diff --git a/.clang-format b/.clang-format
index 5ef5743b77c95f..395292ab2ac634 100644
--- a/.clang-format
+++ b/.clang-format
@@ -282,6 +282,7 @@ ForEachMacros:
   - 'for_each_dpcm_fe'
   - 'for_each_drhd_unit'
   - 'for_each_dss_dev'
+  - 'for_each_early_section_nr'
   - 'for_each_efi_memory_desc'
   - 'for_each_efi_memory_desc_in_map'
   - 'for_each_element'
@@ -403,7 +404,6 @@ ForEachMacros:
   - 'for_each_possible_cpu_wrap'
   - 'for_each_present_blessed_reg'
   - 'for_each_present_cpu'
-  - 'for_each_present_section_nr'
   - 'for_each_prime_number'
   - 'for_each_prime_number_from'
   - 'for_each_probe_cache_entry'
diff --git a/.github/packaging/PKGBUILD b/.github/packaging/PKGBUILD
new file mode 100644
index 00000000000000..85dcd59ff2abe7
--- /dev/null
+++ b/.github/packaging/PKGBUILD
@@ -0,0 +1,262 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Maintainer: OpenGamingCollective
+#
+# PKGBUILD for the linux-unstable-ogc kernel.
+#
+# It is designed to be built by the CI job defined in
+# .github/workflows/build-kernel.yml, which stages the following next to this
+# file before invoking makepkg:
+#   - linux.tar.gz     -> tarball of the checked-out kernel tree (no VCS data)
+#   - config           -> Arch linux-headers .config with the OGC
+#                         kernel-packages config fragments already applied
+#   - config.fragment  -> repo-local overrides merged on top of it
+#
+# Based on the official Arch Linux kernel PKGBUILD and on the (known to work
+# with linux-next) https://github.com/NeroReflex/linux-bisector PKGBUILD.
+
+pkgbase=linux-unstable-ogc
+# Placeholder: makepkg refuses an empty pkgver before pkgver() runs; the real
+# version is computed dynamically by pkgver() once sources are extracted.
+pkgver=0.0.0
+pkgrel=1
+pkgdesc='linux-next kernel for the Open Gaming Collective'
+url='https://github.com/OpenGamingCollective/linux-unstable'
+arch=(x86_64)
+license=(GPL2)
+makedepends=(
+  bc
+  cpio
+  gettext
+  libelf
+  pahole
+  perl
+  python
+  tar
+  xz
+  gcc
+  ccache
+  git
+
+  llvm
+  clang
+  lld
+
+  # CONFIG_RUST=y in the base config
+  rust
+  rust-bindgen
+)
+options=('!strip')
+source=(
+  'linux.tar.gz'
+  'config'
+  'config.fragment'
+)
+b2sums=(
+  'SKIP'
+  'SKIP'
+  'SKIP'
+)
+
+export KBUILD_BUILD_HOST=archlinux
+export KBUILD_BUILD_USER=$pkgbase
+export KBUILD_BUILD_TIMESTAMP=""
+export CC="ccache clang"
+export MAKEFLAGS="-j$(nproc)"
+
+_make() {
+  test -s version
+  LLVM=1 LLVM_IAS=1 WERROR=0 KBUILD_BUILD_TIMESTAMP="" make CC="$CC" KERNELRELEASE="$(<version)" "$@"
+}
+
+pkgver() {
+  # Derive the package version straight from the kernel tree:
+  #   <VERSION>.<PATCHLEVEL>.<SUBLEVEL><EXTRAVERSION> + $(cat localversion-next)
+  #   + the git short sha recorded by CI in .build_commit
+  # e.g. "7.2.0" + "-next-20260818" + ".g4ca2fc86" -> "7.2.0-next-20260818.g4ca2fc86".
+  # Pacman does not allow hyphens in pkgver, so they become underscores:
+  #   7.2.0-next-20260818.g4ca2fc86 -> 7.2.0_next_20260818.g4ca2fc86
+  cd "$srcdir/linux"
+  local ver
+  ver="$(make -s kernelversion)$(cat localversion-next 2>/dev/null || true)"
+  if [[ -s .build_commit ]]; then
+    ver+=".g$(<.build_commit)"
+  fi
+  printf '%s\n' "${ver//-/_}"
+}
+
+prepare() {
+  cd "$srcdir/linux"
+
+  # glibc >= 2.42 made strstr/strchr const-correct; some trees hardcode
+  # -Werror in tools/lib/bpf which then fails to compile. Keep builds green.
+  if [[ -f tools/lib/bpf/Makefile ]]; then
+    sed -i 's/ -Werror -Wall/ -Wall/' tools/lib/bpf/Makefile || true
+  fi
+
+  echo "Setting version..."
+  # localversion* files are picked up sorted by name, so the final kernel
+  # release becomes e.g.: 7.2.0-next-20260818-unstable-ogc-g4ca2fc86-1
+  # The -g<sha> suffix comes from the .build_commit file the CI writes into
+  # the tarball (same trick linux-bisector uses with .bisector_commit).
+  local _commit=""
+  if [[ -s .build_commit ]]; then
+    _commit="-g$(<.build_commit)"
+  fi
+  echo "${pkgbase#linux}${_commit}" > localversion.10-pkgname
+  echo "-$pkgrel" > localversion.20-pkgrel
+  LLVM=1 LLVM_IAS=1 make defconfig
+  LLVM=1 LLVM_IAS=1 make -s kernelrelease > version
+  LLVM=1 LLVM_IAS=1 make mrproper
+
+  echo "Setting config..."
+  cp ../config .config
+  _make olddefconfig
+
+  echo "Applying OGC config fragment..."
+  scripts/kconfig/merge_config.sh -m .config ../config.fragment
+  _make olddefconfig
+
+  diff -u ../config .config || :
+
+  echo "Prepared $pkgbase version $(<version)"
+}
+
+build() {
+  cd "$srcdir/linux"
+  _make all
+}
+
+_package() {
+  pkgdesc="The $pkgdesc kernel and modules"
+  depends=(
+    coreutils
+    initramfs
+    kmod
+  )
+  optdepends=(
+    'wireless-regdb: to set the correct wireless channels of your country'
+    'linux-firmware: firmware images needed for some devices'
+  )
+  provides=(
+    KSMBD-MODULE
+    VIRTUALBOX-GUEST-MODULES
+    WIREGUARD-MODULE
+  )
+  replaces=(
+    virtualbox-guest-modules-arch
+    wireguard-arch
+  )
+
+  cd "$srcdir/linux"
+  local modulesdir="$pkgdir/usr/lib/modules/$(<version)"
+
+  echo "Installing boot image..."
+  # systemd expects to find the kernel here to allow hibernation
+  # https://github.com/systemd/systemd/commit/edda44605f06a41fb86b7ab8128dcf99161d2344
+  install -Dm644 "$(_make -s image_name)" "$modulesdir/vmlinuz"
+
+  # Used by mkinitcpio to name the kernel
+  echo "$pkgbase" | install -Dm644 /dev/stdin "$modulesdir/pkgbase"
+
+  echo "Installing modules..."
+  ZSTD_CLEVEL=19 _make INSTALL_MOD_PATH="$pkgdir/usr" INSTALL_MOD_STRIP=1 \
+    DEPMOD=/doesnt/exist modules_install  # Suppress depmod
+
+  # remove build links
+  rm "$modulesdir"/build
+}
+
+_package-headers() {
+  pkgdesc="Headers and scripts for building modules for the $pkgdesc kernel"
+  depends=(pahole)
+
+  cd "$srcdir/linux"
+  local builddir="$pkgdir/usr/lib/modules/$(<version)/build"
+
+  echo "Installing build files..."
+  install -Dt "$builddir" -m644 .config Makefile Module.symvers System.map \
+    localversion.* version vmlinux
+  install -Dt "$builddir/kernel" -m644 kernel/Makefile
+  install -Dt "$builddir/arch/x86" -m644 arch/x86/Makefile
+  cp -t "$builddir" -a scripts
+
+  # required when STACK_VALIDATION is enabled
+  install -Dt "$builddir/tools/objtool" tools/objtool/objtool
+
+  # required when DEBUG_INFO_BTF_MODULES is enabled
+  install -Dt "$builddir/tools/bpf/resolve_btfids" tools/bpf/resolve_btfids/resolve_btfids
+
+  echo "Installing headers..."
+  cp -t "$builddir" -a include
+  cp -t "$builddir/arch/x86" -a arch/x86/include
+  install -Dt "$builddir/arch/x86/kernel" -m644 arch/x86/kernel/asm-offsets.s
+
+  install -Dt "$builddir/drivers/md" -m644 drivers/md/*.h
+  install -Dt "$builddir/net/mac80211" -m644 net/mac80211/*.h
+
+  # https://bugs.archlinux.org/task/13146
+  install -Dt "$builddir/drivers/media/i2c" -m644 drivers/media/i2c/msp3400-driver.h
+
+  # https://bugs.archlinux.org/task/20402
+  install -Dt "$builddir/drivers/media/usb/dvb-usb" -m644 drivers/media/usb/dvb-usb/*.h
+  install -Dt "$builddir/drivers/media/dvb-frontends" -m644 drivers/media/dvb-frontends/*.h
+  install -Dt "$builddir/drivers/media/tuners" -m644 drivers/media/tuners/*.h
+
+  # https://bugs.archlinux.org/task/71392
+  install -Dt "$builddir/drivers/iio/common/hid-sensors" -m644 drivers/iio/common/hid-sensors/*.h
+
+  echo "Installing KConfig files..."
+  find . -name 'Kconfig*' -exec install -Dm644 {} "$builddir/{}" \;
+
+  echo "Removing unneeded architectures..."
+  local arch
+  for arch in "$builddir"/arch/*/; do
+    [[ $arch = */x86/ ]] && continue
+    echo "Removing $(basename "$arch")"
+    rm -r "$arch"
+  done
+
+  echo "Removing documentation..."
+  rm -r "$builddir/Documentation"
+
+  echo "Removing broken symlinks..."
+  find -L "$builddir" -type l -printf 'Removing %P\n' -delete
+
+  echo "Removing loose objects..."
+  find "$builddir" -type f -name '*.o' -printf 'Removing %P\n' -delete
+
+  echo "Stripping build tools..."
+  local file
+  while read -rd '' file; do
+    case "$(file -Sib "$file")" in
+      application/x-sharedlib\;*)      # Libraries (.so)
+        strip -v $STRIP_SHARED "$file" ;;
+      application/x-archive\;*)        # Libraries (.a)
+        strip -v $STRIP_STATIC "$file" ;;
+      application/x-executable\;*)     # Binaries
+        strip -v $STRIP_BINARIES "$file" ;;
+      application/x-pie-executable\;*) # Relocatable binaries
+        strip -v $STRIP_SHARED "$file" ;;
+    esac
+  done < <(find "$builddir" -type f -perm -u+x ! -name vmlinux -print0)
+
+  echo "Stripping vmlinux..."
+  strip -v $STRIP_STATIC "$builddir/vmlinux"
+
+  echo "Adding symlink..."
+  mkdir -p "$pkgdir/usr/src"
+  ln -sr "$builddir" "$pkgdir/usr/src/$pkgbase"
+}
+
+pkgname=(
+  "$pkgbase"
+  "$pkgbase-headers"
+)
+for _p in "${pkgname[@]}"; do
+  eval "package_$_p() {
+    $(declare -f "_package${_p#$pkgbase}")
+    _package${_p#$pkgbase}
+  }"
+done
+
+# vim:set ts=8 sts=2 sw=2 et:
\ No newline at end of file
diff --git a/.github/packaging/boot-smoke-test.sh b/.github/packaging/boot-smoke-test.sh
new file mode 100755
index 00000000000000..f315029982fd72
--- /dev/null
+++ b/.github/packaging/boot-smoke-test.sh
@@ -0,0 +1,243 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# QEMU boot smoke test for the linux-unstable-ogc release builds.
+#
+# Boots a freshly built, *packaged* kernel image under QEMU with a tiny
+# hand-rolled initramfs whose only job is to prove the kernel can
+# decompress, run its initcalls and hand control to userspace: the
+# embedded PID 1 prints a BOOT_OK marker on the serial console and then
+# powers the VM off, which makes QEMU exit by itself (a successful boot
+# never has to wait for the timeout).
+#
+# A boot matrix is exercised, because handhelds/gaming devices ship all
+# sorts of firmware:
+#
+#   bios-pc   SeaBIOS (BIOS) + i440fx "pc" machine, kernel loaded by the
+#             firmware's Linux loader (fw_cfg).
+#   uefi-q35  OVMF (UEFI) + q35 machine, kernel booted through the EFI
+#             stub (this is what Steam Deck / ROG Ally / Legion Go /
+#             Ayaneo class devices actually boot from).
+#
+# Every boot also attaches a spread of boot-relevant PCI devices (ICH9
+# AHCI comes with q35, PIIX IDE with pc, plus NVMe, virtio-blk,
+# virtio-scsi, e1000e and xHCI USB) so as many storage/USB/net boot
+# drivers as the built kernel contains get enumerated and probed.
+#
+# Each distro job (Arch/Fedora/Debian) runs this against the exact image
+# it is about to upload, before uploading it. The release job only runs
+# when every distro job passes, so an unbootable kernel can never be
+# published.
+#
+# Usage:   boot-smoke-test.sh <bzImage>
+# Env:     KREL  expected kernel release string; when set, the boot banner
+#                "Linux version <KREL>" must appear on every console log.
+#          BOOT_SMOKE_TIMEOUT  override the per-boot timeout in seconds
+#                              (default: 300 under KVM, 1200 under TCG).
+# The kernel command line pins loglevel=7: some distro configs default the
+# console to a quieter level (Arch ships CONSOLE_LOGLEVEL_DEFAULT=4), which
+# would suppress the KERN_NOTICE boot banner and make the banner check below
+# fail on an otherwise perfectly bootable kernel.
+# Requires (installed by the calling CI job): qemu-system-x86_64, cpio,
+#          gzip, a C compiler (gcc or clang), an OVMF build for the UEFI
+#          leg (packages: ovmf / edk2-ovmf), coreutils (timeout, find).
+
+set -euo pipefail
+
+die() { echo "::error::$*" >&2; exit 1; }
+
+IMAGE="${1:?usage: boot-smoke-test.sh <bzImage>}"
+[ -s "$IMAGE" ] || die "kernel image '$IMAGE' not found or empty"
+
+for tool in qemu-system-x86_64 cpio gzip timeout sha256sum find truncate; do
+	command -v "$tool" >/dev/null || die "required tool '$tool' not installed"
+done
+CC_BIN="$(command -v gcc || command -v clang || true)"
+[ -n "$CC_BIN" ] || die "no C compiler (gcc or clang) found"
+
+WORK="$(mktemp -d /tmp/boot-smoke.XXXXXX)"
+trap 'rm -rf "$WORK"' EXIT
+
+# ---------------------------------------------------------------------------
+# PID 1 for the test initramfs: freestanding, no libc, raw x86_64 syscalls,
+# so it compiles with gcc or clang on any distro without extra static-libc
+# packages (the Fedora build env has no glibc-static, for example).
+# ---------------------------------------------------------------------------
+cat > "$WORK/init.c" <<'EOF'
+/*
+ * Tiny freestanding PID 1 for the QEMU boot smoke test: print BOOT_OK on
+ * the serial console, then power the VM off so QEMU exits on its own.
+ */
+static long sys3(long nr, long a, long b, long c)
+{
+	long ret;
+
+	__asm__ volatile ("syscall"
+			  : "=a"(ret)
+			  : "a"(nr), "D"(a), "S"(b), "d"(c)
+			  : "rcx", "r11", "memory");
+	return ret;
+}
+
+void _start(void)
+{
+	static const char msg[] =
+		"BOOT_OK: linux-unstable-ogc booted to userspace init\n";
+	long fd;
+
+	/* CONFIG_DEVTMPFS_MOUNT does not apply to initramfs, so mount
+	 * devtmpfs ourselves to get /dev/ttyS0. */
+	sys3(83, (long)"/dev", 0755, 0);				/* mkdir	*/
+	sys3(165, (long)"devtmpfs", (long)"/dev", (long)"devtmpfs");	/* mount	*/
+
+	/* Announce success on the serial port and on stdio (fd 1 exists
+	 * only when the kernel could open /dev/console from the initramfs). */
+	fd = sys3(2, (long)"/dev/ttyS0", 1, 0);				/* open		*/
+	if (fd >= 0)
+		sys3(1, fd, (long)msg, sizeof(msg) - 1);		/* write	*/
+	sys3(1, 1, (long)msg, sizeof(msg) - 1);				/* write	*/
+
+	/* reboot(LINUX_REBOOT_CMD_POWER_OFF) -> ACPI S5 -> QEMU exits. */
+	sys3(169, 0xfee1deadL, 672274793L, 0x4321fedcL);
+	for (;;)
+		sys3(34, 0, 0, 0);					/* pause	*/
+}
+EOF
+
+mkdir -p "$WORK/initramfs/dev"
+"$CC_BIN" -Os -g0 -static -no-pie -nostdlib -ffreestanding \
+	-fno-stack-protector -fno-asynchronous-unwind-tables \
+	-fno-unwind-tables -Wl,-z,noexecstack \
+	-o "$WORK/initramfs/init" "$WORK/init.c"
+# /dev/console lets the kernel wire init's stdio to the console; creating it
+# needs mknod privileges and may fail for unprivileged callers. Harmless: the
+# init above then opens /dev/ttyS0 itself after mounting devtmpfs.
+mknod -m 600 "$WORK/initramfs/dev/console" c 5 1 2>/dev/null || true
+
+(cd "$WORK/initramfs" && find . -print0 | cpio --null -o -H newc --quiet) \
+	| gzip -1 > "$WORK/initrd.img"
+
+# Use KVM when the runner exposes it, software emulation (TCG) otherwise.
+# TCG boots a distro-config kernel in a couple of minutes; the generous
+# timeout also covers hangs, which are exactly what this test must catch.
+if [ -w /dev/kvm ]; then
+	ACCEL="kvm"
+	ACCEL_ARGS=(-accel kvm -cpu host)
+	TMO=300
+else
+	ACCEL="tcg"
+	ACCEL_ARGS=(-accel tcg -cpu max)
+	TMO=1200
+fi
+
+# ---------------------------------------------------------------------------
+# Locate an OVMF build for the UEFI leg. Layouts, by distro:
+#   Debian: /usr/share/OVMF/OVMF_CODE(_4M).fd      (package: ovmf)
+#   Fedora: /usr/share/edk2/ovmf/OVMF_CODE.fd      (package: edk2-ovmf)
+#   Arch:   /usr/share/edk2/x64/OVMF_CODE.4m.fd    (package: edk2-ovmf)
+#   qemu:   /usr/share/qemu/edk2-x86_64-code.fd    (qemu-system-data)
+# CODE and VARS must be the same flash size, hence the paired candidates.
+# ---------------------------------------------------------------------------
+OVMF_PAIR=""
+for pair in \
+	"/usr/share/OVMF/OVMF_CODE_4M.fd:/usr/share/OVMF/OVMF_VARS_4M.fd" \
+	"/usr/share/OVMF/OVMF_CODE.fd:/usr/share/OVMF/OVMF_VARS.fd" \
+	"/usr/share/edk2/x64/OVMF_CODE.4m.fd:/usr/share/edk2/x64/OVMF_VARS.4m.fd" \
+	"/usr/share/edk2/x64/OVMF_CODE.fd:/usr/share/edk2/x64/OVMF_VARS.fd" \
+	"/usr/share/edk2/ovmf/OVMF_CODE.fd:/usr/share/edk2/ovmf/OVMF_VARS.fd" \
+	"/usr/share/ovmf/x64/OVMF_CODE.fd:/usr/share/ovmf/x64/OVMF_VARS.fd" \
+	"/usr/share/qemu/edk2-x86_64-code.fd:/usr/share/qemu/edk2-x86_64-vars.fd" \
+	; do
+	if [ -r "${pair%%:*}" ] && [ -r "${pair##*:}" ]; then
+		OVMF_PAIR="$pair"
+		break
+	fi
+done
+[ -n "$OVMF_PAIR" ] || die "no OVMF (UEFI) firmware found; install the 'ovmf' (Debian) or 'edk2-ovmf' (Arch/Fedora) package"
+OVMF_CODE="${OVMF_PAIR%%:*}"
+cp "${OVMF_PAIR##*:}" "$WORK/OVMF_VARS.fd"	# pflash needs a writable copy
+echo "UEFI firmware: $OVMF_CODE"
+
+# Blank disks so the storage controllers have something to enumerate.
+for d in nvme virtio scsi; do
+	truncate -s 64M "$WORK/disk-$d.img"
+done
+
+# Device spread common to every boot: covers the storage/USB/net drivers a
+# gaming kernel is most likely to boot from. Attached whether or not the
+# kernel contains them; drivers that are modules are simply not probed,
+# which costs nothing.
+DRIVERS_ARGS=(
+	-drive if=none,id=dsk-nvme,format=raw,file="$WORK/disk-nvme.img"
+	-device nvme,drive=dsk-nvme,serial=ogcsmoke
+	-drive if=none,id=dsk-virtio,format=raw,file="$WORK/disk-virtio.img"
+	-device virtio-blk-pci,drive=dsk-virtio
+	-drive if=none,id=dsk-scsi,format=raw,file="$WORK/disk-scsi.img"
+	-device virtio-scsi-pci,id=scsi0
+	-device scsi-hd,drive=dsk-scsi
+	-device e1000e,netdev=net0 -netdev user,id=net0,restrict=on
+	-device qemu-xhci -device usb-tablet
+)
+
+# run_boot <name> <machine args...> — boot, wait for QEMU to exit or the
+# timeout to fire, then fail hard if the marker (or the expected release
+# banner) is missing from the serial console log.
+run_boot() {
+	local name="$1"; shift
+	local serial="$WORK/serial-$name.log"
+	: > "$serial"
+
+	echo "[$name] Booting $(basename "$IMAGE") (sha256 $(sha256sum "$IMAGE" | cut -c1-16)...) with QEMU ($ACCEL, timeout ${TMO}s)"
+	timeout "$TMO" qemu-system-x86_64 "${ACCEL_ARGS[@]}" "$@" \
+		"${DRIVERS_ARGS[@]}" \
+		-m 2048 -smp 2 -nodefaults \
+		-display none -monitor none -no-reboot \
+		-serial "file:$serial" \
+		-kernel "$IMAGE" -initrd "$WORK/initrd.img" \
+		-append "console=ttyS0,115200n8 rdinit=/init panic=-1 nokaslr loglevel=7" \
+		|| true
+	# A panic with panic=-1 reboots instantly and -no-reboot makes QEMU
+	# exit, so both "qemu exited by itself" and "timeout killed it" end up
+	# in the marker check below.
+
+	if ! grep -q "BOOT_OK" "$serial"; then
+		echo "::error::QEMU boot smoke test FAILED in the '$name' configuration: no BOOT_OK marker on the serial console (accel=$ACCEL, timeout=${TMO}s). The kernel is unbootable."
+		echo "----- last 250 lines of the $name guest serial console -----"
+		tail -n 250 "$serial" || true
+		echo "-------------------------------------------------------------"
+		exit 1
+	fi
+	if [ -n "${KREL:-}" ] && ! grep -qF "Linux version ${KREL} " "$serial"; then
+		echo "::error::QEMU boot smoke test FAILED in the '$name' configuration: the booted kernel banner does not advertise release '${KREL}'."
+		# Show what the console actually carried: log lines carry a
+		# "[    0.000000] " timestamp prefix, so anchor-free context of
+		# the early console output is what makes this diagnosable.
+		echo "----- first 25 lines of the $name guest serial console -----"
+		head -n 25 "$serial" | sed 's/\r$//'
+		echo "-------------------------------------------------------------"
+		exit 1
+	fi
+	echo "[$name] Boot smoke test PASSED: kernel booted to userspace init and powered off. Serial console tail:"
+	tail -n 10 "$serial" || true
+}
+
+# 1. Classic BIOS boot: SeaBIOS firmware, i440fx (PIIX IDE built in).
+run_boot bios-pc -machine pc
+
+# 2. UEFI boot: OVMF firmware, q35 (ICH9 AHCI + PCIe built in), kernel
+#    launched through the EFI stub — the path handhelds actually use.
+run_boot uefi-q35 \
+	-machine q35 \
+	-drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \
+	-drive if=pflash,format=raw,file="$WORK/OVMF_VARS.fd"
+
+if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+	{
+		echo '### QEMU boot smoke test'
+		echo
+		echo "- Image: $(basename "$IMAGE") (sha256 $(sha256sum "$IMAGE" | cut -c1-12)...)"
+		echo "- Accelerator: ${ACCEL}, timeout ${TMO}s per boot"
+		echo "- Matrix: bios-pc (SeaBIOS) and uefi-q35 (OVMF EFI stub), NVMe + virtio-blk + virtio-scsi + e1000e + xHCI attached"
+		echo '- Result: BOOT_OK in both configurations - kernel decompressed, booted and reached userspace init'
+	} >> "$GITHUB_STEP_SUMMARY"
+fi
diff --git a/.github/packaging/config.fragment b/.github/packaging/config.fragment
new file mode 100644
index 00000000000000..8d455d78cc2417
--- /dev/null
+++ b/.github/packaging/config.fragment
@@ -0,0 +1,124 @@
+# OGC config overrides for linux-unstable-ogc
+#
+# This fragment is merged LAST, on top of the OGC config (Arch linux-headers
+# base + OpenGamingCollective/kernel-packages config/*.{set,unset} fragments)
+# via scripts/kconfig/merge_config.sh + olddefconfig in the PKGBUILD.
+#
+# Target hardware: modern gaming desktops and handhelds (Steam Deck, ROG Ally,
+# Legion Go, MSI Claw, GPD, Ayaneo, OneXPlayer...). Keep IMU/gyro/accel/IIO
+# drivers, Bluetooth, UVC webcams and all OGC handheld drivers (those live in
+# drivers/hid, drivers/platform/x86 — NOT in staging, so STAGING can go).
+
+# --- Kernel identity --------------------------------------------------------
+# Distro base configs carry their own release suffix (Arch "-arch1", Debian
+# "-1-amd64", ...). It must be empty so that uname -r is identical across
+# the Arch/Fedora/Debian packages: <base>-unstable-ogc-g<sha8>-1 (the suffix
+# itself comes from the localversion.10/20 files written by the packaging).
+CONFIG_LOCALVERSION=""
+# CONFIG_LOCALVERSION_AUTO is not set
+
+# --- Build fixes ------------------------------------------------------------
+# drivers/net/ethernet/cadence/macb_main.c fails to compile in linux-next
+# (implicit declarations of macb_alloc_tieoff/macb_free_tieoff). Cadence
+# MACB/GEM is an ARM SoC NIC, never present on x86_64 gaming hardware.
+# MACB_PCI and MACB_USE_HWSTAMP depend on MACB and drop out automatically.
+# CONFIG_MACB is not set
+
+# --- Build speed ------------------------------------------------------------
+# Keep DEBUG_INFO/DEBUG_INFO_BTF=y (OGC requires them for BPF/sched_ext) but
+# skip running pahole on every module (thousands of invocations saved).
+# CONFIG_DEBUG_INFO_BTF_MODULES is not set
+
+# --- Xen guest support (not running inside a Xen VM) ------------------------
+# CONFIG_XEN is not set
+
+# --- Ancient graphics / buses ------------------------------------------------
+# AGP is a pre-PCIe bus; radeon covers pre-GCN AMD GPUs (12+ years old).
+# CONFIG_AGP is not set
+# CONFIG_DRM_RADEON is not set
+# PCMCIA/CardBus slots (2000s laptops)
+# CONFIG_PCCARD is not set
+# FireWire (IEEE 1394) ports
+# CONFIG_FIREWIRE is not set
+# Parallel port (printers/scanners from the 90s)
+# CONFIG_PARPORT is not set
+# IndustryPack carrier boards
+# CONFIG_IPACK_BUS is not set
+# 1-Wire bus
+# CONFIG_W1 is not set
+# Analog gameport joysticks (pre-USB)
+# CONFIG_GAMEPORT is not set
+# Floppy disk controller
+# CONFIG_BLK_DEV_FD is not set
+# Server BMC management
+# CONFIG_IPMI_HANDLER is not set
+
+# --- Enterprise / datacenter storage ----------------------------------------
+# Fibre Channel HBAs and enterprise RAID controllers
+# CONFIG_SCSI_LPFC is not set
+# CONFIG_SCSI_QLA_FC is not set
+# CONFIG_QEDF is not set
+# CONFIG_FCOE is not set
+# CONFIG_LIBFC is not set
+# CONFIG_MEGARAID_SAS is not set
+# CONFIG_MEGARAID_LEGACY is not set
+# CONFIG_MEGARAID_MAILBOX is not set
+# CONFIG_FUSION is not set
+# CONFIG_SCSI_HPSA is not set
+# CONFIG_SCSI_SMARTPQI is not set
+
+# --- Ancient Ethernet NICs ---------------------------------------------------
+# Vendor menus for 10/100 hardware from the 90s/00s (3Com, Tulip, NatSemi,
+# NE2000-era, SiS 900, VIA Rhine/Velocity, Adaptec starfire). Modern NICs
+# (Intel/Realtek/Marvell/Broadcom/Aquantia) are untouched.
+# CONFIG_NET_VENDOR_3COM is not set
+# CONFIG_NET_VENDOR_ADAPTEC is not set
+# CONFIG_NET_TULIP is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_8390 is not set
+# CONFIG_NET_VENDOR_SIS is not set
+# CONFIG_NET_VENDOR_VIA is not set
+
+# --- Specialized/legacy networking ------------------------------------------
+# CONFIG_ATM is not set
+# CONFIG_RDS is not set
+# CONFIG_TIPC is not set
+# CONFIG_PHONET is not set
+# CONFIG_IEEE802154 is not set
+# CONFIG_CAN is not set
+# CONFIG_NFC is not set
+# CONFIG_INFINIBAND is not set
+
+# --- Analog/digital TV, radio, legacy webcams --------------------------------
+# UVC (USB_VIDEO_CLASS) webcams stay enabled — handhelds use them.
+# CONFIG_MEDIA_ANALOG_TV_SUPPORT is not set
+# CONFIG_MEDIA_DIGITAL_TV_SUPPORT is not set
+# CONFIG_MEDIA_RADIO_SUPPORT is not set
+# CONFIG_USB_GSPCA is not set
+
+# --- Ancient/cluster filesystems ----------------------------------------------
+# Desktop FS (ext4/btrfs/xfs/f2fs/exfat/ntfs3/vfat/nfs/cifs) untouched.
+# CONFIG_MINIX_FS is not set
+# CONFIG_UFS_FS is not set
+# CONFIG_BFS_FS is not set
+# CONFIG_GFS2_FS is not set
+# CONFIG_OCFS2_FS is not set
+# CONFIG_AFS_FS is not set
+
+# --- Chemical / gas / air-quality sensors ------------------------------------
+# IMU/gyro/accelerometer/pressure/temp/humidity IIO drivers are KEPT
+# (handheld controllers need them); only gas/VOC/CO2/PM sensors removed.
+# CONFIG_CCS811 is not set
+# CONFIG_SPS30 is not set
+# CONFIG_PMS7003 is not set
+# CONFIG_SENSIRION_SGP30 is not set
+# CONFIG_SENSIRION_SGP40 is not set
+# CONFIG_SCD30_CORE is not set
+# CONFIG_SCD4X is not set
+# CONFIG_VZ89X is not set
+# CONFIG_BME680 is not set
+
+# --- Staging drivers ----------------------------------------------------------
+# All OGC handheld drivers live in mainline trees (drivers/hid,
+# drivers/platform/x86), not staging.
+# CONFIG_STAGING is not set
\ No newline at end of file
diff --git a/.github/packaging/fedora/kernel.spec b/.github/packaging/fedora/kernel.spec
new file mode 100644
index 00000000000000..2165fefce635a9
--- /dev/null
+++ b/.github/packaging/fedora/kernel.spec
@@ -0,0 +1,279 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# RPM spec for the linux-unstable-ogc kernel (linux-next based).
+#
+# Built by .github/workflows/build-kernel.yml ("fedora" job) which, before
+# invoking rpmbuild:
+#   - substitutes @@KBASEVER@@ / @@KVERDOTTED@@ / @@SHA8@@ placeholders below
+#   - stages SOURCES/linux.tar.gz  (kernel tree contents, root dir "linux/",
+#     no VCS data, localversion-next included)
+#   - stages SOURCES/config        (Fedora kernel-core .config with the OGC
+#                                   kernel-packages fragments and the local
+#                                   config.fragment already applied)
+#
+# Derived from the OpenGamingCollective/kernel-packages fedora/kernel.spec
+# (itself based on CachyOS/Nobara), trimmed down to core/modules/devel only.
+# The kernel is compiled with clang (LLVM=1) exactly like the Arch packages.
+
+%global _default_patch_fuzz 2
+
+# See https://fedoraproject.org/wiki/Changes/SetBuildFlagsBuildCheck
+%if 0%{?fedora} >= 37
+%undefine _auto_set_build_flags
+%endif
+
+%define _build_id_links none
+%define _disable_source_fetch 1
+# no debuginfo generation, no brp strip/mangle of kernel binaries
+%define debug_package %{nil}
+%define __spec_install_post /usr/lib/rpm/brp-compress || :
+
+# ---- substituted by CI ------------------------------------------------------
+%define kbasever @@KBASEVER@@
+%define sha8 @@SHA8@@
+# ----------------------------------------------------------------------------
+
+Version: @@KVERDOTTED@@
+Release: 1.g%{sha8}%{?dist}
+
+%define rpmver %{version}-%{release}
+# Kernel release string (uname -r). Identical to what the localversion* files
+# below produce during the build, and identical to the Arch packages built
+# from the same commit:
+#   <kbasever>-unstable-ogc-g<sha8>-1
+%define kverstr %{kbasever}-unstable-ogc-g%{sha8}-1
+# RPM dependency versions may contain at most ONE hyphen (V-R separator), so
+# the full kernel release string cannot be used as a Provides: version. Use a
+# 1:1 dotted translation for the *-uname-r provides instead.
+%define kverdot %(echo "%{kverstr}" | sed -e "s/-/./g")
+
+Name: kernel-unstable-ogc
+Summary: The linux-next kernel for the Open Gaming Collective
+License: GPLv2
+URL: https://github.com/OpenGamingCollective/linux-unstable
+Group: System Environment/Kernel
+ExclusiveArch: x86_64
+Source0: linux.tar.gz
+Source1: config
+
+BuildRequires: bash, coreutils, make, tar, findutils, gawk, diffutils, m4
+BuildRequires: bc, bison, flex, perl-interpreter, perl-Carp, binutils
+BuildRequires: xz, zstd, kmod, python3
+BuildRequires: elfutils-libelf-devel, elfutils-devel
+BuildRequires: openssl, openssl-devel
+BuildRequires: dwarves, hmaccalc
+# clang toolchain (like the Arch packages)
+BuildRequires: clang, lld, llvm, ccache
+# needed when the base config enables CONFIG_RUST. The bindgen binary
+# package was renamed from rust-bindgen to bindgen in newer Fedora releases;
+# accept either so this spec works across distro versions.
+BuildRequires: rust, rust-src
+BuildRequires: (bindgen or rust-bindgen)
+
+# All kernel make invocations: clang via ccache, deterministic version strings
+%define kmake make CC="ccache clang" LLVM=1 LLVM_IAS=1 WERROR=0 KBUILD_BUILD_HOST=ogc-ci KBUILD_BUILD_USER=kernel-unstable-ogc KBUILD_BUILD_TIMESTAMP=""
+
+%description
+This package is a meta package that pulls in the linux-unstable-ogc kernel
+(a linux-next snapshot for the Open Gaming Collective) and its matching
+modules.
+
+%package core
+Summary: The linux-unstable-ogc kernel (vmlinuz and core files)
+Group: System Environment/Kernel
+Provides: installonlypkg(kernel)
+Provides: %{name}-core-uname-r = %{kverdot}
+Requires: bash, coreutils, kmod
+Requires: /usr/bin/kernel-install
+Requires: %{name}-modules = %{rpmver}
+Recommends: linux-firmware
+%description core
+This package contains the linux-unstable-ogc kernel image (vmlinuz),
+System.map, the build configuration and the module symbol version file.
+
+%package modules
+Summary: Kernel modules to match the linux-unstable-ogc core kernel
+Group: System Environment/Kernel
+Provides: installonlypkg(kernel-module)
+Provides: %{name}-modules-uname-r = %{kverdot}
+Supplements: %{name}-core = %{rpmver}
+# kmod needed for depmod in %%post
+Requires: kmod
+%description modules
+This package provides the kernel modules for the linux-unstable-ogc kernel.
+
+%package devel
+Summary: Development files for building external modules
+Group: Development/System
+AutoReqProv: no
+Requires: findutils, make, perl-interpreter, flex, bison
+Requires: elfutils-libelf-devel, openssl-devel, gcc
+Requires: clang, llvm, lld
+Provides: %{name}-devel-uname-r = %{kverdot}
+Enhances: akmods
+Enhances: dkms
+%description devel
+This package provides the headers, scripts and tooling (objtool,
+resolve_btfids) needed to build out-of-tree kernel modules against the
+linux-unstable-ogc kernel (%{kverstr}).
+
+%prep
+%setup -q -n linux
+
+cp %{SOURCE1} .config
+
+# The Fedora distro config references Fedora-only certificate files that do
+# not exist in this tree; use the ephemeral in-tree key instead.
+scripts/config --set-str SYSTEM_TRUSTED_KEYS ""
+scripts/config --set-str SYSTEM_REVOCATION_KEYS ""
+# CONFIG_MODULE_SIG_KEY points at the Red Hat signing cert in the distro
+# config, which does not exist in this tree. Reset it to the kbuild default
+# ("certs/signing_key.pem"): an ephemeral self-signed key generated during
+# the build and trusted by the kernel itself. The Fedora config sets
+# CONFIG_MODULE_SIG_ALL=y, and with an empty key string scripts/Makefile.modinst
+# resolves sig-key to "./", making sign-file read a directory as the private
+# key (SSL DECODER error) on every module.
+scripts/config --set-str MODULE_SIG_KEY "certs/signing_key.pem"
+
+# Deterministic / distro-agnostic build identity
+scripts/config -u DEFAULT_HOSTNAME
+scripts/config --set-str BUILD_SALT "%{kverstr}"
+
+# Kernel release suffix, same scheme as the Arch packages:
+#   <base><localversion-next>-unstable-ogc-g<sha8>-1
+echo "-unstable-ogc-g%{sha8}" > localversion.10-pkgname
+echo "-1" > localversion.20-pkgrel
+
+# glibc >= 2.42 const-correctness vs -Werror in tools/lib/bpf (used by
+# resolve_btfids when DEBUG_INFO_BTF=y); keep the build green.
+if [ -f tools/lib/bpf/Makefile ]; then
+    sed -i 's/ -Werror -Wall/ -Wall/' tools/lib/bpf/Makefile || true
+fi
+
+%{kmake} olddefconfig
+
+# Fail fast if the release string ever drifts from the spec
+REL="$(make -s kernelrelease)"
+if [ "$REL" != "%{kverstr}" ]; then
+    echo "kernelrelease '$REL' does not match spec kverstr '%{kverstr}'" >&2
+    exit 1
+fi
+cp .config config-linux-unstable-ogc
+
+%build
+%{kmake} %{?_smp_mflags} all
+
+%install
+MODDIR="%{buildroot}/lib/modules/%{kverstr}"
+DEVEL="%{buildroot}%{_prefix}/src/kernels/%{kverstr}"
+
+mkdir -p "%{buildroot}/boot" "$MODDIR"
+
+echo "Installing boot image..."
+ImageName="$(make -s image_name | tail -n 1)"
+install -m 0644 "$ImageName" "$MODDIR/vmlinuz"
+chmod 0755 "$MODDIR/vmlinuz"
+
+echo "Installing modules..."
+# Modules are compressed by modules_install itself (CONFIG_MODULE_COMPRESS_*).
+# depmod runs from %%post at install time.
+%{kmake} %{?_smp_mflags} KERNELRELEASE=%{kverstr} \
+    INSTALL_MOD_PATH=%{buildroot} INSTALL_MOD_STRIP=1 \
+    DEPMOD=/doesnt/exist modules_install
+
+echo "Installing core files..."
+cp System.map "$MODDIR/System.map"
+cp .config    "$MODDIR/config"
+gzip -c9 < Module.symvers > "$MODDIR/symvers.gz"
+(cd "$MODDIR" && sha512hmac vmlinuz > .vmlinuz.hmac)
+
+# ---- kernel-devel -----------------------------------------------------------
+echo "Preparing kernel-devel..."
+rm -f "$MODDIR"/build "$MODDIR"/source
+ln -s "%{_prefix}/src/kernels/%{kverstr}" "$MODDIR/build"
+(cd "$MODDIR" && ln -s build source)
+mkdir -p "$MODDIR"/updates "$MODDIR"/weak-updates "$DEVEL"
+
+find . -type f \( -name 'Makefile*' -o -name 'Kconfig*' \) -print0 \
+    | xargs -0 cp --parents -t "$DEVEL"
+cp -a include "$DEVEL"/
+cp -a arch/x86/include "$DEVEL"/arch/x86/
+if [ -f arch/x86/kernel/module.lds ]; then
+    cp -a --parents arch/x86/kernel/module.lds "$DEVEL"/
+fi
+cp -a scripts "$DEVEL"/
+rm -rf "$DEVEL"/scripts/tracing
+rm -f  "$DEVEL"/scripts/spdxcheck.py
+cp Module.symvers System.map .config "$DEVEL"/
+
+mkdir -p "$DEVEL"/tools/{objtool,bpf/resolve_btfids,lib,build}
+cp -a tools/objtool/objtool                      "$DEVEL"/tools/objtool/ || :
+cp -a tools/bpf/resolve_btfids/resolve_btfids    "$DEVEL"/tools/bpf/resolve_btfids/ || :
+cp -a tools/include                              "$DEVEL"/tools/
+cp -a tools/lib/subcmd                           "$DEVEL"/tools/lib/
+cp -a tools/lib/bpf                              "$DEVEL"/tools/lib/
+cp -a tools/build/Build.include tools/build/fixdep.c "$DEVEL"/tools/build/
+cp -a tools/scripts/utilities.mak                "$DEVEL"/tools/scripts/ 2>/dev/null || :
+cp -a --parents arch/x86/entry/syscalls/syscall_32.tbl "$DEVEL"/
+cp -a --parents arch/x86/entry/syscalls/syscall_64.tbl "$DEVEL"/
+cp -a arch/x86/tools                             "$DEVEL"/arch/x86/
+
+# Drop intermediate build artifacts from devel tree
+find "$DEVEL" \( -name '*.o' -o -name '*.cmd' -o -name '.*.cmd' \) -delete
+
+# Timestamps must line up so external module builds do not rerun kconfig
+touch -r "$DEVEL"/Makefile \
+      "$DEVEL"/include/generated/uapi/linux/version.h \
+      "$DEVEL"/include/config/auto.conf
+
+%post core
+# nothing to do at this point
+
+%posttrans core
+# Runs after ALL packages of this transaction have been installed and their
+# %%post scriptlets (incl. depmod from -modules) have run.
+if [ -x /usr/bin/kernel-install ]; then
+    /usr/bin/kernel-install add %{kverstr} /lib/modules/%{kverstr}/vmlinuz || exit $?
+fi
+if [ -x /usr/sbin/grubby ]; then
+    grubby --set-default="/boot/vmlinuz-%{kverstr}" || :
+fi
+
+%preun core
+if [ "$1" = "0" ] && [ -x /usr/bin/kernel-install ]; then
+    /usr/bin/kernel-install remove %{kverstr} /lib/modules/%{kverstr}/vmlinuz || exit $?
+fi
+
+%post modules
+/sbin/depmod -a %{kverstr}
+
+%files
+# meta package: everything lives in the subpackages
+
+%files core
+%ghost /boot/vmlinuz-%{kverstr}
+%ghost /boot/initramfs-%{kverstr}.img
+/lib/modules/%{kverstr}/vmlinuz
+/lib/modules/%{kverstr}/.vmlinuz.hmac
+/lib/modules/%{kverstr}/System.map
+/lib/modules/%{kverstr}/config
+/lib/modules/%{kverstr}/symvers.gz
+
+%files modules
+/lib/modules/%{kverstr}/
+%exclude /lib/modules/%{kverstr}/vmlinuz
+%exclude /lib/modules/%{kverstr}/.vmlinuz.hmac
+%exclude /lib/modules/%{kverstr}/System.map
+%exclude /lib/modules/%{kverstr}/config
+%exclude /lib/modules/%{kverstr}/symvers.gz
+%exclude /lib/modules/%{kverstr}/build
+%exclude /lib/modules/%{kverstr}/source
+
+%files devel
+/usr/src/kernels/%{kverstr}
+/lib/modules/%{kverstr}/build
+/lib/modules/%{kverstr}/source
+
+%changelog
+* Thu Aug 20 2026 OpenGamingCollective CI
+- Initial linux-unstable-ogc spec, generated by CI from linux-next.
\ No newline at end of file
diff --git a/.github/packaging/merge-fragments.sh b/.github/packaging/merge-fragments.sh
new file mode 100644
index 00000000000000..e8525b2d86ce8e
--- /dev/null
+++ b/.github/packaging/merge-fragments.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Textually merge kernel config fragments into a base .config file.
+#
+# Usage: merge-fragments.sh <base-config> <fragment> [<fragment>...]
+#
+# Recognized fragment line formats (fragments are applied in order, later
+# fragments win):
+#   CONFIG_X=value           set X to value
+#   CONFIG_X                 unset X (kernel-configurator *.unset format)
+#   "# CONFIG_X is not set"  unset X (kconfig fragment format)
+# Blank lines and other comments are ignored.
+#
+# This replicates the semantics of the OpenGamingCollective
+# kernel-configurator action (*.config.set / *.config.unset fragments) and
+# additionally accepts regular kconfig fragment files, so both the OGC
+# fragments and the repo-local config.fragment can be handled uniformly.
+# The final `make olddefconfig` (run by the PKGBUILD / RPM spec) turns the
+# textual result into a consistent kconfig.
+
+set -euo pipefail
+
+if [ "$#" -lt 2 ]; then
+    echo "usage: $0 <base-config> <fragment>..." >&2
+    exit 2
+fi
+
+CFG="$(realpath "$1")"
+shift
+
+set_key() {
+    local key="$1" value="$2"
+    if grep -q "^${key}=" "$CFG"; then
+        sed -i "s|^${key}=.*|${key}=${value}|" "$CFG"
+    elif grep -q "^# ${key} is not set$" "$CFG"; then
+        sed -i "s|^# ${key} is not set\$|${key}=${value}|" "$CFG"
+    else
+        printf '%s=%s\n' "${key}" "${value}" >> "$CFG"
+    fi
+}
+
+unset_key() {
+    local key="$1"
+    if grep -q "^${key}=" "$CFG"; then
+        sed -i "s|^${key}=.*|# ${key} is not set|" "$CFG"
+    fi
+}
+
+for frag in "$@"; do
+    frag="$(realpath "$frag")"
+    while IFS= read -r line || [ -n "$line" ]; do
+        line="${line#"${line%%[![:space:]]*}"}"
+        line="${line%"${line##*[![:space:]]}"}"
+        [ -z "$line" ] && continue
+        if [[ "$line" =~ ^#\ (CONFIG_[A-Za-z0-9_]+)\ is\ not\ set$ ]]; then
+            unset_key "${BASH_REMATCH[1]}"
+        elif [[ "$line" =~ ^(CONFIG_[A-Za-z0-9_]+)=(.*)$ ]]; then
+            set_key "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
+        elif [[ "$line" =~ ^(CONFIG_[A-Za-z0-9_]+)$ ]]; then
+            unset_key "${BASH_REMATCH[1]}"
+        fi
+    done < "$frag"
+done
\ No newline at end of file
diff --git a/.github/workflows/build-arch-packages.yml b/.github/workflows/build-arch-packages.yml
new file mode 100644
index 00000000000000..748a422d732aa4
--- /dev/null
+++ b/.github/workflows/build-arch-packages.yml
@@ -0,0 +1,163 @@
+# Arch Linux packaging, split out of build-kernel.yml.
+# Called (via `uses:`) by build-kernel.yml; runs inside the same workflow
+# run, so the artifacts uploaded here are seen by publish-release.yml.
+
+name: Build Arch Linux packages
+
+on:
+  workflow_call:
+    inputs:
+      krel:
+        description: Full kernel release string (uname -r), e.g. 6.12.0-next-20250101-unstable-ogc-g12345678-1
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+env:
+  # Config fragments maintained by the OGC kernel-packages repository.
+  # The distro base config is extracted from the official Arch
+  # `linux-headers` package — same approach as the kernel-packages
+  # arch.yaml workflow.
+  KERNEL_PACKAGES_RAW: https://raw.githubusercontent.com/OpenGamingCollective/kernel-packages/main
+
+jobs:
+  build:
+    name: Build Arch Linux packages
+    runs-on: ubuntu-latest
+    timeout-minutes: 330
+    container:
+      image: docker.io/archlinux:base-devel
+    env:
+      PKGDIR: /tmp/pkgbuild
+      DISTDIR: /tmp/dist
+      CCACHE_DIR: /ccache
+      CCACHE_MAXSIZE: 10G
+      KREL: ${{ inputs.krel }}
+    steps:
+      - name: Show disk space
+        run: df -h /
+
+      - name: Bootstrap Arch Linux build environment
+        run: |
+          set -euxo pipefail
+          # Refresh keyring first to avoid signature failures on stale images
+          pacman -Sy --needed --noconfirm archlinux-keyring
+          pacman -Su --needed --noconfirm
+          # Everything makepkg needs (mirrors the makedepends of the PKGBUILD).
+          # rust + rust-bindgen are needed because the Arch config enables
+          # CONFIG_RUST=y.
+          pacman -S --needed --noconfirm \
+            bc cpio gettext libelf pahole perl python tar xz zstd \
+            gcc clang llvm lld ccache pigz file curl git \
+            rust rust-bindgen
+          # makepkg refuses to run as root
+          useradd -m build
+          install -d -o build -g build -m 0777 /ccache
+
+      - name: Download staged sources
+        uses: actions/download-artifact@v8
+        with:
+          name: kernel-sources
+          path: /tmp/stage
+
+      - name: Restore compiler cache
+        uses: actions/cache@v6
+        with:
+          path: /ccache
+          key: ccache-arch-${{ github.sha }}
+          restore-keys: |
+            ccache-arch-
+
+      - name: Prepare ccache for the build user
+        run: chown -R build:build /ccache && su build -c 'ccache -s' || true
+
+      - name: Assemble Arch kernel config
+        working-directory: /tmp/stage
+        run: |
+          set -euxo pipefail
+          # ------------------------------------------------------------------
+          # Base config: extract the official Arch Linux kernel .config from
+          # the distro's `linux-headers` package (same method as the OGC
+          # kernel-packages arch.yaml workflow).
+          # ------------------------------------------------------------------
+          CACHE="/tmp/pkgcache"
+          mkdir -p "$CACHE"
+          pacman -Sw --noconfirm --cachedir "$CACHE" linux-headers
+          PKG="$(find "$CACHE" -maxdepth 1 -name 'linux-headers-*.pkg.tar.zst' -type f)"
+          if [ -z "$PKG" ]; then
+            echo "::error::linux-headers package not found in cache"; exit 1
+          fi
+          CFG="$(tar --zstd -tf "$PKG" | grep -E '^usr/lib/modules/[^/]+/build/\.config$' || true)"
+          if [ -z "$CFG" ]; then
+            echo "::error::kernel .config not found inside $PKG"; exit 1
+          fi
+          tar --zstd -xOf "$PKG" "$CFG" > config
+          test -s config
+          rm -rf "$CACHE"
+
+          # ------------------------------------------------------------------
+          # Layer the OGC config fragments from kernel-packages on top, then
+          # the repo-local config.fragment (which wins). Order is critical:
+          # unsets apply first (removing things we explicitly don't want),
+          # then sets apply (enabling things we do want), so explicit enables
+          # can override explicit disables.
+          # ------------------------------------------------------------------
+          for f in arch.config.set ogc.config.set arch.config.unset ogc.config.unset; do
+            curl -fsSL "${KERNEL_PACKAGES_RAW}/config/${f}" -o "${f}"
+          done
+          bash ./merge-fragments.sh config \
+              arch.config.unset ogc.config.unset \
+              arch.config.set ogc.config.set \
+              config.fragment
+          echo "Config after fragment merge (head):"
+          head -n 3 config
+
+      - name: Stage PKGBUILD directory
+        run: |
+          set -euxo pipefail
+          mkdir -p "${PKGDIR}" "${DISTDIR}"
+          cd /tmp/stage
+          cp PKGBUILD config.fragment linux.tar.gz config "${PKGDIR}/"
+          chown -R build:build "${PKGDIR}"
+
+      - name: Build packages with makepkg
+        run: |
+          set -euxo pipefail
+          cd "${PKGDIR}"
+          runuser -u build -- env HOME=/home/build CCACHE_DIR=/ccache \
+            makepkg -f --noconfirm --noprogressbar
+
+          ls -lh ./*.pkg.tar.zst
+          cp -v ./*.pkg.tar.zst "${DISTDIR}/"
+          # Ship the exact .config used for the build as well
+          cp -v "src/linux/.config" "${DISTDIR}/config-arch-${KREL}"
+
+      - name: Boot smoke test in QEMU (release gate)
+        run: |
+          set -euxo pipefail
+          pacman -S --needed --noconfirm qemu-system-x86 edk2-ovmf
+          # ----------------------------------------------------------------
+          # Boot the exact kernel image that ships in the package. This
+          # step fails the job (and with it the release) if the kernel
+          # panics, hangs or never reaches userspace, so an unbootable
+          # kernel is never uploaded or published.
+          # ----------------------------------------------------------------
+          WORK=/tmp/bootsmoke
+          rm -rf "$WORK"; mkdir -p "$WORK"
+          PKG="$(find /tmp/dist -name 'linux-unstable-ogc-*.pkg.tar.zst' ! -name '*-headers-*' | head -n1)"
+          test -n "$PKG"
+          tar --zstd -xf "$PKG" -C "$WORK" "usr/lib/modules/${KREL}/vmlinuz"
+          test -s "$WORK/usr/lib/modules/${KREL}/vmlinuz"
+          bash /tmp/stage/boot-smoke-test.sh "$WORK/usr/lib/modules/${KREL}/vmlinuz"
+
+      - name: Clean build tree (free disk before cache save)
+        run: rm -rf "${PKGDIR}/src" "${PKGDIR}/pkg" ; df -h /
+
+      - name: Upload Arch packages
+        uses: actions/upload-artifact@v7
+        with:
+          name: arch-packages
+          path: /tmp/dist
+          retention-days: 3
diff --git a/.github/workflows/build-debian-packages.yml b/.github/workflows/build-debian-packages.yml
new file mode 100644
index 00000000000000..6eef922dc09eb4
--- /dev/null
+++ b/.github/workflows/build-debian-packages.yml
@@ -0,0 +1,292 @@
+# Debian .deb packaging, split out of build-kernel.yml.
+# Called (via `uses:`) by build-kernel.yml; runs inside the same workflow
+# run, so the artifacts uploaded here are seen by publish-release.yml.
+
+name: Build Debian packages
+
+on:
+  workflow_call:
+    inputs:
+      krel:
+        description: Full kernel release string (uname -r), e.g. 6.12.0-next-20250101-unstable-ogc-g12345678-1
+        type: string
+        required: true
+      kverdot:
+        description: Kernel version with hyphens replaced by dots, for KDEB_PKGVERSION
+        type: string
+        required: true
+      sha8:
+        description: Short (8 char) source commit the packages are built from
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+env:
+  # Config fragments maintained by the OGC kernel-packages repository.
+  # The distro base config comes from the official Debian
+  # linux-config/linux-image packages.
+  KERNEL_PACKAGES_RAW: https://raw.githubusercontent.com/OpenGamingCollective/kernel-packages/main
+
+jobs:
+  build:
+    name: Build Debian packages
+    runs-on: ubuntu-latest
+    timeout-minutes: 330
+    container:
+      image: docker.io/library/debian:trixie
+    env:
+      CCACHE_DIR: /ccache
+      CCACHE_MAXSIZE: 10G
+      KREL: ${{ inputs.krel }}
+      KVERDOT: ${{ inputs.kverdot }}
+      SHA8: ${{ inputs.sha8 }}
+    steps:
+      - name: Show disk space
+        run: df -h /
+
+      - name: Install build tools
+        run: |
+          set -euxo pipefail
+          apt-get update
+          # Satisfies the Build-Depends generated by scripts/package/mkdebian
+          # (debhelper-compat, bc, bison, flex, kmod, libdw/libelf/libssl-dev,
+          # python3, rsync) plus the clang toolchain used by all OGC builds.
+          apt-get install -y --no-install-recommends \
+            build-essential debhelper rsync \
+            bc bison flex python3 kmod \
+            libelf-dev libdw-dev libssl-dev zlib1g-dev \
+            dwarves zstd xz-utils \
+            clang llvm lld ccache curl ca-certificates git \
+            openssl
+          # dpkg-buildpackage refuses to run as root
+          useradd -m builder
+          install -d -o builder -g builder -m 0777 /ccache
+
+      - name: Download staged sources
+        uses: actions/download-artifact@v8
+        with:
+          name: kernel-sources
+          path: /tmp/stage
+
+      - name: Restore compiler cache
+        uses: actions/cache@v6
+        with:
+          path: /ccache
+          key: ccache-debian-${{ github.sha }}
+          restore-keys: |
+            ccache-debian-
+
+      - name: Assemble Debian kernel config
+        working-directory: /tmp/stage
+        run: |
+          set -euxo pipefail
+          # ------------------------------------------------------------------
+          # Base config: the official Debian kernel configuration. It ships in
+          # the small linux-config-<abi> package; fall back to /boot/config-*
+          # of the linux-image-<abi> package if that is unavailable.
+          # ------------------------------------------------------------------
+          ABI="$(apt-cache depends linux-image-amd64 \
+            | awk '/^ *Depends: *linux-image-[0-9]/ {print $2; exit}' \
+            | sed 's/^linux-image-//')"
+          test -n "${ABI}"
+          echo "Debian kernel ABI: ${ABI}"
+          # linux-config is versioned by major.minor only (e.g. 6.12) and
+          # stores the per-flavour config xz-compressed, e.g.
+          # /usr/src/linux-config-6.12/config.amd64_none_amd64.xz
+          KMAJMIN="$(printf '%s' "${ABI}" | cut -d. -f1,2)"
+          mkdir -p /tmp/pkgcfg
+          if apt-get download "linux-config-${KMAJMIN}"; then
+            dpkg-deb -x linux-config-"${KMAJMIN}"_*.deb /tmp/pkgcfg
+            BASE="$(find /tmp/pkgcfg/usr/src -name 'config.amd64_none_amd64.xz' | head -n1)"
+            test -s "${BASE}"
+            xz -dc "${BASE}" > config
+          else
+            apt-get download "linux-image-${ABI}"
+            dpkg-deb -x linux-image-"${ABI}"_*.deb /tmp/pkgcfg
+            BASE="$(find /tmp/pkgcfg/boot -maxdepth 1 -name 'config-*' | head -n1)"
+            test -s "${BASE}"
+            cp "${BASE}" config
+          fi
+          test -s config
+          rm -rf /tmp/pkgcfg
+
+          # ------------------------------------------------------------------
+          # Layer the OGC config fragments and the repo-local config.fragment
+          # (which wins). kernel-packages ships no debian-specific fragments.
+          # Order is critical: unsets apply first, then sets, so explicit
+          # enables can override explicit disables.
+          # ------------------------------------------------------------------
+          for f in ogc.config.set ogc.config.unset; do
+            curl -fsSL "${KERNEL_PACKAGES_RAW}/config/${f}" -o "${f}"
+          done
+          bash ./merge-fragments.sh config \
+              ogc.config.unset \
+              ogc.config.set \
+              config.fragment
+          echo "Config after fragment merge (head):"
+          head -n 3 config
+
+      - name: Ensure pahole >= 1.31 (sched_ext/BTF)
+        run: |
+          set -euxo pipefail
+          VER="$(pahole --version | tr -d 'v')"
+          MAJ="${VER%%.*}"; MIN="${VER#*.}"; MIN="${MIN%%.*}"
+          echo "Installed pahole: ${VER}"
+          if [ "$MAJ" -lt 1 ] || { [ "$MAJ" -eq 1 ] && [ "$MIN" -lt 31 ]; }; then
+            echo "pahole < 1.31 breaks sched_ext; building dwarves 1.31 from source"
+            apt-get install -y --no-install-recommends cmake pkg-config
+            cd /tmp
+            curl -fsSLO https://fedorapeople.org/~acme/dwarves/dwarves-1.31.tar.xz
+            echo "0a7f255ccacf8cc7f8cd119099eb327179b4b3c67cb015af646af6d0cb03054d  dwarves-1.31.tar.xz" | sha256sum -c
+            tar -xf dwarves-1.31.tar.xz
+            cmake -B dwarves-1.31/build -S dwarves-1.31 \
+              -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr
+            make -C dwarves-1.31/build -j"$(nproc)" install
+          fi
+          pahole --version
+
+      - name: Build .deb packages with the in-tree packaging
+        run: |
+          set -euxo pipefail
+          mkdir -p /build
+          tar -xzf /tmp/stage/linux.tar.gz -C /build
+          cd /build/linux
+
+          # Config adjustments + kernel release suffix, identical scheme to
+          # the Arch and Fedora packages (uname -r == ${KREL}).
+          cp /tmp/stage/config .config
+          # Debian's config references distro-only certificate files that do
+          # not exist in this tree; use the ephemeral in-tree key instead.
+          scripts/config --set-str SYSTEM_TRUSTED_KEYS ""
+          scripts/config --set-str SYSTEM_REVOCATION_KEYS ""
+          # CONFIG_MODULE_SIG_KEY points at the Debian signing cert, which
+          # does not exist in this tree. Reset it to the kbuild default
+          # ("certs/signing_key.pem"): an ephemeral self-signed key generated
+          # during the build and trusted by the kernel itself. The Debian
+          # config sets CONFIG_MODULE_SIG_ALL=y, and with an empty key string
+          # scripts/Makefile.modinst resolves sig-key to "./", making
+          # sign-file read a directory as the private key (SSL DECODER error)
+          # on every module.
+          scripts/config --set-str MODULE_SIG_KEY "certs/signing_key.pem"
+          scripts/config -u DEFAULT_HOSTNAME
+          scripts/config --set-str BUILD_SALT "${KREL}"
+          echo "-unstable-ogc-g${SHA8}" > localversion.10-pkgname
+          echo "-1" > localversion.20-pkgrel
+          # The staged tarball carries no VCS data, but mkdebian (via
+          # gen-diff-patch) runs "git diff HEAD" and aborts on failure.
+          # A throwaway repo with everything committed makes that a no-op.
+          git init -q
+          git config user.email "ci@opengamingcollective.org"
+          git config user.name "OpenGamingCollective CI"
+          git add -A
+          git commit -qm "linux-unstable-ogc ${KREL}"
+          chown -R builder:builder /build
+
+          # scripts/setlocalversion appends a "+" whenever a git repo exists
+          # and LOCALVERSION is unset and the HEAD is not at an annotated
+          # version tag. The throwaway repo below would therefore corrupt the
+          # release string. Setting LOCALVERSION to the empty string (as
+          # documented in the script) suppresses that suffix everywhere.
+          runuser -u builder -- env HOME=/home/builder CCACHE_DIR=/ccache \
+            LOCALVERSION= \
+            make CC="ccache clang" LLVM=1 LLVM_IAS=1 WERROR=0 \
+            KBUILD_BUILD_HOST=ogc-ci KBUILD_BUILD_USER=kernel-unstable-ogc \
+            olddefconfig
+
+          # Fail fast if the release string ever drifts from the other distros
+          REL="$(runuser -u builder -- env HOME=/home/builder LOCALVERSION= make -s kernelrelease)"
+          if [ "$REL" != "${KREL}" ]; then
+            echo "kernelrelease '$REL' does not match expected '${KREL}'" >&2
+            exit 1
+          fi
+
+          # Generate the debian/ directory with the tree's own packaging.
+          # mkdebian is invoked directly as a script: a bare "make debian" is
+          # NOT a top-level make target in this tree (only *-pkg patterns are
+          # delegated to scripts/Makefile.package), and going through make
+          # without the exact same CC flags as the olddefconfig above made
+          # kbuild re-sync the config interactively. As a plain sh script it
+          # touches nothing kbuild-related.
+          # KDEB_PKGVERSION must not contain hyphens except the final revision
+          # separator (Debian policy), hence the dotted translation.
+          KDEBVER="${KVERDOT}.unstable.ogc.g${SHA8}-1"
+          runuser -u builder -- env HOME=/home/builder \
+            srctree="$PWD" \
+            ARCH=x86_64 SRCARCH=x86 UTS_MACHINE=x86_64 \
+            KERNELRELEASE="${KREL}" \
+            KCONFIG_CONFIG=.config \
+            KDEB_SOURCENAME=linux-unstable-ogc \
+            KDEB_PKGVERSION="${KDEBVER}" \
+            KDEB_CHANGELOG_DIST=trixie \
+            DEBFULLNAME="OpenGamingCollective CI" \
+            DEBEMAIL="ci@opengamingcollective.org" \
+            sh scripts/package/mkdebian
+          echo "debian arch: $(cat debian/arch)"
+
+          # Kbuild only honours command-line variables, so inject the
+          # clang/ccache toolchain into the generated debian/rules (this is
+          # what "make bindeb-pkg" would otherwise lose). LOCALVERSION= (set,
+          # empty) keeps setlocalversion from appending "+" inside the build.
+          sed -i 's|^make-opts = |make-opts = CC="ccache clang" LLVM=1 LLVM_IAS=1 WERROR=0 LOCALVERSION= KBUILD_BUILD_HOST=ogc-ci KBUILD_BUILD_USER=kernel-unstable-ogc |' debian/rules
+          grep -n '^make-opts' debian/rules
+
+          # Same invocation as "make bindeb-pkg" (scripts/Makefile.package),
+          # but with parallel jobs, --no-check-builddeps (the build deps are
+          # preinstalled above; the generated Build-Depends-Arch also names a
+          # distro-only cross-gcc package that need not exist), and without
+          # the multi-GB debug-symbol package (all other distro jobs ship no
+          # debug packages either).
+          runuser -u builder -- env HOME=/home/builder CCACHE_DIR=/ccache \
+            LOCALVERSION= \
+            DEB_BUILD_PROFILES="pkg.linux-unstable-ogc.nokerneldbg" \
+            dpkg-buildpackage --build=binary --no-pre-clean --unsigned-changes \
+            --no-check-builddeps \
+            -R'make -f debian/rules' -j"$(nproc)" -a"$(cat debian/arch)"
+
+          ls -lh /build/*.deb
+
+      - name: Collect Debian artifacts
+        run: |
+          set -euxo pipefail
+          DISTDIR=/tmp/dist
+          mkdir -p "${DISTDIR}"
+          cp -v "/build/linux-image-${KREL}"_*.deb "${DISTDIR}/"
+          cp -v "/build/linux-headers-${KREL}"_*.deb "${DISTDIR}/"
+          cp -v /build/linux/.config "${DISTDIR}/config-debian-${KREL}"
+          # linux-libc-dev is intentionally NOT shipped: installing it would
+          # replace the distribution's own linux-libc-dev package.
+          ls -lh "${DISTDIR}"
+
+      - name: Boot smoke test in QEMU (release gate)
+        run: |
+          set -euxo pipefail
+          apt-get update -qq
+          # qemu boots the kernel (OVMF provides the UEFI leg); cpio builds
+          # the test initramfs.
+          apt-get install -y --no-install-recommends qemu-system-x86 cpio ovmf
+          # ----------------------------------------------------------------
+          # Boot the exact kernel image that ships in the linux-image .deb.
+          # This step fails the job (and with it the release) if the kernel
+          # panics, hangs or never reaches userspace, so an unbootable
+          # kernel is never uploaded or published.
+          # ----------------------------------------------------------------
+          WORK=/tmp/bootsmoke
+          rm -rf "$WORK"; mkdir -p "$WORK"
+          DEB="$(find /tmp/dist -name "linux-image-${KREL}_*.deb" | head -n1)"
+          test -n "$DEB"
+          dpkg-deb -x "$DEB" "$WORK"
+          IMG="$(find "$WORK" -name 'vmlinuz-*' -type f | head -n1)"
+          test -n "$IMG"
+          bash /tmp/stage/boot-smoke-test.sh "$IMG"
+
+      - name: Clean build tree (free disk before cache save)
+        run: rm -rf /build ; df -h /
+
+      - name: Upload Debian packages
+        uses: actions/upload-artifact@v7
+        with:
+          name: debian-packages
+          path: /tmp/dist
+          retention-days: 3
diff --git a/.github/workflows/build-fedora-packages.yml b/.github/workflows/build-fedora-packages.yml
new file mode 100644
index 00000000000000..0a95d181f8ab77
--- /dev/null
+++ b/.github/workflows/build-fedora-packages.yml
@@ -0,0 +1,207 @@
+# Fedora RPM packaging, split out of build-kernel.yml.
+# Called (via `uses:`) by build-kernel.yml; runs inside the same workflow
+# run, so the artifacts uploaded here are seen by publish-release.yml.
+
+name: Build Fedora RPM packages
+
+on:
+  workflow_call:
+    inputs:
+      krel:
+        description: Full kernel release string (uname -r), e.g. 6.12.0-next-20250101-unstable-ogc-g12345678-1
+        type: string
+        required: true
+      kbase:
+        description: Base kernel version incl. prerelease suffix but no packaging suffix (e.g. 6.12.0-next-20250101)
+        type: string
+        required: true
+      kverdot:
+        description: Kernel version with hyphens replaced by dots, for the RPM Version field
+        type: string
+        required: true
+      sha8:
+        description: Short (8 char) source commit the packages are built from
+        type: string
+        required: true
+
+permissions:
+  contents: read
+
+env:
+  # Config fragments maintained by the OGC kernel-packages repository.
+  # The distro base config is extracted from the official Fedora
+  # `kernel-core` package — same approach as the kernel-packages
+  # fedora.yaml workflow.
+  KERNEL_PACKAGES_RAW: https://raw.githubusercontent.com/OpenGamingCollective/kernel-packages/main
+
+jobs:
+  build:
+    name: Build Fedora RPM packages
+    runs-on: ubuntu-latest
+    timeout-minutes: 330
+    container:
+      image: docker.io/library/fedora:43
+    env:
+      CCACHE_DIR: /ccache
+      CCACHE_MAXSIZE: 10G
+      KREL: ${{ inputs.krel }}
+      KBASE: ${{ inputs.kbase }}
+      KVERDOT: ${{ inputs.kverdot }}
+      SHA8: ${{ inputs.sha8 }}
+    steps:
+      - name: Show disk space
+        run: df -h /
+
+      - name: Install build tools
+        run: |
+          set -euxo pipefail
+          dnf -y install dnf5-plugins rpm-build
+          dnf -y install cpio curl findutils tar gzip
+          mkdir -p /ccache
+
+      - name: Download staged sources
+        uses: actions/download-artifact@v8
+        with:
+          name: kernel-sources
+          path: /tmp/stage
+
+      - name: Restore compiler cache
+        uses: actions/cache@v6
+        with:
+          path: /ccache
+          key: ccache-fedora-${{ github.sha }}
+          restore-keys: |
+            ccache-fedora-
+
+      - name: Assemble Fedora kernel config
+        working-directory: /tmp/stage
+        run: |
+          set -euxo pipefail
+          # ------------------------------------------------------------------
+          # Base config: extract the official Fedora kernel .config from the
+          # distro's `kernel-core` package (same method as the OGC
+          # kernel-packages fedora.yaml workflow).
+          # ------------------------------------------------------------------
+          CACHE="/tmp/pkgcache"
+          mkdir -p "$CACHE"
+          dnf download --destdir "$CACHE" kernel-core
+          RPM="$(find "$CACHE" -maxdepth 1 -name 'kernel-core-*.rpm' -type f)"
+          if [ -z "$RPM" ]; then
+            echo "::error::kernel-core package not found in cache"; exit 1
+          fi
+          CFG="$(rpm -qlp "$RPM" | grep -E '^/lib/modules/[^/]+/config$' | head -n1 || true)"
+          if [ -z "$CFG" ]; then
+            echo "::error::kernel config not found inside $RPM"; exit 1
+          fi
+          rpm2cpio "$RPM" | cpio -i --to-stdout ".${CFG}" > config
+          test -s config
+          rm -rf "$CACHE"
+
+          # ------------------------------------------------------------------
+          # Layer the OGC config fragments on top, then the repo-local
+          # config.fragment (which wins). Order is critical: unsets apply
+          # first (removing things we explicitly don't want), then sets apply
+          # (enabling things we do want), so explicit enables can override
+          # explicit disables.
+          # ------------------------------------------------------------------
+          for f in fedora.config.set ogc.config.set fedora.config.unset ogc.config.unset; do
+            curl -fsSL "${KERNEL_PACKAGES_RAW}/config/${f}" -o "${f}"
+          done
+          bash ./merge-fragments.sh config \
+              fedora.config.unset ogc.config.unset \
+              fedora.config.set ogc.config.set \
+              config.fragment
+          echo "Config after fragment merge (head):"
+          head -n 3 config
+
+      - name: Finalize spec and install build dependencies
+        working-directory: /tmp/stage
+        run: |
+          set -euxo pipefail
+          # Substitute the CI placeholders (must happen before `dnf builddep`
+          # so RPM can parse Version:/Release:).
+          sed -i \
+            -e "s/@@KBASEVER@@/${KBASE}/" \
+            -e "s/@@KVERDOTTED@@/${KVERDOT}/" \
+            -e "s/@@SHA8@@/${SHA8}/" \
+            kernel.spec
+          grep -n '^Version:\|^Release:\|%define kbasever\|%define sha8' kernel.spec
+          ! grep -q '@@' kernel.spec
+
+          # Installs the toolchain from the spec BuildRequires, including the
+          # distro dwarves/pahole. Run BEFORE the pahole check below so a
+          # source-built pahole is not overwritten by builddep.
+          dnf -y builddep kernel.spec
+
+      - name: Ensure pahole >= 1.31 (sched_ext/BTF)
+        working-directory: /tmp
+        run: |
+          set -euxo pipefail
+          VER="$(pahole --version | tr -d 'v')"
+          MAJ="${VER%%.*}"; MIN="${VER#*.}"; MIN="${MIN%%.*}"
+          echo "Installed pahole: ${VER}"
+          if [ "$MAJ" -lt 1 ] || { [ "$MAJ" -eq 1 ] && [ "$MIN" -lt 31 ]; }; then
+            echo "pahole < 1.31 breaks sched_ext; building dwarves 1.31 from source"
+            dnf -y install cmake make gcc elfutils-devel zlib-devel
+            curl -fsSLO https://fedorapeople.org/~acme/dwarves/dwarves-1.31.tar.xz
+            echo "0a7f255ccacf8cc7f8cd119099eb327179b4b3c67cb015af646af6d0cb03054d  dwarves-1.31.tar.xz" | sha256sum -c
+            tar -xf dwarves-1.31.tar.xz
+            cmake -B dwarves-1.31/build -S dwarves-1.31 \
+              -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -D__LIB=lib
+            make -C dwarves-1.31/build -j"$(nproc)" install
+          fi
+          pahole --version
+
+      - name: Build RPMs with rpmbuild
+        working-directory: /tmp
+        run: |
+          set -euxo pipefail
+          TOPDIR=/tmp/rpmbuild
+          mkdir -p "${TOPDIR}"/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}
+          cp /tmp/stage/linux.tar.gz "${TOPDIR}/SOURCES/"
+          cp /tmp/stage/config        "${TOPDIR}/SOURCES/config"
+          cp /tmp/stage/kernel.spec   "${TOPDIR}/SPECS/kernel.spec"
+
+          rpmbuild --define "_topdir ${TOPDIR}" -ba "${TOPDIR}/SPECS/kernel.spec"
+
+          ls -lh "${TOPDIR}"/RPMS/x86_64/
+
+      - name: Collect Fedora artifacts
+        run: |
+          set -euxo pipefail
+          DISTDIR=/tmp/dist
+          mkdir -p "${DISTDIR}"
+          cp -v /tmp/rpmbuild/RPMS/x86_64/*.rpm "${DISTDIR}/"
+          cp -v /tmp/stage/config "${DISTDIR}/config-fedora-${KREL}"
+          ls -lh "${DISTDIR}"
+
+      - name: Boot smoke test in QEMU (release gate)
+        run: |
+          set -euxo pipefail
+          dnf -y install qemu-system-x86 edk2-ovmf
+          # ----------------------------------------------------------------
+          # Boot the exact kernel image that ships in the kernel-core RPM.
+          # This step fails the job (and with it the release) if the kernel
+          # panics, hangs or never reaches userspace, so an unbootable
+          # kernel is never uploaded or published.
+          # ----------------------------------------------------------------
+          WORK=/tmp/bootsmoke
+          rm -rf "$WORK"; mkdir -p "$WORK"
+          RPM="$(find /tmp/dist -name 'kernel-unstable-ogc-core-*.rpm' | head -n1)"
+          test -n "$RPM"
+          # rpm cpio entries are stored with a "./" prefix (same trick as
+          # the config extraction above).
+          rpm2cpio "$RPM" | cpio -idm --quiet -D "$WORK" "./lib/modules/${KREL}/vmlinuz" || true
+          IMG="$WORK/lib/modules/${KREL}/vmlinuz"
+          test -s "$IMG" || { echo "::error::vmlinuz not found inside ${RPM}"; exit 1; }
+          bash /tmp/stage/boot-smoke-test.sh "$IMG"
+
+      - name: Clean build tree (free disk before cache save)
+        run: rm -rf /tmp/rpmbuild/BUILD /tmp/rpmbuild/BUILDROOT ; df -h /
+
+      - name: Upload Fedora packages
+        uses: actions/upload-artifact@v7
+        with:
+          name: fedora-packages
+          path: /tmp/dist
+          retention-days: 3
diff --git a/.github/workflows/build-kernel.yml b/.github/workflows/build-kernel.yml
new file mode 100644
index 00000000000000..8ca98fa697d2d6
--- /dev/null
+++ b/.github/workflows/build-kernel.yml
@@ -0,0 +1,129 @@
+# Entry point of the linux-unstable-ogc build pipeline. The heavy lifting
+# lives in reusable workflows next to this file:
+#
+#   build-arch-packages.yml    - Arch Linux .pkg.tar.zst builds
+#   build-fedora-packages.yml  - Fedora RPM builds
+#   build-debian-packages.yml  - Debian .deb builds
+#   publish-release.yml        - collects artifacts and cuts the GitHub release
+#
+# They are called with `uses:` so everything stays a single workflow run:
+# artifacts uploaded by the build jobs are downloaded by publish-release.yml,
+# and the needs: chain below keeps the release gated on the QEMU boot smoke
+# tests that run inside each build workflow.
+
+name: Build & release linux-unstable-ogc
+
+on:
+  push:
+    branches: [master]
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+
+concurrency:
+  group: kernel-build-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  prepare:
+    name: Prepare sources and version
+    runs-on: ubuntu-latest
+    outputs:
+      kver: ${{ steps.kernel.outputs.kver }}
+      next: ${{ steps.kernel.outputs.next }}
+      sha8: ${{ steps.kernel.outputs.sha8 }}
+      krel: ${{ steps.kernel.outputs.krel }}
+      tag: ${{ steps.kernel.outputs.tag }}
+      kbase: ${{ steps.kernel.outputs.kbase }}
+      kverdot: ${{ steps.kernel.outputs.kverdot }}
+    steps:
+      - name: Checkout kernel sources
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+
+      - name: Compute kernel version
+        id: kernel
+        run: |
+          set -euxo pipefail
+          KVER="$(make -s kernelversion)"
+          NEXT="$(cat localversion-next 2>/dev/null || true)"
+          SHA8="$(git rev-parse --short=8 HEAD)"
+          KREL="${KVER}${NEXT}-unstable-ogc-g${SHA8}-1"
+          TAG="v${KVER}${NEXT}-g${SHA8}"
+          # RPM Version: field must not contain hyphens
+          KVERDOT="$(printf '%s%s' "${KVER}" "${NEXT}" | tr '-' '.')"
+          echo "kver=${KVER}"     >> "$GITHUB_OUTPUT"
+          echo "next=${NEXT}"     >> "$GITHUB_OUTPUT"
+          echo "sha8=${SHA8}"     >> "$GITHUB_OUTPUT"
+          echo "krel=${KREL}"     >> "$GITHUB_OUTPUT"
+          echo "tag=${TAG}"       >> "$GITHUB_OUTPUT"
+          echo "kbase=${KVER}${NEXT}" >> "$GITHUB_OUTPUT"
+          echo "kverdot=${KVERDOT}"   >> "$GITHUB_OUTPUT"
+          echo "Kernel release: ${KREL}"
+          echo "Release tag:    ${TAG}"
+
+      - name: Stage sources and packaging files
+        run: |
+          set -euxo pipefail
+          # Commit marker consumed by the Arch PKGBUILD (same trick as
+          # linux-bisector). Written before the tarball is created so it is
+          # included in it.
+          git rev-parse --short=8 HEAD > .build_commit
+
+          STAGE="/tmp/stage"
+          mkdir -p "${STAGE}"
+          cp .github/packaging/PKGBUILD             "${STAGE}/PKGBUILD"
+          cp .github/packaging/merge-fragments.sh   "${STAGE}/merge-fragments.sh"
+          cp .github/packaging/config.fragment      "${STAGE}/config.fragment"
+          cp .github/packaging/fedora/kernel.spec   "${STAGE}/kernel.spec"
+          cp .github/packaging/boot-smoke-test.sh   "${STAGE}/boot-smoke-test.sh"
+
+          # Source tarball: contents of the repo as ./linux/, without VCS data.
+          tar --exclude-vcs -I 'gzip -1' -cf "${STAGE}/linux.tar.gz" \
+              --transform 's|^\./|linux/|' -C "$GITHUB_WORKSPACE" .
+          ls -lh "${STAGE}"
+
+      - name: Upload staged sources
+        uses: actions/upload-artifact@v7
+        with:
+          name: kernel-sources
+          path: /tmp/stage
+          retention-days: 3
+
+  arch:
+    name: Arch packages
+    needs: prepare
+    uses: ./.github/workflows/build-arch-packages.yml
+    with:
+      krel: ${{ needs.prepare.outputs.krel }}
+
+  fedora:
+    name: Fedora packages
+    needs: prepare
+    uses: ./.github/workflows/build-fedora-packages.yml
+    with:
+      krel: ${{ needs.prepare.outputs.krel }}
+      kbase: ${{ needs.prepare.outputs.kbase }}
+      kverdot: ${{ needs.prepare.outputs.kverdot }}
+      sha8: ${{ needs.prepare.outputs.sha8 }}
+
+  debian:
+    name: Debian packages
+    needs: prepare
+    uses: ./.github/workflows/build-debian-packages.yml
+    with:
+      krel: ${{ needs.prepare.outputs.krel }}
+      kverdot: ${{ needs.prepare.outputs.kverdot }}
+      sha8: ${{ needs.prepare.outputs.sha8 }}
+
+  release:
+    name: Publish GitHub release
+    needs: [prepare, arch, fedora, debian]
+    uses: ./.github/workflows/publish-release.yml
+    with:
+      tag: ${{ needs.prepare.outputs.tag }}
+      krel: ${{ needs.prepare.outputs.krel }}
+      kver: ${{ needs.prepare.outputs.kver }}
+      next: ${{ needs.prepare.outputs.next }}
diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml
new file mode 100644
index 00000000000000..e9f641a6b36644
--- /dev/null
+++ b/.github/workflows/publish-release.yml
@@ -0,0 +1,157 @@
+# Release publisher, split out of build-kernel.yml.
+# Called (via `uses:`) by build-kernel.yml after all three distro builds
+# (each gated on its QEMU boot smoke test) have succeeded.
+
+name: Publish linux-unstable-ogc release
+
+on:
+  workflow_call:
+    inputs:
+      tag:
+        description: Release tag, e.g. v6.12.0-next-20250101-g12345678
+        type: string
+        required: true
+      krel:
+        description: Full kernel release string (uname -r)
+        type: string
+        required: true
+      kver:
+        description: Base kernel version from `make kernelversion`
+        type: string
+        required: true
+      next:
+        description: localversion-next suffix (may be empty)
+        type: string
+        required: true
+
+permissions:
+  contents: write
+
+env:
+  # Config fragments maintained by the OGC kernel-packages repository
+  # (referenced in the release notes).
+  KERNEL_PACKAGES_RAW: https://raw.githubusercontent.com/OpenGamingCollective/kernel-packages/main
+
+jobs:
+  release:
+    name: Create GitHub release
+    runs-on: ubuntu-latest
+    env:
+      TAG: ${{ inputs.tag }}
+      KREL: ${{ inputs.krel }}
+      KVER: ${{ inputs.kver }}
+      NEXT: ${{ inputs.next }}
+    steps:
+      - name: Download package artifacts
+        uses: actions/download-artifact@v8
+        with:
+          path: /tmp/dist
+          pattern: "*-packages"
+
+      - name: Flatten artifact directory
+        run: |
+          set -euxo pipefail
+          cd /tmp/dist
+          find . -mindepth 2 -maxdepth 2 -type f -exec mv -t . {} +
+          find . -mindepth 1 -type d -delete
+          ls -lh
+
+      - name: Generate checksums
+        run: |
+          set -euxo pipefail
+          cd /tmp/dist
+          sha256sum * > SHA256SUMS
+          cat SHA256SUMS
+
+      - name: Create GitHub release and upload packages
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euxo pipefail
+
+          # Idempotency: if a stale release exists for this tag (e.g. re-run
+          # of the same commit), remove it so this run can recreate it.
+          if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" 2>/dev/null; then
+            echo "Deleting existing release $TAG, will recreate it"
+            gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
+          fi
+          # In case a tag without a release is left over, drop it too.
+          gh api -X DELETE "repos/${GITHUB_REPOSITORY}/git/refs/tags/${TAG}" >/dev/null 2>&1 || true
+
+          NOTES="$(mktemp)"
+          {
+            echo "Automated build of linux-unstable-ogc."
+            echo
+            echo "- Source commit: https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}"
+            echo "- Kernel release: ${KREL}"
+            echo "- Upstream version: ${KVER}${NEXT}"
+            echo "- Base configs: Arch \`linux-headers\` + Fedora \`kernel-core\`, plus [OGC kernel-packages fragments](${KERNEL_PACKAGES_RAW}/config)"
+            echo "- Compiler: clang / LLVM=1 (with ccache)"
+            echo "- Boot smoke test: every packaged kernel (Arch, Fedora, Debian) was booted in QEMU and reached userspace init before publishing"
+            echo
+            echo "### Artifacts"
+            echo
+            echo "#### Arch Linux"
+            echo
+            echo '- `linux-unstable-ogc` — kernel image and modules'
+            echo '- `linux-unstable-ogc-headers` — headers for building external modules'
+            echo "- \`config-arch-${KREL}\` — the exact .config used for this build"
+            echo
+            echo "#### Fedora"
+            echo
+            echo '- `kernel-unstable-ogc-core` — kernel image (vmlinuz) and core files'
+            echo '- `kernel-unstable-ogc-modules` — kernel modules'
+            echo '- `kernel-unstable-ogc-devel` — headers for building external modules'
+            echo "- \`config-fedora-${KREL}\` — the exact .config used for this build"
+            echo
+            echo "#### Debian (and derivatives)"
+            echo
+            echo "- \`linux-image-${KREL}\` — kernel image and modules"
+            echo "- \`linux-headers-${KREL}\` — headers for building external modules"
+            echo "- \`config-debian-${KREL}\` — the exact .config used for this build"
+            echo
+            echo '- `SHA256SUMS` — checksums of all artifacts'
+            echo
+            echo "### Install"
+            echo
+            echo "Arch Linux:"
+            echo
+            echo '```sh'
+            echo 'sudo pacman -U linux-unstable-ogc-headers-*.pkg.tar.zst linux-unstable-ogc-*.pkg.tar.zst'
+            echo '```'
+            echo
+            echo "Fedora:"
+            echo
+            echo '```sh'
+            echo 'sudo dnf install ./kernel-unstable-ogc-core-*.rpm ./kernel-unstable-ogc-modules-*.rpm'
+            echo '```'
+            echo
+            echo "Debian and derivatives:"
+            echo
+            echo '```sh'
+            echo 'sudo apt install ./linux-image-*.deb ./linux-headers-*.deb'
+            echo '```'
+            echo
+            echo "> The initramfs is generated automatically on install (mkinitcpio hooks"
+            echo "> on Arch, kernel-install/dracut on Fedora, initramfs-tools hooks on Debian)."
+          } > "$NOTES"
+
+          cd /tmp/dist
+          # Upload every artifact produced by the distro jobs (package files
+          # plus the config-* files). SHA256SUMS itself is included by ./*.
+          gh release create "$TAG" \
+            --repo "$GITHUB_REPOSITORY" \
+            --target "$GITHUB_SHA" \
+            --title "linux-unstable-ogc ${KREL}" \
+            --notes-file "$NOTES" \
+            ./*
+
+      - name: Summary
+        run: |
+          {
+            echo "## linux-unstable-ogc build"
+            echo
+            echo "- Kernel release: \`${KREL}\`"
+            echo "- Release: https://github.com/${GITHUB_REPOSITORY}/releases/tag/${TAG}"
+            echo "- QEMU boot smoke test: passed (Arch, Fedora and Debian kernel images booted to userspace init)"
+          } >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/workflows/sync-linux-next.yml b/.github/workflows/sync-linux-next.yml
new file mode 100644
index 00000000000000..45b236c3f24cb6
--- /dev/null
+++ b/.github/workflows/sync-linux-next.yml
@@ -0,0 +1,115 @@
+name: Sync linux-next -> master (replay fork commits)
+
+on:
+  schedule:
+    - cron: "0 2 * * *"   # every day at 02:00 UTC
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+
+concurrency:
+  group: sync-linux-next
+  cancel-in-progress: false
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout fork
+        uses: actions/checkout@v7
+        with:
+          ref: master
+          fetch-depth: 0
+
+      - name: Configure upstream
+        run: |
+          git remote remove upstream || true
+          git remote add upstream https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
+          git fetch --no-tags upstream --prune
+
+      - name: Replay fork commits on top of upstream/master
+        run: |
+          set -euo pipefail
+
+          git checkout master
+
+          # REQUIRED for cherry-pick commit creation (runner has no identity by default)
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          UP_BASE="upstream/master"
+          git show -s --oneline "$UP_BASE" >/dev/null
+
+          # ------------------------------------------------------------------
+          # Find the upstream snapshot that master's fork commits sit on.
+          #
+          # Anchor: the empty marker commit "ogc: linux-unstable: first
+          # commit" is always the first fork commit ever made. Its parent is
+          # therefore the upstream snapshot the fork was built on.
+          # ------------------------------------------------------------------
+          MARKER="$(git log --format="%H" --grep="^ogc: linux-unstable: first commit$" master)"
+          if [ -z "$MARKER" ]; then
+            echo "::error::Could not find the 'ogc: linux-unstable: first commit' marker; refusing to touch master."
+            exit 1
+          fi
+          BASE="$(git rev-parse "${MARKER}^")"
+          echo "Upstream base: $(git show -s --oneline "$BASE")"
+
+          # If master already sits on the current upstream tip there is
+          # nothing to sync: replaying would only churn the fork commits'
+          # hashes for identical content.
+          if [ "$(git rev-parse "$BASE")" = "$(git rev-parse "$UP_BASE")" ]; then
+            echo "master is already based on the current linux-next tip; nothing to do."
+            exit 0
+          fi
+
+          # Everything on master on top of the upstream base = the fork
+          # commits (including non-"ogc:"-prefixed ones, e.g. squash-merged
+          # PRs). --no-merges matches the cherry-pick loop below and makes
+          # merge commits replay as their constituent commits.
+          MY_COMMITS="$(git rev-list --reverse --no-merges "${BASE}..master")"
+          if [ -z "${MY_COMMITS// }" ]; then
+            echo "::error::No commits found on top of the upstream base; refusing to reset master (that would delete the fork)."
+            exit 1
+          fi
+
+          # Safety net: a mis-detected base would replay days of upstream
+          # history. The fork accumulates its own commits over time (and PR
+          # squash-merges add to that), so allow a generous 150; anything
+          # beyond that is far more likely a mis-detected base than real fork
+          # commits, so bail out and ask for a manual look instead of
+          # corrupting master.
+          N="$(echo "$MY_COMMITS" | wc -l)"
+          if [ "$N" -gt 150 ]; then
+            echo "::error::Refusing to replay ${N} commits (expected only fork commits). Check master's history (all fork commits must carry the 'ogc:' subject prefix)."
+            exit 1
+          fi
+
+          echo "Replaying ${N} commit(s):"
+          for c in $MY_COMMITS; do
+            echo "  $c  $(git show -s --format=%s "$c")"
+          done
+          echo
+
+          # Reset master to the new upstream tip, then replay the fork commits.
+          git reset --hard "$UP_BASE"
+
+          for c in $MY_COMMITS; do
+            echo "Cherry-picking: $c  $(git show -s --format=%s "$c")"
+
+            if ! git cherry-pick -x --allow-empty "$c"; then
+              git status || true
+              # If we're left mid-cherry-pick, abort to avoid a broken
+              # working state. Nothing was pushed, so master on origin is
+              # still intact.
+              if [ -f .git/CHERRY_PICK_HEAD ]; then
+                git cherry-pick --abort || true
+              fi
+              echo "::error::Cherry-pick failed for $c; nothing was pushed."
+              exit 1
+            fi
+          done
+
+          echo "Done. Pushing updated master."
+          git push --force-with-lease origin master
diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml
new file mode 100644
index 00000000000000..29c4e2fb8f2dbe
--- /dev/null
+++ b/.github/workflows/test-pr.yml
@@ -0,0 +1,227 @@
+name: Test PR (checkpatch + config gate + gcc build)
+
+on:
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: pr-test-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  KERNEL_PACKAGES_RAW: https://raw.githubusercontent.com/OpenGamingCollective/kernel-packages/main
+
+jobs:
+  checks:
+    name: Static checks (checkpatch, new-driver symbols)
+    runs-on: ubuntu-latest
+    outputs:
+      new_symbols: ${{ steps.syms.outputs.syms }}
+    steps:
+      - name: Checkout PR merge result
+        uses: actions/checkout@v7
+        with:
+          # Default ref for pull_request is the merge commit refs/pull/N/merge.
+          # depth 2 fetches the merge commit AND its parents, so HEAD^1
+          # (current master tip) is present and the PR diff is exactly
+          # "what landing this PR changes on master".
+          fetch-depth: 2
+
+      - name: Compute PR patch and new driver symbols
+        id: syms
+        run: |
+          set -euo pipefail
+          # For pull_request, checkout gets the merge commit refs/pull/N/merge:
+          # parent 1 = master tip, parent 2 = PR head. Fall back to
+          # origin/master if HEAD is not a merge commit for any reason.
+          BASE="$(git rev-parse --verify -q HEAD^1 || true)"
+          if [ -z "$BASE" ]; then
+            echo "HEAD^1 unresolvable; falling back to origin/master"
+            git fetch --depth=1 origin master
+            BASE="$(git rev-parse --verify -q FETCH_HEAD || true)"
+          fi
+          if [ -z "$BASE" ]; then
+            echo "::error::Could not determine the base commit for the PR diff."
+            exit 1
+          fi
+          git diff "$BASE" HEAD > /tmp/pr.patch
+
+          if [ ! -s /tmp/pr.patch ]; then
+            echo "Empty patch; nothing to check."
+            echo "syms=" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # Selectable symbols introduced under drivers/ (added
+          # "config FOO" / "menuconfig FOO" lines in any Kconfig file).
+          # NOTE: grep exits 1 on zero matches; under `set -euo pipefail`
+          # that would abort the step, so it is neutralised here only
+          # (git diff / awk failures still propagate).
+          SYMS="$(git diff "$BASE" HEAD -- drivers/ \
+            | { grep -E '^\+[[:space:]]*(menu)?config[[:space:]]+[A-Z0-9_]+' || true; } \
+            | awk '{print $NF}' | sort -u | tr '\n' ' ')"
+          echo "syms=${SYMS}" >> "$GITHUB_OUTPUT"
+          {
+            echo "### PR checks"
+            echo
+            echo "- Changed files: $(git diff --name-only "$BASE" HEAD | wc -l)"
+            echo "- New driver CONFIG symbols: ${SYMS:-none}"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: checkpatch (fail on errors)
+        run: |
+          set -uo pipefail
+          if [ ! -s /tmp/pr.patch ]; then
+            echo "Empty patch; skipping checkpatch."
+            exit 0
+          fi
+          # --no-signoff: internal fork PRs do not require Signed-off-by.
+          perl scripts/checkpatch.pl --no-signoff /tmp/pr.patch \
+            > /tmp/checkpatch.out 2>&1 || true
+          cat /tmp/checkpatch.out
+
+          if grep -q '^ERROR:' /tmp/checkpatch.out; then
+            NERRS="$(grep -c '^ERROR:' /tmp/checkpatch.out)"
+            echo "::error::checkpatch reported ${NERRS} error(s); fix them before merge (warnings do not block)."
+            exit 1
+          fi
+          echo "checkpatch: no errors."
+
+  build:
+    name: Build with GCC (Arch config + fragments)
+    needs: checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 330
+    container:
+      image: docker.io/archlinux:base-devel
+    env:
+      CCACHE_DIR: /ccache
+      CCACHE_MAXSIZE: 10G
+      NEW_SYMBOLS: ${{ needs.checks.outputs.new_symbols }}
+    steps:
+      - name: Show disk space
+        run: df -h /
+
+      - name: Bootstrap build environment
+        run: |
+          set -euxo pipefail
+          pacman -Sy --needed --noconfirm archlinux-keyring
+          pacman -Su --needed --noconfirm
+          # base-devel provides gcc/make/perl; the rest mirrors the makedepends
+          # of the release PKGBUILD minus clang and the rust toolchain.
+          pacman -S --needed --noconfirm \
+            bc cpio gettext libelf pahole perl python tar xz zstd \
+            file curl git ccache
+
+      - name: Checkout PR merge result
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+
+      - name: Restore compiler cache
+        uses: actions/cache@v6
+        with:
+          path: /ccache
+          key: ccache-pr-gcc-${{ github.sha }}
+          restore-keys: |
+            ccache-pr-gcc-
+
+      - name: Assemble test config (same as the Arch release build)
+        run: |
+          set -euxo pipefail
+          # ------------------------------------------------------------------
+          # Base config: official Arch Linux kernel .config extracted from the
+          # distro's linux-headers package (same method as the release build).
+          # ------------------------------------------------------------------
+          CACHE="/tmp/pkgcache"
+          mkdir -p "$CACHE"
+          pacman -Sw --noconfirm --cachedir "$CACHE" linux-headers
+          PKG="$(find "$CACHE" -maxdepth 1 -name 'linux-headers-*.pkg.tar.zst' -type f)"
+          if [ -z "$PKG" ]; then
+            echo "::error::linux-headers package not found in cache"; exit 1
+          fi
+          CFG="$(tar --zstd -tf "$PKG" | grep -E '^usr/lib/modules/[^/]+/build/\.config$' || true)"
+          if [ -z "$CFG" ]; then
+            echo "::error::kernel .config not found inside $PKG"; exit 1
+          fi
+          tar --zstd -xOf "$PKG" "$CFG" > .config
+          test -s .config
+          rm -rf "$CACHE"
+
+          # ------------------------------------------------------------------
+          # OGC kernel-packages fragments + the repo-local config.fragment
+          # (merged last, wins). Same order as the release build.
+          # ------------------------------------------------------------------
+          for f in arch.config.set ogc.config.set arch.config.unset ogc.config.unset; do
+            curl -fsSL "${KERNEL_PACKAGES_RAW}/config/${f}" -o "${f}"
+          done
+          bash .github/packaging/merge-fragments.sh .config \
+              arch.config.set ogc.config.set \
+              arch.config.unset ogc.config.unset \
+              .github/packaging/config.fragment
+
+          # Same fixups as the release packaging...
+          scripts/config --set-str SYSTEM_TRUSTED_KEYS ""
+          scripts/config --set-str SYSTEM_REVOCATION_KEYS ""
+          scripts/config --set-str MODULE_SIG_KEY "certs/signing_key.pem"
+          scripts/config --set-str CONFIG_LOCALVERSION "" || true
+          # ...plus test-only tweaks:
+          # No rust toolchain is installed in this job; the release builds
+          # (clang) compile the rust bits, this job focuses on the C parts.
+          scripts/config -d RUST
+          make olddefconfig
+
+      - name: "Gate: new driver symbols must be enabled in the config"
+        run: |
+          set -euo pipefail
+          if [ -z "${NEW_SYMBOLS// }" ]; then
+            echo "No new driver CONFIG symbols in this PR; skipping gate."
+            exit 0
+          fi
+          echo "Gate symbols: ${NEW_SYMBOLS}"
+
+          FAIL=0
+          for s in ${NEW_SYMBOLS}; do
+            if grep -qE "^CONFIG_${s}=[ym]$" .config; then
+              echo "OK: CONFIG_${s} is enabled."
+            elif grep -q "^# CONFIG_${s} is not set$" .config; then
+              echo "::error::CONFIG_${s} was added by this PR but is disabled in the merged config."
+              echo "::error::Enable it in .github/packaging/config.fragment (CONFIG_${s}=y or =m) if it should ship."
+              FAIL=1
+            elif grep -q "^CONFIG_${s}=" .config; then
+              echo "::error::CONFIG_${s} is set but not to y/m: $(grep "^CONFIG_${s}=" .config)"
+              FAIL=1
+            else
+              echo "::error::CONFIG_${s} is absent from the final .config (its dependencies or its vendor menu are off in the merged config)."
+              echo "::error::If this driver should ship, enable it (and its dependencies) in .github/packaging/config.fragment."
+              FAIL=1
+            fi
+          done
+          if [ "$FAIL" -ne 0 ]; then
+            echo "::error::config gate failed: new drivers must be enabled in the OGC config."
+            exit 1
+          fi
+          echo "config gate passed."
+
+      - name: Build kernel with GCC
+        run: |
+          set -euxo pipefail
+          ccache -s || true
+          make CC="ccache gcc" WERROR=0 -j"$(nproc)" all
+          echo "Kernel release: $(make -s kernelrelease)"
+          ccache -s
+          df -h /
+
+      - name: Summary
+        if: always()
+        run: |
+          {
+            echo "### GCC compile test"
+            echo
+            echo "- Compiler: gcc (Arch Linux) via ccache"
+            echo "- Config: Arch linux-headers base + OGC fragments + repo config.fragment (Rust disabled for this test)"
+            echo "- New driver symbols checked: ${NEW_SYMBOLS:-none}"
+          } >> "$GITHUB_STEP_SUMMARY"
\ No newline at end of file
diff --git a/.mailmap b/.mailmap
index 1f5540bc33f1fb..1e91aae3feef6f 100644
--- a/.mailmap
+++ b/.mailmap
@@ -10,6 +10,7 @@
 #
 # Please keep this list dictionary sorted.
 #
+Aaradhana Sahu <aaradhana.sahu@oss.qualcomm.com> <quic_aarasahu@quicinc.com>
 Aaron Durbin <adurbin@google.com>
 Abel Vesa <abelvesa@kernel.org> <abel.vesa@nxp.com>
 Abel Vesa <abelvesa@kernel.org> <abel.vesa@linaro.org>
@@ -67,6 +68,7 @@ Alex Hung <alexhung@gmail.com> <alex.hung@canonical.com>
 Alex Shi <alexs@kernel.org> <alex.shi@intel.com>
 Alex Shi <alexs@kernel.org> <alex.shi@linaro.org>
 Alex Shi <alexs@kernel.org> <alex.shi@linux.alibaba.com>
+Ali Ahmet Memiş <aliamemis@disroot.org> <ali@iusegentoo.com>
 Alice Mikityanska <alice.kernel@fastmail.im> <maxtram95@gmail.com>
 Alice Mikityanska <alice.kernel@fastmail.im> <maximmi@mellanox.com>
 Alice Mikityanska <alice.kernel@fastmail.im> <maximmi@nvidia.com>
@@ -91,6 +93,7 @@ Andrew Morton <akpm@linux-foundation.org>
 Andrew Murray <amurray@thegoodpenguin.co.uk> <amurray@embedded-bits.co.uk>
 Andrew Murray <amurray@thegoodpenguin.co.uk> <andrew.murray@arm.com>
 Andrew Vasquez <andrew.vasquez@qlogic.com>
+Andrey Golovko <andrey.golovko@gmail.com> <andrey@golovko.me>
 Andrey Konovalov <andreyknvl@gmail.com> <andreyknvl@google.com>
 Andrey Ryabinin <ryabinin.a.a@gmail.com> <a.ryabinin@samsung.com>
 Andrey Ryabinin <ryabinin.a.a@gmail.com> <aryabinin@virtuozzo.com>
@@ -101,6 +104,7 @@ Andy Chiu <andybnac@gmail.com> <andy.chiu@sifive.com>
 Andy Chiu <andybnac@gmail.com> <taochiu@synology.com>
 Andy Shevchenko <andy@kernel.org> <andy@smile.org.ua>
 Andy Shevchenko <andy@kernel.org> <ext-andriy.shevchenko@nokia.com>
+Andy Yan <andyshrk@163.com> <andy.yan@rock-chips.com>
 Anilkumar Kolli <quic_akolli@quicinc.com> <akolli@codeaurora.org>
 Anirudh Ghayal <quic_aghayal@quicinc.com> <aghayal@codeaurora.org>
 Antoine Tenart <atenart@kernel.org> <antoine.tenart@bootlin.com>
@@ -130,7 +134,8 @@ Axel Dyks <xl@xlsigned.net>
 Axel Lin <axel.lin@gmail.com>
 Balakrishna Godavarthi <quic_bgodavar@quicinc.com> <bgodavar@codeaurora.org>
 Banajit Goswami <quic_bgoswami@quicinc.com> <bgoswami@codeaurora.org>
-Baochen Qiang <quic_bqiang@quicinc.com> <bqiang@codeaurora.org>
+Baochen Qiang <baochen.qiang@oss.qualcomm.com> <bqiang@codeaurora.org>
+Baochen Qiang <baochen.qiang@oss.qualcomm.com> <quic_bqiang@quicinc.com>
 Baolin Wang <baolin.wang@linux.alibaba.com> <baolin.wang@linaro.org>
 Baolin Wang <baolin.wang@linux.alibaba.com> <baolin.wang@spreadtrum.com>
 Baolin Wang <baolin.wang@linux.alibaba.com> <baolin.wang@unisoc.com>
@@ -171,6 +176,7 @@ Boris Brezillon <bbrezillon@kernel.org> <b.brezillon.dev@gmail.com>
 Boris Brezillon <bbrezillon@kernel.org> <b.brezillon@overkiz.com>
 Boris Brezillon <bbrezillon@kernel.org> <boris.brezillon@bootlin.com>
 Boris Brezillon <bbrezillon@kernel.org> <boris.brezillon@free-electrons.com>
+Bradley Morgan <brads@mainlining.org> <include@grrlz.net>
 Brendan Higgins <brendan.higgins@linux.dev> <brendanhiggins@google.com>
 Brendan Jackman <brendan.jackman@linux.dev> <jackmanb@google.com>
 Brian Avery <b.avery@hp.com>
@@ -286,6 +292,8 @@ Ed L. Cashin <ecashin@coraid.com>
 Elliot Berman <quic_eberman@quicinc.com> <eberman@codeaurora.org>
 Enric Balletbo i Serra <eballetbo@kernel.org> <enric.balletbo@collabora.com>
 Enric Balletbo i Serra <eballetbo@kernel.org> <eballetbo@iseebcn.com>
+Eric Dumazet <edumazet@kernel.org> <edumazet@google.com>
+Eric Dumazet <edumazet@kernel.org> <eric.dumazet@gmail.com>
 Erik Kaneda <erik.kaneda@intel.com> <erik.schmauss@intel.com>
 Ethan Carter Edwards <ethan@ethancedwards.com> Ethan Edwards <ethancarteredwards@gmail.com>
 Eugen Hristev <ehristev@kernel.org> <eugen.hristev@microchip.com>
@@ -355,6 +363,8 @@ Hans Verkuil <hverkuil@kernel.org> <hverkuil-cisco@xs4all.nl>
 Hans Verkuil <hverkuil@kernel.org> <hansverk@cisco.com>
 Hans Verkuil <hverkuil@kernel.org> <hans.verkuil@cisco.com>
 Hao Ge <hao.ge@linux.dev> <gehao@kylinos.cn>
+Haowen Bai <baihaowen88@gmail.com> <baihaowen@meizu.com>
+Haowen Bai <baihaowen88@gmail.com> <calvin.bai@ugreen.com>
 Harry Yoo <harry.yoo@oracle.com> <42.hyeyoo@gmail.com>
 Harry Yoo <harry@kernel.org> <harry.yoo@oracle.com>
 Heiko Carstens <hca@linux.ibm.com> <h.carstens@de.ibm.com>
@@ -457,7 +467,8 @@ Johan Hovold <johan@kernel.org> <jhovold@gmail.com>
 Johan Hovold <johan@kernel.org> <johan@hovoldconsulting.com>
 John Crispin <john@phrozen.org> <blogic@openwrt.org>
 John Fastabend <john.fastabend@gmail.com> <john.r.fastabend@intel.com>
-John Garry <john.g.garry@oracle.com> <john.garry@huawei.com>
+John Garry <john.garry@linux.dev> <john.garry@huawei.com>
+John Garry <john.garry@linux.dev> <john.g.garry@oracle.com>
 John Keeping <john@keeping.me.uk> <john@metanate.com>
 John Moon <john@jmoon.dev> <quic_johmoo@quicinc.com>
 John Paul Adrian Glaubitz <glaubitz@physik.fu-berlin.de>
@@ -575,6 +586,7 @@ Maheshwar Ajja <quic_majja@quicinc.com> <majja@codeaurora.org>
 Maíra Canal <mairacanal@riseup.net> <maira.canal@usp.br>
 Malathi Gottam <quic_mgottam@quicinc.com> <mgottam@codeaurora.org>
 Manikanta Pubbisetty <quic_mpubbise@quicinc.com> <mpubbise@codeaurora.org>
+Manish Dharanenthiran <manish.dharanenthiran@oss.qualcomm.com> <quic_mdharane@quicinc.com>
 Manivannan Sadhasivam <mani@kernel.org> <manivannanece23@gmail.com>
 Manivannan Sadhasivam <mani@kernel.org> <manivannan.sadhasivam@linaro.org>
 Manoj Basapathi <quic_manojbm@quicinc.com> <manojbm@codeaurora.org>
@@ -764,6 +776,8 @@ Rakesh Pillai <quic_pillair@quicinc.com> <pillair@codeaurora.org>
 Ralf Baechle <ralf@linux-mips.org>
 Ralf Wildenhues <Ralf.Wildenhues@gmx.de>
 Ram Chandra Jangir <quic_rjangir@quicinc.com> <rjangir@codeaurora.org>
+Rameshkumar Sundaram <rameshkumar.sundaram@oss.qualcomm.com> <quic_ramess@quicinc.com>
+Rameshkumar Sundaram <rameshkumar.sundaram@oss.qualcomm.com> <ramess@codeaurora.org>
 Randy Dunlap <rdunlap@infradead.org> <rdunlap@xenotime.net>
 Randy Dunlap <rdunlap@infradead.org> <randy.dunlap@oracle.com>
 Randy Dunlap <rdunlap@infradead.org> <rddunlap@osdl.org>
@@ -862,7 +876,8 @@ Sricharan Ramabadhran <quic_srichara@quicinc.com> <sricharan@codeaurora.org>
 Srinivas Kandagatla <srini@kernel.org> <srinivas.kandagatla@st.com>
 Srinivas Kandagatla <srini@kernel.org> <srinivas.kandagatla@linaro.org>
 Srinivas Ramana <quic_sramana@quicinc.com> <sramana@codeaurora.org>
-Sriram R <quic_srirrama@quicinc.com> <srirrama@codeaurora.org>
+Sriram R <sriram.ramachandran@oss.qualcomm.com> <quic_srirrama@quicinc.com>
+Sriram R <sriram.ramachandran@oss.qualcomm.com> <srirrama@codeaurora.org>
 Sriram Yagnaraman <sriram.yagnaraman@ericsson.com> <sriram.yagnaraman@est.tech>
 Stanislav Fomichev <sdf@fomichev.me> <sdf@google.com>
 Stanislav Fomichev <sdf@fomichev.me> <stfomichev@gmail.com>
@@ -946,7 +961,8 @@ Vadim Fedorenko <vadim.fedorenko@linux.dev> <vfedorenko@novek.ru>
 Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
 Vara Reddy <quic_varar@quicinc.com> <varar@codeaurora.org>
 Varadarajan Narayanan <quic_varada@quicinc.com> <varada@codeaurora.org>
-Vasanthakumar Thiagarajan <quic_vthiagar@quicinc.com> <vthiagar@codeaurora.org>
+Vasanthakumar Thiagarajan <vasanthakumar.thiagarajan@oss.qualcomm.com> <quic_vthiagar@quicinc.com>
+Vasanthakumar Thiagarajan <vasanthakumar.thiagarajan@oss.qualcomm.com> <vthiagar@codeaurora.org>
 Vasily Averin <vasily.averin@linux.dev> <vvs@virtuozzo.com>
 Vasily Averin <vasily.averin@linux.dev> <vvs@openvz.org>
 Vasily Averin <vasily.averin@linux.dev> <vvs@parallels.com>
@@ -975,6 +991,7 @@ Vladimir Davydov <vdavydov.dev@gmail.com> <vdavydov@virtuozzo.com>
 Vlastimil Babka <vbabka@kernel.org> <vbabka@suse.cz>
 WangYuli <wangyuli@aosc.io> <wangyl5933@chinaunicom.cn>
 WangYuli <wangyuli@aosc.io> <wangyuli@deepin.org>
+Wei Wang <weiwan@google.com> <weibunny@fb.com>
 Weiwen Hu <huweiwen@linux.alibaba.com> <sehuww@mail.scut.edu.cn>
 WeiXiong Liao <gmpy.liaowx@gmail.com> <liaoweixiong@allwinnertech.com>
 Wen Gong <quic_wgong@quicinc.com> <wgong@codeaurora.org>
@@ -982,6 +999,8 @@ Wesley Cheng <quic_wcheng@quicinc.com> <wcheng@codeaurora.org>
 Will Deacon <will@kernel.org> <will.deacon@arm.com>
 Wolfram Sang <wsa@kernel.org> <w.sang@pengutronix.de>
 Wolfram Sang <wsa@kernel.org> <wsa@the-dreams.de>
+Xu Xin <xu.xin@linux.dev> <xu.xin16@zte.com.cn>
+Xu Xin <xu.xin@linux.dev> xu xin <cgel.zte@gmail.com>
 Yakir Yang <kuankuan.y@gmail.com> <ykk@rock-chips.com>
 Yanteng Si <si.yanteng@linux.dev> <siyanteng@loongson.cn>
 Ying Huang <huang.ying.caritas@gmail.com> <ying.huang@intel.com>
diff --git a/CREDITS b/CREDITS
index a0bd941ca5dd26..f280ab24476a18 100644
--- a/CREDITS
+++ b/CREDITS
@@ -1,6 +1,7 @@
 	This is at least a partial credits-file of people that have
-	contributed to the Linux project.  It is sorted by name and
-	formatted to allow easy grepping and beautification by
+	contributed to the Linux project.  It is sorted by family name,
+	assuming the used locale is default, e.g. by setting LC_ALL=C,
+	and formatted to allow easy grepping and beautification by
 	scripts.  The fields are: name (N), email (E), web-address
 	(W), PGP key ID and fingerprint (P), description (D), and
 	snail-mail address (S).
@@ -57,7 +58,7 @@ S: Longford, Ireland
 S: Sydney, Australia
 
 N: Tigran A. Aivazian
-E: tigran@aivazian.fsnet.co.uk
+E: aivazian.tigran@gmail.com
 W: http://www.moses.uklinux.net/patches
 D: BFS filesystem
 D: Intel IA32 CPU microcode update support
@@ -344,6 +345,14 @@ D: Hardware spinlock (hwspinlock) subsystem
 D: OMAP hwspinlock driver
 D: OMAP remoteproc driver
 
+N: Muli Ben-Yehuda
+E: mulix@mulix.org
+E: muli@il.ibm.com
+W: http://www.mulix.org
+D: trident OSS sound driver, x86-64 dma-ops and Calgary IOMMU,
+D: KVM and Xen bits and other misc. hackery.
+S: Haifa, Israel
+
 N: Krzysztof Benedyczak
 E: golbi@mat.uni.torun.pl
 W: http://www.mat.uni.torun.pl/~golbi
@@ -362,14 +371,6 @@ S: 2322 37th Ave SW
 S: Seattle, Washington 98126-2010
 S: USA
 
-N: Muli Ben-Yehuda
-E: mulix@mulix.org
-E: muli@il.ibm.com
-W: http://www.mulix.org
-D: trident OSS sound driver, x86-64 dma-ops and Calgary IOMMU,
-D: KVM and Xen bits and other misc. hackery.
-S: Haifa, Israel
-
 N: Johannes Berg
 E: johannes@sipsolutions.net
 W: https://johannes.sipsolutions.net/
@@ -393,14 +394,14 @@ S: 1549 Hiironen Rd.
 S: Brimson, MN  55602
 S: USA
 
-N: Arnd Bergmann
-D: Maintainer of Cell Broadband Engine Architecture
-
 N: Hennus Bergman
 P: 1024/77D50909 76 99 FD 31 91 E1 96 1C  90 BB 22 80 62 F6 BD 63
 D: Author and maintainer of the QIC-02 tape driver
 S: The Netherlands
 
+N: Arnd Bergmann
+D: Maintainer of Cell Broadband Engine Architecture
+
 N: Tomas Berndtsson
 E: tomas@nocrew.org
 W: http://tomas.nocrew.org/
@@ -492,10 +493,6 @@ D: Various fixes (mostly networking)
 S: Montreal, Quebec
 S: Canada
 
-N: Zoltán Böszörményi
-E: zboszor@mail.externet.hu
-D: MTRR emulation with Cyrix style ARR registers, Athlon MTRR support
-
 N: John Boyd
 E: boyd@cis.ohio-state.edu
 D: Co-author of wd7000 SCSI driver
@@ -589,9 +586,6 @@ N: Zach Brown
 E: zab@zabbo.net
 D: maestro pci sound
 
-N: Zefan Li
-D: Contribution to control group stuff
-
 N: David Brownell
 D: Kernel engineer, mentor, and friend.  Maintained USB EHCI and
 D: gadget layers, SPI subsystem, GPIO subsystem, and more than a few
@@ -632,6 +626,10 @@ S: Ravenhorst 58
 S: 2317 AK Leiden
 S: The Netherlands
 
+N: Zoltán Böszörményi
+E: zboszor@mail.externet.hu
+D: MTRR emulation with Cyrix style ARR registers, Athlon MTRR support
+
 N: Michael Callahan
 E: callahan@maths.ox.ac.uk
 D: PPP for Linux
@@ -703,6 +701,10 @@ S: Tamsui town, Taipei county,
 S: Taiwan 251
 S: Republic of China
 
+N: Landen Chao
+E: Landen.Chao@mediatek.com
+D: MT7531 Ethernet switch support
+
 N: Michael Elizabeth Chastain
 E: mec@shout.net
 D: Configure, Menuconfig, xconfig
@@ -715,10 +717,6 @@ D: Media subsystem (V4L/DVB) drivers and core
 D: EDAC drivers and EDAC 3.0 core rework
 S: Brazil
 
-N: Landen Chao
-E: Landen.Chao@mediatek.com
-D: MT7531 Ethernet switch support
-
 N: Raymond Chen
 E: raymondc@microsoft.com
 D: Author of Configure script
@@ -1447,6 +1445,10 @@ D: Made support for modules, ramdisk, generic-serial, etc. optional.
 D: Transformed old user space bdflush into 1st kernel thread - kflushd.
 D: Many other patches, documentation files, mini kernels, utilities, ...
 
+N: Andy Gospodarek
+E: andy@greyhouse.net
+D: Maintenance and contributions to the network interface bonding driver.
+
 N: Masanori GOTO
 E: gotom@debian.or.jp
 D: Workbit NinjaSCSI-32Bi/UDE driver
@@ -1459,10 +1461,6 @@ S: 8124 Constitution Apt. 7
 S: Sterling Heights, Michigan 48313
 S: USA
 
-N: Andy Gospodarek
-E: andy@greyhouse.net
-D: Maintenance and contributions to the network interface bonding driver.
-
 N: Vivek Goyal
 E: vgoyal@redhat.com
 D: KDUMP, KEXEC, and VIRTIO FILE SYSTEM
@@ -1535,6 +1533,14 @@ S: 44 St. Joseph Street, Suite 506
 S: Toronto, Ontario, M4Y 2W4
 S: Canada
 
+N: Nitin Gupta
+E: ngupta@vflare.org
+D: zsmalloc memory allocator and zram block device driver
+
+N: Justin Guyett
+E: jguyett@andrew.cmu.edu
+D: via-rhine net driver hacking
+
 N: Richard Günther
 E: rguenth@tat.physik.uni-tuebingen.de
 W: http://www.tat.physik.uni-tuebingen.de/~rguenth
@@ -1543,14 +1549,6 @@ D: binfmt_misc
 S: 72074 Tübingen
 S: Germany
 
-N: Justin Guyett
-E: jguyett@andrew.cmu.edu
-D: via-rhine net driver hacking
-
-N: Nitin Gupta
-E: ngupta@vflare.org
-D: zsmalloc memory allocator and zram block device driver
-
 N: Danny ter Haar
 E: dth@cistron.nl
 D: /proc/cpuinfo, reboot on panic , kernel pre-patch tester ;)
@@ -1945,16 +1943,6 @@ E: sjenning@redhat.com
 D: Creation and maintenance of zswap
 D: Creation and maintenace of the zbud allocator
 
-N: Jeremy Kerr
-D: Maintainer of SPU File System
-
-N: Michael Kerrisk
-E: mtk.manpages@gmail.com
-W: https://man7.org/
-P: 4096R/3A35CE5E E522 595B 52ED A4E6 BFCC  CB5E 8561 9911 3A35 CE5E
-D: Maintainer of the Linux man-pages project
-D: Linux man pages online, at <https://man7.org/linux/man-pages/>
-
 N: Niels Kristian Bech Jensen
 E: nkbj1970@hotmail.com
 D: Miscellaneous kernel updates and fixes.
@@ -2099,6 +2087,16 @@ S: Keplerstr. 6
 S: 4050 Traun
 S: Austria
 
+N: Jeremy Kerr
+D: Maintainer of SPU File System
+
+N: Michael Kerrisk
+E: mtk.manpages@gmail.com
+W: https://man7.org/
+P: 4096R/3A35CE5E E522 595B 52ED A4E6 BFCC  CB5E 8561 9911 3A35 CE5E
+D: Maintainer of the Linux man-pages project
+D: Linux man pages online, at <https://man7.org/linux/man-pages/>
+
 N: Karl Keyte
 E: karl@koft.com
 D: Disk usage statistics and modifications to line printer driver
@@ -2483,6 +2481,9 @@ D: and much more. He was the maintainer of MD from 2016 to 2018. Shaohua
 D: passed away late 2018, he will be greatly missed.
 W: https://www.spinics.net/lists/raid/msg61993.html
 
+N: Zefan Li
+D: Contribution to control group stuff
+
 N: Stephan Linz
 E: linz@mazet.de
 E: Stephan.Linz@gmx.de
@@ -2542,11 +2543,6 @@ D: misc. kernel hacking and debugging
 S: Cambridge, MA 02139
 S: USA
 
-N: Martin von Löwis
-E: loewis@informatik.hu-berlin.de
-D: script binary format
-D: NTFS driver
-
 N: H.J. Lu
 E: hjl@gnu.ai.mit.edu
 D: GCC + libraries hacker
@@ -2575,6 +2571,11 @@ S: Puistokaari 1 E 18
 S: 00200 Helsinki
 S: Finland
 
+N: Martin von Löwis
+E: loewis@informatik.hu-berlin.de
+D: script binary format
+D: NTFS driver
+
 N: Daniel J. Maas
 E: dmaas@dcine.com
 W: https://www.maasdigital.com
@@ -2625,10 +2626,6 @@ S: PO BOX 220, HFX. CENTRAL
 S: Halifax, Nova Scotia
 S: Canada B3J 3C8
 
-N: Kai Mäkisara
-E: Kai.Makisara@kolumbus.fi
-D: SCSI Tape Driver
-
 N: Asit Mallick
 E: asit.k.mallick@intel.com
 D: Linux/IA-64
@@ -2912,13 +2909,6 @@ S: 12725 SW Millikan Way, Suite 400
 S: Beaverton, Oregon 97005
 S: USA
 
-N: Eberhard Mönkeberg
-E: emoenke@gwdg.de
-D: CDROM driver "sbpcd" (Matsushita/Panasonic/Soundblaster)
-S: Ruhstrathöhe 2 b.
-S: D-37085 Göttingen
-S: Germany
-
 N: Thomas Molina
 E: tmolina@cablespeed.com
 D: bug fixes, documentation, minor hackery
@@ -3005,6 +2995,17 @@ S: Dragonvagen 1 A 13
 S: FIN-00330 Helsingfors
 S: Finland
 
+N: Kai Mäkisara
+E: Kai.Makisara@kolumbus.fi
+D: SCSI Tape Driver
+
+N: Eberhard Mönkeberg
+E: emoenke@gwdg.de
+D: CDROM driver "sbpcd" (Matsushita/Panasonic/Soundblaster)
+S: Ruhstrathöhe 2 b.
+S: D-37085 Göttingen
+S: Germany
+
 N: Matija Nalis
 E: mnalis@jagor.srce.hr
 E: mnalis@voyager.hr
@@ -3113,6 +3114,11 @@ N: Peter Oruba
 D: AMD Microcode loader driver
 S: Germany
 
+N: Pierre Ossman
+D: MMC maintainer 2006-2009
+D: First SDIO implementation
+S: Sweden
+
 N: Jens Osterkamp
 E: jens@de.ibm.com
 D: Maintainer of Spidernet network driver for Cell
@@ -3200,13 +3206,6 @@ S: RR #5, 497 Pole Line Road
 S: Thunder Bay, Ontario
 S: CANADA P7C 5M9
 
-N: Inaky Perez-Gonzalez
-E: inaky.perez-gonzalez@intel.com
-E: linux-wimax@intel.com
-E: inakypg@yahoo.com
-D: WiMAX stack
-D: Intel Wireless WiMAX Connection 2400 driver
-
 N: Yuri Per
 E: yuri@pts.mipt.ru
 D: Some smbfs fixes
@@ -3214,6 +3213,13 @@ S: Demonstratsii 8-382
 S: Tula 300000
 S: Russia
 
+N: Inaky Perez-Gonzalez
+E: inaky.perez-gonzalez@intel.com
+E: linux-wimax@intel.com
+E: inakypg@yahoo.com
+D: WiMAX stack
+D: Intel Wireless WiMAX Connection 2400 driver
+
 N: Thomas Petazzoni
 E: thomas.petazzoni@bootlin.com
 D: Driver for the Marvell Armada 370/XP network unit.
@@ -3328,14 +3334,14 @@ N: Frederic Potter
 E: fpotter@cirpack.com
 D: Some PCI kernel support
 
-N: Rui Prior
-E: rprior@inescn.pt
-D: ATM device driver for NICStAR based cards
-
 N: Roopa Prabhu
 E: roopa@nvidia.com
 D: Bridge co-maintainer, vxlan and networking contributor
 
+N: Rui Prior
+E: rprior@inescn.pt
+D: ATM device driver for NICStAR based cards
+
 N: Stefan Probst
 E: sp@caldera.de
 D: The Linux Support Team Erlangen, 1993-97
@@ -4023,12 +4029,6 @@ S: C/ Federico Garcia Lorca 1 10-A
 S: Sevilla 41005
 S: Spain
 
-N: Björn Töpel
-E: bjorn@kernel.org
-D: AF_XDP
-S: Gothenburg
-S: Sweden
-
 N: Linus Torvalds
 E: torvalds@linux-foundation.org
 D: Original kernel hacker
@@ -4082,17 +4082,6 @@ D: Linux-Workshop Köln (aka LUG Cologne, Germany), Installfests
 S: Tacitusstr. 6
 S: D-50968 Köln
 
-N: Tsu-Sheng Tsao
-E: tsusheng@scf.usc.edu
-D: IGMP (Internet Group Management Protocol) version 2
-S: 2F 14 ALY 31 LN 166 SEC 1 SHIH-PEI RD
-S: Taipei
-S: Taiwan 112
-S: Republic of China
-S: 24335 Delta Drive
-S: Diamond Bar, California 91765
-S: USA
-
 N: Theodore Ts'o
 E: tytso@mit.edu
 D: Random Linux hacker
@@ -4109,6 +4098,17 @@ S: 1 Amherst Street
 S: Cambridge, Massachusetts 02139
 S: USA
 
+N: Tsu-Sheng Tsao
+E: tsusheng@scf.usc.edu
+D: IGMP (Internet Group Management Protocol) version 2
+S: 2F 14 ALY 31 LN 166 SEC 1 SHIH-PEI RD
+S: Taipei
+S: Taiwan 112
+S: Republic of China
+S: 24335 Delta Drive
+S: Diamond Bar, California 91765
+S: USA
+
 N: Luben Tuikov
 E: Luben Tuikov <ltuikov89@gmail.com>
 D: Maintainer of the DRM GPU Scheduler
@@ -4135,6 +4135,12 @@ S: 44 Campbell Park Crescent
 S: Edinburgh EH13 0HT
 S: United Kingdom
 
+N: Björn Töpel
+E: bjorn@kernel.org
+D: AF_XDP
+S: Gothenburg
+S: Sweden
+
 N: Thomas Uhl
 E: uhl@sun1.rz.fh-heilbronn.de
 D: Application programmer
diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband
index 7ba1161034291c..726a38470e5854 100644
--- a/Documentation/ABI/stable/sysfs-class-infiniband
+++ b/Documentation/ABI/stable/sysfs-class-infiniband
@@ -422,124 +422,6 @@ Description:
 				information)
 		=============== =============================================
 
-
-sysfs interface for Intel IB driver qib
----------------------------------------
-
-What:		/sys/class/infiniband/qibX/version
-What:		/sys/class/infiniband/qibX/hw_rev
-What:		/sys/class/infiniband/qibX/hca_type
-What:		/sys/class/infiniband/qibX/board_id
-What:		/sys/class/infiniband/qibX/boardversion
-What:		/sys/class/infiniband/qibX/nctxts
-What:		/sys/class/infiniband/qibX/localbus_info
-What:		/sys/class/infiniband/qibX/tempsense
-What:		/sys/class/infiniband/qibX/serial
-What:		/sys/class/infiniband/qibX/nfreectxts
-What:		/sys/class/infiniband/qibX/chip_reset
-Date:		May, 2010
-KernelVersion:	v2.6.35
-Contact:	linux-rdma@vger.kernel.org
-Description:
-		=============== ======================================================
-		version:	(RO) Display version information of installed software
-				and drivers.
-
-		hw_rev:		(RO) Hardware revision number
-
-		hca_type:	(RO) Host channel adapter type
-
-		board_id:	(RO) Manufacturing board id
-
-		boardversion:	(RO) Current version of the chip architecture
-
-		nctxts:		(RO) Return the number of user ports (contexts)
-				available
-
-		localbus_info:	(RO) Human readable localbus info
-
-		tempsense:	(RO) Display temp sense registers in decimal
-
-		serial:		(RO) Serial number of the HCA
-
-		nfreectxts:	(RO) The number of free user ports (contexts)
-				available.
-
-		chip_reset:	(WO) Reset the chip if possible by writing
-				"reset" to this file. Only allowed if no user
-				contexts are open that use chip resources.
-		=============== ======================================================
-
-
-What:		/sys/class/infiniband/qibX/ports/<N>/sl2vl/[0-15]
-Date:		May, 2010
-KernelVersion:	v2.6.35
-Contact:	linux-rdma@vger.kernel.org
-Description:
-		(RO) The directory contains 16 files numbered 0-15 that specify
-		the Service Level (SL). Listing the SL files returns the Virtual
-		Lane (VL) as programmed by the SL.
-
-What:		/sys/class/infiniband/qibX/ports/<N>/CCMgtA/cc_settings_bin
-What:		/sys/class/infiniband/qibX/ports/<N>/CCMgtA/cc_table_bin
-Date:		May, 2010
-KernelVersion:	v2.6.35
-Contact:	linux-rdma@vger.kernel.org
-Description:
-		Per-port congestion control. Both are binary attributes.
-
-		=============== ================================================
-		cc_table_bin	(RO) Congestion control table size followed by
-				table entries.
-
-		cc_settings_bin (RO) Congestion settings: port control, control
-				map and an array of 16 entries for the
-				congestion entries - increase, timer, event log
-				trigger threshold and the minimum injection rate
-				delay.
-		=============== ================================================
-
-What:		/sys/class/infiniband/qibX/ports/<N>/linkstate/loopback
-What:		/sys/class/infiniband/qibX/ports/<N>/linkstate/led_override
-What:		/sys/class/infiniband/qibX/ports/<N>/linkstate/hrtbt_enable
-What:		/sys/class/infiniband/qibX/ports/<N>/linkstate/status
-What:		/sys/class/infiniband/qibX/ports/<N>/linkstate/status_str
-Date:		May, 2010
-KernelVersion:	v2.6.35
-Contact:	linux-rdma@vger.kernel.org
-Description:
-		[to be documented]
-
-		=============== ===============================================
-		loopback:	(WO)
-		led_override:	(WO)
-		hrtbt_enable:	(RW)
-		status:		(RO)
-
-		status_str:	(RO) Displays information about the link state,
-				possible cable/switch problems, and hardware
-				errors. Possible states are- "Initted",
-				"Present", "IB_link_up", "IB_configured" or
-				"Fatal_Hardware_Error".
-		=============== ===============================================
-
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/rc_resends
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/seq_naks
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/rdma_seq
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/rnr_naks
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/other_naks
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/rc_timeouts
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/look_pkts
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/pkt_drops
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/dma_wait
-What:		/sys/class/infiniband/qibX/ports/<N>/diag_counters/unaligned
-Date:		May, 2010
-KernelVersion:	v2.6.35
-Contact:	linux-rdma@vger.kernel.org
-Description:
-		[to be documented]
-
-
 sysfs interface for Mellanox Connect-IB HCA driver mlx5
 -------------------------------------------------------
 
diff --git a/Documentation/ABI/stable/sysfs-driver-firmware-zynqmp b/Documentation/ABI/stable/sysfs-driver-firmware-zynqmp
index c3fec3c835af36..214e9ed4564e55 100644
--- a/Documentation/ABI/stable/sysfs-driver-firmware-zynqmp
+++ b/Documentation/ABI/stable/sysfs-driver-firmware-zynqmp
@@ -141,32 +141,44 @@ Description:
 
 		Usage:
 
-		Select over temperature config ID to enable/disable feature
+		Select over temperature config ID to enable/disable feature::
+
 		# echo 1 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 
-		Check over temperature config ID is selected or not
+		Check over temperature config ID is selected or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
+
 		The expected result is 1.
 
-		Select over temperature config ID to configure OT limit
+		Select over temperature config ID to configure OT limit::
+
 		# echo 2 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 
-		Check over temperature config ID is selected or not
+		Check over temperature config ID is selected or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
+
 		The expected result is 2.
 
-		Select external watchdog config ID to enable/disable feature
+		Select external watchdog config ID to enable/disable feature::
+
 		# echo 3 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 
-		Check external watchdog config ID is selected or not
+		Check external watchdog config ID is selected or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
+
 		The expected result is 3.
 
-		Select external watchdog config ID to configure time interval
+		Select external watchdog config ID to configure time interval::
+
 		# echo 4 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 
-		Check external watchdog config ID is selected or not
+		Check external watchdog config ID is selected or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
+
 		The expected result is 4.
 
 Users:		Xilinx
@@ -205,52 +217,70 @@ Description:
 
 		Usage:
 
-		Enable over temperature feature
+		Enable over temperature feature::
+
 		# echo 1 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 		# echo 1 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
 
-		Check whether the over temperature feature is enabled or not
+		Check whether the over temperature feature is enabled or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
+
 		The expected result is 1.
 
-		Disable over temperature feature
+		Disable over temperature feature::
+
 		# echo 1 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 		# echo 0 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
 
-		Check whether the over temperature feature is disabled or not
+		Check whether the over temperature feature is disabled or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
+
 		The expected result is 0.
 
-		Configure over temperature limit to 50 Degree Celsius
+		Configure over temperature limit to 50 Degree Celsius::
+
 		# echo 2 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 		# echo 50 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
 
-		Check whether the over temperature limit is configured or not
+		Check whether the over temperature limit is configured or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
+
 		The expected result is 50.
 
-		Enable external watchdog feature
+		Enable external watchdog feature::
+
 		# echo 3 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 		# echo 1 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
 
-		Check whether the external watchdog feature is enabled or not
+		Check whether the external watchdog feature is enabled or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
+
 		The expected result is 1.
 
-		Disable external watchdog feature
+		Disable external watchdog feature::
+
 		# echo 3 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 		# echo 0 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
 
-		Check whether the external watchdog feature is disabled or not
+		Check whether the external watchdog feature is disabled or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
+
 		The expected result is 0.
 
-		Configure external watchdog timer interval to 500ms
+		Configure external watchdog timer interval to 500ms::
+
 		# echo 4 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_id
 		# echo 500 > /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
 
-		Check whether the external watchdog timer interval is configured or not
+		Check whether the external watchdog timer interval is configured or not::
+
 		# cat /sys/devices/platform/firmware\:zynqmp-firmware/feature_config_value
+
 		The expected result is 500.
 
 Users:		Xilinx
diff --git a/Documentation/ABI/testing/debugfs-iio-ad9910 b/Documentation/ABI/testing/debugfs-iio-ad9910
new file mode 100644
index 00000000000000..7083a20d5a40e5
--- /dev/null
+++ b/Documentation/ABI/testing/debugfs-iio-ad9910
@@ -0,0 +1,23 @@
+What:		/sys/kernel/debug/iio/iio:deviceX/frequency_source
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Read-only. Reports the channel label that is read from the
+		<dir>_<type>[Y]_label attribute, which corresponds to the on-chip
+		signal generator that is currently driving the output frequency.
+
+What:		/sys/kernel/debug/iio/iio:deviceX/phase_source
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Read-only. Reports the channel label that is read from the
+		<dir>_<type>[Y]_label attribute, which corresponds to the on-chip
+		signal generator that is currently driving the output phase.
+
+What:		/sys/kernel/debug/iio/iio:deviceX/amplitude_source
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Read-only. Reports the channel label that is read from the
+		<dir>_<type>[Y]_label attribute, which corresponds to the on-chip
+		signal generator that is currently driving the output amplitude.
diff --git a/Documentation/ABI/testing/debugfs-iio-backend b/Documentation/ABI/testing/debugfs-iio-backend
index 01ab94469432c8..2a2f2e5508c05e 100644
--- a/Documentation/ABI/testing/debugfs-iio-backend
+++ b/Documentation/ABI/testing/debugfs-iio-backend
@@ -8,13 +8,13 @@ What:		/sys/kernel/debug/iio/iio:deviceX/backendY/direct_reg_access
 KernelVersion:	6.11
 Contact:	linux-iio@vger.kernel.org
 Description:
-		Directly access the registers of backend Y. Typical usage is:
+		Directly access the registers of backend Y. Typical usage is::
 
-		Reading address 0x50
-		echo 0x50 > direct_reg_access
-		cat direct_reg_access
+		  # Reading address 0x50
+		  echo 0x50 > direct_reg_access
+		  cat direct_reg_access
 
-		Writing address 0x50
-		echo 0x50 0x3 > direct_reg_access
-		//readback address 0x50
-		cat direct_reg_access
+		  # Writing address 0x50
+		  echo 0x50 0x3 > direct_reg_access
+		  # readback address 0x50
+		  cat direct_reg_access
diff --git a/Documentation/ABI/testing/debugfs-tpmi b/Documentation/ABI/testing/debugfs-tpmi
index c493a1403d2f30..09212978fc411c 100644
--- a/Documentation/ABI/testing/debugfs-tpmi
+++ b/Documentation/ABI/testing/debugfs-tpmi
@@ -25,9 +25,11 @@ Description:
 Allows to write at any offset. It doesn't check for Read/Write access
 as hardware will not allow to write at read-only memory. This write is
 at offset multiples of 4. The format is instance,offset,contents.
-Example:
-echo 0,0x20,0xff > mem_write
-echo 1,64,64 > mem_write
+Example::
+
+  echo 0,0x20,0xff > mem_write
+  echo 1,64,64 > mem_write
+
 Users:		Debugging, any user space test suite
 
 What:		/sys/kernel/debug/tpmi-<n>/plr/domain<n>/status
diff --git a/Documentation/ABI/testing/ppc-memtrace b/Documentation/ABI/testing/ppc-memtrace
index 9606aed33137fc..2de9fd18715e03 100644
--- a/Documentation/ABI/testing/ppc-memtrace
+++ b/Documentation/ABI/testing/ppc-memtrace
@@ -3,7 +3,7 @@ Date:		Aug 2017
 KernelVersion:	4.14
 Contact:	linuxppc-dev@lists.ozlabs.org
 Description:	This folder contains the relevant debugfs files for the
-		hardware trace macro to use. CONFIG_PPC64_HARDWARE_TRACING
+		hardware trace macro to use. CONFIG_PPC_MEMTRACE
 		must be set.
 
 What:		/sys/kernel/debug/powerpc/memtrace/enable
diff --git a/Documentation/ABI/testing/se-cdev b/Documentation/ABI/testing/se-cdev
new file mode 100644
index 00000000000000..0a353caeaab2b9
--- /dev/null
+++ b/Documentation/ABI/testing/se-cdev
@@ -0,0 +1,45 @@
+What:		/dev/<se>_mu[0-9]+_ch[0-9]+
+Date:		Mar 2025
+KernelVersion:	6.8
+Contact:	linux-imx@nxp.com, pankaj.gupta@nxp.com
+Description:
+		NXP offers multiple hardware IP(s) for secure enclaves like EdgeLock-
+		Enclave(ELE), SECO. The character device file descriptors
+		/dev/<se>_mu*_ch* are the interface between userspace NXP's secure-
+		enclave shared library and the kernel driver.
+
+		The ioctl(2)-based ABI is defined and documented in
+		[include]<uapi/linux/se_ioctl.h>
+		ioctl(s) are used primarily for:
+
+			- shared memory management
+			- allocation of I/O buffers
+			- getting mu info
+			- setting a dev-ctx as receiver to receive all the commands from FW
+			- getting SoC info
+			- send command and receive command response
+
+		The following file operations are supported:
+
+		open(2)
+		  Currently the only useful flags are O_RDWR.
+
+		read(2)
+		  Every read() from the opened character device context is waiting on
+		  wait_for_completion_interruptible_timeout, that gets set by the
+		  registered mailbox callback function, indicating a message received
+		  from the firmware on message-unit.
+
+		write(2)
+		  Every write() to the opened character device context needs to acquire
+		  `fops_lock` + `se_if_cmd_lock` lock before sending message on to the
+		  message unit.
+
+		close(2)
+		  Stops and frees up the I/O contexts that were associated
+		  with the file descriptor.
+
+Users:		https://github.com/nxp-imx/imx-secure-enclave.git,
+		https://github.com/nxp-imx/imx-smw.git,
+		crypto/skcipher,
+		drivers/nvmem/imx-ocotp-ele.c
diff --git a/Documentation/ABI/testing/sysfs-auxdisplay-linedisp b/Documentation/ABI/testing/sysfs-auxdisplay-linedisp
index 55f1b559e84ebe..ac9e96448b9601 100644
--- a/Documentation/ABI/testing/sysfs-auxdisplay-linedisp
+++ b/Documentation/ABI/testing/sysfs-auxdisplay-linedisp
@@ -11,7 +11,8 @@ Description:
 
 		Writing an empty string clears the display.
 
-		Example:
+		Example::
+
 		  echo "Hello World" > message
 		  cat message			# Returns "Hello World\n"
 
@@ -23,7 +24,8 @@ Description:
 		Read-only attribute showing the character width capacity of
 		the line display device. Messages longer than this will scroll.
 
-		Example:
+		Example::
+
 		  cat num_chars		# Returns "16\n" for 16-char display
 
 What:		/sys/.../scroll_step_ms
@@ -35,7 +37,8 @@ Description:
 
 		Setting to 0 disables scrolling. Default is 500ms.
 
-		Example:
+		Example::
+
 		  echo "250" > scroll_step_ms	# 4Hz scrolling
 		  cat scroll_step_ms		# Returns "250\n"
 
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index a4f5595722ade7..2b6f57041763de 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -442,6 +442,14 @@ What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY_i_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY_q_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_currentY_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_current_offset
+What:		/sys/bus/iio/devices/iio:deviceX/in_phaseY_offset
+What:		/sys/bus/iio/devices/iio:deviceX/in_phase_offset
+What:		/sys/bus/iio/devices/iio:deviceX/out_phaseY_offset
+What:		/sys/bus/iio/devices/iio:deviceX/out_phase_offset
+What:		/sys/bus/iio/devices/iio:deviceX/in_frequencyY_offset
+What:		/sys/bus/iio/devices/iio:deviceX/in_frequency_offset
+What:		/sys/bus/iio/devices/iio:deviceX/out_frequencyY_offset
+What:		/sys/bus/iio/devices/iio:deviceX/out_frequency_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_tempY_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_temp_offset
 What:		/sys/bus/iio/devices/iio:deviceX/in_pressureY_offset
@@ -475,10 +483,22 @@ What:		/sys/bus/iio/devices/iio:deviceX/in_voltage_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_voltage-voltage_scale
 What:		/sys/bus/iio/devices/iio:deviceX/out_voltageY_scale
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_phaseY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_phase_scale
+What:		/sys/bus/iio/devices/iio:deviceX/out_phaseY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/out_phase_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_frequencyY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_frequency_scale
+What:		/sys/bus/iio/devices/iio:deviceX/out_frequencyY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/out_frequency_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_currentY_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_currentY_supply_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_current_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_current_q_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_altcurrentY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_altcurrent_scale
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_scale
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrent_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_accel_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_accel_peak_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_anglvel_scale
@@ -507,6 +527,7 @@ What:		/sys/bus/iio/devices/iio:deviceX/in_intensity_z_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_intensity_red_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_intensity_green_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_intensity_blue_scale
+What:		/sys/bus/iio/devices/iio:deviceX/in_intensity_clear_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_concentration_co2_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_volumeflow_scale
 What:		/sys/bus/iio/devices/iio:deviceX/in_volumeflowY_scale
@@ -762,6 +783,8 @@ What:		/sys/bus/iio/devices/iio:deviceX/out_voltageY_powerdown_mode
 What:		/sys/bus/iio/devices/iio:deviceX/out_voltage_powerdown_mode
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_powerdown_mode
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltage_powerdown_mode
+What:		/sys/bus/iio/devices/iio:deviceX/out_currentY_powerdown_mode
+What:		/sys/bus/iio/devices/iio:deviceX/out_current_powerdown_mode
 KernelVersion:	2.6.38
 Contact:	linux-iio@vger.kernel.org
 Description:
@@ -774,6 +797,7 @@ Description:
 		6kohm_to_gnd: connected to ground via a 6kOhm resistor,
 		7.7kohm_to_gnd: connected to ground via a 7.7kOhm resistor,
 		10kohm_to_gnd: connected to ground via a 10kOhm resistor,
+		15kohm_to_gnd: connected to ground via a 15kOhm resistor,
 		16kohm_to_gnd: connected to ground via a 16kOhm resistor,
 		20kohm_to_gnd: connected to ground via a 20kOhm resistor,
 		32kohm_to_gnd: connected to ground via a 32kOhm resistor,
@@ -802,6 +826,10 @@ What:		/sys/bus/iio/devices/iio:deviceX/out_voltageY_powerdown
 What:		/sys/bus/iio/devices/iio:deviceX/out_voltage_powerdown
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_powerdown
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltage_powerdown
+What:		/sys/bus/iio/devices/iio:deviceX/out_currentY_powerdown
+What:		/sys/bus/iio/devices/iio:deviceX/out_current_powerdown
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_powerdown
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrent_powerdown
 KernelVersion:	2.6.38
 Contact:	linux-iio@vger.kernel.org
 Description:
@@ -812,6 +840,7 @@ Description:
 		together.
 
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_frequency
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_frequency
 KernelVersion:	3.4.0
 Contact:	linux-iio@vger.kernel.org
 Description:
@@ -838,8 +867,10 @@ Description:
 		all the other channels, since it involves changing the VCO
 		fundamental output frequency.
 
+What:		/sys/bus/iio/devices/iio:deviceX/in_altcurrentY_phase
 What:		/sys/bus/iio/devices/iio:deviceX/in_altvoltageY_i_phase
 What:		/sys/bus/iio/devices/iio:deviceX/in_altvoltageY_q_phase
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_phase
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_phase
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_i_phase
 What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_q_phase
@@ -853,6 +884,7 @@ Description:
 		channel.
 
 What:		/sys/bus/iio/devices/iio:deviceX/out_currentY_raw
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_raw
 Date:		May 2012
 KernelVersion:	3.5
 Contact:	Johan Hovold <jhovold@gmail.com>
@@ -1739,6 +1771,7 @@ Description:
 
 What:		/sys/bus/iio/devices/iio:deviceX/in_currentY_raw
 What:		/sys/bus/iio/devices/iio:deviceX/in_currentY_supply_raw
+What:		/sys/bus/iio/devices/iio:deviceX/in_altcurrentY_raw
 KernelVersion:	3.17
 Contact:	linux-iio@vger.kernel.org
 Description:
@@ -2129,7 +2162,9 @@ Description:
 
 		Raw counter device counters direction for channel Y.
 
+What:		/sys/bus/iio/devices/iio:deviceX/in_altcurrentY_label
 What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY_label
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_label
 What:		/sys/bus/iio/devices/iio:deviceX/out_voltageY_label
 KernelVersion:	5.8
 Contact:	linux-iio@vger.kernel.org
@@ -2139,6 +2174,50 @@ Description:
 		specific attributes. This is useful for userspace to be able to
 		better identify an individual channel.
 
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY_parent
+What:		/sys/bus/iio/devices/iio:deviceX/out_voltageY_parent
+What:		/sys/bus/iio/devices/iio:deviceX/in_altvoltageY_parent
+What:		/sys/bus/iio/devices/iio:deviceX/out_altvoltageY_parent
+What:		/sys/bus/iio/devices/iio:deviceX/in_currentY_parent
+What:		/sys/bus/iio/devices/iio:deviceX/out_currentY_parent
+What:		/sys/bus/iio/devices/iio:deviceX/in_altcurrentY_parent
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_parent
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Read-only attribute containing the sysfs name prefix of the parent
+		channel for hierarchical channel relationships (e.g. "out_altvoltage0").
+		The prefix uniquely identifies the parent channel and can be used by
+		userspace to organize channels in tree-like structures that reflects
+		virtual or logical relationships between them.
+
+		For example, an IIO device may expose a primary output channel as
+		"out_voltage0" together with a set of profile-based or mode-based
+		subchannels used to configure or compose the output signal. These
+		subchannels may carry common attributes such as frequency, phase, etc.
+		They do not represent physical outputs, but rather logical/virtual
+		channels grouping configurations that affect the primary output.
+		Each subchannel exposes a <dir>_<type>[Y]_parent attribute reading
+		"out_voltage0", allowing userspace to group controls under the output
+		channel they belong to rather than treating them as unrelated top-level
+		channels.
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_frequencyY_raw
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Raw (unscaled) frequency reading from channel Y.
+		Units after application of scale and offset are Hz.
+
+What:		/sys/bus/iio/devices/iio:deviceX/out_frequencyY_raw
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Raw (unscaled, no bias etc.) output frequency for channel Y.
+		The number must always be specified and unique if the output
+		corresponds to a single channel. Units after application of
+		scale and offset are Hz.
+
 What:		/sys/bus/iio/devices/iio:deviceX/in_phaseY_raw
 KernelVersion:	4.18
 Contact:	linux-iio@vger.kernel.org
@@ -2146,6 +2225,15 @@ Description:
 		Raw (unscaled) phase difference reading from channel Y.
 		Units after application of scale and offset are radians.
 
+What:		/sys/bus/iio/devices/iio:deviceX/out_phaseY_raw
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Raw (unscaled, no bias etc.) output phase for channel Y.
+		The number must always be specified and unique if the output
+		corresponds to a single channel. Units after application of
+		scale and offset are radians.
+
 What:		/sys/bus/iio/devices/iio:deviceX/in_massconcentration_pm1_input
 What:		/sys/bus/iio/devices/iio:deviceX/in_massconcentrationY_pm1_input
 What:		/sys/bus/iio/devices/iio:deviceX/in_massconcentration_pm2p5_input
@@ -2380,6 +2468,7 @@ Description:
 		* "sinc4+lp" - Sinc4 + Low Pass Filter.
 		* "sinc4+sinc1" - Sinc4 + averaging by 8. Low 1st conversion
 		  time.
+		* "sinc4+sinc1+pf1" - Sinc4 + Sinc1 + device specific Post Filter 1.
 		* "sinc4+rej60" - Sinc4 + 60Hz rejection.
 		* "sinc5" - The digital sinc5 filter. Excellent noise
 		  performance
@@ -2401,6 +2490,32 @@ Description:
 		Specifies which filter type apply to the channel. The possible
 		values are given by the filter_type_available attribute.
 
+What:		/sys/bus/iio/devices/iio:deviceX/in_resistanceY_settlingtime
+What:		/sys/bus/iio/devices/iio:deviceX/in_tempY_settlingtime
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY_settlingtime
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY-voltageZ_settlingtime
+KernelVersion:	7.4
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Set/read total settling time in seconds for a device to settle
+		after starting the first conversion after any settings have
+		changed. The data for the conversion is outputted at the end of
+		this time.
+
+		Depending on the device and how it is used, this settling time
+		may apply to more than just the first conversion, e.g. to every
+		conversion in a single-shot conversion mode or when a device
+		setting changes after each conversion (input chopping).
+
+What:		/sys/bus/iio/devices/iio:deviceX/in_resistanceY_settlingtime_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_tempY_settlingtime_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY_settlingtime_available
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY-voltageZ_settlingtime_available
+KernelVersion:	7.4
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Available settlingtime values in IIO range or list format.
+
 What:		/sys/.../events/in_proximity_thresh_either_runningperiod
 KernelVersion:	6.6
 Contact:	linux-iio@vger.kernel.org
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-adc b/Documentation/ABI/testing/sysfs-bus-iio-adc
new file mode 100644
index 00000000000000..e1309b371efb81
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-adc
@@ -0,0 +1,9 @@
+What:		/sys/bus/iio/devices/iio:deviceX/in_resistanceY_burnoutraw
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY_burnoutraw
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltageY-voltageZ_burnoutraw
+KernelVersion:	7.4
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Raw value from channel Y read using a single conversion with
+		the channel burnout current enabled. This is typically used
+		for diagnostic purposes to detect an open or shorted input.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-frequency-ad9910 b/Documentation/ABI/testing/sysfs-bus-iio-frequency-ad9910
new file mode 100644
index 00000000000000..4e25f538af4add
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-frequency-ad9910
@@ -0,0 +1,31 @@
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_dwell_en
+What:		/sys/bus/iio/devices/iio:deviceX/out_frequencyY_dwell_en
+What:		/sys/bus/iio/devices/iio:deviceX/out_phaseY_dwell_en
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		For a channel that produces parametric sweeps, this attribute controls
+		the sweep behavior at the configured limit. It enables dwell mode at a
+		sweep limit when set to 1, i.e., after the sweep is complete the output
+		value stays at the limit. Otherwise (setting this value to 0), the sweep
+		may stop or restart from the initial position, or even continue by
+		reversing the sweep direction.
+
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_raw_roc
+What:		/sys/bus/iio/devices/iio:deviceX/out_frequencyY_raw_roc
+What:		/sys/bus/iio/devices/iio:deviceX/out_phaseY_raw_roc
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		For a channel that produces parametric sweeps, this attribute controls
+		the rate of change of the parameter in raw units per second (slope).
+		This value may be influenced by the channel sampling_frequency attribute
+		if available. Multiplying this value by the scale attribute of the
+		channel yields the rate of change in physical units per second.
+
+What:		/sys/bus/iio/devices/iio:deviceX/out_altcurrentY_raw_roc_available
+KernelVersion:	7.3
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Lists the available roc values for the channel. Values are
+		space-separated in ascending order.
diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
index 55ea1db749a180..be9126e5689400 100644
--- a/Documentation/ABI/testing/sysfs-bus-pci
+++ b/Documentation/ABI/testing/sysfs-bus-pci
@@ -244,10 +244,16 @@ Contact:	Narendra K <narendra_k@dell.com>, linux-bugs@dell.com
 Description:
 		Reading this attribute will provide the firmware
 		given name (SMBIOS type 41 string or ACPI _DSM string) of
-		the PCI device.	The attribute will be created only
-		if the firmware	has given a name to the PCI device.
-		ACPI _DSM string name will be given priority if the
-		system firmware provides SMBIOS type 41 string also.
+		the PCI device. The attribute will be created only if the
+		firmware naming mechanism is implemented for the device:
+		a SMBIOS type 41 record with a non-empty reference
+		designation, or a Device Name _DSM that returns the name
+		as a string or a buffer. The value read is empty when the
+		firmware implements the _DSM but gives the device no name,
+		which per PCI Firmware r3.3, sec 4.6.7 the firmware reports
+		as a NULL string. ACPI _DSM string name will be given
+		priority if the system firmware provides SMBIOS type 41
+		string also.
 Users:
 		Userspace applications interested in knowing the
 		firmware assigned name of the PCI device.
@@ -486,19 +492,19 @@ Description:
 		These files provide an interface to PCIe Resizable BAR support.
 		A file is created for each BAR resource (N) supported by the
 		PCIe Resizable BAR extended capability of the device.  Reading
-		each file exposes the bitmap of available resource sizes:
+		each file exposes the bitmap of available resource sizes::
 
-		# cat resource1_resize
-		00000000000001c0
+		  # cat resource1_resize
+		  00000000000001c0
 
 		The bitmap represents supported resource sizes for the BAR,
 		where bit0 = 1MB, bit1 = 2MB, bit2 = 4MB, etc.  In the above
 		example the device supports 64MB, 128MB, and 256MB BAR sizes.
 
 		When writing the file, the user provides the bit position of
-		the desired resource size, for example:
+		the desired resource size, for example::
 
-		# echo 7 > resource1_resize
+		  # echo 7 > resource1_resize
 
 		This indicates to set the size value corresponding to bit 7,
 		128MB.  The resulting size is 2 ^ (bit# + 20).  This definition
@@ -600,7 +606,7 @@ Description:
 
 		As all DOE devices must support the DOE discovery feature,
 		if DOE is supported you will at least see the doe_discovery
-		file, with this contents:
+		file, with this contents::
 
 		  # cat doe_features/doe_discovery
 		  0001:00
@@ -608,7 +614,7 @@ Description:
 		If the device supports other features you will see other
 		files as well. For example if CMA/SPDM and secure CMA/SPDM
 		are supported the doe_features directory will look like
-		this:
+		this::
 
 		  # ls doe_features
 		  0001:01        0001:02        doe_discovery
diff --git a/Documentation/ABI/testing/sysfs-bus-usb b/Documentation/ABI/testing/sysfs-bus-usb
index baf4bf6d32ee37..5213117fd957e6 100644
--- a/Documentation/ABI/testing/sysfs-bus-usb
+++ b/Documentation/ABI/testing/sysfs-bus-usb
@@ -83,7 +83,8 @@ Description:
 		idVendor idProduct.	After successfully
 		removing an ID, the driver will no longer support the
 		device.  This is useful to ensure auto probing won't
-		match the driver to the device.  For example:
+		match the driver to the device.  For example::
+
 		# echo "046d c315" > /sys/bus/usb/drivers/foo/remove_id
 
 		Reading from this file will list the dynamically added
diff --git a/Documentation/ABI/testing/sysfs-class-firmware-attributes b/Documentation/ABI/testing/sysfs-class-firmware-attributes
index 2713efa509b465..40e35fe97a7d82 100644
--- a/Documentation/ABI/testing/sysfs-class-firmware-attributes
+++ b/Documentation/ABI/testing/sysfs-class-firmware-attributes
@@ -397,22 +397,25 @@ Description:
 		unlimited attributes.
 
 		Read the attribute to check what save mode is enabled (single or bulk).
-		E.g:
-		# cat /sys/class/firmware-attributes/thinklmi/attributes/save_settings
-		single
+		E.g::
+
+		  # cat /sys/class/firmware-attributes/thinklmi/attributes/save_settings
+		  single
 
 		Write the attribute with 'bulk' to enable bulk save mode.
-		Write the attribute with 'single' to enable saving, after every attribute set.
+		Write the attribute with 'single' to enable saving after every attribute set.
 		The default setting is single mode.
-		E.g:
-		# echo bulk > /sys/class/firmware-attributes/thinklmi/attributes/save_settings
+		E.g::
+
+		  # echo bulk > /sys/class/firmware-attributes/thinklmi/attributes/save_settings
 
 		When in bulk mode write 'save' to trigger a save of all currently modified attributes.
-		Note, once a save has been triggered, in bulk mode, attributes can no longer be set and
+		Note, once a save has been triggered in bulk mode, attributes can no longer be set and
 		will return a permissions error. This is to prevent users hitting the 48+ save limitation
 		(which requires entering the BIOS to clear the error condition)
-		E.g:
-		# echo save > /sys/class/firmware-attributes/thinklmi/attributes/save_settings
+		E.g::
+
+		  # echo save > /sys/class/firmware-attributes/thinklmi/attributes/save_settings
 
 What:		/sys/class/firmware-attributes/*/attributes/debug_cmd
 Date:		July 2021
diff --git a/Documentation/ABI/testing/sysfs-class-hwmon b/Documentation/ABI/testing/sysfs-class-hwmon
index b185bdfc7186a7..34df6d7bd5f85e 100644
--- a/Documentation/ABI/testing/sysfs-class-hwmon
+++ b/Documentation/ABI/testing/sysfs-class-hwmon
@@ -61,6 +61,18 @@ Description:
 		take drastic action such as power down or reset. At the very
 		least, it should report a fault.
 
+What:		/sys/class/hwmon/hwmonX/inY_lemergency
+Description:
+		Voltage emergency min value.
+
+		Unit: millivolt
+
+		RW
+
+		If voltage drops to or below this limit, the system is
+		expected to take drastic action such as immediate power
+		down or reset. At the very least, it should report a fault.
+
 What:		/sys/class/hwmon/hwmonX/inY_max
 Description:
 		Voltage max value.
@@ -81,6 +93,18 @@ Description:
 		take drastic action such as power down or reset. At the very
 		least, it should report a fault.
 
+What:		/sys/class/hwmon/hwmonX/inY_emergency
+Description:
+		Voltage emergency max value.
+
+		Unit: millivolt
+
+		RW
+
+		If voltage reaches or exceeds this limit, the system is expected
+		to take drastic action such as immediate power down or reset.
+		At the very least, it should report a fault.
+
 What:		/sys/class/hwmon/hwmonX/inY_input
 Description:
 		Voltage input value.
@@ -647,6 +671,18 @@ Description:
 
 		RW
 
+What:		/sys/class/hwmon/hwmonX/currY_emergency
+Description:
+		Current emergency high value.
+
+		Unit: milliampere
+
+		RW
+
+		If a current reaches or exceeds this limit, the system is
+		expected to take drastic action such as immediate power down
+		or reset. At the very least, it should report a fault.
+
 What:		/sys/class/hwmon/hwmonX/currY_input
 Description:
 		Current input value
diff --git a/Documentation/ABI/testing/sysfs-class-led b/Documentation/ABI/testing/sysfs-class-led
index d4c918cc11a127..ea113fed10efaf 100644
--- a/Documentation/ABI/testing/sysfs-class-led
+++ b/Documentation/ABI/testing/sysfs-class-led
@@ -28,7 +28,9 @@ Description:
 		Writing 0 to this file clears active trigger.
 
 		Writing non-zero to this file while trigger is active changes the
-		top brightness trigger is going to use.
+		top brightness trigger is going to use, unless the current
+		trigger is offloaded to hardware (in which case writing any
+		value will deactivate the trigger).
 
 
 
@@ -78,6 +80,30 @@ Description:
 		(which would often be configured in the device tree for the
 		hardware).
 
+What:		/sys/class/leds/<led>/trigger_may_offload_to_hw
+Date:		September 2026
+KernelVersion:	7.4
+Contact:	linux-leds@vger.kernel.org
+Description:
+		Names and states of triggers that may be offloaded to hardware.
+		Such triggers are also called "hardware control trigger" in some
+		context.
+
+		Only exists when the LED supports trigger offload.
+
+		Reading this file returns a list of triggers that are capable to
+		be offloaded. The optional brackets around the trigger name
+		indicate the state of the current trigger:
+
+		- `foo_trigger`: the trigger is not selected, or selected but
+		  falls back to software blink for some reasons, e.g.,
+		  incompatible trigger parameters.
+		- `[foo_trigger]`: the trigger is selected and offloaded to
+		  hardware.
+
+		The "netdev" trigger also provides a custom attribute to
+		indicate its state, see `/sys/class/leds/<led>/offloaded`.
+
 What:		/sys/class/leds/<led>/inverted
 Date:		January 2011
 KernelVersion:	2.6.38
diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-netdev b/Documentation/ABI/testing/sysfs-class-led-trigger-netdev
index ed46b37ab8a284..203ea58396ed50 100644
--- a/Documentation/ABI/testing/sysfs-class-led-trigger-netdev
+++ b/Documentation/ABI/testing/sysfs-class-led-trigger-netdev
@@ -75,6 +75,9 @@ Description:
 		If 1, the LED blinking in requested mode is offloaded to
 		hardware.
 
+		LED trigger core also provides a generic attribute for this
+		purpose, see `/sys/class/leds/<led>/trigger_may_offload_to_hw`.
+
 What:		/sys/class/leds/<led>/link_10
 Date:		Jun 2023
 KernelVersion:	6.5
diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power
index 5641f1fd5fd6f0..2cf3f53a652b89 100644
--- a/Documentation/ABI/testing/sysfs-class-power
+++ b/Documentation/ABI/testing/sysfs-class-power
@@ -365,9 +365,12 @@ Contact:	linux-pm@vger.kernel.org
 Description:
 		Represents a battery percentage level, above which charging will
 		stop. Not all hardware is capable of setting this to an arbitrary
-		percentage. Drivers will round written values to the nearest
-		supported value. Reading back the value will show the actual
-		threshold set by the driver.
+		value, instead providing different minimum, maximum, or step
+		values. Drivers will round written values to the nearest supported
+		value. Reading back the value will show the actual threshold set
+		by the driver. For hardware that only supports a single fixed
+		value, use charge_types with a value of "Long Life" (vs "Standard")
+		instead'
 
 		Access: Read, Write
 
@@ -398,8 +401,9 @@ Description:
 			when to start and stop charging. Advanced users
 			can use this to drastically extend battery life.
 		Long Life:
-			The charger reduces its charging rate in order to
-			prolong the battery health.
+			The charger firmware reduces its charging rate and/or
+			maximum charging percentage to a hardware specified
+			fixed limit in order to prolong the battery health.
 		Bypass:
 			The charger bypasses the charging path around the
 			integrated converter allowing for a "smart" wall
@@ -590,6 +594,31 @@ Description:
 
 		Valid values: 0 - 100 (percent)
 
+What:		/sys/class/power_supply/<supply_name>/load_switch
+Date:		June 2026
+Contact:	linux-pm@vger.kernel.org
+Description:
+		Devices can control the connection from power to system load.
+		For example, with a field-effect transistor between a battery
+		and the system load (BATFET). This entry controls such switch.
+		The obvious states are "on" and "off", i.e. there is a
+		connection or not, respectively. However, many devices can also
+		enter special modes such as "low-power", "shipping" or "deep
+		sleep". In these modes the switch is usually off and the
+		quiescent current quite low.
+
+		Access: Read, Write
+
+		Valid values:
+
+		============= ==================================
+		"Unknown" (0) Unknown mode.
+		"On"      (1) Power is connected to the load.
+		"Off"     (2) Power is disconnected to the load.
+		"Standby" (3) Low-power mode.
+		"Ship"    (4) Ship mode.
+		============= ==================================
+
 **USB Properties**
 
 What:		/sys/class/power_supply/<supply_name>/input_current_limit
diff --git a/Documentation/ABI/testing/sysfs-class-power-ltc4162l b/Documentation/ABI/testing/sysfs-class-power-ltc4162l
index ba30db93052bf0..ad46ad834af059 100644
--- a/Documentation/ABI/testing/sysfs-class-power-ltc4162l
+++ b/Documentation/ABI/testing/sysfs-class-power-ltc4162l
@@ -77,6 +77,8 @@ Description:
 		The ship mode, when armed, activates once the input voltage
 		drops below 1V.
 
+		Note: use /sys/class/power_supply/ltc4162-l/load_switch instead.
+
 		Access: Read, Write
 
 		Valid values: 0 (disable) or 1 (enable)
diff --git a/Documentation/ABI/testing/sysfs-class-power-mp2629 b/Documentation/ABI/testing/sysfs-class-power-mp2629
index 914d67caac0d79..a85f926ce9c269 100644
--- a/Documentation/ABI/testing/sysfs-class-power-mp2629
+++ b/Documentation/ABI/testing/sysfs-class-power-mp2629
@@ -1,4 +1,4 @@
-What:		/sys/class/power_supply/mp2629_battery/batt_impedance_compen
+What:		/sys/class/power_supply/mp2629_battery/batt_impedance_compensation
 Date:		April 2020
 KernelVersion:	5.7
 Description:
diff --git a/Documentation/ABI/testing/sysfs-class-power-rt9467 b/Documentation/ABI/testing/sysfs-class-power-rt9467
index 619b7c45d145df..bfbfd78ec1c190 100644
--- a/Documentation/ABI/testing/sysfs-class-power-rt9467
+++ b/Documentation/ABI/testing/sysfs-class-power-rt9467
@@ -13,6 +13,8 @@ Description:
 		'Disable' also can help to leave it, but it's more like to
 		abort the action before the device really enter shipping mode.
 
+		Note: use /sys/class/power_supply/rt9467-*/load_switch instead.
+
 		Access: Read, Write
 		Valid values:
 		- 1: enabled
diff --git a/Documentation/ABI/testing/sysfs-class-power-rt9471 b/Documentation/ABI/testing/sysfs-class-power-rt9471
index 0a390ee5ac21c3..90cf94c1cbf6b6 100644
--- a/Documentation/ABI/testing/sysfs-class-power-rt9471
+++ b/Documentation/ABI/testing/sysfs-class-power-rt9471
@@ -11,6 +11,8 @@ Description:
 		mode. 'Disable' also can help to leave it, but it's more like to abort
 		the action before the device really enter shipping mode.
 
+		Note: use /sys/class/power_supply/rt9471-*/load_switch instead.
+
 		Access: Read, Write
 		Valid values:
 		- 1: enabled
diff --git a/Documentation/ABI/testing/sysfs-class-spi-master b/Documentation/ABI/testing/sysfs-class-spi-master
index 0a524fcd96d255..4e1e8a7ca83c6f 100644
--- a/Documentation/ABI/testing/sysfs-class-spi-master
+++ b/Documentation/ABI/testing/sysfs-class-spi-master
@@ -17,7 +17,8 @@ Description:	(WO) Instantiate a new SPI device on bus B, where B
 		with delete_device; platform and DT devices are not
 		affected.
 
-		Example:
+		Example::
+
 		# echo spidev 0 > /sys/class/spi_master/spi0/new_device
 		# echo spidev 0 10000000 > /sys/class/spi_master/spi0/new_device
 		# echo spidev 0 10000000 3 > /sys/class/spi_master/spi0/new_device
@@ -30,5 +31,6 @@ Description:	(WO) Remove a SPI device previously created via
 		new_device. Takes a single parameter: the chip select
 		number of the device to remove.
 
-		Example:
+		Example::
+
 		# echo 0 > /sys/class/spi_master/spi0/delete_device
diff --git a/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa b/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa
index 364b1ba412427e..3b3bae2369d492 100644
--- a/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa
+++ b/Documentation/ABI/testing/sysfs-devices-platform-soc-ipa
@@ -75,20 +75,6 @@ Description:
 		the ID of the AP endpoint on which packets destined
 		for the embedded modem are sent.
 
-What:		.../XXXXXXX.ipa/endpoint_id/monitor_rx
-Date:		July 2022
-KernelVersion:	v5.19
-Contact:	Alex Elder <elder@kernel.org>
-Description:
-		The .../XXXXXXX.ipa/endpoint_id/monitor_rx file contains
-		the ID of the AP endpoint on which IPA "monitor" data is
-		received.  The monitor endpoint supplies replicas of
-		packets that enter the IPA hardware for processing.
-		Each replicated packet is preceded by a fixed-size "ODL"
-		header (see .../XXXXXXX.ipa/feature/monitor, above).
-		Large packets are truncated, to reduce the bandwidth
-		required to provide the monitor function.
-
 What:		.../XXXXXXX.ipa/modem/
 Date:		June 2021
 KernelVersion:	v5.14
diff --git a/Documentation/ABI/testing/sysfs-devices-platform-trackpoint b/Documentation/ABI/testing/sysfs-devices-platform-trackpoint
index df11901a6b3df5..7954434b0434ac 100644
--- a/Documentation/ABI/testing/sysfs-devices-platform-trackpoint
+++ b/Documentation/ABI/testing/sysfs-devices-platform-trackpoint
@@ -5,7 +5,7 @@ Contact:	linux-input@vger.kernel.org
 Description:
 		(RW) Trackpoint sensitivity.
 
-What:		/sys/devices/platform/i8042/.../intertia
+What:		/sys/devices/platform/i8042/.../inertia
 Date:		Aug, 2005
 KernelVersion:	2.6.14
 Contact:	linux-input@vger.kernel.org
diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu
index 82d10d556cc89f..545687ebadccc7 100644
--- a/Documentation/ABI/testing/sysfs-devices-system-cpu
+++ b/Documentation/ABI/testing/sysfs-devices-system-cpu
@@ -335,12 +335,16 @@ Description:	Performance Limited
 		Read to check if platform throttling (thermal/power/current
 		limits) caused delivered performance to fall below the
 		requested level. A non-zero value indicates throttling occurred.
+		If firmware provides the register but Linux cannot access it
+		safely, reads return the literal string "<unsupported>".
 
-		Write the bitmask of bits to clear:
-
-		  - 0x1 = clear bit 0 (desired performance excursion)
-		  - 0x2 = clear bit 1 (minimum performance excursion)
-		  - 0x3 = clear both bits
+		Write 0x3 to clear both bits. Selectively clearing one bit is
+		not supported because ACPI does not define the effect of writing
+		one to these status bits and an interlocked read-modify-write is
+		not generally available.
+		Writing 0x3 returns -EOPNOTSUPP when firmware describes the
+		register using an access which Linux cannot clear safely; reads
+		may remain available in that case.
 
 		The platform sets these bits; OSPM can only clear them.
 
@@ -689,15 +693,21 @@ Description:	Umwait control
 			  Low order two bits must be zero.
 
 What:		/sys/devices/system/cpu/sev
+		/sys/devices/system/cpu/sev/sev_status
 		/sys/devices/system/cpu/sev/vmpl
 Date:		May 2024
 Contact:	Linux kernel mailing list <linux-kernel@vger.kernel.org>
 Description:	Secure Encrypted Virtualization (SEV) information
 
-		This directory is only present when running as an SEV-SNP guest.
+		This directory is only present when running as an SEV guest.
+
+		sev_status: Reports the value of the SEV_STATUS MSR which
+			    enumerates the enabled features of an SEV
+			    environment.
 
 		vmpl: Reports the Virtual Machine Privilege Level (VMPL) at which
-		      the SEV-SNP guest is running.
+		      the SEV-SNP guest is running. This file is only present
+		      when running as an SEV-SNP guest.
 
 
 What:		/sys/devices/system/cpu/svm
@@ -806,3 +816,17 @@ Date:		Nov 2022
 Contact:	Linux kernel mailing list <linux-kernel@vger.kernel.org>
 Description:
 		(RO) the list of CPUs that can be brought online.
+
+What:		/sys/devices/system/cpu/preferred
+Date:		Sep 2026
+Contact:	Linux kernel mailing list <linux-kernel@vger.kernel.org>
+Description:
+		(RO) the list of preferred CPUs applicable in
+		paravirtualized environments.
+
+		The steal governor driver dynamically adjusts this mask
+		based on observed steal time. Scheduling tasks on
+		CPUs outside of this list may lead to performance
+		degradations due to underlying physical CPU contention.
+
+		See Documentation/scheduler/sched-paravirt.rst for more details.
diff --git a/Documentation/ABI/testing/sysfs-driver-aspeed-uart-routing b/Documentation/ABI/testing/sysfs-driver-aspeed-uart-routing
index 910df0e5815abf..b0b713c855175f 100644
--- a/Documentation/ABI/testing/sysfs-driver-aspeed-uart-routing
+++ b/Documentation/ABI/testing/sysfs-driver-aspeed-uart-routing
@@ -8,9 +8,10 @@ Description:	Selects the RX source of the UARTx device.
 		selected option marked by brackets "[]". The list of available options
 		depends on the selected file.
 
-		e.g.
-		cat /sys/bus/platform/drivers/aspeed-uart-routing/\*.uart_routing/uart1
-		[io1] io2 io3 io4 uart2 uart3 uart4 io6
+		E.g.::
+
+		  cat /sys/bus/platform/drivers/aspeed-uart-routing/\*.uart_routing/uart1
+		  [io1] io2 io3 io4 uart2 uart3 uart4 io6
 
 		In this case, UART1 gets its input from IO1 (physical serial port 1).
 
diff --git a/Documentation/ABI/testing/sysfs-driver-intel-xe-gpu b/Documentation/ABI/testing/sysfs-driver-intel-xe-gpu
new file mode 100644
index 00000000000000..4dc4afc3961d9d
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-intel-xe-gpu
@@ -0,0 +1,10 @@
+What:		/sys/bus/pci/drivers/xe/.../device_uid
+Date:		October 2026
+KernelVersion:	7.4
+Contact:	intel-xe@lists.freedesktop.org
+Description:
+		RO. Unique 64-bit identifier of the GPU SoC device, exposed as
+		hexadecimal value.
+
+		This sysfs file is present only on supported Intel Xe platforms.
+		Accessible only to users with administrative privileges.
diff --git a/Documentation/ABI/testing/sysfs-driver-qat_kpt b/Documentation/ABI/testing/sysfs-driver-qat_kpt
index c6480ea1fa4f8b..80831ce3d2ca9d 100644
--- a/Documentation/ABI/testing/sysfs-driver-qat_kpt
+++ b/Documentation/ABI/testing/sysfs-driver-qat_kpt
@@ -14,7 +14,8 @@ Contact:	qat-linux@intel.com
 Description:
 		(RW) Enables or disables Key Protection Technology (KPT).
 
-		Write 1 to enable KPT, or 0 to disable it.
+		Write 1 to enable KPT, or 0 to disable it. The device must be
+		in the down state before changing the KPT state.
 
 		Example usage::
 
diff --git a/Documentation/ABI/testing/sysfs-driver-xdata b/Documentation/ABI/testing/sysfs-driver-xdata
index f574e8e6dca21d..5bcdcba3a5c95f 100644
--- a/Documentation/ABI/testing/sysfs-driver-xdata
+++ b/Documentation/ABI/testing/sysfs-driver-xdata
@@ -9,15 +9,19 @@ Description:	Allows the user to enable the PCIe traffic generator which
 
 		Write y/1/on to enable, n/0/off to disable
 
-		Usage e.g.
+		Usage e.g.::
+
 		 echo 1 > /sys/class/misc/dw-xdata-pcie.<device>/write
-		or
+
+		or::
+
 		 echo 0 > /sys/class/misc/dw-xdata-pcie.<device>/write
 
 		The user can read the current PCIe link throughput generated
 		through this generator in MB/s.
 
-		Usage e.g.
+		Usage e.g.::
+
 		 cat /sys/class/misc/dw-xdata-pcie.<device>/write
 		 204
 
@@ -34,15 +38,19 @@ Description:	Allows the user to enable the PCIe traffic generator which
 
 		Write y/1/on to enable, n/0/off to disable
 
-		Usage e.g.
+		Usage e.g.::
+
 		 echo 1 > /sys/class/misc/dw-xdata-pcie.<device>/read
-		or
+
+		or::
+
 		 echo 0 > /sys/class/misc/dw-xdata-pcie.<device>/read
 
 		The user can read the current PCIe link throughput generated
 		through this generator in MB/s.
 
-		Usage e.g.
+		Usage e.g.::
+
 		 cat /sys/class/misc/dw-xdata-pcie.<device>/read
 		 199
 
diff --git a/Documentation/ABI/testing/sysfs-driver-xilinx-tmr-manager b/Documentation/ABI/testing/sysfs-driver-xilinx-tmr-manager
index 57b9b68a73ee9a..8d07c982d125f9 100644
--- a/Documentation/ABI/testing/sysfs-driver-xilinx-tmr-manager
+++ b/Documentation/ABI/testing/sysfs-driver-xilinx-tmr-manager
@@ -3,14 +3,16 @@ Date:          Nov 2022
 Contact:       appana.durga.kedareswara.rao@amd.com
 Description:   This control file provides the fault detection count.
                This file cannot be written.
-               Example:
-               # cat /sys/devices/platform/amba_pl/44a10000.tmr_manager/errcnt
-               1
+               Example::
+
+                 # cat /sys/devices/platform/amba_pl/44a10000.tmr_manager/errcnt
+                 1
 
 What:          /sys/devices/platform/amba_pl/<dev>/dis_block_break
 Date:          Nov 2022
 Contact:       appana.durga.kedareswara.rao@amd.com
-Description:   Write any value to it, This control file enables the break signal.
+Description:   Write any value to it. This control file enables the break signal.
                This file is write only.
-               Example:
-               # echo <any value> > /sys/devices/platform/amba_pl/44a10000.tmr_manager/dis_block_break
+               Example::
+
+                 # echo <any value> > /sys/devices/platform/amba_pl/44a10000.tmr_manager/dis_block_break
diff --git a/Documentation/ABI/testing/sysfs-firmware-cca b/Documentation/ABI/testing/sysfs-firmware-cca
new file mode 100644
index 00000000000000..d545b752353049
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-firmware-cca
@@ -0,0 +1,7 @@
+What:		/sys/firmware/cca/realm_guest
+Date:		September 2026
+Contact:	Linux ARM Kernel Mailing list <linux-arm-kernel@lists.infradead.org>
+Description:	Read-only. Present only when the kernel is running as an
+		Arm Confidential Compute Architecture (CCA) Realm guest, in
+		which case it reads 1. If the file does not exist, the kernel
+		is not running as a Realm guest.
diff --git a/Documentation/ABI/testing/sysfs-firmware-lefi-boardinfo b/Documentation/ABI/testing/sysfs-firmware-lefi-boardinfo
index 5e3f6148c52e18..abea10ace557e0 100644
--- a/Documentation/ABI/testing/sysfs-firmware-lefi-boardinfo
+++ b/Documentation/ABI/testing/sysfs-firmware-lefi-boardinfo
@@ -12,19 +12,18 @@ Description:
 		requirement and specific for Loongson64, so only add a new
 		boardinfo.c file in arch/mips/loongson64.
 
-		For example:
+		For example::
 
-		[loongson@linux ~]$ cat /sys/firmware/lefi/boardinfo
-		Board Info
-		Manufacturer            : LEMOTE
-		Board Name              : LEMOTE-LS3A4000-7A1000-1w-V01-pc
-		Family                  : LOONGSON3
+		 [loongson@linux ~]$ cat /sys/firmware/lefi/boardinfo
+		 Board Info
+		 Manufacturer            : LEMOTE
+		 Board Name              : LEMOTE-LS3A4000-7A1000-1w-V01-pc
+		 Family                  : LOONGSON3
 
-		BIOS Info
-		Vendor                  : Kunlun
-		Version                 : Kunlun-A1901-V4.1.3-20200414093938
-		ROM Size                : 4 KB
-		Release Date            : 2020-04-14
+		 BIOS Info
+		 Vendor                  : Kunlun
+		 Version                 : Kunlun-A1901-V4.1.3-20200414093938
+		 Release Date            : 2020-04-14
 
 		By the way, using dmidecode command can get the similar info if there
 		exists SMBIOS in firmware, but the fact is that there is no SMBIOS on
diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs
index 0cebc89799dd17..c4746c416ac2bb 100644
--- a/Documentation/ABI/testing/sysfs-fs-f2fs
+++ b/Documentation/ABI/testing/sysfs-fs-f2fs
@@ -1020,3 +1020,10 @@ Contact:	"Daeho Jeong" <daehojeong@google.com>
 Description:	This is a read-only entry to show the upper bound section number
 		for pinned files. Pinned files will only be allocated within
 		sections 0 to pinned_area_max_secno - 1.
+
+What:		/sys/fs/f2fs/<disk>/cache_wb_interval
+Date:		August 2026
+Contact:	"Chao Yu" <chao@kernel.org>
+Description:	This is a writable entry to control writeback interval of
+		f2fs_writeback-x:y, the range is [100, 30000], by default the value
+		is 5000, unit is ms.
diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon
index e675a57145e36d..55df688ea596ff 100644
--- a/Documentation/ABI/testing/sysfs-kernel-mm-damon
+++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon
@@ -3,7 +3,7 @@ Date:		Mar 2022
 Contact:	SJ Park <sj@kernel.org>
 Description:	Interface for Data Access MONitoring (DAMON).  Contains files
 		for controlling DAMON.  For more details on DAMON itself,
-		please refer to Documentation/admin-guide/mm/damon/index.rst.
+		please refer to Documentation/mm/damon/index.rst.
 
 What:		/sys/kernel/mm/damon/admin/
 Date:		Mar 2022
@@ -173,6 +173,19 @@ Contact:	SJ Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		per-probe attribute weight.
 
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/preps/nr_preps
+Date:		Jun 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing a number 'N' to this file creates the number of
+		directories for each DAMON probing preparation action named '0'
+		to 'N-1' under the preps/ directory.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/preps/<R>/prep_action
+Date:		Jun 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	Writing to and reading from this file sets and gets the probing
+		preparation action.
+
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/nr_filters
 Date:		May 2026
 Contact:	SJ Park <sj@kernel.org>
@@ -193,6 +206,20 @@ Description:	If 'memcg' is written to the 'type' file, writing to and
 		reading from this file sets and gets the path to the memory
 		cgroup of the interest.
 
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/min
+Date:		Sep 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	If 'hugepage_size' is written to the 'type' file, writing to and
+		reading from this file sets and gets the minimum size of the
+		huge page of the interest.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/max
+Date:		Sep 2026
+Contact:	SJ Park <sj@kernel.org>
+Description:	If 'hugepage_size' is written to the 'type' file, writing to and
+		reading from this file sets and gets the maximum size of the
+		huge page of the interest.
+
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/probes/<P>/filters/<F>/matching
 Date:		May 2026
 Contact:	SJ Park <sj@kernel.org>
diff --git a/Documentation/ABI/testing/sysfs-platform-intel-ifs b/Documentation/ABI/testing/sysfs-platform-intel-ifs
index 41b4d5b1e21c6d..11cd2f412d8172 100644
--- a/Documentation/ABI/testing/sysfs-platform-intel-ifs
+++ b/Documentation/ABI/testing/sysfs-platform-intel-ifs
@@ -10,8 +10,10 @@ Description:	Write <cpu#> to trigger IFS test for one online core.
 		Note that the test is per core. The cpu# can be
 		for any thread on the core. Running on one thread
 		completes the test for the core containing that thread.
-		Example: to test the core containing cpu5: echo 5 >
-		/sys/devices/virtual/misc/intel_ifs_<N>/run_test
+		Example: to test the core containing cpu5::
+
+		  echo 5 > /sys/devices/virtual/misc/intel_ifs_<N>/run_test
+
 Devices:	all
 
 What:		/sys/devices/virtual/misc/intel_ifs_<N>/status
diff --git a/Documentation/ABI/testing/sysfs-platform-ts5500 b/Documentation/ABI/testing/sysfs-platform-ts5500
deleted file mode 100644
index e685957caa1236..00000000000000
--- a/Documentation/ABI/testing/sysfs-platform-ts5500
+++ /dev/null
@@ -1,54 +0,0 @@
-What:		/sys/devices/platform/ts5500/adc
-Date:		January 2013
-KernelVersion:	3.7
-Contact:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-Description:
-		Indicates the presence of an A/D Converter. If it is present,
-		it will display "1", otherwise "0".
-
-What:		/sys/devices/platform/ts5500/ereset
-Date:		January 2013
-KernelVersion:	3.7
-Contact:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-Description:
-		Indicates the presence of an external reset. If it is present,
-		it will display "1", otherwise "0".
-
-What:		/sys/devices/platform/ts5500/id
-Date:		January 2013
-KernelVersion:	3.7
-Contact:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-Description:
-		Product ID of the TS board. TS-5500 ID is 0x60.
-
-What:		/sys/devices/platform/ts5500/jumpers
-Date:		January 2013
-KernelVersion:	3.7
-Contact:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-Description:
-		Bitfield showing the jumpers' state. If a jumper is present,
-		the corresponding bit is set. For instance, 0x0e means jumpers
-		2, 3 and 4 are set.
-
-What:		/sys/devices/platform/ts5500/name
-Date:		July 2014
-KernelVersion:	3.16
-Contact:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-Description:
-		Model name of the TS board, e.g. "TS-5500".
-
-What:		/sys/devices/platform/ts5500/rs485
-Date:		January 2013
-KernelVersion:	3.7
-Contact:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-Description:
-		Indicates the presence of the RS485 option. If it is present,
-		it will display "1", otherwise "0".
-
-What:		/sys/devices/platform/ts5500/sram
-Date:		January 2013
-KernelVersion:	3.7
-Contact:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-Description:
-		Indicates the presence of the SRAM option. If it is present,
-		it will display "1", otherwise "0".
diff --git a/Documentation/ABI/testing/sysfs-timecard b/Documentation/ABI/testing/sysfs-timecard
index 3ae41b7634ac9e..a0b7172851d4e2 100644
--- a/Documentation/ABI/testing/sysfs-timecard
+++ b/Documentation/ABI/testing/sysfs-timecard
@@ -11,6 +11,35 @@ Contact:	Jonathan Lemon <jonathan.lemon@gmail.com>
 Description:	This directory contains the attributes of the Nth timecard
 		registered.
 
+What:		/sys/class/timecard/ocpN/cpld_status
+Date:		July 2026
+Contact:	Sagi Maimon <maimon.sagi@gmail.com>
+Description:	(RO, root only) The flags set in the status register of the
+		TAP CPLD, space separated, or an empty line when none is
+		set.  Only present on ADVA x1 TAP boards (PCI ID
+		0xad5a:0x0410).
+
+		==========  ================================================
+		done        the configuration flash holds a valid image and
+			    it is active
+		busy        an internal operation is in progress
+		failed      the last ISC operation failed
+		==========  ================================================
+
+		For example "busy" while an operation runs, or "busy failed"
+		if a failure is latched while another one is in progress.
+
+		A read arbitrates for the shared I2C bus and reprograms the
+		on-card mux, so it is restricted to root.  The Lattice device
+		ID of the CPLD is read by the driver shortly after probe,
+		and again after a successful CPLD update, and reported from
+		that cached value as the fixed "cpld.id" version by
+		devlink dev info.
+
+		New CPLD firmware is programmed with devlink dev flash,
+		selecting the "fw.cpld" component; see
+		Documentation/networking/devlink/ptp_ocp.rst.
+
 What:		/sys/class/timecard/ocpN/available_clock_sources
 Date:		September 2021
 Contact:	Jonathan Lemon <jonathan.lemon@gmail.com>
@@ -156,9 +185,11 @@ Description:	(RW) Used to start the signal generator, and summarize
 
 		    period [duty [phase [polarity]]]
 
-		echo 500000000 > signal       # 1/2 second period
-		echo 1000000 40 100 > signal
-		echo 0 > signal               # turn off generator
+		::
+
+		  echo 500000000 > signal       # 1/2 second period
+		  echo 1000000 40 100 > signal
+		  echo 0 > signal               # turn off generator
 
 		Period and phase are specified in nanoseconds.  Duty cycle is
 		a percentage from 1-99.  Polarity is 1 or 0.
diff --git a/Documentation/Makefile b/Documentation/Makefile
index 377a449656c817..ee852afe9bb998 100644
--- a/Documentation/Makefile
+++ b/Documentation/Makefile
@@ -74,6 +74,10 @@ htmldocs-redirects: $(srctree)/Documentation/.renames.txt
 refcheckdocs:
 	$(Q)cd $(srctree); tools/docs/documentation-file-ref-check
 
+testdocs:
+	$(Q)PYTHONPYCACHEPREFIX="$(PYTHONPYCACHEPREFIX)" \
+		$(PYTHON3) $(srctree)/tools/unittests/run.py
+
 cleandocs:
 	$(Q)rm -rf $(BUILDDIR)
 
@@ -95,6 +99,7 @@ dochelp:
 	@echo  '                    (will connect to external hosts)'
 	@echo  '  refcheckdocs    - check for references to non-existing files under'
 	@echo  '                    Documentation'
+	@echo  '  testdocs        - run the unit tests of the documentation tools'
 	@echo  '  cleandocs       - clean all generated files'
 	@echo
 	@echo  '  make SPHINXDIRS="s1 s2" [target] Generate only docs of folder s1, s2'
diff --git a/Documentation/PCI/controller/index.rst b/Documentation/PCI/controller/index.rst
index c15e2c9ac855a9..096d1c3d9807ad 100644
--- a/Documentation/PCI/controller/index.rst
+++ b/Documentation/PCI/controller/index.rst
@@ -8,4 +8,3 @@ PCI Native Host Bridge and Endpoint Drivers
    :maxdepth: 2
 
    pci-controller-drivers
-   rcar-pcie-firmware
diff --git a/Documentation/PCI/controller/rcar-pcie-firmware.rst b/Documentation/PCI/controller/rcar-pcie-firmware.rst
deleted file mode 100644
index 67d3bf66e31585..00000000000000
--- a/Documentation/PCI/controller/rcar-pcie-firmware.rst
+++ /dev/null
@@ -1,32 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-=================================================
-Firmware of PCIe controller for Renesas R-Car V4H
-=================================================
-
-Renesas R-Car V4H (r8a779g0) has a PCIe controller, requiring a specific
-firmware download during startup.
-
-However, Renesas currently cannot distribute the firmware free of charge.
-
-The firmware file "104_PCIe_fw_addr_data_ver1.05.txt" (note that the file name
-might be different between different datasheet revisions) can be found in the
-datasheet encoded as text, and as such, the file's content must be converted
-back to binary form. This can be achieved using the following example script:
-
-.. code-block:: sh
-
-	$ awk '/^\s*0x[0-9A-Fa-f]{4}\s+0x[0-9A-Fa-f]{4}/ { print substr($2,5,2) substr($2,3,2) }' \
-		104_PCIe_fw_addr_data_ver1.05.txt | \
-			xxd -p -r > rcar_gen4_pcie.bin
-
-Once the text content has been converted into a binary firmware file, verify
-its checksum as follows:
-
-.. code-block:: sh
-
-	$ sha1sum rcar_gen4_pcie.bin
-	1d0bd4b189b4eb009f5d564b1f93a79112994945  rcar_gen4_pcie.bin
-
-The resulting binary file called "rcar_gen4_pcie.bin" should be placed in the
-"/lib/firmware" directory before the driver runs.
diff --git a/Documentation/PCI/index.rst b/Documentation/PCI/index.rst
index 5d720d2a415e01..23fb737ac969ea 100644
--- a/Documentation/PCI/index.rst
+++ b/Documentation/PCI/index.rst
@@ -20,3 +20,4 @@ PCI Bus Subsystem
    controller/index
    boot-interrupts
    tph
+   liveupdate
diff --git a/Documentation/PCI/liveupdate.rst b/Documentation/PCI/liveupdate.rst
new file mode 100644
index 00000000000000..96b1d7f5df3a03
--- /dev/null
+++ b/Documentation/PCI/liveupdate.rst
@@ -0,0 +1,35 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+===========================
+PCI Support for Live Update
+===========================
+
+.. kernel-doc:: drivers/pci/liveupdate.c
+   :doc: PCI Live Update
+
+Driver API
+==========
+
+.. kernel-doc:: drivers/pci/liveupdate.c
+   :export:
+
+Internal API
+============
+
+.. kernel-doc:: drivers/pci/liveupdate.c
+   :internal:
+
+Live Update ABI
+===============
+
+.. kernel-doc:: include/linux/kho/abi/pci.h
+   :doc: PCI File-Lifecycle Bound (FLB) Live Update ABI
+
+.. kernel-doc:: include/linux/kho/abi/pci.h
+   :internal:
+
+See Also
+========
+
+ * :doc:`/core-api/liveupdate`
+ * :doc:`/core-api/kho/index`
diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst
index d7c8eff6331740..a986358a5c2795 100644
--- a/Documentation/RCU/stallwarn.rst
+++ b/Documentation/RCU/stallwarn.rst
@@ -122,10 +122,11 @@ warnings:
 	of RCU CPU stall warnings, eventually leading to the realization
 	that the CPU had failed.
 
-The RCU, RCU-sched, RCU-tasks, and RCU-tasks-trace implementations have
-CPU stall warning.  Note that SRCU does *not* have CPU stall warnings.
-Please note that RCU only detects CPU stalls when there is a grace period
-in progress.  No grace period, no CPU stall warnings.
+The RCU and RCU-tasks implementations have CPU stall warning.  Note that
+SRCU does *not* have CPU stall warnings, and now that RCU-tasks-trace
+is implemented in terms of SRCU, neither does it.  Please note that
+RCU only detects CPU stalls when there is a grace period in progress.
+No grace period, no CPU stall warnings.
 
 To diagnose the cause of the stall, inspect the stack traces.
 The offending function will usually be near the top of the stack.
@@ -213,21 +214,16 @@ RCU_STALL_RAT_DELAY
 rcupdate.rcu_task_stall_timeout
 -------------------------------
 
-	This boot/sysfs parameter controls the RCU-tasks and
-	RCU-tasks-trace stall warning intervals.  A value of zero or less
-	suppresses RCU-tasks stall warnings.  A positive value sets the
-	stall-warning interval in seconds.  An RCU-tasks stall warning
-	starts with the line:
+	This boot/sysfs parameter controls the RCU-tasks stall warning
+	interval.  A value of zero or less suppresses RCU-tasks stall
+	warnings.  A positive value sets the stall-warning interval
+	in seconds.  An RCU-tasks stall warning starts with the line:
 
 		INFO: rcu_tasks detected stalls on tasks:
 
 	And continues with the output of sched_show_task() for each
 	task stalling the current RCU-tasks grace period.
 
-	An RCU-tasks-trace stall warning starts (and continues) similarly:
-
-		INFO: rcu_tasks_trace detected stalls on tasks
-
 
 Interpreting RCU's CPU Stall-Detector "Splats"
 ==============================================
diff --git a/Documentation/admin-guide/LSM/LoadPin.rst b/Documentation/admin-guide/LSM/LoadPin.rst
index dd3ca68b5df188..a90fd9dac9606f 100644
--- a/Documentation/admin-guide/LSM/LoadPin.rst
+++ b/Documentation/admin-guide/LSM/LoadPin.rst
@@ -23,9 +23,9 @@ sysctl allows for easy testing on systems with a mutable filesystem.)
 It's also possible to exclude specific file types from LoadPin using kernel
 command line option "``loadpin.exclude``". By default, all files are
 included, but they can be excluded using kernel command line option such
-as "``loadpin.exclude=kernel-module,kexec-image``". This allows to use
+as "``loadpin.exclude=kernel-module,kexec-image``". This allows using
 different mechanisms such as ``CONFIG_MODULE_SIG`` and
-``CONFIG_KEXEC_VERIFY_SIG`` to verify kernel module and kernel image while
-still use LoadPin to protect the integrity of other files kernel loads. The
-full list of valid file types can be found in ``kernel_read_file_str``
-defined in ``include/linux/kernel_read_file.h``.
+``CONFIG_KEXEC_SIG`` to verify kernel modules and kernel images while
+still using LoadPin to protect the integrity of other files the kernel
+loads. The full list of valid file types can be found in
+``kernel_read_file_str`` defined in ``include/linux/kernel_read_file.h``.
diff --git a/Documentation/admin-guide/LSM/index.rst b/Documentation/admin-guide/LSM/index.rst
index b44ef68f6e4da6..9518495edfbc32 100644
--- a/Documentation/admin-guide/LSM/index.rst
+++ b/Documentation/admin-guide/LSM/index.rst
@@ -6,9 +6,11 @@ The Linux Security Module (LSM) framework provides a mechanism for
 various security checks to be hooked by new kernel extensions. The name
 "module" is a bit of a misnomer since these extensions are not actually
 loadable kernel modules. Instead, they are selectable at build-time via
-CONFIG_DEFAULT_SECURITY and can be overridden at boot-time via the
-``"security=..."`` kernel command line argument, in the case where multiple
-LSMs were built into a given kernel.
+CONFIG_LSM, an ordered list of the LSMs to enable, and can be
+overridden at boot-time via the ``"lsm=..."`` kernel command line
+argument. The ``"security=..."`` kernel command line argument remains
+available to choose a legacy "major" security module, but has been
+deprecated by the ``"lsm=..."`` parameter.
 
 The primary users of the LSM interface are Mandatory Access Control
 (MAC) extensions which provide a comprehensive security policy. Examples
@@ -25,9 +27,15 @@ man-pages project.
 A list of the active security modules can be found by reading
 ``/sys/kernel/security/lsm``. This is a comma separated list, and
 will always include the capability module. The list reflects the
-order in which checks are made. The capability module will always
-be first, followed by any "minor" modules (e.g. Yama) and then
-the one "major" module (e.g. SELinux) if there is one configured.
+order in which checks are made. The capability module will be
+first, unless CONFIG_SECURITY_LOCKDOWN_LSM_EARLY is enabled, in
+which case the lockdown module will precede it. The integrity
+modules (e.g. IMA and EVM), if enabled in the kernel
+configuration, are always placed at the end of the list. Any
+other "minor" modules (e.g. Yama) and the one "major" module
+(e.g. SELinux), if there is one configured, appear in between,
+in the order given by CONFIG_LSM or the ``"lsm=..."`` kernel
+command line parameter.
 
 Process attributes associated with "major" security modules should
 be accessed and maintained using the special files in ``/proc/.../attr``.
diff --git a/Documentation/admin-guide/LSM/ipe.rst b/Documentation/admin-guide/LSM/ipe.rst
index a756d81585317c..bebf14fc241166 100644
--- a/Documentation/admin-guide/LSM/ipe.rst
+++ b/Documentation/admin-guide/LSM/ipe.rst
@@ -502,11 +502,11 @@ The following table lists the error codes that may appear in the errno field whi
 Event Examples::
 
    type=1404 audit(1653425689.008:55): enforcing=0 old_enforcing=1 auid=4294967295 ses=4294967295 enabled=1 old-enabled=1 lsm=ipe res=1
-   type=1300 audit(1653425689.008:55): arch=c000003e syscall=1 success=yes exit=2 a0=1 a1=55c1065e5c60 a2=2 a3=0 items=0 ppid=405 pid=441 auid=0 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=)
+   type=1300 audit(1653425689.008:55): arch=c000003e syscall=1 success=yes exit=2 a0=1 a1=55c1065e5c60 a2=2 a3=0 items=0 ppid=405 pid=441 auid=0 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0
    type=1327 audit(1653425689.008:55): proctitle="-bash"
 
    type=1404 audit(1653425689.008:55): enforcing=1 old_enforcing=0 auid=4294967295 ses=4294967295 enabled=1 old-enabled=1 lsm=ipe res=1
-   type=1300 audit(1653425689.008:55): arch=c000003e syscall=1 success=yes exit=2 a0=1 a1=55c1065e5c60 a2=2 a3=0 items=0 ppid=405 pid=441 auid=0 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=)
+   type=1300 audit(1653425689.008:55): arch=c000003e syscall=1 success=yes exit=2 a0=1 a1=55c1065e5c60 a2=2 a3=0 items=0 ppid=405 pid=441 auid=0 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0
    type=1327 audit(1653425689.008:55): proctitle="-bash"
 
 This record will always be emitted in conjunction with a ``AUDITSYSCALL`` record for the ``write`` syscall.
diff --git a/Documentation/admin-guide/RAS/main.rst b/Documentation/admin-guide/RAS/main.rst
index 5a45db32c49b09..c1040e5a5b3537 100644
--- a/Documentation/admin-guide/RAS/main.rst
+++ b/Documentation/admin-guide/RAS/main.rst
@@ -623,7 +623,7 @@ Under ``/sys/devices/system/edac/pci`` are control and attribute files as
 follows:
 
 
-- ``check_pci_parity`` - Enable/Disable PCI Parity checking control file
+- ``check_pci_errors`` - Enable/Disable PCI Parity checking control file
 
 	This control file enables or disables the PCI Bus Parity scanning
 	operation. Writing a 1 to this file enables the scanning. Writing
@@ -631,11 +631,11 @@ follows:
 
 	Enable::
 
-		echo "1" >/sys/devices/system/edac/pci/check_pci_parity
+		echo "1" >/sys/devices/system/edac/pci/check_pci_errors
 
 	Disable::
 
-		echo "0" >/sys/devices/system/edac/pci/check_pci_parity
+		echo "0" >/sys/devices/system/edac/pci/check_pci_errors
 
 
 - ``pci_parity_count`` - Parity Count
@@ -725,15 +725,15 @@ Module parameters
 
 	module/kernel parameter::
 
-			edac_panic_on_pci_pe=[0|1]
+			edac_pci_panic_on_pe=[0|1]
 
 	Enable::
 
-		echo "1" > /sys/module/edac_core/parameters/edac_panic_on_pci_pe
+		echo "1" > /sys/module/edac_core/parameters/edac_pci_panic_on_pe
 
 	Disable::
 
-		echo "0" > /sys/module/edac_core/parameters/edac_panic_on_pci_pe
+		echo "0" > /sys/module/edac_core/parameters/edac_pci_panic_on_pe
 
 
 
diff --git a/Documentation/admin-guide/binfmt-misc.rst b/Documentation/admin-guide/binfmt-misc.rst
index d26b63a27c259b..9e84b877d06dea 100644
--- a/Documentation/admin-guide/binfmt-misc.rst
+++ b/Documentation/admin-guide/binfmt-misc.rst
@@ -19,6 +19,9 @@ To actually register a new binary type, you have to set up a string looking like
 ``:name:type:offset:magic:mask:interpreter:flags`` (where you can choose the
 ``:`` upon your needs) and echo it to ``/proc/sys/fs/binfmt_misc/register``.
 
+The first character of the string is its field delimiter and can be any
+ASCII punctuation character other than the backslash ``\``.
+
 Here is what the fields mean:
 
 - ``name``
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 86a2a0099178ea..7fe950425216cc 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -1321,6 +1321,10 @@ All memory amounts are in bytes.  If a value which is not aligned to
 PAGE_SIZE is written, the value may be rounded up to the closest
 PAGE_SIZE multiple when read back.
 
+For the limit files described below, an empty or all-whitespace
+write is accepted and sets the limit to 0.  To disable a limit,
+write "max"; to set it to zero explicitly, write "0".
+
   memory.current
 	A read-only single value file which exists on non-root
 	cgroups.
@@ -2536,16 +2540,55 @@ Cpuset Interface Files
 	A read-only multiple values file which exists on all
 	cpuset-enabled cgroups.
 
-	It lists the onlined CPUs that are actually granted to this
-	cgroup by its parent.  These CPUs are allowed to be used by
-	tasks within the current cgroup.
+	It lists the active CPUs available to tasks in this cgroup.
+
+	For a cgroup that is not a valid partition root, an empty
+	"cpuset.cpus" makes "cpuset.cpus.effective" show all CPUs
+	available from the parent cgroup. Otherwise, it is a subset
+	of "cpuset.cpus" unless none of the requested CPUs can be
+	granted. In that case, it is treated like an empty
+	"cpuset.cpus".
+
+	For example::
 
-	If "cpuset.cpus" is empty, the "cpuset.cpus.effective" file shows
-	all the CPUs from the parent cgroup that can be available to
-	be used by this cgroup.  Otherwise, it should be a subset of
-	"cpuset.cpus" unless none of the CPUs listed in "cpuset.cpus"
-	can be granted.  In this case, it will be treated just like an
-	empty "cpuset.cpus".
+	  P (cpuset.cpus.effective=0-3)
+	  |
+	  +-- C0 (member):       cpuset.cpus=1-2
+	  |                      cpuset.cpus.effective=1-2
+	  +-- C1 (invalid root): cpuset.cpus=4-5
+	  |                      cpuset.cpus.effective=0-3
+	  \-- C2 (member):       cpuset.cpus=<empty>
+	                         cpuset.cpus.effective=0-3
+
+	C0 gets CPUs 1-2 because they are available from P.
+	None of C1's requested CPUs 4-5 are available from P, so C1 uses P's
+	effective CPUs 0-3 instead.
+	C2 has an empty "cpuset.cpus" and therefore also uses P's effective
+	CPUs 0-3.
+
+	For a valid non-root partition root, "cpuset.cpus.effective" contains
+	the active CPUs from "cpuset.cpus.exclusive.effective" except
+	those granted to valid child partition roots. If
+	"cpuset.cpus.exclusive" is set to a value different from
+	"cpuset.cpus", the effective CPUs need not be a subset of
+	"cpuset.cpus".
+
+	For example, if CPUs 2-5 are allocated to P for exclusive use,
+	CPUs 2-4 are active, CPU 5 is offline, and none of CPUs 2-5 are
+	assigned to a valid child partition root::
+
+	  top (root)
+	  |
+	  \-- P (valid partition root)
+	      cpuset.cpus=0-3
+	      cpuset.cpus.exclusive=2-5
+	      cpuset.cpus.exclusive.effective=2-5
+	      cpuset.cpus.effective=2-4
+
+	P has 2-5 in "cpuset.cpus.exclusive.effective", including the
+	offline CPU 5. CPU 5 is excluded from "cpuset.cpus.effective",
+	leaving 2-4. The result also shows that "cpuset.cpus.effective"
+	need not be a subset of "cpuset.cpus", which is 0-3.
 
 	Its value will be affected by CPU hotplug events.
 
@@ -2657,8 +2700,9 @@ Cpuset Interface Files
 	A read-only and root cgroup only multiple values file.
 
 	This file shows the set of all isolated CPUs used in existing
-	isolated partitions. It will be empty if no isolated partition
-	is created.
+	isolated partitions or isolated at boot time via the "domain"
+	flag of "isolcpus". The latter remain listed even after any
+	partition using them is released.
 
   cpuset.cpus.partition
 	A read-write single value file which exists on non-root
diff --git a/Documentation/admin-guide/cpu-isolation.rst b/Documentation/admin-guide/cpu-isolation.rst
index 8c65d03fd28c2a..b9f13a92867343 100644
--- a/Documentation/admin-guide/cpu-isolation.rst
+++ b/Documentation/admin-guide/cpu-isolation.rst
@@ -353,5 +353,5 @@ Some tools may also be useful for higher level analysis:
   latency and noise in the system. For example Documentation/tools/rtla/rtla-osnoise.rst
   runs a kernel tracer that analyzes and output a summary of the noises.
 
-- dynticks-testing does something similar to rtla-osnoise but in userspace. It is available
-  at git://git.kernel.org/pub/scm/linux/kernel/git/frederic/dynticks-testing.git
+- cpunoise does something similar to rtla-osnoise but in userspace. It is available
+  at https://git.kernel.org/pub/scm/linux/kernel/git/frederic/cpunoise.git
diff --git a/Documentation/admin-guide/device-mapper/verity.rst b/Documentation/admin-guide/device-mapper/verity.rst
index eb9475d7e1965a..9e9d4b8157c031 100644
--- a/Documentation/admin-guide/device-mapper/verity.rst
+++ b/Documentation/admin-guide/device-mapper/verity.rst
@@ -163,6 +163,11 @@ root_hash_sig_key_desc <key_description>
     also gain new certificates at run time if they are signed by a certificate
     already in the secondary trusted keyring.
 
+    Whether a signature is required for every dm-verity device is controlled by
+    the dm_verity.require_signatures parameter which defaults to off. Setting
+    DM_VERITY_VERIFY_ROOTHASH_SIG_FORCE makes it default to on in which case it
+    can no longer be turned off.
+
 try_verify_in_tasklet
     If verity hashes are in cache and the IO size does not exceed the limit,
     verify data blocks in bottom half instead of workqueue. This option can
diff --git a/Documentation/admin-guide/devices.txt b/Documentation/admin-guide/devices.txt
index f544399675fa3c..855f2cad0535e5 100644
--- a/Documentation/admin-guide/devices.txt
+++ b/Documentation/admin-guide/devices.txt
@@ -2388,18 +2388,9 @@
 
  171 char	Reserved for IEEE 1394 (Firewire)
 
- 172 char	Moxa Intellio serial card
-		  0 = /dev/ttyMX0	First Moxa port
-		  1 = /dev/ttyMX1	Second Moxa port
-		    ...
-		127 = /dev/ttyMX127	128th Moxa port
-		128 = /dev/moxactl	Moxa control port
+ 172 char
 
- 173 char	Moxa Intellio serial card - alternate devices
-		  0 = /dev/cumx0	Callout device for ttyMX0
-		  1 = /dev/cumx1	Callout device for ttyMX1
-		    ...
-		127 = /dev/cumx127	Callout device for ttyMX127
+ 173 char
 
  174 char	SmartIO serial card
 		  0 = /dev/ttySI0	First SmartIO port
diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst
index 7663c610fe9014..5f1df6d080508a 100644
--- a/Documentation/admin-guide/kdump/vmcoreinfo.rst
+++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst
@@ -325,7 +325,7 @@ NR_FREE_PAGES
 On linux-2.6.21 or later, the number of free pages is in
 vm_stat[NR_FREE_PAGES]. Used to get the number of free pages.
 
-PG_lru|PG_private|PG_swapcache|PG_swapbacked|PG_hwpoison|PG_head_mask
+PG_lru|PG_swapcache|PG_swapbacked|PG_hwpoison|PG_head_mask
 --------------------------------------------------------------------------
 
 Page attributes. These flags are used to filter various unnecessary for
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 33cd30996e47ef..b0fcdf663389c8 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -501,13 +501,6 @@ Kernel parameters
 			disable
 			  Disable amd-pstate preferred core.
 
-	amd_dynamic_epp=
-			[X86]
-			disable
-			  Disable amd-pstate dynamic EPP.
-			enable
-			  Enable amd-pstate dynamic EPP.
-
 	amijoy.map=	[HW,JOY] Amiga joystick support
 			Map of devices attached to JOY0DAT and JOY1DAT
 			Format: <a>,<b>
@@ -589,9 +582,10 @@ Kernel parameters
 	arm64.nosve	[ARM64] Unconditionally disable Scalable Vector
 			Extension support
 
-	ataflop=	[HW,M68k]
+	arm64.nowfxt    [ARM64] Unconditionally disable Wait For Event with
+			Timeout and Wait For Interrupt with Timeout support
 
-	atarimouse=	[HW,MOUSE] Atari Mouse
+	ataflop=	[HW,M68k]
 
 	atkbd.extra=	[HW] Enable extra LEDs and keys on IBM RapidAccess,
 			EzKey and similar keyboards
@@ -626,13 +620,6 @@ Kernel parameters
 			Format: <int> (must be >=0)
 			Default: 64
 
-	bau=		[X86_UV] Enable the BAU on SGI UV.  The default
-			behavior is to disable the BAU (i.e. bau=0).
-			Format: { "0" | "1" }
-			0 - Disable the BAU.
-			1 - Enable the BAU.
-			unset - Disable the BAU.
-
 	bdev_allow_write_mounted=
 			Format: <bool>
 			Control the ability to open a mounted block device
@@ -669,6 +656,22 @@ Kernel parameters
 
 			See Documentation/admin-guide/bootconfig.rst
 
+	bpf.keyring_unsealed=
+			[BPF] When set to 1, leave the bpf keyring unsealed
+			after initialization so that userspace can provision
+			keys. Once the keyring is restricted it becomes active
+			and can be used for BPF program signature verification.
+
+			Setting this also means that the bpf keyring is the
+			only non-system keyring a loader may select for the
+			rest of the boot: caller-supplied user/session
+			keyrings are refused with -EPERM, whether or not
+			provisioning actually completed. The system keyrings
+			stay selectable. Leaving it unset keeps the prior
+			behaviour, where a caller-supplied keyring is allowed.
+
+			See Documentation/bpf/signing.rst
+
 	bttv.card=	[HW,V4L] bttv (bt848 + bt878 based grabber cards)
 	bttv.radio=	Most important insmod options are available as
 			kernel args too.
@@ -1587,14 +1590,6 @@ Kernel parameters
 			PCI device even when its classcode is not of the
 			UART class.
 
-	edac_report=	[HW,EDAC] Control how to report EDAC event
-			Format: {"on" | "off" | "force"}
-			on: enable EDAC to report H/W event. May be overridden
-			by other higher priority error reporting module.
-			off: disable H/W event reporting through EDAC.
-			force: enforce the use of EDAC to report H/W event.
-			default: on.
-
 	edd=		[EDD]
 			Format: {"off" | "on" | "skip[mbr]"}
 
@@ -1664,12 +1659,6 @@ Kernel parameters
 			to discrete, to make X server driver able to add WB
 			entry later. This parameter enables that.
 
-	enable_timer_pin_1 [X86]
-			Enable PIN 1 of APIC timer
-			Can be useful to work around chipset bugs
-			(in particular on some ATI chipsets).
-			The kernel tries to set a reasonable default.
-
 	enforcing=	[SELINUX] Set initial enforcing status.
 			Format: {"0" | "1"}
 			See security/selinux/Kconfig help text.
@@ -1964,6 +1953,102 @@ Kernel parameters
 			for 64-bit NUMA, off otherwise.
 			Format: 0 | 1 (for off | on)
 
+	hazptrtorture.defer_modulus= [KNL]
+			Defer release of the hazard pointer on average
+			one out of the specified number of times.  Zero
+			disables.  Negative one defaults to 1,000 times
+			nr_cpu_ids, unless hazptr.kthread_do_pending_ms is
+			equal to zero, in which case it instead defaults
+			to zero (disabled).
+
+	hazptrtorture.irq_acquire= [KNL]
+			Acquire hazard pointers from an irq handler on average
+			one out of the specified number of times.  Zero
+			disables.  Negative one defaults to 1,000 times
+			nr_cpu_ids.
+
+	hazptrtorture.irq_release= [KNL]
+			Release hazard pointers from an irq handler on average
+			one out of the specified number of times.  Zero
+			disables.  Negative one defaults to 1,000 times
+			nr_cpu_ids.
+
+	hazptrtorture.kthread_do_pending_ms= [KNL]
+			Interval between cleanup of pending (deferred)
+			hazard-pointer releases in milliseconds.
+			Zero disables.	Negative one defaults to three
+			milliseconds, unless hazptr.kthread_do_pending_ms
+			is equal to zero, in which case it instead
+			defaults to zero (disabled).
+
+	hazptrtorture.nreaders= [KNL]
+			Number of hazard-pointer reader kthreads, each
+			of which repeatedly acquires and releases hazard
+			pointers.  If the value is zero, one reader
+			is spawned.  If the value is less than zero,
+			the absolute value is multiplied by the number
+			of online CPUs at initialization time.
+
+	hazptrtorture.nwriters= [KNL]
+			Controls whether or not there is a writer.
+			There is at most one writer, so this value must
+			be zero or one.
+
+	hazptrtorture.onoff_holdoff= [KNL]
+			Set time (s) after boot for CPU-hotplug testing.
+
+	hazptrtorture.onoff_interval= [KNL]
+			Set time (jiffies) between CPU-hotplug operations,
+			or zero to disable CPU-hotplug testing.
+
+	hazptrtorture.preempt_duration= [KNL]
+			Set duration (in milliseconds) of preemptions
+			by a high-priority FIFO real-time task.  Set to
+			zero (the default) to disable.	The CPUs to
+			preempt are selected randomly from the set that
+			are online at a given point in time.  Races with
+			CPUs going offline are ignored, with that attempt
+			at preemption skipped.
+
+	hazptrtorture.preempt_interval= [KNL]
+			Set interval (in milliseconds, defaulting to one
+			second) between preemptions by a high-priority
+			FIFO real-time task.  This delay is mediated
+			by an hrtimer and is further fuzzed to avoid
+			inadvertent synchronizations.
+
+	hazptrtorture.reader_sleep_us= [KNL]
+			Set occassional read-side sleep time in
+			microseconds, defaulting to zero for disabled.
+
+	hazptrtorture.shuffle_interval= [KNL]
+			Set task-shuffle interval (seconds).  Shuffling
+			tasks allows some CPUs to go into dyntick-idle
+			mode during the hazptrtorture test.
+
+	hazptrtorture.shutdown_secs= [KNL]
+			Set time (s) after boot system shutdown.  This
+			is useful for hands-off automated testing.
+
+	hazptrtorture.stat_interval= [KNL]
+			Time (s) between statistics printk()s.
+
+	hazptrtorture.stutter= [KNL]
+			Time (s) to stutter testing, for example,
+			specifying three seconds (the default) causes
+			the test to run for three seconds, wait
+			for five seconds, and so on.  This tests the
+			hazard-pointers primitives' ability to transition
+			abruptly to and from idle.  Setting this to zero
+			disables stuttering.
+
+	hazptrtorture.torture_type= [KNL]
+			Specify the hazard-pointers implementation
+			to test.
+
+	hazptrtorture.verbose= [KNL]
+			Enable additional printk() statements.
+
 	hd=		[EIDE] (E)IDE hard drive subsystem geometry
 			Format: <cyl>,<head>,<sect>
 
@@ -3031,11 +3116,12 @@ Kernel parameters
 			"0" | "off" | "n" - kexec handover is disabled
 			"1" | "on" | "y" - kexec handover is enabled
 
-	kho_scratch=	[KEXEC,EARLY]
+	kho_bootmem=	[KEXEC,EARLY]
 			Format: ll[KMG],mm[KMG],nn[KMG] | nn%
-			Defines the size of the KHO scratch region. The KHO
-			scratch regions are physically contiguous memory
-			ranges that can only be used for non-kernel
+
+			Defines the size of the KHO boot memory region. The
+			KHO boot memory regions are physically contiguous
+			memory ranges that can only be used for non-kernel
 			allocations. That way, even when memory is heavily
 			fragmented with handed over memory, the kexeced
 			kernel will always have enough contiguous ranges to
@@ -3044,10 +3130,10 @@ Kernel parameters
 			It is possible to specify the exact amount of
 			memory in the form of "ll[KMG],mm[KMG],nn[KMG]"
 			where the first parameter defines the size of a low
-			memory scratch area, the second parameter defines
-			the size of a global scratch area and the third
+			memory bootmem area, the second parameter defines
+			the size of a global bootmem area and the third
 			parameter defines the size of additional per-node
-			scratch areas.  The form "nn%" defines scale factor
+			bootmem areas.  The form "nn%" defines scale factor
 			(in percents) of memory that was used during boot.
 
 	kmac=		[MIPS] Korina ethernet MAC address.
@@ -3156,6 +3242,32 @@ Kernel parameters
 	kvm.enable_vmware_backdoor=[KVM] Support VMware backdoor PV interface.
 				   Default is false (don't support).
 
+	kvm.gmem_in_place_conversion=
+			[KVM] Controls whether KVM enables in-place conversion
+			support for guest_memfd and tracks the private/shared
+			state of memory per guest_memfd instead of per VM.
+
+			If enabled, KVM enables the KVM_SET_MEMORY_ATTRIBUTES2
+			ioctl on guest_memfd file descriptors and disables the
+			legacy VM-scoped KVM_SET_MEMORY_ATTRIBUTES ioctl for
+			private memory state tracking. Only the
+			KVM_MEMORY_ATTRIBUTE_PRIVATE attribute moves to
+			per-guest_memfd tracking; other attributes remain
+			per-VM.
+
+			This parameter toggles KVM's in-place conversion
+			capability support. Whether a VMM uses separate backends
+			or out-of-place memory management is determined by
+			userspace VMM design.
+
+			Note, this parameter is only available when
+			CONFIG_KVM_VM_MEMORY_ATTRIBUTES=y. When
+			CONFIG_KVM_VM_MEMORY_ATTRIBUTES is not set, in-place
+			conversion is enabled (and cannot be disabled) if KVM
+			is built with private memory support.
+
+			Default is N (off).
+
 	kvm.nx_huge_pages=
 			[KVM] Controls the software workaround for the
 			X86_BUG_ITLB_MULTIHIT bug.
@@ -3491,7 +3603,7 @@ Kernel parameters
 
 			* SATA link speed limit: 1.5Gbps or 3.0Gbps.
 
-			* Transfer mode: pio[0-7], mwdma[0-4] and udma[0-7].
+			* Transfer mode: pio[0-6], mwdma[0-4] and udma[0-7].
 			  udma[/][16,25,33,44,66,100,133] notation is also
 			  allowed.
 
@@ -3509,7 +3621,7 @@ Kernel parameters
 
 			* [no]ncqtrim: Enable or disable queued DSM TRIM.
 
-			* [no]ncqati: Enable or disable NCQ trim on ATI chipset.
+			* [no]ncqati: Enable or disable NCQ on ATI chipsets.
 
 			* [no]trim: Enable or disable (unqueued) TRIM.
 
@@ -3540,11 +3652,9 @@ Kernel parameters
 
 			* max_sec_128: Set transfer size limit to 128 sectors.
 
-			* max_sec_1024: Set or clear transfer size limit to
-			  1024 sectors.
+			* max_sec_1024: Set transfer size limit to 1024 sectors.
 
-			* max_sec_lba48: Set or clear transfer size limit to
-			  65535 sectors.
+			* max_sec_lba48: Set transfer size limit to 65535 sectors.
 
 			* external: Mark port as external (hotplug-capable).
 
@@ -4075,27 +4185,6 @@ Kernel parameters
 			Enable or disable the microcode minimal revision
 			enforcement for the runtime microcode loader.
 
-	mini2440=	[ARM,HW,KNL]
-			Format:[0..2][b][c][t]
-			Default: "0tb"
-			MINI2440 configuration specification:
-			0 - The attached screen is the 3.5" TFT
-			1 - The attached screen is the 7" TFT
-			2 - The VGA Shield is attached (1024x768)
-			Leaving out the screen size parameter will not load
-			the TFT driver, and the framebuffer will be left
-			unconfigured.
-			b - Enable backlight. The TFT backlight pin will be
-			linked to the kernel VESA blanking code and a GPIO
-			LED. This parameter is not necessary when using the
-			VGA shield.
-			c - Enable the s3c camera interface.
-			t - Reserved for enabling touchscreen support. The
-			touchscreen support is not enabled in the mainstream
-			kernel as of 2.6.30, a preliminary port can be found
-			in the "bleeding edge" mini2440 support kernel at
-			https://repo.or.cz/w/linux-2.6/mini2440.git
-
 	mitigations=
 			[X86,PPC,S390,ARM64,EARLY] Control optional mitigations for
 			CPU vulnerabilities.  This is a set of curated,
@@ -4597,9 +4686,6 @@ Kernel parameters
 
 	nomce		[X86-32] Disable Machine Check Exception
 
-	nomfgpt		[X86-32] Disable Multi-Function General Purpose
-			Timer usage (for AMD Geode machines).
-
 	nomodeset	Disable kernel modesetting. Most systems' firmware
 			sets up a display mode and provides framebuffer memory
 			for output. With nomodeset, DRM and fbdev drivers will
@@ -4812,11 +4898,6 @@ Kernel parameters
 			waiting for the ACK, so if this is set too high
 			interrupts *may* be lost!
 
-	omap_mux=	[OMAP] Override bootloader pin multiplexing.
-			Format: <mux_mode0.mode_name=value>...
-			For example, to override I2C bus2:
-			omap_mux=i2c2_scl.i2c2_scl=0x100,i2c2_sda.i2c2_sda=0x100
-
 	onenand.bdry=	[HW,MTD] Flex-OneNAND Boundary Configuration
 
 			Format: [die0_boundary][,die0_lock][,die1_boundary][,die1_lock]
@@ -5133,7 +5214,12 @@ Kernel parameters
 				explicitly which ones they are.
 		assign-busses	[X86] Always assign all PCI bus
 				numbers ourselves, overriding
-				whatever the firmware may have done.
+				whatever the firmware may have done. Ignored
+				if any device was preserved across a Live
+				Update, where the kernel must preserve the
+				PCI topology (including bus numbers) to
+				avoid interrupting ongoing memory transactions
+				of preserved devices.
 		usepirqmask	[X86] Honor the possible IRQ mask stored
 				in the BIOS $PIR table. This is needed on
 				some systems with broken BIOSes, notably
@@ -5326,8 +5412,6 @@ Kernel parameters
 		nomsi	Do not use MSI for native PCIe PME signaling (this makes
 			all PCIe root ports use INTx for all services).
 
-	pcmv=		[HW,PCMCIA] BadgePAD 4
-
 	pd_ignore_unused
 			[PM]
 			Keep all power-domains already enabled by bootloader on,
@@ -5453,8 +5537,8 @@ Kernel parameters
 			lazy - Scheduler controlled. Similar to full but instead
 			       of preempting the task immediately, the task gets
 			       one HZ tick time to yield itself before the
-			       preemption will be forced. One preemption is when the
-			       task returns to user space.
+			       preemption will be forced. One such preemption
+			       point is when the task returns to user space.
 
 	print-fatal-signals=
 			[KNL] debug: print fatal signals
@@ -5662,9 +5746,12 @@ Kernel parameters
 			there is an ongoing too-long CSD-lock wait.
 
 	rcutree.do_rcu_barrier=	[KNL]
-			Request a call to rcu_barrier().  This is
-			throttled so that userspace tests can safely
-			hammer on the sysfs variable if they so choose.
+			Wait for deferred kfree_rcu() frees and ordinary
+			call_rcu() callbacks queued before this request to
+			complete.  This does not prevent new work from being
+			queued concurrently.  Requests are throttled so that
+			userspace tests can safely hammer on the sysfs
+			variable if they so choose.
 			If triggered before the RCU grace-period machinery
 			is fully active, this will error out with EAGAIN.
 
@@ -6136,6 +6223,13 @@ Kernel parameters
 			stress RCU, they don't participate in the actual
 			test, hence the "fake".
 
+	rcutorture.nmi_calls= [KNL]
+			Enable issuing RCU callbacks from an NMI, on the
+			RCU flavors that support it, to exercise the
+			any-context callback path.  Requires
+			CONFIG_PERF_EVENTS and a hardware PMU; without
+			both, nothing is issued.  Defaults to enabled.
+
 	rcutorture.nocbs_nthreads= [KNL]
 			Set number of RCU callback-offload togglers.
 			Zero (the default) disables toggling.
@@ -6365,6 +6459,13 @@ Kernel parameters
 			period to instead use normal non-expedited
 			grace-period processing.
 
+	rcupdate.rcu_task_collapse_debug= [KNL]
+			Enable debugging prints that record when RCU Tasks
+			and RCU Tasks Trace expand to per-CPU callback
+			queuing and collapse back to CPU-0 queuing.
+			This is default-disabled due to the fact that
+			some workloads can make it quite noisy.
+
 	rcupdate.rcu_task_collapse_lim= [KNL]
 			Set the maximum number of callbacks present
 			at the beginning of a grace period that allows
@@ -6611,6 +6712,9 @@ Kernel parameters
 
 			reserve_mem=12M:4096:oops ramoops.mem_name=oops
 
+			Both the size and the alignment must be multiples of
+			the page size.
+
 	reservetop=	[X86-32,EARLY]
 			Format: nn[KMG]
 			Reserves a hole at the top of the kernel virtual
diff --git a/Documentation/admin-guide/laptops/thinkpad-acpi.rst b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
index db4588af02781d..60bf6868b84965 100644
--- a/Documentation/admin-guide/laptops/thinkpad-acpi.rst
+++ b/Documentation/admin-guide/laptops/thinkpad-acpi.rst
@@ -583,7 +583,8 @@ Events that are propagated by the driver to userspace:
 0x6030		System thermal table changed
 0x6032		Thermal Control command set completion  (DYTC, Windows)
 0x6040		Nvidia Optimus/AC adapter related (TO BE VERIFIED)
-0x60C0		X1 Yoga 2016, Tablet mode status changed
+0x60C0		X1 Yoga 2016 tablet mode or X1 Fold 16 Gen 1 keyboard
+		attachment state changed
 0x60F0		Thermal Transformation changed (GMTS, Windows)
 ======		=====================================================
 
@@ -1658,6 +1659,29 @@ if damage detected:
 The property is read-only. If feature is not supported then sysfs
 attribute is not created.
 
+X1 Fold keyboard attachment
+---------------------------
+
+sysfs attribute: keyboard_attached_on_screen
+
+ThinkPad X1 Fold 16 Gen 1 systems can report whether the keyboard is
+currently attached on-screen.
+
+This attribute is currently supported on ThinkPad X1 Fold 16 Gen 1
+systems only.
+
+The command to check the keyboard attachment state is::
+
+	cat /sys/devices/platform/thinkpad_acpi/keyboard_attached_on_screen
+
+The values are:
+
+- 0 if the keyboard is not attached on-screen
+- 1 if the keyboard is attached on-screen
+
+The property is read-only. If feature is not supported then the sysfs
+attribute is not created.
+
 Multiple Commands, Module Parameters
 ------------------------------------
 
diff --git a/Documentation/admin-guide/media/vivid.rst b/Documentation/admin-guide/media/vivid.rst
index 034ca7c77fb973..6741f25220fca6 100644
--- a/Documentation/admin-guide/media/vivid.rst
+++ b/Documentation/admin-guide/media/vivid.rst
@@ -6,7 +6,7 @@ The Virtual Video Test Driver (vivid)
 This driver emulates video4linux hardware of various types: video capture, video
 output, vbi capture and output, metadata capture and output, radio receivers and
 transmitters, touch capture and a software defined radio receiver. In addition a
-simple framebuffer device is available for testing capture and output overlays.
+simple framebuffer device is available for testing output overlays.
 
 Up to 64 vivid instances can be created, each with up to 16 inputs and 16 outputs.
 
@@ -35,7 +35,7 @@ This document describes the features implemented by this driver:
 - Raw and Sliced VBI capture and output support
 - Radio receiver and transmitter support, including RDS support
 - Software defined radio (SDR) support
-- Capture and output overlay support
+- Output overlay support
 - Metadata capture and output support
 - Touch capture support
 
diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst
index da5f9afd08aef8..ba47255448564b 100644
--- a/Documentation/admin-guide/mm/damon/usage.rst
+++ b/Documentation/admin-guide/mm/damon/usage.rst
@@ -74,8 +74,11 @@ comma (",").
     │ │ │ │ │ │ nr_regions/min,max
     │ │ │ │ │ │ :ref:`probes <damon_usage_sysfs_probes>`/nr_probes
     │ │ │ │ │ │ │ 0/weight
+    │ │ │ │ │ │ │ │ preps/nr_preps
+    │ │ │ │ │ │ │ │ │ 0/prep_action
+    │ │ │ │ │ │ │ │ │ ...
     │ │ │ │ │ │ │ │ filters/nr_filters
-    │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path
+    │ │ │ │ │ │ │ │ │ 0/type,matching,allow,path,min,max
     │ │ │ │ │ │ │ │ │ ...
     │ │ │ │ │ │ │ ...
     │ │ │ │ │ :ref:`targets <sysfs_targets>`/nr_targets
@@ -283,9 +286,18 @@ In the beginning, this directory has only one file, ``nr_probes``.  Writing a
 number (``N``) to the file creates the number of child directories named ``0``
 to ``N-1``.  Each directory represents each monitoring probe.
 
-In each probe directory, one directory, ``filters`` exists.  The directory
-contains files for installing filters for the probe, that is used to determine
-the data attribute for the probe.
+In each probe directory, two directories, ``preps`` and ``filters`` exist.  The
+directories contain files for installing probing preparation actions and
+filters for the probe, that are used to determine the data attribute for the
+probe.
+
+In the beginning, ``preps`` directory has only one file, ``nr_preps``.
+Writing a number (``N``) to the file creates the number of child directories
+named ``0`` to ``N-1``.  Each directory represents each preparation action.
+Each directory has one file, ``prep_action``.  The preparation action can be
+selected by writing the name of the action to the ``prep_action`` file.  Refer
+to the :ref:`design doc <damon_design_data_attrs_monitoring>` for the list of
+supported actions.
 
 Each probe directory also contains ``weight`` file.  Reading from and writing
 to the file gets and sets the :ref:`attributes-only monitoring
@@ -296,6 +308,8 @@ Writing a number (``N``) to the file creates the number of child directories
 named ``0`` to ``N-1``.  Each directory represents each filter and works in a
 way similar to that for :ref:`DAMOS filter <sysfs_filters>`.  When the filter
 ``type`` is ``memcg``, ``path`` file acts as ``memcg_path`` for :ref:`DAMOS
+filter <sysfs_filters>`.  When the filter ``type`` is ``hugepage_size``,
+``min`` and ``max`` files act as files of the same names for :ref:`DAMOS
 filter <sysfs_filters>`.
 
 .. _sysfs_targets:
@@ -539,6 +553,10 @@ and ``damon_target_idx``.  To ``type`` file, you can write the type of the
 filter.  Refer to :ref:`the design doc <damon_design_damos_filters>` for
 available type names, their meaning and on what layer those are handled.
 
+For ``probe_hits_wsum`` type, you can specify the minimum and maximum probe
+hits weighted sum value for the filter to ``min`` and ``max`` files,
+respectively.
+
 For ``memcg`` type, you can specify the memory cgroup of the interest by
 writing the path of the memory cgroup from the cgroups mount point to
 ``memcg_path`` file.  For ``addr`` type, you can specify the start and end
diff --git a/Documentation/admin-guide/mm/kho.rst b/Documentation/admin-guide/mm/kho.rst
index 2c26e560bd78a3..902d171a780afc 100644
--- a/Documentation/admin-guide/mm/kho.rst
+++ b/Documentation/admin-guide/mm/kho.rst
@@ -20,10 +20,10 @@ need to enable if you would like to preserve their respective state across
 kexec.
 
 To use KHO, please boot the kernel with the ``kho=on`` command line
-parameter. You may use ``kho_scratch`` parameter to define size of the
-scratch regions. For example ``kho_scratch=16M,512M,256M`` will reserve a
-16 MiB low memory scratch area, a 512 MiB global scratch region, and 256 MiB
-per NUMA node scratch regions on boot.
+parameter. You may use ``kho_bootmem`` parameter to define size of the
+boot memory regions. For example ``kho_bootmem=16M,512M,256M`` will reserve a
+16 MiB low memory bootmem area, a 512 MiB global bootmem region, and 256 MiB
+per NUMA node bootmem regions on boot.
 
 Perform a KHO kexec
 ===================
@@ -96,15 +96,15 @@ stabilized.
     current KHO state in this file. Kexec user space tooling can use this
     as input file for the KHO payload image.
 
-``/sys/kernel/debug/kho/out/scratch_len``
-    Lengths of KHO scratch regions, which are physically contiguous
+``/sys/kernel/debug/kho/out/bootmem_len``
+    Lengths of KHO bootmem regions, which are physically contiguous
     memory regions that will always stay available for future kexec
     allocations. Kexec user space tools can use this file to determine
     where it should place its payload images.
 
-``/sys/kernel/debug/kho/out/scratch_phys``
-    Physical locations of KHO scratch regions. Kexec user space tools
-    can use this file in conjunction to scratch_phys to determine where
+``/sys/kernel/debug/kho/out/bootmem_phys``
+    Physical locations of KHO bootmem regions. Kexec user space tools
+    can use this file in conjunction to bootmem_len to determine where
     it should place its payload images.
 
 ``/sys/kernel/debug/kho/out/sub_fdts/``
diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst
index ad8e7a41f3b5d2..c329ca747b8c47 100644
--- a/Documentation/admin-guide/mm/ksm.rst
+++ b/Documentation/admin-guide/mm/ksm.rst
@@ -174,7 +174,7 @@ advisor_mode
         The section about ``advisor`` explains in detail how the scan time
         advisor works.
 
-adivsor_max_cpu
+advisor_max_cpu
         specifies the upper limit of the cpu percent usage of the ksmd
         background thread. The default is 70.
 
@@ -183,10 +183,10 @@ advisor_target_scan_time
         pages. The default value is 200 seconds.
 
 advisor_min_pages_to_scan
-        specifies the lower limit of the ``pages_to_scan`` parameter of the
-        scan time advisor. The default is 500.
+        specifies the initial value of the ``pages_to_scan`` parameter of
+        the scan time advisor. The default is 500.
 
-adivsor_max_pages_to_scan
+advisor_max_pages_to_scan
         specifies the upper limit of the ``pages_to_scan`` parameter of the
         scan time advisor. The default is 30000.
 
diff --git a/Documentation/admin-guide/nfs/pnfs-scsi-server.rst b/Documentation/admin-guide/nfs/pnfs-scsi-server.rst
index b202508d281d54..a3074450ac158f 100644
--- a/Documentation/admin-guide/nfs/pnfs-scsi-server.rst
+++ b/Documentation/admin-guide/nfs/pnfs-scsi-server.rst
@@ -16,7 +16,7 @@ addition to the MDS.  As of now the file system needs to sit directly on the
 exported LUN, striping or concatenation of LUNs on the MDS and clients
 is not supported yet.
 
-On a server built with CONFIG_NFSD_SCSI, the pNFS SCSI volume support is
+On a server built with CONFIG_NFSD_SCSILAYOUT, the pNFS SCSI volume support is
 automatically enabled if the file system is exported using the "pnfs"
 option and the underlying SCSI device support persistent reservations.
 On the client make sure the kernel has the CONFIG_PNFS_BLOCK option
diff --git a/Documentation/admin-guide/parport.rst b/Documentation/admin-guide/parport.rst
index ad3f9b8a11e19d..4aef45262bd49f 100644
--- a/Documentation/admin-guide/parport.rst
+++ b/Documentation/admin-guide/parport.rst
@@ -273,7 +273,7 @@ If that works fine, try with ``io=0x378 irq=7`` (adjust for your
 hardware), to make it use interrupt-driven in-software protocol.
 
 If **that** works fine, then one of the hardware modes isn't working
-right.  Enable ``CONFIG_FIFO`` (no, it isn't a module option,
+right.  Enable ``CONFIG_PARPORT_PC_FIFO`` (no, it isn't a module option,
 and yes, it should be), set the port to ECP mode in the BIOS and note
 the DMA channel, and try with::
 
diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst
index 6fd72ba52df20d..4c5d3d8acf645b 100644
--- a/Documentation/admin-guide/pm/amd-pstate.rst
+++ b/Documentation/admin-guide/pm/amd-pstate.rst
@@ -481,12 +481,6 @@ For systems that support ``amd-pstate`` preferred core, the core rankings will
 always be advertised by the platform. But OS can choose to ignore that via the
 kernel parameter ``amd_prefcore=disable``.
 
-``amd_dynamic_epp``
-
-When AMD pstate is in auto mode, dynamic EPP will control whether the kernel
-autonomously changes the EPP mode. The default is disabled. It can be enabled
-with the kernel parameter ``amd_dynamic_epp=enable``.
-
 User Space Interface in ``sysfs`` - General
 ===========================================
 
diff --git a/Documentation/admin-guide/pm/cpuidle.rst b/Documentation/admin-guide/pm/cpuidle.rst
index be4c1120e3f0b6..e91ebd432f6fd9 100644
--- a/Documentation/admin-guide/pm/cpuidle.rst
+++ b/Documentation/admin-guide/pm/cpuidle.rst
@@ -316,7 +316,7 @@ interval" value.
 
 Now, the governor is ready to walk the list of idle states and choose one of
 them.  For this purpose, it compares the target residency of each state with
-the predicted idle duration and the exit latency of it with the with the latency
+the predicted idle duration and the exit latency of it with the latency
 limit coming from the power management quality of service, or `PM QoS <cpu-pm-qos_>`_,
 framework.  It selects the state with the target residency closest to the predicted
 idle duration, but still below it, and exit latency that does not exceed the
diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index ffea61d448ebb1..608055d7ccc4bc 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -459,8 +459,9 @@ hung_task_warnings
 ==================
 
 The maximum number of warnings to report. During a check interval
-if a hung task is detected, this value is decreased by 1.
-When this value reaches 0, no more warnings will be reported.
+if a hung task is detected, the internal warning budget is decreased by 1.
+When this budget reaches 0, no more detailed warnings will be reported. The
+warning budget is reset to the configured limit when no hung task is found.
 This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled.
 
 -1: report an infinite number of warnings.
@@ -1334,19 +1335,6 @@ seccomp
 See Documentation/userspace-api/seccomp_filter.rst.
 
 
-sg-big-buff
-===========
-
-This file shows the size of the generic SCSI (sg) buffer.
-You can't tune it just yet, but you could change it on
-compile time by editing ``include/scsi/sg.h`` and changing
-the value of ``SG_BIG_BUFF``.
-
-There shouldn't be any reason to change this value. If
-you can come up with one, you probably know what you
-are doing anyway :)
-
-
 shmall
 ======
 
@@ -1609,8 +1597,10 @@ If a value outside of this range is written to ``threads-max`` an
 timer_migration
 ===============
 
-When set to a non-zero value, attempt to migrate timers away from idle cpus to
-allow them to remain in low power states longer.
+When set to a non-zero value, attempt to migrate high-resolution timers from
+nohz isolated (nohz_full) to housekeeping CPUs.
+See Documentation/admin-guide/cpu-isolation.rst
+and Documentation/admin-guide/kernel-parameters.rst.
 
 Default is set (1).
 
diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst
index e586e17fc7a582..fe43e85959583e 100644
--- a/Documentation/admin-guide/sysctl/net.rst
+++ b/Documentation/admin-guide/sysctl/net.rst
@@ -413,6 +413,20 @@ new netns has been created.
 
 Default : 0  (for compatibility reasons)
 
+neigh_inherit_init_net
+----------------------
+
+Controls if a new network namespace should inherit all current
+settings under /proc/sys/net/ipv{4,6}/neigh/default/.
+
+By default, the value is set to 1, and all settings are inherited
+from init_net.
+
+If set to 0, both IPv4 and IPv6 settings are reset to their default
+values.
+
+Default : 1  (for compatibility reasons)
+
 txrehash
 --------
 
diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst
index 5b318d17aa4b44..5f4a056e4dc222 100644
--- a/Documentation/admin-guide/sysctl/vm.rst
+++ b/Documentation/admin-guide/sysctl/vm.rst
@@ -438,7 +438,7 @@ zone[i]'s protection[j] is calculated by following expression::
     = (total sums of managed_pages from zone[i+1] to zone[j] on the node)
       / lowmem_reserve_ratio[i];
   (i = j):
-     (should not be protected. = 0;
+     (should not be protected. = 0)
   (i > j):
      (not necessary, but looks 0)
 
diff --git a/Documentation/admin-guide/tainted-kernels.rst b/Documentation/admin-guide/tainted-kernels.rst
index 9ead927a37c0f7..a208811ad52568 100644
--- a/Documentation/admin-guide/tainted-kernels.rst
+++ b/Documentation/admin-guide/tainted-kernels.rst
@@ -74,35 +74,36 @@ a particular type of taint. It's best to leave that to the aforementioned
 script, but if you need something quick you can use this shell command to check
 which bits are set::
 
-	$ for i in $(seq 20); do echo $(($i-1)) $(($(cat /proc/sys/kernel/tainted)>>($i-1)&1));done
+	$ for i in $(seq 0 20); do echo $i $(($(cat /proc/sys/kernel/tainted)>>$i&1));done
 
 Table for decoding tainted state
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-===  ===  ======  ========================================================
-Bit  Log  Number  Reason that got the kernel tainted
-===  ===  ======  ========================================================
-  0  G/P       1  proprietary module was loaded
-  1  _/F       2  module was force loaded
-  2  _/S       4  kernel running on an out of specification system
-  3  _/R       8  module was force unloaded
-  4  _/M      16  processor reported a Machine Check Exception (MCE)
-  5  _/B      32  bad page referenced or some unexpected page flags
-  6  _/U      64  taint requested by userspace application
-  7  _/D     128  kernel died recently, i.e. there was an OOPS or BUG
-  8  _/A     256  ACPI table overridden by user
-  9  _/W     512  kernel issued warning
- 10  _/C    1024  staging driver was loaded
- 11  _/I    2048  workaround for bug in platform firmware applied
- 12  _/O    4096  externally-built ("out-of-tree") module was loaded
- 13  _/E    8192  unsigned module was loaded
- 14  _/L   16384  soft lockup occurred
- 15  _/K   32768  kernel has been live patched
- 16  _/X   65536  auxiliary taint, defined for and used by distros
- 17  _/T  131072  kernel was built with the struct randomization plugin
- 18  _/N  262144  an in-kernel test has been run
- 19  _/J  524288  userspace used a mutating debug operation in fwctl
-===  ===  ======  ========================================================
+===  ===  =======  ========================================================
+Bit  Log  Number   Reason that got the kernel tainted
+===  ===  =======  ========================================================
+  0  G/P        1  proprietary module was loaded
+  1  _/F        2  module was force loaded
+  2  _/S        4  kernel running on an out of specification system
+  3  _/R        8  module was force unloaded
+  4  _/M       16  processor reported a Machine Check Exception (MCE)
+  5  _/B       32  bad page referenced or some unexpected page flags
+  6  _/U       64  taint requested by userspace application
+  7  _/D      128  kernel died recently, i.e. there was an OOPS or BUG
+  8  _/A      256  ACPI table overridden by user
+  9  _/W      512  kernel issued warning
+ 10  _/C     1024  staging driver was loaded
+ 11  _/I     2048  workaround for bug in platform firmware applied
+ 12  _/O     4096  externally-built ("out-of-tree") module was loaded
+ 13  _/E     8192  unsigned module was loaded
+ 14  _/L    16384  soft lockup occurred
+ 15  _/K    32768  kernel has been live patched
+ 16  _/X    65536  auxiliary taint, defined for and used by distros
+ 17  _/T   131072  kernel was built with the struct randomization plugin
+ 18  _/N   262144  an in-kernel test has been run
+ 19  _/J   524288  userspace used a mutating debug operation in fwctl
+ 20  _/Y  1048576  device was manually bound or unbound from a driver
+===  ===  =======  ========================================================
 
 Note: The character ``_`` is representing a blank in this table to make reading
 easier.
@@ -189,3 +190,6 @@ More detailed explanation for tainting
  19) ``J`` if userspace opened /dev/fwctl/* and performed a FWTCL_RPC_DEBUG_WRITE
      to use the devices debugging features. Device debugging features could
      cause the device to malfunction in undefined ways.
+
+ 20) ``Y`` If userspace wrote to the `bind` or `unbind` sysfs files and
+     successfully bound or removed a device from a driver.
diff --git a/Documentation/admin-guide/verify-bugs-and-bisect-regressions.rst b/Documentation/admin-guide/verify-bugs-and-bisect-regressions.rst
index 7d38393f31fb16..91ebb631a25418 100644
--- a/Documentation/admin-guide/verify-bugs-and-bisect-regressions.rst
+++ b/Documentation/admin-guide/verify-bugs-and-bisect-regressions.rst
@@ -163,7 +163,7 @@ will be considered the 'good' release and used to prepare the .config file.
      section 'Complementary tasks: cleanup during and after the process'
      below.
 
-  d) Once your finished the bisection, put a few things away::
+  d) Once you have finished the bisection, put a few things away::
 
        cd ~/linux/
        git bisect log > ~/bisect-log
@@ -178,9 +178,9 @@ will be considered the 'good' release and used to prepare the .config file.
        ./scripts/config --set-str CONFIG_LOCALVERSION '-local-cafec0cacaca0-reverted'
 
     This is optional, as some commits are impossible to revert. But if the
-    second command worked flawlessly, build, install, and boot one more kernel
-    kernel; just this time skip the first command copying the base .config file
-    over, as that already has been taken care off.
+    second command worked flawlessly, build, install, and boot one more kernel;
+    just this time skip the first command copying the base .config file over,
+    as that already has been taken care off.
 
 * **Complementary tasks**: cleanup during and after the process.
 
@@ -229,10 +229,10 @@ depends on your issue:
 Execute all steps till the end of *segment 1* to **verify if your kernel problem
 is present in code supported by Linux kernel developers**. If it is, you are all
 set to report the bug -- unless it did not happen with earlier kernel versions,
-as then your want to at least continue with *segment 2* to **check if the issue
+as then you want to at least continue with *segment 2* to **check if the issue
 qualifies as regression** which receive priority treatment. Depending on the
 outcome you then are ready to report a bug or submit a preliminary regression
-report; instead of the latter your could also head straight on and follow
+report; instead of the latter you could also head straight on and follow
 *segment 3* to **perform a bisection** for a full-fledged regression report
 developers are obliged to act upon.
 
@@ -550,7 +550,7 @@ be a waste of time. [:ref:`details <introlatestcheck_bisref>`]
       cd ~/linux/
       git switch --discard-changes --detach stable/linux-6.1.y
 
-    Your series is unsupported, if is not listed or carrying a 'end of life'
+    Your series is unsupported if it is not listed or it is carrying a 'end of life'
     tag. In that case you might want to check if a successor series (say
     linux-6.2.y) or mainline (see next point) fix the bug.
 
@@ -1183,9 +1183,9 @@ Space requirements
 The numbers mentioned are rough estimates with a big extra charge to be on the
 safe side, so often you will need less.
 
-If you have space constraints, be sure to hay attention to the :ref:`step about
-debug symbols' <debugsymbols_bissbs>` and its :ref:`accompanying reference
-section' <debugsymbols_bisref>`, as disabling then will reduce the consumed disk
+If you have space constraints, be sure to pay attention to the :ref:`step about
+debug symbols <debugsymbols_bissbs>` and its :ref:`accompanying reference
+section <debugsymbols_bisref>`, as disabling them will reduce the consumed disk
 space by quite a few gigabytes.
 
 [:ref:`back to step-by-step guide <diskspace_bissbs>`]
@@ -1254,8 +1254,8 @@ distributions:
       kernel-install-tools libelf-devel make modutils openssl openssl-devel \
       perl-base zlib-devel rpm-build ncurses-devel qt6-base-devel
 
-These commands install a few packages that are often, but not always needed. You
-for example might want to skip installing the development headers for ncurses,
+These commands install a few packages that are often, but not always needed. For
+example, you might want to skip installing the development headers for ncurses,
 which you will only need in case you later might want to adjust the kernel build
 configuration using make the targets 'menuconfig' or 'nconfig'; likewise omit
 the headers of Qt6 if you do not plan to adjust the .config using 'xconfig'.
@@ -1407,7 +1407,7 @@ Occasionally odd things happen when trying to use a config file prepared for one
 kernel (say 6.1) on an older mainline release -- especially if it is much older
 (say 5.15). That's one of the reasons why the previous step in the guide told
 you to boot the kernel where everything works. If you manually add a .config
-file you thus want to ensure it's from the working kernel and not from a one
+file you thus want to ensure it's from the working kernel and not from one
 that shows the regression.
 
 In case you want to build kernels for another machine, locate its kernel build
diff --git a/Documentation/arch/arm64/booting.rst b/Documentation/arch/arm64/booting.rst
index 13ef311dace83f..dae5ff9fae2653 100644
--- a/Documentation/arch/arm64/booting.rst
+++ b/Documentation/arch/arm64/booting.rst
@@ -465,6 +465,7 @@ Before jumping into the kernel, the following conditions must be met:
     - HDFGWTR2_EL2.nPMICNTR_EL0 (bit 2) must be initialised to 0b1.
     - HDFGWTR2_EL2.nPMICFILTR_EL0 (bit 3) must be initialised to 0b1.
     - HDFGWTR2_EL2.nPMUACR_EL1 (bit 4) must be initialised to 0b1.
+    - HDFGWTR2_EL2.nPMZR_EL0 (bit 21) must be initialised to 0b1.
 
   For CPUs with SPE data source filtering (FEAT_SPE_FDS):
 
@@ -588,7 +589,7 @@ following manner:
 
 - CPUs with a "spin-table" enable-method must have a 'cpu-release-addr'
   property in their cpu node.  This property identifies a
-  naturally-aligned 64-bit zero-initalised memory location.
+  naturally-aligned 64-bit zero-initialised memory location.
 
   These CPUs should spin outside of the kernel in a reserved area of
   memory (communicated to the kernel by a /memreserve/ region in the
diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst
index ac3248b9f2f3bb..25afa79266a470 100644
--- a/Documentation/arch/arm64/silicon-errata.rst
+++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -53,6 +53,9 @@ stable kernels.
 | Allwinner      | A64/R18         | UNKNOWN1        | SUN50I_ERRATUM_UNKNOWN1     |
 +----------------+-----------------+-----------------+-----------------------------+
 +----------------+-----------------+-----------------+-----------------------------+
+| Altera         | SoCFPGA Agilex5 | 2.1.23          |ALTERA_ERRATUM_AGILEX5_2_1_23|
++----------------+-----------------+-----------------+-----------------------------+
++----------------+-----------------+-----------------+-----------------------------+
 | Ampere         | AmpereOne       | AC03_CPU_38     | AMPERE_ERRATUM_AC03_CPU_38  |
 +----------------+-----------------+-----------------+-----------------------------+
 | Ampere         | AmpereOne       | AC03_CPU_57     | N/A                         |
diff --git a/Documentation/arch/powerpc/vas-api.rst b/Documentation/arch/powerpc/vas-api.rst
index 1d0d055356e3cc..f74ea9fbc8a093 100644
--- a/Documentation/arch/powerpc/vas-api.rst
+++ b/Documentation/arch/powerpc/vas-api.rst
@@ -136,7 +136,7 @@ a connection with NX co-processor engine:
 	follows::
 
 		#define VAS_MAGIC 'v'
-		#define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 1,
+		#define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 0x20,
 						struct vas_tx_win_open_attr)
 
 		struct vas_tx_win_open_attr attr;
@@ -265,7 +265,7 @@ Simple example
 		{
 			int rc, fd;
 			void *addr;
-			struct vas_setup_attr txattr;
+			struct vas_tx_win_open_attr txattr;
 
 			fd = open("/dev/crypto/nx-gzip", O_RDWR);
 			if (fd < 0) {
diff --git a/Documentation/arch/powerpc/vmemmap_dedup.rst b/Documentation/arch/powerpc/vmemmap_dedup.rst
index dc4db59fdf87b4..8286acbca9bc57 100644
--- a/Documentation/arch/powerpc/vmemmap_dedup.rst
+++ b/Documentation/arch/powerpc/vmemmap_dedup.rst
@@ -19,82 +19,28 @@ With 1G PUD level mapping, we require 16384 struct pages and a single 64K
 vmemmap page can contain 1024 struct pages (64K/sizeof(struct page)). Hence we
 require 16 64K pages in vmemmap to map the struct page for 1G PUD level mapping.
 
-Here's how things look like on device-dax after the sections are populated::
- +-----------+ ---virt_to_page---> +-----------+   mapping to   +-----------+
- |           |                     |     0     | -------------> |     0     |
- |           |                     +-----------+                +-----------+
- |           |                     |     1     | -------------> |     1     |
- |           |                     +-----------+                +-----------+
- |           |                     |     2     | ----------------^ ^ ^ ^ ^ ^
- |           |                     +-----------+                   | | | | |
- |           |                     |     3     | ------------------+ | | | |
- |           |                     +-----------+                     | | | |
- |           |                     |     4     | --------------------+ | | |
- |    PUD    |                     +-----------+                       | | |
- |   level   |                     |     .     | ----------------------+ | |
- |  mapping  |                     +-----------+                         | |
- |           |                     |     .     | ------------------------+ |
- |           |                     +-----------+                           |
- |           |                     |     15    | --------------------------+
- |           |                     +-----------+
- |           |
- |           |
- |           |
- +-----------+
-
-
 With 4K page size, 2M PMD level mapping requires 512 struct pages and a single
 4K vmemmap page contains 64 struct pages(4K/sizeof(struct page)). Hence we
 require 8 4K pages in vmemmap to map the struct page for 2M pmd level mapping.
 
-Here's how things look like on device-dax after the sections are populated::
-
- +-----------+ ---virt_to_page---> +-----------+   mapping to   +-----------+
- |           |                     |     0     | -------------> |     0     |
- |           |                     +-----------+                +-----------+
- |           |                     |     1     | -------------> |     1     |
- |           |                     +-----------+                +-----------+
- |           |                     |     2     | ----------------^ ^ ^ ^ ^ ^
- |           |                     +-----------+                   | | | | |
- |           |                     |     3     | ------------------+ | | | |
- |           |                     +-----------+                     | | | |
- |           |                     |     4     | --------------------+ | | |
- |    PMD    |                     +-----------+                       | | |
- |   level   |                     |     5     | ----------------------+ | |
- |  mapping  |                     +-----------+                         | |
- |           |                     |     6     | ------------------------+ |
- |           |                     +-----------+                           |
- |           |                     |     7     | --------------------------+
- |           |                     +-----------+
- |           |
- |           |
- |           |
- +-----------+
-
-With 1G PUD level mapping, we require 262144 struct pages and a single 4K
-vmemmap page can contain 64 struct pages (4K/sizeof(struct page)). Hence we
-require 4096 4K pages in vmemmap to map the struct pages for 1G PUD level
-mapping.
-
-Here's how things look like on device-dax after the sections are populated::
-
- +-----------+ ---virt_to_page---> +-----------+   mapping to   +-----------+
- |           |                     |     0     | -------------> |     0     |
- |           |                     +-----------+                +-----------+
- |           |                     |     1     | -------------> |     1     |
- |           |                     +-----------+                +-----------+
- |           |                     |     2     | ----------------^ ^ ^ ^ ^ ^
- |           |                     +-----------+                   | | | | |
- |           |                     |     3     | ------------------+ | | | |
- |           |                     +-----------+                     | | | |
- |           |                     |     4     | --------------------+ | | |
- |    PUD    |                     +-----------+                       | | |
- |   level   |                     |     .     | ----------------------+ | |
- |  mapping  |                     +-----------+                         | |
- |           |                     |     .     | ------------------------+ |
- |           |                     +-----------+                           |
- |           |                     |   4095    | --------------------------+
- |           |                     +-----------+
+Here's how things look on device-dax after vmemmap-optimized sections are
+populated. ``N`` is the number of vmemmap pages required by the DAX mapping
+above::
+
+  Device DAX                  vmemmap pages (N pages)         backing page frames
+ +-----------+ ---virt_to_page---> +-----------+   mapping to   +-------------+
+ |           |                     |     0     | -------------> |      0      |
+ |           |                     +-----------+                +-------------+
+ |           |                     |     1     | ------+
+ |           |                     +-----------+       |
+ |           |                     |     2     | ------+
+ |           |                     +-----------+       |
+ |           |                     |     .     | ------+        +-------------+
+ |  PMD/PUD  |                     +-----------+       |        | A single,   |
+ |   level   |                     |     .     | ------+------> | per-zone    |
+ |  mapping  |                     +-----------+       |        | shared tail |
+ |           |                     |   N - 1   | ------+        | page        |
+ |           |                     +-----------+                +-------------+
  |           |
  |           |
  |           |
diff --git a/Documentation/arch/s390/pci.rst b/Documentation/arch/s390/pci.rst
index 80f4ba19315994..565434626fb5ca 100644
--- a/Documentation/arch/s390/pci.rst
+++ b/Documentation/arch/s390/pci.rst
@@ -67,7 +67,7 @@ Entries specific to zPCI functions and entries that hold zPCI information.
 
   A physical function that currently supports a virtual function cannot be
   powered off until all virtual functions are removed with:
-  echo 0 > /sys/bus/pci/devices/DDDD:BB:dd.f/sriov_numvf
+  echo 0 > /sys/bus/pci/devices/DDDD:BB:dd.f/sriov_numvfs
 
 * /sys/bus/pci/devices/DDDD:BB:dd.f/:
 
diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst
index 3303499ad4c6f8..a36ea2bd4301d5 100644
--- a/Documentation/arch/x86/tdx.rst
+++ b/Documentation/arch/x86/tdx.rst
@@ -200,6 +200,27 @@ reflects the TCB of the currently running TDX module and therefore
 changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD
 launch time and is not affected.
 
+Dynamic PAMT
+------------
+
+The Physical Address Metadata Table (PAMT) is metadata in which the TDX
+module keeps data about each physical page (think struct page). Space
+for it is allocated by the VMM, consumes up to about 0.4% of system
+memory and needs to be supplied to the TDX module when the TDX module is
+first loaded.
+
+Dynamic PAMT is an add-on feature that allows a VMM to dynamically
+allocate the part of the PAMT which tracks 4KB pages. This reduces the
+amount of memory that TDX consumes while TDs are not in use.
+
+When Dynamic PAMT is in use, dmesg shows it like::
+
+  [..] virt/tdx: Enable Dynamic PAMT
+  [..] virt/tdx: 10092 KB allocated for PAMT
+  [..] virt/tdx: TDX-Module initialized
+
+Dynamic PAMT is enabled automatically if supported.
+
 TDX Interaction to Other Kernel Components
 ------------------------------------------
 
diff --git a/Documentation/arch/x86/xstate.rst b/Documentation/arch/x86/xstate.rst
index cec05ac464c121..e2944f744255ba 100644
--- a/Documentation/arch/x86/xstate.rst
+++ b/Documentation/arch/x86/xstate.rst
@@ -172,3 +172,57 @@ are extended to control the guest permission:
 
 Note that some VMMs may have already established a set of supported state
 components. These options are not presumed to support any particular VMM.
+
+Signal Frame Layout and Portability
+-----------------------------------
+
+The signal frame is designed to be self-describing and portable. This is
+especially important for checkpoint/restore tools like CRIU, which may restore
+a process on a different host than where it was checkpointed. A signal frame
+created on a machine with fewer CPU features can be successfully restored on a
+machine with more CPU features, but not vice-versa.
+
+Signal Frame Software Reserved Bytes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+On CPUs supporting XSAVE, bytes 464..511 in the 512-byte FXSAVE/FXRSTOR frame
+are reserved for software use and contain ``struct _fpx_sw_bytes`` (defined in
+``<uapi/asm/sigcontext.h>``)::
+
+    struct _fpx_sw_bytes {
+        __u32 magic1;
+        __u32 extended_size;
+        __u64 xfeatures;
+        __u32 xstate_size;
+        __u32 padding[7];
+    };
+
+- ``magic1``: Set to ``FP_XSTATE_MAGIC1`` (``0x46505853U``) if an extended
+  xstate context is present; 0 for a legacy frame.
+- ``extended_size``: The total size allocated on the stack for the frame,
+  measured from the ``fpstate`` pointer. In 32-bit signal frames, this also
+  includes the 112-byte legacy FPU state prefix of ``struct _fpstate_32``.
+- ``xfeatures``: The mask of xstate features saved in the frame.
+- ``xstate_size``: The actual size of the xstate context for the enabled
+  features (including the 512-byte FXSAVE area and the 64-byte XSAVE header).
+
+The kernel uses ``xstate_size`` in conjunction with the pointer to the xstate
+context to locate the ``FP_XSTATE_MAGIC2`` (``0x46505845U``) marker right after
+the xstate context (at ``xstate_context + xstate_size``). In 64-bit signal frames,
+the ``fpstate`` pointer points directly to the xstate context. In 32-bit signal
+frames (including 32-bit compat tasks on 64-bit kernels), the ``fpstate``
+pointer points to ``struct _fpstate_32``, which contains the 112-byte legacy
+FPU state followed by the 512-byte FXSR state (and any extended xstate). Since
+there is no standalone UAPI structure defined for just the 112-byte legacy
+state, the xstate context starts at ``fpstate + 112`` (and ``extended_size``
+spans the entire allocation from ``fpstate``).
+
+Portability Constraints
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Signal frame portability is constrained by the architectural XSAVE layout.
+Restoration is supported only if the destination host supports all features
+present in the frame and uses matching component offsets and sizes for them.
+While layout compatibility is generally maintained across CPUs from the same
+vendor, differences can occur across vendors or if the XSAVE space of a
+deprecated feature (e.g. MPX) is repurposed for a newer feature (e.g. APX).
diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst
index 49da2147b7d86b..6717bfacd13826 100644
--- a/Documentation/block/biovecs.rst
+++ b/Documentation/block/biovecs.rst
@@ -135,6 +135,7 @@ Usage of helpers:
 	bio_first_bvec_all()
 	bio_first_page_all()
 	bio_first_folio_all()
+	bio_last_bvec_all()
 
 * The following helpers iterate over single-page segment. The passed 'struct
   bio_vec' will contain a single-page IO vector during the iteration::
diff --git a/Documentation/bpf/bpf_design_QA.rst b/Documentation/bpf/bpf_design_QA.rst
index eb19c945f4d578..98d70b8a2533b7 100644
--- a/Documentation/bpf/bpf_design_QA.rst
+++ b/Documentation/bpf/bpf_design_QA.rst
@@ -221,9 +221,14 @@ newer kernels. BPF programs need to change accordingly when this happens.
 
 Q: How much stack space a BPF program uses?
 -------------------------------------------
-A: Currently all program types are limited to 512 bytes of stack
-space, but the verifier computes the actual amount of stack used
-and both interpreter and most JITed code consume necessary amount.
+A: A program may use up to 2 KiB of stack, combined over its call
+chain, when the JIT of the architecture reports support for large
+stacks (currently x86-64 and arm64); a single function may use all of
+it, and every frame of a program running on a private stack gets the
+whole amount. Elsewhere, and for programs verified for the interpreter,
+the limit is 512 bytes; the interpreter itself never runs a frame
+larger than that. The verifier computes the actual amount of stack
+used and both interpreter and most JITed code consume necessary amount.
 
 Q: Can BPF be offloaded to HW?
 ------------------------------
diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst
index 3f05f17990ade7..29de1222c3e73f 100644
--- a/Documentation/bpf/btf.rst
+++ b/Documentation/bpf/btf.rst
@@ -88,6 +88,9 @@ sequentially and type id is assigned to each recognized type starting from id
     #define BTF_KIND_DECL_TAG       17      /* Decl Tag     */
     #define BTF_KIND_TYPE_TAG       18      /* Type Tag     */
     #define BTF_KIND_ENUM64         19      /* Enumeration up to 64-bit values */
+    #define BTF_KIND_LOC_PARAM      20      /* Location description (register, const etc) */
+    #define BTF_KIND_LOC_PROTO      21      /* Set of location parameters for site */
+    #define BTF_KIND_LOCSEC         22      /* Section with site descriptions */
 
 Note that the type section encodes debug info, not just pure types.
 ``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram.
@@ -104,11 +107,13 @@ Each type contains the following common data::
          *             decl_tag and type_tag
          */
         __u32 info;
-        /* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64.
+        /* "size" is used by INT, ENUM, STRUCT, UNION, ENUM64 and
+         * LOC_PARAM.
          * "size" tells the size of the type it is describing.
          *
          * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
-         * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG.
+         * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. It is unused by
+         * LOC_PROTO and LOCSEC.
          * "type" is a type_id referring to another type.
          */
         union {
@@ -416,7 +421,7 @@ If the function has variable arguments, the last parameter is encoded with
   * ``info.vlen``: 0
   * ``type``: the type of the variable
 
-``btf_type`` is followed by a single ``struct btf_variable`` with the
+``btf_type`` is followed by a single ``struct btf_var`` with the
 following data::
 
     struct btf_var {
@@ -563,6 +568,80 @@ The ``btf_enum64`` encoding:
 If the original enum value is signed and the size is less than 8,
 that value will be sign extended into 8 bytes.
 
+2.2.20 BTF_KIND_LOC_PARAM
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``struct btf_type`` encoding requirement:
+  * ``name_off``: 0
+  * ``info.kind_flag``: 0
+  * ``info.kind``: BTF_KIND_LOC_PARAM
+  * ``info.vlen``: number of 32-bit location value words
+  * ``size``: size in bytes of the represented parameter: 1 through 16
+
+``btf_type`` is followed by a ``struct btf_loc_param`` and ``info.vlen``
+number of 32-bit value words.::
+
+    struct btf_loc_param {
+        __u32 flags;
+        __u32 values[];
+    };
+
+The ``flags`` field describes how to interpret ``values``:
+
+  * ``BTF_LOC_PARAM_CONST`` describes a constant; the value is stored in
+    low-word, high-word order when it requires 64 bits.
+  * ``BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_CONST`` describes an address offset
+    relative to the runtime base address of the kernel or module image.
+  * ``BTF_LOC_PARAM_REG`` with one word describes a register number; with two
+    words it describes a multi-register parameter.
+  * ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET`` describes an address held in
+    a register plus an offset. Adding ``BTF_LOC_PARAM_DEREF`` dereferences
+    that address. ``BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF`` with one word
+    dereferences the value held in the register.
+  * ``BTF_LOC_PARAM_SIGNED`` makes a constant or offset signed. A constant's
+    signed width is ``size``. For a register-relative offset, its signed
+    width is the number of offset value words times 32 bits.
+
+2.2.21 BTF_KIND_LOC_PROTO
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``struct btf_type`` encoding requirement:
+  * ``name_off``: 0
+  * ``info.kind_flag``: 0
+  * ``info.kind``: BTF_KIND_LOC_PROTO
+  * ``info.vlen``: number of function parameter locations
+  * ``type``: 0
+
+``btf_type`` is followed by ``info.vlen`` number of ``__u32`` BTF type IDs.
+Each entry corresponds to a function parameter at an inline site. An entry is
+either 0, meaning that no location information is available, or the type ID
+of a ``BTF_KIND_LOC_PARAM``.
+
+2.2.22 BTF_KIND_LOCSEC
+~~~~~~~~~~~~~~~~~~~~~~
+
+``struct btf_type`` encoding requirement:
+  * ``name_off``: offset to a valid ELF section name
+  * ``info.kind_flag``: 0
+  * ``info.kind``: BTF_KIND_LOCSEC
+  * ``info.vlen``: number of inline sites in the section
+  * ``type``: 0
+
+``btf_type`` is followed by ``info.vlen`` number of ``struct btf_loc``.::
+
+    struct btf_loc {
+        __u32 func;
+        __u32 loc_proto;
+        __u32 offset;
+    };
+
+The ``func`` field is the non-zero type ID of the ``BTF_KIND_FUNC`` being
+described. ``loc_proto`` is the non-zero type ID of the associated
+``BTF_KIND_LOC_PROTO``. ``offset`` is the location address offset relative to
+the runtime base address of the ELF section associated with the LOCSEC.
+For example, a LOCSEC named ``inline.text`` contains records for ``.text``
+whose offsets are relative to the runtime base of that section.
+
 2.3 Constant Values
 -------------------
 
diff --git a/Documentation/bpf/clang-notes.rst b/Documentation/bpf/clang-notes.rst
index 2c872a1ee08e94..3ccc7b09d19e62 100644
--- a/Documentation/bpf/clang-notes.rst
+++ b/Documentation/bpf/clang-notes.rst
@@ -23,8 +23,11 @@ For CPU versions prior to 3, Clang v7.0 and later can enable ``BPF_ALU`` support
 Jump instructions
 =================
 
-If ``-O0`` is used, Clang will generate the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d)
-instruction, which is not supported by the Linux kernel verifier.
+Clang generates the ``BPF_CALL | BPF_X | BPF_JMP`` (0x8d) instruction for calls
+through a function pointer. The Linux kernel verifier accepts it only when it
+can prove that the register holds the address of a static BPF function, see
+Documentation/bpf/linux-notes.rst. If ``-O0`` is used, Clang will generate this
+instruction for helper calls as well, which is not supported.
 
 Atomic operations
 =================
diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst
index 85f73e0bbd0ff7..73578a4c3b1fbc 100644
--- a/Documentation/bpf/kfuncs.rst
+++ b/Documentation/bpf/kfuncs.rst
@@ -164,19 +164,32 @@ suffix should be used.
 2.3.3 __uninit Annotation
 -------------------------
 
-This annotation is used to indicate that the argument will be treated as
-uninitialized.
+Use ``__uninit`` on a pointer parameter for an output buffer whose incoming
+contents the kfunc does not read.
 
-An example is given below::
+For generic memory buffers, the kfunc may leave bytes untouched, including
+on error returns. The writable range is determined by the pointed-to type
+or the associated ``__sz`` or ``__szk`` size argument.
+
+The annotation does not change the accepted pointer types. A stack-backed
+struct passed as a generic memory buffer must still be scalar-only.
+
+Generic stack buffers may be uninitialized before the call. The call does
+not make previously uninitialized bytes readable unless the program is
+allowed to read uninitialized stack memory (normally requiring
+``CAP_PERFMON``). Other callers must initialize those bytes themselves
+before reading them. Stack bounds and variable-offset restrictions still
+apply.
 
-        __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr_kern *ptr__uninit)
+For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a
+dynptr in the supplied storage. For example::
+
+        __bpf_kfunc int bpf_dynptr_from_skb(..., struct bpf_dynptr *ptr__uninit)
         {
         ...
         }
 
-Here, the dynptr will be treated as an uninitialized dynptr. Without this
-annotation, the verifier will reject the program if the dynptr passed in is
-not initialized.
+Without this annotation, a dynptr argument must already be initialized.
 
 2.3.4 __nullable Annotation
 ---------------------------
@@ -315,6 +328,29 @@ However, there is no obligation to prove to the verifier that such a pointer is
 non-NULL before use, in-line with existing semantics of arena pointers used in
 a program (or obtained from any other source).
 
+2.3.9 __ctx_out Annotation
+--------------------------
+
+This annotation is used to indicate that the argument is an output
+parameter of the attached hook, passed through from the program's
+context. The verifier requires the register to be a trusted read-only
+pointer to fixed-size memory, which can only be produced by loading an
+argument described by the program's ctx_arg_info from the context. The
+program itself cannot write through the pointer; the kfunc may.
+
+An example is given below::
+
+        __bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs,
+                                             int *xattr_count__ctx_out,
+                                             ...)
+        {
+        ...
+        }
+
+In this case, a program attached to the ``inode_init_security`` LSM hook
+can pass the hook's own ``xattr_count`` argument through to the kfunc,
+which claims xattr slots by writing through it.
+
 .. _BPF_kfunc_nodef:
 
 2.4 Using an existing kernel function
@@ -486,6 +522,16 @@ Example usage in BPF program:
 	/* note that the last argument is omitted */
         bpf_task_work_schedule_signal(task, &work->tw, &arrmap, task_work_callback);
 
+2.5.10 KF_PERFMON flag
+----------------------
+
+The KF_PERFMON flag is used for kfuncs that can expose kernel memory or kernel
+addresses to the BPF program, for example by reading through a pointer that the
+verifier does not check. Calling such a kfunc requires CAP_PERFMON, or
+CAP_SYS_ADMIN, in the same way that the equivalent BPF helpers are gated in
+bpf_base_func_proto(). A program loaded with CAP_BPF alone is rejected at load
+time.
+
 2.6 Registering the kfuncs
 --------------------------
 
@@ -536,7 +582,7 @@ For example, for the following type definition:
 
 The verifier would allow a ``struct bpf_cpumask *`` to be passed to a kfunc
 taking a ``cpumask_t *`` (which is a typedef of ``struct cpumask *``). For
-instance, both ``struct cpumask *`` and ``struct bpf_cpmuask *`` can be passed
+instance, both ``struct cpumask *`` and ``struct bpf_cpumask *`` can be passed
 to bpf_cpumask_test_cpu().
 
 In some cases, this type-aliasing behavior is not desired. ``struct
@@ -575,6 +621,80 @@ is also covered by this recovery. A kfunc handed an arena pointer may
 therefore access up to ``GUARD_SZ / 2`` past it without bounds-checking
 against the arena. Larger accesses must verify the range explicitly.
 
+2.9 kfunc Return Values
+-----------------------
+
+A kfunc may return a scalar, a pointer, or a small struct or union by
+value. A scalar or pointer of up to 8 bytes is returned in R0, as usual.
+
+A struct or union returned by value must be composed only of scalars and arena
+pointers, where a scalar is an integer or an enum and an arena pointer is one
+carrying the ``btf_type_tag("arena")`` attribute. Those may be nested in
+structs and unions and in arrays of any number of dimensions, in any
+combination, as long as what the nesting bottoms out in is a scalar or an arena
+pointer. Its bytes are handed back to the program as the raw contents of R0
+(and R2), so a member of any other pointer type would be laundered into a
+scalar and escape the verifier's pointer provenance and reference tracking. A
+struct or union with such a member is therefore rejected at load time, and so
+is one with a floating-point member, which the ABI may not return in R0:R2 at
+all.
+
+An arena pointer member is handed back as a scalar too, but nothing is lost by
+that. The program must ``cast_kern()`` the value before it can be used, and the
+verifier allows that cast on any scalar, so a laundered arena address gives the
+program no reach it did not already have. The result is confined to the
+program's arena in either case.
+
+A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a
+struct or union of scalars and arena pointers, or an ``__int128``. Such a
+value is returned in the register pair R0:R2, matching the convention LLVM
+uses for the BPF target: the first 8 bytes in R0 and the second 8 bytes in R2.
+A struct or union of 8 bytes or less is returned in R0 alone.
+
+::
+
+        struct bpf_pair { __u64 a, b; };   /* 16 bytes */
+
+        __bpf_kfunc struct bpf_pair bpf_kfunc_get_pair(void)
+        {
+                struct bpf_pair p = { .a = 1, .b = 2 };
+
+                return p;      /* p.a in R0, p.b in R2 */
+        }
+
+Returning a value in the R0:R2 pair requires the JIT to place the second
+half of the return value into R2, which not every architecture supports
+right now. A kfunc with a return value larger than 8 bytes is therefore
+rejected at load time on a JIT that does not advertise this capability (see
+``bpf_jit_supports_kfunc_ret_reg_pair()``), and such a program is never run
+by the interpreter. A return value larger than 16 bytes is not supported.
+
+The same R0:R2 convention applies to a BPF subprogram, global or static, that
+returns an ``__int128`` or a struct or union larger than 8 bytes. It is only
+used when the program is JITed, since the interpreter propagates only R0 out of
+a subprogram: without a JIT the return value stays in R0 alone, and a caller
+reading R2 is rejected for reading an uninitialized register. A global
+subprogram is verified in isolation, so its by-value struct or union return is
+restricted to scalars and arena pointers just like a kfunc's; a static
+subprogram is verified inline and has no such restriction. The main program is
+not covered: its return value is the program's exit code, read out of R0
+alone, so a declared upper half is never looked at.
+
+A global subprogram must leave a scalar in *every* register of the pair, so
+both halves of the returned value have to be assigned. Leaving the upper half
+uninitialized is not merely untidy: the compiler is then free to leave R2
+holding whatever it happened to hold, which for a subprogram taking a pointer
+argument is typically that pointer. Handing the caller an unknown scalar built
+from a pointer is a leak, so the verifier rejects it with::
+
+        At subprogram exit the register R2 is not a scalar value (...)
+
+Initialize the whole return value, for example ``struct pair p = {};``, to
+avoid this. A static subprogram is exempt from the scalar-only rule: it is
+verified inline, so an unassigned R2 is simply passed back to the caller as
+uninitialized and only a caller that reads it fails. A stack pointer left in
+R2 is still rejected there, just as one in R0 is.
+
 .. _BPF_kfunc_lifecycle_expectations:
 
 3. kfunc lifecycle expectations
diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/linux-notes.rst
index 00d2693de025a8..6c036b54a29f2a 100644
--- a/Documentation/bpf/linux-notes.rst
+++ b/Documentation/bpf/linux-notes.rst
@@ -15,10 +15,33 @@ Byte swap instructions
 Jump instructions
 =================
 
-``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), where the helper function
-integer would be read from a specified register, is not currently supported
-by the verifier.  Any programs with this instruction will fail to load
-until such support is added.
+``BPF_CALL | BPF_X | BPF_JMP`` (0x8d), ``callx dst``, performs an indirect
+call of a BPF function whose address is held in the ``dst`` register. The
+``src``, ``offset`` and ``imm`` fields are reserved and must be zero.
+
+The address of a BPF function gets into a register in one of two ways:
+
+* it is loaded by a 64-bit immediate instruction with ``src`` =
+  ``BPF_PSEUDO_FUNC``;
+* it is read, with a 64-bit load, from a frozen read-only array map, that no
+  other program uses, that holds its read-only data: tables of functions,
+  structures of operations, vtables, where pointers to functions may be mixed
+  with other data. In the map a pointer to a function is the offset in bytes
+  of its first instruction in the program, and that is how the verifier
+  recognizes it. It is replaced with the address of the function when
+  the program is loaded. The program reads it from there, which requires
+  ``CAP_PERFMON``.
+
+In both cases only static functions can be referenced. Therefore all functions
+that can be called indirectly are known to the verifier before it starts to
+analyze the program, and ``callx`` is verified as a direct call of every
+function that ``dst`` may point to at that instruction. The same rules apply:
+the calls can not be recursive, and the depth of the call chain and its
+combined stack size are limited.
+
+Calling helper or kernel functions through a register, indirect calls of global
+functions, and tail calls in functions that are called via ``callx`` are not
+supported. ``callx`` requires the BPF JIT.
 
 Maps
 ====
diff --git a/Documentation/bpf/signing.rst b/Documentation/bpf/signing.rst
index e73eaaebd8b157..27fdad3df96f1f 100644
--- a/Documentation/bpf/signing.rst
+++ b/Documentation/bpf/signing.rst
@@ -254,21 +254,24 @@ returned. Only after the program has fully loaded, at the next hook
 (``security_bpf_prog()``), does ``BPF_SIG_VERIFIED`` carry its full meaning:
 validly signed *and* fully verified.
 
-A more realistic admission policy than "is it signed at all": accept programs
-signed by a system keyring, accept a user-keyring signature only if the
-key/keyring it was verified against is on an explicit allowlist, and emit a
-tamper-evident record of every decision so that even denied attempts are
-auditable. (Illustrative - error checking elided.)
+A more realistic admission policy than "is it signed at all": base trust in
+the bpf keyring or an explicit allowlist of a caller-supplied staging key/
+keyring, and emit a record of every decision including denied attempts.
+(Illustrative - error checking elided.)
 
 .. code-block:: c
 
-    /* Serials of user keys/keyrings we additionally trust. */
+    /*
+     * Serials of caller-supplied keyrings we are willing to stage. Empty
+     * on a system that has committed to the bpf keyring, where the kernel
+     * refuses them anyway.
+     */
     struct {
             __uint(type, BPF_MAP_TYPE_HASH);
             __type(key, __s32);             /* keyring_serial */
             __type(value, __u8);
             __uint(max_entries, 64);
-    } trusted_user_keys SEC(".maps");
+    } staging_keys SEC(".maps");
 
     /* Audit stream consumed by a userspace logger. */
     struct {
@@ -291,11 +294,18 @@ auditable. (Illustrative - error checking elided.)
             if (kernel)
                     return 0;                       /* trust in-kernel loads */
 
-            if (verdict != BPF_SIG_VERIFIED)
+            if (verdict != BPF_SIG_VERIFIED) {
                     ret = -EPERM;                   /* must be validly signed */
-            else if (ktype == BPF_SIG_KEYRING_USER &&
-                     !bpf_map_lookup_elem(&trusted_user_keys, &serial))
-                    ret = -EPERM;                   /* key/keyring not allowlisted */
+            } else switch (ktype) {
+            case BPF_SIG_KEYRING_BPF:
+                    break;
+            case BPF_SIG_KEYRING_USER:
+                    if (!bpf_map_lookup_elem(&staging_keys, &serial))
+                            ret = -EPERM;
+                    break;
+            default:
+                    ret = -EPERM;                   /* keyring not in policy */
+            }
 
             d = bpf_ringbuf_reserve(&audit, sizeof(*d), 0);
             if (d) {
@@ -309,6 +319,11 @@ auditable. (Illustrative - error checking elided.)
             return ret;
     }
 
+Such a policy is what makes a caller-supplied keyring usable at all on a system
+that does not boot with ``bpf.keyring_unsealed=1``: the allowlist bounds which
+staged keys count, and the LSM itself has to protect them from being tampered
+with.
+
 Observing a verified load: ``security_bpf_prog()``
 --------------------------------------------------
 
@@ -381,8 +396,9 @@ that verdict covered all of its exclusive maps, rejecting any that did not - so
 a deny-by-default admission policy needs no second enforcement point. Use
 ``security_bpf_prog()`` to record or finally gate the verified programs once
 they carry an id. The ``verdict``, ``keyring_type`` and ``keyring_serial`` fields
-let a policy distinguish, for example, "verified and signed by a builtin key"
-from "verified by a user key". A policy LSM such as IPE could consume the same
+let a policy distinguish "verified against the operator's bpf keyring" from
+"verified against a keyring the loader supplied itself", which is the
+distinction that matters most. A policy LSM such as IPE could consume the same
 hooks to enforce system policy without writing any BPF, though none implements
 this today.
 
@@ -390,33 +406,158 @@ Keyrings
 ========
 
 ``keyring_id`` selects the trusted keyring the PKCS#7 signature is verified
-against. The well-known ids ``0`` (builtin), ``VERIFY_USE_SECONDARY_KEYRING``
-and ``VERIFY_USE_PLATFORM_KEYRING`` select the corresponding system keyrings;
-any other value is treated as the serial of a user/session key or keyring.
-The keyring is looked up first, before the signature bytes are examined, so a
-signature naming a non-existent keyring is rejected up front, and a failed
-verification aborts the load - so a program that loads successfully with a
-signature always has consistent keyring fields recorded.
+against. Four values are well-known; anything else is resolved as a caller-
+supplied key or keyring, named either by its serial or by one of the other
+``KEY_SPEC_*`` ids:
+
+.. list-table::
+   :header-rows: 1
+
+   * - ``keyring_id``
+     - Keyring
+   * - ``0``
+     - builtin trusted keyring
+   * - ``VERIFY_USE_SECONDARY_KEYRING`` (``1``)
+     - secondary trusted keyring
+   * - ``VERIFY_USE_PLATFORM_KEYRING`` (``2``)
+     - platform keyring
+   * - ``KEY_SPEC_BPF_KEYRING`` (``-9``)
+     - the bpf keyring
+   * - anything else
+     - a caller-supplied user/session key or keyring
+
+The keyring is resolved first, before the signature bytes are examined, so a
+signature naming a keyring that cannot be used is rejected up front, and a
+failed verification aborts the load - a program that loads successfully with
+a signature therefore always has consistent keyring fields recorded.
+
+The bpf keyring
+---------------
+
+A system keyring needs a kernel rebuild or a vouched-for enrollment to rotate a
+key, and grants BPF-signing trust to keys trusted for everything else in the
+kernel too. A caller-supplied keyring, at the other extreme, is filled by the
+very process that loads the program and so carries no trust of its own.
+
+The bpf keyring fills that gap and is the trust anchor which a signed BPF
+deployment should be built on top of: a keyring named ``.bpf``, selected with
+``KEY_SPEC_BPF_KEYRING``, that an operator provisions at boot with a key
+scoped to BPF program loading and nothing else in the kernel's trust hierarchy.
+It is owned by the operator rather than by the loader, and rotatable across a
+reboot without touching the kernel image. It is modelled after the dm-verity
+keyring (see ``dm_verity.keyring_unsealed=``) and provisioned the same way: an
+initrd runs the ``keyctl`` steps below before handing off to the rootfs.
+
+Provisioning
+~~~~~~~~~~~~
+
+The keyring is created during ``late_initcall`` and is **sealed empty** by
+default: it carries a reject-all restriction, so no key can ever be added and
+``KEY_SPEC_BPF_KEYRING`` fails with ``-ENOKEY`` for the whole boot.
+
+``bpf.keyring_unsealed=1`` leaves it unrestricted at init so the initrd can
+provision it. The keyring is not linked into any process keyring, but the
+``KEY_SPEC_BPF_KEYRING`` special key id addresses it directly, so its serial
+does not have to be looked up first. Steps would be as follows::
+
+    keyctl padd asymmetric "" -9 < signing_key.der
+    keyctl restrict_keyring -9
+
+Both steps are required: the keyring is consulted only once it is **non-empty
+and restricted**. An unrestricted keyring is ignored even when it holds keys,
+so a half-provisioned keyring is inert rather than a weaker trust anchor, and a
+load naming it fails with ``-ENOKEY`` and a verifier log. Restricting cannot
+be undone.
+
+More than one key is enrolled by repeating the ``keyctl padd`` step; the
+restriction is applied once, after the last of them::
+
+    for key in /etc/bpf/keys/*.der; do
+        keyctl padd asymmetric "" -9 < $key
+    done
+
+    keyctl restrict_keyring -9
+    keyctl show -9
+
+The restriction bounds what can be added, never what can be taken away. A key
+that is already enrolled can still be unlinked, and the keyring cleared or
+revoked, by anything running as root. That does not weaken the anchor, since
+a keyring left empty is no longer consulted and a load naming it fails with
+``-ENOKEY``, but it does take signed loading out until the next boot. Dropping
+the user permissions the keyring no longer needs would close that; as a third
+step in the initrd::
+
+    keyctl setperm -9 0x08030000
+
+What remains is ``KEY_POS_SEARCH`` for the in-kernel search during verification,
+plus ``KEY_USR_VIEW`` and ``KEY_USR_READ`` so the keyring stays visible in
+``/proc/keys``. ``KEY_SPEC_BPF_KEYRING`` names the keyring but conveys no rights
+of its own - the keyring is not possessed by anyone, so the ``KEY_POS_*`` bits
+are only ever reached by the in-kernel search, and userspace is left with what
+the ``KEY_USR_*`` bits grant. Once they are dropped, addressing it by the serial
+``/proc/keys`` reports gets no further than the special id does.
+
+Provisioning has to complete before control passes to the rootfs. The keyring
+takes any number of keys for as long as it is unrestricted, so it is the
+restriction that bounds the enrolled set, not the first enrollment. Before
+the initrd hands off control, it must therefore restrict the keyring after
+the last enrollment.
+
+Enforcement
+~~~~~~~~~~~
+
+``bpf.keyring_unsealed=1`` states that the bpf keyring is *the* trust anchor for
+this boot, so it does more than unseal. From the first program load onwards a
+caller-supplied user/session keyring is refused with ``-EPERM`` and a verifier
+log message, whether or not provisioning ever completed. The system keyrings
+stay selectable.
+
+Enforcement is readable at ``/sys/module/bpf/parameters/keyring_unsealed``, and
+read-only there: the flag is ``__ro_after_init`` behind a 0444 parameter. It is
+also derived from the boot flag rather than from the keyring's runtime state,
+so there is no window early in boot during which a caller-supplied keyring is
+still accepted.
+
+Caller-supplied keyrings are for staging
+----------------------------------------
+
+A ``keyring_id`` naming a user or session key or keyring is a *staging*
+mechanism, not a trust anchor: it is filled by the same userspace that loads the
+program, so verifying against it establishes only that the loader signed what it
+loaded. Its purpose is to let software installed onto a running system - whose
+signing key is not enrolled anywhere yet - run signed until that key reaches the
+bpf keyring on the next boot.
+
+A system that has committed to the bpf keyring refuses this path outright (see
+`Enforcement`_). A system that has not can still allow it, but a policy must
+never treat ``BPF_SIG_KEYRING_USER`` as equivalent to the bpf or system
+keyrings; it should allowlist the specific serials it is willing to stage and
+pair that with a BPF LSM policy protecting those keys from tampering, as in
+`Enforcement via LSMs`_.
+
+Recorded fields
+---------------
 
 Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect:
 
 ``keyring_type`` (``enum bpf_sig_keyring``)
     Classified purely from ``keyring_id`` whenever the program is signed:
     ``BPF_SIG_KEYRING_BUILTIN``, ``_SECONDARY``, ``_PLATFORM`` for the system
-    keyrings, or ``_USER`` for a user/session keyring. It is
-    ``BPF_SIG_KEYRING_NONE`` for an unsigned program.
+    keyrings, ``_BPF`` for the bpf keyring, or ``_USER`` for a caller-supplied
+    user/session keyring. It is ``BPF_SIG_KEYRING_NONE`` for an unsigned
+    program.
 
 ``keyring_serial`` (``s32``)
     Set **only** on a successful verification, to the serial of the
-    **user/session key or keyring** that ``keyring_id`` resolved to - the
+    **caller-supplied key or keyring** that ``keyring_id`` resolved to - the
     object the signature was verified against, not the individual asymmetric
     key inside it that matched the signer. Passing
     ``KEY_SPEC_SESSION_KEYRING``, for example, records the session keyring's
-    serial. The system keyrings are trusted as a whole and expose no serial
-    here, so the serial is ``0`` for builtin, secondary and platform
-    signatures, and ``0`` for unsigned programs. In other words, a non-zero
-    ``keyring_serial`` is exactly "verified against the user key/keyring with
-    this serial".
+    serial. The system keyrings and the bpf keyring are trusted as a whole and
+    expose no serial here, so the serial is ``0`` for them, and ``0`` for
+    unsigned programs. A non-zero ``keyring_serial`` is therefore exactly
+    "verified against the caller-supplied key/keyring with this serial", which
+    is exactly the case a policy has to scrutinise.
 
 .. list-table::
    :header-rows: 1
@@ -436,16 +577,51 @@ Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect:
    * - ``VERIFY_USE_PLATFORM_KEYRING``
      - ``BPF_SIG_KEYRING_PLATFORM``
      - ``0``
-   * - other (a user/session key serial)
+   * - ``KEY_SPEC_BPF_KEYRING``
+     - ``BPF_SIG_KEYRING_BPF``
+     - ``0``
+   * - other (a caller-supplied key serial)
      - ``BPF_SIG_KEYRING_USER``
      - serial of the resolved key/keyring
 
-Producing a signed object
-==========================
+Producing and loading a signed object
+=====================================
+
+Generating a signing key
+------------------------
+
+Signing is algorithm agnostic: the algorithm comes from the X.509 certificate
+and the PKCS#7 ``SignerInfo``. Anything the X.509 and PKCS#7 parsers understand
+works with no BPF-side change. Both examples below read the certificate request
+config from ``x509.genkey``, for which ``certs/x509.genkey`` serves as a
+template (see Documentation/admin-guide/module-signing.rst). RSA::
+
+    openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \
+            -config x509.genkey -outform PEM \
+            -out signing_key.pem -keyout signing_key.pem
+    openssl x509 -in signing_key.pem -outform der -out signing_key.der
+
+ML-DSA-87 (FIPS-204), which needs openssl 3.5 or later, and both
+``CONFIG_CRYPTO_MLDSA`` and ``CONFIG_CRYPTO_SHA512`` in the kernel - the latter
+for the signedAttrs digest described below, which ``CONFIG_CRYPTO_MLDSA`` does
+not select. Note the absence of a digest option: ML-DSA hashes the message
+itself, so openssl ignores an explicit digest for it::
+
+    openssl req -new -nodes -utf8 -days 36500 -batch -x509 \
+            -newkey ML-DSA-87 -config x509.genkey -outform PEM \
+            -out signing_key.pem -keyout signing_key.pem
+    openssl x509 -in signing_key.pem -outform der -out signing_key.der
+
+``bpftool`` handles the following internally: openssl before 4.0 cannot combine
+ML-DSA with ``CMS_NOATTR``, so it falls back to signedAttrs, where only SHA-512
+is permitted. This mirrors what module signing does as well.
+
+Signing
+-------
 
 ``bpftool`` generates and signs a light skeleton in one step::
 
-    bpftool gen skeleton -L -S -k <private_key.pem> -i <certificate.x509> \
+    bpftool gen skeleton -L -S -k signing_key.pem -i signing_key.der \
             obj.bpf.o > obj.lskel.h
 
 ``-L`` selects the light-skeleton (``gen_loader``) backend and ``-S`` enables
@@ -454,12 +630,36 @@ signing; ``-k`` and ``-i`` supply the signing key and its X.509 certificate.
 reconstructs - and also computes ``excl_prog_hash`` as the digest of the loader
 instructions so the metadata map can be bound to the loader. The signature and
 hash are embedded in the generated header; the certificate is used only for
-signing and is not included. Loading the skeleton performs the
-create/populate/freeze/load sequence described above.
+signing and is not included.
+
+Loading
+-------
+
+The generated skeleton exposes ``keyring_id``, which selects the keyring the
+kernel verifies against. Set it between open and load; loading then performs
+the create/populate/freeze/load sequence described above::
 
-At runtime the trusted public key must be present in the chosen keyring (for
-example added to the session keyring, or built into the kernel's builtin trusted
-keyring) for verification to succeed.
+    struct obj *skel = obj__open();
+
+    skel->keyring_id = KEY_SPEC_BPF_KEYRING;
+    err = obj__load(skel);
+
+For the staging case the same object is loaded against a keyring the caller
+populated itself, which only works on a system that has not set
+``bpf.keyring_unsealed=1``::
+
+    /*
+     * Staging only: this keyring is under the loader's own control and
+     * carries no trust of its own. See "Caller-supplied keyrings are for
+     * staging".
+     */
+    key_id = add_key("asymmetric", "", der, der_sz, KEY_SPEC_SESSION_KEYRING);
+    skel->keyring_id = KEY_SPEC_SESSION_KEYRING;
+    err = obj__load(skel);
+
+Either way the trusted public key must already be in the chosen keyring for
+verification to succeed. For the bpf keyring that enrollment happens once at
+boot, see `Provisioning`_.
 
 UAPI reference
 ==============
@@ -487,6 +687,13 @@ UAPI reference
     The map content is not hashed separately at all - it is covered, as bytes,
     by the program signature.
 
+Kernel command line:
+
+``bpf.keyring_unsealed=``
+    Set to ``1`` to leave the bpf keyring unsealed for provisioning, and to make
+    it the only non-system keyring a loader may select for the rest of the boot
+    (see `The bpf keyring`_).
+
 Notes and limitations
 ======================
 
@@ -495,3 +702,5 @@ Notes and limitations
   exceed it.
 - The metadata container is a single-element array map, accessed through
   ``map_direct_value_addr``.
+- The bpf keyring needs ``CONFIG_KEYS``; without it there is no bpf keyring
+  and ``KEY_SPEC_BPF_KEYRING`` never resolves.
diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst
index 6de26d1c6a9ad5..f8b3f59a3d5f71 100644
--- a/Documentation/core-api/cpu_hotplug.rst
+++ b/Documentation/core-api/cpu_hotplug.rst
@@ -607,9 +607,9 @@ ONLINE section for notifications on online and offline operation::
    if (ret)
         return ret;
    ....
-   cpuhp_remove_instance(state, &inst1->node);
+   cpuhp_state_remove_instance(state, &inst1->node);
    ....
-   cpuhp_remove_instance(state, &inst2->node);
+   cpuhp_state_remove_instance(state, &inst2->node);
    ....
    cpuhp_remove_multi_state(state);
 
diff --git a/Documentation/core-api/debug-objects.rst b/Documentation/core-api/debug-objects.rst
index ac926fd55a64a9..708775fc581b74 100644
--- a/Documentation/core-api/debug-objects.rst
+++ b/Documentation/core-api/debug-objects.rst
@@ -129,7 +129,7 @@ When the real object is not yet tracked by debugobjects then the
 fixup_activate function is called if available. This is necessary to
 allow the legitimate activation of statically allocated and initialized
 objects. The fixup function checks whether the object is valid and calls
-the debug_objects_init() function to initialize the tracking of this
+the debug_object_init() function to initialize the tracking of this
 object.
 
 When the activation is legitimate, then the state of the associated
diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst
index ba23a472f79484..69d4ebe4f96db0 100644
--- a/Documentation/core-api/dma-api.rst
+++ b/Documentation/core-api/dma-api.rst
@@ -154,7 +154,7 @@ others should not be larger than the returned value.
 ::
 
 	size_t
-	dma_opt_mapping_size(struct device *dev);
+	dma_max_opt_mapping_size(struct device *dev);
 
 Returns the maximum optimal size of a mapping for the device.
 
diff --git a/Documentation/core-api/dma-attributes.rst b/Documentation/core-api/dma-attributes.rst
index eee743184acdec..b07126b5b61675 100644
--- a/Documentation/core-api/dma-attributes.rst
+++ b/Documentation/core-api/dma-attributes.rst
@@ -34,7 +34,7 @@ such mapping is non-trivial task and consumes very limited resources
 (like kernel virtual address space or dma consistent address space).
 Buffers allocated with this attribute can be only passed to user space
 by calling dma_mmap_attrs(). By using this API, you are guaranteeing
-that you won't dereference the pointer returned by dma_alloc_attr(). You
+that you won't dereference the pointer returned by dma_alloc_attrs(). You
 can treat it as a cookie that must be passed to dma_mmap_attrs() and
 dma_free_attrs(). Make sure that both of these also get this attribute
 set on each call.
diff --git a/Documentation/core-api/dma-isa-lpc.rst b/Documentation/core-api/dma-isa-lpc.rst
index 17b193603f0ae1..296838ff295a19 100644
--- a/Documentation/core-api/dma-isa-lpc.rst
+++ b/Documentation/core-api/dma-isa-lpc.rst
@@ -115,17 +115,18 @@ sure that all data has been transferred.
 
 Example::
 
-	int flags, residue;
+	unsigned long flags;
+	int residue;
 
 	flags = claim_dma_lock();
 
-	clear_dma_ff();
+	clear_dma_ff(channel);
 
 	set_dma_mode(channel, DMA_MODE_WRITE);
 	set_dma_addr(channel, phys_addr);
 	set_dma_count(channel, num_bytes);
 
-	dma_enable(channel);
+	enable_dma(channel);
 
 	release_dma_lock(flags);
 
@@ -133,9 +134,9 @@ Example::
 
 	flags = claim_dma_lock();
 
-	dma_disable(channel);
+	disable_dma(channel);
 
-	residue = dma_get_residue(channel);
+	residue = get_dma_residue(channel);
 	if (residue != 0)
 		printk(KERN_ERR "driver: Incomplete DMA transfer!"
 			" %d bytes left!\n", residue);
diff --git a/Documentation/core-api/housekeeping.rst b/Documentation/core-api/housekeeping.rst
index ccb0a88b9cb365..71ba5d86f24929 100644
--- a/Documentation/core-api/housekeeping.rst
+++ b/Documentation/core-api/housekeeping.rst
@@ -9,7 +9,7 @@ extreme workloads can't stand, such as in some DPDK usecases.
 
 The kernel work moved away by CPU isolation is commonly described as
 "housekeeping" because it includes ground work that performs cleanups,
-statistics maintainance and actions relying on them, memory release,
+statistics maintenance and actions relying on them, memory release,
 various deferrals etc...
 
 Sometimes housekeeping is just some unbound work (unbound workqueues,
diff --git a/Documentation/core-api/irq/irq-affinity.rst b/Documentation/core-api/irq/irq-affinity.rst
index 9cb460cf60b616..671604c7395fe1 100644
--- a/Documentation/core-api/irq/irq-affinity.rst
+++ b/Documentation/core-api/irq/irq-affinity.rst
@@ -53,7 +53,7 @@ Now lets restrict that IRQ to CPU(4-7).
 	--- hell ping statistics ---
 	2779 packets transmitted, 2777 packets received, 0% packet loss
 	round-trip min/avg/max = 0.1/0.5/585.4 ms
-	[root@moon 44]# cat /proc/interrupts |  'CPU\|44:'
+	[root@moon 44]# cat /proc/interrupts | grep 'CPU\|44:'
 		CPU0       CPU1       CPU2       CPU3      CPU4       CPU5        CPU6       CPU7
 	44:       1068       1785       1785       1783      1784       1069        1070       1069   IO-APIC-level  eth1
 
diff --git a/Documentation/core-api/irq/irqflags-tracing.rst b/Documentation/core-api/irq/irqflags-tracing.rst
index bdd208259fb30b..a3c77046ae6bf8 100644
--- a/Documentation/core-api/irq/irqflags-tracing.rst
+++ b/Documentation/core-api/irq/irqflags-tracing.rst
@@ -9,12 +9,8 @@ that it gives interested subsystems an opportunity to be notified of
 every hardirqs-off/hardirqs-on, softirqs-off/softirqs-on event that
 happens in the kernel.
 
-CONFIG_TRACE_IRQFLAGS_SUPPORT is needed for CONFIG_PROVE_SPIN_LOCKING
-and CONFIG_PROVE_RW_LOCKING to be offered by the generic lock debugging
-code. Otherwise only CONFIG_PROVE_MUTEX_LOCKING and
-CONFIG_PROVE_RWSEM_LOCKING will be offered on an architecture - these
-are locking APIs that are not used in IRQ context. (the one exception
-for rwsems is worked around)
+CONFIG_TRACE_IRQFLAGS_SUPPORT is needed for CONFIG_PROVE_LOCKING to be
+offered by the generic lock debugging code.
 
 Architecture support for this is certainly not in the "trivial"
 category, because lots of lowlevel assembly code deal with irq-flags
diff --git a/Documentation/core-api/kho/index.rst b/Documentation/core-api/kho/index.rst
index 320914a42178e3..e78a2bbe2be1d5 100644
--- a/Documentation/core-api/kho/index.rst
+++ b/Documentation/core-api/kho/index.rst
@@ -13,8 +13,8 @@ Kexec HandOver (KHO) is a mechanism that allows Linux to preserve memory
 regions, which could contain serialized system states, across kexec.
 
 KHO uses :ref:`flattened device tree (FDT) <kho_fdt>` to pass information about
-the preserved state from pre-exec kernel to post-kexec kernel and :ref:`scratch
-memory regions <kho_scratch>` to ensure integrity of the preserved memory.
+the preserved state from pre-exec kernel to post-kexec kernel and :ref:`boot
+memory regions <kho_bootmem>` to ensure integrity of the preserved memory.
 
 .. _kho_fdt:
 
@@ -40,31 +40,31 @@ and post-kexec kernels. This ABI is defined by header files in
 
    abi.rst
 
-.. _kho_scratch:
+.. _kho_bootmem:
 
-Scratch Regions
-===============
+Boot Memory Regions
+===================
 
 To boot into kexec, we need to have a physically contiguous memory range that
 contains no handed over memory. Kexec then places the target kernel and initrd
 into that region. The new kernel exclusively uses this region for memory
 allocations before during boot up to the initialization of the page allocator.
 
-We guarantee that we always have such regions through the scratch regions: On
-first boot KHO allocates several physically contiguous memory regions. Since
+We guarantee that we always have such regions through the boot memory regions:
+On first boot KHO allocates several physically contiguous memory regions. Since
 after kexec these regions will be used by early memory allocations, there is a
-scratch region per NUMA node plus a scratch region to satisfy allocations
-requests that do not require particular NUMA node assignment.
-By default, size of the scratch region is calculated based on amount of memory
-allocated during boot. The ``kho_scratch`` kernel command line option may be
-used to explicitly define size of the scratch regions.
-The scratch regions are declared as CMA when page allocator is initialized so
-that their memory can be used during system lifetime. CMA gives us the
-guarantee that no handover pages land in that region, because handover pages
-must be at a static physical memory location and CMA enforces that only
-movable pages can be located inside.
-
-After KHO kexec, we ignore the ``kho_scratch`` kernel command line option and
+boot memory region per NUMA node plus a boot memory region to satisfy
+allocations requests that do not require particular NUMA node assignment. By
+default, size of the boot memory region is calculated based on amount of memory
+allocated during boot. The ``kho_bootmem`` kernel command line option may be
+used to explicitly define size of the boot memory regions. The boot memory
+regions are declared as CMA when page allocator is initialized so that their
+memory can be used during system lifetime. CMA gives us the guarantee that no
+handover pages land in that region, because handover pages must be at a static
+physical memory location and CMA enforces that only movable pages can be located
+inside.
+
+After KHO kexec, we ignore the ``kho_bootmem`` kernel command line option and
 instead reuse the exact same region that was originally allocated. This allows
 us to recursively execute any amount of KHO kexecs. Because we used this region
 for boot memory allocations and as target memory for kexec blobs, some parts
diff --git a/Documentation/core-api/liveupdate.rst b/Documentation/core-api/liveupdate.rst
index 5a292d0f37064d..2bce2644eba223 100644
--- a/Documentation/core-api/liveupdate.rst
+++ b/Documentation/core-api/liveupdate.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _luo:
+
 ========================
 Live Update Orchestrator
 ========================
@@ -18,6 +20,8 @@ LUO Preserving File Descriptors
 .. kernel-doc:: kernel/liveupdate/luo_file.c
    :doc: LUO File Descriptors
 
+.. _flb:
+
 LUO File Lifecycle Bound Global Data
 ====================================
 .. kernel-doc:: kernel/liveupdate/luo_flb.c
@@ -70,3 +74,4 @@ See Also
 
 - :doc:`Live Update uAPI </userspace-api/liveupdate>`
 - :doc:`/core-api/kho/index`
+- :doc:`PCI </PCI/liveupdate>`
diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst
index 12bccfb6aac1f7..836342fb1eec56 100644
--- a/Documentation/core-api/maple_tree.rst
+++ b/Documentation/core-api/maple_tree.rst
@@ -214,11 +214,10 @@ Advanced Allocating Nodes
 -------------------------
 
 Allocations are usually handled internally to the tree, however if allocations
-need to occur before a write occurs then calling mas_expected_entries() will
-allocate the worst-case number of needed nodes to insert the provided number of
-ranges.  This also causes the tree to enter mass insertion mode.  Once
-insertions are complete calling mas_destroy() on the maple state will free the
-unused allocations.
+need to occur before a write occurs then calling mas_preallocate() will
+allocate the nodes needed to store the provided entry.  The entry is then
+stored with mas_store_prealloc().  If the store is abandoned, calling
+mas_destroy() on the maple state will free the unused allocations.
 
 .. _maple-tree-advanced-locks:
 
diff --git a/Documentation/core-api/memory-allocation.rst b/Documentation/core-api/memory-allocation.rst
index 0f19dd52432394..648222b6c85763 100644
--- a/Documentation/core-api/memory-allocation.rst
+++ b/Documentation/core-api/memory-allocation.rst
@@ -19,10 +19,18 @@ Diversity of the allocation APIs combined with the numerous GFP flags
 makes the question "How should I allocate memory?" not that easy to
 answer, although very likely you should use
 
+::
+
+  kzalloc_obj(<VAR_OR_TYPE>);
+
+if you need memory for an object and
+
 ::
 
   kzalloc(<size>, GFP_KERNEL);
 
+if you need memory for a buffer.
+
 Of course there are cases when other allocation APIs and different GFP
 flags must be used.
 
@@ -139,10 +147,13 @@ allocate memory for an array, there are kmalloc_array() and kcalloc()
 helpers. The helpers struct_size(), array_size() and array3_size() can
 be used to safely calculate object sizes without overflowing.
 
-The maximal size of a chunk that can be allocated with `kmalloc` is
-limited. The actual limit depends on the hardware and the kernel
-configuration, but it is a good practice to use `kmalloc` for objects
-smaller than page size.
+Since 7.0 there are type aware kmalloc-family helpers that let you safely and
+conveniently allocate a single object or arrays of objects with kzalloc_obj()
+and kmalloc_obj() and their array versions kzalloc_objs() and
+kmalloc_objs(). These helpers only need the type of the object that should be
+allocated and the count of elements in the array for the array versions.
+
+As of v7.2, vast majority of the memory allocations use kzalloc_obj().
 
 The address of a chunk allocated with `kmalloc` is aligned to at least
 ARCH_KMALLOC_MINALIGN bytes. For sizes which are a power of two, the
@@ -154,9 +165,16 @@ Chunks allocated with kmalloc() can be resized with krealloc(). Similarly
 to kmalloc_array(): a helper for resizing arrays is provided in the form of
 krealloc_array().
 
-For large allocations you can use vmalloc() and vzalloc(), or directly
-request pages from the page allocator. The memory allocated by `vmalloc`
-and related functions is not physically contiguous.
+`kmalloc` always allocates physically contiguous memory and the maximal size of
+a chunk that can be allocated with `kmalloc` is limited by `KMALLOC_MAX_SIZE`,
+which matches the page allocator's MAX_PAGE_ORDER limit.
+
+Internally, the slab allocator differentiates allocations of different orders
+and delegates larger allocations to the page allocator, but for the users of
+`kmalloc` family it is entirely transparent.
+
+For large allocations that do not require physically contiguous memory you can
+use vmalloc() and vzalloc() family.
 
 If you are not sure whether the allocation size is too large for
 `kmalloc`, it is possible to use kvmalloc() and its derivatives. It will
diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst
index c1d03a5a2a192e..e135b61bc87da8 100644
--- a/Documentation/core-api/mm-api.rst
+++ b/Documentation/core-api/mm-api.rst
@@ -52,6 +52,15 @@ Virtually Contiguous Mappings
 .. kernel-doc:: mm/vmalloc.c
    :export:
 
+Kernel Page Table Permissions
+=============================
+
+.. kernel-doc:: include/linux/set_memory.h
+   :doc: Kernel page table permissions
+
+.. kernel-doc:: include/linux/set_memory.h
+   :internal:
+
 File Mapping and Page Cache
 ===========================
 
diff --git a/Documentation/core-api/pin_user_pages.rst b/Documentation/core-api/pin_user_pages.rst
index c16ca163b55e3c..e0acedbd1d4860 100644
--- a/Documentation/core-api/pin_user_pages.rst
+++ b/Documentation/core-api/pin_user_pages.rst
@@ -226,15 +226,6 @@ will be pinned longterm, and whose data will be accessed.
 
 Unit testing
 ============
-This file::
-
- tools/testing/selftests/mm/gup_test.c
-
-has the following new calls to exercise the new pin*() wrapper functions:
-
-* PIN_FAST_BENCHMARK (./gup_test -a)
-* PIN_BASIC_TEST (./gup_test -b)
-
 You can monitor how many total dma-pinned pages have been acquired and released
 since the system was booted, via two new /proc/vmstat entries: ::
 
diff --git a/Documentation/core-api/real-time/differences.rst b/Documentation/core-api/real-time/differences.rst
index a129570dab5a3f..6f5be9f0da2e7a 100644
--- a/Documentation/core-api/real-time/differences.rst
+++ b/Documentation/core-api/real-time/differences.rst
@@ -119,12 +119,20 @@ timers initialized with the HRTIMER_MODE_SOFT flag, which are executed in
 softirq context.
 
 On a PREEMPT_RT kernel, this behavior is reversed: hrtimers are executed in
-softirq context by default, typically within the ktimersd thread. This thread
+softirq context by default, typically within the ktimers thread. This thread
 runs at the lowest real-time priority, ensuring it executes before any
 SCHED_OTHER tasks but does not interfere with higher-priority real-time
 threads. To explicitly request execution in hard interrupt context on
 PREEMPT_RT, the timer must be marked with the HRTIMER_MODE_HARD flag.
 
+Userland sleepers usually deploy a hrtimer to guarantee a precise wakeup
+time. The timer is initialized with hrtimer_setup_sleeper_on_stack(), which
+distinguishes between real-time and regular tasks. The hrtimer of a task
+without a real-time priority is handled in soft interrupt context, but for
+real-time priorities HRTIMER_MODE_HARD is used. This ensures that real-time
+tasks are woken up as soon as possible while ordinary tasks cannot block the
+CPU with a thundering herd of wakeups.
+
 Memory allocation
 -----------------
 
diff --git a/Documentation/core-api/swiotlb.rst b/Documentation/core-api/swiotlb.rst
index 71b4e4c27eb5c0..211cc3499315a2 100644
--- a/Documentation/core-api/swiotlb.rst
+++ b/Documentation/core-api/swiotlb.rst
@@ -107,7 +107,7 @@ A single allocation from swiotlb is limited to IO_TLB_SIZE * IO_TLB_SEGSIZE
 bytes, which is 256 KiB with current definitions. When a device's DMA settings
 are such that the device might use swiotlb, the maximum size of a DMA segment
 must be limited to that 256 KiB. This value is communicated to higher-level
-kernel code via dma_map_mapping_size() and swiotlb_max_mapping_size(). If the
+kernel code via dma_max_mapping_size() and swiotlb_max_mapping_size(). If the
 higher-level code fails to account for this limit, it may make requests that
 are too large for swiotlb, and get a "swiotlb full" error.
 
diff --git a/Documentation/core-api/this_cpu_ops.rst b/Documentation/core-api/this_cpu_ops.rst
index 533ac5dd575076..367706d1714b1c 100644
--- a/Documentation/core-api/this_cpu_ops.rst
+++ b/Documentation/core-api/this_cpu_ops.rst
@@ -150,10 +150,10 @@ preemptible code are addressed by raw_cpu_ptr(), but such use cases need
 to handle cases where two different CPUs are accessing the same per cpu
 variable, which might well be that of a third CPU.  These use cases are
 typically performance optimizations.  For example, SRCU implements a pair
-of counters as a pair of per-CPU variables, and rcu_read_lock_nmisafe()
+of counters as a pair of per-CPU variables, and srcu_read_lock_nmisafe()
 uses raw_cpu_ptr() to get a pointer to some CPU's counter, and uses
-atomic_inc_long() to handle migration between the raw_cpu_ptr() and
-the atomic_inc_long().
+atomic_long_inc() to handle migration between the raw_cpu_ptr() and
+the atomic_long_inc().
 
 Per cpu variables and offsets
 -----------------------------
diff --git a/Documentation/core-api/xarray.rst b/Documentation/core-api/xarray.rst
index c6c91cbd0c3ce4..82876fbb3bf289 100644
--- a/Documentation/core-api/xarray.rst
+++ b/Documentation/core-api/xarray.rst
@@ -127,6 +127,8 @@ by using xa_set_mark() and remove the mark from an entry by calling
 xa_clear_mark().  You can ask whether any entry in the XArray has a
 particular mark set by calling xa_marked().  Erasing an entry from the
 XArray causes all marks associated with that entry to be cleared.
+Storing a new entry that replaces an existing entry keeps the marks
+associated with the index intact.
 
 Setting or clearing a mark on any index of a multi-index entry will
 affect all indices covered by that entry.  Querying the mark on any
@@ -490,7 +492,7 @@ entry at every index to ``NULL`` and dissolve the tie.  A multi-index
 entry can be split into entries occupying smaller ranges by calling
 xas_split_alloc() without the xa_lock held, followed by taking the lock
 and calling xas_split() or calling xas_try_split() with xa_lock. The
-difference between xas_split_alloc()+xas_split() and xas_try_alloc() is
+difference between xas_split_alloc()+xas_split() and xas_try_split() is
 that xas_split_alloc() + xas_split() split the entry from the original
 order to the new order in one shot uniformly, whereas xas_try_split()
 iteratively splits the entry containing the index non-uniformly.
diff --git a/Documentation/crypto/libcrypto-zeroization.rst b/Documentation/crypto/libcrypto-zeroization.rst
new file mode 100644
index 00000000000000..76b65067116347
--- /dev/null
+++ b/Documentation/crypto/libcrypto-zeroization.rst
@@ -0,0 +1,150 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Crypto Key Zeroization
+======================
+
+This document describes the conventions for zeroizing crypto structures in the
+kernel.
+
+Note: the kernel follows traditional cryptographic terminology by using
+the term "zeroizing" to mean erasing sensitive parameters to prevent
+their disclosure if the system is later compromised.  This distinguishes
+it from zeroing memory for other purposes such as initialization.
+
+.. contents::
+
+Overview
+--------
+
+Cryptographic key material and intermediate state (such as HMAC contexts) must
+be zeroized after use to prevent sensitive data from lingering on the stack or
+heap, where it could be leaked through memory disclosure vulnerabilities,
+crash dumps, or cold-boot attacks.
+
+For memory that has been allocated with kmalloc() or a similar function,
+kfree_sensitive() should be used instead of kfree() to release the memory.
+
+For other cases, the kernel provides memzero_explicit() for clearing the
+memory.  Unlike plain memset(), memzero_explicit() is guaranteed not
+to be optimized away by the compiler, even when the memory being cleared
+appears to be dead.
+
+The crypto library builds on memzero_explicit() by providing typed
+zeroization helpers for each key and context structure.  These helpers serve
+two purposes:
+
+1. They make __cleanup() annotations possible, so that structures on
+   the stack are automatically zeroized when they go out of scope.
+
+2. They improve readability by replacing ``memzero_explicit(&key, sizeof(key))``
+   with a self-documenting call like ``aes_zeroize_key(&key)``.
+
+
+What to zeroize
+---------------
+
+The following types of structures hold sensitive material and should be
+zeroized after use:
+
+- **Key structures** (e.g. ``struct aes_key``, ``struct hmac_sha256_key``):
+  contain expanded round keys or prepared key material.
+
+- **HMAC/MAC context structures** (e.g. ``struct hmac_sha256_ctx``,
+  ``struct aes_cmac_ctx``): contain inner and outer hash states derived from
+  the key.
+
+- **Hash context structures** (e.g. ``struct sha256_ctx``): may contain
+  sensitive data being hashed.
+
+Not all of these require explicit cleanup by callers.  Many ``..._final()``
+functions already zeroize their context internally (see `Automatic vs. manual
+zeroization`_ below).
+
+
+Zeroization helpers
+-------------------
+
+Each crypto structure that callers may need to zeroize should have a
+corresponding inline helper function.  The naming convention is::
+
+    <algorithm>_zeroize_<type>(struct <algorithm>_<type> *p);
+
+For example::
+
+    void aes_zeroize_key(struct aes_key *key);
+    void aes_zeroize_enckey(struct aes_enckey *key);
+    void hmac_sha256_zeroize_ctx(struct hmac_sha256_ctx *ctx);
+    void aes_cmac_zeroize_key(struct aes_cmac_key *key);
+    void aes_cmac_zeroize_ctx(struct aes_cmac_ctx *ctx);
+
+Each helper is a ``static inline`` function in the algorithm's header that
+wraps ``memzero_explicit()``, for example::
+
+    static inline void hmac_sha256_zeroize_ctx(struct hmac_sha256_ctx *ctx)
+    {
+            memzero_explicit(ctx, sizeof(*ctx));
+    }
+
+
+Using __cleanup for automatic zeroization
+-----------------------------------------
+
+The preferred way to zeroize stack-allocated key and context structures is
+with the __cleanup() attribute.  This ensures zeroization happens on all
+exit paths, including error returns and early exits. For example::
+
+    static int my_aesxts_setkey(..., const u8 *key, unsigned int len)
+    {
+        struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx);
+        ...
+
+        /* Only half of the key data is cipher key */
+        keylen = (len >> 1);
+        ret = aes_expandkey(&aes, key, keylen);
+        if (ret)
+                return ret;
+
+        ... do something with the cipher key ...
+
+        /* The other half is the tweak key */
+        ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen);
+        if (ret)
+                return ret;  /* <-- Could leak cipher key without __cleanup */
+
+        ... do something with the tweak key ...
+
+        /* No need for memzero_explicit() at the end thanks to the __cleanup */
+        return 0;
+    }
+
+Note that __cleanup() attributes should not be used in functions that use
+"goto" statements. The benefit of cleanup helpers is the removal of "gotos",
+and that "goto" statements can jump between scopes, so the expectation is
+that usage of "goto" and cleanup helpers is never mixed in the same function.
+
+
+Automatic vs. manual zeroization
+--------------------------------
+
+Many ``..._final()`` functions in the crypto library automatically zeroize
+their context before returning.  When this is the case, the kernel-doc for the
+function documents it::
+
+    After finishing, this zeroizes @ctx.  So the caller does not need to do it.
+
+In these cases, callers on simple code paths (where ``..._final()`` is always
+reached) do not need to add __cleanup() or explicit zeroization.
+However, __cleanup() is still recommended whenever there are error paths
+that bypass ``..._final()``, as it ensures zeroization on all paths.
+
+For algorithms where ``..._final()`` does *not* zeroize the context (such as
+the SHAKE XOFs, where ``shake_squeeze()`` can be called multiple times),
+callers must explicitly zeroize the context by calling the appropriate helper
+or using __cleanup(), for example::
+
+    struct shake_ctx ctx __cleanup(shake_zeroize_ctx);
+
+    shake256_init(&ctx);
+    shake_update(&ctx, data, data_len);
+    shake_squeeze(&ctx, out, out_len);
+    /* ctx is automatically zeroized at end of scope */
diff --git a/Documentation/crypto/libcrypto.rst b/Documentation/crypto/libcrypto.rst
index e911e05215979c..494541c8a8e01b 100644
--- a/Documentation/crypto/libcrypto.rst
+++ b/Documentation/crypto/libcrypto.rst
@@ -127,6 +127,8 @@ The crypto library uses standard KUnit tests.  Like many of the kernel's other
 KUnit tests, they are included in the set of tests that is run by
 ``tools/testing/kunit/kunit.py run --alltests``.
 
+For more information about KUnit, see Documentation/dev-tools/kunit/start.rst.
+
 A ``.kunitconfig`` file is also provided to run just the crypto library tests.
 For example, here's how to run them in user-mode Linux:
 
@@ -148,6 +150,17 @@ emulate the correct type of hardware for the code to be reached.
 Since correctness is essential in cryptographic code, new architecture-optimized
 code is accepted only if it can be tested in QEMU.
 
+Most of the crypto KUnit tests also include benchmarks.  To enable these, enable
+``CONFIG_CRYPTO_LIB_BENCHMARK=y`` (in addition to the tests themselves).  The
+benchmark results are printed to the kernel log when the test runs.
+
+Of course, the crypto KUnit tests can also be run on real hardware.  Note that
+it is generally still possible to test and benchmark non-default code paths in
+this case (for example, the software implementation of AES when the CPU has
+hardware-accelerated AES), since on many architectures the kernel supports
+disabling CPU features via the kernel command line.  For example, on x86,
+the ``clearcpuid=aes`` kernel command line option disables AES acceleration.
+
 Note: the crypto library also includes FIPS 140 self-tests.  These are
 lightweight, are designed specifically to meet FIPS 140 requirements, and exist
 *only* to meet those requirements.  Normal testing done by kernel developers and
@@ -165,4 +178,5 @@ API documentation
    libcrypto-signature
    libcrypto-unauth-encryption
    libcrypto-utils
+   libcrypto-zeroization
    sha3
diff --git a/Documentation/dev-tools/kmemleak.rst b/Documentation/dev-tools/kmemleak.rst
index d1b690b1716967..c0d32937234251 100644
--- a/Documentation/dev-tools/kmemleak.rst
+++ b/Documentation/dev-tools/kmemleak.rst
@@ -66,9 +66,14 @@ Memory scanning parameters can be modified at run-time by writing to the
 Kmemleak can also be disabled at boot-time by passing ``kmemleak=off`` on
 the kernel command line.
 
-Memory may be allocated or freed before kmemleak is initialised and
-these actions are stored in an early log buffer. The size of this buffer
-is configured via the CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE option.
+Memory may be allocated or freed before kmemleak is initialised, so a
+static pool of metadata objects is used to track those allocations. Once
+kmemleak is fully initialised the pool becomes an emergency reserve, used
+whenever a metadata object cannot be allocated from the slab. The number
+of objects in the pool is configured via the
+CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE option. Exhausting it at run time
+prints "Cannot allocate a kmemleak_object structure" and disables
+kmemleak.
 
 If CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF are enabled, the kmemleak is
 disabled by default. Passing ``kmemleak=on`` on the kernel command
@@ -185,7 +190,8 @@ reported by kmemleak because values found during the memory scanning
 point to such objects. To reduce the number of false negatives, kmemleak
 provides the kmemleak_ignore, kmemleak_scan_area, kmemleak_no_scan and
 kmemleak_erase functions (see above). The task stacks also increase the
-amount of false negatives and their scanning is not enabled by default.
+amount of false negatives and their scanning is enabled by default; it
+can be turned off with ``stack=off``.
 
 The false positives are objects wrongly reported as being memory leaks
 (orphan). For objects known not to be leaks, kmemleak provides the
@@ -195,12 +201,12 @@ longer be scanned.
 
 Some of the reported leaks are only transient, especially on SMP
 systems, because of pointers temporarily stored in CPU registers or
-stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 1000) representing
+stacks. Kmemleak defines MSECS_MIN_AGE (defaulting to 5000) representing
 the minimum age of an object to be reported as a memory leak.
 
 The ``min_unref_scans`` module parameter requires an object to be seen
 unreferenced in that many consecutive scans before it is reported. It
-defaults to 2 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the
+defaults to 3 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled, where the
 periodic scan thread confirms a leak on its own, and to 1 otherwise. A
 value of 1 preserves the historical behaviour; higher values filter the
 transient false positives described above, at the cost of delaying genuine
@@ -212,8 +218,10 @@ Limitations and Drawbacks
 -------------------------
 
 The main drawback is the reduced performance of memory allocation and
-freeing. To avoid other penalties, the memory scanning is only performed
-when the /sys/kernel/debug/kmemleak file is read. Anyway, this tool is
+freeing. To avoid other penalties, the memory scanning is performed by a
+periodic thread rather than on every allocation. Reading the
+/sys/kernel/debug/kmemleak file only lists the objects found by the last
+scan; writing ``scan`` to it triggers a new one. Anyway, this tool is
 intended for debugging purposes where the performance might not be the
 most important requirement.
 
diff --git a/Documentation/devicetree/bindings/arm/altera.yaml b/Documentation/devicetree/bindings/arm/altera.yaml
index 3030cf46fe74cc..5fce1161210e7c 100644
--- a/Documentation/devicetree/bindings/arm/altera.yaml
+++ b/Documentation/devicetree/bindings/arm/altera.yaml
@@ -112,8 +112,10 @@ properties:
               - intel,socfpga-agilex5-socdk
               - intel,socfpga-agilex5-socdk-013b
               - intel,socfpga-agilex5-socdk-debug
+              - intel,socfpga-agilex5-socdk-emmc
               - intel,socfpga-agilex5-socdk-modular
               - intel,socfpga-agilex5-socdk-nand
+              - intel,socfpga-agilex5-socdk-tsn-cfg2
           - const: intel,socfpga-agilex5
 
       - description: Agilex72 boards
diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml
index fb277c96f341a6..7c3e7131a5e7f1 100644
--- a/Documentation/devicetree/bindings/arm/apple.yaml
+++ b/Documentation/devicetree/bindings/arm/apple.yaml
@@ -82,6 +82,10 @@ description: |
   - iPhone 8 Plus
   - iPhone X
 
+  Devices based on the "A18 Pro" SoC:
+
+  - MacBook Neo (A18 Pro, 2026)
+
   Devices based on the "M1" SoC:
 
   - Mac mini (M1, 2020)
@@ -103,6 +107,15 @@ description: |
   - MacBook Pro (14-inch, M3, 2023)
   - iMac (24-inch, M3, 2023)
 
+  Devices based on the "M4" SoC:
+
+  - MacBook Pro (14-inch, M4, 2024)
+  - iMac (24-inch, 2x USB-C, M4, 2024)
+  - iMac (24-inch, 4x USB-C, M4, 2024)
+  - MacBook Air (13-inch, M4, 2025)
+  - MacBook Air (15-inch, M4, 2025)
+  - Mac mini (M4, 2024)
+
   Devices based on the "M1 Pro", "M1 Max" and "M1 Ultra" SoCs:
 
   - MacBook Pro (14-inch, M1 Pro, 2021)
@@ -291,6 +304,13 @@ properties:
           - const: apple,t8015
           - const: apple,arm-platform
 
+      - description: Apple A18 Pro SoC based platforms
+        items:
+          - enum:
+              - apple,j700  # MacBook Neo (A18 Pro, 2026)
+          - const: apple,t8140
+          - const: apple,arm-platform
+
       - description: Apple M1 SoC based platforms
         items:
           - enum:
@@ -323,6 +343,18 @@ properties:
           - const: apple,t8122
           - const: apple,arm-platform
 
+      - description: Apple M4 SoC based platforms
+        items:
+          - enum:
+              - apple,j604  # MacBook Pro (14-inch, M4, 2024)
+              - apple,j623  # iMac (24-inch, 2x USB-C, M4, 2024)
+              - apple,j624  # iMac (24-inch, 4x USB-C, M4, 2024)
+              - apple,j713  # MacBook Air (13-inch, M4, 2025)
+              - apple,j715  # MacBook Air (15-inch, M4, 2025)
+              - apple,j773g # Mac mini (M4, 2024)
+          - const: apple,t8132
+          - const: apple,arm-platform
+
       - description: Apple M1 Pro SoC based platforms
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml
index 1d2001836eb487..bf21a5b3a4acc7 100644
--- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml
+++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml
@@ -41,6 +41,8 @@ properties:
               - apple,t6030-pmgr
               - apple,t6031-pmgr
               - apple,t8122-pmgr
+              - apple,t8132-pmgr
+              - apple,t8140-pmgr
           - const: apple,t8103-pmgr
           - const: syscon
           - const: simple-mfd
diff --git a/Documentation/devicetree/bindings/arm/arm,cci-400.yaml b/Documentation/devicetree/bindings/arm/arm,cci-400.yaml
index d28303d909e11b..a4f086c2f9c438 100644
--- a/Documentation/devicetree/bindings/arm/arm,cci-400.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,cci-400.yaml
@@ -45,7 +45,11 @@ patternProperties:
 
     properties:
       compatible:
-        const: arm,cci-400-ctrl-if
+        oneOf:
+          - const: arm,cci-400-ctrl-if
+          - items:
+              - const: arm,cci-400-ctrl-if
+              - const: syscon
 
       interface-type:
         enum:
diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml
index 949444aba1f8b3..f4101f94241a53 100644
--- a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml
+++ b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml
@@ -308,7 +308,7 @@ examples:
             arm,trig-in-sigs = <0 1>;
             arm,trig-in-types = <PE_DBGTRIGGER
                                  PE_PMUIRQ>;
-            arm,trig-out-sigs = <0 1 2 >;
+            arm,trig-out-sigs = <0 1 2>;
             arm,trig-out-types = <PE_EDBGREQ
                                   PE_DBGRESTART
                                   PE_CTIIRQ>;
diff --git a/Documentation/devicetree/bindings/arm/bcm/bcm2835.yaml b/Documentation/devicetree/bindings/arm/bcm/bcm2835.yaml
index 2729a542c4f35a..7ad6583f7fae01 100644
--- a/Documentation/devicetree/bindings/arm/bcm/bcm2835.yaml
+++ b/Documentation/devicetree/bindings/arm/bcm/bcm2835.yaml
@@ -29,6 +29,12 @@ properties:
               - raspberrypi,5-model-b
           - const: brcm,bcm2712
 
+      - description: BCM2712-d0 stepping based Boards
+        items:
+          - enum:
+              - raspberrypi,5-model-b-d0
+          - const: brcm,bcm2712-d0
+
       - description: BCM2835 based Boards
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml
index 5be89c5840262d..96136b70dacd9d 100644
--- a/Documentation/devicetree/bindings/arm/cpus.yaml
+++ b/Documentation/devicetree/bindings/arm/cpus.yaml
@@ -86,6 +86,8 @@ properties:
           - apple,avalanche
           - apple,blizzard
           - apple,cyclone
+          - apple,donan-e
+          - apple,donan-p
           - apple,everest
           - apple,firestorm
           - apple,hurricane-zephyr
@@ -93,6 +95,8 @@ properties:
           - apple,mistral
           - apple,monsoon
           - apple,sawtooth
+          - apple,tahiti-e
+          - apple,tahiti-p
           - apple,twister
           - apple,typhoon
           - arm,arm710t
@@ -212,6 +216,7 @@ properties:
           - qcom,kryo465
           - qcom,kryo468
           - qcom,kryo470
+          - qcom,kryo475
           - qcom,kryo485
           - qcom,kryo560
           - qcom,kryo570
@@ -227,6 +232,9 @@ properties:
           - qcom,oryon-2-1
           - qcom,oryon-2-2
           - qcom,oryon-2-3
+          - qcom,oryon-3-0
+          - qcom,oryon-3-1
+          - qcom,oryon-3-2
           - qcom,scorpion
           - samsung,mongoose-m2
           - samsung,mongoose-m3
diff --git a/Documentation/devicetree/bindings/arm/fsl.yaml b/Documentation/devicetree/bindings/arm/fsl.yaml
index 656f2596e1d532..e563d96e4ff7ce 100644
--- a/Documentation/devicetree/bindings/arm/fsl.yaml
+++ b/Documentation/devicetree/bindings/arm/fsl.yaml
@@ -950,6 +950,23 @@ properties:
           - const: toradex,colibri-imx7d-emmc
           - const: fsl,imx7d
 
+      - description: i.MX7D Variscite Boards based on VAR-SOM-MX7 Module
+        oneOf:
+          - items:
+              - enum:
+                  - variscite,var-som-imx7d-emmc-mx7customboard
+                  - variscite,var-som-imx7d-emmc-wm8731-mx7customboard
+                  - variscite,var-som-imx7d-nand-mx7customboard
+                  - variscite,var-som-imx7d-nand-wm8731-mx7customboard
+              - const: variscite,var-som-imx7d
+              - const: fsl,imx7d
+          - items:
+              - enum:
+                  - variscite,var-som-imx7d-v2-emmc-mx7customboard
+                  - variscite,var-som-imx7d-v2-nand-mx7customboard
+              - const: variscite,var-som-imx7d-v2
+              - const: fsl,imx7d
+
       - description: i.MX7ULP based Boards
         items:
           - enum:
@@ -1511,6 +1528,21 @@ properties:
               - fsl,imx91-11x11-frdm-s    # FRDM i.MX91S Development Board
           - const: fsl,imx91
 
+      - description: i.MX91 boards with Toradex Lino Modules and Verdin Adapter
+        items:
+          - enum:
+              - toradex,lino-imx91-verdin-dahlia # Lino iMX91 Module on Verdin Dahlia Board
+              - toradex,lino-imx91-verdin-dev    # Lino iMX91 Module on Verdin Dev Board
+          - const: toradex,lino-imx91-verdin     # Lino iMX91 Module on Verdin Adapter
+          - const: toradex,lino-imx91
+          - const: fsl,imx91
+
+      - description: i.MX91 boards with Toradex OSM Modules
+        items:
+          - const: toradex,osm-imx91-dev # OSM iMX91 Module on OSM Dev Board
+          - const: toradex,osm-imx91
+          - const: fsl,imx91
+
       - description: i.MX93 based Boards
         items:
           - enum:
@@ -1519,6 +1551,22 @@ properties:
               - fsl,imx93-11x11-frdm      # i.MX93 11x11 FRDM Board
               - fsl,imx93-14x14-evk       # i.MX93 14x14 EVK Board
               - fsl,imx93-wireless-evk    # i.MX93 and IW610G WLCSP (Wi-Fi + BLE + 802.15.4) SiP EVK Board
+              - fsl,imx93-wireless-frdm   # i.MX93 and IW610G WLCSP (Wi-Fi + BLE + 802.15.4) SiP FRDM Board
+          - const: fsl,imx93
+
+      - description: i.MX93 boards with Toradex Lino Modules and Verdin Adapter
+        items:
+          - enum:
+              - toradex,lino-imx93-verdin-dahlia # Lino iMX93 Module on Verdin Dahlia Board
+              - toradex,lino-imx93-verdin-dev    # Lino iMX93 Module on Verdin Dev Board
+          - const: toradex,lino-imx93-verdin     # Lino iMX93 Module on Verdin Adapter
+          - const: toradex,lino-imx93
+          - const: fsl,imx93
+
+      - description: i.MX93 boards with Toradex OSM Modules
+        items:
+          - const: toradex,osm-imx93-dev # OSM iMX93 Module on OSM Dev Board
+          - const: toradex,osm-imx93
           - const: fsl,imx93
 
       - description: i.MX94 based Boards
@@ -1542,6 +1590,7 @@ properties:
         items:
           - enum:
               - fsl,imx952-evk            # i.MX952 EVK Board
+              - fsl,imx952-frdm           # i.MX952 FRDM Board
           - const: fsl,imx952
 
       - description: PHYTEC i.MX 95 FPSC based Boards
@@ -1559,6 +1608,12 @@ properties:
           - const: toradex,aquila-imx95     # Aquila iMX95 Module
           - const: fsl,imx95
 
+      - description: Toradex Boards with OSM iMX95 Modules
+        items:
+          - const: toradex,osm-imx95-dev # Toradex OSM iMX95 Module on Toradex OSM Development Board
+          - const: toradex,osm-imx95     # Toradex OSM iMX95 Module
+          - const: fsl,imx95
+
       - description: Toradex Boards with SMARC iMX95 Modules
         items:
           - const: toradex,smarc-imx95-dev # Toradex SMARC iMX95 on Toradex SMARC Development Board
@@ -1737,9 +1792,11 @@ properties:
               - fsl,vf610
               - fsl,vf610m4
 
-      - description: Toradex Colibri VF50 Module on Colibri Evaluation Board
+      - description: Toradex Colibri VF50 Module on a carrier board
         items:
-          - const: toradex,vf500-colibri_vf50-on-eval
+          - enum:
+              - toradex,vf500-colibri_vf50-on-eval
+              - toradex,vf500-colibri-vf50-on-iris
           - const: toradex,vf500-colibri_vf50
           - const: fsl,vf500
 
@@ -1751,9 +1808,11 @@ properties:
               - phytec,vf610-cosmic       # PHYTEC Cosmic/Cosmic+ Board
           - const: fsl,vf610
 
-      - description: Toradex Colibri VF61 Module on Colibri Evaluation Board
+      - description: Toradex Colibri VF61 Module on a carrier board
         items:
-          - const: toradex,vf610-colibri_vf61-on-eval
+          - enum:
+              - toradex,vf610-colibri_vf61-on-eval
+              - toradex,vf610-colibri-vf61-on-iris
           - const: toradex,vf610-colibri_vf61
           - const: fsl,vf610
 
@@ -1930,6 +1989,7 @@ properties:
       - description: LX2160A based Boards
         items:
           - enum:
+              - freemobile,nbxv3
               - fsl,lx2160a-bluebox3
               - fsl,lx2160a-bluebox3-rev-a
               - fsl,lx2160a-qds
diff --git a/Documentation/devicetree/bindings/arm/google.yaml b/Documentation/devicetree/bindings/arm/google.yaml
index 99961e5282e561..b5977e20f39373 100644
--- a/Documentation/devicetree/bindings/arm/google.yaml
+++ b/Documentation/devicetree/bindings/arm/google.yaml
@@ -13,27 +13,16 @@ description: |
   ARM platforms using SoCs designed by Google branded "Tensor" used in Pixel
   devices.
 
-  Currently upstream this is devices using "gs101" SoC which is found in Pixel
-  6, Pixel 6 Pro and Pixel 6a.
-
-  Google have a few different names for the SoC:
-  - Marketing name ("Tensor")
-  - Codename ("Whitechapel")
-  - SoC ID ("gs101")
-  - Die ID ("S5P9845")
-
-  Likewise there are a couple of names for the actual device
-  - Marketing name ("Pixel 6")
-  - Codename ("Oriole")
-
-  Devicetrees should use the lowercased SoC ID and lowercased board codename,
-  e.g. gs101 and gs101-oriole.
+  Currently upstream this is devices using gs101 SoC which is found in Pixel
+  6, Pixel 6 Pro and Pixel 6a and Laguna SoC canonically referred to as lga
+  found in Pixel 10, Pixel 10 Pro and Pixel 10 Pro XL.
 
 properties:
   $nodename:
     const: '/'
   compatible:
     oneOf:
+      # Google Tensor G1 AKA gs101 AKA whitechapel AKA Die ID S5P9845 boards
       - description: Google Pixel 6 or 6 Pro (Oriole or Raven)
         items:
           - enum:
@@ -41,14 +30,35 @@ properties:
               - google,gs101-raven
           - const: google,gs101
 
-  # Bootloader requires empty ect node to be present
+      # Google Tensor G5 AKA lga (laguna) SoC and boards
+      - description:
+          Google Pixel 10, 10 Pro, 10 Pro XL (Frankel, Blazer,
+          Mustang).
+        items:
+          - enum:
+              - google,lga-blazer
+              - google,lga-frankel
+              - google,lga-mustang
+          - const: google,lga
+
   ect:
     type: object
     additionalProperties: false
 
-required:
-  - ect
-
 additionalProperties: true
 
+allOf:
+  # Bootloader requires empty ect node to be present
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: google,gs101
+    then:
+      required:
+        - ect
+    else:
+      properties:
+        ect: false
+
 ...
diff --git a/Documentation/devicetree/bindings/arm/mediatek.yaml b/Documentation/devicetree/bindings/arm/mediatek.yaml
index cd4040ad34377a..180e6e4948e73e 100644
--- a/Documentation/devicetree/bindings/arm/mediatek.yaml
+++ b/Documentation/devicetree/bindings/arm/mediatek.yaml
@@ -122,6 +122,7 @@ properties:
           - const: mediatek,mt7988a
       - items:
           - enum:
+              - amazon,ford
               - mediatek,mt8127-moose
           - const: mediatek,mt8127
       - items:
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml
index 9aa39b002361d9..5ede7d277488c9 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.yaml
@@ -19,7 +19,7 @@ properties:
           - enum:
               - mediatek,mt2701-audsys
               - mediatek,mt6765-audsys
-              - mediatek,mt6779-audsys
+              - mediatek,mt6779-audio
               - mediatek,mt7622-audsys
               - mediatek,mt8167-audsys
               - mediatek,mt8173-audsys
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt
deleted file mode 100644
index 7de43bf41fdc5f..00000000000000
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-MediaTek g3dsys controller
-============================
-
-The MediaTek g3dsys controller provides various clocks and reset controller to
-the GPU.
-
-Required Properties:
-
-- compatible: Should be:
-	- "mediatek,mt2701-g3dsys", "syscon":
-		for MT2701 SoC
-	- "mediatek,mt7623-g3dsys", "mediatek,mt2701-g3dsys", "syscon":
-		for MT7623 SoC
-- #clock-cells: Must be 1
-- #reset-cells: Must be 1
-
-The g3dsys controller uses the common clk binding from
-Documentation/devicetree/bindings/clock/clock-bindings.txt
-The available clocks are defined in dt-bindings/clock/mt*-clk.h.
-
-Example:
-
-g3dsys: clock-controller@13000000 {
-	compatible = "mediatek,mt7623-g3dsys",
-		     "mediatek,mt2701-g3dsys",
-		     "syscon";
-	reg = <0 0x13000000 0 0x200>;
-	#clock-cells = <1>;
-	#reset-cells = <1>;
-};
diff --git a/Documentation/devicetree/bindings/arm/omap/mpu.txt b/Documentation/devicetree/bindings/arm/omap/mpu.txt
deleted file mode 100644
index e41490e6979c3d..00000000000000
--- a/Documentation/devicetree/bindings/arm/omap/mpu.txt
+++ /dev/null
@@ -1,54 +0,0 @@
-* TI - MPU (Main Processor Unit) subsystem
-
-The MPU subsystem contain one or several ARM cores
-depending of the version.
-The MPU contain CPUs, GIC, L2 cache and a local PRCM.
-
-Required properties:
-- compatible : Should be "ti,omap3-mpu" for OMAP3
-               Should be "ti,omap4-mpu" for OMAP4
-	       Should be "ti,omap5-mpu" for OMAP5
-- ti,hwmods: "mpu"
-
-Optional properties:
-- sram:	Phandle to the ocmcram node
-
-am335x and am437x only:
-- pm-sram: Phandles to ocmcram nodes to be used for power management.
-	   First should be type 'protect-exec' for the driver to use to copy
-	   and run PM functions, second should be regular pool to be used for
-	   data region for code. See Documentation/devicetree/bindings/sram/sram.yaml
-	   for more details.
-
-Examples:
-
-- For an OMAP5 SMP system:
-
-mpu {
-    compatible = "ti,omap5-mpu";
-    ti,hwmods = "mpu"
-};
-
-- For an OMAP4 SMP system:
-
-mpu {
-    compatible = "ti,omap4-mpu";
-    ti,hwmods = "mpu";
-};
-
-
-- For an OMAP3 monocore system:
-
-mpu {
-    compatible = "ti,omap3-mpu";
-    ti,hwmods = "mpu";
-};
-
-- For an AM335x system:
-
-mpu {
-	compatible = "ti,omap3-mpu";
-	ti,hwmods = "mpu";
-	pm-sram = <&pm_sram_code
-		   &pm_sram_data>;
-};
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
index f3f3feac4ce289..335a8e1a5a9ebf 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-ctcu.yaml
@@ -32,6 +32,7 @@ properties:
               - qcom,glymur-ctcu
               - qcom,kaanapali-ctcu
               - qcom,qcs8300-ctcu
+              - qcom,shikra-ctcu
               - qcom,sm8750-ctcu
               - qcom,x1e80100-ctcu
           - const: qcom,sa8775p-ctcu
diff --git a/Documentation/devicetree/bindings/arm/qcom-soc.yaml b/Documentation/devicetree/bindings/arm/qcom-soc.yaml
index e7160b63d8145e..179aadc86a9f02 100644
--- a/Documentation/devicetree/bindings/arm/qcom-soc.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom-soc.yaml
@@ -30,7 +30,7 @@ select:
       contains:
         oneOf:
           - pattern: "^qcom,.*(apq|ipq|mdm|msm|qcm|qcs|q[dr]u|sa|sar|sc|sd[amx]|sm|x1[ep])[0-9]+.*$"
-          - pattern: "^qcom,.*(eliza|kaanapali|glymur|hawi|mahua|maili|milos|nord|shikra).*$"
+          - pattern: "^qcom,.*(eliza|kaanapali|kuno|glymur|hawi|mahua|maili|milos|nord|shikra).*$"
   required:
     - compatible
 
@@ -42,7 +42,7 @@ properties:
         - pattern: "^qcom,(apq|ipq|mdm|msm|qcm|qcs|q[dr]u|sa|sc|sd[amx]|sm|x1[ep])[0-9]+(pro)?-.*$"
         - pattern: "^qcom,sar[0-9]+[a-z]?-.*$"
         - pattern: "^qcom,(sa|sc)8[0-9]+[a-z][a-z]?-.*$"
-        - pattern: "^qcom,(eliza|kaanapali|glymur|hawi|mahua|maili|milos|nord|shikra)-.+$"
+        - pattern: "^qcom,(eliza|kaanapali|kuno|glymur|hawi|mahua|maili|milos|nord|shikra)-.+$"
 
         # Legacy namings - variations of existing patterns/compatibles are OK,
         # but do not add completely new entries to these:
diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml
index ff8a35df7862db..121ca557ce1eff 100644
--- a/Documentation/devicetree/bindings/arm/qcom.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom.yaml
@@ -86,8 +86,12 @@ properties:
 
       - items:
           - enum:
+              - asus,zenbook-a14-ux3407na
               - asus,zenbook-a16-ux3607oa
+              - hp,elitebook-x-g2q
+              - lenovo,yoga-slim7x-gen11
               - qcom,glymur-crd
+              - qcom,glymur-qcb
           - const: qcom,glymur
 
       - items:
@@ -175,11 +179,28 @@ properties:
               - qcom,kaanapali-qrd
           - const: qcom,kaanapali
 
+      - items:
+          - enum:
+              - qcom,kalambo-crd
+          - const: qcom,kalambo
+
+      - items:
+          - enum:
+              - qcom,kuno-idp
+          - const: qcom,kuno
+
       - items:
           - enum:
               - qcom,mahua-crd
           - const: qcom,mahua
 
+      - items:
+          - enum:
+              - lenovo,thinkpad-t14s-gen7-lcd
+              - lenovo,thinkpad-t14s-gen7-oled
+          - const: lenovo,thinkpad-t14s-gen7
+          - const: qcom,mahua
+
       - description: Sierra Wireless MangOH Green with WP8548 Module
         items:
           - const: swir,mangoh-green-wp8548
@@ -289,9 +310,15 @@ properties:
               - wingtech,wt82918hdhw39
           - const: qcom,msm8939
 
+      - items:
+          - enum:
+              - generalmobile,shamrock
+          - const: qcom,msm8952
+
       - items:
           - enum:
               - flipkart,rimob
+              - motorola,deen
               - motorola,potter
               - xiaomi,daisy
               - xiaomi,mido
@@ -347,6 +374,7 @@ properties:
 
       - items:
           - enum:
+              - leeco,s2
               - longcheer,l9360
           - const: qcom,msm8976
 
@@ -402,6 +430,12 @@ properties:
               - xiaomi,sagit
           - const: qcom,msm8998
 
+      - items:
+          - enum:
+              - qcom,iq10-rrd
+              - qcom,sa8797p-ride
+          - const: qcom,nord
+
       - description: Qualcomm Technologies, Inc. Robotics RB1
         items:
           - enum:
@@ -433,6 +467,7 @@ properties:
       - items:
           - enum:
               - arduino,monza
+              - qcom,monaco-ac-evk
               - qcom,monaco-evk
               - qcom,qcs8300-ride
           - const: qcom,qcs8300
@@ -1033,6 +1068,7 @@ properties:
       - items:
           - enum:
               - fxtec,pro1x
+              - xiaomi,lemon
           - const: qcom,sm6115
 
       - items:
@@ -1068,6 +1104,8 @@ properties:
 
       - items:
           - enum:
+              - samsung,a52q
+              - samsung,a72q
               - xiaomi,curtana
               - xiaomi,joyeuse
           - const: qcom,sm7125
@@ -1086,6 +1124,7 @@ properties:
           - enum:
               - motorola,dubai
               - nothing,spacewar
+              - xiaomi,lisa
               - xiaomi,taoyao
           - const: qcom,sm7325
 
@@ -1154,6 +1193,18 @@ properties:
 
       - items:
           - enum:
+              - ayntec,odin2
+              - ayntec,odin2mini
+              - ayntec,odin2portal
+              - ayntec,thor
+              - retroidpocket,nova
+              - retroidpocket,rp6
+          - const: qcom,qcs8550
+          - const: qcom,sm8550
+
+      - items:
+          - enum:
+              - ayaneo,pocket-ds
               - qcom,qcs8550-rb5gen2
           - const: qcom,qcs8550
           - const: qcom,sm8550
@@ -1164,6 +1215,7 @@ properties:
               - qcom,sm8650-hdk
               - qcom,sm8650-mtp
               - qcom,sm8650-qrd
+              - valve,deckard
           - const: qcom,sm8650
 
       - items:
@@ -1223,6 +1275,13 @@ properties:
           - const: qcom,hamoa-iot-som
           - const: qcom,x1e80100
 
+      - items:
+          - enum:
+              - advantech,som-db5830
+          - const: advantech,som-6820
+          - const: qcom,iq-x7181-evk-som
+          - const: qcom,x1e80100
+
       - items:
           - const: microsoft,denali-oled
           - const: microsoft,denali
@@ -1333,6 +1392,7 @@ allOf:
               - qcom,msm8917
               - qcom,msm8937
               - qcom,msm8939
+              - qcom,msm8952
               - qcom,msm8953
               - qcom,msm8956
               - qcom,msm8992
diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml
index a799bb843f74fc..81744fbee98a35 100644
--- a/Documentation/devicetree/bindings/arm/rockchip.yaml
+++ b/Documentation/devicetree/bindings/arm/rockchip.yaml
@@ -216,6 +216,13 @@ properties:
           - const: embedfire,lubancat-4
           - const: rockchip,rk3588s
 
+      - description: EmbedFire LubanCat 5IO
+        items:
+          - enum:
+              - embedfire,lubancat-5io
+          - const: embedfire,lubancat-5-btb
+          - const: rockchip,rk3588
+
       - description: Engicam PX30.Core C.TOUCH 2.0
         items:
           - const: engicam,px30-core-ctouch2
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
index 273464400477c7..db19a686f490e7 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
@@ -232,6 +232,7 @@ properties:
       - description: Exynos850 based boards
         items:
           - enum:
+              - samsung,a217f                   # Samsung Galaxy A21s (SM-A217F)
               - winlink,e850-96                 # WinLink E850-96
           - const: samsung,exynos850
 
diff --git a/Documentation/devicetree/bindings/arm/stm32/stm32.yaml b/Documentation/devicetree/bindings/arm/stm32/stm32.yaml
index c6af3a46364fc0..3ee6d7fd53504f 100644
--- a/Documentation/devicetree/bindings/arm/stm32/stm32.yaml
+++ b/Documentation/devicetree/bindings/arm/stm32/stm32.yaml
@@ -196,12 +196,17 @@ properties:
           - const: ultratronik,stm32mp157c-ultra-fly-sbc
           - const: st,stm32mp157
 
-      - description: ST STM32MP257 based Boards
+      - description: ST STM32MP215 based Boards
         items:
           - enum:
-              - st,stm32mp257f-dk
-              - st,stm32mp257f-ev1
-          - const: st,stm32mp257
+              - st,stm32mp215f-dk
+          - const: st,stm32mp215
+
+      - description: DH STM32MP231 DHCOS SoM based Boards
+        items:
+          - const: dh,stm32mp231a-dhcos-bb
+          - const: dh,stm32mp231a-dhcos-som
+          - const: st,stm32mp231
 
       - description: ST STM32MP235 based Boards
         items:
@@ -209,11 +214,31 @@ properties:
               - st,stm32mp235f-dk
           - const: st,stm32mp235
 
-      - description: ST STM32MP215 based Boards
+      - description: DH STM32MP251 DHCOS SoM based Boards
+        items:
+          - const: dh,stm32mp251a-dhcos-bb
+          - const: dh,stm32mp251a-dhcos-som
+          - const: st,stm32mp251
+
+      - description: DH STM32MP255 DHCOS SoM based Boards
+        items:
+          - const: dh,stm32mp255c-dhcos-dhsbc
+          - const: dh,stm32mp255c-dhcos-som
+          - const: st,stm32mp255
+
+      - description: Engicam MicroGEA STM32MP257 SoM based Boards
         items:
           - enum:
-              - st,stm32mp215f-dk
-          - const: st,stm32mp215
+              - engicam,microgea-stm32mp257d-rmm
+          - const: engicam,microgea-stm32mp257
+          - const: st,stm32mp257
+
+      - description: ST STM32MP257 based Boards
+        items:
+          - enum:
+              - st,stm32mp257f-dk
+              - st,stm32mp257f-ev1
+          - const: st,stm32mp257
 
 additionalProperties: true
 
diff --git a/Documentation/devicetree/bindings/arm/sunxi.yaml b/Documentation/devicetree/bindings/arm/sunxi.yaml
index f04db28c630ac5..28437c47ac01bf 100644
--- a/Documentation/devicetree/bindings/arm/sunxi.yaml
+++ b/Documentation/devicetree/bindings/arm/sunxi.yaml
@@ -933,6 +933,16 @@ properties:
           - const: tbs-biometrics,a711
           - const: allwinner,sun8i-a83t
 
+      - description: Teclast P80 Tablet
+        items:
+          - const: teclast,p80
+          - const: allwinner,sun50i-a133
+
+      - description: Teclast P85T Tablet
+        items:
+          - const: teclast,p85t
+          - const: allwinner,sun55i-a523
+
       - description: Topwise A721 Tablet
         items:
           - const: topwise,a721
diff --git a/Documentation/devicetree/bindings/arm/ti/k3.yaml b/Documentation/devicetree/bindings/arm/ti/k3.yaml
index eebb3523303c05..9a0d62234058a7 100644
--- a/Documentation/devicetree/bindings/arm/ti/k3.yaml
+++ b/Documentation/devicetree/bindings/arm/ti/k3.yaml
@@ -229,6 +229,13 @@ properties:
               - ti,j722s-evm
           - const: ti,j722s
 
+      - description: K3 AM67 SoC PHYTEC phyBOARD-Rigel
+        items:
+          - enum:
+              - phytec,am6754-phyboard-rigel
+          - const: phytec,am67-phycore-som
+          - const: ti,j722s
+
       - description: K3 J742S2 SoC
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/ti/ti,omap-mpu.yaml b/Documentation/devicetree/bindings/arm/ti/ti,omap-mpu.yaml
new file mode 100644
index 00000000000000..abd8c64e8e9f74
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/ti/ti,omap-mpu.yaml
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/ti/ti,omap-mpu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI MPU (Main Processor Unit) subsystem
+
+maintainers:
+  - Benoit Cousson <b-cousson@ti.com>
+
+description:
+  The MPU subsystem contains one or several ARM cores depending on the version.
+  The MPU contains CPUs, GIC, L2 cache and a local PRCM.
+
+properties:
+  compatible:
+    enum:
+      - ti,omap3-mpu
+      - ti,omap4-mpu
+      - ti,omap5-mpu
+
+  ti,hwmods:
+    $ref: /schemas/types.yaml#/definitions/string
+    description: Name of the hwmod associated with the MPU
+    const: mpu
+    deprecated: true
+
+  sram:
+    description: ocmcram node
+
+  pm-sram:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    description: ocmcram nodes for power management. For am335x and am437x.
+    items:
+      - items:
+          - description: ocmcram node of type protect-exec for PM functions
+      - items:
+          - description: ocmcram node of regular pool to be used for data region
+
+required:
+  - compatible
+
+additionalProperties: false
+
+examples:
+  - |
+    mpu {
+        compatible = "ti,omap5-mpu";
+        ti,hwmods = "mpu";
+    };
+
+  - |
+    mpu {
+        compatible = "ti,omap3-mpu";
+        ti,hwmods = "mpu";
+        pm-sram = <&pm_sram_code &pm_sram_data>;
+    };
diff --git a/Documentation/devicetree/bindings/arm/vexpress-config.yaml b/Documentation/devicetree/bindings/arm/vexpress-config.yaml
index 41c53e3acc12cf..b04e8e0521815f 100644
--- a/Documentation/devicetree/bindings/arm/vexpress-config.yaml
+++ b/Documentation/devicetree/bindings/arm/vexpress-config.yaml
@@ -139,6 +139,10 @@ patternProperties:
 
   "^regulator-.+$":
     $ref: /schemas/regulator/regulator.yaml#
+    description:
+      If regulator-min-microvolt and regulator-max-microvolt do not both
+      define a range, the supply is treated as fixed and only its voltage
+      can be read.
     properties:
       compatible:
         const: arm,vexpress-volt
diff --git a/Documentation/devicetree/bindings/bus/omap-ocp2scp.txt b/Documentation/devicetree/bindings/bus/omap-ocp2scp.txt
deleted file mode 100644
index 18729f6fe1e5fb..00000000000000
--- a/Documentation/devicetree/bindings/bus/omap-ocp2scp.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-* OMAP OCP2SCP - ocp interface to scp interface
-
-properties:
-- compatible : Should be "ti,am437x-ocp2scp" for AM437x processor
-	       Should be "ti,omap-ocp2scp" for all others
-- reg : Address and length of the register set for the device
-- #address-cells, #size-cells : Must be present if the device has sub-nodes
-- ranges : the child address space are mapped 1:1 onto the parent address space
-- ti,hwmods : must be "ocp2scp_usb_phy"
-
-Sub-nodes:
-All the devices connected to ocp2scp are described using sub-node to ocp2scp
-
-ocp2scp@4a0ad000 {
-	compatible = "ti,omap-ocp2scp";
-	reg = <0x4a0ad000 0x1f>;
-	#address-cells = <1>;
-	#size-cells = <1>;
-	ranges;
-	ti,hwmods = "ocp2scp_usb_phy";
-
-	subnode1 {
-	...
-	};
-
-	subnode2 {
-	...
-	};
-};
diff --git a/Documentation/devicetree/bindings/bus/ti,omap-ocp2scp.yaml b/Documentation/devicetree/bindings/bus/ti,omap-ocp2scp.yaml
new file mode 100644
index 00000000000000..a8852185144dc0
--- /dev/null
+++ b/Documentation/devicetree/bindings/bus/ti,omap-ocp2scp.yaml
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/bus/ti,omap-ocp2scp.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI OMAP OCP to SCP bus bridge
+
+maintainers:
+  - Kishon Vijay Abraham I <kishon@ti.com>
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - const: ti,am437x-ocp2scp
+          - const: ti,omap-ocp2scp
+      - const: ti,omap-ocp2scp
+
+  reg:
+    maxItems: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  ranges: true
+
+  ti,hwmods:
+    $ref: /schemas/types.yaml#/definitions/string
+    description: Name of the hwmod associated with the device.
+    deprecated: true
+
+required:
+  - compatible
+  - "#address-cells"
+  - "#size-cells"
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,am437x-ocp2scp
+    then:
+      properties:
+        reg: false
+    else:
+      required:
+        - reg
+
+additionalProperties:
+  type: object
+
+examples:
+  - |
+    ocp2scp@4a0ad000 {
+        compatible = "ti,omap-ocp2scp";
+        reg = <0x4a0ad000 0x1f>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges;
+
+        phy@80 {
+            compatible = "ti,omap-usb2";
+            reg = <0x80 0x58>;
+            ctrl-module = <&omap_control_usb2phy>;
+            clocks = <&usb_phy_cm_clk32k>;
+            clock-names = "wkupclk";
+            #phy-cells = <0>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
index a1340226a0d036..2141e5d4de63bd 100644
--- a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
+++ b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
@@ -26,6 +26,7 @@ properties:
           - qcom,hawi-llcc
           - qcom,ipq5424-llcc
           - qcom,kaanapali-llcc
+          - qcom,nord-llcc
           - qcom,qcs615-llcc
           - qcom,qcs8300-llcc
           - qcom,qdu1000-llcc
@@ -56,11 +57,11 @@ properties:
 
   reg:
     minItems: 1
-    maxItems: 14
+    maxItems: 18
 
   reg-names:
     minItems: 1
-    maxItems: 14
+    maxItems: 18
 
   interrupts:
     maxItems: 1
@@ -355,6 +356,56 @@ allOf:
             - const: llcc3_base
             - const: llcc_broadcast_base
             - const: llcc_broadcast_and_base
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,nord-llcc
+    then:
+      properties:
+        reg:
+          items:
+            - description: LLCC0 base register region
+            - description: LLCC1 base register region
+            - description: LLCC2 base register region
+            - description: LLCC3 base register region
+            - description: LLCC4 base register region
+            - description: LLCC5 base register region
+            - description: LLCC6 base register region
+            - description: LLCC7 base register region
+            - description: LLCC8 base register region
+            - description: LLCC9 base register region
+            - description: LLCC10 base register region
+            - description: LLCC11 base register region
+            - description: LLCC12 base register region
+            - description: LLCC13 base register region
+            - description: LLCC14 base register region
+            - description: LLCC15 base register region
+            - description: LLCC broadcast OR register region
+            - description: LLCC broadcast AND register region
+        reg-names:
+          items:
+            - const: llcc0_base
+            - const: llcc1_base
+            - const: llcc2_base
+            - const: llcc3_base
+            - const: llcc4_base
+            - const: llcc5_base
+            - const: llcc6_base
+            - const: llcc7_base
+            - const: llcc8_base
+            - const: llcc9_base
+            - const: llcc10_base
+            - const: llcc11_base
+            - const: llcc12_base
+            - const: llcc13_base
+            - const: llcc14_base
+            - const: llcc15_base
+            - const: llcc_broadcast_base
+            - const: llcc_broadcast_and_base
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml b/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml
index eb24a5687639e0..fc37786ac7ed8a 100644
--- a/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml
+++ b/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml
@@ -23,6 +23,7 @@ description: |
 
   All these identifiers can be found in:
   [1]: <include/dt-bindings/clock/en7523-clk.h>.
+  [2]: <include/dt-bindings/soc/airoha,scu-ssr.h>.
 
   The clocks are provided inside a system controller node.
 
@@ -50,6 +51,12 @@ properties:
     description: ID of the controller reset line
     const: 1
 
+  '#phy-cells':
+    description:
+      The first cell indicates the serdes phy number, see [2] for the
+      available serdes port.
+    const: 1
+
 required:
   - compatible
   - reg
@@ -65,12 +72,23 @@ allOf:
         reg:
           minItems: 2
 
+        '#phy-cells': false
+
+  - if:
+      properties:
+        compatible:
+          const: econet,en751221-scu
+    then:
+      properties:
+        reg:
+          maxItems: 1
+
+        '#phy-cells': false
+
   - if:
       properties:
         compatible:
-          enum:
-            - airoha,en7581-scu
-            - econet,en751221-scu
+          const: airoha,en7581-scu
     then:
       properties:
         reg:
diff --git a/Documentation/devicetree/bindings/clock/anlogic,dr1v90-cru.yaml b/Documentation/devicetree/bindings/clock/anlogic,dr1v90-cru.yaml
new file mode 100644
index 00000000000000..11a92588e49494
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/anlogic,dr1v90-cru.yaml
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/anlogic,dr1v90-cru.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Anlogic DR1V90 Clock and Reset Unit (CRU)
+
+maintainers:
+  - Junhui Liu <junhui.liu@pigmoral.tech>
+
+properties:
+  compatible:
+    const: anlogic,dr1v90-cru
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Main oscillator
+      - description: External CAN clock
+      - description: External WDT clock
+
+  clock-names:
+    items:
+      - const: osc
+      - const: can_ext
+      - const: wdt_ext
+
+  "#clock-cells":
+    const: 1
+    description:
+      Refer <dt-bindings/clock/anlogic,dr1v90-cru.h> for valid indices.
+
+  "#reset-cells":
+    const: 1
+    description:
+      Refer <dt-bindings/reset/anlogic,dr1v90-cru.h> for valid indices.
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - "#clock-cells"
+  - "#reset-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    clock-controller@f8801000 {
+        compatible = "anlogic,dr1v90-cru";
+        reg = <0xf8801000 0x400>;
+        clocks = <&osc>, <&can_ext>, <&wdt_ext>;
+        clock-names = "osc", "can_ext", "wdt_ext";
+        #clock-cells = <1>;
+        #reset-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/clock/clk-palmas-clk32kg-clocks.txt b/Documentation/devicetree/bindings/clock/clk-palmas-clk32kg-clocks.txt
deleted file mode 100644
index 4208886d834a03..00000000000000
--- a/Documentation/devicetree/bindings/clock/clk-palmas-clk32kg-clocks.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-* Palmas 32KHz clocks *
-
-Palmas device has two clock output pins for 32KHz, KG and KG_AUDIO.
-
-This binding uses the common clock binding ./clock-bindings.txt.
-
-Required properties:
-- compatible :	"ti,palmas-clk32kg" for clk32kg clock
-		"ti,palmas-clk32kgaudio" for clk32kgaudio clock
-- #clock-cells : shall be set to 0.
-
-Optional property:
-- ti,external-sleep-control: The external enable input pins controlled the
-	enable/disable of clocks.  The external enable input pins ENABLE1,
-	ENABLE2 and NSLEEP. The valid values for the external pins are:
-		PALMAS_EXT_CONTROL_PIN_ENABLE1 for ENABLE1 pin
-		PALMAS_EXT_CONTROL_PIN_ENABLE2 for ENABLE2 pin
-		PALMAS_EXT_CONTROL_PIN_NSLEEP for NSLEEP pin
-	Option 0 or missing this property means the clock is enabled/disabled
-	via register access and these pins do not have any control.
-	The macros of external control pins for DTS is defined at
-	dt-bindings/mfd/palmas.h
-
-Example:
-	#include <dt-bindings/mfd/palmas.h>
-	...
-	palmas: tps65913@58 {
-		...
-		clk32kg: palmas_clk32k@0 {
-			compatible = "ti,palmas-clk32kg";
-			#clock-cells = <0>;
-			ti,external-sleep-control = <PALMAS_EXT_CONTROL_PIN_NSLEEP>;
-		};
-		...
-	};
diff --git a/Documentation/devicetree/bindings/clock/clock-nexus-node.yaml b/Documentation/devicetree/bindings/clock/clock-nexus-node.yaml
new file mode 100644
index 00000000000000..ff3283bd5fb117
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/clock-nexus-node.yaml
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/clock-nexus-node.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Clock Nexus
+
+description: >
+  A nexus node allows to remap a clock specifier list in a consumer node
+  through a container or a connector node in a generic way. With this
+  remapping, the consumer node needs to know only about the nexus
+  node. Resources behind the nexus node are decoupled by the nexus node
+  itself.
+
+maintainers:
+  - Miquel Raynal <miquel.raynal@bootlin.com>
+
+select: true
+
+properties:
+  clock-map:
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+
+dependentRequired:
+  clock-map: ['#clock-cells']
+
+additionalProperties: true
+
+# See the original pwm-nexus-node.yaml description for examples
diff --git a/Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml b/Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml
index 27403b4c52d621..fbbf1b3f17904a 100644
--- a/Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml
+++ b/Documentation/devicetree/bindings/clock/nxp,imx95-blk-ctl.yaml
@@ -39,6 +39,18 @@ properties:
       ID in its "clocks" phandle cell. See
       include/dt-bindings/clock/nxp,imx95-clock.h
 
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 1
+
+patternProperties:
+  '^formatter@[0-9a-f]+$':
+    type: object
+    $ref: /schemas/media/fsl,imx95-csi-formatter.yaml#
+    unevaluatedProperties: false
+
 required:
   - compatible
   - reg
@@ -46,6 +58,23 @@ required:
   - power-domains
   - clocks
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: nxp,imx95-camera-csr
+    then:
+      required:
+        - '#address-cells'
+        - '#size-cells'
+    else:
+      properties:
+        '#address-cells': false
+        '#size-cells': false
+      patternProperties:
+        '^formatter@[0-9a-f]+$': false
+
 additionalProperties: false
 
 examples:
@@ -57,4 +86,46 @@ examples:
       clocks = <&scmi_clk 114>;
       power-domains = <&scmi_devpd 21>;
     };
+
+  - |
+    #include <dt-bindings/clock/nxp,imx95-clock.h>
+
+    syscon@4ac10000 {
+      compatible = "nxp,imx95-camera-csr", "syscon";
+      reg = <0x4ac10000 0x10000>;
+      #address-cells = <1>;
+      #size-cells = <1>;
+      #clock-cells = <1>;
+      clocks = <&scmi_clk 62>;
+      power-domains = <&scmi_devpd 3>;
+
+      formatter@20 {
+        compatible = "fsl,imx95-csi-formatter";
+        reg = <0x20 0x100>;
+        clocks = <&cameramix_csr IMX95_CLK_CAMBLK_CSI2_FOR0>;
+        power-domains = <&scmi_devpd 3>;
+
+        ports {
+          #address-cells = <1>;
+          #size-cells = <0>;
+
+          port@0 {
+            reg = <0>;
+
+            endpoint {
+              remote-endpoint = <&mipi_csi_0_out>;
+            };
+
+          };
+
+          port@1 {
+            reg = <1>;
+
+            endpoint {
+              remote-endpoint = <&isi_in_2>;
+            };
+          };
+        };
+      };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8952.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8952.yaml
new file mode 100644
index 00000000000000..035f2f7451cd8f
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8952.yaml
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,gcc-msm8952.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Global Clock & Reset Controller on MSM8952
+
+maintainers:
+  - Muzaffer Kadir <muzafferkadir@mainlining.org>
+
+description: |
+  Qualcomm global clock control module provides the clocks, resets and power
+  domains on MSM8952.
+
+  See also:
+    include/dt-bindings/clock/qcom,gcc-msm8952.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,gcc-msm8952
+
+  clocks:
+    items:
+      - description: XO source
+      - description: Always-on XO source
+      - description: DSI phy instance 0 dsi clock
+      - description: DSI phy instance 0 byte clock
+
+  power-domains:
+    items:
+      - description: CX power domain
+
+required:
+  - compatible
+  - clocks
+  - power-domains
+  - '#power-domain-cells'
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
+    #include <dt-bindings/clock/qcom,gcc-msm8952.h>
+    #include <dt-bindings/clock/qcom,rpmcc.h>
+    #include <dt-bindings/power/qcom-rpmpd.h>
+
+    clock-controller@1800000 {
+        compatible = "qcom,gcc-msm8952";
+        reg = <0x01800000 0x80000>;
+        clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>,
+                 <&rpmcc RPM_SMD_XO_A_CLK_SRC>,
+                 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>,
+                 <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>;
+        power-domains = <&rpmpd RPMPD_VDDCX>;
+        #clock-cells = <1>;
+        #reset-cells = <1>;
+        #power-domain-cells = <1>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml
index a7523a4143419f..41767eb1165101 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml
@@ -31,11 +31,13 @@ properties:
     maxItems: 5
 
   power-domains:
-    maxItems: 1
+    items:
+      - description: CX power domain
 
 required:
   - compatible
   - '#power-domain-cells'
+  - power-domains
 
 allOf:
   - $ref: qcom,gcc.yaml#
@@ -85,6 +87,7 @@ examples:
   # Example for GCC for SDM845:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,gcc-sdm845";
       reg = <0x100000 0x1f0000>;
@@ -94,6 +97,7 @@ examples:
                <&pcie0_lane>,
                <&pcie1_lane>;
       clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk", "pcie_0_pipe_clk", "pcie_1_pipe_clk";
+      power-domains = <&rpmhpd SDM845_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml
index 819e855eaf9af0..098a108b81234f 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml
@@ -31,10 +31,15 @@ properties:
       - const: bi_tcxo_ao
       - const: sleep_clk
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
   - clock-names
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -45,12 +50,14 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,gcc-sm6350";
       reg = <0x00100000 0x1f0000>;
       clocks = <&rpmhcc RPMH_CXO_CLK>,
                <&rpmhcc RPMH_CXO_CLK_A>,
                <&sleep_clk>;
+      power-domains = <&rpmhpd RPMHPD_CX>;
       clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk";
       #clock-cells = <1>;
       #reset-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml
index 5f3f69fe9ddbed..82d67b0d82d07f 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml
@@ -30,10 +30,15 @@ properties:
       - const: bi_tcxo
       - const: sleep_clk
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
   - clock-names
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -44,12 +49,14 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,gcc-sm8150";
       reg = <0x00100000 0x1f0000>;
       clocks = <&rpmhcc RPMH_CXO_CLK>,
                <&sleep_clk>;
       clock-names = "bi_tcxo", "sleep_clk";
+      power-domains = <&rpmhpd SM8150_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml
index f4cd5a509c6053..5f9aae72824524 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml
@@ -32,10 +32,15 @@ properties:
       - const: bi_tcxo_ao
       - const: sleep_clk
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
   - clock-names
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -46,6 +51,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,gcc-sm8250";
       reg = <0x00100000 0x1f0000>;
@@ -53,6 +59,7 @@ examples:
                <&rpmhcc RPMH_CXO_CLK_A>,
                <&sleep_clk>;
       clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk";
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml
index 97ffae3b55227d..7f3947db5807e1 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml
@@ -51,10 +51,15 @@ properties:
       - const: usb3_uni_phy_sec_gcc_usb30_pipe_clk # Optional clock
     minItems: 2
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
   - clock-names
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -65,12 +70,14 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,gcc-sm8350";
       reg = <0x00100000 0x1f0000>;
       clocks = <&rpmhcc RPMH_CXO_CLK>,
                <&sleep_clk>;
       clock-names = "bi_tcxo", "sleep_clk";
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml
index 3169ac05e1d8ac..793ee9e3ff10a3 100644
--- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml
@@ -47,10 +47,15 @@ properties:
       - const: usb3_phy_wrapper_gcc_usb30_pipe_clk # Optional clock
     minItems: 2
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
   - clock-names
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -61,11 +66,13 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,gcc-sm8450";
       reg = <0x00100000 0x001f4200>;
       clocks = <&rpmhcc RPMH_CXO_CLK>, <&sleep_clk>;
       clock-names = "bi_tcxo", "sleep_clk";
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,hawi-gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,hawi-gpucc.yaml
new file mode 100644
index 00000000000000..b7179d53166f8c
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,hawi-gpucc.yaml
@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,hawi-gpucc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Graphics Clock & Reset Controller on Hawi
+
+maintainers:
+  - Dongfang Zhao <dongfang.zhao@oss.qualcomm.com>
+
+description: |
+  Qualcomm graphics clock control module provides the clocks, resets and power
+  domains on Qualcomm SoCs.
+
+  See also: include/dt-bindings/clock/qcom,hawi-gpucc.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,hawi-gpucc
+      - qcom,maili-gpucc
+
+  clocks:
+    items:
+      - description: Board XO source
+      - description: GPLL0 main branch source
+      - description: GPLL0 div branch source
+
+  power-domains:
+    items:
+      - description: A phandle to the MX power-domain
+      - description: A phandle to the CX power-domain
+      - description: A phandle to the GMXC power-domain
+
+  required-opps:
+    items:
+      - description: MX performance points
+      - description: CX performance points
+      - description: GMXC performance points
+
+required:
+  - compatible
+  - clocks
+  - power-domains
+  - required-opps
+  - '#power-domain-cells'
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,hawi-gcc.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
+
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        clock-controller@3d90000 {
+            compatible = "qcom,hawi-gpucc";
+            reg = <0x0 0x03d90000 0x0 0x9800>;
+            clocks = <&bi_tcxo_div2>,
+                     <&gcc GCC_GPU_GPLL0_CLK_SRC>,
+                     <&gcc GCC_GPU_GPLL0_DIV_CLK_SRC>;
+            power-domains = <&rpmhpd RPMHPD_MX>,
+                            <&rpmhpd RPMHPD_CX>,
+                            <&rpmhpd RPMHPD_GMXC>;
+            required-opps = <&rpmhpd_opp_low_svs>,
+                            <&rpmhpd_opp_low_svs>,
+                            <&rpmhpd_opp_low_svs_d3>;
+            #clock-cells = <1>;
+            #reset-cells = <1>;
+            #power-domain-cells = <1>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml
index a4f9af8fa187b2..91e17f126a9bad 100644
--- a/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml
@@ -25,6 +25,7 @@ properties:
   compatible:
     enum:
       - qcom,ipq5018-cmn-pll
+      - qcom,ipq5210-cmn-pll
       - qcom,ipq5332-cmn-pll
       - qcom,ipq5424-cmn-pll
       - qcom,ipq6018-cmn-pll
diff --git a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
index e868963f659b65..94f84d5294c6df 100644
--- a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
@@ -4,25 +4,32 @@
 $id: http://devicetree.org/schemas/clock/qcom,kaanapali-gxclkctl.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Qualcomm Graphics power domain Controller on Kaanapali
+title: Qualcomm Graphics GX Power Domain Controller
 
 maintainers:
   - Taniya Das <taniya.das@oss.qualcomm.com>
 
 description: |
-  Qualcomm GX(graphics) is a clock controller which has PLLs, clocks and
-  Power domains (GDSC). This module provides the power domains control
-  of gxclkctl on Qualcomm SoCs which helps the recovery of Graphics subsystem.
+  The Qualcomm graphics GX clock controller (GXCLKCTL) contains PLLs, clocks
+  and power domains (GDSC). This module provides the power domain control of
+  GXCLKCTL on Qualcomm SoCs, which helps the recovery of the graphics
+  subsystem.
 
   See also:
     include/dt-bindings/clock/qcom,kaanapali-gxclkctl.h
 
 properties:
   compatible:
-    enum:
-      - qcom,glymur-gxclkctl
-      - qcom,kaanapali-gxclkctl
-      - qcom,sm8750-gxclkctl
+    oneOf:
+      - enum:
+          - qcom,glymur-gxclkctl
+          - qcom,kaanapali-gxclkctl
+          - qcom,sm8750-gxclkctl
+      - items:
+          - enum:
+              - qcom,hawi-gxclkctl
+              - qcom,maili-gxclkctl
+          - const: qcom,kaanapali-gxclkctl
 
   power-domains:
     description:
diff --git a/Documentation/devicetree/bindings/clock/qcom,kuno-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,kuno-gcc.yaml
new file mode 100644
index 00000000000000..7ae37033a1af7d
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,kuno-gcc.yaml
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,kuno-gcc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Global Clock & Reset Controller on Kuno
+
+maintainers:
+  - Hardeep Sharma <hardeep.sharma@oss.qualcomm.com>
+
+description: |
+  Qualcomm global clock control module provides the clocks and resets on Kuno.
+
+  See also: include/dt-bindings/clock/qcom,kuno-gcc.h
+
+properties:
+  compatible:
+    const: qcom,kuno-gcc
+
+  clocks:
+    items:
+      - description: Board XO source
+      - description: Board active-only XO source
+      - description: Sleep clock source
+      - description: PCIe pipe clock source
+
+required:
+  - compatible
+  - clocks
+  - '#power-domain-cells'
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,rpmh.h>
+    clock-controller@80000 {
+        compatible = "qcom,kuno-gcc";
+        reg = <0x00080000 0x1f4200>;
+        clocks = <&rpmhcc RPMH_CXO_CLK>,
+                 <&rpmhcc RPMH_CXO_CLK_A>,
+                 <&sleep_clk>,
+                 <&pcie_pipe_clk>;
+        #clock-cells = <1>;
+        #reset-cells = <1>;
+        #power-domain-cells = <1>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml
index 707b25d2c11e60..0655cea72524e3 100644
--- a/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,milos-camcc.yaml
@@ -8,16 +8,23 @@ title: Qualcomm Camera Clock & Reset Controller on Milos
 
 maintainers:
   - Luca Weiss <luca.weiss@fairphone.com>
+  - Taniya Das <taniya.das@oss.qualcomm.com>
 
 description: |
   Qualcomm camera clock control module provides the clocks, resets and power
   domains on Milos.
 
-  See also: include/dt-bindings/clock/qcom,milos-camcc.h
+  See also:
+    include/dt-bindings/clock/qcom,eliza-cambistmclkcc.h
+    include/dt-bindings/clock/qcom,eliza-camcc.h
+    include/dt-bindings/clock/qcom,milos-camcc.h
 
 properties:
   compatible:
-    const: qcom,milos-camcc
+    enum:
+      - qcom,eliza-cambistmclkcc
+      - qcom,eliza-camcc
+      - qcom,milos-camcc
 
   clocks:
     items:
@@ -29,12 +36,33 @@ properties:
     items:
       - description: Interconnect path to enable the MultiMedia NoC
 
+  power-domains:
+    items:
+      - description: CX power domain
+      - description: MX power domain
+
+  required-opps:
+    items:
+      - description: CX performance point
+      - description: MX performance point
+
 required:
   - compatible
   - clocks
+  - power-domains
+  - required-opps
 
 allOf:
   - $ref: qcom,gcc.yaml#
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              const: qcom,eliza-cambistmclkcc
+    then:
+      required:
+        - '#power-domain-cells'
 
 unevaluatedProperties: false
 
@@ -43,6 +71,7 @@ examples:
     #include <dt-bindings/clock/qcom,milos-gcc.h>
     #include <dt-bindings/interconnect/qcom,icc.h>
     #include <dt-bindings/interconnect/qcom,milos-rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@adb0000 {
         compatible = "qcom,milos-camcc";
         reg = <0x0adb0000 0x40000>;
@@ -51,6 +80,12 @@ examples:
                  <&gcc GCC_CAMERA_AHB_CLK>;
         interconnects = <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS
                          &mmss_noc SLAVE_MNOC_HF_MEM_NOC QCOM_ICC_TAG_ALWAYS>;
+
+        power-domains = <&rpmhpd RPMHPD_CX>,
+                        <&rpmhpd RPMHPD_MX>;
+        required-opps = <&rpmhpd_opp_low_svs>,
+                        <&rpmhpd_opp_low_svs>;
+
         #clock-cells = <1>;
         #reset-cells = <1>;
         #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,milos-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,milos-videocc.yaml
index 14c31efe1308aa..08898f1d74e4ef 100644
--- a/Documentation/devicetree/bindings/clock/qcom,milos-videocc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,milos-videocc.yaml
@@ -8,16 +8,21 @@ title: Qualcomm Video Clock & Reset Controller on Milos
 
 maintainers:
   - Luca Weiss <luca.weiss@fairphone.com>
+  - Taniya Das <taniya.das@oss.qualcomm.com>
 
 description: |
   Qualcomm video clock control module provides the clocks, resets and power
   domains on Milos.
 
-  See also: include/dt-bindings/clock/qcom,milos-videocc.h
+  See also:
+    include/dt-bindings/clock/qcom,eliza-videocc.h
+    include/dt-bindings/clock/qcom,milos-videocc.h
 
 properties:
   compatible:
-    const: qcom,milos-videocc
+    enum:
+      - qcom,eliza-videocc
+      - qcom,milos-videocc
 
   clocks:
     items:
@@ -26,9 +31,22 @@ properties:
       - description: Sleep clock source
       - description: Video AHB clock from GCC
 
+  power-domains:
+    items:
+      - description: CX power domain
+      - description: MX power domain
+
+  required-opps:
+    items:
+      - description: CX performance point
+      - description: MX performance point
+
 required:
   - compatible
   - clocks
+  - power-domains
+  - required-opps
+  - '#power-domain-cells'
 
 allOf:
   - $ref: qcom,gcc.yaml#
@@ -38,6 +56,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,milos-gcc.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@aaf0000 {
         compatible = "qcom,milos-videocc";
         reg = <0x0aaf0000 0x10000>;
@@ -45,6 +64,12 @@ examples:
                  <&bi_tcxo_ao_div2>,
                  <&sleep_clk>,
                  <&gcc GCC_VIDEO_AHB_CLK>;
+
+        power-domains = <&rpmhpd RPMHPD_CX>,
+                        <&rpmhpd RPMHPD_MX>;
+        required-opps = <&rpmhpd_opp_low_svs>,
+                        <&rpmhpd_opp_low_svs>;
+
         #clock-cells = <1>;
         #reset-cells = <1>;
         #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,qcs615-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,qcs615-gcc.yaml
index 4a828e102d25ba..c872485315637e 100644
--- a/Documentation/devicetree/bindings/clock/qcom,qcs615-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,qcs615-gcc.yaml
@@ -31,10 +31,15 @@ properties:
       - const: bi_tcxo_ao
       - const: sleep_clk
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
   - clock-names
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -45,6 +50,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,qcs615-gcc";
       reg = <0x00100000 0x1f0000>;
@@ -52,6 +58,7 @@ examples:
                <&rpmhcc RPMH_CXO_CLK_A>,
                <&sleep_clk>;
       clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk";
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,qcs8300-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,qcs8300-gcc.yaml
index 081bc452081f96..a2f6e97f184818 100644
--- a/Documentation/devicetree/bindings/clock/qcom,qcs8300-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,qcs8300-gcc.yaml
@@ -33,9 +33,14 @@ properties:
       - description: UFS Phy Tx symbol 0 clock source
       - description: USB3 Phy wrapper pipe clock source
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -46,6 +51,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,qcs8300-gcc";
       reg = <0x00100000 0xc7018>;
@@ -59,6 +65,7 @@ examples:
                <&ufs_phy_rx_symbol_1_clk>,
                <&ufs_phy_tx_symbol_0_clk>,
                <&usb3_phy_wrapper_gcc_usb30_prim_pipe_clk>;
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml
index 17332aa558862e..aa532f208f5af3 100644
--- a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml
@@ -56,6 +56,11 @@ properties:
               - qcom,rpmcc-shikra
           - const: qcom,rpmcc-qcm2290
           - const: qcom,rpmcc
+      - items:
+          - enum:
+              - qcom,rpmcc-msm8952
+          - const: qcom,rpmcc-msm8976
+          - const: qcom,rpmcc
 
   '#clock-cells':
     const: 1
diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
index 2b446aca5207c9..5c9b653bc2aa59 100644
--- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
@@ -22,6 +22,7 @@ properties:
           - qcom,glymur-rpmh-clk
           - qcom,hawi-rpmh-clk
           - qcom,kaanapali-rpmh-clk
+          - qcom,kuno-rpmh-clk
           - qcom,milos-rpmh-clk
           - qcom,nord-rpmh-clk
           - qcom,qcs615-rpmh-clk
@@ -39,6 +40,7 @@ properties:
           - qcom,sdx75-rpmh-clk
           - qcom,sm4450-rpmh-clk
           - qcom,sm6350-rpmh-clk
+          - qcom,sm7250-rpmh-clk
           - qcom,sm8150-rpmh-clk
           - qcom,sm8250-rpmh-clk
           - qcom,sm8350-rpmh-clk
diff --git a/Documentation/devicetree/bindings/clock/qcom,sdx75-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sdx75-gcc.yaml
index 29a0b29bcb8161..75a38e9f32483b 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sdx75-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sdx75-gcc.yaml
@@ -38,9 +38,14 @@ properties:
       - description: PCIE Pipe clock source
       - description: USB3 phy wrapper pipe clock source
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -51,6 +56,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@80000 {
       compatible = "qcom,sdx75-gcc";
       reg = <0x80000 0x1f7400>;
@@ -59,6 +65,7 @@ examples:
                <&emac1_sgmiiphy_mac_rclk>, <&emac1_sgmiiphy_mac_tclk>, <&emac1_sgmiiphy_rclk>,
                <&emac1_sgmiiphy_tclk>, <&pcie20_phy_aux_clk>, <&pcie_1_pipe_clk>,
                <&pcie_2_pipe_clk>, <&pcie_pipe_clk>, <&usb3_phy_wrapper_gcc_usb30_pipe_clk>;
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm4450-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm4450-gcc.yaml
index 9cfe859bacc939..e143300e7be920 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm4450-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm4450-gcc.yaml
@@ -29,9 +29,14 @@ properties:
       - description: UFS Phy Tx symbol 0 clock source
       - description: USB3 Phy wrapper pipe clock source
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -42,12 +47,14 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,sm4450-gcc";
       reg = <0x00100000 0x001f4200>;
       clocks = <&rpmhcc RPMH_CXO_CLK>, <&sleep_clk>,
                <&ufs_mem_phy 0>, <&ufs_mem_phy 1>,
                <&ufs_mem_phy 2>, <&usb_1_qmpphy>;
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm6375-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm6375-gcc.yaml
index 66dfa72fa9751e..b4ad2ad14d85e5 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm6375-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm6375-gcc.yaml
@@ -4,10 +4,13 @@
 $id: http://devicetree.org/schemas/clock/qcom,sm6375-gcc.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Qualcomm Global Clock & Reset Controller on SM6375
+title: Qualcomm Global Clock & Reset Controller on SM6375 and SM7150
 
 maintainers:
   - Konrad Dybcio <konradybcio@kernel.org>
+  - Bjorn Andersson <andersson@kernel.org>
+  - Danila Tikhonov <danila@jiaxyga.com>
+  - David Wronek <davidwronek@gmail.com>
 
 description: |
   Qualcomm global clock control module provides the clocks, resets and power
@@ -20,7 +23,9 @@ allOf:
 
 properties:
   compatible:
-    const: qcom,sm6375-gcc
+    enum:
+      - qcom,sm6375-gcc
+      - qcom,sm7150-gcc
 
   clocks:
     items:
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm7150-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm7150-gcc.yaml
deleted file mode 100644
index 3878808f811ee2..00000000000000
--- a/Documentation/devicetree/bindings/clock/qcom,sm7150-gcc.yaml
+++ /dev/null
@@ -1,53 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/clock/qcom,sm7150-gcc.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Qualcomm Global Clock & Reset Controller on SM7150
-
-maintainers:
-  - Bjorn Andersson <andersson@kernel.org>
-  - Danila Tikhonov <danila@jiaxyga.com>
-  - David Wronek <davidwronek@gmail.com>
-
-description: |
-  Qualcomm global clock control module provides the clocks, resets and power
-  domains on SM7150
-
-  See also: include/dt-bindings/clock/qcom,sm7150-gcc.h
-
-properties:
-  compatible:
-    const: qcom,sm7150-gcc
-
-  clocks:
-    items:
-      - description: Board XO source
-      - description: Board XO Active-Only source
-      - description: Sleep clock source
-
-required:
-  - compatible
-  - clocks
-  - '#power-domain-cells'
-
-allOf:
-  - $ref: qcom,gcc.yaml#
-
-unevaluatedProperties: false
-
-examples:
-  - |
-    #include <dt-bindings/clock/qcom,rpmh.h>
-    clock-controller@100000 {
-      compatible = "qcom,sm7150-gcc";
-      reg = <0x00100000 0x001f0000>;
-      clocks = <&rpmhcc RPMH_CXO_CLK>,
-               <&rpmhcc RPMH_CXO_CLK_A>,
-               <&sleep_clk>;
-      #clock-cells = <1>;
-      #reset-cells = <1>;
-      #power-domain-cells = <1>;
-    };
-...
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml
index b41247b257b193..64f5002931f240 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-camcc.yaml
@@ -16,8 +16,10 @@ description: |
   domains on SM8450.
 
   See also:
+    include/dt-bindings/clock/qcom,hawi-camcc.h
     include/dt-bindings/clock/qcom,kaanapali-camcc.h
     include/dt-bindings/clock/qcom,kaanapali-cambistmclkcc.h
+    include/dt-bindings/clock/qcom,nord-camcc.h
     include/dt-bindings/clock/qcom,sm8450-camcc.h
     include/dt-bindings/clock/qcom,sm8550-camcc.h
     include/dt-bindings/clock/qcom,sm8650-camcc.h
@@ -26,15 +28,24 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - qcom,kaanapali-cambistmclkcc
-      - qcom,kaanapali-camcc
-      - qcom,sm8450-camcc
-      - qcom,sm8475-camcc
-      - qcom,sm8550-camcc
-      - qcom,sm8650-camcc
-      - qcom,sm8750-cambistmclkcc
-      - qcom,sm8750-camcc
+    oneOf:
+      - enum:
+          - qcom,hawi-camcc
+          - qcom,kaanapali-cambistmclkcc
+          - qcom,kaanapali-camcc
+          - qcom,maili-camcc
+          - qcom,nord-camcc
+          - qcom,sm8450-camcc
+          - qcom,sm8475-camcc
+          - qcom,sm8550-camcc
+          - qcom,sm8650-camcc
+          - qcom,sm8750-cambistmclkcc
+          - qcom,sm8750-camcc
+      - items:
+          - enum:
+              - qcom,hawi-cambistmclkcc
+              - qcom,maili-cambistmclkcc
+          - const: qcom,kaanapali-cambistmclkcc
 
   clocks:
     items:
@@ -43,6 +54,10 @@ properties:
       - description: Board active XO source
       - description: Sleep clock source
 
+  interconnects:
+    items:
+      - description: Interconnect path to enable the MultiMedia NoC
+
   power-domains:
     description:
       Power domains required for the clock controller to operate
@@ -78,12 +93,24 @@ allOf:
     then:
       required:
         - required-opps
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,hawi-camcc
+              - qcom,maili-camcc
+    then:
+      required:
+        - interconnects
 
 unevaluatedProperties: false
 
 examples:
   - |
     #include <dt-bindings/clock/qcom,gcc-sm8450.h>
+    #include <dt-bindings/interconnect/qcom,icc.h>
+    #include <dt-bindings/interconnect/qcom,sm8450.h>
     #include <dt-bindings/clock/qcom,rpmh.h>
     #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@ade0000 {
@@ -97,6 +124,8 @@ examples:
                       <&rpmhpd RPMHPD_MXC>;
       required-opps = <&rpmhpd_opp_low_svs>,
                       <&rpmhpd_opp_low_svs>;
+      interconnects = <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS
+                      &mmss_noc SLAVE_MNOC_HF_MEM_NOC QCOM_ICC_TAG_ALWAYS>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml
index 2a2d37c59442de..7eca456211e476 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-gpucc.yaml
@@ -15,6 +15,7 @@ description: |
   domains on Qualcomm SoCs.
 
   See also:
+    include/dt-bindings/clock/qcom,eliza-gpucc.h
     include/dt-bindings/clock/qcom,glymur-gpucc.h
     include/dt-bindings/clock/qcom,kaanapali-gpucc.h
     include/dt-bindings/clock/qcom,milos-gpucc.h
@@ -32,6 +33,7 @@ description: |
 properties:
   compatible:
     enum:
+      - qcom,eliza-gpucc
       - qcom,glymur-gpucc
       - qcom,kaanapali-gpucc
       - qcom,milos-gpucc
@@ -75,6 +77,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,eliza-gpucc
               - qcom,sm8750-gpucc
     then:
       required:
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
index 910c6d5672b865..e985a450451d5e 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-videocc.yaml
@@ -19,6 +19,7 @@ description: |
     include/dt-bindings/clock/qcom,hawi-videocc.h
     include/dt-bindings/clock/qcom,kaanapali-videocc.h
     include/dt-bindings/clock/qcom,maili-videocc.h
+    include/dt-bindings/clock/qcom,nord-videocc.h
     include/dt-bindings/clock/qcom,sm8450-videocc.h
     include/dt-bindings/clock/qcom,sm8650-videocc.h
     include/dt-bindings/clock/qcom,sm8750-videocc.h
@@ -31,6 +32,7 @@ properties:
       - qcom,hawi-videocc
       - qcom,kaanapali-videocc
       - qcom,maili-videocc
+      - qcom,nord-videocc
       - qcom,sm8450-videocc
       - qcom,sm8475-videocc
       - qcom,sm8550-videocc
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8550-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8550-gcc.yaml
index c4e9b9bb63f51b..f9423aff5dea2f 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8550-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8550-gcc.yaml
@@ -31,9 +31,14 @@ properties:
       - description: UFS Phy Tx symbol 0 clock source
       - description: USB3 Phy wrapper pipe clock source
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -44,6 +49,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,sm8550-gcc";
       reg = <0x00100000 0x001f4200>;
@@ -55,6 +61,7 @@ examples:
                <&ufs_mem_phy 1>,
                <&ufs_mem_phy 2>,
                <&usb_1_qmpphy>;
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
index 1f3332275c1728..f199655790daf6 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
@@ -36,7 +36,6 @@ properties:
               - qcom,sm8550-tcsr
               - qcom,sm8650-tcsr
               - qcom,sm8750-tcsr
-              - qcom,x1e80100-tcsr
           - const: syscon
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8650-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8650-gcc.yaml
index c7143e2abc80c2..543105b277d563 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8650-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8650-gcc.yaml
@@ -32,9 +32,14 @@ properties:
       - description: UFS Phy Tx symbol 0 clock source
       - description: USB3 Phy wrapper pipe clock source
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -45,6 +50,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,sm8650-gcc";
       reg = <0x00100000 0x001f4200>;
@@ -58,6 +64,7 @@ examples:
                <&ufs_mem_phy 1>,
                <&ufs_mem_phy 2>,
                <&usb_1_qmpphy>;
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml
index 0114d347b26ff0..59da3eb2b07382 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8750-gcc.yaml
@@ -34,9 +34,14 @@ properties:
       - description: UFS Phy Tx symbol 0 clock source
       - description: USB3 Phy wrapper pipe clock source
 
+  power-domains:
+    items:
+      - description: CX power domain
+
 required:
   - compatible
   - clocks
+  - power-domains
   - '#power-domain-cells'
 
 allOf:
@@ -47,6 +52,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
     clock-controller@100000 {
       compatible = "qcom,sm8750-gcc";
       reg = <0x00100000 0x001f4200>;
@@ -58,6 +64,7 @@ examples:
                <&ufs_mem_phy 1>,
                <&ufs_mem_phy 2>,
                <&usb_1_qmpphy>;
+      power-domains = <&rpmhpd RPMHPD_CX>;
       #clock-cells = <1>;
       #reset-cells = <1>;
       #power-domain-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,x1e80100-tcsr.yaml b/Documentation/devicetree/bindings/clock/qcom,x1e80100-tcsr.yaml
new file mode 100644
index 00000000000000..8163c679fc54ae
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,x1e80100-tcsr.yaml
@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,x1e80100-tcsr.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm TCSR Clock Controller on X1E80100
+
+maintainers:
+  - Bjorn Andersson <andersson@kernel.org>
+  - Taniya Das <taniya.das@oss.qualcomm.com>
+
+description: |
+  Qualcomm TCSR clock control module provides the clocks on X1E80100
+
+  See also:
+  - include/dt-bindings/clock/qcom,x1e80100-tcsr.h
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - qcom,x1e80100-tcsr
+      - const: syscon
+
+  clocks:
+    items:
+      - description: TCXO pad clock
+
+  reg:
+    maxItems: 1
+
+  '#clock-cells':
+    const: 1
+
+  vdda-qrefrpt0-0p9-supply: true
+  vdda-qrefrpt1-0p9-supply: true
+  vdda-qrefrpt2-0p9-supply: true
+  vdda-qrefrpt3-0p9-supply: true
+  vdda-qrefrpt4-0p9-supply: true
+  vdda-qrefrx0-0p9-supply: true
+  vdda-qrefrx1-0p9-supply: true
+  vdda-qrefrx2-0p9-supply: true
+  vdda-qrefrx3-0p9-supply: true
+  vdda-qrefrx4-0p9-supply: true
+  vdda-qreftx0-0p9-supply: true
+  vdda-qreftx0-1p2-supply: true
+  vdda-qreftx1-0p9-supply: true
+  vdda-qreftx1-1p2-supply: true
+  vdda-refgen0-0p9-supply: true
+  vdda-refgen0-1p2-supply: true
+  vdda-refgen2-0p9-supply: true
+  vdda-refgen2-1p2-supply: true
+
+required:
+  - compatible
+  - clocks
+  - '#clock-cells'
+  - vdda-qrefrpt0-0p9-supply
+  - vdda-qrefrpt1-0p9-supply
+  - vdda-qrefrpt2-0p9-supply
+  - vdda-qrefrpt3-0p9-supply
+  - vdda-qrefrpt4-0p9-supply
+  - vdda-qrefrx0-0p9-supply
+  - vdda-qrefrx1-0p9-supply
+  - vdda-qrefrx2-0p9-supply
+  - vdda-qrefrx3-0p9-supply
+  - vdda-qrefrx4-0p9-supply
+  - vdda-qreftx0-0p9-supply
+  - vdda-qreftx0-1p2-supply
+  - vdda-qreftx1-0p9-supply
+  - vdda-qreftx1-1p2-supply
+  - vdda-refgen0-0p9-supply
+  - vdda-refgen0-1p2-supply
+  - vdda-refgen2-0p9-supply
+  - vdda-refgen2-1p2-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,rpmh.h>
+
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      clock-controller@1fc0000 {
+        compatible = "qcom,x1e80100-tcsr", "syscon";
+        reg = <0x0 0x1fc0000 0x0 0x30000>;
+        clocks = <&rpmhcc RPMH_CXO_CLK>;
+        #clock-cells = <1>;
+        vdda-qrefrpt0-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrpt1-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrpt2-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrpt3-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrpt4-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrx0-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrx1-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrx2-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrx3-0p9-supply = <&vreg_l1a>;
+        vdda-qrefrx4-0p9-supply = <&vreg_l1a>;
+        vdda-qreftx0-0p9-supply = <&vreg_l1a>;
+        vdda-qreftx0-1p2-supply = <&vreg_l2a>;
+        vdda-qreftx1-0p9-supply = <&vreg_l1a>;
+        vdda-qreftx1-1p2-supply = <&vreg_l2a>;
+        vdda-refgen0-0p9-supply = <&vreg_l1a>;
+        vdda-refgen0-1p2-supply = <&vreg_l2a>;
+        vdda-refgen2-0p9-supply = <&vreg_l1a>;
+        vdda-refgen2-1p2-supply = <&vreg_l2a>;
+      };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos5515-cmu.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos5515-cmu.yaml
new file mode 100644
index 00000000000000..1be8962cda1c8a
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/samsung,exynos5515-cmu.yaml
@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/samsung,exynos5515-cmu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Samsung Exynos5515 SoC clock controller
+
+maintainers:
+  - Aiden Isik <aidenisik+git@member.fsf.org>
+  - Chanwoo Choi <cw00.choi@samsung.com>
+  - Krzysztof Kozlowski <krzk@kernel.org>
+
+description: |
+  Exynos5515 clock controller is comprised of several CMU units, generating
+  clocks for different domains. Those CMU units are modeled as separate device
+  tree nodes, and might depend on each other. The root clock in that root tree
+  is an external clock: OSCCLK (26 MHz). This external clock must be defined
+  as a fixed-rate clock in dts.
+
+  CMU_TOP is a top-level CMU, where all base clocks are prepared using PLLs and
+  dividers; all other clocks of function blocks (other CMUs) are usually
+  derived from CMU_TOP.
+
+  Each clock is assigned an identifier and client nodes can use this identifier
+  to specify the clock which they consume. All clocks available for usage
+  in clock consumer nodes are defined as preprocessor macros in
+  'include/dt-bindings/clock/samsung,exynos5515-cmu.h' header.
+
+properties:
+  compatible:
+    enum:
+      - samsung,exynos5515-cmu-fsys
+      - samsung,exynos5515-cmu-peri
+      - samsung,exynos5515-cmu-top
+
+  clocks:
+    minItems: 1
+    maxItems: 5
+
+  clock-names:
+    minItems: 1
+    maxItems: 5
+
+  "#clock-cells":
+    const: 1
+
+  reg:
+    maxItems: 1
+
+  samsung,sysreg:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      Phandle to system registers interface.
+
+required:
+  - compatible
+  - clocks
+  - clock-names
+  - "#clock-cells"
+  - reg
+  - samsung,sysreg
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - samsung,exynos5515-cmu-fsys
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26MHz)
+            - description: CMU_FSYS BUS clock (from CMU_TOP)
+            - description: CMU_FSYS MMC card clock (from CMU_TOP)
+            - description: CMU_FSYS MMC embedded clock (from CMU_TOP)
+            - description: CMU_FSYS USB20DRD clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: bus
+            - const: mmc_card
+            - const: mmc_embd
+            - const: usb20drd
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - samsung,exynos5515-cmu-peri
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26MHz)
+            - description: CMU_PERI BUS clock (from CMU_TOP)
+            - description: CMU_PERI IP clock (from CMU_TOP)
+            - description: CMU_PERI UART clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: bus
+            - const: ip
+            - const: uart
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: samsung,exynos5515-cmu-top
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26 MHz)
+
+        clock-names:
+          items:
+            - const: oscclk
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/samsung,exynos5515-cmu.h>
+
+    cmu_top: clock-controller@15410000 {
+        compatible = "samsung,exynos5515-cmu-top";
+        reg = <0x15410000 0x10000>;
+        #clock-cells = <1>;
+
+        clocks = <&oscclk>;
+        clock-names = "oscclk";
+        samsung,sysreg = <&sysreg_top>;
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/clock/ti,am62-audio-refclk.yaml b/Documentation/devicetree/bindings/clock/ti,am62-audio-refclk.yaml
index b2e40bd39a3af4..6c8fb0793070db 100644
--- a/Documentation/devicetree/bindings/clock/ti,am62-audio-refclk.yaml
+++ b/Documentation/devicetree/bindings/clock/ti,am62-audio-refclk.yaml
@@ -11,8 +11,12 @@ maintainers:
 
 properties:
   compatible:
-    items:
-      - const: ti,am62-audio-refclk
+    oneOf:
+      - items:
+          - const: ti,am62-audio-refclk
+      - items:
+          - const: ti,j721s2-audio-refclk
+          - const: ti,am62-audio-refclk
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/clock/ti,palmas-clk32kg.yaml b/Documentation/devicetree/bindings/clock/ti,palmas-clk32kg.yaml
new file mode 100644
index 00000000000000..c2a49be9b95acd
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/ti,palmas-clk32kg.yaml
@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/ti,palmas-clk32kg.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments Palmas 32KHz Clocks
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+description:
+  The Palmas device family has two clock output pins for 32KHz,
+  KG and KG_AUDIO.
+
+allOf:
+  - $ref: /schemas/clock/clock.yaml#
+
+properties:
+  compatible:
+    enum:
+      - ti,palmas-clk32kg
+      - ti,palmas-clk32kgaudio
+
+  "#clock-cells":
+    const: 0
+
+  ti,external-sleep-control:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [ 0, 1, 2, 3 ]
+    description: |
+      The external enable input pins control the enable/disable of clocks.
+      The valid values (defined in dt-bindings/mfd/palmas.h) are:
+        1: PALMAS_EXT_CONTROL_PIN_ENABLE1
+        2: PALMAS_EXT_CONTROL_PIN_ENABLE2
+        3: PALMAS_EXT_CONTROL_PIN_NSLEEP
+      Option 0 or missing this property means the clock is enabled/disabled
+      via register access and these pins do not have any control.
+
+required:
+  - compatible
+  - "#clock-cells"
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/mfd/palmas.h>
+
+    pmic {
+        clock-controller {
+            compatible = "ti,palmas-clk32kg";
+            #clock-cells = <0>;
+            ti,external-sleep-control = <PALMAS_EXT_CONTROL_PIN_NSLEEP>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/clock/ti/adpll.txt b/Documentation/devicetree/bindings/clock/ti/adpll.txt
deleted file mode 100644
index 3122360adcf3c0..00000000000000
--- a/Documentation/devicetree/bindings/clock/ti/adpll.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-Binding for Texas Instruments ADPLL clock.
-
-This binding uses the common clock binding[1]. It assumes a
-register-mapped ADPLL with two to three selectable input clocks
-and three to four children.
-
-[1] Documentation/devicetree/bindings/clock/clock-bindings.txt
-
-Required properties:
-- compatible : shall be one of "ti,dm814-adpll-s-clock" or
-  "ti,dm814-adpll-lj-clock" depending on the type of the ADPLL
-- #clock-cells : from common clock binding; shall be set to 1.
-- clocks : link phandles of parent clocks clkinp and clkinpulow, note
-  that the adpll-s-clock also has an optional clkinphif
-- reg : address and length of the register set for controlling the ADPLL.
-
-Examples:
-	adpll_mpu_ck: adpll@40 {
-		#clock-cells = <1>;
-		compatible = "ti,dm814-adpll-s-clock";
-		reg = <0x40 0x40>;
-		clocks = <&devosc_ck &devosc_ck &devosc_ck>;
-		clock-names = "clkinp", "clkinpulow", "clkinphif";
-		clock-output-names = "481c5040.adpll.dcoclkldo",
-				     "481c5040.adpll.clkout",
-				     "481c5040.adpll.clkoutx2",
-				     "481c5040.adpll.clkouthif";
-	};
-
-	adpll_dsp_ck: adpll@80 {
-		#clock-cells = <1>;
-		compatible = "ti,dm814-adpll-lj-clock";
-		reg = <0x80 0x30>;
-		clocks = <&devosc_ck &devosc_ck>;
-		clock-names = "clkinp", "clkinpulow";
-		clock-output-names = "481c5080.adpll.dcoclkldo",
-				     "481c5080.adpll.clkout",
-				     "481c5080.adpll.clkoutldo";
-	};
diff --git a/Documentation/devicetree/bindings/clock/ti/davinci/pll.txt b/Documentation/devicetree/bindings/clock/ti/davinci/pll.txt
deleted file mode 100644
index c9894538315b1a..00000000000000
--- a/Documentation/devicetree/bindings/clock/ti/davinci/pll.txt
+++ /dev/null
@@ -1,96 +0,0 @@
-Binding for TI DaVinci PLL Controllers
-
-The PLL provides clocks to most of the components on the SoC. In addition
-to the PLL itself, this controller also contains bypasses, gates, dividers,
-an multiplexers for various clock signals.
-
-Required properties:
-- compatible: shall be one of:
-	- "ti,da850-pll0" for PLL0 on DA850/OMAP-L138/AM18XX
-	- "ti,da850-pll1" for PLL1 on DA850/OMAP-L138/AM18XX
-- reg: physical base address and size of the controller's register area.
-- clocks: phandles corresponding to the clock names
-- clock-names: names of the clock sources - depends on compatible string
-	- for "ti,da850-pll0", shall be "clksrc", "extclksrc"
-	- for "ti,da850-pll1", shall be "clksrc"
-
-Optional properties:
-- ti,clkmode-square-wave: Indicates that the board is supplying a square
-	wave input on the OSCIN pin instead of using a crystal oscillator.
-	This property is only valid when compatible = "ti,da850-pll0".
-
-
-Optional child nodes:
-
-pllout
-	Describes the main PLL clock output (before POSTDIV). The node name must
-	be "pllout".
-
-	Required properties:
-	- #clock-cells: shall be 0
-
-sysclk
-	Describes the PLLDIVn divider clocks that provide the SYSCLKn clock
-	domains. The node name must be "sysclk". Consumers of this node should
-	use "n" in "SYSCLKn" as the index parameter for the clock cell.
-
-	Required properties:
-	- #clock-cells: shall be 1
-
-auxclk
-	Describes the AUXCLK output of the PLL. The node name must be "auxclk".
-	This child node is only valid when compatible = "ti,da850-pll0".
-
-	Required properties:
-	- #clock-cells: shall be 0
-
-obsclk
-	Describes the OBSCLK output of the PLL. The node name must be "obsclk".
-
-	Required properties:
-	- #clock-cells: shall be 0
-
-
-Examples:
-
-	pll0: clock-controller@11000 {
-		compatible = "ti,da850-pll0";
-		reg = <0x11000 0x1000>;
-		clocks = <&ref_clk>, <&pll1_sysclk 3>;
-		clock-names = "clksrc", "extclksrc";
-		ti,clkmode-square-wave;
-
-		pll0_pllout: pllout {
-			#clock-cells = <0>;
-		};
-
-		pll0_sysclk: sysclk {
-			#clock-cells = <1>;
-		};
-
-		pll0_auxclk: auxclk {
-			#clock-cells = <0>;
-		};
-
-		pll0_obsclk: obsclk {
-			#clock-cells = <0>;
-		};
-	};
-
-	pll1: clock-controller@21a000 {
-		compatible = "ti,da850-pll1";
-		reg = <0x21a000 0x1000>;
-		clocks = <&ref_clk>;
-		clock-names = "clksrc";
-
-		pll0_sysclk: sysclk {
-			#clock-cells = <1>;
-		};
-
-		pll0_obsclk: obsclk {
-			#clock-cells = <0>;
-		};
-	};
-
-Also see:
-- Documentation/devicetree/bindings/clock/clock-bindings.txt
diff --git a/Documentation/devicetree/bindings/clock/ti/davinci/ti,da850-pll.yaml b/Documentation/devicetree/bindings/clock/ti/davinci/ti,da850-pll.yaml
new file mode 100644
index 00000000000000..c6563290a702b4
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/ti/davinci/ti,da850-pll.yaml
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/ti/davinci/ti,da850-pll.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI DaVinci PLL Controllers
+
+maintainers:
+  - David Lechner <david@lechnology.com>
+
+description:
+  The PLL provides clocks to most of the components on the SoC. In addition to
+  the PLL itself, this controller also contains bypasses, gates, dividers, an
+  multiplexers for various clock signals.
+
+properties:
+  compatible:
+    enum:
+      - ti,da850-pll0
+      - ti,da850-pll1
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    items:
+      - description: Primary reference clock input
+      - description: External clock source
+
+  clock-names:
+    minItems: 1
+    items:
+      - const: clksrc
+      - const: extclksrc
+
+  ti,clkmode-square-wave:
+    type: boolean
+    description:
+      Indicates that the board is supplying a square wave input on the OSCIN
+      pin instead of using a crystal oscillator.
+
+  pllout:
+    type: object
+    description:
+      Main PLL clock output before POSTDIV.
+
+    properties:
+      '#clock-cells':
+        const: 0
+
+    required:
+      - '#clock-cells'
+
+    additionalProperties: false
+
+  sysclk:
+    type: object
+    description:
+      PLLDIVn divider clocks that provide the SYSCLKn clock domains. Consumers
+      of this node should use "n" in "SYSCLKn" as the index parameter for the
+      clock cell.
+
+    properties:
+      '#clock-cells':
+        const: 1
+
+    required:
+      - '#clock-cells'
+
+    additionalProperties: false
+
+  auxclk:
+    type: object
+    description:
+      AUXCLK output of the PLL
+
+    properties:
+      '#clock-cells':
+        const: 0
+
+    required:
+      - '#clock-cells'
+
+    additionalProperties: false
+
+  obsclk:
+    type: object
+    description:
+      the OBSCLK output of the PLL.
+
+    properties:
+      '#clock-cells':
+        const: 0
+
+    required:
+      - '#clock-cells'
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+
+additionalProperties: false
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          const: ti,da850-pll1
+    then:
+      properties:
+        clocks:
+          maxItems: 1
+        clock-names:
+          maxItems: 1
+        ti,clkmode-square-wave: false
+        auxclk: false
+
+examples:
+  - |
+    clock-controller@11000 {
+        compatible = "ti,da850-pll0";
+        reg = <0x11000 0x1000>;
+        clocks = <&ref_clk>, <&pll1_sysclk 3>;
+        clock-names = "clksrc", "extclksrc";
+        ti,clkmode-square-wave;
+
+        pllout {
+            #clock-cells = <0>;
+        };
+
+        sysclk {
+            #clock-cells = <1>;
+        };
+
+        auxclk {
+            #clock-cells = <0>;
+        };
+
+        obsclk {
+            #clock-cells = <0>;
+        };
+    };
+
+  - |
+    clock-controller@21a000 {
+        compatible = "ti,da850-pll1";
+        reg = <0x21a000 0x1000>;
+        clocks = <&ref_clk>;
+        clock-names = "clksrc";
+
+        sysclk {
+            #clock-cells = <1>;
+        };
+
+        obsclk {
+            #clock-cells = <0>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/clock/ti/dra7-atl.txt b/Documentation/devicetree/bindings/clock/ti/dra7-atl.txt
deleted file mode 100644
index 68504079f99fa1..00000000000000
--- a/Documentation/devicetree/bindings/clock/ti/dra7-atl.txt
+++ /dev/null
@@ -1,94 +0,0 @@
-Device Tree Clock bindings for ATL (Audio Tracking Logic) of DRA7 SoC.
-
-The ATL IP is used to generate clock to be used to synchronize baseband and
-audio codec. A single ATL IP provides four ATL clock instances sharing the same
-functional clock but can be configured to provide different clocks.
-ATL can maintain a clock averages to some desired frequency based on the bws/aws
-signals - can compensate the drift between the two ws signal.
-
-In order to provide the support for ATL and its output clocks (which can be used
-internally within the SoC or external components) two sets of bindings is needed:
-
-Clock tree binding:
-This binding uses the common clock binding[1].
-To be able to integrate the ATL clocks with DT clock tree.
-Provides ccf level representation of the ATL clocks to be used by drivers.
-Since the clock instances are part of a single IP this binding is used as a node
-for the DT clock tree, the IP driver is needed to handle the actual configuration
-of the IP.
-
-[1] Documentation/devicetree/bindings/clock/clock-bindings.txt
-
-Required properties:
-- compatible : shall be "ti,dra7-atl-clock"
-- #clock-cells : from common clock binding; shall be set to 0.
-- clocks : link phandles to functional clock of ATL
-
-Binding for the IP driver:
-This binding is used to configure the IP driver which is going to handle the
-configuration of the IP for the ATL clock instances.
-
-Required properties:
-- compatible : shall be "ti,dra7-atl"
-- reg : base address for the ATL IP
-- ti,provided-clocks : List of phandles to the clocks associated with the ATL
-- clocks : link phandles to functional clock of ATL
-- clock-names : Shall be set to "fck"
-- ti,hwmods : Shall be set to "atl"
-
-Optional properties:
-Configuration of ATL instances:
-- atl{0/1/2/3} {
-	- bws : Baseband word select signal selection
-	- aws : Audio word select signal selection
-};
-
-For valid word select signals, see the dt-bindings/clock/ti-dra7-atl.h include
-file.
-
-Examples:
-/* clock bindings for atl provided clocks */
-atl_clkin0_ck: atl_clkin0_ck {
-	#clock-cells = <0>;
-	compatible = "ti,dra7-atl-clock";
-	clocks = <&atl_gfclk_mux>;
-};
-
-atl_clkin1_ck: atl_clkin1_ck {
-	#clock-cells = <0>;
-	compatible = "ti,dra7-atl-clock";
-	clocks = <&atl_gfclk_mux>;
-};
-
-atl_clkin2_ck: atl_clkin2_ck {
-	#clock-cells = <0>;
-	compatible = "ti,dra7-atl-clock";
-	clocks = <&atl_gfclk_mux>;
-};
-
-atl_clkin3_ck: atl_clkin3_ck {
-	#clock-cells = <0>;
-	compatible = "ti,dra7-atl-clock";
-	clocks = <&atl_gfclk_mux>;
-};
-
-/* binding for the IP */
-atl: atl@4843c000 {
-	compatible = "ti,dra7-atl";
-	reg = <0x4843c000 0x3ff>;
-	ti,hwmods = "atl";
-	ti,provided-clocks = <&atl_clkin0_ck>, <&atl_clkin1_ck>,
-				<&atl_clkin2_ck>, <&atl_clkin3_ck>;
-	clocks = <&atl_gfclk_mux>;
-	clock-names = "fck";
-};
-
-#include <dt-bindings/clock/ti-dra7-atl.h>
-
-&atl {
-
-	atl2 {
-		bws = <DRA7_ATL_WS_MCASP2_FSX>;
-		aws = <DRA7_ATL_WS_MCASP3_FSX>;
-	};
-};
diff --git a/Documentation/devicetree/bindings/clock/ti/fapll.txt b/Documentation/devicetree/bindings/clock/ti/fapll.txt
deleted file mode 100644
index 88986ef39ddd24..00000000000000
--- a/Documentation/devicetree/bindings/clock/ti/fapll.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-Binding for Texas Instruments FAPLL clock.
-
-This binding uses the common clock binding[1]. It assumes a
-register-mapped FAPLL with usually two selectable input clocks
-(reference clock and bypass clock), and one or more child
-syntesizers.
-
-[1] Documentation/devicetree/bindings/clock/clock-bindings.txt
-
-Required properties:
-- compatible : shall be "ti,dm816-fapll-clock"
-- #clock-cells : from common clock binding; shall be set to 0.
-- clocks : link phandles of parent clocks (clk-ref and clk-bypass)
-- reg : address and length of the register set for controlling the FAPLL.
-
-Examples:
-	main_fapll: main_fapll {
-		#clock-cells = <1>;
-		compatible = "ti,dm816-fapll-clock";
-		reg = <0x400 0x40>;
-		clocks = <&sys_clkin_ck &sys_clkin_ck>;
-		clock-indices = <1>, <2>, <3>, <4>, <5>,
-				<6>, <7>;
-		clock-output-names = "main_pll_clk1",
-				     "main_pll_clk2",
-				     "main_pll_clk3",
-				     "main_pll_clk4",
-				     "main_pll_clk5",
-				     "main_pll_clk6",
-				     "main_pll_clk7";
-	};
diff --git a/Documentation/devicetree/bindings/clock/ti/ti,dm814-adpll-clock.yaml b/Documentation/devicetree/bindings/clock/ti/ti,dm814-adpll-clock.yaml
new file mode 100644
index 00000000000000..9676ab2d870594
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/ti/ti,dm814-adpll-clock.yaml
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/ti/ti,dm814-adpll-clock.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments DM814x ADPLL Clock
+
+maintainers:
+  - Tony Lindgren <tony@atomide.com>
+
+description:
+  The ADPLL is a register-mapped clock generator found on TI DM814x SoCs. It
+  provides multiple output clocks from two or three selectable input clocks and
+  three to four children.
+
+properties:
+  compatible:
+    enum:
+      - ti,dm814-adpll-s-clock
+      - ti,dm814-adpll-lj-clock
+
+  reg:
+    maxItems: 1
+
+  '#clock-cells':
+    const: 1
+
+  clocks:
+    minItems: 2
+    maxItems: 3
+
+  clock-names:
+    minItems: 2
+    items:
+      - const: clkinp
+      - const: clkinpulow
+      - const: clkinphif
+
+  clock-output-names:
+    minItems: 3
+    maxItems: 4
+
+required:
+  - compatible
+  - reg
+  - '#clock-cells'
+  - clocks
+  - clock-output-names
+
+if:
+  properties:
+    compatible:
+      contains:
+        const: ti,dm814-adpll-s-clock
+then:
+  properties:
+    clocks:
+      minItems: 3
+    clock-names:
+      minItems: 3
+    clock-output-names:
+      minItems: 4
+
+else:
+  properties:
+    clocks:
+      maxItems: 2
+    clock-names:
+      maxItems: 2
+    clock-output-names:
+      maxItems: 3
+
+additionalProperties: false
+
+examples:
+  - |
+    adpll@40 {
+        compatible = "ti,dm814-adpll-s-clock";
+        reg = <0x40 0x40>;
+        #clock-cells = <1>;
+        clocks = <&devosc_ck &devosc_ck &devosc_ck>;
+        clock-names = "clkinp", "clkinpulow", "clkinphif";
+        clock-output-names = "481c5040.adpll.dcoclkldo",
+                             "481c5040.adpll.clkout",
+                             "481c5040.adpll.clkoutx2",
+                             "481c5040.adpll.clkouthif";
+    };
diff --git a/Documentation/devicetree/bindings/clock/ti/ti,dm816-fapll-clock.yaml b/Documentation/devicetree/bindings/clock/ti/ti,dm816-fapll-clock.yaml
new file mode 100644
index 00000000000000..423b7405e398e8
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/ti/ti,dm816-fapll-clock.yaml
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/ti/ti,dm816-fapll-clock.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments FAPLL clock
+
+maintainers:
+  - Tony Lindgren <tony@atomide.com>
+  - Tero Kristo <kristo@kernel.org>
+
+description:
+  FAPLL is register-mapped with usually two selectable input clocks
+  (reference clock and bypass clock), and one or more child synthesizers.
+
+properties:
+  compatible:
+    const: ti,dm816-fapll-clock
+
+  "#clock-cells":
+    const: 1
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: parent clock clk-ref
+      - description: parent clock clk-bypass
+
+  clock-indices:
+    description:
+      Maps child synthesizers to physical indices 1-7. Index 0 is physically
+      reserved for the Master PLL.
+    minItems: 1
+    maxItems: 7
+    items:
+      minimum: 1
+      maximum: 7
+
+  clock-output-names:
+    description:
+      Names of the child synthesizer outputs. The length of this array must
+      match clock-indices.
+    minItems: 1
+    maxItems: 7
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - "#clock-cells"
+  - clock-indices
+  - clock-output-names
+
+additionalProperties: false
+
+examples:
+  - |
+    clock-controller@400 {
+        #clock-cells = <1>;
+        compatible = "ti,dm816-fapll-clock";
+        reg = <0x400 0x40>;
+        clocks = <&sys_clkin_ck &sys_clkin_ck>;
+        clock-indices = <1>, <2>, <3>, <4>, <5>, <6>, <7>;
+        clock-output-names = "main_pll_clk1", "main_pll_clk2", "main_pll_clk3",
+                             "main_pll_clk4", "main_pll_clk5", "main_pll_clk6",
+                             "main_pll_clk7";
+
+    };
diff --git a/Documentation/devicetree/bindings/clock/ti/ti,dra7-atl-clock.yaml b/Documentation/devicetree/bindings/clock/ti/ti,dra7-atl-clock.yaml
new file mode 100644
index 00000000000000..c0795f4abcfaa8
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/ti/ti,dra7-atl-clock.yaml
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/ti/ti,dra7-atl-clock.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI DRA7 ATL (Audio Tracking Logic) clock output
+
+maintainers:
+  - Peter Ujfalusi <peter.ujfalusi@ti.com>
+
+properties:
+  compatible:
+    const: ti,dra7-atl-clock
+
+  "#clock-cells":
+    const: 0
+
+  clocks:
+    maxItems: 1
+
+  clock-output-names:
+    maxItems: 1
+
+required:
+  - compatible
+  - "#clock-cells"
+  - clocks
+
+additionalProperties: false
+
+examples:
+  - |
+    atl_clkin0_ck {
+        compatible = "ti,dra7-atl-clock";
+        #clock-cells = <0>;
+        clocks = <&atl_gfclk_mux>;
+    };
diff --git a/Documentation/devicetree/bindings/clock/ti/ti,dra7-atl.yaml b/Documentation/devicetree/bindings/clock/ti/ti,dra7-atl.yaml
new file mode 100644
index 00000000000000..f4bb233be83dff
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/ti/ti,dra7-atl.yaml
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/ti/ti,dra7-atl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI DRA7 ATL (Audio Tracking Logic) Clock Controller
+
+maintainers:
+  - Peter Ujfalusi <peter.ujfalusi@ti.com>
+
+description:
+  The ATL IP is used to generate clock to be used to synchronize baseband and
+  audio codec. A single ATL IP provides four ATL clock instances sharing the
+  same functional clock but can be configured to provide different clocks.
+  ATL can maintain a clock averages to some desired frequency based on the
+  bws/aws signals - can compensate the drift between the two ws signal.
+
+  See also Documentation/devicetree/bindings/clock/ti/ti,dra7-atl-clock.yaml
+
+properties:
+  compatible:
+    const: ti,dra7-atl
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    items:
+      - const: fck
+
+  ti,provided-clocks:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    description:
+      List of phandles to the clocks associated with the ATL.
+    minItems: 4
+    maxItems: 4
+    items:
+      maxItems: 1
+
+  ti,hwmods:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      Name of the hwmod associated with the ATL.
+    deprecated: true
+    const: atl
+
+patternProperties:
+  "^atl[0-3]$":
+    type: object
+    additionalProperties: false
+    description:
+      Configuration of ATL instances.
+
+    properties:
+      bws:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description:
+          Baseband word select signal selection.
+        maximum: 15
+
+      aws:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description:
+          Audio word select signal selection.
+        maximum: 15
+
+    required:
+      - bws
+      - aws
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - ti,provided-clocks
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/ti-dra7-atl.h>
+
+    atl@4843c000 {
+        compatible = "ti,dra7-atl";
+        reg = <0x4843c000 0x3ff>;
+        clocks = <&atl_gfclk_mux>;
+        clock-names = "fck";
+        ti,provided-clocks = <&atl_clkin0_ck>, <&atl_clkin1_ck>,
+                             <&atl_clkin2_ck>, <&atl_clkin3_ck>;
+
+        atl2 {
+            bws = <DRA7_ATL_WS_MCASP2_FSX>;
+            aws = <DRA7_ATL_WS_MCASP3_FSX>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml b/Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml
deleted file mode 100644
index 2bfac9d1c020a2..00000000000000
--- a/Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml
+++ /dev/null
@@ -1,134 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/crypto/hisilicon,hip06-sec.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Hisilicon hip06/hip07 Security Accelerator
-
-maintainers:
-  - Jonathan Cameron <Jonathan.Cameron@huawei.com>
-
-properties:
-  compatible:
-    enum:
-      - hisilicon,hip06-sec
-      - hisilicon,hip07-sec
-
-  reg:
-    items:
-      - description: Registers for backend processing engines
-      - description: Registers for common functionality
-      - description: Registers for queue 0
-      - description: Registers for queue 1
-      - description: Registers for queue 2
-      - description: Registers for queue 3
-      - description: Registers for queue 4
-      - description: Registers for queue 5
-      - description: Registers for queue 6
-      - description: Registers for queue 7
-      - description: Registers for queue 8
-      - description: Registers for queue 9
-      - description: Registers for queue 10
-      - description: Registers for queue 11
-      - description: Registers for queue 12
-      - description: Registers for queue 13
-      - description: Registers for queue 14
-      - description: Registers for queue 15
-
-  interrupts:
-    items:
-      - description: SEC unit error queue interrupt
-      - description: Completion interrupt for queue 0
-      - description: Error interrupt for queue 0
-      - description: Completion interrupt for queue 1
-      - description: Error interrupt for queue 1
-      - description: Completion interrupt for queue 2
-      - description: Error interrupt for queue 2
-      - description: Completion interrupt for queue 3
-      - description: Error interrupt for queue 3
-      - description: Completion interrupt for queue 4
-      - description: Error interrupt for queue 4
-      - description: Completion interrupt for queue 5
-      - description: Error interrupt for queue 5
-      - description: Completion interrupt for queue 6
-      - description: Error interrupt for queue 6
-      - description: Completion interrupt for queue 7
-      - description: Error interrupt for queue 7
-      - description: Completion interrupt for queue 8
-      - description: Error interrupt for queue 8
-      - description: Completion interrupt for queue 9
-      - description: Error interrupt for queue 9
-      - description: Completion interrupt for queue 10
-      - description: Error interrupt for queue 10
-      - description: Completion interrupt for queue 11
-      - description: Error interrupt for queue 11
-      - description: Completion interrupt for queue 12
-      - description: Error interrupt for queue 12
-      - description: Completion interrupt for queue 13
-      - description: Error interrupt for queue 13
-      - description: Completion interrupt for queue 14
-      - description: Error interrupt for queue 14
-      - description: Completion interrupt for queue 15
-      - description: Error interrupt for queue 15
-
-  dma-coherent: true
-
-  iommus:
-    maxItems: 1
-
-required:
-  - compatible
-  - reg
-  - interrupts
-  - dma-coherent
-
-additionalProperties: false
-
-examples:
-  - |
-    bus {
-        #address-cells = <2>;
-        #size-cells = <2>;
-
-        crypto@400d2000000 {
-            compatible = "hisilicon,hip07-sec";
-            reg = <0x400 0xd0000000 0x0 0x10000
-                  0x400 0xd2000000 0x0 0x10000
-                  0x400 0xd2010000 0x0 0x10000
-                  0x400 0xd2020000 0x0 0x10000
-                  0x400 0xd2030000 0x0 0x10000
-                  0x400 0xd2040000 0x0 0x10000
-                  0x400 0xd2050000 0x0 0x10000
-                  0x400 0xd2060000 0x0 0x10000
-                  0x400 0xd2070000 0x0 0x10000
-                  0x400 0xd2080000 0x0 0x10000
-                  0x400 0xd2090000 0x0 0x10000
-                  0x400 0xd20a0000 0x0 0x10000
-                  0x400 0xd20b0000 0x0 0x10000
-                  0x400 0xd20c0000 0x0 0x10000
-                  0x400 0xd20d0000 0x0 0x10000
-                  0x400 0xd20e0000 0x0 0x10000
-                  0x400 0xd20f0000 0x0 0x10000
-                  0x400 0xd2100000 0x0 0x10000>;
-            interrupts = <576 4>,
-                        <577 1>, <578 4>,
-                        <579 1>, <580 4>,
-                        <581 1>, <582 4>,
-                        <583 1>, <584 4>,
-                        <585 1>, <586 4>,
-                        <587 1>, <588 4>,
-                        <589 1>, <590 4>,
-                        <591 1>, <592 4>,
-                        <593 1>, <594 4>,
-                        <595 1>, <596 4>,
-                        <597 1>, <598 4>,
-                        <599 1>, <600 4>,
-                        <601 1>, <602 4>,
-                        <603 1>, <604 4>,
-                        <605 1>, <606 4>,
-                        <607 1>, <608 4>;
-            dma-coherent;
-            iommus = <&p1_smmu_alg_a 0x600>;
-        };
-    };
diff --git a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml
index c731fbdc2fe0fa..7b11cb5e5f6d1f 100644
--- a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml
+++ b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml
@@ -119,11 +119,11 @@ examples:
         #size-cells = <0>;
 
         panel@0 {
-                compatible = "bananapi,lhr050h41", "ilitek,ili9881c";
-                reg = <0>;
-                power-supply = <&reg_display>;
-                reset-gpios = <&r_pio 0 5 1>; /* PL05 */
-                backlight = <&pwm_bl>;
+            compatible = "bananapi,lhr050h41", "ilitek,ili9881c";
+            reg = <0>;
+            power-supply = <&reg_display>;
+            reset-gpios = <&r_pio 0 5 1>; /* PL05 */
+            backlight = <&pwm_bl>;
         };
 
         port {
diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml
index 9aca38a58a1629..8f8aae9f37887b 100644
--- a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml
+++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml
@@ -35,6 +35,9 @@ properties:
     minItems: 1
     maxItems: 2
 
+  iommus:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-v3d.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-v3d.yaml
index c55a8217de2504..1063e686647cad 100644
--- a/Documentation/devicetree/bindings/display/brcm,bcm2835-v3d.yaml
+++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-v3d.yaml
@@ -27,6 +27,9 @@ properties:
   power-domains:
     maxItems: 1
 
+  resets:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/display/bridge/analogix,dp.yaml b/Documentation/devicetree/bindings/display/bridge/analogix,dp.yaml
index 62f0521b09245b..ecf206871cdde1 100644
--- a/Documentation/devicetree/bindings/display/bridge/analogix,dp.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/analogix,dp.yaml
@@ -42,13 +42,22 @@ properties:
     properties:
       port@0:
         $ref: /schemas/graph.yaml#/properties/port
-        description:
-          Input node to receive pixel data.
+        description: Input node to receive pixel data.
 
       port@1:
-        $ref: /schemas/graph.yaml#/properties/port
-        description:
-          Port node with one endpoint connected to a dp-connector node.
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
+        description: Port node with one endpoint connected to sink device node.
+        properties:
+          endpoint:
+            $ref: /schemas/media/video-interfaces.yaml#
+            unevaluatedProperties: false
+            properties:
+              data-lanes:
+                minItems: 1
+                maxItems: 4
+                items:
+                  enum: [ 0, 1, 2, 3 ]
 
     required:
       - port@0
diff --git a/Documentation/devicetree/bindings/display/bridge/fsl,imx8qxp-pxl2dpi.yaml b/Documentation/devicetree/bindings/display/bridge/fsl,imx8qxp-pxl2dpi.yaml
index e4e77fad05f1c5..af9ad07ef08b9f 100644
--- a/Documentation/devicetree/bindings/display/bridge/fsl,imx8qxp-pxl2dpi.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/fsl,imx8qxp-pxl2dpi.yaml
@@ -84,8 +84,8 @@ examples:
                 };
 
                 mipi_lvds_0_pxl2dpi_dc_pixel_link1: endpoint@1 {
-                     reg = <1>;
-                     remote-endpoint = <&dc_pixel_link1_mipi_lvds_0_pxl2dpi>;
+                    reg = <1>;
+                    remote-endpoint = <&dc_pixel_link1_mipi_lvds_0_pxl2dpi>;
                 };
             };
 
diff --git a/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml b/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml
index 7f380879fffdfe..e5a8870bb76a62 100644
--- a/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml
@@ -28,9 +28,11 @@ properties:
     const: ldb
 
   reg:
+    minItems: 1
     maxItems: 2
 
   reg-names:
+    minItems: 1
     items:
       - const: ldb
       - const: lvds
@@ -83,15 +85,6 @@ allOf:
         ports:
           properties:
             port@2: false
-  - if:
-      not:
-        properties:
-          compatible:
-            contains:
-              const: fsl,imx6sx-ldb
-    then:
-      required:
-        - reg-names
 
   - if:
       properties:
@@ -100,7 +93,19 @@ allOf:
             const: fsl,imx6sx-ldb
     then:
       properties:
+        reg:
+          maxItems: 1
+        reg-names:
+          maxItems: 1
         nxp,enable-termination-resistor: false
+    else:
+      required:
+        - reg-names
+      properties:
+        reg:
+          minItems: 2
+        reg-names:
+          minItems: 2
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt8912b.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt8912b.yaml
index 988351f3cd018d..2d2dc9c046fb4f 100644
--- a/Documentation/devicetree/bindings/display/bridge/lontium,lt8912b.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt8912b.yaml
@@ -107,11 +107,11 @@ examples:
           };
 
           port@1 {
-              reg = <1>;
+            reg = <1>;
 
-              endpoint {
-                  remote-endpoint = <&hdmi_in>;
-              };
+            endpoint {
+              remote-endpoint = <&hdmi_in>;
+            };
           };
         };
       };
diff --git a/Documentation/devicetree/bindings/display/bridge/megachips,stdp2690-ge-b850v3-fw.yaml b/Documentation/devicetree/bindings/display/bridge/megachips,stdp2690-ge-b850v3-fw.yaml
index dfa6ff6f115e55..74539132bf940a 100644
--- a/Documentation/devicetree/bindings/display/bridge/megachips,stdp2690-ge-b850v3-fw.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/megachips,stdp2690-ge-b850v3-fw.yaml
@@ -104,7 +104,7 @@ examples:
                     endpoint {
                         remote-endpoint = <&stdp2690_in>;
                     };
-               };
+                };
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/display/bridge/nxp,tda998x.yaml b/Documentation/devicetree/bindings/display/bridge/nxp,tda998x.yaml
index 1205c8e9de329b..0ad55ff8e33080 100644
--- a/Documentation/devicetree/bindings/display/bridge/nxp,tda998x.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/nxp,tda998x.yaml
@@ -106,7 +106,7 @@ examples:
             video-ports = <0x230145>;
 
             #sound-dai-cells = <1>;
-                         /* DAI-format / AP_ENA reg value */
+            /* DAI-format / AP_ENA reg value */
             audio-ports = <TDA998x_SPDIF 0x04>,
                           <TDA998x_I2S 0x03>;
 
diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,r8a779g0-dsc.yaml b/Documentation/devicetree/bindings/display/bridge/renesas,r8a779g0-dsc.yaml
new file mode 100644
index 00000000000000..6ce2444409a2da
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/bridge/renesas,r8a779g0-dsc.yaml
@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/bridge/renesas,r8a779g0-dsc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Renesas R-Car DSC Display Stream Compression
+
+maintainers:
+  - Marek Vasut <marek.vasut+renesas@mailbox.org>
+
+description: |
+  This binding describes the VESA DSC Display Stream Compression encoder
+  embedded in the Renesas R-Car V4H SoC. The encoder supports all DSC1.1
+  encoding mechanisms, configurable bits-per-pixel, resolution up to 8k.
+
+properties:
+  compatible:
+    const: renesas,r8a779g0-dsc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 1
+
+  resets:
+    maxItems: 1
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: R-Car DU input port
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: R-Car DSI output port
+
+    required:
+      - port@0
+      - port@1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - interrupts
+  - power-domains
+  - resets
+  - ports
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/r8a779g0-cpg-mssr.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/power/r8a779g0-sysc.h>
+
+    bridge@feb8d000 {
+        compatible = "renesas,r8a779g0-dsc";
+        reg = <0xfeb8d000 0x400>;
+        interrupts = <GIC_SPI 559 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&cpg CPG_MOD 2819>;
+        power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+        resets = <&cpg 2819>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                dsc_in_dsi1: endpoint {
+                    remote-endpoint = <&du_out_dsi1>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                dsc_out_dsi1: endpoint {
+                    remote-endpoint = <&dsi1_in>;
+                };
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/display/bridge/sil,sii9022.yaml b/Documentation/devicetree/bindings/display/bridge/sil,sii9022.yaml
index 17ea06719b56c7..b869552596f416 100644
--- a/Documentation/devicetree/bindings/display/bridge/sil,sii9022.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/sil,sii9022.yaml
@@ -128,7 +128,7 @@ examples:
             cvcc12-supply = <&v1v2_hdmi>;
 
             #sound-dai-cells = <0>;
-            sil,i2s-data-lanes = < 0 1 2 >;
+            sil,i2s-data-lanes = <0 1 2>;
             clocks = <&mclk>;
             clock-names = "mclk";
 
diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358764.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358764.yaml
index 8666074005142d..23c5b254426b0b 100644
--- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358764.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358764.yaml
@@ -81,7 +81,7 @@ examples:
                 port@1 {
                     reg = <1>;
                     lvds_ep: endpoint {
-                      remote-endpoint = <&panel_ep>;
+                        remote-endpoint = <&panel_ep>;
                     };
                 };
             };
diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml
index 258dd9cfd7709c..8a0d8a66a9b090 100644
--- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml
@@ -161,9 +161,9 @@ examples:
                 dsi0_out: endpoint {
                     remote-endpoint = <&d2l_in_test>;
                 };
-             };
-         };
-     };
+            };
+        };
+    };
 
   - |
     /* For dual-link LVDS display panel */
@@ -227,7 +227,7 @@ examples:
                 dsi0_out_dual: endpoint {
                     remote-endpoint = <&d2l_in_dual>;
                 };
-             };
-         };
-     };
+            };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/display/faraday,tve200.yaml b/Documentation/devicetree/bindings/display/faraday,tve200.yaml
index b09628b69177c6..43dbe2778d6bf5 100644
--- a/Documentation/devicetree/bindings/display/faraday,tve200.yaml
+++ b/Documentation/devicetree/bindings/display/faraday,tve200.yaml
@@ -54,7 +54,7 @@ examples:
     display-controller@6a000000 {
       compatible = "faraday,tve200";
       reg = <0x6a000000 0x1000>;
-      interrupts = <13 IRQ_TYPE_EDGE_RISING>;
+      interrupts = <13 IRQ_TYPE_LEVEL_HIGH>;
       resets = <&syscon GEMINI_RESET_TVC>;
       clocks = <&syscon GEMINI_CLK_GATE_TVC>,
                <&syscon GEMINI_CLK_TVC>;
diff --git a/Documentation/devicetree/bindings/display/fsl,lcdif.yaml b/Documentation/devicetree/bindings/display/fsl,lcdif.yaml
index 2dd0411ec65161..2b123ddf068415 100644
--- a/Documentation/devicetree/bindings/display/fsl,lcdif.yaml
+++ b/Documentation/devicetree/bindings/display/fsl,lcdif.yaml
@@ -182,6 +182,7 @@ allOf:
             contains:
               enum:
                 - fsl,imx28-lcdif
+                - fsl,imx6ul-lcdif
     then:
       properties:
         dmas: false
diff --git a/Documentation/devicetree/bindings/display/himax,hx8357.yaml b/Documentation/devicetree/bindings/display/himax,hx8357.yaml
index 34c3b89bf0033c..bafcd4418d5aa5 100644
--- a/Documentation/devicetree/bindings/display/himax,hx8357.yaml
+++ b/Documentation/devicetree/bindings/display/himax,hx8357.yaml
@@ -74,5 +74,5 @@ examples:
             dc-gpios = <&gpio0 25 GPIO_ACTIVE_HIGH>;
             rotation = <90>;
             backlight = <&backlight>;
-       };
+        };
     };
diff --git a/Documentation/devicetree/bindings/display/imx/fsl,imx-lcdc.yaml b/Documentation/devicetree/bindings/display/imx/fsl,imx-lcdc.yaml
index c2b29622bceb43..84610b5690c490 100644
--- a/Documentation/devicetree/bindings/display/imx/fsl,imx-lcdc.yaml
+++ b/Documentation/devicetree/bindings/display/imx/fsl,imx-lcdc.yaml
@@ -110,7 +110,7 @@ examples:
 
         port {
             parallel_out: endpoint {
-              remote-endpoint = <&panel_in>;
+                remote-endpoint = <&panel_in>;
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/display/imx/fsl,imx6q-ldb.yaml b/Documentation/devicetree/bindings/display/imx/fsl,imx6q-ldb.yaml
index 1646f41d8f728a..cee1b504adf91d 100644
--- a/Documentation/devicetree/bindings/display/imx/fsl,imx6q-ldb.yaml
+++ b/Documentation/devicetree/bindings/display/imx/fsl,imx6q-ldb.yaml
@@ -150,44 +150,44 @@ examples:
 
         /* Using an of-graph endpoint link to connect the panel */
         lvds-channel@0 {
-                reg = <0>;
-                #address-cells = <1>;
-                #size-cells = <0>;
+            reg = <0>;
+            #address-cells = <1>;
+            #size-cells = <0>;
 
-                port@0 {
-                    reg = <0>;
+            port@0 {
+                reg = <0>;
 
-                    endpoint {
-                        remote-endpoint = <&ipu_di0_lvds0>;
-                    };
+                endpoint {
+                    remote-endpoint = <&ipu_di0_lvds0>;
                 };
+            };
 
-                port@2 {
-                    reg = <2>;
+            port@2 {
+                reg = <2>;
 
-                    endpoint {
-                        remote-endpoint = <&panel_in>;
-                    };
-               };
+                endpoint {
+                    remote-endpoint = <&panel_in>;
+                };
+            };
         };
 
         /* Using display-timings and fsl,data-mapping/width instead */
         lvds-channel@1 {
-                reg = <1>;
-                #address-cells = <1>;
-                #size-cells = <0>;
-                fsl,data-mapping = "spwg";
-                fsl,data-width = <24>;
+            reg = <1>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+            fsl,data-mapping = "spwg";
+            fsl,data-width = <24>;
 
-                display-timings {/* ... */
-                };
+            display-timings {/* ... */
+            };
 
-                port@1 {
-                     reg = <1>;
+            port@1 {
+                reg = <1>;
 
-                     endpoint {
-                         remote-endpoint = <&ipu_di1_lvds1>;
-                     };
+                endpoint {
+                    remote-endpoint = <&ipu_di1_lvds1>;
                 };
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,ethdr.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,ethdr.yaml
index 98db47894eebf0..6c0edbae96d795 100644
--- a/Documentation/devicetree/bindings/display/mediatek/mediatek,ethdr.yaml
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,ethdr.yaml
@@ -158,51 +158,51 @@ examples:
         #size-cells = <2>;
 
         hdr-engine@1c114000 {
-                compatible = "mediatek,mt8195-disp-ethdr";
-                reg = <0 0x1c114000 0 0x1000>,
-                      <0 0x1c115000 0 0x1000>,
-                      <0 0x1c117000 0 0x1000>,
-                      <0 0x1c119000 0 0x1000>,
-                      <0 0x1c11a000 0 0x1000>,
-                      <0 0x1c11b000 0 0x1000>,
-                      <0 0x1c11c000 0 0x1000>;
-                reg-names = "mixer", "vdo_fe0", "vdo_fe1", "gfx_fe0", "gfx_fe1",
-                            "vdo_be", "adl_ds";
-                mediatek,gce-client-reg = <&gce0 SUBSYS_1c11XXXX 0x4000 0x1000>,
-                                          <&gce0 SUBSYS_1c11XXXX 0x5000 0x1000>,
-                                          <&gce0 SUBSYS_1c11XXXX 0x7000 0x1000>,
-                                          <&gce0 SUBSYS_1c11XXXX 0x9000 0x1000>,
-                                          <&gce0 SUBSYS_1c11XXXX 0xa000 0x1000>,
-                                          <&gce0 SUBSYS_1c11XXXX 0xb000 0x1000>,
-                                          <&gce0 SUBSYS_1c11XXXX 0xc000 0x1000>;
-                clocks = <&vdosys1 CLK_VDO1_DISP_MIXER>,
-                         <&vdosys1 CLK_VDO1_HDR_VDO_FE0>,
-                         <&vdosys1 CLK_VDO1_HDR_VDO_FE1>,
-                         <&vdosys1 CLK_VDO1_HDR_GFX_FE0>,
-                         <&vdosys1 CLK_VDO1_HDR_GFX_FE1>,
-                         <&vdosys1 CLK_VDO1_HDR_VDO_BE>,
-                         <&vdosys1 CLK_VDO1_26M_SLOW>,
-                         <&vdosys1 CLK_VDO1_HDR_VDO_FE0_DL_ASYNC>,
-                         <&vdosys1 CLK_VDO1_HDR_VDO_FE1_DL_ASYNC>,
-                         <&vdosys1 CLK_VDO1_HDR_GFX_FE0_DL_ASYNC>,
-                         <&vdosys1 CLK_VDO1_HDR_GFX_FE1_DL_ASYNC>,
-                         <&vdosys1 CLK_VDO1_HDR_VDO_BE_DL_ASYNC>,
-                         <&topckgen CLK_TOP_ETHDR>;
-                clock-names = "mixer", "vdo_fe0", "vdo_fe1", "gfx_fe0", "gfx_fe1",
-                              "vdo_be", "adl_ds", "vdo_fe0_async", "vdo_fe1_async",
-                              "gfx_fe0_async", "gfx_fe1_async","vdo_be_async",
-                              "ethdr_top";
-                power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS1>;
-                iommus = <&iommu_vpp M4U_PORT_L3_HDR_DS>,
-                         <&iommu_vpp M4U_PORT_L3_HDR_ADL>;
-                interrupts = <GIC_SPI 517 IRQ_TYPE_LEVEL_HIGH 0>; /* disp mixer */
-                resets = <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE0_DL_ASYNC>,
-                         <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE1_DL_ASYNC>,
-                         <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE0_DL_ASYNC>,
-                         <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE1_DL_ASYNC>,
-                         <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_BE_DL_ASYNC>;
-                reset-names = "vdo_fe0_async", "vdo_fe1_async", "gfx_fe0_async",
-                              "gfx_fe1_async", "vdo_be_async";
+            compatible = "mediatek,mt8195-disp-ethdr";
+            reg = <0 0x1c114000 0 0x1000>,
+                  <0 0x1c115000 0 0x1000>,
+                  <0 0x1c117000 0 0x1000>,
+                  <0 0x1c119000 0 0x1000>,
+                  <0 0x1c11a000 0 0x1000>,
+                  <0 0x1c11b000 0 0x1000>,
+                  <0 0x1c11c000 0 0x1000>;
+            reg-names = "mixer", "vdo_fe0", "vdo_fe1", "gfx_fe0", "gfx_fe1",
+                        "vdo_be", "adl_ds";
+            mediatek,gce-client-reg = <&gce0 SUBSYS_1c11XXXX 0x4000 0x1000>,
+                                      <&gce0 SUBSYS_1c11XXXX 0x5000 0x1000>,
+                                      <&gce0 SUBSYS_1c11XXXX 0x7000 0x1000>,
+                                      <&gce0 SUBSYS_1c11XXXX 0x9000 0x1000>,
+                                      <&gce0 SUBSYS_1c11XXXX 0xa000 0x1000>,
+                                      <&gce0 SUBSYS_1c11XXXX 0xb000 0x1000>,
+                                      <&gce0 SUBSYS_1c11XXXX 0xc000 0x1000>;
+            clocks = <&vdosys1 CLK_VDO1_DISP_MIXER>,
+                     <&vdosys1 CLK_VDO1_HDR_VDO_FE0>,
+                     <&vdosys1 CLK_VDO1_HDR_VDO_FE1>,
+                     <&vdosys1 CLK_VDO1_HDR_GFX_FE0>,
+                     <&vdosys1 CLK_VDO1_HDR_GFX_FE1>,
+                     <&vdosys1 CLK_VDO1_HDR_VDO_BE>,
+                     <&vdosys1 CLK_VDO1_26M_SLOW>,
+                     <&vdosys1 CLK_VDO1_HDR_VDO_FE0_DL_ASYNC>,
+                     <&vdosys1 CLK_VDO1_HDR_VDO_FE1_DL_ASYNC>,
+                     <&vdosys1 CLK_VDO1_HDR_GFX_FE0_DL_ASYNC>,
+                     <&vdosys1 CLK_VDO1_HDR_GFX_FE1_DL_ASYNC>,
+                     <&vdosys1 CLK_VDO1_HDR_VDO_BE_DL_ASYNC>,
+                     <&topckgen CLK_TOP_ETHDR>;
+            clock-names = "mixer", "vdo_fe0", "vdo_fe1", "gfx_fe0", "gfx_fe1",
+                          "vdo_be", "adl_ds", "vdo_fe0_async", "vdo_fe1_async",
+                          "gfx_fe0_async", "gfx_fe1_async","vdo_be_async",
+                          "ethdr_top";
+            power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS1>;
+            iommus = <&iommu_vpp M4U_PORT_L3_HDR_DS>,
+                     <&iommu_vpp M4U_PORT_L3_HDR_ADL>;
+            interrupts = <GIC_SPI 517 IRQ_TYPE_LEVEL_HIGH 0>; /* disp mixer */
+            resets = <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE0_DL_ASYNC>,
+                     <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_FE1_DL_ASYNC>,
+                     <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE0_DL_ASYNC>,
+                     <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_GFX_FE1_DL_ASYNC>,
+                     <&vdosys1 MT8195_VDOSYS1_SW1_RST_B_HDR_VDO_BE_DL_ASYNC>;
+            reset-names = "vdo_fe0_async", "vdo_fe1_async", "gfx_fe0_async",
+                          "gfx_fe1_async", "vdo_be_async";
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi-ddc.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi-ddc.yaml
index bd8f7b8ae0ff25..966127e1ee63c2 100644
--- a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi-ddc.yaml
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi-ddc.yaml
@@ -15,10 +15,13 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - mediatek,mt7623-hdmi-ddc
-      - mediatek,mt8167-hdmi-ddc
-      - mediatek,mt8173-hdmi-ddc
+    oneOf:
+      - const: mediatek,mt8173-hdmi-ddc
+      - items:
+          - enum:
+              - mediatek,mt7623-hdmi-ddc
+              - mediatek,mt8167-hdmi-ddc
+          - const: mediatek,mt8173-hdmi-ddc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.yaml
index b90b6d18a82838..9af876329e6a66 100644
--- a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.yaml
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.yaml
@@ -112,24 +112,24 @@ examples:
         mediatek,syscon-hdmi = <&mmsys 0x900>;
 
         ports {
-          #address-cells = <1>;
-          #size-cells = <0>;
+            #address-cells = <1>;
+            #size-cells = <0>;
 
-          port@0 {
-            reg = <0>;
+            port@0 {
+                reg = <0>;
 
-            hdmi0_in: endpoint {
-              remote-endpoint = <&dpi0_out>;
+                hdmi0_in: endpoint {
+                    remote-endpoint = <&dpi0_out>;
+                };
             };
-          };
 
-          port@1 {
-            reg = <1>;
+            port@1 {
+                reg = <1>;
 
-            hdmi0_out: endpoint {
-              remote-endpoint = <&hdmi_con_in>;
+                hdmi0_out: endpoint {
+                    remote-endpoint = <&hdmi_con_in>;
+                };
             };
-          };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,il79900a.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,il79900a.yaml
index 02f7fb1f16dc77..f475302b87b0ca 100644
--- a/Documentation/devicetree/bindings/display/panel/ilitek,il79900a.yaml
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,il79900a.yaml
@@ -14,10 +14,15 @@ allOf:
 
 properties:
   compatible:
-    items:
-      - enum:
-          - tianma,tl121bvms07-00
-      - const: ilitek,il79900a
+    oneOf:
+      - items:
+          - enum:
+              - raspberrypi,dsi-10inch
+          - const: ilitek,il79600a
+      - items:
+          - enum:
+              - tianma,tl121bvms07-00
+          - const: ilitek,il79900a
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili7836a.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili7836a.yaml
new file mode 100644
index 00000000000000..962cf257ed8d3a
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili7836a.yaml
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/ilitek,ili7836a.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Ilitek ILI7836A-based OLED panels
+
+maintainers:
+  - Neil Armstrong <neil.armstrong@linaro.org>
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - const: retroidpocket,nova-panel
+      - const: ilitek,ili7836a
+
+  reg:
+    maxItems: 1
+    description: DSI virtual channel
+
+  vddio-supply: true
+  avdd-supply: true
+
+  backlight: true
+  port: true
+  reset-gpios: true
+  rotation: true
+
+required:
+  - compatible
+  - reg
+  - vddio-supply
+  - avdd-supply
+  - reset-gpios
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        panel@0 {
+            compatible = "retroidpocket,nova-panel", "ilitek,ili7836a";
+            reg = <0>;
+            vddio-supply = <&vdd_disp_1v8>;
+            avdd-supply = <&vdd_disp2_2v8>;
+            reset-gpios = <&tlmm 133 GPIO_ACTIVE_HIGH>;
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt36532.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt36532.yaml
new file mode 100644
index 00000000000000..c1ab793dd718d0
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/novatek,nt36532.yaml
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/novatek,nt36532.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Novatek NT36532 based DSI display Panels
+
+maintainers:
+  - Junjie Cao <junjie.cao@linux.dev>
+
+description: |
+  The NT36532 IC from Novatek is a Driver IC used to drive MIPI-DSI panels.
+
+allOf:
+  - $ref: panel-common-dual.yaml#
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - csot,ppc100hb1-1
+      - const: novatek,nt36532
+
+  reg:
+    maxItems: 1
+
+  reset-gpios:
+    maxItems: 1
+    description: phandle of gpio for reset line - This should be 8mA
+
+  vddio-supply:
+    description: regulator that supplies the I/O voltage
+
+  ports: true
+
+required:
+  - compatible
+  - reg
+  - vddio-supply
+  - reset-gpios
+  - ports
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "csot,ppc100hb1-1", "novatek,nt36532";
+            reg = <0>;
+
+            reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+
+            vddio-supply = <&vreg_l12b_1p8>;
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    panel_in_0: endpoint {
+                        remote-endpoint = <&dsi0_out>;
+                    };
+                };
+
+                port@1 {
+                    reg = <1>;
+                    panel_in_1: endpoint {
+                        remote-endpoint = <&dsi1_out>;
+                    };
+                };
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple-lvds-dual-ports.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple-lvds-dual-ports.yaml
index 8a2f6feafd3750..68beb476fb487e 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple-lvds-dual-ports.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple-lvds-dual-ports.yaml
@@ -44,6 +44,8 @@ properties:
       - auo,t215hvn01
         # BOE AV123Z7M-N17 12.3" (1920x720) LVDS TFT LCD panel
       - boe,av123z7m-n17
+        # BOE DV215FHM-R01 21.5" FHD (1920x1080) dual-channel LVDS panel
+      - boe,dv215fhm-r01
         # InnoLux 15.6" FHD (1920x1080) TFT LCD panel
       - innolux,g156hce-l01
         # Kaohsiung Opto-Electronics Inc. 10.1" WUXGA (1920 x 1200) LVDS TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
index 3d1cc11ba3aa42..eb49832b26c46e 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
@@ -338,6 +338,8 @@ properties:
       - tpk,f07a-0102
         # TPK U.S.A. LLC Fusion 10.1" 1024 x 600 (WSVGA) LCD panel with capacitive touch
       - tpk,f10a-0102
+        # Truly Semiconductors Limited 3.5" QVGA (320 x 240) TFT LCD panel
+      - truly,tft320240-229-e
         # United Radiant Technology UMSH-8596MD-xT 7.0" WVGA TFT LCD panel
         # Supported are LVDS versions (-11T, -19T) and parallel ones
         # (-T, -1T, -7T, -20T).
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml
index e081c84a932b6b..6db87f7d70d85d 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,ams495qa01.yaml
@@ -51,7 +51,7 @@ examples:
 
             port {
                 mipi_in_panel: endpoint {
-                  remote-endpoint = <&mipi_out_panel>;
+                    remote-endpoint = <&mipi_out_panel>;
                 };
             };
         };
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa5x01-ams561ra01.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa5x01-ams561ra01.yaml
index b271de575e1507..f04702de1101ff 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa5x01-ams561ra01.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e8aa5x01-ams561ra01.yaml
@@ -34,7 +34,7 @@ required:
   - compatible
   - reg
 
-additionalProperties: false
+unevaluatedProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml
index 6548f157fd9663..39dafe75a680e3 100644
--- a/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml
+++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,analogix-dp.yaml
@@ -160,6 +160,7 @@ examples:
           reg = <1>;
 
           edp_out_panel: endpoint {
+            data-lanes = <0 1>;
             remote-endpoint = <&panel_in_edp>;
           };
         };
diff --git a/Documentation/devicetree/bindings/display/sitronix,st7567.yaml b/Documentation/devicetree/bindings/display/sitronix,st7567.yaml
index 2eb6d00b5a2563..c007718541f6a1 100644
--- a/Documentation/devicetree/bindings/display/sitronix,st7567.yaml
+++ b/Documentation/devicetree/bindings/display/sitronix,st7567.yaml
@@ -64,5 +64,5 @@ examples:
                 vfront-porch = <0>;
                 vsync-len = <0>;
             };
-          };
-     };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/solomon,ssd1351.yaml b/Documentation/devicetree/bindings/display/solomon,ssd1351.yaml
new file mode 100644
index 00000000000000..80850c2ab5b3ef
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/solomon,ssd1351.yaml
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/solomon,ssd1351.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Solomon SSD1351 OLED Display Controller
+
+maintainers:
+  - Amit Barzilai <amit.barzilai22@gmail.com>
+  - Javier Martinez Canillas <javierm@redhat.com>
+
+allOf:
+  - $ref: solomon,ssd-common.yaml#
+
+properties:
+  compatible:
+    enum:
+      - solomon,ssd1351
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        oled@0 {
+            compatible = "solomon,ssd1351";
+            reg = <0x0>;
+            reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>;
+            dc-gpios = <&gpio2 8 GPIO_ACTIVE_HIGH>;
+            spi-max-frequency = <10000000>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml
index 53560052aaf00d..1b5f4317dab28b 100644
--- a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml
+++ b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml
@@ -101,35 +101,35 @@ examples:
         #size-cells = <0>;
 
         ports {
-              #address-cells = <1>;
-              #size-cells = <0>;
-
-              port@0 {
-                    reg = <0>;
-                    dsi_in: endpoint {
-                        remote-endpoint = <&ltdc_ep1_out>;
-                    };
-              };
-
-              port@1 {
-                    reg = <1>;
-                    dsi_out: endpoint {
-                        remote-endpoint = <&panel_in>;
-                    };
-              };
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                dsi_in: endpoint {
+                    remote-endpoint = <&ltdc_ep1_out>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                dsi_out: endpoint {
+                    remote-endpoint = <&panel_in>;
+                };
+            };
         };
 
         panel@0 {
-              compatible = "orisetech,otm8009a";
-              reg = <0>;
-              reset-gpios = <&gpioe 4 GPIO_ACTIVE_LOW>;
-              power-supply = <&v3v3>;
-
-              port {
-                    panel_in: endpoint {
-                        remote-endpoint = <&dsi_out>;
-                    };
-              };
+            compatible = "orisetech,otm8009a";
+            reg = <0>;
+            reset-gpios = <&gpioe 4 GPIO_ACTIVE_LOW>;
+            power-supply = <&v3v3>;
+
+            port {
+                panel_in: endpoint {
+                    remote-endpoint = <&dsi_out>;
+                };
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml b/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml
index 77058a5ccf6828..81c7772de25bae 100644
--- a/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml
+++ b/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml
@@ -119,9 +119,9 @@ examples:
         resets = <&rcc LTDC_R>;
 
         port {
-             ltdc_out_dsi: endpoint {
-                     remote-endpoint = <&dsi_in>;
-             };
+            ltdc_out_dsi: endpoint {
+                remote-endpoint = <&dsi_in>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/display/st,stm32mp25-lvds.yaml b/Documentation/devicetree/bindings/display/st,stm32mp25-lvds.yaml
index 14e042156179cb..f8e43c5a656af3 100644
--- a/Documentation/devicetree/bindings/display/st,stm32mp25-lvds.yaml
+++ b/Documentation/devicetree/bindings/display/st,stm32mp25-lvds.yaml
@@ -114,14 +114,14 @@ examples:
             port@0 {
                 reg = <0>;
                 lvds_in: endpoint {
-                   remote-endpoint = <&ltdc_ep1_out>;
+                    remote-endpoint = <&ltdc_ep1_out>;
                 };
             };
 
             port@1 {
                 reg = <1>;
                 lvds_out0: endpoint {
-                   remote-endpoint = <&lvds_panel_in>;
+                    remote-endpoint = <&lvds_panel_in>;
                 };
             };
         };
diff --git a/Documentation/devicetree/bindings/dma/apple,admac.yaml b/Documentation/devicetree/bindings/dma/apple,admac.yaml
index 6a200cbd7d0280..5b4fd8348f99fc 100644
--- a/Documentation/devicetree/bindings/dma/apple,admac.yaml
+++ b/Documentation/devicetree/bindings/dma/apple,admac.yaml
@@ -10,7 +10,7 @@ description: |
   Apple's Audio DMA Controller (ADMAC) is used to fetch and store audio samples
   on SoCs from the "Apple Silicon" family.
 
-  The controller has been seen with up to 24 channels. Even-numbered channels
+  The controller has been seen with up to 32 channels. Even-numbered channels
   are TX-only, odd-numbered are RX-only. Individual channels are coupled to
   fixed device endpoints.
 
@@ -33,6 +33,12 @@ properties:
               - apple,t8103-admac
               - apple,t8112-admac
           - const: apple,admac
+      - items:
+          - enum:
+              - apple,t6030-admac
+              - apple,t6031-admac
+          - const: apple,t8122-admac
+      - const: apple,t8122-admac
 
   reg:
     maxItems: 1
@@ -43,7 +49,7 @@ properties:
       Clients specify a single cell with channel number.
 
   dma-channels:
-    maximum: 24
+    maximum: 32
 
   interrupts:
     minItems: 4
diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
index 0a30a455b0ee54..bfd6d67a98e623 100644
--- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
+++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml
@@ -14,6 +14,27 @@ description:
 
 allOf:
   - $ref: dma-controller.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ultrarisc,dp1000-axi-dma
+    then:
+      properties:
+        reg:
+          maxItems: 1
+        interrupts:
+          minItems: 8
+        dma-channels:
+          const: 8
+        snps,dma-masters:
+          const: 1
+        snps,data-width:
+          const: 4
+        snps,priority:
+          minItems: 8
+        snps,block-size:
+          minItems: 8
 
 properties:
   compatible:
@@ -27,6 +48,7 @@ properties:
           - enum:
               - altr,agilex5-axi-dma
               - sophgo,cv1800b-axi-dma
+              - ultrarisc,dp1000-axi-dma
           - const: snps,axi-dma-1.01a
 
   reg:
diff --git a/Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml b/Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml
new file mode 100644
index 00000000000000..873cedd1427d54
--- /dev/null
+++ b/Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/dma/ti,dra7-dma-crossbar.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments DMA Crossbar (DMA request router)
+
+maintainers:
+  - Vignesh Raghavendra <vigneshr@ti.com>
+  - Bhargav Joshi <j.bhargav.u@gmail.com>
+
+description:
+  When requesting channel via ti,dra7-dma-crossbar, the DMA client must request
+  the DMA event number as crossbar ID (input to the DMA crossbar). For
+  ti,am335x-edma-crossbar the meaning of parameters of dmas for clients dmas =
+  <&edma_xbar 12 0 1>; where <12> is the DMA request number, <0> is the TC the
+  event should be assigned and <1> is the mux selection for in the crossbar.
+  When mux 0 is used the DMA channel can be requested directly from edma node.
+
+properties:
+  compatible:
+    enum:
+      - ti,dra7-dma-crossbar
+      - ti,am335x-edma-crossbar
+
+  reg:
+    maxItems: 1
+
+  "#dma-cells":
+    minimum: 1
+    maximum: 3
+
+  dma-requests:
+    minimum: 1
+    maximum: 256
+
+  dma-masters:
+    maxItems: 1
+
+  ti,dma-safe-map:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Safe routing value for unused request lines
+
+  ti,reserved-dma-request-ranges:
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    description:
+      DMA request ranges which should not be used when mapping xbar input to
+      DMA request, they are either allocated to be used by for example the DSP
+      or they are used as memcpy channels in eDMA.
+    items:
+      items:
+        - description: starting DMA request line number
+        - description: number of consecutive lines to reserve
+
+required:
+  - compatible
+  - reg
+  - "#dma-cells"
+  - dma-requests
+  - dma-masters
+
+allOf:
+  - $ref: dma-router.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,am335x-edma-crossbar
+    then:
+      properties:
+        "#dma-cells":
+          const: 3
+
+    else:
+      properties:
+        "#dma-cells":
+          enum: [1, 2]
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    dma-router@4a002b78 {
+        compatible = "ti,dra7-dma-crossbar";
+        reg = <0x4a002b78 0xfc>;
+        #dma-cells = <1>;
+        dma-requests = <205>;
+        ti,dma-safe-map = <0>;
+        /* Protect the sDMA request ranges: 10-14 and 100-126 */
+        ti,reserved-dma-request-ranges = <10 5>, <100 27>;
+        dma-masters = <&sdma>;
+    };
diff --git a/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt b/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt
deleted file mode 100644
index 1f9831540c9791..00000000000000
--- a/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt
+++ /dev/null
@@ -1,68 +0,0 @@
-Texas Instruments DMA Crossbar (DMA request router)
-
-Required properties:
-- compatible:	"ti,dra7-dma-crossbar" for DRA7xx DMA crossbar
-		"ti,am335x-edma-crossbar" for AM335x and AM437x
-- reg:		Memory map for accessing module
-- #dma-cells:	Should be set to match with the DMA controller's dma-cells
-		for ti,dra7-dma-crossbar and <3> for ti,am335x-edma-crossbar.
-- dma-requests:	Number of DMA requests the crossbar can receive
-- dma-masters:	phandle pointing to the DMA controller
-
-The DMA controller node need to have the following poroperties:
-- dma-requests:	Number of DMA requests the controller can handle
-
-Optional properties:
-- ti,dma-safe-map: Safe routing value for unused request lines
-- ti,reserved-dma-request-ranges: DMA request ranges which should not be used
-		when mapping xbar input to DMA request, they are either
-		allocated to be used by for example the DSP or they are used as
-		memcpy channels in eDMA.
-
-Notes:
-When requesting channel via ti,dra7-dma-crossbar, the DMA client must request
-the DMA event number as crossbar ID (input to the DMA crossbar).
-
-For ti,am335x-edma-crossbar: the meaning of parameters of dmas for clients:
-dmas = <&edma_xbar 12 0 1>; where <12> is the DMA request number, <0> is the TC
-the event should be assigned and <1> is the mux selection for in the crossbar.
-When mux 0 is used the DMA channel can be requested directly from edma node.
-
-Example:
-
-/* DMA controller */
-sdma: dma-controller@4a056000 {
-	compatible = "ti,omap4430-sdma";
-	reg = <0x4a056000 0x1000>;
-	interrupts =	<GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
-	#dma-cells = <1>;
-	dma-channels = <32>;
-	dma-requests = <127>;
-};
-
-/* DMA crossbar */
-sdma_xbar: dma-router@4a002b78 {
-	compatible = "ti,dra7-dma-crossbar";
-	reg = <0x4a002b78 0xfc>;
-	#dma-cells = <1>;
-	dma-requests = <205>;
-	ti,dma-safe-map = <0>;
-	/* Protect the sDMA request ranges: 10-14 and 100-126 */
-	ti,reserved-dma-request-ranges = <10 5>, <100 27>;
-	dma-masters = <&sdma>;
-};
-
-/* DMA client */
-uart1: serial@4806a000 {
-	compatible = "ti,omap4-uart";
-	reg = <0x4806a000 0x100>;
-	interrupts-extended = <&gic GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
-	ti,hwmods = "uart1";
-	clock-frequency = <48000000>;
-	/* Requesting crossbar input 49 and 50 */
-	dmas = <&sdma_xbar 49>, <&sdma_xbar 50>;
-	dma-names = "tx", "rx";
-};
diff --git a/Documentation/devicetree/bindings/dma/ti/ti,cppi41.yaml b/Documentation/devicetree/bindings/dma/ti/ti,cppi41.yaml
new file mode 100644
index 00000000000000..a8c28e8ada072d
--- /dev/null
+++ b/Documentation/devicetree/bindings/dma/ti/ti,cppi41.yaml
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/dma/ti/ti,cppi41.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI CPPI41 DMA Controller
+
+maintainers:
+  - Sebastian Andrzej Siewior <bigeasy@linutronix.de>
+
+properties:
+  compatible:
+    enum:
+      - ti,am3359-cppi41
+      - ti,da830-cppi41
+
+  reg:
+    minItems: 3
+    maxItems: 4
+
+  reg-names:
+    minItems: 3
+    maxItems: 4
+
+  interrupts:
+    maxItems: 1
+
+  interrupt-names:
+    const: glue
+
+  "#dma-cells":
+    const: 2
+
+  dma-channels:
+    description: Number of DMA channels supported.
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  dma-requests:
+    description: Number of DMA requests supported.
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  power-domains:
+    maxItems: 1
+
+  "#dma-channels":
+    description: Deprecated, use dma-channels instead.
+    deprecated: true
+
+  "#dma-requests":
+    description: Deprecated, use dma-requests instead.
+    deprecated: true
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - "#dma-cells"
+  - dma-channels
+
+allOf:
+  - $ref: /schemas/dma/dma-controller.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,am3359-cppi41
+    then:
+      properties:
+        reg:
+          minItems: 4
+        reg-names:
+          items:
+            - const: glue
+            - const: controller
+            - const: scheduler
+            - const: queuemgr
+        power-domains: false
+      required:
+        - interrupt-names
+        - interrupts
+        - dma-requests
+    else:
+      properties:
+        reg:
+          maxItems: 3
+        reg-names:
+          items:
+            - const: controller
+            - const: scheduler
+            - const: queuemgr
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    dma-controller@7402000 {
+        compatible = "ti,am3359-cppi41";
+        reg = <0x47400000 0x1000>,
+              <0x47402000 0x1000>,
+              <0x47403000 0x1000>,
+              <0x47404000 0x4000>;
+        reg-names = "glue", "controller", "scheduler", "queuemgr";
+        interrupts = <17>;
+        interrupt-names = "glue";
+        #dma-cells = <2>;
+        dma-channels = <30>;
+        dma-requests = <256>;
+    };
diff --git a/Documentation/devicetree/bindings/dts-coding-style.rst b/Documentation/devicetree/bindings/dts-coding-style.rst
index 63648db377e184..441436b7fa8044 100644
--- a/Documentation/devicetree/bindings/dts-coding-style.rst
+++ b/Documentation/devicetree/bindings/dts-coding-style.rst
@@ -135,8 +135,9 @@ The above-described ordering follows this approach:
 3. Status is the last information to annotate that device node is or is not
    finished (board resources are needed).
 
-The individual properties inside each group shall use natural sort order by
-the property name.
+The individual properties inside each group shall use usually natural sort
+order by the property name, with exceptions of logical grouping of properties,
+e.g. supplies.
 
 Example::
 
diff --git a/Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml b/Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml
index ac5a08f8f76dfd..aa23382595c226 100644
--- a/Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml
+++ b/Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml
@@ -20,8 +20,11 @@ properties:
     oneOf:
       - items:
           - enum:
+              - lenovo,yoga-slim7x-ec
               - qcom,glymur-crd-ec
+              - qcom,glymur-qcb-ec
               - qcom,hamoa-iot-evk-ec
+              - qcom,purwa-iot-evk-ec
           - const: qcom,hamoa-crd-ec
       - enum:
           - qcom,hamoa-crd-ec
diff --git a/Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml b/Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml
new file mode 100644
index 00000000000000..fa81adbf9b80df
--- /dev/null
+++ b/Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/firmware/fsl,imx-se.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP i.MX HW Secure Enclave(s) EdgeLock Enclave
+
+maintainers:
+  - Pankaj Gupta <pankaj.gupta@nxp.com>
+
+description: |
+  NXP's SoC may contain one or multiple embedded secure-enclave HW
+  IP(s) like i.MX EdgeLock Enclave, V2X etc. These NXP's HW IP(s)
+  enables features like
+    - Hardware Security Module (HSM),
+    - Security Hardware Extension (SHE), and
+    - Vehicular to Anything (V2X)
+
+  Communication interface to the secure-enclaves(se) is based on the
+  messaging unit(s).
+
+properties:
+  compatible:
+    enum:
+      - fsl,imx8ulp-se-ele-hsm
+      - fsl,imx93-se-ele-hsm
+      - fsl,imx95-se-ele-hsm
+
+  mboxes:
+    items:
+      - description: mailbox phandle to send message to se firmware
+      - description: mailbox phandle to receive message from se firmware
+
+  mbox-names:
+    items:
+      - const: tx
+      - const: rx
+
+  memory-region:
+    maxItems: 1
+
+  sram:
+    maxItems: 1
+
+required:
+  - compatible
+  - mboxes
+  - mbox-names
+
+allOf:
+  # memory-region
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - fsl,imx8ulp-se-ele-hsm
+              - fsl,imx93-se-ele-hsm
+    then:
+      required:
+        - memory-region
+    else:
+      properties:
+        memory-region: false
+
+  # sram
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - fsl,imx8ulp-se-ele-hsm
+    then:
+      required:
+        - sram
+
+    else:
+      properties:
+        sram: false
+
+additionalProperties: false
+
+examples:
+  - |
+    secure-enclave {
+      compatible = "fsl,imx95-se-ele-hsm";
+      mboxes = <&ele_mu0 0 0>, <&ele_mu0 1 0>;
+      mbox-names = "tx", "rx";
+    };
+...
diff --git a/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.yaml b/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.yaml
index b42cfa78b28b55..1af8ee3f1f3b59 100644
--- a/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.yaml
+++ b/Documentation/devicetree/bindings/firmware/intel,stratix10-svc.yaml
@@ -55,6 +55,12 @@ properties:
       reserved memory region for the service layer driver to
       communicate with the secure device manager.
 
+  interrupts:
+    maxItems: 1
+    description:
+      This interrupt is used by the Secure Device Manager (SDM) to signal
+      completion of an asynchronous operation to service layer driver.
+
   iommus:
     maxItems: 1
 
@@ -82,6 +88,8 @@ additionalProperties: false
 
 examples:
   - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
     reserved-memory {
       #address-cells = <2>;
       #size-cells = <2>;
@@ -99,6 +107,8 @@ examples:
         compatible = "intel,stratix10-svc";
         method = "smc";
         memory-region = <&service_reserved>;
+        interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-parent = <&intc0>;
 
         fpga-mgr {
           compatible = "intel,stratix10-soc-fpga-mgr";
diff --git a/Documentation/devicetree/bindings/firmware/nvidia,tegra186-bpmp.yaml b/Documentation/devicetree/bindings/firmware/nvidia,tegra186-bpmp.yaml
index 3c44fe607e1281..acdc5c1d3d8bb9 100644
--- a/Documentation/devicetree/bindings/firmware/nvidia,tegra186-bpmp.yaml
+++ b/Documentation/devicetree/bindings/firmware/nvidia,tegra186-bpmp.yaml
@@ -18,11 +18,9 @@ description: |
   which can create the interprocessor communication (IPC) between the
   CPU and BPMP.
 
-  This node is a mailbox consumer. See the following files for details
-  of the mailbox subsystem, and the specifiers implemented by the
-  relevant provider(s):
+  This node is a mailbox consumer. See the following file for details
+  of the mailbox specifiers implemented by the relevant provider(s):
 
-    - .../mailbox/mailbox.txt
     - .../mailbox/nvidia,tegra186-hsp.yaml
 
   This node is a clock, power domain, and reset provider. See the
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
index 037d4ad8f718db..40a267c54b00ca 100644
--- a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
+++ b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
@@ -37,6 +37,7 @@ properties:
           - qcom,scm-ipq9574
           - qcom,scm-ipq9650
           - qcom,scm-kaanapali
+          - qcom,scm-kuno
           - qcom,scm-maili
           - qcom,scm-mdm9607
           - qcom,scm-milos
@@ -44,6 +45,7 @@ properties:
           - qcom,scm-msm8660
           - qcom,scm-msm8916
           - qcom,scm-msm8937
+          - qcom,scm-msm8952
           - qcom,scm-msm8953
           - qcom,scm-msm8960
           - qcom,scm-msm8974
@@ -152,6 +154,7 @@ allOf:
               - qcom,scm-msm8660
               - qcom,scm-msm8916
               - qcom,scm-msm8937
+              - qcom,scm-msm8952
               - qcom,scm-msm8953
               - qcom,scm-msm8960
               - qcom,scm-msm8974
@@ -198,6 +201,7 @@ allOf:
               - qcom,scm-msm8226
               - qcom,scm-msm8916
               - qcom,scm-msm8937
+              - qcom,scm-msm8952
               - qcom,scm-msm8953
               - qcom,scm-msm8974
               - qcom,scm-msm8976
diff --git a/Documentation/devicetree/bindings/fuse/nvidia,tegra20-fuse.yaml b/Documentation/devicetree/bindings/fuse/nvidia,tegra20-fuse.yaml
index 02f0b04623776d..bf9923b6385279 100644
--- a/Documentation/devicetree/bindings/fuse/nvidia,tegra20-fuse.yaml
+++ b/Documentation/devicetree/bindings/fuse/nvidia,tegra20-fuse.yaml
@@ -22,6 +22,7 @@ properties:
           - nvidia,tegra186-efuse
           - nvidia,tegra194-efuse
           - nvidia,tegra234-efuse
+          - nvidia,tegra264-efuse
 
       - items:
           - const: nvidia,tegra132-efuse
diff --git a/Documentation/devicetree/bindings/gpio/axiado,ax3005-sgpio.yaml b/Documentation/devicetree/bindings/gpio/axiado,ax3005-sgpio.yaml
new file mode 100644
index 00000000000000..47ae1285d4829f
--- /dev/null
+++ b/Documentation/devicetree/bindings/gpio/axiado,ax3005-sgpio.yaml
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/gpio/axiado,ax3005-sgpio.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Axiado AX3005 SGPIO Controller
+
+maintainers:
+  - Petar Stepanovic <pstepanovic@axiado.com>
+  - SriNavmani A <srinavmani@axiado.com>
+  - Prasad Bolisetty <pbolisetty@axiado.com>
+
+description: |
+  The Axiado SGPIO controller provides a serial GPIO expansion interface
+  that multiplexes multiple GPIO signals over a small set of physical lines.
+
+  The controller serializes output values on a data output line (DOUT) and
+  samples input values from a data input line (DIN) in synchronization with
+  a shift clock (SCLK). A latch signal (LATCH) updates the output states
+  after each transfer cycle.
+
+  SGPIO positions are organized in banks of 32, with registers controlling
+  output values, input sampling, and interrupt status. The number of SGPIO
+  positions supported depends on the controller configuration.
+
+  The controller detects changes on input GPIOs. Interrupt events are
+  aggregated and signaled through a single interrupt line to the parent
+  interrupt controller.
+
+  Register access is performed over the APB bus. The SGPIO shift clock is
+  derived from the APB clock using a programmable divider.
+
+  Each SGPIO position provides one input and one output, and both are
+  exposed as GPIO lines. Position N is described by two consecutive lines:
+  line 2 * N is the input and line 2 * N + 1 is the output. Directions are
+  fixed by the hardware, so even lines are always inputs and odd lines are
+  always outputs.
+
+  Only input lines can generate interrupts. Rising, falling, and both-edge
+  interrupts are supported.
+
+properties:
+  compatible:
+    const: axiado,ax3005-sgpio
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    description: APB interface clock
+    maxItems: 1
+
+  bus-frequency:
+    description: Desired SGPIO shift clock frequency in Hz
+    minimum: 1
+
+  gpio-controller: true
+
+  '#gpio-cells':
+    const: 2
+
+  interrupt-controller: true
+
+  '#interrupt-cells':
+    const: 2
+
+  interrupts:
+    maxItems: 1
+
+  ngpios:
+    description: |
+      Total number of GPIO lines provided by the controller. Each SGPIO
+      position wired up on the board contributes one input and one output
+      line, so this is twice the number of positions and must be even. It
+      may be lower than the maximum supported by the hardware.
+    minimum: 2
+    maximum: 1024
+    multipleOf: 2
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - bus-frequency
+  - gpio-controller
+  - '#gpio-cells'
+  - interrupts
+  - interrupt-controller
+  - '#interrupt-cells'
+  - ngpios
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    gpio@33201c00 {
+        compatible = "axiado,ax3005-sgpio";
+        reg = <0x33201c00 0x400>;
+        clocks = <&apb_clk>;
+        bus-frequency = <1000000>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+        interrupt-parent = <&gic500>;
+        interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>;
+        ngpios = <256>;
+    };
diff --git a/Documentation/devicetree/bindings/gpio/delta,tn48m-gpio.yaml b/Documentation/devicetree/bindings/gpio/delta,tn48m-gpio.yaml
index e3e668a120917e..698cc14b027f4d 100644
--- a/Documentation/devicetree/bindings/gpio/delta,tn48m-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/delta,tn48m-gpio.yaml
@@ -14,7 +14,7 @@ description: |
   details see ../mfd/delta,tn48m-cpld.yaml.
 
   Delta TN48M has an onboard Lattice CPLD that is used as an GPIO expander.
-  It provides 12 pins in total, they are input-only or ouput-only type.
+  It provides 12 pins in total, they are input-only or output-only type.
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/gpio/fsl,qoriq-gpio.yaml b/Documentation/devicetree/bindings/gpio/fsl,qoriq-gpio.yaml
index 4cb2a6b9fabfbe..1e61e17b0ee35e 100644
--- a/Documentation/devicetree/bindings/gpio/fsl,qoriq-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/fsl,qoriq-gpio.yaml
@@ -63,6 +63,13 @@ properties:
       GPIO registers are used as little endian. If not
       present registers are used as big endian by default.
 
+patternProperties:
+  "^.+-hog(-[0-9]+)?$":
+    type: object
+
+    required:
+      - gpio-hog
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml b/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml
index de24bb361e9f64..fffeb5ec0645d1 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml
@@ -45,6 +45,12 @@ properties:
   power-domains:
     maxItems: 1
 
+patternProperties:
+  "^.+-hog(-[0-9]+)?$":
+    type: object
+    required:
+      - gpio-hog
+
 allOf:
   - if:
       properties:
diff --git a/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml b/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml
index adeb3b3a2902d8..fac53e65909b9b 100644
--- a/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml
@@ -91,6 +91,9 @@ properties:
       - nvidia,tegra264-gpio
       - nvidia,tegra264-gpio-uphy
       - nvidia,tegra264-gpio-aon
+      - nvidia,tegra268-gpio
+      - nvidia,tegra268-gpio-uphy
+      - nvidia,tegra268-gpio-aon
 
   reg-names:
     items:
@@ -169,6 +172,8 @@ allOf:
               - nvidia,tegra256-gpio
               - nvidia,tegra264-gpio
               - nvidia,tegra264-gpio-uphy
+              - nvidia,tegra268-gpio
+              - nvidia,tegra268-gpio-uphy
     then:
       properties:
         interrupts:
@@ -185,6 +190,7 @@ allOf:
               - nvidia,tegra234-gpio-aon
               - nvidia,tegra238-gpio-aon
               - nvidia,tegra264-gpio-aon
+              - nvidia,tegra268-gpio-aon
     then:
       properties:
         interrupts:
@@ -201,6 +207,9 @@ allOf:
               - nvidia,tegra264-gpio
               - nvidia,tegra264-gpio-uphy
               - nvidia,tegra264-gpio-aon
+              - nvidia,tegra268-gpio
+              - nvidia,tegra268-gpio-uphy
+              - nvidia,tegra268-gpio-aon
     then:
       required:
         - wakeup-parent
diff --git a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml
index b18f8f0ca0ae40..33c25df7b815c1 100644
--- a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml
@@ -40,6 +40,8 @@ properties:
 
   gpio-controller: true
 
+  gpio-ranges: true
+
   ngpios:
     minimum: 1
     maximum: 32
diff --git a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml
index d32e103a64aacc..95f5175c7a9da0 100644
--- a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml
@@ -56,6 +56,11 @@ properties:
               - renesas,gpio-r8a779h0     # R-Car V4M
           - const: renesas,rcar-gen4-gpio # R-Car Gen4
 
+      - items:
+          - enum:
+              - renesas,gpio-r8a78000     # R-Car X5H
+          - const: renesas,rcar-gen5-gpio # R-Car Gen5
+
   reg:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml
index 9db9f84ad964b8..ece2b637cd4140 100644
--- a/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml
+++ b/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml
@@ -34,6 +34,7 @@ properties:
               - rockchip,rk3562-mali
               - rockchip,rk3568-mali
               - rockchip,rk3576-mali
+              - samsung,exynos850-mali
           - const: arm,mali-bifrost # Mali Bifrost GPU model/revision is fully discoverable
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml
index c8d0d9192d92ca..d68fe8c7d010fb 100644
--- a/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml
+++ b/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml
@@ -112,7 +112,13 @@ properties:
     maxItems: 1
 
   resets:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
+
+  reset-names:
+    items:
+      - const: ao_g3d
+      - const: media_g3d
 
   operating-points-v2: true
 
@@ -130,6 +136,23 @@ required:
 additionalProperties: false
 
 allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: hisilicon,hi6220-mali
+    then:
+      properties:
+        resets:
+          minItems: 2
+      required:
+        - reset-names
+    else:
+      properties:
+        resets:
+          maxItems: 1
+        reset-names: false
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml b/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml
index b9cdfe52b62ff3..619faf33214bdc 100644
--- a/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml
+++ b/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml
@@ -25,6 +25,7 @@ properties:
       - items:
           - enum:
               - samsung,exynos2200-chipid
+              - samsung,exynos5515-chipid
               - samsung,exynos7885-chipid
               - samsung,exynos8895-chipid
               - samsung,exynos9610-chipid
diff --git a/Documentation/devicetree/bindings/hwmon/adi,ltc2991.yaml b/Documentation/devicetree/bindings/hwmon/adi,ltc2991.yaml
index 1ff44cb22ef42b..2210764a81bbef 100644
--- a/Documentation/devicetree/bindings/hwmon/adi,ltc2991.yaml
+++ b/Documentation/devicetree/bindings/hwmon/adi,ltc2991.yaml
@@ -49,7 +49,7 @@ patternProperties:
 
       shunt-resistor-micro-ohms:
         description:
-          The value of curent sense resistor in micro ohms. Pin configuration is
+          The value of current sense resistor in micro ohms. Pin configuration is
           set for differential input pair.
 
       adi,temperature-enable:
diff --git a/Documentation/devicetree/bindings/hwmon/axiado,ax3000-pwm-fan.yaml b/Documentation/devicetree/bindings/hwmon/axiado,ax3000-pwm-fan.yaml
new file mode 100644
index 00000000000000..76a3e12fc656e4
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/axiado,ax3000-pwm-fan.yaml
@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/axiado,ax3000-pwm-fan.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Axiado AX3000 and AX3005 PWM Fan Controller
+
+maintainers:
+  - Petar Stepanovic <pstepanovic@axiado.com>
+  - Akhila Kavi <akavi@axiado.com>
+  - Prasad Bolisetty <pbolisetty@axiado.com>
+
+description:
+  The Axiado AX3000 and AX3005 fan controllers use a hardware tachometer
+  block to measure fan speed. Fan speed is controlled through a PWM signal
+  supplied by an external PWM controller.
+
+properties:
+  compatible:
+    oneOf:
+      - const: axiado,ax3000-pwm-fan
+      - items:
+          - const: axiado,ax3005-pwm-fan
+          - const: axiado,ax3000-pwm-fan
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  fan:
+    type: object
+    $ref: fan-common.yaml#
+    unevaluatedProperties: false
+    properties:
+      pulses-per-revolution:
+        minimum: 1
+        default: 2
+
+    required:
+      - pwms
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - fan
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    fan-controller@33001400 {
+        compatible = "axiado,ax3000-pwm-fan";
+        reg = <0x33001400 0x400>;
+        interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clk>;
+
+        fan {
+            pwms = <&pwm0 0 400>;
+            pulses-per-revolution = <2>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/hwmon/axiado,ax3000-tsadc.yaml b/Documentation/devicetree/bindings/hwmon/axiado,ax3000-tsadc.yaml
new file mode 100644
index 00000000000000..4b56e4fd9ef97d
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/axiado,ax3000-tsadc.yaml
@@ -0,0 +1,45 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/axiado,ax3000-tsadc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Axiado AX3000 and AX3005 TSADC
+
+maintainers:
+  - Petar Stepanovic <pstepanovic@axiado.com>
+  - Akhila Kavi <akavi@axiado.com>
+  - Prasad Bolisetty <pbolisetty@axiado.com>
+
+description: |
+  The Axiado AX3000 and AX3005 Temperature Sensor ADC (TSADC) controllers
+  monitor the on-die temperature of the SoC.
+
+properties:
+  compatible:
+    oneOf:
+      - const: axiado,ax3000-tsadc
+      - items:
+          - const: axiado,ax3005-tsadc
+          - const: axiado,ax3000-tsadc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+
+additionalProperties: false
+
+examples:
+  - |
+    temperature-sensor@33021000 {
+        compatible = "axiado,ax3000-tsadc";
+        reg = <0x33021000 0x100>;
+        clocks = <&refclk>;
+    };
diff --git a/Documentation/devicetree/bindings/hwmon/national,lm63.yaml b/Documentation/devicetree/bindings/hwmon/national,lm63.yaml
new file mode 100644
index 00000000000000..4c8d0a15fbd12a
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/national,lm63.yaml
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/national,lm63.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: LM63-compatible temperature sensors with integrated fan control
+
+maintainers:
+  - Guenter Roeck <linux@roeck-us.net>
+
+description: |
+  The LM63 family provides local and remote temperature monitoring together
+  with fan speed monitoring and PWM fan control. Supported devices include
+  the National Semiconductor LM63, LM64 and LM96163, and the Sensylink
+  CTF2301.
+
+allOf:
+  - $ref: hwmon-common.yaml#
+
+properties:
+  compatible:
+    enum:
+      - national,lm63
+      - national,lm64
+      - national,lm96163
+      - sensylink,ctf2301
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  "#cooling-cells":
+    const: 2
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        temperature-sensor@4c {
+            compatible = "national,lm63";
+            reg = <0x4c>;
+            #cooling-cells = <2>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/hwmon/national,lm90.yaml b/Documentation/devicetree/bindings/hwmon/national,lm90.yaml
index 164068ba069d73..a8b2a24501b354 100644
--- a/Documentation/devicetree/bindings/hwmon/national,lm90.yaml
+++ b/Documentation/devicetree/bindings/hwmon/national,lm90.yaml
@@ -113,6 +113,23 @@ allOf:
       properties:
         ti,extended-range-enable: false
 
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              enum:
+                - adi,adt7481
+                - dallas,max6695
+                - dallas,max6696
+    then:
+      patternProperties:
+        "^channel@[0-1]$":
+          properties:
+            reg:
+              enum: [0, 1]
+        "channel@2": false
+
   - if:
       properties:
         compatible:
@@ -134,6 +151,11 @@ allOf:
         "^channel@([0-2])$":
           properties:
             temperature-offset-millicelsius: false
+    else:
+      patternProperties:
+        "channel@0":
+          properties:
+            temperature-offset-millicelsius: false
 
   - if:
       properties:
@@ -149,7 +171,7 @@ allOf:
               - onnn,nct1008
     then:
       patternProperties:
-        "^channel@([0-2])$":
+        "^channel@([1-2])$":
           properties:
             temperature-offset-millicelsius:
               maximum: 127750
@@ -172,7 +194,7 @@ allOf:
               - winbond,w83l771
     then:
       patternProperties:
-        "^channel@([0-2])$":
+        "^channel@([1-2])$":
           properties:
             temperature-offset-millicelsius:
               maximum: 127875
@@ -186,7 +208,7 @@ allOf:
               - ti,tmp461
     then:
       patternProperties:
-        "^channel@([0-2])$":
+        "^channel@([1-2])$":
           properties:
             temperature-offset-millicelsius:
               maximum: 127937
diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/adi,max20826.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/adi,max20826.yaml
new file mode 100644
index 00000000000000..e3bb51d6d1d642
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/pmbus/adi,max20826.yaml
@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/pmbus/adi,max20826.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices MAX20826 PMBus Voltage Regulator
+
+maintainers:
+  - Nuno Sá <nuno.sa@analog.com>
+
+description:
+  Analog Devices MAX20826 and similar devices are Dual-Output Multiphase
+  High-Current Controller.
+
+properties:
+  compatible:
+    enum:
+      - adi,max20826
+      - adi,max20855b
+      - adi,max20908
+      - adi,max20912
+      - adi,max20916
+
+  reg:
+    maxItems: 1
+
+  vdd3p3-supply:
+    description:
+      3.3V bias supply. It is the input of the internal 1.8V linear regulator
+      which supplies the controller.
+
+  avren-gpios:
+    description: GPIO pin to enable/disable the output voltage on Rail A.
+    maxItems: 1
+
+  bvren-gpios:
+    description: GPIO pin to enable/disable the output voltage on Rail B.
+    maxItems: 1
+
+  adi,rail-a-high-speed:
+    type: boolean
+    description:
+      The output voltage of Rail A is controlled by the high-speed processor
+      voltage-control interface the device powered up in (PWMVID, SVID, SVI3 or
+      AVSBus). If not given, PMBus controls the output voltage of Rail A.
+
+  adi,rail-b-high-speed:
+    type: boolean
+    description:
+      Same as adi,rail-a-high-speed but for Rail B.
+
+  regulators:
+    type: object
+    description:
+      List of regulators provided by this controller. vout0 is Rail A and
+      vout1 is Rail B.
+
+    patternProperties:
+      "^vout[0-1]$":
+        $ref: /schemas/regulator/regulator.yaml#
+        type: object
+        unevaluatedProperties: false
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - vdd3p3-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        regulator@64 {
+            compatible = "adi,max20826";
+            reg = <0x64>;
+
+            vdd3p3-supply = <&vdd3p3>;
+
+            avren-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>;
+            bvren-gpios = <&gpio 1 GPIO_ACTIVE_HIGH>;
+
+            adi,rail-a-high-speed;
+
+            regulators {
+                vout0 {
+                    regulator-name = "vcore_a";
+                };
+
+                vout1 {
+                    regulator-name = "vcore_b";
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38740.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38740.yaml
new file mode 100644
index 00000000000000..6733e44c37f602
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38740.yaml
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/pmbus/infineon,tda38740.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Infineon TDA38740 and TDA38725 PMBus Synchronous Buck Regulator
+
+maintainers:
+  - Colin Huang <u8813345@gmail.com>
+
+description: |
+  The Infineon TDA38740/TDA38725 is a 40A/25A single-voltage synchronous
+  buck regulator with a PMBus interface designed for industrial use.
+
+  Datasheet:
+  https://www.infineon.com/row/public/documents/24/49/infineon-tda38740-tda38725-datasheet-en.pdf
+
+properties:
+  compatible:
+    oneOf:
+      - const: infineon,tda38740
+      - items:
+          - const: infineon,tda38725
+          - const: infineon,tda38740
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        regulator@40 {
+            compatible = "infineon,tda38725",
+                         "infineon,tda38740";
+            reg = <0x40>;
+
+            interrupt-parent = <&sgpiom0>;
+            interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
+        };
+    };
+
diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/isil,isl68137.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/isil,isl68137.yaml
index 8216cdf758d8cc..a604b67b764dd5 100644
--- a/Documentation/devicetree/bindings/hwmon/pmbus/isil,isl68137.yaml
+++ b/Documentation/devicetree/bindings/hwmon/pmbus/isil,isl68137.yaml
@@ -65,6 +65,8 @@ properties:
           - enum:
               - renesas,raa228942
               - renesas,raa228943
+              - renesas,raa229639
+              - renesas,raa229640
           - const: renesas,raa228244
 
   reg:
diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/ti,tps25990.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/ti,tps25990.yaml
index f4115870e45094..63ccb67576df8b 100644
--- a/Documentation/devicetree/bindings/hwmon/pmbus/ti,tps25990.yaml
+++ b/Documentation/devicetree/bindings/hwmon/pmbus/ti,tps25990.yaml
@@ -5,18 +5,20 @@
 $id: http://devicetree.org/schemas/hwmon/pmbus/ti,tps25990.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Texas Instruments TPS25990 Stackable eFuse
+title: Texas Instruments Stackable eFuses
 
 maintainers:
   - Jerome Brunet <jbrunet@baylibre.com>
 
 description:
-  The TI TPS25990 is an integrated, high-current circuit
+  The TI TPS25990 and TPS1689 are integrated, high-current circuit
   protection and power management device with PMBUS interface
 
 properties:
   compatible:
-    const: ti,tps25990
+    enum:
+      - ti,tps1689
+      - ti,tps25990
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/hwmon/ti,tmp102.yaml b/Documentation/devicetree/bindings/hwmon/ti,tmp102.yaml
index 96b2e4969f78a1..c0b747a9ce0de4 100644
--- a/Documentation/devicetree/bindings/hwmon/ti,tmp102.yaml
+++ b/Documentation/devicetree/bindings/hwmon/ti,tmp102.yaml
@@ -4,15 +4,21 @@
 $id: http://devicetree.org/schemas/hwmon/ti,tmp102.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: TMP102 temperature sensor
+title: TMP102/TMP103/TMP110/TMP113 temperature sensor
 
 maintainers:
   - Krzysztof Kozlowski <krzk@kernel.org>
 
 properties:
   compatible:
-    enum:
-      - ti,tmp102
+    oneOf:
+      - items:
+          - enum:
+              - ti,tmp110
+              - ti,tmp113
+          - const: ti,tmp102
+      - const: ti,tmp102
+      - const: ti,tmp103
 
   interrupts:
     maxItems: 1
@@ -25,10 +31,10 @@ properties:
       A descriptive name for this channel, like "ambient" or "psu".
 
   "#thermal-sensor-cells":
-    const: 1
+    enum: [0, 1]
 
   vcc-supply:
-    description: Power supply for tmp102
+    description: Power supply for the sensor
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/hwmon/ti,tmp401.yaml b/Documentation/devicetree/bindings/hwmon/ti,tmp401.yaml
index 0e8ddf0ad7890d..f283b567d5b6cd 100644
--- a/Documentation/devicetree/bindings/hwmon/ti,tmp401.yaml
+++ b/Documentation/devicetree/bindings/hwmon/ti,tmp401.yaml
@@ -50,6 +50,9 @@ properties:
     minimum: 0
     maximum: 15
 
+  "#thermal-sensor-cells":
+    const: 1
+
 allOf:
   - if:
       properties:
diff --git a/Documentation/devicetree/bindings/hwmon/vexpress.txt b/Documentation/devicetree/bindings/hwmon/vexpress.txt
deleted file mode 100644
index 4a4df4ffc460be..00000000000000
--- a/Documentation/devicetree/bindings/hwmon/vexpress.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-Versatile Express hwmon sensors
--------------------------------
-
-Requires node properties:
-- "compatible" value : one of
-	"arm,vexpress-volt"
-	"arm,vexpress-amp"
-	"arm,vexpress-temp"
-	"arm,vexpress-power"
-	"arm,vexpress-energy"
-- "arm,vexpress-sysreg,func" when controlled via vexpress-sysreg
-  (see Documentation/devicetree/bindings/arm/vexpress-config.yaml
-  for more details)
-
-Optional node properties:
-- label : string describing the monitored value
-
-Example:
-	energy@0 {
-		compatible = "arm,vexpress-energy";
-		arm,vexpress-sysreg,func = <13 0>;
-		label = "A15 Jcore";
-	};
diff --git a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml
index d39f5e3f1df4c7..641f24b0ef4b6f 100644
--- a/Documentation/devicetree/bindings/i2c/apple,i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/apple,i2c.yaml
@@ -27,6 +27,7 @@ properties:
               - apple,t6030-i2c
               - apple,t6031-i2c
               - apple,t8122-i2c
+              - apple,t8132-i2c
           - const: apple,t8103-i2c
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml
index c4efcef591337b..2a9e723a724f51 100644
--- a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml
@@ -34,6 +34,7 @@ properties:
               - allwinner,sun50i-h616-i2c
               - allwinner,sun50i-r329-i2c
               - allwinner,sun55i-a523-i2c
+              - allwinner,sun60i-a733-i2c
           - const: allwinner,sun8i-v536-i2c
           - const: allwinner,sun6i-a31-i2c
       - const: marvell,mv64xxx-i2c
diff --git a/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml b/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml
index f9ede03c328706..4148081e3d84ed 100644
--- a/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml
@@ -83,14 +83,14 @@ properties:
 
   dmas:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
     description:
       Must contain a list of pairs of references to DMA specifiers, one for
       transmission, and one for reception.
 
   dma-names:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
     items:
       enum:
         - tx
diff --git a/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml
index 467bdcbb85389d..8a7c0a44601b3a 100644
--- a/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml
@@ -21,6 +21,20 @@ allOf:
       properties:
         reg:
           maxItems: 1
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ultrarisc,dp1000-i2c
+    then:
+      properties:
+        clocks:
+          minItems: 2
+        clock-names:
+          minItems: 2
+      required:
+        - clocks
+        - clock-names
 
 properties:
   compatible:
@@ -44,6 +58,7 @@ properties:
               - mscc,ocelot-i2c
               - sophgo,sg2044-i2c
               - thead,th1520-i2c
+              - ultrarisc,dp1000-i2c
           - const: snps,designware-i2c
 
   reg:
diff --git a/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml b/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml
index 658ae92fa86df4..57caebe7af3591 100644
--- a/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml
+++ b/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Xilinx IIC controller
 
 maintainers:
-  - info@mocean-labs.com
+  - Michal Simek <michal.simek@amd.com>
 
 allOf:
   - $ref: /schemas/i2c/i2c-controller.yaml#
@@ -52,8 +52,8 @@ examples:
     axi_iic_0: i2c@40800000 {
       compatible = "xlnx,xps-iic-2.00.a";
       clocks = <&clkc 15>;
-      interrupts = < 1 2 >;
-      reg = < 0x40800000 0x10000 >;
+      interrupts = <1 2>;
+      reg = <0x40800000 0x10000>;
 
       #size-cells = <0>;
       #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml
index b6ba7ad1a8d5dd..3ce5fb1be57478 100644
--- a/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml
+++ b/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml
@@ -4,21 +4,23 @@
 $id: http://devicetree.org/schemas/iio/accel/adi,adis16201.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: ADIS16201 Dual Axis Inclinometer and similar
+title: ADIS16201 Inclinometer and similar
 
 maintainers:
   - Jonathan Cameron <Jonathan.Cameron@huawei.com>
 
 description: |
-  Two similar parts from external interface point of view.
+  Three similar parts from external interface point of view.
   SPI interface.
     https://www.analog.com/en/products/adis16201.html
+    https://www.analog.com/en/products/adis16203.html
     https://www.analog.com/en/products/adis16209.html
 
 properties:
   compatible:
     enum:
       - adi,adis16201
+      - adi,adis16203
       - adi,adis16209
 
   reg:
diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml
index f10d98d34cb83e..115170eac05877 100644
--- a/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml
+++ b/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Analog Devices ADXL367 3-Axis Digital Accelerometer
 
 maintainers:
-  - Cosmin Tanislav <cosmin.tanislav@analog.com>
+  - Antoniu Miclaus <antoniu.miclaus@analog.com>
 
 description: |
   The ADXL367 is an ultralow power, 3-axis MEMS accelerometer.
@@ -33,7 +33,14 @@ properties:
     maxItems: 1
 
   interrupts:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
+
+  interrupt-names:
+    minItems: 1
+    items:
+      - enum: [INT1, INT2]
+      - const: INT2
 
   vdd-supply: true
   vddio-supply: true
@@ -61,6 +68,7 @@ examples:
         reg = <0x53>;
         interrupt-parent = <&gpio>;
         interrupts = <25 IRQ_TYPE_EDGE_RISING>;
+        interrupt-names = "INT1";
       };
     };
   - |
@@ -76,5 +84,6 @@ examples:
         spi-max-frequency = <1000000>;
         interrupt-parent = <&gpio>;
         interrupts = <25 IRQ_TYPE_EDGE_RISING>;
+        interrupt-names = "INT1";
       };
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
index db8467b98e67df..c8ac92e6f8e128 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
@@ -41,6 +41,7 @@ properties:
       - adi,ad4880
       - adi,ad4883
       - adi,ad4884
+      - adi,ad4885
 
   reg:
     minItems: 1
@@ -105,6 +106,7 @@ allOf:
               - adi,ad4880
               - adi,ad4883
               - adi,ad4884
+              - adi,ad4885
     then:
       properties:
         reg:
@@ -127,15 +129,15 @@ examples:
         #size-cells = <0>;
 
         adc@0 {
-          compatible = "adi,ad4080";
-          reg = <0>;
-          spi-max-frequency = <10000000>;
-          vdd33-supply = <&vdd33>;
-          vddldo-supply = <&vddldo>;
-          vrefin-supply = <&vrefin>;
-          clocks = <&cnv>;
-          clock-names = "cnv";
-          io-backends = <&iio_backend>;
+            compatible = "adi,ad4080";
+            reg = <0>;
+            spi-max-frequency = <10000000>;
+            vdd33-supply = <&vdd33>;
+            vddldo-supply = <&vddldo>;
+            vrefin-supply = <&vrefin>;
+            clocks = <&cnv>;
+            clock-names = "cnv";
+            io-backends = <&iio_backend>;
         };
     };
   - |
@@ -144,15 +146,15 @@ examples:
         #size-cells = <0>;
 
         adc@0 {
-          compatible = "adi,ad4880";
-          reg = <0>, <1>;
-          spi-max-frequency = <10000000>;
-          vdd33-supply = <&vdd33>;
-          vddldo-supply = <&vddldo>;
-          vrefin-supply = <&vrefin>;
-          clocks = <&cnv>;
-          clock-names = "cnv";
-          io-backends = <&iio_backend_cha>, <&iio_backend_chb>;
+            compatible = "adi,ad4880";
+            reg = <0>, <1>;
+            spi-max-frequency = <10000000>;
+            vdd33-supply = <&vdd33>;
+            vddldo-supply = <&vddldo>;
+            vrefin-supply = <&vrefin>;
+            clocks = <&cnv>;
+            clock-names = "cnv";
+            io-backends = <&iio_backend_cha>, <&iio_backend_chb>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml
index cc38617bb8295c..a4ab6adef1c68f 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml
@@ -8,7 +8,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Analog Devices AD4130 family ADCs
 
 maintainers:
-  - Cosmin Tanislav <cosmin.tanislav@analog.com>
+  - Marcelo Schmitt <marcelo.schmitt@analog.com>
+  - Antoniu Miclaus <antoniu.miclaus@analog.com>
 
 description: |
   Bindings for the Analog Devices AD4130 family ADCs.
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4851.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4851.yaml
index b107322e0ea391..49708ce0b2308c 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad4851.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4851.yaml
@@ -142,13 +142,13 @@ examples:
             io-backends = <&iio_backend>;
 
             channel@0 {
-              reg = <0>;
-              diff-channels = <0 0>;
-              bipolar;
+                reg = <0>;
+                diff-channels = <0 0>;
+                bipolar;
             };
 
             channel@1 {
-              reg = <1>;
+                reg = <1>;
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7173.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7173.yaml
index 62d906e2499732..e178007b9a2285 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad7173.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7173.yaml
@@ -443,7 +443,7 @@ examples:
       #address-cells = <1>;
       #size-cells = <0>;
 
-       adc@0 {
+      adc@0 {
         compatible = "adi,ad4111";
         reg = <0>;
 
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml
index dfa2d7fa9fb37e..7b72d14e104c25 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml
@@ -238,9 +238,9 @@ examples:
             };
 
             regulators {
-              vcm_reg: vcm-output {
-                regulator-name = "ad7768-1-vcm";
-              };
+                vcm_reg: vcm-output {
+                    regulator-name = "ad7768-1-vcm";
+                };
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7768.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7768.yaml
new file mode 100644
index 00000000000000..d8e489fa1f6ffb
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7768.yaml
@@ -0,0 +1,270 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/adi,ad7768.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices AD7768 and AD7768-4 ADC
+
+maintainers:
+  - Janani Sunil <janani.sunil@analog.com>
+
+description: |
+  The AD7768 is an 8-channel, 24-bit simultaneous sampling ADC with configurable
+  power and performance modes. The AD7768-4 is a 4-channel version.
+
+  This binding covers SPI mode only (register access via SPI). Pin control mode,
+  where configuration is set via hardware pins rather than SPI registers, is not
+  supported.
+
+  The io-backend represents the high-speed data output interface, consuming the
+  DOUTx data lines, DCLK (data clock), and DRDY (data ready) signals.
+
+  Datasheet at:
+    https://www.analog.com/media/en/technical-documentation/data-sheets/ad7768-ad7768-4.pdf
+
+  HDL reference design:
+    https://github.com/analogdevicesinc/hdl/tree/main/projects/ad7768evb
+
+properties:
+  compatible:
+    enum:
+      - adi,ad7768
+      - adi,ad7768-4
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    description:
+      Describes the type of master clock (MCLK) attached to the XTAL1 and
+      XTAL2/MCLK pins. A CMOS clock is supplied on the XTAL2/MCLK pin, a
+      crystal oscillator is connected between the XTAL1 and XTAL2 pins, and an
+      LVDS differential clock is supplied on the XTAL1/XTAL2 pin pair. CMOS and
+      LVDS clocks must operate from 1.15 MHz to 34 MHz. Crystals must operate
+      from 8 MHz to 34 MHz.
+    enum:
+      - mclk
+      - xtal
+      - lvds
+
+  avdd1-supply:
+    description:
+      Analog supply. Its voltage is used to derive the supply-dependent VCM
+      output level, so it must be a regulator that can report its voltage.
+
+  avss-supply:
+    description:
+      Negative analog supply. If omitted, AVSS is assumed to be at system
+      ground.
+
+  avdd2-supply: true
+
+  iovdd-supply: true
+
+  ref1p-supply: true
+
+  ref1n-supply:
+    description:
+      Negative reference input supply for REF1. If omitted, REF1- is assumed
+      to be connected to ground.
+
+  ref2p-supply: true
+
+  ref2n-supply:
+    description:
+      Negative reference input supply for REF2. If omitted, REF2- is assumed
+      to be connected to ground.
+
+  reset-gpios:
+    maxItems: 1
+    description: GPIO connected to the active-low RESET pin
+
+  start-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the START pin. An active-low pulse on this pin
+      synchronizes the device to other AD7768 devices. When not used, this
+      pin must be tied to a logic high level through a pull-up resistor.
+      When used, SYNC_OUT must be wired back to SYNC_IN on all devices in
+      the synchronization chain.
+
+  sync-in-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the SYNC_IN pin. Receives the synchronization pulse
+      generated on SYNC_OUT in response to a START pulse. Must be synchronous
+      to the MCLK clock domain. In multi-device or daisy-chained systems,
+      two successive synchronization pulses must be applied to guarantee all
+      ADCs are synchronized.
+
+  gpio-controller: true
+
+  '#gpio-cells':
+    const: 2
+
+  adi,data-lines-number:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [1, 2, 4, 8]
+    description:
+      Number of DOUTx lines used for the high-speed data output interface
+      (distinct from the SPI control interface). AD7768 supports 1, 2, or 8
+      lines; AD7768-4 supports 1 or 4 lines. Defaults to the maximum number
+      of lines for the selected variant if omitted.
+
+  regulators:
+    type: object
+    description: Container for the on-chip VCM regulator.
+    additionalProperties: false
+
+    properties:
+      vcm-output:
+        type: object
+        description:
+          Buffered common-mode voltage output on the VCM pin. Use
+          regulator-always-on when no consumer node references this output.
+          Channel 0 must be present and enabled while VCM is used, since
+          putting channel 0 in standby disables the VCM output.
+        $ref: /schemas/regulator/regulator.yaml#
+        unevaluatedProperties: false
+
+  io-backends:
+    maxItems: 1
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 0
+
+patternProperties:
+  "^channel@[0-7]$":
+    type: object
+    description:
+      Represents the external channels which are connected to the device.
+      AD7768 supports channels 0-7, AD7768-4 supports channels 0-3.
+
+    properties:
+      reg:
+        minimum: 0
+        maximum: 7
+        description: The channel number
+
+      adi,prechargebuf-pos-enable:
+        type: boolean
+        description: Enable positive input precharge buffer
+
+      adi,prechargebuf-neg-enable:
+        type: boolean
+        description: Enable negative input precharge buffer
+
+      adi,refbuf-pos-enable:
+        type: boolean
+        description: Enable positive reference buffer
+
+      adi,refbuf-neg-enable:
+        type: boolean
+        description: Enable negative reference buffer
+
+    required:
+      - reg
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - avdd1-supply
+  - avdd2-supply
+  - iovdd-supply
+  - ref1p-supply
+  - ref2p-supply
+  - io-backends
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: adi,ad7768-4
+    then:
+      properties:
+        adi,data-lines-number:
+          enum: [1, 4]
+      patternProperties:
+        "^channel@[0-3]$":
+          properties:
+            reg:
+              maximum: 3
+        "^channel@[4-7]$": false
+    else:
+      properties:
+        adi,data-lines-number:
+          enum: [1, 2, 8]
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@0 {
+            compatible = "adi,ad7768";
+            reg = <0>;
+            spi-max-frequency = <1000000>;
+
+            avdd1-supply = <&avdd1_regulator>;
+            avss-supply = <&avss_regulator>;
+            avdd2-supply = <&avdd2_regulator>;
+            iovdd-supply = <&iovdd_regulator>;
+            ref1p-supply = <&ref1p_regulator>;
+            ref1n-supply = <&ref1n_regulator>;
+            ref2p-supply = <&ref2p_regulator>;
+            ref2n-supply = <&ref2n_regulator>;
+            reset-gpios = <&gpio0 86 GPIO_ACTIVE_LOW>;
+
+            clocks = <&ad7768_mclk>;
+            clock-names = "mclk";
+
+            adi,data-lines-number = <8>;
+            gpio-controller;
+            #gpio-cells = <2>;
+
+            io-backends = <&iio_backend>;
+
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            channel@0 {
+                reg = <0>;
+                adi,prechargebuf-pos-enable;
+                adi,refbuf-pos-enable;
+            };
+
+            channel@1 {
+                reg = <1>;
+                adi,prechargebuf-pos-enable;
+                adi,prechargebuf-neg-enable;
+                adi,refbuf-pos-enable;
+                adi,refbuf-neg-enable;
+            };
+
+            channel@7 {
+                reg = <7>;
+                adi,prechargebuf-pos-enable;
+                adi,refbuf-pos-enable;
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7779.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7779.yaml
index ba3f7b2bd6cf1e..37e25a6196e72d 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad7779.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7779.yaml
@@ -122,14 +122,14 @@ examples:
         #size-cells = <0>;
 
         adc@0 {
-          compatible = "adi,ad7779";
-          reg = <0>;
-          start-gpios = <&gpio0 87 GPIO_ACTIVE_LOW>;
-          reset-gpios = <&gpio0 93 GPIO_ACTIVE_LOW>;
-          interrupt-parent = <&intc>;
-          interrupts = <0 56 IRQ_TYPE_LEVEL_HIGH>;
-          interrupt-names = "adc_rdy";
-          clocks = <&adc_clk>;
+            compatible = "adi,ad7779";
+            reg = <0>;
+            start-gpios = <&gpio0 87 GPIO_ACTIVE_LOW>;
+            reset-gpios = <&gpio0 93 GPIO_ACTIVE_LOW>;
+            interrupt-parent = <&intc>;
+            interrupts = <0 56 IRQ_TYPE_LEVEL_HIGH>;
+            interrupt-names = "adc_rdy";
+            clocks = <&adc_clk>;
         };
     };
 
@@ -140,13 +140,13 @@ examples:
         #size-cells = <0>;
 
         adc@0 {
-          compatible = "adi,ad7779";
-          reg = <0>;
-          start-gpios = <&gpio0 87 GPIO_ACTIVE_LOW>;
-          reset-gpios = <&gpio0 93 GPIO_ACTIVE_LOW>;
-          clocks = <&adc_clk>;
-          io-backends = <&iio_backend>;
-          adi,num-lanes = <4>;
+            compatible = "adi,ad7779";
+            reg = <0>;
+            start-gpios = <&gpio0 87 GPIO_ACTIVE_LOW>;
+            reset-gpios = <&gpio0 93 GPIO_ACTIVE_LOW>;
+            clocks = <&adc_clk>;
+            io-backends = <&iio_backend>;
+            adi,num-lanes = <4>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ade9000.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ade9000.yaml
index f22eba0250ee5c..0971a29bf37209 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ade9000.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ade9000.yaml
@@ -5,21 +5,23 @@
 $id: http://devicetree.org/schemas/iio/adc/adi,ade9000.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Analog Devices ADE9000 High Performance, Polyphase Energy Metering
+title: Analog Devices ADE9000/ADE9078 High Performance, Polyphase Energy Metering
 
 maintainers:
   - Antoniu Miclaus <antoniu.miclaus@analog.com>
 
 description: |
-  The ADE9000 is a highly accurate, fully integrated, multiphase energy and power
-  quality monitoring device. Superior analog performance and a digital signal
-  processing (DSP) core enable accurate energy monitoring over a wide dynamic
-  range. An integrated high end reference ensures low drift over temperature
-  with a combined drift of less than ±25 ppm/°C maximum for the entire channel
-  including a programmable gain amplifier (PGA) and an analog-to-digital
-  converter (ADC).
+  The ADE9000 and ADE9078 are highly accurate, fully integrated, multiphase
+  energy and power quality monitoring devices. Superior analog performance and a
+  digital signal processing (DSP) core enable accurate energy monitoring over a
+  wide dynamic range. An integrated high end reference ensures low drift over
+  temperature with a combined drift of less than ±25 ppm/°C maximum for the
+  entire channel including a programmable gain amplifier (PGA) and an
+  analog-to-digital converter (ADC). The ADE9078 does not provide the on-chip
+  dip/swell detection available on the ADE9000.
 
   https://www.analog.com/media/en/technical-documentation/data-sheets/ADE9000.pdf
+  https://www.analog.com/media/en/technical-documentation/data-sheets/ADE9078.pdf
 
 $ref: /schemas/spi/spi-peripheral-props.yaml#
 
@@ -27,6 +29,7 @@ properties:
   compatible:
     enum:
       - adi,ade9000
+      - adi,ade9078
 
   reg:
     maxItems: 1
@@ -67,6 +70,17 @@ required:
   - reg
   - vdd-supply
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: adi,ade9078
+    then:
+      properties:
+        spi-max-frequency:
+          maximum: 10000000
+
 unevaluatedProperties: false
 
 examples:
@@ -75,20 +89,20 @@ examples:
     #include <dt-bindings/interrupt-controller/irq.h>
 
     spi {
-      #address-cells = <1>;
-      #size-cells = <0>;
-
-      adc@0 {
-          compatible = "adi,ade9000";
-          reg = <0>;
-          spi-max-frequency = <7000000>;
-
-          #clock-cells = <0>;
-          reset-gpios = <&gpio 4 GPIO_ACTIVE_LOW>;
-          interrupts = <2 IRQ_TYPE_EDGE_FALLING>, <3 IRQ_TYPE_EDGE_FALLING>, <4 IRQ_TYPE_EDGE_FALLING>;
-          interrupt-names = "irq0", "irq1", "dready";
-          interrupt-parent = <&gpio>;
-          clocks = <&ext_clock_24576khz>;
-          vdd-supply = <&vdd_reg>;
-      };
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@0 {
+            compatible = "adi,ade9000";
+            reg = <0>;
+            spi-max-frequency = <7000000>;
+
+            #clock-cells = <0>;
+            reset-gpios = <&gpio 4 GPIO_ACTIVE_LOW>;
+            interrupts = <2 IRQ_TYPE_EDGE_FALLING>, <3 IRQ_TYPE_EDGE_FALLING>, <4 IRQ_TYPE_EDGE_FALLING>;
+            interrupt-names = "irq0", "irq1", "dready";
+            interrupt-parent = <&gpio>;
+            clocks = <&ext_clock_24576khz>;
+            vdd-supply = <&vdd_reg>;
+        };
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,max40080.yaml b/Documentation/devicetree/bindings/iio/adc/adi,max40080.yaml
new file mode 100644
index 00000000000000..b0ec9159a2939e
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/adi,max40080.yaml
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/adi,max40080.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices MAX40080 bidirectional current-sense amplifier
+
+maintainers:
+  - Ciprian Hegbeli <ciprian.hegbeli@analog.com>
+  - Stefan Popa <stefan.popa@analog.com>
+
+description: |
+  The MAX40080 is a high-precision, bidirectional current-sense amplifier with
+  an integrated 12-bit ADC and an I2C/SMBus interface. It measures the voltage
+  across an external shunt resistor and the input bus voltage, and stores the
+  results in an internal FIFO.
+
+  Datasheet:
+    https://www.analog.com/max40080
+
+properties:
+  compatible:
+    const: adi,max40080
+
+  reg:
+    maxItems: 1
+
+  "#io-channel-cells":
+    const: 1
+
+  vdd-supply:
+    description: Power supply for the device.
+
+  interrupts:
+    maxItems: 1
+    description: ALERT_ output for overcurrent/undervoltage threshold events.
+
+  shunt-resistor-micro-ohms:
+    description:
+      Value of the current-sense shunt resistor connected between the RS+ and
+      RS- inputs. Used to scale the reported current.
+
+required:
+  - compatible
+  - reg
+  - shunt-resistor-micro-ohms
+  - vdd-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@20 {
+            compatible = "adi,max40080";
+            reg = <0x20>;
+            #io-channel-cells = <1>;
+            vdd-supply = <&vdd>;
+            shunt-resistor-micro-ohms = <100000>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/axiado,ax3000-saradc.yaml b/Documentation/devicetree/bindings/iio/adc/axiado,ax3000-saradc.yaml
new file mode 100644
index 00000000000000..b910852aa56f28
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/axiado,ax3000-saradc.yaml
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/axiado,ax3000-saradc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Axiado AX3000/AX3005 Successive Approximation Register ADC
+
+description:
+  The Axiado AX3000/AX3005 SAR ADC is a 10-bit ADC with sixteen input
+  channels on AX3000 and eight input channels on AX3005.
+
+maintainers:
+  - Petar Stepanovic <pstepanovic@axiado.com>
+  - Akhila Kavi <akavi@axiado.com>
+  - Prasad Bolisetty <pbolisetty@axiado.com>
+
+properties:
+  compatible:
+    enum:
+      - axiado,ax3000-saradc
+      - axiado,ax3005-saradc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: saradc
+
+  '#io-channel-cells':
+    const: 1
+
+  vref-supply:
+    description: Reference voltage regulator supplying the ADC
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - '#io-channel-cells'
+  - vref-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      adc@806a0000 {
+        compatible = "axiado,ax3000-saradc";
+        reg = <0x0 0x806a0000 0x0 0x400>;
+        clocks = <&pclk>;
+        clock-names = "saradc";
+        vref-supply = <&vref_reg>;
+        #io-channel-cells = <1>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml
index c884b6e0376707..67445ad2bb3b1f 100644
--- a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml
@@ -4,7 +4,7 @@
 $id: http://devicetree.org/schemas/iio/adc/lltc,ltc2497.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Linear Technology / Analog Devices LTC2497 and LTC2309 ADC
+title: Linear Technology / Analog Devices LTC2497 and similar ADCs
 
 maintainers:
   - Michael Hennerich <michael.hennerich@analog.com>
diff --git a/Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml b/Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml
index 4cba856e8d4732..bad780d1bbfb52 100644
--- a/Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml
@@ -118,22 +118,22 @@ examples:
         #size-cells = <0>;
 
         adc@1e {
-              compatible = "maxim,max34409";
-              reg = <0x1e>;
-              powerdown-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>;
-              powerdown-status-gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>;
-
-              #address-cells = <1>;
-              #size-cells = <0>;
-
-              channel@0 {
-                  reg = <0x0>;
-                  maxim,rsense-val-micro-ohms = <5000>;
-              };
-
-              channel@1 {
-                  reg = <0x1>;
-                  maxim,rsense-val-micro-ohms = <10000>;
-             };
+            compatible = "maxim,max34409";
+            reg = <0x1e>;
+            powerdown-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>;
+            powerdown-status-gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>;
+
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            channel@0 {
+                reg = <0x0>;
+                maxim,rsense-val-micro-ohms = <5000>;
+            };
+
+            channel@1 {
+                reg = <0x1>;
+                maxim,rsense-val-micro-ohms = <10000>;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/mediatek,mt2701-auxadc.yaml b/Documentation/devicetree/bindings/iio/adc/mediatek,mt2701-auxadc.yaml
index d9e825e5054fe5..750e1e837fb887 100644
--- a/Documentation/devicetree/bindings/iio/adc/mediatek,mt2701-auxadc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/mediatek,mt2701-auxadc.yaml
@@ -38,6 +38,7 @@ properties:
           - const: mediatek,mt7986-auxadc
       - items:
           - enum:
+              - mediatek,mt6572-auxadc
               - mediatek,mt6893-auxadc
               - mediatek,mt8183-auxadc
               - mediatek,mt8186-auxadc
diff --git a/Documentation/devicetree/bindings/iio/adc/nxp,lpc1850-adc.yaml b/Documentation/devicetree/bindings/iio/adc/nxp,lpc1850-adc.yaml
index 70b38038a0805f..1e2df3dd15d1a4 100644
--- a/Documentation/devicetree/bindings/iio/adc/nxp,lpc1850-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/nxp,lpc1850-adc.yaml
@@ -56,6 +56,6 @@ examples:
             clocks = <&ccu1 CLK_APB3_ADC0>;
             vref-supply = <&reg_vdda>;
             resets = <&rgu 40>;
-         };
+        };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/adc/renesas,r9a09g077-adc.yaml b/Documentation/devicetree/bindings/iio/adc/renesas,r9a09g077-adc.yaml
index dc0206b282315d..5b496729022e19 100644
--- a/Documentation/devicetree/bindings/iio/adc/renesas,r9a09g077-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/renesas,r9a09g077-adc.yaml
@@ -34,8 +34,6 @@ properties:
       - description: A/D scan end interrupt for Group C
       - description: Window A compare match
       - description: Window B compare match
-      - description: Compare match
-      - description: Compare mismatch
 
   interrupt-names:
     items:
@@ -44,8 +42,6 @@ properties:
       - const: gcadi
       - const: cmpai
       - const: cmpbi
-      - const: wcmpm
-      - const: wcmpum
 
   clocks:
     items:
@@ -57,6 +53,18 @@ properties:
       - const: adclk
       - const: pclk
 
+  dmas:
+    minItems: 1
+    maxItems: 3
+    description:
+      Must contain a list of references to DMA RX specifiers.
+
+  dma-names:
+    minItems: 1
+    maxItems: 3
+    items:
+      const: rx
+
   power-domains:
     maxItems: 1
 
@@ -107,11 +115,9 @@ examples:
                    <GIC_SPI 709 IRQ_TYPE_EDGE_RISING>,
                    <GIC_SPI 710 IRQ_TYPE_EDGE_RISING>,
                    <GIC_SPI 711 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SPI 712 IRQ_TYPE_LEVEL_HIGH>,
-                   <GIC_SPI 855 IRQ_TYPE_EDGE_RISING>,
-                   <GIC_SPI 856 IRQ_TYPE_EDGE_RISING>;
+                   <GIC_SPI 712 IRQ_TYPE_LEVEL_HIGH>;
       interrupt-names = "adi", "gbadi", "gcadi",
-                        "cmpai", "cmpbi", "wcmpm", "wcmpum";
+                        "cmpai", "cmpbi";
       clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKL>,
                <&cpg CPG_MOD 225>;
       clock-names = "adclk", "pclk";
diff --git a/Documentation/devicetree/bindings/iio/adc/renesas,rzg2l-adc.yaml b/Documentation/devicetree/bindings/iio/adc/renesas,rzg2l-adc.yaml
index 40341d5417264f..f08f5439d1272e 100644
--- a/Documentation/devicetree/bindings/iio/adc/renesas,rzg2l-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/renesas,rzg2l-adc.yaml
@@ -25,7 +25,9 @@ properties:
               - renesas,r9a07g054-adc   # RZ/V2L
           - const: renesas,rzg2l-adc
       - items:
-          - const: renesas,r9a08g045-adc  # RZ/G3S
+          - enum:
+              - renesas,r9a08g045-adc   # RZ/G3S
+              - renesas,r9a08g046-adc1  # RZ/G3L ADC for TSU
 
   reg:
     maxItems: 1
@@ -121,6 +123,19 @@ allOf:
             reg:
               maximum: 7
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a08g046-adc1
+    then:
+      patternProperties:
+        "^channel@[0-7]$": false
+        "^channel@[8]$":
+          properties:
+            reg:
+              const: 8
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml
index f0551d66591629..0cce357cb01de9 100644
--- a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml
@@ -12,19 +12,19 @@ maintainers:
 properties:
   compatible:
     oneOf:
-      - const: rockchip,saradc
-      - const: rockchip,rk3066-tsadc
-      - const: rockchip,rk3399-saradc
-      - const: rockchip,rk3528-saradc
+      - enum:
+          - rockchip,saradc
+          - rockchip,rk3066-tsadc
+          - rockchip,rk3399-saradc
+          - rockchip,rk3528-saradc
+          - rockchip,rk3562-saradc
+          - rockchip,rk3588-saradc
+          - rockchip,rv1106-saradc
       - items:
           - const: rockchip,rk3506-saradc
           - const: rockchip,rk3528-saradc
-      - const: rockchip,rk3562-saradc
-      - const: rockchip,rk3588-saradc
       - items:
-          - enum:
-              - rockchip,rk3576-saradc
-              - rockchip,rv1106-saradc
+          - const: rockchip,rk3576-saradc
           - const: rockchip,rk3588-saradc
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml
index 718f633c6e04c7..777a75c21d7f7f 100644
--- a/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml
@@ -66,6 +66,11 @@ patternProperties:
           - minimum: 0
             maximum: 7
 
+      label:
+        $ref: /schemas/types.yaml#/definitions/string
+        description:
+          A descriptive name for this channel, like "vcc_ram" or "CH3".
+
       ti,gain:
         $ref: /schemas/types.yaml#/definitions/uint32
         minimum: 0
@@ -125,6 +130,7 @@ examples:
                 reg = <4>;
                 ti,gain = <3>;
                 ti,datarate = <5>;
+                label = "CH3";
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads1100.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads1100.yaml
index 970ccab15e1ec4..28c5e2dd0ad626 100644
--- a/Documentation/devicetree/bindings/iio/adc/ti,ads1100.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/ti,ads1100.yaml
@@ -4,19 +4,23 @@
 $id: http://devicetree.org/schemas/iio/adc/ti,ads1100.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: TI ADS1100/ADS1000 single channel I2C analog to digital converter
+title: TI ADS1100 and similar single channel I2C Analog to Digital Converters
 
 maintainers:
   - Mike Looijmans <mike.looijmans@topic.nl>
 
 description: |
-  Datasheet at: https://www.ti.com/lit/gpn/ads1100
+  Datasheets:
+    - https://www.ti.com/lit/gpn/ads1000
+    - https://www.ti.com/lit/gpn/ads1100
+    - https://www.ti.com/lit/gpn/ads1110
 
 properties:
   compatible:
     enum:
-      - ti,ads1100
       - ti,ads1000
+      - ti,ads1100
+      - ti,ads1110
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads112c04.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads112c04.yaml
new file mode 100644
index 00000000000000..95a123cd7b67c0
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/ti,ads112c04.yaml
@@ -0,0 +1,149 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/ti,ads112c04.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments ADS112C04 ADC
+
+maintainers:
+  - Kyle Hsieh <kylehsieh1995@gmail.com>
+
+description:
+  The ADS112C04 (16-bit) are precision analog-to-digital converters (ADCs)
+  with an I2C interface. They feature a flexible input multiplexer, a
+  low-noise programmable gain amplifier (PGA), two programmable excitation
+  current sources, a voltage reference, and a precision temperature sensor.
+
+properties:
+  compatible:
+    enum:
+      - ti,ads112c04
+
+  reg:
+    maxItems: 1
+    description: I2C address of the device.
+
+  interrupts:
+    maxItems: 1
+    description:
+      Data ready (DRDY) interrupt output. DRDY pulses low when a conversion
+      result is ready and returns high once the result is latched, so an
+      edge trigger should be used.
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+  reset-gpios:
+    maxItems: 1
+    description: GPIO connected to the RESET pin. Active low.
+
+  avdd-supply: true
+  dvdd-supply: true
+
+  refp-supply: true
+  refn-supply: true
+
+  ti,refp-refn-resistor-ohms:
+    description: Resistance of the external resistor between REFP and REFN.
+
+patternProperties:
+  "^channel@[0-9a-b]$":
+    $ref: adc.yaml
+    unevaluatedProperties: false
+    properties:
+      reg:
+        items:
+          - maximum: 11
+
+      single-channel:
+        maximum: 3
+
+      diff-channels:
+        items:
+          maximum: 3
+
+      excitation-channels:
+        maxItems: 2
+        items:
+          maximum: 5
+        description:
+          Additionally to AIN0-AIN3, IDAC1/IDAC2 can also be routed to
+          REFP0 (4) and REFN0 (5).
+
+      excitation-current-nanoamp:
+        enum: [10000, 50000, 100000, 250000, 500000, 1000000, 1500000]
+
+      burn-out-current-nanoamp:
+        const: 10000
+
+      reference-sources:
+        items:
+          - enum: [internal, external, avdd]
+        default: [ internal ]
+
+    dependencies:
+      excitation-channels: [ excitation-current-nanoamp ]
+      excitation-current-nanoamp: [ excitation-channels ]
+
+    oneOf:
+      - required: [ single-channel ]
+      - required: [ diff-channels ]
+
+required:
+  - compatible
+  - reg
+  - avdd-supply
+  - dvdd-supply
+
+dependencies:
+  refn-supply: [ refp-supply ]
+
+oneOf:
+  - required:
+      - refp-supply
+  - required:
+      - ti,refp-refn-resistor-ohms
+  - properties:
+      refp-supply: false
+      refn-supply: false
+      ti,refp-refn-resistor-ohms: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/gpio/gpio.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@40 {
+            compatible = "ti,ads112c04";
+            reg = <0x40>;
+            interrupt-parent = <&gpio>;
+            interrupts = <12 IRQ_TYPE_EDGE_FALLING>;
+
+            reset-gpios = <&gpio 13 GPIO_ACTIVE_LOW>;
+            avdd-supply = <&vdd_3v3_reg>;
+            dvdd-supply = <&vdd_3v3_reg>;
+            refp-supply = <&vref_reg>;
+
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            channel@0 {
+                reg = <0>;
+                diff-channels = <0>, <1>;
+            };
+
+            channel@1 {
+                reg = <1>;
+                single-channel = <2>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads1298.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads1298.yaml
index 71f9f9b745cbc8..afcc683db1490e 100644
--- a/Documentation/devicetree/bindings/iio/adc/ti,ads1298.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/ti,ads1298.yaml
@@ -65,15 +65,15 @@ examples:
         #size-cells = <0>;
 
         adc@1 {
-          reg = <1>;
-          compatible = "ti,ads1298";
-          label = "ads1298-1-ecg";
-          avdd-supply = <&reg_iso_5v_a>;
-          clocks = <&clk_ads1298>;
-          interrupt-parent = <&gpio0>;
-          interrupts = <78 IRQ_TYPE_EDGE_FALLING>;
-          spi-max-frequency = <20000000>;
-          spi-cpha;
+            reg = <1>;
+            compatible = "ti,ads1298";
+            label = "ads1298-1-ecg";
+            avdd-supply = <&reg_iso_5v_a>;
+            clocks = <&clk_ads1298>;
+            interrupt-parent = <&gpio0>;
+            interrupts = <78 IRQ_TYPE_EDGE_FALLING>;
+            spi-max-frequency = <20000000>;
+            spi-cpha;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/addac/adi,ad74115.yaml b/Documentation/devicetree/bindings/iio/addac/adi,ad74115.yaml
index 2a04906531fb0e..6aa86cb56032af 100644
--- a/Documentation/devicetree/bindings/iio/addac/adi,ad74115.yaml
+++ b/Documentation/devicetree/bindings/iio/addac/adi,ad74115.yaml
@@ -7,7 +7,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Analog Devices AD74115H device
 
 maintainers:
-  - Cosmin Tanislav <cosmin.tanislav@analog.com>
+  - Marcelo Schmitt <marcelo.schmitt@analog.com>
+  - Antoniu Miclaus <antoniu.miclaus@analog.com>
 
 description: |
   The AD74115H is a single-channel software configurable input/output
diff --git a/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml b/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml
index 590ea7936ad7c5..5318a00e766905 100644
--- a/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml
+++ b/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml
@@ -7,7 +7,8 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Analog Devices AD74412R/AD74413R device
 
 maintainers:
-  - Cosmin Tanislav <cosmin.tanislav@analog.com>
+  - Marcelo Schmitt <marcelo.schmitt@analog.com>
+  - Antoniu Miclaus <antoniu.miclaus@analog.com>
 
 description: |
   The AD74412R and AD74413R are quad-channel software configurable input/output
diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml
index 2fe098619772f9..bf0278335583f0 100644
--- a/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml
+++ b/Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml
@@ -10,13 +10,13 @@ maintainers:
   - Kim Seer Paller <kimseer.paller@analog.com>
 
 description: |
-  The AD3530/AD3530R (8-channel), AD3531/AD3531R (4-channel), and AD3532/AD3532R
-  (16-channel) are low-power, 16-bit, buffered voltage output digital-to-analog
-  converters (DACs) with software-programmable gain controls, providing
-  full-scale output spans of 2.5V or 5V for reference voltages of 2.5V. These
-  devices operate from a single 2.7V to 5.5V supply and are guaranteed monotonic
-  by design. The "R" variants include a 2.5V, 5ppm/°C internal reference, which
-  is disabled by default.
+  The AD3530/AD3530R (8-channel), AD3531/AD3531R (4-channel), AD3532/AD3532R
+  (16-channel), 16-bit and AD3536R (16-channel), 12-bit are low-power, buffered
+  voltage output digital-to-analog converters (DACs) with software-programmable
+  gain controls, providing full-scale output spans of 2.5V or 5V for reference
+  voltages of 2.5V. These devices operate from a single 2.7V to 5.5V supply and
+  are guaranteed monotonic by design. The "R" variants include a 2.5V, 5ppm/°C
+  internal reference, which is disabled by default.
   Datasheet can be found here:
   https://www.analog.com/media/en/technical-documentation/data-sheets/ad3530_ad530r.pdf
   https://www.analog.com/media/en/technical-documentation/data-sheets/ad3531-ad3531r.pdf
@@ -31,6 +31,7 @@ properties:
       - adi,ad3531r
       - adi,ad3532
       - adi,ad3532r
+      - adi,ad3536r
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5529r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5529r.yaml
new file mode 100644
index 00000000000000..9c390d460e5ff1
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5529r.yaml
@@ -0,0 +1,254 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/dac/adi,ad5529r.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices AD5529R 16-Channel 12/16-bit High Voltage DAC
+
+maintainers:
+  - Janani Sunil <janani.sunil@analog.com>
+
+description: |
+  The AD5529R is a 16-channel, 12-bit or 16-bit, high voltage, buffered voltage
+  output digital-to-analog converter (DAC) with an integrated precision reference.
+  The device operates from unipolar and bipolar supplies. It is guaranteed
+  monotonic and has built-in rail-to-rail output buffers that can source or
+  sink up to 25mA.
+
+  Specifications:
+  * 16 independent 12-bit or 16-bit DAC channels
+  * Independently programmable output ranges: 0V to 5V, 0V to 10V, 0V to 20V,
+    0V to 40V, ±5V, ±10V, ±15V, and ±20V
+  * The device supports SPI communication with Mode 0 and Mode 3.
+  * 4.096V precision reference, 12ppm/°C maximum
+  * Built-in function generation: Toggle, Sinusoidal Dither, and Ramp waveforms
+  * Multiplexer for output voltage, load current sense and die temperature
+
+  Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/ad5529r.pdf
+
+properties:
+  compatible:
+    enum:
+      - adi,ad5529r-16   # 16-bit variant
+      - adi,ad5529r-12   # 12-bit variant
+
+  reg:
+    maxItems: 1
+
+  spi-max-frequency:
+    maximum: 25000000
+    description:
+      Maximum SPI frequency. The device supports SPI Mode 0 and Mode 3.
+      Read operations are limited to 25MHz maximum.
+
+  reset-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the RESET pin. Active low. When asserted low,
+      performs a power-on reset and initializes the device to its default state.
+
+  clear-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the CLEAR pin. Active low. When asserted low,
+      clears all DAC data registers without affecting configuration settings.
+
+  interrupts:
+    maxItems: 1
+    description:
+      Interrupt connected to the ALARM pin. Active low interrupt output
+      for overtemperature conditions, SPI CRC errors, and function completion.
+
+  pwms:
+    minItems: 1
+    maxItems: 4
+    description:
+      PWM signals connected to the TG0-TG3 toggle pins. Pulsing these pins
+      based on trigger edge settings allows selected DACs to be updated
+      synchronously for digital function generation.
+
+  pwm-names:
+    minItems: 1
+    maxItems: 4
+    items:
+      enum: [ tg0, tg1, tg2, tg3 ]
+
+  io-channels:
+    maxItems: 1
+    description:
+      ADC channel connected to the MUXOUT pin for monitoring output voltage,
+      load current sense, and die temperature.
+
+  io-channel-names:
+    const: muxout
+
+  vdd-io-supply:
+    description: Digital power supply (1.08V to 1.98V)
+
+  vdd-5v-supply:
+    description: Analog power supply (4.75V to 5.25V)
+
+  vdd-hv0-supply:
+    description:
+      High voltage positive supply (7V to 45V) for channels 0 to 3.
+
+  vdd-hv1-supply:
+    description:
+      High voltage positive supply (7V to 45V) for channels 4 to 7.
+
+  vdd-hv2-supply:
+    description:
+      High voltage positive supply (7V to 45V) for channels 8 to 11.
+
+  vdd-hv3-supply:
+    description:
+      High voltage positive supply (7V to 45V) for channels 12 to 15.
+
+  vss-hv0-supply:
+    description:
+      High voltage negative supply (-22.5V to 0V) for channels 0 to 3.
+
+  vss-hv1-supply:
+    description:
+      High voltage negative supply (-22.5V to 0V) for channels 4 to 7.
+
+  vss-hv2-supply:
+    description:
+      High voltage negative supply (-22.5V to 0V) for channels 8 to 11.
+
+  vss-hv3-supply:
+    description:
+      High voltage negative supply (-22.5V to 0V) for channels 12 to 15.
+
+  vref-supply:
+    description:
+      External voltage reference supply (4.056V to 4.136V, typically 4.096V).
+      When specified, the device uses external reference mode and the VREF pin
+      becomes an input. The device uses the internal 4.096V precision reference
+      otherwise.
+
+  spi-device-addr:
+    minItems: 1
+    maxItems: 4
+    items:
+      enum: [0, 1, 2, 3]
+    default: 0
+    description:
+      Hardware address of each device, selected by the ID0 and ID1 pins.
+      Up to four AD5529R devices can share a single SPI chip select.
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+patternProperties:
+  "^channel@([0-9a-f]{1,2})$":
+    $ref: dac.yaml
+    type: object
+    description: Child nodes for individual channel configuration
+
+    properties:
+      reg:
+        description:
+          Flattened channel number across all AD5529R devices sharing the chip
+          select. Within each 16-channel block, the physical channel number is reg % 16.
+        minimum: 0
+        maximum: 63
+
+      output-range-microvolt:
+        description:
+          Output voltage range for this channel as [min, max] in
+          microvolts.
+        oneOf:
+          - items:
+              - const: 0
+                default: 0
+              - enum: [5000000, 10000000, 20000000, 40000000]
+                default: 5000000
+          - items:
+              - const: -5000000
+              - const: 5000000
+          - items:
+              - const: -10000000
+              - const: 10000000
+          - items:
+              - const: -15000000
+              - const: 15000000
+          - items:
+              - const: -20000000
+              - const: 20000000
+
+    required:
+      - reg
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - vdd-io-supply
+  - vdd-5v-supply
+  - vdd-hv0-supply
+  - vdd-hv1-supply
+  - vdd-hv2-supply
+  - vdd-hv3-supply
+
+dependencies:
+  spi-cpha: [ spi-cpol ]
+  spi-cpol: [ spi-cpha ]
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        dac@0 {
+            compatible = "adi,ad5529r-16";
+            reg = <0>;
+            spi-max-frequency = <25000000>;
+            spi-device-addr = <0>;
+
+            vdd-io-supply = <&vdd_io_regulator>;
+            vdd-5v-supply = <&vdd_5v_regulator>;
+            vdd-hv0-supply = <&vdd_hv0_regulator>;
+            vdd-hv1-supply = <&vdd_hv1_regulator>;
+            vdd-hv2-supply = <&vdd_hv2_regulator>;
+            vdd-hv3-supply = <&vdd_hv3_regulator>;
+            vss-hv0-supply = <&vss_hv0_regulator>;
+            vss-hv1-supply = <&vss_hv1_regulator>;
+            vss-hv2-supply = <&vss_hv2_regulator>;
+            vss-hv3-supply = <&vss_hv3_regulator>;
+
+            reset-gpios = <&gpio0 87 GPIO_ACTIVE_LOW>;
+
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            channel@0 {
+                reg = <0>;
+                output-range-microvolt = <0 5000000>;
+            };
+
+            channel@1 {
+                reg = <1>;
+                output-range-microvolt = <(-10000000) 10000000>;
+            };
+
+            channel@2 {
+                reg = <2>;
+                output-range-microvolt = <0 40000000>;
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5710r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5710r.yaml
new file mode 100644
index 00000000000000..cdcc3da4f4ea0d
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5710r.yaml
@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/dac/adi,ad5710r.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices AD5710R/AD5711R 8-channel 12-/16-bit Configurable IDAC/VDAC
+
+maintainers:
+  - Kim Seer Paller <kimseer.paller@analog.com>
+
+description: |
+  The AD5710R (16-bit) and AD5711R (12-bit) are 8-channel, low-power,
+  configurable current/voltage output digital-to-analog converters (DACs) with
+  an on-chip 2.5V, 3ppm/°C reference. Each channel can be independently
+  configured as a voltage output (0V to VREF or 0V to 2 x VREF) or a current
+  output (0mA to 50mA). These devices operate from a single 2.7V to 5.5V
+  supply and are guaranteed monotonic by design.
+
+  Datasheet can be found here:
+  https://www.analog.com/media/en/technical-documentation/data-sheets/ad5711r-ad5710r.pdf
+
+properties:
+  compatible:
+    enum:
+      - adi,ad5710r
+      - adi,ad5711r
+
+  reg:
+    maxItems: 1
+
+  spi-max-frequency:
+    maximum: 20000000
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 0
+
+  vdd-supply:
+    description: Power Supply Input.
+
+  iovdd-supply:
+    description: Digital Power Supply Input.
+
+  io-channels:
+    description:
+      ADC channel used to monitor internal die temperature, output voltages, and
+      current of a selected channel via the MUXOUT pin.
+    maxItems: 1
+
+  ref-supply:
+    description:
+      Reference Input/Output. The voltage at the REF pin sets the full-scale
+      range of all channels. If not provided the internal reference is used and
+      also provided on the VREF pin.
+
+  reset-gpios:
+    description:
+      Active low signal that is falling edge sensitive. When it is deasserted,
+      the digital core initialization is performed and all DAC registers except
+      the Interface Configuration A register are reset to their default values.
+    maxItems: 1
+
+  ldac-gpios:
+    description:
+      LDAC pin to be used as a hardware trigger to update the DAC channels. If
+      not present, the DAC channels are updated by Software LDAC.
+    maxItems: 1
+
+  adi,range-double:
+    description:
+      Configure the output range for all channels. If the property is present,
+      the output will range from 0V to 2 x Vref. If the property is not present,
+      the output will range from 0V to Vref.
+    type: boolean
+
+patternProperties:
+  "^channel@[0-7]$":
+    $ref: /schemas/iio/dac/dac.yaml#
+    type: object
+    description:
+      Represents the external channels which are connected to the DAC.
+
+    properties:
+      reg:
+        description: Channel number
+        minimum: 0
+        maximum: 7
+
+      adi,ch-func:
+        description:
+          Channel output type. The values are defined in
+          <dt-bindings/iio/addac/adi,ad74413r.h>. Use
+          CH_FUNC_VOLTAGE_OUTPUT (1) for voltage output or
+          CH_FUNC_CURRENT_OUTPUT (2) for current output.
+        $ref: /schemas/types.yaml#/definitions/uint32
+        enum: [1, 2]
+
+    required:
+      - reg
+      - adi,ch-func
+
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+  - iovdd-supply
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/iio/addac/adi,ad74413r.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        dac@0 {
+            compatible = "adi,ad5710r";
+            reg = <0>;
+            spi-max-frequency = <1000000>;
+            vdd-supply = <&vdd>;
+            iovdd-supply = <&iovdd>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            channel@0 {
+                reg = <0>;
+                adi,ch-func = <CH_FUNC_VOLTAGE_OUTPUT>;
+            };
+
+            channel@1 {
+                reg = <1>;
+                adi,ch-func = <CH_FUNC_CURRENT_OUTPUT>;
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml b/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml
index d131f136bd1516..a86d094f0d4b54 100644
--- a/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml
+++ b/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml
@@ -4,58 +4,92 @@
 $id: http://devicetree.org/schemas/iio/dac/microchip,mcp47feb02.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Microchip MCP47F(E/V)B(0/1/2)(1/2/4/8) DAC with I2C Interface Families
+title: Microchip MCP4(7/8)F(E/V)B(0/1/2)(1/2/4/8) DAC with I2C/SPI Interface Families
 
 maintainers:
   - Ariana Lazar <ariana.lazar@microchip.com>
 
 description: |
-  Datasheet for MCP47FEB01, MCP47FEB11, MCP47FEB21, MCP47FEB02, MCP47FEB12,
-  MCP47FEB22 can be found here:
-    https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005375A.pdf
-  Datasheet for MCP47FVB01, MCP47FVB11, MCP47FVB21, MCP47FVB02, MCP47FVB12,
-  MCP47FVB22 can be found here:
-    https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005405A.pdf
-  Datasheet for MCP47FEB04, MCP47FEB14, MCP47FEB24, MCP47FEB08, MCP47FEB18,
-  MCP47FEB28, MCP47FVB04, MCP47FVB14, MCP47FVB24, MCP47FVB08, MCP47FVB18,
-  MCP47FVB28 can be found here:
-    https://ww1.microchip.com/downloads/aemDocuments/documents/MSLD/ProductDocuments/DataSheets/MCP47FXBX48-Data-Sheet-DS200006368A.pdf
-
-  +------------+--------------+-------------+-------------+------------+
-  | Device     |  Resolution  |   Channels  | Vref number | Memory     |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FEB01 |     8-bit    |      1      |      1      |   EEPROM   |
-  | MCP47FEB11 |    10-bit    |      1      |      1      |   EEPROM   |
-  | MCP47FEB21 |    12-bit    |      1      |      1      |   EEPROM   |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FEB02 |     8-bit    |      2      |      1      |   EEPROM   |
-  | MCP47FEB12 |    10-bit    |      2      |      1      |   EEPROM   |
-  | MCP47FEB22 |    12-bit    |      2      |      1      |   EEPROM   |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FVB01 |     8-bit    |      1      |      1      |      RAM   |
-  | MCP47FVB11 |    10-bit    |      1      |      1      |      RAM   |
-  | MCP47FVB21 |    12-bit    |      1      |      1      |      RAM   |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FVB02 |     8-bit    |      2      |      1      |      RAM   |
-  | MCP47FVB12 |    10-bit    |      2      |      1      |      RAM   |
-  | MCP47FVB22 |    12-bit    |      2      |      1      |      RAM   |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FVB04 |     8-bit    |      4      |      2      |      RAM   |
-  | MCP47FVB14 |    10-bit    |      4      |      2      |      RAM   |
-  | MCP47FVB24 |    12-bit    |      4      |      2      |      RAM   |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FVB08 |     8-bit    |      8      |      2      |      RAM   |
-  | MCP47FVB18 |    10-bit    |      8      |      2      |      RAM   |
-  | MCP47FVB28 |    12-bit    |      8      |      2      |      RAM   |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FEB04 |     8-bit    |      4      |      2      |   EEPROM   |
-  | MCP47FEB14 |    10-bit    |      4      |      2      |   EEPROM   |
-  | MCP47FEB24 |    12-bit    |      4      |      2      |   EEPROM   |
-  |------------|--------------|-------------|-------------|------------|
-  | MCP47FEB08 |     8-bit    |      8      |      2      |   EEPROM   |
-  | MCP47FEB18 |    10-bit    |      8      |      2      |   EEPROM   |
-  | MCP47FEB28 |    12-bit    |      8      |      2      |   EEPROM   |
-  +------------+--------------+-------------+-------------+------------+
+  Datasheets for MCP47F(E/V)B(0/1/2)(1/2/4/8) DAC with I2C Interface Families:
+   Datasheet for MCP47FEB01, MCP47FEB02, MCP47FEB11, MCP47FEB12, MCP47FEB21,
+   MCP47FEB22 can be found here:
+     https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005375A.pdf
+   Datasheet for MCP47FVB01, MCP47FVB02, MCP47FVB11, MCP47FVB12, MCP47FVB21,
+   MCP47FVB22 can be found here:
+     https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005405A.pdf
+   Datasheet for MCP47FEB04, MCP47FEB08, MCP47FEB14, MCP47FEB18, MCP47FEB24,
+   MCP47FEB28, MCP47FVB04, MCP47FVB08, MCP47FVB14, MCP47FVB18, MCP47FVB24,
+   MCP47FVB28 can be found here:
+     https://ww1.microchip.com/downloads/aemDocuments/documents/MSLD/ProductDocuments/DataSheets/MCP47FXBX48-Data-Sheet-DS200006368A.pdf
+   Datasheets for MCP48F(E/V)B(0/1/2)(1/2/4/8) DAC with SPI Interface Families:
+   Datasheet for MCP48FEB01, MCP48FEB02, MCP48FEB11, MCP48FEB12, MCP48FEB21,
+   MCP48FEB22 can be found here:
+     https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005429B.pdf
+   Datasheet for MCP48FVB01, MCP48FVB02, MCP48FVB11, MCP48FVB12, MCP48FVB21,
+   MCP48FVB22 can be found here:
+     https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005466A.pdf
+   Datasheet for MCP48FEB04, MCP48FEB14, MCP48FEB24, MCP48FEB08, MCP48FEB18,
+   MCP48FEB28, MCP48FVB04, MCP48FVB14, MCP48FVB24, MCP48FVB08, MCP48FVB18,
+   MCP48FVB28 can be found here:
+     https://ww1.microchip.com/downloads/aemDocuments/documents/MSLD/ProductDocuments/DataSheets/MCP48FXBX4-8-Family-Data-Sheet-DS20006362A.pdf
+
+  +------------+--------------+-------------+-------------+------------+------------+
+  | Device     |  Resolution  |   Channels  | Vref number |   Memory   | Interface  |
+  |------------|--------------|-------------|-------------|------------|------------|
+  | MCP47FEB01 |              |      1      |      1      |   EEPROM   |    I2C     |
+  | MCP47FEB02 |              |      2      |      1      |   EEPROM   |    I2C     |
+  | MCP47FEB04 |              |      4      |      2      |   EEPROM   |    I2C     |
+  | MCP47FEB08 |     8-bit    |      8      |      2      |   EEPROM   |    I2C     |
+  | MCP48FEB01 |              |      1      |      1      |   EEPROM   |    SPI     |
+  | MCP48FEB02 |              |      2      |      1      |   EEPROM   |    SPI     |
+  | MCP48FEB04 |              |      4      |      2      |   EEPROM   |    SPI     |
+  | MCP48FEB08 |              |      8      |      2      |   EEPROM   |    SPI     |
+  |------------|--------------|-------------|-------------|------------|------------|
+  | MCP47FEB11 |              |      1      |      1      |   EEPROM   |    I2C     |
+  | MCP47FEB12 |              |      2      |      1      |   EEPROM   |    I2C     |
+  | MCP47FEB14 |              |      4      |      2      |   EEPROM   |    I2C     |
+  | MCP47FEB18 |    10-bit    |      8      |      2      |   EEPROM   |    I2C     |
+  | MCP48FEB11 |              |      1      |      1      |   EEPROM   |    SPI     |
+  | MCP48FEB12 |              |      2      |      1      |   EEPROM   |    SPI     |
+  | MCP48FEB14 |              |      4      |      2      |   EEPROM   |    SPI     |
+  | MCP48FEB18 |              |      8      |      2      |   EEPROM   |    SPI     |
+  |------------|--------------|-------------|-------------|------------|------------|
+  | MCP47FEB21 |              |      1      |      1      |   EEPROM   |    I2C     |
+  | MCP47FEB22 |              |      2      |      1      |   EEPROM   |    I2C     |
+  | MCP47FEB24 |              |      4      |      2      |   EEPROM   |    I2C     |
+  | MCP47FEB28 |    12-bit    |      8      |      2      |   EEPROM   |    I2C     |
+  | MCP48FEB21 |              |      1      |      1      |   EEPROM   |    SPI     |
+  | MCP48FEB22 |              |      2      |      1      |   EEPROM   |    SPI     |
+  | MCP48FEB24 |              |      4      |      2      |   EEPROM   |    SPI     |
+  | MCP48FEB28 |              |      8      |      2      |   EEPROM   |    SPI     |
+  |------------|--------------|-------------|-------------|------------|------------|
+  | MCP47FVB01 |              |      1      |      1      |   RAM      |    I2C     |
+  | MCP47FVB02 |              |      2      |      1      |   RAM      |    I2C     |
+  | MCP47FVB04 |              |      4      |      2      |   RAM      |    I2C     |
+  | MCP47FVB08 |     8-bit    |      8      |      2      |   RAM      |    I2C     |
+  | MCP48FVB01 |              |      1      |      1      |   RAM      |    SPI     |
+  | MCP48FVB02 |              |      2      |      1      |   RAM      |    SPI     |
+  | MCP48FVB04 |              |      4      |      2      |   RAM      |    SPI     |
+  | MCP48FVB08 |              |      8      |      2      |   RAM      |    SPI     |
+  |------------|--------------|-------------|-------------|------------|------------|
+  | MCP47FVB11 |              |      1      |      1      |   RAM      |    I2C     |
+  | MCP47FVB12 |              |      2      |      1      |   RAM      |    I2C     |
+  | MCP47FVB14 |              |      4      |      2      |   RAM      |    I2C     |
+  | MCP47FVB18 |    10-bit    |      8      |      2      |   RAM      |    I2C     |
+  | MCP48FVB11 |              |      1      |      1      |   RAM      |    SPI     |
+  | MCP48FVB12 |              |      2      |      1      |   RAM      |    SPI     |
+  | MCP48FVB14 |              |      4      |      2      |   RAM      |    SPI     |
+  | MCP48FVB18 |              |      8      |      2      |   RAM      |    SPI     |
+  |------------|--------------|-------------|-------------|------------|------------|
+  | MCP47FVB21 |              |      1      |      1      |   RAM      |    I2C     |
+  | MCP47FVB22 |              |      2      |      1      |   RAM      |    I2C     |
+  | MCP47FVB24 |              |      4      |      2      |   RAM      |    I2C     |
+  | MCP47FVB28 |    12-bit    |      8      |      2      |   RAM      |    I2C     |
+  | MCP48FVB21 |              |      1      |      1      |   RAM      |    SPI     |
+  | MCP48FVB22 |              |      2      |      1      |   RAM      |    SPI     |
+  | MCP48FVB24 |              |      4      |      2      |   RAM      |    SPI     |
+  | MCP48FVB28 |              |      8      |      2      |   RAM      |    SPI     |
+  +------------+--------------+-------------+-------------+------------+------------+
 
 properties:
   compatible:
@@ -84,10 +118,37 @@ properties:
       - microchip,mcp47feb08
       - microchip,mcp47feb18
       - microchip,mcp47feb28
+      - microchip,mcp48feb01
+      - microchip,mcp48feb02
+      - microchip,mcp48feb04
+      - microchip,mcp48feb08
+      - microchip,mcp48feb11
+      - microchip,mcp48feb12
+      - microchip,mcp48feb14
+      - microchip,mcp48feb18
+      - microchip,mcp48feb21
+      - microchip,mcp48feb22
+      - microchip,mcp48feb24
+      - microchip,mcp48feb28
+      - microchip,mcp48fvb01
+      - microchip,mcp48fvb02
+      - microchip,mcp48fvb04
+      - microchip,mcp48fvb08
+      - microchip,mcp48fvb11
+      - microchip,mcp48fvb12
+      - microchip,mcp48fvb14
+      - microchip,mcp48fvb18
+      - microchip,mcp48fvb21
+      - microchip,mcp48fvb22
+      - microchip,mcp48fvb24
+      - microchip,mcp48fvb28
 
   reg:
     maxItems: 1
 
+  spi-max-frequency:
+    maximum: 10000000
+
   "#address-cells":
     const: 1
 
@@ -178,6 +239,21 @@ required:
   - vdd-supply
 
 allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            pattern: "^microchip,mcp48f[ev]b[0-2][1248]$"
+    then:
+      $ref: /schemas/spi/spi-peripheral-props.yaml#
+      dependencies:
+        spi-cpha: [ spi-cpol ]
+        spi-cpol: [ spi-cpha ]
+    else:
+      properties:
+        spi-max-frequency: false
+        spi-cpha: false
+        spi-cpol: false
   - if:
       properties:
         compatible:
@@ -189,6 +265,12 @@ allOf:
               - microchip,mcp47fvb01
               - microchip,mcp47fvb11
               - microchip,mcp47fvb21
+              - microchip,mcp48feb01
+              - microchip,mcp48feb11
+              - microchip,mcp48feb21
+              - microchip,mcp48fvb01
+              - microchip,mcp48fvb11
+              - microchip,mcp48fvb21
     then:
       properties:
         lat1-gpios: false
@@ -211,6 +293,12 @@ allOf:
               - microchip,mcp47fvb02
               - microchip,mcp47fvb12
               - microchip,mcp47fvb22
+              - microchip,mcp48feb02
+              - microchip,mcp48feb12
+              - microchip,mcp48feb22
+              - microchip,mcp48fvb02
+              - microchip,mcp48fvb12
+              - microchip,mcp48fvb22
     then:
       properties:
         lat1-gpios: false
@@ -233,6 +321,12 @@ allOf:
               - microchip,mcp47feb04
               - microchip,mcp47feb14
               - microchip,mcp47feb24
+              - microchip,mcp48feb04
+              - microchip,mcp48feb14
+              - microchip,mcp48feb24
+              - microchip,mcp48fvb04
+              - microchip,mcp48fvb14
+              - microchip,mcp48fvb24
     then:
       patternProperties:
         "^channel@[0-3]$":
@@ -251,6 +345,12 @@ allOf:
               - microchip,mcp47feb08
               - microchip,mcp47feb18
               - microchip,mcp47feb28
+              - microchip,mcp48feb08
+              - microchip,mcp48feb18
+              - microchip,mcp48feb28
+              - microchip,mcp48fvb08
+              - microchip,mcp48fvb18
+              - microchip,mcp48fvb28
     then:
       patternProperties:
         "^channel@[0-7]$":
@@ -272,7 +372,7 @@ allOf:
       properties:
         microchip,vref1-buffered: false
 
-additionalProperties: false
+unevaluatedProperties: false
 
 examples:
   - |
@@ -299,4 +399,29 @@ examples:
             };
         };
     };
+  - |
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        dac@0 {
+          compatible = "microchip,mcp48feb02";
+          reg = <0x0>;
+          vdd-supply = <&vdac_vdd>;
+          vref-supply = <&vref_reg>;
+          spi-max-frequency = <10000000>;
+
+          #address-cells = <1>;
+          #size-cells = <0>;
+          channel@0 {
+            reg = <0>;
+            label = "Adjustable_voltage_ch0";
+          };
+
+          channel@1 {
+            reg = <0x1>;
+            label = "Adjustable_voltage_ch1";
+          };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/iio/frequency/adi,ad9910.yaml b/Documentation/devicetree/bindings/iio/frequency/adi,ad9910.yaml
new file mode 100644
index 00000000000000..4eebbd388b27c8
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/frequency/adi,ad9910.yaml
@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/frequency/adi,ad9910.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices AD9910 Direct Digital Synthesizer
+
+maintainers:
+  - Rodrigo Alencar <rodrigo.alencar@analog.com>
+
+description: >
+  The AD9910 is a 1 GSPS direct digital synthesizer (DDS) with an integrated
+  14-bit DAC. It features single tone mode with 8 configurable profiles,
+  a digital ramp generator, RAM control, OSK, and a parallel data port for
+  high-speed streaming.
+
+  https://www.analog.com/en/products/ad9910.html
+
+properties:
+  compatible:
+    const: adi,ad9910
+
+  reg:
+    maxItems: 1
+
+  spi-max-frequency:
+    maximum: 70000000
+
+  clocks:
+    minItems: 1
+    items:
+      - description: Reference clock (REF_CLK).
+      - description: Optional synchronization clock (SYNC_IN).
+
+  clock-names:
+    minItems: 1
+    items:
+      - const: ref_clk
+      - const: sync_in
+
+  '#clock-cells':
+    const: 1
+
+  clock-output-names:
+    items:
+      - const: sync_clk
+      - const: pdclk
+      - const: sync_out
+
+  interrupts:
+    minItems: 1
+    maxItems: 2
+    description:
+      Requires interrupt-names property with the same number of items. The
+      supported interrupts are 'drover' (digital ramp generator limit) and
+      'ram_swp_ovr' (end of RAM sweep).
+
+  interrupt-names:
+    minItems: 1
+    maxItems: 2
+    items:
+      enum: [ drover, ram_swp_ovr ]
+
+  dvdd-io33-supply:
+    description: 3.3V Digital I/O supply.
+
+  avdd33-supply:
+    description: 3.3V Analog DAC supply.
+
+  dvdd18-supply:
+    description: 1.8V Digital Core supply.
+
+  avdd18-supply:
+    description: 1.8V Analog Core supply.
+
+  reset-gpios:
+    maxItems: 1
+    description:
+      GPIOs controlling the Main Device reset.
+
+  io-reset-gpios:
+    maxItems: 1
+    description:
+      GPIO controlling the I/O_RESET pin.
+
+  powerdown-gpios:
+    maxItems: 1
+    description:
+      GPIO controlling the EXT_PWR_DWN pin.
+
+  update-gpios:
+    maxItems: 1
+    description:
+      GPIO controlling the I/O_UPDATE pin.
+
+  profile-gpios:
+    minItems: 3
+    maxItems: 3
+    description:
+      GPIOs controlling the PROFILE[2:0] pins for profile selection.
+
+  sync-err-gpios:
+    maxItems: 1
+    description:
+      GPIO used to read SYNC_SMP_ERR pin status.
+
+  lock-detect-gpios:
+    maxItems: 1
+    description:
+      GPIO used to read PLL_LOCK pin status.
+
+  pwms:
+    maxItems: 1
+    description:
+      PWM used to drive the DRCTL pin, which controls the digital ramp
+      generator slope direction.
+
+  pwm-names:
+    items:
+      - const: drctl
+
+  adi,pll-enable:
+    type: boolean
+    description:
+      Enables the internal PLL. Applicable when external loop filter components
+      are connected to the PLL_LOOP_FILTER pin and low reference clock
+      frequencies are used, often provided by a crystal or by a single-ended
+      on-board oscillator.
+
+  adi,charge-pump-current-microamp:
+    minimum: 212
+    maximum: 387
+    default: 212
+    description:
+      PLL charge pump current in microamps. Only applicable when the internal
+      PLL is enabled. The value is rounded to the nearest supported step. This
+      value depends mostly on the loop filter design.
+
+  adi,refclk-out-drive-strength:
+    $ref: /schemas/types.yaml#/definitions/string
+    enum: [ disabled, low, medium, high ]
+    default: disabled
+    description:
+      Reference clock output (DRV0) drive strength. Only applicable when
+      the internal PLL is enabled.
+
+dependencies:
+  adi,charge-pump-current-microamp: [ 'adi,pll-enable' ]
+  adi,refclk-out-drive-strength: [ 'adi,pll-enable' ]
+  lock-detect-gpios: [ 'adi,pll-enable' ]
+  interrupts: [ interrupt-names ]
+  pwms: [ pwm-names ]
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - '#clock-cells'
+  - dvdd-io33-supply
+  - avdd33-supply
+  - dvdd18-supply
+  - avdd18-supply
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        dds@0 {
+            compatible = "adi,ad9910";
+            reg = <0>;
+            spi-max-frequency = <1000000>;
+
+            #clock-cells = <1>;
+            clock-output-names = "sync_clk", "pdclk", "sync_out";
+
+            clocks = <&ad9910_refclk>;
+            clock-names = "ref_clk";
+
+            dvdd-io33-supply = <&vdd_io33>;
+            avdd33-supply = <&vdd_a33>;
+            dvdd18-supply = <&vdd_d18>;
+            avdd18-supply = <&vdd_a18>;
+
+            reset-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>;
+            io-reset-gpios = <&gpio 1 GPIO_ACTIVE_HIGH>;
+            powerdown-gpios = <&gpio 2 GPIO_ACTIVE_HIGH>;
+            update-gpios = <&gpio 3 GPIO_ACTIVE_HIGH>;
+            profile-gpios = <&gpio 4 GPIO_ACTIVE_HIGH>,
+                            <&gpio 5 GPIO_ACTIVE_HIGH>,
+                            <&gpio 6 GPIO_ACTIVE_HIGH>;
+
+            pwms = <&pwm 0 1000000 0>;
+            pwm-names = "drctl";
+
+            adi,pll-enable;
+            adi,charge-pump-current-microamp = <387>;
+            adi,refclk-out-drive-strength = "disabled";
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml b/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml
index 81b6e85decd52c..9e22b603d47fb7 100644
--- a/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml
+++ b/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml
@@ -26,16 +26,20 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - invensense,icm42600
-      - invensense,icm42602
-      - invensense,icm42605
-      - invensense,icm42607
-      - invensense,icm42607p
-      - invensense,icm42622
-      - invensense,icm42631
-      - invensense,icm42686
-      - invensense,icm42688
+    oneOf:
+      - enum:
+          - invensense,icm42600
+          - invensense,icm42602
+          - invensense,icm42605
+          - invensense,icm42607
+          - invensense,icm42607p
+          - invensense,icm42622
+          - invensense,icm42631
+          - invensense,icm42686
+          - invensense,icm42688
+      - items:
+          - const: invensense,icm42630
+          - const: invensense,icm42631
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/light/adux1020.yaml b/Documentation/devicetree/bindings/iio/light/adux1020.yaml
index d7d14f2f1c2057..83abe984a1da1c 100644
--- a/Documentation/devicetree/bindings/iio/light/adux1020.yaml
+++ b/Documentation/devicetree/bindings/iio/light/adux1020.yaml
@@ -40,10 +40,10 @@ examples:
         #size-cells = <0>;
 
         adux1020@64 {
-                compatible = "adi,adux1020";
-                reg = <0x64>;
-                interrupt-parent = <&msmgpio>;
-                interrupts = <24 IRQ_TYPE_LEVEL_HIGH>;
+            compatible = "adi,adux1020";
+            reg = <0x64>;
+            interrupt-parent = <&msmgpio>;
+            interrupts = <24 IRQ_TYPE_LEVEL_HIGH>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/amstaos,tsl2591.yaml b/Documentation/devicetree/bindings/iio/light/amstaos,tsl2591.yaml
index 83b88c6a243d8d..512625d625a882 100644
--- a/Documentation/devicetree/bindings/iio/light/amstaos,tsl2591.yaml
+++ b/Documentation/devicetree/bindings/iio/light/amstaos,tsl2591.yaml
@@ -45,6 +45,6 @@ examples:
             compatible = "amstaos,tsl2591";
             reg = <0x29>;
             interrupts = <20 IRQ_TYPE_EDGE_FALLING>;
-       };
+        };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/capella,cm32181.yaml b/Documentation/devicetree/bindings/iio/light/capella,cm32181.yaml
new file mode 100644
index 00000000000000..e64085a9cb2b19
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/light/capella,cm32181.yaml
@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/light/capella,cm32181.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Capella CM32181 ambient light sensor
+
+maintainers:
+  - Kevin Tsai <ktsai@capellamicro.com>
+
+properties:
+  compatible:
+    const: capella,cm32181
+
+  reg:
+    maxItems: 1
+
+  vdd-supply: true
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        light-sensor@10 {
+            compatible = "capella,cm32181";
+            reg = <0x10>;
+            vdd-supply = <&vdd_3v3>;
+            interrupt-parent = <&gpio>;
+            interrupts = <17 IRQ_TYPE_EDGE_RISING>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/light/isl29018.yaml b/Documentation/devicetree/bindings/iio/light/isl29018.yaml
index 0ea278b07d1c0b..9524f0a1d19591 100644
--- a/Documentation/devicetree/bindings/iio/light/isl29018.yaml
+++ b/Documentation/devicetree/bindings/iio/light/isl29018.yaml
@@ -50,9 +50,9 @@ examples:
         #size-cells = <0>;
 
         sensor@44 {
-                compatible = "isil,isl29018";
-                reg = <0x44>;
-                interrupts-extended = <&msmgpio 61 IRQ_TYPE_LEVEL_HIGH>;
+            compatible = "isil,isl29018";
+            reg = <0x44>;
+            interrupts-extended = <&msmgpio 61 IRQ_TYPE_LEVEL_HIGH>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/liteon,ltr501.yaml b/Documentation/devicetree/bindings/iio/light/liteon,ltr501.yaml
index c8074f180a7936..20abd7da080d67 100644
--- a/Documentation/devicetree/bindings/iio/light/liteon,ltr501.yaml
+++ b/Documentation/devicetree/bindings/iio/light/liteon,ltr501.yaml
@@ -9,15 +9,14 @@ title: LiteON LTR501 I2C Proximity and Light sensor
 maintainers:
   - Nikita Travkin <nikita@trvn.ru>
 
-allOf:
-  - $ref: ../common.yaml#
-
 properties:
   compatible:
     enum:
+      - liteon,ltr301
+      - liteon,ltr303
+      - liteon,ltr329
       - liteon,ltr501
       - liteon,ltr559
-      - liteon,ltr301
 
   reg:
     maxItems: 1
@@ -36,6 +35,31 @@ required:
   - compatible
   - reg
 
+allOf:
+  - $ref: ../common.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - liteon,ltr329
+    then:
+      properties:
+        interrupts: false
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - liteon,ltr301
+              - liteon,ltr303
+              - liteon,ltr329
+    then:
+      properties:
+        proximity-near-level: false
+
 examples:
   - |
     #include <dt-bindings/interrupt-controller/irq.h>
diff --git a/Documentation/devicetree/bindings/iio/light/noa1305.yaml b/Documentation/devicetree/bindings/iio/light/noa1305.yaml
index fe7bfe1adbda5b..e1c64c1519322a 100644
--- a/Documentation/devicetree/bindings/iio/light/noa1305.yaml
+++ b/Documentation/devicetree/bindings/iio/light/noa1305.yaml
@@ -39,8 +39,8 @@ examples:
         #size-cells = <0>;
 
         light@39 {
-                compatible = "onnn,noa1305";
-                reg = <0x39>;
+            compatible = "onnn,noa1305";
+            reg = <0x39>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/stk33xx.yaml b/Documentation/devicetree/bindings/iio/light/stk33xx.yaml
index e4341fdced98ca..f655a570aac7be 100644
--- a/Documentation/devicetree/bindings/iio/light/stk33xx.yaml
+++ b/Documentation/devicetree/bindings/iio/light/stk33xx.yaml
@@ -4,14 +4,14 @@
 $id: http://devicetree.org/schemas/iio/light/stk33xx.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: |
-  Sensortek STK33xx I2C Ambient Light and Proximity sensor
+title: Sensortek STK33xx/STK36C61 I2C Ambient Light and Proximity sensor
 
 maintainers:
   - Jonathan Cameron <jic23@kernel.org>
 
-description: |
-  Ambient light and proximity sensor over an i2c interface.
+description:
+  Ambient light and proximity sensor over an i2c interface. The STK36C61
+  additionally provides RGBC colour channels.
 
 allOf:
   - $ref: ../common.yaml#
@@ -26,6 +26,7 @@ properties:
       - items:
           - enum:
               - sensortek,stk3013
+              - sensortek,stk36c61
           - const: sensortek,stk3310
 
   reg:
@@ -52,11 +53,11 @@ examples:
         #size-cells = <0>;
 
         stk3310@48 {
-                compatible = "sensortek,stk3310";
-                reg = <0x48>;
-                proximity-near-level = <25>;
-                interrupt-parent = <&gpio1>;
-                interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+            compatible = "sensortek,stk3310";
+            reg = <0x48>;
+            proximity-near-level = <25>;
+            interrupt-parent = <&gpio1>;
+            interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/tsl2583.yaml b/Documentation/devicetree/bindings/iio/light/tsl2583.yaml
index 7b92ba8cbb9fbd..aed4ab710e4634 100644
--- a/Documentation/devicetree/bindings/iio/light/tsl2583.yaml
+++ b/Documentation/devicetree/bindings/iio/light/tsl2583.yaml
@@ -41,8 +41,8 @@ examples:
         #size-cells = <0>;
 
         light-sensor@29 {
-                compatible = "amstaos,tsl2581";
-                reg = <0x29>;
+            compatible = "amstaos,tsl2581";
+            reg = <0x29>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/tsl2772.yaml b/Documentation/devicetree/bindings/iio/light/tsl2772.yaml
index 9921ccaa64a06c..38a3c82cb87901 100644
--- a/Documentation/devicetree/bindings/iio/light/tsl2772.yaml
+++ b/Documentation/devicetree/bindings/iio/light/tsl2772.yaml
@@ -74,13 +74,13 @@ examples:
         #size-cells = <0>;
 
         sensor@39 {
-                compatible = "amstaos,tsl2772";
-                reg = <0x39>;
-                interrupts-extended = <&msmgpio 61 IRQ_TYPE_EDGE_FALLING>;
-                vdd-supply = <&pm8941_l17>;
-                vddio-supply = <&pm8941_lvs1>;
-                amstaos,proximity-diodes = <0>;
-                led-max-microamp = <100000>;
+            compatible = "amstaos,tsl2772";
+            reg = <0x39>;
+            interrupts-extended = <&msmgpio 61 IRQ_TYPE_EDGE_FALLING>;
+            vdd-supply = <&pm8941_l17>;
+            vddio-supply = <&pm8941_lvs1>;
+            amstaos,proximity-diodes = <0>;
+            led-max-microamp = <100000>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/upisemi,us5182.yaml b/Documentation/devicetree/bindings/iio/light/upisemi,us5182.yaml
index dd78abe0ec8df6..979d7e7822900a 100644
--- a/Documentation/devicetree/bindings/iio/light/upisemi,us5182.yaml
+++ b/Documentation/devicetree/bindings/iio/light/upisemi,us5182.yaml
@@ -69,7 +69,7 @@ examples:
         light-sensor@39 {
             compatible = "upisemi,usd5182";
             reg = <0x39>;
-            upisemi,glass-coef = < 1000 >;
+            upisemi,glass-coef = <1000>;
             upisemi,dark-ths = /bits/ 16 <170 200 512 512 800 2000 4000 8000>;
             upisemi,upper-dark-gain = /bits/ 8 <0x00>;
             upisemi,lower-dark-gain = /bits/ 8 <0x16>;
diff --git a/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml b/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml
index 0041e1db68388d..2eb1932f7b14a1 100644
--- a/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml
+++ b/Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml
@@ -4,7 +4,8 @@
 $id: http://devicetree.org/schemas/iio/light/vishay,veml6030.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: VEML3235, VEML3328, VEML6030, VEML6035 and VEML7700 Ambient Light Sensors (ALS)
+title: VEML3235, VEML3328, VEML6030, VEML6031x00 series, VEML6035
+  and VEML7700 Ambient Light Sensors (ALS)
 
 maintainers:
   - Rishi Gupta <gupt21@gmail.com>
@@ -23,6 +24,8 @@ description: |
     https://www.vishay.com/docs/80131/veml3235.pdf
     https://www.vishay.com/docs/84968/veml3328.pdf
     https://www.vishay.com/docs/84366/veml6030.pdf
+    https://www.vishay.com/docs/80007/veml6031x00.pdf
+    https://www.vishay.com/docs/80008/veml6031x01.pdf
     https://www.vishay.com/docs/84889/veml6035.pdf
     https://www.vishay.com/docs/84286/veml7700.pdf
 
@@ -32,6 +35,10 @@ properties:
       - vishay,veml3235
       - vishay,veml3328
       - vishay,veml6030
+      - vishay,veml6031x00
+      - vishay,veml6031x01
+      - vishay,veml60311x00
+      - vishay,veml60311x01
       - vishay,veml6035
       - vishay,veml7700
 
@@ -69,6 +76,8 @@ allOf:
       properties:
         compatible:
           enum:
+            - vishay,veml6031x00
+            - vishay,veml6031x01
             - vishay,veml6035
     then:
       properties:
@@ -82,6 +91,8 @@ allOf:
           enum:
             - vishay,veml3235
             - vishay,veml3328
+            - vishay,veml60311x00
+            - vishay,veml60311x01
             - vishay,veml7700
     then:
       properties:
@@ -101,10 +112,10 @@ examples:
         #size-cells = <0>;
 
         light-sensor@10 {
-                compatible = "vishay,veml6030";
-                reg = <0x10>;
-                interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
-                vdd-supply = <&vdd>;
+            compatible = "vishay,veml6030";
+            reg = <0x10>;
+            interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
+            vdd-supply = <&vdd>;
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/pressure/infineon,dps310.yaml b/Documentation/devicetree/bindings/iio/pressure/infineon,dps310.yaml
index e5d1e6c4893936..1912a1b0b4baf2 100644
--- a/Documentation/devicetree/bindings/iio/pressure/infineon,dps310.yaml
+++ b/Documentation/devicetree/bindings/iio/pressure/infineon,dps310.yaml
@@ -45,10 +45,10 @@ examples:
         #size-cells = <0>;
 
         dps: pressure-sensor@76 {
-          compatible = "infineon,dps310";
-          reg = <0x76>;
-          #io-channel-cells = <0>;
-          vdd-supply = <&vref1>;
-          vddio-supply = <&vref2>;
+            compatible = "infineon,dps310";
+            reg = <0x76>;
+            #io-channel-cells = <0>;
+            vdd-supply = <&vref1>;
+            vddio-supply = <&vref2>;
         };
     };
diff --git a/Documentation/devicetree/bindings/iio/proximity/pulsedlight,lidar-lite-v2.yaml b/Documentation/devicetree/bindings/iio/proximity/pulsedlight,lidar-lite-v2.yaml
new file mode 100644
index 00000000000000..dd739319b9ebb9
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/proximity/pulsedlight,lidar-lite-v2.yaml
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/proximity/pulsedlight,lidar-lite-v2.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Pulsedlight/Garmin LIDAR-Lite range-finding sensors
+
+maintainers:
+  - Matt Ranostay <matt@ranostay.sg>
+  - Rodrigo Gobbi <rodrigo.gobbi.7@gmail.com>
+
+description: |
+  This binding covers the I2C interface of the Pulsedlight/Garmin
+  LIDAR-Lite v2 and v3 laser rangefinders.
+
+  Besides I2C, the mode control pin can also trigger acquisitions and
+  report the measured distance through a PWM signal, without using I2C
+  at all. That usage is bidirectional and non-cyclic, and is not
+  described by this binding. The same pin can also be used, while I2C
+  remains the active interface, as a plain status output signalling
+  when an acquisition has completed.
+
+  Datasheets:
+    https://static.garmin.com/pumac/LIDAR_Lite_v3_Operation_Manual_and_Technical_Specifications.pdf
+    https://github.com/PulsedLight3D/LIDAR-Lite-Documentation/blob/master/Docs/LIDAR-Lite-v2-Docs.pdf
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - grmn,lidar-lite-v3
+          - const: pulsedlight,lidar-lite-v2
+      - const: pulsedlight,lidar-lite-v2
+
+  reg:
+    maxItems: 1
+
+  powerdown-gpios:
+    description: GPIO that can be driven low to shut off power to the device.
+    maxItems: 1
+
+  vdd-supply: true
+
+  interrupts:
+    description:
+      Mode control pin used as a status output, driven while the device is
+      busy performing an acquisition. Can be used to signal completion
+      instead of polling over I2C.
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        lidar@62 {
+            compatible = "grmn,lidar-lite-v3", "pulsedlight,lidar-lite-v2";
+            reg = <0x62>;
+            vdd-supply = <&vdd_5v0>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/proximity/vishay,vcnl3020.yaml b/Documentation/devicetree/bindings/iio/proximity/vishay,vcnl3020.yaml
index fbd3a2e3228013..f84aece886ab97 100644
--- a/Documentation/devicetree/bindings/iio/proximity/vishay,vcnl3020.yaml
+++ b/Documentation/devicetree/bindings/iio/proximity/vishay,vcnl3020.yaml
@@ -57,8 +57,8 @@ examples:
         #size-cells = <0>;
 
         proximity@13 {
-              compatible = "vishay,vcnl3020";
-              reg = <0x13>;
-              vishay,led-current-microamp = <200000>;
+            compatible = "vishay,vcnl3020";
+            reg = <0x13>;
+            vishay,led-current-microamp = <200000>;
         };
     };
diff --git a/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml b/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml
index 13e5f29f058849..09b809f01550ba 100644
--- a/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml
+++ b/Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml
@@ -17,6 +17,7 @@ description: |
     - LTC2986
     - LTM2985
 
+  https://www.analog.com/media/en/technical-documentation/data-sheets/adt7604.pdf
   https://www.analog.com/media/en/technical-documentation/data-sheets/2983fc.pdf
   https://www.analog.com/media/en/technical-documentation/data-sheets/2984fb.pdf
   https://www.analog.com/media/en/technical-documentation/data-sheets/29861fa.pdf
diff --git a/Documentation/devicetree/bindings/input/cpcap-pwrbutton.txt b/Documentation/devicetree/bindings/input/cpcap-pwrbutton.txt
deleted file mode 100644
index 0dd0076daf71f1..00000000000000
--- a/Documentation/devicetree/bindings/input/cpcap-pwrbutton.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Motorola CPCAP on key
-
-This module is part of the CPCAP. For more details about the whole
-chip see Documentation/devicetree/bindings/mfd/motorola-cpcap.txt.
-
-This module provides a simple power button event via an Interrupt.
-
-Required properties:
-- compatible: should be one of the following
-   - "motorola,cpcap-pwrbutton"
-- interrupts: irq specifier for CPCAP's ON IRQ
-
-Example:
-
-&cpcap {
-	cpcap_pwrbutton: pwrbutton {
-		compatible = "motorola,cpcap-pwrbutton";
-		interrupts = <23 IRQ_TYPE_NONE>;
-	};
-};
diff --git a/Documentation/devicetree/bindings/input/matrix-keymap.yaml b/Documentation/devicetree/bindings/input/matrix-keymap.yaml
index ce910e4ac823ef..bef8f98135357e 100644
--- a/Documentation/devicetree/bindings/input/matrix-keymap.yaml
+++ b/Documentation/devicetree/bindings/input/matrix-keymap.yaml
@@ -41,8 +41,7 @@ examples:
   - |
     keypad {
         /* ... */
-        linux,keymap = < 0x00030012
-                         0x0102003a >;
+        linux,keymap = <0x00030012 0x0102003a>;
         keypad,num-rows = <2>;
         keypad,num-columns = <8>;
     };
diff --git a/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml b/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml
index 914dd3283df330..20fa41f691ed61 100644
--- a/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml
+++ b/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml
@@ -26,6 +26,7 @@ properties:
       - const: mediatek,mt6779-keypad
       - items:
           - enum:
+              - mediatek,mt6572-keypad
               - mediatek,mt6873-keypad
               - mediatek,mt8183-keypad
               - mediatek,mt8365-keypad
diff --git a/Documentation/devicetree/bindings/input/motorola,cpcap-pwrbutton.yaml b/Documentation/devicetree/bindings/input/motorola,cpcap-pwrbutton.yaml
new file mode 100644
index 00000000000000..77a3e5a47d1ae8
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/motorola,cpcap-pwrbutton.yaml
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/motorola,cpcap-pwrbutton.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Motorola CPCAP PMIC power key
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+description:
+  This module is part of the Motorola CPCAP MFD device. For more details
+  see Documentation/devicetree/bindings/mfd/motorola,cpcap.yaml. The
+  power key is represented as a sub-node of the PMIC node on the device
+  tree.
+
+properties:
+  compatible:
+    const: motorola,cpcap-pwrbutton
+
+  interrupts:
+    items:
+      - description: CPCAP's ON interrupt
+
+required:
+  - compatible
+  - interrupts
+
+additionalProperties: false
+
+...
diff --git a/Documentation/devicetree/bindings/input/omap-keypad.txt b/Documentation/devicetree/bindings/input/omap-keypad.txt
deleted file mode 100644
index 34ed1c60ff95ce..00000000000000
--- a/Documentation/devicetree/bindings/input/omap-keypad.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-* TI's Keypad Controller device tree bindings
-
-TI's Keypad controller is used to interface a SoC with a matrix-type
-keypad device. The keypad controller supports multiple row and column lines.
-A key can be placed at each intersection of a unique row and a unique column.
-The keypad controller can sense a key-press and key-release and report the
-event using a interrupt to the cpu.
-
-This binding is based on the matrix-keymap binding with the following
-changes:
-
-keypad,num-rows and keypad,num-columns are required.
-
-Required SoC Specific Properties:
-- compatible: should be one of the following
-   - "ti,omap4-keypad": For controllers compatible with omap4 keypad
-      controller.
-
-Optional Properties specific to linux:
-- linux,keypad-no-autorepeat: do no enable autorepeat feature.
-
-Example:
-	keypad@4ae1c000{
-		compatible = "ti,omap4-keypad";
-		keypad,num-rows = <2>;
-		keypad,num-columns = <8>;
-		linux,keypad-no-autorepeat;
-	};
diff --git a/Documentation/devicetree/bindings/input/samsung,s3c6410-keypad.yaml b/Documentation/devicetree/bindings/input/samsung,s3c6410-keypad.yaml
index a53569aa0ee774..2498c62444a133 100644
--- a/Documentation/devicetree/bindings/input/samsung,s3c6410-keypad.yaml
+++ b/Documentation/devicetree/bindings/input/samsung,s3c6410-keypad.yaml
@@ -37,6 +37,10 @@ properties:
 
   wakeup-source: true
 
+  keypad,num-columns: true
+  keypad,num-rows: true
+  linux,keymap: true
+
   linux,input-no-autorepeat:
     type: boolean
     description:
@@ -81,12 +85,33 @@ patternProperties:
       - keypad,row
       - linux,code
 
+dependencies:
+  linux,keymap: [ "keypad,num-columns", "keypad,num-rows" ]
+
 required:
   - compatible
   - reg
   - interrupts
-  - samsung,keypad-num-columns
-  - samsung,keypad-num-rows
+
+allOf:
+  - $ref: input.yaml#
+  - $ref: matrix-keymap.yaml#
+  - if:
+      required:
+        - linux,keymap
+    then:
+      properties:
+        samsung,keypad-num-columns: false
+        samsung,keypad-num-rows: false
+      patternProperties:
+        '^key-[0-9a-z]+$': false
+    else:
+      properties:
+        keypad,num-columns: false
+        keypad,num-rows: false
+      required:
+        - samsung,keypad-num-columns
+        - samsung,keypad-num-rows
 
 additionalProperties: false
 
@@ -94,8 +119,9 @@ examples:
   - |
     #include <dt-bindings/clock/exynos4.h>
     #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/input/input.h>
 
-    keypad@100a0000 {
+    keypad1@100a0000 {
         compatible = "samsung,s5pv210-keypad";
         reg = <0x100a0000 0x100>;
         interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
@@ -119,3 +145,24 @@ examples:
             linux,code = <3>;
         };
     };
+  - |
+    #include <dt-bindings/clock/exynos4.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/input/input.h>
+
+    keypad2@100a0000 {
+        compatible = "samsung,s5pv210-keypad";
+        reg = <0x100a0000 0x100>;
+        interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clock CLK_KEYIF>;
+        clock-names = "keypad";
+
+        keypad,num-rows = <2>;
+        keypad,num-columns = <8>;
+        linux,keymap = <
+          MATRIX_KEY(0, 3, 2)
+          MATRIX_KEY(0, 4, 3)
+        >;
+        linux,input-no-autorepeat;
+        wakeup-source;
+    };
diff --git a/Documentation/devicetree/bindings/input/ti,omap4-keypad.yaml b/Documentation/devicetree/bindings/input/ti,omap4-keypad.yaml
new file mode 100644
index 00000000000000..5e0c101f9ba0fb
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/ti,omap4-keypad.yaml
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/ti,omap4-keypad.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments Keypad Controller
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+description:
+  TI's Keypad controller is used to interface a SoC with a matrix-type
+  keypad device. The keypad controller supports multiple row and column lines.
+  A key can be placed at each intersection of a unique row and a unique column.
+  The keypad controller can sense a key-press and key-release and report the
+  event using a interrupt to the cpu.
+
+allOf:
+  - $ref: /schemas/input/matrix-keymap.yaml#
+
+properties:
+  compatible:
+    const: ti,omap4-keypad
+
+  reg:
+    maxItems: 1
+
+  reg-names:
+    const: mpu
+
+  interrupts:
+    maxItems: 1
+
+  linux,input-no-autorepeat:
+    type: boolean
+    description: Do not enable autorepeat feature.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - keypad,num-rows
+  - keypad,num-columns
+  - linux,keymap
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    keypad@4ae1c000 {
+        compatible = "ti,omap4-keypad";
+        reg = <0x4ae1c000 0x400>;
+        reg-names = "mpu";
+        interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+        keypad,num-rows = <2>;
+        keypad,num-columns = <8>;
+        linux,keymap = <0x00000011   /* KEY_W */
+                        0x0001001f>; /* KEY_S */
+        linux,input-no-autorepeat;
+    };
diff --git a/Documentation/devicetree/bindings/input/ti,tca8418.yaml b/Documentation/devicetree/bindings/input/ti,tca8418.yaml
index 624a1830d0b0d2..a8897c5c4d9d52 100644
--- a/Documentation/devicetree/bindings/input/ti,tca8418.yaml
+++ b/Documentation/devicetree/bindings/input/ti,tca8418.yaml
@@ -46,16 +46,15 @@ examples:
             interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
             keypad,num-rows = <4>;
             keypad,num-columns = <4>;
-            linux,keymap = < MATRIX_KEY(0x00, 0x01, BTN_0)
-                             MATRIX_KEY(0x00, 0x00, BTN_1)
-                             MATRIX_KEY(0x01, 0x01, BTN_2)
-                             MATRIX_KEY(0x01, 0x00, BTN_3)
-                             MATRIX_KEY(0x02, 0x00, BTN_4)
-                             MATRIX_KEY(0x00, 0x03, BTN_5)
-                             MATRIX_KEY(0x00, 0x02, BTN_6)
-                             MATRIX_KEY(0x01, 0x03, BTN_7)
-                             MATRIX_KEY(0x01, 0x02, BTN_8)
-                             MATRIX_KEY(0x02, 0x02, BTN_9)
-            >;
+            linux,keymap = <MATRIX_KEY(0x00, 0x01, BTN_0)
+                            MATRIX_KEY(0x00, 0x00, BTN_1)
+                            MATRIX_KEY(0x01, 0x01, BTN_2)
+                            MATRIX_KEY(0x01, 0x00, BTN_3)
+                            MATRIX_KEY(0x02, 0x00, BTN_4)
+                            MATRIX_KEY(0x00, 0x03, BTN_5)
+                            MATRIX_KEY(0x00, 0x02, BTN_6)
+                            MATRIX_KEY(0x01, 0x03, BTN_7)
+                            MATRIX_KEY(0x01, 0x02, BTN_8)
+                            MATRIX_KEY(0x02, 0x02, BTN_9)>;
         };
     };
diff --git a/Documentation/devicetree/bindings/input/touchscreen/sis,9200-ts.yaml b/Documentation/devicetree/bindings/input/touchscreen/sis,9200-ts.yaml
new file mode 100644
index 00000000000000..5c4806348fcb54
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/touchscreen/sis,9200-ts.yaml
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/touchscreen/sis,9200-ts.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: SiS 9200 Family I2C Touchscreen Controller
+
+maintainers:
+  - Mika Penttilä <mpenttil@redhat.com>
+
+allOf:
+  - $ref: touchscreen.yaml#
+
+properties:
+  compatible:
+    const: sis,9200-ts
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  attn-gpios:
+    maxItems: 1
+    description: GPIO pin used as the attention line
+
+  reset-gpios:
+    maxItems: 1
+    description: GPIO pin used to reset the controller
+
+  wakeup-source: true
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        touchscreen@5c {
+            compatible = "sis,9200-ts";
+            reg = <0x5c>;
+            pinctrl-names = "default";
+            pinctrl-0 = <&pinctrl_sis>;
+            interrupt-parent = <&gpio3>;
+            interrupts = <19 IRQ_TYPE_EDGE_FALLING>;
+            attn-gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;
+            reset-gpios = <&gpio2 30 GPIO_ACTIVE_LOW>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/input/touchscreen/sis_i2c.txt b/Documentation/devicetree/bindings/input/touchscreen/sis_i2c.txt
deleted file mode 100644
index 8f5322e0102404..00000000000000
--- a/Documentation/devicetree/bindings/input/touchscreen/sis_i2c.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-* SiS I2C Multiple Touch Controller
-
-Required properties:
-- compatible: must be "sis,9200-ts"
-- reg: i2c slave address
-- interrupts: touch controller interrupt (see interrupt
-  binding [0])
-
-Optional properties:
-- pinctrl-names: should be "default" (see pinctrl binding [1]).
-- pinctrl-0: a phandle pointing to the pin settings for the
-  device (see pinctrl binding [1]).
-- attn-gpios: the gpio pin used as attention line
-- reset-gpios: the gpio pin used to reset the controller
-- wakeup-source: touchscreen can be used as a wakeup source
-
-[0]: Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
-[1]: Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
-
-Example:
-
-	sis9255@5c  {
-		compatible = "sis,9200-ts";
-		reg = <0x5c>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_sis>;
-		interrupt-parent = <&gpio3>;
-		interrupts = <19 IRQ_TYPE_EDGE_FALLING>;
-		irq-gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;
-		reset-gpios = <&gpio2 30 GPIO_ACTIVE_LOW>;
-	};
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,kuno-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,kuno-rpmh.yaml
new file mode 100644
index 00000000000000..20d169b31142ee
--- /dev/null
+++ b/Documentation/devicetree/bindings/interconnect/qcom,kuno-rpmh.yaml
@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interconnect/qcom,kuno-rpmh.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm RPMh Network-On-Chip Interconnect on Kuno SoC
+
+maintainers:
+  - Hardeep Sharma <hardeep.sharma@oss.qualcomm.com>
+
+description: |
+  RPMh interconnect providers support system bandwidth requirements through
+  RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is
+  able to communicate with the BCM through the Resource State Coordinator (RSC)
+  associated with each execution environment. Provider nodes must point to at
+  least one RPMh device child node pertaining to their RSC and each provider
+  can map to multiple RPMh resources.
+
+  See also: include/dt-bindings/interconnect/qcom,kuno.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,kuno-aggre-noc
+      - qcom,kuno-clk-virt
+      - qcom,kuno-cnoc-main
+      - qcom,kuno-dc-noc
+      - qcom,kuno-mc-virt
+      - qcom,kuno-mem-noc
+      - qcom,kuno-pcie-anoc
+      - qcom,kuno-system-noc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    maxItems: 2
+
+required:
+  - compatible
+
+allOf:
+  - $ref: qcom,rpmh-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,kuno-clk-virt
+              - qcom,kuno-mc-virt
+    then:
+      properties:
+        reg: false
+    else:
+      required:
+        - reg
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: qcom,kuno-aggre-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: System NoC USB SF AXI clock
+            - description: IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: qcom,kuno-system-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,kuno-aggre-noc
+              - qcom,kuno-system-noc
+    then:
+      required:
+        - clocks
+    else:
+      properties:
+        clocks: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,kuno-gcc.h>
+    #include <dt-bindings/clock/qcom,rpmh.h>
+
+    clk_virt: interconnect-0 {
+        compatible = "qcom,kuno-clk-virt";
+        #interconnect-cells = <1>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+
+    aggre_noc: interconnect@1640000 {
+        compatible = "qcom,kuno-aggre-noc";
+        reg = <0x01640000 0x33400>;
+        clocks = <&gcc GCC_SYS_NOC_USB_SF_AXI_CLK>,
+                 <&rpmhcc RPMH_IPA_CLK>;
+        #interconnect-cells = <1>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
index 21b4e04727de5f..f6969910bded9d 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
@@ -50,6 +50,7 @@ properties:
           - enum:
               - qcom,eliza-llcc-bwmon
               - qcom,hawi-llcc-bwmon
+              - qcom,milos-cpu-bwmon
               - qcom,qcs615-llcc-bwmon
               - qcom,qcs8300-llcc-bwmon
               - qcom,sa8775p-llcc-bwmon
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml b/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
index 3b8ebe17a9760c..cd6685683bab8e 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,osm-l3.yaml
@@ -31,6 +31,7 @@ properties:
       - items:
           - enum:
               - qcom,eliza-epss-l3
+              - qcom,milos-epss-l3
               - qcom,sa8775p-epss-l3
               - qcom,sc7280-epss-l3
               - qcom,sc8280xp-epss-l3
diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml
index 726a6720e05f3c..e3cadc5ad39d59 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic2.yaml
@@ -45,6 +45,8 @@ properties:
           - enum:
               - apple,t6030-aic3
               - apple,t6031-aic3
+              - apple,t8132-aic3
+              - apple,t8140-aic3
           - const: apple,t8122-aic3
       - const: apple,t8122-aic3
 
diff --git a/Documentation/devicetree/bindings/interrupt-controller/chrp,open-pic.yaml b/Documentation/devicetree/bindings/interrupt-controller/chrp,open-pic.yaml
index 642738512f3ce1..294af57a7c3721 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/chrp,open-pic.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/chrp,open-pic.yaml
@@ -12,8 +12,9 @@ maintainers:
 description:
   This binding specifies what properties must be available in the device tree
   representation of an Open PIC compliant interrupt controller.  This binding is
-  based on the binding defined for Open PIC in [1] and is a superset of that
-  binding.
+  based on the binding defined for Open PIC in the Devicetree Specification
+  (available from https://www.devicetree.org/specifications/) and is a superset
+  of that binding.
 
 properties:
   compatible:
@@ -24,7 +25,7 @@ properties:
       - const: chrp,open-pic
 
   device_type:
-    const: open-pci
+    const: open-pic
     deprecated: true
 
   reg:
diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
index 8162a49d49a669..be4dc95e391613 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
@@ -30,6 +30,7 @@ properties:
           - qcom,glymur-pdc
           - qcom,hawi-pdc
           - qcom,kaanapali-pdc
+          - qcom,kuno-pdc
           - qcom,maili-pdc
           - qcom,milos-pdc
           - qcom,nord-pdc
@@ -58,6 +59,7 @@ properties:
           - qcom,sm8650-pdc
           - qcom,sm8750-pdc
           - qcom,x1e80100-pdc
+          - qcom,x1p42100-pdc
       - const: qcom,pdc
 
   reg:
diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml
index 47ec7fa52c3ac6..6efbe732b4ceef 100644
--- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml
+++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml
@@ -44,10 +44,12 @@ properties:
       Optional since not all IOMMUs are attached to a clock gate.
 
   '#iommu-cells':
-    const: 1
+    enum: [ 1, 5 ]
     description:
-      Has to be one. The single cell describes the stream id emitted by
-      a master to the IOMMU.
+      Has to be one or five. The first cell describes the stream id emitted by
+      a master to the IOMMU. The optional second and third cell describe the
+      aperture start of a master's DMA window and fourth and fifth cell describe
+      the size of DMA window.
 
   power-domains:
     maxItems: 1
@@ -66,11 +68,11 @@ examples:
       compatible = "apple,t8103-dart";
       reg = <0x82f80000 0x4000>;
       interrupts = <1 781 4>;
-      #iommu-cells = <1>;
+      #iommu-cells = <5>;
     };
 
     master1 {
-      iommus = <&dart1 0>;
+      iommus = <&dart1 0 0x100 0x0 0x10 0x0>;
     };
 
   - |+
diff --git a/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml
new file mode 100644
index 00000000000000..5ca8ce469ec317
--- /dev/null
+++ b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Broadcom BCM2712 IOMMU
+
+maintainers:
+  - Daniel Drake <dan@reactivated.net>
+  - Florian Fainelli <florian.fainelli@broadcom.com>
+  - Jim Quinlan <james.quinlan@broadcom.com>
+
+description:
+  The BCM2712 SoC features multiple independent IOMMU instances providing
+  address translation for multimedia hardware blocks, such as the VC6 display
+  pipeline, camera receivers, and ISP. Each IOMMU has its own page tables,
+  control registers, and local TLB, and is additionally linked to a centralized
+  L2 TLB (IOMMUC).
+
+properties:
+  compatible:
+    const: brcm,bcm2712-iommu
+
+  reg:
+    maxItems: 1
+
+  '#iommu-cells':
+    const: 0
+
+  brcm,iommu-cache:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      Phandle to the shared IOMMU cache (IOMMUC). Each IOMMU instance caches
+      translated descriptors in this centralized L2 TLB cache block and must
+      trigger cache invalidation commands on it when updating or unmapping page
+      table entries.
+
+required:
+  - compatible
+  - reg
+  - '#iommu-cells'
+  - brcm,iommu-cache
+
+additionalProperties: false
+
+examples:
+  - |
+    iommu@5200 {
+        compatible = "brcm,bcm2712-iommu";
+        reg = <0x5200 0x80>;
+        brcm,iommu-cache = <&iommuc>;
+        #iommu-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml
new file mode 100644
index 00000000000000..47c584ad667ae6
--- /dev/null
+++ b/Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iommu/brcm,bcm2712-iommuc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Broadcom BCM2712 Shared IOMMU Cache (IOMMUC)
+
+maintainers:
+  - Daniel Drake <dan@reactivated.net>
+  - Florian Fainelli <florian.fainelli@broadcom.com>
+  - Jim Quinlan <james.quinlan@broadcom.com>
+
+description:
+  The BCM2712 IOMMUC is a centralized Translation Lookaside Buffer (TLB) cache
+  which accelerates address translation across the SoC's IOMMU devices. If an
+  address mapping is not found in the IOMMU's local TLB cache, then this
+  IOMMUC is consulted as a L2 TLB. The SoC includes a single IOMMUC which is
+  shared between all IOMMUs, and it must be explicitly invalidated when
+  modifying or unmapping IOMMU page tables.
+
+properties:
+  compatible:
+    const: brcm,bcm2712-iommuc
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    iommuc@5b00 {
+        compatible = "brcm,bcm2712-iommuc";
+        reg = <0x5b00 0x80>;
+    };
diff --git a/Documentation/devicetree/bindings/leds/adi,ltc3208.yaml b/Documentation/devicetree/bindings/leds/adi,ltc3208.yaml
new file mode 100644
index 00000000000000..c6697816b7d985
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/adi,ltc3208.yaml
@@ -0,0 +1,183 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright (c) 2026 Analog Devices, Inc.
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/adi,ltc3208.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: LTC3208 Multidisplay LED Controller from Linear Technologies (Now Analog Devices)
+
+maintainers:
+  - Jan Carlo Roleda <jancarlo.roleda@analog.com>
+
+description:
+  The LTC3208 is a multidisplay LED controller that can support up to 1A to all
+  connected LEDs.
+
+  The datasheet for this device can be found in
+  https://www.analog.com/en/products/ltc3208.html
+
+properties:
+  compatible:
+    const: adi,ltc3208
+
+  reg:
+    maxItems: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+  adi,disable-camhl-pin:
+    type: boolean
+    description:
+      Configures whether the external CAMHL pin is disabled.
+      If disabled then the output pins associated with CAM will always select
+      the CAM register's high half-byte brightness.
+
+  adi,cfg-enrgbs-pin:
+    type: boolean
+    description:
+      Configures which channel the ENRGBS pin toggles when it receives a signal.
+      ENRGBS pin controls the SUB channel's output pins if this is set,
+      or RGB channel's output pins if this is unset.
+
+  adi,disable-rgb-aux4-dropout:
+    type: boolean
+    description:
+      Configures the RGB and AUX4 dropout signals to be disabled.
+
+  adi,aux1-channel:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      LED Channel that the AUX1 output pin mirrors its brightness level from.
+    enum: [aux, main, sub, cam]
+    default: aux
+
+  adi,aux2-channel:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      LED Channel that the AUX2 output pin mirrors its brightness level from.
+    enum: [aux, main, sub, cam]
+    default: aux
+
+  adi,aux3-channel:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      LED Channel that the AUX3 output pin mirrors its brightness level from.
+    enum: [aux, main, sub, cam]
+    default: aux
+
+  adi,aux4-channel:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      LED Channel that the AUX4 output pin mirrors its brightness level from.
+    enum: [aux, main, sub, cam]
+    default: aux
+
+patternProperties:
+  "^led@[0-7]$":
+    type: object
+    $ref: /schemas/leds/common.yaml#
+    unevaluatedProperties: false
+    properties:
+      reg:
+        description:
+          LED Channel Number. each channel maps to a specific channel group used
+          to configure the brightness level of the output pins corresponding to
+          the channel.
+        enum:
+          - 0 # Main Channel (8-bit brightness)
+          - 1 # Sub Channel (8-bit brightness)
+          - 2 # AUX Channel (4-bit brightness)
+          - 3 # Camera Channel, Low-side byte (4-bit brightness)
+          - 4 # Camera Channel, High-side byte (4-bit brightness)
+          - 5 # Red Channel (4-bit brightness)
+          - 6 # Blue Channel (4-bit brightness)
+          - 7 # Green Channel (4-bit brightness)
+    required:
+      - reg
+
+required:
+  - compatible
+  - reg
+  - "#address-cells"
+  - "#size-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/leds/common.h>
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      led-controller@1b {
+        compatible = "adi,ltc3208";
+        reg = <0x1b>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+        adi,disable-camhl-pin;
+        adi,cfg-enrgbs-pin;
+        adi,disable-rgb-aux4-dropout;
+
+        /* MAIN */
+        led@0 {
+          reg = <0>;
+          function = LED_FUNCTION_ACTIVITY;
+          color = <LED_COLOR_ID_WHITE>;
+        };
+
+        /* SUB */
+        led@1 {
+          reg = <1>;
+          function = LED_FUNCTION_ACTIVITY;
+          color = <LED_COLOR_ID_WHITE>;
+        };
+
+        /* AUX */
+        led@2 {
+          reg = <2>;
+          function = LED_FUNCTION_ACTIVITY;
+          color = <LED_COLOR_ID_WHITE>;
+        };
+
+        /* CAMLO */
+        led@3 {
+          reg = <3>;
+          function = LED_FUNCTION_FLASH;
+          color = <LED_COLOR_ID_WHITE>;
+        };
+
+        /* CAMHI */
+        led@4 {
+          reg = <4>;
+          function = LED_FUNCTION_FLASH;
+          color = <LED_COLOR_ID_WHITE>;
+        };
+
+        /* RED */
+        led@5 {
+          reg = <5>;
+          function = LED_FUNCTION_INDICATOR;
+          color = <LED_COLOR_ID_RED>;
+        };
+
+        /* BLUE */
+        led@6 {
+          reg = <6>;
+          function = LED_FUNCTION_INDICATOR;
+          color = <LED_COLOR_ID_BLUE>;
+        };
+
+        /* GREEN */
+        led@7 {
+          reg = <7>;
+          function = LED_FUNCTION_INDICATOR;
+          color = <LED_COLOR_ID_GREEN>;
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/leds/backlight/ti,lm3533-backlight.yaml b/Documentation/devicetree/bindings/leds/backlight/ti,lm3533-backlight.yaml
new file mode 100644
index 00000000000000..a25846e9b3752f
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/backlight/ti,lm3533-backlight.yaml
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/backlight/ti,lm3533-backlight.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI LM3533 high voltage series LED strings
+
+description:
+  This is part of the TI LM3533 MFD device. It represents two high voltage series
+  LED strings for display backlight controlled by the TI LM3533.
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+allOf:
+  - $ref: /schemas/leds/backlight/common.yaml#
+
+properties:
+  compatible:
+    const: ti,lm3533-backlight
+
+  reg:
+    description: Control bank selection (0 = bank A, 1 = bank B).
+    maximum: 1
+
+  led-max-microamp:
+    description: maximum current in uA with a 800 uA step.
+    minimum: 5000
+    maximum: 29800
+    default: 5000
+
+  led-sources:
+    description: |
+      HVLED strings associated with this control bank:
+        0 - HVLED1
+        1 - HVLED2
+    minItems: 1
+    maxItems: 2
+    items:
+      maximum: 1
+
+  ti,pwm-config-mask:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      Control Bank PWM Configuration Register mask that allows to configure
+      PWM input in Zones 0-4
+      BIT(0) - PWM Input is enabled
+      BIT(1) - PWM Input is enabled in Zone 0
+      BIT(2) - PWM Input is enabled in Zone 1
+      BIT(3) - PWM Input is enabled in Zone 2
+      BIT(4) - PWM Input is enabled in Zone 3
+      BIT(5) - PWM Input is enabled in Zone 4
+    default: 0
+
+  ti,linear-mapping-mode:
+    description:
+      Enable linear mapping mode. If disabled, then it will use exponential
+      mapping mode in which the ramp up/down appears to have a more uniform
+      transition to the human eye.
+    type: boolean
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+# see ti,lm3533.yaml for an example
diff --git a/Documentation/devicetree/bindings/leds/leds-cpcap.txt b/Documentation/devicetree/bindings/leds/leds-cpcap.txt
deleted file mode 100644
index ebf7cdc7f70c2d..00000000000000
--- a/Documentation/devicetree/bindings/leds/leds-cpcap.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-Motorola CPCAP PMIC LEDs
-------------------------
-
-This module is part of the CPCAP. For more details about the whole
-chip see Documentation/devicetree/bindings/mfd/motorola-cpcap.txt.
-
-Requires node properties:
-- compatible: should be one of
-   * "motorola,cpcap-led-mdl"		(Main Display Lighting)
-   * "motorola,cpcap-led-kl"		(Keyboard Lighting)
-   * "motorola,cpcap-led-adl"		(Aux Display Lighting)
-   * "motorola,cpcap-led-red"		(Red Triode)
-   * "motorola,cpcap-led-green"		(Green Triode)
-   * "motorola,cpcap-led-blue"		(Blue Triode)
-   * "motorola,cpcap-led-cf"		(Camera Flash)
-   * "motorola,cpcap-led-bt"		(Bluetooth)
-   * "motorola,cpcap-led-cp"		(Camera Privacy LED)
-- label: see Documentation/devicetree/bindings/leds/common.txt
-- vdd-supply: A phandle to the regulator powering the LED
-
-Example:
-
-&cpcap {
-	cpcap_led_red: red-led {
-		compatible = "motorola,cpcap-led-red";
-		label = "cpcap:red";
-		vdd-supply = <&sw5>;
-	};
-};
diff --git a/Documentation/devicetree/bindings/leds/motorola,cpcap-leds.yaml b/Documentation/devicetree/bindings/leds/motorola,cpcap-leds.yaml
new file mode 100644
index 00000000000000..c8e7b88a05ccff
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/motorola,cpcap-leds.yaml
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/motorola,cpcap-leds.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Motorola CPCAP PMIC LEDs
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+description:
+  This module is part of the Motorola CPCAP MFD device. For more details
+  see Documentation/devicetree/bindings/mfd/motorola,cpcap.yaml. LEDs are
+  represented as sub-nodes of the PMIC node on the device tree.
+
+allOf:
+  - $ref: /schemas/leds/common.yaml#
+
+properties:
+  compatible:
+    enum:
+      - motorola,cpcap-led-adl # Display Lighting
+      - motorola,cpcap-led-blue # Blue Triode
+      - motorola,cpcap-led-bt # Bluetooth
+      - motorola,cpcap-led-cf # Camera Flash
+      - motorola,cpcap-led-cp # Camera Privacy LED
+      - motorola,cpcap-led-green # Green Triode
+      - motorola,cpcap-led-kl # Keyboard Lighting
+      - motorola,cpcap-led-mdl # Main Display Lighting
+      - motorola,cpcap-led-red # Red Triode
+
+  vdd-supply: true
+
+required:
+  - compatible
+  - label
+  - vdd-supply
+
+unevaluatedProperties: false
+
+...
diff --git a/Documentation/devicetree/bindings/leds/ti,lm3533-leds.yaml b/Documentation/devicetree/bindings/leds/ti,lm3533-leds.yaml
new file mode 100644
index 00000000000000..d8dbefef3323e6
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/ti,lm3533-leds.yaml
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/ti,lm3533-leds.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI LM3533 low voltage control banks for individual LEDs
+
+description:
+  This is part of the TI LM3533 MFD device. It represents four low voltage
+  control banks for individual LEDs provided by the TI LM3533.
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+allOf:
+  - $ref: /schemas/leds/common.yaml#
+
+properties:
+  compatible:
+    const: ti,lm3533-leds
+
+  reg:
+    description:
+      Control bank selection (2 = bank C, 3 = bank D, 4 = bank E, 5 = bank F).
+    minimum: 2
+    maximum: 5
+
+  led-max-microamp:
+    description: maximum current in uA with a 800 uA step.
+    minimum: 5000
+    maximum: 29800
+    default: 5000
+
+  led-sources:
+    description: |
+      LVLED associated with this control bank. May be more than 1 source per bank.
+        0 - LVLED1
+        1 - LVLED2
+        2 - LVLED3
+        3 - LVLED4
+        4 - LVLED5
+    minItems: 1
+    maxItems: 5
+    items:
+      maximum: 4
+
+  ti,pwm-config-mask:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      Control Bank PWM Configuration Register mask that allows to configure
+      PWM input in Zones 0-4
+      BIT(0) - PWM Input is enabled
+      BIT(1) - PWM Input is enabled in Zone 0
+      BIT(2) - PWM Input is enabled in Zone 1
+      BIT(3) - PWM Input is enabled in Zone 2
+      BIT(4) - PWM Input is enabled in Zone 3
+      BIT(5) - PWM Input is enabled in Zone 4
+    default: 0
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+# see ti,lm3533.yaml for an example
diff --git a/Documentation/devicetree/bindings/leds/ti,lm3533.yaml b/Documentation/devicetree/bindings/leds/ti,lm3533.yaml
new file mode 100644
index 00000000000000..bdc05a9f3d6635
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/ti,lm3533.yaml
@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/ti,lm3533.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI LM3533 Complete Lighting Power Solution
+
+description: >
+  The LM3533 is a complete power source for backlight, keypad, and indicator LEDs
+  in smartphone handsets. The high-voltage inductive boost converter provides the
+  power for two high voltage series LED strings for display backlight and four low
+  voltage control banks for individual LEDs. Additionally, LM3533 features an
+  interface for an external light sensor.
+
+  https://www.ti.com/product/LM3533
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+properties:
+  compatible:
+    const: ti,lm3533
+
+  reg:
+    maxItems: 1
+
+  enable-gpios:
+    description: GPIO connected to the HWEN pin.
+    maxItems: 1
+
+  vin-supply:
+    description: Supply connected to the IN line (2.7 V to 5.5 V).
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 0
+
+  ti,boost-ovp-microvolt:
+    description: boost OVP select (16V, 24V, 32V, 40V)
+    enum: [ 16000000, 24000000, 32000000, 40000000 ]
+    default: 16000000
+
+  ti,boost-freq-hz:
+    description: boost frequency select (500KHz or 1MHz)
+    enum: [ 500000, 1000000 ]
+    default: 500000
+
+  light-sensor:
+    type: object
+    additionalProperties: false
+
+    properties:
+      compatible:
+        const: ti,lm3533-als
+
+      interrupts:
+        maxItems: 1
+
+      ti,resistor-ohms:
+        description:
+          Internal configuration resistor value when light sensor is in
+          Analog Sensor mode and PWM mode is disabled. The expectation is
+          the input is a current from the external analog light sensor and
+          this is used to convert it to a voltage within the target range.
+        minimum: 1575
+        maximum: 200000
+
+      ti,pwm-mode:
+        type: boolean
+        description:
+          Switch for mode in which light sensor interface is running. If
+          this property is set then the light sensor interface is running
+          in PWM mode, internal resistor value is set to high-impedance (0)
+          and ti,resistor-ohms property is ignored.
+
+    required:
+      - compatible
+
+    oneOf:
+      - required:
+          - ti,resistor-ohms
+      - required:
+          - ti,pwm-mode
+
+patternProperties:
+  "^backlight@[01]$":
+    $ref: /schemas/leds/backlight/ti,lm3533-backlight.yaml#
+
+  "^led@[2-5]$":
+    $ref: /schemas/leds/ti,lm3533-leds.yaml#
+
+required:
+  - compatible
+  - reg
+  - '#address-cells'
+  - '#size-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        led-controller@36 {
+            compatible = "ti,lm3533";
+            reg = <0x36>;
+
+            enable-gpios = <&gpio 110 GPIO_ACTIVE_HIGH>;
+            vin-supply = <&vdd_3v3_bat>;
+
+            ti,boost-ovp-microvolt = <24000000>;
+            ti,boost-freq-hz = <500000>;
+
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            backlight@0 {
+                compatible = "ti,lm3533-backlight";
+                reg = <0>;
+
+                default-brightness = <113>;
+
+                led-max-microamp = <23400>;
+                led-sources = <0 1>;
+            };
+
+            led@2 {
+                compatible = "ti,lm3533-leds";
+                reg = <2>;
+
+                led-max-microamp = <23400>;
+                led-sources = <0 1>;
+            };
+
+            led@4 {
+                compatible = "ti,lm3533-leds";
+                reg = <4>;
+
+                led-max-microamp = <23400>;
+                led-sources = <2>;
+            };
+
+            led@5 {
+                compatible = "ti,lm3533-leds";
+                reg = <5>;
+
+                led-max-microamp = <23400>;
+                led-sources = <3 4>;
+            };
+
+            light-sensor {
+                compatible = "ti,lm3533-als";
+
+                interrupt-parent = <&gpio>;
+                interrupts = <80 IRQ_TYPE_LEVEL_LOW>;
+
+                ti,pwm-mode;
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml b/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml
index 87d31963c1b73c..f58b0d41218665 100644
--- a/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml
+++ b/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml
@@ -17,9 +17,6 @@ description: |
   hold four 32-bit messages; when a FIFO is full, clients must wait before
   attempting more transmissions.
 
-  Refer to ./mailbox.txt for generic information about mailbox device-tree
-  bindings.
-
 properties:
   compatible:
     oneOf:
diff --git a/Documentation/devicetree/bindings/mailbox/altera-mailbox.txt b/Documentation/devicetree/bindings/mailbox/altera-mailbox.txt
index c4dd93f1fed243..87749d251f0939 100644
--- a/Documentation/devicetree/bindings/mailbox/altera-mailbox.txt
+++ b/Documentation/devicetree/bindings/mailbox/altera-mailbox.txt
@@ -31,12 +31,12 @@ Example:
 
 Mailbox client
 ===============
-"mboxes" and the optional "mbox-names" (please see
-Documentation/devicetree/bindings/mailbox/mailbox.txt for details). Each value
-of the mboxes property should contain a phandle to the mailbox controller
-device node and second argument is the channel index. It must be 0 (hardware
-support only one channel).The equivalent "mbox-names" property value can be
-used to give a name to the communication channel to be used by the client user.
+The common mailbox binding properties "mboxes" and the optional "mbox-names"
+can be used. Each value of the mboxes property should contain a phandle to
+the mailbox controller device node and second argument is the channel index.
+It must be 0 (hardware support only one channel). The equivalent "mbox-names"
+property value can be used to give a name to the communication channel to be
+used by the client user.
 
 Example:
 	mclient0: mclient0@400 {
diff --git a/Documentation/devicetree/bindings/mailbox/hisilicon,hi3660-mailbox.txt b/Documentation/devicetree/bindings/mailbox/hisilicon,hi3660-mailbox.txt
index 3e5b4537407d17..ec2da5c9e6472c 100644
--- a/Documentation/devicetree/bindings/mailbox/hisilicon,hi3660-mailbox.txt
+++ b/Documentation/devicetree/bindings/mailbox/hisilicon,hi3660-mailbox.txt
@@ -35,7 +35,7 @@ Client
 
 Required properties:
 - compatible		: See the client docs
-- mboxes		: Standard property to specify a Mailbox (See ./mailbox.txt)
+- mboxes		: Standard property to specify a Mailbox
 			  Cells must match 'mbox-cells' (See Controller docs above)
 
 Optional properties
diff --git a/Documentation/devicetree/bindings/mailbox/hisilicon,hi6220-mailbox.txt b/Documentation/devicetree/bindings/mailbox/hisilicon,hi6220-mailbox.txt
index 044b17f3a77a6b..de9fbf722bee0b 100644
--- a/Documentation/devicetree/bindings/mailbox/hisilicon,hi6220-mailbox.txt
+++ b/Documentation/devicetree/bindings/mailbox/hisilicon,hi6220-mailbox.txt
@@ -55,7 +55,7 @@ Mailbox client
 Required properties:
 --------------------
 - compatible:		Many (See the client docs).
-- mboxes:		Standard property to specify a Mailbox (See ./mailbox.txt)
+- mboxes:		Standard property to specify a Mailbox
 			Cells must match 'mbox-cells' (See Mailbox Device Node above).
 
 Optional Properties:
diff --git a/Documentation/devicetree/bindings/mailbox/mailbox.txt b/Documentation/devicetree/bindings/mailbox/mailbox.txt
deleted file mode 100644
index af8ecee2ac687f..00000000000000
--- a/Documentation/devicetree/bindings/mailbox/mailbox.txt
+++ /dev/null
@@ -1,60 +0,0 @@
-* Generic Mailbox Controller and client driver bindings
-
-Generic binding to provide a way for Mailbox controller drivers to
-assign appropriate mailbox channel to client drivers.
-
-* Mailbox Controller
-
-Required property:
-- #mbox-cells: Must be at least 1. Number of cells in a mailbox
-		specifier.
-
-Example:
-	mailbox: mailbox {
-		...
-		#mbox-cells = <1>;
-	};
-
-
-* Mailbox Client
-
-Required property:
-- mboxes: List of phandle and mailbox channel specifiers.
-
-Optional property:
-- mbox-names: List of identifier strings for each mailbox channel.
-- shmem : List of phandle pointing to the shared memory(SHM) area between the
-	  users of these mailboxes for IPC, one for each mailbox. This shared
-	  memory can be part of any memory reserved for the purpose of this
-	  communication between the mailbox client and the remote.
-
-
-Example:
-	pwr_cntrl: power {
-		...
-		mbox-names = "pwr-ctrl", "rpc";
-		mboxes = <&mailbox 0 &mailbox 1>;
-	};
-
-Example with shared memory(shmem):
-
-	sram: sram@50000000 {
-		compatible = "mmio-sram";
-		reg = <0x50000000 0x10000>;
-
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges = <0 0x50000000 0x10000>;
-
-		cl_shmem: shmem@0 {
-			compatible = "client-shmem";
-			reg = <0x0 0x200>;
-		};
-	};
-
-	client@2e000000 {
-		...
-		mboxes = <&mailbox 0>;
-		shmem = <&cl_shmem>;
-		..
-	};
diff --git a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
index 03359479d926a6..2185920e1b5d21 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
+++ b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
@@ -21,6 +21,7 @@ properties:
               - qcom,glymur-cpucp-mbox
               - qcom,hawi-cpucp-mbox
               - qcom,kaanapali-cpucp-mbox
+              - qcom,maili-cpucp-mbox
               - qcom,nord-cpucp-mbox
               - qcom,sm8750-cpucp-mbox
           - const: qcom,x1e80100-cpucp-mbox
diff --git a/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml b/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml
index 8504ceb64806bc..7e3b7b68841cf6 100644
--- a/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml
+++ b/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml
@@ -53,8 +53,7 @@ description: |
   ==============
   A device needing to communicate with a target processor device should specify
   them using the common mailbox binding properties, "mboxes" and the optional
-  "mbox-names" (please see Documentation/devicetree/bindings/mailbox/mailbox.txt
-  for details). Each value of the mboxes property should contain a phandle to
+  "mbox-names". Each value of the mboxes property should contain a phandle to
   the mailbox controller device node and an args specifier that will be the
   phandle to the intended sub-mailbox child node to be used for communication.
   The equivalent "mbox-names" property value can be used to give a name to the
diff --git a/Documentation/devicetree/bindings/media/amlogic,c3-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/amlogic,c3-mipi-csi2.yaml
index b0129beab0c374..2f11e5f4583950 100644
--- a/Documentation/devicetree/bindings/media/amlogic,c3-mipi-csi2.yaml
+++ b/Documentation/devicetree/bindings/media/amlogic,c3-mipi-csi2.yaml
@@ -96,8 +96,8 @@ examples:
                   <0x0 0xff01a000 0x0 0x100>;
             reg-names = "aphy", "dphy", "host";
             power-domains = <&pwrc PWRC_C3_MIPI_ISP_WRAP_ID>;
-            clocks =  <&clkc_periphs CLKID_VAPB>,
-                      <&clkc_periphs CLKID_CSI_PHY0>;
+            clocks = <&clkc_periphs CLKID_VAPB>,
+                     <&clkc_periphs CLKID_CSI_PHY0>;
             clock-names = "vapb", "phy0";
             assigned-clocks = <&clkc_periphs CLKID_VAPB>,
                               <&clkc_periphs CLKID_CSI_PHY0>;
diff --git a/Documentation/devicetree/bindings/media/arm,mali-c55.yaml b/Documentation/devicetree/bindings/media/arm,mali-c55.yaml
index fc4fcd19922aad..39af7ffc70b406 100644
--- a/Documentation/devicetree/bindings/media/arm,mali-c55.yaml
+++ b/Documentation/devicetree/bindings/media/arm,mali-c55.yaml
@@ -79,7 +79,7 @@ examples:
 
       port {
         isp_in: endpoint {
-            remote-endpoint = <&csi2_rx_out>;
+          remote-endpoint = <&csi2_rx_out>;
         };
       };
     };
diff --git a/Documentation/devicetree/bindings/media/atmel,isc.yaml b/Documentation/devicetree/bindings/media/atmel,isc.yaml
index dc8aba698d7801..5c84f372e81ae4 100644
--- a/Documentation/devicetree/bindings/media/atmel,isc.yaml
+++ b/Documentation/devicetree/bindings/media/atmel,isc.yaml
@@ -102,12 +102,12 @@ examples:
         clock-output-names = "isc-mck";
 
         port {
-                isc_0: endpoint {
-                       remote-endpoint = <&ov7740_0>;
-                       hsync-active = <1>;
-                       vsync-active = <0>;
-                       pclk-sample = <1>;
-                       bus-width = <8>;
-                };
+            isc_0: endpoint {
+                remote-endpoint = <&ov7740_0>;
+                hsync-active = <1>;
+                vsync-active = <0>;
+                pclk-sample = <1>;
+                bus-width = <8>;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/media/brcm,bcm2835-unicam.yaml b/Documentation/devicetree/bindings/media/brcm,bcm2835-unicam.yaml
index 5fb5d60f069a28..bfec4daccd0a01 100644
--- a/Documentation/devicetree/bindings/media/brcm,bcm2835-unicam.yaml
+++ b/Documentation/devicetree/bindings/media/brcm,bcm2835-unicam.yaml
@@ -117,11 +117,11 @@ examples:
         power-domains = <&power RPI_POWER_DOMAIN_UNICAM1>;
         brcm,num-data-lanes = <2>;
         port {
-                csi1_ep: endpoint {
-                        remote-endpoint = <&imx219_0>;
-                        bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
-                        data-lanes = <1 2>;
-                };
+            csi1_ep: endpoint {
+                remote-endpoint = <&imx219_0>;
+                bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+                data-lanes = <1 2>;
+            };
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/media/cdns,csi2rx.yaml b/Documentation/devicetree/bindings/media/cdns,csi2rx.yaml
index 6ed9a5621064ae..cbc6249e8803b9 100644
--- a/Documentation/devicetree/bindings/media/cdns,csi2rx.yaml
+++ b/Documentation/devicetree/bindings/media/cdns,csi2rx.yaml
@@ -160,50 +160,50 @@ examples:
         phy-names = "dphy";
 
         ports {
-                #address-cells = <1>;
-                #size-cells = <0>;
+            #address-cells = <1>;
+            #size-cells = <0>;
 
-                port@0 {
-                    reg = <0>;
+            port@0 {
+                reg = <0>;
 
-                    csi2rx_in_sensor: endpoint {
-                        remote-endpoint = <&sensor_out_csi2rx>;
-                        clock-lanes = <0>;
-                        data-lanes = <1 2>;
-                    };
+                csi2rx_in_sensor: endpoint {
+                    remote-endpoint = <&sensor_out_csi2rx>;
+                    clock-lanes = <0>;
+                    data-lanes = <1 2>;
                 };
+            };
 
-                port@1 {
-                    reg = <1>;
+            port@1 {
+                reg = <1>;
 
-                    csi2rx_out_grabber0: endpoint {
-                        remote-endpoint = <&grabber0_in_csi2rx>;
-                    };
+                csi2rx_out_grabber0: endpoint {
+                    remote-endpoint = <&grabber0_in_csi2rx>;
                 };
+            };
 
-                port@2 {
-                    reg = <2>;
+            port@2 {
+                reg = <2>;
 
-                    csi2rx_out_grabber1: endpoint {
-                        remote-endpoint = <&grabber1_in_csi2rx>;
-                    };
+                csi2rx_out_grabber1: endpoint {
+                    remote-endpoint = <&grabber1_in_csi2rx>;
                 };
+            };
 
-                port@3 {
-                    reg = <3>;
+            port@3 {
+                reg = <3>;
 
-                    csi2rx_out_grabber2: endpoint {
-                        remote-endpoint = <&grabber2_in_csi2rx>;
-                    };
+                csi2rx_out_grabber2: endpoint {
+                    remote-endpoint = <&grabber2_in_csi2rx>;
                 };
+            };
 
-                port@4 {
-                    reg = <4>;
+            port@4 {
+                reg = <4>;
 
-                    csi2rx_out_grabber3: endpoint {
-                        remote-endpoint = <&grabber3_in_csi2rx>;
-                    };
+                csi2rx_out_grabber3: endpoint {
+                    remote-endpoint = <&grabber3_in_csi2rx>;
                 };
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/media/fsl,imx95-csi-formatter.yaml b/Documentation/devicetree/bindings/media/fsl,imx95-csi-formatter.yaml
new file mode 100644
index 00000000000000..58c4e1cc056bd2
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/fsl,imx95-csi-formatter.yaml
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/fsl,imx95-csi-formatter.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: i.MX95 CSI Pixel Formatter
+
+maintainers:
+  - Guoniu Zhou <guoniu.zhou@nxp.com>
+
+description:
+  The CSI pixel formatting module found on i.MX95 uses packet info, pixel
+  and non-pixel data from the CSI-2 host controller and reformat them to
+  match Pixel Link(PL) definition.
+
+properties:
+  compatible:
+    const: fsl,imx95-csi-formatter
+
+  reg:
+    maxItems: 1
+    description: Register offset and size within the parent syscon
+
+  clocks:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 1
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description:
+          Input port, connects to MIPI CSI-2 receiver output (IDI interface)
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description:
+          Output port, connects to ISI input via Pixel Link (PL)
+
+    required:
+      - port@0
+      - port@1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - power-domains
+  - ports
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/nxp,imx95-clock.h>
+
+    formatter@20 {
+        compatible = "fsl,imx95-csi-formatter";
+        reg = <0x20 0x100>;
+        clocks = <&cameramix_csr IMX95_CLK_CAMBLK_CSI2_FOR0>;
+        power-domains = <&scmi_devpd 3>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+
+                endpoint {
+                    remote-endpoint = <&mipi_csi_0_out>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+
+                endpoint {
+                    remote-endpoint = <&isi_in_2>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/media/i2c/galaxycore,gc05a2.yaml b/Documentation/devicetree/bindings/media/i2c/galaxycore,gc05a2.yaml
index 0e7a7b5ac89f61..99d728c47cfbee 100644
--- a/Documentation/devicetree/bindings/media/i2c/galaxycore,gc05a2.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/galaxycore,gc05a2.yaml
@@ -88,7 +88,7 @@ examples:
         #size-cells = <0>;
 
         sensor@37 {
-            compatible =  "galaxycore,gc05a2";
+            compatible = "galaxycore,gc05a2";
             reg = <0x37>;
 
             clocks = <&gc05a2_clk>;
diff --git a/Documentation/devicetree/bindings/media/i2c/himax,hm1246.yaml b/Documentation/devicetree/bindings/media/i2c/himax,hm1246.yaml
new file mode 100644
index 00000000000000..9994105552e7eb
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/himax,hm1246.yaml
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright 2025 Matthias Fend <matthias.fend@emfend.at>
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/himax,hm1246.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Himax HM1246-AWD 1/3.7-Inch megapixel SoC image sensor
+
+maintainers:
+  - Matthias Fend <matthias.fend@emfend.at>
+
+description:
+  The Himax HM1246-AWD is a 1/3.7-Inch CMOS image sensor SoC with an active
+  array size of 1296 x 976. It is programmable through an I2C interface and
+  connected via parallel bus.
+
+allOf:
+  - $ref: /schemas/media/video-interface-devices.yaml#
+
+properties:
+  compatible:
+    const: himax,hm1246
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    description: Input reference clock (6 - 27 MHz)
+    maxItems: 1
+
+  reset-gpios:
+    description: Active low XSHUTDOWN pin
+    maxItems: 1
+
+  avdd-supply:
+    description: Power for analog circuit (3.0 - 3.6 V)
+
+  iovdd-supply:
+    description: Power for I/O circuit (1.7 - 3.6 V)
+
+  dvdd-supply:
+    description: Power for digital circuit (1.5 / 1.8 V)
+
+  port:
+    $ref: /schemas/graph.yaml#/$defs/port-base
+    additionalProperties: false
+    description: Parallel video output port
+
+    properties:
+      endpoint:
+        $ref: /schemas/media/video-interfaces.yaml#
+        unevaluatedProperties: false
+
+        properties:
+          bus-width:
+            default: 10
+
+          hsync-active:
+            default: 1
+
+          vsync-active:
+            default: 1
+
+          pclk-sample:
+            default: 0
+
+        required:
+          - link-frequencies
+
+    required:
+      - endpoint
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - avdd-supply
+  - iovdd-supply
+  - dvdd-supply
+  - port
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interfaces.h>
+    #include <dt-bindings/media/video-interface-devices.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        sensor@24 {
+            compatible =  "himax,hm1246";
+            reg = <0x24>;
+
+            clocks = <&hm1246_clk>;
+
+            reset-gpios = <&gpio0 0 GPIO_ACTIVE_LOW>;
+
+            avdd-supply = <&hm1246_avdd>;
+            iovdd-supply = <&hm1246_iovdd>;
+            dvdd-supply = <&hm1246_dvdd>;
+
+            orientation = <MEDIA_ORIENTATION_EXTERNAL>;
+            rotation = <0>;
+
+            port {
+                endpoint {
+                    remote-endpoint = <&isp_par_in>;
+                    bus-width = <10>;
+                    hsync-active = <1>; /* active high */
+                    vsync-active = <1>; /* active high */
+                    pclk-sample = <1>; /* sample on rising edge */
+                    link-frequencies = /bits/ 64 <42200000>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/media/i2c/hynix,hi846.yaml b/Documentation/devicetree/bindings/media/i2c/hynix,hi846.yaml
index 1a57f2aa198228..b7bc6ba26e6ea3 100644
--- a/Documentation/devicetree/bindings/media/i2c/hynix,hi846.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/hynix,hi846.yaml
@@ -86,6 +86,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -102,7 +103,7 @@ examples:
             vddio-supply = <&reg_camera_vddio>;
             reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>;
             shutdown-gpios = <&gpio5 4 GPIO_ACTIVE_LOW>;
-            orientation = <0>;
+            orientation = <MEDIA_ORIENTATION_FRONT>;
             rotation = <0>;
 
             port {
diff --git a/Documentation/devicetree/bindings/media/i2c/ite,it6625.yaml b/Documentation/devicetree/bindings/media/i2c/ite,it6625.yaml
new file mode 100644
index 00000000000000..756fe655f534f8
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/ite,it6625.yaml
@@ -0,0 +1,175 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/ite,it6625.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ITE IT6625/IT6626 HDMI to dual MIPI CSI-2 bridge
+
+maintainers:
+  - Hermes Wu <Hermes.wu@ite.com.tw>
+
+description:
+  The ITE IT6625 and IT6626 are HDMI to MIPI CSI-2 bridge devices.
+  IT6625 supports an HDMI 2.0 input and converts it to one or two D-PHY CSI
+  outputs, while IT6626 supports an HDMI 2.1 input and converts it to one or
+  two C/D-PHY CSI outputs. The bridges are programmable through I2C and
+  expose two selectable CSI-2 output ports and an HDMI input port. The
+  bridge can operate in split mode or clone mode.
+
+properties:
+  compatible:
+    enum:
+      - ite,it6625
+      - ite,it6626
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  reset-gpios:
+    description:
+      GPIO connected to the active-low reset line.
+    maxItems: 1
+
+  vcc10-supply:
+    description: 1.0V core supply
+
+  vdd33-supply:
+    description: 3.3V supply
+
+  ovdd-supply:
+    description: I/O supply voltage
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
+        description: CSI-2 output port MIPI0
+
+        properties:
+          endpoint:
+            $ref: /schemas/media/video-interfaces.yaml#
+            unevaluatedProperties: false
+
+            properties:
+              data-lanes:
+                minItems: 1
+                maxItems: 4
+
+              bus-type:
+                enum:
+                  - 1 # MEDIA_BUS_TYPE_CSI2_CPHY
+                  - 4 # MEDIA_BUS_TYPE_CSI2_DPHY
+
+            required:
+              - data-lanes
+
+      port@1:
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
+        description: CSI-2 output port MIPI1
+
+        properties:
+          endpoint:
+            $ref: /schemas/media/video-interfaces.yaml#
+            unevaluatedProperties: false
+
+            properties:
+              data-lanes:
+                minItems: 1
+                maxItems: 4
+
+              bus-type:
+                enum:
+                  - 1 # MEDIA_BUS_TYPE_CSI2_CPHY
+                  - 4 # MEDIA_BUS_TYPE_CSI2_DPHY
+
+            required:
+              - data-lanes
+
+      port@2:
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
+        description: HDMI input port
+
+    required:
+      - port@0
+
+required:
+  - compatible
+  - reg
+  - ports
+  - vcc10-supply
+  - vdd33-supply
+  - ovdd-supply
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ite,it6626
+    then:
+      properties:
+        ports:
+          properties:
+            port@0:
+              properties:
+                endpoint:
+                  required:
+                    - bus-type
+            port@1:
+              properties:
+                endpoint:
+                  required:
+                    - bus-type
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      hdmi-bridge@4c {
+        compatible = "ite,it6625";
+        reg = <0x4c>;
+        interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+
+        reset-gpios = <&gpio 2 GPIO_ACTIVE_LOW>;
+
+        vcc10-supply = <&vcc10_reg>;
+        vdd33-supply = <&vdd33_reg>;
+        ovdd-supply = <&ovdd_reg>;
+
+        ports {
+          #address-cells = <1>;
+          #size-cells = <0>;
+
+          port@0 {
+            reg = <0>;
+            csi_out0: endpoint {
+              remote-endpoint = <&csi2_rx0>;
+              bus-type = <4>; /* MEDIA_BUS_TYPE_CSI2_DPHY */
+              data-lanes = <1 2 3 4>;
+            };
+          };
+
+          port@2 {
+            reg = <2>;
+            endpoint {
+              remote-endpoint = <&hdmi_con>;
+            };
+          };
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,os02g10.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,os02g10.yaml
new file mode 100644
index 00000000000000..e240974dcc9a1c
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,os02g10.yaml
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/ovti,os02g10.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: OmniVision OS02G10 Image Sensor
+
+maintainers:
+  - Tarang Raval <tarang.raval@siliconsignals.io>
+
+description:
+  The OmniVision OS02G10 is a 2MP (1920x1080) color CMOS image sensor controlled
+  through an I2C-compatible SCCB bus. It outputs RAW10 format data and supports
+  a 2-lane MIPI interface.
+
+properties:
+  compatible:
+    const: ovti,os02g10
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: XCLK clock
+
+  avdd-supply:
+    description: Analog Domain Power Supply (2.8v)
+
+  dovdd-supply:
+    description: I/O Domain Power Supply (1.8v)
+
+  dvdd-supply:
+    description: Digital core Power Supply (1.5v)
+
+  reset-gpios:
+    maxItems: 1
+    description: Reset Pin GPIO Control (active low)
+
+  port:
+    description: MIPI CSI-2 transmitter port
+    $ref: /schemas/graph.yaml#/$defs/port-base
+    additionalProperties: false
+
+    properties:
+      endpoint:
+        $ref: /schemas/media/video-interfaces.yaml#
+        unevaluatedProperties: false
+
+        required:
+          - link-frequencies
+
+    required:
+      - endpoint
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - avdd-supply
+  - dovdd-supply
+  - dvdd-supply
+  - port
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        camera-sensor@3c {
+            compatible = "ovti,os02g10";
+            reg = <0x3c>;
+            clocks = <&os02g10_clk>;
+            reset-gpios = <&gpio1 7 GPIO_ACTIVE_LOW>;
+
+            avdd-supply = <&os02g10_avdd_2v8>;
+            dvdd-supply = <&os02g10_dvdd_1v5>;
+            dovdd-supply = <&os02g10_dovdd_1v8>;
+
+            port {
+                cam_out: endpoint {
+                    remote-endpoint = <&mipi_in_cam>;
+                    data-lanes = <1 2>;
+                    link-frequencies = /bits/ 64 <720000000>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,ov08d10.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,ov08d10.yaml
index 6f2017c7512566..b9c61395b24faf 100644
--- a/Documentation/devicetree/bindings/media/i2c/ovti,ov08d10.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,ov08d10.yaml
@@ -69,6 +69,7 @@ examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
     #include <dt-bindings/media/video-interfaces.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -84,7 +85,7 @@ examples:
             avdd-supply = <&ov08d10_vdda_2v8>;
             dvdd-supply = <&ov08d10_vddd_1v2>;
 
-            orientation = <2>;
+            orientation = <MEDIA_ORIENTATION_EXTERNAL>;
             rotation = <0>;
 
             reset-gpios = <&gpio 1 GPIO_ACTIVE_LOW>;
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,ov2680.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,ov2680.yaml
index 634d3b821b8c75..83f84a7828bff2 100644
--- a/Documentation/devicetree/bindings/media/i2c/ovti,ov2680.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,ov2680.yaml
@@ -90,22 +90,22 @@ examples:
         #size-cells = <0>;
 
         ov2680: camera-sensor@36 {
-                compatible = "ovti,ov2680";
-                reg = <0x36>;
-                clocks = <&osc>;
-                clock-names = "xvclk";
-                reset-gpios = <&gpio1 3 GPIO_ACTIVE_LOW>;
-
-                DOVDD-supply = <&sw2_reg>;
-                DVDD-supply = <&sw2_reg>;
-                AVDD-supply = <&reg_peri_3p15v>;
-
-                port {
-                        ov2680_to_mipi: endpoint {
-                                remote-endpoint = <&mipi_from_sensor>;
-                                link-frequencies = /bits/ 64 <330000000>;
-                        };
+            compatible = "ovti,ov2680";
+            reg = <0x36>;
+            clocks = <&osc>;
+            clock-names = "xvclk";
+            reset-gpios = <&gpio1 3 GPIO_ACTIVE_LOW>;
+
+            DOVDD-supply = <&sw2_reg>;
+            DVDD-supply = <&sw2_reg>;
+            AVDD-supply = <&reg_peri_3p15v>;
+
+            port {
+                ov2680_to_mipi: endpoint {
+                    remote-endpoint = <&mipi_from_sensor>;
+                    link-frequencies = /bits/ 64 <330000000>;
                 };
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,ov2732.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,ov2732.yaml
index 814fc568c550af..b6b5145802e736 100644
--- a/Documentation/devicetree/bindings/media/i2c/ovti,ov2732.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,ov2732.yaml
@@ -94,9 +94,9 @@ examples:
 
             port {
                 camera_out: endpoint {
-                  data-lanes = <1 2>;
-                  link-frequencies = /bits/ 64 <360000000>;
-                  remote-endpoint = <&mipi_in_camera>;
+                    data-lanes = <1 2>;
+                    link-frequencies = /bits/ 64 <360000000>;
+                    remote-endpoint = <&mipi_in_camera>;
                 };
             };
         };
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,ov4689.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,ov4689.yaml
index d96199031b66c5..fcd617848ce316 100644
--- a/Documentation/devicetree/bindings/media/i2c/ovti,ov4689.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,ov4689.yaml
@@ -96,6 +96,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -114,7 +115,7 @@ examples:
             powerdown-gpios = <&pio 107 GPIO_ACTIVE_LOW>;
             reset-gpios = <&pio 109 GPIO_ACTIVE_LOW>;
 
-            orientation = <2>;
+            orientation = <MEDIA_ORIENTATION_EXTERNAL>;
             rotation = <0>;
 
             port {
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,ov5675.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,ov5675.yaml
index ad07204057f979..6df62fd0c0c0dd 100644
--- a/Documentation/devicetree/bindings/media/i2c/ovti,ov5675.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,ov5675.yaml
@@ -85,6 +85,7 @@ examples:
   - |
     #include <dt-bindings/clock/px30-cru.h>
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
     #include <dt-bindings/pinctrl/rockchip.h>
 
     i2c {
@@ -108,7 +109,7 @@ examples:
             dovdd-supply = <&vcc_2v8>;
 
             rotation = <90>;
-            orientation = <0>;
+            orientation = <MEDIA_ORIENTATION_FRONT>;
 
             port {
                 ucam_out: endpoint {
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,ov5693.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,ov5693.yaml
index 3368b3bd8ef2f0..b2eb63d7d40a45 100644
--- a/Documentation/devicetree/bindings/media/i2c/ovti,ov5693.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,ov5693.yaml
@@ -82,6 +82,8 @@ properties:
         unevaluatedProperties: false
 
         properties:
+          clock-noncontinuous: true
+
           link-frequencies: true
 
           data-lanes:
@@ -103,6 +105,7 @@ examples:
   - |
     #include <dt-bindings/clock/px30-cru.h>
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
     #include <dt-bindings/pinctrl/rockchip.h>
 
     i2c {
@@ -126,7 +129,7 @@ examples:
             dovdd-supply = <&vcc_2v8>;
 
             rotation = <90>;
-            orientation = <0>;
+            orientation = <MEDIA_ORIENTATION_FRONT>;
 
             port {
                 ucam_out: endpoint {
diff --git a/Documentation/devicetree/bindings/media/i2c/ovti,ov64a40.yaml b/Documentation/devicetree/bindings/media/i2c/ovti,ov64a40.yaml
index 2b6143aff39139..24787c9aa1557b 100644
--- a/Documentation/devicetree/bindings/media/i2c/ovti,ov64a40.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/ovti,ov64a40.yaml
@@ -72,6 +72,7 @@ unevaluatedProperties: false
 examples:
   - |
       #include <dt-bindings/gpio/gpio.h>
+      #include <dt-bindings/media/video-interface-devices.h>
 
       i2c {
           #address-cells = <1>;
@@ -87,7 +88,7 @@ examples:
               powerdown-gpios = <&gpio1 9 GPIO_ACTIVE_HIGH>;
               reset-gpios = <&gpio1 10 GPIO_ACTIVE_LOW>;
               rotation = <180>;
-              orientation = <2>;
+              orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 
               port {
                   endpoint {
diff --git a/Documentation/devicetree/bindings/media/i2c/sony,imx111.yaml b/Documentation/devicetree/bindings/media/i2c/sony,imx111.yaml
index 20f48d5e9b2d8c..56fb5f18f07bf7 100644
--- a/Documentation/devicetree/bindings/media/i2c/sony,imx111.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/sony,imx111.yaml
@@ -69,6 +69,7 @@ examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
     #include <dt-bindings/media/video-interfaces.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -84,7 +85,7 @@ examples:
             dvdd-supply = <&camera_vddd_1v2>;
             avdd-supply = <&camera_vdda_2v7>;
 
-            orientation = <1>;
+            orientation = <MEDIA_ORIENTATION_BACK>;
             rotation = <90>;
 
             nvmem = <&eeprom>;
diff --git a/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml b/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
index d9cdfda699bfee..6699c238ee0e6c 100644
--- a/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/sony,imx355.yaml
@@ -81,6 +81,7 @@ examples:
   - |
     #include <dt-bindings/clock/qcom,camcc-sdm845.h>
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -105,7 +106,7 @@ examples:
             pinctrl-0 = <&cam_front_default>;
 
             rotation = <270>;
-            orientation = <0>;
+            orientation = <MEDIA_ORIENTATION_FRONT>;
 
             port {
                 cam_front_endpoint: endpoint {
diff --git a/Documentation/devicetree/bindings/media/i2c/sony,imx415.yaml b/Documentation/devicetree/bindings/media/i2c/sony,imx415.yaml
index 7c11e871dca67b..69a37ff68db3be 100644
--- a/Documentation/devicetree/bindings/media/i2c/sony,imx415.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/sony,imx415.yaml
@@ -86,6 +86,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -98,7 +99,7 @@ examples:
             clocks = <&clock_cam>;
             dvdd-supply = <&vcc1v1_cam>;
             lens-focus = <&vcm>;
-            orientation = <2>;
+            orientation = <MEDIA_ORIENTATION_EXTERNAL>;
             ovdd-supply = <&vcc1v8_cam>;
             reset-gpios = <&gpio_expander 14 GPIO_ACTIVE_LOW>;
             rotation = <180>;
diff --git a/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml b/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml
index 58b1f9e85a9d1c..dfd87beba065c5 100644
--- a/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml
@@ -106,6 +106,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -124,7 +125,7 @@ examples:
             reset-gpios = <&gpio 5 GPIO_ACTIVE_LOW>;
             st,leds = <2>;
 
-            orientation = <2>;
+            orientation = <MEDIA_ORIENTATION_EXTERNAL>;
             rotation = <0>;
 
             port {
diff --git a/Documentation/devicetree/bindings/media/i2c/st,vd56g3.yaml b/Documentation/devicetree/bindings/media/i2c/st,vd56g3.yaml
index c6673b8539dbcf..48db22ca4a7e64 100644
--- a/Documentation/devicetree/bindings/media/i2c/st,vd56g3.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/st,vd56g3.yaml
@@ -107,6 +107,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -125,7 +126,7 @@ examples:
             reset-gpios = <&gpio 5 GPIO_ACTIVE_LOW>;
             st,leds = <6>;
 
-            orientation = <2>;
+            orientation = <MEDIA_ORIENTATION_EXTERNAL>;
             rotation = <0>;
 
             port {
diff --git a/Documentation/devicetree/bindings/media/i2c/thine,thp7312.yaml b/Documentation/devicetree/bindings/media/i2c/thine,thp7312.yaml
index bc339a7374b224..4a66cb71137209 100644
--- a/Documentation/devicetree/bindings/media/i2c/thine,thp7312.yaml
+++ b/Documentation/devicetree/bindings/media/i2c/thine,thp7312.yaml
@@ -173,6 +173,7 @@ examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
     #include <dt-bindings/media/video-interfaces.h>
+    #include <dt-bindings/media/video-interface-devices.h>
 
     i2c {
         #address-cells = <1>;
@@ -196,7 +197,7 @@ examples:
             vddgpio-0-supply = <&vsys_v4p2>;
             vddgpio-1-supply = <&vsys_v4p2>;
 
-            orientation = <0>;
+            orientation = <MEDIA_ORIENTATION_FRONT>;
             rotation = <0>;
 
             sensors {
diff --git a/Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.txt b/Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.txt
deleted file mode 100644
index 59102edcf01ea5..00000000000000
--- a/Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.txt
+++ /dev/null
@@ -1,48 +0,0 @@
-* Toshiba TC358743 HDMI-RX to MIPI CSI2-TX Bridge
-
-The Toshiba TC358743 HDMI-RX to MIPI CSI2-TX (H2C) is a bridge that converts
-a HDMI stream to MIPI CSI-2 TX. It is programmable through I2C.
-
-Required Properties:
-
-- compatible: value should be "toshiba,tc358743"
-- clocks, clock-names: should contain a phandle link to the reference clock
-		       source, the clock input is named "refclk".
-
-Optional Properties:
-
-- reset-gpios: gpio phandle GPIO connected to the reset pin
-- interrupts: GPIO connected to the interrupt pin
-- data-lanes: should be <1 2 3 4> for four-lane operation,
-	      or <1 2> for two-lane operation
-- clock-lanes: should be <0>
-- clock-noncontinuous: Presence of this boolean property decides whether the
-		       MIPI CSI-2 clock is continuous or non-continuous.
-- link-frequencies: List of allowed link frequencies in Hz. Each frequency is
-		    expressed as a 64-bit big-endian integer. The frequency
-		    is half of the bps per lane due to DDR transmission.
-
-For further information on the MIPI CSI-2 endpoint node properties, see
-Documentation/devicetree/bindings/media/video-interfaces.txt.
-
-Example:
-
-	tc358743@f {
-		compatible = "toshiba,tc358743";
-		reg = <0x0f>;
-		clocks = <&hdmi_osc>;
-		clock-names = "refclk";
-		reset-gpios = <&gpio6 9 GPIO_ACTIVE_LOW>;
-		interrupt-parent = <&gpio2>;
-		interrupts = <5 IRQ_TYPE_LEVEL_HIGH>;
-
-		port {
-			tc358743_out: endpoint {
-				remote-endpoint = <&mipi_csi2_in>;
-				data-lanes = <1 2 3 4>;
-				clock-lanes = <0>;
-				clock-noncontinuous;
-				link-frequencies = /bits/ 64 <297000000>;
-			};
-		};
-	};
diff --git a/Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.yaml b/Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.yaml
new file mode 100644
index 00000000000000..29dd1d58f766f9
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.yaml
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/toshiba,tc358743.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Toshiba TC358743 HDMI-RX to MIPI CSI2-TX Bridge
+
+maintainers:
+  - Hans Verkuil <hverkuil@kernel.org>
+
+description:
+  The Toshiba TC358743 HDMI-RX to MIPI CSI2-TX (H2C) is a bridge that converts
+  a HDMI stream to MIPI CSI-2 TX. It is programmable through I2C.
+
+properties:
+  compatible:
+    const: toshiba,tc358743
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: refclk
+
+  reset-gpios:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  port:
+    $ref: /schemas/graph.yaml#/$defs/port-base
+    unevaluatedProperties: false
+
+    properties:
+      endpoint:
+        $ref: /schemas/media/video-interfaces.yaml#
+        unevaluatedProperties: false
+
+        properties:
+          data-lanes:
+            oneOf:
+              - items:
+                  - const: 1
+                  - const: 2
+              - items:
+                  - const: 1
+                  - const: 2
+                  - const: 3
+                  - const: 4
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - port
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        bridge@f {
+            compatible = "toshiba,tc358743";
+            reg = <0x0f>;
+            clocks = <&hdmi_osc>;
+            clock-names = "refclk";
+            reset-gpios = <&gpio6 9 GPIO_ACTIVE_LOW>;
+            interrupt-parent = <&gpio2>;
+            interrupts = <5 IRQ_TYPE_LEVEL_HIGH>;
+
+            port {
+                 endpoint {
+                     remote-endpoint = <&mipi_csi2_in>;
+                     data-lanes = <1 2 3 4>;
+                     clock-lanes = <0>;
+                     clock-noncontinuous;
+                     link-frequencies = /bits/ 64 <297000000>;
+                 };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/media/img,e5010-jpeg-enc.yaml b/Documentation/devicetree/bindings/media/img,e5010-jpeg-enc.yaml
index 085020cb9e6142..b6f80098beb7d2 100644
--- a/Documentation/devicetree/bindings/media/img,e5010-jpeg-enc.yaml
+++ b/Documentation/devicetree/bindings/media/img,e5010-jpeg-enc.yaml
@@ -61,15 +61,15 @@ examples:
     #include <dt-bindings/interrupt-controller/irq.h>
 
     soc {
-      #address-cells = <2>;
-      #size-cells = <2>;
-      jpeg-encoder@fd20000 {
-          compatible = "img,e5010-jpeg-enc";
-          reg = <0x00 0xfd20000 0x00 0x100>,
-                <0x00 0xfd20200 0x00 0x200>;
-          reg-names = "core", "mmu";
-          clocks = <&k3_clks 201 0>;
-          power-domains = <&k3_pds 201 TI_SCI_PD_EXCLUSIVE>;
-          interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
-      };
+        #address-cells = <2>;
+        #size-cells = <2>;
+        jpeg-encoder@fd20000 {
+            compatible = "img,e5010-jpeg-enc";
+            reg = <0x00 0xfd20000 0x00 0x100>,
+                  <0x00 0xfd20200 0x00 0x200>;
+            reg-names = "core", "mmu";
+            clocks = <&k3_clks 201 0>;
+            power-domains = <&k3_pds 201 TI_SCI_PD_EXCLUSIVE>;
+            interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+        };
     };
diff --git a/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml b/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml
index 4505e66876a299..551e483f3e517f 100644
--- a/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml
+++ b/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml
@@ -160,7 +160,7 @@ examples:
 
     vcodec_enc_vp8: vcodec@19002000 {
       compatible = "mediatek,mt8173-vcodec-enc-vp8";
-      reg =  <0x19002000 0x1000>; /* VENC_LT_SYS */
+      reg = <0x19002000 0x1000>; /* VENC_LT_SYS */
       interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_LOW>;
       iommus = <&iommu M4U_PORT_VENC_RCPU_SET2>,
              <&iommu M4U_PORT_VENC_REC_FRM_SET2>,
diff --git a/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml b/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml
index a4aacd3eb189bb..76878050cfd9d5 100644
--- a/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml
+++ b/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml
@@ -72,8 +72,8 @@ examples:
       compatible = "mediatek,mt2701-jpgdec";
       reg = <0x15004000 0x1000>;
       interrupts = <GIC_SPI 143 IRQ_TYPE_LEVEL_LOW>;
-      clocks =  <&imgsys CLK_IMG_JPGDEC_SMI>,
-                <&imgsys CLK_IMG_JPGDEC>;
+      clocks = <&imgsys CLK_IMG_JPGDEC_SMI>,
+               <&imgsys CLK_IMG_JPGDEC>;
       clock-names = "jpgdec-smi",
                     "jpgdec";
       power-domains = <&scpsys MT2701_POWER_DOMAIN_ISP>;
diff --git a/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml b/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml
index 5b15f8977f67f7..3bc1ccb2589386 100644
--- a/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml
+++ b/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml
@@ -67,7 +67,7 @@ examples:
                    "mediatek,mtk-jpgenc";
       reg = <0x1500a000 0x1000>;
       interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_LOW>;
-      clocks =  <&imgsys CLK_IMG_VENC>;
+      clocks = <&imgsys CLK_IMG_VENC>;
       clock-names = "jpgenc";
       power-domains = <&scpsys MT2701_POWER_DOMAIN_ISP>;
       iommus = <&iommu MT2701_M4U_PORT_JPGENC_RDMA>,
diff --git a/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml b/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml
index b3a345fc646432..d06987680ed8f2 100644
--- a/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml
+++ b/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml
@@ -149,22 +149,22 @@ examples:
         clock-names = "pclk", "scck";
 
         ports {
-               #address-cells = <1>;
-               #size-cells = <0>;
-               port@0 {
-                       reg = <0>; /* must be 0, first child port */
-                       csi2dc_in: endpoint { /* input from IDI interface */
-                               bus-type = <4>; /* MIPI CSI2 D-PHY */
-                               remote-endpoint = <&csi2host_out>;
-                       };
-               };
-
-               port@1 {
-                       reg = <1>; /* must be 1, second child port */
-                       csi2dc_out: endpoint {
-                               remote-endpoint = <&xisc_in>; /* output to sensor controller */
-                       };
-               };
+            #address-cells = <1>;
+            #size-cells = <0>;
+            port@0 {
+                reg = <0>; /* must be 0, first child port */
+                csi2dc_in: endpoint { /* input from IDI interface */
+                    bus-type = <4>; /* MIPI CSI2 D-PHY */
+                    remote-endpoint = <&csi2host_out>;
+                };
+            };
+
+            port@1 {
+                reg = <1>; /* must be 1, second child port */
+                csi2dc_out: endpoint {
+                    remote-endpoint = <&xisc_in>; /* output to sensor controller */
+                };
+            };
         };
     };
 
@@ -181,18 +181,18 @@ examples:
         dma-names = "rx";
 
         ports {
-               #address-cells = <1>;
-               #size-cells = <0>;
-               port@0 {
-                       reg = <0>; /* must be 0, first child port */
-                       csi2dc_input: endpoint { /* input from IDI interface */
-                               remote-endpoint = <&csi2host_out>;
-                       };
-               };
-
-               port@1 {
-                       reg = <1>;
-               };
+            #address-cells = <1>;
+            #size-cells = <0>;
+            port@0 {
+                reg = <0>; /* must be 0, first child port */
+                csi2dc_input: endpoint { /* input from IDI interface */
+                    remote-endpoint = <&csi2host_out>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/media/microchip,xisc.yaml b/Documentation/devicetree/bindings/media/microchip,xisc.yaml
index 25f5f79d40ce8f..6a9e56c486c25e 100644
--- a/Documentation/devicetree/bindings/media/microchip,xisc.yaml
+++ b/Documentation/devicetree/bindings/media/microchip,xisc.yaml
@@ -118,12 +118,12 @@ examples:
         clock-output-names = "isc-mck";
 
         port {
-                xisc_in: endpoint {
-                       bus-type = <MEDIA_BUS_TYPE_PARALLEL>;
-                       remote-endpoint = <&csi2dc_out>;
-                       hsync-active = <1>;
-                       vsync-active = <1>;
-                       bus-width = <12>;
-                };
+            xisc_in: endpoint {
+                bus-type = <MEDIA_BUS_TYPE_PARALLEL>;
+                remote-endpoint = <&csi2dc_out>;
+                hsync-active = <1>;
+                vsync-active = <1>;
+                bus-width = <12>;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml b/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml
index 6ba668aa633d73..4727dd7040a849 100644
--- a/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml
+++ b/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml
@@ -95,7 +95,7 @@ examples:
 
     jpegdec: jpegdec@58400000 {
         compatible = "nxp,imx8qxp-jpgdec";
-        reg = <0x58400000 0x00050000 >;
+        reg = <0x58400000 0x00050000>;
         clocks = <&img_jpeg_dec_lpcg IMX_LPCG_CLK_0>,
                  <&img_jpeg_dec_lpcg IMX_LPCG_CLK_4>;
         interrupts = <GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH>,
@@ -111,7 +111,7 @@ examples:
 
     jpegenc: jpegenc@58450000 {
         compatible = "nxp,imx8qm-jpgenc", "nxp,imx8qxp-jpgenc";
-        reg = <0x58450000 0x00050000 >;
+        reg = <0x58450000 0x00050000>;
         clocks = <&img_jpeg_enc_lpcg IMX_LPCG_CLK_0>,
                  <&img_jpeg__lpcg IMX_LPCG_CLK_4>;
         interrupts = <GIC_SPI 305 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/Documentation/devicetree/bindings/media/nxp,imx8mq-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/nxp,imx8mq-mipi-csi2.yaml
index 4fcfc4fd35651e..9eee67ed268590 100644
--- a/Documentation/devicetree/bindings/media/nxp,imx8mq-mipi-csi2.yaml
+++ b/Documentation/devicetree/bindings/media/nxp,imx8mq-mipi-csi2.yaml
@@ -220,7 +220,7 @@ examples:
             port@0 {
                 reg = <0>;
 
-                imx8mm_mipi_csi_in: endpoint {
+                mipi_csi_in: endpoint {
                     remote-endpoint = <&imx477_out>;
                     data-lanes = <1 2 3 4>;
                 };
@@ -229,7 +229,7 @@ examples:
             port@1 {
                 reg = <1>;
 
-                imx8mm_mipi_csi_out: endpoint {
+                mipi_csi_out: endpoint {
                     remote-endpoint = <&csi_in>;
                 };
             };
diff --git a/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml b/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml
index 56c16458e3bb4d..389b545c9631c2 100644
--- a/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml
+++ b/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml
@@ -119,26 +119,26 @@ examples:
                  <&mmcc VIDEO_MAXI_CLK>;
         clock-names = "core", "iface", "bus", "mbus";
         power-domains = <&mmcc VENUS_GDSC>;
-        iommus =  <&venus_smmu 0x00>,
-                  <&venus_smmu 0x01>,
-                  <&venus_smmu 0x0a>,
-                  <&venus_smmu 0x07>,
-                  <&venus_smmu 0x0e>,
-                  <&venus_smmu 0x0f>,
-                  <&venus_smmu 0x08>,
-                  <&venus_smmu 0x09>,
-                  <&venus_smmu 0x0b>,
-                  <&venus_smmu 0x0c>,
-                  <&venus_smmu 0x0d>,
-                  <&venus_smmu 0x10>,
-                  <&venus_smmu 0x11>,
-                  <&venus_smmu 0x21>,
-                  <&venus_smmu 0x28>,
-                  <&venus_smmu 0x29>,
-                  <&venus_smmu 0x2b>,
-                  <&venus_smmu 0x2c>,
-                  <&venus_smmu 0x2d>,
-                  <&venus_smmu 0x31>;
+        iommus = <&venus_smmu 0x00>,
+                 <&venus_smmu 0x01>,
+                 <&venus_smmu 0x0a>,
+                 <&venus_smmu 0x07>,
+                 <&venus_smmu 0x0e>,
+                 <&venus_smmu 0x0f>,
+                 <&venus_smmu 0x08>,
+                 <&venus_smmu 0x09>,
+                 <&venus_smmu 0x0b>,
+                 <&venus_smmu 0x0c>,
+                 <&venus_smmu 0x0d>,
+                 <&venus_smmu 0x10>,
+                 <&venus_smmu 0x11>,
+                 <&venus_smmu 0x21>,
+                 <&venus_smmu 0x28>,
+                 <&venus_smmu 0x29>,
+                 <&venus_smmu 0x2b>,
+                 <&venus_smmu 0x2c>,
+                 <&venus_smmu 0x2d>,
+                 <&venus_smmu 0x31>;
         memory-region = <&venus_mem>;
 
         video-decoder {
diff --git a/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml b/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml
index 2d1662ef522b7b..bdf03a944fd7b5 100644
--- a/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml
+++ b/Documentation/devicetree/bindings/media/qcom,x1e80100-camss.yaml
@@ -289,19 +289,19 @@ examples:
                           "vfe_lite_cphy_rx",
                           "vfe_lite_csid";
 
-           interrupts = <GIC_SPI 464 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 466 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 431 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 468 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 359 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 477 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 478 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 479 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 122 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 465 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 467 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 469 IRQ_TYPE_EDGE_RISING>,
-                        <GIC_SPI 360 IRQ_TYPE_EDGE_RISING>;
+            interrupts = <GIC_SPI 464 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 466 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 431 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 468 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 359 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 477 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 478 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 479 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 122 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 465 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 467 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 469 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 360 IRQ_TYPE_EDGE_RISING>;
 
             interrupt-names = "csid0",
                               "csid1",
diff --git a/Documentation/devicetree/bindings/media/raspberrypi,pispbe.yaml b/Documentation/devicetree/bindings/media/raspberrypi,pispbe.yaml
index 1fc62a1d8edaf1..36fde927382a68 100644
--- a/Documentation/devicetree/bindings/media/raspberrypi,pispbe.yaml
+++ b/Documentation/devicetree/bindings/media/raspberrypi,pispbe.yaml
@@ -53,11 +53,11 @@ examples:
         #address-cells = <2>;
         #size-cells = <2>;
 
-        isp@880000  {
-             compatible = "brcm,bcm2712-pispbe", "raspberrypi,pispbe";
-             reg = <0x10 0x00880000 0x0 0x4000>;
-             interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
-             clocks = <&firmware_clocks 7>;
-             iommus = <&iommu2>;
+        isp@880000 {
+            compatible = "brcm,bcm2712-pispbe", "raspberrypi,pispbe";
+            reg = <0x10 0x00880000 0x0 0x4000>;
+            interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+            clocks = <&firmware_clocks 7>;
+            iommus = <&iommu2>;
         };
     };
diff --git a/Documentation/devicetree/bindings/media/samsung,fimc.yaml b/Documentation/devicetree/bindings/media/samsung,fimc.yaml
index 1bfba84f8854ff..1329d4ee070570 100644
--- a/Documentation/devicetree/bindings/media/samsung,fimc.yaml
+++ b/Documentation/devicetree/bindings/media/samsung,fimc.yaml
@@ -188,13 +188,13 @@ examples:
         /* ... CSIS 1 */
 
         fimc-lite@b90000 {
-              compatible = "samsung,exynos4212-fimc-lite";
-              reg = <0xb90000 0x1000>;
-              interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
-              power-domains = <&pd_isp>;
-              clocks = <&isp_clock CLK_ISP_FIMC_LITE0>;
-              clock-names = "flite";
-              iommus = <&sysmmu_fimc_lite0>;
+            compatible = "samsung,exynos4212-fimc-lite";
+            reg = <0xb90000 0x1000>;
+            interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
+            power-domains = <&pd_isp>;
+            clocks = <&isp_clock CLK_ISP_FIMC_LITE0>;
+            clock-names = "flite";
+            iommus = <&sysmmu_fimc_lite0>;
         };
 
         /* ... FIMC-LITE 1 */
diff --git a/Documentation/devicetree/bindings/media/snps,dw-hdmi-rx.yaml b/Documentation/devicetree/bindings/media/snps,dw-hdmi-rx.yaml
index b7f6c87d0e06ae..b80660da105053 100644
--- a/Documentation/devicetree/bindings/media/snps,dw-hdmi-rx.yaml
+++ b/Documentation/devicetree/bindings/media/snps,dw-hdmi-rx.yaml
@@ -78,6 +78,13 @@ properties:
       The phandle of the syscon node for the Video Output GRF register
       to enable EDID transfer through SDAIN and SCLIN.
 
+  "#sound-dai-cells":
+    const: 1
+    description:
+      The HDMI RX controller has two digital audio interfaces, one for
+      I2S and one for S/PDIF. The DAI cell selects the interface, 0 for
+      I2S and 1 for S/PDIF.
+
 required:
   - compatible
   - reg
@@ -90,7 +97,10 @@ required:
   - pinctrl-0
   - hpd-gpios
 
-additionalProperties: false
+allOf:
+  - $ref: /schemas/sound/dai-common.yaml#
+
+unevaluatedProperties: false
 
 examples:
   - |
@@ -129,4 +139,5 @@ examples:
       pinctrl-0 = <&hdmim1_rx_cec &hdmim1_rx_hpdin &hdmim1_rx_scl &hdmim1_rx_sda &hdmirx_5v_detection>;
       pinctrl-names = "default";
       hpd-gpios = <&gpio1 22 GPIO_ACTIVE_LOW>;
+      #sound-dai-cells = <1>;
     };
diff --git a/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml b/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
index 7c2ddd27780f61..cb0116176c3fd8 100644
--- a/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
+++ b/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
@@ -120,14 +120,14 @@ examples:
         dma-names = "tx";
 
         port {
-             dcmi_0: endpoint {
-                   remote-endpoint = <&ov5640_0>;
-                   bus-type = <MEDIA_BUS_TYPE_PARALLEL>;
-                   bus-width = <8>;
-                   hsync-active = <0>;
-                   vsync-active = <0>;
-                   pclk-sample = <1>;
-             };
+            dcmi_0: endpoint {
+                remote-endpoint = <&ov5640_0>;
+                bus-type = <MEDIA_BUS_TYPE_PARALLEL>;
+                bus-width = <8>;
+                hsync-active = <0>;
+                vsync-active = <0>;
+                pclk-sample = <1>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/media/st,stm32-dcmipp.yaml b/Documentation/devicetree/bindings/media/st,stm32-dcmipp.yaml
index 162a0c526d5d22..0bf7c674dd97bc 100644
--- a/Documentation/devicetree/bindings/media/st,stm32-dcmipp.yaml
+++ b/Documentation/devicetree/bindings/media/st,stm32-dcmipp.yaml
@@ -120,13 +120,13 @@ examples:
         clocks = <&rcc DCMIPP_K>;
 
         port {
-             endpoint {
-                   remote-endpoint = <&mipid02_2>;
-                   bus-width = <8>;
-                   hsync-active = <0>;
-                   vsync-active = <0>;
-                   pclk-sample = <0>;
-             };
+            endpoint {
+                remote-endpoint = <&mipid02_2>;
+                bus-width = <8>;
+                hsync-active = <0>;
+                vsync-active = <0>;
+                pclk-sample = <0>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/media/ti,cal.yaml b/Documentation/devicetree/bindings/media/ti,cal.yaml
index f1a940a110d260..c223b4f2d75a70 100644
--- a/Documentation/devicetree/bindings/media/ti,cal.yaml
+++ b/Documentation/devicetree/bindings/media/ti,cal.yaml
@@ -136,17 +136,17 @@ examples:
         ti,camerrx-control = <&scm_conf 0xE94>;
 
         ports {
-              #address-cells = <1>;
-              #size-cells = <0>;
-
-              csi2_0: port@0 {
-                    reg = <0>;
-                    csi2_phy0: endpoint {
-                           remote-endpoint = <&csi2_cam0>;
-                           clock-lanes = <0>;
-                           data-lanes = <1 2>;
-                    };
-              };
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            csi2_0: port@0 {
+                reg = <0>;
+                csi2_phy0: endpoint {
+                    remote-endpoint = <&csi2_cam0>;
+                    clock-lanes = <0>;
+                    data-lanes = <1 2>;
+                };
+            };
         };
     };
 
@@ -156,21 +156,21 @@ examples:
         #size-cells = <0>;
 
         camera-sensor@3c {
-               compatible = "ovti,ov5640";
-               reg = <0x3c>;
-               AVDD-supply = <&reg_2p8v>;
-               DOVDD-supply = <&reg_1p8v>;
-               DVDD-supply = <&reg_1p5v>;
-               clocks = <&clk_ov5640_fixed>;
-               clock-names = "xclk";
-
-               port {
-                    csi2_cam0: endpoint {
-                            remote-endpoint = <&csi2_phy0>;
-                            clock-lanes = <0>;
-                            data-lanes = <1 2>;
-                    };
-               };
+            compatible = "ovti,ov5640";
+            reg = <0x3c>;
+            AVDD-supply = <&reg_2p8v>;
+            DOVDD-supply = <&reg_1p8v>;
+            DVDD-supply = <&reg_1p5v>;
+            clocks = <&clk_ov5640_fixed>;
+            clock-names = "xclk";
+
+            port {
+                csi2_cam0: endpoint {
+                    remote-endpoint = <&csi2_phy0>;
+                    clock-lanes = <0>;
+                    data-lanes = <1 2>;
+                };
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/media/video-interface-devices.yaml b/Documentation/devicetree/bindings/media/video-interface-devices.yaml
index a81d2a155fe668..c9c3f4f1671969 100644
--- a/Documentation/devicetree/bindings/media/video-interface-devices.yaml
+++ b/Documentation/devicetree/bindings/media/video-interface-devices.yaml
@@ -392,17 +392,22 @@ properties:
       The orientation of a device (typically an image sensor or a flash LED)
       describing its mounting position relative to the usage orientation of the
       system where the device is installed on.
+      See include/dt-bindings/media/video-interface-devices.h.
+
     $ref: /schemas/types.yaml#/definitions/uint32
     enum:
-        # Front. The device is mounted on the front facing side of the system. For
-        # mobile devices such as smartphones, tablets and laptops the front side
-        # is the user facing side.
+        # MEDIA_ORIENTATION_FRONT
+        # The device is mounted on the front facing side of the system. For
+        # mobile devices such as smartphones, tablets and laptops the front
+        # side is the user facing side.
       - 0
-        # Back. The device is mounted on the back side of the system, which is
+        # MEDIA_ORIENTATION_BACK
+        # The device is mounted on the back side of the system, which is
         # defined as the opposite side of the front facing one.
       - 1
-        # External. The device is not attached directly to the system but is
-        # attached in a way that allows it to move freely.
+        # MEDIA_ORIENTATION_EXTERNAL
+        # The device is not attached directly to the system but is attached in
+        # a way that allows it to move freely.
       - 2
 
 additionalProperties: true
diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml
index 0762e0ff66ef02..33c989f7fc3307 100644
--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml
@@ -16,7 +16,7 @@ description: |
   MediaTek SMI have two generations of HW architecture, here is the list
   which generation the SoCs use:
   generation 1: mt2701 and mt7623.
-  generation 2: mt2712, mt6779, mt8167, mt8173, mt8183, mt8186, mt8188, mt8192 and mt8195.
+  generation 2: mt2712, mt6779, mt8167, mt8173, mt8183, mt8186, mt8188, mt8189, mt8192 and mt8195.
 
   There's slight differences between the two SMI, for generation 2, the
   register which control the iommu port is at each larb's register base. But
@@ -25,6 +25,21 @@ description: |
   SMI generation 1 to transform the smi clock into emi clock domain, but that is
   not needed for SMI generation 2.
 
+  The smi-common connects with smi-larb and IOMMU. The maximum inputs number of
+  a smi-common is 8. In SMI generation 2, the engines number may be over 8.
+  In this case, we use a smi-sub-common to merge some larbs.
+  The block diagram something is like:
+
+                          IOMMU
+                           | |
+                       smi-common
+              ---------------------------
+               |          |           ...
+              larb0   sub-common      ...  <-max number is 8
+                   ----------------
+                    |     |    ...
+                  larb1 larbX  ...  <-max number is 8
+
 properties:
   compatible:
     oneOf:
@@ -40,6 +55,8 @@ properties:
           - mediatek,mt8186-smi-common
           - mediatek,mt8188-smi-common-vdo
           - mediatek,mt8188-smi-common-vpp
+          - mediatek,mt8189-smi-common
+          - mediatek,mt8189-smi-sub-common
           - mediatek,mt8192-smi-common
           - mediatek,mt8195-smi-common-vdo
           - mediatek,mt8195-smi-common-vpp
@@ -108,6 +125,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - mediatek,mt8189-smi-sub-common
               - mediatek,mt8195-smi-sub-common
     then:
       required:
@@ -157,6 +175,7 @@ allOf:
             - mediatek,mt6795-smi-common
             - mediatek,mt8167-smi-common
             - mediatek,mt8173-smi-common
+            - mediatek,mt8189-smi-common
 
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
index 2e7fac4b50945d..9a5dafd7c07e5a 100644
--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
@@ -27,6 +27,7 @@ properties:
           - mediatek,mt8183-smi-larb
           - mediatek,mt8186-smi-larb
           - mediatek,mt8188-smi-larb
+          - mediatek,mt8189-smi-larb
           - mediatek,mt8192-smi-larb
           - mediatek,mt8195-smi-larb
 
@@ -85,6 +86,7 @@ allOf:
             - mediatek,mt8183-smi-larb
             - mediatek,mt8186-smi-larb
             - mediatek,mt8188-smi-larb
+            - mediatek,mt8189-smi-larb
             - mediatek,mt8195-smi-larb
 
     then:
@@ -119,6 +121,7 @@ allOf:
               - mediatek,mt6779-smi-larb
               - mediatek,mt8186-smi-larb
               - mediatek,mt8188-smi-larb
+              - mediatek,mt8189-smi-larb
               - mediatek,mt8192-smi-larb
               - mediatek,mt8195-smi-larb
 
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml
index 9398aae4909333..8e0c67d671f917 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml
@@ -16,9 +16,13 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - nvidia,tegra114-emc
-      - nvidia,tegra124-emc
+    oneOf:
+      - enum:
+          - nvidia,tegra114-emc
+          - nvidia,tegra124-emc
+      - items:
+          - const: nvidia,tegra132-emc
+          - const: nvidia,tegra124-emc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml
index f8747cebb6808c..2f1ffd27e1d918 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml
@@ -22,6 +22,7 @@ properties:
     enum:
       - nvidia,tegra114-mc
       - nvidia,tegra124-mc
+      - nvidia,tegra132-mc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml b/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
index 3cbc0dc12c319f..d2de227ae7a6de 100644
--- a/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
+++ b/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
@@ -159,6 +159,23 @@ properties:
     required:
       - compatible
 
+  efuse:
+    type: object
+    unevaluatedProperties: false
+    description:
+      The efuse is responsible for storing calibration data, such as thermal
+      sensor calibration.
+
+    properties:
+      compatible:
+        const: mediatek,mt6323-efuse
+
+      nvmem-layout:
+        $ref: /schemas/nvmem/layouts/nvmem-layout.yaml#
+
+    required:
+      - compatible
+
   leds:
     type: object
     additionalProperties: false
@@ -307,6 +324,10 @@ examples:
         interrupt-controller;
         #interrupt-cells = <2>;
 
+        efuse {
+          compatible = "mediatek,mt6323-efuse";
+        };
+
         leds {
             compatible = "mediatek,mt6323-led";
             #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/mfd/motorola,cpcap.yaml b/Documentation/devicetree/bindings/mfd/motorola,cpcap.yaml
new file mode 100644
index 00000000000000..da7abe2ec9126a
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/motorola,cpcap.yaml
@@ -0,0 +1,408 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/motorola,cpcap.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Motorola CPCAP PMIC MFD
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+properties:
+  compatible:
+    oneOf:
+      - enum:
+          - motorola,mapphone-cpcap
+          - motorola,mot-cpcap
+
+      - items:
+          - const: motorola,cpcap
+          - const: st,6556002
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  interrupt-controller: true
+
+  "#interrupt-cells":
+    const: 2
+
+  spi-max-frequency:
+    maximum: 9600000
+
+  spi-cs-high: true
+  spi-cpol: true
+  spi-cpha: true
+
+  adc:
+    $ref: /schemas/iio/adc/motorola,cpcap-adc.yaml#
+
+  audio-codec:
+    type: object
+    additionalProperties: false
+
+    properties:
+      interrupts:
+        items:
+          - description: headset detect interrupt
+          - description: microphone bias 2 detect interrupt
+
+      interrupt-names:
+        items:
+          - const: hs
+          - const: mb2
+
+      "#sound-dai-cells":
+        const: 1
+
+      VAUDIO-supply:
+        description:
+          Codec power supply, usually VAUDIO regulator of CPCAP.
+
+      ports:
+        $ref: /schemas/graph.yaml#/properties/ports
+
+        properties:
+          port@0:
+            $ref: /schemas/graph.yaml#/properties/port
+            description: port connected to the Stereo HiFi DAC
+
+          port@1:
+            $ref: /schemas/graph.yaml#/properties/port
+            description: port connected to the Voice DAC
+
+        required:
+          - port@0
+          - port@1
+
+    required:
+      - interrupts
+      - interrupt-names
+      - "#sound-dai-cells"
+
+  battery:
+    $ref: /schemas/power/supply/cpcap-battery.yaml#
+
+  charger:
+    $ref: /schemas/power/supply/cpcap-charger.yaml#
+
+  key-power:
+    $ref: /schemas/input/motorola,cpcap-pwrbutton.yaml#
+
+  phy:
+    $ref: /schemas/phy/motorola,cpcap-usb-phy.yaml#
+
+  regulator:
+    $ref: /schemas/regulator/motorola,cpcap-regulator.yaml#
+
+  rtc:
+    $ref: /schemas/rtc/motorola,cpcap-rtc.yaml#
+
+patternProperties:
+  "^led(-[a-z]+)?$":
+    $ref: /schemas/leds/motorola,cpcap-leds.yaml#
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - interrupt-controller
+  - "#interrupt-cells"
+  - spi-max-frequency
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/input/linux-event-codes.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        cpcap: pmic@0 {
+            compatible = "motorola,cpcap", "st,6556002";
+            reg = <0>; /* cs0 */
+
+            interrupt-parent = <&gpio1>;
+            interrupts = <7 IRQ_TYPE_EDGE_RISING>;
+
+            interrupt-controller;
+            #interrupt-cells = <2>;
+
+            spi-max-frequency = <3000000>;
+            spi-cs-high;
+
+            spi-cpol;
+            spi-cpha;
+
+            cpcap_adc: adc {
+                compatible = "motorola,cpcap-adc";
+
+                interrupt-parent = <&cpcap>;
+                interrupts = <8 IRQ_TYPE_NONE>;
+                interrupt-names = "adcdone";
+
+                #io-channel-cells = <1>;
+            };
+
+            cpcap_audio: audio-codec {
+                interrupt-parent = <&cpcap>;
+                interrupts = <9 IRQ_TYPE_NONE>, <10 IRQ_TYPE_NONE>;
+                interrupt-names = "hs", "mb2";
+
+                VAUDIO-supply = <&vdd_audio>;
+
+                #sound-dai-cells = <1>;
+
+                ports {
+                    #address-cells = <1>;
+                    #size-cells = <0>;
+
+                    /* HiFi */
+                    port@0 {
+                        reg = <0>;
+
+                        cpcap_audio_codec0: endpoint {
+                        };
+                    };
+
+                    /* Voice */
+                    port@1 {
+                        reg = <1>;
+
+                        cpcap_audio_codec1: endpoint {
+                        };
+                    };
+                };
+            };
+
+            cpcap_battery: battery {
+                compatible = "motorola,cpcap-battery";
+
+                interrupt-parent = <&cpcap>;
+                interrupts = <6 IRQ_TYPE_NONE>, <5 IRQ_TYPE_NONE>,
+                             <3 IRQ_TYPE_NONE>, <20 IRQ_TYPE_NONE>,
+                             <54 IRQ_TYPE_NONE>, <57 IRQ_TYPE_NONE>;
+                interrupt-names = "eol", "lowbph", "lowbpl",
+                                  "chrgcurr1", "battdetb", "cccal";
+
+                io-channels = <&cpcap_adc 0>, <&cpcap_adc 1>,
+                              <&cpcap_adc 5>, <&cpcap_adc 6>;
+                io-channel-names = "battdetb", "battp",
+                                   "chg_isense", "batti";
+                power-supplies = <&cpcap_charger>;
+            };
+
+            cpcap_charger: charger {
+                compatible = "motorola,mapphone-cpcap-charger";
+
+                interrupt-parent = <&cpcap>;
+                interrupts = <13 IRQ_TYPE_NONE>, <12 IRQ_TYPE_NONE>,
+                             <29 IRQ_TYPE_NONE>, <28 IRQ_TYPE_NONE>,
+                             <22 IRQ_TYPE_NONE>, <21 IRQ_TYPE_NONE>,
+                             <20 IRQ_TYPE_NONE>, <19 IRQ_TYPE_NONE>,
+                             <54 IRQ_TYPE_NONE>;
+                interrupt-names = "chrg_det", "rvrs_chrg", "chrg_se1b",
+                                  "se0conn", "rvrs_mode", "chrgcurr2",
+                                  "chrgcurr1", "vbusvld", "battdetb";
+
+                mode-gpios = <&gpio3 29 GPIO_ACTIVE_LOW>,
+                             <&gpio3 23 GPIO_ACTIVE_LOW>;
+
+                io-channels = <&cpcap_adc 0>, <&cpcap_adc 1>,
+                              <&cpcap_adc 2>, <&cpcap_adc 5>,
+                              <&cpcap_adc 6>;
+                io-channel-names = "battdetb", "battp",
+                                   "vbus", "chg_isense",
+                                   "batti";
+            };
+
+            key-power {
+                compatible = "motorola,cpcap-pwrbutton";
+
+                interrupt-parent = <&cpcap>;
+                interrupts = <23 IRQ_TYPE_NONE>;
+            };
+
+            led-red {
+                compatible = "motorola,cpcap-led-red";
+                vdd-supply = <&vdd_led>;
+                label = "status-led::red";
+            };
+
+            led-green {
+                compatible = "motorola,cpcap-led-green";
+                vdd-supply = <&vdd_led>;
+                label = "status-led::green";
+            };
+
+            led-blue {
+                compatible = "motorola,cpcap-led-blue";
+                vdd-supply = <&vdd_led>;
+                label = "status-led::blue";
+            };
+
+            cpcap_usb2_phy: phy {
+                compatible = "motorola,cpcap-usb-phy";
+
+                pinctrl-0 = <&usb_gpio_mux_sel1>, <&usb_gpio_mux_sel2>;
+                pinctrl-1 = <&usb_ulpi_pins>;
+                pinctrl-2 = <&usb_utmi_pins>;
+                pinctrl-3 = <&uart3_pins>;
+                pinctrl-names = "default", "ulpi", "utmi", "uart";
+                #phy-cells = <0>;
+
+                interrupts-extended =
+                    <&cpcap 15 IRQ_TYPE_NONE>, <&cpcap 14 IRQ_TYPE_NONE>,
+                    <&cpcap 28 IRQ_TYPE_NONE>, <&cpcap 19 IRQ_TYPE_NONE>,
+                    <&cpcap 18 IRQ_TYPE_NONE>, <&cpcap 17 IRQ_TYPE_NONE>,
+                    <&cpcap 16 IRQ_TYPE_NONE>, <&cpcap 49 IRQ_TYPE_NONE>,
+                    <&cpcap 48 IRQ_TYPE_NONE>;
+                interrupt-names = "id_ground", "id_float", "se0conn",
+                                  "vbusvld", "sessvld", "sessend",
+                                  "se1", "dm", "dp";
+
+                mode-gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>,
+                             <&gpio1 0 GPIO_ACTIVE_HIGH>;
+
+                io-channels = <&cpcap_adc 2>, <&cpcap_adc 7>;
+                io-channel-names = "vbus", "id";
+
+                vusb-supply = <&avdd_usb>;
+            };
+
+            regulator {
+                compatible = "motorola,cpcap-regulator";
+
+                regulators {
+                    vdd_cpu: SW1 {
+                        regulator-name = "vdd_cpu";
+                        regulator-min-microvolt = <750000>;
+                        regulator-max-microvolt = <1125000>;
+                        regulator-enable-ramp-delay = <1500>;
+                        regulator-always-on;
+                        regulator-boot-on;
+                    };
+
+                    vdd_core: SW2 {
+                        regulator-name = "vdd_core";
+                        regulator-min-microvolt = <950000>;
+                        regulator-max-microvolt = <1300000>;
+                        regulator-enable-ramp-delay = <1500>;
+                        regulator-always-on;
+                        regulator-boot-on;
+                    };
+
+                    vdd_1v8_vio: SW3 {
+                        regulator-name = "vdd_1v8_vio";
+                        regulator-min-microvolt = <1800000>;
+                        regulator-max-microvolt = <1800000>;
+                        regulator-enable-ramp-delay = <0>;
+                        regulator-always-on;
+                        regulator-boot-on;
+                    };
+
+                    vdd_aon: SW4 {
+                        regulator-name = "vdd_aon";
+                        regulator-min-microvolt = <950000>;
+                        regulator-max-microvolt = <1300000>;
+                        regulator-enable-ramp-delay = <1500>;
+                        regulator-always-on;
+                        regulator-boot-on;
+                    };
+
+                    vdd_led: SW5 {
+                        regulator-name = "vdd_led";
+                        regulator-min-microvolt = <5050000>;
+                        regulator-max-microvolt = <5050000>;
+                        regulator-enable-ramp-delay = <1500>;
+                        regulator-boot-on;
+                    };
+
+                    vdd_hvio: VHVIO {
+                        regulator-name = "vdd_hvio";
+                        regulator-min-microvolt = <2775000>;
+                        regulator-max-microvolt = <2775000>;
+                        regulator-enable-ramp-delay = <1000>;
+                    };
+
+                    vcore_emmc: VSDIO {
+                        regulator-name = "vcore_emmc";
+                        regulator-min-microvolt = <1500000>;
+                        regulator-max-microvolt = <3000000>;
+                        regulator-enable-ramp-delay = <1000>;
+                        regulator-always-on;
+                        regulator-boot-on;
+                    };
+
+                    avdd_dsi_csi: VCSI {
+                        regulator-name = "avdd_dsi_csi";
+                        regulator-min-microvolt = <1200000>;
+                        regulator-max-microvolt = <1200000>;
+                        regulator-enable-ramp-delay = <1000>;
+                        regulator-boot-on;
+                    };
+
+                    avdd_3v3_periph: VWLAN2 {
+                        regulator-name = "avdd_3v3_periph";
+                        regulator-min-microvolt = <2775000>;
+                        regulator-max-microvolt = <3300000>;
+                        regulator-enable-ramp-delay = <1000>;
+                        regulator-boot-on;
+                    };
+
+                    vddio_usd: VSIMCARD {
+                        regulator-name = "vddio_usd";
+                        regulator-min-microvolt = <1800000>;
+                        regulator-max-microvolt = <2900000>;
+                        regulator-enable-ramp-delay = <1000>;
+                        regulator-boot-on;
+                    };
+
+                    vdd_haptic: VVIB {
+                        regulator-name = "vdd_haptic";
+                        regulator-min-microvolt = <1300000>;
+                        regulator-max-microvolt = <3000000>;
+                        regulator-enable-ramp-delay = <1000>;
+                    };
+
+                    avdd_usb: VUSB {
+                        regulator-name = "avdd_usb";
+                        regulator-min-microvolt = <3300000>;
+                        regulator-max-microvolt = <3300000>;
+                        regulator-enable-ramp-delay = <1000>;
+                        regulator-always-on;
+                        regulator-boot-on;
+                    };
+
+                    vdd_audio: VAUDIO {
+                        regulator-name = "vdd_audio";
+                        regulator-min-microvolt = <2775000>;
+                        regulator-max-microvolt = <2775000>;
+                        regulator-enable-ramp-delay = <1000>;
+                        regulator-always-on;
+                        regulator-boot-on;
+                    };
+                };
+            };
+
+            cpcap_rtc: rtc {
+                compatible = "motorola,cpcap-rtc";
+
+                interrupt-parent = <&cpcap>;
+                interrupts = <39 IRQ_TYPE_NONE>, <26 IRQ_TYPE_NONE>;
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/mfd/motorola-cpcap.txt b/Documentation/devicetree/bindings/mfd/motorola-cpcap.txt
deleted file mode 100644
index 18c3fc26ca9323..00000000000000
--- a/Documentation/devicetree/bindings/mfd/motorola-cpcap.txt
+++ /dev/null
@@ -1,78 +0,0 @@
-Motorola CPCAP PMIC device tree binding
-
-Required properties:
-- compatible		: One or both of "motorola,cpcap" or "ste,6556002"
-- reg			: SPI chip select
-- interrupts		: The interrupt line the device is connected to
-- interrupt-controller	: Marks the device node as an interrupt controller
-- #interrupt-cells	: The number of cells to describe an IRQ, should be 2
-- #address-cells	: Child device offset number of cells, should be 1
-- #size-cells		: Child device size number of cells, should be 0
-- spi-max-frequency	: Typically set to 3000000
-- spi-cs-high		: SPI chip select direction
-
-Optional subnodes:
-
-The sub-functions of CPCAP get their own node with their own compatible values,
-which are described in the following files:
-
-- Documentation/devicetree/bindings/power/supply/cpcap-battery.yaml
-- Documentation/devicetree/bindings/power/supply/cpcap-charger.yaml
-- Documentation/devicetree/bindings/regulator/cpcap-regulator.txt
-- Documentation/devicetree/bindings/phy/motorola,cpcap-usb-phy.yaml
-- Documentation/devicetree/bindings/input/cpcap-pwrbutton.txt
-- Documentation/devicetree/bindings/rtc/cpcap-rtc.txt
-- Documentation/devicetree/bindings/leds/leds-cpcap.txt
-- Documentation/devicetree/bindings/iio/adc/motorola,cpcap-adc.yaml
-
-The only exception is the audio codec. Instead of a compatible value its
-node must be named "audio-codec".
-
-Required properties for the audio-codec subnode:
-
-- #sound-dai-cells = <1>;
-- interrupts		: should contain jack detection interrupts, with headset
-			  detect interrupt matching "hs" and microphone bias 2
-			  detect interrupt matching "mb2" in interrupt-names.
-- interrupt-names	: Contains "hs", "mb2"
-
-The audio-codec provides two DAIs. The first one is connected to the
-Stereo HiFi DAC and the second one is connected to the Voice DAC.
-
-Example:
-
-&mcspi1 {
-	cpcap: pmic@0 {
-		compatible = "motorola,cpcap", "ste,6556002";
-		reg = <0>;	/* cs0 */
-		interrupt-parent = <&gpio1>;
-		interrupts = <7 IRQ_TYPE_EDGE_RISING>;
-		interrupt-controller;
-		#interrupt-cells = <2>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-		spi-max-frequency = <3000000>;
-		spi-cs-high;
-
-		audio-codec {
-			#sound-dai-cells = <1>;
-			interrupts-extended = <&cpcap 9 0>, <&cpcap 10 0>;
-			interrupt-names = "hs", "mb2";
-
-			/* HiFi */
-			port@0 {
-				endpoint {
-					remote-endpoint = <&cpu_dai1>;
-				};
-			};
-
-			/* Voice */
-			port@1 {
-				endpoint {
-					remote-endpoint = <&cpu_dai2>;
-				};
-			};
-		};
-	};
-};
-
diff --git a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
index 5e0ed30a9a466e..f6534e9e9912ec 100644
--- a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
+++ b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml
@@ -74,6 +74,7 @@ properties:
           - qcom,pm8994
           - qcom,pm8998
           - qcom,pma8084
+          - qcom,pmau0102
           - qcom,pmc8180
           - qcom,pmc8180c
           - qcom,pmc8380
diff --git a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
index c3889f974ed3a4..a021e4dea486cc 100644
--- a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
+++ b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
@@ -60,6 +60,7 @@ properties:
           - qcom,tcsr-msm8916
           - qcom,tcsr-msm8917
           - qcom,tcsr-msm8937
+          - qcom,tcsr-msm8952
           - qcom,tcsr-msm8953
           - qcom,tcsr-msm8960
           - qcom,tcsr-msm8974
diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd71815-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd71815-pmic.yaml
index 4fc78efaa5504a..0a67de05467462 100644
--- a/Documentation/devicetree/bindings/mfd/rohm,bd71815-pmic.yaml
+++ b/Documentation/devicetree/bindings/mfd/rohm,bd71815-pmic.yaml
@@ -15,6 +15,9 @@ description: |
   for LED and a 500 mA single-cell linear charger. Also included is a Coulomb
   counter, a real-time clock (RTC), and a 32.768 kHz clock gate and two GPOs.
 
+allOf:
+  - $ref: rohm,pmic-pins.yaml#
+
 properties:
   compatible:
     const: rohm,bd71815
@@ -44,11 +47,7 @@ properties:
   clock-output-names:
     const: bd71815-32k-out
 
-  rohm,clkout-open-drain:
-    description: clk32kout mode. Set to 1 for "open-drain" or 0 for "cmos".
-    $ref: /schemas/types.yaml#/definitions/uint32
-    minimum: 0
-    maximum: 1
+  rohm,clkout-open-drain: true
 
   rohm,charger-sense-resistor-micro-ohms:
     minimum: 10000
diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd71828-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd71828-pmic.yaml
index 09e7d68e92bf3d..e5043dec990d6c 100644
--- a/Documentation/devicetree/bindings/mfd/rohm,bd71828-pmic.yaml
+++ b/Documentation/devicetree/bindings/mfd/rohm,bd71828-pmic.yaml
@@ -15,6 +15,9 @@ description: |
   single-cell linear charger. Also included is a Coulomb counter, a real-time
   clock (RTC), and a 32.768 kHz clock gate.
 
+allOf:
+  - $ref: rohm,pmic-pins.yaml#
+
 properties:
   compatible:
     oneOf:
@@ -49,11 +52,7 @@ properties:
   clock-output-names:
     const: bd71828-32k-out
 
-  rohm,clkout-open-drain:
-    description: clk32kout mode. Set to 1 for "open-drain" or 0 for "cmos".
-    $ref: /schemas/types.yaml#/definitions/uint32
-    minimum: 0
-    maximum: 1
+  rohm,clkout-open-drain: true
 
   rohm,charger-sense-resistor-micro-ohms:
     minimum: 10000
diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml
index b094542339e839..52d0f6e4ef0fbc 100644
--- a/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml
+++ b/Documentation/devicetree/bindings/mfd/rohm,bd72720-pmic.yaml
@@ -15,6 +15,9 @@ description:
   and a 3000 mA switching charger. ICs also include a Coulomb counter,
   a real-time clock (RTC), GPIOs and a 32.768 kHz clock gate.
 
+allOf:
+  - $ref: rohm,pmic-pins.yaml#
+
 # In addition to the properties found from the charger node, PMICs
 # use properties from a static battery node. Please see the:
 # Documentation/devicetree/bindings/power/supply/battery.yaml
@@ -80,10 +83,7 @@ properties:
   clock-output-names:
     const: bd71828-32k-out
 
-  rohm,clkout-open-drain:
-    description: clk32kout mode. Set to 1 for "open-drain" or 0 for "cmos".
-    $ref: /schemas/types.yaml#/definitions/uint32
-    maximum: 1
+  rohm,clkout-open-drain: true
 
   rohm,charger-sense-resistor-micro-ohms:
     minimum: 10000
@@ -103,14 +103,6 @@ properties:
     $ref: /schemas/leds/rohm,bd71828-leds.yaml
 
   rohm,pin-fault_b:
-    $ref: /schemas/types.yaml#/definitions/string
-    description:
-      PMIC has an OTP option to use fault_b-pin for different
-      purposes. Set this property accordingly. OTP options are
-      OTP0 - bi-directional FAULT_B or READY indicator depending on a
-      'sub option'
-      OTP1 - GPO
-      OTP2 - Power sequencer output.
     enum:
       - faultb
       - readyind
@@ -119,14 +111,6 @@ properties:
 
 patternProperties:
   "^rohm,pin-dvs[0-1]$":
-    $ref: /schemas/types.yaml#/definitions/string
-    description:
-      PMIC has 4 different OTP options to determine the use of dvs<X>-pins.
-      OTP0 - regulator RUN state control.
-      OTP1 - GPI.
-      OTP2 - GPO.
-      OTP3 - Power sequencer output.
-      This property specifies the use of the pin.
     enum:
       - dvs-input
       - gpi
@@ -134,11 +118,6 @@ patternProperties:
       - pwrseq
 
   "^rohm,pin-exten[0-1]$":
-    $ref: /schemas/types.yaml#/definitions/string
-    description: PMIC has an OTP option to use exten0-pin for different
-      purposes. Set this property accordingly.
-      OTP0 - GPO
-      OTP1 - Power sequencer output.
     enum:
       - gpo
       - pwrseq
diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd73800-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd73800-pmic.yaml
new file mode 100644
index 00000000000000..8371e7f2ab8489
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/rohm,bd73800-pmic.yaml
@@ -0,0 +1,218 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/rohm,bd73800-pmic.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ROHM BD73800 Power Management Integrated Circuit
+
+maintainers:
+  - Matti Vaittinen <mazziesaccount@gmail.com>
+
+description:
+  BD73800GW is a single-chip power management IC for battery-powered
+  portable devices. It integrates 8 buck converters, 4 LDOs and a current
+  sense amplifier with ADC. Also included is a Real Time Clock (RTC) and a
+  32.768 kHz clock gate. Depending on the OTP configuration the BD73800
+  may also have interrupt controller and GPIOs.
+
+  There are also different variants called BD71851 and BD71885 which are
+  almost identical from the software point of view.
+
+allOf:
+  - $ref: rohm,pmic-pins.yaml#
+
+properties:
+  compatible:
+    oneOf:
+      - const: rohm,bd73800
+
+      - items:
+          - enum:
+              - rohm,bd71851
+              - rohm,bd71885
+          - const: rohm,bd73800
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  # The GPIO1, CLKOUT (GPIO2), FAULT_B and EXTEN_OUT pins can be
+  # configured to interrupt pins by OTP.
+  interrupt-controller: true
+
+  "#interrupt-cells":
+    const: 1
+    description:
+      The IRQ number. 0 is GPIO1, 1 CLKOUT (GPIO2), 2 FAULT_B and 3 EXTEN_OUT.
+      NOTE, A pin can operate as IRQ source only when the OTP
+      configuration for it has been set to GPI.
+
+  gpio-controller: true
+
+  "#gpio-cells":
+    const: 2
+
+  rohm,pin-gpio1:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      Indicate if the GPIO1 pin has been set to GPI or GPO at manufacturing.
+      The GPIO1, CLKOUT, FAULT_B and EXTEN_OUT pins may be configured for a
+      specific purpose (like ADC input, 32.768 clk output, fault indicator or
+      delivering power sequence to a companion PMIC when multiple PMICs are
+      used) - but also to be either a GPO or GPI. (When used as a GPI the pin
+      can also be used as an IRQ source). The pin purpose is determined by
+      OTP (One Time Programmable memory), typically during device
+      manufacturing. The OTP can't be read at runtime so device-tree should
+      describe the GPIO pins.
+    enum: [gpi, gpo]
+
+  rohm,pin-clkout:
+    enum: [gpi, gpo]
+
+  rohm,pin-fault_b:
+    enum: [gpi, gpo]
+
+  rohm,pin-exten:
+    enum: [gpi, gpo]
+
+  clocks:
+    description:
+      The CLKOUT pin may have its purpose overridden by OTP configuration. It
+      is possible the BD73800 does not output a clock signal. Hence the clk
+      properties are optional.
+    maxItems: 1
+
+  "#clock-cells":
+    const: 0
+
+  clock-output-names:
+    maxItems: 1
+
+  rohm,clkout-open-drain: true
+
+  regulators:
+    $ref: /schemas/regulator/rohm,bd73800-regulator.yaml
+    description:
+      List of child nodes that specify the regulators.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - regulators
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/leds/common.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        pmic: pmic@4b {
+            compatible = "rohm,bd73800";
+            reg = <0x4b>;
+
+            interrupt-parent = <&gpio1>;
+            interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
+
+            clocks = <&osc 0>;
+            #clock-cells = <0>;
+            clock-output-names = "bd73800-32k-out";
+
+            gpio-controller;
+            #gpio-cells = <2>;
+
+            rohm,pin-gpio1 = "gpo";
+            rohm,pin-exten = "gpi";
+
+            regulators {
+                buck1: buck1 {
+                    regulator-name = "buck1";
+                    regulator-min-microvolt = <800000>;
+                    regulator-max-microvolt = <2000000>;
+                    regulator-always-on;
+                    regulator-ramp-delay = <1250>;
+                    rohm,dvs-run-voltage = <1150000>;
+                    rohm,dvs-suspend-voltage = <950000>;
+                };
+                buck2: buck2 {
+                    regulator-name = "buck2";
+                    regulator-min-microvolt = <800000>;
+                    regulator-max-microvolt = <2000000>;
+                    regulator-always-on;
+                    regulator-ramp-delay = <1250>;
+                    rohm,dvs-run-voltage = <1150000>;
+                    rohm,dvs-suspend-voltage = <950000>;
+                };
+                buck3: buck3 {
+                    regulator-name = "buck3";
+                    regulator-min-microvolt = <1200000>;
+                    regulator-max-microvolt = <2700000>;
+                    rohm,dvs-run-voltage = <1150000>;
+                    rohm,dvs-idle-voltage = <1100000>;
+                    rohm,dvs-suspend-voltage = <950000>;
+                    regulator-always-on;
+                };
+                buck4: buck4 {
+                    regulator-name = "buck4";
+                    regulator-min-microvolt = <1100000>;
+                    regulator-max-microvolt = <1850000>;
+                    regulator-always-on;
+                };
+                buck5: buck5 {
+                    regulator-name = "buck5";
+                    regulator-min-microvolt = <1800000>;
+                    regulator-max-microvolt = <3300000>;
+                    regulator-always-on;
+                };
+                buck6: buck6 {
+                    regulator-name = "buck6";
+                    regulator-min-microvolt = <1800000>;
+                    regulator-max-microvolt = <3300000>;
+                    regulator-always-on;
+                };
+                buck7: buck7 {
+                    regulator-name = "buck7";
+                    regulator-min-microvolt = <1800000>;
+                    regulator-max-microvolt = <3300000>;
+                    regulator-always-on;
+                };
+                buck8: buck8 {
+                    regulator-name = "buck8";
+                    regulator-min-microvolt = <500000>;
+                    regulator-max-microvolt = <1300000>;
+                    regulator-always-on;
+                };
+                ldo1: ldo1 {
+                    regulator-name = "ldo1";
+                    rohm,ldo-range-high;
+                    regulator-min-microvolt = <800000>;
+                    regulator-max-microvolt = <3300000>;
+                    regulator-always-on;
+                };
+                ldo2: ldo2 {
+                    regulator-name = "ldo2";
+                    regulator-min-microvolt = <800000>;
+                    regulator-max-microvolt = <3300000>;
+                    regulator-always-on;
+                };
+                ldo3: ldo3 {
+                    regulator-name = "ldo3";
+                    regulator-min-microvolt = <800000>;
+                    regulator-max-microvolt = <3300000>;
+                    regulator-always-on;
+                };
+                ldo4: ldo4 {
+                    regulator-name = "ldo4";
+                    regulator-min-microvolt = <650000>;
+                    regulator-max-microvolt = <1750000>;
+                    regulator-always-on;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mfd/rohm,pmic-pins.yaml b/Documentation/devicetree/bindings/mfd/rohm,pmic-pins.yaml
new file mode 100644
index 00000000000000..0d1082804f5a13
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/rohm,pmic-pins.yaml
@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/rohm,pmic-pins.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ROHM PMICs' OTP configured pin descriptions
+
+maintainers:
+  - Matti Vaittinen <mazziesaccount@gmail.com>
+
+description:
+  Collection of properties which are commonly used to describe various
+  ROHM PMIC's pins.
+
+properties:
+  rohm,clkout-open-drain:
+    description: clk32kout mode. Set to 1 for "open-drain" or 0 for "cmos"
+      a.k.a "push-pull".
+    $ref: /schemas/types.yaml#/definitions/uint32
+    maximum: 1
+
+  rohm,pin-clkout:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      The IC has different OTP options to use clkout-pin for different
+      purposes. Set this property to indicate the OTP variant.
+
+      Many of the ROHM ICs have pins, which function can be defined in
+      OTP. Some of them provide no means to detect the pin configuration
+      at run-time, leaving software unaware of which features the IC
+      really has. These properties can be used to inform which pins
+      provide which functionality. IOW, these properties aren't used to
+      configure pin, they are used to tell what functionality pins on
+      this specific IC have.
+
+    # Supported functionalities depend on the exact IC model. Following
+    # values are already defined: gpi, gpo.
+    # Please, re-use them when adding new ICs.
+
+  rohm,pin-fault_b:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      The IC has different OTP options to use fault_b-pin for different
+      purposes. Set this property to indicate the OTP variant.
+
+    # Supported functionalities depend on the exact IC model. Following
+    # values are already defined: gpi, gpo, faultb, readyind, pwrseq.
+    # Please, re-use them when adding new ICs.
+
+patternProperties:
+  "^rohm,pin-dvs[0-1]$":
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      The IC has different OTP options to use dvs<X>-pins for different
+      purposes. Set this property to indicate the OTP variant.
+    # Supported functionalities depend on the exact IC model. Following
+    # values are already defined: dvs-input, gpi, gpo, pwrseq.
+    # Please, re-use them when adding new ICs.
+
+  "^rohm,pin-exten([0-1])?$":
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      The IC has an OTP option to use exten -pin(s) for different purposes.
+      Set this property to indicate the OTP variant.
+    # Supported functionalities depend on the exact IC model. Following
+    # values are already defined: gpi, gpo, pwrseq.
+    # Please, re-use them when adding new ICs.
+
+# The schema is expected to be reference by IC schemas, which may introduce
+# other properties that must be allowed.
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml b/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml
index 0fdfbfdfe88a36..d819954ee7317a 100644
--- a/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml
+++ b/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml
@@ -312,6 +312,33 @@ properties:
         items:
           - const: audioclk
 
+      '#sound-dai-cells':
+        description: Selects one of the two digital audio interfaces
+        const: 1
+
+      ports:
+        $ref: /schemas/sound/audio-graph-port.yaml#/definitions/ports
+
+      stericsson,amic1-type-single-ended:
+        description: Configure analog microphone 1 as single-ended
+        type: boolean
+
+      stericsson,amic2-type-single-ended:
+        description: Configure analog microphone 2 as single-ended
+        type: boolean
+
+      stericsson,amic1a-bias-vamic2:
+        description: Supply analog microphone 1A from VAMIC2 instead of VAMIC1
+        type: boolean
+
+      stericsson,amic1b-bias-vamic2:
+        description: Supply analog microphone 1B from VAMIC2 instead of VAMIC1
+        type: boolean
+
+      stericsson,amic2-bias-vamic1:
+        description: Supply analog microphone 2 from VAMIC1 instead of VAMIC2
+        type: boolean
+
       stericsson,earpeice-cmv:
         description: Earpiece voltage
         $ref: /schemas/types.yaml#/definitions/uint32
@@ -319,11 +346,14 @@ properties:
 
     required:
       - compatible
+      - '#sound-dai-cells'
 
     additionalProperties: false
 
   regulator:
     description: Node describing the AB8500 internal regulators.
+      LDO means low-dropout regulator. SMPS means switched-mode power
+      supply; the SMPS regulators described here are buck converters.
     type: object
 
     properties:
@@ -439,16 +469,117 @@ properties:
         $ref: /schemas/regulator/regulator.yaml#
         unevaluatedProperties: false
 
+    allOf:
+      - if:
+          properties:
+            compatible:
+              const: stericsson,ab8500-regulator
+        then:
+          properties:
+            ab8500_buck_smps1:
+              description: Analog buck converter for the VSMPS1 1.2V I/O
+                and memory supply.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8500_buck_smps2:
+              description: Analog buck converter for the VSMPS2 1.8V I/O
+                and memory supply.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8500_buck_smps3:
+              description: Analog buck converter for VSMPS3, which supplies
+                the VSAFE power domain.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8500_buck_arm:
+              description: Analog buck converter for the VARM processor
+                supply.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8500_buck_ape:
+              description: Analog buck converter for the VAPE application
+                processor supply.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8500_buck_mod:
+              description: Analog buck converter for the VMOD modem supply.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+      - if:
+          properties:
+            compatible:
+              const: stericsson,ab8505-regulator
+        then:
+          properties:
+            ab8505_buck_smpsa:
+              description: Analog buck converter for the VSMPSA 1.2V I/O
+                and memory supply. The selector range depends on OTP
+                configuration.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8505_buck_smpsb:
+              description: Analog buck converter for the VSMPSB 1.8V I/O
+                and memory supply. The selector range depends on OTP
+                configuration.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8505_buck_safe:
+              description: Analog buck converter for the VSAFE permanent
+                logic and DDR supply. The selector range depends on OTP
+                configuration.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8505_buck_arm:
+              description: Analog buck converter for the VARM processor
+                supply, with a 0.6V to 1.39375V selector range.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8505_buck_smpsc:
+              description: Analog buck converter for the VSMPSC multimedia
+                and graphics supply. The selector range depends on OTP
+                configuration.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
+            ab8505_buck_smpsm:
+              description: Analog buck converter for the VSMPSM modem
+                supply. The selector range depends on OTP configuration.
+              type: object
+              $ref: /schemas/regulator/regulator.yaml#
+              unevaluatedProperties: false
+
     required:
       - compatible
 
-    additionalProperties: false
+    unevaluatedProperties: false
 
   regulator-external:
-    description: Node describing the AB8500 external regulators. This
-      concerns the autonomous regulators VSMPS1, VSMPS2 and VSMPS3
-      that are normally controlled by external electronics but also
-      sometimes need to be explicitly controlled by software.
+    description: Node describing AB8500 control interfaces for external
+      regulators. AB8500 controls these external devices through the
+      ExtSupply1Ena, ExtSupply2Ena and ExtSupply3Ena signals and associated
+      low-power-mode signals. They are distinct from the VSMPS1, VSMPS2 and
+      VSMPS3 buck converters integrated into AB8500.
     type: object
 
     properties:
@@ -456,19 +587,22 @@ properties:
         const: stericsson,ab8500-ext-regulator
 
       ab8500_ext1:
-        description: The voltage for the VSMPS1 external regulator
+        description: External regulator controlled through ExtSupply1Ena and
+          ExtSupply12LPn
         type: object
         $ref: /schemas/regulator/regulator.yaml#
         unevaluatedProperties: false
 
       ab8500_ext2:
-        description: The voltage for the VSMPS2 external regulator
+        description: External regulator controlled through ExtSupply2Ena and
+          ExtSupply12LPn
         type: object
         $ref: /schemas/regulator/regulator.yaml#
         unevaluatedProperties: false
 
       ab8500_ext3:
-        description: The voltage for the VSMPS3 external regulator
+        description: External regulator controlled through ExtSupply3Ena and
+          ExtSupply3LPn
         type: object
         $ref: /schemas/regulator/regulator.yaml#
         unevaluatedProperties: false
diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml
index 945a168b613db1..33e0b1bad5ab48 100644
--- a/Documentation/devicetree/bindings/mfd/syscon.yaml
+++ b/Documentation/devicetree/bindings/mfd/syscon.yaml
@@ -107,6 +107,7 @@ properties:
               - nxp,s32g3-gpr
               - qcom,apq8064-mmss-sfpb
               - qcom,apq8064-sps-sic
+              - qcom,msm8960-sps-sic
               - rockchip,px30-qos
               - rockchip,rk3036-qos
               - rockchip,rk3066-qos
diff --git a/Documentation/devicetree/bindings/mfd/ti,keystone-devctrl.yaml b/Documentation/devicetree/bindings/mfd/ti,keystone-devctrl.yaml
new file mode 100644
index 00000000000000..564dfc8a731f27
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/ti,keystone-devctrl.yaml
@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/ti,keystone-devctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments Keystone Device State Control
+
+maintainers:
+  - Ivan Khoronzhuk <ivan.khoronzhuk@ti.com>
+
+description:
+  The Keystone II devices have a set of registers that are used to control the
+  status of its peripherals. This node is intended to allow access to this
+  functionality.
+
+properties:
+  compatible:
+    items:
+      - const: ti,keystone-devctrl
+      - const: syscon
+      - const: simple-mfd
+
+  reg:
+    maxItems: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  ranges: true
+
+patternProperties:
+  "^gpio@[0-9a-f]+$":
+    type: object
+    description: Keystone DSP GPIO controller
+    required:
+      - compatible
+
+  "^interrupt-controller@[0-9a-f]+$":
+    type: object
+    description: Keystone DSP to ARM interrupt controller
+    required:
+      - compatible
+
+  "^reset-controller@[0-9a-f]+$":
+    type: object
+    description: Keystone reset controller
+    required:
+      - compatible
+
+required:
+  - compatible
+  - reg
+  - "#address-cells"
+  - "#size-cells"
+  - ranges
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    device-state-control@2620000 {
+        compatible = "ti,keystone-devctrl", "syscon", "simple-mfd";
+        reg = <0x02620000 0x1000>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges = <0x0 0x02620000 0x1000>;
+
+        interrupt-controller@2a0 {
+            compatible = "ti,keystone-irq";
+            reg = <0x2a0 0x4>;
+            interrupts = <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>;
+            interrupt-controller;
+            #interrupt-cells = <1>;
+            ti,syscon-dev = <&devctrl 0x2a0>;
+        };
+
+        reset-controller@328 {
+            compatible = "ti,keystone-reset";
+            reg = <0x328 0x10>;
+            ti,syscon-pll = <&pllctrl 0xe4>;
+            ti,syscon-dev = <&devctrl 0x328>;
+            ti,wdt-list = <0>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mfd/ti,tps61050.yaml b/Documentation/devicetree/bindings/mfd/ti,tps61050.yaml
new file mode 100644
index 00000000000000..974ef78d6e26ea
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/ti,tps61050.yaml
@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/ti,tps61050.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments TPS61050/TPS61052 Boost Converter
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+description:
+  The TP61050/TPS61052 is a high-power "white LED driver". The device
+  provides LED, GPIO and regulator functionalities.
+
+properties:
+  compatible:
+    enum:
+      - ti,tps61050
+      - ti,tps61052
+
+  reg:
+    maxItems: 1
+
+  regulator:
+    type: object
+    $ref: /schemas/regulator/regulator.yaml#
+    description: Puts the chip in regulator mode.
+    unevaluatedProperties: false
+
+  led:
+    type: object
+    $ref: /schemas/leds/common.yaml#
+    description: Puts the chip in LED mode.
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+
+allOf:
+  - not:
+      required:
+        - regulator
+        - led
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        tps61052@33 {
+            compatible = "ti,tps61052";
+            reg = <0x33>;
+        };
+    };
+
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        tps61052@33 {
+            compatible = "ti,tps61052";
+            reg = <0x33>;
+
+            regulator {
+                regulator-min-microvolt = <5000000>;
+                regulator-max-microvolt = <5000000>;
+                regulator-always-on;
+            };
+        };
+    };
+
+  - |
+    #include <dt-bindings/leds/common.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        tps61052@33 {
+            compatible = "ti,tps61052";
+            reg = <0x33>;
+
+            led {
+                color = <LED_COLOR_ID_WHITE>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mfd/ti,tps65910.yaml b/Documentation/devicetree/bindings/mfd/ti,tps65910.yaml
index f1a76f88fc0cdd..baf95acd8a70ef 100644
--- a/Documentation/devicetree/bindings/mfd/ti,tps65910.yaml
+++ b/Documentation/devicetree/bindings/mfd/ti,tps65910.yaml
@@ -37,6 +37,8 @@ properties:
 
   interrupt-controller: true
 
+  wakeup-source: true
+
   '#interrupt-cells':
     description: Specifies the IRQ number and flags
     const: 2
@@ -163,6 +165,12 @@ patternProperties:
         vcc7-supply: VRTC input.
         vccio-supply: VIO input.
 
+  "^.+-hog$":
+    type: object
+
+    required:
+      - gpio-hog
+
 required:
   - compatible
   - reg
@@ -208,7 +216,7 @@ examples:
             compatible = "ti,tps65910";
             reg = <0x2d>;
             interrupt-parent = <&intc>;
-            interrupts = < 0 118 0x04 >;
+            interrupts = <0 118 0x04>;
 
             #gpio-cells = <2>;
             gpio-controller;
diff --git a/Documentation/devicetree/bindings/mfd/ti,twl6040.yaml b/Documentation/devicetree/bindings/mfd/ti,twl6040.yaml
new file mode 100644
index 00000000000000..5efcd79b527c3c
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/ti,twl6040.yaml
@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/ti,twl6040.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments TWL6040 family audio codec
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+description:
+  The TWL6040s are 8-channel high quality low-power audio codecs providing
+  audio, vibra and GPO functionality on OMAP4+ platforms. They are connected
+  to the host processor via i2c for commands, McPDM for audio data and
+  commands.
+
+properties:
+  compatible:
+    enum:
+      - ti,twl6040
+      - ti,twl6041
+
+  reg:
+    const: 0x4b
+
+  interrupts:
+    maxItems: 1
+
+  gpio-controller: true
+
+  "#gpio-cells":
+    const: 1
+
+  "#clock-cells":
+    const: 0
+
+  ti,audpwron-gpio:
+    maxItems: 1
+    description: Power on GPIO line for the twl6040
+
+  vio-supply:
+    description: Regulator for the twl6040 VIO supply
+
+  v2v1-supply:
+    description: Regulator for the twl6040 V2V1 supply
+
+  enable-active-high:
+    type: boolean
+    description: To power on the twl6040 during boot.
+
+  clocks:
+    minItems: 1
+    maxItems: 2
+
+  clock-names:
+    minItems: 1
+    maxItems: 2
+    items:
+      enum:
+        - clk32k
+        - mclk
+
+  vddvibl-supply:
+    description: Regulator for the left vibra motor
+
+  vddvibr-supply:
+    description: Regulator for the right vibra motor
+
+  vibra:
+    type: object
+    additionalProperties: false
+    properties:
+      ti,vibldrv-res:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: Resistance parameter for left driver
+
+      ti,vibrdrv-res:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: Resistance parameter for right driver
+
+      ti,viblmotor-res:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: Resistance parameter for left motor
+
+      ti,vibrmotor-res:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: Resistance parameter for right motor
+
+      ti,vddvibl-uV:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: vddvibl default voltage if it needs to be changed
+
+      ti,vddvibr-uV:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: vddvibr default voltage if it needs to be changed
+
+    required:
+      - ti,vibldrv-res
+      - ti,vibrdrv-res
+      - ti,viblmotor-res
+      - ti,vibrmotor-res
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - "#clock-cells"
+  - vio-supply
+  - v2v1-supply
+
+allOf:
+  - if:
+      required:
+        - vibra
+    then:
+      required:
+        - vddvibl-supply
+        - vddvibr-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        audio-codec@4b {
+            compatible = "ti,twl6040";
+            reg = <0x4b>;
+            interrupts = <0 119 IRQ_TYPE_LEVEL_HIGH>;
+            interrupt-parent = <&gic>;
+            gpio-controller;
+            #gpio-cells = <1>;
+            #clock-cells = <0>;
+            ti,audpwron-gpio = <&gpio4 31 0>;
+
+            vio-supply = <&v1v8>;
+            v2v1-supply = <&v2v1>;
+            enable-active-high;
+
+            /* regulators for vibra motor */
+            vddvibl-supply = <&vbat>;
+            vddvibr-supply = <&vbat>;
+
+            vibra {
+                ti,vibldrv-res = <8>;
+                ti,vibrdrv-res = <3>;
+                ti,viblmotor-res = <10>;
+                ti,vibrmotor-res = <10>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mfd/ti-keystone-devctrl.txt b/Documentation/devicetree/bindings/mfd/ti-keystone-devctrl.txt
deleted file mode 100644
index 71a1f5963936b8..00000000000000
--- a/Documentation/devicetree/bindings/mfd/ti-keystone-devctrl.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-* Device tree bindings for Texas Instruments keystone device state control
-
-The Keystone II devices have a set of registers that are used to control
-the status of its peripherals. This node is intended to allow access to
-this functionality.
-
-Required properties:
-
-- compatible:		"ti,keystone-devctrl", "syscon"
-
-- reg:			contains offset/length value for device state control
-			registers space.
-
-Example:
-
-devctrl: device-state-control@02620000 {
-	compatible = "ti,keystone-devctrl", "syscon";
-	reg = <0x02620000 0x1000>;
-};
diff --git a/Documentation/devicetree/bindings/mfd/tps6105x.txt b/Documentation/devicetree/bindings/mfd/tps6105x.txt
deleted file mode 100644
index dc448a9d5b4d3a..00000000000000
--- a/Documentation/devicetree/bindings/mfd/tps6105x.txt
+++ /dev/null
@@ -1,62 +0,0 @@
-* Device tree bindings for TI TPS61050/61052 Boost Converters
-
-The TP61050/TPS61052 is a high-power "white LED driver". The
-device provides LED, GPIO and regulator functionalities.
-
-Required properties:
-- compatible:		"ti,tps61050" or "ti,tps61052"
-- reg:			Specifies the I2C slave address
-
-Optional sub-node:
-
-This subnode selects the chip's operational mode.
-There can be at most one single available subnode.
-
-- regulator: presence of this sub-node puts the chip in regulator mode.
-	see ../regulator/regulator.yaml
-
-- led: presence of this sub-node puts the chip in led mode.
-	Optional properties:
-	- function : see ../leds/common.txt
-	- color    : see ../leds/common.txt
-	- label    : see ../leds/common.txt
-			(deprecated)
-
-Example (GPIO operation only):
-
-i2c0 {
-	tps61052@33 {
-		compatible = "ti,tps61052";
-		reg = <0x33>;
-	};
-};
-
-Example (GPIO + regulator operation):
-
-i2c0 {
-	tps61052@33 {
-		compatible = "ti,tps61052";
-		reg = <0x33>;
-
-		regulator {
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-			regulator-always-on;
-		};
-	};
-};
-
-Example (GPIO + led operation):
-
-#include <dt-bindings/leds/common.h>
-
-i2c0 {
-	tps61052@33 {
-		compatible = "ti,tps61052";
-		reg = <0x33>;
-
-		led {
-			color = <LED_COLOR_ID_WHITE>;
-		};
-	};
-};
diff --git a/Documentation/devicetree/bindings/mfd/twl6040.txt b/Documentation/devicetree/bindings/mfd/twl6040.txt
deleted file mode 100644
index dfd8683ede0cf9..00000000000000
--- a/Documentation/devicetree/bindings/mfd/twl6040.txt
+++ /dev/null
@@ -1,67 +0,0 @@
-Texas Instruments TWL6040 family
-
-The TWL6040s are 8-channel high quality low-power audio codecs providing audio,
-vibra and GPO functionality on OMAP4+ platforms.
-They are connected to the host processor via i2c for commands, McPDM for audio
-data and commands.
-
-Required properties:
-- compatible : "ti,twl6040" for twl6040, "ti,twl6041" for twl6041
-- reg: must be 0x4b for i2c address
-- interrupts: twl6040 has one interrupt line connecteded to the main SoC
-- gpio-controller:
-- #gpio-cells = <1>: twl6040 provides GPO lines.
-- #clock-cells = <0>; twl6040 is a provider of pdmclk which is used by McPDM
-- twl6040,audpwron-gpio: Power on GPIO line for the twl6040
-
-- vio-supply: Regulator for the twl6040 VIO supply
-- v2v1-supply: Regulator for the twl6040 V2V1 supply
-
-Optional properties, nodes:
-- enable-active-high: To power on the twl6040 during boot.
-- clocks: phandle to the clk32k and/or to mclk clock provider
-- clock-names: Must be "clk32k" for the 32K clock and "mclk" for the MCLK.
-
-Vibra functionality
-Required properties:
-- vddvibl-supply: Regulator for the left vibra motor
-- vddvibr-supply: Regulator for the right vibra motor
-- vibra { }: Configuration section for vibra parameters containing the following
-	     properties:
-- ti,vibldrv-res: Resistance parameter for left driver
-- ti,vibrdrv-res: Resistance parameter for right driver
-- ti,viblmotor-res: Resistance parameter for left motor
-- ti,viblmotor-res: Resistance parameter for right motor
-
-Optional properties within vibra { } section:
-- vddvibl_uV: If the vddvibl default voltage need to be changed
-- vddvibr_uV: If the vddvibr default voltage need to be changed
-
-Example:
-&i2c1 {
-	twl6040: twl@4b {
-		compatible = "ti,twl6040";
-
-		interrupts = <0 119 4>;
-		interrupt-parent = <&gic>;
-		twl6040,audpwron-gpio = <&gpio4 31 0>;
-
-		vio-supply = <&v1v8>;
-		v2v1-supply = <&v2v1>;
-		enable-active-high;
-
-		/* regulators for vibra motor */
-		vddvibl-supply = <&vbat>;
-		vddvibr-supply = <&vbat>;
-
-		vibra {
-			/* Vibra driver, motor resistance parameters */
-			ti,vibldrv-res = <8>;
-			ti,vibrdrv-res = <3>;
-			ti,viblmotor-res = <10>;
-			ti,vibrmotor-res = <10>;
-		};
-	};
-};
-
-/include/ "twl6040.dtsi"
diff --git a/Documentation/devicetree/bindings/mmc/amlogic,meson-gx-mmc.yaml b/Documentation/devicetree/bindings/mmc/amlogic,meson-gx-mmc.yaml
index 976f36de2091c2..12342b1d78e569 100644
--- a/Documentation/devicetree/bindings/mmc/amlogic,meson-gx-mmc.yaml
+++ b/Documentation/devicetree/bindings/mmc/amlogic,meson-gx-mmc.yaml
@@ -15,6 +15,23 @@ maintainers:
 
 allOf:
   - $ref: mmc-controller.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: amlogic,t7-mmc
+    then:
+      properties:
+        clocks:
+          minItems: 4
+        clock-names:
+          minItems: 4
+    else:
+      properties:
+        clocks:
+          maxItems: 3
+        clock-names:
+          maxItems: 3
 
 properties:
   compatible:
@@ -38,13 +55,23 @@ properties:
       - description: card detect
 
   clocks:
-    maxItems: 3
+    minItems: 3
+    items:
+      - description: core clock
+      - description: clock input 0 of the internal mux
+      - description: clock input 1 of the internal mux
+      - description:
+          clock of the pipeline stage inserted in the bus path between
+          the controller and the DRAM. Without it, the controller cannot
+          complete DMA transfers.
 
   clock-names:
+    minItems: 3
     items:
       - const: core
       - const: clkin0
       - const: clkin1
+      - const: pipeline
 
   resets:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/mmc/cdns,sd6hc.yaml b/Documentation/devicetree/bindings/mmc/cdns,sd6hc.yaml
new file mode 100644
index 00000000000000..d5ea2717904b41
--- /dev/null
+++ b/Documentation/devicetree/bindings/mmc/cdns,sd6hc.yaml
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mmc/cdns,sd6hc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Cadence SD6HC SD/SDIO/eMMC Host Controller
+
+maintainers:
+  - Tanmay Kathpalia <tanmay.kathpalia@altera.com>
+
+description:
+  Cadence SD6HC is a sixth-generation SD/SDIO/eMMC host controller with an
+  interface to a Cadence Combo PHY. The controller has separate card interface
+  and bus interface clock domains.
+
+properties:
+  compatible:
+    items:
+      - const: altr,agilex5-sd6hc
+      - const: cdns,sd6hc
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: card interface unit (CIU / SDMCLK) clock
+      - description: bus interface unit (BIU) clock
+
+  clock-names:
+    items:
+      - const: ciu
+      - const: biu
+
+  resets:
+    items:
+      - description: SDHC controller reset
+      - description: Combo PHY reset
+      - description: SDMMC OCP/AXI reset
+
+  reset-names:
+    items:
+      - const: sdhc-reset
+      - const: combophy
+      - const: sdmmc-ocp
+
+  iommus:
+    maxItems: 1
+
+  dma-coherent: true
+
+  cdns,iocell-input-delay-ps:
+    description:
+      Input delay across the IO cells, used for SD6HC PHY timing calculations.
+    minimum: 0
+    maximum: 20000
+    default: 2500
+
+  cdns,iocell-output-delay-ps:
+    description:
+      Output delay across the IO cells, used for SD6HC PHY timing calculations.
+    minimum: 0
+    maximum: 20000
+    default: 2500
+
+  cdns,delay-element-ps:
+    description:
+      DLL delay element size, used for SD6HC PHY timing calculations.
+    minimum: 1
+    maximum: 1000
+    default: 24
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - resets
+  - reset-names
+
+allOf:
+  - $ref: sdhci-common.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    mmc@10808000 {
+        compatible = "altr,agilex5-sd6hc", "cdns,sd6hc";
+        reg = <0x10808000 0x1000>;
+        interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clkmgr 0>, <&clkmgr 1>;
+        clock-names = "ciu", "biu";
+        resets = <&rst 0>, <&rst 1>, <&rst 2>;
+        reset-names = "sdhc-reset", "combophy", "sdmmc-ocp";
+        iommus = <&smmu 5>;
+        dma-coherent;
+    };
diff --git a/Documentation/devicetree/bindings/mmc/mmc-controller-common.yaml b/Documentation/devicetree/bindings/mmc/mmc-controller-common.yaml
index 3d7195e9461c30..8b9b3b0361296f 100644
--- a/Documentation/devicetree/bindings/mmc/mmc-controller-common.yaml
+++ b/Documentation/devicetree/bindings/mmc/mmc-controller-common.yaml
@@ -275,6 +275,16 @@ properties:
       not available.
     default: 10
 
+  power-off-delay-us:
+    description:
+      Delay in microseconds after card power is deasserted during a power
+      cycle to allow time for proper power discharge. Larger values can be
+      configured to work around hardware issues such as slow RC discharge
+      on MMC VDD rails.
+    minimum: 0
+    maximum: 10000000
+    default: 1000
+
   supports-cqe:
     $ref: /schemas/types.yaml#/definitions/flag
     description:
diff --git a/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml b/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml
index b3fcc1673c1069..cc43a3ea1d41e6 100644
--- a/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml
+++ b/Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml
@@ -26,6 +26,7 @@ properties:
               - qcom,ipq4019-sdhci
               - qcom,ipq8074-sdhci
               - qcom,msm8226-sdhci
+              - qcom,msm8952-sdhci
               - qcom,msm8953-sdhci
               - qcom,msm8974-sdhci
               - qcom,msm8976-sdhci
@@ -38,6 +39,7 @@ properties:
       - items:
           - enum:
               - qcom,eliza-sdhci
+              - qcom,glymur-sdhci
               - qcom,hawi-sdhci
               - qcom,ipq5018-sdhci
               - qcom,ipq5210-sdhci
@@ -47,6 +49,7 @@ properties:
               - qcom,ipq9574-sdhci
               - qcom,ipq9650-sdhci
               - qcom,kaanapali-sdhci
+              - qcom,maili-sdhci
               - qcom,milos-sdhci
               - qcom,qcm2290-sdhci
               - qcom,qcs404-sdhci
diff --git a/Documentation/devicetree/bindings/mux/reg-mux.yaml b/Documentation/devicetree/bindings/mux/reg-mux.yaml
index dc4be092fc2fcf..c8de11a90d9065 100644
--- a/Documentation/devicetree/bindings/mux/reg-mux.yaml
+++ b/Documentation/devicetree/bindings/mux/reg-mux.yaml
@@ -22,7 +22,7 @@ properties:
   reg: true
 
   '#mux-control-cells':
-    const: 1
+    enum: [ 0, 1 ]
 
   mux-reg-masks:
     $ref: /schemas/types.yaml#/definitions/uint32-matrix
diff --git a/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml b/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml
index 323a669fa98236..2cae0133b3cd4c 100644
--- a/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml
+++ b/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml
@@ -21,6 +21,7 @@ select:
           - allwinner,sun8i-r40-gmac
           - allwinner,sun8i-v3s-emac
           - allwinner,sun50i-a64-emac
+          - allwinner,sun50i-h616-emac1
           - allwinner,sun55i-a523-gmac200
   required:
     - compatible
@@ -33,6 +34,7 @@ properties:
       - const: allwinner,sun8i-r40-gmac
       - const: allwinner,sun8i-v3s-emac
       - const: allwinner,sun50i-a64-emac
+      - const: allwinner,sun50i-h616-emac1
       - items:
           - enum:
               - allwinner,sun20i-d1-emac
@@ -91,6 +93,17 @@ required:
 
 allOf:
   - $ref: snps,dwmac.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: allwinner,sun50i-h616-emac1
+
+    then:
+      properties:
+        phy-mode:
+          const: rmii
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml b/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml
index db93436c96fa6a..d8f90d36d24698 100644
--- a/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml
+++ b/Documentation/devicetree/bindings/net/altr,socfpga-stmmac.yaml
@@ -13,8 +13,6 @@ description:
   This binding describes the Altera SOCFPGA SoC implementation of the
   Synopsys DWMAC for the Cyclone5, Arria5, Stratix10, Agilex5 and Agilex7
   families of chips.
-  # TODO: Determine how to handle the Arria10 reset-name, stmmaceth-ocp, that
-  # does not validate against net/snps,dwmac.yaml.
 
 select:
   properties:
diff --git a/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml b/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml
index 95501e858e6f82..612d21123aaad3 100644
--- a/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml
+++ b/Documentation/devicetree/bindings/net/bluetooth/brcm,bluetooth.yaml
@@ -31,6 +31,7 @@ properties:
           - brcm,bcm43540-bt
           - brcm,bcm4335a0
           - brcm,bcm4349-bt
+          - brcm,bcm4384-bt
           - cypress,cyw4373a0-bt
           - infineon,cyw55572-bt
 
diff --git a/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6750-bt.yaml b/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6750-bt.yaml
index 8606a45ac9b978..79522409d70915 100644
--- a/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6750-bt.yaml
+++ b/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6750-bt.yaml
@@ -12,8 +12,14 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - qcom,wcn6750-bt
+    oneOf:
+      - items:
+          - enum:
+              - qcom,wcn6755-bt
+          - const: qcom,wcn6750-bt
+
+      - enum:
+          - qcom,wcn6750-bt
 
   enable-gpios:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml b/Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml
index a3db6d594c8c9c..7fcc564e820ceb 100644
--- a/Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml
+++ b/Documentation/devicetree/bindings/net/brcm,asp-v2.0.yaml
@@ -123,7 +123,7 @@ examples:
             phy0: ethernet-phy@1 {
                 reg = <1>;
             };
-       };
+        };
 
         mdio@ce14 {
             compatible = "brcm,asp-v2.1-mdio";
diff --git a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
index f0b5bea2458d59..da9bc170cd543e 100644
--- a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
+++ b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
@@ -87,18 +87,6 @@ examples:
     #include <dt-bindings/interrupt-controller/irq.h>
     #include <dt-bindings/clock/cortina,gemini-clock.h>
     #include <dt-bindings/reset/cortina,gemini-reset.h>
-    mdio0: mdio {
-      #address-cells = <1>;
-      #size-cells = <0>;
-      phy0: ethernet-phy@1 {
-        reg = <1>;
-        device_type = "ethernet-phy";
-      };
-      phy1: ethernet-phy@3 {
-        reg = <3>;
-        device_type = "ethernet-phy";
-      };
-    };
 
     ethernet@60000000 {
         compatible = "cortina,gemini-ethernet";
@@ -110,28 +98,28 @@ examples:
         ranges;
 
         gmac0: ethernet-port@0 {
-                compatible = "cortina,gemini-ethernet-port";
-                reg = <0x60008000 0x2000>, /* Port 0 DMA/TOE */
-                      <0x6000a000 0x2000>; /* Port 0 GMAC */
-                interrupt-parent = <&intcon>;
-                interrupts = <1 IRQ_TYPE_LEVEL_HIGH>;
-                resets = <&syscon GEMINI_RESET_GMAC0>;
-                clocks = <&syscon GEMINI_CLK_GATE_GMAC0>;
-                clock-names = "PCLK";
-                phy-mode = "rgmii";
-                phy-handle = <&phy0>;
+            compatible = "cortina,gemini-ethernet-port";
+            reg = <0x60008000 0x2000>, /* Port 0 DMA/TOE */
+                  <0x6000a000 0x2000>; /* Port 0 GMAC */
+            interrupt-parent = <&intcon>;
+            interrupts = <1 IRQ_TYPE_LEVEL_HIGH>;
+            resets = <&syscon GEMINI_RESET_GMAC0>;
+            clocks = <&syscon GEMINI_CLK_GATE_GMAC0>;
+            clock-names = "PCLK";
+            phy-mode = "rgmii";
+            phy-handle = <&phy0>;
         };
 
         gmac1: ethernet-port@1 {
-                compatible = "cortina,gemini-ethernet-port";
-                reg = <0x6000c000 0x2000>, /* Port 1 DMA/TOE */
-                      <0x6000e000 0x2000>; /* Port 1 GMAC */
-                interrupt-parent = <&intcon>;
-                interrupts = <2 IRQ_TYPE_LEVEL_HIGH>;
-                resets = <&syscon GEMINI_RESET_GMAC1>;
-                clocks = <&syscon GEMINI_CLK_GATE_GMAC1>;
-                clock-names = "PCLK";
-                phy-mode = "rgmii";
-                phy-handle = <&phy1>;
+            compatible = "cortina,gemini-ethernet-port";
+            reg = <0x6000c000 0x2000>, /* Port 1 DMA/TOE */
+                  <0x6000e000 0x2000>; /* Port 1 GMAC */
+            interrupt-parent = <&intcon>;
+            interrupts = <2 IRQ_TYPE_LEVEL_HIGH>;
+            resets = <&syscon GEMINI_RESET_GMAC1>;
+            clocks = <&syscon GEMINI_CLK_GATE_GMAC1>;
+            clock-names = "PCLK";
+            phy-mode = "rgmii";
+            phy-handle = <&phy1>;
         };
     };
diff --git a/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml b/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml
index 8d4a3a9a33fcc1..4ed13870ed3ac0 100644
--- a/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml
@@ -23,6 +23,7 @@ properties:
       - microchip,ksz8864  # 4-port version of KSZ8895 family switch
       - microchip,ksz8873
       - microchip,ksz8895  # 5-port version of KSZ8895 family switch
+      - microchip,ksz8995xa
       - microchip,ksz9477
       - microchip,ksz9897
       - microchip,ksz9896
diff --git a/Documentation/devicetree/bindings/net/dsa/motorcomm,yt921x.yaml b/Documentation/devicetree/bindings/net/dsa/motorcomm,yt921x.yaml
index 33a6552e46fc0d..93ba5b181205b9 100644
--- a/Documentation/devicetree/bindings/net/dsa/motorcomm,yt921x.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/motorcomm,yt921x.yaml
@@ -21,6 +21,9 @@ description: |
 
   Any port can be used as the CPU port.
 
+  Each port has at most 3 LEDs connected and can be declared using the standard
+  LEDs structure.
+
 properties:
   compatible:
     const: motorcomm,yt9215
@@ -59,6 +62,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/leds/common.h>
 
     mdio {
         #address-cells = <1>;
@@ -113,6 +117,25 @@ examples:
                     label = "lan1";
                     phy-mode = "internal";
                     phy-handle = <&sw_phy0>;
+
+                    leds {
+                        #address-cells = <1>;
+                        #size-cells = <0>;
+
+                        led@0 {
+                            reg = <0>;
+                            color = <LED_COLOR_ID_WHITE>;
+                            function = LED_FUNCTION_LAN;
+                            default-state = "keep";
+                        };
+
+                        led@1 {
+                            reg = <1>;
+                            color = <LED_COLOR_ID_AMBER>;
+                            function = LED_FUNCTION_LAN;
+                            default-state = "keep";
+                        };
+                    };
                 };
 
                 ethernet-port@1 {
diff --git a/Documentation/devicetree/bindings/net/dsa/realtek.yaml b/Documentation/devicetree/bindings/net/dsa/realtek.yaml
index 473facd87a622f..b7bb60af623ca6 100644
--- a/Documentation/devicetree/bindings/net/dsa/realtek.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/realtek.yaml
@@ -8,6 +8,20 @@ title: Realtek switches for unmanaged switches
 
 allOf:
   - $ref: dsa.yaml#/$defs/ethernet-ports
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              const: realtek,rtl8365mb
+    then:
+      properties:
+        avddh-supply: false
+        avddl-supply: false
+        dvddio-supply: false
+        dvddio1-supply: false
+        dvddl-supply: false
+        pllvddl-supply: false
 
 maintainers:
   - Linus Walleij <linusw@kernel.org>
@@ -62,6 +76,24 @@ properties:
   resets:
     maxItems: 1
 
+  avddh-supply:
+    description: AVDDH high-voltage analog supply.
+
+  avddl-supply:
+    description: AVDDL low-voltage analog supply.
+
+  dvddio-supply:
+    description: DVDDIO digital I/O supply.
+
+  dvddio1-supply:
+    description: DVDDIO_1 secondary digital I/O supply for the RGMII bank.
+
+  dvddl-supply:
+    description: DVDDL digital core supply.
+
+  pllvddl-supply:
+    description: PLLVDDL PLL analog supply.
+
   realtek,disable-leds:
     type: boolean
     description: |
@@ -243,6 +275,7 @@ examples:
     platform {
             ethernet-switch {
                     compatible = "realtek,rtl8365mb";
+                    avddh-supply = <&avddh>;
                     mdc-gpios = <&gpio1 16 GPIO_ACTIVE_HIGH>;
                     mdio-gpios = <&gpio1 17 GPIO_ACTIVE_HIGH>;
                     reset-gpios = <&gpio5 0 GPIO_ACTIVE_LOW>;
diff --git a/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml b/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml
index ea285ef3e64f9b..fab6aa6237e93a 100644
--- a/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml
@@ -127,8 +127,8 @@ examples:
                 phy-mode = "internal";
 
                 fixed-link {
-                  speed = <1000>;
-                  full-duplex;
+                    speed = <1000>;
+                    full-duplex;
                 };
             };
         };
diff --git a/Documentation/devicetree/bindings/net/ethernet-phy.yaml b/Documentation/devicetree/bindings/net/ethernet-phy.yaml
index c3ebb3af8b528b..df50c4c33d76e9 100644
--- a/Documentation/devicetree/bindings/net/ethernet-phy.yaml
+++ b/Documentation/devicetree/bindings/net/ethernet-phy.yaml
@@ -269,6 +269,22 @@ properties:
       have to be placed on board. When omitted, the PHYs default will be left as
       is.
 
+  marvell,reg-init:
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    deprecated: true
+    description:
+      Register initialization tuples for Marvell PHYs.
+    items:
+      items:
+        - description: register page
+          maximum: 0xffff
+        - description: register number
+          maximum: 31
+        - description: mask of bits to preserve
+          maximum: 0xffff
+        - description: value to apply
+          maximum: 0xffff
+
   leds:
     type: object
 
diff --git a/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml b/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml
index ef1e30a48c910a..5dcb49f2f3b1f2 100644
--- a/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml
+++ b/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml
@@ -125,10 +125,6 @@ required:
   - reg
   - fsl,fman-ports
 
-dependencies:
-  pcs-handle-names:
-    - pcs-handle
-
 allOf:
   - $ref: ethernet-controller.yaml#
   - if:
diff --git a/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml b/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml
index 62c1da36a2b5a2..d1522216d32b94 100644
--- a/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml
+++ b/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml
@@ -91,9 +91,9 @@ examples:
         snps,tx-queues-to-use = <2>;
         snps,tx-sched-wrr;
         queue0 {
-           snps,weight = <0x10>;
-           snps,dcb-algorithm;
-           snps,priority = <0x0>;
+            snps,weight = <0x10>;
+            snps,dcb-algorithm;
+            snps,priority = <0x0>;
         };
 
         queue1 {
diff --git a/Documentation/devicetree/bindings/net/mdio-gpio.yaml b/Documentation/devicetree/bindings/net/mdio-gpio.yaml
index eb4171a1940e47..36e51aaae9c0b1 100644
--- a/Documentation/devicetree/bindings/net/mdio-gpio.yaml
+++ b/Documentation/devicetree/bindings/net/mdio-gpio.yaml
@@ -45,13 +45,13 @@ examples:
     };
 
     mdio0: mdio {
-      compatible = "virtual,mdio-gpio";
-      #address-cells = <1>;
-      #size-cells = <0>;
-      gpios = <&qe_pio_a 11>,
-              <&qe_pio_c 6>;
-      ethphy0: ethernet-phy@0 {
-        reg = <0>;
-      };
+        compatible = "virtual,mdio-gpio";
+        #address-cells = <1>;
+        #size-cells = <0>;
+        gpios = <&qe_pio_a 11>,
+                <&qe_pio_c 6>;
+        ethphy0: ethernet-phy@0 {
+            reg = <0>;
+        };
     };
 ...
diff --git a/Documentation/devicetree/bindings/net/microchip,lan8650.yaml b/Documentation/devicetree/bindings/net/microchip,lan8650.yaml
index 766ff58147ae36..1b543ce3076322 100644
--- a/Documentation/devicetree/bindings/net/microchip,lan8650.yaml
+++ b/Documentation/devicetree/bindings/net/microchip,lan8650.yaml
@@ -40,6 +40,11 @@ properties:
       Event.
     maxItems: 1
 
+  reset-gpios:
+    description:
+      GPIO connected to the active-low RESET_N pin of the MAC-PHY.
+    maxItems: 1
+
   spi-max-frequency:
     minimum: 15000000
     maximum: 25000000
diff --git a/Documentation/devicetree/bindings/net/mscc,vsc7514-switch.yaml b/Documentation/devicetree/bindings/net/mscc,vsc7514-switch.yaml
index 86a9c3fc76c890..d825130b54b403 100644
--- a/Documentation/devicetree/bindings/net/mscc,vsc7514-switch.yaml
+++ b/Documentation/devicetree/bindings/net/mscc,vsc7514-switch.yaml
@@ -197,10 +197,10 @@ examples:
             <0x71040000 0x10000>,
             <0x71050000 0x10000>,
             <0x71060000 0x10000>;
-            reg-names = "sys", "rew", "qs", "ptp", "port0", "port1",
-            "port2", "port3", "port4", "port5", "port6",
-            "port7", "port8", "port9", "port10", "qsys",
-            "ana", "s0", "s1", "s2";
+      reg-names = "sys", "rew", "qs", "ptp", "port0", "port1",
+                  "port2", "port3", "port4", "port5", "port6",
+                  "port7", "port8", "port9", "port10", "qsys",
+                  "ana", "s0", "s1", "s2";
 
       ethernet-ports {
         #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml
index 215f14d1897d21..48e4798a039b02 100644
--- a/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml
+++ b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml
@@ -81,8 +81,9 @@ properties:
   phy-mode:
     contains:
       enum:
-        - usxgmii
         - 10gbase-kr
+        - 10gbase-r
+        - usxgmii
 
   mdio:
     $ref: mdio.yaml#
@@ -90,6 +91,9 @@ properties:
     description:
       Optional node for embedded MDIO controller.
 
+allOf:
+  - $ref: snps,dwmac-common.yaml#
+
 required:
   - compatible
   - reg
@@ -103,7 +107,7 @@ required:
   - phy-handle
   - phy-mode
 
-additionalProperties: false
+unevaluatedProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/net/qcom,ipa.yaml b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
index 68ec76fe447356..38e3830512feb3 100644
--- a/Documentation/devicetree/bindings/net/qcom,ipa.yaml
+++ b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
@@ -246,14 +246,14 @@ examples:
         qcom,remote-pid = <5>;
 
         ipa_smp2p_out: ipa-ap-to-modem {
-                qcom,entry-name = "ipa";
-                #qcom,smem-state-cells = <1>;
+            qcom,entry-name = "ipa";
+            #qcom,smem-state-cells = <1>;
         };
 
         ipa_smp2p_in: ipa-modem-to-ap {
-                qcom,entry-name = "ipa";
-                interrupt-controller;
-                #interrupt-cells = <2>;
+            qcom,entry-name = "ipa";
+            interrupt-controller;
+            #interrupt-cells = <2>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/net/realtek,rtl82xx.yaml b/Documentation/devicetree/bindings/net/realtek,rtl82xx.yaml
index 45033c31a2d51f..07b39ef1fc6244 100644
--- a/Documentation/devicetree/bindings/net/realtek,rtl82xx.yaml
+++ b/Documentation/devicetree/bindings/net/realtek,rtl82xx.yaml
@@ -91,8 +91,8 @@ examples:
         #size-cells = <0>;
 
         ethphy1: ethernet-phy@1 {
-                reg = <1>;
-                realtek,clkout-disable;
-                realtek,aldps-enable;
+            reg = <1>;
+            realtek,clkout-disable;
+            realtek,aldps-enable;
         };
     };
diff --git a/Documentation/devicetree/bindings/net/realtek,rtl9301-mdio.yaml b/Documentation/devicetree/bindings/net/realtek,rtl9301-mdio.yaml
index 271e05bae9c574..67e0b23a8470ea 100644
--- a/Documentation/devicetree/bindings/net/realtek,rtl9301-mdio.yaml
+++ b/Documentation/devicetree/bindings/net/realtek,rtl9301-mdio.yaml
@@ -4,7 +4,7 @@
 $id: http://devicetree.org/schemas/net/realtek,rtl9301-mdio.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Realtek RTL9300 MDIO Controller
+title: Realtek Otto Switches Ethernet MDIO Controller
 
 maintainers:
   - Chris Packham <chris.packham@alliedtelesis.co.nz>
@@ -12,6 +12,16 @@ maintainers:
 properties:
   compatible:
     oneOf:
+      - items:
+          - enum:
+              - realtek,rtl8381-mdio
+              - realtek,rtl8382-mdio
+          - const: realtek,rtl8380-mdio
+      - items:
+          - enum:
+              - realtek,rtl8392-mdio
+              - realtek,rtl8393-mdio
+          - const: realtek,rtl8391-mdio
       - items:
           - enum:
               - realtek,rtl9302b-mdio
@@ -24,6 +34,8 @@ properties:
               - realtek,rtl9313-mdio
           - const: realtek,rtl9311-mdio
       - enum:
+          - realtek,rtl8380-mdio
+          - realtek,rtl8391-mdio
           - realtek,rtl9301-mdio
           - realtek,rtl9311-mdio
 
diff --git a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
index 1e00ef5b3acda5..fad002492421a5 100644
--- a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
+++ b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
@@ -97,6 +97,8 @@ properties:
 
   phy-handle: true
 
+  ptp-timer: true
+
   '#address-cells':
     description: Number of address cells for the MDIO bus.
     const: 1
@@ -305,6 +307,15 @@ allOf:
             items:
               - const: fck
               - const: refclk
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              const: renesas,etheravb-rcar-gen4
+    then:
+      properties:
+        ptp-timer: false
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/net/snps,dwmac-common.yaml b/Documentation/devicetree/bindings/net/snps,dwmac-common.yaml
new file mode 100644
index 00000000000000..a38ff314b3cd1f
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/snps,dwmac-common.yaml
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/snps,dwmac-common.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Synopsys DesignWare MAC common properties
+
+maintainers:
+  - Alexandre Torgue <alexandre.torgue@foss.st.com>
+  - Giuseppe Cavallaro <peppe.cavallaro@st.com>
+  - Jose Abreu <joabreu@synopsys.com>
+
+properties:
+  stmmac-axi-config:
+    type: object
+    unevaluatedProperties: false
+    description:
+      AXI BUS Mode parameters.
+
+    properties:
+      snps,lpi_en:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          enable Low Power Interface
+
+      snps,xit_frm:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          unlock on WoL
+
+      snps,wr_osr_lmt:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description:
+          max write outstanding req. limit
+
+      snps,rd_osr_lmt:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description:
+          max read outstanding req. limit
+
+      snps,blen:
+        $ref: /schemas/types.yaml#/definitions/uint32-array
+        description:
+          this is a vector of supported burst length.
+        minItems: 7
+        maxItems: 7
+
+      snps,fb:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          fixed-burst
+
+  snps,axi-config:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      AXI BUS Mode parameters. Phandle to a node that can contain the
+      following properties
+        * snps,lpi_en, enable Low Power Interface
+        * snps,xit_frm, unlock on WoL
+        * snps,wr_osr_lmt, max write outstanding req. limit
+        * snps,rd_osr_lmt, max read outstanding req. limit
+        * snps,blen, this is a vector of supported burst length.
+        * snps,fb, fixed-burst
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/net/snps,dwmac.yaml b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
index 2449311c6d28ed..2218a492817fd3 100644
--- a/Documentation/devicetree/bindings/net/snps,dwmac.yaml
+++ b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
@@ -57,6 +57,7 @@ properties:
         - allwinner,sun8i-r40-gmac
         - allwinner,sun8i-v3s-emac
         - allwinner,sun50i-a64-emac
+        - allwinner,sun50i-h616-emac1
         - amlogic,meson6-dwmac
         - amlogic,meson8b-dwmac
         - amlogic,meson8m2-dwmac
@@ -119,6 +120,7 @@ properties:
         - starfive,jh7110-dwmac
         - tesla,fsd-ethqos
         - thead,th1520-gmac
+        - ultrarisc,dp1000-gmac
 
   reg:
     minItems: 1
@@ -174,7 +176,7 @@ properties:
     minItems: 1
     items:
       - description: GMAC stmmaceth reset
-      - description: AHB reset
+      - description: AHB or OCP reset
 
   reset-names:
     oneOf:
@@ -182,7 +184,7 @@ properties:
           - enum: [stmmaceth, ahb]
       - items:
           - const: stmmaceth
-          - const: ahb
+          - enum: [ahb, stmmaceth-ocp]
 
   power-domains:
     maxItems: 1
@@ -195,18 +197,6 @@ properties:
       can be passive (no SW requirement), and requires that the MAC operate
       in a different mode than the PHY in order to function.
 
-  snps,axi-config:
-    $ref: /schemas/types.yaml#/definitions/phandle
-    description:
-      AXI BUS Mode parameters. Phandle to a node that can contain the
-      following properties
-        * snps,lpi_en, enable Low Power Interface
-        * snps,xit_frm, unlock on WoL
-        * snps,wr_osr_lmt, max write outstanding req. limit
-        * snps,rd_osr_lmt, max read outstanding req. limit
-        * snps,blen, this is a vector of supported burst length.
-        * snps,fb, fixed-burst
-
   snps,mtl-rx-config:
     $ref: /schemas/types.yaml#/definitions/phandle
     description:
@@ -558,45 +548,6 @@ properties:
     required:
       - compatible
 
-  stmmac-axi-config:
-    type: object
-    unevaluatedProperties: false
-    description:
-      AXI BUS Mode parameters.
-
-    properties:
-      snps,lpi_en:
-        $ref: /schemas/types.yaml#/definitions/flag
-        description:
-          enable Low Power Interface
-
-      snps,xit_frm:
-        $ref: /schemas/types.yaml#/definitions/flag
-        description:
-          unlock on WoL
-
-      snps,wr_osr_lmt:
-        $ref: /schemas/types.yaml#/definitions/uint32
-        description:
-          max write outstanding req. limit
-
-      snps,rd_osr_lmt:
-        $ref: /schemas/types.yaml#/definitions/uint32
-        description:
-          max read outstanding req. limit
-
-      snps,blen:
-        $ref: /schemas/types.yaml#/definitions/uint32-array
-        description:
-          this is a vector of supported burst length.
-        minItems: 7
-        maxItems: 7
-
-      snps,fb:
-        $ref: /schemas/types.yaml#/definitions/flag
-        description:
-          fixed-burst
-
 required:
   - compatible
   - reg
@@ -610,6 +561,7 @@ dependencies:
 
 allOf:
   - $ref: ethernet-controller.yaml#
+  - $ref: snps,dwmac-common.yaml#
   - if:
       properties:
         compatible:
@@ -622,6 +574,7 @@ allOf:
                 - allwinner,sun8i-r40-gmac
                 - allwinner,sun8i-v3s-emac
                 - allwinner,sun50i-a64-emac
+                - allwinner,sun50i-h616-emac1
                 - loongson,ls2k-dwmac
                 - loongson,ls7a-dwmac
                 - ingenic,jz4775-mac
diff --git a/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.yaml b/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.yaml
index 4eb63b303cffc3..8c8622160abb42 100644
--- a/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.yaml
+++ b/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.yaml
@@ -121,22 +121,22 @@ examples:
   - |
     ether: ethernet@65000000 {
         compatible = "socionext,uniphier-ld20-ave4";
-                reg = <0x65000000 0x8500>;
-                interrupts = <0 66 4>;
-                phy-mode = "rgmii";
-                phy-handle = <&ethphy>;
-                clock-names = "ether";
-                clocks = <&sys_clk 6>;
-                reset-names = "ether";
-                resets = <&sys_rst 6>;
-                socionext,syscon-phy-mode = <&soc_glue 0>;
-
-                mdio {
-                        #address-cells = <1>;
-                        #size-cells = <0>;
-
-                        ethphy: ethernet-phy@1 {
-                                reg = <1>;
-                        };
-                };
+        reg = <0x65000000 0x8500>;
+        interrupts = <0 66 4>;
+        phy-mode = "rgmii";
+        phy-handle = <&ethphy>;
+        clock-names = "ether";
+        clocks = <&sys_clk 6>;
+        reset-names = "ether";
+        resets = <&sys_rst 6>;
+        socionext,syscon-phy-mode = <&soc_glue 0>;
+
+        mdio {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            ethphy: ethernet-phy@1 {
+                reg = <1>;
+            };
         };
+    };
diff --git a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml
index 8b5da602a2e8d0..4d24dadfaa5dec 100644
--- a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml
+++ b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml
@@ -192,50 +192,50 @@ examples:
         interrupt-names = "rx_thresh", "rx", "tx", "misc";
 
         ethernet-ports {
-                #address-cells = <1>;
-                #size-cells = <0>;
-
-                cpsw_port1: port@1 {
-                        reg = <1>;
-                        label = "port1";
-                        mac-address = [ 00 00 00 00 00 00 ];
-                        phys = <&phy_gmii_sel 1>;
-                        phy-handle = <&ethphy0_sw>;
-                        phy-mode = "rgmii";
-                        ti,dual-emac-pvid = <1>;
-                };
-
-                cpsw_port2: port@2 {
-                        reg = <2>;
-                        label = "wan";
-                        mac-address = [ 00 00 00 00 00 00 ];
-                        phys = <&phy_gmii_sel 2>;
-                        phy-handle = <&ethphy1_sw>;
-                        phy-mode = "rgmii";
-                        ti,dual-emac-pvid = <2>;
-                };
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            cpsw_port1: port@1 {
+                reg = <1>;
+                label = "port1";
+                mac-address = [ 00 00 00 00 00 00 ];
+                phys = <&phy_gmii_sel 1>;
+                phy-handle = <&ethphy0_sw>;
+                phy-mode = "rgmii";
+                ti,dual-emac-pvid = <1>;
+            };
+
+            cpsw_port2: port@2 {
+                reg = <2>;
+                label = "wan";
+                mac-address = [ 00 00 00 00 00 00 ];
+                phys = <&phy_gmii_sel 2>;
+                phy-handle = <&ethphy1_sw>;
+                phy-mode = "rgmii";
+                ti,dual-emac-pvid = <2>;
+            };
         };
 
         davinci_mdio_sw: mdio@1000 {
-                compatible = "ti,cpsw-mdio","ti,davinci_mdio";
-                reg = <0x1000 0x100>;
-                clocks = <&gmac_clkctrl DRA7_GMAC_GMAC_CLKCTRL 0>;
-                clock-names = "fck";
-                #address-cells = <1>;
-                #size-cells = <0>;
-                bus_freq = <1000000>;
-
-                ethphy0_sw: ethernet-phy@0 {
-                        reg = <0>;
-                };
-
-                ethphy1_sw: ethernet-phy@1 {
-                        reg = <1>;
-                };
+            compatible = "ti,cpsw-mdio","ti,davinci_mdio";
+            reg = <0x1000 0x100>;
+            clocks = <&gmac_clkctrl DRA7_GMAC_GMAC_CLKCTRL 0>;
+            clock-names = "fck";
+            #address-cells = <1>;
+            #size-cells = <0>;
+            bus_freq = <1000000>;
+
+            ethphy0_sw: ethernet-phy@0 {
+                reg = <0>;
+            };
+
+            ethphy1_sw: ethernet-phy@1 {
+                reg = <1>;
+            };
         };
 
         cpts {
-                clocks = <&gmac_clkctrl DRA7_GMAC_GMAC_CLKCTRL 25>;
-                clock-names = "cpts";
+            clocks = <&gmac_clkctrl DRA7_GMAC_GMAC_CLKCTRL 25>;
+            clock-names = "cpts";
         };
     };
diff --git a/Documentation/devicetree/bindings/net/ultrarisc,dp1000-gmac.yaml b/Documentation/devicetree/bindings/net/ultrarisc,dp1000-gmac.yaml
new file mode 100644
index 00000000000000..13029302877ba8
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/ultrarisc,dp1000-gmac.yaml
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/ultrarisc,dp1000-gmac.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: UltraRISC DP1000 DWMAC Ethernet controller
+
+maintainers:
+  - Jia Wang <wangjia@ultrarisc.com>
+
+description:
+  The DP1000 integrates a Synopsys DesignWare GMAC controller from the
+  5.20 series. The SoC integration adds fixed RGMII TX and RX clock delays
+  which cannot be disabled. Consequently, a functional RGMII link requires
+  a PCB without TX or RX clock delays, represented by "rgmii-id".
+
+select:
+  properties:
+    compatible:
+      contains:
+        const: ultrarisc,dp1000-gmac
+  required:
+    - compatible
+
+properties:
+  compatible:
+    items:
+      - const: ultrarisc,dp1000-gmac
+      - const: snps,dwmac-5.20
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: stmmaceth
+
+  interrupts:
+    maxItems: 1
+
+  interrupt-names:
+    const: macirq
+
+  phy-mode:
+    const: rgmii-id
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - interrupts
+  - interrupt-names
+  - phy-mode
+
+allOf:
+  - $ref: snps,dwmac.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    ethernet@38000000 {
+        compatible = "ultrarisc,dp1000-gmac", "snps,dwmac-5.20";
+        reg = <0x38000000 0x1000000>;
+        clocks = <&clk 0>;
+        clock-names = "stmmaceth";
+        interrupts = <84>;
+        interrupt-names = "macirq";
+        phy-mode = "rgmii-id";
+    };
diff --git a/Documentation/devicetree/bindings/net/wireless/marvell,sd8787.yaml b/Documentation/devicetree/bindings/net/wireless/marvell,sd8787.yaml
index 930b700b73d007..9f905ef5c2c07b 100644
--- a/Documentation/devicetree/bindings/net/wireless/marvell,sd8787.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/marvell,sd8787.yaml
@@ -59,6 +59,12 @@ properties:
     description: Calibration data for sub-band 3 in the 5GHz band.
     maxItems: 502
 
+  marvell,invalid-reg-hint-in-rom:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      The regulatory hint in ROM conflicts with platform configuration.
+      It should be ignored, so the platform regulatory domain can be used.
+
   marvell,wakeup-pin:
     $ref: /schemas/types.yaml#/definitions/uint32
     description:
diff --git a/Documentation/devicetree/bindings/net/wireless/mediatek,mt76.yaml b/Documentation/devicetree/bindings/net/wireless/mediatek,mt76.yaml
index ae6b97cdc44ba4..bf4116b057d26d 100644
--- a/Documentation/devicetree/bindings/net/wireless/mediatek,mt76.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/mediatek,mt76.yaml
@@ -325,22 +325,22 @@ examples:
           r0 {
             regdomain = "FCC";
             txpower-5g {
-               b0 {
-                   channels = <36 48>;
-                   rates-ofdm = /bits/ 8 <23 23 23 23 23 23 23 23>;
-                   rates-mcs = /bits/ 8 <1 23 23 23 23 23 23 23 23 23 23>,
-                               /bits/ 8 <3 22 22 22 22 22 22 22 22 22 22>;
-                   rates-ru = /bits/ 8 <3 22 22 22 22 22 22 22 22 22 22 22 22>,
-                              /bits/ 8 <4 20 20 20 20 20 20 20 20 20 20 20 20>;
-               };
-               b1 {
-                   channels = <100 181>;
-                   rates-ofdm = /bits/ 8 <14 14 14 14 14 14 14 14>;
-                   rates-mcs = /bits/ 8  <4 14 14 14 14 14 14 14 14 14 14>;
-                   txs-delta = <12 9 6>;
-                   rates-ru = /bits/ 8  <7 14 14 14 14 14 14 14 14 14 14 14 14>;
-               };
-             };
+              b0 {
+                channels = <36 48>;
+                rates-ofdm = /bits/ 8 <23 23 23 23 23 23 23 23>;
+                rates-mcs = /bits/ 8 <1 23 23 23 23 23 23 23 23 23 23>,
+                            /bits/ 8 <3 22 22 22 22 22 22 22 22 22 22>;
+                rates-ru = /bits/ 8 <3 22 22 22 22 22 22 22 22 22 22 22 22>,
+                           /bits/ 8 <4 20 20 20 20 20 20 20 20 20 20 20 20>;
+              };
+              b1 {
+                channels = <100 181>;
+                rates-ofdm = /bits/ 8 <14 14 14 14 14 14 14 14>;
+                rates-mcs = /bits/ 8  <4 14 14 14 14 14 14 14 14 14 14>;
+                txs-delta = <12 9 6>;
+                rates-ru = /bits/ 8  <7 14 14 14 14 14 14 14 14 14 14 14 14>;
+              };
+            };
           };
         };
       };
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml
index e667d78b3ab3fa..0cb065265df4c9 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml
@@ -92,6 +92,22 @@ properties:
 
   ieee80211-freq-limit: true
 
+  nvmem-cells:
+    minItems: 1
+    maxItems: 3
+    description:
+      References to nvmem cells for MAC address and/or calibration data.
+      Supported cell names are mac-address, calibration, and pre-calibration.
+
+  nvmem-cell-names:
+    minItems: 1
+    maxItems: 3
+    items:
+      enum:
+        - mac-address
+        - calibration
+        - pre-calibration
+
   qcom,calibration-data:
     $ref: /schemas/types.yaml#/definitions/uint8-array
     description:
@@ -408,4 +424,4 @@ examples:
                           "msi15",
                           "legacy";
         ieee80211-freq-limit = <5470000 5875000>;
-      };
+    };
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
index d4aa56e2f82335..929a3d60aaefe1 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
@@ -16,11 +16,17 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - qcom,ipq8074-wifi
-      - qcom,ipq6018-wifi
-      - qcom,wcn6750-wifi
-      - qcom,ipq5018-wifi
+    oneOf:
+      - items:
+          - enum:
+              - qcom,wcn6755-wifi
+          - const: qcom,wcn6750-wifi
+
+      - enum:
+          - qcom,ipq8074-wifi
+          - qcom,ipq6018-wifi
+          - qcom,wcn6750-wifi
+          - qcom,ipq5018-wifi
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath12k-wsi.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ath12k-wsi.yaml
index 589960144fe1d5..fa64c8aa56f51c 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath12k-wsi.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath12k-wsi.yaml
@@ -58,13 +58,6 @@ properties:
       String to uniquely identify variant of the calibration data for designs
       with colliding bus and device ids
 
-  qcom,ath12k-calibration-variant:
-    $ref: /schemas/types.yaml#/definitions/string
-    deprecated: true
-    description:
-      String to uniquely identify variant of the calibration data for designs
-      with colliding bus and device ids
-
   qcom,wsi-controller:
     $ref: /schemas/types.yaml#/definitions/flag
     description:
diff --git a/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml b/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml
index 6c0888ae4c4e17..60dfe257eadc26 100644
--- a/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml
@@ -50,6 +50,9 @@ properties:
       used, this property is required. When SDIO is used, the "in-band"
       interrupt provided by the SDIO bus is used unless an interrupt is defined
       in the Device Tree.
+
+      The hardware suffers from a limitation when this property is used with
+      SDIO. OOB IRQ with SDIO should only be used for debug.
     maxItems: 1
 
   reset-gpios:
diff --git a/Documentation/devicetree/bindings/net/wireless/ti,wl1251.txt b/Documentation/devicetree/bindings/net/wireless/ti,wl1251.txt
deleted file mode 100644
index 88fd28d15eaca2..00000000000000
--- a/Documentation/devicetree/bindings/net/wireless/ti,wl1251.txt
+++ /dev/null
@@ -1,64 +0,0 @@
-* Texas Instruments wl1251 wireless lan controller
-
-The wl1251 chip can be connected via SPI or via SDIO. This
-document describes the binding for the SPI connected chip.
-
-Required properties:
-- compatible :        Should be "ti,wl1251"
-- reg :               Chip select address of device
-- spi-max-frequency : Maximum SPI clocking speed of device in Hz
-- interrupts :        Should contain interrupt line
-- vio-supply :        phandle to regulator providing VIO
-
-Optional properties:
-- ti,wl1251-has-eeprom : boolean, the wl1251 has an eeprom connected, which
-                         provides configuration data (calibration, MAC, ...)
-- ti,power-gpio :	 GPIO connected to chip's PMEN pin if operated in
-			 SPI mode
-- Please consult Documentation/devicetree/bindings/spi/spi-bus.txt
-  for optional SPI connection related properties,
-
-Examples:
-
-&spi1 {
-	wl1251@0 {
-		compatible = "ti,wl1251";
-
-		reg = <0>;
-		spi-max-frequency = <48000000>;
-		spi-cpol;
-		spi-cpha;
-
-		interrupt-parent = <&gpio2>;
-		interrupts = <10 IRQ_TYPE_NONE>; /* gpio line 42 */
-
-		vio-supply = <&vio>;
-		ti,power-gpio = <&gpio3 23 GPIO_ACTIVE_HIGH>; /* 87 */
-	};
-};
-
-&mmc3 {
-	vmmc-supply = <&wlan_en>;
-
-	bus-width = <4>;
-	non-removable;
-	ti,non-removable;
-	cap-power-off-card;
-
-	pinctrl-names = "default";
-	pinctrl-0 = <&mmc3_pins>;
-
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	wlan: wifi@1 {
-		compatible = "ti,wl1251";
-
-		reg = <1>;
-
-		interrupt-parent = <&gpio1>;
-		interrupts = <21 IRQ_TYPE_LEVEL_HIGH>;	/* GPIO_21 */
-
-		ti,wl1251-has-eeprom;
-	};
-};
diff --git a/Documentation/devicetree/bindings/net/wireless/ti,wl1251.yaml b/Documentation/devicetree/bindings/net/wireless/ti,wl1251.yaml
new file mode 100644
index 00000000000000..6ce82c4a2c0b28
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/wireless/ti,wl1251.yaml
@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/wireless/ti,wl1251.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments WL1251 Wireless LAN Controller
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+properties:
+  compatible:
+    const: ti,wl1251
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  vio-supply:
+    description:
+      Phandle to the regulator providing VIO. Required when connected via SPI.
+
+  ti,power-gpio:
+    maxItems: 1
+    description:
+      GPIO connected to chip's PMEN pin if operated in SPI mode.
+
+  ti,wl1251-has-eeprom:
+    type: boolean
+    description:
+      The wl1251 has an EEPROM connected, which provides configuration
+      data (calibration, MAC, ...).
+
+  clocks:
+    maxItems: 1
+
+  spi-max-frequency: true
+  spi-cpol: true
+  spi-cpha: true
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/gpio/gpio.h>
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        wifi@0 {
+            compatible = "ti,wl1251";
+            reg = <0>;
+            spi-max-frequency = <48000000>;
+            spi-cpol;
+            spi-cpha;
+            interrupt-parent = <&gpio2>;
+            interrupts = <10 IRQ_TYPE_NONE>;
+            vio-supply = <&vio>;
+            ti,power-gpio = <&gpio3 23 GPIO_ACTIVE_HIGH>;
+        };
+    };
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    mmc {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        wifi@1 {
+            compatible = "ti,wl1251";
+            reg = <1>;
+            interrupt-parent = <&gpio1>;
+            interrupts = <21 IRQ_TYPE_LEVEL_HIGH>;
+            ti,wl1251-has-eeprom;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/net/wiznet,w5100.yaml b/Documentation/devicetree/bindings/net/wiznet,w5100.yaml
new file mode 100644
index 00000000000000..b0d1e1d699dc63
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/wiznet,w5100.yaml
@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/wiznet,w5100.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Wiznet w5100/w5200/w5500 SPI Ethernet Controller
+
+maintainers:
+  - Arthur Crepin Leblond <arthur@marmottus.net>
+
+description: |
+  This is a standalone 10/100 MBit Ethernet controller with SPI interface.
+
+properties:
+  compatible:
+    enum:
+      - wiznet,w5100
+      - wiznet,w5200
+      - wiznet,w5500
+
+  interrupts:
+    minItems: 1
+    description:
+      The main interrupt (INT pin) is mandatory on all devices. The w5500
+      additionally supports an optional second interrupt for link status changes
+      (LINKLED pin). If it is not specified, link state changes will not be
+      reported.
+    items:
+      - description: Main interrupt (INT pin on the device).
+      - description:
+          Optional interrupt for link status changes (LINKLED pin on w5500).
+
+  reg:
+    maxItems: 1
+
+  spi-max-frequency:
+    description:
+      According to the datasheets, w5500 and w5200 allow a maximum of 80 MHz
+      while w5100 is reported by users to allow a maximum of 14 MHz. However,
+      board designs may need to limit this value.
+    maximum: 80000000
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+allOf:
+  - $ref: /schemas/net/ethernet-controller.yaml#
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              enum:
+                - wiznet,w5500
+    then:
+      properties:
+        interrupts:
+          maxItems: 1
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    /* Example (for Raspberry Pi with pin control stuff for GPIO irq) */
+
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        ethernet@0 {
+            compatible = "wiznet,w5500";
+            reg = <0>;
+            interrupt-parent = <&gpio>;
+            interrupts = <25 IRQ_TYPE_LEVEL_LOW>,
+                         <24 IRQ_TYPE_EDGE_BOTH>;
+            local-mac-address = [ a0 b0 c0 d0 e0 f0 ];
+            pinctrl-0 = <&eth1_pins>;
+            pinctrl-names = "default";
+            spi-max-frequency = <30000000>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/net/wiznet,w5x00.txt b/Documentation/devicetree/bindings/net/wiznet,w5x00.txt
deleted file mode 100644
index e9665798c4be63..00000000000000
--- a/Documentation/devicetree/bindings/net/wiznet,w5x00.txt
+++ /dev/null
@@ -1,50 +0,0 @@
-* Wiznet w5x00
-
-This is a standalone 10/100 MBit Ethernet controller with SPI interface.
-
-For each device connected to a SPI bus, define a child node within
-the SPI master node.
-
-Required properties:
-- compatible: Should be one of the following strings:
-	      "wiznet,w5100"
-	      "wiznet,w5200"
-	      "wiznet,w5500"
-- reg: Specify the SPI chip select the chip is wired to.
-- interrupts: Specify the interrupt index within the interrupt controller (referred
-              to above in interrupt-parent) and interrupt type. w5x00 natively
-              generates falling edge interrupts, however, additional board logic
-              might invert the signal.
-- pinctrl-names: List of assigned state names, see pinctrl binding documentation.
-- pinctrl-0: List of phandles to configure the GPIO pin used as interrupt line,
-             see also generic and your platform specific pinctrl binding
-             documentation.
-
-Optional properties:
-- spi-max-frequency: Maximum frequency of the SPI bus when accessing the w5500.
-  According to the w5500 datasheet, the chip allows a maximum of 80 MHz, however,
-  board designs may need to limit this value.
-- local-mac-address: See ethernet.txt in the same directory.
-
-
-Example (for Raspberry Pi with pin control stuff for GPIO irq):
-
-&spi {
-	ethernet@0: w5500@0 {
-		compatible = "wiznet,w5500";
-		reg = <0>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&eth1_pins>;
-		interrupt-parent = <&gpio>;
-		interrupts = <25 IRQ_TYPE_EDGE_FALLING>;
-		spi-max-frequency = <30000000>;
-	};
-};
-
-&gpio {
-	eth1_pins: eth1_pins {
-		brcm,pins = <25>;
-		brcm,function = <0>; /* in */
-		brcm,pull = <0>; /* none */
-	};
-};
diff --git a/Documentation/devicetree/bindings/net/x-powers,acx00-ephy-package.yaml b/Documentation/devicetree/bindings/net/x-powers,acx00-ephy-package.yaml
new file mode 100644
index 00000000000000..781695d9d9f14e
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/x-powers,acx00-ephy-package.yaml
@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/x-powers,acx00-ephy-package.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: X-Powers AC200 and AC300 Ethernet PHY packages
+
+maintainers:
+  - James Hilliard <james.hilliard1@gmail.com>
+
+description:
+  The AC200 and AC300 contain compatible Fast Ethernet PHY blocks which report
+  the same Clause 22 PHY ID and use the same link-side register layout. The
+  package control paths differ. AC200 package control registers are accessed
+  through the companion IC's I2C register interface, while AC300 control
+  registers occupy a second Clause 22 address fixed at 16 plus the link PHY
+  address.
+
+  Systems with a known package use the corresponding AC200 or AC300 compatible.
+  Systems which can contain either package use the ACx00 compatible and a
+  packed NVMEM configuration field to select the backend at runtime.
+
+  The link PHY cannot report its ID until the package control sequence has
+  completed, so its child node supplies the documented PHY ID. The package and
+  child use the same base address. The AC200 driver programs that address into
+  the package, while the AC300 driver derives its control address from it.
+
+properties:
+  compatible:
+    enum:
+      - x-powers,ac200-ephy-package
+      - x-powers,ac300-ephy-package
+      - x-powers,acx00-ephy-package
+
+  reg:
+    maximum: 31
+
+  clocks:
+    maxItems: 1
+    description: AC300 input clock, running at 24, 25, or 27 MHz
+
+  vcc-supply:
+    description:
+      3.3 V package supply. This is connected to EPHY_VCC on AC200 and VCC1
+      on AC300.
+
+  nvmem-cells:
+    maxItems: 1
+    description:
+      Packed little-endian EPHY configuration value, with the least-significant
+      byte first. Bits 3 through 0 contain the analog calibration code, bits 7
+      through 4 are reserved, bit 8 selects AC300 when set and AC200 when clear,
+      and bit 9 requests the AC300 low-calibration tuning sequence. Bits 15
+      through 10 are reserved.
+
+      AC300 and selectable ACx00 packages require the full configuration
+      field, including bits 8 and 9 even when their values are zero, returned
+      as two bytes. A fixed AC200 package may provide a one-byte calibration
+      value or a two-byte configuration with bit 8 clear, or omit the cell
+      and use its internal eFuse calibration copy.
+
+  nvmem-cell-names:
+    const: configuration
+
+  x-powers,ac200:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      Phandle to the AC200 companion IC described by the x-powers,ac200 MFD
+      binding. Its I2C interface provides access to the package control
+      registers used to power and configure the Ethernet PHY.
+
+  x-powers,xmii-rx-clock-inverted:
+    type: boolean
+    description:
+      Invert the xMII receive clock inside the PHY. This is required by some
+      SoC integrations, including the Allwinner H6 AC200 connection.
+
+patternProperties:
+  ^ethernet-phy@[a-f0-9]+$:
+    $ref: ethernet-phy.yaml#
+    properties:
+      compatible:
+        const: ethernet-phy-id0044.1400
+    required:
+      - compatible
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - vcc-supply
+
+dependentRequired:
+  nvmem-cells: [ nvmem-cell-names ]
+
+allOf:
+  - $ref: ethernet-phy-package.yaml#
+  - if:
+      properties:
+        compatible:
+          const: x-powers,ac200-ephy-package
+    then:
+      properties:
+        clocks: false
+      required:
+        - x-powers,ac200
+  - if:
+      properties:
+        compatible:
+          const: x-powers,ac300-ephy-package
+    then:
+      properties:
+        x-powers,ac200: false
+      required:
+        - clocks
+        - nvmem-cells
+        - nvmem-cell-names
+  - if:
+      properties:
+        compatible:
+          const: x-powers,acx00-ephy-package
+    then:
+      required:
+        - clocks
+        - nvmem-cells
+        - nvmem-cell-names
+        - x-powers,ac200
+  - if:
+      properties:
+        compatible:
+          enum:
+            - x-powers,ac300-ephy-package
+            - x-powers,acx00-ephy-package
+    then:
+      properties:
+        reg:
+          maximum: 7
+      patternProperties:
+        ^ethernet-phy@[a-f0-9]+$:
+          properties:
+            reg:
+              maximum: 7
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    mdio {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        ethernet-phy-package@0 {
+            compatible = "x-powers,acx00-ephy-package";
+            reg = <0>;
+            #address-cells = <1>;
+            #size-cells = <0>;
+            clocks = <&pwm 5>;
+            vcc-supply = <&reg_3v3>;
+            nvmem-cells = <&ephy_configuration>;
+            nvmem-cell-names = "configuration";
+            x-powers,ac200 = <&ac200>;
+
+            ethernet-phy@0 {
+                compatible = "ethernet-phy-id0044.1400";
+                reg = <0>;
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
index bc0a0898b9e47b..566304ed4c7869 100644
--- a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
+++ b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
@@ -31,6 +31,7 @@ properties:
           - qcom,ipq9574-qfprom
           - qcom,ipq9650-qfprom
           - qcom,kaanapali-qfprom
+          - qcom,maili-qfprom
           - qcom,milos-qfprom
           - qcom,msm8226-qfprom
           - qcom,msm8916-qfprom
diff --git a/Documentation/devicetree/bindings/nvmem/socionext,uniphier-efuse.yaml b/Documentation/devicetree/bindings/nvmem/socionext,uniphier-efuse.yaml
index e27cbae2d63a0f..d26b3d2a3eba07 100644
--- a/Documentation/devicetree/bindings/nvmem/socionext,uniphier-efuse.yaml
+++ b/Documentation/devicetree/bindings/nvmem/socionext,uniphier-efuse.yaml
@@ -16,7 +16,11 @@ allOf:
 
 properties:
   compatible:
-    const: socionext,uniphier-efuse
+    oneOf:
+      - const: socionext,uniphier-efuse
+      - items:
+          - const: ti,am62p-efuse
+          - const: socionext,uniphier-efuse
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.yaml b/Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.yaml
index d073c51c2b9a99..e0aacecd652d59 100644
--- a/Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.yaml
+++ b/Documentation/devicetree/bindings/nvmem/zii,rave-sp-eeprom.yaml
@@ -27,7 +27,7 @@ properties:
     $ref: /schemas/types.yaml#/definitions/string
     description:
       Unique EEPROM identifier describing its function in the
-      system. Will be used as created NVMEM deivce's name.
+      system. Will be used as created NVMEM device's name.
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml b/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml
index e863519f31617e..b6801057e31e66 100644
--- a/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/hisilicon,kirin-pcie.yaml
@@ -160,7 +160,7 @@ examples:
               reg = <0x2800 0 0 0 0>;
               compatible = "pciclass,0604";
               device_type = "pci";
-              reset-gpios = <&gpio27 4 0 >;
+              reset-gpios = <&gpio27 4 0>;
               #address-cells = <3>;
               #size-cells = <2>;
               ranges;
@@ -170,7 +170,7 @@ examples:
               reg = <0x03800 0 0 0 0>;
               compatible = "pciclass,0604";
               device_type = "pci";
-              reset-gpios = <&gpio25 2 0 >;
+              reset-gpios = <&gpio25 2 0>;
               #address-cells = <3>;
               #size-cells = <2>;
               ranges;
diff --git a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
deleted file mode 100644
index d099f3476cccdd..00000000000000
--- a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
+++ /dev/null
@@ -1,670 +0,0 @@
-NVIDIA Tegra PCIe controller
-
-Required properties:
-- compatible: Must be:
-  - "nvidia,tegra20-pcie": for Tegra20
-  - "nvidia,tegra30-pcie": for Tegra30
-  - "nvidia,tegra124-pcie": for Tegra124 and Tegra132
-  - "nvidia,tegra210-pcie": for Tegra210
-  - "nvidia,tegra186-pcie": for Tegra186
-- power-domains: To ungate power partition by BPMP powergate driver. Must
-  contain BPMP phandle and PCIe power partition ID. This is required only
-  for Tegra186.
-- device_type: Must be "pci"
-- reg: A list of physical base address and length for each set of controller
-  registers. Must contain an entry for each entry in the reg-names property.
-- reg-names: Must include the following entries:
-  "pads": PADS registers
-  "afi": AFI registers
-  "cs": configuration space region
-- interrupts: A list of interrupt outputs of the controller. Must contain an
-  entry for each entry in the interrupt-names property.
-- interrupt-names: Must include the following entries:
-  "intr": The Tegra interrupt that is asserted for controller interrupts
-  "msi": The Tegra interrupt that is asserted when an MSI is received
-- bus-range: Range of bus numbers associated with this controller
-- #address-cells: Address representation for root ports (must be 3)
-  - cell 0 specifies the bus and device numbers of the root port:
-    [23:16]: bus number
-    [15:11]: device number
-  - cell 1 denotes the upper 32 address bits and should be 0
-  - cell 2 contains the lower 32 address bits and is used to translate to the
-    CPU address space
-- #size-cells: Size representation for root ports (must be 2)
-- ranges: Describes the translation of addresses for root ports and standard
-  PCI regions. The entries must be 6 cells each, where the first three cells
-  correspond to the address as described for the #address-cells property
-  above, the fourth cell is the physical CPU address to translate to and the
-  fifth and six cells are as described for the #size-cells property above.
-  - The first two entries are expected to translate the addresses for the root
-    port registers, which are referenced by the assigned-addresses property of
-    the root port nodes (see below).
-  - The remaining entries setup the mapping for the standard I/O, memory and
-    prefetchable PCI regions. The first cell determines the type of region
-    that is setup:
-    - 0x81000000: I/O memory region
-    - 0x82000000: non-prefetchable memory region
-    - 0xc2000000: prefetchable memory region
-  Please refer to the standard PCI bus binding document for a more detailed
-  explanation.
-- #interrupt-cells: Size representation for interrupts (must be 1)
-- interrupt-map-mask and interrupt-map: Standard PCI IRQ mapping properties
-  Please refer to the standard PCI bus binding document for a more detailed
-  explanation.
-- clocks: Must contain an entry for each entry in clock-names.
-  See ../clocks/clock-bindings.txt for details.
-- clock-names: Must include the following entries:
-  - pex
-  - afi
-  - pll_e
-  - cml (not required for Tegra20)
-- resets: Must contain an entry for each entry in reset-names.
-  See ../reset/reset.txt for details.
-- reset-names: Must include the following entries:
-  - pex
-  - afi
-  - pcie_x
-
-Optional properties:
-- pinctrl-names: A list of pinctrl state names. Must contain the following
-  entries:
-  - "default": active state, puts PCIe I/O out of deep power down state
-  - "idle": puts PCIe I/O into deep power down state
-- pinctrl-0: phandle for the default/active state of pin configurations.
-- pinctrl-1: phandle for the idle state of pin configurations.
-
-Required properties on Tegra124 and later (deprecated):
-- phys: Must contain an entry for each entry in phy-names.
-- phy-names: Must include the following entries:
-  - pcie
-
-These properties are deprecated in favour of per-lane PHYs define in each of
-the root ports (see below).
-
-Power supplies for Tegra20:
-- avdd-pex-supply: Power supply for analog PCIe logic. Must supply 1.05 V.
-- vdd-pex-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
-- avdd-pex-pll-supply: Power supply for dedicated (internal) PCIe PLL. Must
-  supply 1.05 V.
-- avdd-plle-supply: Power supply for PLLE, which is shared with SATA. Must
-  supply 1.05 V.
-- vddio-pex-clk-supply: Power supply for PCIe clock. Must supply 3.3 V.
-
-Power supplies for Tegra30:
-- Required:
-  - avdd-pex-pll-supply: Power supply for dedicated (internal) PCIe PLL. Must
-    supply 1.05 V.
-  - avdd-plle-supply: Power supply for PLLE, which is shared with SATA. Must
-    supply 1.05 V.
-  - vddio-pex-ctl-supply: Power supply for PCIe control I/O partition. Must
-    supply 1.8 V.
-  - hvdd-pex-supply: High-voltage supply for PCIe I/O and PCIe output clocks.
-    Must supply 3.3 V.
-- Optional:
-  - If lanes 0 to 3 are used:
-    - avdd-pexa-supply: Power supply for analog PCIe logic. Must supply 1.05 V.
-    - vdd-pexa-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
-  - If lanes 4 or 5 are used:
-    - avdd-pexb-supply: Power supply for analog PCIe logic. Must supply 1.05 V.
-    - vdd-pexb-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
-
-Power supplies for Tegra124:
-- Required:
-  - avddio-pex-supply: Power supply for analog PCIe logic. Must supply 1.05 V.
-  - dvddio-pex-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
-  - hvdd-pex-supply: High-voltage supply for PCIe I/O and PCIe output clocks.
-    Must supply 3.3 V.
-  - vddio-pex-ctl-supply: Power supply for PCIe control I/O partition. Must
-    supply 2.8-3.3 V.
-
-Power supplies for Tegra210:
-- Required:
-  - hvddio-pex-supply: High-voltage supply for PCIe I/O and PCIe output
-    clocks. Must supply 1.8 V.
-  - dvddio-pex-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
-  - vddio-pex-ctl-supply: Power supply for PCIe control I/O partition. Must
-    supply 1.8 V.
-
-Power supplies for Tegra186:
-- Required:
-  - dvdd-pex-supply: Power supply for digital PCIe I/O. Must supply 1.05 V.
-  - hvdd-pex-pll-supply: High-voltage supply for PLLE (shared with USB3). Must
-    supply 1.8 V.
-  - hvdd-pex-supply: High-voltage supply for PCIe I/O and PCIe output clocks.
-    Must supply 1.8 V.
-  - vddio-pexctl-aud-supply: Power supply for PCIe side band signals. Must
-    supply 1.8 V.
-
-Root ports are defined as subnodes of the PCIe controller node.
-
-Required properties:
-- device_type: Must be "pci"
-- assigned-addresses: Address and size of the port configuration registers
-- reg: PCI bus address of the root port
-- #address-cells: Must be 3
-- #size-cells: Must be 2
-- ranges: Sub-ranges distributed from the PCIe controller node. An empty
-  property is sufficient.
-- nvidia,num-lanes: Number of lanes to use for this port. Valid combinations
-  are:
-  - Root port 0 uses 4 lanes, root port 1 is unused.
-  - Both root ports use 2 lanes.
-
-Required properties for Tegra124 and later:
-- phys: Must contain an phandle to a PHY for each entry in phy-names.
-- phy-names: Must include an entry for each active lane. Note that the number
-  of entries does not have to (though usually will) be equal to the specified
-  number of lanes in the nvidia,num-lanes property. Entries are of the form
-  "pcie-N": where N ranges from 0 to the value specified in nvidia,num-lanes.
-
-Examples:
-=========
-
-Tegra20:
---------
-
-SoC DTSI:
-
-	pcie-controller@80003000 {
-		compatible = "nvidia,tegra20-pcie";
-		device_type = "pci";
-		reg = <0x80003000 0x00000800   /* PADS registers */
-		       0x80003800 0x00000200   /* AFI registers */
-		       0x90000000 0x10000000>; /* configuration space */
-		reg-names = "pads", "afi", "cs";
-		interrupts = <0 98 0x04   /* controller interrupt */
-		              0 99 0x04>; /* MSI interrupt */
-		interrupt-names = "intr", "msi";
-
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0>;
-		interrupt-map = <0 0 0 0 &intc GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
-
-		bus-range = <0x00 0xff>;
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		ranges = <0x82000000 0 0x80000000 0x80000000 0 0x00001000   /* port 0 registers */
-			  0x82000000 0 0x80001000 0x80001000 0 0x00001000   /* port 1 registers */
-			  0x81000000 0 0          0x82000000 0 0x00010000   /* downstream I/O */
-			  0x82000000 0 0xa0000000 0xa0000000 0 0x10000000   /* non-prefetchable memory */
-			  0xc2000000 0 0xb0000000 0xb0000000 0 0x10000000>; /* prefetchable memory */
-
-		clocks = <&tegra_car 70>, <&tegra_car 72>, <&tegra_car 118>;
-		clock-names = "pex", "afi", "pll_e";
-		resets = <&tegra_car 70>, <&tegra_car 72>, <&tegra_car 74>;
-		reset-names = "pex", "afi", "pcie_x";
-		status = "disabled";
-
-		pci@1,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82000800 0 0x80000000 0 0x1000>;
-			reg = <0x000800 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-
-			ranges;
-
-			nvidia,num-lanes = <2>;
-		};
-
-		pci@2,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82001000 0 0x80001000 0 0x1000>;
-			reg = <0x001000 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-
-			ranges;
-
-			nvidia,num-lanes = <2>;
-		};
-	};
-
-Board DTS:
-
-	pcie-controller@80003000 {
-		status = "okay";
-
-		vdd-supply = <&pci_vdd_reg>;
-		pex-clk-supply = <&pci_clk_reg>;
-
-		/* root port 00:01.0 */
-		pci@1,0 {
-			status = "okay";
-
-			/* bridge 01:00.0 (optional) */
-			pci@0,0 {
-				reg = <0x010000 0 0 0 0>;
-
-				#address-cells = <3>;
-				#size-cells = <2>;
-
-				device_type = "pci";
-
-				/* endpoint 02:00.0 */
-				pci@0,0 {
-					reg = <0x020000 0 0 0 0>;
-				};
-			};
-		};
-	};
-
-Note that devices on the PCI bus are dynamically discovered using PCI's bus
-enumeration and therefore don't need corresponding device nodes in DT. However
-if a device on the PCI bus provides a non-probeable bus such as I2C or SPI,
-device nodes need to be added in order to allow the bus' children to be
-instantiated at the proper location in the operating system's device tree (as
-illustrated by the optional nodes in the example above).
-
-Tegra30:
---------
-
-SoC DTSI:
-
-	pcie-controller@3000 {
-		compatible = "nvidia,tegra30-pcie";
-		device_type = "pci";
-		reg = <0x00003000 0x00000800   /* PADS registers */
-		       0x00003800 0x00000200   /* AFI registers */
-		       0x10000000 0x10000000>; /* configuration space */
-		reg-names = "pads", "afi", "cs";
-		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH   /* controller interrupt */
-			      GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-		interrupt-names = "intr", "msi";
-
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0>;
-		interrupt-map = <0 0 0 0 &intc GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
-
-		bus-range = <0x00 0xff>;
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		ranges = <0x82000000 0 0x00000000 0x00000000 0 0x00001000   /* port 0 configuration space */
-			  0x82000000 0 0x00001000 0x00001000 0 0x00001000   /* port 1 configuration space */
-			  0x82000000 0 0x00004000 0x00004000 0 0x00001000   /* port 2 configuration space */
-			  0x81000000 0 0          0x02000000 0 0x00010000   /* downstream I/O */
-			  0x82000000 0 0x20000000 0x20000000 0 0x08000000   /* non-prefetchable memory */
-			  0xc2000000 0 0x28000000 0x28000000 0 0x18000000>; /* prefetchable memory */
-
-		clocks = <&tegra_car TEGRA30_CLK_PCIE>,
-			 <&tegra_car TEGRA30_CLK_AFI>,
-			 <&tegra_car TEGRA30_CLK_PLL_E>,
-			 <&tegra_car TEGRA30_CLK_CML0>;
-		clock-names = "pex", "afi", "pll_e", "cml";
-		resets = <&tegra_car 70>,
-			 <&tegra_car 72>,
-			 <&tegra_car 74>;
-		reset-names = "pex", "afi", "pcie_x";
-		status = "disabled";
-
-		pci@1,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82000800 0 0x00000000 0 0x1000>;
-			reg = <0x000800 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <2>;
-		};
-
-		pci@2,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82001000 0 0x00001000 0 0x1000>;
-			reg = <0x001000 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <2>;
-		};
-
-		pci@3,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82001800 0 0x00004000 0 0x1000>;
-			reg = <0x001800 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <2>;
-		};
-	};
-
-Board DTS:
-
-	pcie-controller@3000 {
-		status = "okay";
-
-		avdd-pexa-supply = <&ldo1_reg>;
-		vdd-pexa-supply = <&ldo1_reg>;
-		avdd-pexb-supply = <&ldo1_reg>;
-		vdd-pexb-supply = <&ldo1_reg>;
-		avdd-pex-pll-supply = <&ldo1_reg>;
-		avdd-plle-supply = <&ldo1_reg>;
-		vddio-pex-ctl-supply = <&sys_3v3_reg>;
-		hvdd-pex-supply = <&sys_3v3_pexs_reg>;
-
-		pci@1,0 {
-			status = "okay";
-		};
-
-		pci@3,0 {
-			status = "okay";
-		};
-	};
-
-Tegra124:
----------
-
-SoC DTSI:
-
-	pcie-controller@1003000 {
-		compatible = "nvidia,tegra124-pcie";
-		device_type = "pci";
-		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
-		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
-		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
-		reg-names = "pads", "afi", "cs";
-		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
-			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-		interrupt-names = "intr", "msi";
-
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0>;
-		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
-
-		bus-range = <0x00 0xff>;
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
-			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
-			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
-			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
-			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
-
-		clocks = <&tegra_car TEGRA124_CLK_PCIE>,
-			 <&tegra_car TEGRA124_CLK_AFI>,
-			 <&tegra_car TEGRA124_CLK_PLL_E>,
-			 <&tegra_car TEGRA124_CLK_CML0>;
-		clock-names = "pex", "afi", "pll_e", "cml";
-		resets = <&tegra_car 70>,
-			 <&tegra_car 72>,
-			 <&tegra_car 74>;
-		reset-names = "pex", "afi", "pcie_x";
-		status = "disabled";
-
-		pci@1,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
-			reg = <0x000800 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <2>;
-		};
-
-		pci@2,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
-			reg = <0x001000 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <1>;
-		};
-	};
-
-Board DTS:
-
-	pcie-controller@1003000 {
-		status = "okay";
-
-		avddio-pex-supply = <&vdd_1v05_run>;
-		dvddio-pex-supply = <&vdd_1v05_run>;
-		avdd-pex-pll-supply = <&vdd_1v05_run>;
-		hvdd-pex-supply = <&vdd_3v3_lp0>;
-		hvdd-pex-pll-e-supply = <&vdd_3v3_lp0>;
-		vddio-pex-ctl-supply = <&vdd_3v3_lp0>;
-		avdd-pll-erefe-supply = <&avdd_1v05_run>;
-
-		/* Mini PCIe */
-		pci@1,0 {
-			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
-			phy-names = "pcie-0";
-			status = "okay";
-		};
-
-		/* Gigabit Ethernet */
-		pci@2,0 {
-			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>;
-			phy-names = "pcie-0";
-			status = "okay";
-		};
-	};
-
-Tegra210:
----------
-
-SoC DTSI:
-
-	pcie-controller@1003000 {
-		compatible = "nvidia,tegra210-pcie";
-		device_type = "pci";
-		reg = <0x0 0x01003000 0x0 0x00000800   /* PADS registers */
-		       0x0 0x01003800 0x0 0x00000800   /* AFI registers */
-		       0x0 0x02000000 0x0 0x10000000>; /* configuration space */
-		reg-names = "pads", "afi", "cs";
-		interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
-			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-		interrupt-names = "intr", "msi";
-
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0>;
-		interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
-
-		bus-range = <0x00 0xff>;
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		ranges = <0x82000000 0 0x01000000 0x0 0x01000000 0 0x00001000   /* port 0 configuration space */
-			  0x82000000 0 0x01001000 0x0 0x01001000 0 0x00001000   /* port 1 configuration space */
-			  0x81000000 0 0x0        0x0 0x12000000 0 0x00010000   /* downstream I/O (64 KiB) */
-			  0x82000000 0 0x13000000 0x0 0x13000000 0 0x0d000000   /* non-prefetchable memory (208 MiB) */
-			  0xc2000000 0 0x20000000 0x0 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
-
-		clocks = <&tegra_car TEGRA210_CLK_PCIE>,
-			 <&tegra_car TEGRA210_CLK_AFI>,
-			 <&tegra_car TEGRA210_CLK_PLL_E>,
-			 <&tegra_car TEGRA210_CLK_CML0>;
-		clock-names = "pex", "afi", "pll_e", "cml";
-		resets = <&tegra_car 70>,
-			 <&tegra_car 72>,
-			 <&tegra_car 74>;
-		reset-names = "pex", "afi", "pcie_x";
-		status = "disabled";
-
-		pci@1,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
-			reg = <0x000800 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <4>;
-		};
-
-		pci@2,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
-			reg = <0x001000 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <1>;
-		};
-	};
-
-Board DTS:
-
-	pcie-controller@1003000 {
-		status = "okay";
-
-		avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
-		hvddio-pex-supply = <&vdd_1v8>;
-		dvddio-pex-supply = <&vdd_pex_1v05>;
-		dvdd-pex-pll-supply = <&vdd_pex_1v05>;
-		hvdd-pex-pll-e-supply = <&vdd_1v8>;
-		vddio-pex-ctl-supply = <&vdd_1v8>;
-
-		pci@1,0 {
-			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-0}>,
-			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-1}>,
-			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-2}>,
-			       <&{/padctl@7009f000/pads/pcie/lanes/pcie-3}>;
-			phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
-			status = "okay";
-		};
-
-		pci@2,0 {
-			phys = <&{/padctl@7009f000/pads/pcie/lanes/pcie-4}>;
-			phy-names = "pcie-0";
-			status = "okay";
-		};
-	};
-
-Tegra186:
----------
-
-SoC DTSI:
-
-	pcie@10003000 {
-		compatible = "nvidia,tegra186-pcie";
-		power-domains = <&bpmp TEGRA186_POWER_DOMAIN_PCX>;
-		device_type = "pci";
-		reg = <0x0 0x10003000 0x0 0x00000800   /* PADS registers */
-		       0x0 0x10003800 0x0 0x00000800   /* AFI registers */
-		       0x0 0x40000000 0x0 0x10000000>; /* configuration space */
-		reg-names = "pads", "afi", "cs";
-
-		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
-			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
-		interrupt-names = "intr", "msi";
-
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0>;
-		interrupt-map = <0 0 0 0 &gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
-
-		bus-range = <0x00 0xff>;
-		#address-cells = <3>;
-		#size-cells = <2>;
-
-		ranges = <0x82000000 0 0x10000000 0x0 0x10000000 0 0x00001000   /* port 0 configuration space */
-			  0x82000000 0 0x10001000 0x0 0x10001000 0 0x00001000   /* port 1 configuration space */
-			  0x82000000 0 0x10004000 0x0 0x10004000 0 0x00001000   /* port 2 configuration space */
-			  0x81000000 0 0x0        0x0 0x50000000 0 0x00010000   /* downstream I/O (64 KiB) */
-			  0x82000000 0 0x50100000 0x0 0x50100000 0 0x07F00000   /* non-prefetchable memory (127 MiB) */
-			  0xc2000000 0 0x58000000 0x0 0x58000000 0 0x28000000>; /* prefetchable memory (640 MiB) */
-
-		clocks = <&bpmp TEGRA186_CLK_AFI>,
-			 <&bpmp TEGRA186_CLK_PCIE>,
-			 <&bpmp TEGRA186_CLK_PLLE>;
-		clock-names = "afi", "pex", "pll_e";
-
-		resets = <&bpmp TEGRA186_RESET_AFI>,
-			 <&bpmp TEGRA186_RESET_PCIE>,
-			 <&bpmp TEGRA186_RESET_PCIEXCLK>;
-		reset-names = "afi", "pex", "pcie_x";
-
-		status = "disabled";
-
-		pci@1,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82000800 0 0x10000000 0 0x1000>;
-			reg = <0x000800 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <2>;
-		};
-
-		pci@2,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82001000 0 0x10001000 0 0x1000>;
-			reg = <0x001000 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <1>;
-		};
-
-		pci@3,0 {
-			device_type = "pci";
-			assigned-addresses = <0x82001800 0 0x10004000 0 0x1000>;
-			reg = <0x001800 0 0 0 0>;
-			status = "disabled";
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-
-			nvidia,num-lanes = <1>;
-		};
-	};
-
-Board DTS:
-
-	pcie@10003000 {
-		status = "okay";
-
-		dvdd-pex-supply = <&vdd_pex>;
-		hvdd-pex-pll-supply = <&vdd_1v8>;
-		hvdd-pex-supply = <&vdd_1v8>;
-		vddio-pexctl-aud-supply = <&vdd_1v8>;
-
-		pci@1,0 {
-			nvidia,num-lanes = <4>;
-			status = "okay";
-		};
-
-		pci@2,0 {
-			nvidia,num-lanes = <0>;
-			status = "disabled";
-		};
-
-		pci@3,0 {
-			nvidia,num-lanes = <1>;
-			status = "disabled";
-		};
-	};
diff --git a/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.yaml b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.yaml
new file mode 100644
index 00000000000000..fcc7875e685300
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.yaml
@@ -0,0 +1,532 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pci/nvidia,tegra20-pcie.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NVIDIA Tegra PCIe controller
+
+maintainers:
+  - Thierry Reding <thierry.reding@gmail.com>
+  - Jon Hunter <jonathanh@nvidia.com>
+
+properties:
+  compatible:
+    oneOf:
+      - enum:
+          - nvidia,tegra20-pcie
+          - nvidia,tegra30-pcie
+          - nvidia,tegra124-pcie
+          - nvidia,tegra210-pcie
+          - nvidia,tegra186-pcie
+
+      - items:
+          - const: nvidia,tegra132-pcie
+          - const: nvidia,tegra124-pcie
+
+  reg:
+    items:
+      - description: PADS registers
+      - description: AFI registers
+      - description: configuration space region
+
+  reg-names:
+    items:
+      - const: pads
+      - const: afi
+      - const: cs
+
+  interrupts:
+    items:
+      - description: PCIe controller interrupt
+      - description: MSI controller interrupt
+
+  interrupt-names:
+    items:
+      - const: intr
+      - const: msi
+
+  clocks:
+    items:
+      - description: PCI controller clock
+      - description: AFI interface clock
+      - description: reference PLL clock
+      - description: CML clock
+    minItems: 3
+
+  clock-names:
+    items:
+      - const: pex
+      - const: afi
+      - const: pll_e
+      - const: cml
+    minItems: 3
+
+  resets:
+    items:
+      - description: AFI interface reset
+      - description: PCI controller reset
+      - description: PCI bus reset
+
+  reset-names:
+    items:
+      - const: pex
+      - const: afi
+      - const: pcie_x
+
+  interconnects:
+    items:
+      - description: AFI memory read client
+      - description: AFI memory write client
+
+  interconnect-names:
+    items:
+      - const: dma-mem # read
+      - const: write
+
+  iommus:
+    maxItems: 1
+
+  operating-points-v2:
+    $ref: /schemas/types.yaml#/definitions/phandle
+
+  power-domains:
+    items:
+      - description: phandle to the core power domain
+
+  pinctrl-names:
+    items:
+        # active state, puts PCIe I/O out of deep power down state
+      - const: default
+        # puts PCIe I/O into deep power down state
+      - const: idle
+
+patternProperties:
+  "^pci@1?[0-9a-f](,[0-7])?$":
+    description: Root ports are defined as subnodes of the PCIe controller
+      node.
+
+      Note that devices on the PCI bus are dynamically discovered using PCI's
+      bus enumeration and therefore don't need corresponding device nodes in
+      DT. However if a device on the PCI bus provides a non-probeable bus such
+      as I2C or SPI, device nodes need to be added in order to allow the bus'
+      children to be instantiated at the proper location in the operating
+      system's device tree (as illustrated by the optional nodes in the
+      examples below).
+
+    $ref: /schemas/pci/pci-pci-bridge.yaml#
+    unevaluatedProperties: false
+
+    properties:
+      nvidia,num-lanes:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description: |
+          Number of lanes to use for this port. Valid combinations
+          are:
+
+          - Root port 0 uses 4 lanes, root port 1 is unused.
+          - Both root ports use 2 lanes.
+        enum: [ 0, 1, 2, 4 ]
+
+      phys:
+        description: Must contain an phandle to a PHY for each entry in
+          phy-names.
+        minItems: 1
+        maxItems: 4
+
+      phy-names:
+        description: Must include an entry for each active lane. Note that the
+          number of entries does not have to (though usually will) be equal to
+          the specified number of lanes in the nvidia,num-lanes property.
+          Entries are of the form "pcie-N", where N ranges from 0 to the value
+          specified in nvidia,num-lanes.
+        items:
+          enum:
+            - pcie-0
+            - pcie-1
+            - pcie-2
+            - pcie-3
+        minItems: 1
+        maxItems: 4
+
+required:
+  - reg
+  - reg-names
+  - interrupts
+  - interrupt-names
+  - clocks
+  - clock-names
+  - resets
+  - reset-names
+
+unevaluatedProperties: false
+
+allOf:
+  - $ref: /schemas/pinctrl/pinctrl-consumer.yaml#
+  - $ref: /schemas/pci/pci-host-bridge.yaml#
+  - $ref: pci-iommu.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: nvidia,tegra20-pcie
+    then:
+      properties:
+        clocks:
+          maxItems: 3
+
+        clock-names:
+          maxItems: 3
+
+        avdd-pex-supply:
+          description: Power supply for analog PCIe logic. Must supply 1.05 V.
+
+        vdd-pex-supply:
+          description: Power supply for digital PCIe I/O. Must supply 1.05 V.
+
+        avdd-pex-pll-supply:
+          description: Power supply for dedicated (internal) PCIe PLL. Must
+            supply 1.05 V.
+
+        avdd-plle-supply:
+          description: Power supply for PLLE, which is shared with SATA. Must
+            supply 1.05 V.
+
+        vddio-pex-clk-supply:
+          description: Power supply for PCIe clock. Must supply 3.3 V.
+
+      required:
+        - avdd-pex-supply
+        - vdd-pex-supply
+        - avdd-pex-pll-supply
+        - avdd-plle-supply
+        - vddio-pex-clk-supply
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: nvidia,tegra30-pcie
+    then:
+      properties:
+        avdd-pex-pll-supply:
+          description: Power supply for dedicated (internal) PCIe PLL. Must
+            supply 1.05 V.
+
+        avdd-plle-supply:
+          description: Power supply for PLLE, which is shared with SATA. Must
+            supply 1.05 V.
+
+        vddio-pex-ctl-supply:
+          description: Power supply for PCIe control I/O partition. Must
+            supply 1.8 V.
+
+        hvdd-pex-supply:
+          description: High-voltage supply for PCIe I/O and PCIe output
+            clocks. Must supply 3.3 V.
+
+        avdd-pexa-supply:
+          description: Power supply for analog PCIe logic. Must supply 1.05 V.
+            Required if lanes 0 through 3 are used.
+
+        vdd-pexa-supply:
+          description: Power supply for digital PCIe I/O. Must supply 1.05 V.
+            Required if lanes 0 through 3 are used.
+
+        avdd-pexb-supply:
+          description: Power supply for analog PCIe logic. Must supply 1.05 V.
+            Required if lanes 4 and 5 are used.
+
+        vdd-pexb-supply:
+          description: Power supply for digital PCIe I/O. Must supply 1.05 V.
+            Required if lanes 4 and 5 are used.
+
+      required:
+        - avdd-pex-pll-supply
+        - avdd-plle-supply
+        - vddio-pex-ctl-supply
+        - hvdd-pex-supply
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: nvidia,tegra124-pcie
+    then:
+      properties:
+        phys:
+          minItems: 1
+          maxItems: 1
+          deprecated: true
+
+        phy-names:
+          items:
+            - const: pcie
+          deprecated: true
+
+        avddio-pex-supply:
+          description: Power supply for analog PCIe logic. Must supply 1.05 V.
+
+        dvddio-pex-supply:
+          description: Power supply for digital PCIe I/O. Must supply 1.05 V.
+
+        hvdd-pex-supply:
+          description: High-voltage supply for PCIe I/O and PCIe output
+            clocks. Must supply 3.3 V.
+
+        vddio-pex-ctl-supply:
+          description: Power supply for PCIe control I/O partition. Must
+            supply 2.8-3.3 V.
+
+        avdd-pex-pll-supply:
+          deprecated: true
+
+        hvdd-pex-pll-e-supply:
+          deprecated: true
+
+        avdd-pll-erefe-supply:
+          deprecated: true
+
+      required:
+        - avddio-pex-supply
+        - dvddio-pex-supply
+        - hvdd-pex-supply
+        - vddio-pex-ctl-supply
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: nvidia,tegra210-pcie
+    then:
+      properties:
+        hvddio-pex-supply:
+          description: High-voltage supply for PCIe I/O and PCIe output
+            clocks. Must supply 1.8 V.
+
+        dvddio-pex-supply:
+          description: Power supply for digital PCIe I/O. Must supply 1.05 V.
+
+        vddio-pex-ctl-supply:
+          description: Power supply for PCIe control I/O partition. Must
+            supply 1.8 V.
+
+        avdd-pll-uerefe-supply:
+          deprecated: true
+
+        dvdd-pex-pll-supply:
+          deprecated: true
+
+        hvdd-pex-pll-e-supply:
+          deprecated: true
+
+      required:
+        - hvddio-pex-supply
+        - dvddio-pex-supply
+        - vddio-pex-ctl-supply
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: nvidia,tegra186-pcie
+    then:
+      properties:
+        clocks:
+          maxItems: 3
+
+        clock-names:
+          maxItems: 3
+
+        power-domains:
+          description: To ungate power partition by BPMP powergate driver.
+            Must contain BPMP phandle and PCIe power partition ID.
+
+        dvdd-pex-supply:
+          description: Power supply for digital PCIe I/O. Must supply 1.05 V.
+
+        hvdd-pex-pll-supply:
+          description: High-voltage supply for PLLE (shared with USB3). Must
+            supply 1.8 V.
+
+        hvdd-pex-supply:
+          description: High-voltage supply for PCIe I/O and PCIe output
+            clocks. Must supply 1.8 V.
+
+        vddio-pexctl-aud-supply:
+          description: Power supply for PCIe side band signals. Must supply
+            1.8 V.
+
+      required:
+        - dvdd-pex-supply
+        - hvdd-pex-pll-supply
+        - hvdd-pex-supply
+        - vddio-pexctl-aud-supply
+
+examples:
+  # Tegra20
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    pcie@80003000 {
+        compatible = "nvidia,tegra20-pcie";
+        device_type = "pci";
+        reg = <0x80003000 0x00000800>, /* PADS registers */
+              <0x80003800 0x00000200>, /* AFI registers */
+              <0x90000000 0x10000000>; /* configuration space */
+        reg-names = "pads", "afi", "cs";
+        interrupts = <0 98 0x04>, /* controller interrupt */
+                     <0 99 0x04>; /* MSI interrupt */
+        interrupt-names = "intr", "msi";
+
+        #interrupt-cells = <1>;
+        interrupt-map-mask = <0 0 0 0>;
+        interrupt-map = <0 0 0 0 &intc GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+        bus-range = <0x00 0xff>;
+        #address-cells = <3>;
+        #size-cells = <2>;
+
+        ranges = <0x02000000 0 0x80000000 0x80000000 0 0x00001000>, /* port 0 registers */
+                 <0x02000000 0 0x80001000 0x80001000 0 0x00001000>, /* port 1 registers */
+                 <0x01000000 0 0          0x82000000 0 0x00010000>, /* downstream I/O */
+                 <0x02000000 0 0xa0000000 0xa0000000 0 0x10000000>, /* non-prefetchable memory */
+                 <0x42000000 0 0xb0000000 0xb0000000 0 0x10000000>; /* prefetchable memory */
+
+        clocks = <&tegra_car 70>, <&tegra_car 72>, <&tegra_car 118>;
+        clock-names = "pex", "afi", "pll_e";
+        resets = <&tegra_car 70>, <&tegra_car 72>, <&tegra_car 74>;
+        reset-names = "pex", "afi", "pcie_x";
+
+        avdd-pex-supply = <&pci_vdd_reg>;
+        vdd-pex-supply = <&pci_vdd_reg>;
+        avdd-pex-pll-supply = <&pci_vdd_reg>;
+        avdd-plle-supply = <&pci_vdd_reg>;
+        vddio-pex-clk-supply = <&pci_clk_reg>;
+
+        /* root port 00:01.0 */
+        pci@1,0 {
+            device_type = "pci";
+            assigned-addresses = <0x82000800 0 0x80000000 0 0x1000>;
+            reg = <0x000800 0 0 0 0>;
+            bus-range = <0x00 0xff>;
+
+            #address-cells = <3>;
+            #size-cells = <2>;
+
+            ranges;
+
+            nvidia,num-lanes = <2>;
+
+            /* bridge 01:00.0 (optional) */
+            pci@0,0 {
+                reg = <0x010000 0 0 0 0>;
+                bus-range = <0x00 0xff>;
+
+                #address-cells = <3>;
+                #size-cells = <2>;
+                ranges;
+
+                device_type = "pci";
+
+                /* endpoint 02:00.0 */
+                ethernet@0,0 {
+                    reg = <0x020000 0 0 0 0>;
+                };
+            };
+        };
+
+        pci@2,0 {
+            device_type = "pci";
+            assigned-addresses = <0x82001000 0 0x80001000 0 0x1000>;
+            reg = <0x001000 0 0 0 0>;
+            bus-range = <0x00 0xff>;
+
+            #address-cells = <3>;
+            #size-cells = <2>;
+
+            ranges;
+
+            nvidia,num-lanes = <2>;
+        };
+    };
+
+  # Tegra210
+  - |
+    #include <dt-bindings/clock/tegra210-car.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    pcie@1003000 {
+        compatible = "nvidia,tegra210-pcie";
+        device_type = "pci";
+        reg = <0x01003000 0x00000800>, /* PADS registers */
+              <0x01003800 0x00000800>, /* AFI registers */
+              <0x02000000 0x10000000>; /* configuration space */
+        reg-names = "pads", "afi", "cs";
+        interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+                     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+        interrupt-names = "intr", "msi";
+
+        #interrupt-cells = <1>;
+        interrupt-map-mask = <0 0 0 0>;
+        interrupt-map = <0 0 0 0 &gic GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+
+        bus-range = <0x00 0xff>;
+        #address-cells = <3>;
+        #size-cells = <2>;
+
+        ranges = <0x02000000 0 0x01000000 0x01000000 0 0x00001000>, /* port 0 configuration space */
+                 <0x02000000 0 0x01001000 0x01001000 0 0x00001000>, /* port 1 configuration space */
+                 <0x01000000 0 0x0        0x12000000 0 0x00010000>, /* downstream I/O (64 KiB) */
+                 <0x02000000 0 0x13000000 0x13000000 0 0x0d000000>, /* non-prefetchable memory (208 MiB) */
+                 <0x42000000 0 0x20000000 0x20000000 0 0x20000000>; /* prefetchable memory (512 MiB) */
+
+        clocks = <&tegra_car TEGRA210_CLK_PCIE>,
+                 <&tegra_car TEGRA210_CLK_AFI>,
+                 <&tegra_car TEGRA210_CLK_PLL_E>,
+                 <&tegra_car TEGRA210_CLK_CML0>;
+        clock-names = "pex", "afi", "pll_e", "cml";
+        resets = <&tegra_car 70>,
+                 <&tegra_car 72>,
+                 <&tegra_car 74>;
+        reset-names = "pex", "afi", "pcie_x";
+
+        avdd-pll-uerefe-supply = <&avdd_1v05_pll>;
+        hvddio-pex-supply = <&vdd_1v8>;
+        dvddio-pex-supply = <&vdd_pex_1v05>;
+        dvdd-pex-pll-supply = <&vdd_pex_1v05>;
+        hvdd-pex-pll-e-supply = <&vdd_1v8>;
+        vddio-pex-ctl-supply = <&vdd_1v8>;
+
+        pci@1,0 {
+            device_type = "pci";
+            assigned-addresses = <0x82000800 0 0x01000000 0 0x1000>;
+            reg = <0x000800 0 0 0 0>;
+            bus-range = <0x00 0xff>;
+
+            #address-cells = <3>;
+            #size-cells = <2>;
+            ranges;
+
+            nvidia,num-lanes = <4>;
+
+            phys = <&phy_pcie0>, <&phy_pcie1>, <&phy_pcie2>, <&phy_pcie3>;
+            phy-names = "pcie-0", "pcie-1", "pcie-2", "pcie-3";
+        };
+
+        pci@2,0 {
+            device_type = "pci";
+            assigned-addresses = <0x82001000 0 0x01001000 0 0x1000>;
+            reg = <0x001000 0 0 0 0>;
+            bus-range = <0x00 0xff>;
+
+            #address-cells = <3>;
+            #size-cells = <2>;
+            ranges;
+
+            nvidia,num-lanes = <1>;
+
+            phys = <&phy_pcie4>;
+            phy-names = "pcie-0";
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml
index b23293314a6d05..ce177f0eca753b 100644
--- a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml
+++ b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-ep.yaml
@@ -56,11 +56,14 @@ properties:
     maxItems: 1
 
   resets:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
 
   reset-names:
+    minItems: 1
     items:
       - const: pwr
+      - const: core
 
   max-link-speed:
     maximum: 4
@@ -109,8 +112,8 @@ examples:
             clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
             clock-names = "core", "ref";
             power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
-            resets = <&cpg 624>;
-            reset-names = "pwr";
+            resets = <&cpg 624>,<&cpg 1121>;
+            reset-names = "pwr", "core";
             max-link-speed = <4>;
             num-lanes = <2>;
             max-functions = /bits/ 8 <2>;
diff --git a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml
index 64f456c6a75a18..362c43a1a15d09 100644
--- a/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml
+++ b/Documentation/devicetree/bindings/pci/rcar-gen4-pci-host.yaml
@@ -10,30 +10,20 @@ title: Renesas R-Car Gen4 PCIe Host
 maintainers:
   - Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>
 
-allOf:
-  - $ref: snps,dw-pcie.yaml#
-
 properties:
   compatible:
-    items:
-      - enum:
-          - renesas,r8a779f0-pcie      # R-Car S4-8
-          - renesas,r8a779g0-pcie      # R-Car V4H
-          - renesas,r8a779h0-pcie      # R-Car V4M
-      - const: renesas,rcar-gen4-pcie  # R-Car Gen4
-
-  reg:
-    maxItems: 7
-
-  reg-names:
-    items:
-      - const: dbi
-      - const: dbi2
-      - const: atu
-      - const: dma
-      - const: app
-      - const: phy
-      - const: config
+    oneOf:
+      - items:
+          - enum:
+              - renesas,r8a779f0-pcie      # R-Car S4-8
+              - renesas,r8a779g0-pcie      # R-Car V4H
+              - renesas,r8a779h0-pcie      # R-Car V4M
+          - const: renesas,rcar-gen4-pcie  # R-Car Gen4
+
+      - items:
+          - enum:
+              - renesas,r8a78000-pcie4     # R-Car X5H PCIe4
+          - const: renesas,rcar-gen5-pcie4 # R-Car Gen5 PCIe4
 
   interrupts:
     maxItems: 4
@@ -57,11 +47,14 @@ properties:
     maxItems: 1
 
   resets:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
 
   reset-names:
+    minItems: 1
     items:
       - const: pwr
+      - const: core
 
   max-link-speed:
     maximum: 4
@@ -84,6 +77,46 @@ required:
   - resets
   - reset-names
 
+allOf:
+  - $ref: snps,dw-pcie.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,rcar-gen4-pcie
+    then:
+      properties:
+        reg:
+          maxItems: 7
+
+        reg-names:
+          items:
+            - const: dbi
+            - const: dbi2
+            - const: atu
+            - const: dma
+            - const: app
+            - const: phy
+            - const: config
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,rcar-gen5-pcie4
+    then:
+      properties:
+        reg:
+          maxItems: 6
+
+        reg-names:
+          items:
+            - const: dbi
+            - const: dbi2
+            - const: atu
+            - const: dma
+            - const: app
+            - const: config
+
 unevaluatedProperties: false
 
 examples:
@@ -111,8 +144,8 @@ examples:
             clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
             clock-names = "core", "ref";
             power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
-            resets = <&cpg 624>;
-            reset-names = "pwr";
+            resets = <&cpg 624>,<&cpg 1121>;
+            reset-names = "pwr", "core";
             max-link-speed = <4>;
             num-lanes = <2>;
             #address-cells = <3>;
diff --git a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
index 09d78bd1a5770e..7827d81680fed3 100644
--- a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
@@ -13,7 +13,7 @@ description: |
   PCIe host controller found in Renesas RZ/G3S and similar SoCs complies
   with PCIe Base Specification 4.0 and supports different link speeds
   depending on the SoC variant:
-    - Gen2 (5 GT/s): RZ/G3S
+    - Gen2 (5 GT/s): RZ/G3{L,S}
     - Gen3 (8 GT/s): RZ/G3E, RZ/V2H(P), RZ/V2N
 
 properties:
@@ -21,6 +21,7 @@ properties:
     oneOf:
       - enum:
           - renesas,r9a08g045-pcie # RZ/G3S
+          - renesas,r9a08g046-pcie # RZ/G3L
           - renesas,r9a09g047-pcie # RZ/G3E
           - renesas,r9a09g057-pcie # RZ/V2H(P)
       - items:
@@ -87,14 +88,18 @@ properties:
   interrupt-controller: true
 
   clocks:
+    minItems: 2
     items:
       - description: System clock
       - description: PM control clock or clock for L1 substate handling
+      - description: PM control clock
 
   clock-names:
+    minItems: 2
     items:
       - const: aclk
       - enum: [pm, pmu]
+      - const: pmu
 
   resets:
     minItems: 1
@@ -215,6 +220,8 @@ allOf:
           maxItems: 16
         interrupt-names:
           maxItems: 16
+        clocks:
+          maxItems: 2
         clock-names:
           items:
             - const: aclk
@@ -223,6 +230,28 @@ allOf:
           minItems: 7
         reset-names:
           minItems: 7
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a08g046-pcie
+    then:
+      properties:
+        interrupts:
+          maxItems: 16
+        interrupt-names:
+          maxItems: 16
+        clocks:
+          minItems: 3
+        clock-names:
+          items:
+            - const: aclk
+            - const: pm
+            - const: pmu
+        resets:
+          maxItems: 1
+        reset-names:
+          maxItems: 1
   - if:
       properties:
         compatible:
@@ -236,6 +265,8 @@ allOf:
           minItems: 23
         interrupt-names:
           minItems: 23
+        clocks:
+          maxItems: 2
         clock-names:
           items:
             - const: aclk
diff --git a/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml b/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml
index f084830c6d0cb3..c4ebb99f50c37a 100644
--- a/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml
+++ b/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml
@@ -31,6 +31,11 @@ properties:
     description:
       GPIO controlling the RESX# pin.
 
+  gpio-controller: true
+
+  '#gpio-cells':
+    const: 2
+
   vdd18-supply: true
 
   vdd09-supply: true
@@ -128,7 +133,7 @@ examples:
             ranges;
             bus-range = <0x01 0xff>;
 
-            pcie@0,0 {
+            tc9563: pcie@0,0 {
                 compatible = "pci1179,0623";
 
                 reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -149,6 +154,9 @@ examples:
 
                 resx-gpios = <&gpio 1 GPIO_ACTIVE_LOW>;
 
+                gpio-controller;
+                #gpio-cells = <2>;
+
                 pcie@1,0 {
                     compatible = "pciclass,0604";
                     reg = <0x20800 0x0 0x0 0x0 0x0>;
@@ -158,6 +166,8 @@ examples:
                     ranges;
                     bus-range = <0x03 0xff>;
 
+                    reset-gpios = <&tc9563 2 GPIO_ACTIVE_LOW>;
+
                     toshiba,no-dfe-support;
                 };
 
@@ -170,6 +180,8 @@ examples:
                     ranges;
                     bus-range = <0x04 0xff>;
 
+                    reset-gpios = <&tc9563 3 GPIO_ACTIVE_LOW>;
+
                     toshiba,tx-amplitude-microvolt = <10>;
                 };
 
diff --git a/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml b/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml
index 9823456addea31..a28a88dc8e8839 100644
--- a/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml
+++ b/Documentation/devicetree/bindings/pci/xilinx-versal-cpm.yaml
@@ -41,6 +41,10 @@ properties:
     description:
       Maps a Requester ID to an MSI controller and associated MSI sideband data.
 
+  resets:
+    description: Reset for the PCIe controller.
+    maxItems: 1
+
   ranges:
     maxItems: 2
 
@@ -61,6 +65,17 @@ properties:
 
       interrupt-controller: true
 
+patternProperties:
+  '^pcie@[0-2],0$':
+    type: object
+    $ref: /schemas/pci/pci-pci-bridge.yaml#
+
+    properties:
+      reg:
+        maxItems: 1
+
+    unevaluatedProperties: false
+
 required:
   - reg
   - reg-names
@@ -76,6 +91,9 @@ unevaluatedProperties: false
 
 examples:
   - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/reset/xlnx-versal-resets.h>
+
     versal {
         #address-cells = <2>;
         #size-cells = <2>;
@@ -99,6 +117,16 @@ examples:
             reg = <0x0 0xfca10000 0x0 0x1000>,
                   <0x6 0x00000000 0x0 0x10000000>;
             reg-names = "cpm_slcr", "cfg";
+            resets = <&versal_reset VERSAL_RST_PCIE_CORE0>;
+
+            pcie@0,0 {
+                device_type = "pci";
+                reg = <0x0 0x0 0x0 0x0 0x0>;
+                reset-gpios = <&gpio1 38 GPIO_ACTIVE_LOW>;
+                #address-cells = <3>;
+                #size-cells = <2>;
+                ranges;
+            };
             pcie_intc_0: interrupt-controller {
                     #address-cells = <0>;
                     #interrupt-cells = <1>;
@@ -127,6 +155,16 @@ examples:
                   <0x06 0x00000000 0x00 0x1000000>,
                   <0x00 0xfce20000 0x00 0x1000000>;
             reg-names = "cpm_slcr", "cfg", "cpm_csr";
+            resets = <&versal_reset VERSAL_RST_PCIE_CORE0>;
+
+            pcie@0,0 {
+                device_type = "pci";
+                reg = <0x0 0x0 0x0 0x0 0x0>;
+                reset-gpios = <&gpio1 38 GPIO_ACTIVE_LOW>;
+                #address-cells = <3>;
+                #size-cells = <2>;
+                ranges;
+            };
 
             pcie_intc_1: interrupt-controller {
                 #address-cells = <0>;
diff --git a/Documentation/devicetree/bindings/phy/mediatek,mt8195-dp-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,mt8195-dp-phy.yaml
new file mode 100644
index 00000000000000..1c2dc4f03cd6dd
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/mediatek,mt8195-dp-phy.yaml
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/mediatek,mt8195-dp-phy.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek SoC DisplayPort Transmitter PHY
+
+maintainers:
+  - AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+
+properties:
+  compatible:
+    enum:
+      - mediatek,mt8195-dp-phy
+      - mediatek,mt8196-edp-phy
+
+  reg:
+    maxItems: 1
+
+  "#phy-cells":
+    const: 0
+
+  nvmem-cells:
+    description: PHY calibrations from eFuse for optimal EYE signal pattern
+    items:
+      - description: PHY-Global Reference Bias trim
+      - description: PHY-Global AUX Transmitter clock impedance adjustment
+      - description: Lane 0 Transmitter impedance selection (P-MOSFET)
+      - description: Lane 0 Transmitter impedance selection (N-MOSFET)
+      - description: Lane 1 Transmitter impedance selection (P-MOSFET)
+      - description: Lane 1 Transmitter impedance selection (N-MOSFET)
+      - description: Lane 2 Transmitter impedance selection (P-MOSFET)
+      - description: Lane 2 Transmitter impedance selection (N-MOSFET)
+      - description: Lane 3 Transmitter impedance selection (P-MOSFET)
+      - description: Lane 3 Transmitter impedance selection (N-MOSFET)
+
+  nvmem-cell-names:
+    items:
+      - const: rbias-trim
+      - const: impedance-txclk
+      - const: impedance-lane0p
+      - const: impedance-lane0n
+      - const: impedance-lane1p
+      - const: impedance-lane1n
+      - const: impedance-lane2p
+      - const: impedance-lane2n
+      - const: impedance-lane3p
+      - const: impedance-lane3n
+
+  power-domains:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    phy@1c500000 {
+        compatible = "mediatek,mt8195-dp-phy";
+        reg = <0x1c500000 0x2000>;
+        #phy-cells = <0>;
+        nvmem-cells = <&edp_glb_bias_trim>, <&edp_clktx_impsel>,
+                      <&edp_imp_ln0_pmos>, <&edp_imp_ln0_nmos>,
+                      <&edp_imp_ln1_pmos>, <&edp_imp_ln1_nmos>,
+                      <&edp_imp_ln2_pmos>, <&edp_imp_ln2_nmos>,
+                      <&edp_imp_ln3_pmos>, <&edp_imp_ln3_nmos>;
+        nvmem-cell-names = "rbias-trim", "impedance-txclk",
+                           "impedance-lane0p", "impedance-lane0n",
+                           "impedance-lane1p", "impedance-lane1n",
+                           "impedance-lane2p", "impedance-lane2n",
+                           "impedance-lane3p", "impedance-lane3n";
+    };
diff --git a/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml
index ff5c77ef11765c..8b240cfd537db3 100644
--- a/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml
+++ b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml
@@ -90,6 +90,7 @@ properties:
       - items:
           - enum:
               - mediatek,mt8188-tphy
+              - mediatek,mt8189-tphy
               - mediatek,mt8195-tphy
           - const: mediatek,generic-tphy-v3
       - const: mediatek,mt2701-u3phy
diff --git a/Documentation/devicetree/bindings/phy/qcom,m31-eusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,m31-eusb2-phy.yaml
index c0e7e2963ce6a8..613fa5cfe9a70f 100644
--- a/Documentation/devicetree/bindings/phy/qcom,m31-eusb2-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,m31-eusb2-phy.yaml
@@ -21,6 +21,7 @@ properties:
               - qcom,glymur-m31-eusb2-phy
               - qcom,hawi-m31-eusb2-phy
               - qcom,kaanapali-m31-eusb2-phy
+              - qcom,maili-m31-eusb2-phy
           - const: qcom,sm8750-m31-eusb2-phy
       - const: qcom,sm8750-m31-eusb2-phy
 
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml
index b353b7bfb839ce..d54202f6a897de 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml
@@ -36,6 +36,7 @@ properties:
       - qcom,sdm845-qmp-pcie-phy
       - qcom,sdx55-qmp-pcie-phy
       - qcom,sdx65-qmp-gen4x2-pcie-phy
+      - qcom,shikra-qmp-gen2x1-pcie-phy
       - qcom,sm8150-qmp-gen3x1-pcie-phy
       - qcom,sm8150-qmp-gen3x2-pcie-phy
       - qcom,sm8250-qmp-gen3x1-pcie-phy
@@ -169,6 +170,7 @@ allOf:
               - qcom,sdm845-qhp-pcie-phy
               - qcom,sdm845-qmp-pcie-phy
               - qcom,sdx55-qmp-pcie-phy
+              - qcom,shikra-qmp-gen2x1-pcie-phy
               - qcom,sm8150-qmp-gen3x1-pcie-phy
               - qcom,sm8150-qmp-gen3x2-pcie-phy
               - qcom,sm8250-qmp-gen3x1-pcie-phy
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
index 8adfc78108fef3..43e90995709f97 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
@@ -16,6 +16,10 @@ description:
 properties:
   compatible:
     oneOf:
+      - items:
+          - enum:
+              - qcom,maili-qmp-usb3-dp-phy
+          - const: qcom,hawi-qmp-usb3-dp-phy
       - items:
           - enum:
               - qcom,eliza-qmp-usb3-dp-phy
@@ -53,12 +57,7 @@ properties:
 
   clock-names:
     minItems: 4
-    items:
-      - const: aux
-      - const: ref
-      - const: com_aux
-      - const: usb3_pipe
-      - const: cfg_ahb
+    maxItems: 5
 
   power-domains:
     maxItems: 1
@@ -189,21 +188,59 @@ allOf:
   - if:
       properties:
         compatible:
-          enum:
-            - qcom,sc7180-qmp-usb3-dp-phy
-            - qcom,sdm845-qmp-usb3-dp-phy
+          contains:
+            enum:
+              - qcom,x1e80100-qmp-usb3-dp-phy
     then:
       properties:
         clocks:
           maxItems: 5
         clock-names:
+          items:
+            - const: aux
+            - const: ref
+            - const: com_aux
+            - const: usb3_pipe
+            - const: p2rr2p_pipe
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,sc7180-qmp-usb3-dp-phy
+              - qcom,sdm845-qmp-usb3-dp-phy
+    then:
+      properties:
+        clocks:
           maxItems: 5
-    else:
+        clock-names:
+          items:
+            - const: aux
+            - const: ref
+            - const: com_aux
+            - const: usb3_pipe
+            - const: cfg_ahb
+
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              enum:
+                - qcom,x1e80100-qmp-usb3-dp-phy
+                - qcom,sc7180-qmp-usb3-dp-phy
+                - qcom,sdm845-qmp-usb3-dp-phy
+    then:
       properties:
         clocks:
           maxItems: 4
         clock-names:
-          maxItems: 4
+          items:
+            - const: aux
+            - const: ref
+            - const: com_aux
+            - const: usb3_pipe
 
   - if:
       properties:
diff --git a/Documentation/devicetree/bindings/phy/qcom,x1e80100-csi2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,x1e80100-csi2-phy.yaml
new file mode 100644
index 00000000000000..dc64af97da6c65
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/qcom,x1e80100-csi2-phy.yaml
@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/qcom,x1e80100-csi2-phy.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm X1E80100 SoC CSI2 PHY
+
+maintainers:
+  - Bryan O'Donoghue <bod@kernel.org>
+
+description:
+  Qualcomm MIPI CSI2 C-PHY/D-PHY combination PHY. Connects MIPI CSI2 sensors
+  to Qualcomm's Camera CSI Decoder. The PHY supports both C-PHY and D-PHY
+  modes.
+
+properties:
+  compatible:
+    const: qcom,x1e80100-csi2-phy
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 3
+
+  clock-names:
+    items:
+      - const: core
+      - const: timer
+      - const: ahb
+
+  interrupts:
+    maxItems: 1
+
+  operating-points-v2: true
+
+  opp-table:
+    type: object
+
+  power-domains:
+    items:
+      - description: Titan Top GDSC - Titan ISP Block, Global Distributed Switch Controller.
+      - description: MMCX voltage rail
+      - description: MXC or MXA voltage rail
+
+  power-domain-names:
+    items:
+      - const: top
+      - const: mmcx
+      - const: mx
+
+  vdda-0p9-supply:
+    description: Phandle to a 0.9V regulator supply to a PHY.
+
+  vdda-1p2-supply:
+    description: Phandle to 1.2V regulator supply to a PHY.
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        description:
+          Sensor input. Always present. A single sensor is described by a
+          single endpoint with one to four data lanes, numbered 1-4. The
+          clock lane is position 0.
+
+          DPHY split mode, where two independent sensors share the PHY, is
+          described by two endpoints. endpoint@0 uses data lanes 1 or 1-2,
+          clock lane position 0; endpoint@1 uses data lane 3, clock lane
+          position 4.
+
+        unevaluatedProperties: false
+
+        patternProperties:
+          "^endpoint(@[01])?$":
+            $ref: /schemas/media/video-interfaces.yaml#
+            unevaluatedProperties: false
+            properties:
+              bus-type:
+                enum:
+                  - 1 # MEDIA_BUS_TYPE_CSI2_CPHY
+                  - 4 # MEDIA_BUS_TYPE_CSI2_DPHY
+                default: 4
+
+              data-lanes:
+                minItems: 1
+                maxItems: 4
+                items:
+                  minimum: 1
+                  maximum: 4
+                description:
+                  Data lane positions following this endpoint's clock
+                  lane. Positions are a per-port logical numbering, not
+                  package pad or ball positions.
+
+            required:
+              - data-lanes
+              - remote-endpoint
+
+        allOf:
+          - if:
+              required:
+                - endpoint@1
+            then:
+              properties:
+                endpoint@0:
+                  properties:
+                    data-lanes:
+                      maxItems: 2
+                      items:
+                        maximum: 2
+                endpoint@1:
+                  properties:
+                    data-lanes:
+                      items:
+                        - const: 3
+              required:
+                - endpoint@0
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: Output to the CAMSS CSID controller.
+
+    required:
+      - port@0
+      - port@1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - interrupts
+  - operating-points-v2
+  - power-domains
+  - power-domain-names
+  - vdda-0p9-supply
+  - vdda-1p2-supply
+  - ports
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,x1e80100-camcc.h>
+    #include <dt-bindings/clock/qcom,x1e80100-gcc.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/media/video-interfaces.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
+
+    phy@ace4000 {
+        compatible = "qcom,x1e80100-csi2-phy";
+        reg = <0x0ace4000 0x2000>;
+
+        clocks = <&camcc CAM_CC_CSIPHY0_CLK>,
+                 <&camcc CAM_CC_CSI0PHYTIMER_CLK>,
+                 <&camcc CAM_CC_CORE_AHB_CLK>;
+        clock-names = "core",
+                      "timer",
+                      "ahb";
+
+        interrupts = <GIC_SPI 477 IRQ_TYPE_EDGE_RISING>;
+
+        operating-points-v2 = <&csiphy_opp_table>;
+
+        power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>,
+                        <&rpmhpd RPMHPD_MMCX>,
+                        <&rpmhpd RPMHPD_MX>;
+        power-domain-names = "top",
+                             "mmcx",
+                             "mx";
+
+        vdda-0p9-supply = <&vreg_l2c_0p9>;
+        vdda-1p2-supply = <&vreg_l1c_1p2>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                csiphy0_in: endpoint {
+                    bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+                    data-lanes = <1 2 3 4>;
+                    remote-endpoint = <&sensor_out>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                csiphy0_out: endpoint {
+                    remote-endpoint = <&csid_in>;
+                };
+            };
+        };
+
+        csiphy_opp_table: opp-table {
+            compatible = "operating-points-v2";
+
+            opp-300000000 {
+                opp-hz = /bits/ 64 <300000000>;
+                required-opps = <&rpmhpd_opp_low_svs_d1>,
+                                <&rpmhpd_opp_low_svs_d1>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/phy/ti,am335x-usb-phy.yaml b/Documentation/devicetree/bindings/phy/ti,am335x-usb-phy.yaml
new file mode 100644
index 00000000000000..b28cd42ece5e04
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/ti,am335x-usb-phy.yaml
@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/ti,am335x-usb-phy.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI AM335x USB PHY
+
+maintainers:
+  - Sebastian Andrzej Siewior <bigeasy@linutronix.de>
+
+properties:
+  compatible:
+    const: ti,am335x-usb-phy
+
+  reg:
+    maxItems: 1
+
+  reg-names:
+    const: phy
+
+  ti,ctrl_mod:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: Reference to the reset module node.
+
+  "#phy-cells":
+    const: 0
+
+  vcc-supply:
+    description: Phandle to the regulator that provides power to the PHY.
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - ti,ctrl_mod
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    usb-phy@47401300 {
+        compatible = "ti,am335x-usb-phy";
+        reg = <0x47401300 0x100>;
+        reg-names = "phy";
+        ti,ctrl_mod = <&ctrl_mod>;
+        #phy-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/phy/ti,tcan104x-can.yaml b/Documentation/devicetree/bindings/phy/ti,tcan104x-can.yaml
index 9f5c37ca6496a1..0cf5fa21106b67 100644
--- a/Documentation/devicetree/bindings/phy/ti,tcan104x-can.yaml
+++ b/Documentation/devicetree/bindings/phy/ti,tcan104x-can.yaml
@@ -71,10 +71,11 @@ allOf:
   - if:
       properties:
         compatible:
-          enum:
-            - nxp,tjr1443
-            - ti,tcan1042
-            - ti,tcan1043
+          contains:
+            enum:
+              - nxp,tjr1443
+              - ti,tcan1042
+              - ti,tcan1043
     then:
       properties:
         '#phy-cells':
diff --git a/Documentation/devicetree/bindings/pinctrl/allwinner,sun55i-a523-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/allwinner,sun55i-a523-pinctrl.yaml
index f87b8274cc372f..d7d0ba32ced4dc 100644
--- a/Documentation/devicetree/bindings/pinctrl/allwinner,sun55i-a523-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/allwinner,sun55i-a523-pinctrl.yaml
@@ -4,7 +4,7 @@
 $id: http://devicetree.org/schemas/pinctrl/allwinner,sun55i-a523-pinctrl.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Allwinner A523 Pin Controller
+title: Allwinner A523 & A733 Pin Controller
 
 maintainers:
   - Andre Przywara <andre.przywara@arm.com>
@@ -28,6 +28,8 @@ properties:
     enum:
       - allwinner,sun55i-a523-pinctrl
       - allwinner,sun55i-a523-r-pinctrl
+      - allwinner,sun60i-a733-pinctrl
+      - allwinner,sun60i-a733-r-pinctrl
 
   reg:
     maxItems: 1
@@ -126,6 +128,7 @@ allOf:
         compatible:
           enum:
             - allwinner,sun55i-a523-pinctrl
+            - allwinner,sun60i-a733-pinctrl
 
     then:
       properties:
@@ -138,6 +141,7 @@ allOf:
         compatible:
           enum:
             - allwinner,sun55i-a523-r-pinctrl
+            - allwinner,sun60i-a733-r-pinctrl
 
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml
index f08d2c4f57841e..bc6d41b63b5d1f 100644
--- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml
@@ -23,6 +23,7 @@ properties:
               - apple,t6030-pinctrl
               - apple,t6031-pinctrl
               - apple,t8122-pinctrl
+              - apple,t8132-pinctrl
           - const: apple,t8103-pinctrl
       - items:
           # Do not add additional SoC to this list.
diff --git a/Documentation/devicetree/bindings/pinctrl/awinic,aw9523-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/awinic,aw9523-pinctrl.yaml
index 98c310a3c63cda..1c09c0fd493004 100644
--- a/Documentation/devicetree/bindings/pinctrl/awinic,aw9523-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/awinic,aw9523-pinctrl.yaml
@@ -108,32 +108,32 @@ examples:
         #size-cells = <0>;
 
         aw9523: gpio-expander@58 {
-                compatible = "awinic,aw9523-pinctrl";
-                reg = <0x58>;
-                interrupt-parent = <&tlmm>;
-                interrupts = <50 IRQ_TYPE_EDGE_FALLING>;
-                gpio-controller;
-                #gpio-cells = <2>;
-                gpio-ranges = <&tlmm 0 0 16>;
-                interrupt-controller;
-                #interrupt-cells = <2>;
-                reset-gpios = <&tlmm 51 GPIO_ACTIVE_HIGH>;
-
-                keyboard-matrix-col-pins {
-                        pins = "gpio8", "gpio9", "gpio10", "gpio11",
-                               "gpio12", "gpio13", "gpio14", "gpio15";
-                        function = "gpio";
-                        input-disable;
-                        output-low;
-                };
-
-                keyboard-matrix-row-pins {
-                        pins = "gpio0", "gpio1", "gpio2", "gpio3",
-                               "gpio4", "gpio5", "gpio6", "gpio7";
-                        function = "gpio";
-                        bias-pull-up;
-                        drive-open-drain;
-                        input-enable;
-                };
+            compatible = "awinic,aw9523-pinctrl";
+            reg = <0x58>;
+            interrupt-parent = <&tlmm>;
+            interrupts = <50 IRQ_TYPE_EDGE_FALLING>;
+            gpio-controller;
+            #gpio-cells = <2>;
+            gpio-ranges = <&tlmm 0 0 16>;
+            interrupt-controller;
+            #interrupt-cells = <2>;
+            reset-gpios = <&tlmm 51 GPIO_ACTIVE_HIGH>;
+
+            keyboard-matrix-col-pins {
+                pins = "gpio8", "gpio9", "gpio10", "gpio11",
+                       "gpio12", "gpio13", "gpio14", "gpio15";
+                function = "gpio";
+                input-disable;
+                output-low;
+            };
+
+            keyboard-matrix-row-pins {
+                pins = "gpio0", "gpio1", "gpio2", "gpio3",
+                       "gpio4", "gpio5", "gpio6", "gpio7";
+                function = "gpio";
+                bias-pull-up;
+                drive-open-drain;
+                input-enable;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/pinctrl/cix,sky1-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/cix,sky1-pinctrl.yaml
index 8ed53496c386f7..4ab1f00775ccec 100644
--- a/Documentation/devicetree/bindings/pinctrl/cix,sky1-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/cix,sky1-pinctrl.yaml
@@ -86,6 +86,6 @@ examples:
                 pinmux = <CIX_PAD_GPIO012_FUNC_GPIO012>;
                 bias-pull-up;
                 drive-strength = <8>;
-           };
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/pinctrl/econet,en7528-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/econet,en7528-pinctrl.yaml
new file mode 100644
index 00000000000000..3ce1a5309d0989
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/econet,en7528-pinctrl.yaml
@@ -0,0 +1,190 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/econet,en7528-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: EcoNet EN7528 Pin Controller
+
+maintainers:
+  - Lorenzo Bianconi <lorenzo@kernel.org>
+
+description:
+  The EcoNet EN7528 pin controller is used to control SoC pins. It shares the
+  IP with the Airoha pin controllers, but its IOMUX and pin configuration
+  registers have a different layout.
+
+properties:
+  compatible:
+    const: econet,en7528-pinctrl
+
+  airoha,chip-scu:
+    description: phandle to the chip SCU syscon
+    $ref: /schemas/types.yaml#/definitions/phandle
+
+  interrupts:
+    maxItems: 1
+
+  gpio-controller: true
+
+  '#gpio-cells':
+    const: 2
+
+  gpio-ranges:
+    maxItems: 1
+
+  interrupt-controller: true
+
+  '#interrupt-cells':
+    const: 2
+
+patternProperties:
+  '-pins$':
+    type: object
+
+    patternProperties:
+      '^mux(-|$)':
+        type: object
+
+        description:
+          pinmux configuration nodes.
+
+        $ref: /schemas/pinctrl/pinmux-node.yaml
+
+        properties:
+          function:
+            description:
+              A string containing the name of the function to mux to the group.
+            enum: [gpio, jtag, pcie_reset, pcm, pcm_spi, phy1_led0,
+                   phy1_led1, phy2_led0, phy2_led1, phy3_led0, phy3_led1,
+                   phy4_led0, phy4_led1, pnand, pon, pwm, sipo, spi, uart]
+
+          groups:
+            description:
+              An array of strings. Each string contains the name of a group.
+            items:
+              enum: [pon, sipo, sipo_rclk, uart2, uart2_cts_rts, uart3,
+                     ejtag, pcm1, pcm2, spi_quad, spi_cs1, pcm_spi,
+                     pcm_spi_int, pcm_spi_rst, pcm_spi_cs1, pcm_spi_cs2,
+                     pcm_spi_cs3, pcm_spi_cs4, pcm_spi_cs5, pcm_spi_cs6,
+                     pcm_spi_cs7, pnand, gpio0, gpio1, gpio2, gpio3, gpio4,
+                     gpio5, gpio6, gpio7, gpio8, gpio9, gpio10, gpio11,
+                     gpio12, gpio13, gpio14, gpio15, gpio16, gpio17,
+                     gpio18, gpio19, gpio20, gpio21, gpio22, gpio23,
+                     gpio24, gpio25, gpio26, gpio27, gpio28, gpio29,
+                     gpio30, gpio31, gpio32, gpio33, gpio34, gpio35,
+                     gpio36, gpio37, gpio38, gpio39, gpio40, gpio41,
+                     pcie_reset0, pcie_reset1]
+
+        required:
+          - function
+          - groups
+
+        additionalProperties: false
+
+      '^conf(-|$)':
+        type: object
+
+        description:
+          pinconf configuration nodes.
+
+        $ref: /schemas/pinctrl/pincfg-node.yaml
+
+        properties:
+          pins:
+            description:
+              An array of strings. Each string contains the name of a pin.
+            items:
+              enum: [gpio0, gpio1, gpio2, gpio3, gpio4, gpio5, gpio6,
+                     gpio7, gpio8, gpio9, gpio10, gpio11, gpio12, gpio13,
+                     gpio14, gpio15, gpio16, gpio17, gpio18, gpio19,
+                     gpio20, gpio21, gpio22, gpio23, gpio24, gpio25,
+                     gpio26, gpio27, gpio28, gpio29, gpio30, gpio31,
+                     gpio32, gpio33, gpio34, gpio35, gpio36, gpio37,
+                     gpio38, gpio39, pcie_reset0, pcie_reset1, i2c_sda,
+                     i2c_scl, uart_txd, uart_rxd, spi_cs0, spi_clk,
+                     spi_mosi, spi_miso]
+            minItems: 1
+            maxItems: 50
+
+          bias-disable: true
+
+          bias-pull-up: true
+
+          bias-pull-down: true
+
+          input-enable: true
+
+          output-enable: true
+
+          output-low: true
+
+          output-high: true
+
+          drive-strength:
+            description:
+              Selects the drive strength for the pin, in mA.
+            enum: [2, 4]
+
+        required:
+          - pins
+
+        additionalProperties: false
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - airoha,chip-scu
+  - interrupts
+  - gpio-controller
+  - gpio-ranges
+  - "#gpio-cells"
+  - interrupt-controller
+  - "#interrupt-cells"
+
+allOf:
+  - $ref: pinctrl.yaml#
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/mips-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    pinctrl {
+      compatible = "econet,en7528-pinctrl";
+
+      airoha,chip-scu = <&chip_scu>;
+
+      interrupt-parent = <&gic>;
+      interrupts = <GIC_SHARED 10 IRQ_TYPE_LEVEL_HIGH>;
+
+      gpio-controller;
+      #gpio-cells = <2>;
+
+      interrupt-controller;
+      #interrupt-cells = <2>;
+
+      gpio-ranges = <&pinctrl 0 0 42>;
+
+      uart2-pins {
+        mux {
+          function = "uart";
+          groups = "uart2";
+        };
+
+        conf {
+          pins = "gpio28", "gpio29";
+          bias-pull-up;
+        };
+      };
+
+      pwm-pins {
+        mux {
+          function = "pwm";
+          groups = "gpio30";
+        };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml
index 846e110062b2ad..0a69ed9880e4bd 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx35-pinctrl.yaml
@@ -179,7 +179,7 @@ examples:
 
         pinctrl_uart4: uart4grp {
             fsl,pins =
-                <0x288 0x658 0x000 0x3 0x0	0x140>,
-                <0x28c 0x65c 0x938 0x3 0x3	0x140>;
+                <0x288 0x658 0x000 0x3 0x0 0x140>,
+                <0x28c 0x65c 0x938 0x3 0x3 0x140>;
         };
     };
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.yaml
index 7bd723ab128143..7275215918bbbf 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.yaml
@@ -95,8 +95,8 @@ examples:
 
       pinctrl_uart5: uart5grp {
         fsl,pins =
-          <0x0160 0x03D0 0x0714 0x1 0x0	0x7e>,
-          <0x0164 0x03D4 0x0000 0x1 0x0	0x76>;
+          <0x0160 0x03D0 0x0714 0x1 0x0 0x7e>,
+          <0x0164 0x03D4 0x0000 0x1 0x0 0x76>;
       };
     };
   - |
@@ -106,8 +106,8 @@ examples:
       fsl,input-sel = <&iomuxc>;
 
       pinctrl_gpio_lpsr: gpio1-grp {
-          fsl,pins =
-            <0x0008 0x0038 0x0000 0x0 0x0	0x59>,
-            <0x000C 0x003C 0x0000 0x0 0x0	0x59>;
-        };
+        fsl,pins =
+          <0x0008 0x0038 0x0000 0x0 0x0 0x59>,
+          <0x000C 0x003C 0x0000 0x0 0x0 0x59>;
+      };
     };
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8m-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx8m-pinctrl.yaml
index 6068be11dfe28c..2341633943c7e5 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx8m-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx8m-pinctrl.yaml
@@ -82,8 +82,8 @@ examples:
 
         pinctrl_uart2: uart2grp {
             fsl,pins =
-                <0x23C 0x4A4 0x4FC 0x0 0x0	0x140>,
-                <0x240 0x4A8 0x000 0x0 0x0	0x140>;
+                <0x23C 0x4A4 0x4FC 0x0 0x0 0x140>,
+                <0x240 0x4A8 0x000 0x0 0x0 0x140>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx8ulp-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx8ulp-pinctrl.yaml
index 7dcf681271d3ad..dc86bf6e580f82 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx8ulp-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx8ulp-pinctrl.yaml
@@ -74,8 +74,8 @@ examples:
 
         pinctrl_lpuart5: lpuart5grp {
             fsl,pins =
-                <0x0138 0x08F0 0x4 0x3	0x3>,
-                <0x013C 0x08EC 0x4 0x3	0x3>;
+                <0x0138 0x08F0 0x4 0x3 0x3>,
+                <0x013C 0x08EC 0x4 0x3 0x3>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx9-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imx9-pinctrl.yaml
index 96e7b699527373..333b2e7fdaac98 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx9-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx9-pinctrl.yaml
@@ -78,8 +78,8 @@ examples:
 
         pinctrl_uart3: uart3grp {
             fsl,pins =
-                <0x48 0x1f8 0x41c 0x1 0x0	0x49>,
-                <0x4c 0x1fc 0x418 0x1 0x0	0x49>;
+                <0x48 0x1f8 0x41c 0x1 0x0 0x49>,
+                <0x4c 0x1fc 0x418 0x1 0x0 0x49>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1050.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1050.yaml
index db5fe66ad8733d..854fbcd8e78a3a 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1050.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1050.yaml
@@ -73,7 +73,7 @@ examples:
 
         pinctrl_lpuart1: lpuart1grp {
             fsl,pins =
-              <0x0EC 0x2DC 0x000 0x2 0x0	0xf1>,
-              <0x0F0 0x2E0 0x000 0x2 0x0	0xf1>;
+              <0x0EC 0x2DC 0x000 0x2 0x0 0xf1>,
+              <0x0F0 0x2E0 0x000 0x2 0x0 0xf1>;
         };
     };
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1170.yaml b/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1170.yaml
index 2e880b3e537c13..988db95075436b 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1170.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imxrt1170.yaml
@@ -71,7 +71,7 @@ examples:
         reg = <0x400e8000 0x4000>;
         pinctrl_lpuart1: lpuart1grp {
             fsl,pins =
-              <0x16C 0x3B0 0x620 0x0 0x0  0xf1>,
-              <0x170 0x3B4 0x61C 0x0 0x0	0xf1>;
+              <0x16C 0x3B0 0x620 0x0 0x0 0xf1>,
+              <0x170 0x3B4 0x61C 0x0 0x0 0xf1>;
         };
     };
diff --git a/Documentation/devicetree/bindings/pinctrl/intel,lgm-io.yaml b/Documentation/devicetree/bindings/pinctrl/intel,lgm-io.yaml
index 1144ca2896e336..96c405711f65e2 100644
--- a/Documentation/devicetree/bindings/pinctrl/intel,lgm-io.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/intel,lgm-io.yaml
@@ -64,13 +64,13 @@ examples:
         reg = <0xe2880000 0x100000>;
 
         uart0-pins {
-             pins = <64>, /* UART_RX0 */
-                    <65>; /* UART_TX0 */
-             function = "CONSOLE_UART0";
-             pinmux = <1>,
-                      <1>;
-             groups = "CONSOLE_UART0";
-          };
+            pins = <64>, /* UART_RX0 */
+                   <65>; /* UART_TX0 */
+            function = "CONSOLE_UART0";
+            pinmux = <1>,
+                     <1>;
+            groups = "CONSOLE_UART0";
+        };
     };
 
 ...
diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml
index 3db2438fadc78b..4bcee7741fcd71 100644
--- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml
@@ -221,8 +221,8 @@ examples:
 
             i2c0_pins_a: i2c0-pins {
                 pins1 {
-                  pinmux = <PINMUX_GPIO48__FUNC_SCL5>,
-                           <PINMUX_GPIO49__FUNC_SDA5>;
+                    pinmux = <PINMUX_GPIO48__FUNC_SCL5>,
+                             <PINMUX_GPIO49__FUNC_SDA5>;
                     mediatek,pull-up-adv = <3>;
                     drive-strength-microamp = <1000>;
                 };
diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8195-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8195-pinctrl.yaml
index 33cb71775db979..0bcafa78acb2df 100644
--- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8195-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8195-pinctrl.yaml
@@ -234,7 +234,7 @@ examples:
   - |
     #include <dt-bindings/pinctrl/mt8195-pinfunc.h>
     #include <dt-bindings/interrupt-controller/arm-gic.h>
-    #
+
     pio: pinctrl@10005000 {
       compatible = "mediatek,mt8195-pinctrl";
       reg = <0x10005000 0x1000>,
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt
deleted file mode 100644
index c28d4eb83b7687..00000000000000
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt
+++ /dev/null
@@ -1,105 +0,0 @@
-Palmas Pincontrol bindings
-
-The pins of Palmas device can be set on different option and provides
-the configuration for Pull UP/DOWN, open drain etc.
-
-Required properties:
-- compatible: It must be one of following:
-  - "ti,palmas-pinctrl" for Palma series of the pincontrol.
-  - "ti,tps65913-pinctrl" for Palma series device TPS65913.
-  - "ti,tps80036-pinctrl" for Palma series device TPS80036.
-
-Please refer to pinctrl-bindings.txt in this directory for details of the
-common pinctrl bindings used by client devices, including the meaning of the
-phrase "pin configuration node".
-
-Palmas's pin configuration nodes act as a container for an arbitrary number of
-subnodes. Each of these subnodes represents some desired configuration for a
-list of pins. This configuration can include the mux function to select on
-those pin(s), and various pin configuration parameters, such as pull-up,
-open drain.
-
-The name of each subnode is not important; all subnodes should be enumerated
-and processed purely based on their content.
-
-Each subnode only affects those parameters that are explicitly listed. In
-other words, a subnode that lists a mux function but no pin configuration
-parameters implies no information about any pin configuration parameters.
-Similarly, a pin subnode that describes a pullup parameter implies no
-information about e.g. the mux function.
-
-Optional properties:
-- ti,palmas-enable-dvfs1: Enable DVFS1. Configure pins for DVFS1 mode.
-	Selection primary or secondary function associated to I2C2_SCL_SCE,
-	I2C2_SDA_SDO pin/pad for DVFS1 interface
-- ti,palmas-enable-dvfs2: Enable DVFS2. Configure pins for DVFS2 mode.
-	Selection primary or secondary function associated to GPADC_START
-	and SYSEN2 pin/pad for DVFS2 interface
-- ti,palmas-override-powerhold: This is applicable for PMICs for which
-	GPIO7 is configured in POWERHOLD mode which has higher priority
-	over DEV_ON bit and keeps the PMIC supplies on even after the DEV_ON
-	bit is turned off. This property enables driver to over ride the
-	POWERHOLD value to GPIO7 so as to turn off the PMIC in power off
-	scenarios. So for GPIO7 if ti,palmas-override-powerhold is set
-	then the GPIO_7 field should never be muxed to anything else.
-	It should be set to POWERHOLD by default and only in case of
-	power off scenarios the driver will over ride the mux value.
-
-This binding uses the following generic properties as defined in
-pinctrl-bindings.txt:
-
-Required: pins
-Options: function, bias-disable, bias-pull-up, bias-pull-down,
-	 drive-open-drain.
-
-Note that many of these properties are only valid for certain specific pins.
-See the Palmas device datasheet for complete details regarding which pins
-support which functionality.
-
-Valid values for pin names are:
-	gpio0, gpio1, gpio2, gpio3, gpio4, gpio5, gpio6, gpio7, gpio8, gpio9,
-	gpio10, gpio11, gpio12, gpio13, gpio14, gpio15, vac, powergood,
-	nreswarm, pwrdown, gpadc_start, reset_in, nsleep, enable1, enable2,
-	int.
-
-Valid value of function names are:
-	gpio, led, pwm, regen, sysen, clk32kgaudio, id, vbus_det, chrg_det,
-	vac, vacok, powergood, usb_psel, msecure, pwrhold, int, nreswarm,
-	simrsto, simrsti, low_vbat, wireless_chrg1, rcm, pwrdown, gpadc_start,
-	reset_in, nsleep, enable.
-
-There are 4 special functions: opt0, opt1, opt2 and opt3. If any of these
-functions is selected then directly pins register will be written with 0, 1, 2
-or 3 respectively if it is valid for that pins or list of pins.
-
-Example:
-	palmas: tps65913 {
-		....
-		pinctrl {
-			compatible = "ti,tps65913-pinctrl";
-			ti,palmas-enable-dvfs1;
-			pinctrl-names = "default";
-			pinctrl-0 = <&palmas_pins_state>;
-
-			palmas_pins_state: pinmux {
-				gpio0 {
-					pins = "gpio0";
-					function = "id";
-					bias-pull-up;
-				};
-
-				vac {
-					pins = "vac";
-					function = "vacok";
-					bias-pull-down;
-				};
-
-				gpio5 {
-					pins = "gpio5";
-					function = "opt0";
-					drive-open-drain = <1>;
-				};
-			};
-		};
-		....
-	};
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,hawi-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,hawi-lpass-lpi-pinctrl.yaml
new file mode 100644
index 00000000000000..af68581750cfd6
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,hawi-lpass-lpi-pinctrl.yaml
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,hawi-lpass-lpi-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Hawi SoC LPASS LPI TLMM
+
+maintainers:
+  - Krzysztof Kozlowski <krzk@kernel.org>
+  - Srinivas Kandagatla <srinivas.kandagatla@oss.qualcomm.com>
+  - Prasad Kumpatla <prasad.kumpatla@oss.qualcomm.com>
+
+description:
+  Top Level Mode Multiplexer pin controller in the Low Power Audio SubSystem
+  (LPASS) Low Power Island (LPI) of Qualcomm Hawi SoC.
+
+properties:
+  compatible:
+    const: qcom,hawi-lpass-lpi-pinctrl
+
+  reg:
+    items:
+      - description: LPASS LPI TLMM Control and Status registers
+
+  clocks:
+    items:
+      - description: LPASS LPR(Low Power Resource) voting clock
+      - description: LPASS Audio voting clock
+
+  clock-names:
+    items:
+      - const: lpr
+      - const: audio
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-hawi-lpass-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-hawi-lpass-state"
+        additionalProperties: false
+
+$defs:
+  qcom-hawi-lpass-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,lpass-lpi-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          pattern: "^gpio([0-9]|1[0-9]|2[0-2])$"
+
+      function:
+        enum: [dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic3_clk,
+               dmic3_data, dmic4_clk, dmic4_data, ext_mclk1_a, ext_mclk1_b,
+               ext_mclk1_c, ext_mclk1_d, ext_mclk1_e, gpio, i2s0_clk,
+               i2s0_data, i2s0_ws, i2s1_clk, i2s1_data, i2s1_ws, i2s2_clk,
+               i2s2_data, i2s2_ws, i2s3_clk, i2s3_data, i2s3_ws,
+               lpass_lpi_dbg_clk, qca_swr_clk, qca_swr_data, slimbus_clk,
+               slimbus_data, swr_rx_clk, swr_rx_data, swr_tx_clk, swr_tx_clk1,
+               swr_tx_data, va_i2s0_clk, va_i2s0_data, va_i2s0_ws,
+               wsa2_swr_clk, wsa2_swr_data, wsa_swr_clk, wsa_swr_data]
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+
+allOf:
+  - $ref: qcom,lpass-lpi-common.yaml#
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+
+    lpass_tlmm: pinctrl@7f60000 {
+        compatible = "qcom,hawi-lpass-lpi-pinctrl";
+        reg = <0x07f60000 0x20000>;
+
+        clocks = <&q6prmcc LPASS_HW_LPR_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+                 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+        clock-names = "lpr", "audio";
+
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&lpass_tlmm 0 0 23>;
+
+        tx-swr-sleep-clk-state {
+            pins = "gpio0";
+            function = "swr_tx_clk";
+            drive-strength = <2>;
+            bias-pull-down;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,hawi-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,hawi-tlmm.yaml
index 3b3961789860bf..2b1354e212e5b0 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,hawi-tlmm.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,hawi-tlmm.yaml
@@ -113,8 +113,8 @@ examples:
         #interrupt-cells = <2>;
 
         qup-uart7-state {
-          pins = "gpio62", "gpio63";
-          function = "qup1_se7";
+            pins = "gpio62", "gpio63";
+            function = "qup1_se7";
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,kaanapali-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,kaanapali-tlmm.yaml
index 53534a07a1f0e2..ea7d4768483df5 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,kaanapali-tlmm.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,kaanapali-tlmm.yaml
@@ -120,8 +120,8 @@ examples:
         #interrupt-cells = <2>;
 
         qup-uart7-state {
-          pins = "gpio62", "gpio63";
-          function = "qup1_se7";
+            pins = "gpio62", "gpio63";
+            function = "qup1_se7";
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,kuno-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,kuno-tlmm.yaml
new file mode 100644
index 00000000000000..2250526dfae24d
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,kuno-tlmm.yaml
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,kuno-tlmm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. Kuno TLMM block
+
+maintainers:
+  - Hardeep Sharma <hardeep.sharma@oss.qualcomm.com>
+
+description:
+  Top Level Mode Multiplexer pin controller in Qualcomm Kuno SoC.
+
+allOf:
+  - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
+
+properties:
+  compatible:
+    const: qcom,kuno-tlmm
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  gpio-reserved-ranges:
+    minItems: 1
+    maxItems: 55
+
+  gpio-line-names:
+    maxItems: 110
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-kuno-tlmm-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-kuno-tlmm-state"
+        additionalProperties: false
+
+$defs:
+  qcom-kuno-tlmm-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          pattern: "^gpio([0-9]|[1-9][0-9]|10[0-9])$"
+        minItems: 1
+        maxItems: 36
+
+      function:
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+        enum: [ gpio, audio_ref_clk, coex_uart, ebi2_lcd_a,
+                ebi2_lcd_cs, ebi2_lcd_reset, ebi2_lcd_te, emac_mdc, emac_mdio,
+                emac_pps_in, emac_ptp_aux, emac_ptp_pps, gcc_gp1_clk,
+                gcc_gp2_clk, gcc_gp3_clk, mi2s0_data0, mi2s0_data1, mi2s0_sck,
+                mi2s0_ws, mi2s1_data0, mi2s1_data1, mi2s1_sck, mi2s1_ws,
+                mi2s_mclk, nav_gpio, pci_e_rst,
+                pcie_clkreq_n, pll_bist_sync, pll_clk_aux, qdss_cti_trig0,
+                qdss_cti_trig1, qdss_cti_trig1_mire, qdss_gpio_traceclk,
+                qdss_gpio_tracectl,
+                qup0_se0, qup0_se1, qup0_se2, qup0_se3_mira, qup0_se3_mirb,
+                qup0_se4, sdc4_clk, sdc4_cmd,
+                sdc4_data, sdc4_tb_trig, sgmii_phy_intr, spmi_coex_clk,
+                spmi_coex_data, spmi_vgi_hwevent, uim1_clk, uim1_data,
+                uim1_present, uim1_reset, usb2phy_ac_en ]
+
+    required:
+      - pins
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    tlmm: pinctrl@f000000 {
+        compatible = "qcom,kuno-tlmm";
+        reg = <0x0f000000 0x400000>;
+        interrupts = <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&tlmm 0 0 110>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+
+        spmi-state {
+            pins = "gpio78", "gpio79";
+            function = "spmi_vgi_hwevent";
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml
index 64fe90b2391bd4..a0b56adb3388aa 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,maili-tlmm.yaml
@@ -113,8 +113,8 @@ examples:
         #interrupt-cells = <2>;
 
         qup-uart7-state {
-          pins = "gpio62", "gpio63";
-          function = "qup1_se7";
+            pins = "gpio62", "gpio63";
+            function = "qup1_se7";
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,msm8952-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,msm8952-pinctrl.yaml
new file mode 100644
index 00000000000000..d367bb078aff75
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,msm8952-pinctrl.yaml
@@ -0,0 +1,146 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,msm8952-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm MSM8952 TLMM pin controller
+
+maintainers:
+  - Muzaffer Kadir <muzafferkadir@mainlining.org>
+
+description:
+  Top Level Mode Multiplexer pin controller in Qualcomm MSM8952 SoC.
+
+properties:
+  compatible:
+    const: qcom,msm8952-pinctrl
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  gpio-reserved-ranges:
+    minItems: 1
+    maxItems: 66
+
+  gpio-line-names:
+    maxItems: 134
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-msm8952-tlmm-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-msm8952-tlmm-state"
+        additionalProperties: false
+
+$defs:
+  qcom-msm8952-tlmm-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          oneOf:
+            - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-2][0-9]|13[0-3])$"
+            - enum: [ qdsd_clk, qdsd_cmd, qdsd_data0, qdsd_data1, qdsd_data2,
+                      qdsd_data3, sdc1_clk, sdc1_cmd, sdc1_data, sdc1_rclk,
+                      sdc2_clk, sdc2_cmd, sdc2_data ]
+        minItems: 1
+        maxItems: 36
+
+      function:
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+
+        enum: [ gpio, blsp_uart1, blsp_spi1, adsp_ext, blsp_i2c1, prng_rosc, qdss_cti_trig_out_b0,
+                qdss_cti_trig_out_a1, blsp_spi2, blsp_uart2, ldo_update, qdss_tracedata_b,
+                pbs1, pwr_modem_enabled_b, blsp_i2c3, gcc_gp2_clk_b, atest_combodac_to_gpio_native,
+                ldo_en, blsp_i2c2, gcc_gp1_clk_b, atest_gpsadc_dtest0_native, blsp_spi3,
+                blsp_uart3, sec_mi2s, pbs0, gcc_gp3_clk_b, blsp_spi4, blsp_uart4, pbs2,
+                pwr_nav_enabled_b, pwr_crypto_enabled_b, blsp_i2c4, blsp_spi5, blsp_uart5,
+                qdss_traceclk_a, atest_bbrx1, qdss_tracectl_b, qdss_cti_trig_in_b0, blsp_i2c6,
+                qdss_traceclk_b, atest_wlan0, m_voc, qdss_cti_trig_in_a0, atest_bbrx0, blsp_i2c5,
+                qdss_tracectl_a, atest_gpsadc_dtest1_native, qdss_tracedata_a, blsp_spi6,
+                blsp_uart6, atest_wlan1, mdp_vsync, pri_mi2s_mclk_a, sec_mi2s_mclk_a,
+                cam_mclk, cci_i2c, pwr_modem_enabled_a, cci_timer0, cci_timer1, pwr_nav_enabled_a,
+                pwr_crypto_enabled_a, cci_async, sd_write, qdss_cti_trig_out_a0, gcc_gp1_clk_a,
+                gcc_gp2_clk_a, gcc_gp3_clk_a, blsp6_spi, qdss_cti_trig_in_b1, uim_batt,
+                qdss_cti_trig_out_b1, uim1_data, uim1_clk, uim1_reset, uim1_present, uim2_data,
+                uim2_clk, uim2_reset, uim2_present, uim3_data, usb_fs, uim3_clk, uim3_reset,
+                uim3_present, atest_char3, dbg_out, bimc_dte0, sec_mi2s_mclk_b, pri_mi2s,
+                atest_char1, ebi_cdc, atest_char0, cdc_pdm0, pri_mi2s_mclk_b, lpass_slimbus,
+                lpass_slimbus0, lpass_slimbus1, wcss_bt, atest_char2, ebi_ch0, wcss_wlan2,
+                wcss_wlan1, wcss_wlan0, wcss_wlan, wcss_fm, ext_lpass, gcc_tlmm, dmic0_clk,
+                dmic0_data, qdss_cti_trig_in_a1, blsp_spi7, blsp_uart7, pri_mi2s_ws, wsa_io,
+                blsp_i2c7, blsp_spi8, blsp_uart8, blsp_i2c8, gcc_plltest, nav_pps_in_a,
+                pa_indicator, nav_pps_in_b, nav_pps, modem_tsync, nav_tsync, ssbi_wtr1,
+                gsm1_tx, gsm0_tx, atest_char, atest_tsens, bimc_dte1,
+                gp_mn, gp_pdm_0a, gp_pdm_0b, gp_pdm_1a, gp_pdm_1b,
+                gp_pdm_2a, gp_pdm_2b, gp0_clk, gp1_clk]
+
+    required:
+      - pins
+
+allOf:
+  - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    msmgpio: pinctrl@1000000 {
+        compatible = "qcom,msm8952-pinctrl";
+        reg = <0x01000000 0x300000>;
+        interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+        gpio-controller;
+        gpio-ranges = <&msmgpio 0 0 134>;
+        #gpio-cells = <2>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+
+        blsp1-uart2-sleep-state {
+            pins = "gpio4", "gpio5";
+            function = "gpio";
+
+            drive-strength = <2>;
+            bias-pull-down;
+        };
+
+        spi1-default-state {
+            spi-pins {
+                pins = "gpio0", "gpio1", "gpio3";
+                function = "blsp_spi1";
+
+                drive-strength = <12>;
+                bias-disable;
+            };
+
+            cs-pins {
+                pins = "gpio2";
+                function = "blsp_spi1";
+
+                drive-strength = <2>;
+                bias-disable;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml
index 4bb511719f3130..56758a85ead8d0 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml
@@ -67,7 +67,7 @@ $defs:
           Specify the alternative function to be configured for the specified
           pins.
 
-        enum: [ aoss_cti, atest_char, atest_usb20, atest_usb21,
+        enum: [ gpio, aoss_cti, atest_char, atest_usb20, atest_usb21,
                 aud_intfc0_clk, aud_intfc0_data, aud_intfc0_ws,
                 aud_intfc10_clk, aud_intfc10_data, aud_intfc10_ws,
                 aud_intfc1_clk, aud_intfc1_data, aud_intfc1_ws,
@@ -98,9 +98,10 @@ $defs:
                 pcie3_clk_req_n, phase_flag, pll_bist_sync, pll_clk_aux,
                 prng_rosc0, prng_rosc1, pwrbrk_i_n, qdss, qdss_cti, qspi,
                 qup0_se0, qup0_se1, qup0_se2, qup0_se3, qup0_se4, qup0_se5,
-                qup1_se0, qup1_se1, qup1_se3, qup1_se2, qup1_se4, qup1_se5,
+                qup1_se0, qup1_se1, qup1_se2_01, qup1_se2_23, qup1_se3_01,
+                qup1_se3_23, qup1_se4, qup1_se5,
                 qup1_se6, qup2_se0, qup2_se1, qup2_se2, qup2_se3, qup2_se4,
-                qup2_se5, qup2_se6,
+                qup2_se5, qup2_se6, qup3_se0_mira, qup3_se0_mirb,
                 sailss_ospi, sdc4_clk, sdc4_cmd, sdc4_data, smb_alert,
                 smb_alert_n, smb_clk, smb_dat, tb_trig_sdc4, tmess_prng0,
                 tmess_prng1, tsc_timer, tsense_pwm, usb0_hs,
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml
index c27e24cb206fcd..77be0690c751aa 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml
@@ -57,6 +57,7 @@ properties:
           - qcom,pm8994-gpio
           - qcom,pm8998-gpio
           - qcom,pma8084-gpio
+          - qcom,pmau0102-gpio
           - qcom,pmc8180-gpio
           - qcom,pmc8180c-gpio
           - qcom,pmc8380-gpio
@@ -281,6 +282,7 @@ allOf:
               - qcom,pm8150l-gpio
               - qcom,pm8550-gpio
               - qcom,pm8550b-gpio
+              - qcom,pmau0102-gpio
               - qcom,pmc8180c-gpio
               - qcom,pmp8074-gpio
               - qcom,pms405-gpio
@@ -496,6 +498,7 @@ $defs:
                  - gpio1-gpio22 for pm8994
                  - gpio1-gpio26 for pm8998
                  - gpio1-gpio22 for pma8084
+                 - gpio1-gpio12 for pmau0102
                  - gpio1-gpio14 for pmcx0102
                  - gpio1-gpio4 for pmd8028
                  - gpio1-gpio4 for pmg1110
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-tlmm.yaml
index ed344deaf8b9e4..ebc3a8b34a4c62 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-tlmm.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-tlmm.yaml
@@ -27,6 +27,9 @@ properties:
 
   gpio-reserved-ranges: true
 
+  gpio-line-names:
+    maxItems: 230
+
 patternProperties:
   "-state$":
     oneOf:
diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml
index fb1fe1ea759fc6..ec5d6d52f07bc7 100644
--- a/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml
@@ -129,13 +129,15 @@ additionalProperties:
           enum: [ 33, 50, 66, 100 ]
         power-source:
           description: I/O voltage in millivolt.
-          enum: [ 1800, 2500, 3300 ]
+          enum: [ 1200, 1800, 2500, 3300 ]
         slew-rate: true
         gpio-hog: true
         gpios: true
         input: true
         input-enable: true
+        input-disable: true
         output-enable: true
+        output-disable: true
         output-high: true
         output-low: true
         line-name: true
diff --git a/Documentation/devicetree/bindings/pinctrl/semtech,sx1501q.yaml b/Documentation/devicetree/bindings/pinctrl/semtech,sx1501q.yaml
index 39d7dad3313b6f..f7a7551d7f50fa 100644
--- a/Documentation/devicetree/bindings/pinctrl/semtech,sx1501q.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/semtech,sx1501q.yaml
@@ -244,8 +244,8 @@ examples:
             interrupt-controller;
 
             gpio1-cfg {
-                  pins = "gpio1";
-                  bias-pull-up;
+                pins = "gpio1";
+                bias-pull-up;
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml
index 94b868c7ceb19e..0084350ec579e8 100644
--- a/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml
@@ -339,7 +339,7 @@ examples:
         sdcard-pins {
             function = "SD_CARD";
             groups = "SD_CARD";
-            sunplus,pins = < SPPCTL_IOPAD(91, SPPCTL_PCTL_G_GPIO, 0, 0) >;
+            sunplus,pins = <SPPCTL_IOPAD(91, SPPCTL_PCTL_G_GPIO, 0, 0)>;
         };
 
         hdmi_A_tx1-pins {
diff --git a/Documentation/devicetree/bindings/pinctrl/ti,palmas-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/ti,palmas-pinctrl.yaml
new file mode 100644
index 00000000000000..11c2d429633cb9
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/ti,palmas-pinctrl.yaml
@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/ti,palmas-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI Palmas Pincontrol
+
+maintainers:
+  - Laxman Dewangan <ldewangan@nvidia.com>
+
+description:
+  The pins of Palmas device can be set on different option and provides
+  the configuration for Pull UP/DOWN, open drain etc.
+
+allOf:
+  - $ref: pinctrl.yaml#
+
+properties:
+  compatible:
+    enum:
+      - ti,palmas-pinctrl
+      - ti,tps65913-pinctrl
+      - ti,tps80036-pinctrl
+
+  ti,palmas-enable-dvfs1:
+    type: boolean
+    description:
+      Enable DVFS1. Configure pins for DVFS1 mode.
+      Selection primary or secondary function associated to I2C2_SCL_SCE,
+      I2C2_SDA_SDO pin/pad for DVFS1 interface.
+
+  ti,palmas-enable-dvfs2:
+    type: boolean
+    description:
+      Enable DVFS2. Configure pins for DVFS2 mode.
+      Selection primary or secondary function associated to GPADC_START
+      and SYSEN2 pin/pad for DVFS2 interface.
+
+  ti,palmas-override-powerhold:
+    type: boolean
+    description:
+      This is applicable for PMICs for which GPIO7 is configured in POWERHOLD
+      mode which has higher priority over DEV_ON bit and keeps the PMIC
+      supplies on even after the DEV_ON bit is turned off. This property enables
+      driver to over ride the POWERHOLD value to GPIO7 so as to turn off the PMIC
+      in power off scenarios. So for GPIO7 if ti,palmas-override-powerhold is set
+      then the GPIO_7 field should never be muxed to anything else.
+      It should be set to POWERHOLD by default and only in case of
+      power off scenarios the driver will over ride the mux value.
+
+patternProperties:
+  '^(pinmux|pins?-[a-z0-9-]+)$':
+    type: object
+    additionalProperties: false
+    patternProperties:
+      '^(pin.*|gpio[0-9]+|vac|powergood|nreswarm|pwrdown|gpadc_start|reset_in|nsleep|enable[12]|int)$':
+        type: object
+        allOf:
+          - $ref: /schemas/pinctrl/pinmux-node.yaml
+          - $ref: /schemas/pinctrl/pincfg-node.yaml
+        unevaluatedProperties: false
+        properties:
+          pins:
+            items:
+              enum:
+                - gpio0
+                - gpio1
+                - gpio2
+                - gpio3
+                - gpio4
+                - gpio5
+                - gpio6
+                - gpio7
+                - gpio8
+                - gpio9
+                - gpio10
+                - gpio11
+                - gpio12
+                - gpio13
+                - gpio14
+                - gpio15
+                - vac
+                - powergood
+                - nreswarm
+                - pwrdown
+                - gpadc_start
+                - reset_in
+                - nsleep
+                - enable1
+                - enable2
+                - int
+
+          function:
+            description:
+              Selects the mux function. There are 4 special functions opt0,
+              opt1, opt2 and opt3. If any of these functions is selected then
+              directly pins register will be written with 0, 1, 2 or 3
+              respectively if it is valid for that pin or list of pins.
+            enum:
+              - gpio
+              - led
+              - pwm
+              - regen
+              - sysen
+              - clk32kgaudio
+              - id
+              - vbus_det
+              - chrg_det
+              - vac
+              - vacok
+              - powergood
+              - usb_psel
+              - msecure
+              - pwrhold
+              - int
+              - nreswarm
+              - simrsto
+              - simrsti
+              - low_vbat
+              - wireless_chrg1
+              - rcm
+              - pwrdown
+              - gpadc_start
+              - reset_in
+              - nsleep
+              - enable
+              - opt0
+              - opt1
+              - opt2
+              - opt3
+
+        required:
+          - pins
+
+required:
+  - compatible
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    pmic {
+        pinctrl {
+            compatible = "ti,tps65913-pinctrl";
+            ti,palmas-enable-dvfs1;
+            pinctrl-names = "default";
+            pinctrl-0 = <&palmas_pins_state>;
+
+            pinmux {
+                gpio0 {
+                    pins = "gpio0";
+                    function = "id";
+                    bias-pull-up;
+                };
+
+                vac {
+                    pins = "vac";
+                    function = "vacok";
+                    bias-pull-down;
+                };
+
+                gpio5 {
+                    pins = "gpio5";
+                    function = "opt0";
+                    drive-open-drain = <1>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml
index ce66fd15ff9c30..300d38f87f2d86 100644
--- a/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml
@@ -317,37 +317,37 @@ examples:
     #include <dt-bindings/pinctrl/pinctrl-zynqmp.h>
     zynqmp_firmware: zynqmp-firmware {
         pinctrl0: pinctrl {
-          compatible = "xlnx,zynqmp-pinctrl";
-
-          pinctrl_uart1_default: uart1-default {
-             mux {
-                 groups = "uart0_4_grp", "uart0_5_grp";
-                 function = "uart0";
-             };
-
-             conf {
-                groups = "uart0_4_grp";
-                slew-rate = <SLEW_RATE_SLOW>;
-                power-source = <IO_STANDARD_LVCMOS18>;
-             };
-
-             conf-rx {
-                pins = "MIO18";
-                bias-pull-up;
-             };
-
-             conf-tx {
-                pins = "MIO19";
-                bias-disable;
-                input-schmitt-disable;
-             };
-          };
+            compatible = "xlnx,zynqmp-pinctrl";
+
+            pinctrl_uart1_default: uart1-default {
+                mux {
+                    groups = "uart0_4_grp", "uart0_5_grp";
+                    function = "uart0";
+                };
+
+                conf {
+                    groups = "uart0_4_grp";
+                    slew-rate = <SLEW_RATE_SLOW>;
+                    power-source = <IO_STANDARD_LVCMOS18>;
+                };
+
+                conf-rx {
+                    pins = "MIO18";
+                    bias-pull-up;
+                };
+
+                conf-tx {
+                    pins = "MIO19";
+                    bias-disable;
+                    input-schmitt-disable;
+                };
+            };
         };
     };
 
     uart1 {
-         pinctrl-names = "default";
-         pinctrl-0 = <&pinctrl_uart1_default>;
+        pinctrl-names = "default";
+        pinctrl-0 = <&pinctrl_uart1_default>;
     };
 
 ...
diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml
index e269f433524efc..dcffdd12c8c779 100644
--- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml
+++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml
@@ -48,6 +48,8 @@ properties:
               - apple,t6030-pmgr-pwrstate
               - apple,t6031-pmgr-pwrstate
               - apple,t8122-pmgr-pwrstate
+              - apple,t8132-pmgr-pwrstate
+              - apple,t8140-pmgr-pwrstate
           - const: apple,t8103-pmgr-pwrstate
 
   reg:
diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
index 0a45e1f5506212..8ee1c25a7c9d06 100644
--- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
+++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
@@ -21,6 +21,7 @@ properties:
           - qcom,glymur-rpmhpd
           - qcom,hawi-rpmhpd
           - qcom,kaanapali-rpmhpd
+          - qcom,kuno-rpmhpd
           - qcom,mdm9607-rpmpd
           - qcom,milos-rpmhpd
           - qcom,msm8226-rpmpd
@@ -63,6 +64,7 @@ properties:
           - qcom,sm6350-rpmhpd
           - qcom,sm6375-rpmpd
           - qcom,sm7150-rpmhpd
+          - qcom,sm7250-rpmhpd
           - qcom,sm8150-rpmhpd
           - qcom,sm8250-rpmhpd
           - qcom,sm8350-rpmhpd
@@ -74,6 +76,7 @@ properties:
       - items:
           - enum:
               - qcom,msm8937-rpmpd
+              - qcom,msm8952-rpmpd
           - const: qcom,msm8917-rpmpd
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml
index b67aa170b2c1ef..68b4696d7c5995 100644
--- a/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml
+++ b/Documentation/devicetree/bindings/power/renesas,rcar-sysc.yaml
@@ -13,8 +13,11 @@ maintainers:
 description: |
   The R-Car (RZ/G) System Controller provides power management for the CPU
   cores and various coprocessors.
-  The power domain IDs for consumers are defined in header files:
-  include/dt-bindings/power/r8*-sysc.h
+  The power domain IDs for consumers are defined in the below header files
+  (where the asterisk serves as a wildcard matching the specific SoC
+  identifier):
+  include/dt-bindings/power/r8*-sysc.h or
+  include/dt-bindings/power/renesas,r8*-sysc.h
 
 properties:
   compatible:
diff --git a/Documentation/devicetree/bindings/power/reset/xlnx,zynqmp-power.yaml b/Documentation/devicetree/bindings/power/reset/xlnx,zynqmp-power.yaml
index 079ad977b90780..60f0882f53810f 100644
--- a/Documentation/devicetree/bindings/power/reset/xlnx,zynqmp-power.yaml
+++ b/Documentation/devicetree/bindings/power/reset/xlnx,zynqmp-power.yaml
@@ -27,12 +27,9 @@ properties:
       mailbox controller device node and an args specifier
       that will be the phandle to the intended sub-mailbox
       child node to be used for communication. See
-      Documentation/devicetree/bindings/mailbox/mailbox.txt
-      for more details about the generic mailbox controller
-      and client driver bindings. Also see
-      Documentation/devicetree/bindings/mailbox/ \
-      xlnx,zynqmp-ipi-mailbox.txt for typical controller that
-      is used to communicate with this System controllers.
+      Documentation/devicetree/bindings/mailbox/xlnx,zynqmp-ipi-mailbox.yaml
+      for typical controller that is used to communicate with
+      this System controllers.
     items:
       - description: tx channel
       - description: rx channel
diff --git a/Documentation/devicetree/bindings/power/riscv,rpmi-device-power.yaml b/Documentation/devicetree/bindings/power/riscv,rpmi-device-power.yaml
new file mode 100644
index 00000000000000..5afdc40df7e017
--- /dev/null
+++ b/Documentation/devicetree/bindings/power/riscv,rpmi-device-power.yaml
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/power/riscv,rpmi-device-power.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: RISC-V RPMI device power service group based power domain controller
+
+maintainers:
+  - Joshua Yeong <joshua.yeong@starfivetech.com>
+
+description: |
+  The RISC-V Platform Management Interface (RPMI) [1] defines a
+  messaging protocol which is modular and extensible. The supervisor
+  software can send/receive RPMI messages via SBI MPXY extension [2]
+  or some dedicated supervisor-mode RPMI transport.
+
+  The RPMI specification [1] defines device power service group for
+  accessing and controlling the power state of platform devices managed
+  by a platform microcontroller. The supervisor software can access RPMI
+  device power service group via SBI MPXY channel or some dedicated
+  supervisor-mode RPMI transport.
+
+  ===========================================
+  References
+  ===========================================
+
+  [1] RISC-V Platform Management Interface (RPMI) v1.0 (or higher)
+      https://github.com/riscv-non-isa/riscv-rpmi/releases
+
+  [2] RISC-V Supervisor Binary Interface (SBI) v3.0 (or higher)
+      https://github.com/riscv-non-isa/riscv-sbi-doc/releases
+
+properties:
+  compatible:
+    description:
+      Intended for use by the supervisor software.
+    const: riscv,rpmi-device-power
+
+  mboxes:
+    maxItems: 1
+    description:
+      Mailbox channel of the underlying RPMI transport or SBI message proxy channel.
+
+  "#power-domain-cells":
+    const: 1
+    description:
+      Platform specific DOMAIN_ID as defined by the RISC-V Platform Management
+      Interface (RPMI) specification.
+
+required:
+  - compatible
+  - mboxes
+  - "#power-domain-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    power-controller {
+        compatible = "riscv,rpmi-device-power";
+        mboxes = <&mpxy_mbox 0x1002 0x0>;
+        #power-domain-cells = <1>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/power/riscv,rpmi-mpxy-device-power.yaml b/Documentation/devicetree/bindings/power/riscv,rpmi-mpxy-device-power.yaml
new file mode 100644
index 00000000000000..2b7df66ba172b7
--- /dev/null
+++ b/Documentation/devicetree/bindings/power/riscv,rpmi-mpxy-device-power.yaml
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/power/riscv,rpmi-mpxy-device-power.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: RISC-V RPMI device power service group based message proxy
+
+maintainers:
+  - Joshua Yeong <joshua.yeong@starfivetech.com>
+
+description: |
+  The RISC-V Platform Management Interface (RPMI) [1] defines a
+  messaging protocol which is modular and extensible. The supervisor
+  software can send/receive RPMI messages via SBI MPXY extension [2]
+  or some dedicated supervisor-mode RPMI transport.
+
+  The RPMI specification [1] defines device power service group for
+  accessing and controlling the power state of platform devices managed
+  by a platform microcontroller. The SBI implementation (machine mode
+  firmware or hypervisor) can implement an SBI MPXY channel to allow RPMI
+  device power service group access to the supervisor software.
+
+  ===========================================
+  References
+  ===========================================
+
+  [1] RISC-V Platform Management Interface (RPMI) v1.0 (or higher)
+      https://github.com/riscv-non-isa/riscv-rpmi/releases
+
+  [2] RISC-V Supervisor Binary Interface (SBI) v3.0 (or higher)
+      https://github.com/riscv-non-isa/riscv-sbi-doc/releases
+
+properties:
+  compatible:
+    description:
+      Intended for use by the SBI implementation.
+    const: riscv,rpmi-mpxy-device-power
+
+  mboxes:
+    maxItems: 1
+    description:
+      Mailbox channel of the underlying RPMI transport.
+
+  riscv,sbi-mpxy-channel-id:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      The SBI MPXY channel id to be used for providing RPMI access to
+      the supervisor software.
+
+required:
+  - compatible
+  - mboxes
+  - riscv,sbi-mpxy-channel-id
+
+additionalProperties: false
+
+examples:
+  - |
+    device-power-service {
+        compatible = "riscv,rpmi-mpxy-device-power";
+        mboxes = <&rpmi_shmem_mbox 0x9>;
+        riscv,sbi-mpxy-channel-id = <0x1002>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/power/supply/bq27xxx.yaml b/Documentation/devicetree/bindings/power/supply/bq27xxx.yaml
index bc05400186cf1c..79d1a3c8c6859a 100644
--- a/Documentation/devicetree/bindings/power/supply/bq27xxx.yaml
+++ b/Documentation/devicetree/bindings/power/supply/bq27xxx.yaml
@@ -47,6 +47,7 @@ properties:
       - ti,bq27441
       - ti,bq27621
       - ti,bq27z561
+      - ti,bq27z746
       - ti,bq28z610
       - ti,bq34z100
       - ti,bq78z100
@@ -91,7 +92,6 @@ allOf:
               - ti,bq27742
               - ti,bq27545
               - ti,bq27411
-              - ti,bq27z561
               - ti,bq28z610
               - ti,bq34z100
               - ti,bq78z100
diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml
index dd95037ea5eefb..ae8baa5f7bcdc4 100644
--- a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml
+++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml
@@ -19,6 +19,7 @@ properties:
           - apple,t8103-fpwm
           - apple,t8112-fpwm
           - apple,t8122-fpwm
+          - apple,t8132-fpwm
           - apple,t6000-fpwm
           - apple,t6020-fpwm
           - apple,t6030-fpwm
diff --git a/Documentation/devicetree/bindings/pwm/pwm-tipwmss.txt b/Documentation/devicetree/bindings/pwm/pwm-tipwmss.txt
deleted file mode 100644
index 4633697fbda1f5..00000000000000
--- a/Documentation/devicetree/bindings/pwm/pwm-tipwmss.txt
+++ /dev/null
@@ -1,58 +0,0 @@
-TI SOC based PWM Subsystem
-
-Required properties:
-- compatible: Must be "ti,<soc>-pwmss".
-  for am33xx  - compatible = "ti,am33xx-pwmss";
-  for am4372  - compatible = "ti,am4372-pwmss","ti,am33xx-pwmss";
-  for dra746 - compatible = "ti,dra746-pwmss", "ti,am33xx-pwmss"
-
-- reg: physical base address and size of the registers map.
-- address-cells: Specify the number of u32 entries needed in child nodes.
-		  Should set to 1.
-- size-cells: specify number of u32 entries needed to specify child nodes size
-		in reg property. Should set to 1.
-- ranges: describes the address mapping of a memory-mapped bus. Should set to
-	   physical address map of child's base address, physical address within
-	   parent's address  space and length of the address map. For am33xx,
-	   3 set of child register maps present, ECAP register space, EQEP
-	   register space, EHRPWM register space.
-
-Also child nodes should also populated under PWMSS DT node.
-
-Example:
-epwmss0: epwmss@48300000 { /* PWMSS for am33xx */
-	compatible = "ti,am33xx-pwmss";
-	reg = <0x48300000 0x10>;
-	ti,hwmods = "epwmss0";
-	#address-cells = <1>;
-	#size-cells = <1>;
-	ranges = <0x48300100 0x48300100 0x80   /* ECAP */
-		  0x48300180 0x48300180 0x80   /* EQEP */
-		  0x48300200 0x48300200 0x80>; /* EHRPWM */
-
-	/* child nodes go here */
-};
-
-epwmss0: epwmss@48300000 { /* PWMSS for am4372 */
-	compatible = "ti,am4372-pwmss","ti,am33xx-pwmss"
-	reg = <0x48300000 0x10>;
-	ti,hwmods = "epwmss0";
-	#address-cells = <1>;
-	#size-cells = <1>;
-	ranges = <0x48300100 0x48300100 0x80   /* ECAP */
-		  0x48300180 0x48300180 0x80   /* EQEP */
-		  0x48300200 0x48300200 0x80>; /* EHRPWM */
-
-	/* child nodes go here */
-};
-
-epwmss0: epwmss@4843e000 { /* PWMSS for DRA7xx */
-	compatible = "ti,dra746-pwmss", "ti,am33xx-pwmss";
-	reg = <0x4843e000 0x30>;
-	ti,hwmods = "epwmss0";
-	#address-cells = <1>;
-	#size-cells = <1>;
-	ranges;
-
-	/* child nodes go here */
-};
diff --git a/Documentation/devicetree/bindings/pwm/ti,am33xx-pwmss.yaml b/Documentation/devicetree/bindings/pwm/ti,am33xx-pwmss.yaml
new file mode 100644
index 00000000000000..89ce6b02b70c62
--- /dev/null
+++ b/Documentation/devicetree/bindings/pwm/ti,am33xx-pwmss.yaml
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pwm/ti,am33xx-pwmss.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments SoC PWM Subsystem
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+properties:
+  compatible:
+    oneOf:
+      - const: ti,am33xx-pwmss
+      - items:
+          - enum:
+              - ti,am4372-pwmss
+              - ti,dra746-pwmss
+          - const: ti,am33xx-pwmss
+
+  reg:
+    maxItems: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  ranges: true
+
+patternProperties:
+  "^(ecap|eqep|pwm|counter)@[0-9a-f]+$":
+    type: object
+    description: Child nodes for ECAP, EQEP (counter), and EHRPWM
+    required:
+      - compatible
+
+required:
+  - compatible
+  - reg
+  - "#address-cells"
+  - "#size-cells"
+  - ranges
+
+additionalProperties: false
+
+examples:
+  - |
+    pwm@48300000 {
+        compatible = "ti,am33xx-pwmss";
+        reg = <0x48300000 0x10>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges = <0x48300100 0x48300100 0x80>,   /* ECAP */
+                 <0x48300180 0x48300180 0x80>,   /* EQEP */
+                 <0x48300200 0x48300200 0x80>;   /* EHRPWM */
+
+        pwm@48300100 {
+            compatible = "ti,am3352-ecap";
+            #pwm-cells = <3>;
+            reg = <0x48300100 0x80>;
+            clocks = <&l4ls_gclk>;
+            clock-names = "fck";
+        };
+
+        counter@48300180 {
+            compatible = "ti,am3352-eqep";
+            reg = <0x48300180 0x80>;
+            clocks = <&l4ls_gclk>;
+            interrupts = <79>;
+        };
+
+        pwm@48300200 {
+            compatible = "ti,am3352-ehrpwm";
+            #pwm-cells = <3>;
+            reg = <0x48300200 0x80>;
+            clocks = <&ehrpwm0_tbclk>, <&l4ls_gclk>;
+            clock-names = "tbclk", "fck";
+        };
+    };
diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77620-regulator.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77620-regulator.yaml
index 7118c34961ba01..0f2901cab8afec 100644
--- a/Documentation/devicetree/bindings/regulator/maxim,max77620-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/maxim,max77620-regulator.yaml
@@ -10,17 +10,17 @@ maintainers:
   - Svyatoslav Ryhel <clamor95@gmail.com>
 
 description:
-  Device has multiple DCDC(sd[0-3]) and LDOs(ldo[0-8]). The input supply
-  of these regulators are defined under parent device node. Details of
-  regulator properties are defined as child node under sub-node "regulators"
-  which is child node of device node.
+  Device has multiple DCDC (sd[0-3], plus sd4 on MAX20024) and LDOs
+  (ldo[0-8]). The input supply of these regulators are defined under parent
+  device node. Details of regulator properties are defined as child node under
+  sub-node "regulators" which is child node of device node.
 
 patternProperties:
-  "^in-(sd[0-3]|ldo(0-1|2|3-5|4-6|7-8))-supply$":
+  "^in-(sd[0-4]|ldo(0-1|2|3-5|4-6|7-8))-supply$":
     $ref: /schemas/types.yaml#/definitions/phandle
     description: Input supply for DCDC or LDO
 
-  "^(sd[0-3]|ldo[0-8])$":
+  "^(sd[0-4]|ldo[0-8])$":
     type: object
     $ref: /schemas/regulator/regulator.yaml#
     unevaluatedProperties: false
diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml
index 6d6bbfbd26d4b9..3c02f98a38e9ce 100644
--- a/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml
+++ b/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml
@@ -62,6 +62,6 @@ examples:
                     regulator-max-microvolt = <3587500>;
                 };
             };
-       };
-     };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77838.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77838.yaml
index bed36af5493df2..bbfc06fbf547f1 100644
--- a/Documentation/devicetree/bindings/regulator/maxim,max77838.yaml
+++ b/Documentation/devicetree/bindings/regulator/maxim,max77838.yaml
@@ -63,6 +63,6 @@ examples:
                     regulator-max-microvolt = <1800000>;
                 };
             };
-       };
-     };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/regulator/mediatek,mt6380-regulator.yaml b/Documentation/devicetree/bindings/regulator/mediatek,mt6380-regulator.yaml
new file mode 100644
index 00000000000000..0b3c604da7271f
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/mediatek,mt6380-regulator.yaml
@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/mediatek,mt6380-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek MT6380 Regulator
+
+maintainers:
+  - Sean Wang <sean.wang@mediatek.com>
+
+properties:
+  compatible:
+    const: mediatek,mt6380-regulator
+
+patternProperties:
+  '^buck-(vcore1?|vrf)$':
+    description: BUCK regulators
+    $ref: regulator.yaml#
+    unevaluatedProperties: false
+
+    required:
+      - regulator-min-microvolt
+      - regulator-max-microvolt
+      - regulator-ramp-delay
+
+  '^ldo-v([mat]|phy|ddr)$':
+    type: object
+    description: LDO regulators
+    $ref: regulator.yaml#
+    unevaluatedProperties: false
+
+    required:
+      - regulator-min-microvolt
+      - regulator-max-microvolt
+      - regulator-ramp-delay
+
+required:
+  - compatible
+
+additionalProperties: false
+
+examples:
+  - |
+    regulators {
+        compatible = "mediatek,mt6380-regulator";
+
+        buck-vcore1 {
+            regulator-name = "vcore1";
+            regulator-min-microvolt = <600000>;
+            regulator-max-microvolt = <1393750>;
+            regulator-ramp-delay = <6250>;
+            regulator-always-on;
+            regulator-boot-on;
+        };
+
+        buck-vcore {
+            regulator-name = "vcore";
+            regulator-min-microvolt = <600000>;
+            regulator-max-microvolt = <1393750>;
+            regulator-ramp-delay = <6250>;
+        };
+
+        buck-vrf {
+            regulator-name = "vrf";
+            regulator-min-microvolt = <1200000>;
+            regulator-max-microvolt = <1575000>;
+            regulator-ramp-delay = <0>;
+            regulator-always-on;
+            regulator-boot-on;
+        };
+
+        ldo-vm {
+            regulator-name = "vm";
+            regulator-min-microvolt = <1050000>;
+            regulator-max-microvolt = <1400000>;
+            regulator-ramp-delay = <0>;
+            regulator-always-on;
+            regulator-boot-on;
+        };
+
+        ldo-va {
+            regulator-name = "va";
+            regulator-min-microvolt = <2200000>;
+            regulator-max-microvolt = <3300000>;
+            regulator-ramp-delay = <0>;
+            regulator-always-on;
+            regulator-boot-on;
+        };
+
+        ldo-vphy {
+            regulator-name = "vphy";
+            regulator-min-microvolt = <1800000>;
+            regulator-max-microvolt = <1800000>;
+            regulator-ramp-delay = <0>;
+            regulator-always-on;
+            regulator-boot-on;
+        };
+
+        ldo-vddr {
+            regulator-name = "vddr";
+            regulator-min-microvolt = <1240000>;
+            regulator-max-microvolt = <1840000>;
+            regulator-ramp-delay = <0>;
+            regulator-always-on;
+            regulator-boot-on;
+        };
+
+        ldo-vt {
+            regulator-name = "vt";
+            regulator-min-microvolt = <2200000>;
+            regulator-max-microvolt = <3300000>;
+            regulator-ramp-delay = <0>;
+            regulator-always-on;
+            regulator-boot-on;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/regulator/mps,mp5416.yaml b/Documentation/devicetree/bindings/regulator/mps,mp5416.yaml
index f825ee9efd8105..4b05e07f553780 100644
--- a/Documentation/devicetree/bindings/regulator/mps,mp5416.yaml
+++ b/Documentation/devicetree/bindings/regulator/mps,mp5416.yaml
@@ -53,26 +53,26 @@ examples:
         #size-cells = <0>;
 
         pmic@69 {
-          compatible = "mps,mp5416";
-          reg = <0x69>;
+            compatible = "mps,mp5416";
+            reg = <0x69>;
 
-          regulators {
+            regulators {
 
-            buck1 {
-             regulator-name = "buck1";
-             regulator-min-microvolt = <600000>;
-             regulator-max-microvolt = <2187500>;
-             regulator-min-microamp = <3800000>;
-             regulator-max-microamp = <6800000>;
-             regulator-boot-on;
-            };
+                buck1 {
+                    regulator-name = "buck1";
+                    regulator-min-microvolt = <600000>;
+                    regulator-max-microvolt = <2187500>;
+                    regulator-min-microamp = <3800000>;
+                    regulator-max-microamp = <6800000>;
+                    regulator-boot-on;
+                };
 
-            ldo2 {
-             regulator-name = "ldo2";
-             regulator-min-microvolt = <800000>;
-             regulator-max-microvolt = <3975000>;
+                ldo2 {
+                    regulator-name = "ldo2";
+                    regulator-min-microvolt = <800000>;
+                    regulator-max-microvolt = <3975000>;
+                };
             };
-         };
-       };
-     };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/regulator/mps,mp886x.yaml b/Documentation/devicetree/bindings/regulator/mps,mp886x.yaml
index 374a4f6b1e235a..68f64e86e4764c 100644
--- a/Documentation/devicetree/bindings/regulator/mps,mp886x.yaml
+++ b/Documentation/devicetree/bindings/regulator/mps,mp886x.yaml
@@ -4,17 +4,30 @@
 $id: http://devicetree.org/schemas/regulator/mps,mp886x.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Monolithic Power Systems MP8867/MP8869 voltage regulator
+title: Monolithic Power Systems MP8864/MP8867/MP8869 voltage regulator
 
 maintainers:
   - Jisheng Zhang <jszhang@kernel.org>
 
 allOf:
   - $ref: regulator.yaml#
+  - if:
+      properties:
+        compatible:
+          const: mps,mp8864
+    then:
+      properties:
+        mps,switch-frequency-hz:
+          enum: [600000, 850000, 1100000, 1600000]
+    else:
+      properties:
+        mps,switch-frequency-hz:
+          enum: [500000, 750000, 1000000, 1250000, 1500000]
 
 properties:
   compatible:
     enum:
+      - mps,mp8864
       - mps,mp8867
       - mps,mp8869
 
@@ -33,7 +46,6 @@ properties:
 
   mps,switch-frequency-hz:
     description: The valid switch frequency in Hertz.
-    enum: [500000, 750000, 1000000, 1250000, 1500000]
 
 required:
   - compatible
@@ -50,13 +62,13 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         regulator@62 {
-          compatible = "mps,mp8869";
-          regulator-name = "vcpu";
-          regulator-min-microvolt = <800000>;
-          regulator-max-microvolt = <1150000>;
-          enable-gpios = <&porta 1 GPIO_ACTIVE_LOW>;
-          mps,fb-voltage-divider = <80 240>;
-          reg = <0x62>;
+            compatible = "mps,mp8869";
+            regulator-name = "vcpu";
+            regulator-min-microvolt = <800000>;
+            regulator-max-microvolt = <1150000>;
+            enable-gpios = <&porta 1 GPIO_ACTIVE_LOW>;
+            mps,fb-voltage-divider = <80 240>;
+            reg = <0x62>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/regulator/mps,mpq4210.yaml b/Documentation/devicetree/bindings/regulator/mps,mpq4210.yaml
new file mode 100644
index 00000000000000..c1914aa1a9d623
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/mps,mpq4210.yaml
@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/mps,mpq4210.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Monolithic Power Systems MPQ4210 buck-boost regulator
+
+maintainers:
+  - Tapio Reijonen <tapio.reijonen@vaisala.com>
+
+description:
+  The MPQ4210 is a synchronous four-switch buck-boost controller with an I2C
+  interface. The output voltage is programmed through an 11-bit feedback
+  reference DAC and scaled by an external feedback resistor divider.
+
+  The controller ramps the feedback reference at 38, 50, 75 or 150 mV/ms.
+  The divider scales the ramp rate along with the voltage, so the output
+  ramp rates that regulator-ramp-delay can select are those values
+  multiplied by (R1 + R2) / R2.
+
+allOf:
+  - $ref: regulator.yaml#
+
+properties:
+  compatible:
+    const: mps,mpq4210
+
+  reg:
+    maxItems: 1
+
+  enable-gpios:
+    description: GPIO connected to the EN pin. If absent, EN is assumed to
+      be permanently asserted.
+    maxItems: 1
+
+  mps,fb-voltage-divider-ohms:
+    description: An array of two integers containing the resistor values R1
+      and R2 of the output feedback voltage divider.
+    maxItems: 2
+
+required:
+  - compatible
+  - reg
+  - mps,fb-voltage-divider-ohms
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        regulator@64 {
+            compatible = "mps,mpq4210";
+            reg = <0x64>;
+            regulator-name = "vout";
+            regulator-min-microvolt = <4000000>;
+            regulator-max-microvolt = <20000000>;
+            enable-gpios = <&gpio0 2 GPIO_ACTIVE_LOW>;
+            /* R1 = 82 kOhm, R2 = 9.1 kOhm */
+            mps,fb-voltage-divider-ohms = <82000 9100>;
+            /* 50 mV/ms reference ramp x (82000 + 9100) / 9100 */
+            regulator-ramp-delay = <500>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/regulator/mps,mpq7920.yaml b/Documentation/devicetree/bindings/regulator/mps,mpq7920.yaml
index 0d34af98403f84..726aaeac054ebf 100644
--- a/Documentation/devicetree/bindings/regulator/mps,mpq7920.yaml
+++ b/Documentation/devicetree/bindings/regulator/mps,mpq7920.yaml
@@ -88,30 +88,30 @@ examples:
         #size-cells = <0>;
 
         pmic@69 {
-          compatible = "mps,mpq7920";
-          reg = <0x69>;
-
-          regulators {
-            mps,switch-freq = /bits/ 8 <1>;
-
-            buck1 {
-             regulator-name = "buck1";
-             regulator-min-microvolt = <400000>;
-             regulator-max-microvolt = <3587500>;
-             regulator-min-microamp = <460000>;
-             regulator-max-microamp = <7600000>;
-             regulator-boot-on;
-             mps,buck-ovp-disable;
-             mps,buck-phase-delay = /bits/ 8 <2>;
-             mps,buck-softstart = /bits/ 8 <1>;
+            compatible = "mps,mpq7920";
+            reg = <0x69>;
+
+            regulators {
+                mps,switch-freq = /bits/ 8 <1>;
+
+                buck1 {
+                    regulator-name = "buck1";
+                    regulator-min-microvolt = <400000>;
+                    regulator-max-microvolt = <3587500>;
+                    regulator-min-microamp = <460000>;
+                    regulator-max-microamp = <7600000>;
+                    regulator-boot-on;
+                    mps,buck-ovp-disable;
+                    mps,buck-phase-delay = /bits/ 8 <2>;
+                    mps,buck-softstart = /bits/ 8 <1>;
+                };
+
+                ldo2 {
+                    regulator-name = "ldo2";
+                    regulator-min-microvolt = <650000>;
+                    regulator-max-microvolt = <3587500>;
+                };
             };
-
-            ldo2 {
-             regulator-name = "ldo2";
-             regulator-min-microvolt = <650000>;
-             regulator-max-microvolt = <3587500>;
-            };
-         };
-       };
-     };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/regulator/mt6380-regulator.txt b/Documentation/devicetree/bindings/regulator/mt6380-regulator.txt
deleted file mode 100644
index 0058441f16d2c9..00000000000000
--- a/Documentation/devicetree/bindings/regulator/mt6380-regulator.txt
+++ /dev/null
@@ -1,89 +0,0 @@
-MediaTek MT6380 Regulator
-
-All voltage regulators provided by the MT6380 PMIC are described as the
-subnodes of the MT6380 regulators node. Each regulator is named according
-to its regulator type, buck-<name> and ldo-<name>. The definition for each
-of these nodes is defined using the standard binding for regulators at
-Documentation/devicetree/bindings/regulator/regulator.txt.
-
-The valid names for regulators are:
-BUCK:
-  buck-core1, buck-vcore, buck-vrf
-LDO:
-  ldo-vm ,ldo-va , ldo-vphy, ldo-vddr, ldo-vt
-
-Example:
-
-	regulators {
-		compatible = "mediatek,mt6380-regulator";
-
-		mt6380_vcpu_reg: buck-vcore1 {
-			regulator-name = "vcore1";
-			regulator-min-microvolt = < 600000>;
-			regulator-max-microvolt = <1393750>;
-			regulator-ramp-delay = <6250>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-
-		mt6380_vcore_reg: buck-vcore {
-			regulator-name = "vcore";
-			regulator-min-microvolt = <600000>;
-			regulator-max-microvolt = <1393750>;
-			regulator-ramp-delay = <6250>;
-		};
-
-		mt6380_vrf_reg: buck-vrf {
-			regulator-name = "vrf";
-			regulator-min-microvolt = <1200000>;
-			regulator-max-microvolt = <1575000>;
-			regulator-ramp-delay = <0>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-
-		mt6380_vm_reg: ldo-vm {
-			regulator-name = "vm";
-			regulator-min-microvolt = <1050000>;
-			regulator-max-microvolt = <1400000>;
-			regulator-ramp-delay = <0>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-
-		mt6380_va_reg: ldo-va {
-			regulator-name = "va";
-			regulator-min-microvolt = <2200000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-ramp-delay = <0>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-
-		mt6380_vphy_reg: ldo-vphy {
-			regulator-name = "vphy";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			regulator-ramp-delay = <0>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-
-		mt6380_vddr_reg: ldo-vddr {
-			regulator-name = "vddr";
-			regulator-min-microvolt = <1240000>;
-			regulator-max-microvolt = <1840000>;
-			regulator-ramp-delay = <0>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-
-		mt6380_vt_reg: ldo-vt {
-			regulator-name = "vt";
-			regulator-min-microvolt = <2200000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-ramp-delay = <0>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-	};
diff --git a/Documentation/devicetree/bindings/regulator/nexperia,nex10000ub.yaml b/Documentation/devicetree/bindings/regulator/nexperia,nex10000ub.yaml
new file mode 100644
index 00000000000000..3c0db59b8c1f68
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/nexperia,nex10000ub.yaml
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/nexperia,nex10000ub.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Nexperia NEX10000UB dual output LCD bias power supply
+
+maintainers:
+  - Neil Armstrong <neil.armstrong@linaro.org>
+
+description:
+  The NEX10000UB is designed to supply positive/negative supplies for display Panels.
+
+properties:
+  compatible:
+    enum:
+      - nexperia,nex10000ub
+
+  reg:
+    maxItems: 1
+
+  vin-supply: true
+
+patternProperties:
+  "^(vpos|vneg)$":
+    type: object
+    $ref: regulator.yaml#
+    unevaluatedProperties: false
+
+    properties:
+      enable-gpios:
+        maxItems: 1
+
+    required:
+      - enable-gpios
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        regulator@3e {
+            compatible = "nexperia,nex10000ub";
+            reg = <0x3e>;
+            vin-supply = <&supply>;
+
+            vpos {
+                regulator-name = "outp";
+                regulator-boot-on;
+                enable-gpios = <&gpio 23 GPIO_ACTIVE_HIGH>;
+            };
+
+            vneg {
+                regulator-name = "outn";
+                regulator-boot-on;
+                enable-gpios = <&gpio 40 GPIO_ACTIVE_HIGH>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/regulator/nxp,pf0900.yaml b/Documentation/devicetree/bindings/regulator/nxp,pf0900.yaml
index 8c8fc2cd4cedb0..38fe80749c3c7e 100644
--- a/Documentation/devicetree/bindings/regulator/nxp,pf0900.yaml
+++ b/Documentation/devicetree/bindings/regulator/nxp,pf0900.yaml
@@ -160,4 +160,4 @@ examples:
                 };
             };
         };
-     };
+    };
diff --git a/Documentation/devicetree/bindings/regulator/onnn,fan53880.yaml b/Documentation/devicetree/bindings/regulator/onnn,fan53880.yaml
index b5181719daa167..1618e8f1c4c425 100644
--- a/Documentation/devicetree/bindings/regulator/onnn,fan53880.yaml
+++ b/Documentation/devicetree/bindings/regulator/onnn,fan53880.yaml
@@ -82,6 +82,6 @@ examples:
                     regulator-max-microvolt = <1200000>;
                 };
             };
-       };
-     };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/regulator/pwm-regulator.yaml b/Documentation/devicetree/bindings/regulator/pwm-regulator.yaml
index 80ecf938b74933..a7f48d33bad5ad 100644
--- a/Documentation/devicetree/bindings/regulator/pwm-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/pwm-regulator.yaml
@@ -116,7 +116,7 @@ examples:
         regulator-max-microvolt = <1114000>;
         regulator-name = "vdd_logic";
 
-                /* Voltage Duty-Cycle */
+        /* Voltage Duty-Cycle */
         voltage-table = <1114000 0>,
             <1095000 10>,
             <1076000 20>,
diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
index eed2ce7fa86114..9115885e7985b0 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
@@ -187,6 +187,22 @@ allOf:
       patternProperties:
         "^vdd-s[1-8]-supply$": true
 
+  - if:
+      properties:
+        compatible:
+          enum:
+            - qcom,pm6350-rpmh-regulators
+    then:
+      properties:
+        vdd-l2-l5-l7-supply: true
+        vdd-l3-l6-l8-l9-l10-supply: true
+        vdd-l12-l14-supply: true
+        vdd-l15-l19-l22-supply: true
+        vdd-l18-l20-l21-supply: true
+      patternProperties:
+        "^vdd-l([14]|1[1367])-supply$": true
+        "^vdd-s[1-6]-supply$": true
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/regulator/richtek,rtmv20-regulator.yaml b/Documentation/devicetree/bindings/regulator/richtek,rtmv20-regulator.yaml
index fec3d396ca50e0..1526cd071b6169 100644
--- a/Documentation/devicetree/bindings/regulator/richtek,rtmv20-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/richtek,rtmv20-regulator.yaml
@@ -151,9 +151,9 @@ examples:
         richtek,vsync-polarity-high;
 
         lsw {
-                regulator-name = "rtmv20,lsw";
-                regulator-min-microamp = <0>;
-                regulator-max-microamp = <6000000>;
+          regulator-name = "rtmv20,lsw";
+          regulator-min-microamp = <0>;
+          regulator-max-microamp = <6000000>;
         };
       };
     };
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd73800-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd73800-regulator.yaml
new file mode 100644
index 00000000000000..2db4585272c530
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd73800-regulator.yaml
@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/rohm,bd73800-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ROHM BD73800 Power Management Integrated Circuit regulators
+
+maintainers:
+  - Matti Vaittinen <mazziesaccount@gmail.com>
+
+description: |
+  This module is part of the ROHM BD73800 MFD device. For more details
+  see Documentation/devicetree/bindings/mfd/rohm,bd73800-pmic.yaml.
+
+  The regulator controller is represented as a sub-node of the PMIC node
+  on the device tree.
+
+  Regulator nodes should be named to buck<number> and ldo<number>.
+  The valid names for BD73800 regulator nodes are
+  buck1, buck2, buck3, buck4, buck5, buck6, buck7, buck8
+  ldo1, ldo2, ldo3, ldo4
+
+patternProperties:
+  "^buck[1-8]$":
+    type: object
+    description:
+      Properties for a single BUCK regulator.
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
+
+    properties:
+      regulator-name:
+        pattern: "^buck[1-8]$"
+        description:
+          should be "buck1", ..., "buck8"
+
+      rohm,dvs-run-voltage:
+        minimum: 0
+        maximum: 3500000
+
+      rohm,dvs-idle-voltage:
+        minimum: 0
+        maximum: 3500000
+
+      rohm,dvs-suspend-voltage:
+        minimum: 0
+        maximum: 3500000
+
+        # BUCKs 1,2,3,4 and 8 support voltages 0.5 - 1.3V
+        # BUCK 5 supports voltages 0.3 - 1.3V
+        # BUCKs 6 and 7 support voltages 1.5 - 3.5V
+
+    required:
+      - regulator-name
+
+    unevaluatedProperties: false
+
+  "^ldo[1-4]$":
+    type: object
+    description:
+      Properties for single LDO regulator.
+    allOf:
+      - $ref: regulator.yaml#
+      - $ref: rohm,pmic-states.yaml#
+
+    properties:
+      regulator-name:
+        pattern: "^ldo[1-4]$"
+        description:
+          should be "ldo1", ..., "ldo4"
+
+      rohm,dvs-run-voltage:
+        minimum: 0
+        maximum: 1
+
+      rohm,dvs-idle-voltage:
+        minimum: 0
+        maximum: 1
+
+      rohm,dvs-suspend-voltage:
+        minimum: 0
+        maximum: 1
+
+      rohm,ldo-range-high:
+        type: boolean
+        description:
+          LDO1 and LDO3 voltage ranges can be "high" or "low" depending on
+          OTP. Indicate that the "high" range is used. See comment below
+          for voltages.
+
+        # LDOs 2 and 4 support voltages 0.75 - 3.3V
+        # LDOs 1 and 3 may support different ranges depending on OTP.
+        # either 0.6 - 1.8V or 0.75 - 3.3V.
+
+    unevaluatedProperties: false
+
+additionalProperties: false
diff --git a/Documentation/devicetree/bindings/regulator/silergy,sy8824x.yaml b/Documentation/devicetree/bindings/regulator/silergy,sy8824x.yaml
index 82af4d65617706..0da920f84f892b 100644
--- a/Documentation/devicetree/bindings/regulator/silergy,sy8824x.yaml
+++ b/Documentation/devicetree/bindings/regulator/silergy,sy8824x.yaml
@@ -35,10 +35,10 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         regulator@60 {
-          compatible = "silergy,sy8824c";
-          regulator-min-microvolt = <800000>;
-          regulator-max-microvolt = <1150000>;
-          reg = <0x60>;
+            compatible = "silergy,sy8824c";
+            regulator-min-microvolt = <800000>;
+            regulator-max-microvolt = <1150000>;
+            reg = <0x60>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/regulator/silergy,sy8827n.yaml b/Documentation/devicetree/bindings/regulator/silergy,sy8827n.yaml
index b222adabc7b494..77da91c28c190f 100644
--- a/Documentation/devicetree/bindings/regulator/silergy,sy8827n.yaml
+++ b/Documentation/devicetree/bindings/regulator/silergy,sy8827n.yaml
@@ -39,8 +39,8 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         regulator@60 {
-          compatible = "silergy,sy8827n";
-          reg = <0x60>;
+            compatible = "silergy,sy8827n";
+            reg = <0x60>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/regulator/ti,tps65023.yaml b/Documentation/devicetree/bindings/regulator/ti,tps65023.yaml
new file mode 100644
index 00000000000000..928698595f526c
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/ti,tps65023.yaml
@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/ti,tps65023.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments TPS65023 Regulator
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+properties:
+  compatible:
+    enum:
+      - ti,tps65020
+      - ti,tps65021
+      - ti,tps65023
+
+  reg:
+    maxItems: 1
+
+  regulators:
+    type: object
+    description: >
+      List of child nodes that specify the regulator initialization data.
+      Must be named after their hardware counterparts: VDCDC[1-3] and LDO[1-2].
+
+    patternProperties:
+      "^(VDCDC[1-3]|LDO[1-2])$":
+        type: object
+        $ref: regulator.yaml#
+        unevaluatedProperties: false
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - regulators
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        tps65023@48 {
+            compatible = "ti,tps65023";
+            reg = <0x48>;
+
+            regulators {
+                VDCDC1 {
+                    regulator-name = "vdd_mpu";
+                    regulator-always-on;
+                    regulator-min-microvolt = <1200000>;
+                    regulator-max-microvolt = <1200000>;
+                };
+
+                VDCDC2 {
+                    regulator-name = "vdd_core";
+                    regulator-always-on;
+                    regulator-min-microvolt = <3300000>;
+                    regulator-max-microvolt = <3300000>;
+                };
+
+                VDCDC3 {
+                    regulator-name = "vdd_io";
+                    regulator-always-on;
+                    regulator-min-microvolt = <1800000>;
+                    regulator-max-microvolt = <1800000>;
+                };
+
+                LDO1 {
+                    regulator-name = "vdd_usb18";
+                    regulator-always-on;
+                    regulator-min-microvolt = <1800000>;
+                    regulator-max-microvolt = <1800000>;
+                };
+
+                LDO2 {
+                    regulator-name = "vdd_usb33";
+                    regulator-always-on;
+                    regulator-min-microvolt = <3300000>;
+                    regulator-max-microvolt = <3300000>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/regulator/ti,tps6586x.yaml b/Documentation/devicetree/bindings/regulator/ti,tps6586x.yaml
new file mode 100644
index 00000000000000..1f7c7b35066f8f
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/ti,tps6586x.yaml
@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/ti,tps6586x.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI TPS6586x family of regulators
+
+maintainers:
+  - Thierry Reding <thierry.reding@avionic-design.de>
+
+properties:
+  compatible:
+    const: ti,tps6586x
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  "#gpio-cells":
+    const: 2
+
+  gpio-controller: true
+
+  sys-supply:
+    description: The input supply for SYS.
+
+  ti,system-power-controller:
+    type: boolean
+    description:
+      Whether or not this PMIC is controlling the system power.
+
+  regulators:
+    type: object
+    additionalProperties: false
+    description:
+      list of regulators provided by this controller, LDO5 and LDO_RTC is
+      supplied by SYS regulator internally and driver take care of making
+      proper parent child relationship.
+
+    properties:
+      sys:
+        type: object
+        $ref: regulator.yaml#
+        unevaluatedProperties: false
+
+    patternProperties:
+      "^ldo([0-9]|_rtc)$":
+        type: object
+        allOf:
+          - $ref: /schemas/regulator/regulator.yaml#
+          - $ref: /schemas/regulator/nvidia,tegra-regulators-coupling.yaml#
+        unevaluatedProperties: false
+
+      "^sm[0-2]$":
+        type: object
+        allOf:
+          - $ref: /schemas/regulator/regulator.yaml#
+          - $ref: /schemas/regulator/nvidia,tegra-regulators-coupling.yaml#
+        unevaluatedProperties: false
+
+patternProperties:
+  "^vin-sm[0-2]-supply$":
+    description: The input supply for the corresponding SM (SM0, SM1, or SM2).
+
+  "^vinldo(01|23|4|678|9)-supply$":
+    description: The input supply for the corresponding LDO or group of LDOs.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - regulators
+  - "#gpio-cells"
+  - gpio-controller
+  - sys-supply
+  - vin-sm0-supply
+  - vin-sm1-supply
+  - vin-sm2-supply
+  - vinldo01-supply
+  - vinldo23-supply
+  - vinldo4-supply
+  - vinldo678-supply
+  - vinldo9-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      pmic@34 {
+          compatible = "ti,tps6586x";
+          reg = <0x34>;
+          interrupts = <0 88 0x4>;
+
+          #gpio-cells = <2>;
+          gpio-controller;
+
+          ti,system-power-controller;
+
+          sys-supply = <&vdd_5v0_sys>;
+          vin-sm0-supply = <&sys_reg>;
+          vin-sm1-supply = <&sys_reg>;
+          vin-sm2-supply = <&sys_reg>;
+          vinldo01-supply = <&sm2_reg>;
+          vinldo23-supply = <&sm2_reg>;
+          vinldo4-supply = <&sm2_reg>;
+          vinldo678-supply = <&sm2_reg>;
+          vinldo9-supply = <&sm2_reg>;
+
+          regulators {
+              sys {
+                  regulator-name = "vdd_sys";
+                  regulator-boot-on;
+                  regulator-always-on;
+              };
+
+              sm0 {
+                  regulator-min-microvolt = < 725000>;
+                  regulator-max-microvolt = <1500000>;
+                  regulator-boot-on;
+                  regulator-always-on;
+              };
+
+              sm1 {
+                  regulator-min-microvolt = < 725000>;
+                  regulator-max-microvolt = <1500000>;
+                  regulator-boot-on;
+                  regulator-always-on;
+              };
+
+              sm2 {
+                  regulator-min-microvolt = <3000000>;
+                  regulator-max-microvolt = <4550000>;
+                  regulator-boot-on;
+                  regulator-always-on;
+              };
+
+              ldo0 {
+                  regulator-name = "PCIE CLK";
+                  regulator-min-microvolt = <3300000>;
+                  regulator-max-microvolt = <3300000>;
+              };
+
+              ldo1 {
+                  regulator-min-microvolt = < 725000>;
+                  regulator-max-microvolt = <1500000>;
+              };
+
+              ldo2 {
+                  regulator-min-microvolt = < 725000>;
+                  regulator-max-microvolt = <1500000>;
+              };
+
+              ldo3 {
+                  regulator-min-microvolt = <1250000>;
+                  regulator-max-microvolt = <3300000>;
+              };
+
+              ldo4 {
+                  regulator-min-microvolt = <1700000>;
+                  regulator-max-microvolt = <2475000>;
+              };
+
+              ldo5 {
+                  regulator-min-microvolt = <1250000>;
+                  regulator-max-microvolt = <3300000>;
+              };
+
+              ldo6 {
+                  regulator-min-microvolt = <1250000>;
+                  regulator-max-microvolt = <3300000>;
+              };
+
+              ldo7 {
+                  regulator-min-microvolt = <1250000>;
+                  regulator-max-microvolt = <3300000>;
+              };
+
+              ldo8 {
+                  regulator-min-microvolt = <1250000>;
+                  regulator-max-microvolt = <3300000>;
+              };
+
+              ldo9 {
+                  regulator-min-microvolt = <1250000>;
+                  regulator-max-microvolt = <3300000>;
+              };
+          };
+      };
+    };
diff --git a/Documentation/devicetree/bindings/regulator/tps65023.txt b/Documentation/devicetree/bindings/regulator/tps65023.txt
deleted file mode 100644
index a4714e4da370bc..00000000000000
--- a/Documentation/devicetree/bindings/regulator/tps65023.txt
+++ /dev/null
@@ -1,60 +0,0 @@
-TPS65023 family of regulators
-
-Required properties:
-- compatible: Must be one of the following.
-	"ti,tps65020",
-	"ti,tps65021",
-	"ti,tps65023",
-- reg: I2C slave address
-- regulators: list of regulators provided by this controller, must be named
-  after their hardware counterparts: VDCDC[1-3] and LDO[1-2]
-- regulators: This is the list of child nodes that specify the regulator
-  initialization data for defined regulators. The definition for each of
-  these nodes is defined using the standard binding for regulators found at
-  Documentation/devicetree/bindings/regulator/regulator.txt.
-
-Each regulator is defined using the standard binding for regulators.
-
-Example:
-
-	tps65023@48 {
-		compatible = "ti,tps65023";
-		reg = <0x48>;
-
-		regulators {
-			VDCDC1 {
-				regulator-name = "vdd_mpu";
-				regulator-always-on;
-				regulator-min-microvolt = <1200000>;
-				regulator-max-microvolt = <1200000>;
-			};
-
-			VDCDC2 {
-				regulator-name = "vdd_core";
-				regulator-always-on;
-				regulator-min-microvolt = <3300000>;
-				regulator-max-microvolt = <3300000>;
-			};
-
-			VDCDC3 {
-				regulator-name = "vdd_io";
-				regulator-always-on;
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-			};
-
-			LDO1 {
-				regulator-name = "vdd_usb18";
-				regulator-always-on;
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-			};
-
-			LDO2 {
-				regulator-name = "vdd_usb33";
-				regulator-always-on;
-				regulator-min-microvolt = <3300000>;
-				regulator-max-microvolt = <3300000>;
-			};
-		};
-	};
diff --git a/Documentation/devicetree/bindings/regulator/tps6586x.txt b/Documentation/devicetree/bindings/regulator/tps6586x.txt
deleted file mode 100644
index 8b40cac24d9395..00000000000000
--- a/Documentation/devicetree/bindings/regulator/tps6586x.txt
+++ /dev/null
@@ -1,135 +0,0 @@
-TPS6586x family of regulators
-
-Required properties:
-- compatible: "ti,tps6586x"
-- reg: I2C slave address
-- interrupts: the interrupt outputs of the controller
-- #gpio-cells: number of cells to describe a GPIO
-- gpio-controller: mark the device as a GPIO controller
-- regulators: A node that houses a sub-node for each regulator within the
-  device. Each sub-node is identified using the node's name (or the deprecated
-  regulator-compatible property if present), with valid values listed below.
-  The content of each sub-node is defined by the standard binding for
-  regulators; see regulator.txt.
-  sys, sm[0-2], ldo[0-9] and ldo_rtc
-- sys-supply: The input supply for SYS.
-- vin-sm0-supply: The input supply for the SM0.
-- vin-sm1-supply: The input supply for the SM1.
-- vin-sm2-supply: The input supply for the SM2.
-- vinldo01-supply: The input supply for the LDO1 and LDO2
-- vinldo23-supply: The input supply for the LDO2 and LDO3
-- vinldo4-supply: The input supply for the LDO4
-- vinldo678-supply: The input supply for the LDO6, LDO7 and LDO8
-- vinldo9-supply: The input supply for the LDO9
-
-Optional properties:
-- ti,system-power-controller: Telling whether or not this pmic is controlling
-  the system power.
-
-Each regulator is defined using the standard binding for regulators.
-
-Note: LDO5 and LDO_RTC is supplied by SYS regulator internally and driver
-      take care of making proper parent child relationship.
-
-Example:
-
-	pmu: tps6586x@34 {
-		compatible = "ti,tps6586x";
-		reg = <0x34>;
-		interrupts = <0 88 0x4>;
-
-		#gpio-cells = <2>;
-		gpio-controller;
-
-		ti,system-power-controller;
-
-		sys-supply = <&some_reg>;
-		vin-sm0-supply = <&some_reg>;
-		vin-sm1-supply = <&some_reg>;
-		vin-sm2-supply = <&some_reg>;
-		vinldo01-supply = <...>;
-		vinldo23-supply = <...>;
-		vinldo4-supply = <...>;
-		vinldo678-supply = <...>;
-		vinldo9-supply = <...>;
-
-		regulators {
-			sys_reg: sys {
-				regulator-name = "vdd_sys";
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			sm0_reg: sm0 {
-				regulator-min-microvolt = < 725000>;
-				regulator-max-microvolt = <1500000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			sm1_reg: sm1 {
-				regulator-min-microvolt = < 725000>;
-				regulator-max-microvolt = <1500000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			sm2_reg: sm2 {
-				regulator-min-microvolt = <3000000>;
-				regulator-max-microvolt = <4550000>;
-				regulator-boot-on;
-				regulator-always-on;
-			};
-
-			ldo0_reg: ldo0 {
-				regulator-name = "PCIE CLK";
-				regulator-min-microvolt = <3300000>;
-				regulator-max-microvolt = <3300000>;
-			};
-
-			ldo1_reg: ldo1 {
-				regulator-min-microvolt = < 725000>;
-				regulator-max-microvolt = <1500000>;
-			};
-
-			ldo2_reg: ldo2 {
-				regulator-min-microvolt = < 725000>;
-				regulator-max-microvolt = <1500000>;
-			};
-
-			ldo3_reg: ldo3 {
-				regulator-min-microvolt = <1250000>;
-				regulator-max-microvolt = <3300000>;
-			};
-
-			ldo4_reg: ldo4 {
-				regulator-min-microvolt = <1700000>;
-				regulator-max-microvolt = <2475000>;
-			};
-
-			ldo5_reg: ldo5 {
-				regulator-min-microvolt = <1250000>;
-				regulator-max-microvolt = <3300000>;
-			};
-
-			ldo6_reg: ldo6 {
-				regulator-min-microvolt = <1250000>;
-				regulator-max-microvolt = <3300000>;
-			};
-
-			ldo7_reg: ldo7 {
-				regulator-min-microvolt = <1250000>;
-				regulator-max-microvolt = <3300000>;
-			};
-
-			ldo8_reg: ldo8 {
-				regulator-min-microvolt = <1250000>;
-				regulator-max-microvolt = <3300000>;
-			};
-
-			ldo9_reg: ldo9 {
-				regulator-min-microvolt = <1250000>;
-				regulator-max-microvolt = <3300000>;
-			};
-		};
-	};
diff --git a/Documentation/devicetree/bindings/regulator/vexpress.txt b/Documentation/devicetree/bindings/regulator/vexpress.txt
deleted file mode 100644
index 1c2e92c7831eab..00000000000000
--- a/Documentation/devicetree/bindings/regulator/vexpress.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-Versatile Express voltage regulators
-------------------------------------
-
-Requires node properties:
-- "compatible" value: "arm,vexpress-volt"
-- "arm,vexpress-sysreg,func" when controlled via vexpress-sysreg
-  (see Documentation/devicetree/bindings/arm/vexpress-config.yaml
-  for more details)
-
-Required regulator properties:
-- "regulator-name"
-- "regulator-always-on"
-
-Optional regulator properties:
-- "regulator-min-microvolt"
-- "regulator-max-microvolt"
-
-See Documentation/devicetree/bindings/regulator/regulator.txt
-for more details about the regulator properties.
-
-When no "regulator-[min|max]-microvolt" properties are defined,
-the device is treated as fixed (or rather "read-only") regulator.
-
-Example:
-	volt@0 {
-		compatible = "arm,vexpress-volt";
-		arm,vexpress-sysreg,func = <2 0>;
-		regulator-name = "Cores";
-		regulator-min-microvolt = <800000>;
-		regulator-max-microvolt = <1050000>;
-		regulator-always-on;
-	};
diff --git a/Documentation/devicetree/bindings/remoteproc/mtk,scp.yaml b/Documentation/devicetree/bindings/remoteproc/mtk,scp.yaml
index bdbb12118da435..2f0a28c7a24cc0 100644
--- a/Documentation/devicetree/bindings/remoteproc/mtk,scp.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/mtk,scp.yaml
@@ -28,11 +28,11 @@ properties:
     description:
       Should contain the address ranges for memory regions SRAM, CFG, and,
       on some platforms, L1TCM.
-    minItems: 2
+    minItems: 1
     maxItems: 3
 
   reg-names:
-    minItems: 2
+    minItems: 1
     maxItems: 3
 
   clocks:
@@ -171,6 +171,7 @@ allOf:
     then:
       properties:
         reg:
+          minItems: 2
           maxItems: 2
         reg-names:
           items:
@@ -185,6 +186,7 @@ allOf:
     then:
       properties:
         reg:
+          minItems: 3
           maxItems: 3
         reg-names:
           items:
@@ -195,17 +197,31 @@ allOf:
       properties:
         compatible:
           enum:
-            - mediatek,mt8188-scp-dual
             - mediatek,mt8195-scp-dual
     then:
       properties:
         reg:
+          minItems: 2
           maxItems: 2
         reg-names:
           items:
             - const: cfg
             - const: l1tcm
 
+  - if:
+      properties:
+        compatible:
+          enum:
+            - mediatek,mt8188-scp-dual
+    then:
+      properties:
+        reg:
+          minItems: 1
+          maxItems: 1
+        reg-names:
+          items:
+            - const: cfg
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
index 11faf655f530fb..f75cab6b4ed95e 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
@@ -77,6 +77,15 @@ properties:
       channels and devices related to the ADSP.
     unevaluatedProperties: false
 
+  '#cooling-cells':
+    description: |
+      Cooling device with three cells:
+        Cell 0: Cooling device id as defined in
+                include/dt-bindings/thermal/qcom,pas.h
+        Cell 1: Minimum cooling state
+        Cell 2: Maximum cooling state
+    const: 3
+
   glink-edge:
     $ref: /schemas/remoteproc/qcom,glink-edge.yaml#
     description:
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
index c58e8a6c7fe106..99e9aeeb4f62c9 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sm8550-pas.yaml
@@ -18,7 +18,7 @@ properties:
     oneOf:
       - enum:
           - qcom,eliza-cdsp-pas
-          - qcom,sdx75-mpss-pas
+          - qcom,hawi-cdsp-pas
           - qcom,sm8550-adsp-pas
           - qcom,sm8550-cdsp-pas
           - qcom,sm8550-mpss-pas
@@ -28,6 +28,10 @@ properties:
           - qcom,sm8750-mpss-pas
           - qcom,x1e80100-adsp-pas
           - qcom,x1e80100-cdsp-pas
+      - items:
+          - enum:
+              - qcom,maili-cdsp-pas
+          - const: qcom,hawi-cdsp-pas
       - items:
           - enum:
               - qcom,eliza-adsp-pas
@@ -40,13 +44,17 @@ properties:
       - items:
           - enum:
               - qcom,glymur-cdsp-pas
-              - qcom,hawi-cdsp-pas
               - qcom,kaanapali-cdsp-pas
-              - qcom,maili-cdsp-pas
           - const: qcom,sm8550-cdsp-pas
       - items:
           - const: qcom,sm8750-cdsp-pas
           - const: qcom,sm8650-cdsp-pas
+      - items:
+          - enum:
+              - qcom,hawi-mpss-pas
+              - qcom,maili-mpss-pas
+              - qcom,sdx75-mpss-pas
+          - const: qcom,sm8650-mpss-pas
 
   reg:
     maxItems: 1
@@ -203,9 +211,9 @@ allOf:
   - if:
       properties:
         compatible:
-          enum:
-            - qcom,sdx75-mpss-pas
-            - qcom,sm8650-mpss-pas
+          contains:
+            enum:
+              - qcom,sm8650-mpss-pas
     then:
       properties:
         interrupts:
@@ -253,11 +261,11 @@ allOf:
   - if:
       properties:
         compatible:
-          enum:
-            - qcom,sdx75-mpss-pas
-            - qcom,sm8550-mpss-pas
-            - qcom,sm8650-mpss-pas
-            - qcom,sm8750-mpss-pas
+          contains:
+            enum:
+              - qcom,sm8550-mpss-pas
+              - qcom,sm8650-mpss-pas
+              - qcom,sm8750-mpss-pas
     then:
       properties:
         power-domains:
@@ -273,6 +281,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,hawi-cdsp-pas
               - qcom,sm8550-cdsp-pas
               - qcom,sm8650-cdsp-pas
               - qcom,x1e80100-cdsp-pas
diff --git a/Documentation/devicetree/bindings/remoteproc/ti,am3352-wkup-m3.yaml b/Documentation/devicetree/bindings/remoteproc/ti,am3352-wkup-m3.yaml
index 9e8a03acdec4e8..997be06ceeb17b 100644
--- a/Documentation/devicetree/bindings/remoteproc/ti,am3352-wkup-m3.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/ti,am3352-wkup-m3.yaml
@@ -62,7 +62,6 @@ required:
 
 dependencies:
   resets: [reset-names]
-  reset-names: [resets]
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/serial/8250.yaml b/Documentation/devicetree/bindings/serial/8250.yaml
index 8c2c177faf89c4..50d5c977afa3d4 100644
--- a/Documentation/devicetree/bindings/serial/8250.yaml
+++ b/Documentation/devicetree/bindings/serial/8250.yaml
@@ -100,19 +100,25 @@ allOf:
 properties:
   compatible:
     oneOf:
-      - const: ns8250
-      - const: ns16450
-      - const: ns16550
-      - const: ns16550a
-      - const: ns16850
-      - const: aspeed,ast2400-vuart
-      - const: aspeed,ast2500-vuart
-      - const: intel,xscale-uart
-      - const: mrvl,pxa-uart
-      - const: nuvoton,wpcm450-uart
-      - const: nuvoton,npcm750-uart
-      - const: nvidia,tegra20-uart
-      - const: nxp,lpc3220-uart
+      - enum:
+          - ns8250
+          - ns16450
+          - ns16550
+          - ns16550a
+          - ns16850
+          - airoha,an7581-hsuart
+          - airoha,en7523-uart
+          - aspeed,ast2400-vuart
+          - aspeed,ast2500-vuart
+          - intel,xscale-uart
+          - mrvl,pxa-uart
+          - nuvoton,npcm750-uart
+          - nuvoton,wpcm450-uart
+          - nvidia,tegra20-uart
+          - nxp,lpc3220-uart
+      - items:
+          - const: airoha,an7581-uart
+          - const: airoha,en7523-uart
       - items:
           - enum:
               - exar,xr16l2552
diff --git a/Documentation/devicetree/bindings/serial/cdns,uart.yaml b/Documentation/devicetree/bindings/serial/cdns,uart.yaml
index 9d3e5c1d850227..11292de42c67b3 100644
--- a/Documentation/devicetree/bindings/serial/cdns,uart.yaml
+++ b/Documentation/devicetree/bindings/serial/cdns,uart.yaml
@@ -19,6 +19,7 @@ properties:
       - items:
           - enum:
               - axiado,ax3000-uart
+              - axiado,ax3005-uart
               - xlnx,zynqmp-uart
           - const: cdns,uart-r1p12
 
diff --git a/Documentation/devicetree/bindings/serial/renesas,hscif.yaml b/Documentation/devicetree/bindings/serial/renesas,hscif.yaml
index 4b3f98a46cd9de..409cab32de3d89 100644
--- a/Documentation/devicetree/bindings/serial/renesas,hscif.yaml
+++ b/Documentation/devicetree/bindings/serial/renesas,hscif.yaml
@@ -97,14 +97,14 @@ properties:
 
   dmas:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
     description:
       Must contain a list of pairs of references to DMA specifiers, one for
       transmission, and one for reception.
 
   dma-names:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
     items:
       enum:
         - tx
diff --git a/Documentation/devicetree/bindings/serial/renesas,scif.yaml b/Documentation/devicetree/bindings/serial/renesas,scif.yaml
index 82f54446835e6a..b465c93c8357fd 100644
--- a/Documentation/devicetree/bindings/serial/renesas,scif.yaml
+++ b/Documentation/devicetree/bindings/serial/renesas,scif.yaml
@@ -145,14 +145,14 @@ properties:
 
   dmas:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
     description:
       Must contain a list of pairs of references to DMA specifiers, one for
       transmission, and one for reception.
 
   dma-names:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
     items:
       enum:
         - tx
diff --git a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
index 664305de4986f9..86dc2591931120 100644
--- a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
+++ b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
@@ -80,6 +80,7 @@ properties:
               - starfive,jh7100-hsuart
               - starfive,jh7100-uart
               - starfive,jh7110-uart
+              - ti,tda54-uart
               - ultrarisc,dp1000-uart
           - const: snps,dw-apb-uart
       - const: snps,dw-apb-uart
diff --git a/Documentation/devicetree/bindings/soc/hisilicon/hisilicon,hip05-cpld.yaml b/Documentation/devicetree/bindings/soc/hisilicon/hisilicon,hip05-cpld.yaml
new file mode 100644
index 00000000000000..eaaa8f917bddd0
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/hisilicon/hisilicon,hip05-cpld.yaml
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/soc/hisilicon/hisilicon,hip05-cpld.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: HiSilicon HIP05 CPLD registers
+
+maintainers:
+  - Wei Xu <xuwei5@hisilicon.com>
+
+properties:
+  compatible:
+    const: hisilicon,hip05-cpld
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    local-bus {
+        #address-cells = <2>;
+        #size-cells = <1>;
+
+        cpld@100000000 {
+            compatible = "hisilicon,hip05-cpld";
+            reg = <0x1 0x0 0x100>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx-iomuxc-gpr.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx-iomuxc-gpr.yaml
index 721a67e84c137a..49813adce71c70 100644
--- a/Documentation/devicetree/bindings/soc/imx/fsl,imx-iomuxc-gpr.yaml
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx-iomuxc-gpr.yaml
@@ -47,10 +47,23 @@ properties:
   reg:
     maxItems: 1
 
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 1
+
+  ranges: true
+
   mux-controller:
     type: object
     $ref: /schemas/mux/reg-mux.yaml
 
+  bridge@18:
+    type: object
+    $ref: /schemas/display/bridge/fsl,ldb.yaml#
+    unevaluatedProperties: false
+
 patternProperties:
   "^ipu[12]_csi[01]_mux$":
     type: object
@@ -67,6 +80,19 @@ allOf:
       patternProperties:
         '^ipu[12]_csi[01]_mux$': false
 
+  - if:
+      properties:
+        compatible:
+          not:
+            contains:
+              const: fsl,imx6sx-iomuxc-gpr
+    then:
+      properties:
+        bridge@18: false
+        '#address-cells': false
+        '#size-cells': false
+        ranges: false
+
 additionalProperties: false
 
 required:
@@ -87,4 +113,40 @@ examples:
         };
     };
 
+  - |
+    #include <dt-bindings/clock/imx6sx-clock.h>
+
+    syscon@20e4000 {
+        compatible = "fsl,imx6sx-iomuxc-gpr", "fsl,imx6q-iomuxc-gpr", "syscon", "simple-mfd";
+        reg = <0x020e4000 0x4000>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges;
+
+        bridge@18 {
+            compatible = "fsl,imx6sx-ldb";
+            reg = <0x18 0x4>;
+            clocks = <&clks IMX6SX_CLK_LDB_DI0>;
+            clock-names = "ldb";
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+
+                    endpoint {
+                    };
+                };
+
+                port@1 {
+                    reg = <1>;
+
+                    endpoint {
+                    };
+                };
+            };
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/soc/mediatek/mediatek,mt2701-g3dsys.yaml b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mt2701-g3dsys.yaml
new file mode 100644
index 00000000000000..ad3df062d5db9a
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mt2701-g3dsys.yaml
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/soc/mediatek/mediatek,mt2701-g3dsys.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek G3D System Controller
+
+maintainers:
+  - Sean Wang <sean.wang@mediatek.com>
+  - Ryder Lee <ryder.lee@mediatek.com>
+
+description: |
+  The MediaTek G3D system controller provides clocks and reset control
+  for the GPU subsystem on MediaTek SoCs.
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - const: mediatek,mt2701-g3dsys
+          - const: syscon
+      - items:
+          - const: mediatek,mt7623-g3dsys
+          - const: mediatek,mt2701-g3dsys
+          - const: syscon
+
+  reg:
+    maxItems: 1
+
+  "#clock-cells":
+    const: 1
+
+  "#reset-cells":
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - "#clock-cells"
+  - "#reset-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+     soc {
+       #address-cells = <2>;
+       #size-cells = <2>;
+       g3dsys: syscon@13000000 {
+         compatible = "mediatek,mt7623-g3dsys",
+                      "mediatek,mt2701-g3dsys",
+                      "syscon";
+         reg = <0 0x13000000 0 0x200>;
+         #clock-cells = <1>;
+         #reset-cells = <1>;
+       };
+     };
diff --git a/Documentation/devicetree/bindings/soc/mediatek/mediatek,pwrap.yaml b/Documentation/devicetree/bindings/soc/mediatek/mediatek,pwrap.yaml
index e7c4a3984c601f..79b755858bad47 100644
--- a/Documentation/devicetree/bindings/soc/mediatek/mediatek,pwrap.yaml
+++ b/Documentation/devicetree/bindings/soc/mediatek/mediatek,pwrap.yaml
@@ -105,6 +105,9 @@ properties:
   pmic:
     type: object
 
+  regulators:
+    $ref: /schemas/regulator/mediatek,mt6380-regulator.yaml#
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/soc/nuvoton/nuvoton,npcm-gcr.yaml b/Documentation/devicetree/bindings/soc/nuvoton/nuvoton,npcm-gcr.yaml
index 23e7e4ea01ff73..9f6f009ec991a9 100644
--- a/Documentation/devicetree/bindings/soc/nuvoton/nuvoton,npcm-gcr.yaml
+++ b/Documentation/devicetree/bindings/soc/nuvoton/nuvoton,npcm-gcr.yaml
@@ -17,13 +17,17 @@ description:
 
 properties:
   compatible:
-    items:
-      - enum:
-          - nuvoton,wpcm450-gcr
-          - nuvoton,npcm750-gcr
-          - nuvoton,npcm845-gcr
-      - const: syscon
-      - const: simple-mfd
+    oneOf:
+      - items:
+          - enum:
+              - nuvoton,npcm845-gcr
+          - const: syscon
+      - items:
+          - enum:
+              - nuvoton,wpcm450-gcr
+              - nuvoton,npcm750-gcr
+          - const: syscon
+          - const: simple-mfd
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
index d2ae2ae87a9574..e94c4fd34177a0 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
@@ -71,7 +71,7 @@ properties:
     maxItems: 1
     description:
       Reference to the mailbox representing the outgoing doorbell in APCS for
-      this client, as described in mailbox/mailbox.txt
+      this client.
 
   "#clock-cells":
     const: 0
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml
index e51acdcaafaf6f..01250b556985b3 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml
@@ -120,7 +120,9 @@ allOf:
       properties:
         qcom,glink-channels:
           items:
-            - const: adsp_apps
+            - enum:
+                - adsp_apps
+                - modem_apps
         power-domains: false
     else:
       properties:
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml
index 686a7ef2f48af1..b384a5983d4335 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml
@@ -18,15 +18,22 @@ description:
 
 properties:
   compatible:
-    enum:
-      - qcom,rpmh-stats
-      - qcom,sdm845-rpmh-stats
-      - qcom,rpm-stats
-      # For older RPM firmware versions with fixed offset for the sleep stats
-      - qcom,apq8084-rpm-stats
-      - qcom,msm8226-rpm-stats
-      - qcom,msm8916-rpm-stats
-      - qcom,msm8974-rpm-stats
+    oneOf:
+      - enum:
+          - qcom,rpmh-stats
+          - qcom,sdm845-rpmh-stats
+          - qcom,rpm-stats
+          # For older RPM firmware versions with fixed offset for the sleep stats
+          - qcom,apq8084-rpm-stats
+          - qcom,msm8226-rpm-stats
+          - qcom,msm8916-rpm-stats
+          - qcom,msm8974-rpm-stats
+      # SoC-specific compatibles that also read subsystem-level stats,
+      # falling back to "qcom,rpm-stats" for SoC-level stats only
+      - items:
+          - enum:
+              - qcom,shikra-rpm-stats
+          - const: qcom,rpm-stats
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas.yaml
index eeef34e3d7d070..1673005a3d930e 100644
--- a/Documentation/devicetree/bindings/soc/renesas/renesas.yaml
+++ b/Documentation/devicetree/bindings/soc/renesas/renesas.yaml
@@ -508,6 +508,14 @@ properties:
               - renesas,r9a07g043u12 # RZ/G2UL Type-2
           - const: renesas,r9a07g043
 
+      - description: SolidRun RZ/G2UL based boards
+        items:
+          - enum:
+              - solidrun,rzg2ul-hummingboard-ripple # SolidRun RZ/G2UL HummingBoard Ripple
+          - const: solidrun,rzg2ul-sr-som
+          - const: renesas,r9a07g043u12 # RZ/G2UL Type-2
+          - const: renesas,r9a07g043
+
       - description: RZ/G2{L,LC} (R9A07G044)
         items:
           - enum:
@@ -519,6 +527,25 @@ properties:
               - renesas,r9a07g044l2 # Dual Cortex-A55 RZ/G2L
           - const: renesas,r9a07g044
 
+      - description: SolidRun RZ/G2L based boards
+        items:
+          - enum:
+              - solidrun,rzg2l-hummingboard-iiot   # SolidRun RZ/G2L HummingBoard IIoT
+              - solidrun,rzg2l-hummingboard-pro    # SolidRun RZ/G2L HummingBoard Pro
+              - solidrun,rzg2l-hummingboard-ripple # SolidRun RZ/G2L HummingBoard Ripple
+          - const: solidrun,rzg2l-sr-som
+          - const: renesas,r9a07g044l2 # Dual Cortex-A55 RZ/G2L
+          - const: renesas,r9a07g044
+
+      - description: SolidRun RZ/G2LC based boards
+        items:
+          - enum:
+              - solidrun,rzg2lc-hummingboard-iiot   # SolidRun RZ/G2LC HummingBoard IIoT
+              - solidrun,rzg2lc-hummingboard-ripple # SolidRun RZ/G2LC HummingBoard Ripple
+          - const: solidrun,rzg2lc-sr-som
+          - const: renesas,r9a07g044c2 # Dual Cortex-A55 RZ/G2LC
+          - const: renesas,r9a07g044
+
       - items:
           - enum:
               # MYIR Remi Pi SBC (MYB-YG2LX-REMI)
@@ -535,6 +562,16 @@ properties:
               - renesas,r9a07g054l2 # Dual Cortex-A55 RZ/V2L
           - const: renesas,r9a07g054
 
+      - description: SolidRun RZ/V2L based boards
+        items:
+          - enum:
+              - solidrun,rzv2l-hummingboard-iiot   # SolidRun RZ/V2L HummingBoard IIoT
+              - solidrun,rzv2l-hummingboard-pro    # SolidRun RZ/V2L HummingBoard Pro
+              - solidrun,rzv2l-hummingboard-ripple # SolidRun RZ/V2L HummingBoard Ripple
+          - const: solidrun,rzv2l-sr-som
+          - const: renesas,r9a07g054l2 # Dual Cortex-A55 RZ/V2L
+          - const: renesas,r9a07g054
+
       - description: RZ/G3S (R9A08G045)
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/soc/ti/ti,j721e-system-controller.yaml b/Documentation/devicetree/bindings/soc/ti/ti,j721e-system-controller.yaml
index f3bd0be3b279fb..364be49f8c7c71 100644
--- a/Documentation/devicetree/bindings/soc/ti/ti,j721e-system-controller.yaml
+++ b/Documentation/devicetree/bindings/soc/ti/ti,j721e-system-controller.yaml
@@ -53,9 +53,11 @@ patternProperties:
 
   "^clock-controller@[0-9a-f]+$":
     type: object
-    $ref: /schemas/clock/ti,am654-ehrpwm-tbclk.yaml#
+    oneOf:
+      - $ref: /schemas/clock/ti,am62-audio-refclk.yaml#
+      - $ref: /schemas/clock/ti,am654-ehrpwm-tbclk.yaml#
     description:
-      Clock provider for TI EHRPWM nodes.
+      Clock provider for TI EHRPWM or Audio Reference Clock nodes.
 
   "phy@[0-9a-f]+$":
     type: object
diff --git a/Documentation/devicetree/bindings/sound/adi,adau1372.yaml b/Documentation/devicetree/bindings/sound/adi,adau1372.yaml
index 9a7ff50a0a22d1..532550de62d7ce 100644
--- a/Documentation/devicetree/bindings/sound/adi,adau1372.yaml
+++ b/Documentation/devicetree/bindings/sound/adi,adau1372.yaml
@@ -52,11 +52,11 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         audio-codec@3c {
-                compatible = "adi,adau1372";
-                reg = <0x3c>;
-                #sound-dai-cells = <0>;
-                clock-names = "mclk";
-                clocks = <&adau1372z_xtal>;
+            compatible = "adi,adau1372";
+            reg = <0x3c>;
+            #sound-dai-cells = <0>;
+            clock-names = "mclk";
+            clocks = <&adau1372z_xtal>;
         };
     };
 
diff --git a/Documentation/devicetree/bindings/sound/adi,adau7118.yaml b/Documentation/devicetree/bindings/sound/adi,adau7118.yaml
index 11f59c29b5751f..8272cdaf66a85d 100644
--- a/Documentation/devicetree/bindings/sound/adi,adau7118.yaml
+++ b/Documentation/devicetree/bindings/sound/adi,adau7118.yaml
@@ -68,20 +68,20 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         adau7118_codec: audio-codec@14 {
-                compatible = "adi,adau7118";
-                reg = <0x14>;
-                #sound-dai-cells = <0>;
-                iovdd-supply = <&supply>;
-                dvdd-supply = <&supply>;
-                adi,pdm-clk-map = <1 1 0 0>;
-                adi,decimation-ratio = <16>;
+            compatible = "adi,adau7118";
+            reg = <0x14>;
+            #sound-dai-cells = <0>;
+            iovdd-supply = <&supply>;
+            dvdd-supply = <&supply>;
+            adi,pdm-clk-map = <1 1 0 0>;
+            adi,decimation-ratio = <16>;
         };
     };
 
     /* example with hw standalone mode */
     adau7118_codec_hw: adau7118-codec-hw {
-            compatible = "adi,adau7118";
-            #sound-dai-cells = <0>;
-            iovdd-supply = <&supply>;
-            dvdd-supply = <&supply>;
+        compatible = "adi,adau7118";
+        #sound-dai-cells = <0>;
+        iovdd-supply = <&supply>;
+        dvdd-supply = <&supply>;
     };
diff --git a/Documentation/devicetree/bindings/sound/airoha,an7581-afe.yaml b/Documentation/devicetree/bindings/sound/airoha,an7581-afe.yaml
new file mode 100644
index 00000000000000..80d9e87f147044
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/airoha,an7581-afe.yaml
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/airoha,an7581-afe.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Airoha AFE PCM controller for AN7581
+
+maintainers:
+  - Christian Marangi <ansuelsmth@gmail.com>
+
+properties:
+  compatible:
+    const: airoha,an7581-afe
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    afe@1fbe2200 {
+        compatible = "airoha,an7581-afe";
+        reg = <0x1fbe2200 0x9000>;
+
+        interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/airoha,an7581-wm8960.yaml b/Documentation/devicetree/bindings/sound/airoha,an7581-wm8960.yaml
new file mode 100644
index 00000000000000..8886fbb6b43ccf
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/airoha,an7581-wm8960.yaml
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/airoha,an7581-wm8960.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Airoha AN7581 sound card with WM8960 codec
+
+maintainers:
+  - Christian Marangi <ansuelsmth@gmail.com>
+
+allOf:
+  - $ref: sound-card-common.yaml#
+
+properties:
+  compatible:
+    const: airoha,an7581-wm8960-sound
+
+  platform:
+    type: object
+
+    additionalProperties: false
+
+    properties:
+      sound-dai:
+        items:
+          - description: The phandle of AN7581 platform.
+
+    required:
+      - sound-dai
+
+  codec:
+    type: object
+
+    additionalProperties: false
+
+    properties:
+      sound-dai:
+        items:
+          - description: The phandle of WM8960 i2c codec.
+
+    required:
+      - sound-dai
+
+required:
+  - compatible
+  - audio-routing
+  - platform
+  - codec
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    sound {
+        compatible = "airoha,an7581-wm8960-sound";
+        model = "an7581-wm8960";
+        audio-routing =
+            "Headphone", "HP_L",
+            "Headphone", "HP_R",
+            "LINPUT1", "AMIC",
+            "RINPUT1", "AMIC";
+
+        platform {
+            sound-dai = <&afe>;
+        };
+
+        codec {
+            sound-dai = <&wm8960>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml
index 6fdf605ad59ce9..2be3b34e4f5fed 100644
--- a/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml
+++ b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml
@@ -94,20 +94,20 @@ examples:
         audio-routing = "I2S ENCODER I2S IN", "I2S FIFO Playback";
 
         dai-link-0 {
-               sound-dai = <&i2s_fifo>;
+            sound-dai = <&i2s_fifo>;
         };
 
         dai-link-1 {
-                sound-dai = <&i2s_encoder>;
-                dai-format = "i2s";
-                mclk-fs = <256>;
+            sound-dai = <&i2s_encoder>;
+            dai-format = "i2s";
+            mclk-fs = <256>;
 
-                codec-0 {
-                        sound-dai = <&codec0>;
-                };
+            codec-0 {
+                sound-dai = <&codec0>;
+            };
 
-                codec-1 {
-                        sound-dai = <&codec1>;
-                };
+            codec-1 {
+                sound-dai = <&codec1>;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4619.yaml b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4619.yaml
index d412531ef9a2ba..7a7534493e6bb3 100644
--- a/Documentation/devicetree/bindings/sound/asahi-kasei,ak4619.yaml
+++ b/Documentation/devicetree/bindings/sound/asahi-kasei,ak4619.yaml
@@ -34,6 +34,12 @@ properties:
     $ref: audio-graph-port.yaml#
     unevaluatedProperties: false
 
+  powerdown-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the 'PDN' pin, if any. Needs to be set low to
+      power down the codec, hence the GPIO must be GPIO_ACTIVE_LOW.
+
 required:
   - compatible
   - reg
@@ -42,6 +48,8 @@ unevaluatedProperties: false
 
 examples:
   - |
+    #include <dt-bindings/gpio/gpio.h>
+
     i2c {
         #address-cells = <1>;
         #size-cells = <0>;
@@ -49,14 +57,16 @@ examples:
             compatible = "asahi-kasei,ak4619";
             reg = <0x10>;
 
+            powerdown-gpios = <&gpio1 0 GPIO_ACTIVE_LOW>;
+
             clocks = <&rcar_sound>;
             clock-names = "mclk";
 
             #sound-dai-cells = <0>;
             port {
-                 ak4619_endpoint: endpoint {
-                       remote-endpoint = <&rsnd_endpoint>;
-                  };
+                ak4619_endpoint: endpoint {
+                    remote-endpoint = <&rsnd_endpoint>;
+                };
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/sound/audio-graph-port.yaml b/Documentation/devicetree/bindings/sound/audio-graph-port.yaml
index d1cbfc5edd3ac3..565d723d039851 100644
--- a/Documentation/devicetree/bindings/sound/audio-graph-port.yaml
+++ b/Documentation/devicetree/bindings/sound/audio-graph-port.yaml
@@ -39,6 +39,7 @@ definitions:
     allOf:
       - $ref: /schemas/graph.yaml#/$defs/endpoint-base
       - $ref: /schemas/sound/dai-params.yaml#
+      - $ref: /schemas/sound/tdm-slot.yaml#
     properties:
       mclk-fs:
         $ref: simple-card.yaml#/definitions/mclk-fs
@@ -48,6 +49,9 @@ definitions:
       bitclock-inversion:
         description: dai-link uses bit clock inversion
         $ref: /schemas/types.yaml#/definitions/flag
+      continuous-clock:
+        description: dai-link bit clock runs continuously rather than gated
+        $ref: /schemas/types.yaml#/definitions/flag
       frame-master:
         description: Indicates dai-link frame master.
         oneOf:
@@ -82,12 +86,6 @@ definitions:
             - msb
             - lsb
 
-      dai-tdm-slot-num:
-        description: Number of slots in use.
-        $ref: /schemas/types.yaml#/definitions/uint32
-      dai-tdm-slot-width:
-        description: Width in bits for each slot.
-        $ref: /schemas/types.yaml#/definitions/uint32
       dai-tdm-slot-width-map:
         description: Mapping of sample widths to slot widths. For hardware
           that cannot support a fixed slot width or a slot width always
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs35l45.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs35l45.yaml
index 70f6c62aedca84..1ba809888a88c4 100644
--- a/Documentation/devicetree/bindings/sound/cirrus,cs35l45.yaml
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs35l45.yaml
@@ -53,6 +53,24 @@ properties:
     maximum: 3
     default: 2
 
+  cirrus,sync-lsw-txid:
+    description:
+      Transmitter ID to use for long software (LSW) data packets
+      on the SYNC bus.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 0
+    maximum: 7
+    default: 0
+
+  cirrus,sync-sw-txid:
+    description:
+      Transmitter ID to use for short software (SW) data packets
+      on the SYNC bus.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 0
+    maximum: 7
+    default: 0
+
 patternProperties:
   "^cirrus,gpio-ctrl[1-3]$":
     description:
@@ -135,25 +153,27 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
 
-        cs35l45: cs35l45@2 {
-          #sound-dai-cells = <1>;
-          compatible = "cirrus,cs35l45";
-          reg = <2>;
-          spi-max-frequency = <5000000>;
-          vdd-a-supply = <&dummy_vreg>;
-          vdd-batt-supply = <&dummy_vreg>;
-          reset-gpios = <&gpio 110 0>;
-          cirrus,asp-sdout-hiz-ctrl = <(CS35L45_ASP_TX_HIZ_UNUSED |
-                                        CS35L45_ASP_TX_HIZ_DISABLED)>;
-          cirrus,gpio-ctrl1 {
-             gpio-ctrl = <0x2>;
-          };
-          cirrus,gpio-ctrl2 {
-             gpio-ctrl = <0x2>;
-          };
-          cirrus,gpio-ctrl3 {
-             gpio-ctrl = <0x1>;
-             gpio-dir = <0x1>;
-          };
+        cs35l45: audio-codec@2 {
+            #sound-dai-cells = <1>;
+            compatible = "cirrus,cs35l45";
+            reg = <2>;
+            spi-max-frequency = <5000000>;
+            vdd-a-supply = <&dummy_vreg>;
+            vdd-batt-supply = <&dummy_vreg>;
+            reset-gpios = <&gpio 110 0>;
+            cirrus,asp-sdout-hiz-ctrl = <(CS35L45_ASP_TX_HIZ_UNUSED |
+                                          CS35L45_ASP_TX_HIZ_DISABLED)>;
+            cirrus,sync-lsw-txid = <0x1>;
+            cirrus,sync-sw-txid = <0x1>;
+            cirrus,gpio-ctrl1 {
+                gpio-ctrl = <0x2>;
+            };
+            cirrus,gpio-ctrl2 {
+                gpio-ctrl = <0x2>;
+            };
+            cirrus,gpio-ctrl3 {
+                gpio-ctrl = <0x1>;
+                gpio-dir = <0x1>;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs4270.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs4270.yaml
index 336e1177369428..61106294a1d568 100644
--- a/Documentation/devicetree/bindings/sound/cirrus,cs4270.yaml
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs4270.yaml
@@ -49,11 +49,11 @@ unevaluatedProperties: false
 examples:
   - |
     i2c {
-      #address-cells = <1>;
-      #size-cells = <0>;
+        #address-cells = <1>;
+        #size-cells = <0>;
 
-      codec@48 {
-          compatible = "cirrus,cs4270";
-          reg = <0x48>;
-      };
+        codec@48 {
+            compatible = "cirrus,cs4270";
+            reg = <0x48>;
+        };
     };
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml
index af599d8735e260..ab4c7d22dc0ebe 100644
--- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml
@@ -199,28 +199,28 @@ examples:
   - |
     #include <dt-bindings/sound/cs42l42.h>
     i2c {
-      #address-cells = <1>;
-      #size-cells = <0>;
-
-      cs42l42: cs42l42@48 {
-          compatible = "cirrus,cs42l42";
-          reg = <0x48>;
-          VA-supply = <&dummy_vreg>;
-          VP-supply = <&dummy_vreg>;
-          VCP-supply = <&dummy_vreg>;
-          VD_FILT-supply = <&dummy_vreg>;
-          VL-supply = <&dummy_vreg>;
-
-          reset-gpios = <&axi_gpio_0 1 0>;
-          interrupt-parent = <&gpio0>;
-          interrupts = <55 8>;
-
-          cirrus,ts-inv = <CS42L42_TS_INV_DIS>;
-          cirrus,ts-dbnc-rise = <CS42L42_TS_DBNCE_1000>;
-          cirrus,ts-dbnc-fall = <CS42L42_TS_DBNCE_0>;
-          cirrus,btn-det-init-dbnce = <100>;
-          cirrus,btn-det-event-dbnce = <10>;
-          cirrus,bias-lvls = <0x0F 0x08 0x04 0x01>;
-          cirrus,hs-bias-ramp-rate = <CS42L42_HSBIAS_RAMP_SLOW>;
-      };
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        audio-codec@48 {
+            compatible = "cirrus,cs42l42";
+            reg = <0x48>;
+            VA-supply = <&dummy_vreg>;
+            VP-supply = <&dummy_vreg>;
+            VCP-supply = <&dummy_vreg>;
+            VD_FILT-supply = <&dummy_vreg>;
+            VL-supply = <&dummy_vreg>;
+
+            reset-gpios = <&axi_gpio_0 1 0>;
+            interrupt-parent = <&gpio0>;
+            interrupts = <55 8>;
+
+            cirrus,ts-inv = <CS42L42_TS_INV_DIS>;
+            cirrus,ts-dbnc-rise = <CS42L42_TS_DBNCE_1000>;
+            cirrus,ts-dbnc-fall = <CS42L42_TS_DBNCE_0>;
+            cirrus,btn-det-init-dbnce = <100>;
+            cirrus,btn-det-event-dbnce = <10>;
+            cirrus,bias-lvls = <0x0F 0x08 0x04 0x01>;
+            cirrus,hs-bias-ramp-rate = <CS42L42_HSBIAS_RAMP_SLOW>;
+        };
     };
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml
index 7f8338e8ae369b..94062c1d9d45f7 100644
--- a/Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml
@@ -43,14 +43,14 @@ examples:
     #include <dt-bindings/gpio/gpio.h>
     #include <dt-bindings/interrupt-controller/irq.h>
     i2c {
-      #address-cells = <1>;
-      #size-cells = <0>;
-
-      jack_codec: codec@4b {
-          compatible = "cirrus,cs42l84";
-          reg = <0x4b>;
-          reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_LOW>;
-          interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>;
-          #sound-dai-cells = <0>;
-      };
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        jack_codec: codec@4b {
+            compatible = "cirrus,cs42l84";
+            reg = <0x4b>;
+            reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_LOW>;
+            interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>;
+            #sound-dai-cells = <0>;
+        };
     };
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42xx8.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42xx8.yaml
index a1ae548c4b7b5e..b5cdc7e537f8d9 100644
--- a/Documentation/devicetree/bindings/sound/cirrus,cs42xx8.yaml
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs42xx8.yaml
@@ -75,37 +75,37 @@ unevaluatedProperties: false
 examples:
   - |
     i2c {
-      #address-cells = <1>;
-      #size-cells = <0>;
-
-      codec@48 {
-          compatible = "cirrus,cs42888";
-          reg = <0x48>;
-          clocks = <&codec_mclk 0>;
-          clock-names = "mclk";
-          VA-supply = <&reg_audio>;
-          VD-supply = <&reg_audio>;
-          VLS-supply = <&reg_audio>;
-          VLC-supply = <&reg_audio>;
-          reset-gpios = <&gpio 1>;
-      };
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        codec@48 {
+            compatible = "cirrus,cs42888";
+            reg = <0x48>;
+            clocks = <&codec_mclk 0>;
+            clock-names = "mclk";
+            VA-supply = <&reg_audio>;
+            VD-supply = <&reg_audio>;
+            VLS-supply = <&reg_audio>;
+            VLC-supply = <&reg_audio>;
+            reset-gpios = <&gpio 1>;
+        };
     };
 
     spi {
-      #address-cells = <1>;
-      #size-cells = <0>;
-      cs-gpios = <&gpio 8 0>;
-
-      codec@0 {
-          compatible = "cirrus,cs42888";
-          reg = <0>;
-          spi-max-frequency = <6000000>;
-          clocks = <&codec_mclk 0>;
-          clock-names = "mclk";
-          VA-supply = <&reg_audio>;
-          VD-supply = <&reg_audio>;
-          VLS-supply = <&reg_audio>;
-          VLC-supply = <&reg_audio>;
-          reset-gpios = <&gpio 1>;
-      };
+        #address-cells = <1>;
+        #size-cells = <0>;
+        cs-gpios = <&gpio 8 0>;
+
+        codec@0 {
+            compatible = "cirrus,cs42888";
+            reg = <0>;
+            spi-max-frequency = <6000000>;
+            clocks = <&codec_mclk 0>;
+            clock-names = "mclk";
+            VA-supply = <&reg_audio>;
+            VD-supply = <&reg_audio>;
+            VLS-supply = <&reg_audio>;
+            VLC-supply = <&reg_audio>;
+            reset-gpios = <&gpio 1>;
+        };
     };
diff --git a/Documentation/devicetree/bindings/sound/davinci-evm-audio.txt b/Documentation/devicetree/bindings/sound/davinci-evm-audio.txt
deleted file mode 100644
index 963e100514c27d..00000000000000
--- a/Documentation/devicetree/bindings/sound/davinci-evm-audio.txt
+++ /dev/null
@@ -1,49 +0,0 @@
-* Texas Instruments SoC audio setups with TLV320AIC3X Codec
-
-Required properties:
-- compatible : "ti,da830-evm-audio" : forDM365/DA8xx/OMAPL1x/AM33xx
-- ti,model : The user-visible name of this sound complex.
-- ti,audio-codec : The phandle of the TLV320AIC3x audio codec
-- ti,mcasp-controller : The phandle of the McASP controller
-- ti,audio-routing : A list of the connections between audio components.
-  Each entry is a pair of strings, the first being the connection's sink,
-  the second being the connection's source. Valid names for sources and
-  sinks are the codec's pins, and the jacks on the board:
-
-Optional properties:
-- ti,codec-clock-rate : The Codec Clock rate (in Hz) applied to the Codec.
-- clocks : Reference to the master clock
-- clock-names : The clock should be named "mclk"
-- Either codec-clock-rate or the codec-clock reference has to be defined. If
-  the both are defined the driver attempts to set referenced clock to the
-  defined rate and takes the rate from the clock reference.
-
-  Board connectors:
-
-  * Headphone Jack
-  * Line Out
-  * Mic Jack
-  * Line In
-
-
-Example:
-
-sound {
-	compatible = "ti,da830-evm-audio";
-	ti,model = "DA830 EVM";
-	ti,audio-codec = <&tlv320aic3x>;
-	ti,mcasp-controller = <&mcasp1>;
-	ti,codec-clock-rate = <12000000>;
-	ti,audio-routing =
-		"Headphone Jack",       "HPLOUT",
-		"Headphone Jack",       "HPROUT",
-		"Line Out",             "LLOUT",
-		"Line Out",             "RLOUT",
-		"MIC3L",                "Mic Bias 2V",
-		"MIC3R",                "Mic Bias 2V",
-		"Mic Bias 2V",          "Mic Jack",
-		"LINE1L",               "Line In",
-		"LINE2L",               "Line In",
-		"LINE1R",               "Line In",
-		"LINE2R",               "Line In";
-};
diff --git a/Documentation/devicetree/bindings/sound/dialog,da7219.yaml b/Documentation/devicetree/bindings/sound/dialog,da7219.yaml
index 19137abdba3e5d..852f27e9dffea3 100644
--- a/Documentation/devicetree/bindings/sound/dialog,da7219.yaml
+++ b/Documentation/devicetree/bindings/sound/dialog,da7219.yaml
@@ -195,44 +195,44 @@ examples:
   - |
     #include <dt-bindings/interrupt-controller/irq.h>
     i2c {
-      #address-cells = <1>;
-      #size-cells = <0>;
-
-      codec: da7219@1a {
-          compatible = "dlg,da7219";
-          reg = <0x1a>;
-
-          interrupt-parent = <&gpio6>;
-          interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
-
-          VDD-supply = <&vdd_reg>;
-          VDDMIC-supply = <&vddmic_reg>;
-          VDDIO-supply = <&vddio_reg>;
-
-          #clock-cells = <1>;
-          clock-output-names = "da7219-dai-wclk", "da7219-dai-bclk";
-
-          clocks = <&clks 201>;
-          clock-names = "mclk";
-
-          dlg,micbias-lvl = <2600>;
-          dlg,mic-amp-in-sel = "diff";
-
-          da7219_aad {
-              dlg,btn-cfg = <50>;
-              dlg,mic-det-thr = <500>;
-              dlg,jack-ins-deb = <20>;
-              dlg,jack-ins-det-pty = "low";
-              dlg,jack-det-rate = "32_64";
-              dlg,jack-rem-deb = <1>;
-
-              dlg,a-d-btn-thr = <0xa>;
-              dlg,d-b-btn-thr = <0x16>;
-              dlg,b-c-btn-thr = <0x21>;
-              dlg,c-mic-btn-thr = <0x3E>;
-
-              dlg,btn-avg = <4>;
-              dlg,adc-1bit-rpt = <1>;
-          };
-      };
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        codec: audio-codec@1a {
+            compatible = "dlg,da7219";
+            reg = <0x1a>;
+
+            interrupt-parent = <&gpio6>;
+            interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+
+            VDD-supply = <&vdd_reg>;
+            VDDMIC-supply = <&vddmic_reg>;
+            VDDIO-supply = <&vddio_reg>;
+
+            #clock-cells = <1>;
+            clock-output-names = "da7219-dai-wclk", "da7219-dai-bclk";
+
+            clocks = <&clks 201>;
+            clock-names = "mclk";
+
+            dlg,micbias-lvl = <2600>;
+            dlg,mic-amp-in-sel = "diff";
+
+            da7219_aad {
+                dlg,btn-cfg = <50>;
+                dlg,mic-det-thr = <500>;
+                dlg,jack-ins-deb = <20>;
+                dlg,jack-ins-det-pty = "low";
+                dlg,jack-det-rate = "32_64";
+                dlg,jack-rem-deb = <1>;
+
+                dlg,a-d-btn-thr = <0xa>;
+                dlg,d-b-btn-thr = <0x16>;
+                dlg,b-c-btn-thr = <0x21>;
+                dlg,c-mic-btn-thr = <0x3E>;
+
+                dlg,btn-avg = <4>;
+                dlg,adc-1bit-rpt = <1>;
+            };
+        };
     };
diff --git a/Documentation/devicetree/bindings/sound/esstech,es9039q2m.yaml b/Documentation/devicetree/bindings/sound/esstech,es9039q2m.yaml
new file mode 100644
index 00000000000000..15db30af4b6c5c
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/esstech,es9039q2m.yaml
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/esstech,es9039q2m.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ESS Technology ES9039Q2M audio DAC
+
+maintainers:
+  - Karl Asseily <karl@asseily.com>
+
+description:
+  The ES9039Q2M is a 32-bit two-channel audio DAC supporting PCM to 768 kHz,
+  DSD64 to DSD1024, DoP and S/PDIF. It is controlled over I2C or SPI when the
+  MODE pin selects software mode; this binding covers the I2C interface. The
+  part contains an asynchronous sample rate converter, so the system clock need
+  not be synchronous with the audio bit or frame clocks.
+
+  The four supplies are all nominally 3.3V and the datasheet requires a set
+  order - AVDD first, then VCCA about 200us later, then AVCC_DAC1 and
+  AVCC_DAC2, and the reverse on the way down. DVDD, the part's 1.2V rail, is
+  on-chip and needs only its decoupling capacitor - it is not a supply input
+  and has no property here.
+
+allOf:
+  - $ref: dai-common.yaml#
+
+properties:
+  compatible:
+    const: esstech,es9039q2m
+
+  reg:
+    description:
+      I2C address, selected by the ADDR0 and ADDR1 pins - 0x48, 0x49, 0x4a or
+      0x4b for GND/GND, GND/AVDD, AVDD/GND and AVDD/AVDD respectively.
+    enum: [ 0x48, 0x49, 0x4a, 0x4b ]
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: mclk
+
+  avdd-supply:
+    description:
+      3.3V supply for the part's internal digital regulator, pin AVDD. It is
+      the first rail up and the last down, per the ordering described above.
+
+  vcca-supply:
+    description: 3.3V analogue supply, pin VCCA.
+
+  avcc-dac1-supply:
+    description:
+      3.3V reference supply for the channel 1 analogue output stage, pin
+      AVCC_DAC1. The datasheet allows this rail to move by up to 15% where a
+      calibrating regulator tracks the DAC's output impedance.
+
+  avcc-dac2-supply:
+    description:
+      3.3V reference supply for the channel 2 analogue output stage, pin
+      AVCC_DAC2, with the same 15% allowance as channel 1.
+
+  "#sound-dai-cells":
+    const: 0
+
+required:
+  - compatible
+  - reg
+  - avdd-supply
+  - vcca-supply
+  - avcc-dac1-supply
+  - avcc-dac2-supply
+  - "#sound-dai-cells"
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        audio-codec@48 {
+            compatible = "esstech,es9039q2m";
+            reg = <0x48>;
+            #sound-dai-cells = <0>;
+            clocks = <&dac_mclk>;
+            clock-names = "mclk";
+            avdd-supply = <&dac_3v3>;
+            vcca-supply = <&dac_3v3>;
+            avcc-dac1-supply = <&dac_avcc1>;
+            avcc-dac2-supply = <&dac_avcc2>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/sound/everest,es8316.yaml b/Documentation/devicetree/bindings/sound/everest,es8316.yaml
index f4ff23120c5b55..9060358a9080c6 100644
--- a/Documentation/devicetree/bindings/sound/everest,es8316.yaml
+++ b/Documentation/devicetree/bindings/sound/everest,es8316.yaml
@@ -84,6 +84,11 @@ properties:
   "#sound-dai-cells":
     const: 0
 
+  everest,jack-detect-inverted:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Defined to invert the jack detection.
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/sound/foursemi,fs2105s.yaml b/Documentation/devicetree/bindings/sound/foursemi,fs2105s.yaml
index 4da735317e0f28..17c938a565e5af 100644
--- a/Documentation/devicetree/bindings/sound/foursemi,fs2105s.yaml
+++ b/Documentation/devicetree/bindings/sound/foursemi,fs2105s.yaml
@@ -63,7 +63,7 @@ properties:
       a. Register initialization settings
       b. DSP effect parameters
       c. Multi-scene sound effect configurations(optional)
-      It's gernerated by FourSemi's tuning tool.
+      It's generated by FourSemi's tuning tool.
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/sound/fsl,easrc.yaml b/Documentation/devicetree/bindings/sound/fsl,easrc.yaml
index d5727f8bfb0b56..a53ec4f6f1c08e 100644
--- a/Documentation/devicetree/bindings/sound/fsl,easrc.yaml
+++ b/Documentation/devicetree/bindings/sound/fsl,easrc.yaml
@@ -55,6 +55,14 @@ properties:
       - const: imx/easrc/easrc-imx8mn.bin
     description: The coefficient table for the filters
 
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+    patternProperties:
+      "^port@[0-3]$":
+        $ref: audio-graph-port.yaml#
+        unevaluatedProperties: false
+        description: port for an independent conversion path
+
   fsl,asrc-rate:
     $ref: /schemas/types.yaml#/definitions/uint32
     minimum: 8000
@@ -106,4 +114,41 @@ examples:
         firmware-name = "imx/easrc/easrc-imx8mn.bin";
         fsl,asrc-rate = <8000>;
         fsl,asrc-format = <2>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                playback-only;
+                endpoint {
+                    remote-endpoint = <&fe00_ep>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                playback-only;
+                endpoint {
+                    remote-endpoint = <&fe01_ep>;
+                };
+            };
+
+            port@2 {
+                reg = <2>;
+                capture-only;
+                endpoint {
+                    remote-endpoint = <&fe02_ep>;
+                };
+            };
+
+            port@3 {
+                reg = <3>;
+                capture-only;
+                endpoint {
+                    remote-endpoint = <&fe03_ep>;
+                };
+            };
+        };
     };
diff --git a/Documentation/devicetree/bindings/sound/fsl,imx-asrc.yaml b/Documentation/devicetree/bindings/sound/fsl,imx-asrc.yaml
index 608defc93c1e9f..67d2826eef3fb5 100644
--- a/Documentation/devicetree/bindings/sound/fsl,imx-asrc.yaml
+++ b/Documentation/devicetree/bindings/sound/fsl,imx-asrc.yaml
@@ -78,9 +78,19 @@ properties:
   power-domains:
     maxItems: 1
 
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+    patternProperties:
+      "^port@[0-2]$":
+        $ref: audio-graph-port.yaml#
+        unevaluatedProperties: false
+        description: port for an independent conversion path
+
   port:
     $ref: audio-graph-port.yaml#
     unevaluatedProperties: false
+    deprecated: true
+    description: deprecated, use ports instead
 
   fsl,asrc-rate:
     $ref: /schemas/types.yaml#/definitions/uint32
@@ -125,6 +135,10 @@ required:
   - fsl,asrc-width
 
 allOf:
+  - not:
+      required:
+        - port
+        - ports
   - $ref: dai-common.yaml#
   - if:
       properties:
@@ -180,11 +194,32 @@ examples:
         fsl,asrc-rate = <48000>;
         fsl,asrc-width = <16>;
 
-        port {
-            playback-only;
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                playback-only;
+                endpoint {
+                    remote-endpoint = <&fe00_ep>;
+                };
+            };
+
+            port@1 {
+                reg = <1>;
+                capture-only;
+                endpoint {
+                    remote-endpoint = <&fe01_ep>;
+                };
+            };
 
-            asrc_endpoint: endpoint {
-                remote-endpoint = <&fe00_ep>;
+            port@2 {
+                reg = <2>;
+                capture-only;
+                endpoint {
+                    remote-endpoint = <&fe02_ep>;
+                };
             };
-       };
+        };
     };
diff --git a/Documentation/devicetree/bindings/sound/fsl,sai.yaml b/Documentation/devicetree/bindings/sound/fsl,sai.yaml
index ba65b3f3d0662b..eb0b9d8d907f0c 100644
--- a/Documentation/devicetree/bindings/sound/fsl,sai.yaml
+++ b/Documentation/devicetree/bindings/sound/fsl,sai.yaml
@@ -260,13 +260,13 @@ examples:
                 playback-only;
 
                 sai1_endpoint0: endpoint {
-                   dai-tdm-slot-num = <8>;
-                   dai-tdm-slot-width = <32>;
-                   dai-tdm-slot-width-map = <32 8 32>;
-                   dai-format = "dsp_a";
-                   bitclock-master;
-                   frame-master;
-                   remote-endpoint = <&mcodec01_ep>;
+                    dai-tdm-slot-num = <8>;
+                    dai-tdm-slot-width = <32>;
+                    dai-tdm-slot-width-map = <32 8 32>;
+                    dai-format = "dsp_a";
+                    bitclock-master;
+                    frame-master;
+                    remote-endpoint = <&mcodec01_ep>;
                 };
             };
 
diff --git a/Documentation/devicetree/bindings/sound/hisilicon,hi6210-i2s.yaml b/Documentation/devicetree/bindings/sound/hisilicon,hi6210-i2s.yaml
index 5171f984630bec..b126982647241e 100644
--- a/Documentation/devicetree/bindings/sound/hisilicon,hi6210-i2s.yaml
+++ b/Documentation/devicetree/bindings/sound/hisilicon,hi6210-i2s.yaml
@@ -48,6 +48,10 @@ properties:
       The dai cell indexes reference the following interfaces:
         0: S2 interface
 
+  ports:
+    $ref: audio-graph-port.yaml#/definitions/ports
+    unevaluatedProperties: false
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
index 950e3eab2942e1..27636466ba6385 100644
--- a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
+++ b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
@@ -105,10 +105,10 @@ examples:
             format = "i2s";
             fsl,mclk-equal-bclk;
             cpu {
-                 sound-dai = <&sai1>;
+                sound-dai = <&sai1>;
             };
             codec {
-                 sound-dai = <&ak4458_1>, <&ak4458_2>;
+                sound-dai = <&ak4458_1>, <&ak4458_2>;
             };
         };
         fe-dai-link {
diff --git a/Documentation/devicetree/bindings/sound/imx-audmux.yaml b/Documentation/devicetree/bindings/sound/imx-audmux.yaml
index dab45c310670b5..501af1add3c122 100644
--- a/Documentation/devicetree/bindings/sound/imx-audmux.yaml
+++ b/Documentation/devicetree/bindings/sound/imx-audmux.yaml
@@ -101,10 +101,10 @@ examples:
         mux-ssi1 {
             fsl,audmux-port = <0>;
             fsl,port-config = <
-                IMX_AUDMUX_V2_PTCR_SYN		0
-                IMX_AUDMUX_V2_PTCR_TFSEL(2)	0
-                IMX_AUDMUX_V2_PTCR_TCSEL(2)	0
-                IMX_AUDMUX_V2_PTCR_TFSDIR	0
+                IMX_AUDMUX_V2_PTCR_SYN      0
+                IMX_AUDMUX_V2_PTCR_TFSEL(2) 0
+                IMX_AUDMUX_V2_PTCR_TCSEL(2) 0
+                IMX_AUDMUX_V2_PTCR_TFSDIR   0
                 IMX_AUDMUX_V2_PTCR_TCLKDIR      IMX_AUDMUX_V2_PDCR_RXDSEL(2)
               >;
         };
diff --git a/Documentation/devicetree/bindings/sound/invensense,ics43432.yaml b/Documentation/devicetree/bindings/sound/invensense,ics43432.yaml
index 7bd984817aa94a..054f80390b8d3d 100644
--- a/Documentation/devicetree/bindings/sound/invensense,ics43432.yaml
+++ b/Documentation/devicetree/bindings/sound/invensense,ics43432.yaml
@@ -42,10 +42,10 @@ examples:
         compatible = "invensense,ics43432";
 
         port {
-          endpoint {
-            remote-endpoint = <&i2s1_endpoint>;
-            dai-format = "i2s";
-          };
+            endpoint {
+                remote-endpoint = <&i2s1_endpoint>;
+                dai-format = "i2s";
+            };
         };
 
     };
diff --git a/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml b/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml
index dc7f4afbb7775f..69ea611d4250a1 100644
--- a/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml
+++ b/Documentation/devicetree/bindings/sound/loongson,ls-audio-card.yaml
@@ -79,7 +79,7 @@ examples:
             sound-dai = <&i2s>;
         };
         codec {
-             sound-dai = <&es8323>;
+            sound-dai = <&es8323>;
         };
     };
 
@@ -104,6 +104,6 @@ examples:
         };
 
         codec {
-             sound-dai = <&es8388>;
+            sound-dai = <&es8388>;
         };
     };
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml
index 30f331366566e1..10df1e7c7eb966 100644
--- a/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml
@@ -32,7 +32,7 @@ properties:
     maxItems: 1
 
   clocks:
-    minItems: 34
+    minItems: 33
     items:
       - description: audio infra sys clock
       - description: top audio mux 1
@@ -74,7 +74,7 @@ properties:
       - description: audio APLL root pd
 
   clock-names:
-    minItems: 34
+    minItems: 33
     items:
       - const: infra_sys_audio_clk
       - const: top_audio_mux1_sel
@@ -132,8 +132,10 @@ allOf:
     then:
       properties:
         clocks:
-          maxItems: 34
+          minItems: 33
+          maxItems: 33
         clock-names:
-          maxItems: 34
+          minItems: 33
+          maxItems: 33
 
 additionalProperties: false
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt8173-afe-pcm.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt8173-afe-pcm.yaml
index d8993b5d457a90..4c66160e9e1b95 100644
--- a/Documentation/devicetree/bindings/sound/mediatek,mt8173-afe-pcm.yaml
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt8173-afe-pcm.yaml
@@ -84,15 +84,15 @@ examples:
                  <&topckgen CLK_TOP_I2S2_M_SEL>,
                  <&topckgen CLK_TOP_I2S3_M_SEL>,
                  <&topckgen CLK_TOP_I2S3_B_SEL>;
-         clock-names = "infra_sys_audio_clk",
-                       "top_pdn_audio",
-                       "top_pdn_aud_intbus",
-                       "bck0",
-                       "bck1",
-                       "i2s0_m",
-                       "i2s1_m",
-                       "i2s2_m",
-                       "i2s3_m",
-                       "i2s3_b";
-          memory-region = <&afe_dma_mem>;
+        clock-names = "infra_sys_audio_clk",
+                      "top_pdn_audio",
+                      "top_pdn_aud_intbus",
+                      "bck0",
+                      "bck1",
+                      "i2s0_m",
+                      "i2s1_m",
+                      "i2s2_m",
+                      "i2s3_m",
+                      "i2s3_b";
+        memory-region = <&afe_dma_mem>;
     };
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt8183-audio.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt8183-audio.yaml
index 031b0fa7b4dc1b..261247ca2c8733 100644
--- a/Documentation/devicetree/bindings/sound/mediatek,mt8183-audio.yaml
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt8183-audio.yaml
@@ -182,47 +182,47 @@ examples:
                  <&topckgen CLK_TOP_APLL12_DIV4>,
                  <&topckgen CLK_TOP_APLL12_DIVB>,
                  <&clk26m>;
-      clock-names = "aud_afe_clk",
-                    "aud_dac_clk",
-                    "aud_dac_predis_clk",
-                    "aud_adc_clk",
-                    "aud_adc_adda6_clk",
-                    "aud_apll22m_clk",
-                    "aud_apll24m_clk",
-                    "aud_apll1_tuner_clk",
-                    "aud_apll2_tuner_clk",
-                    "aud_i2s1_bclk_sw",
-                    "aud_i2s2_bclk_sw",
-                    "aud_i2s3_bclk_sw",
-                    "aud_i2s4_bclk_sw",
-                    "aud_tdm_clk",
-                    "aud_tml_clk",
-                    "aud_infra_clk",
-                    "mtkaif_26m_clk",
-                    "top_mux_audio",
-                    "top_mux_aud_intbus",
-                    "top_syspll_d2_d4",
-                    "top_mux_aud_1",
-                    "top_apll1_ck",
-                    "top_mux_aud_2",
-                    "top_apll2_ck",
-                    "top_mux_aud_eng1",
-                    "top_apll1_d8",
-                    "top_mux_aud_eng2",
-                    "top_apll2_d8",
-                    "top_i2s0_m_sel",
-                    "top_i2s1_m_sel",
-                    "top_i2s2_m_sel",
-                    "top_i2s3_m_sel",
-                    "top_i2s4_m_sel",
-                    "top_i2s5_m_sel",
-                    "top_apll12_div0",
-                    "top_apll12_div1",
-                    "top_apll12_div2",
-                    "top_apll12_div3",
-                    "top_apll12_div4",
-                    "top_apll12_divb",
-                    "top_clk26m_clk";
+        clock-names = "aud_afe_clk",
+                      "aud_dac_clk",
+                      "aud_dac_predis_clk",
+                      "aud_adc_clk",
+                      "aud_adc_adda6_clk",
+                      "aud_apll22m_clk",
+                      "aud_apll24m_clk",
+                      "aud_apll1_tuner_clk",
+                      "aud_apll2_tuner_clk",
+                      "aud_i2s1_bclk_sw",
+                      "aud_i2s2_bclk_sw",
+                      "aud_i2s3_bclk_sw",
+                      "aud_i2s4_bclk_sw",
+                      "aud_tdm_clk",
+                      "aud_tml_clk",
+                      "aud_infra_clk",
+                      "mtkaif_26m_clk",
+                      "top_mux_audio",
+                      "top_mux_aud_intbus",
+                      "top_syspll_d2_d4",
+                      "top_mux_aud_1",
+                      "top_apll1_ck",
+                      "top_mux_aud_2",
+                      "top_apll2_ck",
+                      "top_mux_aud_eng1",
+                      "top_apll1_d8",
+                      "top_mux_aud_eng2",
+                      "top_apll2_d8",
+                      "top_i2s0_m_sel",
+                      "top_i2s1_m_sel",
+                      "top_i2s2_m_sel",
+                      "top_i2s3_m_sel",
+                      "top_i2s4_m_sel",
+                      "top_i2s5_m_sel",
+                      "top_apll12_div0",
+                      "top_apll12_div1",
+                      "top_apll12_div2",
+                      "top_apll12_div3",
+                      "top_apll12_div4",
+                      "top_apll12_divb",
+                      "top_clk26m_clk";
     };
 
 ...
diff --git a/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml b/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml
index 037f21443ad14c..d5b358838e2a8a 100644
--- a/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml
+++ b/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml
@@ -167,30 +167,30 @@ examples:
                 "HDMI1", "TX";
 
         hs-playback-dai-link {
-                link-name = "I2S0";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&da7219>;
-                };
+            link-name = "I2S0";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&da7219>;
+            };
         };
 
         hs-capture-dai-link {
-                link-name = "I2S1";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&da7219>;
-                };
+            link-name = "I2S1";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&da7219>;
+            };
         };
 
         spk-dp-playback-dai-link {
-                link-name = "I2S3";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&anx_bridge_dp>, <&max98357a>;
-                };
+            link-name = "I2S3";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&anx_bridge_dp>, <&max98357a>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml b/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml
index ed93f18ef985cf..7f553575c73d3f 100644
--- a/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml
+++ b/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml
@@ -171,30 +171,30 @@ examples:
                 "HDMI1", "TX";
 
         hs-playback-dai-link {
-                link-name = "I2S0";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&rt5682s 0>;
-                };
+            link-name = "I2S0";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&rt5682s 0>;
+            };
         };
 
         hs-capture-dai-link {
-                link-name = "I2S1";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&rt5682s 0>;
-                };
+            link-name = "I2S1";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&rt5682s 0>;
+            };
         };
 
         spk-hdmi-playback-dai-link {
-                link-name = "I2S3";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&it6505dptx>, <&rt1019p>;
-                };
+            link-name = "I2S3";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&it6505dptx>, <&rt1019p>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml b/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml
index c4e68f31aaabd7..841aab0178e3fc 100644
--- a/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml
+++ b/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml
@@ -165,38 +165,38 @@ examples:
                 "Speakers", "Speaker";
 
         spk-playback-dai-link {
-                link-name = "I2S3";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&rt1015p>;
-                };
+            link-name = "I2S3";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&rt1015p>;
+            };
         };
 
         hs-playback-dai-link {
-                link-name = "I2S8";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&rt5682 0>;
-                };
+            link-name = "I2S8";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&rt5682 0>;
+            };
         };
 
         hs-capture-dai-link {
-                link-name = "I2S9";
-                dai-format = "i2s";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&rt5682 0>;
-                };
+            link-name = "I2S9";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&rt5682 0>;
+            };
         };
 
         displayport-dai-link {
-                link-name = "TDM";
-                dai-format = "dsp_a";
-                codec {
-                        sound-dai = <&anx_bridge_dp>;
-                };
+            link-name = "TDM";
+            dai-format = "dsp_a";
+            codec {
+                sound-dai = <&anx_bridge_dp>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/sound/mt8195-mt6359.yaml b/Documentation/devicetree/bindings/sound/mt8195-mt6359.yaml
index 356e1feee96209..c6bd286780c9d4 100644
--- a/Documentation/devicetree/bindings/sound/mt8195-mt6359.yaml
+++ b/Documentation/devicetree/bindings/sound/mt8195-mt6359.yaml
@@ -156,46 +156,46 @@ examples:
                 "Ext Spk", "Speaker";
 
         mm-dai-link {
-                link-name = "ETDM1_IN_BE";
-                mediatek,clk-provider = "cpu";
+            link-name = "ETDM1_IN_BE";
+            mediatek,clk-provider = "cpu";
         };
 
         hs-playback-dai-link {
-                link-name = "ETDM1_OUT_BE";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&headset_codec>;
-                };
+            link-name = "ETDM1_OUT_BE";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&headset_codec>;
+            };
         };
 
         hs-capture-dai-link {
-                link-name = "ETDM2_IN_BE";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&headset_codec>;
-                };
+            link-name = "ETDM2_IN_BE";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&headset_codec>;
+            };
         };
 
         spk-playback-dai-link {
-                link-name = "ETDM2_OUT_BE";
-                mediatek,clk-provider = "cpu";
-                codec {
-                        sound-dai = <&spk_amplifier>;
-                };
+            link-name = "ETDM2_OUT_BE";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&spk_amplifier>;
+            };
         };
 
         hdmi-dai-link {
-                link-name = "ETDM3_OUT_BE";
-                codec {
-                        sound-dai = <&hdmi_tx>;
-                };
+            link-name = "ETDM3_OUT_BE";
+            codec {
+                sound-dai = <&hdmi_tx>;
+            };
         };
 
         displayport-dai-link {
-                link-name = "DPTX_BE";
-                codec {
-                        sound-dai = <&dp_tx>;
-                };
+            link-name = "DPTX_BE";
+            codec {
+                sound-dai = <&dp_tx>;
+            };
         };
     };
 
diff --git a/Documentation/devicetree/bindings/sound/nuvoton,nau8360.yaml b/Documentation/devicetree/bindings/sound/nuvoton,nau8360.yaml
new file mode 100644
index 00000000000000..3edab944a4e996
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/nuvoton,nau8360.yaml
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/nuvoton,nau8360.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Nuvoton NAU83G60 Stereo Class-D Amplifier with DSP
+
+description:
+  Stereo Class-D Amplifier with DSP and I/V-sense.
+  This device supports I2C.
+
+maintainers:
+  - Neo Chang <YLCHANG2@nuvoton.com>
+
+properties:
+  compatible:
+    enum:
+      - nuvoton,nau8360
+
+  reg:
+    description: The I2C address is determined by the hardware pins.
+    enum: [0x1a, 0x1b, 0x4a, 0x4b]
+
+  "#sound-dai-cells":
+    const: 0
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: mclk
+
+  firmware-name:
+    description:
+      List of DSP core firmware filenames. Both Left and Right DSP firmware
+      must be provided and loaded. In PBTL mode, the Left DSP must use the
+      same binary file as the Right DSP.
+    items:
+      - description: Left DSP core firmware filename.
+      - description: Right DSP core firmware filename.
+
+  nuvoton,pbtl-enable:
+    type: boolean
+    description: NAU83G60 supports PBTL mode for mono output.
+
+  nuvoton,dac-cur-enable:
+    type: boolean
+    description:
+      Adjust DAC output current to match speaker impedance and prevent
+      hardware damage. +3.2dB when present, 0dB by default.
+
+  nuvoton,dsp-rx-slot-mapping:
+    description:
+      Configures the TDM slot routing for DSP RX functions.
+      Each integer assigns a specific DSP internal function to a TDM slot.
+      The indices of the array correspond to the following hardware functions
+        (0) DACL
+        (1) DACR
+        (2) ANCL
+        (3) ANCR
+      Assigned slots must be unique, except 255 which disables the function.
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    minItems: 4
+    maxItems: 4
+    items:
+      enum: [ 0, 1, 2, 3, 4, 5, 6, 7, 255 ]
+
+  nuvoton,dsp-tx-slot-mapping:
+    description:
+      Configures the TDM slot routing for DSP TX functions.
+      Each integer assigns a specific DSP internal function to a TDM slot.
+      The indices of the array correspond to the following hardware functions
+        (0) AECL (Acoustic echo cancellation left)
+        (1) AECR (Acoustic echo cancellation right)
+        (2) ISNSL (Current sense left)
+        (3) ISNSR (Current sense right)
+        (4) VSNSL (Voltage sense left)
+        (5) VSNSR (Voltage sense right)
+        (6) TJ (Junction temperature)
+        (7) VBAT (Battery voltage)
+      Assigned slots must be unique, except 255 which disables the function.
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    minItems: 8
+    maxItems: 8
+    items:
+      enum: [ 0, 1, 2, 3, 4, 5, 6, 7, 255 ]
+
+required:
+  - compatible
+  - reg
+  - firmware-name
+
+allOf:
+  - $ref: dai-common.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        codec@1a {
+            compatible = "nuvoton,nau8360";
+            reg = <0x1a>;
+            #sound-dai-cells = <0>;
+            firmware-name = "NAU83G60.kcs.bin.l", "NAU83G60.kcs.bin.r";
+            nuvoton,dac-cur-enable;
+            nuvoton,pbtl-enable;
+            nuvoton,dsp-rx-slot-mapping = <0 1 2 3>;
+            nuvoton,dsp-tx-slot-mapping = <0 1 2 3 4 5 6 7>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6apm-lpass-dais.yaml b/Documentation/devicetree/bindings/sound/qcom,q6apm-lpass-dais.yaml
index 4878c424ef030e..78587139f155f8 100644
--- a/Documentation/devicetree/bindings/sound/qcom,q6apm-lpass-dais.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,q6apm-lpass-dais.yaml
@@ -75,11 +75,11 @@ examples:
         #size-cells = <0>;
 
         dai@10 {
-           reg = <PRIMARY_MI2S_RX>;
-           clocks = <&q6prmcc LPASS_CLK_ID_PRI_MI2S_IBIT
-                              LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
-                    <&q6prmcc LPASS_CLK_ID_MCLK_1
-                              LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
-           clock-names = "bclk", "mclk";
+            reg = <PRIMARY_MI2S_RX>;
+            clocks = <&q6prmcc LPASS_CLK_ID_PRI_MI2S_IBIT
+                               LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+                     <&q6prmcc LPASS_CLK_ID_MCLK_1
+                               LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+            clock-names = "bclk", "mclk";
         };
     };
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml
index 3b03e2acd67e08..63f53837bd9fb2 100644
--- a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml
@@ -108,10 +108,15 @@ patternProperties:
           properties:
             reg:
               contains:
-                # TDM DAI ID range from PRIMARY_TDM_RX_0 - QUINARY_TDM_TX_7
+                # TDM DAI ID range from PRIMARY_TDM_RX_0 - QUINARY_TDM_TX_7 and
+                # AIF_TDM_RX_0 - AIF_TDM_TX_12
                 items:
-                  minimum: 24
-                  maximum: 103
+                  oneOf:
+                    - minimum: 24
+                      maximum: 103
+                    - minimum: 179
+                      maximum: 204
+
         then:
           required:
             - qcom,tdm-sync-mode
@@ -127,7 +132,8 @@ patternProperties:
               contains:
                 # MI2S DAI ID range PRIMARY_MI2S_RX - QUATERNARY_MI2S_TX and
                 # QUINARY_MI2S_RX - QUINARY_MI2S_TX and
-                # LPI_MI2S_RX_0 - LPI_MI2S_TX_6
+                # LPI_MI2S_RX_0 - LPI_MI2S_TX_6 and
+                # AIF_MI2S_RX_0 - AIF_MI2S_TX_12
                 items:
                   oneOf:
                     - minimum: 16
@@ -136,6 +142,8 @@ patternProperties:
                       maximum: 128
                     - minimum: 137
                       maximum: 152
+                    - minimum: 153
+                      maximum: 178
         then:
           required:
             - qcom,sd-lines
diff --git a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
index 1536fcd96d6873..fc7bccdace0f42 100644
--- a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
@@ -54,6 +54,7 @@ properties:
           - qcom,sm8250-sndcard
           - qcom,sm8450-sndcard
           - qcom,x1e80100-sndcard
+          - thundercomm,qcs6490-rubikpi3-sndcard
 
   audio-routing:
     $ref: /schemas/types.yaml#/definitions/non-unique-string-array
diff --git a/Documentation/devicetree/bindings/sound/qcom,wcd9378-sdw.yaml b/Documentation/devicetree/bindings/sound/qcom,wcd9378-sdw.yaml
new file mode 100644
index 00000000000000..62d85b1dca7f2e
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,wcd9378-sdw.yaml
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/qcom,wcd9378-sdw.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm WCD9378 (Tambora) SoundWire codec
+
+maintainers:
+  - Srinivas Kandagatla <srinivas.kandagatla@oss.qualcomm.com>
+
+description:
+  Qualcomm WCD9378 (Tambora) SoundWire codec. The same silicon can be
+  fused for a mobile (non-SDCA) mode or an SDCA-compliant mode; the
+  qcom,sdca-compliant property selects the latter and is required by
+  this driver.
+
+allOf:
+  - $ref: dai-common.yaml#
+
+properties:
+  compatible:
+    const: sdw20217011000
+
+  reg:
+    maxItems: 1
+
+  qcom,sdca-compliant:
+    description:
+      Present when the codec is fused as SDCA-compliant. The mobile
+      (non-SDCA) variant carries the same class-ID compatible and is
+      handled by a different driver.
+    type: boolean
+
+  qcom,port-mapping:
+    description:
+      Each entry maps a slave data port to a master data port.
+      Entries are in order starting from slave port 1; slave port 0
+      is reserved and not represented. SDCA-compliant mode uses 8
+      entries covering DP1..DP8.
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    minItems: 8
+    maxItems: 8
+
+  reset-gpios:
+    description: GPIO used to release the codec from reset.
+    maxItems: 1
+
+  vdd-buck-supply:
+    description: 1.8 V analog buck supply.
+
+  vdd-rxtx-supply:
+    description: 1.8 V RX/TX supply.
+
+  vdd-io-supply:
+    description: Digital I/O supply.
+
+  vdd-mic-bias-supply:
+    description: Mic bias supply.
+
+  '#sound-dai-cells':
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - qcom,sdca-compliant
+  - qcom,port-mapping
+  - reset-gpios
+  - vdd-buck-supply
+  - vdd-rxtx-supply
+  - vdd-io-supply
+  - vdd-mic-bias-supply
+  - '#sound-dai-cells'
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    soundwire@7630000 {
+        reg = <0x07630000 0x10000>;
+        #address-cells = <2>;
+        #size-cells = <0>;
+
+        wcd9378c_sdw: audio-codec@0,3 {
+            compatible = "sdw20217011000";
+            reg = <0 3>;
+
+            qcom,sdca-compliant;
+
+            qcom,port-mapping = <2 2 3 4 5 6 7 8>;
+
+            reset-gpios = <&tlmm 191 GPIO_ACTIVE_LOW>;
+
+            vdd-buck-supply = <&vreg_l15b_1p8>;
+            vdd-rxtx-supply = <&vreg_l15b_1p8>;
+            vdd-io-supply = <&vreg_l18b_1p2>;
+            vdd-mic-bias-supply = <&vreg_bob1>;
+
+            #sound-dai-cells = <1>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/sound/realtek,rt5677.yaml b/Documentation/devicetree/bindings/sound/realtek,rt5677.yaml
index ae27ae78b1b221..f76aad0d30168b 100644
--- a/Documentation/devicetree/bindings/sound/realtek,rt5677.yaml
+++ b/Documentation/devicetree/bindings/sound/realtek,rt5677.yaml
@@ -44,6 +44,14 @@ properties:
   '#gpio-cells':
     const: 2
 
+  clocks:
+    items:
+      - description: phandle and clock specifier for the codec MCLK1 input.
+
+  clock-names:
+    items:
+      - const: mclk1
+
   realtek,pow-ldo2-gpio:
     maxItems: 1
     description: CODEC's POW_LDO2 pin.
@@ -127,6 +135,8 @@ examples:
             interrupts = <3 IRQ_TYPE_LEVEL_HIGH>;
             gpio-controller;
             #gpio-cells = <2>;
+            clocks = <&osc>;
+            clock-names = "mclk1";
             realtek,pow-ldo2-gpio = <&gpio 3 GPIO_ACTIVE_HIGH>;
             realtek,reset-gpio = <&gpio 3 GPIO_ACTIVE_LOW>;
             realtek,in1-differential;
diff --git a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
index e8a2acb926460d..d3516fb1b1300d 100644
--- a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
+++ b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
@@ -432,14 +432,14 @@ examples:
                       "ssi.1", "ssi.0";
 
         rcar_sound,dvc {
-               dvc0: dvc-0 {
-                    dmas = <&audma0 0xbc>;
-                    dma-names = "tx";
-               };
-               dvc1: dvc-1 {
-                    dmas = <&audma0 0xbe>;
-                    dma-names = "tx";
-               };
+            dvc0: dvc-0 {
+                dmas = <&audma0 0xbc>;
+                dma-names = "tx";
+            };
+            dvc1: dvc-1 {
+                dmas = <&audma0 0xbe>;
+                dma-names = "tx";
+            };
         };
 
         rcar_sound,mix {
diff --git a/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml b/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml
index 1394f78281fc25..d696fb33d9b8e7 100644
--- a/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml
+++ b/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml
@@ -95,20 +95,20 @@ examples:
     ssi0: ssi@10049c00 {
         compatible = "renesas,r9a07g044-ssi",
                      "renesas,rz-ssi";
-            reg = <0x10049c00 0x400>;
-            interrupts = <GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 327 IRQ_TYPE_EDGE_RISING>,
-                         <GIC_SPI 328 IRQ_TYPE_EDGE_RISING>;
-            interrupt-names = "int_req", "dma_rx", "dma_tx";
-            clocks = <&cpg CPG_MOD R9A07G044_SSI0_PCLK2>,
-                     <&cpg CPG_MOD R9A07G044_SSI0_PCLK_SFR>,
-                     <&audio_clk1>,
-                     <&audio_clk2>;
-            clock-names = "ssi", "ssi_sfr", "audio_clk1", "audio_clk2";
-            power-domains = <&cpg>;
-            resets = <&cpg R9A07G044_SSI0_RST_M2_REG>;
-            dmas = <&dmac 0x2655>,
-                   <&dmac 0x2656>;
-            dma-names = "tx", "rx";
-            #sound-dai-cells = <0>;
+        reg = <0x10049c00 0x400>;
+        interrupts = <GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 327 IRQ_TYPE_EDGE_RISING>,
+                     <GIC_SPI 328 IRQ_TYPE_EDGE_RISING>;
+        interrupt-names = "int_req", "dma_rx", "dma_tx";
+        clocks = <&cpg CPG_MOD R9A07G044_SSI0_PCLK2>,
+                 <&cpg CPG_MOD R9A07G044_SSI0_PCLK_SFR>,
+                 <&audio_clk1>,
+                 <&audio_clk2>;
+        clock-names = "ssi", "ssi_sfr", "audio_clk1", "audio_clk2";
+        power-domains = <&cpg>;
+        resets = <&cpg R9A07G044_SSI0_RST_M2_REG>;
+        dmas = <&dmac 0x2655>,
+               <&dmac 0x2656>;
+        dma-names = "tx", "rx";
+        #sound-dai-cells = <0>;
     };
diff --git a/Documentation/devicetree/bindings/sound/stericsson,ux500-msp-i2s.yaml b/Documentation/devicetree/bindings/sound/stericsson,ux500-msp-i2s.yaml
new file mode 100644
index 00000000000000..bdfcd5376cfe25
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/stericsson,ux500-msp-i2s.yaml
@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/stericsson,ux500-msp-i2s.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ST-Ericsson Ux500 Multichannel Serial Port
+
+maintainers:
+  - Linus Walleij <linusw@kernel.org>
+
+description:
+  The Ux500 Multichannel Serial Port (MSP) is a synchronous serial audio
+  interface supporting I2S and PCM/TDM protocols. Data is transferred to and
+  from memory using the Ux500 DMA controller.
+
+allOf:
+  - $ref: dai-common.yaml#
+
+properties:
+  compatible:
+    const: stericsson,ux500-msp-i2s
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  v-ape-supply:
+    description: Analog power engine supply
+
+  dmas:
+    minItems: 1
+    maxItems: 2
+
+  dma-names:
+    minItems: 1
+    maxItems: 2
+    items:
+      enum: [ rx, tx ]
+
+  clocks:
+    items:
+      - description: MSP functional clock
+      - description: MSP APB bus clock
+
+  clock-names:
+    items:
+      - const: msp
+      - const: apb_pclk
+
+  resets:
+    maxItems: 1
+
+  '#sound-dai-cells':
+    const: 0
+
+  port:
+    $ref: audio-graph-port.yaml#
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - v-ape-supply
+  - dmas
+  - dma-names
+  - clocks
+  - clock-names
+  - resets
+  - '#sound-dai-cells'
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/reset/stericsson,db8500-prcc-reset.h>
+
+    msp@80123000 {
+        compatible = "stericsson,ux500-msp-i2s";
+        reg = <0x80123000 0x1000>;
+        interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+        v-ape-supply = <&db8500_vape_reg>;
+        dmas = <&dma 31 0 0x12>, <&dma 31 0 0x10>;
+        dma-names = "rx", "tx";
+        clocks = <&prcc_kclk 1 3>, <&prcc_pclk 1 3>;
+        clock-names = "msp", "apb_pclk";
+        resets = <&prcc_reset DB8500_PRCC_1 DB8500_PRCC_1_RESET_MSP0>;
+        #sound-dai-cells = <0>;
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/ti,da830-evm-audio.yaml b/Documentation/devicetree/bindings/sound/ti,da830-evm-audio.yaml
new file mode 100644
index 00000000000000..b4a1cf67582b60
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/ti,da830-evm-audio.yaml
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/ti,da830-evm-audio.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments SoC audio setups with TLV320AIC3X Codec
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+properties:
+  compatible:
+    const: ti,da830-evm-audio
+
+  ti,model:
+    $ref: /schemas/types.yaml#/definitions/string
+    description: The user-visible name of this sound complex.
+
+  ti,audio-codec:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: The phandle of the TLV320AIC3x audio codec
+
+  ti,mcasp-controller:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: The phandle of the McASP controller
+
+  ti,audio-routing:
+    $ref: /schemas/types.yaml#/definitions/non-unique-string-array
+    description:
+      A list of the connections between audio components. Each entry is a
+      pair of strings, the first being the connection's sink, the second
+      being the connection's source. Valid names for sources and sinks
+      are the codec's pins, and the jacks on the board (Headphone Jack,
+      Line Out, Mic Jack, Line In).
+
+  ti,codec-clock-rate:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: The Codec Clock rate (in Hz) applied to the Codec.
+
+  clocks:
+    maxItems: 1
+    description: Reference to the master clock
+
+  clock-names:
+    const: mclk
+
+required:
+  - compatible
+  - ti,model
+  - ti,audio-codec
+  - ti,mcasp-controller
+  - ti,audio-routing
+
+anyOf:
+  - required:
+      - ti,codec-clock-rate
+  - required:
+      - clocks
+
+additionalProperties: false
+
+examples:
+  - |
+    sound {
+        compatible = "ti,da830-evm-audio";
+        ti,model = "DA830 EVM";
+        ti,audio-codec = <&tlv320aic3x>;
+        ti,mcasp-controller = <&mcasp1>;
+        ti,codec-clock-rate = <12000000>;
+        ti,audio-routing =
+            "Headphone Jack",       "HPLOUT",
+            "Headphone Jack",       "HPROUT",
+            "Line Out",             "LLOUT",
+            "Line Out",             "RLOUT",
+            "MIC3L",                "Mic Bias 2V",
+            "MIC3R",                "Mic Bias 2V",
+            "Mic Bias 2V",          "Mic Jack",
+            "LINE1L",               "Line In",
+            "LINE2L",               "Line In",
+            "LINE1R",               "Line In",
+            "LINE2R",               "Line In";
+    };
diff --git a/Documentation/devicetree/bindings/sound/ti,tas2781.yaml b/Documentation/devicetree/bindings/sound/ti,tas2781.yaml
index aa5a317b5a3618..50c7792b24f811 100644
--- a/Documentation/devicetree/bindings/sound/ti,tas2781.yaml
+++ b/Documentation/devicetree/bindings/sound/ti,tas2781.yaml
@@ -219,9 +219,7 @@ allOf:
   - if:
       properties:
         compatible:
-          contains:
-            enum:
-              - ti,tas2781
+          const: ti,tas2781
     then:
       properties:
         reg:
diff --git a/Documentation/devicetree/bindings/sound/ti,tas5805m.yaml b/Documentation/devicetree/bindings/sound/ti,tas5805m.yaml
index c2c2835a9e1dbd..f2ab274b02cd8b 100644
--- a/Documentation/devicetree/bindings/sound/ti,tas5805m.yaml
+++ b/Documentation/devicetree/bindings/sound/ti,tas5805m.yaml
@@ -45,13 +45,13 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         tas5805m: tas5805m@2c {
-                reg = <0x2c>;
-                compatible = "ti,tas5805m";
+            reg = <0x2c>;
+            compatible = "ti,tas5805m";
 
-                pvdd-supply = <&audiopwr>;
-                pdn-gpios = <&tlmm 160 0>;
+            pvdd-supply = <&audiopwr>;
+            pdn-gpios = <&tlmm 160 0>;
 
-                ti,dsp-config-name = "mono_pbtl_48khz";
+            ti,dsp-config-name = "mono_pbtl_48khz";
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/sound/ti,tlv320dac3100.yaml b/Documentation/devicetree/bindings/sound/ti,tlv320dac3100.yaml
index 10299064cbc690..3bdaab1e50841f 100644
--- a/Documentation/devicetree/bindings/sound/ti,tlv320dac3100.yaml
+++ b/Documentation/devicetree/bindings/sound/ti,tlv320dac3100.yaml
@@ -121,6 +121,6 @@ examples:
             AVDD-supply = <&regulator>;
             IOVDD-supply = <&regulator>;
             DVDD-supply = <&regulator>;
-       };
+        };
     };
 
diff --git a/Documentation/devicetree/bindings/sound/ti,tpa6130a2.yaml b/Documentation/devicetree/bindings/sound/ti,tpa6130a2.yaml
index a42bf9bde69400..5634b601ce65c8 100644
--- a/Documentation/devicetree/bindings/sound/ti,tpa6130a2.yaml
+++ b/Documentation/devicetree/bindings/sound/ti,tpa6130a2.yaml
@@ -50,6 +50,6 @@ examples:
             reg = <0x60>;
             Vdd-supply = <&vmmc2>;
             power-gpio = <&gpio4 2 GPIO_ACTIVE_HIGH>;
-       };
+        };
     };
 
diff --git a/Documentation/devicetree/bindings/sound/ux500-mop500.txt b/Documentation/devicetree/bindings/sound/ux500-mop500.txt
deleted file mode 100644
index 48e071c96b465e..00000000000000
--- a/Documentation/devicetree/bindings/sound/ux500-mop500.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-* MOP500 Audio Machine Driver
-
-This node is responsible for linking together all ux500 Audio Driver components.
-
-Required properties:
- - compatible              : "stericsson,snd-soc-mop500"
-
-Non-standard properties:
- - stericsson,cpu-dai      : Phandle to the CPU-side DAI
- - stericsson,audio-codec  : Phandle to the Audio CODEC
- - stericsson,card-name    : Over-ride default card name
-
-Example:
-
-	sound {
-		compatible = "stericsson,snd-soc-mop500";
-
-		stericsson,cpu-dai = <&msp1 &msp3>;
-		stericsson,audio-codec = <&codec>;
-	};
-
-	msp1: msp@80124000 {
-		compatible = "stericsson,ux500-msp-i2s";
-		reg = <0x80124000 0x1000>;
-		interrupts = <0 62 0x4>;
-		v-ape-supply = <&db8500_vape_reg>;
-	};
-
-	msp3: msp@80125000 {
-		compatible = "stericsson,ux500-msp-i2s";
-		reg = <0x80125000 0x1000>;
-		interrupts = <0 62 0x4>;
-		v-ape-supply = <&db8500_vape_reg>;
-	};
-
-	codec: ab8500-codec {
-		compatible = "stericsson,ab8500-codec";
-		stericsson,earpeice-cmv = <950>; /* Units in mV. */
-	};
diff --git a/Documentation/devicetree/bindings/sound/ux500-msp.txt b/Documentation/devicetree/bindings/sound/ux500-msp.txt
deleted file mode 100644
index 7dd1b96160f597..00000000000000
--- a/Documentation/devicetree/bindings/sound/ux500-msp.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-* ux500 MSP (CPU-side Digital Audio Interface)
-
-Required properties:
- - compatible       :"stericsson,ux500-msp-i2s"
- - reg              : Physical base address and length of the device's registers.
-
-Optional properties:
- - interrupts       : The interrupt output from the device.
- - <name>-supply    : Phandle to the regulator <name> supply
-
-Example:
-
-	sound {
-		compatible = "stericsson,snd-soc-mop500";
-
-		stericsson,platform-pcm-dma = <&pcm>;
-		stericsson,cpu-dai = <&msp1 &msp3>;
-		stericsson,audio-codec = <&codec>;
-	};
-
-	pcm: ux500-pcm {
-		compatible = "stericsson,ux500-pcm";
-	};
-
-	msp1: msp@80124000 {
-		compatible = "stericsson,ux500-msp-i2s";
-		reg = <0x80124000 0x1000>;
-		interrupts = <0 62 0x4>;
-		v-ape-supply = <&db8500_vape_reg>;
-	};
-
-	msp3: msp@80125000 {
-		compatible = "stericsson,ux500-msp-i2s";
-		reg = <0x80125000 0x1000>;
-		interrupts = <0 62 0x4>;
-		v-ape-supply = <&db8500_vape_reg>;
-	};
-
-	codec: ab8500-codec {
-		compatible = "stericsson,ab8500-codec";
-		stericsson,earpeice-cmv = <950>; /* Units in mV. */
-	};
diff --git a/Documentation/devicetree/bindings/spi/amlogic,a4-spisg.yaml b/Documentation/devicetree/bindings/spi/amlogic,a4-spisg.yaml
index 9bfb8089f7ea2f..427d8883009cc9 100644
--- a/Documentation/devicetree/bindings/spi/amlogic,a4-spisg.yaml
+++ b/Documentation/devicetree/bindings/spi/amlogic,a4-spisg.yaml
@@ -11,12 +11,11 @@ maintainers:
   - Xianwei Zhao <xianwei.zhao@amlogic.com>
   - Sunny Luo <sunny.luo@amlogic.com>
 
-allOf:
-  - $ref: spi-controller.yaml#
-
 properties:
   compatible:
-    const: amlogic,a4-spisg
+    enum:
+      - amlogic,a4-spisg
+      - amlogic,a9-spisg
 
   reg:
     maxItems: 1
@@ -35,6 +34,16 @@ properties:
   resets:
     maxItems: 1
 
+  amlogic,mo-idle-output:
+    description: |
+      Controls the MOSI output level when the controller is idle.
+      0 - Drive MOSI low
+      1 - Drive MOSI high
+      2 - Follow the last transmitted bit
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [0, 1, 2]
+    default: 0
+
 required:
   - compatible
   - reg
@@ -42,6 +51,19 @@ required:
   - clocks
   - clock-names
 
+allOf:
+  - $ref: spi-controller.yaml#
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              enum:
+                - amlogic,a9-spisg
+    then:
+      properties:
+        amlogic,mo-idle-output: false
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml b/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml
index 4e28e6e9d8e058..f0ba15402a4c14 100644
--- a/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml
+++ b/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml
@@ -16,10 +16,15 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - amlogic,meson-gx-spicc # SPICC controller on Amlogic GX and compatible SoCs
-      - amlogic,meson-axg-spicc # SPICC controller on Amlogic AXG and compatible SoCs
-      - amlogic,meson-g12a-spicc # SPICC controller on Amlogic G12A and compatible SoCs
+    oneOf:
+      - items:
+          - enum:
+              - amlogic,t7-spicc
+          - const: amlogic,meson-g12a-spicc
+      - enum:
+          - amlogic,meson-gx-spicc # SPICC controller on Amlogic GX and compatible SoCs
+          - amlogic,meson-axg-spicc # SPICC controller on Amlogic AXG and compatible SoCs
+          - amlogic,meson-g12a-spicc # SPICC controller on Amlogic G12A and compatible SoCs
 
   interrupts:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/spi/aspeed,ast2600-fmc.yaml b/Documentation/devicetree/bindings/spi/aspeed,ast2600-fmc.yaml
index 80e542624cc62e..ad830aebb7980a 100644
--- a/Documentation/devicetree/bindings/spi/aspeed,ast2600-fmc.yaml
+++ b/Documentation/devicetree/bindings/spi/aspeed,ast2600-fmc.yaml
@@ -62,21 +62,21 @@ examples:
         interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 
         flash@0 {
-            reg = < 0 >;
+            reg = <0>;
             compatible = "jedec,spi-nor";
             spi-max-frequency = <50000000>;
             spi-rx-bus-width = <2>;
         };
 
         flash@1 {
-            reg = < 1 >;
+            reg = <1>;
             compatible = "jedec,spi-nor";
             spi-max-frequency = <50000000>;
             spi-rx-bus-width = <2>;
         };
 
         flash@2 {
-            reg = < 2 >;
+            reg = <2>;
             compatible = "jedec,spi-nor";
             spi-max-frequency = <50000000>;
             spi-rx-bus-width = <2>;
diff --git a/Documentation/devicetree/bindings/spi/microchip,pic32mzda-spi.yaml b/Documentation/devicetree/bindings/spi/microchip,pic32mzda-spi.yaml
new file mode 100644
index 00000000000000..4669b521fa1fce
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/microchip,pic32mzda-spi.yaml
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/microchip,pic32mzda-spi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip PIC32MZDA SPI Controller
+
+maintainers:
+  - Thomas Bogendoerfer <tsbogend@alpha.franken.de>
+
+allOf:
+  - $ref: spi-controller.yaml#
+
+properties:
+  compatible:
+    const: microchip,pic32mzda-spi
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    items:
+      - description: Fault interrupt
+      - description: Receive interrupt
+      - description: Transmit interrupt
+
+  interrupt-names:
+    items:
+      - const: fault
+      - const: rx
+      - const: tx
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    items:
+      - const: mck0
+
+  cs-gpios:
+    maxItems: 1
+
+  dmas:
+    items:
+      - description: RX DMA channel
+      - description: TX DMA channel
+
+  dma-names:
+    items:
+      - const: spi-rx
+      - const: spi-tx
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - interrupt-names
+  - clocks
+  - clock-names
+  - cs-gpios
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    #include <dt-bindings/gpio/gpio.h>
+
+    spi@1f821000 {
+        compatible = "microchip,pic32mzda-spi";
+        reg = <0x1f821000 0x200>;
+        interrupts = <109 IRQ_TYPE_LEVEL_HIGH>,
+                     <110 IRQ_TYPE_LEVEL_HIGH>,
+                     <111 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-names = "fault", "rx", "tx";
+        clocks = <&PBCLK2>;
+        clock-names = "mck0";
+        cs-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>;
+        dmas = <&dma 134>, <&dma 135>;
+        dma-names = "spi-rx", "spi-tx";
+    };
diff --git a/Documentation/devicetree/bindings/spi/microchip,spi-pic32.txt b/Documentation/devicetree/bindings/spi/microchip,spi-pic32.txt
deleted file mode 100644
index 79de379f4dc067..00000000000000
--- a/Documentation/devicetree/bindings/spi/microchip,spi-pic32.txt
+++ /dev/null
@@ -1,34 +0,0 @@
-Microchip PIC32 SPI Master controller
-
-Required properties:
-- compatible: Should be "microchip,pic32mzda-spi".
-- reg: Address and length of register space for the device.
-- interrupts: Should contain all three spi interrupts in sequence
-              of <fault-irq>, <receive-irq>, <transmit-irq>.
-- interrupt-names: Should be "fault", "rx", "tx" in order.
-- clocks: Phandle of the clock generating SPI clock on the bus.
-- clock-names: Should be "mck0".
-- cs-gpios: Specifies the gpio pins to be used for chipselects.
-            See: Documentation/devicetree/bindings/spi/spi-bus.txt
-
-Optional properties:
-- dmas: Two or more DMA channel specifiers following the convention outlined
-        in Documentation/devicetree/bindings/dma/dma.txt
-- dma-names: Names for the dma channels. There must be at least one channel
-             named "spi-tx" for transmit and named "spi-rx" for receive.
-
-Example:
-
-spi1: spi@1f821000 {
-        compatible = "microchip,pic32mzda-spi";
-        reg = <0x1f821000 0x200>;
-        interrupts = <109 IRQ_TYPE_LEVEL_HIGH>,
-                     <110 IRQ_TYPE_LEVEL_HIGH>,
-                     <111 IRQ_TYPE_LEVEL_HIGH>;
-        interrupt-names = "fault", "rx", "tx";
-        clocks = <&PBCLK2>;
-        clock-names = "mck0";
-        cs-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>;
-        dmas = <&dma 134>, <&dma 135>;
-        dma-names = "spi-rx", "spi-tx";
-};
diff --git a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml
index e0c7047ae8adb6..41e423a42654d0 100644
--- a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml
+++ b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml
@@ -54,6 +54,11 @@ properties:
               - renesas,msiof-r8a779h0      # R-Car V4M
           - const: renesas,rcar-gen4-msiof  # generic R-Car Gen4
                                             # compatible device
+      - items:
+          - enum:
+              - renesas,msiof-r8a78000      # R-Car X5H
+          - const: renesas,rcar-gen5-msiof  # generic R-Car Gen5
+                                            # compatible device
       - items:
           - const: renesas,sh-msiof  # deprecated
 
@@ -100,11 +105,11 @@ properties:
 
   dmas:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
 
   dma-names:
     minItems: 2
-    maxItems: 4
+    maxItems: 6
     items:
       enum: [ tx, rx ]
 
diff --git a/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml b/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml
index 880a9f62456678..b3ab89f70df9a2 100644
--- a/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml
+++ b/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml
@@ -91,6 +91,14 @@ properties:
       The delay from the default sample time before the actual
       sample of the rxd input signal occurs.
 
+      This describes the board rather than the peripheral, namely the extra
+      delay this particular design needs, for example because of the flight time
+      of the clock and data signals between controller and peripheral.
+
+      Timing parameters of the peripheral itself, such as its
+      clock-to-output-valid time, are the same on every board using that chip
+      and should be described with the chip.
+
   spi-tx-bus-width:
     description:
       Bus width to the SPI bus used for write transfers.
@@ -142,6 +150,13 @@ properties:
     minItems: 2
     maxItems: 4
 
+  spi-device-addr:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description:
+      Device addresses used when multiple peripherals share a single chip
+      select. The array allows one logical peripheral to comprise multiple
+      physical devices, with one address per physical device.
+
   st,spi-midi-ns:
     deprecated: true
     description: |
diff --git a/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml b/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml
index 1593a1183a367b..44c864fd5fc7a9 100644
--- a/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml
+++ b/Documentation/devicetree/bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml
@@ -27,6 +27,7 @@ properties:
           - enum:
               - qcom,hawi-spmi-pmic-arb
               - qcom,kaanapali-spmi-pmic-arb
+              - qcom,maili-spmi-pmic-arb
           - const: qcom,glymur-spmi-pmic-arb
       - enum:
           - qcom,glymur-spmi-pmic-arb
diff --git a/Documentation/devicetree/bindings/sram/qcom,imem.yaml b/Documentation/devicetree/bindings/sram/qcom,imem.yaml
index b4f082f2af3a35..2f120376a8f478 100644
--- a/Documentation/devicetree/bindings/sram/qcom,imem.yaml
+++ b/Documentation/devicetree/bindings/sram/qcom,imem.yaml
@@ -18,7 +18,6 @@ properties:
     items:
       - enum:
           - qcom,apq8064-imem
-          - qcom,ipq5424-imem
           - qcom,msm8226-imem
           - qcom,msm8974-imem
           - qcom,msm8976-imem
diff --git a/Documentation/devicetree/bindings/sram/sram.yaml b/Documentation/devicetree/bindings/sram/sram.yaml
index b83fc6831947c9..6ee0b14647222c 100644
--- a/Documentation/devicetree/bindings/sram/sram.yaml
+++ b/Documentation/devicetree/bindings/sram/sram.yaml
@@ -37,12 +37,14 @@ properties:
         - qcom,eliza-imem
         - qcom,glymur-imem
         - qcom,hawi-imem
+        - qcom,ipq5424-imem
         - qcom,kaanapali-imem
         - qcom,maili-imem
         - qcom,milos-imem
         - qcom,rpm-msg-ram
         - qcom,shikra-imem
         - rockchip,rk3288-pmu-sram
+        - samsung,exynos850-acpm-sram
 
   reg:
     maxItems: 1
@@ -159,6 +161,7 @@ if:
           enum:
             - qcom,rpm-msg-ram
             - rockchip,rk3288-pmu-sram
+            - samsung,exynos850-acpm-sram
 then:
   required:
     - "#address-cells"
diff --git a/Documentation/devicetree/bindings/thermal/qcom,pm8775-mbg-tm.yaml b/Documentation/devicetree/bindings/thermal/qcom,pm8775-mbg-tm.yaml
index 2e084d040625c0..ef8361a553891f 100644
--- a/Documentation/devicetree/bindings/thermal/qcom,pm8775-mbg-tm.yaml
+++ b/Documentation/devicetree/bindings/thermal/qcom,pm8775-mbg-tm.yaml
@@ -21,7 +21,12 @@ description:
 
 properties:
   compatible:
-    const: qcom,pm8775-mbg-tm
+    oneOf:
+      - const: qcom,pm8775-mbg-tm
+      - items:
+          - enum:
+              - qcom,pmau0102-mbg-tm
+          - const: qcom,pm8775-mbg-tm
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/thermal/renesas,r9a08g045-tsu.yaml b/Documentation/devicetree/bindings/thermal/renesas,r9a08g045-tsu.yaml
index 573e2b9d37524b..20fc1a8000e33e 100644
--- a/Documentation/devicetree/bindings/thermal/renesas,r9a08g045-tsu.yaml
+++ b/Documentation/devicetree/bindings/thermal/renesas,r9a08g045-tsu.yaml
@@ -17,7 +17,9 @@ $ref: thermal-sensor.yaml#
 
 properties:
   compatible:
-    const: renesas,r9a08g045-tsu
+    enum:
+      - renesas,r9a08g045-tsu
+      - renesas,r9a08g046-tsu
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/thermal/thermal-idle.yaml b/Documentation/devicetree/bindings/thermal/thermal-idle.yaml
index 1b77d542a7b8dd..6b4b16580adfb7 100644
--- a/Documentation/devicetree/bindings/thermal/thermal-idle.yaml
+++ b/Documentation/devicetree/bindings/thermal/thermal-idle.yaml
@@ -137,7 +137,7 @@ examples:
                 cooling-maps {
                     map0 {
                         trip = <&cpu_alert1>;
-                        cooling-device = <&cpu_b0_therm 0 15 >,
+                        cooling-device = <&cpu_b0_therm 0 15>,
                                          <&cpu_b1_therm 0 15>;
                     };
 
diff --git a/Documentation/devicetree/bindings/thermal/ti,omap-bandgap.yaml b/Documentation/devicetree/bindings/thermal/ti,omap-bandgap.yaml
new file mode 100644
index 00000000000000..091b7be28461be
--- /dev/null
+++ b/Documentation/devicetree/bindings/thermal/ti,omap-bandgap.yaml
@@ -0,0 +1,177 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/thermal/ti,omap-bandgap.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments OMAP SCM bandgap
+
+maintainers:
+  - Eduardo Valentin <edubezval@gmail.com>
+
+description:
+  In the System Control Module, OMAP supplies a voltage reference and a
+  temperature sensor feature that are gathered in the band gap voltage and
+  temperature sensor (VBGAPTS) module. The band gap provides current and
+  voltage reference for its internal circuits and other analog IP blocks. The
+  analog-to-digital converter (ADC) produces an output value that is
+  proportional to the silicon temperature.
+
+properties:
+  compatible:
+    enum:
+      - ti,omap34xx-bandgap
+      - ti,omap36xx-bandgap
+      - ti,omap4430-bandgap
+      - ti,omap4460-bandgap
+      - ti,omap4470-bandgap
+      - ti,omap5430-bandgap
+      - ti,dra752-bandgap
+
+  reg:
+    minItems: 1
+    maxItems: 6
+
+  interrupts:
+    maxItems: 1
+
+  gpios:
+    maxItems: 1
+    description:
+      used to inform which GPIO line the tshut signal is routed to. The
+      informed GPIO will be treated as an IRQ;
+
+  "#thermal-sensor-cells":
+    enum: [0, 1]
+
+required:
+  - compatible
+  - reg
+
+allOf:
+  - $ref: thermal-sensor.yaml#
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - ti,omap34xx-bandgap
+              - ti,omap36xx-bandgap
+
+    then:
+      properties:
+        reg:
+          minItems: 1
+          maxItems: 1
+
+        interrupts: false
+
+        gpios: false
+
+        "#thermal-sensor-cells":
+          const: 0
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,omap4430-bandgap
+
+    then:
+      properties:
+        reg:
+          minItems: 2
+          maxItems: 2
+
+        interrupts: false
+
+        "#thermal-sensor-cells":
+          const: 0
+
+      required:
+        - gpios
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - ti,omap4460-bandgap
+              - ti,omap4470-bandgap
+
+    then:
+      properties:
+        reg:
+          minItems: 3
+          maxItems: 3
+
+        "#thermal-sensor-cells":
+          const: 0
+
+      required:
+        - interrupts
+        - gpios
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,omap5430-bandgap
+
+    then:
+      properties:
+        reg:
+          minItems: 4
+          maxItems: 4
+
+        gpios: false
+
+        "#thermal-sensor-cells":
+          const: 1
+
+      required:
+        - interrupts
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,dra752-bandgap
+
+    then:
+      properties:
+        reg:
+          minItems: 6
+          maxItems: 6
+
+        gpios: false
+
+        "#thermal-sensor-cells":
+          const: 1
+
+      required:
+        - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    bandgap@48002524 {
+        compatible = "ti,omap34xx-bandgap";
+        reg = <0x48002524 0x4>;
+        #thermal-sensor-cells = <0>;
+    };
+
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    bandgap@4a0021e0 {
+        compatible = "ti,dra752-bandgap";
+        reg = <0x4a0021e0 0xc>, <0x4a00232c 0xc>,
+              <0x4a002380 0x2c>, <0x4a0023c0 0x3c>,
+              <0x4a002564 0x8>, <0x4a002574 0x50>;
+        interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>; /* talert */
+        #thermal-sensor-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/thermal/ti_soc_thermal.txt b/Documentation/devicetree/bindings/thermal/ti_soc_thermal.txt
deleted file mode 100644
index 6299dd8de339ae..00000000000000
--- a/Documentation/devicetree/bindings/thermal/ti_soc_thermal.txt
+++ /dev/null
@@ -1,88 +0,0 @@
-* Texas Instrument OMAP SCM bandgap bindings
-
-In the System Control Module, OMAP supplies a voltage reference
-and a temperature sensor feature that are gathered in the band
-gap voltage and temperature sensor (VBGAPTS) module. The band
-gap provides current and voltage reference for its internal
-circuits and other analog IP blocks. The analog-to-digital
-converter (ADC) produces an output value that is proportional
-to the silicon temperature.
-
-Required properties:
-- compatible : Should be:
-  - "ti,omap34xx-bandgap" : for OMAP34xx bandgap
-  - "ti,omap36xx-bandgap" : for OMAP36xx bandgap
-  - "ti,omap4430-bandgap" : for OMAP4430 bandgap
-  - "ti,omap4460-bandgap" : for OMAP4460 bandgap
-  - "ti,omap4470-bandgap" : for OMAP4470 bandgap
-  - "ti,omap5430-bandgap" : for OMAP5430 bandgap
-- interrupts : this entry should indicate which interrupt line
-the talert signal is routed to;
-Specific:
-- gpios : this entry should be used to inform which GPIO
-line the tshut signal is routed to. The informed GPIO will
-be treated as an IRQ;
-- regs : this entry must also be specified and it is specific
-to each bandgap version, because the mapping may change from
-soc to soc, apart of depending on available features.
-
-Example:
-OMAP34xx:
-bandgap {
-	reg = <0x48002524 0x4>;
-	compatible = "ti,omap34xx-bandgap";
-};
-
-OMAP36xx:
-bandgap {
-	reg = <0x48002524 0x4>;
-	compatible = "ti,omap36xx-bandgap";
-};
-
-OMAP4430:
-bandgap {
-	reg = <0x4a002260 0x4 0x4a00232C 0x4>;
-	compatible = "ti,omap4430-bandgap";
-};
-
-OMAP4460:
-bandgap {
-	reg = <0x4a002260 0x4
-		0x4a00232C 0x4
-		0x4a002378 0x18>;
-	compatible = "ti,omap4460-bandgap";
-	interrupts = <0 126 4>; /* talert */
-	gpios = <&gpio3 22 0>; /* tshut */
-};
-
-OMAP4470:
-bandgap {
-	reg = <0x4a002260 0x4
-		0x4a00232C 0x4
-		0x4a002378 0x18>;
-	compatible = "ti,omap4470-bandgap";
-	interrupts = <0 126 4>; /* talert */
-	gpios = <&gpio3 22 0>; /* tshut */
-};
-
-OMAP5430:
-bandgap {
-	reg = <0x4a0021e0 0xc
-		0x4a00232c 0xc
-		0x4a002380 0x2c
-		0x4a0023C0 0x3c>;
-	compatible = "ti,omap5430-bandgap";
-	interrupts = <0 126 4>; /* talert */
-};
-
-DRA752:
-bandgap {
-	reg = <0x4a0021e0 0xc
-		0x4a00232c 0xc
-		0x4a002380 0x2c
-		0x4a0023C0 0x3c
-		0x4a002564 0x8
-		0x4a002574 0x50>;
-	compatible = "ti,dra752-bandgap";
-	interrupts = <0 126 4>; /* talert */
-};
diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml
index 2de8eb09cb7d14..a8b2b9deade4cf 100644
--- a/Documentation/devicetree/bindings/trivial-devices.yaml
+++ b/Documentation/devicetree/bindings/trivial-devices.yaml
@@ -62,6 +62,8 @@ properties:
             # Aosong temperature & humidity sensors with I2C interface
           - aosong,aht10
           - aosong,aht20
+          - aosong,am2315
+          - aosong,am2320
           - aosong,dht20
             # Arduino microcontroller interface over SPI on UnoQ board
           - arduino,unoq-mcu
@@ -69,6 +71,8 @@ properties:
           - asteralabs,pt5161l
             # ATECC508A - i2c h/w elliptic curve crypto module
           - atmel,atecc508a
+            # ATECC608A - i2c h/w elliptic curve crypto module
+          - atmel,atecc608a
             # ATECC608B - i2c h/w elliptic curve crypto module
           - atmel,atecc608b
             # ATSHA204 - i2c h/w symmetric crypto module
@@ -77,8 +81,6 @@ properties:
           - atmel,atsha204a
             # BPA-RS600: Power Supply
           - blutek,bpa-rs600
-            # CM32181: Ambient Light Sensor
-          - capella,cm32181
             # CM3232: Ambient Light Sensor
           - capella,cm3232
             # CM3323: Ambient Light Sensor
@@ -174,6 +176,7 @@ properties:
             # Infineon Multi-phase Digital VR Controller xdpe1a2g7b
           - infineon,xdpe1a2g5b
           - infineon,xdpe1a2g7b
+          - infineon,xdpe1a2g7c
             # Injoinic IP5108 2.0A Power Bank IC with I2C
           - injoinic,ip5108
             # Injoinic IP5109 2.1A Power Bank IC with I2C
@@ -186,7 +189,7 @@ properties:
           - injoinic,ip5306
             # Inspur Power System power supply unit version 1
           - inspur,ipsps1
-            # Intel common redudant power supply crps185
+            # Intel common redundant power supply crps185
           - intel,crps185
             # Intersil ISL29028 Ambient Light and Proximity Sensor
           - isil,isl29028
@@ -368,16 +371,10 @@ properties:
           - mps,mpq82d00
             # Murata D1U74T-W power supply unit
           - murata,d1u74t
-            # Temperature sensor with integrated fan control
-          - national,lm63
-            # Temperature sensor with integrated fan control
-          - national,lm64
             # Temperature sensor
           - national,lm95235
             # Temperature sensor
           - national,lm95245
-            # Temperature sensor with integrated fan control
-          - national,lm96163
             # Serial Interface ACPI-Compatible Microprocessor System Hardware Monitor
           - national,lm80
             # Serial Interface ACPI-Compatible Microprocessor System Hardware Monitor
@@ -400,8 +397,6 @@ properties:
           - onnn,adt7462
             # 48-Lane, 12-Port PCI Express Gen 2 (5.0 GT/s) Switch
           - plx,pex8648
-            # Pulsedlight LIDAR range-finding sensor
-          - pulsedlight,lidar-lite-v2
             # Renesas HS3001 Temperature and Relative Humidity Sensors
           - renesas,hs3001
             # Renesas ISL29501 time-of-flight sensor
@@ -414,6 +409,8 @@ properties:
           - samsung,24ad0xd1
             # Samsung Exynos SoC SATA PHY I2C device
           - samsung,exynos-sataphy-i2c
+            # SDMC 1024bit EEPROM + 128bit OTP memory for key
+          - sdmc,dm2016
             # Semtech sx1301 baseband processor
           - semtech,sx1301
             # Sensirion multi-pixel gas sensor with I2C interface
@@ -427,6 +424,8 @@ properties:
           - sensirion,sht21
           - sensirion,sht25
           - sensirion,sht4x
+            # Sensirion temperature sensor with I2C interface
+          - sensirion,sts40
             # Sensortek 3 axis accelerometer
           - sensortek,stk8312
             # Sensortek 3 axis accelerometer
@@ -504,16 +503,18 @@ properties:
           - ti,lm74
             # Temperature sensor with integrated fan control
           - ti,lm96000
-            # Low Power Digital Temperature Sensor with SMBUS/Two Wire Serial Interface
-          - ti,tmp103
             # Thermometer with SPI interface
           - ti,tmp121
           - ti,tmp122
           - ti,tmp125
             # TI DC-DC converter on PMBus
           - ti,tps40400
+            # TI Dual channel DCAP+ multiphase controller TPS53622
+          - ti,tps53622
             # TI DCAP+ multiphase controller
           - ti,tps53647
+            # TI Dual channel DCAP+ multiphase controller TPS53659
+          - ti,tps53659
             # TI DCAP+ multiphase controller
           - ti,tps53667
             # TI Dual channel DCAP+ multiphase controller TPS53676 with AVSBus
@@ -526,6 +527,8 @@ properties:
           - ti,tps53685
             # TI Dual channel DCAP+ multiphase controller TPS53688
           - ti,tps53688
+            # TI Dual channel DCAP+ multiphase controller TPS536C7
+          - ti,tps536c7
             # TI DC-DC converters on PMBus
           - ti,tps544b20
           - ti,tps544b25
diff --git a/Documentation/devicetree/bindings/usb/am33xx-usb.txt b/Documentation/devicetree/bindings/usb/am33xx-usb.txt
deleted file mode 100644
index 654ffc62d0131c..00000000000000
--- a/Documentation/devicetree/bindings/usb/am33xx-usb.txt
+++ /dev/null
@@ -1,200 +0,0 @@
-  AM33xx MUSB
-~~~~~~~~~~~~~~~
-- compatible: ti,am33xx-usb
-- reg: offset and length of the usbss register sets
-- ti,hwmods : must be "usb_otg_hs"
-
-The glue layer contains multiple child nodes. It is required to have
-at least a control module node, USB node and a PHY node. The second USB
-node and its PHY node are optional. The DMA node is also optional.
-
-Reset module
-~~~~~~~~~~~~
-- compatible: ti,am335x-usb-ctrl-module
-- reg: offset and length of the "USB control registers" in the "Control
-  Module" block. A second offset and length for the USB wake up control
-  in the same memory block.
-- reg-names: "phy_ctrl" for the "USB control registers" and "wakeup" for
-  the USB wake up control register.
-
-USB PHY
-~~~~~~~
-compatible: ti,am335x-usb-phy
-reg: offset and length of the "USB PHY" register space
-ti,ctrl_mod: reference to the "reset module" node
-reg-names: phy
-The PHY should have a "phy" alias numbered properly in the alias
-node.
-
-USB
-~~~
-- compatible: ti,musb-am33xx
-- reg: offset and length of "USB Controller Registers", and offset and
-  length of "USB Core" register space.
-- reg-names: control for the ""USB Controller Registers" and "mc" for
-  "USB Core" register space
-- interrupts: USB interrupt number
-- interrupt-names: mc
-- dr_mode: Should be one of "host", "peripheral" or "otg".
-- mentor,multipoint: Should be "1" indicating the musb controller supports
-  multipoint. This is a MUSB configuration-specific setting.
-- mentor,num-eps: Specifies the number of endpoints. This is also a
-  MUSB configuration-specific setting. Should be set to "16"
-- mentor,ram-bits: Specifies the ram address size. Should be set to "12"
-- mentor,power: Should be "500". This signifies the controller can supply up to
-  500mA when operating in host mode.
-- phys: reference to the USB phy
-- dmas: specifies the dma channels
-- dma-names: specifies the names of the channels. Use "rxN" for receive
-  and "txN" for transmit endpoints. N specifies the endpoint number.
-
-The controller should have an "usb" alias numbered properly in the alias
-node.
-
-DMA
-~~~
-- compatible: ti,am3359-cppi41
-- reg: offset and length of the following register spaces: USBSS, USB
-  CPPI DMA Controller, USB CPPI DMA Scheduler, USB Queue Manager
-- reg-names: glue, controller, scheduler, queuemgr
-- #dma-cells: should be set to 2. The first number represents the
-  endpoint number (0 … 14 for endpoints 1 … 15 on instance 0 and 15 … 29
-  for endpoints 1 … 15 on instance 1). The second number is 0 for RX and
-  1 for TX transfers.
-- dma-channels: should be set to 30 representing the 15 endpoints for
-  each USB instance.
-- #dma-channels: deprecated
-
-Example:
-~~~~~~~~
-The following example contains all the nodes as used on am335x-evm:
-
-aliases {
-	usb0 = &usb0;
-	usb1 = &usb1;
-	phy0 = &usb0_phy;
-	phy1 = &usb1_phy;
-};
-
-usb: usb@47400000 {
-	compatible = "ti,am33xx-usb";
-	reg = <0x47400000 0x1000>;
-	ranges;
-	#address-cells = <1>;
-	#size-cells = <1>;
-	ti,hwmods = "usb_otg_hs";
-
-	ctrl_mod: control@44e10000 {
-		compatible = "ti,am335x-usb-ctrl-module";
-		reg = <0x44e10620 0x10
-			0x44e10648 0x4>;
-		reg-names = "phy_ctrl", "wakeup";
-	};
-
-	usb0_phy: usb-phy@47401300 {
-		compatible = "ti,am335x-usb-phy";
-		reg = <0x47401300 0x100>;
-		reg-names = "phy";
-		ti,ctrl_mod = <&ctrl_mod>;
-		#phy-cells = <0>;
-	};
-
-	usb0: usb@47401000 {
-		compatible = "ti,musb-am33xx";
-		reg = <0x47401400 0x400
-			0x47401000 0x200>;
-		reg-names = "mc", "control";
-
-		interrupts = <18>;
-		interrupt-names = "mc";
-		dr_mode = "otg"
-		mentor,multipoint = <1>;
-		mentor,num-eps = <16>;
-		mentor,ram-bits = <12>;
-		mentor,power = <500>;
-		phys = <&usb0_phy>;
-
-		dmas = <&cppi41dma  0 0 &cppi41dma  1 0
-			&cppi41dma  2 0 &cppi41dma  3 0
-			&cppi41dma  4 0 &cppi41dma  5 0
-			&cppi41dma  6 0 &cppi41dma  7 0
-			&cppi41dma  8 0 &cppi41dma  9 0
-			&cppi41dma 10 0 &cppi41dma 11 0
-			&cppi41dma 12 0 &cppi41dma 13 0
-			&cppi41dma 14 0 &cppi41dma  0 1
-			&cppi41dma  1 1 &cppi41dma  2 1
-			&cppi41dma  3 1 &cppi41dma  4 1
-			&cppi41dma  5 1 &cppi41dma  6 1
-			&cppi41dma  7 1 &cppi41dma  8 1
-			&cppi41dma  9 1 &cppi41dma 10 1
-			&cppi41dma 11 1 &cppi41dma 12 1
-			&cppi41dma 13 1 &cppi41dma 14 1>;
-		dma-names =
-			"rx1", "rx2", "rx3", "rx4", "rx5", "rx6", "rx7",
-			"rx8", "rx9", "rx10", "rx11", "rx12", "rx13",
-			"rx14", "rx15",
-			"tx1", "tx2", "tx3", "tx4", "tx5", "tx6", "tx7",
-			"tx8", "tx9", "tx10", "tx11", "tx12", "tx13",
-			"tx14", "tx15";
-	};
-
-	usb1_phy: usb-phy@47401b00 {
-		compatible = "ti,am335x-usb-phy";
-		reg = <0x47401b00 0x100>;
-		reg-names = "phy";
-		ti,ctrl_mod = <&ctrl_mod>;
-		#phy-cells = <0>;
-	};
-
-	usb1: usb@47401800 {
-		compatible = "ti,musb-am33xx";
-		reg = <0x47401c00 0x400
-			0x47401800 0x200>;
-		reg-names = "mc", "control";
-		interrupts = <19>;
-		interrupt-names = "mc";
-		dr_mode = "host"
-		mentor,multipoint = <1>;
-		mentor,num-eps = <16>;
-		mentor,ram-bits = <12>;
-		mentor,power = <500>;
-		phys = <&usb1_phy>;
-
-		dmas = <&cppi41dma 15 0 &cppi41dma 16 0
-			&cppi41dma 17 0 &cppi41dma 18 0
-			&cppi41dma 19 0 &cppi41dma 20 0
-			&cppi41dma 21 0 &cppi41dma 22 0
-			&cppi41dma 23 0 &cppi41dma 24 0
-			&cppi41dma 25 0 &cppi41dma 26 0
-			&cppi41dma 27 0 &cppi41dma 28 0
-			&cppi41dma 29 0 &cppi41dma 15 1
-			&cppi41dma 16 1 &cppi41dma 17 1
-			&cppi41dma 18 1 &cppi41dma 19 1
-			&cppi41dma 20 1 &cppi41dma 21 1
-			&cppi41dma 22 1 &cppi41dma 23 1
-			&cppi41dma 24 1 &cppi41dma 25 1
-			&cppi41dma 26 1 &cppi41dma 27 1
-			&cppi41dma 28 1 &cppi41dma 29 1>;
-		dma-names =
-			"rx1", "rx2", "rx3", "rx4", "rx5", "rx6", "rx7",
-			"rx8", "rx9", "rx10", "rx11", "rx12", "rx13",
-			"rx14", "rx15",
-			"tx1", "tx2", "tx3", "tx4", "tx5", "tx6", "tx7",
-			"tx8", "tx9", "tx10", "tx11", "tx12", "tx13",
-			"tx14", "tx15";
-	};
-
-	cppi41dma: dma-controller@7402000 {
-		compatible = "ti,am3359-cppi41";
-		reg =  <0x47400000 0x1000
-			0x47402000 0x1000
-			0x47403000 0x1000
-			0x47404000 0x4000>;
-		reg-names = "glue", "controller", "scheduler", "queuemgr";
-		interrupts = <17>;
-		interrupt-names = "glue";
-		#dma-cells = <2>;
-		dma-channels = <30>;
-		dma-requests = <256>;
-	};
-};
diff --git a/Documentation/devicetree/bindings/usb/da8xx-usb.txt b/Documentation/devicetree/bindings/usb/da8xx-usb.txt
deleted file mode 100644
index fb2027a7d80dd8..00000000000000
--- a/Documentation/devicetree/bindings/usb/da8xx-usb.txt
+++ /dev/null
@@ -1,81 +0,0 @@
-TI DA8xx MUSB
-~~~~~~~~~~~~~
-For DA8xx/OMAP-L1x/AM17xx/AM18xx platforms.
-
-Required properties:
-~~~~~~~~~~~~~~~~~~~~
- - compatible : Should be set to "ti,da830-musb".
-
- - reg: Offset and length of the USB controller register set.
-
- - interrupts: The USB interrupt number.
-
- - interrupt-names: Should be set to "mc".
-
- - dr_mode: The USB operation mode. Should be one of "host", "peripheral" or "otg".
-
- - phys: Phandle for the PHY device
-
- - phy-names: Should be "usb-phy"
-
- - dmas: specifies the dma channels
-
- - dma-names: specifies the names of the channels. Use "rxN" for receive
-   and "txN" for transmit endpoints. N specifies the endpoint number.
-
-Optional properties:
-~~~~~~~~~~~~~~~~~~~~
- - vbus-supply: Phandle to a regulator providing the USB bus power.
-
-DMA
-~~~
-- compatible: ti,da830-cppi41
-- reg: offset and length of the following register spaces: CPPI DMA Controller,
-  CPPI DMA Scheduler, Queue Manager
-- reg-names: "controller", "scheduler", "queuemgr"
-- #dma-cells: should be set to 2. The first number represents the
-  channel number (0 … 3 for endpoints 1 … 4).
-  The second number is 0 for RX and 1 for TX transfers.
-- dma-channels: should be set to 4 representing the 4 endpoints.
-- #dma-channels: deprecated
-
-Example:
-	usb_phy: usb-phy {
-		compatible = "ti,da830-usb-phy";
-		#phy-cells = <0>;
-	};
-	usb0: usb@200000 {
-		compatible = "ti,da830-musb";
-		reg = <0x00200000 0x1000>;
-		ranges;
-		#address-cells = <1>;
-		#size-cells = <1>;
-		interrupts = <58>;
-		interrupt-names = "mc";
-
-		dr_mode = "host";
-		vbus-supply = <&usb_vbus>;
-		phys = <&usb_phy 0>;
-		phy-names = "usb-phy";
-
-		dmas = <&cppi41dma 0 0 &cppi41dma 1 0
-			&cppi41dma 2 0 &cppi41dma 3 0
-			&cppi41dma 0 1 &cppi41dma 1 1
-			&cppi41dma 2 1 &cppi41dma 3 1>;
-		dma-names =
-			"rx1", "rx2", "rx3", "rx4",
-			"tx1", "tx2", "tx3", "tx4";
-
-
-		cppi41dma: dma-controller@201000 {
-			compatible = "ti,da830-cppi41";
-			reg =  <0x201000 0x1000
-				0x202000 0x1000
-				0x204000 0x4000>;
-			reg-names = "controller", "scheduler", "queuemgr";
-			interrupts = <58>;
-			#dma-cells = <2>;
-			dma-channels = <4>;
-		};
-
-	};
diff --git a/Documentation/devicetree/bindings/usb/generic-ehci.yaml b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
index ae9fb70d0212cf..908b02d9175e09 100644
--- a/Documentation/devicetree/bindings/usb/generic-ehci.yaml
+++ b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
@@ -55,6 +55,10 @@ properties:
               - rockchip,rk3588-ehci
               - snps,hsdk-v1.0-ehci
               - socionext,uniphier-ehci
+              - st,stm32mp13-ehci
+              - st,stm32mp15-ehci
+              - st,stm32mp23-ehci
+              - st,stm32mp25-ehci
           - const: generic-ehci
       - items:
           - enum:
@@ -80,6 +84,9 @@ properties:
     minItems: 1
     maxItems: 2
 
+  access-controllers:
+    maxItems: 1
+
   dma-coherent: true
 
   interrupts:
@@ -193,6 +200,23 @@ allOf:
     else:
       properties:
         clock-names: false
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - st,stm32mp23-ehci
+              - st,stm32mp25-ehci
+    then:
+      properties:
+        clocks:
+          items:
+            - description: PHY clock
+            - description: Bus clock
+
+      required:
+        - access-controllers
+        - clocks
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/usb/generic-ohci.yaml b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
index 322808aaa28390..7a8c3c23181534 100644
--- a/Documentation/devicetree/bindings/usb/generic-ohci.yaml
+++ b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
@@ -49,6 +49,10 @@ properties:
               - ingenic,jz4740-ohci
               - rockchip,rk3588-ohci
               - snps,hsdk-v1.0-ohci
+              - st,stm32mp13-ohci
+              - st,stm32mp15-ohci
+              - st,stm32mp23-ohci
+              - st,stm32mp25-ohci
           - const: generic-ohci
       - enum:
           - generic-ohci
@@ -65,6 +69,9 @@ properties:
   reg:
     maxItems: 1
 
+  access-controllers:
+    maxItems: 1
+
   dma-coherent: true
 
   interrupts:
@@ -213,6 +220,23 @@ allOf:
         clocks:
           minItems: 1
           maxItems: 3
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - st,stm32mp23-ohci
+              - st,stm32mp25-ohci
+    then:
+      properties:
+        clocks:
+          items:
+            - description: PHY clock
+            - description: Bus clock
+
+      required:
+        - access-controllers
+        - clocks
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/usb/gpio-sbu-mux.yaml b/Documentation/devicetree/bindings/usb/gpio-sbu-mux.yaml
index 793662f6f3bff4..c0e60848d8abb8 100644
--- a/Documentation/devicetree/bindings/usb/gpio-sbu-mux.yaml
+++ b/Documentation/devicetree/bindings/usb/gpio-sbu-mux.yaml
@@ -22,6 +22,7 @@ properties:
           - nxp,cbdtu02043
           - onnn,fsusb42
           - onnn,fsusb43l10x
+          - onnn,fusb340
           - pericom,pi3usb102
           - ti,tmuxhs4212
       - const: gpio-sbu-mux
diff --git a/Documentation/devicetree/bindings/usb/ohci-da8xx.txt b/Documentation/devicetree/bindings/usb/ohci-da8xx.txt
deleted file mode 100644
index 24a826d5015ec5..00000000000000
--- a/Documentation/devicetree/bindings/usb/ohci-da8xx.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-DA8XX USB OHCI controller
-
-Required properties:
-
- - compatible: Should be "ti,da830-ohci"
- - reg:        Should contain one register range i.e. start and length
- - interrupts: Description of the interrupt line
- - phys:       Phandle for the PHY device
- - phy-names:  Should be "usb-phy"
-
-Optional properties:
- - vbus-supply: phandle of regulator that controls vbus power / over-current
-
-Example:
-
-ohci: usb@225000 {
-        compatible = "ti,da830-ohci";
-        reg = <0x225000 0x1000>;
-        interrupts = <59>;
-        phys = <&usb_phy 1>;
-        phy-names = "usb-phy";
-        vbus-supply = <&reg_usb_ohci>;
-};
diff --git a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
index ea60f7220afe3b..f6bbf28397d27c 100644
--- a/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
+++ b/Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
@@ -27,6 +27,7 @@ properties:
           - qcom,eliza-dwc3
           - qcom,glymur-dwc3
           - qcom,glymur-dwc3-mp
+          - qcom,hawi-dwc3
           - qcom,ipq4019-dwc3
           - qcom,ipq5018-dwc3
           - qcom,ipq5210-dwc3
@@ -40,6 +41,7 @@ properties:
           - qcom,maili-dwc3
           - qcom,milos-dwc3
           - qcom,msm8953-dwc3
+          - qcom,msm8974-dwc3
           - qcom,msm8994-dwc3
           - qcom,msm8996-dwc3
           - qcom,msm8998-dwc3
@@ -206,6 +208,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,hawi-dwc3
               - qcom,ipq5210-dwc3
               - qcom,ipq5424-dwc3
               - qcom,ipq9574-dwc3
@@ -268,6 +271,7 @@ allOf:
           contains:
             enum:
               - qcom,ipq8074-dwc3
+              - qcom,msm8974-dwc3
               - qcom,qdu1000-dwc3
     then:
       properties:
@@ -453,6 +457,23 @@ allOf:
             - const: qusb2_phy
             - const: ss_phy_irq
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,msm8974-dwc3
+    then:
+      properties:
+        interrupts:
+          minItems: 3
+          maxItems: 3
+        interrupt-names:
+          items:
+            - const: dwc_usb3
+            - const: pwr_event
+            - const: hs_phy_irq
+
   - if:
       properties:
         compatible:
@@ -547,6 +568,7 @@ allOf:
           contains:
             enum:
               - qcom,eliza-dwc3
+              - qcom,hawi-dwc3
               - qcom,ipq4019-dwc3
               - qcom,ipq8064-dwc3
               - qcom,kaanapali-dwc3
diff --git a/Documentation/devicetree/bindings/usb/ti,am335x-usb-ctrl-module.yaml b/Documentation/devicetree/bindings/usb/ti,am335x-usb-ctrl-module.yaml
new file mode 100644
index 00000000000000..f8ac0af745c6c3
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ti,am335x-usb-ctrl-module.yaml
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/ti,am335x-usb-ctrl-module.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI AM335x USB Control Module
+
+maintainers:
+  - Sebastian Andrzej Siewior <bigeasy@linutronix.de>
+
+properties:
+  compatible:
+    const: ti,am335x-usb-ctrl-module
+
+  reg:
+    maxItems: 2
+
+  reg-names:
+    items:
+      - const: phy_ctrl
+      - const: wakeup
+
+required:
+  - compatible
+  - reg
+  - reg-names
+
+additionalProperties: false
+
+examples:
+  - |
+    control@44e10000 {
+        compatible = "ti,am335x-usb-ctrl-module";
+        reg = <0x44e10620 0x10>,
+              <0x44e10648 0x4>;
+        reg-names = "phy_ctrl", "wakeup";
+    };
diff --git a/Documentation/devicetree/bindings/usb/ti,am33xx-usb.yaml b/Documentation/devicetree/bindings/usb/ti,am33xx-usb.yaml
new file mode 100644
index 00000000000000..0b36b9ddfb4a94
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ti,am33xx-usb.yaml
@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/ti,am33xx-usb.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI AM33xx USB Wrapper
+
+description:
+  The glue layer contains multiple child nodes. It is required to have at least
+  a USB node and a PHY node. The second USB node and its PHY node are optional.
+  The DMA node is also optional.
+
+maintainers:
+  - Sebastian Andrzej Siewior <bigeasy@linutronix.de>
+
+properties:
+  compatible:
+    const: ti,am33xx-usb
+
+  reg:
+    maxItems: 1
+
+  ranges: true
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  ti,hwmods:
+    $ref: /schemas/types.yaml#/definitions/string
+    description: Name of the hwmod associated with the USB module.
+    const: usb_otg_hs
+    deprecated: true
+
+patternProperties:
+  "^(usb-)?phy@[0-9a-f]+$":
+    $ref: /schemas/phy/ti,am335x-usb-phy.yaml#
+
+  "^usb@[0-9a-f]+$":
+    $ref: /schemas/usb/ti,musb-am33xx.yaml#
+
+  "^dma-controller@[0-9a-f]+$":
+    $ref: /schemas/dma/ti/ti,cppi41.yaml#
+
+required:
+  - compatible
+  - reg
+  - ranges
+  - "#address-cells"
+  - "#size-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    usb@47400000 {
+        compatible = "ti,am33xx-usb";
+        reg = <0x47400000 0x1000>;
+        ranges;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ti,hwmods = "usb_otg_hs";
+
+        usb-phy@47401300 {
+            compatible = "ti,am335x-usb-phy";
+            reg = <0x47401300 0x100>;
+            reg-names = "phy";
+            ti,ctrl_mod = <&ctrl_mod>;
+            #phy-cells = <0>;
+        };
+
+        usb@47401000 {
+            compatible = "ti,musb-am33xx";
+            reg = <0x47401400 0x400>,
+                  <0x47401000 0x200>;
+            reg-names = "mc", "control";
+            interrupts = <18>;
+            interrupt-names = "mc";
+            dr_mode = "otg";
+            mentor,multipoint = <1>;
+            mentor,num-eps = <16>;
+            mentor,ram-bits = <12>;
+            mentor,power = <500>;
+            phys = <&usb0_phy>;
+        };
+
+        dma-controller@7402000 {
+            compatible = "ti,am3359-cppi41";
+            reg = <0x47400000 0x1000>,
+                  <0x47402000 0x1000>,
+                  <0x47403000 0x1000>,
+                  <0x47404000 0x4000>;
+            reg-names = "glue", "controller", "scheduler", "queuemgr";
+            interrupts = <17>;
+            interrupt-names = "glue";
+            #dma-cells = <2>;
+            dma-channels = <30>;
+            dma-requests = <256>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/usb/ti,da830-musb.yaml b/Documentation/devicetree/bindings/usb/ti,da830-musb.yaml
new file mode 100644
index 00000000000000..fefae735b34d70
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ti,da830-musb.yaml
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/ti,da830-musb.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI DA8xx MUSB
+
+maintainers:
+  - Bin Liu <b-liu@ti.com>
+
+allOf:
+  - $ref: /schemas/usb/usb-drd.yaml#
+
+properties:
+  compatible:
+    const: ti,da830-musb
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  interrupt-names:
+    const: mc
+
+  phys:
+    maxItems: 1
+
+  phy-names:
+    const: usb-phy
+
+  dmas:
+    minItems: 1
+    maxItems: 8
+
+  dma-names:
+    minItems: 1
+    maxItems: 8
+    items:
+      pattern: "^(rx|tx)[1-4]$"
+
+  vbus-supply:
+    description: Phandle to a regulator providing the USB bus power.
+
+  clocks:
+    maxItems: 1
+
+  clock-ranges: true
+
+  ranges: true
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+patternProperties:
+  "^dma-controller@[0-9a-f]+$":
+    $ref: /schemas/dma/ti/ti,cppi41.yaml#
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - interrupt-names
+  - phys
+  - phy-names
+  - ranges
+  - "#address-cells"
+  - "#size-cells"
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    usb@200000 {
+        compatible = "ti,da830-musb";
+        reg = <0x00200000 0x1000>;
+        ranges;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        interrupts = <58>;
+        interrupt-names = "mc";
+
+        dr_mode = "host";
+        vbus-supply = <&usb_vbus>;
+        phys = <&usb_phy 0>;
+        phy-names = "usb-phy";
+
+        clocks = <&psc1 1>;
+        clock-ranges;
+
+        dmas = <&cppi41dma 0 0>, <&cppi41dma 1 0>,
+               <&cppi41dma 2 0>, <&cppi41dma 3 0>,
+               <&cppi41dma 0 1>, <&cppi41dma 1 1>,
+               <&cppi41dma 2 1>, <&cppi41dma 3 1>;
+        dma-names = "rx1", "rx2", "rx3", "rx4",
+                    "tx1", "tx2", "tx3", "tx4";
+
+        dma-controller@201000 {
+            compatible = "ti,da830-cppi41";
+            reg = <0x201000 0x1000>,
+                  <0x202000 0x1000>,
+                  <0x204000 0x4000>;
+            reg-names = "controller", "scheduler", "queuemgr";
+            interrupts = <58>;
+            #dma-cells = <2>;
+            dma-channels = <4>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/usb/ti,da830-ohci.yaml b/Documentation/devicetree/bindings/usb/ti,da830-ohci.yaml
new file mode 100644
index 00000000000000..148b9d0d83da58
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ti,da830-ohci.yaml
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/ti,da830-ohci.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: DA8XX USB OHCI controller
+
+maintainers:
+  - Axel Haslam <ahaslam@baylibre.com>
+
+allOf:
+  - $ref: usb-hcd.yaml#
+
+properties:
+  compatible:
+    const: ti,da830-ohci
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  phys:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  phy-names:
+    const: usb-phy
+
+  vbus-supply:
+    description:
+      phandle of regulator that controls vbus power / over-current
+
+  oc-gpios:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - phys
+  - phy-names
+  - clocks
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    usb@225000 {
+        compatible = "ti,da830-ohci";
+        reg = <0x225000 0x1000>;
+        interrupts = <59>;
+        phys = <&usb_phy 1>;
+        phy-names = "usb-phy";
+        clocks = <&psc1 2>;
+        vbus-supply = <&reg_usb_ohci>;
+    };
diff --git a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml
index 06099e93c6c30d..654a982666ddb5 100644
--- a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml
+++ b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml
@@ -25,6 +25,15 @@ properties:
   interrupts:
     maxItems: 1
 
+  vcc33-supply:
+    description: 3.3V supply (VCC33 pin), powering the SuperSpeed 2:1 MUX.
+
+  vdd5-supply:
+    description:
+      5V supply (VDD5 pin), powering the CC controller and sourcing VCONN.
+      VDD5 has to be stable for at least tVDD5V_PG (2ms) before VCC33 starts
+      ramping up, unless ENn_CC is held high while both rails ramp up.
+
   id-gpios:
     description:
       An input gpio for USB ID pin. Upon detecting a UFP device, HD3SS3220
@@ -68,6 +77,8 @@ examples:
             reg = <0x47>;
             interrupt-parent = <&gpio6>;
             interrupts = <3>;
+            vcc33-supply = <&vcc3v3_control>;
+            vdd5-supply = <&vcc5v0_device_s0>;
 
             ports {
                 #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/usb/ti,musb-am33xx.yaml b/Documentation/devicetree/bindings/usb/ti,musb-am33xx.yaml
new file mode 100644
index 00000000000000..cc01edc75d11e1
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/ti,musb-am33xx.yaml
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/ti,musb-am33xx.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI AM33xx/DM816 MUSB
+
+maintainers:
+  - Ravi B <ravibabu@ti.com>
+  - Ajay Kumar Gupta <ajay.gupta@ti.com>
+
+allOf:
+  - $ref: /schemas/usb/usb-drd.yaml#
+
+properties:
+  compatible:
+    enum:
+      - ti,musb-am33xx
+      - ti,musb-dm816
+
+  reg:
+    maxItems: 2
+
+  reg-names:
+    items:
+      - const: mc
+      - const: control
+
+  interrupts:
+    minItems: 1
+    maxItems: 2
+
+  interrupt-names:
+    minItems: 1
+    items:
+      - const: mc
+      - const: vbus
+
+  mentor,multipoint:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Indicates the musb controller supports multipoint.
+    enum: [0, 1]
+
+  mentor,num-eps:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Specifies the number of endpoints.
+    const: 16
+
+  mentor,ram-bits:
+    description: Specifies the ram address size.
+    const: 12
+
+  mentor,power:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Controller supply capacity (in mA) when operating in host mode.
+    const: 500
+
+  phys:
+    maxItems: 1
+
+  dmas:
+    minItems: 1
+    maxItems: 30
+
+  dma-names:
+    minItems: 1
+    maxItems: 30
+    items:
+      pattern: "^(rx|tx)([1-9]|1[0-5])$"
+
+  phy-names:
+    const: usb2-phy
+
+  interface-type:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Describes the type of interface between the controller and the PHY.
+    enum: [ 0, 1 ]
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - interrupts
+  - interrupt-names
+  - dr_mode
+  - mentor,multipoint
+  - mentor,num-eps
+  - mentor,ram-bits
+  - mentor,power
+  - phys
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    usb@47401000 {
+        compatible = "ti,musb-am33xx";
+        reg = <0x47401400 0x400>,
+              <0x47401000 0x200>;
+        reg-names = "mc", "control";
+        interrupts = <18>;
+        interrupt-names = "mc";
+        dr_mode = "otg";
+        mentor,multipoint = <1>;
+        mentor,num-eps = <16>;
+        mentor,ram-bits = <12>;
+        mentor,power = <500>;
+        phys = <&usb0_phy>;
+        dmas = <&cppi41dma 0 0 &cppi41dma 1 0>;
+        dma-names = "rx1", "tx1";
+    };
+
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index ba200296937343..5ddeae3a697e21 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -327,6 +327,8 @@ patternProperties:
     description: ChipOne
   "^chipspark,.*":
     description: ChipSPARK
+  "^chipwealth,.*":
+    description: Chip Wealth Technology
   "^chongzhou,.*":
     description: Shenzhen Chongzhou Electronic Technology Co., Ltd
   "^chrontel,.*":
@@ -559,6 +561,8 @@ patternProperties:
     description: Seiko Epson Corp.
   "^esp,.*":
     description: Espressif Systems Co. Ltd.
+  "^esstech,.*":
+    description: ESS Technology, Inc.
   "^est,.*":
     description: ESTeem Wireless Modems
   "^eswin,.*":
@@ -626,6 +630,8 @@ patternProperties:
     description: Freebox SAS
   "^freecom,.*":
     description: Freecom Gmbh
+  "^freemobile,.*":
+    description: Free Mobile
   "^frida,.*":
     description: Shenzhen Frida LCD Co., Ltd.
   "^friendlyarm,.*":
@@ -662,6 +668,8 @@ patternProperties:
     description: Gemei Digital Technology Co., Ltd.
   "^gemtek,.*":
     description: Gemtek Technology Co., Ltd.
+  "^generalmobile,.*":
+    description: General Mobile Fzc.
   "^genesys,.*":
     description: Genesys Logic, Inc.
   "^genexis,.*":
@@ -949,6 +957,8 @@ patternProperties:
     description: Shenzen LC Technology Co., Ltd.
   "^leadtek,.*":
     description: Shenzhen Leadtek Technology Co., Ltd.
+  "^leeco,.*":
+    description: Leshi Holding Beijing.
   "^leez,.*":
     description: Leez
   "^lego,.*":
@@ -1192,6 +1202,8 @@ patternProperties:
     description: New Vision Display (Shenzhen) Co., Ltd.
   "^nexbox,.*":
     description: Nexbox
+  "^nexperia,.*":
+    description: Nexperia B.V.
   "^nextthing,.*":
     description: Next Thing Co.
   "^ni,.*":
@@ -1419,6 +1431,8 @@ patternProperties:
     description: Renesas Electronics Corporation
   "^rervision,.*":
     description: Shenzhen Rervision Technology Co., Ltd.
+  "^retroidpocket,.*":
+    description: Retroid Pocket
   "^retronix,.*":
     description: Retronix Technology Inc.
   "^revotics,.*":
@@ -1495,6 +1509,8 @@ patternProperties:
     description: Sensirion AG
   "^sensortek,.*":
     description: Sensortek Technology Corporation
+  "^sensylink,.*":
+    description: Sensylink Microelectronics Technology Co., Ltd.
   "^sercomm,.*":
     description: Sercomm (Suzhou) Corporation
   "^sff,.*":
@@ -1694,6 +1710,8 @@ patternProperties:
     description: Techvision Intelligent Technology Co., Ltd
   "^techwell,.*":
     description: Techwell, Inc.
+  "^teclast,.*":
+    description: Teclast
   "^teejet,.*":
     description: TeeJet
   "^teltonika,.*":
diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
index ca4cf01db59f7e..1af924157f7ca6 100644
--- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
@@ -22,6 +22,7 @@ properties:
               - apple,t6031-wdt
               - apple,t8122-wdt
               - apple,t8132-wdt
+              - apple,t8140-wdt
           - const: apple,t8103-wdt
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml b/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml
index 953629cb9558b1..f9b725592c27c5 100644
--- a/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml
@@ -26,6 +26,7 @@ properties:
           - mediatek,mt6795-wdt
           - mediatek,mt7986-wdt
           - mediatek,mt7988-wdt
+          - mediatek,mt8167-wdt
           - mediatek,mt8183-wdt
           - mediatek,mt8186-wdt
           - mediatek,mt8188-wdt
@@ -40,6 +41,7 @@ properties:
               - mediatek,mt7622-wdt
               - mediatek,mt7623-wdt
               - mediatek,mt7629-wdt
+              - mediatek,mt8127-wdt
               - mediatek,mt8173-wdt
               - mediatek,mt8188-wdt
               - mediatek,mt8189-wdt
diff --git a/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml
index 975c5aa4d747f5..322b6921c147f2 100644
--- a/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml
@@ -48,6 +48,17 @@ properties:
   resets:
     maxItems: 1
 
+  renesas,sysc:
+    description:
+      System controller registers control the start/stop of the WDT, and halt debug.
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      - items:
+          - description: phandle to system controller
+          - description: watchdog IP instance index
+            minimum: 0
+            maximum: 5
+
   timeout-sec: true
 
 required:
@@ -73,15 +84,29 @@ allOf:
           minItems: 2
         clock-names:
           minItems: 2
+        renesas,sysc: false
     else:
       properties:
         clocks:
           maxItems: 1
         clock-names:
           maxItems: 1
-        reg:
-          minItems: 2
         resets: false
+      allOf:
+        - if:
+            required:
+              - renesas,sysc
+          then:
+            properties:
+              reg:
+                maxItems: 1
+          else:
+            properties:
+              reg:
+                description: Deprecated. Use the renesas,sysc property along with
+                             the watchdog IP instance index instead.
+                minItems: 2
+                deprecated: true
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml b/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml
index 41aee1655b0c22..a32c478315779b 100644
--- a/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml
@@ -22,6 +22,7 @@ properties:
           - samsung,s3c6410-wdt                   # for S3C6410, S5PV210 and Exynos4
           - samsung,exynos5250-wdt                # for Exynos5250
           - samsung,exynos5420-wdt                # for Exynos5420
+          - samsung,exynos5515-wdt
           - samsung,exynos7-wdt                   # for Exynos7
           - samsung,exynos850-wdt                 # for Exynos850
           - samsung,exynos990-wdt                 # for Exynos990
@@ -57,7 +58,7 @@ properties:
     $ref: /schemas/types.yaml#/definitions/phandle
     description:
       Phandle to the PMU system controller node (in case of Exynos5250,
-      Exynos5420, Exynos7, Exynos850, Exynos990 and gs101).
+      Exynos5420, Exynos5515, Exynos7, Exynos850, Exynos990 and gs101).
 
 required:
   - compatible
@@ -93,6 +94,26 @@ allOf:
         - samsung,cluster-index
         - samsung,syscon-phandle
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - samsung,exynos5515-wdt
+    then:
+      properties:
+        clocks:
+          items:
+            - description: Bus clock, used for register interface
+            - description: Source clock (driving watchdog counter)
+        clock-names:
+          items:
+            - const: watchdog
+            - const: watchdog_src
+        samsung,cluster-index: false
+      required:
+        - samsung,syscon-phandle
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/of_unittest.rst b/Documentation/devicetree/of_unittest.rst
index 6ed6e329196445..5ea8ee0e81effb 100644
--- a/Documentation/devicetree/of_unittest.rst
+++ b/Documentation/devicetree/of_unittest.rst
@@ -125,7 +125,7 @@ Figure 1: Generic structure of un-flattened device tree
 
 
 Before executing OF unittest, it is required to attach the test data to
-machine's device tree (if present). So, when selftest_data_add() is called,
+machine's device tree (if present). So, when unittest_data_add() is called,
 at first it reads the flattened device tree data linked into the kernel image
 via the following kernel symbols::
 
diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst
index 1c148fe8e1f9c4..48f62e59c455f1 100644
--- a/Documentation/doc-guide/kernel-doc.rst
+++ b/Documentation/doc-guide/kernel-doc.rst
@@ -422,7 +422,8 @@ Domain`_ references.
   Function reference.
 
 ``@parameter``
-  Name of a function parameter. (No cross-referencing, just formatting.)
+  Name of a function parameter, struct member, union member, or
+  enum value. (No cross-referencing, just formatting.)
 
 ``%CONST``
   Name of a constant. (No cross-referencing, just formatting.)
@@ -563,8 +564,8 @@ identifiers: *[ function/type ...]*
   Include documentation for each *function* and *type* in *source*.
   If no *function* is specified, the documentation for all functions
   and types in the *source* will be included.
-  *type* can be a ``struct``, ``union``, ``enum``, ``typedef`` or ``var``
-  identifier.
+  *type* can be a ``struct``, ``union``, ``enum``, ``typedef``, ``var``,
+  or ``define`` identifier.
 
   Examples::
 
diff --git a/Documentation/doc-guide/parse-headers.rst b/Documentation/doc-guide/parse-headers.rst
index a7bb01ff04eb0b..eb36c92e9a062c 100644
--- a/Documentation/doc-guide/parse-headers.rst
+++ b/Documentation/doc-guide/parse-headers.rst
@@ -8,26 +8,27 @@ between the code and the documentation. Adding cross-references for
 userspace API files has an additional advantage: Sphinx will generate warnings
 if a symbol is not found at the documentation. That helps to keep the
 uAPI documentation in sync with the Kernel changes.
-The :ref:`parse_headers.py <parse_headers>` provides a way to generate such
-cross-references. It has to be called via Makefile, while building the
-documentation. Please see ``Documentation/userspace-api/media/Makefile`` for an example
-about how to use it inside the Kernel tree.
+The :ref:`parse-headers.py <parse_headers>` provides a way to generate such
+cross-references. During the documentation build, the same parser is used by
+the ``kernel-include`` directive with the ``:generate-cross-refs:`` option.
+Please see ``Documentation/userspace-api/media/cec/cec-header.rst`` for an
+example about how to use it inside the Kernel tree.
 
 .. _parse_headers:
 
-tools/docs/parse_headers.py
+tools/docs/parse-headers.py
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 NAME
 ****
 
-parse_headers.py - parse a C file, in order to identify functions, structs,
+parse-headers.py - parse a C file, in order to identify functions, structs,
 enums and defines and create cross-references to a Sphinx book.
 
 USAGE
 *****
 
-parse-headers.py [-h] [-d] [-t] ``FILE_IN`` ``FILE_OUT`` ``FILE_RULES``
+parse-headers.py [-h] [-d] [-t] ``FILE_IN`` ``FILE_OUT`` [``FILE_RULES``]
 
 SYNOPSIS
 ********
diff --git a/Documentation/driver-api/cxl/platform/acpi/dsdt.rst b/Documentation/driver-api/cxl/platform/acpi/dsdt.rst
index b4583b01d67d36..73b23d82aff0a4 100644
--- a/Documentation/driver-api/cxl/platform/acpi/dsdt.rst
+++ b/Documentation/driver-api/cxl/platform/acpi/dsdt.rst
@@ -25,4 +25,5 @@ Example Compute Express Link Host Bridge ::
             ...
             Name (_UID, 0x05)  // _UID: Unique ID
             ...
-      }
+        }
+    }
diff --git a/Documentation/driver-api/dmaengine/provider.rst b/Documentation/driver-api/dmaengine/provider.rst
index f4ed98f701c918..22d235fb478f15 100644
--- a/Documentation/driver-api/dmaengine/provider.rst
+++ b/Documentation/driver-api/dmaengine/provider.rst
@@ -321,7 +321,7 @@ to use.
 - DESC_METADATA_CLIENT
 
   The metadata buffer is allocated/provided by the client driver and it is
-  attached (via the dmaengine_desc_attach_metadata() helper to the descriptor.
+  attached (via the dmaengine_desc_attach_metadata() helper) to the descriptor.
 
   From the DMA driver the following is expected for this mode:
 
diff --git a/Documentation/driver-api/early-userspace/early_userspace_support.rst b/Documentation/driver-api/early-userspace/early_userspace_support.rst
index 60d1e1bc9413bc..1070f82e35788f 100644
--- a/Documentation/driver-api/early-userspace/early_userspace_support.rst
+++ b/Documentation/driver-api/early-userspace/early_userspace_support.rst
@@ -55,9 +55,9 @@ user root (0).  INITRAMFS_ROOT_GID can be set to a group ID that needs
 to be mapped to group root (0).
 
 A source file must be directives in the format required by the
-usr/gen_init_cpio utility (run 'usr/gen_init_cpio -h' to get the
+scripts/gen_init_cpio utility (run 'scripts/gen_init_cpio -h' to get the
 file format).  The directives in the file will be passed directly to
-usr/gen_init_cpio.
+scripts/gen_init_cpio.
 
 When a combination of directories and files are specified then the
 initramfs image will be an aggregate of all of them.  In this way a user
@@ -69,17 +69,17 @@ early userspace image can be built by an unprivileged user.
 
 As a technical note, when directories and files are specified, the
 entire CONFIG_INITRAMFS_SOURCE is passed to
-usr/gen_initramfs.sh.  This means that CONFIG_INITRAMFS_SOURCE
+scripts/gen_initramfs.sh.  This means that CONFIG_INITRAMFS_SOURCE
 can really be interpreted as any legal argument to
 gen_initramfs.sh.  If a directory is specified as an argument then
 the contents are scanned, uid/gid translation is performed, and
-usr/gen_init_cpio file directives are output.  If a file is
-specified as an argument to usr/gen_initramfs.sh then the
+scripts/gen_init_cpio file directives are output.  If a file is
+specified as an argument to scripts/gen_initramfs.sh then the
 contents of the file are simply copied to the output.  All of the output
 directives from directory scanning and file contents copying are
-processed by usr/gen_init_cpio.
+processed by scripts/gen_init_cpio.
 
-See also 'usr/gen_initramfs.sh -h'.
+See also 'scripts/gen_initramfs.sh -h'.
 
 Where's this all leading?
 =========================
@@ -142,7 +142,7 @@ b) some device and filesystem drivers built as modules and stored in an
 
 c) using initramfs.  The call to prepare_namespace() must be skipped.
    This means that a binary must do all the work.  Said binary can be stored
-   into initramfs either via modifying usr/gen_init_cpio.c or via the new
+   into initramfs either via modifying scripts/gen_init_cpio.c or via the new
    initrd format, an cpio archive.  It must be called "/init".  This binary
    is responsible to do all the things prepare_namespace() would do.
 
diff --git a/Documentation/driver-api/firmware/other_interfaces.rst b/Documentation/driver-api/firmware/other_interfaces.rst
index 06ac89adaafba2..984ee3ecc8dc27 100644
--- a/Documentation/driver-api/firmware/other_interfaces.rst
+++ b/Documentation/driver-api/firmware/other_interfaces.rst
@@ -49,3 +49,241 @@ of the requests on to a secure monitor (EL3).
 
 .. kernel-doc:: drivers/firmware/stratix10-svc.c
    :export:
+
+NXP Secure Enclave Firmware Interface
+--------------------------------------
+
+Introduction
+~~~~~~~~~~~~
+NXP i.MX hardware IPs such as EdgeLock Enclave (ELE) and V2X create an
+embedded secure enclave within the SoC boundary to enable features like:
+
+- Hardware Security Module (HSM)
+- Security Hardware Extension (SHE)
+- Vehicular to Anything (V2X)
+
+Each of the above features is enabled through a dedicated NXP hardware IP
+on the SoC. A single SoC may contain more than one such hardware IP, that
+is, more than one secure enclave can coexist.
+
+NXP SoCs with such secure enclave (SE) IPs are:
+i.MX93, i.MX8ULP
+
+To communicate with one or more coexisting SEs on the SoC, there are
+dedicated messaging units (MU) per SE. Each coexisting SE can have one or
+more exclusive MUs dedicated to itself. No MU is shared between two SEs.
+MU communication is realized using the mailbox driver. Each secure enclave
+can serve multiple clients by virtue of these exclusive MUs, and can
+distinguish transactions from different clients based on the MU used and
+the core security state. The communication between clients and secure
+enclaves uses a command/response mechanism. Each client can expose a
+specific set of secure enclave features to higher layers, based on the
+commands it supports. For example, a secure enclave can simultaneously
+serve an OP-TEE TA and a Linux middleware client. Each client exposes a
+specific set of secure enclave features based on its supported command set.
+
+NXP Secure Enclave (SE) Interface
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+MUs are not shared between SEs. For an SoC like i.MX95, which has multiple
+SEs such as HSM, V2X-HSM, and V2X-SHE, all SEs and their ``se-if``
+interfaces dedicated to a particular SE are enumerated and provisioned
+using the single compatible node ``fsl,imx95-se``.
+
+Each ``se-if`` comprises two layers:
+
+- (C_DEV Layer) User-space software-access interface.
+- (Service Layer) OS-level software-access interface.
+
+::
+
+   +--------------------------------------------+
+   |            Character Device(C_DEV)         |
+   |                                            |
+   |   +---------+ +---------+     +---------+  |
+   |   | misc #1 | | misc #2 | ... | misc #n |  |
+   |   |  dev    | |  dev    |     | dev     |  |
+   |   +---------+ +---------+     +---------+  |
+   |        +-------------------------+         |
+   |        |   Misc. Dev Sync Logic  |         |
+   |        +-------------------------+         |
+   |                                            |
+   +--------------------------------------------+
+
+::
+
+   +--------------------------------------------+
+   |               Service Layer                |
+   |                                            |
+   |      +-----------------------------+       |
+   |      | Message Serialization Logic |       |
+   |      +-----------------------------+       |
+   |          +---------------+                 |
+   |          |  imx-mailbox  |                 |
+   |          |   mailbox.c   |                 |
+   |          +---------------+                 |
+   |                                            |
+   +--------------------------------------------+
+
+- service layer:
+  This layer ensures the communication protocol defined for interaction
+  with firmware.
+
+  The firmware communication protocol provides two guarantees:
+
+  - Serializing the messages to be sent over an MU.
+  - Firmware can handle one command message at a time.
+
+- c_dev:
+  This layer offers character device contexts, created as
+  ``/dev/<se>_mux_chx``. Using multiple device contexts multiplexed over
+  a single MU, userspace applications can use file operations (fops) such
+  as ``write`` and ``read`` to send a command message and read back the
+  response to/from firmware. These fops use the service layer API to
+  communicate with firmware.
+
+  Misc-device (``/dev/<se>_mux_chn``) synchronization protocol::
+
+                                Non-Secure               +   Secure
+                                                         |
+                                                         |
+                +-----------+      +-------------+       |
+                | se_ctrl.c +<---->+imx-mailbox.c|       |
+                |           |      |  mailbox.c  +<-->+------+    +------+
+                +-----+-----+      +-------------+    | MU X +<-->+ ELE |
+                      |                               +------+    +------+
+                      +----------------+                 |
+                      |                |                 |
+                      v                v                 |
+                  logical           logical              |
+                  receiver          waiter               |
+                     +                 +                 |
+                     |                 |                 |
+                     |                 |                 |
+                     |            +----+------+          |
+                     |            |           |          |
+                     |            |           |          |
+              device_ctx     device_ctx     device_ctx   |
+                                                         |
+                User 0        User 1       User Y        |
+                +------+      +------+     +------+      |
+                |misc.c|      |misc.c|     |misc.c|      |
+   kernel space +------+      +------+     +------+      |
+                                                         |
+   +---------------------------------------------------- |
+                    |             |           |          |
+   userspace   /dev/ele_muXch0    |           |          |
+                          /dev/ele_muXch1     |          |
+                                        /dev/ele_muXchY  |
+                                                         |
+
+When a user sends a command to the firmware, it registers its
+``device_ctx`` as a waiter of a response from firmware.
+
+The secure enclave firmware manages storage over a Linux filesystem.
+For this, ``c_dev`` provisions a dedicated device context called the
+command-receiver.
+
+
+ELE_STORAGE_OPEN_REQ concurrency and command-receiver exclusivity
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A userspace-created ``dev_ctx`` becomes the storage subordinate to FW by
+being registered as the command-receiver via
+``set_dev_ctx_as_command_receiver()``. FW sends NVM callback commands to
+this ``dev_ctx`` via that priv's MU, on which the ``dev_ctx`` was created.
+Once a userspace ``dev_ctx`` on one priv is registered as the
+command-receiver, after opening the storage handle with FW, FW enforces a
+global one-storage-instance limit: a concurrent ELE_STORAGE_OPEN_REQ
+arriving over this or any other priv's MU is rejected by FW itself, before
+any storage handle is allocated. The driver uses three layers of
+protection; the sequence below shows how a concurrent race is handled
+safely::
+
+  Userspace A        Kernel (se_ctrl)          FW (ELE)        Userspace B
+      |                    |                      |                  |
+      |--ioctl(SEND_RCV)-->|                      |                  |
+      |            [advisory check under          |                  |
+      |             modify_lock: no receiver]     |                  |
+      |            [acquire se_if_cmd_lock]       |--ioctl(SEND_RCV)->
+      |                    |                      |  [advisory check  |
+      |                    |                      |   passes: TOCTOU] |
+      |                    |                      |  [blocks on       |
+      |                    |                      |   se_if_cmd_lock] |
+      |            ele_msg_send_rcv()             |                  |
+      |                    |--STORAGE_OPEN_REQ--->|                  |
+      |                    |<--STORAGE_OPEN_RSP---|                  |
+      |            fw_api_specific_ops():         |                  |
+      |            set_dev_ctx_as_command_receiver|                  |
+      |            [re-check under modify_lock]   |                  |
+      |            [release se_if_cmd_lock]       |                  |
+      |<--ioctl 0----------|                      |                  |
+      |                    |                      |  [B gets lock]   |
+      |                    |                      |--STORAGE_OPEN_REQ->
+      |                    |                      |  [FW rejects:    |
+      |                    |                      |   one storage    |
+      |                    |                      |   at a time]     |
+      |                    |                      |<--ERROR_RSP------|
+      |                    |  se_val_rsp_hdr_n_status: -EPERM        |
+      |                    |  fw_api_specific_ops not called         |
+      |                    |<--ioctl -EPERM-to B------------------->|
+
+The three protection layers are:
+
+1. Advisory early check (under modify_lock, before send): fast-path
+   rejection if a receiver is already registered. Not the final gate
+   because modify_lock is released before the MU send (TOCTOU window).
+
+2. ``se_if_cmd_lock``: held for the entire send+receive cycle, so only
+   one ioctl command is in flight on a given MU at a time.
+
+3. ``set_dev_ctx_as_command_receiver()`` re-checks under modify_lock
+   after the response arrives. If two callers race past layer 1, FW
+   itself rejects the second ELE_STORAGE_OPEN_REQ before any handle is
+   allocated.
+
+Signal handling after a completed hardware operation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A signal may arrive after the firmware has already executed the command
+and delivered its response into the MU receive buffer. The driver detects
+this case and preserves state before returning ``-EINTR``::
+
+  Userspace            Kernel (se_ctrl)          FW (ELE)
+      |                    |                      |
+      |--ioctl(SEND_RCV)-->|                      |
+      |            ele_msg_send_rcv()             |
+      |                    |--CMD_REQ------------>|
+      |  [signal arrives]  |                      | (FW executes,
+      |                    |                      |  allocates handle,
+      |                    |                      |  writes response)
+      |                    |<--CMD_RSP------------|
+      |  wait_for_completion_interruptible()      |
+      |  wakes: signal seen -> -ERESTARTSYS       |
+      |                    |                      |
+      |            [response is in rx_msg:        |
+      |             validate with                 |
+      |             se_val_rsp_hdr_n_status()]    |
+      |            [fw_api_specific_ops()         |
+      |             (is_cmd_interrupted=true):    |
+      |             for SESSION_OPEN: record      |
+      |             handle, close session via     |
+      |             se_close_session(), clear;    |
+      |             for STORAGE_OPEN: record      |
+      |             handle, close storage via     |
+      |             se_close_storage(), return 0] |
+      |            err = -EINTR                   |
+      |            (not -ERESTARTSYS:             |
+      |             prevents VFS auto-restart)    |
+      |<--ioctl -EINTR-----|                      |
+      |                    |                      |
+      | [signal handler runs; userspace decides   |
+      |  whether to re-issue; FW handle tracked   |
+      |  or cleaned up; no firmware resource leak]|
+
+Returning ``-EINTR`` instead of ``-ERESTARTSYS`` is intentional: the VFS
+would transparently restart an ioctl on ``-ERESTARTSYS``, re-running the
+command with already-zeroed shared input buffers. ``-EINTR`` lets userspace
+enter its signal handler and decide whether to reissue the command.
+
+.. kernel-doc:: drivers/firmware/imx/se_ctrl.c
+   :export:
diff --git a/Documentation/driver-api/gpio/board.rst b/Documentation/driver-api/gpio/board.rst
index 4ac1e12cf8728d..83b74843ecf258 100644
--- a/Documentation/driver-api/gpio/board.rst
+++ b/Documentation/driver-api/gpio/board.rst
@@ -55,7 +55,7 @@ looked up by the gpiod functions internally) used in the device tree. With above
 
 Internally, the GPIO subsystem prefixes the GPIO suffix ("gpios" or "gpio")
 with the string passed in con_id to get the resulting string
-(``snprintf(... "%s-%s", con_id, gpio_suffixes[]``).
+(``snprintf(... "%s-%s", con_id, gpio_suffixes[])``).
 
 ACPI
 ----
diff --git a/Documentation/driver-api/gpio/driver.rst b/Documentation/driver-api/gpio/driver.rst
index a4f160b950892e..3e53374c7e9fba 100644
--- a/Documentation/driver-api/gpio/driver.rst
+++ b/Documentation/driver-api/gpio/driver.rst
@@ -134,6 +134,13 @@ ending up in the pin control back-end "behind" the GPIO controller, usually
 closer to the actual pins. This way the pin controller can manage the below
 listed GPIO configurations.
 
+The optional .get_config() callback reads a configuration back: the packed
+parameter to query goes in, its bare argument comes out, the way
+pinctrl_gpio_get_config() answers. gpiochip_generic_get_config() is its pin
+control backed counterpart. Nothing in gpiolib calls it; it is for the GPIO
+driver's own use, for example to learn the direction of a line when the pin
+controller owns it.
+
 If a pin controller back-end is used, the GPIO controller or hardware
 description needs to provide "GPIO ranges" mapping the GPIO line offsets to pin
 numbers on the pin controller so they can properly cross-reference each other.
diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst
index 6601a258690f83..aaf5d76ed074ed 100644
--- a/Documentation/driver-api/index.rst
+++ b/Documentation/driver-api/index.rst
@@ -47,6 +47,7 @@ of interest to most developers working on device drivers.
    vfio-mediated-device
    vfio
    vfio-pci-device-specific-driver-acceptance
+   vfio-selftests
 
 Bus-level documentation
 =======================
@@ -139,6 +140,7 @@ Subsystem-specific APIs
    sm501
    soundwire/index
    spi
+   steal-governor
    surface_aggregator/index
    switchtec
    sync_file
diff --git a/Documentation/driver-api/libata.rst b/Documentation/driver-api/libata.rst
index 28b8437f6e4fa2..c130b480180fbc 100644
--- a/Documentation/driver-api/libata.rst
+++ b/Documentation/driver-api/libata.rst
@@ -524,8 +524,8 @@ EH will know to handle it later. Then it calls low level libata driver's
 :c:func:`error_handler` callback.
 
 When the :c:func:`error_handler` callback is invoked it stops BMDMA and
-completes the qc. Note that as we're currently in EH, we cannot call
-scsi_done. As described in SCSI EH doc, a recovered scmd should be
+completes the qc. Note that as we're currently in EH, scsi_done() has
+no effect. As described in SCSI EH doc, a recovered scmd should be
 either retried with :c:func:`scsi_queue_insert` or finished with
 :c:func:`scsi_finish_command`. Here, we override ``qc->scsidone`` with
 :c:func:`scsi_finish_command` and calls :c:func:`ata_qc_complete`.
diff --git a/Documentation/driver-api/media/tx-rx.rst b/Documentation/driver-api/media/tx-rx.rst
index 7df2407817b351..9b231fa0216a5d 100644
--- a/Documentation/driver-api/media/tx-rx.rst
+++ b/Documentation/driver-api/media/tx-rx.rst
@@ -104,7 +104,11 @@ where
    * - k
      - 16 for D-PHY and 7 for C-PHY.
 
-Information on whether D-PHY or C-PHY is used, and the value of ``nr_of_lanes``, can be obtained from the OF endpoint configuration.
+Information on whether D-PHY or C-PHY is used as well as the value of
+``nr_of_lanes`` can be obtained from the V4L2 endpoint configuration; see
+:c:func:`v4l2_fwnode_endpoint_alloc_parse()`,
+:c:func:`v4l2_fwnode_endpoint_parse()` and
+:c:func:`v4l2_get_active_data_lanes()`.
 
 .. note::
 
diff --git a/Documentation/driver-api/mmc/mmc-async-req.rst b/Documentation/driver-api/mmc/mmc-async-req.rst
index 0f7197c9c3b529..d9d4ac74b8460b 100644
--- a/Documentation/driver-api/mmc/mmc-async-req.rst
+++ b/Documentation/driver-api/mmc/mmc-async-req.rst
@@ -23,7 +23,7 @@ MMC request.
 MMC block driver
 ================
 
-The mmc_blk_issue_rw_rq() in the MMC block driver is made non-blocking.
+The mmc_blk_mq_issue_rw_rq() in the MMC block driver is made non-blocking.
 
 The increase in throughput is proportional to the time it takes to
 prepare (major part of preparations are dma_map_sg() and dma_unmap_sg())
@@ -34,21 +34,20 @@ platform. In power save mode, when clocks run on a lower frequency, the DMA
 preparation may cost even more. As long as these slower preparations are run
 in parallel with the transfer performance won't be affected.
 
-Details on measurements from IOZone and mmc_test
-================================================
+MMC core API
+============
 
-https://wiki.linaro.org/WorkingGroups/Kernel/Specs/StoragePerfMMC-async-req
+The preparation of a request is separated from starting the transfer, so
+that a host can prepare a request while another one is still in progress:
 
-MMC core API extension
-======================
-
-There is one new public function mmc_start_req().
-
-It starts a new MMC command request for a host. The function isn't
-truly non-blocking. If there is an ongoing async request it waits
-for completion of that request and starts the new one and returns. It
-doesn't wait for the new request to complete. If there is no ongoing
-request it starts the new request and returns immediately.
+ * mmc_pre_req() prepares a request before it is started. It may be called
+   while another request is running on the host.
+ * mmc_start_request() starts a prepared request without waiting for it to
+   complete. mmc_wait_for_req() starts a request and waits for it to
+   complete as well.
+ * mmc_post_req() releases the resources allocated by mmc_pre_req() after
+   the request has completed. It may likewise run while another request is
+   active.
 
 MMC host extensions
 ===================
@@ -59,40 +58,3 @@ to before and after the actual mmc_host_ops.request() function is called.
 
 In the DMA case pre_req() may do dma_map_sg() and prepare the DMA
 descriptor, and post_req() runs the dma_unmap_sg().
-
-Optimize for the first request
-==============================
-
-The first request in a series of requests can't be prepared in parallel
-with the previous transfer, since there is no previous request.
-
-The argument is_first_req in pre_req() indicates that there is no previous
-request. The host driver may optimize for this scenario to minimize
-the performance loss. A way to optimize for this is to split the current
-request in two chunks, prepare the first chunk and start the request,
-and finally prepare the second chunk and start the transfer.
-
-Pseudocode to handle is_first_req scenario with minimal prepare overhead::
-
-  if (is_first_req && req->size > threshold)
-     /* start MMC transfer for the complete transfer size */
-     mmc_start_command(MMC_CMD_TRANSFER_FULL_SIZE);
-
-     /*
-      * Begin to prepare DMA while cmd is being processed by MMC.
-      * The first chunk of the request should take the same time
-      * to prepare as the "MMC process command time".
-      * If prepare time exceeds MMC cmd time
-      * the transfer is delayed, guesstimate max 4k as first chunk size.
-      */
-      prepare_1st_chunk_for_dma(req);
-      /* flush pending desc to the DMAC (dmaengine.h) */
-      dma_issue_pending(req->dma_desc);
-
-      prepare_2nd_chunk_for_dma(req);
-      /*
-       * The second issue_pending should be called before MMC runs out
-       * of the first chunk. If the MMC runs out of the first data chunk
-       * before this call, the transfer is delayed.
-       */
-      dma_issue_pending(req->dma_desc);
diff --git a/Documentation/driver-api/mmc/mmc-tools.rst b/Documentation/driver-api/mmc/mmc-tools.rst
index eee1c2ccfa8f19..728d3aa67eda67 100644
--- a/Documentation/driver-api/mmc/mmc-tools.rst
+++ b/Documentation/driver-api/mmc/mmc-tools.rst
@@ -32,6 +32,7 @@ The mmc-utils tools can do the following:
  - Write to rpmb device from data file.
  - Enable the eMMC cache feature.
  - Disable the eMMC cache feature.
+ - Print the eMMC health status.
  - Print and parse CID data.
  - Print and parse CSD data.
  - Print and parse SCR data.
diff --git a/Documentation/driver-api/mtd/spi-nor.rst b/Documentation/driver-api/mtd/spi-nor.rst
index 747a326fb6c0d5..5441fa7d4f04ed 100644
--- a/Documentation/driver-api/mtd/spi-nor.rst
+++ b/Documentation/driver-api/mtd/spi-nor.rst
@@ -118,7 +118,7 @@ section, after the ``---`` marker.
     write size		1
     page size		256
     address nbytes	3
-    flags		HAS_LOCK | HAS_16BIT_SR | SOFT_RESET | SWP_IS_VOLATILE
+    flags		HAS_LOCK | SOFT_RESET | SWP_IS_VOLATILE
 
     opcodes
      read		0xeb
diff --git a/Documentation/driver-api/pci/p2pdma.rst b/Documentation/driver-api/pci/p2pdma.rst
index d3f406cca69409..63cff9e4d2c9ac 100644
--- a/Documentation/driver-api/pci/p2pdma.rst
+++ b/Documentation/driver-api/pci/p2pdma.rst
@@ -167,9 +167,11 @@ In this case the initiator and target pci_devices are known and the P2P subsyste
 is used to determine the mapping type. The phys_addr_t-based DMA API is used to
 establish the dma_addr_t.
 
-Lifecycle is controlled by DMABUF move_notify(). When the exporting driver wants
+Lifecycle is controlled by DMABUF revocation. When the exporting driver wants
 to remove() it must deliver an invalidation shutdown to all DMABUF importing
-drivers through move_notify() and synchronously DMA unmap all the MMIO.
+drivers through dma_buf_invalidate_mappings() and synchronously DMA unmap all
+the MMIO. Importers unable to complete that unmap within bounded time have to
+be rejected when they attach, which dma_buf_attach_revocable() checks for.
 
 No importing driver can continue to have a DMA map to the MMIO after the
 exporting driver has destroyed its p2p_provider.
diff --git a/Documentation/driver-api/pwm.rst b/Documentation/driver-api/pwm.rst
index 0d27a40f581879..b6b67f7d490537 100644
--- a/Documentation/driver-api/pwm.rst
+++ b/Documentation/driver-api/pwm.rst
@@ -177,7 +177,7 @@ and pwm_put() may not be called from an atomic context.
 Most functions in the PWM consumer API might sleep and so must not be called
 from atomic context. The notable exception is pwm_apply_atomic() which has the
 same semantics as pwm_apply_might_sleep() but can be called from atomic context.
-(The price for that is that it doesn't work for all PWM devices, use
+The price for that is that it doesn't work for all PWM devices. Use
 pwm_might_sleep() to check if a given PWM supports atomic operation.
 
 Locking in the PWM core ensures that callbacks related to a single chip are
diff --git a/Documentation/driver-api/reset.rst b/Documentation/driver-api/reset.rst
index 7a6571849664f0..932699c57d1ed4 100644
--- a/Documentation/driver-api/reset.rst
+++ b/Documentation/driver-api/reset.rst
@@ -213,7 +213,7 @@ devm_reset_controller_register().
    :internal:
 
 .. kernel-doc:: drivers/reset/core.c
-   :functions: of_reset_simple_xlate
+   :functions: fwnode_reset_simple_xlate
                reset_controller_register
                reset_controller_unregister
                devm_reset_controller_register
diff --git a/Documentation/driver-api/serial/driver.rst b/Documentation/driver-api/serial/driver.rst
index c1db6a1a67c4ec..975b0ba53e982d 100644
--- a/Documentation/driver-api/serial/driver.rst
+++ b/Documentation/driver-api/serial/driver.rst
@@ -70,7 +70,7 @@ Other functions
 
 .. kernel-doc:: drivers/tty/serial/serial_core.c
    :identifiers: uart_update_timeout uart_get_baud_rate uart_get_divisor
-           uart_match_port uart_write_wakeup uart_register_driver
+           uart_write_wakeup uart_register_driver
            uart_unregister_driver uart_suspend_port uart_resume_port
            uart_add_one_port uart_remove_one_port uart_console_write
            uart_parse_earlycon uart_parse_options uart_set_options
diff --git a/Documentation/driver-api/serial/serial-rs485.rst b/Documentation/driver-api/serial/serial-rs485.rst
index f53043d2107159..98d8d83718c6d5 100644
--- a/Documentation/driver-api/serial/serial-rs485.rst
+++ b/Documentation/driver-api/serial/serial-rs485.rst
@@ -50,7 +50,10 @@ RS485 Serial Communications
    matching to the current configuration.
 
 .. kernel-doc:: include/uapi/linux/serial.h
-   :identifiers: serial_rs485 uart_get_rs485_mode
+   :identifiers: serial_rs485
+
+.. kernel-doc:: drivers/tty/serial/serial_core.c
+   :identifiers: uart_get_rs485_mode
 
 4. Usage from user-level
 ========================
diff --git a/Documentation/driver-api/steal-governor.rst b/Documentation/driver-api/steal-governor.rst
new file mode 100644
index 00000000000000..3817eedb38d787
--- /dev/null
+++ b/Documentation/driver-api/steal-governor.rst
@@ -0,0 +1,151 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Steal Governor
+==============
+
+:Author: Shrikanth Hegde <sshegde@linux.ibm.com>
+
+Introduction
+============
+
+The steal governor is aimed at mitigating the Noisy Neighbour problem
+which occurs in paravirtualized environments with CPU overcommit.
+The performance of a workload running in one VM gets degraded by
+the activity of other VMs on the same host. As a result, all VMs
+collectively make slower forward progress.
+
+In such systems, high utilization in all VMs causes the hypervisor to
+frequently preempt vCPUs. This vCPU preemption is expensive.
+To mitigate this, the kernel aims to restrict workloads to a subset of
+Preferred CPUs to reduce physical CPU contention.
+A detailed explanation of Preferred CPUs is available in
+``Documentation/scheduler/sched-paravirt.rst``.
+
+The steal governor selects ``CONFIG_PREFERRED_CPU=y`` which enables the
+scheduler core infrastructure to move the tasks to Preferred CPUs where
+possible. The driver controls the policy decisions regarding the state of
+preferred CPUs. That is, this driver decides which CPUs are preferred
+and which CPUs are non-preferred.
+
+The driver code is available at ``drivers/virt/steal_governor.c``.
+
+Core idea
+=========
+
+steal time is an indication available today in Guest which shows contention
+for underlying physical CPU. Use it as a hint in the guest to fold the
+workload to a reduced set of vCPUs. When there is contention, steal time
+will show up in all the guests. When each guest honors the hint and folds
+the workload to a smaller set of vCPUs (Preferred CPUs), it reduces the
+contention and thereby reduces vCPU preemption.
+This is achieved without any cross-guest communication.
+
+Steal governor driver effectively does:
+
+1. Periodically computes the steal ratio using accumulated steal time
+   across possible CPUs, normalized by the number of active CPUs.
+
+2. If steal ratio is greater than high threshold, reduce the number of
+   preferred CPUs by 1 core. Ensure at least one core is left always.
+   Skip changing the state of offline CPUs in that core.
+
+3. If steal ratio is less than or equal to low threshold, increase the
+   number of preferred CPUs by 1 core. If preferred is same as active,
+   nothing to be done. Skip changing the state of offline CPUs.
+   This helps to handle cases where few CPUs are offline in a core and
+   those offline CPUs will not be marked as preferred.
+
+4. Ensure preferred CPUs is always subset of active CPUs.
+   On feature disable it is same as active CPUs.
+
+This feature works best only when all the VMs enable the feature as
+it is a co-operative scheme. If a specific VM doesn't enable this feature
+it may end up with more CPUs than others, still should lead to better
+performance when seen from system view. Those who enable this driver must
+ensure it is enabled in all VMs.
+
+Note that this driver is strictly intended for actual guests; for example,
+loading this module in a privileged VM like Xen Dom0 is blocked.
+
+Workload considerations
+=======================
+
+The steal governor is useful for workloads where vCPU preemption has
+costs beyond the lost CPU time, such as lock-holder preemption, critical
+sections, communicating threads, and cache or TLB disruption.
+
+Pure CPU-time workloads with independent workers may not benefit and
+could see a small regression due to additional guest scheduling overhead.
+
+Module Parameters
+=================
+
+interval_ms
+-----------
+
+How often steal governor checks for steal time.
+Default: 1000 i.e. 1 second. Value should be in between 100ms to 100sec.
+
+This controls how fast steal governor driver reacts to changes to the
+contention of physical CPUs. Since it does a fair amount of work, setting
+too low may have overhead. Setting it too high might render it ineffective.
+
+low_threshold
+-------------
+
+lower threshold value in percentage * 100.
+Default: 200, i.e. 2% steal is considered as low threshold.
+Can't be higher than high_threshold.
+
+This determines what values should be considered as nil/no steal values.
+When steal governor sees steal ratio is less than or equal to this value,
+it will increase the preferred CPUs by 1 core.
+Using zero might cause oscillations.
+
+high_threshold
+--------------
+
+higher threshold value in percentage * 100
+Default: 500, i.e. 5% steal is considered as high threshold.
+Can't be lower than low_threshold. Must be less than 10000.
+
+This determines what values should be considered as high steal values.
+When steal governor sees steal ratio is higher than this value, it will
+reduce the preferred CPUs by 1 core.
+
+Limitations of default values
+-----------------------------
+
+Because of the vast diversity in VM configurations and different
+architectures, the default thresholds may not be optimal for all systems.
+Users may need to tune these parameters based on the system under
+test to achieve the best results.
+
+The governor sums the steal time across all possible CPUs, which ensures
+the accumulated steal time remains a monotonically increasing value.
+However, to calculate the effective steal ratio, it divides this sum
+by the number of active CPUs. Because only active CPUs contribute to
+the steal time delta, this prevents threshold dilution on sparsely
+populated systems.
+
+The driver reduces/increases preferred CPUs by core-level. This could provide
+faster convergence for hypervisors such as powerVM. But on KVM and Xen
+convergence could be slower depending on the configuration.
+Using a smaller interval_ms could help one to expedite it.
+
+Reasons for CONFIG_STEAL_GOVERNOR=m
+===================================
+
+Selecting this driver makes CONFIG_PREFERRED_CPU=y. That makes configs
+driven by user preference. Though one can have CONFIG_STEAL_GOVERNOR=y,
+It is recommended to build CONFIG_STEAL_GOVERNOR=m due to below reasons:
+
+1. Doing periodic work has additional overheads. Enabling this driver
+   in systems where steal time cannot happen is of no use. There is no
+   benefit with additional overheads in such systems.
+
+2. This works well when all VMs work in a co-operative manner. When an
+   administrative user enables it in one VM, he/she will likely enable
+   it in all VMs.
+
+3. User can tweak the module parameters by reloading the module.
diff --git a/Documentation/driver-api/thermal/cpu-idle-cooling.rst b/Documentation/driver-api/thermal/cpu-idle-cooling.rst
index c2a7ca676853f9..b9d4c91a7f1588 100644
--- a/Documentation/driver-api/thermal/cpu-idle-cooling.rst
+++ b/Documentation/driver-api/thermal/cpu-idle-cooling.rst
@@ -161,7 +161,7 @@ tree. So with the idle injection mechanism, we want an average power
 specific OPP and idle another amount of time. That could be put in a
 equation::
 
- P(opp)target = ((Trunning x (P(opp)running) + (Tidle x P(opp)idle)) /
+ P(opp)target = (Trunning x (P(opp)running) + (Tidle x P(opp)idle)) /
 			(Trunning + Tidle)
 
   ...
diff --git a/Documentation/driver-api/vfio-selftests.rst b/Documentation/driver-api/vfio-selftests.rst
new file mode 100644
index 00000000000000..9d39a22519d6ab
--- /dev/null
+++ b/Documentation/driver-api/vfio-selftests.rst
@@ -0,0 +1,252 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==============
+VFIO Selftests
+==============
+
+VFIO selftests are built on top of the kernel's selftests framework and are
+located in ``tools/testing/selftests/vfio``.
+
+VFIO selftests enable kernel developers to write and run tests that take the
+form of userspace programs that interact with VFIO and IOMMUFD uAPIs. VFIO
+selftests can be used to write functional tests for new features, regression
+tests for bugs, and performance tests for optimizations.
+
+These tests are designed to interact with real PCI devices, i.e. they do not
+rely on mocking out or faking any behavior in the kernel. This allows the tests
+to exercise not only VFIO but also IOMMUFD, the IOMMU driver, interrupt
+remapping, IRQ handling, etc.
+
+Running Tests
+=============
+
+Running VFIO selftests requires a device bound to the ``vfio-pci`` driver.
+There are scripts in ``tools/testing/selftests/vfio/scripts/`` to help with
+setting one up.
+
+Picking a Device
+----------------
+
+Some VFIO selftests require a device with a supported driver implementation in
+the selftests library (see `Driver Framework`_), while other tests (such as
+basic PCI configuration space or reset tests) can use any arbitrary PCI device.
+
+This section will be expanded in the future to guide developers on how to find
+a device with a supported driver. In the meantime, the selftests library
+includes driver support for the following devices:
+
+- Intel IGB Gigabit Ethernet controllers
+- Intel IOAT (I/O Acceleration Technology) DMA engines
+- Intel DSA (Data Streaming Accelerator)
+- NVIDIA Falcon engines (found in many NVIDIA GPUs, e.g., GTX 1080 or RTX 2080)
+
+Note that using Intel DSA devices requires setting the module parameter
+``vfio_pci.disable_denylist=Y`` to allow ``vfio-pci`` to bind to DSA devices.
+
+Setting up Devices
+------------------
+
+.. code-block:: sh
+
+    tools/testing/selftests/vfio/scripts/setup.sh 0000:01:00.0
+
+The script handles unbinding the device from its current driver and binding it
+to ``vfio-pci``. Metadata about this device is stored in
+``/tmp/vfio-selftests-devices`` so that the device can be cleaned up later
+(bound back to its original driver).
+
+Execution
+---------
+
+VFIO selftests expect the PCI device BDF string as their final command-line
+argument. e.g.
+
+.. code-block:: sh
+
+    tools/testing/selftests/vfio/vfio_pci_device_test 0000:01:00.0
+
+Alternatively, the PCI device BDF can be passed via the ``VFIO_SELFTESTS_BDF``
+environment variable, which can be useful when running all tests:
+
+.. code-block:: sh
+
+	export VFIO_SELFTESTS_BDF=0000:01:00.0
+	make -C tools/testing/selftests TARGETS=vfio install
+	tools/testing/selftests/kselftest_install/run_kselftest.sh -c vfio
+
+Cleanup
+-------
+
+.. code-block:: sh
+
+    tools/testing/selftests/vfio/scripts/cleanup.sh
+
+This script handles unbinding all previously setup devices from ``vfio-pci``
+and binding them back to their original driver. Specific devices can be cleaned
+up by passing their BDF string to the cleanup script.
+
+Writing Tests
+=============
+
+VFIO selftests leverage the ``kselftest_harness.h`` macros to structure setup
+and teardown scenarios. A basic test involves:
+
+1. Declaring a test FIXTURE containing the device and IOMMU state.
+2. Generating test variants that span the different IOMMU modes utilizing
+   ``FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES()``.
+3. Initializing the device and binding the default driver ops.
+
+For a complete example of writing a VFIO selftest, refer to
+``tools/testing/selftests/vfio/vfio_pci_device_test.c``.
+
+VFIO Selftests Library (libvfio)
+================================
+
+The selftests directory contains a helper library compiled from the ``lib/``
+subdirectory. It provides structures and helper functions to handle standard
+operations like opening VFIO devices, creating VFIO containers, and interacting
+with VFIO and IOMMUFD.
+
+All VFIO selftests have libvfio linked in by default. Tests can access the
+library by including ``<libvfio.h>``.
+
+Core Objects
+------------
+
+The VFIO selftests framework revolves around two distinct core objects:
+``struct iommu`` and ``struct vfio_pci_device``. It is important to understand
+the distinction and relationship between them:
+
+- ``struct iommu`` represents an IOMMU domain and the API used to program it
+  (e.g. legacy VFIO Type1 vs IOMMUFD). It is responsible for managing the I/O
+  address space, handling DMA mappings, and translating between host virtual
+  addresses (HVA) and IO virtual addresses (IOVA).
+
+- ``struct vfio_pci_device`` represents a single PCI device. It encapsulates
+  the VFIO device file descriptor, handles access to the PCI configuration
+  space, manages memory-mapped BARs, and drives device interrupts.
+
+A typical test initializes an IOMMU instance first, then passes it when
+initializing one or more PCI devices, linking the devices to the shared IOMMU
+domain.
+
+``struct iommu``
+^^^^^^^^^^^^^^^^
+
+The ``struct iommu`` abstracts away the complexity of managing VFIO containers,
+IOMMU groups, and IOMMUFD contexts. A test allocates its IOMMU handle using
+``iommu_init()`` passing one of the supported IOMMU modes:
+
+.. code-block:: c
+
+    struct iommu *iommu = iommu_init(MODE_IOMMUFD);
+    ...
+    iommu_cleanup(iommu);
+
+The selftest framework is designed to work uniformly across all standard kernel
+IOMMU APIs. The defined modes (in ``iommu.h``) include:
+
+- ``MODE_VFIO_TYPE1_IOMMU``
+- ``MODE_VFIO_TYPE1V2_IOMMU``
+- ``MODE_IOMMUFD_COMPAT_TYPE1``
+- ``MODE_IOMMUFD_COMPAT_TYPE1V2``
+- ``MODE_IOMMUFD``
+
+VFIO selftests can replicate their test cases across all possible IOMMU modes
+using FIXTURE_VARIANT_ADD_ALL_IOMMU_MODES() to ensure kernel IOMMU APIs remain
+compatible.
+
+``struct vfio_pci_device``
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A PCI device is represented as a ``struct vfio_pci_device``. The device
+representation is initialized using ``vfio_pci_device_init()``, which takes a
+target BDF string, opens the device in VFIO, and binds it to an existing
+``struct iommu`` instance:
+
+.. code-block:: c
+
+    struct vfio_pci_device *device = vfio_pci_device_init(device_bdf, iommu);
+    ...
+    vfio_pci_device_cleanup(device);
+
+``struct iova_allocator``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To facilitate DMA mappings without hardcoding IOVAs (IO virtual addresses) that
+might conflict with reserved platform address ranges, the library provides a
+``struct iova_allocator``.
+
+The IOVA allocator queries the underlying IOMMU (via ``struct iommu``) for its
+valid target IOVA ranges. Tests can safely carve out unique contiguous chunks
+of IOVA space for device DMA by repeatedly calling:
+
+.. code-block:: c
+
+    struct iova_allocator *allocator = iova_allocator_init(iommu);
+    iova_t addr = iova_allocator_alloc(allocator, size);
+
+Driver Framework
+----------------
+
+The primary goal of VFIO selftests is to verify the correctness of the kernel
+code handling VFIO devices, including VFIO core, IOMMUFD, IOMMU drivers, page
+pinning, DMA mapping/unmapping, interrupt remapping, and IRQ delivery.
+
+Because different physical devices have vastly different programming interfaces
+for triggering work, the selftests library implements a standard driver
+framework.  This framework abstracts diverse hardware endpoints (such as Intel
+IGB, IOAT, or DSA) behind a uniform interface, allowing tests to provoke a
+broad range of devices to generate DMA operations and interrupts in a
+standardized manner.
+
+By exercising DMA and interrupts with real endpoints, tests can thoroughly
+stress and validate the underlying kernel paths, such as IOMMU page table
+management, dirty page tracking, Device TLB and IOTLB invalidations, and
+interrupt delivery under authentic hardware conditions.
+
+All endpoint driver implementations populate a common set of operations defined
+in ``struct vfio_pci_driver_ops``:
+
+- ``probe()``: Validate whether the driver supports the current PCI device
+  (e.g. matching vendor/device IDs).
+- ``init()``: Set up specific control registers or ring-buffer data structures.
+- ``remove()``: Clean up structures before destruction.
+- ``memcpy_start()``: Initiate DMA copies via the device.
+- ``memcpy_wait()``: Poll and wait until previously initiated copies complete
+  safely.
+- ``send_msi()``: Provoke the device to signal an MSI interrupt.
+
+Tests are required to set up a dedicated DMA region for the driver to use
+within ``struct vfio_pci_driver`` to serve as the driver's main memory
+allocation region (e.g. used for in-memory descriptor rings and hardware
+buffers). Tests must provision and map this DMA region before completing driver
+initialization.
+
+For a complete example of interacting with the driver framework from a test
+perspective, refer to ``tools/testing/selftests/vfio/vfio_pci_driver_test.c``.
+
+When contributing a new driver implementation to the selftests library, the
+primary acceptance criteria is that ``vfio_pci_driver_test`` passes using the
+new driver and target device.
+
+Integration with KVM Selftests
+------------------------------
+
+libvfio is not strictly limited to VFIO selftests, it is also leveraged within
+KVM selftests (``tools/testing/selftests/kvm/``). KVM selftests build their own
+copy of libvfio and link against it, enabling libvfio to be used in KVM
+selftests.
+
+This integration enables testing the interaction between KVM and VFIO and
+IOMMUFD. For example, testing the end-to-end delivery of device interrupts into
+running vCPUs.
+
+Contribution Guidelines
+=======================
+
+- Prefer the shortlog prefix ``vfio: selftests: ...`` for commits that modify
+  files in ``tools/testing/selftests/vfio/``.
+- Follow the Coding Style, Comments, Changelogs, and Function References
+  guidelines from the
+  :doc:`KVM x86 maintainers handbook </process/maintainer-kvm-x86>`.
diff --git a/Documentation/features/locking/cmpxchg-local/arch-support.txt b/Documentation/features/locking/cmpxchg-local/arch-support.txt
index 2c3a4b91f16df3..127e8159adaa6b 100644
--- a/Documentation/features/locking/cmpxchg-local/arch-support.txt
+++ b/Documentation/features/locking/cmpxchg-local/arch-support.txt
@@ -12,7 +12,7 @@
     |       arm64: |  ok  |
     |        csky: | TODO |
     |     hexagon: | TODO |
-    |   loongarch: | TODO |
+    |   loongarch: |  ok  |
     |        m68k: | TODO |
     |  microblaze: | TODO |
     |        mips: | TODO |
diff --git a/Documentation/filesystems/befs.rst b/Documentation/filesystems/befs.rst
index a22f603b2938ea..c1dbfe9c95d292 100644
--- a/Documentation/filesystems/befs.rst
+++ b/Documentation/filesystems/befs.rst
@@ -44,8 +44,8 @@ implementation.
 Which is it, BFS or BEFS?
 =========================
 Be, Inc said, "BeOS Filesystem is officially called BFS, not BeFS".
-But Unixware Boot Filesystem is called bfs, too. And they are already in
-the kernel. Because of this naming conflict, on Linux the BeOS
+But the UnixWare Boot Filesystem is called bfs, too, and it was already
+in the kernel. Because of this naming conflict, on Linux the BeOS
 filesystem is called befs.
 
 How to Install
diff --git a/Documentation/filesystems/bfs.rst b/Documentation/filesystems/bfs.rst
deleted file mode 100644
index ce14b901880729..00000000000000
--- a/Documentation/filesystems/bfs.rst
+++ /dev/null
@@ -1,60 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-========================
-BFS Filesystem for Linux
-========================
-
-The BFS filesystem is used by SCO UnixWare OS for the /stand slice, which
-usually contains the kernel image and a few other files required for the
-boot process.
-
-In order to access /stand partition under Linux you obviously need to
-know the partition number and the kernel must support UnixWare disk slices
-(CONFIG_UNIXWARE_DISKLABEL config option). However BFS support does not
-depend on having UnixWare disklabel support because one can also mount
-BFS filesystem via loopback::
-
-    # losetup /dev/loop0 stand.img
-    # mount -t bfs /dev/loop0 /mnt/stand
-
-where stand.img is a file containing the image of BFS filesystem.
-When you have finished using it and umounted you need to also deallocate
-/dev/loop0 device by::
-
-    # losetup -d /dev/loop0
-
-You can simplify mounting by just typing::
-
-    # mount -t bfs -o loop stand.img /mnt/stand
-
-this will allocate the first available loopback device (and load loop.o
-kernel module if necessary) automatically. If the loopback driver is not
-loaded automatically, make sure that you have compiled the module and
-that modprobe is functioning. Beware that umount will not deallocate
-/dev/loopN device if /etc/mtab file on your system is a symbolic link to
-/proc/mounts. You will need to do it manually using "-d" switch of
-losetup(8). Read losetup(8) manpage for more info.
-
-To create the BFS image under UnixWare you need to find out first which
-slice contains it. The command prtvtoc(1M) is your friend::
-
-    # prtvtoc /dev/rdsk/c0b0t0d0s0
-
-(assuming your root disk is on target=0, lun=0, bus=0, controller=0). Then you
-look for the slice with tag "STAND", which is usually slice 10. With this
-information you can use dd(1) to create the BFS image::
-
-    # umount /stand
-    # dd if=/dev/rdsk/c0b0t0d0sa of=stand.img bs=512
-
-Just in case, you can verify that you have done the right thing by checking
-the magic number::
-
-    # od -Ad -tx4 stand.img | more
-
-The first 4 bytes should be 0x1badface.
-
-If you have any patches, questions or suggestions regarding this BFS
-implementation please contact the author:
-
-Tigran Aivazian <aivazian.tigran@gmail.com>
diff --git a/Documentation/filesystems/exfat.rst b/Documentation/filesystems/exfat.rst
new file mode 100644
index 00000000000000..ce5c9344a7b23d
--- /dev/null
+++ b/Documentation/filesystems/exfat.rst
@@ -0,0 +1,117 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==================================
+The Linux exFAT filesystem driver
+==================================
+
+
+.. Table of contents
+
+   - Overview
+   - Utilities support
+   - Supported mount options
+
+
+Overview
+========
+
+exFAT is a filesystem designed for removable storage and other devices that
+need to store large files.  The Linux exFAT filesystem driver provides read
+and write support for exFAT volumes.
+
+To mount an exFAT volume, use the ``exfat`` filesystem type::
+
+  mount -t exfat /dev/sdX1 /mnt
+
+
+Utilities support
+=================
+
+The exfatprogs project provides userspace utilities for creating, checking,
+repairing, inspecting, and tuning exFAT filesystems.  Use exfatprogs when
+creating or checking an exFAT filesystem.  For example, use ``mkfs.exfat``
+to create a filesystem and ``fsck.exfat`` to check or repair one.
+
+The project is available at:
+
+  https://github.com/exfatprogs/exfatprogs
+
+
+Supported mount options
+=======================
+
+The exFAT driver supports the following mount options:
+
+======================= ====================================================
+uid=
+gid=                    Set the owner and group of all files and
+                        directories.  The default is the uid and gid of
+                        the process mounting the filesystem.
+
+umask=                  Set the permission mask for files and directories.
+                        The default is the umask of the process mounting
+                        the filesystem.
+
+dmask=                  Set the permission mask for directories.
+
+fmask=                  Set the permission mask for files.
+
+allow_utime=            Control the permission check for changing file
+                        timestamps.  Only permission bits 0022 are used.
+                        Permission bit 0020 allows members of the file's
+                        group to change timestamps, and permission bit 0002
+                        allows other users to change timestamps.  The
+                        default is derived from dmask (``~dmask & 0022``).
+
+iocharset=name          Character set used to convert between user-visible
+                        filenames and the UTF-16 character encoding used by
+                        exFAT.  The default is
+                        CONFIG_EXFAT_DEFAULT_IOCHARSET, which is ``utf8``
+                        unless changed at kernel configuration time.  Use
+                        ``iocharset=utf8`` for UTF-8 filename handling.
+
+errors=                 Specify exFAT behavior on filesystem errors.  The
+                        value must be ``panic``, ``continue``, or
+                        ``remount-ro``.  These respectively panic, continue
+                        without changing the filesystem, or remount the
+                        filesystem read-only.  The default is
+                        ``remount-ro``.
+
+discard                 Issue discard/TRIM requests to the block device
+                        when clusters are freed.  This is disabled by
+                        default.  ``nodiscard`` disables it explicitly.
+
+keep_last_dots          Keep trailing periods in path components during
+                        lookup.  Without this option, trailing periods are
+                        stripped.  Existing entries with trailing periods
+                        can be accessed when this option is enabled, but
+                        creating new entries with trailing periods is
+                        rejected.
+
+sys_tz                  Use the system timezone as the UTC offset when an
+                        exFAT timestamp does not contain a valid timezone
+                        offset.  This takes precedence over time_offset.
+
+time_offset=minutes     Set the UTC offset, in minutes, used when an exFAT
+                        timestamp does not contain a valid timezone offset.
+                        Values from -1440 to 1440 are accepted.  The default
+                        is 0.  This option is ignored when sys_tz is set.
+
+zero_size_dir           Create directories with zero size and without
+                        allocating a cluster.  This is disabled by default;
+                        the default behavior allocates a cluster for a new
+                        directory.  ``nozero_size_dir`` disables it
+                        explicitly.
+======================= ====================================================
+
+
+Deprecated mount options
+------------------------
+
+The following options are accepted for compatibility but should not be used:
+
+``utf8``
+  Deprecated.  Use ``iocharset=utf8`` instead.
+
+``debug``, ``namecase=``, ``codepage=``
+  Deprecated and ignored by the exFAT driver.
diff --git a/Documentation/filesystems/fuse/fuse.rst b/Documentation/filesystems/fuse/fuse.rst
index 0fbd5a03fdc94c..063d99959396dc 100644
--- a/Documentation/filesystems/fuse/fuse.rst
+++ b/Documentation/filesystems/fuse/fuse.rst
@@ -173,10 +173,10 @@ the error set to EINTR.
 It is also possible that there's a race between processing the
 original request and its INTERRUPT request.  There are two possibilities:
 
-  1. The INTERRUPT request is processed before the original request is
+  1) The INTERRUPT request is processed before the original request is
      processed
 
-  2. The INTERRUPT request is processed after the original request has
+  2) The INTERRUPT request is processed after the original request has
      been answered
 
 If the filesystem cannot find the original request, it should wait for
@@ -239,9 +239,9 @@ How are requirements fulfilled?
 
  A) The mount owner could gain elevated privileges by either:
 
-    1. creating a filesystem containing a device file, then opening this device
+    1) creating a filesystem containing a device file, then opening this device
 
-    2. creating a filesystem containing a suid or sgid application, then executing this application
+    2) creating a filesystem containing a suid or sgid application, then executing this application
 
     The solution is not to allow opening device files and ignore
     setuid and setgid bits when executing programs.  To ensure this
diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst
index 734a45e516675c..1100130ccf0a80 100644
--- a/Documentation/filesystems/index.rst
+++ b/Documentation/filesystems/index.rst
@@ -75,7 +75,6 @@ Documentation for filesystem implementations.
    autofs
    autofs-mount-control
    befs
-   bfs
    btrfs
    ceph
    coda
@@ -87,6 +86,7 @@ Documentation for filesystem implementations.
    ecryptfs
    efivarfs
    erofs
+   exfat
    ext2
    ext3
    ext4/index
diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst
index 844d65eb47a5fb..f7363ad770c766 100644
--- a/Documentation/filesystems/locking.rst
+++ b/Documentation/filesystems/locking.rst
@@ -27,7 +27,7 @@ prototypes::
 	int (*d_init)(struct dentry *);
 	void (*d_release)(struct dentry *);
 	void (*d_iput)(struct dentry *, struct inode *);
-	char *(*d_dname)((struct dentry *dentry, char *buffer, int buflen);
+	char *(*d_dname)(struct dentry *dentry, char *buffer, int buflen);
 	struct vfsmount *(*d_automount)(struct path *path);
 	int (*d_manage)(const struct path *, bool);
 	struct dentry *(*d_real)(struct dentry *, enum d_real_type type);
@@ -61,23 +61,23 @@ inode_operations
 
 prototypes::
 
-	int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t);
+	int (*create) (const struct mnt_idmap *, struct inode *,struct dentry *,umode_t);
 	struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
 	int (*link) (struct dentry *,struct inode *,struct dentry *);
 	int (*unlink) (struct inode *,struct dentry *);
-	int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,const char *);
-	struct dentry *(*mkdir) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t);
+	int (*symlink) (const struct mnt_idmap *, struct inode *,struct dentry *,const char *);
+	struct dentry *(*mkdir) (const struct mnt_idmap *, struct inode *,struct dentry *,umode_t);
 	int (*rmdir) (struct inode *,struct dentry *);
-	int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t,dev_t);
-	int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *,
+	int (*mknod) (const struct mnt_idmap *, struct inode *,struct dentry *,umode_t,dev_t);
+	int (*rename) (const struct mnt_idmap *, struct inode *, struct dentry *,
 			struct inode *, struct dentry *, unsigned int);
 	int (*readlink) (struct dentry *, char __user *,int);
 	const char *(*get_link) (struct dentry *, struct inode *, struct delayed_call *);
 	void (*truncate) (struct inode *);
-	int (*permission) (struct mnt_idmap *, struct inode *, int, unsigned int);
+	int (*permission) (const struct mnt_idmap *, struct inode *, int, unsigned int);
 	struct posix_acl * (*get_inode_acl)(struct inode *, int, bool);
-	int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *);
-	int (*getattr) (struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int);
+	int (*setattr) (const struct mnt_idmap *, struct dentry *, struct iattr *);
+	int (*getattr) (const struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int);
 	ssize_t (*listxattr) (struct dentry *, char *, size_t);
 	int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, u64 len);
 	void (*update_time)(struct inode *inode, enum fs_update_time type,
@@ -86,12 +86,12 @@ prototypes::
 	int (*atomic_open)(struct inode *, struct dentry *,
 				struct file *, unsigned open_flag,
 				umode_t create_mode);
-	int (*tmpfile) (struct mnt_idmap *, struct inode *,
+	int (*tmpfile) (const struct mnt_idmap *, struct inode *,
 			struct file *, umode_t);
-	int (*fileattr_set)(struct mnt_idmap *idmap,
+	int (*fileattr_set)(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct file_kattr *fa);
 	int (*fileattr_get)(struct dentry *dentry, struct file_kattr *fa);
-	struct posix_acl * (*get_acl)(struct mnt_idmap *, struct dentry *, int);
+	struct posix_acl * (*get_acl)(const struct mnt_idmap *, struct dentry *, int);
 	struct offset_ctx *(*get_offset_ctx)(struct inode *inode);
 
 locking rules:
@@ -148,7 +148,7 @@ prototypes::
 		   struct inode *inode, const char *name, void *buffer,
 		   size_t size);
 	int (*set)(const struct xattr_handler *handler,
-                   struct mnt_idmap *idmap,
+                   const struct mnt_idmap *idmap,
                    struct dentry *dentry, struct inode *inode, const char *name,
                    const void *buffer, size_t size, int flags);
 
diff --git a/Documentation/filesystems/mmap_prepare.rst b/Documentation/filesystems/mmap_prepare.rst
index 82c99c95ad854e..a476e1006bf126 100644
--- a/Documentation/filesystems/mmap_prepare.rst
+++ b/Documentation/filesystems/mmap_prepare.rst
@@ -164,5 +164,86 @@ pointer. These are:
   sufficient entries in the page array to cover the entire range of the
   described VMA.
 
+* mmap_action_map_discontig_kernel_pages() - Maps a discontiguous range of
+  `struct page` pointers over the VMA. They must span from the start of the VMA,
+  but may terminate prior to the end (leaving the remainder unmapped).
+
 **NOTE:** The ``action`` field should never normally be manipulated directly,
 rather you ought to use one of these helpers.
+
+Discontiguous Actions
+=====================
+
+Some actions can be performed across discontiguous ranges.
+
+Map kernel pages
+----------------
+
+To map kernel pages discontiguously, you must provide hooks using ``struct
+discontig_kernel_page_ops``:
+
+.. code-block:: C
+
+    struct discontig_kernel_page_ops {
+        int (*init)(void *vm_private_data, void **private);
+        int (*get)(struct discontig_kernel_page_state *state);
+    };
+
+The ``init`` hook is optional and allows state to be established before the
+operation starts, for instance taking a reference count. Nothing is invoked
+after the operation, so ``init`` must not leave locks held, and state that must
+be released once the mapping goes away should be released in
+``vm_ops->close``.
+
+The ``init`` hook, if provided, is invoked prior to the operation starting. It
+may update what is pointed to by ``vm_private_data`` and/or ``private``. If an
+error is returned, then the operation is aborted. The ``private`` field can be
+reassigned.
+
+**NOTE:** The operation may sleep between invocations of ``get``, so locks
+needed to stabilise state must be taken and released within each hook.
+
+The ``get`` handler is the key means through which the operation is
+executed. The current state of the operation is provided through ``struct
+discontig_kernel_page_state``:
+
+.. code-block:: C
+
+    struct discontig_kernel_page_state {
+        /* Map state. */
+        unsigned long start;            /* Start address of VMA. */
+        unsigned long end;              /* End address of VMA. */
+        unsigned long addr;             /* The current address to be mapped. */
+        pgoff_t pgoff;                  /* The current pgoff to be mapped. */
+        unsigned long nr_pages_mapped;  /* The number of pages mapped. */
+        unsigned long nr_pages_remain;  /* The number of pages remaining. */
+
+        /* User-defined state. */
+        void *vm_private_data;          /* VMA private data. */
+        void *private;                  /* Mapping private data. */
+
+        /* Users should not touch these, use discontig_kernel_map_*() helpers. */
+        ... internal fields ...
+    };
+
+With ``private`` being an additional user-controllable state variable,
+initialised via ``mmap_action_map_discontig_kernel_pages()``, and
+``vm_private_data`` being equal to the ``desc->private_data`` field set in
+the ``mmap_prepare()`` hook.
+
+In the ``get`` hook, the user must choose how to map kernel pages:
+
+* ``discontig_kernel_map_abort()`` - Call this to abort the operation, whatever
+  has been mapped so far will be retained, the rest of the mapping will SIGBUS
+  if accessed.
+* ``discontig_kernel_map_page()`` - Maps a single page, correctly handling
+  compound pages (if the compound page is bigger than the remaining pages in the
+  VMA, then only those pages that fit will be mapped). For a compound page, the
+  head page must be passed.
+* ``discontig_kernel_map_page_range()`` - Map an array of pages of a specified
+  size. Note that if the number of pages specified exceeds the VMA size then an
+  error will arise.
+
+If an error arises after ``init`` succeeded, the core unmaps the VMA, invoking
+``vm_ops->close`` if set, which is therefore the place to release any state
+that ``init`` established.
diff --git a/Documentation/filesystems/netfs_library.rst b/Documentation/filesystems/netfs_library.rst
index ddd799df6ce363..0c9786ffe19240 100644
--- a/Documentation/filesystems/netfs_library.rst
+++ b/Documentation/filesystems/netfs_library.rst
@@ -195,7 +195,7 @@ structure is defined::
 		struct inode inode;
 		const struct netfs_request_ops *ops;
 		struct fscache_cookie * cache;
-		loff_t remote_i_size;
+		loff_t _remote_i_size;
 		unsigned long flags;
 		...
 	};
@@ -229,11 +229,14 @@ filesystem:
    Local caching cookie, or NULL if no caching is enabled.  This field does not
    exist if fscache is disabled.
 
- * ``remote_i_size``
+ * ``_remote_i_size``
 
    The size of the file on the server.  This differs from inode->i_size if
    local modifications have been made but not yet written back.
 
+   Use netfs_read_remote_i_size() and netfs_write_remote_i_size() to access
+   this field.  Hold inode->i_lock when writing it.
+
  * ``flags``
 
    A set of flags, some of which the filesystem might be interested in:
diff --git a/Documentation/filesystems/ntfs3.rst b/Documentation/filesystems/ntfs3.rst
index 2b86a9b3a6de30..e1d86dcc3316c0 100644
--- a/Documentation/filesystems/ntfs3.rst
+++ b/Documentation/filesystems/ntfs3.rst
@@ -109,6 +109,29 @@ this table marked with no it means default is without **no**.
        Kernel. Not to be confused with NTFS ACLs. The option specified as acl
        enables support for POSIX ACLs.
 
+   * - ads
+     - Enable access to alternate data streams, i.e. named $DATA
+       attributes. A stream is addressed by appending a colon and the
+       stream name to the file name. Reading the pseudo-stream
+       ``query_streams`` returns the names of the streams attached to a
+       file, one per line::
+
+         cat file:query_streams   # list stream names
+         cat file:ads1            # read a stream
+         touch file:ads2          # create a stream on an existing file
+         rm file:ads1             # remove a stream
+
+       Creating a file together with a stream in a single call, and
+       renaming a stream, are not supported. Enabled by default.
+
+   * - nocase
+     - Perform file name lookups case-insensitively.
+
+   * - delalloc
+     - Delay block allocation for buffered writes until writeback,
+       rather than allocating as each write is issued. This lets the
+       driver allocate larger contiguous runs and reduces fragmentation.
+
 Todo list
 =========
 - Full journaling support over JBD. Currently journal replaying is supported
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
index 60880eb0c49d1a..c38f5df755bb50 100644
--- a/Documentation/filesystems/porting.rst
+++ b/Documentation/filesystems/porting.rst
@@ -348,7 +348,7 @@ simply of return 1.  Note that all actual eviction work is done by caller after
 As before, clear_inode() must be called exactly once on each call of
 ->evict_inode() (as it used to be for each call of ->delete_inode()).  Unlike
 before, if you are using inode-associated metadata buffers (i.e.
-mark_buffer_dirty_inode()), it's your responsibility to call
+mmb_mark_buffer_dirty()), it's your responsibility to call
 invalidate_inode_buffers() before clear_inode().
 
 NOTE: checking i_nlink in the beginning of ->write_inode() and bailing out
@@ -673,7 +673,7 @@ watch out, since that shortcut is no longer valid.
 they used to - they just take it exclusive.  However, ->lookup() may be
 called with parent locked shared.  Its instances must not
 
-	* use d_instantiate) and d_rehash() separately - use d_add() or
+	* use d_instantiate() and d_rehash() separately - use d_add() or
 	  d_splice_alias() instead.
 	* use d_rehash() alone - call d_add(new_dentry, NULL) instead.
 	* in the unlikely case when (read-only) access to filesystem
@@ -1203,16 +1203,16 @@ will fail-safe.
 
 ---
 
-** mandatory**
+**mandatory**
 
 lookup_one(), lookup_one_unlocked(), lookup_one_positive_unlocked() now
 take a qstr instead of a name and len.  These, not the "one_len"
 versions, should be used whenever accessing a filesystem from outside
-that filesysmtem, through a mount point - which will have a mnt_idmap.
+that filesystem, through a mount point - which will have a mnt_idmap.
 
 ---
 
-** mandatory**
+**mandatory**
 
 Functions try_lookup_one_len(), lookup_one_len(),
 lookup_one_len_unlocked() and lookup_positive_unlocked() have been
@@ -1229,7 +1229,7 @@ already been performed such as after vfs_path_parent_lookup()
 
 ---
 
-** mandatory**
+**mandatory**
 
 d_hash_and_lookup() is no longer exported or available outside the VFS.
 Use try_lookup_noperm() instead.  This adds name validation and takes
@@ -1370,7 +1370,7 @@ similar.
 
 ---
 
-** mandatory**
+**mandatory**
 
 lock_rename(), lock_rename_child(), unlock_rename() are no
 longer available.  Use start_renaming() or similar.
@@ -1409,3 +1409,16 @@ use only if you have no alternative.
 The .create inode_operation no longer receives the 'excl' arg.  It must
 always assume the file does not already exist.  If the filesystem needs
 to be involved in non-exclusive create, it should provide atomic_open.
+
+---
+
+**mandatory**
+
+All struct mnt_idmap pointers handed to filesystems are const now.
+->create(), ->mkdir(), ->mknod(), ->symlink(), ->rename(), ->setattr(),
+->getattr(), ->permission(), ->tmpfile(), ->get_acl(), ->set_acl() and
+->fileattr_set() as well as the xattr ->set() handler and the vfs_*()
+helpers take a const struct mnt_idmap *. mnt_idmap() and file_mnt_idmap()
+return one. The idmapping is immutable so nothing should have modified it
+anyway. References are taken and dropped via mnt_idmap_get() and
+mnt_idmap_put() as before, both accept a const pointer.
diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst
index c102b62023cdc1..fa7a468e8edf2f 100644
--- a/Documentation/filesystems/proc.rst
+++ b/Documentation/filesystems/proc.rst
@@ -440,7 +440,7 @@ ioctl()-based API that gives ability to flexibly and efficiently query and
 filter individual VMAs. This interface is binary and is meant for more
 efficient and easy programmatic use. `struct procmap_query`, defined in
 linux/fs.h UAPI header, serves as an input/output argument to the
-`PROCMAP_QUERY` ioctl() command. See comments in linus/fs.h UAPI header for
+`PROCMAP_QUERY` ioctl() command. See comments in linux/fs.h UAPI header for
 details on query semantics, supported flags, data returned, and general API
 usage information.
 
@@ -576,14 +576,14 @@ encoded manner. The codes are the following:
 
     ==    =============================================================
     rd    readable
-    wr    writeable
+    wr    writable
     ex    executable
     sh    shared
     mr    may read
     mw    may write
     me    may execute
     ms    may share
-    gd    stack segment growns down
+    gd    stack segment grows down
     pf    pure PFN range
     lo    pages are locked in memory
     io    memory mapped I/O area
@@ -719,7 +719,7 @@ size, in KB, that is backing the mapping up.
 
 Note that some kernel configurations do not track the precise number of times
 a page part of a larger allocation (e.g., THP) is mapped. In these
-configurations, "mapmax" might corresponds to the average number of mappings
+configurations, "mapmax" might correspond to the average number of mappings
 per page in such a larger allocation instead.
 
 1.2 Kernel data
@@ -1963,6 +1963,10 @@ For example::
   $ echo 0x7 > /proc/self/coredump_filter
   $ ./some_program
 
+If the coredump socket protocol is used a coredump server can select memory
+types to include dynamically. See COREDUMP_MEMORY_TYPES in
+include/uapi/linux/coredump.h.
+
 3.5	/proc/<pid>/mountinfo - Information about mounts
 --------------------------------------------------------
 
@@ -2008,7 +2012,7 @@ For more information on mount propagation see:
 These files provide a method to access a task's comm value. It also allows for
 a task to set its own or one of its thread siblings comm value. The comm value
 is limited in size compared to the cmdline value, so writing anything longer
-then the kernel's TASK_COMM_LEN (currently 16 chars, including the NUL
+than the kernel's TASK_COMM_LEN (currently 16 chars, including the NUL
 terminator) will result in a truncated comm value.
 
 
diff --git a/Documentation/filesystems/ramfs-rootfs-initramfs.rst b/Documentation/filesystems/ramfs-rootfs-initramfs.rst
index 165117a721ceee..4263fe06115de1 100644
--- a/Documentation/filesystems/ramfs-rootfs-initramfs.rst
+++ b/Documentation/filesystems/ramfs-rootfs-initramfs.rst
@@ -155,7 +155,7 @@ specification such as the following example::
   dir /mnt 755 0 0
   file /init initramfs/init.sh 755 0 0
 
-Run "usr/gen_init_cpio" (after the kernel build) to get a usage message
+Run "scripts/gen_init_cpio" (after the kernel build) to get a usage message
 documenting the above file format.
 
 One advantage of the configuration file is that root access is not required to
@@ -167,9 +167,9 @@ Documentation/driver-api/early-userspace/early_userspace_support.rst for more de
 The kernel does not depend on external cpio tools.  If you specify a
 directory instead of a configuration file, the kernel's build infrastructure
 creates a configuration file from that directory (usr/Makefile calls
-usr/gen_initramfs.sh), and proceeds to package up that directory
-using the config file (by feeding it to usr/gen_init_cpio, which is created
-from usr/gen_init_cpio.c).  The kernel's build-time cpio creation code is
+scripts/gen_initramfs.sh), and proceeds to package up that directory
+using the config file (by feeding it to scripts/gen_init_cpio, which is created
+from scripts/gen_init_cpio.c).  The kernel's build-time cpio creation code is
 entirely self-contained, and the kernel's boot-time extractor is also
 (obviously) self-contained.
 
@@ -313,7 +313,7 @@ the above threads) is:
 2) The cpio archive format chosen by the kernel is simpler and cleaner (and
    thus easier to create and parse) than any of the (literally dozens of)
    various tar archive formats.  The complete initramfs archive format is
-   explained in buffer-format.rst, created in usr/gen_init_cpio.c, and
+   explained in buffer-format.rst, created in scripts/gen_init_cpio.c, and
    extracted in init/initramfs.c.  All three together come to less than 26k
    total of human-readable text.
 
diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst
index e4b66af55ffba0..b52795e03303f1 100644
--- a/Documentation/filesystems/resctrl.rst
+++ b/Documentation/filesystems/resctrl.rst
@@ -236,12 +236,11 @@ with respect to allocation:
 		user can request.
 
 "bandwidth_gran":
-		The granularity in which the memory bandwidth
-		percentage is allocated. The allocated
-		b/w percentage is rounded off to the next
-		control step available on the hardware. The
-		available bandwidth control steps are:
-		min_bandwidth + N * bandwidth_gran.
+		The approximate granularity in which the memory bandwidth
+		percentage is allocated. The allocated bandwidth percentage is
+		rounded up or down to the closest control step available on the
+		hardware. The available hardware steps are no larger than this
+		value.
 
 "delay_linear":
 		Indicates if the delay scale is linear or
@@ -643,7 +642,7 @@ When monitoring is enabled all MON groups will also contain:
 	during execution of instructions summed across all logical CPUs on a
 	package for the current monitoring group.
 
-	"activity" also reports a floating point value (in Farads).  This provides
+	"activity" also reports a floating point value (in nanofarads).  This provides
 	an estimate of work done independent of the frequency that the CPUs used
 	for execution.
 
@@ -881,8 +880,10 @@ The minimum bandwidth percentage value for each cpu model is predefined
 and can be looked up through "info/MB/min_bandwidth". The bandwidth
 granularity that is allocated is also dependent on the cpu model and can
 be looked up at "info/MB/bandwidth_gran". The available bandwidth
-control steps are: min_bw + N * bw_gran. Intermediate values are rounded
-to the next control step available on the hardware.
+control steps are, approximately, min_bw + N * bw_gran.  The steps may
+appear irregular due to rounding to an exact percentage: bw_gran is the
+maximum interval between the percentage values corresponding to any two
+adjacent steps in the hardware.
 
 The bandwidth throttling is a core specific mechanism on some of Intel
 SKUs. Using a high bandwidth and a low bandwidth setting on two threads
diff --git a/Documentation/filesystems/sharedsubtree.rst b/Documentation/filesystems/sharedsubtree.rst
index 8b7dc915908377..8bae6d9a7e0479 100644
--- a/Documentation/filesystems/sharedsubtree.rst
+++ b/Documentation/filesystems/sharedsubtree.rst
@@ -564,8 +564,8 @@ f) Unmount semantics
    where 'A' is a mount mounted on mount 'B' at dentry 'b'.
 
    If mount 'B' is shared, then all most-recently-mounted mounts at dentry
-   'b' on mounts that receive propagation from mount 'B' and does not have
-   sub-mounts within them are unmounted.
+   'b' on mounts that receive propagation from mount 'B' are unmounted as
+   well, if every mount below them is also unmounted.
 
    Example: Let's say 'B1', 'B2', 'B3' are shared mounts that propagate to
    each other.
@@ -584,10 +584,18 @@ f) Unmount semantics
 
    So all 'C1', 'C2' and 'C3' should be unmounted.
 
-   If any of 'C2' or 'C3' has some child mounts, then that mount is not
-   unmounted, but all other mounts are unmounted. However if 'C1' is told
-   to be unmounted and 'C1' has some sub-mounts, the umount operation is
-   failed entirely.
+   If any of 'C2' or 'C3' has a child mount that cannot be unmounted
+   then that mount is not unmounted. But all other mounts are unmounted.
+   A child mount that is itself unmounted by the same unmount
+   propagation does not keep its parent mounted. However if 'C1' is
+   supposed to be unmounted and 'C1' has some sub-mounts, the unmount
+   fails.
+
+   A lazy umount (MNT_DETACH) takes a whole tree. Every mount of the
+   tree then propagates its unmount from its own parent as described
+   above, so the mounts that receive propagation lose the corresponding
+   trees as well. Documentation/filesystems/propagate_umount.txt has the
+   precise rules, including the ones for locked mounts.
 
 g) Clone Namespace
 
diff --git a/Documentation/filesystems/smb/ksmbd.rst b/Documentation/filesystems/smb/ksmbd.rst
index 672c5d3892ff91..728cfca1320561 100644
--- a/Documentation/filesystems/smb/ksmbd.rst
+++ b/Documentation/filesystems/smb/ksmbd.rst
@@ -82,10 +82,12 @@ Signing Update                 Supported.
 Pre-authentication integrity   Supported.
 SMB3 encryption(CCM, GCM)      Supported. (CCM/GCM128 and CCM/GCM256 supported)
 SMB direct(RDMA)               Supported.
-SMB3 Multi-channel             Partially Supported. Planned to implement
-                               replay/retry mechanisms for future.
+SMB3 Multi-channel             Supported.
 Receive Side Scaling mode      Supported.
 SMB3.1.1 POSIX extension       Supported.
+MSDFS                          Planned for future.
+Continuous Availability (CA)   Under development.
+AD/DC                          Under development.
 ACLs                           Partially Supported. only DACLs available, SACLs
                                (auditing) is planned for the future. For
                                ownership (SIDs) ksmbd generates random subauth
@@ -98,8 +100,9 @@ ACLs                           Partially Supported. only DACLs available, SACLs
                                member.
 Kerberos                       Supported.
 Durable handle v1,v2           Supported.
-Persistent handle              Planned for future.
-SMB2 notify                    Planned for future.
+Persistent handle              Under development.
+Resilient handle               Planned for future.
+SMB2 notify                    Under development.
 Sparse file support            Supported.
 DCE/RPC support                Partially Supported. a few calls(NetShareEnumAll,
                                NetServerGetInfo, SAMR, LSARPC) that are needed
@@ -113,7 +116,8 @@ ksmbd/nfsd interoperability    Planned for future. The features that ksmbd
                                support are Leases, Notify, ACLs and Share modes.
 SMB3.1.1 Compression           Supported.
 SMB3.1.1 over QUIC             Planned for future.
-Signing/Encryption over RDMA   Planned for future.
+Signing over RDMA              Under development.
+Encryption over RDMA           Supported.
 SMB3.1.1 GMAC signing support  Planned for future.
 ============================== =================================================
 
diff --git a/Documentation/filesystems/squashfs.rst b/Documentation/filesystems/squashfs.rst
index 45653b3228f972..d6397961d16ba2 100644
--- a/Documentation/filesystems/squashfs.rst
+++ b/Documentation/filesystems/squashfs.rst
@@ -177,9 +177,9 @@ or if the compressed block was larger than the uncompressed block.
 
 Inodes are packed into the metadata blocks, and are not aligned to block
 boundaries, therefore inodes overlap compressed blocks.  Inodes are identified
-by a 48-bit number which encodes the location of the compressed metadata block
-containing the inode, and the byte offset into that block where the inode is
-placed (<block, offset>).
+by a 64-bit number: the upper 48 bits encode the location of the compressed
+metadata block containing the inode, and the lower 16 bits give the byte offset
+into that block where the inode is placed (<block, offset>).
 
 To maximise compression there are different inodes for each file type
 (regular file, directory, device, etc.), the inode contents and length
diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst
index d3a93eec3945f8..1824462a437117 100644
--- a/Documentation/filesystems/vfs.rst
+++ b/Documentation/filesystems/vfs.rst
@@ -117,7 +117,7 @@ members are defined:
 		const struct fs_parameter_spec *parameters;
 		void (*kill_sb) (struct super_block *);
 		struct module *owner;
-		struct file_system_type * next;
+		struct hlist_node list;
 		struct hlist_head fs_supers;
 
 		struct lock_class_key s_lock_key;
@@ -415,33 +415,33 @@ As of kernel 2.6.22, the following members are defined:
 .. code-block:: c
 
 	struct inode_operations {
-		int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, umode_t);
+		int (*create) (const struct mnt_idmap *, struct inode *,struct dentry *, umode_t);
 		struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
 		int (*link) (struct dentry *,struct inode *,struct dentry *);
 		int (*unlink) (struct inode *,struct dentry *);
-		int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,const char *);
-		struct dentry *(*mkdir) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t);
+		int (*symlink) (const struct mnt_idmap *, struct inode *,struct dentry *,const char *);
+		struct dentry *(*mkdir) (const struct mnt_idmap *, struct inode *,struct dentry *,umode_t);
 		int (*rmdir) (struct inode *,struct dentry *);
-		int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *,umode_t,dev_t);
-		int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *,
+		int (*mknod) (const struct mnt_idmap *, struct inode *,struct dentry *,umode_t,dev_t);
+		int (*rename) (const struct mnt_idmap *, struct inode *, struct dentry *,
 			       struct inode *, struct dentry *, unsigned int);
 		int (*readlink) (struct dentry *, char __user *,int);
 		const char *(*get_link) (struct dentry *, struct inode *,
 					 struct delayed_call *);
-		int (*permission) (struct mnt_idmap *, struct inode *, int);
+		int (*permission) (const struct mnt_idmap *, struct inode *, int);
 		struct posix_acl * (*get_inode_acl)(struct inode *, int, bool);
-		int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *);
-		int (*getattr) (struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int);
+		int (*setattr) (const struct mnt_idmap *, struct dentry *, struct iattr *);
+		int (*getattr) (const struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int);
 		ssize_t (*listxattr) (struct dentry *, char *, size_t);
 		void (*update_time)(struct inode *inode, enum fs_update_time type,
 				    int flags);
 		void (*sync_lazytime)(struct inode *inode);
 		int (*atomic_open)(struct inode *, struct dentry *, struct file *,
 				   unsigned open_flag, umode_t create_mode);
-		int (*tmpfile) (struct mnt_idmap *, struct inode *, struct file *, umode_t);
-		struct posix_acl * (*get_acl)(struct mnt_idmap *, struct dentry *, int);
-	        int (*set_acl)(struct mnt_idmap *, struct dentry *, struct posix_acl *, int);
-		int (*fileattr_set)(struct mnt_idmap *idmap,
+		int (*tmpfile) (const struct mnt_idmap *, struct inode *, struct file *, umode_t);
+		struct posix_acl * (*get_acl)(const struct mnt_idmap *, struct dentry *, int);
+	        int (*set_acl)(const struct mnt_idmap *, struct dentry *, struct posix_acl *, int);
+		int (*fileattr_set)(const struct mnt_idmap *idmap,
 				    struct dentry *dentry, struct file_kattr *fa);
 		int (*fileattr_get)(struct dentry *dentry, struct file_kattr *fa);
 	        struct offset_ctx *(*get_offset_ctx)(struct inode *inode);
@@ -507,8 +507,8 @@ otherwise noted.
 	dentry before the first mkdir returns.
 
 	If there is any chance this could happen, then the new inode
-	should be d_drop()ed and attached with d_splice_alias().  The
-	returned dentry (if any) should be returned by ->mkdir().
+	should be attached with d_splice_alias().  The returned
+	dentry (if any) should be returned by ->mkdir().
 
 ``rmdir``
 	called by the rmdir(2) system call.  Only required if you want
@@ -649,8 +649,8 @@ Writeback.
 
 The first can be used independently to the others.  The VM can try to
 release clean pages in order to reuse them.  To do this it can call
-->release_folio on clean folios with the private
-flag set.  Clean pages without PagePrivate and with no external references
+->release_folio on clean folios with folio->private set. Clean pages
+without folio->private set and with no external references
 will be released without notice being given to the address_space.
 
 To achieve this functionality, pages need to be placed on an LRU with
@@ -674,7 +674,7 @@ filemap_fdatawait_range, to wait for all writeback to complete.
 
 An address_space handler may attach extra information to a page,
 typically using the 'private' field in the 'struct page'.  If such
-information is attached, the PG_Private flag should be set.  This will
+information is attached, non-NULL 'private' field will
 cause various VM routines to make extra calls into the address_space
 handler to deal with that data.
 
diff --git a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst
index 3d9233f403dbb1..14767ce9fad43f 100644
--- a/Documentation/filesystems/xfs/xfs-online-fsck-design.rst
+++ b/Documentation/filesystems/xfs/xfs-online-fsck-design.rst
@@ -1973,8 +1973,7 @@ provide loading and storing of array elements at arbitrary array indices.
 Gaps are defined to be null records, and null records are defined to be a
 sequence of all zero bytes.
 Null records are detected by calling ``xfarray_element_is_null``.
-They are created either by calling ``xfarray_unset`` to null out an existing
-record or by never storing anything to an array index.
+They are created by never storing anything to an array index.
 
 The second type of caller handles records that are not indexed by position
 and do not require multiple updates to a record.
@@ -1991,9 +1990,7 @@ The typical use case here is constructing space extent reference counts from
 reverse mapping information.
 Records can be put in the bag in any order, they can be removed from the bag
 at any time, and uniqueness of records is left to callers.
-The ``xfarray_store_anywhere`` function is used to insert a record in any
-null record slot in the bag; and the ``xfarray_unset`` function removes a
-record from the bag.
+Note: Bags are now implemented with in-memory btrees for faster access.
 
 Iterating Array Elements
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -2643,11 +2640,7 @@ generate refcount information from reverse mapping records.
       refcount record associating the block number range that we just walked to
       the size of the bag.
 
-The bag-like structure in this case is a type 2 xfarray as discussed in the
-:ref:`xfarray access patterns<xfarray_access_patterns>` section.
-Reverse mappings are added to the bag using ``xfarray_store_anywhere`` and
-removed via ``xfarray_unset``.
-Bag members are examined through ``xfarray_iter`` loops.
+The bag-like structure in this case is an in-memory btree.
 
 Case Study: Rebuilding File Fork Mapping Indices
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/Documentation/firmware-guide/acpi/apei/einj.rst b/Documentation/firmware-guide/acpi/apei/einj.rst
index 7d8435d35a18bf..70fa59bd97d8b1 100644
--- a/Documentation/firmware-guide/acpi/apei/einj.rst
+++ b/Documentation/firmware-guide/acpi/apei/einj.rst
@@ -180,8 +180,10 @@ function are specified using param1::
 	| segment |   bus  | device | function | reserved |
 	+-------------------------------------------------+
 
-Anyway, you get the idea, if there's doubt just take a look at the code
-in drivers/acpi/apei/einj.c.
+Anyway, you get the idea, if there's doubt just take a look at the code:
+drivers/acpi/apei/einj-core.c holds the core logic, while CXL-specific
+handling lives in drivers/acpi/apei/einj-cxl.c and the shared plumbing
+in drivers/acpi/apei/apei-internal.h.
 
 An ACPI 5.0 BIOS may also allow vendor-specific errors to be injected.
 In this case a file named vendor will contain identifying information
diff --git a/Documentation/gpu/amdgpu/index.rst b/Documentation/gpu/amdgpu/index.rst
index b2ab182236efb1..ba2ee732786728 100644
--- a/Documentation/gpu/amdgpu/index.rst
+++ b/Documentation/gpu/amdgpu/index.rst
@@ -23,4 +23,5 @@ Next (GCN), Radeon DNA (RDNA), and Compute DNA (CDNA) architectures.
    debugfs
    process-isolation
    amdgpu-glossary
+   ualink
    ptl
diff --git a/Documentation/gpu/amdgpu/ualink.rst b/Documentation/gpu/amdgpu/ualink.rst
new file mode 100644
index 00000000000000..e15c8621e5a1a7
--- /dev/null
+++ b/Documentation/gpu/amdgpu/ualink.rst
@@ -0,0 +1,75 @@
+==============
+UALink Support
+==============
+
+Overview
+========
+
+Connected GPUs in a pod can directly access the remove memory on another GPU
+over UALink.  Unlike RMDA, there is no copy involved; it is direct loads/stores
+over the fabric.  Shared memory can only be accessed by a remote GPU if the
+memory was exported and the importer has been authorized. For the memory to be
+shared, it must be part of a unified physical address space shared between
+nodes.  This address space is called NPA (Nework Physical Address) space.  This
+address space is partitioned between the GPUs so that each GPU has its own
+segment of the address space in which to export its memory.  Each GPU maintains
+a dedicated set of page tables for their NPA space similar to GPUVM.  Note that
+this mechanism only allows for GPU access to remote memory.  The remote memory
+is not CPU accessible.
+
+Exported memory is pinned.  This is similar to how dma-bufs in VRAM are pinned
+for P2P access.  Remote TLB shootdowns from the exporter are used when the
+exported memory is freed in order to remove access by remote GPUs.
+
+To access remote memory, the driver can map NPA addresses into its per process
+GPUVM page tables just like local memory.  Applications use opaque handles to
+represent remote memory.  GPUs in a pod communicate with eachother directly to
+exchange NPA addresses between importers and exporters.  If a node goes offline
+or is reset, their peers will clean up any remaining refrences that are lost
+when that happens.  NPA addresses are exchanged directly between the kernel
+drivers using the scale up fabric.  NPA addresses are never exposed to user
+space.
+
+On the importer, the NPA space is like another physical address space. NPA
+addresses can be used as physical addresses for GPUVM to provide GPU virtual
+addresses to the memory for processes using the GPU.
+
+On the exporter, the NPA space provides a way to expose discontiguous local
+memory as a contiguous address range for remote GPUs.  This allows the exporter
+to locally manage the pages mapped into the NPA space.
+
+Remote NPAs are managed like another device specific TTM pool similar to
+doorbells or VRAM, however they cannot be CPU mapped.
+
+
+User Interface
+==============
+Two IOCTLs are provided to export and import remote memory.
+
+Export Memory
+-------------
+To export memory, a UALINK handle must be created for an allocation that can be
+shared with another node in the pod.  To do this the exporter calls the GEM
+UALink IOCTL with the GEM handle to the buffer it wants to export.  The IOCTL
+returns a unique 128 bit handle which can be shared with the remote host.
+Calling export on the same GEM handle always returns the same UALink handle.
+The UALink handle is destroyed when the GEM object reference count reaches 0.
+
+Import Memory
+-------------
+To import remote memory, the UALink handle from the remote node must be
+converted from a UALink handle to a local GEM object which represents the local
+reference to the NPA space on the importer.  If the memory has already been
+imported, it just returns a new reference to the existing GEM object.  If not,
+the importer queries the exporter to get the NPA address.  Once it has that, the
+importer can create the GEM to represent the NPA space used by the allocation.
+The GEM object is then exported to the caller as a dma-buf. The dma-buf is
+leveraged for dynamic attachment which provides the ability to revoke access
+when necessary.
+
+
+Device to Device Communications
+===============================
+
+Devices communicate via a protocol implemented in firmware.  Mesages sent to a
+remote node generate an interrupt on that node for servicing.
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index 94cfc26aceccc3..400cd01127133e 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -118,12 +118,21 @@ fbdev Helper Functions Reference
 .. kernel-doc:: drivers/gpu/drm/drm_fb_helper.c
    :export:
 
-format Helper Functions Reference
+Draw Helper Functions Reference
 =================================
 
+Format-Conversion Helper Reference
+----------------------------------
+
 .. kernel-doc:: drivers/gpu/drm/drm_format_helper.c
    :export:
 
+Panic Helper Reference
+----------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_panic_helper.c
+   :export:
+
 Framebuffer DMA Helper Functions Reference
 ==========================================
 
diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst
index 0dd440a1494615..8988fd64369b89 100644
--- a/Documentation/gpu/drm-kms.rst
+++ b/Documentation/gpu/drm-kms.rst
@@ -421,6 +421,9 @@ Plane Panic Functions Reference
 .. kernel-doc:: include/drm/drm_panic.h
    :internal:
 
+.. kernel-doc:: drivers/gpu/drm/drm_panic_internal.h
+   :internal:
+
 .. kernel-doc:: drivers/gpu/drm/drm_panic.c
    :export:
 
diff --git a/Documentation/gpu/drm-ras.rst b/Documentation/gpu/drm-ras.rst
index 83c21853b74b60..c4ce24067d4b24 100644
--- a/Documentation/gpu/drm-ras.rst
+++ b/Documentation/gpu/drm-ras.rst
@@ -56,6 +56,11 @@ User space tools can:
   ``node-id`` and ``error-id`` as parameters.
 * Clear specific error counters with the ``clear-error-counter`` command, using both
   ``node-id`` and ``error-id`` as parameters.
+* Subscribe to the ``error-report`` multicast group to receive ``error-event``.
+* Query specific error counter threshold with the ``get-error-threshold`` command, using both
+  ``node-id`` and ``error-id`` as parameters.
+* Set specific error counter threshold with the ``set-error-threshold`` command, using
+  ``node-id``, ``error-id`` and ``error-threshold`` as parameters.
 
 YAML-based Interface
 --------------------
@@ -111,3 +116,37 @@ Example: Clear an error counter for a given node
 
     sudo ynl --family drm_ras --do clear-error-counter --json '{"node-id":0, "error-id":1}'
     None
+
+Example: Subscribe to ``error-report`` multicast group
+
+.. code-block:: bash
+
+    sudo ynl --family drm_ras --output-json --subscribe error-report
+
+.. code-block:: json
+
+    {
+        "name": "error-event",
+        "msg": {
+            "device-name": "0000:03:00.0",
+            "node-id": 1,
+            "node-name": "uncorrectable-errors",
+            "error-id": 1,
+            "error-name": "error_name1",
+            "error-value": 1
+        }
+    }
+
+Example: Query error threshold of a given counter
+
+.. code-block:: bash
+
+    sudo ynl --family drm_ras --do get-error-threshold --json '{"node-id":0, "error-id":1}'
+    {'error-id': 1, 'error-name': 'error_name1', 'error-threshold': 16}
+
+Example: Set error threshold of a given counter
+
+.. code-block:: bash
+
+    sudo ynl --family drm_ras --do set-error-threshold --json '{"node-id":0, "error-id":1, "error-threshold":8}'
+    None
diff --git a/Documentation/gpu/drm-uapi.rst b/Documentation/gpu/drm-uapi.rst
index 93df92c4ac8cf8..52255247a6db96 100644
--- a/Documentation/gpu/drm-uapi.rst
+++ b/Documentation/gpu/drm-uapi.rst
@@ -424,7 +424,7 @@ needed.
 Recovery
 --------
 
-Current implementation defines four recovery methods, out of which, drivers
+Current implementation defines several recovery methods, out of which, drivers
 can use any one, multiple or none. Method(s) of choice will be sent in the
 uevent environment as ``WEDGED=<method1>[,..,<methodN>]`` in order of less to
 more side-effects. See the section `Vendor Specific Recovery`_
@@ -434,15 +434,16 @@ method is unknown, ``WEDGED=unknown`` will be sent instead.
 Userspace consumers can parse this event and attempt recovery as per the
 following expectations.
 
-    =============== ========================================
+    =============== =========================================
     Recovery method Consumer expectations
-    =============== ========================================
+    =============== =========================================
     none            optional telemetry collection
     rebind          unbind + bind driver
     bus-reset       unbind + bus reset/re-enumeration + bind
     vendor-specific vendor specific recovery method
+    cold-reset      remove device + slot power cycle + rescan
     unknown         consumer policy
-    =============== ========================================
+    =============== =========================================
 
 No Recovery
 -----------
@@ -453,6 +454,17 @@ debug purpose in order to root cause the hang. This is useful because the first
 hang is usually the most critical one which can result in consequential hangs
 or complete wedging.
 
+Cold Reset Recovery
+-------------------
+
+When ``WEDGED=cold-reset`` is sent, it indicates that the device has
+encountered an error condition that cannot be resolved through other
+recovery methods such as driver rebind or bus reset, and requires a complete
+device power cycle to restore functionality.
+
+This method is used by devices that are plugged directly into the PCIe slot
+which supports removing the power.
+
 Vendor Specific Recovery
 ------------------------
 
@@ -516,7 +528,7 @@ Example - rebind
 
 Udev rule::
 
-    SUBSYSTEM=="drm", ENV{WEDGED}=="rebind", DEVPATH=="*/drm/card[0-9]",
+    SUBSYSTEM=="drm", ENV{WEDGED}=="rebind", DEVPATH=="*/drm/card[0-9]", \
     RUN+="/path/to/rebind.sh $env{DEVPATH}"
 
 Recovery script::
@@ -530,6 +542,77 @@ Recovery script::
     echo -n $DEVICE > $DRIVER/unbind
     echo -n $DEVICE > $DRIVER/bind
 
+Example - cold-reset
+--------------------
+
+Udev rule::
+
+    SUBSYSTEM=="drm", ENV{WEDGED}=="cold-reset", DEVPATH=="*/drm/card[0-9]", \
+    RUN+="/path/to/cold-reset.sh $env{DEVPATH}"
+
+Recovery script::
+
+    #!/bin/sh
+    die() { echo "ERROR: $*" >&2; exit 1; }
+
+    [ -n "$1" ] || die "Usage: $0 <device-path>"
+
+    PCI_DEVS=/sys/bus/pci/devices
+    PCI_SLOTS=/sys/bus/pci/slots
+
+    syspath=$(readlink -f "/sys/$1/device" 2>/dev/null || readlink -f "/sys/$1" 2>/dev/null)
+    [ -n "$syspath" ] || die "cannot resolve sysfs path for: $1"
+
+    dev=$(basename "$syspath")
+    [ -e "$PCI_DEVS/$dev" ] || die "not a PCI device: $dev"
+    echo "device : $dev"
+
+    slot=""
+    walk=$(dirname "$(readlink -f "$PCI_DEVS/$dev")")
+
+    while true; do
+        ancestor=$(basename "$walk")
+        case "$ancestor" in pci*) break ;; esac  # reached the virtual bus root
+
+        ancestor_nofn=${ancestor%.*}  # strip function: 0000:03:01.0 -> 0000:03:01
+
+        for f in "$PCI_SLOTS"/*/address; do
+            [ -f "$f" ] || continue
+            addr=$(cat "$f")
+            case "$ancestor_nofn" in
+                *"$addr") slot=$(basename "$(dirname "$f")"); break ;;
+            esac
+        done
+
+        if [ -n "$slot" ] && [ -e "$PCI_SLOTS/$slot/power" ]; then
+            echo "slot   : $slot (port $ancestor)"
+            break
+        fi
+        slot=""
+        walk=$(dirname "$walk")
+    done
+
+    [ -n "$slot" ] || die "no hotplug slot with power control found in PCIe topology"
+
+    # Cold reset: remove the device, cut slot power, restore power, rescan.
+    echo "Removing $dev..."
+    [ -e "$PCI_DEVS/$dev" ] && echo 1 > "$PCI_DEVS/$dev/remove"
+
+    echo "Powering off slot $slot..."
+    echo 0 > "$PCI_SLOTS/$slot/power"
+
+    echo "Powering on slot $slot..."
+    echo 1 > "$PCI_SLOTS/$slot/power"
+
+    echo "Rescanning PCI bus..."
+    echo 1 > /sys/bus/pci/rescan
+
+    if [ -e "$PCI_DEVS/$dev" ]; then
+        echo "Done: $dev is back online."
+    else
+        echo "WARNING: $dev did not re-appear after rescan."
+    fi
+
 Customization
 -------------
 
diff --git a/Documentation/gpu/nova/core/fsp.rst b/Documentation/gpu/nova/core/fsp.rst
index 52d618d22bb89d..60c647181da033 100644
--- a/Documentation/gpu/nova/core/fsp.rst
+++ b/Documentation/gpu/nova/core/fsp.rst
@@ -106,6 +106,8 @@ All FSP messages share a common header format consisting of two 32-bit words:
 - Bit 30: EOM (End of Message)
 - Bits 29:28: Packet sequence number
 - Bits 23:16: Source Endpoint ID
+- Bits 15:8: Destination Endpoint ID
+- Bits 3:0: MCTP header version (1)
 
 **NVDM header** (NVIDIA Vendor Defined Message):
 
diff --git a/Documentation/gpu/nova/core/pramin.rst b/Documentation/gpu/nova/core/pramin.rst
new file mode 100644
index 00000000000000..f50b052d73ba16
--- /dev/null
+++ b/Documentation/gpu/nova/core/pramin.rst
@@ -0,0 +1,128 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=========================
+PRAMIN aperture mechanism
+=========================
+
+.. note::
+   The following description is approximate and current as of the Ampere
+   family. It may change for future generations and is intended to assist in
+   understanding the driver code.
+
+Introduction
+============
+
+PRAMIN is a hardware aperture mechanism that provides CPU access to GPU Video
+RAM (VRAM) before the GPU's Memory Management Unit (MMU) and page tables are
+initialized. This 1 MiB sliding window, located at a fixed offset within BAR0,
+is essential for setting up page tables and other critical GPU data structures
+without relying on the GPU's MMU.
+
+Architecture Overview
+=====================
+
+The PRAMIN aperture mechanism is logically implemented by the GPU's PBUS (PCIe
+Bus Controller Unit) and provides a CPU-accessible window into VRAM through the
+PCIe interface::
+
+    +-----------------+    PCIe     +------------------------------+
+    |      CPU        |<----------->|           GPU                |
+    +-----------------+             |                              |
+                                    |  +----------------------+    |
+                                    |  |       PBUS           |    |
+                                    |  |  (Bus Controller)    |    |
+                                    |  |                      |    |
+                                    |  |  +--------------+ <------------ [1]
+                                    |  |  |   PRAMIN     |    |    |
+                                    |  |  |   Window     |    |    |
+                                    |  |  |   (1 MiB)    |    |    |
+                                    |  |  +--------------+    |    |
+                                    |  |         |            |    |
+                                    |  +---------|------------+    |
+                                    |            |                 |
+                                    |            v                 |
+                                    |  +----------------------+ <------- [2]
+                                    |  |       VRAM           |    |
+                                    |  |    (Several GiB)     |    |
+                                    |  |                      |    |
+                                    |  |   FB[0x0000000000]   |    |
+                                    |  |          ...         |    |
+                                    |  |   FB[0xFFFFFFFFFF]   |    |
+                                    |  +----------------------+    |
+                                    +------------------------------+
+
+    [1] Window starts at BAR0 + 0x700000.
+    [2] Program PRAMIN to any 64 KiB-aligned VRAM boundary.
+
+PBUS is responsible for, among other things, handling MMIO
+accesses to the BAR registers.
+
+PRAMIN Window Operation
+=======================
+
+The PRAMIN window provides a 1 MiB sliding aperture that can be repositioned
+over the entire VRAM address space using the ``NV_PBUS_BAR0_WINDOW`` register.
+
+Window Control Mechanism
+-------------------------
+
+::
+
+    NV_PBUS_BAR0_WINDOW Register (0x1700):
+    +-------+--------+--------------------------------------+
+    | 31:26 | 25:24  |               23:0                   |
+    | RSVD  | TARGET |            BASE_ADDR                 |
+    |       |        |        (bits 39:16 of VRAM address)  |
+    +-------+--------+--------------------------------------+
+
+    The 24-bit BASE_ADDR field encodes bits [39:16] of the target VRAM address,
+    providing 40-bit (1 TiB) address space coverage with 64 KiB alignment.
+
+    TARGET field (bits 25:24):
+    - 0x0: VRAM (Video Memory)
+    - 0x1: Reserved (unused)
+    - 0x2: SYS_MEM_COH (Coherent System Memory)
+    - 0x3: SYS_MEM_NONCOH (Non-coherent System Memory)
+
+.. note::
+   Nova only uses TARGET=VRAM (0x0) for video memory access. The SYS_MEM
+   target values are documented here for hardware completeness but are
+   not used by the driver.
+
+64 KiB Alignment Requirement
+----------------------------
+
+The PRAMIN window must be aligned to 64 KiB boundaries in VRAM. This is enforced
+by the ``BASE_ADDR`` field representing bits [39:16] of the target address::
+
+    VRAM Address Calculation:
+    actual_vram_addr = (BASE_ADDR << 16) + pramin_offset
+    Where:
+    - BASE_ADDR: 24-bit value from NV_PBUS_BAR0_WINDOW[23:0]
+    - pramin_offset: 20-bit offset within the PRAMIN window [0x00000-0xFFFFF]
+
+    Example Window Positioning:
+    +---------------------------------------------------------+
+    |                    VRAM Space                           |
+    |                                                         |
+    |  0x0000000000 +-----------------+ <-- 64 KiB aligned    |
+    |               | PRAMIN Window   |                       |
+    |               |    (1 MiB)      |                       |
+    |  0x00000FFFFF +-----------------+                       |
+    |                                                         |
+    |       |              ^                                  |
+    |       |              | Window can slide                 |
+    |       v              | to any 64 KiB-aligned boundary   |
+    |                                                         |
+    |  0x0123400000 +-----------------+ <-- 64 KiB aligned    |
+    |               | PRAMIN Window   |                       |
+    |               |    (1 MiB)      |                       |
+    |  0x01234FFFFF +-----------------+                       |
+    |                                                         |
+    |                       ...                               |
+    |                                                         |
+    |  0xFFFFF00000 +-----------------+ <-- 64 KiB aligned    |
+    |               | PRAMIN Window   |                       |
+    |               |    (1 MiB)      |                       |
+    |  0xFFFFFFFFFF +-----------------+                       |
+    +---------------------------------------------------------+
diff --git a/Documentation/gpu/nova/core/todo.rst b/Documentation/gpu/nova/core/todo.rst
index d5130b2b08fb72..a01c362b1be066 100644
--- a/Documentation/gpu/nova/core/todo.rst
+++ b/Documentation/gpu/nova/core/todo.rst
@@ -33,7 +33,7 @@ A good example from nova-core would be the ``Chipset`` enum type, which defines
 the value ``AD102``. When probing the GPU the value ``0x192`` can be read from a
 certain register indication the chipset AD102. Hence, the enum value ``AD102``
 should be derived from the number ``0x192``. Currently, nova-core uses a custom
-implementation (``Chipset::from_u32`` for this.
+implementation (``Chipset::from_u32``) for this.
 
 Instead, it would be desirable to have something like the ``FromPrimitive``
 trait [1] from the num crate.
diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst
index 2afa58e8f08de2..59b2062384985e 100644
--- a/Documentation/gpu/nova/index.rst
+++ b/Documentation/gpu/nova/index.rst
@@ -34,3 +34,4 @@ vGPU manager VFIO driver and the nova-drm driver.
    core/fwsec
    core/falcon
    core/tlv
+   core/pramin
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index 14cf37590fc7e0..a437ae5c71f473 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -29,21 +29,6 @@ refactorings already and are an expert in the specific area
 Subsystem-wide refactorings
 ===========================
 
-Open-code drm_simple_encoder_init()
------------------------------------
-
-The helper drm_simple_encoder_init() was supposed to simplify encoder
-initialization. Instead it only added an intermediate layer between atomic
-modesetting and the DRM driver.
-
-The task here is to remove drm_simple_encoder_init(). Search for a driver
-that calls drm_simple_encoder_init() and inline the helper. The driver will
-also need its own instance of drm_encoder_funcs.
-
-Contact: Thomas Zimmermann, respective driver maintainer
-
-Level: Easy
-
 Replace struct drm_simple_display_pipe with regular atomic helpers
 ------------------------------------------------------------------
 
@@ -499,21 +484,30 @@ Contact: Douglas Anderson <dianders@chromium.org>
 
 Level: Starter
 
-Remove devm_drm_put_bridge()
-----------------------------
+Remove panel_bridge API functions
+---------------------------------
 
-Due to how the panel bridge handles the drm_bridge object lifetime, special
-care must be taken to dispose of the drm_bridge object when the
-panel_bridge is removed. This is currently managed using
-devm_drm_put_bridge(), but that is an unsafe, temporary workaround. To fix
-that, the DRM panel lifetime needs to be reworked. After the rework is
-done, remove devm_drm_put_bridge() and the TODO in
-drm_panel_bridge_remove().
+Now that a drm_bridge is embedded into every drm_panel, the panel_bridge is
+not needed anymore. Its APIs are just accessing the embedded drm_bridge
+transparently, and the are deprecated.
 
-Contact: Maxime Ripard <mripard@kernel.org>,
-         Luca Ceresoli <luca.ceresoli@bootlin.com>
+The funcions to remove are:
+ - drm_panel_bridge_add_typed()
+ - devm_drm_of_get_bridge()
+ - drmm_of_get_bridge()
+ - drm_panel_bridge_remove()
+ - drmm_drm_panel_bridge_release()
+ - devm_drm_panel_bridge_release()
+ - their direct or indirect callers
 
-Level: Intermediate
+Convert users of those functions and eventually remove them when unused, or
+make them static drm_panel.c functions when used internally by the
+``drm_panel.c`` code.
+
+Contact: Luca Ceresoli <luca.ceresoli@bootlin.com>
+         Maxime Ripard <mripard@kernel.org>,
+
+Level: Easy
 
 Convert users of of_drm_find_bridge() to of_drm_find_and_get_bridge()
 ---------------------------------------------------------------------
diff --git a/Documentation/gpu/xe/index.rst b/Documentation/gpu/xe/index.rst
index 665c0e93601c50..0247a255f7e65a 100644
--- a/Documentation/gpu/xe/index.rst
+++ b/Documentation/gpu/xe/index.rst
@@ -35,3 +35,4 @@ The display, or :ref:`drm-kms`, support for drm/xe is provided by
    xe-drm-usage-stats.rst
    xe_configfs
    xe_gt_stats
+   xe_sigid
diff --git a/Documentation/gpu/xe/xe_migrate.rst b/Documentation/gpu/xe/xe_migrate.rst
index f92faec0ac9431..d297ee53a5829f 100644
--- a/Documentation/gpu/xe/xe_migrate.rst
+++ b/Documentation/gpu/xe/xe_migrate.rst
@@ -6,3 +6,6 @@ Migrate Layer
 
 .. kernel-doc:: drivers/gpu/drm/xe/xe_migrate_doc.h
    :doc: Migrate Layer
+
+.. kernel-doc:: drivers/gpu/drm/xe/xe_migrate.c
+   :doc: ULLS (Ultra Low Latency Submission) for migration jobs
diff --git a/Documentation/gpu/xe/xe_sigid.rst b/Documentation/gpu/xe/xe_sigid.rst
new file mode 100644
index 00000000000000..45d84a62f185b4
--- /dev/null
+++ b/Documentation/gpu/xe/xe_sigid.rst
@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+========
+Xe SIGID
+========
+
+.. kernel-doc:: drivers/gpu/drm/xe/abi/xe_sigid_abi.h
+   :doc: Xe Error Signatures (SIGID)
+
+Signature Identifiers
+=====================
+
+.. kernel-doc:: drivers/gpu/drm/xe/abi/xe_sigid_abi.h
+   :internal:
diff --git a/Documentation/hwmon/arctic_fan_controller.rst b/Documentation/hwmon/arctic_fan_controller.rst
index b5be88ae464d7d..60ac09a9cea3a9 100644
--- a/Documentation/hwmon/arctic_fan_controller.rst
+++ b/Documentation/hwmon/arctic_fan_controller.rst
@@ -1,4 +1,4 @@
-.. SPDX-License-Identifier: GPL-2.0-or-later
+.. SPDX-License-Identifier: GPL-2.0-or-later OR BSD-2-Clause
 
 Kernel driver arctic_fan_controller
 =====================================
@@ -29,18 +29,16 @@ Usage notes
 Since it is a USB device, hotplug is supported. The device is autodetected.
 
 The device does not support GET_REPORT, so the driver cannot read back the
-current hardware PWM state at probe time. The cached PWM values (readable
-via pwm[1-10]) start at 0 and reflect only values that have been
-successfully written. Because each OUT report carries all 10 channel values,
-writing a single channel also sends the cached values for all other channels.
-Users should set all channels to the desired values before relying on the
-cached state.
-
-On system suspend, the device may lose power and reset its PWM channels to
-hardware defaults. The driver clears its cached duty values on resume so
-that reads reflect the unknown hardware state rather than stale pre-suspend
-values. Userspace is responsible for re-applying the desired duty cycles
-after resume.
+current hardware PWM state at probe time. Each OUT report carries all 10
+channels, so pwm[1-10] is a host cache. It starts at the MCU factory
+default of 40% (sysfs 102). After a successful write, the cache reflects
+that value. Users should set all channels to the desired values before
+relying on the cached state.
+
+On system suspend, the device may lose power and reset PWM to the factory
+default. The driver restores the cache to 40% on resume. If the device
+kept power across suspend, userspace should re-apply the desired duty
+cycles.
 
 Sysfs entries
 -------------
@@ -51,6 +49,6 @@ pwm[1-10]        PWM duty cycle (0-255). Write: sends an OUT report setting the
                  duty cycle (scaled from 0-255 to 0-100% for the device);
                  the cached value is updated only after the device ACKs the
                  command with a success status. Read: returns the last
-                 successfully written value; initialized to 0 at driver load
-                 and after resume (hardware state unknown).
+                 successfully written value; initialized to 102 (40%) at
+                 driver load and after resume (MCU factory default).
 ================ ==============================================================
diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst
index 2c68a343a2edde..7032c40731d900 100644
--- a/Documentation/hwmon/asus_ec_sensors.rst
+++ b/Documentation/hwmon/asus_ec_sensors.rst
@@ -45,6 +45,7 @@ Supported boards:
  * ROG STRIX X570-E GAMING WIFI II
  * ROG STRIX X570-F GAMING
  * ROG STRIX X570-I GAMING
+ * ROG STRIX X670E-A GAMING WIFI
  * ROG STRIX X670E-E GAMING WIFI
  * ROG STRIX X670E-I GAMING WIFI
  * ROG STRIX X870-F GAMING WIFI
@@ -54,6 +55,7 @@ Supported boards:
  * ROG STRIX X870E-H GAMING WIFI7
  * ROG STRIX Z390-E GAMING
  * ROG STRIX Z390-F GAMING
+ * ROG STRIX Z490-A GAMING
  * ROG STRIX Z490-F GAMING
  * ROG STRIX Z690-A GAMING WIFI D4
  * ROG STRIX Z690-E GAMING WIFI
diff --git a/Documentation/hwmon/asus_rog_ryujin.rst b/Documentation/hwmon/asus_rog_ryujin.rst
index b0d7ce8dd921f0..3ebab230e8cafa 100644
--- a/Documentation/hwmon/asus_rog_ryujin.rst
+++ b/Documentation/hwmon/asus_rog_ryujin.rst
@@ -6,6 +6,7 @@ Kernel driver asus_rog_ryujin
 Supported devices:
 
 * ASUS ROG RYUJIN II 360
+* ASUS ROG RYUJIN III 360
 * ASUS ROG RYUJIN III EXTREME
 * ASUS ROG RYUJIN III EVA EDITION
 * ASUS ROG RYUJIN III WHITE EDITION
diff --git a/Documentation/hwmon/axiado-pwm-fan.rst b/Documentation/hwmon/axiado-pwm-fan.rst
new file mode 100644
index 00000000000000..d110e170b0df27
--- /dev/null
+++ b/Documentation/hwmon/axiado-pwm-fan.rst
@@ -0,0 +1,38 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver axiado-pwm-fan
+============================
+
+Supported chips:
+
+  * Axiado AX3000
+  * Axiado AX3005
+
+    Datasheet: Not publicly available
+
+Author:
+
+  * Petar Stepanovic <pstepanovic@axiado.com>
+
+Description
+-----------
+
+The Axiado AX3000 and AX3005 PWM fan controllers measure the rotational
+speed of one fan using a hardware tachometer block. Fan speed is controlled
+through a PWM signal supplied by an external PWM controller.
+
+The number of tachometer pulses generated per fan revolution is configured
+through the ``pulses-per-revolution`` devicetree property. If the property
+is not specified, the driver uses two pulses per revolution.
+
+Sysfs attributes
+----------------
+
+The driver provides the following standard hwmon attributes:
+
+=============== ====== =====================================================
+fan1_input      RO     Fan speed in revolutions per minute (RPM).
+
+pwm1            RW     Relative PWM control value from 0 to 255. A value of
+                       255 selects the maximum PWM duty cycle.
+=============== ====== =====================================================
diff --git a/Documentation/hwmon/axiado-tsadc.rst b/Documentation/hwmon/axiado-tsadc.rst
new file mode 100644
index 00000000000000..b8fd0aeacc1f8d
--- /dev/null
+++ b/Documentation/hwmon/axiado-tsadc.rst
@@ -0,0 +1,41 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver axiado-tsadc
+==========================
+
+Supported chips:
+
+  * Axiado AX3000 and AX3005 on-chip temperature sensor (TSADC)
+
+    Prefix: 'axiado_tsadc'
+
+    Addresses scanned: -
+
+    Datasheet: Not publicly available
+
+Authors:
+
+  - Petar Stepanovic <pstepanovic@axiado.com>
+
+Description
+-----------
+
+This driver supports the Temperature Sensor ADC (TSADC) block found on Axiado
+AX3000 and AX3005 SoCs. The block monitors the on-die temperature of the SoC.
+
+An SoC may instantiate several TSADC blocks at different locations on the die.
+Each block is described by its own device tree node and is registered as a
+separate hwmon device with a single temperature channel.
+
+The sensor is configured for continuous conversion at probe time, averaging
+16 samples per conversion. The averaged 12-bit ADC code is converted to
+millidegrees Celsius using a lookup table with linear interpolation between
+entries. The table covers -40 to +125 degrees Celsius; readings outside that
+range are clamped to the nearest supported temperature.
+
+sysfs entries
+-------------
+
+======================= =======================================================
+temp1_input             Die temperature in millidegrees Celsius (read-only)
+======================= =======================================================
diff --git a/Documentation/hwmon/honor-fmi.rst b/Documentation/hwmon/honor-fmi.rst
new file mode 100644
index 00000000000000..a42a1dd532e86b
--- /dev/null
+++ b/Documentation/hwmon/honor-fmi.rst
@@ -0,0 +1,32 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+Kernel driver honor-fmi
+=======================
+
+Supported systems:
+
+  * HONOR FMI-XX
+
+Author: Nikita Dubrovskih <testname142@gmail.com>
+
+Description
+-----------
+
+The driver provides read-only monitoring of the fan speed on the HONOR FMI-XX.
+The system firmware implements a ``GFNS`` ACPI method which returns the speed
+of one of two firmware fan channels in RPM. Embedded Controller access and
+serialization are handled by the firmware method.
+
+The driver does not expose fan control or direct Embedded Controller access.
+
+Sysfs entries
+-------------
+
+The following attributes are supported:
+
+======================= ======= =============================================
+Name                    Perm    Description
+======================= ======= =============================================
+``fan1_input``          RO      Fan channel 0 speed in RPM
+``fan2_input``          RO      Fan channel 1 speed in RPM
+======================= ======= =============================================
diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst
index 9955a525436ae6..199ca79621e94a 100644
--- a/Documentation/hwmon/index.rst
+++ b/Documentation/hwmon/index.rst
@@ -51,6 +51,8 @@ Hardware Monitoring Kernel Drivers
    asus_ec_sensors
    asus_rog_ryujin
    asus_wmi_sensors
+   axiado-pwm-fan
+   axiado-tsadc
    bcm54140
    bel-pfe
    bpa-rs600
@@ -90,6 +92,7 @@ Hardware Monitoring Kernel Drivers
    gxp-fan-ctrl
    hac300s
    hih6130
+   honor-fmi
    hp-wmi-sensors
    hs3001
    htu31
@@ -165,6 +168,7 @@ Hardware Monitoring Kernel Drivers
    max197
    max20730
    max20751
+   max20826
    max20830
    max20860a
    max31722
@@ -186,6 +190,7 @@ Hardware Monitoring Kernel Drivers
    mcp3021
    mcp9982
    menf21bmc
+   minisforum-um780xtx
    mlxreg-fan
    mp2856
    mp2869
@@ -263,6 +268,7 @@ Hardware Monitoring Kernel Drivers
    sy7636a-hwmon
    tc654
    tc74
+   tda38740
    thmc50
    tmp102
    tmp103
diff --git a/Documentation/hwmon/it87.rst b/Documentation/hwmon/it87.rst
index fc1c90b023ae6e..c33ba8a0749a8a 100644
--- a/Documentation/hwmon/it87.rst
+++ b/Documentation/hwmon/it87.rst
@@ -11,6 +11,14 @@ Supported chips:
 
     Datasheet: Not publicly available
 
+  * IT8613E
+
+    Prefix: 'it8613'
+
+    Addresses scanned: from Super I/O config space (8 I/O ports)
+
+    Datasheet: Not publicly available
+
   * IT8620E
 
     Prefix: 'it8620'
diff --git a/Documentation/hwmon/k10temp.rst b/Documentation/hwmon/k10temp.rst
index 91b99adc6c487f..9680c4f3383c36 100644
--- a/Documentation/hwmon/k10temp.rst
+++ b/Documentation/hwmon/k10temp.rst
@@ -28,8 +28,6 @@ Supported chips:
 
 * AMD Family 17h processors: "Zen", "Zen 2"
 
-* AMD Family 18h processors: "Hygon Dhyana"
-
 * AMD Family 19h processors: "Zen 3"
 
   Prefix: 'k10temp'
@@ -128,7 +126,7 @@ it.
 Models from 17h family report relative temperature, the driver aims to
 compensate and report the real temperature.
 
-On Family 17h and Family 18h CPUs, additional temperature sensors may report
-Core Complex Die (CCD) temperatures. Up to 8 such temperatures are reported
-as temp{3..10}_input, labeled Tccd{1..8}. Actual support depends on the CPU
+On Family 17h CPUs, additional temperature sensors may report Core Complex
+Die (CCD) temperatures. Up to 8 such temperatures are reported as
+temp{3..10}_input, labeled Tccd{1..8}. Actual support depends on the CPU
 variant.
diff --git a/Documentation/hwmon/lm63.rst b/Documentation/hwmon/lm63.rst
index 9e27367d74058f..c40bd0dfe591c4 100644
--- a/Documentation/hwmon/lm63.rst
+++ b/Documentation/hwmon/lm63.rst
@@ -33,6 +33,16 @@ Supported chips:
 
 	       http://www.national.com/pf/LM/LM96163.html
 
+  * Sensylink CTF2301
+
+    Prefix: 'ctf2301'
+
+    Addresses scanned: none
+
+    Datasheet: https://www.sensylink.com/upload/1/net.sensylink.portal/1689557281035.pdf
+
+    Register description: https://github.com/TroyMitchell911/ctf2301-datasheet
+
 
 Author: Jean Delvare <jdelvare@suse.de>
 
@@ -62,6 +72,8 @@ value have to be masked out. The value is still 16 bit in width.
 
 All temperature values are given in degrees Celsius. Resolution is 1.0
 degree for the local temperature, 0.125 degree for the remote temperature.
+The CTF2301 local temperature input and limit have a resolution of 0.0625
+degree.
 
 The fan speed is measured using a tachometer. Contrary to most chips which
 store the value in an 8-bit register and have a selectable clock divider
@@ -93,3 +105,13 @@ support these GPIO lines at present.
 The LM96163 is an enhanced version of LM63 with improved temperature accuracy
 and better PWM resolution. For LM96163, the external temperature sensor type is
 configurable as CPU embedded diode(1) or 3904 transistor(2).
+
+The CTF2301 is register-compatible with the LM63 family and provides 12 fan
+control lookup table entries. It supports 8-bit PWM resolution when configured
+for a 22.5 kHz PWM frequency.
+
+Device tree nodes may declare ``#cooling-cells`` to register the fan controller
+with the thermal framework. The cooling state is mapped to the existing
+``pwm1`` range from 0 to 255. The driver keeps the current automatic or manual
+fan control mode during probe and switches to manual control when the thermal
+framework first requests a cooling state.
diff --git a/Documentation/hwmon/max20826.rst b/Documentation/hwmon/max20826.rst
new file mode 100644
index 00000000000000..bcabbc64c0398a
--- /dev/null
+++ b/Documentation/hwmon/max20826.rst
@@ -0,0 +1,139 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver max20826
+======================
+
+Supported chips:
+
+  * Analog Devices MAX20826
+
+    Prefix: 'max20826'
+
+    Addresses scanned: -
+
+    Datasheet: Datasheet is not publicly available.
+
+  * Analog Devices MAX20855B
+
+    Prefix: 'max20855b'
+
+    Addresses scanned: -
+
+    Datasheet: Datasheet is not publicly available.
+
+  * Analog Devices MAX20908
+
+    Prefix: 'max20908'
+
+    Addresses scanned: -
+
+    Datasheet: Datasheet is not publicly available.
+
+  * Analog Devices MAX20912
+
+    Prefix: 'max20912'
+
+    Addresses scanned: -
+
+    Datasheet: Datasheet is not publicly available.
+
+  * Analog Devices MAX20916
+
+    Prefix: 'max20916'
+
+    Addresses scanned: -
+
+    Datasheet: Datasheet is not publicly available.
+
+Author:
+
+  - Nuno Sá <nuno.sa@analog.com>
+
+
+Description
+-----------
+
+This driver supports hardware monitoring for Analog Devices MAX20826,
+MAX20855B, MAX20908, MAX20912, and MAX20916 multiphase voltage regulator
+controllers with PMBus interface.
+
+The devices are dual-loop, multiphase controllers. Depending on the device and
+configuration, the high-speed processor voltage-control interface can be
+Nvidia PWMVID, Intel SVID, AMD SVI3, or AVSBus. PMBus is used for monitoring,
+configuration, status, and fault reporting.
+
+The driver detects whether the device uses PMBus page mode or direct address
+mode. In direct address mode, rail B is accessed at the rail A I2C address plus
+one. If rail B is present, the driver exposes a second PMBus page.
+
+The driver detects the active number of phases and exposes per-phase input and
+output current attributes through the PMBus virtual phase support.
+
+Usage Notes
+-----------
+
+This driver does not auto-detect devices. You will have to instantiate the
+devices explicitly. Please see Documentation/i2c/instantiating-devices.rst for
+details.
+
+The optional ``avren`` and ``bvren`` GPIOs may be provided to control the rail A
+and rail B hardware enable pins. If regulator support is enabled, the rails are
+also registered through the PMBus regulator framework.
+
+The source of the output voltage of each rail, i.e. either the high-speed
+processor voltage-control interface or PMBus, is normally selected once, before
+the processor boots. Hence it is described in firmware with the
+``adi,rail-a-high-speed`` and ``adi,rail-b-high-speed`` properties. If the
+property is not given for a rail, PMBus controls its output voltage.
+
+Sysfs entries
+-------------
+
+The following attributes are supported. Limits, alarms, and per-phase entries
+are exposed depending on device capabilities, rail configuration, and detected
+phase count.
+
+=========================== ================================================
+in1_label                   "vin"
+in1_input                   Measured input voltage
+in1_alarm                   Input voltage alarm
+in[2-3]_label               "vout[1-2]"
+in[2-3]_input               Measured output voltage
+in[2-3]_alarm               Output voltage alarm
+currX_label                 "iinN", "iinN.P", "ioutN", or "ioutN.P"
+currX_input                 Measured input/output current
+currX_alarm                 Current alarm
+powerX_label                "pinN" or "poutN"
+powerX_input                Measured input/output power
+temp[1-2]_input             Measured temperature
+temp[1-2]_alarm             Temperature alarm
+=========================== ================================================
+
+Notes
+-----
+
+``N`` is the rail number, starting at 1. ``P`` is the phase number, starting at
+0. The exact ``currX`` indices depend on the number of present rails and on the
+detected phase count.
+
+Debugfs entries
+---------------
+
+In addition to the entries provided by the PMBus core, the following entries are
+available for debug purposes only. They are not part of any ABI and may change
+or go away at any time.
+
+=========================== ================================================
+in[2-3]_high_speed_en       Enable high-speed voltage-control interface
+in_high_speed_bus           Active high-speed voltage-control interface
+=========================== ================================================
+
+``in[2-3]_high_speed_en`` is a per-rail read/write entry matching the standard
+hwmon voltage channels for the output rails. ``in2_high_speed_en`` controls rail
+A / ``vout1``. ``in3_high_speed_en`` controls rail B / ``vout2`` and is only
+present if rail B is detected. Writing 1 selects the high-speed voltage-control
+interface for that rail; writing 0 selects PMBus voltage control.
+
+``in_high_speed_bus`` is a read-only entry reporting the high-speed interface
+used by the device, for example ``Nvidia PWMVID``, ``Intel SVID``, ``AMD SVI3``,
+or ``AVSBus``.
diff --git a/Documentation/hwmon/max34440.rst b/Documentation/hwmon/max34440.rst
index 866f22b88e9b36..8ce10aa425d5db 100644
--- a/Documentation/hwmon/max34440.rst
+++ b/Documentation/hwmon/max34440.rst
@@ -27,6 +27,14 @@ Supported chips:
 
     Datasheet: -
 
+  * ADI ADPM12300
+
+    Prefixes: 'adpm12300'
+
+    Addresses scanned: -
+
+    Datasheet: -
+
   * Maxim MAX34440
 
     Prefixes: 'max34440'
@@ -105,11 +113,11 @@ This driver supports multiple devices: hardware monitoring for Maxim MAX34440
 PMBus 6-Channel Power-Supply Manager, MAX34441 PMBus 5-Channel Power-Supply
 Manager and Intelligent Fan Controller, and MAX34446 PMBus Power-Supply Data
 Logger; PMBus Voltage Monitor and Sequencers for MAX34451, MAX34452, MAX34460,
-and MAX34461; PMBus DC/DC Power Module ADPM12160, ADPM12200, and ADPM12250. The
-MAX34451 and MAX34452 support monitoring voltage or current of 16 channels based
-on GIN pins. The MAX34460 supports 12 voltage channels, and the MAX34461 supports
-16 voltage channels. The ADPM12160, ADPM12200, and ADPM12250 also monitor both
-input and output of voltage and current.
+and MAX34461; PMBus DC/DC Power Module ADPM12160, ADPM12200, ADPM12250 and
+ADPM12300. The MAX34451 and MAX34452 support monitoring voltage or current of 16
+channels based on GIN pins. The MAX34460 supports 12 voltage channels, and the
+MAX34461 supports 16 voltage channels. The ADPM12160, ADPM12200, ADPM12250 and
+ADPM12300 also monitor both input and output of voltage and current.
 
 The driver is a client driver to the core PMBus driver. Please see
 Documentation/hwmon/pmbus.rst for details on PMBus client drivers.
@@ -167,8 +175,8 @@ in[1-6]_reset_history	Write any value to reset history.
 .. note::
 
     - MAX34446 only supports in[1-4].
-    - ADPM12160, ADPM12200, and ADPM12250 only supports in[1-2]. Label is "vin1"
-      and "vout1" respectively.
+    - ADPM12160, ADPM12200, ADPM12250, and ADPM12300 only supports in[1-2].
+      Label is "vin1" and "vout1" respectively.
 
 Curr
 ~~~~
@@ -190,9 +198,9 @@ curr[1-6]_reset_history	Write any value to reset history.
 
     - in6 and curr6 attributes only exist for MAX34440.
     - MAX34446 only supports curr[1-4].
-    - For ADPM12160, ADPM12200, and ADPM12250, curr[1] is "iin1"
+    - For ADPM12160, ADPM12200, ADPM12250, and ADPM12300, curr[1] is "iin1"
     - For ADPM12160, and ADPM12200 curr[2-6] are "iout[1-5]".
-    - For ADPM12250, curr[2-4] are "iout[1-3]".
+    - For ADPM12250, and ADPM12300 curr[2-4] are "iout[1-3]".
 
 Power
 ~~~~~
@@ -228,7 +236,7 @@ temp[1-8]_reset_history	Write any value to reset history.
 .. note::
    - temp7 and temp8 attributes only exist for MAX34440.
    - MAX34446 only supports temp[1-3].
-   - ADPM12160, ADPM12200, and ADPM12250 only supports temp[1].
+   - ADPM12160, ADPM12200, ADPM12250, and ADPM12300 only supports temp[1].
 
 
 .. note::
diff --git a/Documentation/hwmon/minisforum-um780xtx.rst b/Documentation/hwmon/minisforum-um780xtx.rst
new file mode 100644
index 00000000000000..8237d525a9c7d3
--- /dev/null
+++ b/Documentation/hwmon/minisforum-um780xtx.rst
@@ -0,0 +1,68 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+====================================
+Kernel driver minisforum-um780xtx
+====================================
+
+Supported systems:
+
+  * Minisforum UM780 XTX
+
+    * DMI product name: ``Venus series``
+    * Mainboard: ``F7BSD``, revision ``1.1``
+    * BIOS version: ``1.06``
+
+Author: Sebastián Peyrott <speyrott@gmail.com>
+
+Description
+-----------
+
+This driver exposes hardware monitoring and fan-control data cached by the
+IT5571E embedded controller. It uses the ACPI EC transport and only binds to
+the exact system and firmware identity listed above.
+
+The two temperature channels are the values used by the EC's fan-control
+loops. Their physical sensor placement is not known. The CPU channel is the
+filtered AMD SB-TSI temperature, while the system channel is an external
+thermistor input.
+
+The fan tachometers are read through OEM EC commands. Since each byte is
+returned by a separate command, the driver uses a high-low-high sequence and
+retries if the high byte changes.
+
+Sysfs entries
+-------------
+
+==========================  ==============================================
+``temp1_input``             CPU-fan control temperature
+``temp2_input``             System-fan control temperature
+``fan1_input``              CPU fan speed in RPM
+``fan2_input``              System fan speed in RPM
+``pwm1_enable``             CPU profile: 2 is OEM B1, 3 is OEM B2
+``pwm2_auto_point1_temp``   Off-to-low system-fan transition temperature
+``pwm2_auto_point2_temp``   Low-to-high system-fan transition temperature
+==========================  ==============================================
+
+CPU fan profiles
+----------------
+
+Values 2 and 3 of ``pwm1_enable`` select the two complete automatic profiles
+implemented by the firmware. Writing either value reloads the whole CPU fan
+curve, including firmware state which is not visible through the ACPI EC
+window. Other values are rejected.
+
+System fan thresholds
+---------------------
+
+The two writable system-fan temperatures are rounded to whole degrees Celsius
+and clamped to preserve strict ordering between both visible thresholds and
+the firmware's internal third threshold. The third threshold does not select
+a new PWM target and is therefore not exposed as another auto point.
+
+Resume state preservation
+-------------------------
+
+Fan settings are held in EC RAM. On the supported firmware, initialization
+during s2idle resume reloads the factory system-fan curve. The driver retains
+the last coherent CPU profile and system-fan thresholds selected through its
+interface and restores them synchronously at resume.
diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst
index 45eec9dd349aaa..c47b0709c5e7cf 100644
--- a/Documentation/hwmon/nct6683.rst
+++ b/Documentation/hwmon/nct6683.rst
@@ -67,6 +67,10 @@ ASRock X670E			NCT6686D EC firmware version 1.0 build 05/19/22
 ASRock B650 Steel Legend WiFi	NCT6686D EC firmware version 1.0 build 11/09/23
 ASRock Z590 Taichi		NCT6686D EC firmware version 1.0 build 01/25/21
 MSI B550			NCT6687D EC firmware version 1.0 build 05/07/20
+MSI B550M			NCT6687D EC firmware version 1.0 build 05/07/20
+MSI B650			NCT6687D EC firmware version 0.0 build 09/27/22
+MSI B650M			NCT6687D EC firmware version 0.0 build 09/27/22
+MSI B850M			NCT6687D EC firmware version 0.0 build 03/10/25
 MSI X670-P			NCT6687D EC firmware version 0.0 build 09/27/22
 MSI X870E			NCT6687D EC firmware version 0.0 build 11/13/24
 ===============================	===============================================
diff --git a/Documentation/hwmon/nct6775.rst b/Documentation/hwmon/nct6775.rst
index 41b21bed7e278d..d2dedade0606a4 100644
--- a/Documentation/hwmon/nct6775.rst
+++ b/Documentation/hwmon/nct6775.rst
@@ -16,6 +16,38 @@ Supported chips:
 
     Datasheet: Available from the Nuvoton web site
 
+  * Nuvoton NCT6112D/NCT6114D/NCT6116D
+
+    Prefix: 'nct6116'
+
+    Addresses scanned: ISA address retrieved from Super I/O registers
+
+    Datasheet: Available from the Nuvoton web site
+
+  * Nuvoton NCT6122D/NCT6126D
+
+    Prefix: 'nct6126'
+
+    Addresses scanned: ISA address retrieved from Super I/O registers
+
+    Datasheet: Available from the Nuvoton web site
+
+  * Nuvoton NCT6112D/NCT6114D/NCT6116D
+
+    Prefix: 'nct6116'
+
+    Addresses scanned: ISA address retrieved from Super I/O registers
+
+    Datasheet: Available from the Nuvoton web site
+
+  * Nuvoton NCT6122D/NCT6126D
+
+    Prefix: 'nct6126'
+
+    Addresses scanned: ISA address retrieved from Super I/O registers
+
+    Datasheet: Available from the Nuvoton web site
+
   * Nuvoton NCT5572D/NCT6771F/NCT6772F/NCT6775F/W83677HG-I
 
     Prefix: 'nct6775'
diff --git a/Documentation/hwmon/sht4x.rst b/Documentation/hwmon/sht4x.rst
index ba094ad0e2816f..b9564632a1be2d 100644
--- a/Documentation/hwmon/sht4x.rst
+++ b/Documentation/hwmon/sht4x.rst
@@ -15,6 +15,16 @@ Supported Chips:
 
       English: https://www.sensirion.com/fileadmin/user_upload/customers/sensirion/Dokumente/2_Humidity_Sensors/Datasheets/Sensirion_Humidity_Sensors_SHT4x_Datasheet.pdf
 
+  * Sensirion STS4X
+
+    Prefix: 'sts4x'
+
+    Addresses scanned: None
+
+    Datasheet:
+
+      English: https://sensirion.com/resource/datasheet/sts4x
+
 Author: Navin Sankar Velliangiri <navin@linumiz.com>
 
 
@@ -22,9 +32,10 @@ Description
 -----------
 
 This driver implements support for the Sensirion SHT4x chip, a humidity
-and temperature sensor. Temperature is measured in degree celsius, relative
-humidity is expressed as a percentage. In sysfs interface, all values are
-scaled by 1000, i.e. the value for 31.5 degrees celsius is 31500.
+and temperature sensor, and the Sensirion STS4x chip, a temperature sensor.
+Temperature is measured in degree celsius, relative humidity is expressed as a
+percentage (on SHT4x only). In sysfs interface, all values are scaled by 1000,
+i.e. the value for 31.5 degrees celsius is 31500.
 
 Usage Notes
 -----------
diff --git a/Documentation/hwmon/socfpga-hwmon.rst b/Documentation/hwmon/socfpga-hwmon.rst
index e5da42556a627a..664103a38195a3 100644
--- a/Documentation/hwmon/socfpga-hwmon.rst
+++ b/Documentation/hwmon/socfpga-hwmon.rst
@@ -7,6 +7,7 @@ Supported chips:
 
  * Altera Stratix 10 SoC FPGA
  * Altera Agilex SoC FPGA
+ * Altera Agilex 5 SoC FPGA
 
 Authors:
       - Nazim Amirul <muhammad.nazim.amirul.nazle.asmade@altera.com>
@@ -32,3 +33,54 @@ driver based on the service layer compatible string:
 
   * intel,stratix10-svc
   * intel,agilex-svc
+  * intel,agilex5-svc
+
+Channel mappings are fixed in the driver (not described in DT). The tables
+below list the SDM page/channel encodings used for each family.
+
+Temperature channels
+~~~~~~~~~~~~~~~~~~~~
+
+==========  ====  =======  =================================
+Family      Page  Channel  Label
+==========  ====  =======  =================================
+Stratix 10  0     0        Main Die SDM
+Agilex      0     0        Main Die SDM
+Agilex      1     0        Main Die corner bottom left max
+Agilex      2     0        Main Die corner top left max
+Agilex      3     0        Main Die corner bottom right max
+Agilex      4     0        Main Die corner top right max
+Agilex 5    0     0        Main Die SDM
+Agilex 5    1     0        Main Die corner bottom left max
+Agilex 5    3     0        Main Die corner bottom right max
+Agilex 5    4     0        Main Die corner top right max
+==========  ====  =======  =================================
+
+Agilex 5 omits SDM temperature channel 2 (top-left corner on Agilex)
+because that sensor is not present in hardware. The remaining sensors keep
+the same channel numbers as Agilex.
+
+Voltage channels
+~~~~~~~~~~~~~~~~
+
+==========  ====  =======  =================
+Family      Page  Channel  Label
+==========  ====  =======  =================
+Stratix 10  0     2        0.8V VCC
+Stratix 10  0     3        1.8V VCCIO_SDM
+Stratix 10  0     6        0.9V VCCERAM
+Agilex      0     2        0.8V VCC
+Agilex      0     3        1.8V VCCIO_SDM
+Agilex      0     4        1.8V VCCPT
+Agilex      0     5        1.2V VCCCRCORE
+Agilex      0     6        0.9V VCCH
+Agilex      0     7        0.8V VCCL
+Agilex 5    0     2        0.8V VCC
+Agilex 5    0     3        1.8V VCCIO_SDM
+Agilex 5    0     4        1.8V VCCPT
+Agilex 5    0     5        1.2V VCCCRCORE
+Agilex 5    0     6        0.9V VCCH
+Agilex 5    0     7        0.8V VCCL
+==========  ====  =======  =================
+
+Agilex 5 reuses the Agilex voltage SDM page/channel layout and labels.
diff --git a/Documentation/hwmon/sysfs-interface.rst b/Documentation/hwmon/sysfs-interface.rst
index 94e1bbce172a33..c5ac4d3665dc43 100644
--- a/Documentation/hwmon/sysfs-interface.rst
+++ b/Documentation/hwmon/sysfs-interface.rst
@@ -47,7 +47,7 @@ There is only one value per file, unlike the older /proc specification.
 The common scheme for files naming is: <type><number>_<item>. Usual
 types for sensor chips are "in" (voltage), "temp" (temperature) and
 "fan" (fan). Usual items are "input" (measured value), "max" (high
-threshold, "min" (low threshold). Numbering usually starts from 1,
+threshold), "min" (low threshold). Numbering usually starts from 1,
 except for voltages which start from 0 (because most data sheets use
 this). A number is always used for elements that can be present more
 than once, even if there is a single element of the given type on the
@@ -121,12 +121,18 @@ Voltages
 `in[0-*]_lcrit`
 		Voltage critical min value.
 
+`in[0-*]_lemergency`
+		Voltage emergency min value.
+
 `in[0-*]_max`
 		Voltage max value.
 
 `in[0-*]_crit`
 		Voltage critical max value.
 
+`in[0-*]_emergency`
+		Voltage emergency max value.
+
 `in[0-*]_input`
 		Voltage input value.
 
@@ -332,6 +338,9 @@ Currents
 `curr[1-*]_crit`
 		Current critical high value.
 
+`curr[1-*]_emergency`
+		Current emergency high value.
+
 `curr[1-*]_input`
 		Current input value.
 
@@ -527,12 +536,15 @@ implementation.
 +-------------------------------+-----------------------+
 | **`in[0-*]_min_alarm`,	| Limit alarm		|
 | `in[0-*]_max_alarm`,		|			|
+| `in[0-*]_lemergency_alarm`,	|			|
 | `in[0-*]_lcrit_alarm`,	|   - 0: no alarm	|
 | `in[0-*]_crit_alarm`,		|   - 1: alarm		|
+| `in[0-*]_emergency_alarm`,	|			|
 | `curr[1-*]_min_alarm`,	|			|
 | `curr[1-*]_max_alarm`,	| RO			|
 | `curr[1-*]_lcrit_alarm`,	|			|
 | `curr[1-*]_crit_alarm`,	|			|
+| `curr[1-*]_emergency_alarm`,	|			|
 | `power[1-*]_cap_alarm`,	|			|
 | `power[1-*]_max_alarm`,	|			|
 | `power[1-*]_crit_alarm`,	|			|
diff --git a/Documentation/hwmon/tda38740.rst b/Documentation/hwmon/tda38740.rst
new file mode 100644
index 00000000000000..55ce12f149143b
--- /dev/null
+++ b/Documentation/hwmon/tda38740.rst
@@ -0,0 +1,65 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+====================================
+Kernel driver tda38740
+====================================
+
+Supported chips:
+  * Infineon TDA38740
+  * Infineon TDA38725
+
+    Prefix: 'tda38740'
+
+    Addresses scanned: -
+
+    Datasheet:
+      https://www.infineon.com/row/public/documents/24/49/infineon-tda38740-tda38725-datasheet-en.pdf
+
+Author: Colin Huang <u8813345@gmail.com>
+
+Description
+-----------
+
+The TDA38740 and TDA38725 are single-output synchronous buck regulators with a
+PMBus interface. They expose basic voltage, current, power and temperature
+telemetry through the standard PMBus hwmon interfaces.
+
+Sysfs entries
+-------------
+
+======================= =======================================================
+curr1_alarm		Input current alarm
+curr1_input		Measured input current
+curr1_label		"iin"
+
+curr2_crit		Critical maximum output current
+curr2_crit_alarm	Output current critical high alarm
+curr2_input		Measured output current
+curr2_label		"iout"
+
+in1_crit		Critical maximum input voltage
+in1_crit_alarm		Input voltage critical high alarm
+in1_input		Measured input voltage
+in1_label		"vin"
+
+in2_crit		Critical maximum output voltage
+in2_crit_alarm		Output voltage critical high alarm
+in2_input		Measured output voltage
+in2_label		"vout"
+in2_lcrit		Critical minimum output voltage
+in2_lcrit_alarm		Output voltage critical low alarm
+
+power1_alarm		Input power alarm
+power1_input		Measured input power
+power1_label		"pin"
+
+power2_input		Measured output power
+power2_label		"pout"
+
+temp1_crit		Critical high temperature
+temp1_crit_alarm	Chip temperature critical high alarm
+temp1_input		Measured temperature
+temp1_max		Maximum temperature
+temp1_max_alarm		Chip temperature high alarm
+======================= =======================================================
+
diff --git a/Documentation/hwmon/tps25990.rst b/Documentation/hwmon/tps25990.rst
index 04faec780d2628..e8bc9a550bda39 100644
--- a/Documentation/hwmon/tps25990.rst
+++ b/Documentation/hwmon/tps25990.rst
@@ -9,26 +9,31 @@ Supported chips:
 
     Prefix: 'tps25990'
 
-  * Datasheet
+    Datasheet: Publicly available at Texas Instruments website: https://www.ti.com/lit/gpn/tps25990
 
-    Publicly available at Texas Instruments website: https://www.ti.com/lit/gpn/tps25990
+  * TI TPS1689
+
+    Prefix: 'tps1689'
+
+    Datasheet: Publicly available at Texas Instruments website: https://www.ti.com/lit/gpn/tps1689
 
 Author:
 
 	Jerome Brunet <jbrunet@baylibre.com>
+	Stoyan Bogdanov <sbogdanov@baylibre.com>
 
 Description
 -----------
 
-This driver implements support for TI TPS25990 eFuse.
+This driver implements support for TI TPS25990 and TI TPS1689 eFuse chips.
 This is an integrated, high-current circuit protection and power
 management device with PMBUS interface
 
-Device compliant with:
+Devices are compliant with:
 
 - PMBus rev 1.3 interface.
 
-Device supports direct format for reading input voltages,
+Devices supports direct format for reading input voltages,
 output voltage, input current, input power and temperature.
 
 Due to the specificities of the chip, all history reset attributes
diff --git a/Documentation/hwmon/tps53679.rst b/Documentation/hwmon/tps53679.rst
index dd5e4a37375d11..3d584197b3a03c 100644
--- a/Documentation/hwmon/tps53679.rst
+++ b/Documentation/hwmon/tps53679.rst
@@ -3,6 +3,14 @@ Kernel driver tps53679
 
 Supported chips:
 
+  * Texas Instruments TPS53622
+
+    Prefix: 'tps53622'
+
+    Addresses scanned: -
+
+    Datasheet: https://www.ti.com/lit/gpn/TPS53622
+
   * Texas Instruments TPS53647
 
     Prefix: 'tps53647'
@@ -11,6 +19,14 @@ Supported chips:
 
     Datasheet: https://www.ti.com/lit/gpn/tps53647
 
+  * Texas Instruments TPS53659
+
+    Prefix: 'tps53659'
+
+    Addresses scanned: -
+
+    Datasheet: https://www.ti.com/lit/gpn/TPS53659
+
   * Texas Instruments TPS53667
 
     Prefix: 'tps53667'
@@ -59,6 +75,14 @@ Supported chips:
 
     Datasheet: Available under NDA
 
+  * Texas Instruments TPS536C7
+
+    Prefix: 'tps536c7'
+
+    Addresses scanned: -
+
+    Datasheet: https://www.ti.com/lit/gpn/TPS536C7
+
 
 Authors:
 	Vadim Pasternak <vadimp@mellanox.com>
@@ -69,7 +93,12 @@ Description
 -----------
 
 Chips in this series are multi-phase step-down converters with one or two
-output channels and up to 8 phases per channel.
+output channels and up to 12 phases in total, depending on the device.
+
+For TPS536C7 only aggregate per-channel telemetry is exposed; the
+individual per-phase output currents the hardware can report are not.
+Attributes for the second output channel are present only when channel B
+is configured.
 
 
 Usage Notes
@@ -108,7 +137,7 @@ in1_crit_alarm		Input voltage critical high alarm.
 in[N]_label		"vout[1-2]"
 
 			- TPS53647, TPS53667: N=2
-			- TPS53679, TPS53588: N=2,3
+			- TPS53622, TPS53659, TPS53679, TPS53688, TPS536C7: N=2,3
 
 in[N]_input		Measured output voltage.
 
@@ -135,7 +164,7 @@ in[N]_crit_alarm	Output voltage critical high alarm.
 temp[N]_input		Measured temperature.
 
 			- TPS53647, TPS53667: N=1
-			- TPS53679, TPS53681, TPS53588: N=1,2
+			- TPS53622, TPS53659, TPS53679, TPS53681, TPS53688, TPS536C7: N=1,2
 
 temp[N]_max		Maximum temperature.
 
@@ -152,7 +181,7 @@ power1_input		Measured input power.
 power[N]_label		"pout[1-2]".
 
 			- TPS53647, TPS53667: N=2
-			- TPS53676, TPS53679, TPS53681, TPS53588: N=2,3
+			- TPS53622, TPS53659, TPS53676, TPS53679, TPS53681, TPS53688, TPS536C7: N=2,3
 
 power[N]_input		Measured output power.
 
@@ -175,7 +204,7 @@ curr[N]_label		"iout[1-2]" or "iout1.[0-5]".
 			telemetry supported on TPS53676 and TPS53681 only.
 
 			- TPS53647, TPS53667: N=2
-			- TPS53679, TPS53588: N=2,3
+			- TPS53622, TPS53659, TPS53679, TPS53688, TPS536C7: N=2,3
 			- TPS53676: N=2-8
 			- TPS53681: N=2-9
 
diff --git a/Documentation/hwmon/vexpress.rst b/Documentation/hwmon/vexpress.rst
index 7fc99f75b3b166..84b2a33e2d40fa 100644
--- a/Documentation/hwmon/vexpress.rst
+++ b/Documentation/hwmon/vexpress.rst
@@ -38,4 +38,4 @@ via a custom interface, abstracted by the "vexpress_config" API.
 
 As these devices are non-discoverable, they must be described in a Device
 Tree passed to the kernel. Details of the DT binding for them can be found
-in Documentation/devicetree/bindings/hwmon/vexpress.txt.
+in Documentation/devicetree/bindings/arm/vexpress-config.yaml.
diff --git a/Documentation/hwmon/yogafan.rst b/Documentation/hwmon/yogafan.rst
index 6395c94f4a6b06..239c7b5761caf6 100644
--- a/Documentation/hwmon/yogafan.rst
+++ b/Documentation/hwmon/yogafan.rst
@@ -85,27 +85,30 @@ immediately to ensure the user knows the fan has stopped.
 
 ::
 
- MODEL (DMI PN) | FAMILY / SERIES  | EC OFFSET | FULL ACPI OBJECT PATH          | WIDTH  | MULTiplier
- ----------------------------------------------------------------------------------------------------
- 82N7           | Yoga 14cACN      | 0x06      | \_SB.PCI0.LPC0.EC0.FANS        |  8-bit | 100
- 80V2 / 81C3    | Yoga 710/720     | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
- 83E2 / 83DN    | Yoga Pro 7/9     | 0xFE      | \_SB.PCI0.LPC0.EC0.FANS        |  8-bit | 100
- 82A2 / 82A3    | Yoga Slim 7      | 0x06      | \_SB.PCI0.LPC0.EC0.FANS        |  8-bit | 100
- 81YM / 82FG    | IdeaPad 5        | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
- 82JW / 82JU    | Legion 5 (AMD)   | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FANS (Fan1) | 16-bit | 1
- 82JW / 82JU    | Legion 5 (AMD)   | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FA2S (Fan2) | 16-bit | 1
- 82WQ           | Legion 7i (Int)  | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FANS (Fan1) | 16-bit | 1
- 82WQ           | Legion 7i (Int)  | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FA2S (Fan2) | 16-bit | 1
- 82XV / 83DV    | LOQ 15/16        | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FANS /FA2S  | 16-bit | 1
- 83AK           | ThinkBook G6     | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
- 81X1           | Flex 5           | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
- 83KF           | XiaoXinPro 13ARE | 0x06/0xFE | \_SB.PCI0.LPC0.EC0.FANS/FA2S   |  8-bit | 100
- 82KU           | IdeaPad 3 15ALC6 | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
- 83RU           | Legion Pro 7 16  | 0x03/0x06 | \_SB.PCI0.LPC0.EC0.FANS/FA2S   |  8-bit | 100
- 83KF           | Yoga Pro 7 14IAH | 0x06      | \_SB.PC00.LPCB.EC0.FANS        |  8-bit | 100
- 83BS           | Yoga 7 16ARP8    | 0x03/0x06 | \_SB.PCI0.LPC0.EC0.FANS/FA2S   |  8-bit | 100
- *Legacy*       | Pre-2020 Models  | 0x06      | \_SB.PCI0.LPC.EC.FAN0          |  8-bit | 100
- ----------------------------------------------------------------------------------------------------
+ MODEL (DMI PN) | FAMILY / SERIES     | EC OFFSET | FULL ACPI OBJECT PATH          | WIDTH  | MULTiplier
+ -------------------------------------------------------------------------------------------------------
+ 82N7           | Yoga 14cACN 2021    | 0x06      | \_SB.PCI0.LPC0.EC0.FANS        |  8-bit | 100
+ 81TD           | Yoga 740-15IML      | 0x06      | \_SB.PCI0.LPCB.EC0.FANS        |  8-bit | 100
+ 82KU           | IdeaPad 3 15ALC6 Ub | 0x06      | \_SB.PCI0.LPC0.EC0.FANS/FA2S   |  8-bit | 100
+ 80V2 / 81C3    | Yoga 710/720        | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
+ 83E2           | Yoga Pro 7          | 0xFE      | \_SB.PCI0.LPC0.EC0.FANS        |  8-bit | 100
+ 83DN           | Yoga Pro 9 16IMH9   | 0x06/0xFE | \_SB.PC00.LPCB.EC0.FANS/FA2S   |  8-bit | 100
+ 82A2 / 82A3    | Yoga Slim 7         | 0x06      | \_SB.PCI0.LPC0.EC0.FANS        |  8-bit | 100
+ 81YM / 82FG    | IdeaPad 5           | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
+ 82JW / 82JU    | Legion 5 (AMD)      | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FANS (Fan1) | 16-bit | 1
+ 82JW / 82JU    | Legion 5 (AMD)      | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FA2S (Fan2) | 16-bit | 1
+ 82WQ           | Legion 7i (Int)     | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FANS (Fan1) | 16-bit | 1
+ 82WQ           | Legion 7i (Int)     | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FA2S (Fan2) | 16-bit | 1
+ 82XV / 83DV    | LOQ 15/16           | 0xFE/0xFF | \_SB.PCI0.LPC0.EC0.FANS /FA2S  | 16-bit | 1
+ 83AK           | ThinkBook G6        | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
+ 81X1           | Flex 5              | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
+ 83KF           | XiaoXinPro 13ARE    | 0x06/0xFE | \_SB.PCI0.LPC0.EC0.FANS/FA2S   |  8-bit | 100
+ 82KU           | IdeaPad 3 15ALC6    | 0x06      | \_SB.PCI0.LPC0.EC0.FAN0        |  8-bit | 100
+ 83RU           | Legion Pro 7 16     | 0x03/0x06 | \_SB.PCI0.LPC0.EC0.FANS/FA2S   |  8-bit | 100
+ 83KF           | Yoga Pro 7 14IAH    | 0x06      | \_SB.PC00.LPCB.EC0.FANS        |  8-bit | 100
+ 83BS           | Yoga 7 16ARP8       | 0x03/0x06 | \_SB.PCI0.LPC0.EC0.FANS/FA2S   |  8-bit | 100
+ *Legacy*       | Pre-2020 Models     | 0x06      | \_SB.PCI0.LPC.EC.FAN0          |  8-bit | 100
+ -------------------------------------------------------------------------------------------------------
 
 METHODOLOGY & IDENTIFICATION:
 
@@ -158,6 +161,7 @@ Contributors & DSDT Providers:
 - **Sarbajit Sarkar** (Lenovo LOQ 15IAX9)
 - **HinataKato** (XiaoXin Pro 13ARE 2020 - 83KF)
 - **PenPenIsGod** (IdeaPad 3 15ALC6 - 82KU & Legion Pro 7 16AFR10H - 83RU)
+- **wizard-28** (IdeaPad 3 15ALC6 Ub - 82KU)
 - **unlockxiaom** (Legion Pro 7 16AFR10H - 83RU, Yoga Pro 7 14IAH10 - 83KF, ThinkCentre M80q)
 - **Phani Pavan K** (Yoga Pro 7 14IAH10 - 83KF)
 - **Splarkszter** (Yoga 7 16ARP8 - 83BS)
diff --git a/Documentation/iio/ad7768.rst b/Documentation/iio/ad7768.rst
new file mode 100644
index 00000000000000..9f5fbcae717323
--- /dev/null
+++ b/Documentation/iio/ad7768.rst
@@ -0,0 +1,275 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+=============
+AD7768 driver
+=============
+
+ADC driver for Analog Devices Inc. AD7768 and AD7768-4 devices. The module name
+is ``ad7768``.
+
+Supported devices
+=================
+
+The following chips are supported by this driver:
+
+* `AD7768 <https://www.analog.com/en/products/ad7768.html>`_ -
+  8-channel, 24-bit simultaneous sampling ADC
+* `AD7768-4 <https://www.analog.com/en/products/ad7768-4.html>`_ -
+  4-channel, 24-bit simultaneous sampling ADC
+
+Supported features
+==================
+
+Clock sources
+-------------
+
+The master clock source is selected using the ``clock-names`` device tree
+property:
+
+* ``mclk`` selects a CMOS clock on XTAL2/MCLK. CLK_SEL must be low and XTAL1
+  must be connected to DGND.
+* ``xtal`` selects a crystal connected between XTAL1 and XTAL2. CLK_SEL must
+  be high. The driver drives FILTER/GPIO4 high and keeps channel 4 on the
+  AD7768 or channel 2 on the AD7768-4 enabled to maintain crystal excitation.
+* ``lvds`` selects a differential clock on XTAL1 and XTAL2. CLK_SEL must be
+  high. The driver drives FILTER/GPIO4 low before enabling the LVDS input.
+
+GPIO4 is reserved when ``xtal`` or ``lvds`` is selected.
+
+Power modes
+-----------
+
+The AD7768 family supports three power and performance modes:
+
+* **Low power mode** - Optimized for lowest power consumption
+* **Median mode** - Balanced power and performance
+* **Fast mode** - Highest performance with maximum sampling rates
+
+The driver initializes the device in fast mode and uses the maximum fast-mode
+output data rate as the default sampling frequency.
+
+When buffered capture starts, the driver selects the lowest-noise mode that can
+produce the requested output data rates for all enabled channels. Where output
+data rates overlap, fast mode is preferred over median mode, and median mode is
+preferred over low power mode. This prioritizes the lower RMS noise and higher
+dynamic range offered by a faster mode at the same output data rate.
+
+Data output configuration
+-------------------------
+
+The devices support flexible serial data output configurations:
+
+AD7768 data lines
+^^^^^^^^^^^^^^^^^
+
+* 1 data line (DOUT0) - Standard single-lane output
+* 2 data lines (DOUT0, DOUT1) - Dual-lane output for higher throughput
+* 8 data lines (DOUT0-DOUT7) - Maximum throughput, one line per channel
+
+AD7768-4 data lines
+^^^^^^^^^^^^^^^^^^^
+
+* 1 data line (DOUT0) - Standard single-lane output
+* 4 data lines (DOUT0-DOUT3) - Maximum throughput, one line per channel
+
+The number of data lines can be configured via the ``adi,data-lines-number``
+device tree property. If omitted, the driver uses the maximum supported by the
+selected variant: eight lines for AD7768 and four lines for AD7768-4.
+
+Channel configuration
+---------------------
+
+Each channel can be individually configured with:
+
+Channel modes
+^^^^^^^^^^^^^
+
+* **Mode A** - First set of filter and decimation settings
+* **Mode B** - Second set of filter and decimation settings
+
+The hardware provides two mode profiles (A and B), each holding one
+(frequency, filter) combination. When buffered capture is started,
+enabled channels are grouped by their configured (frequency, filter)
+pair. Up to two distinct groups are supported; the driver automatically
+assigns each group to a mode slot and programs the hardware accordingly.
+If the enabled channels request more than two distinct pairs, buffer setup
+fails and capture does not start.
+
+Precharge and reference buffers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Per-channel buffer control for optimal signal integrity:
+
+* Positive input precharge buffer (``adi,prechargebuf-pos-enable``)
+* Negative input precharge buffer (``adi,prechargebuf-neg-enable``)
+* Positive reference buffer (``adi,refbuf-pos-enable``)
+* Negative reference buffer (``adi,refbuf-neg-enable``)
+
+Common mode voltage
+-------------------
+
+The VCM pin provides a buffered common-mode voltage output used to bias
+the analog inputs. The driver exposes this as a standard voltage regulator
+provider under a ``regulators`` subnode in the device tree. Supported
+output voltage levels are:
+
+* (AVDD1 - AVSS) / 2 - Mid-supply (hardware default), reported as half the
+  voltage provided by ``avdd1-supply``
+* 1,650,000 µV - 1.65V
+* 2,500,000 µV - 2.5V
+* 2,140,000 µV - 2.14V
+
+The regulator can be enabled and disabled at runtime using the standard
+regulator framework interfaces.
+
+The VCM circuitry is associated with channel 0. When VCM is used externally,
+``channel@0`` must be present in the device tree and channel 0 must remain
+enabled in the active scan mask. Placing channel 0 in standby disables the VCM
+output.
+
+Filter types
+------------
+
+Two digital filter types are available:
+
+* **Wideband** - Optimized for wide bandwidth applications
+* **Sinc5** - Fifth-order sinc filter for high rejection of out-of-band noise
+
+IIO backend support
+-------------------
+
+The driver integrates with IIO backends (e.g., AXI ADC) for high-speed data
+capture and DMA operations. Features include:
+
+* Automatic channel enable/disable based on scan mask
+* CRC on data interface. CRC replaces the header every 4th output sample.
+* High-throughput buffered data acquisition
+
+GPIO controller
+---------------
+
+The AD7768 includes a 5-pin GPIO controller for auxiliary digital I/O
+operations. The GPIO pins can be configured as inputs or outputs.
+
+GPIO4 is reserved when a crystal or LVDS master clock is used. GPIO0 through
+GPIO3 remain available in these configurations.
+
+Device attributes
+=================
+
+The following IIO attributes are available for each enabled channel:
+
+Voltage scale
+-------------
+
+.. list-table::
+   :header-rows: 1
+   :widths: 40 60
+
+   * - Attribute
+     - Description
+   * - ``in_voltage<N>_scale``
+     - Scale in millivolts per least significant bit for signed 24-bit data,
+       calculated as twice the differential reference voltage used by channel
+       N divided by 2\ :sup:`24`.
+
+Sampling frequency
+------------------
+
+.. list-table::
+   :header-rows: 1
+   :widths: 40 60
+
+   * - Attribute
+     - Description
+   * - ``in_voltage<N>_sampling_frequency``
+     - Requested sampling frequency in Hz for channel N. Enabled channels are
+       grouped into up to two profiles at capture time.
+   * - ``in_voltage<N>_sampling_frequency_available``
+     - Available sampling frequencies in Hz for channel N across all power
+       modes, based on the master clock frequency. Buffer setup fails if no
+       single power mode supports the frequencies requested by all enabled
+       channels.
+
+Filter configuration
+---------------------
+
+.. list-table::
+   :header-rows: 1
+   :widths: 40 60
+
+   * - Attribute
+     - Description
+   * - ``in_voltage<N>_filter_type``
+     - Requested filter type for channel N: "wideband" or "sinc5". It is
+       grouped with sampling frequency at capture time.
+   * - ``in_voltage<N>_filter_type_available``
+     - Available filter types for channel N: "wideband sinc5".
+
+Per-channel calibration
+-----------------------
+
+.. list-table::
+   :header-rows: 1
+   :widths: 40 60
+
+   * - Attribute
+     - Description
+   * - ``in_voltage<N>_calibbias``
+     - Signed 24-bit two's complement channel offset register value.
+   * - ``in_voltage<N>_calibscale``
+     - Raw unsigned 24-bit channel gain register value.
+   * - ``in_voltage<N>_convdelay``
+     - Per-channel conversion delay. The driver exposes the sync phase offset
+       value in seconds with picosecond precision. Resolution and valid range
+       depend on the decimation ratio in use (see datasheet
+       Table 32).
+
+Device buffers
+==============
+
+This driver supports IIO buffered data acquisition through IIO backends.
+When used with compatible backends like the AXI ADC, it provides:
+
+* High-speed simultaneous sampling across all enabled channels
+* Hardware-driven data capture
+* DMA-based data transfer for minimal CPU overhead
+* CRC error detection
+
+See :doc:`iio_devbuf` for more information about IIO device buffers.
+
+Example usage
+=============
+
+.. code-block:: bash
+
+	# Read current sampling frequency for channel 0
+	cat /sys/bus/iio/devices/iio:device0/in_voltage0_sampling_frequency
+
+	# Update sampling frequency for channel 0
+	echo 8000 > /sys/bus/iio/devices/iio:device0/in_voltage0_sampling_frequency
+
+	# Read current filter type for channel 0
+	cat /sys/bus/iio/devices/iio:device0/in_voltage0_filter_type
+
+	# List available filter types for channel 0
+	cat /sys/bus/iio/devices/iio:device0/in_voltage0_filter_type_available
+
+	# Update filter type for channel 0 to wideband
+	echo wideband > /sys/bus/iio/devices/iio:device0/in_voltage0_filter_type
+
+	# Buffer setup fails if enabled channels request more than two distinct
+	# (sampling frequency, filter type) combinations.
+
+	# Read calibration scale for channel 0
+	cat /sys/bus/iio/devices/iio:device0/in_voltage0_calibscale
+
+	# Read conversion delay for channel 0
+	cat /sys/bus/iio/devices/iio:device0/in_voltage0_convdelay
+
+
+Unimplemented features
+======================
+
+* CRC message every 16 samples (CRC_SEL configuration) - currently only
+  supports CRC every 4 samples
diff --git a/Documentation/iio/ad9910.rst b/Documentation/iio/ad9910.rst
new file mode 100644
index 00000000000000..aab569f561dbc4
--- /dev/null
+++ b/Documentation/iio/ad9910.rst
@@ -0,0 +1,792 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+=============
+AD9910 driver
+=============
+
+Direct Digital Synthesizer (DDS) driver for the Analog Devices Inc. AD9910.
+The module name is ``ad9910``.
+
+* `AD9910 <https://www.analog.com/en/products/ad9910.html>`_
+
+The AD9910 is a 1 GSPS DDS with a 14-bit DAC, controlled over SPI. The driver
+exposes the device through a hierarchy of typed IIO output channels. The root
+``phy`` channel controls the system clock and full-scale output current.
+Sub-channels provide independent control over single tone profiles, parallel
+port modulation, digital ramp generation (DRG), RAM playback and output shift
+keying (OSK).
+
+
+Channel hierarchy
+=================
+
+The driver exposes the following IIO output channels, each identified by a
+unique channel number and a human-readable label. The ``phy`` channel is the
+root of the hierarchy. Changing its ``sampling_frequency`` reconfigures the
+system clock (SYSCLK) which affects all other channels.
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Channel
+     - Label
+     - Parent
+     - Description
+
+   * - ``out_altcurrent100``
+     - ``phy``
+     -
+     - Physical output: system clock and full-scale output current (:math:`I_{FS}`).
+       See `Physical channel`_.
+
+   * - ``out_altcurrent110`` ... ``out_altcurrent117``
+     - ``profile0`` ... ``profile7``
+     - ``phy``
+     - Single tone control: frequency, phase, amplitude, enable.
+       See `Single Tone mode`_.
+
+   * - ``out_altcurrent120``
+     - ``parallel_amplitude``
+     - ``phy``
+     - Parallel port amplitude channel.
+       See `Parallel Port mode`_.
+
+   * - ``out_phase120``
+     - ``parallel_phase``
+     - ``phy``
+     - Parallel port phase channel.
+
+   * - ``out_frequency120``
+     - ``parallel_frequency``
+     - ``phy``
+     - Parallel port frequency channel: ``scale`` sets the FM gain
+       (power-of-2 multiplier), ``offset`` sets the base FTW.
+
+   * - ``out_altcurrent121``
+     - ``parallel_polar_amplitude``
+     - ``phy``
+     - Parallel polar amplitude channel: ``scale`` is the amplitude
+       resolution, ``offset`` is the amplitude bias (lower 6 bits of ASF).
+
+   * - ``out_phase121``
+     - ``parallel_polar_phase``
+     - ``phy``
+     - Parallel polar phase channel: ``scale`` is the phase resolution,
+       ``offset`` is the phase bias (lower 8 bits of POW).
+
+   * - ``out_frequency130``
+     - ``drg_frequency``
+     - ``phy``
+     - DRG frequency channel: ``en`` selects and enables the DRG with
+       frequency as the ramp target.
+       See `Digital ramp generator (DRG)`_.
+
+   * - ``out_frequency131``
+     - ``drg_rising_frequency``
+     - ``drg_frequency``
+     - DRG frequency rising-ramp parameters: limit code, dwell enable,
+       ramp clock, rate of change.
+
+   * - ``out_frequency132``
+     - ``drg_falling_frequency``
+     - ``drg_frequency``
+     - DRG frequency falling-ramp parameters: limit code, dwell enable,
+       ramp clock, rate of change.
+
+   * - ``out_phase130``
+     - ``drg_phase``
+     - ``phy``
+     - DRG phase channel: ``en`` selects and enables the DRG with phase
+       as the ramp target.
+       See `Digital ramp generator (DRG)`_.
+
+   * - ``out_phase131``
+     - ``drg_rising_phase``
+     - ``drg_phase``
+     - DRG phase rising-ramp parameters: limit code, dwell enable, ramp clock,
+       rate of change.
+
+   * - ``out_phase132``
+     - ``drg_falling_phase``
+     - ``drg_phase``
+     - DRG phase falling-ramp parameters: limit code, dwell enable, ramp clock,
+       rate of change.
+
+   * - ``out_altcurrent130``
+     - ``drg_amplitude``
+     - ``phy``
+     - DRG amplitude channel: ``en`` selects and enables the DRG with
+       amplitude as the ramp target.
+       See `Digital ramp generator (DRG)`_.
+
+   * - ``out_altcurrent131``
+     - ``drg_rising_amplitude``
+     - ``drg_amplitude``
+     - DRG amplitude rising-ramp parameters: limit code, dwell enable,
+       ramp clock, rate of change.
+
+   * - ``out_altcurrent132``
+     - ``drg_falling_amplitude``
+     - ``drg_amplitude``
+     - DRG amplitude falling-ramp parameters: limit code, dwell enable,
+       ramp clock, rate of change.
+
+   * - ``out_altcurrent140``
+     - ``ram``
+     - ``phy``
+     - RAM playback: enable, frequency, phase and sampling frequency for
+       the active profile. See `RAM mode`_.
+
+   * - ``out_altcurrent150``
+     - ``osk``
+     - ``phy``
+     - Output shift keying (OSK): enable, amplitude code, ramp rate,
+       rate of change. See `Output Shift Keying (OSK)`_.
+
+DDS modes
+=========
+
+The AD9910 supports multiple modes of operation that can be configured
+independently or in combination. Each DDS core parameter (frequency, phase
+and amplitude) can come from different sources, but only one is active at a
+time. This activation depends on a priority list, which is based on the enable
+and destination configurations for such modes. The following tables are
+extracted from the AD9910 datasheet and summarize the control parameters for
+each mode and their priority when multiple sources are enabled simultaneously:
+
+.. flat-table:: DDS Frequency Control
+   :header-rows: 1
+
+   * - Priority
+     - Data Source
+     - Conditions
+
+   * - Highest Priority
+     - RAM
+     - RAM enabled and data destination is frequency
+
+   * -
+     - DRG
+     - DRG enabled and data destination is frequency
+
+   * -
+     - Parallel data and Frequency Tuning Word, FTW (frequency_offset)
+     - Parallel data port enabled and data destination is frequency
+
+   * -
+     - FTW register (frequency)
+     - RAM enabled and data destination is not frequency
+
+   * - Lowest Priority
+     - FTW (frequency) in single tone channel for the active profile
+     - All other cases
+
+.. flat-table:: DDS Phase Control
+   :header-rows: 1
+
+   * - Priority
+     - Data Source
+     - Conditions
+
+   * - Highest Priority
+     - RAM
+     - RAM enabled and data destination is phase or polar
+
+   * -
+     - DRG
+     - DRG enabled and data destination is phase
+
+   * -
+     - Parallel data port
+     - Parallel data port enabled and data destination is phase
+
+   * -
+     - Parallel data port and Phase Offset Word, POW register LSBs (phase_offset)
+     - Parallel data port enabled and data destination is polar
+
+   * -
+     - POW register (phase)
+     - RAM enabled and destination is not phase nor polar
+
+   * - Lowest Priority
+     - POW (phase) in single tone channel for the active profile
+     - All other cases
+
+.. flat-table:: DDS Amplitude Control
+   :header-rows: 1
+
+   * - Priority
+     - Data Source
+     - Conditions
+
+   * - Highest Priority
+     - Amplitude Scale Factor, ASF register and OSK generator
+     - OSK enabled
+
+   * -
+     - RAM
+     - RAM enabled and data destination is amplitude or polar
+
+   * -
+     - DRG
+     - DRG enabled and data destination is amplitude
+
+   * -
+     - Parallel data port
+     - Parallel data port enabled and data destination is amplitude
+
+   * -
+     - Parallel data port and ASF register LSBs (scale_offset)
+     - Parallel data port enabled and data destination is polar
+
+   * - Lowest Priority
+     - ASF (scale) in single tone channel for the active profile
+     - (Amplitude scale is already enabled by default)
+
+While debugging or testing, the debug attributes ``frequency_source``,
+``phase_source`` and ``amplitude_source`` can be used to read the label of
+the channel that is actively controlling the correspondent DDS parameter,
+which reflects the priority list described above.
+
+Single Tone mode
+----------------
+
+Single tone is the baseline operating mode. The ``profileY`` channels
+provide enable, frequency, phase and amplitude control:
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Attribute
+     - Unit
+     - Description
+
+   * - ``en``
+     - boolean (0 or 1)
+     - Enable/disable profile Y. Only one profile can be active at a
+       time. When enabling a profile it disables the current active profile.
+       Disabling an active profile brings the device to a powered down state.
+
+   * - ``frequency``
+     - Hz
+     - Output frequency. Range :math:`[0, f_{SYSCLK} / 2)`. Stored in the
+       profile's frequency tuning word (FTW).
+
+   * - ``phase``
+     - rad
+     - Phase offset. Range :math:`[0, 2\pi)`. Stored in the profile's phase
+       offset word (POW).
+
+   * - ``raw``
+     - integer
+     - Amplitude scale factor code. Range :math:`[0, 16383]`. Stored in the
+       profile's amplitude scale factor (ASF) register. The physical output
+       amplitude is ``raw * scale`` where ``scale`` is read from the ``phy``
+       channel.
+
+Profile switching is allowed while RAM mode is enabled. In that case single
+tone parameters are stored in a shadow register and are not written to
+hardware until RAM mode is disabled.
+
+Usage examples
+^^^^^^^^^^^^^^
+
+Configure a 100 MHz tone in profile 2 and set it as the active profile:
+
+.. code-block:: bash
+
+  echo 100000000 > /sys/bus/iio/devices/iio\:device0/out_altcurrent112_frequency
+  echo 0 > /sys/bus/iio/devices/iio\:device0/out_altcurrent112_phase
+  echo 16383 > /sys/bus/iio/devices/iio\:device0/out_altcurrent112_raw
+
+  # Activate profile 2
+  echo 1 > /sys/bus/iio/devices/iio\:device0/out_altcurrent112_en
+
+Read back the current single tone frequency:
+
+.. code-block:: bash
+
+  cat /sys/bus/iio/devices/iio\:device0/out_altcurrent112_frequency
+
+Parallel Port mode
+------------------
+
+The parallel port allows real-time modulation of DDS parameters through a
+16-bit external data bus. The driver exposes separate typed channels for each
+modulation target.
+
+Non-polar modulation
+^^^^^^^^^^^^^^^^^^^^
+
+In non-polar mode each DDS parameter is controlled by an independent 16-bit bus
+input. The parallel port channels expose the resolution and base offset of their
+respective bus inputs:
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Channel
+     - Attribute
+     - Unit
+     - Description
+
+   * - ``out_phase120``
+     - ``scale``
+     - rad
+     - Phase resolution per parallel bus LSB. Fixed at :math:`\pi / 2^{15}`
+       rad/LSB (full 16-bit POW resolution).
+
+   * - ``out_frequency120``
+     - ``scale``
+     - Hz
+     - Outputs the frequency scale evaluated as
+       :math:`f_{SYSCLK} \cdot FM / 2^{32}`. Assuming that :math:`f_{SYSCLK}` is
+       fixed, it is used to configure the modulation gain :math:`FM`, which is a
+       power-of-2 multiplier in range :math:`[1, 32768]`. Writing to this
+       attribute rounds up to the nearest :math:`FM` power of 2.
+
+   * - ``out_frequency120``
+     - ``offset``
+     - Hz
+     - Outputs the frequency offset in raw units evaluated as :math:`FTW / FM`.
+       Assuming that :math:`FM` is fixed, it is used to configure the FTW
+       register, which is a 32-bit unsigned integer.
+
+Polar modulation
+^^^^^^^^^^^^^^^^
+
+In polar mode a single 16-bit bus word carries both amplitude (high byte)
+and phase (low byte). The ``parallel_polar_amplitude`` and
+``parallel_polar_phase`` channels configure the bias applied to the low-order
+bits of the ASF and POW registers respectively:
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Channel
+     - Attribute
+     - Unit
+     - Description
+
+   * - ``out_altcurrent121``
+     - ``scale``
+     - mA/LSB
+     - Full-scale amplitude resolution (read-only, fixed at :math:`I_{FS} / 2^{8}`).
+
+   * - ``out_altcurrent121``
+     - ``offset``
+     - fractional (raw units)
+     - Amplitude bias. Lower 6 bits of ASF register. Range :math:`[0, 1)`.
+
+   * - ``out_phase121``
+     - ``scale``
+     - rad/LSB
+     - Phase resolution (read-only, fixed at :math:`\pi / 2^{7}`).
+
+   * - ``out_phase121``
+     - ``offset``
+     - fractional (raw units)
+     - Phase bias. Lower 8 bits of POW register. Range :math:`[0, 1)`.
+
+Usage examples
+^^^^^^^^^^^^^^
+
+Set parallel port frequency modulation with a modulation gain of 16 and a 50 MHz
+offset:
+
+.. code-block:: bash
+
+  # f_SYSCLK = 1 GHz, FM = 16
+  # frequency scale = f_SYSCLK * FM / 2^32 = 3.725290298
+  echo 3.725290298 > /sys/bus/iio/devices/iio\:device0/out_frequency120_scale
+  # frequency offset = 50 MHz / scale = 50e6 / 13421772.8
+  echo 13421772.8 > /sys/bus/iio/devices/iio\:device0/out_frequency120_offset
+
+One should choose a frequency scale that allows all the desired frequencies
+to be represented in the 16-bit bus range, i.e.,
+:math:`scale = (f_{max} - f_{min}) / 2^{16}`.
+
+
+Digital ramp generator (DRG)
+----------------------------
+
+The DRG produces linear frequency, phase or amplitude sweeps using dedicated
+hardware. The active ramp target (destination) is selected by enabling the
+corresponding typed channel at channel number 130:
+
+- ``out_frequency130`` (label ``drg_frequency``) — ramp targets frequency
+- ``out_phase130`` (label ``drg_phase``) — ramp targets phase
+- ``out_altcurrent130`` (label ``drg_amplitude``) — ramp targets amplitude
+
+Writing ``en=1`` to one of these channels enables the DRG and switches its
+destination. Writing ``en=0`` disables the DRG if the channel is the current
+active destination; writing to an already-inactive destination is a no-op.
+
+Each destination channel also exposes a read-only ``scale`` attribute
+reporting the physical quantity per ramp register LSB, which allows converting
+raw limit codes to physical values.
+
+The six ramp channels:
+
+- ``out_frequency131`` (label ``drg_rising_frequency``)
+- ``out_frequency132`` (label ``drg_falling_frequency``)
+- ``out_phase131`` (label ``drg_rising_phase``)
+- ``out_phase132`` (label ``drg_falling_phase``)
+- ``out_altcurrent131`` (label ``drg_rising_amplitude``)
+- ``out_altcurrent132`` (label ``drg_falling_amplitude``)
+
+configure ascending and descending ramp parameters.
+
+Destination channel attributes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Attribute
+     - Unit
+     - Description
+
+   * - ``en``
+     - boolean
+     - Enable the DRG with this channel as the active destination. Only one
+       destination can be active at a time.
+
+   * - ``scale``
+     - Hz/LSB, rad/LSB or mA/LSB
+     - Read-only. Physical quantity per raw units. Multiply a ramp
+       rising/falling channel ``raw`` value by this scale to get the physical
+       ramp target.
+
+Ramp channel attributes
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Attribute
+     - Unit
+     - Description
+
+   * - ``dwell_en``
+     - boolean
+     - Enable dwell at the ramp limit. When disabled, the ramp
+       auto-transitions at this limit without waiting for the DRCTL pin.
+       Disabling both creates a bidirectional continuous ramp (triangular
+       pattern). Other combinations create single-shot ramps at the DRCTL
+       pin transition.
+
+   * - ``raw``
+     - integer (64-bit)
+     - Ramp limit expressed as a raw DRG register code in
+       :math:`[0, 2^{32}-1]`. The physical value is ``raw * scale`` where
+       ``scale`` is read from the active destination channel.
+
+   * - ``sampling_frequency``
+     - Hz
+     - Ramp clock rate. Controlled by an integer divider; the written value
+       is adjusted to the nearest supported rate.
+
+   * - ``raw_roc``
+     - /s
+     - Rate of change. Number of register codes advanced per second, computed
+       from the hardware step size and the current ramp clock. Writing
+       requires ``sampling_frequency`` to be configured first.
+
+Usage examples
+^^^^^^^^^^^^^^
+
+Configure a frequency sweep from 40 MHz to 60 MHz with a rate of change of
+25 GHz/s:
+
+.. code-block:: bash
+
+  # Disable dwell on both limits for a bidirectional continuous ramp
+  echo 0 > /sys/bus/iio/devices/iio\:device0/out_frequency131_dwell_en
+  echo 0 > /sys/bus/iio/devices/iio\:device0/out_frequency132_dwell_en
+
+  # Set ramp rate at 250 MHz
+  echo 250000000 > /sys/bus/iio/devices/iio\:device0/out_frequency131_sampling_frequency
+  echo 250000000 > /sys/bus/iio/devices/iio\:device0/out_frequency132_sampling_frequency
+
+  # read the frequency scale to convert physical values to raw units
+  cat /sys/bus/iio/devices/iio\:device0/out_frequency130_scale
+  0.232830643650
+
+  # 40 MHz / 0.232830643650 = 171798692
+  echo 171798692 > /sys/bus/iio/devices/iio\:device0/out_frequency131_raw
+  # 60 MHz / 0.232830643650 = 257698038
+  echo 257698038 > /sys/bus/iio/devices/iio\:device0/out_frequency132_raw
+
+  # 25 GHz/s / 0.232830643650 = 107374182402
+  echo 107374182402 > /sys/bus/iio/devices/iio\:device0/out_frequency131_raw_roc
+  echo 107374182402 > /sys/bus/iio/devices/iio\:device0/out_frequency132_raw_roc
+
+  # Enable the DRG with frequency as the destination
+  echo 1 > /sys/bus/iio/devices/iio\:device0/out_frequency130_en
+
+RAM mode
+--------
+
+The AD9910 contains a 1024 x 32-bit RAM that can be loaded with waveform data
+and played back to modulate frequency, phase, amplitude, or polar (phase +
+amplitude) parameters.
+
+RAM control channel attributes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Attribute
+     - Unit
+     - Description
+
+   * - ``en``
+     - boolean
+     - Enable/disable RAM playback. Toggling swaps profile registers between
+       single tone and RAM configurations across all 8 profiles.
+
+   * - ``frequency``
+     - Hz
+     - Frequency tuning word used as the single tone frequency when
+       RAM destination is not ``frequency``. Range: :math:`[0, f_{SYSCLK} / 2)`.
+
+   * - ``phase``
+     - rad
+     - Phase offset word used as the single tone phase when RAM destination
+       is not ``phase``. Range: :math:`[0, 2\pi)`.
+
+   * - ``sampling_frequency``
+     - Hz
+     - RAM playback step rate of the active profile, which controls how fast
+       the address counter advances. Controlled by an integer divider; the
+       written value is adjusted to the nearest supported rate.
+
+Loading RAM data
+^^^^^^^^^^^^^^^^
+
+RAM data is loaded through the firmware upload framework. The driver registers
+a firmware upload sysfs entry named ``iio_deviceX:ram``. The firmware data
+follows a binary format (version 1) with an 80-byte header followed by data
+words. All fields are big-endian.
+
+.. flat-table:: RAM firmware header (80 bytes)
+   :header-rows: 1
+
+   * - Offset
+     - Size
+     - Field
+     - Description
+
+   * - 0
+     - 4
+     - ``magic``
+     - Magic number: ``0x00AD9910``
+
+   * - 4
+     - 2
+     - ``version``
+     - Format version: ``0x0001``
+
+   * - 6
+     - 2
+     - ``wcount``
+     - Number of 32-bit RAM data words (0--1024)
+
+   * - 8
+     - 4
+     - ``crc``
+     - CRC32 checksum over ``cfr1``, ``profiles`` and ``words``
+
+   * - 12
+     - 4
+     - ``cfr1``
+     - CFR1 register value. Only RAM-relevant bits are used:
+       bits [30:29] set data destination (00: frequency, 01: phase,
+       10: amplitude, 11: polar); bits [20:17] set internal profile
+       control (see datasheet Table 14)
+
+   * - 16
+     - 64
+     - ``profiles[0..7]``
+     - 8 sets of 8-byte RAM profile configurations (see below)
+
+   * - 80
+     - 4 x wcount
+     - ``words[]``
+     - RAM data words in reverse order
+
+Each 8-byte profile entry contains:
+
+.. flat-table:: RAM profile entry (8 bytes)
+   :header-rows: 1
+
+   * - Bits
+     - Field
+     - Description
+
+   * - [55:40]
+     - Address step rate
+     - Controls playback speed for this profile
+
+   * - [39:30]
+     - End address
+     - Last RAM address for this profile
+
+   * - [23:14]
+     - Start address
+     - First RAM address for this profile
+
+   * - [5]
+     - No-dwell high
+     - No-dwell at high limit (ramp-up mode)
+
+   * - [3]
+     - Zero-crossing
+     - Zero-crossing enable (direct-switch mode)
+
+   * - [2:0]
+     - Operating mode
+     - 000: direct switch, 001: ramp-up, 010: bidirectional,
+       011: bidirectional continuous, 100: ramp-up continuous
+
+Usage examples
+^^^^^^^^^^^^^^
+
+Configure RAM mode with firmware data and enable it:
+
+.. code-block:: bash
+
+  # Load RAM data via firmware upload
+  echo 1 > /sys/class/firmware/iio\:device0\:ram/loading
+  cat ad9910-ram.bin > /sys/class/firmware/iio\:device0\:ram/data
+  echo 0 > /sys/class/firmware/iio\:device0\:ram/loading
+
+  # Enable RAM mode
+  echo 1 > /sys/bus/iio/devices/iio\:device0/out_altcurrent140_en
+
+Output Shift Keying (OSK)
+-------------------------
+
+OSK controls the output amplitude envelope, allowing the output to be ramped
+on/off rather than switched abruptly.
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Attribute
+     - Unit
+     - Description
+
+   * - ``en``
+     - boolean (0 or 1)
+     - Enable/disable OSK.
+
+   * - ``raw``
+     - integer
+     - Target amplitude code. 14-bit ASF field. Range: :math:`[0, 16383]`.
+       The physical output amplitude is ``raw * scale`` where ``scale`` is read
+       from the ``phy`` channel.
+
+   * - ``raw_roc``
+     - /s
+     - Amplitude ramp rate. Writing a non-zero value enables automatic OSK
+       and selects the closest hardware step size. Writing ``0`` disables
+       automatic ramping (manual control via ``raw``). Writing the maximum
+       available value enables pin-controlled immediate transition.
+
+   * - ``raw_roc_available``
+     - /s
+     - Lists the available ``raw_roc`` values based on the current
+       ``sampling_frequency``. The first value is always ``0`` (disabled) and
+       the last value corresponds to pin-controlled immediate mode.
+
+   * - ``sampling_frequency``
+     - Hz
+     - OSK ramp clock. Controlled by an integer divider; the written value
+       is adjusted to the nearest supported rate.
+
+Usage examples
+^^^^^^^^^^^^^^
+
+Enable OSK with automatic ramping:
+
+.. code-block:: bash
+
+  # Set ramp rate 1MHz
+  echo 1000000 > /sys/bus/iio/devices/iio\:device0/out_altcurrent150_sampling_frequency
+
+  # Check available rate of change values
+  cat /sys/bus/iio/devices/iio\:device0/out_altcurrent150_raw_roc_available
+  0 1000000 2000000 4000000 8000000 16383000000
+
+  # Enable automatic OSK with a rate of change of 8000000 raw units/s
+  echo 8000000 > /sys/bus/iio/devices/iio\:device0/out_altcurrent150_raw_roc
+
+  # Enable OSK
+  echo 1 > /sys/bus/iio/devices/iio\:device0/out_altcurrent150_en
+
+Enable pin-controlled immediate OSK:
+
+.. code-block:: bash
+
+  # Enable OSK in manual mode (no ramp)
+  echo 0 > /sys/bus/iio/devices/iio\:device0/out_altcurrent150_raw_roc
+  echo 1 > /sys/bus/iio/devices/iio\:device0/out_altcurrent150_en
+
+  # Set target amplitude to full scale
+  echo 16383 > /sys/bus/iio/devices/iio\:device0/out_altcurrent150_raw
+
+Physical channel
+================
+
+The ``phy`` channel provides device-level control:
+
+.. flat-table::
+   :header-rows: 1
+
+   * - Attribute
+     - Unit
+     - Description
+
+   * - ``sampling_frequency``
+     - Hz
+     - System clock (SYSCLK) frequency. When the internal PLL is enabled
+       (via the ``adi,pll-enable`` devicetree property), configures the PLL
+       multiplier (Range: :math:`[420, 1000]` MHz). Without PLL, the reference
+       clock can only be divided by 2.
+
+   * - ``scale``
+     - mA/LSB
+     - Full-scale DAC output current per amplitude code LSB, which is evaluated
+       as :math:`I_{FS} / 2^{14}`. Shared across all ``altcurrent`` channels.
+       Setting this attribute reconfigures the auxiliary DAC full-scale code and
+       updates the effective amplitude resolution for single tone profiles,
+       DRG amplitude ramps and OSK.
+
+   * - ``powerdown``
+     - boolean (0 or 1)
+     - Software power-down. Writing 1 powers down the digital core, DAC,
+       reference clock input and auxiliary DAC simultaneously.
+
+Usage examples
+--------------
+
+Set the system clock to 1 GHz:
+
+.. code-block:: bash
+
+  echo 1000000000 > /sys/bus/iio/devices/iio\:device0/out_altcurrent100_sampling_frequency
+
+Read current system clock frequency:
+
+.. code-block:: bash
+
+  cat /sys/bus/iio/devices/iio\:device0/out_altcurrent100_sampling_frequency
+
+Power down the device:
+
+.. code-block:: bash
+
+  echo 1 > /sys/bus/iio/devices/iio\:device0/out_altcurrent100_powerdown
diff --git a/Documentation/iio/ade9000.rst b/Documentation/iio/ade9000.rst
index c9ff702a42512b..f416600052237f 100644
--- a/Documentation/iio/ade9000.rst
+++ b/Documentation/iio/ade9000.rst
@@ -1,24 +1,30 @@
 .. SPDX-License-Identifier: GPL-2.0
 
-===============
-ADE9000 driver
-===============
+======================
+ADE9000/ADE9078 driver
+======================
 
-This driver supports Analog Device's ADE9000 energy measurement IC on SPI bus.
+This driver supports Analog Devices' ADE9000 and ADE9078 energy measurement
+ICs on SPI bus.
 
 1. Supported devices
 ====================
 
 * `ADE9000 <https://www.analog.com/media/en/technical-documentation/data-sheets/ADE9000.pdf>`_
-
-The ADE9000 is a highly accurate, fully integrated, multiphase energy and power
-quality monitoring device. Superior analog performance and a digital signal
-processing (DSP) core enable accurate energy monitoring over a wide dynamic
-range. An integrated high end reference ensures low drift over temperature
-with a combined drift of less than ±25 ppm/°C maximum for the entire channel
-including a programmable gain amplifier (PGA) and an analog-to-digital
+* `ADE9078 <https://www.analog.com/media/en/technical-documentation/data-sheets/ADE9078.pdf>`_
+
+The ADE9000 and ADE9078 are highly accurate, fully integrated, multiphase energy
+and power quality monitoring devices. Superior analog performance and a digital
+signal processing (DSP) core enable accurate energy monitoring over a wide
+dynamic range. An integrated high end reference ensures low drift over
+temperature with a combined drift of less than ±25 ppm/°C maximum for the entire
+channel including a programmable gain amplifier (PGA) and an analog-to-digital
 converter (ADC).
 
+The ADE9078 does not provide the on-chip dip/swell (sag) detection available on
+the ADE9000, so the RMS voltage swell/dip events described below are only
+available on the ADE9000.
+
 2. Device attributes
 ====================
 
diff --git a/Documentation/iio/adis16475.rst b/Documentation/iio/adis16475.rst
index 89a388490ab735..0e004f016199f0 100644
--- a/Documentation/iio/adis16475.rst
+++ b/Documentation/iio/adis16475.rst
@@ -4,7 +4,7 @@
 ADIS16475 driver
 ================
 
-This driver supports Analog Device's IMUs on SPI bus.
+This driver supports Analog Devices' IMUs on SPI bus.
 
 1. Supported devices
 ====================
diff --git a/Documentation/iio/adis16480.rst b/Documentation/iio/adis16480.rst
index cce5f3e0174158..7c259070aaaf39 100644
--- a/Documentation/iio/adis16480.rst
+++ b/Documentation/iio/adis16480.rst
@@ -4,7 +4,7 @@
 ADIS16480 driver
 ================
 
-This driver supports Analog Device's IMUs on SPI bus.
+This driver supports Analog Devices' IMUs on SPI bus.
 
 1. Supported devices
 ====================
diff --git a/Documentation/iio/adis16550.rst b/Documentation/iio/adis16550.rst
index c9bbc0a857b03b..c0d5a975e6ca04 100644
--- a/Documentation/iio/adis16550.rst
+++ b/Documentation/iio/adis16550.rst
@@ -4,7 +4,7 @@
 ADIS16550 driver
 ================
 
-This driver supports Analog Device's IMUs on SPI bus.
+This driver supports Analog Devices' IMUs on SPI bus.
 
 1. Supported devices
 ====================
diff --git a/Documentation/iio/adxl313.rst b/Documentation/iio/adxl313.rst
index 3641193bf64778..c448f10745e72a 100644
--- a/Documentation/iio/adxl313.rst
+++ b/Documentation/iio/adxl313.rst
@@ -4,7 +4,7 @@
 ADXL313 driver
 ===============
 
-This driver supports Analog Device's ADXL313 on SPI/I2C bus.
+This driver supports Analog Devices' ADXL313 on SPI/I2C bus.
 
 1. Supported devices
 ====================
diff --git a/Documentation/iio/adxl345.rst b/Documentation/iio/adxl345.rst
index 0aa33a85241280..28c71fbd45e5f0 100644
--- a/Documentation/iio/adxl345.rst
+++ b/Documentation/iio/adxl345.rst
@@ -4,7 +4,7 @@
 ADXL345 driver
 ===============
 
-This driver supports Analog Device's ADXL345/375 on SPI/I2C bus.
+This driver supports Analog Devices' ADXL345/375 on SPI/I2C bus.
 
 1. Supported Devices
 ====================
diff --git a/Documentation/iio/adxl380.rst b/Documentation/iio/adxl380.rst
index 654d4c0e84a197..969ded211cc939 100644
--- a/Documentation/iio/adxl380.rst
+++ b/Documentation/iio/adxl380.rst
@@ -4,7 +4,7 @@
 ADXL380 driver
 ===============
 
-This driver supports Analog Device's ADXL380/382 on SPI/I2C bus.
+This driver supports Analog Devices' ADXL380/382 on SPI/I2C bus.
 
 1. Supported devices
 ====================
@@ -39,13 +39,13 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``.
 +---------------------------------------------------+----------------------------------------------------------+
 | in_accel_scale                                    | Scale for the accelerometer channels.                    |
 +---------------------------------------------------+----------------------------------------------------------+
-| in_accel_filter_high_pass_3db_frequency           | Low pass filter bandwidth.                               |
+| in_accel_filter_high_pass_3db_frequency           | High pass filter bandwidth.                              |
 +---------------------------------------------------+----------------------------------------------------------+
-| in_accel_filter_high_pass_3db_frequency_available | Available low pass filter bandwidth configurations.      |
+| in_accel_filter_high_pass_3db_frequency_available | Available high pass filter bandwidth configurations.     |
 +---------------------------------------------------+----------------------------------------------------------+
-| in_accel_filter_low_pass_3db_frequency            | High pass filter bandwidth.                              |
+| in_accel_filter_low_pass_3db_frequency            | Low pass filter bandwidth.                               |
 +---------------------------------------------------+----------------------------------------------------------+
-| in_accel_filter_low_pass_3db_frequency_available  | Available high pass filter bandwidth configurations.     |
+| in_accel_filter_low_pass_3db_frequency_available  | Available low pass filter bandwidth configurations.      |
 +---------------------------------------------------+----------------------------------------------------------+
 | in_accel_x_calibbias                              | Calibration offset for the X-axis accelerometer channel. |
 +---------------------------------------------------+----------------------------------------------------------+
diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst
index b02b879b053a1b..d0fb32f4b80f07 100644
--- a/Documentation/iio/index.rst
+++ b/Documentation/iio/index.rst
@@ -29,7 +29,9 @@ Industrial I/O Kernel Drivers
    ad7380
    ad7606
    ad7625
+   ad7768
    ad7944
+   ad9910
    ade9000
    adf41513
    adis16475
diff --git a/Documentation/input/devices/yealink.rst b/Documentation/input/devices/yealink.rst
index bb5a1aafeca2a0..53191a3c83caf1 100644
--- a/Documentation/input/devices/yealink.rst
+++ b/Documentation/input/devices/yealink.rst
@@ -120,9 +120,10 @@ Reading /sys/../lineX will return the format string with its current value.
 Writing to /sys/../lineX will set the corresponding LCD line.
 
  - Excess characters are ignored.
- - If less characters are written than allowed, the remaining digits are
+ - If fewer characters than allowed are written, the remaining digits are
    unchanged.
- - The tab '\t'and '\n' char does not overwrite the original content.
+ - The tab ``\t`` and newline ``\n`` characters do not overwrite the original
+   content.
  - Writing a space to an icon will always hide its content.
 
   Example::
@@ -205,7 +206,7 @@ Troubleshooting
 :Q: Module yealink compiled and installed without any problem but phone
     is not initialized and does not react to any actions.
 :A: If you see something like:
-    hiddev0: USB HID v1.00 Device [Yealink Network Technology Ltd. VOIP USB Phone
+    hiddev0: USB HID v1.00 Device [Yealink Network Technology Ltd. VOIP USB Phone]
     in dmesg, it means that the hid driver has grabbed the device first. Try to
     load module yealink before any other usb hid driver. Please see the
     instructions provided by your distribution on module configuration.
diff --git a/Documentation/input/notifier.rst b/Documentation/input/notifier.rst
index 824379399e611e..e7facd1a1003e3 100644
--- a/Documentation/input/notifier.rst
+++ b/Documentation/input/notifier.rst
@@ -31,7 +31,7 @@ In a rough C snippet, we have::
     kbd_keycode(keycode) {
 	...
 	params.value = keycode;
-	if (notifier_call_chain(KBD_KEYCODE,&params) == NOTIFY_STOP)
+	if ((notifier_call_chain(KBD_KEYCODE,&params) == NOTIFY_STOP)
 	    || !bound) {
 		notifier_call_chain(KBD_UNBOUND_KEYCODE,&params);
 		return;
diff --git a/Documentation/kbuild/kbuild.rst b/Documentation/kbuild/kbuild.rst
index 5a9013bacfb75c..e71a692a559ac2 100644
--- a/Documentation/kbuild/kbuild.rst
+++ b/Documentation/kbuild/kbuild.rst
@@ -110,6 +110,23 @@ HOSTLDLIBS
 ----------
 Additional libraries to link against when building host programs.
 
+KGZIP
+-----
+The gzip program used for compressed modules, packaging and the embedded
+kernel configuration. Defaults to gzip.
+
+KPGZIP
+------
+The gzip program used for the compressed kernel image. Defaults to pigz (a
+parallel implementation of gzip), run under scripts/jobserver-exec so that it
+uses the job slots make has free, if pigz is installed, otherwise to
+``KGZIP``. Set ``KPGZIP=`` on the make command line to use another program.
+
+KBZIP2, KLZOP, LZMA, LZ4, XZ, ZSTD
+----------------------------------
+The compressor programs for the other formats. Each defaults to the program
+of the same name.
+
 .. _userkbuildflags:
 
 USERCFLAGS
diff --git a/Documentation/kbuild/kconfig-language.rst b/Documentation/kbuild/kconfig-language.rst
index d9338407c1c68d..00402d43e0dc53 100644
--- a/Documentation/kbuild/kconfig-language.rst
+++ b/Documentation/kbuild/kconfig-language.rst
@@ -113,11 +113,22 @@ applicable everywhere (see syntax).
 
 - type definition + default value::
 
-	"def_bool"/"def_tristate" <expr> ["if" <expr>]
+	"def_bool" <expr> ["if" <expr>]
+	"def_tristate" <expr> ["if" <expr>]
+	"def_string" <expr> ["if" <expr>]
+	"def_int" <expr> ["if" <expr>]
+	"def_hex" <expr> ["if" <expr>]
 
   This is a shorthand notation for a type definition plus a value.
   Optionally dependencies for this default value can be added with "if".
 
+  The shorthand supplies the type once, and is otherwise an ordinary
+  default: it is the first entry of the list described above, so any
+  further "default" entries still apply when its "if" is not met. Since
+  that leaves the type definition inside one arm of a list, spelling the
+  type out on its own line reads better for a symbol with several
+  defaults.
+
 - dependencies: "depends on" <expr> ["if" <expr>]
 
   This defines a dependency for this menu entry. If multiple
diff --git a/Documentation/kbuild/kconfig-macro-language.rst b/Documentation/kbuild/kconfig-macro-language.rst
index 6163467f6ae4f0..c427bece3148a6 100644
--- a/Documentation/kbuild/kconfig-macro-language.rst
+++ b/Documentation/kbuild/kconfig-macro-language.rst
@@ -225,7 +225,7 @@ not work::
             $(MY_TYPE) "foo"
             default y
 
-Obviously from the design, $(shell command) is expanded in the textual
+Obviously from the design, $(shell,command) is expanded in the textual
 substitution phase. You cannot pass symbols to the 'shell' function.
 
 The following does not work as expected::
@@ -236,12 +236,12 @@ The following does not work as expected::
             default "-mlittle-endian" if CPU_LITTLE_ENDIAN
 
     config CC_HAS_ENDIAN_FLAG
-            def_bool $(shell $(srctree)/scripts/gcc-check-flag ENDIAN_FLAG)
+            def_bool $(shell,$(srctree)/scripts/gcc-check-flag ENDIAN_FLAG)
 
 Instead, you can do like follows so that any function call is statically
 expanded::
 
     config CC_HAS_ENDIAN_FLAG
             bool
-            default $(shell $(srctree)/scripts/gcc-check-flag -mbig-endian) if CPU_BIG_ENDIAN
-            default $(shell $(srctree)/scripts/gcc-check-flag -mlittle-endian) if CPU_LITTLE_ENDIAN
+            default $(shell,$(srctree)/scripts/gcc-check-flag -mbig-endian) if CPU_BIG_ENDIAN
+            default $(shell,$(srctree)/scripts/gcc-check-flag -mlittle-endian) if CPU_LITTLE_ENDIAN
diff --git a/Documentation/leds/leds-class.rst b/Documentation/leds/leds-class.rst
index 3913966cfdac04..464e54cabf52ca 100644
--- a/Documentation/leds/leds-class.rst
+++ b/Documentation/leds/leds-class.rst
@@ -242,6 +242,9 @@ ops and needs to declare specific support for the supported triggers.
 
 With hw control we refer to the LED driven by hardware.
 
+A sysfs attribute `trigger_may_offload_to_hw` is provided for userspace
+to query supported triggers and their states.
+
 LED driver must define the following value to support hw control:
 
     - hw_control_trigger:
@@ -298,6 +301,15 @@ LED driver must implement the following API to support hw control:
                 Returns a pointer to a struct device or NULL if nothing
                 is currently attached.
 
+LED trigger should implement the following API to indicate hw control:
+    - hw_offloaded:
+                return a boolean indicating if the trigger is currently
+                offloaded to hardware.
+
+                If a trigger doesn't implement this callback, the default
+                value will be true for private triggers and false for generic
+                ones.
+
 LED driver can activate additional modes by default to workaround the
 impossibility of supporting each different mode on the supported trigger.
 Examples are hardcoding the blink speed to a set interval, enable special
@@ -311,9 +323,69 @@ the end use hw_control_set to activate hw control.
 A trigger can use hw_control_get to check if a LED is already in hw control
 and init their flags.
 
+Alternatively, a private trigger can be implemented along with the LED driver if
+the LED's hardware control doesn't fit any generic trigger. To associate the
+private trigger with the LED classdev, their `trigger_type` must be the same. To
+declare that the private trigger provides hardware control for the associated
+LED classdev, set the `hw_control_trigger` string to the trigger's name. Since
+both the LED classdev and the private trigger are in the same LED driver, it's
+not necessary for them to coordinate via `hw_control_*` callbacks.
+
 When the LED is in hw control, no software blink is possible and doing so
 will effectively disable hw control.
 
+Hardware-initiated trigger transition
+=====================================
+
+Some hardware can autonomously activate/deactivate hardware control. After that,
+the LED hardware notifies the LED driver.
+
+If the driver can detect such transitions and thus wants to notify the LED core
+to update the current trigger then the `LED_TRIG_HW_CHANGED` flag must be set in
+flags before registering. To update the current trigger accordingly, call
+`led_trigger_notify_hw_control_changed` on the LED classdev.
+
+This capability is restricted to the LED device's private trigger. The private
+trigger must have been properly registered (see above) and named after
+`hw_control_trigger`.
+
+Only two transitions are defined:
+
+- "none" => private trigger:
+        This happens when the hardware autonomously activates hardware control
+        and when "none" (i.e., no trigger) is currently active. If the private
+        trigger is already active when the method is called, this is essentially
+        a no-op.
+
+        The activation sequence for the private trigger will be executed as
+        normal.
+
+        The LED driver and its private trigger must be able to handle the
+        activation sequence even if the hardware is currently in hardware
+        control.
+
+        If error occurs in the activation sequence, the LED Trigger core reverts
+        the effective trigger to "none".
+
+- private trigger => "none"
+        This happens when the hardware autonomously deactivates hardware control
+        and when the private trigger is currently active. If "none" (i.e., no
+        trigger) is active when the method is called, this is essentially a
+        no-op.
+
+        The deactivation sequence for the private trigger will be executed as
+        normal, except that the current LED brightness is retained. The reason
+        for keeping the brightness unchanged is that some hardware may choose a
+        specific brightness instead of simply turning off the LED after
+        autonomously deactivating hardware control.
+
+        The LED driver and its private trigger must be able to handle the
+        deactivation sequence even if the hardware is not currently in hardware
+        control.
+
+If the current trigger is neither the private trigger nor "none", no transition
+will be made.
+
 Known Issues
 ============
 
diff --git a/Documentation/leds/leds-st1202.rst b/Documentation/leds/leds-st1202.rst
index ed32eb3a27d450..28479a11238992 100644
--- a/Documentation/leds/leds-st1202.rst
+++ b/Documentation/leds/leds-st1202.rst
@@ -20,6 +20,11 @@ brightness (PWM) and duration must be written to hw_pattern.
 - Min pattern duration: 22 ms
 - Max pattern duration: 5610 ms
 
+If the device tree sets max-brightness, that value replaces 255 as the
+upper limit, and a step equal to it drives the channel at full scale.
+The hardware cannot go above 255, so with a larger max-brightness every
+step from 255 upwards is full scale.
+
 The format of the hardware pattern values should be:
 "brightness duration brightness duration ..."
 
diff --git a/Documentation/livepatch/api.rst b/Documentation/livepatch/api.rst
index 78944b63d74b52..2a98c517da9da0 100644
--- a/Documentation/livepatch/api.rst
+++ b/Documentation/livepatch/api.rst
@@ -27,4 +27,7 @@ Object Types
 ============
 
 .. kernel-doc:: include/linux/livepatch.h
-   :identifiers: klp_patch klp_object klp_func klp_callbacks klp_state
+   :identifiers: klp_patch klp_object klp_func klp_state
+
+.. kernel-doc:: include/linux/livepatch_external.h
+   :identifiers: klp_callbacks
diff --git a/Documentation/misc-devices/spear-pcie-gadget.rst b/Documentation/misc-devices/spear-pcie-gadget.rst
index 09b9d6c7ac1545..13333a20944c39 100644
--- a/Documentation/misc-devices/spear-pcie-gadget.rst
+++ b/Documentation/misc-devices/spear-pcie-gadget.rst
@@ -37,9 +37,9 @@ read behavior of nodes:
 -----------------------
 
 =============== ==============================================================
-link 		gives ltssm status.
-int_type 	type of supported interrupt
-no_of_msi 	zero if MSI is not enabled by host. A positive value is the
+link		gives ltssm status.
+int_type	type of supported interrupt
+no_of_msi	zero if MSI is not enabled by host. A positive value is the
 		number of MSI vector granted.
 vendor_id	returns programmed vendor id (hex)
 device_id	returns programmed device id(hex)
@@ -53,7 +53,7 @@ write behavior of nodes:
 ------------------------
 
 =============== ================================================================
-link 		write UP to enable ltsmm DOWN to disable
+link		write UP to enable ltsmm DOWN to disable
 int_type	write interrupt type to be configured and (int_type could be
 		INTA, MSI or NO_INT). Select MSI only when you have programmed
 		no_of_msi node.
@@ -65,7 +65,7 @@ device_id	write device id(hex) to be programmed.
 bar0_size	write size of bar0 in hex. default bar0 size is 1000 (hex)
 		bytes.
 bar0_address	write	address of bar0 mapped area in hex. (default mapping of
-		bar0 is SYSRAM1(E0800000). Always program bar size before bar
+		bar0 is SYSRAM1(E0800000)). Always program bar size before bar
 		address. Kernel might modify bar size and address for alignment,
 		so read back bar size and address after writing to cross check.
 bar0_rw_offset	write offset of bar0 for which	bar0_data will write value.
diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst
index aed6cb1cf48310..e82390e77a70ae 100644
--- a/Documentation/mm/damon/design.rst
+++ b/Documentation/mm/damon/design.rst
@@ -249,7 +249,7 @@ For each ``aggregation interval``, it compares the access frequencies
 sum of the two regions' sizes is smaller than the size of total regions divided
 by the ``minimum number of regions``, DAMON merges the two regions.  If the
 resulting number of total regions is still higher than ``maximum number of
-regions``, it repeats the merging with increasing access frequenceis difference
+regions``, it repeats the merging with increasing access frequencies difference
 threshold until the upper-limit of the number of regions is met, or the
 threshold becomes higher than possible maximum value (``aggregation interval``
 divided by ``sampling interval``).   Then, after it reports and clears the
@@ -293,8 +293,15 @@ registration is made by specifying a probe per attribute.  Each of the probe
 specifies a rule to determine if a given memory region has the related
 attribute.  The rule is constructed with multiple filters.  The filters work
 same to :ref:`DAMOS filters <damon_design_damos_filters>` except the supported
-filter types.  Currently only ``anon`` and ``memcg`` filter types are supported
-for data attributes monitoring.
+filter types.  Currently below filter types are supported.
+
+- ``anon``: Same to that for DAMOS filters.
+- ``memcg``: Same to that for DAMOS filters.
+- ``pgidle_unset``: Matches if the page for the memory is marked as not
+  access-idle.
+- ``pgidle_set``: Matches if the page for the memory is marked as access-idle.
+- ``hugepage_size``: Matches if the page for the memory is a part of a hugepage
+  of a given size range.
 
 If such probes are registered, DAMON executes the probes for each region's
 sampling memory when it does the access :ref:`sampling
@@ -305,6 +312,13 @@ Users can therefore know how much of a given DAMON region has a specific data
 attribute by reading the per-region per-probe probe hits counter after each
 aggregation interval.
 
+Users can optionally register probing preparation actions per probe.  If such
+actions are registered, DAMON applies the actions to each region's sampling
+memory before starting the next sampling interval.  Currently only one action,
+``set_pgidle`` is supported.  The action marks the page for the probing target
+memory as access-idle.  This can be useful to be used together with
+``pgidle_unset`` or ``pgidle_set`` probe filter.
+
 This is a sampling based mechanism.  Hence, it is lightweight but the output
 may include some measurement errors.  The output should be used with good
 understanding of statistics.
@@ -431,7 +445,7 @@ users to set the aimed amount of access events to observe via DAMON within
 given time interval.  The target can be specified by the user as a ratio of
 DAMON-observed access events to the theoretical maximum amount of the events
 (``access_bp``) that measured within a given number of aggregations
-(``aggrs``).
+(``aggrs``).  The ratio is in basis point (bp or 1/10,000).
 
 The DAMON-observed access events are calculated in byte granularity based on
 DAMON :ref:`region assumption <damon_design_region_based_sampling>`.  For
@@ -682,9 +696,11 @@ There are two such tuning algorithms that users can select as they need.
   This is the default selection.  If unsure, use this.
 - ``temporal``: More straightforward algorithm.  Tries to achieve the goal as
   fast as possible, using maximum allowed quota, but only for a temporal short
-  time.  When the quota is under-achieved, this algorithm keeps tuning quota to
-  a maximum allowed one.  Once the quota is [over]-achieved, this sets the
-  quota zero.  Useful for deterministic control required environments.
+  time.  When the goal is under-achieved, this algorithm keeps tuning quota to
+  a maximum allowed one.  Once the goal is [over-]achieved, this sets the
+  quota zero.  Useful for deterministic control required environments.  Note
+  that the zero quota is a valid quota, and therefore ``qt_exceeds`` :ref:`stat
+  <damon_design_damos_stat>` will keep increasing in this case.
 
 The goal can be specified with five parameters, namely ``target_metric``,
 ``target_value``, ``current_value``, ``nid`` and ``path``.  The auto-tuning
@@ -701,7 +717,8 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to
   in microseconds that measured from last quota reset to next quota reset.
   DAMOS does the measurement on its own, so only ``target_value`` need to be
   set by users at the initial time.  In other words, DAMOS does self-feedback.
-- ``node_mem_used_bp``: Specific NUMA node's used memory ratio in bp (1/10,000).
+- ``node_mem_used_bp``: Specific NUMA node's used memory ratio in basis point
+  (bp or 1/10,000).
 - ``node_mem_free_bp``: Specific NUMA node's free memory ratio in bp (1/10,000).
 - ``node_memcg_used_bp``: Specific cgroup's node used memory ratio for a
   specific NUMA node, in bp (1/10,000).
@@ -713,6 +730,8 @@ mechanism tries to make ``current_value`` of ``target_metric`` be same to
   bp (1/10,000).
 - ``node_eligible_mem_bp``: Scheme target access pattern-eligible memory ratio
   of a node in bp (1/10,000).
+- ``hugepage_mem_bp``: Total huge page to total used memory ratio in bp
+  (1/10,000).
 
 ``nid`` is optionally required for ``node_mem_used_bp``, ``node_mem_free_bp``,
 ``node_memcg_used_bp``, ``node_memcg_free_bp`` and ``node_eligible_mem_bp`` to
@@ -809,6 +828,9 @@ Below ``type`` of filters are currently supported.
         - Applied to pages that belonging to a given address range.
     - target
         - Applied to pages that belonging to a given DAMON monitoring target.
+    - probe_hits_wsum
+        - Matches to monitoring regions having a given range of :ref:`probe
+          hits weighted sum <damon_design_attrs_only_monitoring>` value.
 - Operations layer handled, supported by only ``paddr`` operations set.
     - anon
         - Applied to pages that containing data that not stored in files.
@@ -846,8 +868,8 @@ scheme's execution.
 - ``nr_applied``: Total number of regions that the scheme is applied.
 - ``sz_applied``: Total size of regions that the scheme is applied.
 - ``qt_exceeds``: Total number of times the quota of the scheme has exceeded.
-- ``nr_snapshots``: Total number of DAMON snapshots that the scheme is tried to
-  be applied.
+- ``nr_snapshots``: Total number of DAMON snapshots that the scheme is
+  completely tried to be applied.
 - ``max_nr_snapshots``: Upper limit of ``nr_snapshots``.
 
 "A scheme is tried to be applied to a region" means DAMOS core logic determined
@@ -871,11 +893,12 @@ action is ``pageout`` while all pages of the region are unreclaimable, applying
 the action to the region will fail.
 
 Unlike normal stats, ``max_nr_snapshots`` is set by users.  If it is set as
-non-zero and ``nr_snapshots`` be same to or greater than ``nr_snapshots``, the
-scheme is deactivated.
+non-zero and ``nr_snapshots`` equals or is greater than ``max_nr_snapshots``,
+the scheme is deactivated.  Note that, unlike watermarks, even if a scheme's
+``nr_snapshots`` reaches ``max_nr_snapshots``, monitoring will not stop.
 
 To know how user-space can read the stats via :ref:`DAMON sysfs interface
-<sysfs_interface>`, refer to :ref:s`stats <sysfs_stats>` part of the
+<sysfs_interface>`, refer to :ref:`stats <sysfs_schemes_stats>` part of the
 documentation.
 
 Regions Walking
diff --git a/Documentation/mm/damon/maintainer-profile.rst b/Documentation/mm/damon/maintainer-profile.rst
index fb2fa00cc9aa1b..a7c1352339378c 100644
--- a/Documentation/mm/damon/maintainer-profile.rst
+++ b/Documentation/mm/damon/maintainer-profile.rst
@@ -106,18 +106,9 @@ AI Review
 
 For patches that are publicly posted to DAMON mailing list
 (damon@lists.linux.dev), AI reviews of the patches will be available at
-sashiko.dev.  The reviews could also be sent as mails to the author of the
-patch.
-
-Patch authors are encouraged to check the AI reviews and share their opinions.
-The sharing could be done as a reply to the mail thread.  Consider reducing the
-recipients list for such sharing, since some people are not really interested
-in AI reviews.  As a rule of thumb, drop stable@vger.kernel.org and individuals
-except DAMON maintainer.
-
-`hkml` also provides a `feature
-<https://github.com/sjp38/hackermail/blob/master/USAGE.md#forwarding-sashikodev-statuscomments-to-mailing-list>`_
-for such sharing.  Please feel free to use the feature.
+sashiko.dev.  The reviews will also be sent as replies to the author of the
+patch and the mailing list.
 
-It is only an optional recommendation.  DAMON maintainer could also ask any
-question about the AI reviews, though.
+Patch authors are encouraged to check the AI reviews and share their opinions
+by replying on the mail thread.  It is only an optional recommendation.  DAMON
+maintainer could also ask any question about the AI reviews, though.
diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst
index 13a79f5d092c0e..e9ae5cd82163af 100644
--- a/Documentation/mm/index.rst
+++ b/Documentation/mm/index.rst
@@ -25,6 +25,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`.
 
    physical_memory
    page_tables
+   kernel-page-tables
    process_addrs
    bootmem
    page_allocation
diff --git a/Documentation/mm/kernel-page-tables.rst b/Documentation/mm/kernel-page-tables.rst
new file mode 100644
index 00000000000000..c16de34b4f79d8
--- /dev/null
+++ b/Documentation/mm/kernel-page-tables.rst
@@ -0,0 +1,410 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==================
+Kernel Page Tables
+==================
+
+Introduction
+============
+
+The kernel page tables are created early during boot and, unlike the page
+tables of user processes, most of them remain static throughout the system
+lifetime.
+
+Every architecture has a direct map (also called linear map) that maps the
+physical memory at a fixed offset, so that a physical address can be
+translated to a kernel virtual address with simple arithmetic. On most
+architectures the direct map is a part of the kernel page tables, with a few
+exceptions described in the `Direct map`_ section below.
+
+On 32-bit systems with high memory the direct map covers only a part of the
+physical memory, see Documentation/mm/highmem.rst.
+
+The vmalloc area, present on every architecture with an MMU, is used for
+allocations of virtually contiguous memory whose backing pages are not
+necessarily physically contiguous, and for mapping of the device memory. Its
+page tables are created and torn down at runtime, see
+Documentation/mm/vmalloc.rst.
+
+Architectures that use the `SPARSEMEM_VMEMMAP` memory model reserve a range of
+kernel address space for the memory map, so that `struct page` objects appear
+as a virtually contiguous array indexed by the page frame number, see
+Documentation/mm/memory-model.rst.
+
+Besides these, the kernel image may be mapped in a dedicated part of the
+kernel address space rather than accessed through the direct map. In that
+case its mapping is an alias of the direct map of the physical memory the
+image occupies.
+
+The rest of the kernel address space is architecture specific. For instance,
+x86 has a region for the EFI runtime services and s390 has a region for the
+code that has to run in the 31-bit addressing mode.
+
+Direct map
+==========
+
+On most architectures the direct map is an ordinary part of the kernel page
+tables. It is created early during boot with the largest pages the hardware
+and the kernel configuration allow.
+
+Several architectures are different.
+
+MIPS
+----
+
+MIPS does not map the physical memory with page tables at all. Instead, a
+part of the kernel virtual address space is a window into the physical
+address space: the hardware translates the addresses that fall into that
+window by a fixed transformation of the address bits, without walking the
+page tables and without using the TLB. The memory attributes, such as
+cacheability and the privilege level required to access the memory, are a
+property of the window rather than of an individual page.
+
+There are no page table entries describing the direct map, so its properties
+cannot be changed for an individual page. On 32-bit systems the window covers
+only 512 MiB of the physical memory, so everything above that is high memory.
+
+LoongArch
+---------
+
+Like MIPS, LoongArch maps the physical memory with a hardware window. The
+window covers 256 TiB of the physical address space on 64-bit and 512 MiB on
+32-bit systems, and everything above that is high memory.
+
+The window occupies the lower part of the kernel address space. The upper
+part, which includes the vmalloc area, is mapped with kernel page tables.
+
+PowerPC with the hash MMU
+-------------------------
+
+On 64-bit PowerPC systems with the hash MMU the direct map does not exist in
+the Linux page tables. It is installed into the hardware hash page table early
+during boot.
+
+Modifying such mappings requires updating the hash page table directly, and
+the hash MMU code implements this only for the kernel image permissions,
+`debug_pagealloc` and KFENCE.
+
+With the radix MMU the direct map is a part of the ordinary kernel page
+tables.
+
+Modifying the kernel page tables
+================================
+
+Except for the vmalloc area, the kernel page tables are mostly static. Still,
+there are cases when the permissions of existing kernel mappings have to be
+updated, for instance when a module is loaded and its text becomes read-only
+and executable, or when a page is temporarily removed from the direct map to
+reduce its exposure.
+
+There are two families of functions for this, both declared in
+`include/linux/set_memory.h`:
+
+* `set_memory_*()` change permissions of an arbitrary kernel mapping. They
+  take a kernel virtual address and the number of pages.
+
+* `set_direct_map_*()` change permissions of the direct mapping for the range
+  of page frames starting at the page represented by a `struct page`. They take
+  a `struct page` pointer and the number of pages.
+
+Architectures that implement `set_memory()` select `CONFIG_ARCH_HAS_SET_MEMORY`
+
+Architectures that implement `set_direct_map()` select
+`CONFIG_ARCH_HAS_SET_DIRECT_MAP`.
+
+Common semantics
+----------------
+
+Ranges
+~~~~~~
+
+The `set_memory()` functions expect a range described by a start address and a
+number of pages. The address must be page aligned and the entire range must be
+covered by page table entries the architecture knows how to update.
+
+The entries may be marked as not present, set_memory_p() and set_memory_valid()
+exist exactly to bring such a mapping back.
+
+When a range does not qualify, an architecture will usually say so by returning
+an error and sometimes by a WARN()ing as well, unless it prefers to keep it to
+itself and return success, see `Architecture specific differences`_.
+
+The `set_direct_map()` functions expect a range described by the first
+`struct page` and a number of pages, and they update the direct map starting
+at that page.
+
+The pages that follow the first one are updated regardless of what they are, so
+the caller has to make sure that the range does not extend beyond the memory it
+owns.
+
+Some architectures cannot split a large mapping, and they reject a range that
+is a part of one, see `Architecture specific differences`_.
+
+Calling `set_memory()` with the number of pages set to zero is a no-op that
+returns success, except on arm64, where doing nothing to the wrong address is
+still an error.
+
+Aliases
+~~~~~~~
+
+A physical page may be mapped several times, for instance in the direct map
+and in the vmalloc area, and the permissions of these mappings may differ.
+
+Whether the direct map alias is updated by a `set_memory()` call, and
+which permission bits make it there, is entirely up to the architecture, and
+there is not much agreement between them, see
+`Architecture specific differences`_.
+
+Relying on that is a gamble; code that needs the direct map alias to change
+should say so with the `set_direct_map()` APIs.
+
+Failures and partial updates
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Both families of functions return 0 on success and a negative error code on
+failure. The most common failures are `-EINVAL` for a range that cannot be
+handled and `-ENOMEM` when splitting a large mapping fails to allocate a page
+table.
+
+Some of the range checks happen upfront, so their failure leaves the page
+tables unchanged.
+
+**The update is not atomic and there is no rollback.**
+
+The architecture implementations walk the range and update the page tables as
+they go, and they stop at the first entry that cannot be updated. When an error
+is returned, an arbitrary prefix of the range may have been updated already,
+and the same is true for the direct map alias when the architecture updates it.
+
+For example, when a range spans two large mappings and splitting the second
+one fails because there is no memory for a page table, the first one is
+already split and updated by the time the error is returned.
+
+None of the APIs inform the caller where in the range they failed, so reverting
+such a partial update is possible in principle but unreliable in practice.
+
+The caller may try to restore the original permissions over the entire range,
+but that revert goes through the very code that has just failed, which does
+not inspire much confidence.
+
+The callers should therefore be prepared to give up on the memory in question:
+leak it or panic, but never return it to the allocator before the permissions
+are restored and never assume that the requested permissions are in effect.
+Neither option is appealing, but both beat handing out a page whose
+permissions nobody knows.
+
+TLB flushing
+~~~~~~~~~~~~
+
+The `set_memory()` functions flush the TLB for the affected range before
+they return, so that the new permissions are in effect for every CPU.
+
+The `set_direct_map()` functions have `_noflush` in their names because when
+they were first introduced on x86, the intention was that the TLB flushing
+could be optimized by letting the caller handle it.
+
+For example, vfree() batches the TLB flushes for the areas allocated with
+`VM_FLUSH_RESET_PERMS`, folding the flush of the direct map into the flush it
+has to do for the vmalloc mapping anyway.
+
+Some architectures flush the TLB in the `_noflush` functions anyway, so the
+name is best read as a suggestion. It does not make the flush by the caller
+unnecessary, it only makes it more expensive.
+
+A caller that changes the permissions to more restrictive ones must flush the
+TLB itself.
+
+Context
+~~~~~~~
+
+Architectures use different locking mechanisms to synchronize kernel page table
+updates, and both families of functions may sleep, for instance when they
+allocate memory to split a large mapping.
+
+The caller cannot presume it is safe to call these APIs from an atomic context.
+
+The `set_direct_map()` functions must not be called for high memory pages,
+which have no direct map alias to update.
+
+Unimplemented APIs silently succeed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When an architecture does not implement these APIs, the generic stubs in
+`include/linux/set_memory.h` return 0, that is, they report success for work
+they have no intention of doing.
+
+The same happens inside several architecture implementations. The arm64
+`set_direct_map()` functions return 0 when can_set_direct_map() is false, and
+the LoongArch `set_memory()` functions return 0 for the addresses in its
+windowed direct mapping, which is not backed by page tables at all.
+
+Returning success without actually updating the page tables is a deliberate
+trade-off that keeps the callers free of `#ifdef`\ s, but they have to realize:
+
+**a return value of 0 does not imply that the permissions were actually
+changed.**
+
+For best-effort hardening that is good enough. When correctness or security
+depends on the permissions, the caller has to make sure the architecture
+really implements what it needs. For instance, secretmem depends on
+`CONFIG_ARCH_HAS_SET_DIRECT_MAP` and calls can_set_direct_map() at runtime.
+
+Architecture specific differences
+=================================
+
+The APIs are implemented by seven architectures and, beyond the common
+semantics described above, their behaviour differs in several respects.
+
+Which of the APIs are implemented:
+
+=========  =====================  =========================
+Arch       `ARCH_HAS_SET_MEMORY`  `ARCH_HAS_SET_DIRECT_MAP`
+=========  =====================  =========================
+arm        yes                    no
+arm64      yes                    yes
+loongarch  yes                    yes
+powerpc    yes                    no
+riscv      yes (MMU only)         yes (MMU only)
+s390       yes                    yes
+x86        yes                    yes
+=========  =====================  =========================
+
+Only set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() are
+available everywhere, and even these are not universal: some architectures
+restrict the ranges that can be modified, for instance arm64 rejects direct map
+addresses and accepts only addresses in vmalloc space. The architectures that
+may run on hardware without an execute permission bit, like x86 and s390,
+silently skip the update of the executable bit.
+
+set_memory_rox() has a generic implementation that calls set_memory_ro() and
+set_memory_x() in turn; PowerPC, s390 and x86 override it with a single-pass
+version.
+
+Several architectures define additional APIs. Some of those may have identical
+semantics but different names. For example, making a mapping present or not
+present is spelled differently: set_memory_p() and set_memory_np() on x86 and
+PowerPC, set_memory_valid() on arm and arm64.
+
+The direct map and the kernel image are normally mapped with the largest
+possible pages, and changing the permissions of a single page inside such a
+mapping requires splitting it, which not every architecture can do.
+
+arm
+---
+
+* Does not implement `set_direct_map()`.
+* Provides set_memory_valid().
+* set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() accept
+  only vmalloc and module addresses.
+* set_memory_valid() accepts any address.
+* Does not update mapping aliases.
+
+arm64
+-----
+
+* Provides set_memory_valid().
+* Provides the memory encryption helpers, which are effective only when the
+  kernel runs as a confidential guest.
+* set_memory_ro(), set_memory_rw(), set_memory_x() and set_memory_nx() accept
+  only vmalloc and module addresses:
+
+  - the range must fit in the VM area that contains its start
+  - the VM area must have `VM_ALLOC` set and `VM_ALLOW_HUGE_VMAP` clear
+
+* set_memory_valid() accepts any address.
+* The encryption helpers accept only direct map addresses.
+* The `set_memory()` functions propagate the read-only and the read-write
+  changes to the direct map alias when `rodata=full` is in effect.
+* Splits leaf mappings before the update on the hardware that supports it.
+  The split itself may be partial when it fails midway, but the permissions are
+  left untouched in that case.
+  Without support for splitting large mappings, a range that covers a leaf
+  entry only partially fails with a WARN()ing and `-EINVAL`.
+  If a range spans one or more full leaf entries and a partial leaf entry, the
+  permissions of the full leaf entries are updated before the failure.
+* The `set_direct_map()` functions return 0 without doing anything when the
+  direct map cannot be modified, see can_set_direct_map().
+* Skips the TLB flush in `set_memory()` when the update only turns an invalid
+  mapping into a valid one.
+* Does not flush TLB in `set_direct_map()`.
+
+LoongArch
+---------
+
+* Accepts only the addresses above the hardware window and silently returns
+  success for the rest, see `Direct map`_.
+* Does not update mapping aliases.
+* Does not split anything: a leaf entry is updated as a whole, which changes
+  the permissions of the entire large mapping.
+* Flushes the TLB in `set_direct_map()`.
+
+PowerPC
+-------
+
+* Does not implement `set_direct_map()`.
+* Provides set_memory_np() and set_memory_p().
+* Rejects huge vmalloc mappings.
+* With the hash MMU on 64-bit systems accepts nothing but the vmalloc and the
+  I/O regions.
+* With the radix MMU accepts direct map addresses, but still cannot split a
+  large mapping.
+* Does not update mapping aliases.
+
+riscv
+-----
+
+* Implements both APIs only when the MMU is enabled.
+* Provides set_memory_rw_nx().
+* The `set_memory()` functions accept any mapped kernel address, including the
+  direct map, but a vmalloc range must have the `pages` array of its VM area
+  populated, which rules out vmap() and ioremap() mappings.
+* On 64-bit systems the `set_memory()` functions update the direct map alias of
+  a vmalloc range, including the executable bit.
+* Does not split vmalloc ranges: a leaf entry is updated as a whole, which
+  changes the permissions of the entire large mapping.
+* Splits the direct map on 64-bit systems.
+* Flushes the TLB in `set_direct_map()`.
+
+s390
+----
+
+* Provides set_memory_4k(), set_memory_rwnx() and the
+  `__set_memory_*(start, end)` variants that take a range rather than a page
+  count.
+* The `set_memory()` functions accept any mapped kernel address, including the
+  direct map.
+* Skips the update of the executable bit when the hardware has no support for
+  it.
+* The `set_memory()` functions propagate only the read-only and read-write
+  changes to the direct map alias of a `VM_ALLOC` area, and deliberately not
+  the executable bit.
+* Splits leaf PUD and PMD entries when the range is not aligned to them or when
+  set_memory_4k() is requested.
+* Updates the page table entries with instructions that invalidate the
+  corresponding TLB entries, so no separate flush is needed anywhere.
+
+x86
+---
+
+* Provides the largest set of operations on top of the common ones:
+
+  - the cache attribute helpers: set_memory_uc(), set_memory_wc(),
+    set_memory_wb()
+  - presence control: set_memory_np() and set_memory_p()
+  - set_memory_4k()
+  - set_memory_global() and set_memory_nonglobal()
+  - the array variants that operate on `struct page` arrays or arrays of
+    virtual addresses
+  - memory encryption: set_memory_encrypted() and set_memory_decrypted()
+
+* The `set_memory()` functions accept any mapped kernel address, including the
+  direct map, and silently succeed for the unmapped holes inside it.
+* Does nothing in set_memory_x() and set_memory_nx() when the CPU has no
+  execute permission bit.
+* The `set_memory()` functions apply the change to the direct map alias and,
+  for the kernel image, to the high kernel mapping. The NX bit is never
+  propagated, so that the direct map stays non-executable.
+* Splits large mappings on demand and can collapse them back when the
+  permissions become uniform again.
+* Does not flush the TLB in `set_direct_map()`, but splitting a large
+  mapping flushes it anyway.
diff --git a/Documentation/mm/ksm.rst b/Documentation/mm/ksm.rst
index 2b4f72f1f95343..5a80573162def0 100644
--- a/Documentation/mm/ksm.rst
+++ b/Documentation/mm/ksm.rst
@@ -78,7 +78,7 @@ The frequency of such scans is defined by
 Reference
 ---------
 .. kernel-doc:: mm/ksm.c
-   :functions: mm_slot ksm_scan stable_node rmap_item
+   :functions: ksm_mm_slot ksm_scan ksm_stable_node ksm_rmap_item
 
 --
 Izik Eidus,
diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst
index a6bd3fe6423ad6..bd027377dff6ee 100644
--- a/Documentation/mm/page_owner.rst
+++ b/Documentation/mm/page_owner.rst
@@ -199,6 +199,7 @@ Usage
 		-p		Sort by pid.
 		-P		Sort by tgid.
 		-n		Sort by task command name.
+		-M		Sort by module name.
 		-r		Sort by memory release time.
 		-s		Sort by stack trace.
 		-t		Sort by times (default).
@@ -240,8 +241,10 @@ Usage
 					group ID numbers appear in <tgidlist>.
 		--name <cmdlist>	Select by task command name. This selects the blocks whose
 					task command name appear in <cmdlist>.
+		--module <modlist>	Select by module name. This selects the blocks whose
+					module name appear in <modlist>.
 
-		<pidlist>, <tgidlist>, <cmdlist> are single arguments in the form of a comma-separated list,
+		<pidlist>, <tgidlist>, <cmdlist>, <modlist> are single arguments in the form of a comma-separated list,
 		which offers a way to specify individual selecting rules.
 
 
@@ -249,6 +252,7 @@ Usage
 				./page_owner_sort <input> <output> --pid=1
 				./page_owner_sort <input> <output> --tgid=1,2,3
 				./page_owner_sort <input> <output> --name name1,name2
+				./page_owner_sort <input> <output> --module xfs,ext4
 
 STANDARD FORMAT SPECIFIERS
 ==========================
@@ -265,6 +269,7 @@ STANDARD FORMAT SPECIFIERS
 	ft		free_ts		timestamp of the page when it was released
 	at		alloc_ts	timestamp of the page when it was allocated
 	ator		allocator	memory allocator for pages
+	mod		module		kernel module name
 
   For --cull option:
 
@@ -275,6 +280,7 @@ STANDARD FORMAT SPECIFIERS
 	f		free		whether the page has been released or not
 	st		stacktrace	stack trace of the page allocation
 	ator		allocator	memory allocator for pages
+	mod		module		kernel module name
 
 Filtering page_owner output
 ============================
diff --git a/Documentation/mm/physical_memory.rst b/Documentation/mm/physical_memory.rst
index a09407d7297368..2e67e8b23a994a 100644
--- a/Documentation/mm/physical_memory.rst
+++ b/Documentation/mm/physical_memory.rst
@@ -480,6 +480,12 @@ General
   ``present_pages`` should use ``get_online_mems()`` to get a stable value. It
   is initialized by ``calculate_node_totalpages()``.
 
+``pages_with_online_memmap``
+  Pages within the zone that have an online memory map: present pages and
+  memory holes whose memory map has been initialized and
+  ``pfn_to_online_page()`` succeeds. See the comment for
+  ``pages_with_online_memmap`` in ``include/linux/mmzone.h`` for more details.
+
 ``present_early_pages``
   The present pages existing within the zone located on memory available since
   early boot, excluding hotplugged memory. Defined only when
diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst
index a7296f251799cb..78231995e49012 100644
--- a/Documentation/mm/process_addrs.rst
+++ b/Documentation/mm/process_addrs.rst
@@ -537,6 +537,17 @@ We establish basic locking rules when interacting with page tables:
 * When changing a page table entry the page table lock for that page table
   **must** be held, except if you can safely assume nobody can access the page
   tables concurrently (such as on invocation of :c:func:`!free_pgtables`).
+* Page tables may be *walked* under RCU alone, as page tables are freed only
+  after an RCU grace period has elapsed. However, any entry found must be
+  revalidated after the page table lock is taken (such as the
+  :c:func:`!pmd_same` recheck performed by :c:func:`!pte_offset_map_lock`)
+  before it is acted upon. Changing an entry requires the page table lock
+  and one of the locks that excludes teardown (any one of the mmap, VMA or
+  rmap locks).
+* When traversing page tables under RCU alone it is important to take care
+  when operating upon leaf entries - if the value is operated upon (for
+  instance getting the folio associated with a PTE) an appropriate lock must
+  be taken to prevent concurrent modification.
 * Reads from and writes to page table entries must be *appropriately*
   atomic. See the section on atomicity below for details.
 * Populating previously empty entries requires that the mmap or VMA locks are
diff --git a/Documentation/mm/vmemmap_dedup.rst b/Documentation/mm/vmemmap_dedup.rst
index 9fa8642ded483b..8c287ae3f86cc8 100644
--- a/Documentation/mm/vmemmap_dedup.rst
+++ b/Documentation/mm/vmemmap_dedup.rst
@@ -1,4 +1,3 @@
-
 .. SPDX-License-Identifier: GPL-2.0
 
 =========================================
@@ -192,32 +191,7 @@ to 4 on HugeTLB pages.
 
 There's no remapping of vmemmap given that device-dax memory is not part of
 System RAM ranges initialized at boot. Thus the tail page deduplication
-happens at a later stage when we populate the sections. HugeTLB reuses the
-the head vmemmap page representing, whereas device-dax reuses the tail
-vmemmap page. This results in only half of the savings compared to HugeTLB.
-
-Deduplicated tail pages are not mapped read-only.
+happens at a later stage when we populate the sections.
 
-Here's how things look like on device-dax after the sections are populated::
-
- +-----------+ ---virt_to_page---> +-----------+   mapping to   +-----------+
- |           |                     |     0     | -------------> |     0     |
- |           |                     +-----------+                +-----------+
- |           |                     |     1     | -------------> |     1     |
- |           |                     +-----------+                +-----------+
- |           |                     |     2     | ----------------^ ^ ^ ^ ^ ^
- |           |                     +-----------+                   | | | | |
- |           |                     |     3     | ------------------+ | | | |
- |           |                     +-----------+                     | | | |
- |           |                     |     4     | --------------------+ | | |
- |    PMD    |                     +-----------+                       | | |
- |   level   |                     |     5     | ----------------------+ | |
- |  mapping  |                     +-----------+                         | |
- |           |                     |     6     | ------------------------+ |
- |           |                     +-----------+                           |
- |           |                     |     7     | --------------------------+
- |           |                     +-----------+
- |           |
- |           |
- |           |
- +-----------+
+Deduplicated tail pages are not mapped read-only. The mapping layout is the same
+as HugeTLB.
diff --git a/Documentation/netlink/netlink-raw.yaml b/Documentation/netlink/netlink-raw.yaml
index 4c436b59a34be5..18ccfe05048ae1 100644
--- a/Documentation/netlink/netlink-raw.yaml
+++ b/Documentation/netlink/netlink-raw.yaml
@@ -176,20 +176,23 @@ properties:
               struct:
                 description: Name of the nested struct type.
                 type: string
-            if:
-              properties:
-                type:
-                  const: pad
-            then:
-              required: [ len ]
-            if:
-              properties:
-                type:
-                  const: binary
-            then:
-              oneOf:
-                - required: [ len ]
-                - required: [ struct ]
+            allOf:
+              -
+                if:
+                  properties:
+                    type:
+                      const: pad
+                then:
+                  required: [ len ]
+              -
+                if:
+                  properties:
+                    type:
+                      const: binary
+                then:
+                  oneOf:
+                    - required: [ len ]
+                    - required: [ struct ]
         # End genetlink-legacy
 
   attribute-sets:
diff --git a/Documentation/netlink/specs/conntrack.yaml b/Documentation/netlink/specs/conntrack.yaml
index b1eb102ab84324..56c13705243b3f 100644
--- a/Documentation/netlink/specs/conntrack.yaml
+++ b/Documentation/netlink/specs/conntrack.yaml
@@ -72,14 +72,14 @@ definitions:
     type: enum
     entries:
       - none
-      - cloned
+      - closed
       - cookie-wait
       - cookie-echoed
       - established
       - shutdown-sent
-      - shutdown-received
+      - shutdown-recd
       - shutdown-ack-sent
-      - shutdown-heartbeat-sent
+      - heartbeat-sent
   -
     name: nf-ct-status
     type: flags
@@ -116,9 +116,11 @@ attribute-sets:
       -
         name: packets-old
         type: u32
+        doc: obsolete
       -
         name: bytes-old
         type: u32
+        doc: obsolete
       -
         name: pad
         type: pad
@@ -292,6 +294,12 @@ attribute-sets:
         name: help-name
         type: string
         doc: helper name
+      -
+        name: help-info
+        type: binary
+        doc: >-
+          Private data of a user space helper, only present for helpers
+          registered via nfnetlink_cthelper with a non-zero data length.
   -
     name: nat-proto-attrs
     attributes:
@@ -314,16 +322,16 @@ attribute-sets:
         name: nat-v4-maxip
         type: u32
         byte-order: big-endian
+      -
+        name: nat-proto
+        type: nest
+        nested-attributes: nat-proto-attrs
       -
         name: nat-v6-minip
         type: binary
       -
         name: nat-v6-maxip
         type: binary
-      -
-        name: nat-proto
-        type: nest
-        nested-attributes: nat-proto-attrs
   -
     name: seqadj-attrs
     attributes:
@@ -345,6 +353,20 @@ attribute-sets:
       -
         name: secctx-name
         type: string
+  -
+    name: timestamp-attrs
+    attributes:
+      -
+        name: start
+        type: u64
+        byte-order: big-endian
+      -
+        name: stop
+        type: u64
+        byte-order: big-endian
+      -
+        name: pad
+        type: pad
   -
     name: synproxy-attrs
     attributes:
@@ -371,6 +393,12 @@ attribute-sets:
         name: reply-flags
         type: u32
         doc: bitmask of tuple fields to filter on, reply direction
+      -
+        name: zone
+        type: flag
+        doc: >-
+          Filter on conntrack zone id; requires the top-level zone
+          (``CTA_ZONE``) attribute.
   -
     name: conntrack-attrs
     attributes:
@@ -445,7 +473,7 @@ attribute-sets:
         nested-attributes: seqadj-attrs
       -
         name: secmark
-        type: binary
+        type: u32
         doc: obsolete
       -
         name: zone
@@ -458,8 +486,8 @@ attribute-sets:
         nested-attributes: secctx-attrs
       -
         name: timestamp
-        type: u64
-        byte-order: big-endian
+        type: nest
+        nested-attributes: timestamp-attrs
       -
         name: mark-mask
         type: u32
@@ -510,7 +538,7 @@ attribute-sets:
         name: invalid
         type: u32
         byte-order: big-endian
-        doc: obsolete
+        doc: packets which could not be tracked
       -
         name: ignore
         type: u32
@@ -582,20 +610,18 @@ operations:
             - status
             - protoinfo
             - help
-            - nat-src
-            - nat-dst
             - timeout
             - mark
             - counters-orig
             - counters-reply
             - use
             - id
-            - nat-dst
             - tuple-master
             - seq-adj-orig
             - seq-adj-reply
             - zone
             - secctx
+            - timestamp
             - labels
             - synproxy
       dump:
@@ -618,20 +644,18 @@ operations:
             - status
             - protoinfo
             - help
-            - nat-src
-            - nat-dst
             - timeout
             - mark
             - counters-orig
             - counters-reply
             - use
             - id
-            - nat-dst
             - tuple-master
             - seq-adj-orig
             - seq-adj-reply
             - zone
             - secctx
+            - timestamp
             - labels
             - synproxy
     -
@@ -645,8 +669,8 @@ operations:
         reply:
           value: 0x104
           attributes:
-            - searched
             - found
+            - invalid
             - insert
             - insert-failed
             - drop
diff --git a/Documentation/netlink/specs/devlink.yaml b/Documentation/netlink/specs/devlink.yaml
index 38b1190f3d269e..4706de62839236 100644
--- a/Documentation/netlink/specs/devlink.yaml
+++ b/Documentation/netlink/specs/devlink.yaml
@@ -158,7 +158,7 @@ definitions:
       -
         name: entry
   -
-    type: enum
+    type: flags
     name: resource-scope
     entries:
       -
@@ -174,6 +174,18 @@ definitions:
         value: 1
       -
         name: fw-activate
+  -
+    type: enum
+    name: reload-limit
+    entries:
+      -
+        name: unspec
+        doc: no constraints
+      -
+        name: no-reset
+        doc: >-
+          No reset allowed, no down time allowed, no link flap and no
+          configuration is lost.
   -
     type: enum
     name: param-cmode
@@ -775,7 +787,7 @@ attribute-sets:
       -
         name: reload-limits
         type: bitfield32
-        enum: reload-action
+        enum: reload-limit
         enum-as-flags: true
       -
         name: dev-stats
@@ -793,6 +805,7 @@ attribute-sets:
       -
         name: reload-stats-limit
         type: u8
+        enum: reload-limit
       -
         name: reload-stats-value
         type: u32
@@ -845,13 +858,14 @@ attribute-sets:
         name: linecard-supported-types
         type: nest
         nested-attributes: dl-linecard-supported-types
-
-      # TODO: fill in the attributes in between
-
+      -
+        name: nested-devlink
+        type: nest
+        multi-attr: true
+        nested-attributes: dl-nested-devlink
       -
         name: selftests
         type: nest
-        value: 176
         nested-attributes: dl-selftest-id
       -
         name: rate-tx-priority
@@ -889,7 +903,6 @@ attribute-sets:
         name: resource-scope-mask
         type: u32
         enum: resource-scope
-        enum-as-flags: true
         doc: |
           Bitmask selecting which resource classes to include in a
           resource-dump response. Bit 0 (dev) selects device-level
@@ -898,7 +911,7 @@ attribute-sets:
       -
         name: parent-dev
         type: nest
-        nested-attributes: dl-parent-dev
+        nested-attributes: dl-nested-devlink
         doc: |
           Identifies the devlink instance which owns the parent rate node.
           Used with rate-set and rate-new to parent a rate object to a node on
@@ -978,6 +991,49 @@ attribute-sets:
         type: bitfield32
         enum: port-fn-attr-cap
         enum-as-flags: true
+      -
+        name: devlink
+        type: nest
+        nested-attributes: dl-nested-devlink
+        doc: Handle of the peer devlink instance instantiated for this function.
+      -
+        name: max-io-eqs
+        type: u32
+
+  -
+    name: dl-port-function-set
+    subset-of: dl-port-function
+    doc: |
+      Port function attributes that can be configured; opstate and the
+      devlink handle are read-only.
+    attributes:
+      -
+        name: hw-addr
+      -
+        name: state
+      -
+        name: caps
+      -
+        name: max-io-eqs
+
+  -
+    name: dl-port-set
+    subset-of: devlink
+    doc: Attributes accepted by the port-set request.
+    attributes:
+      -
+        name: bus-name
+      -
+        name: dev-name
+      -
+        name: index
+      -
+        name: port-index
+      -
+        name: port-type
+      -
+        name: port-function
+        nested-attributes: dl-port-function-set
 
   -
     name: dl-dpipe-tables
@@ -1008,6 +1064,8 @@ attribute-sets:
         name: dpipe-table-resource-id
       -
         name: dpipe-table-resource-units
+      -
+        name: pad
 
   -
     name: dl-dpipe-table-matches
@@ -1042,6 +1100,8 @@ attribute-sets:
         name: dpipe-entry-action-values
       -
         name: dpipe-entry-counter
+      -
+        name: pad
 
   -
     name: dl-dpipe-entry-match-values
@@ -1180,6 +1240,8 @@ attribute-sets:
         name: resource-unit
       -
         name: resource-occ
+      -
+        name: pad
 
   -
     name: dl-resource-list
@@ -1207,6 +1269,7 @@ attribute-sets:
     attributes:
       -
         name: region-snapshot
+        multi-attr: true
 
   -
     name: dl-region-snapshot
@@ -1221,6 +1284,7 @@ attribute-sets:
     attributes:
       -
         name: region-chunk
+        multi-attr: true
 
   -
     name: dl-region-chunk
@@ -1230,6 +1294,8 @@ attribute-sets:
         name: region-chunk-data
       -
         name: region-chunk-addr
+      -
+        name: pad
 
   -
     name: dl-fmsg
@@ -1270,6 +1336,8 @@ attribute-sets:
         name: health-reporter-auto-dump
       -
         name: health-reporter-burst-period
+      -
+        name: pad
 
   -
     name: dl-attr-stats
@@ -1284,6 +1352,10 @@ attribute-sets:
       -
         name: stats-rx-dropped
         type: u64
+      -
+        name: pad
+        type: pad
+        value: 61
 
   -
     name: dl-trap-metadata
@@ -1303,6 +1375,7 @@ attribute-sets:
     attributes:
       -
         name: linecard-type
+        multi-attr: true
 
   -
     name: dl-selftest-id
@@ -1330,6 +1403,52 @@ attribute-sets:
   -
     name: dl-parent-dev
     subset-of: devlink
+    doc: |
+      Devlink handle accepted as the parent-dev input; the netns id the
+      kernel reports back is not accepted, the parent is always resolved
+      in the caller's netns.
+    attributes:
+      -
+        name: bus-name
+      -
+        name: dev-name
+      -
+        name: index
+
+  -
+    name: dl-rate-set
+    subset-of: devlink
+    doc: Attributes accepted by the rate-set and rate-new requests.
+    attributes:
+      -
+        name: bus-name
+      -
+        name: dev-name
+      -
+        name: index
+      -
+        name: port-index
+      -
+        name: rate-node-name
+      -
+        name: rate-tx-share
+      -
+        name: rate-tx-max
+      -
+        name: rate-tx-priority
+      -
+        name: rate-tx-weight
+      -
+        name: rate-parent-node-name
+      -
+        name: rate-tc-bws
+      -
+        name: parent-dev
+        nested-attributes: dl-parent-dev
+
+  -
+    name: dl-nested-devlink
+    subset-of: devlink
     attributes:
       -
         name: bus-name
@@ -1337,6 +1456,8 @@ attribute-sets:
         name: dev-name
       -
         name: index
+      -
+        name: netns-id
 
 operations:
   enum-model: directional
@@ -1363,6 +1484,7 @@ operations:
             - index
             - reload-failed
             - dev-stats
+            - nested-devlink
       dump:
         reply: *get-reply
 
@@ -1388,13 +1510,12 @@ operations:
         request:
           attributes: *dev-id-attrs
         reply:
-          value: 3  # due to a bug, port dump returns DEVLINK_CMD_NEW
           attributes: *port-id-attrs
 
     -
       name: port-set
       doc: Set devlink port instances.
-      attribute-set: devlink
+      attribute-set: dl-port-set
       dont-validate: [strict]
       flags: [admin-perm]
       do:
@@ -1901,7 +2022,7 @@ operations:
         post: devlink-nl-post-doit
         request:
           value: 42
-          attributes: &region-id-attrs
+          attributes:
             - bus-name
             - dev-name
             - index
@@ -1909,7 +2030,15 @@ operations:
             - region-name
         reply: &region-get-reply
           value: 42
-          attributes: *region-id-attrs
+          attributes:
+            - bus-name
+            - dev-name
+            - index
+            - port-index
+            - region-name
+            - region-size
+            - region-max-snapshots
+            - region-snapshots
       dump:
         request:
           attributes: *dev-id-attrs
@@ -1975,6 +2104,7 @@ operations:
             - index
             - port-index
             - region-name
+            - region-chunks
 
     -
       name: port-param-get
@@ -2305,7 +2435,7 @@ operations:
     -
       name: rate-set
       doc: Set rate instances.
-      attribute-set: devlink
+      attribute-set: dl-rate-set
       dont-validate: [strict]
       flags: [admin-perm]
       do:
@@ -2316,6 +2446,7 @@ operations:
             - bus-name
             - dev-name
             - index
+            - port-index
             - rate-node-name
             - rate-tx-share
             - rate-tx-max
@@ -2328,7 +2459,7 @@ operations:
     -
       name: rate-new
       doc: Create rate instances.
-      attribute-set: devlink
+      attribute-set: dl-rate-set
       dont-validate: [strict]
       flags: [admin-perm]
       do:
@@ -2374,14 +2505,22 @@ operations:
         post: devlink-nl-post-doit
         request:
           value: 78
-          attributes: &linecard-id-attrs
+          attributes:
             - bus-name
             - dev-name
             - index
             - linecard-index
         reply: &linecard-get-reply
           value: 80
-          attributes: *linecard-id-attrs
+          attributes:
+            - bus-name
+            - dev-name
+            - index
+            - linecard-index
+            - linecard-state
+            - linecard-type
+            - linecard-supported-types
+            - nested-devlink
       dump:
         request:
           attributes: *dev-id-attrs
diff --git a/Documentation/netlink/specs/dpll.yaml b/Documentation/netlink/specs/dpll.yaml
index cdc8c7b456df89..8fa789a502732a 100644
--- a/Documentation/netlink/specs/dpll.yaml
+++ b/Documentation/netlink/specs/dpll.yaml
@@ -31,8 +31,7 @@ definitions:
       -
         name: unlocked
         doc: |
-          dpll was not yet locked to any valid input (or forced by setting
-          DPLL_A_MODE to DPLL_MODE_DETACHED)
+          dpll was not yet locked to any valid input
         value: 1
       -
         name: locked
@@ -550,6 +549,8 @@ attribute-sets:
     attributes:
       -
         name: parent-id
+      -
+        name: pad
       -
         name: direction
       -
@@ -576,6 +577,8 @@ attribute-sets:
     name: frequency-range
     subset-of: pin
     attributes:
+      -
+        name: pad
       -
         name: frequency-min
       -
@@ -632,6 +635,7 @@ operations:
             - mode-supported
             - lock-status
             - lock-status-error
+            - clock-quality-level
             - temp
             - clock-id
             - type
@@ -700,11 +704,9 @@ operations:
       doc: |
         Get list of pins and its attributes.
 
-        - dump request without any attributes given - list all the pins in the
-          system
-        - dump request with target dpll - list all the pins registered with
-          a given dpll device
-        - do request with target dpll and target pin - single pin attributes
+        - dump request - list all the registered, available pins in the
+          system, the request takes no attributes
+        - do request with target pin - single pin attributes
       attribute-set: pin
       flags: [admin-perm]
 
@@ -741,9 +743,6 @@ operations:
             - measured-frequency
 
       dump:
-        request:
-          attributes:
-            - id
         reply: *pin-attrs
 
     -
@@ -759,9 +758,6 @@ operations:
           attributes:
             - id
             - frequency
-            - direction
-            - prio
-            - state
             - parent-device
             - parent-pin
             - phase-adjust
diff --git a/Documentation/netlink/specs/drm_ras.yaml b/Documentation/netlink/specs/drm_ras.yaml
index e113056f8c016c..4c2ac9a1ba3f58 100644
--- a/Documentation/netlink/specs/drm_ras.yaml
+++ b/Documentation/netlink/specs/drm_ras.yaml
@@ -69,6 +69,37 @@ attribute-sets:
         name: error-value
         type: u32
         doc: Current value of the requested error counter.
+      -
+        name: error-threshold
+        type: u32
+        doc: Error threshold of the counter.
+  -
+    name: error-event-attrs
+    attributes:
+      -
+        name: device-name
+        type: string
+        doc: Device (PCI BDF, UUID) that reported the error.
+      -
+        name: node-id
+        type: u32
+        doc: ID of the node that reported the error.
+      -
+        name: node-name
+        type: string
+        doc: Name of the node that reported the error.
+      -
+        name: error-id
+        type: u32
+        doc: ID of the error counter.
+      -
+        name: error-name
+        type: string
+        doc: Name of the error.
+      -
+        name: error-value
+        type: u32
+        doc: Current value of the error counter.
 
 operations:
   list:
@@ -124,3 +155,52 @@ operations:
       do:
         request:
           attributes: *id-attrs
+    -
+      name: error-event
+      doc: >-
+           Report an error event to userspace.
+           The event includes the device, node and error information
+           of the error that triggered the event.
+      attribute-set: error-event-attrs
+      mcgrp: error-report
+      event:
+        attributes:
+          - device-name
+          - node-id
+          - node-name
+          - error-id
+          - error-name
+          - error-value
+    -
+      name: get-error-threshold
+      doc: >-
+           Retrieve error threshold of a given counter.
+           The response includes the id, the name, and current threshold
+           of the counter.
+      attribute-set: error-counter-attrs
+      flags: [admin-perm]
+      do:
+        request:
+          attributes: *id-attrs
+        reply:
+          attributes:
+            - error-id
+            - error-name
+            - error-threshold
+    -
+      name: set-error-threshold
+      doc: >-
+           Set error threshold of a given counter.
+      attribute-set: error-counter-attrs
+      flags: [admin-perm]
+      do:
+        request:
+          attributes:
+            - node-id
+            - error-id
+            - error-threshold
+
+mcast-groups:
+  list:
+    -
+      name: error-report
diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml
index 5dd4d1b5d94bbb..a66594ad9e7391 100644
--- a/Documentation/netlink/specs/ethtool.yaml
+++ b/Documentation/netlink/specs/ethtool.yaml
@@ -47,6 +47,7 @@ definitions:
       -
         name: started
         doc: The firmware flashing process has started.
+        value: 1
       -
         name: in-progress
         doc: The firmware flashing process is in progress.
@@ -89,6 +90,7 @@ definitions:
       -
         name: power-not-available
         doc: Group of power_not_available states
+        value: 8
       -
         name: short-detected
         doc: Group of short_detected states
@@ -97,7 +99,7 @@ definitions:
     enum-name: phy-upstream
     header: linux/ethtool.h
     type: enum
-    name-prefix: phy-upstream
+    name-prefix: phy-upstream-
     entries: [mac, phy]
   -
     name: tcp-data-split
diff --git a/Documentation/netlink/specs/fou.yaml b/Documentation/netlink/specs/fou.yaml
index 331f1b342b3adb..3e6a38e312025c 100644
--- a/Documentation/netlink/specs/fou.yaml
+++ b/Documentation/netlink/specs/fou.yaml
@@ -44,12 +44,14 @@ attribute-sets:
       -
         name: type
         type: u8
+        enum: encap-type
       -
         name: remcsum-nopartial
         type: flag
       -
         name: local-v4
         type: u32
+        byte-order: big-endian
       -
         name: local-v6
         type: binary
@@ -58,6 +60,7 @@ attribute-sets:
       -
         name: peer-v4
         type: u32
+        byte-order: big-endian
       -
         name: peer-v6
         type: binary
@@ -90,6 +93,7 @@ operations:
         request: &all_attrs
           attributes:
             - port
+            - af
             - ipproto
             - type
             - remcsum-nopartial
diff --git a/Documentation/netlink/specs/handshake.yaml b/Documentation/netlink/specs/handshake.yaml
index ffec12b467597d..0498ada2dd97a3 100644
--- a/Documentation/netlink/specs/handshake.yaml
+++ b/Documentation/netlink/specs/handshake.yaml
@@ -22,7 +22,8 @@ definitions:
     type: enum
     name: handler-class
     value-start: 0
-    entries: [none, tlshd, max]
+    entries: [none, tlshd]
+    render-max: true
   -
     type: enum
     name: msg-type
@@ -67,7 +68,7 @@ attribute-sets:
         enum: auth
       -
         name: peer-identity
-        type: u32
+        type: s32
         multi-attr: true
       -
         name: certificate
@@ -79,7 +80,7 @@ attribute-sets:
         type: string
       -
         name: keyring
-        type: u32
+        type: s32
   -
     name: done
     attributes:
@@ -93,7 +94,7 @@ attribute-sets:
         type: s32
       -
         name: remote-auth
-        type: u32
+        type: s32
         multi-attr: true
 
 operations:
diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml
index 3e3f03bd5c29b8..4303593e5003cf 100644
--- a/Documentation/netlink/specs/netdev.yaml
+++ b/Documentation/netlink/specs/netdev.yaml
@@ -80,7 +80,7 @@ definitions:
       -
         name: tx-checksum
         doc: >-
-          L3 checksum HW offload is supported by the driver.
+          L4 checksum HW offload is supported by the driver.
       -
         name: tx-launch-time-fifo
         doc: >-
@@ -118,7 +118,8 @@ attribute-sets:
         enum: xdp-act
       -
         name: xdp-zc-max-segs
-        doc: max fragment count supported by ZC driver
+        doc: max buffer count per packet supported by ZC driver,
+             including the head buffer
         type: u32
         checks:
           min: 1
@@ -851,6 +852,8 @@ operations:
       name: bind-tx
       doc: Bind dmabuf to netdev for TX
       attribute-set: dmabuf
+      # Intentionally unprivileged (no admin-perm / uns-admin-perm); see
+      # comment above netdev_nl_bind_tx_doit().
       do:
         request:
           attributes:
diff --git a/Documentation/netlink/specs/nl80211.yaml b/Documentation/netlink/specs/nl80211.yaml
index d9fdd66b497e26..efc783fb8fdfce 100644
--- a/Documentation/netlink/specs/nl80211.yaml
+++ b/Documentation/netlink/specs/nl80211.yaml
@@ -97,7 +97,7 @@ definitions:
       - unexpected-frame
       - probe-client
       - register-beacons
-      - unexpected-4-addr-frame
+      - unexpected-4addr-frame
       - set-noack-map
       - ch-switch-notify
       - start-p2p-device
@@ -202,7 +202,7 @@ definitions:
       - tdls-channel-switch
       - scan-random-mac-addr
       - sched-scan-random-mac-addr
-      - no-random-mac-addr
+      - nd-random-mac-addr
   -
     name: channel-type
     type: enum
diff --git a/Documentation/netlink/specs/nlctrl.yaml b/Documentation/netlink/specs/nlctrl.yaml
index 8b4472a6aa36a0..a400365cdbeb73 100644
--- a/Documentation/netlink/specs/nlctrl.yaml
+++ b/Documentation/netlink/specs/nlctrl.yaml
@@ -42,6 +42,16 @@ definitions:
       - bitfield32
       - sint
       - uint
+  -
+    name: policy-byte-order
+    doc: |
+      Byte order of an integer attribute. Zero is left unused so that it
+      can be taken to mean host byte order.
+    enum-name: netlink-policy-byte-order
+    type: enum
+    value-start: 1
+    entries:
+      - big-endian
 
 attribute-sets:
   -
@@ -146,12 +156,17 @@ attribute-sets:
       -
         name: bitfield32-mask
         type: u32
+      -
+        name: pad
+        type: pad
       -
         name: mask
         type: u64
       -
-        name: pad
-        type: pad
+        name: byte-order
+        doc: Byte order of the attribute, absent means host byte order.
+        type: u32
+        enum: policy-byte-order
   -
     name: op-policy-attrs
     name-prefix: ctrl-attr-policy-
@@ -176,6 +191,7 @@ operations:
         request:
           value: 3
           attributes:
+            - family-id
             - family-name
         reply: &all-attrs
           value: 1
diff --git a/Documentation/netlink/specs/psp.yaml b/Documentation/netlink/specs/psp.yaml
index e9c2ee7e28e026..f3266763c32532 100644
--- a/Documentation/netlink/specs/psp.yaml
+++ b/Documentation/netlink/specs/psp.yaml
@@ -38,6 +38,8 @@ attribute-sets:
         doc: |
           ifindex of the main netdevice linked to the PSP device,
           or the ifindex to associate with the PSP device.
+          Only reported to the network namespace the main netdevice
+          lives in, an ifindex has no meaning outside of it.
         type: u32
       -
         name: psp-versions-cap
diff --git a/Documentation/netlink/specs/rt-link.yaml b/Documentation/netlink/specs/rt-link.yaml
index 99f6fba456ccc5..81b58633911d63 100644
--- a/Documentation/netlink/specs/rt-link.yaml
+++ b/Documentation/netlink/specs/rt-link.yaml
@@ -25,17 +25,17 @@ definitions:
       -
         name: loopback
       -
-        name: point-to-point
+        name: pointopoint
       -
-        name: no-trailers
+        name: notrailers
       -
         name: running
       -
-        name: no-arp
+        name: noarp
       -
         name: promisc
       -
-        name: all-multi
+        name: allmulti
       -
         name: master
       -
@@ -45,7 +45,7 @@ definitions:
       -
         name: portsel
       -
-        name: auto-media
+        name: automedia
       -
         name: dynamic
       -
@@ -309,6 +309,7 @@ definitions:
     entries:
       -
         name: forwarding
+        value: 1
       -
         name: mc-forwarding
       -
@@ -494,6 +495,10 @@ definitions:
         name: ndisc-evict-nocarrier
       -
         name: accept-untracked-na
+      -
+        name: accept-ra-min-lft
+      -
+        name: force-forwarding
   -
     name: ifla-icmp6-stats
     enum-name:
@@ -527,11 +532,21 @@ definitions:
       -
         name: indelivers
       -
-        name: outforwdatagrams
+        name: noectpkts
+      -
+        name: ect1pkts
+      -
+        name: ect0pkts
+      -
+        name: cepkts
+      -
+        name: outrequests
       -
         name: outpkts
       -
         name: outoctets
+      -
+        name: outforwdatagrams
       -
         name: inhdrerrors
       -
@@ -582,14 +597,6 @@ definitions:
         name: outbcastoctets
       -
         name: csumerrors
-      -
-        name: noectpkts
-      -
-        name: ect1-pkts
-      -
-        name: ect0-pkts
-      -
-        name: cepkts
       -
         name: reasm-overlaps
   - name: br-boolopt-multi
@@ -835,7 +842,7 @@ definitions:
   -
     name: ovpn-mode
     enum-name: ovpn-mode
-    name-prefix: ovpn-mode
+    name-prefix: ovpn-mode-
     type: enum
     entries:
       - p2p
@@ -1087,6 +1094,10 @@ attribute-sets:
       -
         name: tailroom
         type: u16
+      -
+        name: pacing-offload
+        type: u32
+        doc: Enable EDT pacing offload (0 - disabled, 1 - enabled).
   -
     name: prop-list-link-attrs
     subset-of: link-attrs
@@ -2220,7 +2231,7 @@ attribute-sets:
         name: conf
         type: binary
         sub-type: u32
-        doc: u32 indexed by ipv6-devconf - 1 on output, on input it's a nest
+        doc: u32 indexed by ipv6-devconf on output, input is not yet implemented
       -
         name: stats
         type: binary
@@ -2555,6 +2566,7 @@ operations:
             - devlink-port
             - gso-ipv4-max-size
             - gro-ipv4-max-size
+            - pacing-offload
       dump:
         request:
           value: 18
diff --git a/Documentation/netlink/specs/tc.yaml b/Documentation/netlink/specs/tc.yaml
index 2e663333a27982..b6ccc645b0e336 100644
--- a/Documentation/netlink/specs/tc.yaml
+++ b/Documentation/netlink/specs/tc.yaml
@@ -42,7 +42,7 @@ definitions:
       - skip-hw
       - skip-sw
       - in-hw
-      - not-in-nw
+      - not-in-hw
       - verbose
   -
     name: flower-key-ctrl-flags
diff --git a/Documentation/netlink/specs/tcp_metrics.yaml b/Documentation/netlink/specs/tcp_metrics.yaml
index 13144aeed31a52..779e8a22b765e9 100644
--- a/Documentation/netlink/specs/tcp_metrics.yaml
+++ b/Documentation/netlink/specs/tcp_metrics.yaml
@@ -32,7 +32,7 @@ attribute-sets:
         name: addr-ipv6
         type: binary
         checks:
-          min-len: 16
+          exact-len: 16
         byte-order: big-endian
         display-hint: ipv6
       -
@@ -63,7 +63,7 @@ attribute-sets:
         name: fopen-cookie
         type: binary
         checks:
-          min-len: tcp-fastopen-cookie-max
+          max-len: tcp-fastopen-cookie-max
       -
         name: saddr-ipv4
         type: u32
@@ -73,7 +73,7 @@ attribute-sets:
         name: saddr-ipv6
         type: binary
         checks:
-          min-len: 16
+          exact-len: 16
         byte-order: big-endian
         display-hint: ipv6
       -
@@ -93,14 +93,12 @@ attribute-sets:
         name: rtt
         type: u32
         doc: |
-          Round Trip Time (RTT), in msecs with 3 bits fractional
-          (left-shift by 3 to get the msec value).
+          Round Trip Time (RTT), in msecs with 3 bits fractional.
       -
         name: rttvar
         type: u32
         doc: |
-          Round Trip Time VARiance (RTT), in msecs with 2 bits fractional
-          (left-shift by 2 to get the msec value).
+          Round Trip Time VARiance (RTTVAR), in msecs with 2 bits fractional.
       -
         name: ssthresh
         type: u32
@@ -117,14 +115,12 @@ attribute-sets:
         name: rtt-us
         type: u32
         doc: |
-          Round Trip Time (RTT), in usecs, with 3 bits fractional
-          (left-shift by 3 to get the msec value).
+          Round Trip Time (RTT), in usecs, with 3 bits fractional.
       -
         name: rttvar-us
         type: u32
         doc: |
-          Round Trip Time (RTT), in usecs, with 2 bits fractional
-          (left-shift by 3 to get the msec value).
+          Round Trip Time VARiance (RTTVAR), in usecs, with 2 bits fractional.
 
 operations:
   list:
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst b/Documentation/networking/device_drivers/ethernet/index.rst
index d9980c84487a54..403ae9c2b06a9d 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -48,6 +48,7 @@ Contents:
    meta/fbnic
    microsoft/netvsc
    mucse/rnpgbe
+   nebula-matrix/nbl
    netronome/nfp
    pensando/ionic
    pensando/ionic_rdma
diff --git a/Documentation/networking/device_drivers/ethernet/intel/i40e.rst b/Documentation/networking/device_drivers/ethernet/intel/i40e.rst
index 53d9d5829d699f..e01f0adda3f632 100644
--- a/Documentation/networking/device_drivers/ethernet/intel/i40e.rst
+++ b/Documentation/networking/device_drivers/ethernet/intel/i40e.rst
@@ -692,7 +692,7 @@ queues: for each tc, <num queues>@<offset> (e.g. queues 16@0 16@16 assigns
 16 queues to tc0 at offset 0 and 16 queues to tc1 at offset 16. Max total
 number of queues for all tcs is 64 or number of cores, whichever is lower.)
 
-hw 1 mode channel: ‘channel’ with ‘hw’ set to 1 is a new new hardware
+hw 1 mode channel: ‘channel’ with ‘hw’ set to 1 is a new hardware
 offload mode in mqprio that makes full use of the mqprio options, the
 TCs, the queue configurations, and the QoS parameters.
 
diff --git a/Documentation/networking/device_drivers/ethernet/intel/iavf.rst b/Documentation/networking/device_drivers/ethernet/intel/iavf.rst
index eb926c3bd4cdd8..5840569917f51b 100644
--- a/Documentation/networking/device_drivers/ethernet/intel/iavf.rst
+++ b/Documentation/networking/device_drivers/ethernet/intel/iavf.rst
@@ -171,7 +171,7 @@ queues: for each tc, <num queues>@<offset> (e.g. queues 16@0 16@16 assigns
 16 queues to tc0 at offset 0 and 16 queues to tc1 at offset 16. Max total
 number of queues for all tcs is 64 or number of cores, whichever is lower.)
 
-hw 1 mode channel: ‘channel’ with ‘hw’ set to 1 is a new new hardware
+hw 1 mode channel: ‘channel’ with ‘hw’ set to 1 is a new hardware
 offload mode in mqprio that makes full use of the mqprio options, the
 TCs, the queue configurations, and the QoS parameters.
 
diff --git a/Documentation/networking/device_drivers/ethernet/microsoft/netvsc.rst b/Documentation/networking/device_drivers/ethernet/microsoft/netvsc.rst
index fc5acd427a5d06..616f808e885b4f 100644
--- a/Documentation/networking/device_drivers/ethernet/microsoft/netvsc.rst
+++ b/Documentation/networking/device_drivers/ethernet/microsoft/netvsc.rst
@@ -116,5 +116,3 @@ XDP support
   before running XDP::
 
 	ethtool -K eth0 lro off
-
-  XDP_REDIRECT action is not yet supported.
diff --git a/Documentation/networking/device_drivers/ethernet/nebula-matrix/nbl.rst b/Documentation/networking/device_drivers/ethernet/nebula-matrix/nbl.rst
new file mode 100644
index 00000000000000..5540291cf37665
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/nebula-matrix/nbl.rst
@@ -0,0 +1,28 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+================================================================
+Linux Base Driver for Nebula-matrix m18110-NIC/m18000-NIC family
+================================================================
+
+Overview:
+=========
+The m18110-NIC/m18000-NIC (marketed as SNIC s1000) is a series
+of network interface cards for the Data Center Area.
+
+This driver provides the core infrastructure for m18110/m18000-NIC
+devices, including:
+
+- PCI device enumeration and resource (BAR) management
+- Firmware command interface via PF mailbox
+- Channel-based communication between driver and firmware
+- Device initialization and teardown
+
+Support
+=======
+
+For more information about m18110-NIC/m18000-NIC, please visit the following URL:
+https://www.nebula-matrix.com/snic_s1000_en
+
+If an issue is identified with the released source code on the supported kernel
+with a supported adapter, email the specific information related to the issue to
+open@nebula-matrix.com.
diff --git a/Documentation/networking/device_drivers/ethernet/stmicro/stmmac.rst b/Documentation/networking/device_drivers/ethernet/stmicro/stmmac.rst
index 5d46e50361294e..ea347b7e93a25f 100644
--- a/Documentation/networking/device_drivers/ethernet/stmicro/stmmac.rst
+++ b/Documentation/networking/device_drivers/ethernet/stmicro/stmmac.rst
@@ -69,7 +69,6 @@ The following features are available in this driver:
  - Pulse-Per-Second Output (PPS)
  - MDIO Clause 22 / Clause 45 Interface
  - MAC Loopback
- - ARP Offloading
  - Automatic CRC / PAD Insertion and Checking
  - Checksum Offload for Received and Transmitted Packets
  - Standard or Jumbo Ethernet Packets
diff --git a/Documentation/networking/devlink/index.rst b/Documentation/networking/devlink/index.rst
index d4a83fdcff7fe6..1af780c811ee25 100644
--- a/Documentation/networking/devlink/index.rst
+++ b/Documentation/networking/devlink/index.rst
@@ -99,6 +99,7 @@ parameters, info versions, and other features it supports.
    nfp
    octeontx2
    prestera
+   ptp_ocp
    qed
    sfc
    stmmac
diff --git a/Documentation/networking/devlink/ptp_ocp.rst b/Documentation/networking/devlink/ptp_ocp.rst
new file mode 100644
index 00000000000000..f94b759d9cd659
--- /dev/null
+++ b/Documentation/networking/devlink/ptp_ocp.rst
@@ -0,0 +1,75 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+========================
+ptp_ocp devlink support
+========================
+
+This document describes the devlink features implemented by the ``ptp_ocp``
+device driver: the info versions it reports and its flash update support.
+
+Info versions
+=============
+
+The ``ptp_ocp`` driver reports the following versions
+
+.. list-table:: devlink info versions implemented
+   :widths: 5 5 90
+
+   * - Name
+     - Type
+     - Description
+   * - ``fw``
+     - running
+     - Version of the firmware running on the card.  Reported as ``loader``
+       instead when the card is running the firmware loader.
+   * - ``board.id``
+     - fixed
+     - Board identifier, read from the on-card EEPROM.
+   * - ``cpld.id``
+     - fixed
+     - Lattice device ID (IDCODE) of the TAP CPLD, formatted as ``0x%08x``.
+       Only present on ADVA TimeCard X1 boards, which are the only ones
+       carrying that CPLD.  Reading it claims the shared I2C bus and
+       reprograms the on-card mux, so the driver does that from its own
+       worker and reports the cached value here; the version is omitted
+       until that read has succeeded.  The read is made once per binding
+       and again after a successful CPLD update.
+   * - ``fw.cpld``
+     - running
+     - USERCODE of the image programmed into the TAP CPLD, formatted as
+       ``0x%08x``.  Read together with ``cpld.id`` and reported the same
+       way.  This is the component name to pass to ``devlink dev flash``
+       to update the CPLD.
+
+Flash update
+============
+
+The driver implements ``devlink dev flash`` for two separate targets,
+selected with the component name.
+
+.. list-table:: Flash components
+   :widths: 20 80
+
+   * - Component
+     - Description
+   * - (none)
+     - The card's own flash, written through the SPI controller the driver
+       exposes.  The card runs the new image after its next reset.
+   * - ``fw.cpld``
+     - The configuration flash of the TAP CPLD on ADVA TimeCard X1 boards,
+       programmed over I2C with the MachXO3 in-system programming commands
+       and activated with a REFRESH, so the new image runs immediately.
+       The image is the raw configuration bitstream.  The only check the
+       driver makes is that its length is a non-zero multiple of the
+       16-byte page size, so a container such as ``.jed`` has to be
+       converted first rather than passed through - one whose length
+       happens to be a multiple of 16 would be programmed as if it were
+       a bitstream.
+
+Programming the CPLD claims the shared I2C bus for the whole cycle, so
+reads of the card's EEPROM block until it completes.  Progress is reported
+with the standard devlink status notifications.
+
+Example::
+
+    $ devlink dev flash pci/0000:02:00.0 file adva-cpld.bin component fw.cpld
diff --git a/Documentation/networking/dsa/dsa.rst b/Documentation/networking/dsa/dsa.rst
index 42a99f5dfa2e2b..7edfdd555f0581 100644
--- a/Documentation/networking/dsa/dsa.rst
+++ b/Documentation/networking/dsa/dsa.rst
@@ -864,7 +864,7 @@ The drivers which act upon the ``dsa_db`` argument in ``port_fdb_add``,
 DSA associates each offloaded bridge and each offloaded LAG with a one-based ID
 (``struct dsa_bridge :: num``, ``struct dsa_lag :: id``) for the purposes of
 refcounting addresses on shared ports. Drivers may piggyback on DSA's numbering
-scheme (the ID is readable through ``db->bridge.num`` and ``db->lag.id`` or may
+scheme (the ID is readable through ``db->bridge.num`` and ``db->lag.id``) or may
 implement their own.
 
 Only the drivers which declare support for FDB isolation are notified of FDB
diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst
index e92abf45faf50d..393f2b6d38636a 100644
--- a/Documentation/networking/ethtool-netlink.rst
+++ b/Documentation/networking/ethtool-netlink.rst
@@ -131,7 +131,7 @@ For compact form, ``ETHTOOL_A_BITSET_SIZE`` and ``ETHTOOL_A_BITSET_VALUE`` are
 mandatory. ``ETHTOOL_A_BITSET_MASK`` attribute is mandatory if
 ``ETHTOOL_A_BITSET_NOMASK`` is not set (bitset represents a value/mask pair);
 if ``ETHTOOL_A_BITSET_NOMASK`` is not set, ``ETHTOOL_A_BITSET_MASK`` is not
-allowed (bitset represents a single bitmap.
+allowed (bitset represents a single bitmap).
 
 Kernel bit set length may differ from userspace length if older application is
 used on newer kernel or vice versa. If userspace bitmap is longer, an error is
diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst
index f7af0286341c9b..685c84cf543de8 100644
--- a/Documentation/networking/ip-sysctl.rst
+++ b/Documentation/networking/ip-sysctl.rst
@@ -3225,18 +3225,19 @@ drop_unsolicited_na - BOOLEAN
 	Default: 0 (disabled).
 
 accept_untracked_na - INTEGER
-	Define behavior for accepting neighbor advertisements from devices that
-	are absent in the neighbor cache:
+	Define behavior for accepting neighbor advertisements for IPv6 addresses
+	that are absent from the neighbor cache or whose entries are in FAILED
+	state:
 
-	- 0 - (default) Do not accept unsolicited and untracked neighbor
-	  advertisements.
+	- 0 - (default) Do not create new neighbor cache entries or update
+	  FAILED entries from neighbor advertisements.
 
-	- 1 - Add a new neighbor cache entry in STALE state for routers on
-	  receiving a neighbor advertisement (either solicited or unsolicited)
-	  with target link-layer address option specified if no neighbor entry
-	  is already present for the advertised IPv6 address. Without this knob,
-	  NAs received for untracked addresses (absent in neighbor cache) are
-	  silently ignored.
+	- 1 - For routers, add a new neighbor cache entry or update an existing
+	  FAILED entry to STALE upon receiving a neighbor advertisement (either
+	  solicited or unsolicited) with the target link-layer address option
+	  specified. Without this knob, NAs received for untracked addresses
+	  (absent from the neighbor cache or in FAILED state) are silently
+	  ignored.
 
 	  This is as per router-side behavior documented in RFC9131.
 
@@ -3251,9 +3252,10 @@ accept_untracked_na - INTEGER
 	  used in conjunction with the ndisc_notify setting on the host to
 	  satisfy this prerequisite.
 
-	- 2 - Extend option (1) to add a new neighbor cache entry only if the
-	  source IP address is in the same subnet as an address configured on
-	  the interface that received the neighbor advertisement.
+	- 2 - Extend option (1) to add a new neighbor cache entry or update a
+	  FAILED entry only if the source IP address is in the same subnet as
+	  an address configured on the interface that received the neighbor
+	  advertisement.
 
 enhanced_dad - BOOLEAN
 	Include a nonce option in the IPv6 neighbor solicitation messages used for
diff --git a/Documentation/networking/mctp.rst b/Documentation/networking/mctp.rst
index c628cb5406d2d7..1b4bffa1ed74d7 100644
--- a/Documentation/networking/mctp.rst
+++ b/Documentation/networking/mctp.rst
@@ -19,7 +19,7 @@ networks.
 
 An interface (or "link") is an instance of an MCTP physical transport binding
 (as defined by DSP0236, section 3.2.47), likely connected to a specific hardware
-device. This is represented as a ``struct netdevice``.
+device. This is represented as a ``struct net_device``.
 
 A network defines a unique address space for MCTP endpoints by endpoint-ID
 (described by DSP0236, section 3.2.31). A network has a user-visible identifier
@@ -296,7 +296,7 @@ There are a few possible packet flows in the MCTP stack:
 	: route lookup
 	-> rt->output() (== mctp_route_input)
 	   : sk_key lookup
-	   : stores skb in struct sk_key->reasm_head
+	   : stores skb in struct mctp_sk_key->reasm_head
 
 	mctp_pkttype_receive()
 	: route lookup
diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst
index 512f6d6fa3d8f8..8eceaa80b6860e 100644
--- a/Documentation/networking/net_cachelines/net_device.rst
+++ b/Documentation/networking/net_cachelines/net_device.rst
@@ -11,6 +11,7 @@ Type                                Name                        fastpath_tx_acce
 unsigned_long:32                    priv_flags                  read_mostly                             __dev_queue_xmit(tx)
 unsigned_long:1                     lltx                        read_mostly                             HARD_TX_LOCK,HARD_TX_TRYLOCK,HARD_TX_UNLOCK(tx)
 unsigned_long:2                     netmem_tx:2;                read_mostly
+unsigned_long:1                     pacing_offload              read_mostly                             sch_fq
 char                                name[16]
 struct netdev_name_node*            name_node
 struct dev_ifalias*                 ifalias
diff --git a/Documentation/networking/netconsole.rst b/Documentation/networking/netconsole.rst
index 4ab5d7b05cf102..694bca4803d50b 100644
--- a/Documentation/networking/netconsole.rst
+++ b/Documentation/networking/netconsole.rst
@@ -141,6 +141,11 @@ The interface exposes these parameters of a netconsole target to userspace:
 	transmit_errors	Number of packet send errors		(read-only)
 	=============== =================================       ============
 
+local_ip is optional: if it is left unset, netconsole derives a source
+address from the egress interface when the target is enabled. Writing any
+value to it, including 0.0.0.0, uses that value verbatim and disables the
+automatic selection.
+
 The "enabled" attribute is also used to control whether the parameters of
 a target can be updated or not -- you can modify the parameters of only
 disabled targets (i.e. if "enabled" is 0).
diff --git a/Documentation/networking/netdevices.rst b/Documentation/networking/netdevices.rst
index db71d4283032a3..6a87f09e701585 100644
--- a/Documentation/networking/netdevices.rst
+++ b/Documentation/networking/netdevices.rst
@@ -222,13 +222,6 @@ ndo_stop:
 	Context: process
 	Note: netif_running() is guaranteed false
 
-ndo_do_ioctl:
-	Synchronization: rtnl_lock() semaphore.
-
-	This is only called by network subsystems internally,
-	not by user space calling ioctl as it was in before
-	linux-5.14.
-
 ndo_siocbond:
 	Synchronization: rtnl_lock() semaphore. In addition, netdev instance
 	lock if the driver implements queue management or shaper API.
@@ -292,15 +285,21 @@ ndo_set_rx_mode:
 	in process context.
 
 ndo_set_rx_mode_async:
-	Synchronization: rtnl_lock() semaphore. In addition, netdev instance
-	lock if the driver implements queue management or shaper API.
+	Synchronization: netdev instance lock for "ops locked" drivers,
+	rtnl_lock() semaphore for all other drivers.
 	Context: process (from a work queue)
 	Notes: Async version of ndo_set_rx_mode which runs in process
 	context. Receives snapshots of the unicast and multicast address lists.
 
 ndo_change_rx_flags:
-	Synchronization: rtnl_lock() semaphore. In addition, netdev instance
-	lock if the driver implements queue management or shaper API.
+	Synchronization: netdev instance lock for "ops locked" drivers,
+	rtnl_lock() semaphore for all other drivers.
+
+ndo_work:
+	Synchronization: netdev instance lock for "ops locked" drivers,
+	rtnl_lock() semaphore for all other drivers.
+	Context: process (from a work queue)
+	Notes: Runs deferred work scheduled with netdev_work_sched().
 
 ndo_setup_tc:
 	Locking depends on ``tc_setup_type``. For most types the callback
diff --git a/Documentation/networking/nf_conntrack-sysctl.rst b/Documentation/networking/nf_conntrack-sysctl.rst
index 35f889259fcddb..4426cce2a73f35 100644
--- a/Documentation/networking/nf_conntrack-sysctl.rst
+++ b/Documentation/networking/nf_conntrack-sysctl.rst
@@ -44,7 +44,7 @@ nf_conntrack_events - BOOLEAN
 
 nf_conntrack_expect_max - INTEGER
 	Maximum size of expectation table.  Default value is
-	nf_conntrack_buckets / 256. Minimum is 1.
+	nf_conntrack_buckets / 64. Minimum is 1.
 
 nf_conntrack_frag6_high_thresh - INTEGER
 	default 262144
diff --git a/Documentation/networking/page_pool.rst b/Documentation/networking/page_pool.rst
index 817f8b78d24604..d0be2b800a060a 100644
--- a/Documentation/networking/page_pool.rst
+++ b/Documentation/networking/page_pool.rst
@@ -83,7 +83,7 @@ that NAPI context, the page pool may safely access consumer-side resources
 		 page_pool_get_dma_addr page_pool_get_dma_dir
 
 .. kernel-doc:: net/core/page_pool.c
-   :identifiers: page_pool_put_page_bulk
+   :identifiers: page_pool_put_netmem_bulk
 
 DMA sync
 --------
diff --git a/Documentation/networking/phonet.rst b/Documentation/networking/phonet.rst
index d705cc5b09fcf6..e5551fc45c7865 100644
--- a/Documentation/networking/phonet.rst
+++ b/Documentation/networking/phonet.rst
@@ -172,7 +172,7 @@ two endpoints directly, using connect() on the active side. This is
 intended to support the newer Nokia Wireless Modem API, as found in
 e.g. the Nokia Slim Modem in the ST-Ericsson U8500 platform::
 
-  struct sockaddr_spn spn;
+  struct sockaddr_pn spn;
   int fd;
 
   fd = socket(PF_PHONET, SOCK_SEQPACKET, PN_PROTO_PIPE);
diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst
index 0170c9d4dc5e8b..d53067ab1a6136 100644
--- a/Documentation/networking/phy.rst
+++ b/Documentation/networking/phy.rst
@@ -511,24 +511,27 @@ Board Fixups
 Sometimes the specific interaction between the platform and the PHY requires
 special handling.  For instance, to change where the PHY's clock input is,
 or to add a delay to account for latency issues in the data path.  In order
-to support such contingencies, the PHY Layer allows platform code to register
-fixups to be run when the PHY is brought up (or subsequently reset).
+to support such contingencies, the PHY Layer allows platform init code to
+register fixups to be run when the PHY is brought up (or subsequently reset).
+
+Note: This is legacy functionality and not supposed to be used in new code.
+      Fixup's should be handled in the PHY driver.
 
 When the PHY Layer brings up a PHY it checks to see if there are any fixups
 registered for it, matching based on UID (contained in the PHY device's phy_id
 field) and the bus identifier (contained in phydev->dev.bus_id).  Both must
-match, however two constants, PHY_ANY_ID and PHY_ANY_UID, are provided as
-wildcards for the bus ID and UID, respectively.
+match, however NULL for the bus identifier and 0 for phy_uid_mask can be
+used to disable the respective type of matching.
 
 When a match is found, the PHY layer will invoke the run function associated
 with the fixup.  This function is passed a pointer to the phy_device of
-interest.  It should therefore only operate on that PHY.
+interest. It should therefore only operate on that PHY.
 
 The platform code can register the fixup using one of::
 
- int phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask,
+ void __init phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask,
 		int (*run)(struct phy_device *));
- int phy_register_fixup_for_id(const char *phy_id,
+ void __init phy_register_fixup_for_id(const char *phy_id,
 		int (*run)(struct phy_device *));
 
 Standards
diff --git a/Documentation/networking/proc_net_tcp.rst b/Documentation/networking/proc_net_tcp.rst
index 7d9dfe36af45b6..4fa2e58429b958 100644
--- a/Documentation/networking/proc_net_tcp.rst
+++ b/Documentation/networking/proc_net_tcp.rst
@@ -30,16 +30,16 @@ up into 3 parts because of the length of the line)::
       |        |----------------------> receive-queue
       |-------------------------------> transmit-queue
 
-   1000        0 54165785 4 cd1e6040 25 4 27 3 -1
-    |          |    |     |    |     |  | |  | |--> slow start size threshold,
-    |          |    |     |    |     |  | |  |      or -1 if the threshold
-    |          |    |     |    |     |  | |  |      is >= 0xFFFF
-    |          |    |     |    |     |  | |  |----> sending congestion window
-    |          |    |     |    |     |  | |-------> (ack.quick<<1)|ack.pingpong
-    |          |    |     |    |     |  |---------> Predicted tick of soft clock
-    |          |    |     |    |     |              (delayed ACK control data)
-    |          |    |     |    |     |------------> retransmit timeout
-    |          |    |     |    |------------------> location of socket in memory
+   1000        0 54165785 4  0  25 4 27 3 -1
+    |          |    |     |  |  |  | |  | |-------> slow start size threshold,
+    |          |    |     |  |  |  | |  |           or -1 if the threshold
+    |          |    |     |  |  |  | |  |           is >= 0xFFFF
+    |          |    |     |  |  |  | |  |---------> sending congestion window
+    |          |    |     |  |  |  | |------------> (ack.quick<<1)|ack.pingpong
+    |          |    |     |  |  |  |--------------> Predicted tick of soft clock
+    |          |    |     |  |  |                   (delayed ACK control data)
+    |          |    |     |  |  |-----------------> retransmit timeout
+    |          |    |     |  |--------------------> always 0 (was memory location).
     |          |    |     |-----------------------> socket reference count
     |          |    |-----------------------------> inode
     |          |----------------------------------> unanswered 0-window probes
diff --git a/Documentation/networking/radiotap-headers.rst b/Documentation/networking/radiotap-headers.rst
index 1a1bd1ec065002..aadc7bc6b43b82 100644
--- a/Documentation/networking/radiotap-headers.rst
+++ b/Documentation/networking/radiotap-headers.rst
@@ -133,7 +133,7 @@ its prototypes available in include/net/cfg80211.h.  You use it like this::
 
 		    case IEEE80211_RADIOTAP_ANTENNA:
 			    /* radiotap uses 0 for 1st ant */
-			    antenna = *iterator.this_arg);
+			    antenna = *iterator.this_arg;
 			    break;
 
 		    case IEEE80211_RADIOTAP_DBM_TX_POWER:
diff --git a/Documentation/networking/scaling.rst b/Documentation/networking/scaling.rst
index 6c261eb48845a4..6c9836000a8d15 100644
--- a/Documentation/networking/scaling.rst
+++ b/Documentation/networking/scaling.rst
@@ -48,6 +48,15 @@ count is not a power of two. NICs should provide an indirection table
 at least 4 times larger than the queue count. 4x table results in ~16%
 imbalance between the queues, which is acceptable for most applications.
 
+The Toeplitz hash is linear over GF(2), so the quality of the hash key
+matters as much as its randomness. For a key made of uniformly random
+bytes, the q lowest order bits of a given header field fail to spread
+flows over all 2^q queues with probability 1/2, and a burst of connections
+picking nearly consecutive ephemeral ports then lands on a fraction of the
+queues while the others stay idle. The netdev_rss_key_fill() helper draws
+a random key that is free of this defect; drivers should use it rather
+than seeding a key of their own.
+
 Some NICs support symmetric RSS hashing where, if the IP (source address,
 destination address) and TCP/UDP (source port, destination port) tuples
 are swapped, the computed hash is the same. This is beneficial in some
diff --git a/Documentation/power/runtime_pm.rst b/Documentation/power/runtime_pm.rst
index a53ab09c37d53a..39fdeeda7a1e92 100644
--- a/Documentation/power/runtime_pm.rst
+++ b/Documentation/power/runtime_pm.rst
@@ -203,103 +203,12 @@ rules:
 3. Runtime PM Device Fields
 ===========================
 
-The following device runtime PM fields are present in 'struct dev_pm_info', as
-defined in include/linux/pm.h:
+Device PM fields are found in 'struct dev_pm_info', as defined in
+include/linux/pm.h. Many of those fields track runtime PM configuration and
+state.
 
-  `struct timer_list suspend_timer;`
-    - timer used for scheduling (delayed) suspend and autosuspend requests
-
-  `unsigned long timer_expires;`
-    - timer expiration time, in jiffies (if this is different from zero, the
-      timer is running and will expire at that time, otherwise the timer is not
-      running)
-
-  `struct work_struct work;`
-    - work structure used for queuing up requests (i.e. work items in pm_wq)
-
-  `wait_queue_head_t wait_queue;`
-    - wait queue used if any of the helper functions needs to wait for another
-      one to complete
-
-  `spinlock_t lock;`
-    - lock used for synchronization
-
-  `atomic_t usage_count;`
-    - the usage counter of the device
-
-  `atomic_t child_count;`
-    - the count of 'active' children of the device
-
-  `unsigned int ignore_children;`
-    - if set, the value of child_count is ignored (but still updated)
-
-  `unsigned int disable_depth;`
-    - used for disabling the helper functions (they work normally if this is
-      equal to zero); the initial value of it is 1 (i.e. runtime PM is
-      initially disabled for all devices)
-
-  `int runtime_error;`
-    - if set, there was a fatal error (one of the callbacks returned error code
-      as described in Section 2), so the helper functions will not work until
-      this flag is cleared; this is the error code returned by the failing
-      callback
-
-  `unsigned int idle_notification;`
-    - if set, ->runtime_idle() is being executed
-
-  `unsigned int request_pending;`
-    - if set, there's a pending request (i.e. a work item queued up into pm_wq)
-
-  `enum rpm_request request;`
-    - type of request that's pending (valid if request_pending is set)
-
-  `unsigned int deferred_resume;`
-    - set if ->runtime_resume() is about to be run while ->runtime_suspend() is
-      being executed for that device and it is not practical to wait for the
-      suspend to complete; means "start a resume as soon as you've suspended"
-
-  `enum rpm_status runtime_status;`
-    - the runtime PM status of the device; this field's initial value is
-      RPM_SUSPENDED, which means that each device is initially regarded by the
-      PM core as 'suspended', regardless of its real hardware status
-
-  `enum rpm_status last_status;`
-    - the last runtime PM status of the device captured before disabling runtime
-      PM for it (invalid initially and when disable_depth is 0)
-
-  `unsigned int runtime_auto;`
-    - if set, indicates that the user space has allowed the device driver to
-      power manage the device at run time via the /sys/devices/.../power/control
-      `interface;` it may only be modified with the help of the
-      pm_runtime_allow() and pm_runtime_forbid() helper functions
-
-  `unsigned int no_callbacks;`
-    - indicates that the device does not use the runtime PM callbacks (see
-      Section 8); it may be modified only by the pm_runtime_no_callbacks()
-      helper function
-
-  `unsigned int irq_safe;`
-    - indicates that the ->runtime_suspend() and ->runtime_resume() callbacks
-      will be invoked with the spinlock held and interrupts disabled
-
-  `unsigned int use_autosuspend;`
-    - indicates that the device's driver supports delayed autosuspend (see
-      Section 9); it may be modified only by the
-      pm_runtime{_dont}_use_autosuspend() helper functions
-
-  `unsigned int timer_autosuspends;`
-    - indicates that the PM core should attempt to carry out an autosuspend
-      when the timer expires rather than a normal suspend
-
-  `int autosuspend_delay;`
-    - the delay time (in milliseconds) to be used for autosuspend
-
-  `unsigned long last_busy;`
-    - the time (in jiffies) when the pm_runtime_mark_last_busy() helper
-      function was last called for this device; used in calculating inactivity
-      periods for autosuspend
-
-All of the above fields are members of the 'power' member of 'struct device'.
+.. kernel-doc:: include/linux/pm.h
+   :identifiers: dev_pm_info
 
 4. Runtime PM Device Helper Functions
 =====================================
@@ -307,219 +216,10 @@ All of the above fields are members of the 'power' member of 'struct device'.
 The following runtime PM helper functions are defined in
 drivers/base/power/runtime.c and include/linux/pm_runtime.h:
 
-  `void pm_runtime_init(struct device *dev);`
-    - initialize the device runtime PM fields in 'struct dev_pm_info'
-
-  `void pm_runtime_remove(struct device *dev);`
-    - make sure that the runtime PM of the device will be disabled after
-      removing the device from device hierarchy
-
-  `int pm_runtime_idle(struct device *dev);`
-    - execute the subsystem-level idle callback for the device; returns an
-      error code on failure, where -EINPROGRESS means that ->runtime_idle() is
-      already being executed; if there is no callback or the callback returns 0
-      then run pm_runtime_autosuspend(dev) and return its result
-
-  `int pm_runtime_suspend(struct device *dev);`
-    - execute the subsystem-level suspend callback for the device; returns 0 on
-      success, 1 if the device's runtime PM status was already 'suspended', or
-      error code on failure, where -EAGAIN or -EBUSY means it is safe to attempt
-      to suspend the device again in future and -EACCES means that
-      'power.disable_depth' is different from 0
-
-  `int pm_runtime_autosuspend(struct device *dev);`
-    - same as pm_runtime_suspend() except that a call to
-      pm_runtime_mark_last_busy() is made and an autosuspend is scheduled for
-      the appropriate time and 0 is returned
-
-  `int pm_runtime_resume(struct device *dev);`
-    - execute the subsystem-level resume callback for the device; returns 0 on
-      success, 1 if the device's runtime PM status is already 'active' (also if
-      'power.disable_depth' is nonzero, but the status was 'active' when it was
-      changing from 0 to 1) or error code on failure, where -EAGAIN means it may
-      be safe to attempt to resume the device again in future, but
-      'power.runtime_error' should be checked additionally, and -EACCES means
-      that the callback could not be run, because 'power.disable_depth' was
-      different from 0
-
-  `int pm_runtime_resume_and_get(struct device *dev);`
-    - run pm_runtime_resume(dev) and if successful, increment the device's
-      usage counter; returns 0 on success (whether or not the device's
-      runtime PM status was already 'active') or the error code from
-      pm_runtime_resume() on failure.
-
-  `int pm_request_idle(struct device *dev);`
-    - submit a request to execute the subsystem-level idle callback for the
-      device (the request is represented by a work item in pm_wq); returns 0 on
-      success or error code if the request has not been queued up
-
-  `int pm_request_autosuspend(struct device *dev);`
-    - Call pm_runtime_mark_last_busy() and schedule the execution of the
-      subsystem-level suspend callback for the device when the autosuspend delay
-      expires
-
-  `int pm_schedule_suspend(struct device *dev, unsigned int delay);`
-    - schedule the execution of the subsystem-level suspend callback for the
-      device in future, where 'delay' is the time to wait before queuing up a
-      suspend work item in pm_wq, in milliseconds (if 'delay' is zero, the work
-      item is queued up immediately); returns 0 on success, 1 if the device's PM
-      runtime status was already 'suspended', or error code if the request
-      hasn't been scheduled (or queued up if 'delay' is 0); if the execution of
-      ->runtime_suspend() is already scheduled and not yet expired, the new
-      value of 'delay' will be used as the time to wait
-
-  `int pm_request_resume(struct device *dev);`
-    - submit a request to execute the subsystem-level resume callback for the
-      device (the request is represented by a work item in pm_wq); returns 0 on
-      success, 1 if the device's runtime PM status was already 'active', or
-      error code if the request hasn't been queued up
-
-  `void pm_runtime_get_noresume(struct device *dev);`
-    - increment the device's usage counter
-
-  `int pm_runtime_get(struct device *dev);`
-    - increment the device's usage counter, run pm_request_resume(dev) and
-      return its result
-
-  `int pm_runtime_get_sync(struct device *dev);`
-    - increment the device's usage counter, run pm_runtime_resume(dev) and
-      return its result;
-      note that it does not drop the device's usage counter on errors, so
-      consider using pm_runtime_resume_and_get() instead of it, especially
-      if its return value is checked by the caller, as this is likely to
-      result in cleaner code.
-
-  `int pm_runtime_get_if_in_use(struct device *dev);`
-    - return -EINVAL if 'power.disable_depth' is nonzero; otherwise, if the
-      runtime PM status is RPM_ACTIVE and the runtime PM usage counter is
-      nonzero, increment the counter and return 1; otherwise return 0 without
-      changing the counter
-
-  `int pm_runtime_get_if_active(struct device *dev);`
-    - return -EINVAL if 'power.disable_depth' is nonzero; otherwise, if the
-      runtime PM status is RPM_ACTIVE, increment the counter and
-      return 1; otherwise return 0 without changing the counter
-
-  `void pm_runtime_put_noidle(struct device *dev);`
-    - decrement the device's usage counter
-
-  `int pm_runtime_put(struct device *dev);`
-    - decrement the device's usage counter; if the result is 0 then run
-      pm_request_idle(dev) and return its result
-
-  `int pm_runtime_put_autosuspend(struct device *dev);`
-    - set the power.last_busy field to the current time and decrement the
-      device's usage counter; if the result is 0 then run
-      pm_request_autosuspend(dev) and return its result
-
-  `int __pm_runtime_put_autosuspend(struct device *dev);`
-    - decrement the device's usage counter; if the result is 0 then run
-      pm_request_autosuspend(dev) and return its result
-
-  `int pm_runtime_put_sync(struct device *dev);`
-    - decrement the device's usage counter; if the result is 0 then run
-      pm_runtime_idle(dev) and return its result
-
-  `int pm_runtime_put_sync_suspend(struct device *dev);`
-    - decrement the device's usage counter; if the result is 0 then run
-      pm_runtime_suspend(dev) and return its result
-
-  `int pm_runtime_put_sync_autosuspend(struct device *dev);`
-    - set the power.last_busy field to the current time and decrement the
-      device's usage counter; if the result is 0 then run
-      pm_runtime_autosuspend(dev) and return its result
-
-  `void pm_runtime_enable(struct device *dev);`
-    - decrement the device's 'power.disable_depth' field; if that field is equal
-      to zero, the runtime PM helper functions can execute subsystem-level
-      callbacks described in Section 2 for the device
-
-  `int pm_runtime_disable(struct device *dev);`
-    - increment the device's 'power.disable_depth' field (if the value of that
-      field was previously zero, this prevents subsystem-level runtime PM
-      callbacks from being run for the device), make sure that all of the
-      pending runtime PM operations on the device are either completed or
-      canceled; returns 1 if there was a resume request pending and it was
-      necessary to execute the subsystem-level resume callback for the device
-      to satisfy that request, otherwise 0 is returned
-
-  `void pm_runtime_barrier(struct device *dev);`
-    - check if there's a resume request pending for the device and resume it
-      (synchronously) in that case, cancel any other pending runtime PM requests
-      regarding it and wait for all runtime PM operations on it in progress to
-      complete
-
-  `void pm_suspend_ignore_children(struct device *dev, bool enable);`
-    - set/unset the power.ignore_children flag of the device
-
-  `int pm_runtime_set_active(struct device *dev);`
-    - clear the device's 'power.runtime_error' flag, set the device's runtime
-      PM status to 'active' and update its parent's counter of 'active'
-      children as appropriate (it is only valid to use this function if
-      'power.runtime_error' is set or 'power.disable_depth' is greater than
-      zero); it will fail and return error code if the device has a parent
-      which is not active and the 'power.ignore_children' flag of which is unset
-
-  `void pm_runtime_set_suspended(struct device *dev);`
-    - clear the device's 'power.runtime_error' flag, set the device's runtime
-      PM status to 'suspended' and update its parent's counter of 'active'
-      children as appropriate (it is only valid to use this function if
-      'power.runtime_error' is set or 'power.disable_depth' is greater than
-      zero)
-
-  `bool pm_runtime_active(struct device *dev);`
-    - return true if the device's runtime PM status is 'active' or its
-      'power.disable_depth' field is not equal to zero, or false otherwise
-
-  `bool pm_runtime_suspended(struct device *dev);`
-    - return true if the device's runtime PM status is 'suspended' and its
-      'power.disable_depth' field is equal to zero, or false otherwise
-
-  `bool pm_runtime_status_suspended(struct device *dev);`
-    - return true if the device's runtime PM status is 'suspended'
-
-  `void pm_runtime_no_callbacks(struct device *dev);`
-    - set the power.no_callbacks flag for the device and remove the runtime
-      PM attributes from /sys/devices/.../power (or prevent them from being
-      added when the device is registered)
-
-  `void pm_runtime_irq_safe(struct device *dev);`
-    - set the power.irq_safe flag for the device, causing the runtime-PM
-      callbacks to be invoked with interrupts off
-
-  `bool pm_runtime_is_irq_safe(struct device *dev);`
-    - return true if power.irq_safe flag was set for the device, causing
-      the runtime-PM callbacks to be invoked with interrupts off
-
-  `void pm_runtime_mark_last_busy(struct device *dev);`
-    - set the power.last_busy field to the current time
-
-  `void pm_runtime_use_autosuspend(struct device *dev);`
-    - set the power.use_autosuspend flag, enabling autosuspend delays; call
-      pm_runtime_get_sync if the flag was previously cleared and
-      power.autosuspend_delay is negative
-
-  `void pm_runtime_dont_use_autosuspend(struct device *dev);`
-    - clear the power.use_autosuspend flag, disabling autosuspend delays;
-      decrement the device's usage counter if the flag was previously set and
-      power.autosuspend_delay is negative; call pm_runtime_idle
-
-  `void pm_runtime_set_autosuspend_delay(struct device *dev, int delay);`
-    - set the power.autosuspend_delay value to 'delay' (expressed in
-      milliseconds); if 'delay' is negative then runtime suspends are
-      prevented; if power.use_autosuspend is set, pm_runtime_get_sync may be
-      called or the device's usage counter may be decremented and
-      pm_runtime_idle called depending on if power.autosuspend_delay is
-      changed to or from a negative value; if power.use_autosuspend is clear,
-      pm_runtime_idle is called
-
-  `unsigned long pm_runtime_autosuspend_expiration(struct device *dev);`
-    - calculate the time when the current autosuspend delay period will expire,
-      based on power.last_busy and power.autosuspend_delay; if the delay time
-      is 1000 ms or larger then the expiration time is rounded up to the
-      nearest second; returns 0 if the delay period has already expired or
-      power.use_autosuspend isn't set, otherwise returns the expiration time
-      in jiffies
+.. kernel-doc:: drivers/base/power/runtime.c
+   :export:
+
+.. kernel-doc:: include/linux/pm_runtime.h
 
 It is safe to execute the following helper functions from interrupt context:
 
@@ -728,67 +428,8 @@ Subsystems may wish to conserve code space by using the set of generic power
 management callbacks provided by the PM core, defined in
 driver/base/power/generic_ops.c:
 
-  `int pm_generic_runtime_suspend(struct device *dev);`
-    - invoke the ->runtime_suspend() callback provided by the driver of this
-      device and return its result, or return 0 if not defined
-
-  `int pm_generic_runtime_resume(struct device *dev);`
-    - invoke the ->runtime_resume() callback provided by the driver of this
-      device and return its result, or return 0 if not defined
-
-  `int pm_generic_suspend(struct device *dev);`
-    - if the device has not been suspended at run time, invoke the ->suspend()
-      callback provided by its driver and return its result, or return 0 if not
-      defined
-
-  `int pm_generic_suspend_noirq(struct device *dev);`
-    - if pm_runtime_suspended(dev) returns "false", invoke the ->suspend_noirq()
-      callback provided by the device's driver and return its result, or return
-      0 if not defined
-
-  `int pm_generic_resume(struct device *dev);`
-    - invoke the ->resume() callback provided by the driver of this device and,
-      if successful, change the device's runtime PM status to 'active'
-
-  `int pm_generic_resume_noirq(struct device *dev);`
-    - invoke the ->resume_noirq() callback provided by the driver of this device
-
-  `int pm_generic_freeze(struct device *dev);`
-    - if the device has not been suspended at run time, invoke the ->freeze()
-      callback provided by its driver and return its result, or return 0 if not
-      defined
-
-  `int pm_generic_freeze_noirq(struct device *dev);`
-    - if pm_runtime_suspended(dev) returns "false", invoke the ->freeze_noirq()
-      callback provided by the device's driver and return its result, or return
-      0 if not defined
-
-  `int pm_generic_thaw(struct device *dev);`
-    - if the device has not been suspended at run time, invoke the ->thaw()
-      callback provided by its driver and return its result, or return 0 if not
-      defined
-
-  `int pm_generic_thaw_noirq(struct device *dev);`
-    - if pm_runtime_suspended(dev) returns "false", invoke the ->thaw_noirq()
-      callback provided by the device's driver and return its result, or return
-      0 if not defined
-
-  `int pm_generic_poweroff(struct device *dev);`
-    - if the device has not been suspended at run time, invoke the ->poweroff()
-      callback provided by its driver and return its result, or return 0 if not
-      defined
-
-  `int pm_generic_poweroff_noirq(struct device *dev);`
-    - if pm_runtime_suspended(dev) returns "false", run the ->poweroff_noirq()
-      callback provided by the device's driver and return its result, or return
-      0 if not defined
-
-  `int pm_generic_restore(struct device *dev);`
-    - invoke the ->restore() callback provided by the driver of this device and,
-      if successful, change the device's runtime PM status to 'active'
-
-  `int pm_generic_restore_noirq(struct device *dev);`
-    - invoke the ->restore_noirq() callback provided by the device's driver
+.. kernel-doc:: drivers/base/power/generic_ops.c
+   :export:
 
 These functions are the defaults used by the PM core if a subsystem doesn't
 provide its own callbacks for ->runtime_idle(), ->runtime_suspend(),
diff --git a/Documentation/power/userland-swsusp.rst b/Documentation/power/userland-swsusp.rst
index 1cf62d80a9ca10..b00b850148b2c8 100644
--- a/Documentation/power/userland-swsusp.rst
+++ b/Documentation/power/userland-swsusp.rst
@@ -4,9 +4,9 @@ Documentation for userland software suspend interface
 
 	(C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
 
-First, the warnings at the beginning of swsusp.txt still apply.
+First, the warnings at the beginning of swsusp.rst still apply.
 
-Second, you should read the FAQ in swsusp.txt _now_ if you have not
+Second, you should read the FAQ in swsusp.rst _now_ if you have not
 done it already.
 
 Now, to use the userland interface for software suspend you need special
diff --git a/Documentation/process/1.Intro.rst b/Documentation/process/1.Intro.rst
index 2c93caea069f0a..847fbe76b6a445 100644
--- a/Documentation/process/1.Intro.rst
+++ b/Documentation/process/1.Intro.rst
@@ -42,7 +42,7 @@ avoid problems at this important stage.  Developers are cautioned against
 assuming that the job is done when a patch is merged into the mainline.
 
 :ref:`development_advancedtopics` introduces a couple of "advanced" topics:
-managing patches with git and reviewing patches posted by others.
+managing patches with Git and reviewing patches posted by others.
 
 :ref:`development_conclusion` concludes the document with pointers to sources
 for more information on kernel development.
diff --git a/Documentation/process/2.Process.rst b/Documentation/process/2.Process.rst
index 77f3f80e7cd7e1..a8e09ca4e7c709 100644
--- a/Documentation/process/2.Process.rst
+++ b/Documentation/process/2.Process.rst
@@ -223,11 +223,11 @@ of the kernel they manage; they are the ones who will (usually) accept a
 patch for inclusion into the mainline kernel.
 
 Subsystem maintainers each manage their own version of the kernel source
-tree, usually (but certainly not always) using the git source management
-tool.  Tools like git (and related tools like quilt or mercurial) allow
+tree, usually (but certainly not always) using the Git source management
+tool.  Tools like Git (and related tools like Quilt or Mercurial) allow
 maintainers to track a list of patches, including authorship information
 and other metadata.  At any given time, the maintainer can identify which
-patches in his or her repository are not found in the mainline.
+patches in their repository are not found in the mainline.
 
 When the merge window opens, top-level maintainers will ask Linus to "pull"
 the patches they have selected for merging from their repositories.  If
@@ -343,13 +343,13 @@ are well beyond the scope of this document, but there is space for a few
 pointers.
 
 By far the dominant source code management system used by the kernel
-community is git.  Git is one of a number of distributed version control
+community is Git.  Git is one of a number of distributed version control
 systems being developed in the free software community.  It is well tuned
 for kernel development, in that it performs quite well when dealing with
 large repositories and large numbers of patches.  It also has a reputation
 for being difficult to learn and use, though it has gotten better over
-time.  Some sort of familiarity with git is almost a requirement for kernel
-developers; even if they do not use it for their own work, they'll need git
+time.  Some sort of familiarity with Git is almost a requirement for kernel
+developers; even if they do not use it for their own work, they'll need Git
 to keep up with what other developers (and the mainline) are doing.
 
 Git is now packaged by almost all Linux distributions.  There is a home
@@ -359,12 +359,12 @@ page at:
 
 That page has pointers to documentation and tutorials.
 
-Among the kernel developers who do not use git, the most popular choice is
+Among the kernel developers who do not use Git, the most popular choice is
 almost certainly Mercurial:
 
 	https://www.selenic.com/mercurial/
 
-Mercurial shares many features with git, but it provides an interface which
+Mercurial shares many features with Git, but it provides an interface which
 many find easier to use.
 
 The other tool worth knowing about is Quilt:
@@ -374,9 +374,9 @@ The other tool worth knowing about is Quilt:
 Quilt is a patch management system, rather than a source code management
 system.  It does not track history over time; it is, instead, oriented
 toward tracking a specific set of changes against an evolving code base.
-Some major subsystem maintainers use quilt to manage patches intended to go
+Some major subsystem maintainers use Quilt to manage patches intended to go
 upstream.  For the management of certain kinds of trees (-mm, for example),
-quilt is the best tool for the job.
+Quilt is the best tool for the job.
 
 
 Mailing lists
diff --git a/Documentation/process/3.Early-stage.rst b/Documentation/process/3.Early-stage.rst
index 894a920041c624..87fa7875e3369e 100644
--- a/Documentation/process/3.Early-stage.rst
+++ b/Documentation/process/3.Early-stage.rst
@@ -138,7 +138,7 @@ the place to start.  That file tends to not always be up to date, though,
 and not all subsystems are represented there.  The person listed in the
 MAINTAINERS file may, in fact, not be the person who is actually acting in
 that role currently.  So, when there is doubt about who to contact, a
-useful trick is to use git (and "git log" in particular) to see who is
+useful trick is to use Git (and "git log" in particular) to see who is
 currently active within the subsystem of interest.  Look at who is writing
 patches, and who, if anybody, is attaching Signed-off-by lines to those
 patches.  Those are the people who will be best placed to help with a new
diff --git a/Documentation/process/5.Posting.rst b/Documentation/process/5.Posting.rst
index 07d7dbed13ec6a..976cf45a565ad6 100644
--- a/Documentation/process/5.Posting.rst
+++ b/Documentation/process/5.Posting.rst
@@ -21,7 +21,7 @@ There is a constant temptation to avoid posting patches before they are
 completely "ready."  For simple patches, that is not a problem.  If the
 work being done is complex, though, there is a lot to be gained by getting
 feedback from the community before the work is complete.  So you should
-consider posting in-progress work, or even making a git tree available so
+consider posting in-progress work, or even making a Git tree available so
 that interested developers can catch up with your work at any time.
 
 When posting code which is not yet considered ready for inclusion, it is a
@@ -71,7 +71,7 @@ even in the short term.
 
 Patches must be prepared against a specific version of the kernel.  As a
 general rule, a patch should be based on the current mainline as found in
-Linus's git tree.  When basing on mainline, start with a well-known release
+Linus's Git tree.  When basing on mainline, start with a well-known release
 point - a stable or -rc release - rather than branching off the mainline at
 an arbitrary spot.
 
@@ -233,7 +233,7 @@ the patch. Each of these uses this format::
 
 The tags in common use are:
 
- - Signed-off-by: this is a developer's certification that he or she has
+ - Signed-off-by: this is a developer's certification that they have
    the right to submit the patch for inclusion into the kernel.  It is an
    agreement to the Developer's Certificate of Origin, the full text of
    which can be found in :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
@@ -320,7 +320,7 @@ copies should go to:
    the MAINTAINERS file is the first place to look for these people.
 
  - Other developers who have been working in the same area - especially
-   those who might be working there now.  Using git to see who else has
+   those who might be working there now.  Using Git to see who else has
    modified the files you are working on can be helpful.
 
  - If you are responding to a bug report or a feature request, copy the
@@ -362,7 +362,7 @@ that the patches, themselves, have complete changelog information.
 
 In general, the second and following parts of a multi-part patch should be
 sent as a reply to the first part so that they all thread together at the
-receiving end.  Tools like git and quilt have commands to mail out a set of
+receiving end.  Tools like Git and Quilt have commands to mail out a set of
 patches with the proper threading.  If you have a long series, though, and
-are using git, please stay away from the --chain-reply-to option to avoid
+are using Git, please stay away from the --chain-reply-to option to avoid
 creating exceptionally deep nesting.
diff --git a/Documentation/process/6.Followthrough.rst b/Documentation/process/6.Followthrough.rst
index 66fa400c6d9405..ebb0e4cc95840e 100644
--- a/Documentation/process/6.Followthrough.rst
+++ b/Documentation/process/6.Followthrough.rst
@@ -114,7 +114,7 @@ What happens next
 If a patch is considered to be a good thing to add to the kernel, and once
 most of the review issues have been resolved, the next step is usually
 entry into a subsystem maintainer's tree.  How that works varies from one
-subsystem to the next; each maintainer has his or her own way of doing
+subsystem to the next; each maintainer has their own way of doing
 things.  In particular, there may be more than one tree - one, perhaps,
 dedicated to patches planned for the next merge window, and another for
 longer-term work.
diff --git a/Documentation/process/7.AdvancedTopics.rst b/Documentation/process/7.AdvancedTopics.rst
index 185651d87f2af6..066655e5d536ce 100644
--- a/Documentation/process/7.AdvancedTopics.rst
+++ b/Documentation/process/7.AdvancedTopics.rst
@@ -8,7 +8,7 @@ works.  There is still more to learn, however!  This section will cover a
 number of topics which can be helpful for developers wanting to become a
 regular part of the Linux kernel development process.
 
-Managing patches with git
+Managing patches with Git
 -------------------------
 
 The use of distributed version control for the kernel began in early 2002,
@@ -17,17 +17,17 @@ application.  While BitKeeper was controversial, the approach to software
 version management it embodied most certainly was not.  Distributed version
 control enabled an immediate acceleration of the kernel development
 project.  In current times, there are several free alternatives to
-BitKeeper.  For better or for worse, the kernel project has settled on git
+BitKeeper.  For better or for worse, the kernel project has settled on Git
 as its tool of choice.
 
-Managing patches with git can make life much easier for the developer,
+Managing patches with Git can make life much easier for the developer,
 especially as the volume of those patches grows.  Git also has its rough
 edges and poses certain hazards; it is a young and powerful tool which is
 still being civilized by its developers.  This document will not attempt to
-teach the reader how to use git; that would be sufficient material for a
-long document in its own right.  Instead, the focus here will be on how git
+teach the reader how to use Git; that would be sufficient material for a
+long document in its own right.  Instead, the focus here will be on how Git
 fits into the kernel development process in particular.  Developers who
-wish to come up to speed with git will find more information at:
+wish to come up to speed with Git will find more information at:
 
 	https://git-scm.com/
 
@@ -36,20 +36,20 @@ wish to come up to speed with git will find more information at:
 and on various tutorials found on the web.
 
 The first order of business is to read the above sites and get a solid
-understanding of how git works before trying to use it to make patches
+understanding of how Git works before trying to use it to make patches
 available to others.  A git-using developer should be able to obtain a copy
 of the mainline repository, explore the revision history, commit changes to
-the tree, use branches, etc.  An understanding of git's tools for the
+the tree, use branches, etc.  An understanding of Git's tools for the
 rewriting of history (such as rebase) is also useful.  Git comes with its
-own terminology and concepts; a new user of git should know about refs,
+own terminology and concepts; a new user of Git should know about refs,
 remote branches, the index, fast-forward merges, pushes and pulls, detached
 heads, etc.  It can all be a little intimidating at the outset, but the
 concepts are not that hard to grasp with a bit of study.
 
-Using git to generate patches for submission by email can be a good
+Using Git to generate patches for submission by email can be a good
 exercise while coming up to speed.
 
-When you are ready to start putting up git trees for others to look at, you
+When you are ready to start putting up Git trees for others to look at, you
 will, of course, need a server that can be pulled from.  Setting up such a
 server with git-daemon is relatively straightforward if you have a system
 which is accessible to the Internet.  Otherwise, free, public hosting sites
@@ -57,9 +57,9 @@ which is accessible to the Internet.  Otherwise, free, public hosting sites
 developers can get an account on kernel.org, but those are not easy to come
 by; see https://kernel.org/faq/ for more information.
 
-The normal git workflow involves the use of a lot of branches.  Each line
+The normal Git workflow involves the use of a lot of branches.  Each line
 of development can be separated into a separate "topic branch" and
-maintained independently.  Branches in git are cheap, there is no reason to
+maintained independently.  Branches in Git are cheap, there is no reason to
 not make free use of them.  And, in any case, you should not do your
 development in any branch which you intend to ask others to pull from.
 Publicly-available branches should be created with care; merge in patches
@@ -72,7 +72,7 @@ say, or which has some other sort of obvious bug) can be fixed in place or
 made to disappear from the history entirely.  A patch series can be
 rewritten as if it had been written on top of today's mainline, even though
 you have been working on it for months.  Changes can be transparently
-shifted from one branch to another.  And so on.  Judicious use of git's
+shifted from one branch to another.  And so on.  Judicious use of Git's
 ability to revise history can help in the creation of clean patch sets with
 fewer problems.
 
@@ -111,16 +111,16 @@ perform test merges in a private branch.  The git "rerere" tool can be
 useful in such situations; it remembers how merge conflicts were resolved
 so that you don't have to do the same work twice.
 
-One of the biggest recurring complaints about tools like git is this: the
+One of the biggest recurring complaints about tools like Git is this: the
 mass movement of patches from one repository to another makes it easy to
 slip in ill-advised changes which go into the mainline below the review
 radar.  Kernel developers tend to get unhappy when they see that kind of
-thing happening; putting up a git tree with unreviewed or off-topic patches
+thing happening; putting up a Git tree with unreviewed or off-topic patches
 can affect your ability to get trees pulled in the future.  Quoting Linus:
 
 ::
 
-	You can send me patches, but for me to pull a git patch from you, I
+	You can send me patches, but for me to pull a Git patch from you, I
 	need to know that you know what you're doing, and I need to be able
 	to trust things *without* then having to go and check every
 	individual change by hand.
@@ -130,7 +130,7 @@ can affect your ability to get trees pulled in the future.  Quoting Linus:
 To avoid this kind of situation, ensure that all patches within a given
 branch stick closely to the associated topic; a "driver fixes" branch
 should not be making changes to the core memory management code.  And, most
-importantly, do not use a git tree to bypass the review process.  Post an
+importantly, do not use a Git tree to bypass the review process.  Post an
 occasional summary of the tree to the relevant list, and, when the time is
 right, request that the tree be included in linux-next.
 
diff --git a/Documentation/process/backporting.rst b/Documentation/process/backporting.rst
index 0de9eacd46a73a..abc5f8925a7893 100644
--- a/Documentation/process/backporting.rst
+++ b/Documentation/process/backporting.rst
@@ -40,8 +40,8 @@ edit the patch to make it apply.
 
 It is strongly recommended to instead find an appropriate base version
 where the patch applies cleanly and *then* cherry-pick it over to your
-destination tree, as this will make git output conflict markers and let
-you resolve conflicts with the help of git and any other conflict
+destination tree, as this will make Git output conflict markers and let
+you resolve conflicts with the help of Git and any other conflict
 resolution tools you might prefer to use. For example, if you want to
 apply a patch that just arrived on LKML to an older stable kernel, you
 can apply it to the most recent mainline kernel and then cherry-pick it
@@ -54,7 +54,7 @@ problem with applying the patch to the "wrong" base is that it may pull
 in more unrelated changes in the context of the diff when cherry-picking
 it to the older branch.
 
-A good reason to prefer ``git cherry-pick`` over ``git am`` is that git
+A good reason to prefer ``git cherry-pick`` over ``git am`` is that Git
 knows the precise history of an existing commit, so it will know when
 code has moved around and changed the line numbers; this in turn makes
 it less likely to apply the patch to the wrong place (which can result
@@ -69,7 +69,7 @@ article will assume that you are doing a plain ``git cherry-pick``.
 .. _b4: https://people.kernel.org/monsieuricon/introducing-b4-and-patch-attestation
 .. _b4 presentation: https://youtu.be/mF10hgVIx9o?t=2996
 
-Once you have the patch in git, you can go ahead and cherry-pick it into
+Once you have the patch in Git, you can go ahead and cherry-pick it into
 your source tree. Don't forget to cherry-pick with ``-x`` if you want a
 written record of where the patch came from!
 
@@ -101,7 +101,7 @@ backporting from contains patches not in the branch you are backporting
 to. However, the reverse is also possible. In any case, the result is a
 conflict that needs to be resolved.
 
-If your attempted cherry-pick fails with a conflict, git automatically
+If your attempted cherry-pick fails with a conflict, Git automatically
 edits the files to include so-called conflict markers showing you where
 the conflict is and how the two branches have diverged. Resolving the
 conflict typically means editing the end result in such a way that it
@@ -128,7 +128,7 @@ pointers to various tools that you could use:
 -  `IntelliJ <https://www.jetbrains.com/help/idea/resolve-conflicts.html>`__
 -  `VSCode <https://code.visualstudio.com/docs/editor/versioncontrol>`__
 
-To configure git to work with these, see ``git mergetool --help`` or
+To configure Git to work with these, see ``git mergetool --help`` or
 the official `git-mergetool documentation`_.
 
 .. _git-mergetool documentation: https://git-scm.com/docs/git-mergetool
@@ -326,7 +326,7 @@ style, which looks like this::
     this is what the patch wants it to be after being applied
     >>>>>>> <commit> (title)
 
-As you can see, this has 3 parts instead of 2, and includes what git
+As you can see, this has 3 parts instead of 2, and includes what Git
 expected to find there but didn't. It is *highly recommended* to use
 this conflict style as it makes it much clearer what the patch actually
 changed; i.e., it allows you to compare the before-and-after versions
@@ -382,7 +382,7 @@ Dealing with file renames
 
 One of the most annoying things that can happen while backporting a
 patch is discovering that one of the files being patched has been
-renamed, as that typically means git won't even put in conflict markers,
+renamed, as that typically means Git won't even put in conflict markers,
 but will just throw up its hands and say (paraphrased): "Unmerged path!
 You do the work..."
 
@@ -393,7 +393,7 @@ other hand, if the change is big or complicated, you definitely don't
 want to do it by hand.
 
 As a first pass, you can try something like this, which will lower the
-rename detection threshold to 30% (by default, git uses 50%, meaning
+rename detection threshold to 30% (by default, Git uses 50%, meaning
 that two files need to have at least 50% in common for it to consider
 an add-delete pair to be a potential rename)::
 
diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst
index 0aa232b117b549..28f2ca63318e61 100644
--- a/Documentation/process/changes.rst
+++ b/Documentation/process/changes.rst
@@ -55,6 +55,7 @@ nfs-utils              1.0.5            showmount --version
 openssl & libcrypto    1.0.0            openssl version
 pahole                 1.26             pahole --version
 pcmciautils            004              pccardctl -V
+pigz (optional)        2.4              pigz --version
 PPP                    2.4.0            pppd --version
 procps                 3.2.0            ps --version
 Python                 3.9.x            python3 --version
@@ -206,6 +207,13 @@ GNU AWK
 GNU AWK is needed if you want kernel builds to generate address range data for
 builtin modules (CONFIG_BUILTIN_MODULE_RANGES).
 
+pigz (optional)
+---------------
+
+pigz is a parallel implementation of gzip. If it is installed the compressed
+kernel image is produced with it rather than with gzip, see ``KPGZIP`` in
+Documentation/kbuild/kbuild.rst.
+
 System utilities
 ****************
 
diff --git a/Documentation/process/embargoed-hardware-issues.rst b/Documentation/process/embargoed-hardware-issues.rst
index d07f16c3c7b82a..2c244a6d14634e 100644
--- a/Documentation/process/embargoed-hardware-issues.rst
+++ b/Documentation/process/embargoed-hardware-issues.rst
@@ -187,7 +187,7 @@ security issues in the past.
 
 The mailing list operates in the same way as normal Linux development.
 Patches are posted, discussed, and reviewed and if agreed upon, applied to
-a non-public git repository which is only accessible to the participating
+a non-public Git repository which is only accessible to the participating
 developers via a secure connection. The repository contains the main
 development branch against the mainline kernel and backport branches for
 stable kernel versions as necessary.
diff --git a/Documentation/process/handling-regressions.rst b/Documentation/process/handling-regressions.rst
index c71b5d403f0c9c..6958a2a8b03411 100644
--- a/Documentation/process/handling-regressions.rst
+++ b/Documentation/process/handling-regressions.rst
@@ -283,7 +283,7 @@ sure your patch description makes this aspect obvious. Once the change is
 merged, tell the Linux kernel's regression tracker and the regressions mailing
 list about the risk, so everyone has the change on the radar in case reports
 trickle in. Depending on the risk, you also might want to ask the subsystem
-maintainer to mention the issue in his mainline pull request.
+maintainer to mention the issue in their mainline pull request.
 
 What else is there to known about regressions?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/Documentation/process/howto.rst b/Documentation/process/howto.rst
index 9438e03d6f503f..8c60138d03f271 100644
--- a/Documentation/process/howto.rst
+++ b/Documentation/process/howto.rst
@@ -305,7 +305,7 @@ The maintainers of the various kernel subsystems --- and also many
 kernel subsystem developers --- expose their current state of
 development in source repositories.  That way, others can see what is
 happening in the different areas of the kernel.  In areas where
-development is rapid, a developer may be asked to base his submissions
+development is rapid, a developer may be asked to base their submissions
 onto such a subsystem kernel tree so that conflicts between the
 submission and other already ongoing work are avoided.
 
diff --git a/Documentation/process/maintainer-pgp-guide.rst b/Documentation/process/maintainer-pgp-guide.rst
index 652dfbe64102c3..9f6fff2a78e247 100644
--- a/Documentation/process/maintainer-pgp-guide.rst
+++ b/Documentation/process/maintainer-pgp-guide.rst
@@ -25,16 +25,16 @@ communication channels between developers via PGP-signed email exchange.
 
 The Linux kernel source code is available in two main formats:
 
-- Distributed source repositories (git)
+- Distributed source repositories (Git)
 - Periodic release snapshots (tarballs)
 
-Both git repositories and tarballs carry PGP signatures of the kernel
+Both Git repositories and tarballs carry PGP signatures of the kernel
 developers who create official kernel releases. These signatures offer a
 cryptographic guarantee that downloadable versions made available via
 kernel.org or any other mirrors are identical to what these developers
 have on their workstations. To this end:
 
-- git repositories provide PGP signatures on all tags
+- Git repositories provide PGP signatures on all tags
 - tarballs provide detached PGP signatures with all downloads
 
 .. _devs_not_infra:
@@ -660,12 +660,12 @@ impersonate you without having access to your PGP keys.
 
 .. _`nothing to do with it`: https://github.com/jayphelps/git-blame-someone-else
 
-Configure git to use your PGP key
+Configure Git to use your PGP key
 ---------------------------------
 
 If you only have one secret key in your keyring, then you don't really
 need to do anything extra, as it becomes your default key.  However, if
-you happen to have multiple secret keys, you can tell git which key
+you happen to have multiple secret keys, you can tell Git which key
 should be used (``[fpr]`` is the fingerprint of your key)::
 
     $ git config --global user.signingKey [fpr]
@@ -677,8 +677,8 @@ To create a signed tag, pass the ``-s`` switch to the tag command::
 
     $ git tag -s [tagname]
 
-Our recommendation is to always sign git tags, as this allows other
-developers to ensure that the git repository they are pulling from has
+Our recommendation is to always sign Git tags, as this allows other
+developers to ensure that the Git repository they are pulling from has
 not been maliciously altered.
 
 How to verify signed tags
@@ -689,7 +689,7 @@ To verify a signed tag, use the ``verify-tag`` command::
     $ git verify-tag [tagname]
 
 If you are pulling a tag from another fork of the project repository,
-git should automatically verify the signature at the tip you're pulling
+Git should automatically verify the signature at the tip you're pulling
 and show you the results during the merge operation::
 
     $ git pull [url] tags/sometag
@@ -703,15 +703,15 @@ The merge message will contain something like this::
     # gpg: Signature made [...]
     # gpg: Good signature from [...]
 
-If you are verifying someone else's git tag, you will first need to
+If you are verifying someone else's Git tag, you will first need to
 import their PGP key. Please refer to the ":ref:`verify_identities`"
 section below.
 
-Configure git to always sign annotated tags
+Configure Git to always sign annotated tags
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Chances are, if you're creating an annotated tag, you'll want to sign
-it. To force git to always sign annotated tags, you can set a global
+it. To force Git to always sign annotated tags, you can set a global
 configuration option::
 
     $ git config --global tag.forceSignAnnotated true
@@ -722,15 +722,15 @@ How to work with signed commits
 It is also possible to create signed commits, but they have limited
 usefulness in Linux kernel development. The kernel contribution workflow
 relies on sending in patches, and converting commits to patches does not
-preserve git commit signatures. Furthermore, when rebasing your own
+preserve Git commit signatures. Furthermore, when rebasing your own
 repository on a newer upstream, PGP commit signatures will end up
 discarded. For this reason, most kernel developers don't bother signing
 their commits and will ignore signed commits in any external
 repositories that they rely upon in their work.
 
-That said, if you have your working git tree publicly available at some
-git hosting service (kernel.org, infradead.org, ozlabs.org, or others),
-then the recommendation is that you sign all your git commits even if
+That said, if you have your working Git tree publicly available at some
+Git hosting service (kernel.org, infradead.org, ozlabs.org, or others),
+then the recommendation is that you sign all your Git commits even if
 upstream developers do not directly benefit from this practice.
 
 We recommend this for the following reasons:
@@ -752,10 +752,10 @@ command (it's capital ``-S`` due to collision with another flag)::
 
     $ git commit -S
 
-Configure git to always sign commits
+Configure Git to always sign commits
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can tell git to always sign commits::
+You can tell Git to always sign commits::
 
     git config --global commit.gpgSign true
 
@@ -790,7 +790,7 @@ Installing and configuring patatt
 Patatt is packaged for many distributions already, so please check there
 first. You can also install it from pypi using "``pip install patatt``".
 
-If you already have your PGP key configured with git (via the
+If you already have your PGP key configured with Git (via the
 ``user.signingKey`` configuration parameter), then patatt requires no
 further configuration. You can start signing your patches by installing
 the git-send-email hook in the repository you want::
@@ -902,7 +902,7 @@ the new default in GnuPG v2). To set it, add (or modify) the
 Using the kernel.org web of trust repository
 --------------------------------------------
 
-Kernel.org maintains a git repository with developers' public keys as a
+Kernel.org maintains a Git repository with developers' public keys as a
 replacement for replicating keyserver networks that have gone mostly
 dark in the past few years. The full documentation for how to set up
 that repository as your source of public keys can be found here:
diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst
index 7ae79452e1b46d..4c28a239e8cb58 100644
--- a/Documentation/process/submitting-patches.rst
+++ b/Documentation/process/submitting-patches.rst
@@ -456,7 +456,7 @@ When to use Acked-by:, Cc:, and Co-developed-by:
 ------------------------------------------------
 
 The Signed-off-by: tag indicates that the signer was involved in the
-development of the patch, or that he/she was in the patch's delivery path.
+development of the patch, or that they were in the patch's delivery path.
 
 If a person was not directly involved in the preparation or handling of a
 patch but wishes to signify and record their approval of it then they can
diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst
index 17ce8d76befc1b..791178ac8bec68 100644
--- a/Documentation/scheduler/index.rst
+++ b/Documentation/scheduler/index.rst
@@ -23,5 +23,7 @@ Scheduler
     sched-stats
     sched-ext
     sched-debug
+    sched-paravirt
+    sched-preemption
 
     text_files
diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst
index 794ae80b3ba308..13ba2cb7831c69 100644
--- a/Documentation/scheduler/sched-ext.rst
+++ b/Documentation/scheduler/sched-ext.rst
@@ -503,6 +503,19 @@ and edge cases, to name a few examples:
   class, in which case it will exit the tick-dispatch loop even though it is runnable
   and has a non-zero slice.
 
+* Under proxy execution, sched_ext continues to observe the donor as the
+  current scheduling context. Accordingly, ``ops.running()`` and
+  ``ops.stopping()`` report when the donor's scheduling context becomes active
+  and inactive, even when the donor is blocked and a lock owner executes on its
+  behalf. The physical execution context is intentionally not reported through
+  these callbacks.
+
+  A blocked donor enters a running session only after proxy resolution finds
+  an execution context. The session remains active if only the physical
+  execution context changes while the donor remains the same. Running sessions
+  are tracked so that ``ops.running()`` and ``ops.stopping()`` remain paired
+  and are not emitted recursively.
+
 See the "Scheduling Cycle" section for a more detailed description of how
 a freshly woken up task gets on a CPU.
 
diff --git a/Documentation/scheduler/sched-paravirt.rst b/Documentation/scheduler/sched-paravirt.rst
new file mode 100644
index 00000000000000..311cdbf2722b56
--- /dev/null
+++ b/Documentation/scheduler/sched-paravirt.rst
@@ -0,0 +1,67 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. _sched-paravirt:
+
+Preferred CPUs
+==============
+
+In paravirtualized environments CPU overcommit is a common scenario.
+i.e. the sum of virtual CPUs (vCPUs) of all VMs is greater than number of
+physical CPUs (pCPUs). Under such conditions when all or many VMs have
+high utilization, hypervisor won't be able to satisfy the CPU requirement
+and has to context switch within or across VMs. The hypervisor needs to
+preempt one vCPU to run another. This is called vCPU preemption.
+This is more expensive compared to task context switch within a vCPU, since
+hypervisor lacks vCPU context and could preempt a critical section which
+slows forward progress.
+
+In such cases it is better that combined vCPU demand from all VMs is reduced
+by not using some of the vCPUs in each VM. vCPUs where workload can be safely
+scheduled which won't increase any contention for pCPU are called
+"Preferred CPUs".
+
+One of the main design constructs is that preferred CPUs are always
+a subset of active CPUs. In most cases preferred CPUs will be same as
+active CPUs. When there is pCPU contention, Preferred CPUs will reduce
+based on the steal time. When the pCPU contention goes away as indicated
+by steal time, Preferred CPUs could become same as active CPUs again.
+The policy decisions are to be taken by driver.
+For example, steal_governor. Look at its documentation for more
+details. (``drivers/virt/steal_governor.c``)
+
+Scheduling decisions such as wakeup, pushing the task etc, need this
+CPU state info. This is maintained in ``cpu_preferred_mask``.
+vCPUs which are not in ``cpu_preferred_mask`` should be treated as vCPUs which
+should not be used at this moment provided it doesn't break user affinity.
+
+This is achieved by:
+
+1. Selecting a preferred CPU at wakeup using fallback mechanism.
+2. Pushing the task away from non-preferred CPU at tick.
+3. Selecting only preferred CPUs for load balance.
+
+``/sys/devices/system/cpu/preferred`` prints the current ``cpu_preferred_mask``
+in cpulist format.
+
+Notes:
+
+1. This feature is available under ``CONFIG_PREFERRED_CPU``. Driver which
+   makes decisions should enable it. For example, steal_governor driver
+   (``CONFIG_STEAL_GOVERNOR``). On enabling the driver, CPU preferred state
+   can change based on steal time. Without the driver, preferred CPUs is
+   same as active CPUs.
+
+2. This feature works for the FAIR class only.
+
+3. A pinned task, which can't be moved to preferred CPUs will continue
+   to run based on its affinity. But no load balancing happens if it is affined
+   only on non-preferred CPUs.
+
+4. Decision to change the preferred CPU state is driven by the kernel.
+   Hence it shouldn't break user affinities. One of the main reasons why
+   CPU hotplug or Isolated cpuset partitions was not a solution.
+
+5. This feature works best only when all the Guest VMs enable the feature as
+   it is a co-operative scheme. If a specific VM doesn't enable this feature
+   it may end up with more CPUs than others, still should lead to better
+   performance when seen from system view.
+   Users who enable this driver must ensure it is enabled in all Guest VMs.
diff --git a/Documentation/scheduler/sched-preemption.rst b/Documentation/scheduler/sched-preemption.rst
new file mode 100644
index 00000000000000..b48e53587a7afa
--- /dev/null
+++ b/Documentation/scheduler/sched-preemption.rst
@@ -0,0 +1,87 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================
+Scheduler preemption
+=====================
+
+The kernel can be built to run kernel code either uninterruptibly, or with
+varying degrees of preemptibility.  These are the *preemption models*.
+They apply to kernel code only: a task running in user space is always
+preempted by the scheduler, whichever model is selected.
+
+When CONFIG_PREEMPT_DYNAMIC is enabled the preemption model can additionally
+be selected at boot time with the ``preempt=`` command line parameter, without
+rebuilding the kernel.  See
+Documentation/admin-guide/kernel-parameters.txt for the parameter itself.
+The active model can also be read and changed through
+/sys/kernel/debug/sched/preempt, which exists for debugging purposes and may
+change.
+
+The models selectable at runtime are:
+
+	=========	====================================================
+	none		No preemption of kernel code other than at explicit
+			``cond_resched()`` / blocking points.
+	voluntary	As ``none``, plus ``might_sleep()`` sites.
+	full		Any section that is not explicitly preempt disabled
+			may be preempted at any time.
+	lazy		As ``full``, except that the scheduling request is
+			delayed until the return to user space or the next
+			tick, whichever comes first.  This delay does not
+			apply to real-time tasks.
+	=========	====================================================
+
+Not every model is available on every kernel.  An architecture that provides
+lazy-preempt support offers only ``full`` and ``lazy``, and ``none`` and
+``voluntary`` are rejected there.  A PREEMPT_RT kernel always offers ``full``,
+and ``lazy`` as well if the architecture provides it.
+
+The model that is actually active is reported in the boot log::
+
+	Dynamic Preempt: full
+
+Honouring a scheduling request
+==============================
+
+A wakeup makes a task runnable, and the scheduler then decides whether it
+should run.  If there is an idle CPU which is suitable then the task is moved
+there.  Otherwise the scheduler has to decide which task to preempt, and the
+task that has to leave the CPU gets a reschedule flag.  In the tracing output
+this is the ``need-resched`` column, ``N`` for TIF_NEED_RESCHED and ``l`` for
+TIF_NEED_RESCHED_LAZY.
+
+The kernel then has to honour that request.  A task executing in user space
+can always be preempted; a task executing in kernel space can only be
+preempted where it is safe to do so, and how that is decided is what
+separates the models:
+
+``none``
+	Code paths with long loops contain explicit ``cond_resched()`` calls,
+	which perform the scheduling.
+
+``voluntary``
+	As ``none``, and functions which are known to be able to block gain
+	a ``cond_resched()``-style scheduling point as well, through
+	``might_sleep()``.
+
+``full``
+	Explicit preemption points are no longer involved.  The kernel tracks
+	whether it may be preempted and schedules once the request can be
+	honoured.
+
+``lazy``
+	As ``full``, with one difference: the request the fair scheduler sets
+	is TIF_NEED_RESCHED_LAZY, which is not honoured immediately even where
+	it could be.  It is delayed until the task returns to user space, so
+	that the in-kernel work runs to completion and every lock has been
+	dropped before the CPU is given up.  If the task has not scheduled on
+	its own by then, the request is turned into a full TIF_NEED_RESCHED on
+	the next HZ tick; this is visible as ``B`` in the ``need-resched``
+	column.
+
+Real-time tasks always use TIF_NEED_RESCHED, so the delay above does not
+apply to them.  ``lazy`` is therefore not a replacement for PREEMPT_RT: it
+keeps most of the responsiveness of ``full`` for SCHED_NORMAL tasks while
+letting them run to completion, which avoids leaving a lock behind that
+another task will immediately ask for, and avoids pushing cache-hot data out
+that would have to be pulled back in.
diff --git a/Documentation/scsi/scsi_mid_low_api.rst b/Documentation/scsi/scsi_mid_low_api.rst
index 7f59dff43eb549..5489e9d07ab7e1 100644
--- a/Documentation/scsi/scsi_mid_low_api.rst
+++ b/Documentation/scsi/scsi_mid_low_api.rst
@@ -1148,12 +1148,8 @@ The scsi_cmnd structure is defined in include/scsi/scsi_cmnd.h
 Locks
 =====
 Each struct Scsi_Host instance has a spin_lock called struct
-Scsi_Host::default_lock which is initialized in scsi_host_alloc() [found in
-hosts.c]. Within the same function the struct Scsi_Host::host_lock pointer
-is initialized to point at default_lock.  Thereafter lock and unlock
-operations performed by the mid level use the struct Scsi_Host::host_lock
-pointer.  Previously drivers could override the host_lock pointer but
-this is not allowed anymore.
+Scsi_Host::host_lock which is initialized in scsi_host_alloc() [found in
+hosts.c].
 
 
 Autosense
diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst
index e500ec84e3ef72..689eccf6a5921a 100644
--- a/Documentation/sound/alsa-configuration.rst
+++ b/Documentation/sound/alsa-configuration.rst
@@ -2407,6 +2407,12 @@ quirk_flags
           the requested rate.  A device advertising a single rate is otherwise
           never sent the request at all, and some require it before streaming
           will start.
+        * bit 33: ``swap_rates``
+          The clock source applies a requested rate to the other base-rate
+          family, i.e. asking for 44100 makes the device run at 48000 and vice
+          versa (same for 88200 <-> 96000 and 176400 <-> 192000).  The driver
+          sends the partner rate in SET_CUR so the device ends up running at the
+          requested one.
 
 This module supports multiple devices, autoprobe and hotplugging.
 
diff --git a/Documentation/sound/cards/hdspm.rst b/Documentation/sound/cards/hdspm.rst
index 5373e51ed076f7..deba922679422f 100644
--- a/Documentation/sound/cards/hdspm.rst
+++ b/Documentation/sound/cards/hdspm.rst
@@ -6,9 +6,11 @@ Software Interface ALSA-DSP MADI Driver
 
 2004 - winfried ritsch
 
+driver name - snd-hdspm
+
 
 Full functionality has been added to the driver. Since some of
-the Controls and startup-options  are ALSA-Standard and only the
+the Controls and startup options are ALSA-Standard, only the
 special Controls are described and discussed below.
 
 
@@ -18,7 +20,7 @@ Hardware functionality
 Audio transmission
 ------------------
 
-* number of channels --  depends on transmission mode
+* number of channels -- depends on transmission mode
 
 		The number of channels chosen is from 1..Nmax. The reason to
 		use for a lower number of channels is only resource allocation,
@@ -29,29 +31,29 @@ Audio transmission
 * Single Speed -- 1..64 channels 
 
 .. note::
-		 (Note: Choosing the 56channel mode for transmission or as
+		 Choosing the 56-channel mode for transmission or as
 		 receiver, only 56 are transmitted/received over the MADI, but
 		 all 64 channels are available for the mixer, so channel count
-		 for the driver)
+		 for the driver.
 
 * Double Speed -- 1..32 channels
 
 .. note::
-		 Note: Choosing the 56-channel mode for
-		 transmission/receive-mode , only 28 are transmitted/received
+		 Choosing the 56-channel mode for
+		 transmission/receive-mode, only 28 are transmitted/received
 		 over the MADI, but all 32 channels are available for the mixer,
-		 so channel count for the driver
+		 so channel count for the driver.
 
 
 * Quad Speed -- 1..16 channels 
 
 .. note::
 		 Choosing the 56-channel mode for
-		 transmission/receive-mode , only 14 are transmitted/received
+		 transmission/receive-mode, only 14 are transmitted/received
 		 over the MADI, but all 16 channels are available for the mixer,
-		 so channel count for the driver
+		 so channel count for the driver.
 
-* Format -- signed 32 Bit Little Endian (SNDRV_PCM_FMTBIT_S32_LE)
+* Format -- signed 32-bit Little Endian (SNDRV_PCM_FMTBIT_S32_LE)
 
 * Sample Rates --
 
@@ -67,30 +69,6 @@ Audio transmission
 
 * fragments -- 2
 
-* Hardware-pointer -- 2 Modi
-
-
-		 The Card supports the readout of the actual Buffer-pointer,
-		 where DMA reads/writes. Since of the bulk mode of PCI it is only
-		 64 Byte accurate. SO it is not really usable for the
-		 ALSA-mid-level functions (here the buffer-ID gives a better
-		 result), but if MMAP is used by the application. Therefore it
-		 can be configured at load-time with the parameter
-		 precise-pointer.
-
-
-.. hint::
-		 (Hint: Experimenting I found that the pointer is maximum 64 to
-		 large never to small. So if you subtract 64 you always have a
-		 safe pointer for writing, which is used on this mode inside
-		 ALSA. In theory now you can get now a latency as low as 16
-		 Samples, which is a quarter of the interrupt possibilities.)
-
-   * Precise Pointer -- off
-					interrupt used for pointer-calculation
-				
-   * Precise Pointer -- on
-					hardware pointer used.
 
 Controller
 ----------
@@ -104,7 +82,7 @@ hwdep-interface.
 Also all 128+256 Peak and RMS-Meter can be accessed via the
 hwdep-interface. Since it could be a performance problem always
 copying and converting Peak and RMS-Levels even if you just need
-one, I decided to export the hardware structure, so that of
+one, I decided to export the hardware structure, so that if
 needed some driver-guru can implement a memory-mapping of mixer
 or peak-meters over ioctl, or also to do only copying and no
 conversion. A test-application shows the usage of the controller.
@@ -112,9 +90,9 @@ conversion. A test-application shows the usage of the controller.
 * Latency Controls --- not implemented !!!
 
 .. note::
-	   Note: Within the windows-driver the latency is accessible of a
-	   control-panel, but buffer-sizes are controlled with ALSA from
-	   hwparams-calls and should not be changed in run-state, I did not
+	   Within the Windows driver the latency is accessible via
+	   Control Panel, but buffer sizes are controlled with ALSA from
+	   hwparams calls and should not be changed in run-state. I did not
 	   implement it here.
 
 
@@ -127,12 +105,12 @@ conversion. A test-application shows the usage of the controller.
   * Values -- "Master" "Slave"
 
 .. note::
-		  !!!! This is a hardware-function but is in conflict with the
-		  Clock-source controller, which is a kind of ALSA-standard. I
+		  !!!! This is a hardware function but is in conflict with the
+		  Clock-source controller, which is a kind of ALSA-standard. It
 		  makes sense to set the card to a special mode (master at some
-		  frequency or slave), since even not using an Audio-application
+		  frequency or slave), since even not using an Audio application
 		  a studio should have working synchronisations setup. So use
-		  Clock-source-controller instead !!!!
+		  Clock-source controller instead !!!!
 
 * Clock Source  
 
@@ -145,11 +123,11 @@ conversion. A test-application shows the usage of the controller.
     "Internal 96.0 kHz"
 
 		 Choose between Master at a specific Frequency and so also the
-		 Speed-mode or Slave (Autosync). Also see  "Preferred Sync Ref"
+		 Speed-mode or Slave (Autosync). Also see "Preferred Sync Ref".
 
 .. warning::
-       !!!! This is no pure hardware function but was implemented by
-       ALSA by some ALSA-drivers before, so I use it also. !!!
+       !!!! This is not a pure hardware function but was implemented by
+       ALSA by some ALSA drivers before, so I use it also. !!!
 
 
 * Preferred Sync Ref
@@ -165,7 +143,7 @@ conversion. A test-application shows the usage of the controller.
 		 chosen. If it is not available another is used if possible.
 
 .. note::
-		 Note: Since MADI has a much higher bit-rate than word-clock, the
+		 Since MADI has a much higher bit-rate than word-clock, the
 		 card should synchronise better in MADI Mode. But since the
 		 RME-PLL is very good, there are almost no problems with
 		 word-clock too. I never found a difference.
@@ -179,12 +157,12 @@ conversion. A test-application shows the usage of the controller.
 
   * Values -- 0 1
 
-		 Using 64-channel-modus (1) or 56-channel-modus for
+		 Using 64-channel mode (1) or 56-channel mode for
 		 MADI-transmission (0).
 
 
 .. note::
-		 Note: This control is for output only. Input-mode is detected
+		 This control is for output only. Input mode is detected
 		 automatically from hardware sending MADI.
 
 
@@ -200,7 +178,7 @@ conversion. A test-application shows the usage of the controller.
 		 Don't use to lower 5 Audio-bits on AES as additional Bits.
         
 
-* Safe Mode oder Auto Input
+* Safe Mode or Auto Input
 
   * Name -- "Safe Mode"
 
@@ -222,7 +200,7 @@ conversion. A test-application shows the usage of the controller.
   * Values -- optical coaxial
 
 
-		 Choosing the Input, optical or coaxial. If Safe-mode is active,
+		 Choosing the Input, optical or coaxial. If Safe mode is active,
 		 this is the preferred Input.
 
 Mixer
@@ -240,7 +218,7 @@ Mixer
 		 Here as a first value the channel-index is taken to get/set the
 		 corresponding mixer channel, where 0-63 are the input to output
 		 fader and 64-127 the playback to outputs fader. Value 0
-		 is channel muted 0 and 32768 an amplification of  1.
+		 is channel muted 0 and 32768 an amplification of 1.
 
 * Chn 1-64
 
@@ -250,14 +228,14 @@ Mixer
 
 * Line Out
 
-  * Name  -- "Line Out"
+  * Name -- "Line Out"
 
   * Access -- Read Write
 
   * Values -- 0 1
 
 		 Switching on and off the analog out, which has nothing to do
-		 with mixing or routing. the analog outs reflects channel 63,64.
+		 with mixing or routing. The analog outs reflects channel 63,64.
 
 
 Information (only read access)
@@ -280,7 +258,7 @@ Information (only read access)
 
 
 		 Should be "Autosync Rate", but Name used is
-		 ALSA-Scheme. External Sample frequency liked used on Autosync is
+		 ALSA-Scheme. External Sample frequency like used on Autosync is
 		 reported.
 
 
@@ -315,19 +293,19 @@ Information (only read access)
 
 		 Sync-Reference is either "WordClock", "MADI" or none.
 
-* RX 64ch --- noch nicht implementiert
+* RX 64ch --- not yet implemented
 
-       MADI-Receiver is in 64 channel mode oder 56 channel mode.
+       MADI-Receiver is in 64-channel mode or 56-channel mode.
 
 
-* AB_inp   --- not tested 
+* AB_inp --- not tested
 
 		 Used input for Auto-Input.
 
 
 * actual Buffer Position --- not implemented
 
-	   !!! this is a ALSA internal function, so no control is used !!!
+	   !!! This is an ALSA internal function, so no control is used !!!
 
 
 
@@ -352,22 +330,15 @@ Calling Parameter
 
      note: ALSA-standard
 
-* precise_ptr int array (min = 1, max = 8)
-
-     Enable precise pointer, or disable.
-
-.. note::
-     note: Use only when the application supports this (which is a special case).
-
 * line_outs_monitor int array (min = 1, max = 8)
 
      Send playback streams to analog outs by default.
 
 .. note::
-	  note: each playback channel is mixed to the same numbered output
+	  Each playback channel is mixed to the same numbered output
 	  channel (routed). This is against the ALSA-convention, where all
 	  channels have to be muted on after loading the driver, but was
-	  used before on other cards, so i historically use it again)
+	  used before on other cards, so I historically use it again.
 
 
 
@@ -375,5 +346,5 @@ Calling Parameter
 
      Enable Analog Out on Channel 63/64 by default.
 
-.. note ::
-      note: here the analog output is enabled (but not routed).
+.. note::
+      Here the analog output is enabled (but not routed).
diff --git a/Documentation/spi/multiple-data-lanes.rst b/Documentation/spi/multiple-data-lanes.rst
index 69cb532d052fb1..802aece35d56c5 100644
--- a/Documentation/spi/multiple-data-lanes.rst
+++ b/Documentation/spi/multiple-data-lanes.rst
@@ -127,7 +127,7 @@ field to indicate which mode they want to use for a given transfer.
 
 The possible values for this field have the following semantics:
 
-- :c:macro:`SPI_MULTI_BUS_MODE_SINGLE`: Only use the first lane. Other lanes are
+- :c:macro:`SPI_MULTI_LANE_MODE_SINGLE`: Only use the first lane. Other lanes are
     ignored. This means that it is operating just like a conventional SPI
     peripheral. This is the default, so it does not need to be explicitly set.
 
@@ -149,7 +149,7 @@ The possible values for this field have the following semantics:
         ----------   ----------------   ----------
             SDO 0    0-0-0-1-0-0-0-1    SDI 0
 
-- :c:macro:`SPI_MULTI_BUS_MODE_MIRROR`: Send a single data word over all of the
+- :c:macro:`SPI_MULTI_LANE_MODE_MIRROR`: Send a single data word over all of the
     lanes at the same time. This only makes sense for writes and not
     for reads.
 
@@ -160,7 +160,7 @@ The possible values for this field have the following semantics:
         struct spi_transfer xfer = {
             .tx_buf = tx_buf,
             .len = 1,
-            .multi_lane_mode = SPI_MULTI_BUS_MODE_MIRROR,
+            .multi_lane_mode = SPI_MULTI_LANE_MODE_MIRROR,
         };
 
         spi_sync_transfer(spi, &xfer, 1);
@@ -172,7 +172,7 @@ The possible values for this field have the following semantics:
             SDO 0    0-0-0-1-0-0-0-1    SDI 0
             SDO 1    0-0-0-1-0-0-0-1    SDI 1
 
-- :c:macro:`SPI_MULTI_BUS_MODE_STRIPE`: Send or receive two different data words
+- :c:macro:`SPI_MULTI_LANE_MODE_STRIPE`: Send or receive two different data words
     at the same time, one on each lane. This means that the buffer needs to be
     sized to hold data for all lanes. Data is interleaved in the buffer, with
     the first word corresponding to lane 0, the second to lane 1, and so on.
@@ -185,7 +185,7 @@ The possible values for this field have the following semantics:
         struct spi_transfer xfer = {
             .rx_buf = rx_buf,
             .len = 2,
-            .multi_lane_mode = SPI_MULTI_BUS_MODE_STRIPE,
+            .multi_lane_mode = SPI_MULTI_LANE_MODE_STRIPE,
         };
 
         spi_sync_transfer(spi, &xfer, 1);
diff --git a/Documentation/staging/rpmsg.rst b/Documentation/staging/rpmsg.rst
index 63612b7ee120ad..017867383fd943 100644
--- a/Documentation/staging/rpmsg.rst
+++ b/Documentation/staging/rpmsg.rst
@@ -299,3 +299,20 @@ by the bus, and can then start sending messages to the remote service.
 
 The plan is also to add static creation of rpmsg channels via the virtio
 config space, but it's not implemented yet.
+
+Configurable buffer sizes
+=========================
+
+By default each rpmsg buffer is 512 bytes, and the same size is used for both
+the receive and transmit directions. Remote processors that need a different
+size (for example a larger MTU, or different RX and TX queue sizes) can
+advertise it through the virtio device config space by setting the
+VIRTIO_RPMSG_F_BUFSZ feature bit.
+
+When this feature is negotiated, the driver reads struct virtio_rpmsg_config
+from the config space. This structure provides the size of a single buffer for
+each direction (TX and RX). The device should advertise TX/RX buffer sizes
+that already satisfy the alignment constraints of its buffers.
+
+If the feature is not negotiated, the default 512-byte buffers are used for
+both directions.
diff --git a/Documentation/timers/hpet.rst b/Documentation/timers/hpet.rst
index c9d05d3caaca97..254d0bb2b802fe 100644
--- a/Documentation/timers/hpet.rst
+++ b/Documentation/timers/hpet.rst
@@ -23,8 +23,37 @@ The driver supports detection of HPET driver allocation and initialization
 of the HPET before the driver module_init routine is called.  This enables
 platform code which uses timer 0 or 1 as the main timer to intercept HPET
 initialization.  An example of this initialization can be found in
-arch/x86/kernel/hpet.c.
+``arch/x86/kernel/hpet.c``.
 
 The driver provides a userspace API which resembles the API found in the
 RTC driver framework.  An example user space program is provided in
-file:samples/timers/hpet_example.c
+``samples/timers/hpet_example.c``
+
+Userspace API and ioctl Commands
+================================
+
+* ``HPET_IE_ON`` / ``HPET_IE_OFF``
+  Enables or disables the interrupt generation for the timer channel.
+
+* ``HPET_INFO``
+  It is used to query the capabilities and current status of the HPET.
+  When the ioctl call is made, the kernel populates the ``struct hpet_info``
+  structure and returns it to userspace:
+
+  .. code-block:: c
+
+     struct hpet_info {
+         unsigned long hi_ireqfreq;	/* Hz */
+         unsigned long hi_flags;	/* information */
+         unsigned short hi_hpet;
+         unsigned short hi_timer;
+     };
+
+* ``HPET_EPI`` / ``HPET_DPI``
+  Enables or disables periodic interrupts. When activated, the timer
+  triggers continuously at the specified frequency.
+
+* ``HPET_IRQFREQ``
+  Sets the interrupt frequency for the periodic timer. This request
+  takes an unsigned long argument specifying the desired frequency in Hz.
+
diff --git a/Documentation/timers/no_hz.rst b/Documentation/timers/no_hz.rst
index 7fe8ef9718d8e3..4006cab64079b4 100644
--- a/Documentation/timers/no_hz.rst
+++ b/Documentation/timers/no_hz.rst
@@ -204,9 +204,9 @@ but do not see any change in your workload's behavior.  Is this because
 your workload isn't affected that much by OS jitter, or is it because
 something else is in the way?  This section helps answer this question
 by providing a simple OS-jitter test suite, which is available on branch
-master of the following git archive:
+master of the following git repository:
 
-git://git.kernel.org/pub/scm/linux/kernel/git/frederic/dynticks-testing.git
+https://git.kernel.org/pub/scm/linux/kernel/git/frederic/cpunoise.git
 
 Clone this archive and follow the instructions in the README file.
 This test procedure will produce a trace that will allow you to evaluate
diff --git a/Documentation/trace/events-landlock.rst b/Documentation/trace/events-landlock.rst
index af9267cca47d9d..304a8d06273ee3 100644
--- a/Documentation/trace/events-landlock.rst
+++ b/Documentation/trace/events-landlock.rst
@@ -6,7 +6,7 @@ Landlock Trace Events
 =====================
 
 :Author: Mickaël Salaün
-:Date: August 2026
+:Date: September 2026
 
 Landlock emits trace events for sandbox lifecycle operations and access
 denials.  These events can be consumed by ftrace (for human-readable
@@ -31,8 +31,8 @@ Landlock trace events are organized in four categories:
 **Syscall events** are emitted during Landlock system calls:
 
 - ``landlock_create_ruleset``: a new ruleset is created
-- ``landlock_add_rule_fs``: a filesystem rule is added to a ruleset
-- ``landlock_add_rule_net``: a network port rule is added to a ruleset
+- ``landlock_add_rule_path_beneath``: a filesystem rule is added to a ruleset
+- ``landlock_add_rule_net_port``: a network port rule is added to a ruleset
 - ``landlock_create_domain``: a new domain is created from a ruleset
 - ``landlock_enforce_domain``: a domain is enforced on a thread
 
@@ -47,8 +47,8 @@ Landlock trace events are organized in four categories:
 
 **Rule evaluation events** are emitted during rule matching:
 
-- ``landlock_check_rule_fs``: a filesystem rule is evaluated
-- ``landlock_check_rule_net``: a network port rule is evaluated
+- ``landlock_check_rule_inode``: an inode-keyed rule is evaluated
+- ``landlock_check_rule_net_port``: a network-port-keyed rule is evaluated
 
 **Lifecycle events**:
 
@@ -141,10 +141,10 @@ in some field formats:
   treated as untrusted input and escaped in the trace text output so it
   cannot inject field separators or control characters.
 
-- **Other party's domain**: A scope or ptrace denial compares the
-  subject's denying domain (``domain=``, always the enforcing domain and
-  never the current task) with the other party's domain, so these
-  tracepoints also report the other party's domain as a scalar ID:
+- **Other party's domain**: A scope or ptrace denial compares the subject's
+  denying domain (``domain=``), which is the enforcing domain and not
+  necessarily the current task's domain, with the other party's domain.
+  These tracepoints also report the other party's domain as a scalar ID:
   ``tracee_domain=`` (ptrace), ``target_domain=`` (signal), and
   ``peer_domain=`` (abstract unix socket).  It is ``0`` when the other
   party is unsandboxed, and otherwise a domain ID that a consumer resolves
@@ -189,7 +189,7 @@ rather than the caller's, so correlate those to the syscall by domain ID.
 Interpreting check_rule events
 ==============================
 
-The ``check_rule_fs`` and ``check_rule_net`` events expose the per-layer
+The ``check_rule_inode`` and ``check_rule_net_port`` events expose the per-layer
 rule evaluation, which is useful for understanding *why* a specific
 access is allowed or denied.
 
@@ -231,9 +231,10 @@ contribution, not the final decision:
   through another matching rule, or the right is denied and appears in the
   ``blockers=`` field of the corresponding ``deny_access`` event.
 
-To reconstruct the decision for an object, aggregate the ``grants=``
-groups of all ``check_rule`` events emitted for that object during the
-check.
+For an access check that a consumer can delimit, aggregate the ``grants=``
+groups of all matching ``check_rule`` events.  These events do not carry a
+request ID; use their execution context and generic tracepoints to separate
+concurrent or successive checks of the same object.
 
 .. note::
 
@@ -244,29 +245,31 @@ check.
 
 For example, a program sandboxed with read and execute access to the
 whole filesystem reads ``/etc/passwd``; both the ``execve()`` and the
-read match the rule covering ``/`` (inode 2), so ``check_rule_fs`` fires
+read match the rule covering ``/`` (inode 2), so ``check_rule_inode`` fires
 with the requested rights intersected against what that rule grants.
 The ``access_request=`` mask includes ``truncate`` because the file-open hook
 evaluates that optional right alongside the required access, but the
 rule does not grant it, so ``truncate`` never appears in ``grants=``::
 
-  cat-127 [...] landlock_check_rule_fs: domain=1e40cb56f access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file}
-  cat-127 [...] landlock_check_rule_fs: domain=1e40cb56f access_request=read_file|truncate dev=0:17 ino=2 grants={read_file}
+  cat-127 [...] landlock_check_rule_inode: domain=1e40cb56f access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file}
+  cat-127 [...] landlock_check_rule_inode: domain=1e40cb56f access_request=read_file|truncate dev=0:17 ino=2 grants={read_file}
 
 The ``[...]`` replaces the ftrace CPU, flags, and timestamp columns.  A
 single ``grants=`` group means the enforcing domain has one layer.  With
 two nested sandboxes that each grant the same rights, the rule spans both
 layers, so ``grants=`` has one group per layer::
 
-  cat-128 [...] landlock_check_rule_fs: domain=184788b52 access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file,execute|read_file}
-  cat-128 [...] landlock_check_rule_fs: domain=184788b52 access_request=read_file|truncate dev=0:17 ino=2 grants={read_file,read_file}
+  cat-128 [...] landlock_check_rule_inode: domain=184788b52 access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file,execute|read_file}
+  cat-128 [...] landlock_check_rule_inode: domain=184788b52 access_request=read_file|truncate dev=0:17 ino=2 grants={read_file,read_file}
 
 eBPF access
 ===========
 
-eBPF programs attached via ``BPF_RAW_TRACEPOINT`` can access the
-tracepoint arguments directly through BTF.  The arguments include both
-standard kernel objects and Landlock-internal objects:
+BTF-enabled raw tracepoint programs attached through libbpf
+``SEC("tp_btf/...")`` sections receive typed callback arguments.  The
+event prototypes in `Event reference`_ document their argument layouts.
+The arguments include both standard kernel objects and Landlock-internal
+objects:
 
 - Standard kernel objects (``struct task_struct``, ``struct sock``,
   ``struct path``, ``struct dentry``) can be used with existing BPF
@@ -277,7 +280,7 @@ standard kernel objects and Landlock-internal objects:
   Internal struct layouts may change between kernel versions; use CO-RE
   for field relocation.
 
-A stateful eBPF program observes the full event stream and maintains
+A stateful eBPF program attached before sandbox construction can maintain
 per-domain state in BPF maps:
 
 1. On ``landlock_create_domain``: record the domain ID and parent (the
@@ -293,7 +296,10 @@ per-domain state in BPF maps:
    final statistics.
 
 This approach requires no kernel modification and no Landlock-specific
-BPF helpers.  The Landlock IDs serve as correlation keys across events.
+BPF helpers.  Landlock IDs serve as correlation keys within one boot.
+Records exported through tracing or BPF buffers can be lost, and records
+from different CPUs are not globally ordered, so consumers must detect and
+reconcile incomplete state.
 
 Audit filtering equivalence
 ===========================
diff --git a/Documentation/translations/pt_BR/admin-guide/README.rst b/Documentation/translations/pt_BR/admin-guide/README.rst
new file mode 100644
index 00000000000000..1843f89b12ccfa
--- /dev/null
+++ b/Documentation/translations/pt_BR/admin-guide/README.rst
@@ -0,0 +1,382 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _pt_BR_readme:
+
+Versão 6.x do kernel Linux <http://kernel.org/>
+===============================================
+
+Estas são as notas de lançamento da versão 6 do Linux. Leia-as com atenção,
+pois elas dizem do que se trata tudo isso, explicam como instalar o kernel e
+o que fazer se algo der errado.
+
+O que é o Linux?
+----------------
+
+  O Linux é um clone do sistema operacional Unix, escrito do zero por Linus
+  Torvalds com a ajuda de uma equipe pouco organizada de hackers espalhados
+  pela Internet. Ele busca a conformidade com o POSIX e com a Single UNIX
+  Specification.
+
+  Possui todos os recursos que você esperaria de um Unix moderno e completo,
+  incluindo multitarefa real, memória virtual, bibliotecas compartilhadas,
+  carregamento sob demanda (demand loading), executáveis compartilhados com
+  cópia-na-escrita (copy-on-write), gerenciamento de memória adequado e rede
+  multipilha, incluindo IPv4 e IPv6.
+
+  É distribuído sob a GNU General Public License v2 --- veja o arquivo
+  COPYING que o acompanha para mais detalhes.
+
+Em qual hardware ele roda?
+--------------------------
+
+  Embora tenha sido originalmente desenvolvido primeiro para PCs de 32 bits
+  baseados em x86 (386 ou superior), hoje o Linux também roda (pelo menos)
+  nas arquiteturas Compaq Alpha AXP, Sun SPARC e UltraSPARC, Motorola 68000,
+  PowerPC, PowerPC64, ARM, Hitachi SuperH, Cell, IBM S/390, MIPS, HP PA-RISC,
+  Intel IA-64, DEC VAX, AMD x86-64 Xtensa e ARC.
+
+  O Linux é facilmente portável para a maioria das arquiteturas de propósito
+  geral de 32 ou 64 bits, desde que possuam uma unidade de gerenciamento de
+  memória paginada (PMMU) e um port do compilador C da GNU (gcc), parte da
+  GNU Compiler Collection (GCC). O Linux também já foi portado para diversas
+  arquiteturas sem PMMU, embora a funcionalidade fique, obviamente, um tanto
+  limitada.
+  O Linux também foi portado para si mesmo. Você pode agora executar o kernel
+  como uma aplicação de espaço de usuário --- isso se chama UserMode Linux
+  (UML).
+
+Documentação
+------------
+
+ - Há muita documentação disponível, tanto em formato eletrônico na Internet
+   quanto em livros, tanto específica do Linux quanto referente a questões
+   gerais do UNIX. Eu recomendaria procurar nos subdiretórios de documentação
+   de qualquer site FTP do Linux pelos livros do LDP (Linux Documentation
+   Project). Este README não pretende ser a documentação do sistema: existem
+   fontes muito melhores disponíveis.
+
+ - Existem vários arquivos README no subdiretório Documentation/: eles
+   normalmente contêm notas de instalação específicas do kernel para alguns
+   drivers, por exemplo. Por favor, leia o arquivo
+   Documentation/translations/pt_BR/process/changes.rst, pois ele contém
+   informações sobre os problemas que podem resultar da atualização do seu
+   kernel.
+
+Instalando o código-fonte do kernel
+-----------------------------------
+
+ - Se você instalar as fontes completas, coloque o tarball do kernel em um
+   diretório no qual você tenha permissões (por exemplo, o seu diretório
+   pessoal) e o descompacte::
+
+     xz -cd linux-6.x.tar.xz | tar xvf -
+
+   Substitua "X" pelo número da versão do kernel mais recente.
+
+   NÃO use a área /usr/src/linux! Essa área possui um conjunto (geralmente
+   incompleto) de cabeçalhos do kernel que são usados pelos arquivos de
+   cabeçalho da biblioteca. Eles devem corresponder à biblioteca, e não ser
+   bagunçados por qualquer que seja o kernel-do-dia.
+
+ - Você também pode atualizar entre versões 6.x aplicando patches. Os patches
+   são distribuídos no formato xz. Para instalar aplicando patches, obtenha
+   todos os arquivos de patch mais novos, entre no diretório de nível
+   superior do código-fonte do kernel (linux-6.x) e execute::
+
+     xz -cd ../patch-6.x.xz | patch -p1
+
+   Substitua "x" para todas as versões maiores que a versão "x" da sua árvore
+   de fontes atual, **em ordem**, e deve dar tudo certo. Talvez você queira
+   remover os arquivos de backup (algum-nome-de-arquivo~ ou
+   algum-nome-de-arquivo.orig) e certificar-se de que não há patches que
+   falharam (algum-nome-de-arquivo# ou algum-nome-de-arquivo.rej). Se
+   houver, ou você ou eu cometemos um erro.
+
+   Diferentemente dos patches para os kernels 6.x, os patches para os kernels
+   6.x.y (também conhecidos como kernels -stable) não são incrementais; em vez
+   disso, eles se aplicam diretamente ao kernel 6.x base. Por exemplo, se o
+   seu kernel base é o 6.0 e você quer aplicar o patch 6.0.3, você não deve
+   aplicar antes os patches 6.0.1 e 6.0.2. Da mesma forma, se você está
+   rodando a versão 6.0.2 do kernel e quer saltar para a 6.0.3, você deve
+   primeiro reverter o patch 6.0.2 (ou seja, patch -R) **antes** de aplicar o
+   patch 6.0.3. Você pode ler mais sobre isso em
+   Documentation/translations/pt_BR/process/applying-patches.rst.
+
+   Como alternativa, o script patch-kernel pode ser usado para automatizar
+   esse processo. Ele determina a versão atual do kernel e aplica quaisquer
+   patches encontrados::
+
+     linux/scripts/patch-kernel linux
+
+   O primeiro argumento no comando acima é a localização do código-fonte do
+   kernel. Os patches são aplicados a partir do diretório atual, mas um
+   diretório alternativo pode ser especificado como segundo argumento.
+
+ - Certifique-se de que não há arquivos .o e dependências obsoletos espalhados
+   por aí::
+
+     cd linux
+     make mrproper
+
+   Agora você deve ter as fontes instaladas corretamente.
+
+Requisitos de software
+----------------------
+
+   Compilar e executar os kernels 6.x exige versões atualizadas de vários
+   pacotes de software. Consulte
+   Documentation/translations/pt_BR/process/changes.rst para os números de
+   versão mínimos exigidos e como obter atualizações desses pacotes. Esteja
+   ciente de que usar versões excessivamente antigas desses pacotes pode
+   causar erros indiretos, muito difíceis de rastrear; portanto, não presuma
+   que basta atualizar os pacotes quando problemas óbvios surgirem durante a
+   compilação ou a execução.
+
+Diretório de compilação do kernel
+---------------------------------
+
+   Ao compilar o kernel, todos os arquivos de saída serão, por padrão,
+   armazenados junto com o código-fonte do kernel.
+   Usar a opção ``make O=output/dir`` permite especificar um local alternativo
+   para os arquivos de saída (incluindo o .config).
+   Exemplo::
+
+     código-fonte do kernel:    /usr/src/linux-6.x
+     diretório de compilação:   /home/nome/build/kernel
+
+   Para configurar e compilar o kernel, use::
+
+     cd /usr/src/linux-6.x
+     make O=/home/nome/build/kernel menuconfig
+     make O=/home/nome/build/kernel
+     sudo make O=/home/nome/build/kernel modules_install install
+
+   Observe: se a opção ``O=output/dir`` for usada, ela deve ser usada em todas
+   as invocações do make.
+
+Configurando o kernel
+---------------------
+
+   Não pule esta etapa, mesmo que você esteja atualizando apenas uma versão
+   menor. Novas opções de configuração são adicionadas a cada lançamento, e
+   problemas estranhos aparecerão se os arquivos de configuração não estiverem
+   preparados como esperado. Se você quiser levar a sua configuração existente
+   para uma nova versão com o mínimo de trabalho, use ``make oldconfig``, que
+   perguntará apenas as respostas para as novas questões.
+
+ - Comandos alternativos de configuração são::
+
+     "make config"      Interface em texto puro.
+
+     "make menuconfig"  Menus coloridos, listas de seleção e diálogos, em
+                        modo texto.
+
+     "make nconfig"     Menus coloridos aprimorados, em modo texto.
+
+     "make xconfig"     Ferramenta de configuração baseada em Qt.
+
+     "make gconfig"     Ferramenta de configuração baseada em GTK.
+
+     "make oldconfig"   Assume o padrão para todas as questões com base no
+                        conteúdo do seu arquivo ./.config existente e
+                        pergunta sobre os novos símbolos de configuração.
+
+     "make olddefconfig"
+                        Como o anterior, mas define os novos símbolos com
+                        seus valores padrão, sem perguntar.
+
+     "make defconfig"   Cria um arquivo ./.config usando os valores padrão
+                        dos símbolos, vindos de
+                        arch/$ARCH/configs/defconfig ou de
+                        arch/$ARCH/configs/${PLATFORM}_defconfig,
+                        dependendo da arquitetura.
+
+     "make ${PLATFORM}_defconfig"
+                        Cria um arquivo ./.config usando os valores padrão
+                        dos símbolos vindos de
+                        arch/$ARCH/configs/${PLATFORM}_defconfig.
+                        Use "make help" para obter uma lista de todas as
+                        plataformas disponíveis para a sua arquitetura.
+
+     "make allyesconfig"
+                        Cria um arquivo ./.config definindo os valores dos
+                        símbolos como 'y' sempre que possível.
+
+     "make allmodconfig"
+                        Cria um arquivo ./.config definindo os valores dos
+                        símbolos como 'm' sempre que possível.
+
+     "make allnoconfig" Cria um arquivo ./.config definindo os valores dos
+                        símbolos como 'n' sempre que possível.
+
+     "make randconfig"  Cria um arquivo ./.config definindo os valores dos
+                        símbolos aleatoriamente.
+
+     "make localmodconfig" Cria uma configuração baseada na configuração
+                           atual e nos módulos carregados (lsmod). Desativa
+                           qualquer opção de módulo que não seja necessária
+                           para os módulos carregados.
+
+                           Para criar um localmodconfig para outra máquina,
+                           armazene o lsmod daquela máquina em um arquivo e
+                           o passe como parâmetro LSMOD.
+
+                           Além disso, você pode preservar módulos em certas
+                           pastas ou arquivos kconfig especificando seus
+                           caminhos no parâmetro LMC_KEEP.
+
+                   alvo$ lsmod > /tmp/mylsmod
+                   alvo$ scp /tmp/mylsmod host:/tmp
+
+                   host$ make LSMOD=/tmp/mylsmod \
+                           LMC_KEEP="drivers/usb:drivers/gpu:fs" \
+                           localmodconfig
+
+                           O acima também funciona em compilação cruzada.
+
+     "make localyesconfig" Semelhante ao localmodconfig, exceto que converte
+                           todas as opções de módulo em opções embutidas
+                           (=y). Você também pode preservar módulos com o
+                           LMC_KEEP.
+
+     "make kvm_guest.config"   Habilita opções adicionais para suporte a
+                               kernel convidado do kvm.
+
+     "make xen.config"   Habilita opções adicionais para suporte a kernel
+                         convidado dom0 do xen.
+
+     "make tinyconfig"  Configura o menor kernel possível.
+
+   Você pode encontrar mais informações sobre o uso das ferramentas de
+   configuração do kernel Linux em Documentation/kbuild/kconfig.rst.
+
+ - NOTAS sobre o ``make config``:
+
+    - Ter drivers desnecessários deixará o kernel maior e, em algumas
+      circunstâncias, pode levar a problemas: sondar uma placa controladora
+      inexistente pode confundir as suas outras controladoras.
+
+    - Um kernel com emulação matemática compilada ainda usará o
+      coprocessador, se houver um presente: a emulação matemática
+      simplesmente nunca será usada nesse caso. O kernel ficará um pouco
+      maior, mas funcionará em máquinas diferentes, independentemente de
+      terem ou não um coprocessador matemático.
+
+    - Os detalhes de configuração de "kernel hacking" normalmente resultam em
+      um kernel maior ou mais lento (ou ambos), e podem até tornar o kernel
+      menos estável, ao configurar algumas rotinas para tentar ativamente
+      quebrar código ruim e encontrar problemas no kernel (kmalloc()).
+      Portanto, você provavelmente deve responder 'n' às questões sobre
+      recursos de "development", "experimental" ou "debugging".
+
+Compilando o kernel
+-------------------
+
+ - Certifique-se de ter pelo menos o gcc 8.1 disponível.
+   Para mais informações, consulte
+   Documentation/translations/pt_BR/process/changes.rst.
+
+ - Execute um ``make`` para criar uma imagem compactada do kernel. Também é
+   possível executar ``make install`` se você tiver o lilo instalado ou se a
+   sua distribuição possuir um script de instalação reconhecido pelo
+   instalador do kernel. A maioria das distribuições populares terá um script
+   de instalação reconhecido. Talvez você queira verificar antes a
+   configuração da sua distribuição.
+
+   Para fazer a instalação de fato, você precisa ser root, mas nada da
+   compilação normal deve exigir isso. Não tome o nome de root em vão.
+
+ - Se você configurou qualquer parte do kernel como ``modules``, também terá
+   que executar ``make modules_install``.
+
+ - Saída detalhada (verbose) da compilação do kernel:
+
+   Normalmente, o sistema de compilação do kernel roda em um modo bastante
+   silencioso (mas não totalmente). No entanto, às vezes você ou outros
+   desenvolvedores do kernel precisam ver os comandos de compilação, de
+   ligação (link) ou outros exatamente como são executados. Para isso, use o
+   modo de compilação "verbose". Isso é feito passando ``V=1`` ao comando
+   ``make``, por exemplo::
+
+     make V=1 all
+
+   Para que o sistema de compilação também informe o motivo da recompilação
+   de cada alvo, use ``V=2``. O padrão é ``V=0``.
+
+ - Mantenha um kernel de backup à mão, caso algo dê errado. Isso é
+   especialmente verdadeiro para as versões de desenvolvimento, já que cada
+   novo lançamento contém código novo que não foi depurado. Certifique-se de
+   manter também um backup dos módulos correspondentes àquele kernel. Se
+   você estiver instalando um novo kernel com o mesmo número de versão do seu
+   kernel em funcionamento, faça um backup do seu diretório de módulos antes
+   de executar um ``make modules_install``.
+
+   Como alternativa, antes de compilar, use a opção de configuração do kernel
+   "LOCALVERSION" para acrescentar um sufixo único à versão normal do kernel.
+   A LOCALVERSION pode ser definida no menu "General Setup".
+
+ - Para inicializar o seu novo kernel, você precisará copiar a imagem do
+   kernel (por exemplo, .../linux/arch/x86/boot/bzImage após a compilação)
+   para o local onde o seu kernel inicializável habitual se encontra.
+
+ - Inicializar um kernel diretamente de um dispositivo de armazenamento, sem
+   a ajuda de um gerenciador de inicialização como o LILO ou o GRUB, não é
+   mais suportado na BIOS (sistemas não EFI). Em sistemas UEFI/EFI, no
+   entanto, você pode usar o EFISTUB, que permite à placa-mãe inicializar
+   diretamente no kernel. Em estações de trabalho e desktops modernos,
+   geralmente recomenda-se usar um gerenciador de inicialização, pois podem
+   surgir dificuldades com múltiplos kernels e com o secure boot.
+   Para mais detalhes sobre o EFISTUB, veja
+   "Documentation/admin-guide/efi-stub.rst".
+
+ - É importante observar que, desde 2016, o LILO (LInux LOader) não está mais
+   em desenvolvimento ativo, embora, por ter sido extremamente popular,
+   apareça com frequência na documentação. Alternativas populares incluem
+   GRUB2, rEFInd, Syslinux, systemd-boot ou EFISTUB. Por diversas razões, não
+   é recomendável usar software que não esteja mais em desenvolvimento ativo.
+
+ - É provável que a sua distribuição inclua um script de instalação e que
+   executar ``make install`` seja tudo o que é necessário. Caso não seja
+   assim, você terá que identificar o seu gerenciador de inicialização e
+   consultar a documentação dele, ou configurar a sua EFI.
+
+Instruções legadas do LILO
+--------------------------
+
+
+ - Se você usa o LILO, as imagens do kernel são especificadas no arquivo
+   /etc/lilo.conf. O arquivo de imagem do kernel geralmente é /vmlinuz,
+   /boot/vmlinuz, /bzImage ou /boot/bzImage. Para usar o novo kernel, salve
+   uma cópia da imagem antiga e copie a nova imagem por cima da antiga.
+   Então, você DEVE EXECUTAR O LILO NOVAMENTE para atualizar o mapa de
+   carregamento! Se não fizer isso, não conseguirá inicializar a nova imagem
+   do kernel.
+
+ - Reinstalar o LILO geralmente é uma questão de executar /sbin/lilo. Talvez
+   você queira editar o /etc/lilo.conf para especificar uma entrada para a
+   sua imagem antiga do kernel (digamos, /vmlinux.old), caso a nova não
+   funcione. Veja a documentação do LILO para mais informações.
+
+ - Após reinstalar o LILO, deve estar tudo pronto. Desligue o sistema,
+   reinicie e aproveite!
+
+ - Se algum dia você precisar alterar o dispositivo raiz padrão, o modo de
+   vídeo, etc. na imagem do kernel, use as opções de inicialização do seu
+   gerenciador de inicialização onde for apropriado. Não é necessário
+   recompilar o kernel para alterar esses parâmetros.
+
+ - Reinicie com o novo kernel e aproveite.
+
+
+Se algo der errado
+------------------
+
+Se você tiver problemas que pareçam ser causados por bugs do kernel, por
+favor, siga as instruções em
+'Documentation/admin-guide/reporting-issues.rst'.
+
+Dicas sobre como entender os relatórios de bugs do kernel estão em
+'Documentation/admin-guide/bug-hunting.rst'. Mais sobre depuração do kernel
+com o gdb está em
+'Documentation/process/debugging/gdb-kernel-debugging.rst' e
+'Documentation/process/debugging/kgdb.rst'.
diff --git a/Documentation/translations/pt_BR/admin-guide/devices.rst b/Documentation/translations/pt_BR/admin-guide/devices.rst
new file mode 100644
index 00000000000000..614a32c33968e2
--- /dev/null
+++ b/Documentation/translations/pt_BR/admin-guide/devices.rst
@@ -0,0 +1,278 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _pt_BR_admin_devices:
+
+Dispositivos alocados no Linux (versão 4.x+)
+============================================
+
+Esta lista é a Linux Device List, o registro oficial dos números de
+dispositivo alocados e dos nós do diretório ``/dev`` para o sistema
+operacional Linux.
+
+A versão deste documento em lanana.org não é mais mantida. Esta versão, no
+kernel Linux mainline, é o documento mestre. Atualizações devem ser enviadas
+como patches aos mantenedores do kernel (veja o documento
+:ref:`Documentation/translations/pt_BR/process/submitting-patches.rst <pt_BR_submittingpatches>`).
+Explore especificamente as seções intituladas "CHAR and MISC DRIVERS" e
+"BLOCK LAYER" no arquivo MAINTAINERS para encontrar os mantenedores certos
+a envolver para dispositivos de caractere e de bloco.
+
+Este documento é incluído por referência no Filesystem Hierarchy Standard
+(FHS). O FHS está disponível em https://www.pathname.com/fhs/.
+
+Alocações marcadas com (68k/Amiga) aplicam-se somente ao Linux/68k na
+plataforma Amiga. Alocações marcadas com (68k/Atari) aplicam-se somente ao
+Linux/68k na plataforma Atari.
+
+Este documento está em domínio público. Os autores solicitam, no entanto,
+que versões semanticamente alteradas não sejam distribuídas sem permissão
+dos autores, presumindo-se que os autores possam ser contatados sem um
+esforço desarrazoado.
+
+
+.. attention::
+
+  AUTORES DE DRIVERS DE DISPOSITIVO, POR FAVOR LEIAM ISTO
+
+  O Linux agora possui amplo suporte à alocação dinâmica de numeração de
+  dispositivos e pode usar ``sysfs`` e ``udev`` (``systemd``) para lidar com
+  as necessidades de nomenclatura. Ainda existem algumas exceções na área de
+  dispositivos seriais e de inicialização. Antes de pedir um número de
+  dispositivo, certifique-se de que você realmente precisa de um.
+
+  Para ter um número maior (major) alocado, ou um número menor (minor) nas
+  situações em que isso se aplica (por exemplo, busmice), por favor envie um
+  patch aos autores conforme indicado acima.
+
+  Mantenha a descrição do dispositivo *no mesmo formato desta lista*. A razão
+  para isso é que essa é a única maneira que encontramos de garantir que
+  temos todas as informações necessárias para publicar o seu dispositivo e
+  evitar conflitos.
+
+  Por fim, às vezes temos que bancar a "polícia do espaço de nomes". Por
+  favor, não se ofenda. Frequentemente recebemos submissões de nomes em
+  ``/dev`` que fatalmente causariam conflitos no futuro. Estamos tentando
+  evitar chegar a uma situação em que teríamos que sofrer uma mudança
+  incompatível para a frente. Portanto, por favor, consulte-nos **antes** de
+  tornar públicos, de qualquer forma, os nomes e números do seu dispositivo
+  --- pelo menos até o ponto em que seria minimamente difícil alterá-los.
+
+  Sua cooperação é apreciada.
+
+.. include:: ../../../admin-guide/devices.txt
+   :literal:
+
+Entradas adicionais do diretório ``/dev/``
+------------------------------------------
+
+Esta seção detalha as entradas adicionais que devem ou podem existir no
+diretório /dev. É preferível que os links simbólicos usem a mesma forma
+(absoluta ou relativa) indicada aqui. Os links são classificados como
+"físicos" (hard) ou "simbólicos", dependendo do tipo de link preferido; se
+possível, o tipo de link indicado deve ser usado.
+
+Links obrigatórios
+++++++++++++++++++
+
+Estes links devem existir em todos os sistemas:
+
+=============== =============== =============== ==============================
+/dev/fd         /proc/self/fd   simbólico       Descritores de arquivo
+/dev/stdin      fd/0            simbólico       Descritor de arquivo da stdin
+/dev/stdout     fd/1            simbólico       Descritor de arquivo da stdout
+/dev/stderr     fd/2            simbólico       Descritor de arquivo da stderr
+/dev/nfsd       socksys         simbólico       Exigido pelo iBCS-2
+/dev/X0R        null            simbólico       Exigido pelo iBCS-2
+=============== =============== =============== ==============================
+
+Observação: ``/dev/X0R`` é <letra X>-<dígito 0>-<letra R>.
+
+Links recomendados
+++++++++++++++++++
+
+Recomenda-se que estes links existam em todos os sistemas:
+
+
+=============== =============== =============== ==========================
+/dev/core       /proc/kcore     simbólico       Compatibilidade retroativa
+/dev/ramdisk    ram0            simbólico       Compatibilidade retroativa
+/dev/ftape      qft0            simbólico       Compatibilidade retroativa
+/dev/bttv0      video0          simbólico       Compatibilidade retroativa
+/dev/radio      radio0          simbólico       Compatibilidade retroativa
+/dev/i2o*       /dev/i2o/*      simbólico       Compatibilidade retroativa
+=============== =============== =============== ==========================
+
+Os nomes alternativos ``/dev/scd?``, sugeridos anteriormente para os
+``/dev/sr?`` de CD-ROM e outras unidades ópticas (que usam comandos SCSI),
+foram removidos na versão 174 do ``udev``, lançada em 2011.
+
+Links definidos localmente
+++++++++++++++++++++++++++
+
+Os links a seguir podem ser estabelecidos localmente para se adequar à
+configuração do sistema. Isto é meramente uma tabulação da prática
+existente, e não constitui uma recomendação. No entanto, se existirem, eles
+devem ter os seguintes usos.
+
+=============== ================= =============== ==============================
+/dev/mouse      porta de mouse    simbólico       Dispositivo de mouse atual
+/dev/tape       unidade de fita   simbólico       Dispositivo de fita atual
+/dev/cdrom      unidade de CD-ROM simbólico       Dispositivo de CD-ROM atual
+/dev/scanner    scanner           simbólico       Scanner atual
+/dev/modem      porta de modem    simbólico       Dispositivo de discagem atual
+/dev/root       dispositivo raiz  simbólico       Sistema de arquivos raiz atual
+/dev/swap       área de swap      simbólico       Dispositivo de swap atual
+=============== ================= =============== ==============================
+
+O ``/dev/modem`` não deve ser usado para um modem que suporte tanto discagem
+de entrada quanto de saída, pois isso tende a causar problemas com arquivos
+de trava (lock). Se existir, o ``/dev/modem`` deve apontar para o
+dispositivo TTY primário apropriado (o uso dos dispositivos alternativos de
+saída é obsoleto).
+
+Para dispositivos SCSI, o ``/dev/tape`` e o ``/dev/cdrom`` devem apontar
+para os dispositivos *cozidos* (*cooked*) (``/dev/st*`` e ``/dev/sr*``,
+respectivamente), enquanto o ``/dev/scanner`` deve apontar para o
+dispositivo SCSI genérico apropriado (``/dev/sg*``).
+
+O ``/dev/mouse`` pode apontar para um dispositivo TTY serial primário, um
+dispositivo de mouse em hardware, ou um socket para um programa de driver de
+mouse (por exemplo, ``/dev/gpmdata``).
+
+Sockets e pipes
++++++++++++++++
+
+Sockets e pipes nomeados não transitórios podem existir em /dev. As entradas
+comuns são:
+
+=============== ======== =============================
+/dev/printer    socket   socket local do lpd
+/dev/log        socket   socket local do syslog
+/dev/gpmdata    socket   multiplexador de mouse do gpm
+=============== ======== =============================
+
+Pontos de montagem
+++++++++++++++++++
+
+Os nomes a seguir são reservados para a montagem de sistemas de arquivos
+especiais sob /dev. Esses sistemas de arquivos especiais fornecem interfaces
+do kernel que não podem ser fornecidas com nós de dispositivo padrão.
+
+=============== ======== ==================================================
+/dev/pts        devpts   Sistema de arquivos de escravos PTY
+/dev/shm        tmpfs    Acesso de manutenção à memória compartilhada POSIX
+=============== ======== ==================================================
+
+Dispositivos de terminal
+------------------------
+
+Dispositivos de terminal, ou TTY, são uma classe especial de dispositivos de
+caractere. Um dispositivo de terminal é qualquer dispositivo que possa atuar
+como terminal de controle de uma sessão; isso inclui consoles virtuais,
+portas seriais e pseudoterminais (PTYs).
+
+Todos os dispositivos de terminal compartilham um conjunto comum de
+capacidades conhecidas como disciplinas de linha (line disciplines); estas
+incluem a disciplina de linha de terminal comum, bem como os modos SLIP e
+PPP.
+
+Todos os dispositivos de terminal são nomeados de forma semelhante; esta
+seção explica a nomenclatura e o uso dos vários tipos de TTY. Observe que as
+convenções de nomenclatura incluem várias verrugas históricas; algumas delas
+são específicas do Linux, algumas foram herdadas de outros sistemas, e
+algumas refletem o Linux tendo superado uma convenção emprestada.
+
+Um sinal de cerquilha (``#``) em um nome de dispositivo é usado aqui para
+indicar um número decimal sem zeros à esquerda.
+
+Consoles virtuais e o dispositivo de console
+++++++++++++++++++++++++++++++++++++++++++++
+
+Consoles virtuais são telas de terminal em tela cheia no monitor de vídeo do
+sistema. Consoles virtuais são nomeados ``/dev/tty#``, com a numeração
+começando em ``/dev/tty1``; o ``/dev/tty0`` é o console virtual atual.
+O ``/dev/tty0`` é o dispositivo que deve ser usado para acessar a placa de
+vídeo do sistema naquelas arquiteturas para as quais os dispositivos de
+frame buffer (``/dev/fb*``) não são aplicáveis. Não use o ``/dev/console``
+para esse fim.
+
+O dispositivo de console, ``/dev/console``, é o dispositivo para o qual as
+mensagens do sistema devem ser enviadas, e no qual os logins devem ser
+permitidos em modo monousuário. A partir do Linux 2.1.71, o ``/dev/console``
+é gerenciado pelo kernel; para versões anteriores, ele deve ser um link
+simbólico para o ``/dev/tty0``, para um console virtual específico como o
+``/dev/tty1``, ou para um dispositivo primário de porta serial (``tty*``,
+não ``cu*``), dependendo da configuração do sistema.
+
+Portas seriais
+++++++++++++++
+
+Portas seriais são portas seriais RS-232 e qualquer dispositivo que simule
+uma, seja em hardware (como modems internos) ou em software (como o driver
+ISDN). No Linux, cada porta serial possui dois nomes de dispositivo: o
+primário, ou de chamada de entrada (callin), e o alternativo, ou de chamada
+de saída (callout). Cada tipo de dispositivo é indicado por uma letra
+diferente. Para qualquer letra X, os nomes dos dispositivos são
+``/dev/ttyX#`` e ``/dev/cux#``, respectivamente; por razões históricas, o
+``/dev/ttyS#`` e o ``/dev/ttyC#`` correspondem ao ``/dev/cua#`` e ao
+``/dev/cub#``. No futuro, deve-se esperar que múltiplas letras sejam usadas;
+todas as letras serão maiúsculas para o dispositivo "tty" (por exemplo,
+``/dev/ttyDP#``) e minúsculas para o dispositivo "cu" (por exemplo,
+``/dev/cudp#``).
+
+Os nomes ``/dev/ttyQ#`` e ``/dev/cuq#`` são reservados para uso local.
+
+Os dispositivos alternativos provêem exclusão baseada no kernel e padrões um
+tanto diferentes dos dispositivos primários. Seu principal propósito é
+permitir o uso de portas seriais com programas sem suporte inerente a portas
+seriais, ou com suporte defeituoso. Seu uso é obsoleto, e eles podem ser
+removidos em uma versão futura do Linux.
+
+A arbitragem de portas seriais é provida pelo uso de arquivos de trava com
+os nomes ``/var/lock/LCK..ttyX#``. O conteúdo do arquivo de trava deve ser o
+PID do processo que está travando, como um número ASCII.
+
+É prática comum instalar links como /dev/modem, que apontam para portas
+seriais. Para garantir o travamento adequado na presença desses links,
+recomenda-se que o software persiga os links simbólicos e trave todos os
+nomes possíveis; adicionalmente, recomenda-se que um arquivo de trava seja
+instalado com o dispositivo alternativo correspondente. Para evitar
+impasses (deadlocks), recomenda-se que as travas sejam adquiridas na
+seguinte ordem, e liberadas na ordem inversa:
+
+        1. O nome do link simbólico, se houver (``/var/lock/LCK..modem``)
+        2. O nome "tty" (``/var/lock/LCK..ttyS2``)
+        3. O nome do dispositivo alternativo (``/var/lock/LCK..cua2``)
+
+No caso de links simbólicos aninhados, os arquivos de trava devem ser
+instalados na ordem em que os links simbólicos são resolvidos.
+
+Sob nenhuma circunstância uma aplicação deve manter uma trava enquanto
+espera que outra seja liberada. Além disso, aplicações que tentam criar
+arquivos de trava para os nomes de dispositivo alternativos correspondentes
+devem levar em conta a possibilidade de serem usadas em um TTY que não seja
+de porta serial, para o qual nenhum dispositivo alternativo existiria.
+
+Pseudoterminais (PTYs)
+++++++++++++++++++++++
+
+Pseudoterminais, ou PTYs, são usados para criar sessões de login ou para
+prover outras capacidades que exijam uma disciplina de linha TTY (incluindo
+capacidade de SLIP ou PPP) a processos arbitrários geradores de dados. Cada
+PTY tem um lado mestre, nomeado ``/dev/pty[p-za-e][0-9a-f]``, e um lado
+escravo, nomeado ``/dev/tty[p-za-e][0-9a-f]``. O kernel arbitra o uso dos
+PTYs permitindo que cada lado mestre seja aberto apenas uma vez.
+
+Uma vez que o lado mestre tenha sido aberto, o dispositivo escravo
+correspondente pode ser usado da mesma maneira que qualquer dispositivo TTY.
+Os dispositivos mestre e escravo são conectados pelo kernel, gerando o
+equivalente a um pipe bidirecional com capacidades de TTY.
+
+Versões recentes dos kernels Linux e da GNU libc contêm suporte ao esquema
+de nomenclatura System V/Unix98 para PTYs, que atribui um dispositivo comum,
+``/dev/ptmx``, a todos os mestres (abri-lo lhe dará automaticamente um PTY
+não atribuído anteriormente) e um subdiretório, ``/dev/pts``, para os
+escravos; os escravos são nomeados com inteiros decimais (``/dev/pts/#`` em
+nossa notação). Isso remove o problema de esgotar o espaço de nomes e
+permite que o kernel crie automaticamente os nós de dispositivo para os
+escravos sob demanda, usando o sistema de arquivos "devpts".
diff --git a/Documentation/translations/pt_BR/admin-guide/index.rst b/Documentation/translations/pt_BR/admin-guide/index.rst
new file mode 100644
index 00000000000000..471b76f00e006b
--- /dev/null
+++ b/Documentation/translations/pt_BR/admin-guide/index.rst
@@ -0,0 +1,190 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. raw:: latex
+
+	\renewcommand\thesection*
+	\renewcommand\thesubsection*
+
+==================================================
+Guia do usuário e do administrador do kernel Linux
+==================================================
+
+A seguir está uma coleção de documentos voltados ao usuário que foram
+adicionados ao kernel ao longo do tempo. Ainda há pouca ordem ou organização
+geral aqui --- este material não foi escrito para ser um documento único e
+coerente! Com sorte, as coisas melhorarão rapidamente com o tempo.
+
+Guias gerais para a administração do kernel
+-------------------------------------------
+
+Esta seção inicial contém informações gerais, incluindo o arquivo README que
+descreve o kernel como um todo, documentação sobre os parâmetros do kernel,
+etc.
+
+.. toctree::
+   :maxdepth: 1
+
+   README
+   devices
+
+Todolist:
+
+*   features
+
+Uma grande parte da interface administrativa do kernel são os sistemas de
+arquivos virtuais /proc e sysfs; estes documentos descrevem como interagir
+com eles.
+
+Todolist:
+
+*   sysfs-rules
+*   sysctl/index
+*   cputopology
+*   abi
+
+Documentação relacionada à segurança:
+
+Todolist:
+
+*   hw-vuln/index
+*   LSM/index
+*   perf-security
+
+Inicializando o kernel
+----------------------
+
+Todolist:
+
+*   bootconfig
+*   kernel-parameters
+*   efi-stub
+*   initrd
+
+Rastreando e identificando problemas
+------------------------------------
+
+Aqui está um conjunto de documentos voltados a usuários que estão tentando
+rastrear problemas e bugs em particular.
+
+Todolist:
+
+*   reporting-issues
+*   reporting-regressions
+*   quickly-build-trimmed-linux
+*   verify-bugs-and-bisect-regressions
+*   bug-hunting
+*   bug-bisect
+*   tainted-kernels
+*   ramoops
+*   dynamic-debug-howto
+*   init
+*   kdump/index
+*   perf/index
+*   pstore-blk
+*   clearing-warn-once
+*   kernel-per-CPU-kthreads
+*   lockup-watchdogs
+*   RAS/index
+*   sysrq
+
+Subsistemas centrais do kernel
+------------------------------
+
+Estes documentos descrevem interfaces de administração dos subsistemas
+centrais do kernel, que provavelmente são de interesse em quase qualquer
+sistema.
+
+Todolist:
+
+*   cgroup-v2
+*   cgroup-v1/index
+*   cpu-isolation
+*   cpu-load
+*   mm/index
+*   module-signing
+*   namespaces/index
+*   numastat
+*   pm/index
+*   syscall-user-dispatch
+
+Suporte a formatos binários não nativos. Observe que alguns destes
+documentos são ... antigos ...
+
+Todolist:
+
+*   binfmt-misc
+*   java
+*   mono
+
+Administração da camada de blocos e de sistemas de arquivos
+-----------------------------------------------------------
+
+Todolist:
+
+*   bcache
+*   binderfs
+*   blockdev/index
+*   cifs/index
+*   device-mapper/index
+*   ext4
+*   filesystem-monitoring
+*   nfs/index
+*   iostats
+*   jfs
+*   md
+*   ufs
+*   xfs
+
+Guias específicos de dispositivos
+---------------------------------
+
+Como configurar o seu hardware dentro do sistema Linux.
+
+Todolist:
+
+*   acpi/index
+*   aoe/index
+*   auxdisplay/index
+*   braille-console
+*   btmrvl
+*   dell_rbu
+*   edid
+*   gpio/index
+*   hw_random
+*   laptops/index
+*   lcd-panel-cgram
+*   media/index
+*   nvme-multipath
+*   parport
+*   pnp
+*   rapidio
+*   rtc
+*   serial-console
+*   svga
+*   thermal/index
+*   thunderbolt
+*   vga-softcursor
+*   video-output
+
+Análise de carga de trabalho
+----------------------------
+
+Este é o início de uma seção com informações de interesse para
+desenvolvedores de aplicações e integradores de sistemas que fazem análise
+do kernel Linux para aplicações críticas de segurança. Documentos que dão
+suporte à análise das interações do kernel com as aplicações, e às
+expectativas dos principais subsistemas do kernel, serão encontrados aqui.
+
+Todolist:
+
+*   workload-tracing
+
+Todo o resto
+------------
+
+Alguns documentos difíceis de categorizar e geralmente obsoletos.
+
+Todolist:
+
+*   ldm
+*   unicode
diff --git a/Documentation/translations/pt_BR/index.rst b/Documentation/translations/pt_BR/index.rst
index 462267e92f2f2a..5557c5e4e25012 100644
--- a/Documentation/translations/pt_BR/index.rst
+++ b/Documentation/translations/pt_BR/index.rst
@@ -67,3 +67,13 @@ kernel e sobre como ver seu trabalho integrado.
    :maxdepth: 1
 
    process/index
+
+Guias do usuário e do administrador
+===================================
+
+Os guias para usuários e administradores do kernel Linux.
+
+.. toctree::
+   :maxdepth: 1
+
+   admin-guide/index
diff --git a/Documentation/translations/pt_BR/process/2.Process.rst b/Documentation/translations/pt_BR/process/2.Process.rst
index 5ff35f10aac799..0019b7404d33a7 100644
--- a/Documentation/translations/pt_BR/process/2.Process.rst
+++ b/Documentation/translations/pt_BR/process/2.Process.rst
@@ -75,16 +75,16 @@ Como exemplo, veja como ocorreu o ciclo de desenvolvimento da versão 5.4
 (todas as datas são de 2019):
 
 	==============  ===============================
-	Setembro 15	5.3 Lançamento estável do 5.3
+	Setembro 15	Lançamento estável do 5.3
 	Setembro 30	5.4-rc1, fechamento da janela de integração
 	Outubro 6	5.4-rc2
 	Outubro 13	5.4-rc3
 	Outubro 20	5.4-rc4
-	October 27	5.4-rc5
+	Outubro 27	5.4-rc5
 	Novembro 3	5.4-rc6
 	Novembro 10	5.4-rc7
 	Novembro 17	5.4-rc8
-	Novembro 24	5.4 Lançamento estável do 5.4
+	Novembro 24	Lançamento estável do 5.4
 	==============  ===============================
 
 Como os desenvolvedores decidem quando encerrar o ciclo de desenvolvimento
@@ -330,7 +330,7 @@ A árvore de fontes do kernel contém o diretório drivers/staging/, onde reside
 muitos subdiretórios para drivers ou sistemas de arquivos que estão a caminho
 de serem adicionados à árvore do kernel. Eles permanecem em drivers/staging/
 enquanto ainda precisam de mais trabalho; uma vez concluídos, podem ser movidos
-para o kernel proper Esta é uma maneira de acompanhar drivers que não estão à
+para o kernel propriamente dito. Esta é uma maneira de acompanhar drivers que não estão à
 altura dos padrões de codificação ou de qualidade do kernel Linux, mas que as
 pessoas podem querer usar e acompanhar o desenvolvimento.
 
@@ -433,7 +433,7 @@ Existem algumas dicas que podem ajudar na sobrevivência na lista linux-kernel:
   caixa de entrada principal. É preciso ser capaz de ignorar o fluxo de e-mails
   por períodos prolongados de tempo.
 
-- Não tente acompanhar cada conversa ninguém mais faz isso. É importante
+- Não tente acompanhar cada conversa, ninguém mais faz isso. É importante
   filtrar tanto pelo tópico de interesse (embora note que conversas longas
   podem se desviar do assunto original sem que a linha de assunto do e-mail
   seja alterada) quanto pelas pessoas que estão participando.
@@ -455,13 +455,13 @@ Existem algumas dicas que podem ajudar na sobrevivência na lista linux-kernel:
 - Use respostas intercaladas, o que torna sua resposta mais fácil
   de ler. (ou seja, evite o "top-posting" — a prática de colocar sua resposta
   acima do texto citado ao qual você está respondendo). Para mais detalhes, veja
-  :ref:`Documentation/process/submitting-patches.rst <interleaved_replies>`.
+  :ref:`Documentation/process/submitting-patches.rst <pt_BR_interleaved_replies>`.
 
 - Pergunte na lista de discussão correta. A lista linux-kernel pode até ser o
   ponto de encontro geral, mas não é o melhor lugar para encontrar desenvolvedores
   de todos os subsistemas.
 
-O último ponto, encontrar a lista de discussão correta  é um lugar comum
+O último ponto, encontrar a lista de discussão correta, é um lugar comum
 onde os desenvolvedores iniciantes costumam errar. Alguém que faça uma pergunta
 relacionada a redes na lista linux-kernel quase certamente receberá uma
 sugestão educada para perguntar na lista netdev em seu lugar, já que essa é a
diff --git a/Documentation/translations/pt_BR/process/3.Early-stage.rst b/Documentation/translations/pt_BR/process/3.Early-stage.rst
index 74e741766b46bd..86c228353084bf 100644
--- a/Documentation/translations/pt_BR/process/3.Early-stage.rst
+++ b/Documentation/translations/pt_BR/process/3.Early-stage.rst
@@ -74,7 +74,7 @@ Discussão inicial
 
 Ao planejar um projeto de desenvolvimento do kernel, faz todo o sentido realizar
 discussões com a comunidade antes de iniciar a implementação. A comunicação
-inicial pode economizar tempo e problemas de várias maneiras:number of ways:
+inicial pode economizar tempo e problemas de várias maneiras:
 
  - Pode muito bem ser que o problema já seja tratado pelo kernel de maneiras
    que você não compreendeu. O kernel Linux é grande e possui uma série de
@@ -121,8 +121,8 @@ comunidade do kernel. Alguns exemplos incluem:
    consideradas inseguras e não confiáveis. Essa preocupação (entre outras)
    manteve o AppArmor fora do kernel principal (*mainline*) por anos.
 
-In each of these cases, a great deal of pain and extra work could have been
-avoided with some early discussion with the kernel developers.
+Em cada um desses casos, muito sofrimento e trabalho extra poderiam ter sido
+evitados com alguma discussão inicial com os desenvolvedores do kernel.
 
 
 Como encontrar os mantenedores?
@@ -200,8 +200,8 @@ prosseguir, mantendo a comunidade informada à medida que avança.
 Obter a aprovação oficial
 -------------------------
 
-Se o seu trabalho estiver sendo realizado em um ambiente corporativo  como é o
-caso da maior parte do trabalho no kernel do Linux —, você deve, obviamente, ter
+Se o seu trabalho estiver sendo realizado em um ambiente corporativo, como é o
+caso da maior parte do trabalho no kernel do Linux, você deve, obviamente, ter
 a permissão de gerentes devidamente autorizados antes de poder publicar os
 planos ou o código da sua empresa em uma lista de discussão pública.
 A publicação de código que não tenha sido liberado para lançamento sob uma
diff --git a/Documentation/translations/pt_BR/process/4.Coding.rst b/Documentation/translations/pt_BR/process/4.Coding.rst
index ca4c74774a9109..7c7567165725da 100644
--- a/Documentation/translations/pt_BR/process/4.Coding.rst
+++ b/Documentation/translations/pt_BR/process/4.Coding.rst
@@ -197,8 +197,8 @@ a ferramenta certa para o trabalho. Códigos que mostrem falta de atenção à
 concorrência terão um caminho difícil para entrar no mainline.
 
 
-Regressions
-***********
+Regressões
+**********
 
 Um perigo final que vale a pena mencionar é este: pode ser tentador fazer uma
 alteração (que pode trazer grandes melhorias) que faça algo quebrar para os
@@ -258,7 +258,7 @@ Note que nem todos os avisos do compilador ficam ativados por padrão. Compile o
 kernel com "make KCFLAGS=-W" para obter o conjunto completo.
 
 O kernel fornece várias opções de configuração que ativam recursos de
-depuração; a maioria delas é encontrada no submanu "kernel hacking". Várias
+depuração; a maioria delas é encontrada no submenu "kernel hacking". Várias
 dessas opções devem ser ativadas para qualquer kernel usado para fins de
 desenvolvimento ou teste. Em particular, você deve ativar:
 
diff --git a/Documentation/translations/pt_BR/process/5.Posting.rst b/Documentation/translations/pt_BR/process/5.Posting.rst
index 820a56b661dbb8..2ac7e5557805e4 100644
--- a/Documentation/translations/pt_BR/process/5.Posting.rst
+++ b/Documentation/translations/pt_BR/process/5.Posting.rst
@@ -37,7 +37,7 @@ Antes de criar patches
 ----------------------
 
 Há uma série de coisas que devem ser feitas antes de você considerar o envio
-de patches para la comunidade de desenvolvimento. Elas incluem:
+de patches para a comunidade de desenvolvimento. Elas incluem:
 
  - Teste o código tanto quanto puder. Faça uso das ferramentas de depuração
    do kernel, garanta que o kernel seja compilado com todas as combinações
@@ -106,7 +106,7 @@ que podem ajudar consideravelmente:
 
  - Como uma forma de reafirmar a diretriz acima: não misture diferentes tipos de
    alterações no mesmo patch. Se um único patch corrige uma falha crítica de
-   segurança, reorganiza algumas estruturas e reformatará o código, há uma grande
+   segurança, reorganiza algumas estruturas e reformata o código, há uma grande
    chance de que ele seja ignorado e a correção importante seja perdida.
 
  - Cada patch deve resultar em um kernel que compile e funcione corretamente; se
@@ -199,7 +199,7 @@ a alteração em um sistema de controle de versão. Ele será seguido por:
    diff associará os nomes das funções às alterações, tornando o patch resultante
    mais fácil de ser lido por outras pessoas.
 
-As tags  já mencionadas brevemente acima são usados para fornecer
+Os marcadores (tags) já mencionados brevemente acima são usados para fornecer
 informações sobre como o patch surgiu. Eles são descritos em detalhes no
 documento :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`;
 o que se segue aqui é um breve resumo.
@@ -215,7 +215,7 @@ documento com uma especificação implementada pelo patch::
 
   Link: https://example.com/somewhere.html  optional-other-stuff
 
-De acordo com as orientações do Pinguim-Chefe, um marcador Link
+De acordo com as orientações do Pinguim-Chefe, um marcador Link:
 só deve ser adicionado a um commit se ele levar a informações úteis que não
 são encontradas no próprio commit.
 
@@ -279,7 +279,7 @@ Os marcadores de uso comum são:
 
 Tenha cuidado ao adicionar os marcadores mencionados acima aos seus patches, pois
 todos, exceto Cc:, Reported-by: e Suggested-by:, precisam de permissão explícita
-fontes da pessoa nomeada. Para esses três, a permissão implícita é suficiente se
+da pessoa nomeada. Para esses três, a permissão implícita é suficiente se
 a pessoa contribuiu para o kernel Linux usando esse nome e endereço de e-mail de
 acordo com os arquivos do lore ou o histórico de commits — e, no caso de
 Reported-by: e Suggested-by:, se fizeram o relato ou a sugestão publicamente.
@@ -322,7 +322,7 @@ kernel incentiva as pessoas a pecarem pelo excesso, enviando cópias demais; nã
 assuma que as pessoas relevantes verão sua publicação nas listas de discussão. Em
 particular, as cópias devem ir para:
 
-- O(s) mantenedor(es) do(s) subsistema(s) afetado(s). Como descrito antes, o
+ - O(s) mantenedor(es) do(s) subsistema(s) afetado(s). Como descrito antes, o
    arquivo MAINTAINERS é o primeiro lugar para procurar por essas pessoas.
 
  - Outros desenvolvedores que estiveram trabalhando na mesma área — especialmente
diff --git a/Documentation/translations/pt_BR/process/6.Followthrough.rst b/Documentation/translations/pt_BR/process/6.Followthrough.rst
index d6bdaa2cb8a4df..dbdf479bb063c6 100644
--- a/Documentation/translations/pt_BR/process/6.Followthrough.rst
+++ b/Documentation/translations/pt_BR/process/6.Followthrough.rst
@@ -58,7 +58,17 @@ fácil se você mantiver algumas coisas em mente:
   de codificação e pedidos para refatorar parte do seu código em seções
   compartilhadas do kernel. Uma das funções dos mantenedores é manter as coisas
   com a mesma aparência. Às vezes, isso significa que aquele truque inteligente
-  (*clever hack*) em seu driver para contornar um problema
+  (*clever hack*) em seu driver para contornar um problema na verdade precisa
+  se tornar um recurso generalizado do kernel, pronto para a próxima vez.
+
+O que tudo isso significa é que, quando os revisores lhe enviam comentários,
+você precisa prestar atenção às observações técnicas que eles estão fazendo.
+Não deixe que a forma como eles se expressam, ou o seu próprio orgulho, impeçam
+que isso aconteça. Quando você receber comentários de revisão em um patch,
+reserve um tempo para entender o que o revisor está tentando dizer. Se
+possível, corrija as coisas que o revisor está pedindo para você corrigir. E
+responda ao revisor: agradeça-o e descreva como você responderá às suas
+perguntas.
 
 Note que você não precisa concordar com todas as mudanças sugeridas pelos
 revisores. Se você acredita que o revisor entendeu mal o seu código, explique
@@ -67,7 +77,7 @@ sugerida, descreva-a e justifique a sua solução para o problema. Se as suas
 explicações fizerem sentido, o revisor as aceitará. Contudo, caso a sua
 explicação não seja persuasiva — especialmente se outros começarem a concordar
 com o revisor —, reserve um tempo para repensar as coisas. Pode ser fácil ficar
-ceguificado por sua própria solução para um problema, a ponto de não perceber
+cego por sua própria solução para um problema, a ponto de não perceber
 que algo está fundamentalmente errado ou que, talvez, você não esteja sequer
 resolvendo o problema certo.
 
diff --git a/Documentation/translations/pt_BR/process/8.Conclusion.rst b/Documentation/translations/pt_BR/process/8.Conclusion.rst
index d5af31e7c4d9b0..2696ea16756e2f 100644
--- a/Documentation/translations/pt_BR/process/8.Conclusion.rst
+++ b/Documentation/translations/pt_BR/process/8.Conclusion.rst
@@ -21,6 +21,7 @@ encontradas através do índice do kernel do LWN em:
 	https://lwn.net/Kernel/Index/
 
 Além disso, um recurso valioso para os desenvolvedores do kernel é:
+
 	https://kernelnewbies.org/
 
 E, claro, não se deve esquecer o https://kernel.org/, o local definitivo
@@ -29,7 +30,7 @@ para informações sobre os lançamentos do kernel.
 Há uma série de livros sobre o desenvolvimento do kernel:
 
 	Linux Device Drivers, 3rd Edition (Jonathan Corbet, Alessandro
-	Rubini, and Greg Kroah-Hartman).  Online at
+	Rubini, e Greg Kroah-Hartman). Disponível online em
 	https://lwn.net/Kernel/LDD3/.
 
 	Linux Kernel Development (Robert Love).
diff --git a/Documentation/translations/pt_BR/process/adding-syscalls.rst b/Documentation/translations/pt_BR/process/adding-syscalls.rst
index cdf8b5033765a1..99607ee5f94987 100644
--- a/Documentation/translations/pt_BR/process/adding-syscalls.rst
+++ b/Documentation/translations/pt_BR/process/adding-syscalls.rst
@@ -7,7 +7,7 @@ Adicionando uma Nova Chamada de Sistema
 Este documento descreve o que está envolvido na adição de uma nova chamada de
 sistema (system call) ao kernel Linux, indo além dos conselhos normais de
 submissão em
-:ref:`Documentation/process/submitting-patches.rst <submittingpatches>`.
+:ref:`Documentation/translations/pt_BR/process/submitting-patches.rst <pt_BR_submittingpatches>`.
 
 
 Alternativas às Chamadas de Sistema
@@ -122,10 +122,10 @@ como o handle (identificador) para esse objeto -- não invente um novo tipo de
 handle de objeto para o espaço do usuário quando o kernel já possui mecanismos
 e semânticas bem definidas para o uso de descritores de arquivo.
 
-Se a sua nova chamada de sistema (2) de fato retornar un novo descritor de
+Se a sua nova chamada de sistema xyzzy(2) de fato retornar um novo descritor de
 arquivo, então o argumento de flags deve incluir um valor que seja equivalente
 a definir ``O_CLOEXEC`` no novo FD. Isso torna possível para o espaço do usuário
-fechar a janela de tempo entre a chamada ``()`` e a execução de
+fechar a janela de tempo entre a chamada ``xyzzy()`` e a execução de
 ``fcntl(fd, F_SETFD, FD_CLOEXEC)``, onde um ``fork()`` e ``execve()`` inesperados
 em outra thread poderiam vazar um descritor para o programa executado. (Contudo,
 resista à tentação de reutilizar o valor real da constante ``O_CLOEXEC``, pois
@@ -138,7 +138,7 @@ deve considerar o que significa usar a família de chamadas de sistema
 pronto para leitura ou escrita é a maneira normal de o kernel indicar ao espaço
 do usuário que um evento ocorreu no objeto correspondente do kernel.
 
-Se a sua nova chamada de sistema (2) envolver um argumento de nome de arquivo
+Se a sua nova chamada de sistema xyzzy(2) envolver um argumento de nome de arquivo
 (filename)::
 
     int sys_xyzzy(const char __user *path, ..., unsigned int flags);
@@ -152,18 +152,18 @@ o arquivo em questão; em particular, permite que o espaço do usuário solicite
 funcionalidade para um descritor de arquivo já aberto usando a flag
 ``AT_EMPTY_PATH``, fornecendo efetivamente uma operação fxyzzy(3) de graça::
 
- - xyzzyat(AT_FDCWD, path, ..., 0) é equivalente a (path,...)
+ - xyzzyat(AT_FDCWD, path, ..., 0) é equivalente a xyzzy(path,...)
  - xyzzyat(fd, "", ..., AT_EMPTY_PATH) é equivalente a fxyzzy(fd, ...)
 
 (Para mais detalhes sobre a justificativa das chamadas \*at(), veja a página de
 manual :manpage:`openat(2)`; para um exemplo de AT_EMPTY_PATH, veja a página de
 manual :manpage:`fstatat(2)`.)
 
-Se a sua nova chamada de sistema (2) envolver um parâmetro que descreve um
+Se a sua nova chamada de sistema xyzzy(2) envolver um parâmetro que descreve um
 deslocamento (offset) dentro de um arquivo, mude o seu tipo para ``loff_t`` para
 que offsets de 64 bits possam ser suportados mesmo em arquiteturas de 32 bits.
 
-Se a sua nova chamada de sistema (2) envolver funcionalidades privilegiadas,
+Se a sua nova chamada de sistema xyzzy(2) envolver funcionalidades privilegiadas,
 ela precisa ser governada pelo bit de capacidade (capability) do Linux apropriado
 (verificado com uma chamada a ``capable()``), conforme descrito na página de
 manual :manpage:`capabilities(7)`. Escolha um bit de capacidade existente que governe
@@ -172,7 +172,7 @@ apenas uma vaga relação sob o mesmo bit, pois isso vai contra o propósito das
 capabilities de dividir o poder do root. Em particular, evite adicionar novos
 usos para a capacidade ``CAP_SYS_ADMIN``, que já é excessivamente generalista.
 
-Se a sua nova chamada de sistema (2) manipular um processo diferente do
+Se a sua nova chamada de sistema xyzzy(2) manipular um processo diferente do
 processo que a chamou, ela deve ser restrita (usando uma chamada a
 ``ptrace_may_access()``) para que apenas um processo chamador com as mesmas
 permissões do processo alvo, ou com as capacidades necessárias, possa manipular
@@ -211,7 +211,7 @@ kernel, devem sempre ser enviadas com cópia (cc'ed) para linux-api@vger.kernel.
 Implementação Genérica de Chamadas de Sistema
 ---------------------------------------------
 
-O ponto de entrada principal para a sua nova chamada de sistema (2) será chamado
+O ponto de entrada principal para a sua nova chamada de sistema xyzzy(2) será chamado
 de ``sys_xyzzy()``, mas você deve adicionar esse ponto de entrada com a macro
 ``SYSCALL_DEFINEn()`` apropriada, em vez de fazer isso explicitamente. O 'n'
 indica o número de argumentos da chamada de sistema, e a macro recebe o nome da
@@ -242,7 +242,7 @@ O arquivo ``kernel/sys_ni.c`` fornece uma implementação de stub de fallback pa
 cada chamada de sistema, retornando ``-ENOSYS``. Adicione a sua nova chamada de
 sistema aqui também::
 
-    COND_SYSCALL(sys_xyzzy);
+    COND_SYSCALL(xyzzy);
 
 A sua nova funcionalidade de kernel, e a chamada de sistema que a controla, deve
 normalmente ser opcional, portanto adicione uma opção ``CONFIG`` (tipicamente em
@@ -259,7 +259,7 @@ normalmente ser opcional, portanto adicione uma opção ``CONFIG`` (tipicamente
 Para resumir, você precisa de um commit que inclua:
 
  - Opção ``CONFIG`` para a nova função, normalmente em ``init/Kconfig``
- - ``SYSCALL_DEFINEn(, ...)`` para o ponto de entrada
+ - ``SYSCALL_DEFINEn(xyzzy, ...)`` para o ponto de entrada
  - Protótipo correspondente em ``include/linux/syscalls.h``
  - Entrada na tabela genérica em ``include/uapi/asm-generic/unistd.h``
  - Stub de fallback em ``kernel/sys_ni.c``
@@ -289,7 +289,7 @@ ajustar ``arch/*/kernel/Makefile.syscalls``.
 Como o ``scripts/syscall.tbl`` serve como uma tabela de syscall comum para
 múltiplas arquiteturas, uma nova entrada é necessária nesta tabela::
 
-    468   common        sys_xyzzy
+    468   common   xyzzy     sys_xyzzy
 
 Note que adicionar uma entrada ao ``scripts/syscall.tbl`` com a ABI "common"
 também afeta todas as arquiteturas que compartilham essa tabela. Para alterações
@@ -304,7 +304,7 @@ correspondentes também devem ser feitas em ``arch/*/kernel/Makefile.syscalls``:
 Para resumir, você precisa de um commit que inclua:
 
  - Opção ``CONFIG`` para a nova função, normalmente em ``init/Kconfig``
- - ``SYSCALL_DEFINEn(, ...)`` para o ponto de entrada
+ - ``SYSCALL_DEFINEn(xyzzy, ...)`` para o ponto de entrada
  - Protótipo correspondente em ``include/linux/syscalls.h``
  - Nova entrada em ``scripts/syscall.tbl``
  - (Se necessário) Atualizações de Makefile em ``arch/*/kernel/Makefile.syscalls``
@@ -320,11 +320,11 @@ de sistema não seja especial de alguma forma (veja abaixo), isso envolve uma
 entrada "common" (para x86_64 e x32) em
 ``arch/x86/entry/syscalls/syscall_64.tbl``::
 
-    333   common        sys_xyzzy
+    333   common   xyzzy     sys_xyzzy
 
 e uma entrada "i386" em ``arch/x86/entry/syscalls/syscall_32.tbl``::
 
-    380   i386          sys_xyzzy
+    380   i386     xyzzy     sys_xyzzy
 
 Novamente, esses números estão sujeitos a alterações caso ocorram conflitos na
 janela de mesclagem (merge window) relevante.
@@ -414,7 +414,7 @@ a versão compat; a entrada em ``include/uapi/asm-generic/unistd.h`` deve usar
 
 Para resumir, você precisa de:
 
- - uma macro ``COMPAT_SYSCALL_DEFINEn(, ...)`` para o ponto de entrada compat
+ - uma macro ``COMPAT_SYSCALL_DEFINEn(xyzzy, ...)`` para o ponto de entrada compat
  - protótipo correspondente em ``include/linux/compat.h``
  - (se necessário) struct de mapeamento de 32 bits em ``include/linux/compat.h``
  - instância de ``__SC_COMP``, e não de ``__SYSCALL``, em
@@ -433,11 +433,11 @@ Você precisa estender a entrada em ``scripts/syscall.tbl`` com uma coluna extra
 para indicar que um programa de espaço do usuário de 32 bits rodando em um
 kernel de 64 bits deve atingir o ponto de entrada compat::
 
-    468   common          sys_xyzzy    compat_sys_xyzzy
+    468   common     xyzzy     sys_xyzzy    compat_sys_xyzzy
 
 Para resumir, você precisa de:
 
- - ``COMPAT_SYSCALL_DEFINEn(, ...)`` para o ponto de entrada compat
+ - ``COMPAT_SYSCALL_DEFINEn(xyzzy, ...)`` para o ponto de entrada compat
  - Protótipo correspondente em ``include/linux/compat.h``
  - Modificação da entrada em ``scripts/syscall.tbl`` para incluir uma coluna
    "compat" extra
@@ -454,7 +454,7 @@ compatibilidade voltadas para o espaço do usuário de 32 bits (AArch32):
 ``arch/arm64/tools/syscall_32.tbl``. Você precisa adicionar uma linha adicional
 a esta tabela especificando o ponto de entrada compat::
 
-    468   common          sys_xyzzy    compat_sys_xyzzy
+    468   common     xyzzy     sys_xyzzy    compat_sys_xyzzy
 
 
 Chamadas de Sistema de Compatibilidade (x86)
@@ -467,7 +467,7 @@ Primeiro, a entrada em ``arch/x86/entry/syscalls/syscall_32.tbl`` ganha uma
 coluna extra para indicar que um programa de espaço do usuário de 32 bits rodando
 em um kernel de 64 bits deve atingir o ponto de entrada compat::
 
-    380   i386          sys_xyzzy    __ia32_compat_sys_xyzzy
+    380   i386     xyzzy     sys_xyzzy    __ia32_compat_sys_xyzzy
 
 Segundo, você precisa definir o que deve acontecer para a versão da ABI x32 da
 nova chamada de sistema. Há uma escolha aqui: o layout dos argumentos deve
@@ -479,9 +479,9 @@ corresponder à versão de 32 bits, e a entrada em
 ``arch/x86/entry/syscalls/syscall_64.tbl`` é dividida para que os programas x32
 atinjam o wrapper de compatibilidade::
 
-    333   64            sys_xyzzy
+    333   64       xyzzy     sys_xyzzy
     ...
-    555   x32           __x32_compat_sys_xyzzy
+    555   x32      xyzzy     __x32_compat_sys_xyzzy
 
 Se não houver ponteiros envolvidos, então é preferível reutilizar a chamada de
 sistema de 64 bits para a ABI x32 (e, consequentemente, a entrada em
@@ -518,14 +518,14 @@ Para x86_64, isso é implementado como um ponto de entrada ``stub_xyzzy`` em
 ``arch/x86/entry/entry_64.S``, e a entrada correspondente na tabela de syscalls
 (``arch/x86/entry/syscalls/syscall_64.tbl``) é ajustada para refletir::
 
-    333   common        stub_xyzzy
+    333   common   xyzzy     stub_xyzzy
 
 O equivalente para programas de 32 bits executados em um kernel de 64 bits é
 normalmente chamado de ``stub32_xyzzy`` e implementado em
 ``arch/x86/entry/entry_64_compat.S``, com o respectivo ajuste na tabela de
 syscalls em ``arch/x86/entry/syscalls/syscall_32.tbl``::
 
-    380   i386          sys_xyzzy    stub32_xyzzy
+    380   i386     xyzzy     sys_xyzzy    stub32_xyzzy
 
 Se a chamada de sistema precisar de uma camada de compatibilidade (como na
 seção anterior), a versão ``stub32_`` precisará chamar a versão
@@ -579,12 +579,16 @@ espaço do usuário, o cabeçalho correspondente precisará ser instalado para
 compilar o teste.
 
 Certifique-se de que o autoteste seja executado com sucesso em todas as
-arquiteturas suportadas. Por exemplo, verifique se ele funciona quando compitado
+arquiteturas suportadas. Por exemplo, verifique se ele funciona quando compilado
 como um programa ABI x86_64 (-m64), x86_32 (-m32) e x32 (-mx32).
 
 Para testes mais extensos e minuciosos de novas funcionalidades, você também
 deve considerar a adição de testes ao Linux Test Project ou ao projeto
-xfstests para alterações relacionadas
+xfstests para alterações relacionadas a sistemas de arquivos.
+
+ - https://linux-test-project.github.io/
+ - git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git
+
 
 Página de Manual (Man Page)
 ---------------------------
diff --git a/Documentation/translations/pt_BR/process/applying-patches.rst b/Documentation/translations/pt_BR/process/applying-patches.rst
index 313401bc23357c..143603b991f777 100644
--- a/Documentation/translations/pt_BR/process/applying-patches.rst
+++ b/Documentation/translations/pt_BR/process/applying-patches.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _pt_BR_applying_patches:
+
 Aplicando Patches ao Kernel Linux
 +++++++++++++++++++++++++++++++++
 
@@ -12,7 +14,7 @@ Autor Original:
    manualmente, você quase certamente desejará considerar o uso do Git.
 
 Uma pergunta feita com frequência na Linux Kernel Mailing List é como aplicar
-an patch ao kernel ou, mais especificamente, a qual kernel base um patch para
+um patch ao kernel ou, mais especificamente, a qual kernel base um patch para
 uma das muitas árvores/branches deve ser aplicado. Esperamos que este documento
 explique isso a você.
 
@@ -171,7 +173,7 @@ fazer a alteração para fazê-la caber).
 O arquivo resultante pode ou não estar correto, dependendo do motivo pelo qual o
 arquivo estava diferente do esperado.
 
-Isso geralmente acontece se você tentar aplicar un patch que foi gerado contra uma
+Isso geralmente acontece se você tentar aplicar um patch que foi gerado contra uma
 versão de kernel diferente daquela que você está tentando modificar.
 
 Se você receber uma mensagem como ``Hunk #3 FAILED at 2387.``, significa que o
diff --git a/Documentation/translations/pt_BR/process/backporting.rst b/Documentation/translations/pt_BR/process/backporting.rst
index ce3f9fb4fc5ba8..afcff708570095 100644
--- a/Documentation/translations/pt_BR/process/backporting.rst
+++ b/Documentation/translations/pt_BR/process/backporting.rst
@@ -357,7 +357,7 @@ Processo de resolução
 ~~~~~~~~~~~~~~~~~~~~~
 
 Às vezes, a coisa mais fácil a fazer é apenas remover tudo, exceto a primeira
-parteda do conflito, deixando o arquivo essencialmente inalterado, e aplicar
+parte do conflito, deixando o arquivo essencialmente inalterado, e aplicar
 as alterações manualmente. Talvez o patch esteja alterando um argumento de
 chamada de função de ``0`` para ``1``, enquanto uma alteração conflitante
 adicionou um parâmetro totalmente novo (e insignificante) ao final da lista de
@@ -403,7 +403,7 @@ de volta (``git mv`` e commitando novamente) e, finalmente, esmagar (squash) o
 resultado usando ``git rebase -i`` (veja o `tutorial de rebase`_) para que ele
 apareça como um único commit quando você terminar.
 
-.. _tutorial de rebase: [https://medium.com/@slamflipstrom/a-beginners-guide-to-squashing-commits-with-git-rebase-8185cf6e62ec](https://medium.com/@slamflipstrom/a-beginners-guide-to-squashing-commits-with-git-rebase-8185cf6e62ec)
+.. _tutorial de rebase: https://medium.com/@slamflipstrom/a-beginners-guide-to-squashing-commits-with-git-rebase-8185cf6e62ec
 
 Pegadinhas
 ----------
diff --git a/Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst b/Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst
index 866c9f7e7a12d1..c40886e3d3ceeb 100644
--- a/Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst
+++ b/Documentation/translations/pt_BR/process/code-of-conduct-interpretation.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _pt_BR_code_of_conduct_interpretation:
+
 Interpretação do Código de Conduta do Kernel Linux
 ==================================================
 
diff --git a/Documentation/translations/pt_BR/process/code-of-conduct.rst b/Documentation/translations/pt_BR/process/code-of-conduct.rst
index 1ab171bf21cffd..8a3da70b7e6bf6 100644
--- a/Documentation/translations/pt_BR/process/code-of-conduct.rst
+++ b/Documentation/translations/pt_BR/process/code-of-conduct.rst
@@ -84,5 +84,5 @@ disponível em https://www.contributor-covenant.org/version/1/4/code-of-conduct.
 Interpretação
 =============
 
-Consulte o documento :ref:`code_of_conduct_interpretation` para entender como a
-comunidade do kernel Linux interpretará este documento.
+Consulte o documento :ref:`pt_BR_code_of_conduct_interpretation` para
+entender como a comunidade do kernel Linux interpretará este documento.
diff --git a/Documentation/translations/pt_BR/process/coding-assistants.rst b/Documentation/translations/pt_BR/process/coding-assistants.rst
new file mode 100644
index 00000000000000..2055af34a90819
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/coding-assistants.rst
@@ -0,0 +1,60 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+IA Assistente de código
++++++++++++++++++++++++
+
+Esta documentação fornece um guia para ferramentas de IA e desenvolvedores que
+usam IA como assistente para contribuir para o kernel do Linux.
+
+Ferramentas de IA que ajudam desenvolvedores do Linux kernel devem seguir os
+padrões de desenvolvimento do kernel:
+
+* Documentation/process/development-process.rst
+* Documentation/process/coding-style.rst
+* Documentation/process/submitting-patches.rst
+
+Para guias e conteúdo sobre códigos gerados por assistentes de IA veja:
+
+* Documentation/process/generated-content.rst
+
+Licenças e Requisitos Legais
+============================
+
+Todas as contribuições devem estar de acordo com os requisitos de licença do
+kernel:
+
+* Todo código deve ser compatível apenas com GPL-2.0
+* Use identificadores SPDX de licença apropriados
+* Veja Documentation/process/license-rules.rst para mais detalhes
+
+Assinatura e certificado de origem do desenvolvedor
+===================================================
+
+Agentes de IA NÃO DEVEM adicionar tags Signed-off-by. Apenas pessoas
+podem legalmente certificar o Certificado de Origem do Desenvolvedor (DCO).
+A pessoa que envia é responsável por:
+
+* Revisar todo código gerado por IA
+* Garantir conformidade com os requisitos de licença
+* Acrescentar sua própria tag Signed-off-by para certificar o DCO
+* Assumir toda responsabilidade pela contribuição
+
+Atribuições
+===========
+
+Quando ferramentas de IA contribuírem para o desenvolvimento do kernel,
+a atribuição adequada ajuda a rastrear a função da IA no processo de
+desenvolvimento. Contribuições devem incluir a tag de assistência seguindo este
+formato::
+
+    Assisted-by: LLM [FERRAMENTA1] [FERRAMENTA2]
+
+* ``[FERRAMENTA1] [FERRAMENTA2]`` são ferramentas de análise especializada
+    opcionais (Exemplo: coccinelle, sparse, smatch, clang-tidy)
+
+Ferramentas básicas de desenvolvimento (git, gcc, make, editors) não são
+listadas.
+
+Exemplo::
+
+  Assisted-by: LLM coccinelle sparse
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/coding-style.rst b/Documentation/translations/pt_BR/process/coding-style.rst
new file mode 100644
index 00000000000000..ac9bf0a716bca5
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/coding-style.rst
@@ -0,0 +1,1320 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _pt_BR_codingstyle:
+
+Estilo de codificação do kernel Linux
+=====================================
+
+Este é um breve documento descrevendo o estilo de codificação preferido
+para o kernel Linux. O estilo de codificação é muito pessoal, e eu não
+**forçarei** minhas opiniões a ninguém, mas isso é o que vale para tudo
+que eu tenha que manter, e eu preferiria isso para a maioria das outras
+coisas também. Por favor, considere pelo menos os pontos aqui apresentados.
+
+Primeiramente, eu sugiro imprimir uma cópia dos padrões de codificação do GNU,
+e NÃO lê-la. Queime-as, é um grande gesto simbólico.
+
+De qualquer forma, aqui vai:
+
+
+1) Indentação
+-------------
+
+As tabulações têm 8 caracteres, e portanto as indentações também têm 8
+caracteres. Há movimentos heréticos que tentam fazer com que as indentações
+tenham 4 (ou até 2!) caracteres de profundidade, e isso é semelhante a
+tentar definir o valor de PI como 3.
+
+Justificativa: A ideia central da indentação é definir claramente onde um
+bloco de controle começa e termina. Especialmente quando você esteve olhando
+para a tela por 20 horas seguidas, você vai achar muito mais fácil ver como
+a indentação funciona se ela for mais ampla.
+
+Agora, algumas pessoas afirmarão que ter indentações de 8 caracteres faz
+o código se deslocar demais para a direita e dificultam a leitura em um
+terminal de 80 caracteres. A resposta é que, se você precisar de mais de 3
+níveis de indentação, você já está em apuros de qualquer forma, e deve
+corrigir seu programa.
+
+Em resumo, indentações de 8 caracteres deixam as coisas mais fáceis de ler, e
+têm o benefício adicional de avisar quando você está aninhando suas funções
+em excesso. Atenda a esse aviso.
+
+A maneira preferida de aliviar vários níveis de indentação em uma instrução
+``switch`` é alinhar o ``switch`` e seus rótulos subordinados ``case`` na
+mesma coluna, em vez de indentar duplamente os rótulos ``case``. Por exemplo:
+
+.. code-block:: c
+
+	switch (suffix) {
+	case 'G':
+	case 'g':
+		mem <<= 30;
+		break;
+	case 'M':
+	case 'm':
+		mem <<= 20;
+		break;
+	case 'K':
+	case 'k':
+		mem <<= 10;
+		fallthrough;
+	default:
+		break;
+	}
+
+Não coloque múltiplas instruções em uma única linha, a menos que você tenha
+algo para esconder:
+
+.. code-block:: c
+
+	if (condition) do_this;
+	  do_something_everytime;
+
+Não use vírgulas para evitar usar chaves:
+
+.. code-block:: c
+
+	if (condition)
+		do_this(), do_that();
+
+Sempre use chaves para múltiplas instruções:
+
+.. code-block:: c
+
+	if (condition) {
+		do_this();
+		do_that();
+	}
+
+Também não coloque múltiplas atribuições em uma única linha. O estilo de
+codificação do kernel é extremamente simples. Evite expressões complicadas.
+
+
+Fora de comentários, documentação e, exceto em Kconfig, espaços nunca são
+usados para indentação, e o exemplo acima foi deliberadamente quebrado.
+
+Obtenha um editor decente e não deixe espaços em branco no final das linhas.
+
+
+2) Quebrando linhas longas e strings
+------------------------------------
+
+O estilo de codificação trata principalmente da legibilidade e da
+manutenibilidade usando ferramentas comumente disponíveis.
+
+O limite preferido para o comprimento de uma única linha é de 80 colunas.
+
+Instruções com mais de 80 colunas devem ser quebradas em partes sensatas,
+a menos que exceder 80 colunas aumente significativamente a legibilidade e
+não esconda informações.
+
+Os descendentes são sempre substancialmente mais curtos do que o pai e são
+colocados substancialmente à direita. Um estilo muito usado é alinhar os
+descendentes ao parêntese de abertura de uma função.
+
+Essas mesmas regras são aplicadas aos cabeçalhos de função com uma lista de
+argumentos longa.
+
+No entanto, nunca quebre strings visíveis ao usuário, como mensagens
+``printk``, porque isso prejudica a capacidade de fazer ``grep`` nelas.
+
+
+3) Posicionamento de chaves e espaços
+-------------------------------------
+
+A outra questão que sempre surge no estilo em C é o posicionamento das
+chaves. Ao contrário do tamanho da indentação, há poucos motivos técnicos
+para escolher uma estratégia de posicionamento em vez de outra, mas a forma
+preferida, como nos mostraram os profetas Kernighan e Ritchie, é colocar a
+chave de abertura no final da linha e a chave de fechamento no início, assim:
+
+.. code-block:: c
+
+	if (x is true) {
+		we do y
+	}
+
+Isso se aplica a todos os blocos de instruções que não sejam funções (if,
+switch, for, while, do). Por exemplo:
+
+.. code-block:: c
+
+	switch (action) {
+	case KOBJ_ADD:
+		return "add";
+	case KOBJ_REMOVE:
+		return "remove";
+	case KOBJ_CHANGE:
+		return "change";
+	default:
+		return NULL;
+	}
+
+No entanto, há um caso especial: as funções, que têm a chave de abertura no
+início da linha seguinte, assim:
+
+.. code-block:: c
+
+	int function(int x)
+	{
+		body of function
+	}
+
+Pessoas heréticas por todo o mundo afirmaram que essa inconsistência é ...
+bem ... inconsistente, mas todas as pessoas de bom senso sabem que (a) K&R
+estão **corretos** e (b) K&R estão certos. Além disso, as funções são
+especiais de qualquer forma (você não pode aninhá-las em C).
+
+Observe que a chave de fechamento fica vazia em uma linha própria, **exceto**
+nos casos em que ela é seguida por uma continuação da mesma instrução, ou
+seja, um ``while`` em um do-statement ou um ``else`` em um if-statement, como
+neste exemplo:
+
+.. code-block:: c
+
+	do {
+		body of do-loop
+	} while (condition);
+
+e
+
+.. code-block:: c
+
+	if (x == y) {
+		..
+	} else if (x > y) {
+		...
+	} else {
+		....
+	}
+
+Justificativa: K&R.
+
+Além disso, observe que esse posicionamento de chaves também minimiza o
+número de linhas vazias (ou quase vazias), sem qualquer perda de
+legibilidade. Assim, como o suprimento de linhas novas na sua tela não é um
+recurso renovável (pense em telas de terminal de 25 linhas), você tem mais
+linhas vazias para colocar comentários.
+
+Não use chaves desnecessariamente quando uma única instrução basta.
+
+.. code-block:: c
+
+	if (condition)
+		action();
+
+e
+
+.. code-block:: c
+
+	if (condition)
+		do_this();
+	else
+		do_that();
+
+Isso não se aplica se apenas um ramo de uma instrução condicional for uma
+única instrução; nesse último caso, use chaves em ambos os ramos:
+
+.. code-block:: c
+
+	if (condition) {
+		do_this();
+		do_that();
+	} else {
+		otherwise();
+	}
+
+Além disso, use chaves quando um laço contiver mais de uma instrução simples:
+
+.. code-block:: c
+
+	while (condition) {
+		if (test)
+			do_something();
+	}
+
+3.1) Espaços
+************
+
+O estilo do kernel Linux para o uso de espaços depende (em grande parte) do
+uso de função versus palavra-chave. Use um espaço após (a maioria das)
+palavras-chave. As exceções notáveis são ``sizeof``, ``typeof``, ``alignof`` e
+``__attribute__``, que parecem um pouco com funções (e geralmente são usadas
+com parênteses no Linux, embora não sejam obrigatórias na linguagem, como em:
+``sizeof info`` depois que ``struct fileinfo info;`` é declarado).
+
+Então use um espaço após estas palavras-chave::
+
+	if, switch, case, for, do, while
+
+mas não com ``sizeof``, ``typeof``, ``alignof`` ou ``__attribute__``. Por
+exemplo,
+
+.. code-block:: c
+
+
+	s = sizeof(struct file);
+
+Não adicione espaços ao redor (dentro) de expressões entre parênteses. Este
+exemplo é **ruim**:
+
+.. code-block:: c
+
+
+	s = sizeof( struct file );
+
+Ao declarar dados de ponteiro ou uma função que retorna um tipo de ponteiro, o
+uso preferido de ``*`` fica adjacente ao nome dos dados ou ao nome da função e
+não adjacente ao nome do tipo. Exemplos:
+
+.. code-block:: c
+
+
+	char *linux_banner;
+	unsigned long long memparse(char *ptr, char **retptr);
+	char *match_strdup(substring_t *s);
+
+Use um espaço em volta (de cada lado) da maioria dos operadores binários e
+ternários, como qualquer um destes::
+
+	=  +  -  <  >  *  /  %  |  &  ^  <=  >=  ==  !=  ?  :
+
+mas sem espaço após operadores unários::
+
+	&  *  +  -  ~  !  sizeof  typeof  alignof  __attribute__  defined
+
+sem espaço antes dos operadores unários pós-fixados de incremento e
+decremento::
+
+	++  --
+
+sem espaço após os operadores unários prefixados de incremento e
+decremento::
+
+	++  --
+
+e sem espaço ao redor dos operadores de membro de estrutura ``.`` e ``->``.
+
+Não deixe espaços em branco no final das linhas. Alguns editores com
+indentação ``inteligente`` inserem espaços no início das novas linhas conforme
+apropriado, para que você possa começar a digitar a próxima linha de código
+imediatamente. No entanto, alguns desses editores não removem o espaço em
+branco se você acabar não colocando uma linha de código ali, como quando deixa
+uma linha em branco. Como resultado, você termina com linhas contendo espaço
+em branco no final.
+
+O Git avisará você sobre patches que introduzem espaço em branco no final e
+pode remover esse espaço automaticamente para você; entretanto, se você
+aplicar uma série de patches, isso pode fazer com que patches posteriores da
+série falhem ao alterar suas linhas de contexto.
+
+
+4) Nomeação
+-----------
+
+C é uma linguagem espartana, e suas convenções de nomenclatura devem seguir o
+mesmo caminho. Ao contrário dos programadores em Modula-2 e Pascal, os
+programadores em C não usam nomes bonitinhos como ThisVariableIsATemporaryCounter.
+Um programador em C chamaria essa variável de ``tmp``, o que é muito mais fácil
+de escrever e não é menos fácil de entender.
+
+ENTRETANTO, embora nomes em camelCase sejam desencorajados, nomes descritivos
+para variáveis globais são essenciais. Chamar uma função global de ``foo`` é
+um crime.
+
+Variáveis GLOBAIS (a serem usadas somente se você **realmente** precisar) devem
+ter nomes descritivos, assim como as funções globais. Se você tiver uma função
+que conta o número de usuários ativos, você deve chamá-la de
+``count_active_users()`` ou algo parecido; você **não** deve chamá-la de
+``cntusr()``.
+
+Codificar o tipo de uma função no nome (a chamada notação Húngara) é absurdo -
+o compilador conhece os tipos de qualquer forma e pode verificar isso, e isso
+só confunde o programador.
+
+Nomes de variáveis locais devem ser curtos e diretos. Se você tiver algum
+contador inteiro aleatório de loop, provavelmente deve ser chamado de ``i``.
+Chamá-lo de ``loop_counter`` é improdutivo, se não houver chance de ser mal
+interpretado. Da mesma forma, ``tmp`` pode ser praticamente qualquer tipo de
+variável usada para manter um valor temporário.
+
+Se você tem medo de misturar os nomes de variáveis locais, você tem outro
+problema, chamado síndrome de desequilíbrio de hormônio de crescimento da
+função. Veja o capítulo 6 (Funções).
+
+Para nomes de símbolos e documentação, evite introduzir o uso novo de
+'master / slave' (ou 'slave' independente de 'master') e 'blacklist /
+whitelist'.
+
+Substituições recomendadas para 'master / slave' são:
+    '{primary,main} / {secondary,replica,subordinate}'
+    '{initiator,requester} / {target,responder}'
+    '{controller,host} / {device,worker,proxy}'
+    'leader / follower'
+    'director / performer'
+
+Substituições recomendadas para 'blacklist/whitelist' são:
+    'denylist / allowlist'
+    'blocklist / passlist'
+
+Exceções para introduzir novos usos são manter uma ABI/API do espaço do
+usuário, ou ao atualizar código para um hardware ou especificação de
+protocolo existente (a partir de 2020) que exija esses termos. Para novas
+especificações, traduza o uso da terminologia na especificação para o padrão
+de codificação do kernel quando possível.
+
+5) Tipos definidos (typedefs)
+-----------------------------
+
+Por favor, não use coisas como ``vps_t``.
+É um **erro** usar ``typedef`` para estruturas e ponteiros. Quando você vê
+
+.. code-block:: c
+
+
+	vps_t a;
+
+no código-fonte, o que isso significa?
+Em contraste, se diz
+
+.. code-block:: c
+
+	struct virtual_container *a;
+
+você consegue dizer o que ``a`` é.
+
+Muitas pessoas pensam que ``typedef``s ``ajudam na legibilidade``. Não é bem
+assim. Eles são úteis apenas para:
+
+ (a) objetos totalmente opacos (onde o ``typedef`` é usado ativamente para
+     **ocultar** o que o objeto é).
+
+     Exemplo: ``pte_t`` etc. Objetos opacos que você só pode acessar usando
+     as funções de acesso apropriadas.
+
+     .. note::
+
+       Opacidade e ``funções de acesso`` não são boas em si mesmas.
+       A razão pela qual as temos para coisas como ``pte_t`` etc. é que
+       realmente existe absolutamente **zero** informação acessível de forma portátil
+       ali.
+
+ (b) tipos inteiros claros, em que a abstração **ajuda** a evitar confusão
+     sobre se é ``int`` ou ``long``.
+
+     ``u8/u16/u32`` são typedefs perfeitamente aceitáveis, embora se
+     encaixem melhor na categoria (d) do que aqui.
+
+     .. note::
+
+       Novamente - precisa haver uma **razão** para isso. Se algo é
+       ``unsigned long``, não há motivo para fazer
+
+	typedef unsigned long myflags_t;
+
+     mas se houver uma razão clara para que em certas circunstâncias possa
+     ser ``unsigned int`` e em outras configurações possa ser ``unsigned
+     long``, então, claro, use um ``typedef``.
+
+ (c) quando você usa ``sparse`` para criar literalmente um **novo** tipo para
+     verificação de tipos.
+
+ (d) novos tipos idênticos aos tipos padrão do C99, em certas
+     circunstâncias excepcionais.
+
+     Embora levasse apenas um curto período para os olhos e o cérebro se
+     acostumarem aos tipos padrão como ``uint32_t``, algumas pessoas ainda
+     se opõem ao seu uso.
+
+     Portanto, os tipos específicos do Linux ``u8/u16/u32/u64`` e seus
+     equivalentes assinados, que são idênticos aos tipos padrão, são
+     permitidos -- embora não sejam obrigatórios em código novo seu.
+
+     Ao editar código existente que já usa um ou outro conjunto de tipos, você
+     deve seguir as escolhas existentes nesse código.
+
+ (e) tipos seguros para uso em espaço do usuário.
+
+     Em certas estruturas visíveis ao espaço do usuário, não podemos exigir
+     tipos C99 nem usar a forma ``u32`` acima. Portanto, usamos ``__u32`` e
+     tipos semelhantes em todas as estruturas compartilhadas com o espaço do
+     usuário.
+
+Pode haver outros casos também, mas a regra básica deve ser: NUNCA use um
+``typedef`` a menos que você consiga encaixar claramente em uma dessas regras.
+
+Em geral, um ponteiro, ou uma estrutura com elementos que podem ser acessados
+diretamente, **nunca** deve ser um ``typedef``.
+
+
+6) Funções
+----------
+
+As funções devem ser curtas e diretas, e fazer apenas uma coisa. Elas devem
+caber em uma ou duas telas de texto (o tamanho de tela ISO/ANSI é 80x24,
+como todos sabem), e fazer uma coisa e fazê-la bem.
+
+O comprimento máximo de uma função é inversamente proporcional à complexidade
+e ao nível de indentação dessa função. Então, se você tiver uma função
+conceitualmente simples que seja apenas uma longa (mas simples) instrução
+``switch``, em que você precisa fazer várias pequenas coisas para muitos casos
+diferentes, é aceitável ter uma função mais longa.
+
+No entanto, se você tiver uma função complexa e suspeitar que um estudante
+do primeiro ano do ensino médio, menos talentoso, talvez nem entenda do que
+se trata a função, você deve obedecer aos limites máximos ainda mais de
+perto. Use funções auxiliares com nomes descritivos (você pode pedir ao
+compilador para incorporá-las em linha se achar que é crítico para o
+desempenho, e provavelmente ele fará um trabalho melhor do que você faria).
+
+Outra medida da função é o número de variáveis locais. Elas não devem exceder
+5-10, ou algo está errado. Reflita sobre a função e divida em partes menores.
+Um cérebro humano geralmente consegue rastrear facilmente cerca de 7 coisas
+diferentes; qualquer quantidade acima disso o confunde. Você sabe que é
+brilhante, mas talvez queira entender o que fez daqui a 2 semanas.
+
+Nos arquivos de código-fonte, separe as funções com uma linha em branco. Se a
+função for exportada, a macro **EXPORT** para ela deve seguir imediatamente
+depois da linha da chave de fechamento da função. Por exemplo:
+
+.. code-block:: c
+
+	int system_is_up(void)
+	{
+		return system_state == SYSTEM_RUNNING;
+	}
+	EXPORT_SYMBOL(system_is_up);
+
+6.1) Protótipos de função
+*************************
+
+Nos protótipos de função, inclua nomes de parâmetros junto com seus tipos de
+dados. Embora isso não seja obrigatório pela linguagem C, é preferido no Linux
+porque é uma maneira simples de adicionar informações valiosas para o leitor.
+
+Não use a palavra-chave ``extern`` em declarações de função, pois isso torna
+as linhas mais longas e não é estritamente necessário.
+
+Ao escrever protótipos de função, por favor mantenha a `ordem dos elementos
+regular <https://lore.kernel.org/mm-commits/CAHk-=wiOCLRny5aifWNhr621kYrJwhfURsa0vFPeUEm8mF0ufg@mail.gmail.com/>`_.
+Por exemplo, usando este exemplo de declaração de função::
+
+ __init void * __must_check action(enum magic value, size_t size, u8 count,
+				   char *fmt, ...) __printf(4, 5) __malloc;
+
+A ordem preferida dos elementos para um protótipo de função é:
+
+- classe de armazenamento (abaixo, ``static __always_inline``, observando que
+  ``__always_inline`` é tecnicamente um atributo, mas é tratado como ``inline``)
+- atributos de classe de armazenamento (aqui, ``__init`` -- ou seja,
+  declarações de seção, mas também coisas como ``__cold``)
+- tipo de retorno (aqui, ``void *``)
+- atributos do tipo de retorno (aqui, ``__must_check``)
+- nome da função (aqui, ``action``)
+- parâmetros da função (aqui, ``(enum magic value, size_t size, u8 count,
+  char *fmt, ...)``, observando que os nomes dos parâmetros devem sempre ser
+  incluídos)
+- atributos dos parâmetros da função (aqui, ``__printf(4, 5)``)
+- atributos de comportamento da função (aqui, ``__malloc``)
+
+Observe que, para uma **definição** de função (ou seja, o corpo real da
+função), o compilador não permite atributos de parâmetro da função depois dos
+parâmetros da função. Nesses casos, eles devem vir depois dos atributos da
+classe de armazenamento (por exemplo, observe a posição alterada de
+``__printf(4, 5)`` abaixo, em comparação com o exemplo de **declaração**
+acima)::
+
+ static __always_inline __init __printf(4, 5) void * __must_check action(enum magic value,
+		size_t size, u8 count, char *fmt, ...) __malloc
+ {
+	...
+ }
+
+7) Saída centralizada de funções
+--------------------------------
+
+Embora seja depreciado por algumas pessoas, o equivalente da instrução
+``goto`` é usado com frequência pelos compiladores na forma da instrução de
+salto incondicional.
+
+A instrução ``goto`` é útil quando uma função sai de vários pontos e é
+necessária alguma tarefa comum, como limpeza. Se não for necessária nenhuma
+limpeza, basta retornar diretamente.
+
+Escolha nomes de rótulos que indiquem o que o ``goto`` faz ou por que ele
+existe. Um exemplo de nome bom poderia ser ``out_free_buffer:`` se o goto
+liberar ``buffer``. Evite usar nomes do GW-BASIC como ``err1:`` e ``err2:``,
+porque você teria que renumerá-los se adicionasse ou removesse caminhos de
+saída, e isso também torna a correção mais difícil de verificar.
+
+A justificativa para usar gotos é:
+
+- instruções incondicionais são mais fáceis de entender e seguir
+- o aninhamento é reduzido
+- erros por não atualizar pontos de saída individuais ao fazer
+  modificações são evitados
+- economiza o trabalho do compilador de otimizar e remover código redundante ;)
+
+.. code-block:: c
+
+	int fun(int a)
+	{
+		int result = 0;
+		char *buffer;
+
+		buffer = kmalloc(SIZE, GFP_KERNEL);
+		if (!buffer)
+			return -ENOMEM;
+
+		if (condition1) {
+			while (loop1) {
+				...
+			}
+			result = 1;
+			goto out_free_buffer;
+		}
+		...
+	out_free_buffer:
+		kfree(buffer);
+		return result;
+	}
+
+Um tipo comum de bug do qual você deve estar ciente é o ``one err bugs``
+(o "bug de um erro"), que se parece com isto:
+
+.. code-block:: c
+
+	err:
+		kfree(foo->bar);
+		kfree(foo);
+		return ret;
+
+O problema neste código é que, em alguns caminhos de saída, ``foo`` é NULL.
+Normalmente, a correção é dividir em dois rótulos de erro
+``err_free_bar:`` e ``err_free_foo:``:
+
+.. code-block:: c
+
+	err_free_bar:
+		kfree(foo->bar);
+	err_free_foo:
+		kfree(foo);
+		return ret;
+
+Idealmente, você deve simular erros para testar todos os caminhos de saída.
+
+
+8) Comentários
+--------------
+
+Comentários são bons, mas também há o perigo de comentar demais. NUNCA tente
+explicar COMO o seu código funciona em um comentário: é muito melhor escrever
+o código de forma que o **funcionamento** seja óbvio, e é um desperdício de
+tempo explicar código mal escrito.
+
+Em geral, você quer que seus comentários digam O QUE o seu código faz, não
+COMO. Também tente evitar colocar comentários dentro do corpo de uma função:
+se a função for tão complexa que você precisa comentar partes separadas dela,
+provavelmente você deveria voltar ao capítulo 6 por um tempo. Você pode fazer
+pequenos comentários para notar ou avisar sobre algo particularmente esperto
+(ou feio), mas tente evitar excesso. Em vez disso, coloque os comentários no
+início da função, dizendo às pessoas o que ela faz e, possivelmente, POR QUE
+ela faz isso.
+
+Ao comentar as funções da API do kernel, por favor use o formato kernel-doc.
+Veja os arquivos em :ref:`Documentation/doc-guide/ <doc_guide>` e
+``tools/docs/kernel-doc`` para detalhes. Observe que o perigo de comentar em
+excesso se aplica aos comentários kernel-doc da mesma forma. Não adicione
+kernel-doc genérico que apenas repete o que já é óbvio pela assinatura da
+função.
+
+O estilo preferido para comentários longos (em várias linhas) é:
+
+.. code-block:: c
+
+	/*
+	 * Este é o estilo preferido para comentários em várias linhas
+	 * no código-fonte do kernel Linux.
+	 * Por favor, use-o de forma consistente.
+	 *
+	 * Descrição: uma coluna de asteriscos à esquerda,
+	 * com linhas de início e fim quase vazias.
+	 */
+
+Também é importante comentar dados, sejam tipos básicos ou tipos derivados.
+Para isso, use apenas uma declaração de dado por linha (sem vírgulas para
+múltiplas declarações de dados). Isso deixa espaço para um pequeno comentário
+em cada item explicando seu uso.
+
+
+9) Você fez uma bagunça
+-----------------------
+
+Tudo bem, todos fazemos isso. Você provavelmente foi informado por seu
+auxiliar de longa data em Unix que o ``GNU emacs`` formata automaticamente os
+arquivos-fonte em C para você, e você percebeu que ele realmente faz isso, mas as
+configurações padrão que ele usa são menos do que desejáveis (na verdade,
+elas são piores do que digitação aleatória - um número infinito de macacos
+digitando no GNU emacs nunca faria um bom programa).
+
+Então, você pode ou se livrar do GNU emacs, ou mudar para usar valores mais
+sãos. Para fazer isso, você pode colocar o seguinte no seu arquivo .emacs:
+
+.. code-block:: elisp
+
+  (defun c-lineup-arglist-tabs-only (ignored)
+    "Line up argument lists by tabs, not spaces"
+    (let* ((anchor (c-langelem-pos c-syntactic-element))
+          (column (c-langelem-2nd-pos c-syntactic-element))
+          (offset (- (1+ column) anchor))
+          (steps (floor offset c-basic-offset)))
+      (* (max steps 1)
+         c-basic-offset)))
+
+  (dir-locals-set-class-variables
+   'linux-kernel
+   '((c-mode . (
+          (c-basic-offset . 8)
+          (c-label-minimum-indentation . 0)
+          (c-offsets-alist . (
+                  (arglist-close         . c-lineup-arglist-tabs-only)
+                  (arglist-cont-nonempty .
+                      (c-lineup-gcc-asm-reg c-lineup-arglist-tabs-only))
+                  (arglist-intro         . +)
+                  (brace-list-intro      . +)
+                  (c                     . c-lineup-C-comments)
+                  (case-label            . 0)
+                  (comment-intro         . c-lineup-comment)
+                  (cpp-define-intro      . +)
+                  (cpp-macro             . -1000)
+                  (cpp-macro-cont        . +)
+                  (defun-block-intro     . +)
+                  (else-clause           . 0)
+                  (func-decl-cont        . +)
+                  (inclass               . +)
+                  (inher-cont            . c-lineup-multi-inher)
+                  (knr-argdecl-intro     . 0)
+                  (label                 . -1000)
+                  (statement             . 0)
+                  (statement-block-intro . +)
+                  (statement-case-intro  . +)
+                  (statement-cont        . +)
+                  (substatement          . +)
+                  ))
+          (indent-tabs-mode . t)
+          (show-trailing-whitespace . t)
+          ))))
+
+  (dir-locals-set-directory-class
+   (expand-file-name "~/src/linux-trees")
+   'linux-kernel)
+
+Isso fará o emacs funcionar melhor com o estilo de codificação do kernel para
+arquivos C abaixo de ``~/src/linux-trees``.
+
+Mas, mesmo que você falhe em fazer o emacs formatar de maneira sensata, nem
+tudo está perdido: use ``indent``.
+
+Agora, novamente, o GNU indent tem as mesmas configurações sem cérebro do GNU
+emacs, e é por isso que você precisa dar a ele algumas opções de linha de
+comando. No entanto, isso não é tão ruim, porque até os criadores do GNU
+indent reconhecem a autoridade do K&R (as pessoas do GNU não são más, apenas
+estão gravemente equivocadas nessa questão), então você apenas dá ao indent as
+opções ``-kr -i8`` (que significa ``K&R, indentações de 8 caracteres``), ou
+usa ``scripts/Lindent``, que indenta no estilo mais recente.
+
+``indent`` tem muitas opções, e especialmente quando se trata de reformatação
+de comentários, você pode querer dar uma olhada na página de manual. Mas
+lembre-se: ``indent`` não é uma solução para programação ruim.
+
+Observe que você também pode usar a ferramenta ``clang-format`` para ajudá-lo
+com essas regras, para reformatar rapidamente partes do seu código
+automaticamente e revisar arquivos completos para detectar erros de estilo de
+codificação, erros de digitação e possíveis melhorias. Também é útil para
+ordenar ``#includes``, alinhar variáveis/macros, reorganizar texto e outras
+tarefas semelhantes. Consulte o arquivo
+:ref:`Documentation/dev-tools/clang-format.rst <clangformat>`
+para obter mais detalhes.
+
+Algumas configurações básicas do editor, como indentação e finais de linha,
+serão definidas automaticamente se você estiver usando um editor compatível com
+o EditorConfig. Consulte o site oficial do EditorConfig para obter mais
+informação: https://editorconfig.org/
+
+10) Arquivos de configuração Kconfig
+------------------------------------
+
+Para todos os arquivos de configuração Kconfig* em toda a árvore de origem,
+a indentação é um pouco diferente. Linhas sob uma definição ``config`` são
+indentadas com uma tabulação, enquanto o texto de ajuda é indentado com mais
+dois espaços. Exemplo::
+
+  config AUDIT
+	bool "Auditing support"
+	depends on NET
+	help
+	  Enable auditing infrastructure that can be used with another
+	  kernel subsystem, such as SELinux (which requires this for
+	  logging of avc messages output).  Does not do system-call
+	  auditing without CONFIG_AUDITSYSCALL.
+
+Recursos seriamente perigosos (como suporte de gravação para certos sistemas
+de arquivos) devem anunciar isso de forma proeminente na string do prompt::
+
+  config ADFS_FS_RW
+	bool "ADFS write support (DANGEROUS)"
+	depends on ADFS_FS
+	...
+
+Para documentação completa sobre os arquivos de configuração, consulte o
+arquivo Documentation/kbuild/kconfig-language.rst.
+
+
+11) Estruturas de dados
+-----------------------
+
+Estruturas de dados que tenham visibilidade fora do ambiente monothread em que
+são criadas e destruídas devem ter contadores de referência. No kernel, coleta
+de lixo não existe (e fora do kernel, a coleta de lixo é lenta e
+ineficiente), o que significa que você absolutamente **precisa** contar todas
+as referências de uso.
+
+Contagem de referência significa que você pode evitar bloqueios e permite que
+múltiplos usuários tenham acesso à estrutura de dados em paralelo - sem se
+preocupar com a estrutura desaparecendo debaixo deles do nada apenas porque
+eles dormiram ou fizeram outra coisa por um tempo.
+
+Observe que bloqueio **não** substitui contagem de referência. O bloqueio é
+usado para manter estruturas de dados coerentes, enquanto a contagem de
+referência é uma técnica de gerenciamento de memória. Normalmente, ambos são
+necessários, e não devem ser confundidos entre si.
+
+Muitas estruturas de dados podem, de fato, ter dois níveis de contagem de
+referência, quando existem usuários de diferentes ``classes``. A contagem da
+subclasse conta o número de usuários da subclasse e decrementa a contagem
+global apenas uma vez quando a contagem da subclasse chega a zero.
+
+Exemplos desse tipo de ``multi-level-reference-counting`` podem ser encontrados
+em gerenciamento de memória (``struct mm_struct``: mm_users e mm_count) e em
+código de sistema de arquivos (``struct super_block``: s_count e s_active).
+
+Lembre-se: se outra thread puder encontrar sua estrutura de dados, e você não
+tiver contagem de referência nela, quase certamente há um bug.
+
+
+12) Macros, enums e RTL
+-----------------------
+
+Nomes de macros que definem constantes e rótulos em enums são maiúsculos.
+
+.. code-block:: c
+
+	#define CONSTANT 0x12345
+
+Enums são preferidos quando várias constantes relacionadas são definidas.
+
+Nomes de macro em maiúsculas são apreciados, mas macros que se parecem com
+funções podem ser nomeadas em minúsculas.
+
+Em geral, funções inline são preferíveis a macros que se parecem com funções.
+
+Macros com múltiplas instruções devem ser envoltas em um bloco do-while:
+
+.. code-block:: c
+
+	#define macrofun(a, b, c) \
+		do { \
+			if (a == 5) \
+				do_this(b, c); \
+		} while (0)
+
+Macros do tipo função com parâmetros não usados devem ser substituídas por
+funções estáticas inline para evitar o problema de variáveis não usadas:
+
+.. code-block:: c
+
+	static inline void fun(struct foo *foo)
+	{
+	}
+
+Devido a práticas históricas, muitos arquivos ainda empregam a abordagem
+"cast para (void)" para avaliar parâmetros. No entanto, esse método não é
+aconselhável.
+Funções inline resolvem o problema de "expressão com efeitos colaterais
+avaliada mais de uma vez", contornam problemas de variáveis não usadas e, por
+algum motivo, geralmente são mais bem documentadas do que macros.
+
+.. code-block:: c
+
+	/*
+	 * Evite fazer isto sempre que possível e prefira funções estáticas
+	 * inline
+	 */
+	#define macrofun(foo) do { (void) (foo); } while (0)
+
+Coisas a evitar ao usar macros:
+
+1) macros que afetam o fluxo de controle:
+
+.. code-block:: c
+
+	#define FOO(x) \
+		do { \
+			if (blah(x) < 0) \
+				return -EBUGGERED; \
+		} while (0)
+
+é uma ideia **muito** ruim. Ela parece uma chamada de função, mas sai da
+função ``calling``; não quebre os parsers internos de quem lerá o código.
+
+2) macros que dependem de ter uma variável local com um nome mágico:
+
+.. code-block:: c
+
+	#define FOO(val) bar(index, val)
+
+pode parecer uma boa coisa, mas é confuso pra caramba para quem lê o código e
+é propenso a quebrar com mudanças aparentemente inocentes.
+
+3) macros com argumentos usados como l-values: FOO(x) = y; vai te morder se
+alguém, por exemplo, transformar FOO em uma função inline.
+
+4) esquecer da precedência: macros que definem constantes usando expressões
+devem colocar a expressão entre parênteses. Cuidado com problemas semelhantes
+com macros que usam parâmetros.
+
+.. code-block:: c
+
+	#define CONSTANT 0x4000
+	#define CONSTEXP (CONSTANT | 3)
+
+5) colisões de namespace ao definir variáveis locais em macros que se
+parecem com funções:
+
+.. code-block:: c
+
+	#define FOO(x)				\
+	({					\
+		typeof(x) ret;			\
+		ret = calc_ret(x);		\
+		(ret);				\
+	})
+
+``ret`` é um nome comum para uma variável local - ``__foo_ret`` tem menos
+chance de colidir com uma variável existente.
+
+O manual do cpp trata de macros de forma exaustiva. O manual interno do gcc
+também cobre o RTL, que é usado frequentemente com linguagem de montagem no
+kernel.
+
+
+13) Imprimindo mensagens do kernel
+----------------------------------
+
+Desenvolvedores do kernel gostam de ser vistos como letrados. Preste atenção à
+ortografia das mensagens do kernel para causar uma boa impressão. Não use
+contrações incorretas como ``dont``; use ``do not`` ou ``don't`` em vez
+disso. Faça as mensagens concisas, claras e inequívocas.
+
+Mensagens do kernel não precisam terminar com ponto.
+
+Imprimir números entre parênteses (%d) não agrega valor e deve ser evitado.
+
+Há vários macros de diagnóstico do modelo de driver em <linux/dev_printk.h>
+que você deve usar para garantir que as mensagens sejam correspondidas ao
+dispositivo e driver corretos e sejam marcadas com o nível certo: ``dev_err()``,
+``dev_warn()``, ``dev_info()`` e assim por diante. Para mensagens que não
+estão associadas a um device específico, <linux/printk.h> define
+``pr_notice()``, ``pr_info()``, ``pr_warn()``, ``pr_err()`` etc. Quando os
+drivers funcionam corretamente, eles ficam silenciosos, então prefira usar
+``dev_dbg``/``pr_debug`` a menos que algo esteja errado.
+
+Encontrar boas mensagens de depuração pode ser um desafio; e, uma vez que
+você tenha essas mensagens, elas podem ajudar bastante para solução de
+problemas remota. No entanto, a impressão de mensagens de depuração é tratada
+diferentemente da impressão de outras mensagens não de depuração. Enquanto as
+outras funções ``pr_XXX()`` imprimem incondicionalmente, ``pr_debug()`` não;
+ela é compilada fora por padrão, a menos que ``DEBUG`` seja definido ou
+``CONFIG_DYNAMIC_DEBUG`` esteja configurado. Isso também vale para
+``dev_dbg()``, e uma convenção relacionada usa ``VERBOSE_DEBUG`` para adicionar
+mensagens ``dev_vdbg()`` às já habilitadas por ``DEBUG``.
+
+Muitos subsistemas têm opções de depuração do Kconfig para ativar ``-DDEBUG``
+no Makefile correspondente; em outros casos, arquivos específicos fazem
+``#define DEBUG``. E quando uma mensagem de depuração deve ser impressa
+incondicionalmente, por exemplo, se ela já estiver dentro de uma seção
+``#ifdef`` relacionada à depuração, pode-se usar ``printk(KERN_DEBUG ...)``.
+
+
+14) Alocando memória
+--------------------
+
+O kernel fornece os seguintes alocadores de memória de uso geral:
+``kmalloc()``, ``kzalloc()``, ``kmalloc_objs()``, ``kzalloc_objs()``,
+``vmalloc()`` e ``vzalloc()``. Consulte a documentação da API para obter mais
+informações sobre eles. :ref:`Documentation/core-api/memory-allocation.rst
+<memory_allocation>`
+
+A forma preferida de passar o tamanho de uma estrutura é a seguinte:
+
+.. code-block:: c
+
+	p = kmalloc_obj(*p, ...);
+
+A forma alternativa em que o nome da estrutura é escrito explicitamente piora a
+legibilidade e cria oportunidade para um bug quando o tipo da variável ponteiro
+é alterado, mas o ``sizeof`` correspondente passado para um alocador de memória
+não é.
+
+Casting do valor de retorno, que é um ponteiro ``void``, é redundante. A
+conversão de ponteiro ``void`` para qualquer outro tipo de ponteiro é garantida
+pela linguagem de programação C.
+
+A forma preferida para alocar um array é a seguinte:
+
+.. code-block:: c
+
+	p = kmalloc_objs(*p, n, ...);
+
+A forma preferida para alocar um array zerado é a seguinte:
+
+.. code-block:: c
+
+	p = kzalloc_objs(*p, n, ...);
+
+As duas formas verificam estouro no tamanho de alocação ``n * sizeof(...)`` e
+retornam ``NULL`` se isso ocorrer.
+
+Essas funções genéricas de alocação emitem um dump de pilha em caso de falha
+quando usadas sem ``__GFP_NOWARN``, então não há utilidade em emitir uma
+mensagem de falha adicional quando ``NULL`` é retornado.
+
+15) A doença do inline
+----------------------
+
+Parece haver uma percepção errônea comum de que o gcc tem uma opção mágica de
+aceleração chamada ``inline``. Embora o uso de ``inline`` possa ser apropriado
+(por exemplo, como uma forma de substituir macros; veja o Capítulo 12), muitas
+vezes não é. O uso abundante da palavra-chave ``inline`` leva a um kernel
+muito maior, o que, por sua vez, torna o sistema mais lento como um todo, por
+causa de uma maior ocupação de i-cache para a CPU e simplesmente porque há menos
+memória disponível para o ``pagecache``. Pense nisso: uma falha no pagecache
+causa um seek no disco, que facilmente leva 5 milissegundos. Há MUITOS ciclos
+de CPU que podem entrar nesses 5 milissegundos.
+
+Uma regra prática razoável é não colocar ``inline`` em funções com mais de 3
+linhas de código. Uma exceção a essa regra são os casos em que um parâmetro é
+conhecido como uma constante em tempo de compilação, e como resultado dessa
+constância você *sabe* que o compilador será capaz de otimizar grande parte da
+sua função em tempo de compilação. Para um bom exemplo desse caso posterior,
+veja a função inline ``kmalloc()``.
+
+Muitas pessoas argumentam que adicionar ``inline`` a funções ``static`` usadas
+apenas uma vez é sempre uma vantagem, porque não há custo de espaço. Embora
+isso seja tecnicamente correto, o gcc é capaz de fazer esse inline
+automaticamente sem ajuda, e a questão de manutenção de remover o ``inline``
+quando um segundo usuário aparece supera o valor potencial da dica que diz ao
+gcc para fazer algo que ele faria de qualquer forma.
+
+
+16) Valores e nomes de retorno de função
+----------------------------------------
+
+Funções podem retornar valores de vários tipos, e um dos mais comuns é um valor
+que indica se a função teve sucesso ou falhou. Esse valor pode ser representado
+como um inteiro de código de erro (-Exxx = falha, 0 = sucesso) ou como um
+booleano ``succeeded`` (0 = falha, diferente de zero = sucesso).
+
+Misturar esses dois tipos de representação é uma fonte fértil de bugs difíceis
+de encontrar. Se a linguagem C incluísse uma distinção forte entre inteiros e
+booleanos, o compilador encontraria esses erros para nós... mas não inclui. Para
+ajudar a evitar esses bugs, siga sempre esta convenção::
+
+	Se o nome de uma função for uma ação ou um comando imperativo,
+	a função deve retornar um inteiro de código de erro. Se o nome
+	for um predicado, a função deve retornar um booleano de "sucesso".
+
+Por exemplo, ``add work`` é um comando, e a função ``add_work()`` retorna 0
+para sucesso ou -EBUSY para falha. Da mesma forma, ``PCI device present`` é um
+predicado, e a função ``pci_dev_present()`` retorna 1 se encontrar um device
+correspondente ou 0 se não encontrar.
+
+Todas as funções ``EXPORT`` devem respeitar esta convenção, e assim também
+devem todas as funções públicas. Funções privadas (``static``) não precisam,
+mas é recomendável que o façam.
+
+Funções cujo valor de retorno é o resultado real de um cálculo, em vez de uma
+indicação de se o cálculo teve sucesso, não estão sujeitas a essa regra.
+Normalmente, elas indicam falha retornando algum resultado fora do intervalo.
+Exemplos típicos seriam funções que retornam ponteiros; elas usam ``NULL`` ou o
+mecanismo ``ERR_PTR`` para informar falha.
+
+
+17) Usando bool
+---------------
+
+O tipo ``bool`` do kernel Linux é um alias do tipo C99 ``_Bool``. Valores
+``bool`` só podem avaliar para 0 ou 1, e conversão implícita ou explícita para
+``bool`` converte automaticamente o valor para verdadeiro ou falso. Ao usar
+tipos ``bool``, a construção ``!!`` não é necessária, o que elimina uma classe
+de bugs.
+
+Ao trabalhar com valores ``bool``, as definições ``true`` e ``false`` devem ser
+usadas em vez de 1 e 0.
+
+Tipos de retorno de função ``bool`` e variáveis locais na pilha são sempre
+válidos quando apropriados. O uso de ``bool`` é encorajado para melhorar a
+legibilidade e muitas vezes é uma opção melhor do que ``int`` para armazenar
+valores booleanos.
+
+Não use ``bool`` se o layout da linha de cache ou o tamanho do valor importar,
+porque seu tamanho e alinhamento variam conforme a arquitetura compilada.
+Estruturas otimizadas para alinhamento e tamanho não devem usar ``bool``.
+
+Se uma estrutura tiver muitos valores verdadeiro/falso, considere consolidá-los
+em um ``bitfield`` com membros de 1 bit, ou usar um tipo de largura fixa
+apropriado, como ``u8``.
+
+Da mesma forma, para argumentos de função, muitos valores verdadeiro/falso podem
+ser consolidados em um único argumento de sinalizadores bit a bit, e
+``flags`` muitas vezes pode ser uma alternativa mais legível se os pontos de
+chamada tiverem constantes verdadeiras/falsas "nuas".
+
+Caso contrário, o uso limitado de ``bool`` em estruturas e argumentos pode
+melhorar a legibilidade.
+
+18) Não reinventando as macros do kernel
+----------------------------------------
+
+Existem muitos arquivos de cabeçalho em ``include/linux/`` que contêm várias
+macros que você deve usar em vez de escrever explicitamente alguma variante
+delas. Por exemplo, se você precisa calcular o comprimento de um array, aproveite
+a macro
+
+.. code-block:: c
+
+	#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+
+que é definida em ``array_size.h``.
+
+Da mesma forma, se você precisar calcular o tamanho de um membro de alguma
+estrutura, use
+
+.. code-block:: c
+
+	#define sizeof_field(t, f) (sizeof(((t*)0)->f))
+
+que é definida em ``stddef.h``.
+
+Também existem macros ``min()`` e ``max()`` definidas em ``minmax.h`` que fazem
+verificação estrita de tipos se você precisar delas. Sinta-se à vontade para
+explorar os arquivos de cabeçalho para ver o que já está definido e não deve
+ser reproduzido no seu código.
+
+
+19) Modelines do editor e outros restos
+---------------------------------------
+
+Alguns editores podem interpretar informações de configuração embutidas em
+arquivos de origem, indicadas por marcadores especiais. Por exemplo, o emacs
+interpreta linhas marcadas assim:
+
+.. code-block:: c
+
+	-*- mode: c -*-
+
+Ou assim:
+
+.. code-block:: c
+
+	/*
+	Local Variables:
+	compile-command: "gcc -DMAGIC_DEBUG_FLAG foo.c"
+	End:
+	*/
+
+O Vim interpreta marcadores que parecem com isto:
+
+.. code-block:: c
+
+	/* vim:set sw=8 noet */
+
+Não inclua nenhum desses em arquivos de origem. As pessoas têm suas próprias
+configurações pessoais de editor, e seus arquivos de origem não devem
+substituí-las. Isso inclui marcadores para indentação e configuração de modo.
+As pessoas podem usar seu próprio modo personalizado, ou podem ter algum outro
+método mágico para fazer a indentação funcionar corretamente.
+
+
+20) Montagem inline
+-------------------
+
+Em código específico de arquitetura, pode ser necessário usar montagem inline
+para interagir com a funcionalidade da CPU ou da plataforma. Não hesite em
+fazê-lo quando necessário. No entanto, não use montagem inline de forma
+gratuita quando o C puder fazer o trabalho. Você pode e deve mexer em hardware
+em C quando possível.
+
+Considere escrever funções auxiliares simples que encapsulem partes comuns de
+montagem inline, em vez de escrevê-las repetidamente com pequenas variações.
+Lembre-se de que a montagem inline pode usar parâmetros C.
+
+Funções grandes e não triviais de montagem devem ir para arquivos ``.S``, com
+protótipos C correspondentes definidos em arquivos de cabeçalho C. Os
+protótipos C para funções de montagem devem usar ``asmlinkage``.
+
+Você pode precisar marcar sua instrução ``asm`` como ``volatile`` para impedir
+que o GCC a remova se o GCC não perceber efeitos colaterais. No entanto, você
+nem sempre precisa fazer isso, e fazê-lo desnecessariamente pode limitar a
+otimização.
+
+Ao escrever uma única instrução de montagem inline contendo várias instruções,
+coloque cada instrução em uma linha separada em uma string separada e termine
+cada string, exceto a última, com ``\n\t`` para indentar corretamente a
+próxima instrução na saída de montagem:
+
+.. code-block:: c
+
+	asm ("magic %reg1, #42\n\t"
+	     "more_magic %reg2, %reg3"
+	     : /* outputs */ : /* inputs */ : /* clobbers */);
+
+
+21) Compilação condicional
+--------------------------
+
+Sempre que possível, não use condicionais do pré-processador (#if, #ifdef) em
+arquivos ``.c``; isso torna o código mais difícil de ler e a lógica mais difícil
+de seguir. Em vez disso, use esses condicionais em um arquivo de cabeçalho que
+defina funções para uso nesses arquivos ``.c``, fornecendo versões de stub sem
+efeito no caso ``#else``, e então chame essas funções incondicionalmente nos
+arquivos ``.c``. O compilador evitará gerar qualquer código para as chamadas de
+stub, produzindo resultados idênticos, mas a lógica permanecerá fácil de
+seguir.
+
+Prefira compilar funções inteiras fora do código, em vez de partes de funções
+ou partes de expressões. Em vez de colocar um ``ifdef`` em uma expressão,
+extraia parte ou toda a expressão para uma função auxiliar separada e aplique a
+condicional a essa função.
+
+Se você tiver uma função ou variável que pode potencialmente ficar sem uso em
+uma configuração específica, e o compilador avisaria sobre a definição ficar sem
+uso, marque a definição como ``__maybe_unused`` em vez de envolvê-la em uma
+condicional do pré-processador. (No entanto, se uma função ou variável
+*sempre* ficar sem uso, elimine-a.)
+
+Dentro do código, quando possível, use a macro ``IS_ENABLED`` para converter um
+símbolo Kconfig em uma expressão booleana C e usá-la em uma condicional C
+normal:
+
+.. code-block:: c
+
+	if (IS_ENABLED(CONFIG_SOMETHING)) {
+		...
+	}
+
+O compilador reduzirá a condicional a um valor constante e incluirá ou
+excluirá o bloco de código assim como com um ``#ifdef``, então isso não
+adicionará nenhum custo de runtime. No entanto, essa abordagem ainda permite
+que o compilador C veja o código dentro do bloco e verifique sua correção
+(sintaxe, tipos, referências de símbolo etc.). Portanto, você ainda precisa usar
+um ``#ifdef`` se o código dentro do bloco referenciar símbolos que não existirão
+se a condição não for atendida.
+
+No final de qualquer bloco ``#if`` ou ``#ifdef`` não trivial (mais de algumas
+linhas), coloque um comentário após o ``#endif`` na mesma linha, indicando a
+expressão condicional usada. Por exemplo:
+
+.. code-block:: c
+
+	#ifdef CONFIG_SOMETHING
+	...
+	#endif /* CONFIG_SOMETHING */
+
+
+22) Não derrube o kernel
+------------------------
+
+Em geral, a decisão de derrubar o kernel pertence ao usuário, e não ao
+desenvolvedor do kernel.
+
+Evite ``panic()``
+*****************
+
+``panic()`` deve ser usado com cuidado e principalmente apenas durante a inicialização
+do sistema. ``panic()`` é, por exemplo, aceitável ao ficar sem memória durante
+a inicialização e não ser possível continuar.
+
+Use ``WARN()`` em vez de ``BUG()``
+**********************************
+
+Não adicione novo código que use nenhuma das variantes de ``BUG()``, como
+``BUG()``, ``BUG_ON()`` ou ``VM_BUG_ON()``. Em vez disso, use uma variante de
+``WARN*()``, preferencialmente ``WARN_ON_ONCE()``, e possivelmente com código de
+recuperação. O código de recuperação não é obrigatório se não houver uma
+maneira razoável de pelo menos recuperar parcialmente.
+
+"Sou preguiçoso para tratar erros" não é uma desculpa para usar ``BUG()``.
+Corrupções internas graves sem como continuar ainda podem usar ``BUG()``, mas
+precisam de uma boa justificativa.
+
+Use ``WARN_ON_ONCE()`` em vez de ``WARN()`` ou ``WARN_ON()``
+************************************************************
+
+``WARN_ON_ONCE()`` geralmente é preferido em relação a ``WARN()`` ou
+``WARN_ON()``, porque é comum que uma dada condição de aviso, se ocorrer,
+ocorra várias vezes. Isso pode encher e sobrescrever o log do kernel e até
+diminuir o sistema o suficiente para que o registro excessivo vire um problema
+adicional.
+
+Não emita ``WARN`` levianamente
+*******************************
+
+``WARN*()`` foi concebido para situações inesperadas, em que "isso nunca devia
+acontecer". Macros ``WARN*()`` não devem ser usadas para nada que se espere que
+aconteça durante a operação normal. Esses não são asserts de pré- ou pós-
+condição, por exemplo. Novamente: ``WARN*()`` não deve ser usado para uma
+condição que se espera que seja acionada facilmente, por exemplo, por ações do
+espaço do usuário. ``pr_warn_once()`` é uma alternativa possível, se você
+precisar notificar o usuário sobre um problema.
+
+Não se preocupe com usuários de ``panic_on_warn``
+*************************************************
+
+Mais algumas palavras sobre ``panic_on_warn``: lembre-se de que
+``panic_on_warn`` é uma opção disponível do kernel, e muitos usuários a
+habilitam. É por isso que existe um texto "Não emita WARN levianamente" acima.
+Entretanto, a existência de usuários de ``panic_on_warn`` não é uma razão válida
+para evitar o uso judicioso de ``WARN*()``. Isso ocorre porque quem habilita
+``panic_on_warn`` explicitamente pediu ao kernel para travar se um ``WARN*()``
+for disparado, e esses usuários devem estar preparados para lidar com as
+consequências de um sistema que tem uma chance um pouco maior de travar.
+
+Use ``BUILD_BUG_ON()`` para assertivas em tempo de compilação
+*************************************************************
+
+O uso de ``BUILD_BUG_ON()`` é aceitável e encorajado, porque é uma assertiva
+em tempo de compilação que não tem efeito em tempo de execução.
+
+Apêndice I) Referências
+-----------------------
+
+The C Programming Language, Second Edition
+by Brian W. Kernighan and Dennis M. Ritchie.
+Prentice Hall, Inc., 1988.
+ISBN 0-13-110362-8 (paperback), 0-13-110370-9 (hardback).
+
+The Practice of Programming
+by Brian W. Kernighan and Rob Pike.
+Addison-Wesley, Inc., 1999.
+ISBN 0-201-61586-X.
+
+Manuais GNU - onde em conformidade com K&R e este texto - para cpp, gcc,
+gcc internals e indent, todos disponíveis em https://www.gnu.org/manual/
+
+WG14 é o grupo de trabalho de padronização internacional para a linguagem de
+programação C, URL: http://www.open-std.org/JTC1/SC22/WG14/
+
+Kernel CodingStyle, by greg@kroah.com at OLS 2002:
+http://www.kroah.com/linux/talks/ols_2002_kernel_codingstyle_talk/html/
diff --git a/Documentation/translations/pt_BR/process/cve.rst b/Documentation/translations/pt_BR/process/cve.rst
index 18f4b11be3690b..83977eda34c47a 100644
--- a/Documentation/translations/pt_BR/process/cve.rst
+++ b/Documentation/translations/pt_BR/process/cve.rst
@@ -18,7 +18,7 @@ essas atribuições.
 A equipe de desenvolvedores do kernel Linux tem a capacidade de atribuir CVEs
 para possíveis problemas de segurança do kernel Linux. Essa atribuição é
 independente do processo normal de relato de bugs de segurança do kernel
-Linux, descrito em :ref:`securitybugs`.
+Linux, descrito em :ref:`pt_BR_securitybugs`.
 
 Uma lista de todos os CVEs atribuídos ao kernel Linux pode ser encontrada nos
 arquivos da lista de discussão linux-cve, como visto em
@@ -51,7 +51,7 @@ ele é SOMENTE para atribuição de CVEs a correções que já estejam em árvor
 kernel lançadas. Se você acredita ter encontrado um problema de segurança
 ainda
 não corrigido, por favor siga o processo normal de relato de bugs de segurança do kernel
-Linux, descrito em :ref:`securitybugs`.
+Linux, descrito em :ref:`pt_BR_securitybugs`.
 
 Nenhum CVE será atribuído automaticamente para problemas de segurança ainda
 não corrigidos no kernel Linux; a atribuição só acontecerá automaticamente
diff --git a/Documentation/translations/pt_BR/process/debugging/index.rst b/Documentation/translations/pt_BR/process/debugging/index.rst
new file mode 100644
index 00000000000000..b223fae34622f6
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/debugging/index.rst
@@ -0,0 +1,73 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=======================================================
+Dicas de depuração para desenvolvedores do Kernel Linux
+=======================================================
+
+Guias gerais
+------------
+
+Todolist:
+
+*   driver_development_debugging_guide
+*   gdb-kernel-debugging
+*   kgdb
+*   userspace_debugging_guide
+
+Guias específicos de subsistemas
+--------------------------------
+
+Todolist:
+
+*   media_specific_debugging_guide
+
+Dicas gerais de depuração
+-------------------------
+
+Dependendo do problema, um conjunto diferente de ferramentas está disponível
+para rastrear o problema ou até mesmo para perceber se há algum problema em
+primeiro lugar.
+
+Como primeiro passo, você precisa descobrir que tipo de problema você deseja
+depurar. Dependendo da resposta, sua metodologia e escolha de ferramentas podem
+variar.
+
+Preciso depurar com acesso limitado?
+------------------------------------
+
+Você possui acesso limitado à máquina ou não consegue parar a execução em
+andamento?
+
+Nesse caso, sua capacidade de depuração depende do suporte de depuração
+embutido no kernel fornecido pela distribuição.
+O :doc:`/process/debugging/userspace_debugging_guide` fornece uma breve visão
+geral sobre uma variedade de ferramentas de depuração possíveis nessa situação.
+Você pode verificar a capacidade do seu kernel, na maioria dos casos, olhando o
+arquivo de configuração dentro do diretório /boot.
+
+Eu tenho acesso root ao sistema?
+--------------------------------
+
+Você consegue facilmente substituir o módulo em questão ou instalar um novo
+kernel?
+
+Nesse caso, sua gama de ferramentas disponíveis é muito maior. Você
+pode encontrar as ferramentas
+no :doc:`/process/debugging/driver_development_debugging_guide`.
+
+A temporização é um fator?
+--------------------------
+
+É importante entender se o problema que você deseja depurar se manifesta
+de forma consistente (ou seja, para um determinado conjunto de entradas, você
+sempre obtém a mesma saída incorreta) ou de forma inconsistente. Se ele se
+manifestar de forma inconsistente, algum fator de temporização pode estar em
+jogo. Se a inserção de atrasos no código alterar o comportamento, é bastante
+provável que a temporização seja um fator determinante.
+
+Quando a temporização altera o resultado da execução do código, o uso de um
+simples printk() para fins de depuração pode não funcionar; uma alternativa
+semelhante é usar trace_printk(), que registra as mensagens de depuração no
+arquivo de rastreamento, em vez de no log do kernel.
+
+**Copyright** ©2024 : Collabora
diff --git a/Documentation/translations/pt_BR/process/development-process.rst b/Documentation/translations/pt_BR/process/development-process.rst
index d303ab92b2d35a..e1441cec8b1acc 100644
--- a/Documentation/translations/pt_BR/process/development-process.rst
+++ b/Documentation/translations/pt_BR/process/development-process.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _pt_BR_development_process_main:
+
 Guia para o Processo de Desenvolvimento do Kernel
 =================================================
 
diff --git a/Documentation/translations/pt_BR/process/embargoed-hardware-issues.rst b/Documentation/translations/pt_BR/process/embargoed-hardware-issues.rst
new file mode 100644
index 00000000000000..ae1fda0880cc1e
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/embargoed-hardware-issues.rst
@@ -0,0 +1,362 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Problemas de hardware sob embargo
+=================================
+
+Escopo
+------
+
+Problemas de hardware que resultam em problemas de segurança formam uma categoria
+de bugs de segurança diferente dos bugs de software puros que afetam apenas o
+kernel do Linux.
+
+Problemas de hardware como Meltdown, Spectre, L1TF, etc., devem ser tratados
+de maneira diferente porque geralmente afetam todos os Sistemas Operacionais ("OS")
+e, portanto, exigem coordenação entre diferentes fornecedores de SO, distribuições,
+fabricantes de silício, integradores de hardware e outras partes. Para alguns
+dos problemas, as mitigações de software podem depender de atualizações de
+microcódigo ou firmware, o que requer ainda mais coordenação.
+
+.. _pt_BR_Contact:
+
+Contato
+-------
+
+A equipe de segurança de hardware do kernel Linux é separada da equipe regular
+de segurança do kernel Linux.
+
+A equipe lida apenas com o desenvolvimento de correções para problemas de
+segurança de hardware sob embargo. Relatos de bugs de segurança de software puro
+no kernel Linux não são tratados por esta equipe, e o autor do relato será
+orientado a contatar a equipe regular de segurança do kernel Linux
+(:ref:`Documentation/admin-guide/ <securitybugs>`) em vez disso.
+
+A equipe pode ser contatada por e-mail em <hardware-security@kernel.org>. Esta
+é uma lista privada de oficiais de segurança que ajudarão você a coordenar uma
+correção de acordo com o nosso processo documentado.
+
+A lista é criptografada e o e-mail para a lista pode ser enviado criptografado
+por PGP ou S/MIME, e deve ser assinado com a chave PGP ou certificado S/MIME do
+autor do relato. A chave PGP e o certificado S/MIME da equipe estão disponíveis
+nas seguintes URLs:
+
+  - PGP: https://www.kernel.org/static/files/hardware-security.asc
+  - S/MIME: https://www.kernel.org/static/files/hardware-security.crt
+
+Embora os problemas de segurança de hardware sejam frequentemente tratados pelo
+fabricante de silício afetado, nós acolhemos o contato de pesquisadores ou
+indivíduos que tenham identificado uma falha potencial de hardware.
+
+Oficiais de segurança de hardware
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A equipe atual de oficiais de segurança de hardware:
+
+  - Linus Torvalds (Fellow da Linux Foundation)
+  - Greg Kroah-Hartman (Fellow da Linux Foundation)
+  - Thomas Gleixner (Fellow da Linux Foundation)
+
+Operação das listas de e-mail
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As listas de e-mail criptografadas que são usadas em nosso processo são
+hospedadas na infraestrutura de TI da Linux Foundation. Ao fornecer este
+serviço, os membros da equipe de operações de TI da Linux Foundation têm,
+tecnicamente, a capacidade de acessar as informações sob embargo, mas são
+obrigados à confidencialidade por seu contrato de trabalho. O pessoal de TI
+da Linux Foundation também é responsável por operar e gerenciar o restante da
+infraestrutura do kernel.org.
+
+O atual diretor de infraestrutura de projetos de TI da Linux Foundation é
+Konstantin Ryabitsev.
+
+
+Acordos de não divulgação
+-------------------------
+
+A equipe de segurança de hardware do kernel Linux não é um órgão formal e,
+portanto, é incapaz de celebrar quaisquer acordos de não divulgação. A
+comunidade do kernel está ciente da natureza sensível de tais problemas e
+oferece um Memorando de Entendimento em vez disso.
+
+
+Memorando de Entendimento
+-------------------------
+
+A comunidade do kernel Linux compreende profundamente a necessidade de manter
+os problemas de segurança de hardware sob embargo para a coordenação entre
+diferentes fornecedores de SO, distribuidores, fabricantes de silício e outras
+partes.
+
+A comunidade do kernel Linux lidou com sucesso com problemas de segurança de
+hardware no passado e possui os mecanismos necessários para permitir o
+desenvolvimento compatível com a comunidade sob restrições de embargo.
+
+A comunidade do kernel Linux possui uma equipe dedicada de segurança de hardware
+para o contato inicial, que supervisiona o processo de tratamento de tais
+problemas sob as regras de embargo.
+
+A equipe de segurança de hardware identifica os desenvolvedores (especialistas no
+domínio) que formarão a equipe de resposta inicial para um problema específico.
+A equipe de resposta inicial pode trazer outros desenvolvedores (especialistas no
+domínio) para resolver o problema da melhor maneira técnica.
+
+Todos os desenvolvedores envolvidos comprometem-se a aderir às regras de embargo
+e a manter as informações recebidas em sigilo. A violação do compromisso levará à
+exclusão imediata do problema atual e à remoção de todas as listas de e-mail
+relacionadas. Além disso, a equipe de segurança de hardware também excluirá o
+infrator de futuros problemas. O impacto dessa consequência é um impedimento
+altamente eficaz em nossa comunidade. Caso ocorra uma violação, a equipe de
+segurança de hardware informará as partes envolvidas imediatamente. Se você ou
+qualquer outra pessoa tomar conhecimento de uma potencial violação, por favor,
+relate-a imediatamente aos oficiais de segurança de hardware.
+
+
+Processo
+^^^^^^^^
+
+Devido à natureza globalmente distribuída do desenvolvimento do kernel Linux,
+reuniões presenciais são quase impossíveis para lidar com problemas de
+segurança de hardware. Conferências telefônicas são difíceis de coordenar devido
+a fusos horários e outros fatores, devendo ser usadas apenas quando estritamente
+necessário. O e-mail criptografado tem se mostrado o método de comunicação mais
+eficiente e seguro para esses tipos de problema.
+
+Início da divulgação
+"""""""""""""""""""""
+
+A divulgação começa enviando um e-mail para a equipe de segurança de hardware
+do kernel Linux, conforme a seção Contato acima. Este contato inicial deve
+conter uma descrição do problema e uma lista de qualquer silício afetado
+conhecido. Se a sua organização constrói ou distribui o hardware afetado,
+incentivamos você a considerar também quais outros hardwares podem ser
+afetados. A parte que faz a divulgação é responsável por contatar os
+fabricantes de silício afetados em tempo hábil.
+
+A equipe de segurança de hardware fornecerá uma lista de e-mail criptografada
+específica para o incidente, que será usada para a discussão inicial com o
+relator, divulgação posterior e coordenação de correções.
+
+A equipe de segurança de hardware fornecerá à parte divulgadora uma lista de
+desenvolvedores (especialistas no domínio) que devem ser informados inicialmente
+sobre o problema após confirmar com os desenvolvedores que eles aderirão a
+este Memorando de Entendimento e ao processo documentado. Esses desenvolvedores
+formam a equipe de resposta inicial e serão responsáveis por lidar com o
+problema após o contato inicial. A equipe de segurança de hardware apoia a
+equipe de resposta, mas não está necessariamente envolvida no processo de
+desenvolvimento de mitigações.
+
+Embora desenvolvedores individuais possam estar cobertos por um acordo de não
+divulgação por meio de seu empregador, eles não podem celebrar acordos
+individuais de não divulgação em seu papel como desenvolvedores do kernel
+Linux. No entanto, eles concordarão em aderir a este processo documentado e ao
+Memorando de Entendimento.
+
+A parte divulgadora deve fornecer uma lista de contatos para todas as outras
+entidades que já foram, ou devem ser, informadas sobre o problema. Isso serve
+a vários propósitos:
+
+ - A lista de entidades informadas permite a comunicação em toda a
+   indústria, por exemplo, outros fornecedores de SO, fornecedores de HW, etc.
+
+ - As entidades informadas podem ser contatadas para indicar especialistas
+   que devem participar do desenvolvimento da mitigação.
+
+ - Se um especialista necessário para lidar com um problema for funcionário
+   de uma entidade listada ou membro de uma entidade listada, as equipes de
+   resposta podem solicitar a inclusão desse especialista por parte daquela
+   entidade. Isso garante que o especialista também faça parte da equipe de
+   resposta da entidade.
+
+Divulgação
+""""""""""
+
+A parte divulgadora fornece informações detalhadas à equipe de resposta inicial
+por meio da lista de e-mail criptografada específica.
+
+A partir de nossa experiência, a documentação técnica desses problemas costuma
+ser um ponto de partida suficiente, e esclarecimentos técnicos adicionais são
+melhor feitos por e-mail.
+
+Desenvolvimento de mitigações
+""""""""""""""""""""""""""""""
+
+A equipe de resposta inicial configura uma lista de e-mail criptografada ou
+reaproveita uma já existente, se apropriado.
+
+O uso de uma lista de e-mail é próximo ao processo normal de desenvolvimento
+do Linux e tem sido usado com sucesso para desenvolver mitigações para vários
+problemas de segurança de hardware no passado.
+
+A lista de e-mail opera da mesma forma que o desenvolvimento normal do Linux.
+Os patches são publicados, discutidos, revisados e, se aprovados, aplicados a
+um repositório git não público que é acessível apenas aos desenvolvedores
+participantes por meio de uma conexão segura. O repositório contém o ramo
+(branch) principal de desenvolvimento contra o kernel mainline e ramos de
+retroporte (backport) para versões estáveis do kernel conforme necessário.
+
+A equipe de resposta inicial identificará outros especialistas da comunidade
+de desenvolvedores do kernel Linux conforme necessário. Qualquer parte
+envolvida pode sugerir a inclusão de outros especialistas, cada um dos quais
+estará sujeito aos mesmos requisitos descritos acima.
+
+A inclusão de especialistas pode ocorrer a qualquer momento no processo de
+desenvolvimento e precisa ser tratada em tempo hábil.
+
+Se um especialista for funcionário ou membro de uma entidade na lista de
+divulgação fornecida pela parte divulgadora, a participação será solicitada
+à entidade relevante.
+
+Caso contrário, a parte divulgadora será informada sobre a participação
+dos especialistas. Os especialistas são cobertos pelo Memorando de Entendimento
+e a parte divulgadora é solicitada a reconhecer a participação deles. No caso
+de a parte divulgadora ter um motivo convincente para se opor, qualquer
+objeção deve ser levantada no prazo de cinco dias úteis e resolvida com a
+equipe do incidente imediatamente. Se a parte divulgadora não reagir dentro
+de cinco dias úteis, isso é considerado como reconhecimento tácito.
+
+Após a equipe do incidente reconhecer ou resolver uma objeção, o especialista
+é informado e integrado ao processo de desenvolvimento.
+
+Os participantes da lista não podem se comunicar sobre o problema fora da
+lista de e-mail privada. Os participantes da lista não podem usar nenhum
+recurso compartilhado (por exemplo, fazendas de compilação do empregador,
+sistemas de IC, etc.) ao trabalhar em patches.
+
+Acesso antecipado
+"""""""""""""""""
+
+Os patches discutidos e desenvolvidos na lista não podem ser distribuídos a
+nenhum indivíduo que não seja membro da equipe de resposta, nem a nenhuma outra
+organização.
+
+Para permitir que os fornecedores de silício afetados trabalhem com suas equipes
+internas e parceiros da indústria em testes, validação e logística, a seguinte
+exceção é fornecida:
+
+    Representantes designados dos fornecedores de silício afetados têm permissão
+    para repassar os patches a qualquer momento para a equipe de resposta do
+    fornecedor de silício. O representante deve notificar a equipe de resposta
+    do kernel sobre o repasse. O fornecedor de silício afetado deve possuir e
+    manter seu próprio processo de segurança documentado para quaisquer patches
+    compartilhados com sua equipe de resposta que seja consistente com esta
+    política.
+
+    A equipe de resposta do fornecedor de silício pode distribuir esses patches
+    aos seus parceiros da indústria e às suas equipes internas sob o processo
+    de segurança documentado do fornecedor de silício. O feedback dos parceiros
+    da indústria retorna ao fornecedor de silício e é comunicado por ele à
+    equipe de resposta do kernel.
+
+    O repasse para a equipe de resposta do fornecedor de silício remove
+    qualquer responsabilidade civil ou legal da equipe de resposta do kernel
+    em relação à divulgação prematura que ocorra devido ao envolvimento das
+    equipes internas ou parceiros da indústria do fornecedor de silício. O
+    fornecedor de silício garante esta liberação de responsabilidade ao
+    concordar com este processo.
+
+Lançamento coordenado
+"""""""""""""""""""""
+
+As partes envolvidas negociarão a data e a hora em que o embargo termina. Nesse
+ponto, as mitigações preparadas são publicadas nas árvores de kernel relevantes.
+Não há processo de pré-notificação: as mitigações são publicadas publicamente e
+disponibilizadas para todos ao mesmo tempo.
+
+Embora entendamos que problemas de segurança de hardware exijam tempo de embargo
+coordenado, o tempo de embargo deve ser restrito ao mínimo necessário para que
+todas as partes envolvidas desenvolvam, testem e preparem suas mitigações.
+Estender o tempo de embargo artificialmente para cumprir datas de palestras em
+conferências ou outros motivos não técnicos cria mais trabalho e ônus para os
+desenvolvedores e equipes de resposta envolvidos, pois os patches precisam ser
+mantidos atualizados para acompanhar o desenvolvimento contínuo do kernel
+upstream, o que pode criar alterações conflitantes.
+
+Atribuição de CVE
+""""""""""""""""""
+
+Nem a equipe de segurança de hardware nem a equipe de resposta inicial atribuem
+CVEs, nem os CVEs são necessários para o processo de desenvolvimento. Se os CVEs
+forem fornecidos pela parte divulgadora, eles poderão ser usados para fins de
+documentação.
+
+Embaixadores do processo
+------------------------
+
+Para obter assistência com este processo, estabelecemos embaixadores em várias
+organizações, que podem responder a perguntas sobre ou fornecer orientações
+acerca do processo de relatórios e tratamento posterior. Os embaixadores não
+estão envolvidos na divulgação de um problema específico, a menos que seja
+solicitado por uma equipe de resposta ou por uma parte divulgada envolvida.
+A lista atual de embaixadores:
+
+  ============= ========================================================
+  AMD           Tom Lendacky <thomas.lendacky@amd.com>
+  Ampere        Darren Hart <darren@os.amperecomputing.com>
+  ARM           Catalin Marinas <catalin.marinas@arm.com>
+  IBM Power     Madhavan Srinivasan <maddy@linux.ibm.com>
+  IBM Z         Christian Borntraeger <borntraeger@de.ibm.com>
+  Intel         Tony Luck <tony.luck@intel.com>
+  Qualcomm      Trilok Soni <quic_tsoni@quicinc.com>
+  RISC-V        Palmer Dabbelt <palmer@dabbelt.com>
+  Samsung       Javier González <javier.gonz@samsung.com>
+
+  Microsoft     James Morris <jamorris@linux.microsoft.com>
+  Xen           Andrew Cooper <andrew.cooper3@citrix.com>
+
+  Canonical     John Johansen <john.johansen@canonical.com>
+  Debian        Ben Hutchings <ben@decadent.org.uk>
+  Oracle        Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
+  Red Hat       Josh Poimboeuf <jpoimboe@redhat.com>
+  SUSE          Jiri Kosina <jkosina@suse.cz>
+
+  Google        Kees Cook <keescook@chromium.org>
+
+  LLVM          Nick Desaulniers <ndesaulniers@google.com>
+  ============= ========================================================
+
+Se você quiser que sua organização seja adicionada à lista de embaixadores,
+entre em contato com a equipe de segurança de hardware. O embaixador indicado
+deve compreender e apoiar totalmente o nosso processo e, idealmente, estar bem
+conectado na comunidade do kernel Linux.
+
+Listas de e-mail criptografadas
+-------------------------------
+
+Usamos listas de e-mail criptografadas para comunicação. O princípio de
+operação dessas listas é que o e-mail enviado para a lista é criptografado
+com a chave PGP da lista ou com o certificado S/MIME da lista. O software
+da lista de e-mail descriptografa o e-mail e o recriptografa individualmente
+para cada assinante com a chave PGP ou certificado S/MIME do assinante.
+Detalhes sobre o software da lista de e-mail e a configuração usada para
+garantir a segurança das listas e a proteção dos dados podem ser encontrados
+aqui: https://korg.wiki.kernel.org/userdoc/remail.
+
+Listas de chaves
+^^^^^^^^^^^^^^^^
+
+Para o contato inicial, consulte a seção :ref:`pt_BR_Contact` acima. Para listas de
+e-mail específicas de incidentes, a chave e o certificado S/MIME são transmitidos
+aos assinantes por e-mail enviado a partir da lista específica.
+
+Inscrição em listas específicas de incidentes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A inscrição em listas específicas de incidentes é gerenciada pelas equipes de
+resposta. As partes informadas que desejam participar da comunicação enviam
+uma lista de potenciais especialistas para a equipe de resposta, para que esta
+possa validar as solicitações de inscrição.
+
+Cada assinante precisa enviar uma solicitação de inscrição para a equipe de
+resposta por e-mail. O e-mail deve estar assinado com a chave PGP ou o certificado
+S/MIME do assinante. Se uma chave PGP for utilizada, ela deve estar disponível
+em um servidor de chaves público e, idealmente, conectada à teia de confiança
+(web of trust) PGP do kernel Linux. Veja também:
+https://www.kernel.org/signature.html.
+
+A equipe de resposta verifica se a solicitação do assinante é válida e o
+adiciona à lista. Após a inscrição, o assinante receberá e-mails da lista de
+e-mail que são assinados com a chave PGP da lista ou com o certificado S/MIME
+da lista. O cliente de e-mail do assinante pode extrair a chave PGP ou o
+certificado S/MIME da assinatura para que o assinante possa enviar e-mails
+criptografados para a lista.
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/howto.rst b/Documentation/translations/pt_BR/process/howto.rst
index bcedee7273fd87..54ea7fad8d7534 100644
--- a/Documentation/translations/pt_BR/process/howto.rst
+++ b/Documentation/translations/pt_BR/process/howto.rst
@@ -67,7 +67,8 @@ Questões Legais
 O código-fonte do kernel Linux é lançado sob a GPL. Por favor, veja o arquivo
 COPYING no diretório principal da árvore de fontes. As regras de licenciamento
 do kernel Linux e como usar os identificadores `SPDX <https://spdx.org/>`_ no
-código-fonte estão descritas em :ref:`Documentation/process/license-rules.rst <kernel_licensing>`.
+código-fonte estão descritas em
+:ref:`Documentation/translations/pt_BR/process/license-rules.rst <pt_BR_kernel_licensing>`.
 Se você tiver mais perguntas sobre a licença, por favor, entre em contato com
 um advogado e não pergunte na lista de discussão do kernel Linux. As pessoas
 nas listas de discussão não são advogados e você não deve confiar em suas
@@ -94,7 +95,7 @@ a lista linux-api@vger.kernel.org.
 Aqui está uma lista de arquivos que estão na árvore de fontes do kernel e
 que são de leitura obrigatória:
 
-  :ref:`Documentation/admin-guide/README.rst <readme>`
+  :ref:`Documentation/translations/pt_BR/admin-guide/README.rst <pt_BR_readme>`
     Este arquivo fornece um breve histórico sobre o kernel Linux e descreve
     o que é necessário fazer para configurar e compilar o kernel. Pessoas
     que são novas no kernel devem começar por aqui.
@@ -104,14 +105,14 @@ que são de leitura obrigatória:
     software que são necessários para compilar e executar o kernel com
     sucesso.
 
-  :ref:`Documentation/process/coding-style.rst <codingstyle>`
+  :ref:`Documentation/translations/pt_BR/process/coding-style.rst <pt_BR_codingstyle>`
     Este documento descreve o estilo de codificação do kernel Linux e parte
     da fundamentação por trás dele. Espera-se que todo código novo siga as
     diretrizes deste documento. A maioria dos mantenedores apenas aceitará
     patches se essas regras forem seguidas, e muitas pessoas apenas
     revisarão o código se ele estiver no estilo adequado.
 
-  :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+  :ref:`Documentation/translations/pt_BR/process/submitting-patches.rst <pt_BR_submittingpatches>`
     Este arquivo descreve em detalhes explícitos como criar e enviar
     um patch com sucesso, incluindo (mas não limitado a):
 
@@ -144,12 +145,12 @@ Outras excelentes descrições de como criar patches adequadamente são:
     do Linux e é muito importante para pessoas que estão migrando para o
     Linux vindas do desenvolvimento em outros Sistemas Operacionais.
 
-  :ref:`Documentation/process/security-bugs.rst <securitybugs>`
+  :ref:`Documentation/translations/pt_BR/process/security-bugs.rst <pt_BR_securitybugs>`
     Se você acredita ter encontrado um problema de segurança no kernel Linux,
     por favor, siga os passos descritos neste documento para ajudar a
     notificar os desenvolvedores do kernel e auxiliar na resolução do problema.
 
-  :ref:`Documentation/process/management-style.rst <managementstyle>`
+  :ref:`Documentation/translations/pt_BR/process/management-style.rst <pt_BR_managementstyle>`
     Este documento descreve como os mantenedores do kernel Linux operam e o
     ethos compartilhado por trás de suas metodologias. Esta é uma leitura
     importante para qualquer pessoa nova no desenvolvimento do kernel (ou
@@ -162,12 +163,12 @@ Outras excelentes descrições de como criar patches adequadamente são:
     versões estáveis (stable) do kernel e o que fazer se você desejar que
     uma alteração seja incluída em um desses lançamentos.
 
-  :ref:`Documentation/process/kernel-docs.rst <kernel_docs>`
+  :ref:`Documentation/translations/pt_BR/process/kernel-docs.rst <pt_BR_kernel_docs>`
     Uma lista de documentação externa que pertence ao desenvolvimento do
     kernel. Por favor, consulte esta lista caso não encontre o que está
     procurando dentro da documentação interna do kernel.
 
-  :ref:`Documentation/process/applying-patches.rst <applying_patches>`
+  :ref:`Documentation/translations/pt_BR/process/applying-patches.rst <pt_BR_applying_patches>`
     Uma boa introdução descrevendo exatamente o que é um patch e como
     aplicá-lo aos diferentes ramos (branches) de desenvolvimento do kernel.
 
@@ -435,7 +436,7 @@ individualmente, em vez de escrever tudo no topo do e-mail.
 
 Se você adicionar patches ao seu e-mail, certifique-se de que sejam texto
 puro legível, conforme declarado em
-:ref:`Documentation/process/submitting-patches.rst <submittingpatches>`.
+:ref:`Documentation/translations/pt_BR/process/submitting-patches.rst <pt_BR_submittingpatches>`.
 Os desenvolvedores do kernel não querem lidar com anexos ou patches
 compactados; eles podem querer comentar linhas individuais do seu patch,
 o que só funciona dessa forma. Certifique-se de usar um programa de
diff --git a/Documentation/translations/pt_BR/process/index.rst b/Documentation/translations/pt_BR/process/index.rst
index eda2a3fc5166f2..d072f7e46767e2 100644
--- a/Documentation/translations/pt_BR/process/index.rst
+++ b/Documentation/translations/pt_BR/process/index.rst
@@ -26,6 +26,7 @@ sua entrada na comunidade do kernel.
 
    Como começar <howto>
    Guia do Processo de Desenvolvimento <development-process>
+   Enviando patches: o guia essencial para colocar o seu código no kernel <submitting-patches>
    Lista de verificação para submissão de patches do kernel Linux <submit-checklist>
 
 Ferramentas e guias técnicos para desenvolvedores do kernel
@@ -38,10 +39,12 @@ devem estar familiarizados.
    :maxdepth: 1
 
    Requisitos mínimos <changes>
+   Estilo de codificação do kernel Linux <coding-style>
    Informações sobre clientes de email para Linux <email-clients>
    Como aplicar patches <applying-patches>
    Backporting e resolução de conflitos <backporting>
    Adicionando uma nova chamada de Sistema <adding-syscalls>
+   Por que a classe de tipo "volatile" não deve ser usada <volatile-considered-harmful>
    Como não Deixar as ioctls malfeitas <botching-up-ioctls>
 
 Guias de políticas e declarações de desenvolvedores
@@ -57,8 +60,12 @@ Estas são as regras pelas quais tentamos viver na comunidade do kernel
    Código de Conduta de Compromisso do Colaborador <code-of-conduct>
    Interpretação do Código de Conduta do Kernel Linux <code-of-conduct-interpretation>
    Modelos de Maturidade para Contribuição no Kernel Linux <contribution-maturity-model.rst>
+   Declaração de Aplicação do Kernel Linux <kernel-enforcement-statement>
    Declaração sobre Drivers do Kernel <kernel-driver-statement>
+   A interface de drivers do kernel Linux <stable-api-nonsense>
    Estilo de gerenciamento do kernel Linux <management-style>
+   Assistentes de código <coding-assistants>
+   O manual da árvore tip <maintainer-tip>
    Conclave (Continuidade do projeto) <conclave>
 
 Lidando com bugs
@@ -71,7 +78,9 @@ gerenciamento de bugs e vulnerabilidades.
 .. toctree::
    :maxdepth: 1
 
+   Dicas de depuração para desenvolvedores do Kernel Linux <debugging/index>
    Falhas de segurança <security-bugs>
+   Problemas de hardware sob embargo <embargoed-hardware-issues>
    CVEs <cve>
 
 Informações para mantenedores
@@ -88,6 +97,7 @@ mantenedores de subsistemas.
    Processo do subsistema SoC <maintainer-soc>
    Conformidade de DTS para SoC <maintainer-soc-clean-dts>
    Processo do subsistema KVM x86 <maintainer-kvm-x86>
+   Subsistema de Devicetree e Open Firmware <maintainer-devicetree>
 
 Outros materiais
 ----------------
diff --git a/Documentation/translations/pt_BR/process/kernel-docs.rst b/Documentation/translations/pt_BR/process/kernel-docs.rst
index 3c8d80ffa567f8..6235b1e7bde0de 100644
--- a/Documentation/translations/pt_BR/process/kernel-docs.rst
+++ b/Documentation/translations/pt_BR/process/kernel-docs.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _pt_BR_kernel_docs:
+
 Índice de Documentação Adicional do Kernel
 ==========================================
 
diff --git a/Documentation/translations/pt_BR/process/kernel-enforcement-statement.rst b/Documentation/translations/pt_BR/process/kernel-enforcement-statement.rst
new file mode 100644
index 00000000000000..3833c433c36b12
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/kernel-enforcement-statement.rst
@@ -0,0 +1,163 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Declaração de Aplicação do Kernel Linux
+---------------------------------------
+
+Como desenvolvedores do kernel Linux, temos um grande interesse em como nosso
+software é usado e como a licença do nosso software é aplicada. A conformidade
+com as obrigações de compartilhamento recíproco da GPL-2.0 é crítica para a
+sustentabilidade a longo prazo do nosso software e da nossa comunidade.
+
+Embora exista o direito de aplicar os interesses de direitos autorais separados nas
+contribuições feitas à nossa comunidade, compartilhamos o interesse em garantir
+que ações de aplicação individuais sejam conduzidas de uma maneira que beneficie
+nossa comunidade e não tenham um impacto negativo não intencional na saúde e no
+crescimento do nosso ecossistema de software. A fim de deter ações de aplicação
+inúteis, concordamos que é do melhor interesse da nossa comunidade de
+desenvolvimento assumir o seguinte compromisso com os usuários do kernel Linux,
+em nosso nome e em nome de quaisquer sucessores dos nossos interesses de
+direitos autorais:
+
+    Não obstante as disposições de rescisão da GPL-2.0, concordamos que
+    é do melhor interesse da nossa comunidade de desenvolvimento adotar as
+    seguintes disposições da GPL-3.0 como permissões adicionais sob nossa
+    licença com relação a qualquer reivindicação não defensiva de direitos sob a
+    licença.
+
+	No entanto, se você cessar toda violação desta Licença, sua licença
+	de um detentor de direitos autorais específico será restabelecida (a)
+	provisoriamente, a menos e até que o detentor de direitos autorais encerre
+	explícita e definitivamente sua licença, e (b) permanentemente, se o
+	detentor de direitos autorais não notificar você sobre a violação por
+	algum meio razoável antes de 60 dias após a cessação.
+
+	Além disso, sua licença de um detentor de direitos autorais específico é
+	restabelecida permanentemente se o detentor de direitos autorais
+	notificá-lo da violação por algum meio razoável, se esta for a primeira
+	vez que você recebe um aviso de violação desta Licença (para qualquer
+	trabalho) daquele detentor de direitos autorais, e você sanar a violação
+	antes de 30 dias após o recebimento do aviso.
+
+Nossa intenção ao fornecer essas garantias é incentivar um maior uso do
+software. Queremos que empresas e indivíduos usem, modifiquem e distribuam
+este software. Queremos trabalhar com os usuários de forma aberta e transparente
+para eliminar qualquer incerteza sobre nossas expectativas em relação à
+conformidade ou aplicação que possa limitar a adoção do nosso software. Vemos
+a ação judicial como um último recurso, a ser iniciada apenas quando outros
+esforços da comunidade falharem em resolver o problema.
+
+Por fim, uma vez que um problema de não conformidade seja resolvido, esperamos
+que o usuário se sinta bem-vindo para se juntar a nós em nossos esforços neste
+projeto. Trabalhando juntos, seremos mais fortes.
+
+Exceto onde indicado abaixo, falamos apenas por nós mesmos, e não por qualquer
+empresa para a qual possamos trabalhar hoje, no passado ou no futuro.
+
+  - Bjorn Andersson (Linaro)
+  - Andrea Arcangeli
+  - Neil Armstrong
+  - Jens Axboe
+  - Pablo Neira Ayuso
+  - Khalid Aziz
+  - Ralf Baechle
+  - Felipe Balbi
+  - Arnd Bergmann
+  - Ard Biesheuvel
+  - Tim Bird
+  - Paolo Bonzini
+  - Christian Borntraeger
+  - Mark Brown (Linaro)
+  - Paul Burton
+  - Javier Martinez Canillas
+  - Rob Clark
+  - Kees Cook (Google)
+  - Jonathan Corbet
+  - Dennis Dalessandro
+  - Vivien Didelot (Savoir-faire Linux)
+  - Hans de Goede
+  - Mel Gorman (SUSE)
+  - Sven Eckelmann
+  - Alex Elder (Linaro)
+  - Fabio Estevam
+  - Larry Finger
+  - Bhumika Goyal
+  - Andy Gross
+  - Juergen Gross
+  - Shawn Guo
+  - Ulf Hansson
+  - Stephen Hemminger (Microsoft)
+  - Tejun Heo
+  - Rob Herring
+  - Masami Hiramatsu
+  - Michal Hocko
+  - Simon Horman
+  - Johan Hovold (Hovold Consulting AB)
+  - Christophe JAILLET
+  - Olof Johansson
+  - Lee Jones (Linaro)
+  - Heiner Kallweit
+  - Srinivas Kandagatla
+  - Jan Kara
+  - Shuah Khan (Samsung)
+  - David Kershner
+  - Jaegeuk Kim
+  - Namhyung Kim
+  - Colin Ian King
+  - Jeff Kirsher
+  - Greg Kroah-Hartman (Linux Foundation)
+  - Christian König
+  - Vinod Koul
+  - Krzysztof Kozlowski
+  - Viresh Kumar
+  - Aneesh Kumar K.V
+  - Julia Lawall
+  - Doug Ledford
+  - Chuck Lever (Oracle)
+  - Daniel Lezcano
+  - Shaohua Li
+  - Xin Long
+  - Tony Luck
+  - Catalin Marinas (Arm Ltd)
+  - Mike Marshall
+  - Chris Mason
+  - Paul E. McKenney
+  - Arnaldo Carvalho de Melo
+  - David S. Miller
+  - Ingo Molnar
+  - Kuninori Morimoto
+  - Trond Myklebust
+  - Martin K. Petersen (Oracle)
+  - Borislav Petkov
+  - Jiri Pirko
+  - Josh Poimboeuf
+  - Sebastian Reichel (Collabora)
+  - Guenter Roeck
+  - Joerg Roedel
+  - Leon Romanovsky
+  - Steven Rostedt (VMware)
+  - Frank Rowand
+  - Ivan Safonov
+  - Anna Schumaker
+  - Jes Sorensen
+  - K.Y. Srinivasan
+  - David Sterba (SUSE)
+  - Heiko Stuebner
+  - Jiri Kosina (SUSE)
+  - Willy Tarreau
+  - Dmitry Torokhov
+  - Linus Torvalds
+  - Thierry Reding
+  - Rik van Riel
+  - Luis R. Rodriguez
+  - Geert Uytterhoeven (Glider bvba)
+  - Eduardo Valentin (Amazon.com)
+  - Daniel Vetter
+  - Linus Walleij
+  - Richard Weinberger
+  - Dan Williams
+  - Rafael J. Wysocki
+  - Arvind Yadav
+  - Masahiro Yamada
+  - Wei Yongjun
+  - Lv Zheng
+  - Marc Zyngier (Arm Ltd)
diff --git a/Documentation/translations/pt_BR/process/license-rules.rst b/Documentation/translations/pt_BR/process/license-rules.rst
index 1e395dfea8759c..7a801b2a4f4e2e 100644
--- a/Documentation/translations/pt_BR/process/license-rules.rst
+++ b/Documentation/translations/pt_BR/process/license-rules.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _pt_BR_kernel_licensing:
+
 Regras de licenciamento do kernel Linux
 =======================================
 
diff --git a/Documentation/translations/pt_BR/process/maintainer-devicetree.rst b/Documentation/translations/pt_BR/process/maintainer-devicetree.rst
new file mode 100644
index 00000000000000..be26d89defdff6
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/maintainer-devicetree.rst
@@ -0,0 +1,76 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=========================================
+Subsistema de Devicetree e Open Firmware
+=========================================
+
+Outros documentos sobre o processo
+----------------------------------
+
+Consulte os documentos em Documentation/devicetree/bindings/ para saber como
+escrever bindings de Devicetree corretamente e como enviar patches.
+
+Revisão e tratamento de patches
+-------------------------------
+
+Os patches sob responsabilidade dos mantenedores de Devicetree são processados
+de formas distintas, conforme o tipo de patch:
+
+1. Código central de drivers OF, por exemplo, drivers/of/:
+   os patches são revisados e aplicados pelos mantenedores de DT.
+
+2. Bindings de Devicetree:
+   os patches são revisados pelos mantenedores de DT, mas devem ser aplicados
+   pelos mantenedores do subsistema, exceto em alguns casos. Consulte também
+   *Para mantenedores do kernel* em
+   Documentation/devicetree/bindings/submitting-patches.rst.
+
+3. DTS e drivers:
+   os mantenedores de DT podem fazer comentários, mas, em geral, não se espera
+   uma revisão. Os DTS devem passar nas verificações de esquema
+   (dtbs_check) ou, ao menos, não gerar novos avisos.
+
+Patchwork
+~~~~~~~~~
+
+Os mantenedores de Devicetree revisam patches usando o Patchwork; portanto, o
+status atual de um patch pode ser consultado por lá. Em submissões típicas de
+drivers, o Patchwork recebe toda a série de patches, mas normalmente apenas
+alguns patches são bindings de Devicetree e, assim, revisados pelos
+mantenedores de DT.
+
+Explicação dos status do Patchwork:
+
+ - **New**: ainda não processado pelo conjunto de ferramentas de automação.
+ - **Needs ACK**: aguardando revisão dos mantenedores de DT.
+ - **Handled Elsewhere**: patch não relacionado a DT; não será revisado aqui.
+ - **RFC**: o patch provavelmente foi ignorado por ser um RFC incompleto.
+ - **Changes Requested**: o patch foi revisado e os mantenedores de DT esperam
+   alterações.
+ - **Accepted**: o patch foi revisado e aplicado pelos mantenedores de DT em
+   sua árvore.
+ - **Not Applicable**: o patch foi revisado e provavelmente está em boas
+   condições, com uma tag *Reviewed-by* ou *Acked-by* fornecida, mas os
+   mantenedores de DT esperam que outra pessoa o aplique.
+
+Nova revisão e pings de patches
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Devido ao alto volume de e-mails, os mantenedores de Devicetree não leem todas
+as mensagens que recebem; em vez disso, eles dependem do Patchwork durante o
+processo de revisão. Além disso, muitas vezes deixam de lado patches que já
+foram revisados.
+
+Como resultado, os mantenedores podem não perceber:
+
+1. Perguntas sobre patches já revisados.
+2. Pings, por exemplo, quando um patch foi revisado pelos mantenedores de DT,
+   mas ainda não foi aplicado pelos mantenedores do subsistema.
+
+Esses casos podem ser tratados das seguintes formas:
+
+1. Enviando um ping aos mantenedores de DT no canal de IRC.
+2. Removendo a tag *Acked-by* ou *Reviewed-by* do mantenedor de DT ao enviar
+   uma nova versão da série de patches, junto com uma explicação no changelog
+   do patch sobre o motivo da remoção da tag e o que se espera dos mantenedores
+   de DT.
diff --git a/Documentation/translations/pt_BR/process/maintainer-handbooks.rst b/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
index b7aab788ffdbf2..1307306501ab1c 100644
--- a/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
+++ b/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
@@ -5,8 +5,8 @@ Notas sobre o processo de desenvolvimento de subsistemas e mantenedores
 
 O propósito deste documento é fornecer informações específicas de
 subsistemas que são suplementares ao manual geral do processo de
-desenvolvimento.
-:ref:`Documentation/process <development_process_main>`.
+desenvolvimento
+:ref:`Documentation/translations/pt_BR/process <pt_BR_development_process_main>`.
 
 Para desenvolvedores, veja abaixo todos os guias específicos de
 subsistemas conhecidos. Se o subsistema para o qual você está
diff --git a/Documentation/translations/pt_BR/process/maintainer-kvm-x86.rst b/Documentation/translations/pt_BR/process/maintainer-kvm-x86.rst
index 6480ff08b9d8be..ef133d32c517cb 100644
--- a/Documentation/translations/pt_BR/process/maintainer-kvm-x86.rst
+++ b/Documentation/translations/pt_BR/process/maintainer-kvm-x86.rst
@@ -122,7 +122,7 @@ Quando se trata de estilo, nomenclatura, padrões, etc., a consistência é a
 prioridade número um no KVM x86. Se tudo mais falhar, siga o que já existe.
 
 Com algumas ressalvas listadas abaixo, siga o estilo de codificação preferido
-dos mantenedores da árvore "tip" (:ref:`maintainer-tip-coding-style`), já que
+dos mantenedores da árvore "tip" (:ref:`pt_BR_maintainer-tip-coding-style`), já que
 patches/séries frequentemente tocam tanto arquivos do KVM quanto arquivos x86
 não-KVM, ou seja, atraem a atenção de mantenedores do KVM *e* da árvore "tip".
 
@@ -206,7 +206,7 @@ Novos tópicos surgem ocasionalmente, mas, por favor, inicie uma discussão na
 lista se desejar propor a introdução de um novo tópico; ou seja, não aja por
 conta própria.
 
-Veja :ref:`the_canonical_patch_format` para mais informações, com uma ressalva:
+Veja :ref:`pt_BR_the_canonical_patch_format` para mais informações, com uma ressalva:
 não trate o limite de 70-75 caracteres como um limite absoluto e rígido. Em
 vez disso, use 75 caracteres como um limite firme, mas não rígido, e use 80
 caracteres como um limite intransponível. Ou seja, permita que o shortlog
@@ -218,7 +218,7 @@ Changelog
 O mais importante: escreva os changelogs usando o modo imperativo e evite o uso
 de pronomes.
 
-Veja :ref:`describe_changes` para mais informações, com uma ressalva: comece com
+Veja :ref:`pt_BR_describe_changes` para mais informações, com uma ressalva: comece com
 uma breve descrição das mudanças reais e, em seguida, apresente o contexto e o
 histórico. Note! Esta ordem entra em conflito direto com a abordagem preferida
 da árvore "tip"! Por favor, siga o estilo preferido da árvore "tip" ao enviar
@@ -431,5 +431,5 @@ Bugs que podem ser explorados pelo convidado (guest) para atacar o hospedeiro
 (host) (kernel ou espaço do usuário), ou que podem ser explorados por uma VM
 aninhada (nested) contra o *seu* próprio hospedeiro (L2 atacando L1), são de
 interesse particular para o KVM. Por favor, siga o protocolo em
-:ref:`securitybugs` se você suspeitar que um bug possa levar a um escape,
-vazamento de dados, etc.
+:ref:`pt_BR_securitybugs` se você suspeitar que um bug possa levar a um
+escape, vazamento de dados, etc.
diff --git a/Documentation/translations/pt_BR/process/maintainer-soc-clean-dts.rst b/Documentation/translations/pt_BR/process/maintainer-soc-clean-dts.rst
index a7e7bf0f106f45..7256c71534a791 100644
--- a/Documentation/translations/pt_BR/process/maintainer-soc-clean-dts.rst
+++ b/Documentation/translations/pt_BR/process/maintainer-soc-clean-dts.rst
@@ -8,8 +8,9 @@ Visão Geral
 -----------
 
 As plataformas SoC ou subarquiteturas devem seguir todas as regras de
-Documentation/process/maintainer-soc.rst. Este documento, referenciado em
-MAINTAINERS, impõe requisitos adicionais listados abaixo.
+Documentation/translations/pt_BR/process/maintainer-soc.rst. Este
+documento, referenciado em MAINTAINERS, impõe requisitos adicionais
+listados abaixo.
 
 Conformidade Estrita com DT Schema de DTS e dtc
 -----------------------------------------------
diff --git a/Documentation/translations/pt_BR/process/maintainer-tip.rst b/Documentation/translations/pt_BR/process/maintainer-tip.rst
new file mode 100644
index 00000000000000..e583bad4282d61
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/maintainer-tip.rst
@@ -0,0 +1,847 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+O manual da árvore tip
+======================
+
+O que é a árvore tip?
+---------------------
+
+A árvore tip é uma coleção de vários subsistemas e áreas de
+desenvolvimento. A árvore tip é tanto uma árvore de desenvolvimento direto quanto uma
+árvore de agregação para várias árvores de sub-mantenedores. A URL gitweb da árvore tip
+é: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
+
+A árvore tip contém os seguintes subsistemas:
+
+   - **Arquitetura x86**
+
+     O desenvolvimento da arquitetura x86 ocorre na árvore tip, exceto
+     pelas partes específicas do KVM e XEN no x86, que são mantidas nos
+     subsistemas correspondentes e roteadas diretamente para a mainline a partir
+     dali. Ainda é uma boa prática enviar Cc para os mantenedores x86 nos
+     patches do KVM e XEN específicos para x86.
+
+     Alguns subsistemas x86 têm seus próprios mantenedores além dos
+     mantenedores gerais do x86. Por favor, envie Cc para os mantenedores gerais do x86 em
+     patches que toquem em arquivos em arch/x86, mesmo quando não forem indicados
+     pelo arquivo MAINTAINER.
+
+     Note que ``x86@kernel.org`` não é uma lista de discussão. É meramente um
+     alias de e-mail que distribui mensagens para a equipe de mantenedores de nível superior
+     do x86. Por favor, sempre envie Cc para a lista de discussão do Linux Kernel (LKML)
+     ``linux-kernel@vger.kernel.org``, caso contrário, seu e-mail acabará apenas nas
+     caixas de entrada privadas dos mantenedores.
+
+   - **Scheduler**
+
+     O desenvolvimento do scheduler ocorre na árvore -tip, na
+     branch sched/core - com ocasionais árvores de subtópicos para
+     conjuntos de patches em progresso.
+
+   - **Locking e atomics**
+
+     O desenvolvimento de locking (incluindo atomics e outras primitivas de
+     sincronização que estão conectadas ao locking) ocorre na árvore -tip,
+     na branch locking/core - com ocasionais árvores de subtópicos
+     para conjuntos de patches em progresso.
+
+   - **Subsistema genérico de interrupções e drivers de chip de interrupção**:
+
+     - o desenvolvimento do núcleo de interrupções ocorre na branch irq/core
+
+     - o desenvolvimento do driver de chip de interrupção também ocorre na branch
+       irq/core, mas os patches geralmente são aplicados em uma árvore de mantenedor
+       separada e depois agregados na irq/core
+
+   - **Tempo, timers, timekeeping, NOHZ e drivers de chip relacionados**:
+
+     - o desenvolvimento do timekeeping, núcleo clocksource, NTP e alarmtimer
+       ocorre na branch timers/core, mas os patches geralmente são aplicados em
+       uma árvore de mantenedor separada e depois agregados na timers/core
+
+     - o desenvolvimento do driver clocksource/event ocorre na branch
+       timers/core, mas os patches são em sua maioria aplicados em uma árvore de mantenedor
+       separada e depois agregados na timers/core
+
+   - **Núcleo de contadores de desempenho, suporte a arquitetura e ferramentas**:
+
+     - o desenvolvimento do núcleo perf e suporte a arquitetura ocorre na
+       branch perf/core
+
+     - o desenvolvimento de ferramentas perf ocorre na árvore do mantenedor
+       de ferramentas perf e é agregado à árvore tip.
+
+   - **Núcleo de hotplug de CPU**
+
+   - **Núcleo RAS**
+
+     Em sua maioria, os patches RAS específicos para x86 são coletados na branch
+     ras/core da árvore tip.
+
+   - **Núcleo EFI**
+
+     Desenvolvimento EFI na árvore git efi. Os patches coletados são
+     agregados na branch efi/core da árvore tip.
+
+   - **RCU**
+
+     O desenvolvimento do RCU ocorre na árvore linux-rcu. As mudanças resultantes
+     são agregadas na branch core/rcu da árvore tip.
+
+   - **Vários componentes de código do núcleo**:
+
+       - debugobjects
+
+       - objtool
+
+       - partes e peças aleatórias
+
+
+Notas de submissão de patch
+---------------------------
+
+Selecionando a árvore/branch
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Em geral, o desenvolvimento contra o head da branch master da árvore tip é
+adequado, mas para os subsistemas que são mantidos separadamente, possuem sua
+própria árvore git e são apenas agregados na árvore tip, o desenvolvimento deve
+ocorrer contra a árvore ou branch do subsistema relevante.
+
+Correções de bugs que visam a mainline devem sempre ser aplicáveis contra a
+árvore do kernel mainline. Potenciais conflitos contra mudanças que já estão
+na fila da árvore tip são resolvidos pelos mantenedores.
+
+Assunto do patch
+^^^^^^^^^^^^^^^^
+
+O formato preferido da árvore tip para prefixos de assunto do patch é
+'subsys/component:', ex. 'x86/apic:', 'x86/mm/fault:', 'sched/fair:',
+'genirq/core:'. Por favor, não use nomes de arquivos ou caminhos de arquivos completos como
+prefixo. 'git log path/to/file' deve lhe dar uma dica razoável na maioria
+dos casos.
+
+A descrição condensada do patch na linha de assunto deve começar com uma
+letra maiúscula e deve ser escrita em tom imperativo.
+
+
+Changelog
+^^^^^^^^^
+
+As regras gerais sobre changelogs no :ref:`Guia de submissão de patches
+<pt_BR_describe_changes>`, se aplicam.
+
+Os mantenedores da árvore tip valorizam seguir essas regras, especialmente no
+pedido para escrever changelogs no modo imperativo e não personificando
+o código ou sua execução. Isso não é apenas um capricho dos
+mantenedores. Changelogs escritos em palavras abstratas são mais precisos e
+tendem a ser menos confusos do que aqueles escritos em forma de romances.
+
+Também é útil estruturar o changelog em vários parágrafos e não
+juntar tudo em um só. Uma boa estrutura é explicar
+o contexto, o problema e a solução em parágrafos separados e nesta
+ordem.
+
+Exemplos para ilustração:
+
+  Exemplo 1::
+
+    x86/intel_rdt/mbm: Corrigir o manipulador de overflow do MBM durante hot cpu
+
+    Quando uma CPU está morrendo, cancelamos o worker e agendamos um novo worker em uma
+    CPU diferente no mesmo domínio. Mas se o timer já está prestes a
+    expirar (digamos 0.99s) então essencialmente dobramos o intervalo.
+
+    Modificamos o tratamento de hot cpu para cancelar o trabalho atrasado na cpu
+    que está morrendo e executar o worker imediatamente em uma cpu diferente no mesmo domínio. Não
+    fazemos o flush do worker porque o worker de overflow do MBM reagenda o
+    worker na mesma CPU e escaneia a domain->cpu_mask para obter o ponteiro
+    do domínio.
+
+  Versão melhorada::
+
+    x86/intel_rdt/mbm: Corrigir o manipulador de overflow do MBM durante hotplug de CPU
+
+    Quando uma CPU está morrendo, o worker de overflow é cancelado e reagendado em uma
+    CPU diferente no mesmo domínio. Mas se o timer já estiver prestes a
+    expirar isso essencialmente dobra o intervalo, o que pode resultar em um overflow
+    não detectado.
+
+    Cancele o worker de overflow e reagende-o imediatamente em uma CPU diferente
+    no mesmo domínio. O trabalho também poderia sofrer um flush, mas isso iria
+    reagendá-lo na mesma CPU.
+
+  Exemplo 2::
+
+    time: POSIX CPU timers: Garantir que a variável seja inicializada
+
+    Se cpu_timer_sample_group retornar -EINVAL, ela não terá escrito em
+    *sample. Checar o valor de retorno de cpu_timer_sample_group previne o
+    uso potencial de um valor não inicializado de now no bloco seguinte.
+    Dado um clock_idx inválido, o código anterior poderia caso contrário sobrescrever
+    *oldval de maneira indefinida. Isso agora é prevenido. Também exploramos
+    o curto-circuito do && para amostrar o timer apenas se o resultado for
+    realmente usado para atualizar *oldval.
+
+  Versão melhorada::
+
+    posix-cpu-timers: Tornar set_process_cpu_timer() mais robusto
+
+    Como o valor de retorno de cpu_timer_sample_group() não é checado,
+    compiladores e checadores estáticos podem legitimamente avisar sobre um uso potencial
+    da variável não inicializada 'now'. Isso não é um problema de tempo de execução pois todos
+    os locais de chamada passam ids de clock válidos.
+
+    Além disso, cpu_timer_sample_group() é invocado incondicionalmente mesmo quando o
+    resultado não é usado porque *oldval é NULL.
+
+    Torne a invocação condicional e cheque o valor de retorno.
+
+  Exemplo 3::
+
+    A entidade também pode ser usada para outros propósitos.
+
+    Vamos renomeá-la para ser mais genérica.
+
+  Versão melhorada::
+
+    A entidade também pode ser usada para outros propósitos.
+
+    Renomeie para ser mais genérica.
+
+
+Para cenários complexos, especialmente condições de corrida (race conditions) e problemas
+de ordenação de memória, é valioso descrever o cenário com uma tabela que mostra
+o paralelismo e a ordem temporal dos eventos. Aqui está um exemplo::
+
+    CPU0                            CPU1
+    free_irq(X)                     interrupt X
+                                    spin_lock(desc->lock)
+                                    wake irq thread()
+                                    spin_unlock(desc->lock)
+    spin_lock(desc->lock)
+    remove action()
+    shutdown_irq()
+    release_resources()             thread_handler()
+    spin_unlock(desc->lock)           access released resources.
+                                      ^^^^^^^^^^^^^^^^^^^^^^^^^
+    synchronize_irq()
+
+O Lockdep fornece uma saída útil semelhante para descrever um possível cenário
+de deadlock::
+
+    CPU0                                    CPU1
+    rtmutex_lock(&rcu->rt_mutex)
+      spin_lock(&rcu->rt_mutex.wait_lock)
+                                            local_irq_disable()
+                                            spin_lock(&timer->it_lock)
+                                            spin_lock(&rcu->mutex.wait_lock)
+    --> Interrupt
+        spin_lock(&timer->it_lock)
+
+Referências a funções em changelogs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Quando uma função é mencionada no changelog, seja no corpo do texto ou na
+linha de assunto, por favor use o formato 'nome_da_funcao()'. Omitir os
+parênteses após o nome da função pode ser ambíguo::
+
+  Subject: subsys/component: Make reservation_count static
+
+  reservation_count is only used in reservation_stats. Make it static.
+
+A variante com parênteses é mais precisa::
+
+  Subject: subsys/component: Make reservation_count() static
+
+  reservation_count() is only called from reservation_stats(). Make it
+  static.
+
+
+Backtraces em changelogs
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Veja :ref:`pt_BR_backtraces`.
+
+Ordenação das tags de commit
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Para ter uma visão uniforme das tags de commit, os mantenedores da tip usam o
+seguinte esquema de ordenação de tags:
+
+ - Fixes: 12+char-SHA1 ("sub/sys: Original subject line")
+
+   Uma tag Fixes deve ser adicionada mesmo para alterações que não precisam ser
+   portadas de volta (backported) para kernels estáveis, ou seja, quando abordar um
+   problema recém-introduzido que afeta apenas a árvore tip ou o head atual da linha principal (mainline). Estas tags
+   são úteis para identificar o commit original e são muito mais valiosas
+   do que mencionar de forma proeminente o commit que introduziu um problema no
+   próprio texto do changelog, porque elas podem ser automaticamente
+   extraídas.
+
+   O exemplo a seguir ilustra a diferença::
+
+     Commit
+
+       abcdef012345678 ("x86/xxx: Replace foo with bar")
+
+     deixou uma instância não utilizada da variável foo. Remova-a.
+
+     Signed-off-by: J.Dev <j.dev@mail>
+
+   Por favor, diga em vez disso::
+
+     A recente substituição de foo por bar deixou uma instância não utilizada da
+     variável foo. Remova-a.
+
+     Fixes: abcdef012345678 ("x86/xxx: Replace foo with bar")
+     Signed-off-by: J.Dev <j.dev@mail>
+
+   O último coloca as informações sobre o patch em foco e
+   as complementa com a referência ao commit que introduziu o problema,
+   em vez de colocar o foco no commit original em primeiro lugar.
+
+ - Reported-by: ``Reporter <reporter@mail>``
+
+ - Closes: ``URL or Message-ID of the bug report this is fixing``
+
+ - Originally-by: ``Original author <original-author@mail>``
+
+ - Suggested-by: ``Suggester <suggester@mail>``
+
+ - Co-developed-by: ``Co-author <co-author@mail>``
+
+   Signed-off-by: ``Co-author <co-author@mail>``
+
+   Note que Co-developed-by e Signed-off-by do(s) co-autor(es) devem
+   vir em pares.
+
+ - Signed-off-by: ``Author <author@mail>``
+
+   O primeiro Signed-off-by (SOB) após o último par Co-developed-by/SOB é o
+   SOB do autor, ou seja, a pessoa marcada como autora pelo git.
+
+ - Signed-off-by: ``Patch handler <handler@mail>``
+
+   SOBs após o SOB do autor são de pessoas que lidam e transportam
+   o patch, mas não estiveram envolvidas no desenvolvimento. As cadeias de SOB devem
+   refletir a rota **real** que um patch tomou conforme foi propagado para nós,
+   com a primeira entrada de SOB sinalizando a autoria principal de um único
+   autor. Acks devem ser dados como linhas Acked-by e aprovações de revisão
+   como linhas Reviewed-by.
+
+   Se o manipulador fez modificações no patch ou no changelog, então
+   isso deve ser mencionado **após** o texto do changelog e **acima**
+   de todas as tags de commit no seguinte formato::
+
+     ... o texto do changelog termina.
+
+     [ handler: Substituiu foo por bar e atualizou o changelog ]
+
+     First-tag: .....
+
+   Observe as duas novas linhas vazias que separam o texto do changelog e as
+   tags de commit daquele aviso.
+
+   Se um patch for enviado para a lista de discussão por um manipulador, então o autor tem
+   que ser notado na primeira linha do changelog com::
+
+     From: Author <author@mail>
+
+     O texto do changelog começa aqui....
+
+   assim a autoria é preservada. A linha 'From:' tem que ser seguida
+   por uma nova linha vazia. Se essa linha 'From:' estiver faltando, então o patch
+   seria atribuído à pessoa que o enviou (transportou, manipulou).
+   A linha 'From:' é automaticamente removida quando o patch é aplicado
+   e não aparece no changelog final do git. Ela meramente afeta
+   a informação de autoria do commit resultante do Git.
+
+ - Tested-by: ``Tester <tester@mail>``
+
+ - Reviewed-by: ``Reviewer <reviewer@mail>``
+
+ - Acked-by: ``Acker <acker@mail>``
+
+ - Cc: ``cc-ed-person <person@mail>``
+
+   Se o patch deve ser portado para stable, então por favor adicione uma tag '``Cc:
+   stable@vger.kernel.org``', mas não coloque em Cc o stable ao enviar o seu
+   e-mail.
+
+ - Link: ``https://link/to/information``
+
+   Para se referir a um e-mail postado nas listas de discussão do kernel, por favor
+   use o URL de redirecionamento lore.kernel.org::
+
+     Link: https://lore.kernel.org/email-message-id@here
+
+   Esta URL deve ser usada ao se referir a tópicos de lista de discussão relevantes,
+   conjuntos de patches relacionados, ou outras threads de discussão notáveis.
+   Uma maneira conveniente de associar os trailers ``Link:`` com a mensagem de commit
+   é usar a notação de colchetes semelhante ao markdown, por exemplo::
+
+     A similar approach was attempted before as part of a different
+     effort [1], but the initial implementation caused too many
+     regressions [2], so it was backed out and reimplemented.
+
+     Link: https://lore.kernel.org/some-msgid@here # [1]
+     Link: https://bugzilla.example.org/bug/12345  # [2]
+
+   Você também pode usar os trailers ``Link:`` para indicar a origem do
+   patch ao aplicá-lo em sua árvore git. Neste caso, por favor use o
+   domínio dedicado ``patch.msgid.link`` em vez de ``lore.kernel.org``.
+   Esta prática torna possível que as ferramentas automatizadas identifiquem
+   qual link usar para recuperar o envio do patch original. Por
+   exemplo::
+
+     Link: https://patch.msgid.link/patch-source-message-id@here
+
+Por favor não use tags combinadas, ex. ``Reported-and-tested-by``, pois
+elas apenas complicam a extração automatizada de tags.
+
+
+Links para documentação
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Fornecer links para a documentação no changelog é uma grande ajuda para depuração e
+análise posteriores. Infelizmente, os URLs costumam quebrar muito rapidamente
+porque as empresas reestruturam seus sites frequentemente. Exceções não 'voláteis'
+incluem o Intel SDM e o AMD APM.
+
+Portanto, para documentos 'voláteis', por favor crie uma entrada no bugzilla do kernel
+https://bugzilla.kernel.org e anexe uma cópia desses documentos
+à entrada do bugzilla. Finalmente, forneça o URL da entrada do bugzilla no
+changelog.
+
+Reenvio de patch ou lembretes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Veja :ref:`pt_BR_resend_reminders`.
+
+Janela de merge
+^^^^^^^^^^^^^^^
+
+Por favor, não espere que os patches sejam revisados ou mesclados pelos mantenedores da árvore tip
+em torno ou durante a janela de merge. As árvores ficam fechadas
+para todos, exceto correções urgentes, durante esse tempo. Elas reabrem assim que a janela de merge
+fecha e um novo kernel -rc1 é lançado.
+
+Grandes séries devem ser enviadas em estado mesclável (mergeable state) *pelo* *menos* uma semana
+antes da janela de merge abrir. Exceções são feitas para correções de bugs e
+*às vezes* para pequenos drivers independentes para novos hardwares ou patches minimamente
+invasivos para ativação de hardware.
+
+Durante a janela de merge, os mantenedores se concentram em seguir as
+alterações upstream, corrigir problemas resultantes da janela de merge, coletar correções de bugs, e
+se permitir um respiro. Por favor, respeite isso.
+
+Os chamados branches _urgent_ serão mesclados na linha principal (mainline) durante a
+fase de estabilização de cada versão.
+
+
+Git
+^^^
+
+Os mantenedores da árvore tip aceitam pull requests do git de mantenedores que fornecem
+alterações de subsistema para agregação na árvore tip.
+
+Pull requests para novos envios de patches normalmente não são aceitos e não
+substituem o envio adequado de patch para a lista de discussão. O principal motivo para
+isso é que o fluxo de trabalho de revisão é baseado em e-mail.
+
+Se você enviar uma série maior de patches, é útil fornecer um branch git
+em um repositório privado que permita que pessoas interessadas façam pull da
+série facilmente para testes. A maneira usual de oferecer isso é uma URL do git na carta de apresentação (cover letter)
+da série de patches.
+
+Testes
+^^^^^^
+
+O código deve ser testado antes de ser enviado para os mantenedores da árvore tip. Qualquer coisa
+além de alterações menores deve ser construída, inicializada e testada com
+opções abrangentes (e pesadas) de depuração do kernel ativadas.
+
+Essas opções de depuração podem ser encontradas em kernel/configs/x86_debug.config
+e podem ser adicionadas a uma configuração de kernel existente executando:
+
+	make x86_debug.config
+
+Algumas dessas opções são específicas do x86 e podem ser deixadas de fora ao testar
+em outras arquiteturas.
+
+.. _pt_BR_maintainer-tip-coding-style:
+
+Notas de estilo de código
+-------------------------
+
+Estilo de comentário
+^^^^^^^^^^^^^^^^^^^^
+
+Frases em comentários começam com uma letra maiúscula.
+
+Comentários de linha única::
+
+	/* Este é um comentário de linha única */
+
+Comentários de várias linhas::
+
+	/*
+	 * This is a properly formatted
+	 * multi-line comment.
+	 *
+	 * Larger multi-line comments should be split into paragraphs.
+	 */
+
+Sem comentários no fim da linha (veja abaixo):
+
+  Por favor, abstenha-se de usar comentários no fim da linha. Comentários no fim da linha atrapalham o
+  fluxo de leitura em quase todos os contextos, mas especialmente em código::
+
+	if (somecondition_is_true) /* Não coloque um comentário aqui */
+		dostuff(); /* Nem aqui */
+
+	seed = MAGIC_CONSTANT; /* Nem aqui */
+
+  Use comentários independentes em vez disso::
+
+	/* Esta condição não é óbvia sem um comentário */
+	if (somecondition_is_true) {
+		/* Isso realmente precisa ser documentado */
+		dostuff();
+	}
+
+	/* Esta inicialização mágica precisa de um comentário. Talvez não? */
+	seed = MAGIC_CONSTANT;
+
+  Use o estilo C++, comentários no fim da linha ao documentar structs em headers para
+  alcançar um layout mais compacto e melhor legibilidade::
+
+        // eax
+        u32     x2apic_shift    :  5, // Número de bits para deslocar o ID APIC para a direita
+                                      // para o ID de topologia no próximo nível
+                                : 27; // Reservado
+        // ebx
+        u32     num_processors  : 16, // Número de processadores no nível atual
+                                : 16; // Reservado
+
+  versus::
+
+	/* eax */
+	        /*
+	         * Número de bits para deslocar o ID APIC para a direita para o ID de topologia
+	         * no próximo nível
+	         */
+         u32     x2apic_shift    :  5,
+		 /* Reservado */
+				 : 27;
+
+	/* ebx */
+		/* Número de processadores no nível atual */
+	u32     num_processors  : 16,
+		/* Reservado */
+				: 16;
+
+Comente as coisas importantes:
+
+  Comentários devem ser adicionados onde a operação não é óbvia. Documentar
+  o óbvio é apenas uma distração::
+
+	/* Decrementa o refcount e verifica por zero */
+	if (refcount_dec_and_test(&p->refcnt)) {
+		do;
+		lots;
+		of;
+		magic;
+		things;
+	}
+
+  Em vez disso, os comentários devem explicar os detalhes não óbvios e documentar
+  as restrições::
+
+	if (refcount_dec_and_test(&p->refcnt)) {
+		/*
+		 * Explicação muito boa de por que as coisas mágicas abaixo
+		 * precisam ser feitas, restrições de ordenação e locking,
+		 * etc..
+		 */
+		do;
+		lots;
+		of;
+		magic;
+		/* Precisa ser a última operação porque ... */
+		things;
+	}
+
+Comentários de documentação de função:
+
+  Para documentar funções e seus argumentos por favor use o formato kernel-doc
+  e não comentários de formato livre::
+
+	/**
+	 * magic_function - Faz muitas coisas mágicas
+	 * @magic:	Ponteiro para os dados mágicos nos quais operar
+	 * @offset:	Deslocamento no array de dados de @magic
+	 *
+	 * Explicação profunda das coisas misteriosas feitas com @magic junto
+         * com a documentação dos valores de retorno.
+	 *
+	 * Note que os descritores de argumento acima estão dispostos
+	 * de forma tabular.
+	 */
+
+  Isto se aplica especialmente a funções visíveis globalmente e funções
+  inline em arquivos de cabeçalho públicos. Pode ser um exagero usar o formato
+  kernel-doc para cada função (estática) que precisa de uma pequena explicação. O
+  uso de nomes de funções descritivos frequentemente substitui esses pequenos comentários.
+  Aplique o bom senso como sempre.
+
+
+Documentando requisitos de locking
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  Documentar requisitos de locking é uma coisa boa, mas comentários não
+  são necessariamente a melhor escolha. Em vez de escrever::
+
+	/* Caller must hold foo->lock */
+	void func(struct foo *foo)
+	{
+		...
+	}
+
+  Por favor, use::
+
+	void func(struct foo *foo)
+	{
+		lockdep_assert_held(&foo->lock);
+		...
+	}
+
+  Em kernels PROVE_LOCKING, lockdep_assert_held() emite um aviso
+  se o chamador não detém o lock. Comentários não podem fazer isso.
+
+Regras de chaves
+^^^^^^^^^^^^^^^^
+
+Chaves devem ser omitidas apenas se a instrução que se segue a 'if', 'for',
+'while' etc. for verdadeiramente uma única linha::
+
+	if (foo)
+		do_something();
+
+O seguinte não é considerado uma instrução de linha única mesmo
+que o C não exija chaves::
+
+	for (i = 0; i < end; i++)
+		if (foo[i])
+			do_something(foo[i]);
+
+Adicionar chaves ao redor do loop externo melhora o fluxo de leitura::
+
+	for (i = 0; i < end; i++) {
+		if (foo[i])
+			do_something(foo[i]);
+	}
+
+
+Declarações de variáveis
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+A ordem preferida das declarações de variáveis no início de uma
+função é a ordem de árvore de abeto invertida (reverse fir tree order)::
+
+	struct long_struct_name *descriptive_name;
+	unsigned long foo, bar;
+	unsigned int tmp;
+	int ret;
+
+O que está acima é mais rápido de analisar do que a ordem invertida::
+
+	int ret;
+	unsigned int tmp;
+	unsigned long foo, bar;
+	struct long_struct_name *descriptive_name;
+
+E ainda mais do que uma ordem aleatória::
+
+	unsigned long foo, bar;
+	int ret;
+	struct long_struct_name *descriptive_name;
+	unsigned int tmp;
+
+Também por favor tente agregar variáveis do mesmo tipo em uma única
+linha. Não há sentido em desperdiçar espaço na tela::
+
+	unsigned long a;
+	unsigned long b;
+	unsigned long c;
+	unsigned long d;
+
+É realmente suficiente fazer::
+
+	unsigned long a, b, c, d;
+
+Por favor, evite também introduzir divisões de linha em declarações de variáveis::
+
+	struct long_struct_name *descriptive_name = container_of(bar,
+						      struct long_struct_name,
+	                                              member);
+	struct foobar foo;
+
+É muito melhor mover a inicialização para uma linha separada após as
+declarações::
+
+	struct long_struct_name *descriptive_name;
+	struct foobar foo;
+
+	descriptive_name = container_of(bar, struct long_struct_name, member);
+
+
+Tipos de variáveis
+^^^^^^^^^^^^^^^^^^
+
+Por favor use os tipos u8, u16, u32, u64 adequados para variáveis que são destinadas
+a descrever hardware ou são usadas como argumentos para funções que acessam
+hardware. Estes tipos definem claramente a largura em bits e evitam
+truncamento, expansão e confusão entre 32/64 bits.
+
+u64 também é recomendado em código que se tornaria ambíguo para kernels
+de 32 bits quando 'unsigned long' fosse usado em vez disso. Embora em tais
+situações 'unsigned long long' pudesse ser usado também, u64 é mais curto
+e também mostra claramente que a operação requer uma largura de 64 bits
+independente da CPU alvo.
+
+Por favor use 'unsigned int' em vez de 'unsigned'.
+
+
+Constantes
+^^^^^^^^^^
+
+Por favor, não use números (hexa)decimais literais em código ou inicializadores.
+Ou use defines adequados que tenham nomes descritivos ou considere usar
+um enum.
+
+
+Declarações e inicializadores de struct
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As declarações de struct devem alinhar os nomes dos membros da struct de forma
+tabular::
+
+	struct bar_order {
+		unsigned int	guest_id;
+		int		ordered_item;
+		struct menu	*menu;
+	};
+
+Por favor, evite documentar os membros da struct dentro da declaração, pois
+isso frequentemente resulta em comentários formatados de maneira estranha e os membros da struct
+ficam ofuscados::
+
+	struct bar_order {
+		unsigned int	guest_id; /* ID único do convidado */
+		int		ordered_item;
+		/* Ponteiro para uma instância de menu que contém todas as bebidas */
+		struct menu	*menu;
+	};
+
+Em vez disso, por favor considere usar o formato kernel-doc em um comentário precedendo
+a declaração da struct, que é mais fácil de ler e tem a vantagem adicional
+de incluir a informação na documentação do kernel, por exemplo, da
+seguinte forma::
+
+
+	/**
+	 * struct bar_order - Descrição de um pedido de bar
+	 * @guest_id:		ID único do convidado
+	 * @ordered_item:	O número do item do menu
+	 * @menu:		Ponteiro para o menu do qual o item
+	 *  			foi pedido
+	 *
+	 * Informação suplementar para usar a struct.
+	 *
+	 * Note que os descritores dos membros da struct acima estão dispostos
+	 * de forma tabular.
+	 */
+	struct bar_order {
+		unsigned int	guest_id;
+		int		ordered_item;
+		struct menu	*menu;
+	};
+
+Inicializadores de struct estáticos devem usar inicializadores C99 e também devem ser
+alinhados de forma tabular::
+
+	static struct foo statfoo = {
+		.a		= 0,
+		.plain_integer	= CONSTANT_DEFINE_OR_ENUM,
+		.bar		= &statbar,
+	};
+
+Note que embora a sintaxe C99 permita a omissão da vírgula final,
+nós recomendamos o uso de uma vírgula na última linha porque isso torna
+o reordenamento e a adição de novas linhas mais fáceis, e também torna tais
+patches futuros ligeiramente mais fáceis de ler.
+
+Quebras de linha
+^^^^^^^^^^^^^^^^
+
+Restringir o comprimento da linha a 80 caracteres torna código profundamente indentado difícil de
+ler. Considere dividir o código em funções auxiliares para evitar quebra de
+linha excessiva.
+
+A regra de 80 caracteres não é uma regra estrita, então por favor use bom senso ao
+quebrar linhas. Especialmente strings de formato nunca devem ser divididas.
+
+Ao dividir declarações de funções ou chamadas de funções, então por favor alinhe
+o primeiro argumento na segunda linha com o primeiro argumento na primeira
+linha::
+
+  static int long_function_name(struct foobar *barfoo, unsigned int id,
+				unsigned int offset)
+  {
+
+	if (!id) {
+		ret = longer_function_name(barfoo, DEFAULT_BARFOO_ID,
+					   offset);
+	...
+
+Namespaces
+^^^^^^^^^^
+
+Namespaces de funções/variáveis melhoram a legibilidade e permitem
+grepping fácil. Estes namespaces são prefixos de string para nomes
+de funções e variáveis visíveis globalmente, incluindo inlines. Estes prefixos devem
+combinar o subsistema e o nome do componente como 'x86_comp\_',
+'sched\_', 'irq\_', e 'mutex\_'.
+
+Isso também inclui funções estáticas de escopo de arquivo que são imediatamente colocadas
+em templates de driver visíveis globalmente - é útil que esses símbolos
+também carreguem um bom prefixo, para legibilidade do backtrace.
+
+Prefixos de namespace podem ser omitidos para funções e variáveis
+estáticas locais. Funções verdadeiramente locais, chamadas apenas por outras funções locais,
+podem ter nomes descritivos mais curtos - nossa preocupação principal é a facilidade de grepping
+e a legibilidade do backtrace.
+
+Por favor note que os prefixos 'xxx_vendor\_' e 'vendor_xxx\_' não são
+úteis para funções estáticas em arquivos específicos de fornecedores. Afinal,
+já está claro que o código é específico do fornecedor. Além disso, nomes
+de fornecedores devem ser apenas para funcionalidades verdadeiramente específicas de fornecedores.
+
+Como sempre, aplique o bom senso e vise a consistência e a legibilidade.
+
+
+Notificações de commit
+----------------------
+
+A árvore tip é monitorada por um bot por novos commits. O bot envia um email
+para cada novo commit para uma lista de discussão dedicada
+(``linux-tip-commits@vger.kernel.org``) e coloca em Cc todas as pessoas que são
+mencionadas em uma das tags de commit. Ele usa o ID da mensagem de email da
+tag Link no final da lista de tags para definir o cabeçalho de email In-Reply-To para que
+a mensagem seja encadeada corretamente com o email de submissão do patch.
+
+Os mantenedores e submantenedores tip tentam responder ao remetente
+ao fazer o merge de um patch, mas às vezes eles esquecem ou isso não se encaixa no
+fluxo de trabalho do momento. Embora a mensagem do bot seja puramente mecânica, ela
+também implica em um 'Obrigado! Aplicado.'.
diff --git a/Documentation/translations/pt_BR/process/management-style.rst b/Documentation/translations/pt_BR/process/management-style.rst
index b92f8705c30fd1..9d067e04650a15 100644
--- a/Documentation/translations/pt_BR/process/management-style.rst
+++ b/Documentation/translations/pt_BR/process/management-style.rst
@@ -1,13 +1,16 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _pt_BR_managementstyle:
+
 Estilo de gerenciamento do kernel Linux
 =======================================
 
 Este é um documento curto descrevendo o estilo de gerenciamento preferido (ou
 inventado, dependendo de quem você perguntar) para o kernel do Linux. Ele se
-destina a espelhar o documento :ref:`process/coding-style.rst <codingstyle>` em
-algum grau, e foi escrito principalmente para evitar responder [#f1]_ as mesmas
-(ou semelhantes) perguntas repetidamente.
+destina a espelhar o documento
+:ref:`Documentation/translations/pt_BR/process/coding-style.rst <pt_BR_codingstyle>`
+em algum grau, e foi escrito principalmente para evitar responder [#f1]_ as
+mesmas (ou semelhantes) perguntas repetidamente.
 
 Estilo de gerenciamento é muito pessoal e muito mais difícil de quantificar do
 que simples regras de estilo de codificação, então este documento pode ou não ter
diff --git a/Documentation/translations/pt_BR/process/security-bugs.rst b/Documentation/translations/pt_BR/process/security-bugs.rst
index 72c771869566bd..321575657b2824 100644
--- a/Documentation/translations/pt_BR/process/security-bugs.rst
+++ b/Documentation/translations/pt_BR/process/security-bugs.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _pt_BR_securitybugs:
+
 Falhas de segurança
 ===================
 
@@ -59,11 +61,11 @@ Além disso, as seguintes informações são altamente desejáveis:
     mantenedores, mesmo que a correção acabe não sendo a correta, pois ajuda a
     entender o bug. Ao propor uma correção testada, por favor, formate-a
     sempre de uma maneira que possa ser mesclada imediatamente (consulte
-    Documentation/process/submitting-patches.rst). Isso evitará algumas trocas
-    de mensagens caso ela seja aceita, e você receberá o crédito por
-    encontrar e corrigir o problema. Observe que, neste caso, apenas uma tag
-    ``Signed-off-by:`` é necessária, sem ``Reported-by:`` quando o relator e
-    o autor forem a mesma pessoa.
+    Documentation/translations/pt_BR/process/submitting-patches.rst). Isso
+    evitará algumas trocas de mensagens caso ela seja aceita, e você receberá
+    o crédito por encontrar e corrigir o problema. Observe que, neste caso,
+    apenas uma tag ``Signed-off-by:`` é necessária, sem ``Reported-by:``
+    quando o relator e o autor forem a mesma pessoa.
 
   * **mitigações**: com muita frequência, durante a análise de um bug,
     surgem algumas maneiras de mitigar o problema. É útil compartilhá-las,
@@ -228,8 +230,8 @@ a tornar esses relatórios desnecessariamente difíceis de lidar:
     Se a correção não puder ser testada porque depende de hardware raro ou de
     protocolos de rede quase extintos, é provável que o problema não seja um
     bug de segurança. Em qualquer caso, se uma correção for proposta, ela deve
-    aderir a Documentation/process/submitting-patches.rst e incluir uma tag
-    'Fixes:' designando o commit que introduziu o bug.
+    aderir a Documentation/translations/pt_BR/process/submitting-patches.rst
+    e incluir uma tag 'Fixes:' designando o commit que introduziu o bug.
 
 A falha em considerar estes pontos expõe seu relatório ao risco de ser
 ignorado.
@@ -280,7 +282,7 @@ entender e corrigir a vulnerabilidade de segurança.
 Por favor, envie e-mails em **texto simples** sem anexos, sempre que possível.
 É muito mais difícil ter uma discussão com citações de contexto sobre um
 problema complexo se todos os detalhes estiverem ocultos em anexos. Pense nisso
-como uma :doc:`regular path submission </../../../process/submitting-patches>`
+como uma :doc:`submissão pelo caminho normal <submitting-patches>`
 (mesmo que você ainda não tenha um patch): descreva o problema e o impacto,
 liste as etapas de reprodução e siga com uma proposta de correção, tudo em
 texto simples. Relatórios formatados em Markdown, HTML e RST são
@@ -288,8 +290,9 @@ particularmente malvistos, pois são bastante difíceis de ler por humanos e
 incentivam o uso de visualizadores dedicados, às vezes online, o que por
 definição não é aceitável para um relatório de segurança confidencial. Note
 que alguns clientes de e-mail tendem a corromper a formatação de texto simples
-por padrão; por favor, consulte Documentation/process/email-clients.rst para
-mais informações.
+por padrão; por favor, consulte
+Documentation/translations/pt_BR/process/email-clients.rst para mais
+informações.
 
 Divulgação e informações sob embargo
 ------------------------------------
@@ -359,7 +362,7 @@ A equipe de segurança não atribui CVEs, nem os exigimos para relatórios ou
 correções, pois isso pode complicar desnecessariamente o processo e adiar o
 tratamento do bug. Se um relator desejar que um identificador CVE seja
 atribuído para um problema confirmado, ele pode entrar em contato com a
-:doc:`kernel CVE assignment team<../../../process/cve>` para obter um.
+:doc:`equipe de atribuição de CVEs do kernel <cve>` para obter um.
 
 Acordo de não divulgação
 ------------------------
diff --git a/Documentation/translations/pt_BR/process/stable-api-nonsense.rst b/Documentation/translations/pt_BR/process/stable-api-nonsense.rst
new file mode 100644
index 00000000000000..c5d0e643eb3231
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/stable-api-nonsense.rst
@@ -0,0 +1,208 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+A interface de drivers do kernel Linux
+=======================================
+
+(todas as suas perguntas respondidas e mais algumas)
+
+Greg Kroah-Hartman <greg@kroah.com>
+
+Este texto foi escrito para tentar explicar por que o Linux **não possui uma
+interface binária do kernel nem uma interface estável do kernel**.
+
+.. note::
+
+   Observe que este artigo descreve as interfaces **internas do kernel**, e não
+   as interfaces entre o kernel e o espaço de usuário.
+
+   A interface entre o kernel e o espaço de usuário é aquela utilizada pelos
+   aplicativos: a interface de chamadas de sistema (syscalls). Essa
+   interface é **muito** estável ao longo do tempo e não será quebrada. Tenho
+   programas antigos, compilados em uma versão do kernel anterior à 0.9 e alguma
+   coisa, que ainda funcionam perfeitamente na versão mais recente do kernel
+   2.6. Essa é a interface cuja estabilidade os usuários e desenvolvedores de
+   aplicativos podem considerar garantida.
+
+
+Resumo executivo
+----------------
+
+Você acha que quer uma interface estável do kernel, mas, na verdade, não quer,
+e nem sabe disso. O que você realmente quer é um driver que continue
+funcionando de maneira estável, e isso só é possível se o seu driver estiver
+na árvore principal do kernel. Você também obtém muitos outros benefícios se
+o seu driver fizer parte da árvore principal do kernel. São esses benefícios
+que ajudaram a tornar o Linux um sistema operacional tão robusto, estável e
+maduro — justamente a razão pela qual você o está usando.
+
+
+Introdução
+----------
+
+Apenas quem escreve drivers para o kernel precisa se preocupar com as mudanças
+nas interfaces internas do kernel. Para a grande maioria das pessoas, essas
+interfaces nem sequer são visíveis e tampouco são motivo de preocupação.
+
+Antes de mais nada, não abordarei **nenhuma** questão jurídica relacionada a
+código-fonte fechado, código-fonte oculto, blobs binários, wrappers de
+código-fonte ou qualquer outro termo usado para descrever drivers do kernel
+cujo código-fonte não seja disponibilizado sob a GPL. Consulte um advogado
+caso tenha alguma dúvida jurídica. Sou programador e, portanto, descreverei
+aqui apenas as questões técnicas (isso não significa que as questões
+jurídicas sejam pouco importantes; elas são reais e você precisa estar sempre
+ciente delas).
+
+Portanto, há dois tópicos principais: interfaces binárias do kernel e
+interfaces estáveis de código-fonte do kernel. Ambos dependem um do outro,
+mas discutiremos primeiro a parte referente às interfaces binárias para
+deixá-la de lado.
+
+
+Interface binária do kernel
+---------------------------
+
+Supondo que tivéssemos uma interface estável de código-fonte para o kernel,
+uma interface binária surgiria naturalmente também, certo? Errado. Considere
+os seguintes fatos sobre o kernel Linux:
+
+  - Dependendo da versão do compilador C utilizada, diferentes estruturas de
+    dados do kernel terão diferentes alinhamentos e poderão até mesmo incluir
+    funções de maneiras distintas (por exemplo, tornando determinadas funções
+    inline ou não). A organização das funções individuais não é tão importante,
+    mas as diferenças no preenchimento das estruturas de dados são muito
+    importantes.
+  - Dependendo das opções selecionadas durante a compilação do kernel, uma
+    grande variedade de comportamentos pode ser assumida pelo kernel:
+
+      - diferentes estruturas podem conter campos diferentes;
+      - algumas funções podem nem sequer ser implementadas (por exemplo,
+        determinados bloqueios são completamente eliminados durante a
+        compilação em kernels sem SMP);
+      - a memória dentro do kernel pode ser alinhada de maneiras diferentes,
+        dependendo das opções de compilação.
+  - O Linux é executado em uma grande variedade de arquiteturas de
+    processadores. Não há como drivers binários compilados para uma arquitetura
+    funcionarem corretamente em outra.
+
+Vários desses problemas podem ser contornados simplesmente compilando o módulo
+para uma configuração específica e exata do kernel, utilizando exatamente o
+mesmo compilador C empregado na compilação do kernel. Isso é suficiente caso
+você queira fornecer um módulo para uma determinada versão de uma distribuição
+Linux específica. Porém, multiplique essa única compilação pelo número de
+distribuições Linux existentes e pelo número de versões suportadas de cada
+distribuição e você rapidamente terá um pesadelo de diferentes opções de
+compilação em diferentes versões. Além disso, cada versão de uma distribuição
+Linux contém vários kernels, cada um ajustado para diferentes tipos de
+hardware (diferentes tipos de processadores e diferentes opções). Portanto,
+mesmo para uma única versão, você precisará criar várias versões do seu módulo.
+
+Acredite em mim: com o tempo, você enlouquecerá se tentar oferecer suporte a
+esse tipo de distribuição. Aprendi isso da maneira mais difícil há muito
+tempo...
+
+Interfaces estáveis de código-fonte do kernel
+----------------------------------------------
+
+Esse é um tópico um pouco mais "volátil" se você conversar com alguém que está
+tentando manter atualizado, ao longo do tempo, um driver do kernel Linux que
+não está na árvore principal do kernel.
+
+O desenvolvimento do kernel Linux é contínuo e ocorre em ritmo acelerado,
+sem desacelerar. Por isso, os desenvolvedores do kernel encontram bugs nas
+interfaces existentes ou descobrem maneiras melhores de fazer as coisas.
+Quando isso acontece, eles corrigem as interfaces atuais para que funcionem
+melhor. Nesse processo, nomes de funções podem mudar, estruturas podem crescer
+ou diminuir e parâmetros de funções podem ser reformulados. Quando isso
+acontece, todos os locais dentro do kernel que utilizam essa interface são
+corrigidos ao mesmo tempo, garantindo que tudo continue funcionando
+corretamente.
+
+Como exemplos específicos disso, as interfaces USB internas do kernel
+passaram por pelo menos três reformulações diferentes ao longo da existência
+desse subsistema. Essas reformulações foram feitas para resolver diversos
+problemas:
+
+  - Uma mudança de um modelo síncrono de fluxos de dados para um modelo
+    assíncrono. Isso reduziu a complexidade de vários drivers e aumentou a
+    taxa de transferência de todos os drivers USB, de modo que atualmente
+    executamos quase todos os dispositivos USB na maior velocidade possível.
+  - Foi feita uma mudança na maneira como os pacotes de dados eram alocados
+    pelos drivers USB a partir do núcleo USB, de modo que todos os drivers
+    passaram a precisar fornecer mais informações ao núcleo USB, corrigindo
+    diversos deadlocks documentados.
+
+Isso contrasta fortemente com vários sistemas operacionais de código fechado,
+que tiveram de manter suas interfaces USB antigas ao longo do tempo. Isso
+permite que novos desenvolvedores utilizem acidentalmente interfaces antigas
+e façam as coisas de maneira inadequada, prejudicando a estabilidade do
+sistema operacional.
+
+Em ambos os casos, todos os desenvolvedores concordaram que essas eram
+mudanças importantes que precisavam ser feitas, e elas foram realizadas com
+relativamente pouco esforço. Se o Linux tivesse de garantir a preservação de
+uma interface de código-fonte estável, uma nova interface teria de ser criada,
+enquanto a interface antiga e defeituosa teria de continuar sendo mantida ao
+longo do tempo, resultando em trabalho adicional para os desenvolvedores USB.
+Como todos os desenvolvedores USB do Linux realizam esse trabalho em seu
+próprio tempo, pedir que programadores façam trabalho extra, sem nenhum
+benefício e gratuitamente, não é uma possibilidade.
+
+Questões de segurança também são muito importantes para o Linux. Quando um
+problema de segurança é encontrado, ele é corrigido em um período muito curto.
+Em diversas ocasiões, isso fez com que interfaces internas do kernel fossem
+reformuladas para impedir que o problema de segurança ocorresse. Quando isso
+acontece, todos os drivers que utilizam essas interfaces também são corrigidos
+ao mesmo tempo, garantindo que o problema de segurança seja resolvido e não
+possa reaparecer acidentalmente no futuro. Se as interfaces internas não
+pudessem ser alteradas, não seria possível corrigir esse tipo de problema de
+segurança e garantir que ele não voltasse a ocorrer.
+
+As interfaces do kernel são aprimoradas ao longo do tempo. Se ninguém estiver
+utilizando uma determinada interface, ela é removida. Isso garante que o
+kernel permaneça o menor possível e que todas as interfaces existentes possam
+ser testadas da melhor maneira possível (é praticamente impossível testar
+adequadamente a validade de interfaces que não são utilizadas).
+
+
+O que fazer
+-----------
+
+Então, se você possui um driver do kernel Linux que não está na árvore
+principal do kernel, o que você, como desenvolvedor, deve fazer? Distribuir
+um driver binário para cada versão diferente do kernel em cada distribuição
+é um pesadelo, e tentar acompanhar uma interface do kernel que está em
+constante mudança também é uma tarefa difícil.
+
+Simples: coloque seu driver na árvore principal do kernel (lembre-se de que
+estamos falando aqui de drivers distribuídos sob uma licença compatível com
+a GPL; se seu código não se enquadra nessa categoria, boa sorte, você está
+por conta própria aqui, seu parasita). Se seu driver estiver na árvore e uma
+interface do kernel mudar, ele será corrigido pela própria pessoa que realizou
+a alteração no kernel. Isso garante que seu driver continue sempre compilável
+e funcionando ao longo do tempo, exigindo muito pouco esforço de sua parte.
+
+Os excelentes efeitos colaterais de ter seu driver na árvore principal do
+kernel são:
+
+  - A qualidade do driver aumentará, enquanto os custos de manutenção
+    (para o desenvolvedor original) diminuirão.
+  - Outros desenvolvedores adicionarão funcionalidades ao seu driver.
+  - Outras pessoas encontrarão e corrigirão bugs no seu driver.
+  - Outras pessoas encontrarão oportunidades de otimização no seu driver.
+  - Outras pessoas atualizarão o driver para você quando mudanças em
+    interfaces externas exigirem isso.
+  - O driver será automaticamente distribuído por todas as distribuições
+    Linux, sem que seja necessário pedir às distribuições que o adicionem.
+
+Como o Linux oferece suporte, "pronto para uso", a um número maior de
+dispositivos diferentes do que qualquer outro sistema operacional, e oferece
+suporte a esses dispositivos em mais arquiteturas de processadores diferentes
+do que qualquer outro sistema operacional, esse modelo comprovado de
+desenvolvimento deve estar fazendo alguma coisa certa :)
+
+
+------
+
+Agradecimentos a Randy Dunlap, Andrew Morton, David Brownell, Hanna Linder,
+Robert Love e Nishanth Aravamudan pela revisão e pelos comentários sobre
+este documento.
diff --git a/Documentation/translations/pt_BR/process/submit-checklist.rst b/Documentation/translations/pt_BR/process/submit-checklist.rst
index 003fc956832df6..3e0ee38b9a72de 100644
--- a/Documentation/translations/pt_BR/process/submit-checklist.rst
+++ b/Documentation/translations/pt_BR/process/submit-checklist.rst
@@ -8,7 +8,7 @@ Aqui estão algumas coisas básicas que os desenvolvedores devem fazer se
 quiserem ver suas submissões de patches de kernel aceitas mais rapidamente.
 
 Estas diretrizes vão além da documentação fornecida em
-:ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+:ref:`Documentation/translations/pt_BR/process/submitting-patches.rst <pt_BR_submittingpatches>`
 e em outros locais sobre o envio de patches para o kernel Linux.
 
 Revise seu código
@@ -19,7 +19,7 @@ Revise seu código
    os que você usa de forma indireta.
 
 2) Verifique o estilo geral do seu patch conforme detalhado em
-   :ref:`Documentation//process/coding-style.rst <codingstyle>`.
+   :ref:`Documentation/translations/pt_BR/process/coding-style.rst <pt_BR_codingstyle>`.
 
 3) Todas as barreiras de memória {por exemplo, ``barrier()``, ``rmb()``,
    ``wmb()``} precisam de um comentário no código-fonte que explique a
diff --git a/Documentation/translations/pt_BR/process/submitting-patches.rst b/Documentation/translations/pt_BR/process/submitting-patches.rst
new file mode 100644
index 00000000000000..aad926c17fb454
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/submitting-patches.rst
@@ -0,0 +1,963 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _pt_BR_submittingpatches:
+
+Enviando patches: o guia essencial para colocar o seu código no kernel
+======================================================================
+
+Para uma pessoa ou empresa que deseja enviar uma mudança para o
+kernel Linux, o processo pode, por vezes, ser intimidador se você não
+estiver familiarizado com "o sistema". Este texto é uma coleção de sugestões
+que podem aumentar muito as chances de sua mudança ser aceita.
+
+Este documento contém um grande número de sugestões em um formato relativamente
+conciso. Para informações detalhadas sobre como funciona o processo de
+desenvolvimento do kernel, consulte Documentation/process/development-process.rst.
+Além disso, leia Documentation/process/submit-checklist.rst
+para uma lista de itens a serem verificados antes de enviar o código.
+Para patches de binding de device tree, leia
+Documentation/devicetree/bindings/submitting-patches.rst.
+
+Esta documentação assume que você está usando o ``git`` para preparar seus
+patches. Se você não está familiarizado com o ``git``, é muito recomendado que
+você aprenda a usá-lo, ele tornará a sua vida como um desenvolvedor do kernel e,
+em geral, muito mais fácil.
+
+Alguns subsistemas e árvores de mantenedores possuem informações adicionais
+sobre seus fluxos de trabalho e expectativas, consulte
+Documentation/process/maintainer-handbooks.rst.
+
+Obtenha uma árvore de código-fonte atual
+----------------------------------------
+
+Se você não tiver um repositório com o código-fonte atual do kernel em mãos,
+use o ``git`` para obter um. Você vai querer começar com o repositório mainline,
+que pode ser obtido com::
+
+  git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
+
+Note, no entanto, que você pode não querer desenvolver diretamente na
+árvore mainline. A maioria dos mantenedores de subsistemas mantém suas
+próprias árvores e desejam ver os patches preparados em relação a essas árvores.
+Consulte a entrada **T:** do subsistema no arquivo MAINTAINERS para encontrar
+essa árvore, ou simplesmente pergunte ao mantenedor se a árvore não estiver
+listada lá.
+
+.. _pt_BR_describe_changes:
+
+Descreva as suas mudanças
+-------------------------
+
+Descreva o seu problema. Seja o seu patch uma correção de bug de uma linha ou
+5000 linhas de um novo recurso, deve haver um problema subjacente que o motivou
+a fazer esse trabalho. Convença o revisor de que existe um problema que vale a
+pena corrigir e que faz sentido que ele leia além do primeiro parágrafo.
+
+Descreva o impacto visível ao usuário. Travamentos e bloqueios diretos são
+bastante convincentes, mas nem todos os bugs são tão evidentes. Mesmo que o
+problema tenha sido identificado durante a revisão do código, descreva o impacto
+que você acredita que ele pode ter sobre os usuários. Tenha em mente que a
+maioria das instalações Linux executa kernels de árvores estáveis secundárias
+ou árvores específicas de fornecedores/produtos que selecionam apenas patches
+específicos do upstream, então inclua qualquer coisa que possa ajudar a
+direcionar sua mudança downstream: circunstâncias provocadoras, trechos do
+dmesg, descrições do travamento, regressões de desempenho, picos de latência,
+bloqueios, etc.
+
+Quantifique as otimizações e compensações. Se você afirma haver melhorias no
+desempenho, consumo de memória, uso da pilha ou tamanho do binário, inclua
+números que as comprovem. Mas também descreva custos que não são óbvios.
+Otimizações geralmente não são gratuitas, sendo trocas entre CPU, memória e
+legibilidade; ou, quando se trata de heurísticas, entre diferentes cargas de
+trabalho. Descreva as desvantagens esperadas da sua otimização para que o
+revisor possa pesar os custos contra os benefícios.
+
+Uma vez estabelecido o problema, descreva o que você está efetivamente fazendo
+sobre ele, com detalhes técnicos. É importante descrever a mudança em inglês
+claro para o revisor verificar que o código está se comportando como
+você pretendia.
+
+O mantenedor agradecerá se você escrever a descrição do seu patch em uma forma
+que possa ser facilmente inserida no sistema de gerenciamento de código-fonte do
+Linux, o ``git``, como uma "mensagem de commit". Veja
+:ref:`pt_BR_the_canonical_patch_format`.
+
+Resolva apenas um problema por patch. Se a sua descrição começar a ficar longa,
+isso é um sinal de que você provavelmente precisa dividir o seu patch.
+Consulte :ref:`pt_BR_split_changes`.
+
+Quando você enviar ou reenviar um patch ou uma série de patches, inclua a
+descrição completa do patch e a justificativa para ele. Não diga apenas
+que esta é a versão N do patch (ou série). Não espere que o mantenedor
+do subsistema consulte versões anteriores do patch ou URLs de referência
+para encontrar a descrição do patch e colocá-la no patch.
+Ou seja, o patch (ou a série) e sua descrição devem ser autossuficientes.
+Isso beneficia tanto os mantenedores quanto os revisores. Alguns revisores
+provavelmente nem chegaram a receber as versões anteriores do patch.
+
+Descreva suas alterações no modo imperativo, por exemplo, "faça xyzzy executar
+frotz" em vez de "[Este patch] faz xyzzy executar frotz" ou "[Eu] mudei xyzzy
+para executar frotz", como se você estivesse dando ordens à base de código para
+mudar o seu comportamento.
+
+Se você quiser se referir a um commit específico, não se refira apenas ao
+ID SHA-1 do commit. Por favor, inclua também o resumo de uma linha do
+commit, para tornar mais fácil para os revisores saberem sobre o que se trata.
+Exemplo::
+
+	Commit e21d2170f36602ae2708 ("video: remove unnecessary
+	platform_set_drvdata()") removed the unnecessary
+	platform_set_drvdata(), but left the variable "dev" unused,
+	delete it.
+
+Você também deve ter a certeza de usar pelo menos os primeiros doze caracteres do
+ID SHA-1. O repositório do kernel possui um número *muito* grande de objetos, o que torna as
+colisões com IDs mais curtos uma possibilidade real. Tenha em mente que, mesmo que
+não haja colisão com o seu ID de seis caracteres agora, essa condição pode
+mudar daqui a cinco anos.
+
+Se discussões relacionadas ou qualquer outra informação de contexto por trás da mudança
+puderem ser encontradas na web, adicione tags 'Link:' apontando para isso. Se o patch é o
+resultado de algumas discussões anteriores na lista de e-mails ou algo documentado na
+web, aponte para ele.
+
+Ao criar links para arquivos de listas de e-mails, de preferência use o serviço
+de arquivo de mensagens lore.kernel.org. Para criar a URL do link, use o
+conteúdo do cabeçalho ``Message-ID`` da mensagem, sem os colchetes angulares
+circundantes. Por exemplo::
+
+    Link: https://lore.kernel.org/30th.anniversary.repost@klaava.Helsinki.FI
+
+Por favor, verifique o link para se certificar de que ele está realmente
+funcionando e aponta para a mensagem relevante.
+
+No entanto, tente tornar a sua explicação compreensível sem recursos
+externos. Além de fornecer um URL para um arquivo da lista de e-mails ou bug,
+resuma os pontos relevantes da discussão que levaram ao patch conforme enviado.
+
+Caso o seu patch corrija um bug, use a tag 'Closes:' com um URL que referencie o
+relato nos arquivos da lista de e-mails ou em um rastreador público de bugs. Por exemplo::
+
+	Closes: https://example.com/issues/1234
+
+Alguns rastreadores de bugs têm a capacidade de fechar os problemas
+automaticamente quando um commit com tal tag é aplicado. Alguns bots que monitoram as listas de
+e-mails também podem rastrear tais tags e tomar certas ações. Rastreadores de bugs
+privados e URLs inválidos são proibidos.
+
+Se o seu patch corrige um bug em um commit específico, por exemplo, você encontrou um problema usando
+``git bisect``, por favor, use a tag 'Fixes:' com pelo menos os primeiros 12
+caracteres do ID SHA-1 e o resumo de uma linha. Não divida a tag em
+várias linhas, as tags estão isentas da regra de "quebra de linha nas 75 colunas" para
+simplificar os scripts de parsing. Por exemplo::
+
+	Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
+
+As seguintes configurações do ``git config`` podem ser usadas para adicionar um formato aprimorado para
+exibir o estilo acima nos comandos ``git log`` ou ``git show``::
+
+	[core]
+		abbrev = 12
+	[pretty]
+		fixes = Fixes: %h (\"%s\")
+
+Um exemplo de chamada::
+
+	$ git log -1 --pretty=fixes 54a4f0239f2e
+	Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
+
+.. _pt_BR_split_changes:
+
+Separe as suas mudanças
+-----------------------
+
+Separe cada **mudança lógica** em um patch separado.
+
+Por exemplo, se as suas alterações incluírem tanto correções de bugs quanto melhorias
+de desempenho para um único driver, separe essas alterações em dois
+ou mais patches. Se as suas alterações incluírem uma atualização de API e um novo
+driver que utiliza essa nova API, separe-os em dois patches.
+
+Por outro lado, se você fizer uma única alteração em vários arquivos,
+agrupe essas alterações em um único patch. Assim, uma única mudança
+lógica está contida em um único patch.
+
+O ponto a lembrar é que cada patch deve fazer uma mudança facilmente compreendida
+que possa ser verificada pelos revisores. Cada patch deve ser justificável
+por seus próprios méritos.
+
+Se um patch depender de outro patch para que uma mudança seja
+completa, não tem problema. Simplesmente note **"this patch depends on patch X"**
+na descrição do seu patch.
+
+Ao dividir a sua mudança em uma série de patches, tome um cuidado especial para
+garantir que o kernel compile e seja executado adequadamente após cada patch da
+série. Desenvolvedores que usam o ``git bisect`` para rastrear um problema podem acabar
+dividindo a sua série de patches em qualquer ponto; eles não ficarão gratos se você
+introduzir bugs no meio do processo.
+
+Se você não conseguir condensar o seu conjunto de patches em um conjunto menor
+de patches, então publique, digamos, apenas uns 15 de cada vez e aguarde pela
+revisão e integração.
+
+
+
+Verifique o estilo das suas mudanças
+------------------------------------
+
+Verifique o seu patch quanto a violações básicas de estilo, cujos detalhes podem ser
+encontrados em Documentation/process/coding-style.rst.
+Não fazer isso simplesmente desperdiça
+o tempo dos revisores e fará com que o seu patch seja rejeitado, provavelmente
+sem sequer ser lido.
+
+Uma exceção significativa é quando se move código de um arquivo para
+outro -- neste caso você não deve modificar o código movido no
+mesmo patch que o move. Isso delineia claramente o ato de
+mover o código e as suas alterações. Isso ajuda muito a revisão das
+diferenças reais e permite que as ferramentas rastreiem melhor o histórico do
+próprio código.
+
+Verifique os seus patches com o verificador de estilo de patch antes de os submeter
+(scripts/checkpatch.pl). Note, porém, que o verificador de estilo deve ser
+visto como um guia, e não como um substituto para o julgamento humano. Se o seu
+código parecer melhor com uma violação, provavelmente é melhor deixá-lo como está.
+
+O verificador emite relatórios em três níveis:
+ - ERROR: coisas que muito provavelmente estão erradas
+ - WARNING: coisas que requerem uma revisão cuidadosa
+ - CHECK: coisas que requerem reflexão
+
+Você deve ser capaz de justificar todas as violações que permanecerem no seu
+patch.
+
+Selecione os destinatários do seu patch
+---------------------------------------
+
+Você deve sempre copiar o(s) mantenedor(es) e a(s) lista(s) do subsistema
+apropriado(s) em qualquer patch para o código que eles mantêm; dê uma
+olhada no arquivo MAINTAINERS e no histórico de revisão do código-fonte
+para ver quem são esses mantenedores. O script scripts/get_maintainer.pl
+pode ser muito útil nesta etapa (passe os caminhos para seus patches
+como argumentos para scripts/get_maintainer.pl). Se você não conseguir
+encontrar um mantenedor para o subsistema em que está trabalhando,
+Andrew Morton (akpm@linux-foundation.org) serve como um mantenedor de
+último recurso.
+
+linux-kernel@vger.kernel.org deve ser usado por padrão para todos os
+patches, mas o volume dessa lista fez com que vários desenvolvedores a
+ignorassem. Por favor, não envie spam para listas e pessoas não
+relacionadas.
+
+Muitas listas relacionadas ao kernel estão hospedadas em kernel.org;
+você pode encontrar uma lista delas em https://subspace.kernel.org.
+Existem listas relacionadas ao kernel hospedadas em outros lugares
+também, no entanto.
+
+Linus Torvalds é o árbitro final de todas as mudanças aceitas no
+kernel do Linux. Seu endereço de e-mail é <torvalds@linux-foundation.org>.
+Ele recebe muitos e-mails e, neste momento, muito poucos patches passam
+por Linus diretamente, então, normalmente, você deve fazer o seu melhor
+para -evitar- enviar e-mails para ele.
+
+Se você tiver um patch que corrija um bug de segurança explorável,
+envie esse patch para security@kernel.org. Para bugs severos, um
+curto embargo pode ser considerado para permitir que os distribuidores
+disponibilizem o patch aos usuários; em tais casos, obviamente, o
+patch não deve ser enviado a nenhuma lista pública. Veja também
+Documentation/process/security-bugs.rst.
+
+Patches que corrigem um bug severo em um kernel já lançado devem ser
+direcionados aos mantenedores stable (estáveis), colocando uma linha como esta::
+
+  Cc: stable@vger.kernel.org
+
+na área de sign-off do seu patch (note, NÃO como um destinatário de e-mail).
+Você também deve ler Documentation/process/stable-kernel-rules.rst
+além deste documento.
+
+Se as alterações afetarem as interfaces userland-kernel,
+por favor, envie ao mantenedor das MAN-PAGES (como listado no arquivo MAINTAINERS)
+um patch para as páginas de manual, ou pelo menos uma notificação da alteração,
+para que alguma informação chegue às páginas de manual. Mudanças na API
+do espaço de usuário também devem ser copiadas para linux-api@vger.kernel.org.
+
+
+Sem MIME, sem links, sem compressão, sem anexos. Apenas texto puro
+------------------------------------------------------------------
+
+Linus e outros desenvolvedores do kernel precisam ser capazes de ler e
+comentar as mudanças que você está enviando. É importante que um
+desenvolvedor do kernel seja capaz de "citar" suas mudanças, usando
+ferramentas de e-mail padrão, para que eles possam comentar em partes
+específicas do seu código.
+
+Por esse motivo, todos os patches devem ser enviados por e-mail "inline". A
+maneira mais fácil de fazer isso é com ``git send-email``, que é
+fortemente recomendado. Um tutorial interativo para ``git send-email``
+está disponível em https://git-send-email.io.
+
+Se você optar por não usar ``git send-email``:
+
+.. warning::
+
+  Tenha cuidado com a quebra de linha do seu editor corrompendo seu patch,
+  se você optar por recortar e colar o seu patch.
+
+Não anexe o patch como um anexo MIME, comprimido ou não.
+Muitos aplicativos populares de e-mail nem sempre transmitirão um
+anexo MIME como texto puro, tornando impossível comentar o seu
+código. Um anexo MIME também leva um pouco mais de tempo para Linus
+processar, diminuindo a probabilidade da sua alteração anexada em MIME
+ser aceita.
+
+Exceção: Se o seu cliente de e-mail estiver danificando os patches,
+alguém pode pedir que você os reenvie usando MIME.
+
+Veja Documentation/process/email-clients.rst para dicas sobre como
+configurar seu cliente de e-mail para que ele envie seus patches intocados.
+
+Responda aos comentários de revisão
+-----------------------------------
+
+Seu patch quase certamente receberá comentários dos revisores sobre maneiras
+pelas quais o patch pode ser melhorado, na forma de uma resposta ao seu
+e-mail. Você deve responder a esses comentários; ignorar revisores é uma
+boa maneira de ser ignorado em troca. Você pode simplesmente responder aos
+e-mails deles para responder aos seus comentários. Comentários de revisão
+ou perguntas que não levam a uma alteração no código devem quase certamente
+resultar em um comentário ou entrada no changelog para que o próximo
+revisor entenda melhor o que está acontecendo.
+
+Certifique-se de dizer aos revisores quais alterações você está fazendo e
+de agradecê-los pelo tempo dedicado. A revisão de código é um processo
+cansativo e demorado, e os revisores às vezes ficam mal-humorados. Mesmo
+nesse caso, no entanto, responda educadamente e resolva os problemas que
+eles apontaram. Ao enviar uma próxima versão, adicione um ``changelog do patch``
+à carta de apresentação (cover letter) ou aos patches individuais,
+explicando a diferença em relação ao envio anterior (veja
+:ref:`pt_BR_the_canonical_patch_format`).
+Notifique as pessoas que comentaram no seu patch sobre as novas versões
+adicionando-as à lista de CC dos patches.
+
+Veja Documentation/process/email-clients.rst para recomendações sobre
+clientes de e-mail e etiqueta de listas de discussão.
+
+.. _pt_BR_interleaved_replies:
+
+Use respostas intercaladas e aparadas em discussões por e-mail
+--------------------------------------------------------------
+O top-posting (responder no topo) é fortemente desencorajado em
+discussões de desenvolvimento do kernel do Linux. Respostas
+intercaladas (ou "inline") tornam as conversas muito mais fáceis de
+acompanhar. Para mais detalhes, veja:
+https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
+
+Como é frequentemente citado na lista de discussão::
+
+  A: http://en.wikipedia.org/wiki/Top_post
+  Q: Onde encontro informações sobre essa coisa chamada top-posting?
+  A: Porque bagunça a ordem em que as pessoas normalmente leem o texto.
+  Q: Por que o top-posting é algo tão ruim?
+  A: Top-posting.
+  Q: Qual é a coisa mais irritante no e-mail?
+
+Da mesma forma, por favor, apare (corte) todas as citações
+desnecessárias que não são relevantes para a sua resposta. Isso torna
+as respostas mais fáceis de encontrar, e economiza tempo e espaço. Para
+mais detalhes, veja: http://daringfireball.net/2007/07/on_top ::
+
+  A: Não.
+  Q: Devo incluir citações após minha resposta?
+
+.. _pt_BR_resend_reminders:
+
+Não desanime - nem fique impaciente
+-----------------------------------
+
+Depois de ter enviado a sua alteração, seja paciente e espere. Os
+revisores são pessoas ocupadas e podem não chegar ao seu patch
+imediatamente.
+
+Era uma vez, patches costumavam desaparecer no vazio sem comentários,
+mas o processo de desenvolvimento funciona de forma mais suave do que
+isso agora. Você deve receber comentários dentro de algumas semanas
+(normalmente 2-3); se isso não acontecer, certifique-se de que você
+enviou seus patches para o lugar certo. Espere por no mínimo uma
+semana antes de reenviar ou dar um "ping" nos revisores -
+possivelmente mais tempo durante períodos ocupados, como as janelas
+de mesclagem (merge windows).
+
+Também não há problema em reenviar o patch ou a série de patches após
+algumas semanas com a palavra "RESEND" adicionada à linha de Assunto::
+
+   [PATCH Vx RESEND] sub/sys: Resumo condensado do patch
+
+Não adicione "RESEND" quando você estiver enviando uma versão
+modificada do seu patch ou série de patches - "RESEND" se aplica
+apenas ao reenvio de um patch ou série de patches que não foram
+modificados de forma alguma em relação ao envio anterior.
+
+
+Inclua PATCH no Assunto
+-----------------------
+
+Devido ao alto tráfego de e-mails para Linus e para a linux-kernel, é
+uma convenção comum prefixar a sua linha de Assunto com [PATCH]. Isso
+permite que Linus e outros desenvolvedores do kernel distingam
+mais facilmente os patches de outras discussões por e-mail.
+
+O ``git send-email`` fará isso por você automaticamente.
+
+
+Assine seu trabalho - o Certificado de Origem do Desenvolvedor
+--------------------------------------------------------------
+
+Para melhorar o rastreamento de quem fez o que, especialmente com patches
+que podem percolar até o seu local de descanso final no kernel através de
+várias camadas de mantenedores, nós introduzimos um procedimento de
+"sign-off" nos patches que estão sendo enviados por e-mail.
+
+O sign-off é uma linha simples no final da explicação do patch, que
+certifica que você o escreveu ou que de outra forma tem o direito de
+repassá-lo como um patch de código aberto. As regras são bem simples:
+se você pode certificar o seguinte::
+
+        Certificado de Origem do Desenvolvedor 1.1
+
+        Ao fazer uma contribuição para este projeto, eu certifico que:
+
+        (a) A contribuição foi criada no todo ou em parte por mim e eu
+            tenho o direito de enviá-la sob a licença de código aberto
+            indicada no arquivo; ou
+
+        (b) A contribuição baseia-se em trabalho anterior que, até onde eu
+            sei, é coberto por uma licença de código aberto apropriada
+            e eu tenho o direito, sob essa licença, de enviar esse
+            trabalho com modificações, tenham sido criadas no todo ou
+            em parte por mim, sob a mesma licença de código aberto (a menos que eu
+            tenha permissão para enviar sob uma licença diferente), conforme
+            indicado no arquivo; ou
+
+        (c) A contribuição foi fornecida diretamente a mim por alguma outra
+            pessoa que certificou (a), (b) ou (c) e eu não a modifiquei.
+
+        (d) Eu entendo e concordo que este projeto e a contribuição
+            são públicos e que um registro da contribuição (incluindo todas
+            as informações pessoais que eu envio com ela, incluindo meu
+            sign-off) é mantido indefinidamente e pode ser redistribuído de forma
+            consistente com este projeto ou com a(s) licença(s) de código
+            aberto envolvida(s).
+
+então você apenas adiciona uma linha dizendo::
+
+	Signed-off-by: Random J Developer <random@developer.example.org>
+
+usando uma identidade conhecida (desculpe, sem contribuições anônimas.)
+Isso será feito para você automaticamente se você usar o ``git commit -s``.
+As reversões também devem incluir "Signed-off-by". O ``git revert -s``
+faz isso por você.
+
+Algumas pessoas também colocam tags extras no final. Elas serão
+apenas ignoradas por enquanto, mas você pode fazer isso para marcar
+procedimentos internos da empresa ou apenas para apontar algum
+detalhe especial sobre o sign-off.
+
+Quaisquer outros SoBs (Signed-off-by:'s) seguindo o SoB do autor
+são de pessoas que manusearam e transportaram o patch, mas não
+estiveram envolvidas no seu desenvolvimento. As cadeias de SoB devem
+refletir a rota **real** que um patch percorreu à medida que foi
+propagado aos mantenedores e, finalmente, para Linus, com a primeira
+entrada de SoB sinalizando a autoria principal de um único autor.
+
+
+Quando usar Acked-by:, Cc: e Co-developed-by:
+---------------------------------------------
+
+A tag Signed-off-by: indica que o signatário esteve envolvido no
+desenvolvimento do patch, ou que ele/ela estava no caminho de
+entrega do patch.
+
+Se uma pessoa não esteve diretamente envolvida na preparação ou manuseio de um
+patch, mas deseja manifestar e registrar sua aprovação, ela pode
+pedir para ter uma linha Acked-by: adicionada ao changelog do patch.
+
+Acked-by: destina-se a ser usado por aqueles responsáveis ou envolvidos com o
+código afetado de uma forma ou de outra. Mais comumente, o mantenedor quando esse
+mantenedor não contribuiu nem encaminhou o patch.
+
+Acked-by: também pode ser usado por outras partes interessadas, como pessoas com conhecimento de
+domínio (por exemplo, o autor original do código sendo modificado), revisores
+do lado do espaço de usuário para um patch uAPI do kernel ou usuários-chave de um recurso. Opcionalmente,
+nestes casos, pode ser útil adicionar um "# Sufixo" para esclarecer seu significado::
+
+	Acked-by: The Stakeholder <stakeholder@example.org> # As primary user
+
+Acked-by: não é tão formal quanto Signed-off-by:. É um registro de que o avaliador
+pelo menos revisou o patch e indicou aceitação. Por isso, os responsáveis pela fusão de
+patches às vezes converterão manualmente um "sim, parece bom para mim" de um avaliador
+em um Acked-by: (mas note que geralmente é melhor pedir um
+ack explícito).
+
+Acked-by: também é menos formal do que Reviewed-by:. Por exemplo, mantenedores podem
+usá-lo para sinalizar que estão de acordo com a inclusão de um patch, mas podem não tê-lo
+revisado tão minuciosamente como se um Reviewed-by: fosse fornecido. Da mesma forma, um
+usuário-chave pode não ter realizado uma revisão técnica do patch, mas ainda assim estar
+satisfeito com a abordagem geral, o recurso ou a interface voltada para o usuário.
+
+Acked-by: não indica necessariamente o reconhecimento de todo o patch.
+Por exemplo, se um patch afeta vários subsistemas e tem um Acked-by: de
+um mantenedor de subsistema, isso geralmente indica o reconhecimento apenas
+da parte que afeta o código desse mantenedor. O bom senso deve ser usado aqui.
+Em caso de dúvida, as pessoas devem consultar a discussão original nos arquivos da
+lista de discussão. Um "# Sufixo" também pode ser usado neste caso para esclarecer.
+
+Se uma pessoa teve a oportunidade de comentar em um patch, mas não
+forneceu tais comentários, você pode opcionalmente adicionar uma tag ``Cc:`` ao patch.
+Esta tag documenta que partes potencialmente interessadas foram incluídas na
+discussão. Note que esta é uma de apenas três tags que você pode usar
+sem a permissão explícita da pessoa nomeada (veja 'Marcar pessoas requer
+permissão' abaixo para detalhes).
+
+Co-developed-by: afirma que o patch foi co-criado por múltiplos desenvolvedores;
+é usado para dar atribuição a coautores (além do autor
+atribuído pela tag From:) quando várias pessoas trabalham em um único patch. Como
+Co-developed-by: denota autoria, cada Co-developed-by: deve ser imediatamente
+seguido por um Signed-off-by: do coautor associado. O procedimento padrão de assinatura
+se aplica, ou seja, a ordem das tags Signed-off-by: deve refletir a
+história cronológica do patch na medida do possível, independentemente se
+o autor for atribuído via From: ou Co-developed-by:. Notavelmente, o último
+Signed-off-by: deve ser sempre o do desenvolvedor que está enviando o patch.
+
+Note que a tag From: é opcional quando o autor no From: também é a pessoa (e
+e-mail) listada na linha From: do cabeçalho do e-mail.
+
+Exemplo de um patch enviado pelo autor do From:::
+
+	<changelog>
+
+	Co-developed-by: First Co-Author <first@coauthor.example.org>
+	Signed-off-by: First Co-Author <first@coauthor.example.org>
+	Co-developed-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+
+Exemplo de um patch enviado por um autor do Co-developed-by:::
+
+	From: From Author <from@author.example.org>
+
+	<changelog>
+
+	Co-developed-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+	Co-developed-by: Submitting Co-Author <sub@coauthor.example.org>
+	Signed-off-by: Submitting Co-Author <sub@coauthor.example.org>
+
+
+Usando Reported-by:, Tested-by:, Reviewed-by:, Suggested-by: e Fixes:
+---------------------------------------------------------------------
+
+A tag Reported-by dá crédito às pessoas que encontram bugs e os relatam e
+espera-se que isso as inspire a nos ajudar novamente no futuro. A tag destina-se a
+bugs; por favor, não a use para dar crédito a solicitações de recursos. A tag deve ser
+seguida por uma tag Closes: apontando para o relato, a menos que o relato não
+esteja disponível na web. A tag Link: pode ser usada em vez de Closes: se o patch
+corrigir uma parte do(s) problema(s) sendo relatado(s). Note que a tag Reported-by é
+uma de apenas três tags que você pode usar sem a permissão explícita da
+pessoa nomeada (veja 'Marcar pessoas requer permissão' abaixo para detalhes).
+
+Uma tag Tested-by: indica que o patch foi testado com sucesso (em
+algum ambiente) pela pessoa nomeada. Esta tag informa aos mantenedores que
+algum teste foi realizado, fornece um meio para localizar testadores para
+patches futuros e garante crédito para os testadores.
+
+Reviewed-by:, por sua vez, indica que o patch foi revisado e considerado
+aceitável de acordo com a Declaração do Revisor::
+
+	Declaração de supervisão do revisor
+
+	Ao oferecer minha tag Reviewed-by:, eu declaro que:
+
+	 (a) Eu realizei uma revisão técnica deste patch para
+	     avaliar sua adequação e prontidão para inclusão no
+	     kernel mainline.
+
+	 (b) Quaisquer problemas, preocupações ou perguntas relacionadas ao patch
+	     foram comunicadas de volta ao remetente. Eu estou satisfeito
+	     com a resposta do remetente aos meus comentários.
+
+	 (c) Embora possa haver coisas que poderiam ser melhoradas com este
+	     envio, eu acredito que é, neste momento, (1) uma
+	     modificação que vale a pena para o kernel, e (2) livre de problemas
+	     conhecidos que argumentariam contra sua inclusão.
+
+	 (d) Embora eu tenha revisado o patch e acredite que seja sólido, eu
+	     não faço (a menos que explicitamente declarado em outro lugar)
+	     garantias de que alcançará seu propósito
+	     declarado ou funcionará adequadamente em qualquer situação.
+
+Uma tag Reviewed-by é uma declaração de opinião de que o patch é uma
+modificação apropriada do kernel sem nenhum problema técnico sério
+restante. Qualquer revisor interessado (que tenha feito o trabalho e seja uma
+pessoa com identidade conhecida) pode oferecer uma tag Reviewed-by para um patch. Esta tag
+serve para dar crédito aos revisores e para informar os mantenedores do grau de
+revisão que foi feito no patch. Tags Reviewed-by:, quando fornecidas por
+revisores conhecidos por entender a área de assunto e realizar revisões completas,
+normalmente aumentarão a probabilidade de seu patch entrar no kernel.
+
+Ambas as tags Tested-by e Reviewed-by, uma vez recebidas na lista de discussão do testador
+ou revisor, devem ser adicionadas pelo autor aos patches aplicáveis ao enviar as
+próximas versões. No entanto, se o patch mudou substancialmente na versão
+seguinte, essas tags podem não ser mais aplicáveis e, portanto, devem ser removidas.
+Normalmente, a remoção das tags Acked-by, Tested-by ou Reviewed-by de alguém deve ser
+mencionada no changelog do patch com uma explicação (após o separador '---').
+
+Uma tag Suggested-by: indica que a ideia do patch foi sugerida pela pessoa
+nomeada e garante crédito à pessoa pela ideia: se creditarmos diligentemente
+nossos relatores de ideias, eles serão, com sorte, inspirados a nos ajudar novamente no
+futuro. Note que esta é uma de apenas três tags que você pode usar sem
+permissão explícita da pessoa nomeada (veja 'Marcar pessoas requer
+permissão' abaixo para detalhes).
+
+Uma tag Fixes: indica que o patch corrige um bug em um commit anterior. Ela
+é usada para facilitar a determinação de onde um problema se originou, o que pode ajudar
+na revisão da correção de um bug. Esta tag também auxilia a equipe do kernel estável a determinar
+quais versões do kernel estável devem receber sua correção. Este é o método preferido
+para indicar um bug corrigido pelo patch. Veja :ref:`pt_BR_describe_changes`
+para mais detalhes.
+
+Nota: Anexar uma tag Fixes: não subverte o processo de regras do kernel
+estável, nem o requisito de enviar em Cc: para stable@vger.kernel.org em todos os patches
+candidatos estáveis. Para mais informações, por favor, leia
+Documentation/process/stable-kernel-rules.rst.
+
+Por fim, embora fornecer tags seja bem-vindo e tipicamente muito apreciado, por favor
+note que os signatários (ou seja, remetentes e mantenedores) podem usar sua discrição ao
+aplicar as tags oferecidas.
+
+
+Marcar pessoas requer permissão
+-------------------------------
+
+Tenha cuidado ao adicionar as tags mencionadas acima aos seus patches, pois todas
+exceto Cc:, Reported-by: e Suggested-by: precisam de permissão explícita da
+pessoa nomeada. Para essas três, a permissão implícita é suficiente se a pessoa
+contribuiu para o kernel Linux usando esse nome e endereço de e-mail de acordo
+com os arquivos do lore ou o histórico de commits -- e no caso de Reported-by:
+e Suggested-by: tenha feito o relato ou sugestão em público. Note que o
+bugzilla.kernel.org é um local público nesse sentido, mas os endereços de e-mail
+usados lá são privados; portanto, não os exponha em tags, a menos que a pessoa
+os tenha usado em contribuições anteriores.
+
+Usando Assisted-by:
+-------------------
+
+Se você usou qualquer tipo de ferramenta avançada de codificação na criação do seu patch,
+você precisa reconhecer esse uso adicionando uma tag Assisted-by. A falha em
+fazer isso pode impedir a aceitação do seu trabalho. Por favor, veja
+Documentation/process/coding-assistants.rst para detalhes sobre o
+reconhecimento de assistentes de codificação.
+
+
+.. _pt_BR_the_canonical_patch_format:
+
+O formato canônico do patch
+---------------------------
+
+Esta seção descreve como o próprio patch deve ser formatado. Note
+que, se você tiver seus patches armazenados em um repositório ``git``, a formatação
+adequada do patch pode ser obtida com ``git format-patch``. As ferramentas não podem criar
+o texto necessário, no entanto, portanto, leia as instruções abaixo de qualquer maneira.
+
+Linha de Assunto
+^^^^^^^^^^^^^^^^
+
+A linha de assunto canônica do patch é::
+
+    Assunto: [PATCH 001/123] subsistema: frase de resumo
+
+O corpo canônico da mensagem do patch contém o seguinte:
+
+  - Uma linha ``from`` especificando o autor do patch, seguida por uma linha
+    vazia (necessário apenas se a pessoa enviando o patch não for o autor).
+
+  - O corpo da explicação, com quebra de linha em 75 colunas, que será
+    copiado para o changelog permanente para descrever este patch.
+
+  - Uma linha vazia.
+
+  - As linhas ``Signed-off-by:``, descritas acima, que também
+    irão para o changelog.
+
+  - Uma linha de marcador contendo simplesmente ``---``.
+
+  - Quaisquer comentários adicionais não adequados para o changelog.
+
+  - O próprio patch (saída do ``diff``).
+
+O formato da linha de Assunto torna muito fácil classificar os e-mails
+alfabeticamente pela linha de assunto - praticamente qualquer leitor de e-mail
+suportará isso - pois, como o número de sequência é preenchido com zeros,
+a classificação numérica e alfabética é a mesma.
+
+O ``subsystem`` no Assunto do e-mail deve identificar qual
+área ou subsistema do kernel está recebendo o patch.
+
+A ``frase de resumo`` no Assunto do e-mail deve descrever de forma concisa
+o patch que esse e-mail contém. A ``frase de resumo`` não deve ser um nome de arquivo.
+Não use a mesma ``frase de resumo`` para cada patch em uma série de patches inteira (onde uma ``série
+de patches`` é uma sequência ordenada de múltiplos patches relacionados).
+
+Tenha em mente que a ``frase de resumo`` do seu e-mail se torna um
+identificador globalmente único para aquele patch. Ela se propaga por todo o caminho
+até o changelog do ``git``. A ``frase de resumo`` pode ser usada posteriormente em
+discussões de desenvolvedores que se referem ao patch. As pessoas vão querer
+pesquisar no Google pela ``frase de resumo`` para ler a discussão sobre esse
+patch. Também será a única coisa que as pessoas poderão ver rapidamente
+quando, dois ou três meses depois, estiverem passando por talvez
+milhares de patches usando ferramentas como ``gitk`` ou ``git log
+--oneline``.
+
+Por essas razões, o ``resumo`` não deve ter mais de 70-75
+caracteres, e deve descrever tanto o que o patch altera, quanto
+por que o patch pode ser necessário. É um desafio ser
+sucinto e descritivo, mas é isso que um resumo bem escrito
+deve fazer.
+
+A ``frase de resumo`` pode ser prefixada por tags delimitadas por colchetes
+retos: "Assunto: [PATCH <tag>...] <frase de resumo>". As tags não
+são consideradas parte da frase de resumo, mas descrevem como o patch
+deve ser tratado. Tags comuns podem incluir um descritor de versão se
+as múltiplas versões do patch tiverem sido enviadas em resposta a
+comentários (ou seja, "v1, v2, v3"), ou "RFC" para indicar um pedido de
+comentários.
+
+Se houver quatro patches em uma série de patches, os patches individuais podem
+ser numerados assim: 1/4, 2/4, 3/4, 4/4. Isso garante que os desenvolvedores
+entendam a ordem na qual os patches devem ser aplicados e que
+eles tenham revisado ou aplicado todos os patches na série de patches.
+
+Aqui estão alguns bons exemplos de Assuntos::
+
+    Subject: [PATCH 2/5] ext2: improve scalability of bitmap searching
+    Subject: [PATCH v2 01/27] x86: fix eflags tracking
+    Subject: [PATCH v2] sub/sys: Condensed patch summary
+    Subject: [PATCH v2 M/N] sub/sys: Condensed patch summary
+
+Linha From
+^^^^^^^^^^
+
+A linha ``from`` deve ser a primeira linha no corpo da mensagem,
+e tem a forma:
+
+        From: Patch Author <author@example.com>
+
+A linha ``from`` especifica quem será creditado como o autor do
+patch no changelog permanente. Se a linha ``from`` estiver faltando,
+então a linha ``From:`` do cabeçalho do e-mail será usada para determinar
+o autor do patch no changelog.
+
+O autor pode indicar sua afiliação ou o patrocinador do trabalho
+adicionando o nome de uma organização às linhas ``from`` e ``SoB``,
+por exemplo:
+
+	From: Patch Author (Company) <author@example.com>
+
+Corpo da Explicação
+^^^^^^^^^^^^^^^^^^^
+
+O corpo da explicação será commitado no changelog
+permanente da fonte, então deve fazer sentido para um leitor competente que já
+esqueceu há muito tempo os detalhes imediatos da discussão que podem ter levado a
+este patch. Incluir sintomas da falha que o patch aborda
+(mensagens de log do kernel, mensagens oops, etc.) é especialmente útil para
+pessoas que possam estar pesquisando nas mensagens de commit procurando pelo patch
+aplicável. O texto deve ser escrito com detalhes suficientes para que, quando lido
+semanas, meses ou até anos depois, possa dar ao leitor os detalhes
+necessários para compreender o raciocínio do **por que** o patch foi criado.
+
+Se um patch corrige uma falha de compilação, pode não ser necessário incluir
+_todas_ as falhas de compilação; apenas o suficiente para que seja provável que
+alguém pesquisando pelo patch possa encontrá-lo. Como na ``frase de resumo``,
+é importante ser tanto sucinto quanto descritivo.
+
+.. _pt_BR_backtraces:
+
+Backtraces em mensagens de commit
+"""""""""""""""""""""""""""""""""
+
+Backtraces ajudam a documentar a cadeia de chamadas que leva a um problema. No entanto,
+nem todos os backtraces são úteis. Por exemplo, as cadeias de chamadas iniciais de boot são
+únicas e óbvias. Copiar a saída dmesg completa verbatim, no entanto,
+adiciona informações que distraem, como timestamps, listas de módulos, dumps de
+registradores e pilhas.
+
+Portanto, os backtraces mais úteis devem destilar as informações
+relevantes do dump, o que facilita o foco no problema
+real. Aqui está um exemplo de um backtrace bem aparado::
+
+  unchecked MSR access error: WRMSR to 0xd51 (tried to write 0x0000000000000064)
+  at rIP: 0xffffffffae059994 (native_write_msr+0x4/0x20)
+  Call Trace:
+  mba_wrmsr
+  update_domains
+  rdtgroup_mkdir
+
+Comentários
+^^^^^^^^^^^
+
+A linha marcadora ``---`` serve ao propósito essencial de marcar para
+as ferramentas de manipulação de patches onde a mensagem do changelog termina.
+
+Um bom uso para os comentários adicionais após o marcador ``---`` é
+para um ``diffstat``, para mostrar quais arquivos mudaram, e o número de
+linhas inseridas e excluídas por arquivo. Um ``diffstat`` é especialmente útil
+em patches maiores. Se você for incluir um ``diffstat`` após o
+marcador ``---``, por favor, use as opções do ``diffstat`` ``-p 1 -w 70`` para que
+os nomes dos arquivos sejam listados a partir do topo da árvore de código-fonte do kernel e não
+usem muito espaço horizontal (cabem facilmente em 80 colunas, talvez com algum
+recuo). (o ``git`` gera diffstats apropriados por padrão.)
+
+Outros comentários relevantes apenas para o momento ou para o mantenedor, não
+adequados para o changelog permanente, também devem ir aqui. Um bom
+exemplo de tais comentários podem ser ``changelogs do patch`` que descrevem
+o que mudou entre as versões v1 e v2 do patch.
+
+Por favor, coloque esta informação **após** a linha ``---`` que separa
+o changelog do restante do patch. A informação da versão não
+faz parte do changelog que é commitado na árvore git. É
+informação adicional para os revisores. Se for colocada acima das
+tags de commit, precisará de interação manual para removê-la. Se estiver abaixo
+da linha separadora, ela é automaticamente removida ao aplicar o
+patch. Se disponíveis, adicionar links para as versões anteriores do patch (por exemplo,
+link do arquivo lore.kernel.org) é recomendado para ajudar os revisores::
+
+  <commit message>
+  ...
+  Signed-off-by: Author <author@mail>
+  ---
+  V2 -> V3: Removed redundant helper function
+  V1 -> V2: Cleaned up coding style and addressed review comments
+
+  v2: https://lore.kernel.org/bar
+  v1: https://lore.kernel.org/foo
+
+  path/to/file | 5+++--
+  ...
+
+Veja mais detalhes sobre o formato de patch adequado nas seguintes
+referências.
+
+
+Cabeçalhos In-Reply-To explícitos
+---------------------------------
+
+Pode ser útil adicionar manualmente cabeçalhos In-Reply-To: a um patch
+(por exemplo, ao usar ``git send-email``) para associar o patch com
+discussões relevantes anteriores, por exemplo, para vincular uma correção de bug ao e-mail com
+o relatório do bug. No entanto, para uma série de múltiplos patches, geralmente é
+melhor evitar usar In-Reply-To: para vincular a versões mais antigas da
+série. Desta forma, múltiplas versões do patch não se tornam uma
+floresta incontrolável de referências nos clientes de e-mail. Se um link for
+útil, você pode usar o redirecionador https://lore.kernel.org/ (por exemplo, no
+texto do e-mail de capa) para vincular a uma versão anterior da série de patches.
+
+
+Informações sobre a árvore base
+-------------------------------
+
+Quando outros desenvolvedores recebem seus patches e iniciam o processo de revisão,
+é absolutamente necessário que eles saibam qual é o commit/branch
+base no qual seu trabalho se aplica, considerando a enorme quantidade de
+árvores de mantenedores presentes hoje em dia. Note novamente a entrada **T:** no
+arquivo MAINTAINERS explicado acima.
+
+Isso é ainda mais importante para processos automatizados de CI que tentam
+executar uma série de testes a fim de estabelecer a qualidade da sua
+submissão antes que o mantenedor inicie a revisão.
+
+Se você estiver usando ``git format-patch`` para gerar seus patches, você pode
+incluir automaticamente as informações da árvore base em sua submissão ao
+usar a flag ``--base``. A maneira mais fácil e conveniente de usar
+esta opção é com branches de tópicos (topical branches)::
+
+    $ git checkout -t -b my-topical-branch master
+    Branch 'my-topical-branch' set up to track local branch 'master'.
+    Switched to a new branch 'my-topical-branch'
+
+    [perform your edits and commits]
+
+    $ git format-patch --base=auto --cover-letter -o outgoing/ master
+    outgoing/0000-cover-letter.patch
+    outgoing/0001-First-Commit.patch
+    outgoing/...
+
+Quando você abrir ``outgoing/0000-cover-letter.patch`` para edição, você
+notará que ele terá o trailer ``base-commit:`` bem no
+final, o qual fornece ao revisor e às ferramentas de CI informações suficientes
+para realizar o ``git am`` adequadamente sem se preocupar com conflitos::
+
+    $ git checkout -b patch-review [base-commit-id]
+    Switched to a new branch 'patch-review'
+    $ git am patches.mbox
+    Applying: First Commit
+    Applying: ...
+
+Por favor, veja ``man git-format-patch`` para mais informações sobre esta
+opção.
+
+.. note::
+
+    A funcionalidade ``--base`` foi introduzida no git versão 2.9.0.
+
+Se você não estiver usando git para formatar seus patches, você ainda pode incluir
+o mesmo trailer ``base-commit`` para indicar o hash do commit da árvore
+na qual seu trabalho se baseia. Você deve adicioná-lo na cover
+letter (carta de apresentação) ou no primeiro patch da série e ele deve ser colocado
+abaixo da linha ``---`` ou bem no final de todo o outro
+conteúdo, logo antes da sua assinatura de e-mail.
+
+Certifique-se de que o commit base está em uma árvore oficial de mantenedor/mainline
+e não em alguma árvore interna acessível apenas por você - caso contrário seria
+inútil.
+
+Ferramentas
+-----------
+
+Muitos dos aspectos técnicos deste processo podem ser automatizados usando
+b4, documentado em <https://b4.docs.kernel.org/en/latest/>. Isso pode
+ajudar com coisas como rastreamento de dependências, execução do checkpatch e
+com a formatação e o envio de e-mails.
+
+Referências
+-----------
+
+Andrew Morton, "The perfect patch" (tpp).
+  <https://www.ozlabs.org/~akpm/stuff/tpp.txt>
+
+Jeff Garzik, "Linux kernel patch submission format".
+  <https://web.archive.org/web/20180829112450/http://linux.yyz.us/patch-format.html>
+
+Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer".
+  <http://www.kroah.com/log/linux/maintainer.html>
+
+  <http://www.kroah.com/log/linux/maintainer-02.html>
+
+  <http://www.kroah.com/log/linux/maintainer-03.html>
+
+  <http://www.kroah.com/log/linux/maintainer-04.html>
+
+  <http://www.kroah.com/log/linux/maintainer-05.html>
+
+  <http://www.kroah.com/log/linux/maintainer-06.html>
+
+Kernel Documentation/process/coding-style.rst
+
+Linus Torvalds's mail on the canonical patch format:
+  <https://lore.kernel.org/r/Pine.LNX.4.58.0504071023190.28951@ppc970.osdl.org>
+
+Andi Kleen, "On submitting kernel patches"
+  Some strategies to get difficult or controversial changes in.
+
+  http://halobates.de/on-submitting-patches.pdf
diff --git a/Documentation/translations/pt_BR/process/volatile-considered-harmful.rst b/Documentation/translations/pt_BR/process/volatile-considered-harmful.rst
new file mode 100644
index 00000000000000..b8774c0bae77f8
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/volatile-considered-harmful.rst
@@ -0,0 +1,130 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Por que a classe de tipo "volatile" não deve ser usada
+--------------------------------------------------------
+
+Programadores C frequentemente interpretam volatile como uma indicação de
+que uma variável pode ser alterada fora da thread de execução atual; como
+resultado, às vezes são tentados a usá-la no código do kernel quando
+estruturas de dados compartilhadas estão sendo utilizadas. Em outras
+palavras, há quem trate tipos volatile como uma espécie de variável
+atômica simplificada, mas não são. O uso de volatile no código do
+kernel quase nunca é correto; este documento explica o porquê.
+
+O ponto-chave a entender sobre volatile é que seu propósito é suprimir
+otimizações, o que quase nunca é o que realmente se deseja fazer. No kernel,
+é necessário proteger estruturas de dados compartilhadas contra acessos
+concorrentes indesejados, o que é uma tarefa bastante diferente. O processo
+de proteção contra concorrência indesejada também evita, de forma mais
+eficiente, quase todos os problemas relacionados a otimizações.
+
+Assim como volatile, as primitivas do kernel que tornam seguro o acesso
+concorrente a dados (spinlocks, mutexes, barreiras de memória etc.) são
+projetadas para evitar otimizações indesejadas. Se forem usadas
+corretamente, também não haverá necessidade de usar volatile. Se
+volatile ainda for necessário, quase certamente há algum bug no código.
+Em código de kernel corretamente escrito, volatile só serve para deixar
+as coisas mais lentas.
+
+Considere um bloco típico de código do kernel::
+
+    spin_lock(&the_lock);
+    do_something_on(&shared_data);
+    do_something_else_with(&shared_data);
+    spin_unlock(&the_lock);
+
+Se todo o código seguir as regras de bloqueio, o valor de shared_data não
+poderá mudar inesperadamente enquanto the_lock estiver mantido. Qualquer
+outro código que queira manipular esses dados estará aguardando o bloqueio.
+As primitivas de spinlock atuam como barreiras de memória (são escritas
+explicitamente para isso), o que significa que os acessos aos dados não
+serão otimizados de forma a atravessar essas barreiras. Assim, o compilador
+até pode achar que sabe qual será o valor de shared_data, mas a chamada a
+spin_lock(), por atuar como barreira de memória, o forçará a esquecer tudo
+o que sabia. Não haverá problemas de otimização nos acessos a esses dados.
+
+Se shared_data fosse declarada volatile, o bloqueio ainda seria
+necessário. No entanto, o compilador também seria impedido de otimizar o
+acesso a shared_data _dentro_ da seção crítica, justamente quando sabemos
+que ninguém mais pode estar manipulando esses dados. Enquanto o bloqueio
+estiver mantido, shared_data não é volatile. Ao lidar com dados
+compartilhados, um bloqueio adequado torna volatile desnecessário e
+potencialmente prejudicial.
+
+A classe de armazenamento volatile foi originalmente concebida para
+registradores de E/S mapeados em memória. No kernel, os acessos a esses
+registradores também devem ser protegidos por bloqueios, mas também não se
+deseja que o compilador "otimize" esses acessos dentro de uma seção crítica.
+Contudo, no kernel, os acessos à memória de E/S são sempre feitos por meio de
+funções de acesso; acessar diretamente a memória de E/S via ponteiros é
+desencorajado e não funciona em todas as arquiteturas. Essas funções de
+acesso são implementadas de modo a impedir otimizações indesejadas e,
+portanto, mais uma vez, volatile é desnecessário.
+
+Outra situação em que se pode ser tentado a usar volatile é quando o
+processador fica em espera ocupada pelo valor de uma variável. A forma
+correta de realizar essa espera ocupada é::
+
+    while (my_variable != what_i_want)
+        cpu_relax();
+
+A chamada a cpu_relax() pode reduzir o consumo de energia da CPU ou ceder
+recursos a um processador lógico gêmeo hyperthreaded; ela também atua
+como barreira para o compilador e, portanto, mais uma vez, volatile é
+desnecessário. Naturalmente, a espera ocupada já é, por si só, uma prática
+geralmente antissocial.
+
+Ainda existem algumas situações raras em que volatile faz sentido no
+kernel:
+
+  - As funções de acesso mencionadas acima podem usar volatile em
+    arquiteturas nas quais o acesso direto à memória de E/S funciona.
+    Essencialmente, cada chamada a uma função de acesso torna-se uma
+    pequena seção crítica por si só e garante que o acesso ocorra conforme
+    esperado pelo programador.
+  - Código assembly inline que modifica memória, mas não tem outros
+    efeitos colaterais visíveis, corre o risco de ser removido pelo GCC.
+    Adicionar a palavra-chave volatile às instruções asm impede essa
+    remoção.
+  - A variável jiffies é especial, pois pode ter um valor diferente a cada
+    vez que é referenciada, mas pode ser lida sem qualquer bloqueio
+    especial. Portanto, jiffies pode ser volatile, mas a adição de
+    outras variáveis desse tipo é fortemente desencorajada. Nesse
+    sentido, jiffies é considerada um problema de "legado idiota" (nas
+    palavras de Linus); corrigi-la daria mais trabalho do que valeria a
+    pena.
+  - Ponteiros para estruturas de dados em memória coerente que possam ser
+    modificadas por dispositivos de E/S podem, às vezes, ser
+    legitimamente volatile. Um buffer circular usado por um adaptador
+    de rede, no qual esse adaptador altera ponteiros para indicar quais
+    descritores já foram processados, é um exemplo desse tipo de
+    situação.
+
+Na maior parte do código, nenhuma das justificativas acima para o uso de
+volatile se aplica. Como resultado, o uso de volatile provavelmente
+será considerado um bug e fará com que o código seja submetido a uma análise
+mais rigorosa. Desenvolvedores que se sintam tentados a usar volatile devem
+dar um passo atrás e pensar no que realmente estão tentando alcançar.
+
+Patches para remover variáveis volatile são, em geral, bem-vindos, desde
+que venham acompanhados de uma justificativa que demonstre que as questões
+de concorrência foram devidamente analisadas.
+
+
+Referências
+===========
+
+[1] https://lwn.net/Articles/233481/
+
+[2] https://lwn.net/Articles/233482/
+
+Créditos
+========
+
+Motivação original e pesquisa por Randy Dunlap
+
+Escrito por Jonathan Corbet
+
+Melhorias a partir de comentários de Satyam Sharma, Johannes Stezenbach,
+Jesper Juhl, Heikki Orsila, H. Peter Anvin, Philipp Hahn e Stefan
+Richter.
diff --git a/Documentation/translations/zh_CN/admin-guide/README.rst b/Documentation/translations/zh_CN/admin-guide/README.rst
index 7c2ffe7e87c7e6..ee6dd3782f4a33 100644
--- a/Documentation/translations/zh_CN/admin-guide/README.rst
+++ b/Documentation/translations/zh_CN/admin-guide/README.rst
@@ -83,7 +83,7 @@ Linux内核6.x版本 <http://kernel.org/>
    补丁，则不应先应用6.0.1和6.0.2的补丁。类似地，如果您运行的是6.0.2内核，
    并且希望跳转到6.0.3，那么在应用6.0.3补丁之前，必须首先撤销6.0.2补丁
    （即patch -R）。更多关于这方面的内容，请阅读
-   :ref:`Documentation/process/applying-patches.rst <applying_patches>` 。
+   Documentation/translations/zh_CN/process/applying-patches.rst 。
 
    或者，脚本 patch-kernel 可以用来自动化这个过程。它能确定当前内核版本并
    应用找到的所有补丁::
diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst
index 6f8676a50b3811..77bbfa7c25dcb2 100644
--- a/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst
+++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/index.rst
@@ -9,12 +9,12 @@
 
 :校译:
 
-============
-监测数据访问
-============
+========================================
+DAMON：数据访问监测和访问感知系统操作
+========================================
 
-:doc:`DAMON </mm/damon/index>` 允许轻量级的数据访问监测。使用DAMON，
-用户可以分析他们系统的内存访问模式，并优化它们。
+:doc:`DAMON <../../../mm/damon/index>` 是一个 Linux 内核子系统，用于高效的
+数据访问监测和访问感知系统操作。
 
 .. toctree::
    :maxdepth: 2
@@ -23,7 +23,4 @@
    usage
    reclaim
    lru_sort
-
-
-
-
+   stat
diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/lru_sort.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/lru_sort.rst
index 03d33c710604c9..6af0dd59ef5db6 100644
--- a/Documentation/translations/zh_CN/admin-guide/mm/damon/lru_sort.rst
+++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/lru_sort.rst
@@ -48,7 +48,7 @@ DAMON_LRU_SORT使用DAMON寻找热页（范围内的页面访问频率高于用
 
 为了让系统管理员打开或者关闭并且调节指定的系统，DAMON_LRU_SORT设计了模块参数。
 这意味着，你可以添加 ``damon_lru_sort.<parameter>=<value>`` 到内核的启动命令行
-参数，或者在 ``/sys/modules/damon_lru_sort/parameters/<parameter>`` 写入正确的
+参数，或者在 ``/sys/module/damon_lru_sort/parameters/<parameter>`` 写入正确的
 值。
 
 下边是每个参数的描述
@@ -71,6 +71,41 @@ commit_inputs
 ，DAMON_LRU_SORT会再次读取除了 ``enabled`` 之外的参数。读取完成后，这个参数会被
 设置为 ``N`` 。如果在读取时发现有无效参数，DAMON_LRU_SORT会被关闭。
 
+一旦向该参数写入 ``Y``，用户在再次读取 ``commit_inputs`` 返回 ``N`` 之前，
+不得写入任何参数。如果用户违反该规则，内核可能表现出未定义行为。
+
+active_mem_bp
+-------------
+
+期望的活跃内存与[非]活跃内存比率，单位为 bp（1/10,000）。
+
+在保持其他配额设置的上限的同时，DAMON_LRU_SORT 会自动增减配额的有效水平，
+目标是让热/冷内存的 LRU [降低]优先级处理产生该活跃内存与[非]活跃内存比率。
+值为零表示禁用该自动调优功能。
+
+默认禁用。
+
+autotune_monitoring_intervals
+-----------------------------
+
+如果该参数设置为 ``Y``，DAMON_LRU_SORT 会自动调优 DAMON 的采样和聚集间隔。
+自动调优的目标是在每个 DAMON 快照中捕获有意义数量的访问事件，同时将采样
+间隔限制在最小 5 毫秒、最大 10 秒。将其设置为 ``N`` 会禁用自动调优。
+
+默认禁用。
+
+filter_young_pages
+------------------
+
+相应地为 LRU [降低]优先级处理过滤[非]年轻页。
+
+如果设置该参数，则在每次 LRU [降低]优先级处理操作前再次检查页级访问
+（年轻性）。如果该页自上次检查以来未被访问（不年轻），则跳过 LRU 提高
+优先级操作。如果该页自上次检查以来已被访问（年轻），则跳过 LRU 降低优先级
+操作。当该参数分别设置为 ``Y`` 或 ``N`` 时，会启用或禁用该功能。
+
+默认禁用。
+
 hot_thres_access_freq
 ---------------------
 
@@ -163,6 +198,8 @@ min_nr_regions
 对冷内存区域监测的最小数量。这个值可以作为监测质量的下限。不过，这个值设置的过
 大会增加开销。更多细节请参考DAMON文档 (:doc:`usage`) 。默认值为10。
 
+请注意，该值必须为 3 或更高。该下限的理由请参考设计文档的 :ref:`监测 <damon_design_monitoring_zh_CN>` 章节。
+
 max_nr_regions
 --------------
 
@@ -176,14 +213,34 @@ monitor_region_start
 
 目标内存区域的起始物理地址。
 
-DAMON_LRU_SORT要处理的目标内存区域的起始物理地址。默认，使用系统最大内存。
+DAMON_LRU_SORT要处理的目标内存区域的起始物理地址。默认，使用系统的整个物理内存。
 
 monitor_region_end
 ------------------
 
 目标内存区域的结束物理地址。
 
-DAMON_LRU_SORT要处理的目标内存区域的结束物理地址。默认，使用系统最大内存。
+DAMON_LRU_SORT要处理的目标内存区域的结束物理地址。默认，使用系统的整个物理内存。
+
+addr_unit
+---------
+
+内存地址和字节数的缩放因子。
+
+该参数用于设置和获取 DAMON_RECLAIM 的 DAMON 实例的 :ref:`地址单位 <damon_design_addr_unit_zh_CN>` 参数。
+
+``monitor_region_start`` 和 ``monitor_region_end`` 应以该单位提供。例如，
+假设 ``addr_unit``、``monitor_region_start`` 和 ``monitor_region_end``
+分别设置为 ``1024``、``0`` 和 ``10``。那么 DAMON_LRU_SORT 将处理从地址零
+开始、长度为 10 KiB 的物理地址范围（以字节表示为
+``[0 * 1024, 10 * 1024)``）。
+
+带有 ``bytes_`` 前缀的统计参数也使用该单位。例如，假设 ``addr_unit``、
+``bytes_lru_sort_tried_hot_regions`` 和 ``bytes_lru_sorted_hot_regions`` 的值
+分别为 ``1024``、``42`` 和 ``32``。那么这表示 DAMON_LRU_SORT 尝试对
+42 KiB 热内存进行 LRU 排序，并总共成功对其中 32 KiB 内存进行了 LRU 排序。
+
+如果不确定，只使用默认值（``1``）并忘记这个参数即可。
 
 kdamond_pid
 -----------
@@ -252,7 +309,7 @@ LRU的优先级的提升，同时降低那些超过120秒无人访问的内存
 进展且空闲内存低于20%，再次让DAMON_LRU_SORT停止工作，以此回退到以LRU链表为基础
 以页面为单位的内存回收上。 ::
 
-    # cd /sys/modules/damon_lru_sort/parameters
+    # cd /sys/module/damon_lru_sort/parameters
     # echo 500 > hot_thres_access_freq
     # echo 120000000 > cold_min_age
     # echo 10 > quota_ms
@@ -261,3 +318,6 @@ LRU的优先级的提升，同时降低那些超过120秒无人访问的内存
     # echo 400 > wmarks_mid
     # echo 200 > wmarks_low
     # echo Y > enabled
+
+请注意，该模块（damon_lru_sort）不能与其他基于 DAMON 的专用模块同时运行。
+更多细节请参考 :ref:`DAMON 设计文档的专用模块互斥性 <damon_design_special_purpose_modules_exclusivity_zh_CN>`。
diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst
index d14ba32f77885e..1e9375947295ca 100644
--- a/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst
+++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/reclaim.rst
@@ -58,6 +58,19 @@ enabled
 DAMON_RECLAIM。注意，由于基于水位的激活条件，DAMON_RECLAIM不能进行真正的监测和回收。
 这一点请参考下面关于水位参数的描述。
 
+commit_inputs
+-------------
+
+让 DAMON_RECLAIM 再次读取除 ``enabled`` 外的输入参数。
+
+DAMON_RECLAIM 运行期间更新的输入参数默认不会应用。一旦该参数被设置为
+``Y``，DAMON_RECLAIM 会再次读取除 ``enabled`` 外的参数值。重新读取完成后，
+该参数会被设置为 ``N``。如果重新读取时发现无效参数，DAMON_RECLAIM 会被
+禁用。
+
+一旦向该参数写入 ``Y``，用户在再次读取 ``commit_inputs`` 返回 ``N`` 之前，
+不得写入任何参数。如果用户违反该规则，内核可能表现出未定义行为。
+
 min_age
 -------
 
@@ -68,6 +81,15 @@ min_age
 
 默认为120秒。
 
+autotune_monitoring_intervals
+-----------------------------
+
+如果该参数设置为 ``Y``，DAMON_RECLAIM 会自动调优 DAMON 的采样和聚集间隔。
+自动调优的目标是在每个 DAMON 快照中捕获有意义数量的访问事件，同时将采样
+间隔限制在最小 5 毫秒、最大 10 秒。将其设置为 ``N`` 会禁用自动调优。
+
+默认禁用。
+
 quota_ms
 --------
 
@@ -98,6 +120,29 @@ quota_reset_interval_ms
 
 默认为1秒。
 
+quota_mem_pressure_us
+---------------------
+
+期望的内存压力停滞时间水平，单位为微秒。
+
+在保持其他配额设置的上限的同时，DAMON_RECLAIM 会自动增减配额的有效水平，
+目标是产生该水平的内存压力。系统范围的 ``some`` 内存 PSI 会按每个配额重置
+间隔（``quota_reset_interval_ms``）以微秒为单位收集，并与该值比较，以判断
+目标是否满足。值为零表示禁用该自动调优功能。
+
+默认禁用。
+
+quota_autotune_feedback
+-----------------------
+
+用户可指定的有效配额自动调优反馈。
+
+在保持其他配额设置的上限的同时，DAMON_RECLAIM 会自动增减配额的有效水平，
+目标是从用户接收到值为 ``10,000`` 的反馈。DAMON_RECLAIM 假定反馈值和配额
+成正比。值为零表示禁用该自动调优功能。
+
+默认禁用。
+
 wmarks_interval
 ---------------
 
@@ -149,6 +194,8 @@ min_nr_regions
 DAMON用于冷内存监测的最小监测区域数。这可以用来设置监测质量的下限。但是，设
 置的太高可能会导致监测开销的增加。更多细节请参考DAMON文档 (:doc:`usage`) 。
 
+请注意，该值必须为 3 或更高。该下限的理由请参考设计文档的 :ref:`监测 <damon_design_monitoring_zh_CN>` 章节。
+
 max_nr_regions
 --------------
 
@@ -163,7 +210,7 @@ monitor_region_start
 目标内存区域的物理地址起点。
 
 DAMON_RECLAIM将对其进行工作的内存区域的起始物理地址。也就是说，DAMON_RECLAIM
-将在这个区域中找到冷的内存区域并进行回收。默认情况下，该区域使用最大系统内存区。
+将在这个区域中找到冷的内存区域并进行回收。默认情况下，该区域使用系统的整个物理内存。
 
 monitor_region_end
 ------------------
@@ -171,7 +218,35 @@ monitor_region_end
 目标内存区域的结束物理地址。
 
 DAMON_RECLAIM将对其进行工作的内存区域的末端物理地址。也就是说，DAMON_RECLAIM将
-在这个区域内找到冷的内存区域并进行回收。默认情况下，该区域使用最大系统内存区。
+在这个区域内找到冷的内存区域并进行回收。默认情况下，该区域使用系统的整个物理内存。
+
+addr_unit
+---------
+
+内存地址和字节数的缩放因子。
+
+该参数用于设置和获取 DAMON_RECLAIM 的 DAMON 实例的 :ref:`地址单位 <damon_design_addr_unit_zh_CN>` 参数。
+
+``monitor_region_start`` 和 ``monitor_region_end`` 应以该单位提供。例如，
+假设 ``addr_unit``、``monitor_region_start`` 和 ``monitor_region_end``
+分别设置为 ``1024``、``0`` 和 ``10``。那么 DAMON_RECLAIM 将处理从地址零
+开始、长度为 10 KiB 的物理地址范围（以字节表示为
+``[0 * 1024, 10 * 1024)``）。
+
+``bytes_reclaim_tried_regions`` 和 ``bytes_reclaimed_regions`` 也使用该单位。
+例如，假设 ``addr_unit``、``bytes_reclaim_tried_regions`` 和
+``bytes_reclaimed_regions`` 的值分别为 ``1024``、``42`` 和 ``32``。那么这
+表示 DAMON_RECLAIM 总共尝试回收 42 KiB 内存，并成功回收了 32 KiB 内存。
+
+如果不确定，只使用默认值（``1``）并忘记这个参数即可。
+
+skip_anon
+---------
+
+跳过匿名页回收。
+
+如果该参数设置为 ``Y``，DAMON_RECLAIM 不会回收匿名页。默认值为 ``N``。
+
 
 kdamond_pid
 -----------
@@ -223,6 +298,9 @@ DAMON_RECLAIM再次什么都不做，这样我们就可以退回到基于LRU列
     # echo 200 > wmarks_low
     # echo Y > enabled
 
+请注意，该模块（damon_reclaim）不能与其他基于 DAMON 的专用模块同时运行。
+更多细节请参考 :ref:`DAMON 设计文档的专用模块互斥性 <damon_design_special_purpose_modules_exclusivity_zh_CN>`。
+
 .. [1] https://research.google/pubs/pub48551/
 .. [2] https://lwn.net/Articles/787611/
 .. [3] https://www.kernel.org/doc/html/latest/mm/free_page_reporting.html
diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/start.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/start.rst
index cff7b6f98c5941..1a7f3382dd158d 100644
--- a/Documentation/translations/zh_CN/admin-guide/mm/damon/start.rst
+++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/start.rst
@@ -35,18 +35,63 @@ https://github.com/damonitor/damo找到。下面的例子假设DAMO在你的$PAT
 这并不是强制性的。
 
 因为DAMO使用了DAMON的sysfs接口（详情请参考:doc:`usage`），你应该确保
-:doc:`sysfs </filesystems/sysfs>` 被挂载。
+``sysfs`` 被挂载。
+
+
+拍摄数据访问模式快照
+====================
+
+下面的命令展示程序在执行时刻的内存访问模式。::
+
+    $ git clone https://github.com/sjp38/masim; cd masim; make
+    $ sudo damo start "./masim ./configs/stairs.cfg --quiet"
+    $ sudo damo report access
+    heatmap: 641111111000000000000000000000000000000000000000000000[...]33333333333333335557984444[...]7
+    # min/max temperatures: -1,840,000,000, 370,010,000, column size: 3.925 MiB
+    0   addr 86.182 TiB   size 8.000 KiB   access 0 %   age 14.900 s
+    1   addr 86.182 TiB   size 8.000 KiB   access 60 %  age 0 ns
+    2   addr 86.182 TiB   size 3.422 MiB   access 0 %   age 4.100 s
+    3   addr 86.182 TiB   size 2.004 MiB   access 95 %  age 2.200 s
+    4   addr 86.182 TiB   size 29.688 MiB  access 0 %   age 14.100 s
+    5   addr 86.182 TiB   size 29.516 MiB  access 0 %   age 16.700 s
+    6   addr 86.182 TiB   size 29.633 MiB  access 0 %   age 17.900 s
+    7   addr 86.182 TiB   size 117.652 MiB access 0 %   age 18.400 s
+    8   addr 126.990 TiB  size 62.332 MiB  access 0 %   age 9.500 s
+    9   addr 126.990 TiB  size 13.980 MiB  access 0 %   age 5.200 s
+    10  addr 126.990 TiB  size 9.539 MiB   access 100 % age 3.700 s
+    11  addr 126.990 TiB  size 16.098 MiB  access 0 %   age 6.400 s
+    12  addr 127.987 TiB  size 132.000 KiB access 0 %   age 2.900 s
+    total size: 314.008 MiB
+    $ sudo damo stop
+
+上面示例的第一条命令下载并构建一个名为 ``masim`` 的人工内存访问
+生成程序。第二条命令要求 DAMO 用给定命令启动该程序，并让 DAMON
+监测新启动的进程。第三条命令从 DAMON 取回该进程当前被监测访问
+模式的快照，并以人类可读的格式显示该模式。
+
+输出的第一行以单行热图格式显示各区域的相对访问温度（热度）。热图
+中的每一列表示被监测虚拟地址空间上大小相同的区域。列在该行中的
+位置和列上的数字表示该区域的相对位置和访问温度。``[...]`` 表示
+虚拟地址空间中未映射的巨大区域。第二行显示帮助理解热图的附加信息。
+
+从第三行开始，输出的每一行显示进程的哪个虚拟地址范围
+（``addr XX size XX``）被访问得有多频繁（``access XX %``），以及
+持续了多长时间（``age XX``）。例如，大小约为 9.5 MiB 的第十一个
+区域在最近 3.7 秒内被访问得最频繁。最后，第四条命令停止 DAMON。
+
+请注意，DAMON 不仅能监测虚拟地址空间，还能监测包括物理地址空间在内
+的多种地址空间。
+
 
 记录数据访问模式
 ================
 
 下面的命令记录了一个程序的内存访问模式，并将监测结果保存到文件中。 ::
 
-    $ git clone https://github.com/sjp38/masim
-    $ cd masim; make; ./masim ./configs/zigzag.cfg &
+    $ ./masim ./configs/zigzag.cfg &
     $ sudo damo record -o damon.data $(pidof masim)
 
-命令的前两行下载了一个人工内存访问生成器程序并在后台运行。生成器将重复地逐一访问两个
+第一行命令再次运行该人工内存访问生成器程序。生成器将重复地逐一访问两个
 100 MiB大小的内存区域。你可以用你的真实工作负载来代替它。最后一行要求 ``damo`` 将
 访问模式记录在 ``damon.data`` 文件中。
 
@@ -57,7 +102,7 @@ https://github.com/damonitor/damo找到。下面的例子假设DAMO在你的$PAT
 你可以在heatmap中直观地看到这种模式，显示哪个内存区域（X轴）何时被访问（Y轴）以及访
 问的频率（数字）。::
 
-    $ sudo damo report heats --heatmap stdout
+    $ sudo damo report heatmap
     22222222222222222222222222222222222222211111111111111111111111111111111111111100
     44444444444444444444444444444444444444434444444444444444444444444444444444443200
     44444444444444444444444444444444444444433444444444444444444444444444444444444200
@@ -117,8 +162,8 @@ https://github.com/damonitor/damo找到。下面的例子假设DAMO在你的$PAT
 数据访问模式感知的内存管理
 ==========================
 
-以下三个命令使每一个大小>=4K的内存区域在你的工作负载中没有被访问>=60秒，就会被换掉。 ::
+以下命令使每一个大小>=4K的内存区域在你的工作负载中没有被访问>=60秒，就会被换掉。 ::
 
-    $ echo "#min-size max-size min-acc max-acc min-age max-age action" > test_scheme
-    $ echo "4K        max      0       0       60s     max     pageout" >> test_scheme
-    $ damo schemes -c test_scheme <pid of your workload>
+    $ sudo damo start --damos_access_rate 0 0 --damos_sz_region 4K max \
+                      --damos_age 60s max --damos_action pageout \
+                      --target_pid <pid of your workload>
diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/stat.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/stat.rst
new file mode 100644
index 00000000000000..129748a5ea1f06
--- /dev/null
+++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/stat.rst
@@ -0,0 +1,94 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. include:: ../../../disclaimer-zh_CN.rst
+
+:Original: Documentation/admin-guide/mm/damon/stat.rst
+
+:翻译:
+
+ Doehyun Baek <doehyunbaek@gmail.com>
+
+====================
+数据访问监测结果统计
+====================
+
+数据访问监测结果统计（DAMON_STAT）是一个静态内核模块，旨在用于简单的
+访问模式监测。它使用 DAMON 监测系统整个物理内存上的访问，并提供简化的
+访问监测结果统计，即空闲时间百分位数和估计的内存带宽。
+
+.. _damon_stat_monitoring_accuracy_overhead_zh_CN:
+
+监测精度和开销
+==============
+
+DAMON_STAT 使用监测间隔
+:ref:`自动调优 <damon_design_monitoring_intervals_autotuning_zh_CN>` 来提高
+精度并最小化开销。它会自动调优间隔，目标是在每个快照中捕获 4 % 的
+可观测访问事件，同时将得到的采样间隔限制在最小 5 毫秒、最大 10 秒。
+在少数生产服务器系统上，它的结果是只消耗 0.x % 的单个 CPU 时间，同时
+捕获质量合理的访问模式。调优得到的间隔可以通过
+``aggr_interval_us`` :ref:`参数 <damon_stat_aggr_interval_us_zh_CN>` 获取。
+
+接口：模块参数
+==============
+
+要使用这个功能，首先应确保你的系统运行在构建时启用了
+``CONFIG_DAMON_STAT=y`` 的内核上。通过将
+``CONFIG_DAMON_STAT_ENABLED_DEFAULT`` 设置为 true，可以在构建时默认
+启用该功能。
+
+为了让系统管理员在启动时和/或运行时启用或禁用它，并读取监测结果，
+DAMON_STAT 提供了模块参数。下面的章节描述这些参数。
+
+enabled
+-------
+
+启用或禁用 DAMON_STAT。
+
+你可以将该参数的值设置为 ``Y`` 来启用 DAMON_STAT。设置为 ``N`` 会
+禁用 DAMON_STAT。默认值由 ``CONFIG_DAMON_STAT_ENABLED_DEFAULT`` 构建
+配置选项设置。
+
+请注意，该模块（damon_stat）不能与其他基于 DAMON 的专用模块同时运行。
+更多细节请参考 :ref:`DAMON 设计文档的专用模块互斥性 <damon_design_special_purpose_modules_exclusivity_zh_CN>`。
+
+.. _damon_stat_aggr_interval_us_zh_CN:
+
+aggr_interval_us
+----------------
+
+自动调优后的聚集时间间隔，单位是微秒。
+
+用户可以读取 DAMON_STAT 使用的 DAMON 实例的聚集间隔。它会被
+:ref:`自动调优 <damon_stat_monitoring_accuracy_overhead_zh_CN>`，因此该值
+会动态变化。
+
+estimated_memory_bandwidth
+--------------------------
+
+系统的估计内存带宽消耗（字节/秒）。
+
+DAMON_STAT 读取当前 DAMON 结果快照上的观测访问事件，并将其转换为以
+字节/秒为单位的内存带宽消耗估计。得到的指标通过这个只读参数向用户
+公开。由于 DAMON 使用采样，所以这只是访问强度的估计，而不是精确的
+内存带宽。
+
+memory_idle_ms_percentiles
+--------------------------
+
+系统的逐字节空闲时间（毫秒）百分位数。
+
+DAMON_STAT 基于当前 DAMON 结果快照，计算内存中每个字节到现在为止未被
+访问的时间（空闲时间）。对于访问频率（nr_accesses）大于零的区域，当前
+访问频率水平保持的时间乘以 ``-1``，就是该区域每个字节的空闲时间。如果
+某个区域的访问频率（nr_accesses）为零，则该区域保持零访问频率的时间
+（age）就是该区域每个字节的空闲时间。然后，DAMON_STAT 通过这个只读参数
+公开空闲时间值的百分位数。读取该参数会返回 101 个以毫秒为单位、用逗号
+分隔的空闲时间值。每个值分别表示第 0、第 1、第 2、第 3、……、第 99 和
+第 100 百分位的空闲时间。
+
+kdamond_pid
+-----------
+
+DAMON 线程的 PID。
+
+如果 DAMON_STAT 已启用，这将成为工作线程的 PID。否则为 -1。
diff --git a/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst b/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst
index 9d7cb51be4931b..f2fe1c8025fae1 100644
--- a/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst
+++ b/Documentation/translations/zh_CN/admin-guide/mm/damon/usage.rst
@@ -15,22 +15,27 @@
 
 DAMON 为不同的用户提供了下面这些接口。
 
+- *专用 DAMON 模块。*
+  :ref:`这 <damon_modules_special_purpose_zh_CN>` 是为那些正在构建、发布和/或管理
+  带有专用 DAMON 用法的内核的人准备的。使用它，用户可以在构建、启动或运行时，
+  以简单的方式为给定目的使用 DAMON 的主要功能。
 - *DAMON用户空间工具。*
   `这 <https://github.com/damonitor/damo>`_ 为有这特权的人， 如系统管理员，希望有一个刚好
   可以工作的人性化界面。
   使用它，用户可以以人性化的方式使用DAMON的主要功能。不过，它可能不会为特殊情况进行高度调整。
-  它同时支持虚拟和物理地址空间的监测。更多细节，请参考它的 `使用文档
+  更多细节，请参考它的 `使用文档
   <https://github.com/damonitor/damo/blob/next/USAGE.md>`_。
 - *sysfs接口。*
-  :ref:`这 <sysfs_interface>` 是为那些希望更高级的使用DAMON的特权用户空间程序员准备的。
+  :ref:`这 <sysfs_interface_zh_CN>` 是为那些希望更高级的使用DAMON的特权用户空间程序员准备的。
   使用它，用户可以通过读取和写入特殊的sysfs文件来使用DAMON的主要功能。因此，你可以编写和使
   用你个性化的DAMON sysfs包装程序，代替你读/写sysfs文件。  `DAMON用户空间工具
-  <https://github.com/damonitor/damo>`_ 就是这种程序的一个例子  它同时支持虚拟和物理地址
-  空间的监测。
+  <https://github.com/damonitor/damo>`_ 就是这种程序的一个例子。
 - *内核空间编程接口。*
-  :doc:`这 </mm/damon/api>` 这是为内核空间程序员准备的。使用它，用户可以通过为你编写内
+  :doc:`这 <../../../mm/damon/api>` 这是为内核空间程序员准备的。使用它，用户可以通过为你编写内
   核空间的DAMON应用程序，最灵活有效地利用DAMON的每一个功能。你甚至可以为各种地址空间扩展DAMON。
-  详细情况请参考接口 :doc:`文件 </mm/damon/api>`。
+  详细情况请参考接口 :doc:`文件 <../../../mm/damon/api>`。
+
+.. _sysfs_interface_zh_CN:
 
 sysfs接口
 =========
@@ -51,39 +56,61 @@ DAMON的sysfs接口是在定义 ``CONFIG_DAMON_SYSFS`` 时建立的。它在其s
 ------------
 
 DAMON sysfs接口的文件层次结构如下图所示。在下图中，父子关系用缩进表示，每个目录有
-``/`` 后缀，每个目录中的文件用逗号（","）分开。 ::
-
-    /sys/kernel/mm/damon/admin
-    │ kdamonds/nr_kdamonds
-    │ │ 0/state,pid
-    │ │ │ contexts/nr_contexts
-    │ │ │ │ 0/operations
-    │ │ │ │ │ monitoring_attrs/
+``/`` 后缀，每个目录中的文件用逗号（","）分开。
+
+.. parsed-literal::
+
+    :ref:`/sys/kernel/mm/damon <sysfs_root_zh_CN>`/admin
+    │ :ref:`kdamonds <sysfs_kdamonds_zh_CN>`/nr_kdamonds
+    │ │ :ref:`0 <sysfs_kdamond_zh_CN>`/state,pid,refresh_ms
+    │ │ │ :ref:`contexts <sysfs_contexts_zh_CN>`/nr_contexts
+    │ │ │ │ :ref:`0 <sysfs_context_zh_CN>`/avail_operations,operations,addr_unit,
+    │ │ │ │   pause
+    │ │ │ │ │ :ref:`monitoring_attrs <sysfs_monitoring_attrs_zh_CN>`/
     │ │ │ │ │ │ intervals/sample_us,aggr_us,update_us
+    │ │ │ │ │ │ │ intervals_goal/access_bp,aggrs,min_sample_us,max_sample_us
     │ │ │ │ │ │ nr_regions/min,max
-    │ │ │ │ │ targets/nr_targets
-    │ │ │ │ │ │ 0/pid_target
-    │ │ │ │ │ │ │ regions/nr_regions
-    │ │ │ │ │ │ │ │ 0/start,end
+    │ │ │ │ │ │ :ref:`probes <damon_usage_sysfs_probes_zh_CN>`/nr_probes
+    │ │ │ │ │ │ │ 0/filters/nr_filters
+    │ │ │ │ │ │ │ │ 0/type,matching,allow,path
+    │ │ │ │ │ │ │ │ ...
+    │ │ │ │ │ │ │ ...
+    │ │ │ │ │ :ref:`targets <sysfs_targets_zh_CN>`/nr_targets
+    │ │ │ │ │ │ :ref:`0 <sysfs_target_zh_CN>`/pid_target,obsolete_target
+    │ │ │ │ │ │ │ :ref:`regions <sysfs_regions_zh_CN>`/nr_regions
+    │ │ │ │ │ │ │ │ :ref:`0 <sysfs_region_zh_CN>`/start,end
     │ │ │ │ │ │ │ │ ...
     │ │ │ │ │ │ ...
-    │ │ │ │ │ schemes/nr_schemes
-    │ │ │ │ │ │ 0/action
-    │ │ │ │ │ │ │ access_pattern/
+    │ │ │ │ │ :ref:`schemes <sysfs_schemes_zh_CN>`/nr_schemes
+    │ │ │ │ │ │ :ref:`0 <sysfs_scheme_zh_CN>`/action,target_nid,apply_interval_us
+    │ │ │ │ │ │ │ :ref:`access_pattern <sysfs_access_pattern_zh_CN>`/
     │ │ │ │ │ │ │ │ sz/min,max
     │ │ │ │ │ │ │ │ nr_accesses/min,max
     │ │ │ │ │ │ │ │ age/min,max
-    │ │ │ │ │ │ │ quotas/ms,bytes,reset_interval_ms
+    │ │ │ │ │ │ │ :ref:`quotas <sysfs_quotas_zh_CN>`/ms,bytes,reset_interval_ms,
+    │ │ │ │ │ │ │     effective_bytes,goal_tuner,
+    │ │ │ │ │ │ │     fail_charge_num,fail_charge_denom
     │ │ │ │ │ │ │ │ weights/sz_permil,nr_accesses_permil,age_permil
-    │ │ │ │ │ │ │ watermarks/metric,interval_us,high,mid,low
-    │ │ │ │ │ │ │ stats/nr_tried,sz_tried,nr_applied,sz_applied,qt_exceeds
-    │ │ │ │ │ │ │ tried_regions/
-    │ │ │ │ │ │ │ │ 0/start,end,nr_accesses,age
+    │ │ │ │ │ │ │ │ :ref:`goals <sysfs_schemes_quota_goals_zh_CN>`/nr_goals
+    │ │ │ │ │ │ │ │ │ 0/target_metric,target_value,current_value,nid,path
+    │ │ │ │ │ │ │ :ref:`watermarks <sysfs_watermarks_zh_CN>`/metric,interval_us,high,mid,low
+    │ │ │ │ │ │ │ :ref:`{core_,ops_,}filters <sysfs_filters_zh_CN>`/nr_filters
+    │ │ │ │ │ │ │ │ 0/type,matching,allow,memcg_path,addr_start,addr_end,damon_target_idx,min,max
+    │ │ │ │ │ │ │ :ref:`dests <damon_sysfs_dests_zh_CN>`/nr_dests
+    │ │ │ │ │ │ │ │ 0/id,weight
+    │ │ │ │ │ │ │ :ref:`stats <sysfs_schemes_stats_zh_CN>`/nr_tried,sz_tried,nr_applied,sz_applied,sz_ops_filter_passed,qt_exceeds,nr_snapshots,max_nr_snapshots
+    │ │ │ │ │ │ │ :ref:`tried_regions <sysfs_schemes_tried_regions_zh_CN>`/total_bytes
+    │ │ │ │ │ │ │ │ 0/start,end,nr_accesses,age,sz_filter_passed
+    │ │ │ │ │ │ │ │ │ probes
+    │ │ │ │ │ │ │ │ │ │ 0/hits
+    │ │ │ │ │ │ │ │ │ │ ...
     │ │ │ │ │ │ │ │ ...
     │ │ │ │ │ │ ...
     │ │ │ │ ...
     │ │ ...
 
+.. _sysfs_root_zh_CN:
+
 根
 --
 
@@ -91,58 +118,100 @@ DAMON sysfs接口的根是 ``<sysfs>/kernel/mm/damon/`` ，它有一个名为 ``
 目录。该目录包含特权用户空间程序控制DAMON的文件。拥有根权限的用户空间工具或deamons可以
 使用这个目录。
 
+.. _sysfs_kdamonds_zh_CN:
+
 kdamonds/
 ---------
 
-与监测相关的信息包括请求规格和结果被称为DAMON上下文。DAMON用一个叫做kdamond的内核线程
-执行每个上下文，多个kdamonds可以并行运行。
-
-在 ``admin`` 目录下，有一个目录，即``kdamonds``，它有控制kdamonds的文件存在。在开始
+在 ``admin`` 目录下，有一个目录，即``kdamonds``，它有控制kdamonds的文件存在（更多
+细节请参考 :ref:`设计 <damon_design_execution_model_and_data_structures_zh_CN>`）。在开始
 时，这个目录只有一个文件，``nr_kdamonds``。向该文件写入一个数字（``N``），就会创建名为
 ``0`` 到 ``N-1`` 的子目录数量。每个目录代表每个kdamond。
 
+.. _sysfs_kdamond_zh_CN:
+
 kdamonds/<N>/
 -------------
 
-在每个kdamond目录中，存在两个文件（``state`` 和 ``pid`` ）和一个目录( ``contexts`` )。
+在每个kdamond目录中，存在三个文件（``state`` 、 ``pid`` 和 ``refresh_ms`` ）和一个目录( ``contexts`` )。
 
 读取 ``state`` 时，如果kdamond当前正在运行，则返回 ``on`` ，如果没有运行则返回 ``off`` 。
-写入 ``on`` 或 ``off`` 使kdamond处于状态。向 ``state`` 文件写 ``update_schemes_stats`` ，
-更新kdamond的每个基于DAMON的操作方案的统计文件的内容。关于统计信息的细节，请参考
-:ref:`stats section <sysfs_schemes_stats>`. 将 ``update_schemes_tried_regions`` 写到
-``state`` 文件，为kdamond的每个基于DAMON的操作方案，更新基于DAMON的操作方案动作的尝试区域目录。
-将`clear_schemes_tried_regions`写入`state`文件，清除kdamond的每个基于DAMON的操作方案的动作
-尝试区域目录。 关于基于DAMON的操作方案动作尝试区域目录的细节，请参考:ref:tried_regions 部分
-<sysfs_schemes_tried_regions>`。
+
+用户可以向 ``state`` 文件写入以下kdamond命令。
+
+- ``on``：开始运行。
+- ``off``：停止运行。
+- ``commit``：再次读取除 ``state`` 文件之外的 sysfs 文件中的用户输入。
+  如果没有指定目标区域，也会忽略监测 :ref:`目标区域 <sysfs_regions_zh_CN>`
+  输入。
+- ``update_tuned_intervals``：用自动调优后的 ``sampling interval`` 和
+  ``aggregation interval`` 更新该 kdamond 的 ``sample_us`` 和
+  ``aggr_us`` 文件内容。更多细节请参考 :ref:`intervals_goal 章节
+  <damon_usage_sysfs_monitoring_intervals_goal_zh_CN>`。
+- ``commit_schemes_quota_goals``：读取基于 DAMON 的操作方案的
+  :ref:`配额目标 <sysfs_schemes_quota_goals_zh_CN>`。
+- ``update_schemes_stats``：更新该 kdamond 的每个基于 DAMON 的操作方案的
+  统计文件内容。关于统计信息的细节，请参考 :ref:`stats 章节
+  <sysfs_schemes_stats_zh_CN>`。
+- ``update_schemes_tried_regions``：为该 kdamond 的每个基于 DAMON 的
+  操作方案，更新基于 DAMON 的操作方案动作尝试区域目录。关于基于 DAMON
+  的操作方案动作尝试区域目录的细节，请参考 :ref:`tried_regions 章节
+  <sysfs_schemes_tried_regions_zh_CN>`。
+- ``update_schemes_tried_bytes``：只更新 ``.../tried_regions/total_bytes``
+  文件。
+- ``clear_schemes_tried_regions``：为该 kdamond 的每个基于 DAMON 的
+  操作方案，清除基于 DAMON 的操作方案动作尝试区域目录。
+- ``update_schemes_effective_quotas``：更新该 kdamond 的每个基于 DAMON
+  的操作方案的 ``effective_bytes`` 文件内容。更多细节请参考
+  :ref:`quotas 目录 <sysfs_quotas_zh_CN>`。
 
 如果状态为 ``on``，读取 ``pid`` 显示kdamond线程的pid。
 
+用户可以要求内核周期性地更新显示自动调优参数和 DAMOS 统计信息的文件，
+而不是手动向 ``state`` 文件写入 ``update_tuned_intervals`` 这类关键字。
+为此，用户应将所需的更新时间间隔（毫秒）写入 ``refresh_ms`` 文件。如果
+该间隔为零，则禁用周期性更新。读取该文件会显示当前设置的时间间隔。
+
 ``contexts`` 目录包含控制这个kdamond要执行的监测上下文的文件。
 
+.. _sysfs_contexts_zh_CN:
+
 kdamonds/<N>/contexts/
 ----------------------
 
 在开始时，这个目录只有一个文件，即 ``nr_contexts`` 。向该文件写入一个数字( ``N`` )，就会创
-建名为``0`` 到 ``N-1`` 的子目录数量。每个目录代表每个监测背景。目前，每个kdamond只支持
+建名为``0`` 到 ``N-1`` 的子目录数量。每个目录代表每个监测上下文（更多细节请参考
+:ref:`设计 <damon_design_execution_model_and_data_structures_zh_CN>`）。目前，每个kdamond只支持
 一个上下文，所以只有 ``0`` 或 ``1`` 可以被写入文件。
 
+.. _sysfs_context_zh_CN:
+
 contexts/<N>/
 -------------
 
-在每个上下文目录中，存在一个文件(``operations``)和三个目录(``monitoring_attrs``,
+在每个上下文目录中，存在四个文件(``avail_operations``、``operations``、
+``addr_unit`` 和 ``pause``)和三个目录(``monitoring_attrs``,
 ``targets``, 和 ``schemes``)。
 
-DAMON支持多种类型的监测操作，包括对虚拟地址空间和物理地址空间的监测。你可以通过向文件
-中写入以下关键词之一，并从文件中读取，来设置和获取DAMON将为上下文使用何种类型的监测操作。
+DAMON支持多种 :ref:`监测操作 <damon_design_configurable_operations_set_zh_CN>` 类型，
+包括对虚拟地址空间和物理地址空间的监测。你可以通过读取 ``avail_operations`` 文件，
+获取当前运行内核上可用的监测操作集列表。根据内核配置，该文件会列出不同的可用
+操作集。所有可用操作集及其简要说明请参考 :ref:`设计 <damon_operations_set_zh_CN>`。
+
+你可以通过向 ``operations`` 文件写入 ``avail_operations`` 文件中列出的一个关键词，
+并从 ``operations`` 文件中读取，来设置和获取DAMON将为上下文使用何种类型的监测操作。
+
+``addr_unit`` 文件用于设置和获取该操作集的 :ref:`地址单位 <damon_design_addr_unit_zh_CN>` 参数。
+
+``pause`` 文件用于设置和获取该上下文的 :ref:`暂停请求 <damon_design_execution_model_and_data_structures_zh_CN>` 参数。
 
- - vaddr: 监测特定进程的虚拟地址空间
- - paddr: 监视系统的物理地址空间
+.. _sysfs_monitoring_attrs_zh_CN:
 
 contexts/<N>/monitoring_attrs/
 ------------------------------
 
 用于指定监测属性的文件，包括所需的监测质量和效率，都在 ``monitoring_attrs`` 目录中。
-具体来说，这个目录下有两个目录，即 ``intervals`` 和 ``nr_regions`` 。
+具体来说，这个目录下有三个目录，即 ``intervals``、``nr_regions`` 和 ``probes``。
 
 在 ``intervals`` 目录下，存在DAMON的采样间隔(``sample_us``)、聚集间隔(``aggr_us``)
 和更新间隔(``update_us``)三个文件。你可以通过写入和读出这些文件来设置和获取微秒级的值。
@@ -150,7 +219,44 @@ contexts/<N>/monitoring_attrs/
 在 ``nr_regions`` 目录下，有两个文件分别用于DAMON监测区域的下限和上限（``min`` 和 ``max`` ），
 这两个文件控制着监测的开销。你可以通过向这些文件的写入和读出来设置和获取这些值。
 
-关于间隔和监测区域范围的更多细节，请参考设计文件 (:doc:`/mm/damon/design`)。
+关于间隔和监测区域范围的更多细节，请参考设计文件 (:ref:`设计 <damon_design_monitoring_zh_CN>`)。
+
+.. _damon_usage_sysfs_monitoring_intervals_goal_zh_CN:
+
+contexts/<N>/monitoring_attrs/intervals/intervals_goal/
+-------------------------------------------------------
+
+在 ``intervals`` 目录下，还存在一个用于自动调优 ``sample_us`` 和
+``aggr_us`` 的目录，即 ``intervals_goal`` 目录。在该目录下，存在四个
+用于自动调优控制的文件，即 ``access_bp``、``aggrs``、``min_sample_us``
+和 ``max_sample_us``。关于调优机制的内部实现，请参考该功能的
+:ref:`设计文档 <damon_design_monitoring_intervals_autotuning_zh_CN>`。读取和写入
+``intervals_goal`` 目录下的四个文件，会显示并更新 :ref:`设计文档 <damon_design_monitoring_intervals_autotuning_zh_CN>` 中描述的同名调优参数。调优
+从用户设置的 ``sample_us`` 和 ``aggr_us`` 开始。向 ``state`` 文件写入
+``update_tuned_intervals`` 后，可以从 ``sample_us`` 和 ``aggr_us`` 文件
+读取应用调优后的两个间隔的当前值。
+
+.. _damon_usage_sysfs_probes_zh_CN:
+
+contexts/<N>/monitoring_attrs/probes/
+-------------------------------------
+
+用于注册 :ref:`数据属性监测 <damon_design_data_attrs_monitoring_zh_CN>` 探针的目录。
+
+开始时，该目录只有一个文件 ``nr_probes``。向该文件写入一个数字
+（``N``）会创建数量为该数字的子目录，命名为 ``0`` 到 ``N-1``。每个
+目录表示一个监测探针。
+
+在每个探针目录中，存在一个目录 ``filters``。该目录包含为探针安装过滤器
+的文件，该过滤器用于确定探针的数据属性。
+
+开始时，``filters`` 目录只有一个文件 ``nr_filters``。向该文件写入一个
+数字（``N``）会创建数量为该数字的子目录，命名为 ``0`` 到 ``N-1``。每个
+目录表示一个过滤器，其工作方式类似于 :ref:`DAMOS 过滤器
+<sysfs_filters_zh_CN>`。当过滤器 ``type`` 为 ``memcg`` 时，``path`` 文件
+作为 :ref:`DAMOS 过滤器 <sysfs_filters_zh_CN>` 的 ``memcg_path`` 使用。
+
+.. _sysfs_targets_zh_CN:
 
 contexts/<N>/targets/
 ---------------------
@@ -158,30 +264,43 @@ contexts/<N>/targets/
 在开始时，这个目录只有一个文件 ``nr_targets`` 。向该文件写入一个数字(``N``)，就可以创建
 名为 ``0`` 到 ``N-1`` 的子目录的数量。每个目录代表每个监测目标。
 
+.. _sysfs_target_zh_CN:
+
 targets/<N>/
 ------------
 
-在每个目标目录中，存在一个文件(``pid_target``)和一个目录(``regions``)。
+在每个目标目录中，存在两个文件（``pid_target`` 和 ``obsolete_target``）和一个目录(``regions``)。
 
 如果你把 ``vaddr`` 写到 ``contexts/<N>/operations`` 中，每个目标应该是一个进程。你
 可以通过将进程的pid写到 ``pid_target`` 文件中来指定DAMON的进程。
 
+用户可以通过向 ``obsolete_target`` 文件写入非零值并提交它（向 ``state``
+文件写入 ``commit``），有选择地移除目标数组中间的目标。DAMON 会从内部
+目标数组中移除匹配的目标。用户负责重新构建目标目录，使其正确表示变更后的
+内部目标数组。
+
+
+.. _sysfs_regions_zh_CN:
+
 targets/<N>/regions
 -------------------
 
-当使用 ``vaddr`` 监测操作集时（ ``vaddr`` 被写入 ``contexts/<N>/operations`` 文
-件），DAMON自动设置和更新监测目标区域，这样就可以覆盖目标进程的整个内存映射。然而，用户可
-能希望将初始监测区域设置为特定的地址范围。
+在使用 ``fvaddr`` 或 ``paddr`` 监测操作集时，用户需要设置监测目标地址
+范围。在使用 ``vaddr`` 操作集时，这不是必须的，但用户可以选择将初始
+监测区域设置为特定地址范围。更多细节请参考 :ref:`设计 <damon_design_vaddr_target_regions_construction_zh_CN>`。
 
-相反，当使用 ``paddr`` 监测操作集时，DAMON不会自动设置和更新监测目标区域（ ``paddr``
-被写入 ``contexts/<N>/operations`` 中）。因此，在这种情况下，用户应该自己设置监测目标
-区域。
-
-在这种情况下，用户可以按照自己的意愿明确设置初始监测目标区域，将适当的值写入该目录下的文件。
+对于这类情况，用户可以按照自己的需要，向该目录下的文件写入适当的值，
+显式设置初始监测目标区域。
 
 开始时，这个目录只有一个文件， ``nr_regions`` 。向该文件写入一个数字(``N``)，就可以创
 建名为 ``0`` 到  ``N-1`` 的子目录。每个目录代表每个初始监测目标区域。
 
+如果在线提交新的 DAMON 参数时（向 :ref:`kdamond <sysfs_kdamond_zh_CN>` 的
+``state`` 文件写入 ``commit``）``nr_regions`` 为零，则提交逻辑会忽略
+目标区域。换言之，该目标当前的监测结果会被保留。
+
+.. _sysfs_region_zh_CN:
+
 regions/<N>/
 ------------
 
@@ -190,6 +309,8 @@ regions/<N>/
 
 每个区域不应该与其他区域重叠。 目录“N”的“结束”应等于或小于目录“N+1”的“开始”。
 
+.. _sysfs_schemes_zh_CN:
+
 contexts/<N>/schemes/
 ---------------------
 
@@ -200,99 +321,249 @@ contexts/<N>/schemes/
 在开始时，这个目录只有一个文件，``nr_schemes``。向该文件写入一个数字(``N``)，就可以
 创建名为``0``到``N-1``的子目录的数量。每个目录代表每个基于DAMON的操作方案。
 
+.. _sysfs_scheme_zh_CN:
+
 schemes/<N>/
 ------------
 
-在每个方案目录中，存在五个目录(``access_pattern``、``quotas``、``watermarks``、
-``stats`` 和 ``tried_regions``)和一个文件(``action``)。
+在每个方案目录中，存在九个目录（``access_pattern``、``quotas``、
+``watermarks``、``core_filters``、``ops_filters``、``filters``、
+``dests``、``stats`` 和 ``tried_regions``）以及三个文件（``action``、
+``target_nid`` 和 ``apply_interval_us``）。
 
-``action`` 文件用于设置和获取你想应用于具有特定访问模式的内存区域的动作。可以写入文件
-和从文件中读取的关键词及其含义如下。
+``action`` 文件用于设置和获取方案的 :ref:`动作 <damon_design_damos_action_zh_CN>`。可写入和读取该文件的关键字及其含义，与
+:ref:`设计文档 <damon_design_damos_action_zh_CN>` 中列表的内容相同。
 
- - ``willneed``: 对有 ``MADV_WILLNEED`` 的区域调用 ``madvise()`` 。
- - ``cold``: 对具有 ``MADV_COLD`` 的区域调用 ``madvise()`` 。
- - ``pageout``: 为具有 ``MADV_PAGEOUT`` 的区域调用 ``madvise()`` 。
- - ``hugepage``: 为带有 ``MADV_HUGEPAGE`` 的区域调用 ``madvise()`` 。
- - ``nohugepage``: 为带有 ``MADV_NOHUGEPAGE`` 的区域调用 ``madvise()``。
- - ``lru_prio``: 在其LRU列表上对区域进行优先排序。
- - ``lru_deprio``: 对区域的LRU列表进行降低优先处理。
- - ``stat``: 什么都不做，只计算统计数据
+``target_nid`` 文件用于设置迁移目标节点，只有当 ``action`` 为
+``migrate_hot`` 或 ``migrate_cold`` 时才有意义。
+
+``apply_interval_us`` 文件用于以微秒为单位设置和获取方案的
+:ref:`apply_interval <damon_design_damos_zh_CN>`。
+
+.. _sysfs_access_pattern_zh_CN:
 
 schemes/<N>/access_pattern/
 ---------------------------
 
-每个基于DAMON的操作方案的目标访问模式由三个范围构成，包括以字节为单位的区域大小、每个
-聚合区间的监测访问次数和区域年龄的聚合区间数。
+用于给定基于 DAMON 的操作方案的目标访问 :ref:`模式
+<damon_design_damos_access_pattern_zh_CN>` 的目录。
 
 在 ``access_pattern`` 目录下，存在三个目录（ ``sz``, ``nr_accesses``, 和 ``age`` ），
 每个目录有两个文件（``min`` 和 ``max`` ）。你可以通过向  ``sz``, ``nr_accesses``, 和
 ``age``  目录下的 ``min`` 和 ``max`` 文件分别写入和读取来设置和获取给定方案的访问模式。
+请注意，``min`` 和 ``max`` 形成闭区间。
+
+.. _sysfs_quotas_zh_CN:
 
 schemes/<N>/quotas/
 -------------------
 
-每个 ``动作`` 的最佳 ``目标访问模式`` 取决于工作负载，所以不容易找到。更糟糕的是，将某些动作
-的方案设置得过于激进会造成严重的开销。为了避免这种开销，用户可以为每个方案限制时间和大小配额。
-具体来说，用户可以要求DAMON尽量只使用特定的时间（``时间配额``）来应用动作，并且在给定的时间间
-隔（``重置间隔``）内，只对具有目标访问模式的内存区域应用动作，而不使用特定数量（``大小配额``）。
+用于给定基于 DAMON 的操作方案的 :ref:`配额 <damon_design_damos_quotas_zh_CN>` 的目录。
+
+在 ``quotas`` 目录下，存在七个文件（``ms``、``bytes``、
+``reset_interval_ms``、``effective_bytes``、``goal_tuner``、
+``fail_charge_num`` 和 ``fail_charge_denom``）以及两个目录（``weights``
+和 ``goals``）。
 
-当预计超过配额限制时，DAMON会根据 ``目标访问模式`` 的大小、访问频率和年龄，对找到的内存区域
-进行优先排序。为了进行个性化的优先排序，用户可以为这三个属性设置权重。
+你可以分别向这三个文件写入值，设置以毫秒为单位的 ``time quota``、以字节
+为单位的 ``size quota`` 以及以毫秒为单位的 ``reset interval``。随后，
+DAMON 会尝试在 ``reset_interval_ms`` 内，只使用最多 ``time quota``
+毫秒把 ``action`` 应用于符合 ``access_pattern`` 的内存区域，并且只将该
+动作应用于最多 ``bytes`` 字节的内存区域。将 ``ms`` 和 ``bytes`` 都设置
+为零会禁用配额限制，除非至少设置了一个 :ref:`目标
+<sysfs_schemes_quota_goals_zh_CN>`。
 
-在 ``quotas`` 目录下，存在三个文件（``ms``, ``bytes``, ``reset_interval_ms``）和一个
-目录(``weights``)，其中有三个文件(``sz_permil``, ``nr_accesses_permil``, 和
-``age_permil``)。
+你可以向 ``goal_tuner`` 文件写入算法名称，设置要使用的基于目标的有效
+配额自动调优算法。读取该文件会返回当前选择的调优算法。关于该功能的背景
+设计以及可选择算法的名称，请参考 :ref:`自动配额调优目标 <damon_design_damos_quotas_auto_tuning_zh_CN>` 的设计文档。关于目标设置，请参考
+:ref:`goals 目录 <sysfs_schemes_quota_goals_zh_CN>`。
 
-你可以设置以毫秒为单位的 ``时间配额`` ，以字节为单位的 ``大小配额`` ，以及以毫秒为单位的 ``重
-置间隔`` ，分别向这三个文件写入数值。你还可以通过向 ``weights`` 目录下的三个文件写入数值来设
-置大小、访问频率和年龄的优先权，单位为千分之一。
+你可以分别向 ``fail_charge_num`` 和 ``fail_charge_denom`` 文件写入该比率的
+分子和分母，设置动作失败内存配额计费比率。读取这些文件会返回当前设置的值。
+关于该比率功能的更多细节，请参考 :ref:`设计 <damon_design_damos_quotas_failed_memory_charging_ratio_zh_CN>`。
+
+时间配额会在内部转换为大小配额。在转换后的大小配额和用户指定的大小配额之间，
+会应用较小者。根据用户指定的 :ref:`目标
+<sysfs_schemes_quota_goals_zh_CN>`，有效大小配额会被进一步调整。读取
+``effective_bytes`` 会返回当前有效大小配额。该文件不会实时更新，所以用户
+应通过向相关 ``kdamonds/<N>/state`` 文件写入特殊关键字
+``update_schemes_effective_quotas``，要求 DAMON sysfs 接口为统计信息更新该
+文件内容。
+
+在 ``weights`` 目录下，存在三个文件（``sz_permil``、
+``nr_accesses_permil`` 和 ``age_permil``）。你可以向 ``weights`` 目录下的
+三个文件写入值，设置大小、访问频率和年龄的 :ref:`优先级权重 <damon_design_damos_quotas_prioritization_zh_CN>`，单位为千分之一。
+
+.. _sysfs_schemes_quota_goals_zh_CN:
+
+schemes/<N>/quotas/goals/
+-------------------------
+
+用于给定基于 DAMON 的操作方案的 :ref:`自动配额调优目标 <damon_design_damos_quotas_auto_tuning_zh_CN>` 的目录。
+
+开始时，该目录只有一个文件 ``nr_goals``。向该文件写入一个数字（``N``）
+会创建数量为该数字的子目录，命名为 ``0`` 到 ``N-1``。每个目录表示一个
+目标和当前达成情况。在多个反馈中，会使用最好的一个。
+
+每个目标目录包含五个文件，即 ``target_metric``、``target_value``、
+``current_value``、``nid`` 和 ``path``。用户可以通过写入和读取每个文件，
+设置和获取 :ref:`设计文档 <damon_design_damos_quotas_auto_tuning_zh_CN>` 中指定的
+配额自动调优目标的五个参数。因为内核不会更新 ``current_value``，所以只有
+当 ``target_metric`` 为 ``user_input`` 时，读取它才有意义。请注意，用户还应
+向 :ref:`kdamond 目录 <sysfs_kdamond_zh_CN>` 的 ``state`` 文件写入
+``commit_schemes_quota_goals``，以便将反馈传递给 DAMON。
+
+.. _sysfs_watermarks_zh_CN:
 
 schemes/<N>/watermarks/
 -----------------------
 
-为了便于根据系统状态激活和停用每个方案，DAMON提供了一个称为水位的功能。该功能接收五个值，称为
-``度量`` 、``间隔`` 、``高`` 、``中`` 、``低`` 。``度量值`` 是指可以测量的系统度量值，如
-自由内存比率。如果系统的度量值 ``高`` 于memoent的高值或 ``低`` 于低值，则该方案被停用。如果
-该值低于 ``中`` ，则该方案被激活。
+用于给定基于 DAMON 的操作方案的 :ref:`水位 <damon_design_damos_watermarks_zh_CN>` 的目录。
+
+在 watermarks 目录下，存在五个文件（``metric``、``interval_us``、``high``、
+``mid`` 和 ``low``），用于设置度量指标、度量指标检查之间的时间间隔以及三个
+水位。你可以分别通过写入和读取这些文件来设置和获取这五个值。
+
+可写入 ``metric`` 文件的关键字及其含义如下。
+
+ - none：忽略水位
+ - free_mem_rate：系统空闲内存率（千分之一）
+
+``interval_us`` 应以微秒为单位写入。
+
+.. _sysfs_filters_zh_CN:
+
+schemes/<N>/{core\_,ops\_,}filters/
+-----------------------------------
+
+用于给定基于 DAMON 的操作方案的 :ref:`过滤器 <damon_design_damos_filters_zh_CN>` 的目录。
+
+``core_filters`` 和 ``ops_filters`` 目录分别用于由 DAMON 核心层和操作集层
+处理的过滤器。``filters`` 目录可用于安装过滤器，而不管它们由哪一层处理。
+``core_filters`` 和 ``ops_filters`` 请求的过滤器会先于 ``filters`` 中的
+过滤器安装。这三个目录具有相同的文件。
+
+使用 ``filters`` 目录会使过滤器求值顺序难以预期。因此，``filters`` 目录
+已被弃用。它仍然可以工作，但计划在不久的将来移除。用户应改用
+``core_filters`` 和 ``ops_filters`` 目录。
+
+开始时，该目录只有一个文件 ``nr_filters``。向该文件写入一个数字（``N``）
+会创建数量为该数字的子目录，命名为 ``0`` 到 ``N-1``。每个目录表示一个
+过滤器。过滤器按数字顺序求值。
+
+每个过滤器目录包含九个文件，即 ``type``、``matching``、``allow``、
+``memcg_path``、``addr_start``、``addr_end``、``min``、``max`` 和
+``damon_target_idx``。你可以向 ``type`` 文件写入过滤器类型。关于可用
+类型名称、它们的含义以及它们由哪一层处理，请参考 :ref:`设计文档 <damon_design_damos_filters_zh_CN>`。
 
-在水位目录下，存在五个文件(``metric``, ``interval_us``,``high``, ``mid``, and ``low``)
-用于设置每个值。你可以通过向这些文件的写入来分别设置和获取这五个值。
+对于 ``memcg`` 类型，你可以向 ``memcg_path`` 文件写入从 cgroups 挂载点
+开始的内存 cgroup 路径，指定感兴趣的内存 cgroup。对于 ``addr`` 类型，
+你可以分别向 ``addr_start`` 和 ``addr_end`` 文件写入范围（左闭右开区间）
+的起始和结束地址。对于 ``hugepage_size`` 类型，你可以分别向 ``min`` 和
+``max`` 文件写入范围（闭区间）的最小和最大大小。对于 ``target`` 类型，
+你可以向 ``damon_target_idx`` 文件写入 DAMON 上下文的监测目标列表中目标的
+索引。
 
-可以写入 ``metric`` 文件的关键词和含义如下。
+你可以向 ``matching`` 文件写入 ``Y`` 或 ``N``，指定过滤器是否用于匹配
+``type`` 的内存。你可以向 ``allow`` 文件写入 ``Y`` 或 ``N``，指定是否允许
+将动作应用于满足 ``type`` 和 ``matching`` 的内存。
 
- - none: 忽略水位
- - free_mem_rate: 系统的自由内存率（千分比）。
+例如，下面的操作将 DAMOS 动作限制为只应用于除 ``/having_care_already`` 之外
+所有内存 cgroup 的非匿名页。::
 
-``interval`` 应以微秒为单位写入。
+    # cd ops_filters/0/
+    # echo 2 > nr_filters
+    # # disallow anonymous pages
+    echo anon > 0/type
+    echo Y > 0/matching
+    echo N > 0/allow
+    # # further filter out all cgroups except one at '/having_care_already'
+    echo memcg > 1/type
+    echo /having_care_already > 1/memcg_path
+    echo Y > 1/matching
+    echo N > 1/allow
+
+更多细节请参考 :ref:`DAMOS 过滤器设计文档 <damon_design_damos_filters_zh_CN>`，
+其中包括不同 ``allow`` 的多个过滤器如何工作、每个过滤器何时受支持，以及
+统计信息上的差异。
+
+.. _damon_sysfs_dests_zh_CN:
+
+schemes/<N>/dests/
+------------------
+
+用于指定给定基于 DAMON 的操作方案动作的目的地的目录。如果给定方案的动作
+不支持多个目的地，则该目录会被忽略。只有 ``DAMOS_MIGRATE_{HOT,COLD}`` 动作
+支持多个目的地。
+
+开始时，该目录只有一个文件 ``nr_dests``。向该文件写入一个数字（``N``）
+会创建数量为该数字的子目录，命名为 ``0`` 到 ``N-1``。每个目录表示一个
+动作目的地。
+
+每个目的地目录包含两个文件，即 ``id`` 和 ``weight``。用户可以向 ``id``
+文件写入目的地标识符，并从中读取它。对于 ``DAMOS_MIGRATE_{HOT,COLD}`` 动作，
+应将迁移目的节点的节点 id 写入 ``id`` 文件。用户可以向 ``weight`` 文件写入
+该目的地在给定目的地中的权重，并从中读取它。权重可以是任意整数。当 DAMOS
+将动作应用于内存区域的每个实体时，它会根据目的地的相对权重选择该动作的
+目的地。
+
+.. _sysfs_schemes_stats_zh_CN:
 
 schemes/<N>/stats/
 ------------------
 
-DAMON统计每个方案被尝试应用的区域的总数量和字节数，每个方案被成功应用的区域的两个数字，以及
-超过配额限制的总数量。这些统计数据可用于在线分析或调整方案。
+DAMON 为每个方案统计信息。这些统计信息可用于方案的在线分析或调优。关于
+这些统计信息的更多细节，请参考 :ref:`设计文档 <damon_design_damos_stat_zh_CN>`。
+
+可以分别通过读取 ``stats`` 目录下的文件（``nr_tried``、``sz_tried``、
+``nr_applied``、``sz_applied``、``sz_ops_filter_passed``、``qt_exceeds``、
+``nr_snapshots`` 和 ``max_nr_snapshots``）来获取这些统计信息。
 
-可以通过读取 ``stats`` 目录下的文件(``nr_tried``, ``sz_tried``, ``nr_applied``,
-``sz_applied``, 和 ``qt_exceeds``)）分别检索这些统计数据。这些文件不是实时更新的，所以
-你应该要求DAMON sysfs接口通过在相关的 ``kdamonds/<N>/state`` 文件中写入一个特殊的关键字
-``update_schemes_stats`` 来更新统计信息的文件内容。
+默认情况下，这些文件不会实时更新。用户应要求 DAMON sysfs 接口使用
+``refresh_ms`` 周期性地更新这些文件，或者向相关 ``kdamonds/<N>/state`` 文件
+写入特殊关键字 ``update_schemes_stats`` 来进行一次性更新。更多细节请参考
+:ref:`kdamond 目录 <sysfs_kdamond_zh_CN>`。
+
+.. _sysfs_schemes_tried_regions_zh_CN:
 
 schemes/<N>/tried_regions/
 --------------------------
 
-当一个特殊的关键字 ``update_schemes_tried_regions`` 被写入相关的 ``kdamonds/<N>/state``
-文件时，DAMON会在这个目录下创建从 ``0`` 开始命名的整数目录。每个目录包含的文件暴露了关于每个
-内存区域的详细信息，在下一个 :ref:`聚集区间 <sysfs_monitoring_attrs>`，相应的方案的 ``动作``
-已经尝试在这个目录下应用。这些信息包括地址范围、``nr_accesses`` 以及区域的 ``年龄`` 。
+该目录初始时有一个文件 ``total_bytes``。
+
+当特殊关键字 ``update_schemes_tried_regions`` 被写入相关
+``kdamonds/<N>/state`` 文件时，DAMON 会更新 ``total_bytes`` 文件，使读取
+它时返回该方案尝试区域的总大小，并在该目录下创建从 ``0`` 开始以整数命名的
+目录。每个目录包含一些文件，暴露该目录下对应方案的 ``action`` 在对应方案的
+下一个 :ref:`应用间隔 <damon_design_damos_zh_CN>` 内尝试应用的每个内存区域的详细
+信息。这些信息包括该区域的地址范围、``nr_accesses`` 和 ``age``。
+
+向相关 ``kdamonds/<N>/state`` 文件写入 ``update_schemes_tried_bytes`` 只会
+更新 ``total_bytes`` 文件，而不会创建子目录。
+
+当另一个特殊关键字 ``clear_schemes_tried_regions`` 被写入相关
+``kdamonds/<N>/state`` 文件时，这些目录会被删除。
+
+该目录的预期用途是调查方案行为，以及类似查询的高效数据访问监测结果获取。
+特别是对于后一种用例，用户可以将 ``action`` 设置为 ``stat``，并将
+``access pattern`` 设置为他们感兴趣、想要查询的模式。
 
-当另一个特殊的关键字 ``clear_schemes_tried_regions`` 被写入相关的 ``kdamonds/<N>/state``
-文件时，这些目录将被删除。
+.. _sysfs_schemes_tried_region_zh_CN:
 
 tried_regions/<N>/
 ------------------
 
-在每个区域目录中，你会发现四个文件(``start``, ``end``, ``nr_accesses``, and ``age``)。
-读取这些文件将显示相应的基于DAMON的操作方案 ``动作`` 试图应用的区域的开始和结束地址、``nr_accesses``
-和 ``年龄`` 。
+在每个区域目录中，你会发现五个文件（``start``、``end``、``nr_accesses``、
+``age`` 和 ``sz_filter_passed``）。读取这些文件将显示相应的基于DAMON的操作方案
+``action`` 试图应用的区域属性。
+
+tried_regions/<N>/probes/
+-------------------------
+
+在每个区域目录中，还存在一个目录（``probes``）。在该目录中，存在命名为
+``0`` 到 ``N-1`` 的子目录。``N`` 是已安装探针的数量。在每个数字命名的
+目录中，存在一个文件（``hits``）。读取该文件会显示该区域的数据属性监测
+探针命中正样本的数量。
 
 用例
 ~~~~
@@ -330,16 +601,46 @@ tried_regions/<N>/
 请注意，我们强烈建议使用用户空间的工具，如 `damo <https://github.com/damonitor/damo>`_ ，
 而不是像上面那样手动读写文件。以上只是一个例子。
 
+.. _tracepoint_zh_CN:
 
-监测结果的监测点
+监测结果的跟踪点
 ================
 
-DAMON通过一个tracepoint ``damon:damon_aggregated`` 提供监测结果.  当监测开启时，你可
-以记录追踪点事件，并使用追踪点支持工具如perf显示结果。比如说::
+用户可以通过 :ref:`tried_regions <sysfs_schemes_tried_regions_zh_CN>` 获取
+监测结果。该接口适合获取快照，但用于完整记录所有监测结果时可能效率不高。
+为此，提供了两个跟踪点，即 ``damon:damon_aggregated`` 和
+``damon:damos_before_apply``。``damon:damon_aggregated`` 提供完整的监测
+结果，而 ``damon:damos_before_apply`` 提供每个基于 DAMON 的操作方案
+（:ref:`DAMOS <damon_design_damos_zh_CN>`）将要应用到的区域的监测结果。因此，
+``damon:damos_before_apply`` 更适合记录 DAMOS 的内部行为，或者基于 DAMOS
+目标访问 :ref:`模式 <damon_design_damos_access_pattern_zh_CN>` 的类似查询的高效
+监测结果记录。
 
-    # echo on > monitor_on_DEPRECATED
+监测开启时，你可以记录跟踪点事件，并使用跟踪点支持工具如 ``perf`` 显示结果。比如说::
+
+    # echo on > kdamonds/0/state
     # perf record -e damon:damon_aggregated &
     # sleep 5
     # kill 9 $(pidof perf)
-    # echo off > monitor_on_DEPRECATED
+    # echo off > kdamonds/0/state
     # perf script
+    kdamond.0 46568 [027] 79357.842179: damon:damon_aggregated: target_id=0 nr_regions=11 122509119488-135708762112: 0 864
+    [...]
+
+``perf script`` 输出的每一行表示一个监测区域。前五个字段与其他跟踪点输出
+一样。第六个字段 ``target_id=X`` 显示该区域的监测目标 id。第七个字段
+``nr_regions=X`` 显示该目标的监测区域总数。第八个字段 ``X-Y:`` 显示
+该区域的起始地址 ``X`` 和结束地址 ``Y``，单位为字节。第九个字段 ``X``
+显示该区域的 ``nr_accesses`` （关于该计数器的更多细节请参考 :ref:`设计 <damon_design_region_based_sampling_zh_CN>`）。最后，第十个字段 ``X`` 显示该区域
+的 ``age`` （关于该计数器的更多细节请参考 :ref:`设计 <damon_design_age_tracking_zh_CN>`）。
+
+如果事件是 ``damon:damos_before_apply``，``perf script`` 输出大致如下::
+
+    kdamond.0 47293 [000] 80801.060214: damon:damos_before_apply: ctx_idx=0 scheme_idx=0 target_idx=0 nr_regions=11 121932607488-135128711168: 0 136
+    [...]
+
+输出的每一行表示在被跟踪的时间点，每个基于 DAMON 的操作方案即将应用到的
+一个监测区域。前五个字段与通常一样。除了 ``damon_aggregated`` 跟踪点的
+输出之外，它还显示方案所属 DAMON 上下文在该上下文的 kdamond 的上下文列表
+中的索引（``ctx_idx=X``），以及该方案在该上下文的方案列表中的索引
+（``scheme_idx=X``）。
diff --git a/Documentation/translations/zh_CN/mm/damon/design.rst b/Documentation/translations/zh_CN/mm/damon/design.rst
index 16e3db34a7dd18..b7790bcea9b903 100644
--- a/Documentation/translations/zh_CN/mm/damon/design.rst
+++ b/Documentation/translations/zh_CN/mm/damon/design.rst
@@ -13,35 +13,83 @@
 设计
 ====
 
-可配置的层
-==========
 
-DAMON提供了数据访问监控功能，同时使其准确性和开销可控。基本的访问监控需要依赖于目标地址空间
-并为之优化的基元。另一方面，作为DAMON的核心，准确性和开销的权衡机制是在纯逻辑空间中。DAMON
-将这两部分分离在不同的层中，并定义了它的接口，以允许各种低层次的基元实现与核心逻辑的配置。
+.. _damon_design_execution_model_and_data_structures_zh_CN:
 
-由于这种分离的设计和可配置的接口，用户可以通过配置核心逻辑和适当的低级基元实现来扩展DAMON的
-任何地址空间。如果没有提供合适的，用户可以自己实现基元。
+执行模型和数据结构
+==================
 
-例如，物理内存、虚拟内存、交换空间、那些特定的进程、NUMA节点、文件和支持的内存设备将被支持。
-另外，如果某些架构或设备支持特殊的优化访问检查基元，这些基元将很容易被配置。
+与监测相关的信息，包括监测请求规格和基于 DAMON 的操作方案，都存储在名为
+DAMON ``context`` 的数据结构中。DAMON 使用名为 ``kdamond`` 的内核线程执行
+每个上下文。多个 kdamond 可以并行运行，用于不同类型的监测。
 
+要了解用户空间如何进行配置并启动或停止 DAMON，请参考 :ref:`DAMON sysfs
+接口 <sysfs_interface_zh_CN>` 文档。
 
-特定地址空间基元的参考实现
-==========================
+用户还可以请求暂停和恢复每个上下文的执行。当上下文被暂停时，kdamond 除了应用
+在线参数更新外不做任何事情。
 
-基本访问监测的低级基元被定义为两部分。:
+要了解用户空间如何暂停或恢复每个上下文，请参考 :ref:`DAMON sysfs 上下文
+<sysfs_context_zh_CN>` 使用文档。
 
-1. 确定地址空间的监测目标地址范围
-2. 目标空间中特定地址范围的访问检查。
+总体架构
+========
 
-DAMON目前为物理和虚拟地址空间提供了基元的实现。下面两个小节描述了这些工作的方式。
+DAMON 子系统由三层构成，包括
 
+- :ref:`操作集 <damon_operations_set_zh_CN>`：实现依赖于给定监测目标地址空间
+  和可用软硬件基元集合的 DAMON 基本操作，
+- :ref:`核心 <damon_core_logic_zh_CN>`：在操作集层之上，实现包括监测开销/准确性
+  控制和访问感知系统操作在内的核心逻辑，以及
+- :ref:`模块 <damon_modules_zh_CN>`：在核心层之上，实现用于各种目的、并向用户空间
+  提供接口的内核模块。
+
+
+.. _damon_operations_set_zh_CN:
+
+操作集层
+========
+
+.. _damon_design_configurable_operations_set_zh_CN:
+
+为了进行数据访问监测和其他低层工作，DAMON 需要一组针对给定目标地址空间、
+并依赖于和优化于该地址空间的特定操作实现。例如，下面两个用于访问监测的操作
+依赖于地址空间。
+
+1. 确定该地址空间的监测目标地址范围。
+2. 检查目标空间中特定地址范围的访问。
+
+DAMON 将这些实现整合到称为 DAMON 操作集的层中，并定义它和上层之间的接口。
+上层专用于 DAMON 的核心逻辑，包括控制监测准确性和开销的机制。
+
+因此，DAMON 可以通过配置核心逻辑使用适当的操作集，轻松扩展到任意地址空间
+和/或可用硬件功能。如果给定目的没有可用操作集，也可以按照层间接口实现新的
+操作集。
+
+例如，物理内存、虚拟内存、交换空间、特定进程、NUMA 节点、文件和后端内存设备
+都可以被支持。另外，如果某些架构或设备支持特殊的优化访问检查功能，也可以很容易
+进行配置。
+
+DAMON 当前提供以下三个操作集。下面三个小节描述它们如何工作。
+
+ - vaddr：监测特定进程的虚拟地址空间
+ - fvaddr：监测固定虚拟地址范围
+ - paddr：监测系统的物理地址空间
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 进行配置，
+请参考文档的 :ref:`operations <sysfs_context_zh_CN>` 文件部分。
+
+
+.. _damon_design_vaddr_target_regions_construction_zh_CN:
 
 基于VMA的目标地址范围构造
 -------------------------
 
-这仅仅是针对虚拟地址空间基元的实现。对于物理地址空间，只是要求用户手动设置监控目标地址范围。
+``vaddr`` DAMON 操作集的一种机制，会自动初始化并更新监测目标地址区域，
+以覆盖目标进程的整个内存映射。
+
+该机制仅用于 ``vaddr`` 操作集。对于 ``fvaddr`` 和 ``paddr`` 操作集，
+用户需要手动设置监测目标地址范围。
 
 在进程的超级巨大的虚拟地址空间中，只有小部分被映射到物理内存并被访问。因此，跟踪未映射的地
 址区域只是一种浪费。然而，由于DAMON可以使用自适应区域调整机制来处理一定程度的噪声，所以严
@@ -50,7 +98,7 @@ DAMON目前为物理和虚拟地址空间提供了基元的实现。下面两个
 
 出于这个原因，这个实现将复杂的映射转换为三个不同的区域，覆盖地址空间的每个映射区域。这三个
 区域之间的两个空隙是给定地址空间中两个最大的未映射区域。这两个最大的未映射区域是堆和最上面
-的mmap()区域之间的间隙，以及在大多数情况下最下面的mmap()区域和堆之间的间隙。因为这些间隙
+的mmap()区域之间的间隙，以及在大多数情况下最下面的mmap()区域和栈之间的间隙。因为这些间隙
 在通常的地址空间中是异常巨大的，排除这些间隙就足以做出合理的权衡。下面详细说明了这一点::
 
     <heap>
@@ -69,23 +117,52 @@ DAMON目前为物理和虚拟地址空间提供了基元的实现。下面两个
 找到相关的PTE访问位的方式。虚拟地址的实现是为该地址的目标任务查找页表，而物理地址的实现则
 是查找与该地址有映射关系的每一个页表。通过这种方式，实现者找到并清除下一个采样目标地址的位，
 并检查该位是否在一个采样周期后再次设置。这可能会干扰其他使用访问位的内核子系统，即空闲页跟
-踪和回收逻辑。为了避免这种干扰，DAMON使其与空闲页面跟踪相互排斥，并使用 ``PG_idle`` 和
-``PG_young`` 页面标志来解决与回收逻辑的冲突，就像空闲页面跟踪那样。
+踪和回收逻辑。DAMON 不做任何事情来避免干扰空闲页跟踪，因此处理这种干扰是系统管理员的责任。
+不过，它会像空闲页跟踪一样，使用 ``PG_idle`` 和 ``PG_young`` 页面标志解决与回收逻辑的冲突。
+
+
+.. _damon_design_addr_unit_zh_CN:
+
+地址单位
+--------
 
+DAMON 核心层使用 ``unsigned long`` 类型表示监测目标地址范围。在某些情况下，
+给定操作集的地址空间可能过大，无法用该类型处理。带有大物理地址扩展的
+ARM（32 位）就是一个例子。对于这类情况，提供了一个称为 ``address unit`` 的
+逐操作集参数。它表示一个缩放因子，需要乘以核心层地址，才能计算给定地址
+空间中的真实地址。是否支持 ``address unit`` 参数取决于每个操作集实现。
+``paddr`` 是唯一支持该参数的操作集实现。
 
-独立于地址空间的核心机制
-========================
+如果该值小于 ``PAGE_SIZE``，则只能使用 2 的幂。
+
+
+.. _damon_core_logic_zh_CN:
+
+核心逻辑
+========
+
+.. _damon_design_monitoring_zh_CN:
+
+监测
+----
 
 下面四个部分分别描述了DAMON的核心机制和五个监测属性，即 ``采样间隔`` 、 ``聚集间隔`` 、
 ``更新间隔`` 、 ``最小区域数`` 和 ``最大区域数`` 。
 
+请注意，``最小区域数`` 必须为 3 或更高。这是因为虚拟地址空间监测被设计为至少处理三个区域，
+以容纳普通虚拟地址空间中常见的两个大型未映射区域。虽然对 ``paddr`` 这类其他操作集来说，
+这一限制可能并非严格必要，但为了保持一致性，目前会对所有 DAMON 操作强制执行。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置这些属性，
+请参考文档的 :ref:`monitoring_attrs <sysfs_monitoring_attrs_zh_CN>` 部分。
+
 
 访问频率监测
-------------
+~~~~~~~~~~~~
 
 DAMON的输出显示了在给定的时间内哪些页面的访问频率是多少。访问频率的分辨率是通过设置
 ``采样间隔`` 和 ``聚集间隔`` 来控制的。详细地说，DAMON检查每个 ``采样间隔`` 对每
-个页面的访问，并将结果汇总。换句话说，计算每个页面的访问次数。在每个 ``聚合间隔`` 过
+个页面的访问，并将结果汇总。换句话说，计算每个页面的访问次数。在每个 ``聚集间隔`` 过
 去后，DAMON调用先前由用户注册的回调函数，以便用户可以阅读聚合的结果，然后再清除这些结
 果。这可以用以下简单的伪代码来描述::
 
@@ -103,38 +180,564 @@ DAMON的输出显示了在给定的时间内哪些页面的访问频率是多少
 这种机制的监测开销将随着目标工作负载规模的增长而任意增加。
 
 
+.. _damon_design_region_based_sampling_zh_CN:
+
 基于区域的抽样调查
-------------------
+~~~~~~~~~~~~~~~~~~
 
 为了避免开销的无限制增加，DAMON将假定具有相同访问频率的相邻页面归入一个区域。只要保持
 这个假设（一个区域内的页面具有相同的访问频率），该区域内就只需要检查一个页面。因此，对
 于每个 ``采样间隔`` ，DAMON在每个区域中随机挑选一个页面，等待一个 ``采样间隔`` ，检
-查该页面是否同时被访问，如果被访问则增加该区域的访问频率。因此，监测开销是可以通过设置
-区域的数量来控制的。DAMON允许用户设置最小和最大的区域数量来进行权衡。
+查该页面是否同时被访问，如果被访问则增加该区域的访问频率计数器。该计数器称为区域的
+``nr_accesses``。因此，监测开销是可以通过设置区域的数量来控制的。DAMON允许用户设置最小
+和最大的区域数量来进行权衡。
 
 然而，如果假设没有得到保证，这个方案就不能保持输出的质量。
 
 
+.. _damon_design_adaptive_regions_adjustment_zh_CN:
+
 适应性区域调整
---------------
+~~~~~~~~~~~~~~
 
 即使最初的监测目标区域被很好地构建以满足假设（同一区域内的页面具有相似的访问频率），数
 据访问模式也会被动态地改变。这将导致监测质量下降。为了尽可能地保持假设，DAMON根据每个
 区域的访问频率自适应地进行合并和拆分。
 
-对于每个 ``聚集区间`` ，它比较相邻区域的访问频率，如果频率差异较小，就合并这些区域。
-然后，在它报告并清除每个区域的聚合接入频率后，如果区域总数不超过用户指定的最大区域数，
-它将每个区域拆分为两个或三个区域。
+对于每个 ``聚集间隔`` ，它比较相邻区域的访问频率（``nr_accesses``）。如果差异较小，
+并且两个区域的大小之和小于总区域大小除以 ``最小区域数``，DAMON 就会合并这两个区域。如果
+合并后的总区域数仍然高于 ``最大区域数``，它会增大访问频率差异阈值并重复合并，直到满足区域
+数上限，或者阈值高于可能的最大值（``聚集间隔`` 除以 ``采样间隔``）。然后，在它报告并清除
+每个区域的聚合访问频率后，如果拆分后的区域总数不超过用户指定的最大区域数，它会把每个区域
+拆分为两个或三个区域。
 
 通过这种方式，DAMON提供了其最佳的质量和最小的开销，同时保持了用户为其权衡设定的界限。
 
+.. _damon_design_age_tracking_zh_CN:
+
+年龄跟踪
+~~~~~~~~
+
+通过分析监测结果，用户还可以发现某个区域当前的访问模式已经保持了多长时间。这可用于更好地理解
+访问模式。例如，可以利用访问频率和时近性实现页面放置算法。为了让这种访问模式保持时间分析更容
+易，DAMON 在每个区域中维护另一个名为 ``age`` 的计数器。对于每个 ``聚集间隔``，DAMON 检查该
+区域的大小和访问频率（``nr_accesses``）是否发生了显著变化。如果发生了变化，该计数器会被重置为
+零。否则，该计数器会增加。
+
+
+.. _damon_design_data_attrs_monitoring_zh_CN:
+
+数据属性监测
+~~~~~~~~~~~~
+
+数据访问模式只是数据属性的一种类型。在某些用例中，用户需要了解更多数据属性
+信息。例如，用户可能需要知道给定热或冷内存区域中有多少由匿名页支持，或者属于
+某个特定 cgroup。为此，提供了数据属性监测功能。
+
+使用该功能时，用户可以把感兴趣的数据属性注册到 DAMON :ref:`上下文
+<damon_design_execution_model_and_data_structures_zh_CN>` 中。注册通过为每个属性
+指定一个探针完成。每个探针指定一个规则，用于判断给定内存区域是否具有相关
+属性。该规则由多个过滤器构成。除支持的过滤器类型不同外，这些过滤器的工作方式
+与 :ref:`DAMOS 过滤器 <damon_design_damos_filters_zh_CN>` 相同。目前，数据属性
+监测只支持 ``anon`` 和 ``memcg`` 过滤器类型。
+
+如果注册了这类探针，DAMON 在进行访问 :ref:`采样
+<damon_design_region_based_sampling_zh_CN>` 时，会对每个区域的采样内存执行这些
+探针。每个 :ref:`聚集间隔 <damon_design_monitoring_zh_CN>` 中被识别为具有数据
+属性（命中探针）的样本数量，会计入每区域、每探针的计数器。因此，用户可以在
+每个聚集间隔后读取每区域、每探针的探针命中计数器，从而了解给定 DAMON 区域
+有多少具有特定数据属性。
+
+这是基于采样的机制。因此，它很轻量，但输出可能包含一些测量误差。用户应在
+充分理解统计意义的基础上使用输出。
+
+另一种更高精度的方式是使用 ``stat`` :ref:`动作
+<damon_design_damos_action_zh_CN>` 的 :ref:`DAMOS 过滤器
+<damon_design_damos_filters_zh_CN>`，并使用 ``sz_ops_filter_passed`` :ref:`统计
+<damon_design_damos_stat_zh_CN>`。这种方式以页级别提供数据属性信息。不过，由于
+它在页级别操作，开销与内存大小成正比。
 
 动态目标空间更新处理
---------------------
+~~~~~~~~~~~~~~~~~~~~
 
 监测目标地址范围可以动态改变。例如，虚拟内存可以动态地被映射和解映射。物理内存可以被
 热插拔。
 
 由于在某些情况下变化可能相当频繁，DAMON允许监控操作检查动态变化，包括内存映射变化，
 并仅在用户指定的时间间隔（ ``更新间隔`` ）中的每个时间段，将其应用于监控操作相关的
-数据结构，如抽象的监控目标内存区。
\ No newline at end of file
+数据结构，如抽象的监控目标内存区。
+
+用户空间可以通过 DAMON sysfs 接口和/或跟踪点获取监测结果。更多细节请分别参考
+:ref:`DAMOS 尝试区域 <sysfs_schemes_tried_regions_zh_CN>` 和 :ref:`跟踪点 <tracepoint_zh_CN>`
+文档。
+
+
+.. _damon_design_monitoring_params_tuning_guide_zh_CN:
+
+监测参数调优指南
+~~~~~~~~~~~~~~~~
+
+简而言之，应设置 ``聚集间隔``，使其能够为使用目的捕获有意义数量的访问。访问数量可以用聚集后
+监测结果快照中各区域的 ``nr_accesses`` 和 ``age`` 来衡量。该间隔的默认值 ``100ms`` 在许多情况
+下被证明过短。应按 ``聚集间隔`` 的比例设置 ``采样间隔``。默认推荐比例为 ``1/20``。
+
+``聚集间隔`` 应设置为工作负载可在该间隔内为监测目的产生一定数量访问的时间间隔。如果该间隔过短，
+只能捕获少量访问。结果是，监测结果会看起来像所有内容都同样只是很少被访问。对许多目的而言，这
+将毫无用处。不过，如果该间隔过长，根据给定目的的时间尺度，区域通过 :ref:`区域调整机制
+<damon_design_adaptive_regions_adjustment_zh_CN>` 收敛所需的时间可能过长。如果工作负载实际只产生
+很少访问，而用户却认为监测目的所需的访问数量很高，就可能发生这种情况。对于这种情况，应仔细重
+新考虑每个 ``聚集间隔`` 要捕获的目标访问数量。还要注意，捕获的访问数量不仅用 ``nr_accesses``
+表示，也用 ``age`` 表示。例如，即使监测结果中的每个区域都显示 ``nr_accesses`` 为零，仍然可以
+用 ``age`` 值作为时近性信息来区分区域。
+
+因此，``聚集间隔`` 的最佳值取决于工作负载的访问密集程度。用户应根据每个聚集后的监测结果快照
+中捕获的访问数量来调优该间隔。
+
+请注意，该间隔的默认值是 100 毫秒，在许多情况下都太短，尤其是在大型系统上。
+
+``采样间隔`` 定义每次聚集的分辨率。如果它设置得过大，监测结果会看起来像每个区域都同样很少被
+访问，或者同样频繁地被访问。也就是说，区域将无法根据访问模式区分，因此结果在许多用例中都会无
+用。如果 ``采样间隔`` 过小，它不会降低分辨率，但会增加监测开销。如果它已经足以为给定目的提供
+足够的监测结果分辨率，就不应再不必要地降低。建议将它按 ``聚集间隔`` 的比例设置。默认比例设为
+``1/20``，并且仍然推荐该比例。
+
+基于手动调优指南，DAMON 提供了更直观的、基于调节项的间隔自动调优机制。更多细节请参考
+:ref:`该功能的设计文档 <damon_design_monitoring_intervals_autotuning_zh_CN>`。
+
+基于上述指南的示例调优，请参考英文文档 Documentation/mm/damon/monitoring_intervals_tuning_example.rst。
+
+
+.. _damon_design_monitoring_intervals_autotuning_zh_CN:
+
+监测间隔自动调优
+~~~~~~~~~~~~~~~~
+
+DAMON 基于 :ref:`调优指南的思路 <damon_design_monitoring_params_tuning_guide_zh_CN>`，提供了
+对 ``采样间隔`` 和 ``聚集间隔`` 的自动调优机制。该调优机制允许用户设置希望 DAMON 在给定时间间隔
+内观测到的访问事件数量目标。用户可以把该目标指定为 DAMON 观测到的访问事件数量与理论最大事件数
+量之间的比例（``access_bp``），该比例在给定数量的聚集中测量（``aggrs``）。
+
+DAMON 观测到的访问事件基于 DAMON :ref:`区域假设 <damon_design_region_based_sampling_zh_CN>`
+按字节粒度计算。例如，如果发现大小为 ``X`` 字节、``nr_accesses`` 为 ``Y`` 的区域，就意味着
+DAMON 观测到了 ``X * Y`` 个访问事件。该区域的理论最大访问事件也以相同方式计算，但会把 ``Y``
+替换为理论最大 ``nr_accesses``，即 ``聚集间隔 / 采样间隔``。
+
+该机制会计算 ``aggrs`` 次聚集期间的访问事件比例。如果观测到的访问比例低于或高于目标值，就按
+相同比例增大或减小 ``采样间隔`` 和 ``聚集间隔``。间隔变化比例按当前采样比例与目标比例之间的
+距离决定。
+
+用户还可以通过两个参数（``min_sample_us`` 和 ``max_sample_us``）进一步设置调优机制可设置的
+最小和最大 ``采样间隔``。由于调优机制总是以相同比例改变 ``采样间隔`` 和 ``聚集间隔``，所以
+每次调优变化后的最小和最大 ``聚集间隔`` 也可以自动一起设置。
+
+该调优默认关闭，需要由用户显式设置。根据经验法则和帕累托原则，推荐使用 4% 的访问样本比例目标。
+请注意，这里应用了两次帕累托原则（80/20 规则）。也就是说，假设以 4%（20% 的 20%）的 DAMON
+观测访问事件比例（来源），捕获 64%（80% 乘以 80%）的真实访问事件（结果）。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 使用该功能，请参考文档
+的 :ref:`intervals_goal <damon_usage_sysfs_monitoring_intervals_goal_zh_CN>` 部分。
+
+
+.. _damon_design_damos_zh_CN:
+
+操作方案
+--------
+
+数据访问监测的一个常见目的，是实现访问感知的系统效率优化。例如：
+
+    换出超过两分钟未被访问的内存区域
+
+或者：
+
+    对大于 2 MiB 且显示高访问频率超过一分钟的内存区域使用 THP。
+
+这类方案的一种直接做法是基于剖析的优化。也就是说，使用 DAMON 获取工作负载
+或系统的数据访问监测结果，通过分析监测结果找到具有特殊特征的内存区域，并针对
+这些区域进行系统操作变更。这些变更可以通过修改软件（应用程序和/或内核）或向
+其提供建议来完成，也可以通过重新配置硬件来完成。离线和在线两种方式都可以使用。
+
+其中，在运行时向内核提供建议会比较灵活且有效，因此被广泛使用。不过，实现这类
+方案可能引入不必要的冗余和低效。如果关注的类型很常见，剖析可能是冗余的。在
+内核和用户空间之间交换包括监测结果和操作建议在内的信息，也可能效率较低。
+
+为了让用户通过移交这些工作来减少这种冗余和低效，DAMON 提供了一个名为基于
+数据访问监测的操作方案（DAMOS）的功能。它允许用户以高层次指定期望的方案。
+对于这类规格，DAMON 会开始监测，找到具有感兴趣访问模式的区域，并在每个用户
+指定的时间间隔（称为 ``apply_interval``）把用户期望的操作动作应用于这些区域。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置
+``apply_interval``，请参考文档的 :ref:`apply_interval_us <sysfs_scheme_zh_CN>`
+部分。
+
+
+.. _damon_design_damos_action_zh_CN:
+
+操作动作
+~~~~~~~~
+
+用户希望应用到其感兴趣区域的管理动作。例如，换出、为下一次回收受害者选择提高
+优先级、建议 ``khugepaged`` 折叠或拆分，或者什么也不做而只收集区域统计信息。
+
+支持的动作列表在 DAMOS 中定义，但每个动作的实现位于 DAMON 操作集层，因为实现
+通常依赖于监测目标地址空间。例如，换出特定虚拟地址范围的代码会不同于换出物理
+地址范围的代码。监测操作实现集也不被要求支持列表中的所有动作。因此，特定 DAMOS
+动作是否可用取决于选择一起使用的操作集。
+
+支持的动作列表、含义以及支持每个动作的 DAMON 操作集如下。
+
+ - ``willneed``：对带有 ``MADV_WILLNEED`` 的区域调用 ``madvise()``。
+   ``vaddr`` 和 ``fvaddr`` 操作集支持。
+ - ``cold``：对带有 ``MADV_COLD`` 的区域调用 ``madvise()``。
+   ``vaddr`` 和 ``fvaddr`` 操作集支持。
+ - ``pageout``：回收该区域。
+   ``vaddr``、``fvaddr`` 和 ``paddr`` 操作集支持。
+ - ``hugepage``：对带有 ``MADV_HUGEPAGE`` 的区域调用 ``madvise()``。
+   ``vaddr`` 和 ``fvaddr`` 操作集支持。当禁用 TRANSPARENT_HUGEPAGE 时，
+   应用该动作只会失败。
+ - ``nohugepage``：对带有 ``MADV_NOHUGEPAGE`` 的区域调用 ``madvise()``。
+   ``vaddr`` 和 ``fvaddr`` 操作集支持。当禁用 TRANSPARENT_HUGEPAGE 时，
+   应用该动作只会失败。
+ - ``collapse``：对带有 ``MADV_COLLAPSE`` 的区域调用 ``madvise()``。
+   ``vaddr`` 和 ``fvaddr`` 操作集支持。当禁用 TRANSPARENT_HUGEPAGE 时，
+   应用该动作只会失败。
+ - ``lru_prio``：提高该区域在其 LRU 链表上的优先级。
+   ``paddr`` 操作集支持。
+ - ``lru_deprio``：降低该区域在其 LRU 链表上的优先级。
+   ``paddr`` 操作集支持。
+ - ``migrate_hot``：迁移区域，并优先迁移更热的区域。
+   ``vaddr``、``fvaddr`` 和 ``paddr`` 操作集支持。
+ - ``migrate_cold``：迁移区域，并优先迁移更冷的区域。
+   ``vaddr``、``fvaddr`` 和 ``paddr`` 操作集支持。
+ - ``stat``：什么也不做，只统计信息。
+   所有操作集都支持。
+
+把除 ``stat`` 外的动作应用到某个区域，会被认为改变了该区域的特征。因此，
+当任何这类动作被应用到区域时，DAMOS 会重置这些区域的 age。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置
+动作，请参考文档的 :ref:`action <sysfs_scheme_zh_CN>` 部分。
+
+
+.. _damon_design_damos_access_pattern_zh_CN:
+
+目标访问模式
+~~~~~~~~~~~~
+
+方案感兴趣的访问模式。这些模式由 DAMON 监测结果提供的属性构成，具体包括
+大小、访问频率和年龄。用户可以通过设置这三个属性的最小值和最大值，描述
+自己感兴趣的访问模式。如果某个区域的这三个属性都在范围内，DAMOS 就把它
+分类为该方案感兴趣的区域之一。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置
+访问模式，请参考文档的 :ref:`access_pattern <sysfs_access_pattern_zh_CN>`
+部分。
+
+
+.. _damon_design_damos_quotas_zh_CN:
+
+配额
+~~~~
+
+DAMOS 的开销上界控制功能。如果目标访问模式没有正确调优，DAMOS 可能带来高开销。
+例如，如果发现一个具有感兴趣访问模式的巨大内存区域，把方案动作应用到该巨大区域
+的所有页面可能消耗不可接受的大量系统资源。通过调优访问模式来防止这类问题可能
+很有挑战，特别是在工作负载的访问模式高度动态时。
+
+为缓解这种情况，DAMOS 提供了一个称为配额的开销上界控制功能。它允许用户指定
+DAMOS 可用于应用动作的时间上限，和/或在用户指定的时间段内可应用动作的最大内存
+区域字节数。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置基本
+配额，请参考文档的 :ref:`quotas <sysfs_quotas_zh_CN>` 部分。
+
+
+.. _damon_design_damos_quotas_prioritization_zh_CN:
+
+优先级
+^^^^^^
+
+一种在配额限制下作出良好决策的机制。当由于配额限制而无法把动作应用到所有感兴趣
+区域时，DAMOS 会对区域排序，并只把动作应用到优先级足够高、且不会超过配额的区域。
+
+每个动作的优先级机制应该不同。例如，很少被访问（更冷）的内存区域应在 page-out
+方案动作中被优先处理。相反，对于大页折叠方案动作，更冷的区域应被降低优先级。
+因此，每个动作的优先级机制与动作一起，在各个 DAMON 操作集中实现。
+
+虽然实现取决于 DAMON 操作集，但通常会使用区域的访问模式属性来计算优先级。有些
+用户可能希望针对自己的特定场景个性化这些机制。例如，有些用户可能希望该机制更重视
+时近性（``age``）而不是访问频率（``nr_accesses``）。DAMOS 允许用户指定每个访问模式
+属性的权重，并把这些信息传递给底层机制。不过，权重如何被尊重，甚至是否被尊重，
+都取决于底层优先级机制实现。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置优先级
+权重，请参考文档的 :ref:`weights <sysfs_quotas_zh_CN>` 部分。
+
+
+.. _damon_design_damos_quotas_failed_memory_charging_ratio_zh_CN:
+
+动作失败内存计费比率
+^^^^^^^^^^^^^^^^^^^^
+
+对给定区域执行 DAMOS 动作时，区域中某些内存子集可能失败。例如，如果动作是
+``pageout``，而该区域包含一些不可回收页，则把该动作应用到这些页会失败。
+这类失败动作应用消耗的系统资源量通常不同于成功动作应用。对于这类情况，用户可以
+为失败内存设置不同的计费比率。该比率可以用 ``fail_charge_num`` 和
+``fail_charge_denom`` 参数指定。这两个参数分别表示比率的分子和分母。只有当
+``fail_charge_denom`` 不为零时，该功能才启用。
+
+例如，假设某个 DAMOS 动作被应用到大小为 1,000 MiB 的区域。该动作只成功应用到
+该区域的 700 MiB。``fail_charge_num`` 和 ``fail_charge_denom`` 分别设置为 ``1``
+和 ``1024``。那么只会计费 700 MiB 和 300 KiB 的大小（``700 MiB + 300 MiB * 1 /
+1024``）。
+
+
+.. _damon_design_damos_quotas_auto_tuning_zh_CN:
+
+面向目标的反馈驱动自动调优
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+自动反馈驱动的配额调优。用户可以不设置绝对配额值，而是指定自己感兴趣的指标，
+以及希望该指标达到的目标值。随后，DAMOS 会自动调优对应方案的激进程度（配额）。
+例如，如果 DAMOS 未达到目标，DAMOS 会自动增加配额。如果 DAMOS 超过目标，则会
+减少配额。
+
+用户可以按需要选择两种这样的调优算法。
+
+- ``consist``：基于比例反馈环的算法。尝试找到一个应持续保持的最佳配额，以便持续
+  达成目标。适用于动态和长时间运行环境中的内核内部操作。这是默认选择。如果不确定，
+  请使用它。
+- ``temporal``：更直接的算法。尝试尽快达成目标，使用允许的最大配额，但只持续短暂
+  时间。当配额未达成时，该算法持续把配额调优到允许的最大值。一旦配额[超过]达成，
+  它就把配额设置为零。适用于需要确定性控制的环境。
+
+目标可以用五个参数指定，即 ``target_metric``、``target_value``、``current_value``、
+``nid`` 和 ``path``。自动调优机制尝试使 ``target_metric`` 的 ``current_value``
+等于 ``target_value``。
+
+- ``user_input``：用户提供的值。用户可以使用任何自己感兴趣的指标作为该值。用户空间
+  主工作负载的延迟或吞吐量、空闲内存率或内存压力停滞时间（PSI）等系统指标都可以
+  作为示例。请注意，在这种情况下，用户应自己显式设置 ``current_value``。换言之，
+  用户应反复提供反馈。
+- ``some_mem_psi_us``：系统范围的 ``some`` 内存压力停滞信息，单位为微秒，从上一次
+  配额重置到下一次配额重置之间测量。DAMOS 自行进行测量，所以用户只需要在初始时
+  设置 ``target_value``。换言之，DAMOS 会自反馈。
+- ``node_mem_used_bp``：特定 NUMA 节点的已用内存比率，单位为 bp（1/10,000）。
+- ``node_mem_free_bp``：特定 NUMA 节点的空闲内存比率，单位为 bp（1/10,000）。
+- ``node_memcg_used_bp``：特定 cgroup 在特定 NUMA 节点上的节点已用内存比率，
+  单位为 bp（1/10,000）。
+- ``node_memcg_free_bp``：特定 cgroup 在特定 NUMA 节点上的节点未用内存比率，
+  单位为 bp（1/10,000）。
+- ``active_mem_bp``：活跃内存相对活跃 + 非活跃（LRU）内存大小的比率，单位为 bp
+  （1/10,000）。
+- ``inactive_mem_bp``：非活跃内存相对活跃 + 非活跃（LRU）内存大小的比率，单位为 bp
+  （1/10,000）。
+- ``node_eligible_mem_bp``：节点中符合方案目标访问模式条件的内存比率，单位为 bp
+  （1/10,000）。
+
+``nid`` 仅在使用 ``node_mem_used_bp``、``node_mem_free_bp``、
+``node_memcg_used_bp``、``node_memcg_free_bp`` 和 ``node_eligible_mem_bp`` 指标时
+必须提供，用于指定具体 NUMA 节点。
+
+``path`` 仅在使用 ``node_memcg_used_bp`` 和 ``node_memcg_free_bp`` 指标时必须提供，
+用于指定 cgroup 路径。该值应为从 cgroup 挂载点起算的内存 cgroup 路径。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置调优
+目标指标、目标值和/或当前值，请参考文档的 :ref:`quota goals
+<sysfs_schemes_quota_goals_zh_CN>` 部分。
+
+
+.. _damon_design_damos_watermarks_zh_CN:
+
+水位
+~~~~
+
+条件式 DAMOS（去）激活自动化。用户可能希望 DAMOS 只在特定情况下运行。例如，
+当保证有足够空闲内存时，运行主动回收方案只会消耗不必要的系统资源。为避免这类
+消耗，用户需要手动监测一些指标（例如空闲内存率），并打开或关闭 DAMON/DAMOS。
+
+DAMOS 允许用户使用三个水位移交这类工作。它允许用户配置自己感兴趣的指标，以及
+三个水位值，即高、中和低。如果指标值高于高水位或低于低水位，该方案会被停用。
+如果指标值低于中水位但高于低水位，该方案会被激活。如果所有方案都被水位停用，
+监测也会被停用。在这种情况下，DAMON 工作线程只会周期性地检查水位，因此几乎
+不会产生开销。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置水位，
+请参考文档的 :ref:`watermarks <sysfs_watermarks_zh_CN>` 部分。
+
+
+.. _damon_design_damos_filters_zh_CN:
+
+过滤器
+~~~~~~
+
+非访问模式的目标内存区域过滤。如果用户运行自己编写的程序或拥有好的剖析工具，
+他们可能比内核知道更多信息，例如未来的访问模式或针对特定内存类型的一些特殊需求。
+例如，一些用户可能知道只有匿名页会影响其程序性能。他们也可以拥有一组延迟关键进程
+列表。
+
+为了让用户用这类特殊知识优化 DAMOS 方案，DAMOS 提供了一个称为 DAMOS 过滤器的功能。
+该功能允许用户为每个方案设置任意数量的过滤器。每个过滤器指定：
+
+- 一种内存类型（``type``），
+- 它是针对该类型的内存，还是针对该类型以外的所有内存（``matching``），以及
+- 是否允许（包含）或拒绝（排除）把方案动作应用到该内存（``allow``）。
+
+为了高效处理过滤器，一些类型的过滤器由核心层处理，而其他类型由操作集处理。因此，
+在后一种情况下，是否支持过滤器类型取决于 DAMON 操作集。对于核心层处理的过滤器，
+被过滤器排除的内存区域不会计入方案已尝试应用的区域。相反，如果内存区域被操作集层
+处理的过滤器过滤，它会计入方案已尝试。这种差异会影响统计信息。
+
+安装多个过滤器时，由核心层处理的一组过滤器首先求值。之后，由操作层处理的一组过滤器
+求值。每组过滤器内部按安装顺序求值。如果某部分内存匹配某个过滤器，后续过滤器会被
+忽略。如果该部分因为没有匹配任何过滤器而通过过滤器求值阶段，则是否对其应用方案动作
+取决于最后一个过滤器的 allow 类型。如果最后一个过滤器是允许型，该部分内存会被拒绝，
+反之亦然。
+
+例如，假设按顺序安装了 1）一个允许匿名页的过滤器，和 2）另一个拒绝年轻页的过滤器。
+如果某个符合方案动作应用条件的区域中的页面是匿名页，不论它是否年轻，方案动作都会应用
+到该页，因为它匹配第一个允许过滤器。如果该页不是匿名页但年轻，则方案动作不会应用，
+因为第二个拒绝过滤器阻止了它。如果该页既不是匿名页也不年轻，由于没有匹配过滤器，
+该页会通过过滤器求值阶段，并且动作会应用到该页。
+
+当前支持以下 ``type`` 的过滤器。
+
+- 核心层处理
+    - addr
+        - 应用于属于给定地址范围的页面。
+    - target
+        - 应用于属于给定 DAMON 监测目标的页面。
+- 操作层处理，只有 ``paddr`` 操作集支持。
+    - anon
+        - 应用于包含未存储在文件中的数据的页面。
+    - active
+        - 应用于活跃页。
+    - memcg
+        - 应用于属于给定 cgroup 的页面。
+    - young
+        - 应用于自方案上次访问检查以来被访问过的页面。
+    - hugepage_size
+        - 应用于以给定大小范围管理的页面。
+    - unmapped
+        - 应用于未映射的页面。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 设置过滤器，
+请参考文档的 :ref:`filters <sysfs_filters_zh_CN>` 部分。
+
+.. _damon_design_damos_stat_zh_CN:
+
+统计信息
+~~~~~~~~
+
+DAMOS 行为的统计信息，用于帮助监测、调优和调试 DAMOS。
+
+DAMOS 从方案执行开始起，为每个方案统计以下信息。
+
+- ``nr_tried``：方案尝试应用的区域总数。
+- ``sz_tried``：方案尝试应用的区域总大小。
+- ``sz_ops_filter_passed``：通过操作集层处理的 DAMOS 过滤器的总字节数。
+- ``nr_applied``：方案已应用的区域总数。
+- ``sz_applied``：方案已应用的区域总大小。
+- ``qt_exceeds``：方案配额被超过的总次数。
+- ``nr_snapshots``：方案尝试应用的 DAMON 快照总数。
+- ``max_nr_snapshots``：``nr_snapshots`` 的上限。
+
+“方案尝试应用到某个区域”表示 DAMOS 核心逻辑判断该区域符合应用方案
+:ref:`动作 <damon_design_damos_action_zh_CN>` 的条件。核心逻辑处理的
+:ref:`访问模式 <damon_design_damos_access_pattern_zh_CN>`、:ref:`配额
+<damon_design_damos_quotas_zh_CN>`、:ref:`水位 <damon_design_damos_watermarks_zh_CN>`
+和 :ref:`过滤器 <damon_design_damos_filters_zh_CN>` 可能影响这一判断。核心逻辑
+只会请求底层 :ref:`操作集 <damon_operations_set_zh_CN>` 对该区域应用动作，因此
+该动作是否真正应用并不明确。这就是它被称为“尝试”的原因。
+
+“方案应用到某个区域”表示 :ref:`操作集 <damon_operations_set_zh_CN>` 已经把动作
+应用到该区域的至少一部分。操作集处理的 :ref:`过滤器
+<damon_design_damos_filters_zh_CN>`、:ref:`动作 <damon_design_damos_action_zh_CN>`
+类型以及该区域中的页面类型都可能影响这一点。例如，如果过滤器设置为排除匿名页且
+该区域只有匿名页，或者动作是 ``pageout`` 而该区域的所有页面都不可回收，则把动作
+应用到该区域会失败。
+
+不同于普通统计信息，``max_nr_snapshots`` 由用户设置。如果它设置为非零，且
+``nr_snapshots`` 等于或大于 ``max_nr_snapshots``，该方案会被停用。
+
+要了解用户空间如何通过 :ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 读取统计
+信息，请参考文档的 :ref:`stats <sysfs_schemes_stats_zh_CN>` 部分。
+
+
+区域遍历
+~~~~~~~~
+
+DAMOS 功能允许用户访问刚刚应用了 DAMOS 动作的每个区域。使用该功能，DAMON
+:ref:`API <damon_design_api_zh_CN>` 允许用户访问这些区域的完整属性，包括访问监测结果以及
+区域内部通过 DAMOS 过滤器的内存量。:ref:`DAMON sysfs 接口 <sysfs_interface_zh_CN>` 也允许
+用户通过特殊 :ref:`文件 <sysfs_schemes_tried_regions_zh_CN>` 读取这些数据。
+
+.. _damon_design_api_zh_CN:
+
+应用编程接口
+------------
+
+用于内核空间数据访问感知应用的编程接口。DAMON 是一个框架，所以它本身什么也不做。相反，
+它只帮助子系统和模块等其他内核组件使用 DAMON 的核心功能构建它们的数据访问感知应用。为此，
+DAMON 通过其应用编程接口，即 ``include/linux/damon.h``，向其他内核组件暴露所有功能。接口
+细节请参考 API :doc:`文档 <api>`。
+
+
+.. _damon_modules_zh_CN:
+
+模块
+====
+
+由于 DAMON 的核心是供内核组件使用的框架，它本身不向用户空间提供任何直接接口。相反，这些接口
+应由每个使用 DAMON API 的内核组件来实现。DAMON 子系统本身实现了这类 DAMON API 用户模块，
+它们分别用于通用目的的 DAMON 控制和特定目的的数据访问感知系统操作，并为用户空间提供稳定的应
+用二进制接口（ABI）。用户空间可以使用这些接口构建高效的数据访问感知应用程序。
+
+
+通用目的用户接口模块
+--------------------
+
+DAMON 模块为运行时的通用目的 DAMON 用法提供用户空间 ABI。
+
+与许多其他 ABI 一样，这些模块会在类似 ``sysfs`` 的伪文件系统上创建文件，允许用户通过写入和读
+取这些文件来向 DAMON 指定请求并获取回答。作为这类 I/O 的响应，DAMON 用户接口模块会通过 DAMON
+API 按用户请求控制 DAMON 并获取结果，然后把结果返回给用户空间。
+
+这些 ABI 是为用户空间应用程序开发而设计的，而不是为人工手动操作而设计的。建议人工用户使用这类
+用户空间工具。一个用 Python 编写的此类用户空间工具可在 Github
+（https://github.com/damonitor/damo）、Pypi（https://pypistats.org/packages/damo）以及多个发行
+版（https://repology.org/project/damo/versions）中获取。
+
+目前，该类型有一个模块可用，即 ``DAMON sysfs interface``。关于接口细节，请参考 ABI
+:ref:`文档 <sysfs_interface_zh_CN>`。
+
+
+.. _damon_modules_special_purpose_zh_CN:
+
+专用访问感知内核模块
+--------------------
+
+DAMON 模块为特定目的的 DAMON 用法提供用户空间 ABI。
+
+DAMON 用户接口模块用于在运行时完整控制所有 DAMON 功能。对于每种专用的、系统范围的数据访问感知
+系统操作，例如主动回收或 LRU 链表均衡，可以通过移除该特定目的不需要的调节项来简化接口，并扩展
+为启动时甚至编译时控制。用于该用途的 DAMON 控制参数默认值也需要针对该目的进行优化。
+
+为支持这些场景，DAMON 还提供了更多使用 DAMON API 的内核模块，它们提供更简单且更优化的用户空间
+接口。目前提供了用于访问监测统计、主动回收和 LRU 链表操作的三个模块。更多细节请阅读这些模块的
+使用文档（:doc:`../../admin-guide/mm/damon/stat`,
+:doc:`../../admin-guide/mm/damon/reclaim` 和
+:doc:`../../admin-guide/mm/damon/lru_sort`）。
+
+.. _damon_design_special_purpose_modules_exclusivity_zh_CN:
+
+请注意，这些模块当前以互斥方式运行。如果其中一个模块已经在运行，其他模块在收到启动请求时将返回
+``-EBUSY``。
+
+示例 DAMON 模块
+----------------
+
+DAMON 模块提供 DAMON 内核 API 用法示例。
+
+内核程序员可以使用 DAMON 内核 API 构建自己的专用或通用目的 DAMON 模块。为了帮助他们容易理解
+如何使用 DAMON 内核 API，Linux 源码树的 ``samples/damon/`` 目录下提供了一些示例模块。请注意，
+这些模块不是为实际产品使用而开发的，而只是为了展示如何以简单方式使用 DAMON 内核 API。
diff --git a/Documentation/translations/zh_CN/networking/driver.rst b/Documentation/translations/zh_CN/networking/driver.rst
new file mode 100644
index 00000000000000..acb0863c6b0de0
--- /dev/null
+++ b/Documentation/translations/zh_CN/networking/driver.rst
@@ -0,0 +1,138 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/networking/driver.rst
+
+:翻译:
+
+	陈思为 Siwei Chen <businiaoanka@anka1.top>
+
+================
+Softnet 驱动问题
+================
+
+探测指南
+========
+
+地址校验
+--------
+
+为设备获取的任何硬件层地址都应进行校验。例如，以太网地址应使用
+linux/etherdevice.h 中的 is_valid_ether_addr() 进行检查。
+
+关闭/停止指南
+=============
+
+静默
+----
+
+ndo_stop 例程被调用后，硬件不得再收发任何数据。所有在途报文都必须中止。
+如有必要，应轮询或等待任何复位命令完成。
+
+自动关闭
+--------
+
+若设备仍处于 UP 状态，ndo_stop 例程将由 unregister_netdevice 调用。
+
+发送路径指南
+============
+
+提前停止队列
+------------
+
+ndo_start_xmit 方法在任何正常情况下都不得返回 NETDEV_TX_BUSY。
+除非设备确实无法预先判断其发送功能何时会繁忙，否则返回该值被视为严重错误。
+
+相反，驱动必须正确地维护队列。例如，对于实现分散-聚集
+（scatter-gather）的驱动，这意味着：
+
+.. code-block:: c
+
+	static u32 drv_tx_avail(struct drv_ring *dr)
+	{
+		u32 used = READ_ONCE(dr->prod) - READ_ONCE(dr->cons);
+
+		return dr->tx_ring_size - (used & dr->tx_ring_mask);
+	}
+
+	static netdev_tx_t drv_hard_start_xmit(struct sk_buff *skb,
+					       struct net_device *dev)
+	{
+		struct drv *dp = netdev_priv(dev);
+		struct netdev_queue *txq;
+		struct drv_ring *dr;
+		int idx;
+
+		idx = skb_get_queue_mapping(skb);
+		dr = dp->tx_rings[idx];
+		txq = netdev_get_tx_queue(dev, idx);
+
+		//...
+		/* 这应当是极为罕见的竞争——记录下来。 */
+		if (drv_tx_avail(dr) <= skb_shinfo(skb)->nr_frags + 1) {
+			netif_tx_stop_queue(txq);
+			netdev_warn(dev, "Tx Ring full when queue awake!\n");
+			return NETDEV_TX_BUSY;
+		}
+
+		//... 将报文排队到网卡 ...
+
+		netdev_tx_sent_queue(txq, skb->len);
+
+		//... 使用 WRITE_ONCE() 更新 tx 生产者索引 ...
+
+		if (!netif_txq_maybe_stop(txq, drv_tx_avail(dr),
+					  MAX_SKB_FRAGS + 1, 2 * MAX_SKB_FRAGS))
+			dr->stats.stopped++;
+
+		//...
+		return NETDEV_TX_OK;
+	}
+
+然后在 TX 回收事件处理的末尾：
+
+.. code-block:: c
+
+	//... 使用 WRITE_ONCE() 更新 tx 消费者索引 ...
+
+	netif_txq_completed_wake(txq, cmpl_pkts, cmpl_bytes,
+				 drv_tx_avail(dr), 2 * MAX_SKB_FRAGS);
+
+无锁队列停止/唤醒辅助宏
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. kernel-doc:: include/net/netdev_queues.h
+   :doc: Lockless queue stopping / waking helpers.
+
+netif_txq_maybe_stop()、netif_txq_try_stop() 等标准宏已经过充分测试，
+请优先使用它们，而非本地同步方案。
+
+无独占所有权
+------------
+
+ndo_start_xmit 方法不得修改被克隆 SKB 的共享部分。
+
+及时完成
+--------
+
+请谨记：一旦 ndo_start_xmit 方法返回 NETDEV_TX_OK，
+驱动就有责任在有限的时间内释放该 SKB。
+
+例如，这意味着不允许你的 TX 缓解方案在没有新 TX 报文发送时，
+让 TX 报文永远“滞留”在 TX 环中而不被回收。
+这种错误会使等待发送缓冲区空间释放的套接字发生死锁。
+
+若 ndo_start_xmit 方法返回 NETDEV_TX_BUSY，
+则不得保留对该 SKB 的任何引用，也不得尝试释放它。
+
+错误消息报告
+============
+
+许多驱动配置接口会向驱动传递一个 Netlink 扩展 ACK（``extack``）对象
+（直接作为参数，或作为参数结构体的成员）。驱动应尽量通过 ``extack`` 对象
+报告大多数错误。表示系统或设备行为异常、处于不良状态的系统级异常，则应继续
+报告到系统日志。
+
+消息应 **要么** 通过 ``extack`` 传递， **要么** 写入系统日志。驱动不应
+试图将同一信息同时报告到两处。
diff --git a/Documentation/translations/zh_CN/networking/index.rst b/Documentation/translations/zh_CN/networking/index.rst
index 333e9f6cafffbf..bf22d22b64db81 100644
--- a/Documentation/translations/zh_CN/networking/index.rst
+++ b/Documentation/translations/zh_CN/networking/index.rst
@@ -21,7 +21,10 @@
    :maxdepth: 1
 
    msg_zerocopy
+   driver
+   ipv6
    napi
+   secid
    vxlan
    netif-msg
    xfrm_proc
@@ -29,6 +32,8 @@
    alias
    mptcp-sysctl
    generic-hdlc
+   sriov
+   team
    timestamping
 
 Todolist:
@@ -74,7 +79,6 @@ Todolist:
 *   dctcp
 *   devmem
 *   dns_resolver
-*   driver
 *   eql
 *   fib_trie
 *   filter
@@ -87,7 +91,6 @@ Todolist:
 *   ip_dynaddr
 *   ipsec
 *   ip-sysctl
-*   ipv6
 *   ipvlan
 *   ipvs-sysctl
 *   kcm
@@ -124,10 +127,8 @@ Todolist:
 *   representors
 *   rxrpc
 *   sctp
-*   secid
 *   seg6-sysctl
 *   smc-sysctl
-*   sriov
 *   statistics
 *   strparser
 *   switchdev
@@ -136,7 +137,6 @@ Todolist:
 *   tc-queue-filters
 *   tcp_ao
 *   tcp-thin
-*   team
 *   tipc
 *   tproxy
 *   tuntap
diff --git a/Documentation/translations/zh_CN/networking/ipv6.rst b/Documentation/translations/zh_CN/networking/ipv6.rst
new file mode 100644
index 00000000000000..3726b6fdc460f5
--- /dev/null
+++ b/Documentation/translations/zh_CN/networking/ipv6.rst
@@ -0,0 +1,76 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/networking/ipv6.rst
+
+:翻译:
+
+	陈思为 Siwei Chen <businiaoanka@anka1.top>
+
+====
+IPv6
+====
+
+
+ipv6 模块的选项在加载时以参数形式提供。
+
+模块选项可作为 insmod 或 modprobe 命令的命令行参数给出，但通常在
+``/etc/modules.d/*.conf`` 配置文件中，或在发行版特定的配置文件中指定。
+
+可用的 ipv6 模块参数如下所列。若未指定某参数，则使用其默认值。
+
+参数如下：
+
+disable
+
+	指定是否在加载 IPv6 模块的同时禁用其全部功能。当另一模块依赖于 IPv6
+	模块已加载，但又不需要任何 IPv6 地址或操作时，可使用此选项。
+
+	可选值及其效果如下：
+
+	0
+		启用 IPv6。
+
+		这是默认值。
+
+	1
+		禁用 IPv6。
+
+		接口不会添加 IPv6 地址，也无法打开 IPv6 套接字。
+
+		需要重启才能启用 IPv6。
+
+autoconf
+
+	指定是否在所有接口上启用 IPv6 地址自动配置。当不希望根据路由器通告
+	（Router Advertisement）中收到的前缀自动生成地址时，可使用此选项。
+
+	可选值及其效果如下：
+
+	0
+		所有接口均禁用 IPv6 地址自动配置。
+
+		仅添加 IPv6 回环地址（::1）与链路本地地址。
+
+	1
+		所有接口均启用 IPv6 地址自动配置。
+
+		这是默认值。
+
+disable_ipv6
+
+	指定是否在所有接口上禁用 IPv6。
+	当不需要任何 IPv6 地址时，可使用此选项。
+
+	可选值及其效果如下：
+
+	0
+		在所有接口上启用 IPv6。
+
+		这是默认值。
+
+	1
+		在所有接口上禁用 IPv6。
+
+		接口不会添加 IPv6 地址。
diff --git a/Documentation/translations/zh_CN/networking/secid.rst b/Documentation/translations/zh_CN/networking/secid.rst
new file mode 100644
index 00000000000000..94d9e7fb5d1336
--- /dev/null
+++ b/Documentation/translations/zh_CN/networking/secid.rst
@@ -0,0 +1,24 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/networking/secid.rst
+
+:翻译:
+
+	陈思为 Siwei Chen <businiaoanka@anka1.top>
+
+=================
+LSM/SeLinux secid
+=================
+
+flowi 结构体：
+
+flowi 结构体中的 secid 成员在 LSM（如 SELinux）中用于表示该流的标签。
+该流的标签目前用于选择相匹配的带标签 xfrm。
+
+若为出向流，标签派生自套接字（若有）；如果该流是为响应某个入站报文而生成的，
+则标签派生自相应的入站报文（例如 TCP 复位、timewait ACK 等）。
+在特殊情况下，也可酌情让标签派生自其他来源，如进程上下文、设备等。
+
+若为入向流，标签派生自报文所使用的 IPSec 安全关联（若存在）。
diff --git a/Documentation/translations/zh_CN/networking/sriov.rst b/Documentation/translations/zh_CN/networking/sriov.rst
new file mode 100644
index 00000000000000..d5e16a099b070b
--- /dev/null
+++ b/Documentation/translations/zh_CN/networking/sriov.rst
@@ -0,0 +1,34 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/networking/sriov.rst
+
+:翻译:
+
+	陈思为 Siwei Chen <businiaoanka@anka1.top>
+
+===============
+网卡 SR-IOV API
+===============
+
+强烈建议现代网卡专注于实现 ``switchdev`` 模型
+（参见 `以太网交换机设备驱动模型（switchdev）`_）
+来配置 SR-IOV 功能的转发与安全。
+
+传统 API
+========
+
+旧的 SR-IOV API 在 ``rtnetlink`` Netlink 协议族中实现，是 ``RTM_GETLINK`` 和
+``RTM_SETLINK`` 命令的一部分。在驱动侧，它由若干 ``ndo_set_vf_*`` 和
+``ndo_get_vf_*`` 回调组成。
+
+由于传统 API 与协议栈其余部分集成不佳，该 API 被视为已冻结；不再接受任何新功能或扩展。
+新驱动不应实现那些不常用的回调；
+即以下回调禁止使用：
+
+ - ``ndo_get_vf_port``
+ - ``ndo_set_vf_port``
+ - ``ndo_set_vf_rss_query_en``
+
+.. _`以太网交换机设备驱动模型（switchdev）`: https://docs.kernel.org/networking/switchdev.html#switchdev
diff --git a/Documentation/translations/zh_CN/networking/team.rst b/Documentation/translations/zh_CN/networking/team.rst
new file mode 100644
index 00000000000000..e6cf19a245a1d3
--- /dev/null
+++ b/Documentation/translations/zh_CN/networking/team.rst
@@ -0,0 +1,16 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/networking/team.rst
+
+:翻译:
+
+	陈思为 Siwei Chen <businiaoanka@anka1.top>
+
+====================
+Team（网络接口聚合）
+====================
+
+Team 设备由用户空间通过 libteam 库驱动，该库位于：
+	https://github.com/jpirko/libteam
diff --git a/Documentation/translations/zh_CN/process/applying-patches.rst b/Documentation/translations/zh_CN/process/applying-patches.rst
new file mode 100644
index 00000000000000..c056b241b0cdad
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/applying-patches.rst
@@ -0,0 +1,390 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/process/applying-patches.rst
+
+:翻译:
+
+ 袁维杰 Weijie Yuan <wy@wyuan.org>
+
+将补丁应用到 Linux 内核
++++++++++++++++++++++++
+
+原文作者：
+	Jesper Juhl，2005 年 8 月
+
+.. note::
+
+   本文档已经过时。绝大多数情况下，你几乎肯定应该考虑使用 Git，而不是
+   手工运行 ``patch``。
+
+Linux 内核邮件列表中经常有人询问如何给内核打补丁；更确切地说，为众多
+内核树或分支之一制作的补丁，应当以哪个版本的内核为基础来应用。希望本文
+能够解答这些问题。
+
+除了说明如何应用和撤销补丁，本文还会简要介绍不同的内核树，并举例说明
+如何应用各个内核树特有的补丁。
+
+
+什么是补丁？
+============
+
+补丁是一份短小的文本文档，其中记录了源码树两个不同版本之间的改动。
+补丁由 ``diff`` 程序生成。
+
+要正确应用补丁，你必须知道它是基于哪个版本生成的，又会把源码树更新成
+哪个版本。这两项信息应当记录在补丁文件的元数据中，或者能够根据文件名
+推断出来。
+
+
+如何应用或撤销补丁？
+====================
+
+使用 ``patch`` 程序应用补丁。patch 程序读取 diff 文件（也称补丁
+文件），再按照文件中的描述修改源码树。
+
+Linux 内核补丁以存放内核源码目录的父目录为基准生成。
+
+因此，补丁文件中的路径会带有生成补丁时所用的内核源码目录名（也可能是
+“a/”和“b/”之类的其他目录名）。
+
+这个目录名一般不会恰好与本机的内核源码目录名相同（不过，它往往有助于
+判断未注明版本的补丁是基于哪个版本生成的）。应用补丁时，应先进入内核
+源码目录，再去掉补丁文件所列路径中的第一个组成部分（``patch`` 的
+``-p1`` 参数正是用于此目的）。
+
+要撤销先前应用的补丁，请给 patch 加上 -R 参数。也就是说，如果应用
+补丁时执行的是::
+
+	patch -p1 < ../patch-x.y.z
+
+那么可以这样撤销它::
+
+	patch -R -p1 < ../patch-x.y.z
+
+
+如何把补丁或 diff 文件交给 ``patch``？
+======================================
+
+和 Linux 及其他类 UNIX 操作系统中的许多操作一样，这件事也有好几种
+做法。
+
+下文的所有示例都使用如下语法，通过标准输入（stdin）把未压缩的文件
+交给 patch::
+
+	patch -p1 < path/to/patch-x.y.z
+
+如果你只想照着下文的示例操作，无意了解 patch 的其他用法，那么读到
+这里就可以跳过本节余下的内容。
+
+patch 也能通过 -i 参数取得待处理文件的名称，例如::
+
+	patch -p1 -i path/to/patch-x.y.z
+
+如果补丁文件以 gzip 或 xz 格式压缩，而你不想先解压再应用，可以这样
+把它交给 patch::
+
+	xzcat path/to/patch-x.y.z.xz | patch -p1
+	bzcat path/to/patch-x.y.z.gz | patch -p1
+
+如果想在应用前手工解压补丁文件（下文的示例均假定你已经这样做了），
+只需对文件运行 gunzip 或 xz，例如::
+
+	gunzip patch-x.y.z.gz
+	xz -d patch-x.y.z.xz
+
+执行后会留下一个纯文本的 patch-x.y.z 文件。你可以按自己的喜好，
+通过标准输入或 ``-i`` 参数把它交给 patch。
+
+patch 还有几个很实用的参数：``-s`` 会让 patch 除错误外不输出任何
+信息，以免错误信息很快滚出屏幕；``--dry-run`` 只列出将会发生的事情，
+并不实际修改文件；最后，``--verbose`` 会让 patch 输出更详细的处理
+信息。
+
+
+应用补丁时的常见错误
+====================
+
+应用补丁文件时，patch 会用多种方式检查文件是否合理。
+
+例如，patch 会检查文件是否像一个有效的补丁文件，也会检查待修改代码
+周围的内容是否与补丁提供的上下文相符。这只是它所做的两项基本检查。
+
+如果遇到不太对劲的地方，patch 有两种选择：拒绝应用改动并中止，或者
+稍作调整，设法把补丁应用上去。
+
+patch 会尝试调整的一种情况是：上下文和待修改的行全都匹配，只有行号
+不同。例如，补丁要修改文件中部的内容，但文件开头附近由于某种原因增删
+了几行，就会出现这种情况。此时所有内容其实都能对上，只是整体向前或
+向后移动了一些，patch 通常会相应调整行号并应用补丁。
+
+每当 patch 必须略作调整才能匹配时，它会提示补丁是在经过模糊匹配
+（**fuzz**）后应用的。你应当谨慎对待这样的改动：patch 很可能处理
+正确，但它并不保证每次都能正确处理，有时会得到错误的结果。
+
+如果某项改动无法通过模糊匹配调整，patch 会直接拒绝这项改动，并留下
+一个扩展名为 ``.rej`` 的文件（拒绝文件）。你可以阅读这个文件，准确
+了解哪项改动未能应用，再视需要手工处理。
+
+假如内核源码中没有第三方补丁，只有来自 kernel.org 的补丁，而且补丁
+应用顺序正确，源码文件也没有经过自行修改，那么 patch 绝不应报告模糊
+匹配或拒绝应用。如果还是看到这类信息，你的本地源码树或补丁文件很可能
+已经损坏。此时应先尝试重新下载补丁；如果问题依旧，建议从 kernel.org
+重新下载完整的全新源码树，从头再来。
+
+下面进一步看看 patch 可能输出的几种信息。
+
+如果 patch 停下来并显示 ``File to patch:`` 提示，说明它找不到待修改
+的文件。最可能的原因是你忘记指定 -p1，或者当前目录不对。偶尔也会
+遇到必须用 ``-p0`` 而不是 ``-p1`` 应用的补丁（查看补丁文件便能判断
+是否如此；如果确实如此，是补丁制作者犯了错，但并不致命）。
+
+如果看到 ``Hunk #2 succeeded at 1887 with fuzz 2 (offset 7 lines).``
+或类似信息，说明 patch 必须调整改动的位置才能应用；在这个例子中，
+实际改动位置必须与预期位置相差 7 行才能匹配。
+
+最终得到的文件可能正确，也可能不正确，取决于文件为何与预期不符。
+
+尝试应用一个基于其他内核版本生成的补丁时，经常会出现这种情况。
+
+如果看到 ``Hunk #3 FAILED at 2387.`` 之类的信息，说明补丁未能正确
+应用，patch 程序也无法通过模糊匹配完成应用。此时会生成一个 ``.rej``
+文件，其中包含导致应用失败的改动；还会生成一个 ``.orig`` 文件，其中
+是未能修改的原始内容。
+
+如果看到 ``Reversed (or previously applied) patch detected!  Assume
+-R? [n]``，说明 patch 检测到补丁中的改动似乎已经存在。
+
+如果你确实已经应用过这个补丁，只是不慎又应用了一次，请回答 [n]o 并
+中止应用。如果先前应用过补丁，现在本来就想撤销它，只是忘了指定 -R，
+则可以在这里回答 [**y**]es，让 patch 替你撤销。
+
+如果补丁制作者在生成补丁时颠倒了源目录和目标目录，也会出现这条信息；
+在这种情况下，撤销补丁实际上才是在应用补丁。
+
+``patch: **** unexpected end of file in patch`` 或 ``patch unexpectedly
+ends in middle of line`` 之类的信息，表示 patch 无法理解收到的文件。
+可能是文件下载不完整，可能是你没有先解压便把压缩补丁交给了 patch，
+也可能是补丁文件在途经某个邮件客户端或邮件传输代理时遭到破坏，例如
+一行过长的内容被拆成了两行。这类警告通常很容易处理，只需把被拆开的
+两行重新连接（拼接）起来即可。
+
+如前所述，如果把 kernel.org 提供的补丁应用到版本正确、未经修改的
+源码树上，这些错误绝不应出现。因此，如果应用 kernel.org 的补丁时
+遇到这些错误，应当认为补丁文件或源码树已经损坏；建议重新下载完整的
+内核源码树和要应用的补丁，从头再来。
+
+
+除了 ``patch``，还有其他选择吗？
+================================
+
+有。
+
+可以使用 ``interdiff`` 程序（http://cyberelk.net/tim/patchutils/），
+为两个补丁之间的差异生成一个补丁，然后应用生成的补丁。
+
+这样便能一步从 5.7.2 更新到 5.7.3。interdiff 的 -z 参数甚至允许
+直接传入以 gzip 或 bzip2 格式压缩的补丁，无需使用 zcat、bzcat，也
+无需手工解压。
+
+下面是一步从 5.7.2 更新到 5.7.3 的方法::
+
+	interdiff -z ../patch-5.7.2.gz ../patch-5.7.3.gz | patch -p1
+
+尽管 interdiff 能省去一两个步骤，一般仍建议按常规方法多执行这些步骤，
+因为 interdiff 在某些情况下可能产生错误结果。
+
+另一个选择是 ``ketchup``。它是一个自动下载并应用补丁的 Python 脚本
+（https://www.selenic.com/ketchup/）。
+
+其他好用的工具还有：diffstat，用于显示补丁改动的摘要；lsdiff，用于
+简要列出补丁文件影响的文件，还可以选择同时显示各段补丁的起始行号；
+grepdiff，用于在补丁中查找与指定正则表达式匹配的内容，并列出这些内容
+所在的文件。
+
+
+可以从哪里下载补丁？
+====================
+
+补丁可从 https://kernel.org/ 获取。网站首页提供了最新补丁的链接，
+各类补丁也有各自固定的存放位置。
+
+5.x.y（-stable）补丁和 5.x 补丁位于
+
+	https://www.kernel.org/pub/linux/kernel/v5.x/
+
+5.x.y 增量补丁位于
+
+	https://www.kernel.org/pub/linux/kernel/v5.x/incr/
+
+-rc 补丁并不存放在 Web 服务器上，而是根据下面这样的 git 标签按需
+生成
+
+	https://git.kernel.org/torvalds/p/v5.1-rc1/v5.0
+
+稳定版的 -rc 补丁位于
+
+	https://www.kernel.org/pub/linux/kernel/v5.x/stable-review/
+
+
+5.x 内核
+========
+
+这些是 Linus 发布的基础稳定版本，其中版本号最大者最新。
+
+如果发现回归或其他严重缺陷，会在这个基础版本之上发布 -stable 修复
+补丁（见下文）。每当新的 5.x 基础内核发布时，还会提供一个补丁，用来
+表示上一个 5.x 内核与新内核之间的差异。
+
+要应用从 5.6 更新到 5.7 的补丁，可以按下面的步骤操作。请注意，这类
+补丁**不能**应用到 5.x.y 内核之上，只能应用到基础 5.x 内核之上；
+如果要从 5.x.y 更新到 5.x+1，必须先撤销 5.x.y 补丁。
+
+下面是两个示例::
+
+	# 从 5.6 更新到 5.7
+
+	$ cd ~/linux-5.6		# 进入内核源码目录
+	$ patch -p1 < ../patch-5.7	# 应用 5.7 补丁
+	$ cd ..
+	$ mv linux-5.6 linux-5.7	# 重命名源码目录
+
+	# 从 5.6.1 更新到 5.7
+
+	$ cd ~/linux-5.6.1		# 进入内核源码目录
+	$ patch -p1 -R < ../patch-5.6.1	# 撤销 5.6.1 补丁
+					# 源码目录现在是 5.6
+	$ patch -p1 < ../patch-5.7	# 应用新的 5.7 补丁
+	$ cd ..
+	$ mv linux-5.6.1 linux-5.7	# 重命名源码目录
+
+
+5.x.y 内核
+==========
+
+版本号由三段数字组成的是 -stable 内核。它们包含规模较小但至关重要的
+修复，用于解决某个 5.x 内核中发现的安全问题或严重回归。
+
+对于想使用最新的稳定内核，又无意帮助测试开发版或实验版的用户，推荐
+选择这个分支。
+
+如果没有 5.x.y 内核可用，则版本号最大的 5.x 内核就是当前稳定内核。
+
+-stable 团队既提供普通补丁，也提供增量补丁。下面说明如何应用这两类
+补丁。
+
+普通补丁
+~~~~~~~~
+
+这类补丁不是增量补丁。例如，5.7.3 补丁不能应用到 5.7.2 内核源码
+之上，而应应用到基础 5.7 内核源码之上。
+
+因此，要把 5.7.3 补丁应用到现有的 5.7.2 内核源码，必须先撤销
+5.7.2 补丁（回到基础 5.7 内核源码），再应用新的 5.7.3 补丁。
+
+下面是一个简单的示例::
+
+	$ cd ~/linux-5.7.2		# 进入内核源码目录
+	$ patch -p1 -R < ../patch-5.7.2	# 撤销 5.7.2 补丁
+	$ patch -p1 < ../patch-5.7.3	# 应用新的 5.7.3 补丁
+	$ cd ..
+	$ mv linux-5.7.2 linux-5.7.3	# 重命名内核源码目录
+
+增量补丁
+~~~~~~~~
+
+增量补丁则不同：它们不应用到基础 5.x 内核之上，而应用到前一个稳定版
+内核（5.x.y-1）之上。
+
+下面是应用增量补丁的示例::
+
+	$ cd ~/linux-5.7.2		# 进入内核源码目录
+	$ patch -p1 < ../patch-5.7.2-3	# 应用新的 5.7.3 补丁
+	$ cd ..
+	$ mv linux-5.7.2 linux-5.7.3	# 重命名内核源码目录
+
+
+-rc 内核
+========
+
+这些是候选发布版内核。每当 Linus 认为当前的 git 树（git 是内核的
+源码管理工具）处于相当合理、足以测试的状态时，就会发布这样的开发版内核。
+
+这些内核并不稳定；如果打算运行，就要预料到它们偶尔会出故障。不过，
+在几个主要开发分支中，这是最稳定的一个，而且它最终会成为下一个稳定版
+内核，所以让尽可能多的人参与测试十分重要。
+
+如果你想帮助测试开发版内核，但不想运行真正具有实验性的内容，这个分支
+很合适（有关真正实验性的内容，请参阅下文介绍 -next 和 -mm 内核的
+章节）。
+
+-rc 补丁不是增量补丁；与上文介绍的 5.x.y 补丁一样，它们应用到基础
+5.x 内核之上。-rcN 后缀前面的内核版本号，表示这个 -rc 内核最终会
+成为哪个版本。
+
+因此，5.8-rc5 表示它是 5.8 内核的第五个候选发布版，这个补丁应当
+应用到 5.7 内核源码之上。
+
+下面是三个应用这类补丁的示例::
+
+	# 第一个示例：从 5.7 更新到 5.8-rc3
+
+	$ cd ~/linux-5.7			# 进入 5.7 源码目录
+	$ patch -p1 < ../patch-5.8-rc3		# 应用 5.8-rc3 补丁
+	$ cd ..
+	$ mv linux-5.7 linux-5.8-rc3		# 重命名源码目录
+
+	# 接着从 5.8-rc3 更新到 5.8-rc5
+
+	$ cd ~/linux-5.8-rc3			# 进入 5.8-rc3 目录
+	$ patch -p1 -R < ../patch-5.8-rc3	# 撤销 5.8-rc3 补丁
+	$ patch -p1 < ../patch-5.8-rc5		# 应用新的 5.8-rc5 补丁
+	$ cd ..
+	$ mv linux-5.8-rc3 linux-5.8-rc5	# 重命名源码目录
+
+	# 最后尝试从 5.7.3 更新到 5.8-rc5
+
+	$ cd ~/linux-5.7.3			# 进入内核源码目录
+	$ patch -p1 -R < ../patch-5.7.3		# 撤销 5.7.3 补丁
+	$ patch -p1 < ../patch-5.8-rc5		# 应用新的 5.8-rc5 补丁
+	$ cd ..
+	$ mv linux-5.7.3 linux-5.8-rc5		# 重命名源码目录
+
+
+-mm 补丁与 linux-next 树
+========================
+
+-mm 补丁是 Andrew Morton 发布的实验性补丁。
+
+过去，-mm 树还用于测试子系统补丁；现在这项工作由
+`linux-next` （https://www.kernel.org/doc/man-pages/linux-next.html）
+树承担。子系统维护者先把补丁推送到 linux-next，再在合并窗口期间直接
+发送给 Linus。
+
+-mm 补丁是新功能和其他实验性补丁的试验场，这些补丁并未通过子系统树
+合并。一旦这类补丁在 -mm 中经受一段时间的检验并证明自身价值，Andrew
+就会将其提交给 Linus，以纳入主线。
+
+linux-next 树每天更新，其中包括 -mm 补丁。二者始终处于变化之中，
+含有许多实验性功能、大量不适合主线的调试补丁等，是本文所述各分支中
+实验性最强的。
+
+这些补丁不适合用在必须保持稳定的系统上，运行它们的风险高于其他任何
+分支（务必备有最新的备份；运行任何实验性内核都应如此，运行 -mm 补丁
+或 linux-next 树中的内核时尤其如此）。
+
+我们非常欢迎大家测试 -mm 补丁和 linux-next，因为测试的意义就是在改动
+合入更稳定的 Linus 主线树之前，排查并消除回归、崩溃、数据损坏缺陷、
+构建失败以及其他各种缺陷。
+
+不过，-mm 和 linux-next 的测试者必须明白，这些树发生故障的频率高于
+其他任何内核树。
+
+
+至此，各种内核树已经介绍完毕。希望你现在已经清楚如何应用各类补丁，
+并能帮助测试内核。
+
+感谢 Randy Dunlap、Rolf Eike Beer、Linus Torvalds、Bodo Eggert、
+Johannes Stezenbach、Grant Coady、Pavel Machek，以及其他可能被我
+遗漏的人，感谢他们对本文档的审阅和贡献。
diff --git a/Documentation/translations/zh_CN/process/howto.rst b/Documentation/translations/zh_CN/process/howto.rst
index cc47be356dd323..89af3f8f6e2b2b 100644
--- a/Documentation/translations/zh_CN/process/howto.rst
+++ b/Documentation/translations/zh_CN/process/howto.rst
@@ -142,7 +142,7 @@ Linux内核代码中包含有大量的文档。这些文档对于学习如何与
     有助于内核开发的外部文档列表。如果你在内核自带的文档中没有找到你想找
     的内容，可以查看这些文档。
 
-  :ref:`Documentation/process/applying-patches.rst <applying_patches>`
+  :doc:`Documentation/translations/zh_CN/process/applying-patches.rst <applying-patches>`
     关于补丁是什么以及如何将它打在不同内核开发分支上的好介绍
 
 内核还拥有大量从代码自动生成或者从 ReStructuredText(ReST) 标记生成的文档，
diff --git a/Documentation/translations/zh_CN/process/index.rst b/Documentation/translations/zh_CN/process/index.rst
index 3bcb3bdaf5332d..3a36ca7d9333b7 100644
--- a/Documentation/translations/zh_CN/process/index.rst
+++ b/Documentation/translations/zh_CN/process/index.rst
@@ -82,13 +82,13 @@ TODOLIST:
    :maxdepth: 1
 
    magic-number
+   applying-patches
    volatile-considered-harmful
    ../arch/riscv/patch-acceptance
    ../core-api/unaligned-memory-access
 
 TODOLIST:
 
-* applying-patches
 * backporting
 * adding-syscalls
 * botching-up-ioctls
diff --git a/Documentation/translations/zh_TW/glossary.rst b/Documentation/translations/zh_TW/glossary.rst
new file mode 100644
index 00000000000000..15ed1be54416b0
--- /dev/null
+++ b/Documentation/translations/zh_TW/glossary.rst
@@ -0,0 +1,168 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _tw_glossary:
+
+術語表
+======
+
+:作者: 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
+
+本術語表列出Linux核心繁體中文（zh_TW）翻譯所使用的術語。zh_TW翻譯以臺灣
+慣用的資訊術語為準。新增或更新翻譯時，請依照本表選擇用詞，以維持整個
+zh_TW文件樹的一致性；若遇到本表未收錄的術語，歡迎提交補丁補充。
+
+下表同時列出常見的簡體中文（zh_CN）譯法，方便從zh_CN翻譯轉換或對照時
+使用。本表的zh_CN欄位為對照參考，其用詞經核對核心文件樹zh_CN譯文中的
+實際用法，非引用外部術語表。
+
+一般術語
+--------
+
+========================  ==========================  ====================
+英文                      zh_TW                       zh_CN（對照）
+========================  ==========================  ====================
+kernel                    核心                        内核
+user / user space         使用者／使用者空間          用户／用户空间
+software / hardware       軟體／硬體                  软件／硬件
+firmware                  韌體                        固件
+operating system          作業系統                    操作系统
+distribution              發行版                      发行版
+community                 社群                        社区
+project                   專案                        项目
+documentation             文件                        文档
+file                      檔案                        文件
+folder / directory        資料夾／目錄                文件夹／目录
+default                   預設                        默认／缺省
+support                   支援                        支持
+information               資訊                        信息
+message                   訊息                        消息
+data                      資料                        数据
+quality                   品質                        质量
+performance               效能                        性能
+network                   網路                        网络
+the Internet              網際網路                    因特网
+server                    伺服器                      服务器
+digital                   數位                        数字
+computer                  電腦                        计算机
+integrate                 整合                        集成
+create                    建立                        创建
+access                    存取                        访问
+via / through             透過                        通过
+========================  ==========================  ====================
+
+程式設計術語
+------------
+
+========================  ==========================  ====================
+英文                      zh_TW                       zh_CN（對照）
+========================  ==========================  ====================
+source code               原始程式碼／原始碼          源代码
+code                      程式碼                      代码
+program                   程式                        程序
+process                   行程                        进程
+thread                    執行緒                      线程
+scheduler / schedule      排程器／排程                调度器／调度
+queue                     佇列                        队列
+memory                    記憶體                      内存
+cache                     快取                        缓存
+interface                 介面                        接口
+port                      埠                          端口
+register                  暫存器                      寄存器
+stack                     堆疊                        堆栈
+function                  函式                        函数
+function call             呼叫                        调用
+callback                  回呼                        回调
+return value              回傳值                      返回值
+macro                     巨集                        宏
+enum                      列舉                        枚举
+variable                  變數                        变量
+pointer                   指標                        指针
+array                     陣列                        数组
+linked list               鏈結串列                    链表
+loop                      迴圈                        循环
+declaration               宣告                        声明
+identifier                識別字                      标识符
+character / string        字元／字串                  字符／字符串
+byte                      位元組                      字节
+boolean                   布林                        布尔
+binary                    二進位                      二进制
+object                    物件                        对象
+type                      型別／類型                  类型
+module                    模組                        模块
+component                 元件                        组件
+build                     建置                        构建
+compile / assembler       編譯／組譯器                编译／汇编器
+debug                     除錯                        调试
+optimize                  最佳化                      优化
+implement                 實作                        实现
+global / local            全域／區域                  全局／局部
+constant                  常數                        常量
+comment (in code)         註解                        注释
+indentation               縮排                        缩进
+inline                    行內                        内联
+generate                  產生                        生成
+load / loader             載入／載入器                加载／加载器
+export / import           匯出／匯入                  导出／导入
+link / linker             連結／連結器                链接／链接器
+repository                儲存庫                      存储库／仓库
+distributed               分散式                      分布式
+header file               標頭檔                      头文件
+configuration file        設定檔                      配置文件
+file system               檔案系統                    文件系统
+settings / configuration  設定                        设置／配置
+========================  ==========================  ====================
+
+介面與其他術語
+--------------
+
+========================  ==========================  ====================
+英文                      zh_TW                       zh_CN（對照）
+========================  ==========================  ====================
+menu                      選單                        菜单
+window                    視窗                        窗口
+toolbar                   工具列                      工具栏
+field                     欄位                        字段
+icon                      圖示                        图标
+mouse / cursor            滑鼠／游標                  鼠标／光标
+paste                     貼上                        粘贴
+print / printer           列印／印表機                打印／打印机
+disk (hard disk)          磁碟（硬碟）                磁盘（硬盘）
+text / plain text         文字／純文字                文本／纯文本
+mailbox                   信箱                        邮箱
+account                   帳戶／帳號                  帐户
+tracking                  追蹤                        跟踪
+advanced                  進階                        高级
+free software             自由軟體                    自由软件／免费软件
+========================  ==========================  ====================
+
+特定用語說明
+------------
+
+- **access 與 visit**：access（資料、記憶體、資源的存取）譯為「存取」；
+  visit（造訪網站、頁面或連結）譯為「造訪」。兩者在 zh_CN 皆作「訪問」，
+  zh_TW 依語意區分。
+
+- **Fellow**：現有翻譯將 Linux Foundation Fellow 譯為「院士」。「院士」在
+  臺灣多指中央研究院等學術機構的頭銜，「研究員」也不完全對等，實務上亦常
+  直接保留英文。此譯法尚在討論中，暫維持現有譯法，待社群取得共識後再統一。
+
+- **mailing list**：譯為「郵件列表」（zh_CN 亦作「邮件列表」）。
+
+- **-stable 與穩定版**：核心的 ``-stable`` 樹保留英文原名 ``-stable``；泛指
+  時譯為「穩定版核心」。既有翻譯中「穩定版」與 ``-stable`` 混用者不強制
+  統一，新增或更新的翻譯請套用此慣例。
+
+用字慣例
+--------
+
+除術語之外，zh_TW翻譯採用臺灣標準用字，例如「為」（不用「爲」）、「裡」
+（不用「裏」）、「著」（不用「着」）、「才」（不用「纔」）、「啟」（不用
+「啓」）、「只」（不用「隻」）與「發布」（不用「發佈」）。
+
+標點符號沿用現有文件樹的全形標點。每行寬度沿用原文慣例（每行約75個半形
+字元寬）。
+
+中文與英文、數字相鄰時不加空格，例如「Linux核心」而非「Linux 核心」；此為
+現有zh_TW文件樹的主要慣例。本規則適用於新增的翻譯與改寫過的段落，既有檔案
+不另行回頭修改。行內標記（``literal``、:ref: 等）前後仍依reStructuredText
+的需要保留必要的空白。
diff --git a/Documentation/translations/zh_TW/index.rst b/Documentation/translations/zh_TW/index.rst
index 660a74d2023cd0..95809012a9efc3 100644
--- a/Documentation/translations/zh_TW/index.rst
+++ b/Documentation/translations/zh_TW/index.rst
@@ -119,9 +119,10 @@ TODOList:
 術語表
 ------
 
-TODOList:
+.. toctree::
+   :maxdepth: 1
 
-* glossary
+   glossary
 
 
 索引和表格
diff --git a/Documentation/translations/zh_TW/process/1.Intro.rst b/Documentation/translations/zh_TW/process/1.Intro.rst
index 345c4cbe9b5555..2639ab99793f18 100644
--- a/Documentation/translations/zh_TW/process/1.Intro.rst
+++ b/Documentation/translations/zh_TW/process/1.Intro.rst
@@ -12,6 +12,7 @@
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
  胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+ 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_development_process_intro:
 
@@ -21,179 +22,183 @@
 內容提要
 --------
 
-本節的其餘部分涵蓋了內核開發的過程，以及開發人員及其僱主在這方面可能遇到的
-各種問題。有很多原因使內核代碼應被合併到正式的（“主線”）內核中，包括對用戶
-的自動可用性、多種形式的社區支持以及影響內核開發方向的能力。提供給Linux內核
-的代碼必須在與GPL兼容的許可證下可用。
+本節的其餘部分涵蓋了核心開發的過程，以及開發人員及其僱主在這方面可能遇到的
+各種問題。有很多原因使核心程式碼應被合併到正式的（“主線”）核心中，包括對使用者
+的自動可用性、多種形式的社群支援以及影響核心開發方向的能力。提供給Linux核心
+的程式碼必須在與GPL相容的許可證下可用。
 
-:ref:`tw_development_process` 介紹了開發過程、內核發佈週期和合並窗口的機制。
-涵蓋了補丁開發、審查和合並週期中的各個階段。還有一些關於工具和郵件列表的討論？
-鼓勵希望開始內核開發的開發人員跟蹤並修復缺陷以作爲初步練習。
+:ref:`tw_development_process` 介紹了開發過程、核心發布週期和合併視窗的機制。
+涵蓋了補丁開發、審查和合併週期中的各個階段。還有一些關於工具和郵件列表的討論。
+鼓勵希望開始核心開發的開發人員追蹤並修復缺陷以作為初步練習。
 
 
-:ref:`tw_development_early_stage` 包括項目的早期規劃，重點是儘快讓開發社區
+:ref:`tw_development_early_stage` 包括專案的早期規劃，重點是儘快讓開發社群
 參與進來。
 
-:ref:`tw_development_coding` 是關於編程過程的；介紹了其他開發人員遇到的幾個
-陷阱。也涵蓋了對補丁的一些要求，並且介紹了一些工具，這些工具有助於確保內核
+:ref:`tw_development_coding` 是關於程式設計過程的；介紹了其他開發人員遇到的幾個
+陷阱。也涵蓋了對補丁的一些要求，並且介紹了一些工具，這些工具有助於確保核心
 補丁是正確的。
 
-:ref:`tw_development_posting` 描述發佈補丁以供評審的過程。爲了讓開發社區能
+:ref:`tw_development_posting` 描述發布補丁以供評審的過程。為了讓開發社群能
 認真對待，補丁必須被正確格式化和描述，並且必須發送到正確的地方。遵循本節中的
 建議有助於確保您的工作能被較好地接納。
 
-:ref:`tw_development_followthrough` 介紹了發佈補丁之後發生的事情；工作在這時
+:ref:`tw_development_followthrough` 介紹了發布補丁之後發生的事情；工作在這時
 還遠遠沒有完成。與審閱者一起工作是開發過程中的一個重要部分；本節提供了一些
 關於如何在這個重要階段避免問題的提示。當補丁被合併到主線中時，開發人員要注意
 不要假定任務已經完成。
 
-:ref:`tw_development_advancedtopics` 介紹了兩個“高級”主題：使用Git管理補丁
-和查看其他人發佈的補丁。
+:ref:`tw_development_advancedtopics` 介紹了兩個“進階”主題：使用Git管理補丁
+和查看其他人發布的補丁。
 
-:ref:`tw_development_conclusion` 總結了有關內核開發的更多信息，附帶有相關資源
-鏈接。
+:ref:`tw_development_conclusion` 總結了有關核心開發的更多資訊，附帶有相關資源
+連結。
 
-這個文檔是關於什麼的
+這個文件是關於什麼的
 --------------------
 
-Linux內核有超過800萬行代碼，每個版本的貢獻者超過1000人，是現存最大、最活躍的
-免費軟件項目之一。從1991年開始，這個內核已經發展成爲一個最好的操作系統組件，
-運行在袖珍數字音樂播放器、臺式電腦、現存最大的超級計算機以及所有類型的系統上。
-它是一種適用於幾乎任何情況的健壯、高效和可擴展的解決方案。
-
-隨着Linux的發展，希望參與其開發的開發人員（和公司）的數量也在增加。硬件供應商
-希望確保Linux能夠很好地支持他們的產品，使這些產品對Linux用戶具有吸引力。嵌入
-式系統供應商使用Linux作爲集成產品的組件，希望Linux能夠儘可能地勝任手頭的任務。
-分銷商和其他基於Linux的軟件供應商切實關心Linux內核的功能、性能和可靠性。最終
-用戶也常常希望修改Linux，使之能更好地滿足他們的需求。
-
-Linux最引人注目的特性之一是這些開發人員可以訪問它；任何具備必要技能的人都可以
-改進Linux並影響其開發方向。專有產品不能提供這種開放性，這是自由軟件的一個特點。
-如果有什麼不同的話，那就是內核比大多數其他自由軟件項目更開放。一個典型的三個
-月內核開發週期可以涉及1000多個開發人員，他們爲100多個不同的公司（或者根本不
-隸屬公司）工作。
-
-與內核開發社區合作並不是特別困難。但儘管如此，仍有許多潛在的貢獻者在嘗試做
-內核工作時遇到了困難。內核社區已經發展出自己獨特的操作方式，使其能夠在每天
-都要更改數千行代碼的環境中順利運行（並生成高質量的產品）。因此，Linux內核開發
-過程與專有的開發模式有很大的不同也就不足爲奇了。
-
-對於新開發人員來說，內核的開發過程可能會讓人感到奇怪和恐懼，但這背後有充分的
-理由和堅實的經驗。一個不瞭解內核社區工作方式的開發人員（或者更糟的是，他們
-試圖拋棄或規避之）會得到令人沮喪的體驗。開發社區在幫助那些試圖學習的人的同時，
+Linux核心有超過800萬行程式碼，每個版本的貢獻者超過1000人，是現存最大、最活躍的
+自由軟體專案之一。從1991年開始，這個核心已經發展成為一個最好的作業系統元件，
+執行在袖珍數位音樂播放器、桌上型電腦、現存最大的超級電腦以及所有類型的系統上。
+它是一種適用於幾乎任何情況的強健、高效和可擴展的解決方案。
+
+隨著Linux的發展，希望參與其開發的開發人員（和公司）的數量也在增加。硬體供應商
+希望確保Linux能夠很好地支援他們的產品，使這些產品對Linux使用者具有吸引力。嵌入
+式系統供應商使用Linux作為整合產品的元件，希望Linux能夠儘可能地勝任手頭的任務。
+分銷商和其他基於Linux的軟體供應商切實關心Linux核心的功能、效能和可靠性。最終
+使用者也常常希望修改Linux，使之能更好地滿足他們的需求。
+
+Linux最引人注目的特性之一是這些開發人員可以存取它；任何具備必要技能的人都
+可以改進Linux並影響其開發方向。專有產品不能提供這種開放性，這是自由軟體的
+一個特點。如果有什麼不同的話，那就是核心比大多數其他自由軟體專案更開放。一
+個典型的三個月核心開發週期可以涉及1000多個開發人員，他們為100多個不同的公
+司（或者根本不隸屬公司）工作。
+
+與核心開發社群合作並不是特別困難。但儘管如此，仍有許多潛在的貢獻者在嘗試做
+核心工作時遇到了困難。核心社群已經發展出自己獨特的操作方式，使其能夠在每天
+都要更改數千行程式碼的環境中順利執行（並產生高品質的產品）。因此，Linux核
+心開發過程與專有的開發模式有很大的不同也就不足為奇了。
+
+對於新開發人員來說，核心的開發過程可能會讓人感到奇怪和恐懼，但這背後有充分的
+理由和堅實的經驗。一個不瞭解核心社群工作方式的開發人員（或者更糟的是，他們
+試圖拋棄或規避之）會得到令人沮喪的體驗。開發社群在幫助那些試圖學習的人的同時，
 沒有時間幫助那些不願意傾聽或不關心開發過程的人。
 
 希望閱讀本文的人能夠避免這種令人沮喪的經歷。這些材料很長，但閱讀它們時所做的
-努力會在短時間內得到回報。開發社區總是需要能讓內核變更好的開發人員；下面的
-文字應該幫助您或爲您工作的人員加入我們的社區。
+努力會在短時間內得到回報。開發社群總是需要能讓核心變更好的開發人員；下面的
+文字應該幫助您或為您工作的人員加入我們的社群。
 
 致謝
 ----
 
-本文檔由Jonathan Corbet <corbet@lwn.net> 撰寫。以下人員的建議使之更爲完善：
+本文件由Jonathan Corbet <corbet@lwn.net> 撰寫。以下人員的建議使之更為完善：
 Johannes Berg, James Berry, Alex Chiang, Roland Dreier, Randy Dunlap,
 Jake Edge, Jiri Kosina, Matt Mackall, Arthur Marsh, Amanda McPherson,
 Andrew Morton, Andrew Price, Tsugikazu Shibata 和 Jochen Voß 。
 
-這項工作得到了Linux基金會的支持，特別感謝Amanda McPherson，他看到了這項工作
+這項工作得到了Linux基金會的支援，特別感謝Amanda McPherson，他看到了這項工作
 的價值並將其變成現實。
 
-代碼進入主線的重要性
---------------------
+程式碼進入主線的重要性
+----------------------
 
-有些公司和開發人員偶爾會想，爲什麼他們要費心學習如何與內核社區合作，並將代碼
-放入主線內核（“主線”是由Linus Torvalds維護的內核，Linux發行商將其用作基礎）。
-在短期內，貢獻代碼看起來像是一種可以避免的開銷；維護獨立代碼並直接支持用戶
-似乎更容易。事實上，保持代碼獨立（“樹外”）是在經濟上是錯誤的。
+有些公司和開發人員偶爾會想，為什麼他們要費心學習如何與核心社群合作，並將程
+式碼放入主線核心（“主線”是由Linus Torvalds維護的核心，Linux發行商將其用作
+基礎）。在短期內，貢獻程式碼看起來像是一種可以避免的開銷；維護獨立程式碼並
+直接支援使用者似乎更容易。事實上，保持程式碼獨立（“樹外”）是在經濟上是錯誤
+的。
 
-爲了說明樹外代碼成本，下面給出內核開發過程的一些相關方面；本文稍後將更詳細地
+為了說明樹外程式碼成本，下面給出核心開發過程的一些相關方面；本文稍後將更詳細地
 討論其中的大部分內容。請考慮：
 
-- 所有Linux用戶都可以使用合併到主線內核中的代碼。它將自動出現在所有啓用它的
-  發行版上。無需驅動程序磁盤、額外下載，也不需要爲多個發行版的多個版本提供
-  支持；這一切將方便所有開發人員和用戶。併入主線解決了大量的分發和支持問題。
+- 所有Linux使用者都可以使用合併到主線核心中的程式碼。它將自動出現在所有啟
+  用它的發行版上。無需驅動程式磁碟、額外下載，也不需要為多個發行版的多個版
+  本提供支援；這一切將方便所有開發人員和使用者。併入主線解決了大量的分發和
+  支援問題。
 
-- 當內核開發人員努力維護一個穩定的用戶空間接口時，內核內部API處於不斷變化之中。
-  不維持穩定的內部接口是一個慎重的設計決策；它允許在任何時候進行基本的改進，
-  併產出更高質量的代碼。但該策略導致結果是，若要使用新的內核，任何樹外代碼都
-  需要持續的維護。維護樹外代碼會需要大量的工作才能使代碼保持正常運行。
+- 當核心開發人員努力維護一個穩定的使用者空間介面時，核心內部API處於不斷變
+  化之中。不維持穩定的內部介面是一個慎重的設計決策；它允許在任何時候進行基
+  本的改進，並產出更高品質的程式碼。但該策略導致結果是，若要使用新的核心，
+  任何樹外程式碼都需要持續的維護。維護樹外程式碼會需要大量的工作才能使程式
+  碼保持正常執行。
 
-  相反，位於主線中的代碼不需要這樣做，因爲基本規則要求進行API更改的任何開發
-  人員也必須修復由於該更改而破壞的任何代碼。因此，合併到主線中的代碼大大降低
+  相反，位於主線中的程式碼不需要這樣做，因為基本規則要求進行API更改的任何開發
+  人員也必須修復由於該更改而破壞的任何程式碼。因此，合併到主線中的程式碼大大降低
   了維護成本。
 
-- 除此之外，內核中的代碼通常會被其他開發人員改進。您授權的用戶社區和客戶對您
-  產品的改進可能會令人驚喜。
+- 除此之外，核心中的程式碼通常會被其他開發人員改進。您授權的使用者社群和客
+  戶對您產品的改進可能會令人驚喜。
 
-- 內核代碼在合併到主線之前和之後都要經過審查。無論原始開發人員的技能有多強，
-  這個審查過程總是能找到改進代碼的方法。審查經常發現嚴重的錯誤和安全問題。
-  對於在封閉環境中開發的代碼尤其如此；這種代碼從外部開發人員的審查中獲益匪淺。
-  樹外代碼是低質量代碼。
+- 核心程式碼在合併到主線之前和之後都要經過審查。無論原始開發人員的技能有多
+  強，這個審查過程總是能找到改進程式碼的方法。審查經常發現嚴重的錯誤和安全
+  問題。對於在封閉環境中開發的程式碼尤其如此；這種程式碼從外部開發人員的審
+  查中獲益匪淺。樹外程式碼是低品質程式碼。
 
-- 參與開發過程是您影響內核開發方向的方式。旁觀者的抱怨會被聽到，但是活躍的
-  開發人員有更強的聲音——並且能夠實現使內核更好地滿足其需求的更改。
+- 參與開發過程是您影響核心開發方向的方式。旁觀者的抱怨會被聽到，但是活躍的
+  開發人員有更強的聲音——並且能夠實作使核心更好地滿足其需求的更改。
 
-- 當單獨維護代碼時，總是存在第三方爲類似功能提供不同實現的可能性。如果發生
-  這種情況，合併代碼將變得更加困難——甚至成爲不可能。之後，您將面臨以下令人
-  不快的選擇：（1）無限期地維護樹外的非標準特性，或（2）放棄代碼並將用戶遷移
-  到樹內版本。
+- 當單獨維護程式碼時，總是存在第三方為類似功能提供不同實作的可能性。如果發
+  生這種情況，合併程式碼將變得更加困難——甚至成為不可能。之後，您將面臨以下
+  令人不快的選擇：（1）無限期地維護樹外的非標準特性，或（2）放棄程式碼並將
+  使用者遷移到樹內版本。
 
-- 代碼的貢獻是使整個流程工作的根本。通過貢獻代碼，您可以向內核添加新功能，並
-  提供其他內核開發人員使用的功能和示例。如果您已經爲Linux開發了代碼（或者正在
-  考慮這樣做），那麼您顯然對這個平臺的持續成功感興趣；貢獻代碼是確保成功的
-  最好方法之一。
+- 程式碼的貢獻是使整個流程工作的根本。透過貢獻程式碼，您可以向核心添加新功
+  能，並提供其他核心開發人員使用的功能和範例。如果您已經為Linux開發了程式
+  碼（或者正在考慮這樣做），那麼您顯然對這個平臺的持續成功感興趣；貢獻程式
+  碼是確保成功的最好方法之一。
 
-上述所有理由都適用於任何樹外內核代碼，包括以專有的、僅二進制形式分發的代碼。
-然而，在考慮任何類型的純二進制內核代碼分佈之前，還需要考慮其他因素。包括：
+上述所有理由都適用於任何樹外核心程式碼，包括以專有的、僅二進位形式分發的程
+式碼。然而，在考慮任何類型的純二進位核心程式碼分發之前，還需要考慮其他因素。
+包括：
 
-- 圍繞專有內核模塊分發的法律問題其實較爲模糊；相當多的內核版權所有者認爲，
-  大多數僅二進制的模塊是內核的派生產品，因此，它們的分發違反了GNU通用公共
-  許可證（下面將詳細介紹）。本文作者不是律師，本文檔中的任何內容都不可能被
-  視爲法律建議。封閉源代碼模塊的真實法律地位只能由法院決定。但不管怎樣，困擾
-  這些模塊的不確定性仍然存在。
+- 圍繞專有核心模組分發的法律問題其實較為模糊；相當多的核心版權所有者認為，
+  大多數僅二進位的模組是核心的派生產品，因此，它們的分發違反了GNU通用公共
+  許可證（下面將詳細介紹）。本文作者不是律師，本文件中的任何內容都不可能被
+  視為法律建議。封閉原始程式碼模組的真實法律地位只能由法院決定。但不管怎樣，
+  困擾這些模組的不確定性仍然存在。
 
-- 二進制模塊大大增加了調試內核問題的難度，以至於大多數內核開發人員甚至都不會
-  嘗試。因此，只分發二進制模塊將使您的用戶更難從社區獲得支持。
+- 二進位模組大大增加了除錯核心問題的難度，以至於大多數核心開發人員甚至都不會
+  嘗試。因此，只分發二進位模組將使您的使用者更難從社群獲得支援。
 
-- 對於僅二進制的模塊的發行者來說，支持也更加困難，他們必須爲他們希望支持的
-  每個發行版和每個內核版本提供不同版本的模塊。爲了提供較爲全面的覆蓋範圍，
-  可能需要一個模塊的幾十個構建，並且每次升級內核時，您的用戶都必須單獨升級
-  這些模塊。
+- 對於僅二進位的模組的發行者來說，支援也更加困難，他們必須為他們希望支援的
+  每個發行版和每個核心版本提供不同版本的模組。為了提供較為全面的覆蓋範圍，
+  可能需要一個模組的幾十個建置，並且每次升級核心時，您的使用者都必須單獨升級
+  這些模組。
 
-- 上面提到的關於代碼評審的所有問題都更加存在於封閉源代碼中。由於該代碼根本
-  不可得，因此社區無法對其進行審查，毫無疑問，它將存在嚴重問題。
+- 上面提到的關於程式碼評審的所有問題都更加存在於封閉原始程式碼中。由於該程
+  式碼根本不可得，因此社群無法對其進行審查，毫無疑問，它將存在嚴重問題。
 
-尤其是嵌入式系統的製造商，可能會傾向於忽視本節中所說的大部分內容；因爲他們
-相信自己正在商用一種使用凍結內核版本的獨立產品，在發佈後不需要再進行開發。
-這個論點忽略了廣泛的代碼審查的價值以及允許用戶向產品添加功能的價值。但這些
-產品的商業壽命有限，之後必須發佈新版本的產品。在這一點上，代碼在主線上並得到
+尤其是嵌入式系統的製造商，可能會傾向於忽視本節中所說的大部分內容；因為他們
+相信自己正在商用一種使用凍結核心版本的獨立產品，在發布後不需要再進行開發。
+這個論點忽略了廣泛的程式碼審查的價值以及允許使用者向產品添加功能的價值。但這些
+產品的商業壽命有限，之後必須發布新版本的產品。在這一點上，程式碼在主線上並得到
 良好維護的供應商將能夠更好地佔位，以使新產品快速上市。
 
 許可
 ----
 
-代碼是根據一些許可證提供給Linux內核的，但是所有代碼都必須與GNU通用公共許可
-證（GPLV2）的版本2兼容，該版本是覆蓋整個內核分發的許可證。在實踐中，這意味
-着所有代碼貢獻都由GPLv2（可選地，語言允許在更高版本的GPL下分發）或3子句BSD
-許可（New BSD License，譯者注）覆蓋。任何不包含在兼容許可證中的貢獻都不會
-被接受到內核中。
+程式碼是根據一些許可證提供給Linux核心的，但是所有程式碼都必須與GNU通用公共許可
+證（GPLv2）的版本2相容，該版本是覆蓋整個核心分發的許可證。在實踐中，這意味
+著所有程式碼貢獻都由GPLv2（可選地，語言允許在更高版本的GPL下分發）或3子句BSD
+許可（New BSD License，譯者注）覆蓋。任何不包含在相容許可證中的貢獻都不會
+被接受到核心中。
 
-貢獻給內核的代碼不需要（或請求）版權分配。合併到主線內核中的所有代碼都保留
-其原始所有權；因此，內核現在擁有數千個所有者。
+貢獻給核心的程式碼不需要（或請求）版權分配。合併到主線核心中的所有程式碼都保留
+其原始所有權；因此，核心現在擁有數千個所有者。
 
-這種所有權結構也暗示着，任何改變內核許可的嘗試都註定會失敗。很少有實際情況
-可以獲得所有版權所有者的同意（或者從內核中刪除他們的代碼）。因此，尤其是在
+這種所有權結構也暗示著，任何改變核心許可的嘗試都註定會失敗。很少有實際情況
+可以獲得所有版權所有者的同意（或者從核心中刪除他們的程式碼）。因此，尤其是在
 可預見的將來，許可證不大可能遷移到GPL的版本3。
 
-所有貢獻給內核的代碼都必須是合法的免費軟件。因此，不接受匿名（或化名）貢獻
-者的代碼。所有貢獻者都需要在他們的代碼上“sign off（簽發）”，聲明代碼可以
-在GPL下與內核一起分發。無法提供未被其所有者許可爲免費軟件的代碼，或可能爲
-內核造成版權相關問題的代碼（例如，由缺乏適當保護的反向工程工作派生的代碼）
-不能被接受。
+所有貢獻給核心的程式碼都必須是合法的自由軟體。因此，不接受身分不明或匿名的
+貢獻者的程式碼。所有貢獻者都需要在他們的程式碼上“sign off（簽發）”，聲明程
+式碼可以在GPL下與核心一起分發。無法提供未被其所有者許可為自由軟體的程式碼，
+或可能為核心造成版權相關問題的程式碼（例如，由缺乏適當保護的反向工程工作派
+生的程式碼）不能被接受。
 
-有關版權問題的提問在Linux開發郵件列表中很常見。這樣的問題通常會得到不少答案，
-但請記住，回答這些問題的人不是律師，不能提供法律諮詢。如果您有關於Linux源代碼
-的法律問題，沒有什麼可以代替諮詢瞭解這一領域的律師。依賴從技術郵件列表中獲得
-的答案是一件冒險的事情。
+有關版權問題的提問在Linux開發郵件列表中很常見。這樣的問題通常會得到不少答
+案，但請記住，回答這些問題的人不是律師，不能提供法律諮詢。如果您有關於
+Linux原始程式碼的法律問題，沒有什麼可以代替諮詢瞭解這一領域的律師。依賴從
+技術郵件列表中獲得的答案是一件冒險的事情。
 
 
diff --git a/Documentation/translations/zh_TW/process/2.Process.rst b/Documentation/translations/zh_TW/process/2.Process.rst
index f45ddba6238ff3..36b109622ea6ba 100644
--- a/Documentation/translations/zh_TW/process/2.Process.rst
+++ b/Documentation/translations/zh_TW/process/2.Process.rst
@@ -12,64 +12,60 @@
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
  胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+ 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_development_process:
 
 開發流程如何進行
 ================
 
-90年代早期的Linux內核開發是一件相當鬆散的事情，涉及的用戶和開發人員相對較少。
-由於擁有數以百萬計的用戶羣，且每年有大約2000名開發人員參與進來，內核因此必須
-發展出許多既定流程來保證開發的順利進行。要參與到流程中來，需要對此流程的進行
-方式有一個紮實的理解。
+90年代早期的Linux核心開發是一件相當鬆散的事情，涉及的使用者和開發人員相對
+較少。由於擁有數以百萬計的使用者羣，且每年有大約2000名開發人員參與進來，核
+心因此必須發展出許多既定流程來保證開發的順利進行。要參與到流程中來，需要對
+此流程的進行方式有一個紮實的理解。
 
 總覽
 ----
 
-內核開發人員使用一個鬆散的基於時間的發佈過程，每兩到三個月發佈一次新的主要
-內核版本。最近的發佈歷史記錄如下：
+Linux核心使用一種鬆散的、基於時間的滾動發布開發模式。每兩到三個月就會有
+一個新的主要核心版本發布（作為範例，我們稱之為9.x） [1]_ ，它帶來新特性、
+內部API更改等等。一個典型的版本可以包含大約13000個變更集，變更了幾十萬行
+程式碼。最近的版本及其日期可以在
+`維基百科 <https://en.wikipedia.org/wiki/Linux_kernel_version_history>`_
+找到。
 
-	======  =================
-	5.0	2019年3月3日
-	5.1	2019年5月5日
-	5.2	2019年7月7日
-	5.3	2019年9月15日
-	5.4	2019年11月24日
-	5.5	2020年1月6日
-	======  =================
-
-每個5.x版本都是一個主要的內核版本，具有新特性、內部API更改等等。一個典型的5.x
-版本包含大約13000個變更集，變更了幾十萬行代碼。因此，5.x是Linux內核開發的前
-沿；內核使用滾動開發模型，不斷集成重大變化。
+.. [1] 嚴格來說，Linux核心並不使用語意化版本編號方案，而是以9.x這一對
+       數字作為一個整體來標識主要發布版本。每次發布時x會遞增，只有當x被
+       認為足夠大時才會遞增9（例如，Linux 5.0是繼Linux 4.20之後發布的）。
 
 對於每個版本的補丁合併，遵循一個相對簡單的規則。在每個開發週期的開頭，“合併
-窗口”被打開。這時，被認爲足夠穩定（並且被開發社區接受）的代碼被合併到主線內
+視窗”被開啟。這時，被認為足夠穩定（並且被開發社群接受）的程式碼被合併到主線內
 核中。在這段時間內，新開發週期的大部分變更（以及所有主要變更）將以接近每天
 1000次變更（“補丁”或“變更集”）的速度合併。
 
-（順便說一句，值得注意的是，合併窗口期間集成的更改並不是憑空產生的；它們是經
+（順便說一句，值得注意的是，合併視窗期間整合的更改並不是憑空產生的；它們是經
 提前收集、測試和分級的。稍後將詳細描述該過程的工作方式。）
 
-合併窗口持續大約兩週。在這段時間結束時，Linus Torvalds將聲明窗口已關閉，並
-釋放第一個“rc”內核。例如，對於目標爲5.6的內核，在合併窗口結束時發生的釋放
-將被稱爲5.6-rc1。-rc1 版本是一個信號，表示合併新特性的時間已經過去，穩定下一
-個內核的時間已經到來。
+合併視窗持續大約兩週。在這段時間結束時，Linus Torvalds將聲明視窗已關閉，並
+釋放第一個“rc”核心。例如，對於目標為9.x的核心，在合併視窗結束時發生的釋放
+將被稱為9.x-rc1。-rc1 版本是一個信號，表示合併新特性的時間已經過去，穩定下一
+個核心的時間已經到來。
 
 在接下來的6到10周內，只有修復問題的補丁才應該提交給主線。有時會允許更大的
-更改，但這種情況很少發生；試圖在合併窗口外合併新功能的開發人員往往受不到
-友好的接待。一般來說，如果您錯過了給定特性的合併窗口，最好的做法是等待下一
-個開發週期。（偶爾會對未支持硬件的驅動程序進行例外；如果它們不改變已有代碼，
+更改，但這種情況很少發生；試圖在合併視窗外合併新功能的開發人員往往受不到
+友好的接待。一般來說，如果您錯過了給定特性的合併視窗，最好的做法是等待下一
+個開發週期。（偶爾會對未支援硬體的驅動程式進行例外；如果它們不改變已有程式碼，
 則不會導致迴歸，應該可以隨時被安全地加入）。
 
-隨着修復程序進入主線，補丁速度將隨着時間的推移而變慢。Linus大約每週發佈一次
-新的-rc內核；在內核被認爲足夠穩定並最終發佈前，一般會達到-rc6到-rc9之間。
+隨著修復程式進入主線，補丁速度將隨著時間的推移而變慢。Linus大約每週發布一次
+新的-rc核心；在核心被認為足夠穩定並最終發布前，一般會達到-rc6到-rc9之間。
 然後，整個過程又重新開始了。
 
-例如，這裏是5.4的開發週期進行情況（2019年）:
+例如，這裡是5.4的開發週期進行情況（2019年）:
 
 	==============  ==============================
-	九月 15         5.3 穩定版發佈
-	九月 30         5.4-rc1 合併窗口關閉
+	九月 15         5.3 穩定版發布
+	九月 30         5.4-rc1 合併視窗關閉
 	十月 6          5.4-rc2
 	十月 13         5.4-rc3
 	十月 20         5.4-rc4
@@ -77,26 +73,26 @@
 	十一月 3        5.4-rc6
 	十一月 10       5.4-rc7
 	十一月 17       5.4-rc8
-	十一月 24       5.4 穩定版發佈
+	十一月 24       5.4 穩定版發布
 	==============  ==============================
 
-開發人員如何決定何時結束開發週期並創建穩定版本？最重要的指標是以前版本的
-迴歸列表。不歡迎出現任何錯誤，但是那些破壞了以前能工作的系統的錯誤被認爲是
+開發人員如何決定何時結束開發週期並建立穩定版本？最重要的指標是以前版本的
+迴歸列表。不歡迎出現任何錯誤，但是那些破壞了以前能工作的系統的錯誤被認為是
 特別嚴重的。因此，導致迴歸的補丁是不受歡迎的，很可能在穩定期內刪除。
 
-開發人員的目標是在穩定發佈之前修復所有已知的迴歸。在現實世界中，這種完美是
-很難實現的；在這種規模的項目中，變數太多了。需要說明的是，延遲最終版本只會
-使問題變得更糟；等待下一個合併窗口的更改將變多，導致下次出現更多的迴歸錯誤。
-因此，大多數5.x內核都有一些已知的迴歸錯誤，不過，希望沒有一個是嚴重的。
+開發人員的目標是在穩定發布之前修復所有已知的迴歸。在現實世界中，這種完美是
+很難實作的；在這種規模的專案中，變數太多了。需要說明的是，延遲最終版本只會
+使問題變得更糟；等待下一個合併視窗的更改將變多，導致下次出現更多的迴歸錯誤。
+因此，大多數核心版本都有一些已知的迴歸錯誤，不過，希望沒有一個是嚴重的。
 
-一旦一個穩定的版本發佈，它的持續維護工作就被移交給“穩定團隊”，目前由
-Greg Kroah-Hartman領導。穩定團隊將使用5.x.y編號方案不定期地發佈穩定版本的
-更新。要合入更新版本，補丁必須（1）修復一個重要的缺陷，且（2）已經合併到
-下一個開發版本主線中。內核通常會在其初始版本後的一個以上的開發週期內收到
-穩定版更新。例如，5.2內核的歷史如下（2019年）：
+一旦一個穩定的版本發布，它的持續維護工作就被移交給“穩定團隊”，目前由Greg
+Kroah-Hartman和Sasha Levin組成。穩定團隊將使用9.x.y編號方案不定期地發布穩
+定版本的更新。要合入更新版本，補丁必須（1）修復一個重要的缺陷，且（2）已經
+合併到下一個開發版本主線中。核心通常會在其初始版本後的一個以上的開發週期內
+收到穩定版更新。例如，5.2核心的歷史如下（2019年）：
 
 	==============  ===============================
-        七月 7 	        5.2 穩定版發佈
+        七月 7 	        5.2 穩定版發布
 	七月 13	        5.2.1
 	七月 21	        5.2.2
 	七月 26	        5.2.3
@@ -108,36 +104,29 @@ Greg Kroah-Hartman領導。穩定團隊將使用5.x.y編號方案不定期地發
 
 5.2.21是5.2版本的最終穩定更新。
 
-有些內核被指定爲“長期”內核；它們將得到更長時間的支持。在本文中，當前的長期
-內核及其維護者是：
+有些核心被指定為“長期”核心；它們將得到更長時間的支援。當前的長期核心
+版本及其維護者的列表，請參考以下連結：
 
-	======  ================================	================
-	3.16	Ben Hutchings				（長期穩定內核）
-	4.4	Greg Kroah-Hartman & Sasha Levin	（長期穩定內核）
-	4.9	Greg Kroah-Hartman & Sasha Levin
-	4.14	Greg Kroah-Hartman & Sasha Levin
-	4.19	Greg Kroah-Hartman & Sasha Levin
-	5.4	Greg Kroah-Hartman & Sasha Levin
-	======  ================================	================
+	https://www.kernel.org/category/releases.html
 
-長期支持內核的選擇純粹是維護人員是否有需求和時間來維護該版本的問題。
-目前還沒有爲即將發佈的任何特定版本提供長期支持的已知計劃。
+長期支援核心的選擇純粹是維護人員是否有需求和時間來維護該版本的問題。
 
 補丁的生命週期
 --------------
 
-補丁不會直接從開發人員的鍵盤進入主線內核。相反，有一個稍微複雜（如果有些非
-正式）的過程，旨在確保對每個補丁進行質量審查，並確保每個補丁實現了一個在主線
-中需要的更改。對於小的修復，這個過程可能會很快完成，，而對於較大或有爭議的
-變更，可能會持續數年。許多開發人員的沮喪來自於對這個過程缺乏理解或者試圖繞過它。
+補丁不會直接從開發人員的鍵盤進入主線核心。相反，有一個稍微複雜（如果有些非
+正式）的過程，旨在確保對每個補丁進行品質審查，並確保每個補丁實作了一個在主
+線中需要的更改。對於小的修復，這個過程可能會很快完成，，而對於較大或有爭議
+的變更，可能會持續數年。許多開發人員的沮喪來自於對這個過程缺乏理解或者試圖
+繞過它。
 
-爲了減少這種挫敗，本文將描述補丁如何進入內核。下面的介紹以一種較爲理想化的
+為了減少這種挫敗，本文將描述補丁如何進入核心。下面的介紹以一種較為理想化的
 方式描述了這個過程。更詳細的過程將在後面的章節中介紹。
 
 補丁通常要經歷以下階段：
 
 - 設計。這就是補丁的真正需求——以及滿足這些需求的方式——所在。設計工作通常
-  是在不涉及社區的情況下完成的，但是如果可能的話，最好是在公開的情況下完成
+  是在不涉及社群的情況下完成的，但是如果可能的話，最好是在公開的情況下完成
   這項工作；這樣可以節省很多稍後再重新設計的時間。
 
 - 早期評審。補丁被髮布到相關的郵件列表中，列表中的開發人員會回覆他們可能有
@@ -150,74 +139,75 @@ Greg Kroah-Hartman領導。穩定團隊將使用5.x.y編號方案不定期地發
   問題。
 
 - 請注意，大多數維護人員也有日常工作，因此合併補丁可能不是他們的最優先工作。
-  如果您的補丁得到了需要更改的反饋，那麼您應該進行這些更改，或者解釋爲何
+  如果您的補丁得到了需要更改的反饋，那麼您應該進行這些更改，或者解釋為何
   不應該進行這些更改。如果您的補丁沒有評審意見，也沒有被其相應的子系統或
-  驅動程序維護者接受，那麼您應該堅持不懈地將補丁更新到當前內核使其可被正常
-  應用，並不斷地發送它以供審查和合並。
+  驅動程式維護者接受，那麼您應該堅持不懈地將補丁更新到當前核心使其可被正常
+  應用，並不斷地發送它以供審查和合併。
 
-- 合併到主線。最終，一個成功的補丁將被合併到由LinusTorvalds管理的主線存儲庫
+- 合併到主線。最終，一個成功的補丁將被合併到由Linus Torvalds管理的主線儲存庫
   中。此時可能會出現更多的評論和/或問題；對開發人員來說應對這些問題並解決
   出現的任何問題仍很重要。
 
-- 穩定版發佈。大量用戶可能受此補丁影響，因此可能再次出現新的問題。
+- 穩定版發布。大量使用者可能受此補丁影響，因此可能再次出現新的問題。
 
-- 長期維護。雖然開發人員在合併代碼後可能會忘記代碼，但這種行爲往往會給開發
-  社區留下不良印象。合併代碼消除了一些維護負擔，因爲其他人將修復由API更改
-  引起的問題。但是，如果代碼要長期保持可用，原始開發人員應該繼續爲代碼負責。
+- 長期維護。雖然開發人員在合併程式碼後可能會忘記程式碼，但這種行為往往會給
+  開發社群留下不良印象。合併程式碼消除了一些維護負擔，因為其他人將修復由
+  API更改引起的問題。但是，如果程式碼要長期保持可用，原始開發人員應該繼續
+  為程式碼負責。
 
-內核開發人員（或他們的僱主）犯的最大錯誤之一是試圖將流程簡化爲一個“合併到
+核心開發人員（或他們的僱主）犯的最大錯誤之一是試圖將流程簡化為一個“合併到
 主線”步驟。這種方法總是會讓所有相關人員感到沮喪。
 
-補丁如何進入內核
+補丁如何進入核心
 ----------------
 
-只有一個人可以將補丁合併到主線內核存儲庫中：Linus Torvalds。但是，在進入
-2.6.38內核的9500多個補丁中，只有112個（大約1.3%）是由Linus自己直接選擇的。
-內核項目已經發展到一個沒有一個開發人員可以在沒有支持的情況下檢查和選擇每個
-補丁的規模。內核開發人員處理這種增長的方式是使用圍繞信任鏈構建的助理系統。
+只有一個人可以將補丁合併到主線核心儲存庫中：Linus Torvalds。但是，在進入
+2.6.38核心的9500多個補丁中，只有112個（大約1.3%）是由Linus自己直接選擇的。
+核心專案已經發展到一個沒有一個開發人員可以在沒有支援的情況下檢查和選擇每個
+補丁的規模。核心開發人員處理這種增長的方式是使用圍繞信任鏈建置的助理系統。
 
-內核代碼庫在邏輯上被分解爲一組子系統：網絡、特定體系結構支持、內存管理、視
-頻設備等。大多數子系統都有一個指定的維護人員，其總體負責該子系統中的代碼。
-這些子系統維護者（鬆散地）是他們所管理的內核部分的“守門員”；他們（通常）
-會接受一個補丁以包含到主線內核中。
+核心程式碼庫在邏輯上被分解為一組子系統：網路、特定體系結構支援、記憶體管理、視
+頻設備等。大多數子系統都有一個指定的維護人員，其總體負責該子系統中的程式碼。
+這些子系統維護者（鬆散地）是他們所管理的核心部分的“守門員”；他們（通常）
+會接受一個補丁以包含到主線核心中。
 
-子系統維護人員每個人都管理着自己版本的內核源代碼樹，通常（並非總是）使用Git。
-Git等工具（以及Quilt或Mercurial等相關工具）允許維護人員跟蹤補丁列表，包括作者
-信息和其他元數據。在任何給定的時間，維護人員都可以確定他或她的存儲庫中的哪
-些補丁在主線中找不到。
+子系統維護人員每個人都管理著自己版本的核心原始程式碼樹，通常（並非總是）使
+用Git。Git等工具（以及Quilt或Mercurial等相關工具）允許維護人員追蹤補丁列表，
+包括作者資訊和其他元資料。在任何給定的時間，維護人員都可以確定他或她的儲存
+庫中的哪些補丁在主線中找不到。
 
-當合並窗口打開時，頂級維護人員將要求Linus從存儲庫中“拉出”他們爲合併選擇
-的補丁。如果Linus同意，補丁流將流向他的存儲庫，成爲主線內核的一部分。
+當合併視窗開啟時，頂級維護人員將要求Linus從儲存庫中“拉出”他們為合併選擇
+的補丁。如果Linus同意，補丁流將流向他的儲存庫，成為主線核心的一部分。
 Linus對拉取中接收到的特定補丁的關注程度各不相同。很明顯，有時他看起來很
 關注。但是一般來說，Linus相信子系統維護人員不會向上遊發送壞補丁。
 
-子系統維護人員反過來也可以從其他維護人員那裏獲取補丁。例如，網絡樹是由首先
-在專用於網絡設備驅動程序、無線網絡等的樹中積累的補丁構建的。此存儲鏈可以
-任意長，但很少超過兩個或三個鏈接。由於鏈中的每個維護者都信任那些管理較低
-級別樹的維護者，所以這個過程稱爲“信任鏈”。
+子系統維護人員反過來也可以從其他維護人員那裡獲取補丁。例如，網路樹是由首先
+在專用於網路設備驅動程式、無線網路等的樹中積累的補丁建置的。此儲存鏈可以
+任意長，但很少超過兩個或三個連結。由於鏈中的每個維護者都信任那些管理較低
+級別樹的維護者，所以這個過程稱為“信任鏈”。
 
-顯然，在這樣的系統中，獲取內核補丁取決於找到正確的維護者。直接向Linus發送
+顯然，在這樣的系統中，獲取核心補丁取決於找到正確的維護者。直接向Linus發送
 補丁通常不是正確的方法。
 
 Next 樹
 -------
 
-子系統樹鏈引導補丁流到內核，但它也提出了一個有趣的問題：如果有人想查看爲
-下一個合併窗口準備的所有補丁怎麼辦？開發人員將感興趣的是，還有什麼其他的
-更改有待解決，以瞭解是否存在需要擔心的衝突；例如，更改核心內核函數原型的
-修補程序將與使用該函數舊形式的任何其他修補程序衝突。審查人員和測試人員希望
-在所有這些變更到達主線內核之前，能夠訪問它們的集成形式的變更。您可以從所有
+子系統樹鏈引導補丁流到核心，但它也提出了一個有趣的問題：如果有人想查看為
+下一個合併視窗準備的所有補丁怎麼辦？開發人員將感興趣的是，還有什麼其他的
+更改有待解決，以瞭解是否存在需要擔心的衝突；例如，更改核心核心函式原型的
+修補程式將與使用該函式舊形式的任何其他修補程式衝突。審查人員和測試人員希望
+在所有這些變更到達主線核心之前，能夠存取它們的整合形式的變更。您可以從所有
 相關的子系統樹中提取更改，但這將是一項複雜且容易出錯的工作。
 
-解決方案以-next樹的形式出現，在這裏子系統樹被收集以供測試和審查。這些樹中
-由Andrew Morton維護的較老的一個，被稱爲“-mm”（用於內存管理，創建時爲此）。
--mm 樹集成了一長串子系統樹中的補丁；它還包含一些旨在幫助調試的補丁。
+解決方案以-next樹的形式出現，在這裡子系統樹被收集以供測試和審查。這些樹中
+由Andrew Morton維護的較老的一個，被稱為“-mm”（用於記憶體管理，建立時為此）。
+-mm 樹整合了一長串子系統樹中的補丁；它還包含一些旨在幫助除錯的補丁。
 
 除此之外，-mm 還包含大量由Andrew直接選擇的補丁。這些補丁可能已經發布在郵件
-列表上，或者它們可能應用於內核中未指定子系統樹的部分。同時，-mm 作爲最後
+列表上，或者它們可能應用於核心中未指定子系統樹的部分。同時，-mm 作為最後
 手段的子系統樹；如果沒有其他明顯的路徑可以讓補丁進入主線，那麼它很可能最
 終選擇-mm 樹。累積在-mm 中的各種補丁最終將被轉發到適當的子系統樹，或者直接
-發送到Linus。在典型的開發週期中，大約5-10%的補丁通過-mm 進入主線。
+發送到Linus。在典型的開發週期中，大約5-10%的補丁透過-mm 進入主線。
 
 當前-mm 補丁可在“mmotm”（-mm of the moment）目錄中找到：
 
@@ -225,102 +215,103 @@ Next 樹
 
 然而，使用MMOTM樹可能會十分令人頭疼；它甚至可能無法編譯。
 
-下一個週期補丁合併的主要樹是linux-next，由Stephen Rothwell 維護。根據設計
-linux-next 是下一個合併窗口關閉後主線的快照。linux-next樹在Linux-kernel 和
-Linux-next 郵件列表中發佈，可從以下位置下載：
+下一個週期補丁合併的主要樹是linux-next，由Mark Brown維護。根據設計
+linux-next 是下一個合併視窗關閉後主線的快照。linux-next樹在Linux-kernel 和
+Linux-next 郵件列表中發布，可從以下位置下載：
 
         https://www.kernel.org/pub/linux/kernel/next/
 
-Linux-next 已經成爲內核開發過程中不可或缺的一部分；在一個給定的合併窗口中合併
-的所有補丁都應該在合併窗口打開之前的一段時間內找到進入Linux-next 的方法。
+Linux-next 已經成為核心開發過程中不可或缺的一部分；在一個給定的合併視窗中合併
+的所有補丁都應該在合併視窗開啟之前的一段時間內找到進入Linux-next 的方法。
 
 Staging 樹
 ----------
 
-內核源代碼樹包含drivers/staging/目錄，其中有許多驅動程序或文件系統的子目錄
-正在被添加到內核樹中。它們在仍然需要更多的修正的時候可以保留在driver/staging/
-目錄中；一旦完成，就可以將它們移到內核中。這是一種跟蹤不符合Linux內核編碼或
-質量標準的驅動程序的方法，人們可能希望使用它們並跟蹤開發。
+核心原始程式碼樹包含drivers/staging/目錄，其中有許多驅動程式或檔案系統的子目錄
+正在被添加到核心樹中。它們在仍然需要更多的修正的時候可以保留在driver/staging/
+目錄中；一旦完成，就可以將它們移到核心中。這是一種追蹤不符合Linux核心編碼或
+品質標準的驅動程式的方法，人們可能希望使用它們並追蹤開發。
 
-Greg Kroah Hartman 目前負責維護staging 樹。仍需要修正的驅動程序將發送給他，
-每個驅動程序在drivers/staging/中都有自己的子目錄。除了驅動程序源文件之外，
-目錄中還應該有一個TODO文件。TODO文件列出了驅動程序需要接受的暫停的工作，
-以及驅動程序的任何補丁都應該抄送的人員列表。當前的規則要求，staging的驅動
-程序必須至少正確編譯。
+Greg Kroah-Hartman 目前負責維護staging樹。仍需要修正的驅動程式將發送給他，
+每個驅動程式在drivers/staging/中都有自己的子目錄。除了驅動程式原始檔之外，
+目錄中還應該有一個TODO檔案。TODO檔案列出了驅動程式需要接受的暫停的工作，
+以及驅動程式的任何補丁都應該抄送的人員列表。當前的規則要求，staging的驅動
+程式必須至少正確編譯。
 
-Staging 是一種讓新的驅動程序進入主線的相對容易的方法，它們會幸運地引起其他
+Staging 是一種讓新的驅動程式進入主線的相對容易的方法，它們會幸運地引起其他
 開發人員的注意，並迅速改進。然而，進入staging並不是故事的結尾；staging中
-沒有看到常規進展的代碼最終將被刪除。經銷商也傾向於相對不願意使用staging驅動
-程序。因此，在成爲一個合適的主線驅動的路上，staging 僅是一箇中轉站。
+沒有看到常規進展的程式碼最終將被刪除。經銷商也傾向於相對不願意使用staging驅動
+程式。因此，在成為一個合適的主線驅動的路上，staging 僅是一箇中轉站。
 
 工具
 ----
 
-從上面的文本可以看出，內核開發過程在很大程度上依賴於在不同方向上聚集補丁的
+從上面的文字可以看出，核心開發過程在很大程度上依賴於在不同方向上聚集補丁的
 能力。如果沒有適當強大的工具，整個系統將無法在任何地方正常工作。關於如何使用
-這些工具的教程遠遠超出了本文檔的範圍，但還是用一點篇幅介紹一些關鍵點。
+這些工具的教程遠遠超出了本文件的範圍，但還是用一點篇幅介紹一些關鍵點。
 
-到目前爲止，內核社區使用的主要源代碼管理系統是git。Git是在自由軟件社區中開發
-的許多分佈式版本控制系統之一。它非常適合內核開發，因爲它在處理大型存儲庫和
-大量補丁時性能非常好。它也以難以學習和使用而著稱，儘管隨着時間的推移它變得
-更好了。對於內核開發人員來說，對Git的某種熟悉幾乎是一種要求；即使他們不將它
-用於自己的工作，他們也需要Git來跟上其他開發人員（以及主線）正在做的事情。
+到目前為止，核心社群使用的主要原始程式碼管理系統是git。Git是在自由軟體社群
+中開發的許多分散式版本控制系統之一。它非常適合核心開發，因為它在處理大型儲
+存庫和大量補丁時效能非常好。它也以難以學習和使用而著稱，儘管隨著時間的推移
+它變得更好了。對於核心開發人員來說，對Git的某種熟悉幾乎是一種要求；即使他
+們不將它用於自己的工作，他們也需要Git來跟上其他開發人員（以及主線）正在做
+的事情。
 
 現在幾乎所有的Linux發行版都打包了Git。Git主頁位於：
 
         https://git-scm.com/
 
-此頁面包含了文檔和教程的鏈接。
+此頁面包含了文件和教程的連結。
 
-在不使用git的內核開發人員中，最流行的選擇幾乎肯定是Mercurial：
+在不使用git的核心開發人員中，最流行的選擇幾乎肯定是Mercurial：
 
         http://www.seleric.com/mercurial/
 
-Mercurial與Git共享許多特性，但它提供了一個界面，許多人覺得它更易於使用。
+Mercurial與Git共享許多特性，但它提供了一個介面，許多人覺得它更易於使用。
 
 另一個值得了解的工具是Quilt:
 
         https://savannah.nongnu.org/projects/quilt
 
-Quilt 是一個補丁管理系統，而不是源代碼管理系統。它不會隨着時間的推移跟蹤歷史；
-相反，它面向根據不斷發展的代碼庫跟蹤一組特定的更改。一些主要的子系統維護人員
-使用Quilt來管理打算向上遊移動的補丁。對於某些樹的管理（例如-mm），quilt 是
-最好的工具。
+Quilt 是一個補丁管理系統，而不是原始程式碼管理系統。它不會隨著時間的推移追
+蹤歷史；相反，它面向根據不斷發展的程式碼庫追蹤一組特定的更改。一些主要的子
+系統維護人員使用Quilt來管理打算向上遊移動的補丁。對於某些樹的管理（例如-mm），
+quilt 是最好的工具。
 
 郵件列表
 --------
 
-大量的Linux內核開發工作是通過郵件列表完成的。如果不加入至少一個某個列表，
-就很難成爲社區中的一個“全功能”成員。但是，Linux郵件列表對開發人員來說也是
+大量的Linux核心開發工作是透過郵件列表完成的。如果不加入至少一個某個列表，
+就很難成為社群中的一個“全功能”成員。但是，Linux郵件列表對開發人員來說也是
 一個潛在的危險，他們可能會被一堆電子郵件淹沒、違反Linux列表上使用的約定，
 或者兩者兼而有之。
 
-大多數內核郵件列表都在vger.kernel.org上運行；主列表位於：
+大多數核心郵件列表都託管在kernel.org上；主列表位於：
 
-        http://vger.kernel.org/vger-lists.html
+        https://subspace.kernel.org
 
-不過，也有一些列表託管在別處；其中一些列表位於
-redhat.com/mailman/listinfo。
+也有一些列表託管在別處；請查閱MAINTAINERS檔案以找到與特定子系統相關的
+列表。
 
-當然，內核開發的核心郵件列表是linux-kernel。這個列表是一個令人生畏的地方：
-每天的信息量可以達到500條，噪音很高，談話技術性很強，且參與者並不總是表現出
-高度的禮貌。但是，沒有其他地方可以讓內核開發社區作爲一個整體聚集在一起；
-不使用此列表的開發人員將錯過重要信息。
+當然，核心開發的核心郵件列表是linux-kernel。這個列表是一個令人生畏的地方：
+每天的資訊量可以達到500條，噪音很高，談話技術性很強，且參與者並不總是表現出
+高度的禮貌。但是，沒有其他地方可以讓核心開發社群作為一個整體聚集在一起；
+不使用此列表的開發人員將錯過重要資訊。
 
 以下一些提示可以幫助在linux-kernel生存：
 
-- 將郵件轉移到單獨的文件夾，而不是主郵箱文件夾。我們必須能夠持續地忽略洪流。
+- 將郵件轉移到單獨的資料夾，而不是主信箱資料夾。我們必須能夠持續地忽略洪流。
 
 - 不要試圖跟上每一次談話——沒人會這樣。重要的是要篩選感興趣的主題（但請注意
   長時間的對話可能會偏離原來的主題，儘管未改變電子郵件的主題）和參與的人。
 
 - 不要回復挑事的人。如果有人試圖激起憤怒，請忽略他們。
 
-- 當回覆Linux內核電子郵件（或其他列表上的電子郵件）時，請爲所有相關人員保留
+- 當回覆Linux核心電子郵件（或其他列表上的電子郵件）時，請為所有相關人員保留
   Cc: 抄送頭。如果沒有確實的理由（如明確的請求），則不應刪除收件人。一定要
   確保你要回復的人在抄送列表中。這個慣例也使你不必在回覆郵件時明確要求被抄送。
 
-- 在提出問題之前，搜索列表存檔（和整個網絡）。有些開發人員可能會對那些顯然
+- 在提出問題之前，搜索列表存檔（和整個網路）。有些開發人員可能會對那些顯然
   沒有完成家庭作業的人感到不耐煩。
 
 - 避免頂部回覆（把你的答案放在你要回復的引文上面的做法）。這會讓你的回答更難
@@ -329,41 +320,41 @@ redhat.com/mailman/listinfo。
 - 在正確的郵件列表發問。linux-kernel 可能是通用的討論場所，但它不是尋找所有
   子系統開發人員的最佳場所。
 
-最後一點——找到正確的郵件列表——是開發人員常出錯的地方。在linux-kernel上
-提出與網絡相關的問題的人幾乎肯定會收到一個禮貌的建議，轉到netdev列表上提出，
-因爲這是大多數網絡開發人員經常出現的列表。還有其他列表可用於scsi、video4linux、
-ide、filesystem等子系統。查找郵件列表的最佳位置是與內核源代碼一起打包的
-MAINTAINERS文件。
+最後一點——找到正確的郵件列表——是開發人員常出錯的地方。在linux-kernel上提出
+與網路相關的問題的人幾乎肯定會收到一個禮貌的建議，轉到netdev列表上提出，因
+為這是大多數網路開發人員經常出現的列表。還有其他列表可用於scsi、
+video4linux、ide、filesystem等子系統。查找郵件列表的最佳位置是與核心原始程
+式碼一起打包的MAINTAINERS檔案。
 
-開始內核開發
+開始核心開發
 ------------
 
-關於如何開始內核開發過程的問題很常見——個人和公司皆然。同樣常見的是失誤，這
+關於如何開始核心開發過程的問題很常見——個人和公司皆然。同樣常見的是失誤，這
 使得關係的開始比本應的更困難。
 
-公司通常希望聘請知名的開發人員來啓動開發團隊。實際上，這是一種有效的技術。
-但它也往往是昂貴的，而且對增加有經驗的內核開發人員的數量沒有多大幫助。考
-慮到時間投入，可以讓內部開發人員加快Linux內核的開發速度。利用這段時間可以
-讓僱主擁有一批既瞭解內核又瞭解公司的開發人員，還可以幫助培訓其他人。從中期
+公司通常希望聘請知名的開發人員來啟動開發團隊。實際上，這是一種有效的技術。
+但它也往往是昂貴的，而且對增加有經驗的核心開發人員的數量沒有多大幫助。考
+慮到時間投入，可以讓內部開發人員加快Linux核心的開發速度。利用這段時間可以
+讓僱主擁有一批既瞭解核心又瞭解公司的開發人員，還可以幫助培訓其他人。從中期
 來看，這通常是更有利可圖的方法。
 
-可以理解的是，單個開發人員往往對起步感到茫然。從一個大型項目開始可能會很
-嚇人；人們往往想先用一些較小的東西來試試水。由此，一些開發人員開始創建修補
+可以理解的是，單個開發人員往往對起步感到茫然。從一個大型專案開始可能會很
+嚇人；人們往往想先用一些較小的東西來試試水。由此，一些開發人員開始建立修補
 拼寫錯誤或輕微編碼風格問題的補丁。不幸的是，這樣的補丁會產生一定程度的噪音，
-這會分散整個開發社區的注意力，因此，它們越來越被人不看重。希望向社區介紹
-自己的新開發人員將無法通過這些方式獲得他們期待的反響。
+這會分散整個開發社群的注意力，因此，它們越來越被人不看重。希望向社群介紹
+自己的新開發人員將無法透過這些方式獲得他們期待的反響。
 
-Andrew Morton 爲有抱負的內核開發人員提供瞭如下建議
+Andrew Morton 為有抱負的核心開發人員提供了如下建議
 
 ::
 
-	所有內核開發者的第一個項目肯定應該是“確保內核在您可以操作的所有
-	機器上始終完美運行”。通常的方法是和其他人一起解決問題（這可能需
-	要堅持！），但就是如此——這是內核開發的一部分。
+	所有核心開發者的第一個專案肯定應該是“確保核心在您可以操作的所有
+	機器上始終完美執行”。通常的方法是和其他人一起解決問題（這可能需
+	要堅持！），但就是如此——這是核心開發的一部分。
 
 (http://lwn.net/Articles/283982/)
 
 在沒有明顯問題需要解決的情況下，通常建議開發人員查看當前的迴歸和開放缺陷
-列表。從來都不缺少需要解決的問題；通過解決這些問題，開發人員將從該過程獲得
-經驗，同時與開發社區的其他成員建立相互尊重。
+列表。從來都不缺少需要解決的問題；透過解決這些問題，開發人員將從該過程獲得
+經驗，同時與開發社群的其他成員建立相互尊重。
 
diff --git a/Documentation/translations/zh_TW/process/5.Posting.rst b/Documentation/translations/zh_TW/process/5.Posting.rst
index 38f3a6d618ebfc..698cde5cd06de9 100644
--- a/Documentation/translations/zh_TW/process/5.Posting.rst
+++ b/Documentation/translations/zh_TW/process/5.Posting.rst
@@ -12,73 +12,78 @@
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
  胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+ 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_development_posting:
 
-發佈補丁
+發布補丁
 ========
 
-您的工作遲早會準備好提交給社區進行審查，並最終包含到主線內核中。毫不稀奇，
-內核開發社區已經發展出一套用於發佈補丁的約定和過程；遵循這些約定和過程將使
-參與其中的每個人的生活更加輕鬆。本文檔試圖描述這些約定的部分細節；更多信息
-也可在以下文檔中找到
-:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <tw_submittingpatches>`
-和 :ref:`Documentation/translations/zh_CN/process/submit-checklist.rst <tw_submitchecklist>`。
+您的工作遲早會準備好提交給社群進行審查，並最終包含到主線核心中。毫不稀奇，
+核心開發社群已經發展出一套用於發布補丁的約定和過程；遵循這些約定和過程將使
+參與其中的每個人的生活更加輕鬆。本文件試圖描述這些約定的部分細節；更多資訊
+也可在以下文件中找到
+:ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
+和
+:ref:`Documentation/translations/zh_TW/process/submit-checklist.rst <tw_submitchecklist>`。
 
 何時寄送
 --------
 
-在補丁完全“準備好”之前，避免發佈補丁是一種持續的誘惑。對於簡單的補丁，這
-不是問題。但是如果正在完成的工作很複雜，那麼在工作完成之前從社區獲得反饋就
-可以獲得很多好處。因此，您應該考慮發佈正在進行的工作，甚至維護一個可用的Git
+在補丁完全“準備好”之前，避免發布補丁是一種持續的誘惑。對於簡單的補丁，這
+不是問題。但是如果正在完成的工作很複雜，那麼在工作完成之前從社群獲得反饋就
+可以獲得很多好處。因此，您應該考慮發布正在進行的工作，甚至維護一個可用的Git
 樹，以便感興趣的開發人員可以隨時趕上您的工作。
 
-當發佈中有尚未準備好被包含的代碼，最好在發佈中說明。還應提及任何有待完成的
-主要工作和任何已知問題。很少有人會願意看那些被認爲是半生不熟的補丁，但是
-那些願意的人會帶着他們的點子來一起幫助你把工作推向正確的方向。
+當發布中有尚未準備好被包含的程式碼，最好在發布中說明。還應提及任何有待完成的
+主要工作和任何已知問題。很少有人會願意看那些被認為是半生不熟的補丁，但是
+那些願意的人會帶著他們的點子來一起幫助你把工作推向正確的方向。
 
-創建補丁之前
+建立補丁之前
 ------------
 
-在考慮將補丁發送到開發社區之前，有許多事情應該做。包括：
+在考慮將補丁發送到開發社群之前，有許多事情應該做。包括：
 
- - 儘可能地測試代碼。利用內核的調試工具，確保內核使用了所有可能的配置選項組合
-   進行構建，使用交叉編譯器爲不同的體系結構進行構建等。
+ - 儘可能地測試程式碼。利用核心的除錯工具，確保核心使用了所有可能的設定選
+   項組合進行建置，使用交叉編譯器為不同的架構進行建置等。添加測試，最好使
+   用現有的測試框架（如KUnit），並將其作為補丁系列中單獨的一員（關於補丁系
+   列，見下一節）。注意，對某些子系統而言這可能是強制性的。例如，函式庫函
+   式（位於lib/下）幾乎在所有地方被廣泛使用，應當有適當的測試。
 
- - 確保您的代碼符合內核代碼風格指南。
+ - 確保您的程式碼符合核心程式碼風格指南。
 
- - 您的更改是否具有性能影響？如果是這樣，您應該運行基準測試來顯示您的變更的
+ - 您的更改是否具有效能影響？如果是這樣，您應該執行基準測試來顯示您的變更的
    影響（或好處）；結果的摘要應該包含在補丁中。
 
- - 確保您有權發佈代碼。如果這項工作是爲僱主完成的，僱主對這項工作具有所有權，
-   並且必須同意根據GPL對其進行發佈。
+ - 確保您有權發布程式碼。如果這項工作是為僱主完成的，僱主對這項工作具有所
+   有權，並且必須同意根據GPL對其進行發布。
 
-一般來說，在發佈代碼之前進行一些額外的思考，幾乎總是能在短時間內得到回報。
+一般來說，在發布程式碼之前進行一些額外的思考，幾乎總是能在短時間內得到回報。
 
 補丁準備
 --------
 
-準備補丁發佈的工作量可能很驚人，但在此嘗試節省時間通常是不明智的，即使在短期
+準備補丁發布的工作量可能很驚人，但在此嘗試節省時間通常是不明智的，即使在短期
 內亦然。
 
-必須針對內核的特定版本準備補丁。一般來說，補丁應該基於Linus的Git樹中的當前
-主線。當以主線爲基礎時，請從一個衆所周知的發佈點開始——如穩定版本或 -rc
-版本發佈點——而不是在一個任意的主線分支點。
+必須針對核心的特定版本準備補丁。一般來說，補丁應該基於Linus的Git樹中的當前
+主線。當以主線為基礎時，請從一個衆所周知的發布點開始——如穩定版本或 -rc
+版本發布點——而不是在一個任意的主線分支點。
 
-也可能需要針對-mm、linux-next或子系統樹生成版本，以便於更廣泛的測試和審查。
+也可能需要針對-mm、linux-next或子系統樹產生版本，以便於更廣泛的測試和審查。
 根據補丁的區域以及其他地方的情況，針對其他樹建立的補丁可能需要大量的工作來
 解決衝突和處理API更改。
 
-只有最簡單的更改才應格式化爲單個補丁；其他所有更改都應作爲一系列邏輯更改進行。
-分割補丁是一門藝術；一些開發人員花了很長時間來弄清楚如何按照社區期望的方式來
+只有最簡單的更改才應格式化為單個補丁；其他所有更改都應作為一系列邏輯更改進行。
+分割補丁是一門藝術；一些開發人員花了很長時間來弄清楚如何按照社群期望的方式來
 分割。不過，這些經驗法則也許有幫助：
 
- - 您發佈的補丁系列幾乎肯定不會是開發過程中版本控制系統中的一系列更改。相反，
+ - 您發布的補丁系列幾乎肯定不會是開發過程中版本控制系統中的一系列更改。相反，
    需要對您所做更改的最終形式加以考慮，然後以有意義的方式進行拆分。開發人員對
    離散的、自包含的更改感興趣，而不是您創造這些更改的原始路徑。
 
- - 每個邏輯上獨立的變更都應該格式化爲單獨的補丁。這些更改可以是小的（如“向
-   此結構體添加字段”）或大的（如添加一個重要的新驅動程序），但它們在概念上
+ - 每個邏輯上獨立的變更都應該格式化為單獨的補丁。這些更改可以是小的（如“向
+   此結構體添加欄位”）或大的（如添加一個重要的新驅動程式），但它們在概念上
    應該是小的，並且可以在一行內簡述。每個補丁都應該做一個特定的、可以單獨
    檢查並驗證它所做的事情的更改。
 
@@ -86,132 +91,164 @@
    一個補丁修復了一個關鍵的安全漏洞，又重新排列了一些結構，還重新格式化了代
    碼，那麼它很有可能會被忽略，從而導致重要的修復丟失。
 
- - 每個補丁都應該能創建一個可以正確地構建和運行的內核；如果補丁系列在中間被
-   斷開，那麼結果仍應是一個正常工作的內核。部分應用一系列補丁是使用
-   “git bisct”工具查找回歸的一個常見場景；如果結果是一個損壞的內核，那麼將使
-   那些從事追蹤問題的高尚工作的開發人員和用戶的生活更加艱難。
+ - 每個補丁都應該能建立一個可以正確地建置和執行的核心；如果補丁系列在中間被
+   斷開，那麼結果仍應是一個正常工作的核心。部分應用一系列補丁是使用
+   “git bisct”工具查找回歸的一個常見場景；如果結果是一個損壞的核心，那麼將使
+   那些從事追蹤問題的高尚工作的開發人員和使用者的生活更加艱難。
 
- - 不要過分分割。一位開發人員曾經將一組針對單個文件的編輯分成500個單獨的補丁
-   發佈，這並沒有使他成爲內核郵件列表中最受歡迎的人。一個補丁可以相當大，
+ - 不要過分分割。一位開發人員曾經將一組針對單一檔案的編輯分成500個單獨的補丁
+   發布，這並沒有使他成為核心郵件列表中最受歡迎的人。一個補丁可以相當大，
    只要它仍然包含一個單一的 *邏輯* 變更。
 
- - 用一系列補丁添加一個全新的基礎設施，但是該設施在系列中的最後一個補丁啓用
+ - 用一系列補丁添加一個全新的基礎設施，但是該設施在系列中的最後一個補丁啟用
    整個變更之前不能使用，這看起來很誘人。如果可能的話，應該避免這種誘惑；
    如果這個系列增加了迴歸，那麼二分法將指出最後一個補丁是導致問題的補丁，
-   即使真正的bug在其他地方。只要有可能，添加新代碼的補丁程序應該立即激活該
-   代碼。
+   即使真正的bug在其他地方。只要有可能，添加新程式碼的補丁程式應該立即激活該
+   程式碼。
 
-創建完美補丁系列的工作可能是一個令人沮喪的過程，在完成“真正的工作”之後需要
+建立完美補丁系列的工作可能是一個令人沮喪的過程，在完成“真正的工作”之後需要
 花費大量的時間和思考。但是如果做得好，花費的時間就是值得的。
 
 補丁格式和更改日誌
 ------------------
 
-所以現在你有了一系列完美的補丁可以發佈，但是這項工作還沒有完成。每個補丁都
-需要被格式化成一條消息，以快速而清晰地將其目的傳達到世界其他地方。爲此，
+所以現在你有了一系列完美的補丁可以發布，但是這項工作還沒有完成。每個補丁都
+需要被格式化成一條訊息，以快速而清晰地將其目的傳達到世界其他地方。為此，
 每個補丁將由以下部分組成：
 
- - 可選的“From”行，表明補丁作者。只有當你通過電子郵件發送別人的補丁時，這一行
-   纔是必須的，但是爲防止疑問加上它也不會有什麼壞處。
+ - 可選的“From”行，表明補丁作者。只有當你透過電子郵件發送別人的補丁時，這一行
+   才是必須的，但是為防止疑問加上它也不會有什麼壞處。
 
- - 一行描述，說明補丁的作用。對於在沒有其他上下文的情況下看到該消息的讀者來說，
-   該消息應足以確定修補程序的範圍；此行將顯示在“short form（簡短格式）”變更
-   日誌中。此消息通常需要先加上子系統名稱前綴，然後是補丁的目的。例如：
+ - 一行描述，說明補丁的作用。對於在沒有其他上下文的情況下看到該訊息的讀者
+   來說，該訊息應足以確定修補程式的範圍；此行將顯示在“short form（簡短格式）”
+   變更日誌中。此訊息通常需要先加上子系統名稱前綴，然後是補丁的目的。例如：
 
    ::
 
         gpio: fix build on CONFIG_GPIO_SYSFS=n
 
- - 一行空白，後接補丁內容的詳細描述。此描述可以是任意需要的長度；它應該說明補丁
-   的作用以及爲什麼它應該應用於內核。
+ - 一行空白，後接補丁內容的詳細描述。此描述可以是任意需要的長度；它應該說
+   明補丁的作用以及為什麼它應該應用於核心。
 
  - 一個或多個標記行，至少有一個由補丁作者的 Signed-off-by 簽名。標記將在下面
    詳細描述。
 
-上面的項目一起構成補丁的變更日誌。寫一則好的變更日誌是一門至關重要但常常被
+上面的專案一起構成補丁的變更日誌。寫一則好的變更日誌是一門至關重要但常常被
 忽視的藝術；值得花一點時間來討論這個問題。當你編寫變更日誌時，你應該記住有
 很多不同的人會讀你的話。其中包括子系統維護人員和審查人員，他們需要決定是否
-應該合併補丁，分銷商和其他維護人員試圖決定是否應該將補丁反向移植到其他內核，
-缺陷搜尋人員想知道補丁是否導致他們正在追查的問題，以及想知道內核如何變化的
-用戶等等。一個好的變更日誌以最直接和最簡潔的方式向所有這些人傳達所需的信息。
+應該合併補丁，分銷商和其他維護人員試圖決定是否應該將補丁反向移植到其他核心，
+缺陷搜尋人員想知道補丁是否導致他們正在追查的問題，以及想知道核心如何變化的
+使用者等等。一個好的變更日誌以最直接和最簡潔的方式向所有這些人傳達所需的資訊。
 
 在結尾，總結行應該描述變更的影響和動機，以及在一行約束條件下可能發生的變化。
-然後，詳細的描述可以詳述這些主題，並提供任何需要的附加信息。如果補丁修復了
+然後，詳細的描述可以詳述這些主題，並提供任何需要的附加資訊。如果補丁修復了
 一個缺陷，請引用引入該缺陷的提交（如果可能，請在引用提交時同時提供其 id 和
 標題）。如果某個問題與特定的日誌或編譯器輸出相關聯，請包含該輸出以幫助其他
-人搜索同一問題的解決方案。如果更改是爲了支持以後補丁中的其他更改，那麼應當
+人搜索同一問題的解決方案。如果更改是為了支援以後補丁中的其他更改，那麼應當
 說明。如果更改了內部API，請詳細說明這些更改以及其他開發人員應該如何響應。
-一般來說，你越把自己放在每個閱讀你變更日誌的人的位置上，變更日誌（和內核
-作爲一個整體）就越好。
+一般來說，你越把自己放在每個閱讀你變更日誌的人的位置上，變更日誌（和核心
+作為一個整體）就越好。
 
-不需要說，變更日誌是將變更提交到版本控制系統時使用的文本。接下來將是：
+不需要說，變更日誌是將變更提交到版本控制系統時使用的文字。接下來將是：
 
- - 補丁本身，採用統一的（“-u”）補丁格式。使用“-p”選項來diff將使函數名與
+ - 補丁本身，採用統一的（“-u”）補丁格式。使用“-p”選項來diff將使函式名與
    更改相關聯，從而使結果補丁更容易被其他人讀取。
 
-上面提到的標籤（tag）用於描述各種開發人員如何與這個補丁的開發相關聯。
-:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <tw_submittingpatches>`
-文檔中對它們進行了詳細描述；下面是一個簡短的總結。每一行的格式如下：
+前面已簡要提到的標籤（tag）用於提供補丁如何產生的線索。
+:ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
+文件中對它們進行了詳細描述；下面是一個簡短的總結。
 
-::
+有一種標籤用於引用引入了本補丁所修復問題的較早提交::
+
+	Fixes: 1f2e3d4c5b6a ("The first line of the commit specified by the first 12 characters of its SHA-1 ID")
+
+另一種標籤用於連結帶有額外背景或細節的網頁，例如導致此補丁的較早討論，或
+一份由此補丁實作的規格文件::
+
+	Link: https://example.com/somewhere.html  optional-other-stuff
+
+依照企鵝老大（Chief Penguin）的指導，只有當Link:標籤指向的有用資訊無法在
+提交本身中找到時，才應該把它加到提交中。
+
+如果URL指向此補丁所修復的公開缺陷報告，請改用“Closes:”標籤::
+
+	Closes: https://example.com/issues/1234  optional-other-stuff
+
+一些缺陷追蹤系統能夠在帶有此類標籤的提交被套用時自動關閉問題。一些監控
+郵件列表的機器人也會追蹤這類標籤並採取相應動作。私有缺陷追蹤系統和無效
+的URL是被禁止的。
+
+另一種標籤用於記錄誰參與了補丁的開發。它們每個都使用如下格式::
 
 	tag: Full Name <email address>  optional-other-stuff
 
 常用的標籤有：
 
- - Signed-off-by: 這是一個開發人員的證明，證明他或她有權提交補丁以包含到內核
-   中。這表明同意開發者來源認證協議，其全文見
-   :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <tw_submittingpatches>`
+ - Signed-off-by: 這是一個開發人員的證明，證明他或她有權提交補丁以包含到核
+   心中。這表明同意開發者來源認證協議，其全文見
+   :ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
    如果沒有合適的簽字，則不能合併到主線中。
 
- - Co-developed-by: 聲明補丁是由多個開發人員共同創建的；當幾個人在一個補丁上
-   工作時，它用於給出共同作者（除了 From: 所給出的作者之外）。由於
-   Co-developed-by: 表示作者身份，所以每個共同開發人，必須緊跟在相關合作作者
-   的Signed-off-by之後。具體內容和示例見以下文件
-   :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <tw_submittingpatches>`
+ - Co-developed-by: 聲明補丁是由多個開發人員共同建立的；當幾個人在一個補丁
+   上工作時，它用於給出共同作者（除了 From: 所給出的作者之外）。由於
+   Co-developed-by: 表示作者身份，所以每個共同開發人，必須緊跟在相關合作作
+   者的Signed-off-by之後。具體內容和範例見以下文件
+   :ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
 
- - Acked-by: 表示另一個開發人員（通常是相關代碼的維護人員）同意補丁適合包含
-   在內核中。
+ - Acked-by: 表示另一個開發人員（通常是相關程式碼的維護人員）同意補丁適合包含
+   在核心中。
 
  - Tested-by: 聲明某人已經測試了補丁並確認它可以工作。
 
- - Reviewed-by: 表示某開發人員已經審查了補丁的正確性；有關詳細信息，請參閱
-   :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <tw_submittingpatches>`
+ - Reviewed-by: 表示某開發人員已經審查了補丁的正確性；有關詳細資訊，請參閱
+   :ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
+
+ - Reported-by: 指定報告此補丁修復的問題的使用者；此標籤用於向測試我們的
+   程式碼並在發現問題時告知我們的人們（他們常常沒有得到應有的重視）表示
+   感謝。注意，此標籤後面應跟隨指向該報告的Closes:標籤，除非該報告無法在
+   網路上取得。如果補丁只修復了所報告問題的一部分，可以用Link:標籤代替
+   Closes:。
 
- - Reported-by: 指定報告此補丁修復的問題的用戶；此標記用於表示感謝。
+ - Suggested-by: 標籤表明補丁的想法是由被指名者建議的，確保其想法獲得
+   讚譽。希望這能激勵他們在未來繼續幫助我們。
 
  - Cc：指定某人收到了補丁的副本，並有機會對此發表評論。
 
-在補丁中添加標籤時要小心：只有Cc:才適合在沒有指定人員明確許可的情況下添加。
+在補丁中添加上述標籤時要小心：除了Cc:、Reported-by:和Suggested-by:之外，
+所有標籤都需要被指名者的明確許可。對於這三個標籤，如果根據lore存檔或提交
+歷史，該人曾以該名字和電子郵件地址對Linux核心做出過貢獻，那麼隱含的許可
+就足夠了——並且對於Reported-by:和Suggested-by:，報告或建議必須是公開作出
+的。注意，就此而言bugzilla.kernel.org是公開場所，但其中使用的電子郵件地址
+是私密的；因此不要在標籤中暴露它們，除非該人在先前的貢獻中使用過。
 
 寄送補丁
 --------
 
 在寄送補丁之前，您還需要注意以下幾點：
 
- - 您確定您的郵件發送程序不會損壞補丁嗎？被郵件客戶端更改空白或修飾了行的補丁
+ - 您確定您的郵件發送程式不會損壞補丁嗎？被郵件客戶端更改空白或修飾了行的補丁
    無法被另一端接受，並且通常不會進行任何詳細檢查。如果有任何疑問，先把補丁寄
    給你自己，讓你自己確定它是完好無損的。
 
-   :ref:`Documentation/translations/zh_CN/process/email-clients.rst <tw_email_clients>`
+   :ref:`Documentation/translations/zh_TW/process/email-clients.rst <tw_email_clients>`
    提供了一些有用的提示，可以讓特定的郵件客戶端正常發送補丁。
 
- - 你確定你的補丁沒有荒唐的錯誤嗎？您應該始終通過scripts/checkpatch.pl檢查
-   補丁程序，並解決它提出的問題。請記住，checkpatch.pl，雖然體現了對內核補丁
+ - 你確定你的補丁沒有荒唐的錯誤嗎？您應該始終透過scripts/checkpatch.pl檢查
+   補丁程式，並解決它提出的問題。請記住，checkpatch.pl，雖然體現了對核心補丁
    應該是什麼樣的大量思考，但它並不比您聰明。如果修復checkpatch.pl給的問題會
-   使代碼變得更糟，請不要這樣做。
+   使程式碼變得更糟，請不要這樣做。
 
-補丁應始終以純文本形式發送。請不要將它們作爲附件發送；這使得審閱者在答覆中更難
-引用補丁的部分。相反，只需將補丁直接放到您的消息中。
+補丁應始終以純文字形式發送。請不要將它們作為附件發送；這使得審閱者在答覆中更難
+引用補丁的部分。相反，只需將補丁直接放到您的訊息中。
 
-寄出補丁時，重要的是將副本發送給任何可能感興趣的人。與其他一些項目不同，內核
-鼓勵人們甚至錯誤地發送過多的副本；不要假定相關人員會看到您在郵件列表中的發佈。
+寄出補丁時，重要的是將副本發送給任何可能感興趣的人。與其他一些專案不同，核心
+鼓勵人們甚至錯誤地發送過多的副本；不要假定相關人員會看到您在郵件列表中的發布。
 尤其是，副本應發送至：
 
- - 受影響子系統的維護人員。如前所述，維護人員文件是查找這些人員的首選地方。
+ - 受影響子系統的維護人員。如前所述，MAINTAINERS檔案是查找這些人員的首選地方。
 
- - 其他在同一領域工作的開發人員，尤其是那些現在可能在那裏工作的開發人員。使用
-   git查看還有誰修改了您正在處理的文件，這很有幫助。
+ - 其他在同一領域工作的開發人員，尤其是那些現在可能在那裡工作的開發人員。使用
+   git查看還有誰修改了您正在處理的檔案，這很有幫助。
 
  - 如果您對某錯誤報告或功能請求做出響應，也可以抄送原始發送人。
 
@@ -221,9 +258,9 @@
    補丁副本也應發到stable@vger.kernel.org 。另外，在補丁本身的標籤中添加一個
    “Cc: stable@vger.kernel.org”；這將使穩定版團隊在修復進入主線時收到通知。
 
-當爲一個補丁選擇接收者時，最好清楚你認爲誰最終會接受這個補丁並將其合併。雖然
+當為一個補丁選擇接收者時，最好清楚你認為誰最終會接受這個補丁並將其合併。雖然
 可以將補丁直接發給Linus Torvalds並讓他合併，但通常情況下不會這樣做。Linus很
-忙，並且有子系統維護人員負責監視內核的特定部分。通常您會希望維護人員合併您的
+忙，並且有子系統維護人員負責監視核心的特定部分。通常您會希望維護人員合併您的
 補丁。如果沒有明顯的維護人員，Andrew Morton通常是最後的補丁接收者。
 
 補丁需要好的主題行。補丁主題行的規範格式如下：
@@ -235,12 +272,12 @@
 其中“nn”是補丁的序號，“mm”是系列中補丁的總數，“subsys”是受影響子系統的
 名稱。當然，一個單獨的補丁可以省略nn/mm。
 
-如果您有一系列重要的補丁，那麼通常發送一個簡介作爲第〇部分。不過，這個約定
-並沒有得到普遍遵循；如果您使用它，請記住簡介中的信息不會進入內核變更日誌。
-因此，請確保補丁本身具有完整的變更日誌信息。
+如果您有一系列重要的補丁，那麼通常發送一個簡介作為第〇部分。不過，這個約定
+並沒有得到普遍遵循；如果您使用它，請記住簡介中的資訊不會進入核心變更日誌。
+因此，請確保補丁本身具有完整的變更日誌資訊。
 
-一般來說，多部分補丁的第二部分和後續部分應作爲對第一部分的回覆發送，以便它們
-在接收端都連接在一起。像git和coilt這樣的工具有命令，可以通過適當的線程發送
+一般來說，多部分補丁的第二部分和後續部分應作為對第一部分的回覆發送，以便它們
+在接收端都連接在一起。像git和coilt這樣的工具有命令，可以透過適當的執行緒發送
 一組補丁。但是，如果您有一長串補丁，並正使用git，請不要使用–-chain-reply-to
-選項，以避免創建過深的嵌套。
+選項，以避免建立過深的嵌套。
 
diff --git a/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst b/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst
index b449d67e3ad92d..8090d482c4b0a9 100644
--- a/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst
+++ b/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst
@@ -12,28 +12,29 @@
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
  胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+ 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_development_advancedtopics:
 
-高級主題
+進階主題
 ========
 
 現在，希望您能夠掌握開發流程的工作方式。然而，還有更多的東西要學！本節將介紹
-一些主題，這些主題對希望成爲Linux內核開發過程常規部分的開發人員有幫助。
+一些主題，這些主題對希望成為Linux核心開發過程常規部分的開發人員有幫助。
 
 使用Git管理補丁
 ---------------
 
-內核使用分佈式版本控制始於2002年初，當時Linus首次開始使用專有的Bitkeeper應用
-程序。雖然BitKeeper存在爭議，但它所體現的軟件版本管理方法卻肯定不是。分佈式
-版本控制可以立即加速內核開發項目。現在有好幾種免費的BitKeeper替代品。
-但無論好壞，內核項目都已經選擇了Git作爲其工具。
+核心使用分散式版本控制始於2002年初，當時Linus首次開始使用專有的Bitkeeper應用
+程式。雖然BitKeeper存在爭議，但它所體現的軟體版本管理方法卻肯定不是。分散式
+版本控制可以立即加速核心開發專案。現在有好幾種自由的BitKeeper替代品。
+但無論好壞，核心專案都已經選擇了Git作為其工具。
 
-使用Git管理補丁可以使開發人員的生活更加輕鬆，尤其是隨着補丁數量的增長。Git也
+使用Git管理補丁可以使開發人員的生活更加輕鬆，尤其是隨著補丁數量的增長。Git也
 有其粗糙的邊角和一定的危險性，它是一個年輕和強大的工具，仍然在其開發人員完善
-中。本文檔不會試圖教會讀者如何使用git；這會是個巨長的文檔。相反，這裏的重點
-將是Git如何特別適合內核開發過程。想要加快用Git速度的開發人員可以在以下網站上
-找到更多信息：
+中。本文件不會試圖教會讀者如何使用git；這會是個巨長的文件。相反，這裡的重點
+將是Git如何特別適合核心開發過程。想要加快用Git速度的開發人員可以在以下網站上
+找到更多資訊：
 
 	https://git-scm.com/
 
@@ -41,44 +42,44 @@
 
 同時網上也能找到各種各樣的教程。
 
-在嘗試使用它生成補丁供他人使用之前，第一要務是閱讀上述網頁，對Git的工作方式
-有一個紮實的瞭解。使用Git的開發人員應能進行拉取主線存儲庫的副本，查詢修訂
+在嘗試使用它產生補丁供他人使用之前，第一要務是閱讀上述網頁，對Git的工作方式
+有一個紮實的瞭解。使用Git的開發人員應能進行拉取主線儲存庫的副本，查詢修訂
 歷史，提交對樹的更改，使用分支等操作。瞭解Git用於重寫歷史的工具（如rebase）
-也很有用。Git有自己的術語和概念；Git的新用戶應該瞭解引用、遠程分支、索引、
-快進合併、推拉、遊離頭等。一開始可能有點嚇人，但這些概念不難通過一點學習來
+也很有用。Git有自己的術語和概念；Git的新使用者應該瞭解引用、遠端分支、索引、
+快進合併、推拉、遊離頭等。一開始可能有點嚇人，但這些概念不難透過一點學習來
 理解。
 
-使用git生成通過電子郵件提交的補丁是提高速度的一個很好的練習。
+使用git產生透過電子郵件提交的補丁是提高速度的一個很好的練習。
 
-當您準備好開始建立Git樹供其他人查看時，無疑需要一個可以從中拉取的服務器。
-如果您有一個可以訪問因特網的系統，那麼使用git-daemon設置這樣的服務器相對
-簡單。同時，免費的公共託管網站（例如github）也開始出現在網絡上。成熟的開發
-人員可以在kernel.org上獲得一個帳戶，但這些帳戶並不容易得到；更多有關信息，
+當您準備好開始建立Git樹供其他人查看時，無疑需要一個可以從中拉取的伺服器。
+如果您有一個可以連上網際網路的系統，那麼使用git-daemon設定這樣的伺服器相對
+簡單。同時，免費的公共託管網站（例如GitHub）也開始出現在網路上。成熟的開發
+人員可以在kernel.org上獲得一個帳戶，但這些帳戶並不容易得到；更多有關資訊，
 請參閱 https://kernel.org/faq/ 。
 
-正常的Git工作流程涉及到許多分支的使用。每一條開發線都可以分爲單獨的“主題
+正常的Git工作流程涉及到許多分支的使用。每一條開發線都可以分為單獨的“主題
 分支”，並獨立維護。Git的分支很容易使用，沒有理由不使用它們。而且，在任何
 情況下，您都不應該在任何您打算讓其他人從中拉取的分支中進行開發。應該小心地
-創建公開可用的分支；當開發分支處於完整狀態並已準備好時(而不是之前）才合併
+建立公開可用的分支；當開發分支處於完整狀態並已準備好時(而不是之前）才合併
 開發分支的補丁。
 
 Git提供了一些強大的工具，可以讓您重寫開發歷史。一個不方便的補丁（比如說，
 一個打破二分法的補丁，或者有其他一些明顯的缺陷）可以在適當的位置修復，或者
 完全從歷史中消失。一個補丁系列可以被重寫，就好像它是在今天的主線上寫的一樣，
 即使你已經花了幾個月的時間在寫它。可以透明地將更改從一個分支轉移到另一個
-分支。等等。明智地使用git修改歷史的能力可以幫助創建問題更少的乾淨補丁集。
+分支。等等。明智地使用git修改歷史的能力可以幫助建立問題更少的乾淨補丁集。
 
-然而，過度使用這種功能可能會導致其他問題，而不僅僅是對創建完美項目歷史的
-簡單癡迷。重寫歷史將重寫該歷史中包含的更改，將經過測試（希望如此）的內核樹
-變爲未經測試的內核樹。除此之外，如果開發人員沒有共享項目歷史，他們就無法
-輕鬆地協作；如果您重寫了其他開發人員拉入他們存儲庫的歷史，您將使這些開發
-人員的生活更加困難。因此，這裏有一個簡單的經驗法則：被導出到其他地方的歷史
-在此後通常被認爲是不可變的。
+然而，過度使用這種功能可能會導致其他問題，而不僅僅是對建立完美專案歷史的
+簡單癡迷。重寫歷史將重寫該歷史中包含的更改，將經過測試（希望如此）的核心樹
+變為未經測試的核心樹。除此之外，如果開發人員沒有共享專案歷史，他們就無法
+輕鬆地協作；如果您重寫了其他開發人員拉入他們儲存庫的歷史，您將使這些開發
+人員的生活更加困難。因此，這裡有一個簡單的經驗法則：被導出到其他地方的歷史
+在此後通常被認為是不可變的。
 
-因此，一旦將一組更改推送到公開可用的服務器上，就不應該重寫這些更改。如果您
+因此，一旦將一組更改推送到公開可用的伺服器上，就不應該重寫這些更改。如果您
 嘗試強制進行無法快進合併的更改（即不共享同一歷史記錄的更改），Git將嘗試強制
 執行此規則。這可能覆蓋檢查，有時甚至需要重寫導出的樹。在樹之間移動變更集以
-避免linux-next中的衝突就是一個例子。但這種行爲應該是罕見的。這就是爲什麼
+避免linux-next中的衝突就是一個例子。但這種行為應該是罕見的。這就是為什麼
 開發應該在私有分支中進行（必要時可以重寫）並且只有在公共分支處於合理的較新
 狀態時才轉移到公共分支中的原因之一。
 
@@ -86,52 +87,53 @@ Git提供了一些強大的工具，可以讓您重寫開發歷史。一個不
 對於一個私有的分支，rebasing 可能是一個很容易跟上另一棵樹的方法，但是一旦
 一棵樹被導出到外界，rebasing就不可取了。一旦發生這種情況，就必須進行完全
 合併（merge）。合併有時是很有意義的，但是過於頻繁的合併會不必要地擾亂歷史。
-在這種情況下建議的做法是不要頻繁合併，通常只在特定的發佈點（如主線-rc發佈）
+在這種情況下建議的做法是不要頻繁合併，通常只在特定的發布點（如主線-rc發布）
 合併。如果您對特定的更改感到緊張，則可以始終在私有分支中執行測試合併。在
 這種情況下，git“rerere”工具很有用；它能記住合併衝突是如何解決的，這樣您
 就不必重複相同的工作。
 
-關於Git這樣的工具的一個最大的反覆抱怨是：補丁從一個存儲庫到另一個存儲庫的
+關於Git這樣的工具的一個最大的反覆抱怨是：補丁從一個儲存庫到另一個儲存庫的
 大量移動使得很容易陷入錯誤建議的變更中，這些變更避開審查雷達進入主線。當內
 核開發人員看到這種情況發生時，他們往往會感到不高興；在Git樹上放置未審閱或
 主題外的補丁可能會影響您將來讓樹被拉取的能力。引用Linus的話:
 
 ::
 
-   你可以給我發補丁，但當我從你那裏拉取一個Git補丁時，我需要知道你清楚
+   你可以給我發補丁，但當我從你那裡拉取一個Git補丁時，我需要知道你清楚
    自己在做什麼，我需要能夠相信事情而 *無需* 手動檢查每個單獨的更改。
 
 （http://lwn.net/Articles/224135/）。
 
-爲了避免這種情況，請確保給定分支中的所有補丁都與相關主題緊密相關；“驅動程序
-修復”分支不應更改核心內存管理代碼。而且，最重要的是，不要使用Git樹來繞過
-審查過程。不時的將樹的摘要發佈到相關的列表中，在合適時候請求linux-next中
+為了避免這種情況，請確保給定分支中的所有補丁都與相關主題緊密相關；“驅動程式
+修復”分支不應更改核心記憶體管理程式碼。而且，最重要的是，不要使用Git樹來繞過
+審查過程。不時的將樹的摘要發布到相關的列表中，在合適時候請求linux-next中
 包含該樹。
 
 如果其他人開始發送補丁以包含到您的樹中，不要忘記審閱它們。還要確保您維護正確
-的作者信息； git “am”工具在這方面做得最好，但是如果補丁通過第三方轉發給您，
+的作者資訊； git “am”工具在這方面做得最好，但是如果補丁透過第三方轉發給您，
 您可能需要在補丁中添加“From:”行。
 
-請求拉取時，請務必提供所有相關信息：樹的位置、要拉取的分支以及拉取將導致的
+請求拉取時，請務必提供所有相關資訊：樹的位置、要拉取的分支以及拉取將導致的
 更改。在這方面 git request-pull 命令非常有用；它將按照其他開發人員所期望的
-格式化請求，並檢查以確保您已記得將這些更改推送到公共服務器。
+格式化請求，並檢查以確保您已記得將這些更改推送到公共伺服器。
 
 審閱補丁
 --------
 
-一些讀者顯然會反對將本節與“高級主題”放在一起，因爲即使是剛開始的內核開發人員
-也應該審閱補丁。當然，沒有比查看其他人發佈的代碼更好的方法來學習如何在內核環境
-中編程了。此外，審閱者永遠供不應求；通過審閱代碼，您可以對整個流程做出重大貢獻。
-
-審查代碼可能是一副令人生畏的圖景，特別是對一個新的內核開發人員來說，他們
-可能會對公開詢問代碼感到緊張，而這些代碼是由那些有更多經驗的人發佈的。不過，
-即使是最有經驗的開發人員編寫的代碼也可以得到改進。也許對（所有）審閱者最好
-的建議是：把審閱評論當成問題而不是批評。詢問“在這條路徑中如何釋放鎖？”
-總是比說“這裏的鎖是錯誤的”更好。
-
-不同的開發人員將從不同的角度審查代碼。部分人會主要關注代碼風格以及代碼行是
-否有尾隨空格。其他人會主要關注補丁作爲一個整體實現的變更是否對內核有好處。
-同時也有人會檢查是否存在鎖問題、堆棧使用過度、可能的安全問題、在其他地方
-發現的代碼重複、足夠的文檔、對性能的不利影響、用戶空間ABI更改等。所有類型
-的檢查，只要它們能引導更好的代碼進入內核，都是受歡迎和值得的。
+一些讀者顯然會反對將本節與“進階主題”放在一起，因為即使是剛開始的核心開發人
+員也應該審閱補丁。當然，沒有比查看其他人發布的程式碼更好的方法來學習如何在
+核心環境中撰寫程式了。此外，審閱者永遠供不應求；透過審閱程式碼，您可以對整
+個流程做出重大貢獻。
+
+審查程式碼可能是一副令人生畏的圖景，特別是對一個新的核心開發人員來說，他們
+可能會對公開詢問程式碼感到緊張，而這些程式碼是由那些有更多經驗的人發布的。
+不過，即使是最有經驗的開發人員編寫的程式碼也可以得到改進。也許對（所有）審
+閱者最好的建議是：把審閱評論當成問題而不是批評。詢問“在這條路徑中如何釋放
+鎖？”總是比說“這裡的鎖是錯誤的”更好。
+
+不同的開發人員將從不同的角度審查程式碼。部分人會主要關注程式碼風格以及程式
+碼行是否有尾隨空格。其他人會主要關注補丁作為一個整體實作的變更是否對核心有
+好處。同時也有人會檢查是否存在鎖問題、堆疊使用過度、可能的安全問題、在其他
+地方發現的程式碼重複、足夠的文件、對效能的不利影響、使用者空間ABI更改等。
+所有類型的檢查，只要它們能引導更好的程式碼進入核心，都是受歡迎和值得的。
 
diff --git a/Documentation/translations/zh_TW/process/8.Conclusion.rst b/Documentation/translations/zh_TW/process/8.Conclusion.rst
index d1634421b62ccd..8f9b4ec6845f64 100644
--- a/Documentation/translations/zh_TW/process/8.Conclusion.rst
+++ b/Documentation/translations/zh_TW/process/8.Conclusion.rst
@@ -11,45 +11,46 @@
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
  胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+ 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_development_conclusion:
 
-更多信息
+更多資訊
 ========
 
-關於Linux內核開發和相關主題的信息來源很多。首先是在內核源代碼分發中找到的
-文檔目錄。頂級
-:ref:`Documentation/translations/zh_CN/process/howto.rst <tw_process_howto>`
+關於Linux核心開發和相關主題的資訊來源很多。首先是在核心原始碼分發中找到的
+文件目錄。頂級
+:ref:`Documentation/translations/zh_TW/process/howto.rst <tw_process_howto>`
 文件是一個重要的起點；
-:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <tw_submittingpatches>`
-也是所有內核開發人員都應該閱讀的內容。許多內部內核API都是使用kerneldoc機制
-記錄的；“make htmldocs”或“make pdfdocs”可用於以HTML或PDF格式生成這些文檔
-（儘管某些發行版提供的tex版本會遇到內部限制，無法正確處理文檔）。
+:ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
+也是所有核心開發人員都應該閱讀的內容。許多內部核心API都是使用kerneldoc機制
+記錄的；“make htmldocs”或“make pdfdocs”可用於以HTML或PDF格式產生這些文件
+（儘管某些發行版提供的tex版本會遇到內部限制，無法正確處理文件）。
 
-不同的網站在各個細節層次上討論內核開發。本文作者想謙虛地建議用 https://lwn.net/
-作爲來源；有關許多特定內核主題的信息可以通過以下網址的 LWN 內核索引找到：
+不同的網站在各個細節層次上討論核心開發。本文作者想謙虛地建議用 https://lwn.net/
+作為來源；有關許多特定核心主題的資訊可以透過以下網址的 LWN 核心索引找到：
 
-  http://lwn.net/kernel/index/
+  https://lwn.net/Kernel/Index/
 
-除此之外，內核開發人員的一個寶貴資源是：
+除此之外，核心開發人員的一個寶貴資源是：
 
   https://kernelnewbies.org/
 
-當然，也不應該忘記 https://kernel.org/ ，這是內核發佈信息的最終位置。
+當然，也不應該忘記 https://kernel.org/ ，這是核心發布資訊的最終位置。
 
-關於內核開發有很多書：
+關於核心開發有很多書：
 
-  《Linux設備驅動程序》第三版（Jonathan Corbet、Alessandro Rubini和Greg Kroah Hartman）
-  線上版本在 http://lwn.net/kernel/ldd3/
+  《Linux設備驅動程序》第三版（Jonathan Corbet、Alessandro Rubini和Greg Kroah-Hartman）
+  線上版本在 https://lwn.net/Kernel/LDD3/
 
-  《Linux內核設計與實現》（Robert Love）
+  《Linux核心設計與實現》（Robert Love）
 
-  《深入理解Linux內核》(Daniel Bovet和Marco Cesati）
+  《深入理解Linux核心》(Daniel Bovet和Marco Cesati）
 
 然而，所有這些書都有一個共同的缺點：它們上架時就往往有些過時，而且已經上架
-一段時間了。不過，在那裏還是可以找到相當多的好信息。
+一段時間了。不過，在那裡還是可以找到相當多的好資訊。
 
-有關git的文檔，請訪問：
+有關git的文件，請造訪：
 
   https://www.kernel.org/pub/software/scm/git/docs/
 
@@ -58,16 +59,17 @@
 結論
 ====
 
-祝賀所有通過這篇冗長的文檔的人。希望它能夠幫助您理解Linux內核是如何開發的，
+祝賀所有通過這篇冗長的文件的人。希望它能夠幫助您理解Linux核心是如何開發的，
 以及您如何參與這個過程。
 
-最後，重要的是參與。任何開源軟件項目都不會超過其貢獻者投入其中的總和。Linux
-內核的發展速度和以前一樣快，因爲它得到了大量開發人員的幫助，他們都在努力使它
-變得更好。內核是一個最成功的例子，說明了當成千上萬的人爲了一個共同的目標一起
+最後，重要的是參與。任何開源軟體專案都不會超過其貢獻者投入其中的總和。Linux
+核心的發展速度和以前一樣快，因為它得到了大量開發人員的幫助，他們都在努力使它
+變得更好。核心是一個最成功的例子，說明了當成千上萬的人為了一個共同的目標一起
 工作時，可以做出什麼。
 
-不過，內核總是可以從更大的開發人員基礎中獲益。總有更多的工作要做。但是同樣
-重要的是，Linux生態系統中的大多數其他參與者可以通過爲內核做出貢獻而受益。使
-代碼進入主線是提高代碼質量、降低維護和分發成本、提高對內核開發方向的影響程度
-等的關鍵。這是一種共贏的局面。啓動你的編輯器，來加入我們吧；你會非常受歡迎的。
+不過，核心總是可以從更大的開發人員基礎中獲益。總有更多的工作要做。但是同樣
+重要的是，Linux生態系統中的大多數其他參與者可以透過為核心做出貢獻而受益。
+使程式碼進入主線是提高程式碼品質、降低維護和分發成本、提高對核心開發方向的
+影響程度等的關鍵。這是一種共贏的局面。啟動你的編輯器，來加入我們吧；你會非
+常受歡迎的。
 
diff --git a/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst b/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst
index fbe66b00132227..897faa825d1221 100644
--- a/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst
+++ b/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst
@@ -5,108 +5,172 @@
 :Original: :ref:`Documentation/process/code-of-conduct-interpretation.rst <code_of_conduct_interpretation>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
              Hu Haowen <2023002089@link.tyut.edu.cn>
+             Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_code_of_conduct_interpretation:
 
-Linux內核貢獻者契約行爲準則解釋
+Linux核心貢獻者契約行為準則解釋
 ===============================
 
-:ref:`tw_code_of_conduct` 準則是一個通用文檔，旨在爲幾乎所有開源社區提供一套規則。
-每個開源社區都是獨一無二的，Linux內核也不例外。因此，本文描述了Linux內核社區中
-如何解釋它。我們也不希望這種解釋隨着時間的推移是靜態的，並將根據需要進行調整。
+:ref:`tw_code_of_conduct` 準則是一個通用文件，旨在為幾乎所有開源社群提供一套規則。
+每個開源社群都是獨一無二的，Linux核心也不例外。因此，本文描述了Linux核心社群中
+如何解釋它。我們也不希望這種解釋隨著時間的推移是靜態的，並將根據需要進行調整。
 
-與開發軟件的“傳統”方法相比，Linux內核開發工作是一個非常個人化的過程。你的貢獻
+與開發軟體的“傳統”方法相比，Linux核心開發工作是一個非常個人化的過程。你的貢獻
 和背後的想法將被仔細審查，往往導致批判和批評。審查將幾乎總是需要改進，材料才
-能包括在內核中。要知道這是因爲所有相關人員都希望看到Linux整體成功的最佳解決方
-案。這個開發過程已經被證明可以創建有史以來最健壯的操作系統內核，我們不想做任何
-事情來導致提交質量和最終結果的下降。
+能包括在核心中。要知道這是因為所有相關人員都希望看到Linux整體成功的最佳解決方
+案。這個開發過程已經被證明可以建立有史以來最強健的作業系統核心，我們不想做任何
+事情來導致提交品質和最終結果的下降。
 
 維護者
 ------
 
-行爲準則多次使用“維護者”一詞。在內核社區中，“維護者”是負責子系統、驅動程序或
-文件的任何人，並在內核源代碼樹的維護者文件中列出。
+行為準則多次使用“維護者”一詞。在核心社群中，“維護者”是負責子系統、驅動程式或
+文件的任何人，並在核心原始程式碼樹的維護者文件中列出。
 
 責任
 ----
 
-《行爲準則》提到了維護人員的權利和責任，這需要進一步澄清。
+《行為準則》提到了維護人員的權利和責任，這需要進一步澄清。
 
-首先，最重要的是，有一個合理的期望是由維護人員通過實例來領導。
+首先，最重要的是，有一個合理的期望是由維護人員透過實例來領導。
 
-也就是說，我們的社區是廣闊的，對維護者沒有新的要求，他們單方面處理其他人在
-他們活躍的社區的行爲。這一責任由我們所有人承擔，最終《行爲準則》記錄了最終的
-上訴路徑，以防有關行爲問題的問題懸而未決。
+也就是說，我們的社群是廣闊的，對維護者沒有新的要求，他們單方面處理其他人在
+他們活躍的社群的行為。這一責任由我們所有人承擔，最終《行為準則》記錄了最終的
+上訴路徑，以防有關行為問題的問題懸而未決。
 
-維護人員應該願意在出現問題時提供幫助，並在需要時與社區中的其他人合作。如果您
+維護人員應該願意在出現問題時提供幫助，並在需要時與社群中的其他人合作。如果您
 不確定如何處理出現的情況，請不要害怕聯繫技術諮詢委員會（TAB）或其他維護人員。
-除非您願意，否則不會將其視爲違規報告。如果您不確定是否該聯繫TAB 或任何其他維
+除非您願意，否則不會將其視為違規報告。如果您不確定是否該聯繫TAB 或任何其他維
 護人員，請聯繫我們的衝突調解人 Mishi Choudhary <mishi@linux.com>。
 
-最後，“善待對方”纔是每個人的最終目標。我們知道每個人都是人，有時我們都會失敗，
-但我們所有人的首要目標應該是努力友好地解決問題。執行行爲準則將是最後的選擇。
+最後，“善待對方”才是每個人的最終目標。我們知道每個人都是人，有時我們都會失敗，
+但我們所有人的首要目標應該是努力友好地解決問題。執行行為準則將是最後的選擇。
 
-我們的目標是創建一個強大的、技術先進的操作系統，以及所涉及的技術複雜性，這自
+我們的目標是建立一個強大的、技術先進的作業系統，以及所涉及的技術複雜性，這自
 然需要專業知識和決策。
 
 所需的專業知識因貢獻領域而異。它主要由上下文和技術複雜性決定，其次由貢獻者和
 維護者的期望決定。
 
-專家的期望和決策都要經過討論，但在最後，爲了取得進展，必須能夠做出決策。這一
-特權掌握在維護人員和項目領導的手中，預計將善意使用。
+專家的期望和決策都要經過討論，但在最後，為了取得進展，必須能夠做出決策。這一
+特權掌握在維護人員和專案領導的手中，預計將善意使用。
 
-因此，設定專業知識期望、作出決定和拒絕不適當的貢獻不被視爲違反行爲準則。
+因此，設定專業知識期望、作出決定和拒絕不適當的貢獻不被視為違反行為準則。
 
 雖然維護人員一般都歡迎新來者，但他們幫助（新）貢獻者克服障礙的能力有限，因此
-他們必須確定優先事項。這也不應被視爲違反了行爲準則。內核社區意識到這一點，並
+他們必須確定優先事項。這也不應被視為違反了行為準則。核心社群意識到這一點，並
 以各種形式提供入門級節目，如 kernelnewbies.org 。
 
 範圍
 ----
 
-Linux內核社區主要在一組公共電子郵件列表上進行交互，這些列表分佈在由多個不同
-公司或個人控制的多個不同服務器上。所有這些列表都在內核源代碼樹中的
-MAINTAINERS 文件中定義。發送到這些郵件列表的任何電子郵件都被視爲包含在行爲
+Linux核心社群主要在一組公共電子郵件列表上進行交互，這些列表分佈在由多個不同
+公司或個人控制的多個不同伺服器上。所有這些列表都在核心原始程式碼樹中的
+MAINTAINERS 文件中定義。發送到這些郵件列表的任何電子郵件都被視為包含在行為
 準則中。
 
-使用 kernel.org bugzilla和其他子系統bugzilla 或bug跟蹤工具的開發人員應該遵循
-行爲準則的指導原則。Linux內核社區沒有“官方”項目電子郵件地址或“官方”社交媒體
-地址。使用kernel.org電子郵件帳戶執行的任何活動必須遵循爲kernel.org發佈的行爲
+使用 kernel.org bugzilla和其他子系統bugzilla 或bug追蹤工具的開發人員應該遵循
+行為準則的指導原則。Linux核心社群沒有“官方”專案電子郵件地址或“官方”社交媒體
+地址。使用kernel.org電子郵件帳戶執行的任何活動必須遵循為kernel.org發布的行為
 準則，就像任何使用公司電子郵件帳戶的個人必須遵循該公司的特定規則一樣。
 
-行爲準則並不禁止在郵件列表消息、內核更改日誌消息或代碼註釋中繼續包含名稱、
-電子郵件地址和相關注釋。
+行為準則並不禁止在郵件列表消息、核心更改日誌消息或程式碼註解中繼續包含名稱、
+電子郵件地址和相關註解。
 
-其他論壇中的互動包括在適用於上述論壇的任何規則中，通常不包括在行爲準則中。
+其他論壇中的互動包括在適用於上述論壇的任何規則中，通常不包括在行為準則中。
 除了在極端情況下可考慮的例外情況。
 
-提交給內核的貢獻應該使用適當的語言。在行爲準則之前已經存在的內容現在不會被
-視爲違反。然而，不適當的語言可以被視爲一個bug；如果任何相關方提交補丁，
-這樣的bug將被更快地修復。當前屬於用戶/內核API的一部分的表達式，或者反映已
-發佈標準或規範中使用的術語的表達式，不被視爲bug。
+提交給核心的貢獻應該使用適當的語言。在行為準則之前已經存在的內容現在不會被
+視為違反。然而，不適當的語言可以被視為一個bug；如果任何相關方提交補丁，
+這樣的bug將被更快地修復。當前屬於使用者/核心API的一部分的表達式，或者反映已
+發布標準或規範中使用的術語的表達式，不被視為bug。
 
 執行
 ----
 
-行爲準則中列出的地址屬於行爲準則委員會。https://kernel.org/code-of-conduct.html
-列出了在任何給定時間接收這些電子郵件的確切成員。成員不能訪問在加入委員會之前
+行為準則中列出的地址屬於行為準則委員會。https://kernel.org/code-of-conduct.html
+列出了在任何給定時間接收這些電子郵件的確切成員。成員不能存取在加入委員會之前
 或離開委員會之後所做的報告。
 
-最初的行爲準則委員會由TAB的志願者以及作爲中立第三方的專業調解人組成。委員會
-的首要任務是建立文件化的流程，並將其公開。
+行為準則委員會由TAB任命的社群志願者成員以及作為中立第三方的專業調解人組成。
+行為準則委員會處理報告所使用的流程各不相同，將取決於個別情況，但本文件可
+作為其所使用之一般流程的說明。
 
 如果報告人不希望將整個委員會納入投訴或關切，可直接聯繫委員會的任何成員，包括
 調解人。
 
-行爲準則委員會根據流程審查案例（見上文），並根據需要和適當與TAB協商，例如請求
-和接收有關內核社區的信息。
+行為準則委員會根據流程審查案例（見上文），並根據需要和適當與TAB協商，例如請求
+和接收有關核心社群的資訊。
 
-委員會做出的任何決定都將提交到表中，以便在必要時與相關維護人員一起執行。行爲
-準則委員會的決定可以通過三分之二的投票推翻。
+任何有關執法建議的決定都將提交給TAB，以便在必要時與相關維護人員一起實施
+執法。一旦TAB以參與表決成員的三分之二多數批准了禁令範圍中列出的一項或多項
+措施，行為準則委員會將執行TAB批准的措施。任何同時在TAB任職的行為準則委員會
+成員不參與對這些措施的表決。
 
-每季度，行爲準則委員會和標籤將提供一份報告，概述行爲準則委員會收到的匿名報告
-及其狀態，以及任何否決決定的細節，包括完整和可識別的投票細節。
+每季度，行為準則委員會和TAB將提供一份報告，概述行為準則委員會收到的匿名
+報告及其狀態，以及任何TAB批准之決定的細節，包括完整和可識別的投票細節。
 
-我們希望在啓動期之後爲行爲準則委員會人員配備建立一個不同的流程。發生此情況時，
-將使用該信息更新此文檔。
+由於我們對行為準則的解釋和執行會隨著時間演變，本文件將在必要時更新，以反映
+任何變化。
+
+不可接受行為之行為準則違規的執法
+--------------------------------
+
+行為準則委員會致力於確保我們的社群持續具有包容性，促進多元的討論和觀點，並
+致力於隨著時間改進這些特質。行為準則委員會收到的大多數報告，源於對開發過程
+以及維護者的角色、責任及其對程式碼接受與否的決定權的錯誤理解。這些報告透過
+澄清開發過程和行為準則的範圍來解決。
+
+不可接受的行為可能在短時間內中斷相互尊重的協作，並對社群的健康造成長期的
+負面影響。當個人在違規發生的場合承認自己的行為並加以糾正時，不可接受的行為
+通常就能得到解決。
+
+當不可接受的行為未能透過社群討論解決時，行為準則委員會會收到相關報告。當
+不可接受的行為對相互尊重的協作關係造成負面影響時，行為準則委員會會採取措施
+恢復有成效且相互尊重的協作。
+
+行為準則委員會有義務對報告和報告者的資訊保密。報告可能來自受害方，也可能
+來自目睹不可接受行為的社群成員。行為準則委員會有責任與所有相關方合作，對這
+類報告進行調查並予以處理。
+
+行為準則委員會與當事人合作，促使其理解以下事項的重要性：修復其行為對受害方
+造成的傷害，以及對社群的長期負面影響。
+
+目標是達成所有各方都能接受的解決方案。如果與當事人的合作未能達到預期的結果，
+行為準則委員會將評估其他措施，例如要求公開道歉以修復傷害。
+
+要求為違規行為公開道歉
+~~~~~~~~~~~~~~~~~~~~~~
+
+行為準則委員會在違規發生的場合公開指出該行為，要求為違規行為公開道歉。
+
+為違規行為公開道歉是重建信任的第一步。信任對於一個以信任和尊重運作的社群的
+持續成功和健康至關重要。
+
+若未為違規行為公開道歉的補救措施
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+行為準則委員會透過向TAB建議補救措施以供批准，決定恢復健康協作的下一步行動。
+
+- 禁止違規者參與核心開發過程，期限最長可達一個完整的核心開發週期。行為準則
+  委員會可以要求公開道歉作為解除禁令的條件。
+
+一段時間內禁令的範圍可能包括：
+
+    a. 拒絕其補丁貢獻和拉取請求
+    b. 透過忽略其貢獻和/或封鎖其電子郵件帳戶，暫停與違規者的協作
+    c. 限制其透過kernel.org平臺（如郵件列表和社交媒體網站）進行交流的能力
+
+一旦TAB以參與表決成員的三分之二多數批准了禁令範圍中列出的一項或多項措施，
+行為準則委員會將與社群、維護人員、子維護人員和kernel.org管理員合作，執行
+TAB批准的措施。任何同時在TAB任職的行為準則委員會成員不參與對這些措施的表決。
+
+行為準則委員會深知要求公開道歉和實施禁令可能對個人造成的負面影響。它也深知
+在發生此類嚴重的公開違規行為時不採取行動可能對社群造成的長期傷害。
+
+TAB批准的補救措施的有效性，取決於社群、維護人員、子維護人員和kernel.org
+管理員在執行這些措施時的信任與合作。
+
+行為準則委員會衷心希望，需要要求公開道歉的不可接受行為，在未來仍然極為罕見。
 
diff --git a/Documentation/translations/zh_TW/process/coding-style.rst b/Documentation/translations/zh_TW/process/coding-style.rst
index 63c78982a1afcd..ef7a4cdbff375e 100644
--- a/Documentation/translations/zh_TW/process/coding-style.rst
+++ b/Documentation/translations/zh_TW/process/coding-style.rst
@@ -18,39 +18,40 @@
  - Li Zefan <lizf@cn.fujitsu.com>
  - Wang Chen <wangchen@cn.fujitsu.com>
  - Hu Haowen <2023002089@link.tyut.edu.cn>
+ - 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
-Linux 內核代碼風格
-==================
+Linux 核心程式碼風格
+====================
 
-這是一個簡短的文檔，描述了 linux 內核的首選代碼風格。代碼風格是因人而異的，
-而且我不願意把自己的觀點強加給任何人，但這就像我去做任何事情都必須遵循的原則
-那樣，我也希望在絕大多數事上保持這種的態度。請 (在寫代碼時) 至少考慮一下這裏
-的代碼風格。
+這是一個簡短的文件，描述了 linux 核心的首選程式碼風格。程式碼風格是因人而
+異的，而且我不願意把自己的觀點強加給任何人，但這就像我去做任何事情都必須遵
+循的原則那樣，我也希望在絕大多數事上保持這種的態度。請 (在寫程式碼時) 至少
+考慮一下這裡的程式碼風格。
 
-首先，我建議你打印一份 GNU 代碼規範，然後不要讀。燒了它，這是一個具有重大象徵
-性意義的動作。
+首先，我建議你列印一份 GNU 程式碼規範，然後不要讀。燒了它，這是一個具有重
+大象徵性意義的動作。
 
 不管怎樣，現在我們開始：
 
 
-1) 縮進
+1) 縮排
 -------
 
-製表符是 8 個字符，所以縮進也是 8 個字符。有些異端運動試圖將縮進變爲 4 (甚至
-2！) 字符深，這幾乎相當於嘗試將圓周率的值定義爲 3。
+製表符是 8 個字元，所以縮排也是 8 個字元。有些異端運動試圖將縮排變為 4 (甚至
+2！) 字元深，這幾乎相當於嘗試將圓周率的值定義為 3。
 
-理由：縮進的全部意義就在於清楚的定義一個控制塊起止於何處。尤其是當你盯着你的
-屏幕連續看了 20 小時之後，你將會發現大一點的縮進會使你更容易分辨縮進。
+理由：縮排的全部意義就在於清楚的定義一個控制塊起止於何處。尤其是當你盯著你的
+屏幕連續看了 20 小時之後，你將會發現大一點的縮排會使你更容易分辨縮排。
 
-現在，有些人會抱怨 8 個字符的縮進會使代碼向右邊移動的太遠，在 80 個字符的終端
-屏幕上就很難讀這樣的代碼。這個問題的答案是，如果你需要 3 級以上的縮進，不管用
-何種方式你的代碼已經有問題了，應該修正你的程序。
+現在，有些人會抱怨 8 個字元的縮排會使程式碼向右邊移動的太遠，在 80 個字元
+的終端屏幕上就很難讀這樣的程式碼。這個問題的答案是，如果你需要 3 級以上的
+縮排，不管用何種方式你的程式碼已經有問題了，應該修正你的程式。
 
-簡而言之，8 個字符的縮進可以讓代碼更容易閱讀，還有一個好處是當你的函數嵌套太
+簡而言之，8 個字元的縮排可以讓程式碼更容易閱讀，還有一個好處是當你的函式嵌套太
 深的時候可以給你警告。留心這個警告。
 
-在 switch 語句中消除多級縮進的首選的方式是讓 ``switch`` 和從屬於它的 ``case``
-標籤對齊於同一列，而不要 ``兩次縮進`` ``case`` 標籤。比如：
+在 switch 語句中消除多級縮排的首選的方式是讓 ``switch`` 和從屬於它的 ``case``
+標籤對齊於同一列，而不要 ``兩次縮排`` ``case`` 標籤。比如：
 
 .. code-block:: c
 
@@ -71,7 +72,7 @@ Linux 內核代碼風格
 		break;
 	}
 
-不要把多個語句放在一行裏，除非你有什麼東西要隱藏：
+不要把多個語句放在一行裡，除非你有什麼東西要隱藏：
 
 .. code-block:: c
 
@@ -94,37 +95,38 @@ Linux 內核代碼風格
 		do_that();
 	}
 
-也不要在一行裏放多個賦值語句。內核代碼風格超級簡單。就是避免可能導致別人誤讀
+也不要在一行裡放多個賦值語句。核心程式碼風格超級簡單。就是避免可能導致別人誤讀
 的表達式。
 
-除了註釋、文檔和 Kconfig 之外，不要使用空格來縮進，前面的例子是例外，是有意爲
+除了註解、文件和 Kconfig 之外，不要使用空格來縮排，前面的例子是例外，是有意為
 之。
 
 選用一個好的編輯器，不要在行尾留空格。
 
 
-2) 把長的行和字符串打散
+2) 把長的行和字串打散
 -----------------------
 
-代碼風格的意義就在於使用平常使用的工具來維持代碼的可讀性和可維護性。
+程式碼風格的意義就在於使用平常使用的工具來維持程式碼的可讀性和可維護性。
 
 每一行的長度的限制是 80 列，我們強烈建議您遵守這個慣例。
 
 長於 80 列的語句要打散成有意義的片段。除非超過 80 列能顯著增加可讀性，並且不
-會隱藏信息。
+會隱藏資訊。
 
-子片段要明顯短於母片段，並明顯靠右。一種非常常用的樣式是將子體與函數左括號對齊。
+子片段要明顯短於母片段，並明顯靠右。一種非常常用的樣式是將子體與函式左括號
+對齊。
 
-這同樣適用於有着很長參數列表的函數頭。
+這同樣適用於有著很長參數列表的函式頭。
 
-然而，絕對不要打散對用戶可見的字符串，例如 printk 信息，因爲這樣就
+然而，絕對不要打散對使用者可見的字串，例如 printk 資訊，因為這樣就
 很難對它們 grep。
 
 
 3) 大括號和空格的放置
 ---------------------
 
-C 語言風格中另外一個常見問題是大括號的放置。和縮進大小不同，選擇或棄用某種放
+C 語言風格中另外一個常見問題是大括號的放置。和縮排大小不同，選擇或棄用某種放
 置策略並沒有多少技術上的原因，不過首選的方式，就像 Kernighan 和 Ritchie 展示
 給我們的，是把起始大括號放在行尾，而把結束大括號放在行首，所以：
 
@@ -134,7 +136,7 @@ C 語言風格中另外一個常見問題是大括號的放置。和縮進大小
 		we do y
 	}
 
-這適用於所有的非函數語句塊 (if, switch, for, while, do)。比如：
+這適用於所有的非函式語句塊 (if, switch, for, while, do)。比如：
 
 .. code-block:: c
 
@@ -149,7 +151,7 @@ C 語言風格中另外一個常見問題是大括號的放置。和縮進大小
 		return NULL;
 	}
 
-不過，有一個例外，那就是函數：函數的起始大括號放置於下一行的開頭，所以：
+不過，有一個例外，那就是函式：函式的起始大括號放置於下一行的開頭，所以：
 
 .. code-block:: c
 
@@ -159,10 +161,10 @@ C 語言風格中另外一個常見問題是大括號的放置。和縮進大小
 	}
 
 全世界的異端可能會抱怨這個不一致性是……呃……不一致，不過所有思維健全的人
-都知道 (a) K&R 是 **正確的** 並且 (b) K&R 是正確的。此外，不管怎樣函數都是特
-殊的 (C 函數是不能嵌套的)。
+都知道 (a) K&R 是 **正確的** 並且 (b) K&R 是正確的。此外，不管怎樣函式都是特
+殊的 (C 函式是不能嵌套的)。
 
-注意結束大括號獨自佔據一行，除非它後面跟着同一個語句的剩餘部分，也就是 do 語
+注意結束大括號獨自佔據一行，除非它後面跟著同一個語句的剩餘部分，也就是 do 語
 句中的 ``while`` 或者 if 語句中的 ``else`` ，像這樣：
 
 .. code-block:: c
@@ -187,7 +189,7 @@ C 語言風格中另外一個常見問題是大括號的放置。和縮進大小
 
 也請注意這種大括號的放置方式也能使空 (或者差不多空的) 行的數量最小化，同時不
 失可讀性。因此，由於你的屏幕上的新行是不可再生資源 (想想 25 行的終端屏幕)，你
-將會有更多的空行來放置註釋。
+將會有更多的空行來放置註解。
 
 當只有一個單獨的語句的時候，不用加不必要的大括號。
 
@@ -219,10 +221,10 @@ C 語言風格中另外一個常見問題是大括號的放置。和縮進大小
 3.1) 空格
 *********
 
-Linux 內核的空格使用方式 (主要) 取決於它是用於函數還是關鍵字。(大多數) 關鍵字
+Linux 核心的空格使用方式 (主要) 取決於它是用於函式還是關鍵字。(大多數) 關鍵字
 後要加一個空格。值得注意的例外是 sizeof, typeof, alignof 和 __attribute__，這
-些關鍵字某些程度上看起來更像函數 (它們在 Linux 裏也常常伴隨小括號而使用，儘管
-在 C 裏這樣的小括號不是必需的，就像 ``struct fileinfo info;`` 聲明過後的
+些關鍵字某些程度上看起來更像函式 (它們在 Linux 裡也常常伴隨小括號而使用，儘管
+在 C 裡這樣的小括號不是必需的，就像 ``struct fileinfo info;`` 宣告過後的
 ``sizeof info``)。
 
 所以在這些關鍵字之後放一個空格::
@@ -236,14 +238,14 @@ Linux 內核的空格使用方式 (主要) 取決於它是用於函數還是關
 
 	s = sizeof(struct file);
 
-不要在小括號裏的表達式兩側加空格。這是一個 **反例** ：
+不要在小括號裡的表達式兩側加空格。這是一個 **反例** ：
 
 .. code-block:: c
 
 	s = sizeof( struct file );
 
-當聲明指針類型或者返回指針類型的函數時， ``*`` 的首選使用方式是使之靠近變量名
-或者函數名，而不是靠近類型名。例子：
+當宣告指標類型或者返回指標類型的函式時， ``*`` 的首選使用方式是使之靠近變數名
+或者函式名，而不是靠近類型名。例子：
 
 .. code-block:: c
 
@@ -269,67 +271,67 @@ Linux 內核的空格使用方式 (主要) 取決於它是用於函數還是關
 
 ``.`` 和 ``->`` 結構體成員操作符前後不加空格。
 
-不要在行尾留空白。有些可以自動縮進的編輯器會在新行的行首加入適量的空白，然後
-你就可以直接在那一行輸入代碼。不過假如你最後沒有在那一行輸入代碼，有些編輯器
-就不會移除已經加入的空白，就像你故意留下一個只有空白的行。包含行尾空白的行就
-這樣產生了。
+不要在行尾留空白。有些可以自動縮排的編輯器會在新行的行首加入適量的空白，然
+後你就可以直接在那一行輸入程式碼。不過假如你最後沒有在那一行輸入程式碼，有
+些編輯器就不會移除已經加入的空白，就像你故意留下一個只有空白的行。包含行尾
+空白的行就這樣產生了。
 
 當 git 發現補丁包含了行尾空白的時候會警告你，並且可以應你的要求去掉行尾空白；
-不過如果你是正在打一系列補丁，這樣做會導致後面的補丁失敗，因爲你改變了補丁的
+不過如果你是正在打一系列補丁，這樣做會導致後面的補丁失敗，因為你改變了補丁的
 上下文。
 
 
 4) 命名
 -------
 
-C 是一個簡樸的語言，你的命名也應該這樣。和 Modula-2 和 Pascal 程序員不同，
-C 程序員不使用類似 ThisVariableIsATemporaryCounter 這樣華麗的名字。C 程序員會
-稱那個變量爲 ``tmp`` ，這樣寫起來會更容易，而且至少不會令其難於理解。
+C 是一個簡樸的語言，你的命名也應該這樣。和 Modula-2 和 Pascal 程式員不同，
+C 程式員不使用類似 ThisVariableIsATemporaryCounter 這樣華麗的名字。C 程式員會
+稱那個變數為 ``tmp`` ，這樣寫起來會更容易，而且至少不會令其難於理解。
 
-不過，雖然混用大小寫的名字是不提倡使用的，但是全局變量還是需要一個具描述性的
-名字。稱一個全局函數爲 ``foo`` 是一個難以饒恕的錯誤。
+不過，雖然混用大小寫的名字是不提倡使用的，但是全域變數還是需要一個具描述性的
+名字。稱一個全域函式為 ``foo`` 是一個難以饒恕的錯誤。
 
-全局變量 (只有當你 **真正** 需要它們的時候再用它) 需要有一個具描述性的名字，就
-像全局函數。如果你有一個可以計算活動用戶數量的函數，你應該叫它
+全域變數 (只有當你 **真正** 需要它們的時候再用它) 需要有一個具描述性的名字，就
+像全域函式。如果你有一個可以計算活動使用者數量的函式，你應該叫它
 ``count_active_users()`` 或者類似的名字，你不應該叫它 ``cntuser()`` 。
 
-在函數名中包含函數類型 (所謂的匈牙利命名法) 是腦子出了問題——編譯器知道那些類
-型而且能夠檢查那些類型，這樣做只能把程序員弄糊塗了。
+在函式名中包含函式類型 (所謂的匈牙利命名法) 是腦子出了問題——編譯器知道那些類
+型而且能夠檢查那些類型，這樣做只能把程式員弄糊塗了。
 
-本地變量名應該簡短，而且能夠表達相關的含義。如果你有一些隨機的整數型的循環計
-數器，它應該被稱爲 ``i`` 。叫它 ``loop_counter`` 並無益處，如果它沒有被誤解的
-可能的話。類似的， ``tmp`` 可以用來稱呼任意類型的臨時變量。
+本地變數名應該簡短，而且能夠表達相關的含義。如果你有一些隨機的整數型的迴圈計
+數器，它應該被稱為 ``i`` 。叫它 ``loop_counter`` 並無益處，如果它沒有被誤解的
+可能的話。類似的， ``tmp`` 可以用來稱呼任意類型的臨時變數。
 
-如果你怕混淆了你的本地變量名，你就遇到另一個問題了，叫做函數增長荷爾蒙失衡綜
-合徵。請看第六章 (函數)。
+如果你怕混淆了你的本地變數名，你就遇到另一個問題了，叫做函式增長荷爾蒙失衡綜
+合徵。請看第六章 (函式)。
 
-對於符號名稱和文檔，避免引入新的“master/slave”（或獨立於“master”的“slave”）
+對於符號名稱和文件，避免引入新的“master/slave”（或獨立於“master”的“slave”）
 和“blacklist/whitelist”。
 
-“master/slave”推薦替換爲：
+“master/slave”推薦替換為：
     '{primary,main} / {secondary,replica,subordinate}'
     '{initiator,requester} / {target,responder}'
     '{controller,host} / {device,worker,proxy}'
     'leader/follower'
     'director/performer'
 
-“blacklist/whitelist”推薦替換爲：
+“blacklist/whitelist”推薦替換為：
     'denylist/allowlist'
     'blocklist/passlist'
 
-引入新用法的例外情況是：維護用戶空間ABI/API，或更新現有（截至2020年）硬件或
-協議規範的代碼時要求這些術語。對於新規範，儘可能將術語的規範用法轉換爲內核
+引入新用法的例外情況是：維護使用者空間ABI/API，或更新現有（截至2020年）硬體或
+協議規範的程式碼時要求這些術語。對於新規範，儘可能將術語的規範用法轉換為核心
 編碼標準。
 
 .. warning::
-	以上主從、黑白名單規則不適用於中文文檔，請勿更改中文術語！
+	以上主從、黑白名單規則不適用於中文文件，請勿更改中文術語！
 
 5) Typedef
 ----------
 
 不要使用類似 ``vps_t`` 之類的東西。
 
-對結構體和指針使用 typedef 是一個 **錯誤** 。當你在代碼裏看到：
+對結構體和指標使用 typedef 是一個 **錯誤** 。當你在程式碼裡看到：
 
 .. code-block:: c
 
@@ -345,34 +347,34 @@ C 程序員不使用類似 ThisVariableIsATemporaryCounter 這樣華麗的名字
 
 你就知道 ``a`` 是什麼了。
 
-很多人認爲 typedef ``能提高可讀性`` 。實際不是這樣的。它們只在下列情況下有用：
+很多人認為 typedef ``能提高可讀性`` 。實際不是這樣的。它們只在下列情況下有用：
 
- (a) 完全不透明的對象 (這種情況下要主動使用 typedef 來 **隱藏** 這個對象實際上
+ (a) 完全不透明的物件 (這種情況下要主動使用 typedef 來 **隱藏** 這個物件實際上
      是什麼)。
 
-     例如： ``pte_t`` 等不透明對象，你只能用合適的訪問函數來訪問它們。
+     例如： ``pte_t`` 等不透明物件，你只能用合適的存取函式來存取它們。
 
      .. note::
 
-       不透明性和“訪問函數”本身是不好的。我們使用 pte_t 等類型的原因在於真
-       的是完全沒有任何共用的可訪問信息。
+       不透明性和“存取函式”本身是不好的。我們使用 pte_t 等類型的原因在於真
+       的是完全沒有任何共用的可存取資訊。
 
  (b) 清楚的整數類型，如此，這層抽象就可以 **幫助** 消除到底是 ``int`` 還是
      ``long`` 的混淆。
 
-     u8/u16/u32 是完全沒有問題的 typedef，不過它們更符合類別 (d) 而不是這裏。
+     u8/u16/u32 是完全沒有問題的 typedef，不過它們更符合類別 (d) 而不是這裡。
 
      .. note::
 
-       要這樣做，必須事出有因。如果某個變量是 ``unsigned long`` ，那麼沒有必要
+       要這樣做，必須事出有因。如果某個變數是 ``unsigned long`` ，那麼沒有必要
 
 	typedef unsigned long myflags_t;
 
      不過如果有一個明確的原因，比如它在某種情況下可能會是一個 ``unsigned int``
-     而在其他情況下可能爲 ``unsigned long`` ，那麼就不要猶豫，請務必使用
+     而在其他情況下可能為 ``unsigned long`` ，那麼就不要猶豫，請務必使用
      typedef。
 
- (c) 當你使用 sparse 按字面的創建一個 **新** 類型來做類型檢查的時候。
+ (c) 當你使用 sparse 按字面的建立一個 **新** 類型來做類型檢查的時候。
 
  (d) 和標準 C99 類型相同的類型，在某些例外的情況下。
 
@@ -380,45 +382,45 @@ C 程序員不使用類似 ThisVariableIsATemporaryCounter 這樣華麗的名字
      是有些人仍然拒絕使用它們。
 
      因此，Linux 特有的等同於標準類型的 ``u8/u16/u32/u64`` 類型和它們的有符號
-     類型是被允許的——儘管在你自己的新代碼中，它們不是強制要求要使用的。
+     類型是被允許的——儘管在你自己的新程式碼中，它們不是強制要求要使用的。
 
-     當編輯已經使用了某個類型集的已有代碼時，你應該遵循那些代碼中已經做出的選
+     當編輯已經使用了某個類型集的已有程式碼時，你應該遵循那些程式碼中已經做出的選
      擇。
 
- (e) 可以在用戶空間安全使用的類型。
+ (e) 可以在使用者空間安全使用的類型。
 
-     在某些用戶空間可見的結構體裏，我們不能要求 C99 類型而且不能用上面提到的
-     ``u32`` 類型。因此，我們在與用戶空間共享的所有結構體中使用 __u32 和類似
+     在某些使用者空間可見的結構體裡，我們不能要求 C99 類型而且不能用上面提到的
+     ``u32`` 類型。因此，我們在與使用者空間共享的所有結構體中使用 __u32 和類似
      的類型。
 
 可能還有其他的情況，不過基本的規則是 **永遠不要** 使用 typedef，除非你可以明
 確的應用上述某個規則中的一個。
 
-總的來說，如果一個指針或者一個結構體裏的元素可以合理的被直接訪問到，那麼它們
+總的來說，如果一個指標或者一個結構體裡的元素可以合理的被直接存取到，那麼它們
 就不應該是一個 typedef。
 
 
-6) 函數
+6) 函式
 -------
 
-函數應該簡短而漂亮，並且只完成一件事情。函數應該可以一屏或者兩屏顯示完 (我們
+函式應該簡短而漂亮，並且只完成一件事情。函式應該可以一屏或者兩屏顯示完 (我們
 都知道 ISO/ANSI 屏幕大小是 80x24)，只做一件事情，而且把它做好。
 
-一個函數的最大長度是和該函數的複雜度和縮進級數成反比的。所以，如果你有一個理
-論上很簡單的只有一個很長 (但是簡單) 的 case 語句的函數，而且你需要在每個 case
-裏做很多很小的事情，這樣的函數儘管很長，但也是可以的。
+一個函式的最大長度是和該函式的複雜度和縮排級數成反比的。所以，如果你有一個理
+論上很簡單的只有一個很長 (但是簡單) 的 case 語句的函式，而且你需要在每個 case
+裡做很多很小的事情，這樣的函式儘管很長，但也是可以的。
 
-不過，如果你有一個複雜的函數，而且你懷疑一個天分不是很高的高中一年級學生可能
-甚至搞不清楚這個函數的目的，你應該嚴格遵守前面提到的長度限制。使用輔助函數，
-併爲之取個具描述性的名字 (如果你覺得它們的性能很重要的話，可以讓編譯器內聯它
-們，這樣的效果往往會比你寫一個複雜函數的效果要好。)
+不過，如果你有一個複雜的函式，而且你懷疑一個天分不是很高的高中一年級學生可能
+甚至搞不清楚這個函式的目的，你應該嚴格遵守前面提到的長度限制。使用輔助函式，
+併為之取個具描述性的名字 (如果你覺得它們的效能很重要的話，可以讓編譯器行內它
+們，這樣的效果往往會比你寫一個複雜函式的效果要好。)
 
-函數的另外一個衡量標準是本地變量的數量。此數量不應超過 5－10 個，否則你的函數
-就有問題了。重新考慮一下你的函數，把它分拆成更小的函數。人的大腦一般可以輕鬆
-的同時跟蹤 7 個不同的事物，如果再增多的話，就會糊塗了。即便你聰穎過人，你也可
+函式的另外一個衡量標準是本地變數的數量。此數量不應超過 5－10 個，否則你的函式
+就有問題了。重新考慮一下你的函式，把它分拆成更小的函式。人的大腦一般可以輕鬆
+的同時追蹤 7 個不同的事物，如果再增多的話，就會糊塗了。即便你聰穎過人，你也可
 能會記不清你 2 個星期前做過的事情。
 
-在源文件裏，使用空行隔開不同的函數。如果該函數需要被導出，它的 **EXPORT** 宏
+在原始檔裡，使用空行隔開不同的函式。如果該函式需要被匯出，它的 **EXPORT** 巨集
 應該緊貼在它的結束大括號之下。比如：
 
 .. code-block:: c
@@ -429,37 +431,38 @@ C 程序員不使用類似 ThisVariableIsATemporaryCounter 這樣華麗的名字
 	}
 	EXPORT_SYMBOL(system_is_up);
 
-6.1) 函數原型
+6.1) 函式原型
 *************
 
-在函數原型中包含參數名和它們的數據類型。雖然 C 語言裏沒有這樣的要求，但在
-Linux 裏這是提倡的做法，因爲這樣可以很簡單的給讀者提供更多的有價值的信息。
+在函式原型中包含參數名和它們的資料類型。雖然 C 語言裡沒有這樣的要求，但在
+Linux 裡這是提倡的做法，因為這樣可以很簡單的給讀者提供更多的有價值的資訊。
 
-不要在函數聲明裏使用 ``extern`` 關鍵字，因爲這會導致代碼行變長，並且不是嚴格
+不要在函式宣告裡使用 ``extern`` 關鍵字，因為這會導致程式碼行變長，並且不是嚴格
 必需的。
 
-寫函數原型時，請保持 `元素順序規則 <https://lore.kernel.org/mm-commits/CAHk-=wiOCLRny5aifWNhr621kYrJwhfURsa0vFPeUEm8mF0ufg@mail.gmail.com/>`_ 。
-例如下列函數聲明::
+寫函式原型時，請保持 `元素順序規則 <https://lore.kernel.org/mm-commits/CAHk-=wiOCLRny5aifWNhr621kYrJwhfURsa0vFPeUEm8mF0ufg@mail.gmail.com/>`_ 。
+例如下列函式宣告::
 
  __init void * __must_check action(enum magic value, size_t size, u8 count,
 				   char *fmt, ...) __printf(4, 5) __malloc;
 
-推薦的函數原型元素順序是：
+推薦的函式原型元素順序是：
 
 - 儲存類型（下方的 ``static __always_inline`` ，注意 ``__always_inline``
   技術上來講是個屬性但被當做 ``inline`` ）
-- 儲存類型屬性（上方的 ``__init`` ——即節聲明，但也像 ``__cold`` ）
+- 儲存類型屬性（上方的 ``__init`` ——即節宣告，但也像 ``__cold`` ）
 - 返回類型（上方的 ``void *`` ）
 - 返回類型屬性（上方的 ``__must_check`` ）
-- 函數名（上方的 ``action`` ）
-- 函數參數（上方的 ``(enum magic value, size_t size, u8 count, char *fmt, ...)`` ，
-  注意必須寫上參數名）
-- 函數參數屬性（上方的 ``__printf(4, 5)`` ）
-- 函數行爲屬性（上方的 ``__malloc`` ）
+- 函式名（上方的 ``action`` ）
+- 函式參數（上方的
+  ``(enum magic value, size_t size, u8 count, char *fmt, ...)`` ，注意必須
+  寫上參數名）
+- 函式參數屬性（上方的 ``__printf(4, 5)`` ）
+- 函式行為屬性（上方的 ``__malloc`` ）
 
-請注意，對於函數 **定義** （即實際函數體），編譯器不允許在函數參數之後添加函
-數參數屬性。在這種情況下，它們應該跟隨存儲類型屬性（例如，與上面的 **聲明**
-示例相比，請注意下面的 ``__printf(4, 5)`` 的位置發生了變化）::
+請注意，對於函式 **定義** （即實際函式體），編譯器不允許在函式參數之後添加函
+數參數屬性。在這種情況下，它們應該跟隨儲存類型屬性（例如，與上面的 **宣告**
+範例相比，請注意下面的 ``__printf(4, 5)`` 的位置發生了變化）::
 
  static __always_inline __init __printf(4, 5) void * __must_check action(enum magic value,
 		size_t size, u8 count, char *fmt, ...) __malloc
@@ -467,26 +470,26 @@ Linux 裏這是提倡的做法，因爲這樣可以很簡單的給讀者提供
 	...
  }
 
-7) 集中的函數退出途徑
+7) 集中的函式退出途徑
 ---------------------
 
 雖然被某些人聲稱已經過時，但是 goto 語句的等價物還是經常被編譯器所使用，具體
 形式是無條件跳轉指令。
 
-當一個函數從多個位置退出，並且需要做一些類似清理的常見操作時，goto 語句就很方
+當一個函式從多個位置退出，並且需要做一些類似清理的常見操作時，goto 語句就很方
 便了。如果並不需要清理操作，那麼直接 return 即可。
 
-選擇一個能夠說明 goto 行爲或它爲何存在的標籤名。如果 goto 要釋放 ``buffer``,
+選擇一個能夠說明 goto 行為或它為何存在的標籤名。如果 goto 要釋放 ``buffer``,
 一個不錯的名字可以是 ``out_free_buffer:`` 。別去使用像 ``err1:`` 和 ``err2:``
-這樣的GW_BASIC 名稱，因爲一旦你添加或刪除了 (函數的) 退出路徑，你就必須對它們
+這樣的GW_BASIC 名稱，因為一旦你添加或刪除了 (函式的) 退出路徑，你就必須對它們
 重新編號，這樣會難以去檢驗正確性。
 
 使用 goto 的理由是：
 
-- 無條件語句容易理解和跟蹤
+- 無條件語句容易理解和追蹤
 - 嵌套程度減小
 - 可以避免由於修改時忘記更新個別的退出點而導致錯誤
-- 讓編譯器省去刪除冗餘代碼的工作 ;)
+- 讓編譯器省去刪除冗餘程式碼的工作 ;)
 
 .. code-block:: c
 
@@ -521,7 +524,7 @@ Linux 裏這是提倡的做法，因爲這樣可以很簡單的給讀者提供
 		kfree(foo);
 		return ret;
 
-這段代碼的錯誤是，在某些退出路徑上 ``foo`` 是 NULL。通常情況下，通過把它分離
+這段程式碼的錯誤是，在某些退出路徑上 ``foo`` 是 NULL。通常情況下，透過把它分離
 成兩個錯誤標籤 ``err_free_bar:`` 和 ``err_free_foo:`` 來修復這個錯誤：
 
 .. code-block:: c
@@ -535,22 +538,23 @@ Linux 裏這是提倡的做法，因爲這樣可以很簡單的給讀者提供
 理想情況下，你應該模擬錯誤來測試所有退出路徑。
 
 
-8) 註釋
+8) 註解
 -------
 
-註釋是好的，不過有過度註釋的危險。永遠不要在註釋裏解釋你的代碼是如何運作的：
-更好的做法是讓別人一看你的代碼就可以明白，解釋寫的很差的代碼是浪費時間。
+註解是好的，不過有過度註解的危險。永遠不要在註解裡解釋你的程式碼是如何運作的：
+更好的做法是讓別人一看你的程式碼就可以明白，解釋寫的很差的程式碼是浪費時間。
 
-一般來說你用註釋告訴別人你的代碼做了什麼，而不是怎麼做的。也請你不要把
-註釋放在一個函數體內部：如果函數複雜到你需要獨立的註釋其中的一部分，你很可能
-需要回到第六章看一看。你可以做一些小注釋來註明或警告某些很聰明 (或者槽糕) 的
-做法，但不要加太多。你應該做的，是把註釋放在函數的頭部，告訴人們它做了什麼，
+一般來說你用註解告訴別人你的程式碼做了什麼，而不是怎麼做的。也請你不要把
+註解放在一個函式體內部：如果函式複雜到你需要獨立的註解其中的一部分，你很可能
+需要回到第六章看一看。你可以做一些小註解來註明或警告某些很聰明 (或者槽糕) 的
+做法，但不要加太多。你應該做的，是把註解放在函式的頭部，告訴人們它做了什麼，
 也可以加上它做這些事情的原因。
 
-當註釋內核 API 函數時，請使用 kernel-doc 格式。詳見
-Documentation/translations/zh_CN/doc-guide/index.rst 和 tools/docs/kernel-doc 。
+當註解核心 API 函式時，請使用 kernel-doc 格式。詳見
+Documentation/translations/zh_CN/doc-guide/index.rst 和
+tools/docs/kernel-doc 。
 
-長 (多行) 註釋的首選風格是：
+長 (多行) 註解的首選風格是：
 
 .. code-block:: c
 
@@ -563,7 +567,7 @@ Documentation/translations/zh_CN/doc-guide/index.rst 和 tools/docs/kernel-doc 
 	 * with beginning and ending almost-blank lines.
 	 */
 
-對於在 net/ 和 drivers/net/ 的文件，首選的長 (多行) 註釋風格有些不同。
+對於在 net/ 和 drivers/net/ 的檔案，首選的長 (多行) 註解風格有些不同。
 
 .. code-block:: c
 
@@ -574,22 +578,22 @@ Documentation/translations/zh_CN/doc-guide/index.rst 和 tools/docs/kernel-doc 
 	 * but there is no initial almost-blank line.
 	 */
 
-註釋數據也是很重要的，不管是基本類型還是衍生類型。爲了方便實現這一點，每一行
-應只聲明一個數據 (不要使用逗號來一次聲明多個數據)。這樣你就有空間來爲每個數據
-寫一段小注釋來解釋它們的用途了。
+註解資料也是很重要的，不管是基本類型還是衍生類型。為了方便實作這一點，每一行
+應只宣告一個資料 (不要使用逗號來一次宣告多個資料)。這樣你就有空間來為每個資料
+寫一段小註解來解釋它們的用途了。
 
 
 9) 你已經把事情弄糟了
 ---------------------
 
-這沒什麼，我們都是這樣。可能你長期使用 Unix 的朋友已經告訴你
-``GNU emacs`` 能自動幫你格式化 C 源代碼，而且你也注意到了，確實是這樣，不過它
-所使用的默認值和我們想要的相去甚遠 (實際上，甚至比隨機打的還要差——無數個猴子
-在 GNU emacs 裏打字永遠不會創造出一個好程序)
+這沒什麼，我們都是這樣。可能你長期使用 Unix 的朋友已經告訴你``GNU emacs``
+能自動幫你格式化 C 原始程式碼，而且你也注意到了，確實是這樣，不過它所使用
+的預設值和我們想要的相去甚遠 (實際上，甚至比隨機打的還要差——無數個猴子在
+GNU emacs 裡打字永遠不會創造出一個好程式)
 *(譯註：Infinite Monkey Theorem)*
 
 所以你要麼放棄 GNU emacs，要麼改變它讓它使用更合理的設定。要採用後一個方案，
-你可以把下面這段粘貼到你的 .emacs 文件裏。
+你可以把下面這段貼上到你的 .emacs 檔案裡。
 
 .. code-block:: elisp
 
@@ -640,31 +644,31 @@ Documentation/translations/zh_CN/doc-guide/index.rst 和 tools/docs/kernel-doc 
    (expand-file-name "~/src/linux-trees")
    'linux-kernel)
 
-這會讓 emacs 在 ``~/src/linux-trees`` 下的 C 源文件獲得更好的內核代碼風格。
+這會讓 emacs 在 ``~/src/linux-trees`` 下的 C 原始檔獲得更好的核心程式碼風格。
 
-不過就算你嘗試讓 emacs 正確的格式化代碼失敗了，也並不意味着你失去了一切：還可
-以用 ``indent`` 。
+不過就算你嘗試讓 emacs 正確的格式化程式碼失敗了，也並不意味著你失去了一切：
+還可以用 ``indent`` 。
 
 不過，GNU indent 也有和 GNU emacs 一樣有問題的設定，所以你需要給它一些命令選
-項。不過，這還不算太糟糕，因爲就算是 GNU indent 的作者也認同 K&R 的權威性
+項。不過，這還不算太糟糕，因為就算是 GNU indent 的作者也認同 K&R 的權威性
 (GNU 的人並不是壞人，他們只是在這個問題上被嚴重的誤導了)，所以你只要給 indent
-指定選項 ``-kr -i8`` (代表 ``K&R，8 字符縮進``)，或使用 ``scripts/Lindent``
-這樣就可以以最時髦的方式縮進源代碼。
+指定選項 ``-kr -i8`` (代表 ``K&R，8 字元縮排``)，或使用 ``scripts/Lindent``
+這樣就可以以最時髦的方式縮排原始程式碼。
 
-``indent`` 有很多選項，特別是重新格式化註釋的時候，你可能需要看一下它的手冊。
-不過記住： ``indent`` 不能修正壞的編程習慣。
+``indent`` 有很多選項，特別是重新格式化註解的時候，你可能需要看一下它的手冊。
+不過記住： ``indent`` 不能修正壞的程式設計習慣。
 
-請注意，您還可以使用 ``clang-format`` 工具幫助您處理這些規則，快速自動重新格
-式化部分代碼，並審閱整個文件以發現代碼風格錯誤、打字錯誤和可能的改進。它還可
-以方便地排序 ``#include`` ，對齊變量/宏，重排文本和其他類似任務。
+請注意，您還可以使用 ``clang-format`` 工具幫助您處理這些規則，快速自動重新
+格式化部分程式碼，並審閱整個檔案以發現程式碼風格錯誤、打字錯誤和可能的改進。
+它還可以方便地排序 ``#include`` ，對齊變數/巨集，重排文字和其他類似任務。
 詳見 Documentation/dev-tools/clang-format.rst 。
 
 
-10) Kconfig 配置文件
+10) Kconfig 設定檔
 --------------------
 
-對於遍佈源碼樹的所有 Kconfig* 配置文件來說，它們縮進方式有所不同。緊挨着
-``config`` 定義的行，用一個製表符縮進，然而 help 信息的縮進則額外增加 2 個空
+對於遍佈源碼樹的所有 Kconfig* 設定文件來說，它們縮排方式有所不同。緊挨著
+``config`` 定義的行，用一個製表符縮排，然而 help 資訊的縮排則額外增加 2 個空
 格。舉個例子::
 
   config AUDIT
@@ -676,7 +680,7 @@ Documentation/translations/zh_CN/doc-guide/index.rst 和 tools/docs/kernel-doc 
 	  logging of avc messages output).  Does not do system-call
 	  auditing without CONFIG_AUDITSYSCALL.
 
-而那些危險的功能 (比如某些文件系統的寫支持) 應該在它們的提示字符串裏顯著的聲
+而那些危險的功能 (比如某些檔案系統的寫支援) 應該在它們的提示字串裡顯著的聲
 明這一點::
 
   config ADFS_FS_RW
@@ -684,49 +688,50 @@ Documentation/translations/zh_CN/doc-guide/index.rst 和 tools/docs/kernel-doc 
 	depends on ADFS_FS
 	...
 
-要查看配置文件的完整文檔，請看 Documentation/kbuild/kconfig-language.rst 。
+要查看設定文件的完整文件，請看 Documentation/kbuild/kconfig-language.rst 。
 
 
-11) 數據結構
+11) 資料結構
 ------------
 
-如果一個數據結構，在創建和銷燬它的單線執行環境之外可見，那麼它必須要有一個引
-用計數器。內核裏沒有垃圾收集 (並且內核之外的垃圾收集慢且效率低下)，這意味着你
-絕對需要記錄你對這種數據結構的使用情況。
+如果一個資料結構，在建立和銷燬它的單線執行環境之外可見，那麼它必須要有一個引
+用計數器。核心裡沒有垃圾收集 (並且核心之外的垃圾收集慢且效率低下)，這意味著你
+絕對需要記錄你對這種資料結構的使用情況。
 
-引用計數意味着你能夠避免上鎖，並且允許多個用戶並行訪問這個數據結構——而不需要
-擔心這個數據結構僅僅因爲暫時不被使用就消失了，那些用戶可能不過是沉睡了一陣或
+引用計數意味著你能夠避免上鎖，並且允許多個使用者並行存取這個資料結構——而不需要
+擔心這個資料結構僅僅因為暫時不被使用就消失了，那些使用者可能不過是沉睡了一陣或
 者做了一些其他事情而已。
 
-注意上鎖 **不能** 取代引用計數。上鎖是爲了保持數據結構的一致性，而引用計數是一
-個內存管理技巧。通常二者都需要，不要把兩個搞混了。
+注意上鎖 **不能** 取代引用計數。上鎖是為了保持資料結構的一致性，而引用計數是一
+個記憶體管理技巧。通常二者都需要，不要把兩個搞混了。
 
-很多數據結構實際上有 2 級引用計數，它們通常有不同 ``類`` 的用戶。子類計數器統
-計子類用戶的數量，每當子類計數器減至零時，全局計數器減一。
+很多資料結構實際上有 2 級引用計數，它們通常有不同 ``類`` 的使用者。子類計
+數器統計子類使用者的數量，每當子類計數器減至零時，全域計數器減一。
 
-這種 ``多級引用計數`` 的例子可以在內存管理 (``struct mm_struct``: mm_users 和
-mm_count)，和文件系統 (``struct super_block``: s_count 和 s_active) 中找到。
+這種 ``多級引用計數`` 的例子可以在記憶體管理 (``struct mm_struct``:
+mm_users 和mm_count)，和檔案系統 (``struct super_block``: s_count 和
+s_active) 中找到。
 
-記住：如果另一個執行線索可以找到你的數據結構，但這個數據結構沒有引用計數器，
-這裏幾乎肯定是一個 bug。
+記住：如果另一個執行線索可以找到你的資料結構，但這個資料結構沒有引用計數器，
+這裡幾乎肯定是一個 bug。
 
 
-12) 宏，枚舉和RTL
------------------
+12) 巨集，列舉和RTL
+-------------------
 
-用於定義常量的宏的名字及枚舉裏的標籤需要大寫。
+用於定義常數的巨集的名字及列舉裡的標籤需要大寫。
 
 .. code-block:: c
 
 	#define CONSTANT 0x12345
 
-在定義幾個相關的常量時，最好用枚舉。
+在定義幾個相關的常數時，最好用列舉。
 
-宏的名字請用大寫字母，不過形如函數的宏的名字可以用小寫字母。
+巨集的名字請用大寫字母，不過形如函式的巨集的名字可以用小寫字母。
 
-通常如果能寫成內聯函數就不要寫成像函數的宏。
+通常如果能寫成行內函式就不要寫成像函式的巨集。
 
-含有多個語句的宏應該被包含在一個 do-while 代碼塊裏：
+含有多個語句的巨集應該被包含在一個 do-while 程式碼塊裡：
 
 .. code-block:: c
 
@@ -736,9 +741,9 @@ mm_count)，和文件系統 (``struct super_block``: s_count 和 s_active) 中
 				do_this(b, c);		\
 		} while (0)
 
-使用宏的時候應避免的事情：
+使用巨集的時候應避免的事情：
 
-1) 影響控制流程的宏：
+1) 影響控制流程的巨集：
 
 .. code-block:: c
 
@@ -748,30 +753,30 @@ mm_count)，和文件系統 (``struct super_block``: s_count 和 s_active) 中
 				return -EBUGGERED;	\
 		} while (0)
 
-**非常** 不好。它看起來像一個函數，不過卻能導致 ``調用`` 它的函數退出；不要打
-亂讀者大腦裏的語法分析器。
+**非常** 不好。它看起來像一個函式，不過卻能導致 ``呼叫`` 它的函式退出；不要打
+亂讀者大腦裡的語法分析器。
 
-2) 依賴於一個固定名字的本地變量的宏：
+2) 依賴於一個固定名字的本地變數的巨集：
 
 .. code-block:: c
 
 	#define FOO(val) bar(index, val)
 
-可能看起來像是個不錯的東西，不過它非常容易把讀代碼的人搞糊塗，而且容易導致看起
-來不相關的改動帶來錯誤。
+可能看起來像是個不錯的東西，不過它非常容易把讀程式碼的人搞糊塗，而且容易導
+致看起來不相關的改動帶來錯誤。
 
-3) 作爲左值的帶參數的宏： FOO(x) = y；如果有人把 FOO 變成一個內聯函數的話，這
+3) 作為左值的帶參數的巨集： FOO(x) = y；如果有人把 FOO 變成一個行內函式的話，這
    種用法就會出錯了。
 
-4) 忘記了優先級：使用表達式定義常量的宏必須將表達式置於一對小括號之內。帶參數
-   的宏也要注意此類問題。
+4) 忘記了優先級：使用表達式定義常數的巨集必須將表達式置於一對小括號之內。帶參數
+   的巨集也要注意此類問題。
 
 .. code-block:: c
 
 	#define CONSTANT 0x4000
 	#define CONSTEXP (CONSTANT | 3)
 
-5) 在宏裏定義類似函數的本地變量時命名衝突：
+5) 在巨集裡定義類似函式的本地變數時命名衝突：
 
 .. code-block:: c
 
@@ -782,45 +787,45 @@ mm_count)，和文件系統 (``struct super_block``: s_count 和 s_active) 中
 		(ret);				\
 	})
 
-ret 是本地變量的通用名字—— __foo_ret 更不容易與一個已存在的變量衝突。
+ret 是本地變數的通用名字—— __foo_ret 更不容易與一個已存在的變數衝突。
 
-cpp 手冊對宏的講解很詳細。gcc internals 手冊也詳細講解了 RTL，內核裏的彙編語
+cpp 手冊對巨集的講解很詳細。gcc internals 手冊也詳細講解了 RTL，核心裡的組譯語
 言經常用到它。
 
 
-13) 打印內核消息
+13) 列印核心訊息
 ----------------
 
-內核開發者應該看起來有文化。請一定注意內核信息的拼寫，以給人良好的印象。
+核心開發者應該看起來有文化。請一定注意核心資訊的拼寫，以給人良好的印象。
 不要用不規範的單詞比如 ``dont``，而要用 ``do not`` 或者 ``don't`` 。保證這些信
 息簡單明瞭、無歧義。
 
-內核信息不必以英文句號結束。
+核心資訊不必以英文句號結束。
 
-在小括號裏打印數字 (%d) 沒有任何價值，應該避免這樣做。
+在小括號裡列印數字 (%d) 沒有任何價值，應該避免這樣做。
 
-<linux/device.h> 裏有一些驅動模型診斷宏，你應該使用它們，以確保信息對應於正確
-的設備和驅動，並且被標記了正確的消息級別。這些宏有：dev_err(), dev_warn(),
-dev_info() 等等。對於那些不和某個特定設備相關連的信息，<linux/printk.h> 定義
-了 pr_notice(), pr_info(), pr_warn(), pr_err() 和其他。
+<linux/device.h> 裡有一些驅動模型診斷巨集，你應該使用它們，以確保資訊對應
+於正確的設備和驅動，並且被標記了正確的訊息級別。這些巨集有：dev_err(),
+dev_warn(),dev_info() 等等。對於那些不和某個特定設備相關連的資訊，
+<linux/printk.h> 定義了 pr_notice(), pr_info(), pr_warn(), pr_err() 和其他。
 
-寫出好的調試信息可以是一個很大的挑戰；一旦你寫出後，這些信息在遠程除錯時能提
-供極大的幫助。然而打印調試信息的處理方式同打印非調試信息不同。其他 pr_XXX()
-函數能無條件地打印，pr_debug() 卻不；默認情況下它不會被編譯，除非定義了 DEBUG
-或設定了 CONFIG_DYNAMIC_DEBUG。實際這同樣是爲了 dev_dbg()，一個相關約定是在一
-個已經開啓了 DEBUG 時，使用 VERBOSE_DEBUG 來添加 dev_vdbg()。
+寫出好的除錯資訊可以是一個很大的挑戰；一旦你寫出後，這些資訊在遠端除錯時能提
+供極大的幫助。然而列印除錯資訊的處理方式同列印非除錯資訊不同。其他 pr_XXX()
+函式能無條件地列印，pr_debug() 卻不；預設情況下它不會被編譯，除非定義了 DEBUG
+或設定了 CONFIG_DYNAMIC_DEBUG。實際這同樣是為了 dev_dbg()，一個相關約定是在一
+個已經開啟了 DEBUG 時，使用 VERBOSE_DEBUG 來添加 dev_vdbg()。
 
-許多子系統擁有 Kconfig 調試選項來開啓對應 Makefile 裏面的 -DDEBUG；在其他
-情況下，特殊文件使用 #define DEBUG。當一條調試信息需要被無條件打印時，例如，
-如果已經包含一個調試相關的 #ifdef 條件，printk(KERN_DEBUG ...) 就可被使用。
+許多子系統擁有 Kconfig 除錯選項來開啟對應 Makefile 裡面的 -DDEBUG；在其他
+情況下，特定檔案使用 #define DEBUG。當一條除錯資訊需要被無條件列印時，例如，
+如果已經包含一個除錯相關的 #ifdef 條件，printk(KERN_DEBUG ...) 就可被使用。
 
 
-14) 分配內存
-------------
+14) 分配記憶體
+--------------
 
-內核提供了下面的一般用途的內存分配函數：
-kmalloc(), kzalloc(), kmalloc_array(), kcalloc(), vmalloc() 和 vzalloc()。
-請參考 API 文檔以獲取有關它們的詳細信息：
+核心提供了下面的一般用途的記憶體分配函式：
+kmalloc(), kzalloc(), kmalloc_objs(), kzalloc_objs(), vmalloc() 和 vzalloc()。
+請參考 API 文件以獲取有關它們的詳細資訊：
 Documentation/translations/zh_CN/core-api/memory-allocation.rst 。
 
 傳遞結構體大小的首選形式是這樣的：
@@ -830,103 +835,106 @@ Documentation/translations/zh_CN/core-api/memory-allocation.rst 。
 	p = kmalloc_obj(*p, ...);
 
 另外一種傳遞方式中，sizeof 的操作數是結構體的名字，這樣會降低可讀性，並且可能
-會引入 bug。有可能指針變量類型被改變時，而對應的傳遞給內存分配函數的 sizeof
+會引入 bug。有可能指標變數類型被改變時，而對應的傳遞給記憶體分配函式的 sizeof
 的結果不變。
 
-強制轉換一個 void 指針返回值是多餘的。C 語言本身保證了從 void 指針到其他任何
-指針類型的轉換是沒有問題的。
+強制轉換一個 void 指標回傳值是多餘的。C 語言本身保證了從 void 指標到其他任何
+指標類型的轉換是沒有問題的。
 
-分配一個數組的首選形式是這樣的：
+分配一個陣列的首選形式是這樣的：
 
 .. code-block:: c
 
-	p = kmalloc_array(n, sizeof(...), ...);
+	p = kmalloc_objs(*p, n, ...);
 
-分配一個零長數組的首選形式是這樣的：
+分配一個零長陣列的首選形式是這樣的：
 
 .. code-block:: c
 
-	p = kcalloc(n, sizeof(...), ...);
+	p = kzalloc_objs(*p, n, ...);
+
+這兩種形式都會檢查分配大小 n * sizeof(...) 是否溢位，如果發生溢位則回傳
+NULL。
 
-兩種形式都會檢查分配 n * sizeof(...) 大小時內存的溢出，如果溢出返回 NULL。
+兩種形式都會檢查分配 n * sizeof(...) 大小時記憶體的溢出，如果溢出返回 NULL。
 
-在沒有 __GFP_NOWARN 的情況下使用時，這些通用分配函數都會在失敗時發起堆棧轉儲，
-因此當返回NULL時，沒有必要發出額外的失敗消息。
+在沒有 __GFP_NOWARN 的情況下使用時，這些通用分配函式都會在失敗時發起堆疊轉儲，
+因此當返回NULL時，沒有必要發出額外的失敗訊息。
 
-15) 內聯弊病
+15) 行內弊病
 ------------
 
-有一個常見的誤解是 ``內聯`` 是 gcc 提供的可以讓代碼運行更快的一個選項。雖然使
-用內聯函數有時候是恰當的 (比如作爲一種替代宏的方式，請看第十二章)，不過很多情
-況下不是這樣。inline 的過度使用會使內核變大，從而使整個系統運行速度變慢。
-因爲體積大內核會佔用更多的指令高速緩存，而且會導致 pagecache 的可用內存減少。
-想象一下，一次 pagecache 未命中就會導致一次磁盤尋址，將耗時 5 毫秒。5 毫秒的
-時間內 CPU 能執行很多很多指令。
-
-一個基本的原則是如果一個函數有 3 行以上，就不要把它變成內聯函數。這個原則的一
-個例外是，如果你知道某個參數是一個編譯時常量，而且因爲這個常量你確定編譯器在
-編譯時能優化掉你的函數的大部分代碼，那仍然可以給它加上 inline 關鍵字。
-kmalloc() 內聯函數就是一個很好的例子。
-
-人們經常主張給 static 的而且只用了一次的函數加上 inline，如此不會有任何損失，
-因爲沒有什麼好權衡的。雖然從技術上說這是正確的，但是實際上這種情況下即使不加
-inline gcc 也可以自動使其內聯。而且其他用戶可能會要求移除 inline，由此而來的
+有一個常見的誤解是 ``行內`` 是 gcc 提供的可以讓程式碼執行更快的一個選項。
+雖然使用行內函式有時候是恰當的 (比如作為一種替代巨集的方式，請看第十二章)，
+不過很多情況下不是這樣。inline 的過度使用會使核心變大，從而使整個系統執行
+速度變慢。因為體積大核心會佔用更多的指令高速快取，而且會導致 pagecache 的
+可用記憶體減少。想象一下，一次 pagecache 未命中就會導致一次磁碟尋址，將耗
+時 5 毫秒。5 毫秒的時間內 CPU 能執行很多很多指令。
+
+一個基本的原則是如果一個函式有 3 行以上，就不要把它變成行內函式。這個原則的一
+個例外是，如果你知道某個參數是一個編譯時常數，而且因為這個常數你確定編譯器在
+編譯時能最佳化掉你的函式的大部分程式碼，那仍然可以給它加上 inline 關鍵字。
+kmalloc() 行內函式就是一個很好的例子。
+
+人們經常主張給 static 的而且只用了一次的函式加上 inline，如此不會有任何損失，
+因為沒有什麼好權衡的。雖然從技術上說這是正確的，但是實際上這種情況下即使不加
+inline gcc 也可以自動使其行內。而且其他使用者可能會要求移除 inline，由此而來的
 爭論會抵消 inline 自身的潛在價值，得不償失。
 
 
-16) 函數返回值及命名
+16) 函式回傳值及命名
 --------------------
 
-函數可以返回多種不同類型的值，最常見的一種是表明函數執行成功或者失敗的值。這樣
-的一個值可以表示爲一個錯誤代碼整數 (-Exxx＝失敗，0＝成功) 或者一個 ``成功``
-布爾值 (0＝失敗，非0＝成功)。
+函式可以返回多種不同類型的值，最常見的一種是表明函式執行成功或者失敗的值。這樣
+的一個值可以表示為一個錯誤程式碼整數 (-Exxx＝失敗，0＝成功) 或者一個 ``成功``
+布林值 (0＝失敗，非0＝成功)。
 
 混合使用這兩種表達方式是難於發現的 bug 的來源。如果 C 語言本身嚴格區分整形和
-布爾型變量，那麼編譯器就能夠幫我們發現這些錯誤... 不過 C 語言不區分。爲了避免
+布林型變數，那麼編譯器就能夠幫我們發現這些錯誤... 不過 C 語言不區分。為了避免
 產生這種 bug，請遵循下面的慣例::
 
-	如果函數的名字是一個動作或者強制性的命令，那麼這個函數應該返回錯誤代
-	碼整數。如果是一個判斷，那麼函數應該返回一個“成功”布爾值。
+	如果函式的名字是一個動作或者強制性的命令，那麼這個函式應該返回錯誤代
+	碼整數。如果是一個判斷，那麼函式應該返回一個“成功”布林值。
 
 比如， ``add work`` 是一個命令，所以 add_work() 在成功時返回 0，在失敗時返回
--EBUSY。類似的，因爲 ``PCI device present`` 是一個判斷，所以 pci_dev_present()
+-EBUSY。類似的，因為 ``PCI device present`` 是一個判斷，所以 pci_dev_present()
 在成功找到一個匹配的設備時應該返回 1，如果找不到時應該返回 0。
 
-所有 EXPORTed 函數都必須遵守這個慣例，所有的公共函數也都應該如此。私有
-(static) 函數不需要如此，但是我們也推薦這樣做。
+所有 EXPORTed 函式都必須遵守這個慣例，所有的公共函式也都應該如此。私有
+(static) 函式不需要如此，但是我們也推薦這樣做。
 
-返回值是實際計算結果而不是計算是否成功的標誌的函數不受此慣例的限制。通常
-他們通過返回一些正常值範圍之外的結果來表示出錯。典型的例子是返回指針的函數，
+回傳值是實際計算結果而不是計算是否成功的標誌的函式不受此慣例的限制。通常
+他們透過返回一些正常值範圍之外的結果來表示出錯。典型的例子是返回指標的函式，
 他們使用 NULL 或者 ERR_PTR 機制來報告錯誤。
 
-17) 使用布爾類型
+17) 使用布林類型
 ----------------
 
-Linux內核布爾（bool）類型是C99 _Bool類型的別名。布爾值只能爲0或1，而對布爾的
-隱式或顯式轉換將自動將值轉換爲true或false。在使用布爾類型時 **不需要** 構造，
+Linux核心布林（bool）類型是C99 _Bool類型的別名。布林值只能為0或1，而對布林的
+隱式或顯式轉換將自動將值轉換為true或false。在使用布林類型時 **不需要** 構造，
 它會消除一類錯誤。
 
-使用布爾值時，應使用true和false定義，而不是1和0。
+使用布林值時，應使用true和false定義，而不是1和0。
 
-布爾函數返回類型和堆棧變量總是可以在適當的時候使用。鼓勵使用布爾來提高可讀性，
-並且布爾值在存儲時通常比“int”更好。
+布林函式返回類型和堆疊變數總是可以在適當的時候使用。鼓勵使用布林來提高可讀性，
+並且布林值在儲存時通常比“int”更好。
 
-如果緩存行佈局或值的大小很重要，請不要使用布爾，因爲其大小和對齊方式根據編譯
-的體系結構而不同。針對對齊和大小進行優化的結構體不應使用布爾。
+如果快取行佈局或值的大小很重要，請不要使用布林，因為其大小和對齊方式根據編譯
+的體系結構而不同。針對對齊和大小進行最佳化的結構體不應使用布林。
 
-如果一個結構體有多個true/false值，請考慮將它們合併爲具有1比特成員的位域，或使
+如果一個結構體有多個true/false值，請考慮將它們合併為具有1比特成員的位域，或使
 用適當的固定寬度類型，如u8。
 
-類似地，對於函數參數，多個true/false值可以合併爲單個按位的“標誌”參數，如果調
-用點具有裸true/false常量，“標誌”參數通常是更具可讀性的替代方法。
+類似地，對於函式參數，多個true/false值可以合併為單個按位的“標誌”參數，如果調
+用點具有裸true/false常數，“標誌”參數通常是更具可讀性的替代方法。
 
-總之，在結構體和參數中有限地使用布爾可以提高可讀性。
+總之，在結構體和參數中有限地使用布林可以提高可讀性。
 
-18) 不要重新發明內核宏
-----------------------
+18) 不要重新發明核心巨集
+------------------------
 
-頭文件 include/linux/kernel.h 包含了一些宏，你應該使用它們，而不要自己寫一些
-它們的變種。比如，如果你需要計算一個數組的長度，使用這個宏
+標頭檔 include/linux/kernel.h 包含了一些巨集，你應該使用它們，而不要自己寫一些
+它們的變種。比如，如果你需要計算一個陣列的長度，使用這個巨集
 
 .. code-block:: c
 
@@ -938,15 +946,15 @@ Linux內核布爾（bool）類型是C99 _Bool類型的別名。布爾值只能
 
 	#define sizeof_field(t, f) (sizeof(((t*)0)->f))
 
-還有可以做嚴格的類型檢查的 min() 和 max() 宏，如果你需要可以使用它們。你可以
-自己看看那個頭文件裏還定義了什麼你可以拿來用的東西，如果有定義的話，你就不應
-在你的代碼裏自己重新定義。
+還有可以做嚴格的類型檢查的 min() 和 max() 巨集，如果你需要可以使用它們。你可以
+自己看看那個標頭檔裡還定義了什麼你可以拿來用的東西，如果有定義的話，你就不應
+在你的程式碼裡自己重新定義。
 
 
 19) 編輯器模式行和其他需要羅嗦的事情
 ------------------------------------
 
-有一些編輯器可以解釋嵌入在源文件裏的由一些特殊標記標明的配置信息。比如，emacs
+有一些編輯器可以解釋嵌入在原始檔裡的由一些特殊標記標明的設定資訊。比如，emacs
 能夠解析被標記成這樣的行：
 
 .. code-block:: c
@@ -969,29 +977,29 @@ Vim 能夠解析這樣的標記：
 
 	/* vim:set sw=8 noet */
 
-不要在源代碼中包含任何這樣的內容。每個人都有他自己的編輯器配置，你的源文件不
-應該覆蓋別人的配置。這包括有關縮進和模式配置的標記。人們可以使用他們自己定製
-的模式，或者使用其他可以產生正確的縮進的巧妙方法。
+不要在原始程式碼中包含任何這樣的內容。每個人都有他自己的編輯器設定，你的原
+始檔不應該覆蓋別人的設定。這包括有關縮排和模式設定的標記。人們可以使用他們
+自己定製的模式，或者使用其他可以產生正確的縮排的巧妙方法。
 
 
-20) 內聯彙編
+20) 行內組譯
 ------------
 
-在特定架構的代碼中，你可能需要內聯彙編與 CPU 和平臺相關功能連接。需要這麼做時
-就不要猶豫。然而，當 C 可以完成工作時，不要平白無故地使用內聯彙編。在可能的情
-況下，你可以並且應該用 C 和硬件溝通。
+在特定架構的程式碼中，你可能需要行內組譯與 CPU 和平臺相關功能連接。需要這
+麼做時就不要猶豫。然而，當 C 可以完成工作時，不要平白無故地使用行內組譯。
+在可能的情況下，你可以並且應該用 C 和硬體溝通。
 
-請考慮去寫捆綁通用位元 (wrap common bits) 的內聯彙編的簡單輔助函數，別去重複
-地寫下只有細微差異內聯彙編。記住內聯彙編可以使用 C 參數。
+請考慮去寫捆綁通用位元 (wrap common bits) 的行內組譯的簡單輔助函式，別去重複
+地寫下只有細微差異行內組譯。記住行內組譯可以使用 C 參數。
 
-大型，有一定複雜度的彙編函數應該放在 .S 文件內，用相應的 C 原型定義在 C 頭文
-件中。彙編函數的 C 原型應該使用 ``asmlinkage`` 。
+大型，有一定複雜度的組譯函式應該放在 .S 檔案內，用相應的 C 原型定義在 C 標頭
+檔中。組譯函式的 C 原型應該使用 ``asmlinkage`` 。
 
-你可能需要把彙編語句標記爲 volatile，用來阻止 GCC 在沒發現任何副作用後就把它
-移除了。你不必總是這樣做，儘管，這不必要的舉動會限制優化。
+你可能需要把組譯語句標記為 volatile，用來阻止 GCC 在沒發現任何副作用後就把它
+移除了。你不必總是這樣做，儘管，這不必要的舉動會限制最佳化。
 
-在寫一個包含多條指令的單個內聯彙編語句時，把每條指令用引號分割而且各佔一行，
-除了最後一條指令外，在每個指令結尾加上 ``\n\t`` ，讓彙編輸出時可以正確地縮進
+在寫一個包含多條指令的單個行內組譯語句時，把每條指令用引號分割而且各佔一行，
+除了最後一條指令外，在每個指令結尾加上 ``\n\t`` ，讓組譯輸出時可以正確地縮排
 下一條指令：
 
 .. code-block:: c
@@ -1004,21 +1012,21 @@ Vim 能夠解析這樣的標記：
 21) 條件編譯
 ------------
 
-只要可能，就不要在 .c 文件裏面使用預處理條件 (#if, #ifdef)；這樣做會讓代碼更難
-閱讀並且更難去跟蹤邏輯。替代方案是，在頭文件中用預處理條件提供給那些 .c 文件
-使用，再給 #else 提供一個空樁 (no-op stub) 版本，然後在 .c 文件內無條件地調用
-那些 (定義在頭文件內的) 函數。這樣做，編譯器會避免爲樁函數 (stub) 的調用生成
-任何代碼，產生的結果是相同的，但邏輯將更加清晰。
+只要可能，就不要在 .c 檔案裡面使用預處理條件 (#if, #ifdef)；這樣做會讓程式
+碼更難閱讀並且更難去追蹤邏輯。替代方案是，在標頭檔中用預處理條件提供給那些
+.c 檔案使用，再給 #else 提供一個空樁 (no-op stub) 版本，然後在 .c 檔案內無
+條件地呼叫那些 (定義在標頭檔內的) 函式。這樣做，編譯器會避免為樁函式 (stub
+) 的呼叫產生任何程式碼，產生的結果是相同的，但邏輯將更加清晰。
 
-最好傾向於編譯整個函數，而不是函數的一部分或表達式的一部分。與其放一個 ifdef
-在表達式內，不如分解出部分或全部表達式，放進一個單獨的輔助函數，並應用預處理
-條件到這個輔助函數內。
+最好傾向於編譯整個函式，而不是函式的一部分或表達式的一部分。與其放一個 ifdef
+在表達式內，不如分解出部分或全部表達式，放進一個單獨的輔助函式，並應用預處理
+條件到這個輔助函式內。
 
-如果你有一個在特定配置中，可能變成未使用的函數或變量，編譯器會警告它定義了但
-未使用，請把它標記爲 __maybe_unused 而不是將它包含在一個預處理條件中。(然而，
-如果一個函數或變量總是未使用，就直接刪除它。)
+如果你有一個在特定設定中，可能變成未使用的函式或變數，編譯器會警告它定義了但
+未使用，請把它標記為 __maybe_unused 而不是將它包含在一個預處理條件中。(然而，
+如果一個函式或變數總是未使用，就直接刪除它。)
 
-在代碼中，儘可能地使用 IS_ENABLED 宏來轉化某個 Kconfig 標記爲 C 的布爾
+在程式碼中，儘可能地使用 IS_ENABLED 巨集來轉化某個 Kconfig 標記為 C 的布林
 表達式，並在一般的 C 條件中使用它：
 
 .. code-block:: c
@@ -1027,13 +1035,13 @@ Vim 能夠解析這樣的標記：
 		...
 	}
 
-編譯器會做常量摺疊，然後就像使用 #ifdef 那樣去包含或排除代碼塊，所以這不會帶
-來任何運行時開銷。然而，這種方法依舊允許 C 編譯器查看塊內的代碼，並檢查它的正
-確性 (語法，類型，符號引用，等等)。因此，如果條件不滿足，代碼塊內的引用符號就
-不存在時，你還是必須去用 #ifdef。
+編譯器會做常數摺疊，然後就像使用 #ifdef 那樣去包含或排除程式碼塊，所以這不
+會帶來任何執行時開銷。然而，這種方法依舊允許 C 編譯器查看塊內的程式碼，並
+檢查它的正確性 (語法，類型，符號引用，等等)。因此，如果條件不滿足，程式碼
+塊內的引用符號就不存在時，你還是必須去用 #ifdef。
 
 在任何有意義的 #if 或 #ifdef 塊的末尾 (超過幾行的)，在 #endif 同一行的後面寫下
-註解，註釋這個條件表達式。例如：
+註解，註解這個條件表達式。例如：
 
 .. code-block:: c
 
@@ -1052,7 +1060,7 @@ ISBN 0-13-110362-8 (平裝), 0-13-110370-9 (精裝).
 
 .. note::
 
-    《C程序設計語言（第2版）》
+    《C程式設計語言（第2版）》
     作者：[美] Brian W. Kernighan / [美] Dennis M. Ritchie
     譯者：徐寶文 / 李志 / 尤晉元（審校）
     出版社：機械工業出版社，2019
@@ -1065,23 +1073,23 @@ ISBN 0-201-61586-X.
 
 .. note::
 
-    《程序設計實踐》
+    《程式設計實踐》
     作者：[美] Brian W. Kernighan / [美] Rob Pike
     出版社：機械工業出版社，2005
     ISBN：9787111091578
 
-    《程序設計實踐》
+    《程式設計實踐》
     作者：[美] Brian W. Kernighan / Rob Pike
     譯者：裘宗燕
     出版社：機械工業出版社，2000
     ISBN：9787111075738
 
-GNU 手冊 - 遵循 K&R 標準和此文本 - cpp, gcc, gcc internals and indent,
+GNU 手冊 - 遵循 K&R 標準和此文字 - cpp, gcc, gcc internals and indent,
 都可以從 https://www.gnu.org/manual/ 找到
 
 WG14 是 C 語言的國際標準化工作組，URL: http://www.open-std.org/JTC1/SC22/WG14/
 
-內核文檔 Documentation/process/coding-style.rst，
+核心文件 Documentation/process/coding-style.rst，
 作者 greg@kroah.com 發表於 OLS 2002：
 http://www.kroah.com/linux/talks/ols_2002_kernel_codingstyle_talk/html/
 
diff --git a/Documentation/translations/zh_TW/process/email-clients.rst b/Documentation/translations/zh_TW/process/email-clients.rst
index 4543c447d797aa..4fae2fdbb3bf84 100644
--- a/Documentation/translations/zh_TW/process/email-clients.rst
+++ b/Documentation/translations/zh_TW/process/email-clients.rst
@@ -16,87 +16,89 @@
  - Xiaochen Wang <wangxiaochen0@gmail.com>
  - yaxinsn <yaxinsn@163.com>
  - Hu Haowen <2023002089@link.tyut.edu.cn>
+ - Chen-Yu Yeh <chenyou910331@gmail.com>
 
-Linux郵件客戶端配置信息
+Linux郵件客戶端設定資訊
 =======================
 
 Git
 ---
 
 現在大多數開發人員使用 ``git send-email`` 而不是常規的電子郵件客戶端。這方面
-的手冊非常好。在接收端，維護人員使用 ``git am`` 加載補丁。
+的手冊非常好。在接收端，維護人員使用 ``git am`` 載入補丁。
 
-如果你是 ``git`` 新手，那麼把你的第一個補丁發送給你自己。將其保存爲包含所有
-標題的原始文本。運行 ``git am raw_email.txt`` ，然後使用 ``git log`` 查看更
+如果你是 ``git`` 新手，那麼把你的第一個補丁發送給你自己。將其保存為包含所有
+標題的原始文字。執行 ``git am raw_email.txt`` ，然後使用 ``git log`` 查看更
 改日誌。如果工作正常，再將補丁發送到相應的郵件列表。
 
 
-通用配置
+通用設定
 --------
 
-Linux內核補丁是通過郵件被提交的，最好把補丁作爲郵件體的內嵌文本。有些維護者
+Linux核心補丁是透過郵件被提交的，最好把補丁作為郵件體的內嵌文字。有些維護者
 接收附件，但是附件的內容格式應該是"text/plain"。然而，附件一般是不贊成的，
-因爲這會使補丁的引用部分在評論過程中變的很困難。
+因為這會使補丁的引用部分在評論過程中變得很困難。
 
-同時也強烈建議在補丁或其他郵件的正文中使用純文本格式。https://useplaintext.email
-有助於瞭解如何配置你喜歡的郵件客戶端，並在您還沒有首選的情況下列出一些推薦的
+同時也強烈建議在補丁或其他郵件的正文中使用純文字格式。https://useplaintext.email
+有助於瞭解如何設定你喜歡的郵件客戶端，並在您還沒有首選的情況下列出一些推薦的
 客戶端。
 
-用來發送Linux內核補丁的郵件客戶端在發送補丁時應該處於文本的原始狀態。例如，
+用來發送Linux核心補丁的郵件客戶端在發送補丁時應該處於文字的原始狀態。例如，
 他們不能改變或者刪除製表符或者空格，甚至是在每一行的開頭或者結尾。
 
-不要通過"format=flowed"模式發送補丁。這樣會引起不可預期以及有害的斷行。
+不要透過"format=flowed"模式發送補丁。這樣會引起不可預期以及有害的斷行。
 
 不要讓你的郵件客戶端進行自動換行。這樣也會破壞你的補丁。
 
-郵件客戶端不能改變文本的字符集編碼方式。要發送的補丁只能是ASCII或者UTF-8編碼
-方式，如果你使用UTF-8編碼方式發送郵件，那麼你將會避免一些可能發生的字符集問題。
+郵件客戶端不能改變文字的字元集編碼方式。要發送的補丁只能是ASCII或者UTF-8編
+碼方式，如果你使用UTF-8編碼方式發送郵件，那麼你將會避免一些可能發生的字元
+集問題。
 
-郵件客戶端應該生成並且保持“References:”或者“In-Reply-To:”郵件頭，這樣郵件會話
+郵件客戶端應該產生並且保持“References:”或者“In-Reply-To:”郵件頭，這樣郵件會話
 就不會中斷。
 
-複製粘帖(或者剪貼粘帖)通常不能用於補丁，因爲製表符會轉換爲空格。使用xclipboard,
-xclip或者xcutsel也許可以，但是最好測試一下或者避免使用複製粘帖。
+複製貼上(或者剪貼貼上)通常不能用於補丁，因為製表符會轉換為空格。使用
+xclipboard,xclip或者xcutsel也許可以，但是最好測試一下或者避免使用複製貼上。
 
 不要在使用PGP/GPG簽名的郵件中包含補丁。這樣會使得很多腳本不能讀取和適用於你的
 補丁。（這個問題應該是可以修復的）
 
-在給內核郵件列表發送補丁之前，給自己發送一個補丁是個不錯的主意，保存接收到的
-郵件，將補丁用'patch'命令打上，如果成功了，再給內核郵件列表發送。
+在給核心郵件列表發送補丁之前，給自己發送一個補丁是個不錯的主意，保存接收到的
+郵件，將補丁用'patch'命令打上，如果成功了，再給核心郵件列表發送。
 
 
 一些郵件客戶端提示
 ------------------
 
-這裏給出一些詳細的MUA配置提示，可以用於給Linux內核發送補丁。這些並不意味是
-所有的軟件包配置總結。
+這裡給出一些詳細的MUA設定提示，可以用於給Linux核心發送補丁。這些並不意味是
+所有的軟體套件設定總結。
 
 說明：
 
-- TUI = 以文本爲基礎的用戶接口
-- GUI = 圖形界面用戶接口
+- TUI = 以文字為基礎的使用者介面
+- GUI = 圖形介面使用者介面
 
 Alpine (TUI)
 ************
 
-配置選項：
+設定選項：
 
-在 :menuselection:`Sending Preferences` 菜單：
+在 :menuselection:`Sending Preferences` 選單：
 
-- :menuselection:`Do Not Send Flowed Text` 必須開啓
+- :menuselection:`Do Not Send Flowed Text` 必須開啟
 - :menuselection:`Strip Whitespace Before Sending` 必須關閉
 
-當寫郵件時，光標應該放在補丁會出現的地方，然後按下 `CTRL-R` 組合鍵，使指
-定的補丁文件嵌入到郵件中。
+當寫郵件時，游標應該放在補丁會出現的地方，然後按下 `CTRL-R` 組合鍵，使指
+定的補丁檔案嵌入到郵件中。
 
 Claws Mail (GUI)
 ****************
 
 可以用，有人用它成功地發過補丁。
 
-用 :menuselection:`Message-->Insert File` (`CTRL-I`) 或外置編輯器插入補丁。
+用 :menuselection:`Message-->Insert File` (`CTRL-I`) 或外部編輯器插入補丁。
 
-若要在Claws編輯窗口重修改插入的補丁，需關閉
+若要在Claws編輯視窗重修改插入的補丁，需關閉
 :menuselection:`Configuration-->Preferences-->Compose-->Wrapping`
 的 `Auto wrapping` 。
 
@@ -107,49 +109,49 @@ Evolution (GUI)
 
 撰寫郵件時：
 從 :menuselection:`格式-->段落樣式-->預格式化` (`CTRL-7`)
-或工具欄選擇 :menuselection:`預格式化` ；
+或工具列選擇 :menuselection:`預格式化` ；
 
 然後使用：
-:menuselection:`插入-->文本文件...` (`ALT-N x`) 插入補丁文件。
+:menuselection:`插入-->文字檔...` (`ALT-N x`) 插入補丁檔案。
 
 你還可以 ``diff -Nru old.c new.c | xclip`` ，選擇 :menuselection:`預格式化` ，
-然後使用鼠標中鍵進行粘帖。
+然後使用滑鼠中鍵進行貼上。
 
 Kmail (GUI)
 ***********
 
 一些開發者成功的使用它發送補丁。
 
-默認撰寫設置禁用HTML格式是合適的；不要啓用它。
+預設撰寫設定禁用HTML格式是合適的；不要啟用它。
 
 當書寫一封郵件的時候，在選項下面不要選擇自動換行。唯一的缺點就是你在郵件中輸
-入的任何文本都不會被自動換行，因此你必須在發送補丁之前手動換行。最簡單的方法
-就是啓用自動換行來書寫郵件，然後把它保存爲草稿。一旦你在草稿中再次打開它，它
+入的任何文字都不會被自動換行，因此你必須在發送補丁之前手動換行。最簡單的方法
+就是啟用自動換行來書寫郵件，然後把它保存為草稿。一旦你在草稿中再次開啟它，它
 已經全部自動換行了，那麼你的郵件雖然沒有選擇自動換行，但是還不會失去已有的自
 動換行。
 
-在郵件的底部，插入補丁之前，放上常用的補丁定界符：三個連字符(``---``)。
+在郵件的底部，插入補丁之前，放上常用的補丁定界符：三個連字元(``---``)。
 
-然後在 :menuselection:`信件` 菜單，選擇 :menuselection:`插入文本文件` ，接
-着選取你的補丁文件。還有一個額外的選項，你可以通過它配置你的創建新郵件工具欄，
-加上 :menuselection:`插入文本文件` 圖標。
+然後在 :menuselection:`信件` 選單，選擇 :menuselection:`插入文字檔` ，接
+著選取你的補丁檔案。還有一個額外的選項，你可以透過它設定你的建立新郵件工具列，
+加上 :menuselection:`插入文字檔` 圖示。
 
-將編輯器窗口拉到足夠寬避免折行。對於KMail 1.13.5 (KDE 4.5.4)，它會在發送郵件
-時對編輯器窗口中顯示折行的地方自動換行。在選項菜單中取消自動換行仍不能解決。
-因此，如果你的補丁中有非常長的行，必須在發送之前把編輯器窗口拉得非常寬。
+將編輯器視窗拉到足夠寬避免折行。對於KMail 1.13.5 (KDE 4.5.4)，它會在發送郵件
+時對編輯器視窗中顯示折行的地方自動換行。在選項選單中取消自動換行仍不能解決。
+因此，如果你的補丁中有非常長的行，必須在發送之前把編輯器視窗拉得非常寬。
 參見：https://bugs.kde.org/show_bug.cgi?id=174034
 
-你可以安全地用GPG簽名附件，但是內嵌補丁最好不要使用GPG簽名它們。作爲內嵌文本
+你可以安全地用GPG簽名附件，但是內嵌補丁最好不要使用GPG簽名它們。作為內嵌文字
 插入的簽名補丁將使其難以從7-bit編碼中提取。
 
 如果你非要以附件的形式發送補丁，那麼就右鍵點擊附件，然後選擇
-:menuselection:`屬性` ，打開 :menuselection:`建議自動顯示` ，使附件內聯更容
+:menuselection:`屬性` ，開啟 :menuselection:`建議自動顯示` ，使附件內聯更容
 易讓讀者看到。
 
-當你要保存將要發送的內嵌文本補丁，你可以從消息列表窗格選擇包含補丁的郵件，然
-後右鍵選擇 :menuselection:`另存爲` 。如果整個電子郵件的組成正確，您可直接將
-其作爲補丁使用。電子郵件以當前用戶可讀寫權限保存，因此您必須 ``chmod`` ，以
-使其在複製到別處時用戶組和其他人可讀。
+當你要保存將要發送的內嵌文字補丁，你可以從訊息列表窗格選擇包含補丁的郵件，然
+後右鍵選擇 :menuselection:`另存新檔` 。如果整個電子郵件的組成正確，您可直接將
+其作為補丁使用。電子郵件以當前使用者可讀寫權限保存，因此您必須 ``chmod`` ，以
+使其在複製到別處時使用者組和其他人可讀。
 
 Lotus Notes (GUI)
 *****************
@@ -167,9 +169,9 @@ Mutt (TUI)
 很多Linux開發人員使用mutt客戶端，這證明它肯定工作得非常漂亮。
 
 Mutt不自帶編輯器，所以不管你使用什麼編輯器，不自動斷行就行。大多數編輯器都有
-:menuselection:`插入文件` 選項，它可以在不改變文件內容的情況下插入文件。
+:menuselection:`插入檔案` 選項，它可以在不改變檔案內容的情況下插入檔案。
 
-用 ``vim`` 作爲mutt的編輯器::
+用 ``vim`` 作為mutt的編輯器::
 
   set editor="vi"
 
@@ -181,21 +183,21 @@ Mutt不自帶編輯器，所以不管你使用什麼編輯器，不自動斷行
 
   :r filename
 
-把補丁插入爲內嵌文本。
-在未設置  ``set paste`` 時(a)ttach工作的很好。
+把補丁插入為內嵌文字。
+在未設定  ``set paste`` 時(a)ttach工作的很好。
 
-你可以通過 ``git format-patch`` 生成補丁，然後用 Mutt發送它們::
+你可以透過 ``git format-patch`` 產生補丁，然後用 Mutt發送它們::
 
     $ mutt -H 0001-some-bug-fix.patch
 
-配置選項：
+設定選項：
 
-它應該以默認設置的形式工作。
-然而，把 ``send_charset`` 設置一下也是一個不錯的主意::
+它應該以預設設定的形式工作。
+然而，把 ``send_charset`` 設定一下也是一個不錯的主意::
 
   set send_charset="us-ascii:utf-8"
 
-Mutt 是高度可配置的。 這裏是個使用mutt通過 Gmail 發送的補丁的最小配置::
+Mutt 是高度可設定的。 這裡是個使用mutt透過 Gmail 發送的補丁的最小設定::
 
   # .muttrc
   # ================  IMAP ====================
@@ -222,7 +224,7 @@ Mutt 是高度可配置的。 這裏是個使用mutt通過 Gmail 發送的補丁
   set from = "username@gmail.com"
   set use_from = yes
 
-Mutt文檔含有更多信息：
+Mutt文件含有更多資訊：
 
     https://gitlab.com/muttmua/mutt/-/wikis/UseCases/Gmail
 
@@ -235,7 +237,7 @@ Pine過去有一些空格刪減問題，但是這些現在應該都被修復了
 
 如果可以，請使用alpine（pine的繼承者）。
 
-配置選項：
+設定選項：
 
 - 最近的版本需要 ``quell-flowed-text``
 - ``no-strip-whitespace-before-send`` 選項也是需要的。
@@ -244,23 +246,23 @@ Pine過去有一些空格刪減問題，但是這些現在應該都被修復了
 Sylpheed (GUI)
 **************
 
-- 內嵌文本可以很好的工作（或者使用附件）。
+- 內嵌文字可以很好的工作（或者使用附件）。
 - 允許使用外部的編輯器。
 - 收件箱較多時非常慢。
-- 如果通過non-SSL連接，無法使用TLS SMTP授權。
-- 撰寫窗口的標尺很有用。
+- 如果透過non-SSL連接，無法使用TLS SMTP授權。
+- 撰寫視窗的標尺很有用。
 - 將地址添加到通訊簿時無法正確理解顯示的名稱。
 
 Thunderbird (GUI)
 *****************
 
-Thunderbird是Outlook的克隆版本，它很容易損壞文本，但也有一些方法強制修正。
+Thunderbird是Outlook的翻版，它很容易損壞文字，但也有一些方法強制修正。
 
-在完成修改後（包括安裝擴展），您需要重新啓動Thunderbird。
+在完成修改後（包括安裝擴展），您需要重新啟動Thunderbird。
 
 - 允許使用外部編輯器：
 
-  使用Thunderbird發補丁最簡單的方法是使用擴展來打開您最喜歡的外部編輯器。
+  使用Thunderbird發補丁最簡單的方法是使用擴展來開啟您最喜歡的外部編輯器。
 
   下面是一些能夠做到這一點的擴展樣例。
 
@@ -270,43 +272,50 @@ Thunderbird是Outlook的克隆版本，它很容易損壞文本，但也有一
 
     https://addons.thunderbird.net/en-GB/thunderbird/addon/external-editor-revived/
 
-    它需要安裝“本地消息主機（native messaging host）”。
-    參見以下文檔:
+    它需要安裝“本地訊息主機（native messaging host）”。
+    參見以下文件:
     https://github.com/Frederick888/external-editor-revived/wiki
 
   - “External Editor”
 
     https://github.com/exteditor/exteditor
 
-    下載並安裝此擴展，然後打開 :menuselection:`新建消息` 窗口, 用
-    :menuselection:`查看-->工具欄-->自定義...` 給它增加一個按鈕，直接點擊此
-    按鈕即可使用外置編輯器。
+    下載並安裝此擴展，然後開啟 :menuselection:`新增訊息` 視窗, 用
+    :menuselection:`查看-->工具列-->自訂...` 給它增加一個按鈕，直接點擊此
+    按鈕即可使用外部編輯器。
 
     請注意，“External Editor”要求你的編輯器不能fork，換句話說，編輯器必須在
-    關閉前不返回。你可能需要傳遞額外的參數或修改編輯器設置。最值得注意的是，
-    如果您使用的是gvim，那麼您必須將 :menuselection:`external editor` 設置的
-    編輯器字段設置爲 ``/usr/bin/gvim --nofork"`` （假設可執行文件在
+    關閉前不返回。你可能需要傳遞額外的參數或修改編輯器設定。最值得注意的是，
+    如果您使用的是gvim，那麼您必須將 :menuselection:`external editor` 設定的
+    編輯器欄位設定為 ``/usr/bin/gvim --nofork"`` （假設可執行檔在
     ``/usr/bin`` ），以傳遞 ``-f`` 參數。如果您正在使用其他編輯器，請閱讀其
     手冊瞭解如何處理。
 
 若要修正內部編輯器，請執行以下操作：
 
-- 修改你的Thunderbird設置，不要使用 ``format=flowed`` ！
-  回到主窗口，按照
-  :menuselection:`主菜單-->首選項-->常規-->配置編輯器...`
-  打開Thunderbird的配置編輯器。
+- 修改你的Thunderbird設定，不要使用 ``format=flowed`` ！
+  回到主視窗，按照
+  :menuselection:`主選單-->偏好設定-->常規-->設定編輯器...`
+  開啟Thunderbird的設定編輯器。
 
-  - 將 ``mailnews.send_plaintext_flowed`` 設爲 ``false``
+  - 將 ``mailnews.send_plaintext_flowed`` 設為 ``false``
 
-  - 將 ``mailnews.wraplength`` 從 ``72`` 改爲 ``0``
+  - 將 ``mailnews.wraplength`` 從 ``72`` 改為 ``0`` ，**或者** 安裝
+    “Toggle Line Wrap”擴展
+
+    https://github.com/jan-kiszka/togglelinewrap
+
+    https://addons.thunderbird.net/thunderbird/addon/toggle-line-wrap
+
+    以便即時控制此設定項。
 
 - 不要寫HTML郵件！
-  回到主窗口，打開
-  :menuselection:`主菜單-->賬戶設置-->你的@郵件.地址-->通訊錄/編寫&地址簿` ，
-  關掉 ``以HTML格式編寫消息`` 。
+  回到主視窗，開啟
+  :menuselection:`主選單-->帳戶設定-->你的@郵件.地址-->通訊錄/編寫&地址簿` ，
+  關掉 ``以HTML格式編寫訊息`` 。
 
-- 只用純文本格式查看郵件！
-  回到主窗口， :menuselection:`主菜單-->查看-->消息體爲-->純文本` ！
+- 只用純文字格式查看郵件！
+  回到主視窗， :menuselection:`主選單-->查看-->訊息體為-->純文字` ！
 
 TkRat (GUI)
 ***********
@@ -318,12 +327,12 @@ Gmail (Web GUI)
 
 不要使用它發送補丁。
 
-Gmail網頁客戶端自動地把製表符轉換爲空格。
+Gmail網頁客戶端自動地把製表符轉換為空格。
 
-雖然製表符轉換爲空格問題可以被外部編輯器解決，但它同時還會使用回車換行把每行
-拆分爲78個字符。
+雖然製表符轉換為空格問題可以被外部編輯器解決，但它同時還會使用回車換行把每行
+拆分為78個字元。
 
-另一個問題是Gmail還會把任何含有非ASCII的字符的消息改用base64編碼，如歐洲人的
+另一個問題是Gmail還會把任何含有非ASCII的字元的訊息改用base64編碼，如歐洲人的
 名字。
 
 
diff --git a/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst b/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst
index 93d21fd8891003..611247c816f42b 100644
--- a/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst
+++ b/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst
@@ -5,115 +5,119 @@
 :Original: :ref:`Documentation/process/embargoed-hardware-issues.rst <embargoed_hardware_issues>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
              Hu Haowen <2023002089@link.tyut.edu.cn>
+             Chen-Yu Yeh <chenyou910331@gmail.com>
 
-被限制的硬件問題
+被限制的硬體問題
 ================
 
 範圍
 ----
 
-導致安全問題的硬件問題與隻影響Linux內核的純軟件錯誤是不同的安全錯誤類別。
+導致安全問題的硬體問題與只影響Linux核心的純軟體錯誤是不同的安全錯誤類別。
 
-必須區別對待諸如熔燬(Meltdown)、Spectre、L1TF等硬件問題，因爲它們通常會影響
-所有操作系統（“OS”），因此需要在不同的OS供應商、發行版、硬件供應商和其他各方
-之間進行協調。對於某些問題，軟件緩解可能依賴於微碼或固件更新，這需要進一步的
-協調。
+必須區別對待諸如熔毀(Meltdown)、Spectre、L1TF等硬體問題，因為它們通常會影
+響所有作業系統（“OS”），因此需要在不同的OS供應商、發行版、晶片供應商、硬體
+整合商和其他各方之間進行協調。對於某些問題，軟體緩解可能依賴於微碼或韌體更
+新，這需要進一步的協調。
 
 .. _tw_Contact:
 
 接觸
 ----
 
-Linux內核硬件安全小組獨立於普通的Linux內核安全小組。
+Linux核心硬體安全小組獨立於普通的Linux核心安全小組。
 
-該小組只負責協調被限制的硬件安全問題。Linux內核中純軟件安全漏洞的報告不由該
-小組處理，報告者將被引導至常規Linux內核安全小組(:ref:`Documentation/admin-guide/
-<securitybugs>`)聯繫。
+該小組只負責協調被限制的硬體安全問題。Linux核心中純軟體安全漏洞的報告不由
+該小組處理，報告者將被引導至常規Linux核心安全小組(
+:ref:`Documentation/admin-guide/<securitybugs>`)聯繫。
 
-可以通過電子郵件 <hardware-security@kernel.org> 與小組聯繫。這是一份私密的安全
-官名單，他們將幫助您根據我們的文檔化流程協調問題。
+可以透過電子郵件 <hardware-security@kernel.org> 與小組聯繫。這是一份私密的安全
+官名單，他們將幫助您根據我們的文件化流程協調問題。
 
-郵件列表是加密的，發送到列表的電子郵件可以通過PGP或S/MIME加密，並且必須使用報告
-者的PGP密鑰或S/MIME證書籤名。該列表的PGP密鑰和S/MIME證書可從
-https://www.kernel.org/.... 獲得。
+郵件列表是加密的，發送到列表的電子郵件可以透過PGP或S/MIME加密，並且必須使
+用報告者的PGP密鑰或S/MIME證書籤名。該列表的PGP密鑰和S/MIME證書可從以下URL
+獲得：
 
-雖然硬件安全問題通常由受影響的硬件供應商處理，但我們歡迎發現潛在硬件缺陷的研究
+  - PGP: https://www.kernel.org/static/files/hardware-security.asc
+  - S/MIME: https://www.kernel.org/static/files/hardware-security.crt
+
+雖然硬體安全問題通常由受影響的晶片供應商處理，但我們歡迎發現潛在硬體缺陷的研究
 人員或個人與我們聯繫。
 
-硬件安全官
+硬體安全官
 ^^^^^^^^^^
 
-目前的硬件安全官小組:
+目前的硬體安全官小組:
 
   - Linus Torvalds（Linux基金會院士）
-  - Greg Kroah Hartman（Linux基金會院士）
+  - Greg Kroah-Hartman（Linux基金會院士）
   - Thomas Gleixner（Linux基金會院士）
 
 郵件列表的操作
 ^^^^^^^^^^^^^^
 
-處理流程中使用的加密郵件列表託管在Linux Foundation的IT基礎設施上。通過提供這項
-服務，Linux基金會的IT基礎設施安全總監在技術上有能力訪問被限制的信息，但根據他
-的僱傭合同，他必須保密。Linux基金會的IT基礎設施安全總監還負責 kernel.org 基礎
-設施。
+處理流程中使用的加密郵件列表託管在Linux基金會的IT基礎設施上。透過提供這項
+服務，Linux基金會IT營運人員在技術上有能力存取被限制的資訊，但根據其僱傭
+合約必須保密。Linux基金會的IT人員還負責營運和管理kernel.org的其餘基礎設施。
 
-Linux基金會目前的IT基礎設施安全總監是 Konstantin Ryabitsev。
+Linux基金會目前的IT專案基礎設施總監是 Konstantin Ryabitsev。
 
 保密協議
 --------
 
-Linux內核硬件安全小組不是正式的機構，因此無法簽訂任何保密協議。核心社區意識到
+Linux核心硬體安全小組不是正式的機構，因此無法簽訂任何保密協議。核心社群意識到
 這些問題的敏感性，並提供了一份諒解備忘錄。
 
 諒解備忘錄
 ----------
 
-Linux內核社區深刻理解在不同操作系統供應商、發行商、硬件供應商和其他各方之間
-進行協調時，保持硬件安全問題處於限制狀態的要求。
+Linux核心社群深刻理解在不同作業系統供應商、發行商、晶片供應商和其他各方之間
+進行協調時，保持硬體安全問題處於限制狀態的要求。
 
-Linux內核社區在過去已經成功地處理了硬件安全問題，並且有必要的機制允許在限制
-限制下進行符合社區的開發。
+Linux核心社群在過去已經成功地處理了硬體安全問題，並且有必要的機制允許在限制
+限制下進行符合社群的開發。
 
-Linux內核社區有一個專門的硬件安全小組負責初始聯繫，並監督在限制規則下處理
+Linux核心社群有一個專門的硬體安全小組負責初始聯繫，並監督在限制規則下處理
 此類問題的過程。
 
-硬件安全小組確定開發人員（領域專家），他們將組成特定問題的初始響應小組。最初
+硬體安全小組確定開發人員（領域專家），他們將組成特定問題的初始響應小組。最初
 的響應小組可以引入更多的開發人員（領域專家）以最佳的技術方式解決這個問題。
 
-所有相關開發商承諾遵守限制規定，並對收到的信息保密。違反承諾將導致立即從當前
-問題中排除，並從所有相關郵件列表中刪除。此外，硬件安全小組還將把違反者排除在
-未來的問題之外。這一後果的影響在我們社區是一種非常有效的威懾。如果發生違規
-情況，硬件安全小組將立即通知相關方。如果您或任何人發現潛在的違規行爲，請立即
-向硬件安全人員報告。
+所有相關開發商承諾遵守限制規定，並對收到的資訊保密。違反承諾將導致立即從當前
+問題中排除，並從所有相關郵件列表中刪除。此外，硬體安全小組還將把違反者排除在
+未來的問題之外。這一後果的影響在我們社群是一種非常有效的威懾。如果發生違規
+情況，硬體安全小組將立即通知相關方。如果您或任何人發現潛在的違規行為，請立即
+向硬體安全人員報告。
 
 流程
 ^^^^
 
-由於Linux內核開發的全球分佈式特性，面對面的會議幾乎不可能解決硬件安全問題。
+由於Linux核心開發的全球分散式特性，面對面的會議幾乎不可能解決硬體安全問題。
 由於時區和其他因素，電話會議很難協調，只能在絕對必要時使用。加密電子郵件已被
 證明是解決此類問題的最有效和最安全的通信方法。
 
 開始披露
 """"""""
 
-披露內容首先通過電子郵件聯繫Linux內核硬件安全小組。此初始聯繫人應包含問題的
-描述和任何已知受影響硬件的列表。如果您的組織製造或分發受影響的硬件，我們建議
-您也考慮哪些其他硬件可能會受到影響。
+披露首先按照上述聯繫方式一節，透過電子郵件聯繫Linux核心硬體安全小組。此
+初始聯繫應包含問題的描述和任何已知受影響晶片的列表。如果您的組織製造或分發
+受影響的硬體，我們建議您也考慮哪些其他硬體可能會受到影響。披露方有責任及時
+聯繫受影響的晶片供應商。
 
-硬件安全小組將提供一個特定於事件的加密郵件列表，用於與報告者進行初步討論、
+硬體安全小組將提供一個特定於事件的加密郵件列表，用於與報告者進行初步討論、
 進一步披露和協調。
 
-硬件安全小組將向披露方提供一份開發人員（領域專家）名單，在與開發人員確認他們
-將遵守本諒解備忘錄和文件化流程後，應首先告知開發人員有關該問題的信息。這些開發
-人員組成初始響應小組，並在初始接觸後負責處理問題。硬件安全小組支持響應小組，
+硬體安全小組將向披露方提供一份開發人員（領域專家）名單，在與開發人員確認他們
+將遵守本諒解備忘錄和文件化流程後，應首先告知開發人員有關該問題的資訊。這些開發
+人員組成初始響應小組，並在初始接觸後負責處理問題。硬體安全小組支援響應小組，
 但不一定參與緩解開發過程。
 
-雖然個別開發人員可能通過其僱主受到保密協議的保護，但他們不能以Linux內核開發
-人員的身份簽訂個別保密協議。但是，他們將同意遵守這一書面程序和諒解備忘錄。
+雖然個別開發人員可能透過其僱主受到保密協議的保護，但他們不能以Linux核心開發
+人員的身份簽訂個別保密協議。但是，他們將同意遵守這一書面程式和諒解備忘錄。
 
 披露方應提供已經或應該被告知該問題的所有其他實體的聯繫人名單。這有幾個目的:
 
-  - 披露的實體列表允許跨行業通信，例如其他操作系統供應商、硬件供應商等。
+  - 披露的實體列表允許跨行業通信，例如其他作業系統供應商、硬體供應商等。
 
   - 可聯繫已披露的實體，指定應參與緩解措施開發的專家。
 
@@ -123,67 +127,97 @@ Linux內核社區有一個專門的硬件安全小組負責初始聯繫，並監
 披露
 """"
 
-披露方通過特定的加密郵件列表向初始響應小組提供詳細信息。
+披露方透過特定的加密郵件列表向初始響應小組提供詳細資訊。
 
-根據我們的經驗，這些問題的技術文檔通常是一個足夠的起點，最好通過電子郵件進行
+根據我們的經驗，這些問題的技術文件通常是一個足夠的起點，最好透過電子郵件進行
 進一步的技術澄清。
 
 緩解開發
 """"""""
 
-初始響應小組設置加密郵件列表，或在適當的情況下重新修改現有郵件列表。
+初始響應小組設定加密郵件列表，或在適當的情況下重新修改現有郵件列表。
 
-使用郵件列表接近於正常的Linux開發過程，並且在過去已經成功地用於爲各種硬件安全
+使用郵件列表接近於正常的Linux開發過程，並且在過去已經成功地用於為各種硬體安全
 問題開發緩解措施。
 
-郵件列表的操作方式與正常的Linux開發相同。發佈、討論和審查修補程序，如果同意，
-則應用於非公共git存儲庫，參與開發人員只能通過安全連接訪問該存儲庫。存儲庫包含
-針對主線內核的主開發分支，並根據需要爲穩定的內核版本提供向後移植分支。
+郵件列表的操作方式與正常的Linux開發相同。發布、討論和審查修補程式，如果同意，
+則應用於非公共git儲存庫，參與開發人員只能透過安全連接存取該儲存庫。儲存庫包含
+針對主線核心的主開發分支，並根據需要為穩定的核心版本提供向後移植分支。
+
+最初的響應小組將根據需要從Linux核心開發人員社群中確定更多的專家。任何相關
+方都可以建議增補專家，每位專家都須符合上述相同的要求。
 
-最初的響應小組將根據需要從Linux內核開發人員社區中確定更多的專家。引進專家可以
-在開發過程中的任何時候發生，需要及時處理。
+引進專家可以在開發過程中的任何時候發生，需要及時處理。
 
 如果專家受僱於披露方提供的披露清單上的實體或其成員，則相關實體將要求其參與。
 
 否則，披露方將被告知專家參與的情況。諒解備忘錄涵蓋了專家，要求披露方確認參與。
 如果披露方有令人信服的理由提出異議，則必須在五個工作日內提出異議，並立即與事件
-小組解決。如果披露方在五個工作日內未作出回應，則視爲默許。
+小組解決。如果披露方在五個工作日內未作出回應，則視為默許。
 
 在確認或解決異議後，專家由事件小組披露，並進入開發過程。
 
+列表參與者不得在私有郵件列表之外交流該問題。列表參與者在處理補丁時不得使用
+任何共享資源（例如僱主的建置農場、CI系統等）。
+
+提前取得
+""""""""
+
+在列表上討論和開發的補丁，既不能分發給任何非響應小組成員的個人，也不能分發
+給任何其他組織。
+
+為了讓受影響的晶片供應商能夠與其內部團隊和產業夥伴合作進行測試、驗證和
+後勤工作，提供了以下例外：
+
+	受影響晶片供應商的指定代表可以在任何時候將補丁移交給該晶片供應商
+	的響應小組。該代表必須將移交一事通知核心響應小組。受影響的晶片
+	供應商必須為與其響應小組共享的任何補丁，制定並維護自己的、與本
+	政策一致的文件化安全流程。
+
+	晶片供應商的響應小組可以依照該晶片供應商的文件化安全流程，將這些
+	補丁分發給其產業夥伴及其內部團隊。來自產業夥伴的反饋會回到晶片
+	供應商，並由晶片供應商轉達給核心響應小組。
+
+	移交給晶片供應商的響應小組後，因晶片供應商的內部團隊或產業夥伴的
+	參與而發生的過早披露，核心響應小組不再承擔任何責任或義務。晶片
+	供應商透過同意本流程來保證這一責任免除。
+
 協調發布
 """"""""
 
-有關各方將協商限制結束的日期和時間。此時，準備好的緩解措施集成到相關的內核樹中
+有關各方將協商限制結束的日期和時間。此時，準備好的緩解措施整合到相關的核心樹中
 併發布。
 
-雖然我們理解硬件安全問題需要協調限制時間，但限制時間應限制在所有有關各方制定、
-測試和準備緩解措施所需的最短時間內。人爲地延長限制時間以滿足會議討論日期或其他
-非技術原因，會給相關的開發人員和響應小組帶來了更多的工作和負擔，因爲補丁需要
-保持最新，以便跟蹤正在進行的上游內核開發，這可能會造成衝突的更改。
+雖然我們理解硬體安全問題需要協調限制時間，但限制時間應限制在所有有關各方制定、
+測試和準備緩解措施所需的最短時間內。人為地延長限制時間以滿足會議討論日期或其他
+非技術原因，會給相關的開發人員和響應小組帶來了更多的工作和負擔，因為補丁需要
+保持最新，以便追蹤正在進行的上游核心開發，這可能會造成衝突的更改。
 
 CVE分配
 """""""
 
-硬件安全小組和初始響應小組都不分配CVE，開發過程也不需要CVE。如果CVE是由披露方
-提供的，則可用於文檔中。
+硬體安全小組和初始響應小組都不分配CVE，開發過程也不需要CVE。如果CVE是由披露方
+提供的，則可用於文件記錄目的。
 
 流程專使
 --------
 
-爲了協助這一進程，我們在各組織設立了專使，他們可以回答有關報告流程和進一步處理
+為了協助這一流程，我們在各組織設立了專使，他們可以回答有關報告流程和進一步處理
 的問題或提供指導。專使不參與特定問題的披露，除非響應小組或相關披露方提出要求。
 現任專使名單:
 
   ============= ========================================================
-  ARM
   AMD		Tom Lendacky <thomas.lendacky@amd.com>
-  IBM
+  Ampere	Darren Hart <darren@os.amperecomputing.com>
+  ARM		Catalin Marinas <catalin.marinas@arm.com>
+  IBM Power	Madhavan Srinivasan <maddy@linux.ibm.com>
+  IBM Z		Christian Borntraeger <borntraeger@de.ibm.com>
   Intel		Tony Luck <tony.luck@intel.com>
   Qualcomm	Trilok Soni <quic_tsoni@quicinc.com>
+  RISC-V	Palmer Dabbelt <palmer@dabbelt.com>
+  Samsung	Javier González <javier.gonz@samsung.com>
 
-  Microsoft	Sasha Levin <sashal@kernel.org>
-  VMware
+  Microsoft	James Morris <jamorris@linux.microsoft.com>
   Xen		Andrew Cooper <andrew.cooper3@citrix.com>
 
   Canonical	John Johansen <john.johansen@canonical.com>
@@ -192,27 +226,28 @@ CVE分配
   Red Hat	Josh Poimboeuf <jpoimboe@redhat.com>
   SUSE		Jiri Kosina <jkosina@suse.cz>
 
-  Amazon
   Google	Kees Cook <keescook@chromium.org>
+
+  LLVM		Nick Desaulniers <ndesaulniers@google.com>
   ============= ========================================================
 
-如果要將您的組織添加到專使名單中，請與硬件安全小組聯繫。被提名的專使必須完全
-理解和支持我們的過程，並且在Linux內核社區中很容易聯繫。
+如果要將您的組織添加到專使名單中，請與硬體安全小組聯繫。被提名的專使必須完全
+理解和支援我們的過程，並且在Linux核心社群中很容易聯繫。
 
 加密郵件列表
 ------------
 
 我們使用加密郵件列表進行通信。這些列表的工作原理是，發送到列表的電子郵件使用
-列表的PGP密鑰或列表的/MIME證書進行加密。郵件列表軟件對電子郵件進行解密，並
-使用訂閱者的PGP密鑰或S/MIME證書爲每個訂閱者分別對其進行重新加密。有關郵件列表
-軟件和用於確保列表安全和數據保護的設置的詳細信息，請訪問:
-https://www.kernel.org/....
+列表的PGP密鑰或列表的S/MIME證書進行加密。郵件列表軟體對電子郵件進行解密，並
+使用訂閱者的PGP密鑰或S/MIME證書為每個訂閱者分別對其進行重新加密。有關郵件列表
+軟體和用於確保列表安全和資料保護的設定的詳細資訊，請見：
+https://korg.wiki.kernel.org/userdoc/remail.
 
-關鍵點
-^^^^^^
+列表密鑰
+^^^^^^^^
 
-初次接觸見 :ref:`zh_Contact`. 對於特定於事件的郵件列表，密鑰和S/MIME證書通過
-特定列表發送的電子郵件傳遞給訂閱者。
+初次接觸見上面的 :ref:`tw_Contact` 一節。 對於特定於事件的郵件列表，密鑰和
+S/MIME證書透過特定列表發送的電子郵件傳遞給訂閱者。
 
 訂閱事件特定列表
 ^^^^^^^^^^^^^^^^
@@ -220,13 +255,13 @@ https://www.kernel.org/....
 訂閱由響應小組處理。希望參與通信的披露方將潛在訂戶的列表發送給響應組，以便
 響應組可以驗證訂閱請求。
 
-每個訂戶都需要通過電子郵件向響應小組發送訂閱請求。電子郵件必須使用訂閱服務器
-的PGP密鑰或S/MIME證書籤名。如果使用PGP密鑰，則必須從公鑰服務器獲得該密鑰，
-並且理想情況下該密鑰連接到Linux內核的PGP信任網。另請參見:
+每個訂戶都需要透過電子郵件向響應小組發送訂閱請求。電子郵件必須使用訂閱者
+的PGP密鑰或S/MIME證書籤名。如果使用PGP密鑰，則必須從公鑰伺服器獲得該密鑰，
+並且理想情況下該密鑰連接到Linux核心的PGP信任網。另請參見:
 https://www.kernel.org/signature.html.
 
-響應小組驗證訂閱者，並將訂閱者添加到列表中。訂閱後，訂閱者將收到來自郵件列表
-的電子郵件，該郵件列表使用列表的PGP密鑰或列表的/MIME證書籤名。訂閱者的電子郵件
-客戶端可以從簽名中提取PGP密鑰或S/MIME證書，以便訂閱者可以向列表發送加密電子
-郵件。
+響應小組驗證訂閱者，並將訂閱者添加到列表中。訂閱後，訂閱者將收到來自郵件列
+表的電子郵件，該郵件列表使用列表的PGP密鑰或列表的S/MIME證書籤名。訂閱者的
+電子郵件客戶端可以從簽名中提取PGP密鑰或S/MIME證書，以便訂閱者可以向列表發
+送加密電子郵件。
 
diff --git a/Documentation/translations/zh_TW/process/howto.rst b/Documentation/translations/zh_TW/process/howto.rst
index 80c416483e7386..8466f6af96cc5e 100644
--- a/Documentation/translations/zh_TW/process/howto.rst
+++ b/Documentation/translations/zh_TW/process/howto.rst
@@ -17,13 +17,14 @@
                    陳琦  Maggie Chen <chenqi@beyondsoft.com>
                    王聰  Wang Cong <xiyou.wangcong@gmail.com>
                    胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+                   葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
-如何參與Linux內核開發
+如何參與Linux核心開發
 =====================
 
-這是一篇將如何參與Linux內核開發的相關問題一網打盡的終極祕笈。它將指導你
-成爲一名Linux內核開發者，並且學會如何同Linux內核開發社區合作。它儘可能不
-包括任何關於內核編程的技術細節，但會給你指引一條獲得這些知識的正確途徑。
+這是一篇將如何參與Linux核心開發的相關問題一網打盡的終極祕笈。它將指導你
+成為一名Linux核心開發者，並且學會如何同Linux核心開發社群合作。它儘可能不
+包括任何關於核心程式設計的技術細節，但會給你指引一條獲得這些知識的正確途徑。
 
 如果這篇文章中的任何內容不再適用，請給文末列出的文件維護者發送補丁。
 
@@ -31,15 +32,15 @@
 入門
 ----
 
-你想了解如何成爲一名Linux內核開發者？或者老闆吩咐你「給這個設備寫個Linux
-驅動程序」？這篇文章的目的就是教會你達成這些目標的全部訣竅，它將描述你需
-要經過的流程以及給出如何同內核社區合作的一些提示。它還將試圖解釋內核社區
-爲何這樣運作。
+你想了解如何成為一名Linux核心開發者？或者老闆吩咐你「給這個設備寫個Linux
+驅動程式」？這篇文章的目的就是教會你達成這些目標的全部訣竅，它將描述你需
+要經過的流程以及給出如何同核心社群合作的一些提示。它還將試圖解釋核心社群
+為何這樣運作。
 
-Linux內核大部分是由C語言寫成的，一些體系結構相關的代碼用到了彙編語言。要
-參與內核開發，你必須精通C語言。除非你想爲某個架構開發底層代碼，否則你並
-不需要了解（任何體系結構的）彙編語言。下面列舉的書籍雖然不能替代紮實的C
-語言教育和多年的開發經驗，但如果需要的話，做爲參考還是不錯的：
+Linux核心大部分是由C語言寫成的，一些體系結構相關的程式碼用到了組譯語言。要
+參與核心開發，你必須精通C語言。除非你想為某個架構開發底層程式碼，否則你並
+不需要了解（任何體系結構的）組譯語言。下面列舉的書籍雖然不能替代紮實的C
+語言教育和多年的開發經驗，但如果需要的話，做為參考還是不錯的：
 
  - "The C Programming Language" by Kernighan and Ritchie [Prentice Hall]
    《C程序設計語言（第2版·新版）》（徐寶文 李志 譯）[機械工業出版社]
@@ -48,67 +49,67 @@ Linux內核大部分是由C語言寫成的，一些體系結構相關的代碼
  - "C:  A Reference Manual" by Harbison and Steele [Prentice Hall]
    《C語言參考手冊（原書第5版）》（邱仲潘 等譯）[機械工業出版社]
 
-Linux內核使用GNU C和GNU工具鏈開發。雖然它遵循ISO C11標準，但也用到了一些
-標準中沒有定義的擴展。內核是自給自足的C環境，不依賴於標準C庫的支持，所以
-並不支持C標準中的部分定義。比如long long類型的大數除法和浮點運算就不允許
-使用。有時候確實很難弄清楚內核對工具鏈的要求和它所使用的擴展，不幸的是目
-前還沒有明確的參考資料可以解釋它們。請查閱gcc信息頁（使用「info gcc」命令
-顯示）獲得一些這方面信息。
+Linux核心使用GNU C和GNU工具鏈開發。雖然它遵循ISO C11標準，但也用到了一些
+標準中沒有定義的擴展。核心是自給自足的C環境，不依賴於標準C庫的支援，所以
+並不支援C標準中的部分定義。比如long long類型的大數除法和浮點運算就不允許
+使用。有時候確實很難弄清楚核心對工具鏈的要求和它所使用的擴展，不幸的是目
+前還沒有明確的參考資料可以解釋它們。請查閱gcc資訊頁（使用「info gcc」命令
+顯示）獲得一些這方面資訊。
 
-請記住你是在學習怎麼和已經存在的開發社區打交道。它由一羣形形色色的人組成，
-他們對代碼、風格和過程有著很高的標準。這些標準是在長期實踐中總結出來的，
+請記住你是在學習怎麼和已經存在的開發社群打交道。它由一羣形形色色的人組成，
+他們對程式碼、風格和過程有著很高的標準。這些標準是在長期實踐中總結出來的，
 適應於地理上分散的大型開發團隊。它們已經被很好得整理成檔，建議你在開發
-之前儘可能多的學習這些標準，而不要期望別人來適應你或者你公司的行爲方式。
+之前儘可能多的學習這些標準，而不要期望別人來適應你或者你公司的行為方式。
 
 
 法律問題
 --------
 
-Linux內核原始碼都是在GPL（通用公共許可證）的保護下發布的。要了解這種許可
-的細節請查看原始碼主目錄下的COPYING文件。Linux內核許可準則和如何使用
+Linux核心原始碼都是在GPL（通用公共許可證）的保護下發布的。要了解這種許可
+的細節請查看原始碼主目錄下的COPYING檔案。Linux核心許可準則和如何使用
 `SPDX <https://spdx.org/>` 標誌符說明在這個文件中
 :ref:`Documentation/translations/zh_TW/process/license-rules.rst <tw_kernel_licensing>`
-如果你對它還有更深入問題請聯繫律師，而不要在Linux內核郵件組上提問。因爲
+如果你對它還有更深入問題請聯繫律師，而不要在Linux核心郵件組上提問。因為
 郵件組裡的人並不是律師，不要期望他們的話有法律效力。
 
-對於GPL的常見問題和解答，請訪問以下連結：
+對於GPL的常見問題和解答，請造訪以下連結：
 	https://www.gnu.org/licenses/gpl-faq.html
 
 
-文檔
+文件
 ----
 
-Linux內核代碼中包含有大量的文檔。這些文檔對於學習如何與內核社區互動有著
-不可估量的價值。當一個新的功能被加入內核，最好把解釋如何使用這個功能的文
-檔也放進內核。當內核的改動導致面向用戶空間的接口發生變化時，最好將相關信
-息或手冊頁(manpages)的補丁發到mtk.manpages@gmail.com，以向手冊頁(manpages)
+Linux核心程式碼中包含有大量的文件。這些文件對於學習如何與核心社群互動有著
+不可估量的價值。當一個新的功能被加入核心，最好把解釋如何使用這個功能的文
+檔也放進核心。當核心的改動導致面向使用者空間的介面發生變化時，最好將相關信
+息或手冊頁(manpages)的補丁發到alx@kernel.org，以向手冊頁(manpages)
 的維護者解釋這些變化。
 
-以下是內核代碼中需要閱讀的文檔：
+以下是核心程式碼中需要閱讀的文件：
   :ref:`Documentation/admin-guide/README.rst <readme>`
-    文件簡要介紹了Linux內核的背景，並且描述了如何配置和編譯內核。內核的
-    新用戶應該從這裡開始。
+    文件簡要介紹了Linux核心的背景，並且描述了如何設定和編譯核心。核心的
+    新使用者應該從這裡開始。
 
 
   :ref:`Documentation/process/changes.rst <changes>`
-    文件給出了用來編譯和使用內核所需要的最小軟體包列表。
+    文件給出了用來編譯和使用核心所需要的最小軟體套件列表。
 
   :ref:`Documentation/translations/zh_TW/process/coding-style.rst <tw_codingstyle>`
-    描述Linux內核的代碼風格和理由。所有新代碼需要遵守這篇文檔中定義的規
+    描述Linux核心的程式碼風格和理由。所有新程式碼需要遵守這篇文件中定義的規
     范。大多數維護者只會接收符合規定的補丁，很多人也只會幫忙檢查符合風格
-    的代碼。
+    的程式碼。
 
   :ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
 
-    這兩份文檔明確描述如何創建和發送補丁，其中包括（但不僅限於)：
+    這兩份文件明確描述如何建立和發送補丁，其中包括（但不僅限於)：
        - 郵件內容
        - 郵件格式
        - 選擇收件人
 
-    遵守這些規定並不能保證提交成功（因爲所有補丁需要通過嚴格的內容和風格
+    遵守這些規定並不能保證提交成功（因為所有補丁需要透過嚴格的內容和風格
     審查），但是忽視他們幾乎就意味著失敗。
 
-    其他關於如何正確地生成補丁的優秀文檔包括：
+    其他關於如何正確地產生補丁的優秀文件包括：
     "The Perfect Patch"
 
         https://www.ozlabs.org/~akpm/stuff/tpp.txt
@@ -118,78 +119,78 @@ Linux內核代碼中包含有大量的文檔。這些文檔對於學習如何與
         https://web.archive.org/web/20180829112450/http://linux.yyz.us/patch-format.html
 
   :ref:`Documentation/translations/zh_TW/process/stable-api-nonsense.rst <tw_stable_api_nonsense>`
-    論證內核爲什麼特意不包括穩定的內核內部API，也就是說不包括像這樣的特
+    論證核心為什麼特意不包括穩定的核心內部API，也就是說不包括像這樣的特
     性：
 
-       - 子系統中間層（爲了兼容性？）
-       - 在不同作業系統間易於移植的驅動程序
-       - 減緩（甚至阻止）內核代碼的快速變化
+       - 子系統中間層（為了相容性？）
+       - 在不同作業系統間易於移植的驅動程式
+       - 減緩（甚至阻止）核心程式碼的快速變化
 
-    這篇文檔對於理解Linux的開發哲學至關重要。對於將開發平台從其他操作系
+    這篇文件對於理解Linux的開發哲學至關重要。對於將開發平台從其他操作系
     統轉移到Linux的人來說也很重要。
 
   :ref:`Documentation/process/security-bugs.rst <securitybugs>`
-    如果你認爲自己發現了Linux內核的安全性問題，請根據這篇文檔中的步驟來
-    提醒其他內核開發者並幫助解決這個問題。
+    如果你認為自己發現了Linux核心的安全性問題，請根據這篇文件中的步驟來
+    提醒其他核心開發者並幫助解決這個問題。
 
   :ref:`Documentation/translations/zh_TW/process/management-style.rst <tw_managementstyle>`
 
-    描述內核維護者的工作方法及其共有特點。這對於剛剛接觸內核開發（或者對
-    它感到好奇）的人來說很重要，因爲它解釋了很多對於內核維護者獨特行爲的
+    描述核心維護者的工作方法及其共有特點。這對於剛剛接觸核心開發（或者對
+    它感到好奇）的人來說很重要，因為它解釋了很多對於核心維護者獨特行為的
     普遍誤解與迷惑。
 
   :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
-    解釋了穩定版內核發布的規則，以及如何將改動放入這些版本的步驟。
+    解釋了穩定版核心發布的規則，以及如何將改動放入這些版本的步驟。
 
   :ref:`Documentation/process/kernel-docs.rst <kernel_docs>`
-    有助於內核開發的外部文檔列表。如果你在內核自帶的文檔中沒有找到你想找
-    的內容，可以查看這些文檔。
+    有助於核心開發的外部文件列表。如果你在核心自帶的文件中沒有找到你想找
+    的內容，可以查看這些文件。
 
   :ref:`Documentation/process/applying-patches.rst <applying_patches>`
-    關於補丁是什麼以及如何將它打在不同內核開發分支上的好介紹
+    關於補丁是什麼以及如何將它打在不同核心開發分支上的好介紹
 
-內核還擁有大量從代碼自動生成或者從 ReStructuredText(ReST) 標記生成的文檔，
-比如這個文檔，它包含內核內部API的全面介紹以及如何妥善處理加鎖的規則。所有
-這些文檔都可以通過運行以下命令從內核代碼中生成爲PDF或HTML文檔::
+核心還擁有大量從程式碼自動產生或者從 ReStructuredText(ReST) 標記產生的文件，
+比如這個文件，它包含核心內部API的全面介紹以及如何妥善處理加鎖的規則。所有
+這些文件都可以透過執行以下命令從核心程式碼中產生為PDF或HTML文件::
 
     make pdfdocs
     make htmldocs
 
-ReST格式的文檔會生成在 Documentation/output. 目錄中。
-它們也可以用下列命令生成 LaTeX 和 ePub 格式文檔::
+ReST格式的文件會產生在 Documentation/output. 目錄中。
+它們也可以用下列命令產生 LaTeX 和 ePub 格式文件::
 
     make latexdocs
     make epubdocs
 
-如何成爲內核開發者
+如何成為核心開發者
 ------------------
-如果你對Linux內核開發一無所知，你應該訪問「Linux內核新手」計劃：
+如果你對Linux核心開發一無所知，你應該造訪「Linux核心新手」計劃：
 
 	https://kernelnewbies.org
 
-它擁有一個可以問各種最基本的內核開發問題的郵件列表（在提問之前一定要記得
+它擁有一個可以問各種最基本的核心開發問題的郵件列表（在提問之前一定要記得
 查找已往的郵件，確認是否有人已經回答過相同的問題）。它還擁有一個可以獲得
-實時反饋的IRC聊天頻道，以及大量對於學習Linux內核開發相當有幫助的文檔。
+實時反饋的IRC聊天頻道，以及大量對於學習Linux核心開發相當有幫助的文件。
 
-網站簡要介紹了原始碼組織結構、子系統劃分以及目前正在進行的項目（包括內核
-中的和單獨維護的）。它還提供了一些基本的幫助信息，比如如何編譯內核和打補
-丁。
+網站簡要介紹了原始碼組織結構、子系統劃分以及目前正在進行的專案（包括核心
+中的和單獨維護的）。它還提供了一些基本的幫助資訊，比如如何編譯核心和打補丁
+。
 
-如果你想加入內核開發社區並協助完成一些任務，卻找不到從哪裡開始，可以訪問
-「Linux內核房管員」計劃：
+如果你想加入核心開發社群並協助完成一些任務，卻找不到從哪裡開始，可以造訪
+「Linux核心房管員」計劃：
 
 	https://kernelnewbies.org/KernelJanitors
 
-這是極佳的起點。它提供一個相對簡單的任務列表，列出內核代碼中需要被重新
-整理或者改正的地方。通過和負責這個計劃的開發者們一同工作，你會學到將補丁
-集成進內核的基本原理。如果還沒有決定下一步要做什麼的話，你還可能會得到方
+這是極佳的起點。它提供一個相對簡單的任務列表，列出核心程式碼中需要被重新
+整理或者改正的地方。透過和負責這個計劃的開發者們一同工作，你會學到將補丁
+整合進核心的基本原理。如果還沒有決定下一步要做什麼的話，你還可能會得到方
 向性的指點。
 
-在真正動手修改內核代碼之前，理解要修改的代碼如何運作是必需的。要達到這個
-目的，沒什麼辦法比直接讀代碼更有效了（大多數花招都會有相應的注釋），而且
-一些特製的工具還可以提供幫助。例如，「Linux代碼交叉引用」項目就是一個值得
+在真正動手修改核心程式碼之前，理解要修改的程式碼如何運作是必需的。要達到這個
+目的，沒什麼辦法比直接讀程式碼更有效了（大多數花招都會有相應的註解），而且
+一些特製的工具還可以提供幫助。例如，「Linux程式碼交叉引用」專案就是一個值得
 特別推薦的幫助工具，它將原始碼顯示在有編目和索引的網頁上。其中一個更新及
-時的內核源碼庫，可以通過以下地址訪問：
+時的核心源碼庫，可以透過以下地址造訪：
 
         https://elixir.bootlin.com/
 
@@ -197,158 +198,159 @@ ReST格式的文檔會生成在 Documentation/output. 目錄中。
 開發流程
 --------
 
-目前Linux內核開發流程包括幾個「主內核分支」和很多子系統相關的內核分支。這
+目前Linux核心開發流程包括幾個「主核心分支」和很多子系統相關的核心分支。這
 些分支包括：
 
-  - Linus 的內核源碼樹
-  - 多個主要版本的穩定版內核樹
-  - 子系統相關的內核樹
-  - linux-next 集成測試樹
+  - Linus 的核心源碼樹
+  - 多個主要版本的穩定版核心樹
+  - 子系統相關的核心樹
+  - linux-next 整合測試樹
 
 
 主線樹
 ------
-主線樹是由Linus Torvalds 維護的。你可以在https://kernel.org 網站或者代碼
+主線樹是由Linus Torvalds 維護的。你可以在https://kernel.org 網站或者程式碼
 庫中下找到它。它的開發遵循以下步驟：
 
-  - 每當一個新版本的內核被發布，爲期兩周的集成窗口將被打開。在這段時間裡
-    維護者可以向Linus提交大段的修改，通常這些修改已經被放到-mm內核中幾個
-    星期了。提交大量修改的首選方式是使用git工具（內核的代碼版本管理工具
-    ，更多的信息可以在 https://git-scm.com/ 獲取），不過使用普通補丁也是
+  - 每當一個新版本的核心被發布，為期兩周的整合視窗將被開啟。在這段時間裡
+    維護者可以向Linus提交大段的修改，通常這些修改已經被放到-mm核心中幾個
+    星期了。提交大量修改的首選方式是使用git工具（核心的程式碼版本管理工具
+    ，更多的資訊可以在 https://git-scm.com/ 獲取），不過使用普通補丁也是
     可以的。
-  - 兩個星期以後-rc1版本內核發布。之後只有不包含可能影響整個內核穩定性的
-    新功能的補丁才可能被接受。請注意一個全新的驅動程序（或者文件系統）有
-    可能在-rc1後被接受是因爲這樣的修改完全獨立，不會影響其他的代碼，所以
-    沒有造成內核退步的風險。在-rc1以後也可以用git向Linus提交補丁，不過所
+  - 兩個星期以後-rc1版本核心發布。之後只有不包含可能影響整個核心穩定性的
+    新功能的補丁才可能被接受。請注意一個全新的驅動程式（或者檔案系統）有
+    可能在-rc1後被接受是因為這樣的修改完全獨立，不會影響其他的程式碼，所以
+    沒有造成核心退步的風險。在-rc1以後也可以用git向Linus提交補丁，不過所
     有的補丁需要同時被發送到相應的公衆郵件列表以徵詢意見。
-  - 當Linus認爲當前的git源碼樹已經達到一個合理健全的狀態足以發布供人測試
+  - 當Linus認為當前的git源碼樹已經達到一個合理健全的狀態足以發布供人測試
     時，一個新的-rc版本就會被發布。計劃是每周都發布新的-rc版本。
-  - 這個過程一直持續下去直到內核被認爲達到足夠穩定的狀態，持續時間大概是
+  - 這個過程一直持續下去直到核心被認為達到足夠穩定的狀態，持續時間大概是
     6個星期。
 
-關於內核發布，值得一提的是Andrew Morton在linux-kernel郵件列表中如是說：
-	「沒有人知道新內核何時會被發布，因爲發布是根據已知bug的情況來決定
+關於核心發布，值得一提的是Andrew Morton在linux-kernel郵件列表中如是說：
+	「沒有人知道新核心何時會被發布，因為發布是根據已知bug的情況來決定
 	的，而不是根據一個事先制定好的時間表。」
 
 子系統特定樹
 ------------
 
-各種內核子系統的維護者——以及許多內核子系統開發人員——在原始碼庫中公開了他們
-當前的開發狀態。這樣，其他人就可以看到內核的不同區域發生了什麼。在開發速度
-很快的領域，可能會要求開發人員將提交的內容建立在這樣的子系統內核樹上，這樣
+各種核心子系統的維護者——以及許多核心子系統開發人員——在原始碼庫中公開了他們
+當前的開發狀態。這樣，其他人就可以看到核心的不同區域發生了什麼。在開發速度
+很快的領域，可能會要求開發人員將提交的內容建立在這樣的子系統核心樹上，這樣
 就避免了提交與其他已經進行的工作之間的衝突。
 
-這些存儲庫中的大多數都是Git樹，但是也有其他的scm在使用，或者補丁隊列被發布
-爲Quilt系列。這些子系統存儲庫的地址列在MAINTAINERS文件中。其中許多可以在
+這些儲存庫中的大多數都是Git樹，但是也有其他的scm在使用，或者補丁佇列被發布
+為Quilt系列。這些子系統儲存庫的地址列在MAINTAINERS檔案中。其中許多可以在
 https://git.kernel.org/上瀏覽。
 
 在將一個建議的補丁提交到這樣的子系統樹之前，需要對它進行審查，審查主要發生
-在郵件列表上（請參見下面相應的部分）。對於幾個內核子系統，這個審查過程是通
-過工具補丁跟蹤的。Patchwork提供了一個Web界面，顯示補丁發布、對補丁的任何評
-論或修訂，維護人員可以將補丁標記爲正在審查、接受或拒絕。大多數補丁網站都列
+在郵件列表上（請參見下面相應的部分）。對於幾個核心子系統，這個審查過程是通
+過工具補丁追蹤的。Patchwork提供了一個Web介面，顯示補丁發布、對補丁的任何評
+論或修訂，維護人員可以將補丁標記為正在審查、接受或拒絕。大多數補丁網站都列
 在 https://patchwork.kernel.org/
 
-Linux-next 集成測試樹
+Linux-next 整合測試樹
 ---------------------
 
-在將子系統樹的更新合併到主線樹之前，需要對它們進行集成測試。爲此，存在一個
-特殊的測試存儲庫，其中幾乎每天都會提取所有子系統樹：
+在將子系統樹的更新合併到主線樹之前，需要對它們進行整合測試。為此，存在一個
+特殊的測試儲存庫，其中幾乎每天都會提取所有子系統樹：
 
-        https://git.kernel.org/？p=linux/kernel/git/next/linux-next.git
+        https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
 
-通過這種方式，Linux-next 對下一個合併階段將進入主線內核的內容給出了一個概要
-展望。非常歡冒險的測試者運行測試Linux-next。
+透過這種方式，Linux-next 對下一個合併階段將進入主線核心的內容給出了一個概要
+展望。非常歡迎冒險的測試者對linux-next進行執行時測試。
 
-多個主要版本的穩定版內核樹
+多個主要版本的穩定版核心樹
 -----------------------------------
-由3個數字組成的內核版本號說明此內核是-stable版本。它們包含內核的相對較小且
-至關重要的修補，這些修補針對安全性問題或者嚴重的內核退步。
+由3個數字組成的核心版本號說明此核心是-stable版本。它們包含核心的相對較小且
+至關重要的修補，這些修補針對安全性問題或者嚴重的核心退步。
 
-這種版本的內核適用於那些期望獲得最新的穩定版內核並且不想參與測試開發版或
-者實驗版的用戶。
+這種版本的核心適用於那些期望獲得最新的穩定版核心並且不想參與測試開發版或
+者實驗版的使用者。
 
-穩定版內核樹版本由「穩定版」小組（郵件地址<stable@vger.kernel.org>）維護，一般
+穩定版核心樹版本由「穩定版」小組（郵件地址<stable@vger.kernel.org>）維護，一般
 隔周發布新版本。
 
-內核源碼中的 :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
-文件具體描述了可被穩定版內核接受的修改類型以及發布的流程。
+核心源碼中的
+:ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+文件具體描述了可被穩定版核心接受的修改類型以及發布的流程。
 
 
 報告bug
 -------
 
-bugzilla.kernel.org是Linux內核開發者們用來跟蹤內核Bug的網站。我們鼓勵用
-戶在這個工具中報告找到的所有bug。如何使用內核bugzilla的細節請訪問：
+bugzilla.kernel.org是Linux核心開發者們用來追蹤核心Bug的網站。我們鼓勵用
+戶在這個工具中報告找到的所有bug。如何使用核心bugzilla的細節請造訪：
 
 	http://test.kernel.org/bugzilla/faq.html
 
-內核源碼主目錄中的:ref:`admin-guide/reporting-bugs.rst <reportingbugs>`
-文件里有一個很好的模板。它指導用戶如何報告可能的內核bug以及需要提供哪些信息
-來幫助內核開發者們找到問題的根源。
+核心源碼主目錄中的:ref:`admin-guide/reporting-bugs.rst <reportingbugs>`
+文件裡有一個很好的模板。它指導使用者如何報告可能的核心bug以及需要提供哪些資訊
+來幫助核心開發者們找到問題的根源。
 
 
 利用bug報告
 -----------
 
-練習內核開發技能的最好辦法就是修改其他人報告的bug。你不光可以幫助內核變
+練習核心開發技能的最好辦法就是修改其他人報告的bug。你不光可以幫助核心變
 得更加穩定，還可以學會如何解決實際問題從而提高自己的技能，並且讓其他開發
-者感受到你的存在。修改bug是贏得其他開發者讚譽的最好辦法，因爲並不是很多
+者感受到你的存在。修改bug是贏得其他開發者讚譽的最好辦法，因為並不是很多
 人都喜歡浪費時間去修改別人報告的bug。
 
-要嘗試修改已知的bug，請訪問 http://bugzilla.kernel.org 網址。
+要嘗試修改已知的bug，請造訪 http://bugzilla.kernel.org 網址。
 
 
 郵件列表
 --------
 
-正如上面的文檔所描述，大多數的骨幹內核開發者都加入了Linux Kernel郵件列
+正如上面的文件所描述，大多數的骨幹核心開發者都加入了Linux Kernel郵件列
 表。如何訂閱和退訂列表的細節可以在這裡找到：
 
-	http://vger.kernel.org/vger-lists.html#linux-kernel
+	https://subspace.kernel.org/subscribing.html
 
 網上很多地方都有這個郵件列表的存檔(archive)。可以使用搜尋引擎來找到這些
 存檔。比如：
 
-	https://lore.kernel.org/lkml/
+	https://lore.kernel.org/linux-kernel/
 
 在發信之前，我們強烈建議你先在存檔中搜索你想要討論的問題。很多已經被詳細
 討論過的問題只在郵件列表的存檔中可以找到。
 
-大多數內核子系統也有自己獨立的郵件列表來協調各自的開發工作。從
-MAINTAINERS文件中可以找到不同話題對應的郵件列表。
+大多數核心子系統也有自己獨立的郵件列表來協調各自的開發工作。從
+MAINTAINERS檔案中可以找到不同話題對應的郵件列表。
 
-很多郵件列表架設在kernel.org伺服器上。這些列表的信息可以在這裡找到：
+很多郵件列表架設在kernel.org伺服器上。這些列表的資訊可以在這裡找到：
 
-	http://vger.kernel.org/vger-lists.html
+	https://subspace.kernel.org
 
-在使用這些郵件列表時，請記住保持良好的行爲習慣。下面的連結提供了與這些列
+在使用這些郵件列表時，請記住保持良好的行為習慣。下面的連結提供了與這些列
 表（或任何其它郵件列表）交流的一些簡單規則，雖然內容有點濫竽充數。
 
-	http://www.albion.com/netiquette/
+	https://subspace.kernel.org/etiquette.html
 
 當有很多人回覆你的郵件時，郵件的抄送列表會變得很長。請不要將任何人從抄送
 列表中刪除，除非你有足夠的理由這麼做。也不要只回復到郵件列表。請習慣於同
-一封郵件接收兩次（一封來自發送者一封來自郵件列表），而不要試圖通過添加一
+一封郵件接收兩次（一封來自發送者一封來自郵件列表），而不要試圖透過添加一
 些奇特的郵件頭來解決這個問題，人們不會喜歡的。
 
 記住保留你所回復內容的上下文和源頭。在你回覆郵件的頂部保留「某某某說到……」
 這幾行。將你的評論加在被引用的段落之間而不要放在郵件的頂部。
 
-如果你在郵件中附帶補丁，請確認它們是可以直接閱讀的純文本（如
+如果你在郵件中附帶補丁，請確認它們是可以直接閱讀的純文字（如
 :ref:`Documentation/translations/zh_TW/process/submitting-patches.rst <tw_submittingpatches>`
-文檔中所述）。內核開發者們不希望遇到附件或者被壓縮了的補丁。只有這樣才能
-保證他們可以直接評論你的每行代碼。請確保你使用的郵件發送程序不會修改空格
+文件中所述）。核心開發者們不希望遇到附件或者被壓縮了的補丁。只有這樣才能
+保證他們可以直接評論你的每行程式碼。請確保你使用的郵件發送程式不會修改空格
 和制表符。一個防範性的測試方法是先將郵件發送給自己，然後自己嘗試是否可以
-順利地打上收到的補丁。如果測試不成功，請調整或者更換你的郵件發送程序直到
-它正確工作爲止。
+順利地打上收到的補丁。如果測試不成功，請調整或者更換你的郵件發送程式直到
+它正確工作為止。
 
 總而言之，請尊重其他的郵件列表訂閱者。
 
 
-同內核社區合作
+同核心社群合作
 ----------------
 
-內核社區的目標就是提供盡善盡美的內核。所以當你提交補丁期望被接受進內核的
+核心社群的目標就是提供盡善盡美的核心。所以當你提交補丁期望被接受進核心的
 時候，它的技術價值以及其他方面都將被評審。那麼你可能會得到什麼呢？
 
   - 批評
@@ -357,9 +359,9 @@ MAINTAINERS文件中可以找到不同話題對應的郵件列表。
   - 要求證明修改的必要性
   - 沉默
 
-要記住，這些是把補丁放進內核的正常情況。你必須學會聽取對補丁的批評和評論，
+要記住，這些是把補丁放進核心的正常情況。你必須學會聽取對補丁的批評和評論，
 從技術層面評估它們，然後要麼重寫你的補丁要麼簡明扼要地論證修改是不必要
-的。如果你發的郵件沒有得到任何回應，請過幾天後再試一次，因爲有時信件會湮
+的。如果你發的郵件沒有得到任何回應，請過幾天後再試一次，因為有時信件會湮
 沒在茫茫信海中。
 
 你不應該做的事情：
@@ -369,8 +371,8 @@ MAINTAINERS文件中可以找到不同話題對應的郵件列表。
   - 忽略別人的評論
   - 沒有按照別人的要求做任何修改就重新提交
 
-在一個努力追尋最好技術方案的社區里，對於一個補丁有多少好處總會有不同的見
-解。你必須要抱著合作的態度，願意改變自己的觀點來適應內核的風格。或者至少
+在一個努力追尋最好技術方案的社群裡，對於一個補丁有多少好處總會有不同的見
+解。你必須要抱著合作的態度，願意改變自己的觀點來適應核心的風格。或者至少
 願意去證明你的想法是有價值的。記住，犯錯誤是允許的，只要你願意朝著正確的
 方案去努力。
 
@@ -378,36 +380,36 @@ MAINTAINERS文件中可以找到不同話題對應的郵件列表。
 不會被接受，也不意味著有人和你作對。你只需要改正所有提出的問題然後重新發
 送你的補丁。
 
-內核社區和公司文化的差異
+核心社群和公司文化的差異
 ------------------------
 
-內核社區的工作模式同大多數傳統公司開發隊伍的工作模式並不相同。下面這些例
+核心社群的工作模式同大多數傳統公司開發隊伍的工作模式並不相同。下面這些例
 子，可以幫助你避免某些可能發生問題：
 用這些話介紹你的修改提案會有好處：
 
     - 它同時解決了多個問題
-    - 它刪除了2000行代碼
+    - 它刪除了2000行程式碼
     - 這是補丁，它已經解釋了我想要說明的
     - 我在5種不同的體系結構上測試過它……
     - 這是一系列小補丁用來……
-    - 這個修改提高了普通機器的性能……
+    - 這個修改提高了普通機器的效能……
 
 應該避免如下的說法：
 
     - 我們在AIX/ptx/Solaris就是這麼做的，所以這麼做肯定是好的……
     - 我做這行已經20年了，所以……
-    - 爲了我們公司賺錢考慮必須這麼做
+    - 為了我們公司賺錢考慮必須這麼做
     - 這是我們的企業產品線所需要的
-    - 這裡是描述我觀點的1000頁設計文檔
+    - 這裡是描述我觀點的1000頁設計文件
     - 這是一個5000行的補丁用來……
-    - 我重寫了現在亂七八糟的代碼，這就是……
+    - 我重寫了現在亂七八糟的程式碼，這就是……
     - 我被規定了最後期限，所以這個補丁需要立刻被接受
 
-另外一個內核社區與大部分傳統公司的軟體開發隊伍不同的地方是無法面對面地交
-流。使用電子郵件和IRC聊天工具做爲主要溝通工具的一個好處是性別和種族歧視
-將會更少。Linux內核的工作環境更能接受婦女和少數族羣，因爲每個人在別人眼
-里只是一個郵件地址。國際化也幫助了公平的實現，因爲你無法通過姓名來判斷人
-的性別。男人有可能叫李麗，女人也有可能叫王剛。大多數在Linux內核上工作過
+另外一個核心社群與大部分傳統公司的軟體開發隊伍不同的地方是無法面對面地交
+流。使用電子郵件和IRC聊天工具做為主要溝通工具的一個好處是性別和種族歧視
+將會更少。Linux核心的工作環境更能接受婦女和少數族羣，因為每個人在別人眼
+里只是一個郵件地址。國際化也幫助了公平的實作，因為你無法透過姓名來判斷人
+的性別。男人有可能叫李麗，女人也有可能叫王剛。大多數在Linux核心上工作過
 並表達過看法的女性對在linux上工作的經歷都給出了正面的評價。
 
 對於一些不習慣使用英語的人來說，語言可能是一個引起問題的障礙。在郵件列表
@@ -418,61 +420,61 @@ MAINTAINERS文件中可以找到不同話題對應的郵件列表。
 拆分修改
 --------
 
-Linux內核社區並不喜歡一下接收大段的代碼。修改需要被恰當地介紹、討論並且
+Linux核心社群並不喜歡一下接收大段的程式碼。修改需要被恰當地介紹、討論並且
 拆分成獨立的小段。這幾乎完全和公司中的習慣背道而馳。你的想法應該在開發最
 開始的階段就讓大家知道，這樣你就可以及時獲得對你正在進行的開發的反饋。這
-樣也會讓社區覺得你是在和他們協作，而不是僅僅把他們當作傾銷新功能的對象。
+樣也會讓社群覺得你是在和他們協作，而不是僅僅把他們當作傾銷新功能的物件。
 無論如何，你不要一次性地向郵件列表發送50封信，你的補丁序列應該永遠用不到
 這麼多。
 
 將補丁拆開的原因如下：
 
-1) 小的補丁更有可能被接受，因爲它們不需要太多的時間和精力去驗證其正確性。
+1) 小的補丁更有可能被接受，因為它們不需要太多的時間和精力去驗證其正確性。
    一個5行的補丁，可能在維護者看了一眼以後就會被接受。而500行的補丁則
    需要數個小時來審查其正確性（所需時間隨補丁大小增加大約呈指數級增長）。
 
-   當出了問題的時候，小的補丁也會讓調試變得非常容易。一個一個補丁地回溯
+   當出了問題的時候，小的補丁也會讓除錯變得非常容易。一個一個補丁地回溯
    將會比仔細剖析一個被打上的大補丁（這個補丁破壞了其他東西）容易得多。
 
 2）不光發送小的補丁很重要，在提交之前重新編排、化簡（或者僅僅重新排列）
    補丁也是很重要的。
 
-這裡有內核開發者Al Viro打的一個比方：
-	「想像一個老師正在給學生批改數學作業。老師並不希望看到學生爲了得
+這裡有核心開發者Al Viro打的一個比方：
+	「想像一個老師正在給學生批改數學作業。老師並不希望看到學生為了得
 	到正確解法所進行的嘗試和產生的錯誤。他希望看到的是最乾淨最優雅的
 	解答。好學生了解這點，絕不會把最終解決之前的中間方案提交上去。」
 
-	內核開發也是這樣。維護者和評審者不希望看到一個人在解決問題時的思
+	核心開發也是這樣。維護者和評審者不希望看到一個人在解決問題時的思
 	考過程。他們只希望看到簡單和優雅的解決方案。
 
-直接給出一流的解決方案，和社區一起協作討論尚未完成的工作，這兩者之間似乎
+直接給出一流的解決方案，和社群一起協作討論尚未完成的工作，這兩者之間似乎
 很難找到一個平衡點。所以最好儘早開始收集有利於你進行改進的反饋；同時也要
-保證修改分成很多小塊，這樣在整個項目都準備好被包含進內核之前，其中的一部
+保證修改分成很多小塊，這樣在整個專案都準備好被包含進核心之前，其中的一部
 分可能會先被接收。
 
-必須了解這樣做是不可接受的：試圖將未完成的工作提交進內核，然後再找時間修
+必須了解這樣做是不可接受的：試圖將未完成的工作提交進核心，然後再找時間修
 復。
 
 
 證明修改的必要性
 ----------------
-除了將補丁拆成小塊，很重要的一點是讓Linux社區了解他們爲什麼需要這樣修改。
+除了將補丁拆成小塊，很重要的一點是讓Linux社群了解他們為什麼需要這樣修改。
 你必須證明新功能是有人需要的並且是有用的。
 
 
 記錄修改
 --------
 
-當你發送補丁的時候，需要特別留意郵件正文的內容。因爲這裡的信息將會做爲補
-丁的修改記錄(ChangeLog)，會被一直保留以備大家查閱。它需要完全地描述補丁，
+當你發送補丁的時候，需要特別留意郵件正文的內容。因為這裡的資訊將會做為補丁
+的修改記錄(ChangeLog)，會被一直保留以備大家查閱。它需要完全地描述補丁，
 包括：
 
-  - 爲什麼需要這個修改
+  - 為什麼需要這個修改
   - 補丁的總體設計
-  - 實現細節
+  - 實作細節
   - 測試結果
 
-想了解它具體應該看起來像什麼，請查閱以下文檔中的「ChangeLog」章節：
+想了解它具體應該看起來像什麼，請查閱以下文件中的「ChangeLog」章節：
   「The Perfect Patch」
   	 https://www.ozlabs.org/~akpm/stuff/tpp.txt
 
@@ -490,7 +492,7 @@ Dunlap和Gerrit Huizenga完善了應該說和不該說的列表。感謝Pat Moch
 Linder, Randy Dunlap, Kay Sievers, Vojtech Pavlik, Jan Kara, Josh Boyer,
 Kees Cook, Andrew Morton, Andi Kleen, Vadim Lobanov, Jesper Juhl, Adrian
 Bunk, Keri Harris, Frans Pop, David A. Wheeler, Junio Hamano, Michael
-Kerrisk和Alex Shepard的評審、建議和貢獻。沒有他們的幫助，這篇文檔是不可
+Kerrisk和Alex Shepard的評審、建議和貢獻。沒有他們的幫助，這篇文件是不可
 能完成的。
 
 
diff --git a/Documentation/translations/zh_TW/process/index.rst b/Documentation/translations/zh_TW/process/index.rst
index 65922d9faa2043..3f84437a2ca911 100644
--- a/Documentation/translations/zh_TW/process/index.rst
+++ b/Documentation/translations/zh_TW/process/index.rst
@@ -10,53 +10,121 @@
 :Original: :ref:`Documentation/process/index.rst <process_index>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
              Hu Haowen <2023002089@link.tyut.edu.cn>
+             Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_process_index:
 
 ========================
-與Linux 內核社區一起工作
+與Linux核心社群一起工作
 ========================
 
-你想成爲Linux內核開發人員嗎？歡迎之至！在學習許多關於內核的技術知識的同時，
-瞭解我們社區的工作方式也很重要。閱讀這些文檔可以讓您以更輕鬆的、麻煩更少的
-方式將更改合併到內核。
+你想成為Linux核心開發人員嗎？歡迎之至！在學習許多關於核心的技術知識的同時，
+瞭解我們社群的工作方式也很重要。閱讀這些文件可以讓您以更輕鬆的、麻煩更少的
+方式將更改合併到核心。
 
-以下是每位開發人員都應閱讀的基本指南：
+核心開發如何運作的介紹
+----------------------
+
+請先閱讀這些文件：理解這裡的內容將使你更順利地進入核心社群。
 
 .. toctree::
    :maxdepth: 1
 
    howto
-   code-of-conduct
-   code-of-conduct-interpretation
+   development-process
    submitting-patches
+   submit-checklist
+
+核心開發者的工具與技術指南
+--------------------------
+
+這是核心開發者應該熟悉的材料集合。
+
+.. toctree::
+   :maxdepth: 1
+
    programming-language
    coding-style
-   development-process
    email-clients
-   license-rules
-   kernel-enforcement-statement
-   kernel-driver-statement
+   volatile-considered-harmful
+
+TODOList:
 
-其它大多數開發人員感興趣的社區指南：
+* changes
+* maintainer-pgp-guide
+* applying-patches
+* backporting
+* adding-syscalls
+* botching-up-ioctls
 
+政策指南與開發者聲明
+--------------------
+
+這些是我們在核心社群（以及更廣範圍）中努力遵循的規則。
 
 .. toctree::
    :maxdepth: 1
 
-   submit-checklist
+   license-rules
+   code-of-conduct
+   code-of-conduct-interpretation
+   kernel-enforcement-statement
+   kernel-driver-statement
    stable-api-nonsense
    stable-kernel-rules
    management-style
+
+TODOList:
+
+* contribution-maturity-model
+* researcher-guidelines
+* generated-content
+* coding-assistants
+* conclave
+
+處理缺陷
+--------
+
+缺陷是無法避免的；正確地處理它們非常重要。下面的文件提供了關於除錯的一般
+建議，並描述了我們處理幾類特殊缺陷——迴歸和安全問題——的政策。
+
+.. toctree::
+   :maxdepth: 1
+
    embargoed-hardware-issues
 
-這些是一些總體性技術指南，由於不大好分類而放在這裏：
+TODOList:
+
+* debugging/index
+* handling-regressions
+* security-bugs
+* threat-model
+* cve
+
+維護者資訊
+----------
+
+如何找到會接受你補丁的人。
+
+TODOList:
+
+* maintainer-handbooks
+* maintainers
+
+其他材料
+--------
+
+這裡是一些大多數開發者會感興趣的其他社群指南：
 
 .. toctree::
    :maxdepth: 1
 
    magic-number
-   volatile-considered-harmful
+
+TODOList:
+
+* kernel-docs
+* deprecated
 
 .. only::  subproject and html
 
@@ -64,4 +132,3 @@
    ====
 
    * :ref:`genindex`
-
diff --git a/Documentation/translations/zh_TW/process/license-rules.rst b/Documentation/translations/zh_TW/process/license-rules.rst
index 594255856b682b..d9054cff505c33 100644
--- a/Documentation/translations/zh_TW/process/license-rules.rst
+++ b/Documentation/translations/zh_TW/process/license-rules.rst
@@ -5,21 +5,22 @@
 :Original: :ref:`Documentation/process/license-rules.rst <kernel_licensing>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
              Hu Haowen <2023002089@link.tyut.edu.cn>
+             Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_kernel_licensing:
 
-Linux內核許可規則
+Linux核心許可規則
 =================
 
-Linux內核根據LICENSES/preferred/GPL-2.0中提供的GNU通用公共許可證版本2
+Linux核心根據LICENSES/preferred/GPL-2.0中提供的GNU通用公共許可證版本2
 （GPL-2.0）的條款提供，並在LICENSES/exceptions/Linux-syscall-note中顯式
-描述了例外的系統調用，如COPYING文件中所述。
+描述了例外的系統呼叫，如COPYING檔案中所述。
 
-此文檔文件提供瞭如何對每個源文件進行註釋以使其許可證清晰明確的說明。
-它不會取代內核的許可證。
+本文件提供了如何對每個原始檔進行註解以使其許可證清晰明確的說明。
+它不會取代核心的許可證。
 
-內核源代碼作爲一個整體適用於COPYING文件中描述的許可證，但是單個源文件可以
-具有不同的與GPL-20兼容的許可證::
+核心原始程式碼作為一個整體適用於COPYING檔案中描述的許可證，但是單個原始檔可以
+具有不同的與GPL-2.0相容的許可證::
 
     GPL-1.0+ : GNU通用公共許可證v1.0或更高版本
     GPL-2.0+ : GNU通用公共許可證v2.0或更高版本
@@ -28,43 +29,47 @@ Linux內核根據LICENSES/preferred/GPL-2.0中提供的GNU通用公共許可證
     LGPL-2.1 : 僅限GNU寬通用公共許可證v2.1
     LGPL-2.1+: GNU寬通用公共許可證v2.1或更高版本
 
-除此之外，個人文件可以在雙重許可下提供，例如一個兼容的GPL變體，或者BSD，
+除此之外，個人檔案可以在雙重許可下提供，例如一個相容的GPL變體，或者BSD，
 MIT等許可。
 
-用戶空間API（UAPI）頭文件描述了用戶空間程序與內核的接口，這是一種特殊情況。
-根據內核COPYING文件中的註釋，syscall接口是一個明確的邊界，它不會將GPL要求
-擴展到任何使用它與內核通信的軟件。由於UAPI頭文件必須包含在創建在Linux內核
-上運行的可執行文件的任何源文件中，因此此例外必須記錄在特別的許可證表述中。
+使用者空間API（UAPI）標頭檔描述了使用者空間程式與核心的介面，這是一種特殊
+情況。根據核心COPYING檔案中的註解，syscall介面是一個明確的邊界，它不會將
+GPL要求擴展到任何使用它與核心通信的軟體。由於UAPI標頭檔必須包含在建立在
+Linux核心上執行的可執行檔案的任何原始檔中，因此此例外必須記錄在特別的許可
+證表述中。
 
-表達源文件許可證的常用方法是將匹配的樣板文本添加到文件的頂部註釋中。由於
-格式，拼寫錯誤等，這些“樣板”很難通過那些在上下文中使用的驗證許可證合規性
+表達原始檔許可證的常用方法是將匹配的樣板文本添加到檔案的頂部註解中。由於
+格式，拼寫錯誤等，這些“樣板”很難透過那些在上下文中使用的驗證許可證合規性
 的工具。
 
-樣板文本的替代方法是在每個源文件中使用軟件包數據交換（SPDX）許可證標識符。
-SPDX許可證標識符是機器可解析的，並且是用於提供文件內容的許可證的精確縮寫。
-SPDX許可證標識符由Linux 基金會的SPDX 工作組管理，並得到了整個行業，工具
-供應商和法律團隊的合作伙伴的一致同意。有關詳細信息，請參閱
+樣板文本的替代方法是在每個原始檔中使用軟體套件資料交換（SPDX）許可證識別碼。
+SPDX許可證識別碼是機器可解析的，並且是用於提供檔案內容的許可證的精確縮寫。
+SPDX許可證識別碼由Linux 基金會的SPDX 工作組管理，並得到了整個行業，工具
+供應商和法律團隊的合作夥伴的一致同意。有關詳細資訊，請參閱
 https://spdx.org/
 
-Linux內核需要所有源文件中的精確SPDX標識符。內核中使用的有效標識符在
-`許可標識符`_ 一節中進行了解釋，並且已可以在
+Linux核心需要所有原始檔中的精確SPDX識別碼。核心中使用的有效識別碼在
+`許可識別碼`_ 一節中進行了解釋，並且已可以在
 https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶許可證
 文本。
 
-許可標識符語法
+許可識別碼語法
 --------------
 
 1.安置:
 
-   內核文件中的SPDX許可證標識符應添加到可包含註釋的文件中的第一行。對於大多
-   數文件，這是第一行，除了那些在第一行中需要'#!PATH_TO_INTERPRETER'的腳本。
-   對於這些腳本，SPDX標識符進入第二行。
+   核心檔案中的SPDX許可證識別碼應添加到可包含註解的檔案中的第一行。對於大多
+   數檔案，這是第一行，除了那些在第一行中需要'#!PATH_TO_INTERPRETER'的腳本。
+   對於這些腳本，SPDX許可證識別碼進入第二行。
+
+   如有需要，許可證識別碼行之後可以接上一行或多行SPDX-FileCopyrightText
+   行。
 
 |
 
 2. 風格:
 
-   SPDX許可證標識符以註釋的形式添加。註釋樣式取決於文件類型::
+   SPDX許可證識別碼以註解的形式添加。註解樣式取決於檔案類型::
 
       C source:	// SPDX-License-Identifier: <SPDX License Expression>
       C header:	/* SPDX-License-Identifier: <SPDX License Expression> */
@@ -73,44 +78,44 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
       .rst:	.. SPDX-License-Identifier: <SPDX License Expression>
       .dts{i}:	// SPDX-License-Identifier: <SPDX License Expression>
 
-   如果特定工具無法處理標準註釋樣式，則應使用工具接受的相應註釋機制。這是在
-   C 頭文件中使用“/\*\*/”樣式註釋的原因。過去在使用生成的.lds文件中觀察到
-   構建被破壞，其中'ld'無法解析C++註釋。現在已經解決了這個問題，但仍然有較
-   舊的彙編程序工具無法處理C++樣式的註釋。
+   如果特定工具無法處理標準註解樣式，則應使用工具接受的相應註解機制。這是在
+   C 標頭檔中使用“/\*\*/”樣式註解的原因。過去在使用產生的.lds檔案中觀察到
+   建置被破壞，其中'ld'無法解析C++註解。現在已經解決了這個問題，但仍然有較
+   舊的組譯器工具無法處理C++樣式的註解。
 
 |
 
 3. 句法:
 
-   <SPDX許可證表達式>是SPDX許可證列表中的SPDX短格式許可證標識符，或者在許可
-   證例外適用時由“WITH”分隔的兩個SPDX短格式許可證標識符的組合。當應用多個許
+   <SPDX許可證表達式>是SPDX許可證列表中的SPDX短格式許可證識別碼，或者在許可
+   證例外適用時由“WITH”分隔的兩個SPDX短格式許可證識別碼的組合。當應用多個許
    可證時，表達式由分隔子表達式的關鍵字“AND”，“OR”組成，並由“（”，“）”包圍。
 
-   帶有“或更高”選項的[L]GPL等許可證的許可證標識符通過使用“+”來表示“或更高”
-   選項來構建。::
+   帶有“或更高”選項的[L]GPL等許可證的許可證識別碼透過使用“+”來表示“或更高”
+   選項來建置。::
 
       // SPDX-License-Identifier: GPL-2.0+
       // SPDX-License-Identifier: LGPL-2.1+
 
-   當需要修正的許可證時，應使用WITH。 例如，linux內核UAPI文件使用表達式::
+   當需要修正的許可證時，應使用WITH。 例如，linux核心UAPI檔案使用表達式::
 
       // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
       // SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note
 
-   其它在內核中使用WITH例外的事例如下::
+   其它在核心中使用WITH例外的事例如下::
 
       // SPDX-License-Identifier: GPL-2.0 WITH mif-exception
       // SPDX-License-Identifier: GPL-2.0+ WITH GCC-exception-2.0
 
-   例外只能與特定的許可證標識符一起使用。有效的許可證標識符列在異常文本文件
-   的標記中。有關詳細信息，請參閱 `許可標識符`_ 一章中的 `例外`_ 。
+   例外只能與特定的許可證識別碼一起使用。有效的許可證識別碼列在異常文本檔案
+   的標記中。有關詳細資訊，請參閱 `許可識別碼`_ 一章中的 `例外`_ 。
 
-   如果文件是雙重許可且只選擇一個許可證，則應使用OR。例如，一些dtsi文件在雙
+   如果檔案是雙重許可且只選擇一個許可證，則應使用OR。例如，一些dtsi檔案在雙
    許可下可用::
 
       // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 
-   內核中雙許可文件中許可表達式的示例::
+   核心中雙許可檔案中許可表達式的範例::
 
       // SPDX-License-Identifier: GPL-2.0 OR MIT
       // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
@@ -119,8 +124,8 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
       // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT
       // SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause OR OpenSSL
 
-   如果文件具有多個許可證，其條款全部適用於使用該文件，則應使用AND。例如，
-   如果代碼是從另一個項目繼承的，並且已經授予了將其放入內核的權限，但原始
+   如果檔案具有多個許可證，其條款全部適用於使用該檔案，則應使用AND。例如，
+   如果程式碼是從另一個專案繼承的，並且已經授予了將其放入核心的權限，但原始
    許可條款需要保持有效::
 
       // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) AND MIT
@@ -129,20 +134,20 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
 
       // SPDX-License-Identifier: GPL-1.0+ AND LGPL-2.1+
 
-許可標識符
+許可識別碼
 ----------
 
-當前使用的許可證以及添加到內核的代碼許可證可以分解爲：
+當前使用的許可證以及添加到核心的程式碼許可證可以分解為：
 
 1. _`優先許可`:
 
-   應儘可能使用這些許可證，因爲它們已知完全兼容並廣泛使用。這些許可證在內核
+   應儘可能使用這些許可證，因為它們已知完全相容並廣泛使用。這些許可證在核心
    目錄::
 
       LICENSES/preferred/
 
-   此目錄中的文件包含完整的許可證文本和 `元標記`_ 。文件名與SPDX許可證標識
-   符相同，後者應用於源文件中的許可證。
+   此目錄中的檔案包含完整的許可證文本和 `元標記`_ 。檔名與SPDX許可證識別碼
+   相同，後者應用於原始檔中的許可證。
 
    例如::
 
@@ -156,28 +161,28 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
 
    _`元標記`:
 
-   許可證文件中必須包含以下元標記：
+   許可證檔案中必須包含以下元標記：
 
    - Valid-License-Identifier:
 
-     一行或多行, 聲明那些許可標識符在項目內有效, 以引用此特定許可的文本。通
-     常這是一個有效的標識符，但是例如對於帶有'或更高'選項的許可證，兩個標識
+     一行或多行, 聲明那些許可識別碼在專案內有效, 以引用此特定許可的文本。通
+     常這是一個有效的識別碼，但是例如對於帶有'或更高'選項的許可證，兩個識別碼
      符都有效。
 
    - SPDX-URL:
 
-     SPDX頁面的URL，其中包含與許可證相關的其他信息.
+     SPDX頁面的URL，其中包含與許可證相關的其他資訊.
 
    - Usage-Guidance:
 
-     使用建議的自由格式文本。該文本必須包含SPDX許可證標識符的正確示例，因爲
-     它們應根據 `許可標識符語法`_ 指南放入源文件中。
+     使用建議的自由格式文本。該文本必須包含SPDX許可證識別碼的正確範例，因為
+     它們應根據 `許可識別碼語法`_ 指南放入原始檔中。
 
    - License-Text:
 
-     此標記之後的所有文本都被視爲原始許可文本
+     此標記之後的所有文本都被視為原始許可文本
 
-   文件格式示例::
+   檔案格式範例::
 
       Valid-License-Identifier: GPL-2.0
       Valid-License-Identifier: GPL-2.0+
@@ -209,12 +214,12 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
 
 2. 不推薦的許可證:
 
-   這些許可證只應用於現有代碼或從其他項目導入代碼。這些許可證在內核目錄::
+   這些許可證只應用於現有程式碼或從其他專案導入程式碼。這些許可證在核心目錄::
 
       LICENSES/other/
 
-   此目錄中的文件包含完整的許可證文本和 `元標記`_ 。文件名與SPDX許可證標識
-   符相同，後者應用於源文件中的許可證。
+   此目錄中的檔案包含完整的許可證文本和 `元標記`_ 。檔名與SPDX許可證識別碼
+   相同，後者應用於原始檔中的許可證。
 
    例如::
 
@@ -230,7 +235,7 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
 
    “其他”許可證的元標籤要求與 `優先許可`_ 的要求相同。
 
-   文件格式示例::
+   檔案格式範例::
 
       Valid-License-Identifier: ISC
       SPDX-URL: https://spdx.org/licenses/ISC.html
@@ -250,50 +255,50 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
 3. _`例外`:
 
    某些許可證可以修改，並允許原始許可證不具有的某些例外權利。這些例外在
-   內核目錄::
+   核心目錄::
 
       LICENSES/exceptions/
 
-   此目錄中的文件包含完整的例外文本和所需的 `例外元標記`_ 。
+   此目錄中的檔案包含完整的例外文本和所需的 `例外元標記`_ 。
 
    例如::
 
       LICENSES/exceptions/Linux-syscall-note
 
-   包含Linux內核的COPYING文件中記錄的Linux系統調用例外，該文件用於UAPI
-   頭文件。例如::
+   包含Linux核心的COPYING檔案中記錄的Linux系統呼叫例外，該檔案用於UAPI
+   標頭檔。例如::
 
       LICENSES/exceptions/GCC-exception-2.0
 
-   包含GCC'鏈接例外'，它允許獨立於其許可證的任何二進制文件與標記有此例外的
-   文件的編譯版本鏈接。這是從GPL不兼容源代碼創建可運行的可執行文件所必需的。
+   包含GCC'連結例外'，它允許獨立於其許可證的任何二進位檔案與標記有此例外的
+   檔案的編譯版本連結。這是從GPL不相容原始程式碼建立可執行的可執行檔案所必需的。
 
    _`例外元標記`:
 
-   以下元標記必須在例外文件中可用：
+   以下元標記必須在例外檔案中可用：
 
    - SPDX-Exception-Identifier:
 
-     一個可與SPDX許可證標識符一起使用的例外標識符。
+     一個可與SPDX許可證識別碼一起使用的例外識別碼。
 
    - SPDX-URL:
 
-     SPDX頁面的URL，其中包含與例外相關的其他信息。
+     SPDX頁面的URL，其中包含與例外相關的其他資訊。
 
    - SPDX-Licenses:
 
-     以逗號分隔的例外可用的SPDX許可證標識符列表。
+     以逗號分隔的例外可用的SPDX許可證識別碼列表。
 
    - Usage-Guidance:
 
-     使用建議的自由格式文本。必須在文本後面加上SPDX許可證標識符的正確示例，
-     因爲它們應根據 `許可標識符語法`_ 指南放入源文件中。
+     使用建議的自由格式文本。必須在文本後面加上SPDX許可證識別碼的正確範例，
+     因為它們應根據 `許可識別碼語法`_ 指南放入原始檔中。
 
    - Exception-Text:
 
-     此標記之後的所有文本都被視爲原始異常文本
+     此標記之後的所有文本都被視為原始異常文本
 
-   文件格式示例::
+   檔案格式範例::
 
       SPDX-Exception-Identifier: Linux-syscall-note
       SPDX-URL: https://spdx.org/licenses/Linux-syscall-note.html
@@ -324,49 +329,51 @@ https://spdx.org/licenses/ 上的官方SPDX許可證列表中檢索，並附帶
         Full exception text
 
 
-所有SPDX許可證標識符和例外都必須在LICENSES子目錄中具有相應的文件。這是允許
+所有SPDX許可證識別碼和例外都必須在LICENSES子目錄中具有相應的檔案。這是允許
 工具驗證（例如checkpatch.pl）以及準備好從源讀取和提取許可證所必需的, 這是
 各種FOSS組織推薦的，例如 `FSFE REUSE initiative <https://reuse.software/>`_.
 
-_`模塊許可`
+_`模組許可`
 -----------------
 
-   可加載內核模塊還需要MODULE_LICENSE（）標記。此標記既不替代正確的源代碼
-   許可證信息（SPDX-License-Identifier），也不以任何方式表示或確定提供模塊
-   源代碼的確切許可證。
+   可載入核心模組還需要MODULE_LICENSE（）標記。此標記既不替代正確的原始程式碼
+   許可證資訊（SPDX-License-Identifier），也不以任何方式表示或確定提供模組
+   原始程式碼的確切許可證。
 
-   此標記的唯一目的是提供足夠的信息，該模塊是否是自由軟件或者是內核模塊加
-   載器和用戶空間工具的專有模塊。
+   此標記的唯一目的是提供足夠的資訊，該模組是否是自由軟體或者是核心模組加
+   載器和使用者空間工具的專有模組。
 
-   MODULE_LICENSE（）的有效許可證字符串是:
+   MODULE_LICENSE（）的有效許可證字串是:
 
     ============================= =============================================
-    "GPL"			  模塊是根據GPL版本2許可的。這並不表示僅限於
+    "GPL"			  模組是根據GPL版本2許可的。這並不表示僅限於
                                   GPL-2.0或GPL-2.0或更高版本之間的任何區別。
-                                  最正確許可證信息只能通過相應源文件中的許可證
-                                  信息來確定
+                                  最正確許可證資訊只能透過相應原始檔中的許可證
+                                  資訊來確定
 
-    "GPL v2"			  和"GPL"相同，它的存在是因爲歷史原因。
+    "GPL v2"			  和"GPL"相同，它的存在是因為歷史原因。
 
-    "GPL and additional rights"   表示模塊源在GPL v2變體和MIT許可下雙重許可的
-                                  歷史變體。請不要在新代碼中使用。
+    "GPL and additional rights"   表示模組源在GPL v2變體和MIT許可下雙重許可的
+                                  歷史變體。請不要在新程式碼中使用。
 
-    "Dual MIT/GPL"		  表達該模塊在GPL v2變體或MIT許可證選擇下雙重
+    "Dual MIT/GPL"		  表達該模組在GPL v2變體或MIT許可證選擇下雙重
                                   許可的正確方式。
 
-    "Dual BSD/GPL"		  該模塊根據GPL v2變體或BSD許可證選擇進行雙重
-                                  許可。 BSD許可證的確切變體只能通過相應源文件
-                                  中的許可證信息來確定。
+    "Dual BSD/GPL"		  該模組根據GPL v2變體或BSD許可證選擇進行雙重
+                                  許可。 BSD許可證的確切變體只能透過相應原始檔
+                                  中的許可證資訊來確定。
 
-    "Dual MPL/GPL"		  該模塊根據GPL v2變體或Mozilla Public License
+    "Dual MPL/GPL"		  該模組根據GPL v2變體或Mozilla Public License
                                   （MPL）選項進行雙重許可。 MPL許可證的確切變體
-                                  只能通過相應的源文件中的許可證信息來確定。
-
-    "Proprietary"		  該模塊屬於專有許可。此字符串僅用於專有的第三
-                                  方模塊，不能用於在內核樹中具有源代碼的模塊。
-                                  以這種方式標記的模塊在加載時會使用'P'標記污
-                                  染內核，並且內核模塊加載器拒絕將這些模塊鏈接
-                                  到使用EXPORT_SYMBOL_GPL（）導出的符號。
+                                  只能透過相應的原始檔中的許可證資訊來確定。
+
+    "Proprietary"		  該模組屬於非GPL2相容的許可。“Proprietary
+                                  （專有）”應僅理解為“該許可證與GPLv2不相容”。
+                                  此字串僅用於非GPL2相容的第三方模組，不能用
+                                  於在核心樹中具有原始程式碼的模組。以這種方式
+                                  標記的模組在載入時會使用'P'標記污染核心，並
+                                  且核心模組載入器拒絕將這些模組連結到使用
+                                  EXPORT_SYMBOL_GPL（）匯出的符號。
     ============================= =============================================
 
 
diff --git a/Documentation/translations/zh_TW/process/programming-language.rst b/Documentation/translations/zh_TW/process/programming-language.rst
index d2c64a5599e83b..a5466047b90c1e 100644
--- a/Documentation/translations/zh_TW/process/programming-language.rst
+++ b/Documentation/translations/zh_TW/process/programming-language.rst
@@ -5,71 +5,59 @@
 :Original: :ref:`Documentation/process/programming-language.rst <programming_language>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
              Hu Haowen <2023002089@link.tyut.edu.cn>
+             Chen-Yu Yeh <chenyou910331@gmail.com>
 
 .. _tw_programming_language:
 
-程序設計語言
-============
+程式語言
+========
 
-內核是用C語言 :ref:`c-language <tw_c-language>` 編寫的。更準確地說，內核通常是用 :ref:`gcc <tw_gcc>`
-在 ``-std=gnu11`` :ref:`gcc-c-dialect-options <tw_gcc-c-dialect-options>` 下編譯的：ISO C11的 GNU 方言
+Linux核心是用C程式語言 [zh_tw_c-language]_ 編寫的。更準確地說，核心通常使
+用``gcc`` [zh_tw_gcc]_ 編譯，並且使用 ``-std=gnu11``
+[zh_tw_gcc-c-dialect-options]_：這是 ISO C11 的 GNU 方言。``clang``
+[zh_tw_clang]_ 也得到了支援，詳見文件：
+:ref:`使用 Clang/LLVM 建置 Linux <kbuild_llvm>`。
 
-這種方言包含對語言 :ref:`gnu-extensions <tw_gnu-extensions>` 的許多擴展，當然，它們許多都在內核中使用。
-
-對於一些體系結構，有一些使用 :ref:`clang <tw_clang>` 和 :ref:`icc <tw_icc>` 編譯內核
-的支持，儘管在編寫此文檔時還沒有完成，仍需要第三方補丁。
+這種方言包含對C語言的許多擴展 [zh_tw_gnu-extensions]_，當然，它們許多都在核心
+中使用。
 
 屬性
 ----
 
-在整個內核中使用的一個常見擴展是屬性（attributes） :ref:`gcc-attribute-syntax <tw_gcc-attribute-syntax>`
-屬性允許將實現定義的語義引入語言實體（如變量、函數或類型），而無需對語言進行
-重大的語法更改（例如添加新關鍵字） :ref:`n2049 <tw_n2049>`
-
-在某些情況下，屬性是可選的（即不支持這些屬性的編譯器仍然應該生成正確的代碼，
-即使其速度較慢或執行的編譯時檢查/診斷次數不夠）
-
-內核定義了僞關鍵字（例如， ``pure`` ），而不是直接使用GNU屬性語法（例如,
-``__attribute__((__pure__))`` ），以檢測可以使用哪些關鍵字和/或縮短代碼, 具體
-請參閱 ``include/linux/compiler_attributes.h``
-
-.. _tw_c-language:
-
-c-language
-   http://www.open-std.org/jtc1/sc22/wg14/www/standards
-
-.. _tw_gcc:
-
-gcc
-   https://gcc.gnu.org
-
-.. _tw_clang:
-
-clang
-   https://clang.llvm.org
+在整個核心中使用的一個常見擴展是屬性（attributes）
+[zh_tw_gcc-attribute-syntax]_。屬性允許將實作定義的語義引入語言實體（如變
+數、函式或型別），而無需對語言進行重大的語法更改（例如添加新關鍵字）
+[zh_tw_n2049]_。
 
-.. _tw_icc:
+在某些情況下，屬性是可選的（即不支援這些屬性的編譯器仍然應該產生正確的程式碼，
+即使其速度較慢或執行的編譯時檢查/診斷次數不夠）。
 
-icc
-   https://software.intel.com/en-us/c-compilers
+核心定義了偽關鍵字（例如， ``__pure`` ），而不是直接使用GNU屬性語法（例如,
+``__attribute__((__pure__))`` ），以檢測可以使用哪些關鍵字和/或縮短程式碼，
+具體請參閱 ``include/linux/compiler_attributes.h``
 
-.. _tw_gcc-c-dialect-options:
-
-c-dialect-options
-   https://gcc.gnu.org/onlinedocs/gcc/C-Dialect-Options.html
-
-.. _tw_gnu-extensions:
-
-gnu-extensions
-   https://gcc.gnu.org/onlinedocs/gcc/C-Extensions.html
-
-.. _tw_gcc-attribute-syntax:
-
-gcc-attribute-syntax
-   https://gcc.gnu.org/onlinedocs/gcc/Attribute-Syntax.html
-
-.. _tw_n2049:
-
-n2049
-   http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2049.pdf
+Rust
+----
 
+核心支援 Rust 程式語言 [zh_tw_rust-language]_，並可以透過設定選項
+``CONFIG_RUST`` 來啟用。Rust 程式碼使用 ``rustc`` [zh_tw_rustc]_ 編譯器在
+``--edition=2021`` [zh_tw_rust-editions]_ 選項下進行編譯。版本（Editions）是
+一種在語言中引入非後向相容的小型變更的方式。
+
+除此之外，核心中還使用了一些不穩定的特性 [zh_tw_rust-unstable-features]_。
+這些不穩定的特性將來可能會發生變化，因此，一個重要的目標是達到僅使用穩定特性
+的程度。
+
+具體請參閱 Documentation/rust/index.rst
+
+.. [zh_tw_c-language] http://www.open-std.org/jtc1/sc22/wg14/www/standards
+.. [zh_tw_gcc] https://gcc.gnu.org
+.. [zh_tw_clang] https://clang.llvm.org
+.. [zh_tw_gcc-c-dialect-options] https://gcc.gnu.org/onlinedocs/gcc/C-Dialect-Options.html
+.. [zh_tw_gnu-extensions] https://gcc.gnu.org/onlinedocs/gcc/C-Extensions.html
+.. [zh_tw_gcc-attribute-syntax] https://gcc.gnu.org/onlinedocs/gcc/Attribute-Syntax.html
+.. [zh_tw_n2049] http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2049.pdf
+.. [zh_tw_rust-language] https://www.rust-lang.org
+.. [zh_tw_rustc] https://doc.rust-lang.org/rustc/
+.. [zh_tw_rust-editions] https://doc.rust-lang.org/edition-guide/editions/
+.. [zh_tw_rust-unstable-features] https://github.com/Rust-for-Linux/linux/issues/2
diff --git a/Documentation/translations/zh_TW/process/stable-kernel-rules.rst b/Documentation/translations/zh_TW/process/stable-kernel-rules.rst
index 2f8f064f86294b..7d286ad54f2eae 100644
--- a/Documentation/translations/zh_TW/process/stable-kernel-rules.rst
+++ b/Documentation/translations/zh_TW/process/stable-kernel-rules.rst
@@ -6,7 +6,7 @@
 
 :Original: :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
 
-如果想評論或更新本文的內容，請直接聯繫原文檔的維護者。如果你使用英文
+如果想評論或更新本文的內容，請直接聯繫原文件的維護者。如果你使用英文
 交流有困難的話，也可以向中文版維護者求助。如果本翻譯更新不及時或者翻
 譯存在問題，請聯繫中文版維護者::
 
@@ -16,53 +16,213 @@
             - 李陽  Li Yang <leoyang.li@nxp.com>
             - Kangkai Yin <e12051@motorola.com>
             - 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+            - 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
-所有你想知道的事情 - 關於linux穩定版發佈
-========================================
-
-關於Linux 2.6穩定版發佈，所有你想知道的事情。
-
-關於哪些類型的補丁可以被接收進入穩定版代碼樹，哪些不可以的規則：
-----------------------------------------------------------------
-
-  - 必須是顯而易見的正確，並且經過測試的。
-  - 連同上下文，不能大於100行。
-  - 必須只修正一件事情。
-  - 必須修正了一個給大家帶來麻煩的真正的bug（不是“這也許是一個問題...”
-    那樣的東西）。
-  - 必須修正帶來如下後果的問題：編譯錯誤（對被標記爲CONFIG_BROKEN的例外），
-    內核崩潰，掛起，數據損壞，真正的安全問題，或者一些類似“哦，這不
-    好”的問題。簡短的說，就是一些致命的問題。
-  - 沒有“理論上的競爭條件”，除非能給出競爭條件如何被利用的解釋。
-  - 不能存在任何的“瑣碎的”修正（拼寫修正，去掉多餘空格之類的）。
-  - 必須被相關子系統的維護者接受。
-  - 必須遵循Documentation/translations/zh_CN/process/submitting-patches.rst裏的規則。
-
-向穩定版代碼樹提交補丁的過程：
-------------------------------
-
-  - 在確認了補丁符合以上的規則後，將補丁發送到stable@vger.kernel.org。
-  - 如果補丁被接受到隊列裏，發送者會收到一個ACK回覆，如果沒有被接受，收
-    到的是NAK回覆。回覆需要幾天的時間，這取決於開發者的時間安排。
-  - 被接受的補丁會被加到穩定版本隊列裏，等待其他開發者的審查。
-  - 安全方面的補丁不要發到這個列表，應該發送到security@kernel.org。
-
-審查週期：
-----------
+所有你想知道的事情 - 關於Linux -stable 版本發布
+===============================================
+
+關於哪些類型的補丁會被接收進入 "-stable" 樹、哪些不會被接收的規則：
+
+- 該補丁或一個等效的修復必須已經存在於Linux主線（上游）。
+- 它必須是顯而易見正確的，並且經過測試的。
+- 連同上下文，它不能大於100行。
+- 它必須遵循
+  :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+  裡的規則。
+- 它必須要麼修復一個困擾人們的真實的缺陷，要麼只是添加一個裝置ID。
+  對於前者，詳細來說：
+
+  - 它修復的問題，像是oops、當機、資料損壞、真實的安全問題、硬體怪癖
+    （hardware quirk）、建置錯誤（但不包括標記為CONFIG_BROKEN的東西），
+    或者一些“喔，這可不好”之類的問題。
+  - 發行版核心的使用者所報告的嚴重問題，如果修復的是顯著的效能或互動性
+    問題，也可以被考慮。由於這些修復不那麼顯而易見，並且有較高的風險引入
+    不易察覺的迴歸，它們應該只由發行版核心的維護者提交，並附上補充說明，
+    給出指向bugzilla條目（如果存在）的連結，以及關於使用者可見影響的額外
+    資訊。
+  - 不接受“這可能是一個問題...”之類的東西，比如“理論上的競爭條件”，除非
+    同時提供了缺陷如何被利用的解釋。
+  - 不接受對使用者沒有好處的“瑣碎”修復（拼寫更改、空白清理等）。
+
+
+向 -stable 樹提交補丁的流程
+---------------------------
+
+.. note::
+
+   安全補丁不應（只）由 -stable 審查流程處理，而應遵循
+   :ref:`Documentation/process/security-bugs.rst <securitybugs>`
+   的流程。
+
+要向 -stable 樹提交更改，有三個選項：
+
+1. 在你隨後提交到主線的補丁的描述中，加上一個“stable標籤”。
+2. 請求穩定版團隊撿取一個已經合併到主線的補丁。
+3. 向穩定版團隊提交一個與已合併到主線的更改等效的補丁。
+
+以下小節更詳細地描述每個選項。
+
+:ref:`tw_option_1` 是 **強烈** 推薦的做法，它最簡單也最常見。
+:ref:`tw_option_2` 主要用於提交時沒有考慮向後移植的更改。 :ref:`tw_option_3`
+是前兩個選項之外的替代方案，用於已合併到主線的補丁需要調整才能套用到較舊系列
+的情況（例如由於API變化）。
+
+使用選項2或3時，可以要求將你的更改包含到特定的穩定版系列中。這麼做時，要確保
+該修復或等效修復適用於、已提交到、或已經存在於所有仍在維護的較新穩定版樹中。
+這是為了防止使用者日後更新時可能遇到的迴歸，例如一個合併於5.19-rc1的修復被
+向後移植到5.10.y，卻沒有移植到5.15.y。
+
+.. _tw_option_1:
+
+選項1
+*****
+
+要讓你提交到主線的補丁之後被自動撿取到穩定版樹，請在簽署（sign-off）區加上
+這個標籤::
+
+  Cc: stable@vger.kernel.org
+
+當修復未公開的漏洞時，請改用 ``Cc: stable@kernel.org``：它可以降低透過
+'git send-email' 意外將修復公開的機會，因為發送到該地址的郵件不會被投遞到
+任何地方。
+
+補丁合併到主線後，它將被套用到穩定版樹，而無需作者或子系統維護者再做任何
+事情。
+
+要向穩定版團隊發送額外的指示，可使用shell風格的行內註解來傳遞任意的或預定義
+的備註：
+
+* 指明揀選（cherry pick）所需的額外補丁前置條件::
+
+    Cc: <stable@vger.kernel.org> # 3.3.x: a1f84a3: sched: Check for idle
+    Cc: <stable@vger.kernel.org> # 3.3.x: 1b9508f: sched: Rate-limit newidle
+    Cc: <stable@vger.kernel.org> # 3.3.x: fd21073: sched: Fix affinity logic
+    Cc: <stable@vger.kernel.org> # 3.3.x
+    Signed-off-by: Ingo Molnar <mingo@elte.hu>
+
+  上面標籤序列的含義為::
+
+    git cherry-pick a1f84a3
+    git cherry-pick 1b9508f
+    git cherry-pick fd21073
+    git cherry-pick <this commit>
+
+  注意，對於一個補丁系列，你不必把系列中已有的補丁列為前置條件。例如，如果
+  你有如下補丁系列::
+
+    patch1
+    patch2
+
+  其中patch2依賴patch1，如果你已經把patch1標記為穩定版收錄，就不必再把它列
+  為patch2的前置條件。
+
+* 指出核心版本的前置條件::
+
+    Cc: <stable@vger.kernel.org> # 3.3.x
+
+  該標籤的含義為::
+
+    git cherry-pick <this commit>
+
+  對每個從指定版本開始的“-stable”樹執行。
+
+  注意，如果穩定版團隊可以從Fixes:標籤推導出適當的版本，則無需這樣標記。
+
+* 延遲補丁的撿取::
 
-  - 當穩定版的維護者決定開始一個審查週期，補丁將被髮送到審查委員會，以
-    及被補丁影響的領域的維護者（除非提交者就是該領域的維護者）並且抄送
-    到linux-kernel郵件列表。
-  - 審查委員會有48小時的時間，用來決定給該補丁回覆ACK還是NAK。
-  - 如果委員會中有成員拒絕這個補丁，或者linux-kernel列表上有人反對這個
-    補丁，並提出維護者和審查委員會之前沒有意識到的問題，補丁會從隊列中
-    丟棄。
-  - 在審查週期結束的時候，那些得到ACK回應的補丁將會被加入到最新的穩定版
-    發佈中，一個新的穩定版發佈就此產生。
-  - 安全性補丁將從內核安全小組那裏直接接收到穩定版代碼樹中，而不是通過
-    通常的審查週期。請聯繫內核安全小組以獲得關於這個過程的更多細節。
-
-審查委員會：
-------------
-  - 由一些自願承擔這項任務的內核開發者，和幾個非志願的組成。
+    Cc: <stable@vger.kernel.org> # after -rc3
+
+* 指出已知的問題::
+
+    Cc: <stable@vger.kernel.org> # see patch description, needs adjustments for <= 6.3
+
+此外，stable標籤還有一種變體，可以讓穩定版團隊的向後移植工具（例如AUTOSEL
+或尋找含有'Fixes:'標籤的提交的腳本）忽略一個更改::
+
+     Cc: <stable+noautosel@kernel.org> # reason goes here, and must be present
+
+.. _tw_option_2:
+
+選項2
+*****
+
+如果補丁已經合併到主線，請發送一封電子郵件到stable@vger.kernel.org，內容
+包含補丁的標題、提交ID、你認為它應該被套用的原因，以及你希望它被套用到哪些
+核心版本。
+
+.. _tw_option_3:
+
+選項3
+*****
+
+在確認補丁符合上述規則後，將補丁發送到stable@vger.kernel.org，並註明你希望
+它被套用到的核心版本。這麼做時，你必須在你所提交補丁的更改日誌中註明上游的
+提交ID，並在提交說明文字上方以單獨一行標註，像這樣::
+
+  commit <sha1> upstream.
+
+或者::
+
+  [ Upstream commit <sha1> ]
+
+如果提交的補丁與原始的上游補丁有出入（例如因為需要為較舊的API調整），則必須
+在補丁描述中非常清楚地記錄並說明理由。
+
+
+提交之後
+--------
+
+當補丁被接受進入佇列後，發送者會收到一個ACK；如果補丁被拒絕，則會收到NAK。
+這個回覆可能需要幾天時間，取決於穩定版團隊成員的日程安排。
+
+如果被接受，補丁將被加入 -stable 佇列，供其他開發人員和相關子系統維護者
+審查。
+
+
+審查週期
+--------
+
+- 當 -stable 維護者決定進行審查週期時，補丁將被發送到審查委員會，以及補丁
+  影響領域的維護者（除非提交者就是該領域的維護者），並抄送到linux-kernel
+  郵件列表。
+- 審查委員會有48小時的時間對補丁作出ACK或NAK。
+- 如果補丁被委員會成員拒絕，或者linux-kernel列表上的成員反對這個補丁並提出
+  了維護者和委員會成員沒有意識到的問題，補丁將從佇列中移除。
+- 通過ACK的補丁將作為釋出候選（-rc）版本的一部分再次發布，以供開發人員和
+  測試人員測試。
+- 通常只會產生一個 -rc 版本，然而如果存在未解決的問題，某些補丁可能會被修改
+  或移除，或者有額外的補丁進入佇列。此後會發布更多的 -rc 版本並加以測試，
+  直到不再發現問題為止。
+- 可以在郵件列表上發送帶有任何所需測試資訊的“Tested-by:”郵件來回覆 -rc
+  版本。“Tested-by:”標籤將被收集並加入到發布提交中。
+- 在審查週期結束時，新的 -stable 版本將被發布，其中包含所有排隊的、經過測試
+  的補丁。
+- 安全補丁將由核心安全團隊直接接受進入 -stable 樹，而不經過正常的審查週期。
+  關於這一流程的更多細節，請聯繫核心安全團隊。
+
+
+樹
+--
+
+- 已完成版本和進行中版本的補丁佇列可以在以下位置找到：
+
+    https://git.kernel.org/pub/scm/linux/kernel/git/stable/stable-queue.git
+
+- 所有穩定版核心的最終定版並打上標籤的版本，可以在以下位置的每個版本各自的
+  分支中找到：
+
+    https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
+
+- 所有穩定版核心版本的釋出候選版本可以在以下位置找到：
+
+    https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git/
+
+  .. warning::
+     -stable-rc 樹是stable-queue樹在某個時間點的快照，會頻繁變動，因此會經常
+     被rebase。它只應被用於測試目的（例如供CI系統使用）。
+
+
+審查委員會
+----------
 
+- 審查委員會由一些自願承擔這項任務的核心開發人員組成，還有幾位不是自願的。
diff --git a/Documentation/translations/zh_TW/process/submitting-patches.rst b/Documentation/translations/zh_TW/process/submitting-patches.rst
index 64de92c07906af..b780f1539b46b2 100644
--- a/Documentation/translations/zh_TW/process/submitting-patches.rst
+++ b/Documentation/translations/zh_TW/process/submitting-patches.rst
@@ -15,37 +15,38 @@
  - 李陽 Li Yang <leoyang.li@nxp.com>
  - 王聰 Wang Cong <xiyou.wangcong@gmail.com>
  - 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+ - 葉宸佑 Chen-Yu Yeh <chenyou910331@gmail.com>
 
 
-提交補丁：如何讓你的改動進入內核
+提交補丁：如何讓你的改動進入核心
 ================================
 
-對於想要將改動提交到 Linux 內核的個人或者公司來說，如果不熟悉“規矩”，
-提交的流程會讓人畏懼。本文檔包含了一系列建議，可以大大提高你
+對於想要將改動提交到 Linux 核心的個人或者公司來說，如果不熟悉“規矩”，
+提交的流程會讓人畏懼。本文件包含了一系列建議，可以大大提高你
 的改動被接受的機會.
 
-本文檔以較爲簡潔的行文給出了大量建議。關於內核開發流程如何進行的詳細信息，
-參見： Documentation/translations/zh_CN/process/development-process.rst 。
-Documentation/translations/zh_CN/process/submit-checklist.rst 給出了一系列
+本文件以較為簡潔的行文給出了大量建議。關於核心開發流程如何進行的詳細資訊，
+參見： Documentation/translations/zh_TW/process/development-process.rst 。
+Documentation/translations/zh_TW/process/submit-checklist.rst 給出了一系列
 提交補丁之前要檢查的事項。設備樹相關的補丁，請參閱
 Documentation/devicetree/bindings/submitting-patches.rst 。
 
-本文檔假設您正在使用 ``git`` 準備你的補丁。如果您不熟悉 ``git`` ，最好學習
-如何使用它，這將使您作爲內核開發人員的生活變得更加輕鬆。
+本文件假設您正在使用 ``git`` 準備你的補丁。如果您不熟悉 ``git`` ，最好學習
+如何使用它，這將使您作為核心開發人員的生活變得更加輕鬆。
 
-部分子系統和維護人員的樹有一些關於其工作流程和要求的額外信息，請參閱
+部分子系統和維護人員的樹有一些關於其工作流程和要求的額外資訊，請參閱
 Documentation/process/maintainer-handbooks.rst 。
 
 獲取當前源碼樹
 --------------
 
-如果您手頭沒有當前內核源代碼的存儲庫，請使用 ``git`` 獲取一份。您需要先獲取
-主線存儲庫，它可以通過以下命令拉取::
+如果您手頭沒有當前核心原始程式碼的儲存庫，請使用 ``git`` 獲取一份。您需要先獲取
+主線儲存庫，它可以透過以下命令拉取::
 
     git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
 
 但是，請注意，您可能不想直接針對主線樹進行開發。大多數子系統維護人員運
-行自己的樹，並希望看到針對這些樹準備的補丁。請參見MAINTAINERS文件中子系
+行自己的樹，並希望看到針對這些樹準備的補丁。請參見MAINTAINERS檔案中子系
 統的 **T:** 項以查找該樹，或者直接詢問維護者該樹是否未在其中列出。
 
 .. _tw_describe_changes:
@@ -57,35 +58,37 @@ Documentation/process/maintainer-handbooks.rst 。
 的問題激勵您完成這項工作。說服審閱者相信有一個問題值得解決，讓他們讀完第一段
 後就能明白這一點。
 
-描述用戶可見的影響。直接崩潰和鎖定是相當有說服力的，但並不是所有的錯誤都那麼
-明目張膽。即使在代碼審閱期間發現了這個問題，也要描述一下您認爲它可能對用戶產
-生的影響。請記住，大多數Linux安裝運行的內核來自二級穩定樹或特定於供應商/產品
-的樹，只從上游精選特定的補丁，因此請包含任何可以幫助您將更改定位到下游的內容：
-觸發的場景、DMESG的摘錄、崩潰描述、性能迴歸、延遲尖峯、鎖定等。
+描述使用者可見的影響。直接崩潰和鎖定是相當有說服力的，但並不是所有的錯誤都
+那麼明目張膽。即使在程式碼審閱期間發現了這個問題，也要描述一下您認為它可能
+對使用者產生的影響。請記住，大多數Linux安裝執行的核心來自二級穩定樹或特定
+於供應商/產品的樹，只從上游精選特定的補丁，因此請包含任何可以幫助您將更改
+定位到下游的內容：觸發的場景、DMESG的摘錄、崩潰描述、效能迴歸、延遲尖峯、
+鎖定等。
 
-質量優化和權衡。如果您聲稱在性能、內存消耗、堆棧佔用空間或二進制大小方面有所
-改進，請包括支持它們的數據。但也要描述不明顯的成本。優化通常不是零成本的，而是
-在CPU、內存和可讀性之間進行權衡；或者，做探索性的工作，在不同的工作負載之間進
-行權衡。請描述優化的預期缺點，以便審閱者可以權衡成本和收益。
+品質最佳化和權衡。如果您聲稱在效能、記憶體消耗、堆疊佔用空間或二進位大小方
+面有所改進，請包括支援它們的資料。但也要描述不明顯的成本。最佳化通常不是零
+成本的，而是在CPU、記憶體和可讀性之間進行權衡；或者，做探索性的工作，在不
+同的工作負載之間進行權衡。請描述最佳化的預期缺點，以便審閱者可以權衡成本和
+收益。
 
 提出問題之後，就要詳細地描述一下您實際在做的技術細節。對於審閱者來說，用簡練的
-英語描述代碼的變化是很重要的，以驗證代碼的行爲是否符合您的意圖。
+英語描述程式碼的變化是很重要的，以驗證程式碼的行為是否符合您的意圖。
 
-如果您將補丁描述寫成“標準格式”，可以很容易地作爲“提交日誌”放入Linux的源代
+如果您將補丁描述寫成“標準格式”，可以很容易地作為“提交日誌”放入Linux的源代
 碼管理系統 ``git`` 中，那麼維護人員將非常感謝您。
-參見 :ref:`zh_the_canonical_patch_format` 。
+參見 :ref:`tw_the_canonical_patch_format` 。
 
 每個補丁只解決一個問題。如果你的描述開始變長，這就表明你可能需要拆分你的補丁。
-請見 :ref:`zh_split_changes` 。
+請見 :ref:`tw_split_changes` 。
 
-提交或重新提交補丁或補丁系列時，請包括完整的補丁說明和理由。不要
-只說這是補丁（系列）的第幾版。不要期望子系統維護人員引用更早的補丁版本或引用
-URL來查找補丁描述並將其放入補丁中。也就是說，補丁（系列）及其描述應該是獨立的。
-這對維護人員和審閱者都有好處。一些審閱者可能甚至沒有收到補丁的早期版本。
+提交或重新提交補丁或補丁系列時，請包括完整的補丁說明和理由。不要只說這是補丁
+（系列）的第幾版。不要期望子系統維護人員引用更早的補丁版本或引用URL來查
+找補丁描述並將其放入補丁中。也就是說，補丁（系列）及其描述應該是獨立的。這
+對維護人員和審閱者都有好處。一些審閱者可能甚至沒有收到補丁的早期版本。
 
 用祈使句描述你的變更，例如“make xyzzy do frotz”而不是“[This patch]make
-xyzzy do frotz”或“[I]changed xyzzy to do frotz”，就好像你在命令代碼庫改變
-它的行爲一樣。
+xyzzy do frotz”或“[I]changed xyzzy to do frotz”，就好像你在命令程式碼庫改變
+它的行為一樣。
 
 如果您想要引用一個特定的提交，不要只引用提交的SHA-1 ID。還請包括提交的一行
 摘要，以便於審閱者瞭解它是關於什麼的。例如::
@@ -95,38 +98,38 @@ xyzzy do frotz”或“[I]changed xyzzy to do frotz”，就好像你在命令
         platform_set_drvdata(), but left the variable "dev" unused,
         delete it.
 
-您還應該確保至少使用前12位SHA-1 ID。內核存儲庫包含 *許多* 對象，使較短的ID
-發生衝突的可能性很大。記住，即使現在不會與您的六個字符ID發生衝突，這種情況
+您還應該確保至少使用前12位SHA-1 ID。核心儲存庫包含 *許多* 物件，使較短的ID
+發生衝突的可能性很大。記住，即使現在不會與您的六個字元ID發生衝突，這種情況
 也可能在五年後改變。
 
-如果該變更的相關討論或背景信息可以在網上查閱，請加上“Link:”標籤指向它。例如
-你的補丁修復了一個缺陷，需要添加一個帶有URL的標籤指向郵件列表存檔或缺陷跟蹤器
-的相關報告；如果該補丁是由一些早先郵件列表討論或網絡上的記錄引起的，請指向它。
+如果該變更的相關討論或背景資訊可以在網上查閱，請加上“Link:”標籤指向它。例如
+你的補丁修復了一個缺陷，需要添加一個帶有URL的標籤指向郵件列表存檔或缺陷追蹤器
+的相關報告；如果該補丁是由一些早先郵件列表討論或網路上的記錄引起的，請指向它。
 
-當鏈接到郵件列表存檔時，請首選lore.kernel.org郵件存檔服務。用郵件中的
-``Message-ID`` 頭（去掉尖括號）可以創建鏈接URL。例如::
+當連結到郵件列表存檔時，請首選lore.kernel.org郵件存檔服務。用郵件中的
+``Message-ID`` 頭（去掉尖括號）可以建立連結URL。例如::
 
-    Link: https://lore.kernel.org/r/30th.anniversary.repost@klaava.Helsinki.FI/
+    Link: https://lore.kernel.org/30th.anniversary.repost@klaava.Helsinki.FI
 
-請檢查該鏈接以確保可用且指向正確的郵件。
+請檢查該連結以確保可用且指向正確的郵件。
 
 不過，在沒有外部資源的情況下，也要儘量讓你的解釋可理解。除了提供郵件列表存檔或
 缺陷的URL之外，還要需要總結該補丁的相關討論要點。
 
-如果補丁修復了特定提交中的錯誤，例如使用 ``git bisct`` 發現了一個問題，請使用
-帶有前12個字符SHA-1 ID的“Fixes:”標籤和單行摘要。爲了簡化解析腳本，不要將該
-標籤拆分爲多行，標籤不受“75列換行”規則的限制。例如::
+如果補丁修復了特定提交中的錯誤，例如使用 ``git bisect`` 發現了一個問題，請使用
+帶有至少前12個字元SHA-1 ID的“Fixes:”標籤和單行摘要。為了簡化解析腳本，不要將該
+標籤拆分為多行，標籤不受“75列換行”規則的限制。例如::
 
   Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
 
-下列 ``git config`` 設置可以讓 ``git log``, ``git show`` 增加上述風格的顯示格式::
+下列 ``git config`` 設定可以讓 ``git log``, ``git show`` 增加上述風格的顯示格式::
 
 	[core]
 		abbrev = 12
 	[pretty]
 		fixes = Fixes: %h (\"%s\")
 
-使用示例::
+使用範例::
 
 	$ git log -1 --pretty=fixes 54a4f0239f2e
 	Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
@@ -138,41 +141,42 @@ xyzzy do frotz”或“[I]changed xyzzy to do frotz”，就好像你在命令
 
 將每個 **邏輯更改** 拆分成一個單獨的補丁。
 
-例如，如果你的改動裏同時有bug修正和性能優化，那麼把這些改動拆分到兩個或
-者更多的補丁文件中。如果你的改動包含對API的修改，並且增加了一個使用該新API
+例如，如果你的改動裡同時有bug修正和效能最佳化，那麼把這些改動拆分到兩個或
+者更多的補丁檔案中。如果你的改動包含對API的修改，並且增加了一個使用該新API
 的驅動，那麼把這些修改分成兩個補丁。
 
-另一方面，如果你將一個單獨的改動做成多個補丁文件，那麼將它們合併成一個
-單獨的補丁文件。這樣一個邏輯上單獨的改動只被包含在一個補丁文件裏。
+另一方面，如果你將一個單獨的改動做成多個補丁檔案，那麼將它們合併成一個
+單獨的補丁檔案。這樣一個邏輯上單獨的改動只被包含在一個補丁檔案裡。
 
 需要記住的一點是，每個補丁的更改都應易於理解，以便審閱者驗證。每個補丁都應該
 對其價值進行闡述。
 
-如果有一個補丁依賴另外一個補丁來完成它的改動，那沒問題。直接在你的補
-丁描述裏指出 **“這個補丁依賴某補丁”** 就好了。
+如果有一個補丁依賴另外一個補丁來完成它的改動，那沒問題。直接在你的補丁
+描述裡指出 **“這個補丁依賴某補丁”** 就好了。
 
-在將您的更改劃分爲一系列補丁時，要特別注意確保內核在應用系列中的每個補丁之後
-都能正常構建和運行。使用 ``git bisect`` 來追蹤問題的開發者可能會在任何地方分
+在將您的更改劃分為一系列補丁時，要特別注意確保核心在應用系列中的每個補丁之後
+都能正常建置和執行。使用 ``git bisect`` 來追蹤問題的開發者可能會在任何地方分
 割你的補丁系列；如果你在中間引入錯誤，他們不會感謝你。
 
 如果你不能將補丁系列濃縮得更小，那麼每次大約發送出15個補丁，然後等待審閱
-和集成。
+和整合。
 
 檢查你的更改風格
 ----------------
 
-檢查您的補丁是否違反了基本樣式規定，詳細信息參見
-Documentation/translations/zh_CN/process/coding-style.rst
+檢查您的補丁是否違反了基本樣式規定，詳細資訊參見
+Documentation/translations/zh_TW/process/coding-style.rst
 中找到。如果不這樣做，只會浪費審閱者的時間，並且會導致你的補丁被拒絕，甚至
 可能沒有被閱讀。
 
-一個重要的例外是在將代碼從一個文件移動到另一個文件時——在這種情況下，您不應
-該在移動代碼的同一個補丁中修改移動的代碼。這清楚地描述了移動代碼和您的更改
-的行爲。這大大有助於審閱實際差異，並允許工具更好地跟蹤代碼本身的歷史。
+一個重要的例外是在將程式碼從一個檔案移動到另一個檔案時——在這種情況下，您不
+應該在移動程式碼的同一個補丁中修改移動的程式碼。這清楚地描述了移動程式碼和
+您的更改的行為。這大大有助於審閱實際差異，並允許工具更好地追蹤程式碼本身的
+歷史。
 
-在提交之前，使用補丁樣式檢查程序檢查補丁（scripts/check patch.pl）。不過，
-請注意，樣式檢查程序應該被視爲一個指南，而不是作爲人類判斷的替代品。如果您
-的代碼看起來更好，但有違規行爲，那麼最好別管它。
+在提交之前，使用補丁樣式檢查程式檢查補丁（scripts/checkpatch.pl）。不過，
+請注意，樣式檢查程式應該被視為一個指南，而不是作為人類判斷的替代品。如果您
+的程式碼看起來更好，但有違規行為，那麼最好別管它。
 
 檢查者報告三個級別：
 
@@ -180,58 +184,56 @@ Documentation/translations/zh_CN/process/coding-style.rst
  - WARNING：需要仔細審閱的事項
  - CHECK：需要思考的事情
 
-您應該能夠判斷您的補丁中存在的所有違規行爲。
+您應該能夠判斷您的補丁中存在的所有違規行為。
 
 選擇補丁收件人
 --------------
 
-您應該總是知會任何補丁相應代碼的子系統維護人員；查看
-維護人員文件和源代碼修訂歷史記錄，以瞭解這些維護人員是誰。腳本
+您應該總是知會任何補丁相應程式碼的子系統維護人員；查看
+MAINTAINERS檔案和原始程式碼修訂歷史記錄，以瞭解這些維護人員是誰。腳本
 scripts/get_maintainer.pl在這個步驟中非常有用。如果您找不到正在工作的子系統
 的維護人員，那麼Andrew Morton（akpm@linux-foundation.org）將充當最後的維護
 人員。
 
-您通常還應該選擇至少一個郵件列表來接收補丁集的副本。linux-kernel@vger.kernel.org
-是所有補丁的默認列表，但是這個列表的流量已經導致了許多開發人員不再看它。
-在MAINTAINERS文件中查找子系統特定的列表；您的補丁可能會在那裏得到更多的關注。
-不過，請不要發送垃圾郵件到無關的列表。
+您通常還應該選擇至少一個郵件列表來接收補丁集的副本。linux-kernel@
+vger.kernel.org是所有補丁的預設列表，但是這個列表的流量已經導致了許多開發
+人員不再看它。在MAINTAINERS檔案中查找子系統特定的列表；您的補丁可能會在那
+裡得到更多的關注。不過，請不要發送垃圾郵件到無關的列表。
 
-許多與內核相關的列表託管在vger.kernel.org上；您可以在
-http://vger.kernel.org/vger-lists.html 上找到它們的列表。不過，也有與內核相關
+許多與核心相關的列表託管在kernel.org上；您可以在
+https://subspace.kernel.org 上找到它們的列表。不過，也有與核心相關
 的列表託管在其他地方。
 
-不要一次發送超過15個補丁到vger郵件列表！！！！
-
-Linus Torvalds是決定改動能否進入 Linux 內核的最終裁決者。他的郵件地址是
+Linus Torvalds是決定改動能否進入 Linux 核心的最終裁決者。他的郵件地址是
 torvalds@linux-foundation.org 。他收到的郵件很多，所以一般來說最好 **別**
 給他發郵件。
 
-如果您有修復可利用安全漏洞的補丁，請將該補丁發送到 security@kernel.org 。對於
-嚴重的bug，可以考慮短期禁令以允許分銷商（有時間）向用戶發佈補丁；在這種情況下，
-顯然不應將補丁發送到任何公共列表。
-參見 Documentation/translations/zh_CN/process/security-bugs.rst 。
+如果您有修復可利用安全漏洞的補丁，請將該補丁發送到 security@kernel.org 。
+對於嚴重的bug，可以考慮短期禁令以允許分銷商（有時間）向使用者發布補丁；在
+這種情況下，顯然不應將補丁發送到任何公共列表。參見
+Documentation/process/security-bugs.rst 。
 
-修復已發佈內核中嚴重錯誤的補丁程序應該抄送給穩定版維護人員，方法是把以下列行
-放進補丁的籤準區（注意，不是電子郵件收件人）::
+修復已發布核心中嚴重錯誤的補丁程式應該抄送給穩定版維護人員，方法是把以下列行
+放進補丁的簽署區（注意，不是電子郵件收件人）::
 
   Cc: stable@vger.kernel.org
 
 除了本文件之外，您還應該閱讀
-Documentation/translations/zh_CN/process/stable-kernel-rules.rst 。
+Documentation/translations/zh_TW/process/stable-kernel-rules.rst 。
 
-如果更改影響到用戶側內核接口，請向手冊頁維護人員（如維護人員文件中所列）發送
-手冊頁補丁，或至少發送更改通知，以便一些信息進入手冊頁。還應將用戶空間API
-更改抄送到 linux-api@vger.kernel.org 。
+如果更改影響到使用者側核心介面，請向手冊頁維護人員（如MAINTAINERS檔案中所
+列）發送手冊頁補丁，或至少發送更改通知，以便一些資訊進入手冊頁。還應將使用
+者空間API更改抄送到 linux-api@vger.kernel.org 。
 
 
-不要MIME編碼，不要鏈接，不要壓縮，不要附件，只要純文本
+不要MIME編碼，不要連結，不要壓縮，不要附件，只要純文字
 ------------------------------------------------------
 
-Linus 和其他的內核開發者需要閱讀和評論你提交的改動。對於內核開發者來說
+Linus 和其他的核心開發者需要閱讀和評論你提交的改動。對於核心開發者來說
 ，可以“引用”你的改動很重要，使用一般的郵件工具，他們就可以在你的
-代碼的任何位置添加評論。
+程式碼的任何位置添加評論。
 
-因爲這個原因，所有的提交的補丁都是郵件中“內嵌”的。最簡單（和推薦）的方法就
+因為這個原因，所有的提交的補丁都是郵件中“內嵌”的。最簡單（和推薦）的方法就
 是使用 ``git send-email`` 。https://git-send-email.io 有 ``git send-email``
 的交互式教程。
 
@@ -239,30 +241,54 @@ Linus 和其他的內核開發者需要閱讀和評論你提交的改動。對
 
 .. warning::
 
-  如果你使用剪切-粘貼你的補丁，小心你的編輯器的自動換行功能破壞你的補丁
+  如果你使用剪切-貼上你的補丁，小心你的編輯器的自動換行功能破壞你的補丁
 
-不要將補丁作爲MIME編碼的附件，不管是否壓縮。很多流行的郵件軟件不
-是任何時候都將MIME編碼的附件當作純文本發送的，這會使得別人無法在你的
-代碼中加評論。另外，MIME編碼的附件會讓Linus多花一點時間來處理，這就
+不要將補丁作為MIME編碼的附件，不管是否壓縮。很多流行的郵件軟體不
+是任何時候都將MIME編碼的附件當作純文字發送的，這會使得別人無法在你的
+程式碼中加評論。另外，MIME編碼的附件會讓Linus多花一點時間來處理，這就
 降低了你的改動被接受的可能性。
 
 例外：如果你的郵路損壞了補丁，那麼有人可能會要求你使用MIME重新發送補丁。
 
-請參閱 Documentation/translations/zh_CN/process/email-clients.rst
-以獲取有關配置電子郵件客戶端以使其不受影響地發送補丁的提示。
+請參閱 Documentation/translations/zh_TW/process/email-clients.rst
+以獲取有關設定電子郵件客戶端以使其不受影響地發送補丁的提示。
 
 回覆審閱意見
 ------------
 
 你的補丁幾乎肯定會得到審閱者對補丁改進方法的評論（以回覆郵件的形式）。您必須
 對這些評論作出回應；讓補丁被忽略的一個好辦法就是忽略審閱者的意見。直接回復郵
-件來回應意見即可。不會導致代碼更改的意見或問題幾乎肯定會帶來註釋或變更日誌的
+件來回應意見即可。不會導致程式碼更改的意見或問題幾乎肯定會帶來註解或變更日誌的
 改變，以便下一個審閱者更好地瞭解正在發生的事情。
 
-一定要告訴審閱者你在做什麼改變，並感謝他們的時間。代碼審閱是一個累人且耗時的
+一定要告訴審閱者你在做什麼改變，並感謝他們的時間。程式碼審閱是一個累人且耗時的
 過程，審閱者有時會變得暴躁。即使在這種情況下，也要禮貌地回應並解決他們指出的
 問題。當發送下一版時，在封面郵件或獨立補丁里加上 ``patch changelog`` 說明與
-前一版本的不同之處（參見 :ref:`zh_the_canonical_patch_format` ）。
+前一版本的不同之處（參見 :ref:`tw_the_canonical_patch_format` ）。
+
+.. _tw_interleaved_replies:
+
+在郵件討論中使用裁剪過的交錯式回覆
+----------------------------------
+
+在Linux核心開發的討論中，強烈不建議置頂回覆（top-posting）。交錯式（或
+“行內”）回覆使對話更容易理解。更多細節參見：
+https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
+
+正如郵件列表中經常被引用的那樣::
+
+  A: http://en.wikipedia.org/wiki/Top_post
+  Q: Where do I find info about this thing called top-posting?
+  A: Because it messes up the order in which people normally read text.
+  Q: Why is top-posting such a bad thing?
+  A: Top-posting.
+  Q: What is the most annoying thing in e-mail?
+
+同樣，請裁剪掉所有與你的回覆無關的引文。這使回覆更容易查找，並節省時間和
+空間。更多細節參見： http://daringfireball.net/2007/07/on_top ::
+
+  A: No.
+  Q: Should I include quotations after my reply?
 
 .. _tw_resend_reminders:
 
@@ -271,54 +297,54 @@ Linus 和其他的內核開發者需要閱讀和評論你提交的改動。對
 
 提交更改後，請耐心等待。審閱者是大忙人，可能無法立即審閱您的補丁。
 
-曾幾何時，補丁曾在沒收到評論的情況下消失在虛空中，但現在開發過程應該更加順利了。
-您應該在一週左右的時間內收到評論；如果沒有收到評論，請確保您已將補丁發送
-到正確的位置。在重新提交或聯繫審閱者之前至少等待一週——在諸如合併窗口之類的
-繁忙時間可能更長。
+曾幾何時，補丁曾在沒收到評論的情況下消失在虛空中，但現在開發過程應該更加順
+利了。您應該在一週左右的時間內收到評論；如果沒有收到評論，請確保您已將補丁
+發送到正確的位置。在重新提交或聯繫審閱者之前至少等待一週——在諸如合併視窗之
+類的繁忙時間可能更長。
 
 在等了幾個星期後，用帶RESEND的主題重發補丁也是可以的::
 
    [PATCH Vx RESEND] sub/sys: Condensed patch summary
 
-當你發佈補丁（系列）修改版的時候，不要加上“RESEND”——“RESEND”只適用於重
+當你發布補丁（系列）修改版的時候，不要加上“RESEND”——“RESEND”只適用於重
 新提交之前未經修改的補丁（系列）。
 
 主題中包含 PATCH
 ----------------
 
 由於到Linus和linux-kernel的電子郵件流量很高，通常會在主題行前面加上[PATCH]
-前綴。這使Linus和其他內核開發人員更容易將補丁與其他電子郵件討論區分開。
+前綴。這使Linus和其他核心開發人員更容易將補丁與其他電子郵件討論區分開。
 
-``git send-email`` 會自動爲你加上。
+``git send-email`` 會自動為你加上。
 
 簽署你的作品——開發者來源認證
 ------------------------------
 
-爲了加強對誰做了何事的追蹤，尤其是對那些透過好幾層維護者才最終到達的補丁，我
-們在通過郵件發送的補丁上引入了“簽署（sign-off）”流程。
+為了加強對誰做了何事的追蹤，尤其是對那些透過好幾層維護者才最終到達的補丁，我
+們在透過郵件發送的補丁上引入了“簽署（sign-off）”流程。
 
-“簽署”是在補丁註釋最後的一行簡單文字，認證你編寫了它或者其他
-人有權力將它作爲開放源代碼的補丁傳遞。規則很簡單：如果你能認證如下信息:
+“簽署”是在補丁註解最後的一行簡單文字，認證你編寫了它或者其他
+人有權力將它作為開放原始程式碼的補丁傳遞。規則很簡單：如果你能認證如下資訊:
 
 開發者來源認證 1.1
 ^^^^^^^^^^^^^^^^^^
 
-對於本項目的貢獻，我認證如下信息：
+對於本專案的貢獻，我認證如下資訊：
 
-       (a) 這些貢獻是完全或者部分的由我創建，我有權利以文件中指出
-           的開放源代碼許可證提交它；或者
+       (a) 這些貢獻是完全或者部分的由我建立，我有權利以文件中指出
+           的開放原始程式碼許可證提交它；或者
 
        (b) 這些貢獻基於以前的工作，據我所知，這些以前的工作受恰當的開放
-           源代碼許可證保護，而且，根據文件中指出的許可證，我有權提交修改後的貢獻，
-           無論是完全還是部分由我創造，這些貢獻都使用同一個開放源代碼許可證
+           原始程式碼許可證保護，而且，根據文件中指出的許可證，我有權提交修改後的貢獻，
+           無論是完全還是部分由我創造，這些貢獻都使用同一個開放原始程式碼許可證
            （除非我被允許用其它的許可證）；或者
 
        (c) 這些貢獻由認證（a），（b）或者（c）的人直接提供給我，而
            且我沒有修改它。
 
-       (d) 我理解並同意這個項目和貢獻是公開的，貢獻的記錄（包括我
-           一起提交的個人記錄，包括sign-off）被永久維護並且可以和這個項目
-           或者開放源代碼的許可證同步地再發行。
+       (d) 我理解並同意這個專案和貢獻是公開的，貢獻的記錄（包括我
+           一起提交的個人記錄，包括sign-off）被永久維護並且可以和這個專案
+           或者開放原始程式碼的許可證同步地再發行。
 
 那麼加入這樣一行::
 
@@ -339,33 +365,47 @@ Linus 和其他的內核開發者需要閱讀和評論你提交的改動。對
 
 Signed-off-by: 標籤表示簽名者參與了補丁的開發，或者他/她在補丁的傳遞路徑中。
 
-如果一個人沒有直接參與補丁的準備或處理，但希望表示並記錄他們對補丁的批准/贊成，
-那麼他們可以要求在補丁的變更日誌中添加一個Acked-by:。
+如果一個人沒有直接參與補丁的準備或處理，但希望表示並記錄他們對補丁的批准/
+贊成，那麼他們可以要求在補丁的變更日誌中添加一個Acked-by:。
+
+Acked-by: 供以某種方式對受影響程式碼負責或與之相關的人使用。最常見的情況是，
+當維護者既沒有貢獻也沒有轉發補丁時，由該維護者使用。
 
-Acked-by: 通常由受影響代碼的維護者使用，當該維護者既沒有貢獻也沒有轉發補丁時。
+Acked-by: 也可以由其他利益相關者使用，例如具有領域知識的人（例如被修改程式
+碼的原作者）、核心uAPI補丁的使用者空間側審閱者，或某項功能的關鍵使用者。在
+這些情況下，可以視需要加上一個“# 後綴”以澄清其含義::
+
+	Acked-by: The Stakeholder <stakeholder@example.org> # As primary user
 
 Acked-by: 不像簽署那樣正式。這是一個記錄，確認人至少審閱了補丁，並表示接受。
-因此，補丁合併有時會手動將Acker的“Yep，looks good to me”轉換爲 Acked-By:（但
+因此，補丁合併有時會手動將Acker的“Yep，looks good to me”轉換為 Acked-By:（但
 請注意，通常最好要求一個明確的Ack）。
 
-Acked-by：不一定表示對整個補丁的確認。例如，如果一個補丁影響多個子系統，並且
-有一個來自某個子系統維護者的Acked-By:，那麼這通常表示只確認影響維護者代碼的部
-分。這裏應該仔細判斷。如有疑問，應參考郵件列表存檔中的原始討論。
+Acked-by: 也不如 Reviewed-by: 正式。例如，維護者可以用它表示他們同意補丁
+合入，但可能沒有像提供Reviewed-by:那樣徹底地審閱過補丁。同樣，關鍵使用者
+可能沒有對補丁進行技術審閱，但他們可能對整體方法、功能或面向使用者的介面
+感到滿意。
+
+Acked-by：不一定表示對整個補丁的確認。例如，如果一個補丁影響多個子系統，並
+且有一個來自某個子系統維護者的Acked-By:，那麼這通常表示只確認影響維護者程
+式碼的部分。這裡應該仔細判斷。如有疑問，應參考郵件列表存檔中的原始討論。在
+這種情況下也可以使用“# 後綴”來澄清。
 
 如果某人本應有機會對補丁進行評論，但沒有提供此類評論，您可以選擇在補丁中添加
-``Cc:`` 這是唯一可以在沒有被該人明確同意的情況下添加的標籤——但它應該表明
-這個人是在補丁上抄送的。此標籤記錄了討論中包含的潛在利益相關方。
+``Cc:`` 標籤。此標籤記錄了討論中包含的潛在利益相關方。注意，這是僅有的三個
+可以在未經被指名者明確許可的情況下使用的標籤之一（詳見下面的“標記他人需要
+許可”）。
 
-Co-developed-by: 聲明補丁是由多個開發人員共同創建的；當幾個人在一個補丁上工
-作時，它用於給出共同作者（除了From:所給出的作者之外）。因爲Co-developed-by:
+Co-developed-by: 聲明補丁是由多個開發人員共同建立的；當幾個人在一個補丁上工
+作時，它用於給出共同作者（除了From:所給出的作者之外）。因為Co-developed-by:
 表示作者身份，所以每個Co-developed-by:必須緊跟在相關合作作者的簽署之後。標準
-簽署程序要求Signed-off-by:標籤的順序應儘可能反映補丁的時間歷史，無論作者是通
+簽署程式要求Signed-off-by:標籤的順序應儘可能反映補丁的時間歷史，無論作者是通
 過From:還是Co-developed-by:表明。值得注意的是，最後一個Signed-off-by:必須是
 提交補丁的開發人員。
 
 注意，如果From:作者也是電子郵件標題的From:行中列出的人，則From:標籤是可選的。
 
-被From:作者提交的補丁示例::
+被From:作者提交的補丁範例::
 
 	<changelog>
 
@@ -375,7 +415,7 @@ Co-developed-by: 聲明補丁是由多個開發人員共同創建的；當幾個
 	Signed-off-by: Second Co-Author <second@coauthor.example.org>
 	Signed-off-by: From Author <from@author.example.org>
 
-被合作開發者提交的補丁示例::
+被合作開發者提交的補丁範例::
 
 	From: From Author <from@author.example.org>
 
@@ -392,67 +432,95 @@ Co-developed-by: 聲明補丁是由多個開發人員共同創建的；當幾個
 -----------------------------------------------------------------
 
 Reported-by: 給那些發現錯誤並報告錯誤的人致謝，它希望激勵他們在將來再次幫助
-我們。請注意，如果bug是以私有方式報告的，那麼在使用Reported-by標籤之前，請
-先請求許可。此標籤是爲Bug設計的；請不要將其用於感謝功能請求。
+我們。注意，Reported-by標籤是僅有的三個可以在未經被指名者明確許可的情況下
+使用的標籤之一（詳見下面的“標記他人需要許可”）。此標籤是為Bug設計的；請不要
+將其用於感謝功能請求。
 
 Tested-by: 標籤表示補丁已由指定的人（在某些環境中）成功測試。這個標籤通知
-維護人員已經執行了一些測試，爲將來的補丁提供了一種定位測試人員的方法，並彰顯測試人員的功勞。
+維護人員已經執行了一些測試，為將來的補丁提供了一種定位測試人員的方法，並彰
+顯測試人員的功勞。
 
-Reviewed-by：根據審閱者的監督聲明，表明該補丁已被審閱並被認爲是可接受的：
+Reviewed-by：根據審閱者的監督聲明，表明該補丁已被審閱並被認為是可接受的：
 
 
 審閱者的監督聲明
 ^^^^^^^^^^^^^^^^
 
-通過提供我的Reviewed-by:標籤，我聲明：
+透過提供我的Reviewed-by:標籤，我聲明：
 
         (a) 我已經對這個補丁進行了一次技術審閱，以評估它是否適合被包含到
-            主線內核中。
+            主線核心中。
 
         (b) 與補丁相關的任何問題、顧慮或問題都已反饋給提交者。我對提交者對
             我的評論的回應感到滿意。
 
-        (c) 雖然這一提交可能仍可被改進，但我相信，此時，（1）對內核
+        (c) 雖然這一提交可能仍可被改進，但我相信，此時，（1）對核心
             進行了有價值的修改，（2）沒有包含爭論中涉及的已知問題。
 
-        (d) 雖然我已經審閱了補丁並認爲它是健全的，但我不會（除非另有明確
-            說明）作出任何保證或擔保它會在任何給定情況下實現其規定的目的
-            或正常運行。
+        (d) 雖然我已經審閱了補丁並認為它是健全的，但我不會（除非另有明確
+            說明）作出任何保證或擔保它會在任何給定情況下實作其規定的目的
+            或正常執行。
 
-Reviewed-by是一種觀點聲明，即補丁是對內核的適當修改，沒有任何遺留的嚴重技術
-問題。任何感興趣的審閱者（完成工作的人）都可以爲一個補丁提供一個Reviewed-by
-標籤。此標籤用於向審閱者提供致謝，並通知維護者補丁的審閱進度。
-當Reviewed-by:標籤由已知了解主題區域並執行徹底檢查的審閱者提供時，通常會增加
-補丁進入內核的可能性。
+Reviewed-by是一種觀點聲明，即補丁是對核心的適當修改，沒有任何遺留的嚴重技
+術問題。任何感興趣的審閱者（完成了審閱工作且具有已知身分的人）都可以為一個
+補丁提供一個Reviewed-by標籤。此標籤用於向審閱者提供致謝，並通知維護者補丁
+的審閱進度。當Reviewed-by:標籤由已知了解主題區域並執行徹底檢查的審閱者提供
+時，通常會增加補丁進入核心的可能性。
 
-一旦從測試人員或審閱者的“Tested-by”和“Reviewed-by”標籤出現在郵件列表中，
-作者應在發送下一個版本時將其添加到適用的補丁中。但是，如果補丁在以下版本中發
-生了實質性更改，這些標籤可能不再適用，因此應該刪除。通常，在補丁更改日誌中
-（在 ``---`` 分隔符之後）應該提到刪除某人的測試者或審閱者標籤。
+一旦從測試人員或審閱者的“Tested-by”和“Reviewed-by”標籤出現在郵件列表中，作
+者應在發送下一個版本時將其添加到適用的補丁中。但是，如果補丁在以下版本中發
+生了實質性更改，這些標籤可能不再適用，因此應該刪除。通常，刪除某人的
+Acked-by、Tested-by或Reviewed-by標籤時，應在補丁更改日誌中（在 ``---`` 分
+隔符之後）提及並附上解釋。
 
 Suggested-by: 表示補丁的想法是由指定的人提出的，並確保將此想法歸功於指定的
-人。請注意，未經許可，不得添加此標籤，特別是如果該想法未在公共論壇上發佈。
-也就是說，如果我們勤快地致謝創意提供者，他們將受到鼓舞，很有希望在未來再次
-幫助我們。
+人：如果我們勤快地致謝創意提供者，他們將受到鼓舞，很有希望在未來再次幫助
+我們。注意，這是僅有的三個可以在未經被指名者明確許可的情況下使用的標籤之一
+（詳見下面的“標記他人需要許可”）。
 
-Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定錯誤的來源，這有助於
-檢查錯誤修復。這個標籤還幫助穩定內核團隊確定應該接收修復的穩定內核版本。這是
-指示補丁修復的錯誤的首選方法。請參閱 :ref:`zh_describe_changes` 瞭解更多信息。
+Fixes: 指示補丁修復了之前提交中的一個缺陷。它可以便於確定問題的來源，這有助於
+檢查錯誤修復。這個標籤還幫助穩定核心團隊確定應該接收修復的穩定核心版本。這是
+指示補丁修復的錯誤的首選方法。請參閱 :ref:`tw_describe_changes` 瞭解更多資訊。
 
 .. note::
 
-  附加Fixes:標籤不會改變穩定內核規則流程，也不改變所有穩定版補丁抄送
-  stable@vger.kernel.org的要求。有關更多信息，請閱讀
-  Documentation/translations/zh_CN/process/stable-kernel-rules.rst 。
+  附加Fixes:標籤不會改變穩定核心規則流程，也不改變所有穩定版補丁抄送
+  stable@vger.kernel.org的要求。有關更多資訊，請閱讀
+  Documentation/translations/zh_TW/process/stable-kernel-rules.rst 。
+
+最後，雖然提供標籤是受歡迎的且通常非常受讚賞，但請注意，簽署者（即提交者和
+維護者）可以自行斟酌是否採用所提供的標籤。
+
+.. _tw_tagging_people:
+
+標記他人需要許可
+----------------
+
+在補丁中添加上述標籤時要小心：除了Cc:、Reported-by:和Suggested-by:之外，
+所有標籤都需要被指名者的明確許可。對於這三個標籤，如果根據lore存檔或提交
+歷史，該人曾以該名字和電子郵件地址對Linux核心做出過貢獻，那麼隱含的許可
+就足夠了——並且對於Reported-by:和Suggested-by:，報告或建議必須是公開作出
+的。注意，就此而言bugzilla.kernel.org是公開場所，但其中使用的電子郵件地址
+是私密的；因此不要在標籤中暴露它們，除非該人在先前的貢獻中使用過。
+
+使用Assisted-by:
+----------------
+
+如果您在建立補丁的過程中使用了任何進階編碼工具，您需要透過添加Assisted-by
+標籤來聲明這一使用。不這樣做可能會妨礙您的工作被接受。關於聲明編碼助手的
+細節，請參見 Documentation/process/coding-assistants.rst 。
 
 .. _tw_the_canonical_patch_format:
 
 標準補丁格式
 ------------
 
-本節描述如何格式化補丁本身。請注意，如果您的補丁存儲在 ``Git`` 存儲庫中，則
-可以使用 ``git format-patch`` 進行正確的補丁格式化。但是，這些工具無法創建
-必要的文本，因此請務必閱讀下面的說明。
+本節描述如何格式化補丁本身。請注意，如果您的補丁儲存在 ``Git`` 儲存庫中，則
+可以使用 ``git format-patch`` 進行正確的補丁格式化。但是，這些工具無法建立
+必要的文字，因此請務必閱讀下面的說明。
+
+主題行
+^^^^^^
 
 標準的補丁標題行是::
 
@@ -460,7 +528,8 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
 
 標準補丁的信體包含如下部分：
 
-  - 一個 ``from`` 行指出補丁作者。後跟空行（僅當發送補丁的人不是作者時才需要）。
+  - 一個 ``from`` 行指出補丁作者。後跟空行（僅當發送補丁的人不是作者時才需
+    要）。
 
   - 說明文字，每行最長75列，這將被複制到永久變更日誌來描述這個補丁。
 
@@ -470,30 +539,30 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
 
   - 只包含 ``---`` 的標記線。
 
-  - 任何其他不適合放在變更日誌的註釋。
+  - 任何其他不適合放在變更日誌的註解。
 
   - 實際補丁（ ``diff`` 輸出）。
 
 標題行的格式，使得對標題行按字母序排序非常的容易——很多郵件客戶端都
-可以支持——因爲序列號是用零填充的，所以按數字排序和按字母排序是一樣的。
+可以支援——因為序列號是用零填充的，所以按數字排序和按字母排序是一樣的。
 
-郵件標題中的“子系統”標識哪個內核子系統將被打補丁。
+郵件標題中的“子系統”標識哪個核心子系統將被打補丁。
 
 郵件標題中的“一句話概述”扼要的描述郵件中的補丁。“一句話概述”
-不應該是一個文件名。對於一個補丁系列（“補丁系列”指一系列的多個相關補
-丁），不要對每個補丁都使用同樣的“一句話概述”。
+不應該是一個檔名。對於一個補丁系列（“補丁系列”指一系列的多個相關補丁
+），不要對每個補丁都使用同樣的“一句話概述”。
 
-記住郵件的“一句話概述”會成爲該補丁的全局唯一標識。它會進入 ``git``
-的改動記錄裏。然後“一句話概述”會被用在開發者的討論裏，用來指代這個補
-丁。用戶將希望通過搜索引擎搜索“一句話概述”來找到那些討論這個補丁的文
+記住郵件的“一句話概述”會成為該補丁的全域唯一標識。它會進入 ``git``
+的改動記錄裡。然後“一句話概述”會被用在開發者的討論裡，用來指代這個補丁
+。使用者將希望透過搜索引擎搜索“一句話概述”來找到那些討論這個補丁的文
 章。當人們在兩三個月後使用諸如 ``gitk`` 或 ``git log --oneline`` 之類
 的工具查看數千個補丁時，也會很快看到它。
 
-出於這些原因，概述必須不超過70-75個字符，並且必須描述補丁的更改以及爲
+出於這些原因，概述必須不超過70-75個字元，並且必須描述補丁的更改以及為
 什麼需要補丁。既要簡潔又要描述性很有挑戰性，但寫得好的概述應該這樣。
 
 概述的前綴可以用方括號括起來：“Subject: [PATCH <tag>...] <概述>”。標記
-不被視爲概述的一部分，而是描述應該如何處理補丁。如果補丁的多個版本已發
+不被視為概述的一部分，而是描述應該如何處理補丁。如果補丁的多個版本已發
 送出來以響應評審（即“v1，v2，v3”）則必須包含版本號，或包含“RFC”以指示
 評審請求。如果一個補丁系列中有四個補丁，那麼各個補丁可以這樣編號：1/4、2/4、
 3/4、4/4。這可以確保開發人員瞭解補丁應用的順序，且
@@ -501,42 +570,78 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
 
 一些標題的例子::
 
-    Subject: [patch 2/5] ext2: improve scalability of bitmap searching
-    Subject: [PATCHv2 001/207] x86: fix eflags tracking
+    Subject: [PATCH 2/5] ext2: improve scalability of bitmap searching
+    Subject: [PATCH v2 01/27] x86: fix eflags tracking
+    Subject: [PATCH v2] sub/sys: Condensed patch summary
+    Subject: [PATCH v2 M/N] sub/sys: Condensed patch summary
+
+From行
+^^^^^^
 
-``From`` 行是信體裏的最上面一行，具有如下格式::
+``From`` 行必須是信體裡的最上面一行，具有如下格式::
 
         From: Patch Author <author@example.com>
 
-``From`` 行指明在永久改動日誌裏，誰會被確認爲作者。如果沒有 ``From`` 行，那
-麼郵件頭裏的 ``From:`` 行會被用來決定改動日誌中的作者。
+``From`` 行指明在永久改動日誌裡，誰會被確認為作者。如果沒有 ``From`` 行，那
+麼郵件頭裡的 ``From:`` 行會被用來決定改動日誌中的作者。
 
-說明文字將會被提交到永久的源代碼改動日誌裏，因此應針對那些早已經不記得和這
-個補丁相關的討論細節的讀者。包括補丁處理的故障症狀（內核日誌消息、oops消息
-等），這對於可能正在搜索提交日誌以查找適用補丁的人特別有用。文本應該寫得如
-此詳細，以便在數週、數月甚至數年後閱讀時，能夠爲讀者提供所需的細節信息，以
-掌握創建補丁的 **原因** 。
+作者可以透過在 ``from`` 行和 ``SoB`` 行中加上組織名稱，來表明其所屬單位
+或工作的贊助者，例如：
+
+	From: Patch Author (Company) <author@example.com>
+
+說明主體
+^^^^^^^^
+
+說明文字將會被提交到永久的原始程式碼改動日誌裡，因此應針對那些早已經不記得和這
+個補丁相關的討論細節的讀者。包括補丁處理的故障症狀（核心日誌訊息、oops訊息
+等），這對於可能正在搜索提交日誌以查找適用補丁的人特別有用。文字應該寫得如
+此詳細，以便在數週、數月甚至數年後閱讀時，能夠為讀者提供所需的細節資訊，以
+掌握建立補丁的 **原因** 。
 
 如果一個補丁修復了一個編譯失敗，那麼可能不需要包含 *所有* 編譯失敗；
 只要足夠讓搜索補丁的人能夠找到它就行了。與概述一樣，既要簡潔又要描述性。
 
-``---`` 標記行對於補丁處理工具要找到哪裏是改動日誌信息的結束，是不可缺少
+
+.. _tw_backtraces:
+
+提交訊息中的回溯（Backtraces）
+""""""""""""""""""""""""""""""
+
+回溯有助於記錄導致問題的呼叫鏈。然而，並非所有回溯都有幫助。例如，早期引導呼
+叫鏈是獨特而明顯的。而逐字複製完整的dmesg輸出則會增加時間戳、模組列表、暫存
+器和堆疊轉儲等分散注意力的資訊。
+
+因此，最有用的回溯應該從轉儲中提取相關資訊，以更容易集中在真實問題上。下面是
+一個剪裁良好的回溯範例::
+
+  unchecked MSR access error: WRMSR to 0xd51 (tried to write 0x0000000000000064)
+  at rIP: 0xffffffffae059994 (native_write_msr+0x4/0x20)
+  Call Trace:
+  mba_wrmsr
+  update_domains
+  rdtgroup_mkdir
+
+附加註解（Commentary）
+^^^^^^^^^^^^^^^^^^^^^^
+
+``---`` 標記行對於補丁處理工具要找到哪裡是改動日誌資訊的結束，是不可缺少
 的。
 
-對於 ``---`` 標記之後的額外註解，一個好的用途就是用來寫 ``diffstat`` ，用來顯
-示修改了什麼文件和每個文件都增加和刪除了多少行。 ``diffstat`` 對於比較大的補
-丁特別有用。
-使用 ``diffstat`` 的選項 ``-p 1 -w 70`` 這樣文件名就會從內核源代碼樹的目錄開始
-，不會佔用太寬的空間（很容易適合80列的寬度，也許會有一些縮進。）
-（ ``git`` 默認會生成合適的diffstat。）
+對於 ``---`` 標記之後的額外註解，一個好的用途就是用來寫 ``diffstat`` ，用
+來顯示修改了什麼檔案和每個檔案都增加和刪除了多少行。 ``diffstat`` 對於比較
+大的補丁特別有用。使用 ``diffstat`` 的選項 ``-p 1 -w 70`` 這樣檔名就會從核
+心原始程式碼樹的目錄開始，不會佔用太寬的空間（很容易適合80列的寬度，也許會
+有一些縮排。）（ ``git`` 預設會產生合適的diffstat。）
 
-其餘那些只適用於當時或者與維護者相關的註解，不合適放到永久的改動日誌裏的，也
-應該放這裏。較好的例子就是 ``補丁更改記錄`` ，記錄了v1和v2版本補丁之間的差異。
+其餘那些只適用於當時或者與維護者相關的註解，不合適放到永久的改動日誌裡的，也
+應該放這裡。較好的例子就是 ``補丁更改記錄`` ，記錄了v1和v2版本補丁之間的差異。
 
-請將此信息放在將變更日誌與補丁的其餘部分分隔開的 ``---`` 行 **之後** 。版本
-信息不是提交到git樹的變更日誌的一部分。只是供審閱人員使用的附加信息。如果將
+請將此資訊放在將變更日誌與補丁的其餘部分分隔開的 ``---`` 行 **之後** 。版本
+資訊不是提交到git樹的變更日誌的一部分。只是供審閱人員使用的附加資訊。如果將
 其放置在提交標記上方，則需要手動交互才能將其刪除。如果它位於分隔線以下，則在
-應用補丁時會自動剝離::
+應用補丁時會自動剝離。如果可以，建議附上指向該補丁先前版本的連結（例如
+lore.kernel.org存檔連結），以幫助審閱者::
 
   <commit message>
   ...
@@ -545,29 +650,14 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
   V2 -> V3: Removed redundant helper function
   V1 -> V2: Cleaned up coding style and addressed review comments
 
+  v2: https://lore.kernel.org/bar
+  v1: https://lore.kernel.org/foo
+
   path/to/file | 5+++--
   ...
 
 在後面的參考資料中能看到正確補丁格式的更多細節。
 
-.. _tw_backtraces:
-
-提交消息中的回溯（Backtraces）
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-回溯有助於記錄導致問題的調用鏈。然而，並非所有回溯都有幫助。例如，早期引導調
-用鏈是獨特而明顯的。而逐字複製完整的dmesg輸出則會增加時間戳、模塊列表、寄存
-器和堆棧轉儲等分散注意力的信息。
-
-因此，最有用的回溯應該從轉儲中提取相關信息，以更容易集中在真實問題上。下面是
-一個剪裁良好的回溯示例::
-
-  unchecked MSR access error: WRMSR to 0xd51 (tried to write 0x0000000000000064)
-  at rIP: 0xffffffffae059994 (native_write_msr+0x4/0x20)
-  Call Trace:
-  mba_wrmsr
-  update_domains
-  rdtgroup_mkdir
 
 .. _tw_explicit_in_reply_to:
 
@@ -575,21 +665,21 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
 -----------------------------
 
 手動添加回復補丁的的郵件頭（In-Reply_To:）是有用的（例如，使用 ``git send-email`` ），
-可以將補丁與以前的相關討論關聯起來，例如，將bug補丁鏈接到電子郵件和bug報告。
-但是，對於多補丁系列，最好避免在回覆時使用鏈接到該系列的舊版本。這樣，
-補丁的多個版本就不會成爲電子郵件客戶端中無法管理的引用樹。如果鏈接有用，
-可以使用 https://lore.kernel.org/ 重定向器（例如，在封面電子郵件文本中）
-鏈接到補丁系列的早期版本。
+可以將補丁與以前的相關討論關聯起來，例如，將bug補丁連結到電子郵件和bug報告。
+但是，對於多補丁系列，最好避免在回覆時使用連結到該系列的舊版本。這樣，
+補丁的多個版本就不會成為電子郵件客戶端中無法管理的引用樹。如果連結有用，
+可以使用 https://lore.kernel.org/ 重定向器（例如，在封面電子郵件文字中）
+連結到補丁系列的早期版本。
 
-給出基礎樹信息
+給出基礎樹資訊
 --------------
 
-當其他開發人員收到您的補丁並開始審閱時，知道應該將您的工作放到代碼樹歷史記錄
-中的什麼位置通常很有用。這對於自動化持續集成流水（CI）特別有用，這些流水線試
-圖運行一系列測試，以便在維護人員開始審閱之前確定提交的質量。
+當其他開發人員收到您的補丁並開始審閱時，知道應該將您的工作放到程式碼樹歷史記錄
+中的什麼位置通常很有用。這對於自動化持續整合流水（CI）特別有用，這些流水線試
+圖執行一系列測試，以便在維護人員開始審閱之前確定提交的品質。
 
-如果您使用 ``git format-patch`` 生成補丁，則可以通過 ``--base`` 標誌在提交中
-自動包含基礎樹信息。使用此選項最簡單、最方便的方法是配合主題分支::
+如果您使用 ``git format-patch`` 產生補丁，則可以透過 ``--base`` 標誌在提交中
+自動包含基礎樹資訊。使用此選項最簡單、最方便的方法是配合主題分支::
 
     $ git checkout -t -b my-topical-branch master
     Branch 'my-topical-branch' set up to track local branch 'master'.
@@ -603,7 +693,7 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
     outgoing/...
 
 當你編輯 ``outgoing/0000-cover-letter.patch`` 時，您會注意到在它的最底部有一
-行 ``base-commit:`` 尾註，它爲審閱者和CI工具提供了足夠的信息以正確執行
+行 ``base-commit:`` 尾註，它為審閱者和CI工具提供了足夠的資訊以正確執行
 ``git am`` 而不必擔心衝突::
 
     $ git checkout -b patch-review [base-commit-id]
@@ -612,7 +702,7 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
     Applying: First Commit
     Applying: ...
 
-有關此選項的更多信息，請參閱 ``man git-format-patch`` 。
+有關此選項的更多資訊，請參閱 ``man git-format-patch`` 。
 
 .. note::
 
@@ -622,16 +712,23 @@ Fixes: 指示補丁修復了之前提交的一個問題。它可以便於確定
 的工作所基於的樹的提交哈希。你應該在封面郵件或系列的第一個補丁中添加它，它應
 該放在 ``---`` 行的下面或所有其他內容之後，即只在你的電子郵件簽名之前。
 
+工具
+----
+
+此流程的許多技術層面都可以使用b4自動化，其說明文件見
+<https://b4.docs.kernel.org/en/latest/>。它可以幫助追蹤依賴關係、執行
+checkpatch，以及格式化和發送郵件。
+
 參考文獻
 --------
 
 Andrew Morton，“完美的補丁”（tpp）
   <https://www.ozlabs.org/~akpm/stuff/tpp.txt>
 
-Jeff Garzik，“Linux內核補丁提交格式”
+Jeff Garzik，“Linux核心補丁提交格式”
   <https://web.archive.org/web/20180829112450/http://linux.yyz.us/patch-format.html>
 
-Greg Kroah-Hartman，“如何惹惱內核子系統維護人員”
+Greg Kroah-Hartman，“如何惹惱核心子系統維護人員”
   <http://www.kroah.com/log/linux/maintainer.html>
 
   <http://www.kroah.com/log/linux/maintainer-02.html>
@@ -644,10 +741,7 @@ Greg Kroah-Hartman，“如何惹惱內核子系統維護人員”
 
   <http://www.kroah.com/log/linux/maintainer-06.html>
 
-不！！！別再發巨型補丁炸彈給linux-kernel@vger.kernel.org的人們了！
-  <https://lore.kernel.org/r/20050711.125305.08322243.davem@davemloft.net>
-
-內核 Documentation/translations/zh_CN/process/coding-style.rst
+核心 Documentation/translations/zh_TW/process/coding-style.rst
 
 Linus Torvalds關於標準補丁格式的郵件
   <https://lore.kernel.org/r/Pine.LNX.4.58.0504071023190.28951@ppc970.osdl.org>
diff --git a/Documentation/usb/usbmon.rst b/Documentation/usb/usbmon.rst
index 6d5ec1e62d0933..667b84d3ae56ac 100644
--- a/Documentation/usb/usbmon.rst
+++ b/Documentation/usb/usbmon.rst
@@ -361,7 +361,7 @@ Then, execute a loop similar to the one written in pseudo-code below::
       ioctl(fd, MON_IOCX_MFETCH, &fetch);   // Process errors, too
       nflush = fetch.nfetch;       // This many packets to flush when done
       for (i = 0; i < nflush; i++) {
-         hdr = (struct ubsmon_packet *) &mmap_area[vec[i]];
+         hdr = (struct usbmon_packet *) &mmap_area[vec[i]];
          if (hdr->type == '@')     // Filler packet
             continue;
          caddr_t data = &mmap_area[vec[i]] + 64;
diff --git a/Documentation/userspace-api/dma-buf-heaps.rst b/Documentation/userspace-api/dma-buf-heaps.rst
index f56b743cdb365d..92099188893e56 100644
--- a/Documentation/userspace-api/dma-buf-heaps.rst
+++ b/Documentation/userspace-api/dma-buf-heaps.rst
@@ -36,7 +36,7 @@ following heaps:
  - A heap will be created for each reusable region in the device tree
    with the ``shared-dma-pool`` compatible, using the full device tree
    node name as its name. The buffer semantics are identical to
-   ``default-cma-region``.
+   ``default_cma_region``.
 
 Naming Convention
 =================
diff --git a/Documentation/userspace-api/futex2.rst b/Documentation/userspace-api/futex2.rst
index 9693f47a7e6287..615616305c57cb 100644
--- a/Documentation/userspace-api/futex2.rst
+++ b/Documentation/userspace-api/futex2.rst
@@ -51,10 +51,10 @@ future extension.
 
 For each entry in ``waiters`` array, the current value at ``uaddr`` is compared
 to ``val``. If it's different, the syscall undo all the work done so far and
-return ``-EAGAIN``. If all tests and verifications succeeds, syscall waits until
+return ``-EWOULDBLOCK``. If all tests and verifications succeeds, syscall waits until
 one of the following happens:
 
-- The timeout expires, returning ``-ETIMEOUT``.
+- The timeout expires, returning ``-ETIMEDOUT``.
 - A signal was sent to the sleeping task, returning ``-ERESTARTSYS``.
 - Some futex at the list was woken, returning the index of some waked futex.
 
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 2fc53093752d13..35f495bc54f7e2 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -181,8 +181,9 @@ Code  Seq#    Include File                                             Comments
 'M'   01-03  drivers/scsi/megaraid/megaraid_sas.h
 'M'   00-0F  drivers/video/fsl-diu-fb.h                                conflict!
 'N'   00-1F  drivers/usb/scanner.h
-'N'   40-7F  drivers/block/nvme.c
-'N'   80-8F  uapi/linux/ntsync.h                                       NT synchronization primitives
+'N'   40-7F  uapi/linux/nvme_ioctl.h
+'N'   80-83  uapi/linux/ntsync.h and uapi/linux/nvme_ioctl.h           conflict!
+'N'   84-8F  uapi/linux/ntsync.h                                       NT synchronization primitives
                                                                        <mailto:wine-devel@winehq.org>
 'O'   00-06  mtd/ubi-user.h                                            UBI
 'P'   all    linux/soundcard.h                                         conflict!
@@ -399,7 +400,7 @@ Code  Seq#    Include File                                             Comments
 0xDD  00-3F                                                            ZFCP device driver see drivers/s390/scsi/
                                                                        <mailto:aherrman@de.ibm.com>
 0xE5  00-3F  linux/fuse.h
-0xEC  00-01  drivers/platform/chrome/cros_ec_dev.h                     ChromeOS EC driver
+0xEC  00-01  linux/platform_data/cros_ec_chardev.h                     ChromeOS EC driver
 0xEE  00-09  uapi/linux/pfrut.h                                        Platform Firmware Runtime Update and Telemetry
 0xF3  00-3F  drivers/usb/misc/sisusbvga/sisusb.h                       sisfb (in development)
                                                                        <mailto:thomas@winischhofer.net>
diff --git a/Documentation/userspace-api/iommufd.rst b/Documentation/userspace-api/iommufd.rst
index f1c4d21e5c5ee2..763b23339480a5 100644
--- a/Documentation/userspace-api/iommufd.rst
+++ b/Documentation/userspace-api/iommufd.rst
@@ -242,7 +242,7 @@ creating the objects and links::
 
       Either a manual IOMMUFD_OBJ_HWPT_PAGING or an IOMMUFD_OBJ_HWPT_NESTED is
       created via the same IOMMU_HWPT_ALLOC uAPI. The difference is at the type
-      of the object passed in via the @pt_id field of struct iommufd_hwpt_alloc.
+      of the object passed in via the @pt_id field of struct iommu_hwpt_alloc.
 
 5. IOMMUFD_OBJ_VIOMMU can be only manually created via the IOMMU_VIOMMU_ALLOC
    uAPI, provided a dev_id (for the device's physical IOMMU to back the vIOMMU)
diff --git a/Documentation/userspace-api/media/drivers/dcmipp.rst b/Documentation/userspace-api/media/drivers/dcmipp.rst
new file mode 100644
index 00000000000000..ed4272da4b68b9
--- /dev/null
+++ b/Documentation/userspace-api/media/drivers/dcmipp.rst
@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+ST DCMIPP driver
+================
+
+The ST DCMIPP driver implements a driver specific control as part of its
+pixelproc subdev.
+
+``V4L2_CID_DCMIPP_PIXELPROC_GAMMA_CORRECTION_ENABLE (boolean)``
+    Enable / disable the gamma correction block.
+
+    The DCMIPP PixelProc stage implements a gamma compression on each R, G, B
+    component, using a static gamma exponent 2.2.
+    The gamma is implemented as a 7-segment linear curve.
diff --git a/Documentation/userspace-api/media/drivers/index.rst b/Documentation/userspace-api/media/drivers/index.rst
index 02967c9b18d6e9..8d2c489b8453e1 100644
--- a/Documentation/userspace-api/media/drivers/index.rst
+++ b/Documentation/userspace-api/media/drivers/index.rst
@@ -30,6 +30,7 @@ For more details see the file COPYING in the source distribution of Linux.
 	camera-sensor
 	ccs
 	cx2341x-uapi
+	dcmipp
 	dw100
 	imx-uapi
 	mali-c55
diff --git a/Documentation/userspace-api/vduse.rst b/Documentation/userspace-api/vduse.rst
index 81479d47c8b91f..d316857ca5bdeb 100644
--- a/Documentation/userspace-api/vduse.rst
+++ b/Documentation/userspace-api/vduse.rst
@@ -11,11 +11,10 @@ to make the device emulation more secure, the emulated vDPA device's
 control path is handled in the kernel and only the data path is
 implemented in the userspace.
 
-Note that only virtio block device is supported by VDUSE framework now,
-which can reduce security risks when the userspace process that implements
-the data path is run by an unprivileged user. The support for other device
-types can be added after the security issue of corresponding device driver
-is clarified or fixed in the future.
+Note that virtio block, network, and filesystem device types are supported
+by the VDUSE framework. Other device types may be added after the security
+implications of the corresponding device driver are clarified or fixed in
+the future.
 
 Create/Destroy VDUSE devices
 ----------------------------
@@ -135,7 +134,8 @@ module as follows:
 		return 0;
 	}
 
-There are now three types of messages introduced by VDUSE framework:
+The following control message types may be delivered via read(2) on
+/dev/vduse/$NAME:
 
 - VDUSE_GET_VQ_STATE: Get the state for virtqueue, userspace should return
   avail index for split virtqueue or the device/driver ring wrap counters and
@@ -151,6 +151,15 @@ There are now three types of messages introduced by VDUSE framework:
   IOVA range, userspace should firstly remove the old mapping, then setup the new
   mapping via the VDUSE_IOTLB_GET_FD ioctl.
 
+- VDUSE_SET_VQ_GROUP_ASID: Notify userspace to change the address space of a
+  virtqueue group (API version 1).
+
+- VDUSE_SET_VQ_READY: Notify userspace that a virtqueue should become ready or
+  not ready (when VDUSE_F_QUEUE_READY is negotiated).
+
+- VDUSE_SUSPEND: Notify userspace that the device is being suspended (when
+  VDUSE_F_SUSPEND is negotiated).
+
 After DRIVER_OK status bit is set via the VDUSE_SET_STATUS message, userspace is
 able to start the dataplane processing as follows:
 
@@ -227,7 +236,7 @@ able to start the dataplane processing as follows:
    described by the descriptors in the descriptor table should be also mapped into
    userspace via the VDUSE_IOTLB_GET_FD ioctl before accessing.
 
-5. Inject an interrupt for specific virtqueue with the VDUSE_INJECT_VQ_IRQ ioctl
+5. Inject an interrupt for specific virtqueue with the VDUSE_VQ_INJECT_IRQ ioctl
    after the used ring is filled.
 
 Enabling ASID (API version 1)
@@ -238,11 +247,11 @@ version 1. Set it up with ioctl(VDUSE_SET_API_VERSION) on `/dev/vduse/control`
 and pass `VDUSE_API_VERSION_1` before creating a new VDUSE instance with
 ioctl(VDUSE_CREATE_DEV).
 
-Afterwards, you can use the member asid of ioctl(VDUSE_VQ_SETUP) argument to
-select the address space of the IOTLB you are querying.  The driver could
-change the address space of any virtqueue group by using the
-VDUSE_SET_VQ_GROUP_ASID VDUSE message type, and the VDUSE instance needs to
-reply with VDUSE_REQ_RESULT_OK if it was possible to change it.
+Afterwards, ioctl(VDUSE_VQ_SETUP) takes a virtqueue group index in
+struct vduse_vq_config::group.  The driver can change the address space
+of any virtqueue group by using the VDUSE_SET_VQ_GROUP_ASID message type,
+and the VDUSE instance needs to reply with VDUSE_REQ_RESULT_OK if it was
+possible to change it.
 
 Similarly, you can use ioctl(VDUSE_IOTLB_GET_FD2) to obtain the file descriptor
 describing an IOVA region of a specific ASID. Example usage:
diff --git a/Documentation/virt/coco/sev-guest.rst b/Documentation/virt/coco/sev-guest.rst
index 93debceb6eb0ef..3371ebc29e8b81 100644
--- a/Documentation/virt/coco/sev-guest.rst
+++ b/Documentation/virt/coco/sev-guest.rst
@@ -38,7 +38,7 @@ along with a description:
       are not detailed, but errors with specific meanings are.
 
 The guest ioctl should be issued on a file descriptor of the /dev/sev-guest
-device.  The ioctl accepts struct snp_user_guest_request. The input and
+device.  The ioctl accepts struct snp_guest_request_ioctl. The input and
 output structure is specified through the req_data and resp_data field
 respectively. If the ioctl fails to execute due to a firmware error, then
 the fw_error code will be set, otherwise fw_error will be set to -1.
diff --git a/Documentation/virt/index.rst b/Documentation/virt/index.rst
index c1f0bbc373156a..142134ed45c00d 100644
--- a/Documentation/virt/index.rst
+++ b/Documentation/virt/index.rst
@@ -9,6 +9,7 @@ Virtualization Support
 
    kvm/index
    uml/user_mode_linux_howto_v2
+   uml/nommu-uml
    paravirt_ops
    guest-halt-polling
    ne_overview
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index e0430cc750c9e5..a48e7239558249 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -117,7 +117,7 @@ description:
       x86 includes both i386 and x86_64.
 
   Type:
-      system, vm, or vcpu.
+      system, vm, vcpu or guest_memfd.
 
   Parameters:
       what parameters are accepted by the ioctl.
@@ -918,7 +918,8 @@ The irq_type field has the following values:
 - KVM_ARM_IRQ_TYPE_SPI:
 	       in-kernel GICv2/GICv3: SPI, irq_id between 32 and 1019 (incl.)
                (the vcpu_index field is ignored)
-	       in-kernel GICv5: SPI, irq_id between 0 and 65535 (incl.)
+	       in-kernel GICv5: SPI, irq_id between 0 and the configured
+	       number of SPIs minus one (1023 maximum)
 - KVM_ARM_IRQ_TYPE_PPI:
 	       in-kernel GICv2/GICv3: PPI, irq_id between 16 and 31 (incl.)
 	       in-kernel GICv5: PPI, irq_id between 0 and 127 (incl.)
@@ -3600,20 +3601,14 @@ Possible features:
 :Parameters: struct kvm_vcpu_init (out)
 :Returns: 0 on success; -1 on error
 
-Errors:
-
-  ======     ==========================================
-  ENODEV     no preferred target available for the host
-  ======     ==========================================
-
 This queries KVM for preferred CPU target type which can be emulated
 by KVM on underlying host.
 
-The ioctl returns struct kvm_vcpu_init instance containing information
-about preferred CPU target type and recommended features for it.  The
-kvm_vcpu_init->features bitmap returned will have feature bits set if
-the preferred target recommends setting these features, but this is
-not mandatory.
+The ioctl returns a struct kvm_vcpu_init instance containing the
+preferred CPU target type. The kvm_vcpu_init->features bitmap is
+returned empty: userspace selects the vCPU features itself, and their
+availability is reported by the capabilities listed under
+KVM_ARM_VCPU_INIT.
 
 The information returned by this ioctl can be used to prepare an instance
 of struct kvm_vcpu_init for KVM_ARM_VCPU_INIT ioctl which will result in
@@ -3676,7 +3671,7 @@ type KVM_X86_REG_TYPE_MSR, but are NOT enumerated via KVM_GET_REG_LIST.
 :Capability: KVM_CAP_ARM_SET_DEVICE_ADDR
 :Architectures: arm64
 :Type: vm ioctl
-:Parameters: struct kvm_arm_device_address (in)
+:Parameters: struct kvm_arm_device_addr (in)
 :Returns: 0 on success, -1 on error
 
 Errors:
@@ -6381,11 +6376,24 @@ mapping for userspace_addr is not required to be valid/populated at the time of
 KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated
 on-demand.
 
-When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes
-userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE
-state.  At VM creation time, all memory is shared, i.e. the PRIVATE attribute
-is '0' for all gfns.  Userspace can control whether memory is shared/private by
-toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed.
+When mapping a gfn into the guest, guest faults are always serviced from
+guest_memfd regardless of whether memory is shared or private.  KVM determines
+shared vs. private based on the state in guest_memfd, which is the sole
+authority on private vs. shared memory.  See :ref:`KVM_CREATE_GUEST_MEMFD` to
+find out more about the creation-time shared/private status.  Userspace can
+control whether memory is shared/private by toggling
+KVM_MEMORY_ATTRIBUTE_PRIVATE via :ref:`KVM_SET_MEMORY_ATTRIBUTES2` as needed.
+
+userspace_addr is expected to be the mmap()-ed address corresponding to the
+right offset within the guest_memfd. Any mismatch between userspace_addr and
+guest_memfd is not validated and is a user error. userspace_addr is only used
+for host-side guest accesses such as kvm_read_guest().
+
+If in-place conversion is disabled, KVM selects shared vs. private based on the
+gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE state.  At VM creation time, all memory is
+shared, i.e. the PRIVATE attribute is '0' for all gfns.  Userspace can control
+whether memory is shared/private by toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via
+KVM_SET_MEMORY_ATTRIBUTES as needed.
 
 S390:
 ^^^^^
@@ -6393,6 +6401,8 @@ S390:
 Returns -EINVAL if the VM has the KVM_VM_S390_UCONTROL flag set.
 Returns -EINVAL if called on a protected VM.
 
+.. _KVM_SET_MEMORY_ATTRIBUTES:
+
 4.141 KVM_SET_MEMORY_ATTRIBUTES
 -------------------------------
 
@@ -6429,6 +6439,10 @@ the state of a gfn/page as needed.
 
 The "flags" field is reserved for future extensions and must be '0'.
 
+See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES2`.
+
+.. _KVM_CREATE_GUEST_MEMFD:
+
 4.142 KVM_CREATE_GUEST_MEMFD
 ----------------------------
 
@@ -6483,10 +6497,10 @@ specified via KVM_CREATE_GUEST_MEMFD.  Currently defined flags:
                                page tables. Private memory cannot.
   ============================ ================================================
 
-When the KVM MMU performs a PFN lookup to service a guest fault and the backing
-guest_memfd has the GUEST_MEMFD_FLAG_MMAP set, then the fault will always be
-consumed from guest_memfd, regardless of whether it is a shared or a private
-fault.
+When the KVM MMU performs a PFN lookup to service a guest fault, the fault will
+always be consumed from guest_memfd, regardless of whether it is a shared or a
+private fault (unless in-place conversion is disabled and the backing
+guest_memfd does not have the GUEST_MEMFD_FLAG_MMAP flag set).
 
 See KVM_SET_USER_MEMORY_REGION2 for additional details.
 
@@ -6494,7 +6508,7 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details.
 ---------------------------
 
 :Capability: KVM_CAP_PRE_FAULT_MEMORY
-:Architectures: none
+:Architectures: x86, s390, arm64
 :Type: vcpu ioctl
 :Parameters: struct kvm_pre_fault_memory (in/out)
 :Returns: 0 if at least one page is processed, < 0 on error
@@ -6502,12 +6516,15 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details.
 Errors:
 
   ========== ===============================================================
+  EAGAIN     A race occurred before progress was made, but a retry may succeed.
   EINVAL     The specified `gpa` and `size` were invalid (e.g. not
              page aligned, causes an overflow, or size is zero), or the VM
              is UCONTROL (s390).
   ENOENT     The specified `gpa` is outside defined memslots.
+  ENOEXEC    The vCPU has not been initialised (arm64).
   EINTR      An unmasked signal is pending and no page was processed.
   EFAULT     The parameter address was invalid.
+  EHWPOISON  A poisoned host page was encountered.
   EOPNOTSUPP Mapping memory for a GPA is unsupported by the
              hypervisor, and/or for the current vCPU state/mode.
   EIO        unexpected error conditions (also causes a WARN)
@@ -6527,7 +6544,17 @@ Errors:
 KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory
 for the current vCPU state.  KVM maps memory as if the vCPU generated a
 stage-2 read page fault, e.g. faults in memory as needed, but doesn't break
-CoW.  On x86, KVM does not mark any newly created stage-2 PTE as Accessed.
+CoW.  On arm64, KVM marks newly created stage-2 PTEs as Accessed, as it
+does for any stage-2 fault, but leaves the Accessed state of existing PTEs
+unchanged.  On x86, KVM does not mark any newly created stage-2 PTE as
+Accessed, and for s390 it is not applicable.
+
+On arm64, a GPA is interpreted as an IPA, and never interpreted as the IPA
+of a nested guest. Pre-faulting only populates canonical stage-2 page
+tables.
+
+The feature is not supported on arm64 if the protected KVM (pKVM) feature
+is enabled.
 
 In the case of confidential VM types where there is an initial set up of
 private guest memory before the guest is 'finalized'/measured, this ioctl
@@ -6542,7 +6569,7 @@ When the ioctl returns, the input values are updated to point to the
 remaining range.  If `size` > 0 on return, the caller can just issue
 the ioctl again with the same `struct kvm_map_memory` argument.
 
-Shadow page tables cannot support this ioctl because they
+On x86, shadow page tables cannot support this ioctl because they
 are indexed by virtual address or nested guest physical address.
 Calling this ioctl when the guest is using shadow page tables (for
 example because it is running a nested guest with nested page tables)
@@ -6663,6 +6690,83 @@ significant bit):
    Userspace should use the defined constants from ``<linux/kvm.h>`` rather
    than hardcoding bit positions.
 
+.. _KVM_SET_MEMORY_ATTRIBUTES2:
+
+4.146 KVM_SET_MEMORY_ATTRIBUTES2
+---------------------------------
+
+:Capability: KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES
+:Architectures: all
+:Type: guest_memfd ioctl
+:Parameters: struct kvm_memory_attributes2 (in/out)
+:Returns: 0 on success, <0 on error
+
+Errors:
+
+  ========== ===============================================================
+  EINVAL     The specified `offset` or `size` was invalid (e.g. not
+             page aligned, causes an overflow, or size is zero).
+  EFAULT     The parameter address was invalid.
+  ENOMEM     Ran out of memory trying to track private/shared state
+  EAGAIN     Pages in the range have outstanding references, see
+             ``error_offset`` below.
+  ========== ===============================================================
+
+KVM_SET_MEMORY_ATTRIBUTES2 is an extension to
+KVM_SET_MEMORY_ATTRIBUTES that supports returning (writing) values to
+userspace.  The original (pre-extension) fields are shared with
+KVM_SET_MEMORY_ATTRIBUTES identically.
+
+Attribute values are shared with KVM_SET_MEMORY_ATTRIBUTES.
+
+::
+
+  struct kvm_memory_attributes2 {
+	union {
+		__u64 address;
+		__u64 offset;
+	};
+	__u64 size;
+	__u64 attributes;
+	__u64 flags;
+	__u64 error_offset;
+	__u64 reserved[11];
+  };
+
+  #define KVM_MEMORY_ATTRIBUTE_PRIVATE           (1ULL << 3)
+
+The capability KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES enumerates the attributes
+that can be set via KVM_SET_MEMORY_ATTRIBUTES2.
+
+Set attributes for a range of offsets within a guest_memfd to
+KVM_MEMORY_ATTRIBUTE_PRIVATE to limit the specified guest_memfd backed
+memory range for guest use. Even if the guest_memfd was created with
+GUEST_MEMFD_FLAG_MMAP, after a successful call to set
+KVM_MEMORY_ATTRIBUTE_PRIVATE, the requested range will not be mappable
+into host userspace and will only be mappable by the guest.
+
+To allow the range to be mappable into host userspace again, call
+KVM_SET_MEMORY_ATTRIBUTES2 on the guest_memfd again with
+KVM_MEMORY_ATTRIBUTE_PRIVATE unset.
+
+KVM does not directly manipulate the memory contents of pages during
+attribute updates. However, the process of setting these attributes,
+which includes operations such as unmapping pages from the host or
+stage-2 page tables, may result in side effects on memory contents
+that vary across different trusted firmware implementations.
+
+If this ioctl returns -EAGAIN, the offset of the page with unexpected
+refcounts will be returned in ``error_offset``. This can occur if
+there are transient refcounts on the pages, taken by other parts of
+the kernel.
+
+Userspace is expected to figure out how to remove all known refcounts
+on the shared pages, such as refcounts taken by get_user_pages(), and
+try the ioctl again. A possible source of these long term refcounts is
+if the guest_memfd memory was pinned in IOMMU page tables.
+
+See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`.
+
 .. _kvm_run:
 
 5. The kvm_run structure
diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst
index 70b9162755c7e8..8a691c8938b625 100644
--- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst
+++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst
@@ -12,12 +12,39 @@ Only one VGIC instance may be instantiated through this API.  The created VGIC
 will act as the VM interrupt controller, requiring emulated user-space devices
 to inject interrupts to the VGIC instead of directly to CPUs.
 
-Creating a guest GICv5 device requires a GICv5 host.  The current VGICv5 device
-only supports PPI interrupts.  These can either be injected from emulated
-in-kernel devices (such as the Arch Timer, or PMU), or via the KVM_IRQ_LINE
-ioctl.
+Creating a guest GICv5 device requires a GICv5 host.  The VGICv5 device supports
+PPI, SPI, and LPI interrupts.  The PPI and SPI interrupts can either be injected
+from emulated in-kernel devices (such as the Arch Timer, or PMU), or via the
+KVM_IRQ_LINE ioctl.  LPIs are not externally injected, but are handled in
+hardware via the LPI IST.  Their pending state is driven directly by the guest.
 
 Groups:
+  KVM_DEV_ARM_VGIC_GRP_ADDR
+   Attributes:
+
+    KVM_VGIC_V5_ADDR_TYPE_IRS (rw, 64-bit)
+      Base address in the guest physical address space of the GICv5 IRS
+      (Interrupt Routing Service) register mappings. Only valid for
+      KVM_DEV_TYPE_ARM_VGIC_V5.  This address needs to be 64K aligned and the
+      region covers 128 KByte - the IRS has a CONFIG_FRAME and a SETLPI_FRAME,
+      each of which is 64 KBytes in size.
+
+      Setting the address of the IRS in GPA space is mandatory before VGIC
+      resources are mapped, as the IRS is responsible for handling SPIs and
+      LPIs. Failure to set the IRS address before the first vCPU run results in
+      an error.
+
+  KVM_DEV_ARM_VGIC_GRP_NR_IRQS
+   Attributes:
+
+    A value describing the number of SPIs for this GIC instance. This is
+    GICv5-specific: unlike GICv2/v3, the value does not include SGIs or PPIs.
+    The value ranges from 32 to KVM's VGICv5 maximum of 1024 SPIs, in
+    increments of 32. If userspace does not set this attribute, KVM uses 32
+    SPIs by default.
+
+    kvm_device_attr.addr points to a __u32 value.
+
   KVM_DEV_ARM_VGIC_GRP_CTRL
    Attributes:
 
@@ -48,3 +75,239 @@ Groups:
     -EFAULT  Invalid guest ram access
     -EBUSY   One or more VCPUS are running
     =======  ========================================================
+
+  KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS
+   Attributes:
+
+    The attr field of kvm_device_attr encodes two values::
+
+      bits:     | 63      ....       32 | 31  ....  16 | 15  ....  0 |
+      values:   |         mpidr         |      RES     |    instr    |
+
+    The mpidr field encodes the CPU ID based on the affinity information in the
+    architecture defined MPIDR, and the field is encoded as follows::
+
+      | 63 .... 56 | 55 .... 48 | 47 .... 40 | 39 .... 32 |
+      |    Aff3    |    Aff2    |    Aff1    |    Aff0    |
+
+    The instr field encodes the system register to access based on the fields
+    defined in the A64 instruction set encoding for system register access
+    (RES means the bits are reserved for future use and should be zero)::
+
+      | 15 ... 14 | 13 ... 11 | 10 ... 7 | 6 ... 3 | 2 ... 0 |
+      |   Op 0    |    Op1    |    CRn   |   CRm   |   Op2   |
+
+    All system regs accessed through this API are (rw, 64-bit) and
+    kvm_device_attr.addr points to a __u64 value.
+
+    KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the
+    CPU specified by the mpidr field.
+
+    The available registers are:
+
+    =======================  ===================================================
+    ICC_ICSR_EL1
+    ICC_PPI_ENABLER0_EL1
+    ICC_PPI_ENABLER1_EL1
+    ICC_PPI_SACTIVER0_EL1    ICC_PPI_CACTIVER0_EL1 is not supported. Writes to
+                             ICC_PPI_SACTIVER0_EL1 are treated as RAW writes of
+                             the underlying state.
+    ICC_PPI_SACTIVER1_EL1    ICC_PPI_CACTIVER1_EL1 is not supported. Writes to
+                             ICC_PPI_SACTIVER1_EL1 are treated as RAW writes of
+                             the underlying state.
+    ICC_PPI_SPENDR0_EL1      ICC_PPI_CPENDR0_EL1 is not supported. Writes to
+                             ICC_PPI_SPENDR0_EL1 are treated as RAW writes of
+                             the underlying state.
+    ICC_PPI_SPENDR1_EL1      ICC_PPI_CPENDR1_EL1 is not supported. Writes to
+                             ICC_PPI_SPENDR1_EL1 are treated as RAW writes of
+                             the underlying state.
+    ICC_PPI_PRIORITYR0_EL1
+    ICC_PPI_PRIORITYR1_EL1
+    ICC_PPI_PRIORITYR2_EL1
+    ICC_PPI_PRIORITYR3_EL1
+    ICC_PPI_PRIORITYR4_EL1
+    ICC_PPI_PRIORITYR5_EL1
+    ICC_PPI_PRIORITYR6_EL1
+    ICC_PPI_PRIORITYR7_EL1
+    ICC_PPI_PRIORITYR8_EL1
+    ICC_PPI_PRIORITYR9_EL1
+    ICC_PPI_PRIORITYR10_EL1
+    ICC_PPI_PRIORITYR11_EL1
+    ICC_PPI_PRIORITYR12_EL1
+    ICC_PPI_PRIORITYR13_EL1
+    ICC_PPI_PRIORITYR14_EL1
+    ICC_PPI_PRIORITYR15_EL1
+    ICC_APR_EL1
+    ICC_CR0_EL1
+    ICC_PCR_EL1
+    =======================  ===================================================
+
+  Errors:
+
+    =======  =============================================================
+    -ENXIO   Getting or setting this register is not supported
+    -EBUSY   VCPU is running, or write attempted after a VCPU has run
+    -EINVAL  Invalid mpidr or register value supplied
+    =======  =============================================================
+
+
+  KVM_DEV_ARM_VGIC_GRP_IRS_REGS
+    Attributes:
+      The attr field of kvm_device_attr encodes the offset of the IRS register,
+      relative to the IRS CONFIG_FRAME base address. This is the address that
+      was provided via KVM_VGIC_V5_ADDR_TYPE_IRS when creating VGICv5 in the
+      first place.
+
+      kvm_device_attr.addr points to a __u64 value whatever the width
+      of the addressed register (32/64 bits). 64 bit registers can only
+      be accessed with full length.
+
+      Writes to read-only registers are ignored by the kernel except for
+      IRS_IDR0 - IRS_IDR2 and IRS_IDR5 - IRS_IDR7, which are sanity checked to
+      ensure that they match a sane config.
+
+      IRS_IDR3 and IRS_IDR4 are RAZ/WI, as nested virtualization is not
+      supported.
+
+      For registers without dedicated userspace accessors, getting or setting a
+      register uses the same emulated MMIO handlers as guest reads/writes.
+      Dedicated userspace accessors may instead save or restore migration state
+      without triggering guest-visible side effects. For example, restoring
+      IRS_IST_BASER only restores the emulated register state; any host LPI IST
+      allocation based on the restored IRS_IST_CFGR and IRS_IST_BASER state
+      happens when KVM_DEV_ARM_VGIC_GRP_IST is restored.
+
+      Once an LPI IST has been allocated, IRS_IST_CFGR is immutable. Userspace
+      may write back its current value, but setting a different value is
+      rejected with -EINVAL.
+
+      Once an LPI IST has been allocated, IRS_IST_BASER is immutable.
+      Userspace may write back its current value, but setting a different value
+      is rejected with -EINVAL.
+
+  Errors:
+
+    =======  =================================================================
+    -ENXIO   Offset does not correspond to any supported register
+    -EFAULT  Invalid user pointer for attr->addr
+    -EINVAL  Offset is not 32-bit aligned for 32-bit MMIO registers, not
+             64-bit aligned for 64-bit registers, or the supplied register
+             value is not compatible with the configured VGICv5 IRS state
+    -EBUSY   VGIC is not initialized, one or more VCPUs are running, or a
+             write is attempted after a VCPU has run
+    =======  =================================================================
+
+  KVM_DEV_ARM_VGIC_GRP_IST
+    Attributes:
+      This interface is used to either save the state of the IRS's Interrupt
+      State Tables (ISTs), or to restore them. A get operation saves IST state,
+      and a set operation restores IST state. kvm_device_attr.attr is reserved
+      and must be zero.
+
+      The VGIC must be initialized before using this interface. Restore must be
+      performed before the VM has run. For restore, userspace must have already
+      restored the IRS state needed to describe any guest LPI IST.
+
+      Saving first asks the IRS to save and quiesce the VM so that interrupt
+      state has been written back to the ISTs. KVM checks that the VM remains
+      quiesced while copying out the SPI and LPI IST state.
+
+      kvm_device_attr.addr points to a struct kvm_vgic_v5_ist::
+
+        struct kvm_vgic_v5_ist {
+                __u64   spi_ist_addr;
+                __u64   spi_ist_size;
+                __u64   lpi_ist_addr;
+                __u64   lpi_ist_size;
+        };
+
+      The SPI and LPI IST buffers contain one little-endian 32-bit IST entry per
+      interrupt, in interrupt number order. Only the architected 32-bit ISTE
+      state is exposed to userspace; host IST layout and metadata are private to
+      KVM.
+
+      The SPI IST buffer is required and its size must be:
+
+        nr_spis * sizeof(__u32)
+
+      where nr_spis is the value returned by KVM_DEV_ARM_VGIC_GRP_NR_IRQS for
+      the VGICv5 device. For VGICv5 this value is the number of SPIs, not the
+      total number of interrupts.
+
+      The LPI IST buffer is required if the guest's IRS_IST_CFGR and
+      IRS_IST_BASER state describes a valid LPI IST, and its size must be:
+
+        BIT(lpi_id_bits) * sizeof(__u32)
+
+      where lpi_id_bits is the LPI ID width configured in IRS_IST_CFGR. If no
+      LPI IST is configured, lpi_ist_addr and lpi_ist_size must both be zero.
+
+    Errors:
+
+      ===========  ============================================================
+      -EBUSY       One or more VCPUs are running, the VGIC is not initialized,
+                   restore was requested after the VM has run, an LPI IST
+                   already exists, or the save operation completed but the VM
+                   did not remain quiesced
+      -EINVAL      attr->addr is NULL, the userspace IST descriptor is missing
+                   a required buffer, supplies a non-zero LPI buffer when no
+                   LPI IST is configured, has a buffer size that does not match
+                   the configured VGICv5 IST state, or an internal VM table
+                   operation rejected the VM state
+      -EFAULT      Invalid user pointer for attr->addr, spi_ist_addr, or
+                   lpi_ist_addr
+      -ENXIO       Required per-VM VGICv5/IST backing state is missing or
+                   inconsistent
+      -ENOMEM      Restoring IST state failed while allocating the host LPI IST
+                   or tracking pending interrupts
+      -ETIMEDOUT   An IRS save/VM operation timed out
+      ===========  ============================================================
+
+IRS Save Sequence:
+------------------
+
+The following operations are required when saving the virtual GICv5 IRS:
+
+a) Save the ISTs by issuing KVM_GET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_IST.
+b) Save the IRS MMIO register state by issuing KVM_GET_DEVICE_ATTR on
+   KVM_DEV_ARM_VGIC_GRP_IRS_REGS.
+
+These two steps may be performed in either order. Saving the ISTs writes all
+IST state to userspace-provided buffers and does not update guest memory.
+
+IRS Restore Sequence:
+---------------------
+
+The following ordering must be followed when restoring the virtual GICv5 and
+IRS:
+
+a) Create vCPUs.
+b) Provide the IRS base address by issuing KVM_SET_DEVICE_ATTR on
+   KVM_DEV_ARM_VGIC_GRP_ADDR
+c) Restore the number of SPIs by issuing KVM_SET_DEVICE_ATTR on
+   KVM_DEV_ARM_VGIC_GRP_NR_IRQS.
+d) Initialise the GIC - this sets up the default state and creates the SPI
+   IST - by issuing KVM_SET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_CTRL with
+   KVM_DEV_ARM_VGIC_CTRL_INIT
+e) Restore the IRS MMIO register state by issuing KVM_SET_DEVICE_ATTR on
+   KVM_DEV_ARM_VGIC_GRP_IRS_REGS. The IRS registers may be restored in any
+   order, but any IRS_IDR* state and the IRS_IST_CFGR/IRS_IST_BASER state
+   describing an LPI IST must be restored before restoring the ISTs. KVM uses
+   the restored IRS_IST_CFGR and IRS_IST_BASER state to allocate the LPI IST
+   during the following step.
+f) Restore the ISTs by issuing KVM_SET_DEVICE_ATTR on
+   KVM_DEV_ARM_VGIC_GRP_IST.
+
+The number of SPIs must be restored before VGIC initialization because
+initialization allocates the SPI state and fixes the SPI range exposed by the
+IRS ID registers.
+
+The IRS's MMIO registers must be restored prior to restoring the ISTs as these
+are used to convey the number of LPIs the guest has configured to KVM.
+
+The various ``*_STATUSR`` registers are observational state in the current KVM
+implementation. Userspace may save them for validation or debugging purposes,
+but they are not required as restore input and do not need to be replayed during
+restore.
+
+Then vCPUs can be started.
diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst
index deb5c51bc00c87..a3f20a4ee4b100 100644
--- a/Documentation/virt/kvm/devices/vcpu.rst
+++ b/Documentation/virt/kvm/devices/vcpu.rst
@@ -135,7 +135,7 @@ after selecting a hardware PMU.
 Request that the VCPU uses the specified hardware PMU when creating guest events
 for the purpose of PMU emulation. The PMU identifier can be read from the "type"
 file for the desired PMU instance under /sys/devices (or, equivalent,
-/sys/bus/even_source). This attribute is particularly useful on heterogeneous
+/sys/bus/event_source). This attribute is particularly useful on heterogeneous
 systems where there are at least two CPU PMUs on the system. The PMU that is set
 for one VCPU will be used by all the other VCPUs. It isn't possible to set a PMU
 if a PMU event filter is already present.
diff --git a/Documentation/virt/kvm/devices/vm.rst b/Documentation/virt/kvm/devices/vm.rst
index a4d39fa1b0834b..78e63eb9994e64 100644
--- a/Documentation/virt/kvm/devices/vm.rst
+++ b/Documentation/virt/kvm/devices/vm.rst
@@ -280,6 +280,24 @@ Allows user space to disable dea key wrapping, clearing the wrapping key.
 :Parameters: none
 :Returns:    0
 
+4.5. ATTRIBUTE: KVM_S390_VM_CRYPTO_ENABLE_APIE (w/o)
+-------------------------------------------------------
+
+Allows user space to enable AP interpretive execution.
+
+:Parameters: none
+:Returns:    0 for success
+	     -EOPNOTSUPP if AP instructions not available for guest
+
+4.6. ATTRIBUTE: KVM_S390_VM_CRYPTO_DISABLE_APIE (w/o)
+-------------------------------------------------------
+
+Allows user space to disable AP interpretive execution.
+
+:Parameters: none
+:Returns:    0 for success
+	     -EOPNOTSUPP if AP instructions not available for guest
+
 5. GROUP: KVM_S390_VM_MIGRATION
 ===============================
 
diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst
index bd04a908a8dbdd..eeb0812cb10aba 100644
--- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst
+++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst
@@ -170,7 +170,7 @@ Returns: 0 on success, -negative on error
 
 ::
 
-        struct kvm_sev_launch_update {
+        struct kvm_sev_launch_update_data {
                 __u64 uaddr;    /* userspace address to be encrypted (must be 16-byte aligned) */
                 __u32 len;      /* length of the data to be encrypted (must be 16-byte aligned) */
         };
@@ -316,7 +316,7 @@ used by the guest owner with the KVM_SEV_LAUNCH_MEASURE.
 If len is zero on entry, the measurement blob length is written to len and
 uaddr is unused.
 
-Parameters (in): struct kvm_sev_attestation
+Parameters (in): struct kvm_sev_attestation_report
 
 Returns: 0 on success, -negative on error
 
@@ -377,7 +377,7 @@ Returns: 0 on success, -negative on error
 
 ::
 
-        struct kvm_sev_launch_send_update_data {
+        struct kvm_sev_send_update_data {
                 __u64 hdr_uaddr;        /* userspace address containing the packet header */
                 __u32 hdr_len;
 
@@ -447,7 +447,7 @@ Returns: 0 on success, -negative on error
 
 ::
 
-        struct kvm_sev_launch_receive_update_data {
+        struct kvm_sev_receive_update_data {
                 __u64 hdr_uaddr;        /* userspace address containing the packet header */
                 __u32 hdr_len;
 
@@ -503,7 +503,8 @@ secrets.
 
 It is required that the GPA ranges initialized by this command have had the
 KVM_MEMORY_ATTRIBUTE_PRIVATE attribute set in advance. See the documentation
-for KVM_SET_MEMORY_ATTRIBUTES for more details on this aspect.
+for KVM_SET_MEMORY_ATTRIBUTES/KVM_SET_MEMORY_ATTRIBUTES2 for more details on
+this aspect.
 
 Upon success, this command is not guaranteed to have processed the entire
 range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of
@@ -511,9 +512,14 @@ range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of
 remaining range that has yet to be processed. The caller should continue
 calling this command until those fields indicate the entire range has been
 processed, e.g. ``len`` is 0, ``gfn_start`` is equal to the last GFN in the
-range plus 1, and ``uaddr`` is the last byte of the userspace-provided source
-buffer address plus 1. In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO,
-``uaddr`` will be ignored completely.
+range plus 1, and ``uaddr`` (if specified) is the last byte of the
+userspace-provided source buffer address plus 1.
+
+In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, ``uaddr`` will be
+ignored completely. For all other page types, ``uaddr`` is optional if in-place
+conversion is enabled (i.e. when the data had been written directly to
+guest_memfd while the page was in the shared state) and is required if in-place
+conversion is disabled.
 
 Parameters (in): struct  kvm_sev_snp_launch_update
 
diff --git a/Documentation/virt/kvm/x86/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst
index 521ecf9a8a361a..e0b6c11f19ceae 100644
--- a/Documentation/virt/kvm/x86/hypercalls.rst
+++ b/Documentation/virt/kvm/x86/hypercalls.rst
@@ -109,7 +109,7 @@ The template for each hypercall is:
 Usage:
 
 a0: guest physical address where host copies
-"struct kvm_clock_offset" structure.
+"struct kvm_clock_pairing" structure.
 
 a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0)
 is supported (corresponding to the host's CLOCK_REALTIME clock).
diff --git a/Documentation/virt/kvm/x86/intel-tdx.rst b/Documentation/virt/kvm/x86/intel-tdx.rst
index 6a222e9d09541d..d8d9409120e614 100644
--- a/Documentation/virt/kvm/x86/intel-tdx.rst
+++ b/Documentation/virt/kvm/x86/intel-tdx.rst
@@ -158,6 +158,10 @@ KVM_TDX_INIT_MEM_REGION
 Initialize @nr_pages TDX guest private memory starting from @gpa with userspace
 provided data from @source_addr. @source_addr must be PAGE_SIZE-aligned.
 
+If guest_memfd in-place conversion is enabled, pass 0 for @source_addr
+to represent "no source page". A source page is required if in-place
+conversion is not enabled or not supported.
+
 Note, before calling this sub command, memory attribute of the range
 [gpa, gpa + nr_pages] needs to be private.  Userspace can use
 KVM_SET_MEMORY_ATTRIBUTES to set the attribute.
diff --git a/Documentation/virt/uml/nommu-uml.rst b/Documentation/virt/uml/nommu-uml.rst
new file mode 100644
index 00000000000000..105e596ae1cc50
--- /dev/null
+++ b/Documentation/virt/uml/nommu-uml.rst
@@ -0,0 +1,54 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. contents:: :local:
+
+Execution model
+===============
+
+When emulating a system with an MMU, UML gives every ``mm_struct`` its own
+host process with its own host address space, and keeps that address space
+in sync with the kernel's page tables by issuing ``mmap()``/``munmap()``
+into it.
+
+NOMMU has no per-process address spaces: there is a single "physical" address
+space that the kernel and every userspace process share.  UML models this with
+a **userspace runner** - an ordinary uml-userspace stub host process, started
+exactly like an MMU userspace process, but with the whole of "physical" memory
+(the physmem file) mapped into it at the kernel's addresses:
+
+- The runner ``exec()``'s the stub just like the MMU case.  The only extra step
+  is that, once it is up, the kernel maps the entire physmem file into it in one
+  go, at the same addresses the kernel uses.
+- Because NOMMU never remaps, that single mapping is all a runner ever needs:
+  there is no per-mm host process, no TLB sync and no ``mmap()`` of individual
+  page-table entries into a child.  Userspace code and data - placed in memory
+  by the loader - are directly executable and accessible.
+- The runner processes hold no per-task state: the kernel loads a task's state
+  on every entry, so there's just one process per CPU.
+
+Running a task's userspace therefore only means pointing a runner at that
+task's code and letting it run until the next syscall or signal
+
+Building and running
+====================
+
+Configure and build the ARCH=um kernel normally, but deselect CONFIG_MMU.
+
+KUnit tests can be run in NOMMU mode with::
+
+   ./tools/testing/kunit/kunit.py run \
+       --kconfig_add CONFIG_MMU=n --kconfig_add CONFIG_KUNIT_UML_PCI=n
+
+Running a normal userspace requires NOMMU-aware binaries.  There is no stock
+x86_64 NOMMU distribution, but a prebuilt Alpine image with musl-libc and
+busybox built for NOMMU is available and can be turned into a root image::
+
+   cid=$(docker create ghcr.io/thehajime/alpine:3.20.3-um-nommu)
+   docker export "$cid" > alpine.tar
+   docker rm "$cid"
+   mkdir alpine-root && tar xf alpine.tar -C alpine-root
+   mke2fs -q -F -t ext4 -d alpine-root alpine.ext4 1200M
+
+Then boot it::
+
+   ./linux ubd0=./alpine.ext4 root=/dev/ubda rw mem=1024m init=/sbin/init
diff --git a/Documentation/watchdog/watchdog-parameters.rst b/Documentation/watchdog/watchdog-parameters.rst
index 502cb6adbeda74..348229ffcb43f0 100644
--- a/Documentation/watchdog/watchdog-parameters.rst
+++ b/Documentation/watchdog/watchdog-parameters.rst
@@ -515,6 +515,8 @@ sbsa_gwdt:
     nowayout:
 	Watchdog cannot be stopped once started
 	(default=kernel config parameter)
+    early_enable:
+	Watchdog is started on module insertion (default=0)
 
 -------------------------------------------------
 
diff --git a/Documentation/wmi/devices/acer-wmi-battery.rst b/Documentation/wmi/devices/acer-wmi-battery.rst
new file mode 100644
index 00000000000000..8aa3e1334490ed
--- /dev/null
+++ b/Documentation/wmi/devices/acer-wmi-battery.rst
@@ -0,0 +1,70 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+====================================================
+Acer WMI Battery interface driver (acer-wmi-battery)
+====================================================
+
+Introduction
+============
+
+Some Acer laptops support setting battery charge control limits and allow a
+user to calibrate their battery.
+
+WMI interface description
+=========================
+
+The WMI interface description can be decoded from the embedded binary MOF (bmof)
+data using the `bmfdec <https://github.com/pali/bmfdec>`_ utility:
+
+::
+
+  [WMI, Dynamic, Provider("WmiProv"), Locale("MS\\0x409"), Description("Class used to control smart battery, Version 2.88"), guid("{79772EC5-04B1-4bfd-843C-61E7F77B6CC9}")]
+  class BatteryControl {
+    [key, read] string InstanceName;
+    [read] boolean Active;
+
+    [WmiMethodId(19), Implemented, read, write, Description("Get battery Information Interface.")] void GetBattInfoInterface([in] uint32 uBatteryInfoIndex, [in] uint32 uBatteryNo, [out] uint32 uReturn);
+    [WmiMethodId(20), Implemented, read, write, Description("Get Battery Health Control Status.")] void GetBatteryHealthControlStatus([in] uint8 uBatteryNo, [in] uint8 uFunctionQuery, [in] uint8 uReserved[2], [out] uint8 uFunctionList, [out] uint8 uReturn[2], [out] uint8 uFunctionStatus[5]);
+    [WmiMethodId(21), Implemented, read, write, Description("Set Battery Health Control.")] void SetBatteryHealthControl([in] uint8 uBatteryNo, [in] uint8 uFunctionMask, [in] uint8 uFunctionStatus, [in] uint8 uReservedIn[5], [out] uint16 uReturn, [out] uint16 uReservedOut);
+    [WmiMethodId(22), Implemented, read, write, Description("Get Battery Function Data.")] void GetBatteryFunctionData([in] uint8 uFunctionMask, [in] uint8 uReservedIn[3], [out] uint8 uReturnCode[2], [out] uint8 uBACStartTime[2], [out] uint8 uBACStopTime[2], [out] uint8 uBACStatus, [out] uint8 uReservedOut[9]);
+    [WmiMethodId(23), Implemented, read, write, Description("Set Battery Function Data.")] void SetBatteryFunctionData([in] uint8 uFunctionMask, [in] uint8 uBACSwitch, [in] uint8 uReservedIn[6], [out] uint16 uReturnCode, [out] uint8 uReservedOut[2]);
+  };
+
+For all methods the ``uBatteryNo`` is always ``0x1``, there are no known Acer
+laptop models with two batteries.
+
+WMI method GetBatteryInfoInterface()
+------------------------------------
+
+Returns additional battery information, the data seems to be based on the
+"Smart Battery Data Specification". The only known value for
+``uBatteryInfoIndex`` is ``0x8``, the battery temperature.
+
+WMI method GetBatteryHealthControlStatus()
+------------------------------------------
+
+This interface does not exist on all Acer laptops, it returns whether battery
+charge limiting or calibration mode is enabled.
+
+When calling this method, ``uFunctionList`` is a bitmask with known values:
+
++--------------------+------------------------------------+
+| Value              | Description                        |
++====================+====================================+
+| 0x01               | Health mode supported              |
++--------------------+------------------------------------+
+| 0x02               | Calibration mode supported         |
++--------------------+------------------------------------+
+
+``uFunctionStatus`` is a 5 byte array known elements are ``0`` for health mode
+and ``1`` for calibration mode. A non-zero value means the mode is enabled.
+
+WMI method SetBatteryHealthControl()
+------------------------------------
+
+This interface does not exist on all Acer laptops, it enables battery
+charge limiting or battery calibration.
+
+The argument ``uFunctionMask`` is known to accept either ``0x01`` for setting
+health mode or ``0x02`` for calibration mode; other values are unknown.
+``uFunctionStatus`` is an uint8 value, a non-zero value enables the given mode.
diff --git a/Documentation/wmi/devices/bitland-mifs-wmi.rst b/Documentation/wmi/devices/bitland-mifs-wmi.rst
index 9e86ecc2993cd5..27c3fd248279b7 100644
--- a/Documentation/wmi/devices/bitland-mifs-wmi.rst
+++ b/Documentation/wmi/devices/bitland-mifs-wmi.rst
@@ -67,7 +67,7 @@ WMI Methods (MICommonInterface)
 
 The ``MICommonInterface`` class (GUID: ``{b60bfb48-3e5b-49e4-a0e9-8cffe1b3434b}``)
 is the primary control interface. It uses a 32-byte buffer for both input
-(``InData``) and output (``OutData``).
+(``InData``) and output (``OutData`` + ``Reserved``).
 
 Method Structure
 ----------------
@@ -77,53 +77,78 @@ The data packet follows a standardized format:
 +----------+------------------------------------------------------------------+
 | Byte     | Description                                                      |
 +==========+==================================================================+
-| 1        | Method Type: Get (0xFA / 250) or Set (0xFB / 251)                |
+| 1 and 2  | Method Type or Return Code                                       |
 +----------+------------------------------------------------------------------+
-| 3        | Command ID (Method Name)                                         |
+| 3 and 4  | Command ID (Method Name)                                         |
 +----------+------------------------------------------------------------------+
-| 4 - 31   | Arguments (for Set) or Return Data (for Get)                     |
+| 5 - 32   | Arguments (for Set) or Return Data (for Get)                     |
 +----------+------------------------------------------------------------------+
 
+Method Types
+------------
+
+The following Method types are understood by the underlying firmware:
+
++--------+---------+
+| Type   | Meaning |
++========+=========+
+| 0xFA00 | Read    |
++--------+---------+
+| 0xFB00 | Write   |
++--------+---------+
+
+Return Codes
+------------
+
+The following Return Codes are know to be returned in response to a WMI method invocation:
+
++--------+--------------------+
+| Code   | Meaning            |
++========+====================+
+| 0x8000 | Success            |
++--------+--------------------+
+| 0xE000 | Invalid Command ID |
++--------+--------------------+
 
 Command IDs
 -----------
 
-The following Command IDs are used in the third byte of the buffer:
+The following Command IDs know to be used on some models:
 
 +----------+-----------------------+------------------------------------------+
 | ID       | Name                  | Values / Description                     |
 +==========+=======================+==========================================+
-| 8        | SystemPerMode         | 0: Balance, 1: Performance, 2: Quiet,    |
+| 0x0800   | SystemPerMode         | 0: Balance, 1: Performance, 2: Quiet,    |
 |          |                       | 3: Full-speed                            |
 +----------+-----------------------+------------------------------------------+
-| 9        | GPUMode               | 0: Hybrid, 1: Discrete, 2: UMA           |
+| 0x0900   | GPUMode               | 0: Hybrid, 1: Discrete, 2: UMA           |
 +----------+-----------------------+------------------------------------------+
-| 10       | KeyboardType          | 0: White, 1: Single RGB, 2: Zone RGB     |
+| 0x0A00   | KeyboardType          | 0: White, 1: Single RGB, 2: Zone RGB     |
 +----------+-----------------------+------------------------------------------+
-| 11       | FnLock                | 0: Off, 1: On                            |
+| 0x0B00   | FnLock                | 0: Off, 1: On                            |
 +----------+-----------------------+------------------------------------------+
-| 12       | TPLock                | 0: Unlock, 1: Lock (Touchpad)            |
+| 0x0C00   | TPLock                | 0: Unlock, 1: Lock (Touchpad)            |
 +----------+-----------------------+------------------------------------------+
-| 13       | CPUGPUSYSFanSpeed     | Returns 12 bytes of fan data:            |
+| 0x0D00   | CPUGPUSYSFanSpeed     | Returns 12 bytes of fan data:            |
 |          |                       | Bytes 4-5: CPU Fan RPM (Little Endian)   |
 |          |                       | Bytes 6-7: GPU Fan RPM (Little Endian)   |
 |          |                       | Bytes 10-11: SYS Fan RPM (Little Endian) |
 +----------+-----------------------+------------------------------------------+
-| 16       | RGBKeyboardMode       | 0: Off, 1: Auto Cyclic, 2: Fixed,        |
+| 0x1000   | RGBKeyboardMode       | 0: Off, 1: Auto Cyclic, 2: Fixed,        |
 |          |                       | 3: Custom                                |
 +----------+-----------------------+------------------------------------------+
-| 17       | RGBKeyboardColor      | Bytes 4, 5, 6: Red, Green, Blue values   |
+| 0x1100   | RGBKeyboardColor      | Bytes 4, 5, 6: Red, Green, Blue values   |
 +----------+-----------------------+------------------------------------------+
-| 18       | RGBKeyboardBrightness | 0-10: Brightness Levels, 128: Auto       |
+| 0x1200   | RGBKeyboardBrightness | 0-10: Brightness Levels, 128: Auto       |
 +----------+-----------------------+------------------------------------------+
-| 19       | SystemAcType          | 1: Type-C, 2: Circular Hole (DC)         |
+| 0x1300   | SystemAcType          | 1: Type-C, 2: Circular Hole (DC)         |
 +----------+-----------------------+------------------------------------------+
-| 20       | MaxFanSpeedSwitch     | Byte 4: Fan Type (0: CPU/GPU, 1: SYS)    |
+| 0x1400   | MaxFanSpeedSwitch     | Byte 4: Fan Type (0: CPU/GPU, 1: SYS)    |
 |          |                       | Byte 5: State (0: Off, 1: On)            |
 +----------+-----------------------+------------------------------------------+
-| 21       | MaxFanSpeed           | Sets manual fan speed duty cycle         |
+| 0x1500   | MaxFanSpeed           | Sets manual fan speed duty cycle         |
 +----------+-----------------------+------------------------------------------+
-| 22       | CPUThermometer        | Returns CPU Temperature                  |
+| 0x1600   | CPUThermometer        | Returns CPU Temperature                  |
 +----------+-----------------------+------------------------------------------+
 
 WMI Events (HID_EVENT20)
@@ -189,7 +214,8 @@ Performance Modes
 Changing the performance mode via Command ID 0x08 (SystemPerMode) affects the
 power limits (PL1/PL2) and fan curves managed by the Embedded Controller (EC).
 Note that the "Full-speed" and "Performance" mode (1, 3) is typically only
-available when the system is connected to a DC power source (not USB-C/PD).
+available when the system is connected to a DC power source (not USB-C/PD on
+model with a dedicated DC connector).
 
 In the driver implementation, switch to performance/full-speed mode without
 DC power connected will throw the EOPNOTSUPP error.
diff --git a/MAINTAINERS b/MAINTAINERS
index cc3cae2e378b34..30c1cdd0fb3854 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -113,7 +113,7 @@ F:	net/6lowpan/
 M:	Johannes Berg <johannes@sipsolutions.net>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/
+W:	https://wireless.docs.kernel.org/en/latest/
 Q:	https://patchwork.kernel.org/project/linux-wireless/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git
@@ -640,6 +640,7 @@ F:	drivers/iio/accel/adxl355_spi.c
 
 ADXL367 THREE-AXIS DIGITAL ACCELEROMETER DRIVER
 M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
+M:	Antoniu Miclaus <antoniu.miclaus@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -767,6 +768,14 @@ F:	Documentation/devicetree/bindings/phy/airoha,en7581-pcie-phy.yaml
 F:	drivers/phy/phy-airoha-pcie-regs.h
 F:	drivers/phy/phy-airoha-pcie.c
 
+AIROHA SOUND DRIVER
+M:	Christian Marangi <ansuelsmth@gmail.com>
+L:	linux-sound@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/sound/airoha,an7581-afe.yaml
+F:	Documentation/devicetree/bindings/sound/airoha,an7581-wm8960.yaml
+F:	sound/soc/mediatek/an7581/*
+
 AIROHA SPI SNFI DRIVER
 M:	Lorenzo Bianconi <lorenzo@kernel.org>
 M:	Ray Liu <ray.liu@airoha.com>
@@ -1218,7 +1227,7 @@ AMD KFD
 M:	Felix Kuehling <Felix.Kuehling@amd.com>
 L:	amd-gfx@lists.freedesktop.org
 S:	Supported
-T:	git https://gitlab.freedesktop.org/agd5f/linux.git
+T:	git https://gitlab.freedesktop.org/drm/amdgpu/kernel.git
 F:	drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd*.[ch]
 F:	drivers/gpu/drm/amd/amdkfd/
 F:	drivers/gpu/drm/amd/include/cik_structs.h
@@ -1468,6 +1477,7 @@ L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/dac/adi,ad3530r.yaml
+F:	Documentation/devicetree/bindings/iio/dac/adi,ad5710r.yaml
 F:	drivers/iio/dac/ad3530r.c
 
 ANALOG DEVICES INC AD3552R DRIVER
@@ -1517,6 +1527,7 @@ F:	drivers/iio/adc/ad4080.c
 
 ANALOG DEVICES INC AD4130 DRIVER
 M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
+M:	Antoniu Miclaus <antoniu.miclaus@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -1571,6 +1582,14 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/adc/adi,ad4851.yaml
 F:	drivers/iio/adc/ad4851.c
 
+ANALOG DEVICES INC AD5529R DRIVER
+M:	Janani Sunil <janani.sunil@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/dac/adi,ad5529r.yaml
+F:	drivers/iio/dac/ad5529r.c
+
 ANALOG DEVICES INC AD5706R DRIVER
 M:	Alexis Czezar Torreno <alexisczezar.torreno@analog.com>
 L:	linux-iio@vger.kernel.org
@@ -1622,6 +1641,7 @@ F:	drivers/iio/dac/ad7293.c
 
 ANALOG DEVICES INC AD74115 DRIVER
 M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
+M:	Antoniu Miclaus <antoniu.miclaus@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -1630,6 +1650,7 @@ F:	drivers/iio/addac/ad74115.c
 
 ANALOG DEVICES INC AD74413R DRIVER
 M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
+M:	Antoniu Miclaus <antoniu.miclaus@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -1657,6 +1678,15 @@ F:	Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml
 F:	drivers/iio/adc/ad7768-1.c
 F:	include/dt-bindings/iio/adc/adi,ad7768-1.h
 
+ANALOG DEVICES INC AD7768 DRIVER
+M:	Janani Sunil <janani.sunil@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/adc/adi,ad7768.yaml
+F:	Documentation/iio/ad7768.rst
+F:	drivers/iio/adc/ad7768.c
+
 ANALOG DEVICES INC AD7780 DRIVER
 M:	Michael Hennerich <Michael.Hennerich@analog.com>
 M:	Renato Lui Geh <renatogeh@gmail.com>
@@ -1702,6 +1732,17 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/dac/adi,ad9739a.yaml
 F:	drivers/iio/dac/ad9739a.c
 
+ANALOG DEVICES INC AD9910 DRIVER
+M:	Rodrigo Alencar <rodrigo.alencar@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/ABI/testing/debugfs-iio-ad9910
+F:	Documentation/ABI/testing/sysfs-bus-iio-frequency-ad9910
+F:	Documentation/devicetree/bindings/iio/frequency/adi,ad9910.yaml
+F:	Documentation/iio/ad9910.rst
+F:	drivers/iio/frequency/ad9910.c
+
 ANALOG DEVICES INC MAX22007 DRIVER
 M:	Janani Sunil <janani.sunil@analog.com>
 L:	linux-iio@vger.kernel.org
@@ -2010,6 +2051,15 @@ M:	Jiaxun Yang <jiaxun.yang@flygoat.com>
 S:	Supported
 F:	drivers/rtc/rtc-goldfish.c
 
+ANLOGIC DR1V90 CRU DRIVER
+M:	Junhui Liu <junhui.liu@pigmoral.tech>
+S:	Maintained
+F:	Documentation/devicetree/bindings/clock/anlogic,dr1v90-cru.yaml
+F:	drivers/clk/anlogic/cru?dr1*
+F:	drivers/reset/reset-dr1v90.c
+F:	include/dt-bindings/clock/anlogic,dr1v90-cru.h
+F:	include/dt-bindings/reset/anlogic,dr1v90-cru.h
+
 AOA (Apple Onboard Audio) ALSA DRIVER
 M:	Johannes Berg <johannes@sipsolutions.net>
 L:	linuxppc-dev@lists.ozlabs.org
@@ -2076,8 +2126,7 @@ S:	Supported
 F:	arch/arm64/boot/dts/apm/
 
 APPLIED MICRO (APM) X-GENE SOC EDAC
-M:	Khuong Dinh <khuong@os.amperecomputing.com>
-S:	Supported
+S:	Orphan
 F:	Documentation/devicetree/bindings/edac/apm,xgene-edac.yaml
 F:	drivers/edac/xgene_edac.c
 
@@ -2177,7 +2226,7 @@ F:	drivers/net/arcnet/
 F:	include/uapi/linux/if_arcnet.h
 
 ARCTIC FAN CONTROLLER DRIVER
-M:	Aureo Serrano de Souza <aureo.serrano@arctic.de>
+M:	ARCTIC (HK) <arctic.it.hk@arctic.de>
 L:	linux-hwmon@vger.kernel.org
 S:	Maintained
 F:	Documentation/hwmon/arctic_fan_controller.rst
@@ -2333,6 +2382,7 @@ ARM MALI TYR DRM DRIVER
 M:	Daniel Almeida <daniel.almeida@collabora.com>
 M:	Alice Ryhl <aliceryhl@google.com>
 L:	dri-devel@lists.freedesktop.org
+L:	rust-for-linux@vger.kernel.org
 S:	Supported
 W:	https://rust-for-linux.com/tyr-gpu-driver
 W:	https://drm.pages.freedesktop.org/maintainer-tools/drm-rust.html
@@ -3432,6 +3482,7 @@ F:	include/linux/soc/qcom/
 ARM/QUALCOMM SUPPORT
 M:	Bjorn Andersson <andersson@kernel.org>
 M:	Konrad Dybcio <konradybcio@kernel.org>
+R:	Abel Vesa <abelvesa@kernel.org>
 L:	linux-arm-msm@vger.kernel.org
 S:	Maintained
 C:	irc://irc.oftc.net/linux-msm
@@ -3454,6 +3505,7 @@ F:	drivers/watchdog/gunyah_wdt.c
 F:	include/dt-bindings/arm/qcom,ids.h
 F:	include/dt-bindings/firmware/qcom,scm.h
 F:	include/dt-bindings/soc/qcom*
+F:	include/dt-bindings/thermal/qcom,pas.h
 F:	include/linux/firmware/qcom
 F:	include/linux/soc/qcom/
 F:	include/soc/qcom/
@@ -3789,6 +3841,7 @@ M:	Vignesh Raghavendra <vigneshr@ti.com>
 M:	Tero Kristo <kristo@kernel.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Supported
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git
 F:	Documentation/devicetree/bindings/arm/ti/k3.yaml
 F:	Documentation/devicetree/bindings/hwinfo/ti,k3-socinfo.yaml
 F:	arch/arm64/boot/dts/ti/Makefile
@@ -3949,10 +4002,14 @@ R:	Mark Rutland <mark.rutland@arm.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git
+F:	Documentation/ABI/testing/sysfs-firmware-cca
 F:	Documentation/arch/arm64/
 F:	arch/arm64/
+F:	drivers/firmware/arm_rmm/
 F:	drivers/virt/coco/arm-cca-guest/
 F:	drivers/virt/coco/pkvm-guest/
+F:	include/linux/arm-rsi-cmds.h
+F:	include/linux/arm-smccc-rsi.h
 F:	tools/testing/selftests/arm64/
 X:	arch/arm64/boot/dts/
 X:	arch/arm64/configs/defconfig
@@ -4260,13 +4317,13 @@ M:	Nick Kossifidis <mickflemm@gmail.com>
 M:	Luis Chamberlain <mcgrof@kernel.org>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/ath5k
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/ath5k.html
 F:	drivers/net/wireless/ath/ath5k/
 
 ATHEROS ATH6KL WIRELESS DRIVER
 L:	linux-wireless@vger.kernel.org
 S:	Orphan
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/ath6kl
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/ath6kl.html
 F:	drivers/net/wireless/ath/ath6kl/
 
 ATI_REMOTE2 DRIVER
@@ -4346,6 +4403,13 @@ F:	drivers/net/ieee802154/at86rf230.h
 F:	drivers/net/ieee802154/atusb.c
 F:	drivers/net/ieee802154/atusb.h
 
+AUDIENT EVO MIXER DRIVER
+M:	Christian Ruppert <arc@gmx.li>
+L:	linux-sound@vger.kernel.org
+S:	Maintained
+F:	sound/usb/mixer_evo.c
+F:	sound/usb/mixer_evo.h
+
 AUDIT SUBSYSTEM
 M:	Paul Moore <paul@paul-moore.com>
 M:	Eric Paris <eparis@redhat.com>
@@ -4439,6 +4503,24 @@ S:	Orphan
 F:	Documentation/devicetree/bindings/sound/axentia,*
 F:	sound/soc/atmel/tse850-pcm5142.c
 
+AXIADO SARADC DRIVER
+M:	Petar Stepanovic <pstepanovic@axiado.com>
+M:	Akhila Kavi <akavi@axiado.com>
+M:	Prasad Bolisetty <pbolisetty@axiado.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/adc/axiado,ax3000-saradc.yaml
+F:	drivers/iio/adc/axiado_saradc.c
+
+AXIADO SGPIO DRIVER
+M:	Petar Stepanovic <pstepanovic@axiado.com>
+M:	SriNavmani A <srinavmani@axiado.com>
+M:	Prasad Bolisetty <pbolisetty@axiado.com>
+L:	linux-gpio@vger.kernel.org
+S:	Supported
+F:	Documentation/devicetree/bindings/gpio/axiado,ax3005-sgpio.yaml
+F:	drivers/gpio/gpio-axiado-sgpio.c
+
 AXIS ARTPEC ARM64 SoC SUPPORT
 M:	Jesper Nilsson <jesper.nilsson@axis.com>
 M:	Lars Persson <lars.persson@axis.com>
@@ -4497,6 +4579,16 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/pwm/adi,axi-pwmgen.yaml
 F:	drivers/pwm/pwm-axi-pwmgen.c
 
+AXIADO PWM FAN CONTROLLER DRIVER
+M:	Petar Stepanovic <pstepanovic@axiado.com>
+M:	Akhila Kavi <akavi@axiado.com>
+M:	Prasad Bolisetty <pbolisetty@axiado.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Supported
+F:	Documentation/devicetree/bindings/hwmon/axiado,ax3000-pwm-fan.yaml
+F:	Documentation/hwmon/axiado-pwm-fan.rst
+F:	drivers/hwmon/axiado-pwm-fan.c
+
 AXIADO SPI DB DRIVER
 M:	Vladimir Moravcevic <vmoravcevic@axiado.com>
 M:	Tzu-Hao Wei <twei@axiado.com>
@@ -4508,6 +4600,16 @@ F:	Documentation/devicetree/bindings/spi/axiado,ax3000-spi.yaml
 F:	drivers/spi/spi-axiado.c
 F:	drivers/spi/spi-axiado.h
 
+AXIADO TSADC DRIVER
+M:	Petar Stepanovic <pstepanovic@axiado.com>
+M:	Akhila Kavi <akavi@axiado.com>
+M:	Prasad Bolisetty <pbolisetty@axiado.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Supported
+F:	Documentation/devicetree/bindings/hwmon/axiado,ax3000-tsadc.yaml
+F:	Documentation/hwmon/axiado-tsadc.rst
+F:	drivers/hwmon/axiado-tsadc.c
+
 AYANEO PLATFORM EC DRIVER
 M:	Antheas Kapenekakis <lkml@antheas.dev>
 L:	platform-driver-x86@vger.kernel.org
@@ -4535,14 +4637,14 @@ B43 WIRELESS DRIVER
 L:	linux-wireless@vger.kernel.org
 L:	b43-dev@lists.infradead.org
 S:	Orphan
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/b43
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/b43.html
 F:	drivers/net/wireless/broadcom/b43/
 
 B43LEGACY WIRELESS DRIVER
 L:	linux-wireless@vger.kernel.org
 L:	b43-dev@lists.infradead.org
 S:	Orphan
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/b43
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/b43.html
 F:	drivers/net/wireless/broadcom/b43legacy/
 
 BACKLIGHT CLASS/SUBSYSTEM
@@ -4627,17 +4729,11 @@ S:	Odd Fixes
 F:	Documentation/block/bfq-iosched.rst
 F:	block/bfq-*
 
-BFS FILE SYSTEM
-M:	"Tigran A. Aivazian" <aivazian.tigran@gmail.com>
-S:	Maintained
-F:	Documentation/filesystems/bfs.rst
-F:	fs/bfs/
-F:	include/uapi/linux/bfs_fs.h
-
 BITMAP API
 M:	Yury Norov <yury.norov@gmail.com>
 R:	Rasmus Villemoes <linux@rasmusvillemoes.dk>
 S:	Maintained
+F:	include/linux/bitfield-fix-width.h
 F:	include/linux/bitfield.h
 F:	include/linux/bitmap-str.h
 F:	include/linux/bitmap.h
@@ -4658,6 +4754,7 @@ F:	lib/find_bit_benchmark.c
 F:	lib/region_alloc_benchmark.c
 F:	lib/test_bitmap.c
 F:	lib/tests/cpumask_kunit.c
+F:	tools/include/linux/bitfield-fix-width.h
 F:	tools/include/linux/bitfield.h
 F:	tools/include/linux/bitmap.h
 F:	tools/include/linux/bits.h
@@ -4936,8 +5033,6 @@ BPF [CRYPTO]
 M:	Vadim Fedorenko <vadim.fedorenko@linux.dev>
 L:	bpf@vger.kernel.org
 S:	Maintained
-F:	crypto/bpf_crypto_skcipher.c
-F:	include/linux/bpf_crypto.h
 F:	kernel/bpf/crypto.c
 
 BPF [DOCUMENTATION] (Related to Standardization)
@@ -5143,6 +5238,7 @@ L:	netdev@vger.kernel.org
 L:	openwrt-devel@lists.openwrt.org (subscribers-only)
 S:	Supported
 F:	Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml
+F:	Documentation/networking/dsa/b53.rst
 F:	drivers/net/dsa/b53/*
 F:	drivers/net/dsa/bcm_sf2*
 F:	include/linux/dsa/brcm.h
@@ -5166,6 +5262,18 @@ N:	bcm2712
 N:	bcm283*
 N:	raspberrypi
 
+BROADCOM BCM2712 IOMMU DRIVER
+M:	Daniel Drake <dan@reactivated.net>
+M:	Florian Fainelli <florian.fainelli@broadcom.com>
+M:	Jim Quinlan <james.quinlan@broadcom.com>
+R:	Broadcom internal kernel review list <bcm-kernel-feedback-list@broadcom.com>
+L:	iommu@lists.linux.dev
+L:	linux-rpi-kernel@lists.infradead.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu*
+F:	drivers/iommu/bcm2712-iommu*
+F:	drivers/iommu/generic_pt/fmt/*bcm2712*
+
 BROADCOM BCM281XX/BCM11XXX/BCM216XX ARM ARCHITECTURE
 M:	Florian Fainelli <florian.fainelli@broadcom.com>
 M:	Ray Jui <rjui@broadcom.com>
@@ -5454,6 +5562,7 @@ F:	include/linux/brcmphy.h
 BROADCOM GENET ETHERNET DRIVER
 M:	Doug Berger <opendmb@gmail.com>
 M:	Florian Fainelli <florian.fainelli@broadcom.com>
+M:	Nicolai Buchwitz <nb@tipi-net.de>
 R:	Broadcom internal kernel review list <bcm-kernel-feedback-list@broadcom.com>
 L:	netdev@vger.kernel.org
 S:	Maintained
@@ -5919,7 +6028,7 @@ CARL9170 LINUX COMMUNITY WIRELESS DRIVER
 M:	Christian Lamparter <chunkeey@googlemail.com>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/carl9170
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/carl9170.html
 F:	drivers/net/wireless/ath/carl9170/
 
 CAVIUM I2C DRIVER
@@ -6500,7 +6609,7 @@ M:	Jerome Brunet <jbrunet+clk@baylibre.com>
 L:	linux-clk@vger.kernel.org
 S:	Maintained
 Q:	http://patchwork.kernel.org/project/linux-clk/list/
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk-next
 F:	Documentation/devicetree/bindings/clock/
 F:	drivers/clk/
 F:	include/dt-bindings/clock/
@@ -6567,6 +6676,7 @@ M:	Vishal Verma <vishal.l.verma@intel.com>
 M:	Dan Williams <djbw@kernel.org>
 R:	Ira Weiny <iweiny@kernel.org>
 R:	Li Ming <ming.li@zohomail.com>
+R:	Richard Cheng <icheng@nvidia.com>
 L:	linux-cxl@vger.kernel.org
 S:	Maintained
 F:	Documentation/driver-api/cxl
@@ -6603,6 +6713,7 @@ S:	Supported
 F:	fs/configfs/
 F:	include/linux/configfs.h
 F:	samples/configfs/
+F:	tools/testing/selftests/filesystems/configfs/
 
 CONFIGFS [RUST]
 M:	Andreas Hindborg <a.hindborg@kernel.org>
@@ -6716,6 +6827,16 @@ F:	tools/testing/selftests/cgroup/test_hugetlb_memcg.c
 F:	tools/testing/selftests/cgroup/test_kmem.c
 F:	tools/testing/selftests/cgroup/test_memcontrol.c
 
+COREBOOT FIRMWARE DRIVERS
+M:	Tzung-Bi Shih <tzungbi@kernel.org>
+R:	Brian Norris <briannorris@chromium.org>
+R:	Julius Werner <jwerner@chromium.org>
+L:	chrome-platform@lists.linux.dev
+S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git
+F:	drivers/firmware/coreboot/
+F:	include/linux/coreboot.h
+
 CORETEMP HARDWARE MONITORING DRIVER
 L:	linux-hwmon@vger.kernel.org
 S:	Orphan
@@ -6979,6 +7100,7 @@ L:	linux-crypto@vger.kernel.org
 S:	Maintained
 T:	git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-next
 T:	git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-fixes
+F:	Documentation/crypto/libcrypto*
 F:	lib/crypto/
 F:	scripts/crypto/
 
@@ -7494,6 +7616,7 @@ F:	rust/kernel/io.rs
 F:	rust/kernel/io/
 F:	rust/kernel/irq.rs
 F:	rust/kernel/irq/
+F:	rust/macros/io/
 
 DEVICE RESOURCE MANAGEMENT HELPERS
 M:	Hans de Goede <hansg@kernel.org>
@@ -7557,11 +7680,12 @@ F:	drivers/iio/chemical/sen0322.c
 
 DH ELECTRONICS DHSOM SOM AND BOARD SUPPORT
 M:	Christoph Niedermaier <cniedermaier@dh-electronics.com>
-M:	Marek Vasut <marex@denx.de>
+M:	Marek Vasut <marex@nabladev.com>
 L:	kernel@dh-electronics.com
 S:	Maintained
 N:	dhcom
 N:	dhcor
+N:	dhcos
 N:	dhsom
 
 DIALOG SEMICONDUCTOR DRIVERS
@@ -7680,6 +7804,7 @@ F:	fs/dlm/
 DMA BUFFER SHARING FRAMEWORK
 M:	Sumit Semwal <sumit.semwal@linaro.org>
 M:	Christian König <christian.koenig@amd.com>
+M:	Philipp Stanner <phasta@kernel.org>
 L:	linux-media@vger.kernel.org
 L:	dri-devel@lists.freedesktop.org
 L:	linaro-mm-sig@lists.linaro.org (moderated for non-subscribers)
@@ -7693,6 +7818,8 @@ F:	include/linux/dma-buf.h
 F:	include/linux/dma-buf/
 F:	include/linux/dma-resv.h
 F:	rust/helpers/dma-resv.c
+F:	rust/helpers/dma_fence.c
+F:	rust/kernel/dma_buf/
 K:	\bdma_(?:buf|fence|resv)\b
 
 DMA GENERIC OFFLOAD ENGINE SUBSYSTEM
@@ -7728,6 +7855,7 @@ F:	include/linux/dma-direct.h
 F:	include/linux/dma-map-ops.h
 F:	include/linux/dma-mapping.h
 F:	include/linux/swiotlb.h
+F:	include/trace/events/dma.h
 F:	kernel/dma/
 
 DMA MAPPING & SCATTERLIST API [RUST]
@@ -7919,6 +8047,7 @@ S:	Supported
 F:	Documentation/devicetree/bindings/dpll/dpll-device.yaml
 F:	Documentation/devicetree/bindings/dpll/dpll-pin.yaml
 F:	Documentation/driver-api/dpll.rst
+F:	Documentation/netlink/specs/dpll.yaml
 F:	drivers/dpll/
 F:	include/linux/dpll.h
 F:	include/uapi/linux/dpll.h
@@ -8168,7 +8297,6 @@ F:	drivers/gpu/drm/tiny/hx8357d.c
 
 DRM DRIVER FOR HYPERV SYNTHETIC VIDEO DEVICE
 M:	Dexuan Cui <decui@microsoft.com>
-M:	Long Li <longli@microsoft.com>
 M:	Saurabh Sengar <ssengar@linux.microsoft.com>
 L:	linux-hyperv@vger.kernel.org
 L:	dri-devel@lists.freedesktop.org
@@ -8340,6 +8468,13 @@ T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	Documentation/devicetree/bindings/display/panel/novatek,nt36523.yaml
 F:	drivers/gpu/drm/panel/panel-novatek-nt36523.c
 
+DRM DRIVER FOR NOVATEK NT36532 PANELS
+M:	Junjie Cao <junjie.cao@linux.dev>
+S:	Maintained
+T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
+F:	Documentation/devicetree/bindings/display/panel/novatek,nt36532.yaml
+F:	drivers/gpu/drm/panel/panel-novatek-nt36532.c
+
 DRM DRIVER FOR NOVATEK NT36672A PANELS
 M:	Sumit Semwal <sumit.semwal@linaro.org>
 S:	Maintained
@@ -8697,7 +8832,9 @@ T:	git https://gitlab.freedesktop.org/drm/rust/kernel.git
 F:	drivers/gpu/drm/nova/
 F:	drivers/gpu/drm/tyr/
 F:	drivers/gpu/nova-core/
+F:	rust/helpers/dma_fence.c
 F:	rust/helpers/gpu.c
+F:	rust/kernel/dma_buf/
 F:	rust/kernel/drm/
 F:	rust/kernel/gpu.rs
 F:	rust/kernel/gpu/
@@ -9069,6 +9206,21 @@ F:	include/drm/drm_gpuvm.h
 F:	rust/helpers/drm_gpuvm.c
 F:	rust/kernel/drm/gpuvm/
 
+DRM GPUSVM
+M:	Matthew Brost <matthew.brost@intel.com>
+R:	Thomas Hellström <thomas.hellstrom@linux.intel.com>
+R:	Himal Prasad Ghimiray <himal.prasad.ghimiray@intel.com>
+R:	Honglei Huang <honghuan@amd.com>
+L:	dri-devel@lists.freedesktop.org
+S:	Supported
+T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
+F:	drivers/gpu/drm/drm_gpusvm.c
+F:	drivers/gpu/drm/drm_pagemap.c
+F:	drivers/gpu/drm/drm_pagemap_util.c
+F:	include/drm/drm_gpusvm.h
+F:	include/drm/drm_pagemap.h
+F:	include/drm/drm_pagemap_util.h
+
 DRM LOG
 M:	Jocelyn Falempe <jfalempe@redhat.com>
 M:	Javier Martinez Canillas <javierm@redhat.com>
@@ -9096,9 +9248,12 @@ S:	Supported
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	drivers/gpu/drm/drm_draw.c
 F:	drivers/gpu/drm/drm_draw_internal.h
-F:	drivers/gpu/drm/drm_panic*.c
-F:	drivers/gpu/drm/tests/drm_panic_test.c
-F:	include/drm/drm_panic*
+F:	drivers/gpu/drm/drm_panic.c
+F:	drivers/gpu/drm/drm_panic_helper.c
+F:	drivers/gpu/drm/drm_panic_internal.h
+F:	drivers/gpu/drm/tests/drm_panic_helper_test.c
+F:	include/drm/drm_panic.h
+F:	include/drm/drm_panic_helper.h
 
 DRM PANIC QR CODE
 M:	Jocelyn Falempe <jfalempe@redhat.com>
@@ -9107,7 +9262,7 @@ L:	dri-devel@lists.freedesktop.org
 L:	rust-for-linux@vger.kernel.org
 S:	Supported
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
-F:	drivers/gpu/drm/drm_panic_qr.rs
+F:	drivers/gpu/drm/drm_panic_helper_qr.rs
 
 DRM PRIVACY-SCREEN CLASS
 M:	Hans de Goede <hansg@kernel.org>
@@ -9117,11 +9272,25 @@ T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	drivers/gpu/drm/drm_privacy_screen*
 F:	include/drm/drm_privacy_screen*
 
+DRM RAS
+M:	Riana Tauro <riana.tauro@intel.com>
+M:	Rodrigo Vivi <rodrigo.vivi@intel.com>
+R:	Aravind Iddamsetty <aravind.iddamsetty@linux.intel.com>
+R:	Raag Jadav <raag.jadav@intel.com>
+L:	dri-devel@lists.freedesktop.org
+S:	Supported
+T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
+F:	Documentation/netlink/specs/drm_ras.yaml
+F:	Documentation/gpu/drm-ras.rst
+F:	drivers/gpu/drm/drm_ras*
+F:	include/drm/drm_ras*
+
 DRM TTM SUBSYSTEM
-M:	Christian Koenig <christian.koenig@amd.com>
-M:	Huang Rui <ray.huang@amd.com>
+M:	Natalie Vock <nat@pixelcluster.dev>
+M:	Arun Pravin <arunpravin.paneerselvam@amd.com>
 R:	Matthew Auld <matthew.auld@intel.com>
 R:	Matthew Brost <matthew.brost@intel.com>
+R:	Thomas Hellström <thomas.hellstrom@linux.intel.com>
 L:	dri-devel@lists.freedesktop.org
 S:	Maintained
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
@@ -9449,6 +9618,11 @@ L:	linux-edac@vger.kernel.org
 S:	Orphan
 F:	drivers/edac/i3000_edac.c
 
+EDAC-I3200
+L:	linux-edac@vger.kernel.org
+S:	Orphan
+F:	drivers/edac/i3200_edac.c
+
 EDAC-I5000
 L:	linux-edac@vger.kernel.org
 S:	Maintained
@@ -9473,9 +9647,8 @@ S:	Maintained
 F:	drivers/edac/i7core_edac.c
 
 EDAC-I82975X
-M:	"Arvind R." <arvino55@gmail.com>
 L:	linux-edac@vger.kernel.org
-S:	Maintained
+S:	Orphan
 F:	drivers/edac/i82975x_edac.c
 
 EDAC-IE31200
@@ -9486,11 +9659,17 @@ F:	drivers/edac/ie31200_edac.c
 
 EDAC-IGEN6
 M:	Tony Luck <tony.luck@intel.com>
-R:	Qiuxu Zhuo <qiuxu.zhuo@intel.com>
+R:	Qiuxu Zhuo <qzhuo@nvidia.com>
 L:	linux-edac@vger.kernel.org
 S:	Maintained
 F:	drivers/edac/igen6_edac.c
 
+EDAC-INTEL-BFF-RESET
+M:	Tony Luck <tony.luck@intel.com>
+L:	linux-edac@vger.kernel.org
+S:	Maintained
+F:	drivers/edac/intel-bff.c
+
 EDAC-MPC85XX
 L:	linux-edac@vger.kernel.org
 S:	Orphan
@@ -9525,7 +9704,7 @@ F:	drivers/edac/qcom_edac.c
 
 EDAC-SBRIDGE
 M:	Tony Luck <tony.luck@intel.com>
-R:	Qiuxu Zhuo <qiuxu.zhuo@intel.com>
+R:	Qiuxu Zhuo <qzhuo@nvidia.com>
 L:	linux-edac@vger.kernel.org
 S:	Maintained
 F:	drivers/edac/sb_edac.c
@@ -9542,6 +9721,11 @@ L:	linux-edac@vger.kernel.org
 S:	Odd Fixes
 F:	drivers/edac/ti_edac.c
 
+EDAC-X38
+L:	linux-edac@vger.kernel.org
+S:	Orphan
+F:	drivers/edac/x38_edac.c
+
 EDIROL UA-101/UA-1000 DRIVER
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	linux-sound@vger.kernel.org
@@ -9699,6 +9883,14 @@ M:	Jeff Layton <jlayton@kernel.org>
 S:	Maintained
 F:	include/linux/errseq.h
 F:	lib/errseq.c
+F:	lib/tests/errseq_kunit.c
+
+ES9039Q2M AUDIO CODEC DRIVER
+M:	Karl Asseily <karl@asseily.com>
+L:	linux-sound@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/sound/esstech,es9039q2m.yaml
+F:	sound/soc/codecs/es9039q2m.c
 
 ESD CAN NETWORK DRIVERS
 M:	Stefan Mätje <stefan.maetje@esd.eu>
@@ -9829,6 +10021,7 @@ R:	Yuezhang Mo <yuezhang.mo@sony.com>
 L:	exfat@lists.linux.dev
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git
+F:	Documentation/filesystems/exfat.rst
 F:	fs/exfat/
 
 EXPRESSWIRE PROTOCOL LIBRARY
@@ -9976,6 +10169,7 @@ F:	net/core/failover.c
 FANOTIFY
 M:	Jan Kara <jack@suse.cz>
 R:	Amir Goldstein <amir73il@gmail.com>
+R:	Matt Bobrowski <matt@bobrowski.net>
 L:	linux-fsdevel@vger.kernel.org
 S:	Maintained
 F:	fs/notify/fanotify/
@@ -10206,9 +10400,8 @@ S:	Maintained
 F:	drivers/nvmem/layouts/fixed-layout.c
 
 FLEXTIMER FTM-QUADDEC DRIVER
-M:	Patrick Havelange <patrick.havelange@essensium.com>
 L:	linux-iio@vger.kernel.org
-S:	Maintained
+S:	Orphan
 F:	Documentation/devicetree/bindings/counter/fsl,ftm-quaddec.yaml
 F:	drivers/counter/ftm-quaddec.c
 
@@ -10350,7 +10543,7 @@ F:	include/video/
 FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER
 M:	Horia Geantă <horia.geanta@nxp.com>
 M:	Pankaj Gupta <pankaj.gupta@nxp.com>
-M:	Gaurav Jain <gaurav.jain@nxp.com>
+M:	Sahil Malhotra <sahil.malhotra@nxp.com>
 L:	linux-crypto@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/crypto/fsl,sec-v4.0*
@@ -10658,7 +10851,8 @@ M:	Jaegeuk Kim <jaegeuk@kernel.org>
 L:	linux-fscrypt@vger.kernel.org
 S:	Supported
 Q:	https://patchwork.kernel.org/project/linux-fscrypt/list/
-T:	git https://git.kernel.org/pub/scm/fs/fscrypt/linux.git
+T:	git https://git.kernel.org/pub/scm/fs/fscrypt/linux.git for-next
+T:	git https://git.kernel.org/pub/scm/fs/fscrypt/linux.git for-current
 F:	Documentation/filesystems/fscrypt.rst
 F:	fs/crypto/
 F:	include/linux/fscrypt.h
@@ -10705,7 +10899,8 @@ M:	Theodore Y. Ts'o <tytso@mit.edu>
 L:	fsverity@lists.linux.dev
 S:	Supported
 Q:	https://patchwork.kernel.org/project/fsverity/list/
-T:	git https://git.kernel.org/pub/scm/fs/fsverity/linux.git
+T:	git https://git.kernel.org/pub/scm/fs/fsverity/linux.git for-next
+T:	git https://git.kernel.org/pub/scm/fs/fsverity/linux.git for-current
 F:	Documentation/filesystems/fsverity.rst
 F:	fs/verity/
 F:	include/linux/fsverity.h
@@ -11136,16 +11331,6 @@ S:	Maintained
 F:	Documentation/networking/device_drivers/ethernet/google/gve.rst
 F:	drivers/net/ethernet/google
 
-GOOGLE FIRMWARE DRIVERS
-M:	Tzung-Bi Shih <tzungbi@kernel.org>
-R:	Brian Norris <briannorris@chromium.org>
-R:	Julius Werner <jwerner@chromium.org>
-L:	chrome-platform@lists.linux.dev
-S:	Maintained
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git
-F:	drivers/firmware/google/
-F:	include/linux/coreboot.h
-
 GOOGLE TENSOR SoC SUPPORT
 M:	Peter Griffin <peter.griffin@linaro.org>
 R:	André Draszik <andre.draszik@linaro.org>
@@ -11160,6 +11345,7 @@ F:	Documentation/devicetree/bindings/phy/google,lga-usb-phy.yaml
 F:	Documentation/devicetree/bindings/soc/google/google,gs101-pmu-intr-gen.yaml
 F:	Documentation/devicetree/bindings/usb/google,lga-dwc3.yaml
 F:	arch/arm64/boot/dts/exynos/google/
+F:	arch/arm64/boot/dts/google/
 F:	drivers/clk/samsung/clk-gs101.c
 F:	drivers/phy/phy-google-usb.c
 F:	drivers/soc/samsung/gs101-pmu.c
@@ -11637,6 +11823,12 @@ L:	linux-input@vger.kernel.org
 S:	Supported
 F:	drivers/hid/hid-playstation.c
 
+HID PULSAR DRIVER
+M:	Nikolas Koesling <nikolas@koesling.info>
+L:	linux-input@vger.kernel.org
+S:	Maintained
+F:	drivers/hid/hid-pulsar.c
+
 HID SENSOR HUB DRIVERS
 M:	Jiri Kosina <jikos@kernel.org>
 M:	Jonathan Cameron <jic23@kernel.org>
@@ -11728,6 +11920,13 @@ W:	http://www.highpoint-tech.com
 F:	Documentation/scsi/hptiop.rst
 F:	drivers/scsi/hptiop.c
 
+HIMAX HM1246 SENSOR DRIVER
+M:	Matthias Fend <matthias.fend@emfend.at>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/i2c/himax,hm1246.yaml
+F:	drivers/media/i2c/hm1246.c
+
 HIMAX HX83112B TOUCHSCREEN SUPPORT
 M:	Job Noorman <job@noorman.info>
 L:	linux-input@vger.kernel.org
@@ -11960,6 +12159,13 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/pressure/honeywell,mprls0025pa.yaml
 F:	drivers/iio/pressure/mprls0025pa*
 
+HONOR FMI-XX HARDWARE MONITOR DRIVER
+M:	Nikita Dubrovskih <testname142@gmail.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Maintained
+F:	Documentation/hwmon/honor-fmi.rst
+F:	drivers/hwmon/honor-fmi.c
+
 HP BIOSCFG DRIVER
 M:	Jorge Lopez <jorge.lopez2@hp.com>
 L:	platform-driver-x86@vger.kernel.org
@@ -12096,6 +12302,7 @@ F:	Documentation/mm/hugetlbfs_reserv.rst
 F:	Documentation/mm/vmemmap_dedup.rst
 F:	fs/hugetlbfs/
 F:	include/linux/hugetlb.h
+F:	include/linux/vmemmap-optimization.h
 F:	include/trace/events/hugetlbfs.h
 F:	mm/hugetlb.c
 F:	mm/hugetlb_cgroup.c
@@ -12163,7 +12370,6 @@ M:	"K. Y. Srinivasan" <kys@microsoft.com>
 M:	Haiyang Zhang <haiyangz@microsoft.com>
 M:	Wei Liu <wei.liu@kernel.org>
 M:	Dexuan Cui <decui@microsoft.com>
-M:	Long Li <longli@microsoft.com>
 L:	linux-hyperv@vger.kernel.org
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git
@@ -12456,6 +12662,7 @@ F:	drivers/crypto/nx/Makefile
 F:	drivers/crypto/nx/nx-842*
 F:	include/linux/sw842.h
 F:	lib/842/
+F:	lib/tests/842_decompress_kunit.c
 
 IBM Power in-Nest Crypto Acceleration
 M:	Breno Leitão <leitao@debian.org>
@@ -12696,6 +12903,7 @@ R:	Nuno Sá <nuno.sa@analog.com>
 R:	Andy Shevchenko <andy@kernel.org>
 L:	linux-iio@vger.kernel.org
 S:	Maintained
+Q:	https://patchwork.kernel.org/project/linux-iio/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git
 F:	Documentation/ABI/testing/configfs-iio*
 F:	Documentation/ABI/testing/sysfs-bus-iio*
@@ -12708,6 +12916,13 @@ F:	include/dt-bindings/iio/
 F:	include/linux/iio/
 F:	tools/iio/
 
+IIO CORE KUNIT TESTS
+M:	Rodrigo Alencar <rodrigo.alencar@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	drivers/iio/test/iio-test-channel-prefix.c
+F:	drivers/iio/test/iio-test-format.c
+
 IIO UNIT CONVERTER
 M:	Peter Rosin <peda@lysator.liu.se>
 L:	linux-iio@vger.kernel.org
@@ -13010,9 +13225,8 @@ S:	Maintained
 F:	drivers/platform/x86/intel/atomisp2/led.c
 
 INTEL BIOS SAR INT1092 DRIVER
-M:	Shravan Sudhakar <s.shravan@intel.com>
 L:	platform-driver-x86@vger.kernel.org
-S:	Maintained
+S:	Orphan
 F:	drivers/platform/x86/intel/int1092/
 
 INTEL BROXTON PMC DRIVER
@@ -13030,6 +13244,8 @@ F:	drivers/scsi/isci/
 
 INTEL COMPUTER VISION SENSING (CVS) DRIVER
 M:	Miguel Vadillo <miguel.vadillo@intel.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+R:	Antti Laakso <antti.laakso@linux.intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	drivers/media/i2c/cvs/
@@ -13217,6 +13433,15 @@ S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
 F:	drivers/iommu/intel/
 
+INTEL IPU BRIDGE
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+M:	Dan Scally <dan.scally@ideasonboard.com>
+R:	Hans de Goede <hansg@kernel.org>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	drivers/media/pci/intel/ipu-bridge.c
+F:	include/media/ipu-bridge.h
+
 INTEL IPU3 CSI-2 CIO2 DRIVER
 M:	Yong Zhi <yong.zhi@intel.com>
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
@@ -13238,6 +13463,8 @@ F:	drivers/staging/media/ipu3/
 
 INTEL IPU6 INPUT SYSTEM DRIVER
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+R:	Antti Laakso <antti.laakso@linux.intel.com>
+R:	"Sapre, Sarang" <sarang.sapre@intel.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
@@ -13474,6 +13701,7 @@ K:	\bSGX_
 INTEL SKYLAKE INT3472 ACPI DEVICE DRIVER
 M:	Daniel Scally <dan.scally@ideasonboard.com>
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+L:	linux-media@vger.kernel.org
 S:	Maintained
 T:	git git://linuxtv.org/media.git
 F:	drivers/platform/x86/intel/int3472/
@@ -13560,7 +13788,7 @@ INTEL WIRELESS WIFI LINK (iwlwifi)
 M:	Miri Korenblit <miriam.rachel.korenblit@intel.com>
 L:	linux-wireless@vger.kernel.org
 S:	Supported
-W:	https://wireless.wiki.kernel.org/en/users/drivers/iwlwifi
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/iwlwifi.html
 T:	git https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git/
 F:	drivers/net/wireless/intel/iwlwifi/
 
@@ -13940,6 +14168,14 @@ T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	Documentation/devicetree/bindings/display/bridge/ite,it66121.yaml
 F:	drivers/gpu/drm/bridge/ite-it66121.c
 
+ITE IT6625 HDMI to MIPI MEDIA DRIVER
+M:	Hermes Wu <Hermes.Wu@ite.com.tw>
+S:	Maintained
+T:	git git://linuxtv.org/media.git
+F:	Documentation/devicetree/bindings/media/i2c/ite,it6625.yaml
+F:	drivers/media/i2c/it6625.c
+F:	include/uapi/linux/it6625.h
+
 IVTV VIDEO4LINUX DRIVER
 M:	Andy Walls <awalls@md.metrocast.net>
 L:	linux-media@vger.kernel.org
@@ -14058,6 +14294,7 @@ Q:	https://patchwork.kernel.org/project/linux-kbuild/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git
 F:	Documentation/kbuild/kconfig*
 F:	scripts/Kconfig.include
+F:	scripts/Kconfig.toolchain
 F:	scripts/kconfig/
 
 KCORE
@@ -14139,6 +14376,7 @@ F:	scripts/basic/
 F:	scripts/clang-tools/
 F:	scripts/container
 F:	scripts/dummy-tools/
+F:	scripts/gen_init*
 F:	scripts/include/
 F:	scripts/install.sh
 F:	scripts/mk*
@@ -14187,7 +14425,8 @@ L:	linux-nfs@vger.kernel.org
 S:	Supported
 P:	Documentation/filesystems/nfs/nfsd-maintainer-entry-profile.rst
 B:	https://bugzilla.kernel.org
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux.git nfsd-testing
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux.git nfsd-next
 F:	Documentation/filesystems/nfs/
 F:	fs/lockd/
 F:	fs/nfs_common/
@@ -14413,6 +14652,7 @@ S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git
 F:	fs/kernfs/
 F:	include/linux/kernfs.h
+F:	tools/testing/selftests/filesystems/kernfs_test.c
 
 KEXEC
 M:	Andrew Morton <akpm@linux-foundation.org>
@@ -14905,7 +15145,7 @@ M:	Damien Le Moal <dlemoal@kernel.org>
 M:	Niklas Cassel <cassel@kernel.org>
 L:	linux-ide@vger.kernel.org
 S:	Maintained
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/libata/linux.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/libata/linux.git for-next
 F:	Documentation/ABI/testing/sysfs-ata
 F:	Documentation/devicetree/bindings/ata/
 F:	drivers/ata/
@@ -15523,6 +15763,14 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/temperature/adi,ltc2983.yaml
 F:	drivers/iio/temperature/ltc2983.c
 
+LTC3208 LED DRIVER
+M:	Jan Carlo Roleda <jancarlo.roleda@analog.com>
+L:	linux-leds@vger.kernel.org
+S:	Maintained
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/leds/adi,ltc3208.yaml
+F:	drivers/leds/leds-ltc3208.c
+
 LTC3220 LED DRIVER
 M:	Edelweise Escala <edelweise.escala@analog.com>
 L:	linux-leds@vger.kernel.org
@@ -15663,7 +15911,7 @@ MAC80211
 M:	Johannes Berg <johannes@sipsolutions.net>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/
+W:	https://wireless.docs.kernel.org/en/latest/
 Q:	https://patchwork.kernel.org/project/linux-wireless/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git
@@ -15988,6 +16236,15 @@ F:	Documentation/devicetree/bindings/hwmon/pmbus/adi,max17616.yaml
 F:	Documentation/hwmon/max17616.rst
 F:	drivers/hwmon/pmbus/max17616.c
 
+MAX20826 HARDWARE MONITOR CONTROLLER DRIVER
+M:	Nuno Sá <nuno.sa@analog.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/hwmon/pmbus/adi,max20826.yaml
+F:	Documentation/hwmon/max20826.rst
+F:	drivers/hwmon/pmbus/max20826.c
+
 MAX20830 HARDWARE MONITOR DRIVER
 M:	Alexis Czezar Torreno <alexisczezar.torreno@analog.com>
 L:	linux-hwmon@vger.kernel.org
@@ -16122,6 +16379,15 @@ L:	linux-iio@vger.kernel.org
 S:	Supported
 F:	drivers/iio/temperature/max30208.c
 
+MAXIM MAX40080 CURRENT SENSE AMPLIFIER DRIVER
+M:	Ciprian Hegbeli <ciprian.hegbeli@analog.com>
+M:	Stefan Popa <stefan.popa@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/adc/adi,max40080.yaml
+F:	drivers/iio/adc/max40080.c
+
 MAXIM MAX7360 KEYPAD LED MFD DRIVER
 M:	Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com>
 S:	Maintained
@@ -16293,7 +16559,7 @@ M:	Ariana Lazar <ariana.lazar@microchip.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 F:	Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml
-F:	drivers/iio/dac/mcp47feb02.c
+F:	drivers/iio/dac/mcp47feb02*
 
 MCP4821 DAC DRIVER
 M:	Anshul Dalal <anshulusr@gmail.com>
@@ -16404,6 +16670,7 @@ M:	Frank Li <Frank.Li@nxp.com>
 M:	Martin Kepplinger-Novakovic <martink@posteo.de>
 R:	Rui Miguel Silva <rmfrfs@gmail.com>
 R:	Purism Kernel Team <kernel@puri.sm>
+R:	Bryan O'Donoghue <bod@kernel.org>
 L:	imx@lists.linux.dev
 L:	linux-media@vger.kernel.org
 S:	Maintained
@@ -16609,6 +16876,17 @@ M:	Sean Wang <sean.wang@mediatek.com>
 S:	Maintained
 F:	drivers/media/rc/mtk-cir.c
 
+MEDIATEK CLOCK DRIVERS
+M:	AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+L:	linux-clk@vger.kernel.org
+L:	linux-mediatek@lists.infradead.org (moderated for non-subscribers)
+S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git
+F:	Documentation/devicetree/bindings/clock/mediatek,*
+F:	drivers/clk/mediatek/
+F:	include/dt-bindings/clock/mediatek,*.h
+F:	include/dt-bindings/clock/mt*-clk.h
+
 MEDIATEK DMA DRIVER
 M:	Sean Wang <sean.wang@mediatek.com>
 L:	dmaengine@vger.kernel.org
@@ -16714,6 +16992,11 @@ S:	Maintained
 F:	drivers/iio/adc/mt6323-auxadc.c
 F:	include/dt-bindings/iio/adc/mediatek,mt6323-auxadc.h
 
+MEDIATEK MT6323 PMIC NVMEM DRIVER
+M:	Roman Vivchar <rva333@protonmail.com>
+S:	Maintained
+F:	drivers/nvmem/mt6323-efuse.c
+
 MEDIATEK MT6735 CLOCK & RESET DRIVERS
 M:	Yassine Oudjana <y.oudjana@protonmail.com>
 L:	linux-clk@vger.kernel.org
@@ -17188,13 +17471,14 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
 F:	mm/gup.c
 F:	mm/gup_test.c
 F:	mm/gup_test.h
+F:	tools/mm/gup_bench.c
+F:	tools/testing/selftests/mm/gup.c
 F:	tools/testing/selftests/mm/gup_longterm.c
-F:	tools/testing/selftests/mm/gup_test.c
 
 MEMORY MANAGEMENT - KSM (Kernel Samepage Merging)
 M:	Andrew Morton <akpm@linux-foundation.org>
 M:	David Hildenbrand <david@kernel.org>
-R:	Xu Xin <xu.xin16@zte.com.cn>
+R:	Xu Xin <xu.xin@linux.dev>
 R:	Chengming Zhou <chengming.zhou@linux.dev>
 L:	linux-mm@kvack.org
 S:	Maintained
@@ -17208,31 +17492,6 @@ F:	include/trace/events/ksm.h
 F:	mm/ksm.c
 F:	mm/mm_slot.h
 
-MEMORY MANAGEMENT - MEMORY POLICY AND MIGRATION
-M:	Andrew Morton <akpm@linux-foundation.org>
-M:	David Hildenbrand <david@kernel.org>
-R:	Zi Yan <ziy@nvidia.com>
-R:	Matthew Brost <matthew.brost@intel.com>
-R:	Joshua Hahn <joshua.hahnjy@gmail.com>
-R:	Rakie Kim <rakie.kim@sk.com>
-R:	Byungchul Park <byungchul@sk.com>
-R:	Gregory Price <gourry@gourry.net>
-R:	Ying Huang <ying.huang@linux.alibaba.com>
-R:	Alistair Popple <apopple@nvidia.com>
-L:	linux-mm@kvack.org
-S:	Maintained
-W:	http://www.linux-mm.org
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
-F:	Documentation/ABI/testing/sysfs-kernel-mm-mempolicy
-F:	Documentation/ABI/testing/sysfs-kernel-mm-mempolicy-weighted-interleave
-F:	include/linux/mempolicy.h
-F:	include/uapi/linux/mempolicy.h
-F:	include/linux/migrate.h
-F:	include/linux/migrate_mode.h
-F:	mm/mempolicy.c
-F:	mm/migrate.c
-F:	mm/migrate_device.c
-
 MEMORY MANAGEMENT - MGLRU (MULTI-GEN LRU)
 M:	Andrew Morton <akpm@linux-foundation.org>
 R:	Kairui Song <kasong@tencent.com>
@@ -17242,6 +17501,8 @@ R:	Barry Song <baohua@kernel.org>
 R:	Axel Rasmussen <axelrasmussen@google.com>
 R:	Yuanchu Xie <yuanchu@google.com>
 R:	Wei Xu <weixugc@google.com>
+R:	Baoquan He <baoquan.he@linux.dev>
+R:	Baolin Wang <baolin.wang@linux.alibaba.com>
 L:	linux-mm@kvack.org
 S:	Maintained
 W:	http://www.linux-mm.org
@@ -17253,6 +17514,25 @@ F:	include/linux/mmzone.h
 F:	mm/vmscan.c
 F:	mm/workingset.c
 
+MEMORY MANAGEMENT - MIGRATION
+M:	Andrew Morton <akpm@linux-foundation.org>
+M:	David Hildenbrand <david@kernel.org>
+R:	Zi Yan <ziy@nvidia.com>
+R:	Matthew Brost <matthew.brost@intel.com>
+R:	Joshua Hahn <joshua.hahnjy@gmail.com>
+R:	Byungchul Park <byungchul@sk.com>
+R:	Gregory Price <gourry@gourry.net>
+R:	Ying Huang <ying.huang@linux.alibaba.com>
+R:	Alistair Popple <apopple@nvidia.com>
+L:	linux-mm@kvack.org
+S:	Maintained
+W:	http://www.linux-mm.org
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
+F:	include/linux/migrate.h
+F:	include/linux/migrate_mode.h
+F:	mm/migrate.c
+F:	mm/migrate_device.c
+
 MEMORY MANAGEMENT - MISC
 M:	Andrew Morton <akpm@linux-foundation.org>
 M:	David Hildenbrand <david@kernel.org>
@@ -17268,15 +17548,15 @@ W:	http://www.linux-mm.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
 F:	Documentation/ABI/testing/sysfs-kernel-mm
 F:	Documentation/ABI/testing/sysfs-kernel-mm-cma
-F:	Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers
 F:	Documentation/ABI/testing/sysfs-kernel-mm-numa
 F:	Documentation/admin-guide/mm/
+F:	Documentation/core-api/memory-allocation.rst
+F:	Documentation/core-api/mm-api.rst
 F:	Documentation/mm/
-F:	drivers/char/mem.c
+F:	mm/char-mem.c
 F:	include/linux/cma.h
 F:	include/linux/dmapool.h
 F:	include/linux/ioremap.h
-F:	include/linux/memory-tiers.h
 F:	include/linux/page_idle.h
 F:	mm/backing-dev.c
 F:	mm/cma.c
@@ -17288,10 +17568,10 @@ F:	mm/early_ioremap.c
 F:	mm/fadvise.c
 F:	mm/ioremap.c
 F:	mm/mapping_dirty_helpers.c
-F:	mm/memory-tiers.c
 F:	mm/page_idle.c
 F:	mm/pgalloc-track.h
 F:	mm/process_vm_access.c
+F:	tools/lib/mm/
 F:	tools/testing/selftests/mm/
 
 MEMORY MANAGEMENT - NUMA MEMBLOCKS AND NUMA EMULATION
@@ -17305,6 +17585,27 @@ F:	mm/numa.c
 F:	mm/numa_emulation.c
 F:	mm/numa_memblks.c
 
+MEMORY MANAGEMENT - NUMA PLACEMENT
+M:	Andrew Morton <akpm@linux-foundation.org>
+M:	David Hildenbrand <david@kernel.org>
+M:	Gregory Price <gourry@gourry.net>
+R:	Zi Yan <ziy@nvidia.com>
+R:	Joshua Hahn <joshua.hahnjy@gmail.com>
+R:	Rakie Kim <rakie.kim@sk.com>
+R:	Ying Huang <ying.huang@linux.alibaba.com>
+L:	linux-mm@kvack.org
+S:	Maintained
+W:	http://www.linux-mm.org
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
+F:	Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers
+F:	Documentation/ABI/testing/sysfs-kernel-mm-mempolicy
+F:	Documentation/ABI/testing/sysfs-kernel-mm-mempolicy-weighted-interleave
+F:	include/linux/memory-tiers.h
+F:	include/linux/mempolicy.h
+F:	include/uapi/linux/mempolicy.h
+F:	mm/memory-tiers.c
+F:	mm/mempolicy.c
+
 MEMORY MANAGEMENT - OOM KILLER
 M:	Michal Hocko <mhocko@suse.com>
 R:	David Rientjes <rientjes@google.com>
@@ -17431,6 +17732,7 @@ F:	Documentation/admin-guide/mm/transhuge.rst
 F:	include/linux/huge_mm.h
 F:	include/linux/khugepaged.h
 F:	include/trace/events/huge_memory.h
+F:	mm/collapse.h
 F:	mm/huge_memory.c
 F:	mm/khugepaged.c
 F:	mm/mm_slot.h
@@ -17477,7 +17779,7 @@ L:	linux-mm@kvack.org
 S:	Maintained
 W:	http://www.linux-mm.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
-F:	drivers/char/mem.c
+F:	mm/char-mem.c
 F:	include/trace/events/mmap.h
 F:	fs/proc/task_mmu.c
 F:	fs/proc/task_nommu.c
@@ -17772,6 +18074,7 @@ F:	Documentation/devicetree/bindings/net/dsa/microchip,lan937x.yaml
 F:	drivers/net/dsa/microchip/*
 F:	include/linux/dsa/ksz_common.h
 F:	include/linux/platform_data/microchip-ksz.h
+F:	net/dsa/tag_ks8995.c
 F:	net/dsa/tag_ksz.c
 
 MICROCHIP LAN743X ETHERNET DRIVER
@@ -17987,10 +18290,9 @@ F:	drivers/misc/atmel-ssc.c
 F:	include/linux/atmel-ssc.h
 
 Microchip Timer Counter Block (TCB) Capture Driver
-M:	Kamel Bouhara <kamel.bouhara@bootlin.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 L:	linux-iio@vger.kernel.org
-S:	Maintained
+S:	Orphan
 F:	drivers/counter/microchip-tcb-capture.c
 F:	include/uapi/linux/counter/microchip-tcb-capture.h
 
@@ -18047,7 +18349,7 @@ F:	include/linux/cciss*.h
 F:	include/uapi/linux/cciss*.h
 
 MICROSOFT MANA RDMA DRIVER
-M:	Long Li <longli@microsoft.com>
+M:	Wei Hu <weh@microsoft.com>
 M:	Konstantin Taranov <kotaranov@microsoft.com>
 L:	linux-rdma@vger.kernel.org
 L:	linux-hyperv@vger.kernel.org
@@ -18178,6 +18480,13 @@ F:	include/linux/min_heap.h
 F:	lib/min_heap.c
 F:	lib/tests/min_heap_kunit.c
 
+MINISFORUM UM780 XTX EC HARDWARE MONITOR DRIVER
+M:	Sebastián Peyrott <speyrott@gmail.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Maintained
+F:	Documentation/hwmon/minisforum-um780xtx.rst
+F:	drivers/hwmon/minisforum-um780xtx.c
+
 MIPI CCS, SMIA AND SMIA++ IMAGE SENSOR DRIVER
 M:	Sakari Ailus <sakari.ailus@linux.intel.com>
 L:	linux-media@vger.kernel.org
@@ -18368,6 +18677,12 @@ F:	scripts/module*
 F:	tools/testing/selftests/kmod/
 F:	tools/testing/selftests/module/
 
+MONOLITHIC POWER SYSTEM MPQ4210 REGULATOR DRIVER
+M:	Tapio Reijonen <tapio.reijonen@vaisala.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/regulator/mps,mpq4210.yaml
+F:	drivers/regulator/mpq4210.c
+
 MONOLITHIC POWER SYSTEM PMIC DRIVER
 M:	Saravanan Sekar <sravanhome@gmail.com>
 S:	Maintained
@@ -18418,10 +18733,10 @@ M:	David Yang <mmyangfl@gmail.com>
 L:	netdev@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/net/dsa/motorcomm,yt921x.yaml
-F:	drivers/net/dsa/yt921x.*
+F:	drivers/net/dsa/motorcomm/
 F:	net/dsa/tag_yt921x.c
 
-MOXA SMARTIO/INDUSTIO/INTELLIO SERIAL CARD
+MOXA SMARTIO/INDUSTIO SERIAL CARD
 M:	Jiri Slaby <jirislaby@kernel.org>
 S:	Maintained
 F:	Documentation/driver-api/tty/moxa-smartio.rst
@@ -18800,6 +19115,16 @@ F:	Documentation/devicetree/bindings/hwmon/nuvoton,nct7363.yaml
 F:	Documentation/hwmon/nct7363.rst
 F:	drivers/hwmon/nct7363.c
 
+NEBULA-MATRIX ETHERNET DRIVER (nebula-matrix)
+M:	Illusion Wang <illusion.wang@nebula-matrix.com>
+M:	Dimon Zhao <dimon.zhao@nebula-matrix.com>
+M:	Alvin Wang <alvin.wang@nebula-matrix.com>
+M:	Sam Chen <sam.chen@nebula-matrix.com>
+L:	netdev@vger.kernel.org
+S:	Maintained
+F:	Documentation/networking/device_drivers/ethernet/nebula-matrix/
+F:	drivers/net/ethernet/nebula-matrix/
+
 NETCONSOLE
 M:	Breno Leitao <leitao@debian.org>
 S:	Maintained
@@ -18882,7 +19207,7 @@ F:	net/core/drop_monitor.c
 NETWORKING DRIVERS
 M:	Andrew Lunn <andrew+netdev@lunn.ch>
 M:	"David S. Miller" <davem@davemloft.net>
-M:	Eric Dumazet <edumazet@google.com>
+M:	Eric Dumazet <edumazet@kernel.org>
 M:	Jakub Kicinski <kuba@kernel.org>
 M:	Paolo Abeni <pabeni@redhat.com>
 L:	netdev@vger.kernel.org
@@ -18922,7 +19247,7 @@ NETWORKING DRIVERS (WIRELESS)
 M:	Johannes Berg <johannes@sipsolutions.net>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/
+W:	https://wireless.docs.kernel.org/en/latest/
 Q:	https://patchwork.kernel.org/project/linux-wireless/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git
@@ -18999,7 +19324,7 @@ K:	struct\s+phy_port|phy_port_
 
 NETWORKING [GENERAL]
 M:	"David S. Miller" <davem@davemloft.net>
-M:	Eric Dumazet <edumazet@google.com>
+M:	Eric Dumazet <edumazet@kernel.org>
 M:	Jakub Kicinski <kuba@kernel.org>
 M:	Paolo Abeni <pabeni@redhat.com>
 R:	Simon Horman <horms@kernel.org>
@@ -19225,7 +19550,7 @@ F:	net/ipv6/seg6*
 F:	tools/testing/selftests/net/srv6*
 
 NETWORKING [TCP]
-M:	Eric Dumazet <edumazet@google.com>
+M:	Eric Dumazet <edumazet@kernel.org>
 M:	Neal Cardwell <ncardwell@google.com>
 R:	Kuniyuki Iwashima <kuniyu@google.com>
 L:	netdev@vger.kernel.org
@@ -19258,7 +19583,7 @@ F:	net/tls/
 F:	tools/testing/selftests/net/tls.c
 
 NETWORKING [SOCKETS]
-M:	Eric Dumazet <edumazet@google.com>
+M:	Eric Dumazet <edumazet@kernel.org>
 M:	Kuniyuki Iwashima <kuniyu@google.com>
 M:	Paolo Abeni <pabeni@redhat.com>
 M:	Willem de Bruijn <willemb@google.com>
@@ -19632,6 +19957,8 @@ F:	include/linux/nvmem-consumer.h
 F:	include/linux/nvmem-provider.h
 
 NXP BLUETOOTH WIRELESS DRIVERS
+M:	Song Xue <song.xue_1@nxp.com>
+M:	Alex Zhou <alex.zhou@nxp.com>
 M:	Amitkumar Karwar <amitkumar.karwar@nxp.com>
 M:	Neeraj Kale <neeraj.sanjaykale@nxp.com>
 S:	Maintained
@@ -19721,6 +20048,14 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml
 F:	drivers/media/platform/nxp/imx-jpeg
 
+NXP i.MX 95 CSI PIXEL FORMATTER V4L2 DRIVER
+M:	Guoniu Zhou <guoniu.zhou@nxp.com>
+L:	imx@lists.linux.dev
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/fsl,imx95-csi-formatter.yaml
+F:	drivers/media/platform/nxp/imx95-csi-formatter.c
+
 NXP i.MX CLOCK DRIVERS
 M:	Abel Vesa <abelvesa@kernel.org>
 R:	Peng Fan <peng.fan@nxp.com>
@@ -20111,6 +20446,14 @@ T:	git git://linuxtv.org/media_tree.git
 F:	Documentation/devicetree/bindings/media/i2c/ovti,og0ve1b.yaml
 F:	drivers/media/i2c/og0ve1b.c
 
+OMNIVISION OS02G10 SENSOR DRIVER
+M:	Tarang Raval <tarang.raval@siliconsignals.io>
+M:	Elgin Perumbilly <elgin.perumbilly@siliconsignals.io>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/i2c/ovti,os02g10.yaml
+F:	drivers/media/i2c/os02g10.c
+
 OMNIVISION OS05B10 SENSOR DRIVER
 M:	Himanshu Bhavani <himanshu.bhavani@siliconsignals.io>
 M:	Elgin Perumbilly <elgin.perumbilly@siliconsignals.io>
@@ -20474,11 +20817,14 @@ F:	Documentation/devicetree/
 F:	Documentation/process/maintainer-devicetree.rst
 F:	arch/*/boot/dts/
 F:	include/dt-bindings/
+F:	scripts/dtc/dt-check-style
+F:	scripts/dtc/dt-style-selftest/
 
 OPENCOMPUTE PTP CLOCK DRIVER
 M:	Vadim Fedorenko <vadim.fedorenko@linux.dev>
 L:	netdev@vger.kernel.org
 S:	Maintained
+F:	Documentation/networking/devlink/ptp_ocp.rst
 F:	drivers/ptp/ptp_ocp.c
 
 OPENCORES I2C BUS DRIVER
@@ -20554,6 +20900,7 @@ ORACLE CLUSTER FILESYSTEM 2 (OCFS2)
 M:	Mark Fasheh <mark@fasheh.com>
 M:	Joel Becker <jlbec@evilplan.org>
 M:	Joseph Qi <joseph.qi@linux.alibaba.com>
+R:	Heming Zhao <heming.zhao@suse.com>
 L:	ocfs2-devel@lists.linux.dev
 S:	Supported
 W:	http://ocfs2.wiki.kernel.org
@@ -20594,7 +20941,7 @@ P54 WIRELESS DRIVER
 M:	Christian Lamparter <chunkeey@googlemail.com>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/p54
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/p54.html
 F:	Documentation/devicetree/bindings/net/wireless/st,stlc4560.yaml
 F:	drivers/net/wireless/intersil/
 
@@ -20950,7 +21297,6 @@ M:	Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>
 L:	linux-pci@vger.kernel.org
 L:	linux-renesas-soc@vger.kernel.org
 S:	Maintained
-F:	Documentation/PCI/controller/rcar-pcie-firmware.rst
 F:	Documentation/devicetree/bindings/pci/*rcar*
 F:	drivers/pci/controller/*rcar*
 F:	drivers/pci/controller/dwc/*rcar*
@@ -21042,6 +21388,21 @@ L:	linux-pci@vger.kernel.org
 S:	Supported
 F:	Documentation/PCI/pci-error-recovery.rst
 
+PCI LIVE UPDATE
+M:	David Matlack <dmatlack@google.com>
+R:	Pasha Tatashin <pasha.tatashin@soleen.com>
+R:	Mike Rapoport <rppt@kernel.org>
+R:	Pratyush Yadav <pratyush@kernel.org>
+L:	kexec@lists.infradead.org
+L:	linux-pci@vger.kernel.org
+S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
+F:	Documentation/PCI/liveupdate.rst
+F:	drivers/pci/liveupdate.c
+F:	drivers/pci/liveupdate.h
+F:	include/linux/kho/abi/pci.h
+F:	include/linux/pci_liveupdate.h
+
 PCI MSI DRIVER FOR ALTERA MSI IP
 L:	linux-pci@vger.kernel.org
 S:	Orphan
@@ -21404,7 +21765,7 @@ F:	tools/lib/perf/
 F:	tools/perf/
 
 PERFORMANCE EVENTS TOOLING ARM64
-R:	John Garry <john.g.garry@oracle.com>
+R:	John Garry <john.garry@linux.dev>
 R:	Will Deacon <will@kernel.org>
 R:	James Clark <james.clark@linaro.org>
 R:	Mike Leach <mike.leach@arm.com>
@@ -22284,7 +22645,7 @@ M:	Jeff Johnson <jjohnson@kernel.org>
 L:	linux-wireless@vger.kernel.org
 L:	ath12k@lists.infradead.org
 S:	Supported
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/ath12k
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/ath12k.html
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git
 F:	drivers/net/wireless/ath/ath12k/
 N:	ath12k
@@ -22294,7 +22655,7 @@ M:	Jeff Johnson <jjohnson@kernel.org>
 L:	linux-wireless@vger.kernel.org
 L:	ath10k@lists.infradead.org
 S:	Supported
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/ath10k
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/ath10k.html
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git
 F:	drivers/net/wireless/ath/ath10k/
 N:	ath10k
@@ -22304,8 +22665,8 @@ M:	Jeff Johnson <jjohnson@kernel.org>
 L:	linux-wireless@vger.kernel.org
 L:	ath11k@lists.infradead.org
 S:	Supported
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/ath11k
-B:	https://wireless.wiki.kernel.org/en/users/Drivers/ath11k/bugreport
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/ath11k.html
+B:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/ath11k/bugreport.html
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git
 F:	drivers/net/wireless/ath/ath11k/
 N:	ath11k
@@ -22314,7 +22675,7 @@ QUALCOMM ATHEROS ATH9K WIRELESS DRIVER
 M:	Toke Høiland-Jørgensen <toke@toke.dk>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/ath9k
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/ath9k.html
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git
 F:	Documentation/devicetree/bindings/net/wireless/qca,ath9k.yaml
 F:	drivers/net/wireless/ath/ath9k/
@@ -22363,6 +22724,7 @@ F:	drivers/media/platform/qcom/camss/
 
 QUALCOMM CLOCK DRIVERS
 M:	Bjorn Andersson <andersson@kernel.org>
+R:	Abel Vesa <abelvesa@kernel.org>
 L:	linux-arm-msm@vger.kernel.org
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git
@@ -22479,6 +22841,15 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml
 F:	drivers/i2c/busses/i2c-qcom-cci.c
 
+QUALCOMM INLINE CRYPTO ENGINE DRIVER
+M:	Abel Vesa <abelvesa@kernel.org>
+L:	linux-arm-msm@vger.kernel.org
+L:	linux-crypto@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
+F:	drivers/soc/qcom/ice.c
+F:	include/soc/qcom/ice.h
+
 QUALCOMM INTERCONNECT BWMON DRIVER
 M:	Krzysztof Kozlowski <krzk@kernel.org>
 L:	linux-arm-msm@vger.kernel.org
@@ -22538,6 +22909,16 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/media/qcom,*-iris.yaml
 F:	drivers/media/platform/qcom/iris/
 
+QUALCOMM MIPI CSI2 PHY DRIVER
+M:	Bryan O'Donoghue <bod@kernel.org>
+L:	linux-phy@lists.infradead.org
+L:	linux-media@vger.kernel.org
+L:	linux-arm-msm@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/phy/qcom,*-csi2-phy.yaml
+F:	drivers/phy/qualcomm/phy-qcom-mipi-csi2*.c
+F:	drivers/phy/qualcomm/phy-qcom-mipi-csi2*.h
+
 QUALCOMM NAND CONTROLLER DRIVER
 M:	Manivannan Sadhasivam <mani@kernel.org>
 L:	linux-mtd@lists.infradead.org
@@ -22581,6 +22962,13 @@ F:	Documentation/devicetree/bindings/net/qcom,ipq9574-ppe.yaml
 F:	Documentation/networking/device_drivers/ethernet/qualcomm/ppe/ppe.rst
 F:	drivers/net/ethernet/qualcomm/ppe/
 
+QUALCOMM QMI (REMOTEPROC THERMAL MITIGATION) TMD
+M:	Gaurav Kohli <gaurav.kohli@oss.qualcomm.com>
+L:	linux-arm-msm@vger.kernel.org
+L:	linux-pm@vger.kernel.org
+S:	Maintained
+F:	drivers/soc/qcom/qmi_tmd.c
+
 QUALCOMM QSEECOM DRIVER
 M:	Maximilian Luz <luzmaximilian@gmail.com>
 L:	linux-arm-msm@vger.kernel.org
@@ -22656,7 +23044,7 @@ M:	Loic Poulain <loic.poulain@oss.qualcomm.com>
 L:	wcn36xx@lists.infradead.org
 L:	linux-wireless@vger.kernel.org
 S:	Supported
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/wcn36xx
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/wcn36xx.html
 F:	drivers/net/wireless/ath/wcn36xx/
 
 QUANTENNA QTNFMAC WIRELESS DRIVER
@@ -22673,7 +23061,7 @@ L:	amd-gfx@lists.freedesktop.org
 S:	Supported
 B:	https://gitlab.freedesktop.org/drm/amd/-/issues
 C:	irc://irc.oftc.net/radeon
-T:	git https://gitlab.freedesktop.org/agd5f/linux.git
+T:	git https://gitlab.freedesktop.org/drm/amdgpu/kernel.git
 F:	Documentation/gpu/amdgpu/
 F:	drivers/gpu/drm/amd/
 F:	drivers/gpu/drm/ci/xfails/amd*
@@ -23415,7 +23803,7 @@ RFKILL
 M:	Johannes Berg <johannes@sipsolutions.net>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
-W:	https://wireless.wiki.kernel.org/
+W:	https://wireless.docs.kernel.org/en/latest/
 Q:	https://patchwork.kernel.org/project/linux-wireless/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git
@@ -23503,7 +23891,8 @@ F:	drivers/iommu/riscv/
 
 RISC-V MICROCHIP SUPPORT
 M:	Conor Dooley <conor.dooley@microchip.com>
-M:	Daire McNamara <daire.mcnamara@microchip.com>
+R:	Valentina Fernandez <valentina.fernandezalanis@microchip.com>
+R:	Daire McNamara <daire.mcnamara@microchip.com>
 L:	linux-riscv@lists.infradead.org
 S:	Supported
 T:	git https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git/ (dts, soc, firmware)
@@ -23583,8 +23972,17 @@ F:	drivers/irqchip/irq-riscv-rpmi-sysmsi.c
 F:	drivers/mailbox/riscv-sbi-mpxy-mbox.c
 F:	include/linux/mailbox/riscv-rpmi-message.h
 
+RISC-V RPMI DEVICE POWER DRIVER
+M:	Joshua Yeong <joshua.yeong@starfivetech.com>
+L:	linux-riscv@lists.infradead.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/power/riscv,rpmi-device-power.yaml
+F:	Documentation/devicetree/bindings/power/riscv,rpmi-mpxy-device-power.yaml
+F:	drivers/pmdomain/riscv/
+
 RISC-V SPACEMIT SoC Support
 M:	Yixun Lan <dlan@kernel.org>
+R:	Troy Mitchell <troy.mitchell@linux.spacemit.com>
 L:	linux-riscv@lists.infradead.org
 L:	spacemit@lists.linux.dev
 S:	Maintained
@@ -23596,11 +23994,12 @@ N:	spacemit
 K:	spacemit
 
 RISC-V TENSTORRENT SoC SUPPORT
-M:	Drew Fustini <dfustini@oss.tenstorrent.com>
+M:	Drew Fustini <fustini@kernel.org>
 M:	Joel Stanley <jms@oss.tenstorrent.com>
+M:	Anirudh Srinivasan <asrinivasan@oss.tenstorrent.com>
 L:	linux-riscv@lists.infradead.org
 S:	Maintained
-T:	git https://github.com/tenstorrent/linux.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git
 F:	Documentation/devicetree/bindings/clock/tenstorrent,atlantis-prcm-rcpu.yaml
 F:	Documentation/devicetree/bindings/riscv/tenstorrent.yaml
 F:	arch/riscv/boot/dts/tenstorrent/
@@ -23817,6 +24216,7 @@ F:	drivers/clk/clk-bd718x7.c
 F:	drivers/gpio/gpio-bd71815.c
 F:	drivers/gpio/gpio-bd71828.c
 F:	drivers/gpio/gpio-bd72720.c
+F:	drivers/gpio/gpio-bd73800.c
 F:	drivers/mfd/rohm-bd71828.c
 F:	drivers/mfd/rohm-bd718x7.c
 F:	drivers/mfd/rohm-bd9576.c
@@ -23836,6 +24236,7 @@ F:	include/linux/mfd/rohm-bd71815.h
 F:	include/linux/mfd/rohm-bd71828.h
 F:	include/linux/mfd/rohm-bd718x7.h
 F:	include/linux/mfd/rohm-bd72720.h
+F:	include/linux/mfd/rohm-bd73800.h
 F:	include/linux/mfd/rohm-bd957x.h
 F:	include/linux/mfd/rohm-bd96801.h
 F:	include/linux/mfd/rohm-bd96802.h
@@ -23969,6 +24370,7 @@ K:	\b(?i:rust)\b
 
 RUST [ALLOC]
 M:	Danilo Krummrich <dakr@kernel.org>
+M:	Alice Ryhl <aliceryhl@google.com>
 R:	Lorenzo Stoakes <ljs@kernel.org>
 R:	Vlastimil Babka <vbabka@kernel.org>
 R:	Liam R. Howlett <liam@infradead.org>
@@ -24472,6 +24874,11 @@ R:	Marc Murphy <marc.murphy@sancloud.com>
 S:	Supported
 F:	arch/arm/boot/dts/ti/omap/am335x-sancloud*
 
+SB1250-DUART BROADCOM SOC SERIAL DRIVER
+M:	"Maciej W. Rozycki" <macro@orcam.me.uk>
+S:	Maintained
+F:	drivers/tty/serial/sb1250-duart.c
+
 SBOM
 M:	Luis Augenstein <luis.augenstein@tngtech.com>
 M:	Maximilian Huber <maximilian.huber@tngtech.com>
@@ -24685,6 +25092,13 @@ L:	linux-mmc@vger.kernel.org
 S:	Maintained
 F:	drivers/mmc/host/sdhci-brcmstb*
 
+SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) CADENCE DRIVER
+M:	Tanmay Kathpalia <tanmay.kathpalia@altera.com>
+L:	linux-mmc@vger.kernel.org
+S:	Supported
+F:	Documentation/devicetree/bindings/mmc/cdns,sd6hc.yaml
+F:	drivers/mmc/host/sdhci-cadence*
+
 SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) DRIVER
 M:	Adrian Hunter <adrian.hunter@intel.com>
 L:	linux-mmc@vger.kernel.org
@@ -24742,7 +25156,9 @@ M:	Sudeep Holla <sudeep.holla@kernel.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 F:	drivers/firmware/smccc/
+F:	include/linux/arm-smccc-bus.h
 F:	include/linux/arm-smccc.h
+F:	include/linux/device-id/arm_smccc.h
 
 SECURITY CONTACT
 M:	Security Officers <security@kernel.org>
@@ -25142,7 +25558,7 @@ SIS I2C TOUCHSCREEN DRIVER
 M:	Mika Penttilä <mpenttil@redhat.com>
 L:	linux-input@vger.kernel.org
 S:	Maintained
-F:	Documentation/devicetree/bindings/input/touchscreen/sis_i2c.txt
+F:	Documentation/devicetree/bindings/input/touchscreen/sis,9200-ts.yaml
 F:	drivers/input/touchscreen/sis_i2c.c
 
 SIS USB2VGA DRIVER
@@ -25182,7 +25598,7 @@ R:	David Rientjes <rientjes@google.com>
 R:	Roman Gushchin <roman.gushchin@linux.dev>
 L:	linux-mm@kvack.org
 S:	Maintained
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mm/slab.git
 F:	Documentation/admin-guide/mm/slab.rst
 F:	Documentation/mm/slab.rst
 F:	include/linux/mempool.h
@@ -25728,6 +26144,14 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/sound/spacemit,k1-i2s.yaml
 F:	sound/soc/spacemit/k1_i2s.c
 
+SPACEMIT RESET CONTROLLER DRIVER
+M:	Guodong Xu <docular.xu@gmail.com>
+L:	linux-riscv@lists.infradead.org
+L:	spacemit@lists.linux.dev
+S:	Maintained
+F:	drivers/reset/spacemit/
+F:	include/dt-bindings/reset/spacemit,k3-resets.h
+
 SPANISH DOCUMENTATION
 M:	Carlos Bilbao <carlos.bilbao@kernel.org>
 R:	Avadhut Naik <avadhut.naik@amd.com>
@@ -26218,6 +26642,15 @@ F:	rust/helpers/jump_label.c
 F:	rust/kernel/generated_arch_static_branch_asm.rs.S
 F:	rust/kernel/jump_label.rs
 
+STEAL GOVERNOR DRIVER
+M:	Shrikanth Hegde <sshegde@linux.ibm.com>
+R:	Yury Norov <yury.norov@gmail.com>
+L:	linux-kernel@vger.kernel.org
+S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core
+F:	Documentation/driver-api/steal-governor.rst
+F:	drivers/virt/steal_governor.c
+
 STI AUDIO (ASoC) DRIVERS
 M:	Arnaud Pouliquen <arnaud.pouliquen@foss.st.com>
 L:	linux-sound@vger.kernel.org
@@ -26548,6 +26981,7 @@ F:	include/linux/soc/amd/isp4_misc.h
 
 SYNOPSYS DESIGNWARE MIPI CSI-2 RECEIVER DRIVER
 M:	Michael Riesch <michael.riesch@collabora.com>
+R:	Bryan O'Donoghue <bod@kernel.org>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/media/rockchip,rk3568-mipi-csi2.yaml
@@ -26605,6 +27039,7 @@ F:	drivers/pmdomain/arm/
 F:	drivers/powercap/arm_scmi_powercap.c
 F:	drivers/regulator/scmi-regulator.c
 F:	drivers/reset/reset-scmi.c
+F:	include/linux/device-id/scmi.h
 F:	include/linux/sc[mp]i_protocol.h
 F:	include/trace/events/scmi.h
 F:	include/uapi/linux/virtio_scmi.h
@@ -26806,11 +27241,6 @@ S:	Maintained
 F:	Documentation/process/contribution-maturity-model.rst
 F:	Documentation/process/researcher-guidelines.rst
 
-TECHNOLOGIC SYSTEMS TS-5500 PLATFORM SUPPORT
-M:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
-S:	Maintained
-F:	arch/x86/platform/ts5500/
-
 TECHNOTREND USB IR RECEIVER
 M:	Sean Young <sean@mess.org>
 L:	linux-media@vger.kernel.org
@@ -26926,9 +27356,11 @@ M:	Luca Ceresoli <luca.ceresoli@bootlin.com>
 L:	linux-media@vger.kernel.org
 L:	linux-tegra@vger.kernel.org
 S:	Maintained
+F:	Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-csi.yaml
 F:	Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.yaml
 F:	Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-vi.yaml
 F:	Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-vip.yaml
+F:	Documentation/devicetree/bindings/display/tegra/nvidia,tegra210-csi.yaml
 F:	drivers/staging/media/tegra-video/
 
 TEGRA XUSB PADCTL DRIVER
@@ -27013,7 +27445,7 @@ TEXAS INSTRUMENTS DMA DRIVERS
 M:	Vignesh Raghavendra <vigneshr@ti.com>
 L:	dmaengine@vger.kernel.org
 S:	Maintained
-F:	Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt
+F:	Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml
 F:	Documentation/devicetree/bindings/dma/ti-edma.txt
 F:	Documentation/devicetree/bindings/dma/ti/
 F:	drivers/dma/ti/
@@ -27238,6 +27670,13 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/adc/ti,ads1119.yaml
 F:	drivers/iio/adc/ti-ads1119.c
 
+TI ADS112C04 ADC DRIVER
+M:	Kyle Hsieh <kylehsieh1995@gmail.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/adc/ti,ads112c04.yaml
+F:	drivers/iio/adc/ti-ads112c04.c
+
 TI ADS112C14 ADC DRIVER
 M:	David Lechner <dlechner@baylibre.com>
 L:	linux-iio@vger.kernel.org
@@ -27510,8 +27949,8 @@ F:	drivers/media/platform/ti/vpe/
 TI WILINK WIRELESS DRIVERS
 L:	linux-wireless@vger.kernel.org
 S:	Orphan
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/wl12xx
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/wl1251
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/wl12xx.html
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/wl1251.html
 F:	drivers/net/wireless/ti/
 
 TIMEKEEPING, CLOCKSOURCE CORE, NTP, ALARMTIMER
@@ -27658,7 +28097,7 @@ TOSHIBA TC358743 DRIVER
 M:	Hans Verkuil <hverkuil@kernel.org>
 L:	linux-media@vger.kernel.org
 S:	Maintained
-F:	Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.txt
+F:	Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.yaml
 F:	drivers/media/i2c/tc358743*
 F:	include/media/i2c/tc358743.h
 
@@ -27998,6 +28437,12 @@ F:	Documentation/devicetree/bindings/clock/ultrarisc,dp1000-clk.yaml
 F:	drivers/clk/ultrarisc/*
 F:	include/dt-bindings/clock/ultrarisc,dp1000-clk.h
 
+ULTRARISC DP1000 DWMAC GLUE LAYER
+M:	Jia Wang <wangjia@ultrarisc.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/net/ultrarisc,dp1000-gmac.yaml
+F:	drivers/net/ethernet/stmicro/stmmac/dwmac-ultrarisc.c
+
 ULTRARISC DP1000 PINCTRL DRIVER
 M:	Jia Wang <wangjia@ultrarisc.com>
 L:	linux-gpio@vger.kernel.org
@@ -28720,6 +29165,7 @@ VFIO SELFTESTS
 M:	David Matlack <dmatlack@google.com>
 L:	kvm@vger.kernel.org
 S:	Maintained
+F:	Documentation/driver-api/vfio-selftests.rst
 F:	tools/testing/selftests/vfio/
 
 VFIO VIRTIO PCI DRIVER
@@ -29095,6 +29541,12 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml
 F:	drivers/iio/light/veml6030.c
 
+VISHAY VEML6031X00 AMBIENT LIGHT SENSOR DRIVER
+M:	Javier Carrasco <javier.carrasco.cruz@gmail.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/light/vishay,veml6030.yaml
+F:	drivers/iio/light/veml6031x00.c
+
 VISHAY VEML6046X00 RGBIR COLOR SENSOR DRIVER
 M:	Andreas Klinger <ak@it-klinger.de>
 S:	Maintained
@@ -29296,11 +29748,6 @@ L:	linux-hwmon@vger.kernel.org
 S:	Maintained
 F:	drivers/hwmon/vt8231.c
 
-VUB300 USB to SDIO/SD/MMC bridge chip
-L:	linux-mmc@vger.kernel.org
-S:	Orphan
-F:	drivers/mmc/host/vub300.c
-
 W1 DALLAS'S 1-WIRE BUS
 M:	Krzysztof Kozlowski <krzk@kernel.org>
 S:	Maintained
@@ -29394,7 +29841,7 @@ F:	drivers/hid/hid-wiimote*
 WILOCITY WIL6210 WIRELESS DRIVER
 L:	linux-wireless@vger.kernel.org
 S:	Orphan
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/wil6210
+W:	https://wireless.docs.kernel.org/en/latest/en/users/drivers/wil6210.html
 F:	drivers/net/wireless/ath/wil6210/
 
 WINBOND CIR DRIVER
@@ -29433,6 +29880,13 @@ M:	Miloslav Trmac <mitr@volny.cz>
 S:	Maintained
 F:	drivers/input/misc/wistron_btns.c
 
+WIZNET W5100 ETHERNET DRIVER
+M:	Arthur Crepin Leblond <arthur@marmottus.net>
+L:	netdev@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/net/wiznet,w5100.yaml
+F:	drivers/net/ethernet/wiznet/
+
 WMI BINARY MOF DRIVER
 M:	Armin Wolf <W_Armin@gmx.de>
 R:	Thomas Weißschuh <linux@weissschuh.net>
@@ -30183,9 +30637,12 @@ F:	tools/testing/selftests/cgroup/test_zswap.c
 
 SENARYTECH AUDIO CODEC DRIVER
 M:	bo liu <bo.liu@senarytech.com>
+M:	qianghua.wang <qianghua.wang@senarytech.com>
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 F:	sound/hda/codecs/senarytech.c
+F:	sound/soc/codecs/sn624x*
+F:	sound/soc/sdw_utils/soc_sdw_senary*
 
 ZTE DINGHAI ETHERNET DRIVER
 M:	Junyang Han <han.junyang@zte.com.cn>
diff --git a/Makefile b/Makefile
index 0f1b80100b4798..7fcef10824cd2a 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@
 VERSION = 7
 PATCHLEVEL = 3
 SUBLEVEL = 0
-EXTRAVERSION = -rc3
+EXTRAVERSION = -rc5
 NAME = Baby Opossum Posse
 
 # *DOCUMENTATION*
@@ -298,6 +298,7 @@ no-dot-config-targets := $(clean-targets) \
 			 %asm-generic kernelversion %src-pkg dt_binding_check \
 			 dt_style_selftest \
 			 outputmakefile rustavailable rustfmt rustfmtcheck \
+			 scripts_gen_init_cpio \
 			 run-command
 no-sync-config-targets := $(no-dot-config-targets) %install modules_sign kernelrelease \
 			  image_name
@@ -568,6 +569,9 @@ LZMA		= lzma
 LZ4		= lz4
 XZ		= xz
 ZSTD		= zstd
+# The kernel image is compressed with pigz, on the job slots make has free,
+# if it is installed. Everything else uses KGZIP.
+KPGZIP		:= $(if $(shell command -v pigz 2>/dev/null),$(PYTHON3) $(abs_srctree)/scripts/jobserver-exec pigz -p %PARALLELISM%,$(KGZIP))
 TAR		= tar
 
 CHECKFLAGS     := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
@@ -648,7 +652,7 @@ export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN LLVM_
 export HOSTRUSTC KBUILD_HOSTRUSTFLAGS
 export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
 export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
-export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD TAR
+export KGZIP KPGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD TAR
 export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS KBUILD_PROCMACROLDFLAGS LDFLAGS_MODULE
 export KBUILD_USERCFLAGS KBUILD_USERLDFLAGS
 
@@ -681,8 +685,8 @@ ifdef building_out_of_srctree
 # outputmakefile generates a Makefile in the output directory, if using a
 # separate output directory. This allows convenient use of make in the
 # output directory.
-# At the same time when output Makefile generated, generate .gitignore to
-# ignore whole output directory
+# At the same time as the output Makefile is generated, generate .gitignore to
+# ignore the whole output directory
 
 ifdef KBUILD_EXTMOD
 print_env_for_makefile = \
@@ -946,8 +950,8 @@ KBUILD_RUSTFLAGS += -Coverflow-checks=$(if $(CONFIG_RUST_OVERFLOW_CHECKS),y,n)
 ifdef CONFIG_CC_IS_GCC
 # gcc-10 renamed --param=allow-store-data-races=0 to
 # -fno-allow-store-data-races.
-KBUILD_CFLAGS	+= $(call cc-option,--param=allow-store-data-races=0)
-KBUILD_CFLAGS	+= $(call cc-option,-fno-allow-store-data-races)
+KBUILD_CFLAGS	+= $(CONFIG_CC_OPT_ALLOW_STORE_DATA_RACES_PARAM)
+KBUILD_CFLAGS	+= $(CONFIG_CC_OPT_NO_ALLOW_STORE_DATA_RACES)
 endif
 
 ifdef CONFIG_READABLE_ASM
@@ -1011,18 +1015,20 @@ endif
 endif
 
 # Explicitly clear padding bits during variable initialization
-KBUILD_CFLAGS += $(call cc-option,-fzero-init-padding-bits=all)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_ZERO_INIT_PADDING_BITS)
 
 # While VLAs have been removed, GCC produces unreachable stack probes
 # for the randomize_kstack_offset feature. Disable it for all compilers.
+# Probed here rather than in Kconfig as clang only accepts it for some
+# sub-targets, so the architecture's flags matter.
 KBUILD_CFLAGS	+= $(call cc-option, -fno-stack-clash-protection)
 
 # Get details on warnings generated due to GCC value tracking.
-KBUILD_CFLAGS	+= $(call cc-option, -fdiagnostics-show-context=2)
+KBUILD_CFLAGS	+= $(CONFIG_CC_OPT_DIAGNOSTICS_SHOW_CONTEXT)
 
 # Show inlining notes for __attribute__((warning/error)) call chains.
 # GCC supports this unconditionally while Clang 23+ provides a flag.
-KBUILD_CFLAGS	+= $(call cc-option, -fdiagnostics-show-inlining-chain)
+KBUILD_CFLAGS	+= $(CONFIG_CC_OPT_DIAGNOSTICS_SHOW_INLINING_CHAIN)
 
 # Clear used registers at func exit (to reduce data lifetime and ROP gadgets).
 ifdef CONFIG_ZERO_CALL_USED_REGS
@@ -1033,7 +1039,7 @@ ifdef CONFIG_FUNCTION_TRACER
 ifdef CONFIG_FTRACE_MCOUNT_USE_CC
   CC_FLAGS_FTRACE	+= -mrecord-mcount
   ifdef CONFIG_HAVE_NOP_MCOUNT
-    ifeq ($(call cc-option-yn, -mnop-mcount),y)
+    ifdef CONFIG_CC_HAS_MNOP_MCOUNT
       CC_FLAGS_FTRACE	+= -mnop-mcount
       CC_FLAGS_USING	+= -DCC_USING_NOP_MCOUNT
     endif
@@ -1051,8 +1057,7 @@ ifdef CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT
   endif
 endif
 ifdef CONFIG_HAVE_FENTRY
-  # s390-linux-gnu-gcc did not support -mfentry until gcc-9.
-  ifeq ($(call cc-option-yn, -mfentry),y)
+  ifdef CONFIG_CC_HAS_MFENTRY
     CC_FLAGS_FTRACE	+= -mfentry
     CC_FLAGS_USING	+= -DCC_USING_FENTRY
   endif
@@ -1144,7 +1149,7 @@ export CC_FLAGS_NO_FPU
 
 ifneq ($(CONFIG_FUNCTION_ALIGNMENT),0)
 # Set the minimal function alignment. Use the newer GCC option
-# -fmin-function-alignment if it is available, or fall back to -falign-funtions.
+# -fmin-function-alignment if it is available, or fall back to -falign-functions.
 # See also CONFIG_CC_HAS_SANE_FUNCTION_ALIGNMENT.
 ifdef CONFIG_CC_HAS_MIN_FUNCTION_ALIGNMENT
 KBUILD_CFLAGS += -fmin-function-alignment=$(CONFIG_FUNCTION_ALIGNMENT)
@@ -1160,7 +1165,7 @@ NOSTDINC_FLAGS += -nostdinc
 # the kernel uses only C99 flexible arrays for dynamically sized trailing
 # arrays. Enforce this for everything that may examine structure sizes and
 # perform bounds checking.
-KBUILD_CFLAGS += $(call cc-option, -fstrict-flex-arrays=3)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_STRICT_FLEX_ARRAYS)
 
 # disable invalid "can't wrap" optimizations for signed / pointers
 KBUILD_CFLAGS	+= -fno-strict-overflow
@@ -1199,7 +1204,10 @@ export CFLAGS_GCOV
 
 # change __FILE__ to the relative path to the source directory
 ifdef building_out_of_srctree
-KBUILD_CPPFLAGS += -fmacro-prefix-map=$(srcroot)/=
+CFLAGS_PREFIX_MAP := -fmacro-prefix-map=$(srcroot)/=
+KBUILD_CPPFLAGS += $(CFLAGS_PREFIX_MAP)
+KBUILD_USERCFLAGS += $(CFLAGS_PREFIX_MAP)
+KBUILD_HOSTCFLAGS += $(CFLAGS_PREFIX_MAP)
 ifeq ($(call rustc-option-yn, --remap-path-scope=macro),y)
 KBUILD_RUSTFLAGS += --remap-path-prefix=$(srcroot)/= --remap-path-scope=macro
 endif
@@ -1240,11 +1248,11 @@ LDFLAGS_vmlinux += --build-id=sha1
 # COMDAT-deduplicated sections. Use --force-group-allocation to resolve these
 # groups when linking modules. The option is available from ld.bfd 2.29 and
 # ld.lld 19.1.0.
-KBUILD_LDFLAGS_MODULE += $(call ld-option,--force-group-allocation)
+KBUILD_LDFLAGS_MODULE += $(CONFIG_LD_OPT_FORCE_GROUP_ALLOCATION)
 
 KBUILD_LDFLAGS	+= -z noexecstack
 ifeq ($(CONFIG_LD_IS_BFD),y)
-KBUILD_LDFLAGS	+= $(call ld-option,--no-warn-rwx-segments)
+KBUILD_LDFLAGS	+= $(CONFIG_LD_OPT_NO_WARN_RWX_SEGMENTS)
 endif
 
 ifeq ($(CONFIG_STRIP_ASM_SYMS),y)
@@ -1262,8 +1270,9 @@ ifdef CONFIG_LD_ORPHAN_WARN
 LDFLAGS_vmlinux += --orphan-handling=$(CONFIG_LD_ORPHAN_WARN_LEVEL)
 endif
 
+# --emit-relocs is added by scripts/link-vmlinux.sh, for the final link only.
 ifneq ($(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS),)
-LDFLAGS_vmlinux	+= --emit-relocs --discard-none
+LDFLAGS_vmlinux	+= --discard-none
 endif
 
 # Align the architecture of userspace programs with the kernel
@@ -1528,9 +1537,9 @@ ifdef CONFIG_HEADERS_INSTALL
 prepare: headers
 endif
 
-PHONY += usr_gen_init_cpio
-usr_gen_init_cpio: scripts_basic
-	$(Q)$(MAKE) $(build)=usr usr/gen_init_cpio
+PHONY += scripts_gen_init_cpio
+scripts_gen_init_cpio: scripts_basic
+	$(Q)$(MAKE) $(build)=scripts scripts/gen_init_cpio
 
 PHONY += scripts_unifdef
 scripts_unifdef: scripts_basic
@@ -1831,7 +1840,7 @@ distclean: mrproper
 # Packaging of the kernel to various formats
 # ---------------------------------------------------------------------------
 
-modules-cpio-pkg: usr_gen_init_cpio
+modules-cpio-pkg: scripts_gen_init_cpio
 
 %src-pkg: FORCE
 	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.package $@
@@ -2002,7 +2011,7 @@ $(help-board-dirs): help-%:
 # ---------------------------------------------------------------------------
 DOC_TARGETS := xmldocs latexdocs pdfdocs htmldocs epubdocs cleandocs \
 	       linkcheckdocs dochelp refcheckdocs texinfodocs infodocs mandocs \
-	       htmldocs-redirects
+	       htmldocs-redirects testdocs
 
 PHONY += $(DOC_TARGETS)
 $(DOC_TARGETS):
diff --git a/Next/SHA1s b/Next/SHA1s
new file mode 100644
index 00000000000000..0ae8ae9e27a2c7
--- /dev/null
+++ b/Next/SHA1s
@@ -0,0 +1,433 @@
+Name		SHA1
+----		----
+origin		551c722f40809618230001baccf219193e22fc5a
+ext4-fixes	981fcc5674e67158d24d23e841523eccba19d0e7
+vfs-brauner-fixes	b78b728e21c32ec4c330b299f657fb1eb02dffc2
+fscrypt-current	cee9395acd8043be0644b25c34bfa86623f2b935
+fsverity-current	cee9395acd8043be0644b25c34bfa86623f2b935
+btrfs-fixes	50a31f73ef35e06375895ba37814acc3bd9479db
+vfs-fixes	49c5d168a3a8f4eb27d44a2a22b7e8a856ca601f
+erofs-fixes	135d84c66f85426299db01a09d93a79a87af18ba
+nfsd-fixes	f76017a7663c4ce5e379f8a8d39f032bdb1fd865
+v9fs-fixes	028ef9c96e96197026887c0f092424679298aae8
+overlayfs-fixes	4549871118cf616eecdd2d939f78e3b9e1dddc48
+fscrypt		a63883d2c3d47ce6d41918c2453a4513b921eaaa
+btrfs		55d05b596923df9445a9f6179b8807d6d4c5a50d
+ceph		dc173b37415e8f738fc4de477490056b479ddc9f
+cifs		f14572c203d57492e1d4e5d7851a3b143e083b82
+configfs	620938e7da8943970ec25e1c8cdbf44ba1d44fa1
+configfs-rust	6dc9fb1516aa4930329ef561f078c7de9257abf2
+ecryptfs	f81cb44f9a4b88d73ee5dec4a1ccdb0232fd2e3f
+dlm		ed9b6a1296f10e4881d93dfe6d76013fbbaeee87
+erofs		a7d28aa0e9b2c983b915d091f9596f0e3b253355
+exfat		216426aff8f69379f03337857551ca0ed3190361
+ext3		ba5855e74bcd761123e39f4708834a0015a74a8b
+ext4		9091c97be34083587a75db174aab51551d8e8543
+f2fs		a4b9fb9e69116e9ce35e1de2ca2bd940552b8d1d
+fsverity	b08e4ee274917074633a41b3598d5541cee2b914
+fuse		29b60c6c561f71784c43d4c5d65565ce46e223c6
+gfs2		d0c4bce31a579216780b990a09250f15e00f38ff
+jfs		dad98c5b2a05ef744af4c884c97066a3c8cdad61
+ksmbd		d30f0c30c87d62a7be7fdad295a01860a0d846ef
+nfs		fd73f4a6659897191fa0d40695fe370925dd3780
+nfs-anna	9bafc322b7fca03717db78615b07bd401367b7c2
+nfsd		ac04dab23b5ff28fc7e41957824c5c439ae99887
+ntfs		708f9d56cacae21aeee98d16bcdd50a66edc04a0
+ntfs3		f3b8ee6c05bed24a06192ea8e4cafcc06946c1bd
+orangefs	2bc09cb0e9e44c7e622b956332ebfb9b638fd5f7
+overlayfs	1f6ee9be92f8df85a8c9a5a78c20fd39c0c21a95
+ubifs		a5e0055eac837a1168c781653943d4a0d9920af3
+v9fs		c60ae98c5aa64021751b38ab1313b19d620bf640
+v9fs-ericvh	028ef9c96e96197026887c0f092424679298aae8
+xfs		b4787e7b9730d52f33cf8dd23b3c42f1237f9753
+zonefs		3a8389d42bdf4213730f4067f8bfa78bae6564ef
+vfs-brauner	84086827932b58e7645d93d970bbc566c4ee408b
+vfs		4dda01b67c8662c5d0c53034974cb8280c575a5d
+mm-fixes	ffd79f732828e5fe7c9f97f9e5930f95db70583b
+fs-current	5a9410e0a34d2e31af3e2c97d2dc428d08e41f39
+kbuild-current	fd73f4a6659897191fa0d40695fe370925dd3780
+clang-fixes-current	df2908090cda368b01ff43709f51890076c56157
+arc-current	f050c3e61d2a1aaece3170d459e4ce5fa2486c12
+arm-current	1039bffd6ae9c75b42b7d148d6c1106134107b66
+arm64-fixes	3872cc6b92940af0f73c6f9a45d65300492d648d
+arm-soc-fixes	4dd1999783d7d12434006289338373e49492dc96
+davinci-current	cee9395acd8043be0644b25c34bfa86623f2b935
+realtek-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+drivers-memory-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+sophgo-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+sophgo-soc-fixes	0af2f6be1b4281385b618cb86ad946eded089ac8
+m68k-current	2f8e3cad53b5c36ab0ed5d3195bfc55c59ea61a5
+powerpc-fixes	93f51579e7df248780214094418f205253383cc5
+s390-fixes	5b76268dac968612f7283d59b539036de955b7d9
+net		99b43ede9e355ba35244cc9470bf1819774ce39d
+bpf		564b7e8e09ec9b47921dc11de0de05da358335d9
+ipsec		868f63c8bfafa9b827168c9126f85264c39c02ec
+netfilter	9c572a83037a7dcd653ba3a9cc468c16b857d0c9
+ipvs		a401a9d547c50ef34db1088cc1fb9a201a7af657
+bluetooth-fixes	86ef0f58bdecdedb3a1240971c56d71b7e4ce3fc
+wireless	6f63e919fe1e335b8abcb3a28bfd4804a98d875a
+ath		6f63e919fe1e335b8abcb3a28bfd4804a98d875a
+iwlwifi		6f63e919fe1e335b8abcb3a28bfd4804a98d875a
+wpan		2b4707a149a55e8fa75c9ef32b359d60f470a566
+rdma-fixes	72d3fcf802c45d00b300f25b848a93c3a2bd7c7e
+sound-current	a154f7b9f197b3d5e41fa3ad62083f5aa93b1fe8
+sound-asoc-fixes	b2047b8cadadccb1c9269ce756c399cd9aef2c82
+regmap-fixes	2e42cade8ff1ff579e77976d9869db4df1feaf74
+regulator-fixes	f3e6ef13e24c9f26dca0d35de57fcdf04f78e378
+spi-fixes	3d743adf090cd4c9a2120c1e02b0482e88aa0d2d
+pci-current	cee9395acd8043be0644b25c34bfa86623f2b935
+driver-core.current	72d3fcf802c45d00b300f25b848a93c3a2bd7c7e
+tty.current	5dad87615c9861cfa366ca984b52f581e861df20
+usb.current	abc36cbda29d8f19cf3a580cd86ca9e865186a41
+usb-serial-fixes	ec06546b43b612cefc851558981b4955c0ef9e46
+phy		93f51579e7df248780214094418f205253383cc5
+staging.current	df2908090cda368b01ff43709f51890076c56157
+iio-fixes	d2f2868d2f487b418f68db2ef5bd7cf41722ee68
+watchdog-fixes	2686e13649a9e846f6c08cf4ada0256ca19c3475
+counter-current	fd73f4a6659897191fa0d40695fe370925dd3780
+char-misc.current	540f55de8c9f79c83f13e44910955faf82b0d79c
+soundwire-fixes	93f51579e7df248780214094418f205253383cc5
+thunderbolt-fixes	395e9f2967a7ac6898026e8f136c369805dc2fd9
+input-current	309731e95917125bbd13626a7a5600490a5bf44f
+crypto-current	10396a2d6d41d594975b6ece712278570c3c970c
+libcrypto-fixes	6d996c3b974c501f347e8bad1644afd1180b2795
+vfio-fixes	e242e974e812e7a47e3088860c80d9492fac314f
+kselftest-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+dmaengine-fixes	93f51579e7df248780214094418f205253383cc5
+backlight-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+mtd-fixes	1c1a342aceec79528b3ba51f376eca2be5928fe7
+mfd-fixes	d5d2d7a8d8be18681a0864f58e3875f1c639e11c
+v4l-dvb-fixes	2579cbe68005f46fc7f8f95364f6b101b07b9d1c
+reset-fixes	71827776667f4e4677a4fa806bcfb24d4b8dd9d7
+mips-fixes	93f51579e7df248780214094418f205253383cc5
+at91-fixes	afb1ecfeda3fc31d6300a176f4854cc62e289bf3
+omap-fixes	2fabd2f406d0cf787be47b3bbeea99b30004033d
+tegra-fixes	52574866649979631b76b601053be903ab2453bd
+kvm-fixes	973ea70393e885e540f714904e51bc6cac80e3d7
+kvms390-fixes	f47190b08b71e8482072978373ee88cb2dfbdaf4
+kvm-arm-fixes	afb4334fb52c7b416bbc9c4b166f6806509f734b
+hwmon-fixes	61406e9cac695b19b979b002d790d346b9f07887
+nvdimm-fixes	a8aec14230322ed8f1e8042b6d656c1631d41163
+cxl-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+dma-mapping-fixes	057e5e07420c753f248f9ce148040ab6dcf8359f
+drivers-x86-fixes	d144a494d81fcf2d1c5cf58b01c655bb8bafc701
+samsung-krzk-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+pinctrl-samsung-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+pinctrl-qcom-fixes	19fc4240358be2a25ce8e0a49a2819588ed61c5d
+devicetree-fixes	21c89ff1fc86ffa517f3a9ca1ba5c48e9e37c1ba
+dt-krzk-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+scsi-fixes	42d1221d321e55afc7bba9109a77aaf5a817c8a3
+drm-fixes	a9ed3aa9b87ee41e8ab3ff471b1331c154767e45
+drm-intel-fixes	c034e8a46e4cb703018fe7e10fe5a3a974d6a7c3
+mmc-fixes	aa0a37b5024d921e96ac4f8f487c8bfbf1f1a582
+rtc-fixes	055ef5ce9f67a6a3a1363fa663007f8196cc0fb8
+gnss-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+hyperv-fixes	ca039df94983c441fdee0e2488263fc744e8ecb6
+risc-v-fixes	e1116094833337fe9480fb9527a59c7637b96118
+riscv-dt-fixes	0f70fd6f4c1d6c1f2ea298d8dd07c4a4d5dff9af
+riscv-soc-fixes	dc59e4fea9d83f03bad6bddf3fa2e52491777482
+fpga-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+spdx		cee9395acd8043be0644b25c34bfa86623f2b935
+gpio-brgl-fixes	ff82fc3a4a1d417dee1229681cc5285986edb1fa
+gpio-intel-fixes	8d37f20173a2760718bb02b58bafc5a7cac93d67
+pinctrl-intel-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+auxdisplay-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+kunit-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+renesas-fixes	8dc2615d5702059b2b71fca6f93c0d7d10ae54cb
+perf-current	aadea57f532882d8bab444646863c7ef8a778ff1
+efi-fixes	d8809f6931065cbbf3554647a50a65a471ab5983
+battery-fixes	a58cbc8b36ec0cd00d2ba3de7d003de818a27523
+iommufd-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+rust-fixes	551c722f40809618230001baccf219193e22fc5a
+w1-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+pmdomain-fixes	4c66c423593e26273ea0d644906ed007630c6f6d
+i2c-andi-fixes	840d8acc87925deb3c75566fde9ca81d2f47f98c
+i2c-rust-fixes	4eb422482ca5d924d7212ad2ca1cb7ea6f5b524d
+sparc-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+clk-fixes	81493c1dd1b1ebecb2843a7815973a5bd9a37e5a
+thead-clk-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+tenstorrent-clk-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+fustini-config-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+pwrseq-fixes	58a00330248154461c52a6f3bd5c01e5b67f9560
+thead-dt-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+ftrace-fixes	1650a1b6cb1ae6cb99bb4fce21b30ebdf9fc238e
+ring-buffer-fixes	057caace5214da3b457bbd295e1a2ad34d3685ea
+trace-fixes	d860c67c051685abb0460b593b193f0f45f4fa92
+tracefs-fixes	07004a8c4b572171934390148ee48c4175c77eed
+spacemit-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+tip-fixes	3ebb3531c6d0d533fddc65fa3bed174291d71956
+kexec-fixes	a901b0778ae82d46084b55237e65d3a7bb5f0c86
+liveupdate-fixes	3a0b8fa2eb36afc88b62a95f33f0c77c71fa5ded
+drm-msm-fixes	a15fac810c76397ec9f62a6fc26c4d7ab6e238a7
+uml-fixes	af421e9aed3920c7ac88c24daa48606c7112feca
+fwctl-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+devsec-tsm-fixes	c3fd16c3b98ed726294feab2f94f876290bf7b61
+drm-rust-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+tenstorrent-dt-fixes	cee9395acd8043be0644b25c34bfa86623f2b935
+nfc-fixes	b61732f47316d45f27706db7812950145d3327b5
+mm-nonmm-hotfixes-stable	a243ede718463c7b481878656f1ff32a0ce0fd54
+mm-nonmm-hotfixes-unstable	844c67369f72de6c89ad783b3fd1d59f8e8f4902
+drm-misc-fixes	e78a9fb7a40c55ec2a70ae39903dde4f898af2ef
+rust		72d3fcf802c45d00b300f25b848a93c3a2bd7c7e
+rust-interop	05f7e89ab9731565d8a62e3b5d1ec206485eeb0b
+rust-alloc	6e8339118040f24ebcc21db22c9c1b7398b7e39e
+rust-io		86731a2a651e58953fc949573895f2fa6d456841
+rust-pin-init	95593ed3616194c9498cc190cd1a3693ff4bca37
+rust-timekeeping	2ea0119f72dba597aa8a98cbdb72c564bfc5cb38
+rust-xarray	c455f19bbe6104debd980bb15515faf716bd81b8
+rust-analyzer	5f45afb8ab04d934fc0601a202f95267ebc20059
+mm		7a6c9825d917f0428982af72a233a36dbf45cf8b
+mm-nonmm-stable	a243ede718463c7b481878656f1ff32a0ce0fd54
+mm-nonmm-unstable	cbb17a7a0afe4f2a766b25e75777f70f47dbc6ba
+kbuild		59ba9b8f8803c132f59db74bdd4b809c30dbe3c7
+clang-fixes	0bb666d5f5a2339a5692afb312c2161df9620503
+clang-format	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+perf		45d15e89a783a0a279b7a54f9018c230127380d7
+compiler-attributes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+dma-mapping	57a57ae077f0f167056fc08f1666678f78e5b147
+asm-generic	adbbd9714f8058730f93c8df5c5bf1679456424b
+alpha		d58041d2c63e09a1c9083e0e9f4151e487c4e16a
+arm		1a89abc009cb5035d0cb4e5ba48d32b94f30e8e4
+arm64		6c360697fd4d17a53ed977a1341add3520a19269
+arm-perf	cee9395acd8043be0644b25c34bfa86623f2b935
+arm-soc		b9bb4134a9583a388704433eef22c82c49963585
+amlogic		a2530c83c458a503e55219966217dfd1c36a385e
+asahi-soc	9378cd5ddcebc06fdd803373487ed1a376532391
+at91		bd49a1c72abbdddbc7b3d691d926623e33a7c1a0
+bmc		cd7d1ef7d74ed5f5a1790a40389bc38382924556
+broadcom	c0f2033a9ce4b843ca3e947536a0607ecf982be1
+cix		a0cffbd8878c55b12dc4555f883adf3372cecd91
+davinci		cee9395acd8043be0644b25c34bfa86623f2b935
+drivers-memory	a22355280361d2a376a2020059a2bae11e6cea10
+fsl		7b3b0598c00e67f2df85d0b4cd99eaa3be6c9bfd
+imx-mxs		5d034f17b24642b2a764c0e3a68b4c6937c9ba75
+mediatek	4bfec57314396c8c816ee78f4803987882b668a7
+mvebu		6dfda79476538a9aad1d2dac45453e62c6939522
+omap		7cdd46c9d6c5a7f4800195af82bb718081e7c84d
+qcom		e081f1e68766ca772101bf1f9e14368166fe0ae7
+realtek		3c778f0c9fa36b6b9a26bd2146df24ed4e830ba4
+renesas		7687cd01a980ac59572031b1e3cbe30bf09e90fa
+reset		d373605cd514837d8a6de3d00c786d4bae6dbaf8
+rockchip	23d8f49fcae5d72b77744c7f349c12462c3bb748
+samsung-krzk	08df370772f32b2bce3f88eed253b6a1b0035e40
+scmi		0dcfd0f40b5bee8583eb59ed546189523452c0ae
+sophgo		76acfee87c74dc0dc7a68a00b70b9c39d1c8428f
+sophgo-soc	c8754c7deab4cbfa947fa2d656cbaf83771828ef
+spacemit	4b98722be2911a922fb29b54adccdee46f6a9f41
+stm32		e6cdc9f49eebe486c256e134be84175865a357e9
+sunxi		afb622bddaa6fc7b6d205fd080b9c0d3ec853431
+tee		a4df08db64528b4b605924a0a1f874ffdb99545f
+tegra		11430072551f5d5a37f58e390b3d042035992ca4
+tenstorrent-dt	cee9395acd8043be0644b25c34bfa86623f2b935
+fustini-config	cee9395acd8043be0644b25c34bfa86623f2b935
+thead-dt	cee9395acd8043be0644b25c34bfa86623f2b935
+ti		ba727be7fe4a64ddc6aa2c178204cf6a36d44cc4
+xilinx		bf126d9aa20f2d8bb1c71f5b7f7ba216178fd80d
+socfpga		04a1d21330501b113ba04efb81724f1c95ad2db8
+clk		b1470947e92aeedd522ddb7363369c956e55d6b6
+clk-imx		39ec460b56b26319d1f31b459e8be7ea34ae9c67
+clk-renesas	21cbd7ec29930f8e7e156e18b81736f099f03849
+thead-clk	cee9395acd8043be0644b25c34bfa86623f2b935
+tenstorrent-clk	c64b54ddb692c30b8fb139a2ead32eb4db7faef5
+csky		abb81e5ce7d995baa41556b8125fa59e28ba3be8
+loongarch	a2628ce4ddb6873e35380a42396d17a66e704a1a
+m68k		af32c3cb72529b14bfb2685bbdaa2b1514496b58
+m68knommu	29036c5910df0f0a4c824d7da5b6254c7d4ce500
+microblaze	6b125c73aaa0dab9ce625437c63b176afe453776
+mips		f552e14f77bfce3e7ee0bb268ab023951baec3ff
+openrisc	6620f5e8c11c4f7e41222a86f5c97150cc5f84a5
+parisc-hd	a5f6df25261b31eb75f61047a890515671d95a29
+powerpc		12d238d584ba485a99e02e033746100fc7099184
+risc-v		a06776565b9e73512e880a95b66c198ae8e7e24a
+riscv-dt	f55a54f07a367ed5c73ae53398a60c01219a6df3
+riscv-soc	00bdd4d4daaae140ff747475ef475facee61a950
+s390		c57ae907d4f9ae822bdcd7b16f2d0dc56fb7973b
+sh		be5a19d95030ffa7a37d2981d0ab4eac8a6c89fa
+sparc		cee9395acd8043be0644b25c34bfa86623f2b935
+uml		2f88f5689de1a039764d00f466209acf0c010eaf
+xtensa		28722ed2527aa63ac1454013defe718c56032886
+fs-next		bf234c28d9e24e3d6c42a6202e3f92fba514c2ae
+printk		3fa6f22c1dc835fe1757b06e7d51d8c424ac978a
+pci		81bebadfa0f7f9337a0b4e8e218731457b7eb9af
+pstore		7c756181175d50200be487affa905e973f6cbaf5
+hid		145c2b2e9a5c0f794fb4009bcb072ab19f8ccfcd
+i2c		8cd9520d35a6c38db6567e97dd93b1f11f185dc6
+i2c-andi	e33eb6c27c095649c09465fbf75ada7d492ba83a
+i2c-rust	61ddec70c9bcc3d4b0471c8683e57d3b03cecf0a
+i3c		bf9ec06c90dcd8ccba116fbc065b5d918de47644
+dmi		1afafbaf749d8e8ec53f8e38efdc731131902b5b
+hwmon-staging	4781ca52761e666cf18b591e6bb0478396c90320
+jc_docs		2d72a4c09867a8f9131afc2e705a728f9fba2417
+v4l-dvb		58348f64125e9a3e44d3abb275ca7f4e6c9641e5
+v4l-dvb-next	adc218676eef25575469234709c2d87185ca223a
+pm		114920a3e82922869f8c27916e852b4ba0d560c4
+cpufreq-arm	d82d896f00e7bf697b61d5df0555941afa8d0657
+cpupower	cee9395acd8043be0644b25c34bfa86623f2b935
+devfreq		9a222650d9e70d0027126e4df5b00b1b5b678a97
+pmdomain	d1a93cf1d3bed4667de58a561afb9682430e4f50
+opp		1f6de65e3314519ce462bfd3a1d29e918f97e4e5
+thermal		e856ca3013b3074fd07cb81c90b1de332f69153c
+rdma		485effd117d0310a7f589defb4f442fa3bb52ae2
+net-next	47a1446725732cd3996edf607e8739334bbf4d78
+bpf-next	acff58e305175df35985082b0e79103a2497f702
+ipsec-next	014d795c73837ea2339a4ea8e8f82c6e959b845d
+mlx5-next	36b1d3299d0b6aa51485cc9a79b8d94948d5f3d4
+netfilter-next	87b80c2f6b05cad9f0ff9136709c62a0f59923e3
+ipvs-next	6ebcf5074cff0402730c6981d2397139fee6322d
+bluetooth	fdd5964bd389936856971647fcac3442117009dc
+wireless-next	21b4248bfa0f410fa22202fc2c82f4808d672cda
+ath-next	21b4248bfa0f410fa22202fc2c82f4808d672cda
+iwlwifi-next	b75dd71d260eee7f77b661fc458c5ce767617680
+wpan-next	a6bfdfcc6711d1d5a92e98644359dedc67c0c858
+wpan-staging	a6bfdfcc6711d1d5a92e98644359dedc67c0c858
+mtd		112a666bd82e96e4a01e0cd8a0fb9c88dd0bce38
+nand		55c5b6d5f59f59a8c98a7effc3195226cc20175c
+spi-nor		68d7115d77a873c28d8c7d2f239c3d9ca83bf39f
+crypto		67aefeccc4101a93627f36abbd049f191b54f903
+libcrypto	b63b9b3d5ddaf0f1767f31aeef7aef08bb36225e
+drm		2abe8e7e33973dda5dd24b89aa3ee52470a78265
+drm-exynos	3a8660878839faadb4f1a6dd72c3179c1df56787
+drm-misc	744f262401cc2e1f3827496c72a47e072d31a852
+amdgpu		76b9706e7fa1a6e374703170128b1be2f590cda7
+drm-intel	c9e608d1247cd4c286a54d7f26188c41e8f0a6c2
+drm-msm		d33622598496c8994c35ba0a8a913de064dfc1a4
+drm-msm-lumag	140b13475302601368c0cf4e193e66126a49feb3
+drm-xe		3fc93d311249d5ad969a520dd3aae73bc007d431
+drm-rust	658c5f0042e6e897338dd04330a363aa7022c85d
+drm-nova	93296e9d9528f0d87f2cf3fee494599060a0f14a
+etnaviv		6bde14ba5f7ef59e103ac317df6cc5ac4291ff4a
+fbdev		9f4c6043f33c95db7e46d6ca198344ce0ca2e6c5
+regmap		117e6a5fd98fb7002e70ba39c3ba393498399982
+sound		d4febca239be3c91498762dd0535a87d4af1eabc
+ieee1394	a6e7c3836b81235df08814bd409f7a23efcfb34d
+sound-asoc	0fbfaeecfd6f5925bfcd7cfcbd62860a9e868619
+modules		d6bfb4a05affc8083a45feefe0bbf715968e0d9e
+input		daae2ab46e0cb612f50ca1d86cf50e5962461ae5
+block		0183ac11c13abc35710b3356919f42806bd961e8
+device-mapper	bec2fa7bdf06d6523e40e05fd93117b0c1c8bd30
+libata		cfce1dc635041d6b9dd5fad5367a331faaf644fe
+pcmcia		b3c26ea81ccc522e77ed0b1707add61fc9206216
+mmc		c2063613bceaaffe864ff781082845d322ffb2e1
+mfd		319633b06ff2bfc2a7a60d2cfcab2e681a343b27
+backlight	5e1631df673f3d5355acd06263a884e00d8d007b
+battery		4fc88ba435dadbc05990951e3f3fbd8ccd2df140
+regulator	26f25a65cda68bab7db1152b1f37d856c8a49ae1
+security	881f19c2ffbc73f351b257e09c71e6059939b770
+apparmor	cee9395acd8043be0644b25c34bfa86623f2b935
+integrity	1dc317438308c79cf90a6f14d3e4116347ea3716
+selinux		f2b37cf384b0111b975239f4dc75930368023058
+smack		fedc88e38ce979a720cd2de042578cb5df3dc8de
+tomoyo		72d3fcf802c45d00b300f25b848a93c3a2bd7c7e
+tpmdd-tpm	015fb29a748342a186f37b602ac70017098c8251
+tpmdd-keys	e7ac8fd885298225076abd92b82c1f31a6bc8320
+watchdog	8b5a9f09037e3c372c4e9f2fbda85fcfbf5ea5f0
+iommu		cec2dc7663e7182c311162163d71b1bbdfdbf09b
+audit		88bd5e852addf09db73a38d93f7911863ef5a104
+devicetree	833aaa4790eee9812269b776eb8672acf1524741
+dt-krzk		dfcea1641506d7ba096a9e6ead5d15f5573206ec
+mailbox		14af7a96afa39f4f3c1972705489b9ba15c01857
+spi		168b675c187d40141944d76cedee8a1061beba68
+tip		1aeb52f7869a680c042fc9ae806281e8f60469f7
+kexec		9d0b028715b007188a61ce70b9656ead84a2b3ef
+liveupdate	5221d141653a83a974595b4e5eedd9d6983ab270
+clockevents	1b8b356b4b06e3a28feb852c04e06caeac07bc87
+edac		898e6a2c5ce0e4aba55d126bbe610f0f77e71226
+ftrace		6dc993d4520fdaa083c8871e5fee443117d87057
+rcu		aebf6c4777d534ccd089b3ccab98d71ea3167a0d
+paulmck		c87605b21fdd40aef7d647baff6685fa1aefee1a
+kvm		d4b7fb647204f0c81dfeae2d1a708e4d858e0c94
+kvm-arm		8c00199d322b9e5e936bb0ff624ac95b01c740fe
+kvms390		044ae0767d8cc1fc2a53930361f05c9ed2c3c081
+kvm-ppc		93f51579e7df248780214094418f205253383cc5
+kvm-riscv	41e81f7e3ef96594fb840445343c0ee7723aa550
+kvm-x86		b378201ccd5280d0fff89bbe55e1eb00620ec0d5
+xen-tip		93f51579e7df248780214094418f205253383cc5
+percpu		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+workqueues	fee1265a725725dac0ff7459f912d69558798b36
+sched-ext	91a186b9e599f08f79abfbdbbbcf22443f966a6a
+drivers-x86	fe5030c8cc7156223f48530e9b49aa87c0305bcd
+chrome-platform	5859f97c6f404d07e89fe9c0d20a318ebf200bd1
+chrome-platform-firmware	0e30b98a545991e440a947fb24f9df4ded9798f0
+hsi		e81250ec6b69248b00d38c523dc6a13efaf38aab
+leds-lj		05b4738b0078f7d6f154f68068a11c8a0635e9df
+ipmi		89a312991dc6e638a36adc43ccb91dbc25504c04
+driver-core	f1850e443b0e4f2429ddf42a8d5033ea54ae8a90
+usb		d58dffe9ee2c8883193959ff4ef995ec07932874
+thunderbolt	a93a8e3200002f0c345fec4c340390e9a60aabc7
+usb-serial	6583f9741341b98ade67aa764794a02fea1eb7db
+tty		bf961847813d9edcb5a9c089714069428d141fa9
+char-misc	315860c4f912990c75ef228775ea8871fc5bcc9a
+coresight	9e3604d7369cfc0110100eb1a0acab1865ee2d18
+fastrpc		ef071c4906eb45d16b60f09154cf0bc6ec8f5435
+fpga		093da48782df3d50953c520626b345ca70af8cbd
+icc		3a6d690152557706cc3d44f6402d40c6c1ea7d74
+iio		a3b3580713f3ac5a32dc2874ee546828977a1d68
+nfc		fd73f4a6659897191fa0d40695fe370925dd3780
+phy-next	c7f2322431cb6d108b18fb4154606b49e2bc50f7
+soundwire	90b63b309fd6c0f192337659e73e3047a67df99f
+extcon		8d3ae59288f1e7d58d76558a6ee96d533bc5019f
+gnss		cee9395acd8043be0644b25c34bfa86623f2b935
+vfio		b30b52c2fb82e6d38ee2845a4d85e8254b44f3d8
+w1		813a5b9a3c9f6cee830b1e23343fbd2359683557
+spmi		8cdeaa50eae8dad34885515f62559ee83e7e8dda
+staging		8444548bd905f22093729065408284a6b46f7eee
+counter-next	edac5cf35699492027fb54e348dc6234b2661617
+mux		ac7bde3c53166656d80e3aacc7d274d3c60a6de4
+dmaengine	0a8dda0a15d3926422d286567f945a05328a4ac6
+cgroup		8deb0752fa76101ff2a0c5cdaf837d95ed0d0046
+scsi		6147f16c23efb58a98fdfc71b794b063dc01c767
+scsi-mkp	f09d2c7485b32adb82336d0d748935c8237a649e
+vhost		8f2c2fb94a01320e5136c6e89a47fb12355ad27c
+rpmsg		7fbd9a5319c33204a719294d90c08abb15a5a46c
+gpio-brgl	c4e74a7058b573617f142e07b4b5bc9eee04f4c8
+gpio-intel	0fc424b6a8ef447450f6087db42abce3a6feb328
+pinctrl		c34fbaaac34ba9a85af0f16a0e1d2b98b2d05b25
+pinctrl-intel	c016587866e573fa8dff50c3bdae9734c4418099
+pinctrl-renesas	0cd4a7b3a4883970df3ae8814d9aabdecf65a81e
+pinctrl-samsung	cee9395acd8043be0644b25c34bfa86623f2b935
+pinctrl-qcom	4d7c9430a26aea6a69521af3aa2d78dd5bc8d3ed
+pwm		e74b9a7ee50071aad25d3989bf985292f90b0c6e
+ktest		932cdaf3e273a2727e77af97f79f12577174c5a0
+kselftest	30af56a227e27b892d34bc44c23298b5274913e3
+kunit		cee9395acd8043be0644b25c34bfa86623f2b935
+kunit-next	e38f53f0482468efd04397f66bda4648b70ac9fa
+livepatching	5d791d3396ca4c9e5fd9d20b49386c20d8179eb8
+rtc		cee9395acd8043be0644b25c34bfa86623f2b935
+nvdimm		e99cb3ecd8334ca21e01ff9a79a916693f58f1fb
+at24		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+ntb		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+seccomp		832b9b176be06a20747a267db6bc0010a24b74f4
+slimbus		4350d705466992dce4a21c382985553bfb568f65
+nvmem		41f42ff4ae134eba8dbe3424cf2ca1824c652edd
+hyperv		be0cfab740e58b70047ef6e7e3d578f00ed5d258
+auxdisplay	f63dc0eea92d07c5bb799a406571b555cd2bbd97
+kgdb		fdbdd0ccb30af18d3b29e714ac8d5ab6163279e0
+hmm		cee9395acd8043be0644b25c34bfa86623f2b935
+cfi		dc59e4fea9d83f03bad6bddf3fa2e52491777482
+mhi		710bf7329abf86e37db529f19c8f394099238892
+cxl		71392a644e88c6bb921cdf12cb2b00adec070fe4
+zstd		65d1f5507ed2c78c64fce40e44e5574a9419eb09
+efi		7eef16311a234b5d77c1493e19c6538a3aa1a203
+unicode		a511442085c140da9cdbe60e3f0fab1c71480801
+random		703749b069d53b55f92499e088b30b031e47b8f9
+landlock	02619311dbfc4351e78e5dc6652532cbe7603661
+sysctl		4991c8b72b564cd16cb48124f880617fd49f1013
+execve		df2908090cda368b01ff43709f51890076c56157
+bitmap		452a6d5b5e0556848a8c28428c8f86874ea4ee02
+hte		30167fadbf87fa901a2fee49c30a5c5a506a43e8
+kspp		760f96b7f54beb8dc6f85d97ac7854fddba86835
+nolibc		da27722051fdf0775d8c5b3e33dc21bcf605dbd4
+iommufd		54dadb030c7e2350957855d3995de05ae02c2e66
+turbostat	ccdfcb7e7ab84573046061ece61bfd2368577f1e
+pwrseq		09baa2f4caab013160eed975db47235e6c930af4
+capabilities-next	507adb6448378155ec89495fed8c3c09d8c83920
+ipe		bcaa4d1d69368b6034b1ceb226ffd0a203a5fbc7
+kcsan		a8488ecbd7ba44d65b912dfe88a73f438eba2447
+crc		cee9395acd8043be0644b25c34bfa86623f2b935
+keys-next	965e9a2cf23b066d8bdeb690dff9cd7089c5f667
+fwctl		cee9395acd8043be0644b25c34bfa86623f2b935
+devsec-tsm	3177779ae17db4c66c851f799505fb95c7530c03
+hisilicon	a5db65458a911daa6f8927264366dab501715dc0
+device-id	995832b2cebe6969d1b42635db698803ee31294d
+kthread		fa39ec4f89f2637ed1cdbcde3656825951787668
+pagemap-headers	e02cb91d4644dfff593146f89e70d2008cc6ac16
diff --git a/Next/Trees b/Next/Trees
new file mode 100644
index 00000000000000..32b5b670b60a62
--- /dev/null
+++ b/Next/Trees
@@ -0,0 +1,433 @@
+Trees included into this release:
+
+Name		Url
+----		---
+origin		https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#master
+ext4-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#fixes
+vfs-brauner-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.fixes
+fscrypt-current	https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-current
+fsverity-current	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-current
+btrfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#next-fixes
+vfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#fixes
+erofs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#fixes
+nfsd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-fixes
+v9fs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#fixes/next
+overlayfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#ovl-fixes
+fscrypt		https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-next
+btrfs		https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#for-next
+ceph		https://github.com/ceph/ceph-client.git#master
+cifs		https://git.manguebit.org/linux.git#cifs-next
+configfs	https://git.kernel.org/pub/scm/linux/kernel/git/leitao/linux.git#configfs-next
+configfs-rust	https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git#configfs-next
+ecryptfs	https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git#next
+dlm		https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git#next
+erofs		https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#dev
+exfat		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git#dev
+ext3		https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git#for_next
+ext4		https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#dev
+f2fs		https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git#dev
+fsverity	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-next
+fuse		https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
+gfs2		https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git#for-next
+jfs		https://github.com/kleikamp/linux-shaggy.git#jfs-next
+ksmbd		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/smb.git#ksmbd-for-next
+nfs		git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git#linux-next
+nfs-anna	git://git.linux-nfs.org/projects/anna/linux-nfs.git#linux-next
+nfsd		https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-next
+ntfs		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git#ntfs-next
+ntfs3		https://github.com/Paragon-Software-Group/linux-ntfs3.git#master
+orangefs	https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git#for-next
+overlayfs	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#overlayfs-next
+ubifs		https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git#next
+v9fs		https://github.com/martinetd/linux#9p-next
+v9fs-ericvh	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#ericvh/for-next
+xfs		https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#for-next
+zonefs		https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git#for-next
+vfs-brauner	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.all
+vfs		https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#for-next
+mm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mm/linux.git#for-next-fixes
+kbuild-current	https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-fixes-for-next
+clang-fixes-current	https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git#clang-fixes-for-current
+arc-current	https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-curr
+arm-current	https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#fixes
+arm64-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/fixes
+arm-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#arm/fixes
+davinci-current	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-current
+realtek-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git#fixes
+drivers-memory-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#fixes
+sophgo-fixes	https://github.com/sophgo/linux.git#fixes
+sophgo-soc-fixes	https://github.com/sophgo/linux.git#soc-fixes
+m68k-current	https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-linus
+powerpc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#fixes
+s390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#fixes
+net		https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git#main
+bpf		https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/#master
+ipsec		https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git#master
+netfilter	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git#main
+ipvs		https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git#main
+bluetooth-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth.git#master
+wireless	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git#for-next
+ath		https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-current
+iwlwifi		https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#fixes
+wpan		https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git#master
+rdma-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-rc
+sound-current	https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-linus
+sound-asoc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-linus
+regmap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-linus
+regulator-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-linus
+spi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-linus
+pci-current	https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#for-linus
+driver-core.current	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-linus
+tty.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-linus
+usb.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-linus
+usb-serial-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-linus
+phy		https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#fixes
+staging.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-linus
+iio-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#fixes-togreg
+watchdog-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#watchdog
+counter-current	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-current
+char-misc.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-linus
+soundwire-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#fixes
+thunderbolt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#fixes
+input-current	https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#for-linus
+crypto-current	https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git#master
+libcrypto-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-fixes
+vfio-fixes	https://github.com/awilliam/linux-vfio.git#for-linus
+kselftest-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#fixes
+dmaengine-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#fixes
+backlight-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-fixes
+mtd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/fixes
+mfd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-fixes
+v4l-dvb-fixes	git://linuxtv.org/media-ci/media-pending.git#fixes
+reset-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux#reset/fixes
+mips-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-fixes
+at91-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-fixes
+omap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#fixes
+tegra-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#fixes
+kvm-fixes	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#master
+kvms390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#master
+kvm-arm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#fixes
+hwmon-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon
+nvdimm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-fixes
+cxl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#fixes
+dma-mapping-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-fixes
+drivers-x86-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#fixes
+samsung-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#fixes
+pinctrl-samsung-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#fixes
+pinctrl-qcom-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pinctrl-qcom/for-current
+devicetree-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#dt/linus
+dt-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#fixes
+scsi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#fixes
+drm-fixes	https://gitlab.freedesktop.org/drm/kernel.git#drm-fixes
+drm-intel-fixes	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next-fixes
+mmc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#fixes
+rtc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-fixes
+gnss-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-linus
+hyperv-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-fixes
+risc-v-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#fixes
+riscv-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-fixes
+riscv-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-fixes
+fpga-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#fixes
+spdx		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git#spdx-linus
+gpio-brgl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-current
+gpio-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#fixes
+pinctrl-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#fixes
+auxdisplay-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#fixes
+kunit-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit-fixes
+renesas-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#fixes
+perf-current	https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git#perf-tools
+efi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#urgent
+battery-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#fixes
+iommufd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-rc
+rust-fixes	https://github.com/Rust-for-Linux/linux.git#rust-fixes
+w1-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#fixes
+pmdomain-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#fixes
+i2c-andi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-fixes
+i2c-rust-fixes	https://github.com/ikrtn/rust-for-linux#rust-i2c-fixes
+sparc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-linus
+clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-fixes
+thead-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-fixes
+tenstorrent-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-fixes
+fustini-config-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#riscv-config-fixes
+pwrseq-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-current
+thead-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-fixes
+ftrace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ftrace/fixes
+ring-buffer-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ring-buffer/fixes
+trace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#trace/fixes
+tracefs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#tracefs/fixes
+spacemit-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux#fixes
+tip-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#tip/urgent
+kexec-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#kexec-fixes
+liveupdate-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#fixes
+drm-msm-fixes	https://gitlab.freedesktop.org/drm/msm.git#msm-fixes
+uml-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#fixes
+fwctl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-rc
+devsec-tsm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#fixes
+drm-rust-fixes	https://gitlab.freedesktop.org/drm/rust/kernel.git#for-linux-next-fixes
+tenstorrent-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-fixes
+nfc-fixes	https://codeberg.org/linux-nfc/linux.git#for-linus
+mm-nonmm-hotfixes-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-hotfixes-stable
+mm-nonmm-hotfixes-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-hotfixes-unstable
+drm-misc-fixes	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next-fixes
+rust		https://github.com/Rust-for-Linux/linux.git#rust-next
+rust-interop	https://github.com/Rust-for-Linux/linux.git#interop-next
+rust-alloc	https://github.com/Rust-for-Linux/linux.git#alloc-next
+rust-io		https://github.com/Rust-for-Linux/linux.git#io-next
+rust-pin-init	https://github.com/Rust-for-Linux/linux.git#pin-init-next
+rust-timekeeping	https://github.com/Rust-for-Linux/linux.git#timekeeping-next
+rust-xarray	https://github.com/Rust-for-Linux/linux.git#xarray-next
+rust-analyzer	https://github.com/Rust-for-Linux/linux.git#rust-analyzer-next
+mm		https://git.kernel.org/pub/scm/linux/kernel/git/mm/linux.git#for-next
+mm-nonmm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-stable
+mm-nonmm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-unstable
+kbuild		https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-for-next
+clang-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git#clang-fixes-for-next
+clang-format	https://github.com/ojeda/linux.git#clang-format
+perf		https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git#perf-tools-next
+compiler-attributes	https://github.com/ojeda/linux.git#compiler-attributes
+dma-mapping	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-for-next
+asm-generic	https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic#master
+alpha		https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git#alpha-next
+arm		https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#for-next
+arm64		https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/core
+arm-perf	https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git#for-next/perf
+arm-soc		https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#for-next
+amlogic		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git#for-next
+asahi-soc	https://github.com/AsahiLinux/linux.git#asahi-soc/for-next
+at91		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-next
+bmc		https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git#for-next
+broadcom	https://github.com/Broadcom/stblinux.git#next
+cix		https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git#for-next
+davinci		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-next
+drivers-memory	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#for-next
+fsl		https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git#soc_fsl
+imx-mxs		https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git#for-next
+mediatek	https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git#for-next
+mvebu		https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git#for-next
+omap		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#for-next
+qcom		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git#for-next
+realtek		https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git#for-next
+renesas		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#next
+reset		https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux#reset/next
+rockchip	https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git#for-next
+samsung-krzk	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#for-next
+scmi		https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git#for-linux-next
+sophgo		https://github.com/sophgo/linux.git#for-next
+sophgo-soc	https://github.com/sophgo/linux.git#soc-for-next
+spacemit	https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux#for-next
+stm32		https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git#stm32-next
+sunxi		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git#sunxi/for-next
+tee		https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git#next
+tegra		https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#for-next
+tenstorrent-dt	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-for-next
+fustini-config	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#riscv-config-for-next
+thead-dt	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-for-next
+ti		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git#ti-next
+xilinx		https://github.com/Xilinx/linux-xlnx.git#for-next
+socfpga		https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git#for-next
+clk		https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-next
+clk-imx		https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git#for-next
+clk-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-clk
+thead-clk	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-for-next
+tenstorrent-clk	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-for-next
+csky		https://github.com/c-sky/csky-linux.git#linux-next
+loongarch	https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git#loongarch-next
+m68k		https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-next
+m68knommu	https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git#for-next
+microblaze	git://git.monstr.eu/linux-2.6-microblaze.git#next
+mips		https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-next
+openrisc	https://github.com/openrisc/linux.git#for-next
+parisc-hd	https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git#for-next
+powerpc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#next
+risc-v		https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#for-next
+riscv-dt	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-for-next
+riscv-soc	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-for-next
+s390		https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#for-next
+sh		https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git#for-next
+sparc		https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-next
+uml		https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#next
+xtensa		https://github.com/jcmvbkbc/linux-xtensa.git#xtensa-for-next
+printk		https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git#for-next
+pci		https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#next
+pstore		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/pstore
+hid		https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git#for-next
+i2c		https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git#i2c/for-next
+i2c-andi	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-next
+i2c-rust	https://github.com/ikrtn/rust-for-linux#rust-i2c-next
+i3c		https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git#i3c/next
+dmi		https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git#dmi-for-next
+hwmon-staging	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon-next
+jc_docs		git://git.lwn.net/linux.git#docs-next
+v4l-dvb		git://linuxtv.org/media-ci/media-pending.git#next
+v4l-dvb-next	git://linuxtv.org/mchehab/media-next.git#master
+pm		https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git#linux-next
+cpufreq-arm	https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#cpufreq/arm/linux-next
+cpupower	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git#cpupower
+devfreq		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git#devfreq-next
+pmdomain	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#next
+opp		https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#opp/linux-next
+thermal		https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git#thermal/linux-next
+rdma		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-next
+net-next	https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git#main
+bpf-next	https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git#for-next
+ipsec-next	https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git#master
+mlx5-next	https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git#mlx5-next
+netfilter-next	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git#main
+ipvs-next	https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git#main
+bluetooth	https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git#master
+wireless-next	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git#for-next
+ath-next	https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-next
+iwlwifi-next	https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#next
+wpan-next	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#master
+wpan-staging	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#staging
+mtd		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/next
+nand		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#nand/next
+spi-nor		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#spi-nor/next
+crypto		https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git#master
+libcrypto	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-next
+drm		https://gitlab.freedesktop.org/drm/kernel.git#drm-next
+drm-exynos	https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git#for-linux-next
+drm-misc	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next
+amdgpu		https://gitlab.freedesktop.org/agd5f/linux.git#drm-next
+drm-intel	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next
+drm-msm		https://gitlab.freedesktop.org/drm/msm.git#msm-next
+drm-msm-lumag	https://gitlab.freedesktop.org/lumag/msm.git#msm-next-lumag
+drm-xe		https://gitlab.freedesktop.org/drm/xe/kernel.git#drm-xe-next
+drm-rust	https://gitlab.freedesktop.org/drm/rust/kernel.git#for-linux-next
+drm-nova	https://gitlab.freedesktop.org/drm/nova.git#nova-next
+etnaviv		https://git.pengutronix.de/git/lst/linux#etnaviv/next
+fbdev		https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git#for-next
+regmap		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-next
+sound		https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-next
+ieee1394	https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git#for-next
+sound-asoc	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-next
+modules		https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git#modules-next
+input		https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#next
+block		https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git#for-next
+device-mapper	https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git#for-next
+libata		https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux#for-next
+pcmcia		https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git#pcmcia-next
+mmc		https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#next
+mfd		https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-next
+backlight	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-next
+battery		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#for-next
+regulator	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-next
+security	https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git#next
+apparmor	https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor#apparmor-next
+integrity	https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity#next-integrity
+selinux		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git#next
+smack		https://github.com/cschaufler/smack-next#next
+tomoyo		git://git.code.sf.net/p/tomoyo/tomoyo.git#master
+tpmdd-tpm	https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#for-next-tpm
+tpmdd-keys	https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#for-next-keys
+watchdog	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#watchdog-next
+iommu		https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git#next
+audit		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git#next
+devicetree	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#for-next
+dt-krzk		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#for-next
+mailbox		https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git#for-next
+spi		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-next
+tip		https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#master
+kexec		https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#kexec-next
+liveupdate	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#next
+clockevents	https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git#timers/drivers/next
+edac		https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git#edac-for-next
+ftrace		https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#for-next
+rcu		https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux#next
+paulmck		https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git#non-rcu/next
+kvm		git://git.kernel.org/pub/scm/virt/kvm/kvm.git#next
+kvm-arm		https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#next
+kvms390		https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#next
+kvm-ppc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#topic/ppc-kvm
+kvm-riscv	https://github.com/kvm-riscv/linux.git#riscv_kvm_next
+kvm-x86		https://github.com/kvm-x86/linux.git#next
+xen-tip		https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git#linux-next
+percpu		https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git#for-next
+workqueues	https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git#for-next
+sched-ext	https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git#for-next
+drivers-x86	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#for-next
+chrome-platform	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-next
+chrome-platform-firmware	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-firmware-next
+hsi		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git#for-next
+leds-lj		https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git#for-leds-next
+ipmi		https://github.com/cminyard/linux-ipmi.git#for-next
+driver-core	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-next
+usb		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-next
+thunderbolt	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#next
+usb-serial	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-next
+tty		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-next
+char-misc	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-next
+coresight	https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git#next
+fastrpc		https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git#for-next
+fpga		https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#for-next
+icc		https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git#icc-next
+iio		https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#togreg
+nfc		https://codeberg.org/linux-nfc/linux.git#for-next
+phy-next	https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#next
+soundwire	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#next
+extcon		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git#extcon-next
+gnss		https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-next
+vfio		https://github.com/awilliam/linux-vfio.git#next
+w1		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#for-next
+spmi		https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git#spmi-next
+staging		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-next
+counter-next	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-next
+mux		https://gitlab.com/peda-linux/mux.git#for-next
+dmaengine	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#next
+cgroup		https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git#for-next
+scsi		https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git#for-next
+scsi-mkp	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#for-next
+vhost		https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git#linux-next
+rpmsg		https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git#for-next
+gpio-brgl	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-next
+gpio-intel	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#for-next
+pinctrl		https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git#for-next
+pinctrl-intel	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#for-next
+pinctrl-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-pinctrl
+pinctrl-samsung	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#for-next
+pinctrl-qcom	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pinctrl-qcom/for-next
+pwm		https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git#pwm/for-next
+ktest		https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git#for-next
+kselftest	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#next
+kunit		https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#test
+kunit-next	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit
+livepatching	https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git#for-next
+rtc		https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-next
+nvdimm		https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-for-next
+at24		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#at24/for-next
+ntb		https://github.com/jonmason/ntb.git#ntb-next
+seccomp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/seccomp
+slimbus		https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git#for-next
+nvmem		https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git#for-next
+hyperv		https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-next
+auxdisplay	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#for-next
+kgdb		https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git#kgdb/for-next
+hmm		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#hmm
+cfi		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#cfi/next
+mhi		https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git#mhi-next
+cxl		https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#next
+zstd		https://github.com/terrelln/linux.git#zstd-next
+efi		https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#next
+unicode		https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git#for-next
+random		https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git#master
+landlock	https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git#next
+sysctl		https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git#sysctl-next
+execve		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/execve
+bitmap		https://github.com/norov/linux.git#bitmap-for-next
+hte		https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git#for-next
+kspp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/kspp
+nolibc		https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git#for-next
+iommufd		https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-next
+turbostat	https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git#next
+pwrseq		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-next
+capabilities-next	https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git#caps-next
+ipe		https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git#next
+kcsan		https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git#next
+crc		https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#crc-next
+keys-next	https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git#keys-next
+fwctl		https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-next
+devsec-tsm	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#next
+hisilicon	https://github.com/hisilicon/linux-hisi.git#for-next
+device-id	https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git#device-id-rework
+kthread		https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git#for-next
+pagemap-headers	git://git.infradead.org/users/willy/pagecache.git#headers
diff --git a/Next/merge.log b/Next/merge.log
new file mode 100644
index 00000000000000..8ca151c441a18b
--- /dev/null
+++ b/Next/merge.log
@@ -0,0 +1,17408 @@
+$ date -R
+Wed, 30 Sep 2026 11:42:57 +0100
+$ git checkout master
+Already on 'master'
+$ git reset --hard stable
+Updating files:  41% (4753/11360)
Updating files:  42% (4772/11360)
Updating files:  43% (4885/11360)
Updating files:  44% (4999/11360)
Updating files:  45% (5112/11360)
Updating files:  46% (5226/11360)
Updating files:  47% (5340/11360)
Updating files:  48% (5453/11360)
Updating files:  49% (5567/11360)
Updating files:  50% (5680/11360)
Updating files:  51% (5794/11360)
Updating files:  52% (5908/11360)
Updating files:  53% (6021/11360)
Updating files:  54% (6135/11360)
Updating files:  55% (6248/11360)
Updating files:  56% (6362/11360)
Updating files:  57% (6476/11360)
Updating files:  58% (6589/11360)
Updating files:  59% (6703/11360)
Updating files:  60% (6816/11360)
Updating files:  61% (6930/11360)
Updating files:  62% (7044/11360)
Updating files:  63% (7157/11360)
Updating files:  64% (7271/11360)
Updating files:  65% (7384/11360)
Updating files:  66% (7498/11360)
Updating files:  67% (7612/11360)
Updating files:  68% (7725/11360)
Updating files:  69% (7839/11360)
Updating files:  70% (7952/11360)
Updating files:  71% (8066/11360)
Updating files:  72% (8180/11360)
Updating files:  73% (8293/11360)
Updating files:  73% (8394/11360)
Updating files:  74% (8407/11360)
Updating files:  75% (8520/11360)
Updating files:  76% (8634/11360)
Updating files:  77% (8748/11360)
Updating files:  78% (8861/11360)
Updating files:  79% (8975/11360)
Updating files:  80% (9088/11360)
Updating files:  81% (9202/11360)
Updating files:  82% (9316/11360)
Updating files:  83% (9429/11360)
Updating files:  84% (9543/11360)
Updating files:  85% (9656/11360)
Updating files:  86% (9770/11360)
Updating files:  87% (9884/11360)
Updating files:  88% (9997/11360)
Updating files:  89% (10111/11360)
Updating files:  90% (10224/11360)
Updating files:  91% (10338/11360)
Updating files:  92% (10452/11360)
Updating files:  93% (10565/11360)
Updating files:  94% (10679/11360)
Updating files:  95% (10792/11360)
Updating files:  96% (10906/11360)
Updating files:  97% (11020/11360)
Updating files:  98% (11133/11360)
Updating files:  99% (11247/11360)
Updating files: 100% (11360/11360)
Updating files: 100% (11360/11360), done.
+HEAD is now at 72d3fcf802c45 Linux 7.3-rc5
+Merging origin/master (551c722f40809 Merge tag 'rtc-7.3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git origin/master
+Updating 72d3fcf802c45..551c722f40809
+Fast-forward (no commit created; -m option ignored)
+ MAINTAINERS                                    | 69 ++++++++++++---------
+ arch/arc/Kconfig                               |  4 --
+ arch/arc/include/asm/atomic64-arcv2.h          |  6 ++
+ arch/arc/include/asm/cmpxchg.h                 |  4 +-
+ arch/arc/kernel/setup.c                        |  1 +
+ arch/arc/kernel/smp.c                          |  1 -
+ drivers/mtd/chips/cfi_cmdset_0001.c            | 42 ++++++++-----
+ drivers/mtd/devices/block2mtd.c                |  2 +-
+ drivers/mtd/devices/mtd_intel_dg.c             |  3 +-
+ drivers/mtd/mtdcore.c                          |  3 +-
+ drivers/mtd/nand/ecc.c                         |  7 +++
+ drivers/mtd/nand/raw/cadence-nand-controller.c |  6 +-
+ drivers/mtd/nand/raw/vf610_nfc.c               | 85 +++++++++++++++++++++-----
+ drivers/mtd/nand/spi/core.c                    | 53 +++++++++++-----
+ drivers/mtd/spi-nor/core.c                     |  2 +-
+ drivers/rtc/dev.c                              |  2 +-
+ drivers/rtc/rtc-ac100.c                        | 39 +++---------
+ drivers/rtc/rtc-efi.c                          | 80 +++++++++++++++++++++++-
+ drivers/rtc/rtc-mpfs.c                         |  6 +-
+ drivers/rtc/rtc-spear.c                        | 11 ++--
+ include/linux/mtd/nand.h                       |  2 +
+ include/uapi/linux/alloc_tag.h                 |  1 +
+ kernel/module/main.c                           |  8 ++-
+ mm/damon/core.c                                |  2 +-
+ mm/kasan/common.c                              | 13 +++-
+ mm/shmem.c                                     |  7 ++-
+ mm/vma.c                                       |  4 +-
+ 27 files changed, 327 insertions(+), 136 deletions(-)
+Merging ext4-fixes/fixes (981fcc5674e67 jbd2: fix deadlock in jbd2_journal_cancel_revoke())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4-fixes/fixes
+Already up to date.
+Merging vfs-brauner-fixes/vfs.fixes (b78b728e21c32 netfs: Fix missing alloc tagging of direct mempool allocations)
+$ git merge -m Merge branch 'vfs.fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner-fixes/vfs.fixes
+Already up to date.
+Merging fscrypt-current/for-current (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt-current/for-current
+Already up to date.
+Merging fsverity-current/for-current (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity-current/for-current
+Already up to date.
+Merging btrfs-fixes/next-fixes (50a31f73ef35e Merge branch 'misc-7.3' into next-fixes)
+$ git merge -m Merge branch 'next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs-fixes/next-fixes
+Merge made by the 'ort' strategy.
+Merging vfs-fixes/fixes (49c5d168a3a8f udf: fix nls leak on udf_fill_super() failure)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs-fixes/fixes
+Auto-merging fs/udf/super.c
+Merge made by the 'ort' strategy.
+Merging erofs-fixes/fixes (135d84c66f854 erofs: add missing buf->off in erofs_bread())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs-fixes/fixes
+Already up to date.
+Merging nfsd-fixes/nfsd-fixes (f76017a7663c4 nfsd: fix handling of NFSEXP_PNFS in the netlink codepath)
+$ git merge -m Merge branch 'nfsd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd-fixes/nfsd-fixes
+Already up to date.
+Merging v9fs-fixes/fixes/next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'fixes/next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-fixes/fixes/next
+Already up to date.
+Merging overlayfs-fixes/ovl-fixes (4549871118cf6 Linux 7.1-rc7)
+$ git merge -m Merge branch 'ovl-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs-fixes/ovl-fixes
+Already up to date.
+Merging fscrypt/for-next (a63883d2c3d47 MAINTAINERS: List the fscrypt branches)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                 |  3 ++-
+ fs/crypto/Kconfig           |  2 +-
+ fs/crypto/crypto.c          |  5 +++++
+ fs/crypto/fscrypt_private.h | 27 ---------------------------
+ fs/crypto/hooks.c           |  9 +++++++++
+ fs/crypto/keyring.c         | 13 +++++++++++++
+ fs/crypto/keysetup_v1.c     |  5 ++---
+ 7 files changed, 32 insertions(+), 32 deletions(-)
+Merging btrfs/for-next (55d05b596923d Merge branch 'for-next-next-v7.3-20260917' into for-next-20260917)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs/for-next
+Merge made by the 'ort' strategy.
+ fs/btrfs/Kconfig                 |    1 +
+ fs/btrfs/bio.c                   |  163 ++---
+ fs/btrfs/bio.h                   |    9 +-
+ fs/btrfs/block-group.c           |  498 ++------------
+ fs/btrfs/block-group.h           |   16 +-
+ fs/btrfs/btrfs_inode.h           |   22 +-
+ fs/btrfs/compression.c           |   61 +-
+ fs/btrfs/ctree.c                 |    4 +-
+ fs/btrfs/delalloc-space.c        |   13 +-
+ fs/btrfs/delayed-inode.c         |  115 +++-
+ fs/btrfs/delayed-inode.h         |   22 +-
+ fs/btrfs/dev-replace.c           |   10 +-
+ fs/btrfs/dir-item.c              |   41 +-
+ fs/btrfs/dir-item.h              |    5 +-
+ fs/btrfs/disk-io.c               |  126 +---
+ fs/btrfs/extent-io-tree.c        |    2 +-
+ fs/btrfs/extent-tree.c           |   10 +
+ fs/btrfs/extent_io.c             |  247 ++++---
+ fs/btrfs/extent_map.c            |    8 +
+ fs/btrfs/file-item.c             |   46 +-
+ fs/btrfs/free-space-cache.c      | 1344 +-------------------------------------
+ fs/btrfs/free-space-cache.h      |   29 +-
+ fs/btrfs/fs.c                    |    2 +-
+ fs/btrfs/fs.h                    |    4 +-
+ fs/btrfs/inode.c                 |  342 ++++------
+ fs/btrfs/ioctl.c                 |   41 +-
+ fs/btrfs/ordered-data.c          |   27 +-
+ fs/btrfs/qgroup.c                |  126 ++--
+ fs/btrfs/qgroup.h                |   16 +-
+ fs/btrfs/raid56.c                |   57 +-
+ fs/btrfs/relocation.c            |    2 +-
+ fs/btrfs/send.c                  |    2 +-
+ fs/btrfs/space-info.c            |    2 -
+ fs/btrfs/space-info.h            |    4 -
+ fs/btrfs/super.c                 |   48 +-
+ fs/btrfs/sysfs.c                 |  129 +---
+ fs/btrfs/tests/extent-io-tests.c |  129 ++--
+ fs/btrfs/transaction.c           |   64 +-
+ fs/btrfs/transaction.h           |   31 +-
+ fs/btrfs/tree-checker.c          |   83 ++-
+ fs/btrfs/tree-log.c              |   11 +-
+ fs/btrfs/verity.c                |   31 +-
+ fs/btrfs/volumes.c               |   55 +-
+ fs/btrfs/volumes.h               |    1 +
+ fs/btrfs/zlib.c                  |   10 +-
+ fs/btrfs/zoned.c                 |   21 +-
+ fs/btrfs/zstd.c                  |   73 ++-
+ include/uapi/linux/btrfs_tree.h  |   25 +-
+ 48 files changed, 1091 insertions(+), 3037 deletions(-)
+Merging ceph/master (dc173b37415e8 ceph: apply nearfull_sync option on remount)
+$ git merge -m Merge branch 'master' of https://github.com/ceph/ceph-client.git ceph/master
+Already up to date.
+Merging cifs/cifs-next (f14572c203d57 Merge tag 'cifs-fixes-7.3-rc5' of https://git.manguebit.org/linux)
+$ git merge -m Merge branch 'cifs-next' of https://git.manguebit.org/linux.git cifs/cifs-next
+Already up to date.
+Merging configfs/configfs-next (620938e7da894 samples: configfs: constify the configfs_attribute structures)
+$ git merge -m Merge branch 'configfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/leitao/linux.git configfs/configfs-next
+Auto-merging MAINTAINERS
+Auto-merging tools/testing/selftests/Makefile
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |   1 +
+ drivers/acpi/acpi_configfs.c                       |   2 +-
+ drivers/gpu/drm/xe/xe_configfs.c                   |   4 +-
+ drivers/virt/coco/guest/report.c                   |   6 +-
+ fs/configfs/configfs_internal.h                    |  10 +-
+ fs/configfs/dir.c                                  |  10 +-
+ fs/configfs/file.c                                 |   6 +-
+ fs/configfs/inode.c                                |   2 +-
+ include/linux/configfs.h                           |  73 ++--
+ rust/kernel/configfs.rs                            |  20 +-
+ samples/configfs/configfs_sample.c                 | 137 +++++-
+ tools/testing/selftests/Makefile                   |   1 +
+ .../selftests/filesystems/configfs/.gitignore      |   2 +
+ .../selftests/filesystems/configfs/Makefile        |   8 +
+ .../testing/selftests/filesystems/configfs/config  |   5 +
+ .../selftests/filesystems/configfs/configfs_test.c | 481 +++++++++++++++++++++
+ 16 files changed, 697 insertions(+), 71 deletions(-)
+ create mode 100644 tools/testing/selftests/filesystems/configfs/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/configfs/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/configfs/config
+ create mode 100644 tools/testing/selftests/filesystems/configfs/configfs_test.c
+Merging configfs-rust/configfs-next (6dc9fb1516aa4 rust: configfs: fix data offset calculation for subsystem callbacks)
+$ git merge -m Merge branch 'configfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git configfs-rust/configfs-next
+Auto-merging rust/kernel/configfs.rs
+Merge made by the 'ort' strategy.
+ rust/kernel/configfs.rs | 107 ++++++++++++++++++++++++++++++++----------------
+ 1 file changed, 71 insertions(+), 36 deletions(-)
+Merging ecryptfs/next (f81cb44f9a4b8 ecryptfs: ecryptfs_kernel.h: clean up kernel-doc comments)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git ecryptfs/next
+Already up to date.
+Merging dlm/next (ed9b6a1296f10 dlm: wait for outstanding SRCU callbacks to complete in exit paths)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git dlm/next
+Merge made by the 'ort' strategy.
+ fs/dlm/config.c       | 30 ++++++++++++++++++++++++++++--
+ fs/dlm/dlm_internal.h |  5 +++++
+ fs/dlm/lock.c         | 18 +++++++++++++-----
+ fs/dlm/lock.h         |  4 ++--
+ fs/dlm/lowcomms.c     |  1 +
+ fs/dlm/midcomms.c     |  1 +
+ fs/dlm/plock.c        | 14 +++++++++++++-
+ fs/dlm/user.c         | 23 +++++++++++++++++++++++
+ 8 files changed, 86 insertions(+), 10 deletions(-)
+Merging erofs/dev (a7d28aa0e9b2c erofs: simplify z_erofs_gbuf_growsize())
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs/dev
+Already up to date.
+Merging exfat/dev (216426aff8f69 MAINTAINERS: add exFAT documentation)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git exfat/dev
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/exfat.rst | 117 ++++++++++++++++++++++++++++++++++++
+ Documentation/filesystems/index.rst |   1 +
+ MAINTAINERS                         |   1 +
+ fs/exfat/dir.c                      |  52 ++++++++++++++--
+ fs/exfat/exfat_fs.h                 |   1 +
+ fs/exfat/fatent.c                   |  19 +++---
+ fs/exfat/iomap.c                    |  28 ++++++++-
+ fs/exfat/namei.c                    |   7 ++-
+ fs/exfat/super.c                    |  30 +++++++--
+ 9 files changed, 234 insertions(+), 22 deletions(-)
+ create mode 100644 Documentation/filesystems/exfat.rst
+Merging ext3/for_next (ba5855e74bcd7 Pull quota shrinker fix.)
+$ git merge -m Merge branch 'for_next' of https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git ext3/for_next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                        |  1 +
+ fs/ext2/Makefile                   |  2 +
+ fs/ext2/balloc.c                   |  4 ++
+ fs/ext2/ext2.h                     | 19 ++++++----
+ fs/ext2/file.c                     |  7 ++++
+ fs/ext2/inode.c                    |  7 ++++
+ fs/ext2/super.c                    | 33 +++++++++-------
+ fs/isofs/compress.c                | 77 +++++++++++++++++++++-----------------
+ fs/isofs/inode.c                   | 46 ++++++++---------------
+ fs/isofs/isofs.h                   |  2 +
+ fs/isofs/rock.c                    | 11 +++++-
+ fs/notify/fanotify/fanotify_user.c |  2 +-
+ fs/ocfs2/quota_local.c             |  4 ++
+ fs/quota/dquot.c                   | 16 ++++++++
+ fs/udf/inode.c                     | 59 ++++++++++++++++++++++++++---
+ fs/udf/misc.c                      | 15 ++++----
+ fs/udf/super.c                     |  5 ++-
+ include/linux/once_lite.h          |  8 ++--
+ mm/shmem_quota.c                   |  5 +++
+ 19 files changed, 217 insertions(+), 106 deletions(-)
+Merging ext4/dev (9091c97be3408 ext4: fix estimate extent index blocks in ext4_ext_index_trans_blocks())
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4/dev
+Already up to date.
+Merging f2fs/dev (a4b9fb9e69116 f2fs: rename nr_pages_to_skip with nr_caches_to_skip)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git f2fs/dev
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-fs-f2fs |    7 +
+ fs/f2fs/Makefile                        |    2 +-
+ fs/f2fs/acl.c                           |   26 +-
+ fs/f2fs/acl.h                           |    8 +-
+ fs/f2fs/cache.c                         |  720 +++++++++++++++++
+ fs/f2fs/cache.h                         |  240 ++++++
+ fs/f2fs/checkpoint.c                    |  558 +++++++-------
+ fs/f2fs/compress.c                      |  183 ++---
+ fs/f2fs/data.c                          |  648 +++++++++++-----
+ fs/f2fs/debug.c                         |   94 ++-
+ fs/f2fs/dir.c                           |  221 +++---
+ fs/f2fs/extent_cache.c                  |   25 +-
+ fs/f2fs/f2fs.h                          |  508 +++++++-----
+ fs/f2fs/file.c                          |  181 ++---
+ fs/f2fs/gc.c                            |  222 +++---
+ fs/f2fs/inline.c                        |  293 +++----
+ fs/f2fs/inode.c                         |  242 +++---
+ fs/f2fs/iostat.h                        |   11 +
+ fs/f2fs/namei.c                         |  114 +--
+ fs/f2fs/node.c                          | 1277 +++++++++++++++----------------
+ fs/f2fs/node.h                          |  191 +++--
+ fs/f2fs/recovery.c                      |  255 +++---
+ fs/f2fs/segment.c                       |  429 ++++++-----
+ fs/f2fs/segment.h                       |   87 ++-
+ fs/f2fs/shrinker.c                      |   16 +-
+ fs/f2fs/super.c                         |  280 +++----
+ fs/f2fs/sysfs.c                         |   32 +-
+ fs/f2fs/verity.c                        |    6 +-
+ fs/f2fs/xattr.c                         |  133 ++--
+ fs/f2fs/xattr.h                         |   23 +-
+ include/linux/f2fs_fs.h                 |  140 ++--
+ include/trace/events/f2fs.h             |   71 ++
+ 32 files changed, 4345 insertions(+), 2898 deletions(-)
+ create mode 100644 fs/f2fs/cache.c
+ create mode 100644 fs/f2fs/cache.h
+Merging fsverity/for-next (b08e4ee274917 MAINTAINERS: List the fsverity branches)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging fuse/for-next (29b60c6c561f7 fuse: make fuse_backing_get() static)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git fuse/for-next
+Auto-merging fs/fuse/file.c
+Merge made by the 'ort' strategy.
+ fs/fuse/Kconfig                                    |   8 +-
+ fs/fuse/Makefile                                   |   2 +-
+ fs/fuse/backing.c                                  |   2 +-
+ fs/fuse/dax.c                                      | 215 +++++++++++----------
+ fs/fuse/dev.c                                      |  50 +++--
+ fs/fuse/dev.h                                      |   2 +-
+ fs/fuse/dev_uring.c                                |   6 +
+ fs/fuse/dir.c                                      |   4 +-
+ fs/fuse/file.c                                     | 129 ++++++++-----
+ fs/fuse/fuse_i.h                                   |  96 +++++----
+ fs/fuse/inode.c                                    |  82 +++++---
+ fs/fuse/ioctl.c                                    |   3 -
+ fs/fuse/iomode.c                                   |   4 +-
+ fs/fuse/notify.c                                   |   4 +-
+ fs/fuse/passthrough.c                              |  15 +-
+ fs/fuse/req.c                                      |   7 +-
+ fs/fuse/virtio_fs.c                                |  28 +--
+ include/uapi/linux/fuse.h                          |  12 +-
+ .../testing/selftests/filesystems/fuse/.gitignore  |   1 +
+ tools/testing/selftests/filesystems/fuse/Makefile  |   2 +-
+ 20 files changed, 380 insertions(+), 292 deletions(-)
+Merging gfs2/for-next (d0c4bce31a579 gfs2: gfs2_create_inode excl cleanup)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git gfs2/for-next
+Merge made by the 'ort' strategy.
+ fs/gfs2/acl.c        |  11 +++--
+ fs/gfs2/aops.c       |  17 +++----
+ fs/gfs2/bmap.c       |  93 ++++++++++++++++++++----------------
+ fs/gfs2/dentry.c     |   6 +--
+ fs/gfs2/dir.c        |  61 +++++++++++++++---------
+ fs/gfs2/export.c     |   7 +--
+ fs/gfs2/file.c       |  72 ++++++++++++++++------------
+ fs/gfs2/glock.c      |  60 +++++++++++++++--------
+ fs/gfs2/glops.c      |  13 +++--
+ fs/gfs2/incore.h     |   8 ++--
+ fs/gfs2/inode.c      | 132 ++++++++++++++++++++++++++++-----------------------
+ fs/gfs2/lock_dlm.c   |   4 +-
+ fs/gfs2/log.c        |   3 +-
+ fs/gfs2/lops.c       |  18 ++++---
+ fs/gfs2/meta_io.c    |   7 +--
+ fs/gfs2/ops_fstype.c |  26 +++++-----
+ fs/gfs2/quota.c      |  57 ++++++++++++----------
+ fs/gfs2/recovery.c   |  19 ++++----
+ fs/gfs2/rgrp.c       | 129 +++++++++++++++++++++++++------------------------
+ fs/gfs2/super.c      |  98 +++++++++++++++++++++++---------------
+ fs/gfs2/trace_gfs2.h |   6 +--
+ fs/gfs2/util.c       |   8 ++--
+ fs/gfs2/xattr.c      |  69 ++++++++++++++++-----------
+ 23 files changed, 528 insertions(+), 396 deletions(-)
+Merging jfs/jfs-next (dad98c5b2a05e jfs: avoid -Wtautological-constant-out-of-range-compare warning again)
+$ git merge -m Merge branch 'jfs-next' of https://github.com/kleikamp/linux-shaggy.git jfs/jfs-next
+Already up to date.
+Merging ksmbd/ksmbd-for-next (d30f0c30c87d6 ksmbd: fix OOB read and cross-share confusion in ksmbd_validate_name_reconnect())
+$ git merge -m Merge branch 'ksmbd-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/smb.git ksmbd/ksmbd-for-next
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/smb/ksmbd.rst |  14 +-
+ fs/smb/common/compress/compress.c       |  16 +-
+ fs/smb/common/fscc.h                    |   5 +-
+ fs/smb/common/smbglob.h                 |   1 +
+ fs/smb/server/Kconfig                   |   2 +
+ fs/smb/server/Makefile                  |   1 +
+ fs/smb/server/compress.c                |   1 +
+ fs/smb/server/connection.c              |   3 +-
+ fs/smb/server/connection.h              |   3 +
+ fs/smb/server/ksmbd_work.c              |   3 -
+ fs/smb/server/ksmbd_work.h              |   5 +-
+ fs/smb/server/mgmt/user_session.c       |   4 +-
+ fs/smb/server/smb2ops.c                 |  33 ++
+ fs/smb/server/smb2pdu.c                 | 690 ++++++++++++++------------------
+ fs/smb/server/smb2pdu.h                 |  22 +-
+ fs/smb/server/smb_common.c              |   4 +-
+ fs/smb/server/smbacl.c                  |   7 +-
+ fs/smb/server/tests/Kconfig             |  15 +
+ fs/smb/server/tests/Makefile            |   4 +
+ fs/smb/server/tests/smbacl_kunit.c      | 301 ++++++++++++++
+ fs/smb/server/transport_tcp.c           |   8 +-
+ fs/smb/server/vfs.c                     |  12 +-
+ fs/smb/server/vfs.h                     |   5 +
+ fs/smb/server/vfs_cache.c               |  66 +--
+ fs/smb/server/vfs_cache.h               |   6 -
+ 25 files changed, 738 insertions(+), 493 deletions(-)
+ create mode 100644 fs/smb/server/tests/Kconfig
+ create mode 100644 fs/smb/server/tests/Makefile
+ create mode 100644 fs/smb/server/tests/smbacl_kunit.c
+$ git am -3 ../patches/0001-ntfs3-Fix-up-merge-with-Linus.patch
+Applying: ntfs3: Fix up merge with Linus
+Using index info to reconstruct a base tree...
+M	fs/ntfs3/file.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/ntfs3/file.c
+No changes -- Patch already applied.
+Merging nfs/linux-next (fd73f4a665989 Linux 7.3-rc3)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git nfs/linux-next
+Already up to date.
+Merging nfs-anna/linux-next (9bafc322b7fca nfs: split up block layout and SCSI layout support)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/anna/linux-nfs.git nfs-anna/linux-next
+Merge made by the 'ort' strategy.
+ fs/nfs/Kconfig                            |  17 +-
+ fs/nfs/blocklayout/Makefile               |   3 +-
+ fs/nfs/blocklayout/blocklayout.c          | 119 ++++++---
+ fs/nfs/blocklayout/dev.c                  |  32 ++-
+ fs/nfs/callback_proc.c                    |  23 +-
+ fs/nfs/callback_xdr.c                     |   9 +-
+ fs/nfs/client.c                           |   6 +-
+ fs/nfs/direct.c                           | 139 +++++-----
+ fs/nfs/filelayout/filelayout.c            |  17 +-
+ fs/nfs/filelayout/filelayoutdev.c         |  19 +-
+ fs/nfs/flexfilelayout/flexfilelayout.c    | 413 +++++++++++++++++++-----------
+ fs/nfs/flexfilelayout/flexfilelayout.h    |  48 ++--
+ fs/nfs/flexfilelayout/flexfilelayoutdev.c | 212 +++++++++------
+ fs/nfs/internal.h                         |   3 +-
+ fs/nfs/netns.h                            |   5 +-
+ fs/nfs/nfs3client.c                       |   9 +-
+ fs/nfs/nfs4_fs.h                          |   3 +
+ fs/nfs/nfs4client.c                       |   8 +-
+ fs/nfs/nfs4proc.c                         | 136 ++++++++++
+ fs/nfs/nfs4state.c                        |   7 +-
+ fs/nfs/nfs4xdr.c                          |   2 +-
+ fs/nfs/pagelist.c                         |  66 ++++-
+ fs/nfs/pnfs.c                             | 376 +++++++++++++++++++++++++--
+ fs/nfs/pnfs.h                             | 102 +++++++-
+ fs/nfs/pnfs_dev.c                         |  30 ++-
+ fs/nfs/pnfs_nfs.c                         | 108 ++++++--
+ fs/nfs/read.c                             |   2 +-
+ fs/nfs/write.c                            |   2 +-
+ include/linux/nfs_fs_sb.h                 |   8 +
+ include/linux/nfs_page.h                  |   8 +-
+ include/linux/nfs_xdr.h                   |   2 +
+ net/sunrpc/xprtsock.c                     |  69 ++---
+ 32 files changed, 1513 insertions(+), 490 deletions(-)
+Merging nfsd/nfsd-next (ac04dab23b5ff NFSD: Return NFSERR_ISDIR for NFSv2 READ and WRITE on a non-regular file)
+$ git merge -m Merge branch 'nfsd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd/nfsd-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/nfs/pnfs-scsi-server.rst |   2 +-
+ MAINTAINERS                                        |   3 +-
+ fs/lockd/svc.c                                     |   1 -
+ fs/lockd/svclock.c                                 |  33 +-
+ fs/lockd/trace.h                                   |   1 -
+ fs/lockd/xdr.h                                     |   2 +-
+ fs/namei.c                                         |   2 +
+ fs/nfs/nfs4file.c                                  |   1 +
+ fs/nfs/super.c                                     |  25 -
+ fs/nfs_common/nfs_ssc.c                            | 126 +++--
+ fs/nfsd/blocklayout.c                              |   1 +
+ fs/nfsd/blocklayoutxdr.c                           |  11 +
+ fs/nfsd/export.c                                   |   8 +-
+ fs/nfsd/export.h                                   |   3 +-
+ fs/nfsd/filecache.c                                |   1 +
+ fs/nfsd/flexfilelayout.c                           |  24 +-
+ fs/nfsd/flexfilelayoutxdr.c                        |   9 +-
+ fs/nfsd/flexfilelayoutxdr.h                        |   8 +-
+ fs/nfsd/localio.c                                  |  10 +-
+ fs/nfsd/lockd.c                                    |   5 +-
+ fs/nfsd/netns.h                                    |  17 +-
+ fs/nfsd/nfs2acl.c                                  |  48 +-
+ fs/nfsd/nfs3acl.c                                  |   1 +
+ fs/nfsd/nfs3proc.c                                 |  79 ++-
+ fs/nfsd/nfs3xdr.c                                  |   4 +
+ fs/nfsd/nfs4acl.c                                  |   1 +
+ fs/nfsd/nfs4callback.c                             |   8 +-
+ fs/nfsd/nfs4ctl.h                                  |  83 +++
+ fs/nfsd/nfs4idmap.c                                |   1 +
+ fs/nfsd/nfs4layouts.c                              |   1 +
+ fs/nfsd/nfs4proc.c                                 | 465 +++++++++-------
+ fs/nfsd/nfs4recover.c                              |   1 +
+ fs/nfsd/nfs4state.c                                | 587 ++++++++++++++++++---
+ fs/nfsd/nfs4xdr.c                                  | 188 +++++--
+ fs/nfsd/nfscache.c                                 |   1 +
+ fs/nfsd/nfsctl.c                                   |  36 +-
+ fs/nfsd/nfsd.h                                     | 237 +--------
+ fs/nfsd/nfserr.h                                   | 159 ++++++
+ fs/nfsd/nfsfh.c                                    |  73 +--
+ fs/nfsd/nfsfh.h                                    |  28 +-
+ fs/nfsd/nfsproc.c                                  |  39 +-
+ fs/nfsd/nfssvc.c                                   |   8 +
+ fs/nfsd/nfsxdr.c                                   |   1 +
+ fs/nfsd/state.h                                    |  46 +-
+ fs/nfsd/trace.h                                    |  45 +-
+ fs/nfsd/vfs.c                                      | 298 +++++------
+ fs/nfsd/vfs.h                                      |  39 +-
+ fs/nfsd/xdr.h                                      |   6 +-
+ fs/nfsd/xdr3.h                                     |   2 +-
+ fs/nfsd/xdr4.h                                     | 156 +-----
+ fs/nfsd/xdr4cb.h                                   |  20 +-
+ include/linux/nfs.h                                |  55 +-
+ include/linux/nfs3.h                               |  43 ++
+ include/linux/nfs4.h                               |   6 +
+ include/linux/nfs_fh.h                             |  63 +++
+ include/linux/nfs_ssc.h                            |  69 +--
+ include/linux/nfsd_ssc.h                           |  38 ++
+ include/linux/nfslocalio.h                         |  11 +-
+ include/linux/sunrpc/svc_xprt.h                    |   5 +-
+ include/trace/misc/nfs.h                           |  13 +-
+ net/sunrpc/svc_xprt.c                              |  36 +-
+ net/sunrpc/svcauth_unix.c                          |   9 +
+ net/sunrpc/svcsock.c                               | 490 ++++++++++-------
+ net/sunrpc/xprtrdma/svc_rdma_recvfrom.c            |  18 +-
+ net/sunrpc/xprtrdma/svc_rdma_transport.c           |   3 +-
+ 65 files changed, 2431 insertions(+), 1382 deletions(-)
+ create mode 100644 fs/nfsd/nfs4ctl.h
+ create mode 100644 fs/nfsd/nfserr.h
+ create mode 100644 include/linux/nfs_fh.h
+ create mode 100644 include/linux/nfsd_ssc.h
+$ git am -3 ../patches/0001-Revert-smb-client-implement-fileattr_get-to-support-.patch
+Applying: Revert "smb: client: implement fileattr_get to support FS_IOC_GETFLAGS"
+Using index info to reconstruct a base tree...
+M	fs/smb/client/cifsfs.c
+M	fs/smb/client/cifsfs.h
+M	fs/smb/client/inode.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/smb/client/inode.c
+Auto-merging fs/smb/client/cifsfs.h
+Auto-merging fs/smb/client/cifsfs.c
+No changes -- Patch already applied.
+Merging ntfs/ntfs-next (708f9d56cacae ntfs: reject non-resident attributes whose sizes exceed their allocation)
+$ git merge -m Merge branch 'ntfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git ntfs/ntfs-next
+Merge made by the 'ort' strategy.
+ fs/ntfs/attrib.c   |  76 ++++-
+ fs/ntfs/attrlist.c |  14 +-
+ fs/ntfs/bitmap.c   |  19 +-
+ fs/ntfs/collate.c  |   8 +-
+ fs/ntfs/dir.c      |  23 +-
+ fs/ntfs/file.c     |  56 ++--
+ fs/ntfs/index.c    |   2 +-
+ fs/ntfs/index.h    |   2 +-
+ fs/ntfs/inode.c    | 131 ++++++--
+ fs/ntfs/iomap.c    |  16 +-
+ fs/ntfs/layout.h   |  19 +-
+ fs/ntfs/mft.c      | 921 +++++++++++++++++++++++++++++++++++------------------
+ fs/ntfs/mft.h      |  19 +-
+ fs/ntfs/mst.c      |   4 +-
+ fs/ntfs/namei.c    |  31 +-
+ fs/ntfs/ntfs.h     |   7 +-
+ fs/ntfs/reparse.c  |   2 +-
+ fs/ntfs/runlist.c  |   5 +-
+ fs/ntfs/super.c    | 391 +++++++++++++++++++----
+ fs/ntfs/unistr.c   |  29 +-
+ fs/ntfs/volume.h   |  15 +
+ 21 files changed, 1266 insertions(+), 524 deletions(-)
+Merging ntfs3/master (f3b8ee6c05bed fs/ntfs3: return -ERANGE for short xattr buffers)
+$ git merge -m Merge branch 'master' of https://github.com/Paragon-Software-Group/linux-ntfs3.git ntfs3/master
+Auto-merging fs/ntfs3/inode.c
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/ntfs3.rst | 23 ++++++++++++++++++
+ fs/ntfs3/attrib.c                   | 47 ++++++++++++++++++++++++++++++-------
+ fs/ntfs3/dir.c                      |  3 +++
+ fs/ntfs3/file.c                     |  3 +++
+ fs/ntfs3/frecord.c                  |  5 ++++
+ fs/ntfs3/fslog.c                    |  4 +++-
+ fs/ntfs3/fsntfs.c                   |  3 +++
+ fs/ntfs3/index.c                    | 13 ++++++++--
+ fs/ntfs3/inode.c                    | 19 ++++++++++-----
+ fs/ntfs3/record.c                   | 16 ++++++++++++-
+ fs/ntfs3/super.c                    |  2 +-
+ fs/ntfs3/xattr.c                    | 14 +++++++----
+ 12 files changed, 127 insertions(+), 25 deletions(-)
+Merging orangefs/for-next (2bc09cb0e9e44 orangefs: don't continue on to gpf if client dies on write.)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git orangefs/for-next
+Merge made by the 'ort' strategy.
+ fs/orangefs/inode.c            | 21 ++++++++++++++++++++-
+ fs/orangefs/orangefs-debugfs.c | 10 ++++++----
+ fs/orangefs/super.c            | 18 ++++++++++++++++++
+ 3 files changed, 44 insertions(+), 5 deletions(-)
+Merging overlayfs/overlayfs-next (1f6ee9be92f8d ovl: make fsync after metadata copy-up opt-in mount option)
+$ git merge -m Merge branch 'overlayfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs/overlayfs-next
+Already up to date.
+Merging ubifs/next (a5e0055eac837 UBI: support per-device wear-leveling threshold)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git ubifs/next
+Already up to date.
+Merging v9fs/9p-next (c60ae98c5aa64 9p: Fix v9fs_issue_write() to update i_size and remote_i_size)
+$ git merge -m Merge branch '9p-next' of https://github.com/martinetd/linux v9fs/9p-next
+Already up to date.
+Merging v9fs-ericvh/ericvh/for-next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'ericvh/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-ericvh/ericvh/for-next
+Already up to date.
+Merging xfs/for-next (b4787e7b9730d Merge remote-tracking branch 'xfs-linux/xfs-7.3-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git xfs/for-next
+Merge made by the 'ort' strategy.
+ .../filesystems/xfs/xfs-online-fsck-design.rst     |  13 +-
+ fs/xfs/libxfs/xfs_attr.c                           |  14 +-
+ fs/xfs/libxfs/xfs_bmap.c                           |   5 +-
+ fs/xfs/libxfs/xfs_bmap.h                           |   2 +-
+ fs/xfs/libxfs/xfs_btree.c                          |  10 +-
+ fs/xfs/libxfs/xfs_btree_mem.c                      |   6 +-
+ fs/xfs/libxfs/xfs_dquot_buf.c                      |  13 +-
+ fs/xfs/libxfs/xfs_exchmaps.c                       |   1 -
+ fs/xfs/libxfs/xfs_exchmaps.h                       |   3 +-
+ fs/xfs/libxfs/xfs_ialloc.c                         |   2 +-
+ fs/xfs/libxfs/xfs_ialloc_btree.c                   |   1 -
+ fs/xfs/libxfs/xfs_ialloc_btree.h                   |   5 +-
+ fs/xfs/libxfs/xfs_inode_fork.c                     |   4 +-
+ fs/xfs/libxfs/xfs_metadir.c                        |  32 +-
+ fs/xfs/libxfs/xfs_metadir.h                        |   5 +-
+ fs/xfs/libxfs/xfs_parent.h                         |   1 -
+ fs/xfs/libxfs/xfs_refcount_btree.c                 |  16 +-
+ fs/xfs/libxfs/xfs_refcount_btree.h                 |  26 ++
+ fs/xfs/libxfs/xfs_rmap.c                           |  13 +-
+ fs/xfs/libxfs/xfs_rmap_btree.c                     |  16 +-
+ fs/xfs/libxfs/xfs_rmap_btree.h                     |  26 ++
+ fs/xfs/libxfs/xfs_rtrefcount_btree.c               | 121 +-----
+ fs/xfs/libxfs/xfs_rtrefcount_btree.h               |   5 +-
+ fs/xfs/libxfs/xfs_rtrmap_btree.c                   | 236 +-----------
+ fs/xfs/libxfs/xfs_rtrmap_btree.h                   |   3 +-
+ fs/xfs/libxfs/xfs_sb.c                             |  40 --
+ fs/xfs/libxfs/xfs_sb.h                             |   1 -
+ fs/xfs/libxfs/xfs_symlink_remote.c                 |   8 +-
+ fs/xfs/libxfs/xfs_symlink_remote.h                 |   2 +-
+ fs/xfs/libxfs/xfs_trans_resv.c                     |  45 +--
+ fs/xfs/libxfs/xfs_trans_space.c                    |   6 +-
+ fs/xfs/libxfs/xfs_types.c                          |   7 +-
+ fs/xfs/libxfs/xfs_types.h                          |   7 +-
+ fs/xfs/scrub/alloc_repair.c                        |  15 +-
+ fs/xfs/scrub/bmap.c                                |  11 +-
+ fs/xfs/scrub/bmap_repair.c                         |   4 +-
+ fs/xfs/scrub/inode_repair.c                        |  32 +-
+ fs/xfs/scrub/orphanage.c                           |   2 +-
+ fs/xfs/scrub/quota.c                               |   2 +-
+ fs/xfs/scrub/quota_repair.c                        |   9 +-
+ fs/xfs/scrub/quotacheck_repair.c                   |  54 ++-
+ fs/xfs/scrub/rtrefcount_repair.c                   |   3 +-
+ fs/xfs/scrub/rtrmap_repair.c                       |   2 +-
+ fs/xfs/scrub/xfarray.c                             | 201 ++++------
+ fs/xfs/scrub/xfarray.h                             |   9 +-
+ fs/xfs/xfs_bmap_item.c                             |   2 +-
+ fs/xfs/xfs_dquot.c                                 |  20 +-
+ fs/xfs/xfs_dquot.h                                 |   2 +-
+ fs/xfs/xfs_exchmaps_item.c                         |   4 +-
+ fs/xfs/xfs_exchrange.c                             |   2 +-
+ fs/xfs/xfs_file.c                                  |  22 +-
+ fs/xfs/xfs_handle.c                                |   2 +-
+ fs/xfs/xfs_inode.c                                 |  20 +-
+ fs/xfs/xfs_ioctl.c                                 | 421 +++++++++++++--------
+ fs/xfs/xfs_ioctl.h                                 |   4 +-
+ fs/xfs/xfs_ioctl32.c                               | 188 +++++----
+ fs/xfs/xfs_qm.c                                    |   6 +-
+ fs/xfs/xfs_rmap_item.c                             |   2 +-
+ fs/xfs/xfs_rtalloc.h                               |  49 ++-
+ fs/xfs/xfs_super.c                                 |   2 +-
+ fs/xfs/xfs_symlink.c                               |   4 +-
+ fs/xfs/xfs_trace.h                                 |  28 +-
+ fs/xfs/xfs_trans_dquot.c                           |   6 +-
+ 63 files changed, 881 insertions(+), 942 deletions(-)
+Merging zonefs/for-next (3a8389d42bdf4 zonefs: handle integer overflow in zonefs_fname_to_fno)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git zonefs/for-next
+Already up to date.
+Merging vfs-brauner/vfs.all (84086827932b5 Merge branch 'vfs-7.4.sync_close' into vfs.all)
+$ git merge -m Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner/vfs.all
+Auto-merging Documentation/filesystems/index.rst
+Auto-merging MAINTAINERS
+Auto-merging drivers/gpio/gpiolib-cdev.c
+Auto-merging fs/9p/vfs_addr.c
+Auto-merging fs/btrfs/btrfs_inode.h
+Auto-merging fs/btrfs/inode.c
+Auto-merging fs/btrfs/ioctl.c
+Auto-merging fs/configfs/configfs_internal.h
+Auto-merging fs/configfs/dir.c
+Auto-merging fs/configfs/inode.c
+Auto-merging fs/dax.c
+Auto-merging fs/debugfs/inode.c
+Auto-merging fs/exec.c
+Auto-merging fs/exfat/exfat_fs.h
+Auto-merging fs/exfat/namei.c
+Auto-merging fs/ext2/ext2.h
+Auto-merging fs/ext2/inode.c
+Auto-merging fs/f2fs/acl.c
+Auto-merging fs/f2fs/acl.h
+Auto-merging fs/f2fs/f2fs.h
+Auto-merging fs/f2fs/file.c
+Auto-merging fs/f2fs/namei.c
+Auto-merging fs/f2fs/xattr.c
+Auto-merging fs/fuse/dir.c
+Auto-merging fs/fuse/file.c
+Auto-merging fs/fuse/fuse_i.h
+Auto-merging fs/fuse/ioctl.c
+Auto-merging fs/fuse/req.c
+Auto-merging fs/gfs2/acl.c
+Auto-merging fs/gfs2/file.c
+Auto-merging fs/gfs2/inode.c
+Auto-merging fs/gfs2/log.c
+Auto-merging fs/gfs2/lops.c
+Auto-merging fs/gfs2/xattr.c
+Auto-merging fs/namei.c
+Auto-merging fs/nfs/Kconfig
+Auto-merging fs/nfs/internal.h
+Auto-merging fs/nfs/nfs4proc.c
+Auto-merging fs/nfsd/vfs.c
+Auto-merging fs/ntfs/file.c
+Auto-merging fs/ntfs/namei.c
+Auto-merging fs/ntfs3/file.c
+Auto-merging fs/ntfs3/inode.c
+Auto-merging fs/ntfs3/xattr.c
+Auto-merging fs/ocfs2/namei.c
+Auto-merging fs/ocfs2/xattr.c
+Auto-merging fs/orangefs/inode.c
+Auto-merging fs/quota/dquot.c
+Auto-merging fs/smb/client/dir.c
+Auto-merging fs/smb/client/transport.c
+Auto-merging fs/smb/server/smb2pdu.c
+CONFLICT (content): Merge conflict in fs/smb/server/smb2pdu.c
+Auto-merging fs/smb/server/smb_common.c
+Auto-merging fs/smb/server/smbacl.c
+Auto-merging fs/smb/server/vfs.c
+CONFLICT (content): Merge conflict in fs/smb/server/vfs.c
+Auto-merging fs/smb/server/vfs.h
+CONFLICT (content): Merge conflict in fs/smb/server/vfs.h
+Auto-merging fs/super.c
+Auto-merging fs/xfs/libxfs/xfs_errortag.h
+Auto-merging fs/xfs/xfs_file.c
+Auto-merging fs/xfs/xfs_handle.c
+Auto-merging fs/xfs/xfs_inode.c
+Auto-merging fs/xfs/xfs_ioctl.c
+Auto-merging fs/xfs/xfs_ioctl.h
+Auto-merging fs/xfs/xfs_super.c
+Auto-merging fs/xfs/xfs_symlink.c
+Auto-merging fs/xfs/xfs_trace.h
+Auto-merging fs/xfs/xfs_zone_alloc.c
+Auto-merging include/linux/sched.h
+Auto-merging kernel/exit.c
+Auto-merging kernel/fork.c
+Auto-merging kernel/signal.c
+Auto-merging mm/shmem.c
+Auto-merging security/selinux/hooks.c
+Auto-merging tools/testing/selftests/Makefile
+Resolved 'fs/smb/server/smb2pdu.c' using previous resolution.
+Resolved 'fs/smb/server/vfs.c' using previous resolution.
+Resolved 'fs/smb/server/vfs.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[fs-next 5eb3fdca6ca76] Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
+$ git diff -M --stat --summary HEAD^..
+ CREDITS                                            |    2 +-
+ Documentation/admin-guide/binfmt-misc.rst          |    3 +
+ Documentation/filesystems/befs.rst                 |    4 +-
+ Documentation/filesystems/bfs.rst                  |   60 -
+ Documentation/filesystems/index.rst                |    1 -
+ Documentation/filesystems/locking.rst              |   24 +-
+ Documentation/filesystems/netfs_library.rst        |    7 +-
+ Documentation/filesystems/porting.rst              |   25 +-
+ Documentation/filesystems/proc.rst                 |    4 +
+ Documentation/filesystems/sharedsubtree.rst        |   20 +-
+ Documentation/filesystems/squashfs.rst             |    6 +-
+ Documentation/filesystems/vfs.rst                  |   30 +-
+ MAINTAINERS                                        |    8 +-
+ arch/mips/configs/malta_defconfig                  |    1 -
+ arch/mips/configs/malta_kvm_defconfig              |    1 -
+ arch/mips/configs/maltaup_xpa_defconfig            |    1 -
+ arch/mips/configs/rm200_defconfig                  |    1 -
+ arch/powerpc/Kconfig                               |    1 -
+ arch/powerpc/configs/fsl-emb-nonhw.config          |    1 -
+ arch/powerpc/configs/ppc6xx_defconfig              |    1 -
+ arch/powerpc/include/asm/elf.h                     |    6 -
+ arch/powerpc/include/asm/spu.h                     |    3 -
+ arch/powerpc/platforms/cell/Kconfig                |    1 -
+ arch/powerpc/platforms/cell/spu_syscalls.c         |   20 -
+ arch/powerpc/platforms/cell/spufs/Makefile         |    1 -
+ arch/powerpc/platforms/cell/spufs/coredump.c       |  183 --
+ arch/powerpc/platforms/cell/spufs/file.c           |  114 --
+ arch/powerpc/platforms/cell/spufs/inode.c          |   14 +-
+ arch/powerpc/platforms/cell/spufs/spufs.h          |   12 -
+ arch/powerpc/platforms/cell/spufs/syscalls.c       |    4 -
+ block/bio-integrity-fs.c                           |   15 +-
+ block/bio-integrity.c                              |    1 +
+ block/bio.c                                        |  219 +--
+ block/blk-map.c                                    |    2 +-
+ block/blk-settings.c                               |    6 +
+ block/fops.c                                       |    3 +-
+ block/partitions/efi.h                             |    8 +-
+ drivers/android/binder/rust_binderfs.c             |    2 +-
+ drivers/android/binderfs.c                         |    2 +-
+ drivers/base/devtmpfs.c                            |  108 +-
+ drivers/gpio/gpiolib-cdev.c                        |   18 +-
+ drivers/gpu/drm/msm/msm_perfcntr.c                 |    4 +-
+ drivers/media/mc/mc-request.c                      |    8 +-
+ drivers/misc/ntsync.c                              |    6 +-
+ fs/9p/acl.c                                        |    4 +-
+ fs/9p/acl.h                                        |    4 +-
+ fs/9p/v9fs.h                                       |    2 +-
+ fs/9p/v9fs_vfs.h                                   |    2 +-
+ fs/9p/vfs_addr.c                                   |    1 -
+ fs/9p/vfs_inode.c                                  |   14 +-
+ fs/9p/vfs_inode_dotl.c                             |   14 +-
+ fs/9p/xattr.c                                      |    2 +-
+ fs/Kconfig                                         |    9 +-
+ fs/Makefile                                        |    1 -
+ fs/adfs/adfs.h                                     |    2 +-
+ fs/adfs/dir.c                                      |    2 +-
+ fs/adfs/inode.c                                    |    2 +-
+ fs/affs/affs.h                                     |   10 +-
+ fs/affs/inode.c                                    |    2 +-
+ fs/affs/namei.c                                    |    8 +-
+ fs/afs/dir.c                                       |   16 +-
+ fs/afs/file.c                                      |    8 +-
+ fs/afs/inode.c                                     |    4 +-
+ fs/afs/internal.h                                  |    8 +-
+ fs/afs/security.c                                  |    2 +-
+ fs/afs/xattr.c                                     |    4 +-
+ fs/aio.c                                           |   11 +-
+ fs/anon_inodes.c                                   |    4 +-
+ fs/attr.c                                          |   16 +-
+ fs/autofs/root.c                                   |   12 +-
+ fs/backing-file.c                                  |    2 +-
+ fs/bad_inode.c                                     |   20 +-
+ fs/bfs/Kconfig                                     |   21 -
+ fs/bfs/Makefile                                    |    8 -
+ fs/bfs/bfs.h                                       |   69 -
+ fs/bfs/dir.c                                       |    4 +-
+ fs/bfs/file.c                                      |  203 --
+ fs/bfs/inode.c                                     |  538 ------
+ fs/binfmt_elf.c                                    |   18 +-
+ fs/binfmt_elf_fdpic.c                              |   14 +-
+ fs/binfmt_misc.c                                   |   12 +-
+ fs/bpf_fs_kfuncs.c                                 |    4 +-
+ fs/btrfs/acl.c                                     |    2 +-
+ fs/btrfs/acl.h                                     |    2 +-
+ fs/btrfs/btrfs_inode.h                             |    2 +-
+ fs/btrfs/inode.c                                   |   24 +-
+ fs/btrfs/ioctl.c                                   |   16 +-
+ fs/btrfs/ioctl.h                                   |    2 +-
+ fs/btrfs/xattr.c                                   |    6 +-
+ fs/buffer.c                                        |  131 +-
+ fs/cachefiles/Kconfig                              |    2 +-
+ fs/cachefiles/interface.c                          |   96 +-
+ fs/cachefiles/internal.h                           |   18 +-
+ fs/cachefiles/io.c                                 |  472 +++--
+ fs/cachefiles/namei.c                              |   34 +-
+ fs/cachefiles/xattr.c                              |   82 +-
+ fs/ceph/Kconfig                                    |    1 +
+ fs/ceph/acl.c                                      |    2 +-
+ fs/ceph/addr.c                                     |    4 +-
+ fs/ceph/dir.c                                      |   10 +-
+ fs/ceph/file.c                                     |    2 +-
+ fs/ceph/inode.c                                    |   10 +-
+ fs/ceph/mds_client.h                               |    2 +-
+ fs/ceph/super.h                                    |   10 +-
+ fs/ceph/xattr.c                                    |    2 +-
+ fs/char_dev.c                                      |    4 +-
+ fs/coda/coda_linux.h                               |    6 +-
+ fs/coda/dir.c                                      |   10 +-
+ fs/coda/inode.c                                    |    4 +-
+ fs/coda/pioctl.c                                   |    4 +-
+ fs/configfs/configfs_internal.h                    |    4 +-
+ fs/configfs/dir.c                                  |    2 +-
+ fs/configfs/inode.c                                |    2 +-
+ fs/configfs/symlink.c                              |    2 +-
+ fs/coredump.c                                      |  558 ++++--
+ fs/dax.c                                           |   26 +-
+ fs/dcache.c                                        |  206 +-
+ fs/debugfs/inode.c                                 |    2 +-
+ fs/devpts/inode.c                                  |    2 +
+ fs/ecryptfs/inode.c                                |   26 +-
+ fs/efivarfs/inode.c                                |    6 +-
+ fs/erofs/inode.c                                   |    2 +-
+ fs/erofs/internal.h                                |    2 +-
+ fs/eventfd.c                                       |    4 +-
+ fs/eventpoll.c                                     |    6 +-
+ fs/exec.c                                          |   28 +-
+ fs/exfat/exfat_fs.h                                |    4 +-
+ fs/exfat/file.c                                    |    6 +-
+ fs/exfat/misc.c                                    |    2 +-
+ fs/exfat/namei.c                                   |    6 +-
+ fs/ext2/acl.c                                      |    2 +-
+ fs/ext2/acl.h                                      |    2 +-
+ fs/ext2/ext2.h                                     |    6 +-
+ fs/ext2/inode.c                                    |    4 +-
+ fs/ext2/ioctl.c                                    |    2 +-
+ fs/ext2/namei.c                                    |   12 +-
+ fs/ext2/xattr.c                                    |    2 +-
+ fs/ext2/xattr_security.c                           |    2 +-
+ fs/ext2/xattr_trusted.c                            |    2 +-
+ fs/ext2/xattr_user.c                               |    2 +-
+ fs/ext4/acl.c                                      |    2 +-
+ fs/ext4/acl.h                                      |    2 +-
+ fs/ext4/ext4.h                                     |   10 +-
+ fs/ext4/ext4_jbd2.c                                |    2 +-
+ fs/ext4/ialloc.c                                   |    2 +-
+ fs/ext4/inode.c                                    |    6 +-
+ fs/ext4/ioctl.c                                    |    6 +-
+ fs/ext4/mmp.c                                      |    2 +-
+ fs/ext4/namei.c                                    |   16 +-
+ fs/ext4/symlink.c                                  |    2 +-
+ fs/ext4/xattr_hurd.c                               |    2 +-
+ fs/ext4/xattr_security.c                           |    2 +-
+ fs/ext4/xattr_trusted.c                            |    2 +-
+ fs/ext4/xattr_user.c                               |    2 +-
+ fs/f2fs/acl.c                                      |    6 +-
+ fs/f2fs/acl.h                                      |    2 +-
+ fs/f2fs/f2fs.h                                     |    8 +-
+ fs/f2fs/file.c                                     |   14 +-
+ fs/f2fs/namei.c                                    |   24 +-
+ fs/f2fs/xattr.c                                    |    4 +-
+ fs/failfs.c                                        |    4 +-
+ fs/fat/fat.h                                       |    4 +-
+ fs/fat/file.c                                      |    6 +-
+ fs/fat/misc.c                                      |    2 +-
+ fs/fat/namei_msdos.c                               |    6 +-
+ fs/fat/namei_vfat.c                                |    6 +-
+ fs/fhandle.c                                       |    2 +-
+ fs/file.c                                          |  321 +++-
+ fs/file_attr.c                                     |    6 +-
+ fs/fs-writeback.c                                  |   23 +-
+ fs/fs_pin.c                                        |    9 +-
+ fs/fuse/acl.c                                      |    4 +-
+ fs/fuse/dir.c                                      |   51 +-
+ fs/fuse/file.c                                     |    2 +-
+ fs/fuse/fuse_i.h                                   |   12 +-
+ fs/fuse/ioctl.c                                    |    2 +-
+ fs/fuse/req.c                                      |    8 +-
+ fs/fuse/xattr.c                                    |    2 +-
+ fs/gfs2/acl.c                                      |    2 +-
+ fs/gfs2/acl.h                                      |    2 +-
+ fs/gfs2/file.c                                     |    2 +-
+ fs/gfs2/inode.c                                    |   16 +-
+ fs/gfs2/inode.h                                    |    4 +-
+ fs/gfs2/log.c                                      |    4 +-
+ fs/gfs2/lops.c                                     |    4 +-
+ fs/gfs2/xattr.c                                    |    2 +-
+ fs/hfs/attr.c                                      |    2 +-
+ fs/hfs/dir.c                                       |    6 +-
+ fs/hfs/hfs_fs.h                                    |    2 +-
+ fs/hfs/inode.c                                     |    2 +-
+ fs/hfsplus/dir.c                                   |   10 +-
+ fs/hfsplus/hfsplus_fs.h                            |    4 +-
+ fs/hfsplus/inode.c                                 |    6 +-
+ fs/hfsplus/xattr.c                                 |    2 +-
+ fs/hfsplus/xattr_security.c                        |    2 +-
+ fs/hfsplus/xattr_trusted.c                         |    2 +-
+ fs/hfsplus/xattr_user.c                            |    2 +-
+ fs/hostfs/hostfs_kern.c                            |   14 +-
+ fs/hpfs/hpfs_fn.h                                  |    2 +-
+ fs/hpfs/inode.c                                    |    2 +-
+ fs/hpfs/namei.c                                    |   10 +-
+ fs/hugetlbfs/inode.c                               |   14 +-
+ fs/inode.c                                         |   14 +-
+ fs/internal.h                                      |   30 +-
+ fs/iomap/bio.c                                     |    4 +-
+ fs/iomap/buffered-io.c                             |    8 +-
+ fs/iomap/direct-io.c                               |   46 +-
+ fs/iomap/ioend.c                                   |  122 +-
+ fs/jbd2/commit.c                                   |   22 +-
+ fs/jbd2/journal.c                                  |   31 +-
+ fs/jbd2/transaction.c                              |    2 +-
+ fs/jffs2/acl.c                                     |    2 +-
+ fs/jffs2/acl.h                                     |    2 +-
+ fs/jffs2/dir.c                                     |   20 +-
+ fs/jffs2/fs.c                                      |    2 +-
+ fs/jffs2/os-linux.h                                |    2 +-
+ fs/jffs2/security.c                                |    2 +-
+ fs/jffs2/xattr_trusted.c                           |    2 +-
+ fs/jffs2/xattr_user.c                              |    2 +-
+ fs/jfs/acl.c                                       |    2 +-
+ fs/jfs/file.c                                      |    2 +-
+ fs/jfs/ioctl.c                                     |    2 +-
+ fs/jfs/jfs_acl.h                                   |    2 +-
+ fs/jfs/jfs_inode.h                                 |    4 +-
+ fs/jfs/namei.c                                     |   10 +-
+ fs/jfs/xattr.c                                     |    4 +-
+ fs/kernfs/dir.c                                    |  231 ++-
+ fs/kernfs/file.c                                   |   47 +-
+ fs/kernfs/inode.c                                  |   10 +-
+ fs/kernfs/kernfs-internal.h                        |   24 +-
+ fs/kernfs/mount.c                                  |   34 +-
+ fs/kernfs/symlink.c                                |   17 +-
+ fs/libfs.c                                         |    8 +-
+ fs/minix/file.c                                    |    2 +-
+ fs/minix/inode.c                                   |    2 +-
+ fs/minix/minix.h                                   |    2 +-
+ fs/minix/namei.c                                   |   12 +-
+ fs/mnt_idmapping.c                                 |   33 +-
+ fs/mount.h                                         |    3 +-
+ fs/namei.c                                         |  153 +-
+ fs/namespace.c                                     |   75 +-
+ fs/netfs/Kconfig                                   |    3 +
+ fs/netfs/Makefile                                  |    2 +-
+ fs/netfs/buffered_read.c                           |  217 ++-
+ fs/netfs/buffered_write.c                          |   69 +-
+ fs/netfs/direct_read.c                             |   14 +-
+ fs/netfs/direct_write.c                            |    8 +-
+ fs/netfs/fscache_cookie.c                          |    8 +-
+ fs/netfs/fscache_internal.h                        |   14 -
+ fs/netfs/fscache_io.c                              |   10 +-
+ fs/netfs/internal.h                                |   70 +-
+ fs/netfs/iterator.c                                |    2 +-
+ fs/netfs/main.c                                    |    1 -
+ fs/netfs/misc.c                                    |   10 +-
+ fs/netfs/objects.c                                 |    7 +-
+ fs/netfs/read_collect.c                            |   14 +-
+ fs/netfs/read_pgpriv2.c                            |   13 +-
+ fs/netfs/read_retry.c                              |    7 +-
+ fs/netfs/read_single.c                             |   50 +-
+ fs/netfs/stats.c                                   |    4 +-
+ fs/netfs/write_collect.c                           |  157 +-
+ fs/netfs/write_issue.c                             |  141 +-
+ fs/netfs/write_retry.c                             |    8 +-
+ fs/nfs/Kconfig                                     |    1 +
+ fs/nfs/dir.c                                       |   12 +-
+ fs/nfs/inode.c                                     |    4 +-
+ fs/nfs/internal.h                                  |   10 +-
+ fs/nfs/namespace.c                                 |    4 +-
+ fs/nfs/nfs3_fs.h                                   |    2 +-
+ fs/nfs/nfs3acl.c                                   |    2 +-
+ fs/nfs/nfs4proc.c                                  |   10 +-
+ fs/nfs/unlink.c                                    |    3 +
+ fs/nfsd/vfs.c                                      |   13 +-
+ fs/nilfs2/inode.c                                  |    4 +-
+ fs/nilfs2/ioctl.c                                  |    2 +-
+ fs/nilfs2/namei.c                                  |   10 +-
+ fs/nilfs2/nilfs.h                                  |    6 +-
+ fs/nls/nls_iso8859-14.c                            |   26 +-
+ fs/nsfs.c                                          |    4 +-
+ fs/ntfs/ea.c                                       |   10 +-
+ fs/ntfs/ea.h                                       |    6 +-
+ fs/ntfs/file.c                                     |    4 +-
+ fs/ntfs/inode.h                                    |    4 +-
+ fs/ntfs/namei.c                                    |   12 +-
+ fs/ntfs3/file.c                                    |    6 +-
+ fs/ntfs3/inode.c                                   |    2 +-
+ fs/ntfs3/namei.c                                   |   10 +-
+ fs/ntfs3/ntfs_fs.h                                 |   16 +-
+ fs/ntfs3/xattr.c                                   |   12 +-
+ fs/ocfs2/acl.c                                     |    2 +-
+ fs/ocfs2/acl.h                                     |    2 +-
+ fs/ocfs2/buffer_head_io.c                          |   12 +-
+ fs/ocfs2/dlmfs/dlmfs.c                             |    6 +-
+ fs/ocfs2/file.c                                    |    6 +-
+ fs/ocfs2/file.h                                    |    6 +-
+ fs/ocfs2/ioctl.c                                   |    2 +-
+ fs/ocfs2/ioctl.h                                   |    2 +-
+ fs/ocfs2/journal.c                                 |   25 +-
+ fs/ocfs2/namei.c                                   |   10 +-
+ fs/ocfs2/xattr.c                                   |    6 +-
+ fs/omfs/dir.c                                      |    6 +-
+ fs/omfs/file.c                                     |    2 +-
+ fs/omfs/inode.c                                    |    4 +-
+ fs/open.c                                          |   50 +-
+ fs/orangefs/acl.c                                  |    2 +-
+ fs/orangefs/inode.c                                |    8 +-
+ fs/orangefs/namei.c                                |    8 +-
+ fs/orangefs/orangefs-kernel.h                      |    8 +-
+ fs/orangefs/xattr.c                                |    2 +-
+ fs/overlayfs/dir.c                                 |   14 +-
+ fs/overlayfs/file.c                                |    2 +-
+ fs/overlayfs/inode.c                               |   16 +-
+ fs/overlayfs/overlayfs.h                           |   14 +-
+ fs/overlayfs/ovl_entry.h                           |    2 +-
+ fs/overlayfs/util.c                                |    4 +-
+ fs/overlayfs/xattrs.c                              |    4 +-
+ fs/pidfs.c                                         |    6 +-
+ fs/pipe.c                                          |    2 +-
+ fs/pnode.c                                         |  108 +-
+ fs/posix_acl.c                                     |   26 +-
+ fs/proc/base.c                                     |   53 +-
+ fs/proc/fd.c                                       |    6 +-
+ fs/proc/fd.h                                       |    2 +-
+ fs/proc/generic.c                                  |    4 +-
+ fs/proc/internal.h                                 |    4 +-
+ fs/proc/proc_net.c                                 |    2 +-
+ fs/proc/proc_sysctl.c                              |    6 +-
+ fs/proc/root.c                                     |    2 +-
+ fs/proc/vmcore.c                                   |   20 +
+ fs/quota/dquot.c                                   |    2 +-
+ fs/ramfs/file-nommu.c                              |    4 +-
+ fs/ramfs/inode.c                                   |   10 +-
+ fs/read_write.c                                    |    8 +-
+ fs/remap_range.c                                   |    2 +-
+ fs/smb/client/cifsacl.c                            |    4 +-
+ fs/smb/client/cifsfs.c                             |    2 +-
+ fs/smb/client/cifsfs.h                             |   16 +-
+ fs/smb/client/cifsproto.h                          |    4 +-
+ fs/smb/client/dir.c                                |    6 +-
+ fs/smb/client/inode.c                              |    8 +-
+ fs/smb/client/link.c                               |    2 +-
+ fs/smb/client/transport.c                          |   13 +-
+ fs/smb/client/xattr.c                              |    2 +-
+ fs/smb/server/ndr.c                                |    2 +-
+ fs/smb/server/ndr.h                                |    2 +-
+ fs/smb/server/oplock.c                             |    2 +-
+ fs/smb/server/smb2pdu.c                            |   22 +-
+ fs/smb/server/smb_common.c                         |    2 +-
+ fs/smb/server/smbacl.c                             |   20 +-
+ fs/smb/server/smbacl.h                             |    8 +-
+ fs/smb/server/vfs.c                                |   44 +-
+ fs/smb/server/vfs.h                                |   32 +-
+ fs/splice.c                                        |   70 +-
+ fs/stat.c                                          |    4 +-
+ fs/super.c                                         |   19 +-
+ fs/tests/.kunitconfig                              |    2 +
+ fs/tests/fdtable_kunit.c                           |   72 +
+ fs/tracefs/event_inode.c                           |    2 +-
+ fs/tracefs/inode.c                                 |    8 +-
+ fs/ubifs/dir.c                                     |   14 +-
+ fs/ubifs/file.c                                    |    4 +-
+ fs/ubifs/ioctl.c                                   |    2 +-
+ fs/ubifs/ubifs.h                                   |    6 +-
+ fs/ubifs/xattr.c                                   |    2 +-
+ fs/udf/file.c                                      |    2 +-
+ fs/udf/namei.c                                     |   12 +-
+ fs/udf/symlink.c                                   |    2 +-
+ fs/ufs/dir.c                                       |    2 +-
+ fs/ufs/inode.c                                     |    2 +-
+ fs/ufs/namei.c                                     |   10 +-
+ fs/ufs/ufs.h                                       |    2 +-
+ fs/vboxsf/dir.c                                    |    8 +-
+ fs/vboxsf/utils.c                                  |    4 +-
+ fs/vboxsf/vfsmod.h                                 |    4 +-
+ fs/xattr.c                                         |   30 +-
+ fs/xfs/libxfs/xfs_errortag.h                       |    6 +-
+ fs/xfs/libxfs/xfs_inode_util.h                     |    2 +-
+ fs/xfs/xfs_acl.c                                   |    2 +-
+ fs/xfs/xfs_acl.h                                   |    2 +-
+ fs/xfs/xfs_aops.c                                  |   13 +-
+ fs/xfs/xfs_buf.c                                   |   11 +
+ fs/xfs/xfs_file.c                                  |   11 +-
+ fs/xfs/xfs_handle.c                                |    6 +-
+ fs/xfs/xfs_inode.c                                 |    4 +-
+ fs/xfs/xfs_inode.h                                 |    2 +-
+ fs/xfs/xfs_ioctl.c                                 |    2 +-
+ fs/xfs/xfs_ioctl.h                                 |    2 +-
+ fs/xfs/xfs_ioend.c                                 |  137 +-
+ fs/xfs/xfs_ioend.h                                 |    2 +
+ fs/xfs/xfs_iops.c                                  |   24 +-
+ fs/xfs/xfs_iops.h                                  |    2 +-
+ fs/xfs/xfs_itable.c                                |    2 +-
+ fs/xfs/xfs_itable.h                                |    2 +-
+ fs/xfs/xfs_mount.h                                 |    8 +
+ fs/xfs/xfs_super.c                                 |    1 +
+ fs/xfs/xfs_symlink.c                               |    2 +-
+ fs/xfs/xfs_symlink.h                               |    2 +-
+ fs/xfs/xfs_sysfs.c                                 |   78 +-
+ fs/xfs/xfs_trace.h                                 |    1 +
+ fs/xfs/xfs_xattr.c                                 |    2 +-
+ fs/xfs/xfs_zone_alloc.c                            |    4 +
+ fs/zonefs/super.c                                  |    2 +-
+ include/linux/binfmts.h                            |    3 +-
+ include/linux/bio-integrity.h                      |    3 +-
+ include/linux/bio.h                                |   11 +-
+ include/linux/blkdev.h                             |    8 +-
+ include/linux/buffer_head.h                        |   86 +-
+ include/linux/capability.h                         |    8 +-
+ include/linux/cleanup.h                            |    7 -
+ include/linux/coredump.h                           |   37 +-
+ include/linux/dax.h                                |   12 -
+ include/linux/dcache.h                             |   38 +-
+ include/linux/fdtable.h                            |   15 +-
+ include/linux/file.h                               |  130 +-
+ include/linux/fileattr.h                           |    2 +-
+ include/linux/fs.h                                 |  127 +-
+ include/linux/fs_context.h                         |    4 +
+ include/linux/fscache-cache.h                      |    2 +-
+ include/linux/fscache.h                            |   53 +-
+ include/linux/iomap.h                              |   38 +-
+ include/linux/lsm_hook_defs.h                      |   25 +-
+ include/linux/mnt_idmapping.h                      |   24 +-
+ include/linux/mount.h                              |    4 +-
+ include/linux/namei.h                              |   19 +-
+ include/linux/netfs.h                              |  112 +-
+ include/linux/nfs_fs.h                             |    6 +-
+ include/linux/posix_acl.h                          |   24 +-
+ include/linux/quotaops.h                           |    6 +-
+ include/linux/sched.h                              |    2 +-
+ include/linux/sched/signal.h                       |   33 +-
+ include/linux/security.h                           |   61 +-
+ include/linux/splice.h                             |    4 +-
+ include/linux/uidgid.h                             |   12 +-
+ include/linux/user_namespace.h                     |   11 +-
+ include/linux/wait_bit.h                           |   26 +
+ include/linux/xattr.h                              |   20 +-
+ include/trace/events/cachefiles.h                  |   81 +-
+ include/trace/events/fscache.h                     |   10 +-
+ include/trace/events/netfs.h                       |  145 +-
+ include/uapi/linux/close_range.h                   |   31 +-
+ include/uapi/linux/coredump.h                      |  149 +-
+ include/uapi/linux/fs.h                            |    2 +-
+ init/Kconfig                                       |   11 +
+ init/initramfs.c                                   |   11 +-
+ io_uring/io-wq.c                                   |    2 +
+ io_uring/mock_file.c                               |    8 +-
+ ipc/mqueue.c                                       |    9 +-
+ kernel/Makefile                                    |    1 +
+ kernel/bpf/bpf_iter.c                              |    6 +-
+ kernel/bpf/inode.c                                 |    6 +-
+ kernel/bpf/token.c                                 |    6 +-
+ kernel/capability.c                                |    4 +-
+ kernel/exit.c                                      |   15 +-
+ kernel/fork.c                                      |   68 +-
+ kernel/kthread.c                                   |    2 +-
+ kernel/pid_namespace.c                             |    3 +-
+ kernel/ptrace.c                                    |    6 +
+ kernel/signal.c                                    |   14 +
+ kernel/tests/.kunitconfig                          |    4 +
+ kernel/tests/user_ns_map_kunit.c                   |   98 +
+ kernel/user_namespace.c                            |   60 +-
+ kernel/utsname.c                                   |    1 -
+ mm/secretmem.c                                     |    2 +-
+ mm/shmem.c                                         |   30 +-
+ mm/userfaultfd.c                                   |    6 +-
+ net/core/scm.c                                     |    8 +-
+ net/handshake/netlink.c                            |    8 +-
+ net/kcm/kcmsock.c                                  |    6 +-
+ net/socket.c                                       |    6 +-
+ net/unix/af_unix.c                                 |    2 +-
+ security/apparmor/apparmorfs.c                     |    2 +-
+ security/apparmor/lsm.c                            |    4 +-
+ security/commoncap.c                               |   10 +-
+ security/integrity/evm/evm_main.c                  |   26 +-
+ security/integrity/ima/ima.h                       |   10 +-
+ security/integrity/ima/ima_api.c                   |    2 +-
+ security/integrity/ima/ima_appraise.c              |   12 +-
+ security/integrity/ima/ima_main.c                  |    6 +-
+ security/integrity/ima/ima_policy.c                |    4 +-
+ security/security.c                                |   49 +-
+ security/selinux/hooks.c                           |   45 +-
+ security/selinux/selinuxfs.c                       |    2 +-
+ security/smack/smack_lsm.c                         |   14 +-
+ tools/include/uapi/linux/coredump.h                |  149 +-
+ tools/testing/selftests/Makefile                   |    3 +
+ .../selftests/clone3/clone3_clear_sighand.c        |    6 +-
+ tools/testing/selftests/core/close_range_test.c    |  951 ++++++++++
+ tools/testing/selftests/coredump/.gitignore        |    2 +
+ tools/testing/selftests/coredump/Makefile          |   11 +-
+ .../selftests/coredump/coredump_notify_signal.h    |   29 +
+ .../coredump/coredump_notify_signal_helper.c       |   46 +
+ .../coredump/coredump_notify_signal_test.c         |  245 +++
+ .../selftests/coredump/coredump_signal_test.c      |  238 +++
+ .../coredump/coredump_socket_protocol_test.c       | 1983 +++++++++++++++-----
+ tools/testing/selftests/coredump/coredump_test.h   |   32 +-
+ .../selftests/coredump/coredump_test_helpers.c     | 1742 ++++++++++++++++-
+ .../selftests/coredump/coredump_test_helpers.h     |   79 +
+ .../selftests/coredump/coredump_worker_test.c      |  447 +++++
+ tools/testing/selftests/exec/Makefile              |    4 +
+ tools/testing/selftests/exec/binfmt_misc_delim.c   |  127 ++
+ tools/testing/selftests/filesystems/.gitignore     |    1 -
+ tools/testing/selftests/filesystems/Makefile       |    2 +-
+ tools/testing/selftests/filesystems/config         |    8 +
+ .../selftests/filesystems/file_stressor/.gitignore |    2 +
+ .../selftests/filesystems/file_stressor/Makefile   |    6 +
+ .../{ => file_stressor}/file_stressor.c            |    0
+ .../selftests/filesystems/file_stressor/settings   |    3 +
+ .../selftests/filesystems/fscontext_ns/.gitignore  |    2 +
+ tools/testing/selftests/filesystems/kernfs_test.c  | 1203 +++++++++++-
+ .../filesystems/mntns_unbindable/Makefile          |    6 +
+ .../mntns_unbindable/mntns_unbindable_test.c       |  227 +++
+ .../selftests/filesystems/openat2/openat2_test.c   |   10 +-
+ .../selftests/filesystems/openat2/resolve_test.c   |    7 +-
+ .../filesystems/statmount/statmount_test.c         |    2 +-
+ .../filesystems/umount_propagation/Makefile        |    6 +
+ .../umount_propagation/umount_propagation_test.c   |  226 +++
+ .../move_mount_set_group_test.c                    |   74 +-
+ tools/testing/selftests/pidfd/pidfd_open_test.c    |    2 +-
+ virt/kvm/guest_memfd.c                             |    2 +-
+ 519 files changed, 12646 insertions(+), 5165 deletions(-)
+ delete mode 100644 Documentation/filesystems/bfs.rst
+ delete mode 100644 arch/powerpc/platforms/cell/spufs/coredump.c
+ delete mode 100644 fs/bfs/Kconfig
+ delete mode 100644 fs/bfs/Makefile
+ delete mode 100644 fs/bfs/bfs.h
+ delete mode 100644 fs/bfs/file.c
+ delete mode 100644 fs/bfs/inode.c
+ delete mode 100644 fs/netfs/fscache_internal.h
+ create mode 100644 fs/tests/.kunitconfig
+ create mode 100644 fs/tests/fdtable_kunit.c
+ create mode 100644 kernel/tests/.kunitconfig
+ create mode 100644 kernel/tests/user_ns_map_kunit.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_notify_signal.h
+ create mode 100644 tools/testing/selftests/coredump/coredump_notify_signal_helper.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_notify_signal_test.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_signal_test.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_test_helpers.h
+ create mode 100644 tools/testing/selftests/coredump/coredump_worker_test.c
+ create mode 100644 tools/testing/selftests/exec/binfmt_misc_delim.c
+ create mode 100644 tools/testing/selftests/filesystems/config
+ create mode 100644 tools/testing/selftests/filesystems/file_stressor/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/file_stressor/Makefile
+ rename tools/testing/selftests/filesystems/{ => file_stressor}/file_stressor.c (100%)
+ create mode 100644 tools/testing/selftests/filesystems/file_stressor/settings
+ create mode 100644 tools/testing/selftests/filesystems/fscontext_ns/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/mntns_unbindable/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/mntns_unbindable/mntns_unbindable_test.c
+ create mode 100644 tools/testing/selftests/filesystems/umount_propagation/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/umount_propagation/umount_propagation_test.c
+$ git am -3 ../patches/0001-ksmbd-Fix-removal-of-type-parameter-from-vfs_path_pa.patch
+Applying: ksmbd: Fix removal of type parameter from vfs_path_parent_lookup()
+Using index info to reconstruct a base tree...
+M	fs/smb/server/vfs.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/smb/server/vfs.c
+No changes -- Patch already applied.
+Merging vfs/for-next (4dda01b67c866 Merge branches 'work.dcache', 'work.dcache-d_add' and 'work.configfs' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs/for-next
+Merge made by the 'ort' strategy.
+Merging mm-fixes/for-next-fixes (ffd79f732828e Merge https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-hotfixes-unstable into for-next-fixes)
+$ git merge -m Merge branch 'for-next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mm/linux.git mm-fixes/for-next-fixes
+Updating 551c722f40809..ffd79f732828e
+Fast-forward (no commit created; -m option ignored)
+ .mailmap                                     |   3 +-
+ MAINTAINERS                                  |   2 +-
+ drivers/char/mem.c                           |   2 +-
+ lib/test_xarray.c                            |  70 ++++++++++++++++++
+ lib/xarray.c                                 |   8 ++-
+ mm/hugetlb.c                                 |  25 +++++--
+ mm/mremap.c                                  |  87 +++++++++++++---------
+ mm/page_alloc.c                              | 103 ++++++++++++++++++---------
+ mm/pgtable-generic.c                         |  20 +++++-
+ mm/slub.c                                    |   7 +-
+ mm/userfaultfd.c                             |   1 +
+ mm/vma.c                                     |  19 ++++-
+ mm/vma.h                                     |   7 +-
+ mm/vmalloc.c                                 |  79 +++++++++++++-------
+ tools/testing/selftests/mm/hugetlb-mmap.c    |   4 +-
+ tools/testing/selftests/mm/uffd-unit-tests.c |  84 ++++++++++++++++++++++
+ tools/testing/vma/tests/vma.c                |  10 +--
+ 17 files changed, 413 insertions(+), 118 deletions(-)
+Merging fs-current (5a9410e0a34d2 Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git)
+$ git merge -m Merge branch 'fs-current' of linux-next fs-current
+Merge made by the 'ort' strategy.
+Merging kbuild-current/kbuild-fixes-for-next (fd73f4a665989 Linux 7.3-rc3)
+$ git merge -m Merge branch 'kbuild-fixes-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild-current/kbuild-fixes-for-next
+Already up to date.
+Merging clang-fixes-current/clang-fixes-for-current (df2908090cda3 Linux 7.3-rc2)
+$ git merge -m Merge branch 'clang-fixes-for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git clang-fixes-current/clang-fixes-for-current
+Already up to date.
+Merging arc-current/for-curr (f050c3e61d2a1 ARC: arch_cmpxchg_relaxed to use size of pointed type not pointer)
+$ git merge -m Merge branch 'for-curr' of https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git arc-current/for-curr
+Already up to date.
+Merging arm-current/fixes (1039bffd6ae9c ARM: 9485/1: mm: acquire mmap write lock around show_pte() for user faults)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm-current/fixes
+Already up to date.
+Merging arm64-fixes/for-next/fixes (3872cc6b92940 arm64: topology: fix arch_freq_get_on_cpu() overflow above 4.19 GHz)
+$ git merge -m Merge branch 'for-next/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64-fixes/for-next/fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/kernel/topology.c | 6 ++----
+ arch/arm64/mm/mmu.c          | 2 +-
+ 2 files changed, 3 insertions(+), 5 deletions(-)
+Merging arm-soc-fixes/arm/fixes (4dd1999783d7d soc: samsung: exynos-pmu: fix use-after-free of interrupt generator node)
+$ git merge -m Merge branch 'arm/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc-fixes/arm/fixes
+Already up to date.
+Merging davinci-current/davinci/for-current (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'davinci/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci-current/davinci/for-current
+Already up to date.
+Merging realtek-fixes/fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git realtek-fixes/fixes
+Already up to date.
+Merging drivers-memory-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory-fixes/fixes
+Already up to date.
+Merging sophgo-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/sophgo/linux.git sophgo-fixes/fixes
+Already up to date.
+Merging sophgo-soc-fixes/soc-fixes (0af2f6be1b428 Linux 6.15-rc1)
+$ git merge -m Merge branch 'soc-fixes' of https://github.com/sophgo/linux.git sophgo-soc-fixes/soc-fixes
+Already up to date.
+Merging m68k-current/for-linus (2f8e3cad53b5c m68k: nfcon: Do not call console_is_registered() in nfcon_device())
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k-current/for-linus
+Already up to date.
+Merging powerpc-fixes/fixes (93f51579e7df2 Linux 7.3-rc4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc-fixes/fixes
+Already up to date.
+Merging s390-fixes/fixes (5b76268dac968 s390/cio: Fix NULL pointer dereference in ccw_device_get_util_str())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390-fixes/fixes
+Already up to date.
+Merging net/main (99b43ede9e355 net: microchip: vcap: stop scanning after deleting key field)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git net/main
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .mailmap                                           |   2 +
+ MAINTAINERS                                        |   9 +-
+ drivers/bluetooth/btintel.c                        |  15 +-
+ drivers/bluetooth/btintel_pcie.c                   |  12 +-
+ drivers/net/ethernet/broadcom/bcmsysport.c         |   8 +-
+ drivers/net/ethernet/broadcom/genet/bcmgenet.c     |  34 +-
+ drivers/net/ethernet/broadcom/genet/bcmgenet.h     |   2 +
+ drivers/net/ethernet/google/gve/gve_tx_dqo.c       |  14 +-
+ drivers/net/ethernet/marvell/octeontx2/af/mcs.c    |   4 +-
+ drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c |  11 +-
+ .../ethernet/marvell/octeontx2/nic/otx2_common.c   |  16 +-
+ .../ethernet/marvell/octeontx2/nic/otx2_common.h   |   9 +
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c        |  38 ++-
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c  |  10 +-
+ drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c |   3 +
+ drivers/net/ethernet/microchip/vcap/vcap_api.c     |   2 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_main.c  |  83 +++--
+ drivers/net/pcs/pcs-rzn1-miic.c                    |   3 +
+ drivers/net/pfcp.c                                 |   3 +
+ drivers/net/phy/qcom/at803x.c                      |   8 +-
+ include/linux/netfilter_netdev.h                   |   2 +-
+ include/linux/rtnetlink.h                          |   7 +-
+ include/linux/skbuff.h                             |   2 +-
+ include/net/bluetooth/hci_core.h                   |   5 +-
+ include/net/gso.h                                  |   8 +
+ include/net/sch_generic.h                          |   9 +
+ net/bluetooth/hci_conn.c                           |  18 +-
+ net/bluetooth/hci_core.c                           |  54 ++-
+ net/bluetooth/hci_debugfs.c                        |   2 +-
+ net/bluetooth/hci_sync.c                           |  19 +-
+ net/bluetooth/rfcomm/tty.c                         |   7 +-
+ net/bluetooth/smp.c                                |   2 +
+ net/core/dev.c                                     |  78 +++--
+ net/core/dev_ioctl.c                               |   3 +-
+ net/core/gso.c                                     |   1 +
+ net/core/page_pool.c                               |   2 +-
+ net/core/skbuff.c                                  |   8 +-
+ net/core/sock.c                                    |   4 +-
+ net/ethtool/tsconfig.c                             |  24 +-
+ net/ipv4/af_inet.c                                 |   3 +
+ net/ipv4/tcp_input.c                               |   6 +
+ net/ipv6/addrconf.c                                |   2 +-
+ net/ipv6/ip6_offload.c                             |   4 +
+ net/ipv6/seg6_local.c                              | 140 ++++++--
+ net/mctp/device.c                                  |   6 +-
+ net/packet/af_packet.c                             |   3 +
+ net/sched/act_skbedit.c                            |   5 +-
+ net/sched/cls_api.c                                |  15 +-
+ net/sched/sch_codel.c                              |   2 +-
+ net/sched/sch_fq_codel.c                           |   4 +-
+ net/tipc/crypto.c                                  |  20 +-
+ rust/kernel/net/netlink.rs                         |   9 +
+ tools/testing/selftests/net/.gitignore             |   1 +
+ tools/testing/selftests/net/Makefile               |   1 +
+ .../selftests/net/packetdrill/tcp_old_ack_ts.pkt   |  22 ++
+ tools/testing/selftests/net/udp_splice_checksum.c  | 378 +++++++++++++++++++++
+ 56 files changed, 940 insertions(+), 222 deletions(-)
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_old_ack_ts.pkt
+ create mode 100644 tools/testing/selftests/net/udp_splice_checksum.c
+Merging bpf/master (564b7e8e09ec9 Merge branch 'bpf-fix-use-after-free-of-progs-detached-from-busy-trampolines')
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/ bpf/master
+Merge made by the 'ort' strategy.
+ arch/arm64/net/bpf_jit_comp.c                      |  37 ++--
+ arch/loongarch/net/bpf_jit.c                       |  60 ++++---
+ arch/powerpc/net/bpf_jit_comp.c                    |  45 ++---
+ arch/riscv/net/bpf_jit_comp64.c                    |  56 ++++--
+ arch/s390/net/bpf_jit_comp.c                       |  46 +++--
+ arch/x86/net/bpf_jit_comp.c                        |  26 +--
+ include/linux/bpf.h                                |  45 ++++-
+ kernel/bpf/core.c                                  |  10 +-
+ kernel/bpf/hashtab.c                               |  17 +-
+ kernel/bpf/syscall.c                               |  19 ++-
+ kernel/bpf/trampoline.c                            |  85 ++++++++--
+ .../selftests/bpf/prog_tests/bpf_mod_race.c        |  34 +---
+ .../selftests/bpf/prog_tests/tramp_prog_detach.c   | 188 +++++++++++++++++++++
+ .../selftests/bpf/progs/tramp_prog_detach.c        |  56 ++++++
+ tools/testing/selftests/bpf/testing_helpers.c      |  28 +++
+ tools/testing/selftests/bpf/testing_helpers.h      |   2 +
+ 16 files changed, 596 insertions(+), 158 deletions(-)
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c
+ create mode 100644 tools/testing/selftests/bpf/progs/tramp_prog_detach.c
+Merging ipsec/master (868f63c8bfafa xfrm: retry inexact policy lookup after node reinsertion)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git ipsec/master
+Merge made by the 'ort' strategy.
+ net/xfrm/xfrm_policy.c | 11 +++++++++--
+ 1 file changed, 9 insertions(+), 2 deletions(-)
+Merging netfilter/main (9c572a83037a7 net/sched: fix potential stack infoleak in em_text_dump())
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git netfilter/main
+Already up to date.
+Merging ipvs/main (a401a9d547c50 Merge branch 'net-macb-fix-two-probe-path-leaks')
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git ipvs/main
+Already up to date.
+Merging bluetooth-fixes/master (86ef0f58bdecd Bluetooth: MGMT: Fix status of pending commands flushed on power off)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth.git bluetooth-fixes/master
+Merge made by the 'ort' strategy.
+ drivers/bluetooth/btintel_pcie.c |   9 ++-
+ include/net/bluetooth/rfcomm.h   |   1 +
+ net/bluetooth/mgmt.c             |   2 +-
+ net/bluetooth/rfcomm/core.c      | 134 +++++++++++++++++++++++++++------------
+ net/bluetooth/rfcomm/sock.c      |   5 +-
+ net/bluetooth/sco.c              |  87 ++++++++++++++++++-------
+ 6 files changed, 174 insertions(+), 64 deletions(-)
+Merging wireless/for-next (6f63e919fe1e3 wifi: mac80211: fix slab-out-of-bounds read in ieee80211_monitor_select_queue())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git wireless/for-next
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath11k/mac.c          |  1 +
+ drivers/net/wireless/ath/ath9k/hif_usb.c       | 21 +++---
+ drivers/net/wireless/ath/ath9k/htc.h           |  1 -
+ drivers/net/wireless/ath/ath9k/htc_drv_init.c  |  7 +-
+ drivers/net/wireless/ath/ath9k/htc_drv_txrx.c  |  8 +--
+ drivers/net/wireless/ath/ath9k/wmi.c           | 28 +++++---
+ drivers/net/wireless/ath/ath9k/wmi.h           |  1 +
+ drivers/net/wireless/intel/iwlegacy/3945-mac.c |  1 +
+ drivers/net/wireless/intersil/p54/fwio.c       | 42 ++++++++++--
+ drivers/net/wireless/silabs/wfx/hif_tx.c       |  4 +-
+ drivers/net/wireless/st/cw1200/bh.c            |  8 ++-
+ drivers/net/wireless/st/cw1200/txrx.c          | 14 ++++
+ drivers/net/wireless/ti/wlcore/main.c          |  4 +-
+ net/mac80211/agg-rx.c                          |  2 +-
+ net/mac80211/chan.c                            | 50 ++++++++++++---
+ net/mac80211/drop.h                            |  1 +
+ net/mac80211/fils_aead.c                       |  8 +++
+ net/mac80211/ieee80211_i.h                     |  2 +-
+ net/mac80211/iface.c                           |  4 +-
+ net/mac80211/mesh_pathtbl.c                    | 30 +++++++--
+ net/mac80211/mlme.c                            |  4 +-
+ net/mac80211/rc80211_minstrel_ht.c             | 89 +++++++++++++++++++++++---
+ net/mac80211/rx.c                              |  6 ++
+ net/mac80211/spectmgmt.c                       |  4 +-
+ net/mac80211/sta_info.c                        |  6 +-
+ net/mac80211/status.c                          | 70 ++++++++++++++++++++
+ net/mac80211/tx.c                              | 45 +++++++++----
+ net/wireless/nl80211.c                         | 11 ++--
+ net/wireless/scan.c                            | 22 ++++---
+ 29 files changed, 395 insertions(+), 99 deletions(-)
+Merging ath/for-current (6f63e919fe1e3 wifi: mac80211: fix slab-out-of-bounds read in ieee80211_monitor_select_queue())
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath/for-current
+Already up to date.
+Merging iwlwifi/fixes (6f63e919fe1e3 wifi: mac80211: fix slab-out-of-bounds read in ieee80211_monitor_select_queue())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi/fixes
+Already up to date.
+Merging wpan/master (2b4707a149a55 net: mctp: i3c: serialize probe with bus removal)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git wpan/master
+Already up to date.
+Merging rdma-fixes/for-rc (72d3fcf802c45 Linux 7.3-rc5)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma-fixes/for-rc
+Already up to date.
+Merging sound-current/for-linus (a154f7b9f197b ALSA: usb-audio: Apply IGNORE_CTL_ERROR quirk to all Audient devices)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound-current/for-linus
+Merge made by the 'ort' strategy.
+ sound/core/seq/seq_compat.c       |  4 ++-
+ sound/hda/codecs/realtek/alc269.c | 58 +++++++++++++++++++++++++++++++++++++--
+ sound/hda/controllers/tegra.c     |  3 +-
+ sound/usb/caiaq/device.c          |  3 ++
+ sound/usb/line6/playback.c        |  7 +++++
+ sound/usb/misc/ua101.c            |  9 ++++++
+ sound/usb/mixer_maps.c            |  4 +++
+ sound/usb/quirks.c                | 16 ++++++++---
+ 8 files changed, 95 insertions(+), 9 deletions(-)
+Merging sound-asoc-fixes/for-linus (b2047b8cadadc ASoC: codecs: lpass-wsa-macro: rewrite the interpolator volume after enabling clocks)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc-fixes/for-linus
+Merge made by the 'ort' strategy.
+ sound/soc/amd/acp-config.c                         | 14 +++++++++++
+ sound/soc/amd/acp/Kconfig                          |  1 +
+ sound/soc/amd/acp/amd-acp70-acpi-match.c           | 20 ++++++++++++++++
+ sound/soc/amd/yc/acp6x-mach.c                      | 14 +++++++++++
+ sound/soc/codecs/Kconfig                           |  4 +---
+ sound/soc/codecs/lpass-wsa-macro.c                 |  3 +++
+ sound/soc/codecs/max98363.c                        |  2 +-
+ sound/soc/codecs/rt1017-sdca-sdw.c                 | 27 +++++++---------------
+ sound/soc/codecs/rt274.c                           |  4 +++-
+ sound/soc/codecs/rt286.c                           |  2 +-
+ sound/soc/codecs/wm8903.c                          |  2 +-
+ sound/soc/intel/boards/bytcr_rt5651.c              | 17 ++++++++++++++
+ sound/soc/intel/common/soc-acpi-intel-lnl-match.c  |  1 +
+ .../soc/intel/common/soc-acpi-intel-sdca-quirks.c  | 16 +++++++++++++
+ .../soc/intel/common/soc-acpi-intel-sdca-quirks.h  |  1 +
+ sound/soc/qcom/lpass-cpu.c                         |  4 ++--
+ sound/soc/sdca/sdca_class_function.c               | 19 ++++++++++-----
+ sound/soc/tegra/tegra186_asrc.c                    |  2 +-
+ 18 files changed, 118 insertions(+), 35 deletions(-)
+Merging regmap-fixes/for-linus (2e42cade8ff1f regmap: irq: Free the irqdomain we create)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap-fixes/for-linus
+Merge made by the 'ort' strategy.
+ drivers/base/regmap/regmap-irq.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging regulator-fixes/for-linus (f3e6ef13e24c9 regulator: pf1550: fix which regulator is notified)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator-fixes/for-linus
+Already up to date.
+Merging spi-fixes/for-linus (3d743adf090cd spi: fsl-qspi: Reprogram the clock rate when the operation frequency changes)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi-fixes/for-linus
+Already up to date.
+Merging pci-current/for-linus (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci-current/for-linus
+Already up to date.
+Merging driver-core.current/driver-core-linus (72d3fcf802c45 Linux 7.3-rc5)
+$ git merge -m Merge branch 'driver-core-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core.current/driver-core-linus
+Already up to date.
+Merging tty.current/tty-linus (5dad87615c986 tty: add break_wait kernel-doc)
+$ git merge -m Merge branch 'tty-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty.current/tty-linus
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                            |   5 +
+ drivers/tty/amiserial.c                |   9 +-
+ drivers/tty/n_gsm.c                    |  47 ++-----
+ drivers/tty/serial/8250/8250_bcm7271.c |   2 +-
+ drivers/tty/serial/8250/8250_omap.c    |  15 ++-
+ drivers/tty/serial/8250/8250_port.c    |   2 +-
+ drivers/tty/serial/kgdboc.c            |   9 ++
+ drivers/tty/serial/ma35d1_serial.c     |   6 +-
+ drivers/tty/serial/qcom_geni_serial.c  | 229 ++++++++++++++++++---------------
+ drivers/tty/serial/serial_core.c       |  76 ++++++++---
+ drivers/tty/serial/vt8500_serial.c     |   1 +
+ drivers/tty/tty_io.c                   |  36 ++++--
+ drivers/tty/tty_port.c                 |  91 ++++++++-----
+ drivers/tty/vcc.c                      |   5 +-
+ drivers/tty/vt/vt.c                    |   2 +
+ include/linux/soc/qcom/geni-se.h       |  15 +--
+ include/linux/tty.h                    |   2 +
+ include/linux/tty_port.h               |  22 +---
+ 18 files changed, 333 insertions(+), 241 deletions(-)
+Merging usb.current/usb-linus (abc36cbda29d8 Merge tag 'usb-serial-7.3-rc4' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial into usb-linus)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb.current/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/thunderbolt/ctl.c             |  63 +++++++++--------
+ drivers/thunderbolt/nhi.c             |   5 ++
+ drivers/thunderbolt/switch.c          |   9 ++-
+ drivers/thunderbolt/tb.c              |  37 ++++++----
+ drivers/thunderbolt/test.c            |  58 +++++++++++++---
+ drivers/thunderbolt/tunnel.c          |  73 +++++++++++++-------
+ drivers/thunderbolt/tunnel.h          |   8 ++-
+ drivers/thunderbolt/xdomain.c         |   1 -
+ drivers/usb/cdns3/cdns3-gadget.c      |   4 ++
+ drivers/usb/chipidea/ci_hdrc_tegra.c  |   4 +-
+ drivers/usb/class/cdc-acm.c           |  36 +++++-----
+ drivers/usb/class/cdc-acm.h           |   1 -
+ drivers/usb/core/hub.c                |   7 +-
+ drivers/usb/dwc2/hcd.c                |   3 +-
+ drivers/usb/dwc3/dwc3-am62.c          |   1 +
+ drivers/usb/gadget/function/f_midi2.c |   4 +-
+ drivers/usb/host/octeon-hcd.c         |  54 +++++++++++++--
+ drivers/usb/host/ohci-da8xx.c         |   2 +
+ drivers/usb/host/ohci-s3c2410.c       |   1 +
+ drivers/usb/host/ohci-spear.c         |   1 +
+ drivers/usb/host/ohci-st.c            |   1 +
+ drivers/usb/serial/bus.c              |   6 +-
+ drivers/usb/serial/cp210x.c           |   1 +
+ drivers/usb/serial/generic.c          |  10 ++-
+ drivers/usb/serial/option.c           |  14 ++++
+ drivers/usb/serial/quatech2.c         |  19 ++++-
+ drivers/usb/serial/ssu100.c           |  18 ++++-
+ drivers/usb/serial/usb-serial.c       | 126 +++++++++++++++++++++++++++-------
+ drivers/usb/serial/xr_serial.c        |  10 +--
+ drivers/usb/storage/sierra_ms.c       |   7 ++
+ drivers/usb/typec/anx7411.c           |   5 +-
+ drivers/usb/typec/tcpm/tcpm.c         |   6 +-
+ drivers/usb/typec/tipd/core.c         |  11 +--
+ drivers/usb/typec/ucsi/displayport.c  |   9 ++-
+ drivers/usb/typec/ucsi/ucsi_acpi.c    |  34 +++++++++
+ drivers/usb/typec/ucsi/ucsi_glink.c   |  15 +++-
+ include/linux/thunderbolt.h           |   3 +
+ include/linux/usb/serial.h            |   1 +
+ 38 files changed, 499 insertions(+), 169 deletions(-)
+Merging usb-serial-fixes/usb-linus (ec06546b43b61 USB: serial: quatech2: fix baud rate overflow)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial-fixes/usb-linus
+Already up to date.
+Merging phy/fixes (93f51579e7df2 Linux 7.3-rc4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy/fixes
+Already up to date.
+Merging staging.current/staging-linus (df2908090cda3 Linux 7.3-rc2)
+$ git merge -m Merge branch 'staging-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging.current/staging-linus
+Already up to date.
+Merging iio-fixes/fixes-togreg (d2f2868d2f487 iio: adc: ad_sigma_delta: fix use-after-free on unbind)
+$ git merge -m Merge branch 'fixes-togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio-fixes/fixes-togreg
+Merge made by the 'ort' strategy.
+ .../bindings/iio/adc/rockchip-saradc.yaml          | 18 +++----
+ drivers/android/binder.c                           |  3 +-
+ drivers/android/binder/node.rs                     | 20 +++++--
+ drivers/android/binder/node/wrapper.rs             | 34 +++++++++++-
+ drivers/android/binder/rust_binderfs.c             |  8 +++
+ drivers/android/binder/thread.rs                   |  8 ++-
+ drivers/android/binderfs.c                         |  3 +-
+ drivers/hid/hid-sensor-hub.c                       |  6 ++-
+ drivers/iio/accel/kionix-kx022a.c                  | 30 ++++++++---
+ drivers/iio/accel/kxcjk-1013.c                     |  2 +-
+ drivers/iio/accel/sca3000.c                        |  2 +-
+ drivers/iio/adc/ad4030.c                           | 11 +++-
+ drivers/iio/adc/ad7173.c                           |  4 +-
+ drivers/iio/adc/ad_sigma_delta.c                   | 31 ++++++-----
+ drivers/iio/adc/ade9000.c                          | 36 +++++++------
+ drivers/iio/adc/adi-axi-adc.c                      |  4 ++
+ drivers/iio/adc/aspeed_adc.c                       |  4 +-
+ drivers/iio/adc/axp288_adc.c                       |  8 +++
+ drivers/iio/adc/max1363.c                          |  8 +++
+ drivers/iio/adc/pac1934.c                          |  4 ++
+ drivers/iio/adc/rohm-bd79124.c                     | 20 ++++---
+ drivers/iio/adc/stm32-adc.c                        | 47 +++++++++-------
+ drivers/iio/adc/sun4i-gpadc-iio.c                  | 10 ++--
+ drivers/iio/adc/xilinx-xadc-core.c                 |  9 ++--
+ drivers/iio/buffer/industrialio-buffer-dmaengine.c | 16 ++++--
+ drivers/iio/cdc/ad7150.c                           |  8 +--
+ .../iio/common/inv_sensors/inv_sensors_timestamp.c | 11 ++--
+ drivers/iio/dac/mcp47a1.c                          |  4 +-
+ drivers/iio/dac/rohm-bd79703.c                     |  3 ++
+ drivers/iio/frequency/adf4377.c                    |  4 +-
+ drivers/iio/frequency/admv1013.c                   |  9 ++--
+ drivers/iio/gyro/adis16136.c                       |  2 +-
+ drivers/iio/health/max30102.c                      | 12 +++--
+ drivers/iio/imu/adis16400.c                        |  2 +-
+ drivers/iio/imu/adis16480.c                        |  4 +-
+ drivers/iio/imu/inv_icm42607/inv_icm42607_core.c   | 38 +++++++++----
+ drivers/iio/industrialio-buffer.c                  |  7 ++-
+ drivers/iio/industrialio-trigger.c                 |  2 +
+ drivers/iio/light/gp2ap020a00f.c                   |  2 +
+ drivers/iio/light/rohm-bu27034.c                   | 62 +++++++++++++---------
+ drivers/iio/pressure/bmp280-core.c                 |  2 +-
+ drivers/iio/pressure/rohm-bm1390.c                 |  2 +-
+ drivers/iio/proximity/aw96103.c                    | 30 ++++++++---
+ drivers/iio/proximity/isl29501.c                   |  2 +-
+ drivers/iio/proximity/pulsedlight-lidar-lite-v2.c  |  4 +-
+ drivers/iio/proximity/sx9324.c                     |  2 +-
+ drivers/iio/proximity/vcnl3020.c                   | 24 ++++++---
+ drivers/iio/proximity/vl53l0x-i2c.c                |  3 ++
+ drivers/virt/nitro_enclaves/ne_misc_dev.c          |  1 +
+ include/linux/hid-sensor-hub.h                     |  2 +
+ 50 files changed, 414 insertions(+), 174 deletions(-)
+Merging watchdog-fixes/watchdog (2686e13649a9e watchdog: mediatek: Apply the driver's mode to a watchdog left running)
+$ git merge -m Merge branch 'watchdog' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git watchdog-fixes/watchdog
+Merge made by the 'ort' strategy.
+ drivers/watchdog/mtk_wdt.c | 43 ++++++++++++++++++++++++-------------------
+ 1 file changed, 24 insertions(+), 19 deletions(-)
+Merging counter-current/counter-current (fd73f4a665989 Linux 7.3-rc3)
+$ git merge -m Merge branch 'counter-current' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-current/counter-current
+Already up to date.
+Merging char-misc.current/char-misc-linus (540f55de8c9f7 Merge tag 'icc-7.3-rc5' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/djakov/icc into char-misc-linus)
+$ git merge -m Merge branch 'char-misc-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc.current/char-misc-linus
+Merge made by the 'ort' strategy.
+ drivers/interconnect/qcom/x1e80100.c | 485 -----------------------------------
+ 1 file changed, 485 deletions(-)
+Merging soundwire-fixes/fixes (93f51579e7df2 Linux 7.3-rc4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire-fixes/fixes
+Already up to date.
+Merging thunderbolt-fixes/fixes (395e9f2967a7a thunderbolt: Disable CL states for the Anker Prime TB5 dock)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt-fixes/fixes
+Auto-merging drivers/thunderbolt/stream.c
+Merge made by the 'ort' strategy.
+ drivers/thunderbolt/domain.c   | 26 +-------------------------
+ drivers/thunderbolt/nhi.c      | 26 --------------------------
+ drivers/thunderbolt/nhi.h      | 21 ++-------------------
+ drivers/thunderbolt/nhi_regs.h |  4 ----
+ drivers/thunderbolt/pci.c      | 23 -----------------------
+ drivers/thunderbolt/quirks.c   |  3 +++
+ drivers/thunderbolt/stream.c   |  3 +++
+ drivers/thunderbolt/xdomain.c  |  4 ++++
+ 8 files changed, 13 insertions(+), 97 deletions(-)
+Merging input-current/for-linus (309731e959171 Input: hp_sdc - shut down kicker timer on module exit)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input-current/for-linus
+Already up to date.
+Merging crypto-current/master (10396a2d6d41d crypto: s390/hmac - Generate intermediate CV for API partial block handling)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git crypto-current/master
+Already up to date.
+Merging libcrypto-fixes/libcrypto-fixes (6d996c3b974c5 crypto: aes - Fix undesired override of some optimized AES modes)
+$ git merge -m Merge branch 'libcrypto-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-fixes/libcrypto-fixes
+Merge made by the 'ort' strategy.
+ arch/x86/crypto/aesni-intel_glue.c |  2 +-
+ crypto/aes.c                       | 51 +++++++++++++++++++++++++++++++++++---
+ 2 files changed, 48 insertions(+), 5 deletions(-)
+Merging vfio-fixes/for-linus (e242e974e812e vfio: selftests: Add luuid to libvfio.mk's list of libraries, not to the Makefile)
+$ git merge -m Merge branch 'for-linus' of https://github.com/awilliam/linux-vfio.git vfio-fixes/for-linus
+Already up to date.
+Merging kselftest-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest-fixes/fixes
+Already up to date.
+Merging dmaengine-fixes/fixes (93f51579e7df2 Linux 7.3-rc4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine-fixes/fixes
+Already up to date.
+Merging backlight-fixes/for-backlight-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'for-backlight-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight-fixes/for-backlight-fixes
+Already up to date.
+Merging mtd-fixes/mtd/fixes (1c1a342aceec7 mtd: spinand: Do not update the QE bit on devices without one)
+$ git merge -m Merge branch 'mtd/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd-fixes/mtd/fixes
+Already up to date.
+Merging mfd-fixes/for-mfd-fixes (d5d2d7a8d8be1 MAINTAINERS: Add a mailing list entry to MFD)
+$ git merge -m Merge branch 'for-mfd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd-fixes/for-mfd-fixes
+Already up to date.
+Merging v4l-dvb-fixes/fixes (2579cbe68005f media: em28xx: use video_unregister_device for radio_dev)
+$ git merge -m Merge branch 'fixes' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/media/usb/em28xx/em28xx-video.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging reset-fixes/reset/fixes (71827776667f4 reset: imx7: Correct polarity of MIPI CSI resets on i.MX8MQ)
+$ git merge -m Merge branch 'reset/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux reset-fixes/reset/fixes
+Already up to date.
+Merging mips-fixes/mips-fixes (93f51579e7df2 Linux 7.3-rc4)
+$ git merge -m Merge branch 'mips-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips-fixes/mips-fixes
+Already up to date.
+Merging at91-fixes/at91-fixes (afb1ecfeda3fc ARM: configs: sama5: enable current Microchip KSZ DSA symbols)
+$ git merge -m Merge branch 'at91-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91-fixes/at91-fixes
+Merge made by the 'ort' strategy.
+ arch/arm/configs/sama5_defconfig | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging omap-fixes/fixes (2fabd2f406d0c ARM: dts: ti/omap: dra7: fix PCIe PHY clock divider definition)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/ti/omap/dra7xx-clocks.dtsi | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging tegra-fixes/fixes (5257486664997 Merge branch for-7.3/arm64/dt into fixes)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git tegra-fixes/fixes
+Merge made by the 'ort' strategy.
+ .../boot/dts/nvidia/tegra264-p4071-0000+p3834.dtsi |  13 ++
+ arch/arm64/boot/dts/nvidia/tegra264.dtsi           | 180 ++++++++++++++++++---
+ drivers/soc/tegra/fuse/tegra-apbmisc.c             |   2 +-
+ drivers/soc/tegra/pmc.c                            |   1 +
+ 4 files changed, 171 insertions(+), 25 deletions(-)
+Merging kvm-fixes/master (973ea70393e88 KVM: SEV: Do cache maintenance on the source VM *before* clearing SEV state)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm-fixes/master
+Merge made by the 'ort' strategy.
+ arch/arm64/kvm/vgic/vgic-init.c                    |  12 +-
+ arch/powerpc/kvm/book3s_hv.c                       |   2 -
+ arch/riscv/kvm/aia_device.c                        |   2 +-
+ arch/s390/kvm/s390/s390.c                          |   5 +-
+ arch/x86/kvm/mmu/mmu.c                             |   6 +
+ arch/x86/kvm/svm/sev.c                             |  57 +++---
+ arch/x86/kvm/svm/svm.c                             |  43 ++--
+ arch/x86/kvm/svm/svm.h                             |   1 +
+ arch/x86/kvm/vmx/nested.c                          |   6 +
+ arch/x86/kvm/vmx/tdx.c                             |   5 -
+ include/linux/kvm_host.h                           |   1 -
+ tools/testing/selftests/kvm/Makefile.kvm           |   1 +
+ .../testing/selftests/kvm/x86/nested_x2apic_test.c | 222 +++++++++++++++++++++
+ virt/kvm/kvm_main.c                                |  35 +---
+ 14 files changed, 302 insertions(+), 96 deletions(-)
+ create mode 100644 tools/testing/selftests/kvm/x86/nested_x2apic_test.c
+Merging kvms390-fixes/master (f47190b08b71e s390/uv: Prevent potential out-of-bounds read)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390-fixes/master
+Already up to date.
+Merging kvm-arm-fixes/fixes (afb4334fb52c7 KVM: arm64: selftests: Check a feature hidden in an ID register is UNDEF)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm-fixes/fixes
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Merge made by the 'ort' strategy.
+ arch/arm64/kvm/emulate-nested.c                    |   3 -
+ arch/arm64/kvm/hyp/include/nvhe/pkvm.h             |   9 +
+ arch/arm64/kvm/hyp/nvhe/hyp-main.c                 |   7 +-
+ arch/arm64/kvm/hyp/nvhe/pkvm.c                     |  25 ++-
+ arch/arm64/kvm/vgic/vgic-its.c                     |  12 +-
+ arch/arm64/kvm/vgic/vgic-v2.c                      |   6 +-
+ arch/arm64/kvm/vgic/vgic-v3.c                      |   6 +-
+ arch/arm64/kvm/vgic/vgic.c                         |  21 ++-
+ tools/testing/selftests/kvm/Makefile.kvm           |   1 +
+ .../testing/selftests/kvm/arm64/hidden_features.c  | 184 +++++++++++++++++++++
+ 10 files changed, 246 insertions(+), 28 deletions(-)
+ create mode 100644 tools/testing/selftests/kvm/arm64/hidden_features.c
+Merging hwmon-fixes/hwmon (61406e9cac695 hwmon: (sht4x) Fix jiffies wraparound in heater-ready check)
+$ git merge -m Merge branch 'hwmon' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-fixes/hwmon
+Merge made by the 'ort' strategy.
+ Documentation/hwmon/k10temp.rst |  8 +++-----
+ drivers/hwmon/coretemp.c        |  5 +++++
+ drivers/hwmon/k10temp.c         |  5 ++---
+ drivers/hwmon/lm70.c            |  8 ++++----
+ drivers/hwmon/lm95245.c         |  6 +++---
+ drivers/hwmon/sht4x.c           | 18 +++++++++---------
+ drivers/hwmon/tmp102.c          |  8 ++++----
+ drivers/hwmon/tmp108.c          |  8 ++++----
+ 8 files changed, 34 insertions(+), 32 deletions(-)
+Merging nvdimm-fixes/libnvdimm-fixes (a8aec14230322 nvdimm/bus: Fix potential use after free in asynchronous initialization)
+$ git merge -m Merge branch 'libnvdimm-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm-fixes/libnvdimm-fixes
+Already up to date.
+Merging cxl-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl-fixes/fixes
+Already up to date.
+Merging dma-mapping-fixes/dma-mapping-fixes (057e5e07420c7 iommu/dma: skip swiotlb bounce for DMA_ATTR_MMIO in iommu_dma_map_phys)
+$ git merge -m Merge branch 'dma-mapping-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping-fixes/dma-mapping-fixes
+Merge made by the 'ort' strategy.
+ drivers/iommu/dma-iommu.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging drivers-x86-fixes/fixes (d144a494d81fc MAINTAINERS: fix sysfs-platform-ayaneo-ec documentation path)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86-fixes/fixes
+Already up to date.
+Merging samsung-krzk-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk-fixes/fixes
+Already up to date.
+Merging pinctrl-samsung-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung-fixes/fixes
+Already up to date.
+Merging pinctrl-qcom-fixes/pinctrl-qcom/for-current (19fc4240358be pinctrl: qcom: spmi-gpio: make direction changes exclusive)
+$ git merge -m Merge branch 'pinctrl-qcom/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pinctrl-qcom-fixes/pinctrl-qcom/for-current
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/qcom/pinctrl-hawi.c      |  1 +
+ drivers/pinctrl/qcom/pinctrl-ipq5018.c   |  4 ++--
+ drivers/pinctrl/qcom/pinctrl-maili.c     |  1 +
+ drivers/pinctrl/qcom/pinctrl-nord.c      |  1 +
+ drivers/pinctrl/qcom/pinctrl-qcs8300.c   |  1 +
+ drivers/pinctrl/qcom/pinctrl-spmi-gpio.c | 37 +++++++++++++++++++++++++-------
+ 6 files changed, 35 insertions(+), 10 deletions(-)
+Merging devicetree-fixes/dt/linus (21c89ff1fc86f of/overlay: don't create "//" paths for fragments targeting the root)
+$ git merge -m Merge branch 'dt/linus' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree-fixes/dt/linus
+Merge made by the 'ort' strategy.
+ drivers/of/overlay.c | 17 +++++++++++++----
+ 1 file changed, 13 insertions(+), 4 deletions(-)
+Merging dt-krzk-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk-fixes/fixes
+Already up to date.
+Merging scsi-fixes/fixes (42d1221d321e5 scsi: megaraid_sas: Protect megasas_get_ctrl_info() in megasas_resume())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-fixes/fixes
+Already up to date.
+Merging drm-fixes/drm-fixes (a9ed3aa9b87ee Merge tag 'drm-misc-fixes-2026-09-24' of https://gitlab.freedesktop.org/drm/misc/kernel into drm-fixes)
+$ git merge -m Merge branch 'drm-fixes' of https://gitlab.freedesktop.org/drm/kernel.git drm-fixes/drm-fixes
+Already up to date.
+Merging drm-intel-fixes/for-linux-next-fixes (c034e8a46e4cb drm/i915/vrr: Disable DC balance by default)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/i915/display/intel_display_params.c | 4 ++++
+ drivers/gpu/drm/i915/display/intel_display_params.h | 1 +
+ drivers/gpu/drm/i915/display/intel_vrr.c            | 4 +++-
+ 3 files changed, 8 insertions(+), 1 deletion(-)
+Merging mmc-fixes/fixes (aa0a37b5024d9 memstick: core: wait for request completion before freeing card)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/memstick/core/memstick.c   | 8 ++------
+ drivers/mmc/host/cavium-octeon.c   | 5 ++++-
+ drivers/mmc/host/cavium-thunderx.c | 5 ++++-
+ drivers/mmc/host/mtk-sd.c          | 1 +
+ drivers/mmc/host/sdhci-sprd.c      | 4 ++++
+ 5 files changed, 15 insertions(+), 8 deletions(-)
+Merging rtc-fixes/rtc-fixes (055ef5ce9f67a rtc: spear: initialize IRQ state before requesting alarm IRQ)
+$ git merge -m Merge branch 'rtc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc-fixes/rtc-fixes
+Already up to date.
+Merging gnss-fixes/gnss-linus (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'gnss-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss-fixes/gnss-linus
+Already up to date.
+Merging hyperv-fixes/hyperv-fixes (ca039df94983c PCI: hv: Warn when wait_for_response() waits indefinitely)
+$ git merge -m Merge branch 'hyperv-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv-fixes/hyperv-fixes
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                         |  2 --
+ arch/x86/hyperv/hv_init.c           |  7 +++---
+ drivers/hv/channel.c                |  2 +-
+ drivers/hv/channel_mgmt.c           | 12 ++++++---
+ drivers/hv/connection.c             |  1 -
+ drivers/hv/mshv_vtl_main.c          |  3 +++
+ drivers/hv/vmbus_drv.c              | 19 ++++++--------
+ drivers/pci/controller/pci-hyperv.c | 49 +++++++++++++++++++++++++++++--------
+ tools/hv/hv_get_dns_info.sh         |  2 +-
+ 9 files changed, 64 insertions(+), 33 deletions(-)
+Merging risc-v-fixes/fixes (e111609483333 riscv: vector: Fix output operands in context save)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/riscv/Kconfig.errata                 | 2 +-
+ arch/riscv/Kconfig.socs                   | 1 +
+ arch/riscv/include/asm/vector.h           | 8 ++++----
+ arch/riscv/include/uapi/asm/vendor/mips.h | 4 +++-
+ 4 files changed, 9 insertions(+), 6 deletions(-)
+Merging riscv-dt-fixes/riscv-dt-fixes (0f70fd6f4c1d6 riscv: dts: microchip: beaglev-fire: remove double definition of gpio interrupts)
+$ git merge -m Merge branch 'riscv-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt-fixes/riscv-dt-fixes
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts | 18 ------------------
+ 1 file changed, 18 deletions(-)
+Merging riscv-soc-fixes/riscv-soc-fixes (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'riscv-soc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc-fixes/riscv-soc-fixes
+Already up to date.
+Merging fpga-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga-fixes/fixes
+Already up to date.
+Merging spdx/spdx-linus (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'spdx-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git spdx/spdx-linus
+Already up to date.
+Merging gpio-brgl-fixes/gpio/for-current (ff82fc3a4a1d4 gpio: xilinx: fix runtime PM leak on request error path)
+$ git merge -m Merge branch 'gpio/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl-fixes/gpio/for-current
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-xilinx.c | 9 +--------
+ 1 file changed, 1 insertion(+), 8 deletions(-)
+Merging gpio-intel-fixes/fixes (8d37f20173a27 gpiolib: acpi: Add quirk for Lenovo IdeaPad Slim 3 15ABR8)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-crystalcove.c    |  1 +
+ drivers/gpio/gpiolib-acpi-quirks.c | 13 +++++++++++++
+ 2 files changed, 14 insertions(+)
+Merging pinctrl-intel-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel-fixes/fixes
+Already up to date.
+Merging auxdisplay-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay-fixes/fixes
+Already up to date.
+Merging kunit-fixes/kunit-fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'kunit-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-fixes/kunit-fixes
+Already up to date.
+Merging renesas-fixes/fixes (8dc2615d57020 arm64: dts: renesas: r8a779f0: Set UFS lane count)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas-fixes/fixes
+Already up to date.
+Merging perf-current/perf-tools (aadea57f53288 perf powerpc-vpadtl: Fix raw_size of DTL samples)
+$ git merge -m Merge branch 'perf-tools' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git perf-current/perf-tools
+Already up to date.
+Merging efi-fixes/urgent (d8809f6931065 efi: sysfb_efi: Extend quirk to cover IdeaPad Duet 3 10IGL5-LTE)
+$ git merge -m Merge branch 'urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi-fixes/urgent
+Already up to date.
+Merging battery-fixes/fixes (a58cbc8b36ec0 power: supply: ds2780: Fix ds2780_get_capacity() returning raw value)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/power/supply/ds2780_battery.c   | 2 +-
+ drivers/power/supply/max17042_battery.c | 2 +-
+ include/linux/power_supply.h            | 6 +++---
+ 3 files changed, 5 insertions(+), 5 deletions(-)
+Merging iommufd-fixes/for-rc (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd-fixes/for-rc
+Already up to date.
+Merging rust-fixes/rust-fixes (551c722f40809 Merge tag 'rtc-7.3-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux)
+$ git merge -m Merge branch 'rust-fixes' of https://github.com/Rust-for-Linux/linux.git rust-fixes/rust-fixes
+Already up to date.
+Merging w1-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1-fixes/fixes
+Already up to date.
+Merging pmdomain-fixes/fixes (4c66c423593e2 pmdomain: rockchip: don't ignore clock lookup errors on attach)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/pmdomain/imx/imx8m-blk-ctrl.c  | 15 +++++++++++++++
+ drivers/pmdomain/rockchip/pm-domains.c | 20 +++++++++++++++++---
+ 2 files changed, 32 insertions(+), 3 deletions(-)
+Merging i2c-andi-fixes/i2c/i2c-fixes (840d8acc87925 i2c: xiic: don't clobber msg->len to signal block-read completion)
+$ git merge -m Merge branch 'i2c/i2c-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-andi-fixes/i2c/i2c-fixes
+Merge made by the 'ort' strategy.
+ drivers/i2c/busses/i2c-xiic.c | 61 ++++++++++++++++++++++++++++++++-----------
+ 1 file changed, 46 insertions(+), 15 deletions(-)
+Merging i2c-rust-fixes/rust-i2c-fixes (4eb422482ca5d rust: i2c: fix I2cAdapter refcounts double increment)
+$ git merge -m Merge branch 'rust-i2c-fixes' of https://github.com/ikrtn/rust-for-linux i2c-rust-fixes/rust-i2c-fixes
+Already up to date.
+Merging sparc-fixes/for-linus (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc-fixes/for-linus
+Already up to date.
+Merging clk-fixes/clk-fixes (81493c1dd1b1e Merge tag 'tags/spacemit-clk-fixes-for-7.3-1' into clk-fixes)
+$ git merge -m Merge branch 'clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk-fixes/clk-fixes
+Merge made by the 'ort' strategy.
+ drivers/clk/spacemit/ccu-k3.c | 92 +++++++++++++++++++++++++++++++++++++++++++
+ drivers/clk/ti/composite.c    | 26 ++++++------
+ 2 files changed, 104 insertions(+), 14 deletions(-)
+Merging thead-clk-fixes/thead-clk-fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'thead-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk-fixes/thead-clk-fixes
+Already up to date.
+Merging tenstorrent-clk-fixes/tenstorrent-clk-fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'tenstorrent-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk-fixes/tenstorrent-clk-fixes
+Already up to date.
+Merging fustini-config-fixes/riscv-config-fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'riscv-config-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git fustini-config-fixes/riscv-config-fixes
+Already up to date.
+Merging pwrseq-fixes/pwrseq/for-current (58a0033024815 power: sequencing: pcie-m2: Add Lenovo ThinkPad T14s gen6 WCN7850 subsystem PCI ids)
+$ git merge -m Merge branch 'pwrseq/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq-fixes/pwrseq/for-current
+Merge made by the 'ort' strategy.
+ drivers/power/sequencing/pwrseq-pcie-m2.c | 2 ++
+ 1 file changed, 2 insertions(+)
+Merging thead-dt-fixes/thead-dt-fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'thead-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt-fixes/thead-dt-fixes
+Already up to date.
+Merging ftrace-fixes/ftrace/fixes (1650a1b6cb1ae fgraph: Check ftrace_pids_enabled on registration for early filtering)
+$ git merge -m Merge branch 'ftrace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace-fixes/ftrace/fixes
+Already up to date.
+Merging ring-buffer-fixes/ring-buffer/fixes (057caace5214d tracing: Create output file from cmd_check_undefined)
+$ git merge -m Merge branch 'ring-buffer/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ring-buffer-fixes/ring-buffer/fixes
+Already up to date.
+Merging trace-fixes/trace/fixes (d860c67c05168 ring-buffer: Check resize_disabled before publishing the new subbuf order)
+$ git merge -m Merge branch 'trace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git trace-fixes/trace/fixes
+Already up to date.
+Merging tracefs-fixes/tracefs/fixes (07004a8c4b572 eventfs: Hold eventfs_mutex and SRCU when remount walks events)
+$ git merge -m Merge branch 'tracefs/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git tracefs-fixes/tracefs/fixes
+Already up to date.
+Merging spacemit-fixes/fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux spacemit-fixes/fixes
+Already up to date.
+Merging tip-fixes/tip/urgent (3ebb3531c6d0d Merge branch into tip/master: 'x86/mm')
+$ git merge -m Merge branch 'tip/urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip-fixes/tip/urgent
+Merge made by the 'ort' strategy.
+ arch/x86/include/asm/pgtable.h         |  6 +++++-
+ arch/x86/include/uapi/asm/ptrace-abi.h |  4 ++--
+ arch/x86/kernel/sys_x86_64.c           | 15 ++++++++++----
+ arch/x86/mm/pat/set_memory.c           | 23 ++++++++++++++++----
+ arch/x86/mm/pgtable.c                  | 38 +++++++++++++---------------------
+ arch/x86/um/asm/ptrace.h               |  4 +---
+ arch/x86/um/ptrace_32.c                |  1 +
+ include/linux/hrtimer_rearm.h          |  2 +-
+ 8 files changed, 54 insertions(+), 39 deletions(-)
+Merging kexec-fixes/kexec-fixes (a901b0778ae82 Merge patch series "kexec: fix probe error codes and error propagation")
+$ git merge -m Merge branch 'kexec-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git kexec-fixes/kexec-fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/kernel/kexec_image.c   |  4 ++--
+ arch/loongarch/kernel/kexec_efi.c |  4 ++--
+ arch/riscv/kernel/kexec_image.c   |  4 ++--
+ kernel/kexec_elf.c                |  4 ++--
+ kernel/kexec_file.c               | 12 +++++++-----
+ 5 files changed, 15 insertions(+), 13 deletions(-)
+Merging liveupdate-fixes/fixes (3a0b8fa2eb36a kho: fix size calculation in kho_preserved_memory_reserve())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git liveupdate-fixes/fixes
+Already up to date.
+Merging drm-msm-fixes/msm-fixes (a15fac810c763 dt-bindings: display/msm: Use consistent indentation in the example)
+$ git merge -m Merge branch 'msm-fixes' of https://gitlab.freedesktop.org/drm/msm.git drm-msm-fixes/msm-fixes
+Already up to date.
+Merging uml-fixes/fixes (af421e9aed392 um: vector: fix use-after-free in vector_mmsg_rx())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml-fixes/fixes
+Already up to date.
+Merging fwctl-fixes/for-rc (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl-fixes/for-rc
+Already up to date.
+Merging devsec-tsm-fixes/fixes (c3fd16c3b98ed virt: tdx-guest: Fix handling of host controlled 'quote' buffer length)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm-fixes/fixes
+Already up to date.
+Merging drm-rust-fixes/for-linux-next-fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-fixes/for-linux-next-fixes
+Already up to date.
+Merging tenstorrent-dt-fixes/tenstorrent-dt-fixes (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'tenstorrent-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt-fixes/tenstorrent-dt-fixes
+Already up to date.
+Merging nfc-fixes/for-linus (b61732f47316d nfc: pn533: fix OOB read in pn533_acr122_is_rx_frame_valid())
+$ git merge -m Merge branch 'for-linus' of https://codeberg.org/linux-nfc/linux.git nfc-fixes/for-linus
+Already up to date.
+Merging mm-nonmm-hotfixes-stable/mm-nonmm-hotfixes-stable (a243ede718463 Merge tag 'mtd/fixes-for-7.3-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux)
+$ git merge -m Merge branch 'mm-nonmm-hotfixes-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-hotfixes-stable/mm-nonmm-hotfixes-stable
+Already up to date.
+Merging mm-nonmm-hotfixes-unstable/mm-nonmm-hotfixes-unstable (844c67369f72d fault-inject: fix dentry leak)
+$ git merge -m Merge branch 'mm-nonmm-hotfixes-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-hotfixes-unstable/mm-nonmm-hotfixes-unstable
+Merge made by the 'ort' strategy.
+ drivers/infiniband/hw/hfi1/fault.c |  1 -
+ fs/fat/fat.h                       |  2 +-
+ fs/fat/fatent.c                    | 21 ++++++++++++++++++---
+ fs/fat/file.c                      |  3 ++-
+ fs/fat/misc.c                      |  6 ++----
+ include/linux/fault-inject.h       | 10 ++++++++--
+ kernel/resource.c                  |  2 +-
+ lib/fault-inject.c                 |  7 +++++--
+ 8 files changed, 37 insertions(+), 15 deletions(-)
+Merging drm-misc-fixes/for-linux-next-fixes (e78a9fb7a40c5 drm/vmwgfx: Add blend mode property)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/bridge/th1520-dw-hdmi.c      |  8 ++++----
+ drivers/gpu/drm/vc4/vc4_drv.c                |  5 ++++-
+ drivers/gpu/drm/vc4/vc4_v3d.c                |  8 ++++++++
+ drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c | 26 +++++++++++++++++++++-----
+ drivers/gpu/drm/vmwgfx/vmwgfx_kms.c          |  8 ++++++++
+ 5 files changed, 45 insertions(+), 10 deletions(-)
+Merging rust/rust-next (72d3fcf802c45 Linux 7.3-rc5)
+$ git merge -m Merge branch 'rust-next' of https://github.com/Rust-for-Linux/linux.git rust/rust-next
+Already up to date.
+Merging rust-interop/interop-next (05f7e89ab9731 Linux 6.19)
+$ git merge -m Merge branch 'interop-next' of https://github.com/Rust-for-Linux/linux.git rust-interop/interop-next
+Already up to date.
+Merging rust-alloc/alloc-next (6e8339118040f rust: alloc: add `NumaNode::id()` accessor)
+$ git merge -m Merge branch 'alloc-next' of https://github.com/Rust-for-Linux/linux.git rust-alloc/alloc-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS               | 1 +
+ rust/kernel/alloc.rs      | 6 ++++++
+ rust/kernel/alloc/kbox.rs | 7 ++++---
+ 3 files changed, 11 insertions(+), 3 deletions(-)
+Merging rust-io/io-next (86731a2a651e5 Linux 6.16-rc3)
+$ git merge -m Merge branch 'io-next' of https://github.com/Rust-for-Linux/linux.git rust-io/io-next
+Already up to date.
+Merging rust-pin-init/pin-init-next (95593ed361619 rust: pin-init: internal: improve diagnostics robustness against panicking)
+$ git merge -m Merge branch 'pin-init-next' of https://github.com/Rust-for-Linux/linux.git rust-pin-init/pin-init-next
+Merge made by the 'ort' strategy.
+ rust/pin-init/internal/src/diagnostics.rs | 169 +++++++--
+ rust/pin-init/internal/src/init.rs        | 546 ++++++++++++++++++++++++------
+ rust/pin-init/internal/src/lib.rs         |  40 ++-
+ rust/pin-init/internal/src/pin_data.rs    | 164 +++++----
+ rust/pin-init/internal/src/util.rs        |  54 +++
+ rust/pin-init/src/__internal.rs           |  64 ++--
+ rust/pin-init/src/lib.rs                  |  82 ++++-
+ 7 files changed, 850 insertions(+), 269 deletions(-)
+ create mode 100644 rust/pin-init/internal/src/util.rs
+Merging rust-timekeeping/timekeeping-next (2ea0119f72dba rust: hrtimer: document handle based design rationale)
+$ git merge -m Merge branch 'timekeeping-next' of https://github.com/Rust-for-Linux/linux.git rust-timekeeping/timekeeping-next
+Auto-merging rust/bindings/lib.rs
+Merge made by the 'ort' strategy.
+ rust/bindings/lib.rs        |   5 ++
+ rust/helpers/helpers.c      |   1 +
+ rust/helpers/math.c         |   8 +++
+ rust/kernel/Kconfig.test    |  10 ++++
+ rust/kernel/time.rs         | 139 ++++++++++++++++++++++++++++++++++++++------
+ rust/kernel/time/delay.rs   |  17 ++++--
+ rust/kernel/time/hrtimer.rs |  93 ++++++++++++++++-------------
+ 7 files changed, 210 insertions(+), 63 deletions(-)
+ create mode 100644 rust/helpers/math.c
+Merging rust-xarray/xarray-next (c455f19bbe610 rust: xarray: add __rust_helper to helpers)
+$ git merge -m Merge branch 'xarray-next' of https://github.com/Rust-for-Linux/linux.git rust-xarray/xarray-next
+Already up to date.
+Merging rust-analyzer/rust-analyzer-next (5f45afb8ab04d scripts: generate_rust_analyzer.py: pass cfg to macros crate)
+$ git merge -m Merge branch 'rust-analyzer-next' of https://github.com/Rust-for-Linux/linux.git rust-analyzer/rust-analyzer-next
+Auto-merging scripts/generate_rust_analyzer.py
+Merge made by the 'ort' strategy.
+ scripts/generate_rust_analyzer.py | 1 +
+ 1 file changed, 1 insertion(+)
+Merging mm/for-next (7a6c9825d917f Merge https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-unstable into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mm/linux.git mm/for-next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/mm/mmu.c
+Auto-merging arch/x86/include/asm/pgtable.h
+Auto-merging arch/x86/mm/pat/set_memory.c
+Auto-merging drivers/usb/cdns3/cdns3-gadget.c
+Merge made by the 'ort' strategy.
+ .clang-format                                      |   2 +-
+ Documentation/ABI/testing/sysfs-kernel-mm-damon    |  29 +-
+ Documentation/admin-guide/cgroup-v2.rst            |   4 +
+ Documentation/admin-guide/kdump/vmcoreinfo.rst     |   2 +-
+ Documentation/admin-guide/kernel-parameters.txt    |   3 +
+ Documentation/admin-guide/mm/damon/usage.rst       |  26 +-
+ Documentation/admin-guide/mm/ksm.rst               |   8 +-
+ Documentation/arch/powerpc/vmemmap_dedup.rst       |  88 +-
+ Documentation/core-api/memory-allocation.rst       |  32 +-
+ Documentation/core-api/mm-api.rst                  |   9 +
+ Documentation/core-api/pin_user_pages.rst          |   9 -
+ Documentation/dev-tools/kmemleak.rst               |  24 +-
+ Documentation/filesystems/mmap_prepare.rst         |  81 ++
+ Documentation/filesystems/vfs.rst                  |   6 +-
+ Documentation/mm/damon/design.rst                  |  49 +-
+ Documentation/mm/damon/maintainer-profile.rst      |  19 +-
+ Documentation/mm/index.rst                         |   1 +
+ Documentation/mm/kernel-page-tables.rst            | 410 ++++++++++
+ Documentation/mm/ksm.rst                           |   2 +-
+ Documentation/mm/page_owner.rst                    |   8 +-
+ Documentation/mm/physical_memory.rst               |   6 +
+ Documentation/mm/process_addrs.rst                 |  11 +
+ Documentation/mm/vmemmap_dedup.rst                 |  32 +-
+ MAINTAINERS                                        |  12 +-
+ arch/Kconfig                                       |   8 -
+ arch/alpha/Kconfig                                 |   1 -
+ arch/alpha/include/asm/pgtable.h                   |   7 -
+ arch/arc/include/asm/pgalloc.h                     |   6 +-
+ arch/arc/include/asm/pgtable-levels.h              |  11 -
+ arch/arm/Kconfig                                   |   2 -
+ arch/arm/Kconfig.debug                             |   2 +-
+ arch/arm/configs/aspeed_g4_defconfig               |   2 +-
+ arch/arm/configs/aspeed_g5_defconfig               |   2 +-
+ arch/arm/configs/shmobile_defconfig                |   2 +-
+ arch/arm/include/asm/pgtable.h                     |   7 -
+ arch/arm/include/asm/ptdump.h                      |   6 +-
+ arch/arm/kernel/traps.c                            |  17 -
+ arch/arm/mm/init.c                                 |   2 +-
+ arch/arm64/Kconfig                                 |   4 +-
+ arch/arm64/include/asm/mte.h                       |   2 -
+ arch/arm64/include/asm/pgtable.h                   |  64 +-
+ arch/arm64/include/asm/set_memory.h                |   5 +-
+ arch/arm64/kvm/mmu.c                               |   4 +-
+ arch/arm64/mm/fault.c                              |  16 +-
+ arch/arm64/mm/mmu.c                                |  48 ++
+ arch/arm64/mm/mteswap.c                            |  32 +-
+ arch/arm64/mm/pageattr.c                           |  24 +-
+ arch/csky/include/asm/pgtable.h                    |   4 -
+ arch/hexagon/include/asm/pgtable.h                 |   3 -
+ arch/loongarch/Kconfig                             |   2 -
+ arch/loongarch/include/asm/pgtable.h               |  14 +-
+ arch/loongarch/include/asm/set_memory.h            |   5 +-
+ arch/loongarch/mm/init.c                           |   4 +-
+ arch/loongarch/mm/pageattr.c                       |  27 +-
+ arch/m68k/Kconfig                                  |   1 +
+ arch/m68k/include/asm/mcf_pgalloc.h                |   5 +-
+ arch/m68k/include/asm/mcf_pgtable.h                |   6 -
+ arch/m68k/include/asm/motorola_pgalloc.h           |   9 +-
+ arch/m68k/include/asm/motorola_pgtable.h           |   8 -
+ arch/m68k/include/asm/sun3_pgtable.h               |   7 -
+ arch/m68k/mm/motorola.c                            | 119 ++-
+ arch/microblaze/include/asm/pgalloc.h              |   2 +-
+ arch/microblaze/include/asm/pgtable.h              |   7 -
+ arch/mips/Kconfig                                  |   1 -
+ arch/mips/include/asm/pgtable-32.h                 |  10 -
+ arch/mips/include/asm/pgtable-64.h                 |  13 -
+ arch/nios2/include/asm/pgtable.h                   |   7 -
+ arch/openrisc/include/asm/pgtable.h                |   7 -
+ arch/parisc/Kconfig                                |   1 -
+ arch/parisc/include/asm/pgtable.h                  |   9 -
+ arch/parisc/kernel/pci-dma.c                       |   6 +-
+ arch/powerpc/Kconfig                               |   3 +-
+ arch/powerpc/configs/ppc64_defconfig               |   2 +-
+ arch/powerpc/include/asm/book3s/32/pgtable.h       |   2 -
+ arch/powerpc/include/asm/book3s/64/pgtable.h       |   7 -
+ arch/powerpc/include/asm/nohash/32/pgtable.h       |   2 -
+ arch/powerpc/include/asm/nohash/64/pgtable-4k.h    |   3 -
+ arch/powerpc/include/asm/nohash/64/pgtable.h       |   5 -
+ arch/powerpc/mm/book3s64/radix_pgtable.c           | 124 +--
+ arch/powerpc/mm/book3s64/radix_tlb.c               |   6 +-
+ arch/powerpc/mm/nohash/e500_hugetlbpage.c          |   2 +-
+ arch/powerpc/mm/nohash/tlb.c                       |   2 +-
+ arch/powerpc/mm/ptdump/ptdump.c                    |   2 +-
+ arch/powerpc/platforms/powernv/Kconfig             |   1 -
+ arch/powerpc/platforms/pseries/Kconfig             |   1 -
+ arch/riscv/Kconfig                                 |   4 +-
+ arch/riscv/include/asm/page.h                      |   6 -
+ arch/riscv/include/asm/pgtable-64.h                |   9 -
+ arch/riscv/include/asm/pgtable.h                   |   4 -
+ arch/riscv/include/asm/set_memory.h                |   5 +-
+ arch/riscv/kvm/mmu.c                               |   2 +-
+ arch/riscv/mm/init.c                               |   1 +
+ arch/riscv/mm/pageattr.c                           |  23 +-
+ arch/riscv/mm/tlbflush.c                           |   2 +-
+ arch/s390/Kconfig                                  |   4 +-
+ arch/s390/configs/debug_defconfig                  |   2 +-
+ arch/s390/configs/defconfig                        |   2 +-
+ arch/s390/include/asm/pgtable.h                    |  11 -
+ arch/s390/include/asm/set_memory.h                 |   5 +-
+ arch/s390/mm/dump_pagetables.c                     |   2 +-
+ arch/s390/mm/gmap_helpers.c                        |   6 +-
+ arch/s390/mm/pageattr.c                            |  20 +-
+ arch/sh/Kconfig                                    |   1 +
+ arch/sh/include/asm/pgalloc.h                      |   6 +-
+ arch/sh/include/asm/pgtable-3level.h               |   3 -
+ arch/sh/include/asm/pgtable_32.h                   |  13 -
+ arch/sh/mm/init.c                                  |  16 +-
+ arch/sh/mm/pgtable.c                               |  20 +
+ arch/sparc/Kconfig                                 |   4 +-
+ arch/sparc/include/asm/pgalloc_32.h                |   7 +-
+ arch/sparc/include/asm/pgalloc_64.h                |   8 -
+ arch/sparc/include/asm/pgtable_32.h                |   3 -
+ arch/sparc/include/asm/pgtable_64.h                |  10 -
+ arch/sparc/include/asm/tlb_64.h                    |   2 -
+ arch/sparc/lib/bitext.c                            |  14 +-
+ arch/sparc/mm/init_64.c                            |   2 +-
+ arch/sparc/mm/srmmu.c                              |  32 +-
+ arch/um/Kconfig                                    |   1 -
+ arch/um/include/asm/pgtable-2level.h               |   7 -
+ arch/um/include/asm/pgtable-4level.h               |  13 -
+ arch/x86/Kconfig                                   |   6 +-
+ arch/x86/configs/x86_64_defconfig                  |   2 +-
+ arch/x86/entry/vdso/vdso32/fake_32bit_build.h      |   2 +-
+ arch/x86/events/intel/bts.c                        |   3 -
+ arch/x86/events/intel/pt.c                         |   6 +-
+ arch/x86/include/asm/pgtable-2level.h              |   5 -
+ arch/x86/include/asm/pgtable-3level.h              |  11 -
+ arch/x86/include/asm/pgtable.h                     |   6 +-
+ arch/x86/include/asm/pgtable_64.h                  |  18 -
+ arch/x86/include/asm/set_memory.h                  |   5 +-
+ arch/x86/include/asm/string_64.h                   |  85 +-
+ arch/x86/kernel/uprobes.c                          |   2 +-
+ arch/x86/mm/pat/set_memory.c                       |  20 +-
+ arch/x86/mm/pti.c                                  |   2 +-
+ arch/xtensa/include/asm/pgtable.h                  |   4 -
+ arch/xtensa/include/asm/tlb.h                      |   2 +-
+ drivers/android/binder/page_range.rs               |  19 +-
+ drivers/android/binder_alloc.c                     |  63 +-
+ drivers/base/memory.c                              |   9 +-
+ drivers/block/zram/zcomp.c                         |  23 +-
+ drivers/block/zram/zcomp.h                         |   4 +-
+ drivers/block/zram/zram_drv.c                      |  82 +-
+ drivers/char/Makefile                              |   2 +-
+ drivers/gpu/drm/drm_gpusvm.c                       |   5 +-
+ drivers/gpu/drm/sti/sti_cursor.c                   |   4 +-
+ drivers/gpu/drm/sti/sti_hqvdp.c                    |   2 +-
+ drivers/gpu/ipu-v3/ipu-image-convert.c             |   2 +-
+ drivers/hid/hid-core.c                             |   4 +-
+ drivers/hsi/clients/cmt_speech.c                   |  35 +-
+ drivers/infiniband/hw/hfi1/file_ops.c              |  84 +-
+ drivers/md/md-bitmap.c                             |  18 +-
+ drivers/media/platform/nxp/imx7-media-csi.c        |   2 +-
+ .../media/platform/nxp/imx8-isi/imx8-isi-video.c   |   2 +-
+ drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c         |   2 +-
+ drivers/nvdimm/pmem.c                              |   2 +-
+ drivers/nvdimm/pmem.h                              |  12 -
+ drivers/scsi/sg.c                                  | 115 ++-
+ drivers/spi/spi-atmel.c                            |   4 +-
+ drivers/spi/spi-ti-qspi.c                          |   2 +-
+ drivers/staging/media/imx/imx-media-utils.c        |   2 +-
+ drivers/usb/cdns3/cdns3-gadget.c                   |   2 +-
+ drivers/usb/gadget/udc/cdns2/cdns2-gadget.c        |   2 +-
+ drivers/usb/gadget/udc/lpc32xx_udc.c               |   2 +-
+ drivers/usb/mon/mon_bin.c                          |  98 ++-
+ drivers/video/fbdev/core/fb_defio.c                |   6 +-
+ drivers/video/fbdev/fsl-diu-fb.c                   |   2 +-
+ drivers/video/fbdev/ssd1307fb.c                    |   2 +
+ drivers/xen/grant-table.c                          |  11 +-
+ fs/Kconfig                                         |   2 +-
+ fs/buffer.c                                        |   8 -
+ fs/ceph/addr.c                                     |   8 +-
+ fs/coredump.c                                      |   6 +-
+ fs/crypto/crypto.c                                 |   2 -
+ fs/erofs/data.c                                    |  16 +-
+ fs/erofs/zdata.c                                   |  13 +-
+ fs/f2fs/compress.c                                 |  35 +-
+ fs/f2fs/data.c                                     |   2 +-
+ fs/f2fs/f2fs.h                                     |  99 +--
+ fs/f2fs/segment.c                                  |   2 +-
+ fs/fuse/dax.c                                      |   2 +-
+ fs/hugetlbfs/inode.c                               |  10 +-
+ fs/iomap/buffered-io.c                             |   3 +-
+ fs/nfs/file.c                                      |   4 +-
+ fs/nfs/write.c                                     |   2 -
+ fs/proc/base.c                                     |   3 +-
+ fs/proc/internal.h                                 |   2 -
+ fs/proc/page.c                                     |   8 +-
+ fs/proc/task_mmu.c                                 | 448 ++++-------
+ fs/proc/vmcore.c                                   |  27 +-
+ fs/ubifs/file.c                                    |   8 +-
+ fs/xfs/libxfs/xfs_btree.c                          |  18 +-
+ fs/xfs/xfs_platform.h                              |   4 -
+ include/asm-generic/pgtable-nop4d.h                |   1 -
+ include/asm-generic/pgtable-nopmd.h                |   1 -
+ include/asm-generic/pgtable-nopud.h                |   1 -
+ include/asm-generic/tlb.h                          |  70 +-
+ include/linux/buffer_head.h                        |   8 +-
+ include/linux/cache.h                              |   1 +
+ include/linux/cgroup.h                             |   3 +
+ include/linux/compaction.h                         |  11 +-
+ include/linux/compiler.h                           |   5 +
+ include/linux/damon.h                              |  98 ++-
+ include/linux/gfp_types.h                          |   2 +-
+ include/linux/huge_mm.h                            |   9 -
+ include/linux/hugetlb.h                            |  50 +-
+ include/linux/hugetlb_inline.h                     |  28 -
+ include/linux/idr.h                                |   5 +-
+ include/linux/kernel-page-flags.h                  |   1 -
+ include/linux/maple_tree.h                         |   4 +-
+ include/linux/memblock.h                           |   1 -
+ include/linux/memcontrol.h                         | 464 ++++++-----
+ include/linux/mm.h                                 | 369 +++++++--
+ include/linux/mm_inline.h                          | 115 ++-
+ include/linux/mm_types.h                           |  66 +-
+ include/linux/mmap_lock.h                          | 106 ++-
+ include/linux/mmzone.h                             | 224 +++---
+ include/linux/page-flags.h                         |  64 +-
+ include/linux/page_counter.h                       |   3 +-
+ include/linux/pagemap.h                            |  68 +-
+ include/linux/pgtable.h                            |  24 +-
+ include/linux/ptdump.h                             |   4 +-
+ include/linux/rmap.h                               |   2 +-
+ include/linux/sched.h                              |   4 +-
+ include/linux/set_memory.h                         | 120 ++-
+ include/linux/shmem_fs.h                           |  12 +-
+ include/linux/string.h                             |  13 +
+ include/linux/swap.h                               |  59 +-
+ include/linux/swapops.h                            |   9 -
+ include/linux/userfaultfd_k.h                      |   1 -
+ include/linux/vmalloc.h                            |   4 +
+ include/linux/vmemmap-optimization.h               | 115 +++
+ include/linux/zsmalloc.h                           |   4 -
+ include/linux/zswap.h                              |   8 +-
+ include/net/mana/mana.h                            |   4 +-
+ include/trace/events/huge_memory.h                 |  18 +-
+ include/trace/events/mmflags.h                     |  45 +-
+ include/trace/events/pagemap.h                     |   2 +-
+ include/trace/events/vmscan.h                      |  14 -
+ init/main.c                                        |   2 +-
+ kernel/bpf/arena.c                                 |   3 +-
+ kernel/bpf/stackmap.c                              |  17 +-
+ kernel/bpf/task_iter.c                             |   2 +-
+ kernel/cgroup/cgroup-internal.h                    |   1 -
+ kernel/configs/debug.config                        |   2 +-
+ kernel/events/core.c                               |   2 +-
+ kernel/events/ring_buffer.c                        |   7 +-
+ kernel/events/uprobes.c                            |   4 +-
+ kernel/fork.c                                      |   2 -
+ kernel/power/snapshot.c                            |   4 +-
+ kernel/sched/fair.c                                |   3 +-
+ kernel/vmcore_info.c                               |   1 -
+ lib/interval_tree_test.c                           |   2 +-
+ lib/region_alloc_benchmark.c                       |   8 +-
+ lib/test_hmm.c                                     |   2 +
+ mm/Kconfig                                         |  82 +-
+ mm/Kconfig.debug                                   |  40 -
+ mm/Makefile                                        |   3 +-
+ mm/alloc_tag.c                                     |   2 +-
+ drivers/char/mem.c => mm/char-mem.c                |  21 +-
+ mm/cma.h                                           |  23 +-
+ mm/collapse.h                                      | 162 ++++
+ mm/compaction.c                                    |  11 +-
+ mm/damon/core.c                                    | 392 ++++++---
+ mm/damon/lru_sort.c                                |  12 +-
+ mm/damon/ops-common.c                              | 195 ++++-
+ mm/damon/ops-common.h                              |  12 +
+ mm/damon/paddr.c                                   |  83 +-
+ mm/damon/reclaim.c                                 |   8 -
+ mm/damon/sysfs-schemes.c                           |  51 +-
+ mm/damon/sysfs.c                                   | 364 ++++++++-
+ mm/damon/tests/core-kunit.h                        | 883 ++++++++++++++++++++-
+ mm/damon/vaddr.c                                   | 299 +++++--
+ mm/debug.c                                         |   4 -
+ mm/execmem.c                                       | 148 ++--
+ mm/filemap.c                                       |  34 +-
+ mm/folio.c                                         |  21 +-
+ mm/gup.c                                           | 222 +++---
+ mm/gup_test.c                                      |  11 +-
+ mm/hmm.c                                           |   3 +-
+ mm/huge_memory.c                                   | 757 +++++++++---------
+ mm/hugetlb.c                                       | 303 ++++---
+ mm/hugetlb_cgroup.c                                |  33 +
+ mm/hugetlb_sysfs.c                                 |  10 +-
+ mm/hugetlb_vmemmap.c                               | 130 +--
+ mm/hugetlb_vmemmap.h                               |  15 +-
+ mm/init-mm.c                                       |   2 -
+ mm/internal.h                                      | 140 ++--
+ mm/interval_tree.c                                 |  33 +
+ mm/kfence/core.c                                   |   2 +-
+ mm/khugepaged.c                                    | 690 ++++++++--------
+ mm/kmemleak.c                                      |  72 +-
+ mm/ksm.c                                           |  27 +-
+ mm/list_lru.c                                      |  13 +-
+ mm/madvise.c                                       | 231 +++++-
+ mm/memblock.c                                      |  27 +-
+ mm/memcontrol-v1.c                                 | 413 +---------
+ mm/memcontrol-v1.h                                 |  31 +-
+ mm/memcontrol.c                                    | 648 ++++++++++-----
+ mm/memfd.c                                         |  31 +-
+ mm/memory-failure.c                                |  31 +-
+ mm/memory-tiers.c                                  |   7 +-
+ mm/memory.c                                        | 266 ++++---
+ mm/memory_hotplug.c                                | 149 ++--
+ mm/mempolicy.c                                     | 191 +++--
+ mm/migrate.c                                       |  21 +-
+ mm/migrate_device.c                                |  18 +-
+ mm/mincore.c                                       |  31 +-
+ mm/mlock.c                                         |  61 +-
+ mm/mm_init.c                                       | 243 +++---
+ mm/mm_init.h                                       |   7 +-
+ mm/mmap.c                                          |   8 +-
+ mm/mmap_lock.c                                     |  61 +-
+ mm/mmu_gather.c                                    |  32 +-
+ mm/mprotect.c                                      |  11 +-
+ mm/mremap.c                                        |  21 +-
+ mm/nommu.c                                         |   6 +-
+ mm/oom_kill.c                                      |  15 +-
+ mm/page-writeback.c                                |   2 +-
+ mm/page_alloc.c                                    | 121 ++-
+ mm/page_alloc.h                                    |   2 +-
+ mm/page_counter.c                                  |  41 +-
+ mm/page_io.c                                       |  59 +-
+ mm/page_isolation.c                                |  40 +-
+ mm/page_owner.c                                    |   6 +-
+ mm/page_table_check.c                              |   6 +-
+ mm/page_vma_mapped.c                               |  11 +-
+ mm/pagewalk.c                                      |   4 +-
+ mm/percpu.c                                        |   9 +-
+ mm/pgtable-generic.c                               |  37 +-
+ mm/rmap.c                                          |  37 +-
+ mm/secretmem.c                                     |   8 +-
+ mm/shmem.c                                         | 418 +++++++---
+ mm/slab.h                                          |  28 +-
+ mm/slab_common.c                                   |   2 +-
+ mm/slub.c                                          | 330 +++++---
+ mm/sparse-vmemmap.c                                | 399 +++++-----
+ mm/sparse.c                                        | 236 ++----
+ mm/sparse.h                                        |  32 +-
+ mm/swap.h                                          |  17 +-
+ mm/swap_state.c                                    |  82 +-
+ mm/swapfile.c                                      | 151 ++--
+ mm/truncate.c                                      | 131 +--
+ mm/userfaultfd.c                                   | 120 +--
+ mm/util.c                                          |  31 +-
+ mm/vma.c                                           | 440 +++++++---
+ mm/vma.h                                           |  65 +-
+ mm/vma_internal.h                                  |   1 -
+ mm/vmalloc.c                                       | 231 +++---
+ mm/vmalloc.h                                       |   2 +-
+ mm/vmpressure.c                                    |   3 -
+ mm/vmscan.c                                        | 629 +++++++++------
+ mm/vmstat.c                                        |  17 +-
+ mm/workingset.c                                    |   7 +-
+ mm/zpdesc.h                                        |   2 +-
+ mm/zsmalloc.c                                      | 101 +--
+ mm/zswap.c                                         | 363 +++++----
+ net/ipv4/tcp.c                                     |  31 +-
+ rust/kernel/mm.rs                                  |  58 +-
+ samples/damon/mtier.c                              |   8 +-
+ samples/damon/prcl.c                               |   5 +-
+ samples/damon/wsse.c                               |   5 +-
+ scripts/gdb/linux/mm.py                            |  22 +-
+ security/selinux/selinuxfs.c                       |  11 +-
+ sound/core/pcm_native.c                            |  48 +-
+ sound/mips/snd-n64.c                               |   2 +-
+ tools/include/linux/compiler.h                     |   5 +
+ tools/include/linux/mm.h                           |   6 +-
+ tools/lib/mm/file_utils.c                          | 106 +++
+ tools/lib/mm/file_utils.h                          |  13 +
+ .../selftests => lib}/mm/hugepage_settings.c       | 186 ++++-
+ .../selftests => lib}/mm/hugepage_settings.h       |  12 +
+ tools/mm/.gitignore                                |   1 +
+ tools/mm/Makefile                                  |  11 +-
+ .../selftests/mm/gup_test.c => mm/gup_bench.c}     | 192 ++---
+ tools/mm/page-types.c                              |   2 -
+ tools/mm/page_owner_sort.c                         | 134 +++-
+ tools/sched_ext/include/scx/common.bpf.h           |   2 -
+ tools/testing/memblock/Makefile                    |   3 +-
+ tools/testing/memblock/README                      |  12 +-
+ tools/testing/memblock/TODO                        |   5 -
+ tools/testing/memblock/asm/dma.h                   |   6 +
+ tools/testing/memblock/main.c                      |   2 +
+ tools/testing/memblock/tests/alloc_low_api.c       | 148 ++++
+ tools/testing/memblock/tests/alloc_low_api.h       |   9 +
+ tools/testing/memblock/tests/basic_api.c           |  20 +-
+ tools/testing/memblock/tests/common.c              |  12 +-
+ tools/testing/radix-tree/idr-test.c                |  20 +-
+ tools/testing/radix-tree/maple.c                   |  12 +-
+ tools/testing/selftests/cgroup/test_zswap.c        |  24 +-
+ tools/testing/selftests/damon/.gitignore           |   1 +
+ tools/testing/selftests/damon/_damon_sysfs.py      | 161 +++-
+ tools/testing/selftests/damon/damon_nr_regions.py  |   3 +
+ .../selftests/damon/damos_apply_interval.py        |   3 +
+ tools/testing/selftests/damon/damos_quota.py       |   3 +
+ tools/testing/selftests/damon/damos_quota_goal.py  |   3 +
+ .../testing/selftests/damon/damos_tried_regions.py |   3 +
+ .../selftests/damon/drgn_dump_damon_status.py      |  32 +
+ tools/testing/selftests/damon/sysfs.py             |  39 +-
+ tools/testing/selftests/damon/sysfs.sh             |  28 +
+ .../selftests/damon/sysfs_memcg_path_leak.sh       |   7 +
+ .../selftests/damon/sysfs_no_op_commit_break.py    |  35 +-
+ .../sysfs_update_schemes_tried_regions_hang.py     |   3 +
+ ..._update_schemes_tried_regions_wss_estimation.py |   3 +
+ tools/testing/selftests/mm/.gitignore              |   2 +
+ tools/testing/selftests/mm/Makefile                |  21 +-
+ tools/testing/selftests/mm/compaction_test.c       |   2 +-
+ tools/testing/selftests/mm/cow.c                   |   1 -
+ tools/testing/selftests/mm/folio_order_check.c     | 122 +++
+ tools/testing/selftests/mm/folio_split_race_test.c |   5 +-
+ tools/testing/selftests/mm/guard-regions.c         |  15 +-
+ tools/testing/selftests/mm/gup.c                   | 262 ++++++
+ tools/testing/selftests/mm/gup_longterm.c          |   1 -
+ tools/testing/selftests/mm/hmm-tests.c             |   7 +-
+ tools/testing/selftests/mm/hugetlb-madvise.c       |   1 -
+ tools/testing/selftests/mm/hugetlb-mmap.c          |   1 -
+ tools/testing/selftests/mm/hugetlb-mremap.c        |   1 -
+ tools/testing/selftests/mm/hugetlb-shm.c           |   1 -
+ tools/testing/selftests/mm/hugetlb-soft-offline.c  |  59 +-
+ tools/testing/selftests/mm/hugetlb_dio.c           |   1 -
+ .../selftests/mm/hugetlb_fault_after_madv.c        |   1 -
+ tools/testing/selftests/mm/hugetlb_madv_vs_map.c   | 142 +++-
+ tools/testing/selftests/mm/khugepaged.c            | 580 ++++++++++++--
+ tools/testing/selftests/mm/khugepaged_race.c       | 538 +++++++++++++
+ tools/testing/selftests/mm/khugepaged_sync_check.c | 179 +++++
+ tools/testing/selftests/mm/ksm_tests.c             |   1 -
+ tools/testing/selftests/mm/memfd_secret.c          |   3 +-
+ tools/testing/selftests/mm/memory-failure.c        | 114 ++-
+ tools/testing/selftests/mm/merge.c                 |  95 +++
+ tools/testing/selftests/mm/migration.c             |   3 +-
+ tools/testing/selftests/mm/mlock-random-test.c     |   1 -
+ tools/testing/selftests/mm/mlock2-tests.c          |   2 +-
+ tools/testing/selftests/mm/mlock2.h                |   8 +-
+ tools/testing/selftests/mm/mremap_test.c           |  44 +-
+ tools/testing/selftests/mm/pagemap_ioctl.c         |  95 +--
+ tools/testing/selftests/mm/pkey-helpers.h          |   7 -
+ tools/testing/selftests/mm/pkey_sighandler_tests.c |   4 +-
+ tools/testing/selftests/mm/prctl_thp_disable.c     |   1 -
+ tools/testing/selftests/mm/protection_keys.c       |   2 +-
+ tools/testing/selftests/mm/run_vmtests.sh          |  90 +--
+ tools/testing/selftests/mm/soft-dirty.c            |   1 -
+ tools/testing/selftests/mm/split_huge_page_test.c  |  90 +--
+ tools/testing/selftests/mm/thuge-gen.c             |   1 -
+ tools/testing/selftests/mm/transhuge-stress.c      |   1 -
+ tools/testing/selftests/mm/uffd-common.c           |   4 +-
+ tools/testing/selftests/mm/uffd-common.h           |   1 -
+ tools/testing/selftests/mm/uffd-wp-mremap.c        |   4 +-
+ tools/testing/selftests/mm/va_high_addr_switch.c   |   1 -
+ tools/testing/selftests/mm/vm_util.c               | 408 ++++++----
+ tools/testing/selftests/mm/vm_util.h               |  19 +-
+ tools/testing/selftests/proc/proc-maps-race.c      | 186 ++++-
+ .../selftests/proc/proc-self-map-files-001.c       |   2 +-
+ .../selftests/proc/proc-self-map-files-002.c       |   2 +-
+ tools/testing/vma/include/dup.h                    |  97 ++-
+ tools/testing/vma/include/stubs.h                  |  14 +-
+ tools/testing/vma/shared.c                         |   9 +
+ tools/testing/vma/tests/merge.c                    |  10 +-
+ tools/testing/vma/tests/mmap.c                     | 105 +++
+ tools/testing/vma/vma_internal.h                   |   1 -
+ 458 files changed, 14599 insertions(+), 8013 deletions(-)
+ create mode 100644 Documentation/mm/kernel-page-tables.rst
+ delete mode 100644 include/linux/hugetlb_inline.h
+ create mode 100644 include/linux/vmemmap-optimization.h
+ rename drivers/char/mem.c => mm/char-mem.c (97%)
+ create mode 100644 mm/collapse.h
+ create mode 100644 tools/lib/mm/file_utils.c
+ create mode 100644 tools/lib/mm/file_utils.h
+ rename tools/{testing/selftests => lib}/mm/hugepage_settings.c (76%)
+ rename tools/{testing/selftests => lib}/mm/hugepage_settings.h (90%)
+ rename tools/{testing/selftests/mm/gup_test.c => mm/gup_bench.c} (50%)
+ delete mode 100644 tools/testing/memblock/TODO
+ create mode 100644 tools/testing/memblock/tests/alloc_low_api.c
+ create mode 100644 tools/testing/memblock/tests/alloc_low_api.h
+ create mode 100644 tools/testing/selftests/mm/folio_order_check.c
+ create mode 100644 tools/testing/selftests/mm/gup.c
+ create mode 100644 tools/testing/selftests/mm/khugepaged_race.c
+ create mode 100644 tools/testing/selftests/mm/khugepaged_sync_check.c
+Merging mm-nonmm-stable/mm-nonmm-stable (a243ede718463 Merge tag 'mtd/fixes-for-7.3-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux)
+$ git merge -m Merge branch 'mm-nonmm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-stable/mm-nonmm-stable
+Already up to date.
+Merging mm-nonmm-unstable/mm-nonmm-unstable (cbb17a7a0afe4 init: simplify initramfs.o build rule)
+$ git merge -m Merge branch 'mm-nonmm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-unstable/mm-nonmm-unstable
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Auto-merging arch/Kconfig
+Auto-merging arch/s390/Kconfig
+Auto-merging fs/proc/base.c
+Auto-merging init/main.c
+Auto-merging kernel/fork.c
+Auto-merging mm/shmem.c
+Merge made by the 'ort' strategy.
+ .mailmap                                           |   4 +
+ CREDITS                                            | 181 ++++++++--------
+ Documentation/admin-guide/sysctl/kernel.rst        |   5 +-
+ MAINTAINERS                                        |   1 +
+ arch/Kconfig                                       |   8 -
+ arch/alpha/include/uapi/asm/setup.h                |   4 +
+ arch/arc/include/asm/setup.h                       |   2 +-
+ arch/arm/include/uapi/asm/setup.h                  |   6 +-
+ arch/arm64/include/uapi/asm/setup.h                |   4 +
+ arch/loongarch/include/uapi/asm/setup.h            |   4 +
+ arch/m68k/include/uapi/asm/setup.h                 |   6 +-
+ arch/microblaze/include/uapi/asm/setup.h           |   4 +
+ arch/mips/include/uapi/asm/setup.h                 |   4 +
+ arch/parisc/include/uapi/asm/setup.h               |   4 +
+ arch/powerpc/include/uapi/asm/setup.h              |   4 +
+ arch/riscv/include/uapi/asm/setup.h                |   4 +
+ arch/s390/Kconfig                                  |   8 -
+ arch/s390/kernel/traps.c                           |   7 +
+ arch/sparc/include/uapi/asm/setup.h                |  10 +-
+ arch/sparc/kernel/setup.c                          |   9 +
+ arch/um/include/asm/setup.h                        |   2 +-
+ arch/x86/include/asm/setup.h                       |   2 +-
+ arch/xtensa/include/uapi/asm/setup.h               |   4 +
+ drivers/media/v4l2-core/v4l2-vp9.c                 |  30 +--
+ drivers/rapidio/devices/rio_mport_cdev.c           |   9 +-
+ drivers/usb/gadget/legacy/inode.c                  |   4 +-
+ drivers/watchdog/hpwdt.c                           |   2 +-
+ fs/fat/inode.c                                     |  23 +-
+ fs/ocfs2/alloc.c                                   |  36 +++-
+ fs/ocfs2/cluster/heartbeat.c                       |  12 +-
+ fs/ocfs2/dir.c                                     |  55 +++++
+ fs/ocfs2/dlmglue.c                                 |  33 ++-
+ fs/ocfs2/dlmglue.h                                 |   5 +-
+ fs/ocfs2/export.c                                  |  10 +-
+ fs/ocfs2/inode.c                                   |  58 ++++-
+ fs/ocfs2/journal.c                                 |   3 +-
+ fs/ocfs2/journal.h                                 |   1 -
+ fs/ocfs2/ocfs2.h                                   |  17 ++
+ fs/ocfs2/quota_local.c                             |  33 ++-
+ fs/ocfs2/refcounttree.c                            |  27 +++
+ fs/ocfs2/suballoc.c                                | 235 +++++++++++++++++---
+ fs/ocfs2/suballoc.h                                |   2 +-
+ fs/ocfs2/super.c                                   |  32 ++-
+ fs/ocfs2/xattr.c                                   | 204 ++++++++++++++----
+ fs/proc/base.c                                     |   8 +-
+ fs/squashfs/fragment.c                             |   6 +-
+ fs/squashfs/squashfs_fs.h                          |   2 +-
+ include/linux/kdev_t.h                             |   2 +-
+ include/linux/minmax.h                             |   6 +-
+ include/linux/panic.h                              |   5 +-
+ include/uapi/asm-generic/setup.h                   |   4 +
+ init/Kconfig                                       |  19 +-
+ init/Makefile                                      |   6 +-
+ init/main.c                                        |  12 +-
+ init/version.c                                     |  11 +-
+ ipc/mqueue.c                                       |   7 +
+ kernel/fork.c                                      |   3 +-
+ kernel/gcov/fs.c                                   |   2 +-
+ kernel/hung_task.c                                 |  55 ++++-
+ kernel/kallsyms.c                                  | 111 ++++++----
+ kernel/kallsyms_internal.h                         |  11 +
+ kernel/kcov.c                                      |   8 +
+ kernel/panic.c                                     | 104 +++++----
+ kernel/resource.c                                  |  14 +-
+ kernel/resource_kunit.c                            |   3 +
+ kernel/taskstats.c                                 |  54 ++---
+ lib/Kconfig                                        |  14 +-
+ lib/Kconfig.debug                                  |  16 +-
+ lib/bootconfig.c                                   |  46 ++--
+ lib/decompress_bunzip2.c                           |   6 +-
+ lib/decompress_unlz4.c                             |   4 +
+ lib/decompress_unxz.c                              |  12 +-
+ lib/dynamic_debug.c                                |  15 +-
+ lib/group_cpus.c                                   |  87 +++++++-
+ lib/klist.c                                        |  16 +-
+ lib/percpu_counter.c                               |  10 +-
+ lib/plist.c                                        |   7 +
+ lib/raid/Kconfig                                   |   2 +
+ lib/raid/raid6/x86/avx2.c                          |   6 +
+ lib/raid/raid6/x86/avx512.c                        |   6 +
+ lib/raid/raid6/x86/recov_avx2.c                    |   2 +
+ lib/raid/raid6/x86/recov_avx512.c                  |   2 +
+ lib/raid/xor/x86/xor-avx.c                         |   1 +
+ lib/string_helpers.c                               |   5 +-
+ lib/tests/Makefile                                 |   1 +
+ lib/tests/errseq_kunit.c                           | 237 +++++++++++++++++++++
+ lib/tests/string_helpers_kunit.c                   |  46 ++--
+ mm/shmem.c                                         |   5 +
+ scripts/checkpatch.pl                              |  38 +++-
+ scripts/checkstack.pl                              |   6 +-
+ scripts/kallsyms.c                                 |  14 +-
+ scripts/spelling.txt                               |   4 +-
+ tools/bootconfig/main.c                            | 104 ++++-----
+ tools/bootconfig/test-bootconfig.sh                |  12 ++
+ tools/testing/selftests/core/unshare_test.c        |  19 +-
+ tools/testing/selftests/filelock/ofdlocks.c        |   2 +-
+ .../filesystems/epoll/epoll_wakeup_test.c          |  32 ++-
+ .../selftests/membarrier/membarrier_test_impl.h    |  17 +-
+ tools/testing/selftests/uevent/uevent_filtering.c  |   8 -
+ 99 files changed, 1785 insertions(+), 588 deletions(-)
+ create mode 100644 lib/tests/errseq_kunit.c
+Merging kbuild/kbuild-for-next (59ba9b8f8803c Merge branch 'kbuild-next-unstable' into kbuild-for-next)
+$ git merge -m Merge branch 'kbuild-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild/kbuild-for-next
+Auto-merging MAINTAINERS
+Auto-merging Makefile
+Auto-merging arch/x86/Kconfig
+Auto-merging arch/x86/Makefile
+Auto-merging init/Kconfig
+Auto-merging scripts/kallsyms.c
+CONFLICT (content): Merge conflict in scripts/kallsyms.c
+Recorded preimage for 'scripts/kallsyms.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'scripts/kallsyms.c'.
+[master b7d4f7554b9d1] Merge branch 'kbuild-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git
+$ git diff -M --stat --summary HEAD^..
+ .../early-userspace/early_userspace_support.rst    |  16 +-
+ .../filesystems/ramfs-rootfs-initramfs.rst         |  10 +-
+ Documentation/kbuild/kbuild.rst                    |  17 +
+ Documentation/kbuild/kconfig-language.rst          |  13 +-
+ Documentation/kbuild/kconfig-macro-language.rst    |   8 +-
+ Documentation/process/changes.rst                  |   8 +
+ MAINTAINERS                                        |   2 +
+ Makefile                                           |  51 +--
+ arch/arm64/kernel/pi/Makefile                      |   2 +-
+ arch/riscv/kernel/pi/Makefile                      |   2 +-
+ arch/x86/Kconfig                                   |  17 +
+ arch/x86/Makefile                                  |  12 +-
+ arch/x86/boot/Makefile                             |   2 +-
+ arch/x86/boot/compressed/Makefile                  |   2 +-
+ drivers/firmware/efi/libstub/Makefile              |   2 +-
+ drivers/firmware/qcom/Kconfig                      |  26 +-
+ drivers/firmware/qcom/qcom_tzmem.c                 |   4 +-
+ fs/erofs/Kconfig                                   |   8 +-
+ include/asm-generic/vmlinux.lds.h                  |   2 +-
+ init/Kconfig                                       | 197 +----------
+ kernel/Makefile                                    |   6 +-
+ scripts/.gitignore                                 |   1 +
+ scripts/Kconfig.include                            |  29 +-
+ scripts/Kconfig.toolchain                          | 272 +++++++++++++++
+ scripts/Makefile                                   |   8 +-
+ scripts/Makefile.gcc-plugins                       |   4 +
+ scripts/Makefile.lib                               |   4 +-
+ scripts/Makefile.package                           |   2 +-
+ scripts/Makefile.vmlinux                           |  20 +-
+ scripts/Makefile.warn                              |  28 +-
+ scripts/check-function-names.sh                    |   3 +-
+ scripts/elf-parse.c                                | 103 +++++-
+ scripts/elf-parse.h                                |  54 +++
+ {usr => scripts}/gen_init_cpio.c                   |   0
+ {usr => scripts}/gen_initramfs.sh                  |   4 +-
+ scripts/jobserver-exec                             |   3 +-
+ scripts/kallsyms-sysmap.c                          | 248 +++++++++++++
+ scripts/kallsyms.c                                 | 382 +++++++++++++++------
+ scripts/kallsyms.h                                 |  44 +++
+ scripts/kconfig/conf.c                             |  16 +-
+ scripts/kconfig/confdata.c                         |   6 +
+ scripts/kconfig/kconfig-sym-check.pl               |   2 +-
+ scripts/kconfig/lexer.l                            |   3 +
+ scripts/kconfig/lkc.h                              |   2 +-
+ scripts/kconfig/lkc_proto.h                        |   1 +
+ scripts/kconfig/menu.c                             |  64 +++-
+ scripts/kconfig/parser.y                           |   8 +-
+ scripts/kconfig/symbol.c                           | 115 +++++--
+ scripts/kconfig/tests/conftest.py                  |  27 +-
+ scripts/kconfig/tests/def_type/Kconfig             |  28 ++
+ scripts/kconfig/tests/def_type/__init__.py         |  18 +
+ scripts/kconfig/tests/def_type/expected_guard_n    |   9 +
+ scripts/kconfig/tests/def_type/expected_guard_y    |  11 +
+ scripts/kconfig/tests/def_type/guard_n.config      |   1 +
+ scripts/kconfig/tests/def_type/guard_y.config      |   1 +
+ scripts/kconfig/tests/err_num_bounds/Kconfig       |  79 +++++
+ scripts/kconfig/tests/err_num_bounds/__init__.py   |  12 +
+ .../kconfig/tests/err_num_bounds/expected_stderr   |  10 +
+ scripts/kconfig/tests/err_num_mismatch/Kconfig     |  38 ++
+ scripts/kconfig/tests/err_num_mismatch/__init__.py |   9 +
+ .../kconfig/tests/err_num_mismatch/expected_stderr |   4 +
+ .../kconfig/tests/err_num_non_numeric_ref/Kconfig  |  75 ++++
+ .../tests/err_num_non_numeric_ref/__init__.py      |   9 +
+ .../tests/err_num_non_numeric_ref/expected_stderr  |  16 +
+ scripts/kconfig/tests/err_recursive_dep/Kconfig    |  24 ++
+ .../tests/err_recursive_dep/expected_stderr        |  13 +
+ .../kconfig/tests/randconfig_probability/Kconfig   |  12 +
+ .../tests/randconfig_probability/__init__.py       | 130 +++++++
+ scripts/kconfig/tests/savedefconfig_range/Kconfig  |  60 ++++
+ .../kconfig/tests/savedefconfig_range/__init__.py  |   8 +
+ scripts/kconfig/tests/savedefconfig_range/config   |   7 +
+ .../tests/savedefconfig_range/expected_defconfig   |   0
+ scripts/kconfig/tests/warn_changed_input/Kconfig   |   5 +
+ .../kconfig/tests/warn_changed_input/__init__.py   |  21 +-
+ .../tests/warn_changed_input/expected_config       |   1 +
+ scripts/kconfig/tests/warn_num_bounds/Kconfig      |  24 ++
+ scripts/kconfig/tests/warn_num_bounds/__init__.py  |  25 ++
+ scripts/kconfig/tests/warn_num_bounds/config       |   7 +
+ .../kconfig/tests/warn_num_bounds/expected_config  |  11 +
+ .../tests/warn_num_bounds/expected_config_stderr   |   3 +
+ .../tests/warn_num_bounds/expected_frontend_config |  11 +
+ .../tests/warn_num_bounds/expected_frontend_stderr |   0
+ scripts/link-vmlinux.sh                            |  24 +-
+ scripts/mkcompile_h                                |   2 +-
+ scripts/mksysmap                                   |  94 -----
+ scripts/remove-stale-files                         |   2 +
+ .../sbom/tests/cmd_graph/test_savedcmd_parser.py   |   2 +-
+ scripts/setlocalversion                            |   3 +-
+ scripts/sorttable.c                                |  15 +-
+ scripts/ver_linux                                  |   1 +
+ tools/lib/python/jobserver.py                      |  13 +
+ tools/testing/selftests/kho/vmtest.sh              |   2 +-
+ tools/testing/selftests/liveupdate/vmtest.sh       |   4 +-
+ tools/testing/selftests/nolibc/Makefile.nolibc     |   2 +-
+ usr/.gitignore                                     |   4 +-
+ usr/Kconfig                                        |   2 +-
+ usr/Makefile                                       |   4 +-
+ 97 files changed, 2091 insertions(+), 625 deletions(-)
+ create mode 100644 scripts/Kconfig.toolchain
+ rename {usr => scripts}/gen_init_cpio.c (100%)
+ rename {usr => scripts}/gen_initramfs.sh (97%)
+ create mode 100644 scripts/kallsyms-sysmap.c
+ create mode 100644 scripts/kallsyms.h
+ create mode 100644 scripts/kconfig/tests/def_type/Kconfig
+ create mode 100644 scripts/kconfig/tests/def_type/__init__.py
+ create mode 100644 scripts/kconfig/tests/def_type/expected_guard_n
+ create mode 100644 scripts/kconfig/tests/def_type/expected_guard_y
+ create mode 100644 scripts/kconfig/tests/def_type/guard_n.config
+ create mode 100644 scripts/kconfig/tests/def_type/guard_y.config
+ create mode 100644 scripts/kconfig/tests/err_num_bounds/Kconfig
+ create mode 100644 scripts/kconfig/tests/err_num_bounds/__init__.py
+ create mode 100644 scripts/kconfig/tests/err_num_bounds/expected_stderr
+ create mode 100644 scripts/kconfig/tests/err_num_mismatch/Kconfig
+ create mode 100644 scripts/kconfig/tests/err_num_mismatch/__init__.py
+ create mode 100644 scripts/kconfig/tests/err_num_mismatch/expected_stderr
+ create mode 100644 scripts/kconfig/tests/err_num_non_numeric_ref/Kconfig
+ create mode 100644 scripts/kconfig/tests/err_num_non_numeric_ref/__init__.py
+ create mode 100644 scripts/kconfig/tests/err_num_non_numeric_ref/expected_stderr
+ create mode 100644 scripts/kconfig/tests/randconfig_probability/Kconfig
+ create mode 100644 scripts/kconfig/tests/randconfig_probability/__init__.py
+ create mode 100644 scripts/kconfig/tests/savedefconfig_range/Kconfig
+ create mode 100644 scripts/kconfig/tests/savedefconfig_range/__init__.py
+ create mode 100644 scripts/kconfig/tests/savedefconfig_range/config
+ create mode 100644 scripts/kconfig/tests/savedefconfig_range/expected_defconfig
+ create mode 100644 scripts/kconfig/tests/warn_num_bounds/Kconfig
+ create mode 100644 scripts/kconfig/tests/warn_num_bounds/__init__.py
+ create mode 100644 scripts/kconfig/tests/warn_num_bounds/config
+ create mode 100644 scripts/kconfig/tests/warn_num_bounds/expected_config
+ create mode 100644 scripts/kconfig/tests/warn_num_bounds/expected_config_stderr
+ create mode 100644 scripts/kconfig/tests/warn_num_bounds/expected_frontend_config
+ create mode 100644 scripts/kconfig/tests/warn_num_bounds/expected_frontend_stderr
+ delete mode 100755 scripts/mksysmap
+Merging clang-fixes/clang-fixes-for-next (0bb666d5f5a23 once_lite: Simplify condition handling and fix context analysis)
+$ git merge -m Merge branch 'clang-fixes-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git clang-fixes/clang-fixes-for-next
+Merge made by the 'ort' strategy.
+ include/linux/once_lite.h | 8 +++++---
+ 1 file changed, 5 insertions(+), 3 deletions(-)
+Merging clang-format/clang-format (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'clang-format' of https://github.com/ojeda/linux.git clang-format/clang-format
+Already up to date.
+Merging perf/perf-tools-next (45d15e89a783a perf symbols: Don't let a module's last symbol overlap the next module)
+$ git merge -m Merge branch 'perf-tools-next' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf/perf-tools-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |    2 +-
+ tools/arch/x86/include/asm/msr-index.h             |    7 +
+ tools/arch/x86/include/uapi/asm/perf_regs.h        |   53 +
+ tools/build/Makefile.feature                       |    8 +-
+ tools/build/feature/Makefile                       |   42 +-
+ tools/build/feature/test-all.c                     |    6 +-
+ tools/build/feature/test-gettid.c                  |    1 +
+ tools/build/feature/test-gtk2-infobar.c            |   12 -
+ tools/build/feature/{test-gtk2.c => test-gtk4.c}   |    4 +-
+ tools/build/feature/test-libdw.c                   |   10 +-
+ tools/build/feature/test-libperl.c                 |   10 -
+ tools/build/feature/test-libpython.c               |   10 -
+ tools/build/feature/test-python-module.c           |   13 +
+ tools/build/feature/test-timerfd.c                 |    2 +-
+ tools/include/tools/dis-asm-compat.h               |    1 +
+ tools/include/uapi/linux/perf_event.h              |   49 +-
+ tools/lib/api/fd/array.c                           |    4 +
+ tools/lib/perf/evlist.c                            |    5 +-
+ tools/lib/perf/include/internal/evsel.h            |    2 +
+ tools/perf/.clang-format                           |    1 +
+ tools/perf/Build                                   |    5 +-
+ tools/perf/Documentation/db-export.txt             |   20 +-
+ tools/perf/Documentation/perf-annotate.txt         |    3 +
+ tools/perf/Documentation/perf-c2c.txt              |   14 +-
+ tools/perf/Documentation/perf-check.txt            |    3 +-
+ tools/perf/Documentation/perf-config.txt           |   10 +
+ tools/perf/Documentation/perf-mem.txt              |    4 +
+ tools/perf/Documentation/perf-record.txt           |   27 +-
+ tools/perf/Documentation/perf-report.txt           |   13 +-
+ tools/perf/Documentation/perf-script-perl.txt      |  216 -
+ tools/perf/Documentation/perf-script-python.txt    |  752 +--
+ tools/perf/Documentation/perf-script.txt           |   82 +-
+ tools/perf/Documentation/perf-stat.txt             |   12 +-
+ tools/perf/Documentation/perf-top.txt              |   11 +
+ tools/perf/Documentation/perf.data-file-format.txt |   13 +
+ tools/perf/Documentation/tips.txt                  |    3 +-
+ tools/perf/Makefile                                |   35 +-
+ tools/perf/Makefile.config                         |  106 +-
+ tools/perf/Makefile.perf                           |  120 +-
+ tools/perf/arch/arm/tests/dwarf-unwind.c           |    1 +
+ tools/perf/arch/arm64/tests/dwarf-unwind.c         |    1 +
+ tools/perf/arch/loongarch/util/header.c            |    2 +-
+ tools/perf/arch/powerpc/tests/dwarf-unwind.c       |    1 +
+ tools/perf/arch/powerpc/util/skip-callchain-idx.c  |    4 +-
+ tools/perf/arch/riscv/Build                        |    1 +
+ tools/perf/arch/riscv/include/arch-tests.h         |    9 +
+ tools/perf/arch/riscv/include/perf_regs.h          |    2 +
+ tools/perf/arch/riscv/tests/Build                  |    4 +
+ tools/perf/arch/riscv/tests/arch-tests.c           |   10 +
+ tools/perf/arch/riscv/tests/dwarf-unwind.c         |   64 +
+ tools/perf/arch/riscv/tests/regs_load.S            |   58 +
+ tools/perf/arch/x86/tests/dwarf-unwind.c           |    4 +
+ tools/perf/arch/x86/tests/topdown.c                |   30 +
+ tools/perf/arch/x86/util/mem-events.c              |   17 +
+ tools/perf/arch/x86/util/mem-events.h              |    2 +
+ tools/perf/arch/x86/util/pmu.c                     |  226 +-
+ tools/perf/arch/x86/util/topdown.c                 |    3 +-
+ tools/perf/bench/futex.h                           |   11 +
+ tools/perf/bench/numa.c                            |    2 +
+ tools/perf/bench/sched-messaging.c                 |    4 +-
+ tools/perf/builtin-annotate.c                      |   96 +-
+ tools/perf/builtin-buildid-list.c                  |   17 +-
+ tools/perf/builtin-c2c.c                           |  229 +-
+ tools/perf/builtin-check.c                         |    3 +-
+ tools/perf/builtin-config.c                        |   18 +-
+ tools/perf/builtin-ftrace.c                        |   28 +-
+ tools/perf/builtin-help.c                          |   30 +-
+ tools/perf/builtin-inject.c                        |    4 +
+ tools/perf/builtin-kmem.c                          |    3 +-
+ tools/perf/builtin-kvm.c                           |    1 +
+ tools/perf/builtin-list.c                          |    2 +-
+ tools/perf/builtin-mem.c                           |    7 +-
+ tools/perf/builtin-record.c                        |   15 +
+ tools/perf/builtin-report.c                        |   43 +-
+ tools/perf/builtin-sched.c                         |    7 +-
+ tools/perf/builtin-script.c                        | 1124 +++--
+ tools/perf/builtin-stat.c                          |    4 +-
+ tools/perf/builtin-timechart.c                     |    5 -
+ tools/perf/builtin-top.c                           |   38 +-
+ tools/perf/builtin-trace.c                         |  271 +-
+ tools/perf/builtin.h                               |    3 +
+ tools/perf/perf.c                                  |  120 +-
+ tools/perf/pmu-events/Build                        |   10 +-
+ tools/perf/pmu-events/amd_metrics.py               |   16 +-
+ .../perf/pmu-events/arch/x86/alderlake/cache.json  |    4 +-
+ .../perf/pmu-events/arch/x86/alderlake/memory.json |   13 +
+ .../perf/pmu-events/arch/x86/alderlake/other.json  |   10 +
+ .../pmu-events/arch/x86/alderlake/pipeline.json    |    6 +-
+ .../perf/pmu-events/arch/x86/alderlaken/cache.json |    4 +-
+ .../perf/pmu-events/arch/x86/arrowlake/cache.json  |    1 +
+ .../perf/pmu-events/arch/x86/arrowlake/other.json  |   19 +
+ .../pmu-events/arch/x86/arrowlake/pipeline.json    |   10 +-
+ .../pmu-events/arch/x86/broadwell/bdw-metrics.json |   51 +-
+ .../arch/x86/broadwell/metricgroups.json           |    3 +-
+ .../arch/x86/broadwellde/bdwde-metrics.json        |   49 +-
+ .../arch/x86/broadwellde/metricgroups.json         |    3 +-
+ .../arch/x86/broadwellx/bdx-metrics.json           |   63 +-
+ .../arch/x86/broadwellx/metricgroups.json          |    3 +-
+ .../arch/x86/cascadelakex/clx-metrics.json         |   94 +-
+ .../arch/x86/cascadelakex/metricgroups.json        |    3 +-
+ .../arch/x86/cascadelakex/uncore-interconnect.json |    6 +
+ .../arch/x86/clearwaterforest/pipeline.json        |    3 +-
+ .../arch/x86/clearwaterforest/uncore-cache.json    |    5 +
+ .../x86/clearwaterforest/uncore-interconnect.json  |    2 +
+ .../pmu-events/arch/x86/emeraldrapids/cache.json   |    2 +-
+ .../pmu-events/arch/x86/emeraldrapids/memory.json  |   35 +
+ .../pmu-events/arch/x86/emeraldrapids/other.json   |    9 +
+ .../arch/x86/emeraldrapids/pipeline.json           |   10 +-
+ .../arch/x86/emeraldrapids/uncore-cache.json       |   42 +
+ .../x86/emeraldrapids/uncore-interconnect.json     |   12 +
+ .../arch/x86/emeraldrapids/uncore-io.json          |   54 +-
+ .../arch/x86/emeraldrapids/uncore-memory.json      |   10 +
+ .../pmu-events/arch/x86/graniterapids/other.json   |    9 +
+ .../arch/x86/graniterapids/pipeline.json           |    6 +-
+ .../arch/x86/graniterapids/uncore-cache.json       |   15 +
+ .../x86/graniterapids/uncore-interconnect.json     |    2 +
+ .../arch/x86/graniterapids/uncore-io.json          |   12 +
+ .../pmu-events/arch/x86/haswell/hsw-metrics.json   |   49 +-
+ .../pmu-events/arch/x86/haswell/metricgroups.json  |    3 +-
+ .../pmu-events/arch/x86/haswellx/hsx-metrics.json  |   61 +-
+ .../pmu-events/arch/x86/haswellx/metricgroups.json |    3 +-
+ .../pmu-events/arch/x86/icelake/icl-metrics.json   |   84 +-
+ tools/perf/pmu-events/arch/x86/icelake/memory.json |   24 +
+ .../pmu-events/arch/x86/icelake/metricgroups.json  |    2 +-
+ tools/perf/pmu-events/arch/x86/icelake/other.json  |    9 +
+ .../perf/pmu-events/arch/x86/icelake/pipeline.json |    9 +-
+ .../pmu-events/arch/x86/icelakex/icx-metrics.json  |  101 +-
+ .../perf/pmu-events/arch/x86/icelakex/memory.json  |   24 +
+ .../pmu-events/arch/x86/icelakex/metricgroups.json |    2 +-
+ tools/perf/pmu-events/arch/x86/icelakex/other.json |    9 +
+ .../pmu-events/arch/x86/icelakex/pipeline.json     |    7 +-
+ .../pmu-events/arch/x86/icelakex/uncore-cache.json |   79 +
+ .../arch/x86/icelakex/uncore-interconnect.json     |   22 +
+ .../pmu-events/arch/x86/ivybridge/ivb-metrics.json |   51 +-
+ .../arch/x86/ivybridge/metricgroups.json           |    3 +-
+ .../pmu-events/arch/x86/ivytown/ivt-metrics.json   |   63 +-
+ .../pmu-events/arch/x86/ivytown/metricgroups.json  |    3 +-
+ .../pmu-events/arch/x86/jaketown/jkt-metrics.json  |   41 +-
+ .../pmu-events/arch/x86/jaketown/metricgroups.json |    3 +-
+ .../perf/pmu-events/arch/x86/lunarlake/cache.json  |    3 +
+ .../pmu-events/arch/x86/lunarlake/pipeline.json    |    2 +
+ tools/perf/pmu-events/arch/x86/mapfile.csv         |   26 +-
+ .../perf/pmu-events/arch/x86/meteorlake/other.json |   10 +
+ .../pmu-events/arch/x86/meteorlake/pipeline.json   |    6 +-
+ tools/perf/pmu-events/arch/x86/novalake/cache.json |  500 +-
+ .../arch/x86/novalake/floating-point.json          |  423 ++
+ .../pmu-events/arch/x86/novalake/frontend.json     |  147 +-
+ .../perf/pmu-events/arch/x86/novalake/memory.json  |   24 +
+ tools/perf/pmu-events/arch/x86/novalake/other.json |  213 +
+ .../pmu-events/arch/x86/novalake/pipeline.json     |  508 +-
+ .../arch/x86/novalake/virtual-memory.json          |  173 +
+ .../pmu-events/arch/x86/pantherlake/cache.json     |    3 +
+ .../pmu-events/arch/x86/pantherlake/other.json     |   10 +
+ .../pmu-events/arch/x86/pantherlake/pipeline.json  |   10 +-
+ .../pmu-events/arch/x86/rocketlake/memory.json     |   24 +
+ .../arch/x86/rocketlake/metricgroups.json          |    2 +-
+ .../perf/pmu-events/arch/x86/rocketlake/other.json |    9 +
+ .../pmu-events/arch/x86/rocketlake/pipeline.json   |    9 +-
+ .../arch/x86/rocketlake/rkl-metrics.json           |   84 +-
+ .../arch/x86/sandybridge/metricgroups.json         |    3 +-
+ .../arch/x86/sandybridge/snb-metrics.json          |   35 +-
+ .../pmu-events/arch/x86/sapphirerapids/cache.json  |    2 +-
+ .../pmu-events/arch/x86/sapphirerapids/memory.json |   35 +
+ .../arch/x86/sapphirerapids/metricgroups.json      |    2 +-
+ .../pmu-events/arch/x86/sapphirerapids/other.json  |    9 +
+ .../arch/x86/sapphirerapids/pipeline.json          |   10 +-
+ .../arch/x86/sapphirerapids/spr-metrics.json       |   93 +-
+ .../arch/x86/sapphirerapids/uncore-cache.json      |   42 +
+ .../x86/sapphirerapids/uncore-interconnect.json    |   12 +
+ .../arch/x86/sapphirerapids/uncore-io.json         |   54 +-
+ .../arch/x86/sapphirerapids/uncore-memory.json     |   10 +
+ .../arch/x86/sierraforest/uncore-cache.json        |    5 +
+ .../arch/x86/sierraforest/uncore-interconnect.json |    2 +
+ .../pmu-events/arch/x86/skylake/metricgroups.json  |    3 +-
+ .../pmu-events/arch/x86/skylake/skl-metrics.json   |   59 +-
+ .../pmu-events/arch/x86/skylakex/metricgroups.json |    3 +-
+ .../pmu-events/arch/x86/skylakex/skx-metrics.json  |   94 +-
+ .../arch/x86/skylakex/uncore-interconnect.json     |    6 +
+ .../arch/x86/snowridgex/uncore-cache.json          |   68 +
+ .../arch/x86/snowridgex/uncore-interconnect.json   |   10 +
+ .../arch/x86/tigerlake/metricgroups.json           |    2 +-
+ .../perf/pmu-events/arch/x86/tigerlake/other.json  |    9 +
+ .../pmu-events/arch/x86/tigerlake/pipeline.json    |    7 +-
+ .../pmu-events/arch/x86/tigerlake/tgl-metrics.json |   82 +-
+ tools/perf/pmu-events/intel_metrics.py             |  268 +-
+ tools/perf/pmu-events/jevents.py                   |  112 +-
+ tools/perf/pmu-events/make_legacy_cache.py         |   34 +-
+ tools/perf/pmu-events/metric.py                    |   60 +-
+ tools/perf/python/SchedGui.py                      |  246 +
+ tools/perf/python/arm-cs-trace-disasm.py           |  440 ++
+ tools/perf/python/check-perf-trace.py              |  223 +
+ tools/perf/python/compaction-times.py              |  362 ++
+ tools/perf/python/counting.py                      |    1 +
+ tools/perf/python/event_analyzing_sample.py        |  317 ++
+ tools/perf/python/export-to-postgresql.py          | 1328 +++++
+ tools/perf/python/export-to-sqlite.py              |  946 ++++
+ tools/perf/python/exported-sql-viewer.py           | 5103 ++++++++++++++++++++
+ tools/perf/python/failed-syscalls-by-pid.py        |  161 +
+ tools/perf/python/failed-syscalls.py               |   94 +
+ tools/perf/python/flamegraph.py                    |  283 ++
+ tools/perf/python/futex-contention.py              |  107 +
+ tools/perf/python/gecko.py                         |  411 ++
+ tools/perf/python/ilist.py                         |   86 +-
+ tools/perf/python/intel-pt-events.py               |  632 +++
+ tools/perf/python/libxed.py                        |  123 +
+ tools/perf/python/mem-phys-addr.py                 |  136 +
+ tools/perf/python/net_dropmonitor.py               |  173 +
+ tools/perf/python/netdev-times.py                  |  494 ++
+ tools/perf/python/parallel-perf.py                 | 1250 +++++
+ tools/perf/python/perf.pyi                         |  122 +-
+ tools/perf/python/perf_live.py                     |   10 +-
+ tools/perf/{scripts => }/python/powerpc-hcalls.py  |  213 +-
+ tools/perf/python/rw-by-file.py                    |  113 +
+ tools/perf/python/rw-by-pid.py                     |  205 +
+ tools/perf/python/rwtop.py                         |  256 +
+ tools/perf/python/sched-migration.py               |  496 ++
+ tools/perf/python/sctop.py                         |  253 +
+ tools/perf/python/stackcollapse.py                 |  145 +
+ tools/perf/python/stat-cpi.py                      |  231 +
+ tools/perf/python/syscall-counts-by-pid.py         |  112 +
+ tools/perf/python/syscall-counts.py                |   95 +
+ tools/perf/python/task-analyzer.py                 |  908 ++++
+ tools/perf/python/tracepoint.py                    |    5 +-
+ tools/perf/python/treport.py                       |  564 +++
+ tools/perf/python/twatch.py                        |   77 +-
+ tools/perf/python/wakeup-latency.py                |  103 +
+ tools/perf/scripts/Build                           |   30 -
+ tools/perf/scripts/install-build-deps.sh           |    8 +-
+ tools/perf/scripts/perl/Perf-Trace-Util/Build      |    9 -
+ tools/perf/scripts/perl/Perf-Trace-Util/Context.c  |  122 -
+ tools/perf/scripts/perl/Perf-Trace-Util/Context.xs |   42 -
+ .../perf/scripts/perl/Perf-Trace-Util/Makefile.PL  |   18 -
+ tools/perf/scripts/perl/Perf-Trace-Util/README     |   59 -
+ .../perl/Perf-Trace-Util/lib/Perf/Trace/Context.pm |   55 -
+ .../perl/Perf-Trace-Util/lib/Perf/Trace/Core.pm    |  192 -
+ .../perl/Perf-Trace-Util/lib/Perf/Trace/Util.pm    |   94 -
+ tools/perf/scripts/perl/Perf-Trace-Util/typemap    |    1 -
+ .../perf/scripts/perl/bin/check-perf-trace-record  |    2 -
+ tools/perf/scripts/perl/bin/failed-syscalls-record |    3 -
+ tools/perf/scripts/perl/bin/failed-syscalls-report |   10 -
+ tools/perf/scripts/perl/bin/rw-by-file-record      |    3 -
+ tools/perf/scripts/perl/bin/rw-by-file-report      |   10 -
+ tools/perf/scripts/perl/bin/rw-by-pid-record       |    2 -
+ tools/perf/scripts/perl/bin/rw-by-pid-report       |    3 -
+ tools/perf/scripts/perl/bin/rwtop-record           |    2 -
+ tools/perf/scripts/perl/bin/rwtop-report           |   20 -
+ tools/perf/scripts/perl/bin/wakeup-latency-record  |    6 -
+ tools/perf/scripts/perl/bin/wakeup-latency-report  |    3 -
+ tools/perf/scripts/perl/check-perf-trace.pl        |  106 -
+ tools/perf/scripts/perl/failed-syscalls.pl         |   47 -
+ tools/perf/scripts/perl/rw-by-file.pl              |  106 -
+ tools/perf/scripts/perl/rw-by-pid.pl               |  184 -
+ tools/perf/scripts/perl/rwtop.pl                   |  203 -
+ tools/perf/scripts/perl/wakeup-latency.pl          |  107 -
+ tools/perf/scripts/python/Perf-Trace-Util/Build    |    4 -
+ .../perf/scripts/python/Perf-Trace-Util/Context.c  |  225 -
+ .../python/Perf-Trace-Util/lib/Perf/Trace/Core.py  |  116 -
+ .../Perf-Trace-Util/lib/Perf/Trace/EventClass.py   |   97 -
+ .../Perf-Trace-Util/lib/Perf/Trace/SchedGui.py     |  184 -
+ .../python/Perf-Trace-Util/lib/Perf/Trace/Util.py  |   92 -
+ tools/perf/scripts/python/arm-cs-trace-disasm.py   |  356 --
+ .../scripts/python/bin/compaction-times-record     |    2 -
+ .../scripts/python/bin/compaction-times-report     |    4 -
+ .../python/bin/event_analyzing_sample-record       |    8 -
+ .../python/bin/event_analyzing_sample-report       |    3 -
+ .../scripts/python/bin/export-to-postgresql-record |    8 -
+ .../scripts/python/bin/export-to-postgresql-report |   29 -
+ .../scripts/python/bin/export-to-sqlite-record     |    8 -
+ .../scripts/python/bin/export-to-sqlite-report     |   29 -
+ .../python/bin/failed-syscalls-by-pid-record       |    3 -
+ .../python/bin/failed-syscalls-by-pid-report       |   10 -
+ tools/perf/scripts/python/bin/flamegraph-record    |    2 -
+ tools/perf/scripts/python/bin/flamegraph-report    |    3 -
+ .../scripts/python/bin/futex-contention-record     |    2 -
+ .../scripts/python/bin/futex-contention-report     |    4 -
+ tools/perf/scripts/python/bin/gecko-record         |    2 -
+ tools/perf/scripts/python/bin/gecko-report         |    7 -
+ .../perf/scripts/python/bin/intel-pt-events-record |   13 -
+ .../perf/scripts/python/bin/intel-pt-events-report |    3 -
+ tools/perf/scripts/python/bin/mem-phys-addr-record |   19 -
+ tools/perf/scripts/python/bin/mem-phys-addr-report |    3 -
+ .../perf/scripts/python/bin/net_dropmonitor-record |    2 -
+ .../perf/scripts/python/bin/net_dropmonitor-report |    4 -
+ tools/perf/scripts/python/bin/netdev-times-record  |    8 -
+ tools/perf/scripts/python/bin/netdev-times-report  |    5 -
+ .../perf/scripts/python/bin/powerpc-hcalls-record  |    2 -
+ .../perf/scripts/python/bin/powerpc-hcalls-report  |    2 -
+ .../perf/scripts/python/bin/sched-migration-record |    2 -
+ .../perf/scripts/python/bin/sched-migration-report |    3 -
+ tools/perf/scripts/python/bin/sctop-record         |    3 -
+ tools/perf/scripts/python/bin/sctop-report         |   24 -
+ tools/perf/scripts/python/bin/stackcollapse-record |    8 -
+ tools/perf/scripts/python/bin/stackcollapse-report |    3 -
+ .../python/bin/syscall-counts-by-pid-record        |    3 -
+ .../python/bin/syscall-counts-by-pid-report        |   10 -
+ .../perf/scripts/python/bin/syscall-counts-record  |    3 -
+ .../perf/scripts/python/bin/syscall-counts-report  |   10 -
+ tools/perf/scripts/python/bin/task-analyzer-record |    2 -
+ tools/perf/scripts/python/bin/task-analyzer-report |    3 -
+ tools/perf/scripts/python/check-perf-trace.py      |   84 -
+ tools/perf/scripts/python/compaction-times.py      |  311 --
+ .../perf/scripts/python/event_analyzing_sample.py  |  192 -
+ tools/perf/scripts/python/export-to-postgresql.py  | 1114 -----
+ tools/perf/scripts/python/export-to-sqlite.py      |  799 ---
+ tools/perf/scripts/python/exported-sql-viewer.py   | 5030 -------------------
+ .../perf/scripts/python/failed-syscalls-by-pid.py  |   79 -
+ tools/perf/scripts/python/flamegraph.py            |  267 -
+ tools/perf/scripts/python/futex-contention.py      |   57 -
+ tools/perf/scripts/python/gecko.py                 |  395 --
+ tools/perf/scripts/python/intel-pt-events.py       |  494 --
+ tools/perf/scripts/python/libxed.py                |  107 -
+ tools/perf/scripts/python/mem-phys-addr.py         |  127 -
+ tools/perf/scripts/python/net_dropmonitor.py       |   78 -
+ tools/perf/scripts/python/netdev-times.py          |  473 --
+ tools/perf/scripts/python/parallel-perf.py         |  989 ----
+ tools/perf/scripts/python/sched-migration.py       |  462 --
+ tools/perf/scripts/python/sctop.py                 |   89 -
+ tools/perf/scripts/python/stackcollapse.py         |  127 -
+ tools/perf/scripts/python/stat-cpi.py              |   79 -
+ tools/perf/scripts/python/syscall-counts-by-pid.py |   75 -
+ tools/perf/scripts/python/syscall-counts.py        |   65 -
+ tools/perf/scripts/python/task-analyzer.py         |  934 ----
+ tools/perf/tests/Build                             |    7 +-
+ tools/perf/tests/builtin-test.c                    |   53 +-
+ tools/perf/tests/code-reading.c                    |   12 +-
+ tools/perf/tests/fdarray.c                         |   32 +
+ tools/perf/tests/hists_cumulate.c                  |    2 +-
+ tools/perf/tests/hists_filter.c                    |    2 +-
+ tools/perf/tests/hists_link.c                      |    2 +-
+ tools/perf/tests/hists_output.c                    |    2 +-
+ tools/perf/tests/hybrid-merge.c                    |  206 +
+ tools/perf/tests/make                              |  114 +-
+ tools/perf/tests/sample-parsing.c                  |  109 +
+ tools/perf/tests/shell/annotate.sh                 |    1 -
+ tools/perf/tests/shell/annotate_weight.sh          |   63 +
+ .../tests/shell/base_probe/test_adding_kernel.sh   |    1 -
+ tools/perf/tests/shell/c2c.sh                      |  114 +
+ tools/perf/tests/shell/coresight/callchain.sh      |    7 +-
+ .../shell/coresight/test_arm_coresight_disasm.sh   |   15 +-
+ tools/perf/tests/shell/data_type_profiling.sh      |   46 +-
+ tools/perf/tests/shell/diff.sh                     |    1 -
+ tools/perf/tests/shell/inject_aslr.sh              |    1 -
+ tools/perf/tests/shell/jitdump-python.sh           |    1 -
+ tools/perf/tests/shell/lib/attr.py                 |   86 +-
+ tools/perf/tests/shell/lib/perf_brstack_max.py     |   40 +
+ .../perf/tests/shell/lib/perf_json_output_lint.py  |   36 +-
+ .../perf/tests/shell/lib/perf_metric_validation.py |   53 +-
+ tools/perf/tests/shell/lib/perf_record.sh          |    2 +-
+ tools/perf/tests/shell/lib/probe_vfs_getname.sh    |   12 +-
+ tools/perf/tests/shell/lib/setup_python.sh         |   29 +-
+ tools/perf/tests/shell/lib/waiting.sh              |   34 +-
+ tools/perf/tests/shell/list.sh                     |    1 -
+ tools/perf/tests/shell/pipe_test.sh                |    1 -
+ tools/perf/tests/shell/probe_vfs_getname.sh        |    2 -
+ tools/perf/tests/shell/python-use.sh               |    1 -
+ .../tests/shell/record+probe_libc_inet_pton.sh     |    2 -
+ .../tests/shell/record+script_probe_vfs_getname.sh |    2 -
+ tools/perf/tests/shell/record.sh                   |  278 +-
+ tools/perf/tests/shell/record_weak_term.sh         |    1 -
+ tools/perf/tests/shell/report_hybrid_merge.sh      |  121 +
+ tools/perf/tests/shell/script.sh                   |   46 +-
+ tools/perf/tests/shell/script_dlfilter.sh          |    1 -
+ tools/perf/tests/shell/script_perl.sh              |  102 -
+ tools/perf/tests/shell/script_python.sh            |  113 -
+ tools/perf/tests/shell/stat+csv_output.sh          |    1 -
+ tools/perf/tests/shell/stat+json_output.sh         |    1 -
+ tools/perf/tests/shell/stat+std_output.sh          |    1 -
+ tools/perf/tests/shell/stat.sh                     |    5 +
+ tools/perf/tests/shell/stat_metrics_cgrp.sh        |    7 +
+ tools/perf/tests/shell/stat_metrics_values.sh      |    1 -
+ tools/perf/tests/shell/test_arm_callgraph_fp.sh    |    1 -
+ tools/perf/tests/shell/test_brstack.sh             |    1 -
+ .../tests/shell/test_check_perf_trace_python.sh    |   87 +
+ .../tests/shell/test_compaction_times_python.sh    |  129 +
+ tools/perf/tests/shell/test_data_symbol.sh         |    7 +-
+ .../shell/test_event_analyzing_sample_python.sh    |   70 +
+ tools/perf/tests/shell/test_event_open_fallback.sh |    2 -
+ .../shell/test_export_to_postgresql_python.sh      |  129 +
+ .../tests/shell/test_export_to_sqlite_python.sh    |  115 +
+ .../shell/test_failed_syscalls_by_pid_python.sh    |  106 +
+ .../tests/shell/test_failed_syscalls_python.sh     |   90 +
+ tools/perf/tests/shell/test_flamegraph_python.sh   |  123 +
+ .../tests/shell/test_futex_contention_python.sh    |  116 +
+ tools/perf/tests/shell/test_gecko_python.sh        |   96 +
+ tools/perf/tests/shell/test_intel_pt.sh            |   46 +-
+ .../tests/shell/test_intel_pt_events_python.sh     |   74 +
+ .../perf/tests/shell/test_mem_phys_addr_python.sh  |  105 +
+ .../tests/shell/test_net_dropmonitor_python.sh     |  103 +
+ tools/perf/tests/shell/test_netdev_times_python.sh |  104 +
+ .../tests/shell/test_perf_data_converter_json.sh   |    1 -
+ .../perf/tests/shell/test_powerpc_hcalls_python.sh |   95 +
+ tools/perf/tests/shell/test_rw_by_file_python.sh   |   69 +
+ tools/perf/tests/shell/test_rw_by_pid_python.sh    |   75 +
+ tools/perf/tests/shell/test_rwtop_python.sh        |   74 +
+ .../tests/shell/test_sched_migration_python.sh     |   72 +
+ tools/perf/tests/shell/test_sctop_python.sh        |   78 +
+ .../perf/tests/shell/test_stackcollapse_python.sh  |   78 +
+ tools/perf/tests/shell/test_stat_cpi_python.sh     |  121 +
+ .../shell/test_syscall_counts_by_pid_python.sh     |   93 +
+ .../perf/tests/shell/test_syscall_counts_python.sh |   82 +
+ tools/perf/tests/shell/test_task_analyzer.sh       |   93 +-
+ tools/perf/tests/shell/test_test_junit_output.sh   |    1 -
+ .../tests/shell/test_uprobe_from_different_cu.sh   |    1 -
+ .../perf/tests/shell/test_wakeup_latency_python.sh |   83 +
+ tools/perf/tests/shell/top.sh                      |   73 +-
+ tools/perf/tests/shell/trace+probe_vfs_getname.sh  |    1 -
+ tools/perf/tests/shell/trace_btf_enum.sh           |    1 -
+ tools/perf/tests/shell/trace_btf_general.sh        |    1 -
+ tools/perf/tests/shell/trace_exit_race.sh          |    1 -
+ tools/perf/tests/shell/trace_ksym_beautifier.sh    |   38 +
+ tools/perf/tests/shell/trace_record_replay.sh      |    2 -
+ tools/perf/tests/shell/trace_summary.sh            |    1 -
+ tools/perf/tests/symbols.c                         |   96 +-
+ tools/perf/tests/tests.h                           |    1 +
+ tools/perf/tests/thread-maps-share.c               |    3 +-
+ tools/perf/tests/tool_pmu.c                        |   79 +
+ tools/perf/tests/workloads/code_with_type.c        |    2 +-
+ tools/perf/trace/beauty/arch_errno_names.sh        |    4 +-
+ tools/perf/trace/beauty/beauty.h                   |    3 +
+ tools/perf/trace/beauty/include/uapi/linux/fs.h    |    2 +-
+ tools/perf/trace/beauty/syscalltbl.sh              |    2 +-
+ tools/perf/ui/browsers/annotate-data.c             |   12 +-
+ tools/perf/ui/browsers/annotate.c                  |   21 +-
+ tools/perf/ui/browsers/hists.c                     |    8 +-
+ tools/perf/ui/browsers/scripts.c                   |  218 +-
+ tools/perf/ui/gtk/annotate.c                       |   36 +-
+ tools/perf/ui/gtk/browser.c                        |   96 +-
+ tools/perf/ui/gtk/gtk.h                            |   16 +-
+ tools/perf/ui/gtk/hists.c                          |   72 +-
+ tools/perf/ui/gtk/progress.c                       |   40 +-
+ tools/perf/ui/gtk/setup.c                          |    5 +-
+ tools/perf/ui/gtk/util.c                           |   97 +-
+ tools/perf/ui/hist.c                               |  248 +-
+ tools/perf/ui/libslang.h                           |    2 +
+ tools/perf/ui/setup.c                              |    2 +-
+ tools/perf/util/Build                              |   10 +-
+ tools/perf/util/addr2line.c                        |   42 +-
+ tools/perf/util/annotate-arch/Build                |    1 +
+ tools/perf/util/annotate-arch/annotate-alpha.c     |  185 +
+ tools/perf/util/annotate-arch/annotate-arm64.c     |  435 +-
+ tools/perf/util/annotate-arch/annotate-loongarch.c |    4 +
+ tools/perf/util/annotate-arch/annotate-powerpc.c   |    9 +
+ tools/perf/util/annotate-arch/annotate-x86.c       |   79 +
+ tools/perf/util/annotate-data.c                    |  303 +-
+ tools/perf/util/annotate-data.h                    |   17 +-
+ tools/perf/util/annotate.c                         |  240 +-
+ tools/perf/util/annotate.h                         |  121 +-
+ tools/perf/util/arm-spe.c                          |   17 +-
+ tools/perf/util/aslr.c                             |   41 +-
+ tools/perf/util/aslr.h                             |    2 +
+ tools/perf/util/auxtrace.c                         |   12 +-
+ tools/perf/util/auxtrace.h                         |    7 +-
+ tools/perf/util/bpf-filter.l                       |    1 +
+ tools/perf/util/bpf_trace_augment.c                |    2 +
+ tools/perf/util/build-id.c                         |   12 +-
+ tools/perf/util/c2c-function.c                     |    9 +-
+ tools/perf/util/c2c.h                              |    2 +
+ tools/perf/util/cache.h                            |   31 -
+ tools/perf/util/capstone.c                         |  186 +-
+ tools/perf/util/config.c                           |   52 +-
+ tools/perf/util/cs-etm.c                           |    2 +-
+ tools/perf/util/debug.c                            |    2 +-
+ tools/perf/util/debuginfo.c                        |   51 +-
+ tools/perf/util/debuginfo.h                        |   13 +-
+ tools/perf/util/disasm.c                           |   28 +-
+ tools/perf/util/disasm.h                           |    6 +
+ tools/perf/util/dso.c                              |  194 +-
+ tools/perf/util/dso.h                              |   59 +-
+ tools/perf/util/dwarf-aux.c                        |  247 +-
+ tools/perf/util/dwarf-aux.h                        |   16 +
+ tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c |   25 +
+ tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c  |    2 +-
+ .../perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c |    2 +-
+ tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c  |    2 +-
+ tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c   |  140 +-
+ tools/perf/util/dwarf-regs.c                       |   11 +-
+ tools/perf/util/env.c                              |    1 +
+ tools/perf/util/env.h                              |   10 +
+ tools/perf/util/evlist.c                           |  236 +-
+ tools/perf/util/evlist.h                           |    3 +
+ tools/perf/util/evsel.c                            |  277 +-
+ tools/perf/util/evsel.h                            |    9 +
+ tools/perf/util/expr.c                             |    5 +-
+ tools/perf/util/genelf_debug.c                     |   28 +-
+ tools/perf/util/header.c                           |  336 +-
+ tools/perf/util/header.h                           |    1 +
+ tools/perf/util/help-unknown-cmd.c                 |   19 +-
+ tools/perf/util/help-unknown-cmd.h                 |    0
+ tools/perf/util/hist.h                             |   21 +-
+ tools/perf/util/hwmon_pmu.c                        |    2 +-
+ tools/perf/util/include/dwarf-regs.h               |    8 +-
+ tools/perf/util/intel-bts.c                        |    2 +-
+ tools/perf/util/intel-pt.c                         |    3 +-
+ tools/perf/util/jitdump.c                          |  204 +-
+ tools/perf/util/libbfd.c                           |   94 +-
+ tools/perf/util/libbfd.h                           |    9 +
+ tools/perf/util/libdw.c                            |   39 +-
+ tools/perf/util/llvm.c                             |   56 +-
+ tools/perf/util/machine.c                          |    4 +-
+ tools/perf/util/machine.h                          |   10 +-
+ tools/perf/util/mem-events.c                       |   94 +-
+ tools/perf/util/mem-events.h                       |    5 +-
+ tools/perf/util/metricgroup.c                      |   37 +-
+ tools/perf/util/parse-events.c                     |    5 +-
+ tools/perf/util/parse-regs-options.c               |  211 +-
+ tools/perf/util/path.c                             |   10 +-
+ tools/perf/util/path.h                             |    6 +-
+ tools/perf/util/perf-regs-arch/perf_regs_x86.c     |  438 +-
+ tools/perf/util/perf_event_attr_fprintf.c          |   13 +
+ tools/perf/util/perf_regs.c                        |   84 +-
+ tools/perf/util/perf_regs.h                        |   21 +-
+ tools/perf/util/pmu.c                              |   48 +-
+ tools/perf/util/pmu.h                              |    3 +
+ tools/perf/util/pmus.c                             |   21 +-
+ tools/perf/util/powerpc-vpadtl.c                   |    2 +-
+ tools/perf/util/probe-event.c                      |    4 +-
+ tools/perf/util/python.c                           | 1042 +++-
+ tools/perf/util/record.h                           |    7 +
+ tools/perf/util/sample.h                           |    5 +
+ tools/perf/util/scripting-engines/Build            |    9 -
+ .../perf/util/scripting-engines/trace-event-perl.c |  770 ---
+ .../util/scripting-engines/trace-event-python.c    | 2224 ---------
+ tools/perf/util/session.c                          |  119 +-
+ tools/perf/util/srcline.c                          |   66 +-
+ tools/perf/util/strbuf.c                           |   14 +-
+ tools/perf/util/symbol.c                           |   23 +-
+ tools/perf/util/symbol_conf.h                      |   16 +-
+ tools/perf/util/synthetic-events.c                 |   56 +-
+ tools/perf/util/thread.c                           |   18 +-
+ tools/perf/util/tool_pmu.c                         |    2 +-
+ tools/perf/util/tp_pmu.c                           |    8 +-
+ tools/perf/util/trace-event-parse.c                |   65 -
+ tools/perf/util/trace-event-scripting.c            |  407 --
+ tools/perf/util/trace-event.c                      |   90 +-
+ tools/perf/util/trace-event.h                      |   73 +-
+ tools/perf/util/trace_augment.h                    |    1 +
+ tools/perf/util/unwind-libdw.c                     |   59 +-
+ tools/perf/util/unwind-libunwind.c                 |   62 +-
+ tools/perf/util/usage.c                            |   34 -
+ tools/perf/util/util.h                             |    4 -
+ 540 files changed, 32637 insertions(+), 23820 deletions(-)
+ delete mode 100644 tools/build/feature/test-gtk2-infobar.c
+ rename tools/build/feature/{test-gtk2.c => test-gtk4.c} (76%)
+ delete mode 100644 tools/build/feature/test-libperl.c
+ delete mode 100644 tools/build/feature/test-libpython.c
+ create mode 100644 tools/build/feature/test-python-module.c
+ delete mode 100644 tools/perf/Documentation/perf-script-perl.txt
+ create mode 100644 tools/perf/arch/riscv/include/arch-tests.h
+ create mode 100644 tools/perf/arch/riscv/tests/Build
+ create mode 100644 tools/perf/arch/riscv/tests/arch-tests.c
+ create mode 100644 tools/perf/arch/riscv/tests/dwarf-unwind.c
+ create mode 100644 tools/perf/arch/riscv/tests/regs_load.S
+ create mode 100755 tools/perf/python/SchedGui.py
+ create mode 100755 tools/perf/python/arm-cs-trace-disasm.py
+ create mode 100755 tools/perf/python/check-perf-trace.py
+ create mode 100755 tools/perf/python/compaction-times.py
+ create mode 100755 tools/perf/python/event_analyzing_sample.py
+ create mode 100755 tools/perf/python/export-to-postgresql.py
+ create mode 100755 tools/perf/python/export-to-sqlite.py
+ create mode 100755 tools/perf/python/exported-sql-viewer.py
+ create mode 100755 tools/perf/python/failed-syscalls-by-pid.py
+ create mode 100755 tools/perf/python/failed-syscalls.py
+ create mode 100755 tools/perf/python/flamegraph.py
+ create mode 100755 tools/perf/python/futex-contention.py
+ create mode 100755 tools/perf/python/gecko.py
+ create mode 100755 tools/perf/python/intel-pt-events.py
+ create mode 100755 tools/perf/python/libxed.py
+ create mode 100755 tools/perf/python/mem-phys-addr.py
+ create mode 100755 tools/perf/python/net_dropmonitor.py
+ create mode 100755 tools/perf/python/netdev-times.py
+ create mode 100755 tools/perf/python/parallel-perf.py
+ rename tools/perf/{scripts => }/python/powerpc-hcalls.py (54%)
+ mode change 100644 => 100755
+ create mode 100755 tools/perf/python/rw-by-file.py
+ create mode 100755 tools/perf/python/rw-by-pid.py
+ create mode 100755 tools/perf/python/rwtop.py
+ create mode 100755 tools/perf/python/sched-migration.py
+ create mode 100755 tools/perf/python/sctop.py
+ create mode 100755 tools/perf/python/stackcollapse.py
+ create mode 100755 tools/perf/python/stat-cpi.py
+ create mode 100755 tools/perf/python/syscall-counts-by-pid.py
+ create mode 100755 tools/perf/python/syscall-counts.py
+ create mode 100755 tools/perf/python/task-analyzer.py
+ create mode 100755 tools/perf/python/treport.py
+ create mode 100755 tools/perf/python/wakeup-latency.py
+ delete mode 100644 tools/perf/scripts/Build
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/Build
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/Context.c
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/Context.xs
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/Makefile.PL
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/README
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/lib/Perf/Trace/Context.pm
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/lib/Perf/Trace/Core.pm
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/lib/Perf/Trace/Util.pm
+ delete mode 100644 tools/perf/scripts/perl/Perf-Trace-Util/typemap
+ delete mode 100644 tools/perf/scripts/perl/bin/check-perf-trace-record
+ delete mode 100644 tools/perf/scripts/perl/bin/failed-syscalls-record
+ delete mode 100644 tools/perf/scripts/perl/bin/failed-syscalls-report
+ delete mode 100644 tools/perf/scripts/perl/bin/rw-by-file-record
+ delete mode 100644 tools/perf/scripts/perl/bin/rw-by-file-report
+ delete mode 100644 tools/perf/scripts/perl/bin/rw-by-pid-record
+ delete mode 100644 tools/perf/scripts/perl/bin/rw-by-pid-report
+ delete mode 100644 tools/perf/scripts/perl/bin/rwtop-record
+ delete mode 100644 tools/perf/scripts/perl/bin/rwtop-report
+ delete mode 100644 tools/perf/scripts/perl/bin/wakeup-latency-record
+ delete mode 100644 tools/perf/scripts/perl/bin/wakeup-latency-report
+ delete mode 100644 tools/perf/scripts/perl/check-perf-trace.pl
+ delete mode 100644 tools/perf/scripts/perl/failed-syscalls.pl
+ delete mode 100644 tools/perf/scripts/perl/rw-by-file.pl
+ delete mode 100644 tools/perf/scripts/perl/rw-by-pid.pl
+ delete mode 100644 tools/perf/scripts/perl/rwtop.pl
+ delete mode 100644 tools/perf/scripts/perl/wakeup-latency.pl
+ delete mode 100644 tools/perf/scripts/python/Perf-Trace-Util/Build
+ delete mode 100644 tools/perf/scripts/python/Perf-Trace-Util/Context.c
+ delete mode 100644 tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Core.py
+ delete mode 100755 tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/EventClass.py
+ delete mode 100644 tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/SchedGui.py
+ delete mode 100644 tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py
+ delete mode 100755 tools/perf/scripts/python/arm-cs-trace-disasm.py
+ delete mode 100644 tools/perf/scripts/python/bin/compaction-times-record
+ delete mode 100644 tools/perf/scripts/python/bin/compaction-times-report
+ delete mode 100644 tools/perf/scripts/python/bin/event_analyzing_sample-record
+ delete mode 100644 tools/perf/scripts/python/bin/event_analyzing_sample-report
+ delete mode 100644 tools/perf/scripts/python/bin/export-to-postgresql-record
+ delete mode 100644 tools/perf/scripts/python/bin/export-to-postgresql-report
+ delete mode 100644 tools/perf/scripts/python/bin/export-to-sqlite-record
+ delete mode 100644 tools/perf/scripts/python/bin/export-to-sqlite-report
+ delete mode 100644 tools/perf/scripts/python/bin/failed-syscalls-by-pid-record
+ delete mode 100644 tools/perf/scripts/python/bin/failed-syscalls-by-pid-report
+ delete mode 100755 tools/perf/scripts/python/bin/flamegraph-record
+ delete mode 100755 tools/perf/scripts/python/bin/flamegraph-report
+ delete mode 100644 tools/perf/scripts/python/bin/futex-contention-record
+ delete mode 100644 tools/perf/scripts/python/bin/futex-contention-report
+ delete mode 100644 tools/perf/scripts/python/bin/gecko-record
+ delete mode 100755 tools/perf/scripts/python/bin/gecko-report
+ delete mode 100644 tools/perf/scripts/python/bin/intel-pt-events-record
+ delete mode 100644 tools/perf/scripts/python/bin/intel-pt-events-report
+ delete mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record
+ delete mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report
+ delete mode 100755 tools/perf/scripts/python/bin/net_dropmonitor-record
+ delete mode 100755 tools/perf/scripts/python/bin/net_dropmonitor-report
+ delete mode 100644 tools/perf/scripts/python/bin/netdev-times-record
+ delete mode 100644 tools/perf/scripts/python/bin/netdev-times-report
+ delete mode 100644 tools/perf/scripts/python/bin/powerpc-hcalls-record
+ delete mode 100644 tools/perf/scripts/python/bin/powerpc-hcalls-report
+ delete mode 100644 tools/perf/scripts/python/bin/sched-migration-record
+ delete mode 100644 tools/perf/scripts/python/bin/sched-migration-report
+ delete mode 100644 tools/perf/scripts/python/bin/sctop-record
+ delete mode 100644 tools/perf/scripts/python/bin/sctop-report
+ delete mode 100755 tools/perf/scripts/python/bin/stackcollapse-record
+ delete mode 100755 tools/perf/scripts/python/bin/stackcollapse-report
+ delete mode 100644 tools/perf/scripts/python/bin/syscall-counts-by-pid-record
+ delete mode 100644 tools/perf/scripts/python/bin/syscall-counts-by-pid-report
+ delete mode 100644 tools/perf/scripts/python/bin/syscall-counts-record
+ delete mode 100644 tools/perf/scripts/python/bin/syscall-counts-report
+ delete mode 100755 tools/perf/scripts/python/bin/task-analyzer-record
+ delete mode 100755 tools/perf/scripts/python/bin/task-analyzer-report
+ delete mode 100644 tools/perf/scripts/python/check-perf-trace.py
+ delete mode 100644 tools/perf/scripts/python/compaction-times.py
+ delete mode 100644 tools/perf/scripts/python/event_analyzing_sample.py
+ delete mode 100644 tools/perf/scripts/python/export-to-postgresql.py
+ delete mode 100644 tools/perf/scripts/python/export-to-sqlite.py
+ delete mode 100755 tools/perf/scripts/python/exported-sql-viewer.py
+ delete mode 100644 tools/perf/scripts/python/failed-syscalls-by-pid.py
+ delete mode 100755 tools/perf/scripts/python/flamegraph.py
+ delete mode 100644 tools/perf/scripts/python/futex-contention.py
+ delete mode 100644 tools/perf/scripts/python/gecko.py
+ delete mode 100644 tools/perf/scripts/python/intel-pt-events.py
+ delete mode 100644 tools/perf/scripts/python/libxed.py
+ delete mode 100644 tools/perf/scripts/python/mem-phys-addr.py
+ delete mode 100755 tools/perf/scripts/python/net_dropmonitor.py
+ delete mode 100644 tools/perf/scripts/python/netdev-times.py
+ delete mode 100755 tools/perf/scripts/python/parallel-perf.py
+ delete mode 100644 tools/perf/scripts/python/sched-migration.py
+ delete mode 100644 tools/perf/scripts/python/sctop.py
+ delete mode 100755 tools/perf/scripts/python/stackcollapse.py
+ delete mode 100644 tools/perf/scripts/python/stat-cpi.py
+ delete mode 100644 tools/perf/scripts/python/syscall-counts-by-pid.py
+ delete mode 100644 tools/perf/scripts/python/syscall-counts.py
+ delete mode 100755 tools/perf/scripts/python/task-analyzer.py
+ create mode 100644 tools/perf/tests/hybrid-merge.c
+ create mode 100755 tools/perf/tests/shell/annotate_weight.sh
+ create mode 100644 tools/perf/tests/shell/lib/perf_brstack_max.py
+ create mode 100755 tools/perf/tests/shell/report_hybrid_merge.sh
+ delete mode 100755 tools/perf/tests/shell/script_perl.sh
+ delete mode 100755 tools/perf/tests/shell/script_python.sh
+ create mode 100755 tools/perf/tests/shell/test_check_perf_trace_python.sh
+ create mode 100755 tools/perf/tests/shell/test_compaction_times_python.sh
+ create mode 100755 tools/perf/tests/shell/test_event_analyzing_sample_python.sh
+ create mode 100755 tools/perf/tests/shell/test_export_to_postgresql_python.sh
+ create mode 100755 tools/perf/tests/shell/test_export_to_sqlite_python.sh
+ create mode 100755 tools/perf/tests/shell/test_failed_syscalls_by_pid_python.sh
+ create mode 100755 tools/perf/tests/shell/test_failed_syscalls_python.sh
+ create mode 100755 tools/perf/tests/shell/test_flamegraph_python.sh
+ create mode 100755 tools/perf/tests/shell/test_futex_contention_python.sh
+ create mode 100755 tools/perf/tests/shell/test_gecko_python.sh
+ create mode 100755 tools/perf/tests/shell/test_intel_pt_events_python.sh
+ create mode 100755 tools/perf/tests/shell/test_mem_phys_addr_python.sh
+ create mode 100755 tools/perf/tests/shell/test_net_dropmonitor_python.sh
+ create mode 100755 tools/perf/tests/shell/test_netdev_times_python.sh
+ create mode 100755 tools/perf/tests/shell/test_powerpc_hcalls_python.sh
+ create mode 100755 tools/perf/tests/shell/test_rw_by_file_python.sh
+ create mode 100755 tools/perf/tests/shell/test_rw_by_pid_python.sh
+ create mode 100755 tools/perf/tests/shell/test_rwtop_python.sh
+ create mode 100755 tools/perf/tests/shell/test_sched_migration_python.sh
+ create mode 100755 tools/perf/tests/shell/test_sctop_python.sh
+ create mode 100755 tools/perf/tests/shell/test_stackcollapse_python.sh
+ create mode 100755 tools/perf/tests/shell/test_stat_cpi_python.sh
+ create mode 100755 tools/perf/tests/shell/test_syscall_counts_by_pid_python.sh
+ create mode 100755 tools/perf/tests/shell/test_syscall_counts_python.sh
+ create mode 100755 tools/perf/tests/shell/test_wakeup_latency_python.sh
+ create mode 100755 tools/perf/tests/shell/trace_ksym_beautifier.sh
+ create mode 100644 tools/perf/util/annotate-arch/annotate-alpha.c
+ delete mode 100644 tools/perf/util/cache.h
+ delete mode 100644 tools/perf/util/help-unknown-cmd.h
+ delete mode 100644 tools/perf/util/scripting-engines/Build
+ delete mode 100644 tools/perf/util/scripting-engines/trace-event-perl.c
+ delete mode 100644 tools/perf/util/scripting-engines/trace-event-python.c
+ delete mode 100644 tools/perf/util/trace-event-scripting.c
+ delete mode 100644 tools/perf/util/usage.c
+Merging compiler-attributes/compiler-attributes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'compiler-attributes' of https://github.com/ojeda/linux.git compiler-attributes/compiler-attributes
+Already up to date.
+Merging dma-mapping/dma-mapping-for-next (57a57ae077f0f MAINTAINERS: Add files to the DMA MAPPING HELPERS entry)
+$ git merge -m Merge branch 'dma-mapping-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping/dma-mapping-for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/iommu/dma-iommu.c
+Auto-merging drivers/nvme/host/pci.c
+Auto-merging mm/page_alloc.c
+Merge made by the 'ort' strategy.
+ Documentation/core-api/dma-api.rst |  2 +-
+ MAINTAINERS                        |  1 +
+ drivers/iommu/dma-iommu.c          | 12 +++++++-----
+ drivers/nvme/host/pci.c            |  2 +-
+ drivers/scsi/scsi_transport_sas.c  | 12 ++++++------
+ include/linux/dma-map-ops.h        |  4 ++--
+ include/linux/dma-mapping.h        |  4 ++--
+ include/linux/gfp.h                |  2 +-
+ include/linux/iommu-dma.h          |  2 +-
+ kernel/dma/contiguous.c            |  6 +++---
+ kernel/dma/direct.c                |  8 +++++---
+ kernel/dma/mapping.c               | 10 +++++-----
+ kernel/dma/ops_helpers.c           | 11 +++++++----
+ mm/page_alloc.c                    |  2 +-
+ 14 files changed, 43 insertions(+), 35 deletions(-)
+Merging asm-generic/master (adbbd9714f805 scripts: headers_install.sh: Remove config leak ignore machinery)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic asm-generic/master
+Already up to date.
+Merging alpha/alpha-next (d58041d2c63e0 MAINTAINERS: Add Magnus Lindholm as maintainer for alpha port)
+$ git merge -m Merge branch 'alpha-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git alpha/alpha-next
+Already up to date.
+Merging arm/for-next (1a89abc009cb5 Merge branches 'fixes' and 'misc' into for-linus)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm/for-next
+Already up to date.
+Merging arm64/for-next/core (6c360697fd4d1 Merge branches 'for-next/misc', 'for-next/be-gone', 'for-next/smccc-bus' and 'for-next/preemptible-this-cpu' into for-next/core)
+$ git merge -m Merge branch 'for-next/core' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64/for-next/core
+Auto-merging Documentation/arch/arm64/booting.rst
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/Kconfig
+Auto-merging arch/arm64/include/asm/io.h
+Auto-merging arch/arm64/kernel/kexec_image.c
+Auto-merging arch/arm64/mm/pageattr.c
+CONFLICT (content): Merge conflict in arch/arm64/mm/pageattr.c
+Auto-merging arch/arm64/net/bpf_jit_comp.c
+Resolved 'arch/arm64/mm/pageattr.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 1e2c745c23532] Merge branch 'for-next/core' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/sysfs-firmware-cca       |   7 +
+ Documentation/arch/arm64/booting.rst               |   2 +-
+ MAINTAINERS                                        |   6 +
+ arch/arm/include/asm/archrandom.h                  |   2 +-
+ arch/arm64/Kconfig                                 |  36 +-
+ arch/arm64/Makefile                                |  12 +-
+ arch/arm64/crypto/aes-ce-ccm-core.S                |   4 +-
+ arch/arm64/crypto/aes-neonbs-core.S                |   4 +-
+ arch/arm64/crypto/ghash-ce-core.S                  |   6 +-
+ arch/arm64/crypto/sm4-ce-cipher-core.S             |   4 +-
+ arch/arm64/include/asm/alternative-macros.h        |  30 +-
+ arch/arm64/include/asm/archrandom.h                |   2 +-
+ arch/arm64/include/asm/assembler.h                 |  31 --
+ arch/arm64/include/asm/atomic_ll_sc.h              |  27 +-
+ arch/arm64/include/asm/atomic_lse.h                |   4 +-
+ arch/arm64/include/asm/compat.h                    |  14 -
+ arch/arm64/include/asm/elf.h                       |  12 -
+ arch/arm64/include/asm/gpr-num.h                   |   9 +
+ arch/arm64/include/asm/io.h                        |   2 +-
+ arch/arm64/include/asm/mem_encrypt.h               |   5 +-
+ arch/arm64/include/asm/mte-kasan.h                 |  10 +-
+ arch/arm64/include/asm/percpu.h                    | 462 ++++++++++++++++-----
+ arch/arm64/include/asm/pgtable-prot.h              |   4 +-
+ arch/arm64/include/asm/preempt.h                   |  31 +-
+ arch/arm64/include/asm/processor.h                 |   2 -
+ arch/arm64/include/asm/ptrace.h                    |  18 +-
+ arch/arm64/include/asm/rsi.h                       |  70 ----
+ arch/arm64/include/asm/stage2_pgtable.h            |   2 +-
+ arch/arm64/include/asm/sysreg.h                    |  52 +--
+ arch/arm64/include/asm/thread_info.h               |   6 +-
+ arch/arm64/include/asm/word-at-a-time.h            |   6 -
+ arch/arm64/include/asm/xwreg.h                     |  16 +
+ arch/arm64/include/uapi/asm/byteorder.h            |   4 -
+ arch/arm64/include/uapi/asm/ptrace.h               |   4 +-
+ arch/arm64/kernel/Makefile                         |   2 +-
+ arch/arm64/kernel/asm-offsets.c                    |   2 +
+ arch/arm64/kernel/cpufeature.c                     |   6 +-
+ arch/arm64/kernel/entry-common.c                   |  38 ++
+ arch/arm64/kernel/entry.S                          |  41 +-
+ arch/arm64/kernel/fpsimd.c                         |  21 +-
+ arch/arm64/kernel/head.S                           |   3 +-
+ arch/arm64/kernel/image.h                          |  18 +-
+ arch/arm64/kernel/kexec_image.c                    |   5 +-
+ arch/arm64/kernel/kgdb.c                           |   6 +-
+ arch/arm64/kernel/ptrace.c                         |   8 +-
+ arch/arm64/kernel/setup.c                          |   2 +-
+ arch/arm64/kernel/signal32.c                       |  13 -
+ arch/arm64/kernel/sys32.c                          |   5 -
+ arch/arm64/kernel/vdso32/Makefile                  |   4 -
+ arch/arm64/kvm/hyp/include/nvhe/spinlock.h         |   4 -
+ arch/arm64/kvm/hyp/nvhe/gen-hyprel.c               |  16 -
+ arch/arm64/lib/csum.c                              |  17 -
+ arch/arm64/lib/memchr.S                            |   2 +-
+ arch/arm64/lib/memcmp.S                            |   2 -
+ arch/arm64/lib/strcmp.S                            |  34 +-
+ arch/arm64/lib/strlen.S                            |  26 --
+ arch/arm64/lib/strncmp.S                           |  93 +----
+ arch/arm64/lib/strnlen.S                           |  16 +-
+ arch/arm64/mm/extable.c                            |   5 -
+ arch/arm64/mm/init.c                               |   3 +-
+ arch/arm64/mm/pageattr.c                           |   4 +-
+ arch/arm64/net/bpf_jit_comp.c                      |  11 -
+ drivers/char/hw_random/arm_smccc_trng.c            |  32 +-
+ drivers/crypto/hisilicon/sec2/sec_main.c           |   2 +-
+ drivers/firmware/Kconfig                           |   1 +
+ drivers/firmware/Makefile                          |   1 +
+ drivers/firmware/arm_rmm/Kconfig                   |  17 +
+ drivers/firmware/arm_rmm/Makefile                  |   2 +
+ .../kernel => drivers/firmware/arm_rmm}/rsi.c      |  79 +++-
+ drivers/firmware/smccc/Makefile                    |   2 +-
+ drivers/firmware/smccc/bus.c                       | 158 +++++++
+ drivers/firmware/smccc/smccc.c                     |  66 ++-
+ drivers/hv/Kconfig                                 |   3 +-
+ drivers/misc/vmw_vmci/Kconfig                      |   2 +-
+ drivers/net/ethernet/google/Kconfig                |   2 +-
+ drivers/net/ethernet/microsoft/Kconfig             |   2 +-
+ drivers/soc/tegra/Kconfig                          |   5 -
+ drivers/virt/coco/arm-cca-guest/Kconfig            |   3 +-
+ drivers/virt/coco/arm-cca-guest/Makefile           |   2 +
+ .../coco/arm-cca-guest/{arm-cca-guest.c => main.c} |  52 +--
+ .../asm/rsi_cmds.h => include/linux/arm-rsi-cmds.h |  76 +++-
+ include/linux/arm-smccc-bus.h                      |  48 +++
+ .../asm/rsi_smc.h => include/linux/arm-smccc-rsi.h |   6 +-
+ include/linux/device-id/arm_smccc.h                |  19 +
+ include/linux/mod_devicetable.h                    |   1 +
+ scripts/mod/devicetable-offsets.c                  |   3 +
+ scripts/mod/file2alias.c                           |   8 +
+ tools/testing/selftests/arm64/fp/fp-ptrace.c       |  19 +-
+ .../testcases/fake_sigreturn_sve_change_vl.c       |   2 +-
+ tools/testing/selftests/rseq/rseq-arm64.h          |   5 -
+ 90 files changed, 1107 insertions(+), 824 deletions(-)
+ create mode 100644 Documentation/ABI/testing/sysfs-firmware-cca
+ delete mode 100644 arch/arm64/include/asm/rsi.h
+ create mode 100644 arch/arm64/include/asm/xwreg.h
+ create mode 100644 drivers/firmware/arm_rmm/Kconfig
+ create mode 100644 drivers/firmware/arm_rmm/Makefile
+ rename {arch/arm64/kernel => drivers/firmware/arm_rmm}/rsi.c (71%)
+ create mode 100644 drivers/firmware/smccc/bus.c
+ rename drivers/virt/coco/arm-cca-guest/{arm-cca-guest.c => main.c} (82%)
+ rename arch/arm64/include/asm/rsi_cmds.h => include/linux/arm-rsi-cmds.h (69%)
+ create mode 100644 include/linux/arm-smccc-bus.h
+ rename arch/arm64/include/asm/rsi_smc.h => include/linux/arm-smccc-rsi.h (98%)
+ create mode 100644 include/linux/device-id/arm_smccc.h
+$ git am -3 ../patches/0001-arm64-fixup-merge-with-dropped-copy-of-code-getting-.patch
+Applying: arm64: fixup merge with dropped copy of code getting kept
+$ git reset HEAD^
+Unstaged changes after reset:
+M	arch/arm64/mm/pageattr.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master b7c7aa664e16b] Merge branch 'for-next/core' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
+ Date: Wed Sep 30 12:47:14 2026 +0100
+Merging arm-perf/for-next/perf (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-next/perf' of https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git arm-perf/for-next/perf
+Already up to date.
+Merging arm-soc/for-next (b9bb4134a9583 soc: document merges)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/arm-soc-for-next-contents.txt | 22 ++++++++++++++++++++++
+ 1 file changed, 22 insertions(+)
+ create mode 100644 arch/arm/arm-soc-for-next-contents.txt
+Merging amlogic/for-next (a2530c83c458a Merge branch 'v7.4/arm64-dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git amlogic/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi               | 15 +++++++++------
+ .../arm64/boot/dts/amlogic/meson-g12b-odroid-go-ultra.dts |  2 +-
+ arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts      |  2 +-
+ arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi               |  2 +-
+ arch/arm64/boot/dts/amlogic/meson-gxl.dtsi                |  2 +-
+ 5 files changed, 13 insertions(+), 10 deletions(-)
+Merging asahi-soc/asahi-soc/for-next (9378cd5ddcebc Merge branch 'apple-soc/drivers-7.4' into asahi-soc/for-next)
+$ git merge -m Merge branch 'asahi-soc/for-next' of https://github.com/AsahiLinux/linux.git asahi-soc/asahi-soc/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/apple.yaml   |   32 +
+ .../devicetree/bindings/arm/apple/apple,pmgr.yaml  |    2 +
+ Documentation/devicetree/bindings/arm/cpus.yaml    |    4 +
+ .../devicetree/bindings/i2c/apple,i2c.yaml         |    1 +
+ .../bindings/interrupt-controller/apple,aic2.yaml  |    2 +
+ .../devicetree/bindings/pinctrl/apple,pinctrl.yaml |    1 +
+ .../bindings/power/apple,pmgr-pwrstate.yaml        |    2 +
+ .../devicetree/bindings/pwm/apple,s5l-fpwm.yaml    |    1 +
+ arch/arm64/Kconfig.platforms                       |    1 +
+ arch/arm64/boot/dts/apple/Makefile                 |    7 +
+ arch/arm64/boot/dts/apple/t6001.dtsi               |   11 +-
+ arch/arm64/boot/dts/apple/t6002.dtsi               |   18 +-
+ arch/arm64/boot/dts/apple/t600x-die0.dtsi          |    3 +-
+ arch/arm64/boot/dts/apple/t600x-dieX.dtsi          |    2 +
+ arch/arm64/boot/dts/apple/t600x-nvme.dtsi          |    2 +
+ arch/arm64/boot/dts/apple/t6021.dtsi               |   10 +-
+ arch/arm64/boot/dts/apple/t6022.dtsi               |   18 +-
+ arch/arm64/boot/dts/apple/t602x-common.dtsi        |    2 +-
+ arch/arm64/boot/dts/apple/t602x-die0.dtsi          |    2 +
+ arch/arm64/boot/dts/apple/t602x-dieX.dtsi          |    2 +
+ arch/arm64/boot/dts/apple/t602x-nvme.dtsi          |    2 +
+ arch/arm64/boot/dts/apple/t6030-pmgr.dtsi          |    1 -
+ arch/arm64/boot/dts/apple/t6030.dtsi               |  277 ++---
+ arch/arm64/boot/dts/apple/t6031-base.dtsi          |  191 ++--
+ arch/arm64/boot/dts/apple/t6031-die0.dtsi          |   63 +-
+ arch/arm64/boot/dts/apple/t6031-dieX.dtsi          |   94 +-
+ arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi     |   18 +-
+ arch/arm64/boot/dts/apple/t6031-pmgr.dtsi          |    6 +
+ arch/arm64/boot/dts/apple/t6031.dtsi               |   16 +-
+ arch/arm64/boot/dts/apple/t6032-j575d.dts          |   10 +-
+ arch/arm64/boot/dts/apple/t6032.dtsi               |  193 ++--
+ arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi     |   15 +-
+ arch/arm64/boot/dts/apple/t8122-j504.dts           |   10 +-
+ arch/arm64/boot/dts/apple/t8122-j613.dts           |    9 +-
+ arch/arm64/boot/dts/apple/t8122-j615.dts           |    9 +-
+ arch/arm64/boot/dts/apple/t8122-jxxx.dtsi          |    5 +-
+ arch/arm64/boot/dts/apple/t8122-pmgr.dtsi          |    1 -
+ arch/arm64/boot/dts/apple/t8122.dtsi               |  256 +++--
+ arch/arm64/boot/dts/apple/t8132-j604.dts           |   35 +
+ arch/arm64/boot/dts/apple/t8132-j623.dts           |   18 +
+ arch/arm64/boot/dts/apple/t8132-j624.dts           |   18 +
+ arch/arm64/boot/dts/apple/t8132-j713.dts           |   35 +
+ arch/arm64/boot/dts/apple/t8132-j715.dts           |   35 +
+ arch/arm64/boot/dts/apple/t8132-j773g.dts          |   25 +
+ arch/arm64/boot/dts/apple/t8132-jxxx.dtsi          |   48 +
+ arch/arm64/boot/dts/apple/t8132-pmgr.dtsi          | 1130 ++++++++++++++++++++
+ arch/arm64/boot/dts/apple/t8132.dtsi               |  467 ++++++++
+ arch/arm64/boot/dts/apple/t8140-j700.dts           |   54 +
+ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi          |  777 ++++++++++++++
+ arch/arm64/boot/dts/apple/t8140.dtsi               |  207 ++++
+ drivers/soc/apple/rtkit-crashlog.c                 |  141 ++-
+ 51 files changed, 3715 insertions(+), 574 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-j604.dts
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-j623.dts
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-j624.dts
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-j713.dts
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-j715.dts
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-j773g.dts
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-jxxx.dtsi
+ create mode 100644 arch/arm64/boot/dts/apple/t8132-pmgr.dtsi
+ create mode 100644 arch/arm64/boot/dts/apple/t8132.dtsi
+ create mode 100644 arch/arm64/boot/dts/apple/t8140-j700.dts
+ create mode 100644 arch/arm64/boot/dts/apple/t8140-pmgr.dtsi
+ create mode 100644 arch/arm64/boot/dts/apple/t8140.dtsi
+Merging at91/at91-next (bd49a1c72abbd Merge branch 'microchip-dt64' into at91-next)
+$ git merge -m Merge branch 'at91-next' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91/at91-next
+Merge made by the 'ort' strategy.
+ arch/arm/mach-at91/pm.h                            |  2 +-
+ arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts | 13 +++++++++++++
+ drivers/clk/at91/clk-main.c                        |  2 +-
+ drivers/soc/atmel/soc.c                            |  4 +---
+ 4 files changed, 16 insertions(+), 5 deletions(-)
+Merging bmc/for-next (cd7d1ef7d74ed Merge branches 'aspeed/drivers', 'aspeed/arm/dt', 'aspeed/fixes/drivers', 'aspeed/maintainers', 'nuvoton/arm/dt', 'nuvoton/arm/fixes' and 'nuvoton/arm64/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git bmc/for-next
+Merge made by the 'ort' strategy.
+Merging broadcom/next (c0f2033a9ce4b Merge branch 'devicetree-arm64/next' into next)
+$ git merge -m Merge branch 'next' of https://github.com/Broadcom/stblinux.git broadcom/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/bcm/bcm2835.yaml       |   6 ++
+ arch/arm/boot/dts/broadcom/bcm-ns.dtsi             |  13 +++
+ .../dts/broadcom/bcm4708-linksys-ea6300-v1.dts     |   4 +
+ .../boot/dts/broadcom/bcm4708-smartrg-sr400ac.dts  |  18 ++++
+ .../boot/dts/broadcom/bcm4709-linksys-ea9200.dts   |  62 +++++++++++
+ .../boot/dts/broadcom/bcm4709-netgear-r8000.dts    |  12 +++
+ .../boot/dts/broadcom/bcm47094-dlink-dir-890l.dts  |   6 +-
+ .../dts/broadcom/bcm47094-linksys-panamera.dts     |   2 +-
+ arch/arm/boot/dts/broadcom/bcm47189-tenda-ac9.dts  |  20 ++++
+ arch/arm/boot/dts/broadcom/bcm53573.dtsi           |   9 +-
+ arch/arm/boot/dts/broadcom/bcm7445.dtsi            |   2 +-
+ arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts |   5 +
+ .../boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi    |  41 +++++++
+ arch/arm64/boot/dts/broadcom/bcm2712.dtsi          |  77 +++++++++++++
+ arch/arm64/boot/dts/broadcom/rp1-common.dtsi       | 119 +++++++++++++++++++++
+ 15 files changed, 392 insertions(+), 4 deletions(-)
+Merging cix/for-next (a0cffbd8878c5 Merge remote-tracking branch 'cix/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git cix/for-next
+Merge made by the 'ort' strategy.
+Merging davinci/davinci/for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'davinci/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci/davinci/for-next
+Already up to date.
+Merging drivers-memory/for-next (a22355280361d memory: stm32_omm: fix child clock leak on set_amcr() error path)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory/for-next
+Merge made by the 'ort' strategy.
+ .../memory-controllers/mediatek,smi-common.yaml    |  21 +-
+ .../memory-controllers/mediatek,smi-larb.yaml      |   3 +
+ drivers/memory/brcmstb_dpfe.c                      |  41 ++-
+ drivers/memory/emif.c                              |   6 +-
+ drivers/memory/emif.h                              |   4 +-
+ drivers/memory/mtk-smi.c                           |  45 +++
+ drivers/memory/renesas-rpc-if.c                    |  87 ++---
+ drivers/memory/samsung/exynos5422-dmc.c            |  11 +-
+ drivers/memory/stm32_omm.c                         |   2 +
+ drivers/memory/tegra/tegra264.c                    | 392 +++++++++++++--------
+ 10 files changed, 398 insertions(+), 214 deletions(-)
+Merging fsl/soc_fsl (7b3b0598c00e6 soc: fsl: cpm1: tsa: Fix clock leak in tsa_of_parse_tdms())
+$ git merge -m Merge branch 'soc_fsl' of https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git fsl/soc_fsl
+Merge made by the 'ort' strategy.
+ drivers/bus/fsl-mc/fsl-mc-bus.c | 47 +++++++++++++++++------------
+ drivers/bus/fsl-mc/fsl-mc-msi.c |  2 +-
+ drivers/soc/fsl/qe/gpio.c       | 67 ++++++++++++++++-------------------------
+ drivers/soc/fsl/qe/tsa.c        |  8 ++---
+ drivers/usb/host/fhci-hcd.c     |  2 +-
+ include/soc/fsl/qe/qe.h         |  6 ++--
+ 6 files changed, 64 insertions(+), 68 deletions(-)
+Merging imx-mxs/for-next (5d034f17b2464 Merge branch 'imx/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git imx-mxs/for-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/se-cdev                  |   45 +
+ Documentation/devicetree/bindings/arm/fsl.yaml     |   68 +-
+ .../bindings/display/bridge/fsl,ldb.yaml           |   23 +-
+ .../devicetree/bindings/display/fsl,lcdif.yaml     |    1 +
+ .../devicetree/bindings/firmware/fsl,imx-se.yaml   |   91 +
+ .../bindings/soc/imx/fsl,imx-iomuxc-gpr.yaml       |   62 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |    2 +
+ .../driver-api/firmware/other_interfaces.rst       |  238 ++
+ arch/arm/boot/dts/nxp/imx/Makefile                 |   49 +
+ arch/arm/boot/dts/nxp/imx/e60k02.dtsi              |    8 +-
+ arch/arm/boot/dts/nxp/imx/e70k02.dtsi              |    8 +-
+ arch/arm/boot/dts/nxp/imx/imx50-kobo-aura.dts      |    7 +-
+ arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts       |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx53-qsb.dts            |    2 +-
+ .../arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-b125pv2.dts       |    2 +
+ arch/arm/boot/dts/nxp/imx/imx6dl-b125v2.dts        |    2 +
+ arch/arm/boot/dts/nxp/imx/imx6dl-mamoj.dts         |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts        |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-prtvt7.dts        |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts        |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx6q-bosch-acc.dts      |    4 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-emcon.dtsi       |    1 -
+ arch/arm/boot/dts/nxp/imx/imx6qdl-gw560x.dtsi      |   20 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-pico.dtsi        |    4 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi    |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-wandboard.dtsi   |    4 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl-zii-rdu2.dtsi    |   10 +-
+ arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi             |   10 +-
+ arch/arm/boot/dts/nxp/imx/imx6sl-kobo-aura2.dts    |    8 +-
+ .../boot/dts/nxp/imx/imx6sl-tolino-shine2hd.dts    |    8 +-
+ arch/arm/boot/dts/nxp/imx/imx6sll-evk.dts          |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx6sll.dtsi             |   11 -
+ arch/arm/boot/dts/nxp/imx/imx6ul-14x14-evk.dtsi    |    1 -
+ .../boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi |   38 +-
+ arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi        |    2 +-
+ .../arm/boot/dts/nxp/imx/imx7-colibri-iris-v2.dtsi |    1 -
+ arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi          |   52 +-
+ ...nel-cap-touch-7inch-parallel-touch-adapter.dtso |   58 +
+ ...olibri-emmc-panel-cap-touch-7inch-parallel.dtso |   43 +
+ ...olibri-emmc-panel-res-touch-7inch-parallel.dtso |   32 +
+ .../boot/dts/nxp/imx/imx7d-colibri-emmc-vga.dtso   |   59 +
+ arch/arm/boot/dts/nxp/imx/imx7d-meerkat96.dts      |    2 +-
+ arch/arm/boot/dts/nxp/imx/imx7d-nitrogen7.dts      |    1 -
+ .../nxp/imx/imx7d-var-som-emmc-mx7customboard.dts  |   23 +
+ .../imx7d-var-som-emmc-wm8731-mx7customboard.dts   |   23 +
+ arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc.dtsi  |   56 +
+ .../dts/nxp/imx/imx7d-var-som-mx7customboard.dtsi  |  358 +++
+ .../nxp/imx/imx7d-var-som-nand-mx7customboard.dts  |   23 +
+ .../imx7d-var-som-nand-wm8731-mx7customboard.dts   |   23 +
+ arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand.dtsi  |   73 +
+ .../imx/imx7d-var-som-v2-emmc-mx7customboard.dts   |   24 +
+ .../imx/imx7d-var-som-v2-nand-mx7customboard.dts   |   24 +
+ arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2.dtsi    |   66 +
+ .../arm/boot/dts/nxp/imx/imx7d-var-som-wm8731.dtsi |  102 +
+ .../arm/boot/dts/nxp/imx/imx7d-var-som-wm8904.dtsi |  118 +
+ arch/arm/boot/dts/nxp/imx/imx7d-var-som.dtsi       |  533 +++++
+ arch/arm/boot/dts/nxp/imx/imx7s-warp.dts           |    2 +-
+ arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi             |   36 +-
+ arch/arm/boot/dts/nxp/ls/ls1021a.dtsi              |   10 +-
+ arch/arm/boot/dts/nxp/vf/Makefile                  |    2 +
+ arch/arm/boot/dts/nxp/vf/vf-colibri-iris.dtsi      |  126 +
+ arch/arm/boot/dts/nxp/vf/vf-colibri.dtsi           |   56 +
+ arch/arm/boot/dts/nxp/vf/vf500-colibri-iris.dts    |   14 +
+ arch/arm/boot/dts/nxp/vf/vf500.dtsi                |    6 +
+ arch/arm/boot/dts/nxp/vf/vf610-colibri-iris.dts    |   14 +
+ arch/arm/boot/dts/nxp/vf/vf610.dtsi                |    3 +
+ arch/arm/include/asm/linkage.h                     |   29 +
+ arch/arm/mach-imx/hardware.h                       |    2 +-
+ arch/arm/mach-imx/mach-imx6q.c                     |   30 -
+ arch/arm/mach-imx/mxc.h                            |    2 +-
+ arch/arm/mach-imx/pm-imx6.c                        |   26 +-
+ arch/arm/mach-imx/suspend-imx6.S                   |    6 +-
+ arch/arm64/boot/dts/freescale/Makefile             |  266 ++-
+ arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi     |    2 +-
+ .../arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dts |   34 +
+ .../boot/dts/freescale/fsl-lx2160a-nbxv3.dtsi      |  330 +++
+ arch/arm64/boot/dts/freescale/fsl-lx216x.dtsi      |   19 +-
+ .../arm64/boot/dts/freescale/imx8-apalis-eval.dtsi |    6 +-
+ .../boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi |    6 +-
+ .../boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi |    6 +-
+ .../arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi |   16 +-
+ arch/arm64/boot/dts/freescale/imx8dxl-evk.dts      |   14 +-
+ arch/arm64/boot/dts/freescale/imx8dxl-sr-som.dtsi  |   14 +-
+ .../dts/freescale/imx8mm-beacon-baseboard.dtsi     |    2 +-
+ .../imx8mm-data-modul-edm-sbc-overlay-cm4.dtso     |   56 +
+ ...odul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi |   59 +
+ ...-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtsi |   99 +
+ ...-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso |   79 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtsi |   69 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso |   59 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtsi |   94 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso |   17 +
+ ...edm-sbc-overlay-edm-mod-imx8mm-lvds-common.dtsi |  118 +
+ ...l-edm-sbc-overlay-edm-mod-imx8mm-lvds-dual.dtsi |   32 +
+ ...sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtsi |   12 +
+ ...sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso |    7 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtsi |   12 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso |    7 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtsi |   12 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso |    7 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtsi |   12 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso |    7 +
+ ...sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtsi |   12 +
+ ...sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso |    7 +
+ ...c-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtsi |   12 +
+ ...c-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso |    7 +
+ ...verlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtsi |   12 +
+ ...verlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso |    7 +
+ ...edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi |   20 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi |   22 +
+ ...odul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtso |   18 +
+ ...imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi |   19 +
+ ...imx8mm-data-modul-edm-sbc-overlay-lvds-5v0.dtsi |   19 +
+ ...mx8mm-data-modul-edm-sbc-overlay-lvds-dual.dtsi |   29 +
+ ...data-modul-edm-sbc-overlay-lvds-g070y2-l01.dtsi |   31 +
+ ...ata-modul-edm-sbc-overlay-lvds-g101ice-l01.dtsi |   31 +
+ ...ata-modul-edm-sbc-overlay-lvds-g121xce-l01.dtsi |   31 +
+ ...ata-modul-edm-sbc-overlay-lvds-g156hce-l01.dtsi |   31 +
+ ...data-modul-edm-sbc-overlay-lvds-g215hvn011.dtsi |   30 +
+ ...ta-modul-edm-sbc-overlay-lvds-mi0700a2t-30.dtsi |   31 +
+ ...modul-edm-sbc-overlay-lvds-mi1010z1t-1cp11.dtsi |   31 +
+ ...8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi |   13 +
+ .../dts/freescale/imx8mm-data-modul-edm-sbc.dts    |   64 +-
+ .../arm64/boot/dts/freescale/imx8mm-kontron-bl.dts |  107 +-
+ arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi |   21 +-
+ .../boot/dts/freescale/imx8mm-verdin-ivy.dtsi      |    4 +-
+ .../dts/freescale/imx8mn-solidsense-n8-compact.dts |    2 -
+ .../freescale/imx8mn-vhip4-evalboard-common.dtsi   |    3 +-
+ .../dts/freescale/imx8mn-vhip4-evalboard-v1.dts    |    8 +
+ .../dts/freescale/imx8mn-vhip4-evalboard-v2.dts    |   28 +-
+ .../imx8mn-vhip4-overlay-eeprom-1000.dtso          |  103 +
+ .../imx8mn-vhip4-overlay-eeprom-1100.dtso          |  103 +
+ .../imx8mn-vhip4-overlay-eeprom-2000.dtso          |  107 +
+ .../imx8mp-data-modul-edm-sbc-overlay-cm7.dtso     |   57 +
+ ...-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso |   67 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso |   46 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso |   17 +
+ ...sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso |    7 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso |    7 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso |    7 +
+ ...bc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso |    7 +
+ ...sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso |   11 +
+ ...c-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso |    7 +
+ ...verlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso |    7 +
+ ...-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi |   35 +
+ ...sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtso |   24 +
+ ...bc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtso |   24 +
+ ...bc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtso |   24 +
+ ...bc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtso |   32 +
+ ...sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtso |   36 +
+ ...c-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtso |   24 +
+ ...verlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtso |   24 +
+ ...edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtso |   41 +
+ ...edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtso |   14 +
+ ...-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi |   79 +
+ ...odul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtso |   97 +
+ ...odul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtso |   69 +
+ .../dts/freescale/imx8mp-data-modul-edm-sbc.dts    |  176 +-
+ .../dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts  |    2 +-
+ .../boot/dts/freescale/imx8mp-icore-mx8mp.dtsi     |    2 +-
+ .../boot/dts/freescale/imx8mp-kontron-dl.dtso      |   11 +
+ .../imx8mp-phyboard-pollux-peb-av-10.dtsi          |    5 +-
+ .../imx8mp-phyboard-pollux-peb-wlbt-07.dtso        |   76 +
+ .../boot/dts/freescale/imx8mp-toradex-smarc.dtsi   |    2 +-
+ ...8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso |    2 +-
+ ...8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso |    2 +-
+ .../boot/dts/freescale/imx8mp-var-dart-sonata.dts  |    1 +
+ .../boot/dts/freescale/imx8mp-verdin-ivy.dtsi      |    4 +-
+ .../boot/dts/freescale/imx8mp-verdin-yavia.dtsi    |    7 +-
+ arch/arm64/boot/dts/freescale/imx8mp.dtsi          |    2 +-
+ arch/arm64/boot/dts/freescale/imx8mq-evk.dts       |    6 +
+ .../boot/dts/freescale/imx8mq-librem5-devkit.dts   |    4 +-
+ arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi  |    4 +-
+ arch/arm64/boot/dts/freescale/imx8mq-nitrogen.dts  |    2 +-
+ .../arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi |   16 +-
+ arch/arm64/boot/dts/freescale/imx8mq.dtsi          |  168 +-
+ arch/arm64/boot/dts/freescale/imx8qm-mek.dts       |   32 +-
+ arch/arm64/boot/dts/freescale/imx8qm-ss-conn.dtsi  |   39 +
+ .../boot/dts/freescale/imx8qm-var-som-symphony.dts |    2 +-
+ arch/arm64/boot/dts/freescale/imx8qm-var-som.dtsi  |    6 +-
+ arch/arm64/boot/dts/freescale/imx8qxp-mek.dts      |   20 +-
+ arch/arm64/boot/dts/freescale/imx8ulp-evk.dts      |    7 +-
+ .../arm64/boot/dts/freescale/imx8ulp-firmware.dtsi |   31 +
+ arch/arm64/boot/dts/freescale/imx8ulp.dtsi         |   12 +-
+ arch/arm64/boot/dts/freescale/imx9-mqs.dtso        |   57 +
+ arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts  |   55 +-
+ .../boot/dts/freescale/imx91-11x11-frdm-s.dts      |    2 -
+ arch/arm64/boot/dts/freescale/imx91-11x11-frdm.dts |    2 -
+ .../freescale/imx91-9x9-qsb-tianma-tm050rdh03.dtso |  104 +
+ arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts    |   52 +
+ .../dts/freescale/imx91-lino-verdin-dahlia.dts     |   24 +
+ .../boot/dts/freescale/imx91-lino-verdin-dev.dts   |   24 +
+ arch/arm64/boot/dts/freescale/imx91-lino.dtsi      |  305 +++
+ .../boot/dts/freescale/imx91-toradex-osm-dev.dts   |   22 +
+ .../boot/dts/freescale/imx91-toradex-osm.dtsi      |  303 +++
+ arch/arm64/boot/dts/freescale/imx91.dtsi           |   11 +
+ arch/arm64/boot/dts/freescale/imx91_93_common.dtsi |    6 +
+ .../boot/dts/freescale/imx93-11x11-evk-common.dtsi |    5 +-
+ arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts  |   54 +-
+ .../dts/freescale/imx93-11x11-frdm-common.dtsi     |  681 ++++++
+ arch/arm64/boot/dts/freescale/imx93-11x11-frdm.dts |  682 +-----
+ arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts  |   59 +-
+ arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts    |    7 +-
+ .../dts/freescale/imx93-imx91-lino-common.dtsi     |  536 +++++
+ .../freescale/imx93-imx91-lino-verdin-common.dtsi  |  175 ++
+ .../imx93-imx91-lino-verdin-dahlia-common.dtsi     |  198 ++
+ .../imx93-imx91-lino-verdin-dev-common.dtsi        |  216 ++
+ .../freescale/imx93-imx91-toradex-osm-common.dtsi  |  526 +++++
+ .../imx93-imx91-toradex-osm-dev-common.dtsi        |  324 +++
+ .../boot/dts/freescale/imx93-kontron-bl-osm-s.dts  |    6 +-
+ .../boot/dts/freescale/imx93-kontron-osm-s.dtsi    |   32 +-
+ .../dts/freescale/imx93-lino-verdin-dahlia.dts     |   24 +
+ .../boot/dts/freescale/imx93-lino-verdin-dev.dts   |   24 +
+ arch/arm64/boot/dts/freescale/imx93-lino.dtsi      |  359 +++
+ .../boot/dts/freescale/imx93-toradex-osm-dev.dts   |   22 +
+ .../boot/dts/freescale/imx93-toradex-osm.dtsi      |  357 +++
+ arch/arm64/boot/dts/freescale/imx93w-frdm.dts      |   23 +
+ arch/arm64/boot/dts/freescale/imx94.dtsi           |    5 +-
+ .../boot/dts/freescale/imx943-evk-sdwifi.dtso      |   25 +
+ arch/arm64/boot/dts/freescale/imx943-evk.dts       |  138 +-
+ arch/arm64/boot/dts/freescale/imx943.dtsi          |    2 +-
+ arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts  |   48 +-
+ arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts |   38 +-
+ arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts  |   55 +-
+ .../boot/dts/freescale/imx95-19x19-frdm-pro.dts    |    2 -
+ .../boot/dts/freescale/imx95-aquila-clover.dts     |   18 +
+ arch/arm64/boot/dts/freescale/imx95-aquila-dev.dts |   18 +
+ arch/arm64/boot/dts/freescale/imx95-aquila.dtsi    |   15 +-
+ .../boot/dts/freescale/imx95-toradex-osm-dev.dts   |  404 ++++
+ .../boot/dts/freescale/imx95-toradex-osm.dtsi      | 1032 +++++++++
+ .../boot/dts/freescale/imx95-toradex-smarc-dev.dts |    8 +
+ .../boot/dts/freescale/imx95-toradex-smarc.dtsi    |    8 +
+ .../boot/dts/freescale/imx95-var-dart-sonata.dts   |    2 +-
+ arch/arm64/boot/dts/freescale/imx95.dtsi           |   18 +-
+ arch/arm64/boot/dts/freescale/imx952-evk.dts       |   77 +-
+ arch/arm64/boot/dts/freescale/imx952-frdm.dts      |    9 +
+ arch/arm64/boot/dts/freescale/imx952-frdm.dtsi     |  726 ++++++
+ arch/arm64/boot/dts/freescale/imx952.dtsi          |   19 +-
+ arch/arm64/boot/dts/freescale/s32g2.dtsi           |    2 +-
+ arch/arm64/boot/dts/freescale/s32g3.dtsi           |    2 +-
+ drivers/firmware/imx/Kconfig                       |   12 +
+ drivers/firmware/imx/Makefile                      |    2 +
+ drivers/firmware/imx/ele_base_msg.c                |  395 ++++
+ drivers/firmware/imx/ele_base_msg.h                |  119 +
+ drivers/firmware/imx/ele_common.c                  | 1012 ++++++++
+ drivers/firmware/imx/ele_common.h                  |  147 ++
+ drivers/firmware/imx/ele_fw_api.c                  |  406 ++++
+ drivers/firmware/imx/ele_fw_api.h                  |  104 +
+ drivers/firmware/imx/ele_msg_addr_field.c          |  619 +++++
+ drivers/firmware/imx/imx-dsp.c                     |    6 +-
+ drivers/firmware/imx/se_ctrl.c                     | 2434 ++++++++++++++++++++
+ drivers/firmware/imx/se_ctrl.h                     |  309 +++
+ include/linux/firmware/imx/se_api.h                |   14 +
+ include/linux/micrel_phy.h                         |    5 -
+ include/soc/imx/cpu.h                              |    2 +-
+ include/uapi/linux/se_ioctl.h                      |   97 +
+ 257 files changed, 19003 insertions(+), 1271 deletions(-)
+ create mode 100644 Documentation/ABI/testing/se-cdev
+ create mode 100644 Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel-touch-adapter.dtso
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel.dtso
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-res-touch-7inch-parallel.dtso
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-vga.dtso
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-mx7customboard.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-wm8731-mx7customboard.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-mx7customboard.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-mx7customboard.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-wm8731-mx7customboard.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-emmc-mx7customboard.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-nand-mx7customboard.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8731.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8904.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx7d-var-som.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/vf/vf-colibri-iris.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/vf/vf500-colibri-iris.dts
+ create mode 100644 arch/arm/boot/dts/nxp/vf/vf610-colibri-iris.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-cm4.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-dual.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-5v0.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-dual.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g070y2-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g101ice-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g121xce-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g156hce-l01.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g215hvn011.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi0700a2t-30.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi1010z1t-1cp11.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1000.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1100.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-2000.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-cm7.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-wlbt-07.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8ulp-firmware.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx9-mqs.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-9x9-qsb-tianma-tm050rdh03.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-lino-verdin-dahlia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-lino-verdin-dev.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-lino.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-toradex-osm-dev.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-toradex-osm.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-11x11-frdm-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-imx91-lino-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dahlia-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dev-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-dev-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-lino-verdin-dahlia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-lino-verdin-dev.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-lino.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-toradex-osm-dev.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-toradex-osm.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93w-frdm.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx95-toradex-osm-dev.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx95-toradex-osm.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx952-frdm.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx952-frdm.dtsi
+ create mode 100644 drivers/firmware/imx/ele_base_msg.c
+ create mode 100644 drivers/firmware/imx/ele_base_msg.h
+ create mode 100644 drivers/firmware/imx/ele_common.c
+ create mode 100644 drivers/firmware/imx/ele_common.h
+ create mode 100644 drivers/firmware/imx/ele_fw_api.c
+ create mode 100644 drivers/firmware/imx/ele_fw_api.h
+ create mode 100644 drivers/firmware/imx/ele_msg_addr_field.c
+ create mode 100644 drivers/firmware/imx/se_ctrl.c
+ create mode 100644 drivers/firmware/imx/se_ctrl.h
+ create mode 100644 include/linux/firmware/imx/se_api.h
+ create mode 100644 include/uapi/linux/se_ioctl.h
+Merging mediatek/for-next (4bfec57314396 Merge branches 'v7.3-next/dts32', 'v7.3-next/dts64' and 'v7.3-next/soc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git mediatek/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/mediatek.yaml          |   1 +
+ arch/arm/boot/dts/mediatek/Makefile                |   1 +
+ arch/arm/boot/dts/mediatek/mt8127-amazon-ford.dts  |  54 ++++++
+ arch/arm/boot/dts/mediatek/mt8127.dtsi             |   7 +
+ arch/arm64/boot/dts/mediatek/mt6878-pinfunc.h      |   2 +-
+ arch/arm64/boot/dts/mediatek/mt6893-pinfunc.h      |   2 +-
+ .../boot/dts/mediatek/mt7986a-bananapi-bpi-r3.dts  |  13 ++
+ arch/arm64/boot/dts/mediatek/mt7986a.dtsi          |   1 +
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso |   1 +
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso |   1 +
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-cn15.dtso |   1 +
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-cn18.dtso |   1 +
+ .../dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi  |  18 ++
+ .../boot/dts/mediatek/mt7988a-bananapi-bpi-r4.dtsi |  13 ++
+ arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi       |   5 +
+ arch/arm64/boot/dts/mediatek/mt8173.dtsi           |  35 +++-
+ arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi     |   5 +
+ arch/arm64/boot/dts/mediatek/mt8186-corsola.dtsi   |   5 +
+ arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi    |   7 +-
+ arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi   |   5 +
+ arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi    |   5 +
+ drivers/soc/mediatek/mt8167-mmsys.h                | 183 ++++++++++++++++++++-
+ drivers/soc/mediatek/mtk-regulator-coupler.c       |   1 +
+ 23 files changed, 353 insertions(+), 14 deletions(-)
+ create mode 100644 arch/arm/boot/dts/mediatek/mt8127-amazon-ford.dts
+Merging mvebu/for-next (6dfda79476538 ARM: dts: marvell: armada-388: use onnn,pca9655 compatible)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git mvebu/for-next
+Merge made by the 'ort' strategy.
+ .../boot/dts/marvell/armada-385-clearfog-gtr.dtsi  |  2 +-
+ arch/arm/boot/dts/marvell/armada-388-clearfog.dtsi |  7 +----
+ arch/arm/boot/dts/marvell/armada-388-helios4.dts   | 36 ++++++++--------------
+ arch/arm/boot/dts/marvell/armada-38x.dtsi          |  1 +
+ arch/arm/boot/dts/marvell/armada-395-gp.dts        |  1 -
+ arch/arm/boot/dts/marvell/armada-39x.dtsi          |  1 +
+ arch/arm/boot/dts/marvell/dove.dtsi                | 22 ++++++-------
+ arch/arm/boot/dts/marvell/kirkwood-6282.dtsi       | 15 +++++----
+ .../dts/marvell/kirkwood-guruplug-server-plus.dts  |  4 +--
+ arch/arm/boot/dts/marvell/kirkwood-lsxl.dtsi       |  4 +--
+ arch/arm/boot/dts/marvell/kirkwood-synology.dtsi   | 12 ++++----
+ .../arm/boot/dts/marvell/orion5x-rd88f5182-nas.dts |  2 +-
+ 12 files changed, 46 insertions(+), 61 deletions(-)
+Merging omap/for-next (7cdd46c9d6c5a Merge branch 'omap-for-v7.4/soc' into tmp/omap-next-20260921.091322)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/input/omap-keypad.txt      | 28 -------
+ .../devicetree/bindings/input/ti,omap4-keypad.yaml | 63 +++++++++++++++
+ .../boot/dts/ti/omap/am335x-boneblack-hdmi.dtsi    |  2 +-
+ arch/arm/boot/dts/ti/omap/am335x-bonegreen-eco.dts |  2 +-
+ arch/arm/boot/dts/ti/omap/am335x-shc.dts           |  4 -
+ arch/arm/boot/dts/ti/omap/am437x-l4.dtsi           |  2 +-
+ arch/arm/boot/dts/ti/omap/am57-pruss.dtsi          | 11 +++
+ arch/arm/boot/dts/ti/omap/am571x-idk.dts           | 65 +++++++++++++++-
+ arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts  |  2 +-
+ .../boot/dts/ti/omap/am57xx-beagle-x15-common.dtsi |  2 +-
+ arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi   |  3 +-
+ arch/arm/boot/dts/ti/omap/dra7-l4.dtsi             |  2 +-
+ .../boot/dts/ti/omap/motorola-mapphone-common.dtsi |  8 +-
+ .../dts/ti/omap/motorola-mapphone-mz607-mz617.dtsi |  2 +
+ arch/arm/boot/dts/ti/omap/omap3-igep.dtsi          |  2 +-
+ arch/arm/boot/dts/ti/omap/omap3-n9.dts             |  5 +-
+ arch/arm/boot/dts/ti/omap/omap3-n900.dts           |  3 +-
+ arch/arm/boot/dts/ti/omap/omap3-n950.dts           |  5 +-
+ arch/arm/boot/dts/ti/omap/omap3-tao3530.dtsi       |  8 +-
+ .../boot/dts/ti/omap/omap4-droid-bionic-xt875.dts  |  2 +
+ arch/arm/boot/dts/ti/omap/omap4-droid4-xt894.dts   |  2 +
+ arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts  |  2 +
+ arch/arm/boot/dts/ti/omap/omap4-l4.dtsi            |  1 +
+ .../dts/ti/omap/omap4-samsung-espresso-common.dtsi |  6 +-
+ arch/arm/boot/dts/ti/omap/omap4-sdp.dts            |  2 +
+ arch/arm/boot/dts/ti/omap/omap4.dtsi               |  6 +-
+ arch/arm/boot/dts/ti/omap/omap5-l4.dtsi            |  1 +
+ arch/arm/boot/dts/ti/omap/omap5.dtsi               |  2 +-
+ arch/arm/configs/multi_v7_defconfig                |  1 +
+ arch/arm/configs/omap2plus_defconfig               |  3 +
+ arch/arm/mach-omap2/control.h                      |  8 +-
+ arch/arm/mach-omap2/ctrl_module_wkup_44xx.h        | 89 ----------------------
+ arch/arm/mach-omap2/soc.h                          |  4 +-
+ arch/arm/mach-omap2/sram.h                         |  4 +-
+ 34 files changed, 195 insertions(+), 157 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/input/omap-keypad.txt
+ create mode 100644 Documentation/devicetree/bindings/input/ti,omap4-keypad.yaml
+ delete mode 100644 arch/arm/mach-omap2/ctrl_module_wkup_44xx.h
+Merging qcom/for-next (e081f1e68766c Merge branches 'arm32-for-7.4', 'arm64-defconfig-for-7.4', 'arm64-fixes-for-7.3', 'arm64-for-7.4', 'clk-fixes-for-7.3', 'clk-for-7.4', 'drivers-fixes-for-7.3' and 'drivers-for-7.4' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git qcom/for-next
+Auto-merging Documentation/devicetree/bindings/arm/cpus.yaml
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging MAINTAINERS
+Auto-merging arch/arm/configs/multi_v7_defconfig
+Auto-merging drivers/firmware/qcom/Kconfig
+Auto-merging drivers/firmware/qcom/qcom_tzmem.c
+Auto-merging include/linux/mod_devicetable.h
+CONFLICT (content): Merge conflict in include/linux/mod_devicetable.h
+Resolved 'include/linux/mod_devicetable.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 3857b8771c684] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/devicetree/bindings/arm/cpus.yaml    |    4 +
+ .../bindings/arm/qcom,coresight-ctcu.yaml          |    1 +
+ .../devicetree/bindings/arm/qcom-soc.yaml          |    4 +-
+ Documentation/devicetree/bindings/arm/qcom.yaml    |   60 +
+ .../devicetree/bindings/cache/qcom,llcc.yaml       |   55 +-
+ .../bindings/clock/qcom,gcc-msm8952.yaml           |   65 +
+ .../devicetree/bindings/clock/qcom,gcc-sdm845.yaml |    6 +-
+ .../devicetree/bindings/clock/qcom,gcc-sm6350.yaml |    7 +
+ .../devicetree/bindings/clock/qcom,gcc-sm8150.yaml |    7 +
+ .../devicetree/bindings/clock/qcom,gcc-sm8250.yaml |    7 +
+ .../devicetree/bindings/clock/qcom,gcc-sm8350.yaml |    7 +
+ .../devicetree/bindings/clock/qcom,gcc-sm8450.yaml |    7 +
+ .../devicetree/bindings/clock/qcom,hawi-gpucc.yaml |   80 +
+ .../bindings/clock/qcom,ipq9574-cmn-pll.yaml       |    1 +
+ .../bindings/clock/qcom,kaanapali-gxclkctl.yaml    |   23 +-
+ .../devicetree/bindings/clock/qcom,kuno-gcc.yaml   |   52 +
+ .../bindings/clock/qcom,milos-camcc.yaml           |   39 +-
+ .../bindings/clock/qcom,milos-videocc.yaml         |   29 +-
+ .../devicetree/bindings/clock/qcom,qcs615-gcc.yaml |    7 +
+ .../bindings/clock/qcom,qcs8300-gcc.yaml           |    7 +
+ .../devicetree/bindings/clock/qcom,rpmcc.yaml      |    5 +
+ .../devicetree/bindings/clock/qcom,rpmhcc.yaml     |    2 +
+ .../devicetree/bindings/clock/qcom,sdx75-gcc.yaml  |    7 +
+ .../devicetree/bindings/clock/qcom,sm4450-gcc.yaml |    7 +
+ .../devicetree/bindings/clock/qcom,sm6375-gcc.yaml |    9 +-
+ .../devicetree/bindings/clock/qcom,sm7150-gcc.yaml |   53 -
+ .../bindings/clock/qcom,sm8450-camcc.yaml          |   47 +-
+ .../bindings/clock/qcom,sm8450-gpucc.yaml          |    3 +
+ .../bindings/clock/qcom,sm8450-videocc.yaml        |    2 +
+ .../devicetree/bindings/clock/qcom,sm8550-gcc.yaml |    7 +
+ .../bindings/clock/qcom,sm8550-tcsr.yaml           |    1 -
+ .../devicetree/bindings/clock/qcom,sm8650-gcc.yaml |    7 +
+ .../devicetree/bindings/clock/qcom,sm8750-gcc.yaml |    7 +
+ .../bindings/clock/qcom,x1e80100-tcsr.yaml         |  114 +
+ .../embedded-controller/qcom,hamoa-crd-ec.yaml     |    3 +
+ .../devicetree/bindings/firmware/qcom,scm.yaml     |    4 +
+ .../bindings/interconnect/qcom,msm8998-bwmon.yaml  |    1 +
+ .../bindings/mailbox/qcom,cpucp-mbox.yaml          |    1 +
+ Documentation/devicetree/bindings/mfd/syscon.yaml  |    1 +
+ .../devicetree/bindings/mmc/qcom,sdhci-msm.yaml    |    1 +
+ .../bindings/net/bluetooth/qcom,wcn6750-bt.yaml    |   10 +-
+ .../bindings/net/wireless/qcom,ath11k.yaml         |   16 +-
+ .../devicetree/bindings/soc/qcom/qcom,apr.yaml     |    4 +-
+ .../devicetree/bindings/soc/qcom/qcom-stats.yaml   |   25 +-
+ .../bindings/spmi/qcom,glymur-spmi-pmic-arb.yaml   |    1 +
+ .../devicetree/bindings/sram/qcom,imem.yaml        |    1 -
+ Documentation/devicetree/bindings/sram/sram.yaml   |    1 +
+ .../devicetree/bindings/usb/qcom,snps-dwc3.yaml    |    3 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |    8 +
+ MAINTAINERS                                        |    2 +
+ .../boot/dts/qcom/qcom-msm8960-sony-huashan.dts    |   15 +
+ arch/arm/boot/dts/qcom/qcom-msm8960.dtsi           |  169 +-
+ arch/arm/boot/dts/qcom/qcom-msm8974.dtsi           |    3 +
+ arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts          |    2 +-
+ arch/arm/configs/multi_v7_defconfig                |    9 -
+ arch/arm/configs/qcom_defconfig                    |   14 -
+ arch/arm64/boot/dts/qcom/Makefile                  |   44 +
+ arch/arm64/boot/dts/qcom/agatti.dtsi               |   89 +-
+ arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi       |   15 +-
+ arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi        |   15 +
+ arch/arm64/boot/dts/qcom/eliza-evk.dtsi            |  243 +
+ arch/arm64/boot/dts/qcom/eliza-mtp.dts             |   34 +
+ arch/arm64/boot/dts/qcom/eliza.dtsi                |  774 ++-
+ .../dts/qcom/glymur-asus-zenbook-a14-ux3407na.dts  | 1140 ++++
+ .../dts/qcom/glymur-asus-zenbook-a16-ux3607oa.dts  |    3 +-
+ arch/arm64/boot/dts/qcom/glymur-crd.dts            |    8 +
+ arch/arm64/boot/dts/qcom/glymur-crd.dtsi           |   36 +-
+ .../boot/dts/qcom/glymur-hp-elitebook-x-g2q.dts    |  940 +++
+ .../boot/dts/qcom/glymur-lenovo-yoga-slim7x.dts    | 1221 ++++
+ arch/arm64/boot/dts/qcom/glymur.dtsi               |  496 +-
+ .../boot/dts/qcom/hamoa-advantech-som-6820.dtsi    |   54 +
+ .../boot/dts/qcom/hamoa-advantech-som-db5830.dts   |  119 +
+ arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts         |  267 +-
+ arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi        |   21 +
+ .../qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts  |   21 +
+ arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi          |    1 +
+ arch/arm64/boot/dts/qcom/hamoa.dtsi                |   44 +-
+ arch/arm64/boot/dts/qcom/hawi-ipcc.h               |   61 +
+ arch/arm64/boot/dts/qcom/hawi-mtp.dts              |  997 +++
+ arch/arm64/boot/dts/qcom/hawi.dtsi                 | 6844 ++++++++++++++++++++
+ arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts     |   10 +
+ arch/arm64/boot/dts/qcom/ipq5018.dtsi              |   21 +-
+ arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi   |    2 +
+ arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts        |    2 +
+ arch/arm64/boot/dts/qcom/ipq5424.dtsi              |   16 +
+ arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi   |    2 +
+ arch/arm64/boot/dts/qcom/ipq9574.dtsi              |    3 +-
+ arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts        |  119 +
+ arch/arm64/boot/dts/qcom/ipq9650.dtsi              |  666 +-
+ arch/arm64/boot/dts/qcom/kaanapali-mtp.dts         |   10 +-
+ arch/arm64/boot/dts/qcom/kaanapali-qrd.dts         |    8 +
+ arch/arm64/boot/dts/qcom/kaanapali.dtsi            |  470 +-
+ arch/arm64/boot/dts/qcom/kodiak.dtsi               | 1444 +++--
+ arch/arm64/boot/dts/qcom/lemans-el2.dtso           |    4 +-
+ .../boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso    |   15 +-
+ arch/arm64/boot/dts/qcom/lemans-evk.dts            |   16 +-
+ arch/arm64/boot/dts/qcom/lemans-pmics.dtsi         |  116 +
+ arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi   |   21 +-
+ arch/arm64/boot/dts/qcom/lemans.dtsi               |   24 +-
+ arch/arm64/boot/dts/qcom/mahua-crd.dts             |    5 +
+ .../qcom/mahua-lenovo-thinkpad-t14s-gen7-lcd.dts   | 1272 ++++
+ arch/arm64/boot/dts/qcom/mahua.dtsi                |   89 +
+ arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts   |  191 +
+ arch/arm64/boot/dts/qcom/milos.dtsi                |  479 +-
+ arch/arm64/boot/dts/qcom/monaco-ac-evk.dts         |  938 +++
+ arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts  |  201 +-
+ .../boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso    |   12 +-
+ arch/arm64/boot/dts/qcom/monaco-evk.dts            |   69 +-
+ arch/arm64/boot/dts/qcom/monaco-pmics.dtsi         |   62 +
+ arch/arm64/boot/dts/qcom/monaco.dtsi               |   15 +-
+ arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts   |   35 +
+ arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts     |    8 +
+ .../boot/dts/qcom/msm8939-longcheer-l9100.dts      |    8 +
+ arch/arm64/boot/dts/qcom/msm8939.dtsi              |   26 +
+ arch/arm64/boot/dts/qcom/msm8976.dtsi              |    2 +-
+ .../boot/dts/qcom/msm8996-oneplus-common.dtsi      |    5 +-
+ .../boot/dts/qcom/msm8996-sony-xperia-tone.dtsi    |    7 +-
+ .../arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi |    6 +-
+ arch/arm64/boot/dts/qcom/msm8996.dtsi              |   45 +-
+ arch/arm64/boot/dts/qcom/msm8998.dtsi              |    6 +-
+ arch/arm64/boot/dts/qcom/nord-embedded.dtsi        | 1928 ++++++
+ arch/arm64/boot/dts/qcom/nord-gearvm.dtsi          | 2848 ++++++++
+ arch/arm64/boot/dts/qcom/nord-ride.dts             |  254 +
+ arch/arm64/boot/dts/qcom/nord-rrd.dts              |  424 ++
+ arch/arm64/boot/dts/qcom/nord.dtsi                 | 4650 +++++++++++++
+ arch/arm64/boot/dts/qcom/pmh0104-hawi.dtsi         |  103 +
+ arch/arm64/boot/dts/qcom/pmi8996.dtsi              |   12 +-
+ arch/arm64/boot/dts/qcom/purwa-iot-evk.dts         |  224 +-
+ arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi        |   21 +
+ arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts |   54 +
+ arch/arm64/boot/dts/qcom/qcm6490-idp.dts           |    6 +
+ .../boot/dts/qcom/qcm6490-particle-tachyon.dts     |    3 +-
+ arch/arm64/boot/dts/qcom/qcs404-evb.dtsi           |    6 +-
+ arch/arm64/boot/dts/qcom/qcs404.dtsi               |    7 +-
+ arch/arm64/boot/dts/qcom/qcs615-ride.dts           |    5 +-
+ .../boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts     |    4 +-
+ .../qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso |   30 +-
+ .../dts/qcom/qcs6490-rb3gen2-vision-mezzanine.dtso |   13 +-
+ arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts       |   24 +-
+ .../dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts  |   16 +-
+ .../qcs6490-thundercomm-rubikpi3-cam1-imx219.dtso  |  115 +
+ .../qcs6490-thundercomm-rubikpi3-cam2-imx219.dtso  |  115 +
+ .../boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts |   25 +-
+ .../boot/dts/qcom/qcs6490-vicharak-axon-mini.dts   |   16 +-
+ arch/arm64/boot/dts/qcom/qcs8300-ride.dts          |    8 +-
+ arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi       |   16 +-
+ .../arm64/boot/dts/qcom/qcs8550-ayntec-common.dtsi | 1819 ++++++
+ .../boot/dts/qcom/qcs8550-ayntec-odin2mini.dts     |   44 +
+ .../boot/dts/qcom/qcs8550-ayntec-odin2portal.dts   |   99 +
+ arch/arm64/boot/dts/qcom/qcs8550-ayntec-thor.dts   |  250 +
+ arch/arm64/boot/dts/qcom/qcs8550-imdt-sbc.dts      |  401 ++
+ arch/arm64/boot/dts/qcom/qcs8550-imdt-som.dtsi     |  315 +
+ arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts       |   24 +-
+ arch/arm64/boot/dts/qcom/qdu1000.dtsi              |    6 +-
+ arch/arm64/boot/dts/qcom/sa8540p-ride.dts          |    4 +-
+ arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts     |    6 +-
+ arch/arm64/boot/dts/qcom/sar2130p.dtsi             |   13 +-
+ .../boot/dts/qcom/sc7180-trogdor-homestar.dtsi     |    2 +-
+ .../qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts |    2 +-
+ .../boot/dts/qcom/sc7180-trogdor-lazor-r1.dts      |    4 +-
+ .../boot/dts/qcom/sc7180-trogdor-pompom-r1.dts     |    4 +-
+ arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts     |    4 +-
+ .../arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts |    7 +-
+ arch/arm64/boot/dts/qcom/sc8180x-primus.dts        |    7 +-
+ arch/arm64/boot/dts/qcom/sc8180x.dtsi              |   24 +-
+ arch/arm64/boot/dts/qcom/sc8280xp-crd.dts          |    3 +
+ .../boot/dts/qcom/sc8280xp-huawei-gaokun3.dts      |   13 +-
+ .../dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts     |   37 +
+ .../boot/dts/qcom/sc8280xp-microsoft-arcata.dts    |    5 +-
+ .../boot/dts/qcom/sc8280xp-microsoft-blackrock.dts |    3 +
+ arch/arm64/boot/dts/qcom/sc8280xp.dtsi             |  690 +-
+ arch/arm64/boot/dts/qcom/sdm630.dtsi               |   16 +-
+ arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi |   50 +-
+ arch/arm64/boot/dts/qcom/sdm670.dtsi               |  271 +
+ arch/arm64/boot/dts/qcom/sdm845-db845c.dts         |   13 +-
+ arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi |   36 +-
+ arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi     |   13 +-
+ arch/arm64/boot/dts/qcom/sdm845-mtp.dts            |   12 +-
+ .../arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi |    7 +-
+ .../boot/dts/qcom/sdm845-oneplus-enchilada.dts     |    5 +
+ arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts |    1 +
+ .../boot/dts/qcom/sdm845-sony-xperia-tama.dtsi     |    2 +
+ .../dts/qcom/sdm845-xiaomi-beryllium-common.dtsi   |    6 +
+ arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts |    6 +
+ arch/arm64/boot/dts/qcom/sdm845.dtsi               |   20 +-
+ arch/arm64/boot/dts/qcom/sdx75.dtsi                |    2 -
+ .../dts/qcom/shikra-cqm-cqs-evk-imx577-camera.dtso |   72 +
+ arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts        |   36 +
+ arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi       |    1 -
+ arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts        |   32 +
+ arch/arm64/boot/dts/qcom/shikra-evk.dtsi           |   41 +
+ .../dts/qcom/shikra-iqs-evk-imx577-camera.dtso     |   72 +
+ arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts        |   32 +
+ arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi       |    3 +-
+ arch/arm64/boot/dts/qcom/shikra.dtsi               | 2034 +++++-
+ arch/arm64/boot/dts/qcom/sm4450.dtsi               |    1 -
+ arch/arm64/boot/dts/qcom/sm6115.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/sm6350.dtsi               |  233 +
+ arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts  |    4 +
+ .../boot/dts/qcom/sm7325-nothing-spacewar.dts      |    2 +
+ arch/arm64/boot/dts/qcom/sm7325-xiaomi-lisa.dts    | 1108 ++++
+ arch/arm64/boot/dts/qcom/sm8150.dtsi               |   25 +-
+ arch/arm64/boot/dts/qcom/sm8250.dtsi               |   49 +-
+ arch/arm64/boot/dts/qcom/sm8350-hdk.dts            |   18 +-
+ .../boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi   |   17 +-
+ arch/arm64/boot/dts/qcom/sm8350.dtsi               |   15 +-
+ arch/arm64/boot/dts/qcom/sm8450.dtsi               |   25 +-
+ arch/arm64/boot/dts/qcom/sm8550-hdk.dts            |   28 +-
+ arch/arm64/boot/dts/qcom/sm8550-mtp.dts            |   16 +-
+ arch/arm64/boot/dts/qcom/sm8550-qrd.dts            |   20 +-
+ arch/arm64/boot/dts/qcom/sm8550-samsung-q5q.dts    |    7 +-
+ .../dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts    |    8 +-
+ arch/arm64/boot/dts/qcom/sm8550.dtsi               |   12 +-
+ .../boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts      |  243 +-
+ arch/arm64/boot/dts/qcom/sm8650-hdk.dts            |   22 +-
+ arch/arm64/boot/dts/qcom/sm8650-mtp.dts            |   16 +-
+ arch/arm64/boot/dts/qcom/sm8650-qrd.dts            |   14 +-
+ arch/arm64/boot/dts/qcom/sm8650-valve-deckard.dts  | 1288 ++++
+ arch/arm64/boot/dts/qcom/sm8650.dtsi               |   18 +-
+ arch/arm64/boot/dts/qcom/sm8750-mtp.dts            |   10 +-
+ arch/arm64/boot/dts/qcom/sm8750-qrd.dts            |    8 +
+ arch/arm64/boot/dts/qcom/sm8750.dtsi               |  407 +-
+ .../boot/dts/qcom/talos-evk-rpi-display-2-5in.dtso |   88 +
+ arch/arm64/boot/dts/qcom/talos-evk-som.dtsi        |    9 +-
+ arch/arm64/boot/dts/qcom/talos-evk.dts             |    2 +-
+ arch/arm64/boot/dts/qcom/talos.dtsi                |   11 +-
+ arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi |   22 +
+ arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi  |   22 +
+ arch/arm64/boot/dts/qcom/x1-crd.dtsi               |   22 +
+ arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi        |   24 +-
+ arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi   |   22 +
+ arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi  |   93 +-
+ arch/arm64/boot/dts/qcom/x1e001de-devkit.dts       |   24 +-
+ .../dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi    |   22 +
+ arch/arm64/boot/dts/qcom/x1e80100-crd.dts          |  119 +
+ .../qcom/x1e80100-dell-inspiron-14-plus-7441.dts   |   31 +-
+ .../boot/dts/qcom/x1e80100-dell-xps13-9345.dts     |   22 +
+ .../dts/qcom/x1e80100-honor-magicbook-art-14.dts   |   24 +-
+ .../boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts  |   44 +
+ .../dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts    |   26 +-
+ .../boot/dts/qcom/x1e80100-microsoft-romulus.dtsi  |   21 +
+ arch/arm64/boot/dts/qcom/x1e80100-qcp.dts          |   22 +
+ .../boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts |   22 +
+ .../boot/dts/qcom/x1p42100-microsoft-sp12in.dts    |   21 +
+ arch/arm64/configs/defconfig                       |  127 +-
+ drivers/clk/qcom/Kconfig                           |  167 +-
+ drivers/clk/qcom/Makefile                          |    9 +
+ drivers/clk/qcom/cambistmclkcc-eliza.c             |  464 ++
+ drivers/clk/qcom/cambistmclkcc-kaanapali.c         |    7 +
+ drivers/clk/qcom/cambistmclkcc-sm8750.c            |    7 +
+ drivers/clk/qcom/camcc-eliza.c                     | 2803 ++++++++
+ drivers/clk/qcom/camcc-glymur.c                    |    7 +
+ drivers/clk/qcom/camcc-hawi.c                      | 3152 +++++++++
+ drivers/clk/qcom/camcc-nord.c                      | 2941 +++++++++
+ drivers/clk/qcom/camcc-sa8775p.c                   |   36 +-
+ drivers/clk/qcom/camcc-sc8280xp.c                  |   13 +-
+ drivers/clk/qcom/camcc-sm7150.c                    |   13 +-
+ drivers/clk/qcom/camcc-sm8150.c                    |   13 +-
+ drivers/clk/qcom/clk-alpha-pll.c                   |    9 +
+ drivers/clk/qcom/clk-rcg2.c                        |   35 +-
+ drivers/clk/qcom/clk-regmap-divider.c              |   16 +-
+ drivers/clk/qcom/clk-regmap-divider.h              |    1 +
+ drivers/clk/qcom/clk-rpmh.c                        |   44 +
+ drivers/clk/qcom/dispcc-sc7280.c                   |   13 +-
+ drivers/clk/qcom/dispcc-sc8280xp.c                 |   18 +-
+ drivers/clk/qcom/dispcc-sm4450.c                   |   15 +-
+ drivers/clk/qcom/dispcc-sm6115.c                   |   13 +-
+ drivers/clk/qcom/dispcc-sm7150.c                   |   13 +-
+ drivers/clk/qcom/dispcc-sm8250.c                   |   13 +-
+ drivers/clk/qcom/dispcc-sm8450.c                   |   40 +-
+ drivers/clk/qcom/dispcc-sm8550.c                   |   13 +-
+ drivers/clk/qcom/dispcc-sm8750.c                   |   19 +-
+ drivers/clk/qcom/dispcc-x1e80100.c                 |   27 +-
+ drivers/clk/qcom/dispcc0-sa8775p.c                 |   15 +-
+ drivers/clk/qcom/dispcc1-sa8775p.c                 |   15 +-
+ drivers/clk/qcom/gcc-eliza.c                       |   19 +-
+ drivers/clk/qcom/gcc-glymur.c                      |   81 +-
+ drivers/clk/qcom/gcc-hawi.c                        |   18 +-
+ drivers/clk/qcom/gcc-ipq5018.c                     |   10 +
+ drivers/clk/qcom/gcc-ipq5210.c                     |    1 +
+ drivers/clk/qcom/gcc-ipq5332.c                     |    1 +
+ drivers/clk/qcom/gcc-ipq5424.c                     |    4 +-
+ drivers/clk/qcom/gcc-ipq9650.c                     |   22 +
+ drivers/clk/qcom/gcc-kaanapali.c                   |   19 +-
+ drivers/clk/qcom/gcc-kuno.c                        | 1473 +++++
+ drivers/clk/qcom/gcc-milos.c                       |   12 +-
+ drivers/clk/qcom/gcc-msm8939.c                     |    4 +
+ drivers/clk/qcom/gcc-msm8952.c                     | 3548 ++++++++++
+ drivers/clk/qcom/gcc-nord.c                        |   12 +-
+ drivers/clk/qcom/gcc-qcm2290.c                     |   12 +-
+ drivers/clk/qcom/gcc-qcs615.c                      |   69 +-
+ drivers/clk/qcom/gcc-qcs8300.c                     |  167 +-
+ drivers/clk/qcom/gcc-qdu1000.c                     |   12 +-
+ drivers/clk/qcom/gcc-sa8775p.c                     |   56 +-
+ drivers/clk/qcom/gcc-sar2130p.c                    |   58 +-
+ drivers/clk/qcom/gcc-sc7180.c                      |   60 +-
+ drivers/clk/qcom/gcc-sc7280.c                      |   67 +-
+ drivers/clk/qcom/gcc-sc8280xp.c                    |   47 +-
+ drivers/clk/qcom/gcc-sdx55.c                       |   43 +-
+ drivers/clk/qcom/gcc-sdx65.c                       |   43 +-
+ drivers/clk/qcom/gcc-sdx75.c                       |   44 +-
+ drivers/clk/qcom/gcc-shikra.c                      |   12 +-
+ drivers/clk/qcom/gcc-sm4450.c                      |   77 +-
+ drivers/clk/qcom/gcc-sm6115.c                      |   12 +-
+ drivers/clk/qcom/gcc-sm6125.c                      |   12 +-
+ drivers/clk/qcom/gcc-sm6375.c                      |   12 +-
+ drivers/clk/qcom/gcc-sm7150.c                      |   72 +-
+ drivers/clk/qcom/gcc-sm8150.c                      |   12 +-
+ drivers/clk/qcom/gcc-sm8250.c                      |   69 +-
+ drivers/clk/qcom/gcc-sm8350.c                      |   65 +-
+ drivers/clk/qcom/gcc-sm8450.c                      |   64 +-
+ drivers/clk/qcom/gcc-sm8550.c                      |   70 +-
+ drivers/clk/qcom/gcc-sm8650.c                      |   72 +-
+ drivers/clk/qcom/gcc-sm8750.c                      |   89 +-
+ drivers/clk/qcom/gcc-x1e80100.c                    |   80 +-
+ drivers/clk/qcom/gpucc-eliza.c                     |  606 ++
+ drivers/clk/qcom/gpucc-glymur.c                    |    2 +-
+ drivers/clk/qcom/gpucc-hawi.c                      |  477 ++
+ drivers/clk/qcom/gpucc-kaanapali.c                 |    2 +-
+ drivers/clk/qcom/gpucc-sar2130p.c                  |   13 +-
+ drivers/clk/qcom/gpucc-sc7280.c                    |   22 +-
+ drivers/clk/qcom/gpucc-sc8280xp.c                  |   15 +-
+ drivers/clk/qcom/gpucc-sm4450.c                    |   17 +-
+ drivers/clk/qcom/gpucc-sm8550.c                    |   15 +-
+ drivers/clk/qcom/gpucc-x1e80100.c                  |   13 +-
+ drivers/clk/qcom/gpucc-x1p42100.c                  |   17 +-
+ drivers/clk/qcom/ipq-cmn-pll.c                     |  538 +-
+ drivers/clk/qcom/lpasscc-sc7280.c                  |   12 +-
+ drivers/clk/qcom/lpasscc-sdm845.c                  |   12 +-
+ drivers/clk/qcom/lpasscc-sm6115.c                  |    4 +-
+ drivers/clk/qcom/lpasscorecc-sc7180.c              |   31 +-
+ drivers/clk/qcom/mmcc-msm8960.c                    |    3 +-
+ drivers/clk/qcom/mmcc-msm8996.c                    |    4 +-
+ drivers/clk/qcom/mmcc-sdm660.c                     |    2 +-
+ drivers/clk/qcom/negcc-nord.c                      |    5 +-
+ drivers/clk/qcom/nwgcc-nord.c                      |    5 +-
+ drivers/clk/qcom/segcc-nord.c                      |   40 +-
+ drivers/clk/qcom/tcsrcc-eliza.c                    |   12 +-
+ drivers/clk/qcom/tcsrcc-glymur.c                   |   12 +-
+ drivers/clk/qcom/tcsrcc-kaanapali.c                |   12 +-
+ drivers/clk/qcom/tcsrcc-sm8550.c                   |   12 +-
+ drivers/clk/qcom/tcsrcc-sm8650.c                   |   12 +-
+ drivers/clk/qcom/tcsrcc-sm8750.c                   |   12 +-
+ drivers/clk/qcom/tcsrcc-x1e80100.c                 |  349 +-
+ drivers/clk/qcom/videocc-eliza.c                   |  404 ++
+ drivers/clk/qcom/videocc-glymur.c                  |   38 +
+ drivers/clk/qcom/videocc-sa8775p.c                 |   17 +-
+ drivers/clk/qcom/videocc-sm6350.c                  |   13 +-
+ drivers/clk/qcom/videocc-sm7150.c                  |   13 +-
+ drivers/clk/qcom/videocc-sm8250.c                  |   15 +-
+ drivers/clk/qcom/videocc-sm8350.c                  |   31 +-
+ drivers/clk/qcom/videocc-sm8750.c                  |   12 +-
+ drivers/clk/renesas/r9a06g032-clocks.c             |    8 +-
+ drivers/clk/renesas/renesas-cpg-mssr.c             |    7 +-
+ drivers/clk/renesas/rzg2l-cpg.c                    |    7 +-
+ drivers/clk/renesas/rzv2h-cpg.c                    |    7 +-
+ drivers/firmware/qcom/Kconfig                      |    5 +-
+ drivers/firmware/qcom/qcom_scm-legacy.c            |    2 +-
+ drivers/firmware/qcom/qcom_scm.c                   |  187 +-
+ drivers/firmware/qcom/qcom_tzmem.c                 |    3 +-
+ drivers/soc/qcom/Kconfig                           |    1 -
+ drivers/soc/qcom/apr.c                             |   75 +-
+ drivers/soc/qcom/llcc-qcom.c                       |  199 +-
+ drivers/soc/qcom/pmic_glink.c                      |    3 +-
+ drivers/soc/qcom/pmic_glink_altmode.c              |    4 +-
+ drivers/soc/qcom/pmic_pdcharger_ulog.c             |    2 +-
+ drivers/soc/qcom/qcom-geni-se.c                    |   65 +-
+ drivers/soc/qcom/qcom_aoss.c                       |    4 +-
+ drivers/soc/qcom/qcom_stats.c                      |    9 +
+ drivers/soc/qcom/rpmh-internal.h                   |   16 +-
+ drivers/soc/qcom/rpmh-rsc.c                        |    4 +-
+ drivers/soc/qcom/rpmh.c                            |   40 +-
+ drivers/soc/qcom/smem_dramc.c                      |   81 +-
+ drivers/soc/qcom/smp2p.c                           |   22 +-
+ drivers/soc/qcom/smsm.c                            |   36 +-
+ drivers/soc/qcom/socinfo.c                         |    3 +
+ drivers/soc/qcom/ubwc_config.c                     |   12 +
+ include/dt-bindings/arm/qcom,ids.h                 |    3 +
+ .../dt-bindings/clock/qcom,eliza-cambistmclkcc.h   |   32 +
+ include/dt-bindings/clock/qcom,eliza-camcc.h       |  151 +
+ include/dt-bindings/clock/qcom,eliza-gpucc.h       |   51 +
+ include/dt-bindings/clock/qcom,eliza-videocc.h     |   37 +
+ include/dt-bindings/clock/qcom,gcc-msm8952.h       |  215 +
+ include/dt-bindings/clock/qcom,hawi-camcc.h        |  165 +
+ include/dt-bindings/clock/qcom,hawi-gpucc.h        |   47 +
+ include/dt-bindings/clock/qcom,ipq5210-cmn-pll.h   |   30 +
+ include/dt-bindings/clock/qcom,kuno-gcc.h          |  100 +
+ include/dt-bindings/clock/qcom,nord-camcc.h        |  167 +
+ include/dt-bindings/clock/qcom,nord-negcc.h        |    1 +
+ include/dt-bindings/clock/qcom,nord-nwgcc.h        |    3 +
+ include/dt-bindings/clock/qcom,nord-segcc.h        |    2 +
+ include/dt-bindings/clock/qcom,qcs8300-gcc.h       |    6 +
+ include/dt-bindings/interconnect/qcom,ipq9650.h    |   28 +
+ include/linux/device-id/apr.h                      |   21 -
+ include/linux/device/driver.h                      |   31 +
+ include/linux/firmware/qcom/qcom_scm.h             |   29 -
+ include/linux/mod_devicetable.h                    |    1 -
+ include/linux/platform_device.h                    |   30 +
+ include/linux/soc/qcom/apr.h                       |    4 +-
+ include/linux/soc/qcom/llcc-qcom.h                 |    6 +
+ include/linux/soc/qcom/ubwc.h                      |    3 +
+ 401 files changed, 60568 insertions(+), 3709 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,gcc-msm8952.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,hawi-gpucc.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,kuno-gcc.yaml
+ delete mode 100644 Documentation/devicetree/bindings/clock/qcom,sm7150-gcc.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,x1e80100-tcsr.yaml
+ create mode 100644 arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a14-ux3407na.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/glymur-hp-elitebook-x-g2q.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/glymur-lenovo-yoga-slim7x.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/hamoa-advantech-som-6820.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/hamoa-advantech-som-db5830.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/hawi-ipcc.h
+ create mode 100644 arch/arm64/boot/dts/qcom/hawi-mtp.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/hawi.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/mahua-lenovo-thinkpad-t14s-gen7-lcd.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/monaco-ac-evk.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/nord-embedded.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/nord-gearvm.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/nord-ride.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/nord-rrd.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/nord.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/pmh0104-hawi.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam1-imx219.dtso
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam2-imx219.dtso
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-ayntec-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2mini.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2portal.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-ayntec-thor.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-imdt-sbc.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/qcs8550-imdt-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-cqm-cqs-evk-imx577-camera.dtso
+ create mode 100644 arch/arm64/boot/dts/qcom/shikra-iqs-evk-imx577-camera.dtso
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-xiaomi-lisa.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sm8650-valve-deckard.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/talos-evk-rpi-display-2-5in.dtso
+ create mode 100644 drivers/clk/qcom/cambistmclkcc-eliza.c
+ create mode 100644 drivers/clk/qcom/camcc-eliza.c
+ create mode 100644 drivers/clk/qcom/camcc-hawi.c
+ create mode 100644 drivers/clk/qcom/camcc-nord.c
+ create mode 100644 drivers/clk/qcom/gcc-kuno.c
+ create mode 100644 drivers/clk/qcom/gcc-msm8952.c
+ create mode 100644 drivers/clk/qcom/gpucc-eliza.c
+ create mode 100644 drivers/clk/qcom/gpucc-hawi.c
+ create mode 100644 drivers/clk/qcom/videocc-eliza.c
+ create mode 100644 include/dt-bindings/clock/qcom,eliza-cambistmclkcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,eliza-camcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,eliza-gpucc.h
+ create mode 100644 include/dt-bindings/clock/qcom,eliza-videocc.h
+ create mode 100644 include/dt-bindings/clock/qcom,gcc-msm8952.h
+ create mode 100644 include/dt-bindings/clock/qcom,hawi-camcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,hawi-gpucc.h
+ create mode 100644 include/dt-bindings/clock/qcom,ipq5210-cmn-pll.h
+ create mode 100644 include/dt-bindings/clock/qcom,kuno-gcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,nord-camcc.h
+ create mode 100644 include/dt-bindings/interconnect/qcom,ipq9650.h
+ delete mode 100644 include/linux/device-id/apr.h
+Merging realtek/for-next (3c778f0c9fa36 arm64: dts: realtek: Add GPIO support for RTD1625)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/yu_chun/linux.git realtek/for-next
+Already up to date.
+Merging renesas/next (7687cd01a980a Merge branches 'renesas-drivers-for-v7.4' and 'renesas-dts-for-v7.4' into renesas-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas/next
+Auto-merging arch/arm/configs/shmobile_defconfig
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/power/qcom,rpmpd.yaml      |   2 +
+ .../bindings/power/renesas,rcar-sysc.yaml          |   7 +-
+ .../bindings/power/riscv,rpmi-device-power.yaml    |  65 +++
+ .../power/riscv,rpmi-mpxy-device-power.yaml        |  65 +++
+ .../devicetree/bindings/soc/renesas/renesas.yaml   |  37 ++
+ arch/arm/boot/dts/renesas/r7s72100-gr-peach.dts    |   2 +
+ .../boot/dts/renesas/r8a7740-armadillo800eva.dts   |   2 +
+ arch/arm/boot/dts/renesas/r8a7743-sk-rzg1m.dts     |   2 +
+ arch/arm/boot/dts/renesas/r8a7743.dtsi             |   6 +-
+ arch/arm/boot/dts/renesas/r8a7744.dtsi             |   6 +-
+ arch/arm/boot/dts/renesas/r8a7745-sk-rzg1e.dts     |   2 +
+ arch/arm/boot/dts/renesas/r8a7745.dtsi             |   6 +-
+ arch/arm/boot/dts/renesas/r8a77470.dtsi            |   6 +-
+ arch/arm/boot/dts/renesas/r8a7790-lager.dts        |  28 +-
+ arch/arm/boot/dts/renesas/r8a7790-stout.dts        |   4 +-
+ arch/arm/boot/dts/renesas/r8a7790.dtsi             |   4 +-
+ arch/arm/boot/dts/renesas/r8a7791-koelsch.dts      |   2 +
+ arch/arm/boot/dts/renesas/r8a7791-porter.dts       |   2 +
+ arch/arm/boot/dts/renesas/r8a7791.dtsi             |   4 +-
+ arch/arm/boot/dts/renesas/r8a7792.dtsi             |   4 +-
+ arch/arm/boot/dts/renesas/r8a7793-gose.dts         |   2 +
+ arch/arm/boot/dts/renesas/r8a7793.dtsi             |   4 +-
+ arch/arm/boot/dts/renesas/r8a7794-alt.dts          |   2 +
+ arch/arm/boot/dts/renesas/r8a7794-silk.dts         |   2 +
+ arch/arm/boot/dts/renesas/r8a7794.dtsi             |   4 +-
+ .../arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts |   1 -
+ .../arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts |  14 +-
+ arch/arm/boot/dts/renesas/r9a06g032.dtsi           |   2 -
+ arch/arm/configs/shmobile_defconfig                |   3 +-
+ arch/arm64/boot/dts/renesas/Makefile               |  83 +++
+ .../dts/renesas/aistarvision-mipi-adapter-2.1.dtsi |   2 +-
+ .../boot/dts/renesas/beacon-renesom-baseboard.dtsi |   6 +-
+ .../arm64/boot/dts/renesas/beacon-renesom-som.dtsi |  12 +-
+ arch/arm64/boot/dts/renesas/cat875.dtsi            |   2 +
+ arch/arm64/boot/dts/renesas/condor-common.dtsi     |   2 +-
+ arch/arm64/boot/dts/renesas/draak.dtsi             |   2 +-
+ arch/arm64/boot/dts/renesas/ebisu.dtsi             |   6 +-
+ arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi  |   8 +-
+ arch/arm64/boot/dts/renesas/hihope-common.dtsi     |   4 +-
+ arch/arm64/boot/dts/renesas/hihope-rev2.dtsi       |   8 +-
+ arch/arm64/boot/dts/renesas/hihope-rev4.dtsi       |   4 +-
+ arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi    |   6 +-
+ arch/arm64/boot/dts/renesas/r8a774a1.dtsi          |  14 +-
+ .../boot/dts/renesas/r8a774a3-hihope-rzg2m-ex.dts  |  14 +
+ .../boot/dts/renesas/r8a774a3-hihope-rzg2m.dts     |  49 ++
+ arch/arm64/boot/dts/renesas/r8a774a3.dtsi          | 302 +++++++++++
+ arch/arm64/boot/dts/renesas/r8a774b1.dtsi          |  12 +-
+ arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts    |   4 +-
+ arch/arm64/boot/dts/renesas/r8a774c0.dtsi          |  12 +-
+ arch/arm64/boot/dts/renesas/r8a774e1.dtsi          |  14 +-
+ arch/arm64/boot/dts/renesas/r8a77951.dtsi          |  14 +-
+ arch/arm64/boot/dts/renesas/r8a77960.dtsi          |  14 +-
+ arch/arm64/boot/dts/renesas/r8a77961.dtsi          |  14 +-
+ arch/arm64/boot/dts/renesas/r8a77965.dtsi          |  12 +-
+ .../renesas/r8a77970-eagle-function-expansion.dtso |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77970-eagle.dts     |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts     |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77970.dtsi          |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts     |   2 +-
+ arch/arm64/boot/dts/renesas/r8a77980.dtsi          |   4 +-
+ arch/arm64/boot/dts/renesas/r8a77990.dtsi          |  10 +-
+ arch/arm64/boot/dts/renesas/r8a77995.dtsi          |   6 +-
+ .../boot/dts/renesas/r8a779a0-falcon-cpu.dtsi      |   2 +-
+ arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts    |   4 +-
+ arch/arm64/boot/dts/renesas/r8a779a0.dtsi          |   2 +-
+ .../boot/dts/renesas/r8a779f0-spider-cpu.dtsi      |   2 +-
+ arch/arm64/boot/dts/renesas/r8a779f0.dtsi          |  52 +-
+ arch/arm64/boot/dts/renesas/r8a779f4-s4sk.dts      |   2 +-
+ .../arm64/boot/dts/renesas/r8a779g0-white-hawk.dts |  94 ++++
+ arch/arm64/boot/dts/renesas/r8a779g0.dtsi          |  85 +++-
+ .../renesas/r8a779g3-sparrow-hawk-fan-argon40.dtso |   1 -
+ .../r8a779g3-sparrow-hawk-ws-2ch-canfd.dtso        | 135 +++++
+ .../boot/dts/renesas/r8a779g3-sparrow-hawk.dts     |  19 +-
+ arch/arm64/boot/dts/renesas/r8a779h0.dtsi          |  34 +-
+ arch/arm64/boot/dts/renesas/r8a779md-geist.dts     |  12 +-
+ arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts  |  12 +-
+ arch/arm64/boot/dts/renesas/r8a78000.dtsi          | 516 ++++++++++++++++++-
+ .../renesas/r9a07g043u12-hummingboard-ripple.dts   |  89 ++++
+ .../dts/renesas/r9a07g044c2-hummingboard-iiot.dts  |  20 +
+ .../renesas/r9a07g044c2-hummingboard-ripple.dts    |  93 ++++
+ .../dts/renesas/r9a07g044l2-hummingboard-iiot.dts  |  16 +
+ .../renesas/r9a07g044l2-hummingboard-ripple.dts    |  18 +
+ .../arm64/boot/dts/renesas/r9a07g044l2-remi-pi.dts |   2 +-
+ .../dts/renesas/r9a07g054l2-hummingboard-iiot.dts  |  16 +
+ .../renesas/r9a07g054l2-hummingboard-ripple.dts    |  18 +
+ arch/arm64/boot/dts/renesas/r9a08g045.dtsi         |   5 +-
+ arch/arm64/boot/dts/renesas/r9a08g046.dtsi         | 335 +++++++++++-
+ arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts |  36 ++
+ arch/arm64/boot/dts/renesas/r9a09g011.dtsi         |   4 +-
+ arch/arm64/boot/dts/renesas/r9a09g047.dtsi         | 126 ++++-
+ arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts |  58 ++-
+ arch/arm64/boot/dts/renesas/r9a09g056.dtsi         |   2 +-
+ arch/arm64/boot/dts/renesas/r9a09g057.dtsi         |   2 +-
+ .../boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts    |   6 +-
+ arch/arm64/boot/dts/renesas/r9a09g077.dtsi         |  89 +++-
+ .../dts/renesas/r9a09g077m44-evk-cn15-lcdc.dtso    |  53 ++
+ .../boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts    |  18 +-
+ arch/arm64/boot/dts/renesas/r9a09g087.dtsi         |  89 +++-
+ .../dts/renesas/r9a09g087m44-evk-cn20-lcdc.dtso    |  63 +++
+ .../boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts    |  32 +-
+ arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi    |  38 +-
+ .../renesas/rzg2l-hummingboard-iiot-common.dtsi    | 560 +++++++++++++++++++++
+ .../renesas/rzg2l-hummingboard-iiot-microsd.dtso   |  26 +
+ ...hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtso |  79 +++
+ .../renesas/rzg2l-hummingboard-iiot-rs485-a.dtso   |  17 +
+ .../renesas/rzg2l-hummingboard-iiot-rs485-b.dtso   |  17 +
+ .../boot/dts/renesas/rzg2l-hummingboard-iiot.dtsi  |  49 ++
+ .../renesas/rzg2l-hummingboard-pulse-common.dtsi   | 116 +++++
+ .../rzg2l-hummingboard-pulse-micro-hdmi.dtsi       |  79 +++
+ .../dts/renesas/rzg2l-hummingboard-ripple.dtsi     | 122 +++++
+ .../boot/dts/renesas/rzg2l-smarc-pinfunction.dtsi  |  16 +-
+ arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi   |  20 +-
+ arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi       |   2 +-
+ arch/arm64/boot/dts/renesas/rzg2l-sr-som-emmc.dtso |  44 ++
+ arch/arm64/boot/dts/renesas/rzg2l-sr-som.dtsi      | 473 +++++++++++++++++
+ .../renesas/rzg2lc-hummingboard-pulse-leds.dtso    |  48 ++
+ .../boot/dts/renesas/rzg2lc-smarc-pinfunction.dtsi |  16 +-
+ arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi  |  20 +-
+ arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi      |   2 +-
+ arch/arm64/boot/dts/renesas/rzg2lc-sr-som.dtsi     | 436 ++++++++++++++++
+ .../renesas/rzg2ul-hummingboard-pulse-leds.dtso    |  48 ++
+ .../boot/dts/renesas/rzg2ul-smarc-pinfunction.dtsi |  16 +-
+ arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi  |  20 +-
+ arch/arm64/boot/dts/renesas/rzg2ul-sr-som.dtsi     | 418 +++++++++++++++
+ arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi   |  25 +
+ arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi   |  25 +-
+ arch/arm64/boot/dts/renesas/rzg3s-smarc-switches.h |   4 +
+ arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi       |   1 -
+ .../boot/dts/renesas/rzt2h-n2h-evk-common.dtsi     |   7 +
+ .../boot/dts/renesas/rzt2h-n2h-evk-du-adv7513.dtsi |  72 +++
+ arch/arm64/boot/dts/renesas/salvator-common.dtsi   |  14 +-
+ arch/arm64/boot/dts/renesas/salvator-xs.dtsi       |   2 +-
+ .../ulcb-kf-audio-graph-card2-mix+split.dtsi       |  18 +-
+ arch/arm64/boot/dts/renesas/ulcb-kf.dtsi           |   2 +-
+ arch/arm64/boot/dts/renesas/ulcb.dtsi              |   8 +-
+ .../boot/dts/renesas/white-hawk-cpu-common.dtsi    |   6 +-
+ drivers/soc/renesas/Kconfig                        |  22 +-
+ drivers/soc/renesas/r9a08g046-sysc.c               |   1 +
+ drivers/soc/renesas/rcar-mfis.c                    |  62 ++-
+ drivers/soc/renesas/rcar-rst.c                     |   1 +
+ drivers/soc/renesas/renesas-soc.c                  |   3 +
+ drivers/soc/renesas/rz-sysc.c                      |   5 +
+ drivers/soc/renesas/rz-sysc.h                      |   2 +
+ .../dt-bindings/clock/renesas,r8a774a3-cpg-mssr.h  |  59 +++
+ include/dt-bindings/clock/renesas,r8a78000-cpg.h   |   2 +
+ include/dt-bindings/power/qcom,rpmhpd.h            |   1 +
+ include/dt-bindings/power/renesas,r8a774a3-sysc.h  |  30 ++
+ 147 files changed, 5734 insertions(+), 422 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/power/riscv,rpmi-device-power.yaml
+ create mode 100644 Documentation/devicetree/bindings/power/riscv,rpmi-mpxy-device-power.yaml
+ create mode 100644 arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m-ex.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r8a774a3.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-ws-2ch-canfd.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a07g043u12-hummingboard-ripple.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-iiot.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-ripple.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-iiot.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-ripple.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-iiot.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-ripple.dts
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a09g077m44-evk-cn15-lcdc.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/r9a09g087m44-evk-cn20-lcdc.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-microsd.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-a.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-b.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-micro-hdmi.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-hummingboard-ripple.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-sr-som-emmc.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2l-sr-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2lc-hummingboard-pulse-leds.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2lc-sr-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2ul-hummingboard-pulse-leds.dtso
+ create mode 100644 arch/arm64/boot/dts/renesas/rzg2ul-sr-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-du-adv7513.dtsi
+ create mode 100644 include/dt-bindings/clock/renesas,r8a774a3-cpg-mssr.h
+ create mode 100644 include/dt-bindings/power/renesas,r8a774a3-sysc.h
+Merging reset/reset/next (d373605cd5148 Merge tag 'reset-fixes-for-v7.0-2' into reset/next)
+$ git merge -m Merge branch 'reset/next' of https://git.kernel.org/pub/scm/linux/kernel/git/pza/linux reset/reset/next
+Already up to date.
+Merging rockchip/for-next (23d8f49fcae5d Merge branch 'v7.4-armsoc/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git rockchip/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/rockchip.yaml          |    7 +
+ arch/arm/boot/dts/rockchip/rk3229-xms6.dts         |    4 +-
+ arch/arm64/boot/dts/rockchip/Makefile              |    6 +
+ .../boot/dts/rockchip/rk3399pro-vmarc-som.dtsi     |    2 +
+ .../boot/dts/rockchip/rk3576-armsom-cm5-io.dts     |    4 +-
+ arch/arm64/boot/dts/rockchip/rk3576.dtsi           |    7 +-
+ arch/arm64/boot/dts/rockchip/rk3588-base.dtsi      |   39 +
+ .../rockchip/rk3588-jaguar-can1-can2-uart4.dtso    |   26 +
+ arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts     |   16 +
+ .../boot/dts/rockchip/rk3588-lubancat-5-btb.dtsi   |  534 ++++++++++
+ .../boot/dts/rockchip/rk3588-lubancat-5io.dts      | 1032 ++++++++++++++++++++
+ .../boot/dts/rockchip/rk3588-nanopc-t6-lts.dts     |   17 -
+ arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi |  175 ++--
+ .../boot/dts/rockchip/rk3588-tiger-haikou.dts      |    4 +
+ arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi     |    5 +
+ .../boot/dts/rockchip/rk3588s-gameforce-ace.dts    |    7 +-
+ 16 files changed, 1760 insertions(+), 125 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3588-jaguar-can1-can2-uart4.dtso
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3588-lubancat-5-btb.dtsi
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3588-lubancat-5io.dts
+Merging samsung-krzk/for-next (08df370772f32 Merge branch 'for-v7.4/google-lga' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk/for-next
+Auto-merging Documentation/devicetree/bindings/sram/sram.yaml
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/Kconfig.platforms
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/google.yaml  |   48 +-
+ .../bindings/arm/samsung/samsung-boards.yaml       |    1 +
+ .../bindings/clock/samsung,exynos5515-cmu.yaml     |  145 ++
+ .../devicetree/bindings/gpu/arm,mali-bifrost.yaml  |    1 +
+ .../bindings/hwinfo/samsung,exynos-chipid.yaml     |    1 +
+ .../bindings/input/samsung,s3c6410-keypad.yaml     |   53 +-
+ Documentation/devicetree/bindings/sram/sram.yaml   |    2 +
+ MAINTAINERS                                        |    1 +
+ arch/arm/boot/dts/samsung/exynos4210-i9100.dts     |    9 +-
+ arch/arm/boot/dts/samsung/exynos4210-trats.dts     |    5 +
+ .../boot/dts/samsung/exynos4210-universal_c210.dts |    5 +
+ arch/arm/boot/dts/samsung/exynos4412-midas.dtsi    |    5 +-
+ arch/arm/mach-exynos/smc.h                         |    4 +-
+ arch/arm/mach-s3c/Kconfig                          |    5 -
+ arch/arm/mach-s3c/Kconfig.s3c64xx                  |    7 -
+ arch/arm/mach-s3c/Makefile.s3c64xx                 |    1 -
+ arch/arm/mach-s3c/devs.c                           |   27 -
+ arch/arm/mach-s3c/devs.h                           |    1 -
+ arch/arm/mach-s3c/gpio-core.h                      |    3 +
+ arch/arm/mach-s3c/gpio-samsung-s3c64xx.h           |    5 +
+ arch/arm/mach-s3c/gpio-samsung.c                   |   72 +-
+ arch/arm/mach-s3c/keypad.h                         |   27 -
+ arch/arm/mach-s3c/mach-crag6410.c                  |  135 +-
+ arch/arm/mach-s3c/setup-keypad-s3c64xx.c           |   20 -
+ arch/arm64/Kconfig.platforms                       |    8 +
+ arch/arm64/boot/dts/Makefile                       |    1 +
+ arch/arm64/boot/dts/exynos/Makefile                |    1 +
+ arch/arm64/boot/dts/exynos/axis/artpec8.dtsi       |   36 +-
+ arch/arm64/boot/dts/exynos/axis/artpec9.dtsi       |   36 +-
+ arch/arm64/boot/dts/exynos/exynos2200-g0s.dts      |    8 +-
+ arch/arm64/boot/dts/exynos/exynos2200-pinctrl.dtsi |    2 -
+ arch/arm64/boot/dts/exynos/exynos2200.dtsi         |    1 -
+ .../boot/dts/exynos/exynos5433-tm2-common.dtsi     |   93 +-
+ arch/arm64/boot/dts/exynos/exynos5433.dtsi         |  108 +-
+ .../arm64/boot/dts/exynos/exynos7870-a2corelte.dts |   39 +-
+ arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts |   34 +-
+ arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts    |   34 +-
+ arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts  |   35 +-
+ arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts |   39 +-
+ arch/arm64/boot/dts/exynos/exynos7870.dtsi         |  108 +-
+ arch/arm64/boot/dts/exynos/exynos850-a217f.dts     |  200 +++
+ arch/arm64/boot/dts/exynos/exynos850.dtsi          |   29 +-
+ arch/arm64/boot/dts/exynos/exynos8855-smdk.dts     |    2 +-
+ arch/arm64/boot/dts/exynos/exynosautov920.dtsi     |    2 +-
+ .../boot/dts/exynos/google/gs101-pixel-common.dtsi |   51 +-
+ arch/arm64/boot/dts/exynos/google/gs101.dtsi       |   96 +-
+ arch/arm64/boot/dts/google/Makefile                |    6 +
+ arch/arm64/boot/dts/google/lga-blazer.dts          |   15 +
+ arch/arm64/boot/dts/google/lga-frankel.dts         |   15 +
+ arch/arm64/boot/dts/google/lga-mustang.dts         |   15 +
+ arch/arm64/boot/dts/google/lga-pixel-common.dtsi   |   22 +
+ arch/arm64/boot/dts/google/lga.dtsi                |  411 ++++++
+ arch/arm64/boot/dts/tesla/fsd-evb.dts              |   10 +-
+ arch/arm64/boot/dts/tesla/fsd-pinctrl.dtsi         |   16 +-
+ arch/arm64/boot/dts/tesla/fsd.dtsi                 | 1162 +++++++--------
+ arch/arm64/configs/defconfig                       |    1 +
+ drivers/clk/samsung/Makefile                       |    1 +
+ drivers/clk/samsung/clk-acpm.c                     |   61 +-
+ drivers/clk/samsung/clk-exynos5515.c               | 1484 ++++++++++++++++++++
+ drivers/clk/samsung/clk-pll.c                      |    1 +
+ drivers/clk/samsung/clk-pll.h                      |    1 +
+ drivers/input/keyboard/samsung-keypad.c            |  200 ++-
+ drivers/soc/samsung/exynos-chipid.c                |    1 +
+ include/dt-bindings/clock/samsung,exynos5515-cmu.h |  191 +++
+ include/linux/input/samsung-keypad.h               |   39 -
+ 65 files changed, 3922 insertions(+), 1276 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/samsung,exynos5515-cmu.yaml
+ delete mode 100644 arch/arm/mach-s3c/keypad.h
+ delete mode 100644 arch/arm/mach-s3c/setup-keypad-s3c64xx.c
+ create mode 100644 arch/arm64/boot/dts/exynos/exynos850-a217f.dts
+ create mode 100644 arch/arm64/boot/dts/google/Makefile
+ create mode 100644 arch/arm64/boot/dts/google/lga-blazer.dts
+ create mode 100644 arch/arm64/boot/dts/google/lga-frankel.dts
+ create mode 100644 arch/arm64/boot/dts/google/lga-mustang.dts
+ create mode 100644 arch/arm64/boot/dts/google/lga-pixel-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/google/lga.dtsi
+ create mode 100644 drivers/clk/samsung/clk-exynos5515.c
+ create mode 100644 include/dt-bindings/clock/samsung,exynos5515-cmu.h
+ delete mode 100644 include/linux/input/samsung-keypad.h
+Merging scmi/for-linux-next (0dcfd0f40b5be Merge branches 'for-next/juno/updates' and 'for-next/scmi/updates' of git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git scmi/for-linux-next
+Auto-merging MAINTAINERS
+Auto-merging scripts/mod/devicetable-offsets.c
+Auto-merging scripts/mod/file2alias.c
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                    |   1 +
+ arch/arm64/boot/dts/arm/fvp-base-revc.dts      |  42 +-
+ drivers/firmware/arm_scmi/bus.c                | 109 ++-
+ drivers/firmware/arm_scmi/common.h             |  61 +-
+ drivers/firmware/arm_scmi/driver.c             | 262 +++++---
+ drivers/firmware/arm_scmi/protocols.h          |   9 +-
+ drivers/firmware/arm_scmi/raw_mode.c           |  30 +-
+ drivers/firmware/arm_scmi/reset.c              |  36 +-
+ drivers/firmware/arm_scmi/transports/Kconfig   |  12 +
+ drivers/firmware/arm_scmi/transports/Makefile  |   2 +
+ drivers/firmware/arm_scmi/transports/mailbox.c |   7 +-
+ drivers/firmware/arm_scmi/transports/optee.c   |   7 +-
+ drivers/firmware/arm_scmi/transports/pcc.c     | 875 +++++++++++++++++++++++++
+ drivers/firmware/arm_scmi/transports/smc.c     |  10 +-
+ drivers/firmware/arm_scmi/transports/virtio.c  |   3 +-
+ include/linux/device-id/scmi.h                 |  17 +
+ include/linux/scmi_protocol.h                  |  20 +-
+ include/trace/events/scmi.h                    |   8 +-
+ scripts/mod/devicetable-offsets.c              |   5 +
+ scripts/mod/file2alias.c                       |  12 +
+ 20 files changed, 1342 insertions(+), 186 deletions(-)
+ create mode 100644 drivers/firmware/arm_scmi/transports/pcc.c
+ create mode 100644 include/linux/device-id/scmi.h
+Merging sophgo/for-next (76acfee87c74d Merge branch 'dt/arm' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/sophgo/linux.git sophgo/for-next
+Merge made by the 'ort' strategy.
+Merging sophgo-soc/soc-for-next (c8754c7deab4c soc: sophgo: cv1800: rtcsys: New driver (handling RTC only))
+$ git merge -m Merge branch 'soc-for-next' of https://github.com/sophgo/linux.git sophgo-soc/soc-for-next
+Already up to date.
+Merging spacemit/for-next (4b98722be2911 Merge branch 'spacemit-misc-for-next' into spacemit-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/spacemit/linux spacemit/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                      |   9 ++
+ arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts   |  35 ++++++-
+ arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts |  65 +++++++++++-
+ arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts | 126 +++++++++++++++++------
+ arch/riscv/boot/dts/spacemit/k3-com260.dtsi      |  17 +++
+ arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi     |  11 ++
+ arch/riscv/boot/dts/spacemit/k3.dtsi             | 123 +++++++++++++++++++++-
+ 7 files changed, 350 insertions(+), 36 deletions(-)
+Merging stm32/stm32-next (e6cdc9f49eebe arm64: dts: st: fix sai3b unit address on stm32mp231)
+$ git merge -m Merge branch 'stm32-next' of https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git stm32/stm32-next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/stm32/stm32.yaml       |  39 +-
+ MAINTAINERS                                        |   3 +-
+ arch/arm/boot/dts/st/stm32mp131.dtsi               |  24 +
+ arch/arm/boot/dts/st/stm32mp135.dtsi               |  44 +-
+ arch/arm/boot/dts/st/stm32mp135f-dk.dts            |  28 +-
+ arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi        |  34 +
+ arch/arm/boot/dts/st/stm32mp151.dtsi               |  20 +-
+ .../arm/boot/dts/st/stm32mp153c-lxa-fairytux2.dtsi |   2 +-
+ arch/arm/boot/dts/st/stm32mp157a-dk1-scmi.dts      |   8 +-
+ arch/arm/boot/dts/st/stm32mp157c-dk2-scmi.dts      |   8 +-
+ arch/arm/boot/dts/st/stm32mp157c-ed1-scmi.dts      |   8 +-
+ arch/arm/boot/dts/st/stm32mp157c-ev1-scmi.dts      |   8 +-
+ arch/arm/boot/dts/st/stm32mp157c-ev1.dts           |   9 +-
+ arch/arm/boot/dts/st/stm32mp157c-lxa-mc1.dts       |   2 +-
+ arch/arm/boot/dts/st/stm32mp157f-dk2-scmi.dtsi     |   1 +
+ arch/arm/boot/dts/st/stm32mp15xc-lxa-tac.dtsi      |   2 +-
+ arch/arm/boot/dts/st/stm32mp15xx-dhcom-som.dtsi    |   2 +
+ arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi          |  35 +-
+ arch/arm64/boot/dts/st/Makefile                    |  23 +
+ arch/arm64/boot/dts/st/stm32mp231.dtsi             | 105 ++-
+ arch/arm64/boot/dts/st/stm32mp235f-dk.dts          |  75 ++
+ arch/arm64/boot/dts/st/stm32mp23xc.dtsi            |   7 +
+ arch/arm64/boot/dts/st/stm32mp23xx-dhcos-bb.dts    |  15 +
+ arch/arm64/boot/dts/st/stm32mp23xx-dhcos-som.dtsi  |  51 ++
+ arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi      | 872 +++++++++++++++++++++
+ arch/arm64/boot/dts/st/stm32mp251.dtsi             |  67 +-
+ arch/arm64/boot/dts/st/stm32mp253.dtsi             |  16 +
+ ...stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtso | 111 +++
+ arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc.dts | 189 +++++
+ .../boot/dts/st/stm32mp257-engicam-microgea.dtsi   |  63 ++
+ ...microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtso |  67 ++
+ ...cam-microgea-rmm-overlay-rk050hr345-ct106a.dtso |  68 ++
+ .../dts/st/stm32mp257d-engicam-microgea-rmm.dts    | 275 +++++++
+ arch/arm64/boot/dts/st/stm32mp257f-dk.dts          |  93 +++
+ arch/arm64/boot/dts/st/stm32mp257f-ev1.dts         | 101 ++-
+ arch/arm64/boot/dts/st/stm32mp25xc.dtsi            |   7 +
+ arch/arm64/boot/dts/st/stm32mp25xx-dhcos-bb.dts    |  15 +
+ arch/arm64/boot/dts/st/stm32mp25xx-dhcos-som.dtsi  |  51 ++
+ arch/arm64/boot/dts/st/stm32mp2xxx-dhcos-som.dtsi  | 452 +++++++++++
+ arch/arm64/configs/defconfig                       |   5 +
+ 40 files changed, 2862 insertions(+), 143 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp23xc.dtsi
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp23xx-dhcos-bb.dts
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp23xx-dhcos-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtso
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc.dts
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp257-engicam-microgea.dtsi
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtso
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a.dtso
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm.dts
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp25xc.dtsi
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp25xx-dhcos-bb.dts
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp25xx-dhcos-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/st/stm32mp2xxx-dhcos-som.dtsi
+Merging sunxi/sunxi/for-next (afb622bddaa6f Merge branches 'sunxi/dt-for-7.4' and 'sunxi/clk-for-7.4' into sunxi/for-next)
+$ git merge -m Merge branch 'sunxi/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git sunxi/sunxi/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/sunxi.yaml   |  10 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |   2 +
+ arch/arm/boot/dts/allwinner/sun7i-a20.dtsi         |   2 +-
+ arch/arm64/boot/dts/allwinner/Makefile             |   1 +
+ .../boot/dts/allwinner/sun50i-a133-teclast-p80.dts | 205 +++++++++++++++++++++
+ drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c           |  37 ++--
+ drivers/clk/sunxi-ng/ccu-sun55i-a523.c             | 128 ++++++-------
+ drivers/clk/sunxi-ng/ccu_common.c                  |   2 +-
+ drivers/clk/sunxi-ng/ccu_div.h                     |  18 +-
+ 9 files changed, 308 insertions(+), 97 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/allwinner/sun50i-a133-teclast-p80.dts
+Merging tee/next (a4df08db64528 Merge branches 'qcomtee_fix_for_v7.3', 'qcomtee_for_v7.4', 'tee_for_v7.4' and 'otpee_for_v7.4' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git tee/next
+Merge made by the 'ort' strategy.
+ drivers/tee/optee/Kconfig            |  1 +
+ drivers/tee/optee/Makefile           |  4 ++--
+ drivers/tee/optee/ffa_abi.c          | 24 +++++++++-------------
+ drivers/tee/optee/notif.c            |  1 -
+ drivers/tee/optee/optee_private.h    | 39 ++++++++++++++++++++++++++++++++++-
+ drivers/tee/optee/smc_abi.c          | 17 ++++++++-------
+ drivers/tee/qcomtee/async.c          |  4 ++--
+ drivers/tee/qcomtee/qcomtee_msg.h    |  2 +-
+ drivers/tee/qcomtee/qcomtee_object.h |  1 +
+ include/uapi/linux/tee.h             | 40 ++++++++++++++++++------------------
+ 10 files changed, 83 insertions(+), 50 deletions(-)
+Merging tegra/for-next (11430072551f5 Merge branch for-7.4/arm64/dt into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git tegra/for-next
+Auto-merging drivers/soc/tegra/pmc.c
+Merge made by the 'ort' strategy.
+ .../bindings/fuse/nvidia,tegra20-fuse.yaml         |   1 +
+ .../memory-controllers/nvidia,tegra124-emc.yaml    |  10 +-
+ .../memory-controllers/nvidia,tegra124-mc.yaml     |   1 +
+ arch/arm/boot/dts/nvidia/tegra124.dtsi             |  16 +-
+ arch/arm64/boot/dts/nvidia/tegra132.dtsi           |  16 +-
+ .../dts/nvidia/tegra186-p3509-0000+p3636-0001.dts  |   8 +-
+ arch/arm64/boot/dts/nvidia/tegra186.dtsi           |  74 +--
+ arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts |   8 +-
+ .../arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi |   8 +-
+ arch/arm64/boot/dts/nvidia/tegra194.dtsi           |  78 +--
+ arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts |  10 +-
+ arch/arm64/boot/dts/nvidia/tegra210.dtsi           |   6 +-
+ .../boot/dts/nvidia/tegra234-p3737-0000+p3701.dtsi |   9 +-
+ .../dts/nvidia/tegra234-p3740-0002+p3701-0008.dts  |  45 +-
+ .../boot/dts/nvidia/tegra234-p3768-0000+p3767.dtsi |   2 +-
+ arch/arm64/boot/dts/nvidia/tegra234.dtsi           |  74 +--
+ arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi     |  57 ++
+ .../boot/dts/nvidia/tegra264-p4071-0000+p3834.dtsi |  93 ++++
+ arch/arm64/boot/dts/nvidia/tegra264.dtsi           | 580 ++++++++++++---------
+ .../boot/dts/nvidia/thermal/nvidia,tegra264-bpmp.h |  20 +
+ drivers/soc/tegra/fuse/fuse-tegra.c                |  11 +-
+ drivers/soc/tegra/fuse/fuse-tegra30.c              | 168 +++++-
+ drivers/soc/tegra/fuse/fuse.h                      |   7 +-
+ drivers/soc/tegra/pmc.c                            |  98 ++++
+ 24 files changed, 972 insertions(+), 428 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/nvidia/thermal/nvidia,tegra264-bpmp.h
+Merging tenstorrent-dt/tenstorrent-dt-for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'tenstorrent-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt/tenstorrent-dt-for-next
+Already up to date.
+Merging fustini-config/riscv-config-for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'riscv-config-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git fustini-config/riscv-config-for-next
+Already up to date.
+Merging thead-dt/thead-dt-for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'thead-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt/thead-dt-for-next
+Already up to date.
+Merging ti/ti-next (ba727be7fe4a6 Merge branches 'ti-drivers-soc-next' and 'ti-k3-dts-next' into ti-next)
+$ git merge -m Merge branch 'ti-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git ti/ti-next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/configs/defconfig
+Auto-merging drivers/firmware/ti_sci.c
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/ti/k3.yaml   |   7 +
+ .../bindings/clock/ti,am62-audio-refclk.yaml       |   8 +-
+ .../soc/ti/ti,j721e-system-controller.yaml         |   6 +-
+ MAINTAINERS                                        |   1 +
+ arch/arm/boot/dts/ti/keystone/keystone-k2e.dtsi    |   2 +-
+ arch/arm/boot/dts/ti/keystone/keystone-k2g.dtsi    |  28 +-
+ arch/arm/boot/dts/ti/keystone/keystone-k2hk.dtsi   |  16 +-
+ arch/arm/boot/dts/ti/keystone/keystone-k2l.dtsi    |  16 +-
+ arch/arm/boot/dts/ti/keystone/keystone.dtsi        |   4 +-
+ arch/arm64/boot/dts/ti/Makefile                    |  12 +
+ arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts           |   2 +-
+ arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi    |   2 +-
+ arch/arm64/boot/dts/ti/k3-am62-pocketbeagle2.dts   |   2 +-
+ arch/arm64/boot/dts/ti/k3-am62-verdin-ivy.dtsi     |  41 +-
+ arch/arm64/boot/dts/ti/k3-am62-verdin-zinnia.dtsi  |  41 +-
+ arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi         |  49 +--
+ arch/arm64/boot/dts/ti/k3-am625-beagleplay.dts     |   6 +-
+ arch/arm64/boot/dts/ti/k3-am625-sk-common.dtsi     |   3 +-
+ .../boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts      |   4 +-
+ arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi      |   2 +-
+ .../boot/dts/ti/k3-am625-var-som-symphony.dts      |   2 +-
+ arch/arm64/boot/dts/ti/k3-am625-var-som.dtsi       |   4 +-
+ arch/arm64/boot/dts/ti/k3-am62a-main.dtsi          |   1 +
+ arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi           |   2 +-
+ arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi   |   2 +-
+ arch/arm64/boot/dts/ti/k3-am62a7-sk.dts            |   6 +-
+ arch/arm64/boot/dts/ti/k3-am62l3-evm.dts           |   4 +-
+ .../boot/dts/ti/k3-am62p-j722s-common-main.dtsi    |  12 +
+ .../boot/dts/ti/k3-am62p-j722s-common-mcu.dtsi     |   2 +-
+ arch/arm64/boot/dts/ti/k3-am62x-phyboard-lyra.dtsi |   2 +-
+ arch/arm64/boot/dts/ti/k3-am62x-sk-common.dtsi     |   2 +-
+ arch/arm64/boot/dts/ti/k3-am654-base-board.dts     |   1 +
+ arch/arm64/boot/dts/ti/k3-am67-phycore-som.dtsi    | 324 ++++++++++++++++
+ .../arm64/boot/dts/ti/k3-am6754-phyboard-rigel.dts | 431 +++++++++++++++++++++
+ ...uila-dsi-to-lvds-v2-panel-cap-touch-10inch.dtso | 161 ++++++++
+ arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi         |   2 +-
+ .../boot/dts/ti/k3-j7200-common-proc-board.dts     |   1 +
+ .../boot/dts/ti/k3-j721s2-common-proc-board.dts    |   1 +
+ arch/arm64/boot/dts/ti/k3-j721s2-evm-audio.dtso    | 157 ++++++++
+ arch/arm64/boot/dts/ti/k3-j721s2-main.dtsi         |  18 +
+ arch/arm64/boot/dts/ti/k3-j722s-evm.dts            |  26 +-
+ .../boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi   |   2 +
+ arch/arm64/boot/dts/ti/k3-pinctrl.h                |  10 +
+ arch/arm64/configs/defconfig                       |  15 +-
+ drivers/firmware/ti_sci.c                          |   6 +-
+ drivers/soc/ti/knav_qmss_queue.c                   |   4 +-
+ include/linux/soc/ti/k3-ringacc.h                  |  31 +-
+ 47 files changed, 1260 insertions(+), 221 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/ti/k3-am67-phycore-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/ti/k3-am6754-phyboard-rigel.dts
+ create mode 100644 arch/arm64/boot/dts/ti/k3-am69-aquila-dsi-to-lvds-v2-panel-cap-touch-10inch.dtso
+ create mode 100644 arch/arm64/boot/dts/ti/k3-j721s2-evm-audio.dtso
+Merging xilinx/for-next (bf126d9aa20f2 Merge branch 'zynqmp/soc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/Xilinx/linux-xlnx.git xilinx/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/xilinx/zynq-7000.dtsi      |  8 ++--
+ arch/arm/boot/dts/xilinx/zynq-parallella.dts |  4 +-
+ drivers/firmware/xilinx/zynqmp-ufs.c         | 70 ++++++++++++++++++++++++++--
+ drivers/firmware/xilinx/zynqmp.c             | 44 ++++++++++++++---
+ drivers/ufs/host/ufs-amd-versal2.c           | 46 +++++-------------
+ include/linux/firmware/xlnx-zynqmp-ufs.h     |  8 ++--
+ include/linux/firmware/xlnx-zynqmp.h         |  5 --
+ 7 files changed, 126 insertions(+), 59 deletions(-)
+Merging socfpga/for-next (04a1d21330501 arm64: dts: socfpga: agilex5: Add SoCDK TSN Config2 board)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git socfpga/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/altera.yaml  |   2 +
+ .../bindings/firmware/intel,stratix10-svc.yaml     |  10 ++
+ arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi  |   4 +
+ arch/arm64/boot/dts/intel/Makefile                 |   2 +
+ arch/arm64/boot/dts/intel/socfpga_agilex.dtsi      |   2 +
+ arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi     |  61 ++++++++++
+ .../arm64/boot/dts/intel/socfpga_agilex5_socdk.dts |  38 +++++-
+ .../boot/dts/intel/socfpga_agilex5_socdk_emmc.dts  | 123 +++++++++++++++++++
+ .../dts/intel/socfpga_agilex5_socdk_tsn_cfg2.dts   | 130 +++++++++++++++++++++
+ 9 files changed, 369 insertions(+), 3 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_emmc.dts
+ create mode 100644 arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_tsn_cfg2.dts
+Merging clk/clk-next (b1470947e92ae Merge branch 'clk-pile' into clk-next)
+$ git merge -m Merge branch 'clk-next' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk/clk-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/clk/at91/clk-main.c
+Auto-merging drivers/clk/clk-scpi.c
+Auto-merging drivers/clk/renesas/r9a06g032-clocks.c
+Auto-merging drivers/clk/renesas/renesas-cpg-mssr.c
+Auto-merging drivers/clk/renesas/rzg2l-cpg.c
+Auto-merging drivers/clk/renesas/rzv2h-cpg.c
+Auto-merging drivers/clk/samsung/clk-pll.c
+Merge made by the 'ort' strategy.
+ .../bindings/clock/airoha,en7523-scu.yaml          |  24 +-
+ .../bindings/clock/anlogic,dr1v90-cru.yaml         |  60 +++
+ .../bindings/clock/clk-palmas-clk32kg-clocks.txt   |  35 --
+ .../bindings/clock/clock-nexus-node.yaml           |  30 ++
+ .../bindings/clock/ti,palmas-clk32kg.yaml          |  56 +++
+ .../devicetree/bindings/clock/ti/adpll.txt         |  39 --
+ .../devicetree/bindings/clock/ti/davinci/pll.txt   |  96 ----
+ .../bindings/clock/ti/davinci/ti,da850-pll.yaml    | 164 ++++++
+ .../devicetree/bindings/clock/ti/dra7-atl.txt      |  94 ----
+ .../devicetree/bindings/clock/ti/fapll.txt         |  31 --
+ .../bindings/clock/ti/ti,dm814-adpll-clock.yaml    |  88 ++++
+ .../bindings/clock/ti/ti,dm816-fapll-clock.yaml    |  71 +++
+ .../bindings/clock/ti/ti,dra7-atl-clock.yaml       |  38 ++
+ .../devicetree/bindings/clock/ti/ti,dra7-atl.yaml  | 100 ++++
+ MAINTAINERS                                        |  22 +-
+ arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts |   4 +
+ arch/riscv/boot/dts/anlogic/dr1v90.dtsi            |  40 +-
+ drivers/clk/Kconfig                                |   5 +
+ drivers/clk/Makefile                               |   8 +-
+ drivers/clk/actions/owl-composite.h                |   1 -
+ drivers/clk/actions/owl-fixed-factor.h             |   2 -
+ drivers/clk/actions/owl-pll.c                      |   2 +-
+ drivers/clk/anlogic/Kconfig                        |  21 +
+ drivers/clk/anlogic/Makefile                       |   7 +
+ drivers/clk/anlogic/cru-dr1v90.c                   | 192 +++++++
+ drivers/clk/anlogic/cru_dr1.c                      | 226 +++++++++
+ drivers/clk/anlogic/cru_dr1.h                      | 117 +++++
+ drivers/clk/aspeed/Kconfig                         |   1 +
+ drivers/clk/aspeed/clk-aspeed.c                    |   2 +-
+ drivers/clk/aspeed/clk-ast2600.c                   |   2 +-
+ drivers/clk/aspeed/clk-ast2700.c                   |   2 +-
+ drivers/clk/at91/clk-audio-pll.c                   |   4 +-
+ drivers/clk/at91/clk-h32mx.c                       |   2 +-
+ drivers/clk/at91/clk-main.c                        |   2 +-
+ drivers/clk/at91/clk-pll.c                         |   2 +-
+ drivers/clk/at91/clk-plldiv.c                      |   2 +-
+ drivers/clk/at91/clk-slow.c                        |   2 +-
+ drivers/clk/at91/clk-smd.c                         |   2 +-
+ drivers/clk/at91/clk-usb.c                         |   6 +-
+ drivers/clk/at91/sckc.c                            |   2 +-
+ drivers/clk/bcm/clk-iproc-armpll.c                 |   2 +-
+ drivers/clk/bcm/clk-iproc-asiu.c                   |   2 +-
+ drivers/clk/bcm/clk-iproc-pll.c                    |   2 +-
+ drivers/clk/bcm/clk-raspberrypi.c                  |   2 +
+ drivers/clk/berlin/berlin2-avpll.c                 |   4 +-
+ drivers/clk/berlin/berlin2-pll.c                   |   2 +-
+ drivers/clk/clk-axi-clkgen.c                       |   2 +-
+ drivers/clk/clk-cdce925.c                          |   2 +-
+ drivers/clk/clk-conf.c                             |  12 +-
+ drivers/clk/clk-cs2000-cp.c                        |   2 +-
+ drivers/clk/clk-en7523.c                           | 218 +++++++-
+ drivers/clk/clk-eyeq.c                             |  31 +-
+ drivers/clk/clk-fractional-divider.c               |   2 +-
+ drivers/clk/clk-gemini.c                           |   2 +-
+ drivers/clk/clk-highbank.c                         |   2 +-
+ drivers/clk/clk-lmk04832.c                         |   6 +-
+ drivers/clk/clk-milbeaut.c                         |  14 +-
+ drivers/clk/clk-nomadik.c                          |   4 +-
+ drivers/clk/clk-npcm7xx.c                          |   2 +-
+ drivers/clk/clk-pwm.c                              |   2 +-
+ drivers/clk/clk-renesas-pcie.c                     |  43 +-
+ drivers/clk/clk-s2mps11.c                          |  12 +-
+ drivers/clk/clk-scpi.c                             |   2 +-
+ drivers/clk/clk-si514.c                            |   2 +-
+ drivers/clk/clk-si521xx.c                          |  43 +-
+ drivers/clk/clk-si5341.c                           |   2 +-
+ drivers/clk/clk-si5351.c                           |  20 +-
+ drivers/clk/clk-si544.c                            |   2 +-
+ drivers/clk/clk-si570.c                            |   2 +-
+ drivers/clk/clk-stm32f4.c                          |   4 +-
+ drivers/clk/clk-stm32h7.c                          |   2 +-
+ drivers/clk/clk-versaclock5.c                      |  12 +-
+ drivers/clk/clk-vt8500.c                           |   4 +-
+ drivers/clk/clk-xgene.c                            |   6 +-
+ drivers/clk/clk.c                                  |  75 ++-
+ drivers/clk/clk.h                                  |   5 +-
+ drivers/clk/clk_kunit_helpers.c                    |  31 ++
+ drivers/clk/clk_test.c                             | 151 +++++-
+ drivers/clk/clkdev.c                               |   4 +-
+ drivers/clk/davinci/Kconfig                        |  10 +
+ drivers/clk/davinci/Makefile                       |   8 +-
+ drivers/clk/davinci/da8xx-cfgchip.c                |   4 +-
+ drivers/clk/davinci/pll.c                          |   2 +-
+ drivers/clk/davinci/psc.c                          |   2 +-
+ drivers/clk/hisilicon/clk-hi3559a.c                |   2 +-
+ drivers/clk/hisilicon/clk-hi3620.c                 |   2 +-
+ drivers/clk/hisilicon/clk-hi6220-stub.c            |   2 +-
+ drivers/clk/hisilicon/clk-hisi-phase.c             |   2 +-
+ drivers/clk/hisilicon/clk-hix5hd2.c                |   2 +-
+ drivers/clk/hisilicon/clkdivider-hi6220.c          |   2 +-
+ drivers/clk/hisilicon/clkgate-separated.c          |   2 +-
+ drivers/clk/imx/clk-busy.c                         |   4 +-
+ drivers/clk/imx/clk-cpu.c                          |   2 +-
+ drivers/clk/imx/clk-divider-gate.c                 |   2 +-
+ drivers/clk/imx/clk-fixup-div.c                    |   2 +-
+ drivers/clk/imx/clk-fixup-mux.c                    |   2 +-
+ drivers/clk/imx/clk-frac-pll.c                     |   2 +-
+ drivers/clk/imx/clk-fracn-gppll.c                  |   2 +-
+ drivers/clk/imx/clk-gate-93.c                      |   2 +-
+ drivers/clk/imx/clk-gate-exclusive.c               |   2 +-
+ drivers/clk/imx/clk-gate2.c                        |   2 +-
+ drivers/clk/imx/clk-lpcg-scu.c                     |   2 +-
+ drivers/clk/imx/clk-pfd.c                          |   2 +-
+ drivers/clk/imx/clk-pfdv2.c                        |   2 +-
+ drivers/clk/imx/clk-pll14xx.c                      |   2 +-
+ drivers/clk/imx/clk-pllv1.c                        |   2 +-
+ drivers/clk/imx/clk-pllv2.c                        |   2 +-
+ drivers/clk/imx/clk-pllv3.c                        |   2 +-
+ drivers/clk/imx/clk-pllv4.c                        |   2 +-
+ drivers/clk/imx/clk-scu.c                          |   4 +-
+ drivers/clk/imx/clk-sscg-pll.c                     |   2 +-
+ drivers/clk/ingenic/cgu.c                          |   2 +-
+ drivers/clk/keystone/gate.c                        |   2 +-
+ drivers/clk/keystone/pll.c                         |   2 +-
+ drivers/clk/keystone/syscon-clk.c                  |   2 +-
+ drivers/clk/kunit_clk_assigned_rates.h             |   4 +-
+ drivers/clk/kunit_clk_parse_clkspec.dtso           |  31 ++
+ drivers/clk/mediatek/Kconfig                       |   1 +
+ drivers/clk/mediatek/clk-cpumux.c                  |   2 +-
+ drivers/clk/mediatek/reset.h                       |   6 +-
+ drivers/clk/mmp/Kconfig                            |   5 +
+ drivers/clk/mmp/clk-apbc.c                         |   2 +-
+ drivers/clk/mmp/clk-apmu.c                         |   2 +-
+ drivers/clk/mmp/clk-frac.c                         |   2 +-
+ drivers/clk/mmp/clk-gate.c                         |   2 +-
+ drivers/clk/mmp/clk-mix.c                          |   2 +-
+ drivers/clk/mmp/clk-pll.c                          |   2 +-
+ drivers/clk/mstar/clk-msc313-mpll.c                |   2 +-
+ drivers/clk/mvebu/ap-cpu-clk.c                     |   6 +-
+ drivers/clk/mvebu/clk-corediv.c                    |   2 +-
+ drivers/clk/mvebu/clk-cpu.c                        |   2 +-
+ drivers/clk/mxs/clk-div.c                          |   2 +-
+ drivers/clk/mxs/clk-frac.c                         |   2 +-
+ drivers/clk/mxs/clk-pll.c                          |   2 +-
+ drivers/clk/mxs/clk-ref.c                          |   2 +-
+ drivers/clk/nxp/clk-lpc18xx-creg.c                 |   2 +-
+ drivers/clk/pistachio/clk-pll.c                    |   2 +-
+ drivers/clk/pistachio/clk.c                        |  15 +-
+ drivers/clk/pistachio/clk.h                        |   1 +
+ drivers/clk/renesas/Kconfig                        |   2 +
+ drivers/clk/renesas/r9a06g032-clocks.c             |   2 +-
+ drivers/clk/renesas/r9a08g046-cpg.c                |  92 +++-
+ drivers/clk/renesas/r9a09g077-cpg.c                | 142 ++++++
+ drivers/clk/renesas/rcar-gen3-cpg.c                |  15 +-
+ drivers/clk/renesas/rcar-gen3-cpg.h                |   3 +-
+ drivers/clk/renesas/renesas-cpg-mssr.c             |   3 +
+ drivers/clk/renesas/renesas-cpg-mssr.h             |   1 +
+ drivers/clk/renesas/rzg2l-cpg.c                    | 554 ++++++++++++++++++++-
+ drivers/clk/renesas/rzg2l-cpg.h                    |  51 +-
+ drivers/clk/renesas/rzv2h-cpg.c                    | 164 +++---
+ drivers/clk/rockchip/clk-cpu.c                     |   2 +-
+ drivers/clk/rockchip/clk-ddr.c                     |   2 +-
+ drivers/clk/rockchip/clk-gate-grf.c                |   2 +-
+ drivers/clk/rockchip/clk-inverter.c                |   2 +-
+ drivers/clk/rockchip/clk-mmc-phase.c               |   2 +-
+ drivers/clk/rockchip/clk-muxgrf.c                  |   2 +-
+ drivers/clk/rockchip/clk-pll.c                     |   2 +-
+ drivers/clk/rockchip/clk.c                         |   2 +-
+ drivers/clk/samsung/clk-cpu.c                      |   2 +-
+ drivers/clk/samsung/clk-exynos-clkout.c            |   8 +-
+ drivers/clk/samsung/clk-pll.c                      |   2 +-
+ drivers/clk/socfpga/clk-gate-a10.c                 |   2 +-
+ drivers/clk/socfpga/clk-gate-s10.c                 |   6 +-
+ drivers/clk/socfpga/clk-gate.c                     |   2 +-
+ drivers/clk/socfpga/clk-periph-a10.c               |   2 +-
+ drivers/clk/socfpga/clk-periph-s10.c               |   8 +-
+ drivers/clk/socfpga/clk-periph.c                   |   2 +-
+ drivers/clk/socfpga/clk-pll-a10.c                  |   2 +-
+ drivers/clk/socfpga/clk-pll-s10.c                  |   8 +-
+ drivers/clk/socfpga/clk-pll.c                      |   2 +-
+ drivers/clk/spear/clk-aux-synth.c                  |   2 +-
+ drivers/clk/spear/clk-frac-synth.c                 |   2 +-
+ drivers/clk/spear/clk-gpt-synth.c                  |   2 +-
+ drivers/clk/spear/clk-vco-pll.c                    |   2 +-
+ drivers/clk/st/clk-flexgen.c                       |   2 +-
+ drivers/clk/st/clkgen-fsyn.c                       |   4 +-
+ drivers/clk/st/clkgen-pll.c                        |   2 +-
+ drivers/clk/starfive/clk-starfive-jh7110-isp.c     |   2 +-
+ drivers/clk/starfive/clk-starfive-jh7110-sys.c     |   3 +
+ drivers/clk/starfive/clk-starfive-jh7110-vout.c    |   6 +-
+ drivers/clk/stm32/clk-stm32mp1.c                   |   4 +-
+ drivers/clk/sunxi/clk-sun4i-tcon-ch1.c             |   2 +-
+ drivers/clk/tegra/clk-audio-sync.c                 |   2 +-
+ drivers/clk/tegra/clk-divider.c                    |   2 +-
+ drivers/clk/tegra/clk-periph-fixed.c               |   2 +-
+ drivers/clk/tegra/clk-periph-gate.c                |   2 +-
+ drivers/clk/tegra/clk-periph.c                     |   2 +-
+ drivers/clk/tegra/clk-pll-out.c                    |   2 +-
+ drivers/clk/tegra/clk-pll.c                        |   2 +-
+ drivers/clk/tegra/clk-sdmmc-mux.c                  |   2 +-
+ drivers/clk/tegra/clk-super.c                      |   4 +-
+ drivers/clk/tegra/clk-tegra-super-cclk.c           |   2 +-
+ drivers/clk/tegra/clk-tegra124-emc.c               |   2 +-
+ drivers/clk/tegra/clk-tegra20-emc.c                |   2 +-
+ drivers/clk/tegra/clk-tegra210-emc.c               |   2 +-
+ drivers/clk/tegra/clk.h                            |   2 +-
+ drivers/clk/ti/adpll.c                             |   1 +
+ drivers/clk/ti/clockdomain.c                       |   2 +-
+ drivers/clk/uniphier/clk-uniphier-cpugear.c        |   2 +-
+ drivers/clk/uniphier/clk-uniphier-fixed-factor.c   |   2 +-
+ drivers/clk/uniphier/clk-uniphier-fixed-rate.c     |   2 +-
+ drivers/clk/uniphier/clk-uniphier-gate.c           |   2 +-
+ drivers/clk/uniphier/clk-uniphier-mux.c            |   2 +-
+ drivers/clk/ux500/clk-prcc.c                       |   2 +-
+ drivers/clk/ux500/clk-prcmu.c                      |   4 +-
+ drivers/clk/ux500/clk-sysctrl.c                    |   2 +-
+ drivers/clk/versatile/clk-icst.c                   |   6 +-
+ drivers/clk/versatile/clk-sp810.c                  |   2 +-
+ drivers/clk/versatile/clk-vexpress-osc.c           |   2 +-
+ drivers/clk/x86/clk-pmc-atom.c                     |   2 +-
+ drivers/clk/xilinx/clk-xlnx-clock-wizard.c         |  18 +-
+ drivers/clk/xilinx/xlnx_vcu.c                      |   2 +-
+ drivers/clk/zynqmp/clk-gate-zynqmp.c               |   2 +-
+ drivers/clk/zynqmp/clk-mux-zynqmp.c                |   2 +-
+ drivers/clk/zynqmp/divider.c                       |   2 +-
+ drivers/clk/zynqmp/pll.c                           |   2 +-
+ drivers/reset/Kconfig                              |  10 +
+ drivers/reset/Makefile                             |   1 +
+ drivers/reset/reset-dr1v90.c                       | 140 ++++++
+ include/dt-bindings/clock/anlogic,dr1v90-cru.h     |  46 ++
+ include/dt-bindings/reset/anlogic,dr1v90-cru.h     |  41 ++
+ include/dt-bindings/soc/airoha,scu-ssr.h           |  11 +
+ include/kunit/clk.h                                |   2 +
+ include/linux/clk-provider.h                       |  25 +-
+ include/linux/clk/tegra.h                          |   2 +-
+ include/linux/clk/ti.h                             |   2 +-
+ rust/kernel/clk.rs                                 |  15 +
+ 227 files changed, 3273 insertions(+), 771 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/anlogic,dr1v90-cru.yaml
+ delete mode 100644 Documentation/devicetree/bindings/clock/clk-palmas-clk32kg-clocks.txt
+ create mode 100644 Documentation/devicetree/bindings/clock/clock-nexus-node.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/ti,palmas-clk32kg.yaml
+ delete mode 100644 Documentation/devicetree/bindings/clock/ti/adpll.txt
+ delete mode 100644 Documentation/devicetree/bindings/clock/ti/davinci/pll.txt
+ create mode 100644 Documentation/devicetree/bindings/clock/ti/davinci/ti,da850-pll.yaml
+ delete mode 100644 Documentation/devicetree/bindings/clock/ti/dra7-atl.txt
+ delete mode 100644 Documentation/devicetree/bindings/clock/ti/fapll.txt
+ create mode 100644 Documentation/devicetree/bindings/clock/ti/ti,dm814-adpll-clock.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/ti/ti,dm816-fapll-clock.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/ti/ti,dra7-atl-clock.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/ti/ti,dra7-atl.yaml
+ create mode 100644 drivers/clk/anlogic/Kconfig
+ create mode 100644 drivers/clk/anlogic/Makefile
+ create mode 100644 drivers/clk/anlogic/cru-dr1v90.c
+ create mode 100644 drivers/clk/anlogic/cru_dr1.c
+ create mode 100644 drivers/clk/anlogic/cru_dr1.h
+ create mode 100644 drivers/clk/davinci/Kconfig
+ create mode 100644 drivers/clk/kunit_clk_parse_clkspec.dtso
+ create mode 100644 drivers/reset/reset-dr1v90.c
+ create mode 100644 include/dt-bindings/clock/anlogic,dr1v90-cru.h
+ create mode 100644 include/dt-bindings/reset/anlogic,dr1v90-cru.h
+ create mode 100644 include/dt-bindings/soc/airoha,scu-ssr.h
+Merging clk-imx/for-next (39ec460b56b26 clk: imx95-blk-ctl: Fix REFCLK rise-fall mismatch on i.MX95)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git clk-imx/for-next
+Already up to date.
+Merging clk-renesas/renesas-clk (21cbd7ec29930 clk: renesas: r8a78000: Add SGASYNCD8_PERW_BUS)
+$ git merge -m Merge branch 'renesas-clk' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git clk-renesas/renesas-clk
+Auto-merging drivers/clk/renesas/renesas-cpg-mssr.c
+Merge made by the 'ort' strategy.
+ drivers/clk/renesas/Kconfig             |   5 +
+ drivers/clk/renesas/Makefile            |   1 +
+ drivers/clk/renesas/r8a774a3-cpg-mssr.c | 168 ++++++++++++++++++++++++++++++++
+ drivers/clk/renesas/r8a779a0-cpg-mssr.c |   6 +-
+ drivers/clk/renesas/r8a779f0-cpg-mssr.c |   4 +-
+ drivers/clk/renesas/r8a779g0-cpg-mssr.c |   4 +-
+ drivers/clk/renesas/r8a779h0-cpg-mssr.c |   8 +-
+ drivers/clk/renesas/r8a78000-cpg.c      |   3 +
+ drivers/clk/renesas/r9a07g043-cpg.c     |  12 +--
+ drivers/clk/renesas/r9a07g044-cpg.c     |  12 +--
+ drivers/clk/renesas/r9a08g045-cpg.c     |   6 +-
+ drivers/clk/renesas/r9a08g046-cpg.c     |  25 ++++-
+ drivers/clk/renesas/rcar-gen4-cpg.c     |  25 +++--
+ drivers/clk/renesas/rcar-gen4-cpg.h     |  14 ++-
+ drivers/clk/renesas/renesas-cpg-mssr.c  |   6 ++
+ drivers/clk/renesas/renesas-cpg-mssr.h  |   1 +
+ 16 files changed, 257 insertions(+), 43 deletions(-)
+ create mode 100644 drivers/clk/renesas/r8a774a3-cpg-mssr.c
+Merging thead-clk/thead-clk-for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'thead-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk/thead-clk-for-next
+Already up to date.
+Merging tenstorrent-clk/tenstorrent-clk-for-next (c64b54ddb692c MAINTAINERS: Update RISC-V Tenstorrent SoC entry)
+$ git merge -m Merge branch 'tenstorrent-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk/tenstorrent-clk-for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                             |  5 +++--
+ drivers/clk/tenstorrent/atlantis-prcm.c | 35 +++++++++++++--------------------
+ 2 files changed, 17 insertions(+), 23 deletions(-)
+Merging csky/linux-next (abb81e5ce7d99 csky: Fix a4/a5 restoration in syscall trace path)
+$ git merge -m Merge branch 'linux-next' of https://github.com/c-sky/csky-linux.git csky/linux-next
+Already up to date.
+Merging loongarch/loongarch-next (a2628ce4ddb68 perf build: Add clang and rust target flags for LoongArch)
+$ git merge -m Merge branch 'loongarch-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git loongarch/loongarch-next
+Already up to date.
+Merging m68k/for-next (af32c3cb72529 m68k: defconfig: Enable ATARI_SVETHLANA and ETHOC)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k/for-next
+Merge made by the 'ort' strategy.
+ arch/m68k/Kconfig.devices            | 12 ++++++
+ arch/m68k/apollo/dn_ints.c           |  6 +++
+ arch/m68k/atari/config.c             | 75 ++++++++++++++++++++++++++++++++++++
+ arch/m68k/configs/amiga_defconfig    |  4 ++
+ arch/m68k/configs/apollo_defconfig   |  3 ++
+ arch/m68k/configs/atari_defconfig    |  5 +++
+ arch/m68k/configs/bvme6000_defconfig |  3 ++
+ arch/m68k/configs/hp300_defconfig    |  3 ++
+ arch/m68k/configs/mac_defconfig      |  3 ++
+ arch/m68k/configs/multi_defconfig    |  6 +++
+ arch/m68k/configs/mvme147_defconfig  |  3 ++
+ arch/m68k/configs/mvme16x_defconfig  |  3 ++
+ arch/m68k/configs/q40_defconfig      |  3 ++
+ arch/m68k/configs/sun3_defconfig     |  3 ++
+ arch/m68k/configs/sun3x_defconfig    |  3 ++
+ arch/m68k/include/asm/atariints.h    |  2 +-
+ arch/m68k/include/asm/irq.h          |  6 +--
+ arch/m68k/include/asm/serial.h       | 21 +++-------
+ drivers/zorro/zorro.c                |  5 +--
+ 19 files changed, 145 insertions(+), 24 deletions(-)
+Merging m68knommu/for-next (29036c5910df0 m68knommu: fix compile breakage for 5407 cleopatra board)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git m68knommu/for-next
+Merge made by the 'ort' strategy.
+ arch/m68k/68000/entry.S              |  3 +-
+ arch/m68k/coldfire/entry.S           |  3 +-
+ arch/m68k/coldfire/m5407.c           |  5 ++++
+ arch/m68k/coldfire/reset.c           |  2 +-
+ arch/m68k/configs/stmark2_defconfig  |  2 ++
+ arch/m68k/include/asm/m53xxacr.h     |  2 +-
+ arch/m68k/include/asm/quicc_simple.h | 53 ------------------------------------
+ 7 files changed, 13 insertions(+), 57 deletions(-)
+ delete mode 100644 arch/m68k/include/asm/quicc_simple.h
+Merging microblaze/next (6b125c73aaa0d microblaze: Enable xilinx dmas and axi emac drivers)
+$ git merge -m Merge branch 'next' of git://git.monstr.eu/linux-2.6-microblaze.git microblaze/next
+Merge made by the 'ort' strategy.
+ arch/microblaze/Kconfig                          |   9 +
+ arch/microblaze/Makefile                         |   8 +-
+ arch/microblaze/configs/mmu_defconfig            |   2 +
+ arch/microblaze/include/asm/entry.h              |  13 +
+ arch/microblaze/include/asm/processor.h          |   2 +-
+ arch/microblaze/include/asm/uaccess.h            |   3 +-
+ arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c    |   2 +-
+ arch/microblaze/kernel/cpu/cpuinfo-static.c      |   2 +-
+ arch/microblaze/kernel/cpu/cpuinfo.c             |   1 +
+ arch/microblaze/kernel/entry.S                   | 348 ++++++++++++-----------
+ arch/microblaze/kernel/hw_exception_handler.S    |   5 +
+ arch/microblaze/kernel/process.c                 |   5 +-
+ arch/microblaze/kernel/ptrace.c                  |   3 +
+ arch/microblaze/kernel/reset.c                   |  22 ++
+ arch/microblaze/kernel/signal.c                  |  26 ++
+ arch/microblaze/kernel/syscalls/syscall.tbl      |   2 +-
+ tools/testing/kunit/qemu_configs/microblazebe.py |  24 ++
+ tools/testing/kunit/qemu_configs/microblazeel.py |  27 ++
+ 18 files changed, 328 insertions(+), 176 deletions(-)
+ create mode 100644 tools/testing/kunit/qemu_configs/microblazebe.py
+ create mode 100644 tools/testing/kunit/qemu_configs/microblazeel.py
+Merging mips/mips-next (f552e14f77bfc mips: dts: pic32: pic32mzda: Remove unused address-cells, size-cells property)
+$ git merge -m Merge branch 'mips-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips/mips-next
+Merge made by the 'ort' strategy.
+ .../ABI/testing/sysfs-firmware-lefi-boardinfo      | 21 +++---
+ arch/mips/Makefile                                 | 32 +++++----
+ arch/mips/alchemy/common/clock.c                   |  8 +--
+ arch/mips/boot/dts/pic32/pic32mzda.dtsi            |  2 -
+ arch/mips/cavium-octeon/dma-octeon.c               |  2 +-
+ arch/mips/cavium-octeon/executive/cvmx-helper.c    | 20 ++++--
+ arch/mips/cavium-octeon/executive/cvmx-l2c.c       | 31 ++++----
+ arch/mips/cavium-octeon/flash_setup.c              |  2 +-
+ arch/mips/cavium-octeon/octeon-memcpy.S            |  2 +-
+ arch/mips/cavium-octeon/setup.c                    | 22 +++++-
+ arch/mips/configs/decstation_64_defconfig          | 44 +++++-------
+ arch/mips/dec/setup.c                              |  4 +-
+ arch/mips/generic/board-jaguar2.its.S              |  2 -
+ arch/mips/generic/board-serval.its.S               |  1 -
+ arch/mips/include/asm/amon.h                       | 12 ----
+ arch/mips/include/asm/cmp.h                        | 10 ---
+ arch/mips/include/asm/dec/reset.h                  |  1 +
+ arch/mips/include/asm/mach-loongson64/boot_param.h |  2 +-
+ arch/mips/include/asm/mips-boards/sead3-addr.h     | 83 ----------------------
+ arch/mips/lib/memcpy.S                             |  2 +-
+ arch/mips/mm/mmap.c                                |  6 +-
+ arch/mips/mm/tlb-r4k.c                             | 21 ++++--
+ arch/mips/n64/init.c                               |  2 +-
+ arch/mips/rb532/gpio.c                             |  7 +-
+ 24 files changed, 133 insertions(+), 206 deletions(-)
+ delete mode 100644 arch/mips/include/asm/amon.h
+ delete mode 100644 arch/mips/include/asm/cmp.h
+ delete mode 100644 arch/mips/include/asm/mips-boards/sead3-addr.h
+Merging openrisc/for-next (6620f5e8c11c4 openrisc: drop unneeded semicolon)
+$ git merge -m Merge branch 'for-next' of https://github.com/openrisc/linux.git openrisc/for-next
+Already up to date.
+Merging parisc-hd/for-next (a5f6df25261b3 parisc: fix typos in comments in parport.c)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git parisc-hd/for-next
+Merge made by the 'ort' strategy.
+ arch/parisc/include/asm/pdcpat.h        | 4 ++--
+ arch/parisc/include/asm/special_insns.h | 2 +-
+ arch/parisc/include/uapi/asm/mman.h     | 2 +-
+ arch/parisc/lib/memcpy.c                | 2 +-
+ drivers/parisc/ccio-dma.c               | 2 +-
+ drivers/parisc/dino.c                   | 2 +-
+ drivers/parisc/pdc_stable.c             | 2 +-
+ drivers/parisc/sba_iommu.c              | 2 +-
+ drivers/parport/parport_gsc.c           | 2 +-
+ 9 files changed, 10 insertions(+), 10 deletions(-)
+Merging powerpc/next (12d238d584ba4 powerpc/sysfs: Remove redundant wait time clamps)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc/next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/ppc-memtrace        |   2 +-
+ arch/powerpc/boot/dts/ac14xx.dts              |  13 ---
+ arch/powerpc/boot/dts/mpc5121.dtsi            |   6 +-
+ arch/powerpc/boot/dts/pdm360ng.dts            |   8 +-
+ arch/powerpc/include/asm/hardirq.h            |  31 +++---
+ arch/powerpc/kernel/dbell.c                   |   2 +-
+ arch/powerpc/kernel/irq.c                     | 131 +++++++++++++-------------
+ arch/powerpc/kernel/sysfs.c                   |   4 +-
+ arch/powerpc/kernel/time.c                    |   6 +-
+ arch/powerpc/kernel/trace/ftrace_entry.S      |   6 ++
+ arch/powerpc/kernel/traps.c                   |  11 +--
+ arch/powerpc/kernel/watchdog.c                |   2 +-
+ arch/powerpc/kvm/guest-state-buffer.c         |   2 +-
+ arch/powerpc/lib/sstep.c                      | 111 +++++++++++-----------
+ arch/powerpc/platforms/512x/Kconfig           |   3 +-
+ arch/powerpc/platforms/512x/Makefile          |   1 -
+ arch/powerpc/platforms/512x/mpc512x_generic.c |   1 +
+ arch/powerpc/platforms/512x/pdm360ng.c        | 126 -------------------------
+ arch/powerpc/platforms/ps3/interrupt.c        |  22 ++---
+ 19 files changed, 182 insertions(+), 306 deletions(-)
+ delete mode 100644 arch/powerpc/platforms/512x/pdm360ng.c
+Merging risc-v/for-next (a06776565b9e7 riscv: process: Use str_supported_unsupported() in compat_mode_detect())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v/for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/Kconfig.errata       |  1 +
+ arch/riscv/errata/mips/Makefile |  6 ++++++
+ arch/riscv/errata/mips/errata.c | 38 +++++++++++++++++++++++---------------
+ arch/riscv/kernel/process.c     |  3 ++-
+ 4 files changed, 32 insertions(+), 16 deletions(-)
+Merging riscv-dt/riscv-dt-for-next (f55a54f07a367 MAINTAINERS: update RISC-V Microchip maintainers)
+$ git merge -m Merge branch 'riscv-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt/riscv-dt-for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging riscv-soc/riscv-soc-for-next (00bdd4d4daaae Merge branch 'riscv-firmware-for-next' into riscv-soc-for-next)
+$ git merge -m Merge branch 'riscv-soc-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc/riscv-soc-for-next
+Merge made by the 'ort' strategy.
+ drivers/firmware/microchip/mpfs-auto-update.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging s390/for-next (c57ae907d4f9a Merge branch 'features' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390/for-next
+Auto-merging arch/s390/kernel/traps.c
+Merge made by the 'ort' strategy.
+ arch/s390/boot/ipl_parm.c            |  13 +-
+ arch/s390/include/asm/ebcdic.h       |   6 +
+ arch/s390/include/asm/entry-percpu.h |  71 ++-----
+ arch/s390/include/asm/idals.h        |   9 +-
+ arch/s390/include/asm/lowcore.h      |   5 +-
+ arch/s390/include/asm/percpu.h       | 354 ++++++++++++++++++++++-------------
+ arch/s390/kernel/ebcdic.c            |  40 +++-
+ arch/s390/kernel/hiperdispatch.c     |   2 +-
+ arch/s390/kernel/irq.c               |  10 +-
+ arch/s390/kernel/nmi.c               |   4 +-
+ arch/s390/kernel/traps.c             |   4 +-
+ arch/s390/lib/spinlock.c             |   4 +
+ drivers/s390/char/keyboard.c         |   6 +-
+ drivers/s390/char/sclp_vt220.c       |  87 +++++++--
+ drivers/s390/cio/chsc.c              |  20 +-
+ drivers/s390/cio/chsc_sch.c          | 300 +++++++++++------------------
+ drivers/s390/cio/cmf.c               |  11 +-
+ drivers/s390/cio/qdio.h              |   2 +-
+ drivers/s390/cio/qdio_main.c         |  23 +--
+ drivers/s390/cio/qdio_setup.c        |  14 +-
+ drivers/s390/cio/qdio_thinint.c      |   2 +-
+ drivers/s390/cio/scm.c               |   7 +-
+ 22 files changed, 538 insertions(+), 456 deletions(-)
+Merging sh/for-next (be5a19d95030f sh: ecovec24: Use static device properties to describe the touchscreen)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git sh/for-next
+Merge made by the 'ort' strategy.
+ arch/sh/boards/mach-ecovec24/setup.c         |  38 ++----
+ arch/sh/boards/mach-x3proto/gpio.c           |  15 ++-
+ arch/sh/boards/mach-x3proto/setup.c          | 165 +++++++++++++--------------
+ arch/sh/include/mach-x3proto/mach/hardware.h |   2 +
+ 4 files changed, 105 insertions(+), 115 deletions(-)
+Merging sparc/for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc/for-next
+Already up to date.
+Merging uml/next (2f88f5689de1a um: fix shutdown __inittext access)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml/next
+Auto-merging arch/um/Kconfig
+CONFLICT (content): Merge conflict in arch/um/Kconfig
+Auto-merging lib/Kconfig.debug
+Resolved 'arch/um/Kconfig' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 2e7c2602f0ee8] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/virt/index.rst            |   1 +
+ Documentation/virt/uml/nommu-uml.rst    |  54 ++++++++
+ arch/um/Kconfig                         |  38 +++--
+ arch/um/Makefile                        |   7 +-
+ arch/um/drivers/Kconfig                 |   3 +
+ arch/um/drivers/cow_user.c              |  34 ++++-
+ arch/um/drivers/mconsole_kern.c         |  14 +-
+ arch/um/drivers/vector_kern.c           |  37 +++--
+ arch/um/drivers/vector_user.c           |   3 +
+ arch/um/drivers/virtio_pcidev.c         |  19 ++-
+ arch/um/drivers/virtio_uml.c            |  26 ++--
+ arch/um/include/asm/common.lds.S        |   2 +
+ arch/um/include/asm/futex.h             |  53 +++++++
+ arch/um/include/asm/mmu.h               |   8 ++
+ arch/um/include/asm/mmu_context.h       |   2 +
+ arch/um/include/asm/perf_event.h        |   7 +
+ arch/um/include/asm/pgtable.h           |  25 ++--
+ arch/um/include/asm/ptrace-generic.h    |   6 +
+ arch/um/include/asm/tlbflush.h          |  20 +++
+ arch/um/include/asm/uaccess.h           |   6 +-
+ arch/um/include/shared/as-layout.h      |   3 +-
+ arch/um/include/shared/kern_util.h      |   3 +-
+ arch/um/include/shared/longjmp.h        |   3 +-
+ arch/um/include/shared/os.h             |   6 +-
+ arch/um/include/shared/skas/stub-data.h |   9 ++
+ arch/um/kernel/Makefile                 |   6 +-
+ arch/um/kernel/dyn.lds.S                |   9 +-
+ arch/um/kernel/mem-pgtable.c            |  55 ++++++++
+ arch/um/kernel/mem.c                    |  47 ++-----
+ arch/um/kernel/physmem.c                |   7 +
+ arch/um/kernel/process.c                |  23 +++-
+ arch/um/kernel/skas/Makefile            |  16 ++-
+ arch/um/kernel/skas/mmu.c               |  25 ++++
+ arch/um/kernel/skas/nommu.c             |  91 ++++++++++++
+ arch/um/kernel/skas/process.c           |  32 +----
+ arch/um/kernel/skas/syscall.c           |   5 +-
+ arch/um/kernel/skas/uaccess.c           | 134 +++++++-----------
+ arch/um/kernel/smp.c                    |   3 +-
+ arch/um/kernel/sysrq.c                  |   2 +-
+ arch/um/kernel/time.c                   |   3 +-
+ arch/um/kernel/tlb.c                    |   7 -
+ arch/um/kernel/trap-mmu.c               | 237 ++++++++++++++++++++++++++++++++
+ arch/um/kernel/trap-nommu.c             |  13 ++
+ arch/um/kernel/trap.c                   | 219 -----------------------------
+ arch/um/kernel/um_arch.c                |  11 +-
+ arch/um/kernel/uml.lds.S                |  11 +-
+ arch/um/os-Linux/main.c                 |  17 ++-
+ arch/um/os-Linux/signal.c               |   2 +-
+ arch/um/os-Linux/skas/process.c         |  11 +-
+ arch/x86/um/Makefile                    |   3 +
+ arch/x86/um/asm/elf.h                   |   8 +-
+ arch/x86/um/asm/required-features.h     |   9 --
+ arch/x86/um/vdso/vma.c                  |  13 +-
+ fs/Kconfig.binfmt                       |   2 +-
+ lib/Kconfig.debug                       |  10 +-
+ lib/kunit/Kconfig                       |   2 +-
+ 56 files changed, 906 insertions(+), 516 deletions(-)
+ create mode 100644 Documentation/virt/uml/nommu-uml.rst
+ create mode 100644 arch/um/include/asm/perf_event.h
+ create mode 100644 arch/um/kernel/mem-pgtable.c
+ create mode 100644 arch/um/kernel/skas/nommu.c
+ create mode 100644 arch/um/kernel/trap-mmu.c
+ create mode 100644 arch/um/kernel/trap-nommu.c
+ delete mode 100644 arch/x86/um/asm/required-features.h
+Merging xtensa/xtensa-for-next (28722ed2527aa xtensa: time: Fix clk reference leak in calibrate_ccount())
+$ git merge -m Merge branch 'xtensa-for-next' of https://github.com/jcmvbkbc/linux-xtensa.git xtensa/xtensa-for-next
+Merge made by the 'ort' strategy.
+ arch/xtensa/include/asm/atomic.h    | 13 -------------
+ arch/xtensa/include/uapi/asm/mman.h |  2 +-
+ arch/xtensa/kernel/entry.S          |  2 +-
+ arch/xtensa/kernel/time.c           |  1 +
+ arch/xtensa/kernel/vectors.S        |  2 +-
+ arch/xtensa/mm/init.c               |  2 +-
+ 6 files changed, 5 insertions(+), 17 deletions(-)
+Merging fs-next (bf234c28d9e24 Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git)
+$ git merge -m Merge branch 'fs-next' of linux-next fs-next
+Auto-merging CREDITS
+Auto-merging Documentation/filesystems/vfs.rst
+Auto-merging MAINTAINERS
+Auto-merging arch/powerpc/Kconfig
+Auto-merging drivers/android/binder/rust_binderfs.c
+Auto-merging drivers/android/binderfs.c
+Auto-merging fs/Kconfig
+Auto-merging fs/buffer.c
+Auto-merging fs/ceph/addr.c
+Auto-merging fs/coredump.c
+CONFLICT (content): Merge conflict in fs/coredump.c
+Auto-merging fs/crypto/crypto.c
+Auto-merging fs/f2fs/compress.c
+Auto-merging fs/f2fs/data.c
+Auto-merging fs/f2fs/f2fs.h
+CONFLICT (content): Merge conflict in fs/f2fs/f2fs.h
+Auto-merging fs/f2fs/segment.c
+Auto-merging fs/fat/fat.h
+Auto-merging fs/fat/file.c
+Auto-merging fs/fat/misc.c
+Auto-merging fs/fuse/dax.c
+CONFLICT (content): Merge conflict in fs/fuse/dax.c
+Auto-merging fs/hugetlbfs/inode.c
+Auto-merging fs/iomap/buffered-io.c
+Auto-merging fs/nfs/write.c
+Auto-merging fs/ocfs2/journal.c
+Auto-merging fs/ocfs2/quota_local.c
+Auto-merging fs/ocfs2/xattr.c
+Auto-merging fs/proc/base.c
+Auto-merging fs/proc/internal.h
+Auto-merging fs/proc/vmcore.c
+Auto-merging fs/ubifs/file.c
+Auto-merging fs/xfs/libxfs/xfs_btree.c
+CONFLICT (content): Merge conflict in fs/xfs/libxfs/xfs_btree.c
+Auto-merging include/linux/buffer_head.h
+Auto-merging include/linux/sched.h
+Auto-merging init/Kconfig
+Auto-merging ipc/mqueue.c
+Auto-merging kernel/Makefile
+Auto-merging kernel/fork.c
+Auto-merging mm/secretmem.c
+Auto-merging mm/shmem.c
+Auto-merging mm/userfaultfd.c
+Auto-merging security/selinux/selinuxfs.c
+Resolved 'fs/coredump.c' using previous resolution.
+Resolved 'fs/f2fs/f2fs.h' using previous resolution.
+Resolved 'fs/fuse/dax.c' using previous resolution.
+Resolved 'fs/xfs/libxfs/xfs_btree.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master c0c20ac78811b] Merge branch 'fs-next' of linux-next
+$ git diff -M --stat --summary HEAD^..
+ CREDITS                                            |    2 +-
+ Documentation/ABI/testing/sysfs-fs-f2fs            |    7 +
+ Documentation/admin-guide/binfmt-misc.rst          |    3 +
+ Documentation/admin-guide/nfs/pnfs-scsi-server.rst |    2 +-
+ Documentation/filesystems/befs.rst                 |    4 +-
+ Documentation/filesystems/bfs.rst                  |   60 -
+ Documentation/filesystems/exfat.rst                |  117 ++
+ Documentation/filesystems/index.rst                |    2 +-
+ Documentation/filesystems/locking.rst              |   24 +-
+ Documentation/filesystems/netfs_library.rst        |    7 +-
+ Documentation/filesystems/ntfs3.rst                |   23 +
+ Documentation/filesystems/porting.rst              |   25 +-
+ Documentation/filesystems/proc.rst                 |    4 +
+ Documentation/filesystems/sharedsubtree.rst        |   20 +-
+ Documentation/filesystems/smb/ksmbd.rst            |   14 +-
+ Documentation/filesystems/squashfs.rst             |    6 +-
+ Documentation/filesystems/vfs.rst                  |   30 +-
+ .../filesystems/xfs/xfs-online-fsck-design.rst     |   13 +-
+ MAINTAINERS                                        |   20 +-
+ arch/mips/configs/malta_defconfig                  |    1 -
+ arch/mips/configs/malta_kvm_defconfig              |    1 -
+ arch/mips/configs/maltaup_xpa_defconfig            |    1 -
+ arch/mips/configs/rm200_defconfig                  |    1 -
+ arch/powerpc/Kconfig                               |    1 -
+ arch/powerpc/configs/fsl-emb-nonhw.config          |    1 -
+ arch/powerpc/configs/ppc6xx_defconfig              |    1 -
+ arch/powerpc/include/asm/elf.h                     |    6 -
+ arch/powerpc/include/asm/spu.h                     |    3 -
+ arch/powerpc/platforms/cell/Kconfig                |    1 -
+ arch/powerpc/platforms/cell/spu_syscalls.c         |   20 -
+ arch/powerpc/platforms/cell/spufs/Makefile         |    1 -
+ arch/powerpc/platforms/cell/spufs/coredump.c       |  183 --
+ arch/powerpc/platforms/cell/spufs/file.c           |  114 --
+ arch/powerpc/platforms/cell/spufs/inode.c          |   14 +-
+ arch/powerpc/platforms/cell/spufs/spufs.h          |   12 -
+ arch/powerpc/platforms/cell/spufs/syscalls.c       |    4 -
+ block/bio-integrity-fs.c                           |   15 +-
+ block/bio-integrity.c                              |    1 +
+ block/bio.c                                        |  219 +--
+ block/blk-map.c                                    |    2 +-
+ block/blk-settings.c                               |    6 +
+ block/fops.c                                       |    3 +-
+ block/partitions/efi.h                             |    8 +-
+ drivers/acpi/acpi_configfs.c                       |    2 +-
+ drivers/android/binder/rust_binderfs.c             |    2 +-
+ drivers/android/binderfs.c                         |    2 +-
+ drivers/base/devtmpfs.c                            |  108 +-
+ drivers/gpio/gpiolib-cdev.c                        |   18 +-
+ drivers/gpu/drm/msm/msm_perfcntr.c                 |    4 +-
+ drivers/gpu/drm/xe/xe_configfs.c                   |    4 +-
+ drivers/media/mc/mc-request.c                      |    8 +-
+ drivers/misc/ntsync.c                              |    6 +-
+ drivers/virt/coco/guest/report.c                   |    6 +-
+ fs/9p/acl.c                                        |    4 +-
+ fs/9p/acl.h                                        |    4 +-
+ fs/9p/v9fs.h                                       |    2 +-
+ fs/9p/v9fs_vfs.h                                   |    2 +-
+ fs/9p/vfs_addr.c                                   |    1 -
+ fs/9p/vfs_inode.c                                  |   14 +-
+ fs/9p/vfs_inode_dotl.c                             |   14 +-
+ fs/9p/xattr.c                                      |    2 +-
+ fs/Kconfig                                         |    9 +-
+ fs/Makefile                                        |    1 -
+ fs/adfs/adfs.h                                     |    2 +-
+ fs/adfs/dir.c                                      |    2 +-
+ fs/adfs/inode.c                                    |    2 +-
+ fs/affs/affs.h                                     |   10 +-
+ fs/affs/inode.c                                    |    2 +-
+ fs/affs/namei.c                                    |    8 +-
+ fs/afs/dir.c                                       |   16 +-
+ fs/afs/file.c                                      |    8 +-
+ fs/afs/inode.c                                     |    4 +-
+ fs/afs/internal.h                                  |    8 +-
+ fs/afs/security.c                                  |    2 +-
+ fs/afs/xattr.c                                     |    4 +-
+ fs/aio.c                                           |   11 +-
+ fs/anon_inodes.c                                   |    4 +-
+ fs/attr.c                                          |   16 +-
+ fs/autofs/root.c                                   |   12 +-
+ fs/backing-file.c                                  |    2 +-
+ fs/bad_inode.c                                     |   20 +-
+ fs/bfs/Kconfig                                     |   21 -
+ fs/bfs/Makefile                                    |    8 -
+ fs/bfs/bfs.h                                       |   69 -
+ fs/bfs/dir.c                                       |    4 +-
+ fs/bfs/file.c                                      |  203 --
+ fs/bfs/inode.c                                     |  538 ------
+ fs/binfmt_elf.c                                    |   18 +-
+ fs/binfmt_elf_fdpic.c                              |   14 +-
+ fs/binfmt_misc.c                                   |   12 +-
+ fs/bpf_fs_kfuncs.c                                 |    4 +-
+ fs/btrfs/Kconfig                                   |    1 +
+ fs/btrfs/acl.c                                     |    2 +-
+ fs/btrfs/acl.h                                     |    2 +-
+ fs/btrfs/bio.c                                     |  163 +-
+ fs/btrfs/bio.h                                     |    9 +-
+ fs/btrfs/block-group.c                             |  498 +----
+ fs/btrfs/block-group.h                             |   16 +-
+ fs/btrfs/btrfs_inode.h                             |   24 +-
+ fs/btrfs/compression.c                             |   61 +-
+ fs/btrfs/ctree.c                                   |    4 +-
+ fs/btrfs/delalloc-space.c                          |   13 +-
+ fs/btrfs/delayed-inode.c                           |  115 +-
+ fs/btrfs/delayed-inode.h                           |   22 +-
+ fs/btrfs/dev-replace.c                             |   10 +-
+ fs/btrfs/dir-item.c                                |   41 +-
+ fs/btrfs/dir-item.h                                |    5 +-
+ fs/btrfs/disk-io.c                                 |  126 +-
+ fs/btrfs/extent-io-tree.c                          |    2 +-
+ fs/btrfs/extent-tree.c                             |   10 +
+ fs/btrfs/extent_io.c                               |  247 ++-
+ fs/btrfs/extent_map.c                              |    8 +
+ fs/btrfs/file-item.c                               |   46 +-
+ fs/btrfs/free-space-cache.c                        | 1344 +------------
+ fs/btrfs/free-space-cache.h                        |   29 +-
+ fs/btrfs/fs.c                                      |    2 +-
+ fs/btrfs/fs.h                                      |    4 +-
+ fs/btrfs/inode.c                                   |  366 ++--
+ fs/btrfs/ioctl.c                                   |   57 +-
+ fs/btrfs/ioctl.h                                   |    2 +-
+ fs/btrfs/ordered-data.c                            |   27 +-
+ fs/btrfs/qgroup.c                                  |  126 +-
+ fs/btrfs/qgroup.h                                  |   16 +-
+ fs/btrfs/raid56.c                                  |   57 +-
+ fs/btrfs/relocation.c                              |    2 +-
+ fs/btrfs/send.c                                    |    2 +-
+ fs/btrfs/space-info.c                              |    2 -
+ fs/btrfs/space-info.h                              |    4 -
+ fs/btrfs/super.c                                   |   48 +-
+ fs/btrfs/sysfs.c                                   |  129 +-
+ fs/btrfs/tests/extent-io-tests.c                   |  129 +-
+ fs/btrfs/transaction.c                             |   64 +-
+ fs/btrfs/transaction.h                             |   31 +-
+ fs/btrfs/tree-checker.c                            |   83 +-
+ fs/btrfs/tree-log.c                                |   11 +-
+ fs/btrfs/verity.c                                  |   31 +-
+ fs/btrfs/volumes.c                                 |   55 +-
+ fs/btrfs/volumes.h                                 |    1 +
+ fs/btrfs/xattr.c                                   |    6 +-
+ fs/btrfs/zlib.c                                    |   10 +-
+ fs/btrfs/zoned.c                                   |   21 +-
+ fs/btrfs/zstd.c                                    |   73 +-
+ fs/buffer.c                                        |  131 +-
+ fs/cachefiles/Kconfig                              |    2 +-
+ fs/cachefiles/interface.c                          |   96 +-
+ fs/cachefiles/internal.h                           |   18 +-
+ fs/cachefiles/io.c                                 |  472 +++--
+ fs/cachefiles/namei.c                              |   34 +-
+ fs/cachefiles/xattr.c                              |   82 +-
+ fs/ceph/Kconfig                                    |    1 +
+ fs/ceph/acl.c                                      |    2 +-
+ fs/ceph/addr.c                                     |    4 +-
+ fs/ceph/dir.c                                      |   10 +-
+ fs/ceph/file.c                                     |    2 +-
+ fs/ceph/inode.c                                    |   10 +-
+ fs/ceph/mds_client.h                               |    2 +-
+ fs/ceph/super.h                                    |   10 +-
+ fs/ceph/xattr.c                                    |    2 +-
+ fs/char_dev.c                                      |    4 +-
+ fs/coda/coda_linux.h                               |    6 +-
+ fs/coda/dir.c                                      |   10 +-
+ fs/coda/inode.c                                    |    4 +-
+ fs/coda/pioctl.c                                   |    4 +-
+ fs/configfs/configfs_internal.h                    |   14 +-
+ fs/configfs/dir.c                                  |   12 +-
+ fs/configfs/file.c                                 |    6 +-
+ fs/configfs/inode.c                                |    4 +-
+ fs/configfs/symlink.c                              |    2 +-
+ fs/coredump.c                                      |  558 ++++--
+ fs/crypto/Kconfig                                  |    2 +-
+ fs/crypto/crypto.c                                 |    5 +
+ fs/crypto/fscrypt_private.h                        |   27 -
+ fs/crypto/hooks.c                                  |    9 +
+ fs/crypto/keyring.c                                |   13 +
+ fs/crypto/keysetup_v1.c                            |    5 +-
+ fs/dax.c                                           |   26 +-
+ fs/dcache.c                                        |  206 +-
+ fs/debugfs/inode.c                                 |    2 +-
+ fs/devpts/inode.c                                  |    2 +
+ fs/dlm/config.c                                    |   30 +-
+ fs/dlm/dlm_internal.h                              |    5 +
+ fs/dlm/lock.c                                      |   18 +-
+ fs/dlm/lock.h                                      |    4 +-
+ fs/dlm/lowcomms.c                                  |    1 +
+ fs/dlm/midcomms.c                                  |    1 +
+ fs/dlm/plock.c                                     |   14 +-
+ fs/dlm/user.c                                      |   23 +
+ fs/ecryptfs/inode.c                                |   26 +-
+ fs/efivarfs/inode.c                                |    6 +-
+ fs/erofs/inode.c                                   |    2 +-
+ fs/erofs/internal.h                                |    2 +-
+ fs/eventfd.c                                       |    4 +-
+ fs/eventpoll.c                                     |    6 +-
+ fs/exec.c                                          |   28 +-
+ fs/exfat/dir.c                                     |   52 +-
+ fs/exfat/exfat_fs.h                                |    5 +-
+ fs/exfat/fatent.c                                  |   19 +-
+ fs/exfat/file.c                                    |    6 +-
+ fs/exfat/iomap.c                                   |   28 +-
+ fs/exfat/misc.c                                    |    2 +-
+ fs/exfat/namei.c                                   |   13 +-
+ fs/exfat/super.c                                   |   30 +-
+ fs/ext2/Makefile                                   |    2 +
+ fs/ext2/acl.c                                      |    2 +-
+ fs/ext2/acl.h                                      |    2 +-
+ fs/ext2/balloc.c                                   |    4 +
+ fs/ext2/ext2.h                                     |   25 +-
+ fs/ext2/file.c                                     |    7 +
+ fs/ext2/inode.c                                    |   11 +-
+ fs/ext2/ioctl.c                                    |    2 +-
+ fs/ext2/namei.c                                    |   12 +-
+ fs/ext2/super.c                                    |   33 +-
+ fs/ext2/xattr.c                                    |    2 +-
+ fs/ext2/xattr_security.c                           |    2 +-
+ fs/ext2/xattr_trusted.c                            |    2 +-
+ fs/ext2/xattr_user.c                               |    2 +-
+ fs/ext4/acl.c                                      |    2 +-
+ fs/ext4/acl.h                                      |    2 +-
+ fs/ext4/ext4.h                                     |   10 +-
+ fs/ext4/ext4_jbd2.c                                |    2 +-
+ fs/ext4/ialloc.c                                   |    2 +-
+ fs/ext4/inode.c                                    |    6 +-
+ fs/ext4/ioctl.c                                    |    6 +-
+ fs/ext4/mmp.c                                      |    2 +-
+ fs/ext4/namei.c                                    |   16 +-
+ fs/ext4/symlink.c                                  |    2 +-
+ fs/ext4/xattr_hurd.c                               |    2 +-
+ fs/ext4/xattr_security.c                           |    2 +-
+ fs/ext4/xattr_trusted.c                            |    2 +-
+ fs/ext4/xattr_user.c                               |    2 +-
+ fs/f2fs/Makefile                                   |    2 +-
+ fs/f2fs/acl.c                                      |   32 +-
+ fs/f2fs/acl.h                                      |   10 +-
+ fs/f2fs/cache.c                                    |  720 +++++++
+ fs/f2fs/cache.h                                    |  240 +++
+ fs/f2fs/checkpoint.c                               |  558 +++---
+ fs/f2fs/compress.c                                 |  183 +-
+ fs/f2fs/data.c                                     |  648 ++++---
+ fs/f2fs/debug.c                                    |   94 +-
+ fs/f2fs/dir.c                                      |  221 ++-
+ fs/f2fs/extent_cache.c                             |   25 +-
+ fs/f2fs/f2fs.h                                     |  516 +++--
+ fs/f2fs/file.c                                     |  195 +-
+ fs/f2fs/gc.c                                       |  222 ++-
+ fs/f2fs/inline.c                                   |  293 +--
+ fs/f2fs/inode.c                                    |  242 +--
+ fs/f2fs/iostat.h                                   |   11 +
+ fs/f2fs/namei.c                                    |  138 +-
+ fs/f2fs/node.c                                     | 1277 ++++++-------
+ fs/f2fs/node.h                                     |  191 +-
+ fs/f2fs/recovery.c                                 |  255 +--
+ fs/f2fs/segment.c                                  |  429 +++--
+ fs/f2fs/segment.h                                  |   87 +-
+ fs/f2fs/shrinker.c                                 |   16 +-
+ fs/f2fs/super.c                                    |  280 +--
+ fs/f2fs/sysfs.c                                    |   32 +-
+ fs/f2fs/verity.c                                   |    6 +-
+ fs/f2fs/xattr.c                                    |  137 +-
+ fs/f2fs/xattr.h                                    |   23 +-
+ fs/failfs.c                                        |    4 +-
+ fs/fat/fat.h                                       |    4 +-
+ fs/fat/file.c                                      |    6 +-
+ fs/fat/misc.c                                      |    2 +-
+ fs/fat/namei_msdos.c                               |    6 +-
+ fs/fat/namei_vfat.c                                |    6 +-
+ fs/fhandle.c                                       |    2 +-
+ fs/file.c                                          |  321 +++-
+ fs/file_attr.c                                     |    6 +-
+ fs/fs-writeback.c                                  |   23 +-
+ fs/fs_pin.c                                        |    9 +-
+ fs/fuse/Kconfig                                    |    8 +-
+ fs/fuse/Makefile                                   |    2 +-
+ fs/fuse/acl.c                                      |    4 +-
+ fs/fuse/backing.c                                  |    2 +-
+ fs/fuse/dax.c                                      |  215 +--
+ fs/fuse/dev.c                                      |   50 +-
+ fs/fuse/dev.h                                      |    2 +-
+ fs/fuse/dev_uring.c                                |    6 +
+ fs/fuse/dir.c                                      |   55 +-
+ fs/fuse/file.c                                     |  131 +-
+ fs/fuse/fuse_i.h                                   |  108 +-
+ fs/fuse/inode.c                                    |   82 +-
+ fs/fuse/ioctl.c                                    |    5 +-
+ fs/fuse/iomode.c                                   |    4 +-
+ fs/fuse/notify.c                                   |    4 +-
+ fs/fuse/passthrough.c                              |   15 +-
+ fs/fuse/req.c                                      |   15 +-
+ fs/fuse/virtio_fs.c                                |   28 +-
+ fs/fuse/xattr.c                                    |    2 +-
+ fs/gfs2/acl.c                                      |   13 +-
+ fs/gfs2/acl.h                                      |    2 +-
+ fs/gfs2/aops.c                                     |   17 +-
+ fs/gfs2/bmap.c                                     |   93 +-
+ fs/gfs2/dentry.c                                   |    6 +-
+ fs/gfs2/dir.c                                      |   61 +-
+ fs/gfs2/export.c                                   |    7 +-
+ fs/gfs2/file.c                                     |   74 +-
+ fs/gfs2/glock.c                                    |   60 +-
+ fs/gfs2/glops.c                                    |   13 +-
+ fs/gfs2/incore.h                                   |    8 +-
+ fs/gfs2/inode.c                                    |  148 +-
+ fs/gfs2/inode.h                                    |    4 +-
+ fs/gfs2/lock_dlm.c                                 |    4 +-
+ fs/gfs2/log.c                                      |    7 +-
+ fs/gfs2/lops.c                                     |   22 +-
+ fs/gfs2/meta_io.c                                  |    7 +-
+ fs/gfs2/ops_fstype.c                               |   26 +-
+ fs/gfs2/quota.c                                    |   57 +-
+ fs/gfs2/recovery.c                                 |   19 +-
+ fs/gfs2/rgrp.c                                     |  129 +-
+ fs/gfs2/super.c                                    |   98 +-
+ fs/gfs2/trace_gfs2.h                               |    6 +-
+ fs/gfs2/util.c                                     |    8 +-
+ fs/gfs2/xattr.c                                    |   71 +-
+ fs/hfs/attr.c                                      |    2 +-
+ fs/hfs/dir.c                                       |    6 +-
+ fs/hfs/hfs_fs.h                                    |    2 +-
+ fs/hfs/inode.c                                     |    2 +-
+ fs/hfsplus/dir.c                                   |   10 +-
+ fs/hfsplus/hfsplus_fs.h                            |    4 +-
+ fs/hfsplus/inode.c                                 |    6 +-
+ fs/hfsplus/xattr.c                                 |    2 +-
+ fs/hfsplus/xattr_security.c                        |    2 +-
+ fs/hfsplus/xattr_trusted.c                         |    2 +-
+ fs/hfsplus/xattr_user.c                            |    2 +-
+ fs/hostfs/hostfs_kern.c                            |   14 +-
+ fs/hpfs/hpfs_fn.h                                  |    2 +-
+ fs/hpfs/inode.c                                    |    2 +-
+ fs/hpfs/namei.c                                    |   10 +-
+ fs/hugetlbfs/inode.c                               |   14 +-
+ fs/inode.c                                         |   14 +-
+ fs/internal.h                                      |   30 +-
+ fs/iomap/bio.c                                     |    4 +-
+ fs/iomap/buffered-io.c                             |    8 +-
+ fs/iomap/direct-io.c                               |   46 +-
+ fs/iomap/ioend.c                                   |  122 +-
+ fs/isofs/compress.c                                |   77 +-
+ fs/isofs/inode.c                                   |   46 +-
+ fs/isofs/isofs.h                                   |    2 +
+ fs/isofs/rock.c                                    |   11 +-
+ fs/jbd2/commit.c                                   |   22 +-
+ fs/jbd2/journal.c                                  |   31 +-
+ fs/jbd2/transaction.c                              |    2 +-
+ fs/jffs2/acl.c                                     |    2 +-
+ fs/jffs2/acl.h                                     |    2 +-
+ fs/jffs2/dir.c                                     |   20 +-
+ fs/jffs2/fs.c                                      |    2 +-
+ fs/jffs2/os-linux.h                                |    2 +-
+ fs/jffs2/security.c                                |    2 +-
+ fs/jffs2/xattr_trusted.c                           |    2 +-
+ fs/jffs2/xattr_user.c                              |    2 +-
+ fs/jfs/acl.c                                       |    2 +-
+ fs/jfs/file.c                                      |    2 +-
+ fs/jfs/ioctl.c                                     |    2 +-
+ fs/jfs/jfs_acl.h                                   |    2 +-
+ fs/jfs/jfs_inode.h                                 |    4 +-
+ fs/jfs/namei.c                                     |   10 +-
+ fs/jfs/xattr.c                                     |    4 +-
+ fs/kernfs/dir.c                                    |  231 ++-
+ fs/kernfs/file.c                                   |   47 +-
+ fs/kernfs/inode.c                                  |   10 +-
+ fs/kernfs/kernfs-internal.h                        |   24 +-
+ fs/kernfs/mount.c                                  |   34 +-
+ fs/kernfs/symlink.c                                |   17 +-
+ fs/libfs.c                                         |    8 +-
+ fs/lockd/svc.c                                     |    1 -
+ fs/lockd/svclock.c                                 |   33 +-
+ fs/lockd/trace.h                                   |    1 -
+ fs/lockd/xdr.h                                     |    2 +-
+ fs/minix/file.c                                    |    2 +-
+ fs/minix/inode.c                                   |    2 +-
+ fs/minix/minix.h                                   |    2 +-
+ fs/minix/namei.c                                   |   12 +-
+ fs/mnt_idmapping.c                                 |   33 +-
+ fs/mount.h                                         |    3 +-
+ fs/namei.c                                         |  155 +-
+ fs/namespace.c                                     |   75 +-
+ fs/netfs/Kconfig                                   |    3 +
+ fs/netfs/Makefile                                  |    2 +-
+ fs/netfs/buffered_read.c                           |  217 ++-
+ fs/netfs/buffered_write.c                          |   69 +-
+ fs/netfs/direct_read.c                             |   14 +-
+ fs/netfs/direct_write.c                            |    8 +-
+ fs/netfs/fscache_cookie.c                          |    8 +-
+ fs/netfs/fscache_internal.h                        |   14 -
+ fs/netfs/fscache_io.c                              |   10 +-
+ fs/netfs/internal.h                                |   70 +-
+ fs/netfs/iterator.c                                |    2 +-
+ fs/netfs/main.c                                    |    1 -
+ fs/netfs/misc.c                                    |   10 +-
+ fs/netfs/objects.c                                 |    7 +-
+ fs/netfs/read_collect.c                            |   14 +-
+ fs/netfs/read_pgpriv2.c                            |   13 +-
+ fs/netfs/read_retry.c                              |    7 +-
+ fs/netfs/read_single.c                             |   50 +-
+ fs/netfs/stats.c                                   |    4 +-
+ fs/netfs/write_collect.c                           |  157 +-
+ fs/netfs/write_issue.c                             |  141 +-
+ fs/netfs/write_retry.c                             |    8 +-
+ fs/nfs/Kconfig                                     |   18 +-
+ fs/nfs/blocklayout/Makefile                        |    3 +-
+ fs/nfs/blocklayout/blocklayout.c                   |  119 +-
+ fs/nfs/blocklayout/dev.c                           |   32 +-
+ fs/nfs/callback_proc.c                             |   23 +-
+ fs/nfs/callback_xdr.c                              |    9 +-
+ fs/nfs/client.c                                    |    6 +-
+ fs/nfs/dir.c                                       |   12 +-
+ fs/nfs/direct.c                                    |  139 +-
+ fs/nfs/filelayout/filelayout.c                     |   17 +-
+ fs/nfs/filelayout/filelayoutdev.c                  |   19 +-
+ fs/nfs/flexfilelayout/flexfilelayout.c             |  413 ++--
+ fs/nfs/flexfilelayout/flexfilelayout.h             |   48 +-
+ fs/nfs/flexfilelayout/flexfilelayoutdev.c          |  212 ++-
+ fs/nfs/inode.c                                     |    4 +-
+ fs/nfs/internal.h                                  |   13 +-
+ fs/nfs/namespace.c                                 |    4 +-
+ fs/nfs/netns.h                                     |    5 +-
+ fs/nfs/nfs3_fs.h                                   |    2 +-
+ fs/nfs/nfs3acl.c                                   |    2 +-
+ fs/nfs/nfs3client.c                                |    9 +-
+ fs/nfs/nfs4_fs.h                                   |    3 +
+ fs/nfs/nfs4client.c                                |    8 +-
+ fs/nfs/nfs4file.c                                  |    1 +
+ fs/nfs/nfs4proc.c                                  |  146 +-
+ fs/nfs/nfs4state.c                                 |    7 +-
+ fs/nfs/nfs4xdr.c                                   |    2 +-
+ fs/nfs/pagelist.c                                  |   66 +-
+ fs/nfs/pnfs.c                                      |  376 +++-
+ fs/nfs/pnfs.h                                      |  102 +-
+ fs/nfs/pnfs_dev.c                                  |   30 +-
+ fs/nfs/pnfs_nfs.c                                  |  108 +-
+ fs/nfs/read.c                                      |    2 +-
+ fs/nfs/super.c                                     |   25 -
+ fs/nfs/unlink.c                                    |    3 +
+ fs/nfs/write.c                                     |    2 +-
+ fs/nfs_common/nfs_ssc.c                            |  126 +-
+ fs/nfsd/blocklayout.c                              |    1 +
+ fs/nfsd/blocklayoutxdr.c                           |   11 +
+ fs/nfsd/export.c                                   |    8 +-
+ fs/nfsd/export.h                                   |    3 +-
+ fs/nfsd/filecache.c                                |    1 +
+ fs/nfsd/flexfilelayout.c                           |   24 +-
+ fs/nfsd/flexfilelayoutxdr.c                        |    9 +-
+ fs/nfsd/flexfilelayoutxdr.h                        |    8 +-
+ fs/nfsd/localio.c                                  |   10 +-
+ fs/nfsd/lockd.c                                    |    5 +-
+ fs/nfsd/netns.h                                    |   17 +-
+ fs/nfsd/nfs2acl.c                                  |   48 +-
+ fs/nfsd/nfs3acl.c                                  |    1 +
+ fs/nfsd/nfs3proc.c                                 |   79 +-
+ fs/nfsd/nfs3xdr.c                                  |    4 +
+ fs/nfsd/nfs4acl.c                                  |    1 +
+ fs/nfsd/nfs4callback.c                             |    8 +-
+ fs/nfsd/nfs4ctl.h                                  |   83 +
+ fs/nfsd/nfs4idmap.c                                |    1 +
+ fs/nfsd/nfs4layouts.c                              |    1 +
+ fs/nfsd/nfs4proc.c                                 |  465 +++--
+ fs/nfsd/nfs4recover.c                              |    1 +
+ fs/nfsd/nfs4state.c                                |  587 +++++-
+ fs/nfsd/nfs4xdr.c                                  |  188 +-
+ fs/nfsd/nfscache.c                                 |    1 +
+ fs/nfsd/nfsctl.c                                   |   36 +-
+ fs/nfsd/nfsd.h                                     |  237 +--
+ fs/nfsd/nfserr.h                                   |  159 ++
+ fs/nfsd/nfsfh.c                                    |   73 +-
+ fs/nfsd/nfsfh.h                                    |   28 +-
+ fs/nfsd/nfsproc.c                                  |   39 +-
+ fs/nfsd/nfssvc.c                                   |    8 +
+ fs/nfsd/nfsxdr.c                                   |    1 +
+ fs/nfsd/state.h                                    |   46 +-
+ fs/nfsd/trace.h                                    |   45 +-
+ fs/nfsd/vfs.c                                      |  311 ++-
+ fs/nfsd/vfs.h                                      |   39 +-
+ fs/nfsd/xdr.h                                      |    6 +-
+ fs/nfsd/xdr3.h                                     |    2 +-
+ fs/nfsd/xdr4.h                                     |  156 +-
+ fs/nfsd/xdr4cb.h                                   |   20 +-
+ fs/nilfs2/inode.c                                  |    4 +-
+ fs/nilfs2/ioctl.c                                  |    2 +-
+ fs/nilfs2/namei.c                                  |   10 +-
+ fs/nilfs2/nilfs.h                                  |    6 +-
+ fs/nls/nls_iso8859-14.c                            |   26 +-
+ fs/notify/fanotify/fanotify_user.c                 |    2 +-
+ fs/nsfs.c                                          |    4 +-
+ fs/ntfs/attrib.c                                   |   76 +-
+ fs/ntfs/attrlist.c                                 |   14 +-
+ fs/ntfs/bitmap.c                                   |   19 +-
+ fs/ntfs/collate.c                                  |    8 +-
+ fs/ntfs/dir.c                                      |   23 +-
+ fs/ntfs/ea.c                                       |   10 +-
+ fs/ntfs/ea.h                                       |    6 +-
+ fs/ntfs/file.c                                     |   60 +-
+ fs/ntfs/index.c                                    |    2 +-
+ fs/ntfs/index.h                                    |    2 +-
+ fs/ntfs/inode.c                                    |  131 +-
+ fs/ntfs/inode.h                                    |    4 +-
+ fs/ntfs/iomap.c                                    |   16 +-
+ fs/ntfs/layout.h                                   |   19 +-
+ fs/ntfs/mft.c                                      |  921 ++++++---
+ fs/ntfs/mft.h                                      |   19 +-
+ fs/ntfs/mst.c                                      |    4 +-
+ fs/ntfs/namei.c                                    |   43 +-
+ fs/ntfs/ntfs.h                                     |    7 +-
+ fs/ntfs/reparse.c                                  |    2 +-
+ fs/ntfs/runlist.c                                  |    5 +-
+ fs/ntfs/super.c                                    |  391 +++-
+ fs/ntfs/unistr.c                                   |   29 +-
+ fs/ntfs/volume.h                                   |   15 +
+ fs/ntfs3/attrib.c                                  |   47 +-
+ fs/ntfs3/dir.c                                     |    3 +
+ fs/ntfs3/file.c                                    |    9 +-
+ fs/ntfs3/frecord.c                                 |    5 +
+ fs/ntfs3/fslog.c                                   |    4 +-
+ fs/ntfs3/fsntfs.c                                  |    3 +
+ fs/ntfs3/index.c                                   |   13 +-
+ fs/ntfs3/inode.c                                   |   21 +-
+ fs/ntfs3/namei.c                                   |   10 +-
+ fs/ntfs3/ntfs_fs.h                                 |   16 +-
+ fs/ntfs3/record.c                                  |   16 +-
+ fs/ntfs3/super.c                                   |    2 +-
+ fs/ntfs3/xattr.c                                   |   26 +-
+ fs/ocfs2/acl.c                                     |    2 +-
+ fs/ocfs2/acl.h                                     |    2 +-
+ fs/ocfs2/buffer_head_io.c                          |   12 +-
+ fs/ocfs2/dlmfs/dlmfs.c                             |    6 +-
+ fs/ocfs2/file.c                                    |    6 +-
+ fs/ocfs2/file.h                                    |    6 +-
+ fs/ocfs2/ioctl.c                                   |    2 +-
+ fs/ocfs2/ioctl.h                                   |    2 +-
+ fs/ocfs2/journal.c                                 |   25 +-
+ fs/ocfs2/namei.c                                   |   10 +-
+ fs/ocfs2/quota_local.c                             |    4 +
+ fs/ocfs2/xattr.c                                   |    6 +-
+ fs/omfs/dir.c                                      |    6 +-
+ fs/omfs/file.c                                     |    2 +-
+ fs/omfs/inode.c                                    |    4 +-
+ fs/open.c                                          |   50 +-
+ fs/orangefs/acl.c                                  |    2 +-
+ fs/orangefs/inode.c                                |   29 +-
+ fs/orangefs/namei.c                                |    8 +-
+ fs/orangefs/orangefs-debugfs.c                     |   10 +-
+ fs/orangefs/orangefs-kernel.h                      |    8 +-
+ fs/orangefs/super.c                                |   18 +
+ fs/orangefs/xattr.c                                |    2 +-
+ fs/overlayfs/dir.c                                 |   14 +-
+ fs/overlayfs/file.c                                |    2 +-
+ fs/overlayfs/inode.c                               |   16 +-
+ fs/overlayfs/overlayfs.h                           |   14 +-
+ fs/overlayfs/ovl_entry.h                           |    2 +-
+ fs/overlayfs/util.c                                |    4 +-
+ fs/overlayfs/xattrs.c                              |    4 +-
+ fs/pidfs.c                                         |    6 +-
+ fs/pipe.c                                          |    2 +-
+ fs/pnode.c                                         |  108 +-
+ fs/posix_acl.c                                     |   26 +-
+ fs/proc/base.c                                     |   53 +-
+ fs/proc/fd.c                                       |    6 +-
+ fs/proc/fd.h                                       |    2 +-
+ fs/proc/generic.c                                  |    4 +-
+ fs/proc/internal.h                                 |    4 +-
+ fs/proc/proc_net.c                                 |    2 +-
+ fs/proc/proc_sysctl.c                              |    6 +-
+ fs/proc/root.c                                     |    2 +-
+ fs/proc/vmcore.c                                   |   20 +
+ fs/quota/dquot.c                                   |   18 +-
+ fs/ramfs/file-nommu.c                              |    4 +-
+ fs/ramfs/inode.c                                   |   10 +-
+ fs/read_write.c                                    |    8 +-
+ fs/remap_range.c                                   |    2 +-
+ fs/smb/client/cifsacl.c                            |    4 +-
+ fs/smb/client/cifsfs.c                             |    2 +-
+ fs/smb/client/cifsfs.h                             |   16 +-
+ fs/smb/client/cifsproto.h                          |    4 +-
+ fs/smb/client/dir.c                                |    6 +-
+ fs/smb/client/inode.c                              |    8 +-
+ fs/smb/client/link.c                               |    2 +-
+ fs/smb/client/transport.c                          |   13 +-
+ fs/smb/client/xattr.c                              |    2 +-
+ fs/smb/common/compress/compress.c                  |   16 +-
+ fs/smb/common/fscc.h                               |    5 +-
+ fs/smb/common/smbglob.h                            |    1 +
+ fs/smb/server/Kconfig                              |    2 +
+ fs/smb/server/Makefile                             |    1 +
+ fs/smb/server/compress.c                           |    1 +
+ fs/smb/server/connection.c                         |    3 +-
+ fs/smb/server/connection.h                         |    3 +
+ fs/smb/server/ksmbd_work.c                         |    3 -
+ fs/smb/server/ksmbd_work.h                         |    5 +-
+ fs/smb/server/mgmt/user_session.c                  |    4 +-
+ fs/smb/server/ndr.c                                |    2 +-
+ fs/smb/server/ndr.h                                |    2 +-
+ fs/smb/server/oplock.c                             |    2 +-
+ fs/smb/server/smb2ops.c                            |   33 +
+ fs/smb/server/smb2pdu.c                            |  710 +++----
+ fs/smb/server/smb2pdu.h                            |   22 +-
+ fs/smb/server/smb_common.c                         |    6 +-
+ fs/smb/server/smbacl.c                             |   27 +-
+ fs/smb/server/smbacl.h                             |    8 +-
+ fs/smb/server/tests/Kconfig                        |   15 +
+ fs/smb/server/tests/Makefile                       |    4 +
+ fs/smb/server/tests/smbacl_kunit.c                 |  301 +++
+ fs/smb/server/transport_tcp.c                      |    8 +-
+ fs/smb/server/vfs.c                                |   52 +-
+ fs/smb/server/vfs.h                                |   33 +-
+ fs/smb/server/vfs_cache.c                          |   66 +-
+ fs/smb/server/vfs_cache.h                          |    6 -
+ fs/splice.c                                        |   70 +-
+ fs/stat.c                                          |    4 +-
+ fs/super.c                                         |   19 +-
+ fs/tests/.kunitconfig                              |    2 +
+ fs/tests/fdtable_kunit.c                           |   72 +
+ fs/tracefs/event_inode.c                           |    2 +-
+ fs/tracefs/inode.c                                 |    8 +-
+ fs/ubifs/dir.c                                     |   14 +-
+ fs/ubifs/file.c                                    |    4 +-
+ fs/ubifs/ioctl.c                                   |    2 +-
+ fs/ubifs/ubifs.h                                   |    6 +-
+ fs/ubifs/xattr.c                                   |    2 +-
+ fs/udf/file.c                                      |    2 +-
+ fs/udf/inode.c                                     |   59 +-
+ fs/udf/misc.c                                      |   15 +-
+ fs/udf/namei.c                                     |   12 +-
+ fs/udf/super.c                                     |    5 +-
+ fs/udf/symlink.c                                   |    2 +-
+ fs/ufs/dir.c                                       |    2 +-
+ fs/ufs/inode.c                                     |    2 +-
+ fs/ufs/namei.c                                     |   10 +-
+ fs/ufs/ufs.h                                       |    2 +-
+ fs/vboxsf/dir.c                                    |    8 +-
+ fs/vboxsf/utils.c                                  |    4 +-
+ fs/vboxsf/vfsmod.h                                 |    4 +-
+ fs/xattr.c                                         |   30 +-
+ fs/xfs/libxfs/xfs_attr.c                           |   14 +-
+ fs/xfs/libxfs/xfs_bmap.c                           |    5 +-
+ fs/xfs/libxfs/xfs_bmap.h                           |    2 +-
+ fs/xfs/libxfs/xfs_btree_mem.c                      |    6 +-
+ fs/xfs/libxfs/xfs_dquot_buf.c                      |   13 +-
+ fs/xfs/libxfs/xfs_errortag.h                       |    6 +-
+ fs/xfs/libxfs/xfs_exchmaps.c                       |    1 -
+ fs/xfs/libxfs/xfs_exchmaps.h                       |    3 +-
+ fs/xfs/libxfs/xfs_ialloc.c                         |    2 +-
+ fs/xfs/libxfs/xfs_ialloc_btree.c                   |    1 -
+ fs/xfs/libxfs/xfs_ialloc_btree.h                   |    5 +-
+ fs/xfs/libxfs/xfs_inode_fork.c                     |    4 +-
+ fs/xfs/libxfs/xfs_inode_util.h                     |    2 +-
+ fs/xfs/libxfs/xfs_metadir.c                        |   32 +-
+ fs/xfs/libxfs/xfs_metadir.h                        |    5 +-
+ fs/xfs/libxfs/xfs_parent.h                         |    1 -
+ fs/xfs/libxfs/xfs_refcount_btree.c                 |   16 +-
+ fs/xfs/libxfs/xfs_refcount_btree.h                 |   26 +
+ fs/xfs/libxfs/xfs_rmap.c                           |   13 +-
+ fs/xfs/libxfs/xfs_rmap_btree.c                     |   16 +-
+ fs/xfs/libxfs/xfs_rmap_btree.h                     |   26 +
+ fs/xfs/libxfs/xfs_rtrefcount_btree.c               |  121 +-
+ fs/xfs/libxfs/xfs_rtrefcount_btree.h               |    5 +-
+ fs/xfs/libxfs/xfs_rtrmap_btree.c                   |  236 +--
+ fs/xfs/libxfs/xfs_rtrmap_btree.h                   |    3 +-
+ fs/xfs/libxfs/xfs_sb.c                             |   40 -
+ fs/xfs/libxfs/xfs_sb.h                             |    1 -
+ fs/xfs/libxfs/xfs_symlink_remote.c                 |    8 +-
+ fs/xfs/libxfs/xfs_symlink_remote.h                 |    2 +-
+ fs/xfs/libxfs/xfs_trans_resv.c                     |   45 +-
+ fs/xfs/libxfs/xfs_trans_space.c                    |    6 +-
+ fs/xfs/libxfs/xfs_types.c                          |    7 +-
+ fs/xfs/libxfs/xfs_types.h                          |    7 +-
+ fs/xfs/scrub/alloc_repair.c                        |   15 +-
+ fs/xfs/scrub/bmap.c                                |   11 +-
+ fs/xfs/scrub/bmap_repair.c                         |    4 +-
+ fs/xfs/scrub/inode_repair.c                        |   32 +-
+ fs/xfs/scrub/orphanage.c                           |    2 +-
+ fs/xfs/scrub/quota.c                               |    2 +-
+ fs/xfs/scrub/quota_repair.c                        |    9 +-
+ fs/xfs/scrub/quotacheck_repair.c                   |   54 +-
+ fs/xfs/scrub/rtrefcount_repair.c                   |    3 +-
+ fs/xfs/scrub/rtrmap_repair.c                       |    2 +-
+ fs/xfs/scrub/xfarray.c                             |  201 +-
+ fs/xfs/scrub/xfarray.h                             |    9 +-
+ fs/xfs/xfs_acl.c                                   |    2 +-
+ fs/xfs/xfs_acl.h                                   |    2 +-
+ fs/xfs/xfs_aops.c                                  |   13 +-
+ fs/xfs/xfs_bmap_item.c                             |    2 +-
+ fs/xfs/xfs_buf.c                                   |   11 +
+ fs/xfs/xfs_dquot.c                                 |   20 +-
+ fs/xfs/xfs_dquot.h                                 |    2 +-
+ fs/xfs/xfs_exchmaps_item.c                         |    4 +-
+ fs/xfs/xfs_exchrange.c                             |    2 +-
+ fs/xfs/xfs_file.c                                  |   33 +-
+ fs/xfs/xfs_handle.c                                |    8 +-
+ fs/xfs/xfs_inode.c                                 |   24 +-
+ fs/xfs/xfs_inode.h                                 |    2 +-
+ fs/xfs/xfs_ioctl.c                                 |  423 +++--
+ fs/xfs/xfs_ioctl.h                                 |    6 +-
+ fs/xfs/xfs_ioctl32.c                               |  188 +-
+ fs/xfs/xfs_ioend.c                                 |  137 +-
+ fs/xfs/xfs_ioend.h                                 |    2 +
+ fs/xfs/xfs_iops.c                                  |   24 +-
+ fs/xfs/xfs_iops.h                                  |    2 +-
+ fs/xfs/xfs_itable.c                                |    2 +-
+ fs/xfs/xfs_itable.h                                |    2 +-
+ fs/xfs/xfs_mount.h                                 |    8 +
+ fs/xfs/xfs_qm.c                                    |    6 +-
+ fs/xfs/xfs_rmap_item.c                             |    2 +-
+ fs/xfs/xfs_rtalloc.h                               |   49 +-
+ fs/xfs/xfs_super.c                                 |    3 +-
+ fs/xfs/xfs_symlink.c                               |    6 +-
+ fs/xfs/xfs_symlink.h                               |    2 +-
+ fs/xfs/xfs_sysfs.c                                 |   78 +-
+ fs/xfs/xfs_trace.h                                 |   29 +-
+ fs/xfs/xfs_trans_dquot.c                           |    6 +-
+ fs/xfs/xfs_xattr.c                                 |    2 +-
+ fs/xfs/xfs_zone_alloc.c                            |    4 +
+ fs/zonefs/super.c                                  |    2 +-
+ include/linux/binfmts.h                            |    3 +-
+ include/linux/bio-integrity.h                      |    3 +-
+ include/linux/bio.h                                |   11 +-
+ include/linux/blkdev.h                             |    8 +-
+ include/linux/buffer_head.h                        |   86 +-
+ include/linux/capability.h                         |    8 +-
+ include/linux/cleanup.h                            |    7 -
+ include/linux/configfs.h                           |   73 +-
+ include/linux/coredump.h                           |   37 +-
+ include/linux/dax.h                                |   12 -
+ include/linux/dcache.h                             |   38 +-
+ include/linux/f2fs_fs.h                            |  140 +-
+ include/linux/fdtable.h                            |   15 +-
+ include/linux/file.h                               |  130 +-
+ include/linux/fileattr.h                           |    2 +-
+ include/linux/fs.h                                 |  127 +-
+ include/linux/fs_context.h                         |    4 +
+ include/linux/fscache-cache.h                      |    2 +-
+ include/linux/fscache.h                            |   53 +-
+ include/linux/iomap.h                              |   38 +-
+ include/linux/lsm_hook_defs.h                      |   25 +-
+ include/linux/mnt_idmapping.h                      |   24 +-
+ include/linux/mount.h                              |    4 +-
+ include/linux/namei.h                              |   19 +-
+ include/linux/netfs.h                              |  112 +-
+ include/linux/nfs.h                                |   55 +-
+ include/linux/nfs3.h                               |   43 +
+ include/linux/nfs4.h                               |    6 +
+ include/linux/nfs_fh.h                             |   63 +
+ include/linux/nfs_fs.h                             |    6 +-
+ include/linux/nfs_fs_sb.h                          |    8 +
+ include/linux/nfs_page.h                           |    8 +-
+ include/linux/nfs_ssc.h                            |   69 +-
+ include/linux/nfs_xdr.h                            |    2 +
+ include/linux/nfsd_ssc.h                           |   38 +
+ include/linux/nfslocalio.h                         |   11 +-
+ include/linux/posix_acl.h                          |   24 +-
+ include/linux/quotaops.h                           |    6 +-
+ include/linux/sched.h                              |    2 +-
+ include/linux/sched/signal.h                       |   33 +-
+ include/linux/security.h                           |   61 +-
+ include/linux/splice.h                             |    4 +-
+ include/linux/sunrpc/svc_xprt.h                    |    5 +-
+ include/linux/uidgid.h                             |   12 +-
+ include/linux/user_namespace.h                     |   11 +-
+ include/linux/wait_bit.h                           |   26 +
+ include/linux/xattr.h                              |   20 +-
+ include/trace/events/cachefiles.h                  |   81 +-
+ include/trace/events/f2fs.h                        |   71 +
+ include/trace/events/fscache.h                     |   10 +-
+ include/trace/events/netfs.h                       |  145 +-
+ include/trace/misc/nfs.h                           |   13 +-
+ include/uapi/linux/btrfs_tree.h                    |   25 +-
+ include/uapi/linux/close_range.h                   |   31 +-
+ include/uapi/linux/coredump.h                      |  149 +-
+ include/uapi/linux/fs.h                            |    2 +-
+ include/uapi/linux/fuse.h                          |   12 +-
+ init/Kconfig                                       |   11 +
+ init/initramfs.c                                   |   11 +-
+ io_uring/io-wq.c                                   |    2 +
+ io_uring/mock_file.c                               |    8 +-
+ ipc/mqueue.c                                       |    2 +-
+ kernel/Makefile                                    |    1 +
+ kernel/bpf/bpf_iter.c                              |    6 +-
+ kernel/bpf/inode.c                                 |    6 +-
+ kernel/bpf/token.c                                 |    6 +-
+ kernel/capability.c                                |    4 +-
+ kernel/exit.c                                      |   15 +-
+ kernel/fork.c                                      |   68 +-
+ kernel/kthread.c                                   |    2 +-
+ kernel/pid_namespace.c                             |    3 +-
+ kernel/ptrace.c                                    |    6 +
+ kernel/signal.c                                    |   14 +
+ kernel/tests/.kunitconfig                          |    4 +
+ kernel/tests/user_ns_map_kunit.c                   |   98 +
+ kernel/user_namespace.c                            |   60 +-
+ kernel/utsname.c                                   |    1 -
+ mm/secretmem.c                                     |    2 +-
+ mm/shmem.c                                         |   30 +-
+ mm/shmem_quota.c                                   |    5 +
+ mm/userfaultfd.c                                   |    6 +-
+ net/core/scm.c                                     |    8 +-
+ net/handshake/netlink.c                            |    8 +-
+ net/kcm/kcmsock.c                                  |    6 +-
+ net/socket.c                                       |    6 +-
+ net/sunrpc/svc_xprt.c                              |   36 +-
+ net/sunrpc/svcauth_unix.c                          |    9 +
+ net/sunrpc/svcsock.c                               |  490 +++--
+ net/sunrpc/xprtrdma/svc_rdma_recvfrom.c            |   18 +-
+ net/sunrpc/xprtrdma/svc_rdma_transport.c           |    3 +-
+ net/sunrpc/xprtsock.c                              |   69 +-
+ net/unix/af_unix.c                                 |    2 +-
+ rust/kernel/configfs.rs                            |  127 +-
+ samples/configfs/configfs_sample.c                 |  137 +-
+ security/apparmor/apparmorfs.c                     |    2 +-
+ security/apparmor/lsm.c                            |    4 +-
+ security/commoncap.c                               |   10 +-
+ security/integrity/evm/evm_main.c                  |   26 +-
+ security/integrity/ima/ima.h                       |   10 +-
+ security/integrity/ima/ima_api.c                   |    2 +-
+ security/integrity/ima/ima_appraise.c              |   12 +-
+ security/integrity/ima/ima_main.c                  |    6 +-
+ security/integrity/ima/ima_policy.c                |    4 +-
+ security/security.c                                |   49 +-
+ security/selinux/hooks.c                           |   45 +-
+ security/selinux/selinuxfs.c                       |    2 +-
+ security/smack/smack_lsm.c                         |   14 +-
+ tools/include/uapi/linux/coredump.h                |  149 +-
+ tools/testing/selftests/Makefile                   |    4 +
+ .../selftests/clone3/clone3_clear_sighand.c        |    6 +-
+ tools/testing/selftests/core/close_range_test.c    |  951 ++++++++++
+ tools/testing/selftests/coredump/.gitignore        |    2 +
+ tools/testing/selftests/coredump/Makefile          |   11 +-
+ .../selftests/coredump/coredump_notify_signal.h    |   29 +
+ .../coredump/coredump_notify_signal_helper.c       |   46 +
+ .../coredump/coredump_notify_signal_test.c         |  245 +++
+ .../selftests/coredump/coredump_signal_test.c      |  238 +++
+ .../coredump/coredump_socket_protocol_test.c       | 1983 +++++++++++++++-----
+ tools/testing/selftests/coredump/coredump_test.h   |   32 +-
+ .../selftests/coredump/coredump_test_helpers.c     | 1742 ++++++++++++++++-
+ .../selftests/coredump/coredump_test_helpers.h     |   79 +
+ .../selftests/coredump/coredump_worker_test.c      |  447 +++++
+ tools/testing/selftests/exec/Makefile              |    4 +
+ tools/testing/selftests/exec/binfmt_misc_delim.c   |  127 ++
+ tools/testing/selftests/filesystems/.gitignore     |    1 -
+ tools/testing/selftests/filesystems/Makefile       |    2 +-
+ tools/testing/selftests/filesystems/config         |    8 +
+ .../selftests/filesystems/configfs/.gitignore      |    2 +
+ .../selftests/filesystems/configfs/Makefile        |    8 +
+ .../testing/selftests/filesystems/configfs/config  |    5 +
+ .../selftests/filesystems/configfs/configfs_test.c |  481 +++++
+ .../selftests/filesystems/file_stressor/.gitignore |    2 +
+ .../selftests/filesystems/file_stressor/Makefile   |    6 +
+ .../{ => file_stressor}/file_stressor.c            |    0
+ .../selftests/filesystems/file_stressor/settings   |    3 +
+ .../selftests/filesystems/fscontext_ns/.gitignore  |    2 +
+ .../testing/selftests/filesystems/fuse/.gitignore  |    1 +
+ tools/testing/selftests/filesystems/fuse/Makefile  |    2 +-
+ tools/testing/selftests/filesystems/kernfs_test.c  | 1203 +++++++++++-
+ .../filesystems/mntns_unbindable/Makefile          |    6 +
+ .../mntns_unbindable/mntns_unbindable_test.c       |  227 +++
+ .../selftests/filesystems/openat2/openat2_test.c   |   10 +-
+ .../selftests/filesystems/openat2/resolve_test.c   |    7 +-
+ .../filesystems/statmount/statmount_test.c         |    2 +-
+ .../filesystems/umount_propagation/Makefile        |    6 +
+ .../umount_propagation/umount_propagation_test.c   |  226 +++
+ .../move_mount_set_group_test.c                    |   74 +-
+ tools/testing/selftests/pidfd/pidfd_open_test.c    |    2 +-
+ virt/kvm/guest_memfd.c                             |    2 +-
+ 861 files changed, 27304 insertions(+), 15917 deletions(-)
+ delete mode 100644 Documentation/filesystems/bfs.rst
+ create mode 100644 Documentation/filesystems/exfat.rst
+ delete mode 100644 arch/powerpc/platforms/cell/spufs/coredump.c
+ delete mode 100644 fs/bfs/Kconfig
+ delete mode 100644 fs/bfs/Makefile
+ delete mode 100644 fs/bfs/bfs.h
+ delete mode 100644 fs/bfs/file.c
+ delete mode 100644 fs/bfs/inode.c
+ create mode 100644 fs/f2fs/cache.c
+ create mode 100644 fs/f2fs/cache.h
+ delete mode 100644 fs/netfs/fscache_internal.h
+ create mode 100644 fs/nfsd/nfs4ctl.h
+ create mode 100644 fs/nfsd/nfserr.h
+ create mode 100644 fs/smb/server/tests/Kconfig
+ create mode 100644 fs/smb/server/tests/Makefile
+ create mode 100644 fs/smb/server/tests/smbacl_kunit.c
+ create mode 100644 fs/tests/.kunitconfig
+ create mode 100644 fs/tests/fdtable_kunit.c
+ create mode 100644 include/linux/nfs_fh.h
+ create mode 100644 include/linux/nfsd_ssc.h
+ create mode 100644 kernel/tests/.kunitconfig
+ create mode 100644 kernel/tests/user_ns_map_kunit.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_notify_signal.h
+ create mode 100644 tools/testing/selftests/coredump/coredump_notify_signal_helper.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_notify_signal_test.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_signal_test.c
+ create mode 100644 tools/testing/selftests/coredump/coredump_test_helpers.h
+ create mode 100644 tools/testing/selftests/coredump/coredump_worker_test.c
+ create mode 100644 tools/testing/selftests/exec/binfmt_misc_delim.c
+ create mode 100644 tools/testing/selftests/filesystems/config
+ create mode 100644 tools/testing/selftests/filesystems/configfs/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/configfs/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/configfs/config
+ create mode 100644 tools/testing/selftests/filesystems/configfs/configfs_test.c
+ create mode 100644 tools/testing/selftests/filesystems/file_stressor/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/file_stressor/Makefile
+ rename tools/testing/selftests/filesystems/{ => file_stressor}/file_stressor.c (100%)
+ create mode 100644 tools/testing/selftests/filesystems/file_stressor/settings
+ create mode 100644 tools/testing/selftests/filesystems/fscontext_ns/.gitignore
+ create mode 100644 tools/testing/selftests/filesystems/mntns_unbindable/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/mntns_unbindable/mntns_unbindable_test.c
+ create mode 100644 tools/testing/selftests/filesystems/umount_propagation/Makefile
+ create mode 100644 tools/testing/selftests/filesystems/umount_propagation/umount_propagation_test.c
+Merging printk/for-next (3fa6f22c1dc83 Merge branch 'for-7.4' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git printk/for-next
+Auto-merging lib/vsprintf.c
+Merge made by the 'ort' strategy.
+ kernel/printk/printk.c            | 94 +++++++++++++++++++++------------------
+ kernel/printk/printk_ringbuffer.c | 17 +++----
+ lib/vsprintf.c                    | 11 ++---
+ 3 files changed, 62 insertions(+), 60 deletions(-)
+Merging pci/next (81bebadfa0f7f Merge branch 'pci/misc')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci/next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
+Auto-merging drivers/pci/controller/dwc/pci-imx6.c
+Auto-merging drivers/pci/endpoint/pci-ep-msi.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-bus-pci            |  14 +-
+ Documentation/PCI/controller/index.rst             |   1 -
+ .../PCI/controller/rcar-pcie-firmware.rst          |  32 -
+ .../bindings/pci/nvidia,tegra20-pcie.txt           | 670 ---------------------
+ .../bindings/pci/nvidia,tegra20-pcie.yaml          | 532 ++++++++++++++++
+ .../devicetree/bindings/pci/rcar-gen4-pci-ep.yaml  |   9 +-
+ .../bindings/pci/rcar-gen4-pci-host.yaml           |  81 ++-
+ .../bindings/pci/renesas,r9a08g045-pcie.yaml       |  33 +-
+ .../devicetree/bindings/pci/toshiba,tc9563.yaml    |  14 +-
+ .../devicetree/bindings/pci/xilinx-versal-cpm.yaml |  38 ++
+ Documentation/driver-api/pci/p2pdma.rst            |   6 +-
+ MAINTAINERS                                        |   1 -
+ arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts       |   7 +-
+ drivers/gpio/Kconfig                               |  11 +
+ drivers/gpio/Makefile                              |   1 +
+ drivers/gpio/gpio-tc9563.c                         |  99 +++
+ drivers/net/wireless/ath/ath10k/Kconfig            |   2 +-
+ drivers/net/wireless/ath/ath10k/pci.c              |  11 +-
+ drivers/net/wireless/ath/ath10k/pci.h              |   5 +-
+ drivers/net/wireless/ath/ath11k/Kconfig            |   2 +-
+ drivers/net/wireless/ath/ath11k/pci.c              |  19 +-
+ drivers/net/wireless/ath/ath11k/pci.h              |   3 +-
+ drivers/net/wireless/ath/ath12k/Kconfig            |   2 +-
+ drivers/net/wireless/ath/ath12k/pci.c              |  19 +-
+ drivers/net/wireless/ath/ath12k/pci.h              |   4 +-
+ .../pci/controller/cadence/pcie-cadence-debugfs.c  |  15 +-
+ drivers/pci/controller/cadence/pcie-cadence-plat.c |   6 +-
+ drivers/pci/controller/cadence/pcie-cadence.c      |  16 +-
+ drivers/pci/controller/cadence/pcie-cadence.h      |   2 -
+ drivers/pci/controller/dwc/pci-dra7xx.c            |  22 +-
+ drivers/pci/controller/dwc/pci-imx6.c              |  94 ++-
+ drivers/pci/controller/dwc/pci-keystone.c          |  32 +-
+ drivers/pci/controller/dwc/pcie-designware-ep.c    |   8 +-
+ drivers/pci/controller/dwc/pcie-designware.h       |   1 +
+ drivers/pci/controller/dwc/pcie-dw-rockchip.c      |   1 +
+ drivers/pci/controller/dwc/pcie-histb.c            |   1 +
+ drivers/pci/controller/dwc/pcie-qcom-ep.c          |   1 +
+ drivers/pci/controller/dwc/pcie-qcom.c             | 176 +++++-
+ drivers/pci/controller/dwc/pcie-rcar-gen4.c        | 325 ++++++++--
+ drivers/pci/controller/dwc/pcie-spacemit-k1.c      |   3 +
+ drivers/pci/controller/dwc/pcie-tegra194.c         |   1 +
+ drivers/pci/controller/pci-tegra.c                 |   1 +
+ drivers/pci/controller/pci-xgene.c                 |  11 +-
+ drivers/pci/controller/pcie-aspeed.c               |   9 +-
+ drivers/pci/controller/pcie-mediatek-gen3.c        |  15 +-
+ drivers/pci/controller/pcie-mediatek.c             |   4 +-
+ drivers/pci/controller/pcie-rockchip-host.c        |   1 +
+ drivers/pci/controller/pcie-rzg3s-host.c           |  76 ++-
+ drivers/pci/controller/pcie-xilinx-cpm.c           |  77 ++-
+ drivers/pci/controller/plda/pcie-starfive.c        |   1 +
+ drivers/pci/controller/vmd.c                       |  57 +-
+ drivers/pci/endpoint/functions/pci-epf-vntb.c      |  93 ++-
+ drivers/pci/endpoint/pci-ep-msi.c                  |  40 +-
+ drivers/pci/hotplug/pciehp_core.c                  |   2 +-
+ drivers/pci/iov.c                                  |  25 -
+ drivers/pci/p2pdma.c                               |  39 +-
+ drivers/pci/pci-label.c                            |  94 ++-
+ drivers/pci/pci.h                                  |  13 +-
+ drivers/pci/pcie/aspm.c                            | 140 +++--
+ drivers/pci/probe.c                                |  15 +-
+ drivers/pci/pwrctrl/Kconfig                        |   2 +
+ drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c           | 321 ++++++----
+ drivers/pci/quirks.c                               | 126 +++-
+ drivers/pci/tph.c                                  |   6 +-
+ include/linux/pci-epf.h                            |   3 +-
+ include/linux/pci.h                                |   7 +-
+ include/linux/soc/qcom/tc9563.h                    |  19 +
+ 67 files changed, 2244 insertions(+), 1273 deletions(-)
+ delete mode 100644 Documentation/PCI/controller/rcar-pcie-firmware.rst
+ delete mode 100644 Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.txt
+ create mode 100644 Documentation/devicetree/bindings/pci/nvidia,tegra20-pcie.yaml
+ create mode 100644 drivers/gpio/gpio-tc9563.c
+ create mode 100644 include/linux/soc/qcom/tc9563.h
+Merging pstore/for-next/pstore (7c756181175d5 pstore: publish big_oops_buf after max_compressed_size)
+$ git merge -m Merge branch 'for-next/pstore' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git pstore/for-next/pstore
+Merge made by the 'ort' strategy.
+ fs/pstore/platform.c | 12 +++++++++---
+ fs/pstore/ram.c      |  1 +
+ fs/pstore/ram_core.c |  6 ++++++
+ 3 files changed, 16 insertions(+), 3 deletions(-)
+Merging hid/for-next (145c2b2e9a5c0 Merge branch 'for-7.3/upstream-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git hid/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |   6 +
+ drivers/hid/Kconfig                                |  21 +-
+ drivers/hid/Makefile                               |   2 +-
+ drivers/hid/amd-sfh-hid/amd_sfh_common.h           |   2 +
+ drivers/hid/amd-sfh-hid/amd_sfh_pcie.c             |  10 +
+ drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c      |  13 +-
+ drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c |  37 +-
+ drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h |   7 +
+ drivers/hid/bpf/hid_bpf_dispatch.c                 |   2 +-
+ .../hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c    |  11 +-
+ drivers/hid/hid-appletb-kbd.c                      |  95 ++-
+ drivers/hid/hid-asus.c                             |  14 +-
+ drivers/hid/hid-google-hammer.c                    |   2 +-
+ drivers/hid/hid-ids.h                              |  15 +
+ drivers/hid/hid-kysona.c                           | 292 --------
+ drivers/hid/hid-lenovo.c                           |  14 +
+ drivers/hid/hid-logitech-dj.c                      |  48 +-
+ drivers/hid/hid-logitech-hidpp.c                   |  51 +-
+ drivers/hid/hid-multitouch.c                       |   2 +-
+ drivers/hid/hid-nintendo.c                         |  14 +-
+ drivers/hid/hid-pulsar.c                           | 765 +++++++++++++++++++++
+ drivers/hid/hid-universal-pidff.c                  |   1 +
+ drivers/hid/i2c-hid/i2c-hid-acpi.c                 |   8 +-
+ drivers/hid/i2c-hid/i2c-hid-core.c                 |  11 +-
+ .../intel-thc-hid/intel-quicki2c/pci-quicki2c.c    |  10 +-
+ .../intel-thc-hid/intel-quicki2c/quicki2c-dev.h    |   2 +
+ .../intel-thc-hid/intel-quicki2c/quicki2c-hid.c    |   6 +-
+ .../intel-thc-hid/intel-quicki2c/quicki2c-hid.h    |   2 +-
+ .../intel-thc-hid/intel-quickspi/pci-quickspi.c    |   5 +-
+ .../intel-thc-hid/intel-quickspi/quickspi-dev.h    |   2 +
+ .../intel-thc-hid/intel-quickspi/quickspi-hid.c    |   6 +-
+ .../intel-thc-hid/intel-quickspi/quickspi-hid.h    |   2 +-
+ .../intel-quickspi/quickspi-protocol.c             |   2 +-
+ drivers/hid/surface-hid/surface_kbd.c              |   4 +-
+ drivers/hid/usbhid/hiddev.c                        |  45 +-
+ drivers/hid/wacom.h                                |   2 +-
+ drivers/hid/wacom_sys.c                            | 196 ++++--
+ drivers/hid/wacom_wac.c                            |  72 +-
+ drivers/hid/wacom_wac.h                            |   6 +-
+ include/linux/hiddev.h                             |   2 +
+ tools/testing/selftests/hid/hid_bpf.c              |  31 +
+ 41 files changed, 1270 insertions(+), 568 deletions(-)
+ delete mode 100644 drivers/hid/hid-kysona.c
+ create mode 100644 drivers/hid/hid-pulsar.c
+Merging i2c/i2c/for-next (8cd9520d35a6c Linux 7.1)
+$ git merge -m Merge branch 'i2c/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git i2c/i2c/for-next
+Already up to date.
+$ git am -3 ../patches/0001-i2c-Fix-up-the-rest-of-the-merge.patch
+Applying: i2c: Fix up the rest of the merge
+Using index info to reconstruct a base tree...
+M	drivers/i2c/i2c-core-base.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/i2c/i2c-core-base.c
+No changes -- Patch already applied.
+Merging i2c-andi/i2c/i2c-next (e33eb6c27c095 Merge branch 'i2c/i2c' into i2c/i2c-next)
+$ git merge -m Merge branch 'i2c/i2c-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-andi/i2c/i2c-next
+Merge made by the 'ort' strategy.
+ .../bindings/i2c/marvell,mv64xxx-i2c.yaml          |  1 +
+ .../devicetree/bindings/i2c/renesas,rcar-i2c.yaml  |  4 +-
+ .../bindings/i2c/snps,designware-i2c.yaml          | 15 ++++
+ .../bindings/i2c/xlnx,xps-iic-2.00.a.yaml          |  2 +-
+ drivers/i2c/Kconfig                                | 13 +++
+ drivers/i2c/busses/i2c-bcm2835.c                   |  2 +-
+ drivers/i2c/busses/i2c-ljca.c                      |  9 --
+ drivers/i2c/busses/i2c-qcom-geni.c                 | 77 +++++++++++------
+ drivers/i2c/i2c-core-acpi.c                        |  1 +
+ drivers/i2c/i2c-core-base.c                        | 43 ++++++++++
+ drivers/i2c/i2c-core-smbus.c                       | 24 ++++--
+ drivers/i2c/i2c-dev.c                              |  3 +
+ drivers/i2c/i2c-mux.c                              | 98 ++++++++++++----------
+ drivers/media/i2c/isl7998x.c                       |  3 +-
+ include/linux/i2c.h                                | 36 +++++++-
+ 15 files changed, 232 insertions(+), 99 deletions(-)
+Merging i2c-rust/rust-i2c-next (61ddec70c9bcc i2c: rust: mark I2cAdapter methods as inline)
+$ git merge -m Merge branch 'rust-i2c-next' of https://github.com/ikrtn/rust-for-linux i2c-rust/rust-i2c-next
+Auto-merging drivers/i2c/busses/i2c-imx-lpi2c.c
+CONFLICT (content): Merge conflict in drivers/i2c/busses/i2c-imx-lpi2c.c
+Auto-merging drivers/i2c/busses/i2c-imx.c
+Auto-merging drivers/i2c/busses/i2c-qcom-geni.c
+Auto-merging rust/kernel/i2c.rs
+Resolved 'drivers/i2c/busses/i2c-imx-lpi2c.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 8b36d0d274658] Merge branch 'rust-i2c-next' of https://github.com/ikrtn/rust-for-linux
+$ git diff -M --stat --summary HEAD^..
+ drivers/i2c/busses/i2c-imx-lpi2c.c | 18 ++++--------------
+ 1 file changed, 4 insertions(+), 14 deletions(-)
+Merging i3c/i3c/next (bf9ec06c90dcd i3c: master: amd: Report direct CCC read length in payload.actual_len)
+$ git merge -m Merge branch 'i3c/next' of https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git i3c/i3c/next
+Auto-merging drivers/i3c/master/amd-i3c-master.c
+Merge made by the 'ort' strategy.
+ drivers/i3c/device.c                               |  15 ++-
+ drivers/i3c/internals.h                            |   2 +
+ drivers/i3c/master.c                               |  91 ++++++++++++--
+ drivers/i3c/master/amd-i3c-master.c                |   4 +-
+ drivers/i3c/master/mipi-i3c-hci/cmd.h              |   4 +-
+ drivers/i3c/master/mipi-i3c-hci/cmd_v1.c           |  28 ++++-
+ drivers/i3c/master/mipi-i3c-hci/cmd_v2.c           |   4 +-
+ drivers/i3c/master/mipi-i3c-hci/core.c             | 106 +++++++++++++---
+ drivers/i3c/master/mipi-i3c-hci/dat.h              |   2 +
+ drivers/i3c/master/mipi-i3c-hci/dat_v1.c           |  13 +-
+ drivers/i3c/master/mipi-i3c-hci/dma.c              | 136 +++++++++++++++------
+ drivers/i3c/master/mipi-i3c-hci/hci.h              |   9 +-
+ drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c |   5 +-
+ include/linux/i3c/device.h                         |   4 +
+ include/linux/i3c/master.h                         |   3 +
+ include/linux/platform_data/mipi-i3c-hci.h         |   4 +
+ 16 files changed, 339 insertions(+), 91 deletions(-)
+Merging dmi/dmi-for-next (1afafbaf749d8 firmware/dmi: Include product_family info to modalias)
+$ git merge -m Merge branch 'dmi-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git dmi/dmi-for-next
+Already up to date.
+Merging hwmon-staging/hwmon-next (4781ca52761e6 hwmon:(pmbus/xdpe1a2g7b) Add support for xdpe1a2g7c controller)
+$ git merge -m Merge branch 'hwmon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-staging/hwmon-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-class-hwmon        |   36 +
+ .../bindings/hwmon/axiado,ax3000-pwm-fan.yaml      |   72 ++
+ .../bindings/hwmon/axiado,ax3000-tsadc.yaml        |   45 +
+ .../devicetree/bindings/hwmon/national,lm63.yaml   |   55 ++
+ .../devicetree/bindings/hwmon/national,lm90.yaml   |   28 +-
+ .../bindings/hwmon/pmbus/adi,max20826.yaml         |  102 ++
+ .../bindings/hwmon/pmbus/infineon,tda38740.yaml    |   55 ++
+ .../bindings/hwmon/pmbus/isil,isl68137.yaml        |    2 +
+ .../bindings/hwmon/pmbus/ti,tps25990.yaml          |    8 +-
+ .../devicetree/bindings/hwmon/ti,tmp102.yaml       |   16 +-
+ .../devicetree/bindings/hwmon/ti,tmp401.yaml       |    3 +
+ .../devicetree/bindings/trivial-devices.yaml       |   17 +-
+ .../devicetree/bindings/vendor-prefixes.yaml       |    2 +
+ Documentation/hwmon/arctic_fan_controller.rst      |   26 +-
+ Documentation/hwmon/asus_ec_sensors.rst            |    2 +
+ Documentation/hwmon/asus_rog_ryujin.rst            |    1 +
+ Documentation/hwmon/axiado-pwm-fan.rst             |   38 +
+ Documentation/hwmon/axiado-tsadc.rst               |   41 +
+ Documentation/hwmon/honor-fmi.rst                  |   32 +
+ Documentation/hwmon/index.rst                      |    6 +
+ Documentation/hwmon/it87.rst                       |    8 +
+ Documentation/hwmon/lm63.rst                       |   22 +
+ Documentation/hwmon/max20826.rst                   |  139 +++
+ Documentation/hwmon/max34440.rst                   |   28 +-
+ Documentation/hwmon/minisforum-um780xtx.rst        |   68 ++
+ Documentation/hwmon/nct6683.rst                    |    4 +
+ Documentation/hwmon/nct6775.rst                    |   32 +
+ Documentation/hwmon/sht4x.rst                      |   17 +-
+ Documentation/hwmon/socfpga-hwmon.rst              |   52 +
+ Documentation/hwmon/sysfs-interface.rst            |   14 +-
+ Documentation/hwmon/tda38740.rst                   |   65 ++
+ Documentation/hwmon/tps25990.rst                   |   15 +-
+ Documentation/hwmon/tps53679.rst                   |   39 +-
+ Documentation/hwmon/vexpress.rst                   |    2 +-
+ Documentation/hwmon/yogafan.rst                    |   46 +-
+ MAINTAINERS                                        |   45 +-
+ drivers/hwmon/Kconfig                              |   57 +-
+ drivers/hwmon/Makefile                             |    4 +
+ drivers/hwmon/abituguru.c                          |    4 +-
+ drivers/hwmon/acpi_power_meter.c                   |    4 +-
+ drivers/hwmon/adm1177.c                            |    2 +-
+ drivers/hwmon/adt7x10.c                            |    2 +-
+ drivers/hwmon/arctic_fan_controller.c              |   22 +-
+ drivers/hwmon/asus-ec-sensors.c                    |    4 +
+ drivers/hwmon/asus_atk0110.c                       |    4 +-
+ drivers/hwmon/asus_rog_ryujin.c                    |    3 +
+ drivers/hwmon/asus_wmi_sensors.c                   |    2 +-
+ drivers/hwmon/axiado-pwm-fan.c                     |  416 ++++++++
+ drivers/hwmon/axiado-tsadc.c                       |  275 ++++++
+ drivers/hwmon/coretemp.c                           |    2 +-
+ drivers/hwmon/dell-smm-hwmon.c                     |   32 +
+ drivers/hwmon/emc2103.c                            |    2 +-
+ drivers/hwmon/hih6130.c                            |    4 +-
+ drivers/hwmon/honor-fmi.c                          |  178 ++++
+ drivers/hwmon/hp-wmi-sensors.c                     |    2 +-
+ drivers/hwmon/hwmon.c                              |    6 +
+ drivers/hwmon/it87.c                               |  308 +++++-
+ drivers/hwmon/k10temp.c                            |    2 +
+ drivers/hwmon/lm63.c                               |  426 ++++++--
+ drivers/hwmon/lm85.c                               |    2 +-
+ drivers/hwmon/lm90.c                               |    5 +
+ drivers/hwmon/minisforum-um780xtx.c                |  501 ++++++++++
+ drivers/hwmon/nct6775-core.c                       |  116 +++
+ drivers/hwmon/nct6775-platform.c                   |   33 +-
+ drivers/hwmon/nct6775.h                            |    4 +-
+ drivers/hwmon/pmbus/Kconfig                        |   41 +-
+ drivers/hwmon/pmbus/Makefile                       |    2 +
+ drivers/hwmon/pmbus/ibm-cffps.c                    |    4 +-
+ drivers/hwmon/pmbus/isl68137.c                     |    4 +
+ drivers/hwmon/pmbus/ltc4286.c                      |   96 +-
+ drivers/hwmon/pmbus/max20826.c                     | 1044 ++++++++++++++++++++
+ drivers/hwmon/pmbus/max34440.c                     |    2 +
+ drivers/hwmon/pmbus/mp2888.c                       |    4 +-
+ drivers/hwmon/pmbus/pmbus.h                        |    7 +
+ drivers/hwmon/pmbus/pmbus_core.c                   |   29 +-
+ drivers/hwmon/pmbus/tda38740.c                     |   64 ++
+ drivers/hwmon/pmbus/tps25990.c                     |  249 +++--
+ drivers/hwmon/pmbus/tps53679.c                     |  146 ++-
+ drivers/hwmon/pmbus/xdpe1a2g7b.c                   |   10 +-
+ drivers/hwmon/pt5161l.c                            |    4 +-
+ drivers/hwmon/pwm-fan.c                            |    2 +-
+ drivers/hwmon/scmi-hwmon.c                         |    3 +-
+ drivers/hwmon/sht4x.c                              |   66 +-
+ drivers/hwmon/socfpga-hwmon.c                      |   22 +
+ drivers/hwmon/spd5118.c                            |   67 +-
+ drivers/hwmon/xgene-hwmon.c                        |    6 +-
+ drivers/hwmon/yogafan.c                            |   48 +
+ include/linux/hwmon.h                              |   12 +
+ 88 files changed, 5146 insertions(+), 391 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/hwmon/axiado,ax3000-pwm-fan.yaml
+ create mode 100644 Documentation/devicetree/bindings/hwmon/axiado,ax3000-tsadc.yaml
+ create mode 100644 Documentation/devicetree/bindings/hwmon/national,lm63.yaml
+ create mode 100644 Documentation/devicetree/bindings/hwmon/pmbus/adi,max20826.yaml
+ create mode 100644 Documentation/devicetree/bindings/hwmon/pmbus/infineon,tda38740.yaml
+ create mode 100644 Documentation/hwmon/axiado-pwm-fan.rst
+ create mode 100644 Documentation/hwmon/axiado-tsadc.rst
+ create mode 100644 Documentation/hwmon/honor-fmi.rst
+ create mode 100644 Documentation/hwmon/max20826.rst
+ create mode 100644 Documentation/hwmon/minisforum-um780xtx.rst
+ create mode 100644 Documentation/hwmon/tda38740.rst
+ create mode 100644 drivers/hwmon/axiado-pwm-fan.c
+ create mode 100644 drivers/hwmon/axiado-tsadc.c
+ create mode 100644 drivers/hwmon/honor-fmi.c
+ create mode 100644 drivers/hwmon/minisforum-um780xtx.c
+ create mode 100644 drivers/hwmon/pmbus/max20826.c
+ create mode 100644 drivers/hwmon/pmbus/tda38740.c
+$ git am -3 ../patches/0001-drivers-hwmon-sht4x.c-Fix-up-merge-issues.patch
+Applying: drivers/hwmon/sht4x.c: Fix up merge issues
+Using index info to reconstruct a base tree...
+M	drivers/hwmon/sht4x.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/hwmon/sht4x.c
+CONFLICT (content): Merge conflict in drivers/hwmon/sht4x.c
+Recorded preimage for 'drivers/hwmon/sht4x.c'
+error: Failed to merge in the changes.
+hint: Use 'git am --show-current-patch=diff' to see the failed patch
+hint: When you have resolved this problem, run "git am --continue".
+hint: If you prefer to skip this patch, run "git am --skip" instead.
+hint: To restore the original branch and stop patching, run "git am --abort".
+hint: Disable this message with "git config advice.mergeConflict false"
+Patch failed at 0001 drivers/hwmon/sht4x.c: Fix up merge issues
+Merging jc_docs/docs-next (2d72a4c09867a docs: kdoc: parse context_lock_struct() as struct declaration)
+$ git merge -m Merge branch 'docs-next' of git://git.lwn.net/linux.git jc_docs/docs-next
+Auto-merging Documentation/ABI/testing/sysfs-bus-pci
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging Documentation/admin-guide/sysctl/kernel.rst
+Auto-merging Documentation/filesystems/locking.rst
+Auto-merging Documentation/filesystems/porting.rst
+Auto-merging Documentation/filesystems/proc.rst
+Auto-merging Makefile
+Merge made by the 'ort' strategy.
+ .../ABI/stable/sysfs-driver-firmware-zynqmp        |   70 +-
+ Documentation/ABI/testing/sysfs-bus-pci            |   14 +-
+ Documentation/ABI/testing/sysfs-bus-usb            |    3 +-
+ .../ABI/testing/sysfs-class-firmware-attributes    |   21 +-
+ .../ABI/testing/sysfs-driver-aspeed-uart-routing   |    7 +-
+ Documentation/ABI/testing/sysfs-driver-xdata       |   20 +-
+ .../ABI/testing/sysfs-driver-xilinx-tmr-manager    |   14 +-
+ Documentation/ABI/testing/sysfs-platform-intel-ifs |    6 +-
+ Documentation/Makefile                             |    5 +
+ Documentation/admin-guide/LSM/LoadPin.rst          |   10 +-
+ Documentation/admin-guide/RAS/main.rst             |   12 +-
+ Documentation/admin-guide/cpu-isolation.rst        |    4 +-
+ Documentation/admin-guide/kernel-parameters.txt    |   58 +-
+ Documentation/admin-guide/parport.rst              |    2 +-
+ Documentation/admin-guide/sysctl/kernel.rst        |   19 +-
+ Documentation/admin-guide/sysctl/vm.rst            |    2 +-
+ .../verify-bugs-and-bisect-regressions.rst         |   26 +-
+ Documentation/arch/powerpc/vas-api.rst             |    4 +-
+ Documentation/core-api/cpu_hotplug.rst             |    4 +-
+ Documentation/core-api/debug-objects.rst           |    2 +-
+ Documentation/core-api/dma-attributes.rst          |    2 +-
+ Documentation/core-api/dma-isa-lpc.rst             |   11 +-
+ Documentation/core-api/housekeeping.rst            |    2 +-
+ Documentation/core-api/irq/irq-affinity.rst        |    2 +-
+ Documentation/core-api/irq/irqflags-tracing.rst    |    8 +-
+ Documentation/core-api/maple_tree.rst              |    9 +-
+ Documentation/core-api/real-time/differences.rst   |   10 +-
+ Documentation/core-api/swiotlb.rst                 |    2 +-
+ Documentation/core-api/this_cpu_ops.rst            |    6 +-
+ Documentation/core-api/xarray.rst                  |    4 +-
+ Documentation/doc-guide/kernel-doc.rst             |    7 +-
+ Documentation/doc-guide/parse-headers.rst          |   15 +-
+ Documentation/driver-api/reset.rst                 |    2 +-
+ Documentation/driver-api/serial/serial-rs485.rst   |    5 +-
+ .../locking/cmpxchg-local/arch-support.txt         |    2 +-
+ Documentation/filesystems/fuse/fuse.rst            |    8 +-
+ Documentation/filesystems/locking.rst              |    2 +-
+ Documentation/filesystems/porting.rst              |    2 +-
+ Documentation/filesystems/proc.rst                 |   10 +-
+ Documentation/input/devices/yealink.rst            |    7 +-
+ Documentation/input/notifier.rst                   |    2 +-
+ Documentation/livepatch/api.rst                    |    5 +-
+ Documentation/misc-devices/spear-pcie-gadget.rst   |   10 +-
+ Documentation/process/1.Intro.rst                  |    2 +-
+ Documentation/process/2.Process.rst                |   20 +-
+ Documentation/process/3.Early-stage.rst            |    2 +-
+ Documentation/process/5.Posting.rst                |   12 +-
+ Documentation/process/6.Followthrough.rst          |    2 +-
+ Documentation/process/7.AdvancedTopics.rst         |   36 +-
+ Documentation/process/backporting.rst              |   18 +-
+ .../process/embargoed-hardware-issues.rst          |    2 +-
+ Documentation/process/handling-regressions.rst     |    2 +-
+ Documentation/process/howto.rst                    |    2 +-
+ Documentation/process/maintainer-pgp-guide.rst     |   38 +-
+ Documentation/process/submitting-patches.rst       |    2 +-
+ Documentation/scheduler/index.rst                  |    1 +
+ Documentation/scheduler/sched-preemption.rst       |   87 ++
+ Documentation/timers/hpet.rst                      |   33 +-
+ Documentation/timers/no_hz.rst                     |    4 +-
+ .../translations/pt_BR/admin-guide/README.rst      |  382 ++++++
+ .../translations/pt_BR/admin-guide/devices.rst     |  278 +++++
+ .../translations/pt_BR/admin-guide/index.rst       |  190 +++
+ Documentation/translations/pt_BR/index.rst         |   10 +
+ .../translations/pt_BR/process/2.Process.rst       |   14 +-
+ .../translations/pt_BR/process/3.Early-stage.rst   |   10 +-
+ .../translations/pt_BR/process/4.Coding.rst        |    6 +-
+ .../translations/pt_BR/process/5.Posting.rst       |   12 +-
+ .../translations/pt_BR/process/6.Followthrough.rst |   14 +-
+ .../translations/pt_BR/process/8.Conclusion.rst    |    3 +-
+ .../translations/pt_BR/process/adding-syscalls.rst |   56 +-
+ .../pt_BR/process/applying-patches.rst             |    6 +-
+ .../translations/pt_BR/process/backporting.rst     |    4 +-
+ .../process/code-of-conduct-interpretation.rst     |    2 +
+ .../translations/pt_BR/process/code-of-conduct.rst |    4 +-
+ .../pt_BR/process/coding-assistants.rst            |   60 +
+ .../translations/pt_BR/process/coding-style.rst    | 1320 ++++++++++++++++++++
+ Documentation/translations/pt_BR/process/cve.rst   |    4 +-
+ .../translations/pt_BR/process/debugging/index.rst |   73 ++
+ .../pt_BR/process/development-process.rst          |    2 +
+ .../pt_BR/process/embargoed-hardware-issues.rst    |  362 ++++++
+ Documentation/translations/pt_BR/process/howto.rst |   19 +-
+ Documentation/translations/pt_BR/process/index.rst |   10 +
+ .../translations/pt_BR/process/kernel-docs.rst     |    2 +
+ .../pt_BR/process/kernel-enforcement-statement.rst |  163 +++
+ .../translations/pt_BR/process/license-rules.rst   |    2 +
+ .../pt_BR/process/maintainer-devicetree.rst        |   76 ++
+ .../pt_BR/process/maintainer-handbooks.rst         |    4 +-
+ .../pt_BR/process/maintainer-kvm-x86.rst           |   10 +-
+ .../pt_BR/process/maintainer-soc-clean-dts.rst     |    5 +-
+ .../translations/pt_BR/process/maintainer-tip.rst  |  847 +++++++++++++
+ .../pt_BR/process/management-style.rst             |    9 +-
+ .../translations/pt_BR/process/security-bugs.rst   |   25 +-
+ .../pt_BR/process/stable-api-nonsense.rst          |  208 +++
+ .../pt_BR/process/submit-checklist.rst             |    4 +-
+ .../pt_BR/process/submitting-patches.rst           |  963 ++++++++++++++
+ .../pt_BR/process/volatile-considered-harmful.rst  |  130 ++
+ .../translations/zh_CN/admin-guide/README.rst      |    2 +-
+ .../zh_CN/admin-guide/mm/damon/index.rst           |   15 +-
+ .../zh_CN/admin-guide/mm/damon/lru_sort.rst        |   68 +-
+ .../zh_CN/admin-guide/mm/damon/reclaim.rst         |   82 +-
+ .../zh_CN/admin-guide/mm/damon/start.rst           |   63 +-
+ .../zh_CN/admin-guide/mm/damon/stat.rst            |   94 ++
+ .../zh_CN/admin-guide/mm/damon/usage.rst           |  521 ++++++--
+ .../translations/zh_CN/mm/damon/design.rst         |  667 +++++++++-
+ .../translations/zh_CN/networking/driver.rst       |  138 ++
+ .../translations/zh_CN/networking/index.rst        |   10 +-
+ .../translations/zh_CN/networking/ipv6.rst         |   76 ++
+ .../translations/zh_CN/networking/secid.rst        |   24 +
+ .../translations/zh_CN/networking/sriov.rst        |   34 +
+ .../translations/zh_CN/networking/team.rst         |   16 +
+ .../zh_CN/process/applying-patches.rst             |  390 ++++++
+ Documentation/translations/zh_CN/process/howto.rst |    2 +-
+ Documentation/translations/zh_CN/process/index.rst |    2 +-
+ Documentation/translations/zh_TW/glossary.rst      |  168 +++
+ Documentation/translations/zh_TW/index.rst         |    5 +-
+ .../translations/zh_TW/process/1.Intro.rst         |  227 ++--
+ .../translations/zh_TW/process/2.Process.rst       |  325 +++--
+ .../translations/zh_TW/process/5.Posting.rst       |  235 ++--
+ .../zh_TW/process/7.AdvancedTopics.rst             |  106 +-
+ .../translations/zh_TW/process/8.Conclusion.rst    |   58 +-
+ .../process/code-of-conduct-interpretation.rst     |  162 ++-
+ .../translations/zh_TW/process/coding-style.rst    |  610 ++++-----
+ .../translations/zh_TW/process/email-clients.rst   |  185 +--
+ .../zh_TW/process/embargoed-hardware-issues.rst    |  209 ++--
+ Documentation/translations/zh_TW/process/howto.rst |  344 ++---
+ Documentation/translations/zh_TW/process/index.rst |  105 +-
+ .../translations/zh_TW/process/license-rules.rst   |  201 +--
+ .../zh_TW/process/programming-language.rst         |   92 +-
+ .../zh_TW/process/stable-kernel-rules.rst          |  244 +++-
+ .../zh_TW/process/submitting-patches.rst           |  568 +++++----
+ Documentation/userspace-api/dma-buf-heaps.rst      |    2 +-
+ Documentation/userspace-api/futex2.rst             |    4 +-
+ Documentation/userspace-api/ioctl/ioctl-number.rst |    7 +-
+ Documentation/userspace-api/iommufd.rst            |    2 +-
+ Documentation/userspace-api/vduse.rst              |   33 +-
+ Documentation/virt/coco/sev-guest.rst              |    2 +-
+ Makefile                                           |    2 +-
+ include/uapi/linux/gpio.h                          |    2 +-
+ security/landlock/syscalls.c                       |    2 +-
+ tools/lib/python/kdoc/c_lex.py                     |    2 +-
+ tools/lib/python/kdoc/kdoc_parser.py               |   11 +-
+ tools/lib/python/kdoc/xforms_lists.py              |    3 +-
+ tools/unittests/test_kdoc_parser.py                |   26 +
+ 143 files changed, 9947 insertions(+), 2187 deletions(-)
+ create mode 100644 Documentation/scheduler/sched-preemption.rst
+ create mode 100644 Documentation/translations/pt_BR/admin-guide/README.rst
+ create mode 100644 Documentation/translations/pt_BR/admin-guide/devices.rst
+ create mode 100644 Documentation/translations/pt_BR/admin-guide/index.rst
+ create mode 100644 Documentation/translations/pt_BR/process/coding-assistants.rst
+ create mode 100644 Documentation/translations/pt_BR/process/coding-style.rst
+ create mode 100644 Documentation/translations/pt_BR/process/debugging/index.rst
+ create mode 100644 Documentation/translations/pt_BR/process/embargoed-hardware-issues.rst
+ create mode 100644 Documentation/translations/pt_BR/process/kernel-enforcement-statement.rst
+ create mode 100644 Documentation/translations/pt_BR/process/maintainer-devicetree.rst
+ create mode 100644 Documentation/translations/pt_BR/process/maintainer-tip.rst
+ create mode 100644 Documentation/translations/pt_BR/process/stable-api-nonsense.rst
+ create mode 100644 Documentation/translations/pt_BR/process/submitting-patches.rst
+ create mode 100644 Documentation/translations/pt_BR/process/volatile-considered-harmful.rst
+ create mode 100644 Documentation/translations/zh_CN/admin-guide/mm/damon/stat.rst
+ create mode 100644 Documentation/translations/zh_CN/networking/driver.rst
+ create mode 100644 Documentation/translations/zh_CN/networking/ipv6.rst
+ create mode 100644 Documentation/translations/zh_CN/networking/secid.rst
+ create mode 100644 Documentation/translations/zh_CN/networking/sriov.rst
+ create mode 100644 Documentation/translations/zh_CN/networking/team.rst
+ create mode 100644 Documentation/translations/zh_CN/process/applying-patches.rst
+ create mode 100644 Documentation/translations/zh_TW/glossary.rst
+Merging v4l-dvb/next (58348f64125e9 media: ipu6: Support upstream sub-devices without get_frame_desc())
+$ git merge -m Merge branch 'next' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb/next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
+Auto-merging arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts
+Auto-merging arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
+Auto-merging arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi
+Auto-merging drivers/media/i2c/isl7998x.c
+Auto-merging drivers/media/pci/intel/ipu-bridge.c
+Auto-merging drivers/media/platform/nxp/imx7-media-csi.c
+Auto-merging drivers/media/usb/em28xx/em28xx-video.c
+Auto-merging drivers/media/v4l2-core/v4l2-ctrls-core.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/media/vivid.rst          |    4 +-
+ .../bindings/clock/nxp,imx95-blk-ctl.yaml          |   71 +
+ .../bindings/media/fsl,imx95-csi-formatter.yaml    |   88 +
+ .../bindings/media/i2c/himax,hm1246.yaml           |  121 +
+ .../devicetree/bindings/media/i2c/hynix,hi846.yaml |    3 +-
+ .../devicetree/bindings/media/i2c/ite,it6625.yaml  |  175 ++
+ .../bindings/media/i2c/ovti,os02g10.yaml           |   94 +
+ .../bindings/media/i2c/ovti,ov08d10.yaml           |    3 +-
+ .../devicetree/bindings/media/i2c/ovti,ov4689.yaml |    3 +-
+ .../devicetree/bindings/media/i2c/ovti,ov5675.yaml |    3 +-
+ .../devicetree/bindings/media/i2c/ovti,ov5693.yaml |    5 +-
+ .../bindings/media/i2c/ovti,ov64a40.yaml           |    3 +-
+ .../devicetree/bindings/media/i2c/sony,imx111.yaml |    3 +-
+ .../devicetree/bindings/media/i2c/sony,imx355.yaml |    3 +-
+ .../devicetree/bindings/media/i2c/sony,imx415.yaml |    3 +-
+ .../devicetree/bindings/media/i2c/st,vd55g1.yaml   |    3 +-
+ .../devicetree/bindings/media/i2c/st,vd56g3.yaml   |    3 +-
+ .../bindings/media/i2c/thine,thp7312.yaml          |    3 +-
+ .../bindings/media/i2c/toshiba,tc358743.txt        |   48 -
+ .../bindings/media/i2c/toshiba,tc358743.yaml       |   93 +
+ .../bindings/media/nxp,imx8mq-mipi-csi2.yaml       |    4 +-
+ .../devicetree/bindings/media/snps,dw-hdmi-rx.yaml |   13 +-
+ .../bindings/media/video-interface-devices.yaml    |   17 +-
+ Documentation/driver-api/media/tx-rx.rst           |    6 +-
+ .../userspace-api/media/drivers/dcmipp.rst         |   14 +
+ .../userspace-api/media/drivers/index.rst          |    1 +
+ MAINTAINERS                                        |   51 +-
+ .../nvidia/tegra30-asus-nexus7-grouper-common.dtsi |    3 +-
+ .../nvidia/tegra30-asus-transformer-common.dtsi    |    3 +-
+ arch/arm/boot/dts/nvidia/tegra30-lg-p895.dts       |    4 +-
+ arch/arm/boot/dts/nvidia/tegra30-lg-x3.dtsi        |    3 +-
+ .../imx8mp-tqma8mpql-mba8mp-ras314-imx219.dtso     |    3 +-
+ arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi  |    3 +-
+ arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts |    3 +-
+ .../dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts     |    3 +-
+ arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi |    3 +-
+ .../r8a779g3-sparrow-hawk-camera-j1-imx219.dtso    |    3 +-
+ .../r8a779g3-sparrow-hawk-camera-j1-imx462.dtso    |    3 +-
+ .../r8a779g3-sparrow-hawk-camera-j2-imx219.dtso    |    3 +-
+ .../r8a779g3-sparrow-hawk-camera-j2-imx462.dtso    |    3 +-
+ arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi      |    3 +-
+ .../rockchip/px30-ringneck-haikou-video-demo.dtso  |    3 +-
+ .../boot/dts/rockchip/rk3399-pinephone-pro.dts     |    5 +-
+ .../rk3588-rock-5b-plus-radxa-cam4k-cam0.dtso      |    3 +-
+ .../rk3588-rock-5b-plus-radxa-cam4k-cam1.dtso      |    3 +-
+ drivers/media/cec/platform/meson/ao-cec-g12a.c     |    2 +-
+ .../extron-da-hd-4k-plus/extron-da-hd-4k-plus.c    |    4 +-
+ drivers/media/common/cypress_firmware.c            |    2 +
+ drivers/media/common/saa7146/saa7146_core.c        |   12 +-
+ drivers/media/common/saa7146/saa7146_video.c       |   11 +
+ drivers/media/dvb-frontends/drxd_map_firm.h        |    4 +-
+ drivers/media/dvb-frontends/stv0900_core.c         |    2 +
+ drivers/media/dvb-frontends/stv090x.c              |    2 +
+ drivers/media/i2c/Kconfig                          |   51 +-
+ drivers/media/i2c/Makefile                         |    3 +
+ drivers/media/i2c/adv7170.c                        |    1 +
+ drivers/media/i2c/adv7175.c                        |    3 +-
+ drivers/media/i2c/adv7180.c                        |    5 +-
+ drivers/media/i2c/adv7183.c                        |    3 +-
+ drivers/media/i2c/adv7343.c                        |   10 +-
+ drivers/media/i2c/adv7393.c                        |   10 +-
+ drivers/media/i2c/adv748x/adv748x-afe.c            |    1 +
+ drivers/media/i2c/adv748x/adv748x-core.c           |   13 +-
+ drivers/media/i2c/adv748x/adv748x-csi2.c           |    1 +
+ drivers/media/i2c/adv748x/adv748x-hdmi.c           |    1 +
+ drivers/media/i2c/adv7511-v4l2.c                   |    1 +
+ drivers/media/i2c/adv7604.c                        |    7 +-
+ drivers/media/i2c/adv7842.c                        |   11 +-
+ drivers/media/i2c/ak881x.c                         |    2 +-
+ drivers/media/i2c/alvium-csi2.c                    |    4 +
+ drivers/media/i2c/ar0521.c                         |    2 +-
+ drivers/media/i2c/ccs/ccs-core.c                   |   12 +-
+ drivers/media/i2c/ccs/ccs-data.h                   |    4 +-
+ drivers/media/i2c/cvs/Kconfig                      |    1 +
+ drivers/media/i2c/cvs/core.c                       |   30 +-
+ drivers/media/i2c/cvs/v4l2.c                       |   96 +-
+ drivers/media/i2c/cx25840/cx25840-core.c           |   39 +-
+ drivers/media/i2c/ds90ub913.c                      |    1 +
+ drivers/media/i2c/ds90ub953.c                      |    1 +
+ drivers/media/i2c/ds90ub960.c                      |    1 +
+ drivers/media/i2c/et8ek8/et8ek8_driver.c           |    1 +
+ drivers/media/i2c/gc0308.c                         |    1 +
+ drivers/media/i2c/gc0310.c                         |    2 +-
+ drivers/media/i2c/gc05a2.c                         |    4 +-
+ drivers/media/i2c/gc08a3.c                         |    4 +-
+ drivers/media/i2c/gc2145.c                         |    2 +
+ drivers/media/i2c/hi556.c                          |    2 +
+ drivers/media/i2c/hi846.c                          |    2 +
+ drivers/media/i2c/hi847.c                          |    3 +-
+ drivers/media/i2c/hm1246.c                         | 1287 +++++++++++
+ drivers/media/i2c/imx111.c                         |    1 +
+ drivers/media/i2c/imx208.c                         |    1 +
+ drivers/media/i2c/imx214.c                         |    4 +-
+ drivers/media/i2c/imx219.c                         |   42 +-
+ drivers/media/i2c/imx258.c                         |    2 +
+ drivers/media/i2c/imx274.c                         |    3 +
+ drivers/media/i2c/imx283.c                         |    2 +
+ drivers/media/i2c/imx290.c                         |    4 +-
+ drivers/media/i2c/imx296.c                         |    7 +-
+ drivers/media/i2c/imx319.c                         |    1 +
+ drivers/media/i2c/imx334.c                         |  135 +-
+ drivers/media/i2c/imx335.c                         |    4 +-
+ drivers/media/i2c/imx355.c                         |    4 +-
+ drivers/media/i2c/imx412.c                         |    3 +-
+ drivers/media/i2c/imx415.c                         |    4 +-
+ drivers/media/i2c/imx471.c                         |    6 +-
+ drivers/media/i2c/imx678.c                         |    2 +-
+ drivers/media/i2c/isl7998x.c                       |    1 +
+ drivers/media/i2c/it6625.c                         | 2391 ++++++++++++++++++++
+ drivers/media/i2c/lt6911uxe.c                      |    5 +-
+ drivers/media/i2c/max9286.c                        |    1 +
+ drivers/media/i2c/max96714.c                       |    1 +
+ drivers/media/i2c/max96717.c                       |    1 +
+ drivers/media/i2c/ml86v7667.c                      |    1 -
+ drivers/media/i2c/mt9m001.c                        |    9 +-
+ drivers/media/i2c/mt9m111.c                        |    3 +
+ drivers/media/i2c/mt9m114.c                        |    6 +
+ drivers/media/i2c/mt9p031.c                        |    3 +
+ drivers/media/i2c/mt9t112.c                        |    3 +
+ drivers/media/i2c/mt9v011.c                        |    1 +
+ drivers/media/i2c/mt9v032.c                        |    3 +
+ drivers/media/i2c/mt9v111.c                        |    1 +
+ drivers/media/i2c/og01a1b.c                        |    8 +-
+ drivers/media/i2c/og0ve1b.c                        |    3 +-
+ drivers/media/i2c/os02g10.c                        |  934 ++++++++
+ drivers/media/i2c/os05b10.c                        |    2 +
+ drivers/media/i2c/ov01a10.c                        |    3 +
+ drivers/media/i2c/ov02a10.c                        |    3 +-
+ drivers/media/i2c/ov02c10.c                        |    1 +
+ drivers/media/i2c/ov02e10.c                        |    1 +
+ drivers/media/i2c/ov08d10.c                        |    1 +
+ drivers/media/i2c/ov08x40.c                        |    1 +
+ drivers/media/i2c/ov13858.c                        |    1 +
+ drivers/media/i2c/ov13b10.c                        |    1 +
+ drivers/media/i2c/ov2640.c                         |    2 +
+ drivers/media/i2c/ov2659.c                         |    1 +
+ drivers/media/i2c/ov2680.c                         |    3 +
+ drivers/media/i2c/ov2685.c                         |    2 +
+ drivers/media/i2c/ov2732.c                         |    4 +-
+ drivers/media/i2c/ov2735.c                         |    5 +-
+ drivers/media/i2c/ov2740.c                         |    1 +
+ drivers/media/i2c/ov4689.c                         |    2 +
+ drivers/media/i2c/ov5640.c                         |    2 +
+ drivers/media/i2c/ov5645.c                         |    4 +-
+ drivers/media/i2c/ov5647.c                         |    6 +
+ drivers/media/i2c/ov5648.c                         |    5 +-
+ drivers/media/i2c/ov5670.c                         |    2 +
+ drivers/media/i2c/ov5675.c                         |    2 +
+ drivers/media/i2c/ov5693.c                         |   28 +
+ drivers/media/i2c/ov5695.c                         |    1 +
+ drivers/media/i2c/ov6211.c                         |    3 +-
+ drivers/media/i2c/ov64a40.c                        |    2 +
+ drivers/media/i2c/ov7251.c                         |    4 +-
+ drivers/media/i2c/ov7670.c                         |    1 +
+ drivers/media/i2c/ov772x.c                         |    4 +-
+ drivers/media/i2c/ov7740.c                         |    1 +
+ drivers/media/i2c/ov8856.c                         |   33 +-
+ drivers/media/i2c/ov8858.c                         |    3 +-
+ drivers/media/i2c/ov8865.c                         |    2 +
+ drivers/media/i2c/ov9282.c                         |    4 +-
+ drivers/media/i2c/ov9640.c                         |    2 +
+ drivers/media/i2c/ov9650.c                         |    1 +
+ drivers/media/i2c/ov9734.c                         |    1 +
+ drivers/media/i2c/rdacm20.c                        |    1 -
+ drivers/media/i2c/rdacm21.c                        |    1 -
+ drivers/media/i2c/rj54n1cb0c.c                     |    3 +
+ drivers/media/i2c/s5c73m3/s5c73m3-core.c           |    2 +
+ drivers/media/i2c/s5k3m5.c                         |    4 +-
+ drivers/media/i2c/s5k5baf.c                        |    3 +
+ drivers/media/i2c/s5k6a3.c                         |    1 +
+ drivers/media/i2c/s5kjn1.c                         |    4 +-
+ drivers/media/i2c/saa6752hs.c                      |    1 +
+ drivers/media/i2c/saa7115.c                        |    1 +
+ drivers/media/i2c/saa717x.c                        |    1 +
+ drivers/media/i2c/st-mipid02.c                     |    1 +
+ drivers/media/i2c/t4ka3.c                          |    4 +-
+ drivers/media/i2c/tc358743.c                       |    6 +-
+ drivers/media/i2c/tc358746.c                       |    1 +
+ drivers/media/i2c/tda1997x.c                       |    3 +-
+ drivers/media/i2c/thp7312.c                        |    1 +
+ drivers/media/i2c/ths7303.c                        |   10 +-
+ drivers/media/i2c/ths8200.c                        |   14 +-
+ drivers/media/i2c/ths8200_regs.h                   |   14 +-
+ drivers/media/i2c/tvp514x.c                        |    1 +
+ drivers/media/i2c/tvp5150.c                        |    3 +-
+ drivers/media/i2c/tvp7002.c                        |    1 +
+ drivers/media/i2c/tw9900.c                         |    1 +
+ drivers/media/i2c/tw9910.c                         |    2 +
+ drivers/media/i2c/vd55g1.c                         |    4 +-
+ drivers/media/i2c/vd56g3.c                         |    4 +-
+ drivers/media/i2c/vgxy61.c                         |    4 +-
+ drivers/media/i2c/wm8739.c                         |    2 +-
+ drivers/media/pci/bt8xx/bttv-driver.c              |    8 +-
+ drivers/media/pci/cobalt/cobalt-driver.c           |    8 +-
+ drivers/media/pci/cobalt/cobalt-v4l2.c             |   10 +-
+ drivers/media/pci/cx18/cx18-av-core.c              |    1 +
+ drivers/media/pci/cx18/cx18-controls.c             |    2 +-
+ drivers/media/pci/cx18/cx18-driver.c               |    9 +-
+ drivers/media/pci/cx18/cx18-ioctl.c                |    2 +-
+ drivers/media/pci/cx18/cx18-queue.c                |    5 +-
+ drivers/media/pci/cx18/cx23418.h                   |    2 +-
+ drivers/media/pci/cx23885/cx23885-core.c           |    4 +
+ drivers/media/pci/cx23885/cx23885-dvb.c            |   14 +-
+ drivers/media/pci/cx23885/cx23885-video.c          |    4 +-
+ drivers/media/pci/cx88/cx88-input.c                |   22 +-
+ drivers/media/pci/cx88/cx88-mpeg.c                 |    9 +-
+ drivers/media/pci/cx88/cx88-video.c                |    8 +-
+ drivers/media/pci/hws/hws.h                        |    4 +-
+ drivers/media/pci/hws/hws_pci.c                    |    6 +-
+ drivers/media/pci/hws/hws_reg.h                    |   10 +-
+ drivers/media/pci/hws/hws_v4l2_ioctl.c             |    5 +-
+ drivers/media/pci/hws/hws_video.c                  |    6 +-
+ drivers/media/pci/intel/ipu-bridge.c               |  148 +-
+ drivers/media/pci/intel/ipu3/ipu3-cio2.c           |    1 +
+ drivers/media/pci/intel/ipu6/Kconfig               |    9 +
+ drivers/media/pci/intel/ipu6/Makefile              |   10 +-
+ drivers/media/pci/intel/ipu6/ipu6-bus.h            |    6 +-
+ drivers/media/pci/intel/ipu6/ipu6-buttress.c       |  588 +++--
+ drivers/media/pci/intel/ipu6/ipu6-buttress.h       |   50 +-
+ drivers/media/pci/intel/ipu6/ipu6-cpd.c            |  201 +-
+ drivers/media/pci/intel/ipu6/ipu6-cpd.h            |   43 +
+ drivers/media/pci/intel/ipu6/ipu6-dma.c            |   24 +-
+ drivers/media/pci/intel/ipu6/ipu6-dma.h            |    2 +
+ drivers/media/pci/intel/ipu6/ipu6-fw-isys.c        |  586 ++++-
+ drivers/media/pci/intel/ipu6/ipu6-fw-isys.h        |   48 +-
+ drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c      |  584 ++++-
+ drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h      |   18 +-
+ drivers/media/pci/intel/ipu6/ipu6-isys-queue.c     |  213 +-
+ drivers/media/pci/intel/ipu6/ipu6-isys-queue.h     |    6 +-
+ drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c    |   23 +-
+ drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h    |    3 +
+ drivers/media/pci/intel/ipu6/ipu6-isys-video.c     |  772 ++-----
+ drivers/media/pci/intel/ipu6/ipu6-isys-video.h     |   62 +-
+ drivers/media/pci/intel/ipu6/ipu6-isys.c           |  607 ++---
+ drivers/media/pci/intel/ipu6/ipu6-isys.h           |  105 +-
+ drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c         |  296 +++
+ drivers/media/pci/intel/ipu6/ipu6-mmu.c            |  133 +-
+ drivers/media/pci/intel/ipu6/ipu6-mmu.h            |  158 +-
+ .../pci/intel/ipu6/ipu6-platform-buttress-regs.h   |  111 +-
+ drivers/media/pci/intel/ipu6/ipu6.c                |  574 +++--
+ drivers/media/pci/intel/ipu6/ipu6.h                |  196 +-
+ drivers/media/pci/intel/ipu6/ipu7-boot.c           |  405 ++++
+ drivers/media/pci/intel/ipu6/ipu7-boot.h           |   46 +
+ drivers/media/pci/intel/ipu6/ipu7-fw-com.c         |   74 +
+ drivers/media/pci/intel/ipu6/ipu7-fw-com.h         |   53 +
+ drivers/media/pci/intel/ipu6/ipu7-fw-isys.c        |  796 +++++++
+ drivers/media/pci/intel/ipu6/ipu7-fw-isys.h        |  296 +++
+ drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.c   | 1072 +++++++++
+ drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.h   |   16 +
+ drivers/media/pci/intel/ipu6/ipu7-isys-csi2-regs.h | 1189 ++++++++++
+ drivers/media/pci/intel/ipu6/ipu7-mmu-hw.c         |  858 +++++++
+ drivers/media/pci/intel/ipu6/ipu7-mmu-hw.h         |  254 +++
+ drivers/media/pci/intel/ipu6/ipu7-platform-regs.h  |   32 +
+ drivers/media/pci/intel/ivsc/mei_ace.c             |    4 +-
+ drivers/media/pci/intel/ivsc/mei_csi.c             |    8 +-
+ drivers/media/pci/ivtv/ivtv-controls.c             |    2 +-
+ drivers/media/pci/ivtv/ivtv-ioctl.c                |    2 +-
+ drivers/media/pci/saa7134/saa7134-core.c           |    8 +-
+ drivers/media/pci/saa7134/saa7134-empress.c        |    4 +-
+ drivers/media/pci/saa7134/saa7134-input.c          |    7 +-
+ drivers/media/pci/saa7146/mxb.c                    |    4 +-
+ drivers/media/pci/saa7164/saa7164-core.c           |   32 +-
+ drivers/media/pci/saa7164/saa7164.h                |    1 -
+ drivers/media/pci/tw68/tw68-core.c                 |   10 +-
+ drivers/media/platform/amd/isp4/isp4_subdev.c      |    1 +
+ drivers/media/platform/amd/isp4/isp4_video.c       |    2 +-
+ .../media/platform/amlogic/c3/isp/c3-isp-core.c    |    1 +
+ .../media/platform/amlogic/c3/isp/c3-isp-resizer.c |    3 +
+ .../amlogic/c3/mipi-adapter/c3-mipi-adap.c         |    1 +
+ .../platform/amlogic/c3/mipi-csi2/c3-mipi-csi2.c   |    1 +
+ drivers/media/platform/arm/mali-c55/mali-c55-isp.c |    3 +
+ .../media/platform/arm/mali-c55/mali-c55-resizer.c |   15 +-
+ drivers/media/platform/arm/mali-c55/mali-c55-tpg.c |    1 +
+ drivers/media/platform/aspeed/aspeed-video.c       |   12 +-
+ drivers/media/platform/atmel/atmel-isi.c           |   11 +-
+ drivers/media/platform/broadcom/bcm2835-unicam.c   |    1 +
+ drivers/media/platform/cadence/cdns-csi2rx.c       |    5 +
+ drivers/media/platform/cadence/cdns-csi2tx.c       |    1 +
+ drivers/media/platform/intel/pxa_camera.c          |    6 +-
+ drivers/media/platform/m2m-deinterlace.c           |    2 +-
+ drivers/media/platform/marvell/Kconfig             |    1 +
+ drivers/media/platform/marvell/cafe-driver.c       |    2 +-
+ drivers/media/platform/marvell/mcam-core.c         |   24 +-
+ drivers/media/platform/marvell/mmp-driver.c        |    5 +-
+ drivers/media/platform/mediatek/mdp/mtk_mdp_ipi.h  |    2 +-
+ .../mediatek/vcodec/decoder/vdec_ipi_msg.h         |    4 +-
+ .../media/platform/microchip/microchip-csi2dc.c    |    2 +
+ .../media/platform/microchip/microchip-isc-base.c  |   82 +-
+ .../media/platform/microchip/microchip-isc-clk.c   |    7 +-
+ .../media/platform/microchip/microchip-isc-regs.h  |   16 +-
+ .../platform/microchip/microchip-isc-scaler.c      |    2 +
+ drivers/media/platform/microchip/microchip-isc.h   |    5 +-
+ .../platform/microchip/microchip-sama5d2-isc.c     |   48 +-
+ .../platform/microchip/microchip-sama7g5-isc.c     |   48 +-
+ drivers/media/platform/nuvoton/npcm-video.c        |   18 +-
+ drivers/media/platform/nxp/Kconfig                 |   16 +
+ drivers/media/platform/nxp/Makefile                |    1 +
+ drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c     |   23 +-
+ drivers/media/platform/nxp/imx-mipi-csis.c         |    3 +-
+ drivers/media/platform/nxp/imx7-media-csi.c        |    1 +
+ .../platform/nxp/imx8-isi/imx8-isi-crossbar.c      |    1 +
+ .../media/platform/nxp/imx8-isi/imx8-isi-pipe.c    |    3 +
+ drivers/media/platform/nxp/imx8mq-mipi-csi2.c      |    1 +
+ drivers/media/platform/nxp/imx95-csi-formatter.c   |  759 +++++++
+ drivers/media/platform/qcom/camss/camss-csid.c     |    3 +-
+ drivers/media/platform/qcom/camss/camss-csiphy.c   |    3 +-
+ drivers/media/platform/qcom/camss/camss-ispif.c    |    3 +-
+ drivers/media/platform/qcom/camss/camss-tpg.c      |    3 +-
+ drivers/media/platform/qcom/camss/camss-vfe.c      |   12 +-
+ drivers/media/platform/qcom/camss/camss.c          |   17 +-
+ drivers/media/platform/raspberrypi/rp1-cfe/csi2.c  |    1 +
+ .../media/platform/raspberrypi/rp1-cfe/pisp-fe.c   |    1 +
+ drivers/media/platform/renesas/rcar-csi2.c         |  385 +++-
+ drivers/media/platform/renesas/rcar-isp/csisp.c    |  228 +-
+ .../media/platform/renesas/rcar-vin/rcar-core.c    |   27 +-
+ drivers/media/platform/renesas/rcar-vin/rcar-dma.c |    2 +-
+ drivers/media/platform/renesas/rcar_drif.c         |    2 +-
+ drivers/media/platform/renesas/renesas-ceu.c       |    7 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-core.c  |    3 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-cru.h   |    2 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-csi2.c  |    3 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-ip.c    |    3 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-video.c |   13 +-
+ .../platform/renesas/rzv2h-ivc/rzv2h-ivc-subdev.c  |    1 +
+ drivers/media/platform/renesas/sh_vou.c            |    6 +-
+ drivers/media/platform/renesas/vsp1/vsp1_brx.c     |    3 +
+ drivers/media/platform/renesas/vsp1/vsp1_dl.c      |    2 +-
+ drivers/media/platform/renesas/vsp1/vsp1_drm.c     |   18 +-
+ drivers/media/platform/renesas/vsp1/vsp1_entity.c  |    4 +-
+ drivers/media/platform/renesas/vsp1/vsp1_entity.h  |    1 +
+ drivers/media/platform/renesas/vsp1/vsp1_histo.c   |    3 +
+ drivers/media/platform/renesas/vsp1/vsp1_hsit.c    |    1 +
+ drivers/media/platform/renesas/vsp1/vsp1_rwpf.c    |    3 +
+ drivers/media/platform/renesas/vsp1/vsp1_sru.c     |    1 +
+ drivers/media/platform/renesas/vsp1/vsp1_uds.c     |    1 +
+ drivers/media/platform/renesas/vsp1/vsp1_uif.c     |    2 +
+ drivers/media/platform/renesas/vsp1/vsp1_vspx.c    |    3 +-
+ .../platform/rockchip/rkcif/rkcif-interface.c      |    3 +
+ .../media/platform/rockchip/rkisp1/rkisp1-csi.c    |    1 +
+ .../media/platform/rockchip/rkisp1/rkisp1-dev.c    |    4 +-
+ .../media/platform/rockchip/rkisp1/rkisp1-isp.c    |    3 +
+ .../platform/rockchip/rkisp1/rkisp1-resizer.c      |    3 +
+ .../platform/samsung/exynos4-is/fimc-capture.c     |    8 +-
+ .../media/platform/samsung/exynos4-is/fimc-isp.c   |    1 +
+ .../media/platform/samsung/exynos4-is/fimc-lite.c  |    3 +
+ .../media/platform/samsung/exynos4-is/mipi-csis.c  |    1 +
+ .../platform/samsung/s3c-camif/camif-capture.c     |    3 +
+ .../media/platform/samsung/s3c-camif/camif-core.c  |    2 +-
+ drivers/media/platform/st/stm32/stm32-csi.c        |    6 +-
+ drivers/media/platform/st/stm32/stm32-dcmi.c       |   31 +-
+ .../media/platform/st/stm32/stm32-dcmipp/Makefile  |    3 +-
+ .../st/stm32/stm32-dcmipp/dcmipp-byteproc.c        |   30 +-
+ .../{dcmipp-bytecap.c => dcmipp-capture.c}         |  600 +++--
+ .../platform/st/stm32/stm32-dcmipp/dcmipp-common.h |   99 +-
+ .../platform/st/stm32/stm32-dcmipp/dcmipp-core.c   |  124 +-
+ .../platform/st/stm32/stm32-dcmipp/dcmipp-input.c  |  127 +-
+ .../platform/st/stm32/stm32-dcmipp/dcmipp-isp.c    |  493 ++++
+ .../st/stm32/stm32-dcmipp/dcmipp-pixelcommon.c     |  181 ++
+ .../st/stm32/stm32-dcmipp/dcmipp-pixelcommon.h     |   42 +
+ .../st/stm32/stm32-dcmipp/dcmipp-pixelproc.c       |  942 ++++++++
+ .../media/platform/sunxi/sun4i-csi/sun4i_v4l2.c    |    1 +
+ .../platform/sunxi/sun6i-csi/sun6i_csi_bridge.c    |  156 +-
+ .../platform/sunxi/sun6i-csi/sun6i_csi_bridge.h    |    9 -
+ .../platform/sunxi/sun6i-csi/sun6i_csi_capture.c   |   27 +-
+ .../sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c        |  108 +-
+ .../sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.h        |    2 -
+ .../sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c    |  113 +-
+ .../sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.h    |    2 -
+ drivers/media/platform/synopsys/dw-mipi-csi2rx.c   |    1 +
+ .../media/platform/synopsys/hdmirx/snps_hdmirx.c   |  423 +++-
+ .../media/platform/synopsys/hdmirx/snps_hdmirx.h   |    8 +
+ drivers/media/platform/ti/Kconfig                  |   11 -
+ drivers/media/platform/ti/am437x/am437x-vpfe.c     |    2 +-
+ drivers/media/platform/ti/cal/cal-camerarx.c       |    1 +
+ drivers/media/platform/ti/cal/cal-video.c          |   19 +-
+ drivers/media/platform/ti/cal/cal.c                |   10 +-
+ drivers/media/platform/ti/davinci/vpif_capture.c   |    5 +-
+ drivers/media/platform/ti/davinci/vpif_display.c   |    3 +
+ .../media/platform/ti/j721e-csi2rx/j721e-csi2rx.c  |   25 +
+ drivers/media/platform/ti/omap3isp/ispccdc.c       |    5 +-
+ drivers/media/platform/ti/omap3isp/ispccp2.c       |    3 +-
+ drivers/media/platform/ti/omap3isp/ispcsi2.c       |    3 +-
+ drivers/media/platform/ti/omap3isp/isppreview.c    |    5 +-
+ drivers/media/platform/ti/omap3isp/ispresizer.c    |    5 +-
+ drivers/media/platform/ti/omap3isp/ispvideo.c      |    4 +-
+ drivers/media/platform/ti/vpe/vip.c                |    4 +-
+ drivers/media/platform/via/via-camera.c            |    4 +-
+ drivers/media/platform/video-mux.c                 |    1 +
+ drivers/media/platform/xilinx/xilinx-csi2rxss.c    |    1 +
+ drivers/media/platform/xilinx/xilinx-tpg.c         |    1 +
+ drivers/media/radio/si4713/radio-usb-si4713.c      |    4 +-
+ drivers/media/radio/si4713/si4713.c                |    2 +-
+ drivers/media/rc/bpf-lirc.c                        |   18 +-
+ drivers/media/rc/ene_ir.c                          |    4 +-
+ drivers/media/rc/fintek-cir.c                      |   13 -
+ drivers/media/rc/fintek-cir.h                      |   22 -
+ drivers/media/rc/imon.c                            |  175 +-
+ drivers/media/rc/ir-hix5hd2.c                      |    5 +-
+ drivers/media/rc/ir-mce_kbd-decoder.c              |    5 +-
+ drivers/media/rc/ir_toy.c                          |    4 +-
+ drivers/media/rc/ite-cir.c                         |    1 -
+ drivers/media/rc/ite-cir.h                         |    1 -
+ drivers/media/rc/lirc_dev.c                        |    5 +
+ drivers/media/rc/mceusb.c                          |    1 -
+ drivers/media/rc/meson-ir-tx.c                     |   20 +-
+ drivers/media/rc/nuvoton-cir.h                     |    3 -
+ drivers/media/rc/rc-ir-raw.c                       |   86 +-
+ drivers/media/rc/rc-loopback.c                     |    5 -
+ drivers/media/rc/rc-main.c                         |  206 +-
+ drivers/media/rc/redrat3.c                         |   32 +-
+ drivers/media/rc/serial_ir.c                       |    2 +-
+ drivers/media/rc/streamzap.c                       |    1 +
+ drivers/media/rc/sunxi-cir.c                       |    2 +-
+ drivers/media/spi/Kconfig                          |    4 -
+ drivers/media/test-drivers/vicodec/codec-fwht.h    |    2 +-
+ drivers/media/test-drivers/vicodec/vicodec-core.c  |   14 +-
+ drivers/media/test-drivers/vidtv/vidtv_bridge.c    |   75 +-
+ drivers/media/test-drivers/vidtv/vidtv_demod.c     |    9 -
+ drivers/media/test-drivers/vim2m.c                 |   33 +-
+ drivers/media/test-drivers/vimc/vimc-debayer.c     |    1 +
+ drivers/media/test-drivers/vimc/vimc-scaler.c      |    3 +
+ drivers/media/test-drivers/vimc/vimc-sensor.c      |    9 +-
+ drivers/media/test-drivers/vivid/vivid-cec.c       |   10 +-
+ drivers/media/test-drivers/vivid/vivid-core.c      |    1 +
+ drivers/media/test-drivers/vivid/vivid-vid-cap.c   |   13 +
+ drivers/media/tuners/tda18250.c                    |    3 +-
+ drivers/media/usb/au0828/au0828-core.c             |    4 +
+ drivers/media/usb/au0828/au0828-dvb.c              |   20 +-
+ drivers/media/usb/cx231xx/cx231xx-417.c            |    2 +-
+ drivers/media/usb/cx231xx/cx231xx-audio.c          |   15 +-
+ drivers/media/usb/cx231xx/cx231xx-cards.c          |    2 +-
+ drivers/media/usb/cx231xx/cx231xx-video.c          |    4 +-
+ drivers/media/usb/cx231xx/cx231xx.h                |    2 +-
+ drivers/media/usb/dvb-usb-v2/mxl111sf-i2c.c        |    4 +-
+ drivers/media/usb/dvb-usb/cxusb-analog.c           |    6 +-
+ drivers/media/usb/dvb-usb/dib0700_core.c           |    2 +-
+ drivers/media/usb/dvb-usb/dvb-usb-firmware.c       |    2 +
+ drivers/media/usb/em28xx/em28xx-camera.c           |    2 +-
+ drivers/media/usb/em28xx/em28xx-video.c            |    4 +
+ drivers/media/usb/go7007/go7007-driver.c           |    5 +-
+ drivers/media/usb/go7007/go7007-usb.c              |    8 +
+ drivers/media/usb/go7007/go7007-v4l2.c             |    2 +-
+ drivers/media/usb/go7007/s2250-board.c             |    1 +
+ drivers/media/usb/gspca/gspca.c                    |    3 +-
+ drivers/media/usb/gspca/ov519.c                    |    2 +-
+ drivers/media/usb/gspca/w996Xcf.c                  |    2 +-
+ drivers/media/usb/hackrf/hackrf.c                  |    7 +-
+ drivers/media/usb/pvrusb2/pvrusb2-hdw.c            |   14 +-
+ drivers/media/usb/usbtv/usbtv-core.c               |    2 -
+ drivers/media/usb/usbtv/usbtv-video.c              |    4 +
+ drivers/media/v4l2-core/v4l2-common.c              |   21 +-
+ drivers/media/v4l2-core/v4l2-ctrls-core.c          |   19 +-
+ drivers/media/v4l2-core/v4l2-mc.c                  |    5 +-
+ drivers/media/v4l2-core/v4l2-subdev.c              |  163 +-
+ drivers/platform/x86/intel/int3472/discrete.c      |   81 +-
+ drivers/platform/x86/intel/int3472/tps68470.c      |    2 +-
+ drivers/staging/media/atomisp/i2c/atomisp-gc2235.c |    1 +
+ drivers/staging/media/atomisp/i2c/atomisp-ov2722.c |    1 +
+ drivers/staging/media/atomisp/pci/atomisp_cmd.c    |   16 +-
+ drivers/staging/media/atomisp/pci/atomisp_csi2.c   |    1 +
+ drivers/staging/media/atomisp/pci/atomisp_subdev.c |    3 +
+ drivers/staging/media/atomisp/pci/atomisp_v4l2.c   |    8 +-
+ .../pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h |    4 +-
+ drivers/staging/media/av7110/av7110.c              |    2 +-
+ drivers/staging/media/av7110/av7110_ir.c           |    2 -
+ drivers/staging/media/av7110/sp8870.c              |    9 +-
+ drivers/staging/media/imx/imx-ic-prp.c             |    2 +-
+ drivers/staging/media/imx/imx-ic-prpencvf.c        |    4 +-
+ drivers/staging/media/imx/imx-media-capture.c      |    1 -
+ drivers/staging/media/imx/imx-media-csc-scaler.c   |    3 +-
+ drivers/staging/media/imx/imx-media-csi.c          |    3 +
+ drivers/staging/media/imx/imx-media-dev.c          |    1 -
+ drivers/staging/media/imx/imx-media-vdic.c         |    1 +
+ drivers/staging/media/imx/imx6-mipi-csi2.c         |    1 +
+ drivers/staging/media/ipu3/ipu3-css.c              |    1 -
+ drivers/staging/media/ipu3/ipu3-v4l2.c             |    3 +
+ drivers/staging/media/ipu7/TODO                    |   28 +-
+ drivers/staging/media/ipu7/ipu7-isys-csi2.c        |    2 +
+ drivers/staging/media/ipu7/ipu7-isys-subdev.c      |    1 +
+ drivers/staging/media/ipu7/ipu7-isys-subdev.h      |    1 +
+ drivers/staging/media/ipu7/ipu7-isys.c             |    1 +
+ drivers/staging/media/ipu7/ipu7.c                  |    7 +
+ drivers/staging/media/max96712/max96712.c          |    1 -
+ .../staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c |    1 +
+ drivers/staging/media/tegra-video/csi.c            |    1 +
+ drivers/staging/media/tegra-video/vi.c             |  102 +-
+ .../dt-bindings/media/video-interface-devices.h    |   13 +
+ include/linux/platform_data/x86/int3472.h          |    2 +
+ include/linux/property.h                           |    5 +
+ include/media/ipu-bridge.h                         |   52 +-
+ include/media/ipu6-pci-table.h                     |    4 +
+ include/media/rc-map.h                             |    2 -
+ include/media/v4l2-common.h                        |   75 +-
+ include/media/v4l2-ctrls.h                         |    6 +
+ include/media/v4l2-dv-timings.h                    |   14 +
+ include/media/v4l2-subdev.h                        |   34 +-
+ include/uapi/linux/it6625.h                        |   25 +
+ include/uapi/linux/media/st/dcmipp_config.h        |   16 +
+ include/uapi/linux/v4l2-controls.h                 |   21 +
+ 499 files changed, 20234 insertions(+), 4318 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/media/fsl,imx95-csi-formatter.yaml
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/himax,hm1246.yaml
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/ite,it6625.yaml
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/ovti,os02g10.yaml
+ delete mode 100644 Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.txt
+ create mode 100644 Documentation/devicetree/bindings/media/i2c/toshiba,tc358743.yaml
+ create mode 100644 Documentation/userspace-api/media/drivers/dcmipp.rst
+ create mode 100644 drivers/media/i2c/hm1246.c
+ create mode 100644 drivers/media/i2c/it6625.c
+ create mode 100644 drivers/media/i2c/os02g10.c
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-boot.c
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-boot.h
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-fw-com.c
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-fw-com.h
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-fw-isys.h
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.c
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.h
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-isys-csi2-regs.h
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-mmu-hw.c
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-mmu-hw.h
+ create mode 100644 drivers/media/pci/intel/ipu6/ipu7-platform-regs.h
+ create mode 100644 drivers/media/platform/nxp/imx95-csi-formatter.c
+ rename drivers/media/platform/st/stm32/stm32-dcmipp/{dcmipp-bytecap.c => dcmipp-capture.c} (53%)
+ create mode 100644 drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-isp.c
+ create mode 100644 drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.c
+ create mode 100644 drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.h
+ create mode 100644 drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelproc.c
+ create mode 100644 include/dt-bindings/media/video-interface-devices.h
+ create mode 100644 include/uapi/linux/it6625.h
+ create mode 100644 include/uapi/linux/media/st/dcmipp_config.h
+Merging v4l-dvb-next/master (adc218676eef2 Linux 6.12)
+$ git merge -m Merge branch 'master' of git://linuxtv.org/mchehab/media-next.git v4l-dvb-next/master
+Already up to date.
+Merging pm/linux-next (114920a3e8292 Merge branch 'thermal-drivers' into linux-next)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git pm/linux-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging arch/arm64/kernel/topology.c
+Auto-merging sound/soc/intel/boards/bytcr_rt5651.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-devices-system-cpu |   14 +-
+ Documentation/admin-guide/kernel-parameters.txt    |    7 -
+ Documentation/admin-guide/pm/amd-pstate.rst        |    6 -
+ Documentation/admin-guide/pm/cpuidle.rst           |    2 +-
+ .../driver-api/thermal/cpu-idle-cooling.rst        |    2 +-
+ Documentation/firmware-guide/acpi/apei/einj.rst    |    6 +-
+ Documentation/power/runtime_pm.rst                 |  379 +---
+ Documentation/power/userland-swsusp.rst            |    4 +-
+ arch/arm64/kernel/topology.c                       |   14 +-
+ arch/x86/kernel/acpi/cppc.c                        |   14 +
+ drivers/acpi/acpi_extlog.c                         |   64 +-
+ drivers/acpi/acpi_mrrm.c                           |   14 +-
+ drivers/acpi/acpi_pcc.c                            |   55 +-
+ drivers/acpi/acpi_video.c                          |   43 +-
+ drivers/acpi/acpica/dsfield.c                      |    3 +-
+ drivers/acpi/acpica/exfield.c                      |   11 +-
+ drivers/acpi/apei/ghes.c                           |   92 +-
+ drivers/acpi/apei/ghes_helpers.c                   |   18 +-
+ drivers/acpi/arm64/amba.c                          |    3 +-
+ drivers/acpi/battery.c                             |  131 +-
+ drivers/acpi/bus.c                                 |    8 +-
+ drivers/acpi/button.c                              |   12 +
+ drivers/acpi/cppc_acpi.c                           | 2223 +++++++++++++++++---
+ drivers/acpi/device_pm.c                           |   77 +-
+ drivers/acpi/fan.h                                 |    2 +-
+ drivers/acpi/fan_core.c                            |  147 +-
+ drivers/acpi/fan_hwmon.c                           |    2 +-
+ drivers/acpi/glue.c                                |  145 +-
+ drivers/acpi/internal.h                            |    1 +
+ drivers/acpi/numa/hmat.c                           |    2 +-
+ drivers/acpi/numa/srat.c                           |    2 +-
+ drivers/acpi/osl.c                                 |    2 +-
+ drivers/acpi/pfr_update.c                          |    2 +-
+ drivers/acpi/power.c                               |   13 +
+ drivers/acpi/processor_driver.c                    |    6 +-
+ drivers/acpi/processor_thermal.c                   |   80 +-
+ drivers/acpi/riscv/cppc.c                          |   26 +-
+ drivers/acpi/sbs.c                                 |    8 +-
+ drivers/acpi/scan.c                                |   54 +-
+ drivers/acpi/sysfs.c                               |    5 +-
+ drivers/acpi/tables.c                              |    3 +
+ drivers/acpi/thermal.c                             |   19 +-
+ drivers/acpi/utils.c                               |   13 +-
+ drivers/acpi/video_detect.c                        |    8 +
+ drivers/acpi/x86/s2idle.c                          |   29 +
+ drivers/base/base.h                                |    2 +
+ drivers/base/bus.c                                 |   67 +-
+ drivers/base/power/clock_ops.c                     |    2 +-
+ drivers/base/power/main.c                          |   31 +-
+ drivers/base/power/runtime-test.c                  |   57 +
+ drivers/base/power/runtime.c                       |   99 +-
+ drivers/clocksource/timer-ti-dm.c                  |    9 +-
+ drivers/cpufreq/amd-pstate-ut.c                    |   23 +-
+ drivers/cpufreq/amd-pstate.c                       |  257 ++-
+ drivers/cpufreq/amd-pstate.h                       |   16 +
+ drivers/cpufreq/cppc_cpufreq.c                     |   73 +-
+ drivers/cpufreq/cpufreq.c                          |    4 +-
+ drivers/cpufreq/cpufreq_conservative.c             |    8 +-
+ drivers/cpufreq/cpufreq_governor.c                 |   29 +-
+ drivers/cpufreq/cpufreq_governor.h                 |    2 +
+ drivers/cpufreq/intel_pstate.c                     |   10 +
+ drivers/cpuidle/cpuidle-tegra.c                    |    2 +-
+ drivers/cpuidle/governors/menu.c                   |    8 +-
+ drivers/cpuidle/governors/teo.c                    |   11 +
+ drivers/cxl/core/ras.c                             |    3 +-
+ drivers/firmware/efi/cper.c                        |   51 +-
+ drivers/idle/intel_idle.c                          |   12 +-
+ drivers/platform/x86/lenovo/yogabook.c             |    4 +-
+ drivers/platform/x86/serdev_helpers.h              |    3 +-
+ drivers/platform/x86/x86-android-tablets/core.c    |    4 +-
+ drivers/pnp/driver.c                               |    4 +-
+ drivers/powercap/intel_rapl_msr.c                  |    5 +-
+ drivers/thermal/cpufreq_cooling.c                  |   14 +-
+ drivers/thermal/devfreq_cooling.c                  |    4 +-
+ drivers/thermal/gov_power_allocator.c              |    7 +-
+ .../int340x_thermal/processor_thermal_device.c     |   14 +-
+ .../int340x_thermal/processor_thermal_soc_slider.c |   28 +-
+ drivers/thermal/intel/intel_powerclamp.c           |   64 +-
+ drivers/thermal/intel/intel_tcc_cooling.c          |    1 +
+ drivers/thermal/qcom/qcom-spmi-adc-tm5.c           |    4 +-
+ drivers/thermal/thermal_core.c                     |   30 +-
+ drivers/thermal/thermal_core.h                     |    3 +-
+ drivers/thermal/thermal_of.c                       |   17 +-
+ drivers/thermal/ti-soc-thermal/ti-bandgap.c        |   12 +-
+ drivers/thunderbolt/acpi.c                         |    5 +-
+ include/acpi/acpi_bus.h                            |   43 +-
+ include/acpi/cppc_acpi.h                           |   18 +-
+ include/acpi/ghes.h                                |    4 +
+ include/acpi/processor.h                           |    6 +-
+ include/cxl/event.h                                |    6 +-
+ include/linux/acpi.h                               |   30 +-
+ include/linux/apple-gmux.h                         |    2 +-
+ include/linux/cpufreq.h                            |    3 +
+ include/linux/device/bus.h                         |    1 +
+ include/linux/pm.h                                 |   93 +
+ include/linux/pm_runtime.h                         |  409 ++--
+ include/linux/thermal.h                            |   19 +-
+ include/uapi/linux/thermal.h                       |    8 +-
+ kernel/cpu_pm.c                                    |    9 +-
+ kernel/power/em_netlink.c                          |  126 +-
+ kernel/power/em_netlink.h                          |    8 +-
+ kernel/power/energy_model.c                        |    8 +-
+ sound/hda/codecs/side-codecs/aw88399_hda.c         |    3 +-
+ sound/soc/amd/acp-es8336.c                         |    2 +-
+ sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c      |    2 +-
+ sound/soc/intel/boards/bytcht_es8316.c             |    4 +-
+ sound/soc/intel/boards/bytcr_rt5640.c              |    4 +-
+ sound/soc/intel/boards/bytcr_rt5651.c              |    4 +-
+ sound/soc/intel/boards/cht_bsw_rt5645.c            |    5 +-
+ sound/soc/intel/boards/sof_cirrus_common.c         |    2 +-
+ sound/soc/intel/boards/sof_es8336.c                |    4 +-
+ sound/soc/loongson/loongson_card.c                 |    3 +-
+ tools/power/pm-graph/sleepgraph.py                 |    2 +-
+ 113 files changed, 3834 insertions(+), 1764 deletions(-)
+Merging cpufreq-arm/cpufreq/arm/linux-next (d82d896f00e7b cpufreq: Use %pe to print error pointers symbolically)
+$ git merge -m Merge branch 'cpufreq/arm/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq-arm/cpufreq/arm/linux-next
+Auto-merging drivers/cpufreq/cppc_cpufreq.c
+CONFLICT (content): Merge conflict in drivers/cpufreq/cppc_cpufreq.c
+Resolved 'drivers/cpufreq/cppc_cpufreq.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 05f0c876ba59d] Merge branch 'cpufreq/arm/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/cpufreq/airoha-cpufreq.c     |  2 +-
+ drivers/cpufreq/bmips-cpufreq.c      |  4 ++--
+ drivers/cpufreq/cppc_cpufreq.c       |  4 ++--
+ drivers/cpufreq/qoriq-cpufreq.c      |  3 +--
+ drivers/cpufreq/rcpufreq_dt.rs       |  1 +
+ drivers/cpufreq/s3c64xx-cpufreq.c    |  5 ++---
+ drivers/cpufreq/sparc-us2e-cpufreq.c | 11 +++++------
+ drivers/cpufreq/sti-cpufreq.c        |  4 ++--
+ drivers/cpufreq/tegra194-cpufreq.c   |  2 +-
+ rust/kernel/cpufreq.rs               | 18 ++++++++++++++----
+ 10 files changed, 31 insertions(+), 23 deletions(-)
+Merging cpupower/cpupower (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'cpupower' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git cpupower/cpupower
+Already up to date.
+Merging devfreq/devfreq-next (9a222650d9e70 PM / devfreq: Fix governor_store() failing when device has no current governor)
+$ git merge -m Merge branch 'devfreq-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git devfreq/devfreq-next
+Auto-merging drivers/devfreq/event/rockchip-dfi.c
+Merge made by the 'ort' strategy.
+ drivers/devfreq/devfreq.c            | 50 +++++++-----------------------------
+ drivers/devfreq/event/rockchip-dfi.c |  4 ++-
+ 2 files changed, 12 insertions(+), 42 deletions(-)
+Merging pmdomain/next (d1a93cf1d3bed pmdomain: Merge branch dt into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/power/qcom,rpmpd.yaml      |   1 +
+ MAINTAINERS                                        |   8 +
+ drivers/cpuidle/cpuidle-psci-domain.c              |   7 +-
+ drivers/cpuidle/cpuidle-psci.c                     |   2 +-
+ drivers/pmdomain/Kconfig                           |   1 +
+ drivers/pmdomain/Makefile                          |   1 +
+ drivers/pmdomain/core.c                            | 163 ++++++-
+ drivers/pmdomain/core.h                            |  18 +
+ drivers/pmdomain/governor.c                        |  45 ++
+ drivers/pmdomain/imx/gpcv2.c                       |  16 +-
+ drivers/pmdomain/imx/imx8m-blk-ctrl.c              |  15 +-
+ drivers/pmdomain/imx/imx93-blk-ctrl.c              |   3 +-
+ drivers/pmdomain/imx/scu-pd.c                      |  10 +-
+ drivers/pmdomain/qcom/rpmhpd.c                     |  41 ++
+ drivers/pmdomain/renesas/Kconfig                   |   4 +
+ drivers/pmdomain/renesas/Makefile                  |   1 +
+ drivers/pmdomain/renesas/r8a774a3-sysc.c           |  45 ++
+ drivers/pmdomain/renesas/r8a7795-sysc.c            |   2 +-
+ drivers/pmdomain/renesas/r8a779a0-sysc.c           |   2 +-
+ drivers/pmdomain/renesas/r8a779f0-sysc.c           |   2 +-
+ drivers/pmdomain/renesas/r8a779g0-sysc.c           |   2 +-
+ drivers/pmdomain/renesas/r8a779h0-sysc.c           |   2 +-
+ drivers/pmdomain/renesas/r8a78000-mdlc.c           |   6 +-
+ drivers/pmdomain/renesas/rcar-sysc.c               |   3 +
+ drivers/pmdomain/renesas/rcar-sysc.h               |  13 +-
+ drivers/pmdomain/riscv/Kconfig                     |  15 +
+ drivers/pmdomain/riscv/Makefile                    |   3 +
+ drivers/pmdomain/riscv/riscv-rpmi-device-power.c   | 485 +++++++++++++++++++++
+ drivers/pmdomain/rockchip/pm-domains.c             |   4 +-
+ include/linux/mailbox/riscv-rpmi-message.h         |  11 +
+ include/linux/pm_domain.h                          |   9 +
+ include/linux/pm_qos.h                             |   9 +
+ kernel/power/qos.c                                 |   4 +-
+ 33 files changed, 886 insertions(+), 67 deletions(-)
+ create mode 100644 drivers/pmdomain/core.h
+ create mode 100644 drivers/pmdomain/renesas/r8a774a3-sysc.c
+ create mode 100644 drivers/pmdomain/riscv/Kconfig
+ create mode 100644 drivers/pmdomain/riscv/Makefile
+ create mode 100644 drivers/pmdomain/riscv/riscv-rpmi-device-power.c
+Merging opp/opp/linux-next (1f6de65e33145 opp: fix use after free in _update_opp_table_clk())
+$ git merge -m Merge branch 'opp/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git opp/opp/linux-next
+Already up to date.
+Merging thermal/thermal/linux-next (e856ca3013b30 thermal/drivers/renesas/rzg3s: Add RZ/G3L TSU support)
+$ git merge -m Merge branch 'thermal/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git thermal/thermal/linux-next
+Merge made by the 'ort' strategy.
+ .../bindings/thermal/qcom,pm8775-mbg-tm.yaml       |   7 +-
+ .../bindings/thermal/renesas,r9a08g045-tsu.yaml    |   4 +-
+ .../bindings/thermal/ti,omap-bandgap.yaml          | 177 +++++++++++++++++++++
+ .../devicetree/bindings/thermal/ti_soc_thermal.txt |  88 ----------
+ drivers/thermal/airoha_thermal.c                   |  24 +--
+ drivers/thermal/imx91_thermal.c                    |   2 +-
+ drivers/thermal/qcom/tsens-v2.c                    |   4 +-
+ drivers/thermal/renesas/rzg3e_thermal.c            |  35 ++--
+ drivers/thermal/renesas/rzg3s_thermal.c            |  38 +++--
+ tools/thermal/tmon/sysfs.c                         |   2 +-
+ tools/thermal/tmon/tmon.h                          |   2 +-
+ tools/thermal/tmon/tui.c                           |   2 +-
+ 12 files changed, 258 insertions(+), 127 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/thermal/ti,omap-bandgap.yaml
+ delete mode 100644 Documentation/devicetree/bindings/thermal/ti_soc_thermal.txt
+Merging rdma/for-next (485effd117d03 RDMA/ionic: Fix double removal of CMB mmap entries in create QP error path)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/infiniband/core/nldev.c
+Auto-merging drivers/infiniband/core/verbs.c
+Auto-merging drivers/infiniband/hw/bnxt_re/main.c
+Auto-merging drivers/infiniband/hw/erdma/erdma_main.c
+Auto-merging drivers/infiniband/hw/erdma/erdma_verbs.c
+Auto-merging drivers/infiniband/hw/hns/hns_roce_debugfs.c
+Auto-merging drivers/infiniband/hw/irdma/verbs.c
+Auto-merging drivers/infiniband/hw/mlx5/main.c
+Auto-merging drivers/infiniband/sw/rxe/rxe_mr.c
+Auto-merging drivers/infiniband/sw/rxe/rxe_odp.c
+Auto-merging drivers/infiniband/sw/rxe/rxe_verbs.c
+CONFLICT (content): Merge conflict in drivers/infiniband/sw/rxe/rxe_verbs.c
+Auto-merging drivers/infiniband/ulp/isert/ib_isert.c
+Auto-merging drivers/infiniband/ulp/rtrs/rtrs-clt.c
+Auto-merging drivers/net/ethernet/microsoft/mana/gdma_main.c
+Recorded preimage for 'drivers/infiniband/sw/rxe/rxe_verbs.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'drivers/infiniband/sw/rxe/rxe_verbs.c'.
+[master e20a0522c63b9] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/stable/sysfs-class-infiniband   | 118 -----
+ MAINTAINERS                                       |   2 +-
+ drivers/infiniband/core/cma.c                     |  34 +-
+ drivers/infiniband/core/cma_trace.h               |   1 +
+ drivers/infiniband/core/device.c                  |   4 +-
+ drivers/infiniband/core/iwcm.c                    |   2 +-
+ drivers/infiniband/core/lag.c                     |   2 +-
+ drivers/infiniband/core/mad_rmpp.c                |   7 +-
+ drivers/infiniband/core/multicast.c               |   7 +-
+ drivers/infiniband/core/nldev.c                   |   9 +-
+ drivers/infiniband/core/restrack.c                |   3 +-
+ drivers/infiniband/core/umem.c                    |  77 ++-
+ drivers/infiniband/core/uverbs_std_types_device.c |   1 +
+ drivers/infiniband/core/uverbs_std_types_wq.c     |   2 +-
+ drivers/infiniband/core/uverbs_uapi.c             |   2 +-
+ drivers/infiniband/core/verbs.c                   |   2 +-
+ drivers/infiniband/hw/bnxt_re/bnxt_re.h           |   4 +
+ drivers/infiniband/hw/bnxt_re/hw_counters.c       |   6 +-
+ drivers/infiniband/hw/bnxt_re/ib_verbs.c          |  69 ++-
+ drivers/infiniband/hw/bnxt_re/main.c              |  26 +-
+ drivers/infiniband/hw/bnxt_re/qplib_res.c         |   9 +-
+ drivers/infiniband/hw/bnxt_re/qplib_sp.c          |   2 +-
+ drivers/infiniband/hw/cxgb4/cm.c                  |   5 +-
+ drivers/infiniband/hw/cxgb4/cq.c                  |   2 +-
+ drivers/infiniband/hw/efa/efa_admin_cmds_defs.h   |   5 +-
+ drivers/infiniband/hw/efa/efa_com_cmd.c           |   9 +-
+ drivers/infiniband/hw/efa/efa_com_cmd.h           |   8 +-
+ drivers/infiniband/hw/efa/efa_verbs.c             |   5 +-
+ drivers/infiniband/hw/erdma/erdma_cq.c            |  18 +-
+ drivers/infiniband/hw/erdma/erdma_main.c          |   2 +-
+ drivers/infiniband/hw/erdma/erdma_qp.c            |  38 +-
+ drivers/infiniband/hw/erdma/erdma_verbs.c         | 583 ++++++++++++----------
+ drivers/infiniband/hw/erdma/erdma_verbs.h         |  66 ++-
+ drivers/infiniband/hw/hfi1/affinity.c             |   2 +-
+ drivers/infiniband/hw/hfi1/firmware.c             |   2 +-
+ drivers/infiniband/hw/hfi1/pcie.c                 |  14 +-
+ drivers/infiniband/hw/hns/hns_roce_bond.c         |   4 +-
+ drivers/infiniband/hw/hns/hns_roce_cq.c           |   3 +-
+ drivers/infiniband/hw/hns/hns_roce_debugfs.c      |  55 ++
+ drivers/infiniband/hw/hns/hns_roce_debugfs.h      |   1 +
+ drivers/infiniband/hw/hns/hns_roce_device.h       |   6 +-
+ drivers/infiniband/hw/hns/hns_roce_hw_v2.c        |  63 ++-
+ drivers/infiniband/hw/hns/hns_roce_mr.c           |  22 +-
+ drivers/infiniband/hw/hns/hns_roce_qp.c           |   2 +-
+ drivers/infiniband/hw/hns/hns_roce_srq.c          |   4 +-
+ drivers/infiniband/hw/ionic/ionic_controlpath.c   |  41 +-
+ drivers/infiniband/hw/ionic/ionic_fw.h            |   2 +
+ drivers/infiniband/hw/ionic/ionic_ibdev.c         |   4 +-
+ drivers/infiniband/hw/ionic/ionic_lif_cfg.c       |   1 +
+ drivers/infiniband/hw/ionic/ionic_lif_cfg.h       |   1 +
+ drivers/infiniband/hw/irdma/ctrl.c                |  23 +-
+ drivers/infiniband/hw/irdma/hw.c                  |  15 +-
+ drivers/infiniband/hw/irdma/icrdma_if.c           |   9 +-
+ drivers/infiniband/hw/irdma/main.c                |   2 +
+ drivers/infiniband/hw/irdma/pble.c                |   8 +-
+ drivers/infiniband/hw/irdma/protos.h              |   3 +-
+ drivers/infiniband/hw/irdma/type.h                |   7 +-
+ drivers/infiniband/hw/irdma/verbs.c               |   7 +-
+ drivers/infiniband/hw/mana/cq.c                   | 396 +++++++++++----
+ drivers/infiniband/hw/mana/device.c               |   3 +-
+ drivers/infiniband/hw/mana/main.c                 |  43 +-
+ drivers/infiniband/hw/mana/mana_ib.h              | 134 ++++-
+ drivers/infiniband/hw/mana/qp.c                   | 150 ++++--
+ drivers/infiniband/hw/mana/shadow_queue.h         |  61 +--
+ drivers/infiniband/hw/mana/wq.c                   |   3 +-
+ drivers/infiniband/hw/mana/wr.c                   | 170 ++++---
+ drivers/infiniband/hw/mlx4/cq.c                   |  17 +-
+ drivers/infiniband/hw/mlx4/mcg.c                  |   4 +-
+ drivers/infiniband/hw/mlx5/cq.c                   |   9 +-
+ drivers/infiniband/hw/mlx5/data_direct.c          |  10 +-
+ drivers/infiniband/hw/mlx5/fs.c                   |  45 +-
+ drivers/infiniband/hw/mlx5/main.c                 |  54 +-
+ drivers/infiniband/hw/mlx5/odp.c                  |   2 +-
+ drivers/infiniband/hw/mlx5/qp.c                   |  32 +-
+ drivers/infiniband/hw/mlx5/qpc.c                  |  20 +-
+ drivers/infiniband/hw/mlx5/srq.c                  |   3 +-
+ drivers/infiniband/hw/mlx5/wr.c                   |   1 +
+ drivers/infiniband/hw/mthca/mthca_cq.c            |   3 +-
+ drivers/infiniband/hw/ocrdma/ocrdma_verbs.c       |   6 +-
+ drivers/infiniband/hw/qedr/verbs.c                |  25 +-
+ drivers/infiniband/hw/usnic/usnic_abi.h           |   2 +-
+ drivers/infiniband/hw/usnic/usnic_ib_verbs.c      |   2 +-
+ drivers/infiniband/hw/usnic/usnic_transport.h     |   2 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c      |   6 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c     |   3 +-
+ drivers/infiniband/sw/rdmavt/qp.c                 |   2 +-
+ drivers/infiniband/sw/rdmavt/srq.c                |   2 +-
+ drivers/infiniband/sw/rxe/rxe_loc.h               |   1 +
+ drivers/infiniband/sw/rxe/rxe_mr.c                |   6 +
+ drivers/infiniband/sw/rxe/rxe_net.c               |  71 ++-
+ drivers/infiniband/sw/rxe/rxe_odp.c               |   4 +-
+ drivers/infiniband/sw/rxe/rxe_req.c               |   2 +-
+ drivers/infiniband/sw/rxe/rxe_resp.c              |   6 +-
+ drivers/infiniband/sw/rxe/rxe_verbs.c             |   6 +
+ drivers/infiniband/sw/siw/siw_main.c              |   5 +-
+ drivers/infiniband/sw/siw/siw_qp_tx.c             |   1 -
+ drivers/infiniband/ulp/ipoib/ipoib_vlan.c         |   2 +-
+ drivers/infiniband/ulp/iser/iscsi_iser.c          |   6 +-
+ drivers/infiniband/ulp/iser/iser_verbs.c          |   2 +-
+ drivers/infiniband/ulp/isert/ib_isert.c           |   2 +-
+ drivers/infiniband/ulp/rtrs/rtrs-clt.c            |  27 +-
+ drivers/infiniband/ulp/srpt/ib_srpt.c             |   2 +-
+ drivers/infiniband/ulp/srpt/ib_srpt.h             |   2 +-
+ drivers/net/ethernet/microsoft/mana/gdma_main.c   |  51 +-
+ include/net/mana/gdma.h                           |  12 +-
+ include/rdma/ib_umem.h                            |   2 +
+ include/rdma/ib_verbs.h                           |   4 +-
+ include/rdma/uverbs_ioctl.h                       |   2 +-
+ include/uapi/rdma/ionic-abi.h                     |   9 +-
+ include/uapi/rdma/mana-abi.h                      |  12 +
+ 110 files changed, 1854 insertions(+), 1024 deletions(-)
+Merging net-next/main (47a1446725732 Merge branch 'mptcp-misc-improvements-for-v7-4')
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git net-next/main
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Auto-merging drivers/net/ethernet/broadcom/bcmsysport.c
+Auto-merging drivers/net/ethernet/broadcom/genet/bcmgenet.c
+Auto-merging drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+Auto-merging drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+Auto-merging drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+Auto-merging drivers/net/wireless/ath/ath11k/Kconfig
+Auto-merging drivers/net/wireless/ath/ath12k/pci.c
+Auto-merging include/linux/skbuff.h
+Auto-merging net/core/dev.c
+Auto-merging net/core/skbuff.c
+Auto-merging net/core/sock.c
+Auto-merging net/ipv4/af_inet.c
+Auto-merging net/ipv4/tcp.c
+Auto-merging net/ipv4/tcp_input.c
+Auto-merging net/ipv6/addrconf.c
+Auto-merging net/mac80211/ieee80211_i.h
+CONFLICT (content): Merge conflict in net/mac80211/ieee80211_i.h
+Auto-merging net/mac80211/iface.c
+Auto-merging net/mac80211/mesh_pathtbl.c
+Auto-merging net/mac80211/mlme.c
+Auto-merging net/mac80211/rx.c
+Auto-merging net/mac80211/tx.c
+CONFLICT (content): Merge conflict in net/mac80211/tx.c
+Auto-merging net/packet/af_packet.c
+Auto-merging net/socket.c
+Auto-merging net/unix/af_unix.c
+Auto-merging net/wireless/nl80211.c
+Auto-merging rust/kernel/net/netlink.rs
+Auto-merging tools/testing/selftests/net/Makefile
+Resolved 'net/mac80211/ieee80211_i.h' using previous resolution.
+Resolved 'net/mac80211/tx.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 88d6478ff3a01] Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git
+$ git diff -M --stat --summary HEAD^..
+ .mailmap                                           |   13 +-
+ .../ABI/testing/sysfs-devices-platform-soc-ipa     |   14 -
+ Documentation/ABI/testing/sysfs-timecard           |   37 +-
+ Documentation/admin-guide/sysctl/net.rst           |   14 +
+ .../bindings/net/allwinner,sun8i-a83t-emac.yaml    |   13 +
+ .../bindings/net/altr,socfpga-stmmac.yaml          |    2 -
+ .../devicetree/bindings/net/brcm,asp-v2.0.yaml     |    2 +-
+ .../bindings/net/cortina,gemini-ethernet.yaml      |   52 +-
+ .../devicetree/bindings/net/dsa/microchip,ksz.yaml |    1 +
+ .../bindings/net/dsa/motorcomm,yt921x.yaml         |   23 +
+ .../devicetree/bindings/net/dsa/realtek.yaml       |   33 +
+ .../bindings/net/dsa/renesas,rzn1-a5psw.yaml       |    4 +-
+ .../devicetree/bindings/net/ethernet-phy.yaml      |   16 +
+ .../devicetree/bindings/net/fsl,fman-dtsec.yaml    |    4 -
+ .../devicetree/bindings/net/intel,dwmac-plat.yaml  |    6 +-
+ .../devicetree/bindings/net/mdio-gpio.yaml         |   16 +-
+ .../devicetree/bindings/net/microchip,lan8650.yaml |    5 +
+ .../bindings/net/mscc,vsc7514-switch.yaml          |    8 +-
+ .../bindings/net/nvidia,tegra234-mgbe.yaml         |    8 +-
+ .../devicetree/bindings/net/qcom,ipa.yaml          |   10 +-
+ .../devicetree/bindings/net/realtek,rtl82xx.yaml   |    6 +-
+ .../bindings/net/realtek,rtl9301-mdio.yaml         |   14 +-
+ .../devicetree/bindings/net/renesas,etheravb.yaml  |   11 +
+ .../devicetree/bindings/net/snps,dwmac-common.yaml |   66 +
+ .../devicetree/bindings/net/snps,dwmac.yaml        |   59 +-
+ .../bindings/net/socionext,uniphier-ave4.yaml      |   32 +-
+ .../devicetree/bindings/net/ti,cpsw-switch.yaml    |   70 +-
+ .../bindings/net/ultrarisc,dp1000-gmac.yaml        |   74 ++
+ .../bindings/net/wireless/marvell,sd8787.yaml      |    6 +
+ .../bindings/net/wireless/mediatek,mt76.yaml       |   32 +-
+ .../bindings/net/wireless/qcom,ath10k.yaml         |    2 +-
+ .../bindings/net/wireless/qcom,ath12k-wsi.yaml     |    7 -
+ .../bindings/net/wireless/silabs,wfx.yaml          |    3 +
+ .../devicetree/bindings/net/wireless/ti,wl1251.txt |   64 -
+ .../bindings/net/wireless/ti,wl1251.yaml           |   85 ++
+ .../devicetree/bindings/net/wiznet,w5100.yaml      |   88 ++
+ .../devicetree/bindings/net/wiznet,w5x00.txt       |   50 -
+ .../bindings/net/x-powers,acx00-ephy-package.yaml  |  170 +++
+ Documentation/netlink/netlink-raw.yaml             |   31 +-
+ Documentation/netlink/specs/conntrack.yaml         |   60 +-
+ Documentation/netlink/specs/devlink.yaml           |  171 ++-
+ Documentation/netlink/specs/dpll.yaml              |   22 +-
+ Documentation/netlink/specs/ethtool.yaml           |    4 +-
+ Documentation/netlink/specs/fou.yaml               |    4 +
+ Documentation/netlink/specs/handshake.yaml         |    9 +-
+ Documentation/netlink/specs/netdev.yaml            |    5 +-
+ Documentation/netlink/specs/nl80211.yaml           |    4 +-
+ Documentation/netlink/specs/nlctrl.yaml            |   20 +-
+ Documentation/netlink/specs/psp.yaml               |    2 +
+ Documentation/netlink/specs/rt-link.yaml           |   44 +-
+ Documentation/netlink/specs/tc.yaml                |    2 +-
+ Documentation/netlink/specs/tcp_metrics.yaml       |   18 +-
+ .../networking/device_drivers/ethernet/index.rst   |    1 +
+ .../device_drivers/ethernet/intel/i40e.rst         |    2 +-
+ .../device_drivers/ethernet/intel/iavf.rst         |    2 +-
+ .../device_drivers/ethernet/microsoft/netvsc.rst   |    2 -
+ .../device_drivers/ethernet/nebula-matrix/nbl.rst  |   28 +
+ .../device_drivers/ethernet/stmicro/stmmac.rst     |    1 -
+ Documentation/networking/devlink/index.rst         |    1 +
+ Documentation/networking/devlink/ptp_ocp.rst       |   75 ++
+ Documentation/networking/dsa/dsa.rst               |    2 +-
+ Documentation/networking/ethtool-netlink.rst       |    2 +-
+ Documentation/networking/ip-sysctl.rst             |   28 +-
+ Documentation/networking/mctp.rst                  |    4 +-
+ .../networking/net_cachelines/net_device.rst       |    1 +
+ Documentation/networking/netconsole.rst            |    5 +
+ Documentation/networking/netdevices.rst            |   21 +-
+ Documentation/networking/nf_conntrack-sysctl.rst   |    2 +-
+ Documentation/networking/page_pool.rst             |    2 +-
+ Documentation/networking/phonet.rst                |    2 +-
+ Documentation/networking/phy.rst                   |   17 +-
+ Documentation/networking/proc_net_tcp.rst          |   20 +-
+ Documentation/networking/radiotap-headers.rst      |    2 +-
+ Documentation/networking/scaling.rst               |    9 +
+ MAINTAINERS                                        |   68 +-
+ drivers/dibs/dibs_loopback.c                       |    5 +-
+ drivers/dpll/dpll_netlink.c                        |    2 +-
+ drivers/dpll/dpll_nl.c                             |   20 +-
+ drivers/infiniband/ulp/ipoib/ipoib_main.c          |   27 +-
+ drivers/net/bonding/bond_alb.c                     |   63 +-
+ drivers/net/can/bxcan.c                            |    2 +-
+ drivers/net/can/c_can/c_can_platform.c             |    4 +-
+ drivers/net/can/flexcan/flexcan-core.c             |    2 +-
+ drivers/net/can/ifi_canfd/ifi_canfd.c              |    2 +-
+ drivers/net/can/m_can/m_can_platform.c             |    2 +-
+ drivers/net/can/sja1000/f81601.c                   |    2 +-
+ drivers/net/can/sja1000/sja1000_platform.c         |    2 +-
+ drivers/net/can/slcan/slcan-core.c                 |    5 +-
+ drivers/net/dsa/Kconfig                            |   19 +-
+ drivers/net/dsa/Makefile                           |    3 +-
+ drivers/net/dsa/b53/b53_mdio.c                     |    2 +-
+ drivers/net/dsa/b53/b53_srab.c                     |    2 +-
+ drivers/net/dsa/bcm_sf2.c                          |    2 +-
+ drivers/net/dsa/hirschmann/hellcreek.c             |    2 +-
+ drivers/net/dsa/ks8995.c                           |  857 ------------
+ drivers/net/dsa/lan9303_i2c.c                      |    2 +-
+ drivers/net/dsa/lan9303_mdio.c                     |    2 +-
+ drivers/net/dsa/lantiq/lantiq_gswip_common.c       |    9 +-
+ drivers/net/dsa/lantiq/mxl-gsw1xx.c                |   31 +-
+ drivers/net/dsa/microchip/Kconfig                  |    1 +
+ drivers/net/dsa/microchip/ksz8.c                   |  252 +++-
+ drivers/net/dsa/microchip/ksz8.h                   |    2 +
+ drivers/net/dsa/microchip/ksz8_reg.h               |    2 +
+ drivers/net/dsa/microchip/ksz_common.c             |  114 +-
+ drivers/net/dsa/microchip/ksz_common.h             |   16 +-
+ drivers/net/dsa/microchip/ksz_dcb.c                |   68 +-
+ drivers/net/dsa/microchip/ksz_spi.c                |   33 +-
+ drivers/net/dsa/motorcomm/Kconfig                  |   17 +
+ drivers/net/dsa/motorcomm/Makefile                 |    7 +
+ drivers/net/dsa/{yt921x.c => motorcomm/chip.c}     |  752 ++---------
+ drivers/net/dsa/{yt921x.h => motorcomm/chip.h}     |  119 +-
+ drivers/net/dsa/motorcomm/leds.c                   |  641 +++++++++
+ drivers/net/dsa/motorcomm/leds.h                   |  118 ++
+ drivers/net/dsa/motorcomm/mdio_bus.c               |  301 +++++
+ drivers/net/dsa/motorcomm/mdio_bus.h               |   54 +
+ drivers/net/dsa/motorcomm/pcs-921x.c               |  247 ++++
+ drivers/net/dsa/motorcomm/pcs.h                    |   13 +
+ drivers/net/dsa/motorcomm/smi.c                    |  180 +++
+ drivers/net/dsa/motorcomm/smi.h                    |   61 +
+ drivers/net/dsa/mt7530-mdio.c                      |   14 +-
+ drivers/net/dsa/mt7530-mmio.c                      |    2 +-
+ drivers/net/dsa/mt7530.c                           |  834 ++++++------
+ drivers/net/dsa/mt7530.h                           |  221 ++--
+ drivers/net/dsa/mv88e6060.c                        |    3 +-
+ drivers/net/dsa/mv88e6xxx/chip.c                   |    9 +-
+ drivers/net/dsa/qca/qca8k-8xxx.c                   |    2 +-
+ drivers/net/dsa/qca/qca8k-common.c                 |   15 +-
+ drivers/net/dsa/realtek/realtek.h                  |    7 +
+ drivers/net/dsa/realtek/rtl8365mb_main.c           |  145 +-
+ drivers/net/dsa/realtek/rtl8366rb.c                |    2 +-
+ drivers/net/dsa/realtek/rtl83xx.c                  |   43 +-
+ drivers/net/dsa/rzn1_a5psw.c                       |    2 +-
+ drivers/net/dsa/sja1105/sja1105_main.c             |    2 +-
+ drivers/net/dsa/vitesse-vsc73xx-spi.c              |    9 +-
+ drivers/net/ethernet/8390/pcnet_cs.c               |    9 +-
+ drivers/net/ethernet/Kconfig                       |    1 +
+ drivers/net/ethernet/Makefile                      |    1 +
+ drivers/net/ethernet/adaptec/starfire.c            |    6 +-
+ drivers/net/ethernet/adi/adin1140.c                |    4 +-
+ drivers/net/ethernet/airoha/airoha_eth.c           |  363 ++++-
+ drivers/net/ethernet/airoha/airoha_eth.h           |   29 +-
+ drivers/net/ethernet/airoha/airoha_regs.h          |   23 +-
+ drivers/net/ethernet/alacritech/slicoss.c          |    1 +
+ drivers/net/ethernet/amd/xgbe/xgbe-mdio.c          |    5 +-
+ drivers/net/ethernet/aquantia/atlantic/aq_nic.c    |    5 +-
+ .../net/ethernet/aquantia/atlantic/aq_pci_func.c   |    2 -
+ drivers/net/ethernet/atheros/alx/main.c            |   13 +-
+ drivers/net/ethernet/atheros/atl1c/atl1c_hw.h      |    2 +-
+ drivers/net/ethernet/atheros/atl1e/atl1e.h         |    2 +-
+ drivers/net/ethernet/atheros/atl1e/atl1e_hw.c      |    2 +-
+ drivers/net/ethernet/atheros/atl1e/atl1e_main.c    |    2 +-
+ drivers/net/ethernet/atheros/atlx/atl1.c           |    2 +-
+ drivers/net/ethernet/atheros/atlx/atl2.c           |    2 +-
+ drivers/net/ethernet/broadcom/asp2/bcmasp.c        |    4 +-
+ drivers/net/ethernet/broadcom/asp2/bcmasp.h        |    5 -
+ drivers/net/ethernet/broadcom/bcm63xx_enet.c       |    2 +-
+ drivers/net/ethernet/broadcom/bcm63xx_enet.h       |    2 +-
+ drivers/net/ethernet/broadcom/bcmsysport.c         |   25 +-
+ drivers/net/ethernet/broadcom/bcmsysport.h         |    2 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x.h        |    4 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_dcb.c    |    4 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h    |    2 +-
+ .../net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h   |    2 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.c   |    8 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c   |    2 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h    |    4 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c  |   18 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt.c          |    4 +-
+ drivers/net/ethernet/broadcom/genet/bcmgenet.c     |   11 +-
+ drivers/net/ethernet/cadence/macb.h                |    7 +-
+ drivers/net/ethernet/cadence/macb_main.c           |  172 ++-
+ drivers/net/ethernet/cadence/macb_ptp.c            |   62 +-
+ .../net/ethernet/cavium/liquidio/cn66xx_device.c   |    2 +-
+ drivers/net/ethernet/cavium/liquidio/lio_core.c    |    2 +-
+ .../net/ethernet/cavium/liquidio/octeon_config.h   |    4 +-
+ drivers/net/ethernet/cavium/liquidio/octeon_iq.h   |    2 +-
+ drivers/net/ethernet/cavium/liquidio/octeon_main.h |    2 +-
+ drivers/net/ethernet/cavium/liquidio/octeon_nic.h  |    2 +-
+ .../net/ethernet/cavium/liquidio/request_manager.c |    2 +-
+ drivers/net/ethernet/cavium/thunder/nic.h          |    2 +-
+ drivers/net/ethernet/cavium/thunder/nic_main.c     |    2 +-
+ drivers/net/ethernet/cisco/enic/enic_main.c        |   19 +-
+ drivers/net/ethernet/cortina/gemini.c              |   32 +-
+ drivers/net/ethernet/cortina/gemini.h              |    2 +-
+ drivers/net/ethernet/davicom/dm9000.c              |    6 +-
+ drivers/net/ethernet/dec/tulip/de2104x.c           |    2 +-
+ drivers/net/ethernet/dec/tulip/interrupt.c         |    4 +-
+ drivers/net/ethernet/dec/tulip/media.c             |    2 +-
+ drivers/net/ethernet/dec/tulip/tulip.h             |    4 +-
+ drivers/net/ethernet/dec/tulip/winbond-840.c       |    2 +-
+ drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c   |    2 +
+ drivers/net/ethernet/freescale/enetc/Kconfig       |    1 +
+ drivers/net/ethernet/freescale/enetc/enetc.c       |   68 +-
+ drivers/net/ethernet/freescale/enetc/enetc.h       |   11 +-
+ .../net/ethernet/freescale/enetc/enetc4_debugfs.c  |   51 +-
+ drivers/net/ethernet/freescale/enetc/enetc4_hw.h   |    1 +
+ drivers/net/ethernet/freescale/enetc/enetc4_pf.c   |   85 +-
+ .../net/ethernet/freescale/enetc/enetc_ethtool.c   |    6 +
+ drivers/net/ethernet/freescale/enetc/enetc_hw.h    |   26 +
+ .../net/ethernet/freescale/enetc/enetc_mailbox.h   |  109 +-
+ drivers/net/ethernet/freescale/enetc/enetc_mdio.c  |    2 +-
+ drivers/net/ethernet/freescale/enetc/enetc_msg.c   |  641 ++++++++-
+ drivers/net/ethernet/freescale/enetc/enetc_pf.c    |   64 +-
+ drivers/net/ethernet/freescale/enetc/enetc_pf.h    |   21 +-
+ .../net/ethernet/freescale/enetc/enetc_pf_common.c |  150 ++-
+ .../net/ethernet/freescale/enetc/enetc_pf_common.h |   19 +
+ drivers/net/ethernet/freescale/enetc/enetc_vf.c    |  453 ++++++-
+ drivers/net/ethernet/freescale/fec.h               |    2 +-
+ drivers/net/ethernet/freescale/fec_main.c          |   18 +-
+ drivers/net/ethernet/freescale/fec_ptp.c           |   24 +-
+ .../net/ethernet/freescale/fs_enet/fs_enet-main.c  |    1 +
+ .../net/ethernet/hisilicon/hibmcge/hbg_common.h    |    2 +-
+ drivers/net/ethernet/hisilicon/hibmcge/hbg_hw.c    |    4 +-
+ drivers/net/ethernet/hisilicon/hibmcge/hbg_irq.c   |    4 +-
+ drivers/net/ethernet/hisilicon/hibmcge/hbg_reg.h   |    4 +-
+ drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c  |   12 +-
+ drivers/net/ethernet/hisilicon/hip04_eth.c         |    5 +-
+ drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c |    2 +-
+ drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c  |    2 +-
+ drivers/net/ethernet/hisilicon/hns/hns_enet.c      |    4 +-
+ drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c |   10 +-
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.c    |   11 +-
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c |    7 +-
+ .../net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c  |    2 +-
+ .../ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c  |    2 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c  |    7 +-
+ drivers/net/ethernet/intel/fm10k/fm10k_pci.c       |    2 -
+ drivers/net/ethernet/intel/i40e/i40e_common.c      |    2 +-
+ drivers/net/ethernet/intel/i40e/i40e_ethtool.c     |   12 +-
+ drivers/net/ethernet/intel/i40e/i40e_main.c        |   70 +-
+ drivers/net/ethernet/intel/i40e/i40e_txrx.c        |    5 +-
+ drivers/net/ethernet/intel/i40e/i40e_txrx.h        |    7 +-
+ drivers/net/ethernet/intel/i40e/i40e_type.h        |    5 +
+ drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c |    5 +-
+ drivers/net/ethernet/intel/i40e/i40e_xsk.c         |   12 +
+ drivers/net/ethernet/intel/ice/Makefile            |    2 +-
+ drivers/net/ethernet/intel/ice/ice.h               |    9 +-
+ drivers/net/ethernet/intel/ice/ice_arfs.c          |    8 +-
+ drivers/net/ethernet/intel/ice/ice_arfs.h          |    2 +-
+ drivers/net/ethernet/intel/ice/ice_dpll.c          |  495 ++++++-
+ drivers/net/ethernet/intel/ice/ice_dpll.h          |    4 +
+ drivers/net/ethernet/intel/ice/ice_ethtool.c       |   14 +-
+ .../{ice_ethtool_fdir.c => ice_ethtool_ntuple.c}   |   88 +-
+ drivers/net/ethernet/intel/ice/ice_fdir.c          |   38 +-
+ drivers/net/ethernet/intel/ice/ice_fdir.h          |   14 +-
+ drivers/net/ethernet/intel/ice/ice_main.c          |   11 +-
+ drivers/net/ethernet/intel/ice/ice_ptp.c           |   82 ++
+ drivers/net/ethernet/intel/ice/ice_ptp.h           |   11 +
+ drivers/net/ethernet/intel/ice/ice_sriov.c         |    2 +-
+ drivers/net/ethernet/intel/ice/ice_tspll.c         |  120 +-
+ drivers/net/ethernet/intel/ice/ice_tspll.h         |    6 +
+ drivers/net/ethernet/intel/ice/ice_txclk.c         |   20 +-
+ drivers/net/ethernet/intel/ice/ice_type.h          |    4 +-
+ drivers/net/ethernet/intel/ice/virt/fdir.c         |   28 +-
+ drivers/net/ethernet/intel/idpf/idpf.h             |   12 +
+ drivers/net/ethernet/intel/idpf/idpf_lib.c         |   19 +
+ drivers/net/ethernet/intel/idpf/idpf_txrx.c        |   71 +-
+ drivers/net/ethernet/intel/idpf/idpf_txrx.h        |    8 +-
+ drivers/net/ethernet/intel/idpf/idpf_virtchnl.c    |   60 +-
+ drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c  |    2 +-
+ drivers/net/ethernet/marvell/mvmdio.c              |    4 +-
+ drivers/net/ethernet/marvell/mvneta.c              |    2 +-
+ drivers/net/ethernet/marvell/mvpp2/mvpp2.h         |    2 +-
+ drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c    |    2 +-
+ drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c     |    8 +-
+ .../ethernet/marvell/octeon_ep/octep_pfvf_mbox.c   |    1 -
+ drivers/net/ethernet/marvell/octeontx2/af/cgx.c    |   25 +-
+ drivers/net/ethernet/marvell/octeontx2/af/cgx.h    |    2 +
+ .../net/ethernet/marvell/octeontx2/af/cn20k/npa.c  |    1 -
+ .../net/ethernet/marvell/octeontx2/af/cn20k/npc.c  |    5 +-
+ .../ethernet/marvell/octeontx2/af/lmac_common.h    |    2 +
+ .../net/ethernet/marvell/octeontx2/af/mcs_reg.h    |    2 -
+ drivers/net/ethernet/marvell/octeontx2/af/npc.h    |    4 +-
+ drivers/net/ethernet/marvell/octeontx2/af/rpm.c    |   40 +-
+ drivers/net/ethernet/marvell/octeontx2/af/rpm.h    |    2 +
+ drivers/net/ethernet/marvell/octeontx2/af/rvu.c    |    4 +-
+ drivers/net/ethernet/marvell/octeontx2/af/rvu.h    |    5 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_cgx.c    |   12 +
+ .../ethernet/marvell/octeontx2/af/rvu_debugfs.c    |    8 +
+ .../net/ethernet/marvell/octeontx2/af/rvu_nix.c    |   33 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_npa.c    |   10 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_npc.c    |    5 +-
+ .../net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c |    1 -
+ drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c |    2 +-
+ .../ethernet/marvell/octeontx2/nic/otx2_common.c   |    8 +-
+ .../ethernet/marvell/octeontx2/nic/otx2_common.h   |    1 -
+ .../ethernet/marvell/octeontx2/nic/otx2_dcbnl.c    |    7 +-
+ .../ethernet/marvell/octeontx2/nic/otx2_ethtool.c  |   18 +
+ .../ethernet/marvell/octeontx2/nic/otx2_flows.c    |    3 -
+ .../net/ethernet/marvell/octeontx2/nic/otx2_pf.c   |  101 +-
+ .../net/ethernet/marvell/octeontx2/nic/otx2_reg.h  |    2 +
+ .../net/ethernet/marvell/octeontx2/nic/otx2_tc.c   |    1 -
+ .../net/ethernet/marvell/octeontx2/nic/otx2_vf.c   |    9 +-
+ drivers/net/ethernet/marvell/octeontx2/nic/qos.h   |    2 -
+ .../ethernet/marvell/prestera/prestera_router.c    |   10 +-
+ drivers/net/ethernet/marvell/pxa168_eth.c          |    4 +-
+ drivers/net/ethernet/mellanox/mlx4/main.c          |    8 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c   |    2 +-
+ .../net/ethernet/mellanox/mlx5/core/en/rep/neigh.c |   29 +-
+ .../net/ethernet/mellanox/mlx5/core/en/tc_priv.h   |   14 +-
+ .../ethernet/mellanox/mlx5/core/en/tc_tun_encap.c  |   23 +-
+ .../ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c  |   11 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c   |    2 +-
+ .../ethernet/mellanox/mlx5/core/en_accel/ipsec.c   |    6 +-
+ .../net/ethernet/mellanox/mlx5/core/en_ethtool.c   |    6 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |    3 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_rx.c    |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_tc.c    |   75 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_tx.c    |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.c  |   20 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |    2 +-
+ .../ethernet/mellanox/mlx5/core/eswitch_offloads.c |   37 +-
+ .../net/ethernet/mellanox/mlx5/core/fpga/conn.c    |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c |    6 +-
+ .../net/ethernet/mellanox/mlx5/core/lag/debugfs.c  |   14 +-
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c  |  154 ++-
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h  |    2 +-
+ .../ethernet/mellanox/mlx5/core/lag/shared_fdb.c   |    3 +-
+ drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c  |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |    8 +-
+ .../ethernet/mellanox/mlx5/core/steering/hws/bwc.c |    4 +-
+ .../mellanox/mlx5/core/steering/hws/send.c         |   34 +-
+ drivers/net/ethernet/mellanox/mlxsw/pci.c          |    7 +-
+ .../ethernet/mellanox/mlxsw/spectrum_nve_vxlan.c   |   19 +-
+ .../net/ethernet/mellanox/mlxsw/spectrum_router.c  |   31 +-
+ .../net/ethernet/mellanox/mlxsw/spectrum_span.c    |   10 +-
+ .../ethernet/mellanox/mlxsw/spectrum_switchdev.c   |   57 +-
+ drivers/net/ethernet/meta/Kconfig                  |   13 +
+ drivers/net/ethernet/meta/Makefile                 |    1 +
+ drivers/net/ethernet/meta/fbnic/fbnic.h            |    2 +
+ drivers/net/ethernet/meta/fbnic/fbnic_devlink.c    |    2 +
+ drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c    |   15 +-
+ drivers/net/ethernet/meta/fbnic/fbnic_fw.c         |    9 +-
+ drivers/net/ethernet/meta/fbnic/fbnic_mac.c        |    9 +-
+ drivers/net/ethernet/meta/fbnic/fbnic_pci.c        |    2 -
+ drivers/net/ethernet/meta/fbnic/fbnic_rpc.c        |    8 +-
+ drivers/net/ethernet/meta/fbnic/fbnic_tlv.c        |   13 +-
+ drivers/net/ethernet/meta/fbnic/fbnic_txrx.c       |   18 +
+ drivers/net/ethernet/meta/fbnic/fbnic_txrx.h       |   10 +
+ drivers/net/ethernet/meta/mpnic/Makefile           |   16 +
+ drivers/net/ethernet/meta/mpnic/mpnic.h            |   66 +
+ drivers/net/ethernet/meta/mpnic/mpnic_csr.h        |  319 +++++
+ drivers/net/ethernet/meta/mpnic/mpnic_init.c       |  556 ++++++++
+ drivers/net/ethernet/meta/mpnic/mpnic_irq.c        |   64 +
+ drivers/net/ethernet/meta/mpnic/mpnic_netdev.c     |  186 +++
+ drivers/net/ethernet/meta/mpnic/mpnic_netdev.h     |   35 +
+ drivers/net/ethernet/meta/mpnic/mpnic_pci.c        |  187 +++
+ drivers/net/ethernet/meta/mpnic/mpnic_txrx.c       | 1395 ++++++++++++++++++++
+ drivers/net/ethernet/meta/mpnic/mpnic_txrx.h       |  151 +++
+ drivers/net/ethernet/microchip/lan865x/lan865x.c   |    4 +-
+ drivers/net/ethernet/microchip/vcap/Kconfig        |    1 -
+ drivers/net/ethernet/microsoft/mana/hw_channel.c   |   24 +-
+ drivers/net/ethernet/microsoft/mana/mana_bpf.c     |    3 -
+ drivers/net/ethernet/nebula-matrix/Kconfig         |   32 +
+ drivers/net/ethernet/nebula-matrix/Makefile        |    6 +
+ drivers/net/ethernet/nebula-matrix/nbl/Makefile    |    7 +
+ drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h  |   31 +
+ .../nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c       |  153 +++
+ .../nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h       |   15 +
+ .../ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h |   31 +
+ .../nebula-matrix/nbl/nbl_include/nbl_def_common.h |   31 +
+ .../nebula-matrix/nbl/nbl_include/nbl_def_hw.h     |   17 +
+ .../nebula-matrix/nbl/nbl_include/nbl_include.h    |   23 +
+ drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c  |  192 +++
+ .../ethernet/netronome/nfp/flower/tunnel_conf.c    |   14 +-
+ drivers/net/ethernet/oa_tc6.c                      |   46 +-
+ .../net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c   |   10 +-
+ .../net/ethernet/pensando/ionic/ionic_bus_pci.c    |    1 +
+ .../net/ethernet/pensando/ionic/ionic_ethtool.c    |   10 +-
+ drivers/net/ethernet/pensando/ionic/ionic_if.h     |   31 +-
+ .../net/ethernet/qlogic/netxen/netxen_nic_ctx.c    |    2 +-
+ drivers/net/ethernet/qlogic/qed/qed_cxt.c          |    2 +-
+ drivers/net/ethernet/qlogic/qed/qed_mcp.h          |    2 +-
+ drivers/net/ethernet/qlogic/qede/qede_ethtool.c    |    6 +-
+ drivers/net/ethernet/qlogic/qla3xxx.c              |   19 +-
+ drivers/net/ethernet/qlogic/qlcnic/qlcnic.h        |    2 +-
+ .../net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c  |    4 +-
+ drivers/net/ethernet/qlogic/qlcnic/qlcnic_ctx.c    |    2 +-
+ drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.c    |    5 +-
+ drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c   |    4 +-
+ .../ethernet/qlogic/qlcnic/qlcnic_sriov_common.c   |    6 +-
+ .../net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c   |    6 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c |   75 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h |    2 +-
+ .../net/ethernet/qualcomm/rmnet/rmnet_handlers.c   |   34 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h    |    9 +-
+ .../ethernet/qualcomm/rmnet/rmnet_map_command.c    |    9 +-
+ .../net/ethernet/qualcomm/rmnet/rmnet_map_data.c   |   14 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c    |   13 +-
+ drivers/net/ethernet/realtek/Kconfig               |    2 +-
+ drivers/net/ethernet/realtek/r8169_main.c          |  706 ++++++++--
+ drivers/net/ethernet/renesas/ravb.h                |   35 +-
+ drivers/net/ethernet/renesas/ravb_main.c           |  253 ++--
+ drivers/net/ethernet/renesas/ravb_ptp.c            |   38 +-
+ drivers/net/ethernet/renesas/rswitch_main.c        |    7 +-
+ drivers/net/ethernet/rocker/rocker_main.c          |    2 +-
+ drivers/net/ethernet/rocker/rocker_ofdpa.c         |    2 +-
+ drivers/net/ethernet/sfc/Kconfig                   |    2 +-
+ drivers/net/ethernet/sfc/tc_counters.c             |    8 +-
+ drivers/net/ethernet/sfc/tc_encap_actions.c        |    4 +-
+ drivers/net/ethernet/smsc/Kconfig                  |    3 +-
+ drivers/net/ethernet/smsc/smc9194.h                |  241 ----
+ drivers/net/ethernet/spacemit/k1_emac.c            |    4 +-
+ drivers/net/ethernet/stmicro/stmmac/Kconfig        |   11 +
+ drivers/net/ethernet/stmicro/stmmac/Makefile       |    1 +
+ drivers/net/ethernet/stmicro/stmmac/common.h       |    5 +
+ .../net/ethernet/stmicro/stmmac/dwmac-eic7700.c    |   13 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c |   10 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c  |   77 +-
+ .../net/ethernet/stmicro/stmmac/dwmac-ultrarisc.c  |   52 +
+ drivers/net/ethernet/stmicro/stmmac/dwmac4.h       |    6 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c  |   79 +-
+ drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c   |    2 +
+ .../net/ethernet/stmicro/stmmac/dwxgmac2_core.c    |   18 -
+ drivers/net/ethernet/stmicro/stmmac/hwif.h         |    3 -
+ drivers/net/ethernet/stmicro/stmmac/stmmac.h       |    7 +
+ .../net/ethernet/stmicro/stmmac/stmmac_ethtool.c   |    3 +-
+ drivers/net/ethernet/stmicro/stmmac/stmmac_main.c  |  176 ++-
+ .../net/ethernet/stmicro/stmmac/stmmac_platform.c  |   17 +-
+ .../net/ethernet/stmicro/stmmac/stmmac_selftests.c |  112 --
+ drivers/net/ethernet/sun/niu.c                     |   10 +-
+ drivers/net/ethernet/ti/am65-cpsw-nuss.c           |    2 +-
+ drivers/net/ethernet/ti/cpsw.c                     |    2 +-
+ drivers/net/ethernet/ti/cpsw_new.c                 |    2 +-
+ drivers/net/ethernet/ti/davinci_mdio.c             |    4 +-
+ drivers/net/ethernet/wangxun/libwx/wx_hw.c         |    2 +-
+ drivers/net/ethernet/wangxun/libwx/wx_ptp.c        |    5 +-
+ drivers/net/ethernet/wangxun/libwx/wx_type.h       |    6 +-
+ drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c     |    6 +-
+ drivers/net/ethernet/wiznet/w5100.c                |  234 +++-
+ drivers/net/ethernet/xilinx/ll_temac_main.c        |    7 +-
+ drivers/net/ethernet/xilinx/xilinx_axienet_main.c  |   12 +-
+ drivers/net/ethernet/xscale/ptp_ixp46x.c           |    1 +
+ drivers/net/fddi/skfp/cfm.c                        |    2 +-
+ drivers/net/fddi/skfp/ecm.c                        |    2 +-
+ drivers/net/fddi/skfp/fplustm.c                    |    4 +-
+ drivers/net/fddi/skfp/h/hwmtm.h                    |    2 +-
+ drivers/net/fddi/skfp/h/sba.h                      |    2 +-
+ drivers/net/fddi/skfp/pcmplc.c                     |    2 +-
+ drivers/net/fddi/skfp/rmt.c                        |    2 +-
+ drivers/net/gtp.c                                  |    2 +-
+ drivers/net/hyperv/netvsc.c                        |   10 +-
+ drivers/net/ieee802154/at86rf230.c                 |    2 +-
+ drivers/net/macsec.c                               |   32 +
+ drivers/net/mdio/Kconfig                           |    4 +-
+ drivers/net/mdio/mdio-bcm-iproc.c                  |    2 +-
+ drivers/net/mdio/mdio-bcm-unimac.c                 |    2 +-
+ drivers/net/mdio/mdio-mux-meson-g12a.c             |    2 +-
+ drivers/net/mdio/mdio-realtek-rtl9300.c            |  426 +++++-
+ drivers/net/netconsole.c                           |  132 +-
+ drivers/net/netdevsim/bus.c                        |    7 +-
+ drivers/net/netdevsim/ethtool.c                    |   53 +
+ drivers/net/netdevsim/ipsec.c                      |   10 +-
+ drivers/net/netdevsim/netdev.c                     |   10 +-
+ drivers/net/netdevsim/netdevsim.h                  |    8 +-
+ drivers/net/netdevsim/psp.c                        |   33 -
+ drivers/net/netkit.c                               |   17 +-
+ drivers/net/pcs/pcs-lynx.c                         |   59 +-
+ drivers/net/pcs/pcs-xpcs-plat.c                    |    2 +-
+ drivers/net/phy/Kconfig                            |   11 +
+ drivers/net/phy/Makefile                           |    3 +-
+ drivers/net/phy/air_en8811h.c                      |   84 +-
+ drivers/net/phy/broadcom.c                         |    7 +
+ drivers/net/phy/dp83848.c                          |    3 +
+ drivers/net/phy/dp83867.c                          |  109 +-
+ drivers/net/phy/mediatek/mtk-phy-lib.c             |   26 +-
+ drivers/net/phy/microchip_t1.c                     |   16 +-
+ drivers/net/phy/motorcomm.c                        |   84 +-
+ drivers/net/phy/mxl-gpy.c                          |    2 +-
+ drivers/net/phy/nxp-c45-tja11xx.c                  |    2 +-
+ drivers/net/phy/phy_device.c                       |  360 +++--
+ drivers/net/phy/phy_fixup.c                        |   98 ++
+ drivers/net/phy/phy_led_triggers.c                 |   11 +-
+ drivers/net/phy/phylib-internal.h                  |    1 +
+ drivers/net/phy/phylink.c                          |  176 ++-
+ drivers/net/phy/qcom/qca808x.c                     |   15 +-
+ drivers/net/phy/realtek/realtek_main.c             |  103 +-
+ drivers/net/phy/sfp.c                              |   21 +-
+ drivers/net/phy/xpowers/Makefile                   |    3 +
+ drivers/net/phy/xpowers/ac200.c                    |  314 +++++
+ drivers/net/phy/xpowers/ac300.c                    |  387 ++++++
+ drivers/net/phy/xpowers/acx00.c                    |  633 +++++++++
+ drivers/net/phy/xpowers/acx00.h                    |   28 +
+ drivers/net/ppp/ppp_synctty.c                      |   20 +-
+ drivers/net/usb/ax88179_178a.c                     |    9 +-
+ drivers/net/usb/ch9200.c                           |   14 +-
+ drivers/net/usb/qmi_wwan.c                         |    1 +
+ drivers/net/usb/r8152.c                            |   51 +-
+ drivers/net/virtio_net.c                           |   33 +-
+ drivers/net/vrf.c                                  |    2 +-
+ drivers/net/vxlan/vxlan_core.c                     |  780 ++++++-----
+ drivers/net/vxlan/vxlan_mdb.c                      |   45 +-
+ drivers/net/vxlan/vxlan_multicast.c                |   94 +-
+ drivers/net/vxlan/vxlan_private.h                  |   34 +-
+ drivers/net/vxlan/vxlan_vnifilter.c                |  206 ++-
+ drivers/net/wan/slic_ds26522.c                     |    2 +-
+ drivers/net/wireless/ath/ath10k/debug.c            |   10 +-
+ drivers/net/wireless/ath/ath10k/snoc.c             |    6 +-
+ drivers/net/wireless/ath/ath11k/Kconfig            |    3 +
+ drivers/net/wireless/ath/ath11k/ahb.c              |   16 +-
+ drivers/net/wireless/ath/ath11k/ce.c               |    2 +-
+ drivers/net/wireless/ath/ath11k/core.c             |   10 +
+ drivers/net/wireless/ath/ath11k/dp.c               |    2 +
+ drivers/net/wireless/ath/ath11k/dp_rx.c            |   21 +-
+ drivers/net/wireless/ath/ath11k/htc.c              |    5 +-
+ drivers/net/wireless/ath/ath12k/ahb.c              |  166 ++-
+ drivers/net/wireless/ath/ath12k/ahb.h              |    9 +
+ drivers/net/wireless/ath/ath12k/ce.c               |    2 +-
+ drivers/net/wireless/ath/ath12k/core.c             |   23 +
+ drivers/net/wireless/ath/ath12k/dp.c               |    2 +-
+ drivers/net/wireless/ath/ath12k/dp_mon.c           |    2 -
+ drivers/net/wireless/ath/ath12k/dp_rx.c            |   20 +-
+ drivers/net/wireless/ath/ath12k/htc.c              |    5 +-
+ drivers/net/wireless/ath/ath12k/hw.h               |   20 +-
+ drivers/net/wireless/ath/ath12k/mac.c              |   45 +-
+ drivers/net/wireless/ath/ath12k/pci.c              |   55 +-
+ drivers/net/wireless/ath/ath12k/qmi.c              |    4 +
+ drivers/net/wireless/ath/ath12k/wifi7/ahb.c        |    2 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c     |   23 +-
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c      |   20 +-
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h      |   12 +-
+ drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c      |   17 +-
+ drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c     |    2 +-
+ drivers/net/wireless/ath/ath12k/wifi7/hw.c         |    9 +-
+ drivers/net/wireless/ath/ath12k/wmi.c              |   22 +-
+ drivers/net/wireless/ath/ath9k/ar9003_calib.c      |    5 +-
+ drivers/net/wireless/ath/ath9k/mci.c               |    5 +-
+ drivers/net/wireless/ath/ath9k/xmit.c              |    2 +-
+ drivers/net/wireless/broadcom/b43legacy/dma.c      |   24 +-
+ drivers/net/wireless/broadcom/b43legacy/dma.h      |    4 +-
+ .../broadcom/brcm80211/brcmfmac/cfg80211.c         |  129 +-
+ .../broadcom/brcm80211/brcmfmac/cfg80211.h         |    5 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/feature.c |    2 +
+ .../wireless/broadcom/brcm80211/brcmfmac/feature.h |    4 +-
+ drivers/net/wireless/intel/iwlwifi/mld/agg.c       |   17 +-
+ drivers/net/wireless/intel/iwlwifi/mld/agg.h       |    4 +-
+ drivers/net/wireless/intel/iwlwifi/mld/rx.c        |   50 +-
+ drivers/net/wireless/intel/iwlwifi/mld/rx.h        |    2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/tests/agg.c |    7 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rx.c        |    2 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c      |    6 +-
+ drivers/net/wireless/marvell/libertas/if_usb.c     |   10 +-
+ drivers/net/wireless/marvell/libertas/mesh.c       |  506 -------
+ drivers/net/wireless/marvell/libertas_tf/if_usb.c  |   10 +-
+ .../net/wireless/marvell/mwifiex/11n_rxreorder.c   |   20 +-
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |    7 +
+ drivers/net/wireless/marvell/mwifiex/main.h        |    2 +-
+ drivers/net/wireless/mediatek/mt76/mac80211.c      |   25 +-
+ .../net/wireless/mediatek/mt76/mt76_connac_mcu.c   |    5 +-
+ drivers/net/wireless/mediatek/mt76/mt7925/mcu.c    |    5 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00pci.c     |  104 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00usb.c     |   67 +-
+ drivers/net/wireless/realtek/rtw88/debug.c         |    5 +-
+ drivers/net/wireless/realtek/rtw89/core.c          |   12 +-
+ drivers/net/wireless/realtek/rtw89/mac.c           |   10 +-
+ drivers/net/wireless/realtek/rtw89/regd.c          |    5 +-
+ drivers/net/wireless/silabs/wfx/bh.c               |    6 +-
+ drivers/net/wireless/silabs/wfx/bus_sdio.c         |    1 +
+ drivers/net/wireless/silabs/wfx/main.c             |   37 +-
+ drivers/net/wireless/ti/wlcore/rx.c                |    5 +-
+ drivers/net/wireless/virtual/mac80211_hwsim.h      |    5 +-
+ drivers/net/wireless/virtual/mac80211_hwsim_main.c |   98 +-
+ drivers/ptp/ptp_idt82p33.c                         |   25 +-
+ drivers/ptp/ptp_idt82p33.h                         |    1 +
+ drivers/ptp/ptp_ocp.c                              |  939 ++++++++++++-
+ drivers/usb/atm/cxacru.c                           |   18 +-
+ drivers/usb/atm/speedtch.c                         |   10 +-
+ include/linux/bnge/hsi.h                           |  136 +-
+ include/linux/ieee80211-eht.h                      |   18 +-
+ include/linux/ieee80211-uhr.h                      |  101 +-
+ include/linux/ieee80211.h                          |    3 +-
+ include/linux/igmp.h                               |    3 +-
+ include/linux/mdio-mux.h                           |    2 +-
+ include/linux/mlx5/eswitch.h                       |    3 +
+ include/linux/net.h                                |   34 +-
+ include/linux/netdevice.h                          |   23 +-
+ include/linux/netpoll.h                            |    5 -
+ include/linux/phy.h                                |   28 +-
+ include/linux/phylink.h                            |    2 +
+ include/linux/platform_data/microchip-ksz.h        |    2 +
+ include/linux/skbuff.h                             |    7 +-
+ include/linux/stmmac.h                             |    2 +
+ include/net/arp.h                                  |   10 +-
+ include/net/bond_alb.h                             |   14 +-
+ include/net/cfg80211.h                             |    6 +-
+ include/net/cfg802154.h                            |    9 +-
+ include/net/dropreason-core.h                      |    6 +
+ include/net/dsa.h                                  |    2 +
+ include/net/ieee80211_radiotap.h                   |   68 +-
+ include/net/inet_connection_sock.h                 |    1 +
+ include/net/ip6_tunnel.h                           |    1 +
+ include/net/ip_tunnels.h                           |    7 +-
+ include/net/mac80211.h                             |   66 +-
+ include/net/mana/hw_channel.h                      |    8 +-
+ include/net/ndisc.h                                |   20 +-
+ include/net/neighbour.h                            |   22 +-
+ include/net/net_namespace.h                        |    4 +
+ include/net/netdev_lock.h                          |   20 +-
+ include/net/netfilter/nf_conntrack_seqadj.h        |    1 +
+ include/net/netmem.h                               |   27 +
+ include/net/netns/netfilter.h                      |    2 +
+ include/net/page_pool/helpers.h                    |   12 +-
+ include/net/phy/realtek_phy.h                      |    7 -
+ include/net/psp/functions.h                        |    5 -
+ include/net/psp/types.h                            |    4 +
+ include/net/route.h                                |    7 +-
+ include/net/tcp.h                                  |   15 +-
+ include/net/vxlan.h                                |   12 +-
+ include/uapi/linux/atm.h                           |    2 +-
+ include/uapi/linux/dpll.h                          |    3 +-
+ include/uapi/linux/handshake.h                     |    5 +-
+ include/uapi/linux/if_link.h                       |    1 +
+ include/uapi/linux/mii.h                           |    2 +-
+ include/uapi/linux/netdev.h                        |    2 +-
+ include/uapi/linux/netfilter/nfnetlink_conntrack.h |    1 +
+ include/uapi/linux/netlink.h                       |   14 +
+ include/uapi/linux/nl80211.h                       |   27 +-
+ net/6lowpan/debugfs.c                              |   10 +-
+ net/8021q/vlan.c                                   |    8 +-
+ net/8021q/vlan.h                                   |   22 +-
+ net/8021q/vlan_core.c                              |    2 +-
+ net/8021q/vlan_dev.c                               |   37 +-
+ net/8021q/vlan_netlink.c                           |   52 +-
+ net/8021q/vlanproc.c                               |   18 +-
+ net/Kconfig                                        |    1 -
+ net/atm/proc.c                                     |    7 +-
+ net/batman-adv/bat_iv_ogm.c                        |   29 +-
+ net/batman-adv/bat_v.c                             |   42 +-
+ net/batman-adv/bridge_loop_avoidance.c             |    6 +-
+ net/batman-adv/distributed-arp-table.c             |   10 +-
+ net/batman-adv/hash.h                              |    2 +-
+ net/batman-adv/mesh-interface.c                    |   12 +-
+ net/batman-adv/multicast.c                         |    6 +-
+ net/batman-adv/translation-table.c                 | 1118 ++++++++++------
+ net/batman-adv/types.h                             |   49 +-
+ net/bridge/br.c                                    |    5 +-
+ net/bridge/br_arp_nd_proxy.c                       |   57 +-
+ net/bridge/br_device.c                             |    8 +-
+ net/bridge/br_forward.c                            |  276 ++--
+ net/bridge/br_input.c                              |    4 +-
+ net/bridge/br_mst.c                                |   28 +-
+ net/bridge/br_multicast.c                          |   18 +-
+ net/bridge/br_netlink.c                            |   10 +-
+ net/bridge/br_netlink_tunnel.c                     |   14 +-
+ net/bridge/br_private.h                            |   94 +-
+ net/bridge/br_sysfs_if.c                           |   10 +-
+ net/bridge/br_vlan.c                               |  188 +--
+ net/bridge/br_vlan_options.c                       |   20 +-
+ net/bridge/br_vlan_tunnel.c                        |    2 +-
+ net/core/dev.c                                     |   37 +
+ net/core/dev.h                                     |    2 +
+ net/core/devmem.c                                  |  229 ++--
+ net/core/devmem.h                                  |   44 +-
+ net/core/gro_cells.c                               |    5 +-
+ net/core/neighbour.c                               |  428 +++---
+ net/core/net-sysfs.c                               |  112 +-
+ net/core/netdev-genl.c                             |   26 +-
+ net/core/netdev_work.c                             |   15 +-
+ net/core/page_pool_priv.h                          |   14 +-
+ net/core/page_pool_user.c                          |    8 +-
+ net/core/pktgen.c                                  |   17 +-
+ net/core/rtnetlink.c                               |   22 +
+ net/core/skbuff.c                                  |   32 +-
+ net/core/sock.c                                    |    4 +-
+ net/core/sysctl_net_core.c                         |   17 +-
+ net/devlink/dev.c                                  |    4 +-
+ net/devlink/netlink_gen.c                          |    9 +-
+ net/devlink/netlink_gen.h                          |    2 +-
+ net/devlink/port.c                                 |   19 +-
+ net/dsa/Kconfig                                    |    6 +
+ net/dsa/Makefile                                   |    1 +
+ net/dsa/tag_ks8995.c                               |  180 +++
+ net/dsa/tag_ksz.c                                  |    5 +-
+ net/ethtool/common.c                               |  174 +++
+ net/ethtool/ioctl.c                                |    9 -
+ net/ethtool/ts.h                                   |    8 +-
+ net/handshake/genl.c                               |    4 +-
+ net/handshake/handshake-test.c                     |    2 +-
+ net/handshake/request.c                            |    2 +-
+ net/hsr/hsr_device.c                               |    2 +-
+ net/hsr/hsr_forward.c                              |    2 +-
+ net/hsr/hsr_netlink.c                              |   37 +-
+ net/ieee802154/6lowpan/tx.c                        |    3 +-
+ net/ieee802154/socket.c                            |    5 +-
+ net/ipv4/af_inet.c                                 |    3 -
+ net/ipv4/arp.c                                     |  137 +-
+ net/ipv4/bpf_tcp_ca.c                              |    5 +-
+ net/ipv4/devinet.c                                 |   18 +-
+ net/ipv4/fib_semantics.c                           |    7 +-
+ net/ipv4/fou_nl.c                                  |    6 +-
+ net/ipv4/igmp.c                                    |   23 +-
+ net/ipv4/inet_connection_sock.c                    |    1 +
+ net/ipv4/inet_hashtables.c                         |   11 +
+ net/ipv4/inet_timewait_sock.c                      |    6 +-
+ net/ipv4/ip_forward.c                              |    2 +-
+ net/ipv4/ip_gre.c                                  |    6 +-
+ net/ipv4/ip_sockglue.c                             |   10 +-
+ net/ipv4/ip_tunnel.c                               |  146 +-
+ net/ipv4/ip_vti.c                                  |    2 +-
+ net/ipv4/ipip.c                                    |    2 +-
+ net/ipv4/ipmr.c                                    |   17 +-
+ net/ipv4/netfilter/arp_tables.c                    |    2 +-
+ net/ipv4/ping.c                                    |    5 +-
+ net/ipv4/raw.c                                     |    4 +-
+ net/ipv4/route.c                                   |    2 +-
+ net/ipv4/tcp.c                                     |    9 +-
+ net/ipv4/tcp_bbr.c                                 |   13 +-
+ net/ipv4/tcp_input.c                               |   19 +-
+ net/ipv4/tcp_ipv4.c                                |   13 +-
+ net/ipv4/tcp_output.c                              |   28 +-
+ net/ipv4/udp.c                                     |   26 +-
+ net/ipv6/addrconf.c                                |   17 +-
+ net/ipv6/af_inet6.c                                |    4 -
+ net/ipv6/ah6.c                                     |    2 +-
+ net/ipv6/datagram.c                                |    5 +-
+ net/ipv6/exthdrs.c                                 |    6 +-
+ net/ipv6/ip6_fib.c                                 |   14 +-
+ net/ipv6/ip6_gre.c                                 |  268 ++--
+ net/ipv6/ip6_output.c                              |    6 +-
+ net/ipv6/ip6_vti.c                                 |    2 +-
+ net/ipv6/ip6mr.c                                   |    5 +-
+ net/ipv6/ipv6_sockglue.c                           |  123 +-
+ net/ipv6/ndisc.c                                   |  193 +--
+ net/ipv6/route.c                                   |   63 +-
+ net/ipv6/sit.c                                     |  480 ++++---
+ net/ipv6/tcp_ipv6.c                                |   12 +-
+ net/ipv6/udp.c                                     |    8 +-
+ net/iucv/af_iucv.c                                 |   20 +-
+ net/key/af_key.c                                   |    5 +-
+ net/llc/llc_conn.c                                 |    2 +-
+ net/mac80211/agg-tx.c                              |    9 +-
+ net/mac80211/ap.c                                  |    3 -
+ net/mac80211/debugfs.c                             |    2 +-
+ net/mac80211/debugfs_netdev.c                      |    8 +-
+ net/mac80211/ht.c                                  |    2 +-
+ net/mac80211/ieee80211_i.h                         |   30 +-
+ net/mac80211/iface.c                               |   12 +-
+ net/mac80211/main.c                                |   13 +-
+ net/mac80211/mesh_pathtbl.c                        |    2 +-
+ net/mac80211/mlme.c                                |   48 +-
+ net/mac80211/offchannel.c                          |   13 +-
+ net/mac80211/rate.c                                |    1 -
+ net/mac80211/rx.c                                  |  470 ++++---
+ net/mac80211/s1g.c                                 |    8 -
+ net/mac80211/scan.c                                |   10 +-
+ net/mac80211/tdls.c                                |    4 +-
+ net/mac80211/tests/chan-mode.c                     |    6 +-
+ net/mac80211/tests/util.c                          |    6 +-
+ net/mac80211/tx.c                                  |  427 +++---
+ net/mac80211/util.c                                |    2 +-
+ net/mac802154/iface.c                              |   10 +-
+ net/mptcp/options.c                                |    6 +-
+ net/mptcp/protocol.h                               |   45 +-
+ net/mptcp/subflow.c                                |   18 +-
+ net/ncsi/ncsi-rsp.c                                |   10 +-
+ net/netfilter/core.c                               |   18 +-
+ net/netfilter/ipset/ip_set_core.c                  |    2 +-
+ net/netfilter/ipvs/ip_vs_sync.c                    |    2 +-
+ net/netfilter/nf_conncount.c                       |    2 +-
+ net/netfilter/nf_conntrack_core.c                  |    5 +-
+ net/netfilter/nf_conntrack_netlink.c               |   19 +-
+ net/netfilter/nf_conntrack_ovs.c                   |    2 +-
+ net/netfilter/nf_conntrack_seqadj.c                |   28 +-
+ net/netfilter/nf_conntrack_standalone.c            |    2 +-
+ net/netfilter/nf_flow_table_core.c                 |   17 +-
+ net/netfilter/nf_hooks_lwtunnel.c                  |    2 +-
+ net/netfilter/nf_log.c                             |    2 +-
+ net/netfilter/nf_nat_core.c                        |   16 +-
+ net/netfilter/nf_synproxy_core.c                   |    6 +-
+ net/netfilter/nfnetlink_acct.c                     |    2 +-
+ net/netfilter/nfnetlink_cthelper.c                 |    3 +-
+ net/netfilter/nfnetlink_cttimeout.c                |    8 +-
+ net/netfilter/nfnetlink_hook.c                     |   74 +-
+ net/netfilter/nfnetlink_osf.c                      |    3 +-
+ net/netfilter/nft_ct.c                             |    5 +-
+ net/netfilter/nft_set_pipapo.c                     |    6 +-
+ net/netfilter/xt_CT.c                              |    6 +-
+ net/netfilter/xt_IDLETIMER.c                       |    8 +-
+ net/netfilter/xt_LED.c                             |    5 +-
+ net/netfilter/xt_RATEEST.c                         |    2 +-
+ net/netfilter/xt_TEE.c                             |    2 +-
+ net/netfilter/xt_hashlimit.c                       |    4 +-
+ net/netfilter/xt_limit.c                           |    2 +-
+ net/netfilter/xt_quota.c                           |    2 +-
+ net/netfilter/xt_recent.c                          |    3 +-
+ net/netfilter/xt_statistic.c                       |    2 +-
+ net/netfilter/xt_string.c                          |    2 +-
+ net/netlink/af_netlink.c                           |    8 +-
+ net/netlink/policy.c                               |   16 +-
+ net/openvswitch/datapath.c                         |   14 +-
+ net/openvswitch/datapath.h                         |    2 +-
+ net/openvswitch/vport-internal_dev.c               |    2 +
+ net/packet/af_packet.c                             |    7 +-
+ net/phonet/socket.c                                |    5 +-
+ net/psp/psp.h                                      |   13 +
+ net/psp/psp_main.c                                 |   29 +-
+ net/psp/psp_nl.c                                   |   51 +-
+ net/psp/psp_sock.c                                 |  115 +-
+ net/rds/ib.h                                       |    4 +-
+ net/rds/ib_cm.c                                    |   29 +-
+ net/sched/act_api.c                                |   15 +-
+ net/sched/act_connmark.c                           |    6 +
+ net/sched/act_ct.c                                 |    2 +-
+ net/sched/act_mirred.c                             |    3 +-
+ net/sched/act_mpls.c                               |   11 +
+ net/sched/act_nat.c                                |    6 +
+ net/sched/act_simple.c                             |    7 +
+ net/sched/act_skbmod.c                             |    9 +
+ net/sched/cls_flower.c                             |   39 +-
+ net/sched/ematch.c                                 |    9 +-
+ net/sched/sch_api.c                                |   92 +-
+ net/sched/sch_fq.c                                 |   71 +-
+ net/sctp/proc.c                                    |    6 +-
+ net/socket.c                                       |   32 +-
+ net/tipc/bearer.c                                  |    6 +-
+ net/tipc/bearer.h                                  |    3 +-
+ net/tipc/monitor.c                                 |    3 +-
+ net/tls/tls_main.c                                 |    2 -
+ net/unix/af_unix.c                                 |    3 +-
+ net/vmw_vsock/af_vsock.c                           |   35 +
+ net/vmw_vsock/af_vsock_tap.c                       |    4 -
+ net/wireless/chan.c                                |   11 +-
+ net/wireless/nl80211.c                             |   38 +-
+ net/x25/af_x25.c                                   |    5 +-
+ rust/kernel/net/netlink.rs                         |    8 +-
+ tools/include/uapi/linux/netdev.h                  |    2 +-
+ tools/net/ynl/Makefile                             |    1 +
+ tools/net/ynl/Makefile.deps                        |    3 +-
+ tools/net/ynl/generated/Makefile                   |    3 +-
+ tools/net/ynl/lib/Makefile                         |    3 +-
+ tools/net/ynl/pyynl/lib/ynl.py                     |    8 +-
+ tools/net/ynl/pyynl/ynl_gen_c.py                   |   15 +-
+ tools/net/ynl/tests/Makefile                       |    2 +-
+ tools/net/ynl/ynltool/Makefile                     |   10 +-
+ tools/testing/selftests/bpf/progs/tcp_ca_kfunc.c   |    8 +-
+ tools/testing/selftests/drivers/net/Makefile       |   11 +-
+ tools/testing/selftests/drivers/net/README.rst     |    9 +
+ .../testing/selftests/drivers/net/bonding/Makefile |    3 +-
+ tools/testing/selftests/drivers/net/dsa/Makefile   |    1 +
+ tools/testing/selftests/drivers/net/gro_hw.py      |   13 +
+ .../selftests/drivers/net/{gro.py => gro_lib.py}   |  190 +--
+ tools/testing/selftests/drivers/net/gro_lro.py     |   14 +
+ tools/testing/selftests/drivers/net/gro_sw.py      |   13 +
+ tools/testing/selftests/drivers/net/hds.py         |   29 -
+ tools/testing/selftests/drivers/net/hw/Makefile    |   18 +-
+ tools/testing/selftests/drivers/net/hw/config      |    1 +
+ .../selftests/drivers/net/hw/devlink_rate_tc_bw.py |    5 +-
+ .../testing/selftests/drivers/net/hw/devmem_lib.py |    4 +-
+ .../drivers/net/hw/{gro_hw.py => gro_stats.py}     |    0
+ tools/testing/selftests/drivers/net/hw/iou-zcrx.c  |  422 ++++--
+ tools/testing/selftests/drivers/net/hw/iou-zcrx.py |   50 +-
+ .../selftests/drivers/net/hw/lib/py/__init__.py    |    7 +-
+ tools/testing/selftests/drivers/net/hw/toeplitz.py |   10 +-
+ tools/testing/selftests/drivers/net/hw/tso.py      |  161 +++
+ tools/testing/selftests/drivers/net/hw/vlan.py     |  127 ++
+ .../selftests/drivers/net/lib/py/__init__.py       |    7 +-
+ tools/testing/selftests/drivers/net/lib/py/env.py  |   19 +
+ tools/testing/selftests/drivers/net/lib/py/feat.py |   43 +
+ .../selftests/drivers/net/netconsole/Makefile      |    3 +-
+ .../selftests/drivers/net/netdevsim/Makefile       |    8 +
+ tools/testing/selftests/drivers/net/pppoe_gro.py   |   45 +
+ tools/testing/selftests/drivers/net/psp.py         |  180 ++-
+ .../testing/selftests/drivers/net/ring_reconfig.py |   26 +-
+ tools/testing/selftests/drivers/net/rss_key.py     |  343 +++++
+ tools/testing/selftests/drivers/net/ruff.toml      |    4 +
+ tools/testing/selftests/drivers/net/settings       |    2 +-
+ tools/testing/selftests/drivers/net/so_txtime.c    |  161 ++-
+ tools/testing/selftests/drivers/net/so_txtime.py   |   80 +-
+ tools/testing/selftests/drivers/net/team/Makefile  |    4 +-
+ .../selftests/drivers/net/virtio_net/Makefile      |    1 +
+ tools/testing/selftests/net/Makefile               |    1 +
+ tools/testing/selftests/net/bind_wildcard.c        |    5 +
+ tools/testing/selftests/net/forwarding/Makefile    |    2 +
+ tools/testing/selftests/net/fou_mcast_encap.sh     |   11 +-
+ tools/testing/selftests/net/icmp_redirect.sh       |   10 +-
+ tools/testing/selftests/net/ip_local_port_range.c  |   52 +-
+ tools/testing/selftests/net/lib/csum.c             |   55 +-
+ tools/testing/selftests/net/lib/gro.c              |  136 +-
+ .../selftests/net/lib/ksft_setup_loopback.sh       |    2 +-
+ tools/testing/selftests/net/lib/py/__init__.py     |    4 +-
+ tools/testing/selftests/net/lib/py/utils.py        |   22 +
+ tools/testing/selftests/net/mptcp/config           |    2 +-
+ .../selftests/net/ndisc_unsolicited_na_test.sh     |  197 ++-
+ tools/testing/selftests/net/netdev_lock.py         |   42 +
+ tools/testing/selftests/net/netfilter/Makefile     |    2 +
+ tools/testing/selftests/net/netfilter/config       |    2 +-
+ .../selftests/net/netfilter/conntrack_dump_flush.c |  145 +-
+ .../net/netfilter/conntrack_icmp_related.sh        |    2 +-
+ tools/testing/selftests/net/openvswitch/config     |    3 +
+ .../selftests/net/openvswitch/openvswitch.sh       |  202 +++
+ .../packetdrill/tcp_rcv_ssthresh_scaling_ratio.pkt |   46 +
+ tools/testing/selftests/net/rtnetlink.py           |  113 +-
+ tools/testing/selftests/net/rtnetlink.sh           |    8 +-
+ tools/testing/selftests/net/ruff.toml              |    4 +
+ tools/testing/selftests/net/so_incoming_cpu.c      |    7 +-
+ .../selftests/net/srv6_end_dt4_l3vpn_test.sh       |   21 +
+ .../selftests/net/srv6_end_dt6_l3vpn_test.sh       |   21 +
+ tools/testing/selftests/net/test_neigh.sh          |   66 +-
+ tools/testing/selftests/net/tun.c                  |    9 +
+ tools/testing/selftests/net/xfrm_policy.sh         |    2 +-
+ tools/testing/vsock/util.c                         |    9 +
+ tools/testing/vsock/util.h                         |    1 +
+ tools/testing/vsock/vsock_uring_test.c             |  391 ++++++
+ 903 files changed, 27189 insertions(+), 10462 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/net/snps,dwmac-common.yaml
+ create mode 100644 Documentation/devicetree/bindings/net/ultrarisc,dp1000-gmac.yaml
+ delete mode 100644 Documentation/devicetree/bindings/net/wireless/ti,wl1251.txt
+ create mode 100644 Documentation/devicetree/bindings/net/wireless/ti,wl1251.yaml
+ create mode 100644 Documentation/devicetree/bindings/net/wiznet,w5100.yaml
+ delete mode 100644 Documentation/devicetree/bindings/net/wiznet,w5x00.txt
+ create mode 100644 Documentation/devicetree/bindings/net/x-powers,acx00-ephy-package.yaml
+ create mode 100644 Documentation/networking/device_drivers/ethernet/nebula-matrix/nbl.rst
+ create mode 100644 Documentation/networking/devlink/ptp_ocp.rst
+ delete mode 100644 drivers/net/dsa/ks8995.c
+ create mode 100644 drivers/net/dsa/motorcomm/Kconfig
+ create mode 100644 drivers/net/dsa/motorcomm/Makefile
+ rename drivers/net/dsa/{yt921x.c => motorcomm/chip.c} (88%)
+ rename drivers/net/dsa/{yt921x.h => motorcomm/chip.h} (94%)
+ create mode 100644 drivers/net/dsa/motorcomm/leds.c
+ create mode 100644 drivers/net/dsa/motorcomm/leds.h
+ create mode 100644 drivers/net/dsa/motorcomm/mdio_bus.c
+ create mode 100644 drivers/net/dsa/motorcomm/mdio_bus.h
+ create mode 100644 drivers/net/dsa/motorcomm/pcs-921x.c
+ create mode 100644 drivers/net/dsa/motorcomm/pcs.h
+ create mode 100644 drivers/net/dsa/motorcomm/smi.c
+ create mode 100644 drivers/net/dsa/motorcomm/smi.h
+ rename drivers/net/ethernet/intel/ice/{ice_ethtool_fdir.c => ice_ethtool_ntuple.c} (96%)
+ create mode 100644 drivers/net/ethernet/meta/mpnic/Makefile
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic.h
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_csr.h
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_init.c
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_irq.c
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_netdev.c
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_netdev.h
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_pci.c
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_txrx.c
+ create mode 100644 drivers/net/ethernet/meta/mpnic/mpnic_txrx.h
+ create mode 100644 drivers/net/ethernet/nebula-matrix/Kconfig
+ create mode 100644 drivers/net/ethernet/nebula-matrix/Makefile
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/Makefile
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h
+ create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c
+ delete mode 100644 drivers/net/ethernet/smsc/smc9194.h
+ create mode 100644 drivers/net/ethernet/stmicro/stmmac/dwmac-ultrarisc.c
+ create mode 100644 drivers/net/phy/phy_fixup.c
+ create mode 100644 drivers/net/phy/xpowers/Makefile
+ create mode 100644 drivers/net/phy/xpowers/ac200.c
+ create mode 100644 drivers/net/phy/xpowers/ac300.c
+ create mode 100644 drivers/net/phy/xpowers/acx00.c
+ create mode 100644 drivers/net/phy/xpowers/acx00.h
+ delete mode 100644 include/net/phy/realtek_phy.h
+ create mode 100644 net/dsa/tag_ks8995.c
+ create mode 100755 tools/testing/selftests/drivers/net/gro_hw.py
+ rename tools/testing/selftests/drivers/net/{gro.py => gro_lib.py} (74%)
+ mode change 100755 => 100644
+ create mode 100755 tools/testing/selftests/drivers/net/gro_lro.py
+ create mode 100755 tools/testing/selftests/drivers/net/gro_sw.py
+ rename tools/testing/selftests/drivers/net/hw/{gro_hw.py => gro_stats.py} (100%)
+ create mode 100755 tools/testing/selftests/drivers/net/hw/vlan.py
+ create mode 100644 tools/testing/selftests/drivers/net/lib/py/feat.py
+ create mode 100755 tools/testing/selftests/drivers/net/pppoe_gro.py
+ create mode 100755 tools/testing/selftests/drivers/net/rss_key.py
+ create mode 100644 tools/testing/selftests/drivers/net/ruff.toml
+ create mode 100755 tools/testing/selftests/net/netdev_lock.py
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_rcv_ssthresh_scaling_ratio.pkt
+ create mode 100644 tools/testing/selftests/net/ruff.toml
+Merging bpf-next/for-next (acff58e305175 libbpf: Fix cleanup on invalid CO-RE relocation offsets)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git bpf-next/for-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/net/bpf_jit_comp.c
+CONFLICT (content): Merge conflict in arch/arm64/net/bpf_jit_comp.c
+Auto-merging arch/riscv/net/bpf_jit_comp64.c
+Auto-merging arch/x86/Kconfig
+Auto-merging arch/x86/net/bpf_jit_comp.c
+CONFLICT (content): Merge conflict in arch/x86/net/bpf_jit_comp.c
+Auto-merging fs/bpf_fs_kfuncs.c
+Auto-merging fs/exec.c
+Auto-merging include/linux/bpf.h
+Auto-merging include/linux/mm.h
+Auto-merging include/net/tcp.h
+Auto-merging kernel/bpf/arena.c
+Auto-merging kernel/bpf/core.c
+Auto-merging kernel/bpf/hashtab.c
+Auto-merging kernel/bpf/syscall.c
+Auto-merging kernel/bpf/trampoline.c
+Auto-merging mm/internal.h
+CONFLICT (content): Merge conflict in mm/internal.h
+Auto-merging mm/memory.c
+Auto-merging mm/nommu.c
+Auto-merging mm/util.c
+Auto-merging net/ipv4/af_inet.c
+Auto-merging net/ipv4/bpf_tcp_ca.c
+Auto-merging net/ipv4/tcp.c
+Auto-merging net/ipv4/tcp_input.c
+Auto-merging net/ipv4/tcp_output.c
+Auto-merging tools/testing/selftests/bpf/testing_helpers.c
+Auto-merging tools/testing/selftests/bpf/testing_helpers.h
+Resolved 'arch/arm64/net/bpf_jit_comp.c' using previous resolution.
+Resolved 'arch/x86/net/bpf_jit_comp.c' using previous resolution.
+Resolved 'mm/internal.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 1d2642c1a986c] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/kernel-parameters.txt    |   16 +
+ Documentation/bpf/bpf_design_QA.rst                |   11 +-
+ Documentation/bpf/btf.rst                          |   85 +-
+ Documentation/bpf/clang-notes.rst                  |    7 +-
+ Documentation/bpf/kfuncs.rst                       |  126 +-
+ Documentation/bpf/linux-notes.rst                  |   31 +-
+ Documentation/bpf/signing.rst                      |  283 +-
+ MAINTAINERS                                        |    2 -
+ arch/arm64/net/bpf_jit_comp.c                      |  172 +-
+ arch/mips/include/asm/uasm.h                       |    8 +
+ arch/mips/mm/uasm-mips.c                           |   12 +
+ arch/mips/mm/uasm.c                                |   24 +-
+ arch/mips/net/bpf_jit_comp.c                       |   57 +-
+ arch/mips/net/bpf_jit_comp.h                       |    6 +-
+ arch/mips/net/bpf_jit_comp32.c                     |  109 +-
+ arch/mips/net/bpf_jit_comp64.c                     |  139 +-
+ arch/parisc/net/bpf_jit.h                          |    2 +
+ arch/parisc/net/bpf_jit_comp32.c                   |   29 +-
+ arch/parisc/net/bpf_jit_comp64.c                   |   35 +-
+ arch/parisc/net/bpf_jit_core.c                     |   14 +
+ arch/riscv/net/bpf_jit.h                           |    4 +
+ arch/riscv/net/bpf_jit_comp64.c                    |  109 +-
+ arch/riscv/net/bpf_jit_core.c                      |    8 +
+ arch/riscv/net/bpf_timed_may_goto.S                |    8 +-
+ arch/x86/Kconfig                                   |    1 +
+ arch/x86/net/bpf_jit_comp.c                        |  501 ++-
+ arch/x86/net/bpf_timed_may_goto.S                  |   10 +-
+ crypto/Makefile                                    |    3 -
+ crypto/bpf_crypto_skcipher.c                       |   83 -
+ drivers/net/ethernet/netronome/nfp/bpf/verifier.c  |    2 +-
+ fs/bpf_fs_kfuncs.c                                 |   99 +
+ fs/exec.c                                          |    7 +-
+ include/linux/bpf-cgroup-defs.h                    |    1 +
+ include/linux/bpf-cgroup.h                         |   28 +
+ include/linux/bpf.h                                |  169 +-
+ include/linux/bpf_crypto.h                         |   24 -
+ include/linux/bpf_lsm.h                            |   11 +-
+ include/linux/bpf_verifier.h                       |  297 +-
+ include/linux/btf.h                                |   23 +-
+ include/linux/filter.h                             |   45 +
+ include/linux/key.h                                |    2 +
+ include/linux/mm.h                                 |    8 +-
+ include/net/tcp.h                                  |  156 +-
+ include/uapi/linux/bpf.h                           |   82 +-
+ include/uapi/linux/btf.h                           |   83 +-
+ include/uapi/linux/keyctl.h                        |    1 +
+ kernel/bpf/Kconfig                                 |   26 +
+ kernel/bpf/Makefile                                |    7 +-
+ kernel/bpf/arena.c                                 |   83 +-
+ kernel/bpf/arraymap.c                              |    2 +-
+ kernel/bpf/backtrack.c                             |  153 +-
+ kernel/bpf/bpf_insn_array.c                        |   19 +-
+ kernel/bpf/bpf_local_storage.c                     |   15 +-
+ kernel/bpf/bpf_lsm.c                               |   26 +-
+ kernel/bpf/bpf_lsm_proto.c                         |   19 +-
+ kernel/bpf/bpf_struct_ops.c                        |  142 +-
+ kernel/bpf/btf.c                                   |  645 ++-
+ kernel/bpf/cfg.c                                   |  309 +-
+ kernel/bpf/cgroup.c                                |  502 ++-
+ kernel/bpf/cnum_defs.h                             |   37 +-
+ kernel/bpf/const_fold.c                            |    6 +-
+ kernel/bpf/core.c                                  |  137 +-
+ kernel/bpf/crypto.c                                |  288 +-
+ kernel/bpf/diagnostics.c                           |   71 +-
+ kernel/bpf/diagnostics.h                           |    1 +
+ kernel/bpf/disasm.c                                |    5 +-
+ kernel/bpf/fixups.c                                |  341 +-
+ kernel/bpf/hashtab.c                               |   63 +-
+ kernel/bpf/helpers.c                               |  259 +-
+ kernel/bpf/keys.c                                  |   73 +
+ kernel/bpf/liveness.c                              |  633 ++-
+ kernel/bpf/log.c                                   |   13 +-
+ kernel/bpf/map_in_map.c                            |    4 +
+ kernel/bpf/map_iter.c                              |    6 +
+ kernel/bpf/queue_stack_maps.c                      |    3 +
+ kernel/bpf/range_tree.c                            |   52 +-
+ kernel/bpf/states.c                                |  108 +-
+ kernel/bpf/stream.c                                |  271 +-
+ kernel/bpf/syscall.c                               |   46 +-
+ kernel/bpf/trampoline.c                            |    8 +-
+ kernel/bpf/verifier.c                              | 4455 +++++++++++++-------
+ kernel/trace/bpf_trace.c                           |    6 +-
+ mm/bpf_memcontrol.c                                |   65 +-
+ mm/internal.h                                      |   13 +-
+ mm/memory.c                                        |   43 +-
+ mm/nommu.c                                         |   43 +-
+ mm/util.c                                          |   60 +
+ net/core/filter.c                                  |   33 +-
+ net/ipv4/Makefile                                  |    1 +
+ net/ipv4/af_inet.c                                 |    1 +
+ net/ipv4/bpf_tcp_ca.c                              |   16 +
+ net/ipv4/bpf_tcp_ops.c                             |  326 ++
+ net/ipv4/tcp.c                                     |    1 +
+ net/ipv4/tcp_bpf.c                                 |    1 -
+ net/ipv4/tcp_input.c                               |   17 +
+ net/ipv4/tcp_output.c                              |  104 +-
+ net/ipv4/tcp_timer.c                               |    1 +
+ net/sched/bpf_qdisc.c                              |    2 -
+ samples/bpf/Makefile                               |    1 +
+ samples/bpf/hash_func01.h                          |   55 -
+ security/bpf/hooks.c                               |    1 +
+ security/keys/process_keys.c                       |   25 +
+ tools/bpf/bpftool/Documentation/bpftool-btf.rst    |    7 +-
+ tools/bpf/bpftool/Documentation/bpftool-cgroup.rst |   25 +-
+ tools/bpf/bpftool/Documentation/bpftool-map.rst    |   14 +-
+ tools/bpf/bpftool/Documentation/bpftool-prog.rst   |   23 +-
+ tools/bpf/bpftool/bash-completion/bpftool          |   65 +-
+ tools/bpf/bpftool/btf.c                            |  337 +-
+ tools/bpf/bpftool/cgroup.c                         |   99 +-
+ tools/bpf/bpftool/common.c                         |   40 +
+ tools/bpf/bpftool/gen.c                            |   26 +-
+ tools/bpf/bpftool/main.c                           |   61 +-
+ tools/bpf/bpftool/main.h                           |    4 +-
+ tools/bpf/bpftool/map.c                            |   95 +-
+ tools/bpf/bpftool/map_perf_ring.c                  |   87 +-
+ tools/bpf/bpftool/prog.c                           |  302 +-
+ tools/bpf/bpftool/sign.c                           |   24 +-
+ tools/bpf/bpftool/skeleton/profiler.bpf.c          |   29 +-
+ tools/include/uapi/linux/bpf.h                     |   82 +-
+ tools/include/uapi/linux/btf.h                     |   83 +-
+ tools/lib/bpf/bpf.c                                |   21 +
+ tools/lib/bpf/bpf.h                                |   27 +-
+ tools/lib/bpf/bpf_gen_internal.h                   |   13 +-
+ tools/lib/bpf/btf.c                                |  424 +-
+ tools/lib/bpf/btf.h                                |   50 +
+ tools/lib/bpf/btf_dump.c                           |  217 +-
+ tools/lib/bpf/btf_iter.c                           |   18 +
+ tools/lib/bpf/elf.c                                |    2 +-
+ tools/lib/bpf/gen_loader.c                         |  188 +-
+ tools/lib/bpf/libbpf.c                             |  670 ++-
+ tools/lib/bpf/libbpf.h                             |   41 +-
+ tools/lib/bpf/libbpf.map                           |   10 +
+ tools/lib/bpf/libbpf_internal.h                    |    6 +-
+ tools/lib/bpf/libbpf_probes.c                      |   11 +-
+ tools/lib/bpf/linker.c                             |   31 +-
+ tools/lib/bpf/skel_internal.h                      |   48 +-
+ tools/lib/bpf/usdt.c                               |    4 +
+ tools/testing/selftests/bpf/.gitignore             |    1 -
+ tools/testing/selftests/bpf/Makefile               |  758 +---
+ tools/testing/selftests/bpf/Makefile.buildvars     |  162 +
+ tools/testing/selftests/bpf/Makefile.runner        |  125 +
+ tools/testing/selftests/bpf/Makefile.skel          |  140 +
+ tools/testing/selftests/bpf/README.rst             |    2 +-
+ tools/testing/selftests/bpf/bench.c                |    6 +
+ .../testing/selftests/bpf/benchs/bench_libarena.c  |  210 +
+ .../selftests/bpf/benchs/run_bench_libarena.sh     |   31 +
+ tools/testing/selftests/bpf/bpf_kfuncs.h           |   19 +-
+ .../selftests/bpf/bpftool_btf_dump_sorted.expected |   48 +
+ .../bpf/bpftool_btf_dump_unsorted.expected         |   48 +
+ tools/testing/selftests/bpf/bpftool_helpers.c      |    9 +-
+ tools/testing/selftests/bpf/bpftool_helpers.h      |    1 +
+ tools/testing/selftests/bpf/btf_helpers.c          |   45 +-
+ tools/testing/selftests/bpf/cgroup_helpers.c       |   67 +
+ tools/testing/selftests/bpf/cgroup_helpers.h       |    4 +
+ tools/testing/selftests/bpf/config                 |    6 +-
+ tools/testing/selftests/bpf/gen_bpf_skel.sh        |   99 +
+ tools/testing/selftests/bpf/libarena/Makefile      |   31 +-
+ .../bpf/libarena/benchs/bench_malloc.bpf.c         |   51 +
+ .../bpf/libarena/include/bpf_arena_spin_lock.h     |    2 +-
+ .../selftests/bpf/libarena/include/bpf_atomic.h    |    2 +-
+ .../selftests/bpf/libarena/include/bpf_may_goto.h  |    1 +
+ .../bpf/libarena/include/libarena/bitmap.h         |   30 +-
+ .../bpf/libarena/include/libarena/common.h         |   71 +
+ .../bpf/libarena/selftests/test_bitmap.bpf.c       |    3 +
+ .../testing/selftests/bpf/libarena/src/asan.bpf.c  |   21 +-
+ .../selftests/bpf/libarena/src/bitmap.bpf.c        |   18 -
+ .../testing/selftests/bpf/libarena/src/buddy.bpf.c |   44 +-
+ .../selftests/bpf/libarena/src/common.bpf.c        |   27 +-
+ tools/testing/selftests/bpf/network_helpers.c      |   46 +-
+ tools/testing/selftests/bpf/network_helpers.h      |   15 +-
+ .../selftests/bpf/prog_tests/aggregate_arg.c       |   11 +
+ .../selftests/bpf/prog_tests/aggregate_ret.c       |   53 +
+ .../testing/selftests/bpf/prog_tests/arena_memcg.c |  196 +
+ .../selftests/bpf/prog_tests/attach_probe.c        |    4 +-
+ .../selftests/bpf/prog_tests/bpf_insn_array.c      |  650 ++-
+ tools/testing/selftests/bpf/prog_tests/bpf_nf.c    |   14 +-
+ .../testing/selftests/bpf/prog_tests/bpf_tcp_ops.c |  560 +++
+ .../selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c     |   77 +
+ .../selftests/bpf/prog_tests/bpf_verif_scale.c     |    2 +-
+ .../selftests/bpf/prog_tests/bpftool_btf_dump.c    |  297 ++
+ .../selftests/bpf/prog_tests/bpftool_ringbuf.c     |  417 ++
+ tools/testing/selftests/bpf/prog_tests/btf.c       |  126 +
+ .../selftests/bpf/prog_tests/btf_dedup_split.c     |  116 +
+ .../testing/selftests/bpf/prog_tests/btf_distill.c |  106 +
+ tools/testing/selftests/bpf/prog_tests/btf_dump.c  |  400 +-
+ .../selftests/bpf/prog_tests/btf_field_iter.c      |   31 +-
+ .../bpf/prog_tests/btf_module_allowlist.c          |  162 +
+ tools/testing/selftests/bpf/prog_tests/btf_write.c |   34 +
+ tools/testing/selftests/bpf/prog_tests/call_rcu.c  |  275 ++
+ .../selftests/bpf/prog_tests/callx_func_ptr_map.c  |  207 +
+ .../selftests/bpf/prog_tests/callx_rodata_lskel.c  |   74 +
+ tools/testing/selftests/bpf/prog_tests/cb_refs.c   |   48 -
+ .../bpf/prog_tests/cgroup_getset_retval.c          |    2 +-
+ .../selftests/bpf/prog_tests/cgroup_mprog_opts.c   |   71 +-
+ .../selftests/bpf/prog_tests/copy_from_user_bprm.c |   65 +
+ .../testing/selftests/bpf/prog_tests/core_reloc.c  |   10 +-
+ .../testing/selftests/bpf/prog_tests/exceptions.c  |    7 +
+ .../selftests/bpf/prog_tests/fexit_bpf2bpf.c       |   22 +-
+ .../bpf/prog_tests/flow_dissector_classification.c |    2 +-
+ .../selftests/bpf/prog_tests/global_data_init.c    |   96 +-
+ tools/testing/selftests/bpf/prog_tests/kasan.c     |  479 +++
+ .../testing/selftests/bpf/prog_tests/kernel_flag.c |   22 +-
+ .../testing/selftests/bpf/prog_tests/kfunc_call.c  |    2 +-
+ .../selftests/bpf/prog_tests/kprobe_multi_test.c   |    3 +-
+ .../selftests/bpf/prog_tests/linked_externs.c      |  120 +
+ .../testing/selftests/bpf/prog_tests/lirc_mode2.c  |  334 ++
+ .../bpf/prog_tests/lsm_inode_init_xattr.c          |  399 ++
+ .../selftests/bpf/prog_tests/lwt_ip_encap.c        |   53 +
+ .../selftests/bpf/prog_tests/may_goto_far.c        |   92 +
+ .../selftests/bpf/prog_tests/may_goto_priv_stack.c |   41 +
+ .../bpf/prog_tests/prog_tests_framework.c          |   23 +
+ .../selftests/bpf/prog_tests/queue_stack_map.c     |   29 +
+ .../selftests/bpf/prog_tests/signed_loader.c       |  600 ++-
+ tools/testing/selftests/bpf/prog_tests/snprintf.c  |    4 +-
+ tools/testing/selftests/bpf/prog_tests/stream.c    |  553 +++
+ .../bpf/prog_tests/struct_ops_private_stack.c      |   31 +
+ tools/testing/selftests/bpf/prog_tests/tailcalls.c |  116 +
+ .../bpf/prog_tests/test_struct_ops_multi_args.c    |   57 +-
+ .../selftests/bpf/prog_tests/test_task_work.c      |    2 +-
+ tools/testing/selftests/bpf/prog_tests/timer.c     |   33 +
+ tools/testing/selftests/bpf/prog_tests/usdt.c      |   54 +
+ tools/testing/selftests/bpf/prog_tests/verifier.c  |   29 +
+ .../selftests/bpf/prog_tests/verify_pkcs7_sig.c    |    4 +-
+ .../selftests/bpf/prog_tests/xdp_adjust_frags.c    |    3 +
+ .../selftests/bpf/prog_tests/xdp_devmap_attach.c   |    5 +
+ .../selftests/bpf/prog_tests/xdp_metadata.c        |    4 +-
+ tools/testing/selftests/bpf/prog_tests/xsk.c       |    2 +-
+ .../selftests/bpf/progs/aggregate_arg_func.c       |  188 +
+ .../selftests/bpf/progs/aggregate_arg_kfunc.c      |  168 +
+ .../selftests/bpf/progs/aggregate_ret_func.c       |  383 ++
+ .../selftests/bpf/progs/aggregate_ret_kfunc.c      |  203 +
+ .../bpf/progs/aggregate_ret_kfunc_arena.c          |  121 +
+ .../selftests/bpf/progs/aggregate_ret_target.c     |   29 +
+ tools/testing/selftests/bpf/progs/arena_kfunc.c    |   20 +-
+ tools/testing/selftests/bpf/progs/arena_memcg.c    |   23 +
+ .../selftests/bpf/progs/async_stack_depth.c        |   75 +
+ .../testing/selftests/bpf/progs/bpf_iter_netlink.c |    4 +-
+ tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c  |   11 +-
+ tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c  |   11 +-
+ tools/testing/selftests/bpf/progs/bpf_iter_udp4.c  |    4 +-
+ tools/testing/selftests/bpf/progs/bpf_iter_udp6.c  |    4 +-
+ tools/testing/selftests/bpf/progs/bpf_iter_unix.c  |    5 +-
+ tools/testing/selftests/bpf/progs/bpf_misc.h       |   22 +-
+ tools/testing/selftests/bpf/progs/bpf_tcp_ops.c    |  141 +
+ .../testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c  |   86 +
+ .../testing/selftests/bpf/progs/bpftool_ringbuf.c  |   47 +
+ .../selftests/bpf/progs/btf__stack_arg_precision.c |    3 +-
+ .../bpf/progs/btf__verifier_stack_arg_order.c      |    3 +-
+ .../selftests/bpf/progs/btf_module_allowlist.c     |   13 +
+ tools/testing/selftests/bpf/progs/call_rcu.c       |  110 +
+ tools/testing/selftests/bpf/progs/call_rcu_fail.c  |  114 +
+ tools/testing/selftests/bpf/progs/callx_rodata.c   |   75 +
+ tools/testing/selftests/bpf/progs/cb_refs.c        |    5 +
+ .../bpf/{ => progs}/cgroup_getset_retval_hooks.h   |    0
+ .../selftests/bpf/progs/cgrp_kfunc_failure.c       |    8 +-
+ .../selftests/bpf/progs/compute_live_registers.c   |   36 +
+ .../selftests/bpf/progs/copy_from_user_bprm.c      |   69 +
+ .../testing/selftests/bpf/progs/cpumask_failure.c  |    4 +-
+ tools/testing/selftests/bpf/progs/dynptr_fail.c    |   14 +-
+ .../selftests/bpf/progs/exceptions_dead_subprog.c  |   40 +
+ .../testing/selftests/bpf/progs/exceptions_fail.c  |    2 +-
+ .../selftests/bpf/progs/freplace_ret_pair.c        |   12 +
+ tools/testing/selftests/bpf/progs/irq.c            |   17 +-
+ tools/testing/selftests/bpf/progs/iters.c          |  388 +-
+ .../selftests/bpf/progs/iters_state_safety.c       |    3 +
+ tools/testing/selftests/bpf/progs/iters_testmod.c  |    7 +-
+ tools/testing/selftests/bpf/progs/kasan.c          |  502 +++
+ tools/testing/selftests/bpf/progs/kasan_harden.c   |   52 +
+ tools/testing/selftests/bpf/progs/linked_arena1.c  |   42 +
+ tools/testing/selftests/bpf/progs/linked_arena2.c  |   22 +
+ tools/testing/selftests/bpf/progs/lirc_mode2.c     |   32 +
+ tools/testing/selftests/bpf/progs/lsm.c            |    5 +-
+ .../selftests/bpf/progs/lsm_inode_init_xattr.c     |   98 +
+ .../bpf/progs/lsm_inode_init_xattr_budget.c        |   37 +
+ .../bpf/progs/lsm_inode_init_xattr_value.c         |   47 +
+ tools/testing/selftests/bpf/progs/map_kptr_fail.c  |   10 +-
+ .../selftests/bpf/progs/may_goto_priv_stack.c      |   52 +
+ .../selftests/bpf/progs/mem_rdonly_untrusted.c     |    3 +-
+ .../selftests/bpf/progs/percpu_alloc_fail.c        |   35 +
+ tools/testing/selftests/bpf/progs/rbtree_fail.c    |    4 +-
+ .../selftests/bpf/progs/refcounted_kptr_fail.c     |    9 +-
+ .../selftests/bpf/progs/res_spin_lock_fail.c       |    2 +-
+ tools/testing/selftests/bpf/progs/stack_arg.c      |    3 +-
+ tools/testing/selftests/bpf/progs/stack_arg_fail.c |   13 +-
+ .../testing/selftests/bpf/progs/stack_arg_kfunc.c  |    3 +-
+ .../selftests/bpf/progs/stack_arg_precision.c      |    4 +-
+ tools/testing/selftests/bpf/progs/stream.c         |   48 +
+ tools/testing/selftests/bpf/progs/stream_fail.c    |    2 +-
+ .../selftests/bpf/progs/struct_ops_multi_args.c    |   14 +-
+ .../bpf/progs/struct_ops_private_stack_fail.c      |   47 +-
+ .../bpf/progs/struct_ops_private_stack_large.c     |   51 +
+ .../selftests/bpf/progs/tailcall_freplace_multi.c  |   27 +
+ .../selftests/bpf/progs/tailcall_large_stack.c     |   62 +
+ .../selftests/bpf/progs/task_kfunc_failure.c       |   10 +-
+ tools/testing/selftests/bpf/progs/task_work_fail.c |    2 +-
+ .../selftests/bpf/progs/test_global_func1.c        |   65 +
+ .../selftests/bpf/progs/test_global_func5.c        |    2 +-
+ .../bpf/progs/test_global_func_deep_stack.c        |   33 +-
+ .../selftests/bpf/progs/test_global_percpu_data.c  |   10 +-
+ .../selftests/bpf/progs/test_kfunc_dynptr_param.c  |    2 +-
+ .../selftests/bpf/progs/test_lirc_mode2_kern.c     |   26 -
+ tools/testing/selftests/bpf/progs/test_snprintf.c  |    4 +-
+ tools/testing/selftests/bpf/progs/timer.c          |   60 +-
+ tools/testing/selftests/bpf/progs/timer_failure.c  |   29 +
+ .../selftests/bpf/progs/verifier_aggregate_arg.c   |  200 +
+ .../selftests/bpf/progs/verifier_aggregate_ret.c   |  179 +
+ tools/testing/selftests/bpf/progs/verifier_arena.c |   48 +
+ .../selftests/bpf/progs/verifier_bpf_fastcall.c    |  169 +
+ tools/testing/selftests/bpf/progs/verifier_bswap.c |    3 +-
+ tools/testing/selftests/bpf/progs/verifier_callx.c | 1085 +++++
+ .../selftests/bpf/progs/verifier_callx_rodata.c    |  902 ++++
+ tools/testing/selftests/bpf/progs/verifier_cfg.c   |   39 +
+ tools/testing/selftests/bpf/progs/verifier_ctx.c   |    2 +-
+ .../selftests/bpf/progs/verifier_global_ptr_args.c |    9 +-
+ .../selftests/bpf/progs/verifier_global_subprogs.c |   33 +-
+ tools/testing/selftests/bpf/progs/verifier_gotol.c |    1 +
+ tools/testing/selftests/bpf/progs/verifier_gotox.c |  125 +-
+ .../bpf/progs/verifier_helper_access_var_len.c     |  336 +-
+ .../bpf/progs/verifier_helper_packet_access.c      |    4 +-
+ .../selftests/bpf/progs/verifier_jit_inline.c      |    2 +-
+ .../bpf/progs/verifier_kfunc_packet_access.c       |   47 +
+ .../selftests/bpf/progs/verifier_kfunc_uninit.c    |  236 ++
+ .../bpf/progs/verifier_kfunc_uninit_multi.c        |  113 +
+ .../selftests/bpf/progs/verifier_large_stack.c     |  425 ++
+ tools/testing/selftests/bpf/progs/verifier_ldsx.c  |   36 +-
+ .../selftests/bpf/progs/verifier_live_stack.c      |  137 +-
+ .../selftests/bpf/progs/verifier_load_acquire.c    |    1 +
+ tools/testing/selftests/bpf/progs/verifier_lsm.c   |   13 +
+ .../selftests/bpf/progs/verifier_lsm_init_xattr.c  |  245 ++
+ .../selftests/bpf/progs/verifier_map_in_map.c      |    3 +-
+ .../bpf/progs/verifier_map_lookup_refine.c         |    2 +-
+ .../selftests/bpf/progs/verifier_may_goto_1.c      |  121 +
+ tools/testing/selftests/bpf/progs/verifier_movsx.c |    3 +-
+ tools/testing/selftests/bpf/progs/verifier_mtu.c   |   88 +
+ .../selftests/bpf/progs/verifier_percpu_addr.c     |    1 +
+ .../selftests/bpf/progs/verifier_private_stack.c   |    1 +
+ .../selftests/bpf/progs/verifier_raw_stack.c       |   25 +
+ .../selftests/bpf/progs/verifier_ref_tracking.c    |    6 +-
+ tools/testing/selftests/bpf/progs/verifier_sdiv.c  |    3 +-
+ tools/testing/selftests/bpf/progs/verifier_sock.c  |   10 +-
+ .../selftests/bpf/progs/verifier_spill_fill.c      |   43 +
+ .../selftests/bpf/progs/verifier_stack_arg.c       |    4 +-
+ .../selftests/bpf/progs/verifier_stack_arg_order.c |    8 +-
+ .../selftests/bpf/progs/verifier_stack_ptr.c       |   53 +
+ .../selftests/bpf/progs/verifier_store_release.c   |    1 +
+ .../selftests/bpf/progs/verifier_tailcall.c        |   57 +
+ .../testing/selftests/bpf/progs/verifier_unpriv.c  |   37 +
+ .../testing/selftests/bpf/progs/verifier_var_off.c |   32 +
+ .../selftests/bpf/progs/verifier_vfs_reject.c      |    6 +-
+ tools/testing/selftests/bpf/progs/verifier_xadd.c  |   27 +
+ .../selftests/bpf/progs/wakeup_source_fail.c       |    2 +-
+ tools/testing/selftests/bpf/progs/wq_failures.c    |    4 +-
+ tools/testing/selftests/bpf/test_btf.h             |    2 +-
+ .../testing/selftests/bpf/test_kmods/bpf_testmod.c |  303 +-
+ .../testing/selftests/bpf/test_kmods/bpf_testmod.h |    5 +
+ .../selftests/bpf/test_kmods/bpf_testmod_kfunc.h   |  136 +
+ tools/testing/selftests/bpf/test_lirc_mode2.sh     |   41 -
+ tools/testing/selftests/bpf/test_lirc_mode2_user.c |  177 -
+ tools/testing/selftests/bpf/test_loader.c          |   86 +-
+ tools/testing/selftests/bpf/test_progs.h           |    3 +
+ tools/testing/selftests/bpf/testing_helpers.c      |   88 +-
+ tools/testing/selftests/bpf/testing_helpers.h      |    4 +
+ tools/testing/selftests/bpf/unpriv_helpers.c       |   35 +-
+ tools/testing/selftests/bpf/unpriv_helpers.h       |    4 +
+ tools/testing/selftests/bpf/usdt_2.c               |   16 +
+ tools/testing/selftests/bpf/verifier/basic_call.c  |    2 +-
+ tools/testing/selftests/bpf/verifier/calls.c       |  132 +-
+ tools/testing/selftests/bpf/verifier/map_kptr.c    |    4 +-
+ tools/testing/selftests/bpf/verify_sig_setup.sh    |   63 +-
+ tools/testing/selftests/bpf/veristat.c             |    4 +-
+ tools/testing/selftests/bpf/vmtest.sh              |   16 +-
+ tools/testing/selftests/bpf/xdp_hw_metadata.c      |    2 +-
+ 372 files changed, 29911 insertions(+), 5208 deletions(-)
+ delete mode 100644 crypto/bpf_crypto_skcipher.c
+ delete mode 100644 include/linux/bpf_crypto.h
+ create mode 100644 kernel/bpf/keys.c
+ create mode 100644 net/ipv4/bpf_tcp_ops.c
+ delete mode 100644 samples/bpf/hash_func01.h
+ create mode 100644 tools/testing/selftests/bpf/Makefile.buildvars
+ create mode 100644 tools/testing/selftests/bpf/Makefile.runner
+ create mode 100644 tools/testing/selftests/bpf/Makefile.skel
+ create mode 100644 tools/testing/selftests/bpf/benchs/bench_libarena.c
+ create mode 100755 tools/testing/selftests/bpf/benchs/run_bench_libarena.sh
+ create mode 100644 tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected
+ create mode 100644 tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected
+ create mode 100755 tools/testing/selftests/bpf/gen_bpf_skel.sh
+ create mode 100644 tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/aggregate_arg.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/aggregate_ret.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_memcg.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_tcp_ops_hdr.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/bpftool_ringbuf.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/btf_module_allowlist.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/call_rcu.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/callx_func_ptr_map.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/callx_rodata_lskel.c
+ delete mode 100644 tools/testing/selftests/bpf/prog_tests/cb_refs.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/copy_from_user_bprm.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/kasan.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/linked_externs.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/lirc_mode2.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/lsm_inode_init_xattr.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/may_goto_far.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/may_goto_priv_stack.c
+ create mode 100644 tools/testing/selftests/bpf/progs/aggregate_arg_func.c
+ create mode 100644 tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c
+ create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_func.c
+ create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c
+ create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c
+ create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_target.c
+ create mode 100644 tools/testing/selftests/bpf/progs/arena_memcg.c
+ create mode 100644 tools/testing/selftests/bpf/progs/bpf_tcp_ops.c
+ create mode 100644 tools/testing/selftests/bpf/progs/bpf_tcp_ops_hdr.c
+ create mode 100644 tools/testing/selftests/bpf/progs/bpftool_ringbuf.c
+ create mode 100644 tools/testing/selftests/bpf/progs/btf_module_allowlist.c
+ create mode 100644 tools/testing/selftests/bpf/progs/call_rcu.c
+ create mode 100644 tools/testing/selftests/bpf/progs/call_rcu_fail.c
+ create mode 100644 tools/testing/selftests/bpf/progs/callx_rodata.c
+ rename tools/testing/selftests/bpf/{ => progs}/cgroup_getset_retval_hooks.h (100%)
+ create mode 100644 tools/testing/selftests/bpf/progs/copy_from_user_bprm.c
+ create mode 100644 tools/testing/selftests/bpf/progs/exceptions_dead_subprog.c
+ create mode 100644 tools/testing/selftests/bpf/progs/freplace_ret_pair.c
+ create mode 100644 tools/testing/selftests/bpf/progs/kasan.c
+ create mode 100644 tools/testing/selftests/bpf/progs/kasan_harden.c
+ create mode 100644 tools/testing/selftests/bpf/progs/linked_arena1.c
+ create mode 100644 tools/testing/selftests/bpf/progs/linked_arena2.c
+ create mode 100644 tools/testing/selftests/bpf/progs/lirc_mode2.c
+ create mode 100644 tools/testing/selftests/bpf/progs/lsm_inode_init_xattr.c
+ create mode 100644 tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_budget.c
+ create mode 100644 tools/testing/selftests/bpf/progs/lsm_inode_init_xattr_value.c
+ create mode 100644 tools/testing/selftests/bpf/progs/may_goto_priv_stack.c
+ create mode 100644 tools/testing/selftests/bpf/progs/struct_ops_private_stack_large.c
+ create mode 100644 tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c
+ create mode 100644 tools/testing/selftests/bpf/progs/tailcall_large_stack.c
+ delete mode 100644 tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_aggregate_arg.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_callx.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_callx_rodata.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_kfunc_packet_access.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_kfunc_uninit.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_kfunc_uninit_multi.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_large_stack.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_lsm_init_xattr.c
+ delete mode 100755 tools/testing/selftests/bpf/test_lirc_mode2.sh
+ delete mode 100644 tools/testing/selftests/bpf/test_lirc_mode2_user.c
+$ git am -3 ../patches/0001-perf-Fixup-for-btf_vlan-API-change.patch
+Applying: perf: Fixup for btf_vlan() API change
+Using index info to reconstruct a base tree...
+M	tools/perf/builtin-trace.c
+M	tools/perf/util/btf.c
+Falling back to patching base and 3-way merge...
+Auto-merging tools/perf/builtin-trace.c
+No changes -- Patch already applied.
+Merging ipsec-next/master (014d795c73837 idpf: fix kernel-doc parameter descriptions)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git ipsec-next/master
+Already up to date.
+Merging mlx5-next/mlx5-next (36b1d3299d0b6 net/mlx5: Add qp_latency_sensitive_disable cap bit)
+$ git merge -m Merge branch 'mlx5-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git mlx5-next/mlx5-next
+Already up to date.
+Merging netfilter-next/main (87b80c2f6b05c net: txgbe: free the fixed-rate clock on cleanup)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git netfilter-next/main
+Already up to date.
+Merging ipvs-next/main (6ebcf5074cff0 net: openvswitch: don't schedule rebalancing if there are no datapaths)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git ipvs-next/main
+Already up to date.
+Merging bluetooth/master (fdd5964bd3899 Bluetooth: btusb: drop BROKEN_EXT_SCAN quirk for 0bda:a728)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git bluetooth/master
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/net/bluetooth/brcm,bluetooth.yaml     |    1 +
+ MAINTAINERS                                        |    2 +
+ drivers/bluetooth/Kconfig                          |   15 +
+ drivers/bluetooth/Makefile                         |    3 +
+ drivers/bluetooth/btbcm.c                          |    6 +-
+ drivers/bluetooth/btintel.c                        |   57 +-
+ drivers/bluetooth/btintel.h                        |   14 +
+ drivers/bluetooth/btintel_pcie.c                   | 1535 ++++++-
+ drivers/bluetooth/btintel_pcie.h                   |  257 +-
+ drivers/bluetooth/btmrvl_main.c                    |    4 +-
+ drivers/bluetooth/btmtk.c                          |  104 +-
+ drivers/bluetooth/btmtk.h                          |    9 +
+ drivers/bluetooth/btmtksdio.c                      |   57 +-
+ drivers/bluetooth/btnxpuart.c                      |    4 +-
+ drivers/bluetooth/{btusb.c => btusb_main.c}        |  369 +-
+ drivers/bluetooth/btusb_qcom.c                     | 4557 ++++++++++++++++++++
+ drivers/bluetooth/btusb_qcom.h                     |   99 +
+ drivers/bluetooth/hci_bcm.c                        |    1 +
+ drivers/bluetooth/hci_bcsp.c                       |   26 +-
+ drivers/bluetooth/hci_h4.c                         |  123 +-
+ drivers/bluetooth/hci_h5.c                         |   11 +-
+ drivers/bluetooth/hci_ll.c                         |    2 +-
+ drivers/bluetooth/hci_serdev.c                     |   46 +-
+ drivers/bluetooth/hci_uart.h                       |   39 +-
+ drivers/bluetooth/virtio_bt.c                      |   21 +-
+ include/net/bluetooth/hci_core.h                   |   49 +
+ include/net/bluetooth/hci_h4.h                     |   60 +
+ include/net/bluetooth/hci_mon.h                    |    2 +
+ include/net/bluetooth/l2cap.h                      |  102 +-
+ net/bluetooth/6lowpan.c                            |   44 +-
+ net/bluetooth/Makefile                             |    2 +-
+ net/bluetooth/bnep/core.c                          |   16 +-
+ net/bluetooth/hci_core.c                           |   47 +-
+ net/bluetooth/hci_event.c                          |   68 +-
+ net/bluetooth/hci_h4.c                             |  160 +
+ net/bluetooth/hci_sock.c                           |    8 +
+ net/bluetooth/hci_sync.c                           |   64 +-
+ net/bluetooth/iso.c                                |   15 +-
+ net/bluetooth/l2cap_core.c                         |  284 +-
+ net/bluetooth/l2cap_sock.c                         |  118 +-
+ net/bluetooth/mgmt.c                               |   14 +-
+ net/bluetooth/rfcomm/core.c                        |    6 +-
+ net/bluetooth/rfcomm/sock.c                        |    5 +-
+ net/bluetooth/sco.c                                |   10 +-
+ net/bluetooth/smp.c                                |    4 +-
+ scripts/context-analysis-suppression.txt           |    1 +
+ 46 files changed, 7675 insertions(+), 766 deletions(-)
+ rename drivers/bluetooth/{btusb.c => btusb_main.c} (94%)
+ create mode 100644 drivers/bluetooth/btusb_qcom.c
+ create mode 100644 drivers/bluetooth/btusb_qcom.h
+ create mode 100644 include/net/bluetooth/hci_h4.h
+ create mode 100644 net/bluetooth/hci_h4.c
+$ git am -3 ../patches/0001-bluetooth-Fix-up-mismerge-due-to-drivers-bluetooth-b.patch
+Applying: bluetooth: Fix up mismerge due to drivers/bluetooth/btusb_main.c
+Using index info to reconstruct a base tree...
+M	drivers/bluetooth/btusb_main.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/bluetooth/btusb_main.c
+CONFLICT (content): Merge conflict in drivers/bluetooth/btusb_main.c
+Recorded preimage for 'drivers/bluetooth/btusb_main.c'
+error: Failed to merge in the changes.
+hint: Use 'git am --show-current-patch=diff' to see the failed patch
+hint: When you have resolved this problem, run "git am --continue".
+hint: If you prefer to skip this patch, run "git am --skip" instead.
+hint: To restore the original branch and stop patching, run "git am --abort".
+hint: Disable this message with "git config advice.mergeConflict false"
+Patch failed at 0001 bluetooth: Fix up mismerge due to drivers/bluetooth/btusb_main.c
+Merging wireless-next/for-next (21b4248bfa0f4 Merge tag 'ath-next-20260927' of git://git.kernel.org/pub/scm/linux/kernel/git/ath/ath)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git wireless-next/for-next
+Auto-merging drivers/net/wireless/st/cw1200/txrx.c
+Auto-merging net/mac80211/mesh_pathtbl.c
+Auto-merging net/mac80211/mlme.c
+Auto-merging net/mac80211/tx.c
+Auto-merging net/wireless/nl80211.c
+Merge made by the 'ort' strategy.
+ .../bindings/net/wireless/qcom,ath10k.yaml         |  16 ++
+ drivers/net/wireless/ath/ath11k/wmi.c              |   9 +
+ drivers/net/wireless/ath/ath12k/core.h             |   9 -
+ drivers/net/wireless/ath/ath12k/debugfs.c          |  78 ++++++-
+ drivers/net/wireless/ath/ath12k/dp.c               |  90 +++++---
+ drivers/net/wireless/ath/ath12k/dp.h               | 114 ++++++---
+ drivers/net/wireless/ath/ath12k/dp_mon.c           |  15 +-
+ drivers/net/wireless/ath/ath12k/dp_mon.h           |   2 +-
+ drivers/net/wireless/ath/ath12k/dp_rx.c            |  33 +--
+ drivers/net/wireless/ath/ath12k/dp_rx.h            |   3 -
+ drivers/net/wireless/ath/ath12k/dp_stats.h         |  25 ++
+ drivers/net/wireless/ath/ath12k/hw.h               |   1 +
+ drivers/net/wireless/ath/ath12k/mac.c              |   5 +-
+ drivers/net/wireless/ath/ath12k/qmi.c              | 102 +++++---
+ drivers/net/wireless/ath/ath12k/qmi.h              |   4 +
+ drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c     |  22 +-
+ drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c      |  52 ++++-
+ drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c      |  39 ++--
+ drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c     |   5 +-
+ drivers/net/wireless/ath/ath12k/wifi7/hw.c         |   7 +
+ drivers/net/wireless/ath/ath6kl/cfg80211.c         |  10 +-
+ drivers/net/wireless/ath/wil6210/cfg80211.c        |  13 +-
+ drivers/net/wireless/broadcom/b43legacy/main.c     |   3 +-
+ .../broadcom/brcm80211/brcmfmac/cfg80211.c         |  11 +-
+ drivers/net/wireless/intel/iwlwifi/mld/d3.c        |   6 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/d3.c        |   6 +-
+ drivers/net/wireless/marvell/libertas/cfg.c        |   6 +-
+ drivers/net/wireless/marvell/libertas/rx.c         |   9 +
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |  20 +-
+ drivers/net/wireless/marvell/mwifiex/sdio.c        |   3 +
+ drivers/net/wireless/microchip/wilc1000/cfg80211.c |  13 +-
+ drivers/net/wireless/morsemicro/mm81x/core.h       |   7 +-
+ drivers/net/wireless/morsemicro/mm81x/fw.c         |   9 +-
+ drivers/net/wireless/morsemicro/mm81x/mac.c        | 181 +++++++++------
+ drivers/net/wireless/morsemicro/mm81x/sdio.c       |   9 +-
+ drivers/net/wireless/morsemicro/mm81x/skbq.c       |   7 +-
+ drivers/net/wireless/nxp/nxpwifi/cfg80211.c        |  11 +-
+ drivers/net/wireless/quantenna/qtnfmac/cfg80211.c  |   8 +-
+ drivers/net/wireless/realtek/rtw89/wow.c           |   6 +-
+ drivers/net/wireless/st/cw1200/txrx.c              |   2 +-
+ drivers/net/wireless/virtual/mac80211_hwsim_main.c |  35 +++
+ drivers/net/wireless/virtual/mac80211_hwsim_nan.c  |  22 ++
+ drivers/net/wireless/virtual/mac80211_hwsim_nan.h  |   3 +
+ drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c  |   9 +-
+ include/linux/ieee80211.h                          |  22 ++
+ include/net/cfg80211.h                             |  45 +++-
+ include/net/ieee80211_radiotap.h                   |  45 +++-
+ include/net/mac80211.h                             |  13 +-
+ include/uapi/linux/nl80211.h                       |  55 ++++-
+ net/mac80211/cfg.c                                 |  54 ++++-
+ net/mac80211/debugfs_key.c                         |  18 +-
+ net/mac80211/ieee80211_i.h                         |   7 +
+ net/mac80211/key.c                                 |  96 ++++++--
+ net/mac80211/key.h                                 |   3 +-
+ net/mac80211/mesh_pathtbl.c                        |  13 +-
+ net/mac80211/mesh_plink.c                          |   4 +-
+ net/mac80211/mlme.c                                |  84 ++++++-
+ net/mac80211/nan.c                                 |  33 +++
+ net/mac80211/offchannel.c                          |  10 +
+ net/mac80211/parse.c                               |   4 +
+ net/mac80211/sta_info.h                            |   2 +
+ net/mac80211/tx.c                                  |   2 +-
+ net/mac80211/util.c                                |  41 ++++
+ net/wireless/ap.c                                  |   7 +-
+ net/wireless/chan.c                                |  33 ++-
+ net/wireless/core.c                                |  87 +++++--
+ net/wireless/core.h                                |  11 +-
+ net/wireless/ibss.c                                |   3 +-
+ net/wireless/nl80211.c                             | 257 +++++++++++++++++----
+ net/wireless/rdev-ops.h                            |  40 +++-
+ net/wireless/reg.c                                 |  13 +-
+ net/wireless/sme.c                                 |  10 +-
+ net/wireless/sysfs.c                               |  12 +-
+ net/wireless/trace.h                               |  62 +++--
+ net/wireless/util.c                                |  40 +++-
+ net/wireless/wext-compat.c                         |   8 +-
+ 76 files changed, 1688 insertions(+), 486 deletions(-)
+ create mode 100644 drivers/net/wireless/ath/ath12k/dp_stats.h
+Merging ath-next/for-next (21b4248bfa0f4 Merge tag 'ath-next-20260927' of git://git.kernel.org/pub/scm/linux/kernel/git/ath/ath)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath-next/for-next
+Already up to date.
+Merging iwlwifi-next/next (b75dd71d260ee wifi: iwlwifi: disable HE ER mode)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi-next/next
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/intel/iwlwifi/Makefile        |   6 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/bz.c        |   8 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/dr.c        |   2 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c     |   4 +
+ drivers/net/wireless/intel/iwlwifi/cfg/sc.c        |   2 +-
+ drivers/net/wireless/intel/iwlwifi/fw/acpi.c       | 306 ++++++++++-----
+ drivers/net/wireless/intel/iwlwifi/fw/acpi.h       |  20 +-
+ .../net/wireless/intel/iwlwifi/fw/api/datapath.h   |  31 +-
+ .../net/wireless/intel/iwlwifi/fw/api/mac-cfg.h    |  38 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/power.h  |  14 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/rx.h     |  23 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/scan.h   |  68 +++-
+ drivers/net/wireless/intel/iwlwifi/fw/api/stats.h  |  60 ++-
+ drivers/net/wireless/intel/iwlwifi/fw/dbg.c        |  51 ++-
+ drivers/net/wireless/intel/iwlwifi/fw/file.h       |   2 +
+ drivers/net/wireless/intel/iwlwifi/fw/init.c       |   6 +-
+ drivers/net/wireless/intel/iwlwifi/fw/regulatory.c |  76 +++-
+ drivers/net/wireless/intel/iwlwifi/fw/regulatory.h |  26 +-
+ drivers/net/wireless/intel/iwlwifi/fw/runtime.h    |  13 +-
+ drivers/net/wireless/intel/iwlwifi/fw/uefi.c       | 412 +++++++++++++++++----
+ drivers/net/wireless/intel/iwlwifi/fw/uefi.h       |  26 ++
+ drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c   |  50 ++-
+ drivers/net/wireless/intel/iwlwifi/iwl-drv.c       |   9 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-io.c        | 231 ------------
+ drivers/net/wireless/intel/iwlwifi/iwl-io.h        |   4 -
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c |   8 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-trans.c     |  45 ++-
+ drivers/net/wireless/intel/iwlwifi/iwl-trans.h     |  12 +-
+ drivers/net/wireless/intel/iwlwifi/mld/agg.c       |   7 +-
+ drivers/net/wireless/intel/iwlwifi/mld/debugfs.c   |   2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/fw.c        |   4 +-
+ drivers/net/wireless/intel/iwlwifi/mld/hcmd.h      |   2 +
+ drivers/net/wireless/intel/iwlwifi/mld/key.c       |   6 +
+ drivers/net/wireless/intel/iwlwifi/mld/link.c      |  14 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.c  |   4 +
+ drivers/net/wireless/intel/iwlwifi/mld/mld.c       |   7 +
+ drivers/net/wireless/intel/iwlwifi/mld/nan.c       |  35 +-
+ drivers/net/wireless/intel/iwlwifi/mld/notif.c     |   6 +-
+ drivers/net/wireless/intel/iwlwifi/mld/power.c     |   2 +-
+ .../net/wireless/intel/iwlwifi/mld/regulatory.c    |  83 ++++-
+ .../net/wireless/intel/iwlwifi/mld/regulatory.h    |   4 +-
+ drivers/net/wireless/intel/iwlwifi/mld/rx.c        |  17 +-
+ drivers/net/wireless/intel/iwlwifi/mld/scan.c      |  15 +-
+ drivers/net/wireless/intel/iwlwifi/mld/stats.c     |  35 +-
+ drivers/net/wireless/intel/iwlwifi/mld/tlc.c       | 184 ++++++---
+ drivers/net/wireless/intel/iwlwifi/mvm/d3.c        |  89 +++--
+ drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c   | 135 +------
+ drivers/net/wireless/intel/iwlwifi/mvm/fw.c        |   7 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c  |   4 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       |   3 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/nvm.c       |  15 +
+ drivers/net/wireless/intel/iwlwifi/mvm/ops.c       |   2 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c      |  26 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/scan.c      |   4 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/sta.c       |   8 +-
+ .../net/wireless/intel/iwlwifi/pcie/ctxt-info-v2.c |  19 +-
+ .../net/wireless/intel/iwlwifi/pcie/ctxt-info.c    |   2 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/drv.c      |  12 +-
+ .../intel/iwlwifi/pcie/{gen1_2 => }/internal.h     |  68 +---
+ .../wireless/intel/iwlwifi/pcie/{gen1_2 => }/rx.c  |   9 +-
+ .../intel/iwlwifi/pcie/{gen1_2 => }/trans-gen2.c   |   4 +-
+ .../intel/iwlwifi/pcie/{gen1_2 => }/trans.c        | 248 +++----------
+ .../intel/iwlwifi/pcie/{gen1_2 => }/tx-gen2.c      |   0
+ .../wireless/intel/iwlwifi/pcie/{gen1_2 => }/tx.c  |  60 ++-
+ 64 files changed, 1623 insertions(+), 1072 deletions(-)
+ rename drivers/net/wireless/intel/iwlwifi/pcie/{gen1_2 => }/internal.h (95%)
+ rename drivers/net/wireless/intel/iwlwifi/pcie/{gen1_2 => }/rx.c (99%)
+ rename drivers/net/wireless/intel/iwlwifi/pcie/{gen1_2 => }/trans-gen2.c (99%)
+ rename drivers/net/wireless/intel/iwlwifi/pcie/{gen1_2 => }/trans.c (95%)
+ rename drivers/net/wireless/intel/iwlwifi/pcie/{gen1_2 => }/tx-gen2.c (100%)
+ rename drivers/net/wireless/intel/iwlwifi/pcie/{gen1_2 => }/tx.c (98%)
+Merging wpan-next/master (a6bfdfcc6711d ieee802154: allow legacy LLSEC ADD/DEL ops to pass strict validation)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-next/master
+Already up to date.
+Merging wpan-staging/staging (a6bfdfcc6711d ieee802154: allow legacy LLSEC ADD/DEL ops to pass strict validation)
+$ git merge -m Merge branch 'staging' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-staging/staging
+Already up to date.
+Merging mtd/mtd/next (112a666bd82e9 mtd: virt-concat: unlink discarded items before freeing)
+$ git merge -m Merge branch 'mtd/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd/mtd/next
+Auto-merging drivers/mtd/chips/cfi_cmdset_0001.c
+Auto-merging drivers/mtd/mtd_virt_concat.c
+Auto-merging drivers/mtd/mtdconcat.c
+Auto-merging drivers/mtd/mtdcore.c
+Auto-merging drivers/mtd/nand/raw/cadence-nand-controller.c
+Auto-merging drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+Merge made by the 'ort' strategy.
+ drivers/mtd/chips/cfi_cmdset_0001.c             |   5 +-
+ drivers/mtd/chips/cfi_cmdset_0002.c             |  15 ++++
+ drivers/mtd/devices/docg3.c                     |  20 ++---
+ drivers/mtd/devices/powernv_flash.c             |   2 +-
+ drivers/mtd/hyperbus/hbmc-am654.c               |   7 +-
+ drivers/mtd/maps/Kconfig                        |  17 ++++
+ drivers/mtd/maps/Makefile                       |   1 +
+ drivers/mtd/maps/amd76xrom.c                    |   2 +
+ drivers/mtd/maps/int0800.c                      | 115 ++++++++++++++++++++++++
+ drivers/mtd/maps/l440gx.c                       |   2 -
+ drivers/mtd/maps/pci.c                          |   2 +-
+ drivers/mtd/mtd_blkdevs.c                       |   5 ++
+ drivers/mtd/mtd_virt_concat.c                   |   2 +
+ drivers/mtd/mtdconcat.c                         |   2 +-
+ drivers/mtd/mtdcore.c                           |   2 +-
+ drivers/mtd/mtdsuper.c                          |   4 +-
+ drivers/mtd/nand/onenand/onenand_base.c         |   2 +-
+ drivers/mtd/nand/raw/cadence-nand-controller.c  |   2 +-
+ drivers/mtd/nand/raw/fsmc_nand.c                |   2 +-
+ drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c      |   2 +-
+ drivers/mtd/nand/raw/intel-nand-controller.c    |   6 +-
+ drivers/mtd/nand/raw/loongson-nand-controller.c |   4 +-
+ drivers/mtd/nand/raw/lpc32xx_mlc.c              |  10 +--
+ drivers/mtd/nand/raw/lpc32xx_slc.c              |  10 +--
+ drivers/mtd/nand/raw/marvell_nand.c             |   7 +-
+ drivers/mtd/nand/raw/mtk_nand.c                 |   2 +-
+ drivers/mtd/nand/raw/nand_base.c                |   2 +-
+ drivers/mtd/nand/raw/nand_legacy.c              |   4 +-
+ drivers/mtd/nand/raw/omap2.c                    |  10 +--
+ drivers/mtd/nand/raw/sh_flctl.c                 |   8 +-
+ drivers/mtd/spi-nor/otp.c                       |   2 +-
+ include/linux/mtd/bbm.h                         |   2 +-
+ include/linux/mtd/rawnand.h                     |   2 +-
+ include/linux/mtd/sh_flctl.h                    |   4 +-
+ 34 files changed, 215 insertions(+), 69 deletions(-)
+ create mode 100644 drivers/mtd/maps/int0800.c
+Merging nand/nand/next (55c5b6d5f59f5 mtd: nand: qpic_common: drop stray empty line from struct 'bam_transaction')
+$ git merge -m Merge branch 'nand/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git nand/nand/next
+Auto-merging drivers/mtd/nand/raw/lpc32xx_mlc.c
+Auto-merging drivers/mtd/nand/raw/lpc32xx_slc.c
+Auto-merging drivers/mtd/nand/raw/omap2.c
+Auto-merging drivers/mtd/nand/spi/core.c
+Merge made by the 'ort' strategy.
+ drivers/mtd/nand/raw/atmel/nand-controller.c       |   6 +-
+ drivers/mtd/nand/raw/brcmnand/brcmnand.c           |   2 +
+ drivers/mtd/nand/raw/fsl_ifc_nand.c                |  34 +---
+ drivers/mtd/nand/raw/internals.h                   |   1 +
+ drivers/mtd/nand/raw/lpc32xx_mlc.c                 |  21 +--
+ drivers/mtd/nand/raw/lpc32xx_slc.c                 |  21 +--
+ drivers/mtd/nand/raw/nand_esmt.c                   |  34 ++++
+ drivers/mtd/nand/raw/nand_ids.c                    |   2 +-
+ drivers/mtd/nand/raw/ndfc.c                        |  28 ++-
+ .../mtd/nand/raw/nuvoton-ma35d1-nand-controller.c  |   1 -
+ drivers/mtd/nand/raw/omap2.c                       |   4 +-
+ drivers/mtd/nand/raw/stm32_fmc2_nand.c             |   8 +-
+ drivers/mtd/nand/raw/sunxi_nand.c                  |  37 ++--
+ drivers/mtd/nand/spi/Makefile                      |   2 +-
+ drivers/mtd/nand/spi/core.c                        |   1 +
+ drivers/mtd/nand/spi/issi.c                        | 187 +++++++++++++++++++++
+ drivers/mtd/nand/spi/winbond.c                     |  87 +++++++++-
+ include/linux/mtd/lpc32xx_mlc.h                    |  17 --
+ include/linux/mtd/lpc32xx_slc.h                    |  17 --
+ include/linux/mtd/nand-qpic-common.h               |   1 -
+ include/linux/mtd/spinand.h                        |   3 +-
+ 21 files changed, 370 insertions(+), 144 deletions(-)
+ create mode 100644 drivers/mtd/nand/spi/issi.c
+ delete mode 100644 include/linux/mtd/lpc32xx_mlc.h
+ delete mode 100644 include/linux/mtd/lpc32xx_slc.h
+Merging spi-nor/spi-nor/next (68d7115d77a87 mtd: spi-nor: sfdp: get the 1-1-8 and 1-8-8 page programs from 4BAIT)
+$ git merge -m Merge branch 'spi-nor/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git spi-nor/spi-nor/next
+Auto-merging drivers/mtd/spi-nor/core.c
+Auto-merging drivers/mtd/spi-nor/otp.c
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/mtd/spi-nor.rst |    2 +-
+ drivers/mtd/spi-nor/atmel.c              |   75 ++-
+ drivers/mtd/spi-nor/core.c               | 1035 ++++++++++++------------------
+ drivers/mtd/spi-nor/core.h               |  144 +++--
+ drivers/mtd/spi-nor/debugfs.c            |   31 +-
+ drivers/mtd/spi-nor/everspin.c           |    7 +-
+ drivers/mtd/spi-nor/gigadevice.c         |   15 +-
+ drivers/mtd/spi-nor/issi.c               |   29 +-
+ drivers/mtd/spi-nor/macronix.c           |   73 ++-
+ drivers/mtd/spi-nor/micron-st.c          |   70 +-
+ drivers/mtd/spi-nor/otp.c                |   16 +-
+ drivers/mtd/spi-nor/sfdp.c               |  255 ++++++--
+ drivers/mtd/spi-nor/sfdp.h               |   23 +-
+ drivers/mtd/spi-nor/spansion.c           |  109 ++--
+ drivers/mtd/spi-nor/sst.c                |   16 +-
+ drivers/mtd/spi-nor/swp.c                |  133 ++--
+ drivers/mtd/spi-nor/sysfs.c              |    4 +-
+ drivers/mtd/spi-nor/winbond.c            |  320 +++++++--
+ drivers/mtd/spi-nor/xmc.c                |    1 +
+ include/linux/mtd/spi-nor.h              |   10 +-
+ 20 files changed, 1310 insertions(+), 1058 deletions(-)
+Merging crypto/master (67aefeccc4101 crypto: ccp - kill v5 IRQ tasklet during teardown)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git crypto/master
+Auto-merging Documentation/devicetree/bindings/trivial-devices.yaml
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/crypto/aes-neonbs-glue.c
+Auto-merging lib/Kconfig.debug
+Auto-merging lib/tests/Makefile
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-driver-qat_kpt     |    3 +-
+ .../bindings/crypto/hisilicon,hip06-sec.yaml       |  134 --
+ .../devicetree/bindings/trivial-devices.yaml       |    2 +
+ MAINTAINERS                                        |   12 +-
+ arch/arm/crypto/aes-neonbs-glue.c                  |    2 +-
+ arch/arm64/boot/dts/hisilicon/hip07.dtsi           |  280 -----
+ arch/arm64/crypto/aes-neonbs-glue.c                |    2 +-
+ arch/powerpc/crypto/vmx.c                          |   14 +-
+ crypto/algapi.c                                    |   23 +-
+ crypto/asymmetric_keys/Kconfig                     |   12 +
+ crypto/asymmetric_keys/Makefile                    |    2 +
+ crypto/asymmetric_keys/pkcs7_parser.c              |    6 +
+ crypto/asymmetric_keys/restrict.c                  |    8 +-
+ crypto/asymmetric_keys/verify_pefile.c             |    5 +
+ crypto/asymmetric_keys/verify_pefile_test.c        |  100 ++
+ crypto/asymmetric_keys/x509_public_key.c           |    6 +-
+ crypto/lskcipher.c                                 |    6 +-
+ crypto/rsassa-pkcs1.c                              |    4 +-
+ crypto/zstd.c                                      |   40 +-
+ drivers/char/hw_random/cctrng.c                    |    7 +-
+ drivers/char/hw_random/imx-rngc.c                  |    8 +-
+ drivers/char/hw_random/jh7110-trng.c               |   39 +-
+ drivers/char/hw_random/omap-rng.c                  |    8 +-
+ drivers/crypto/amcc/crypto4xx_core.c               |   35 +-
+ drivers/crypto/amcc/crypto4xx_trng.c               |   15 +-
+ drivers/crypto/amcc/crypto4xx_trng.h               |    6 +-
+ drivers/crypto/amlogic/amlogic-gxl-cipher.c        |   25 +-
+ drivers/crypto/amlogic/amlogic-gxl-core.c          |   48 +-
+ drivers/crypto/aspeed/aspeed-hace-crypto.c         |    3 +-
+ drivers/crypto/atmel-aes.c                         |    2 +-
+ drivers/crypto/atmel-ecc.c                         |   12 +-
+ drivers/crypto/atmel-sha.c                         |    7 +-
+ drivers/crypto/atmel-tdes.c                        |   74 +-
+ drivers/crypto/cavium/cpt/cptpf_main.c             |   12 +-
+ drivers/crypto/cavium/cpt/cptvf_main.c             |    4 +-
+ drivers/crypto/cavium/cpt/cptvf_mbox.c             |    4 +-
+ drivers/crypto/cavium/nitrox/nitrox_main.c         |    2 +-
+ drivers/crypto/ccp/ccp-dev-v3.c                    |    4 +
+ drivers/crypto/ccp/ccp-dev-v5.c                    |    4 +
+ drivers/crypto/ccp/ccp-dev.c                       |    2 +-
+ drivers/crypto/ccp/dbc.c                           |    4 +-
+ drivers/crypto/ccp/sfs.c                           |    9 +-
+ drivers/crypto/ccp/sp-platform.c                   |    4 +-
+ drivers/crypto/ccree/cc_aead.c                     |    4 +-
+ drivers/crypto/ccree/cc_buffer_mgr.c               |    4 +-
+ drivers/crypto/chelsio/chcr_algo.c                 |    2 +-
+ drivers/crypto/hisilicon/Kconfig                   |   14 -
+ drivers/crypto/hisilicon/Makefile                  |    1 -
+ drivers/crypto/hisilicon/hpre/hpre.h               |    1 -
+ drivers/crypto/hisilicon/hpre/hpre_crypto.c        |   28 +-
+ drivers/crypto/hisilicon/hpre/hpre_main.c          |   47 +-
+ drivers/crypto/hisilicon/sec/Makefile              |    3 -
+ drivers/crypto/hisilicon/sec/sec_algs.c            | 1122 -----------------
+ drivers/crypto/hisilicon/sec/sec_drv.c             | 1307 --------------------
+ drivers/crypto/hisilicon/sec/sec_drv.h             |  428 -------
+ drivers/crypto/hisilicon/zip/dae_main.c            |   23 +-
+ drivers/crypto/inside-secure/eip93/eip93-aead.c    |    3 +-
+ drivers/crypto/inside-secure/eip93/eip93-regs.h    |    2 +-
+ drivers/crypto/inside-secure/safexcel_cipher.c     |   16 +-
+ drivers/crypto/inside-secure/safexcel_hash.c       |    6 +-
+ drivers/crypto/intel/qat/qat_common/adf_cfg.c      |   84 +-
+ .../crypto/intel/qat/qat_common/adf_gen4_vf_mig.c  |    5 +
+ drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c |   20 +-
+ drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h |   24 +-
+ drivers/crypto/intel/qat/qat_common/adf_init.c     |   24 +-
+ .../crypto/intel/qat/qat_common/adf_sysfs_kpt.c    |   24 +-
+ drivers/crypto/intel/qat/qat_common/qat_algs.c     |   18 +-
+ .../crypto/intel/qat/qat_common/qat_asym_algs.c    |   38 +-
+ drivers/crypto/marvell/octeontx/otx_cptvf_algs.c   |    7 +-
+ drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c   |    4 +-
+ drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c |    7 +-
+ drivers/crypto/mxs-dcp.c                           |    3 +
+ drivers/crypto/omap-des.c                          |    2 +
+ drivers/crypto/omap-sham.c                         |    5 +-
+ drivers/crypto/padlock-aes.c                       |   22 +-
+ drivers/crypto/qce/aead.c                          |   11 +-
+ drivers/crypto/qce/sha.c                           |    9 +-
+ drivers/crypto/qce/skcipher.c                      |    9 +-
+ drivers/crypto/rockchip/rk3288_crypto.c            |    4 +-
+ drivers/crypto/s5p-sss.c                           |    7 +-
+ drivers/crypto/sa2ul.c                             |    2 +-
+ drivers/crypto/starfive/jh7110-cryp.c              |   20 +-
+ drivers/crypto/talitos.c                           |   21 +-
+ drivers/crypto/xilinx/zynqmp-aes-gcm.c             |   14 +-
+ drivers/crypto/xilinx/zynqmp-sha.c                 |    2 +-
+ include/crypto/aes.h                               |   13 +
+ include/linux/ccp.h                                |   58 +-
+ include/linux/psp-sev.h                            |   86 +-
+ include/linux/rhashtable-types.h                   |   74 +-
+ include/linux/rhashtable.h                         |   19 +-
+ include/uapi/linux/psp-sev.h                       |   26 +-
+ kernel/padata.c                                    |    2 +-
+ lib/842/842_decompress.c                           |    7 +-
+ lib/Kconfig.debug                                  |   15 +
+ lib/rhashtable.c                                   |   70 +-
+ lib/tests/842_decompress_kunit.c                   |  144 +++
+ lib/tests/Makefile                                 |    1 +
+ 97 files changed, 1057 insertions(+), 3854 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/crypto/hisilicon,hip06-sec.yaml
+ create mode 100644 crypto/asymmetric_keys/verify_pefile_test.c
+ delete mode 100644 drivers/crypto/hisilicon/sec/Makefile
+ delete mode 100644 drivers/crypto/hisilicon/sec/sec_algs.c
+ delete mode 100644 drivers/crypto/hisilicon/sec/sec_drv.c
+ delete mode 100644 drivers/crypto/hisilicon/sec/sec_drv.h
+ create mode 100644 lib/tests/842_decompress_kunit.c
+Merging libcrypto/libcrypto-next (b63b9b3d5ddaf MAINTAINERS: Place libcrypto docs under CRYPTO LIBRARY)
+$ git merge -m Merge branch 'libcrypto-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto/libcrypto-next
+Auto-merging MAINTAINERS
+Auto-merging include/crypto/aes.h
+Merge made by the 'ort' strategy.
+ Documentation/crypto/libcrypto-zeroization.rst | 150 +++++++++++++++++++++++++
+ Documentation/crypto/libcrypto.rst             |  14 +++
+ MAINTAINERS                                    |   1 +
+ include/crypto/aes-ccm.h                       |  22 +++-
+ include/crypto/aes-gcm.h                       |  22 +++-
+ include/crypto/aes-xts.h                       |  13 ++-
+ include/crypto/aes.h                           |  18 +++
+ include/crypto/blake2b.h                       |   9 ++
+ include/crypto/blake2s.h                       |   9 ++
+ include/crypto/md5.h                           |  19 ++++
+ include/crypto/sha1.h                          |  19 ++++
+ include/crypto/sm3.h                           |  10 ++
+ lib/crypto/aes.c                               |  28 ++---
+ lib/crypto/blake2b.c                           |   2 +-
+ lib/crypto/blake2s.c                           |   2 +-
+ lib/crypto/md5.c                               |   2 +-
+ lib/crypto/sha1.c                              |   2 +-
+ lib/crypto/sm3.c                               |   2 +-
+ security/keys/trusted-keys/trusted_tpm1.c      |   2 +-
+ 19 files changed, 318 insertions(+), 28 deletions(-)
+ create mode 100644 Documentation/crypto/libcrypto-zeroization.rst
+Merging drm/drm-next (2abe8e7e33973 Merge tag 'amd-drm-next-7.4-2026-09-24' of https://gitlab.freedesktop.org/drm/amdgpu/kernel into drm-next)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git drm/drm-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/accel/ivpu/ivpu_drv.c
+Auto-merging drivers/accel/ivpu/ivpu_drv.h
+Auto-merging drivers/accel/ivpu/ivpu_job.c
+Auto-merging drivers/accel/ivpu/ivpu_mmu.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+Auto-merging drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
+Auto-merging drivers/gpu/drm/bridge/samsung-dsim.c
+Auto-merging drivers/gpu/drm/clients/drm_fbdev_client.c
+Auto-merging drivers/gpu/drm/drm_gpusvm.c
+Auto-merging drivers/gpu/drm/i915/display/intel_cursor.c
+Auto-merging drivers/gpu/drm/i915/display/intel_display_types.h
+Auto-merging drivers/gpu/drm/i915/display/intel_dp_mst.c
+Auto-merging drivers/gpu/drm/i915/display/intel_psr.c
+Auto-merging drivers/gpu/drm/i915/display/intel_vrr.c
+Auto-merging drivers/gpu/drm/i915/display/skl_universal_plane.c
+Auto-merging drivers/gpu/drm/nouveau/nouveau_connector.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/nouveau/nouveau_connector.c
+Auto-merging drivers/gpu/drm/nouveau/nouveau_drm.c
+Auto-merging drivers/gpu/drm/sti/sti_cursor.c
+Auto-merging drivers/gpu/drm/sti/sti_hqvdp.c
+Auto-merging drivers/gpu/drm/vc4/vc4_drv.c
+Auto-merging drivers/gpu/drm/vc4/vc4_v3d.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/vc4/vc4_v3d.c
+Auto-merging drivers/gpu/drm/virtio/virtgpu_plane.c
+Auto-merging drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+Auto-merging drivers/gpu/drm/xe/regs/xe_gt_regs.h
+Auto-merging drivers/gpu/drm/xe/xe_bo.c
+Auto-merging drivers/gpu/drm/xe/xe_bo.h
+Auto-merging drivers/gpu/drm/xe/xe_configfs.c
+Auto-merging drivers/gpu/drm/xe/xe_tlb_inval.c
+Auto-merging drivers/gpu/drm/xe/xe_vm.c
+Resolved 'drivers/gpu/drm/nouveau/nouveau_connector.c' using previous resolution.
+Resolved 'drivers/gpu/drm/vc4/vc4_v3d.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master de7ab1c71354d] Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git
+$ git diff -M --stat --summary HEAD^..
+ .../bindings/display/brcm,bcm2835-v3d.yaml         |     3 +
+ .../bindings/display/bridge/analogix,dp.yaml       |    19 +-
+ .../display/bridge/renesas,r8a779g0-dsc.yaml       |    96 +
+ .../bindings/display/panel/ilitek,ili7836a.yaml    |    58 +
+ .../bindings/display/panel/novatek,nt36532.yaml    |    83 +
+ .../panel/samsung,s6e8aa5x01-ams561ra01.yaml       |     2 +-
+ .../display/rockchip/rockchip,analogix-dp.yaml     |     1 +
+ .../bindings/display/solomon,ssd1351.yaml          |    42 +
+ Documentation/gpu/amdgpu/index.rst                 |     1 +
+ Documentation/gpu/amdgpu/ualink.rst                |    75 +
+ Documentation/gpu/drm-kms-helpers.rst              |    11 +-
+ Documentation/gpu/drm-kms.rst                      |     3 +
+ Documentation/gpu/drm-ras.rst                      |    39 +
+ Documentation/gpu/drm-uapi.rst                     |    93 +-
+ Documentation/gpu/todo.rst                         |    15 -
+ Documentation/gpu/xe/index.rst                     |     1 +
+ Documentation/gpu/xe/xe_sigid.rst                  |    14 +
+ Documentation/netlink/specs/drm_ras.yaml           |    80 +
+ MAINTAINERS                                        |    55 +-
+ arch/arm/boot/dts/broadcom/bcm2835-common.dtsi     |     1 +
+ drivers/accel/amdxdna/aie2_ctx.c                   |     9 +-
+ drivers/accel/amdxdna/aie2_message.c               |     1 +
+ drivers/accel/amdxdna/amdxdna_gem.c                |   460 +-
+ drivers/accel/amdxdna/amdxdna_gem.h                |     6 +-
+ drivers/accel/ethosu/ethosu_job.c                  |     4 +-
+ drivers/accel/ivpu/ivpu_drv.c                      |    13 +
+ drivers/accel/ivpu/ivpu_drv.h                      |     5 +-
+ drivers/accel/ivpu/ivpu_hw.c                       |    22 +
+ drivers/accel/ivpu/ivpu_hw_ip.c                    |     6 +-
+ drivers/accel/ivpu/ivpu_job.c                      |    92 +-
+ drivers/accel/ivpu/ivpu_job.h                      |     5 +-
+ drivers/accel/ivpu/ivpu_mmu.c                      |    12 +-
+ drivers/accel/qaic/qaic_data.c                     |     8 +-
+ drivers/accel/qaic/qaic_debugfs.c                  |     1 +
+ drivers/accel/qaic/qaic_drv.c                      |     1 +
+ drivers/accel/qaic/qaic_ras.c                      |     1 +
+ drivers/accel/qaic/qaic_ssr.c                      |     1 +
+ drivers/accel/qaic/qaic_timesync.c                 |     1 +
+ drivers/accel/qaic/sahara.c                        |     1 +
+ drivers/dma-buf/dma-buf.c                          |     2 +-
+ drivers/dma-buf/dma-resv.c                         |     2 +-
+ drivers/firmware/efi/sysfb_efi.c                   |     9 -
+ drivers/gpu/buddy.c                                |  1342 +-
+ drivers/gpu/drm/Kconfig                            |     9 +-
+ drivers/gpu/drm/Kconfig.debug                      |     1 +
+ drivers/gpu/drm/Makefile                           |     3 +-
+ drivers/gpu/drm/adp/adp_drv.c                      |     2 +-
+ drivers/gpu/drm/amd/amdgpu/Makefile                |     5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |    12 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c            |     6 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c         |    31 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |     8 +-
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c   |     3 +-
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c    |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c |   151 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h |     6 +-
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c   |     1 +
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c   |    36 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c  |    14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h  |     4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c   |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |     4 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c            |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c        |    15 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c   |     8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |   121 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |   340 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h      |    13 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c        |     5 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |    31 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |     9 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c            |    11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h            |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c            |    17 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h            |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c             |   124 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c            |    37 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h            |    27 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h             |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h            |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c             |    68 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h             |    20 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c            |    35 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h            |    11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_isp.c            |     6 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c            |     5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.h            |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c           |    39 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h           |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c            |    11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.c          |    20 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.h          |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c            |     6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c            |    99 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h            |    11 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h          |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |    11 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.h         |     5 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c            |   588 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h            |    86 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c            |   110 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h            |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c     |     8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h     |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c          |     7 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c           |    19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |    50 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c           |   169 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h           |   165 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma_types.h     |   163 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h          |    45 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c   |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |    58 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h            |     4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c         |  6539 ++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.h         |   427 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c            |    46 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h            |     8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          |    46 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c    |    68 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c           |    42 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h           |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c           |    10 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |   105 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |   122 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c         |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h    |   146 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c          |    35 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c        |     6 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c   |     4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c            |    34 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c            |    18 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h            |     7 +
+ drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h        |    49 +-
+ drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c         |    42 +-
+ drivers/gpu/drm/amd/amdgpu/atom.c                  |    29 +-
+ drivers/gpu/drm/amd/amdgpu/cik.c                   |     7 -
+ drivers/gpu/drm/amd/amdgpu/cik_ih.c                |    12 -
+ drivers/gpu/drm/amd/amdgpu/cik_sdma.c              |    68 +-
+ drivers/gpu/drm/amd/amdgpu/cz_ih.c                 |    12 -
+ drivers/gpu/drm/amd/amdgpu/dce_v10_0.c             |     6 -
+ drivers/gpu/drm/amd/amdgpu/dce_v6_0.c              |     6 -
+ drivers/gpu/drm/amd/amdgpu/dce_v8_0.c              |     6 -
+ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c             |    44 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c             |    44 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c             |    44 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c             |   797 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h         |    39 -
+ drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c              |    26 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c              |    39 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c              |   153 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c              |   146 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c            |   111 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h            |     1 +
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c            |    41 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c        |     9 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c          |     9 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c          |    41 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c           |     9 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c           |     3 +
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c           |     9 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c           |     9 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c           |     9 +-
+ drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c         |     9 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c             |    12 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c             |    18 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c             |    59 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c             |   184 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h             |     2 +
+ drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c              |     7 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c              |     7 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c              |    19 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c              |    14 +-
+ drivers/gpu/drm/amd/amdgpu/iceland_ih.c            |    12 -
+ drivers/gpu/drm/amd/amdgpu/ih_v6_0.c               |    39 +-
+ drivers/gpu/drm/amd/amdgpu/ih_v6_1.c               |    16 +-
+ drivers/gpu/drm/amd/amdgpu/ih_v7_0.c               |    43 +-
+ drivers/gpu/drm/amd/amdgpu/imu_v12_1.c             |    91 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c             |     1 -
+ drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c             |     2 -
+ drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c             |     1 -
+ drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c             |     1 -
+ drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c           |     2 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c           |     2 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c           |     2 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c           |     4 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h           |     8 -
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c           |   265 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c           |     1 -
+ drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c            |    27 +-
+ drivers/gpu/drm/amd/amdgpu/mes_userqueue.c         |    62 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v11_0.c             |    35 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v12_0.c             |    29 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v12_1.c             |   282 +-
+ drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c          |   372 +-
+ drivers/gpu/drm/amd/amdgpu/navi10_ih.c             |    11 +-
+ drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c           |   155 +-
+ drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c             |     2 +-
+ drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c           |    18 +-
+ drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c          |   351 +
+ drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.h          |    32 +
+ drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c             |     1 +
+ drivers/gpu/drm/amd/amdgpu/nv.c                    |     6 -
+ drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h            |   162 +
+ drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c           |   146 +
+ drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c             |   105 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c             |   113 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c             |   124 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c           |   250 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c             |   141 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c             |   133 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c             |   143 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c             |   142 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c             |   114 +-
+ drivers/gpu/drm/amd/amdgpu/si.c                    |     6 -
+ drivers/gpu/drm/amd/amdgpu/si_dma.c                |    38 +-
+ drivers/gpu/drm/amd/amdgpu/si_ih.c                 |     1 -
+ drivers/gpu/drm/amd/amdgpu/soc15.c                 |     6 -
+ drivers/gpu/drm/amd/amdgpu/soc15_common.h          |     6 +-
+ drivers/gpu/drm/amd/amdgpu/soc21.c                 |     6 -
+ drivers/gpu/drm/amd/amdgpu/soc24.c                 |     6 -
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.c              |   493 +-
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.h              |     3 +
+ drivers/gpu/drm/amd/amdgpu/ta_ras_if.h             |     1 +
+ drivers/gpu/drm/amd/amdgpu/tonga_ih.c              |    12 -
+ drivers/gpu/drm/amd/amdgpu/ualink_v1_0.c           |   143 +
+ drivers/gpu/drm/amd/amdgpu/ualink_v1_0.h           |    30 +
+ drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c              |     8 -
+ drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c              |     8 -
+ drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c              |     8 -
+ drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c              |     1 -
+ drivers/gpu/drm/amd/amdgpu/vce_v1_0.c              |     1 -
+ drivers/gpu/drm/amd/amdgpu/vce_v2_0.c              |     1 -
+ drivers/gpu/drm/amd/amdgpu/vce_v3_0.c              |     1 -
+ drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c              |     3 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c              |     2 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c              |     3 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c              |    17 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c              |    24 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c            |    22 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c            |    24 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c            |    24 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c            |    20 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c            |   351 +-
+ drivers/gpu/drm/amd/amdgpu/vega10_ih.c             |     7 -
+ drivers/gpu/drm/amd/amdgpu/vega20_ih.c             |     7 -
+ drivers/gpu/drm/amd/amdgpu/vi.c                    |     6 -
+ drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h     |    90 +-
+ .../gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm |   122 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_device.c            |    18 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |   151 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |     3 +
+ drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c       |     6 +
+ drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c    |    23 +
+ drivers/gpu/drm/amd/amdkfd/kfd_migrate.c           |    53 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c |    31 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c |     3 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |    46 +-
+ .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |    10 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_queue.c             |    10 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_svm.c               |     7 +-
+ drivers/gpu/drm/amd/display/Kconfig                |    12 +-
+ drivers/gpu/drm/amd/display/Makefile               |     1 +
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |   454 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h  |   182 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_backlight.c    |    33 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_backlight.h    |     1 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_color.c    |    85 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c  |   137 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.h  |    29 +
+ .../amd/display/amdgpu_dm/amdgpu_dm_connector.c    |   379 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_connector.h    |    11 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c |    87 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h |     9 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.c   |    63 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.h   |     6 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c  |    88 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c |    47 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h |    15 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c |    47 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c  |    32 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c  |     8 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h  |     9 +
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c  |    21 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h  |     2 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.c    |    73 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.h    |    25 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c    |   109 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h    |    14 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_services.c |    35 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c   |    72 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h   |    16 +
+ .../drm/amd/display/amdgpu_dm/tests/.kunitconfig   |     1 -
+ .../gpu/drm/amd/display/amdgpu_dm/tests/Makefile   |     4 +-
+ .../amdgpu_dm/tests/amdgpu_dm_backlight_test.c     |    53 +-
+ .../amdgpu_dm/tests/amdgpu_dm_colorop_test.c       |   232 +-
+ .../amdgpu_dm/tests/amdgpu_dm_connector_test.c     |  4632 +-
+ .../display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c  |   813 +-
+ .../amdgpu_dm/tests/amdgpu_dm_cursor_test.c        |   736 +
+ .../display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c  |   578 +-
+ .../amdgpu_dm/tests/amdgpu_dm_freesync_test.c      |   429 +
+ .../amdgpu_dm/tests/amdgpu_dm_helpers_test.c       |  1129 +-
+ .../display/amdgpu_dm/tests/amdgpu_dm_irq_test.c   |  2308 +-
+ .../display/amdgpu_dm/tests/amdgpu_dm_ism_test.c   |   436 +-
+ .../amdgpu_dm/tests/amdgpu_dm_mst_types_test.c     |  2791 +-
+ .../display/amdgpu_dm/tests/amdgpu_dm_plane_test.c |  2660 +-
+ .../amdgpu_dm/tests/amdgpu_dm_services_test.c      |   228 +-
+ .../amd/display/amdgpu_dm/tests/amdgpu_dm_test.c   |  4282 +-
+ .../display/amdgpu_dm/tests/amdgpu_dm_wb_test.c    |   334 +
+ drivers/gpu/drm/amd/display/dc/Makefile            |     5 +-
+ .../drm/amd/display/dc/bios/command_table_helper.c |     7 +-
+ .../amd/display/dc/bios/command_table_helper2.c    |     2 +
+ drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile    |     8 +
+ drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c   |     4 +
+ .../amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c |    12 +-
+ .../amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.c   |   181 +
+ .../amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.h   |    29 +
+ .../drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c |     9 +-
+ .../drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c |     6 +-
+ .../amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c   |    12 +-
+ .../amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c |     6 +-
+ .../drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c  |     5 +-
+ .../amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c   |     8 +-
+ .../drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c |     8 +-
+ .../amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c   |    10 +-
+ .../amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c |    10 +-
+ .../amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c |    14 +-
+ .../amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c |    10 +-
+ .../amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c   |    18 +-
+ .../amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c   |    12 +-
+ .../amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c |    19 +-
+ .../amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c   |    33 +-
+ .../amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h   |     5 +
+ .../amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c |    54 +-
+ .../gpu/drm/amd/display/dc/clk_mgr/dcn60/dalsmc.h  |   345 +-
+ .../amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.c   |   341 +-
+ .../amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.h   |    17 +-
+ .../dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.c       |   184 +-
+ .../dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.h       |   167 +-
+ .../display/dc/clk_mgr/dcn60/dcn60_smu_driver_if.h |    78 +
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |   291 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_debug.c     |     2 +
+ .../gpu/drm/amd/display/dc/core/dc_hw_sequencer.c  |  1035 +-
+ .../gpu/drm/amd/display/dc/core/dc_link_exports.c  |    25 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_resource.c  |   243 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_state.c     |     6 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_stream.c    |   168 +-
+ drivers/gpu/drm/amd/display/dc/dc.h                |    64 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c       |   281 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h       |    58 +-
+ drivers/gpu/drm/amd/display/dc/dc_edid_parser.c    |    80 -
+ drivers/gpu/drm/amd/display/dc/dc_hw_types.h       |    21 +-
+ drivers/gpu/drm/amd/display/dc/dc_memory_pool.c    |   275 +
+ drivers/gpu/drm/amd/display/dc/dc_memory_pool.h    |   107 +
+ drivers/gpu/drm/amd/display/dc/dc_probe.h          |     1 +
+ drivers/gpu/drm/amd/display/dc/dc_stream.h         |    14 +-
+ drivers/gpu/drm/amd/display/dc/dc_types.h          |     6 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.c |     1 +
+ .../gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h |     4 +
+ .../drm/amd/display/dc/dccg/dcn201/dcn201_dccg.c   |     1 +
+ .../gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c |     1 +
+ .../gpu/drm/amd/display/dc/dccg/dcn30/dcn30_dccg.c |     2 +
+ .../drm/amd/display/dc/dccg/dcn301/dcn301_dccg.c   |     1 +
+ .../gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c |     1 +
+ .../drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c   |     1 +
+ .../gpu/drm/amd/display/dc/dccg/dcn32/dcn32_dccg.c |     1 +
+ .../gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c |     2 +
+ .../drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c   |     1 +
+ .../gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c |    15 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h |     2 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.c |     5 +
+ .../gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.h |     4 +
+ drivers/gpu/drm/amd/display/dc/dce/dce_abm.c       |     2 +
+ drivers/gpu/drm/amd/display/dc/dce/dce_aux.c       |    12 +-
+ .../gpu/drm/amd/display/dc/dce/dce_clock_source.c  |    11 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c      |   121 -
+ drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c      |     1 +
+ .../gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c  |    10 +-
+ drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c      |     1 +
+ drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h      |     1 +
+ drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c   |    28 +-
+ drivers/gpu/drm/amd/display/dc/dce/dmub_replay.h   |     3 +
+ drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c  |     2 +-
+ .../gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c |     8 +-
+ .../drm/amd/display/dc/dcn301/dcn301_panel_cntl.c  |     1 +
+ .../drm/amd/display/dc/dcn31/dcn31_panel_cntl.c    |     1 +
+ .../gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.c   |     1 +
+ .../amd/display/dc/dio/dcn10/dcn10_link_encoder.c  |    12 +-
+ .../display/dc/dio/dcn10/dcn10_stream_encoder.c    |     2 +
+ .../amd/display/dc/dio/dcn20/dcn20_link_encoder.c  |     2 +
+ .../display/dc/dio/dcn20/dcn20_stream_encoder.c    |     2 +
+ .../display/dc/dio/dcn30/dcn30_dio_link_encoder.c  |     2 +
+ .../dc/dio/dcn30/dcn30_dio_stream_encoder.c        |     2 +
+ .../dc/dio/dcn301/dcn301_dio_link_encoder.c        |     2 +
+ .../display/dc/dio/dcn31/dcn31_dio_link_encoder.c  |     2 +
+ .../dc/dio/dcn314/dcn314_dio_stream_encoder.c      |     2 +
+ .../display/dc/dio/dcn32/dcn32_dio_link_encoder.c  |     2 +
+ .../dc/dio/dcn32/dcn32_dio_stream_encoder.c        |     2 +
+ .../dc/dio/dcn321/dcn321_dio_link_encoder.c        |     2 +
+ .../display/dc/dio/dcn35/dcn35_dio_link_encoder.c  |     2 +
+ .../dc/dio/dcn35/dcn35_dio_stream_encoder.c        |     2 +
+ .../dc/dio/dcn401/dcn401_dio_link_encoder.c        |     2 +
+ .../dc/dio/dcn401/dcn401_dio_stream_encoder.c      |     1 +
+ .../display/dc/dio/dcn42/dcn42_dio_link_encoder.c  |     6 +-
+ .../dc/dio/dcn42/dcn42_dio_stream_encoder.c        |     2 +
+ .../display/dc/dio/dcn60/dcn60_dio_link_encoder.c  |     2 +
+ .../dc/dio/dcn60/dcn60_dio_stream_encoder.c        |     2 +
+ .../display/dc/dio/virtual/virtual_link_encoder.c  |     3 +
+ .../dc/dio/virtual/virtual_stream_encoder.c        |     1 +
+ .../gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c   |     1 -
+ .../gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c   |    18 +-
+ .../amd/display/dc/dml/dcn30/display_mode_vba_30.c |     2 +-
+ .../amd/display/dc/dml/dcn31/display_mode_vba_31.c |     2 +-
+ .../gpu/drm/amd/display/dc/dml/dml1_frl_cap_chk.c  |     8 +-
+ .../gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c   |     1 -
+ drivers/gpu/drm/amd/display/dc/dml2_0/Makefile     |    17 +-
+ .../amd/display/dc/dml2_0/dml21/dml21_wrapper.h    |   106 -
+ .../dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h |    39 +-
+ .../dml21/inc/bounding_boxes/dcn42b_soc_bb.h       |    37 +-
+ .../dml2_0/dml21/inc/bounding_boxes/dcn4_soc_bb.h  |    27 +-
+ .../dml2_0/dml21/inc/bounding_boxes/dcn6_soc_bb.h  |    12 +
+ .../dc/dml2_0/dml21/inc/dml_top_dchub_registers.h  |     1 +
+ .../dml2_0/dml21/inc/dml_top_display_cfg_types.h   |     1 +
+ .../dml2_0/dml21/inc/dml_top_soc_parameter_types.h |    28 +-
+ .../display/dc/dml2_0/dml21/inc/dml_top_types.h    |    14 +
+ .../dc/dml2_0/dml21/src/dml2_cga/dml2_cga_dcn6.c   |     4 +-
+ .../dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c |     4 +
+ .../dml21/src/dml2_core/dml2_core_dcn4_calcs.c     |   222 +-
+ .../dml21/src/dml2_core/dml2_core_dcn4_calcs.h     |     1 +
+ .../src/dml2_core/dml2_core_dcn5_calcs_dchub.c     |    18 +-
+ .../src/dml2_core/dml2_core_dcn5_calcs_dchub.h     |    23 +
+ .../dml2_core/dml2_core_dcn5_calcs_display_pipe.c  |     2 +-
+ .../dml2_core/dml2_core_dcn5_funcs_initialize.c    |     2 +-
+ .../dml2_core_dcn5_funcs_mode_programming.c        |     4 +
+ .../dml2_core/dml2_core_dcn5_funcs_mode_support.c  |    32 +-
+ .../dml21/src/dml2_core/dml2_core_dcn6_calcs.c     |    63 +-
+ .../dml21/src/dml2_core/dml2_core_dcn6_calcs.h     |   468 +
+ .../src/dml2_core/dml2_core_dcn6_calcs_dchub.c     |    86 +-
+ .../dml2_core/dml2_core_dcn6_funcs_initialize.c    |     4 +
+ .../dml2_core_dcn6_funcs_mode_programming.c        |    52 +-
+ .../dml2_core_dcn6_funcs_mode_programming.h        |    31 +
+ .../dml2_core/dml2_core_dcn6_funcs_mode_support.c  |   350 +-
+ .../dml2_core/dml2_core_dcn6_funcs_mode_support.h  |   324 +
+ .../dml21/src/dml2_core/dml2_core_shared_types.h   |    11 +-
+ .../dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c |   106 +-
+ .../dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c       |    11 +-
+ .../alternate_pstate_shared_lib.c                  |    22 +-
+ .../dc/dml2_0/dml21/src/dml2_top/dml2_top_utm.c    |     4 +-
+ .../src/dml2_utm_soc_bb/dml2_utm_soc_bb_dcn6.c     |    15 +-
+ .../dml21/src/inc/dml2_internal_shared_types.h     |    19 +
+ .../gpu/drm/amd/display/dc/dml2_wrapper/Makefile   |    61 +
+ .../dml21_wrapper}/dml21_translation_helper.c      |    46 +-
+ .../dml21_wrapper}/dml21_translation_helper.h      |     7 +-
+ .../dml21_wrapper}/dml21_utils.c                   |    24 +-
+ .../dml21_wrapper}/dml21_utils.h                   |     7 +-
+ .../dml21_wrapper}/dml21_wrapper.c                 |     2 +-
+ .../dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.h  |   107 +
+ .../dml21_wrapper}/dml21_wrapper_fpu.c             |    19 +-
+ .../dml21_wrapper}/dml21_wrapper_fpu.h             |     7 +-
+ .../dml2_dc_resource_mgmt.c                        |    60 +-
+ .../dml2_dc_resource_mgmt.h                        |     0
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_dc_types.h    |     0
+ .../{dml2_0 => dml2_wrapper}/dml2_internal_types.h |     2 +-
+ .../{dml2_0 => dml2_wrapper}/dml2_mall_phantom.c   |    70 +-
+ .../{dml2_0 => dml2_wrapper}/dml2_mall_phantom.h   |     0
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_policy.c      |     2 +-
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_policy.h      |     0
+ .../dml2_translation_helper.c                      |    20 +-
+ .../dml2_translation_helper.h                      |     0
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_utils.c       |    28 +-
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_utils.h       |     0
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_wrapper.c     |     6 +-
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_wrapper.h     |     0
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_wrapper_fpu.c |    45 +-
+ .../dc/{dml2_0 => dml2_wrapper}/dml2_wrapper_fpu.h |     0
+ .../drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c    |     2 +
+ .../gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h |     2 +
+ .../drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c  |    19 +
+ .../gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c   |    20 +-
+ .../gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h   |    14 +-
+ .../gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c   |     1 +
+ .../gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h   |     7 +-
+ drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c        |     8 +
+ .../gpu/drm/amd/display/dc/dsc/dcn60/dcn60_dsc.c   |     4 +-
+ drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c       |     7 +-
+ drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c   |     4 +
+ drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c |     4 +
+ .../dc/hpo/dcn30/dcn30_hpo_frl_stream_encoder.c    |     8 +-
+ .../dc/hpo/dcn401/dcn401_hpo_frl_stream_encoder.c  |     2 +
+ .../dc/hpo/dcn42/dcn42_hpo_frl_stream_encoder.c    |    11 +-
+ .../dc/hpo/dcn60/dcn60_hpo_frl_stream_encoder.c    |   142 +-
+ .../drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c |     2 +
+ .../drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h |    11 +-
+ .../drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c |     7 +-
+ .../amd/display/dc/hubbub/dcn201/dcn201_hubbub.c   |     2 +
+ .../drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c |     2 +
+ .../drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c |     2 +
+ .../amd/display/dc/hubbub/dcn301/dcn301_hubbub.c   |     2 +
+ .../drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c |     6 +-
+ .../drm/amd/display/dc/hubbub/dcn32/dcn32_hubbub.c |     2 +
+ .../drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c |    16 +-
+ .../amd/display/dc/hubbub/dcn401/dcn401_hubbub.c   |     2 +
+ .../drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c |    15 +-
+ .../drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c |   241 +-
+ .../drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.h |    19 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c |     4 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.h |     3 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c |     5 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h |     3 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c |    11 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.h |     2 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c |     7 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.h |     3 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn31/dcn31_hubp.c |     1 +
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c   |   114 +-
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h   |     5 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c |     6 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.c |    28 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.h |     3 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn60/dcn60_hubp.c |     4 +
+ drivers/gpu/drm/amd/display/dc/hwss/Makefile       |     6 +
+ .../gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.c    |    46 +-
+ .../gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h    |    12 +-
+ .../drm/amd/display/dc/hwss/dce110/dce110_hwseq.c  |    35 +-
+ .../drm/amd/display/dc/hwss/dce60/dce60_hwseq.c    |     8 +-
+ .../drm/amd/display/dc/hwss/dce80/dce80_hwseq.c    |     3 +-
+ .../drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c    |   280 +-
+ .../drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.h    |    22 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c |     6 +-
+ .../drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c    |   291 +-
+ .../drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.h    |    26 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c |     5 +-
+ .../drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c  |    61 +-
+ .../drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.h  |     7 +-
+ .../drm/amd/display/dc/hwss/dcn201/dcn201_init.c   |     6 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn21/dcn21_init.c |     5 +-
+ .../drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c    |    57 +-
+ .../drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h    |     8 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c |     5 +-
+ .../drm/amd/display/dc/hwss/dcn301/dcn301_init.c   |     5 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c |     5 +-
+ .../drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c  |     8 +-
+ .../drm/amd/display/dc/hwss/dcn314/dcn314_init.c   |     5 +-
+ .../drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c    |   104 +-
+ .../drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.h    |    10 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c |     5 +-
+ .../drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c    |   128 +-
+ .../drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h    |    14 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c |     5 +-
+ .../drm/amd/display/dc/hwss/dcn351/dcn351_init.c   |     5 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c  |   314 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h  |    24 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_init.c   |     5 +-
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c    |   330 +-
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h    |    21 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c |    10 +-
+ .../drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c    |    19 +-
+ .../drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c    |   244 +-
+ .../drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h    |     5 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c |    12 +-
+ drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h |   259 +-
+ .../drm/amd/display/dc/hwss/hw_sequencer_private.h |    28 +-
+ drivers/gpu/drm/amd/display/dc/inc/clock_source.h  |     4 +-
+ drivers/gpu/drm/amd/display/dc/inc/core_status.h   |     2 +
+ drivers/gpu/drm/amd/display/dc/inc/core_types.h    |    13 +-
+ drivers/gpu/drm/amd/display/dc/inc/custom_float.h  |    21 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/abm.h        |     1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h    |     1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h       |     1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h   |     2 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/dio.h        |     1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h       |    10 -
+ drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h        |     5 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h       |     6 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h  |    10 -
+ .../gpu/drm/amd/display/dc/inc/hw/link_encoder.h   |     1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h        |    65 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/opp.h        |    14 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h    |     1 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/rmcm.h       |   140 +
+ .../gpu/drm/amd/display/dc/inc/hw/stream_encoder.h |     2 +
+ .../drm/amd/display/dc/inc/hw/timing_generator.h   |     1 +
+ drivers/gpu/drm/amd/display/dc/inc/link_service.h  |     9 +-
+ drivers/gpu/drm/amd/display/dc/inc/resource.h      |    20 +-
+ drivers/gpu/drm/amd/display/dc/irq/Makefile        |     2 +
+ .../amd/display/dc/irq/dce110/irq_service_dce110.c |    21 -
+ .../amd/display/dc/irq/dce110/irq_service_dce110.h |     9 -
+ drivers/gpu/drm/amd/display/dc/irq/irq_service.c   |    28 +
+ drivers/gpu/drm/amd/display/dc/irq/irq_service.h   |     9 +
+ .../amd/display/dc/link/hwss/link_hwss_hpo_dp.c    |    23 +-
+ .../gpu/drm/amd/display/dc/link/link_detection.c   |     4 +-
+ drivers/gpu/drm/amd/display/dc/link/link_factory.c |     7 +-
+ .../display/dc/link/protocols/link_dp_capability.c |     3 +-
+ .../dc/link/protocols/link_dp_panel_replay.c       |    36 +-
+ .../dc/link/protocols/link_dp_panel_replay.h       |     2 +-
+ .../dc/link/protocols/link_edp_panel_control.c     |   103 +-
+ .../dc/link/protocols/link_edp_panel_control.h     |     6 +
+ .../amd/display/dc/link/protocols/link_hdmi_frl.c  |    32 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c   |     1 +
+ .../gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c   |     1 +
+ .../gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c   |     1 +
+ .../gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c   |     1 +
+ .../gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c |     1 +
+ .../gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c   |   680 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h   |   714 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.c   |    41 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.h   |   238 -
+ .../gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c   |     2 +
+ .../gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c |    10 +
+ .../gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.h |     1 +
+ .../drm/amd/display/dc/optc/dcn314/dcn314_optc.c   |     1 +
+ .../gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c |     1 +
+ .../gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c |     1 +
+ drivers/gpu/drm/amd/display/dc/os_types.h          |    19 +
+ .../drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c    |     1 +
+ .../drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c    |     5 +
+ drivers/gpu/drm/amd/display/dc/resource/Makefile   |     4 +
+ .../display/dc/resource/dce112/dce112_resource.c   |    62 -
+ .../amd/display/dc/resource/dcn32/dcn32_resource.c |     2 +-
+ .../amd/display/dc/resource/dcn35/dcn35_resource.c |     2 +-
+ .../display/dc/resource/dcn351/dcn351_resource.c   |     2 +-
+ .../amd/display/dc/resource/dcn36/dcn36_resource.c |     2 +-
+ .../display/dc/resource/dcn401/dcn401_resource.c   |     2 +-
+ .../amd/display/dc/resource/dcn42/dcn42_resource.c |    62 +-
+ .../display/dc/resource/dcn42b/dcn42b_resource.c   |   129 +-
+ .../amd/display/dc/resource/dcn60/dcn60_resource.c |    64 +-
+ .../amd/display/dc/resource/dcn60/dcn60_resource.h |     9 +-
+ drivers/gpu/drm/amd/display/dc/rmcm/Makefile       |    44 +
+ .../gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.c |   867 +
+ .../gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.h |   717 +
+ .../gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.c |   251 +
+ .../{dc_edid_parser.h => rmcm/dcn60/dcn60_rmcm.h}  |    23 +-
+ .../dcn401/dcn401_soc_and_ip_translator.c          |     6 +-
+ .../dcn60/dcn60_soc_and_ip_translator.c            |     6 +-
+ drivers/gpu/drm/amd/display/dc/sspl/Makefile       |     5 +-
+ drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c       |   130 +-
+ .../amd/display/dc/sspl/dc_spl_isharp_filters.c    |    10 +-
+ .../amd/display/dc/sspl/dc_spl_scl_easf_filters.c  |    88 +-
+ .../drm/amd/display/dc/sspl/dc_spl_scl_filters.c   |    24 +-
+ drivers/gpu/drm/amd/display/dc/sspl/dc_spl_types.h |    60 +-
+ .../gpu/drm/amd/display/dc/sspl/spl_custom_float.c |   152 -
+ .../gpu/drm/amd/display/dc/sspl/spl_custom_float.h |    29 -
+ .../gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c   |   495 -
+ .../gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h   |   526 -
+ .../gpu/drm/amd/display/dc/sspl/spl_namespace.h    |    17 +
+ drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h |    43 +-
+ drivers/gpu/drm/amd/display/dmub/dmub_srv.h        |    60 +-
+ drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h    |   189 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c  |    74 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h  |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c  |    82 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.h  |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c  |    82 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.h  |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c  |    82 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.h  |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c |    86 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.h |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c  |    86 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h  |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.c  |    86 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.h  |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c    |    63 +-
+ drivers/gpu/drm/amd/display/include/dal_asic_id.h  |     5 +
+ .../drm/amd/display/modules/color/color_gamma.c    |     3 +-
+ .../drm/amd/display/modules/inc/mod_info_packet.h  |     4 +
+ .../gpu/drm/amd/display/modules/inc/mod_power.h    |     4 +
+ .../amd/display/modules/info_packet/info_packet.c  |   109 +
+ .../drm/amd/display/modules/power/power_replay.c   |    13 +
+ drivers/gpu/drm/amd/include/amd_shared.h           |     6 +-
+ .../drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h |     2 +
+ .../amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h    |     4 +-
+ .../amd/include/asic_reg/nbio/nbio_7_11_5_offset.h | 11074 ++++
+ .../include/asic_reg/nbio/nbio_7_11_5_sh_mask.h    | 63248 +++++++++++++++++++
+ drivers/gpu/drm/amd/include/discovery.h            |   179 +-
+ .../amd/include/ivsrcid/mpnht/irqsrcs_mpnht_15_0.h |    30 +
+ drivers/gpu/drm/amd/include/kgd_kfd_interface.h    |     4 +-
+ drivers/gpu/drm/amd/include/kgd_pp_interface.h     |    17 +
+ drivers/gpu/drm/amd/include/v12_structs.h          |     2 +-
+ drivers/gpu/drm/amd/pm/amdgpu_dpm.c                |    26 +-
+ drivers/gpu/drm/amd/pm/amdgpu_pm.c                 |    56 +-
+ drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h            |     1 +
+ drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c         |     6 -
+ drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c         |     7 -
+ drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c   |     6 -
+ .../gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c    |     3 +
+ drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c          |    49 +-
+ drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h      |    23 +-
+ .../pm/swsmu/inc/pmfw_if/smu15_driver_if_v15_0_0.h |    46 -
+ .../amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_ppsmc.h  |     1 +
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h       |     1 +
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h       |     6 -
+ drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c     |     2 -
+ drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c     |     6 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c   |     2 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c  |    24 +
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c   |    10 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h   |     1 +
+ drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c     |     8 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c     |   174 +-
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c   |   270 +-
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c   |    16 +-
+ drivers/gpu/drm/amd/ras/core/Makefile              |     9 +-
+ drivers/gpu/drm/amd/ras/core/aca.c                 |   353 +-
+ drivers/gpu/drm/amd/ras/core/aca.h                 |    44 +-
+ drivers/gpu/drm/amd/ras/core/aca_v1_0.c            |    41 +-
+ drivers/gpu/drm/amd/ras/core/aca_v5_0.c            |   448 +
+ drivers/gpu/drm/amd/ras/core/aca_v5_0.h            |    62 +
+ drivers/gpu/drm/amd/ras/core/cmd.c                 |   147 +-
+ drivers/gpu/drm/amd/ras/core/cmd.h                 |    32 +-
+ drivers/gpu/drm/amd/ras/core/core.c                |   309 +-
+ drivers/gpu/drm/amd/ras/core/eeprom.c              |   504 +-
+ drivers/gpu/drm/amd/ras/core/eeprom.h              |    27 +-
+ drivers/gpu/drm/amd/ras/core/eeprom_fw.c           |   640 +-
+ drivers/gpu/drm/amd/ras/core/eeprom_fw.h           |    67 +-
+ drivers/gpu/drm/amd/ras/core/log_ring.c            |    56 +-
+ drivers/gpu/drm/amd/ras/core/log_ring.h            |    46 +-
+ drivers/gpu/drm/amd/ras/core/ras.h                 |   138 +-
+ drivers/gpu/drm/amd/ras/core/ras_bert.c            |   546 +
+ drivers/gpu/drm/amd/ras/core/ras_bert.h            |    33 +
+ drivers/gpu/drm/amd/ras/core/ras_cper.c            |   844 +-
+ drivers/gpu/drm/amd/ras/core/ras_cper.h            |   177 +-
+ drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c      |   418 +
+ drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.h      |   124 +
+ drivers/gpu/drm/amd/ras/core/ras_gfx.c             |    10 +-
+ drivers/gpu/drm/amd/ras/core/ras_mce.c             |   151 +
+ drivers/gpu/drm/amd/ras/core/ras_mce.h             |    51 +
+ drivers/gpu/drm/amd/ras/core/ras_mp1.c             |   169 +-
+ drivers/gpu/drm/amd/ras/core/ras_mp1.h             |    87 +-
+ drivers/gpu/drm/amd/ras/core/ras_mp1_v13_0.c       |   175 +-
+ drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.c       |   249 +
+ drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.h       |    30 +
+ drivers/gpu/drm/amd/ras/core/ras_nbio.c            |     9 +-
+ drivers/gpu/drm/amd/ras/core/ras_process.c         |    36 +-
+ drivers/gpu/drm/amd/ras/core/ras_psp.c             |   599 +-
+ drivers/gpu/drm/amd/ras/core/ras_psp.h             |   111 +-
+ drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c       |    76 +
+ drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.c       |   133 +
+ drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.h       |    31 +
+ drivers/gpu/drm/amd/ras/core/ras_umc.c             |   635 +-
+ drivers/gpu/drm/amd/ras/core/ras_umc.h             |    52 +-
+ drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c       |   143 +-
+ drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.h       |     3 -
+ drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c       |   220 +
+ drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.h       |    69 +
+ drivers/gpu/drm/amd/ras/core/ta_if.h               |    35 +
+ drivers/gpu/drm/amd/ras/ras_mgr/Makefile           |     6 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.c  |   191 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.h  |    30 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c   |    38 +
+ .../drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c    |   141 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.c   |   267 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.h   |    31 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c   |   357 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h   |     8 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.c   |   105 +
+ .../{amdgpu_ras_mp1_v13_0.h => amdgpu_ras_mp1.h}   |     8 +-
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c |   154 -
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c   |     9 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c   |   100 +-
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c  |    42 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h          |     9 +-
+ .../gpu/drm/arm/display/include/malidp_product.h   |    10 +-
+ drivers/gpu/drm/arm/display/komeda/komeda_crtc.c   |    27 +-
+ drivers/gpu/drm/arm/display/komeda/komeda_plane.c  |    18 +-
+ .../drm/arm/display/komeda/komeda_wb_connector.c   |    33 +
+ drivers/gpu/drm/arm/hdlcd_crtc.c                   |     4 +-
+ drivers/gpu/drm/arm/malidp_crtc.c                  |    19 +-
+ drivers/gpu/drm/arm/malidp_planes.c                |    18 +-
+ drivers/gpu/drm/armada/armada_crtc.c               |     2 +-
+ drivers/gpu/drm/armada/armada_overlay.c            |    39 +-
+ drivers/gpu/drm/armada/armada_plane.c              |    15 +-
+ drivers/gpu/drm/armada/armada_plane.h              |     2 +-
+ drivers/gpu/drm/aspeed/aspeed_gfx.h                |    11 +-
+ drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c           |   203 +-
+ drivers/gpu/drm/aspeed/aspeed_gfx_drv.c            |     3 +-
+ drivers/gpu/drm/ast/ast_dp.c                       |    24 +-
+ drivers/gpu/drm/ast/ast_mode.c                     |    20 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c     |    19 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c    |    33 +-
+ drivers/gpu/drm/bridge/Kconfig                     |     7 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.c |    88 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.h |     4 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c  |    15 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h  |     4 +
+ .../gpu/drm/bridge/cadence/cdns-mhdp8546-core.c    |     1 -
+ drivers/gpu/drm/bridge/chipone-icn6211.c           |     8 +-
+ drivers/gpu/drm/bridge/ite-it6505.c                |     4 +-
+ drivers/gpu/drm/bridge/lontium-lt8713sx.c          |     2 +-
+ drivers/gpu/drm/bridge/lontium-lt9211.c            |     2 +-
+ drivers/gpu/drm/bridge/lontium-lt9611.c            |     6 +-
+ drivers/gpu/drm/bridge/lontium-lt9611uxc.c         |     2 +-
+ drivers/gpu/drm/bridge/samsung-dsim.c              |     4 +-
+ drivers/gpu/drm/bridge/sil-sii8620.c               |     3 +-
+ drivers/gpu/drm/bridge/synopsys/dw-dp.c            |     4 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c       |   209 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi.c          |     2 +-
+ drivers/gpu/drm/bridge/tc358767.c                  |     4 +-
+ drivers/gpu/drm/bridge/ti-sn65dsi83.c              |    79 +-
+ drivers/gpu/drm/clients/drm_fbdev_client.c         |    23 +-
+ drivers/gpu/drm/clients/drm_log.c                  |    54 +-
+ drivers/gpu/drm/display/drm_bridge_connector.c     |   186 +-
+ drivers/gpu/drm/display/drm_hdmi_helper.c          |   304 +
+ drivers/gpu/drm/display/drm_hdmi_state_helper.c    |   239 +-
+ drivers/gpu/drm/display/drm_scdc_helper.c          |   359 +-
+ drivers/gpu/drm/drm_atomic.c                       |    36 +-
+ drivers/gpu/drm/drm_atomic_helper.c                |    10 +-
+ drivers/gpu/drm/drm_atomic_state_helper.c          |    86 -
+ drivers/gpu/drm/drm_atomic_uapi.c                  |     7 +
+ drivers/gpu/drm/drm_bridge.c                       |    27 +-
+ drivers/gpu/drm/drm_bridge_helper.c                |     2 -
+ drivers/gpu/drm/drm_buddy.c                        |     3 +-
+ drivers/gpu/drm/drm_client_event.c                 |    18 +
+ drivers/gpu/drm/drm_colorop.c                      |   107 +
+ drivers/gpu/drm/drm_connector.c                    |   153 +-
+ drivers/gpu/drm/drm_crtc_internal.h                |     2 -
+ drivers/gpu/drm/drm_debugfs.c                      |   157 -
+ drivers/gpu/drm/drm_drv.c                          |    34 +-
+ drivers/gpu/drm/drm_edid.c                         |   237 +-
+ drivers/gpu/drm/drm_gem.c                          |    19 +-
+ drivers/gpu/drm/drm_gem_atomic_helper.c            |   100 +-
+ drivers/gpu/drm/drm_gpusvm.c                       |   410 +-
+ drivers/gpu/drm/drm_kms_helper_common.c            |    14 +
+ drivers/gpu/drm/drm_mode_config.c                  |    21 +-
+ drivers/gpu/drm/drm_modeset_helper.c               |     2 +
+ drivers/gpu/drm/drm_of.c                           |    38 +-
+ drivers/gpu/drm/drm_pagemap.c                      |     5 +-
+ drivers/gpu/drm/drm_panic.c                        |   927 +-
+ drivers/gpu/drm/drm_panic_helper.c                 |   919 +
+ .../{drm_panic_qr.rs => drm_panic_helper_qr.rs}    |     4 +-
+ drivers/gpu/drm/drm_panic_internal.h               |    69 +
+ drivers/gpu/drm/drm_probe_helper.c                 |    15 +-
+ drivers/gpu/drm/drm_ras.c                          |   278 +-
+ drivers/gpu/drm/drm_ras_nl.c                       |    33 +
+ drivers/gpu/drm/drm_ras_nl.h                       |     8 +
+ drivers/gpu/drm/drm_simple_kms_helper.c            |    37 +-
+ drivers/gpu/drm/drm_sysfs.c                        |    32 -
+ drivers/gpu/drm/exynos/exynos_drm_crtc.c           |     2 +-
+ drivers/gpu/drm/exynos/exynos_drm_dpi.c            |     3 +-
+ drivers/gpu/drm/exynos/exynos_drm_plane.c          |    22 +-
+ drivers/gpu/drm/exynos/exynos_drm_vidi.c           |     3 +-
+ drivers/gpu/drm/exynos/exynos_hdmi.c               |     3 +-
+ drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c         |     2 +-
+ drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c        |     2 +-
+ drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c          |    10 +-
+ drivers/gpu/drm/gma500/psb_intel_display.c         |    34 +-
+ drivers/gpu/drm/gud/gud_drv.c                      |     2 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c     |    18 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c    |     5 +-
+ drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c       |     9 +-
+ drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c    |     4 +-
+ drivers/gpu/drm/hyperv/hyperv_drm.h                |     1 -
+ drivers/gpu/drm/hyperv/hyperv_drm_modeset.c        |     4 +-
+ drivers/gpu/drm/hyperv/hyperv_drm_proto.c          |    44 +-
+ drivers/gpu/drm/i915/display/i9xx_plane.c          |     3 +
+ drivers/gpu/drm/i915/display/intel_atomic.c        |     1 +
+ drivers/gpu/drm/i915/display/intel_audio.c         |   153 +
+ drivers/gpu/drm/i915/display/intel_audio_regs.h    |    16 +-
+ drivers/gpu/drm/i915/display/intel_cdclk.c         |   326 +-
+ drivers/gpu/drm/i915/display/intel_cmtg.c          |     2 +-
+ .../gpu/drm/i915/display/intel_crtc_state_dump.c   |     3 +
+ drivers/gpu/drm/i915/display/intel_cursor.c        |   296 +-
+ drivers/gpu/drm/i915/display/intel_cursor_regs.h   |     8 +-
+ drivers/gpu/drm/i915/display/intel_ddi.c           |    36 +-
+ drivers/gpu/drm/i915/display/intel_display.c       |    76 +-
+ drivers/gpu/drm/i915/display/intel_display.h       |     3 +-
+ .../drm/i915/display/intel_display_clock_gating.c  |    67 +-
+ .../drm/i915/display/intel_display_clock_gating.h  |    16 +-
+ .../gpu/drm/i915/display/intel_display_debugfs.c   |    21 +-
+ .../drm/i915/display/intel_display_power_well.c    |     6 +-
+ drivers/gpu/drm/i915/display/intel_display_regs.h  |    18 +-
+ drivers/gpu/drm/i915/display/intel_display_types.h |    17 +-
+ drivers/gpu/drm/i915/display/intel_display_wa.c    |     4 +-
+ drivers/gpu/drm/i915/display/intel_display_wa.h    |     9 -
+ drivers/gpu/drm/i915/display/intel_dp.c            |    52 +-
+ drivers/gpu/drm/i915/display/intel_dp_mst.c        |     4 +
+ drivers/gpu/drm/i915/display/intel_dpll.c          |    22 +-
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.c      |   221 +-
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.h      |    22 +
+ drivers/gpu/drm/i915/display/intel_frontbuffer.c   |     3 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.c          |    53 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.h          |    11 +-
+ drivers/gpu/drm/i915/display/intel_hti.c           |     3 -
+ .../gpu/drm/i915/display/intel_modeset_verify.c    |     1 -
+ drivers/gpu/drm/i915/display/intel_parent.c        |    10 +-
+ drivers/gpu/drm/i915/display/intel_parent.h        |     3 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           |    18 +-
+ drivers/gpu/drm/i915/display/intel_snps_phy.c      |    60 +-
+ drivers/gpu/drm/i915/display/intel_snps_phy.h      |     2 +
+ drivers/gpu/drm/i915/display/intel_tdf.h           |    25 -
+ drivers/gpu/drm/i915/display/intel_vrr.c           |   395 +-
+ drivers/gpu/drm/i915/display/intel_vrr.h           |     2 +
+ drivers/gpu/drm/i915/display/skl_universal_plane.c |     4 +
+ drivers/gpu/drm/i915/gvt/handlers.c                |     2 +-
+ drivers/gpu/drm/i915/i915_dpt.c                    |     1 -
+ drivers/gpu/drm/i915/i915_reg.h                    |     2 +-
+ drivers/gpu/drm/i915/i915_switcheroo.c             |    11 +-
+ drivers/gpu/drm/i915/intel_clock_gating.c          |    30 +-
+ drivers/gpu/drm/i915/intel_gvt_mmio_table.c        |     2 +-
+ drivers/gpu/drm/i915/intel_pcode.c                 |    14 +-
+ drivers/gpu/drm/i915/intel_pcode.h                 |     2 +-
+ drivers/gpu/drm/imagination/pvr_ccb.c              |    43 +-
+ drivers/gpu/drm/imagination/pvr_device.c           |    25 +-
+ drivers/gpu/drm/imagination/pvr_fw.c               |    23 +-
+ drivers/gpu/drm/imagination/pvr_fw.h               |     2 +-
+ drivers/gpu/drm/imagination/pvr_fw_meta.c          |    18 +-
+ drivers/gpu/drm/imagination/pvr_fw_riscv.c         |    40 +-
+ drivers/gpu/drm/imagination/pvr_fw_startstop.c     |    22 +-
+ drivers/gpu/drm/imagination/pvr_fw_trace.c         |     1 -
+ drivers/gpu/drm/imagination/pvr_rogue_cr_defs.h    |   920 +-
+ drivers/gpu/drm/imagination/pvr_rogue_defs.h       |    22 +-
+ drivers/gpu/drm/imagination/pvr_rogue_riscv.h      |     2 -
+ drivers/gpu/drm/imagination/pvr_trace.h            |    14 +-
+ drivers/gpu/drm/imx/dc/dc-crtc.c                   |     2 +-
+ drivers/gpu/drm/imx/dc/dc-kms.c                    |     8 +-
+ drivers/gpu/drm/imx/dc/dc-plane.c                  |     2 +-
+ drivers/gpu/drm/imx/dcss/dcss-crtc.c               |     2 +-
+ drivers/gpu/drm/imx/dcss/dcss-plane.c              |     2 +-
+ drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c             |    18 +-
+ drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c            |    21 +-
+ drivers/gpu/drm/ingenic/ingenic-drm-drv.c          |     4 +-
+ drivers/gpu/drm/ingenic/ingenic-ipu.c              |     2 +-
+ drivers/gpu/drm/kmb/kmb_crtc.c                     |     2 +-
+ drivers/gpu/drm/kmb/kmb_dsi.c                      |     9 +-
+ drivers/gpu/drm/kmb/kmb_plane.c                    |     2 +-
+ drivers/gpu/drm/logicvc/logicvc_crtc.c             |     2 +-
+ drivers/gpu/drm/logicvc/logicvc_layer.c            |     2 +-
+ drivers/gpu/drm/loongson/lsdc_crtc.c               |    30 +-
+ drivers/gpu/drm/loongson/lsdc_plane.c              |     2 +-
+ drivers/gpu/drm/mcde/mcde_display.c                |   272 +-
+ drivers/gpu/drm/mcde/mcde_drm.h                    |    12 +-
+ drivers/gpu/drm/mcde/mcde_drv.c                    |     3 +-
+ drivers/gpu/drm/mcde/mcde_dsi.c                    |     8 +-
+ drivers/gpu/drm/mediatek/mtk_crtc.c                |    18 +-
+ drivers/gpu/drm/mediatek/mtk_dsi.c                 |    10 +-
+ drivers/gpu/drm/mediatek/mtk_plane.c               |    21 +-
+ drivers/gpu/drm/meson/meson_crtc.c                 |     2 +-
+ drivers/gpu/drm/meson/meson_encoder_cvbs.c         |    11 +-
+ drivers/gpu/drm/meson/meson_encoder_dsi.c          |    11 +-
+ drivers/gpu/drm/meson/meson_encoder_hdmi.c         |    11 +-
+ drivers/gpu/drm/meson/meson_overlay.c              |     2 +-
+ drivers/gpu/drm/meson/meson_plane.c                |     2 +-
+ drivers/gpu/drm/mgag200/mgag200_drv.h              |     8 +-
+ drivers/gpu/drm/mgag200/mgag200_mode.c             |    15 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c           |    18 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c          |    18 +-
+ drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c          |     2 +-
+ drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c         |     2 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c          |    20 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c         |    16 +-
+ drivers/gpu/drm/msm/msm_gem_shrinker.c             |    22 +-
+ drivers/gpu/drm/mxsfb/lcdif_kms.c                  |    19 +-
+ drivers/gpu/drm/mxsfb/mxsfb_kms.c                  |     6 +-
+ drivers/gpu/drm/nouveau/dispnv04/dfp.c             |     5 +-
+ drivers/gpu/drm/nouveau/dispnv50/disp.c            |     4 +-
+ drivers/gpu/drm/nouveau/dispnv50/head.c            |    14 +-
+ drivers/gpu/drm/nouveau/dispnv50/headca7d.c        |    21 +-
+ drivers/gpu/drm/nouveau/dispnv50/wndw.c            |    17 +-
+ .../gpu/drm/nouveau/include/nvhw/class/clca7d.h    |     4 +
+ drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h  |     1 +
+ drivers/gpu/drm/nouveau/nouveau_abi16.c            |     4 +
+ drivers/gpu/drm/nouveau/nouveau_acpi.c             |    32 +-
+ drivers/gpu/drm/nouveau/nouveau_acpi.h             |    10 +-
+ drivers/gpu/drm/nouveau/nouveau_bios.c             |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_connector.c        |   152 +-
+ drivers/gpu/drm/nouveau/nouveau_connector.h        |    12 +-
+ drivers/gpu/drm/nouveau/nouveau_dp.c               |     6 +-
+ drivers/gpu/drm/nouveau/nouveau_drm.c              |   114 +-
+ drivers/gpu/drm/nouveau/nouveau_exec.c             |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_fence.c            |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_led.c              |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_svm.c              |    13 +
+ drivers/gpu/drm/nouveau/nouveau_vga.c              |    28 +-
+ drivers/gpu/drm/nouveau/nvkm/engine/device/base.c  |     2 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c    |     2 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h    |     4 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild     |     1 +
+ drivers/gpu/drm/nouveau/nvkm/subdev/pci/mcp79.c    |    35 +
+ drivers/gpu/drm/omapdrm/dss/dsi.c                  |   361 +-
+ drivers/gpu/drm/omapdrm/dss/hdmi.h                 |     6 +
+ drivers/gpu/drm/omapdrm/dss/hdmi4.c                |    29 +-
+ drivers/gpu/drm/omapdrm/dss/hdmi5.c                |    37 +-
+ drivers/gpu/drm/omapdrm/dss/hdmi_common.c          |    14 +
+ drivers/gpu/drm/omapdrm/omap_crtc.c                |    17 +-
+ drivers/gpu/drm/omapdrm/omap_drv.c                 |     1 -
+ drivers/gpu/drm/omapdrm/omap_plane.c               |    13 +-
+ drivers/gpu/drm/panel/Kconfig                      |    25 +
+ drivers/gpu/drm/panel/Makefile                     |     2 +
+ drivers/gpu/drm/panel/panel-edp.c                  |     5 +
+ drivers/gpu/drm/panel/panel-himax-hx83102.c        |    24 +-
+ drivers/gpu/drm/panel/panel-himax-hx83112a.c       |    26 +-
+ drivers/gpu/drm/panel/panel-himax-hx83112b.c       |    23 +-
+ drivers/gpu/drm/panel/panel-himax-hx8394.c         |    26 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili7836a.c      |   297 +
+ drivers/gpu/drm/panel/panel-ilitek-ili9805.c       |    21 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c |    12 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.h |     1 -
+ drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c  |    16 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9806e-spi.c  |     6 -
+ drivers/gpu/drm/panel/panel-ilitek-ili9881c.c      |    15 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9882t.c      |    24 +-
+ drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c       |    19 +-
+ drivers/gpu/drm/panel/panel-jdi-lt070me05000.c     |    32 +-
+ drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c |    20 +-
+ drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c |    20 +-
+ drivers/gpu/drm/panel/panel-novatek-nt36532.c      |   431 +
+ drivers/gpu/drm/panel/panel-samsung-dsi.h          |    38 +
+ drivers/gpu/drm/panel/panel-samsung-s6d16d0.c      |    74 +-
+ drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c      |    20 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c      |    20 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e3fc2x01.c   |    91 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c      |    30 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e3ha8.c      |    90 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c   |    31 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c  |    13 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e63m0-spi.c  |     6 -
+ drivers/gpu/drm/panel/panel-samsung-s6e63m0.c      |    12 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e63m0.h      |     1 -
+ .../drm/panel/panel-samsung-s6e88a0-ams427ap24.c   |    20 +-
+ .../drm/panel/panel-samsung-s6e88a0-ams452ef01.c   |    26 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c      |    19 +-
+ .../gpu/drm/panel/panel-samsung-s6e8fc0-m1906f9.c  |    46 +-
+ drivers/gpu/drm/panel/panel-samsung-sofef00.c      |    35 +-
+ drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c    |    31 +-
+ drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c    |    20 +-
+ drivers/gpu/drm/panel/panel-sony-td4353-jdi.c      |    20 +-
+ .../gpu/drm/panel/panel-sony-tulip-truly-nt35521.c |    20 +-
+ drivers/gpu/drm/panel/panel-visionox-r66451.c      |    23 +-
+ drivers/gpu/drm/panel/panel-visionox-rm69299.c     |    21 +-
+ drivers/gpu/drm/panfrost/panfrost_devfreq.c        |     2 +-
+ drivers/gpu/drm/panfrost/panfrost_device.c         |     1 -
+ drivers/gpu/drm/panfrost/panfrost_drv.c            |     2 +-
+ drivers/gpu/drm/panthor/panthor_device.h           |   286 +-
+ drivers/gpu/drm/panthor/panthor_drv.c              |     1 +
+ drivers/gpu/drm/panthor/panthor_fw.c               |    22 +-
+ drivers/gpu/drm/panthor/panthor_gem.c              |    14 +-
+ drivers/gpu/drm/panthor/panthor_gpu.c              |   117 +-
+ drivers/gpu/drm/panthor/panthor_mmu.c              |    44 +-
+ drivers/gpu/drm/panthor/panthor_mmu.h              |     3 +-
+ drivers/gpu/drm/panthor/panthor_pwr.c              |    24 +-
+ drivers/gpu/drm/panthor/panthor_sched.c            |   591 +-
+ drivers/gpu/drm/panthor/panthor_sched.h            |     5 +
+ drivers/gpu/drm/panthor/panthor_trace.h            |    38 +
+ drivers/gpu/drm/pl111/pl111_display.c              |   207 +-
+ drivers/gpu/drm/pl111/pl111_drm.h                  |     5 +-
+ drivers/gpu/drm/pl111/pl111_drv.c                  |    19 +-
+ drivers/gpu/drm/pl111/pl111_versatile.c            |    18 -
+ drivers/gpu/drm/qxl/qxl_display.c                  |     6 +-
+ drivers/gpu/drm/radeon/atombios_encoders.c         |    36 +
+ drivers/gpu/drm/radeon/radeon_device.c             |     9 +-
+ drivers/gpu/drm/radeon/radeon_fence.c              |     4 +-
+ drivers/gpu/drm/renesas/rcar-du/Kconfig            |    12 +
+ drivers/gpu/drm/renesas/rcar-du/Makefile           |     1 +
+ drivers/gpu/drm/renesas/rcar-du/rcar_dsc.c         |   153 +
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c     |    17 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c  |    17 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c    |    19 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c      |    17 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c    |    51 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c      |    15 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c       |    15 +-
+ drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c  |    12 +-
+ drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c |    19 +-
+ drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c    |    66 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop.c        |    20 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop2.c       |    20 +-
+ drivers/gpu/drm/scheduler/sched_entity.c           |    10 +-
+ drivers/gpu/drm/scheduler/sched_fence.c            |     3 +-
+ drivers/gpu/drm/scheduler/sched_internal.h         |     3 +-
+ drivers/gpu/drm/scheduler/sched_main.c             |     8 +-
+ drivers/gpu/drm/scheduler/tests/tests_basic.c      |     2 +-
+ drivers/gpu/drm/sitronix/st7571.c                  |     2 +-
+ drivers/gpu/drm/sitronix/st7920.c                  |    24 +-
+ drivers/gpu/drm/solomon/ssd130x-spi.c              |    25 +-
+ drivers/gpu/drm/solomon/ssd130x.c                  |   443 +-
+ drivers/gpu/drm/solomon/ssd130x.h                  |    10 +-
+ drivers/gpu/drm/sprd/sprd_dpu.c                    |     4 +-
+ drivers/gpu/drm/sti/sti_crtc.c                     |     2 +-
+ drivers/gpu/drm/sti/sti_cursor.c                   |     2 +-
+ drivers/gpu/drm/sti/sti_gdp.c                      |     2 +-
+ drivers/gpu/drm/sti/sti_hqvdp.c                    |     2 +-
+ drivers/gpu/drm/stm/ltdc.c                         |     6 +-
+ drivers/gpu/drm/sun4i/sun4i_crtc.c                 |     2 +-
+ drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c             |     3 +-
+ drivers/gpu/drm/sun4i/sun4i_layer.c                |    21 +-
+ drivers/gpu/drm/sun4i/sun8i_ui_layer.c             |     2 +-
+ drivers/gpu/drm/sun4i/sun8i_vi_layer.c             |     2 +-
+ drivers/gpu/drm/sysfb/drm_sysfb_helper.h           |    12 +-
+ drivers/gpu/drm/sysfb/drm_sysfb_modeset.c          |    70 +-
+ drivers/gpu/drm/sysfb/efidrm.c                     |    17 +-
+ drivers/gpu/drm/sysfb/ofdrm.c                      |    18 +-
+ drivers/gpu/drm/sysfb/vesadrm.c                    |    18 +-
+ drivers/gpu/drm/tegra/dc.c                         |    18 +-
+ drivers/gpu/drm/tegra/dsi.c                        |    16 +-
+ drivers/gpu/drm/tegra/plane.c                      |    28 +-
+ drivers/gpu/drm/tegra/rgb.c                        |    15 +-
+ drivers/gpu/drm/tests/Makefile                     |     1 +
+ drivers/gpu/drm/tests/drm_connector_test.c         |    42 +-
+ drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c |    56 +-
+ drivers/gpu/drm/tests/drm_kunit_helpers.c          |     4 +-
+ .../{drm_panic_test.c => drm_panic_helper_test.c}  |    75 +-
+ drivers/gpu/drm/tidss/tidss_dispc.c                |    38 +-
+ drivers/gpu/drm/tidss/tidss_dispc.h                |     2 +-
+ drivers/gpu/drm/tidss/tidss_encoder.c              |    10 +-
+ drivers/gpu/drm/tidss/tidss_plane.c                |     2 +
+ drivers/gpu/drm/tilcdc/tilcdc_crtc.c               |     7 +-
+ drivers/gpu/drm/tilcdc/tilcdc_plane.c              |     2 +-
+ drivers/gpu/drm/tiny/appletbdrm.c                  |    14 +-
+ drivers/gpu/drm/tiny/arcpgu.c                      |   201 +-
+ drivers/gpu/drm/tiny/bochs.c                       |     6 +-
+ drivers/gpu/drm/tiny/cirrus-qemu.c                 |     2 +-
+ drivers/gpu/drm/tiny/gm12u320.c                    |   138 +-
+ drivers/gpu/drm/tiny/pixpaper.c                    |     2 +-
+ drivers/gpu/drm/tiny/repaper.c                     |   138 +-
+ drivers/gpu/drm/tiny/sharp-memory.c                |     2 +-
+ drivers/gpu/drm/ttm/ttm_bo.c                       |     3 +-
+ drivers/gpu/drm/ttm/ttm_module.c                   |     1 -
+ drivers/gpu/drm/tve200/tve200_display.c            |   220 +-
+ drivers/gpu/drm/tve200/tve200_drm.h                |     6 +-
+ drivers/gpu/drm/tve200/tve200_drv.c                |    12 +-
+ drivers/gpu/drm/udl/udl_modeset.c                  |     2 +-
+ drivers/gpu/drm/v3d/v3d_drv.h                      |    22 +-
+ drivers/gpu/drm/v3d/v3d_gem.c                      |     3 -
+ drivers/gpu/drm/v3d/v3d_perfmon.c                  |    13 +-
+ drivers/gpu/drm/v3d/v3d_sched.c                    |     4 -
+ drivers/gpu/drm/v3d/v3d_submit.c                   |    19 +-
+ drivers/gpu/drm/vboxvideo/vbox_mode.c              |     4 +-
+ drivers/gpu/drm/vc4/tests/vc4_mock_crtc.c          |     2 +-
+ drivers/gpu/drm/vc4/vc4_crtc.c                     |    14 +-
+ drivers/gpu/drm/vc4/vc4_drv.c                      |     1 -
+ drivers/gpu/drm/vc4/vc4_drv.h                      |    15 +-
+ drivers/gpu/drm/vc4/vc4_gem.c                      |    40 +-
+ drivers/gpu/drm/vc4/vc4_hdmi.c                     |     5 +-
+ drivers/gpu/drm/vc4/vc4_irq.c                      |     7 +-
+ drivers/gpu/drm/vc4/vc4_plane.c                    |    15 +-
+ drivers/gpu/drm/vc4/vc4_txp.c                      |     2 +-
+ drivers/gpu/drm/vc4/vc4_v3d.c                      |    37 +-
+ drivers/gpu/drm/verisilicon/vs_crtc.c              |     2 +-
+ drivers/gpu/drm/verisilicon/vs_cursor_plane.c      |    10 +-
+ drivers/gpu/drm/verisilicon/vs_plane.c             |    34 +-
+ drivers/gpu/drm/verisilicon/vs_plane.h             |     4 +-
+ drivers/gpu/drm/verisilicon/vs_primary_plane.c     |     9 +-
+ drivers/gpu/drm/virtio/virtgpu_display.c           |    14 +-
+ drivers/gpu/drm/virtio/virtgpu_plane.c             |    14 +-
+ drivers/gpu/drm/vkms/tests/gen_yuv_conversion.py   |    87 +
+ drivers/gpu/drm/vkms/tests/vkms_format_test.c      |    40 +-
+ drivers/gpu/drm/vkms/vkms_colorop.c                |    66 +-
+ drivers/gpu/drm/vkms/vkms_composer.c               |    36 +-
+ drivers/gpu/drm/vkms/vkms_crtc.c                   |    18 +-
+ drivers/gpu/drm/vkms/vkms_drv.h                    |     2 +-
+ drivers/gpu/drm/vkms/vkms_formats.c                |   112 +-
+ drivers/gpu/drm/vkms/vkms_formats.h                |     2 +-
+ drivers/gpu/drm/vkms/vkms_plane.c                  |    91 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_kms.c                |    39 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_kms.h                |     4 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c                |     6 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c               |     6 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c               |     6 +-
+ drivers/gpu/drm/xe/Makefile                        |     4 +-
+ drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h      |     1 +
+ drivers/gpu/drm/xe/abi/xe_log_abi.h                |   200 +
+ drivers/gpu/drm/xe/abi/xe_sigid_abi.h              |   172 +
+ drivers/gpu/drm/xe/display/xe_display.c            |    22 +-
+ drivers/gpu/drm/xe/display/xe_display_pcode.c      |     4 +-
+ drivers/gpu/drm/xe/display/xe_display_rpm.c        |     2 -
+ drivers/gpu/drm/xe/display/xe_display_wa.c         |    13 +-
+ drivers/gpu/drm/xe/display/xe_display_wa.h         |     9 +
+ drivers/gpu/drm/xe/display/xe_dsb_buffer.c         |     2 +-
+ drivers/gpu/drm/xe/display/xe_fb_pin.c             |     2 +-
+ drivers/gpu/drm/xe/display/xe_panic.c              |     3 +-
+ drivers/gpu/drm/xe/display/xe_tdf.c                |    15 -
+ drivers/gpu/drm/xe/instructions/xe_mi_commands.h   |     7 +-
+ drivers/gpu/drm/xe/regs/xe_gt_regs.h               |    33 +
+ drivers/gpu/drm/xe/regs/xe_lrc_layout.h            |     2 +
+ drivers/gpu/drm/xe/tests/Makefile                  |     1 +
+ drivers/gpu/drm/xe/tests/xe_any_kunit.c            |   213 +
+ .../gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c    |   303 +
+ drivers/gpu/drm/xe/tests/xe_kunit_helpers.c        |     4 +
+ drivers/gpu/drm/xe/tests/xe_log_kunit.c            |   553 +
+ drivers/gpu/drm/xe/tests/xe_pci.c                  |    25 +-
+ drivers/gpu/drm/xe/xe_any.h                        |   137 +
+ drivers/gpu/drm/xe/xe_bo.c                         |    76 +-
+ drivers/gpu/drm/xe/xe_bo.h                         |     3 +-
+ drivers/gpu/drm/xe/xe_bo_types.h                   |     8 +
+ drivers/gpu/drm/xe/xe_configfs.c                   |    72 +-
+ drivers/gpu/drm/xe/xe_configfs.h                   |     2 +
+ drivers/gpu/drm/xe/xe_debugfs.c                    |    66 +
+ drivers/gpu/drm/xe/xe_debugfs.h                    |     4 +
+ drivers/gpu/drm/xe/xe_defaults.h                   |     1 +
+ drivers/gpu/drm/xe/xe_devcoredump.c                |     4 +-
+ drivers/gpu/drm/xe/xe_device.c                     |   150 +-
+ drivers/gpu/drm/xe/xe_device.h                     |     2 +-
+ drivers/gpu/drm/xe/xe_device_types.h               |    42 +-
+ drivers/gpu/drm/xe/xe_dma_buf.c                    |    22 +-
+ drivers/gpu/drm/xe/xe_drm_ras.c                    |    65 +
+ drivers/gpu/drm/xe/xe_drm_ras.h                    |     3 +
+ drivers/gpu/drm/xe/xe_drm_ras_types.h              |     3 +
+ drivers/gpu/drm/xe/xe_exec_queue.c                 |    78 +-
+ drivers/gpu/drm/xe/xe_exec_queue_types.h           |    28 +-
+ drivers/gpu/drm/xe/xe_execlist.c                   |     4 +-
+ drivers/gpu/drm/xe/xe_ggtt.c                       |    74 +-
+ drivers/gpu/drm/xe/xe_gsc.c                        |     3 +-
+ drivers/gpu/drm/xe/xe_gt.c                         |    24 +-
+ drivers/gpu/drm/xe/xe_gt.h                         |    13 +
+ drivers/gpu/drm/xe/xe_gt_debugfs.c                 |   252 +-
+ drivers/gpu/drm/xe/xe_gt_idle.c                    |    84 +-
+ drivers/gpu/drm/xe/xe_gt_idle.h                    |     1 +
+ drivers/gpu/drm/xe/xe_gt_printk.h                  |     3 +
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c         |   299 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h         |    10 +
+ drivers/gpu/drm/xe/xe_gt_sriov_printk.h            |     3 +
+ drivers/gpu/drm/xe/xe_gt_stats.c                   |     7 +
+ drivers/gpu/drm/xe/xe_gt_stats_types.h             |    22 +
+ drivers/gpu/drm/xe/xe_gt_types.h                   |    29 +
+ drivers/gpu/drm/xe/xe_guc.c                        |    18 +-
+ drivers/gpu/drm/xe/xe_guc_capture.c                |     2 -
+ drivers/gpu/drm/xe/xe_guc_ct.c                     |    95 +-
+ drivers/gpu/drm/xe/xe_guc_ct.h                     |    38 +-
+ drivers/gpu/drm/xe/xe_guc_exec_queue_types.h       |    32 +-
+ drivers/gpu/drm/xe/xe_guc_klv_helpers.c            |    78 +
+ drivers/gpu/drm/xe/xe_guc_klv_helpers.h            |     5 +
+ drivers/gpu/drm/xe/xe_guc_pagefault.c              |    47 +-
+ drivers/gpu/drm/xe/xe_guc_pc.c                     |    15 +-
+ drivers/gpu/drm/xe/xe_guc_submit.c                 |   382 +-
+ drivers/gpu/drm/xe/xe_guc_submit.h                 |     5 +
+ drivers/gpu/drm/xe/xe_guc_tlb_inval.c              |    29 +
+ drivers/gpu/drm/xe/xe_guc_types.h                  |     6 +
+ drivers/gpu/drm/xe/xe_hwmon.c                      |    95 +-
+ drivers/gpu/drm/xe/xe_i2c.c                        |    22 +-
+ drivers/gpu/drm/xe/xe_i2c.h                        |     1 +
+ drivers/gpu/drm/xe/xe_log.c                        |   247 +
+ drivers/gpu/drm/xe/xe_log.h                        |   194 +
+ drivers/gpu/drm/xe/xe_lrc.c                        |    39 +-
+ drivers/gpu/drm/xe/xe_lrc.h                        |     1 +
+ drivers/gpu/drm/xe/xe_mert.c                       |     2 +-
+ drivers/gpu/drm/xe/xe_migrate.c                    |   123 +-
+ drivers/gpu/drm/xe/xe_migrate.h                    |     6 +
+ drivers/gpu/drm/xe/xe_mmio_gem.c                   |     7 +-
+ drivers/gpu/drm/xe/xe_module.c                     |     4 +
+ drivers/gpu/drm/xe/xe_module.h                     |     1 +
+ drivers/gpu/drm/xe/xe_nvm.c                        |     1 +
+ drivers/gpu/drm/xe/xe_oa.c                         |     7 +-
+ drivers/gpu/drm/xe/xe_pagefault.c                  |   826 +-
+ drivers/gpu/drm/xe/xe_pagefault.h                  |   137 +
+ drivers/gpu/drm/xe/xe_pagefault_types.h            |   201 +-
+ drivers/gpu/drm/xe/xe_pci.c                        |    74 +-
+ drivers/gpu/drm/xe/xe_pci_error.c                  |    13 +-
+ drivers/gpu/drm/xe/xe_pci_sriov.c                  |    27 +-
+ drivers/gpu/drm/xe/xe_pcode.c                      |    27 +-
+ drivers/gpu/drm/xe/xe_pcode.h                      |     6 +-
+ drivers/gpu/drm/xe/xe_pm.c                         |    23 +
+ drivers/gpu/drm/xe/xe_pm.h                         |     1 +
+ drivers/gpu/drm/xe/xe_printk.h                     |     3 +
+ drivers/gpu/drm/xe/xe_pt.c                         |    34 +-
+ drivers/gpu/drm/xe/xe_pxp_submit.c                 |     5 +-
+ drivers/gpu/drm/xe/xe_ras.c                        |   245 +-
+ drivers/gpu/drm/xe/xe_ras.h                        |     2 +
+ drivers/gpu/drm/xe/xe_ras_types.h                  |    50 +
+ drivers/gpu/drm/xe/xe_res_cursor.h                 |     5 +-
+ drivers/gpu/drm/xe/xe_ring_ops.c                   |     4 +-
+ drivers/gpu/drm/xe/xe_sriov_packet.c               |    31 +-
+ drivers/gpu/drm/xe/xe_sriov_printk.h               |     3 +
+ drivers/gpu/drm/xe/xe_sriov_vf_ccs.c               |     1 -
+ drivers/gpu/drm/xe/xe_survivability_mode.c         |    85 +-
+ drivers/gpu/drm/xe/xe_svm.c                        |   182 +-
+ drivers/gpu/drm/xe/xe_svm.h                        |    79 +-
+ drivers/gpu/drm/xe/xe_sysctrl.c                    |    92 +
+ drivers/gpu/drm/xe/xe_sysctrl.h                    |     2 +
+ drivers/gpu/drm/xe/xe_sysctrl_event.c              |    28 +-
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h      |    49 +
+ drivers/gpu/drm/xe/xe_tile_printk.h                |     3 +
+ drivers/gpu/drm/xe/xe_tile_sriov_printk.h          |     3 +
+ drivers/gpu/drm/xe/xe_tile_types.h                 |     4 +
+ drivers/gpu/drm/xe/xe_tlb_inval.c                  |    42 +-
+ drivers/gpu/drm/xe/xe_tlb_inval.h                  |     1 +
+ drivers/gpu/drm/xe/xe_tlb_inval_types.h            |    10 +
+ drivers/gpu/drm/xe/xe_ttm_vram_mgr.c               |   600 +-
+ drivers/gpu/drm/xe/xe_ttm_vram_mgr.h               |     3 +
+ drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h         |    40 +
+ drivers/gpu/drm/xe/xe_userptr.c                    |    21 +-
+ drivers/gpu/drm/xe/xe_vm.c                         |   316 +-
+ drivers/gpu/drm/xe/xe_vm_types.h                   |    44 +-
+ drivers/gpu/drm/xe/xe_vram.c                       |   255 +-
+ drivers/gpu/drm/xe/xe_vram.h                       |    10 +
+ drivers/gpu/drm/xe/xe_wa.c                         |     4 +-
+ drivers/gpu/drm/xen/xen_drm_front.h                |     6 +-
+ drivers/gpu/drm/xen/xen_drm_front_kms.c            |   188 +-
+ drivers/gpu/drm/xlnx/zynqmp_kms.c                  |    18 +-
+ drivers/gpu/tests/gpu_buddy_test.c                 |   160 +-
+ drivers/gpu/vga/vga_switcheroo.c                   |    41 +-
+ drivers/video/fbdev/core/fbcon.c                   |     8 -
+ include/drm/display/drm_dp.h                       |     1 +
+ include/drm/display/drm_dp_helper.h                |     6 +
+ include/drm/display/drm_hdmi_helper.h              |    15 +
+ include/drm/display/drm_hdmi_state_helper.h        |    11 +-
+ include/drm/display/drm_scdc.h                     |    21 +-
+ include/drm/display/drm_scdc_helper.h              |   105 +-
+ include/drm/drm_atomic.h                           |    89 +-
+ include/drm/drm_atomic_state_helper.h              |     6 -
+ include/drm/drm_bridge.h                           |    49 +
+ include/drm/drm_client.h                           |    14 +
+ include/drm/drm_client_event.h                     |     3 +
+ include/drm/drm_colorop.h                          |   127 +
+ include/drm/drm_connector.h                        |   216 +-
+ include/drm/drm_crtc.h                             |    12 -
+ include/drm/drm_device.h                           |     1 +
+ include/drm/drm_edid.h                             |     2 +
+ include/drm/drm_gem.h                              |    14 +-
+ include/drm/drm_gem_atomic_helper.h                |    13 +-
+ include/drm/drm_gpusvm.h                           |    71 +-
+ include/drm/drm_mipi_dbi.h                         |     2 +-
+ include/drm/drm_mode_config.h                      |     4 +-
+ include/drm/drm_modeset_helper_vtables.h           |    53 +-
+ include/drm/drm_panic.h                            |   117 +-
+ include/drm/drm_panic_helper.h                     |    41 +
+ include/drm/drm_plane.h                            |    69 +-
+ include/drm/drm_print.h                            |     3 +
+ include/drm/drm_ras.h                              |    34 +
+ include/drm/drm_simple_kms_helper.h                |     7 +-
+ include/drm/drm_sysfs.h                            |     4 -
+ include/drm/intel/display_parent_interface.h       |    15 +-
+ include/drm/ttm/ttm_resource.h                     |     2 +-
+ include/linux/dma-fence.h                          |     2 +-
+ include/linux/gpu_buddy.h                          |   101 +-
+ include/linux/hdmi.h                               |    12 +
+ include/linux/vga_switcheroo.h                     |    29 +-
+ include/sound/omap-hdmi-audio.h                    |     1 +
+ include/uapi/drm/amdgpu_drm.h                      |    41 +
+ include/uapi/drm/drm_mode.h                        |    12 +
+ include/uapi/drm/drm_ras.h                         |    18 +
+ include/uapi/drm/ivpu_accel.h                      |    17 +-
+ include/uapi/drm/xe_drm.h                          |    22 +-
+ kernel/cgroup/dmem.c                               |     4 +-
+ rust/bindings/bindings_helper.h                    |     4 +-
+ rust/kernel/drm/gem/mod.rs                         |    13 +-
+ rust/kernel/drm/gem/shmem.rs                       |     5 +-
+ sound/soc/ti/omap-hdmi.c                           |    51 +-
+ 1337 files changed, 149250 insertions(+), 25200 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/display/bridge/renesas,r8a779g0-dsc.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/ilitek,ili7836a.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/novatek,nt36532.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/solomon,ssd1351.yaml
+ create mode 100644 Documentation/gpu/amdgpu/ualink.rst
+ create mode 100644 Documentation/gpu/xe/xe_sigid.rst
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma_types.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/ualink_v1_0.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/ualink_v1_0.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_smu_driver_if.h
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/dc_edid_parser.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dc_memory_pool.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dc_memory_pool.h
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_wrapper/Makefile
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0/dml21 => dml2_wrapper/dml21_wrapper}/dml21_translation_helper.c (96%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0/dml21 => dml2_wrapper/dml21_wrapper}/dml21_translation_helper.h (94%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0/dml21 => dml2_wrapper/dml21_wrapper}/dml21_utils.c (97%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0/dml21 => dml2_wrapper/dml21_wrapper}/dml21_utils.h (96%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0/dml21 => dml2_wrapper/dml21_wrapper}/dml21_wrapper.c (100%)
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.h
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0/dml21 => dml2_wrapper/dml21_wrapper}/dml21_wrapper_fpu.c (96%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0/dml21 => dml2_wrapper/dml21_wrapper}/dml21_wrapper_fpu.h (95%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_dc_resource_mgmt.c (96%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_dc_resource_mgmt.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_dc_types.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_internal_types.h (99%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_mall_phantom.c (97%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_mall_phantom.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_policy.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_policy.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_translation_helper.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_translation_helper.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_utils.c (98%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_utils.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_wrapper.c (97%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_wrapper.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_wrapper_fpu.c (93%)
+ rename drivers/gpu/drm/amd/display/dc/{dml2_0 => dml2_wrapper}/dml2_wrapper_fpu.h (100%)
+ create mode 100644 drivers/gpu/drm/amd/display/dc/inc/hw/rmcm.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/rmcm/Makefile
+ create mode 100644 drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.c
+ create mode 100644 drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.c
+ rename drivers/gpu/drm/amd/display/dc/{dc_edid_parser.h => rmcm/dcn60/dcn60_rmcm.h} (71%)
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/sspl/spl_namespace.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/ivsrcid/mpnht/irqsrcs_mpnht_15_0.h
+ create mode 100644 drivers/gpu/drm/amd/ras/core/aca_v5_0.c
+ create mode 100644 drivers/gpu/drm/amd/ras/core/aca_v5_0.h
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_bert.c
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_bert.h
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.h
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_mce.c
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_mce.h
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.c
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.h
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.c
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.h
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c
+ create mode 100644 drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.h
+ create mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.c
+ create mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.h
+ create mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.c
+ create mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.h
+ create mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.c
+ rename drivers/gpu/drm/amd/ras/ras_mgr/{amdgpu_ras_mp1_v13_0.h => amdgpu_ras_mp1.h} (86%)
+ delete mode 100644 drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
+ create mode 100644 drivers/gpu/drm/drm_panic_helper.c
+ rename drivers/gpu/drm/{drm_panic_qr.rs => drm_panic_helper_qr.rs} (99%)
+ create mode 100644 drivers/gpu/drm/drm_panic_internal.h
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_tdf.h
+ create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/pci/mcp79.c
+ create mode 100644 drivers/gpu/drm/panel/panel-ilitek-ili7836a.c
+ create mode 100644 drivers/gpu/drm/panel/panel-novatek-nt36532.c
+ create mode 100644 drivers/gpu/drm/panel/panel-samsung-dsi.h
+ create mode 100644 drivers/gpu/drm/renesas/rcar-du/rcar_dsc.c
+ rename drivers/gpu/drm/tests/{drm_panic_test.c => drm_panic_helper_test.c} (77%)
+ create mode 100755 drivers/gpu/drm/vkms/tests/gen_yuv_conversion.py
+ create mode 100644 drivers/gpu/drm/xe/abi/xe_log_abi.h
+ create mode 100644 drivers/gpu/drm/xe/abi/xe_sigid_abi.h
+ create mode 100644 drivers/gpu/drm/xe/display/xe_display_wa.h
+ delete mode 100644 drivers/gpu/drm/xe/display/xe_tdf.c
+ create mode 100644 drivers/gpu/drm/xe/tests/xe_any_kunit.c
+ create mode 100644 drivers/gpu/drm/xe/tests/xe_log_kunit.c
+ create mode 100644 drivers/gpu/drm/xe/xe_any.h
+ create mode 100644 drivers/gpu/drm/xe/xe_log.c
+ create mode 100644 drivers/gpu/drm/xe/xe_log.h
+ create mode 100644 include/drm/drm_panic_helper.h
+Merging drm-exynos/for-linux-next (3a8660878839f Linux 6.18-rc1)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git drm-exynos/for-linux-next
+Already up to date.
+Merging drm-misc/for-linux-next (744f262401cc2 drm/dp/mst: reject DPCD read/write on ports with ddps=0)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc/for-linux-next
+Auto-merging drivers/accel/ivpu/ivpu_job.c
+Auto-merging drivers/gpu/drm/bridge/samsung-dsim.c
+Merge made by the 'ort' strategy.
+ .../display/allwinner,sun6i-a31-mipi-dsi.yaml      |  10 +-
+ .../display/bridge/fsl,imx8qxp-pxl2dpi.yaml        |   4 +-
+ .../bindings/display/bridge/lontium,lt8912b.yaml   |   8 +-
+ .../bridge/megachips,stdp2690-ge-b850v3-fw.yaml    |   2 +-
+ .../bindings/display/bridge/nxp,tda998x.yaml       |   2 +-
+ .../bindings/display/bridge/toshiba,tc358764.yaml  |   2 +-
+ .../bindings/display/bridge/toshiba,tc358775.yaml  |  12 +-
+ .../bindings/display/faraday,tve200.yaml           |   2 +-
+ .../devicetree/bindings/display/himax,hx8357.yaml  |   2 +-
+ .../bindings/display/imx/fsl,imx-lcdc.yaml         |   2 +-
+ .../bindings/display/imx/fsl,imx6q-ldb.yaml        |  54 +-
+ .../bindings/display/mediatek/mediatek,ethdr.yaml  |  90 +--
+ .../bindings/display/mediatek/mediatek,hdmi.yaml   |  24 +-
+ .../bindings/display/panel/ilitek,il79900a.yaml    |  13 +-
+ .../panel/panel-simple-lvds-dual-ports.yaml        |   2 +
+ .../bindings/display/panel/panel-simple.yaml       |   2 +
+ .../bindings/display/panel/samsung,ams495qa01.yaml |   2 +-
+ .../bindings/display/sitronix,st7567.yaml          |   4 +-
+ .../devicetree/bindings/display/st,stm32-dsi.yaml  |  46 +-
+ .../devicetree/bindings/display/st,stm32-ltdc.yaml |   6 +-
+ .../bindings/display/st,stm32mp25-lvds.yaml        |   4 +-
+ Documentation/gpu/todo.rst                         |  33 +-
+ drivers/accel/amdxdna/aie2_ctx.c                   | 117 +++-
+ drivers/accel/amdxdna/aie2_msg_priv.h              |   1 +
+ drivers/accel/amdxdna/aie2_pci.c                   |  15 +-
+ drivers/accel/amdxdna/aie2_pci.h                   |   6 +-
+ drivers/accel/amdxdna/amdxdna_mailbox.c            |   2 +-
+ drivers/accel/amdxdna/npu4_regs.c                  |   1 +
+ drivers/accel/ivpu/ivpu_fw.c                       |  65 +-
+ drivers/accel/ivpu/ivpu_gem.c                      |   3 +
+ drivers/accel/ivpu/ivpu_ipc.c                      |  11 +-
+ drivers/accel/ivpu/ivpu_job.c                      |   6 +-
+ drivers/accel/ivpu/ivpu_ms.c                       |  17 +-
+ drivers/gpu/drm/Makefile                           |   3 +-
+ drivers/gpu/drm/adp/adp-mipi.c                     |   1 +
+ drivers/gpu/drm/arm/display/komeda/komeda_crtc.c   |   1 +
+ drivers/gpu/drm/bridge/Kconfig                     |  18 +-
+ drivers/gpu/drm/bridge/analogix/Kconfig            |   1 +
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.c |  40 +-
+ drivers/gpu/drm/bridge/aux-bridge.c                |   1 +
+ drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c     |   1 +
+ drivers/gpu/drm/bridge/fsl-ldb.c                   |  19 +-
+ drivers/gpu/drm/bridge/imx/imx93-pdfc.c            |   1 +
+ drivers/gpu/drm/bridge/panel.c                     | 563 -----------------
+ drivers/gpu/drm/bridge/samsung-dsim.c              |  24 +-
+ drivers/gpu/drm/bridge/ssd2825.c                   |  23 +-
+ drivers/gpu/drm/bridge/tc358767.c                  |  64 +-
+ drivers/gpu/drm/bridge/tc358768.c                  |  25 +-
+ drivers/gpu/drm/bridge/ti-tdp158.c                 |   1 +
+ drivers/gpu/drm/bridge/waveshare-dsi.c             |  17 +-
+ drivers/gpu/drm/display/drm_bridge_connector.c     |   1 +
+ drivers/gpu/drm/display/drm_dp_mst_topology.c      |   6 +
+ drivers/gpu/drm/drm_bridge.c                       |  17 -
+ drivers/gpu/drm/drm_of.c                           |  63 --
+ drivers/gpu/drm/drm_panel.c                        | 664 +++++++++++++++++++--
+ drivers/gpu/drm/exynos/exynos_dp.c                 |  36 +-
+ drivers/gpu/drm/imagination/pvr_fw.c               | 130 ++--
+ drivers/gpu/drm/imagination/pvr_fw.h               |  18 +-
+ drivers/gpu/drm/imx/dc/dc-kms.c                    |   1 +
+ drivers/gpu/drm/imx/dcss/Kconfig                   |   1 +
+ drivers/gpu/drm/ingenic/Kconfig                    |   1 +
+ drivers/gpu/drm/logicvc/Kconfig                    |   1 +
+ drivers/gpu/drm/mcde/Kconfig                       |   2 +-
+ drivers/gpu/drm/mcde/mcde_display.c                |   1 +
+ drivers/gpu/drm/mcde/mcde_dsi.c                    |  45 +-
+ drivers/gpu/drm/msm/dp/dp_display.c                |   1 +
+ drivers/gpu/drm/msm/dsi/dsi.c                      |   3 +-
+ drivers/gpu/drm/omapdrm/dss/omapdss.h              |   1 -
+ drivers/gpu/drm/omapdrm/dss/output.c               |  42 +-
+ drivers/gpu/drm/panel/Kconfig                      |   2 +-
+ drivers/gpu/drm/panel/panel-ebbg-ft8719.c          |  23 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9882t.c      | 234 +++++++-
+ drivers/gpu/drm/panel/panel-simple.c               |  57 ++
+ drivers/gpu/drm/pl111/Kconfig                      |   1 +
+ drivers/gpu/drm/rockchip/Kconfig                   |   2 +
+ drivers/gpu/drm/rockchip/analogix_dp-rockchip.c    |  11 -
+ drivers/gpu/drm/scheduler/sched_main.c             |   1 -
+ drivers/gpu/drm/solomon/ssd130x.c                  |  30 +-
+ drivers/gpu/drm/stm/Kconfig                        |   1 +
+ drivers/gpu/drm/tegra/rgb.c                        |   1 +
+ drivers/gpu/drm/tidss/Kconfig                      |   1 +
+ drivers/gpu/drm/tve200/Kconfig                     |   2 +-
+ drivers/gpu/drm/tve200/tve200_drm.h                |   1 -
+ drivers/gpu/drm/tve200/tve200_drv.c                |  31 +-
+ include/drm/bridge/analogix_dp.h                   |   1 -
+ include/drm/drm_bridge.h                           |  54 --
+ include/drm/drm_of.h                               |  15 +-
+ include/drm/drm_panel.h                            |  99 ++-
+ 88 files changed, 1577 insertions(+), 1397 deletions(-)
+ delete mode 100644 drivers/gpu/drm/bridge/panel.c
+Merging amdgpu/drm-next (76b9706e7fa1a drm/amd/display: Bump frame warning limit for all builds of dml)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux.git amdgpu/drm-next
+Already up to date.
+Merging drm-intel/for-linux-next (c9e608d1247cd drm/i915/display: Update the CMN_SDP_TL in fastset path)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel/for-linux-next
+Auto-merging drivers/gpu/drm/i915/display/intel_display_params.h
+Auto-merging drivers/gpu/drm/i915/display/intel_display_types.h
+Auto-merging drivers/gpu/drm/i915/display/intel_dp_link_training.c
+Auto-merging drivers/gpu/drm/i915/display/intel_dp_mst.c
+Auto-merging drivers/gpu/drm/i915/display/intel_psr.c
+Auto-merging drivers/gpu/drm/i915/display/intel_vrr.c
+Auto-merging drivers/gpu/drm/i915/display/skl_universal_plane.c
+Auto-merging drivers/gpu/drm/xe/xe_device_types.h
+Auto-merging drivers/gpu/drm/xe/xe_pci.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/xe/xe_pci.c
+Auto-merging drivers/gpu/drm/xe/xe_pm.c
+Auto-merging drivers/gpu/drm/xe/xe_pm.h
+Resolved 'drivers/gpu/drm/xe/xe_pci.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 4711f5c0956c1] Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/i915/kernel.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/gpu/drm/i915/Makefile                      |   1 +
+ drivers/gpu/drm/i915/display/intel_alpm.c          | 166 ++++++++++--
+ drivers/gpu/drm/i915/display/intel_alpm.h          |   3 +
+ drivers/gpu/drm/i915/display/intel_bios.c          | 262 ++++++++++++++++++-
+ drivers/gpu/drm/i915/display/intel_bios.h          |  14 ++
+ drivers/gpu/drm/i915/display/intel_color.c         |  76 ++++--
+ .../gpu/drm/i915/display/intel_color_pipeline.c    |  34 ++-
+ .../gpu/drm/i915/display/intel_crtc_state_dump.c   |   9 +
+ drivers/gpu/drm/i915/display/intel_cx0_phy.c       |   5 +-
+ drivers/gpu/drm/i915/display/intel_ddi.c           |  37 ++-
+ drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c |  90 ++++++-
+ drivers/gpu/drm/i915/display/intel_dip.c           | 215 ++++++++++++++++
+ drivers/gpu/drm/i915/display/intel_dip.h           |  53 ++++
+ drivers/gpu/drm/i915/display/intel_dip_regs.h      |  38 +++
+ drivers/gpu/drm/i915/display/intel_display.c       |  60 ++++-
+ drivers/gpu/drm/i915/display/intel_display_core.h  |   7 +
+ drivers/gpu/drm/i915/display/intel_display_irq.c   |  18 +-
+ .../gpu/drm/i915/display/intel_display_limits.h    |   1 +
+ .../gpu/drm/i915/display/intel_display_params.h    |   5 +
+ drivers/gpu/drm/i915/display/intel_display_power.c |   9 +
+ .../gpu/drm/i915/display/intel_display_power_map.c |  42 +++-
+ drivers/gpu/drm/i915/display/intel_display_rpm.c   |   7 +
+ drivers/gpu/drm/i915/display/intel_display_rpm.h   |   1 +
+ drivers/gpu/drm/i915/display/intel_display_types.h |  20 ++
+ drivers/gpu/drm/i915/display/intel_dp.c            | 279 +++++++++++++++------
+ .../gpu/drm/i915/display/intel_dp_aux_backlight.c  |  78 +++---
+ drivers/gpu/drm/i915/display/intel_dp_hdcp.c       | 176 ++++++-------
+ .../gpu/drm/i915/display/intel_dp_link_training.c  |  54 ++--
+ drivers/gpu/drm/i915/display/intel_dp_mst.c        |   2 +-
+ drivers/gpu/drm/i915/display/intel_dp_test.c       |  77 +++---
+ drivers/gpu/drm/i915/display/intel_fb.c            |   2 +-
+ drivers/gpu/drm/i915/display/intel_fbc.c           |  12 +-
+ drivers/gpu/drm/i915/display/intel_hotplug.c       |   5 +
+ drivers/gpu/drm/i915/display/intel_lspcon.c        |  82 +++---
+ drivers/gpu/drm/i915/display/intel_plane.c         |  53 +++-
+ drivers/gpu/drm/i915/display/intel_psr.c           |  76 +++---
+ drivers/gpu/drm/i915/display/intel_psr_regs.h      |   2 +
+ drivers/gpu/drm/i915/display/intel_vrr.c           |  14 +-
+ drivers/gpu/drm/i915/display/intel_vrr_regs.h      |   6 -
+ drivers/gpu/drm/i915/display/skl_universal_plane.c |  57 +++--
+ drivers/gpu/drm/i915/gt/intel_engine_cs.c          |   2 +-
+ .../gpu/drm/i915/gt/intel_execlists_submission.c   |  17 +-
+ drivers/gpu/drm/i915/gt/intel_reset.c              |   8 +-
+ drivers/gpu/drm/i915/gt/selftest_engine_pm.c       |   8 +-
+ drivers/gpu/drm/i915/gt/uc/intel_guc.h             |   2 +-
+ drivers/gpu/drm/i915/gvt/aperture_gm.c             |   4 +-
+ drivers/gpu/drm/i915/i915_dpt.c                    |   2 -
+ drivers/gpu/drm/i915/i915_drv.h                    |   2 -
+ drivers/gpu/drm/i915/i915_fb_pin.c                 |   6 -
+ drivers/gpu/drm/i915/i915_overlay.c                |   5 -
+ drivers/gpu/drm/i915/i915_request.c                |   2 -
+ drivers/gpu/drm/i915/selftests/igt_atomic.c        |   7 +
+ drivers/gpu/drm/i915/vlv_iosf_sb.c                 |   6 +-
+ drivers/gpu/drm/xe/Makefile                        |   1 +
+ .../gpu/drm/xe/compat-i915-headers/i915_config.h   |  16 --
+ drivers/gpu/drm/xe/display/xe_display.c            |  10 +-
+ drivers/gpu/drm/xe/display/xe_display_rpm.c        |   6 +
+ drivers/gpu/drm/xe/xe_device_types.h               |  13 +
+ drivers/gpu/drm/xe/xe_pci.c                        |  22 +-
+ drivers/gpu/drm/xe/xe_pm.c                         |  45 ++++
+ drivers/gpu/drm/xe/xe_pm.h                         |   2 +
+ include/drm/intel/display_parent_interface.h       |   1 +
+ 62 files changed, 1785 insertions(+), 550 deletions(-)
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dip.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dip.h
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dip_regs.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/i915_config.h
+Merging drm-msm/msm-next (d33622598496c Merge tag 'qcom-drivers-for-7.4' into msm-next-merge-qcom-drivers-for-7.4)
+$ git merge -m Merge branch 'msm-next' of https://gitlab.freedesktop.org/drm/msm.git drm-msm/msm-next
+Merge made by the 'ort' strategy.
+Merging drm-msm-lumag/msm-next-lumag (140b134753026 drm/msm: detach the ARM DMA mapping before attaching our own domain)
+$ git merge -m Merge branch 'msm-next-lumag' of https://gitlab.freedesktop.org/lumag/msm.git drm-msm-lumag/msm-next-lumag
+Already up to date.
+Merging drm-xe/drm-xe-next (3fc93d311249d drm/xe/xe3p: Force non-compressible memory reads to 256B overfetches)
+$ git merge -m Merge branch 'drm-xe-next' of https://gitlab.freedesktop.org/drm/xe/kernel.git drm-xe/drm-xe-next
+Auto-merging drivers/gpu/drm/drm_pagemap.c
+Auto-merging drivers/gpu/drm/xe/Makefile
+Auto-merging drivers/gpu/drm/xe/xe_bo.h
+Auto-merging drivers/gpu/drm/xe/xe_configfs.c
+Auto-merging drivers/gpu/drm/xe/xe_device_types.h
+Auto-merging drivers/gpu/drm/xe/xe_guc_ads.c
+Auto-merging drivers/gpu/drm/xe/xe_pagefault.c
+Auto-merging drivers/gpu/drm/xe/xe_pci.c
+Auto-merging drivers/gpu/drm/xe/xe_vm.c
+Auto-merging drivers/gpu/drm/xe/xe_wa_oob.rules
+CONFLICT (content): Merge conflict in drivers/gpu/drm/xe/xe_wa_oob.rules
+Resolved 'drivers/gpu/drm/xe/xe_wa_oob.rules' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master c2a5eaf3bbac2] Merge branch 'drm-xe-next' of https://gitlab.freedesktop.org/drm/xe/kernel.git
+$ git diff -M --stat --summary HEAD^..
+ .../ABI/testing/sysfs-driver-intel-xe-gpu          |  10 +
+ Documentation/gpu/xe/xe_migrate.rst                |   3 +
+ drivers/gpu/drm/drm_pagemap.c                      |   2 +-
+ drivers/gpu/drm/xe/Makefile                        |   1 +
+ drivers/gpu/drm/xe/abi/guc_klvs_abi.h              |   1 +
+ drivers/gpu/drm/xe/regs/xe_gt_regs.h               |   3 +-
+ drivers/gpu/drm/xe/regs/xe_regs.h                  |   2 +
+ drivers/gpu/drm/xe/xe_bo.c                         |   4 +-
+ drivers/gpu/drm/xe/xe_bo.h                         |  10 +-
+ drivers/gpu/drm/xe/xe_bo_types.h                   |   2 -
+ drivers/gpu/drm/xe/xe_configfs.c                   |  67 ++
+ drivers/gpu/drm/xe/xe_configfs.h                   |   2 +
+ drivers/gpu/drm/xe/xe_cpu_bind.c                   | 296 ++++++++
+ drivers/gpu/drm/xe/xe_cpu_bind.h                   | 111 +++
+ drivers/gpu/drm/xe/xe_devcoredump.c                |  46 +-
+ drivers/gpu/drm/xe/xe_devcoredump.h                |  15 +-
+ drivers/gpu/drm/xe/xe_device.c                     |  14 +
+ drivers/gpu/drm/xe/xe_device_sysfs.c               |  36 +
+ drivers/gpu/drm/xe/xe_device_types.h               |  15 +-
+ drivers/gpu/drm/xe/xe_drm_client.c                 |   2 +-
+ drivers/gpu/drm/xe/xe_exec_queue.c                 | 161 ++---
+ drivers/gpu/drm/xe/xe_exec_queue.h                 |  16 +-
+ drivers/gpu/drm/xe/xe_exec_queue_types.h           |  20 +-
+ drivers/gpu/drm/xe/xe_gt_ccs_mode.c                |   6 -
+ drivers/gpu/drm/xe/xe_guc.c                        |  16 +
+ drivers/gpu/drm/xe/xe_guc.h                        |   2 +
+ drivers/gpu/drm/xe/xe_guc_ads.c                    |   9 +-
+ drivers/gpu/drm/xe/xe_guc_ct.c                     |   6 +-
+ drivers/gpu/drm/xe/xe_guc_engine_activity.c        |   6 +-
+ drivers/gpu/drm/xe/xe_guc_hwconfig.c               |   2 +-
+ drivers/gpu/drm/xe/xe_guc_log.c                    |   7 +-
+ drivers/gpu/drm/xe/xe_guc_pc.c                     |   3 +-
+ drivers/gpu/drm/xe/xe_guc_submit.c                 |  76 +-
+ drivers/gpu/drm/xe/xe_guc_submit_types.h           |   2 +
+ drivers/gpu/drm/xe/xe_i2c.c                        |  22 +-
+ drivers/gpu/drm/xe/xe_i2c.h                        |   1 -
+ drivers/gpu/drm/xe/xe_lrc.c                        |  73 ++
+ drivers/gpu/drm/xe/xe_lrc.h                        |   4 +
+ drivers/gpu/drm/xe/xe_lrc_types.h                  |   4 +
+ drivers/gpu/drm/xe/xe_migrate.c                    | 769 +++++++++-----------
+ drivers/gpu/drm/xe/xe_migrate.h                    |  95 +--
+ drivers/gpu/drm/xe/xe_pagefault.c                  |   3 +
+ drivers/gpu/drm/xe/xe_pci.c                        |   5 +
+ drivers/gpu/drm/xe/xe_pci_sriov.c                  |   3 +
+ drivers/gpu/drm/xe/xe_pci_types.h                  |   4 +-
+ drivers/gpu/drm/xe/xe_pt.c                         | 779 ++++++++++++---------
+ drivers/gpu/drm/xe/xe_pt.h                         |  12 +-
+ drivers/gpu/drm/xe/xe_pt_types.h                   |  49 +-
+ drivers/gpu/drm/xe/xe_ring_ops.c                   |  80 ++-
+ drivers/gpu/drm/xe/xe_ring_ops_types.h             |  24 +
+ drivers/gpu/drm/xe/xe_sched_job.c                  | 101 ++-
+ drivers/gpu/drm/xe/xe_sched_job.h                  |  56 ++
+ drivers/gpu/drm/xe/xe_sched_job_types.h            |  49 +-
+ drivers/gpu/drm/xe/xe_sriov_pf_provision.c         |   4 +-
+ drivers/gpu/drm/xe/xe_sync.c                       |  20 +-
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox.c            |   7 +-
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h      |   3 +
+ drivers/gpu/drm/xe/xe_tlb_inval.c                  |  54 ++
+ drivers/gpu/drm/xe/xe_tlb_inval_job.c              |  28 +-
+ drivers/gpu/drm/xe/xe_tlb_inval_job.h              |   4 +-
+ drivers/gpu/drm/xe/xe_tlb_inval_types.h            |  17 +
+ drivers/gpu/drm/xe/xe_trace.h                      |   2 +-
+ drivers/gpu/drm/xe/xe_tuning.c                     |   4 +
+ drivers/gpu/drm/xe/xe_uc_fw.c                      |   1 +
+ drivers/gpu/drm/xe/xe_vm.c                         | 242 +++----
+ drivers/gpu/drm/xe/xe_vm.h                         |   3 +
+ drivers/gpu/drm/xe/xe_vm_madvise.c                 |   6 -
+ drivers/gpu/drm/xe/xe_vm_types.h                   |  12 +-
+ drivers/gpu/drm/xe/xe_wa_oob.rules                 |   2 +
+ include/drm/intel/pciids.h                         |   1 +
+ 70 files changed, 2238 insertions(+), 1279 deletions(-)
+ create mode 100644 Documentation/ABI/testing/sysfs-driver-intel-xe-gpu
+ create mode 100644 drivers/gpu/drm/xe/xe_cpu_bind.c
+ create mode 100644 drivers/gpu/drm/xe/xe_cpu_bind.h
+$ git am -3 ../patches/0001-drm-xe-Fix-up-merge-issue.patch
+Applying: drm: xe: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+No changes -- Patch already applied.
+Merging drm-rust/for-linux-next (658c5f0042e6e gpu: nova-core: use a single try_pin_init!() block in probe())
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust/for-linux-next
+Auto-merging MAINTAINERS
+Auto-merging rust/bindings/bindings_helper.h
+Auto-merging rust/helpers/helpers.c
+Merge made by the 'ort' strategy.
+ Documentation/gpu/nova/core/fsp.rst                |    2 +
+ Documentation/gpu/nova/core/pramin.rst             |  128 +++
+ Documentation/gpu/nova/core/todo.rst               |    2 +-
+ Documentation/gpu/nova/index.rst                   |    1 +
+ MAINTAINERS                                        |    7 +
+ drivers/gpu/drm/nova/driver.rs                     |   19 +-
+ drivers/gpu/drm/nova/file.rs                       |  119 ++-
+ drivers/gpu/drm/tyr/driver.rs                      |    1 +
+ drivers/gpu/drm/tyr/fw.rs                          |    5 +-
+ drivers/gpu/drm/tyr/gpu.rs                         |    6 +-
+ drivers/gpu/drm/tyr/regs.rs                        |   50 +-
+ drivers/gpu/nova-core/Kconfig                      |   10 +
+ drivers/gpu/nova-core/api.rs                       |   68 ++
+ drivers/gpu/nova-core/driver.rs                    |  109 ++-
+ drivers/gpu/nova-core/falcon.rs                    |  164 ++--
+ drivers/gpu/nova-core/falcon/fsp.rs                |   63 +-
+ drivers/gpu/nova-core/falcon/gsp.rs                |   51 +-
+ drivers/gpu/nova-core/falcon/hal/ga102.rs          |   59 +-
+ drivers/gpu/nova-core/falcon/hal/tu102.rs          |    9 +-
+ drivers/gpu/nova-core/falcon/sec2.rs               |   37 +-
+ drivers/gpu/nova-core/fb.rs                        |   22 +-
+ drivers/gpu/nova-core/fb/hal/gb100.rs              |   59 +-
+ drivers/gpu/nova-core/fb/regs.rs                   |   34 +-
+ drivers/gpu/nova-core/firmware.rs                  |    3 +-
+ drivers/gpu/nova-core/firmware/booter.rs           |    2 +-
+ drivers/gpu/nova-core/firmware/fwsec/bootloader.rs |   30 +-
+ drivers/gpu/nova-core/firmware/gsp.rs              |  119 +--
+ .../gpu/nova-core/firmware/{fsp.rs => gsp_fmc.rs}  |   59 +-
+ drivers/gpu/nova-core/firmware/radix3.rs           |  144 +++
+ drivers/gpu/nova-core/firmware/riscv.rs            |    8 +-
+ drivers/gpu/nova-core/fsp.rs                       |   45 +-
+ drivers/gpu/nova-core/gpu.rs                       |  231 +++--
+ drivers/gpu/nova-core/gpu/regs.rs                  |   86 ++
+ drivers/gpu/nova-core/gsp.rs                       |   39 +-
+ drivers/gpu/nova-core/gsp/boot.rs                  |   26 +-
+ drivers/gpu/nova-core/gsp/cmdq.rs                  |   98 +-
+ drivers/gpu/nova-core/gsp/commands.rs              |   40 +-
+ drivers/gpu/nova-core/gsp/fw.rs                    |   18 +-
+ drivers/gpu/nova-core/gsp/fw/commands.rs           |   28 +
+ drivers/gpu/nova-core/gsp/hal.rs                   |   16 +-
+ drivers/gpu/nova-core/gsp/hal/gh100.rs             |   16 +-
+ drivers/gpu/nova-core/gsp/hal/tu102.rs             |   41 +-
+ drivers/gpu/nova-core/gsp/regs.rs                  |    9 +-
+ drivers/gpu/nova-core/gsp/sequencer.rs             |    6 +-
+ drivers/gpu/nova-core/mctp.rs                      |   16 +-
+ drivers/gpu/nova-core/mm.rs                        |  335 +++++++
+ drivers/gpu/nova-core/mm/bar_user.rs               |  420 ++++++++
+ drivers/gpu/nova-core/mm/hal.rs                    |   56 ++
+ drivers/gpu/nova-core/mm/hal/gb100.rs              |   35 +
+ drivers/gpu/nova-core/mm/hal/gh100.rs              |   35 +
+ drivers/gpu/nova-core/mm/hal/tu102.rs              |   37 +
+ drivers/gpu/nova-core/mm/pagetable.rs              |  424 ++++++++
+ drivers/gpu/nova-core/mm/pagetable/map.rs          |  345 +++++++
+ drivers/gpu/nova-core/mm/pagetable/ver2.rs         |  275 ++++++
+ drivers/gpu/nova-core/mm/pagetable/ver3.rs         |  421 ++++++++
+ drivers/gpu/nova-core/mm/pagetable/walk.rs         |  244 +++++
+ drivers/gpu/nova-core/mm/pramin.rs                 |  312 ++++++
+ drivers/gpu/nova-core/mm/regs.rs                   |   70 ++
+ drivers/gpu/nova-core/mm/tlb.rs                    |  120 +++
+ drivers/gpu/nova-core/mm/vmm.rs                    |  346 +++++++
+ drivers/gpu/nova-core/nova_core.rs                 |    4 +
+ drivers/gpu/nova-core/num.rs                       |    2 +-
+ drivers/gpu/nova-core/regs.rs                      |  257 ++---
+ drivers/gpu/nova-core/selftest.rs                  |   64 ++
+ drivers/gpu/nova-core/vbios.rs                     |   11 +-
+ include/uapi/drm/nova_drm.h                        |  128 +++
+ rust/bindings/bindings_helper.h                    |    1 +
+ rust/helpers/dma_fence.c                           |   49 +
+ rust/helpers/helpers.c                             |    1 +
+ rust/helpers/pci.c                                 |    6 +
+ rust/kernel/auxiliary.rs                           |   17 +-
+ rust/kernel/bitfield.rs                            |    9 +
+ rust/kernel/debugfs.rs                             |   26 +-
+ rust/kernel/debugfs/entry.rs                       |    4 +-
+ rust/kernel/debugfs/file_ops.rs                    |   29 +-
+ rust/kernel/device_id.rs                           |    3 +-
+ rust/kernel/dma.rs                                 |  141 ++-
+ rust/kernel/dma_buf/dma_fence.rs                   | 1022 ++++++++++++++++++++
+ rust/kernel/dma_buf/mod.rs                         |   14 +
+ rust/kernel/io.rs                                  |  220 +++--
+ rust/kernel/io/register.rs                         |  803 ++++-----------
+ rust/kernel/io/resource.rs                         |    8 +
+ rust/kernel/lib.rs                                 |    3 +
+ rust/kernel/maple_tree.rs                          |   30 +-
+ rust/kernel/mem.rs                                 |  234 +++++
+ rust/kernel/pci.rs                                 |   14 +
+ rust/kernel/sync/atomic.rs                         |    4 +-
+ rust/kernel/uaccess.rs                             |   18 +-
+ rust/macros/io/mod.rs                              |    3 +
+ rust/macros/io/register.rs                         |  296 ++++++
+ rust/macros/lib.rs                                 |    9 +
+ samples/rust/rust_dma.rs                           |   30 +-
+ samples/rust/rust_driver_pci.rs                    |    4 +
+ 93 files changed, 7452 insertions(+), 1592 deletions(-)
+ create mode 100644 Documentation/gpu/nova/core/pramin.rst
+ create mode 100644 drivers/gpu/nova-core/api.rs
+ rename drivers/gpu/nova-core/firmware/{fsp.rs => gsp_fmc.rs} (67%)
+ create mode 100644 drivers/gpu/nova-core/firmware/radix3.rs
+ create mode 100644 drivers/gpu/nova-core/gpu/regs.rs
+ create mode 100644 drivers/gpu/nova-core/mm.rs
+ create mode 100644 drivers/gpu/nova-core/mm/bar_user.rs
+ create mode 100644 drivers/gpu/nova-core/mm/hal.rs
+ create mode 100644 drivers/gpu/nova-core/mm/hal/gb100.rs
+ create mode 100644 drivers/gpu/nova-core/mm/hal/gh100.rs
+ create mode 100644 drivers/gpu/nova-core/mm/hal/tu102.rs
+ create mode 100644 drivers/gpu/nova-core/mm/pagetable.rs
+ create mode 100644 drivers/gpu/nova-core/mm/pagetable/map.rs
+ create mode 100644 drivers/gpu/nova-core/mm/pagetable/ver2.rs
+ create mode 100644 drivers/gpu/nova-core/mm/pagetable/ver3.rs
+ create mode 100644 drivers/gpu/nova-core/mm/pagetable/walk.rs
+ create mode 100644 drivers/gpu/nova-core/mm/pramin.rs
+ create mode 100644 drivers/gpu/nova-core/mm/regs.rs
+ create mode 100644 drivers/gpu/nova-core/mm/tlb.rs
+ create mode 100644 drivers/gpu/nova-core/mm/vmm.rs
+ create mode 100644 drivers/gpu/nova-core/selftest.rs
+ create mode 100644 rust/helpers/dma_fence.c
+ create mode 100644 rust/kernel/dma_buf/dma_fence.rs
+ create mode 100644 rust/kernel/dma_buf/mod.rs
+ create mode 100644 rust/kernel/mem.rs
+ create mode 100644 rust/macros/io/mod.rs
+ create mode 100644 rust/macros/io/register.rs
+Merging drm-nova/nova-next (93296e9d9528f gpu: nova-core: vbios: store reference to Device where relevant)
+$ git merge -m Merge branch 'nova-next' of https://gitlab.freedesktop.org/drm/nova.git drm-nova/nova-next
+Already up to date.
+Merging etnaviv/etnaviv/next (6bde14ba5f7ef drm/etnaviv: add optional reset support)
+$ git merge -m Merge branch 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux etnaviv/etnaviv/next
+Already up to date.
+Merging fbdev/for-next (9f4c6043f33c9 fbdev: atafb: avoid cast when assigning buffer address pointer)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git fbdev/for-next
+Merge made by the 'ort' strategy.
+ drivers/video/fbdev/acornfb.c                   | 57 ++++++-------------------
+ drivers/video/fbdev/acornfb.h                   |  9 +---
+ drivers/video/fbdev/atafb.c                     | 28 +-----------
+ drivers/video/fbdev/aty/aty128fb.c              |  2 +-
+ drivers/video/fbdev/aty/atyfb_base.c            |  4 +-
+ drivers/video/fbdev/aty/mach64_cursor.c         |  2 +-
+ drivers/video/fbdev/aty/radeon_base.c           |  4 +-
+ drivers/video/fbdev/aty/radeon_monitor.c        |  4 +-
+ drivers/video/fbdev/aty/radeonfb.h              |  2 +-
+ drivers/video/fbdev/au1100fb.c                  |  2 +-
+ drivers/video/fbdev/core/Kconfig                |  6 ---
+ drivers/video/fbdev/efifb.c                     |  4 +-
+ drivers/video/fbdev/grvga.c                     |  2 +-
+ drivers/video/fbdev/gxt4500.c                   |  1 +
+ drivers/video/fbdev/kyro/STG4000OverlayDevice.c |  2 +-
+ drivers/video/fbdev/kyro/fbdev.c                |  4 +-
+ drivers/video/fbdev/maxinefb.c                  |  2 +-
+ drivers/video/fbdev/mmp/core.c                  |  2 +-
+ drivers/video/fbdev/mmp/hw/mmp_ctrl.c           |  4 +-
+ drivers/video/fbdev/mmp/hw/mmp_ctrl.h           |  4 +-
+ drivers/video/fbdev/mmp/hw/mmp_spi.c            |  2 +-
+ drivers/video/fbdev/nvidia/nvidia.c             |  1 +
+ drivers/video/fbdev/ocfb.c                      |  2 +-
+ drivers/video/fbdev/omap2/omapfb/dss/dispc.c    |  4 +-
+ drivers/video/fbdev/pm2fb.c                     |  2 +-
+ drivers/video/fbdev/s1d13xxxfb.c                |  6 +--
+ drivers/video/fbdev/sh_mobile_lcdcfb.c          | 37 ++++++++++------
+ drivers/video/fbdev/skeletonfb.c                |  6 +--
+ drivers/video/fbdev/sstfb.c                     |  5 ++-
+ drivers/video/fbdev/tgafb.c                     |  6 +--
+ drivers/video/fbdev/tridentfb.c                 |  2 +-
+ drivers/video/fbdev/udlfb.c                     |  5 +++
+ drivers/video/fbdev/via/dvi.c                   |  6 +--
+ drivers/video/fbdev/via/hw.c                    |  2 +-
+ drivers/video/fbdev/via/viafbdev.c              |  4 +-
+ drivers/video/sticore.c                         |  2 +-
+ include/video/mmp_disp.h                        |  6 +--
+ 37 files changed, 100 insertions(+), 143 deletions(-)
+$ git am -3 ../patches/0001-fix-up-for-drm-hyperv-Remove-reference-to-hyperv_fb-.patch
+Applying: fix up for "drm/hyperv: Remove reference to hyperv_fb driver"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/gpu/drm/hyperv/Kconfig
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 5bee4e96fdc0d] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git
+ Date: Wed Sep 30 13:14:38 2026 +0100
+Merging regmap/for-next (117e6a5fd98fb Merge regmap/for-7.4 into regmap-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap/for-next
+Merge made by the 'ort' strategy.
+ drivers/base/regmap/internal.h        |  12 +++
+ drivers/base/regmap/regcache-rbtree.c |   4 +-
+ drivers/base/regmap/regcache.c        |  28 ++----
+ drivers/base/regmap/regmap-debugfs.c  |  24 ++---
+ drivers/base/regmap/regmap-kunit.c    |  98 +++++++++++++++++-
+ drivers/base/regmap/regmap-ram.c      |  26 +++--
+ drivers/base/regmap/regmap-raw-ram.c  |  21 ++--
+ drivers/base/regmap/regmap.c          | 184 +++++++++++-----------------------
+ 8 files changed, 219 insertions(+), 178 deletions(-)
+Merging sound/for-next (d4febca239be3 ALSA: control: Fix UAF in snd_ctl_elem_add() on card disconnect)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/acpi/scan.c
+Auto-merging sound/core/pcm_native.c
+Merge made by the 'ort' strategy.
+ Documentation/sound/alsa-configuration.rst         |   6 +
+ Documentation/sound/cards/hdspm.rst                | 109 ++---
+ MAINTAINERS                                        |   7 +
+ drivers/acpi/scan.c                                |   1 +
+ drivers/platform/x86/serial-multi-instantiate.c    |   1 +
+ include/sound/control.h                            |   2 +-
+ include/sound/emu10k1.h                            |   4 +-
+ include/sound/emux_legacy.h                        |   2 +-
+ include/sound/rawmidi.h                            |   1 +
+ include/uapi/sound/asound.h                        |   3 +-
+ sound/core/control.c                               |   2 +
+ sound/core/init.c                                  |  12 +-
+ sound/core/pcm_native.c                            |   6 +-
+ sound/core/rawmidi.c                               |  83 +++-
+ sound/core/seq/Kconfig                             |   7 +-
+ sound/core/seq/oss/seq_oss.c                       |   3 +
+ sound/core/seq/oss/seq_oss_init.c                  |   2 +-
+ sound/core/timer.c                                 |   1 +
+ sound/core/timer_compat.c                          |   1 +
+ sound/drivers/dummy.c                              |  10 +-
+ sound/drivers/mpu401/mpu401_uart.c                 |   8 +-
+ sound/drivers/serial-generic.c                     |  10 +-
+ sound/firewire/bebob/bebob_hwdep.c                 |   2 +-
+ sound/hda/codecs/conexant.c                        |  67 +++
+ sound/hda/codecs/hdmi/hdmi.c                       |   2 +-
+ sound/hda/codecs/realtek/alc269.c                  | 147 +++++-
+ sound/hda/codecs/realtek/alc882.c                  |   2 +-
+ sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c     |   3 +
+ .../hda/codecs/side-codecs/cs35l41_hda_property.c  |  28 ++
+ sound/hda/codecs/sigmatel.c                        |   6 +-
+ sound/hda/common/codec.c                           |   2 +-
+ sound/hda/controllers/acpi.c                       |  41 ++
+ sound/hda/core/bus.c                               |   2 +-
+ sound/hda/core/component.c                         |   5 +-
+ sound/hda/core/controller.c                        |   5 +-
+ sound/i2c/other/ak4113.c                           |   2 +-
+ sound/i2c/other/ak4114.c                           |   2 +-
+ sound/i2c/other/ak4xxx-adda.c                      |   2 +-
+ sound/isa/opti9xx/opti92x-ad1848.c                 |   2 +-
+ sound/mips/hal2.c                                  |   4 +-
+ sound/mips/sgio2audio.c                            |  13 +-
+ sound/oss/dmasound/dmasound_q40.c                  |  65 ++-
+ sound/pci/ac97/ac97_codec.c                        |   2 +-
+ sound/pci/ac97/ac97_patch.c                        |   2 +-
+ sound/pci/au88x0/au88x0.h                          |   2 +-
+ sound/pci/au88x0/au88x0_mpu401.c                   |   2 +-
+ sound/pci/aw2/aw2-saa7146.c                        |   4 +-
+ sound/pci/cs4281.c                                 |   2 +-
+ sound/pci/cs46xx/cs46xx_lib.c                      |   6 +-
+ sound/pci/cs46xx/dsp_spos_scb_lib.c                |   8 +-
+ sound/pci/echoaudio/echoaudio.c                    |   4 +-
+ sound/pci/echoaudio/echoaudio.h                    |   4 +-
+ sound/pci/lx6464es/lx_core.c                       |   2 +-
+ sound/pci/lx6464es/lx_defs.h                       |   2 +-
+ sound/pci/oxygen/oxygen_pcm.c                      |  37 +-
+ sound/pci/pcxhr/pcxhr_core.c                       |   4 +-
+ sound/pci/riptide/riptide.c                        |   4 +-
+ sound/pci/rme32.c                                  |   2 +-
+ sound/pci/rme9652/hdsp.c                           |   2 +-
+ sound/pci/trident/trident_memory.c                 |   2 +-
+ sound/pci/via82xx.c                                |  11 +-
+ sound/pci/vx222/vx222_ops.c                        |   2 +-
+ sound/pcmcia/vx/vxp_ops.c                          |   2 +-
+ sound/ppc/tumbler.c                                |   2 +-
+ sound/soc/codecs/tas675x.c                         |  10 +-
+ sound/usb/Makefile                                 |   1 +
+ sound/usb/caiaq/Makefile                           |   2 +-
+ sound/usb/caiaq/device.c                           |  12 +
+ sound/usb/caiaq/device.h                           |  13 +
+ sound/usb/caiaq/input.c                            |  29 +-
+ sound/usb/caiaq/lcd.c                              | 258 +++++++++++
+ sound/usb/caiaq/lcd.h                              |   7 +
+ sound/usb/clock.c                                  |  27 ++
+ sound/usb/midi.c                                   |  69 +--
+ sound/usb/mixer.c                                  |   4 +-
+ sound/usb/mixer_evo.c                              | 505 +++++++++++++++++++++
+ sound/usb/mixer_evo.h                              |  12 +
+ sound/usb/mixer_maps.c                             |  13 +
+ sound/usb/mixer_quirks.c                           |   5 +
+ sound/usb/mixer_us16x08.h                          |   2 +-
+ sound/usb/quirks-table.h                           | 253 ++++++++++-
+ sound/usb/quirks.c                                 | 235 +++++++++-
+ sound/usb/usbaudio.h                               |   6 +
+ tools/testing/selftests/alsa/mixer-test.c          | 255 +++++++++++
+ tools/testing/selftests/alsa/utimer-test.c         |   6 +
+ 85 files changed, 2215 insertions(+), 300 deletions(-)
+ create mode 100644 sound/usb/caiaq/lcd.c
+ create mode 100644 sound/usb/caiaq/lcd.h
+ create mode 100644 sound/usb/mixer_evo.c
+ create mode 100644 sound/usb/mixer_evo.h
+Merging ieee1394/for-next (a6e7c3836b812 firewire: cdev: use kzalloc_flex() to allocate structure with byte array)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git ieee1394/for-next
+Merge made by the 'ort' strategy.
+ drivers/firewire/.kunitconfig                |   1 +
+ drivers/firewire/Kconfig                     |  16 ++
+ drivers/firewire/config-rom-generator-test.c | 407 +++++++++++++++++++++++++++
+ drivers/firewire/config-rom-parser-test.c    | 355 +++++++++++++++++++++++
+ drivers/firewire/core-card.c                 |  11 +-
+ drivers/firewire/core-cdev.c                 | 299 ++++++++++----------
+ drivers/firewire/core-device.c               |   4 +
+ drivers/firewire/core-transaction.c          | 146 +++++-----
+ drivers/firewire/device-attribute-test.c     |   4 +-
+ drivers/firewire/ohci-serdes-test.c          |   4 +-
+ drivers/firewire/ohci.c                      | 267 ++++++++++++------
+ drivers/firewire/ohci.h                      |   2 +-
+ drivers/firewire/uapi-test.c                 |  20 ++
+ include/linux/firewire.h                     |  50 ++--
+ include/uapi/linux/firewire-cdev.h           |   4 +-
+ tools/firewire/nosy-dump.c                   |  10 +-
+ 16 files changed, 1252 insertions(+), 348 deletions(-)
+ create mode 100644 drivers/firewire/config-rom-generator-test.c
+ create mode 100644 drivers/firewire/config-rom-parser-test.c
+Merging sound-asoc/for-next (0fbfaeecfd6f5 Merge asoc/for-7.4 into asoc-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging MAINTAINERS
+Auto-merging sound/soc/amd/acp-es8336.c
+Auto-merging sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c
+Auto-merging sound/soc/intel/boards/sof_cirrus_common.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/sound/adi,adau1372.yaml    |   10 +-
+ .../devicetree/bindings/sound/adi,adau7118.yaml    |   22 +-
+ .../bindings/sound/airoha,an7581-afe.yaml          |   41 +
+ .../bindings/sound/airoha,an7581-wm8960.yaml       |   71 +
+ .../bindings/sound/amlogic,gx-sound-card.yaml      |   20 +-
+ .../bindings/sound/asahi-kasei,ak4619.yaml         |   16 +-
+ .../bindings/sound/audio-graph-port.yaml           |   10 +-
+ .../devicetree/bindings/sound/cirrus,cs35l45.yaml  |   60 +-
+ .../devicetree/bindings/sound/cirrus,cs4270.yaml   |   12 +-
+ .../devicetree/bindings/sound/cirrus,cs42l42.yaml  |   42 +-
+ .../devicetree/bindings/sound/cirrus,cs42l84.yaml  |   18 +-
+ .../devicetree/bindings/sound/cirrus,cs42xx8.yaml  |   56 +-
+ .../bindings/sound/davinci-evm-audio.txt           |   49 -
+ .../devicetree/bindings/sound/dialog,da7219.yaml   |   62 +-
+ .../bindings/sound/esstech,es9039q2m.yaml          |   94 +
+ .../devicetree/bindings/sound/everest,es8316.yaml  |    5 +
+ .../bindings/sound/foursemi,fs2105s.yaml           |    2 +-
+ .../devicetree/bindings/sound/fsl,easrc.yaml       |   45 +
+ .../devicetree/bindings/sound/fsl,imx-asrc.yaml    |   45 +-
+ .../devicetree/bindings/sound/fsl,sai.yaml         |   14 +-
+ .../bindings/sound/hisilicon,hi6210-i2s.yaml       |    4 +
+ .../devicetree/bindings/sound/imx-audio-card.yaml  |    4 +-
+ .../devicetree/bindings/sound/imx-audmux.yaml      |    8 +-
+ .../bindings/sound/invensense,ics43432.yaml        |    8 +-
+ .../bindings/sound/loongson,ls-audio-card.yaml     |    4 +-
+ .../bindings/sound/mediatek,mt2701-audio.yaml      |   10 +-
+ .../bindings/sound/mediatek,mt8173-afe-pcm.yaml    |   22 +-
+ .../bindings/sound/mediatek,mt8183-audio.yaml      |   82 +-
+ .../sound/mt8186-mt6366-da7219-max98357.yaml       |   36 +-
+ .../sound/mt8186-mt6366-rt1019-rt5682s.yaml        |   36 +-
+ .../sound/mt8192-mt6359-rt1015-rt5682.yaml         |   46 +-
+ .../devicetree/bindings/sound/mt8195-mt6359.yaml   |   50 +-
+ .../devicetree/bindings/sound/nuvoton,nau8360.yaml |  115 +
+ .../bindings/sound/qcom,q6apm-lpass-dais.yaml      |   12 +-
+ .../bindings/sound/qcom,q6dsp-lpass-ports.yaml     |   16 +-
+ .../devicetree/bindings/sound/qcom,sm8250.yaml     |    1 +
+ .../bindings/sound/qcom,wcd9378-sdw.yaml           |  105 +
+ .../devicetree/bindings/sound/realtek,rt5677.yaml  |   10 +
+ .../devicetree/bindings/sound/renesas,rsnd.yaml    |   16 +-
+ .../devicetree/bindings/sound/renesas,rz-ssi.yaml  |   32 +-
+ .../bindings/sound/stericsson,ux500-msp-i2s.yaml   |   96 +
+ .../bindings/sound/ti,da830-evm-audio.yaml         |   83 +
+ .../devicetree/bindings/sound/ti,tas2781.yaml      |    4 +-
+ .../devicetree/bindings/sound/ti,tas5805m.yaml     |   10 +-
+ .../bindings/sound/ti,tlv320dac3100.yaml           |    2 +-
+ .../devicetree/bindings/sound/ti,tpa6130a2.yaml    |    2 +-
+ .../devicetree/bindings/sound/ux500-mop500.txt     |   39 -
+ .../devicetree/bindings/sound/ux500-msp.txt        |   42 -
+ .../devicetree/bindings/vendor-prefixes.yaml       |    2 +
+ MAINTAINERS                                        |   18 +
+ drivers/firmware/cirrus/cs_dsp.c                   |    2 +-
+ drivers/gpib/fmh_gpib/fmh_gpib.c                   |   10 +-
+ include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h |   52 +
+ include/linux/dmaengine.h                          |    9 -
+ include/linux/firmware/cirrus/cs_dsp.h             |    2 +-
+ include/sound/sdca.h                               |   23 +-
+ include/sound/sdca_class.h                         |   70 +
+ include/sound/simple_card_utils.h                  |    2 +
+ include/sound/soc-component.h                      |   14 +-
+ include/sound/soc-dai.h                            |  179 +-
+ include/sound/soc.h                                |   16 +-
+ include/sound/soc_sdw_utils.h                      |   20 +
+ include/sound/tas2781-dsp.h                        |   13 +
+ include/sound/tas2781.h                            |   24 +
+ sound/hda/core/intel-nhlt.c                        |    4 +-
+ sound/soc/amd/acp-es8336.c                         |    3 +-
+ sound/soc/amd/acp/Kconfig                          |    4 +-
+ sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c      |   12 +
+ sound/soc/amd/acp/soc_amd_sdw_common.h             |   13 +
+ sound/soc/atmel/atmel-pcm-dma.c                    |    2 +-
+ sound/soc/bcm/cygnus-ssp.h                         |    2 +-
+ sound/soc/codecs/88pm860x-codec.c                  |    6 +
+ sound/soc/codecs/Kconfig                           |   49 +
+ sound/soc/codecs/Makefile                          |    8 +
+ sound/soc/codecs/ab8500-codec.c                    |  157 +-
+ sound/soc/codecs/adau1372.c                        |   12 +
+ sound/soc/codecs/adau1373.c                        |   12 +
+ sound/soc/codecs/adau1701.c                        |   15 +-
+ sound/soc/codecs/adau17x1.c                        |   17 +-
+ sound/soc/codecs/adau1977.c                        |   13 +
+ sound/soc/codecs/adau7118.c                        |   12 +
+ sound/soc/codecs/ak4104.c                          |    7 +
+ sound/soc/codecs/ak4118.c                          |    7 +
+ sound/soc/codecs/ak4458.c                          |    9 +
+ sound/soc/codecs/ak4535.c                          |    6 +
+ sound/soc/codecs/ak4619.c                          |   32 +
+ sound/soc/codecs/ak4642.c                          |   14 +-
+ sound/soc/codecs/ak4671.c                          |    7 +
+ sound/soc/codecs/ak5386.c                          |    6 +
+ sound/soc/codecs/ak5558.c                          |    7 +
+ sound/soc/codecs/alc5623.c                         |   11 +
+ sound/soc/codecs/alc5632.c                         |   13 +
+ sound/soc/codecs/arizona-jack.c                    |    1 +
+ sound/soc/codecs/arizona.c                         |   23 +
+ sound/soc/codecs/cpcap.c                           |   87 +-
+ sound/soc/codecs/cs35l33.c                         |    8 +-
+ sound/soc/codecs/cs35l34.c                         |    2 +-
+ sound/soc/codecs/cs35l35.c                         |    7 +
+ sound/soc/codecs/cs35l36.c                         |   56 +
+ sound/soc/codecs/cs35l41.c                         |   10 +
+ sound/soc/codecs/cs35l45-tables.c                  |    4 +
+ sound/soc/codecs/cs35l45.c                         |   97 +-
+ sound/soc/codecs/cs35l45.h                         |   23 +
+ sound/soc/codecs/cs35l56-shared-test.c             |   56 +
+ sound/soc/codecs/cs35l56.c                         |   10 +
+ sound/soc/codecs/cs40l50-codec.c                   |    9 +
+ sound/soc/codecs/cs4234.c                          |    9 +
+ sound/soc/codecs/cs4265.c                          |   12 +
+ sound/soc/codecs/cs4270.c                          |    6 +
+ sound/soc/codecs/cs4271.c                          |    6 +
+ sound/soc/codecs/cs42l42.c                         |   11 +-
+ sound/soc/codecs/cs42l43.c                         |   12 +
+ sound/soc/codecs/cs42l51.c                         |    7 +
+ sound/soc/codecs/cs42l52.c                         |   22 +-
+ sound/soc/codecs/cs42l56.c                         |   17 +-
+ sound/soc/codecs/cs42l73.c                         |   11 +-
+ sound/soc/codecs/cs42l84.c                         |    6 +
+ sound/soc/codecs/cs42xx8.c                         |    8 +
+ sound/soc/codecs/cs43130.c                         |   14 +
+ sound/soc/codecs/cs4341.c                          |    8 +
+ sound/soc/codecs/cs4349.c                          |   12 +
+ sound/soc/codecs/cs48l32.c                         |   23 +
+ sound/soc/codecs/cs530x.c                          |    9 +
+ sound/soc/codecs/cs53l30.c                         |    8 +
+ sound/soc/codecs/cx20442.c                         |    2 +-
+ sound/soc/codecs/cx2072x.c                         |   12 +
+ sound/soc/codecs/da7210.c                          |    9 +-
+ sound/soc/codecs/da7218.c                          |  141 +-
+ sound/soc/codecs/da7219.c                          |   12 +
+ sound/soc/codecs/da732x.c                          |   17 +
+ sound/soc/codecs/da9055.c                          |    8 +
+ sound/soc/codecs/es7134.c                          |    6 +
+ sound/soc/codecs/es7241.c                          |    7 +
+ sound/soc/codecs/es8311.c                          |   27 +-
+ sound/soc/codecs/es8316.c                          |   27 +-
+ sound/soc/codecs/es8323.c                          |   32 +-
+ sound/soc/codecs/es8326.c                          |  181 +-
+ sound/soc/codecs/es8326.h                          |    4 +
+ sound/soc/codecs/es8328.c                          |   20 +-
+ sound/soc/codecs/es8375.c                          |   30 +-
+ sound/soc/codecs/es8389.c                          |   65 +-
+ sound/soc/codecs/es9039q2m.c                       | 2008 +++++++++++++++++
+ sound/soc/codecs/fs210x.c                          |    5 +-
+ sound/soc/codecs/hda.c                             |    7 +-
+ sound/soc/codecs/hdac_hda.c                        |    2 +-
+ sound/soc/codecs/hdac_hdmi.c                       |    2 +-
+ sound/soc/codecs/inno_rk3036.c                     |   12 +
+ sound/soc/codecs/isabelle.c                        |   13 +
+ sound/soc/codecs/lm49453.c                         |   15 +
+ sound/soc/codecs/lpass-rx-macro.c                  |    6 +-
+ sound/soc/codecs/lpass-tx-macro.c                  |    4 +-
+ sound/soc/codecs/lpass-va-macro.c                  |    4 +-
+ sound/soc/codecs/lpass-wsa-macro.c                 |   48 +-
+ sound/soc/codecs/max98088.c                        |   12 +
+ sound/soc/codecs/max98090.c                        |   35 +-
+ sound/soc/codecs/max98095.c                        |   24 +-
+ sound/soc/codecs/max98371.c                        |    7 +
+ sound/soc/codecs/max98373-i2c.c                    |   10 +
+ sound/soc/codecs/max98388.c                        |   10 +
+ sound/soc/codecs/max98390.c                        |   10 +
+ sound/soc/codecs/max98396.c                        |   12 +
+ sound/soc/codecs/max9850.c                         |   11 +
+ sound/soc/codecs/max98520.c                        |   10 +
+ sound/soc/codecs/max9860.c                         |   23 +
+ sound/soc/codecs/max9867.c                         |   10 +
+ sound/soc/codecs/max98925.c                        |    8 +
+ sound/soc/codecs/max98926.c                        |    8 +
+ sound/soc/codecs/max98927.c                        |   14 +-
+ sound/soc/codecs/mc13783.c                         |   12 +
+ sound/soc/codecs/ml26124.c                         |    6 +
+ sound/soc/codecs/mt6359.c                          |    8 +-
+ sound/soc/codecs/nau8325.c                         |   11 +
+ sound/soc/codecs/nau8360-dsp.c                     |  634 ++++++
+ sound/soc/codecs/nau8360-dsp.h                     |  122 ++
+ sound/soc/codecs/nau8360.c                         | 2309 ++++++++++++++++++++
+ sound/soc/codecs/nau8360.h                         |  917 ++++++++
+ sound/soc/codecs/nau8540.c                         |   15 +-
+ sound/soc/codecs/nau8810.c                         |   12 +
+ sound/soc/codecs/nau8821.c                         |   11 +
+ sound/soc/codecs/nau8822.c                         |   27 +-
+ sound/soc/codecs/nau8824.c                         |   11 +
+ sound/soc/codecs/nau8825.c                         |   29 +-
+ sound/soc/codecs/ntp8835.c                         |    7 +
+ sound/soc/codecs/ntp8918.c                         |    7 +
+ sound/soc/codecs/pcm5102a.c                        |   30 +-
+ sound/soc/codecs/pcm6240.c                         |    8 +-
+ sound/soc/codecs/rk3308_codec.c                    |   11 +
+ sound/soc/codecs/rk3328_codec.c                    |    9 +
+ sound/soc/codecs/rt1011.c                          |   44 +-
+ sound/soc/codecs/rt1015.c                          |   91 +-
+ sound/soc/codecs/rt1016.c                          |   14 +-
+ sound/soc/codecs/rt1019.c                          |   10 +
+ sound/soc/codecs/rt1305.c                          |   14 +-
+ sound/soc/codecs/rt1308.c                          |   14 +-
+ sound/soc/codecs/rt1318.c                          |   13 +-
+ sound/soc/codecs/rt1320-sdw.c                      |   12 +-
+ sound/soc/codecs/rt274.c                           |    8 +
+ sound/soc/codecs/rt286.c                           |    8 +
+ sound/soc/codecs/rt298.c                           |    8 +
+ sound/soc/codecs/rt5514-spi.c                      |   37 +-
+ sound/soc/codecs/rt5514.c                          |   39 +-
+ sound/soc/codecs/rt5616.c                          |   23 +-
+ sound/soc/codecs/rt5631.c                          |   10 +
+ sound/soc/codecs/rt5631.h                          |    8 +-
+ sound/soc/codecs/rt5640.c                          |   22 +-
+ sound/soc/codecs/rt5645.c                          |   23 +-
+ sound/soc/codecs/rt5651.c                          |   12 +-
+ sound/soc/codecs/rt5659.c                          |   14 +-
+ sound/soc/codecs/rt5660.c                          |   14 +-
+ sound/soc/codecs/rt5663.c                          |   10 +
+ sound/soc/codecs/rt5665.c                          |   14 +-
+ sound/soc/codecs/rt5668.c                          |   22 +
+ sound/soc/codecs/rt5670.c                          |   12 +-
+ sound/soc/codecs/rt5677-spi.c                      |   21 +-
+ sound/soc/codecs/rt5677.c                          |   10 +
+ sound/soc/codecs/rt5682.c                          |   22 +
+ sound/soc/codecs/rt5682s.c                         |   34 +-
+ sound/soc/codecs/rt712-sdca-dmic.c                 |    4 +-
+ sound/soc/codecs/rt712-sdca-sdw.c                  |    1 +
+ sound/soc/codecs/rt712-sdca-sdw.h                  |    3 +
+ sound/soc/codecs/rt712-sdca.c                      |  136 ++
+ sound/soc/codecs/rt712-sdca.h                      |    6 +
+ sound/soc/codecs/rt721-sdca-sdw.c                  |   15 +
+ sound/soc/codecs/rt721-sdca.c                      |  498 +++--
+ sound/soc/codecs/rt721-sdca.h                      |   17 +
+ sound/soc/codecs/rt766-sdca.c                      |    2 +-
+ sound/soc/codecs/rt9120.c                          |    9 +
+ sound/soc/codecs/rt9123.c                          |   13 +
+ sound/soc/codecs/rtq9124.c                         |    9 +
+ sound/soc/codecs/rtq9128.c                         |    9 +
+ sound/soc/codecs/sgtl5000.c                        |   11 +
+ sound/soc/codecs/si476x.c                          |   18 +
+ sound/soc/codecs/sma1303.c                         |   13 +
+ sound/soc/codecs/sma1307.c                         |   27 +-
+ sound/soc/codecs/sn624x-sdca-sdw.c                 | 1888 ++++++++++++++++
+ sound/soc/codecs/sn624x-sdca.h                     |  152 ++
+ sound/soc/codecs/src4xxx.c                         |    8 +
+ sound/soc/codecs/ssm2518.c                         |   13 +
+ sound/soc/codecs/ssm2602.c                         |   13 +
+ sound/soc/codecs/ssm3515.c                         |   10 +
+ sound/soc/codecs/ssm4567.c                         |   13 +
+ sound/soc/codecs/sta32x.c                          |  116 +-
+ sound/soc/codecs/sta350.c                          |   15 +-
+ sound/soc/codecs/sta529.c                          |    7 +
+ sound/soc/codecs/tas2552.c                         |   14 +
+ sound/soc/codecs/tas2562.c                         |   10 +
+ sound/soc/codecs/tas2764.c                         |   14 +-
+ sound/soc/codecs/tas2770.c                         |   14 +-
+ sound/soc/codecs/tas2780.c                         |   10 +
+ sound/soc/codecs/tas2781-comlib-i2c.c              |    7 +-
+ sound/soc/codecs/tas2781-comlib.c                  |   22 +-
+ sound/soc/codecs/tas2781-fmwlib.c                  |   40 +-
+ sound/soc/codecs/tas2781-i2c.c                     |  320 ++-
+ sound/soc/codecs/tas2783-sdw.c                     |  218 +-
+ sound/soc/codecs/tas2783.h                         |    4 +-
+ sound/soc/codecs/tas5086.c                         |    7 +
+ sound/soc/codecs/tas571x.c                         |    7 +
+ sound/soc/codecs/tas5720.c                         |    9 +
+ sound/soc/codecs/tas6424.c                         |    9 +
+ sound/soc/codecs/tfa9879.c                         |    9 +
+ sound/soc/codecs/tlv320adc3xxx.c                   |   14 +
+ sound/soc/codecs/tlv320adcx140.c                   |   12 +
+ sound/soc/codecs/tlv320aic23.c                     |    9 +
+ sound/soc/codecs/tlv320aic26.c                     |    8 +
+ sound/soc/codecs/tlv320aic31xx.c                   |   41 +-
+ sound/soc/codecs/tlv320aic32x4-clk.c               |    2 +-
+ sound/soc/codecs/tlv320aic32x4.c                   |  102 +-
+ sound/soc/codecs/tlv320aic3x.c                     |   42 +-
+ sound/soc/codecs/tlv320dac33.c                     |   10 +-
+ sound/soc/codecs/tscs454.c                         |   15 +
+ sound/soc/codecs/twl4030.c                         |   80 +-
+ sound/soc/codecs/uda1334.c                         |    6 +
+ sound/soc/codecs/uda1342.c                         |    7 +
+ sound/soc/codecs/uda1380.c                         |   11 +
+ sound/soc/codecs/wcd-mbhc-v2.c                     |    8 +-
+ sound/soc/codecs/wcd9335.c                         |   25 +-
+ sound/soc/codecs/wcd934x.c                         |    2 +-
+ sound/soc/codecs/wcd9378-sdca.c                    | 1080 +++++++++
+ sound/soc/codecs/wcd9378-sdca.h                    |   20 +
+ sound/soc/codecs/wcd9378-sdw.c                     |   51 +
+ sound/soc/codecs/wm2200.c                          |   10 +
+ sound/soc/codecs/wm5100.c                          |   10 +
+ sound/soc/codecs/wm8350.c                          |   24 +
+ sound/soc/codecs/wm8400.c                          |    9 +
+ sound/soc/codecs/wm8510.c                          |   12 +
+ sound/soc/codecs/wm8523.c                          |   13 +
+ sound/soc/codecs/wm8524.c                          |    6 +
+ sound/soc/codecs/wm8580.c                          |   20 +
+ sound/soc/codecs/wm8711.c                          |   13 +
+ sound/soc/codecs/wm8728.c                          |    9 +
+ sound/soc/codecs/wm8731.c                          |   13 +
+ sound/soc/codecs/wm8737.c                          |   11 +
+ sound/soc/codecs/wm8741.c                          |   13 +
+ sound/soc/codecs/wm8750.c                          |   13 +
+ sound/soc/codecs/wm8753.c                          |   20 +
+ sound/soc/codecs/wm8770.c                          |   11 +
+ sound/soc/codecs/wm8776.c                          |   13 +
+ sound/soc/codecs/wm8804.c                          |   15 +-
+ sound/soc/codecs/wm8900.c                          |   18 +
+ sound/soc/codecs/wm8903.c                          |   18 +
+ sound/soc/codecs/wm8904.c                          |   20 +-
+ sound/soc/codecs/wm8940.c                          |   13 +
+ sound/soc/codecs/wm8955.c                          |   42 +-
+ sound/soc/codecs/wm8958-dsp2.c                     |    8 +-
+ sound/soc/codecs/wm8960.c                          |   13 +
+ sound/soc/codecs/wm8961.c                          |   18 +
+ sound/soc/codecs/wm8962.c                          |   74 +-
+ sound/soc/codecs/wm8971.c                          |   13 +
+ sound/soc/codecs/wm8974.c                          |   17 +
+ sound/soc/codecs/wm8978.c                          |   12 +
+ sound/soc/codecs/wm8983.c                          |   12 +
+ sound/soc/codecs/wm8985.c                          |   53 +-
+ sound/soc/codecs/wm8988.c                          |   13 +
+ sound/soc/codecs/wm8990.c                          |    9 +
+ sound/soc/codecs/wm8991.c                          |    9 +
+ sound/soc/codecs/wm8993.c                          |   18 +
+ sound/soc/codecs/wm8994.c                          |   23 +
+ sound/soc/codecs/wm8995.c                          |   61 +-
+ sound/soc/codecs/wm8996.c                          |   14 +-
+ sound/soc/codecs/wm9081.c                          |   18 +
+ sound/soc/codecs/wm9713.c                          |   13 +
+ sound/soc/codecs/wm_hubs.c                         |    2 +-
+ sound/soc/codecs/zl38060.c                         |    6 +
+ sound/soc/dwc/dwc-i2s.c                            |    9 +
+ sound/soc/fsl/Kconfig                              |    2 +-
+ sound/soc/fsl/fsl_asrc.c                           |  110 +-
+ sound/soc/fsl/fsl_asrc_common.h                    |    7 +-
+ sound/soc/fsl/fsl_asrc_dma.c                       |   38 +-
+ sound/soc/fsl/fsl_asrc_m2m.c                       |   17 +-
+ sound/soc/fsl/fsl_easrc.c                          |  125 +-
+ sound/soc/fsl/fsl_mqs.c                            |   14 +
+ sound/soc/fsl/fsl_xcvr.c                           |   28 +-
+ sound/soc/generic/audio-graph-card.c               |    1 +
+ sound/soc/generic/audio-graph-card2.c              |    4 +-
+ sound/soc/generic/simple-card-utils.c              |   23 +-
+ sound/soc/generic/simple-card.c                    |   13 +-
+ sound/soc/hisilicon/hi6210-i2s.c                   |    7 +
+ sound/soc/img/img-i2s-in.c                         |   12 +-
+ sound/soc/img/img-i2s-out.c                        |   13 +-
+ sound/soc/img/img-parallel-out.c                   |    8 +-
+ sound/soc/intel/atom/sst-mfld-dsp.h                |    2 +-
+ sound/soc/intel/atom/sst/sst.h                     |    2 +-
+ sound/soc/intel/atom/sst/sst_acpi.c                |   14 +-
+ sound/soc/intel/avs/boards/hdaudio.c               |    8 +-
+ sound/soc/intel/avs/pcm.c                          |   11 +-
+ sound/soc/intel/avs/topology.c                     |   25 -
+ sound/soc/intel/avs/topology.h                     |    3 -
+ sound/soc/intel/boards/Kconfig                     |    1 +
+ sound/soc/intel/boards/sof_cirrus_common.c         |    2 +-
+ sound/soc/intel/common/soc-acpi-intel-arl-match.c  |   71 +
+ sound/soc/intel/common/soc-acpi-intel-lnl-match.c  |   71 +
+ sound/soc/intel/common/soc-acpi-intel-mtl-match.c  |   71 +
+ sound/soc/intel/common/soc-acpi-intel-ptl-match.c  |   85 +
+ sound/soc/intel/common/sof-function-topology-lib.c |  289 ++-
+ sound/soc/intel/common/sof-function-topology-lib.h |    3 +
+ sound/soc/jz4740/jz4740-i2s.c                      |    7 +
+ sound/soc/kirkwood/kirkwood-i2s.c                  |    7 +
+ sound/soc/loongson/loongson_i2s.c                  |    6 +
+ sound/soc/mediatek/Kconfig                         |   27 +-
+ sound/soc/mediatek/Makefile                        |    1 +
+ sound/soc/mediatek/an7581/Makefile                 |    9 +
+ sound/soc/mediatek/an7581/an7581-afe-common.h      |   49 +
+ sound/soc/mediatek/an7581/an7581-afe-pcm.c         |  515 +++++
+ sound/soc/mediatek/an7581/an7581-dai-etdm.c        |  433 ++++
+ sound/soc/mediatek/an7581/an7581-reg.h             |  114 +
+ sound/soc/mediatek/an7581/an7581-wm8960.c          |  161 ++
+ sound/soc/mediatek/common/mtk-afe-fe-dai.c         |   14 +-
+ sound/soc/mediatek/common/mtk-base-afe.h           |    2 +
+ sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c  |   61 +-
+ sound/soc/mediatek/mt2701/mt2701-afe-pcm.c         |    4 +-
+ sound/soc/mediatek/mt2701/mt2701-cs42448.c         |    4 +-
+ sound/soc/mediatek/mt2701/mt2701-wm8960.c          |    4 +-
+ sound/soc/mediatek/mt6797/mt6797-afe-clk.c         |   23 +-
+ sound/soc/mediatek/mt6797/mt6797-afe-pcm.c         |    8 +-
+ sound/soc/mediatek/mt7986/mt7986-afe-pcm.c         |    8 +-
+ sound/soc/mediatek/mt7986/mt7986-dai-etdm.c        |    2 +-
+ sound/soc/mediatek/mt7986/mt7986-wm8960.c          |    4 +-
+ sound/soc/mediatek/mt8173/mt8173-afe-pcm.c         |   26 +-
+ sound/soc/mediatek/mt8183/mt8183-afe-clk.c         |   31 +-
+ sound/soc/mediatek/mt8183/mt8183-afe-pcm.c         |   21 +-
+ sound/soc/mediatek/mt8186/mt8186-afe-clk.c         |   95 +-
+ sound/soc/mediatek/mt8186/mt8186-afe-gpio.c        |    6 +-
+ sound/soc/mediatek/mt8186/mt8186-afe-pcm.c         |   23 +-
+ sound/soc/mediatek/mt8188/mt8188-afe-clk.c         |  115 +-
+ sound/soc/mediatek/mt8188/mt8188-afe-pcm.c         |   36 +-
+ sound/soc/mediatek/mt8188/mt8188-audsys-clk.c      |   12 +-
+ sound/soc/mediatek/mt8188/mt8188-dai-adda.c        |    5 +-
+ sound/soc/mediatek/mt8188/mt8188-dai-dmic.c        |   10 +-
+ sound/soc/mediatek/mt8189/mt8189-afe-clk.c         |  134 +-
+ sound/soc/mediatek/mt8189/mt8189-afe-pcm.c         |   61 +-
+ sound/soc/mediatek/mt8189/mt8189-dai-i2s.c         |   18 +-
+ sound/soc/mediatek/mt8189/mt8189-dai-tdm.c         |   19 +-
+ sound/soc/mediatek/mt8192/mt8192-afe-clk.c         |  163 +-
+ sound/soc/mediatek/mt8192/mt8192-afe-gpio.c        |   94 +-
+ sound/soc/mediatek/mt8192/mt8192-afe-pcm.c         |   23 +-
+ sound/soc/mediatek/mt8192/mt8192-dai-adda.c        |   44 +-
+ sound/soc/mediatek/mt8192/mt8192-dai-i2s.c         |   20 +-
+ sound/soc/mediatek/mt8192/mt8192-dai-tdm.c         |   18 +-
+ .../mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c  |   37 +-
+ sound/soc/mediatek/mt8195/mt8195-afe-pcm.c         |    2 +-
+ sound/soc/mediatek/mt8196/mt8196-afe-pcm.c         |   14 +-
+ sound/soc/mxs/mxs-saif.c                           |   10 +
+ sound/soc/pxa/mmp-sspa.c                           |    6 +
+ sound/soc/pxa/pxa-ssp.c                            |   11 +
+ sound/soc/pxa/pxa2xx-i2s.c                         |    6 +
+ sound/soc/pxa/pxa2xx-pcm-lib.c                     |    9 +-
+ sound/soc/qcom/common.c                            |   61 +-
+ sound/soc/qcom/common.h                            |    5 +-
+ sound/soc/qcom/qdsp6/q6apm-lpass-dais.c            |    2 +
+ sound/soc/qcom/qdsp6/q6asm-dai.c                   |    2 +-
+ sound/soc/qcom/qdsp6/q6dsp-lpass-ports.c           |   54 +
+ sound/soc/qcom/sc8280xp.c                          |  132 +-
+ sound/soc/qcom/sdm845.c                            |   49 +-
+ sound/soc/renesas/rz-ssi.c                         |    9 +
+ sound/soc/renesas/siu_dai.c                        |    6 +
+ sound/soc/renesas/ssi.c                            |   13 +
+ sound/soc/rockchip/rk3399_gru_sound.c              |    4 +-
+ sound/soc/rockchip/rockchip_i2s.c                  |   25 +-
+ sound/soc/rockchip/rockchip_i2s_tdm.c              |   27 +-
+ sound/soc/rockchip/rockchip_pdm.c                  |   14 +-
+ sound/soc/rockchip/rockchip_sai.c                  |   31 +-
+ sound/soc/samsung/aries_wm8994.c                   |   14 +-
+ sound/soc/samsung/pcm.c                            |   19 +-
+ sound/soc/samsung/spdif.c                          |    8 +-
+ sound/soc/samsung/tm2_wm5110.c                     |   12 +-
+ sound/soc/sdca/Kconfig                             |    6 +-
+ sound/soc/sdca/sdca_asoc.c                         |    4 +-
+ sound/soc/sdca/sdca_class.c                        |  148 +-
+ sound/soc/sdca/sdca_class.h                        |   35 -
+ sound/soc/sdca/sdca_class_function.c               |   50 +-
+ sound/soc/sdca/sdca_device.c                       |    4 +
+ sound/soc/sdca/sdca_fdl.c                          |   18 +-
+ sound/soc/sdca/sdca_functions.c                    |   15 +-
+ sound/soc/sdw_utils/Makefile                       |    4 +-
+ sound/soc/sdw_utils/soc_sdw_rt711.c                |    1 +
+ sound/soc/sdw_utils/soc_sdw_rt_mf_sdca.c           |    6 +
+ sound/soc/sdw_utils/soc_sdw_rt_sdca_jack_common.c  |    8 +
+ sound/soc/sdw_utils/soc_sdw_senary_amp.c           |   80 +
+ sound/soc/sdw_utils/soc_sdw_senary_dmic.c          |   49 +
+ sound/soc/sdw_utils/soc_sdw_senary_sdca.c          |   63 +
+ .../sdw_utils/soc_sdw_senary_sdca_jack_common.c    |  197 ++
+ sound/soc/sdw_utils/soc_sdw_utils.c                |  384 ++++
+ sound/soc/soc-component.c                          |    8 +-
+ sound/soc/soc-compress.c                           |    5 +-
+ sound/soc/soc-core.c                               |  175 +-
+ sound/soc/soc-dai.c                                |  418 +++-
+ sound/soc/soc-dapm.c                               |    9 +-
+ sound/soc/soc-generic-dmaengine-pcm.c              |   10 +-
+ sound/soc/soc-internal.h                           |   39 +
+ sound/soc/soc-ops.c                                |    4 +-
+ sound/soc/soc-pcm.c                                |  144 +-
+ sound/soc/sof/amd/Kconfig                          |    1 +
+ sound/soc/sof/amd/acp-common.c                     |   17 +
+ sound/soc/sof/amd/acp-dsp-offset.h                 |   17 +
+ sound/soc/sof/amd/acp.c                            |  391 +++-
+ sound/soc/sof/amd/acp.h                            |   16 +
+ sound/soc/sof/amd/acp7x.h                          |   37 +
+ sound/soc/sof/amd/pci-acp7x.c                      |    2 +
+ sound/soc/sof/imx/imx-common.c                     |    1 +
+ sound/soc/sof/imx/imx8.c                           |   26 +-
+ sound/soc/sof/intel/apl.c                          |    1 +
+ sound/soc/sof/intel/cnl.c                          |    1 +
+ sound/soc/sof/intel/hda-dai.c                      |   26 +-
+ sound/soc/sof/intel/icl.c                          |    1 +
+ sound/soc/sof/intel/mtl.c                          |    1 +
+ sound/soc/sof/intel/skl.c                          |    1 +
+ sound/soc/sof/intel/tgl.c                          |    1 +
+ sound/soc/sof/ipc4-priv.h                          |   10 +-
+ sound/soc/sof/ipc4-topology.c                      |   65 +-
+ sound/soc/sof/mediatek/mt8186/mt8186.c             |    2 +-
+ sound/soc/sof/mediatek/mt8195/mt8195.c             |    2 +-
+ sound/soc/sof/topology.c                           |    3 +-
+ sound/soc/sprd/sprd-pcm-compress.c                 |    7 +-
+ sound/soc/sprd/sprd-pcm-dma.c                      |   10 +-
+ sound/soc/sti/uniperif_player.c                    |   29 +-
+ sound/soc/sti/uniperif_reader.c                    |   27 +-
+ sound/soc/stm/stm32_i2s.c                          |   12 +
+ sound/soc/stm/stm32_sai_sub.c                      |   15 +
+ sound/soc/stm/stm32_spdifrx.c                      |    2 +-
+ sound/soc/sunxi/sun4i-i2s.c                        |   13 +
+ sound/soc/sunxi/sun8i-codec.c                      |   13 +
+ sound/soc/tegra/tegra20_i2s.c                      |   10 +
+ sound/soc/tegra/tegra210_adx.c                     |   12 +-
+ sound/soc/tegra/tegra210_adx.h                     |    2 +-
+ sound/soc/tegra/tegra210_i2s.c                     |   13 +
+ sound/soc/tegra/tegra30_i2s.c                      |   10 +
+ sound/soc/ti/davinci-i2s.c                         |   16 +-
+ sound/soc/ti/davinci-mcasp.c                       |   17 +-
+ sound/soc/ti/omap-mcbsp.c                          |   14 +-
+ sound/soc/uniphier/aio-cpu.c                       |    9 +
+ sound/soc/uniphier/aio-dma.c                       |    5 +-
+ sound/soc/uniphier/aio.h                           |    2 +-
+ sound/soc/ux500/Kconfig                            |   23 +-
+ sound/soc/ux500/Makefile                           |    3 -
+ sound/soc/ux500/mop500.c                           |  167 --
+ sound/soc/ux500/mop500_ab8500.c                    |  437 ----
+ sound/soc/ux500/mop500_ab8500.h                    |   17 -
+ sound/soc/ux500/ux500_msp_dai.c                    |   33 +-
+ sound/soc/ux500/ux500_msp_i2s.c                    |    2 +-
+ sound/soc/xilinx/xlnx_formatter_pcm.c              |   79 +-
+ sound/soc/xtensa/xtfpga-i2s.c                      |   12 +-
+ 501 files changed, 20889 insertions(+), 3581 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/sound/airoha,an7581-afe.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/airoha,an7581-wm8960.yaml
+ delete mode 100644 Documentation/devicetree/bindings/sound/davinci-evm-audio.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/esstech,es9039q2m.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/nuvoton,nau8360.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/qcom,wcd9378-sdw.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/stericsson,ux500-msp-i2s.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/ti,da830-evm-audio.yaml
+ delete mode 100644 Documentation/devicetree/bindings/sound/ux500-mop500.txt
+ delete mode 100644 Documentation/devicetree/bindings/sound/ux500-msp.txt
+ create mode 100644 include/sound/sdca_class.h
+ create mode 100644 sound/soc/codecs/es9039q2m.c
+ create mode 100644 sound/soc/codecs/nau8360-dsp.c
+ create mode 100644 sound/soc/codecs/nau8360-dsp.h
+ create mode 100644 sound/soc/codecs/nau8360.c
+ create mode 100644 sound/soc/codecs/nau8360.h
+ create mode 100644 sound/soc/codecs/sn624x-sdca-sdw.c
+ create mode 100644 sound/soc/codecs/sn624x-sdca.h
+ create mode 100644 sound/soc/codecs/wcd9378-sdca.c
+ create mode 100644 sound/soc/codecs/wcd9378-sdca.h
+ create mode 100644 sound/soc/codecs/wcd9378-sdw.c
+ create mode 100644 sound/soc/mediatek/an7581/Makefile
+ create mode 100644 sound/soc/mediatek/an7581/an7581-afe-common.h
+ create mode 100644 sound/soc/mediatek/an7581/an7581-afe-pcm.c
+ create mode 100644 sound/soc/mediatek/an7581/an7581-dai-etdm.c
+ create mode 100644 sound/soc/mediatek/an7581/an7581-reg.h
+ create mode 100644 sound/soc/mediatek/an7581/an7581-wm8960.c
+ delete mode 100644 sound/soc/sdca/sdca_class.h
+ create mode 100644 sound/soc/sdw_utils/soc_sdw_senary_amp.c
+ create mode 100644 sound/soc/sdw_utils/soc_sdw_senary_dmic.c
+ create mode 100644 sound/soc/sdw_utils/soc_sdw_senary_sdca.c
+ create mode 100644 sound/soc/sdw_utils/soc_sdw_senary_sdca_jack_common.c
+ create mode 100644 sound/soc/soc-internal.h
+ create mode 100644 sound/soc/sof/amd/acp7x.h
+ delete mode 100644 sound/soc/ux500/mop500.c
+ delete mode 100644 sound/soc/ux500/mop500_ab8500.c
+ delete mode 100644 sound/soc/ux500/mop500_ab8500.h
+Merging modules/modules-next (d6bfb4a05affc module: Bring includes in linux/kmod.h up to date)
+$ git merge -m Merge branch 'modules-next' of https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git modules/modules-next
+Auto-merging fs/coredump.c
+Auto-merging fs/nfsd/nfs4layouts.c
+Auto-merging fs/nfsd/nfs4recover.c
+Auto-merging kernel/reboot.c
+Auto-merging kernel/time/jiffies.c
+Merge made by the 'ort' strategy.
+ arch/x86/kernel/cpu/mce/dev-mcelog.c |  2 +-
+ drivers/block/drbd/drbd_nl.c         |  1 +
+ drivers/greybus/svc_watchdog.c       |  1 +
+ drivers/macintosh/windfarm_core.c    |  1 +
+ drivers/pnp/pnpbios/core.c           |  2 +-
+ drivers/video/fbdev/uvesafb.c        |  1 +
+ fs/coredump.c                        |  2 +-
+ fs/nfs/cache_lib.c                   |  2 +-
+ fs/nfsd/nfs4layouts.c                |  2 +-
+ fs/nfsd/nfs4recover.c                |  1 +
+ fs/ocfs2/stackglue.c                 |  1 +
+ include/linux/kmod.h                 | 12 ++------
+ include/linux/module_symbol.h        |  7 +++--
+ kernel/cgroup/cgroup-v1.c            |  2 +-
+ kernel/module/kallsyms.c             | 59 +++++++++++++++++++-----------------
+ kernel/module/kmod.c                 |  3 +-
+ kernel/power/process.c               |  2 +-
+ kernel/reboot.c                      |  2 +-
+ kernel/time/jiffies.c                |  1 +
+ kernel/umh.c                         |  2 +-
+ lib/kobject_uevent.c                 |  2 +-
+ net/bridge/br_stp_if.c               |  2 +-
+ scripts/faddr2line                   |  2 +-
+ scripts/mod/modpost.h                |  2 +-
+ security/keys/request_key.c          |  2 +-
+ security/tomoyo/common.h             |  2 +-
+ tools/perf/util/symbol.h             |  4 +--
+ 27 files changed, 63 insertions(+), 59 deletions(-)
+Merging input/next (daae2ab46e0cb Input: gscps2 - drop busy-wait and manual interrupt pump on transmit)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/input/touchscreen/sis,9200-ts.yaml    |  61 +++++
+ .../bindings/input/touchscreen/sis_i2c.txt         |  31 ---
+ MAINTAINERS                                        |   2 +-
+ drivers/input/keyboard/snvs_pwrkey.c               |   9 +-
+ drivers/input/keyboard/st-keyscan.c                |  13 +-
+ drivers/input/matrix-keymap.c                      |   7 +
+ drivers/input/serio/gscps2.c                       | 293 +++++++++++----------
+ 7 files changed, 225 insertions(+), 191 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/input/touchscreen/sis,9200-ts.yaml
+ delete mode 100644 Documentation/devicetree/bindings/input/touchscreen/sis_i2c.txt
+Merging block/for-next (0183ac11c13ab Merge branch 'block-7.3' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git block/for-next
+Auto-merging drivers/nvme/host/pci.c
+Auto-merging io_uring/io-wq.c
+Merge made by the 'ort' strategy.
+ arch/m68k/mac/config.c                 |  31 +-
+ block/bdev.c                           |   2 +-
+ block/blk-cgroup.c                     |   5 +-
+ block/blk-core.c                       |   7 +-
+ block/blk-mq.c                         |  63 ++-
+ block/blk-settings.c                   |  18 +-
+ block/blk-zoned.c                      | 872 ++++++++++++++++++++-------------
+ block/blk.h                            |   6 +
+ block/fops.c                           |  45 +-
+ block/mq-deadline.c                    |   2 +-
+ drivers/block/drbd/drbd_nl_gen.c       |   4 -
+ drivers/block/rnull/configfs.rs        |  44 +-
+ drivers/block/rnull/rnull.rs           |  15 +-
+ drivers/block/swim.c                   | 457 +++++++++--------
+ drivers/block/swim3.c                  |   8 +-
+ drivers/block/swim_asm.S               | 340 +++++++------
+ drivers/block/virtio_blk.c             |   1 +
+ drivers/nvme/host/core.c               |  63 ++-
+ drivers/nvme/host/fc.c                 |   2 +-
+ drivers/nvme/host/ioctl.c              |  19 +-
+ drivers/nvme/host/multipath.c          |  19 +-
+ drivers/nvme/host/nvme.h               |   9 +-
+ drivers/nvme/host/pci.c                |   2 +
+ drivers/nvme/host/sysfs.c              |   4 +-
+ drivers/nvme/host/tcp.c                |  39 +-
+ drivers/nvme/target/configfs.c         |   9 +-
+ drivers/nvme/target/core.c             |  35 +-
+ drivers/nvme/target/fabrics-cmd-auth.c |   2 +-
+ drivers/nvme/target/nvmet.h            |   2 +
+ drivers/nvme/target/pci-epf.c          |   3 +
+ include/linux/blkdev.h                 |  11 +-
+ include/linux/io_uring.h               |  32 ++
+ include/linux/io_uring/cmd.h           |  34 +-
+ include/linux/io_uring_types.h         |   2 +
+ io_uring/bpf_filter.c                  |   1 +
+ io_uring/cancel.c                      |  89 +++-
+ io_uring/cancel.h                      |  13 +-
+ io_uring/cmd_net.c                     |  19 +-
+ io_uring/fdinfo.c                      |  11 +-
+ io_uring/io-wq.c                       |  12 +-
+ io_uring/io_uring.c                    | 241 +++++++--
+ io_uring/io_uring.h                    |  31 --
+ io_uring/loop.c                        |   5 +
+ io_uring/notif.c                       |   2 +
+ io_uring/refs.h                        |  27 +
+ io_uring/rsrc.c                        |  42 +-
+ io_uring/rw.c                          |   4 +
+ io_uring/timeout.c                     |  18 +
+ io_uring/timeout.h                     |   2 +
+ io_uring/tw.c                          |   3 +
+ io_uring/uring_cmd.c                   |  23 +-
+ io_uring/uring_cmd.h                   |   2 +-
+ io_uring/zcrx.c                        |   9 +-
+ rust/kernel/block/mq/gen_disk.rs       |  72 +--
+ rust/kernel/block/mq/operations.rs     |  16 +-
+ rust/kernel/block/mq/request.rs        |  16 +-
+ rust/kernel/block/mq/tag_set.rs        |  28 +-
+ tools/testing/selftests/ublk/kublk.c   |   2 +-
+ 58 files changed, 1795 insertions(+), 1100 deletions(-)
+$ git am -3 ../patches/0001-Revert-block-remove-bio_last_bvec_all.patch
+Applying: Revert "block: remove bio_last_bvec_all"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	Documentation/block/biovecs.rst
+M	include/linux/bio.h
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master ac7093d359896] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
+ Date: Wed Sep 30 13:14:56 2026 +0100
+$ git am -3 ../patches/0001-Fixup-for-blk-zoned-mismerge.patch
+Applying: Fixup for blk-zoned mismerge
+Using index info to reconstruct a base tree...
+M	block/blk-zoned.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging device-mapper/for-next (bec2fa7bdf06d dm-verity: add DM_VERITY_VERIFY_ROOTHASH_SIG_FORCE)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git device-mapper/for-next
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/device-mapper/verity.rst |  5 +++
+ drivers/md/Kconfig                                 | 14 ++++++++
+ drivers/md/dm-cache-metadata.c                     |  2 +-
+ drivers/md/dm-clone-metadata.c                     |  3 --
+ drivers/md/dm-crypt.c                              | 23 +++++++------
+ drivers/md/dm-delay.c                              |  3 +-
+ drivers/md/dm-exception-store.h                    |  2 +-
+ drivers/md/dm-mpath.c                              |  5 +--
+ drivers/md/dm-pcache/backing_dev.c                 |  2 +-
+ drivers/md/dm-raid.c                               |  2 +-
+ drivers/md/dm-raid1.c                              |  2 +-
+ drivers/md/dm-unstripe.c                           |  2 +-
+ drivers/md/dm-vdo/block-map.c                      |  2 +-
+ drivers/md/dm-vdo/dm-vdo-target.c                  |  2 +-
+ drivers/md/dm-vdo/indexer/chapter-index.c          | 15 ++++++---
+ drivers/md/dm-vdo/indexer/delta-index.c            | 17 ++++++++--
+ drivers/md/dm-vdo/indexer/volume.c                 |  6 ----
+ drivers/md/dm-verity-verify-sig.c                  |  4 +--
+ drivers/md/dm-writecache.c                         | 39 ++++++++++------------
+ drivers/md/dm-zone.c                               |  6 ++--
+ drivers/md/persistent-data/dm-bitset.c             |  5 +--
+ drivers/md/persistent-data/dm-btree-remove.c       |  2 +-
+ 22 files changed, 92 insertions(+), 71 deletions(-)
+Merging libata/for-next (cfce1dc635041 ata: libata: Fix scsi_done() documentation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux libata/for-next
+Auto-merging drivers/ata/ahci.c
+Auto-merging drivers/ata/libahci.c
+Auto-merging drivers/ata/libahci_platform.c
+Auto-merging drivers/ata/libata-core.c
+Auto-merging drivers/ata/libata-scsi.c
+Auto-merging include/linux/libata.h
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/libata.rst     |   4 +-
+ drivers/ata/Kconfig                     |  17 +-
+ drivers/ata/Makefile                    |   2 +-
+ drivers/ata/ahci.c                      |  19 +-
+ drivers/ata/ahci_brcm.c                 |   1 +
+ drivers/ata/ahci_ceva.c                 |   1 +
+ drivers/ata/ahci_da850.c                |   6 +-
+ drivers/ata/ahci_qoriq.c                |   1 +
+ drivers/ata/ahci_st.c                   |  78 +--
+ drivers/ata/ata_generic.c               |  16 +
+ drivers/ata/libahci.c                   |  29 +-
+ drivers/ata/libahci_platform.c          |   4 +
+ drivers/ata/libata-core.c               |  81 ++-
+ drivers/ata/libata-scsi.c               |   7 +-
+ drivers/ata/libata-sff.c                |  13 +-
+ drivers/ata/pata_arasan_cf.c            |   6 +-
+ drivers/ata/pata_cswarp.c               | 183 +++++++
+ drivers/ata/pata_parport/pata_parport.c |   9 +-
+ drivers/ata/sata_dwc_460ex.c            |   5 +-
+ drivers/ata/sata_fsl.c                  |  26 +-
+ drivers/ata/sata_inic162x.c             | 903 --------------------------------
+ drivers/ata/sata_qstor.c                |   8 +-
+ include/linux/libata.h                  |   1 +
+ 23 files changed, 416 insertions(+), 1004 deletions(-)
+ create mode 100644 drivers/ata/pata_cswarp.c
+ delete mode 100644 drivers/ata/sata_inic162x.c
+Merging pcmcia/pcmcia-next (b3c26ea81ccc5 pcmcia: remove obsolete host controller drivers)
+$ git merge -m Merge branch 'pcmcia-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git pcmcia/pcmcia-next
+Already up to date.
+Merging mmc/next (c2063613bceaa mmc: Merge branch fixes into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc/next
+Auto-merging CREDITS
+Auto-merging Documentation/devicetree/bindings/mmc/qcom,sdhci-msm.yaml
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ CREDITS                                            |    5 +
+ .../bindings/mmc/amlogic,meson-gx-mmc.yaml         |   29 +-
+ .../devicetree/bindings/mmc/cdns,sd6hc.yaml        |  105 +
+ .../bindings/mmc/mmc-controller-common.yaml        |   10 +
+ .../devicetree/bindings/mmc/qcom,sdhci-msm.yaml    |    2 +
+ Documentation/driver-api/mmc/mmc-async-req.rst     |   64 +-
+ Documentation/driver-api/mmc/mmc-tools.rst         |    1 +
+ MAINTAINERS                                        |   12 +-
+ drivers/memstick/core/ms_block.c                   |    2 +-
+ drivers/memstick/host/r592.c                       |    2 +-
+ drivers/mmc/core/block.c                           |    2 +-
+ drivers/mmc/core/core.c                            |    8 +-
+ drivers/mmc/core/core.h                            |   10 +-
+ drivers/mmc/core/crypto.c                          |    2 +-
+ drivers/mmc/core/host.c                            |   39 +-
+ drivers/mmc/core/mmc.c                             |    2 +-
+ drivers/mmc/core/mmc_ops.c                         |   22 +-
+ drivers/mmc/core/sdio_cis.c                        |    5 +-
+ drivers/mmc/host/Kconfig                           |   31 -
+ drivers/mmc/host/Makefile                          |    2 +-
+ drivers/mmc/host/atmel-mci.c                       |    5 +-
+ drivers/mmc/host/bcm2835.c                         |    2 +-
+ drivers/mmc/host/cavium-thunderx.c                 |    5 +-
+ drivers/mmc/host/davinci_mmc.c                     |    6 +-
+ drivers/mmc/host/dw_mmc-rockchip.c                 |    2 +-
+ drivers/mmc/host/dw_mmc.c                          |  316 ++-
+ drivers/mmc/host/dw_mmc.h                          |   18 +-
+ drivers/mmc/host/meson-gx-mmc.c                    |   31 +-
+ drivers/mmc/host/mvsdio.c                          |    2 +-
+ drivers/mmc/host/rtsx_usb_sdmmc.c                  |    5 +
+ .../host/{sdhci-cadence.c => sdhci-cadence-core.c} |  303 ++-
+ drivers/mmc/host/sdhci-cadence-phy-v6.c            |  975 ++++++++
+ drivers/mmc/host/sdhci-cadence.h                   |  118 +
+ drivers/mmc/host/sdhci-msm.c                       |   34 +-
+ drivers/mmc/host/sdhci-of-arasan.c                 |    4 +-
+ drivers/mmc/host/sdhci-of-dwcmshc.c                |   13 +
+ drivers/mmc/host/sdhci-of-esdhc.c                  |    2 +-
+ drivers/mmc/host/sdhci-omap.c                      |    2 +-
+ drivers/mmc/host/sdhci-pci-core.c                  |    8 +-
+ drivers/mmc/host/sdhci-pxav3.c                     |   13 +-
+ drivers/mmc/host/sdhci.h                           |    2 +-
+ drivers/mmc/host/sunplus-mmc.c                     |    5 +
+ drivers/mmc/host/ushc.c                            |    5 +-
+ drivers/mmc/host/vub300.c                          | 2493 --------------------
+ drivers/staging/greybus/sdio.c                     |   11 +-
+ include/linux/mmc/host.h                           |    6 +-
+ 46 files changed, 1823 insertions(+), 2918 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/mmc/cdns,sd6hc.yaml
+ rename drivers/mmc/host/{sdhci-cadence.c => sdhci-cadence-core.c} (67%)
+ create mode 100644 drivers/mmc/host/sdhci-cadence-phy-v6.c
+ create mode 100644 drivers/mmc/host/sdhci-cadence.h
+ delete mode 100644 drivers/mmc/host/vub300.c
+Merging mfd/for-mfd-next (319633b06ff2b dt-bindings: mfd: qcom,tcsr: Add compatible for MSM8952)
+$ git merge -m Merge branch 'for-mfd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd/for-mfd-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/gpio/Kconfig
+Auto-merging drivers/gpio/Makefile
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/input/cpcap-pwrbutton.txt  |  20 -
+ .../bindings/input/motorola,cpcap-pwrbutton.yaml   |  32 ++
+ .../leds/backlight/ti,lm3533-backlight.yaml        |  69 +++
+ .../devicetree/bindings/leds/ti,lm3533-leds.yaml   |  67 +++
+ .../devicetree/bindings/leds/ti,lm3533.yaml        | 169 +++++++
+ .../devicetree/bindings/mfd/motorola,cpcap.yaml    | 408 +++++++++++++++
+ .../devicetree/bindings/mfd/motorola-cpcap.txt     |  78 ---
+ .../devicetree/bindings/mfd/qcom,spmi-pmic.yaml    |   1 +
+ .../devicetree/bindings/mfd/qcom,tcsr.yaml         |   1 +
+ .../devicetree/bindings/mfd/rohm,bd71815-pmic.yaml |   9 +-
+ .../devicetree/bindings/mfd/rohm,bd71828-pmic.yaml |   9 +-
+ .../devicetree/bindings/mfd/rohm,bd72720-pmic.yaml |  29 +-
+ .../devicetree/bindings/mfd/rohm,bd73800-pmic.yaml | 218 ++++++++
+ .../devicetree/bindings/mfd/rohm,pmic-pins.yaml    |  72 +++
+ .../devicetree/bindings/mfd/stericsson,ab8500.yaml | 150 +++++-
+ .../bindings/mfd/ti,keystone-devctrl.yaml          |  90 ++++
+ .../devicetree/bindings/mfd/ti,tps61050.yaml       |  92 ++++
+ .../devicetree/bindings/mfd/ti,tps65910.yaml       |   8 +
+ .../devicetree/bindings/mfd/ti,twl6040.yaml        | 156 ++++++
+ .../bindings/mfd/ti-keystone-devctrl.txt           |  19 -
+ Documentation/devicetree/bindings/mfd/tps6105x.txt |  62 ---
+ Documentation/devicetree/bindings/mfd/twl6040.txt  |  67 ---
+ .../bindings/regulator/rohm,bd73800-regulator.yaml |  99 ++++
+ MAINTAINERS                                        |   2 +
+ drivers/clk/clk-bd718x7.c                          |   8 +
+ drivers/gpio/Kconfig                               |  12 +
+ drivers/gpio/Makefile                              |   1 +
+ drivers/gpio/gpio-bd73800.c                        | 209 ++++++++
+ drivers/iio/light/lm3533-als.c                     | 185 +++----
+ drivers/leds/leds-lm3533.c                         | 121 ++---
+ drivers/mfd/Kconfig                                |  17 +-
+ drivers/mfd/ab8500-core.c                          |   2 +-
+ drivers/mfd/cs42l43-i2c.c                          |  10 +
+ drivers/mfd/cs42l43-sdw.c                          |   9 +
+ drivers/mfd/cs42l43.c                              |  30 +-
+ drivers/mfd/cs42l43.h                              |   1 +
+ drivers/mfd/da903x.c                               |  34 +-
+ drivers/mfd/da9062-core.c                          |  24 +-
+ drivers/mfd/da9150-core.c                          |   8 +-
+ drivers/mfd/intel-lpss.c                           |  24 +-
+ drivers/mfd/intel_quark_i2c_gpio.c                 |  52 +-
+ drivers/mfd/iqs62x.c                               |   8 +-
+ drivers/mfd/khadas-mcu.c                           | 104 +++-
+ drivers/mfd/lm3533-core.c                          | 386 ++++++--------
+ drivers/mfd/lm3533-ctrlbank.c                      |  33 +-
+ drivers/mfd/max77843.c                             |   8 +-
+ drivers/mfd/motorola-cpcap.c                       | 143 +++---
+ drivers/mfd/mt6360-core.c                          |   8 +-
+ drivers/mfd/rk8xx-core.c                           |   2 +-
+ drivers/mfd/rk8xx-i2c.c                            |   4 +-
+ drivers/mfd/rn5t618.c                              |   8 +-
+ drivers/mfd/rohm-bd71828.c                         | 147 +++++-
+ drivers/mfd/ti_am335x_tscadc.c                     |   8 +-
+ drivers/mfd/tps6586x.c                             |   8 +-
+ drivers/mfd/twl-core.c                             |  31 +-
+ drivers/mfd/wcd934x.c                              |   8 +-
+ drivers/mfd/wm831x-auxadc.c                        |   2 +-
+ drivers/regulator/Kconfig                          |   4 +-
+ drivers/regulator/bd71828-regulator.c              | 558 ++++++++++++++++++++-
+ drivers/rtc/rtc-bd70528.c                          |   8 +
+ drivers/thermal/khadas_mcu_fan.c                   | 108 +++-
+ drivers/video/backlight/lm3533_bl.c                | 188 +++----
+ include/linux/mfd/cs42l43-regs.h                   |   1 +
+ include/linux/mfd/da9150/core.h                    |   1 +
+ include/linux/mfd/khadas-mcu.h                     |  19 +-
+ include/linux/mfd/lm3533.h                         |  75 +--
+ include/linux/mfd/motorola-cpcap.h                 |   7 +
+ include/linux/mfd/rohm-bd73800.h                   | 306 +++++++++++
+ include/linux/mfd/rohm-generic.h                   |   1 +
+ 69 files changed, 3747 insertions(+), 1111 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/input/cpcap-pwrbutton.txt
+ create mode 100644 Documentation/devicetree/bindings/input/motorola,cpcap-pwrbutton.yaml
+ create mode 100644 Documentation/devicetree/bindings/leds/backlight/ti,lm3533-backlight.yaml
+ create mode 100644 Documentation/devicetree/bindings/leds/ti,lm3533-leds.yaml
+ create mode 100644 Documentation/devicetree/bindings/leds/ti,lm3533.yaml
+ create mode 100644 Documentation/devicetree/bindings/mfd/motorola,cpcap.yaml
+ delete mode 100644 Documentation/devicetree/bindings/mfd/motorola-cpcap.txt
+ create mode 100644 Documentation/devicetree/bindings/mfd/rohm,bd73800-pmic.yaml
+ create mode 100644 Documentation/devicetree/bindings/mfd/rohm,pmic-pins.yaml
+ create mode 100644 Documentation/devicetree/bindings/mfd/ti,keystone-devctrl.yaml
+ create mode 100644 Documentation/devicetree/bindings/mfd/ti,tps61050.yaml
+ create mode 100644 Documentation/devicetree/bindings/mfd/ti,twl6040.yaml
+ delete mode 100644 Documentation/devicetree/bindings/mfd/ti-keystone-devctrl.txt
+ delete mode 100644 Documentation/devicetree/bindings/mfd/tps6105x.txt
+ delete mode 100644 Documentation/devicetree/bindings/mfd/twl6040.txt
+ create mode 100644 Documentation/devicetree/bindings/regulator/rohm,bd73800-regulator.yaml
+ create mode 100644 drivers/gpio/gpio-bd73800.c
+ create mode 100644 include/linux/mfd/rohm-bd73800.h
+Merging backlight/for-backlight-next (5e1631df673f3 backlight: ili9320/ktd253: Fix typos in comments)
+$ git merge -m Merge branch 'for-backlight-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight/for-backlight-next
+Merge made by the 'ort' strategy.
+ drivers/video/backlight/ili9320.c          | 2 +-
+ drivers/video/backlight/ktd253-backlight.c | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+Merging battery/for-next (4fc88ba435dad power: supply: cros_charge-control: adopt EC charge state on probe)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery/for-next
+Auto-merging drivers/power/supply/max17042_battery.c
+Auto-merging include/linux/power_supply.h
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-class-power        |  39 ++++-
+ .../ABI/testing/sysfs-class-power-ltc4162l         |   2 +
+ Documentation/ABI/testing/sysfs-class-power-mp2629 |   2 +-
+ Documentation/ABI/testing/sysfs-class-power-rt9467 |   2 +
+ Documentation/ABI/testing/sysfs-class-power-rt9471 |   2 +
+ .../devicetree/bindings/power/supply/bq27xxx.yaml  |   2 +-
+ drivers/power/reset/keystone-reset.c               |   3 +-
+ drivers/power/supply/bq24190_charger.c             |  71 +++++++++-
+ drivers/power/supply/bq24257_charger.c             |  43 +++++-
+ drivers/power/supply/bq2515x_charger.c             |  15 +-
+ drivers/power/supply/bq25630_charger.c             |  84 +++++++++++
+ drivers/power/supply/bq25890_charger.c             |   2 +-
+ drivers/power/supply/bq27xxx_battery.c             |  18 ++-
+ drivers/power/supply/bq27xxx_battery_i2c.c         |   2 +
+ drivers/power/supply/charger-manager.c             |   2 +-
+ drivers/power/supply/cros_charge-control.c         | 157 ++++++++++++++++++---
+ drivers/power/supply/da9030_battery.c              |   2 +-
+ drivers/power/supply/ltc4162-l-charger.c           |  56 +++++++-
+ drivers/power/supply/max17042_battery.c            |   2 +-
+ drivers/power/supply/mm8013.c                      |   1 +
+ drivers/power/supply/power_supply_sysfs.c          |  17 +++
+ drivers/power/supply/qcom_smbx.c                   |  98 ++++++++++---
+ drivers/power/supply/rt9467-charger.c              |  54 ++++++-
+ drivers/power/supply/rt9471.c                      |  54 ++++++-
+ drivers/power/supply/s2mu005-battery.c             |   4 +-
+ drivers/power/supply/sbs-charger.c                 |   4 +-
+ drivers/power/supply/smb347-charger.c              |   4 +-
+ include/linux/power/bq27xxx_battery.h              |   1 +
+ include/linux/power_supply.h                       |  18 ++-
+ 29 files changed, 671 insertions(+), 90 deletions(-)
+Merging regulator/for-next (26f25a65cda68 Merge regulator/for-7.4 into regulator-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging MAINTAINERS
+Auto-merging drivers/regulator/Kconfig
+Merge made by the 'ort' strategy.
+ .../regulator/maxim,max77620-regulator.yaml        |  12 +-
+ .../bindings/regulator/maxim,max77826.yaml         |   4 +-
+ .../bindings/regulator/maxim,max77838.yaml         |   4 +-
+ .../regulator/mediatek,mt6380-regulator.yaml       | 117 ++++
+ .../devicetree/bindings/regulator/mps,mp5416.yaml  |  38 +-
+ .../devicetree/bindings/regulator/mps,mp886x.yaml  |  30 +-
+ .../devicetree/bindings/regulator/mps,mpq4210.yaml |  68 ++
+ .../devicetree/bindings/regulator/mps,mpq7920.yaml |  46 +-
+ .../bindings/regulator/mt6380-regulator.txt        |  89 ---
+ .../bindings/regulator/nexperia,nex10000ub.yaml    |  69 ++
+ .../devicetree/bindings/regulator/nxp,pf0900.yaml  |   2 +-
+ .../bindings/regulator/onnn,fan53880.yaml          |   4 +-
+ .../bindings/regulator/pwm-regulator.yaml          |   2 +-
+ .../bindings/regulator/qcom,rpmh-regulator.yaml    |  16 +
+ .../regulator/richtek,rtmv20-regulator.yaml        |   6 +-
+ .../bindings/regulator/silergy,sy8824x.yaml        |   8 +-
+ .../bindings/regulator/silergy,sy8827n.yaml        |   4 +-
+ .../devicetree/bindings/regulator/ti,tps65023.yaml |  90 +++
+ .../devicetree/bindings/regulator/ti,tps6586x.yaml | 196 ++++++
+ .../devicetree/bindings/regulator/tps65023.txt     |  60 --
+ .../devicetree/bindings/regulator/tps6586x.txt     | 135 ----
+ .../bindings/soc/mediatek/mediatek,pwrap.yaml      |   3 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |   2 +
+ MAINTAINERS                                        |   6 +
+ drivers/regulator/88pm886-regulator.c              |  26 +
+ drivers/regulator/Kconfig                          |  21 +-
+ drivers/regulator/Makefile                         |   2 +
+ drivers/regulator/ab8500-ext.c                     |   2 +-
+ drivers/regulator/ab8500.c                         | 748 +++++++++++++++++++--
+ drivers/regulator/axp20x-regulator.c               |   6 +-
+ drivers/regulator/core.c                           |  10 +-
+ drivers/regulator/fixed.c                          |   4 +
+ drivers/regulator/mp886x.c                         |  34 +-
+ drivers/regulator/mpq4210.c                        | 243 +++++++
+ drivers/regulator/nex10000ub-regulator.c           | 147 ++++
+ drivers/regulator/pca9450-regulator.c              |  81 ++-
+ drivers/regulator/pf1550-regulator.c               |  19 +-
+ drivers/regulator/pv88080-regulator.c              |   4 +-
+ drivers/regulator/qcom-rpmh-regulator.c            |  55 +-
+ drivers/regulator/rt6190-regulator.c               |  18 +-
+ drivers/regulator/tps6105x-regulator.c             |   2 +-
+ drivers/regulator/tps65185.c                       |   6 +-
+ include/linux/mfd/88pm886.h                        |   7 +
+ include/linux/regulator/driver.h                   |   2 +-
+ include/linux/regulator/pca9450.h                  |   4 +-
+ kernel/irq/irqdesc.c                               |  10 +-
+ kernel/softirq.c                                   |   4 +-
+ 47 files changed, 1979 insertions(+), 487 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/regulator/mediatek,mt6380-regulator.yaml
+ create mode 100644 Documentation/devicetree/bindings/regulator/mps,mpq4210.yaml
+ delete mode 100644 Documentation/devicetree/bindings/regulator/mt6380-regulator.txt
+ create mode 100644 Documentation/devicetree/bindings/regulator/nexperia,nex10000ub.yaml
+ create mode 100644 Documentation/devicetree/bindings/regulator/ti,tps65023.yaml
+ create mode 100644 Documentation/devicetree/bindings/regulator/ti,tps6586x.yaml
+ delete mode 100644 Documentation/devicetree/bindings/regulator/tps65023.txt
+ delete mode 100644 Documentation/devicetree/bindings/regulator/tps6586x.txt
+ create mode 100644 drivers/regulator/mpq4210.c
+ create mode 100644 drivers/regulator/nex10000ub-regulator.c
+Merging security/next (881f19c2ffbc7 lsm: remove redundant NULL check in security_init())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git security/next
+Auto-merging fs/namei.c
+Auto-merging fs/namespace.c
+Auto-merging include/linux/lsm_hook_defs.h
+CONFLICT (content): Merge conflict in include/linux/lsm_hook_defs.h
+Auto-merging include/linux/ns/ns_common_types.h
+CONFLICT (content): Merge conflict in include/linux/ns/ns_common_types.h
+Auto-merging include/linux/sched.h
+Auto-merging include/linux/security.h
+CONFLICT (content): Merge conflict in include/linux/security.h
+Auto-merging security/security.c
+Auto-merging security/selinux/hooks.c
+Auto-merging security/smack/smack_lsm.c
+Auto-merging tools/testing/selftests/bpf/progs/lsm.c
+Auto-merging tools/testing/selftests/landlock/fs_test.c
+Resolved 'include/linux/lsm_hook_defs.h' using previous resolution.
+Resolved 'include/linux/ns/ns_common_types.h' using previous resolution.
+Resolved 'include/linux/security.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master fb6b2da489e27] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/LSM/index.rst           |  20 +-
+ fs/cachefiles/security.c                          |   4 +-
+ fs/namei.c                                        |  18 +-
+ fs/namespace.c                                    |   3 +-
+ include/linux/cred.h                              |  17 +-
+ include/linux/lsm_audit.h                         |   7 +-
+ include/linux/lsm_hook_defs.h                     |  28 +--
+ include/linux/lsm_hooks.h                         |   1 +
+ include/linux/ns/ns_common_types.h                |   3 +
+ include/linux/sched.h                             |   8 +-
+ include/linux/security.h                          |  78 +++++---
+ include/uapi/linux/nsfs.h                         |   1 +
+ init/init_task.c                                  |   2 +-
+ kernel/auditsc.c                                  |   5 +-
+ kernel/cred.c                                     |   5 +-
+ kernel/nscommon.c                                 |  17 +-
+ kernel/nsproxy.c                                  |   6 +
+ rust/kernel/security.rs                           |   3 +-
+ security/lsm_audit.c                              |   8 +-
+ security/lsm_init.c                               |   9 +-
+ security/security.c                               | 115 +++++++++--
+ security/selinux/hooks.c                          |  25 ++-
+ security/smack/smack_lsm.c                        |   9 +-
+ tools/testing/selftests/bpf/prog_tests/test_lsm.c | 231 ++++++++++++++++++++++
+ tools/testing/selftests/bpf/progs/lsm.c           |  79 ++++++++
+ tools/testing/selftests/landlock/fs_test.c        |  10 +-
+ 26 files changed, 600 insertions(+), 112 deletions(-)
+$ git am -3 ../patches/0001-security-Fix-up-mismerge-and-additional-semantic-iss.patch
+Applying: security: Fix up mismerge and additional semantic issues with vfs-brauner
+Using index info to reconstruct a base tree...
+M	include/linux/lsm_hook_defs.h
+M	security/security.c
+Falling back to patching base and 3-way merge...
+Auto-merging security/security.c
+$ git reset HEAD^
+Unstaged changes after reset:
+M	include/linux/security.h
+M	security/apparmor/af_unix.c
+M	security/apparmor/file.c
+M	security/apparmor/include/af_unix.h
+M	security/apparmor/include/file.h
+M	security/apparmor/include/net.h
+M	security/apparmor/lsm.c
+M	security/apparmor/net.c
+M	security/security.c
+M	security/selinux/hooks.c
+M	security/smack/smack_lsm.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 635e4a5f0833b] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+ Date: Wed Sep 30 13:15:18 2026 +0100
+$ git am -3 ../patches/0001-security-More-merge-fixup-from-the-constification-of.patch
+Applying: security: More merge fixup from the constification of idmap
+$ git reset HEAD^
+Unstaged changes after reset:
+M	security/security.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 65c77b8cc47d9] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+ Date: Wed Sep 30 13:15:18 2026 +0100
+Merging apparmor/apparmor-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'apparmor-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor apparmor/apparmor-next
+Already up to date.
+Merging integrity/next-integrity (1dc317438308c integrity: Replace integrity_audit_message() with integrity_audit_msg())
+$ git merge -m Merge branch 'next-integrity' of https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity integrity/next-integrity
+Auto-merging security/integrity/evm/evm_main.c
+Auto-merging security/integrity/ima/ima_api.c
+Auto-merging security/integrity/ima/ima_appraise.c
+Auto-merging security/integrity/ima/ima_main.c
+Auto-merging security/integrity/ima/ima_policy.c
+Merge made by the 'ort' strategy.
+ security/integrity/evm/evm_main.c         |  9 +++++----
+ security/integrity/ima/ima_api.c          |  8 ++++----
+ security/integrity/ima/ima_appraise.c     |  6 +++---
+ security/integrity/ima/ima_fs.c           |  7 ++++---
+ security/integrity/ima/ima_init.c         |  2 +-
+ security/integrity/ima/ima_main.c         | 13 +++++++------
+ security/integrity/ima/ima_policy.c       | 11 ++++++-----
+ security/integrity/ima/ima_queue.c        |  2 +-
+ security/integrity/ima/ima_queue_keys.c   |  8 ++++----
+ security/integrity/ima/ima_template_lib.c |  2 +-
+ security/integrity/integrity.h            | 18 +++---------------
+ security/integrity/integrity_audit.c      | 12 ++----------
+ 12 files changed, 41 insertions(+), 57 deletions(-)
+Merging selinux/next (f2b37cf384b01 Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git selinux/next
+Auto-merging security/selinux/selinuxfs.c
+Merge made by the 'ort' strategy.
+ security/selinux/avc.c          | 42 ++++++++++++++++-------------
+ security/selinux/include/avc.h  |  7 ++---
+ security/selinux/selinuxfs.c    | 59 ++++++++++++++++++++++++-----------------
+ security/selinux/ss/mls.c       | 38 --------------------------
+ security/selinux/ss/mls.h       |  3 ---
+ security/selinux/ss/mls_types.h |  3 ---
+ security/selinux/ss/policydb.c  | 51 ++++++++++++++++++++++++++---------
+ 7 files changed, 101 insertions(+), 102 deletions(-)
+Merging smack/next (fedc88e38ce97 smack: fix cred UAF in smack_file_send_sigiotask())
+$ git merge -m Merge branch 'next' of https://github.com/cschaufler/smack-next smack/next
+Already up to date.
+Merging tomoyo/master (72d3fcf802c45 Linux 7.3-rc5)
+$ git merge -m Merge branch 'master' of git://git.code.sf.net/p/tomoyo/tomoyo.git tomoyo/master
+Already up to date.
+Merging tpmdd-tpm/for-next-tpm (015fb29a74834 tpm: Disable TPM on null key name mismatch)
+$ git merge -m Merge branch 'for-next-tpm' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd-tpm/for-next-tpm
+Merge made by the 'ort' strategy.
+ drivers/char/tpm/tpm-interface.c | 2 +-
+ drivers/char/tpm/tpm2-cmd.c      | 6 ++++--
+ drivers/char/tpm/tpm2-sessions.c | 1 +
+ 3 files changed, 6 insertions(+), 3 deletions(-)
+Merging tpmdd-keys/for-next-keys (e7ac8fd885298 assoc_array: discard shortcut when collapsing a leaf-only node)
+$ git merge -m Merge branch 'for-next-keys' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd-keys/for-next-keys
+Auto-merging security/keys/trusted-keys/trusted_tpm1.c
+Merge made by the 'ort' strategy.
+ lib/assoc_array.c                         | 36 ++++++++++++--------
+ security/keys/persistent.c                | 55 ++++++++++++++++++-------------
+ security/keys/trusted-keys/trusted_tpm1.c |  4 ++-
+ security/keys/trusted-keys/trusted_tpm2.c |  4 ++-
+ 4 files changed, 60 insertions(+), 39 deletions(-)
+Merging watchdog/watchdog-next (8b5a9f09037e3 dt-bindings: watchdog: apple,wdt: Add t8140 compatible)
+$ git merge -m Merge branch 'watchdog-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git watchdog/watchdog-next
+Auto-merging drivers/watchdog/hpwdt.c
+Auto-merging drivers/watchdog/mtk_wdt.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/watchdog/apple,wdt.yaml    |   1 +
+ .../bindings/watchdog/mediatek,mtk-wdt.yaml        |   2 +
+ .../bindings/watchdog/renesas,r9a09g057-wdt.yaml   |  29 ++++-
+ .../devicetree/bindings/watchdog/samsung-wdt.yaml  |  23 +++-
+ Documentation/watchdog/watchdog-parameters.rst     |   2 +
+ drivers/watchdog/acquirewdt.c                      |   3 +-
+ drivers/watchdog/advantech_ec_wdt.c                |   3 +-
+ drivers/watchdog/advantechwdt.c                    |   5 +-
+ drivers/watchdog/airoha_wdt.c                      |   5 +-
+ drivers/watchdog/alim1535_wdt.c                    |   5 +-
+ drivers/watchdog/alim7101_wdt.c                    |   5 +-
+ drivers/watchdog/arm_smc_wdt.c                     |   3 +-
+ drivers/watchdog/armada_37xx_wdt.c                 |   3 +-
+ drivers/watchdog/aspeed_wdt.c                      |   3 +-
+ drivers/watchdog/at91rm9200_wdt.c                  |   5 +-
+ drivers/watchdog/at91sam9_wdt.c                    |   5 +-
+ drivers/watchdog/atcwdt200_wdt.c                   |   5 +-
+ drivers/watchdog/ath79_wdt.c                       |   5 +-
+ drivers/watchdog/bcm2835_wdt.c                     |   3 +-
+ drivers/watchdog/bcm47xx_wdt.c                     |   5 +-
+ drivers/watchdog/bcm7038_wdt.c                     |   3 +-
+ drivers/watchdog/booke_wdt.c                       |   3 +-
+ drivers/watchdog/cadence_wdt.c                     |  13 +--
+ drivers/watchdog/cgbc_wdt.c                        |   7 +-
+ drivers/watchdog/da9052_wdt.c                      |   5 +-
+ drivers/watchdog/da9055_wdt.c                      |   3 +-
+ drivers/watchdog/da9062_wdt.c                      |   8 +-
+ drivers/watchdog/davinci_wdt.c                     |   7 +-
+ drivers/watchdog/db8500_wdt.c                      |   5 +-
+ drivers/watchdog/dw_wdt.c                          |   3 +-
+ drivers/watchdog/ebc-c384_wdt.c                    |   5 +-
+ drivers/watchdog/eurotechwdt.c                     |   3 +-
+ drivers/watchdog/exar_wdt.c                        |   5 +-
+ drivers/watchdog/f71808e_wdt.c                     |   9 +-
+ drivers/watchdog/gef_wdt.c                         |   3 +-
+ drivers/watchdog/geodewdt.c                        |   5 +-
+ drivers/watchdog/gpio_wdt.c                        |   3 +-
+ drivers/watchdog/hpwdt.c                           |   3 +-
+ drivers/watchdog/i6300esb.c                        |  28 +++--
+ drivers/watchdog/iTCO_wdt.c                        |   5 +-
+ drivers/watchdog/ib700wdt.c                        |   5 +-
+ drivers/watchdog/ibmasr.c                          |   3 +-
+ drivers/watchdog/ie6xx_wdt.c                       |   5 +-
+ drivers/watchdog/imgpdc_wdt.c                      |   5 +-
+ drivers/watchdog/imx2_wdt.c                        |   5 +-
+ drivers/watchdog/imx7ulp_wdt.c                     |   3 +-
+ drivers/watchdog/imx_sc_wdt.c                      |   3 +-
+ drivers/watchdog/indydog.c                         |   3 +-
+ drivers/watchdog/intel_oc_wdt.c                    |   5 +-
+ drivers/watchdog/it87_wdt.c                        |   7 +-
+ drivers/watchdog/jz4740_wdt.c                      |   7 +-
+ drivers/watchdog/keembay_wdt.c                     |  13 +--
+ drivers/watchdog/kempld_wdt.c                      |   7 +-
+ drivers/watchdog/lenovo_se10_wdt.c                 |   5 +-
+ drivers/watchdog/lenovo_se30_wdt.c                 |   5 +-
+ drivers/watchdog/lenovo_se30g2_se60_wdt.c          |   5 +-
+ drivers/watchdog/loongson1_wdt.c                   |   5 +-
+ drivers/watchdog/lpc18xx_wdt.c                     |   5 +-
+ drivers/watchdog/ma35d1_wdt.c                      |   3 +-
+ drivers/watchdog/max63xx_wdt.c                     |   9 +-
+ drivers/watchdog/max77620_wdt.c                    |   3 +-
+ drivers/watchdog/mena21_wdt.c                      |   3 +-
+ drivers/watchdog/menf21bmc_wdt.c                   |   3 +-
+ drivers/watchdog/menz69_wdt.c                      |   3 +-
+ drivers/watchdog/meson_gxbb_wdt.c                  |   5 +-
+ drivers/watchdog/meson_wdt.c                       |   3 +-
+ drivers/watchdog/mixcomwd.c                        |   3 +-
+ drivers/watchdog/mpc8xxx_wdt.c                     |   5 +-
+ drivers/watchdog/msc313e_wdt.c                     |  12 +--
+ drivers/watchdog/mt7621_wdt.c                      |   3 +-
+ drivers/watchdog/mtk_wdt.c                         | 119 ++++++++++++++++++---
+ drivers/watchdog/nct6694_wdt.c                     |   3 +-
+ drivers/watchdog/ni903x_wdt.c                      |   5 +-
+ drivers/watchdog/nic7018_wdt.c                     |   5 +-
+ drivers/watchdog/nv_tco.c                          |   5 +-
+ drivers/watchdog/octeon-wdt-main.c                 |   5 +-
+ drivers/watchdog/of_xilinx_wdt.c                   |   8 +-
+ drivers/watchdog/omap_wdt.c                        |   3 +-
+ drivers/watchdog/orion_wdt.c                       |   3 +-
+ drivers/watchdog/pc87413_wdt.c                     |   9 +-
+ drivers/watchdog/pcwd.c                            |   5 +-
+ drivers/watchdog/pcwd_pci.c                        |   7 +-
+ drivers/watchdog/pcwd_usb.c                        |   7 +-
+ drivers/watchdog/pika_wdt.c                        |   5 +-
+ drivers/watchdog/pm8916_wdt.c                      |   8 +-
+ drivers/watchdog/pnx4008_wdt.c                     |   5 +-
+ drivers/watchdog/pseries-wdt.c                     |   7 +-
+ drivers/watchdog/rc32434_wdt.c                     |   5 +-
+ drivers/watchdog/renesas_wdt.c                     |   3 +-
+ drivers/watchdog/rn5t618_wdt.c                     |   3 +-
+ drivers/watchdog/rt2880_wdt.c                      |   3 +-
+ drivers/watchdog/rti_wdt.c                         |   5 +-
+ drivers/watchdog/rzg2l_wdt.c                       |   3 +-
+ drivers/watchdog/rzv2h_wdt.c                       |   3 +-
+ drivers/watchdog/s32g_wdt.c                        |   5 +-
+ drivers/watchdog/s3c2410_wdt.c                     |  23 +++-
+ drivers/watchdog/sama5d4_wdt.c                     |   5 +-
+ drivers/watchdog/sbc60xxwdt.c                      |   5 +-
+ drivers/watchdog/sbc7240_wdt.c                     |   5 +-
+ drivers/watchdog/sbc8360.c                         |   3 +-
+ drivers/watchdog/sbc_epx_c3.c                      |   3 +-
+ drivers/watchdog/sbsa_gwdt.c                       |  24 ++++-
+ drivers/watchdog/sc1200wdt.c                       |   3 +-
+ drivers/watchdog/sch311x_wdt.c                     |   5 +-
+ drivers/watchdog/shwdt.c                           |   7 +-
+ drivers/watchdog/simatic-ipc-wdt.c                 |   3 +-
+ drivers/watchdog/sl28cpld_wdt.c                    |   3 +-
+ drivers/watchdog/smsc37b787_wdt.c                  |   3 +-
+ drivers/watchdog/softdog.c                         |   5 +-
+ drivers/watchdog/sp5100_tco.c                      |  68 ++++++++++--
+ drivers/watchdog/sp805_wdt.c                       |  14 +--
+ drivers/watchdog/starfive-wdt.c                    |   7 +-
+ drivers/watchdog/stmp3xxx_rtc_wdt.c                |  13 +--
+ drivers/watchdog/stpmic1_wdt.c                     |   3 +-
+ drivers/watchdog/sun4v_wdt.c                       |   5 +-
+ drivers/watchdog/sunplus_wdt.c                     |   3 +-
+ drivers/watchdog/sunxi_wdt.c                       |   3 +-
+ drivers/watchdog/tegra_wdt.c                       |   5 +-
+ drivers/watchdog/tqmx86_wdt.c                      |   3 +-
+ drivers/watchdog/ts4800_wdt.c                      |   3 +-
+ drivers/watchdog/twl4030_wdt.c                     |   3 +-
+ drivers/watchdog/txx9wdt.c                         |   7 +-
+ drivers/watchdog/uniphier_wdt.c                    |   5 +-
+ drivers/watchdog/via_wdt.c                         |   5 +-
+ drivers/watchdog/visconti_wdt.c                    |   3 +-
+ drivers/watchdog/w83627hf_wdt.c                    |   5 +-
+ drivers/watchdog/w83877f_wdt.c                     |   5 +-
+ drivers/watchdog/w83977f_wdt.c                     |   5 +-
+ drivers/watchdog/wafer5823wdt.c                    |   5 +-
+ drivers/watchdog/watchdog_core.c                   |   5 +-
+ drivers/watchdog/watchdog_dev.c                    |   9 +-
+ drivers/watchdog/wdat_wdt.c                        |   5 +-
+ drivers/watchdog/wdt.c                             |   9 +-
+ drivers/watchdog/wdt977.c                          |   7 +-
+ drivers/watchdog/wdt_pci.c                         |   9 +-
+ drivers/watchdog/wm831x_wdt.c                      |   3 +-
+ drivers/watchdog/wm8350_wdt.c                      |   3 +-
+ drivers/watchdog/xen_wdt.c                         |   5 +-
+ drivers/watchdog/xilinx_wwdt.c                     |   5 +-
+ drivers/watchdog/ziirave_wdt.c                     |   3 +-
+ include/dt-bindings/reset/mediatek,mt8167-wdt.h    |  21 ++++
+ 141 files changed, 689 insertions(+), 298 deletions(-)
+ create mode 100644 include/dt-bindings/reset/mediatek,mt8167-wdt.h
+Merging iommu/next (cec2dc7663e71 Merge branches 'apple/dart', 'samsung/exynos', 'riscv', 'broadcom', 'intel/vt-d', 'amd/amd-vi' and 'core' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git iommu/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu.h
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+Merge made by the 'ort' strategy.
+ .../bindings/display/brcm,bcm2835-hvs.yaml         |   3 +
+ .../devicetree/bindings/iommu/apple,dart.yaml      |  12 +-
+ .../bindings/iommu/brcm,bcm2712-iommu.yaml         |  54 ++
+ .../bindings/iommu/brcm,bcm2712-iommuc.yaml        |  40 ++
+ MAINTAINERS                                        |  12 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |   1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |  50 ++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |  12 +
+ drivers/iommu/Kconfig                              |  17 +-
+ drivers/iommu/Makefile                             |   1 +
+ drivers/iommu/amd/amd_iommu.h                      |   3 +
+ drivers/iommu/amd/amd_iommu_types.h                |   7 +
+ drivers/iommu/amd/init.c                           | 169 ++++---
+ drivers/iommu/amd/iommu.c                          | 460 ++++++++++++-----
+ drivers/iommu/apple-dart.c                         |  74 ++-
+ drivers/iommu/bcm2712-iommu-cache.c                |  84 ++++
+ drivers/iommu/bcm2712-iommu-cache.h                |   9 +
+ drivers/iommu/bcm2712-iommu.c                      | 550 +++++++++++++++++++++
+ drivers/iommu/exynos-iommu.c                       |  37 +-
+ drivers/iommu/generic_pt/.kunitconfig              |   1 +
+ drivers/iommu/generic_pt/Kconfig                   |  10 +
+ drivers/iommu/generic_pt/fmt/Makefile              |   2 +
+ drivers/iommu/generic_pt/fmt/bcm2712.h             | 288 +++++++++++
+ drivers/iommu/generic_pt/fmt/defs_bcm2712.h        |  18 +
+ drivers/iommu/generic_pt/fmt/iommu_bcm2712.c       |   6 +
+ drivers/iommu/generic_pt/kunit_iommu_pt.h          |   2 +-
+ drivers/iommu/intel/dmar.c                         |  72 ++-
+ drivers/iommu/intel/iommu.c                        | 156 +++++-
+ drivers/iommu/intel/iommu.h                        |  17 +-
+ drivers/iommu/intel/nested.c                       |   2 +
+ drivers/iommu/iommu-sva.c                          |  14 +-
+ drivers/iommu/iommu.c                              |   4 +
+ drivers/iommu/riscv/iommu.c                        |  12 +
+ include/linux/amd-iommu.h                          |  13 +-
+ include/linux/generic_pt/common.h                  |   6 +
+ include/linux/generic_pt/iommu.h                   |  12 +
+ 36 files changed, 1995 insertions(+), 235 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommu.yaml
+ create mode 100644 Documentation/devicetree/bindings/iommu/brcm,bcm2712-iommuc.yaml
+ create mode 100644 drivers/iommu/bcm2712-iommu-cache.c
+ create mode 100644 drivers/iommu/bcm2712-iommu-cache.h
+ create mode 100644 drivers/iommu/bcm2712-iommu.c
+ create mode 100644 drivers/iommu/generic_pt/fmt/bcm2712.h
+ create mode 100644 drivers/iommu/generic_pt/fmt/defs_bcm2712.h
+ create mode 100644 drivers/iommu/generic_pt/fmt/iommu_bcm2712.c
+Merging audit/next (88bd5e852addf Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git audit/next
+Auto-merging kernel/auditsc.c
+Merge made by the 'ort' strategy.
+ include/linux/audit.h |  6 ++--
+ kernel/audit.c        |  2 +-
+ kernel/audit.h        | 20 ++++++++----
+ kernel/audit_tree.c   | 37 ++++++++++++++++------
+ kernel/audit_watch.c  |  2 ++
+ kernel/auditfilter.c  | 86 +++++++++++++++++++++++++--------------------------
+ kernel/auditsc.c      |  4 ++-
+ 7 files changed, 92 insertions(+), 65 deletions(-)
+Merging devicetree/for-next (833aaa4790eee of/irq: Stop the MSI walk at the first msi-parent)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree/for-next
+Auto-merging Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml
+Auto-merging Documentation/devicetree/bindings/mfd/ti,tps65910.yaml
+Auto-merging Documentation/devicetree/bindings/trivial-devices.yaml
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/arm,cci-400.yaml       |   6 +-
+ .../devicetree/bindings/arm/arm,coresight-cti.yaml |   2 +-
+ .../bindings/arm/mediatek/mediatek,audsys.yaml     |   2 +-
+ .../bindings/arm/mediatek/mediatek,g3dsys.txt      |  30 -
+ Documentation/devicetree/bindings/arm/omap/mpu.txt |  54 --
+ .../devicetree/bindings/arm/ti/ti,omap-mpu.yaml    |  58 ++
+ .../devicetree/bindings/arm/vexpress-config.yaml   |   4 +
+ .../devicetree/bindings/bus/omap-ocp2scp.txt       |  29 -
+ .../devicetree/bindings/bus/ti,omap-ocp2scp.yaml   |  74 ++
+ .../bindings/display/bridge/sil,sii9022.yaml       |   2 +-
+ .../display/mediatek/mediatek,hdmi-ddc.yaml        |  11 +-
+ .../devicetree/bindings/dts-coding-style.rst       |   5 +-
+ .../bindings/firmware/nvidia,tegra186-bpmp.yaml    |   6 +-
+ .../devicetree/bindings/gpio/delta,tn48m-gpio.yaml |   2 +-
+ .../devicetree/bindings/gpu/arm,mali-utgard.yaml   |  25 +-
+ .../devicetree/bindings/hwmon/adi,ltc2991.yaml     |   2 +-
+ .../devicetree/bindings/hwmon/vexpress.txt         |  23 -
+ .../bindings/i2c/xlnx,xps-iic-2.00.a.yaml          |   4 +-
+ .../bindings/iio/light/upisemi,us5182.yaml         |   2 +-
+ .../devicetree/bindings/input/matrix-keymap.yaml   |   3 +-
+ .../devicetree/bindings/input/ti,tca8418.yaml      |  21 +-
+ .../interrupt-controller/chrp,open-pic.yaml        |   7 +-
+ .../bindings/interrupt-controller/qcom,pdc.yaml    |   1 +
+ .../mailbox/allwinner,sun6i-a31-msgbox.yaml        |   3 -
+ .../devicetree/bindings/mailbox/altera-mailbox.txt |  12 +-
+ .../bindings/mailbox/hisilicon,hi3660-mailbox.txt  |   2 +-
+ .../bindings/mailbox/hisilicon,hi6220-mailbox.txt  |   2 +-
+ .../devicetree/bindings/mailbox/mailbox.txt        |  60 --
+ .../bindings/mailbox/ti,omap-mailbox.yaml          |   3 +-
+ .../bindings/media/amlogic,c3-mipi-csi2.yaml       |   4 +-
+ .../devicetree/bindings/media/arm,mali-c55.yaml    |   2 +-
+ .../devicetree/bindings/media/atmel,isc.yaml       |  14 +-
+ .../bindings/media/brcm,bcm2835-unicam.yaml        |  10 +-
+ .../devicetree/bindings/media/cdns,csi2rx.yaml     |  58 +-
+ .../bindings/media/i2c/galaxycore,gc05a2.yaml      |   2 +-
+ .../devicetree/bindings/media/i2c/ovti,ov2680.yaml |  26 +-
+ .../devicetree/bindings/media/i2c/ovti,ov2732.yaml |   6 +-
+ .../bindings/media/img,e5010-jpeg-enc.yaml         |  22 +-
+ .../bindings/media/mediatek,vcodec-encoder.yaml    |   2 +-
+ .../bindings/media/mediatek-jpeg-decoder.yaml      |   4 +-
+ .../bindings/media/mediatek-jpeg-encoder.yaml      |   2 +-
+ .../bindings/media/microchip,csi2dc.yaml           |  52 +-
+ .../devicetree/bindings/media/microchip,xisc.yaml  |  14 +-
+ .../devicetree/bindings/media/nxp,imx8-jpeg.yaml   |   4 +-
+ .../bindings/media/qcom,msm8996-venus.yaml         |  40 +-
+ .../bindings/media/qcom,x1e80100-camss.yaml        |  26 +-
+ .../bindings/media/raspberrypi,pispbe.yaml         |  12 +-
+ .../devicetree/bindings/media/samsung,fimc.yaml    |  14 +-
+ .../devicetree/bindings/media/st,stm32-dcmi.yaml   |  16 +-
+ .../devicetree/bindings/media/st,stm32-dcmipp.yaml |  14 +-
+ .../devicetree/bindings/media/ti,cal.yaml          |  48 +-
+ .../devicetree/bindings/mfd/ti,tps65910.yaml       |   2 +-
+ Documentation/devicetree/bindings/mux/reg-mux.yaml |   2 +-
+ .../bindings/nvmem/zii,rave-sp-eeprom.yaml         |   2 +-
+ .../bindings/pci/hisilicon,kirin-pcie.yaml         |   4 +-
+ .../bindings/pinctrl/sunplus,sp7021-pinctrl.yaml   |   2 +-
+ .../bindings/power/reset/xlnx,zynqmp-power.yaml    |   9 +-
+ .../devicetree/bindings/regulator/vexpress.txt     |  32 -
+ .../soc/hisilicon/hisilicon,hip05-cpld.yaml        |  35 +
+ .../soc/mediatek/mediatek,mt2701-g3dsys.yaml       |  58 ++
+ .../bindings/soc/nuvoton/nuvoton,npcm-gcr.yaml     |  18 +-
+ .../bindings/soc/qcom/qcom,aoss-qmp.yaml           |   2 +-
+ .../bindings/spi/aspeed,ast2600-fmc.yaml           |   6 +-
+ .../devicetree/bindings/thermal/thermal-idle.yaml  |   2 +-
+ .../devicetree/bindings/trivial-devices.yaml       |   4 +-
+ Documentation/devicetree/of_unittest.rst           |   2 +-
+ MAINTAINERS                                        |   2 +
+ drivers/of/fdt_address.c                           |   2 +-
+ drivers/of/irq.c                                   |  39 +-
+ drivers/of/property.c                              |  23 +-
+ include/dt-bindings/clock/agilex-clock.h           |   1 +
+ scripts/dtc/dt-check-style                         | 972 ++++++++++++---------
+ .../dtc/dt-style-selftest/bad/dts-blank-lines.dts  |  37 +
+ .../bad/dts-child-name-order.dtso                  |  33 +
+ .../dtc/dt-style-selftest/bad/dts-cont-align.dts   |  27 +
+ .../dt-style-selftest/bad/dts-digit-node-order.dts |  40 +
+ .../bad/dts-digit-node-order.dtso                  |  41 +
+ .../bad/dts-extend-node-child-name-order.dtso      |  26 +
+ .../bad/dts-extend-node-digit-node-order.dtso      |  34 +
+ .../dtc/dt-style-selftest/bad/dts-line-length.dts  |  22 +
+ .../dtc/dt-style-selftest/bad/dts-node-name.dts    |  60 ++
+ .../dt-style-selftest/bad/dts-property-name.dts    |  28 +
+ .../dt-style-selftest/bad/dts-property-order.dts   |  18 +
+ .../dt-style-selftest/bad/dts-property-order.dtso  |  62 ++
+ .../bad/dts-redundant-ws-strict.dts                |  27 +
+ .../dtc/dt-style-selftest/bad/dts-redundant-ws.dts |  28 +
+ .../dt-style-selftest/bad/dts-redundant-ws.dtso    |   9 +
+ .../dtc/dt-style-selftest/bad/dts-trailing-ws.dts  |  16 +
+ .../dtc/dt-style-selftest/bad/dts-unused-label.dts |  21 +
+ .../dt-style-selftest/bad/yaml-blank-lines.yaml    |  54 ++
+ .../bad/yaml-child-addr-order.yaml                 |   2 +-
+ .../bad/yaml-child-name-order.yaml                 |   2 +-
+ .../dtc/dt-style-selftest/bad/yaml-cont-align.yaml |   9 +-
+ .../bad/yaml-digit-node-order.yaml                 |   2 +-
+ .../dtc/dt-style-selftest/bad/yaml-hex-case.yaml   |   7 +-
+ .../dt-style-selftest/bad/yaml-indent-strict.yaml  |   2 +-
+ .../bad/yaml-label-in-string.yaml                  |   2 +-
+ .../dt-style-selftest/bad/yaml-line-length.yaml    |   5 +-
+ .../dt-style-selftest/bad/yaml-mixed-indent.yaml   |   4 +-
+ .../dt-style-selftest/bad/yaml-multi-close.yaml    |   2 +-
+ .../dtc/dt-style-selftest/bad/yaml-node-close.yaml |   2 +-
+ .../dtc/dt-style-selftest/bad/yaml-node-name.yaml  |  54 ++
+ .../bad/yaml-prop-order-device-type.yaml           |   2 +-
+ .../dtc/dt-style-selftest/bad/yaml-prop-order.yaml |   7 +-
+ .../dt-style-selftest/bad/yaml-prop-pairing.yaml   |   2 +-
+ .../dt-style-selftest/bad/yaml-property-name.yaml  |  46 +
+ .../bad/yaml-redundant-ws-strict.yaml              |  31 +
+ .../dt-style-selftest/bad/yaml-redundant-ws.yaml   |  35 +
+ .../dt-style-selftest/bad/yaml-required-blank.yaml |   2 +-
+ scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml    |   2 +-
+ .../bad/yaml-trailing-comment.yaml                 |   2 +-
+ .../dt-style-selftest/bad/yaml-trailing-ws.yaml    |   7 +-
+ .../bad/yaml-unclosed-comment.yaml                 |   2 +-
+ .../bad/yaml-unit-addr-prefix.yaml                 |   2 +-
+ .../dtc/dt-style-selftest/bad/yaml-unit-addr.yaml  |   2 +-
+ .../dt-style-selftest/bad/yaml-unused-label.yaml   |   2 +-
+ .../bad/yaml-value-ws-multiline.yaml               |   2 +-
+ .../dtc/dt-style-selftest/bad/yaml-value-ws.yaml   |   2 +-
+ .../expected/dts-blank-lines.dts.txt               |   6 +
+ .../expected/dts-child-name-order.dts.txt          |   1 +
+ .../expected/dts-child-name-order.dtso.txt         |   3 +
+ .../expected/dts-cont-align.dts.txt                |  11 +
+ .../expected/dts-digit-node-order.dts.txt          |   2 +
+ .../expected/dts-digit-node-order.dtso.txt         |   2 +
+ .../dts-extend-node-child-name-order.dtso.txt      |   2 +
+ .../dts-extend-node-digit-node-order.dtso.txt      |   2 +
+ .../expected/dts-line-length.dts.txt               |   3 +
+ .../expected/dts-mixed-indent.dts.txt              |   1 +
+ .../expected/dts-node-name.dts.txt                 |  13 +
+ .../expected/dts-property-name.dts.txt             |  14 +
+ .../expected/dts-property-order.dts.txt            |  16 +-
+ .../expected/dts-property-order.dtso.txt           |  12 +
+ .../expected/dts-redundant-ws-strict.dts.txt       |  13 +
+ .../expected/dts-redundant-ws.dts.txt              |  10 +
+ .../expected/dts-redundant-ws.dtso.txt             |   2 +
+ .../expected/dts-trailing-ws.dts.txt               |   3 +
+ .../expected/dts-unused-label.dts.txt              |   2 +
+ .../expected/yaml-blank-lines.yaml.txt             |   6 +
+ .../expected/yaml-cont-align.yaml.txt              |   4 +-
+ .../expected/yaml-hex-case.yaml.txt                |   2 +
+ .../expected/yaml-line-length.yaml.txt             |   1 +
+ .../expected/yaml-mixed-indent.yaml.txt            |   1 +
+ .../expected/yaml-node-name.yaml.txt               |   7 +
+ .../expected/yaml-prop-order.yaml.txt              |   1 +
+ .../expected/yaml-prop-pairing.yaml.txt            |   4 +-
+ .../expected/yaml-property-name.yaml.txt           |  14 +
+ .../expected/yaml-redundant-ws-strict.yaml.txt     |   5 +
+ .../expected/yaml-redundant-ws.yaml.txt            |   4 +
+ .../expected/yaml-trailing-ws.yaml.txt             |   2 +
+ .../expected/yaml-value-ws-multiline.yaml.txt      |   1 +
+ .../good/dts-child-name-order.dtso                 |  44 +
+ .../dtc/dt-style-selftest/good/dts-cont-align.dts  |  14 +-
+ .../good/dts-digit-node-order.dts                  |   3 -
+ .../good/dts-digit-node-order.dtso                 |  59 ++
+ .../good/dts-extend-node-child-name-order.dtso     |  26 +
+ .../good/dts-extend-node-digit-node-order.dtso     |  34 +
+ .../dt-style-selftest/good/dts-property-order.dts  |   8 +
+ .../dt-style-selftest/good/dts-property-order.dtso |  50 ++
+ .../dtc/dt-style-selftest/good/yaml-4space.yaml    |   2 +-
+ .../dt-style-selftest/good/yaml-cont-align.yaml    |  33 +
+ .../good/yaml-tricky-parsing.yaml                  |   2 +-
+ scripts/dtc/dt-style-selftest/run.sh               |   2 +-
+ 162 files changed, 2357 insertions(+), 980 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/arm/mediatek/mediatek,g3dsys.txt
+ delete mode 100644 Documentation/devicetree/bindings/arm/omap/mpu.txt
+ create mode 100644 Documentation/devicetree/bindings/arm/ti/ti,omap-mpu.yaml
+ delete mode 100644 Documentation/devicetree/bindings/bus/omap-ocp2scp.txt
+ create mode 100644 Documentation/devicetree/bindings/bus/ti,omap-ocp2scp.yaml
+ delete mode 100644 Documentation/devicetree/bindings/hwmon/vexpress.txt
+ delete mode 100644 Documentation/devicetree/bindings/mailbox/mailbox.txt
+ delete mode 100644 Documentation/devicetree/bindings/regulator/vexpress.txt
+ create mode 100644 Documentation/devicetree/bindings/soc/hisilicon/hisilicon,hip05-cpld.yaml
+ create mode 100644 Documentation/devicetree/bindings/soc/mediatek/mediatek,mt2701-g3dsys.yaml
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-blank-lines.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-child-name-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-cont-align.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-extend-node-child-name-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-extend-node-digit-node-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-line-length.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-node-name.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-property-name.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-property-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-redundant-ws-strict.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-trailing-ws.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/dts-unused-label.dts
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/yaml-blank-lines.yaml
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/yaml-node-name.yaml
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/yaml-property-name.yaml
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws-strict.yaml
+ create mode 100644 scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws.yaml
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-blank-lines.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dtso.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-cont-align.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dtso.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-extend-node-child-name-order.dtso.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-extend-node-digit-node-order.dtso.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-line-length.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-node-name.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-property-name.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-property-order.dtso.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-redundant-ws-strict.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dtso.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-trailing-ws.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/dts-unused-label.dts.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/yaml-blank-lines.yaml.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/yaml-node-name.yaml.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/yaml-property-name.yaml.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws-strict.yaml.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws.yaml.txt
+ create mode 100644 scripts/dtc/dt-style-selftest/good/dts-child-name-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/good/dts-extend-node-child-name-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/good/dts-extend-node-digit-node-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/good/dts-property-order.dtso
+ create mode 100644 scripts/dtc/dt-style-selftest/good/yaml-cont-align.yaml
+Merging dt-krzk/for-next (dfcea1641506d Merge branches 'next/dt' and 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/actions/owl-s500.dtsi                   | 12 ++++++------
+ arch/arm/boot/dts/arm/vexpress-v2p-ca5s.dts               |  2 +-
+ arch/arm/boot/dts/aspeed/aspeed-bmc-opp-witherspoon.dts   |  2 +-
+ arch/arm/boot/dts/aspeed/aspeed-bmc-supermicro-x11spi.dts |  2 +-
+ arch/arm64/boot/dts/actions/s700.dtsi                     |  6 +++---
+ arch/arm64/boot/dts/actions/s900.dtsi                     |  6 +++---
+ arch/arm64/boot/dts/cavium/thunder-88xx.dtsi              |  6 +++---
+ 7 files changed, 18 insertions(+), 18 deletions(-)
+Merging mailbox/for-next (14af7a96afa39 mailbox: add Axiado AX3005 mailbox driver)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git mailbox/for-next
+Already up to date.
+Merging spi/for-next (168b675c187d4 Merge spi/for-7.4 into spi-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi/for-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-class-spi-master   |   6 +-
+ .../devicetree/bindings/spi/amlogic,a4-spisg.yaml  |  30 +-
+ .../bindings/spi/amlogic,meson-gx-spicc.yaml       |  13 +-
+ .../bindings/spi/microchip,pic32mzda-spi.yaml      |  82 ++++
+ .../bindings/spi/microchip,spi-pic32.txt           |  34 --
+ .../devicetree/bindings/spi/renesas,sh-msiof.yaml  |   9 +-
+ .../bindings/spi/spi-peripheral-props.yaml         |   8 +
+ Documentation/spi/multiple-data-lanes.rst          |  10 +-
+ drivers/spi/Kconfig                                | 364 +++++++--------
+ drivers/spi/spi-amlogic-spisg.c                    | 116 ++++-
+ drivers/spi/spi-ar934x.c                           |  13 +-
+ drivers/spi/spi-bcm2835.c                          |   4 +-
+ drivers/spi/spi-bcm63xx.c                          |   2 +-
+ drivers/spi/spi-dw-core.c                          |  15 +-
+ drivers/spi/spi-geni-qcom.c                        |  93 +++-
+ drivers/spi/spi-imx.c                              |   2 +-
+ drivers/spi/spi-ingenic.c                          |   5 +-
+ drivers/spi/spi-ma35d1-qspi.c                      |  27 +-
+ drivers/spi/spi-mem.c                              |  14 +-
+ drivers/spi/spi-mtk-nor.c                          |   1 +
+ drivers/spi/spi-mxic.c                             |   8 +-
+ drivers/spi/spi-omap2-mcspi.c                      |   4 +-
+ drivers/spi/spi-orion.c                            |   9 +
+ drivers/spi/spi-pic32.c                            |   2 +-
+ drivers/spi/spi-qpic-snand.c                       | 104 ++---
+ drivers/spi/spi-realtek-rtl.c                      |  36 +-
+ drivers/spi/spi-rockchip-sfc.c                     |  21 +-
+ drivers/spi/spi-rspi.c                             |  49 +-
+ drivers/spi/spi-s3c64xx.c                          |   2 +-
+ drivers/spi/spi-sh-msiof.c                         |   6 +-
+ drivers/spi/spi-stm32.c                            |   2 +-
+ drivers/spi/spi-sun6i.c                            |   2 +-
+ drivers/spi/spi-sunplus-sp7021.c                   |  13 +-
+ drivers/spi/spi-tegra210-quad.c                    | 517 ++++++++++++++++++---
+ drivers/spi/spi-virtio.c                           |  22 +-
+ drivers/spi/spi-xilinx.c                           |   2 +-
+ drivers/spi/spi.c                                  |  15 +-
+ include/linux/spi/spi.h                            |   9 +
+ tools/spi/Makefile                                 |   2 +-
+ tools/spi/spidev_test.c                            | 330 +++++++++----
+ 40 files changed, 1396 insertions(+), 607 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/spi/microchip,pic32mzda-spi.yaml
+ delete mode 100644 Documentation/devicetree/bindings/spi/microchip,spi-pic32.txt
+Merging tip/master (1aeb52f7869a6 Merge branch into tip/master: 'x86/tdx')
+  2c6a75adb15f8 ("x86/um: Remove unused <asm/required-features.h> header")
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip/master
+Auto-merging Documentation/ABI/testing/sysfs-devices-system-cpu
+Auto-merging Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
+Auto-merging Documentation/scheduler/index.rst
+CONFLICT (content): Merge conflict in Documentation/scheduler/index.rst
+Auto-merging MAINTAINERS
+Auto-merging arch/Kconfig
+Auto-merging arch/arm64/Kconfig
+Auto-merging arch/arm64/Kconfig.platforms
+Auto-merging arch/arm64/configs/defconfig
+CONFLICT (content): Merge conflict in arch/arm64/configs/defconfig
+Auto-merging arch/arm64/include/asm/preempt.h
+Auto-merging arch/loongarch/Kconfig
+Auto-merging arch/powerpc/Kconfig
+Auto-merging arch/riscv/Kconfig
+Auto-merging arch/s390/Kconfig
+Auto-merging arch/s390/kernel/hiperdispatch.c
+Auto-merging arch/um/kernel/um_arch.c
+Auto-merging arch/x86/Kconfig
+Auto-merging arch/x86/crypto/aesni-intel_glue.c
+Auto-merging arch/x86/include/asm/string_64.h
+Auto-merging arch/x86/kvm/mmu/mmu.c
+Auto-merging arch/x86/kvm/svm/sev.c
+Auto-merging arch/x86/kvm/vmx/tdx.c
+Auto-merging fs/aio.c
+Auto-merging fs/exec.c
+Auto-merging include/linux/sched.h
+Auto-merging io_uring/rw.c
+Auto-merging kernel/events/core.c
+Auto-merging kernel/exit.c
+Auto-merging kernel/sched/fair.c
+Auto-merging net/core/pktgen.c
+Resolved 'Documentation/scheduler/index.rst' using previous resolution.
+Resolved 'arch/arm64/configs/defconfig' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 24bf019cbe7e4] Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/sysfs-devices-system-cpu |  24 +-
+ Documentation/ABI/testing/sysfs-platform-ts5500    |  54 --
+ Documentation/arch/arm64/silicon-errata.rst        |   3 +
+ Documentation/arch/x86/tdx.rst                     |  21 +
+ Documentation/arch/x86/xstate.rst                  |  54 ++
+ .../bindings/interrupt-controller/qcom,pdc.yaml    |   1 +
+ Documentation/driver-api/index.rst                 |   1 +
+ Documentation/driver-api/steal-governor.rst        | 151 ++++
+ Documentation/filesystems/resctrl.rst              |  19 +-
+ Documentation/scheduler/index.rst                  |   1 +
+ Documentation/scheduler/sched-paravirt.rst         |  67 ++
+ MAINTAINERS                                        |  14 +-
+ arch/Kconfig                                       |  38 -
+ arch/arm/kernel/perf_regs.c                        |   8 +-
+ arch/arm64/Kconfig                                 |  10 +-
+ arch/arm64/Kconfig.platforms                       |   1 +
+ arch/arm64/boot/dts/qcom/purwa.dtsi                |   5 +
+ arch/arm64/configs/defconfig                       |   2 -
+ arch/arm64/include/asm/preempt.h                   |  10 -
+ arch/arm64/kernel/paravirt.c                       |   4 +-
+ arch/arm64/kernel/perf_regs.c                      |   8 +-
+ arch/csky/kernel/perf_regs.c                       |   8 +-
+ arch/loongarch/Kconfig                             |   1 -
+ arch/loongarch/kernel/paravirt.c                   |   4 +-
+ arch/loongarch/kernel/perf_regs.c                  |   8 +-
+ arch/mips/kernel/perf_regs.c                       |   8 +-
+ arch/parisc/kernel/perf_regs.c                     |   8 +-
+ arch/powerpc/Kconfig                               |   1 -
+ arch/powerpc/perf/perf_regs.c                      |   2 +-
+ arch/powerpc/platforms/pseries/setup.c             |   4 +-
+ arch/riscv/Kconfig                                 |   1 -
+ arch/riscv/include/asm/smp.h                       |  12 +
+ arch/riscv/kernel/paravirt.c                       |   4 +-
+ arch/riscv/kernel/perf_regs.c                      |   8 +-
+ arch/riscv/kernel/sbi-ipi.c                        |   4 +-
+ arch/riscv/kernel/smp.c                            |  12 -
+ arch/s390/Kconfig                                  |   1 -
+ arch/s390/include/asm/preempt.h                    |  11 -
+ arch/s390/kernel/hiperdispatch.c                   |  10 +-
+ arch/s390/kernel/perf_regs.c                       |   2 +-
+ arch/um/kernel/um_arch.c                           |  78 +-
+ arch/x86/Kconfig                                   |  10 -
+ arch/x86/boot/compressed/error.c                   |  19 -
+ arch/x86/boot/compressed/error.h                   |   1 -
+ arch/x86/boot/compressed/mem.c                     |  42 -
+ arch/x86/boot/compressed/sev.h                     |   2 -
+ arch/x86/boot/compressed/tdx-shared.c              |   2 +
+ arch/x86/boot/early_serial_console.c               |  13 +-
+ arch/x86/boot/string.c                             |  13 +-
+ arch/x86/coco/sev/core.c                           |  32 +-
+ arch/x86/coco/tdx/tdx-shared.c                     |  31 +
+ arch/x86/coco/tdx/tdx.c                            |  41 +-
+ arch/x86/crypto/aegis128-aesni-glue.c              |   3 +-
+ arch/x86/crypto/aesni-intel_glue.c                 |   7 +-
+ arch/x86/crypto/aria_aesni_avx2_glue.c             |  11 +-
+ arch/x86/crypto/aria_aesni_avx_glue.c              |  11 +-
+ arch/x86/crypto/aria_gfni_avx512_glue.c            |  11 +-
+ arch/x86/crypto/camellia_aesni_avx2_glue.c         |  11 +-
+ arch/x86/crypto/camellia_aesni_avx_glue.c          |  11 +-
+ arch/x86/crypto/cast5_avx_glue.c                   |   7 +-
+ arch/x86/crypto/cast6_avx_glue.c                   |   7 +-
+ arch/x86/crypto/serpent_avx2_glue.c                |   9 +-
+ arch/x86/crypto/serpent_avx_glue.c                 |   7 +-
+ arch/x86/crypto/sm4_aesni_avx2_glue.c              |  11 +-
+ arch/x86/crypto/sm4_aesni_avx_glue.c               |  11 +-
+ arch/x86/crypto/twofish_avx_glue.c                 |   6 +-
+ arch/x86/events/amd/uncore.c                       |  39 +-
+ arch/x86/events/core.c                             | 488 ++++++++++-
+ arch/x86/events/intel/core.c                       | 154 ++--
+ arch/x86/events/intel/ds.c                         | 232 ++++--
+ arch/x86/events/intel/lbr.c                        |   2 +-
+ arch/x86/events/perf_event.h                       | 217 ++++-
+ arch/x86/include/asm/cpufeatures.h                 |   5 +-
+ arch/x86/include/asm/cpuid/api.h                   |   2 +-
+ arch/x86/include/asm/fpu/regset.h                  |   6 +-
+ arch/x86/include/asm/fpu/sched.h                   |   6 +-
+ arch/x86/include/asm/fpu/types.h                   |  25 -
+ arch/x86/include/asm/fpu/xstate.h                  |   3 +
+ arch/x86/include/asm/kvm-x86-ops.h                 |   1 +
+ arch/x86/include/asm/kvm_host.h                    |  10 +-
+ arch/x86/include/asm/local.h                       |   4 +-
+ arch/x86/include/asm/math_emu.h                    |  15 -
+ arch/x86/include/asm/msr-index.h                   |  10 +
+ arch/x86/include/asm/perf_event.h                  |  46 +-
+ arch/x86/include/asm/preempt.h                     |  28 -
+ arch/x86/include/asm/processor.h                   |   2 +-
+ arch/x86/include/asm/sev.h                         |   4 +
+ arch/x86/include/asm/shared/string.h               |  52 ++
+ arch/x86/include/asm/shared/tdx.h                  |   6 +
+ arch/x86/include/asm/string.h                      |  21 +-
+ arch/x86/include/asm/string_64.h                   |   1 -
+ arch/x86/include/asm/tdx.h                         |  23 +
+ arch/x86/include/asm/tdx_global_metadata.h         |   9 +-
+ arch/x86/include/asm/traps.h                       |   2 -
+ arch/x86/include/uapi/asm/perf_regs.h              |  53 ++
+ arch/x86/include/uapi/asm/sigcontext.h             |  15 +
+ arch/x86/kernel/asm-offsets.c                      |   1 +
+ arch/x86/kernel/cpu/amd.c                          |  21 +-
+ arch/x86/kernel/cpu/bugs.c                         |  18 +-
+ arch/x86/kernel/cpu/common.c                       |  98 ++-
+ arch/x86/kernel/cpu/microcode/intel-ucode-defs.h   |  91 +-
+ arch/x86/kernel/cpu/mtrr/amd.c                     |   9 +-
+ arch/x86/kernel/cpu/resctrl/ctrlmondata.c          |   6 +
+ arch/x86/kernel/cpu/scattered.c                    |   2 +
+ arch/x86/kernel/cpu/sgx/main.c                     |   8 +-
+ arch/x86/kernel/cpu/vmware.c                       |   4 +-
+ arch/x86/kernel/crash.c                            |   6 +-
+ arch/x86/kernel/fpu/bugs.c                         |   4 -
+ arch/x86/kernel/fpu/core.c                         |  46 +-
+ arch/x86/kernel/fpu/init.c                         |   6 +-
+ arch/x86/kernel/fpu/regset.c                       |   6 -
+ arch/x86/kernel/fpu/signal.c                       | 136 +--
+ arch/x86/kernel/fpu/xstate.c                       |  62 +-
+ arch/x86/kernel/kvm.c                              |   4 +-
+ arch/x86/kernel/perf_regs.c                        | 172 +++-
+ arch/x86/kernel/shstk.c                            |   2 +-
+ arch/x86/kvm/mmu/mmu.c                             |   4 +
+ arch/x86/kvm/svm/sev.c                             |   2 +
+ arch/x86/kvm/vmx/pmu_intel.c                       |  28 +-
+ arch/x86/kvm/vmx/tdx.c                             |  99 ++-
+ arch/x86/kvm/vmx/tdx.h                             |   2 +
+ arch/x86/kvm/vmx/vmx.c                             |  10 +-
+ arch/x86/kvm/vmx/vmx.h                             |  15 +-
+ arch/x86/platform/Makefile                         |   1 -
+ arch/x86/platform/olpc/olpc-xo15-sci.c             |   4 +-
+ arch/x86/platform/pvh/enlighten.c                  |   3 +-
+ arch/x86/platform/ts5500/Makefile                  |   2 -
+ arch/x86/platform/ts5500/ts5500.c                  | 341 --------
+ arch/x86/virt/svm/sev.c                            | 172 +++-
+ arch/x86/virt/vmx/tdx/seamcall_internal.h          |  19 +
+ arch/x86/virt/vmx/tdx/tdx.c                        | 439 ++++++++--
+ arch/x86/virt/vmx/tdx/tdx.h                        |  10 +-
+ arch/x86/virt/vmx/tdx/tdx_global_metadata.c        |  23 +-
+ arch/x86/virt/vmx/tdx/tdxcall.S                    |  10 +-
+ arch/x86/xen/pmu.c                                 |   5 +-
+ drivers/base/cpu.c                                 |  12 +
+ drivers/clocksource/timer-clint.c                  |   4 +-
+ drivers/crypto/ccp/sev-dev.c                       |   2 +
+ drivers/firmware/efi/libstub/x86-stub.c            |  39 +
+ drivers/irqchip/Kconfig                            |   6 +-
+ drivers/irqchip/irq-aclint-sswi.c                  |   4 +-
+ drivers/irqchip/irq-al-fic.c                       |   2 +-
+ drivers/irqchip/irq-gic-v3-its.c                   |   6 +-
+ drivers/irqchip/irq-gic-v3.c                       |  12 +-
+ drivers/irqchip/irq-gic-v5.c                       |   9 +-
+ drivers/irqchip/irq-gic.c                          |   8 +-
+ drivers/irqchip/irq-lan966x-oic.c                  |   1 -
+ drivers/irqchip/irq-mtk-cirq.c                     |   2 +-
+ drivers/irqchip/irq-pruss-intc.c                   |  38 +-
+ drivers/irqchip/irq-riscv-imsic-early.c            |   4 +-
+ drivers/irqchip/irq-riscv-imsic-state.c            |  13 +-
+ drivers/irqchip/irq-riscv-imsic-state.h            |   1 -
+ drivers/irqchip/irq-sifive-plic.c                  |   2 +-
+ drivers/irqchip/irq-vic.c                          |   2 +-
+ drivers/irqchip/qcom-pdc.c                         |   3 +
+ drivers/resctrl/mpam_resctrl.c                     |   5 +
+ drivers/soc/fsl/qe/qe_ports_ic.c                   |   1 -
+ drivers/virt/Kconfig                               |  17 +
+ drivers/virt/Makefile                              |   1 +
+ drivers/virt/coco/tdx-guest/tdx-guest.c            |   6 +-
+ drivers/virt/steal_governor.c                      | 296 +++++++
+ drivers/xen/time.c                                 |   4 +-
+ fs/aio.c                                           |   2 +-
+ fs/exec.c                                          |  13 +-
+ fs/proc/uptime.c                                   |   6 +-
+ fs/resctrl/ctrlmondata.c                           |   6 +-
+ fs/resctrl/pseudo_lock.c                           |   9 +-
+ fs/resctrl/rdtgroup.c                              |   8 +-
+ include/asm-generic/preempt.h                      |  10 -
+ include/linux/bitmap.h                             |  14 +
+ include/linux/cpumask.h                            |  42 +
+ include/linux/hrtimer.h                            |  10 +-
+ include/linux/interrupt.h                          |   6 +-
+ include/linux/irq-entry-common.h                   |  17 +-
+ include/linux/kernel.h                             |  20 -
+ include/linux/kernel_stat.h                        |  11 +
+ include/linux/list.h                               |   6 +-
+ include/linux/perf_event.h                         |  23 +
+ include/linux/perf_regs.h                          |  36 +-
+ include/linux/posix-timers.h                       |  39 +-
+ include/linux/preempt.h                            |  20 +-
+ include/linux/resctrl.h                            |  19 +
+ include/linux/sched.h                              |  32 +-
+ include/linux/sched/cputime.h                      |   6 +-
+ include/linux/sched/task.h                         |   1 -
+ include/linux/wait.h                               |   2 +-
+ include/uapi/linux/perf_event.h                    |  49 +-
+ include/uapi/linux/sched.h                         |   2 +-
+ include/vdso/math64.h                              |  31 +-
+ io_uring/rw.c                                      |   2 +-
+ kernel/Kconfig.kexec                               |   2 +-
+ kernel/Kconfig.preempt                             |  13 +-
+ kernel/cpu.c                                       |   6 +
+ kernel/crash_core.c                                |   2 +-
+ kernel/entry/common.c                              |  17 +-
+ kernel/events/core.c                               | 177 +++-
+ kernel/exit.c                                      |  13 +-
+ kernel/futex/requeue.c                             |   2 +-
+ kernel/futex/waitwake.c                            |   8 +-
+ kernel/irq/irqdomain.c                             |   1 +
+ kernel/locking/rtmutex.c                           |   2 +-
+ kernel/sched/core.c                                | 444 +++++-----
+ kernel/sched/cputime.c                             |   4 +-
+ kernel/sched/deadline.c                            |   9 +-
+ kernel/sched/debug.c                               |  21 +-
+ kernel/sched/ext/ext.c                             |  18 +-
+ kernel/sched/ext/ext.h                             |   7 +
+ kernel/sched/fair.c                                | 131 ++-
+ kernel/sched/idle.c                                |   5 +-
+ kernel/sched/rt.c                                  |   7 +-
+ kernel/sched/sched.h                               |  80 +-
+ kernel/sched/stop_task.c                           |   5 +-
+ kernel/sched/wait.c                                |  22 +-
+ kernel/time/hrtimer.c                              |  23 +-
+ kernel/time/posix-cpu-timers.c                     | 103 ++-
+ kernel/time/posix-timers.c                         |  26 +-
+ kernel/time/posix-timers.h                         |   3 +
+ kernel/time/sleep_timeout.c                        |   4 +-
+ kernel/time/tick-sched.c                           |  30 +-
+ kernel/time/time_test.c                            |  16 +
+ kernel/time/timeconv.c                             |   6 +-
+ kernel/time/timekeeping.c                          |   4 +-
+ kernel/time/timer.c                                |   2 +-
+ kernel/time/timer_migration.c                      |   6 +-
+ kernel/time/vsyscall.c                             |  26 +-
+ lib/bitmap.c                                       |  17 +
+ lib/crc/x86/crc-pclmul-template.h                  |   6 +-
+ lib/crypto/x86/blake2s.h                           |   4 +-
+ lib/crypto/x86/chacha.h                            |   3 +-
+ lib/crypto/x86/nh.h                                |   4 +-
+ lib/crypto/x86/poly1305.h                          |   7 +-
+ lib/crypto/x86/sha1.h                              |   4 +-
+ lib/crypto/x86/sha256.h                            |   4 +-
+ lib/crypto/x86/sha512.h                            |   3 +-
+ lib/crypto/x86/sm3.h                               |   3 +-
+ lib/raid/xor/Makefile                              |   2 +-
+ lib/raid/xor/x86/xor-avx512.c                      | 122 +++
+ lib/raid/xor/x86/xor_arch.h                        |  31 +-
+ lib/vdso/gettimeofday.c                            |   2 +
+ net/core/pktgen.c                                  |   4 +-
+ tools/objtool/Documentation/klp-test-design.txt    | 286 +++++++
+ tools/objtool/Documentation/klp-write-tests.txt    | 266 ++++++
+ tools/objtool/Makefile                             |  15 +-
+ .../tests/generic/fixtures/abs_and_addressable.c   |  44 +
+ tools/objtool/tests/generic/fixtures/basic.c       |  20 +
+ .../objtool/tests/generic/fixtures/changed_data.c  |  16 +
+ .../objtool/tests/generic/fixtures/checksum_data.c | 116 +++
+ .../objtool/tests/generic/fixtures/checksum_insn.c |  78 ++
+ .../tests/generic/fixtures/checksum_position.c     |  42 +
+ .../objtool/tests/generic/fixtures/checksum_skip.c |  47 ++
+ .../objtool/tests/generic/fixtures/cold_function.c |  21 +
+ .../objtool/tests/generic/fixtures/cross_module.c  |  25 +
+ .../tests/generic/fixtures/data_alignment.c        |  29 +
+ .../tests/generic/fixtures/function_removal.c      |  25 +
+ .../tests/generic/fixtures/init_reference.c        |  17 +
+ tools/objtool/tests/generic/fixtures/jump_label.c  |  76 ++
+ tools/objtool/tests/generic/fixtures/klp_funcs.c   |  31 +
+ .../tests/generic/fixtures/local_to_global.c       |  34 +
+ tools/objtool/tests/generic/fixtures/new_data.c    |  23 +
+ .../tests/generic/fixtures/new_export_ref.c        |  35 +
+ .../objtool/tests/generic/fixtures/new_function.c  |  21 +
+ tools/objtool/tests/generic/fixtures/no_modinfo.c  |  11 +
+ .../tests/generic/fixtures/special_section.c       |  24 +
+ .../generic/fixtures/special_section_shared.c      |  31 +
+ tools/objtool/tests/generic/fixtures/static_call.c |  59 ++
+ .../objtool/tests/generic/fixtures/static_local.c  |  17 +
+ .../generic/fixtures/static_local_uncorrelated.c   |  41 +
+ .../objtool/tests/generic/fixtures/switch_rodata.c |  31 +
+ .../tests/generic/fixtures/symid_discarded.c       |  25 +
+ tools/objtool/tests/generic/fixtures/sympos_dup.c  |  32 +
+ .../tests/generic/fixtures/sympos_vmlinux.c        |  40 +
+ .../tests/generic/fixtures/thinlto_ambiguity.c     |  57 ++
+ .../objtool/tests/generic/fixtures/thinlto_local.c |  39 +
+ tools/objtool/tests/generic/fixtures/ubsan_noise.c |  49 ++
+ .../tests/generic/test-abs-and-addressable.sh      |  50 ++
+ tools/objtool/tests/generic/test-basic.sh          |  17 +
+ tools/objtool/tests/generic/test-changed-data.sh   |  18 +
+ tools/objtool/tests/generic/test-checksum-data.sh  |  61 ++
+ tools/objtool/tests/generic/test-checksum-debug.sh |  49 ++
+ tools/objtool/tests/generic/test-checksum-insn.sh  |  49 ++
+ .../tests/generic/test-checksum-position.sh        |  51 ++
+ tools/objtool/tests/generic/test-checksum-skip.sh  |  81 ++
+ tools/objtool/tests/generic/test-checksum-value.sh |  37 +
+ tools/objtool/tests/generic/test-cold-function.sh  |  39 +
+ tools/objtool/tests/generic/test-data-alignment.sh |  40 +
+ .../generic/test-export-symbol-for-modules.sh      |  39 +
+ .../objtool/tests/generic/test-function-removal.sh |  34 +
+ tools/objtool/tests/generic/test-init-reference.sh |  29 +
+ .../tests/generic/test-jump-label-exempt-keys.sh   |  51 ++
+ tools/objtool/tests/generic/test-jump-label-key.sh |  44 +
+ .../tests/generic/test-jump-label-module-key.sh    |  22 +
+ .../generic/test-jump-label-module-static-key.sh   |  45 +
+ .../tests/generic/test-jump-label-new-key.sh       |  51 ++
+ .../tests/generic/test-klp-funcs-content.sh        |  45 +
+ .../tests/generic/test-local-to-global-flip.sh     |  63 ++
+ .../objtool/tests/generic/test-local-vs-export.sh  |  32 +
+ .../objtool/tests/generic/test-missing-checksum.sh |  18 +
+ .../objtool/tests/generic/test-missing-modinfo.sh  |  16 +
+ .../tests/generic/test-modname-normalize.sh        |  26 +
+ tools/objtool/tests/generic/test-module-object.sh  |  31 +
+ .../tests/generic/test-module-vmlinux-reloc.sh     |  40 +
+ tools/objtool/tests/generic/test-new-data.sh       |  27 +
+ tools/objtool/tests/generic/test-new-export-ref.sh |  46 ++
+ tools/objtool/tests/generic/test-new-function.sh   |  16 +
+ tools/objtool/tests/generic/test-post-link.sh      |  42 +
+ .../tests/generic/test-special-section-shared.sh   |  26 +
+ .../objtool/tests/generic/test-special-section.sh  |  20 +
+ .../generic/test-static-call-annotate-stripped.sh  |  42 +
+ .../tests/generic/test-static-call-module-key.sh   |  36 +
+ .../objtool/tests/generic/test-static-call-new.sh  |  45 +
+ .../generic/test-static-local-uncorrelated.sh      |  41 +
+ tools/objtool/tests/generic/test-static-local.sh   |  24 +
+ tools/objtool/tests/generic/test-switch-rodata.sh  |  53 ++
+ .../objtool/tests/generic/test-symid-discarded.sh  |  44 +
+ tools/objtool/tests/generic/test-sympos-vmlinux.sh |  57 ++
+ tools/objtool/tests/generic/test-sympos.sh         |  51 ++
+ .../tests/generic/test-symvers-parse-error.sh      |  23 +
+ .../tests/generic/test-thinlto-ambiguity.sh        |  77 ++
+ tools/objtool/tests/generic/test-thinlto-local.sh  |  48 ++
+ tools/objtool/tests/generic/test-ubsan-noise.sh    |  48 ++
+ tools/objtool/tests/lib.sh                         | 911 +++++++++++++++++++++
+ tools/objtool/tests/run-tests.sh                   | 239 ++++++
+ tools/objtool/tests/x86/fixtures/alt_annotate.c    |  57 ++
+ tools/objtool/tests/x86/fixtures/checksum_alt.c    |  66 ++
+ .../objtool/tests/x86/fixtures/empty_alternative.c |  77 ++
+ tools/objtool/tests/x86/fixtures/kcfi.c            |  39 +
+ .../objtool/tests/x86/fixtures/special_sections.c  |  77 ++
+ .../tests/x86/fixtures/static_call_no_key.c        |  32 +
+ tools/objtool/tests/x86/test-alt-annotation.sh     |  38 +
+ tools/objtool/tests/x86/test-checksum-alt.sh       |  45 +
+ tools/objtool/tests/x86/test-empty-alternative.sh  |  31 +
+ tools/objtool/tests/x86/test-kcfi.sh               |  39 +
+ .../tests/x86/test-manual-klp-static-call.sh       |  40 +
+ tools/objtool/tests/x86/test-special-sections.sh   |  42 +
+ tools/perf/trace/beauty/include/uapi/linux/sched.h |   2 +-
+ .../testing/selftests/timers/clocksource-switch.c  |  23 +-
+ tools/testing/selftests/timers/posix_timers.c      |   7 +-
+ tools/testing/selftests/timers/raw_skew.c          |  12 +
+ tools/testing/selftests/x86/Makefile               |   5 +-
+ .../selftests/x86/sigframe_fpu_portability.c       | 235 ++++++
+ tools/testing/selftests/x86/xstate.c               |  12 -
+ tools/testing/selftests/x86/xstate.h               |  20 +
+ 342 files changed, 10162 insertions(+), 2241 deletions(-)
+ delete mode 100644 Documentation/ABI/testing/sysfs-platform-ts5500
+ create mode 100644 Documentation/driver-api/steal-governor.rst
+ create mode 100644 Documentation/scheduler/sched-paravirt.rst
+ delete mode 100644 arch/x86/include/asm/math_emu.h
+ create mode 100644 arch/x86/include/asm/shared/string.h
+ delete mode 100644 arch/x86/platform/ts5500/Makefile
+ delete mode 100644 arch/x86/platform/ts5500/ts5500.c
+ create mode 100644 drivers/virt/steal_governor.c
+ create mode 100644 lib/raid/xor/x86/xor-avx512.c
+ create mode 100644 tools/objtool/Documentation/klp-test-design.txt
+ create mode 100644 tools/objtool/Documentation/klp-write-tests.txt
+ create mode 100644 tools/objtool/tests/generic/fixtures/abs_and_addressable.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/basic.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/changed_data.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/checksum_data.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/checksum_insn.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/checksum_position.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/checksum_skip.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/cold_function.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/cross_module.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/data_alignment.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/function_removal.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/init_reference.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/jump_label.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/klp_funcs.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/local_to_global.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/new_data.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/new_export_ref.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/new_function.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/no_modinfo.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/special_section.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/special_section_shared.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/static_call.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/static_local.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/static_local_uncorrelated.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/switch_rodata.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/symid_discarded.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/sympos_dup.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/sympos_vmlinux.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/thinlto_ambiguity.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/thinlto_local.c
+ create mode 100644 tools/objtool/tests/generic/fixtures/ubsan_noise.c
+ create mode 100755 tools/objtool/tests/generic/test-abs-and-addressable.sh
+ create mode 100755 tools/objtool/tests/generic/test-basic.sh
+ create mode 100755 tools/objtool/tests/generic/test-changed-data.sh
+ create mode 100755 tools/objtool/tests/generic/test-checksum-data.sh
+ create mode 100755 tools/objtool/tests/generic/test-checksum-debug.sh
+ create mode 100755 tools/objtool/tests/generic/test-checksum-insn.sh
+ create mode 100755 tools/objtool/tests/generic/test-checksum-position.sh
+ create mode 100755 tools/objtool/tests/generic/test-checksum-skip.sh
+ create mode 100755 tools/objtool/tests/generic/test-checksum-value.sh
+ create mode 100755 tools/objtool/tests/generic/test-cold-function.sh
+ create mode 100755 tools/objtool/tests/generic/test-data-alignment.sh
+ create mode 100755 tools/objtool/tests/generic/test-export-symbol-for-modules.sh
+ create mode 100755 tools/objtool/tests/generic/test-function-removal.sh
+ create mode 100755 tools/objtool/tests/generic/test-init-reference.sh
+ create mode 100755 tools/objtool/tests/generic/test-jump-label-exempt-keys.sh
+ create mode 100755 tools/objtool/tests/generic/test-jump-label-key.sh
+ create mode 100755 tools/objtool/tests/generic/test-jump-label-module-key.sh
+ create mode 100755 tools/objtool/tests/generic/test-jump-label-module-static-key.sh
+ create mode 100755 tools/objtool/tests/generic/test-jump-label-new-key.sh
+ create mode 100755 tools/objtool/tests/generic/test-klp-funcs-content.sh
+ create mode 100755 tools/objtool/tests/generic/test-local-to-global-flip.sh
+ create mode 100755 tools/objtool/tests/generic/test-local-vs-export.sh
+ create mode 100755 tools/objtool/tests/generic/test-missing-checksum.sh
+ create mode 100755 tools/objtool/tests/generic/test-missing-modinfo.sh
+ create mode 100755 tools/objtool/tests/generic/test-modname-normalize.sh
+ create mode 100755 tools/objtool/tests/generic/test-module-object.sh
+ create mode 100755 tools/objtool/tests/generic/test-module-vmlinux-reloc.sh
+ create mode 100755 tools/objtool/tests/generic/test-new-data.sh
+ create mode 100755 tools/objtool/tests/generic/test-new-export-ref.sh
+ create mode 100755 tools/objtool/tests/generic/test-new-function.sh
+ create mode 100755 tools/objtool/tests/generic/test-post-link.sh
+ create mode 100755 tools/objtool/tests/generic/test-special-section-shared.sh
+ create mode 100755 tools/objtool/tests/generic/test-special-section.sh
+ create mode 100755 tools/objtool/tests/generic/test-static-call-annotate-stripped.sh
+ create mode 100755 tools/objtool/tests/generic/test-static-call-module-key.sh
+ create mode 100755 tools/objtool/tests/generic/test-static-call-new.sh
+ create mode 100755 tools/objtool/tests/generic/test-static-local-uncorrelated.sh
+ create mode 100755 tools/objtool/tests/generic/test-static-local.sh
+ create mode 100755 tools/objtool/tests/generic/test-switch-rodata.sh
+ create mode 100755 tools/objtool/tests/generic/test-symid-discarded.sh
+ create mode 100755 tools/objtool/tests/generic/test-sympos-vmlinux.sh
+ create mode 100755 tools/objtool/tests/generic/test-sympos.sh
+ create mode 100755 tools/objtool/tests/generic/test-symvers-parse-error.sh
+ create mode 100755 tools/objtool/tests/generic/test-thinlto-ambiguity.sh
+ create mode 100755 tools/objtool/tests/generic/test-thinlto-local.sh
+ create mode 100755 tools/objtool/tests/generic/test-ubsan-noise.sh
+ create mode 100644 tools/objtool/tests/lib.sh
+ create mode 100755 tools/objtool/tests/run-tests.sh
+ create mode 100644 tools/objtool/tests/x86/fixtures/alt_annotate.c
+ create mode 100644 tools/objtool/tests/x86/fixtures/checksum_alt.c
+ create mode 100644 tools/objtool/tests/x86/fixtures/empty_alternative.c
+ create mode 100644 tools/objtool/tests/x86/fixtures/kcfi.c
+ create mode 100644 tools/objtool/tests/x86/fixtures/special_sections.c
+ create mode 100644 tools/objtool/tests/x86/fixtures/static_call_no_key.c
+ create mode 100755 tools/objtool/tests/x86/test-alt-annotation.sh
+ create mode 100755 tools/objtool/tests/x86/test-checksum-alt.sh
+ create mode 100755 tools/objtool/tests/x86/test-empty-alternative.sh
+ create mode 100755 tools/objtool/tests/x86/test-kcfi.sh
+ create mode 100755 tools/objtool/tests/x86/test-manual-klp-static-call.sh
+ create mode 100755 tools/objtool/tests/x86/test-special-sections.sh
+ create mode 100644 tools/testing/selftests/x86/sigframe_fpu_portability.c
+Merging kexec/kexec-next (9d0b028715b00 Merge branch 'kexec-7.4' into kexec-next)
+$ git merge -m Merge branch 'kexec-next' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git kexec/kexec-next
+Auto-merging include/linux/mm.h
+Auto-merging mm/memory-failure.c
+Merge made by the 'ort' strategy.
+ arch/riscv/kernel/kexec_elf.c |  2 +-
+ include/linux/kexec.h         | 15 ---------------
+ include/linux/mm.h            | 14 ++++++++++++++
+ kernel/kexec_core.c           | 10 ++++++++++
+ kernel/kexec_file.c           | 22 ++++++++++++++++++++--
+ mm/memory-failure.c           | 40 ++++++++++++++++++++++++++++++++++++++++
+ 6 files changed, 85 insertions(+), 18 deletions(-)
+Merging liveupdate/next (5221d141653a8 Merge branch 'kho-bootmem' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git liveupdate/next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging drivers/pci/pci.h
+Auto-merging drivers/pci/probe.c
+Auto-merging drivers/pci/quirks.c
+Auto-merging include/linux/memblock.h
+Auto-merging include/linux/pci.h
+Auto-merging mm/Kconfig
+Auto-merging mm/memblock.c
+CONFLICT (content): Merge conflict in mm/memblock.c
+Auto-merging mm/mm_init.c
+CONFLICT (content): Merge conflict in mm/mm_init.c
+Resolved 'mm/memblock.c' using previous resolution.
+Resolved 'mm/mm_init.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 39f8a23bd04b6] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/PCI/index.rst                     |    1 +
+ Documentation/PCI/liveupdate.rst                |   35 +
+ Documentation/admin-guide/kernel-parameters.txt |   22 +-
+ Documentation/admin-guide/mm/kho.rst            |   18 +-
+ Documentation/core-api/kho/index.rst            |   36 +-
+ Documentation/core-api/liveupdate.rst           |    5 +
+ MAINTAINERS                                     |   15 +
+ arch/x86/boot/compressed/kaslr.c                |   12 +-
+ arch/x86/include/uapi/asm/setup_data.h          |    4 +-
+ arch/x86/kernel/e820.c                          |   11 +-
+ arch/x86/kernel/kexec-bzimage64.c               |    6 +-
+ arch/x86/kernel/setup.c                         |    2 +-
+ arch/x86/realmode/init.c                        |    2 +-
+ drivers/firmware/efi/efi-init.c                 |    6 +-
+ drivers/of/fdt.c                                |    8 +-
+ drivers/of/kexec.c                              |   16 +-
+ drivers/pci/Kconfig                             |   15 +
+ drivers/pci/Makefile                            |    1 +
+ drivers/pci/liveupdate.c                        | 1015 +++++++++++++++++++++++
+ drivers/pci/liveupdate.h                        |   62 ++
+ drivers/pci/pci-driver.c                        |    9 +-
+ drivers/pci/pci.c                               |   79 +-
+ drivers/pci/pci.h                               |    5 +
+ drivers/pci/probe.c                             |   16 +-
+ drivers/pci/quirks.c                            |   58 +-
+ include/asm-generic/kexec_handover.h            |    2 +-
+ include/linux/kexec.h                           |    2 +-
+ include/linux/kexec_handover.h                  |   16 +-
+ include/linux/kho/abi/pci.h                     |   66 ++
+ include/linux/liveupdate.h                      |   22 +
+ include/linux/memblock.h                        |   30 +-
+ include/linux/pci.h                             |    7 +
+ include/linux/pci_liveupdate.h                  |   72 ++
+ kernel/kexec_file.c                             |    4 +-
+ kernel/liveupdate/Kconfig                       |    1 -
+ kernel/liveupdate/kexec_handover.c              |  347 ++++----
+ kernel/liveupdate/kexec_handover_debugfs.c      |   24 +-
+ kernel/liveupdate/kexec_handover_internal.h     |    4 +-
+ kernel/liveupdate/luo_file.c                    |   84 ++
+ kernel/liveupdate/luo_internal.h                |   17 +
+ mm/Kconfig                                      |    4 -
+ mm/memblock.c                                   |   56 +-
+ mm/memfd_luo.c                                  |    3 +-
+ mm/mm_init.c                                    |    4 +-
+ tools/testing/memblock/internal.h               |    2 +-
+ 45 files changed, 1871 insertions(+), 355 deletions(-)
+ create mode 100644 Documentation/PCI/liveupdate.rst
+ create mode 100644 drivers/pci/liveupdate.c
+ create mode 100644 drivers/pci/liveupdate.h
+ create mode 100644 include/linux/kho/abi/pci.h
+ create mode 100644 include/linux/pci_liveupdate.h
+Merging clockevents/timers/drivers/next (1b8b356b4b06e clocksource/drivers/armada: Unwind timer clock on init failure)
+$ git merge -m Merge branch 'timers/drivers/next' of https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git clockevents/timers/drivers/next
+Auto-merging drivers/clocksource/timer-ti-dm.c
+Auto-merging drivers/pwm/pwm-samsung.c
+Merge made by the 'ort' strategy.
+Merging edac/edac-for-next (898e6a2c5ce0e Merge ras/edac-drivers into for-next)
+$ git merge -m Merge branch 'edac-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git edac/edac-for-next
+Auto-merging MAINTAINERS
+Auto-merging arch/x86/include/asm/msr-index.h
+Auto-merging drivers/base/cacheinfo.c
+Merge made by the 'ort' strategy.
+ MAINTAINERS                      |  26 +++-
+ arch/x86/include/asm/mce.h       |   5 +
+ arch/x86/include/asm/msr-index.h |   2 +
+ drivers/base/cacheinfo.c         |  17 +++
+ drivers/edac/Kconfig             |  22 +++
+ drivers/edac/Makefile            |   5 +-
+ drivers/edac/altera_edac.c       |  67 ++++-----
+ drivers/edac/bluefield_edac.c    |   4 +-
+ drivers/edac/dummy_edac.c        | 191 +++++++++++++++++++++++++
+ drivers/edac/i10nm_base.c        |   2 +
+ drivers/edac/ie31200_edac.c      |   4 +-
+ drivers/edac/intel-bff.c         | 295 +++++++++++++++++++++++++++++++++++++++
+ drivers/edac/loongson_edac.c     |   4 +-
+ drivers/edac/skx_base.c          |   2 +-
+ drivers/edac/versalnet_edac.c    |   1 +
+ include/linux/cacheinfo.h        |  12 +-
+ 16 files changed, 603 insertions(+), 56 deletions(-)
+ create mode 100644 drivers/edac/dummy_edac.c
+ create mode 100644 drivers/edac/intel-bff.c
+Merging ftrace/for-next (6dc993d4520fd tools/bootconfig: remove unused PAGE_SIZE macro)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace/for-next
+Auto-merging lib/bootconfig.c
+Auto-merging tools/bootconfig/main.c
+Merge made by the 'ort' strategy.
+ include/linux/bootconfig.h |  7 +------
+ lib/bootconfig.c           | 22 +++++++++-------------
+ tools/bootconfig/main.c    |  2 --
+ 3 files changed, 10 insertions(+), 21 deletions(-)
+$ git am -3 ../patches/0001-ftrace-Fix-semantic-conflict-with-mm-tree.patch
+Applying: ftrace: Fix semantic conflict with mm tree
+Using index info to reconstruct a base tree...
+M	kernel/trace/trace_printk.c
+Falling back to patching base and 3-way merge...
+Auto-merging kernel/trace/trace_printk.c
+No changes -- Patch already applied.
+Merging rcu/next (aebf6c4777d53 Merge branches 'misc.2026.09.18a' and 'torture.2026.09.18a' into HEAD)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux rcu/next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Merge made by the 'ort' strategy.
+ Documentation/RCU/stallwarn.rst                    |  22 +-
+ Documentation/admin-guide/kernel-parameters.txt    |  23 +-
+ include/linux/rcuref.h                             |   2 +-
+ include/linux/srcu.h                               |  83 +++-
+ include/linux/srcutiny.h                           |  18 +-
+ include/linux/srcutree.h                           |  19 +-
+ kernel/rcu/Kconfig                                 |   6 +
+ kernel/rcu/rcu.h                                   |  14 +
+ kernel/rcu/rcutorture.c                            | 208 ++++++++-
+ kernel/rcu/srcutiny.c                              | 182 +++++++-
+ kernel/rcu/srcutree.c                              | 464 +++++++++++++++++++--
+ kernel/rcu/tasks.h                                 |  15 +-
+ kernel/rcu/tiny.c                                  | 127 +++++-
+ kernel/rcu/tree.c                                  | 164 +++++++-
+ kernel/rcu/tree.h                                  |   6 +
+ kernel/rcu/tree_exp.h                              |   2 +-
+ kernel/rcu/tree_nocb.h                             |   3 +-
+ kernel/rcu/tree_stall.h                            |  35 +-
+ .../testing/selftests/bpf/prog_tests/rcu_reentry.c |  93 +++++
+ tools/testing/selftests/bpf/progs/rcu_reentry.c    |  51 +++
+ .../testing/selftests/rcutorture/bin/kvm-remote.sh |  22 +-
+ tools/testing/selftests/rcutorture/bin/torture.sh  |  24 ++
+ 22 files changed, 1430 insertions(+), 153 deletions(-)
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/rcu_reentry.c
+ create mode 100644 tools/testing/selftests/bpf/progs/rcu_reentry.c
+Merging paulmck/non-rcu/next (c87605b21fdd4 Merge branches 'csd-lock.2026.09.03a', 'hazptr.2026.09.18a', 'nmi.2026.09.17a' and 'usb-mtu3.2026.09.29a' into HEAD)
+$ git merge -m Merge branch 'non-rcu/next' of https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git paulmck/non-rcu/next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging init/main.c
+Auto-merging kernel/Makefile
+Auto-merging kernel/sched/core.c
+Auto-merging lib/Kconfig.debug
+Auto-merging tools/testing/selftests/rcutorture/bin/torture.sh
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt    |  96 +++
+ arch/x86/kernel/nmi.c                              |  21 +-
+ drivers/usb/mtu3/mtu3_trace.h                      |   4 +-
+ include/linux/hazptr.h                             | 325 +++++++
+ include/linux/torture.h                            |   6 +-
+ init/main.c                                        |   2 +
+ kernel/Makefile                                    |   2 +-
+ kernel/hazptr.c                                    | 284 +++++++
+ kernel/rcu/Kconfig.debug                           |  26 +
+ kernel/rcu/Makefile                                |   1 +
+ kernel/rcu/hazptrtorture.c                         | 946 +++++++++++++++++++++
+ kernel/rcu/refscale.c                              |  43 +
+ kernel/rcu/update.c                                |   3 +-
+ kernel/sched/core.c                                |   2 +
+ kernel/smp.c                                       |  77 +-
+ kernel/torture.c                                   |  38 +-
+ lib/Kconfig.debug                                  |  12 +
+ lib/Makefile                                       |   1 +
+ lib/test_csd_lock.c                                | 173 ++++
+ tools/testing/selftests/rcutorture/bin/kvm.sh      |  10 +-
+ tools/testing/selftests/rcutorture/bin/torture.sh  |  26 +-
+ .../selftests/rcutorture/configs/hazptr/CFLIST     |   2 +
+ .../selftests/rcutorture/configs/hazptr/CFcommon   |   2 +
+ .../selftests/rcutorture/configs/hazptr/NOPREEMPT  |  19 +
+ .../rcutorture/configs/hazptr/NOPREEMPT.boot       |   1 +
+ .../selftests/rcutorture/configs/hazptr/PREEMPT    |  16 +
+ .../rcutorture/configs/hazptr/ver_functions.sh     |  40 +
+ 27 files changed, 2140 insertions(+), 38 deletions(-)
+ create mode 100644 include/linux/hazptr.h
+ create mode 100644 kernel/hazptr.c
+ create mode 100644 kernel/rcu/hazptrtorture.c
+ create mode 100644 lib/test_csd_lock.c
+ create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/CFLIST
+ create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/CFcommon
+ create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT
+ create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot
+ create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT
+ create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh
+Merging kvm/next (d4b7fb647204f Merge branch 'kvm-xen-longmode' into HEAD)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm/next
+Auto-merging arch/x86/include/asm/kvm_host.h
+Auto-merging include/linux/kvm_host.h
+Auto-merging virt/kvm/kvm_main.c
+Merge made by the 'ort' strategy.
+ arch/x86/include/asm/kvm_host.h |   3 +-
+ arch/x86/kvm/xen.c              | 212 ++++++++++++++++++++++++----------------
+ arch/x86/kvm/xen.h              |   5 +
+ include/linux/kvm_host.h        |   2 +
+ virt/kvm/kvm_main.c             |  10 ++
+ virt/kvm/pfncache.c             |  18 ++--
+ 6 files changed, 157 insertions(+), 93 deletions(-)
+$ git am -3 ../patches/kvm-x86-static-cpu-has
+Applying: Signed-off-by: Mark Brown <broonie@kernel.org>
+Using index info to reconstruct a base tree...
+M	arch/x86/kvm/msrs.c
+Falling back to patching base and 3-way merge...
+Auto-merging arch/x86/kvm/msrs.c
+No changes -- Patch already applied.
+Merging kvm-arm/next (8c00199d322b9 Merge branch kvm-arm64/immutable-idregs-7.4 into kvmarm-master/next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm/next
+Auto-merging arch/arm64/include/asm/sysreg.h
+Auto-merging arch/arm64/kernel/pi/Makefile
+Auto-merging arch/arm64/kvm/arm.c
+Auto-merging arch/arm64/kvm/hyp/include/nvhe/pkvm.h
+Auto-merging arch/arm64/kvm/hyp/nvhe/hyp-main.c
+Auto-merging arch/arm64/kvm/hyp/nvhe/pkvm.c
+Auto-merging arch/arm64/kvm/mmu.c
+Auto-merging arch/arm64/kvm/vgic/vgic-init.c
+Auto-merging arch/arm64/kvm/vgic/vgic-its.c
+Auto-merging arch/arm64/kvm/vgic/vgic.c
+Auto-merging arch/s390/kvm/s390/s390.c
+Auto-merging arch/x86/kvm/mmu/mmu.c
+Auto-merging arch/x86/kvm/vmx/tdx.c
+Auto-merging drivers/irqchip/irq-gic-v5.c
+Auto-merging include/linux/kvm_host.h
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Auto-merging virt/kvm/kvm_main.c
+Merge made by the 'ort' strategy.
+ Documentation/virt/kvm/api.rst                     |   40 +-
+ Documentation/virt/kvm/devices/arm-vgic-v5.rst     |  271 ++-
+ Documentation/virt/kvm/devices/vcpu.rst            |    2 +-
+ arch/arm64/include/asm/esr.h                       |   29 +
+ arch/arm64/include/asm/kvm_asm.h                   |    3 +
+ arch/arm64/include/asm/kvm_emulate.h               |   60 +-
+ arch/arm64/include/asm/kvm_hcall.h                 |  247 +++
+ arch/arm64/include/asm/kvm_host.h                  |   81 +-
+ arch/arm64/include/asm/kvm_hyp.h                   |    3 +
+ arch/arm64/include/asm/kvm_mmu.h                   |   16 +-
+ arch/arm64/include/asm/kvm_nested.h                |    7 +
+ arch/arm64/include/asm/kvm_pgtable.h               |    7 +-
+ arch/arm64/include/asm/kvm_pkvm.h                  |    2 +-
+ arch/arm64/include/asm/stacktrace/nvhe.h           |   10 +-
+ arch/arm64/include/asm/sysreg.h                    |   13 +-
+ arch/arm64/include/uapi/asm/kvm.h                  |   15 +
+ arch/arm64/kernel/pi/Makefile                      |    1 +
+ arch/arm64/kernel/vmlinux.lds.S                    |    2 +-
+ arch/arm64/kvm/Kconfig                             |    1 +
+ arch/arm64/kvm/Makefile                            |    3 +-
+ arch/arm64/kvm/arm.c                               |   13 +-
+ arch/arm64/kvm/hyp/entry.S                         |    2 +-
+ arch/arm64/kvm/hyp/include/nvhe/pkvm.h             |    7 +-
+ arch/arm64/kvm/hyp/include/nvhe/trace.h            |    5 +-
+ arch/arm64/kvm/hyp/nvhe/Makefile                   |    1 +
+ arch/arm64/kvm/hyp/nvhe/events.c                   |    2 +
+ arch/arm64/kvm/hyp/nvhe/ffa.c                      |  220 ++-
+ arch/arm64/kvm/hyp/nvhe/hyp-main.c                 |  485 +++--
+ arch/arm64/kvm/hyp/nvhe/hyp.lds.S                  |    1 -
+ arch/arm64/kvm/hyp/nvhe/mem_protect.c              |   10 +-
+ arch/arm64/kvm/hyp/nvhe/mm.c                       |    2 +-
+ arch/arm64/kvm/hyp/nvhe/pkvm.c                     |   12 +-
+ arch/arm64/kvm/hyp/nvhe/stacktrace.c               |    3 +-
+ arch/arm64/kvm/hyp/nvhe/trace.c                    |    6 +-
+ arch/arm64/kvm/hyp/pgtable.c                       |    5 +-
+ arch/arm64/kvm/hyp/vgic-v5-sr.c                    |   45 +
+ arch/arm64/kvm/hyp_trace.c                         |    2 +-
+ arch/arm64/kvm/hypercalls.c                        |    4 +
+ arch/arm64/kvm/mmu.c                               |  420 ++++-
+ arch/arm64/kvm/nested.c                            |  151 +-
+ arch/arm64/kvm/pauth.c                             |    2 +-
+ arch/arm64/kvm/ptdump.c                            |   79 +-
+ arch/arm64/kvm/stacktrace.c                        |    3 -
+ arch/arm64/kvm/sys_regs.c                          |   63 +-
+ arch/arm64/kvm/sys_regs.h                          |    2 +-
+ arch/arm64/kvm/vgic-sys-reg-v5.c                   |  519 ++++++
+ arch/arm64/kvm/vgic/vgic-init.c                    |  160 +-
+ arch/arm64/kvm/vgic/vgic-irqfd.c                   |   18 +-
+ arch/arm64/kvm/vgic/vgic-irs-v5.c                  | 1217 +++++++++++++
+ arch/arm64/kvm/vgic/vgic-its.c                     |    6 +-
+ arch/arm64/kvm/vgic/vgic-kvm-device.c              |  272 ++-
+ arch/arm64/kvm/vgic/vgic-mmio.c                    |    6 +
+ arch/arm64/kvm/vgic/vgic-mmio.h                    |    2 +
+ arch/arm64/kvm/vgic/vgic-v5-tables.c               | 1872 ++++++++++++++++++++
+ arch/arm64/kvm/vgic/vgic-v5-tables.h               |  140 ++
+ arch/arm64/kvm/vgic/vgic-v5.c                      | 1152 +++++++++++-
+ arch/arm64/kvm/vgic/vgic.c                         |   39 +-
+ arch/arm64/kvm/vgic/vgic.h                         |   21 +
+ arch/s390/kvm/s390/s390.c                          |   11 +-
+ arch/x86/kvm/mmu/mmu.c                             |   11 +-
+ arch/x86/kvm/vmx/tdx.c                             |    2 +-
+ drivers/irqchip/irq-gic-v5-irs.c                   |   19 +-
+ drivers/irqchip/irq-gic-v5.c                       |  111 +-
+ include/kvm/arm_vgic.h                             |  213 ++-
+ include/linux/irqchip/arm-gic-v5.h                 |  254 ++-
+ include/linux/irqchip/arm-vgic-info.h              |    5 +
+ include/linux/kvm_host.h                           |    1 +
+ include/linux/trace_remote_event.h                 |    2 +
+ tools/arch/arm64/include/uapi/asm/kvm.h            |   15 +
+ tools/testing/selftests/kvm/Makefile.kvm           |    2 +
+ .../testing/selftests/kvm/arm64/debug-exceptions.c |    2 +-
+ .../testing/selftests/kvm/arm64/external_aborts.c  |    3 +-
+ tools/testing/selftests/kvm/arm64/hypercalls.c     |   31 +-
+ .../selftests/kvm/arm64/nv_pre_fault_memory_test.c |  158 ++
+ tools/testing/selftests/kvm/arm64/sea_to_user.c    |   75 +-
+ tools/testing/selftests/kvm/arm64/set_id_regs.c    |  106 +-
+ tools/testing/selftests/kvm/arm64/vgic_v5.c        | 1849 ++++++++++++++++++-
+ tools/testing/selftests/kvm/include/arm64/gic_v5.h |  105 ++
+ tools/testing/selftests/kvm/include/test_util.h    |    1 +
+ tools/testing/selftests/kvm/lib/test_util.c        |   15 +
+ .../testing/selftests/kvm/pre_fault_memory_test.c  |  150 +-
+ virt/kvm/kvm_main.c                                |   10 +-
+ 82 files changed, 10179 insertions(+), 754 deletions(-)
+ create mode 100644 arch/arm64/include/asm/kvm_hcall.h
+ create mode 100644 arch/arm64/kvm/vgic-sys-reg-v5.c
+ create mode 100644 arch/arm64/kvm/vgic/vgic-irs-v5.c
+ create mode 100644 arch/arm64/kvm/vgic/vgic-v5-tables.c
+ create mode 100644 arch/arm64/kvm/vgic/vgic-v5-tables.h
+ create mode 100644 tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c
+Merging kvms390/next (044ae0767d8cc KVM: s390: Kick PV cpus at the right time for service irqs)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390/next
+Auto-merging arch/s390/kernel/uv.c
+Auto-merging arch/s390/kvm/s390/interrupt.c
+Auto-merging arch/s390/kvm/s390/s390.c
+Auto-merging arch/s390/kvm/s390/s390.h
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Merge made by the 'ort' strategy.
+ Documentation/virt/kvm/devices/vm.rst            |  18 ++++
+ arch/s390/kernel/uv.c                            |  41 +++----
+ arch/s390/kvm/s390/intercept.c                   |  32 +++---
+ arch/s390/kvm/s390/interrupt.c                   | 130 ++++++++++++++++++-----
+ arch/s390/kvm/s390/priv.c                        |  19 ++--
+ arch/s390/kvm/s390/s390.c                        |  12 +--
+ arch/s390/kvm/s390/s390.h                        |   1 +
+ tools/testing/selftests/kvm/Makefile.kvm         |   1 +
+ tools/testing/selftests/kvm/s390/irq_injection.c |  74 +++++++++++++
+ 9 files changed, 257 insertions(+), 71 deletions(-)
+ create mode 100644 tools/testing/selftests/kvm/s390/irq_injection.c
+Merging kvm-ppc/topic/ppc-kvm (93f51579e7df2 Linux 7.3-rc4)
+$ git merge -m Merge branch 'topic/ppc-kvm' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git kvm-ppc/topic/ppc-kvm
+Already up to date.
+Merging kvm-riscv/riscv_kvm_next (41e81f7e3ef96 RISC-V: KVM: Fix HSM hart status error propagation)
+$ git merge -m Merge branch 'riscv_kvm_next' of https://github.com/kvm-riscv/linux.git kvm-riscv/riscv_kvm_next
+Already up to date.
+Merging kvm-x86/next (b378201ccd528 Merge branch 'vmx')
+  9b2f8146fcef9 ("KVM: selftests: Extend nested x2APIC test to validate disabling x2APIC virt")
+  a4bab1c12fd52 ("KVM: selftests: Extend nested x2APIC test to validate using eVMCS for vmcs12")
+$ git merge -m Merge branch 'next' of https://github.com/kvm-x86/linux.git kvm-x86/next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging Documentation/virt/kvm/api.rst
+Auto-merging arch/arm64/kvm/mmu.c
+Auto-merging arch/arm64/kvm/nested.c
+Auto-merging arch/x86/include/asm/cpufeatures.h
+Auto-merging arch/x86/include/asm/kvm-x86-ops.h
+Auto-merging arch/x86/include/asm/kvm_host.h
+Auto-merging arch/x86/kernel/cpu/scattered.c
+Auto-merging arch/x86/kvm/mmu/mmu.c
+Auto-merging arch/x86/kvm/svm/sev.c
+Auto-merging arch/x86/kvm/vmx/tdx.c
+Auto-merging arch/x86/kvm/vmx/vmx.c
+Auto-merging arch/x86/kvm/vmx/vmx.h
+Auto-merging include/linux/kvm_host.h
+Auto-merging tools/testing/selftests/kvm/Makefile.kvm
+Auto-merging tools/testing/selftests/kvm/arm64/hypercalls.c
+Auto-merging tools/testing/selftests/kvm/include/test_util.h
+Auto-merging tools/testing/selftests/kvm/lib/test_util.c
+Auto-merging tools/testing/selftests/kvm/x86/nested_x2apic_test.c
+CONFLICT (content): Merge conflict in tools/testing/selftests/kvm/x86/nested_x2apic_test.c
+Auto-merging virt/kvm/guest_memfd.c
+Auto-merging virt/kvm/kvm_main.c
+Recorded preimage for 'tools/testing/selftests/kvm/x86/nested_x2apic_test.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'tools/testing/selftests/kvm/x86/nested_x2apic_test.c'.
+[master 95683d8e36322] Merge branch 'next' of https://github.com/kvm-x86/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/kernel-parameters.txt    |  26 +
+ Documentation/virt/kvm/api.rst                     | 116 +++-
+ .../virt/kvm/x86/amd-memory-encryption.rst         |  22 +-
+ Documentation/virt/kvm/x86/hypercalls.rst          |   2 +-
+ Documentation/virt/kvm/x86/intel-tdx.rst           |   4 +
+ arch/arm64/kvm/mmu.c                               |   4 +-
+ arch/arm64/kvm/nested.c                            |   4 +-
+ arch/x86/include/asm/cpufeatures.h                 |   1 +
+ arch/x86/include/asm/kvm-x86-ops.h                 |   3 +-
+ arch/x86/include/asm/kvm_host.h                    |  32 +-
+ arch/x86/include/asm/svm.h                         |  18 +-
+ arch/x86/include/asm/vmx.h                         |   8 +
+ arch/x86/include/uapi/asm/svm.h                    |   2 +
+ arch/x86/kernel/cpu/scattered.c                    |   1 +
+ arch/x86/kvm/Kconfig                               |  15 +-
+ arch/x86/kvm/cpuid.c                               |  17 +-
+ arch/x86/kvm/cpuid.h                               |   1 -
+ arch/x86/kvm/hyperv.c                              |   9 +-
+ arch/x86/kvm/kvm_emulate.h                         |   4 +-
+ arch/x86/kvm/lapic.c                               |   4 +-
+ arch/x86/kvm/lapic.h                               |   2 +-
+ arch/x86/kvm/mmu/mmu.c                             | 183 ++++--
+ arch/x86/kvm/mmu/paging_tmpl.h                     |  21 +-
+ arch/x86/kvm/msrs.c                                |   3 +-
+ arch/x86/kvm/pmu.c                                 |  14 +-
+ arch/x86/kvm/regs.c                                |   8 +-
+ arch/x86/kvm/reverse_cpuid.h                       |   2 +-
+ arch/x86/kvm/svm/avic.c                            |  22 +-
+ arch/x86/kvm/svm/nested.c                          |  20 +
+ arch/x86/kvm/svm/sev.c                             | 112 ++--
+ arch/x86/kvm/svm/svm.c                             | 138 ++++-
+ arch/x86/kvm/svm/svm.h                             |   2 +
+ arch/x86/kvm/vmx/common.h                          |  90 ++-
+ arch/x86/kvm/vmx/main.c                            | 153 ++++-
+ arch/x86/kvm/vmx/nested.c                          |  60 +-
+ arch/x86/kvm/vmx/posted_intr.c                     |  24 +-
+ arch/x86/kvm/vmx/posted_intr.h                     |   8 +-
+ arch/x86/kvm/vmx/tdx.c                             | 141 +++--
+ arch/x86/kvm/vmx/vmx.c                             | 302 +++-------
+ arch/x86/kvm/vmx/vmx.h                             |  46 +-
+ arch/x86/kvm/vmx/x86_ops.h                         |   6 +-
+ arch/x86/kvm/x86.c                                 | 577 +++++++++---------
+ arch/x86/kvm/x86.h                                 |  17 +-
+ include/linux/kvm_host.h                           |  89 +--
+ include/trace/events/kvm.h                         |   6 +-
+ include/uapi/linux/kvm.h                           |  16 +
+ tools/testing/selftests/kvm/Makefile.kvm           |   2 +-
+ tools/testing/selftests/kvm/arm64/hypercalls.c     |   2 +-
+ .../testing/selftests/kvm/arm64/page_fault_test.c  |  18 +-
+ .../testing/selftests/kvm/arm64/vgic_lpi_stress.c  |  21 +-
+ tools/testing/selftests/kvm/demand_paging_test.c   |   3 +-
+ tools/testing/selftests/kvm/guest_memfd_test.c     |   2 +-
+ tools/testing/selftests/kvm/include/kvm_util.h     | 245 +++++++-
+ tools/testing/selftests/kvm/include/numaif.h       |  43 ++
+ tools/testing/selftests/kvm/include/test_util.h    |  36 +-
+ tools/testing/selftests/kvm/include/x86/evmcs.h    |  78 +--
+ .../testing/selftests/kvm/include/x86/processor.h  |  13 +-
+ tools/testing/selftests/kvm/include/x86/smm.h      |   2 +-
+ tools/testing/selftests/kvm/include/x86/vmx.h      | 186 +++---
+ tools/testing/selftests/kvm/lib/arm64/processor.c  |   4 +-
+ tools/testing/selftests/kvm/lib/elf.c              |  48 +-
+ tools/testing/selftests/kvm/lib/io.c               | 157 -----
+ tools/testing/selftests/kvm/lib/kvm_util.c         | 350 +++++++----
+ .../selftests/kvm/lib/loongarch/processor.c        |   5 +-
+ tools/testing/selftests/kvm/lib/riscv/processor.c  |   4 +-
+ tools/testing/selftests/kvm/lib/s390/processor.c   |   7 +-
+ tools/testing/selftests/kvm/lib/test_util.c        |   7 -
+ tools/testing/selftests/kvm/lib/userfaultfd_util.c |   6 +-
+ tools/testing/selftests/kvm/lib/x86/memstress.c    |   8 +-
+ tools/testing/selftests/kvm/lib/x86/processor.c    |  23 +-
+ tools/testing/selftests/kvm/lib/x86/vmx.c          | 159 +++--
+ tools/testing/selftests/kvm/memslot_perf_test.c    |   5 +-
+ tools/testing/selftests/kvm/s390/cmma_test.c       |  19 +-
+ tools/testing/selftests/kvm/s390/irq_routing.c     |   2 +-
+ .../testing/selftests/kvm/set_memory_region_test.c |   8 +-
+ tools/testing/selftests/kvm/steal_time.c           |  20 +-
+ tools/testing/selftests/kvm/x86/amx_test.c         |   2 +-
+ tools/testing/selftests/kvm/x86/aperfmperf_test.c  |  10 +-
+ tools/testing/selftests/kvm/x86/cpuid_test.c       |   2 +-
+ .../selftests/kvm/x86/evmcs_smm_controls_test.c    |   6 +-
+ .../testing/selftests/kvm/x86/feature_msrs_test.c  |   2 +-
+ .../testing/selftests/kvm/x86/fix_hypercall_test.c |   2 +-
+ .../kvm/x86/guest_memfd_conversions_test.c         | 512 ++++++++++++++++
+ tools/testing/selftests/kvm/x86/hyperv_clock.c     |   2 +-
+ tools/testing/selftests/kvm/x86/hyperv_evmcs.c     |  87 +--
+ tools/testing/selftests/kvm/x86/hyperv_svm_test.c  |   2 +-
+ tools/testing/selftests/kvm/x86/kvm_buslock_test.c |   8 +-
+ .../selftests/kvm/x86/nested_close_kvm_test.c      |   6 +-
+ .../selftests/kvm/x86/nested_dirty_log_test.c      |   8 +-
+ .../selftests/kvm/x86/nested_emulation_test.c      |  23 +-
+ .../selftests/kvm/x86/nested_exceptions_test.c     |  33 +-
+ .../selftests/kvm/x86/nested_invalid_cr3_test.c    |  14 +-
+ .../selftests/kvm/x86/nested_tdp_fault_test.c      |  16 +-
+ .../selftests/kvm/x86/nested_tsc_adjust_test.c     |  10 +-
+ .../selftests/kvm/x86/nested_tsc_scaling_test.c    |  12 +-
+ .../testing/selftests/kvm/x86/nested_x2apic_test.c |   2 +-
+ .../testing/selftests/kvm/x86/nx_huge_pages_test.c |   3 +-
+ .../kvm/x86/private_mem_conversions_test.c         |  66 ++-
+ .../selftests/kvm/x86/private_mem_kvm_exits_test.c |  36 +-
+ .../kvm/x86/save_restore_pf_stress_test.c          |  14 +-
+ tools/testing/selftests/kvm/x86/set_boot_cpu_id.c  |   2 +-
+ tools/testing/selftests/kvm/x86/set_sregs_test.c   |  11 +
+ tools/testing/selftests/kvm/x86/sev_dbg_test.c     |   6 +-
+ tools/testing/selftests/kvm/x86/sev_smoke_test.c   |   3 +-
+ .../kvm/x86/smaller_maxphyaddr_emulation_test.c    |   9 +-
+ tools/testing/selftests/kvm/x86/smm_test.c         |   4 +-
+ tools/testing/selftests/kvm/x86/state_test.c       |  74 +--
+ .../selftests/kvm/x86/triple_fault_event_test.c    |   8 +-
+ tools/testing/selftests/kvm/x86/tsc_msrs_test.c    |   2 +-
+ .../selftests/kvm/x86/vmx_apic_access_test.c       |  22 +-
+ .../selftests/kvm/x86/vmx_apicv_updates_test.c     |  16 +-
+ .../x86/vmx_exception_with_invalid_guest_state.c   |   2 +-
+ .../kvm/x86/vmx_invalid_nested_guest_state.c       |  12 +-
+ .../selftests/kvm/x86/vmx_nested_la57_state_test.c |  12 +-
+ .../selftests/kvm/x86/vmx_preemption_timer_test.c  |  28 +-
+ tools/testing/selftests/kvm/x86/xapic_ipi_test.c   |  78 +--
+ virt/kvm/Kconfig                                   |   3 -
+ virt/kvm/guest_memfd.c                             | 649 +++++++++++++++++----
+ virt/kvm/guest_memfd.h                             |  19 +-
+ virt/kvm/kvm_main.c                                | 159 +++--
+ 120 files changed, 3827 insertions(+), 2008 deletions(-)
+ delete mode 100644 tools/testing/selftests/kvm/lib/io.c
+ create mode 100644 tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c
+$ git am -3 ../patches/0001-KVM-selftests-Fix-up-semantic-changes.patch
+Applying: KVM: selftests: Fix up semantic changes
+Using index info to reconstruct a base tree...
+M	tools/testing/selftests/kvm/lib/kvm_util.c
+Falling back to patching base and 3-way merge...
+Auto-merging tools/testing/selftests/kvm/lib/kvm_util.c
+No changes -- Patch already applied.
+$ git am -3 ../patches/0001-KVM-selftests-Fix-up-phy_pages_alloc-API-rework-inte.patch
+Applying: KVM: selftests: Fix up phy_pages_alloc() API rework interaction
+$ git reset HEAD^
+Unstaged changes after reset:
+M	tools/testing/selftests/kvm/arm64/vgic_its_save.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 546bb0c1a1012] Merge branch 'next' of https://github.com/kvm-x86/linux.git
+ Date: Wed Sep 30 13:35:02 2026 +0100
+Merging xen-tip/linux-next (93f51579e7df2 Linux 7.3-rc4)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git xen-tip/linux-next
+Already up to date.
+Merging percpu/for-next (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git percpu/for-next
+Already up to date.
+Merging workqueues/for-next (fee1265a72572 Merge branch 'for-7.3-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git workqueues/for-next
+Auto-merging kernel/workqueue.c
+Merge made by the 'ort' strategy.
+ include/trace/events/workqueue.h | 146 ++++++++++
+ kernel/workqueue.c               | 603 ++++++++++++++++++++++++++-------------
+ 2 files changed, 547 insertions(+), 202 deletions(-)
+Merging sched-ext/for-next (91a186b9e599f Merge branch 'for-7.4' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git sched-ext/for-next
+Auto-merging init/Kconfig
+Auto-merging kernel/sched/ext/cid.c
+Auto-merging kernel/sched/ext/ext.c
+Auto-merging kernel/sched/ext/sub.c
+Auto-merging kernel/sched/sched.h
+Auto-merging tools/sched_ext/include/scx/common.bpf.h
+Merge made by the 'ort' strategy.
+ Documentation/scheduler/sched-ext.rst              |  13 +
+ include/linux/sched/ext.h                          |  38 +-
+ init/Kconfig                                       |   2 -
+ kernel/sched/ext/cid.c                             |  67 +-
+ kernel/sched/ext/cid.h                             |  11 +-
+ kernel/sched/ext/ext.c                             | 896 ++++++++++++++++----
+ kernel/sched/ext/ext.h                             |  32 +-
+ kernel/sched/ext/idle.c                            |  14 +-
+ kernel/sched/ext/inlines.h                         |   8 +-
+ kernel/sched/ext/internal.h                        | 105 ++-
+ kernel/sched/ext/sub.c                             | 110 +--
+ kernel/sched/ext/sub.h                             |  38 +-
+ kernel/sched/ext/types.h                           |  28 +-
+ kernel/sched/sched.h                               |  12 +-
+ kernel/sched/syscalls.c                            |   2 +
+ tools/sched_ext/Makefile                           |   2 +-
+ tools/sched_ext/include/scx/common.bpf.h           |   3 +-
+ tools/sched_ext/include/scx/compat.bpf.h           |  14 +
+ tools/sched_ext/include/scx/compat.h               |  17 +-
+ tools/sched_ext/include/scx/enum_defs.autogen.h    |   7 +
+ tools/sched_ext/include/scx/enums.autogen.bpf.h    |  12 +
+ tools/sched_ext/include/scx/enums.autogen.h        |   4 +
+ tools/sched_ext/include/scx/enums_abi.autogen.h    |   6 +-
+ tools/sched_ext/scx_flatcg.bpf.c                   |   2 +-
+ tools/sched_ext/scx_qmap.bpf.c                     |  79 +-
+ tools/sched_ext/scx_qmap.c                         |  13 +-
+ tools/sched_ext/scx_qmap.h                         |   1 +
+ tools/testing/selftests/sched_ext/.gitignore       |   4 +
+ tools/testing/selftests/sched_ext/Makefile         |   7 +-
+ .../testing/selftests/sched_ext/allowed_cpus.bpf.c |  24 +-
+ tools/testing/selftests/sched_ext/allowed_cpus.c   | 123 ++-
+ tools/testing/selftests/sched_ext/config           |   2 +
+ tools/testing/selftests/sched_ext/create_dsq.bpf.c |  35 +
+ tools/testing/selftests/sched_ext/create_dsq.c     |   4 +-
+ .../testing/selftests/sched_ext/enq_blocked.bpf.c  | 116 +++
+ tools/testing/selftests/sched_ext/enq_blocked.c    | 917 +++++++++++++++++++++
+ tools/testing/selftests/sched_ext/enq_blocked.h    |  28 +
+ tools/testing/selftests/sched_ext/hotplug.c        |  34 +-
+ tools/testing/selftests/sched_ext/kick.bpf.c       | 164 ++++
+ tools/testing/selftests/sched_ext/kick.c           | 519 ++++++++++++
+ tools/testing/selftests/sched_ext/kick_test.h      |  29 +
+ tools/testing/selftests/sched_ext/nohz_tick.bpf.c  |  57 +-
+ tools/testing/selftests/sched_ext/nohz_tick.c      | 207 ++++-
+ tools/testing/selftests/sched_ext/nohz_tick_test.h |  13 +
+ tools/testing/selftests/sched_ext/rt_stall.c       |  80 +-
+ tools/testing/selftests/sched_ext/runner.c         |   2 +-
+ .../selftests/sched_ext/test_modules/Makefile      |  13 +
+ .../sched_ext/test_modules/scx_enq_blocked_test.c  | 195 +++++
+ tools/testing/selftests/sched_ext/util.c           | 143 ++++
+ tools/testing/selftests/sched_ext/util.h           |  12 +
+ 50 files changed, 3836 insertions(+), 428 deletions(-)
+ create mode 100644 tools/testing/selftests/sched_ext/enq_blocked.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/enq_blocked.c
+ create mode 100644 tools/testing/selftests/sched_ext/enq_blocked.h
+ create mode 100644 tools/testing/selftests/sched_ext/kick.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/kick.c
+ create mode 100644 tools/testing/selftests/sched_ext/kick_test.h
+ create mode 100644 tools/testing/selftests/sched_ext/nohz_tick_test.h
+ create mode 100644 tools/testing/selftests/sched_ext/test_modules/Makefile
+ create mode 100644 tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c
+Merging drivers-x86/for-next (fe5030c8cc715 Merge branch 'platform-drivers-x86-intel-pmt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/platform/x86/asus-laptop.c
+Auto-merging drivers/platform/x86/hp/hp-wmi.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/debugfs-tpmi             |   8 +-
+ .../admin-guide/laptops/thinkpad-acpi.rst          |  26 +-
+ Documentation/wmi/devices/acer-wmi-battery.rst     |  70 ++++
+ Documentation/wmi/devices/bitland-mifs-wmi.rst     |  64 ++-
+ MAINTAINERS                                        |   3 +-
+ drivers/platform/arm64/acer-aspire1-ec.c           |   2 +-
+ drivers/platform/arm64/huawei-gaokun-ec.c          |   2 +-
+ drivers/platform/arm64/lenovo-thinkpad-t14s.c      |  10 +-
+ drivers/platform/arm64/lenovo-yoga-c630.c          |   2 +-
+ drivers/platform/mellanox/mlxbf-tmfifo.c           |   1 -
+ drivers/platform/mellanox/mlxreg-hotplug.c         |   4 +-
+ .../platform/surface/surface_aggregator_registry.c |   5 +-
+ drivers/platform/x86/Kconfig                       |  12 +
+ drivers/platform/x86/Makefile                      |   1 +
+ drivers/platform/x86/acer-wmi-battery.c            | 450 +++++++++++++++++++++
+ drivers/platform/x86/acer-wmi.c                    |  56 ++-
+ drivers/platform/x86/amd/pmc/pmc-quirks.c          |   9 +
+ drivers/platform/x86/amd/pmc/pmc.c                 |  15 +-
+ drivers/platform/x86/amd/pmf/core.c                |   1 +
+ drivers/platform/x86/amd/pmf/sps.c                 |  10 +-
+ drivers/platform/x86/amd/pmf/util.c                |   1 +
+ drivers/platform/x86/asus-armoury.c                |  29 +-
+ drivers/platform/x86/asus-armoury.h                |  41 ++
+ drivers/platform/x86/asus-laptop.c                 |   2 +-
+ drivers/platform/x86/asus-nb-wmi.c                 |  15 +-
+ drivers/platform/x86/asus-tf103c-dock.c            |   2 +-
+ drivers/platform/x86/asus-wmi.c                    |  97 +++--
+ drivers/platform/x86/asus-wmi.h                    |   1 -
+ drivers/platform/x86/bitland-mifs-wmi.c            | 146 ++++---
+ drivers/platform/x86/dell/alienware-wmi-wmax.c     |   8 +
+ drivers/platform/x86/dell/dell-dw5826e-reset.c     |  30 +-
+ drivers/platform/x86/dell/dell-wmi-aio.c           | 200 ++++-----
+ drivers/platform/x86/hp/hp-bioscfg/bioscfg.c       |   2 +-
+ drivers/platform/x86/hp/hp-wmi.c                   |  27 +-
+ drivers/platform/x86/hp/hp_accel.c                 |   2 +-
+ drivers/platform/x86/huawei-wmi.c                  |   1 +
+ drivers/platform/x86/intel/bxtwc_tmu.c             |   5 +-
+ drivers/platform/x86/intel/bytcrc_pwrsrc.c         |   2 +-
+ drivers/platform/x86/intel/crystal_cove_charger.c  |   2 +-
+ drivers/platform/x86/intel/int0002_vgpio.c         |   4 +-
+ drivers/platform/x86/intel/pmc/core.c              |   1 +
+ drivers/platform/x86/intel/pmt/class.c             |  35 +-
+ drivers/platform/x86/intel/pmt/class.h             |   5 +-
+ drivers/platform/x86/intel/pmt/crashlog.c          | 235 +++++++----
+ drivers/platform/x86/intel/pmt/discovery.c         |   2 +-
+ drivers/platform/x86/intel/pmt/telemetry.c         |   3 +
+ drivers/platform/x86/intel/punit_ipc.c             |   4 +-
+ .../x86/intel/speed_select_if/isst_if_common.c     |  57 ++-
+ .../x86/intel/uncore-frequency/uncore-frequency.c  |   1 +
+ drivers/platform/x86/intel/vsec.c                  |  19 +-
+ drivers/platform/x86/intel/vsec_tpmi.c             |   2 +-
+ drivers/platform/x86/lenovo/ideapad-laptop.c       |   4 +
+ drivers/platform/x86/lenovo/think-lmi.c            |   2 +-
+ drivers/platform/x86/lenovo/thinkpad_acpi.c        | 273 ++++++++-----
+ drivers/platform/x86/msi-laptop.c                  |   2 +-
+ drivers/platform/x86/panasonic-laptop.c            |  14 +-
+ drivers/platform/x86/samsung-laptop.c              |   2 +-
+ drivers/platform/x86/topstar-laptop.c              |   6 +-
+ drivers/platform/x86/uniwill/uniwill-acpi.c        |   2 +-
+ include/linux/intel_vsec.h                         |  14 +-
+ include/linux/platform_data/x86/asus-wmi.h         |   4 +-
+ include/linux/string_choices.h                     |   6 +
+ include/uapi/linux/amd-pmf.h                       |   3 +
+ 63 files changed, 1525 insertions(+), 539 deletions(-)
+ create mode 100644 Documentation/wmi/devices/acer-wmi-battery.rst
+ create mode 100644 drivers/platform/x86/acer-wmi-battery.c
+Merging chrome-platform/for-next (5859f97c6f404 platform/chrome: cros_ec_rpmsg: Fix repeated word 'from' in comment)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform/for-next
+Merge made by the 'ort' strategy.
+ drivers/platform/chrome/cros_ec_ishtp.c        |   2 +-
+ drivers/platform/chrome/cros_ec_proto.c        |  43 +++++--
+ drivers/platform/chrome/cros_ec_proto_test.c   | 158 ++++++++++++++++++++++++-
+ drivers/platform/chrome/cros_ec_rpmsg.c        |   2 +-
+ drivers/platform/chrome/cros_usbpd_notify.c    |   4 +-
+ include/linux/platform_data/cros_ec_commands.h |  72 +++++++++++
+ 6 files changed, 262 insertions(+), 19 deletions(-)
+Merging chrome-platform-firmware/for-firmware-next (0e30b98a54599 firmware: coreboot: Use named initializers for acpi_device_id)
+$ git merge -m Merge branch 'for-firmware-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform-firmware/for-firmware-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/firmware/Kconfig
+Auto-merging drivers/firmware/Makefile
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        | 20 +++---
+ drivers/firmware/Kconfig                           |  2 +-
+ drivers/firmware/Makefile                          |  2 +-
+ drivers/firmware/{google => coreboot}/Kconfig      | 74 ++++++++++++++++------
+ drivers/firmware/{google => coreboot}/Makefile     | 10 +--
+ drivers/firmware/{google => coreboot}/cbmem.c      |  0
+ .../firmware/{google => coreboot}/coreboot_table.c |  4 +-
+ .../firmware/{google => coreboot}/coreboot_table.h |  0
+ .../{google => coreboot}/framebuffer-coreboot.c    |  0
+ drivers/firmware/{google => coreboot}/gsmi.c       |  0
+ .../{google => coreboot}/memconsole-coreboot.c     |  0
+ .../{google => coreboot}/memconsole-x86-legacy.c   |  0
+ drivers/firmware/{google => coreboot}/memconsole.c |  0
+ drivers/firmware/{google => coreboot}/memconsole.h |  6 +-
+ drivers/firmware/{google => coreboot}/vpd.c        |  0
+ drivers/firmware/{google => coreboot}/vpd_decode.c |  0
+ drivers/firmware/{google => coreboot}/vpd_decode.h |  0
+ 17 files changed, 78 insertions(+), 40 deletions(-)
+ rename drivers/firmware/{google => coreboot}/Kconfig (64%)
+ rename drivers/firmware/{google => coreboot}/Makefile (51%)
+ rename drivers/firmware/{google => coreboot}/cbmem.c (100%)
+ rename drivers/firmware/{google => coreboot}/coreboot_table.c (99%)
+ rename drivers/firmware/{google => coreboot}/coreboot_table.h (100%)
+ rename drivers/firmware/{google => coreboot}/framebuffer-coreboot.c (100%)
+ rename drivers/firmware/{google => coreboot}/gsmi.c (100%)
+ rename drivers/firmware/{google => coreboot}/memconsole-coreboot.c (100%)
+ rename drivers/firmware/{google => coreboot}/memconsole-x86-legacy.c (100%)
+ rename drivers/firmware/{google => coreboot}/memconsole.c (100%)
+ rename drivers/firmware/{google => coreboot}/memconsole.h (82%)
+ rename drivers/firmware/{google => coreboot}/vpd.c (100%)
+ rename drivers/firmware/{google => coreboot}/vpd_decode.c (100%)
+ rename drivers/firmware/{google => coreboot}/vpd_decode.h (100%)
+Merging hsi/for-next (e81250ec6b692 hsi: omap_ssi_core: fix missing DMA mask setup for SSI controller device)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git hsi/for-next
+Already up to date.
+Merging leds-lj/for-leds-next (05b4738b0078f leds: trigger: Add led_trigger_notify_hw_control_changed() interface)
+$ git merge -m Merge branch 'for-leds-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git leds-lj/for-leds-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-class-led          |  28 ++-
+ .../ABI/testing/sysfs-class-led-trigger-netdev     |   3 +
+ .../devicetree/bindings/leds/adi,ltc3208.yaml      | 183 ++++++++++++++++
+ .../devicetree/bindings/leds/leds-cpcap.txt        |  29 ---
+ .../bindings/leds/motorola,cpcap-leds.yaml         |  42 ++++
+ Documentation/leds/leds-class.rst                  |  72 +++++++
+ Documentation/leds/leds-st1202.rst                 |   5 +
+ MAINTAINERS                                        |   8 +
+ drivers/leds/Kconfig                               |  12 ++
+ drivers/leds/Makefile                              |   1 +
+ drivers/leds/flash/leds-aat1290.c                  |   5 +-
+ drivers/leds/flash/leds-ktd2692.c                  |  10 +-
+ drivers/leds/led-class-flash.c                     |  21 +-
+ drivers/leds/led-class.c                           |  46 ++--
+ drivers/leds/led-core.c                            |   5 +-
+ drivers/leds/led-triggers.c                        | 177 ++++++++++++++-
+ drivers/leds/leds-cros_ec.c                        |   6 +
+ drivers/leds/leds-is31fl32xx.c                     |  20 +-
+ drivers/leds/leds-lm3692x.c                        |   4 +-
+ drivers/leds/leds-lp8860.c                         |   5 +-
+ drivers/leds/leds-ltc3208.c                        | 234 ++++++++++++++++++++
+ drivers/leds/leds-max77705.c                       |   3 +-
+ drivers/leds/leds-pca9532.c                        |   9 +-
+ drivers/leds/leds-pca963x.c                        |   5 +-
+ drivers/leds/leds-ss4200.c                         |   2 +-
+ drivers/leds/leds-st1202.c                         | 240 ++++++++++++++++-----
+ drivers/leds/leds-syscon.c                         |   9 +-
+ drivers/leds/leds-turris-omnia.c                   |   7 +
+ drivers/leds/leds.h                                |   8 +-
+ drivers/leds/rgb/leds-qcom-lpg.c                   |  38 ++--
+ drivers/leds/trigger/Kconfig                       |  10 +
+ drivers/leds/trigger/ledtrig-netdev.c              |  13 +-
+ include/linux/leds.h                               |  24 +++
+ 33 files changed, 1089 insertions(+), 195 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/leds/adi,ltc3208.yaml
+ delete mode 100644 Documentation/devicetree/bindings/leds/leds-cpcap.txt
+ create mode 100644 Documentation/devicetree/bindings/leds/motorola,cpcap-leds.yaml
+ create mode 100644 drivers/leds/leds-ltc3208.c
+Merging ipmi/for-next (89a312991dc6e Merge tag 'cifs-fixes-7.3-rc2' of https://git.manguebit.org/linux)
+$ git merge -m Merge branch 'for-next' of https://github.com/cminyard/linux-ipmi.git ipmi/for-next
+Already up to date.
+Merging driver-core/driver-core-next (f1850e443b0e4 docs: admin-guide: Handle TAINT_FORCED_BIND when parsing /proc/sys/kernel/tainted)
+$ git merge -m Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core/driver-core-next
+Auto-merging drivers/base/bus.c
+Auto-merging include/linux/panic.h
+Auto-merging kernel/module/main.c
+Auto-merging kernel/panic.c
+Auto-merging lib/kobject_uevent.c
+Auto-merging rust/kernel/auxiliary.rs
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/tainted-kernels.rst | 54 ++++++++++++++-------------
+ arch/powerpc/perf/hv-24x7.c                   |  2 +-
+ arch/x86/kernel/cpu/mce/core.c                |  6 +--
+ drivers/base/bus.c                            |  3 ++
+ drivers/base/core.c                           | 30 +++++++--------
+ drivers/base/node.c                           | 17 ++++-----
+ drivers/base/platform.c                       | 19 +++++-----
+ drivers/base/property.c                       |  2 +-
+ drivers/base/transport_class.c                |  2 +-
+ drivers/perf/alibaba_uncore_drw_pmu.c         |  2 +-
+ drivers/perf/arm-cci.c                        |  2 +-
+ drivers/perf/arm-ccn.c                        |  2 +-
+ drivers/perf/arm_cspmu/arm_cspmu.h            |  2 +-
+ drivers/perf/arm_dsu_pmu.c                    |  2 +-
+ drivers/perf/arm_spe_pmu.c                    |  6 +--
+ drivers/perf/cxl_pmu.c                        | 10 ++---
+ drivers/perf/fsl_imx8_ddr_perf.c              |  2 +-
+ drivers/perf/fujitsu_uncore_pmu.c             |  2 +-
+ drivers/perf/hisilicon/hisi_pcie_pmu.c        |  2 +-
+ drivers/perf/hisilicon/hisi_uncore_pmu.h      |  6 +--
+ drivers/perf/hisilicon/hns3_pmu.c             |  6 +--
+ drivers/perf/nvidia_t410_c2c_pmu.c            |  2 +-
+ drivers/perf/nvidia_t410_cmem_latency_pmu.c   | 12 +++---
+ drivers/perf/qcom_l3_pmu.c                    |  8 ++--
+ drivers/perf/starfive_starlink_pmu.c          |  2 +-
+ drivers/perf/xgene_pmu.c                      |  2 +-
+ include/linux/device.h                        | 14 +++----
+ include/linux/kernfs.h                        |  2 +-
+ include/linux/module.h                        | 10 +++++
+ include/linux/panic.h                         |  3 +-
+ include/trace/events/module.h                 |  3 +-
+ kernel/module/main.c                          | 17 +++++++--
+ kernel/panic.c                                | 10 +++--
+ lib/kobject_uevent.c                          |  2 +-
+ rust/helpers/dma.c                            |  5 +++
+ rust/kernel/auxiliary.rs                      |  8 ++++
+ rust/kernel/scatterlist.rs                    | 12 +++++-
+ tools/debugging/kernel-chktaint               |  8 ++++
+ 38 files changed, 180 insertions(+), 119 deletions(-)
+Merging usb/usb-next (d58dffe9ee2c8 USB: core: amend usb_get_from_anchor() kernel-doc)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb/usb-next
+Auto-merging Documentation/devicetree/bindings/usb/qcom,snps-dwc3.yaml
+Auto-merging arch/arm/boot/dts/st/stm32mp131.dtsi
+Auto-merging arch/arm/boot/dts/st/stm32mp151.dtsi
+Auto-merging arch/arm64/boot/dts/st/stm32mp231.dtsi
+Auto-merging arch/arm64/boot/dts/st/stm32mp251.dtsi
+Auto-merging drivers/net/usb/r8152.c
+Auto-merging drivers/usb/core/hub.c
+Auto-merging drivers/usb/dwc2/hcd.c
+Auto-merging drivers/usb/gadget/udc/lpc32xx_udc.c
+Auto-merging drivers/usb/typec/anx7411.c
+Auto-merging drivers/usb/typec/tcpm/tcpm.c
+Auto-merging drivers/usb/typec/tipd/core.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/dma/ti/ti,cppi41.yaml      |  109 +
+ .../devicetree/bindings/phy/ti,am335x-usb-phy.yaml |   49 +
+ .../devicetree/bindings/usb/am33xx-usb.txt         |  200 -
+ .../devicetree/bindings/usb/da8xx-usb.txt          |   81 -
+ .../devicetree/bindings/usb/generic-ehci.yaml      |   24 +
+ .../devicetree/bindings/usb/generic-ohci.yaml      |   24 +
+ .../devicetree/bindings/usb/gpio-sbu-mux.yaml      |    1 +
+ .../devicetree/bindings/usb/ohci-da8xx.txt         |   23 -
+ .../devicetree/bindings/usb/qcom,snps-dwc3.yaml    |   19 +
+ .../bindings/usb/ti,am335x-usb-ctrl-module.yaml    |   38 +
+ .../devicetree/bindings/usb/ti,am33xx-usb.yaml     |  103 +
+ .../devicetree/bindings/usb/ti,da830-musb.yaml     |  113 +
+ .../devicetree/bindings/usb/ti,da830-ohci.yaml     |   61 +
+ .../devicetree/bindings/usb/ti,hd3ss3220.yaml      |   11 +
+ .../devicetree/bindings/usb/ti,musb-am33xx.yaml    |  113 +
+ Documentation/usb/usbmon.rst                       |    2 +-
+ arch/arm/boot/dts/st/stm32mp131.dtsi               |    4 +-
+ arch/arm/boot/dts/st/stm32mp151.dtsi               |    4 +-
+ arch/arm64/boot/dts/st/stm32mp231.dtsi             |   64 +
+ arch/arm64/boot/dts/st/stm32mp251.dtsi             |   63 +
+ drivers/net/usb/r8152.c                            |    2 +-
+ drivers/usb/cdns3/core.c                           |    4 +-
+ drivers/usb/cdns3/drd.c                            |    4 +-
+ drivers/usb/chipidea/ci_hdrc_imx.c                 |    8 +-
+ drivers/usb/chipidea/otg_fsm.c                     |    2 +-
+ drivers/usb/chipidea/udc.c                         |    2 +-
+ drivers/usb/common/ulpi.c                          |    4 +-
+ drivers/usb/common/usb-conn-gpio.c                 |    8 +-
+ drivers/usb/core/driver.c                          |   10 +-
+ drivers/usb/core/hub.c                             |    2 +-
+ drivers/usb/core/urb.c                             |    4 +-
+ drivers/usb/core/usb.c                             |    2 +-
+ drivers/usb/dwc2/core.c                            |    2 +-
+ drivers/usb/dwc2/core.h                            |    2 +-
+ drivers/usb/dwc2/debugfs.c                         |    6 +-
+ drivers/usb/dwc2/gadget.c                          |   98 +-
+ drivers/usb/dwc2/hcd.c                             |    4 +-
+ drivers/usb/dwc2/hcd_ddma.c                        |    2 +-
+ drivers/usb/dwc3/debugfs.c                         |    6 +-
+ drivers/usb/dwc3/dwc3-google.c                     |    4 +-
+ drivers/usb/dwc3/dwc3-imx.c                        |    2 +-
+ drivers/usb/dwc3/dwc3-imx8mp.c                     |    4 +-
+ drivers/usb/dwc3/dwc3-keystone.c                   |    5 +-
+ drivers/usb/dwc3/dwc3-omap.c                       |    5 +-
+ drivers/usb/fotg210/Kconfig                        |    5 +
+ drivers/usb/fotg210/fotg210-core.c                 |  273 +-
+ drivers/usb/fotg210/fotg210-hcd.c                  | 5666 +-------------------
+ drivers/usb/fotg210/fotg210-hcd.h                  |  689 ---
+ drivers/usb/fotg210/fotg210-udc.c                  |  114 +-
+ drivers/usb/fotg210/fotg210-udc.h                  |    2 +-
+ drivers/usb/fotg210/fotg210.h                      |   26 +-
+ drivers/usb/gadget/composite.c                     |    2 +-
+ drivers/usb/gadget/function/f_mass_storage.c       |    7 +-
+ drivers/usb/gadget/udc/aspeed-vhub/core.c          |    4 +-
+ drivers/usb/gadget/udc/aspeed-vhub/dev.c           |    2 -
+ drivers/usb/gadget/udc/aspeed_udc.c                |    5 +-
+ drivers/usb/gadget/udc/at91_udc.c                  |    1 -
+ drivers/usb/gadget/udc/atmel_usba_udc.c            |    5 +-
+ drivers/usb/gadget/udc/bcm63xx_udc.c               |    1 -
+ drivers/usb/gadget/udc/fsl_udc_core.c              |    3 -
+ drivers/usb/gadget/udc/lpc32xx_udc.c               |    1 -
+ drivers/usb/gadget/udc/r8a66597-udc.c              |    4 +-
+ drivers/usb/gadget/udc/renesas_usbf.c              |    8 +-
+ drivers/usb/gadget/udc/snps_udc_plat.c             |    4 +-
+ drivers/usb/gadget/udc/tegra-xudc.c                |    5 +-
+ drivers/usb/host/Kconfig                           |   13 +
+ drivers/usb/host/ehci-dbg.c                        |    3 +-
+ drivers/usb/host/ehci-hcd.c                        |   31 +-
+ drivers/usb/host/ehci-hub.c                        |  104 +-
+ drivers/usb/host/ehci-ppc-of.c                     |   47 +-
+ drivers/usb/host/ehci-timer.c                      |    3 +-
+ drivers/usb/host/ehci.h                            |   55 +-
+ drivers/usb/host/ohci-dbg.c                        |    2 +-
+ drivers/usb/host/ohci-hub.c                        |    2 +-
+ drivers/usb/host/ohci-ppc-of.c                     |   44 +-
+ drivers/usb/host/pci-quirks.c                      |    1 +
+ drivers/usb/host/xhci-tegra.c                      |    8 +-
+ drivers/usb/image/microtek.c                       |    1 +
+ drivers/usb/misc/apple-mfi-fastcharge.c            |    2 +-
+ drivers/usb/misc/brcmstb-usb-pinmap.c              |    9 +-
+ drivers/usb/misc/onboard_usb_dev.c                 |    2 +-
+ drivers/usb/misc/qcom_eud.c                        |    2 +-
+ drivers/usb/mon/mon_main.c                         |    3 +
+ drivers/usb/mtu3/mtu3_core.c                       |    4 +-
+ drivers/usb/musb/da8xx.c                           |    8 +-
+ drivers/usb/musb/musb_dsps.c                       |   25 +
+ drivers/usb/phy/phy-ab8500-usb.c                   |   12 +-
+ drivers/usb/phy/phy-generic.c                      |    3 +-
+ drivers/usb/phy/phy-gpio-vbus-usb.c                |    5 +-
+ drivers/usb/renesas_usbhs/mod.c                    |    4 +-
+ drivers/usb/storage/alauda.c                       |   95 +-
+ drivers/usb/storage/ene_ub6250.c                   |    2 +
+ drivers/usb/storage/sddr09.c                       |    9 +
+ drivers/usb/storage/usb.c                          |    6 +-
+ drivers/usb/typec/anx7411.c                        |    4 +-
+ drivers/usb/typec/bus.c                            |    4 +-
+ drivers/usb/typec/hd3ss3220.c                      |   36 +-
+ drivers/usb/typec/mux/gpio-sbu-mux.c               |   13 +-
+ drivers/usb/typec/mux/it5205.c                     |    2 +-
+ drivers/usb/typec/tcpm/tcpci_maxim_core.c          |    3 +-
+ drivers/usb/typec/tcpm/tcpci_mt6360.c              |    1 -
+ drivers/usb/typec/tcpm/tcpci_mt6370.c              |    2 +-
+ drivers/usb/typec/tcpm/tcpm.c                      |   57 +-
+ drivers/usb/typec/tipd/core.c                      |    3 +
+ drivers/usb/typec/ucsi/ucsi.c                      |   52 +-
+ drivers/usb/typec/ucsi/ucsi.h                      |    3 +-
+ drivers/usb/usbip/stub_main.c                      |    2 +-
+ include/linux/ulpi/driver.h                        |    4 +-
+ include/linux/usb.h                                |    6 +-
+ include/linux/usb/typec_altmode.h                  |    4 +-
+ 110 files changed, 1791 insertions(+), 7035 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/dma/ti/ti,cppi41.yaml
+ create mode 100644 Documentation/devicetree/bindings/phy/ti,am335x-usb-phy.yaml
+ delete mode 100644 Documentation/devicetree/bindings/usb/am33xx-usb.txt
+ delete mode 100644 Documentation/devicetree/bindings/usb/da8xx-usb.txt
+ delete mode 100644 Documentation/devicetree/bindings/usb/ohci-da8xx.txt
+ create mode 100644 Documentation/devicetree/bindings/usb/ti,am335x-usb-ctrl-module.yaml
+ create mode 100644 Documentation/devicetree/bindings/usb/ti,am33xx-usb.yaml
+ create mode 100644 Documentation/devicetree/bindings/usb/ti,da830-musb.yaml
+ create mode 100644 Documentation/devicetree/bindings/usb/ti,da830-ohci.yaml
+ create mode 100644 Documentation/devicetree/bindings/usb/ti,musb-am33xx.yaml
+ delete mode 100644 drivers/usb/fotg210/fotg210-hcd.h
+Merging thunderbolt/next (a93a8e3200002 thunderbolt: stream: Make read return framing error to the userspace)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt/next
+Auto-merging drivers/thunderbolt/nhi.c
+Auto-merging drivers/thunderbolt/nhi.h
+Auto-merging drivers/thunderbolt/pci.c
+Auto-merging drivers/thunderbolt/quirks.c
+Auto-merging drivers/thunderbolt/stream.c
+Auto-merging drivers/thunderbolt/switch.c
+Auto-merging drivers/thunderbolt/tb.c
+Auto-merging include/linux/thunderbolt.h
+Merge made by the 'ort' strategy.
+ drivers/net/thunderbolt/main.c |   6 +-
+ drivers/thunderbolt/debugfs.c  | 126 ++++++++++++++----
+ drivers/thunderbolt/dma_test.c |  10 +-
+ drivers/thunderbolt/eeprom.c   |  23 +++-
+ drivers/thunderbolt/nhi.c      | 222 ++++++++++++++++++++++---------
+ drivers/thunderbolt/nhi.h      |   2 +
+ drivers/thunderbolt/path.c     |   2 +-
+ drivers/thunderbolt/pci.c      | 113 ++++++++++++++++
+ drivers/thunderbolt/quirks.c   |  18 +++
+ drivers/thunderbolt/sb_regs.h  |   2 +
+ drivers/thunderbolt/stream.c   | 287 +++++++++++++++++++++++++++++++----------
+ drivers/thunderbolt/switch.c   |   4 +
+ drivers/thunderbolt/tb.c       |  78 -----------
+ drivers/thunderbolt/tb.h       |   5 +-
+ drivers/thunderbolt/usb4.c     |  10 +-
+ include/linux/thunderbolt.h    |  47 ++++++-
+ 16 files changed, 703 insertions(+), 252 deletions(-)
+Merging usb-serial/usb-next (6583f9741341b USB: serial: wwan: replace __get_free_page() with kmalloc())
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial/usb-next
+Merge made by the 'ort' strategy.
+ drivers/usb/serial/usb_wwan.c | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+Merging tty/tty-next (bf961847813d9 serial: sifive: fix off-by-one in console port bounds check)
+$ git merge -m Merge branch 'tty-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty/tty-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/tty/serial/8250/8250_port.c
+Auto-merging drivers/tty/serial/kgdboc.c
+Auto-merging drivers/tty/serial/qcom_geni_serial.c
+CONFLICT (content): Merge conflict in drivers/tty/serial/qcom_geni_serial.c
+Auto-merging drivers/tty/serial/serial_core.c
+Auto-merging drivers/tty/tty_port.c
+Auto-merging include/linux/soc/qcom/geni-se.h
+CONFLICT (content): Merge conflict in include/linux/soc/qcom/geni-se.h
+Resolved 'drivers/tty/serial/qcom_geni_serial.c' using previous resolution.
+Resolved 'include/linux/soc/qcom/geni-se.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 473edeccacc76] Merge branch 'tty-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/devices.txt              |   13 +-
+ Documentation/devicetree/bindings/serial/8250.yaml |   32 +-
+ .../devicetree/bindings/serial/cdns,uart.yaml      |    1 +
+ .../devicetree/bindings/serial/renesas,hscif.yaml  |    4 +-
+ .../devicetree/bindings/serial/renesas,scif.yaml   |    4 +-
+ .../bindings/serial/snps-dw-apb-uart.yaml          |    1 +
+ Documentation/driver-api/serial/driver.rst         |    2 +-
+ MAINTAINERS                                        |    2 +-
+ drivers/acpi/acpi_apd.c                            |    1 +
+ drivers/tty/Kconfig                                |   10 -
+ drivers/tty/Makefile                               |    1 -
+ drivers/tty/hvc/hvcs.c                             |    6 +-
+ drivers/tty/moxa.c                                 | 2136 --------------------
+ drivers/tty/pty.c                                  |    5 +-
+ drivers/tty/serdev/serdev-ttyport.c                |    2 +
+ drivers/tty/serial/8250/8250.h                     |   17 +-
+ drivers/tty/serial/8250/8250_airoha.c              |  187 ++
+ drivers/tty/serial/8250/8250_core.c                |   25 +-
+ drivers/tty/serial/8250/8250_dw.c                  |   19 +-
+ drivers/tty/serial/8250/8250_dwlib.c               |    5 +
+ drivers/tty/serial/8250/8250_dwlib.h               |    5 +
+ drivers/tty/serial/8250/8250_hub6.c                |   11 +-
+ drivers/tty/serial/8250/8250_mid.c                 |   15 +-
+ drivers/tty/serial/8250/8250_mxpcie.c              |   30 +-
+ drivers/tty/serial/8250/8250_pci.c                 |    2 +-
+ drivers/tty/serial/8250/8250_platform.c            |    9 +-
+ drivers/tty/serial/8250/8250_pnp.c                 |    4 +-
+ drivers/tty/serial/8250/8250_port.c                |   31 +-
+ drivers/tty/serial/8250/8250_uniphier.c            |   28 +-
+ drivers/tty/serial/8250/Kconfig                    |   15 +-
+ drivers/tty/serial/8250/Makefile                   |    3 +-
+ drivers/tty/serial/Kconfig                         |    1 +
+ drivers/tty/serial/arc_uart.c                      |    2 +-
+ drivers/tty/serial/atmel_serial.c                  |    8 +-
+ drivers/tty/serial/fsl_lpuart.c                    |   18 +-
+ drivers/tty/serial/kgdboc.c                        |    2 +-
+ drivers/tty/serial/qcom_geni_serial.c              |   43 +-
+ drivers/tty/serial/serial_core.c                   |   51 +-
+ drivers/tty/serial/serial_txx9.c                   |   62 +-
+ drivers/tty/serial/sh-sci.c                        |    8 +-
+ drivers/tty/serial/sifive.c                        |    2 +-
+ drivers/tty/sysrq.c                                |    5 +-
+ drivers/tty/tty_port.c                             |    8 +-
+ drivers/tty/vt/keyboard.c                          |    4 +-
+ include/linux/serial_core.h                        |    2 -
+ include/linux/soc/qcom/geni-se.h                   |   12 +-
+ 46 files changed, 460 insertions(+), 2394 deletions(-)
+ delete mode 100644 drivers/tty/moxa.c
+ create mode 100644 drivers/tty/serial/8250/8250_airoha.c
+Merging char-misc/char-misc-next (315860c4f9129 binder: Fix up Kconfig dependancy due to removal of .c code)
+$ git merge -m Merge branch 'char-misc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc/char-misc-next
+CONFLICT (modify/delete): drivers/android/binder.c deleted in char-misc/char-misc-next and modified in HEAD.  Version HEAD of drivers/android/binder.c left in tree.
+Auto-merging drivers/android/binder/node.rs
+Auto-merging drivers/android/binder/node/wrapper.rs
+Auto-merging drivers/android/binder/page_range.rs
+Auto-merging drivers/android/binder/rust_binderfs.c
+Auto-merging drivers/android/binder/thread.rs
+CONFLICT (modify/delete): drivers/android/binder_alloc.c deleted in char-misc/char-misc-next and modified in HEAD.  Version HEAD of drivers/android/binder_alloc.c left in tree.
+CONFLICT (modify/delete): drivers/android/binderfs.c deleted in char-misc/char-misc-next and modified in HEAD.  Version HEAD of drivers/android/binderfs.c left in tree.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git rm -f drivers/android/binder.c
+drivers/android/binderfs.c
+drivers/android/binder_alloc.c
+fatal: pathspec 'drivers/android/binder.c
+drivers/android/binderfs.c
+drivers/android/binder_alloc.c' did not match any files
+$ git commit --no-edit -v -a
+[master 70b76384f1cc2] Merge branch 'char-misc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/android/Kconfig                            |   41 +-
+ drivers/android/Makefile                           |    3 -
+ drivers/android/binder.c                           | 7187 --------------------
+ drivers/android/binder/Makefile                    |    4 +-
+ drivers/android/binder/allocation.rs               |   10 +-
+ drivers/android/binder/defs.rs                     |    3 +-
+ drivers/android/binder/freeze.rs                   |   11 +-
+ drivers/android/binder/node.rs                     |   10 +-
+ drivers/android/binder/node/wrapper.rs             |    2 +-
+ drivers/android/binder/page_range.rs               |    6 +-
+ drivers/android/binder/process.rs                  |   21 +-
+ drivers/android/binder/range_alloc/array.rs        |    2 +-
+ drivers/android/binder/range_alloc/mod.rs          |    2 +-
+ drivers/android/binder/range_alloc/tree.rs         |    2 +-
+ drivers/android/binder/rust_binder_main.rs         |   14 +-
+ drivers/android/binder/rust_binderfs.c             |    2 +-
+ drivers/android/binder/thread.rs                   |   58 +-
+ drivers/android/binder/transaction.rs              |    3 +-
+ drivers/android/binder_alloc.c                     | 1398 ----
+ drivers/android/binder_alloc.h                     |  189 -
+ drivers/android/binder_internal.h                  |  597 --
+ drivers/android/binder_netlink.c                   |   32 -
+ drivers/android/binder_netlink.h                   |   21 -
+ drivers/android/binder_trace.h                     |  448 --
+ drivers/android/binderfs.c                         |  785 ---
+ drivers/android/dbitmap.h                          |  169 -
+ drivers/android/tests/.kunitconfig                 |    7 -
+ drivers/android/tests/Makefile                     |    6 -
+ drivers/android/tests/binder_alloc_kunit.c         |  572 --
+ include/uapi/linux/android/binder.h                |    1 +
+ .../selftests/filesystems/binderfs/binderfs_test.c |    2 +-
+ .../testing/selftests/filesystems/binderfs/config  |    3 +-
+ 32 files changed, 98 insertions(+), 11513 deletions(-)
+ delete mode 100644 drivers/android/binder.c
+ delete mode 100644 drivers/android/binder_alloc.c
+ delete mode 100644 drivers/android/binder_alloc.h
+ delete mode 100644 drivers/android/binder_internal.h
+ delete mode 100644 drivers/android/binder_netlink.c
+ delete mode 100644 drivers/android/binder_netlink.h
+ delete mode 100644 drivers/android/binder_trace.h
+ delete mode 100644 drivers/android/binderfs.c
+ delete mode 100644 drivers/android/dbitmap.h
+ delete mode 100644 drivers/android/tests/.kunitconfig
+ delete mode 100644 drivers/android/tests/Makefile
+ delete mode 100644 drivers/android/tests/binder_alloc_kunit.c
+Merging coresight/next (9e3604d7369cf coresight: etm4x: remove redundant fields in etmv4_save_state)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git coresight/next
+Already up to date.
+Merging fastrpc/for-next (ef071c4906eb4 Merge branches 'fastrpc-fixes' and 'fastrpc-for-7.4' into fastrpc-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git fastrpc/for-next
+Auto-merging drivers/misc/fastrpc.c
+Merge made by the 'ort' strategy.
+ drivers/misc/fastrpc.c | 200 ++++++++++++++++++++++++++-----------------------
+ 1 file changed, 107 insertions(+), 93 deletions(-)
+Merging fpga/for-next (093da48782df3 fpga: altera-cvp: Retry teardown and reset CVP state on failure)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga/for-next
+Merge made by the 'ort' strategy.
+ drivers/fpga/altera-cvp.c       | 43 ++++++++++++++++++++++++++++++++++-------
+ drivers/fpga/dfl.h              |  1 -
+ drivers/fpga/fpga-mgr.c         |  2 +-
+ drivers/fpga/stratix10-soc.c    |  2 +-
+ drivers/fpga/xilinx-selectmap.c | 39 ++++++++++++++++++++++++++++---------
+ 5 files changed, 68 insertions(+), 19 deletions(-)
+Merging icc/icc-next (3a6d690152557 Merge branch 'icc-fixes' into icc-next)
+$ git merge -m Merge branch 'icc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git icc/icc-next
+Merge made by the 'ort' strategy.
+ .../bindings/interconnect/qcom,kuno-rpmh.yaml      | 118 +++
+ .../bindings/interconnect/qcom,osm-l3.yaml         |   1 +
+ drivers/interconnect/core.c                        |  32 +
+ drivers/interconnect/debugfs-client.c              |  81 +-
+ drivers/interconnect/mediatek/Makefile             |   2 +-
+ drivers/interconnect/qcom/Kconfig                  |  57 ++
+ drivers/interconnect/qcom/Makefile                 |   2 +
+ drivers/interconnect/qcom/bcm-voter.c              |  59 +-
+ drivers/interconnect/qcom/bcm-voter.h              |   1 +
+ drivers/interconnect/qcom/eliza.c                  |  18 +-
+ drivers/interconnect/qcom/icc-common.h             |   9 +
+ drivers/interconnect/qcom/icc-rpm.c                | 105 ++-
+ drivers/interconnect/qcom/icc-rpm.h                |   6 +-
+ drivers/interconnect/qcom/icc-rpmh.c               |  71 +-
+ drivers/interconnect/qcom/kuno.c                   | 988 +++++++++++++++++++++
+ drivers/interconnect/qcom/milos.c                  |  66 +-
+ drivers/interconnect/qcom/msm8976.c                |   3 +-
+ drivers/interconnect/qcom/msm8996.c                |   1 +
+ drivers/interconnect/qcom/sm6350.c                 |  56 +-
+ drivers/interconnect/qcom/sm8650.c                 |  70 +-
+ drivers/interconnect/qcom/smd-rpm.c                |   8 +-
+ drivers/interconnect/samsung/exynos.c              |   1 +
+ include/dt-bindings/interconnect/qcom,kuno.h       |  89 ++
+ 23 files changed, 1672 insertions(+), 172 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/interconnect/qcom,kuno-rpmh.yaml
+ create mode 100644 drivers/interconnect/qcom/kuno.c
+ create mode 100644 include/dt-bindings/interconnect/qcom,kuno.h
+Merging iio/togreg (a3b3580713f3a iio: dac: ad5758: Fix the offset calculation)
+$ git merge -m Merge branch 'togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio/togreg
+Auto-merging Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml
+Auto-merging Documentation/devicetree/bindings/trivial-devices.yaml
+Auto-merging MAINTAINERS
+Auto-merging drivers/iio/accel/kionix-kx022a.c
+Auto-merging drivers/iio/accel/kxcjk-1013.c
+Auto-merging drivers/iio/accel/sca3000.c
+Auto-merging drivers/iio/adc/ad7173.c
+Auto-merging drivers/iio/adc/ade9000.c
+CONFLICT (content): Merge conflict in drivers/iio/adc/ade9000.c
+Auto-merging drivers/iio/adc/adi-axi-adc.c
+Auto-merging drivers/iio/adc/pac1934.c
+Auto-merging drivers/iio/adc/stm32-adc.c
+Auto-merging drivers/iio/buffer/industrialio-buffer-dmaengine.c
+Auto-merging drivers/iio/frequency/adf4377.c
+Auto-merging drivers/iio/imu/adis16400.c
+Auto-merging drivers/iio/light/lm3533-als.c
+Auto-merging drivers/iio/pressure/rohm-bm1390.c
+Auto-merging drivers/iio/proximity/aw96103.c
+Resolved 'drivers/iio/adc/ade9000.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 0025a46a07d9b] Merge branch 'togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/debugfs-iio-ad9910       |   23 +
+ Documentation/ABI/testing/debugfs-iio-backend      |   16 +-
+ Documentation/ABI/testing/sysfs-bus-iio            |  115 +
+ Documentation/ABI/testing/sysfs-bus-iio-adc        |    9 +
+ .../ABI/testing/sysfs-bus-iio-frequency-ad9910     |   31 +
+ .../bindings/iio/accel/adi,adis16201.yaml          |    6 +-
+ .../devicetree/bindings/iio/accel/adi,adxl367.yaml |   13 +-
+ .../devicetree/bindings/iio/adc/adi,ad4080.yaml    |   38 +-
+ .../devicetree/bindings/iio/adc/adi,ad4130.yaml    |    3 +-
+ .../devicetree/bindings/iio/adc/adi,ad4851.yaml    |    8 +-
+ .../devicetree/bindings/iio/adc/adi,ad7173.yaml    |    2 +-
+ .../devicetree/bindings/iio/adc/adi,ad7768-1.yaml  |    6 +-
+ .../devicetree/bindings/iio/adc/adi,ad7768.yaml    |  270 +++
+ .../devicetree/bindings/iio/adc/adi,ad7779.yaml    |   30 +-
+ .../devicetree/bindings/iio/adc/adi,ade9000.yaml   |   58 +-
+ .../devicetree/bindings/iio/adc/adi,max40080.yaml  |   65 +
+ .../bindings/iio/adc/axiado,ax3000-saradc.yaml     |   63 +
+ .../devicetree/bindings/iio/adc/lltc,ltc2497.yaml  |    2 +-
+ .../bindings/iio/adc/maxim,max34408.yaml           |   28 +-
+ .../bindings/iio/adc/mediatek,mt2701-auxadc.yaml   |    1 +
+ .../bindings/iio/adc/nxp,lpc1850-adc.yaml          |    2 +-
+ .../bindings/iio/adc/renesas,r9a09g077-adc.yaml    |   22 +-
+ .../bindings/iio/adc/renesas,rzg2l-adc.yaml        |   17 +-
+ .../devicetree/bindings/iio/adc/ti,ads1015.yaml    |    6 +
+ .../devicetree/bindings/iio/adc/ti,ads1100.yaml    |   10 +-
+ .../devicetree/bindings/iio/adc/ti,ads112c04.yaml  |  149 ++
+ .../devicetree/bindings/iio/adc/ti,ads1298.yaml    |   18 +-
+ .../devicetree/bindings/iio/addac/adi,ad74115.yaml |    3 +-
+ .../bindings/iio/addac/adi,ad74413r.yaml           |    3 +-
+ .../devicetree/bindings/iio/dac/adi,ad3530r.yaml   |   15 +-
+ .../devicetree/bindings/iio/dac/adi,ad5529r.yaml   |  254 +++
+ .../devicetree/bindings/iio/dac/adi,ad5710r.yaml   |  145 ++
+ .../bindings/iio/dac/microchip,mcp47feb02.yaml     |  219 +-
+ .../bindings/iio/frequency/adi,ad9910.yaml         |  209 ++
+ .../bindings/iio/imu/invensense,icm42600.yaml      |   24 +-
+ .../devicetree/bindings/iio/light/adux1020.yaml    |    8 +-
+ .../bindings/iio/light/amstaos,tsl2591.yaml        |    2 +-
+ .../bindings/iio/light/capella,cm32181.yaml        |   44 +
+ .../devicetree/bindings/iio/light/isl29018.yaml    |    6 +-
+ .../bindings/iio/light/liteon,ltr501.yaml          |   32 +-
+ .../devicetree/bindings/iio/light/noa1305.yaml     |    4 +-
+ .../devicetree/bindings/iio/light/stk33xx.yaml     |   19 +-
+ .../devicetree/bindings/iio/light/tsl2583.yaml     |    4 +-
+ .../devicetree/bindings/iio/light/tsl2772.yaml     |   14 +-
+ .../bindings/iio/light/vishay,veml6030.yaml        |   21 +-
+ .../bindings/iio/pressure/infineon,dps310.yaml     |   10 +-
+ .../iio/proximity/pulsedlight,lidar-lite-v2.yaml   |   71 +
+ .../bindings/iio/proximity/vishay,vcnl3020.yaml    |    6 +-
+ .../bindings/iio/temperature/adi,ltc2983.yaml      |    1 +
+ .../bindings/spi/spi-peripheral-props.yaml         |    7 +
+ .../devicetree/bindings/trivial-devices.yaml       |    6 +-
+ Documentation/iio/ad7768.rst                       |  275 +++
+ Documentation/iio/ad9910.rst                       |  792 +++++++
+ Documentation/iio/ade9000.rst                      |   26 +-
+ Documentation/iio/adis16475.rst                    |    2 +-
+ Documentation/iio/adis16480.rst                    |    2 +-
+ Documentation/iio/adis16550.rst                    |    2 +-
+ Documentation/iio/adxl313.rst                      |    2 +-
+ Documentation/iio/adxl345.rst                      |    2 +-
+ Documentation/iio/adxl380.rst                      |   10 +-
+ Documentation/iio/index.rst                        |    2 +
+ MAINTAINERS                                        |   74 +-
+ drivers/iio/accel/Kconfig                          |    7 +-
+ drivers/iio/accel/adis16201.c                      |  133 +-
+ drivers/iio/accel/adxl313_spi.c                    |    2 +
+ drivers/iio/accel/adxl367.c                        |   59 +-
+ drivers/iio/accel/adxl372_spi.c                    |    2 +
+ drivers/iio/accel/adxl380.c                        |    7 +
+ drivers/iio/accel/adxl380_spi.c                    |    2 +
+ drivers/iio/accel/bma220_spi.c                     |    3 +-
+ drivers/iio/accel/bma400_core.c                    |    2 -
+ drivers/iio/accel/kionix-kx022a.c                  |   22 +-
+ drivers/iio/accel/kionix-kx022a.h                  |    2 +-
+ drivers/iio/accel/kxcjk-1013.c                     |    1 +
+ drivers/iio/accel/mma8452.c                        |  145 +-
+ drivers/iio/accel/mma9551.c                        |    1 +
+ drivers/iio/accel/mma9553.c                        |    1 +
+ drivers/iio/accel/sca3000.c                        |    2 +
+ drivers/iio/adc/Kconfig                            |   86 +-
+ drivers/iio/adc/Makefile                           |    4 +
+ drivers/iio/adc/ad4080.c                           |   34 +-
+ drivers/iio/adc/ad4134.c                           |    1 -
+ drivers/iio/adc/ad7173.c                           |    3 +-
+ drivers/iio/adc/ad7192.c                           |    3 +
+ drivers/iio/adc/ad7606_spi.c                       |    6 +-
+ drivers/iio/adc/ad7768-1.c                         |    3 +
+ drivers/iio/adc/ad7768.c                           | 1921 ++++++++++++++++
+ drivers/iio/adc/ade9000.c                          |  180 +-
+ drivers/iio/adc/adi-axi-adc.c                      |   21 +
+ drivers/iio/adc/at91-sama5d2_adc.c                 |   10 +-
+ drivers/iio/adc/axiado_saradc.c                    |  277 +++
+ drivers/iio/adc/bcm_iproc_adc.c                    |  101 +-
+ drivers/iio/adc/ltc2497-core.c                     |  297 ++-
+ drivers/iio/adc/ltc2497.c                          |   42 +
+ drivers/iio/adc/ltc2497.h                          |   31 +-
+ drivers/iio/adc/max11205.c                         |    2 +
+ drivers/iio/adc/max14001.c                         |    1 +
+ drivers/iio/adc/max40080.c                         |  574 +++++
+ drivers/iio/adc/mcp3911.c                          |    2 +
+ drivers/iio/adc/meson_saradc.c                     |    2 +-
+ drivers/iio/adc/nxp-sar-adc.c                      |    2 +-
+ drivers/iio/adc/pac1934.c                          |    9 +-
+ drivers/iio/adc/rockchip_saradc.c                  |   16 +
+ drivers/iio/adc/rzg2l_adc.c                        |   29 +-
+ drivers/iio/adc/rzt2h_adc.c                        |  499 ++++-
+ drivers/iio/adc/sophgo-cv1800b-adc.c               |    2 +
+ drivers/iio/adc/stm32-adc.c                        |   14 +-
+ drivers/iio/adc/stm32-dfsdm-adc.c                  |    4 +-
+ drivers/iio/adc/ti-adc128s052.c                    |    8 +-
+ drivers/iio/adc/ti-ads1015.c                       |   30 +
+ drivers/iio/adc/ti-ads1018.c                       |    6 +-
+ drivers/iio/adc/ti-ads1100.c                       |  188 +-
+ drivers/iio/adc/ti-ads112c04.c                     |  524 +++++
+ drivers/iio/adc/ti-ads112c14.c                     | 1274 ++++++++++-
+ drivers/iio/adc/ti-ads131m02.c                     |    2 +
+ drivers/iio/adc/ti-ads7950.c                       |    4 +-
+ drivers/iio/adc/ti_am335x_adc.c                    |    4 +-
+ drivers/iio/amplifiers/ad8366.c                    |    2 +
+ drivers/iio/buffer/industrialio-buffer-dmaengine.c |    8 +-
+ drivers/iio/chemical/ens160_core.c                 |    7 +-
+ drivers/iio/chemical/sps30.c                       |   11 +-
+ .../iio/common/cros_ec_sensors/cros_ec_sensors.c   |    2 +-
+ .../common/cros_ec_sensors/cros_ec_sensors_core.c  |    5 +-
+ drivers/iio/dac/Kconfig                            |   50 +-
+ drivers/iio/dac/Makefile                           |    5 +-
+ drivers/iio/dac/ad3530r.c                          |  321 ++-
+ drivers/iio/dac/ad3552r.c                          |    2 +-
+ drivers/iio/dac/ad5529r.c                          |  546 +++++
+ drivers/iio/dac/ad5755.c                           |    3 +
+ drivers/iio/dac/ad5758.c                           |    8 +-
+ drivers/iio/dac/ad8460.c                           |    2 +-
+ .../iio/dac/{mcp47feb02.c => mcp47feb02-core.c}    |  393 +---
+ drivers/iio/dac/mcp47feb02-i2c.c                   |  145 ++
+ drivers/iio/dac/mcp47feb02-spi.c                   |  145 ++
+ drivers/iio/dac/mcp47feb02.h                       |   41 +
+ drivers/iio/dac/mcp4821.c                          |    7 +-
+ drivers/iio/dummy/iio_simple_dummy.c               |    2 +-
+ drivers/iio/frequency/Kconfig                      |   21 +
+ drivers/iio/frequency/Makefile                     |    1 +
+ drivers/iio/frequency/ad9910.c                     | 2347 ++++++++++++++++++++
+ drivers/iio/frequency/adf4350.c                    |    2 +-
+ drivers/iio/frequency/adf4377.c                    |    3 +
+ drivers/iio/gyro/adxrs290.c                        |    4 +-
+ drivers/iio/gyro/bmg160_core.c                     |    4 +-
+ drivers/iio/gyro/itg3200_buffer.c                  |    3 +-
+ drivers/iio/humidity/Kconfig                       |    8 +-
+ drivers/iio/humidity/am2315.c                      |   34 +-
+ drivers/iio/humidity/ens210.c                      |    4 +-
+ drivers/iio/humidity/hts221_core.c                 |  168 +-
+ drivers/iio/humidity/hts221_i2c.c                  |   11 +-
+ drivers/iio/humidity/hts221_spi.c                  |   11 +-
+ drivers/iio/iio_core.h                             |    7 +
+ drivers/iio/imu/adis16400.c                        |    8 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600.h        |    5 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c  |   33 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c |  114 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_core.c   |   26 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c   |   24 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c    |   16 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c    |   16 +-
+ drivers/iio/imu/inv_mpu6050/inv_mpu_core.c         |    6 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c       |    4 +-
+ drivers/iio/industrialio-backend.c                 |   33 +
+ drivers/iio/industrialio-core.c                    |  255 ++-
+ drivers/iio/industrialio-gts-helper.c              |    2 +-
+ drivers/iio/light/Kconfig                          |   14 +
+ drivers/iio/light/Makefile                         |    1 +
+ drivers/iio/light/apds9306.c                       |    2 +-
+ drivers/iio/light/apds9999.c                       |   12 +-
+ drivers/iio/light/cros_ec_light_prox.c             |    2 +-
+ drivers/iio/light/iqs621-als.c                     |   24 +-
+ drivers/iio/light/isl29018.c                       |    7 +-
+ drivers/iio/light/isl29028.c                       |   33 +-
+ drivers/iio/light/lm3533-als.c                     |   11 +-
+ drivers/iio/light/ltr390.c                         |    6 -
+ drivers/iio/light/ltr501.c                         |  113 +-
+ drivers/iio/light/stk3310.c                        |  203 +-
+ drivers/iio/light/tsl2583.c                        |   15 +-
+ drivers/iio/light/tsl2772.c                        |   16 +-
+ drivers/iio/light/vcnl4000.c                       |    3 +-
+ drivers/iio/light/veml3328.c                       |   53 +-
+ drivers/iio/light/veml6031x00.c                    | 1387 ++++++++++++
+ drivers/iio/magnetometer/ak8975.c                  |  234 +-
+ drivers/iio/position/iqs624-pos.c                  |   24 +-
+ drivers/iio/potentiometer/x9250.c                  |    7 +-
+ drivers/iio/pressure/Kconfig                       |    2 +
+ drivers/iio/pressure/Makefile                      |    2 +
+ drivers/iio/pressure/bmp280-spi.c                  |    2 +
+ drivers/iio/pressure/cros_ec_baro.c                |    2 +-
+ drivers/iio/pressure/dps310.c                      |  313 ++-
+ drivers/iio/pressure/rohm-bm1390.c                 |    6 +-
+ drivers/iio/proximity/aw96103.c                    |    2 +-
+ drivers/iio/temperature/iqs620at-temp.c            |    9 +-
+ drivers/iio/temperature/ltc2983.c                  |   19 +-
+ drivers/iio/temperature/mlx90632.c                 |   10 +-
+ drivers/iio/temperature/mlx90635.c                 |   11 +-
+ drivers/iio/temperature/tmp117.c                   |   18 +-
+ drivers/iio/temperature/tsys01.c                   |   17 +-
+ drivers/iio/temperature/tsys02d.c                  |   12 +-
+ drivers/iio/test/Kconfig                           |   13 +
+ drivers/iio/test/Makefile                          |    1 +
+ drivers/iio/test/iio-test-channel-prefix.c         |  307 +++
+ drivers/iio/test/iio-test-rescale.c                |    4 +-
+ .../iio/Documentation/sysfs-bus-iio-adc-ad7280a    |    2 +-
+ drivers/staging/iio/Kconfig                        |    1 -
+ drivers/staging/iio/Makefile                       |    1 -
+ drivers/staging/iio/accel/Kconfig                  |   19 -
+ drivers/staging/iio/accel/Makefile                 |    6 -
+ drivers/staging/iio/accel/adis16203.c              |  315 ---
+ drivers/staging/iio/adc/ad7816.c                   |   33 +-
+ drivers/staging/iio/frequency/ad9832.c             |    6 +
+ drivers/staging/iio/frequency/ad9834.c             |    6 +
+ include/linux/iio/adc/qcom-adc5-gen3-common.h      |    2 +-
+ include/linux/iio/backend.h                        |    6 +
+ include/linux/iio/iio-gts-helper.h                 |   11 +-
+ include/linux/iio/iio-opaque.h                     |    2 +-
+ include/linux/iio/iio.h                            |   25 +-
+ include/linux/notifier.h                           |    7 +
+ include/uapi/linux/iio/types.h                     |    1 +
+ kernel/notifier.c                                  |  105 +
+ tools/iio/iio_event_monitor.c                      |    2 +
+ 221 files changed, 16048 insertions(+), 2228 deletions(-)
+ create mode 100644 Documentation/ABI/testing/debugfs-iio-ad9910
+ create mode 100644 Documentation/ABI/testing/sysfs-bus-iio-adc
+ create mode 100644 Documentation/ABI/testing/sysfs-bus-iio-frequency-ad9910
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/adi,ad7768.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/adi,max40080.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/axiado,ax3000-saradc.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/ti,ads112c04.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/dac/adi,ad5529r.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/dac/adi,ad5710r.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/frequency/adi,ad9910.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/light/capella,cm32181.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/proximity/pulsedlight,lidar-lite-v2.yaml
+ create mode 100644 Documentation/iio/ad7768.rst
+ create mode 100644 Documentation/iio/ad9910.rst
+ create mode 100644 drivers/iio/adc/ad7768.c
+ create mode 100644 drivers/iio/adc/axiado_saradc.c
+ create mode 100644 drivers/iio/adc/max40080.c
+ create mode 100644 drivers/iio/adc/ti-ads112c04.c
+ create mode 100644 drivers/iio/dac/ad5529r.c
+ rename drivers/iio/dac/{mcp47feb02.c => mcp47feb02-core.c} (69%)
+ create mode 100644 drivers/iio/dac/mcp47feb02-i2c.c
+ create mode 100644 drivers/iio/dac/mcp47feb02-spi.c
+ create mode 100644 drivers/iio/dac/mcp47feb02.h
+ create mode 100644 drivers/iio/frequency/ad9910.c
+ create mode 100644 drivers/iio/light/veml6031x00.c
+ create mode 100644 drivers/iio/test/iio-test-channel-prefix.c
+ delete mode 100644 drivers/staging/iio/accel/Kconfig
+ delete mode 100644 drivers/staging/iio/accel/Makefile
+ delete mode 100644 drivers/staging/iio/accel/adis16203.c
+Merging nfc/for-next (fd73f4a665989 Linux 7.3-rc3)
+$ git merge -m Merge branch 'for-next' of https://codeberg.org/linux-nfc/linux.git nfc/for-next
+Already up to date.
+Merging phy-next/next (c7f2322431cb6 dt-bindings: phy: ti,tcan104x-can: Fix property constrains)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy-next/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/phy/mediatek,mt8195-dp-phy.yaml       |  77 ++
+ .../devicetree/bindings/phy/mediatek,tphy.yaml     |   1 +
+ .../bindings/phy/qcom,m31-eusb2-phy.yaml           |   1 +
+ .../bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml   |   2 +
+ .../phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml         |  59 +-
+ .../bindings/phy/qcom,x1e80100-csi2-phy.yaml       | 209 ++++++
+ .../devicetree/bindings/phy/ti,tcan104x-can.yaml   |   9 +-
+ MAINTAINERS                                        |  10 +
+ drivers/phy/apple/atc.c                            |   2 -
+ drivers/phy/broadcom/phy-brcm-usb.c                |  29 +-
+ drivers/phy/cadence/phy-cadence-sierra.c           |  11 +-
+ drivers/phy/hisilicon/phy-hi3670-pcie.c            |   2 +
+ drivers/phy/mediatek/phy-mtk-dp.c                  | 826 ++++++++++++++++++---
+ drivers/phy/phy-core.c                             | 166 ++++-
+ drivers/phy/qualcomm/Kconfig                       |  15 +
+ drivers/phy/qualcomm/Makefile                      |   5 +
+ drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c | 385 ++++++++++
+ drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c     | 463 ++++++++++++
+ drivers/phy/qualcomm/phy-qcom-mipi-csi2.h          |  97 +++
+ drivers/phy/qualcomm/phy-qcom-qmp-combo.c          | 402 ++++++++--
+ drivers/phy/qualcomm/phy-qcom-qmp-pcie.c           |  73 ++
+ drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h      |   4 +-
+ drivers/phy/samsung/phy-exynos5-usbdrd.c           |  78 +-
+ drivers/phy/socionext/phy-uniphier-usb3hs.c        |   3 +-
+ drivers/phy/starfive/phy-jh7110-dphy-tx.c          |  16 +-
+ include/dt-bindings/phy/phy-qcom-qmp.h             |   1 +
+ include/linux/phy/phy-thunderbolt.h                |  14 +
+ include/linux/phy/phy.h                            |  15 +
+ 28 files changed, 2715 insertions(+), 260 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/phy/mediatek,mt8195-dp-phy.yaml
+ create mode 100644 Documentation/devicetree/bindings/phy/qcom,x1e80100-csi2-phy.yaml
+ create mode 100644 drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c
+ create mode 100644 drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c
+ create mode 100644 drivers/phy/qualcomm/phy-qcom-mipi-csi2.h
+ create mode 100644 include/linux/phy/phy-thunderbolt.h
+Merging soundwire/next (90b63b309fd6c soundwire: Intel: stop sdw clock in system suspend)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire/next
+Auto-merging drivers/soundwire/dmi-quirks.c
+Merge made by the 'ort' strategy.
+ drivers/soundwire/bus.c              | 54 +++++++++++++++++++++++-------------
+ drivers/soundwire/bus_type.c         | 16 +++++++----
+ drivers/soundwire/dmi-quirks.c       | 14 ++++++++++
+ drivers/soundwire/intel.h            |  6 ++--
+ drivers/soundwire/intel_auxdevice.c  | 11 +++++---
+ drivers/soundwire/intel_bus_common.c |  8 +++---
+ drivers/soundwire/qcom.c             | 38 ++++++++++++++++++-------
+ drivers/soundwire/slave.c            | 28 +++++++++++++------
+ drivers/soundwire/stream.c           |  8 +++---
+ include/linux/soundwire/sdw.h        |  4 +++
+ include/linux/soundwire/sdw_intel.h  |  2 +-
+ 11 files changed, 129 insertions(+), 60 deletions(-)
+Merging extcon/extcon-next (8d3ae59288f1e Linux 7.2)
+$ git merge -m Merge branch 'extcon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git extcon/extcon-next
+Already up to date.
+Merging gnss/gnss-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'gnss-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss/gnss-next
+Already up to date.
+Merging vfio/next (b30b52c2fb82e vfio/pci: Restore 8-byte ioeventfd support)
+$ git merge -m Merge branch 'next' of https://github.com/awilliam/linux-vfio.git vfio/next
+Auto-merging Documentation/driver-api/index.rst
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/driver-api/index.rst                 |    1 +
+ Documentation/driver-api/vfio-selftests.rst        |  252 +++
+ MAINTAINERS                                        |    1 +
+ drivers/vfio/cdx/main.c                            |   17 +-
+ drivers/vfio/device_cdev.c                         |   12 +
+ drivers/vfio/fsl-mc/vfio_fsl_mc.c                  |   15 +-
+ drivers/vfio/mdev/mdev_core.c                      |    2 +-
+ drivers/vfio/pci/vfio_pci_rdwr.c                   |    3 -
+ drivers/vfio/platform/vfio_platform_common.c       |   15 +-
+ drivers/vfio/vfio_iommu_type1.c                    |   12 +-
+ drivers/vfio/vfio_main.c                           |    7 -
+ include/linux/mlx5/cq.h                            |   10 -
+ include/linux/mlx5/device.h                        |  231 +--
+ include/linux/mlx5/mlx5_ifc.h                      |  178 ++
+ include/linux/mlx5/mlx5_ifc_macros.h               |  185 ++
+ tools/arch/arm64/include/asm/barrier.h             |    4 +
+ tools/arch/x86/include/asm/barrier.h               |    5 +
+ tools/include/asm-generic/io.h                     |   28 +
+ tools/include/asm/barrier.h                        |    8 +
+ tools/include/linux/stddef.h                       |   10 +
+ .../selftests/kvm/include/arm64/processor.h        |    4 +-
+ tools/testing/selftests/kvm/irq_test.c             |    2 -
+ tools/testing/selftests/vfio/.gitignore            |    1 +
+ tools/testing/selftests/vfio/lib/drivers/dsa/dsa.c |    1 +
+ tools/testing/selftests/vfio/lib/drivers/igb/igb.c |    1 +
+ .../testing/selftests/vfio/lib/drivers/ioat/ioat.c |    1 +
+ .../testing/selftests/vfio/lib/drivers/mlx5/mlx5.c | 1928 ++++++++++++++++++++
+ .../selftests/vfio/lib/drivers/mlx5/mlx5_hw.h      |  114 ++
+ .../selftests/vfio/lib/drivers/mlx5/mlx5_ifc.h     |    1 +
+ .../vfio/lib/drivers/mlx5/mlx5_ifc_fpga.h          |    1 +
+ .../vfio/lib/drivers/mlx5/mlx5_ifc_macros.h        |    1 +
+ .../vfio/lib/drivers/nv_falcon/nv_falcon.c         |    1 +
+ .../vfio/lib/include/libvfio/vfio_pci_device.h     |   11 +
+ .../vfio/lib/include/libvfio/vfio_pci_driver.h     |    6 +
+ tools/testing/selftests/vfio/lib/iova_allocator.c  |    7 +-
+ tools/testing/selftests/vfio/lib/libvfio.mk        |    1 +
+ tools/testing/selftests/vfio/lib/sysfs.c           |    1 +
+ tools/testing/selftests/vfio/lib/vfio_pci_driver.c |    6 +
+ tools/testing/selftests/vfio/settings              |    5 +
+ .../testing/selftests/vfio/vfio_pci_driver_test.c  |    5 +-
+ .../selftests/vfio/vfio_pci_sriov_uapi_test.c      |   35 +
+ 41 files changed, 2850 insertions(+), 279 deletions(-)
+ create mode 100644 Documentation/driver-api/vfio-selftests.rst
+ create mode 100644 include/linux/mlx5/mlx5_ifc_macros.h
+ create mode 100644 tools/include/linux/stddef.h
+ create mode 100644 tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5.c
+ create mode 100644 tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_hw.h
+ create mode 120000 tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc.h
+ create mode 120000 tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_fpga.h
+ create mode 120000 tools/testing/selftests/vfio/lib/drivers/mlx5/mlx5_ifc_macros.h
+ create mode 100644 tools/testing/selftests/vfio/settings
+Merging w1/for-next (813a5b9a3c9f6 w1: fix spelling mistakes in comments across the subsystem)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1/for-next
+Merge made by the 'ort' strategy.
+ drivers/w1/masters/ds2482.c    | 2 +-
+ drivers/w1/masters/ds2490.c    | 4 ++--
+ drivers/w1/masters/omap_hdq.c  | 2 +-
+ drivers/w1/slaves/w1_ds28e17.c | 2 +-
+ drivers/w1/slaves/w1_therm.c   | 2 +-
+ drivers/w1/w1.c                | 2 +-
+ drivers/w1/w1_family.c         | 2 +-
+ drivers/w1/w1_netlink.h        | 2 +-
+ 8 files changed, 9 insertions(+), 9 deletions(-)
+Merging spmi/spmi-next (8cdeaa50eae8d Linux 7.2-rc2)
+$ git merge -m Merge branch 'spmi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git spmi/spmi-next
+Already up to date.
+Merging staging/staging-next (8444548bd905f staging: rtl8723bs: remove unused SSIZE_PTR definition)
+$ git merge -m Merge branch 'staging-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging/staging-next
+Auto-merging drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+Merge made by the 'ort' strategy.
+ drivers/staging/axis-fifo/axis-fifo.c              |    6 +-
+ drivers/staging/fbtft/fb_bd663474.c                |    4 +-
+ drivers/staging/greybus/loopback.c                 |    5 +-
+ drivers/staging/most/video/video.c                 |    1 -
+ drivers/staging/octeon/ethernet-tx.c               |    2 +-
+ drivers/staging/rtl8723bs/core/rtw_ap.c            |    2 +-
+ drivers/staging/rtl8723bs/core/rtw_cmd.c           |   25 +-
+ drivers/staging/rtl8723bs/core/rtw_efuse.c         |  147 +--
+ drivers/staging/rtl8723bs/core/rtw_mlme.c          | 1006 ++++++++++----------
+ drivers/staging/rtl8723bs/core/rtw_mlme_ext.c      |  111 +--
+ drivers/staging/rtl8723bs/core/rtw_recv.c          |   49 +-
+ drivers/staging/rtl8723bs/core/rtw_security.c      |    4 +-
+ drivers/staging/rtl8723bs/core/rtw_wlan_util.c     |    6 +-
+ drivers/staging/rtl8723bs/core/rtw_xmit.c          |   48 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf.c           |    5 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c     |   75 --
+ drivers/staging/rtl8723bs/hal/hal_com.c            |    6 +-
+ drivers/staging/rtl8723bs/hal/hal_intf.c           |    9 +-
+ drivers/staging/rtl8723bs/hal/odm.h                |    9 -
+ drivers/staging/rtl8723bs/hal/odm_DIG.h            |    1 -
+ drivers/staging/rtl8723bs/hal/odm_DynamicTxPower.h |    1 -
+ drivers/staging/rtl8723bs/hal/odm_precomp.h        |    1 -
+ drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c  |    2 -
+ drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c    |    2 +-
+ drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c     |    3 +-
+ drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c     |    3 -
+ drivers/staging/rtl8723bs/hal/sdio_halinit.c       |   24 -
+ drivers/staging/rtl8723bs/include/basic_types.h    |    4 -
+ drivers/staging/rtl8723bs/include/cmd_osdep.h      |    6 +-
+ drivers/staging/rtl8723bs/include/hal_com.h        |    2 +-
+ drivers/staging/rtl8723bs/include/hal_data.h       |   16 -
+ drivers/staging/rtl8723bs/include/hal_intf.h       |    2 +-
+ drivers/staging/rtl8723bs/include/hal_pg.h         |    2 +-
+ drivers/staging/rtl8723bs/include/hal_phy.h        |   16 -
+ drivers/staging/rtl8723bs/include/osdep_service.h  |    4 +-
+ .../rtl8723bs/include/osdep_service_linux.h        |    2 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_hal.h   |    2 +-
+ drivers/staging/rtl8723bs/include/rtw_cmd.h        |   58 +-
+ drivers/staging/rtl8723bs/include/rtw_efuse.h      |    5 +-
+ drivers/staging/rtl8723bs/include/rtw_io.h         |   14 +-
+ drivers/staging/rtl8723bs/include/rtw_ioctl_set.h  |    2 -
+ drivers/staging/rtl8723bs/include/rtw_mlme.h       |   78 +-
+ drivers/staging/rtl8723bs/include/rtw_mlme_ext.h   |   38 +-
+ drivers/staging/rtl8723bs/include/rtw_pwrctrl.h    |   20 +-
+ drivers/staging/rtl8723bs/include/rtw_recv.h       |   69 +-
+ drivers/staging/rtl8723bs/include/rtw_xmit.h       |    8 +-
+ drivers/staging/rtl8723bs/include/sdio_ops.h       |   21 +-
+ drivers/staging/rtl8723bs/include/sta_info.h       |   84 +-
+ drivers/staging/rtl8723bs/include/wlan_bssdef.h    |    8 +-
+ drivers/staging/rtl8723bs/include/xmit_osdep.h     |   14 +-
+ drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c  |   86 +-
+ drivers/staging/rtl8723bs/os_dep/sdio_intf.c       |    4 +-
+ drivers/staging/sm750fb/sm750.c                    |   12 +-
+ drivers/staging/sm750fb/sm750_accel.h              |    4 +-
+ drivers/staging/vme_user/vme_tsi148.c              |    3 +-
+ 55 files changed, 929 insertions(+), 1212 deletions(-)
+Merging counter-next/counter-next (edac5cf356994 MAINTAINERS: Mark ftm-quaddec driver as orphaned)
+$ git merge -m Merge branch 'counter-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-next/counter-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS | 6 ++----
+ 1 file changed, 2 insertions(+), 4 deletions(-)
+Merging mux/for-next (ac7bde3c53166 mux: Add driver for Renesas RZ/V2H VBENCTL VBUS_SEL mux)
+$ git merge -m Merge branch 'for-next' of https://gitlab.com/peda-linux/mux.git mux/for-next
+Merge made by the 'ort' strategy.
+ drivers/mux/Kconfig         | 13 ++++++++
+ drivers/mux/Makefile        |  2 ++
+ drivers/mux/rzv2h-vbenctl.c | 81 +++++++++++++++++++++++++++++++++++++++++++++
+ 3 files changed, 96 insertions(+)
+ create mode 100644 drivers/mux/rzv2h-vbenctl.c
+Merging dmaengine/next (0a8dda0a15d39 dmaengine: bestcomm: use devm_platform_get_and_ioremap_resource() to simplify code)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine/next
+CONFLICT (modify/delete): Documentation/devicetree/bindings/usb/am33xx-usb.txt deleted in HEAD and modified in dmaengine/next.  Version dmaengine/next of Documentation/devicetree/bindings/usb/am33xx-usb.txt left in tree.
+CONFLICT (modify/delete): Documentation/devicetree/bindings/usb/da8xx-usb.txt deleted in HEAD and modified in dmaengine/next.  Version dmaengine/next of Documentation/devicetree/bindings/usb/da8xx-usb.txt left in tree.
+Auto-merging MAINTAINERS
+Auto-merging drivers/dma/dmaengine.c
+Auto-merging drivers/dma/mmp_pdma.c
+Auto-merging drivers/dma/pxa_dma.c
+Auto-merging drivers/dma/sprd-dma.c
+Auto-merging drivers/dma/sun6i-dma.c
+Auto-merging drivers/dma/switchtec_dma.c
+Auto-merging drivers/dma/xilinx/xilinx_dma.c
+Auto-merging include/linux/dmaengine.h
+Automatic merge failed; fix conflicts and then commit the result.
+$ git rm -f Documentation/devicetree/bindings/usb/am33xx-usb.txt
+rm 'Documentation/devicetree/bindings/usb/am33xx-usb.txt'
+$ git commit --no-edit -v -a
+[master e818ae9edd692] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git
+$ git diff -M --stat --summary HEAD^..
+ .../devicetree/bindings/dma/apple,admac.yaml       |  10 +-
+ .../devicetree/bindings/dma/snps,dw-axi-dmac.yaml  |  22 ++
+ .../bindings/dma/ti,dra7-dma-crossbar.yaml         |  94 ++++++
+ .../devicetree/bindings/dma/ti-dma-crossbar.txt    |  68 ----
+ Documentation/driver-api/dmaengine/provider.rst    |   2 +-
+ MAINTAINERS                                        |   2 +-
+ drivers/dma/altera-msgdma.c                        |  10 +-
+ drivers/dma/amba-pl08x.c                           |   4 +-
+ drivers/dma/apple-admac.c                          |  24 ++
+ drivers/dma/arm-dma350.c                           |   2 +-
+ drivers/dma/at_hdmac.c                             | 139 ++++----
+ drivers/dma/at_xdmac.c                             | 159 +++++-----
+ drivers/dma/bestcomm/ata.c                         |   2 -
+ drivers/dma/bestcomm/bestcomm.c                    |  46 +--
+ drivers/dma/bestcomm/gen_bd.c                      |  23 +-
+ drivers/dma/dma-jz4780.c                           |  10 +-
+ drivers/dma/dmaengine.c                            |  34 +-
+ drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c     | 151 +++++----
+ drivers/dma/dw-axi-dmac/dw-axi-dmac.h              |  57 ++--
+ drivers/dma/dw-edma/dw-edma-core.c                 | 349 ++++++++++++++-------
+ drivers/dma/dw-edma/dw-edma-core.h                 |  67 +++-
+ drivers/dma/dw-edma/dw-edma-pcie.c                 |  18 ++
+ drivers/dma/dw-edma/dw-edma-v0-core.c              |  59 ++--
+ drivers/dma/dw-edma/dw-hdma-v0-core.c              |  82 +++--
+ drivers/dma/dw-edma/dw-hdma-v0-debugfs.c           |  17 +-
+ drivers/dma/dw-edma/dw-hdma-v0-regs.h              |  10 -
+ drivers/dma/dw/core.c                              |  47 ++-
+ drivers/dma/ep93xx_dma.c                           |  37 +--
+ drivers/dma/fsl-edma-main.c                        |   5 +-
+ drivers/dma/fsl_raid.c                             |  38 ++-
+ drivers/dma/fsldma.c                               |  22 +-
+ drivers/dma/idma64.c                               |  10 +-
+ drivers/dma/img-mdc-dma.c                          |   2 +-
+ drivers/dma/loongson/loongson1-apb-dma.c           |  23 +-
+ drivers/dma/loongson/loongson2-apb-cmc-dma.c       |  19 +-
+ drivers/dma/loongson/loongson2-apb-dma.c           |   9 +-
+ drivers/dma/mediatek/mtk-hsdma.c                   |  26 +-
+ drivers/dma/mmp_pdma.c                             |   3 +-
+ drivers/dma/moxart-dma.c                           |  23 +-
+ drivers/dma/mv_xor.c                               |   8 +-
+ drivers/dma/nbpfaxi.c                              |   2 +-
+ drivers/dma/owl-dma.c                              |  21 +-
+ drivers/dma/pch_dma.c                              |  41 ++-
+ drivers/dma/pl330.c                                |   4 +-
+ drivers/dma/pxa_dma.c                              |  52 +--
+ drivers/dma/qcom/gpi.c                             |   9 +-
+ drivers/dma/sprd-dma.c                             |  45 +--
+ drivers/dma/st_fdma.c                              |   2 +-
+ drivers/dma/ste_dma40.c                            |  15 +-
+ drivers/dma/stm32/stm32-dma.c                      |  69 ++--
+ drivers/dma/stm32/stm32-dma3.c                     |  99 +++---
+ drivers/dma/stm32/stm32-mdma.c                     |  96 +++---
+ drivers/dma/sun4i-dma.c                            |  17 +-
+ drivers/dma/sun6i-dma.c                            |  62 ++--
+ drivers/dma/switchtec_dma.c                        |  14 +-
+ drivers/dma/tegra186-gpc-dma.c                     |   4 +-
+ drivers/dma/tegra20-apb-dma.c                      |   2 +-
+ drivers/dma/ti/k3-udma.c                           |   6 +-
+ drivers/dma/timb_dma.c                             |  63 ++--
+ drivers/dma/txx9dmac.c                             |  89 +++---
+ drivers/dma/virt-dma.h                             |  16 +
+ drivers/dma/xilinx/xilinx_dma.c                    |  85 ++++-
+ drivers/dma/xilinx/zynqmp_dma.c                    |  74 +++--
+ drivers/pci/controller/dwc/pcie-designware.c       |   1 +
+ include/linux/dma/edma.h                           |   3 +-
+ include/linux/dma/imx-dma.h                        |   5 -
+ include/linux/dmaengine.h                          |  23 +-
+ include/linux/fsl/bestcomm/gen_bd.h                |   8 -
+ include/trace/events/tegra_apb_dma.h               |   6 +-
+ 69 files changed, 1481 insertions(+), 1185 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/dma/ti,dra7-dma-crossbar.yaml
+ delete mode 100644 Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt
+Merging cgroup/for-next (8deb0752fa761 Merge branch 'for-7.4' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git cgroup/for-next
+Auto-merging Documentation/admin-guide/cgroup-v2.rst
+Auto-merging block/blk-cgroup.c
+Auto-merging kernel/cgroup/cgroup-internal.h
+Auto-merging kernel/cgroup/cgroup-v1.c
+Auto-merging kernel/cgroup/dmem.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/cgroup-v2.rst          |  62 +++++++++---
+ block/blk-cgroup.c                               |  11 ++-
+ kernel/cgroup/cgroup-internal.h                  |   2 +-
+ kernel/cgroup/cgroup-v1.c                        |  26 +++--
+ kernel/cgroup/cgroup.c                           | 121 ++++++++++++++---------
+ kernel/cgroup/cpuset.c                           |  77 +++------------
+ kernel/cgroup/debug.c                            |  18 ++--
+ kernel/cgroup/dmem.c                             |   4 +-
+ kernel/cgroup/freezer.c                          |   5 +-
+ kernel/cgroup/namespace.c                        |   2 -
+ tools/testing/selftests/cgroup/lib/cgroup_util.c |  45 +++++++--
+ tools/testing/selftests/cgroup/settings          |   1 +
+ 12 files changed, 210 insertions(+), 164 deletions(-)
+ create mode 100644 tools/testing/selftests/cgroup/settings
+Merging scsi/for-next (6147f16c23efb Merge branch 'misc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git scsi/for-next
+Auto-merging drivers/ata/libata-scsi.c
+CONFLICT (content): Merge conflict in drivers/ata/libata-scsi.c
+Auto-merging drivers/scsi/scsi_scan.c
+Auto-merging drivers/ufs/host/ufs-qcom.c
+Auto-merging drivers/usb/gadget/function/f_mass_storage.c
+Resolved 'drivers/ata/libata-scsi.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 11e052aca8eb2] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/scsi/scsi_mid_low_api.rst      |    8 +-
+ drivers/ata/libata-eh.c                      |   17 +-
+ drivers/ata/libata-sata.c                    |   20 +-
+ drivers/ata/libata-scsi.c                    |  344 ++--
+ drivers/ata/libata.h                         |    5 +-
+ drivers/cdrom/cdrom.c                        |   10 +-
+ drivers/message/fusion/mptfc.c               |    8 +-
+ drivers/s390/scsi/zfcp_erp.c                 |   30 +-
+ drivers/s390/scsi/zfcp_ext.h                 |    2 +-
+ drivers/s390/scsi/zfcp_fsf.c                 |    9 +-
+ drivers/s390/scsi/zfcp_scsi.c                |    6 +-
+ drivers/s390/scsi/zfcp_sysfs.c               |    4 +-
+ drivers/scsi/3w-9xxx.c                       |   20 +-
+ drivers/scsi/3w-sas.c                        |   28 +-
+ drivers/scsi/3w-xxxx.c                       |   27 +-
+ drivers/scsi/53c700.c                        |   16 +-
+ drivers/scsi/BusLogic.c                      |   20 +-
+ drivers/scsi/a100u2w.c                       |    4 +-
+ drivers/scsi/a2091.c                         |    4 +-
+ drivers/scsi/a3000.c                         |    4 +-
+ drivers/scsi/aacraid/commsup.c               |    8 +-
+ drivers/scsi/advansys.c                      |    8 +-
+ drivers/scsi/aha1542.c                       |   20 +-
+ drivers/scsi/aha1740.c                       |   12 +-
+ drivers/scsi/arm/eesox.c                     |    8 +-
+ drivers/scsi/arm/fas216.c                    |   20 +-
+ drivers/scsi/atp870u.c                       |   12 +-
+ drivers/scsi/bfa/bfad_bsg.c                  |    4 +-
+ drivers/scsi/ch.c                            |   55 +-
+ drivers/scsi/constants.c                     |   61 +-
+ drivers/scsi/csiostor/csio_attr.c            |    4 +-
+ drivers/scsi/csiostor/csio_scsi.c            |    4 +-
+ drivers/scsi/dc395x.c                        |    8 +-
+ drivers/scsi/device_handler/scsi_dh_alua.c   |   55 +-
+ drivers/scsi/device_handler/scsi_dh_emc.c    |   28 +-
+ drivers/scsi/device_handler/scsi_dh_hp_sw.c  |   29 +-
+ drivers/scsi/device_handler/scsi_dh_rdac.c   |   55 +-
+ drivers/scsi/esp_scsi.c                      |   32 +-
+ drivers/scsi/fcoe/fcoe.c                     |    4 +-
+ drivers/scsi/fdomain.c                       |   16 +-
+ drivers/scsi/gvp11.c                         |    4 +-
+ drivers/scsi/hosts.c                         |   17 +-
+ drivers/scsi/hpsa.c                          |   69 +-
+ drivers/scsi/hpsa_cmd.h                      |   23 -
+ drivers/scsi/hptiop.c                        |    8 +-
+ drivers/scsi/ibmvscsi/ibmvfc-core.c          |  262 +--
+ drivers/scsi/ibmvscsi/ibmvfc-nvme.c          |    8 +-
+ drivers/scsi/ibmvscsi/ibmvscsi.c             |   86 +-
+ drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c     |    5 +-
+ drivers/scsi/imm.c                           |    4 +-
+ drivers/scsi/initio.c                        |    8 +-
+ drivers/scsi/ipr.c                           |  324 ++--
+ drivers/scsi/ips.c                           |   26 +-
+ drivers/scsi/leapraid/leapraid_func.h        |    7 -
+ drivers/scsi/leapraid/leapraid_os.c          |   15 +-
+ drivers/scsi/libfc/fc_fcp.c                  |    8 +-
+ drivers/scsi/libiscsi.c                      |    3 +-
+ drivers/scsi/libsas/sas_scsi_host.c          |   10 +-
+ drivers/scsi/lpfc/lpfc_els.c                 |   30 +-
+ drivers/scsi/lpfc/lpfc_hbadisc.c             |   44 +-
+ drivers/scsi/lpfc/lpfc_init.c                |   16 +-
+ drivers/scsi/lpfc/lpfc_scsi.c                |   26 +-
+ drivers/scsi/lpfc/lpfc_sli.c                 |    4 +-
+ drivers/scsi/mac53c94.c                      |    8 +-
+ drivers/scsi/megaraid.c                      |    8 +-
+ drivers/scsi/megaraid/mega_common.h          |    2 -
+ drivers/scsi/megaraid/megaraid_mbox.c        |   12 +-
+ drivers/scsi/megaraid/megaraid_sas_base.c    |   18 +-
+ drivers/scsi/mesh.c                          |   24 +-
+ drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h         |   77 +-
+ drivers/scsi/mpi3mr/mpi/mpi30_image.h        |    7 +-
+ drivers/scsi/mpi3mr/mpi/mpi30_ioc.h          |   15 +-
+ drivers/scsi/mpi3mr/mpi/mpi30_transport.h    |    2 +-
+ drivers/scsi/mpi3mr/mpi3mr.h                 |   13 +-
+ drivers/scsi/mpi3mr/mpi3mr_app.c             |   87 +-
+ drivers/scsi/mpi3mr/mpi3mr_fw.c              |  223 ++-
+ drivers/scsi/mpi3mr/mpi3mr_os.c              |  320 ++--
+ drivers/scsi/mpi3mr/mpi3mr_transport.c       |  106 +-
+ drivers/scsi/mpt3sas/mpt3sas_scsih.c         |   69 +-
+ drivers/scsi/mvumi.c                         |   28 +-
+ drivers/scsi/myrb.c                          |   67 +-
+ drivers/scsi/myrs.c                          |   23 +-
+ drivers/scsi/nsp32.c                         |    8 +-
+ drivers/scsi/pcmcia/sym53c500_cs.c           |    8 +-
+ drivers/scsi/pm8001/pm8001_defs.h            |   98 ++
+ drivers/scsi/pm8001/pm8001_hwi.c             |    2 +-
+ drivers/scsi/pm8001/pm8001_hwi.h             |    4 +-
+ drivers/scsi/pm8001/pm80xx_hwi.c             |    2 +-
+ drivers/scsi/pm8001/pm80xx_hwi.h             |  100 +-
+ drivers/scsi/pmcraid.c                       |   84 +-
+ drivers/scsi/ps3rom.c                        |    5 +-
+ drivers/scsi/qla1280.c                       |   44 +-
+ drivers/scsi/qla2xxx/qla_attr.c              |    4 +-
+ drivers/scsi/qla2xxx/qla_init.c              |    4 +-
+ drivers/scsi/qla2xxx/qla_isr.c               |    9 +-
+ drivers/scsi/qla2xxx/qla_target.c            |   99 +-
+ drivers/scsi/qla2xxx/qla_target.h            |    3 -
+ drivers/scsi/qlogicfas408.c                  |    8 +-
+ drivers/scsi/qlogicpti.c                     |   20 +-
+ drivers/scsi/scsi.c                          |   15 +-
+ drivers/scsi/scsi_common.c                   |   24 +-
+ drivers/scsi/scsi_debug.c                    |  514 ++++--
+ drivers/scsi/scsi_debugfs.c                  |    2 +-
+ drivers/scsi/scsi_devinfo.c                  |    5 +-
+ drivers/scsi/scsi_error.c                    |  164 +-
+ drivers/scsi/scsi_ioctl.c                    |    3 +-
+ drivers/scsi/scsi_lib.c                      |  179 +-
+ drivers/scsi/scsi_lib_test.c                 |   95 +-
+ drivers/scsi/scsi_logging.c                  |   22 +-
+ drivers/scsi/scsi_proc.c                     |    9 +-
+ drivers/scsi/scsi_scan.c                     |   38 +-
+ drivers/scsi/scsi_sysfs.c                    |   28 +-
+ drivers/scsi/scsi_transport_fc.c             |  136 +-
+ drivers/scsi/scsi_transport_spi.c            |   12 +-
+ drivers/scsi/sd.c                            |  126 +-
+ drivers/scsi/sd_zbc.c                        |    2 +-
+ drivers/scsi/sense_codes.h                   | 2352 +++++++++++++++++---------
+ drivers/scsi/ses.c                           |   23 +-
+ drivers/scsi/sgiwd93.c                       |    4 +-
+ drivers/scsi/smartpqi/smartpqi_init.c        |   31 +-
+ drivers/scsi/snic/snic_disc.c                |   16 +-
+ drivers/scsi/sr.c                            |    3 +-
+ drivers/scsi/sr_ioctl.c                      |   29 +-
+ drivers/scsi/st.c                            |   35 +-
+ drivers/scsi/stex.c                          |   51 +-
+ drivers/scsi/storvsc_drv.c                   |   15 +-
+ drivers/scsi/sym53c8xx_2/sym_glue.c          |   56 +-
+ drivers/scsi/sym53c8xx_2/sym_hipd.h          |    4 +-
+ drivers/scsi/wd33c93.c                       |    4 +-
+ drivers/scsi/wd719x.c                        |   24 +-
+ drivers/scsi/xen-scsifront.c                 |   40 +-
+ drivers/scsi/zorro7xx.c                      |   31 +-
+ drivers/target/target_core_file.c            |    4 +-
+ drivers/target/target_core_pscsi.c           |   16 +-
+ drivers/target/target_core_spc.c             |   11 +-
+ drivers/target/target_core_transport.c       |   11 +-
+ drivers/target/target_core_ua.c              |   28 +-
+ drivers/target/target_core_ua.h              |    7 +-
+ drivers/ufs/Kconfig                          |    1 +
+ drivers/ufs/core/ufs-debugfs.c               |    4 +-
+ drivers/ufs/core/ufs-rpmb.c                  |   29 +-
+ drivers/ufs/core/ufs-sysfs.c                 |   12 +-
+ drivers/ufs/core/ufshcd.c                    |  189 ++-
+ drivers/ufs/host/ufs-mediatek.c              |    4 +-
+ drivers/ufs/host/ufs-qcom.c                  |   47 +-
+ drivers/ufs/host/ufs-qcom.h                  |    5 +-
+ drivers/ufs/host/ufs-sprd.c                  |    4 +-
+ drivers/usb/gadget/function/f_mass_storage.c |   34 +-
+ drivers/usb/gadget/function/storage_common.h |   69 +-
+ drivers/usb/storage/debug.c                  |   11 +-
+ drivers/usb/storage/debug.h                  |    4 +-
+ drivers/usb/storage/transport.c              |    7 +-
+ drivers/usb/storage/uas.c                    |   12 +-
+ drivers/usb/storage/usb.h                    |    4 +-
+ include/scsi/scsi_cmnd.h                     |    3 +-
+ include/scsi/scsi_common.h                   |   16 +-
+ include/scsi/scsi_dbg.h                      |    6 +-
+ include/scsi/scsi_device.h                   |   26 +-
+ include/scsi/scsi_host.h                     |    9 +-
+ include/scsi/scsi_proto.h                    |   71 +-
+ include/scsi/scsi_sense.h                    |  950 +++++++++++
+ include/trace/events/scsi.h                  |    4 +-
+ 162 files changed, 5925 insertions(+), 3412 deletions(-)
+ create mode 100644 include/scsi/scsi_sense.h
+Merging scsi-mkp/for-next (f09d2c7485b32 scsi: target: core: Use assign_bit() where applicable)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-mkp/for-next
+Auto-merging drivers/ata/libata-scsi.c
+Auto-merging drivers/scsi/mpi3mr/mpi3mr_transport.c
+CONFLICT (content): Merge conflict in drivers/scsi/mpi3mr/mpi3mr_transport.c
+Resolved 'drivers/scsi/mpi3mr/mpi3mr_transport.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 16ea30b5942d3] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/scsi/mpi3mr/mpi3mr_transport.c |   2 +-
+ drivers/target/target_core_user.c      |   5 +-
+ include/scsi/scsi_sense.h              | 236 ++++++++++++++++-----------------
+ 3 files changed, 120 insertions(+), 123 deletions(-)
+$ git am -3 ../patches/device-id-zorro
+Applying: foofof
+Using index info to reconstruct a base tree...
+M	include/linux/device-id/zorro.h
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+$ git am -3 ../patches/0001-libata-Fix-up-semantic-conflict-in-ata_scsi_set_sens.patch
+Applying: libata: Fix up semantic conflict in ata_scsi_set_sense()
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/ata/libata-scsi.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master efb88db687078] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git
+ Date: Wed Sep 30 13:37:18 2026 +0100
+Merging vhost/linux-next (8f2c2fb94a013 vhost/vsock: add VHOST_RESET_OWNER ioctl)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git vhost/linux-next
+Auto-merging arch/um/drivers/virtio_uml.c
+Auto-merging drivers/platform/mellanox/mlxbf-tmfifo.c
+Merge made by the 'ort' strategy.
+ arch/um/drivers/virtio_uml.c             | 10 ++++
+ drivers/platform/mellanox/mlxbf-tmfifo.c | 15 ++++++
+ drivers/remoteproc/remoteproc_core.c     | 12 +++++
+ drivers/remoteproc/remoteproc_virtio.c   | 39 +++++++++++---
+ drivers/s390/virtio/virtio_ccw.c         |  6 +--
+ drivers/vhost/net.c                      |  4 +-
+ drivers/vhost/scsi.c                     |  2 +-
+ drivers/vhost/test.c                     |  4 +-
+ drivers/vhost/vdpa.c                     |  4 +-
+ drivers/vhost/vhost.c                    | 26 ++++++++--
+ drivers/vhost/vhost.h                    |  5 +-
+ drivers/vhost/vsock.c                    | 89 +++++++++++++++++++++++++-------
+ drivers/virtio/virtio.c                  |  2 +
+ drivers/virtio/virtio_pci_legacy.c       |  2 -
+ drivers/virtio/virtio_pci_modern.c       | 15 +++---
+ drivers/virtio/virtio_vdpa.c             | 34 ++++++++++--
+ include/linux/remoteproc.h               |  3 ++
+ 17 files changed, 213 insertions(+), 59 deletions(-)
+Merging rpmsg/for-next (7fbd9a5319c33 Merge branches 'rpmsg-next', 'rproc-fixes' and 'rproc-next' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git rpmsg/for-next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/boot/dts/qcom/sdx75.dtsi
+Auto-merging drivers/remoteproc/remoteproc_core.c
+Auto-merging drivers/soc/qcom/Kconfig
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/remoteproc/mtk,scp.yaml    |  22 +-
+ .../bindings/remoteproc/qcom,pas-common.yaml       |   9 +
+ .../bindings/remoteproc/qcom,sm8550-pas.yaml       |  31 +-
+ .../bindings/remoteproc/ti,am3352-wkup-m3.yaml     |   1 -
+ Documentation/staging/rpmsg.rst                    |  17 +
+ MAINTAINERS                                        |   8 +
+ arch/arm64/boot/dts/qcom/sdx75.dtsi                |   3 +-
+ drivers/remoteproc/Kconfig                         |  11 +-
+ drivers/remoteproc/imx_dsp_rproc.c                 |   7 +-
+ drivers/remoteproc/imx_rproc.c                     |  51 +-
+ drivers/remoteproc/omap_remoteproc.c               |  12 +-
+ drivers/remoteproc/qcom_q6v5_adsp.c                |   2 +-
+ drivers/remoteproc/qcom_q6v5_pas.c                 | 166 +++++-
+ drivers/remoteproc/remoteproc_core.c               |   3 +
+ drivers/remoteproc/remoteproc_debugfs.c            |   6 +-
+ drivers/remoteproc/remoteproc_elf_loader.c         |  23 +-
+ drivers/remoteproc/remoteproc_sysfs.c              |   7 +-
+ drivers/remoteproc/stm32_rproc.c                   |   2 +-
+ drivers/remoteproc/ti_k3_common.c                  |  10 +-
+ drivers/remoteproc/ti_k3_common.h                  |   4 +
+ drivers/remoteproc/ti_k3_dsp_remoteproc.c          |   7 +-
+ drivers/remoteproc/ti_k3_m4_remoteproc.c           |   4 +-
+ drivers/remoteproc/ti_k3_r5_remoteproc.c           |   9 +-
+ drivers/remoteproc/xlnx_r5_remoteproc.c            |  13 +-
+ drivers/rpmsg/virtio_rpmsg_bus.c                   | 156 ++++--
+ drivers/soc/qcom/Kconfig                           |  12 +
+ drivers/soc/qcom/Makefile                          |   1 +
+ drivers/soc/qcom/qmi_tmd.c                         | 595 +++++++++++++++++++++
+ include/dt-bindings/thermal/qcom,pas.h             |  20 +
+ include/linux/omap-mailbox.h                       |   5 +-
+ include/linux/rpmsg/virtio_rpmsg.h                 |  41 ++
+ include/linux/soc/qcom/qmi.h                       |   1 +
+ include/linux/soc/qcom/qmi_tmd.h                   |  36 ++
+ include/uapi/linux/rpmsg.h                         |  15 +-
+ samples/rpmsg/rpmsg_client_sample.c                |  20 +-
+ 35 files changed, 1173 insertions(+), 157 deletions(-)
+ create mode 100644 drivers/soc/qcom/qmi_tmd.c
+ create mode 100644 include/dt-bindings/thermal/qcom,pas.h
+ create mode 100644 include/linux/rpmsg/virtio_rpmsg.h
+ create mode 100644 include/linux/soc/qcom/qmi_tmd.h
+Merging gpio-brgl/gpio/for-next (c4e74a7058b57 gpio: pxa: stop reading gpio_chip::base in direction callbacks)
+$ git merge -m Merge branch 'gpio/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl/gpio/for-next
+Auto-merging MAINTAINERS
+CONFLICT (content): Merge conflict in MAINTAINERS
+Auto-merging drivers/gpio/Kconfig
+Auto-merging drivers/gpio/Makefile
+Auto-merging drivers/gpio/gpio-mvebu.c
+Auto-merging drivers/gpio/gpiolib-acpi-quirks.c
+CONFLICT (content): Merge conflict in drivers/gpio/gpiolib-acpi-quirks.c
+Auto-merging drivers/gpio/gpiolib-cdev.c
+Auto-merging drivers/gpio/gpiolib.c
+Resolved 'MAINTAINERS' using previous resolution.
+Resolved 'drivers/gpio/gpiolib-acpi-quirks.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master ba4d7f800fca0] Merge branch 'gpio/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git
+$ git diff -M --stat --summary HEAD^..
+ .../bindings/gpio/axiado,ax3005-sgpio.yaml         | 112 ++++
+ .../devicetree/bindings/gpio/fsl,qoriq-gpio.yaml   |   7 +
+ .../devicetree/bindings/gpio/gpio-zynq.yaml        |   6 +
+ .../bindings/gpio/nvidia,tegra186-gpio.yaml        |   9 +
+ .../bindings/gpio/realtek,otto-gpio.yaml           |   2 +
+ .../bindings/gpio/renesas,rcar-gpio.yaml           |   5 +
+ Documentation/driver-api/gpio/board.rst            |   2 +-
+ Documentation/driver-api/gpio/driver.rst           |   7 +
+ MAINTAINERS                                        |   9 +
+ drivers/gpio/Kconfig                               |  33 +
+ drivers/gpio/Makefile                              |   2 +
+ drivers/gpio/gpio-ad7768.c                         | 121 ++++
+ drivers/gpio/gpio-adp5585.c                        |  20 +-
+ drivers/gpio/gpio-altera.c                         |   5 +-
+ drivers/gpio/gpio-axiado-sgpio.c                   | 736 +++++++++++++++++++++
+ drivers/gpio/gpio-dln2.c                           |   6 +-
+ drivers/gpio/gpio-eic-sprd.c                       |  17 +-
+ drivers/gpio/gpio-it87.c                           |   2 +-
+ drivers/gpio/gpio-mlxbf2.c                         |   4 +-
+ drivers/gpio/gpio-mlxbf3.c                         |   4 +-
+ drivers/gpio/gpio-mmio.c                           |  65 +-
+ drivers/gpio/gpio-mvebu.c                          |  19 +-
+ drivers/gpio/gpio-omap.c                           |  11 +-
+ drivers/gpio/gpio-pcf857x.c                        |  16 +-
+ drivers/gpio/gpio-pxa.c                            |   4 +-
+ drivers/gpio/gpio-rcar.c                           |  86 ++-
+ drivers/gpio/gpio-realtek-otto.c                   |  21 +-
+ drivers/gpio/gpio-regmap.c                         |  98 ++-
+ drivers/gpio/gpio-rtd1625.c                        |  26 +-
+ drivers/gpio/gpio-siox.c                           |   4 +-
+ drivers/gpio/gpio-sloppy-logic-analyzer.c          |   4 +-
+ drivers/gpio/gpio-tegra186.c                       |  74 +++
+ drivers/gpio/gpiolib-acpi-quirks.c                 |  13 +
+ drivers/gpio/gpiolib-cdev.c                        |   2 +
+ drivers/gpio/gpiolib-kunit.c                       |  14 +-
+ drivers/gpio/gpiolib.c                             |  21 +
+ drivers/pinctrl/freescale/pinctrl-imx.c            |  54 +-
+ drivers/pinctrl/freescale/pinctrl-imx.h            |   4 +
+ drivers/pinctrl/freescale/pinctrl-vf610.c          |   2 +
+ include/linux/gpio/driver.h                        |   9 +
+ include/linux/gpio/regmap.h                        |   2 +
+ include/linux/pinctrl/consumer.h                   |   4 +-
+ tools/gpio/gpio-hammer.c                           |   2 +-
+ 43 files changed, 1532 insertions(+), 132 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/gpio/axiado,ax3005-sgpio.yaml
+ create mode 100644 drivers/gpio/gpio-ad7768.c
+ create mode 100644 drivers/gpio/gpio-axiado-sgpio.c
+Merging gpio-intel/for-next (0fc424b6a8ef4 gpio: wcove: use regmap_assign_bits() for conditional set/clear)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel/for-next
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-wcove.c | 5 +----
+ 1 file changed, 1 insertion(+), 4 deletions(-)
+Merging pinctrl/for-next (c34fbaaac34ba Merge branch 'devel' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git pinctrl/for-next
+Merge made by the 'ort' strategy.
+ .../pinctrl/allwinner,sun55i-a523-pinctrl.yaml     |    6 +-
+ .../bindings/pinctrl/awinic,aw9523-pinctrl.yaml    |   50 +-
+ .../bindings/pinctrl/cix,sky1-pinctrl.yaml         |    2 +-
+ .../bindings/pinctrl/econet,en7528-pinctrl.yaml    |  190 +++
+ .../bindings/pinctrl/fsl,imx35-pinctrl.yaml        |    4 +-
+ .../bindings/pinctrl/fsl,imx7d-pinctrl.yaml        |   12 +-
+ .../bindings/pinctrl/fsl,imx8m-pinctrl.yaml        |    4 +-
+ .../bindings/pinctrl/fsl,imx8ulp-pinctrl.yaml      |    4 +-
+ .../bindings/pinctrl/fsl,imx9-pinctrl.yaml         |    4 +-
+ .../devicetree/bindings/pinctrl/fsl,imxrt1050.yaml |    4 +-
+ .../devicetree/bindings/pinctrl/fsl,imxrt1170.yaml |    4 +-
+ .../devicetree/bindings/pinctrl/intel,lgm-io.yaml  |   14 +-
+ .../bindings/pinctrl/mediatek,mt8183-pinctrl.yaml  |    4 +-
+ .../bindings/pinctrl/mediatek,mt8195-pinctrl.yaml  |    2 +-
+ .../devicetree/bindings/pinctrl/pinctrl-palmas.txt |  105 --
+ .../bindings/pinctrl/qcom,hawi-tlmm.yaml           |    4 +-
+ .../bindings/pinctrl/qcom,kaanapali-tlmm.yaml      |    4 +-
+ .../bindings/pinctrl/qcom,maili-tlmm.yaml          |    4 +-
+ .../bindings/pinctrl/renesas,rzg2l-pinctrl.yaml    |    4 +-
+ .../bindings/pinctrl/semtech,sx1501q.yaml          |    4 +-
+ .../bindings/pinctrl/ti,palmas-pinctrl.yaml        |  170 +++
+ .../bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml      |   46 +-
+ drivers/pinctrl/airoha/Kconfig                     |   15 +-
+ drivers/pinctrl/airoha/Makefile                    |    1 +
+ drivers/pinctrl/airoha/airoha-common.h             |    3 +
+ drivers/pinctrl/airoha/pinctrl-airoha.c            |   33 +-
+ drivers/pinctrl/airoha/pinctrl-an7563.c            |    1 +
+ drivers/pinctrl/airoha/pinctrl-an7581.c            |    1 +
+ drivers/pinctrl/airoha/pinctrl-an7583.c            |    1 +
+ drivers/pinctrl/airoha/pinctrl-en7523.c            |    1 +
+ drivers/pinctrl/airoha/pinctrl-en7528.c            | 1204 ++++++++++++++++++++
+ drivers/pinctrl/core.c                             |    4 +-
+ drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c     |    2 +-
+ drivers/pinctrl/pinconf.c                          |    2 +-
+ drivers/pinctrl/pinctrl-amd.c                      |    8 +-
+ drivers/pinctrl/pinctrl-equilibrium.c              |    2 +-
+ drivers/pinctrl/pinctrl-st.c                       |    2 +-
+ drivers/pinctrl/pinmux.c                           |    2 +-
+ drivers/pinctrl/renesas/pinctrl-rzg2l.c            |  412 ++++---
+ drivers/pinctrl/renesas/pinctrl-rzt2h.c            |  182 +--
+ drivers/pinctrl/stm32/Kconfig                      |    2 +-
+ drivers/pinctrl/sunplus/Kconfig                    |    8 +-
+ drivers/pinctrl/sunxi/Kconfig                      |    5 +
+ drivers/pinctrl/sunxi/Makefile                     |    1 +
+ drivers/pinctrl/sunxi/pinctrl-sun20i-d1.c          |    2 +-
+ drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c      |    3 +-
+ drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c        |    2 +-
+ drivers/pinctrl/sunxi/pinctrl-sun60i-a733.c        |   50 +
+ drivers/pinctrl/sunxi/pinctrl-sunxi.c              |   41 +-
+ drivers/pinctrl/sunxi/pinctrl-sunxi.h              |   71 +-
+ include/linux/pinctrl/consumer.h                   |    1 +
+ 51 files changed, 2258 insertions(+), 449 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/econet,en7528-pinctrl.yaml
+ delete mode 100644 Documentation/devicetree/bindings/pinctrl/pinctrl-palmas.txt
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/ti,palmas-pinctrl.yaml
+ create mode 100644 drivers/pinctrl/airoha/pinctrl-en7528.c
+ create mode 100644 drivers/pinctrl/sunxi/pinctrl-sun60i-a733.c
+Merging pinctrl-intel/for-next (c016587866e57 pinctrl: intel: Try to retrieve driver data for pure platform drivers)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel/for-next
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/intel/pinctrl-intel.c | 51 ++++++++++++++++++++++++-----------
+ drivers/pinctrl/intel/pinctrl-intel.h |  2 +-
+ 2 files changed, 36 insertions(+), 17 deletions(-)
+Merging pinctrl-renesas/renesas-pinctrl (0cd4a7b3a4883 pinctrl: renesas: rzt2h: Add a helper for reading PFC)
+$ git merge -m Merge branch 'renesas-pinctrl' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git pinctrl-renesas/renesas-pinctrl
+Already up to date.
+Merging pinctrl-samsung/for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung/for-next
+Already up to date.
+Merging pinctrl-qcom/pinctrl-qcom/for-next (4d7c9430a26ae pinctrl: qcom: tlmm-test: Add const to reg_names allocation type)
+$ git merge -m Merge branch 'pinctrl-qcom/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pinctrl-qcom/pinctrl-qcom/for-next
+Auto-merging drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
+Merge made by the 'ort' strategy.
+ .../pinctrl/qcom,hawi-lpass-lpi-pinctrl.yaml       |  109 ++
+ .../bindings/pinctrl/qcom,kuno-tlmm.yaml           |  110 ++
+ .../bindings/pinctrl/qcom,msm8952-pinctrl.yaml     |  146 +++
+ .../bindings/pinctrl/qcom,pmic-gpio.yaml           |    3 +
+ .../bindings/pinctrl/qcom,sc8280xp-tlmm.yaml       |    3 +
+ drivers/pinctrl/qcom/Kconfig                       |   10 +
+ drivers/pinctrl/qcom/Kconfig.msm                   |   19 +
+ drivers/pinctrl/qcom/Makefile                      |    3 +
+ drivers/pinctrl/qcom/pinctrl-apq8064.c             |  188 +--
+ drivers/pinctrl/qcom/pinctrl-apq8084.c             |  502 ++++----
+ drivers/pinctrl/qcom/pinctrl-hawi-lpass-lpi.c      |  244 ++++
+ drivers/pinctrl/qcom/pinctrl-ipq4019.c             |  208 ++--
+ drivers/pinctrl/qcom/pinctrl-ipq8064.c             |  208 ++--
+ drivers/pinctrl/qcom/pinctrl-kuno.c                |  801 +++++++++++++
+ drivers/pinctrl/qcom/pinctrl-lpass-lpi.c           |    6 +-
+ drivers/pinctrl/qcom/pinctrl-lpass-lpi.h           |   17 +
+ drivers/pinctrl/qcom/pinctrl-msm.h                 |   25 -
+ drivers/pinctrl/qcom/pinctrl-msm8952.c             | 1257 ++++++++++++++++++++
+ drivers/pinctrl/qcom/pinctrl-spmi-gpio.c           |    1 +
+ drivers/pinctrl/qcom/tlmm-test.c                   |    2 +-
+ 20 files changed, 3282 insertions(+), 580 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,hawi-lpass-lpi-pinctrl.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,kuno-tlmm.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,msm8952-pinctrl.yaml
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-hawi-lpass-lpi.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-kuno.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-msm8952.c
+Merging pwm/pwm/for-next (e74b9a7ee5007 pwm: tiehrpwm: Clear period on disable to allow reconfiguration)
+$ git merge -m Merge branch 'pwm/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git pwm/pwm/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/pwm/pwm-tipwmss.txt        | 58 ---------------
+ .../devicetree/bindings/pwm/ti,am33xx-pwmss.yaml   | 82 ++++++++++++++++++++++
+ Documentation/driver-api/pwm.rst                   |  2 +-
+ drivers/pwm/core.c                                 |  1 +
+ drivers/pwm/pwm-brcmstb.c                          |  2 +-
+ drivers/pwm/pwm-ipq.c                              |  2 +-
+ drivers/pwm/pwm-iqs620a.c                          | 23 +-----
+ drivers/pwm/pwm-loongson.c                         |  2 +
+ drivers/pwm/pwm-lp3943.c                           |  3 +
+ drivers/pwm/pwm-mtk-disp.c                         |  4 +-
+ drivers/pwm/pwm-pca9685.c                          |  2 +-
+ drivers/pwm/pwm-renesas-tpu.c                      | 23 ++++--
+ drivers/pwm/pwm-tiehrpwm.c                         |  3 +
+ 13 files changed, 118 insertions(+), 89 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/pwm/pwm-tipwmss.txt
+ create mode 100644 Documentation/devicetree/bindings/pwm/ti,am33xx-pwmss.yaml
+Merging ktest/for-next (932cdaf3e273a ktest: Add logfile to failure directory)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git ktest/for-next
+Already up to date.
+Merging kselftest/next (30af56a227e27 selftests/ftrace: skip gcov symbols when picking a function to probe)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest/next
+Merge made by the 'ort' strategy.
+ .../ftrace/test.d/kprobe/kprobe_eventname.tc       |   2 +-
+ tools/testing/selftests/resctrl/cat_test.c         |  23 ++-
+ tools/testing/selftests/resctrl/mba_test.c         |   1 +
+ tools/testing/selftests/resctrl/mbm_test.c         |   1 +
+ tools/testing/selftests/resctrl/resctrl.h          |   2 +
+ tools/testing/selftests/resctrl/resctrl_val.c      | 169 +++++++++++----------
+ 6 files changed, 115 insertions(+), 83 deletions(-)
+Merging kunit/test (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'test' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit/test
+Already up to date.
+Merging kunit-next/kunit (e38f53f048246 kunit: Return void from kunit_run_all_tests())
+$ git merge -m Merge branch 'kunit' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-next/kunit
+Merge made by the 'ort' strategy.
+ include/kunit/test.h | 5 ++---
+ lib/kunit/executor.c | 5 ++---
+ 2 files changed, 4 insertions(+), 6 deletions(-)
+Merging livepatching/for-next (5d791d3396ca4 selftests/livepatch: filter debug messages in check_result())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git livepatching/for-next
+Merge made by the 'ort' strategy.
+ include/linux/livepatch.h                      |  4 ++++
+ kernel/livepatch/core.c                        | 10 ++++++++++
+ tools/testing/selftests/livepatch/functions.sh | 12 +++++++++---
+ 3 files changed, 23 insertions(+), 3 deletions(-)
+Merging rtc/rtc-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'rtc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc/rtc-next
+Already up to date.
+Merging nvdimm/libnvdimm-for-next (e99cb3ecd8334 nvdimm-btt: clean up kernel-doc warnings)
+$ git merge -m Merge branch 'libnvdimm-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm/libnvdimm-for-next
+Already up to date.
+Merging at24/at24/for-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'at24/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git at24/at24/for-next
+Already up to date.
+Merging ntb/ntb-next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'ntb-next' of https://github.com/jonmason/ntb.git ntb/ntb-next
+Already up to date.
+Merging seccomp/for-next/seccomp (832b9b176be06 seccomp: restore knotif->state when SECCOMP_ADDFD_FLAG_SEND is interrupted)
+$ git merge -m Merge branch 'for-next/seccomp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git seccomp/for-next/seccomp
+Merge made by the 'ort' strategy.
+ kernel/seccomp.c                              |   7 +-
+ tools/testing/selftests/seccomp/seccomp_bpf.c | 179 ++++++++++++++++++++++++++
+ 2 files changed, 184 insertions(+), 2 deletions(-)
+Merging slimbus/for-next (4350d70546699 Merge branch 'slimbus-for-v7.4' into slimbus-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git slimbus/for-next
+Merge made by the 'ort' strategy.
+ drivers/slimbus/messaging.c     |  2 +-
+ drivers/slimbus/qcom-ngd-ctrl.c | 82 ++++++++++++++++++++++++++++++++++++++++-
+ drivers/slimbus/slimbus.h       | 14 ++++++-
+ include/linux/slimbus.h         |  2 +-
+ 4 files changed, 95 insertions(+), 5 deletions(-)
+Merging nvmem/for-next (41f42ff4ae134 Merge branches 'nvmem-fixes' and 'nvmem-for-v7.4' into nvmem-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git nvmem/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mfd/mediatek,mt6397.yaml   |  21 ++++
+ .../devicetree/bindings/nvmem/qcom,qfprom.yaml     |   1 +
+ .../bindings/nvmem/socionext,uniphier-efuse.yaml   |   6 +-
+ MAINTAINERS                                        |   5 +
+ drivers/nvmem/Kconfig                              |  14 ++-
+ drivers/nvmem/Makefile                             |   2 +
+ drivers/nvmem/core.c                               | 128 ++++++++++++---------
+ drivers/nvmem/internals.h                          |   5 +-
+ drivers/nvmem/layouts.c                            |  13 ++-
+ drivers/nvmem/layouts/onie-tlv.c                   |   1 +
+ drivers/nvmem/layouts/sl28vpd.c                    |   1 +
+ drivers/nvmem/mt6323-efuse.c                       |  83 +++++++++++++
+ drivers/nvmem/rockchip-otp.c                       |  15 ++-
+ 13 files changed, 236 insertions(+), 59 deletions(-)
+ create mode 100644 drivers/nvmem/mt6323-efuse.c
+Merging hyperv/hyperv-next (be0cfab740e58 clocksource: hyper-v: Remove support for stimer interrupts in message mode)
+$ git merge -m Merge branch 'hyperv-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv/hyperv-next
+Already up to date.
+Merging auxdisplay/for-next (f63dc0eea92d0 docs: ABI: auxdisplay: use literal blocks for commands)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay/for-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-auxdisplay-linedisp | 9 ++++++---
+ drivers/auxdisplay/arm-charlcd.c                    | 7 ++-----
+ drivers/auxdisplay/panel.c                          | 5 +----
+ 3 files changed, 9 insertions(+), 12 deletions(-)
+Merging kgdb/kgdb/for-next (fdbdd0ccb30af kdb: remove redundant check for scancode 0xe0)
+$ git merge -m Merge branch 'kgdb/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git kgdb/kgdb/for-next
+Already up to date.
+Merging hmm/hmm (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'hmm' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git hmm/hmm
+Already up to date.
+Merging cfi/cfi/next (dc59e4fea9d83 Linux 7.2-rc1)
+$ git merge -m Merge branch 'cfi/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git cfi/cfi/next
+Already up to date.
+Merging mhi/mhi-next (710bf7329abf8 bus: mhi: host: pci_generic: Add DUN channel configuration)
+$ git merge -m Merge branch 'mhi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git mhi/mhi-next
+Merge made by the 'ort' strategy.
+ drivers/bus/mhi/ep/main.c          | 4 ++--
+ drivers/bus/mhi/host/debugfs.c     | 4 ++--
+ drivers/bus/mhi/host/pci_generic.c | 9 ++++++++-
+ include/linux/mhi.h                | 2 +-
+ 4 files changed, 13 insertions(+), 6 deletions(-)
+$ git am -3 ../patches/0001-fix-up-for-net-qrtr-Drop-the-MHI-auto_queue-feature-.patch
+Applying: fix up for "net: qrtr: Drop the MHI auto_queue feature for IPCR DL channels"
+Using index info to reconstruct a base tree...
+M	drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging cxl/next (71392a644e88c Merge branch 'for-7.4/cxl-misc' into cxl-for-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/cxl/core/ras.c
+Auto-merging include/linux/device.h
+Merge made by the 'ort' strategy.
+ .../driver-api/cxl/platform/acpi/dsdt.rst          |   3 +-
+ MAINTAINERS                                        |   1 +
+ drivers/cxl/acpi.c                                 |  10 +-
+ drivers/cxl/core/cdat.c                            |   8 +-
+ drivers/cxl/core/core.h                            |   8 +-
+ drivers/cxl/core/edac.c                            |  30 ++++--
+ drivers/cxl/core/features.c                        |  58 ++++++----
+ drivers/cxl/core/hdm.c                             | 120 ++++++++++++++++-----
+ drivers/cxl/core/mbox.c                            |   3 +
+ drivers/cxl/core/mce.c                             |   8 +-
+ drivers/cxl/core/pci.c                             |   9 ++
+ drivers/cxl/core/port.c                            |  39 +++++--
+ drivers/cxl/core/ras.c                             |   6 +-
+ drivers/cxl/core/region.c                          | 100 +++++++++++------
+ drivers/cxl/core/region_pmem.c                     |   6 +-
+ drivers/cxl/core/regs.c                            |  14 +++
+ drivers/cxl/cxl.h                                  |  12 +++
+ drivers/cxl/mem.c                                  |  14 ++-
+ drivers/cxl/pci.c                                  |   5 +-
+ drivers/cxl/port.c                                 |   3 +
+ include/linux/device.h                             |   1 +
+ tools/testing/cxl/Kbuild                           |  16 +--
+ tools/testing/cxl/test/cxl.c                       | 117 ++++++++++++++++----
+ 23 files changed, 443 insertions(+), 148 deletions(-)
+Merging zstd/zstd-next (65d1f5507ed2c zstd: Import upstream v1.5.7)
+$ git merge -m Merge branch 'zstd-next' of https://github.com/terrelln/linux.git zstd/zstd-next
+Already up to date.
+Merging efi/next (7eef16311a234 Merge remote-tracking branch 'linux-efi/bootloader-info' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi/next
+Auto-merging drivers/firmware/efi/libstub/Makefile
+Merge made by the 'ort' strategy.
+ drivers/firmware/efi/capsule-loader.c            |   6 +-
+ drivers/firmware/efi/capsule.c                   |   4 +-
+ drivers/firmware/efi/efi.c                       |  33 +++++-
+ drivers/firmware/efi/libstub/Makefile            |   3 +-
+ drivers/firmware/efi/libstub/bli.c               |  87 ++++++++++++++
+ drivers/firmware/efi/libstub/efi-stub-entry.c    |   2 +-
+ drivers/firmware/efi/libstub/efi-stub-helper.c   | 108 +++++++----------
+ drivers/firmware/efi/libstub/efi-stub.c          |   3 +-
+ drivers/firmware/efi/libstub/efistub.h           |  12 +-
+ drivers/firmware/efi/libstub/file.c              |   8 +-
+ drivers/firmware/efi/libstub/gop.c               |  23 ++--
+ drivers/firmware/efi/libstub/kaslr.c             |  19 ++-
+ drivers/firmware/efi/libstub/mem.c               |   2 +-
+ drivers/firmware/efi/libstub/pci.c               |   2 +-
+ drivers/firmware/efi/libstub/printk.c            |  95 ++-------------
+ drivers/firmware/efi/libstub/random.c            |   8 +-
+ drivers/firmware/efi/libstub/riscv.c             |   2 +-
+ drivers/firmware/efi/libstub/smbios.c            |   3 +-
+ drivers/firmware/efi/libstub/tpm.c               |   8 +-
+ drivers/firmware/efi/libstub/unaccepted_memory.c |   2 +-
+ drivers/firmware/efi/libstub/vsprintf.c          | 140 ++++++++---------------
+ drivers/firmware/efi/libstub/x86-stub.c          |  18 +--
+ drivers/firmware/efi/libstub/zboot.c             |   3 +-
+ drivers/firmware/efi/runtime-wrappers.c          |  28 +++++
+ include/linux/efi.h                              |  23 ++++
+ include/linux/ucs2_string.h                      |  11 +-
+ lib/ucs2_string.c                                |  19 +--
+ 27 files changed, 360 insertions(+), 312 deletions(-)
+ create mode 100644 drivers/firmware/efi/libstub/bli.c
+Merging unicode/for-next (a511442085c14 unicode: Properly reject invalid encoding version strings)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git unicode/for-next
+Already up to date.
+Merging random/master (703749b069d53 random: fix vgetrandom_opaque_params kernel-doc)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git random/master
+Merge made by the 'ort' strategy.
+ drivers/virt/vmgenid.c      | 11 +++++++----
+ include/linux/siphash.h     |  8 ++++++--
+ include/uapi/linux/random.h |  2 +-
+ include/vdso/getrandom.h    |  2 +-
+ 4 files changed, 15 insertions(+), 8 deletions(-)
+Merging landlock/next (02619311dbfc4 Merge branch 'lsm/dev' into master)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git landlock/next
+Merge made by the 'ort' strategy.
+Merging sysctl/sysctl-next (4991c8b72b564 time/jiffies: Saturate in mult_hz() instead of wrapping)
+$ git merge -m Merge branch 'sysctl-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git sysctl/sysctl-next
+Auto-merging fs/dcache.c
+Auto-merging fs/proc/proc_sysctl.c
+Auto-merging kernel/time/jiffies.c
+Merge made by the 'ort' strategy.
+ drivers/parport/procfs.c                 |    6 +-
+ fs/dcache.c                              |    2 +-
+ fs/file_table.c                          |    2 +-
+ fs/proc/proc_sysctl.c                    |    2 +-
+ kernel/sysctl.c                          |  429 +++++----
+ kernel/time/jiffies.c                    |    2 +
+ lib/test_sysctl.c                        |   33 +-
+ tools/testing/selftests/sysctl/sysctl.sh | 1546 ++++++++++++------------------
+ 8 files changed, 903 insertions(+), 1119 deletions(-)
+Merging execve/for-next/execve (df2908090cda3 Linux 7.3-rc2)
+$ git merge -m Merge branch 'for-next/execve' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git execve/for-next/execve
+Already up to date.
+Merging bitmap/bitmap-for-next (452a6d5b5e055 bitmap: test bitmap_parselist() with a group size close to UINT_MAX)
+$ git merge -m Merge branch 'bitmap-for-next' of https://github.com/norov/linux.git bitmap/bitmap-for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                              |   2 +
+ include/linux/bitfield-fix-width.h       | 241 +++++++++++++++++++++++++++++++
+ include/linux/bitfield.h                 |  49 +------
+ lib/bitmap-str.c                         |  24 ++-
+ lib/test_bitmap.c                        |   9 ++
+ tools/include/linux/bitfield-fix-width.h | 236 ++++++++++++++++++++++++++++++
+ tools/include/linux/bitfield.h           |  49 +------
+ 7 files changed, 509 insertions(+), 101 deletions(-)
+ create mode 100644 include/linux/bitfield-fix-width.h
+ create mode 100644 tools/include/linux/bitfield-fix-width.h
+Merging hte/for-next (30167fadbf87f hte: tegra194-test: shut down timer before GPIO release)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git hte/for-next
+Merge made by the 'ort' strategy.
+ drivers/hte/hte-tegra194-test.c |  2 +-
+ drivers/hte/hte-tegra194.c      | 14 ++++++--------
+ 2 files changed, 7 insertions(+), 9 deletions(-)
+Merging kspp/for-next/kspp (760f96b7f54be um: fix CONFIG_GCOV for built-in code)
+$ git merge -m Merge branch 'for-next/kspp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git kspp/for-next/kspp
+Auto-merging arch/um/include/asm/common.lds.S
+Merge made by the 'ort' strategy.
+ arch/um/include/asm/common.lds.S              |   1 +
+ drivers/misc/lkdtm/core.c                     |  16 +--
+ fs/signalfd.c                                 |  28 ++++-
+ include/linux/fortify-string.h                |   2 -
+ scripts/coccinelle/api/kmalloc_objs.cocci     | 161 +++++++++++++++++++++-----
+ scripts/gcc-plugins/randomize_layout_plugin.c |  63 +++++++++-
+ 6 files changed, 219 insertions(+), 52 deletions(-)
+Merging nolibc/for-next (da27722051fdf tools/nolibc: add support for pread() and pwrite())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git nolibc/for-next
+Auto-merging tools/testing/selftests/nolibc/Makefile.nolibc
+Merge made by the 'ort' strategy.
+ tools/include/nolibc/Makefile                  |  19 ++-
+ tools/include/nolibc/arch-arm.h                |   7 +-
+ tools/include/nolibc/arch-hexagon.h            | 159 +++++++++++++++++++++++++
+ tools/include/nolibc/arch-mips.h               |   8 +-
+ tools/include/nolibc/arch-parisc.h             |  28 ++++-
+ tools/include/nolibc/arch-powerpc.h            |   7 +-
+ tools/include/nolibc/arch-sh.h                 |  18 +++
+ tools/include/nolibc/arch-x86.h                |   2 +
+ tools/include/nolibc/arch.h                    |   7 ++
+ tools/include/nolibc/compiler.h                |   9 ++
+ tools/include/nolibc/dirent.h                  |  25 +++-
+ tools/include/nolibc/err.h                     |   2 +-
+ tools/include/nolibc/nolibc.h                  |   1 +
+ tools/include/nolibc/stdlib.h                  |   2 +-
+ tools/include/nolibc/sys.h                     |  60 ++++++++--
+ tools/include/nolibc/sys/select.h              |   6 +-
+ tools/include/nolibc/sys/sendfile.h            |  41 +++++++
+ tools/include/nolibc/types.h                   |   2 +-
+ tools/include/nolibc/unistd.h                  |   4 +
+ tools/testing/selftests/nolibc/Makefile.nolibc |  15 ++-
+ tools/testing/selftests/nolibc/nolibc-test.c   | 159 +++++++++++++++++++++++--
+ tools/testing/selftests/nolibc/run-tests.sh    |  30 ++++-
+ 22 files changed, 554 insertions(+), 57 deletions(-)
+ create mode 100644 tools/include/nolibc/arch-hexagon.h
+ create mode 100644 tools/include/nolibc/sys/sendfile.h
+Merging iommufd/for-next (54dadb030c7e2 iommufd: Fix iommufd_hw_capabilities reference in kernel-doc)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd/for-next
+Auto-merging drivers/iommu/intel/nested.c
+Merge made by the 'ort' strategy.
+ .../iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c    | 199 ++++++++++++++++-----
+ drivers/iommu/intel/nested.c                       |  50 +++---
+ drivers/iommu/iommufd/hw_pagetable.c               |  25 +--
+ drivers/iommu/iommufd/selftest.c                   | 163 ++++++++---------
+ include/linux/iommu.h                              |   6 +-
+ include/linux/iommufd.h                            |   2 +
+ include/uapi/linux/iommufd.h                       |   6 +-
+ tools/testing/selftests/iommu/iommufd.c            |  14 +-
+ tools/testing/selftests/iommu/iommufd_fail_nth.c   |  14 +-
+ 9 files changed, 315 insertions(+), 164 deletions(-)
+Merging turbostat/next (ccdfcb7e7ab84 turbostat 2026.09.28)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git turbostat/next
+Merge made by the 'ort' strategy.
+ tools/power/x86/turbostat/turbostat.c | 36 ++++++++++++++++++++++-------------
+ 1 file changed, 23 insertions(+), 13 deletions(-)
+Merging pwrseq/pwrseq/for-next (09baa2f4caab0 Merge tag 'pwrseq-is-controllable-for-v7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux into pwrseq/for-next)
+$ git merge -m Merge branch 'pwrseq/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq/pwrseq/for-next
+Auto-merging drivers/power/sequencing/pwrseq-pcie-m2.c
+Merge made by the 'ort' strategy.
+ drivers/power/sequencing/Kconfig                 |   19 +
+ drivers/power/sequencing/Makefile                |    2 +
+ drivers/power/sequencing/core.c                  |   49 +
+ drivers/power/sequencing/pwrseq-kunit.c          | 1497 ++++++++++++++++++++++
+ drivers/power/sequencing/pwrseq-pcie-m2.c        |   45 +-
+ drivers/power/sequencing/pwrseq-qcom-wcn.c       |   30 +
+ drivers/power/sequencing/pwrseq-renesas-pwrrdy.c |  142 ++
+ include/linux/pwrseq/consumer.h                  |    7 +
+ include/linux/pwrseq/provider.h                  |    8 +
+ 9 files changed, 1797 insertions(+), 2 deletions(-)
+ create mode 100644 drivers/power/sequencing/pwrseq-kunit.c
+ create mode 100644 drivers/power/sequencing/pwrseq-renesas-pwrrdy.c
+Merging capabilities-next/caps-next (507adb6448378 security: commoncap: clarify CAP_FS_SET comment in cap_task_fix_setuid())
+$ git merge -m Merge branch 'caps-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git capabilities-next/caps-next
+Auto-merging security/commoncap.c
+Merge made by the 'ort' strategy.
+ security/commoncap.c | 10 ++++++----
+ 1 file changed, 6 insertions(+), 4 deletions(-)
+$ git am -3 ../patches/0001-sign-file-Fix-up-merge-issue.patch
+Applying: sign-file: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	scripts/sign-file.c
+Falling back to patching base and 3-way merge...
+Auto-merging scripts/sign-file.c
+No changes -- Patch already applied.
+Merging ipe/next (bcaa4d1d69368 ipe: fix invalid sgid value in audit event documentation)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git ipe/next
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/LSM/ipe.rst | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging kcsan/next (a8488ecbd7ba4 kcsan: avoid unintended access checking in NMIs)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git kcsan/next
+Already up to date.
+Merging crc/crc-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'crc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc/crc-next
+Already up to date.
+Merging keys-next/keys-next (965e9a2cf23b0 pkcs7: Change a pr_warn() to pr_warn_once())
+$ git merge -m Merge branch 'keys-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git keys-next/keys-next
+Already up to date.
+Merging fwctl/for-next (cee9395acd804 Linux 7.3-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl/for-next
+Already up to date.
+Merging devsec-tsm/next (3177779ae17db virt: coco: change tsm_class to a const struct)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm/next
+Already up to date.
+Merging hisilicon/for-next (a5db65458a911 Merge branch 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/hisilicon/linux-hisi.git hisilicon/for-next
+Merge made by the 'ort' strategy.
+Merging device-id/device-id-rework (995832b2cebe6 Replace <linux/mod_devicetable.h> by more specific <linux/device-id/*.h> (c files))
+$ git merge -m Merge branch 'device-id-rework' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git device-id/device-id-rework
+Already up to date.
+Merging kthread/for-next (fa39ec4f89f26 doc: Add housekeeping documentation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git kthread/for-next
+Already up to date.
+Merging pagemap-headers/headers (e02cb91d4644d ksm: Remove pagemap.h include)
+$ git merge -m Merge branch 'headers' of git://git.infradead.org/users/willy/pagecache.git pagemap-headers/headers
+Auto-merging arch/alpha/include/asm/pgtable.h
+Auto-merging arch/arc/include/asm/pgtable-levels.h
+Auto-merging arch/microblaze/include/asm/pgtable.h
+Auto-merging arch/sparc/include/asm/pgtable_64.h
+Auto-merging arch/sparc/include/asm/tlb_64.h
+Auto-merging arch/x86/virt/vmx/tdx/tdx.c
+Auto-merging block/fops.c
+Auto-merging drivers/dma/ste_dma40.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+Auto-merging drivers/gpu/drm/i915/display/intel_display_power.c
+Auto-merging drivers/gpu/drm/msm/msm_fb.c
+Auto-merging drivers/gpu/drm/msm/msm_gem_shrinker.c
+Auto-merging drivers/gpu/drm/nouveau/nouveau_dmem.c
+Auto-merging drivers/gpu/drm/omapdrm/dss/hdmi4.c
+Auto-merging drivers/gpu/drm/omapdrm/dss/hdmi5.c
+Auto-merging drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c
+Auto-merging drivers/gpu/drm/panthor/panthor_gem.c
+Auto-merging drivers/hwmon/pmbus/pmbus_core.c
+Auto-merging drivers/i2c/busses/i2c-amd-asf-plat.c
+Auto-merging drivers/i2c/busses/i2c-gxp.c
+Auto-merging drivers/i2c/busses/i2c-k1.c
+Auto-merging drivers/iio/accel/fxls8962af-core.c
+Auto-merging drivers/iio/adc/ti-ads1298.c
+Auto-merging drivers/iio/light/rohm-bu27034.c
+Auto-merging drivers/iio/pressure/bmp280-core.c
+Auto-merging drivers/leds/leds-turris-omnia.c
+Auto-merging drivers/media/platform/st/stm32/stm32-csi.c
+Auto-merging drivers/media/platform/ti/omap3isp/ispvideo.c
+Auto-merging drivers/media/usb/go7007/go7007-v4l2.c
+Auto-merging drivers/media/usb/gspca/gspca.c
+Auto-merging drivers/mfd/88pm886.c
+Auto-merging drivers/mfd/cs42l43.c
+Auto-merging drivers/mfd/twl-core.c
+Auto-merging drivers/mmc/core/core.c
+Auto-merging drivers/mmc/core/host.c
+Auto-merging drivers/mmc/host/sh_mmcif.c
+Auto-merging drivers/net/ethernet/atheros/atl1e/atl1e.h
+Auto-merging drivers/pinctrl/pinctrl-sx150x.c
+Auto-merging drivers/platform/arm64/huawei-gaokun-ec.c
+Auto-merging drivers/platform/arm64/lenovo-yoga-c630.c
+Auto-merging drivers/power/supply/bq25630_charger.c
+Auto-merging drivers/regulator/fixed.c
+Auto-merging drivers/regulator/tps65185.c
+Auto-merging drivers/ufs/host/ufs-mediatek.c
+Auto-merging drivers/ufs/host/ufs-qcom.c
+Auto-merging drivers/usb/typec/mux/it5205.c
+Auto-merging fs/aio.c
+Auto-merging fs/buffer.c
+Auto-merging fs/file_table.c
+Auto-merging fs/inode.c
+Auto-merging fs/nfs/blocklayout/blocklayout.c
+Auto-merging fs/nfs/nfstrace.h
+Auto-merging fs/nfs/pnfs.c
+Auto-merging fs/nfsd/nfs4proc.c
+Auto-merging include/drm/drm_print.h
+Auto-merging include/linux/ceph/libceph.h
+Auto-merging include/linux/i3c/master.h
+Auto-merging include/linux/nfs_fs.h
+Auto-merging include/linux/nfs_page.h
+Auto-merging include/linux/swap.h
+Auto-merging kernel/power/snapshot.c
+Auto-merging net/ceph/osd_client.c
+CONFLICT (content): Merge conflict in net/ceph/osd_client.c
+Auto-merging net/ceph/pagevec.c
+Auto-merging net/sunrpc/auth_gss/gss_krb5_crypto.c
+Auto-merging net/sunrpc/xdr.c
+Auto-merging net/sunrpc/xprtsock.c
+Auto-merging security/commoncap.c
+Auto-merging security/selinux/hooks.c
+Auto-merging security/selinux/selinuxfs.c
+Auto-merging security/smack/smack_lsm.c
+Resolved 'drivers/gpu/drm/amd/amdkfd/kfd_migrate.c' using previous resolution.
+Resolved 'net/ceph/osd_client.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 53d1b34b8f9fe] Merge branch 'headers' of git://git.infradead.org/users/willy/pagecache.git
+$ git diff -M --stat --summary HEAD^..
+ arch/alpha/include/asm/pgtable.h                   |  2 +-
+ arch/arc/include/asm/Kbuild                        |  1 +
+ arch/arc/include/asm/pgtable-levels.h              |  2 ++
+ arch/arc/include/asm/tlb.h                         | 12 ----------
+ arch/arm/include/asm/pgalloc.h                     |  2 --
+ arch/arm/include/asm/tlb.h                         |  2 --
+ arch/arm/mach-pxa/pxa3xx.c                         |  1 +
+ arch/arm64/include/asm/tlb.h                       |  3 ---
+ arch/hexagon/include/asm/tlb.h                     |  1 -
+ arch/microblaze/include/asm/pgtable.h              |  2 +-
+ arch/nios2/include/asm/tlb.h                       |  1 -
+ arch/openrisc/include/asm/Kbuild                   |  1 +
+ arch/openrisc/include/asm/tlb.h                    | 26 ----------------------
+ arch/powerpc/include/asm/tlb.h                     |  2 --
+ arch/sh/include/asm/pgtable.h                      |  2 +-
+ arch/sh/include/asm/tlb.h                          |  1 -
+ arch/sparc/include/asm/pgtable_64.h                |  2 +-
+ arch/sparc/include/asm/tlb_64.h                    |  1 -
+ arch/x86/include/asm/pgalloc.h                     |  1 -
+ arch/x86/virt/vmx/tdx/tdx.c                        |  1 +
+ arch/xtensa/kernel/hibernate.c                     |  1 +
+ block/fops.c                                       |  1 +
+ drivers/char/agp/backend.c                         |  1 -
+ drivers/char/agp/generic.c                         |  1 -
+ drivers/char/agp/intel-agp.c                       |  1 -
+ drivers/char/agp/intel-gtt.c                       |  1 -
+ drivers/char/agp/uninorth-agp.c                    |  3 ++-
+ drivers/dma/ste_dma40.c                            |  1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c   |  1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |  1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |  1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |  3 +--
+ drivers/gpu/drm/amd/amdkfd/kfd_migrate.c           |  1 +
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |  1 -
+ drivers/gpu/drm/display/drm_dp_cec.c               |  1 +
+ drivers/gpu/drm/etnaviv/etnaviv_gpu.c              |  1 +
+ drivers/gpu/drm/i915/display/intel_display_power.c |  1 +
+ drivers/gpu/drm/i915/gem/i915_gem_userptr.c        |  1 +
+ drivers/gpu/drm/msm/msm_fb.c                       |  2 ++
+ drivers/gpu/drm/msm/msm_gem_shrinker.c             |  2 ++
+ drivers/gpu/drm/nouveau/nouveau_dmem.c             |  1 +
+ drivers/gpu/drm/omapdrm/dss/hdmi4.c                |  1 +
+ drivers/gpu/drm/omapdrm/dss/hdmi5.c                |  1 +
+ drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c |  1 +
+ drivers/gpu/drm/panfrost/panfrost_gem.c            |  2 ++
+ drivers/gpu/drm/panthor/panthor_gem.c              |  3 +++
+ drivers/hwmon/pmbus/pmbus_core.c                   |  1 +
+ drivers/i2c/busses/i2c-amd-asf-plat.c              |  1 +
+ drivers/i2c/busses/i2c-gxp.c                       |  1 +
+ drivers/i2c/busses/i2c-k1.c                        |  1 +
+ drivers/i2c/busses/i2c-pasemi-platform.c           |  1 +
+ drivers/iio/accel/fxls8962af-core.c                |  1 +
+ drivers/iio/adc/nct7201.c                          |  1 +
+ drivers/iio/adc/ti-ads1298.c                       |  1 +
+ drivers/iio/light/rohm-bu27034.c                   |  1 +
+ drivers/iio/pressure/bmp280-core.c                 |  1 +
+ drivers/iio/pressure/bmp280.h                      |  1 +
+ drivers/input/misc/aw86927.c                       |  1 +
+ drivers/input/touchscreen/goodix_berlin_core.c     |  1 +
+ drivers/input/touchscreen/hynitron_cstxxx.c        |  1 +
+ drivers/input/touchscreen/imagis.c                 |  1 +
+ drivers/leds/leds-turris-omnia.c                   |  1 +
+ drivers/media/common/videobuf2/frame_vector.c      |  1 -
+ drivers/media/pci/cx18/cx18-driver.h               |  1 -
+ drivers/media/pci/ivtv/ivtv-driver.h               |  2 +-
+ drivers/media/platform/st/stm32/stm32-csi.c        |  1 +
+ drivers/media/platform/ti/omap3isp/ispvideo.c      |  1 -
+ drivers/media/usb/go7007/go7007-v4l2.c             |  1 -
+ drivers/media/usb/gspca/gspca.c                    |  1 -
+ drivers/mfd/88pm886.c                              |  1 +
+ drivers/mfd/abx500-core.c                          |  1 +
+ drivers/mfd/adp5585.c                              |  1 +
+ drivers/mfd/cs40l50-core.c                         |  1 +
+ drivers/mfd/cs42l43.c                              |  1 +
+ drivers/mfd/rt5120.c                               |  1 +
+ drivers/mfd/tps65219.c                             |  1 +
+ drivers/mfd/twl-core.c                             |  2 +-
+ drivers/mmc/core/core.c                            |  1 -
+ drivers/mmc/core/host.c                            |  1 -
+ drivers/mmc/host/renesas_sdhi_internal_dmac.c      |  1 -
+ drivers/mmc/host/renesas_sdhi_sys_dmac.c           |  1 -
+ drivers/mmc/host/sh_mmcif.c                        |  1 -
+ drivers/mmc/host/tmio_mmc.h                        |  1 -
+ drivers/mmc/host/tmio_mmc_core.c                   |  1 -
+ drivers/mmc/host/usdhi6rol0.c                      |  1 -
+ drivers/net/ethernet/atheros/atl1c/atl1c.h         |  1 -
+ drivers/net/ethernet/atheros/atl1e/atl1e.h         |  1 -
+ drivers/net/ethernet/intel/e1000/e1000.h           |  1 -
+ drivers/net/ethernet/intel/e1000e/netdev.c         |  1 -
+ drivers/net/ethernet/intel/igb/igb_main.c          |  1 -
+ drivers/net/ethernet/intel/igbvf/netdev.c          |  1 -
+ drivers/phy/st/phy-stm32-combophy.c                |  1 +
+ drivers/pinctrl/pinctrl-sx150x.c                   |  1 +
+ drivers/platform/arm64/huawei-gaokun-ec.c          |  1 +
+ drivers/platform/arm64/lenovo-yoga-c630.c          |  2 +-
+ .../platform/x86/intel/int3472/clk_and_regulator.c |  1 +
+ drivers/power/supply/bq25630_charger.c             |  1 +
+ drivers/power/supply/rk817_charger.c               |  2 +-
+ drivers/regulator/fixed.c                          |  1 +
+ drivers/regulator/fp9931.c                         |  1 +
+ drivers/regulator/mt6360-regulator.c               |  1 +
+ drivers/regulator/qcom-labibb-regulator.c          |  1 +
+ drivers/regulator/rtq2208-regulator.c              |  1 +
+ drivers/regulator/tps65185.c                       |  1 +
+ drivers/regulator/tps65219-regulator.c             |  1 +
+ drivers/regulator/tps6594-regulator.c              |  1 +
+ drivers/soc/xilinx/zynqmp_power.c                  |  1 +
+ drivers/ufs/host/ufs-mediatek.c                    |  1 +
+ drivers/ufs/host/ufs-qcom.c                        |  1 +
+ drivers/usb/core/hcd.c                             |  1 +
+ drivers/usb/typec/mux/it5205.c                     |  1 +
+ drivers/usb/typec/wusb3801.c                       |  1 +
+ drivers/video/fbdev/core/fb_procfs.c               |  1 +
+ drivers/xen/balloon.c                              |  1 +
+ fs/aio.c                                           |  1 +
+ fs/buffer.c                                        |  1 +
+ fs/file_table.c                                    |  1 +
+ fs/inode.c                                         |  1 +
+ fs/nfs/blocklayout/blocklayout.c                   |  1 +
+ fs/nfs/nfs42proc.c                                 |  1 +
+ fs/nfs/nfstrace.h                                  |  1 +
+ fs/nfs/pnfs.c                                      |  1 +
+ fs/nfsd/nfs4proc.c                                 |  1 +
+ include/drm/drm_print.h                            |  1 +
+ include/drm/ttm/ttm_tt.h                           |  1 -
+ include/linux/balloon.h                            |  1 -
+ include/linux/ceph/libceph.h                       |  1 -
+ include/linux/i3c/master.h                         |  1 +
+ include/linux/ksm.h                                |  1 -
+ include/linux/mempolicy.h                          |  1 -
+ include/linux/nfs_fs.h                             |  1 -
+ include/linux/nfs_page.h                           |  1 -
+ include/linux/suspend.h                            |  1 -
+ include/linux/swap.h                               |  1 -
+ include/sound/cs35l41.h                            |  1 +
+ kernel/power/snapshot.c                            |  1 +
+ net/ceph/osd_client.c                              |  1 -
+ net/ceph/pagevec.c                                 |  1 +
+ net/core/datagram.c                                |  1 -
+ net/rds/rdma.c                                     |  1 -
+ net/sunrpc/auth_gss/auth_gss.c                     |  1 -
+ net/sunrpc/auth_gss/gss_krb5_crypto.c              |  1 -
+ net/sunrpc/auth_gss/gss_krb5_wrap.c                |  1 -
+ net/sunrpc/auth_gss/svcauth_gss.c                  |  1 -
+ net/sunrpc/cache.c                                 |  1 -
+ net/sunrpc/rpc_pipe.c                              |  1 -
+ net/sunrpc/socklib.c                               |  1 -
+ net/sunrpc/xdr.c                                   |  1 -
+ net/sunrpc/xprtsock.c                              |  1 -
+ security/commoncap.c                               |  3 ---
+ security/inode.c                                   |  1 -
+ security/selinux/hooks.c                           |  1 -
+ security/selinux/selinuxfs.c                       |  1 -
+ security/smack/smack_lsm.c                         |  1 -
+ sound/soc/codecs/cs35l41-lib.c                     |  1 +
+ 155 files changed, 98 insertions(+), 118 deletions(-)
+ delete mode 100644 arch/arc/include/asm/tlb.h
+ delete mode 100644 arch/openrisc/include/asm/tlb.h
diff --git a/arch/Kconfig b/arch/Kconfig
index 45c65777236231..1163736de1f996 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -526,13 +526,6 @@ config HAVE_ARCH_JUMP_LABEL
 config HAVE_ARCH_JUMP_LABEL_RELATIVE
 	bool
 
-config MMU_GATHER_TABLE_FREE
-	bool
-
-config MMU_GATHER_RCU_TABLE_FREE
-	bool
-	select MMU_GATHER_TABLE_FREE
-
 config MMU_GATHER_PAGE_SIZE
 	bool
 
@@ -548,7 +541,6 @@ config MMU_GATHER_MERGE_VMAS
 
 config MMU_GATHER_NO_GATHER
 	bool
-	depends on MMU_GATHER_TABLE_FREE
 
 config ARCH_WANT_IRQS_OFF_ACTIVATE_MM
 	bool
@@ -1238,13 +1230,9 @@ config ARCH_MMAP_RND_BITS_MIN
 config ARCH_MMAP_RND_BITS_MAX
 	int
 
-config ARCH_MMAP_RND_BITS_DEFAULT
-	int
-
 config ARCH_MMAP_RND_BITS
 	int "Number of bits to use for ASLR of mmap base address" if EXPERT
 	range ARCH_MMAP_RND_BITS_MIN ARCH_MMAP_RND_BITS_MAX
-	default ARCH_MMAP_RND_BITS_DEFAULT if ARCH_MMAP_RND_BITS_DEFAULT
 	default ARCH_MMAP_RND_BITS_MIN
 	depends on HAVE_ARCH_MMAP_RND_BITS
 	help
@@ -1272,13 +1260,9 @@ config ARCH_MMAP_RND_COMPAT_BITS_MIN
 config ARCH_MMAP_RND_COMPAT_BITS_MAX
 	int
 
-config ARCH_MMAP_RND_COMPAT_BITS_DEFAULT
-	int
-
 config ARCH_MMAP_RND_COMPAT_BITS
 	int "Number of bits to use for ASLR of mmap base address for compatible applications" if EXPERT
 	range ARCH_MMAP_RND_COMPAT_BITS_MIN ARCH_MMAP_RND_COMPAT_BITS_MAX
-	default ARCH_MMAP_RND_COMPAT_BITS_DEFAULT if ARCH_MMAP_RND_COMPAT_BITS_DEFAULT
 	default ARCH_MMAP_RND_COMPAT_BITS_MIN
 	depends on HAVE_ARCH_MMAP_RND_COMPAT_BITS
 	help
@@ -1712,44 +1696,6 @@ config HAVE_STATIC_CALL_INLINE
 	depends on HAVE_STATIC_CALL
 	select OBJTOOL
 
-config HAVE_PREEMPT_DYNAMIC
-	bool
-
-config HAVE_PREEMPT_DYNAMIC_CALL
-	bool
-	depends on HAVE_STATIC_CALL
-	select HAVE_PREEMPT_DYNAMIC
-	help
-	  An architecture should select this if it can handle the preemption
-	  model being selected at boot time using static calls.
-
-	  Where an architecture selects HAVE_STATIC_CALL_INLINE, any call to a
-	  preemption function will be patched directly.
-
-	  Where an architecture does not select HAVE_STATIC_CALL_INLINE, any
-	  call to a preemption function will go through a trampoline, and the
-	  trampoline will be patched.
-
-	  It is strongly advised to support inline static call to avoid any
-	  overhead.
-
-config HAVE_PREEMPT_DYNAMIC_KEY
-	bool
-	depends on HAVE_ARCH_JUMP_LABEL
-	select HAVE_PREEMPT_DYNAMIC
-	help
-	  An architecture should select this if it can handle the preemption
-	  model being selected at boot time using static keys.
-
-	  Each preemption function will be given an early return based on a
-	  static key. This should have slightly lower overhead than non-inline
-	  static calls, as this effectively inlines each trampoline into the
-	  start of its callee. This may avoid redundant work, and may
-	  integrate better with CFI schemes.
-
-	  This will have greater overhead than using inline static calls as
-	  the call to the preemption function cannot be entirely elided.
-
 config ARCH_WANT_LD_ORPHAN_WARN
 	bool
 	help
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index e53ef2d8846360..9063c7bda4e41e 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -42,7 +42,6 @@ config ALPHA
 	select ARCH_STACKWALK
 	select CPU_NO_EFFICIENT_FFS if !ALPHA_EV67
 	select MMU_GATHER_NO_RANGE
-	select MMU_GATHER_RCU_TABLE_FREE
 	select SPARSEMEM_EXTREME if SPARSEMEM
 	select ZONE_DMA
 	select TRACE_IRQFLAGS_SUPPORT
diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h
index 8e00cf9dc39dea..05885a692244af 100644
--- a/arch/alpha/include/asm/pgtable.h
+++ b/arch/alpha/include/asm/pgtable.h
@@ -2,6 +2,7 @@
 #ifndef _ALPHA_PGTABLE_H
 #define _ALPHA_PGTABLE_H
 
+#include <asm/page.h>
 #include <asm-generic/pgtable-nopud.h>
 
 /*
@@ -13,7 +14,6 @@
  */
 #include <linux/mmzone.h>
 
-#include <asm/page.h>
 #include <asm/processor.h>	/* For TASK_SIZE */
 #include <asm/machvec.h>
 #include <asm/setup.h>
@@ -357,13 +357,6 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte)
 	return pte;
 }
 
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
-#define pmd_ERROR(e) \
-	printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
-
 extern void paging_init(void);
 
 /* We have our own get_unmapped_area */
diff --git a/arch/alpha/include/uapi/asm/setup.h b/arch/alpha/include/uapi/asm/setup.h
index f881ea5947cbc1..169f743ef7658d 100644
--- a/arch/alpha/include/uapi/asm/setup.h
+++ b/arch/alpha/include/uapi/asm/setup.h
@@ -2,6 +2,10 @@
 #ifndef _UAPI__ALPHA_SETUP_H
 #define _UAPI__ALPHA_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	256
+#endif
 
 #endif /* _UAPI__ALPHA_SETUP_H */
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index 2ed7186c81c573..80e61175bf526b 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -331,13 +331,9 @@ config ARC_FPU_SAVE_RESTORE
 	  for floating point exceptions and rounding modes. These are
 	  preserved across task context switch when enabled.
 
-config ARC_CANT_LLSC
-	def_bool n
-
 config ARC_HAS_LLSC
 	bool "Insn: LLOCK/SCOND (efficient atomic ops)"
 	default y
-	depends on !ARC_CANT_LLSC
 
 if ISA_ARCV2
 
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild
index 483caacc698844..5a8ad8b9ad5353 100644
--- a/arch/arc/include/asm/Kbuild
+++ b/arch/arc/include/asm/Kbuild
@@ -6,5 +6,6 @@ generic-y += kvm_para.h
 generic-y += mcs_spinlock.h
 generic-y += parport.h
 generic-y += ring_buffer.h
+generic-y += tlb.h
 generic-y += user.h
 generic-y += text-patching.h
diff --git a/arch/arc/include/asm/atomic64-arcv2.h b/arch/arc/include/asm/atomic64-arcv2.h
index 73080a664369b4..1ca4a1047fb378 100644
--- a/arch/arc/include/asm/atomic64-arcv2.h
+++ b/arch/arc/include/asm/atomic64-arcv2.h
@@ -156,6 +156,12 @@ static inline u64 __arch_cmpxchg64_relaxed(volatile void *ptr, u64 old, u64 new)
 }
 #define arch_cmpxchg64_relaxed __arch_cmpxchg64_relaxed
 
+static inline s64 arch_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
+{
+	return arch_cmpxchg64_relaxed(&v->counter, old, new);
+}
+#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed
+
 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new)
 {
 	s64 prev;
diff --git a/arch/arc/include/asm/cmpxchg.h b/arch/arc/include/asm/cmpxchg.h
index 76f43db0890fcd..66b3743c94008f 100644
--- a/arch/arc/include/asm/cmpxchg.h
+++ b/arch/arc/include/asm/cmpxchg.h
@@ -46,7 +46,7 @@
 	__typeof__(*(ptr)) _n_ = (new);					\
 	__typeof__(*(ptr)) _prev_;					\
 									\
-	switch(sizeof((_p_))) {						\
+	switch (sizeof(*(_p_))) {						\
 	case 1:								\
 		_prev_ = (__typeof__(*(ptr)))cmpxchg_emu_u8((volatile u8 *__force)_p_, (uintptr_t)_o_, (uintptr_t)_n_);	\
 		break;							\
@@ -102,7 +102,7 @@
 	__typeof__(ptr) _p_ = (ptr);					\
 	__typeof__(*(ptr)) _val_ = (val);				\
 									\
-	switch(sizeof(*(_p_))) {					\
+	switch (sizeof(*(_p_))) {					\
 	case 4:								\
 		_val_ = __arch_xchg(_p_, _val_);			\
 		break;							\
diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h
index dfae070fe8d556..9b6c37f92e97f3 100644
--- a/arch/arc/include/asm/pgalloc.h
+++ b/arch/arc/include/asm/pgalloc.h
@@ -72,7 +72,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)
 	set_p4d(p4dp, __p4d((unsigned long)pudp));
 }
 
-#define __pud_free_tlb(tlb, pmd, addr)  pud_free((tlb)->mm, pmd)
+#define __pud_free_tlb(tlb, pmd, addr)  tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd))
 
 #endif
 
@@ -83,10 +83,10 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
 	set_pud(pudp, __pud((unsigned long)pmdp));
 }
 
-#define __pmd_free_tlb(tlb, pmd, addr)  pmd_free((tlb)->mm, pmd)
+#define __pmd_free_tlb(tlb, pmd, addr)  tlb_remove_ptdesc((tlb), virt_to_ptdesc(pmd))
 
 #endif
 
-#define __pte_free_tlb(tlb, pte, addr)  pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb, pte, addr)  tlb_remove_ptdesc((tlb), page_ptdesc(pte))
 
 #endif /* _ASM_ARC_PGALLOC_H */
diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h
index c8f9273372c073..7e16b7fa17bc67 100644
--- a/arch/arc/include/asm/pgtable-levels.h
+++ b/arch/arc/include/asm/pgtable-levels.h
@@ -87,6 +87,8 @@
 
 #ifndef __ASSEMBLER__
 
+#include <asm/page.h>
+
 #if CONFIG_PGTABLE_LEVELS > 3
 #include <asm-generic/pgtable-nop4d.h>
 #elif CONFIG_PGTABLE_LEVELS > 2
@@ -98,8 +100,6 @@
 /*
  * 1st level paging: pgd
  */
-#define pgd_ERROR(e) \
-	pr_crit("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
 
 #if CONFIG_PGTABLE_LEVELS > 3
 
@@ -115,9 +115,6 @@
 /*
  * 2nd level paging: pud
  */
-#define pud_ERROR(e) \
-	pr_crit("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
-
 #endif
 
 #if CONFIG_PGTABLE_LEVELS > 2
@@ -137,9 +134,6 @@
 /*
  * 3rd level paging: pmd
  */
-#define pmd_ERROR(e) \
-	pr_crit("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
-
 #define pmd_pfn(pmd)		((pmd_val(pmd) & PMD_MASK) >> PAGE_SHIFT)
 #define pfn_pmd(pfn,prot)	__pmd(((pfn) << PAGE_SHIFT) | pgprot_val(prot))
 
@@ -165,9 +159,6 @@
 /*
  * 4th level paging: pte
  */
-#define pte_ERROR(e) \
-	pr_crit("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
-
 #define PFN_PTE_SHIFT		PAGE_SHIFT
 #define pte_none(x)		(!pte_val(x))
 #define pte_present(x)		(pte_val(x) & _PAGE_PRESENT)
diff --git a/arch/arc/include/asm/setup.h b/arch/arc/include/asm/setup.h
index 1c6db599e1fcc9..60e158d58cec77 100644
--- a/arch/arc/include/asm/setup.h
+++ b/arch/arc/include/asm/setup.h
@@ -9,7 +9,7 @@
 #include <linux/types.h>
 #include <uapi/asm/setup.h>
 
-#define COMMAND_LINE_SIZE 256
+#define COMMAND_LINE_SIZE CONFIG_COMMAND_LINE_SIZE
 
 /*
  * Data structure to map a ID to string
diff --git a/arch/arc/include/asm/tlb.h b/arch/arc/include/asm/tlb.h
deleted file mode 100644
index 975b35d3738d21..00000000000000
--- a/arch/arc/include/asm/tlb.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
- */
-
-#ifndef _ASM_ARC_TLB_H
-#define _ASM_ARC_TLB_H
-
-#include <linux/pagemap.h>
-#include <asm-generic/tlb.h>
-
-#endif /* _ASM_ARC_TLB_H */
diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c
index 7b6a9beba9db6d..8bebd0e6fc2f09 100644
--- a/arch/arc/kernel/setup.c
+++ b/arch/arc/kernel/setup.c
@@ -593,6 +593,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 			   cpu_id);
 	} else {
 		freq = clk_get_rate(cpu_clk);
+		clk_put(cpu_clk);
 	}
 	if (freq)
 		seq_printf(m, "CPU speed\t: %lu.%02lu Mhz\n",
diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c
index 2d99dffed0ce5c..ee210a94ce459d 100644
--- a/arch/arc/kernel/smp.c
+++ b/arch/arc/kernel/smp.c
@@ -12,7 +12,6 @@
 #include <linux/spinlock.h>
 #include <linux/sched/mm.h>
 #include <linux/interrupt.h>
-#include <linux/profile.h>
 #include <linux/mm.h>
 #include <linux/cpu.h>
 #include <linux/irq.h>
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index ffbc7f38613151..0cc289a7184ab1 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -42,7 +42,6 @@ config ARM
 	select ARCH_SUPPORTS_ATOMIC_RMW
 	select ARCH_SUPPORTS_CFI
 	select ARCH_SUPPORTS_HUGETLBFS if ARM_LPAE
-	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_SUPPORTS_RT
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_CMPXCHG_LOCKREF
@@ -135,7 +134,6 @@ config ARM
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
-	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RSEQ
 	select HAVE_RUST if CPU_LITTLE_ENDIAN && CPU_32v7 && !KASAN
diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug
index 366f162e147d11..abcf14f10276be 100644
--- a/arch/arm/Kconfig.debug
+++ b/arch/arm/Kconfig.debug
@@ -17,7 +17,7 @@ config ARM_PTDUMP_DEBUGFS
 	  kernel.
 	  If in doubt, say "N"
 
-config ARM_DEBUG_WX
+config ARM_CHECK_WX
 	bool "Warn on W+X mappings at boot"
 	depends on MMU
 	select ARM_PTDUMP_CORE
diff --git a/arch/arm/arm-soc-for-next-contents.txt b/arch/arm/arm-soc-for-next-contents.txt
new file mode 100644
index 00000000000000..2f1c655335b588
--- /dev/null
+++ b/arch/arm/arm-soc-for-next-contents.txt
@@ -0,0 +1,22 @@
+soc/arm
+
+soc/dt
+
+soc/drivers
+
+soc/defconfig
+
+soc/late
+
+arm/fixes
+	<no branch> (cfc1e9a543e3589ba200795b6e7fd8ef4314efdf)
+		git://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux tags/socfpga_fix_for_v7.3
+	socfpga/fix
+		git://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux tags/socfpga_dts_fix_for_v7.3
+	renesas/fixes
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-fixes-for-v7.3-tag1
+	scmi/fixes
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/scmi-ffa-fixes-7.3
+	<no branch> (406292fd75f95aa3010fec95b5beb5a8b7e3ba3a)
+		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux tags/amlogic-fixes-v7.3-rc
+
diff --git a/arch/arm/boot/dts/actions/owl-s500.dtsi b/arch/arm/boot/dts/actions/owl-s500.dtsi
index 739b4b9cec8c50..f921654d318e7e 100644
--- a/arch/arm/boot/dts/actions/owl-s500.dtsi
+++ b/arch/arm/boot/dts/actions/owl-s500.dtsi
@@ -61,9 +61,9 @@
 	arm-pmu {
 		compatible = "arm,cortex-a9-pmu";
 		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
 	};
 
@@ -247,9 +247,9 @@
 			compatible = "actions,s500-timer";
 			reg = <0xb0168000 0x8000>;
 			interrupts = <GIC_SPI  8 IRQ_TYPE_LEVEL_HIGH>,
-			             <GIC_SPI  9 IRQ_TYPE_LEVEL_HIGH>,
-			             <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
-			             <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+				     <GIC_SPI  9 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "2hz0", "2hz1", "timer0", "timer1";
 		};
 
diff --git a/arch/arm/boot/dts/allwinner/sun7i-a20.dtsi b/arch/arm/boot/dts/allwinner/sun7i-a20.dtsi
index 5f44f09c554528..adb9ee57ac11a8 100644
--- a/arch/arm/boot/dts/allwinner/sun7i-a20.dtsi
+++ b/arch/arm/boot/dts/allwinner/sun7i-a20.dtsi
@@ -873,7 +873,7 @@
 			gmac_rgmii_pins: gmac-rgmii-pins {
 				pins = "PA0", "PA1", "PA2",
 				       "PA3", "PA4", "PA5", "PA6",
-				        "PA7", "PA8", "PA10",
+				       "PA7", "PA8", "PA10",
 				       "PA11", "PA12", "PA13",
 				       "PA15", "PA16";
 				function = "gmac";
diff --git a/arch/arm/boot/dts/arm/vexpress-v2p-ca5s.dts b/arch/arm/boot/dts/arm/vexpress-v2p-ca5s.dts
index e3896253f33e3e..41a98ba33e806c 100644
--- a/arch/arm/boot/dts/arm/vexpress-v2p-ca5s.dts
+++ b/arch/arm/boot/dts/arm/vexpress-v2p-ca5s.dts
@@ -106,7 +106,7 @@
 
 	timer@2c000200 {
 		compatible = "arm,cortex-a5-global-timer",
-		             "arm,cortex-a9-global-timer";
+			     "arm,cortex-a9-global-timer";
 		reg = <0x2c000200 0x20>;
 		interrupts = <1 11 0x304>;
 		clocks = <&cpu_clk>;
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-opp-witherspoon.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-opp-witherspoon.dts
index 89907b628b6552..d235ebc3022ab0 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-opp-witherspoon.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-opp-witherspoon.dts
@@ -213,7 +213,7 @@
 	/*L0-L7*/	"","","","","","","","",
 	/*M0-M7*/	"","","","","","","","",
 	/*N0-N7*/	"presence-ps1","","led-rear-fault","led-rear-power",
-		        "led-rear-id","","","",
+			"led-rear-id","","","",
 	/*O0-O7*/	"","","","","","","","",
 	/*P0-P7*/	"","","","","","","","presence-ps0",
 	/*Q0-Q7*/	"","","","","","","","",
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-supermicro-x11spi.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-supermicro-x11spi.dts
index b961dff388d107..154a2eaf859643 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-supermicro-x11spi.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-supermicro-x11spi.dts
@@ -75,7 +75,7 @@
     pinctrl-names = "default";
     pinctrl-0 = <&pinctrl_rmii1_default>;
     clocks = <&syscon ASPEED_CLK_GATE_MAC1CLK>,
-           <&syscon ASPEED_CLK_MAC1RCLK>;
+	     <&syscon ASPEED_CLK_MAC1RCLK>;
     clock-names = "MACCLK", "RCLK";
     use-ncsi;
 };
diff --git a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi
index bd52de0faa3ea2..707c8693b28299 100644
--- a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi
@@ -442,16 +442,29 @@
 			port@5 {
 				reg = <5>;
 				ethernet = <&gmac0>;
+				phy-mode = "internal";
+
+				fixed-link {
+					speed = <1000>;
+					full-duplex;
+				};
 			};
 
 			port@7 {
 				reg = <7>;
 				ethernet = <&gmac1>;
+				phy-mode = "internal";
+
+				fixed-link {
+					speed = <1000>;
+					full-duplex;
+				};
 			};
 
 			port@8 {
 				reg = <8>;
 				ethernet = <&gmac2>;
+				phy-mode = "internal";
 
 				fixed-link {
 					speed = <1000>;
diff --git a/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi b/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi
index 1e76b290510d3e..39eea8ce10583e 100644
--- a/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm2835-common.dtsi
@@ -139,6 +139,7 @@
 			compatible = "brcm,bcm2835-v3d";
 			reg = <0x7ec00000 0x1000>;
 			interrupts = <1 10>;
+			resets = <&pm BCM2835_RESET_V3D>;
 		};
 
 	};
diff --git a/arch/arm/boot/dts/broadcom/bcm4708-linksys-ea6300-v1.dts b/arch/arm/boot/dts/broadcom/bcm4708-linksys-ea6300-v1.dts
index 0ed25bf71f0dfa..03d5546a147c4b 100644
--- a/arch/arm/boot/dts/broadcom/bcm4708-linksys-ea6300-v1.dts
+++ b/arch/arm/boot/dts/broadcom/bcm4708-linksys-ea6300-v1.dts
@@ -46,3 +46,7 @@
 &usb3_phy {
 	status = "okay";
 };
+
+&usb3 {
+	vcc-gpio = <&chipcommon 10 GPIO_ACTIVE_HIGH>;
+};
diff --git a/arch/arm/boot/dts/broadcom/bcm4708-smartrg-sr400ac.dts b/arch/arm/boot/dts/broadcom/bcm4708-smartrg-sr400ac.dts
index b226bef3369cf7..03f4a6d37a674b 100644
--- a/arch/arm/boot/dts/broadcom/bcm4708-smartrg-sr400ac.dts
+++ b/arch/arm/boot/dts/broadcom/bcm4708-smartrg-sr400ac.dts
@@ -25,6 +25,16 @@
 		      <0x88000000 0x08000000>;
 	};
 
+	nvram@1eff0000 {
+		compatible = "brcm,nvram";
+		reg = <0x1eff0000 0x10000>;
+
+		et0macaddr: et0macaddr {
+			#nvmem-cell-cells = <1>;
+		};
+	};
+
+
 	leds {
 		compatible = "gpio-leds";
 
@@ -118,6 +128,11 @@
 	status = "okay";
 };
 
+&gmac0 {
+	nvmem-cells = <&et0macaddr 0>;
+	nvmem-cell-names = "mac-address";
+};
+
 &srab {
 	status = "okay";
 
@@ -140,6 +155,9 @@
 
 		port@4 {
 			label = "wan";
+
+			nvmem-cells = <&et0macaddr 1>;
+			nvmem-cell-names = "mac-address";
 		};
 
 		port@5 {
diff --git a/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts b/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts
index 87569408bb6959..37593e7582bab8 100644
--- a/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts
+++ b/arch/arm/boot/dts/broadcom/bcm4709-linksys-ea9200.dts
@@ -93,6 +93,63 @@
 	};
 };
 
+&pcie_bridge0 {
+	pcie@0,0 {
+		device_type = "pci";
+		reg = <0x0000 0 0 0 0>;
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		ranges;
+
+		pcie@1,0 {
+			device_type = "pci";
+			reg = <0x800 0 0 0 0>;
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			wifi@0,0 {
+				compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
+				reg = <0x0000 0 0 0 0>;
+				ieee80211-freq-limit = <5170000 5250000>;
+				brcm,ccode-map = "JP-JP-78", "US-Q2-86";
+				nvmem-cells = <&et2macaddr 2>;
+				nvmem-cell-names = "mac-address";
+			};
+		};
+
+		pcie@2,0 {
+			device_type = "pci";
+			reg = <0x1000 0 0 0 0>;
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges;
+
+			wifi@0,0 {
+				compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
+				reg = <0x0000 0 0 0 0>;
+				brcm,ccode-map = "JP-JP-78", "US-Q2-86";
+				nvmem-cells = <&et2macaddr 3>;
+				nvmem-cell-names = "mac-address";
+			};
+		};
+	};
+};
+
+&pcie_bridge1 {
+	wifi@0,0 {
+		compatible = "brcm,bcm4366-fmac", "brcm,bcm4329-fmac";
+		reg = <0x0000 0 0 0 0>;
+		ieee80211-freq-limit = <5735000 5835000>;
+		brcm,ccode-map = "JP-JP-78", "US-Q2-86";
+		nvmem-cells = <&et2macaddr 4>;
+		nvmem-cell-names = "mac-address";
+	};
+};
+
 &usb3_phy {
 	status = "okay";
 };
@@ -105,6 +162,11 @@
 	vcc-gpios = <&chipcommon 14 GPIO_ACTIVE_HIGH>;
 };
 
+&gmac2 {
+	nvmem-cells = <&et2macaddr 0>;
+	nvmem-cell-names = "mac-address";
+};
+
 &srab {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts b/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts
index e85693fba16af4..a4b135d3765936 100644
--- a/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts
+++ b/arch/arm/boot/dts/broadcom/bcm4709-netgear-r8000.dts
@@ -36,6 +36,15 @@
 		      <0x88000000 0x08000000>;
 	};
 
+	nvram@1c080000 {
+		compatible = "brcm,nvram";
+		reg = <0x1c080000 0x180000>;
+
+		et2macaddr: et2macaddr {
+			#nvmem-cell-cells = <1>;
+		};
+	};
+
 	leds {
 		compatible = "gpio-leds";
 
@@ -211,6 +220,9 @@
 
 		port@4 {
 			label = "wan";
+
+			nvmem-cells = <&et2macaddr 1>;
+			nvmem-cell-names = "mac-address";
 		};
 
 		port@5 {
diff --git a/arch/arm/boot/dts/broadcom/bcm47094-dlink-dir-890l.dts b/arch/arm/boot/dts/broadcom/bcm47094-dlink-dir-890l.dts
index 3124dfd01b9447..944d592dee2c24 100644
--- a/arch/arm/boot/dts/broadcom/bcm47094-dlink-dir-890l.dts
+++ b/arch/arm/boot/dts/broadcom/bcm47094-dlink-dir-890l.dts
@@ -114,6 +114,7 @@
 		reg = <0x1e1f0000 0x00010000>;
 
 		et0macaddr: et0macaddr {
+			#nvmem-cell-cells = <1>;
 		};
 	};
 };
@@ -125,7 +126,7 @@
 	 * actually in use on the platform, we use this et0 MAC
 	 * address for et2.
 	 */
-	nvmem-cells = <&et0macaddr>;
+	nvmem-cells = <&et0macaddr 0>;
 	nvmem-cell-names = "mac-address";
 };
 
@@ -190,6 +191,9 @@
 
 		port@4 {
 			label = "wan";
+
+			nvmem-cells = <&et0macaddr 1>;
+			nvmem-cell-names = "mac-address";
 		};
 
 		port@5 {
diff --git a/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts b/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts
index 6c4131b846c12a..bdbd0f09792588 100644
--- a/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts
+++ b/arch/arm/boot/dts/broadcom/bcm47094-linksys-panamera.dts
@@ -146,7 +146,6 @@
 				#address-cells = <1>;
 				#size-cells = <0>;
 				reset-gpios = <&chipcommon 10 GPIO_ACTIVE_LOW>;
-				reset-names = "robo_reset";
 				reg = <0>;
 				dsa,member = <1 0>;
 				pinctrl-names = "default";
@@ -184,6 +183,7 @@
 					sw1_p8: port@8 {
 						reg = <8>;
 						ethernet = <&sw0_p0>;
+						phy-mode = "rgmii";
 						label = "cpu";
 
 						fixed-link {
diff --git a/arch/arm/boot/dts/broadcom/bcm47189-tenda-ac9.dts b/arch/arm/boot/dts/broadcom/bcm47189-tenda-ac9.dts
index 3ac6cac541cacf..8b2ad7683529bd 100644
--- a/arch/arm/boot/dts/broadcom/bcm47189-tenda-ac9.dts
+++ b/arch/arm/boot/dts/broadcom/bcm47189-tenda-ac9.dts
@@ -20,6 +20,16 @@
 		reg = <0x00000000 0x08000000>;
 	};
 
+	nvram@1eff0000 {
+		compatible = "brcm,nvram";
+		reg = <0x1eff0000 0x10000>;
+
+		et0macaddr: et0macaddr {
+			#nvmem-cell-cells = <1>;
+		};
+	};
+
+
 	leds-0 {
 		compatible = "gpio-leds";
 
@@ -106,6 +116,16 @@
 	};
 };
 
+&gmac0 {
+	nvmem-cells = <&et0macaddr 0>;
+	nvmem-cell-names = "mac-address";
+};
+
+&gmac1 {
+	nvmem-cells = <&et0macaddr 1>;
+	nvmem-cell-names = "mac-address";
+};
+
 &switch {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/broadcom/bcm53573.dtsi b/arch/arm/boot/dts/broadcom/bcm53573.dtsi
index 2df80740d181a8..9e40da44aab9e9 100644
--- a/arch/arm/boot/dts/broadcom/bcm53573.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm53573.dtsi
@@ -32,7 +32,7 @@
 		};
 	};
 
-	mpcore@18310000 {
+	mpcore-bus@18310000 {
 		compatible = "simple-bus";
 		ranges = <0x00000000 0x18310000 0x00008000>;
 		#address-cells = <1>;
@@ -224,6 +224,7 @@
 
 						port@5 {
 							reg = <5>;
+							phy-mode = "internal";
 							ethernet = <&gmac1>;
 
 							fixed-link {
@@ -234,7 +235,13 @@
 
 						port@8 {
 							reg = <8>;
+							phy-mode = "internal";
 							ethernet = <&gmac0>;
+
+							fixed-link {
+								speed = <1000>;
+								full-duplex;
+							};
 						};
 					};
 				};
diff --git a/arch/arm/boot/dts/broadcom/bcm7445.dtsi b/arch/arm/boot/dts/broadcom/bcm7445.dtsi
index c6307c7437e3bf..7488781e0301f0 100644
--- a/arch/arm/boot/dts/broadcom/bcm7445.dtsi
+++ b/arch/arm/boot/dts/broadcom/bcm7445.dtsi
@@ -132,7 +132,7 @@
 			interrupt-names = "hif";
 		};
 
-                aon_pm_l2_intc: interrupt-controller@410640 {
+		aon_pm_l2_intc: interrupt-controller@410640 {
 			compatible = "brcm,l2-intc";
 			reg = <0x410640 0x30>;
 			interrupt-controller;
diff --git a/arch/arm/boot/dts/marvell/armada-385-clearfog-gtr.dtsi b/arch/arm/boot/dts/marvell/armada-385-clearfog-gtr.dtsi
index 7aa71a9aa1bbca..16e9a258876ce5 100644
--- a/arch/arm/boot/dts/marvell/armada-385-clearfog-gtr.dtsi
+++ b/arch/arm/boot/dts/marvell/armada-385-clearfog-gtr.dtsi
@@ -235,7 +235,7 @@
 				pinctrl-0 = <&cf_gtr_sdhci_pins>;
 				pinctrl-names = "default";
 				status = "okay";
-				vmmc = <&reg_3p3v>;
+				vmmc-supply = <&reg_3p3v>;
 				wp-inverted;
 			};
 
diff --git a/arch/arm/boot/dts/marvell/armada-388-clearfog.dtsi b/arch/arm/boot/dts/marvell/armada-388-clearfog.dtsi
index f8a06ae4a3c916..4a61e9d1a2c0d2 100644
--- a/arch/arm/boot/dts/marvell/armada-388-clearfog.dtsi
+++ b/arch/arm/boot/dts/marvell/armada-388-clearfog.dtsi
@@ -131,12 +131,7 @@
 	 * 15-SFP_MOD_DEF0
 	 */
 	expander0: gpio-expander@20 {
-		/*
-		 * This is how it should be:
-		 * compatible = "onnn,pca9655", "nxp,pca9555";
-		 * but you can't do this because of the way I2C works.
-		 */
-		compatible = "nxp,pca9555";
+		compatible = "onnn,pca9655";
 		gpio-controller;
 		#gpio-cells = <2>;
 		reg = <0x20>;
diff --git a/arch/arm/boot/dts/marvell/armada-388-helios4.dts b/arch/arm/boot/dts/marvell/armada-388-helios4.dts
index 003b0708e96560..3f0e89832f363f 100644
--- a/arch/arm/boot/dts/marvell/armada-388-helios4.dts
+++ b/arch/arm/boot/dts/marvell/armada-388-helios4.dts
@@ -140,11 +140,13 @@
 
 	usb2_phy: usb2-phy {
 		compatible = "usb-nop-xceiv";
-		vbus-regulator = <&reg_5p0v_usb>;
+		vbus-supply = <&reg_5p0v_usb>;
+		#phy-cells = <0>;
 	};
 
 	usb3_phy: usb3-phy {
 		compatible = "usb-nop-xceiv";
+		#phy-cells = <0>;
 	};
 
 	soc {
@@ -158,14 +160,7 @@
 				 *  6-USB3 power
 				 */
 				expander0: gpio-expander@20 {
-					/*
-					 * This is how it should be:
-					 * compatible = "onnn,pca9655",
-					 *	 "nxp,pca9555";
-					 * but you can't do this because of
-					 * the way I2C works.
-					 */
-					compatible = "nxp,pca9555";
+					compatible = "onnn,pca9655";
 					gpio-controller;
 					#gpio-cells = <2>;
 					reg = <0x20>;
@@ -198,9 +193,9 @@
 				};
 
 				temp_sensor: temp@4c {
-					compatible = "ti,lm75";
+					compatible = "national,lm75";
 					reg = <0x4c>;
-					vcc-supply = <&reg_3p3v>;
+					vs-supply = <&reg_3p3v>;
 				};
 
 				eeprom@53 {
@@ -254,13 +249,6 @@
 				};
 			};
 
-			spi@10680 {
-				pinctrl-0 = <&spi1_pins
-					     &microsom_spi1_cs_pins>;
-				pinctrl-names = "default";
-				status = "okay";
-			};
-
 			sdhci@d8000 {
 				bus-width = <4>;
 				cd-gpios = <&gpio0 20 GPIO_ACTIVE_LOW>;
@@ -269,7 +257,7 @@
 					     &helios_sdhci_cd_pins>;
 				pinctrl-names = "default";
 				status = "okay";
-				vmmc = <&reg_3p3v>;
+				vmmc-supply = <&reg_3p3v>;
 				wp-inverted;
 			};
 
@@ -327,11 +315,13 @@
 					marvell,pins = "mpp48", "mpp55";
 					marvell,function = "gpio";
 				};
-				microsom_spi1_cs_pins: spi1-cs-pins {
-					marvell,pins = "mpp59";
-					marvell,function = "spi1";
-				};
 			};
 		};
 	};
 };
+
+&spi1 {
+	pinctrl-0 = <&spi1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/marvell/armada-38x.dtsi b/arch/arm/boot/dts/marvell/armada-38x.dtsi
index 1d616edda322d6..0618779090d04e 100644
--- a/arch/arm/boot/dts/marvell/armada-38x.dtsi
+++ b/arch/arm/boot/dts/marvell/armada-38x.dtsi
@@ -627,6 +627,7 @@
 					<0x18454 0x4>;
 				interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gateclk 17>;
+				clock-names = "io";
 				mrvl,clk-delay-cycles = <0x1F>;
 				status = "disabled";
 			};
diff --git a/arch/arm/boot/dts/marvell/armada-395-gp.dts b/arch/arm/boot/dts/marvell/armada-395-gp.dts
index 6dd9e9077f8421..9ea447cd35a85e 100644
--- a/arch/arm/boot/dts/marvell/armada-395-gp.dts
+++ b/arch/arm/boot/dts/marvell/armada-395-gp.dts
@@ -59,7 +59,6 @@
 
 			/* CON18 */
 			sdhci@d8000 {
-				clock-frequency = <200000000>;
 				broken-cd;
 				wp-inverted;
 				bus-width = <8>;
diff --git a/arch/arm/boot/dts/marvell/armada-39x.dtsi b/arch/arm/boot/dts/marvell/armada-39x.dtsi
index 6d05835efb4293..077f4b43382219 100644
--- a/arch/arm/boot/dts/marvell/armada-39x.dtsi
+++ b/arch/arm/boot/dts/marvell/armada-39x.dtsi
@@ -381,6 +381,7 @@
 					<0x18454 0x4>;
 				interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&gateclk 17>;
+				clock-names = "io";
 				mrvl,clk-delay-cycles = <0x1F>;
 				status = "disabled";
 			};
diff --git a/arch/arm/boot/dts/marvell/dove.dtsi b/arch/arm/boot/dts/marvell/dove.dtsi
index 062c86361640fd..75e2f9baf66145 100644
--- a/arch/arm/boot/dts/marvell/dove.dtsi
+++ b/arch/arm/boot/dts/marvell/dove.dtsi
@@ -100,12 +100,12 @@
 			msi-parent = <&intc>;
 			bus-range = <0x00 0xff>;
 
-			ranges = <0x82000000 0x0 0x40000 MBUS_ID(0xf0, 0x01) 0x40000 0 0x2000
-			          0x82000000 0x0 0x80000 MBUS_ID(0xf0, 0x01) 0x80000 0 0x2000
-				  0x82000000 0x1 0x0 MBUS_ID(0x04, 0xe8) 0 1 0   /* Port 0.0 Mem */
-				  0x81000000 0x1 0x0 MBUS_ID(0x04, 0xe0) 0 1 0   /* Port 0.0 I/O */
-				  0x82000000 0x2 0x0 MBUS_ID(0x08, 0xe8) 0 1 0   /* Port 1.0 Mem */
-				  0x81000000 0x2 0x0 MBUS_ID(0x08, 0xe0) 0 1 0>; /* Port 1.0 I/O */
+			ranges = <0x82000000 0x0 0x40000 MBUS_ID(0xf0, 0x01) 0x40000 0 0x2000>,
+				 <0x82000000 0x0 0x80000 MBUS_ID(0xf0, 0x01) 0x80000 0 0x2000>,
+				 <0x82000000 0x1 0x0 MBUS_ID(0x04, 0xe8) 0 1 0>,   /* Port 0.0 Mem */
+				 <0x81000000 0x1 0x0 MBUS_ID(0x04, 0xe0) 0 1 0>,   /* Port 0.0 I/O */
+				 <0x82000000 0x2 0x0 MBUS_ID(0x08, 0xe8) 0 1 0>,   /* Port 1.0 Mem */
+				 <0x81000000 0x2 0x0 MBUS_ID(0x08, 0xe0) 0 1 0>; /* Port 1.0 I/O */
 
 			pcie0: pcie@1 {
 				device_type = "pci";
@@ -117,8 +117,8 @@
 
 				#address-cells = <3>;
 				#size-cells = <2>;
-				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
-				          0x81000000 0 0 0x81000000 0x1 0 1 0>;
+				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0>,
+					 <0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
 
 				#interrupt-cells = <1>;
@@ -146,8 +146,8 @@
 
 				#address-cells = <3>;
 				#size-cells = <2>;
-				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
-				          0x81000000 0 0 0x81000000 0x2 0 1 0>;
+				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0>,
+					 <0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
 
 				#interrupt-cells = <1>;
@@ -772,7 +772,7 @@
 
 			gconf: global-config@e802c {
 				compatible = "marvell,dove-global-config",
-				             "syscon";
+					     "syscon";
 				reg = <0xe802c 0x14>;
 			};
 
diff --git a/arch/arm/boot/dts/marvell/kirkwood-6282.dtsi b/arch/arm/boot/dts/marvell/kirkwood-6282.dtsi
index e33723160ce7cf..9f63e3f9eb1c47 100644
--- a/arch/arm/boot/dts/marvell/kirkwood-6282.dtsi
+++ b/arch/arm/boot/dts/marvell/kirkwood-6282.dtsi
@@ -11,14 +11,13 @@
 
 			bus-range = <0x00 0xff>;
 
-			ranges =
-			       <0x82000000 0 0x40000 MBUS_ID(0xf0, 0x01) 0x40000 0 0x00002000
-			        0x82000000 0 0x44000 MBUS_ID(0xf0, 0x01) 0x44000 0 0x00002000
-				0x82000000 0 0x80000 MBUS_ID(0xf0, 0x01) 0x80000 0 0x00002000
-				0x82000000 0x1 0     MBUS_ID(0x04, 0xe8) 0       1 0 /* Port 0.0 MEM */
-				0x81000000 0x1 0     MBUS_ID(0x04, 0xe0) 0       1 0 /* Port 0.0 IO  */
-				0x82000000 0x2 0     MBUS_ID(0x04, 0xd8) 0       1 0 /* Port 1.0 MEM */
-				0x81000000 0x2 0     MBUS_ID(0x04, 0xd0) 0       1 0 /* Port 1.0 IO  */>;
+			ranges = <0x82000000 0 0x40000 MBUS_ID(0xf0, 0x01) 0x40000 0 0x00002000>,
+				 <0x82000000 0 0x44000 MBUS_ID(0xf0, 0x01) 0x44000 0 0x00002000>,
+				 <0x82000000 0 0x80000 MBUS_ID(0xf0, 0x01) 0x80000 0 0x00002000>,
+				 <0x82000000 0x1 0     MBUS_ID(0x04, 0xe8) 0       1 0 /* Port 0.0 MEM */>,
+				 <0x81000000 0x1 0     MBUS_ID(0x04, 0xe0) 0       1 0 /* Port 0.0 IO  */>,
+				 <0x82000000 0x2 0     MBUS_ID(0x04, 0xd8) 0       1 0 /* Port 1.0 MEM */>,
+				 <0x81000000 0x2 0     MBUS_ID(0x04, 0xd0) 0       1 0 /* Port 1.0 IO  */>;
 
 			pcie0: pcie@1,0 {
 				device_type = "pci";
diff --git a/arch/arm/boot/dts/marvell/kirkwood-guruplug-server-plus.dts b/arch/arm/boot/dts/marvell/kirkwood-guruplug-server-plus.dts
index d5aa8b505cc0aa..cc88d532498319 100644
--- a/arch/arm/boot/dts/marvell/kirkwood-guruplug-server-plus.dts
+++ b/arch/arm/boot/dts/marvell/kirkwood-guruplug-server-plus.dts
@@ -104,14 +104,14 @@
 	ethphy0: ethernet-phy@0 {
 		/* Marvell 88E1121R */
 		compatible = "ethernet-phy-id0141.0cb0",
-		             "ethernet-phy-ieee802.3-c22";
+			     "ethernet-phy-ieee802.3-c22";
 		reg = <0>;
 	};
 
 	ethphy1: ethernet-phy@1 {
 		/* Marvell 88E1121R */
 		compatible = "ethernet-phy-id0141.0cb0",
-		             "ethernet-phy-ieee802.3-c22";
+			     "ethernet-phy-ieee802.3-c22";
 		reg = <1>;
 	};
 };
diff --git a/arch/arm/boot/dts/marvell/kirkwood-lsxl.dtsi b/arch/arm/boot/dts/marvell/kirkwood-lsxl.dtsi
index 5e0b139dd4fba8..4206529b4a5217 100644
--- a/arch/arm/boot/dts/marvell/kirkwood-lsxl.dtsi
+++ b/arch/arm/boot/dts/marvell/kirkwood-lsxl.dtsi
@@ -168,8 +168,8 @@
 		compatible = "gpio-fan";
 		pinctrl-0 = <&pmx_fan_low &pmx_fan_high &pmx_fan_lock>;
 		pinctrl-names = "default";
-		gpios = <&gpio0 19 GPIO_ACTIVE_LOW
-		         &gpio0 18 GPIO_ACTIVE_LOW>;
+		gpios = <&gpio0 19 GPIO_ACTIVE_LOW>,
+			<&gpio0 18 GPIO_ACTIVE_LOW>;
 		gpio-fan,speed-map =
 				<0    3>,
 				<1500 2>,
diff --git a/arch/arm/boot/dts/marvell/kirkwood-synology.dtsi b/arch/arm/boot/dts/marvell/kirkwood-synology.dtsi
index 6b7c5218b1fb58..de34bc2446eddb 100644
--- a/arch/arm/boot/dts/marvell/kirkwood-synology.dtsi
+++ b/arch/arm/boot/dts/marvell/kirkwood-synology.dtsi
@@ -281,7 +281,7 @@
 		status = "disabled";
 		compatible = "gpio-fan";
 		pinctrl-0 = <&pmx_fanctrl_32 &pmx_fanctrl_33 &pmx_fanctrl_34
-		             &pmx_fanalarm_35>;
+			     &pmx_fanalarm_35>;
 		pinctrl-names = "default";
 		gpios = <&gpio1 0 GPIO_ACTIVE_HIGH
 			 &gpio1 1 GPIO_ACTIVE_HIGH
@@ -301,7 +301,7 @@
 		status = "disabled";
 		compatible = "gpio-fan";
 		pinctrl-0 = <&pmx_fanctrl_15 &pmx_fanctrl_16 &pmx_fanctrl_17
-		             &pmx_fanalarm_18>;
+			     &pmx_fanalarm_18>;
 		pinctrl-names = "default";
 		gpios = <&gpio0 15 GPIO_ACTIVE_HIGH
 			 &gpio0 16 GPIO_ACTIVE_HIGH
@@ -322,7 +322,7 @@
 		status = "disabled";
 		compatible = "gpio-fan";
 		pinctrl-0 = <&pmx_fanctrl_32 &pmx_fanctrl_33 &pmx_fanctrl_34
-		             &pmx_fanalarm_35>;
+			     &pmx_fanalarm_35>;
 		pinctrl-names = "default";
 		gpios = <&gpio1 0 GPIO_ACTIVE_HIGH
 			 &gpio1 1 GPIO_ACTIVE_HIGH
@@ -343,7 +343,7 @@
 		status = "disabled";
 		compatible = "gpio-fan";
 		pinctrl-0 = <&pmx_fanctrl_15 &pmx_fanctrl_16 &pmx_fanctrl_17
-		             &pmx_fanalarm_18>;
+			     &pmx_fanalarm_18>;
 		pinctrl-names = "default";
 		gpios = <&gpio0 15 GPIO_ACTIVE_HIGH
 			 &gpio0 16 GPIO_ACTIVE_HIGH
@@ -364,7 +364,7 @@
 		status = "disabled";
 		compatible = "gpio-fan";
 		pinctrl-0 = <&pmx_fanctrl_15 &pmx_fanctrl_16 &pmx_fanctrl_17
-		             &pmx_fanalarm_35>;
+			     &pmx_fanalarm_35>;
 		pinctrl-names = "default";
 		gpios = <&gpio0 15 GPIO_ACTIVE_HIGH
 			 &gpio0 16 GPIO_ACTIVE_HIGH
@@ -385,7 +385,7 @@
 		status = "disabled";
 		compatible = "gpio-fan";
 		pinctrl-0 = <&pmx_fanctrl_15 &pmx_fanctrl_16 &pmx_fanctrl_17
-		             &pmx_fanalarm_35 &pmx_fanalarm_44 &pmx_fanalarm_45>;
+			     &pmx_fanalarm_35 &pmx_fanalarm_44 &pmx_fanalarm_45>;
 		pinctrl-names = "default";
 		gpios = <&gpio0 15 GPIO_ACTIVE_HIGH
 			 &gpio0 16 GPIO_ACTIVE_HIGH
diff --git a/arch/arm/boot/dts/marvell/orion5x-rd88f5182-nas.dts b/arch/arm/boot/dts/marvell/orion5x-rd88f5182-nas.dts
index 75ab913b21e5ff..1d803d1d1112d4 100644
--- a/arch/arm/boot/dts/marvell/orion5x-rd88f5182-nas.dts
+++ b/arch/arm/boot/dts/marvell/orion5x-rd88f5182-nas.dts
@@ -22,7 +22,7 @@
 
 	soc {
 		ranges = <MBUS_ID(0xf0, 0x01) 0 0xf1000000 0x100000>,
-		         <MBUS_ID(0x09, 0x00) 0 0xf2200000 0x800>,
+			 <MBUS_ID(0x09, 0x00) 0 0xf2200000 0x800>,
 			 <MBUS_ID(0x01, 0x0f) 0 0xf4000000 0x80000>,
 			 <MBUS_ID(0x01, 0x1d) 0 0xfc000000 0x1000000>;
 	};
diff --git a/arch/arm/boot/dts/mediatek/Makefile b/arch/arm/boot/dts/mediatek/Makefile
index 37c4cded0eae8c..a610bc75c7d934 100644
--- a/arch/arm/boot/dts/mediatek/Makefile
+++ b/arch/arm/boot/dts/mediatek/Makefile
@@ -14,5 +14,6 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += \
 	mt7623n-rfb-emmc.dtb \
 	mt7623n-bananapi-bpi-r2.dtb \
 	mt7629-rfb.dtb \
+	mt8127-amazon-ford.dtb \
 	mt8127-moose.dtb \
 	mt8135-evbp1.dtb
diff --git a/arch/arm/boot/dts/mediatek/mt8127-amazon-ford.dts b/arch/arm/boot/dts/mediatek/mt8127-amazon-ford.dts
new file mode 100644
index 00000000000000..4b138e832795ed
--- /dev/null
+++ b/arch/arm/boot/dts/mediatek/mt8127-amazon-ford.dts
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/dts-v1/;
+#include "mt8127.dtsi"
+
+/ {
+	model = "Amazon Fire 7 2015 (Ford)";
+	compatible = "amazon,ford", "mediatek,mt8127";
+	chassis-type = "tablet";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		stdout-path = "serial0:921600n8";
+
+		framebuffer0: framebuffer@b7a00000 {
+			compatible = "simple-framebuffer";
+			memory-region = <&framebuffer_reserved>;
+			width = <1024>;
+			height = <600>;
+			stride = <(1024 * 2)>;
+			format = "r5g6b5";
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0 0x80000000 0 0x40000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		framebuffer_reserved: framebuffer@b7a00000 {
+			reg = <0 0xb7a00000 0 0x1000000>;
+			no-map;
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&watchdog {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/mediatek/mt8127.dtsi b/arch/arm/boot/dts/mediatek/mt8127.dtsi
index bd61ec7e70c0ca..9bdf55d2da479c 100644
--- a/arch/arm/boot/dts/mediatek/mt8127.dtsi
+++ b/arch/arm/boot/dts/mediatek/mt8127.dtsi
@@ -99,6 +99,13 @@
 		compatible = "simple-bus";
 		ranges;
 
+		watchdog: watchdog@10007000 {
+			compatible = "mediatek,mt8127-wdt", "mediatek,mt6589-wdt";
+			reg = <0 0x10007000 0 0x100>;
+			interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_LOW>;
+			status = "disabled";
+		};
+
 		timer: timer@10008000 {
 			compatible = "mediatek,mt8127-timer",
 					"mediatek,mt6577-timer";
diff --git a/arch/arm/boot/dts/nvidia/tegra124.dtsi b/arch/arm/boot/dts/nvidia/tegra124.dtsi
index ce4efa1de509b7..86956b94ca0b03 100644
--- a/arch/arm/boot/dts/nvidia/tegra124.dtsi
+++ b/arch/arm/boot/dts/nvidia/tegra124.dtsi
@@ -1114,13 +1114,13 @@
 			    "rx3", "tx3", "rx4", "tx4", "rx5", "tx5",
 			    "rx6", "tx6", "rx7", "tx7", "rx8", "tx8",
 			    "rx9", "tx9";
-		ranges;
-		#address-cells = <2>;
-		#size-cells = <2>;
+		ranges = <0x70300000 0x0 0x70300000 0x10000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
 
 		tegra_i2s0: i2s@70301000 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301000 0x0 0x100>;
+			reg = <0x70301000 0x100>;
 			nvidia,ahub-cif-ids = <4 4>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S0>;
 			resets = <&tegra_car 30>;
@@ -1130,7 +1130,7 @@
 
 		tegra_i2s1: i2s@70301100 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301100 0x0 0x100>;
+			reg = <0x70301100 0x100>;
 			nvidia,ahub-cif-ids = <5 5>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S1>;
 			resets = <&tegra_car 11>;
@@ -1140,7 +1140,7 @@
 
 		tegra_i2s2: i2s@70301200 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301200 0x0 0x100>;
+			reg = <0x70301200 0x100>;
 			nvidia,ahub-cif-ids = <6 6>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S2>;
 			resets = <&tegra_car 18>;
@@ -1150,7 +1150,7 @@
 
 		tegra_i2s3: i2s@70301300 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301300 0x0 0x100>;
+			reg = <0x70301300 0x100>;
 			nvidia,ahub-cif-ids = <7 7>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S3>;
 			resets = <&tegra_car 101>;
@@ -1160,7 +1160,7 @@
 
 		tegra_i2s4: i2s@70301400 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301400 0x0 0x100>;
+			reg = <0x70301400 0x100>;
 			nvidia,ahub-cif-ids = <8 8>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S4>;
 			resets = <&tegra_car 102>;
diff --git a/arch/arm/boot/dts/nvidia/tegra30-asus-nexus7-grouper-common.dtsi b/arch/arm/boot/dts/nvidia/tegra30-asus-nexus7-grouper-common.dtsi
index 892d718294dd10..a7fdd194300c03 100644
--- a/arch/arm/boot/dts/nvidia/tegra30-asus-nexus7-grouper-common.dtsi
+++ b/arch/arm/boot/dts/nvidia/tegra30-asus-nexus7-grouper-common.dtsi
@@ -3,6 +3,7 @@
 #include <dt-bindings/input/gpio-keys.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/power/summit,smb347-charger.h>
 #include <dt-bindings/thermal/thermal.h>
 
@@ -991,7 +992,7 @@
 			vdd-supply = <&vddio_cam>;
 			vaa-supply = <&avdd_cam1>;
 
-			orientation = <0>; /* Front camera */
+			orientation = <MEDIA_ORIENTATION_FRONT>;
 
 			assigned-clocks = <&tegra_car TEGRA30_CLK_VI_SENSOR>,
 					  <&tegra_car TEGRA30_CLK_CSUS>;
diff --git a/arch/arm/boot/dts/nvidia/tegra30-asus-transformer-common.dtsi b/arch/arm/boot/dts/nvidia/tegra30-asus-transformer-common.dtsi
index bf1c3a31d40624..76286e15684c28 100644
--- a/arch/arm/boot/dts/nvidia/tegra30-asus-transformer-common.dtsi
+++ b/arch/arm/boot/dts/nvidia/tegra30-asus-transformer-common.dtsi
@@ -3,6 +3,7 @@
 #include <dt-bindings/input/gpio-keys.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/thermal/thermal.h>
 
 #include "tegra30.dtsi"
@@ -1262,7 +1263,7 @@
 			vdd-supply = <&vdd_1v8_cam>;
 			vaa-supply = <&avdd_2v85_fcam>;
 
-			orientation = <0>; /* Front camera */
+			orientation = <MEDIA_ORIENTATION_FRONT>;
 
 			assigned-clocks = <&tegra_car TEGRA30_CLK_VI_SENSOR>,
 					  <&tegra_car TEGRA30_CLK_CSUS>;
diff --git a/arch/arm/boot/dts/nvidia/tegra30-lg-p895.dts b/arch/arm/boot/dts/nvidia/tegra30-lg-p895.dts
index 896639599c1260..28680063bcc05b 100644
--- a/arch/arm/boot/dts/nvidia/tegra30-lg-p895.dts
+++ b/arch/arm/boot/dts/nvidia/tegra30-lg-p895.dts
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0
 /dts-v1/;
 
+#include <dt-bindings/media/video-interface-devices.h>
+
 #include "tegra30-lg-x3.dtsi"
 
 / {
@@ -132,7 +134,7 @@
 			vdd-supply = <&vt_1v8_front>;
 			vaa-supply = <&vt_2v8_front>;
 
-			orientation = <0>; /* Front camera */
+			orientation = <MEDIA_ORIENTATION_FRONT>;
 
 			assigned-clocks = <&tegra_car TEGRA30_CLK_VI_SENSOR>,
 					  <&tegra_car TEGRA30_CLK_CSUS>;
diff --git a/arch/arm/boot/dts/nvidia/tegra30-lg-x3.dtsi b/arch/arm/boot/dts/nvidia/tegra30-lg-x3.dtsi
index 60e8a19aa70eca..77cab695d3da94 100644
--- a/arch/arm/boot/dts/nvidia/tegra30-lg-x3.dtsi
+++ b/arch/arm/boot/dts/nvidia/tegra30-lg-x3.dtsi
@@ -4,6 +4,7 @@
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/mfd/max77620.h>
 #include <dt-bindings/thermal/thermal.h>
 
@@ -1216,7 +1217,7 @@
 			dvdd-supply = <&vdd_1v2_rear>;
 			avdd-supply = <&vdd_2v7_rear>;
 
-			orientation = <1>; /* Rear camera */
+			orientation = <MEDIA_ORIENTATION_BACK>;
 			rotation = <90>;
 
 			nvmem = <&m24c08>;
diff --git a/arch/arm/boot/dts/nxp/imx/Makefile b/arch/arm/boot/dts/nxp/imx/Makefile
index 1a2539fa19b447..c5caca92ea10ca 100644
--- a/arch/arm/boot/dts/nxp/imx/Makefile
+++ b/arch/arm/boot/dts/nxp/imx/Makefile
@@ -419,13 +419,56 @@ dtb-$(CONFIG_SOC_IMX6UL) += \
 	imx6ull-var-som-concerto-full.dtb \
 	imx6ulz-14x14-evk.dtb \
 	imx6ulz-bsh-smm-m2.dtb
+
+imx7d-colibri-emmc-aster-panel-cap-touch-7inch-parallel-dtbs := \
+	imx7d-colibri-emmc-aster.dtb \
+	imx7d-colibri-emmc-panel-cap-touch-7inch-parallel.dtbo
+
+imx7d-colibri-emmc-iris-v2-panel-cap-touch-7inch-parallel-dtbs := \
+	imx7d-colibri-emmc-iris-v2.dtb \
+	imx7d-colibri-emmc-panel-cap-touch-7inch-parallel.dtbo
+
+imx7d-colibri-emmc-eval-v3-panel-cap-touch-7inch-parallel-touch-adapter-dtbs := \
+	imx7d-colibri-emmc-eval-v3.dtb \
+	imx7d-colibri-emmc-panel-cap-touch-7inch-parallel-touch-adapter.dtbo
+
+imx7d-colibri-emmc-eval-v3-panel-res-touch-7inch-parallel-dtbs := \
+	imx7d-colibri-emmc-eval-v3.dtb \
+	imx7d-colibri-emmc-panel-res-touch-7inch-parallel.dtbo
+
+imx7d-colibri-emmc-aster-vga-dtbs := \
+	imx7d-colibri-emmc-aster.dtb \
+	imx7d-colibri-emmc-vga.dtbo
+
+imx7d-colibri-emmc-eval-v3-vga-dtbs := \
+	imx7d-colibri-emmc-eval-v3.dtb \
+	imx7d-colibri-emmc-vga.dtbo
+
+imx7d-colibri-emmc-iris-vga-dtbs := \
+	imx7d-colibri-emmc-iris.dtb \
+	imx7d-colibri-emmc-vga.dtbo
+
+imx7d-colibri-emmc-iris-v2-vga-dtbs := \
+	imx7d-colibri-emmc-iris-v2.dtb \
+	imx7d-colibri-emmc-vga.dtbo
+
 dtb-$(CONFIG_SOC_IMX7D) += \
 	imx7d-cl-som-imx7.dtb \
 	imx7d-colibri-aster.dtb \
 	imx7d-colibri-emmc-aster.dtb \
+	imx7d-colibri-emmc-aster-panel-cap-touch-7inch-parallel.dtb \
+	imx7d-colibri-emmc-aster-vga.dtb \
 	imx7d-colibri-emmc-iris.dtb \
+	imx7d-colibri-emmc-iris-vga.dtb \
 	imx7d-colibri-emmc-iris-v2.dtb \
+	imx7d-colibri-emmc-iris-v2-panel-cap-touch-7inch-parallel.dtb \
+	imx7d-colibri-emmc-iris-v2-vga.dtb \
 	imx7d-colibri-emmc-eval-v3.dtb \
+	imx7d-colibri-emmc-eval-v3-panel-cap-touch-7inch-parallel-touch-adapter.dtb \
+	imx7d-colibri-emmc-eval-v3-panel-res-touch-7inch-parallel.dtb \
+	imx7d-colibri-emmc-eval-v3-vga.dtb \
+	imx7d-colibri-emmc-panel-cap-touch-7inch-parallel.dtbo \
+	imx7d-colibri-emmc-vga.dtbo \
 	imx7d-colibri-eval-v3.dtb \
 	imx7d-colibri-iris.dtb \
 	imx7d-colibri-iris-v2.dtb \
@@ -444,6 +487,12 @@ dtb-$(CONFIG_SOC_IMX7D) += \
 	imx7d-sdb-reva.dtb \
 	imx7d-sdb-sht11.dtb \
 	imx7d-smegw01.dtb \
+	imx7d-var-som-emmc-mx7customboard.dtb \
+	imx7d-var-som-emmc-wm8731-mx7customboard.dtb \
+	imx7d-var-som-nand-mx7customboard.dtb \
+	imx7d-var-som-nand-wm8731-mx7customboard.dtb \
+	imx7d-var-som-v2-emmc-mx7customboard.dtb \
+	imx7d-var-som-v2-nand-mx7customboard.dtb \
 	imx7d-zii-rmu2.dtb \
 	imx7d-zii-rpu2.dtb \
 	imx7s-colibri-aster.dtb \
diff --git a/arch/arm/boot/dts/nxp/imx/e60k02.dtsi b/arch/arm/boot/dts/nxp/imx/e60k02.dtsi
index aac7b9ef762753..ed59b27834a6d3 100644
--- a/arch/arm/boot/dts/nxp/imx/e60k02.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/e60k02.dtsi
@@ -240,13 +240,13 @@
 			};
 
 			/* IR_3V3 */
-			ldo1_reg: LDO1  {
+			ldo1_reg: LDO1 {
 				regulator-name = "LDO1";
 				regulator-boot-on;
 			};
 
 			/* Core1_3V3 */
-			ldo2_reg: LDO2  {
+			ldo2_reg: LDO2 {
 				regulator-name = "LDO2";
 				regulator-always-on;
 				regulator-boot-on;
@@ -259,7 +259,7 @@
 			};
 
 			/* Core5_1V2 */
-			ldo3_reg: LDO3  {
+			ldo3_reg: LDO3 {
 				regulator-name = "LDO3";
 				regulator-always-on;
 				regulator-boot-on;
@@ -310,7 +310,7 @@
 				regulator-boot-on;
 			};
 
-			ldortc1_reg: LDORTC1  {
+			ldortc1_reg: LDORTC1 {
 				regulator-name = "LDORTC1";
 				regulator-boot-on;
 			};
diff --git a/arch/arm/boot/dts/nxp/imx/e70k02.dtsi b/arch/arm/boot/dts/nxp/imx/e70k02.dtsi
index 3bb11c5a63536d..dc7ff108aa8534 100644
--- a/arch/arm/boot/dts/nxp/imx/e70k02.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/e70k02.dtsi
@@ -243,13 +243,13 @@
 				};
 			};
 
-			ldo1_reg: LDO1  {
+			ldo1_reg: LDO1 {
 				regulator-name = "LDO1";
 				regulator-boot-on;
 			};
 
 			/* Core1_3V3 */
-			ldo2_reg: LDO2  {
+			ldo2_reg: LDO2 {
 				regulator-name = "LDO2";
 				regulator-always-on;
 				regulator-boot-on;
@@ -262,7 +262,7 @@
 			};
 
 			/* Core5_1V2 */
-			ldo3_reg: LDO3  {
+			ldo3_reg: LDO3 {
 				regulator-name = "LDO3";
 				regulator-always-on;
 				regulator-boot-on;
@@ -311,7 +311,7 @@
 				regulator-boot-on;
 			};
 
-			ldortc1_reg: LDORTC1  {
+			ldortc1_reg: LDORTC1 {
 				regulator-name = "LDORTC1";
 				regulator-boot-on;
 			};
diff --git a/arch/arm/boot/dts/nxp/imx/imx50-kobo-aura.dts b/arch/arm/boot/dts/nxp/imx/imx50-kobo-aura.dts
index 4725ee241cb114..4d8672ab1c42b4 100644
--- a/arch/arm/boot/dts/nxp/imx/imx50-kobo-aura.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx50-kobo-aura.dts
@@ -75,15 +75,14 @@
 		reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>;
 	};
 
-	sd2_vmmc: gpio-regulator {
-		compatible = "regulator-gpio";
+	sd2_vmmc: regulator-sd2-vmmc {
+		compatible = "regulator-fixed";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_sd2_vmmc>;
 		regulator-name = "vmmc";
-		states = <3300000 0>;
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
-		enable-gpios = <&gpio4 12 GPIO_ACTIVE_LOW>;
+		gpio = <&gpio4 12 GPIO_ACTIVE_LOW>;
 		startup-delay-us = <100000>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts b/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts
index 91c63d1f260436..c17ab36474cd23 100644
--- a/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx51-zii-rdu1.dts
@@ -115,7 +115,7 @@
 		};
 	};
 
-	i2c_gpio: i2c-gpio {
+	i2c_gpio: i2c-0 {
 		compatible = "i2c-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_swi2c>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts b/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts
index 6831836bd726cb..e7e15d2b82c445 100644
--- a/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx53-qsb.dts
@@ -13,7 +13,7 @@
 
 &i2c1 {
 	pmic: dialog@48 {
-		compatible = "dlg,da9053-aa", "dlg,da9052";
+		compatible = "dlg,da9053-aa";
 		reg = <0x48>;
 		interrupt-parent = <&gpio7>;
 		interrupts = <11 IRQ_TYPE_LEVEL_LOW>; /* low-level active IRQ at GPIO7_11 */
diff --git a/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi b/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi
index 6dc70a92d831b4..6eed683d92a83e 100644
--- a/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx53-voipac-dmm-668.dtsi
@@ -128,7 +128,7 @@
 	status = "okay";
 
 	pmic: dialog@48 {
-		compatible = "dlg,da9053-aa", "dlg,da9052";
+		compatible = "dlg,da9053-aa";
 		reg = <0x48>;
 		interrupt-parent = <&gpio7>;
 		interrupts = <11 IRQ_TYPE_LEVEL_LOW>; /* low-level active IRQ at GPIO7_11 */
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-b125pv2.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-b125pv2.dts
index 94625d5d5918e8..c6ee5f9c8bf870 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-b125pv2.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-b125pv2.dts
@@ -26,5 +26,7 @@
 		interrupt-parent = <&gpio5>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 		reset-gpios = <&tca6424a 21 GPIO_ACTIVE_HIGH>;
+		touchscreen-size-x = <1280>;
+		touchscreen-size-y = <800>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-b125v2.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-b125v2.dts
index b3cfa8110ade45..bd4ef091e412b5 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-b125v2.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-b125v2.dts
@@ -26,5 +26,7 @@
 		interrupt-parent = <&gpio5>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 		reset-gpios = <&tca6424a 21 GPIO_ACTIVE_HIGH>;
+		touchscreen-size-x = <1280>;
+		touchscreen-size-y = <800>;
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-mamoj.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-mamoj.dts
index ec5a9bf40677df..30275319902ec4 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-mamoj.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-mamoj.dts
@@ -151,7 +151,7 @@
 		enable-active-high;
 	};
 
-	reg_wl18xx_vmmc:  regulator-wl18xx-vmcc {
+	reg_wl18xx_vmmc: regulator-wl18xx-vmcc {
 		compatible = "regulator-fixed";
 		regulator-name = "vwl1807";
 		pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts
index 635c5a1e1fc7c4..e3496ea9643d39 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-plym2m.dts
@@ -155,7 +155,7 @@
 	touchscreen {
 		compatible = "resistive-adc-touch";
 		io-channels = <&adc_ts 1>, <&adc_ts 3>, <&adc_ts 4>,
-                              <&adc_ts 5>;
+			      <&adc_ts 5>;
 		io-channel-names = "y", "z1", "z2", "x";
 		touchscreen-min-pressure = <64687>;
 		touchscreen-inverted-x;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-prtvt7.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-prtvt7.dts
index 353f7097cb7e02..09381dc980b236 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-prtvt7.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-prtvt7.dts
@@ -288,7 +288,7 @@
 	touchscreen {
 		compatible = "resistive-adc-touch";
 		io-channels = <&adc_ts 1>, <&adc_ts 3>, <&adc_ts 4>,
-                              <&adc_ts 5>;
+			      <&adc_ts 5>;
 		io-channel-names = "y", "z1", "z2", "x";
 		touchscreen-min-pressure = <64687>;
 		touchscreen-x-plate-ohms = <300>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts b/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts
index 18019a6bb3af07..f874b2b11114f3 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6dl-victgo.dts
@@ -116,7 +116,7 @@
 	touchscreen {
 		compatible = "resistive-adc-touch";
 		io-channels = <&adc_ts 1>, <&adc_ts 3>, <&adc_ts 4>,
-                              <&adc_ts 5>;
+			      <&adc_ts 5>;
 		io-channel-names = "y", "z1", "z2", "x";
 		touchscreen-min-pressure = <64687>;
 		touchscreen-inverted-y;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6q-bosch-acc.dts b/arch/arm/boot/dts/nxp/imx/imx6q-bosch-acc.dts
index 929def2bb35ebb..20b6cc74f622ad 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6q-bosch-acc.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6q-bosch-acc.dts
@@ -177,7 +177,7 @@
 		regulator-name = "usb_h2_vbus";
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
-		vin-supply = <&reg_5p0> ;
+		vin-supply = <&reg_5p0>;
 		regulator-always-on;
 	};
 
@@ -207,7 +207,7 @@
 		regulator-name = "vref_dac";
 		regulator-min-microvolt = <20000>;
 		regulator-max-microvolt = <20000>;
-		vin-supply = <&reg_5p0> ;
+		vin-supply = <&reg_5p0>;
 		regulator-boot-on;
 	};
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-emcon.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-emcon.dtsi
index 9f4e746beb2d5c..34ad7fc04f6b8c 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-emcon.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-emcon.dtsi
@@ -733,7 +733,6 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_pcie_ctrl>;
 	reset-gpio = <&gpio7 12 GPIO_ACTIVE_LOW>;
-	disable-gpio = <&gpio2 22 GPIO_ACTIVE_LOW>;
 };
 
 &pwm1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-gw560x.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-gw560x.dtsi
index e9d5bbb43145f8..57f1583c74fb27 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-gw560x.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-gw560x.dtsi
@@ -418,16 +418,16 @@
 		reg = <0x34>;
 		interrupt-parent = <&gpio5>;
 		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
-		linux,keymap = < MATRIX_KEY(0x00, 0x01, BTN_0)
-			         MATRIX_KEY(0x00, 0x00, BTN_1)
-			         MATRIX_KEY(0x01, 0x01, BTN_2)
-			         MATRIX_KEY(0x01, 0x00, BTN_3)
-			         MATRIX_KEY(0x02, 0x00, BTN_4)
-			         MATRIX_KEY(0x00, 0x03, BTN_5)
-			         MATRIX_KEY(0x00, 0x02, BTN_6)
-			         MATRIX_KEY(0x01, 0x03, BTN_7)
-			         MATRIX_KEY(0x01, 0x02, BTN_8)
-			         MATRIX_KEY(0x02, 0x02, BTN_9)
+		linux,keymap = <MATRIX_KEY(0x00, 0x01, BTN_0)
+				MATRIX_KEY(0x00, 0x00, BTN_1)
+				MATRIX_KEY(0x01, 0x01, BTN_2)
+				MATRIX_KEY(0x01, 0x00, BTN_3)
+				MATRIX_KEY(0x02, 0x00, BTN_4)
+				MATRIX_KEY(0x00, 0x03, BTN_5)
+				MATRIX_KEY(0x00, 0x02, BTN_6)
+				MATRIX_KEY(0x01, 0x03, BTN_7)
+				MATRIX_KEY(0x01, 0x02, BTN_8)
+				MATRIX_KEY(0x02, 0x02, BTN_9)
 		>;
 		keypad,num-rows = <4>;
 		keypad,num-columns = <4>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-pico.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-pico.dtsi
index ca4cb986efbc20..0cc05d084f9bf4 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-pico.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-pico.dtsi
@@ -233,7 +233,9 @@
 		pinctrl-0 = <&pinctrl_ov5645>;
 		reg = <0x3c>;
 		clocks = <&clks IMX6QDL_CLK_CKO2>;
-		clock-frequency = <24000000>;
+		assigned-clocks = <&clks IMX6QDL_CLK_CKO2>;
+		assigned-clock-rates = <24000000>;
+
 		vdddo-supply = <&reg_1p8v>;
 		vdda-supply = <&reg_2p8v>;
 		vddd-supply = <&reg_1p5v>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi
index e2631db798a8f4..27a73a3361aeb0 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-skov-cpu.dtsi
@@ -28,7 +28,7 @@
 	iio-hwmon {
 		compatible = "iio-hwmon";
 		io-channels = <&adc 0>, /* 24V */
-		              <&adc 1>; /* temperature */
+			      <&adc 1>; /* temperature */
 	};
 
 	leds {
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-wandboard.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-wandboard.dtsi
index 26489eccd5fbe4..f3aaca039953bf 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-wandboard.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-wandboard.dtsi
@@ -137,7 +137,9 @@
 		pinctrl-0 = <&pinctrl_ov5645>;
 		reg = <0x3c>;
 		clocks = <&clks IMX6QDL_CLK_CKO2>;
-		clock-frequency = <24000000>;
+		assigned-clocks = <&clks IMX6QDL_CLK_CKO2>;
+		assigned-clock-rates = <24000000>;
+
 		vdddo-supply = <&reg_1p8v>;
 		vdda-supply = <&reg_2p8v>;
 		vddd-supply = <&reg_1p5v>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl-zii-rdu2.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl-zii-rdu2.dtsi
index 9ff183e4e069a8..601733c3c2a985 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl-zii-rdu2.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl-zii-rdu2.dtsi
@@ -22,8 +22,8 @@
 		#size-cells = <0>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_mdio1>;
-		gpios = <&gpio6 5 GPIO_ACTIVE_HIGH
-		         &gpio6 4 GPIO_ACTIVE_HIGH>;
+		gpios = <&gpio6 5 GPIO_ACTIVE_HIGH>,
+			<&gpio6 4 GPIO_ACTIVE_HIGH>;
 
 		phy: ethernet-phy@0 {
 			pinctrl-0 = <&pinctrl_rmii_phy_irq>;
@@ -218,9 +218,9 @@
 
 &clks {
 	assigned-clocks = <&clks IMX6QDL_CLK_LDB_DI0_SEL>,
-	                  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
+			  <&clks IMX6QDL_CLK_LDB_DI1_SEL>;
 	assigned-clock-parents = <&clks IMX6QDL_CLK_PLL5_VIDEO_DIV>,
-	                         <&clks IMX6QDL_CLK_PLL5_VIDEO_DIV>;
+				 <&clks IMX6QDL_CLK_PLL5_VIDEO_DIV>;
 };
 
 &cpu0 {
@@ -629,7 +629,7 @@
 		pinctrl-0 = <&pinctrl_ucs1002_pins>;
 		reg = <0x32>;
 		interrupts-extended = <&gpio5 2 IRQ_TYPE_EDGE_BOTH>,
-		                      <&gpio3 21 IRQ_TYPE_EDGE_FALLING>;
+				      <&gpio3 21 IRQ_TYPE_EDGE_FALLING>;
 		interrupt-names = "a_det", "alert";
 	};
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi b/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi
index 9438862b99271c..86c4f7e010d073 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6qdl.dtsi
@@ -924,11 +924,11 @@
 						#power-domain-cells = <0>;
 						power-supply = <&reg_pu>;
 						clocks = <&clks IMX6QDL_CLK_GPU3D_CORE>,
-						         <&clks IMX6QDL_CLK_GPU3D_SHADER>,
-						         <&clks IMX6QDL_CLK_GPU2D_CORE>,
-						         <&clks IMX6QDL_CLK_GPU2D_AXI>,
-						         <&clks IMX6QDL_CLK_OPENVG_AXI>,
-						         <&clks IMX6QDL_CLK_VPU_AXI>;
+							 <&clks IMX6QDL_CLK_GPU3D_SHADER>,
+							 <&clks IMX6QDL_CLK_GPU2D_CORE>,
+							 <&clks IMX6QDL_CLK_GPU2D_AXI>,
+							 <&clks IMX6QDL_CLK_OPENVG_AXI>,
+							 <&clks IMX6QDL_CLK_VPU_AXI>;
 					};
 				};
 			};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6sl-kobo-aura2.dts b/arch/arm/boot/dts/nxp/imx/imx6sl-kobo-aura2.dts
index 657d0f1b6115f0..47b05207574e06 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6sl-kobo-aura2.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6sl-kobo-aura2.dts
@@ -211,14 +211,14 @@
 			};
 
 			/* IR_3V3 */
-			ldo1_reg: LDO1  {
+			ldo1_reg: LDO1 {
 				regulator-name = "LDO1";
 				regulator-always-on;
 				regulator-boot-on;
 			};
 
 			/* Core1_3V3 */
-			ldo2_reg: LDO2  {
+			ldo2_reg: LDO2 {
 				regulator-name = "LDO2";
 				regulator-always-on;
 				regulator-boot-on;
@@ -231,7 +231,7 @@
 			};
 
 			/* Core5_1V2 */
-			ldo3_reg: LDO3  {
+			ldo3_reg: LDO3 {
 				regulator-name = "LDO3";
 				regulator-always-on;
 				regulator-boot-on;
@@ -282,7 +282,7 @@
 				regulator-boot-on;
 			};
 
-			ldortc1_reg: LDORTC1  {
+			ldortc1_reg: LDORTC1 {
 				regulator-name = "LDORTC1";
 				regulator-always-on;
 				regulator-boot-on;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6sl-tolino-shine2hd.dts b/arch/arm/boot/dts/nxp/imx/imx6sl-tolino-shine2hd.dts
index 4c655579f43efb..64a74719c216f6 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6sl-tolino-shine2hd.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6sl-tolino-shine2hd.dts
@@ -276,13 +276,13 @@
 			};
 
 			/* IR_3V3 */
-			ldo1_reg: LDO1  {
+			ldo1_reg: LDO1 {
 				regulator-name = "LDO1";
 				regulator-boot-on;
 			};
 
 			/* Core1_3V3 */
-			ldo2_reg: LDO2  {
+			ldo2_reg: LDO2 {
 				regulator-name = "LDO2";
 				regulator-always-on;
 				regulator-boot-on;
@@ -295,7 +295,7 @@
 			};
 
 			/* Core5_1V2 */
-			ldo3_reg: LDO3  {
+			ldo3_reg: LDO3 {
 				regulator-name = "LDO3";
 				regulator-always-on;
 				regulator-boot-on;
@@ -346,7 +346,7 @@
 				regulator-boot-on;
 			};
 
-			ldortc1_reg: LDORTC1  {
+			ldortc1_reg: LDORTC1 {
 				regulator-name = "LDORTC1";
 				regulator-always-on;
 				regulator-boot-on;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6sll-evk.dts b/arch/arm/boot/dts/nxp/imx/imx6sll-evk.dts
index 814401486792ca..d20d2735cab524 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6sll-evk.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6sll-evk.dts
@@ -633,7 +633,7 @@
 		>;
 	};
 
-	pinctrl_wdog1: wdog1grp	{
+	pinctrl_wdog1: wdog1grp {
 		fsl,pins = <
 			MX6SLL_PAD_WDOG_B__WDOG1_B   0x170b0
 		>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6sll.dtsi b/arch/arm/boot/dts/nxp/imx/imx6sll.dtsi
index c96669605d1d87..929f7815a45465 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6sll.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6sll.dtsi
@@ -624,17 +624,6 @@
 				reg = <0x020e4000 0x4000>;
 			};
 
-			csi: csi@20e8000 {
-				compatible = "fsl,imx6sll-csi", "fsl,imx6s-csi";
-				reg = <0x020e8000 0x4000>;
-				interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&clks IMX6SLL_CLK_DUMMY>,
-					 <&clks IMX6SLL_CLK_CSI>,
-					 <&clks IMX6SLL_CLK_DUMMY>;
-				clock-names = "disp-axi", "csi_mclk", "disp_dcic";
-				status = "disabled";
-			};
-
 			sdma: dma-controller@20ec000 {
 				compatible = "fsl,imx6sll-sdma", "fsl,imx6ul-sdma";
 				reg = <0x020ec000 0x4000>;
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-14x14-evk.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-14x14-evk.dtsi
index 32afe4130e2112..181676cd3b60aa 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-14x14-evk.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-14x14-evk.dtsi
@@ -236,7 +236,6 @@
 	port {
 		parallel_from_ov5640: endpoint {
 			remote-endpoint = <&ov5640_to_parallel>;
-			bus-type = <MEDIA_BUS_TYPE_PARALLEL>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi
index f4c45e964daf8f..0658584c9f11da 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-kontron-bl-common.dtsi
@@ -132,6 +132,22 @@
 	};
 };
 
+&gpio1 {
+	gpio-line-names = "", "", "ADC_REF", "AIN1",
+			  "", "", "ENET2_MDIO", "ENET2_MDC",
+			  "AIN2", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "SD1_CD",
+			  "", "", "", "",
+			  "CAN_ADR0", "CAN_ADR1", "CAN_ADR2", "CAN_ADR3",
+			  "", "", "LED_H14";
+};
+
+&gpio5 {
+	gpio-line-names = "DIN2", "DOUT2", "LED_H16",
+			  "LED_H15", "DIN1", "DOUT1";
+};
+
 &i2c1 {
 	clock-frequency = <100000>;
 	pinctrl-names = "default";
@@ -172,13 +188,6 @@
 	status = "okay";
 };
 
-&uart3 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart3>;
-	uart-has-rtscts;
-	status = "okay";
-};
-
 &uart4 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart4>;
@@ -283,6 +292,11 @@
 			MX6UL_PAD_SNVS_TAMPER4__GPIO5_IO04	0x1b0b0	/* DIN1 */
 			MX6UL_PAD_SNVS_TAMPER1__GPIO5_IO01	0x1b0b0	/* DOUT2 */
 			MX6UL_PAD_SNVS_TAMPER0__GPIO5_IO00	0x1b0b0	/* DIN2 */
+
+			MX6UL_PAD_UART3_TX_DATA__GPIO1_IO24	0x1b0b0	/* CAN_ADR0 (DIP sw 1) */
+			MX6UL_PAD_UART3_RX_DATA__GPIO1_IO25	0x1b0b0	/* CAN_ADR1 (DIP sw 2) */
+			MX6UL_PAD_UART3_CTS_B__GPIO1_IO26	0x1b0b0	/* CAN_ADR2 (DIP sw 3) */
+			MX6UL_PAD_UART3_RTS_B__GPIO1_IO27	0x1b0b0	/* CAN_ADR3 (DIP sw 4) */
 		>;
 	};
 
@@ -334,15 +348,6 @@
 		>;
 	};
 
-	pinctrl_uart3: uart3grp {
-		fsl,pins = <
-			MX6UL_PAD_UART3_TX_DATA__UART3_DCE_TX	0x1b0b1
-			MX6UL_PAD_UART3_RX_DATA__UART3_DCE_RX	0x1b0b1
-			MX6UL_PAD_UART3_CTS_B__UART3_DCE_CTS	0x1b0b1
-			MX6UL_PAD_UART3_RTS_B__UART3_DCE_RTS	0x1b0b1
-		>;
-	};
-
 	pinctrl_uart4: uart4grp {
 		fsl,pins = <
 			MX6UL_PAD_UART4_TX_DATA__UART4_DCE_TX	0x1b0b1
@@ -352,6 +357,7 @@
 
 	pinctrl_usbotg1: usbotg1grp {
 		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO00__ANATOP_OTG1_ID	0x17059
 			MX6UL_PAD_GPIO1_IO04__GPIO1_IO04	0x1b0b0
 		>;
 	};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi
index 192c6a95ae589f..2f6916c0359c5b 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-tx6ul.dtsi
@@ -369,7 +369,7 @@
 	display = <&display>;
 	status = "okay";
 
-	display: disp0 {
+	display: display0 {
 		bits-per-pixel = <32>;
 		bus-width = <24>;
 		status = "okay";
diff --git a/arch/arm/boot/dts/nxp/imx/imx7-colibri-iris-v2.dtsi b/arch/arm/boot/dts/nxp/imx/imx7-colibri-iris-v2.dtsi
index b687727f956a6a..216dac7823df60 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7-colibri-iris-v2.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7-colibri-iris-v2.dtsi
@@ -106,7 +106,6 @@
 /* Colibri MMC/SD, UHS-I capable uSD slot */
 &usdhc1 {
 	cap-power-off-card;
-	/delete-property/ keep-power-in-suspend;
 	/delete-property/ no-1-8-v;
 	vmmc-supply = <&reg_3v3_vmmc>;
 	status = "okay";
diff --git a/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi b/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi
index e1f740edfb763c..353af6b320cb69 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx7-tqma7.dtsi
@@ -46,8 +46,8 @@
 
 		regulators {
 			sw1a_reg: sw1a {
-				regulator-min-microvolt = <700000>;
-				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1225000>;
 				regulator-boot-on;
 				regulator-always-on;
 				regulator-ramp-delay = <6250>;
@@ -55,34 +55,33 @@
 
 			/* use sw1c_reg to align with pfuze100/pfuze200 */
 			sw1c_reg: sw1b {
-				regulator-min-microvolt = <700000>;
-				regulator-max-microvolt = <1475000>;
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1000000>;
 				regulator-boot-on;
 				regulator-always-on;
 				regulator-ramp-delay = <6250>;
 			};
 
 			sw2_reg: sw2 {
-				regulator-min-microvolt = <1500000>;
-				regulator-max-microvolt = <1850000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
 				regulator-boot-on;
 				regulator-always-on;
 			};
 
 			sw3a_reg: sw3 {
-				regulator-min-microvolt = <900000>;
-				regulator-max-microvolt = <1650000>;
+				regulator-min-microvolt = <1350000>;
+				regulator-max-microvolt = <1350000>;
 				regulator-boot-on;
 				regulator-always-on;
 			};
 
 			swbst_reg: swbst {
-				regulator-min-microvolt = <5000000>;
-				regulator-max-microvolt = <5150000>;
+				/* Not connected and left unconfigured */
 			};
 
 			snvs_reg: vsnvs {
-				regulator-min-microvolt = <1000000>;
+				regulator-min-microvolt = <3000000>;
 				regulator-max-microvolt = <3000000>;
 				regulator-boot-on;
 				regulator-always-on;
@@ -95,18 +94,25 @@
 
 			vgen1_reg: vldo1 {
 				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <3300000>;
+				regulator-max-microvolt = <1800000>;
 				regulator-always-on;
 			};
 
 			vgen2_reg: vldo2 {
-				regulator-min-microvolt = <800000>;
-				regulator-max-microvolt = <1550000>;
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1500000>;
 				regulator-always-on;
 			};
 
 			vgen3_reg: vccsd {
-				regulator-min-microvolt = <2850000>;
+				/*
+				 * Specified values refer to 3.3V operation; voltage for 1.8V
+				 * operation is derived from this.
+				 *
+				 * The base board should usually mux GPIO1_IO08 to SD1_VSELECT for
+				 * automatic switching of SD card IO voltage.
+				 */
+				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-always-on;
 			};
@@ -118,19 +124,25 @@
 			};
 
 			vgen5_reg: vldo3 {
-				regulator-min-microvolt = <1800000>;
+				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-always-on;
 			};
 
 			vgen6_reg: vldo4 {
-				regulator-min-microvolt = <1800000>;
+				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-always-on;
 			};
 		};
 	};
 
+	/* NXP SE97BTP with temperature sensor + eeprom, TQMa7x 02xx */
+	se97b: temperature-sensor@1e {
+		compatible = "nxp,se97b", "jedec,jc-42.4-temp";
+		reg = <0x1e>;
+	};
+
 	/* LM75A temperature sensor, TQMa7x 01xx */
 	lm75a: temperature-sensor@48 {
 		compatible = "national,lm75a";
@@ -138,12 +150,6 @@
 		vs-supply = <&vgen4_reg>;
 	};
 
-	/* NXP SE97BTP with temperature sensor + eeprom, TQMa7x 02xx */
-	se97b: temperature-sensor@1e {
-		compatible = "nxp,se97b", "jedec,jc-42.4-temp";
-		reg = <0x1e>;
-	};
-
 	/* ST M24C64 */
 	m24c64: eeprom@50 {
 		compatible = "atmel,24c64";
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel-touch-adapter.dtso b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel-touch-adapter.dtso
new file mode 100644
index 00000000000000..890461f92442e5
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel-touch-adapter.dtso
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex Capacitive Touch Display 7" Parallel connected through the 40-way
+ * Unified Interface Display connector and Toradex Capacitive Touch Adapter.
+ *
+ * Toradex Capacitive Touch Adapter connection with the various carrier boards
+ * is documented in the datasheet [1].
+ *
+ * https://developer.toradex.com/hardware/accessories/displays/capacitive-touch-display-7inch-parallel/
+ * https://www.toradex.com/accessories/capacitive-touch-display-7-inch-parallel
+ * https://docs.toradex.com/104497-7-inch-parallel-capacitive-touch-display-800x480-datasheet.pdf
+ * https://developer.toradex.com/hardware/accessories/add-ons/capacitive-touch-adapter/
+ * https://www.toradex.com/accessories/capacitive-touch-adapter
+ * https://docs.toradex.com/104615-capacitive-touch-adapter-datasheet.pdf [1]
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&atmel_mxt_ts {
+	pinctrl-0 = <&pinctrl_atmel_adapter>;
+	/* SODIMM 28 / TOUCH_INT# */
+	interrupt-parent = <&gpio1>;
+	interrupts = <9 IRQ_TYPE_EDGE_FALLING>;
+	/* SODIMM 30 / TOUCH_RST# */
+	reset-gpios = <&gpio1 10 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+};
+
+&backlight {
+	status = "okay";
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&panel_dpi {
+	compatible = "logictechno,lt161010-2nhc";
+
+	status = "okay";
+};
+
+/* Conflict with SODIMM 28 / TOUCH_INT# */
+&pwm2 {
+	status = "disabled";
+};
+
+/* Conflict with SODIMM 30 / TOUCH_RST# */
+&pwm3 {
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel.dtso b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel.dtso
new file mode 100644
index 00000000000000..f336f7b05a2b71
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-cap-touch-7inch-parallel.dtso
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex Capacitive Touch Display 7" Parallel connected through the 40-way
+ * Unified Interface Display and 10-way Capacitive Touch Interface connectors
+ * (as featured on Iris v2 and Aster v1.1 carrier boards).
+ *
+ * https://docs.toradex.com/104497-7-inch-parallel-capacitive-touch-display-800x480-datasheet.pdf
+ * https://developer.toradex.com/hardware/accessories/displays/capacitive-touch-display-7inch-parallel/
+ * https://www.toradex.com/accessories/capacitive-touch-display-7-inch-parallel
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&atmel_mxt_ts {
+	pinctrl-0 = <&pinctrl_atmel_connector>;
+	/* SODIMM 107 / TOUCH_INT# */
+	interrupt-parent = <&gpio2>;
+	interrupts = <15 IRQ_TYPE_EDGE_FALLING>;
+	/* SODIMM 106 / TOUCH_RST# */
+	reset-gpios = <&gpio2 28 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+};
+
+&backlight {
+	status = "okay";
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&panel_dpi {
+	compatible = "logictechno,lt161010-2nhc";
+
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-res-touch-7inch-parallel.dtso b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-res-touch-7inch-parallel.dtso
new file mode 100644
index 00000000000000..b926cc020e6268
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-panel-res-touch-7inch-parallel.dtso
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex Resistive Touch Display 7" Parallel connected through the 40-way
+ * Unified Interface Display connector.
+ *
+ * https://docs.toradex.com/104498-7-inch-parallel-resistive-touch-display-800x480.pdf
+ * https://developer.toradex.com/hardware/accessories/displays/resistive-touch-display-7inch-parallel/
+ * https://www.toradex.com/accessories/resistive-touch-display
+ */
+
+/dts-v1/;
+/plugin/;
+
+&ad7879_ts {
+	status = "okay";
+};
+
+&backlight {
+	status = "okay";
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&panel_dpi {
+	compatible = "logictechno,lt161010-2nhr";
+
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-vga.dtso b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-vga.dtso
new file mode 100644
index 00000000000000..cda214c138769f
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-colibri-emmc-vga.dtso
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * VGA output connected to the Colibri parallel RGB LCD interface through a
+ * non-programmable video DAC with a DVI connector.
+ *
+ * The VGA output is available on multiple carrier boards (including Iris,
+ * Eval, Aster) without VESA DDC I2C signals.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	vga-encoder {
+		compatible = "dumb-vga-dac";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				vga_enc_in: endpoint {
+					remote-endpoint = <&lcdif_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				vga_enc_out: endpoint {
+					remote-endpoint = <&dvi_connector_in>;
+				};
+			};
+		};
+	};
+
+	dvi-connector {
+		compatible = "dvi-connector";
+		analog;
+
+		port {
+			dvi_connector_in: endpoint {
+				remote-endpoint = <&vga_enc_out>;
+			};
+		};
+	};
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&lcdif_out {
+	remote-endpoint = <&vga_enc_in>;
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-meerkat96.dts b/arch/arm/boot/dts/nxp/imx/imx7d-meerkat96.dts
index f0fda15f30204a..38f886c25509b0 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7d-meerkat96.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-meerkat96.dts
@@ -134,7 +134,7 @@
 &lcdif {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_lcdif>;
-	status = "okay";
+	status = "disabled";
 };
 
 &uart1 {
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-nitrogen7.dts b/arch/arm/boot/dts/nxp/imx/imx7d-nitrogen7.dts
index 2192f105ec81cb..c72859ac43e1a0 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7d-nitrogen7.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-nitrogen7.dts
@@ -284,7 +284,6 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_i2c3_tsc2004>;
 		interrupts-extended = <&gpio3 4 IRQ_TYPE_EDGE_FALLING>;
-		wakeup-gpios = <&gpio3 4 GPIO_ACTIVE_LOW>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-mx7customboard.dts b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-mx7customboard.dts
new file mode 100644
index 00000000000000..594ee2d2d23a5f
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-mx7customboard.dts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * VAR-MX7CustomBoard for VAR-SOM-MX7 with eMMC
+ *
+ * Link: https://variscite.com/carrier-boards/var-mx7customboard/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx7d-var-som.dtsi"
+#include "imx7d-var-som-emmc.dtsi"
+#include "imx7d-var-som-wm8904.dtsi"
+#include "imx7d-var-som-mx7customboard.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-MX7 with eMMC & WM8904 on VAR-MX7CustomBoard";
+	compatible = "variscite,var-som-imx7d-emmc-mx7customboard",
+		     "variscite,var-som-imx7d",
+		     "fsl,imx7d";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-wm8731-mx7customboard.dts b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-wm8731-mx7customboard.dts
new file mode 100644
index 00000000000000..94c4b1fa5d6c36
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc-wm8731-mx7customboard.dts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * VAR-MX7CustomBoard for VAR-SOM-MX7 with eMMC and WM8731
+ *
+ * Link: https://variscite.com/carrier-boards/var-mx7customboard/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx7d-var-som.dtsi"
+#include "imx7d-var-som-emmc.dtsi"
+#include "imx7d-var-som-wm8731.dtsi"
+#include "imx7d-var-som-mx7customboard.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-MX7 with eMMC & WM8731 on VAR-MX7CustomBoard";
+	compatible = "variscite,var-som-imx7d-emmc-wm8731-mx7customboard",
+		     "variscite,var-som-imx7d",
+		     "fsl,imx7d";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc.dtsi
new file mode 100644
index 00000000000000..62f60ec3999282
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-emmc.dtsi
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for eMMC support on Variscite VAR-SOM-MX7
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-7/var-som-mx7/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+&usdhc3 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_sleep>;
+	vmmc-supply = <&vgen3_reg>;
+	assigned-clocks = <&clks IMX7D_USDHC3_ROOT_CLK>;
+	assigned-clock-rates = <400000000>;
+	no-1-8-v;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX7D_PAD_SD3_RESET_B__SD3_RESET_B	0x59
+			MX7D_PAD_SD3_CMD__SD3_CMD		0x59
+			MX7D_PAD_SD3_CLK__SD3_CLK		0x19
+			MX7D_PAD_SD3_DATA0__SD3_DATA0		0x59
+			MX7D_PAD_SD3_DATA1__SD3_DATA1		0x59
+			MX7D_PAD_SD3_DATA2__SD3_DATA2		0x59
+			MX7D_PAD_SD3_DATA3__SD3_DATA3		0x59
+			MX7D_PAD_SD3_DATA4__SD3_DATA4		0x59
+			MX7D_PAD_SD3_DATA5__SD3_DATA5		0x59
+			MX7D_PAD_SD3_DATA6__SD3_DATA6		0x59
+			MX7D_PAD_SD3_DATA7__SD3_DATA7		0x59
+		>;
+	};
+
+	pinctrl_usdhc3_sleep: usdhc3-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_SD3_RESET_B__GPIO6_IO11	0x10
+			MX7D_PAD_SD3_CMD__GPIO6_IO1		0x10
+			MX7D_PAD_SD3_CLK__GPIO6_IO0		0x10
+			MX7D_PAD_SD3_DATA0__GPIO6_IO2		0x10
+			MX7D_PAD_SD3_DATA1__GPIO6_IO3		0x10
+			MX7D_PAD_SD3_DATA2__GPIO6_IO4		0x10
+			MX7D_PAD_SD3_DATA3__GPIO6_IO5		0x10
+			MX7D_PAD_SD3_DATA4__GPIO6_IO6		0x10
+			MX7D_PAD_SD3_DATA5__GPIO6_IO7		0x10
+			MX7D_PAD_SD3_DATA6__GPIO6_IO8		0x10
+			MX7D_PAD_SD3_DATA7__GPIO6_IO9		0x10
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-mx7customboard.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-mx7customboard.dtsi
new file mode 100644
index 00000000000000..d798b021f8e052
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-mx7customboard.dtsi
@@ -0,0 +1,358 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Variscite VAR-MX7CustomBoard carrier board for VAR-SOM-MX7
+ *
+ * Link: https://variscite.com/carrier-boards/var-mx7customboard/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/ {
+	chosen {
+		stdout-path = &uart1;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_keys>;
+
+		key-back {
+			label = "Back";
+			gpios = <&gpio1 11 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_BACK>;
+		};
+
+		key-menu {
+			label = "Menu";
+			gpios = <&gpio2 28 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_MENU>;
+		};
+
+		key-home {
+			label = "Home";
+			gpios = <&gpio2 29 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_HOME>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_leds>;
+
+		led-d10 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			gpios = <&gpio1 13 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	reg_hsic_hub_connect: regulator-hsic-hub-connect {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hsic_hub_connect>;
+		regulator-name = "reg_hsic_hub_connect";
+		gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	reg_hsic_hub_pwr_on: regulator-hsic-hub-pwr-on {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hsic_hub_pwr>;
+		regulator-name = "hsic_hub_pwr_on";
+		gpio = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usb_otg1_vbus: regulator-otg1-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbotg1_pwr>;
+		regulator-name = "usb_otg1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio1 5 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usb_otg2_vbus: regulator-otg2-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbotg2_pwr>;
+		regulator-name = "usb_otg2_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio1 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+&clks {
+	assigned-clocks = <&clks IMX7D_PLL_AUDIO_POST_DIV>;
+	assigned-clock-rates = <884736000>;
+};
+
+&ecspi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	cs-gpios = <&gpio4 23 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	status = "okay";
+};
+
+&i2c2 {
+	/* DS1337 RTC module */
+	rtc@68 {
+		compatible = "dallas,ds1337";
+		reg = <0x68>;
+	};
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	pinctrl-1 = <&pinctrl_i2c4_gpio>;
+	scl-gpios = <&gpio4 14 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio4 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	ft5x06_ts: touchscreen@38 {
+		compatible = "edt,edt-ft5206";
+		reg = <0x38>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_touch>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <30 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <480>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+	};
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie>;
+	reset-gpios = <&gpio2 31 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+	status = "okay";
+};
+
+&snvs_poweroff {
+	status = "okay";
+};
+
+&snvs_pwrkey {
+	status = "okay";
+};
+
+&snvs_rtc {
+	status = "disabled";
+};
+
+/* DEBUG */
+&uart1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_uart1>;
+	pinctrl-1 = <&pinctrl_uart1_sleep>;
+	assigned-clocks = <&clks IMX7D_UART1_ROOT_SRC>;
+	assigned-clock-parents = <&clks IMX7D_OSC_24M_CLK>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	assigned-clocks = <&clks IMX7D_UART2_ROOT_SRC>;
+	assigned-clock-parents = <&clks IMX7D_OSC_24M_CLK>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usbh {
+	disable-over-current;
+	vbus-supply = <&reg_hsic_hub_pwr_on>;
+	status = "okay";
+};
+
+&usbotg1 {
+	dr_mode = "host";
+	disable-over-current;
+	vbus-supply = <&reg_usb_otg1_vbus>;
+	status = "okay";
+};
+
+&usbotg2 {
+	dr_mode = "host";
+	disable-over-current;
+	vbus-supply = <&reg_usb_otg2_vbus>;
+	status = "okay";
+};
+
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc1>, <&pinctrl_usdhc1_gpio>;
+	cd-gpios = <&gpio5 0 GPIO_ACTIVE_LOW>;
+	no-1-8-v;
+	keep-power-in-suspend;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+};
+
+&iomuxc {
+	pinctrl_gpio_keys: gpio-keysgrp {
+		fsl,pins = <
+			MX7D_PAD_EPDC_BDR0__GPIO2_IO28		0x32
+			MX7D_PAD_EPDC_BDR1__GPIO2_IO29		0x32
+			MX7D_PAD_GPIO1_IO11__GPIO1_IO11		0x32
+		>;
+	};
+
+	pinctrl_gpio_leds: gpio-ledsgrp {
+		fsl,pins = <
+			MX7D_PAD_GPIO1_IO13__GPIO1_IO13		0x59
+		>;
+	};
+
+	pinctrl_ecspi2: ecspi2grp {
+		fsl,pins = <
+			MX7D_PAD_ECSPI2_MISO__ECSPI2_MISO	0x2
+			MX7D_PAD_ECSPI2_MOSI__ECSPI2_MOSI	0x2
+			MX7D_PAD_ECSPI2_SCLK__ECSPI2_SCLK	0x2
+			MX7D_PAD_ECSPI2_SS0__GPIO4_IO23		0x2
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX7D_PAD_I2C3_SCL__FLEXCAN2_RX		0x59
+			MX7D_PAD_I2C3_SDA__FLEXCAN2_TX		0x59
+		>;
+	};
+
+	pinctrl_hsic_hub_connect: hsichubconnectgrp {
+		fsl,pins = <
+			MX7D_PAD_GPIO1_IO12__GPIO1_IO12		0x59
+		>;
+	};
+
+	pinctrl_hsic_hub_pwr: hsichubpwrgrp {
+		fsl,pins = <
+			MX7D_PAD_GPIO1_IO10__GPIO1_IO10		0x59
+		>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <
+			MX7D_PAD_I2C4_SDA__I2C4_SDA		0x4000007f
+			MX7D_PAD_I2C4_SCL__I2C4_SCL		0x4000007f
+		>;
+	};
+
+	pinctrl_i2c4_gpio: i2c4gpiogrp {
+		fsl,pins = <
+			MX7D_PAD_I2C4_SDA__GPIO4_IO15		0x4000007f
+			MX7D_PAD_I2C4_SCL__GPIO4_IO14		0x4000007f
+		>;
+	};
+
+	pinctrl_pcie: pciegrp {
+		fsl,pins = <
+			MX7D_PAD_EPDC_PWR_STAT__GPIO2_IO31	0x79
+		>;
+	};
+
+	pinctrl_touch: touchgrp {
+		fsl,pins = <
+			MX7D_PAD_EPDC_PWR_COM__GPIO2_IO30	0x80000000
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX7D_PAD_UART1_TX_DATA__UART1_DCE_TX	0x79
+			MX7D_PAD_UART1_RX_DATA__UART1_DCE_RX	0x79
+		>;
+	};
+
+	pinctrl_uart1_sleep: uart1sleep-grp {
+		fsl,pins = <
+			MX7D_PAD_UART1_TX_DATA__GPIO4_IO1	0x10
+			MX7D_PAD_UART1_RX_DATA__GPIO4_IO0	0x10
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX7D_PAD_LCD_ENABLE__UART2_DCE_TX	0x79
+			MX7D_PAD_LCD_CLK__UART2_DCE_RX		0x79
+			MX7D_PAD_LCD_VSYNC__UART2_DCE_CTS	0x79
+			MX7D_PAD_LCD_HSYNC__UART2_DCE_RTS	0x79
+		>;
+	};
+
+	pinctrl_usdhc1_gpio: usdhc1-gpiogrp {
+		fsl,pins = <
+			MX7D_PAD_SD1_CD_B__GPIO5_IO0		0x59
+			MX7D_PAD_GPIO1_IO08__SD1_VSELECT	0x59
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX7D_PAD_SD1_CMD__SD1_CMD		0x59
+			MX7D_PAD_SD1_CLK__SD1_CLK		0x19
+			MX7D_PAD_SD1_DATA0__SD1_DATA0		0x59
+			MX7D_PAD_SD1_DATA1__SD1_DATA1		0x59
+			MX7D_PAD_SD1_DATA2__SD1_DATA2		0x59
+			MX7D_PAD_SD1_DATA3__SD1_DATA3		0x59
+		>;
+	};
+};
+
+&iomuxc_lpsr {
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO02__PWM2_OUT	0x110b0
+		>;
+	};
+
+	pinctrl_usbotg1_pwr: usbotg1pwrgrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO05__GPIO1_IO5	0x14
+		>;
+	};
+
+	pinctrl_usbotg2_pwr: usbotg2pwrgrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO07__GPIO1_IO7	0x14
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO00__WDOG1_WDOG_B	0x74
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-mx7customboard.dts b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-mx7customboard.dts
new file mode 100644
index 00000000000000..d3fbf3b4b5a3ac
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-mx7customboard.dts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * VAR-MX7CustomBoard for VAR-SOM-MX7 with NAND
+ *
+ * Link: https://variscite.com/carrier-boards/var-mx7customboard/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx7d-var-som.dtsi"
+#include "imx7d-var-som-nand.dtsi"
+#include "imx7d-var-som-wm8904.dtsi"
+#include "imx7d-var-som-mx7customboard.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-MX7 with NAND flash & WM8904 on VAR-MX7CustomBoard";
+	compatible = "variscite,var-som-imx7d-nand-mx7customboard",
+		     "variscite,var-som-imx7d",
+		     "fsl,imx7d";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-wm8731-mx7customboard.dts b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-wm8731-mx7customboard.dts
new file mode 100644
index 00000000000000..603b2dbf4c8ea3
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand-wm8731-mx7customboard.dts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * VAR-MX7CustomBoard for VAR-SOM-MX7 with NAND and WM8731
+ *
+ * Link: https://variscite.com/carrier-boards/var-mx7customboard/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx7d-var-som.dtsi"
+#include "imx7d-var-som-nand.dtsi"
+#include "imx7d-var-som-wm8731.dtsi"
+#include "imx7d-var-som-mx7customboard.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-MX7 with NAND flash & WM8731 on VAR-MX7CustomBoard";
+	compatible = "variscite,var-som-imx7d-nand-wm8731-mx7customboard",
+		     "variscite,var-som-imx7d",
+		     "fsl,imx7d";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand.dtsi
new file mode 100644
index 00000000000000..d2c3b9e6007e28
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-nand.dtsi
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for NAND support on Variscite VAR-SOM-MX7
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-7/var-som-mx7/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+&gpmi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpmi_nand>;
+	status = "okay";
+
+	nand@0 {
+		reg = <0>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "spl";
+				reg = <0x00000000 0x00200000>;
+			};
+
+			partition@200000 {
+				label = "u-boot";
+				reg = <0x00200000 0x00200000>;
+			};
+
+			partition@400000 {
+				label = "u-boot_env";
+				reg = <0x00400000 0x00200000>;
+			};
+
+			partition@600000 {
+				label = "kernel";
+				reg = <0x00600000 0x00c00000>;
+			};
+
+			partition@1200000 {
+				label = "rootfs";
+				reg = <0x01200000 0x3ee00000>;
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_gpmi_nand: gpminandgrp {
+		fsl,pins = <
+			MX7D_PAD_SD3_CLK__NAND_CLE		0x71
+			MX7D_PAD_SD3_CMD__NAND_ALE		0x71
+			MX7D_PAD_SAI1_MCLK__NAND_WP_B		0x71
+			MX7D_PAD_SAI1_TX_BCLK__NAND_CE0_B	0x71
+			MX7D_PAD_SAI1_RX_DATA__NAND_CE1_B	0x71
+			MX7D_PAD_SAI1_TX_DATA__NAND_READY_B	0x74
+			MX7D_PAD_SD3_STROBE__NAND_RE_B		0x71
+			MX7D_PAD_SD3_RESET_B__NAND_WE_B		0x71
+			MX7D_PAD_SD3_DATA0__NAND_DATA00		0x71
+			MX7D_PAD_SD3_DATA1__NAND_DATA01		0x71
+			MX7D_PAD_SD3_DATA2__NAND_DATA02		0x71
+			MX7D_PAD_SD3_DATA3__NAND_DATA03		0x71
+			MX7D_PAD_SD3_DATA4__NAND_DATA04		0x71
+			MX7D_PAD_SD3_DATA5__NAND_DATA05		0x71
+			MX7D_PAD_SD3_DATA6__NAND_DATA06		0x71
+			MX7D_PAD_SD3_DATA7__NAND_DATA07		0x71
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-emmc-mx7customboard.dts b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-emmc-mx7customboard.dts
new file mode 100644
index 00000000000000..1c419251beddcb
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-emmc-mx7customboard.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * VAR-MX7CustomBoard for VAR-SOM-MX7 V2 with eMMC
+ *
+ * Link: https://variscite.com/carrier-boards/var-mx7customboard/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx7d-var-som.dtsi"
+#include "imx7d-var-som-v2.dtsi"
+#include "imx7d-var-som-emmc.dtsi"
+#include "imx7d-var-som-wm8904.dtsi"
+#include "imx7d-var-som-mx7customboard.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-MX7-5G_V2 with eMMC & WM8904 on VAR-MX7CustomBoard";
+	compatible = "variscite,var-som-imx7d-v2-emmc-mx7customboard",
+		     "variscite,var-som-imx7d-v2",
+		     "fsl,imx7d";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-nand-mx7customboard.dts b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-nand-mx7customboard.dts
new file mode 100644
index 00000000000000..68569111f3a306
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2-nand-mx7customboard.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * VAR-MX7CustomBoard for VAR-SOM-MX7 V2 with NAND
+ *
+ * Link: https://variscite.com/carrier-boards/var-mx7customboard/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx7d-var-som.dtsi"
+#include "imx7d-var-som-v2.dtsi"
+#include "imx7d-var-som-nand.dtsi"
+#include "imx7d-var-som-wm8904.dtsi"
+#include "imx7d-var-som-mx7customboard.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-MX7-5G_V2 with NAND flash & WM8904 on VAR-MX7CustomBoard";
+	compatible = "variscite,var-som-imx7d-v2-nand-mx7customboard",
+		     "variscite,var-som-imx7d-v2",
+		     "fsl,imx7d";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2.dtsi
new file mode 100644
index 00000000000000..8aa9d0950a97b3
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-v2.dtsi
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * dtsi for Variscite VAR-SOM-MX7 V2
+ *
+ * Changelog link: https://dev.variscite.com/var-som-mx7/changelog/#var-som-mx7_v2-v20
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/ {
+	model = "Variscite i.MX7 Dual VAR-SOM-MX7 V2";
+	compatible = "variscite,var-som-imx7d-v2", "fsl,imx7d";
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <10000>;
+		reset-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&fec1 {
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+};
+
+&fec2 {
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+};
+
+&pinctrl_wlan {
+	fsl,pins = <
+		MX7D_PAD_LPSR_GPIO1_IO04__GPIO1_IO4	0x09
+	>;
+};
+
+&pinctrl_wlan_sleep {
+	fsl,pins = <
+		MX7D_PAD_LPSR_GPIO1_IO04__GPIO1_IO4	0x10
+	>;
+};
+
+/* BT */
+&uart3 {
+	bluetooth_iw61x: bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+/* WiFi */
+&usdhc2 {
+	/delete-property/ #address-cells;
+	/delete-property/ #size-cells;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	/delete-node/ wifi@1;
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8731.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8731.dtsi
new file mode 100644
index 00000000000000..ab05de43cc53ae
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8731.dtsi
@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for WM8731 support on Variscite VAR-SOM-MX7
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-7/var-som-mx7/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/ {
+	codec_wm8731: sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&sound_master>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&sound_master>;
+		simple-audio-card,name = "wm8731audio";
+		simple-audio-card,routing =
+			"Headphone Jack", "RHPOUT",
+			"Headphone Jack", "LHPOUT",
+			"LLINEIN", "Line Jack",
+			"RLINEIN", "Line Jack",
+			"MICIN", "Mic Bias",
+			"Mic Bias", "Mic Jack";
+		simple-audio-card,widgets =
+			"Headphone", "Headphone Jack",
+			"Line", "Line Jack",
+			"Microphone", "Mic Jack";
+
+		sound_master: simple-audio-card,cpu {
+			sound-dai = <&sai1>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&wm8731>;
+			system-clock-frequency = <12288000>;
+		};
+	};
+};
+
+&i2c2 {
+	wm8731: audio-codec@1a {
+		compatible = "wlf,wm8731";
+		reg = <0x1a>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&vgen6_reg>;
+		DBVDD-supply = <&vgen6_reg>;
+		DCVDD-supply = <&vgen6_reg>;
+		HPVDD-supply = <&vgen6_reg>;
+		clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+		clock-names = "mclk";
+		assigned-clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_SRC>,
+				  <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+		assigned-clock-parents = <&clks IMX7D_PLL_AUDIO_POST_DIV>;
+		assigned-clock-rates = <0>, <12288000>;
+	};
+};
+
+&sai1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_sai1>, <&pinctrl_sai1_mclk>;
+	pinctrl-1 = <&pinctrl_sai1_sleep>, <&pinctrl_sai1_mclk_sleep>;
+	assigned-clocks = <&clks IMX7D_SAI1_ROOT_SRC>,
+			  <&clks IMX7D_SAI1_ROOT_CLK>;
+	assigned-clock-parents = <&clks IMX7D_PLL_AUDIO_POST_DIV>;
+	assigned-clock-rates = <0>, <36864000>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <
+			MX7D_PAD_ENET1_RX_CLK__SAI1_TX_BCLK		0x1f
+			MX7D_PAD_ENET1_CRS__SAI1_TX_SYNC		0x1f
+			MX7D_PAD_ENET1_COL__SAI1_TX_DATA0		0x30
+			MX7D_PAD_ENET1_TX_CLK__SAI1_RX_DATA0		0x1f
+		>;
+	};
+
+	pinctrl_sai1_sleep: sai1-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_ENET1_RX_CLK__GPIO7_IO13		0x10
+			MX7D_PAD_ENET1_CRS__GPIO7_IO14			0x10
+			MX7D_PAD_ENET1_COL__GPIO7_IO15			0x10
+			MX7D_PAD_ENET1_TX_CLK__GPIO7_IO12		0x10
+		>;
+	};
+};
+
+&iomuxc_lpsr {
+	pinctrl_sai1_mclk: sai1-mclkgrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO01__SAI1_MCLK		0x1f
+		>;
+	};
+
+	pinctrl_sai1_mclk_sleep: sai1-mclksleepgrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO01__GPIO1_IO1		0x10
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8904.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8904.dtsi
new file mode 100644
index 00000000000000..b56f3680edcffe
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som-wm8904.dtsi
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for WM8904 support on Variscite VAR-SOM-MX7
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-7/var-som-mx7/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/ {
+	codec_wm8904: sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&dailink_master>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&dailink_master>;
+		simple-audio-card,name = "wm8904-audio";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"IN1L", "Microphone Jack",
+			"IN1R", "Microphone Jack",
+			"Microphone Jack", "MICBIAS";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+		simple-audio-card,mclk-fs = <256>;
+
+		dailink_master: simple-audio-card,codec {
+			sound-dai = <&wm8904>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai1>;
+		};
+	};
+};
+
+&i2c2 {
+	wm8904: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		#sound-dai-cells = <0>;
+		clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_DIV>;
+		clock-names = "mclk";
+		AVDD-supply = <&vgen6_reg>;
+		CPVDD-supply = <&vgen6_reg>;
+		DCVDD-supply = <&vgen6_reg>;
+		DBVDD-supply = <&vgen6_reg>;
+		MICVDD-supply = <&vgen6_reg>;
+		wlf,drc-cfg-names = "default", "peaklimiter", "tradition",
+				    "soft", "music";
+		/*
+		 * Config registers per name, respectively:
+		 * KNEE_IP = 0,   KNEE_OP = 0,     HI_COMP = 1,   LO_COMP = 1
+		 * KNEE_IP = -24, KNEE_OP = -6,    HI_COMP = 1/4, LO_COMP = 1
+		 * KNEE_IP = -42, KNEE_OP = -3,    HI_COMP = 0,   LO_COMP = 1
+		 * KNEE_IP = -45, KNEE_OP = -9,    HI_COMP = 1/8, LO_COMP = 1
+		 * KNEE_IP = -30, KNEE_OP = -10.5, HI_COMP = 1/4, LO_COMP = 1
+		 */
+		wlf,drc-cfg-regs = /bits/ 16 <0x01af 0x3248 0x0000 0x0000>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x0408>,
+				   /bits/ 16 <0x04af 0x324b 0x0028 0x0704>,
+				   /bits/ 16 <0x04af 0x324b 0x0018 0x078c>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x050e>;
+		/* GPIO1 = DMIC_CLK, don't touch others */
+		wlf,gpio-cfg = <0x0018>, <0xffff>, <0xffff>, <0xffff>;
+	};
+};
+
+&sai1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_sai1>, <&pinctrl_sai1_mclk>;
+	pinctrl-1 = <&pinctrl_sai1_sleep>, <&pinctrl_sai1_mclk_sleep>;
+	assigned-clocks = <&clks IMX7D_SAI1_ROOT_SRC>,
+			  <&clks IMX7D_SAI1_ROOT_CLK>;
+	assigned-clock-parents = <&clks IMX7D_PLL_AUDIO_POST_DIV>;
+	assigned-clock-rates = <0>, <12288000>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <
+			MX7D_PAD_ENET1_RX_CLK__SAI1_TX_BCLK		0x1f
+			MX7D_PAD_ENET1_CRS__SAI1_TX_SYNC		0x1f
+			MX7D_PAD_ENET1_COL__SAI1_TX_DATA0		0x30
+			MX7D_PAD_ENET1_TX_CLK__SAI1_RX_DATA0		0x1f
+		>;
+	};
+
+	pinctrl_sai1_sleep: sai1-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_ENET1_RX_CLK__GPIO7_IO13		0x10
+			MX7D_PAD_ENET1_CRS__GPIO7_IO14			0x10
+			MX7D_PAD_ENET1_COL__GPIO7_IO15			0x10
+			MX7D_PAD_ENET1_TX_CLK__GPIO7_IO12		0x10
+		>;
+	};
+};
+
+&iomuxc_lpsr {
+	pinctrl_sai1_mclk: sai1-mclkgrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO01__SAI1_MCLK		0x1f
+		>;
+	};
+
+	pinctrl_sai1_mclk_sleep: sai1-mclksleepgrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO01__GPIO1_IO1		0x10
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7d-var-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx7d-var-som.dtsi
new file mode 100644
index 00000000000000..255cf449007713
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx7d-var-som.dtsi
@@ -0,0 +1,533 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for Variscite VAR-SOM-MX7
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-7/var-som-mx7/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+#include <dt-bindings/leds/common.h>
+#include "imx7d.dtsi"
+
+/ {
+	model = "Variscite i.MX7 Dual VAR-SOM-MX7";
+	compatible = "variscite,var-som-imx7d", "fsl,imx7d";
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x80000000>;
+	};
+
+	reg_phy_vddio: regulator-phy-vddio {
+		compatible = "regulator-fixed";
+		regulator-name = "vddio-1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	reg_vmmc_wl: regulator-vmmc-wl {
+		compatible = "regulator-fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "wlreg_on";
+	};
+
+	reg_vref_1v8: regulator-vref-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vref-1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+};
+
+&adc1 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+&adc2 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+&cpu0 {
+	cpu-supply = <&sw1a_reg>;
+};
+
+&cpu1 {
+	cpu-supply = <&sw1a_reg>;
+};
+
+&ecspi1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	pinctrl-1 = <&pinctrl_ecspi1_sleep>;
+	cs-gpios = <&gpio4 19 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	/* Resistive touch controller */
+	ads7846_ts: touchscreen@0 {
+		reg = <0>;
+		compatible = "ti,ads7846";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_restouch>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <9 IRQ_TYPE_EDGE_FALLING>;
+		spi-max-frequency = <1500000>;
+		pendown-gpio = <&gpio1 9 GPIO_ACTIVE_LOW>;
+		ti,x-min = /bits/ 16 <125>;
+		ti,x-max = /bits/ 16 <4008>;
+		ti,y-min = /bits/ 16 <282>;
+		ti,y-max = /bits/ 16 <3864>;
+		ti,x-plate-ohms = /bits/ 16 <180>;
+		ti,pressure-max = /bits/ 16 <255>;
+		ti,debounce-max = /bits/ 16 <10>;
+		ti,debounce-tol = /bits/ 16 <3>;
+		ti,debounce-rep = /bits/ 16 <1>;
+		ti,settle-delay-usec = /bits/ 16 <150>;
+		ti,keep-vref-on;
+	};
+};
+
+&fec1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_enet1>,
+		    <&pinctrl_enet1_rst>,
+		    <&pinctrl_enet2_rst>;
+	pinctrl-1 = <&pinctrl_enet1_sleep>,
+		    <&pinctrl_enet1_rst_sleep>,
+		    <&pinctrl_enet2_rst_sleep>;
+	phy-supply = <&vgen3_reg>;
+	assigned-clocks = <&clks IMX7D_ENET_PHY_REF_ROOT_SRC>,
+			  <&clks IMX7D_ENET_AXI_ROOT_SRC>,
+			  <&clks IMX7D_ENET1_TIME_ROOT_SRC>,
+			  <&clks IMX7D_ENET1_TIME_ROOT_CLK>,
+			  <&clks IMX7D_ENET_AXI_ROOT_CLK>;
+	assigned-clock-parents = <&clks IMX7D_PLL_ENET_MAIN_25M_CLK>,
+				 <&clks IMX7D_PLL_ENET_MAIN_250M_CLK>,
+				 <&clks IMX7D_PLL_ENET_MAIN_100M_CLK>;
+	assigned-clock-rates = <0>, <0>, <0>, <100000000>, <250000000>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy0>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			reset-gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <20000>;
+			reset-deassert-us = <80000>;
+			vddio-supply = <&reg_phy_vddio>;
+		};
+
+		ethphy1: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			reset-gpios = <&gpio4 3 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <20000>;
+			reset-deassert-us = <80000>;
+			vddio-supply = <&reg_phy_vddio>;
+		};
+	};
+};
+
+&fec2 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_enet2>;
+	pinctrl-1 = <&pinctrl_enet2_sleep>;
+	phy-supply = <&vgen3_reg>;
+	assigned-clocks = <&clks IMX7D_ENET_PHY_REF_ROOT_SRC>,
+			  <&clks IMX7D_ENET_AXI_ROOT_SRC>,
+			  <&clks IMX7D_ENET2_TIME_ROOT_SRC>,
+			  <&clks IMX7D_ENET2_TIME_ROOT_CLK>,
+			  <&clks IMX7D_ENET_AXI_ROOT_CLK>;
+	assigned-clock-parents = <&clks IMX7D_PLL_ENET_MAIN_25M_CLK>,
+				 <&clks IMX7D_PLL_ENET_MAIN_250M_CLK>,
+				 <&clks IMX7D_PLL_ENET_MAIN_100M_CLK>;
+	assigned-clock-rates = <0>, <0>, <0>, <100000000>, <250000000>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy1>;
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	scl-gpios = <&gpio4 8 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio4 9 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pfuze300: pmic@8 {
+		compatible = "fsl,pfuze3000";
+		reg = <0x08>;
+
+		regulators {
+			sw1a_reg: sw1a {
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			/* use sw1c_reg to align with pfuze100/pfuze200 */
+			sw1c_reg: sw1b {
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1475000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw2_reg: sw2 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw3a_reg: sw3 {
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <1650000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			swbst_reg: swbst {
+				regulator-min-microvolt = <5000000>;
+				regulator-max-microvolt = <5150000>;
+			};
+
+			snvs_reg: vsnvs {
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vref_reg: vrefddr {
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vgen1_reg: vldo1 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen2_reg: vldo2 {
+				status = "disabled";
+			};
+
+			vgen3_reg: vccsd {
+				regulator-min-microvolt = <2850000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vgen4_reg: v33 {
+				regulator-min-microvolt = <2850000>;
+				regulator-max-microvolt = <2850000>;
+				regulator-always-on;
+			};
+
+			vgen5_reg: vldo3 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+
+			vgen6_reg: vldo4 {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	scl-gpios = <&gpio4 10 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio4 11 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+};
+
+&reg_1p0d {
+	vin-supply = <&sw2_reg>;
+};
+
+&reg_1p2 {
+	vin-supply = <&sw2_reg>;
+};
+
+/* BT */
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>, <&pinctrl_bt>;
+	assigned-clocks = <&clks IMX7D_UART3_ROOT_SRC>;
+	assigned-clock-parents = <&clks IMX7D_PLL_SYS_MAIN_240M_CLK>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+/* WiFi */
+&usdhc2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_wlan>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_wlan>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_wlan>;
+	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_wlan_sleep>;
+	keep-power-in-suspend;
+	non-removable;
+	vmmc-supply = <&reg_vmmc_wl>;
+	status = "okay";
+
+	brcmf: wifi@1 {
+		compatible = "brcm,bcm4329-fmac";
+		reg = <1>;
+	};
+};
+
+&iomuxc {
+	/* GPIO Used in User Space init script */
+	pinctrl_bt: btgrp {
+		fsl,pins = <
+			MX7D_PAD_GPIO1_IO14__GPIO1_IO14			0x80000000
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX7D_PAD_ECSPI1_MISO__ECSPI1_MISO		0x2
+			MX7D_PAD_ECSPI1_MOSI__ECSPI1_MOSI		0x2
+			MX7D_PAD_ECSPI1_SCLK__ECSPI1_SCLK		0x2
+			MX7D_PAD_ECSPI1_SS0__GPIO4_IO19			0x2
+		>;
+	};
+
+	pinctrl_ecspi1_sleep: ecspi1-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_ECSPI1_MISO__GPIO4_IO18		0x10
+			MX7D_PAD_ECSPI1_MOSI__GPIO4_IO17		0x10
+			MX7D_PAD_ECSPI1_SCLK__GPIO4_IO16		0x10
+		>;
+	};
+
+	pinctrl_enet1: enet1grp {
+		fsl,pins = <
+			MX7D_PAD_SD2_CD_B__ENET1_MDIO			0x3
+			MX7D_PAD_SD2_WP__ENET1_MDC			0x3
+			MX7D_PAD_ENET1_RGMII_TXC__ENET1_RGMII_TXC	0x1
+			MX7D_PAD_ENET1_RGMII_TD0__ENET1_RGMII_TD0	0x1
+			MX7D_PAD_ENET1_RGMII_TD1__ENET1_RGMII_TD1	0x1
+			MX7D_PAD_ENET1_RGMII_TD2__ENET1_RGMII_TD2	0x1
+			MX7D_PAD_ENET1_RGMII_TD3__ENET1_RGMII_TD3	0x1
+			MX7D_PAD_ENET1_RGMII_TX_CTL__ENET1_RGMII_TX_CTL	0x1
+			MX7D_PAD_ENET1_RGMII_RXC__ENET1_RGMII_RXC	0x1
+			MX7D_PAD_ENET1_RGMII_RD0__ENET1_RGMII_RD0	0x1
+			MX7D_PAD_ENET1_RGMII_RD1__ENET1_RGMII_RD1	0x1
+			MX7D_PAD_ENET1_RGMII_RD2__ENET1_RGMII_RD2	0x1
+			MX7D_PAD_ENET1_RGMII_RD3__ENET1_RGMII_RD3	0x1
+			MX7D_PAD_ENET1_RGMII_RX_CTL__ENET1_RGMII_RX_CTL	0x1
+		>;
+	};
+
+	pinctrl_enet1_rst: enet1rstgrp {
+		fsl,pins = <
+			MX7D_PAD_SD2_RESET_B__GPIO5_IO11		0x59
+		>;
+	};
+
+	pinctrl_enet1_sleep: enet1-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_SD2_CD_B__GPIO5_IO9			0x10
+			MX7D_PAD_SD2_WP__GPIO5_IO10			0x10
+			MX7D_PAD_ENET1_RGMII_TXC__GPIO7_IO11		0x10
+			MX7D_PAD_ENET1_RGMII_TD0__GPIO7_IO6		0x10
+			MX7D_PAD_ENET1_RGMII_TD1__GPIO7_IO7		0x10
+			MX7D_PAD_ENET1_RGMII_TD2__GPIO7_IO8		0x10
+			MX7D_PAD_ENET1_RGMII_TD3__GPIO7_IO9		0x10
+			MX7D_PAD_ENET1_RGMII_TX_CTL__GPIO7_IO10		0x10
+			MX7D_PAD_ENET1_RGMII_RXC__GPIO7_IO5		0x10
+			MX7D_PAD_ENET1_RGMII_RD0__GPIO7_IO0		0x10
+			MX7D_PAD_ENET1_RGMII_RD1__GPIO7_IO1		0x10
+			MX7D_PAD_ENET1_RGMII_RD2__GPIO7_IO2		0x10
+			MX7D_PAD_ENET1_RGMII_RD3__GPIO7_IO3		0x10
+			MX7D_PAD_ENET1_RGMII_RX_CTL__GPIO7_IO4		0x10
+		>;
+	};
+
+	pinctrl_enet1_rst_sleep: enet1-rstsleepgrp {
+		fsl,pins = <
+			MX7D_PAD_SD2_RESET_B__GPIO5_IO11		0x10
+		>;
+	};
+
+	pinctrl_enet2: enet2grp {
+		fsl,pins = <
+			MX7D_PAD_EPDC_GDSP__ENET2_RGMII_TXC		0x1
+			MX7D_PAD_EPDC_SDCE2__ENET2_RGMII_TD0		0x1
+			MX7D_PAD_EPDC_SDCE3__ENET2_RGMII_TD1		0x1
+			MX7D_PAD_EPDC_GDCLK__ENET2_RGMII_TD2		0x1
+			MX7D_PAD_EPDC_GDOE__ENET2_RGMII_TD3		0x1
+			MX7D_PAD_EPDC_GDRL__ENET2_RGMII_TX_CTL		0x1
+			MX7D_PAD_EPDC_SDCE1__ENET2_RGMII_RXC		0x1
+			MX7D_PAD_EPDC_SDCLK__ENET2_RGMII_RD0		0x1
+			MX7D_PAD_EPDC_SDLE__ENET2_RGMII_RD1		0x1
+			MX7D_PAD_EPDC_SDOE__ENET2_RGMII_RD2		0x1
+			MX7D_PAD_EPDC_SDSHR__ENET2_RGMII_RD3		0x1
+			MX7D_PAD_EPDC_SDCE0__ENET2_RGMII_RX_CTL		0x1
+		>;
+	};
+
+	pinctrl_enet2_rst: enet2rstgrp {
+		fsl,pins = <
+			MX7D_PAD_UART2_TX_DATA__GPIO4_IO3		0x59
+		>;
+	};
+
+	pinctrl_enet2_sleep: enet2-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_EPDC_GDSP__GPIO2_IO27			0x10
+			MX7D_PAD_EPDC_SDCE2__GPIO2_IO22			0x10
+			MX7D_PAD_EPDC_SDCE3__GPIO2_IO23			0x10
+			MX7D_PAD_EPDC_GDCLK__GPIO2_IO24			0x10
+			MX7D_PAD_EPDC_GDOE__GPIO2_IO25			0x10
+			MX7D_PAD_EPDC_GDRL__GPIO2_IO26			0x10
+			MX7D_PAD_EPDC_SDCE1__GPIO2_IO21			0x10
+			MX7D_PAD_EPDC_SDCLK__GPIO2_IO16			0x10
+			MX7D_PAD_EPDC_SDLE__GPIO2_IO17			0x10
+			MX7D_PAD_EPDC_SDOE__GPIO2_IO18			0x10
+			MX7D_PAD_EPDC_SDSHR__GPIO2_IO19			0x10
+			MX7D_PAD_EPDC_SDCE0__GPIO2_IO20			0x10
+		>;
+	};
+
+	pinctrl_enet2_rst_sleep: enet2-rstsleepgrp {
+		fsl,pins = <
+			MX7D_PAD_UART2_TX_DATA__GPIO4_IO3		0x10
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX7D_PAD_I2C1_SDA__I2C1_SDA			0x4000007f
+			MX7D_PAD_I2C1_SCL__I2C1_SCL			0x4000007f
+		>;
+	};
+
+	pinctrl_i2c1_gpio: i2c1gpiogrp {
+		fsl,pins = <
+			MX7D_PAD_I2C1_SDA__GPIO4_IO9			0x4000007f
+			MX7D_PAD_I2C1_SCL__GPIO4_IO8			0x4000007f
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX7D_PAD_I2C2_SDA__I2C2_SDA			0x4000007f
+			MX7D_PAD_I2C2_SCL__I2C2_SCL			0x4000007f
+		>;
+	};
+
+	pinctrl_i2c2_gpio: i2c2gpiogrp {
+		fsl,pins = <
+			MX7D_PAD_I2C2_SDA__GPIO4_IO11			0x4000007f
+			MX7D_PAD_I2C2_SCL__GPIO4_IO10			0x4000007f
+		>;
+	};
+
+	pinctrl_restouch: restouchgrp {
+		fsl,pins = <
+			MX7D_PAD_GPIO1_IO09__GPIO1_IO9			0x59
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX7D_PAD_UART3_TX_DATA__UART3_DCE_TX		0x79
+			MX7D_PAD_UART3_RX_DATA__UART3_DCE_RX		0x79
+			MX7D_PAD_UART3_CTS_B__UART3_DCE_CTS		0x79
+			MX7D_PAD_UART3_RTS_B__UART3_DCE_RTS		0x79
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX7D_PAD_SD2_CMD__SD2_CMD			0x59
+			MX7D_PAD_SD2_CLK__SD2_CLK			0x19
+			MX7D_PAD_SD2_DATA0__SD2_DATA0			0x59
+			MX7D_PAD_SD2_DATA1__SD2_DATA1			0x59
+			MX7D_PAD_SD2_DATA2__SD2_DATA2			0x59
+			MX7D_PAD_SD2_DATA3__SD2_DATA3			0x59
+		>;
+	};
+
+	pinctrl_usdhc2_sleep: usdhc2-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_SD2_CMD__GPIO5_IO13			0x10
+			MX7D_PAD_SD2_CLK__GPIO5_IO12			0x10
+			MX7D_PAD_SD2_DATA0__GPIO5_IO14			0x10
+			MX7D_PAD_SD2_DATA1__GPIO5_IO15			0x10
+			MX7D_PAD_SD2_DATA2__GPIO5_IO16			0x10
+			MX7D_PAD_SD2_DATA3__GPIO5_IO17			0x10
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX7D_PAD_SD2_CMD__SD2_CMD			0x5a
+			MX7D_PAD_SD2_CLK__SD2_CLK			0x1a
+			MX7D_PAD_SD2_DATA0__SD2_DATA0			0x5a
+			MX7D_PAD_SD2_DATA1__SD2_DATA1			0x5a
+			MX7D_PAD_SD2_DATA2__SD2_DATA2			0x5a
+			MX7D_PAD_SD2_DATA3__SD2_DATA3			0x5a
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX7D_PAD_SD2_CMD__SD2_CMD			0x5b
+			MX7D_PAD_SD2_CLK__SD2_CLK			0x1b
+			MX7D_PAD_SD2_DATA0__SD2_DATA0			0x5b
+			MX7D_PAD_SD2_DATA1__SD2_DATA1			0x5b
+			MX7D_PAD_SD2_DATA2__SD2_DATA2			0x5b
+			MX7D_PAD_SD2_DATA3__SD2_DATA3			0x5b
+		>;
+	};
+};
+
+&iomuxc_lpsr {
+	/* GPIO Used in User Space init script */
+	pinctrl_wlan: wlangrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO04__GPIO1_IO4		0x09
+		>;
+	};
+
+	pinctrl_wlan_sleep: wlan-sleepgrp {
+		fsl,pins = <
+			MX7D_PAD_LPSR_GPIO1_IO04__GPIO1_IO4		0x10
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts b/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts
index 25f38acc53501b..b1989013c1a4a5 100644
--- a/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx7s-warp.dts
@@ -272,7 +272,7 @@
 	status = "okay";
 };
 
-&uart3  {
+&uart3 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart3>;
 	assigned-clocks = <&clks IMX7D_UART3_ROOT_SRC>;
diff --git a/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi b/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi
index 1fda60d62ffee6..fd05479bbffc97 100644
--- a/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/mba6ulx.dtsi
@@ -118,16 +118,6 @@
 		vin-supply = <&reg_mba6ul_3v3>;
 	};
 
-	reg_otg2vbus_5v0: regulator-otg2-vbus-5v0 {
-		compatible = "regulator-fixed";
-		gpio = <&expander_out1 0 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-		regulator-name = "otg2-vbus-supply-5v0";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		vin-supply = <&reg_mpcie>;
-	};
-
 	reserved-memory {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -215,6 +205,8 @@
 			clocks = <&clks IMX6UL_CLK_ENET_REF>;
 			reg = <0>;
 			max-speed = <100>;
+			interrupt-parent = <&expander_in0>;
+			interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
 		};
 
 		ethphy1: ethernet-phy@1 {
@@ -222,6 +214,8 @@
 			clocks = <&clks IMX6UL_CLK_ENET2_REF_125M>;
 			reg = <1>;
 			max-speed = <100>;
+			interrupt-parent = <&expander_in0>;
+			interrupts = <7 IRQ_TYPE_LEVEL_LOW>;
 		};
 	};
 };
@@ -261,18 +255,6 @@
 		gpio-controller;
 		#gpio-cells = <2>;
 		vcc-supply = <&reg_mba6ul_3v3>;
-
-		enet1_int-hog {
-			gpio-hog;
-			gpios = <6 0>;
-			input;
-		};
-
-		enet2_int-hog {
-			gpio-hog;
-			gpios = <7 0>;
-			input;
-		};
 	};
 
 	expander_out1: gpio-expander@22 {
@@ -377,10 +359,18 @@
 /* 7-port usb hub */
 /* id, pwr, oc pins not connected */
 &usbotg2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
 	disable-over-current;
-	vbus-supply = <&reg_otg2vbus_5v0>;
 	dr_mode = "host";
 	status = "okay";
+
+	hub_2_0: hub@1 {
+		compatible = "usb424,2517";
+		reg = <1>;
+		reset-gpios = <&expander_out1 0 GPIO_ACTIVE_LOW>;
+		vdd-supply = <&reg_mba6ul_3v3>;
+	};
 };
 
 &usdhc1 {
diff --git a/arch/arm/boot/dts/nxp/ls/ls1021a.dtsi b/arch/arm/boot/dts/nxp/ls/ls1021a.dtsi
index e0b9ea6dd51005..af518fc08809f1 100644
--- a/arch/arm/boot/dts/nxp/ls/ls1021a.dtsi
+++ b/arch/arm/boot/dts/nxp/ls/ls1021a.dtsi
@@ -721,7 +721,7 @@
 					<GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			queue-group@2d14000  {
+			queue-group@2d14000 {
 				reg = <0x0 0x2d14000 0x0 0x1000>;
 				interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>,
@@ -740,14 +740,14 @@
 			ranges;
 			dma-coherent;
 
-			queue-group@2d50000  {
+			queue-group@2d50000 {
 				reg = <0x0 0x2d50000 0x0 0x1000>;
 				interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			queue-group@2d54000  {
+			queue-group@2d54000 {
 				reg = <0x0 0x2d54000 0x0 0x1000>;
 				interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>,
@@ -766,14 +766,14 @@
 			ranges;
 			dma-coherent;
 
-			queue-group@2d90000  {
+			queue-group@2d90000 {
 				reg = <0x0 0x2d90000 0x0 0x1000>;
 				interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			queue-group@2d94000  {
+			queue-group@2d94000 {
 				reg = <0x0 0x2d94000 0x0 0x1000>;
 				interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm/boot/dts/nxp/vf/Makefile b/arch/arm/boot/dts/nxp/vf/Makefile
index 0a4a7f9dd43e4e..47be8bc318e285 100644
--- a/arch/arm/boot/dts/nxp/vf/Makefile
+++ b/arch/arm/boot/dts/nxp/vf/Makefile
@@ -1,8 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_SOC_VF610) += \
 	vf500-colibri-eval-v3.dtb \
+	vf500-colibri-iris.dtb \
 	vf610-bk4.dtb \
 	vf610-colibri-eval-v3.dtb \
+	vf610-colibri-iris.dtb \
 	vf610m4-colibri.dtb \
 	vf610-cosmic.dtb \
 	vf610m4-cosmic.dtb \
diff --git a/arch/arm/boot/dts/nxp/vf/vf-colibri-iris.dtsi b/arch/arm/boot/dts/nxp/vf/vf-colibri-iris.dtsi
new file mode 100644
index 00000000000000..36e34dd766aee0
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/vf/vf-colibri-iris.dtsi
@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright 2026 Mehmet Fide
+ *
+ * Toradex Iris carrier board V1.1, common part for Colibri VF50 and VF61.
+ *
+ * The Colibri modules are pin compatible, so the carrier signals sit on the
+ * same SODIMM pins as they do for the i.MX modules, whose Iris device trees
+ * are already upstream. The two SODIMM pins below are taken from the Colibri
+ * VFxx datasheet (102 -> PTA12 -> PORT0[5], 104 -> PTD28 -> PORT2[2]).
+ *
+ * An Iris V2.0 carries the same signals on the same pins, so everything here
+ * works on it as well. What it adds is a regulator on SODIMM 100 that can cut
+ * power to the uSD slot, which this file does not describe.
+ */
+
+/ {
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	reg_usbh_vbus: regulator-usbh-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbh1_reg>;
+		regulator-name = "VCC_USB[1-4]";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_LOW>;	/* SODIMM 129, USBH_PEN */
+	};
+};
+
+&gpio0 {
+	/*
+	 * The RS232 transceivers on the carrier are switched on by these two
+	 * lines. Delete the hog to turn a transceiver off from userspace.
+	 * The pin group belongs to the gpio controller rather than to iomuxc,
+	 * which would depend on its own child and log a dependency cycle.
+	 */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_iris_uart1_tx_on>;
+
+	uart1-tx-on-hog {
+		gpio-hog;
+		gpios = <5 GPIO_ACTIVE_HIGH>;	/* SODIMM 102 */
+		output-high;
+	};
+};
+
+&gpio2 {
+	pinctrl-names = "default";
+	/* keep the EXT_IO group from vf-colibri.dtsi alongside ours */
+	pinctrl-0 = <&pinctrl_gpio_ext>, <&pinctrl_iris_uart25_tx_on>;
+
+	uart25-tx-on-hog {
+		gpio-hog;
+		gpios = <2 GPIO_ACTIVE_HIGH>;	/* SODIMM 104 */
+		output-high;
+	};
+};
+
+&iomuxc {
+	pinctrl_iris_uart1_tx_on: irisuart1txongrp {
+		fsl,pins = <
+			VF610_PAD_PTA12__GPIO_5		0x22ef
+		>;
+	};
+
+	pinctrl_iris_uart25_tx_on: irisuart25txongrp {
+		fsl,pins = <
+			VF610_PAD_PTD28__GPIO_66	0x22ef
+		>;
+	};
+};
+
+/* Colibri SSP, on the extension connector */
+&dspi1 {
+	status = "okay";
+};
+
+&esdhc1 {
+	status = "okay";
+};
+
+&fec1 {
+	status = "okay";
+};
+
+&pwm0 {
+	status = "okay";
+};
+
+&pwm1 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	/* M41T0M6 real time clock on the carrier */
+	rtc@68 {
+		compatible = "st,m41t0";
+		reg = <0x68>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&usbdev0 {
+	status = "okay";
+};
+
+&usbh1 {
+	vbus-supply = <&reg_usbh_vbus>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/nxp/vf/vf-colibri.dtsi b/arch/arm/boot/dts/nxp/vf/vf-colibri.dtsi
index 98f9ee1b00306a..a9269857df2c12 100644
--- a/arch/arm/boot/dts/nxp/vf/vf-colibri.dtsi
+++ b/arch/arm/boot/dts/nxp/vf/vf-colibri.dtsi
@@ -170,6 +170,62 @@
 	status = "okay";
 };
 
+&gpio0 {
+	gpio-line-names = "", "", "SODIMM_89", "",
+			  "", "SODIMM_102", "SODIMM_18", "SODIMM_134",
+			  "SODIMM_16", "SODIMM_14", "SODIMM_23", "",
+			  "", "", "SODIMM_47", "SODIMM_190",
+			  "SODIMM_192", "SODIMM_49", "SODIMM_51", "SODIMM_53",
+			  "SODIMM_37", "SODIMM_29", "", "SODIMM_30",
+			  "SODIMM_20", "SODIMM_24", "SODIMM_21", "SODIMM_19",
+			  "SODIMM_94", "SODIMM_81", "SODIMM_28";
+};
+
+&gpio1 {
+	gpio-line-names = "SODIMM_35", "SODIMM_33", "SODIMM_27", "SODIMM_25",
+			  "SODIMM_196", "SODIMM_194", "SODIMM_63", "SODIMM_55",
+			  "SODIMM_65", "SODIMM_45", "SODIMM_43", "SODIMM_73",
+			  "SODIMM_77", "SODIMM_71", "SODIMM_98", "SODIMM_101",
+			  "SODIMM_103", "SODIMM_79", "SODIMM_97", "",
+			  "", "SODIMM_85", "", "",
+			  "", "", "", "",
+			  "", "", "", "SODIMM_106";
+};
+
+&gpio3 {
+	gpio-line-names = "SODIMM_105", "", "SODIMM_93", "",
+			  "", "", "", "SODIMM_95",
+			  "SODIMM_22", "SODIMM_68", "SODIMM_82", "SODIMM_56",
+			  "SODIMM_131", "SODIMM_44", "SODIMM_144", "SODIMM_146",
+			  "SODIMM_52", "SODIMM_54", "SODIMM_66", "SODIMM_64",
+			  "SODIMM_57", "SODIMM_61", "SODIMM_140", "SODIMM_142",
+			  "SODIMM_80", "SODIMM_46", "SODIMM_62", "SODIMM_48",
+			  "SODIMM_74", "SODIMM_50", "SODIMM_136", "SODIMM_138";
+};
+
+&gpio4 {
+	gpio-line-names = "SODIMM_76", "SODIMM_70", "SODIMM_60", "SODIMM_58",
+			  "SODIMM_78", "SODIMM_72", "SODIMM_96";
+};
+
+&gpio2 {
+	/*
+	 * EXT_IO_0..2 belong to no device, so hog the pin group here rather
+	 * than on the pin controller, which would depend on its own child.
+	 */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio_ext>;
+
+	gpio-line-names = "SODIMM_69", "SODIMM_99", "SODIMM_104", "SODIMM_107",
+			  "SODIMM_127", "SODIMM_184", "SODIMM_186", "",
+			  "", "", "", "",
+			  "", "", "", "SODIMM_38",
+			  "SODIMM_36", "SODIMM_34", "SODIMM_32", "SODIMM_129",
+			  "SODIMM_86", "SODIMM_90", "SODIMM_92", "SODIMM_88",
+			  "SODIMM_133", "SODIMM_135", "SODIMM_188", "SODIMM_75",
+			  "SODIMM_100";
+};
+
 &iomuxc {
 	pinctrl_flexcan0: can0grp {
 		fsl,pins = <
diff --git a/arch/arm/boot/dts/nxp/vf/vf500-colibri-iris.dts b/arch/arm/boot/dts/nxp/vf/vf500-colibri-iris.dts
new file mode 100644
index 00000000000000..aa91537cab3777
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/vf/vf500-colibri-iris.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright 2026 Mehmet Fide
+ */
+
+/dts-v1/;
+#include "vf500-colibri.dtsi"
+#include "vf-colibri-iris.dtsi"
+
+/ {
+	model = "Toradex Colibri VF50 on Iris Carrier Board";
+	compatible = "toradex,vf500-colibri-vf50-on-iris", "toradex,vf500-colibri_vf50",
+		     "fsl,vf500";
+};
diff --git a/arch/arm/boot/dts/nxp/vf/vf500.dtsi b/arch/arm/boot/dts/nxp/vf/vf500.dtsi
index 71ccdaa6f269a9..a73e9a80e56882 100644
--- a/arch/arm/boot/dts/nxp/vf/vf500.dtsi
+++ b/arch/arm/boot/dts/nxp/vf/vf500.dtsi
@@ -18,6 +18,12 @@
 		a5_cpu: cpu@0 {
 			compatible = "arm,cortex-a5";
 			device_type = "cpu";
+			d-cache-line-size = <32>;
+			d-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			i-cache-line-size = <32>;
+			i-cache-sets = <512>;
+			i-cache-size = <0x8000>;
 			reg = <0x0>;
 		};
 	};
diff --git a/arch/arm/boot/dts/nxp/vf/vf610-colibri-iris.dts b/arch/arm/boot/dts/nxp/vf/vf610-colibri-iris.dts
new file mode 100644
index 00000000000000..7db5aa490966ba
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/vf/vf610-colibri-iris.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright 2026 Mehmet Fide
+ */
+
+/dts-v1/;
+#include "vf610-colibri.dtsi"
+#include "vf-colibri-iris.dtsi"
+
+/ {
+	model = "Toradex Colibri VF61 on Iris Carrier Board";
+	compatible = "toradex,vf610-colibri-vf61-on-iris", "toradex,vf610-colibri_vf61",
+		     "fsl,vf610";
+};
diff --git a/arch/arm/boot/dts/nxp/vf/vf610.dtsi b/arch/arm/boot/dts/nxp/vf/vf610.dtsi
index 2fba923821d025..ea59e3d6ee49af 100644
--- a/arch/arm/boot/dts/nxp/vf/vf610.dtsi
+++ b/arch/arm/boot/dts/nxp/vf/vf610.dtsi
@@ -14,6 +14,9 @@
 		reg = <0x40006000 0x1000>;
 		cache-unified;
 		cache-level = <2>;
+		cache-line-size = <32>;
+		cache-sets = <2048>;
+		cache-size = <0x80000>;
 		arm,data-latency = <3 3 3>;
 		arm,tag-latency = <2 2 2>;
 	};
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8960-sony-huashan.dts b/arch/arm/boot/dts/qcom/qcom-msm8960-sony-huashan.dts
index 591dc837e60036..7d54ced3641fa9 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8960-sony-huashan.dts
+++ b/arch/arm/boot/dts/qcom/qcom-msm8960-sony-huashan.dts
@@ -95,6 +95,21 @@
 	status = "okay";
 };
 
+&riva {
+	vddcx-supply = <&pm8921_s3>;
+	vddmx-supply = <&pm8921_l24>;
+	vddpx-supply = <&pm8921_s4>;
+
+	status = "okay";
+
+	iris {
+		vdddig-supply = <&pm8921_lvs2>;
+		vddpa-supply = <&pm8921_l10>;
+		vddrfa-supply = <&pm8921_s2>;
+		vddxo-supply = <&pm8921_l4>;
+	};
+};
+
 &rpm {
 	regulators {
 		compatible = "qcom,rpm-pm8921-regulators";
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8960.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8960.dtsi
index fd28401cebb5e1..b9c28e468d32f4 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8960.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8960.dtsi
@@ -5,6 +5,7 @@
 #include <dt-bindings/clock/qcom,gcc-msm8960.h>
 #include <dt-bindings/reset/qcom,gcc-msm8960.h>
 #include <dt-bindings/clock/qcom,lcc-msm8960.h>
+#include <dt-bindings/clock/qcom,rpmcc.h>
 #include <dt-bindings/mfd/qcom-rpm.h>
 #include <dt-bindings/soc/qcom,gsbi.h>
 
@@ -76,11 +77,68 @@
 		};
 	};
 
+	firmware {
+		scm {
+			compatible = "qcom,scm-msm8960", "qcom,scm";
+
+			clocks = <&rpmcc RPM_DAYTONA_FABRIC_CLK>;
+			clock-names = "core";
+		};
+	};
+
 	memory@80000000 {
 		device_type = "memory";
 		reg = <0x80000000 0>;
 	};
 
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		smem_region: smem@80000000 {
+			reg = <0x80000000 0x200000>;
+			no-map;
+		};
+
+		wcnss_mem: wcnss@8f000000 {
+			reg = <0x8f000000 0x700000>;
+			no-map;
+		};
+	};
+
+	smem {
+		compatible = "qcom,smem";
+		memory-region = <&smem_region>;
+
+		hwlocks = <&sfpb_mutex 3>;
+	};
+
+	smsm {
+		compatible = "qcom,smsm";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		qcom,ipc-1 = <&l2cc 8 4>;
+		qcom,ipc-2 = <&l2cc 8 14>;
+		qcom,ipc-3 = <&l2cc 8 23>;
+		qcom,ipc-4 = <&sps_sic_non_secure 0x4094 0>;
+
+		apps_smsm: apps@0 {
+			reg = <0>;
+			#qcom,smem-state-cells = <1>;
+		};
+
+		wcnss_smsm: wcnss@3 {
+			reg = <3>;
+			interrupts = <GIC_SPI 204 IRQ_TYPE_EDGE_RISING>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
 	soc: soc {
 		compatible = "simple-bus";
 		ranges;
@@ -98,6 +156,16 @@
 			interrupt-names = "ack",
 					  "err",
 					  "wakeup";
+
+			rpmcc: clock-controller {
+				compatible = "qcom,rpmcc-apq8064",
+					     "qcom,rpmcc";
+				#clock-cells = <1>;
+				clocks = <&pxo_board>,
+					 <&cxo_board>;
+				clock-names = "pxo",
+					      "cxo";
+			};
 		};
 
 		ssbi: ssbi@500000 {
@@ -257,6 +325,34 @@
 				};
 			};
 
+			riva_bt_default_state: riva-bt-active-state {
+				pins = "gpio28", "gpio29";
+				function = "bt";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			riva_bt_sleep_state: riva-bt-sleep-state {
+				pins = "gpio28", "gpio29";
+				function = "bt";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			riva_wlan_default_state: riva-wlan-active-state {
+				pins = "gpio84", "gpio85", "gpio86", "gpio87", "gpio88";
+				function = "wlan";
+				drive-strength = <6>;
+				bias-pull-down;
+			};
+
+			riva_wlan_sleep_state: riva-wlan-sleep-state {
+				pins = "gpio84", "gpio85", "gpio86", "gpio87", "gpio88";
+				function = "wlan";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
 			sdcc3_default_state: sdcc3-default-state {
 				clk-pins {
 					pins = "sdc3_clk";
@@ -323,6 +419,12 @@
 			};
 		};
 
+		sfpb_mutex: hwlock@1200600 {
+			compatible = "qcom,sfpb-mutex";
+			reg = <0x01200600 0x100>;
+			#hwlock-cells = <1>;
+		};
+
 		intc: interrupt-controller@2000000 {
 			compatible = "qcom,msm-qgic2";
 			reg = <0x02000000 0x1000>,
@@ -390,6 +492,60 @@
 			};
 		};
 
+		riva: riva-pil@3200800 {
+			compatible = "qcom,riva-pil";
+			reg = <0x03200800 0x1000>,
+			      <0x03202000 0x2000>,
+			      <0x03204000 0x100>;
+			reg-names = "ccu",
+				    "dxe",
+				    "pmu";
+			pinctrl-0 = <&riva_wlan_default_state>, <&riva_bt_default_state>;
+			pinctrl-1 = <&riva_wlan_sleep_state>, <&riva_bt_sleep_state>;
+			pinctrl-names = "default", "sleep";
+			interrupts-extended = <&intc GIC_SPI 199 IRQ_TYPE_EDGE_RISING>,
+					      <&wcnss_smsm 6 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog", "fatal";
+			memory-region = <&wcnss_mem>;
+
+			status = "disabled";
+
+			iris {
+				compatible = "qcom,wcn3660";
+				clocks = <&cxo_board>;
+				clock-names = "xo";
+			};
+
+			smd-edge {
+				interrupts = <GIC_SPI 198 IRQ_TYPE_EDGE_RISING>;
+				label = "riva";
+				qcom,ipc = <&l2cc 8 25>;
+				qcom,smd-edge = <6>;
+
+				wcnss {
+					compatible = "qcom,wcnss";
+					qcom,smd-channels = "WCNSS_CTRL";
+					qcom,mmio = <&riva>;
+
+					bluetooth {
+						compatible = "qcom,wcnss-bt";
+					};
+
+					wifi {
+						compatible = "qcom,wcnss-wlan";
+						interrupts = <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>,
+							     <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>;
+						interrupt-names = "tx",
+								  "rx";
+						qcom,smem-states = <&apps_smsm 10>,
+								   <&apps_smsm 9>;
+						qcom,smem-state-names = "tx-enable",
+									"tx-rings-empty";
+					};
+				};
+			};
+		};
+
 		clock-controller@4000000 {
 			compatible = "qcom,mmcc-msm8960";
 			reg = <0x4000000 0x1000>;
@@ -414,6 +570,11 @@
 				      "hdmipll";
 		};
 
+		sps_sic_non_secure: interrupt-controller@12100000 {
+			compatible = "qcom,msm8960-sps-sic", "syscon";
+			reg = <0x12100000 0x10000>;
+		};
+
 		sdcc3: mmc@12180000 {
 			compatible = "arm,pl18x", "arm,primecell";
 			reg = <0x12180000 0x2000>;
@@ -507,8 +668,12 @@
 			reg = <0x12500000 0x200>,
 			      <0x12500200 0x200>;
 			interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&gcc USB_HS1_XCVR_CLK>, <&gcc USB_HS1_H_CLK>;
-			clock-names = "core", "iface";
+			clocks = <&rpmcc RPM_DAYTONA_FABRIC_CLK>,
+				 <&gcc USB_HS1_H_CLK>,
+				 <&gcc USB_HS1_XCVR_CLK>;
+			clock-names = "iface",
+				      "core",
+				      "fs";
 			assigned-clocks = <&gcc USB_HS1_XCVR_CLK>;
 			assigned-clock-rates = <60000000>;
 			resets = <&gcc USB_HS1_RESET>;
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
index 06c0af0aa31766..f300de74c47d46 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
@@ -10,6 +10,7 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/reset/qcom,gcc-msm8974.h>
+#include <dt-bindings/reset/qcom,mmcc-msm8974.h>
 
 / {
 	#address-cells = <1>;
@@ -1935,6 +1936,8 @@
 			interrupt-controller;
 			#interrupt-cells = <1>;
 
+			resets = <&mmcc MDSS_RESET>;
+
 			status = "disabled";
 
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts b/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts
index d23b8ec08af43a..0443a86899a9c9 100644
--- a/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts
+++ b/arch/arm/boot/dts/qcom/qcom-sdx55-t55.dts
@@ -249,7 +249,7 @@
 
 &pcie_rc {
 	perst-gpios = <&tlmm 57 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 53 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 53 GPIO_ACTIVE_LOW>;
 
 	pinctrl-0 = <&pcie_default>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/renesas/r7s72100-gr-peach.dts b/arch/arm/boot/dts/renesas/r7s72100-gr-peach.dts
index 23ddec21768574..2477db9e5aacac 100644
--- a/arch/arm/boot/dts/renesas/r7s72100-gr-peach.dts
+++ b/arch/arm/boot/dts/renesas/r7s72100-gr-peach.dts
@@ -131,5 +131,7 @@
 
 		reset-gpios = <&port4 2 GPIO_ACTIVE_LOW>;
 		reset-delay-us = <5>;
+		reset-assert-us = <25000>;
+		reset-deassert-us = <300>;
 	};
 };
diff --git a/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts b/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts
index 1d56bdef545398..eb65a54d0e5111 100644
--- a/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts
+++ b/arch/arm/boot/dts/renesas/r8a7740-armadillo800eva.dts
@@ -197,6 +197,8 @@
 			     "ethernet-phy-ieee802.3-c22";
 		reg = <0>;
 		reset-gpios = <&pfc 18 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <25000>;
+		reset-deassert-us = <300>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/renesas/r8a7743-sk-rzg1m.dts b/arch/arm/boot/dts/renesas/r8a7743-sk-rzg1m.dts
index 60217797e5345b..dc1b4ca9446225 100644
--- a/arch/arm/boot/dts/renesas/r8a7743-sk-rzg1m.dts
+++ b/arch/arm/boot/dts/renesas/r8a7743-sk-rzg1m.dts
@@ -75,5 +75,7 @@
 		interrupts-extended = <&irqc 0 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio5 22 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
diff --git a/arch/arm/boot/dts/renesas/r8a7743.dtsi b/arch/arm/boot/dts/renesas/r8a7743.dtsi
index c697942387e1d5..4db849b5bc6b91 100644
--- a/arch/arm/boot/dts/renesas/r8a7743.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7743.dtsi
@@ -456,7 +456,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -464,7 +464,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
@@ -477,7 +477,7 @@
 			};
 		};
 
-		icram2:	sram@e6300000 {
+		icram2: sram@e6300000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe6300000 0 0x40000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7744.dtsi b/arch/arm/boot/dts/renesas/r8a7744.dtsi
index fed46345807cb0..1a85bc0dc5f399 100644
--- a/arch/arm/boot/dts/renesas/r8a7744.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7744.dtsi
@@ -456,7 +456,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -464,7 +464,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
@@ -477,7 +477,7 @@
 			};
 		};
 
-		icram2:	sram@e6300000 {
+		icram2: sram@e6300000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe6300000 0 0x40000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7745-sk-rzg1e.dts b/arch/arm/boot/dts/renesas/r8a7745-sk-rzg1e.dts
index 42e82f0697553c..26ee1eec521be5 100644
--- a/arch/arm/boot/dts/renesas/r8a7745-sk-rzg1e.dts
+++ b/arch/arm/boot/dts/renesas/r8a7745-sk-rzg1e.dts
@@ -70,5 +70,7 @@
 		interrupts-extended = <&irqc 8 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio1 24 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
diff --git a/arch/arm/boot/dts/renesas/r8a7745.dtsi b/arch/arm/boot/dts/renesas/r8a7745.dtsi
index 5424a73562ddba..78ff5a9f8f3860 100644
--- a/arch/arm/boot/dts/renesas/r8a7745.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7745.dtsi
@@ -420,7 +420,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -428,7 +428,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
@@ -441,7 +441,7 @@
 			};
 		};
 
-		icram2:	sram@e6300000 {
+		icram2: sram@e6300000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe6300000 0 0x40000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a77470.dtsi b/arch/arm/boot/dts/renesas/r8a77470.dtsi
index c61790e7667f58..02df768bd2bc3e 100644
--- a/arch/arm/boot/dts/renesas/r8a77470.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a77470.dtsi
@@ -285,7 +285,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -293,7 +293,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
@@ -306,7 +306,7 @@
 			};
 		};
 
-		icram2:	sram@e6300000 {
+		icram2: sram@e6300000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe6300000 0 0x20000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7790-lager.dts b/arch/arm/boot/dts/renesas/r8a7790-lager.dts
index 8e766550167557..f7345d19337bb5 100644
--- a/arch/arm/boot/dts/renesas/r8a7790-lager.dts
+++ b/arch/arm/boot/dts/renesas/r8a7790-lager.dts
@@ -302,13 +302,13 @@
 	};
 
 	/*
-	 * IIC0/I2C0 is routed to EXIO connector A, pins 114 (SCL) + 116 (SDA) only.
+	 * IIC0/I2C0 is routed to EXIO connector C, pins 114 (SCL) + 116 (SDA) only.
 	 * We use the I2C demuxer, so the desired IP core can be selected at runtime
-	 * depending on the use case (e.g. DMA with IIC0 or slave support with I2C0).
-	 * Note: For testing the I2C slave feature, it is convenient to connect this
+	 * depending on the use case (e.g. DMA with IIC0 or target support with I2C0).
+	 * Note: For testing the I2C target feature, it is convenient to connect this
 	 * bus with IIC3 on pins 110 (SCL) + 112 (SDA), select I2C0 at runtime, and
-	 * instantiate the slave device at runtime according to the documentation.
-	 * You can then communicate with the slave via IIC3.
+	 * instantiate the target device at runtime according to the documentation.
+	 * You can then communicate with the target via IIC3.
 	 *
 	 * IIC0/I2C0 does not appear to support fallback to GPIO.
 	 */
@@ -690,6 +690,8 @@
 		interrupts-extended = <&irqc0 0 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio5 31 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
@@ -813,46 +815,46 @@
 	cpu0-supply = <&vdd_dvfs>;
 };
 
-&i2c0	{
+&i2c0 {
 	pinctrl-0 = <&i2c0_pins>;
 	pinctrl-names = "i2c-exio0";
 };
 
-&iic0	{
+&iic0 {
 	pinctrl-0 = <&iic0_pins>;
 	pinctrl-names = "i2c-exio0";
 };
 
-&i2c1	{
+&i2c1 {
 	pinctrl-0 = <&i2c1_pins>;
 	pinctrl-names = "i2c-exio1";
 };
 
-&iic1	{
+&iic1 {
 	pinctrl-0 = <&iic1_pins>;
 	pinctrl-names = "i2c-exio1";
 };
 
-&i2c2	{
+&i2c2 {
 	pinctrl-0 = <&i2c2_pins>;
 	pinctrl-names = "i2c-hdmi";
 
 	clock-frequency = <100000>;
 };
 
-&iic2	{
+&iic2 {
 	pinctrl-0 = <&iic2_pins>;
 	pinctrl-names = "i2c-hdmi";
 
 	clock-frequency = <100000>;
 };
 
-&i2c3	{
+&i2c3 {
 	pinctrl-0 = <&i2c3_pins>;
 	pinctrl-names = "i2c-pwr";
 };
 
-&iic3	{
+&iic3 {
 	pinctrl-0 = <&iic3_pins>;
 	pinctrl-names = "i2c-pwr";
 };
diff --git a/arch/arm/boot/dts/renesas/r8a7790-stout.dts b/arch/arm/boot/dts/renesas/r8a7790-stout.dts
index 8ba9d85f103896..95adece7681a4b 100644
--- a/arch/arm/boot/dts/renesas/r8a7790-stout.dts
+++ b/arch/arm/boot/dts/renesas/r8a7790-stout.dts
@@ -213,6 +213,8 @@
 		interrupts-extended = <&irqc0 1 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio3 31 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
@@ -289,7 +291,7 @@
 	cpu0-supply = <&vdd_dvfs>;
 };
 
-&iic2	{
+&iic2 {
 	status = "okay";
 	pinctrl-0 = <&iic2_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/renesas/r8a7790.dtsi b/arch/arm/boot/dts/renesas/r8a7790.dtsi
index 12cce9bdc44991..358c8061290ac7 100644
--- a/arch/arm/boot/dts/renesas/r8a7790.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7790.dtsi
@@ -566,7 +566,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -574,7 +574,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts b/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts
index 48db62e0ff8748..54148a5ddff7b9 100644
--- a/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts
+++ b/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts
@@ -681,6 +681,8 @@
 		interrupts-extended = <&irqc0 0 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio5 22 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/renesas/r8a7791-porter.dts b/arch/arm/boot/dts/renesas/r8a7791-porter.dts
index 811e263452acd9..05989da3be0469 100644
--- a/arch/arm/boot/dts/renesas/r8a7791-porter.dts
+++ b/arch/arm/boot/dts/renesas/r8a7791-porter.dts
@@ -331,6 +331,8 @@
 		interrupts-extended = <&irqc0 0 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio5 22 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/renesas/r8a7791.dtsi b/arch/arm/boot/dts/renesas/r8a7791.dtsi
index 35313e8da42650..3a447a18d03882 100644
--- a/arch/arm/boot/dts/renesas/r8a7791.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7791.dtsi
@@ -493,7 +493,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -501,7 +501,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7792.dtsi b/arch/arm/boot/dts/renesas/r8a7792.dtsi
index fbdbcff1cbed41..a63a28dd2bc784 100644
--- a/arch/arm/boot/dts/renesas/r8a7792.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7792.dtsi
@@ -415,7 +415,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -423,7 +423,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7793-gose.dts b/arch/arm/boot/dts/renesas/r8a7793-gose.dts
index 69d9c674bb0329..9687c1eebfd4d4 100644
--- a/arch/arm/boot/dts/renesas/r8a7793-gose.dts
+++ b/arch/arm/boot/dts/renesas/r8a7793-gose.dts
@@ -621,6 +621,8 @@
 		interrupts-extended = <&irqc0 0 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio5 22 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/renesas/r8a7793.dtsi b/arch/arm/boot/dts/renesas/r8a7793.dtsi
index 1ad50070a1a732..4cb3bf305ab556 100644
--- a/arch/arm/boot/dts/renesas/r8a7793.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7793.dtsi
@@ -468,7 +468,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -476,7 +476,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7794-alt.dts b/arch/arm/boot/dts/renesas/r8a7794-alt.dts
index 5d6d0d8cc4dd8f..92bc4ec4f854fc 100644
--- a/arch/arm/boot/dts/renesas/r8a7794-alt.dts
+++ b/arch/arm/boot/dts/renesas/r8a7794-alt.dts
@@ -383,6 +383,8 @@
 		interrupts-extended = <&irqc0 8 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio1 24 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/renesas/r8a7794-silk.dts b/arch/arm/boot/dts/renesas/r8a7794-silk.dts
index af474b1d9676d7..72829ce0524087 100644
--- a/arch/arm/boot/dts/renesas/r8a7794-silk.dts
+++ b/arch/arm/boot/dts/renesas/r8a7794-silk.dts
@@ -417,6 +417,8 @@
 		interrupts-extended = <&irqc0 8 IRQ_TYPE_LEVEL_LOW>;
 		micrel,led-mode = <1>;
 		reset-gpios = <&gpio1 24 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/renesas/r8a7794.dtsi b/arch/arm/boot/dts/renesas/r8a7794.dtsi
index 7669a67377c989..732e99f830c7ce 100644
--- a/arch/arm/boot/dts/renesas/r8a7794.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7794.dtsi
@@ -413,7 +413,7 @@
 			status = "disabled";
 		};
 
-		icram0:	sram@e63a0000 {
+		icram0: sram@e63a0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63a0000 0 0x12000>;
 			#address-cells = <1>;
@@ -421,7 +421,7 @@
 			ranges = <0 0 0xe63a0000 0x12000>;
 		};
 
-		icram1:	sram@e63c0000 {
+		icram1: sram@e63c0000 {
 			compatible = "mmio-sram";
 			reg = <0 0xe63c0000 0 0x1000>;
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts
index 5626d7fd6c3ee8..49f2def1bb128a 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts
+++ b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-db.dts
@@ -194,7 +194,6 @@
 &gpio2 {
 	pinctrl-0 = <&pins_gpio2>;
 	pinctrl-names = "default";
-	status = "okay";
 };
 
 &i2c2 {
diff --git a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts
index ead379988fb1c6..eb72955047155c 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts
+++ b/arch/arm/boot/dts/renesas/r9a06g032-rzn1d400-eb.dts
@@ -53,8 +53,9 @@
 	};
 };
 
-&gpio2 {
-	status = "okay";
+&gpio2b {
+	pinctrl-0 = <&pins_cn12>;
+	pinctrl-names = "default";
 };
 
 &i2c2 {
@@ -85,6 +86,15 @@
 };
 
 &pinctrl {
+	pins_cn12: pins-cn12 {
+		pinmux = <RZN1_PINMUX(164, RZN1_FUNC_GPIO)>,
+			 <RZN1_PINMUX(165, RZN1_FUNC_GPIO)>,
+			 <RZN1_PINMUX(166, RZN1_FUNC_GPIO)>,
+			 <RZN1_PINMUX(167, RZN1_FUNC_GPIO)>,
+			 <RZN1_PINMUX(168, RZN1_FUNC_GPIO)>,
+			 <RZN1_PINMUX(169, RZN1_FUNC_GPIO)>;
+	};
+
 	pins_eth0: pins-eth0 {
 		pinmux = <RZN1_PINMUX(0, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
 			 <RZN1_PINMUX(1, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
diff --git a/arch/arm/boot/dts/renesas/r9a06g032.dtsi b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
index 19c9bce0a26d69..8645bab7ee2392 100644
--- a/arch/arm/boot/dts/renesas/r9a06g032.dtsi
+++ b/arch/arm/boot/dts/renesas/r9a06g032.dtsi
@@ -145,7 +145,6 @@
 		sysctrl: system-controller@4000c000 {
 			compatible = "renesas,r9a06g032-sysctrl";
 			reg = <0x4000c000 0x1000>;
-			status = "okay";
 			#clock-cells = <1>;
 			#power-domain-cells = <0>;
 
@@ -352,7 +351,6 @@
 			reg = <0x40067000 0x1000>, <0x51000000 0x480>;
 			clocks = <&sysctrl R9A06G032_HCLK_PINCONFIG>;
 			clock-names = "bus";
-			status = "okay";
 		};
 
 		sdio1: mmc@40100000 {
diff --git a/arch/arm/boot/dts/rockchip/rk3229-xms6.dts b/arch/arm/boot/dts/rockchip/rk3229-xms6.dts
index 28333449c43aa0..6fd3e61c89af39 100644
--- a/arch/arm/boot/dts/rockchip/rk3229-xms6.dts
+++ b/arch/arm/boot/dts/rockchip/rk3229-xms6.dts
@@ -48,7 +48,7 @@
 	sdio_pwrseq: sdio-pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		reset-gpios = <&gpio2 26 GPIO_ACTIVE_LOW>,
-		              <&gpio2 29 GPIO_ACTIVE_LOW>;
+			      <&gpio2 29 GPIO_ACTIVE_LOW>;
 	};
 
 	vcc_host: regulator-vcc-host {
@@ -163,7 +163,7 @@
 
 		phy: ethernet-phy@0 {
 			compatible = "ethernet-phy-id1234.d400",
-			             "ethernet-phy-ieee802.3-c22";
+				     "ethernet-phy-ieee802.3-c22";
 			reg = <0>;
 			clocks = <&cru SCLK_MAC_PHY>;
 			phy-is-integrated;
diff --git a/arch/arm/boot/dts/samsung/exynos4210-i9100.dts b/arch/arm/boot/dts/samsung/exynos4210-i9100.dts
index 8a635bee59fa9b..b9a31714381c67 100644
--- a/arch/arm/boot/dts/samsung/exynos4210-i9100.dts
+++ b/arch/arm/boot/dts/samsung/exynos4210-i9100.dts
@@ -169,11 +169,14 @@
 			vdda-supply = <&cam_io_en_reg>;
 			vddreg-supply = <&vt_core_15v_reg>;
 			vddio-supply = <&vtcam_reg>;
+
 			clocks = <&camera 0>;
 			clock-names = "mclk";
+			assigned-clocks = <&camera 0>;
+			assigned-clock-rates = <24000000>;
+
 			stbyn-gpios = <&gpl2 0 GPIO_ACTIVE_LOW>;
 			rstn-gpios = <&gpl2 1 GPIO_ACTIVE_LOW>;
-			clock-frequency = <24000000>;
 
 			port {
 				s5k5bafx_ep: endpoint {
@@ -877,9 +880,13 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&bt_en>, <&bt_res>, <&uart0_data>, <&uart0_fctl>;
 
+	uart-has-rtscts;
+
 	bluetooth {
 		compatible = "brcm,bcm4330-bt";
 
+		max-speed = <3000000>;
+
 		shutdown-gpios = <&gpl0 4 GPIO_ACTIVE_HIGH>;
 		reset-gpios = <&gpl1 0 GPIO_ACTIVE_LOW>;
 		device-wakeup-gpios = <&gpx3 1 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/samsung/exynos4210-trats.dts b/arch/arm/boot/dts/samsung/exynos4210-trats.dts
index 6bd902cb8f4ad2..20fe148387fc23 100644
--- a/arch/arm/boot/dts/samsung/exynos4210-trats.dts
+++ b/arch/arm/boot/dts/samsung/exynos4210-trats.dts
@@ -541,8 +541,13 @@
 	pinctrl-0 = <&uart0_data &uart0_fctl>;
 	pinctrl-names = "default";
 
+	uart-has-rtscts;
+
 	bluetooth {
 		compatible = "brcm,bcm4330-bt";
+
+		max-speed = <3000000>;
+
 		pinctrl-0 = <&bt_shutdown &bt_device_wakeup &bt_host_wakeup>;
 		pinctrl-names = "default";
 		shutdown-gpios = <&gpl1 0 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts b/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts
index 91490693432b68..744039068e8f31 100644
--- a/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts
+++ b/arch/arm/boot/dts/samsung/exynos4210-universal_c210.dts
@@ -634,8 +634,13 @@
 	pinctrl-0 = <&uart0_data &uart0_fctl>;
 	pinctrl-names = "default";
 
+	uart-has-rtscts;
+
 	bluetooth {
 		compatible = "brcm,bcm4330-bt";
+
+		max-speed = <3000000>;
+
 		pinctrl-0 = <&bt_shutdown &bt_device_wakeup &bt_host_wakeup>;
 		pinctrl-names = "default";
 		shutdown-gpios = <&gpe1 4 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi b/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi
index 48245b1665a69c..8b6c2e2baed458 100644
--- a/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi
+++ b/arch/arm/boot/dts/samsung/exynos4412-midas.dtsi
@@ -638,10 +638,13 @@
 		svdda-supply = <&cam_io_reg>;
 		svddio-supply = <&ldo19_reg>;
 		afvdd-supply = <&ldo19_reg>;
-		clock-frequency = <24000000>;
+
 		/* CAM_B_CLKOUT */
 		clocks = <&camera 1>;
 		clock-names = "extclk";
+		assigned-clocks = <&camera 1>;
+		assigned-clock-rates = <24000000>;
+
 		gpios = <&gpm1 6 GPIO_ACTIVE_LOW>;
 
 		port {
diff --git a/arch/arm/boot/dts/st/stm32mp131.dtsi b/arch/arm/boot/dts/st/stm32mp131.dtsi
index 83ae59b73dd099..ae45c2023afeb4 100644
--- a/arch/arm/boot/dts/st/stm32mp131.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp131.dtsi
@@ -139,6 +139,30 @@
 		interrupt-parent = <&intc>;
 		ranges;
 
+		sram1: sram@30000000 {
+			compatible = "mmio-sram";
+			reg = <0x30000000 0x4000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x30000000 0x4000>;
+		};
+
+		sram2: sram@30004000 {
+			compatible = "mmio-sram";
+			reg = <0x30004000 0x2000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x30004000 0x2000>;
+		};
+
+		sram3: sram@30006000 {
+			compatible = "mmio-sram";
+			reg = <0x30006000 0x2000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x30006000 0x2000>;
+		};
+
 		timers2: timer@40000000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1104,7 +1128,7 @@
 		};
 
 		usbh_ohci: usb@5800c000 {
-			compatible = "generic-ohci";
+			compatible = "st,stm32mp13-ohci", "generic-ohci";
 			reg = <0x5800c000 0x1000>;
 			clocks = <&usbphyc>, <&rcc USBH>;
 			resets = <&rcc USBH_R>;
@@ -1113,7 +1137,7 @@
 		};
 
 		usbh_ehci: usb@5800d000 {
-			compatible = "generic-ehci";
+			compatible = "st,stm32mp13-ehci", "generic-ehci";
 			reg = <0x5800d000 0x1000>;
 			clocks = <&usbphyc>, <&rcc USBH>;
 			resets = <&rcc USBH_R>;
diff --git a/arch/arm/boot/dts/st/stm32mp135.dtsi b/arch/arm/boot/dts/st/stm32mp135.dtsi
index 834a4d545fe448..f573335edf6b44 100644
--- a/arch/arm/boot/dts/st/stm32mp135.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp135.dtsi
@@ -6,29 +6,29 @@
 
 #include "stm32mp133.dtsi"
 
-/ {
-	soc {
-		dcmipp: dcmipp@5a000000 {
-			compatible = "st,stm32mp13-dcmipp";
-			reg = <0x5a000000 0x400>;
-			interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
-			resets = <&rcc DCMIPP_R>;
-			clocks = <&rcc DCMIPP_K>;
-			status = "disabled";
+&etzpc {
+	dcmipp: dcmipp@5a000000 {
+		compatible = "st,stm32mp13-dcmipp";
+		reg = <0x5a000000 0x400>;
+		interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
+		resets = <&rcc DCMIPP_R>;
+		clocks = <&rcc DCMIPP_K>;
+		access-controllers = <&etzpc 4>;
+		status = "disabled";
 
-			port {
-			};
+		port {
 		};
+	};
 
-		ltdc: display-controller@5a001000 {
-			compatible = "st,stm32-ltdc";
-			reg = <0x5a001000 0x400>;
-			interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&rcc LTDC_PX>;
-			clock-names = "lcd";
-			resets = <&scmi_reset RST_SCMI_LTDC>;
-			status = "disabled";
-		};
+	ltdc: display-controller@5a001000 {
+		compatible = "st,stm32-ltdc";
+		reg = <0x5a001000 0x400>;
+		interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rcc LTDC_PX>;
+		clock-names = "lcd";
+		resets = <&scmi_reset RST_SCMI_LTDC>;
+		access-controllers = <&etzpc 3>;
+		status = "disabled";
 	};
 };
diff --git a/arch/arm/boot/dts/st/stm32mp135f-dk.dts b/arch/arm/boot/dts/st/stm32mp135f-dk.dts
index 1d6435c59096a0..2d6c5af7e48164 100644
--- a/arch/arm/boot/dts/st/stm32mp135f-dk.dts
+++ b/arch/arm/boot/dts/st/stm32mp135f-dk.dts
@@ -60,17 +60,6 @@
 		};
 	};
 
-	gpio-keys {
-		compatible = "gpio-keys";
-
-		button-user {
-			/* GPIO on PA13 "User button 2 (B2)" */
-			label = "User-PA13";
-			linux,code = <BTN_1>;
-			gpios = <&gpioa 13 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
-		};
-	};
-
 	leds {
 		compatible = "gpio-leds";
 
@@ -190,11 +179,11 @@
 	status = "okay";
 };
 
-&cs_cti_trace {
+&cs_cti_cpu0 {
 	status = "okay";
 };
 
-&cs_cti_cpu0 {
+&cs_cti_trace {
 	status = "okay";
 };
 
@@ -479,9 +468,21 @@
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&spi5_pins_a>;
 	pinctrl-1 = <&spi5_sleep_pins_a>;
+	dmas = <&dmamux1 85 0x400 0x01>,
+	       <&dmamux1 86 0x400 0x01>,
+	       <&mdma 0 0x3 0x1200000a 0 0>;
+	dma-names = "rx", "tx", "rxm2m";
+	sram = <&spi5_dma_pool>;
 	status = "disabled";
 };
 
+&sram2 {
+	spi5_dma_pool: dma-sram@1000 {
+		reg = <0x1000 0x1000>;
+		pool;
+	};
+};
+
 &timers3 {
 	/delete-property/dmas;
 	/delete-property/dma-names;
@@ -628,6 +629,7 @@
 		compatible = "usb424,2514";
 		reg = <1>;
 		vdd-supply = <&scmi_v3v3_sw>;
+		vdda-supply = <&scmi_v3v3_sw>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi b/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi
index aaa91b634c12ff..4480bc52927027 100644
--- a/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15-pinctrl.dtsi
@@ -2873,6 +2873,31 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	spi4_pins_b: spi4-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('E', 12, AF5)>, /* SPI4_SCK */
+				 <STM32_PINMUX('E', 14, AF5)>; /* SPI4_MOSI */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <1>;
+		};
+
+		pins2 {
+			pinmux = <STM32_PINMUX('E', 13, AF5)>; /* SPI4_MISO */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	spi4_sleep_pins_b: spi4-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('E', 12, ANALOG)>, /* SPI4_SCK */
+				 <STM32_PINMUX('E', 13, ANALOG)>, /* SPI4_MISO */
+				 <STM32_PINMUX('E', 14, ANALOG)>; /* SPI4_MOSI */
+		};
+	};
+
 	/omit-if-no-ref/
 	spi5_pins_a: spi5-0 {
 		pins1 {
@@ -2889,6 +2914,15 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	spi5_sleep_pins_a: spi5-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('F', 7, ANALOG)>, /* SPI5_SCK */
+				 <STM32_PINMUX('F', 8, ANALOG)>, /* SPI5_MISO */
+				 <STM32_PINMUX('F', 9, ANALOG)>; /* SPI5_MOSI */
+		};
+	};
+
 	/omit-if-no-ref/
 	stusb1600_pins_a: stusb1600-0 {
 		pins {
diff --git a/arch/arm/boot/dts/st/stm32mp151.dtsi b/arch/arm/boot/dts/st/stm32mp151.dtsi
index 84f68e8563d853..ae31883732388a 100644
--- a/arch/arm/boot/dts/st/stm32mp151.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp151.dtsi
@@ -504,7 +504,7 @@
 		};
 
 		usbh_ohci: usb@5800c000 {
-			compatible = "generic-ohci";
+			compatible = "st,stm32mp15-ohci", "generic-ohci";
 			reg = <0x5800c000 0x1000>;
 			clocks = <&usbphyc>, <&rcc USBH>;
 			resets = <&rcc USBH_R>;
@@ -515,7 +515,7 @@
 		};
 
 		usbh_ehci: usb@5800d000 {
-			compatible = "generic-ehci";
+			compatible = "st,stm32mp15-ehci", "generic-ehci";
 			reg = <0x5800d000 0x1000>;
 			clocks = <&usbphyc>, <&rcc USBH>;
 			resets = <&rcc USBH_R>;
@@ -943,8 +943,8 @@
 				interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc SPI2_K>;
 				resets = <&rcc SPI2_R>;
-				dmas = <&dmamux1 39 0x400 0x05>,
-				       <&dmamux1 40 0x400 0x05>;
+				dmas = <&dmamux1 39 0x400 0x01>,
+				       <&dmamux1 40 0x400 0x01>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 27>;
 				status = "disabled";
@@ -970,8 +970,8 @@
 				interrupts = <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc SPI3_K>;
 				resets = <&rcc SPI3_R>;
-				dmas = <&dmamux1 61 0x400 0x05>,
-				       <&dmamux1 62 0x400 0x05>;
+				dmas = <&dmamux1 61 0x400 0x01>,
+				       <&dmamux1 62 0x400 0x01>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 28>;
 				status = "disabled";
@@ -1301,8 +1301,8 @@
 				interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc SPI1_K>;
 				resets = <&rcc SPI1_R>;
-				dmas = <&dmamux1 37 0x400 0x05>,
-				       <&dmamux1 38 0x400 0x05>;
+				dmas = <&dmamux1 37 0x400 0x01>,
+				       <&dmamux1 38 0x400 0x01>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 52>;
 				status = "disabled";
@@ -1316,8 +1316,8 @@
 				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc SPI4_K>;
 				resets = <&rcc SPI4_R>;
-				dmas = <&dmamux1 83 0x400 0x05>,
-				       <&dmamux1 84 0x400 0x05>;
+				dmas = <&dmamux1 83 0x400 0x01>,
+				       <&dmamux1 84 0x400 0x01>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 53>;
 				status = "disabled";
@@ -1432,8 +1432,8 @@
 				interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&rcc SPI5_K>;
 				resets = <&rcc SPI5_R>;
-				dmas = <&dmamux1 85 0x400 0x05>,
-				       <&dmamux1 86 0x400 0x05>;
+				dmas = <&dmamux1 85 0x400 0x01>,
+				       <&dmamux1 86 0x400 0x01>;
 				dma-names = "rx", "tx";
 				access-controllers = <&etzpc 57>;
 				status = "disabled";
diff --git a/arch/arm/boot/dts/st/stm32mp153c-lxa-fairytux2.dtsi b/arch/arm/boot/dts/st/stm32mp153c-lxa-fairytux2.dtsi
index 7d3a6a3b5d09ea..d30b626a18c20e 100644
--- a/arch/arm/boot/dts/st/stm32mp153c-lxa-fairytux2.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp153c-lxa-fairytux2.dtsi
@@ -28,7 +28,7 @@
 	};
 
 	chosen {
-		stdout-path = &uart4;
+		stdout-path = "serial0:115200n8";
 	};
 
 	backlight: backlight {
diff --git a/arch/arm/boot/dts/st/stm32mp157a-dk1-scmi.dts b/arch/arm/boot/dts/st/stm32mp157a-dk1-scmi.dts
index 847b360f02fccf..53e40e2f776b7e 100644
--- a/arch/arm/boot/dts/st/stm32mp157a-dk1-scmi.dts
+++ b/arch/arm/boot/dts/st/stm32mp157a-dk1-scmi.dts
@@ -51,10 +51,6 @@
 	clocks = <&rcc IWDG2>, <&scmi_clk CK_SCMI_LSI>;
 };
 
-&mdma1 {
-	resets = <&scmi_reset RST_SCMI_MDMA>;
-};
-
 &m4_rproc {
 	/delete-property/ st,syscfg-holdboot;
 	resets = <&scmi_reset RST_SCMI_MCU>,
@@ -62,6 +58,10 @@
 	reset-names = "mcu_rst", "hold_boot";
 };
 
+&mdma1 {
+	resets = <&scmi_reset RST_SCMI_MDMA>;
+};
+
 &optee {
 	interrupt-parent = <&intc>;
 	interrupts = <GIC_PPI 15 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
diff --git a/arch/arm/boot/dts/st/stm32mp157c-dk2-scmi.dts b/arch/arm/boot/dts/st/stm32mp157c-dk2-scmi.dts
index 43280289759d02..0790ed426ebc02 100644
--- a/arch/arm/boot/dts/st/stm32mp157c-dk2-scmi.dts
+++ b/arch/arm/boot/dts/st/stm32mp157c-dk2-scmi.dts
@@ -57,10 +57,6 @@
 	clocks = <&rcc IWDG2>, <&scmi_clk CK_SCMI_LSI>;
 };
 
-&mdma1 {
-	resets = <&scmi_reset RST_SCMI_MDMA>;
-};
-
 &m4_rproc {
 	/delete-property/ st,syscfg-holdboot;
 	resets = <&scmi_reset RST_SCMI_MCU>,
@@ -68,6 +64,10 @@
 	reset-names = "mcu_rst", "hold_boot";
 };
 
+&mdma1 {
+	resets = <&scmi_reset RST_SCMI_MDMA>;
+};
+
 &optee {
 	interrupt-parent = <&intc>;
 	interrupts = <GIC_PPI 15 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
diff --git a/arch/arm/boot/dts/st/stm32mp157c-ed1-scmi.dts b/arch/arm/boot/dts/st/stm32mp157c-ed1-scmi.dts
index 6f27d794d2702c..0a3894aff4aead 100644
--- a/arch/arm/boot/dts/st/stm32mp157c-ed1-scmi.dts
+++ b/arch/arm/boot/dts/st/stm32mp157c-ed1-scmi.dts
@@ -56,10 +56,6 @@
 	clocks = <&rcc IWDG2>, <&scmi_clk CK_SCMI_LSI>;
 };
 
-&mdma1 {
-	resets = <&scmi_reset RST_SCMI_MDMA>;
-};
-
 &m4_rproc {
 	/delete-property/ st,syscfg-holdboot;
 	resets = <&scmi_reset RST_SCMI_MCU>,
@@ -67,6 +63,10 @@
 	reset-names = "mcu_rst", "hold_boot";
 };
 
+&mdma1 {
+	resets = <&scmi_reset RST_SCMI_MDMA>;
+};
+
 &optee {
 	interrupt-parent = <&intc>;
 	interrupts = <GIC_PPI 15 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
diff --git a/arch/arm/boot/dts/st/stm32mp157c-ev1-scmi.dts b/arch/arm/boot/dts/st/stm32mp157c-ev1-scmi.dts
index 6ae391bffee53a..c2b6efb1cbb74e 100644
--- a/arch/arm/boot/dts/st/stm32mp157c-ev1-scmi.dts
+++ b/arch/arm/boot/dts/st/stm32mp157c-ev1-scmi.dts
@@ -61,10 +61,6 @@
 	clocks = <&scmi_clk CK_SCMI_HSE>, <&rcc FDCAN_K>;
 };
 
-&mdma1 {
-	resets = <&scmi_reset RST_SCMI_MDMA>;
-};
-
 &m4_rproc {
 	/delete-property/ st,syscfg-holdboot;
 	resets = <&scmi_reset RST_SCMI_MCU>,
@@ -72,6 +68,10 @@
 	reset-names = "mcu_rst", "hold_boot";
 };
 
+&mdma1 {
+	resets = <&scmi_reset RST_SCMI_MDMA>;
+};
+
 &optee {
 	interrupt-parent = <&intc>;
 	interrupts = <GIC_PPI 15 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
diff --git a/arch/arm/boot/dts/st/stm32mp157c-ev1.dts b/arch/arm/boot/dts/st/stm32mp157c-ev1.dts
index 0e65a1862eb533..c0e274eebf1364 100644
--- a/arch/arm/boot/dts/st/stm32mp157c-ev1.dts
+++ b/arch/arm/boot/dts/st/stm32mp157c-ev1.dts
@@ -81,15 +81,15 @@
 	status = "okay";
 };
 
-&cs_cti_trace {
+&cs_cti_cpu0 {
 	status = "okay";
 };
 
-&cs_cti_cpu0 {
+&cs_cti_cpu1 {
 	status = "okay";
 };
 
-&cs_cti_cpu1 {
+&cs_cti_trace {
 	status = "okay";
 };
 
@@ -439,6 +439,7 @@
 		compatible = "usb424,2514";
 		reg = <1>;
 		vdd-supply = <&v3v3>;
+		vdda-supply = <&v3v3>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/st/stm32mp157c-lxa-mc1.dts b/arch/arm/boot/dts/st/stm32mp157c-lxa-mc1.dts
index eada9cf257be9c..b3d8eb57aa24eb 100644
--- a/arch/arm/boot/dts/st/stm32mp157c-lxa-mc1.dts
+++ b/arch/arm/boot/dts/st/stm32mp157c-lxa-mc1.dts
@@ -33,7 +33,7 @@
 	};
 
 	chosen {
-		stdout-path = &uart4;
+		stdout-path = "serial0:115200n8";
 	};
 
 	led-controller-0 {
diff --git a/arch/arm/boot/dts/st/stm32mp157f-dk2-scmi.dtsi b/arch/arm/boot/dts/st/stm32mp157f-dk2-scmi.dtsi
index 89de85a2eff327..172347769e0251 100644
--- a/arch/arm/boot/dts/st/stm32mp157f-dk2-scmi.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp157f-dk2-scmi.dtsi
@@ -180,6 +180,7 @@
 &usbh_ehci {
 	hub@1 {
 		vdd-supply = <&scmi_v3v3>;
+		vdda-supply = <&scmi_v3v3>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/st/stm32mp15xc-lxa-tac.dtsi b/arch/arm/boot/dts/st/stm32mp15xc-lxa-tac.dtsi
index ab13f0c39892f8..ddb1657cd78575 100644
--- a/arch/arm/boot/dts/st/stm32mp15xc-lxa-tac.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15xc-lxa-tac.dtsi
@@ -33,7 +33,7 @@
 	};
 
 	chosen {
-		stdout-path = &uart4;
+		stdout-path = "serial0:115200n8";
 	};
 
 	led-controller-0 {
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-som.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-som.dtsi
index 4cc633683c6b7a..0a4263e6209d8f 100644
--- a/arch/arm/boot/dts/st/stm32mp15xx-dhcom-som.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dhcom-som.dtsi
@@ -505,9 +505,11 @@
 	 * - mandatory on SoMs without SD voltage translator
 	 */
 	pins1 {
+		/delete-property/ bias-disable;
 		bias-pull-up;
 	};
 	pins2 {
+		/delete-property/ bias-disable;
 		bias-pull-up;
 	};
 };
diff --git a/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi b/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi
index 599ea07bdb19ca..1a07ae09afb2a2 100644
--- a/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15xx-dkx.dtsi
@@ -155,15 +155,15 @@
 	status = "okay";
 };
 
-&cs_cti_trace {
+&cs_cti_cpu0 {
 	status = "okay";
 };
 
-&cs_cti_cpu0 {
+&cs_cti_cpu1 {
 	status = "okay";
 };
 
-&cs_cti_cpu1 {
+&cs_cti_trace {
 	status = "okay";
 };
 
@@ -623,6 +623,32 @@
 	status = "disabled";
 };
 
+&spi4 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi4_pins_b>;
+	pinctrl-1 = <&spi4_sleep_pins_b>;
+	dmas = <&dmamux1 83 0x400 0x01>,
+	       <&dmamux1 84 0x400 0x01>,
+	       <&mdma1 0 0x3 0x1200000a 0 0>;
+	dma-names = "rx", "tx", "rxm2m";
+	sram = <&spi4_dma_pool>;
+	status = "disabled";
+};
+
+&spi5 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi5_pins_a>;
+	pinctrl-1 = <&spi5_sleep_pins_a>;
+	status = "disabled";
+};
+
+&sram4 {
+	spi4_dma_pool: dma-sram@9000 {
+		reg = <0x9000 0x1000>;
+		pool;
+	};
+};
+
 &timers1 {
 	/* spare dmas for other usage */
 	/delete-property/dmas;
@@ -764,6 +790,7 @@
 		compatible = "usb424,2514";
 		reg = <1>;
 		vdd-supply = <&v3v3>;
+		vdda-supply = <&v3v3>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/ti/keystone/keystone-k2e.dtsi b/arch/arm/boot/dts/ti/keystone/keystone-k2e.dtsi
index 662aa33cba1148..083c94b62fc762 100644
--- a/arch/arm/boot/dts/ti/keystone/keystone-k2e.dtsi
+++ b/arch/arm/boot/dts/ti/keystone/keystone-k2e.dtsi
@@ -110,7 +110,7 @@
 		};
 
 		devctrl: device-state-control@2620000 {
-			dspgpio0: keystone_dsp_gpio@240 {
+			dspgpio0: gpio@240 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x240 0x4>;
 				gpio-controller;
diff --git a/arch/arm/boot/dts/ti/keystone/keystone-k2g.dtsi b/arch/arm/boot/dts/ti/keystone/keystone-k2g.dtsi
index dafe485dfe197f..ee03a8555dc75a 100644
--- a/arch/arm/boot/dts/ti/keystone/keystone-k2g.dtsi
+++ b/arch/arm/boot/dts/ti/keystone/keystone-k2g.dtsi
@@ -69,22 +69,16 @@
 		interrupts = <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>;
 	};
 
-	usbphy {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		compatible = "simple-bus";
-
-		usb0_phy: usb-phy@0 {
-			compatible = "usb-nop-xceiv";
-			reg = <0>;
-			status = "disabled";
-		};
+	usb0_phy: usb0-phy {
+		compatible = "usb-nop-xceiv";
+		#phy-cells = <0>;
+		status = "disabled";
+	};
 
-		usb1_phy: usb-phy@1 {
-			compatible = "usb-nop-xceiv";
-			reg = <1>;
-			status = "disabled";
-		};
+	usb1_phy: usb1-phy {
+		compatible = "usb-nop-xceiv";
+		#phy-cells = <0>;
+		status = "disabled";
 	};
 
 	soc0: soc@0 {
@@ -121,7 +115,7 @@
 			#size-cells = <1>;
 			ranges = <0x0 0x02620000 0x1000>;
 
-			kirq0: keystone_irq@2a0 {
+			kirq0: interrupt-controller@2a0 {
 				compatible = "ti,keystone-irq";
 				reg = <0x2a0 0x10>;
 				interrupts = <GIC_SPI 1 IRQ_TYPE_EDGE_RISING>;
@@ -130,7 +124,7 @@
 				ti,syscon-dev = <&devctrl 0x2a0>;
 			};
 
-			dspgpio0: keystone_dsp_gpio@240 {
+			dspgpio0: gpio@240 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x240 0x4>;
 				gpio-controller;
diff --git a/arch/arm/boot/dts/ti/keystone/keystone-k2hk.dtsi b/arch/arm/boot/dts/ti/keystone/keystone-k2hk.dtsi
index 4fdf4b30384f73..b7a9fbd6d59d63 100644
--- a/arch/arm/boot/dts/ti/keystone/keystone-k2hk.dtsi
+++ b/arch/arm/boot/dts/ti/keystone/keystone-k2hk.dtsi
@@ -88,7 +88,7 @@
 		};
 
 		devctrl: device-state-control@2620000 {
-			dspgpio0: keystone_dsp_gpio@240 {
+			dspgpio0: gpio@240 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x240 0x4>;
 				gpio-controller;
@@ -96,7 +96,7 @@
 				gpio,syscon-dev = <&devctrl 0x240>;
 			};
 
-			dspgpio1: keystone_dsp_gpio@244 {
+			dspgpio1: gpio@244 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x244 0x4>;
 				gpio-controller;
@@ -104,7 +104,7 @@
 				gpio,syscon-dev = <&devctrl 0x244>;
 			};
 
-			dspgpio2: keystone_dsp_gpio@248 {
+			dspgpio2: gpio@248 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x248 0x4>;
 				gpio-controller;
@@ -112,7 +112,7 @@
 				gpio,syscon-dev = <&devctrl 0x248>;
 			};
 
-			dspgpio3: keystone_dsp_gpio@24c {
+			dspgpio3: gpio@24c {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x24c 0x4>;
 				gpio-controller;
@@ -120,7 +120,7 @@
 				gpio,syscon-dev = <&devctrl 0x24c>;
 			};
 
-			dspgpio4: keystone_dsp_gpio@250 {
+			dspgpio4: gpio@250 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x250 0x4>;
 				gpio-controller;
@@ -128,7 +128,7 @@
 				gpio,syscon-dev = <&devctrl 0x250>;
 			};
 
-			dspgpio5: keystone_dsp_gpio@254 {
+			dspgpio5: gpio@254 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x254 0x4>;
 				gpio-controller;
@@ -136,7 +136,7 @@
 				gpio,syscon-dev = <&devctrl 0x254>;
 			};
 
-			dspgpio6: keystone_dsp_gpio@258 {
+			dspgpio6: gpio@258 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x258 0x4>;
 				gpio-controller;
@@ -144,7 +144,7 @@
 				gpio,syscon-dev = <&devctrl 0x258>;
 			};
 
-			dspgpio7: keystone_dsp_gpio@25c {
+			dspgpio7: gpio@25c {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x25c 0x4>;
 				gpio-controller;
diff --git a/arch/arm/boot/dts/ti/keystone/keystone-k2l.dtsi b/arch/arm/boot/dts/ti/keystone/keystone-k2l.dtsi
index 330b437b667f66..9830cfffeeab48 100644
--- a/arch/arm/boot/dts/ti/keystone/keystone-k2l.dtsi
+++ b/arch/arm/boot/dts/ti/keystone/keystone-k2l.dtsi
@@ -51,7 +51,7 @@
 			interrupts = <GIC_SPI 432 IRQ_TYPE_EDGE_RISING>;
 		};
 
-		uart3:	serial@2348800 {
+		uart3: serial@2348800 {
 			compatible = "ti,da830-uart", "ns16550a";
 			current-speed = <115200>;
 			reg-shift = <2>;
@@ -290,7 +290,7 @@
 		};
 
 		devctrl: device-state-control@2620000 {
-			dspgpio0: keystone_dsp_gpio@240 {
+			dspgpio0: gpio@240 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x240 0x4>;
 				gpio-controller;
@@ -298,7 +298,7 @@
 				gpio,syscon-dev = <&devctrl 0x240>;
 			};
 
-			dspgpio1: keystone_dsp_gpio@244 {
+			dspgpio1: gpio@244 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x244 0x4>;
 				gpio-controller;
@@ -306,7 +306,7 @@
 				gpio,syscon-dev = <&devctrl 0x244>;
 			};
 
-			dspgpio2: keystone_dsp_gpio@248 {
+			dspgpio2: gpio@248 {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x248 0x4>;
 				gpio-controller;
@@ -314,7 +314,7 @@
 				gpio,syscon-dev = <&devctrl 0x248>;
 			};
 
-			dspgpio3: keystone_dsp_gpio@24c {
+			dspgpio3: gpio@24c {
 				compatible = "ti,keystone-dsp-gpio";
 				reg = <0x24c 0x4>;
 				gpio-controller;
@@ -401,15 +401,15 @@
 };
 
 &spi0 {
-       ti,davinci-spi-num-cs = <5>;
+	num-cs = <5>;
 };
 
 &spi1 {
-       ti,davinci-spi-num-cs = <3>;
+	num-cs = <3>;
 };
 
 &spi2 {
-       ti,davinci-spi-num-cs = <5>;
+	num-cs = <5>;
        /* Pin muxed. Enabled and configured by Bootloader */
        status = "disabled";
 };
diff --git a/arch/arm/boot/dts/ti/keystone/keystone.dtsi b/arch/arm/boot/dts/ti/keystone/keystone.dtsi
index ff16428860a9bd..f803b9e2afd1c1 100644
--- a/arch/arm/boot/dts/ti/keystone/keystone.dtsi
+++ b/arch/arm/boot/dts/ti/keystone/keystone.dtsi
@@ -93,7 +93,7 @@
 			#size-cells = <1>;
 			ranges = <0x0 0x02620000 0x1000>;
 
-			kirq0: keystone_irq@2a0 {
+			kirq0: interrupt-controller@2a0 {
 				compatible = "ti,keystone-irq";
 				reg = <0x2a0 0x4>;
 				interrupts = <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>;
@@ -123,7 +123,7 @@
 			interrupts = <GIC_SPI 277 IRQ_TYPE_EDGE_RISING>;
 		};
 
-		uart1:	serial@2531000 {
+		uart1: serial@2531000 {
 			compatible = "ti,da830-uart", "ns16550a";
 			current-speed = <115200>;
 			reg-shift = <2>;
diff --git a/arch/arm/boot/dts/ti/omap/am335x-boneblack-hdmi.dtsi b/arch/arm/boot/dts/ti/omap/am335x-boneblack-hdmi.dtsi
index 109f08fd174d62..c52bc3324dea9c 100644
--- a/arch/arm/boot/dts/ti/omap/am335x-boneblack-hdmi.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am335x-boneblack-hdmi.dtsi
@@ -99,7 +99,7 @@
 	};
 };
 
-&mcasp0	{
+&mcasp0 {
 	#sound-dai-cells = <0>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&mcasp0_pins>;
diff --git a/arch/arm/boot/dts/ti/omap/am335x-bonegreen-eco.dts b/arch/arm/boot/dts/ti/omap/am335x-bonegreen-eco.dts
index d21118cdb6c2c0..f00abfdd2cbd44 100644
--- a/arch/arm/boot/dts/ti/omap/am335x-bonegreen-eco.dts
+++ b/arch/arm/boot/dts/ti/omap/am335x-bonegreen-eco.dts
@@ -63,7 +63,7 @@
 			buck1: buck1 {
 				regulator-name = "vdd_mpu";
 				regulator-min-microvolt = <925000>;
-				regulator-max-microvolt = <1298500>;
+				regulator-max-microvolt = <1351500>;
 				regulator-boot-on;
 				regulator-always-on;
 			};
diff --git a/arch/arm/boot/dts/ti/omap/am335x-shc.dts b/arch/arm/boot/dts/ti/omap/am335x-shc.dts
index 5974828226082f..2b946823928fba 100644
--- a/arch/arm/boot/dts/ti/omap/am335x-shc.dts
+++ b/arch/arm/boot/dts/ti/omap/am335x-shc.dts
@@ -241,10 +241,6 @@
 	status = "okay";
 };
 
-&rtc {
-	ti,no-init;
-};
-
 &sham {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi b/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi
index 30fcce33f4b7ce..9745e0d420ff7a 100644
--- a/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am437x-l4.dtsi
@@ -902,7 +902,7 @@
 			#size-cells = <1>;
 			ranges = <0x0 0x40000 0x1000>;
 
-			timer2: timer@0  {
+			timer2: timer@0 {
 				compatible = "ti,am4372-timer","ti,am335x-timer";
 				reg = <0x0 0x400>;
 				interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi b/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi
index f7331662560842..1579f284cb6ef9 100644
--- a/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am57-pruss.dtsi
@@ -61,6 +61,17 @@
 				};
 			};
 
+			pruss1_iep: iep@2e000 {
+				compatible = "ti,am5728-icss-iep";
+				reg = <0x2e000 0x31c>;
+				clocks = <&pruss1_iepclk_mux>;
+			};
+
+			pruss1_ecap: ecap@30000 {
+				compatible = "ti,pruss-ecap";
+				reg = <0x30000 0x60>;
+			};
+
 			pruss1_mii_rt: mii-rt@32000 {
 				compatible = "ti,pruss-mii", "syscon";
 				reg = <0x32000 0x58>;
diff --git a/arch/arm/boot/dts/ti/omap/am571x-idk.dts b/arch/arm/boot/dts/ti/omap/am571x-idk.dts
index 02653b440585d8..e74c154b1940e5 100644
--- a/arch/arm/boot/dts/ti/omap/am571x-idk.dts
+++ b/arch/arm/boot/dts/ti/omap/am571x-idk.dts
@@ -165,6 +165,53 @@
 			default-state = "off";
 		};
 	};
+
+	/* Dual mac ethernet application node on icss1 */
+	pruss1-eth {
+		compatible = "ti,am57-prueth";
+		ti,prus = <&pru1_0>, <&pru1_1>;
+		ti,pruss-gp-mux-sel = <0>,	/* GP, default */
+				<4>;	/* MII2, for PRUSS1_MII1 */
+		sram = <&ocmcram1>;
+		ti,mii-rt = <&pruss1_mii_rt>;
+		ti,iep = <&pruss1_iep>;
+		ti,ecap = <&pruss1_ecap>;
+		interrupts = <20 2 2>, <21 3 3>;
+		interrupt-names = "rx_hp", "rx_lp";
+		interrupt-parent = <&pruss1_intc>;
+
+		ethernet-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pruss1_emac0: ethernet-port@0 {
+				reg = <0>;
+				phy-handle = <&pruss1_eth0_phy>;
+				phy-mode = "mii";
+				interrupts = <20 2 2>, <26 6 6>, <23 6 6>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						"hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+
+			pruss1_emac1: ethernet-port@1 {
+				reg = <1>;
+				phy-handle = <&pruss1_eth1_phy>;
+				phy-mode = "mii";
+				interrupts = <21 3 3>, <27 9 7>, <24 9 7>;
+				interrupt-names = "rx", "emac_ptp_tx",
+						"hsr_ptp_tx";
+				/* Filled in by bootloader */
+				local-mac-address = [00 00 00 00 00 00];
+			};
+		};
+	};
+};
+
+&pruss1_iep {
+	interrupt-parent = <&pruss1_intc>;
+	interrupts = <7 7 8>;
+	interrupt-names = "iep_cap_cmp";
 };
 
 &extcon_usb2 {
@@ -210,7 +257,23 @@
 };
 
 &pruss1_mdio {
-	status = "disabled";
+	status = "okay";
+	reset-gpios = <&gpio5 8 GPIO_ACTIVE_LOW>;
+	reset-delay-us = <2>;   /* PHY datasheet states 1uS min */
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pruss1_eth0_phy: ethernet-phy@0 {
+		reg = <0>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <28 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	pruss1_eth1_phy: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
+	};
 };
 
 &pruss2_mdio {
diff --git a/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts b/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts
index 76bfb364777e56..46270ed87377ab 100644
--- a/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts
+++ b/arch/arm/boot/dts/ti/omap/am5729-beagleboneai.dts
@@ -386,7 +386,7 @@
 			};
 		};
 
-		tps659038_rtc: tps659038_rtc {
+		tps659038_rtc: rtc {
 			compatible = "ti,palmas-rtc";
 			interrupt-parent = <&tps659038>;
 			interrupts = <8 IRQ_TYPE_EDGE_FALLING>;
diff --git a/arch/arm/boot/dts/ti/omap/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/ti/omap/am57xx-beagle-x15-common.dtsi
index 87b61a98d5e9a7..e7c186941c230c 100644
--- a/arch/arm/boot/dts/ti/omap/am57xx-beagle-x15-common.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am57xx-beagle-x15-common.dtsi
@@ -360,7 +360,7 @@
 			};
 		};
 
-		tps659038_rtc: tps659038_rtc {
+		tps659038_rtc: rtc {
 			compatible = "ti,palmas-rtc";
 			interrupt-parent = <&tps659038>;
 			interrupts = <8 IRQ_TYPE_EDGE_FALLING>;
diff --git a/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi b/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi
index 5eccff3bb4b675..699141b9a86fdf 100644
--- a/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi
+++ b/arch/arm/boot/dts/ti/omap/am57xx-idk-common.dtsi
@@ -405,7 +405,7 @@
 			};
 		};
 
-		tps659038_rtc: tps659038_rtc {
+		tps659038_rtc: rtc {
 			compatible = "ti,palmas-rtc";
 			interrupt-parent = <&tps659038>;
 			interrupts = <8 IRQ_TYPE_EDGE_FALLING>;
@@ -491,7 +491,6 @@
 
 &rtc {
 	status = "okay";
-	ext-clk-src;
 };
 
 &mac_sw {
diff --git a/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi b/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
index a595745afe5958..51dda169c928b4 100644
--- a/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dra7-l4.dtsi
@@ -3556,7 +3556,7 @@
 			ranges = <0x0 0x38000 0x1000>;
 
 			rtc: rtc@0 {
-				compatible = "ti,am3352-rtc";
+				compatible = "ti,am3352-rtc", "ti,da830-rtc";
 				reg = <0x0 0x100>;
 				interrupts = <GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/ti/omap/dra7xx-clocks.dtsi b/arch/arm/boot/dts/ti/omap/dra7xx-clocks.dtsi
index 0de16ee262cf47..615a8fb4777615 100644
--- a/arch/arm/boot/dts/ti/omap/dra7xx-clocks.dtsi
+++ b/arch/arm/boot/dts/ti/omap/dra7xx-clocks.dtsi
@@ -1376,8 +1376,8 @@
 		clocks = <&apll_pcie_ck>;
 		#clock-cells = <0>;
 		reg = <0x021c>;
+		ti,dividers = <2>, <1>;
 		ti,bit-shift = <8>;
-		ti,max-div = <2>;
 	};
 
 	apll_pcie_clkvcoldo: clock-apll-pcie-clkvcoldo {
diff --git a/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi b/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi
index 28b4f12b82a733..edc84928e9b679 100644
--- a/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi
+++ b/arch/arm/boot/dts/ti/omap/motorola-mapphone-common.dtsi
@@ -101,10 +101,10 @@
 	 */
 	operating-points = <
 		/* kHz    uV */
-	        300000  1025000
-	        600000  1200000
-	        800000  1313000
-	        1008000 1375000
+		300000  1025000
+		600000  1200000
+		800000  1313000
+		1008000 1375000
 		1200000 1375000
 	>;
 };
diff --git a/arch/arm/boot/dts/ti/omap/motorola-mapphone-mz607-mz617.dtsi b/arch/arm/boot/dts/ti/omap/motorola-mapphone-mz607-mz617.dtsi
index a356b3a2f24e94..b97ab290695039 100644
--- a/arch/arm/boot/dts/ti/omap/motorola-mapphone-mz607-mz617.dtsi
+++ b/arch/arm/boot/dts/ti/omap/motorola-mapphone-mz607-mz617.dtsi
@@ -4,6 +4,8 @@
 #include "motorola-mapphone-common.dtsi"
 
 &keypad {
+	status = "okay";
+
 	keypad,num-rows = <8>;
 	keypad,num-columns = <8>;
 	linux,keymap = <MATRIX_KEY(5, 0, KEY_VOLUMEUP)>,
diff --git a/arch/arm/boot/dts/ti/omap/omap3-igep.dtsi b/arch/arm/boot/dts/ti/omap/omap3-igep.dtsi
index 7346cad84edab9..a0717461dee8f1 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-igep.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap3-igep.dtsi
@@ -37,7 +37,7 @@
 	gpmc_pins: gpmc-pins {
 		pinctrl-single,pins = <
 			/* OneNAND seems to require PIN_INPUT on clock. */
-                        OMAP3_CORE1_IOPAD(0x20be, PIN_INPUT | MUX_MODE0)        /* gpmc_clk.gpmc_clk */
+			OMAP3_CORE1_IOPAD(0x20be, PIN_INPUT | MUX_MODE0)        /* gpmc_clk.gpmc_clk */
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/ti/omap/omap3-n9.dts b/arch/arm/boot/dts/ti/omap/omap3-n9.dts
index a3cf3f443785f1..2edc1933449b10 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-n9.dts
+++ b/arch/arm/boot/dts/ti/omap/omap3-n9.dts
@@ -21,8 +21,11 @@
 		reg = <0x10>;
 		/* No reset gpio */
 		vana-supply = <&vaux3>;
+
 		clocks = <&isp 0>;
-		clock-frequency = <9600000>;
+		assigned-clocks = <&isp 0>;
+		assigned-clock-rates = <9600000>;
+
 		flash-leds = <&as3645a_flash &as3645a_indicator>;
 		port {
 			smia_1_1: endpoint {
diff --git a/arch/arm/boot/dts/ti/omap/omap3-n900.dts b/arch/arm/boot/dts/ti/omap/omap3-n900.dts
index 7db73d9bed9e4b..97e9eae2b60e83 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-n900.dts
+++ b/arch/arm/boot/dts/ti/omap/omap3-n900.dts
@@ -792,7 +792,8 @@
 
 		clocks = <&isp 0>;
 		clock-names = "extclk";
-		clock-frequency = <9600000>;
+		assigned-clocks = <&isp 0>;
+		assigned-clock-rates = <9600000>;
 
 		reset-gpio = <&gpio4 6 GPIO_ACTIVE_HIGH>; /* 102 */
 
diff --git a/arch/arm/boot/dts/ti/omap/omap3-n950.dts b/arch/arm/boot/dts/ti/omap/omap3-n950.dts
index b99f97880204da..2864ed8dd6c3d2 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-n950.dts
+++ b/arch/arm/boot/dts/ti/omap/omap3-n950.dts
@@ -74,8 +74,11 @@
 		reg = <0x10>;
 		/* No reset gpio */
 		vana-supply = <&vaux3>;
+
 		clocks = <&isp 0>;
-		clock-frequency = <9600000>;
+		assigned-clocks = <&isp 0>;
+		assigned-clock-rates = <9600000>;
+
 		flash-leds = <&as3645a_flash &as3645a_indicator>;
 		port {
 			smia_1_1: endpoint {
diff --git a/arch/arm/boot/dts/ti/omap/omap3-tao3530.dtsi b/arch/arm/boot/dts/ti/omap/omap3-tao3530.dtsi
index 92a5846048bd7f..f5bfa01ee244d8 100644
--- a/arch/arm/boot/dts/ti/omap/omap3-tao3530.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap3-tao3530.dtsi
@@ -145,10 +145,10 @@
 
 	mcspi3_pins: mcspi3-pins {
 		pinctrl-single,pins = <
-                        OMAP3_CORE1_IOPAD(0x25dc, PIN_OUTPUT | MUX_MODE1)	/* etk_d0.mcspi3_simo gpio14 INPUT | MODE1 */
-                        OMAP3_CORE1_IOPAD(0x25de, PIN_INPUT_PULLUP | MUX_MODE1)	/* etk_d1.mcspi3_somi gpio15 INPUT | MODE1 */
-                        OMAP3_CORE1_IOPAD(0x25e0, PIN_OUTPUT | MUX_MODE1)	/* etk_d2.mcspi3_cs0 gpio16 INPUT | MODE1 */
-                        OMAP3_CORE1_IOPAD(0x25e2, PIN_INPUT | MUX_MODE1)	/* etk_d3.mcspi3_clk gpio17 INPUT | MODE1 */
+			OMAP3_CORE1_IOPAD(0x25dc, PIN_OUTPUT | MUX_MODE1)	/* etk_d0.mcspi3_simo gpio14 INPUT | MODE1 */
+			OMAP3_CORE1_IOPAD(0x25de, PIN_INPUT_PULLUP | MUX_MODE1)	/* etk_d1.mcspi3_somi gpio15 INPUT | MODE1 */
+			OMAP3_CORE1_IOPAD(0x25e0, PIN_OUTPUT | MUX_MODE1)	/* etk_d2.mcspi3_cs0 gpio16 INPUT | MODE1 */
+			OMAP3_CORE1_IOPAD(0x25e2, PIN_INPUT | MUX_MODE1)	/* etk_d3.mcspi3_clk gpio17 INPUT | MODE1 */
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/ti/omap/omap4-droid-bionic-xt875.dts b/arch/arm/boot/dts/ti/omap/omap4-droid-bionic-xt875.dts
index 1d9000f84f1b41..e72ac17fdd96f3 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-droid-bionic-xt875.dts
+++ b/arch/arm/boot/dts/ti/omap/omap4-droid-bionic-xt875.dts
@@ -18,6 +18,8 @@
 };
 
 &keypad {
+	status = "okay";
+
 	keypad,num-rows = <8>;
 	keypad,num-columns = <8>;
 	linux,keymap = <
diff --git a/arch/arm/boot/dts/ti/omap/omap4-droid4-xt894.dts b/arch/arm/boot/dts/ti/omap/omap4-droid4-xt894.dts
index cc3f3e1b65ea98..62f87cf95f74a8 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-droid4-xt894.dts
+++ b/arch/arm/boot/dts/ti/omap/omap4-droid4-xt894.dts
@@ -46,6 +46,8 @@
 };
 
 &keypad {
+	status = "okay";
+
 	keypad,num-rows = <8>;
 	keypad,num-columns = <8>;
 	linux,keymap = <
diff --git a/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts b/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
index e11d1931c42a75..d51913620f06b3 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
+++ b/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
@@ -428,6 +428,8 @@
 };
 
 &keypad {
+	status = "okay";
+
 	pinctrl-names = "default";
 	pinctrl-0 = <&keypad_pins>;
 	keypad,num-rows = <2>;
diff --git a/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi b/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
index c1afc49f456cf0..00f98991cc5c74 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap4-l4.dtsi
@@ -1200,6 +1200,7 @@
 				reg = <0x0 0x80>;
 				interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
 				reg-names = "mpu";
+				status = "disabled";
 			};
 		};
 
diff --git a/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi
index 06651072f0007c..07c76a80cfff31 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap4-samsung-espresso-common.dtsi
@@ -16,7 +16,7 @@
 		#size-cells = <1>;
 		ranges;
 
-		continuous_splash: framebuffer@bef00000{
+		continuous_splash: framebuffer@bef00000 {
 			reg = <0xbef00000 (1024 * 600 * 4)>;
 			no-map;
 		};
@@ -117,7 +117,7 @@
 		enable-active-high;
 	};
 
-	reg_touch_ldo_en: regulator-touch-ldo-en  {
+	reg_touch_ldo_en: regulator-touch-ldo-en {
 		compatible = "regulator-fixed";
 		regulator-name = "touch_ldo_en";
 		regulator-max-microvolt = <2800000>;
@@ -153,7 +153,7 @@
 		};
 	};
 
-	reg_lcd: regulator-lcd	{
+	reg_lcd: regulator-lcd {
 		compatible = "regulator-fixed";
 		regulator-name = "lcd_en";
 		gpios = <&gpio5 7 GPIO_ACTIVE_HIGH>; /* GPIO_135 */
diff --git a/arch/arm/boot/dts/ti/omap/omap4-sdp.dts b/arch/arm/boot/dts/ti/omap/omap4-sdp.dts
index b550105585a1b1..a04234b3be2ac9 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-sdp.dts
+++ b/arch/arm/boot/dts/ti/omap/omap4-sdp.dts
@@ -531,6 +531,8 @@
 };
 
 &keypad {
+	status = "okay";
+
 	keypad,num-rows = <8>;
 	keypad,num-columns = <8>;
 	linux,keymap = <0x00000012	/* KEY_E */
diff --git a/arch/arm/boot/dts/ti/omap/omap4.dtsi b/arch/arm/boot/dts/ti/omap/omap4.dtsi
index 559b2bfe4ca7cd..6ebe5f6deee1d3 100644
--- a/arch/arm/boot/dts/ti/omap/omap4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap4.dtsi
@@ -590,7 +590,7 @@
 					#size-cells = <1>;
 					ranges = <0 0x2000 0x1000>;
 
-					rfbi: encoder@0  {
+					rfbi: encoder@0 {
 						reg = <0 0x1000>;
 						status = "disabled";
 						clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 8>, <&l3_div_ck>;
@@ -679,7 +679,7 @@
 						interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 						status = "disabled";
 						clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 8>,
-						         <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 10>;
+							 <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 10>;
 						clock-names = "fck", "sys_clk";
 
 						#address-cells = <1>;
@@ -716,7 +716,7 @@
 						interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
 						status = "disabled";
 						clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 9>,
-						         <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 10>;
+							 <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 10>;
 						clock-names = "fck", "sys_clk";
 						dmas = <&sdma 76>;
 						dma-names = "audio_tx";
diff --git a/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi b/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
index 72849e1c95b069..af32ca32993022 100644
--- a/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap5-l4.dtsi
@@ -2449,6 +2449,7 @@
 			keypad: keypad@0 {
 				compatible = "ti,omap4-keypad";
 				reg = <0x0 0x400>;
+				status = "disabled";
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/ti/omap/omap5.dtsi b/arch/arm/boot/dts/ti/omap/omap5.dtsi
index 6a66214ad0e2f9..07159b97f5043b 100644
--- a/arch/arm/boot/dts/ti/omap/omap5.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap5.dtsi
@@ -536,7 +536,7 @@
 					#size-cells = <1>;
 					ranges = <0 0x2000 0x1000>;
 
-					rfbi: encoder@0  {
+					rfbi: encoder@0 {
 						compatible = "ti,omap5-rfbi";
 						reg = <0 0x100>;
 						status = "disabled";
diff --git a/arch/arm/boot/dts/xilinx/zynq-7000.dtsi b/arch/arm/boot/dts/xilinx/zynq-7000.dtsi
index 153b8d93cbee2c..715299f5d860ae 100644
--- a/arch/arm/boot/dts/xilinx/zynq-7000.dtsi
+++ b/arch/arm/boot/dts/xilinx/zynq-7000.dtsi
@@ -380,10 +380,10 @@
 			 * "dma4", "dma5", "dma6", "dma7";
 			 */
 			interrupts = <0 13 4>,
-			             <0 14 4>, <0 15 4>,
-			             <0 16 4>, <0 17 4>,
-			             <0 40 4>, <0 41 4>,
-			             <0 42 4>, <0 43 4>;
+				     <0 14 4>, <0 15 4>,
+				     <0 16 4>, <0 17 4>,
+				     <0 40 4>, <0 41 4>,
+				     <0 42 4>, <0 43 4>;
 			#dma-cells = <1>;
 			clocks = <&clkc 27>;
 			clock-names = "apb_pclk";
diff --git a/arch/arm/boot/dts/xilinx/zynq-parallella.dts b/arch/arm/boot/dts/xilinx/zynq-parallella.dts
index 366af4fcf8d907..b4651540935ad0 100644
--- a/arch/arm/boot/dts/xilinx/zynq-parallella.dts
+++ b/arch/arm/boot/dts/xilinx/zynq-parallella.dts
@@ -44,10 +44,10 @@
 	ethernet_phy: ethernet-phy@0 {
 		/* Marvell 88E1318 */
 		compatible = "ethernet-phy-id0141.0e90",
-		             "ethernet-phy-ieee802.3-c22";
+			     "ethernet-phy-ieee802.3-c22";
 		reg = <0>;
 		marvell,reg-init = <0x3 0x10 0xff00 0x1e>,
-		                   <0x3 0x11 0xfff0 0xa>;
+				   <0x3 0x11 0xfff0 0xa>;
 	};
 };
 
diff --git a/arch/arm/configs/aspeed_g4_defconfig b/arch/arm/configs/aspeed_g4_defconfig
index f86dd4ce7d0def..2c9d5a644ae965 100644
--- a/arch/arm/configs/aspeed_g4_defconfig
+++ b/arch/arm/configs/aspeed_g4_defconfig
@@ -249,7 +249,7 @@ CONFIG_DEBUG_INFO_REDUCED=y
 CONFIG_GDB_SCRIPTS=y
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_DEBUG_FS=y
-CONFIG_ARM_DEBUG_WX=y
+CONFIG_ARM_CHECK_WX=y
 CONFIG_SCHED_STACK_END_CHECK=y
 CONFIG_PANIC_ON_OOPS=y
 CONFIG_PANIC_TIMEOUT=-1
diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig
index 45b937419dbdaa..1327a09e163ab8 100644
--- a/arch/arm/configs/aspeed_g5_defconfig
+++ b/arch/arm/configs/aspeed_g5_defconfig
@@ -300,7 +300,7 @@ CONFIG_DEBUG_INFO_REDUCED=y
 CONFIG_GDB_SCRIPTS=y
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_DEBUG_FS=y
-CONFIG_ARM_DEBUG_WX=y
+CONFIG_ARM_CHECK_WX=y
 CONFIG_SCHED_STACK_END_CHECK=y
 CONFIG_PANIC_ON_OOPS=y
 CONFIG_PANIC_TIMEOUT=-1
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index f9b8dbf9318c9d..069d5a49f01cb3 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -800,6 +800,7 @@ CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y
 CONFIG_SOUND=m
 CONFIG_SND=m
+CONFIG_SND_ALOOP=m
 CONFIG_SND_HDA_INPUT_BEEP=y
 CONFIG_SND_HDA_PATCH_LOADER=y
 CONFIG_SND_HDA_TEGRA=m
@@ -1101,15 +1102,6 @@ CONFIG_COMMON_CLK_S2MPS11=m
 CONFIG_CLK_TWL=m
 CONFIG_CLK_RASPBERRYPI=y
 CONFIG_COMMON_CLK_QCOM=y
-CONFIG_QCOM_A53PLL=y
-CONFIG_QCOM_CLK_APCS_MSM8916=y
-CONFIG_QCOM_CLK_RPM=y
-CONFIG_QCOM_CLK_SMD_RPM=y
-CONFIG_APQ_MMCC_8084=y
-CONFIG_MSM_GCC_8660=y
-CONFIG_MSM_GCC_8916=y
-CONFIG_MSM_MMCC_8960=y
-CONFIG_MSM_MMCC_8974=y
 CONFIG_HWSPINLOCK=y
 CONFIG_HWSPINLOCK_OMAP=y
 CONFIG_HWSPINLOCK_QCOM=y
diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
index f827f89567fdf1..5791953b75fff1 100644
--- a/arch/arm/configs/omap2plus_defconfig
+++ b/arch/arm/configs/omap2plus_defconfig
@@ -268,6 +268,8 @@ CONFIG_PCI_ENDPOINT_CONFIGFS=y
 CONFIG_PCI_EPF_TEST=m
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_FW_LOADER_COMPRESS=y
+CONFIG_FW_LOADER_COMPRESS_ZSTD=y
 CONFIG_OMAP_OCP2SCP=y
 CONFIG_CONNECTOR=m
 CONFIG_MTD=y
@@ -523,6 +525,7 @@ CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y
 CONFIG_LOGO=y
 CONFIG_SOUND=m
 CONFIG_SND=m
+CONFIG_SND_ALOOP=m
 CONFIG_SND_OSSEMUL=y
 CONFIG_SND_MIXER_OSS=m
 CONFIG_SND_PCM_OSS=m
diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig
index df0a0ce5b097bc..cc93a32342af50 100644
--- a/arch/arm/configs/qcom_defconfig
+++ b/arch/arm/configs/qcom_defconfig
@@ -206,20 +206,6 @@ CONFIG_DMADEVICES=y
 CONFIG_QCOM_BAM_DMA=y
 CONFIG_STAGING=y
 CONFIG_COMMON_CLK_QCOM=y
-CONFIG_QCOM_A7PLL=y
-CONFIG_QCOM_CLK_APCS_SDX55=y
-CONFIG_QCOM_CLK_RPM=y
-CONFIG_QCOM_CLK_SMD_RPM=y
-CONFIG_QCOM_CLK_RPMH=y
-CONFIG_APQ_MMCC_8084=y
-CONFIG_IPQ_GCC_4019=y
-CONFIG_IPQ_LCC_806X=y
-CONFIG_MSM_GCC_8660=y
-CONFIG_MSM_LCC_8960=y
-CONFIG_MDM_LCC_9615=y
-CONFIG_MSM_MMCC_8960=y
-CONFIG_MSM_MMCC_8974=y
-CONFIG_SDX_GCC_55=y
 CONFIG_HWSPINLOCK=y
 CONFIG_HWSPINLOCK_QCOM=y
 CONFIG_MAILBOX=y
diff --git a/arch/arm/configs/sama5_defconfig b/arch/arm/configs/sama5_defconfig
index 1eafad9139f1d4..6e5f845e95dbf1 100644
--- a/arch/arm/configs/sama5_defconfig
+++ b/arch/arm/configs/sama5_defconfig
@@ -69,8 +69,8 @@ CONFIG_BLK_DEV_SD=y
 # CONFIG_BLK_DEV_BSG is not set
 # CONFIG_SCSI_LOWLEVEL is not set
 CONFIG_NETDEVICES=y
-CONFIG_NET_DSA_MICROCHIP_KSZ9477=m
-CONFIG_NET_DSA_MICROCHIP_KSZ9477_SPI=m
+CONFIG_NET_DSA_MICROCHIP_KSZ_COMMON=m
+CONFIG_NET_DSA_MICROCHIP_KSZ_SPI=m
 # CONFIG_NET_VENDOR_BROADCOM is not set
 CONFIG_MACB=y
 # CONFIG_NET_VENDOR_CIRRUS is not set
diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig
index 6f9696e9fe17dd..58059f943a9b30 100644
--- a/arch/arm/configs/shmobile_defconfig
+++ b/arch/arm/configs/shmobile_defconfig
@@ -48,7 +48,6 @@ CONFIG_MTD_CFI_INTELEXT=y
 CONFIG_MTD_PHYSMAP=y
 CONFIG_MTD_PHYSMAP_OF=y
 CONFIG_MTD_SPI_NOR=y
-CONFIG_EEPROM_AT24=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_ATA=y
 CONFIG_SATA_RCAR=y
@@ -79,6 +78,7 @@ CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
 # CONFIG_SERIAL_8250_PCI is not set
 # CONFIG_SERIAL_8250_EXAR is not set
+# CONFIG_SERIAL_8250_MOXA_PCIE is not set
 CONFIG_SERIAL_8250_DW=y
 CONFIG_SERIAL_8250_EM=y
 # CONFIG_SERIAL_8250_PERICOM is not set
@@ -209,6 +209,7 @@ CONFIG_PWM_RENESAS_RCAR=y
 CONFIG_PWM_RENESAS_TPU=y
 CONFIG_PHY_RCAR_GEN2=y
 CONFIG_PHY_RCAR_GEN3_USB2=y
+CONFIG_EEPROM_AT24=y
 # CONFIG_DNOTIFY is not set
 CONFIG_MSDOS_FS=y
 CONFIG_VFAT_FS=y
@@ -225,4 +226,4 @@ CONFIG_CMA_SIZE_MBYTES=64
 CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DEBUG_FS=y
-CONFIG_ARM_DEBUG_WX=y
+CONFIG_ARM_CHECK_WX=y
diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
index c49ddafc54f34f..f0c8bfd2ac8fc3 100644
--- a/arch/arm/crypto/aes-neonbs-glue.c
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -60,7 +60,7 @@ static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 			unsigned int key_len)
 {
 	struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct crypto_aes_ctx rk;
+	struct crypto_aes_ctx rk __cleanup(aes_zeroize_ctx);
 	int err;
 
 	err = aes_expandkey(&rk, in_key, key_len);
diff --git a/arch/arm/include/asm/archrandom.h b/arch/arm/include/asm/archrandom.h
index cc4714eb1a7556..ee39a03ddf8abe 100644
--- a/arch/arm/include/asm/archrandom.h
+++ b/arch/arm/include/asm/archrandom.h
@@ -2,7 +2,7 @@
 #ifndef _ASM_ARCHRANDOM_H
 #define _ASM_ARCHRANDOM_H
 
-static inline bool __init smccc_probe_trng(void)
+static inline bool smccc_probe_trng(void)
 {
 	return false;
 }
diff --git a/arch/arm/include/asm/linkage.h b/arch/arm/include/asm/linkage.h
index c4670694ada796..bba992f5d7a804 100644
--- a/arch/arm/include/asm/linkage.h
+++ b/arch/arm/include/asm/linkage.h
@@ -9,4 +9,33 @@
   .type name, %function; \
   END(name)
 
+#ifdef __ASSEMBLY__
+
+/*
+ * Variants of SYM_TYPED_START/SYM_TYPED_FUNC_START that align the
+ * function entry itself instead of the kCFI type hash preceding it,
+ * for functions whose entry point must meet an alignment requirement,
+ * such as the 8-byte alignment fncpy() demands of its source.
+ */
+#ifdef CONFIG_CFI
+
+#define SYM_TYPED_START_ALIGNED(name, linkage, align)	\
+	linkage(name) ASM_NL				\
+	.balign align ASM_NL				\
+	.fill (align) - 4, 1, 0 ASM_NL			\
+	__CFI_TYPE(name) ASM_NL				\
+	name:
+
+#else /* CONFIG_CFI */
+
+#define SYM_TYPED_START_ALIGNED(name, linkage, align)	\
+	SYM_START(name, linkage, .balign align)
+
+#endif /* CONFIG_CFI */
+
+#define SYM_TYPED_FUNC_START_ALIGNED(name, align)	\
+	SYM_TYPED_START_ALIGNED(name, SYM_L_GLOBAL, align)
+
+#endif /* __ASSEMBLY__ */
+
 #endif
diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
index a17f01235c29c0..25d78ee33fa77a 100644
--- a/arch/arm/include/asm/pgalloc.h
+++ b/arch/arm/include/asm/pgalloc.h
@@ -7,8 +7,6 @@
 #ifndef _ASMARM_PGALLOC_H
 #define _ASMARM_PGALLOC_H
 
-#include <linux/pagemap.h>
-
 #include <asm/domain.h>
 #include <asm/pgtable-hwdef.h>
 #include <asm/processor.h>
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index 982795cf45637e..8dd17d20faa33f 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -44,13 +44,6 @@
 #define LIBRARY_TEXT_START	0x0c000000
 
 #ifndef __ASSEMBLY__
-extern void __pte_error(const char *file, int line, pte_t);
-extern void __pmd_error(const char *file, int line, pmd_t);
-extern void __pgd_error(const char *file, int line, pgd_t);
-
-#define pte_ERROR(pte)		__pte_error(__FILE__, __LINE__, pte)
-#define pmd_ERROR(pmd)		__pmd_error(__FILE__, __LINE__, pmd)
-#define pgd_ERROR(pgd)		__pgd_error(__FILE__, __LINE__, pgd)
 
 /*
  * This is the lowest virtual address we can permit any user space
diff --git a/arch/arm/include/asm/ptdump.h b/arch/arm/include/asm/ptdump.h
index 46a4575146ee85..3c5245220ecfee 100644
--- a/arch/arm/include/asm/ptdump.h
+++ b/arch/arm/include/asm/ptdump.h
@@ -32,10 +32,10 @@ void ptdump_check_wx(void);
 
 #endif /* CONFIG_ARM_PTDUMP_CORE */
 
-#ifdef CONFIG_ARM_DEBUG_WX
-#define arm_debug_checkwx() ptdump_check_wx()
+#ifdef CONFIG_ARM_CHECK_WX
+#define arm_pgtable_checkwx() ptdump_check_wx()
 #else
-#define arm_debug_checkwx() do { } while (0)
+#define arm_pgtable_checkwx() do { } while (0)
 #endif
 
 #endif /* __ASM_PTDUMP_H */
diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index ea4fbe7b17f6ff..1b8903a1ccf7c9 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -18,8 +18,6 @@
 
 #ifndef CONFIG_MMU
 
-#include <linux/pagemap.h>
-
 #define tlb_flush(tlb)	((void) tlb)
 
 #include <asm-generic/tlb.h>
diff --git a/arch/arm/include/uapi/asm/setup.h b/arch/arm/include/uapi/asm/setup.h
index 8e50e034fec73a..4aa93558af1e7f 100644
--- a/arch/arm/include/uapi/asm/setup.h
+++ b/arch/arm/include/uapi/asm/setup.h
@@ -17,7 +17,11 @@
 
 #include <linux/types.h>
 
-#define COMMAND_LINE_SIZE 1024
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
+#define COMMAND_LINE_SIZE	1024
+#endif
 
 /* The list ends with an ATAG_NONE node. */
 #define ATAG_NONE	0x00000000
diff --git a/arch/arm/kernel/perf_regs.c b/arch/arm/kernel/perf_regs.c
index 0529f90395c94d..838d701adf4d88 100644
--- a/arch/arm/kernel/perf_regs.c
+++ b/arch/arm/kernel/perf_regs.c
@@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 
 #define REG_RESERVED (~((1ULL << PERF_REG_ARM_MAX) - 1))
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask || mask & REG_RESERVED)
 		return -EINVAL;
@@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task)
 	return PERF_SAMPLE_REGS_ABI_32;
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index afbd2ebe5c39dc..ad04c806cc9d8f 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -753,23 +753,6 @@ void __readwrite_bug(const char *fn)
 }
 EXPORT_SYMBOL(__readwrite_bug);
 
-#ifdef CONFIG_MMU
-void __pte_error(const char *file, int line, pte_t pte)
-{
-	pr_err("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte));
-}
-
-void __pmd_error(const char *file, int line, pmd_t pmd)
-{
-	pr_err("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd));
-}
-
-void __pgd_error(const char *file, int line, pgd_t pgd)
-{
-	pr_err("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd));
-}
-#endif
-
 asmlinkage void __div0(void)
 {
 	pr_err("Division by zero in kernel.\n");
diff --git a/arch/arm/mach-at91/pm.h b/arch/arm/mach-at91/pm.h
index 50c3a425d14006..2193e1ab984762 100644
--- a/arch/arm/mach-at91/pm.h
+++ b/arch/arm/mach-at91/pm.h
@@ -25,7 +25,7 @@
 #define AT91_PM_ULP1		0x03
 #define	AT91_PM_BACKUP		0x04
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 struct at91_pm_data {
 	void __iomem *pmc;
 	void __iomem *ramc[2];
diff --git a/arch/arm/mach-exynos/smc.h b/arch/arm/mach-exynos/smc.h
index 5c30feb8f07d9e..5c7730f6259fcb 100644
--- a/arch/arm/mach-exynos/smc.h
+++ b/arch/arm/mach-exynos/smc.h
@@ -32,11 +32,11 @@
 #define SMC_REG_CLASS_SFR_W	(0x1 << 30)
 #define SMC_REG_ID_SFR_W(addr)	(SMC_REG_CLASS_SFR_W | ((addr) >> 2))
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 extern void exynos_smc(u32 cmd, u32 arg1, u32 arg2, u32 arg3);
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 /* op type for SMC_CMD_SAVE and SMC_CMD_SHUTDOWN */
 #define OP_TYPE_CORE		0x0
diff --git a/arch/arm/mach-imx/hardware.h b/arch/arm/mach-imx/hardware.h
index 0760fff39a0b36..71f09648c77784 100644
--- a/arch/arm/mach-imx/hardware.h
+++ b/arch/arm/mach-imx/hardware.h
@@ -7,7 +7,7 @@
 #ifndef __ASM_ARCH_MXC_HARDWARE_H__
 #define __ASM_ARCH_MXC_HARDWARE_H__
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #include <asm/io.h>
 #include <soc/imx/revision.h>
 #endif
diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c
index 7f62009257526f..58f53fba202f44 100644
--- a/arch/arm/mach-imx/mach-imx6q.c
+++ b/arch/arm/mach-imx/mach-imx6q.c
@@ -20,26 +20,6 @@
 #include "cpuidle.h"
 #include "hardware.h"
 
-/* For imx6q sabrelite board: set KSZ9021RN RGMII pad skew */
-static int ksz9021rn_phy_fixup(struct phy_device *phydev)
-{
-	if (IS_BUILTIN(CONFIG_PHYLIB)) {
-		/* min rx data delay */
-		phy_write(phydev, MICREL_KSZ9021_EXTREG_CTRL,
-			0x8000 | MICREL_KSZ9021_RGMII_RX_DATA_PAD_SCEW);
-		phy_write(phydev, MICREL_KSZ9021_EXTREG_DATA_WRITE, 0x0000);
-
-		/* max rx/tx clock delay, min rx/tx control delay */
-		phy_write(phydev, MICREL_KSZ9021_EXTREG_CTRL,
-			0x8000 | MICREL_KSZ9021_RGMII_CLK_CTRL_PAD_SCEW);
-		phy_write(phydev, MICREL_KSZ9021_EXTREG_DATA_WRITE, 0xf0f0);
-		phy_write(phydev, MICREL_KSZ9021_EXTREG_CTRL,
-			MICREL_KSZ9021_RGMII_CLK_CTRL_PAD_SCEW);
-	}
-
-	return 0;
-}
-
 /*
  * fixup for PLX PEX8909 bridge to configure GPIO1-7 as output High
  * as they are used for slots1-7 PERST#
@@ -68,14 +48,6 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_PLX, 0x8609, ventana_pciesw_early_fixup);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_PLX, 0x8606, ventana_pciesw_early_fixup);
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_PLX, 0x8604, ventana_pciesw_early_fixup);
 
-static void __init imx6q_enet_phy_init(void)
-{
-	if (IS_BUILTIN(CONFIG_PHYLIB)) {
-		phy_register_fixup_for_uid(PHY_ID_KSZ9021, MICREL_PHY_ID_MASK,
-				ksz9021rn_phy_fixup);
-	}
-}
-
 static void __init imx6q_1588_init(void)
 {
 	struct device_node *np;
@@ -179,8 +151,6 @@ static void __init imx6q_init_machine(void)
 		imx_print_silicon_rev(cpu_is_imx6dl() ? "i.MX6DL" : "i.MX6Q",
 				imx_get_soc_revision());
 
-	imx6q_enet_phy_init();
-
 	of_platform_default_populate(NULL, NULL, NULL);
 
 	imx_anatop_init();
diff --git a/arch/arm/mach-imx/mxc.h b/arch/arm/mach-imx/mxc.h
index fe2d0f5abfcc24..1d13e3959159e9 100644
--- a/arch/arm/mach-imx/mxc.h
+++ b/arch/arm/mach-imx/mxc.h
@@ -16,7 +16,7 @@
 
 #define IMX_DDR_TYPE_LPDDR2		1
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #ifdef CONFIG_SOC_IMX6SL
 static inline bool cpu_is_imx6sl(void)
diff --git a/arch/arm/mach-imx/pm-imx6.c b/arch/arm/mach-imx/pm-imx6.c
index a671ca498f887f..3c73e2cdb8cebe 100644
--- a/arch/arm/mach-imx/pm-imx6.c
+++ b/arch/arm/mach-imx/pm-imx6.c
@@ -4,6 +4,8 @@
  * Copyright 2011 Linaro Ltd.
  */
 
+#include <linux/build_bug.h>
+#include <linux/cfi.h>
 #include <linux/clk/imx.h>
 #include <linux/delay.h>
 #include <linux/init.h>
@@ -18,6 +20,7 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/stddef.h>
 #include <linux/suspend.h>
 #include <asm/cacheflush.h>
 #include <asm/fncpy.h>
@@ -61,9 +64,9 @@
 #define MX6Q_SUSPEND_OCRAM_SIZE		0x1000
 #define MX6_MAX_MMDC_IO_NUM		33
 
-static void __iomem *ccm_base;
-static void __iomem *suspend_ocram_base;
-static void (*imx6_suspend_in_ocram_fn)(void __iomem *ocram_vbase);
+static void __iomem *ccm_base __ro_after_init;
+static void __iomem *suspend_ocram_base __ro_after_init;
+static void (*imx6_suspend_in_ocram_fn)(void __iomem *ocram_vbase) __ro_after_init;
 
 /*
  * suspend ocram space layout:
@@ -229,8 +232,18 @@ struct imx6_cpu_pm_info {
 	struct imx6_pm_base l2_base;
 	u32 mmdc_io_num; /* Number of MMDC IOs which need saved/restored. */
 	u32 mmdc_io_val[MX6_MAX_MMDC_IO_NUM][2]; /* To save offset and value */
+	u32 cfi_type; /* kCFI type hash of imx6_suspend() */
 } __aligned(8);
 
+/*
+ * The ocram copy of imx6_suspend() starts right after struct imx6_cpu_pm_info,
+ * and the CFI check on the indirect call reads the kCFI type hash from the
+ * four bytes preceding the function entry, so cfi_type must occupy the last
+ * four bytes of the struct, i.e. fit into its tail padding.
+ */
+static_assert(offsetofend(struct imx6_cpu_pm_info, cfi_type) ==
+	      sizeof(struct imx6_cpu_pm_info));
+
 void imx6_set_int_mem_clk_lpm(bool enable)
 {
 	u32 val = readl_relaxed(ccm_base + CGPR);
@@ -568,6 +581,13 @@ static int __init imx6q_suspend_init(const struct imx6_pm_socdata *socdata)
 			mmdc_offset_array[i]);
 	}
 
+	/*
+	 * Mask out the Thumb bit, as cfi_get_func_hash() expects the
+	 * function's actual start address. Returns 0 if CONFIG_CFI=n.
+	 */
+	pm_info->cfi_type =
+		cfi_get_func_hash((void *)((uintptr_t)&imx6_suspend & ~1UL));
+
 	imx6_suspend_in_ocram_fn = fncpy(
 		suspend_ocram_base + sizeof(*pm_info),
 		&imx6_suspend,
diff --git a/arch/arm/mach-imx/suspend-imx6.S b/arch/arm/mach-imx/suspend-imx6.S
index 63ccc2d0e920ac..95d5f723ea0ed9 100644
--- a/arch/arm/mach-imx/suspend-imx6.S
+++ b/arch/arm/mach-imx/suspend-imx6.S
@@ -3,6 +3,7 @@
  * Copyright 2014 Freescale Semiconductor, Inc.
  */
 
+#include <linux/cfi_types.h>
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/asm-offsets.h>
@@ -148,7 +149,8 @@
 
 	.endm
 
-ENTRY(imx6_suspend)
+/* fncpy() requires an 8-byte-aligned entry; see arch/arm/include/asm/fncpy.h */
+SYM_TYPED_FUNC_START_ALIGNED(imx6_suspend, 8)
 	ldr	r1, [r0, #PM_INFO_PBASE_OFFSET]
 	ldr	r2, [r0, #PM_INFO_RESUME_ADDR_OFFSET]
 	ldr	r3, [r0, #PM_INFO_DDR_TYPE_OFFSET]
@@ -329,4 +331,4 @@ resume:
 	resume_mmdc
 
 	ret	lr
-ENDPROC(imx6_suspend)
+SYM_FUNC_END(imx6_suspend)
diff --git a/arch/arm/mach-omap2/control.h b/arch/arm/mach-omap2/control.h
index 7e7440533bf90c..e4f4b169580843 100644
--- a/arch/arm/mach-omap2/control.h
+++ b/arch/arm/mach-omap2/control.h
@@ -18,7 +18,7 @@
 
 #include "am33xx.h"
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #define OMAP242X_CTRL_REGADDR(reg)					\
 		OMAP2_L4_IO_ADDRESS(OMAP242X_CTRL_BASE + (reg))
 #define OMAP243X_CTRL_REGADDR(reg)					\
@@ -36,7 +36,7 @@
 		OMAP2_L4_IO_ADDRESS(OMAP343X_CTRL_BASE + (reg))
 #define AM33XX_CTRL_REGADDR(reg)					\
 		AM33XX_L4_WK_IO_ADDRESS(AM33XX_SCM_BASE + (reg))
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 /*
  * As elsewhere, the "OMAP2_" prefix indicates that the macro is valid for
@@ -503,7 +503,7 @@
 #define		FEAT_NEON_NONE		1
 
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 #ifdef CONFIG_ARCH_OMAP2PLUS
 extern u8 omap_ctrl_readb(u16 offset);
 extern u16 omap_ctrl_readw(u16 offset);
@@ -534,7 +534,7 @@ int omap_control_init(void);
 #define omap_ctrl_writel(x, y)		WARN_ON(1)
 #define omap4_ctrl_pad_writel(x, y)	WARN_ON(1)
 #endif
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
 
 #endif /* __ARCH_ARM_MACH_OMAP2_CONTROL_H */
 
diff --git a/arch/arm/mach-omap2/ctrl_module_wkup_44xx.h b/arch/arm/mach-omap2/ctrl_module_wkup_44xx.h
deleted file mode 100644
index 414f14d1e0a0e1..00000000000000
--- a/arch/arm/mach-omap2/ctrl_module_wkup_44xx.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * OMAP44xx CTRL_MODULE_WKUP registers and bitfields
- *
- * Copyright (C) 2009-2010 Texas Instruments, Inc.
- *
- * Benoit Cousson (b-cousson@ti.com)
- * Santosh Shilimkar (santosh.shilimkar@ti.com)
- *
- * This file is automatically generated from the OMAP hardware databases.
- * We respectfully ask that any modifications to this file be coordinated
- * with the public linux-omap@vger.kernel.org mailing list and the
- * authors above to ensure that the autogeneration scripts are kept
- * up-to-date with the file contents.
- */
-
-#ifndef __ARCH_ARM_MACH_OMAP2_CTRL_MODULE_WKUP_44XX_H
-#define __ARCH_ARM_MACH_OMAP2_CTRL_MODULE_WKUP_44XX_H
-
-
-/* Base address */
-#define OMAP4_CTRL_MODULE_WKUP				0x4a30c000
-
-/* Registers offset */
-#define OMAP4_CTRL_MODULE_WKUP_IP_REVISION		0x0000
-#define OMAP4_CTRL_MODULE_WKUP_IP_HWINFO		0x0004
-#define OMAP4_CTRL_MODULE_WKUP_IP_SYSCONFIG		0x0010
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_0	0x0460
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_1	0x0464
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_2	0x0468
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_3	0x046c
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_4	0x0470
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_5	0x0474
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_6	0x0478
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_7	0x047c
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_8	0x0480
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_9	0x0484
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_10	0x0488
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_11	0x048c
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_12	0x0490
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_13	0x0494
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_14	0x0498
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_15	0x049c
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_16	0x04a0
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_17	0x04a4
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_18	0x04a8
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_19	0x04ac
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_20	0x04b0
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_21	0x04b4
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_22	0x04b8
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_23	0x04bc
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_24	0x04c0
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_25	0x04c4
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_26	0x04c8
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_27	0x04cc
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_28	0x04d0
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_29	0x04d4
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_30	0x04d8
-#define OMAP4_CTRL_MODULE_WKUP_CONF_DEBUG_SEL_TST_31	0x04dc
-
-/* Registers shifts and masks */
-
-/* IP_REVISION */
-#define OMAP4_IP_REV_SCHEME_SHIFT		30
-#define OMAP4_IP_REV_SCHEME_MASK		(0x3 << 30)
-#define OMAP4_IP_REV_FUNC_SHIFT			16
-#define OMAP4_IP_REV_FUNC_MASK			(0xfff << 16)
-#define OMAP4_IP_REV_RTL_SHIFT			11
-#define OMAP4_IP_REV_RTL_MASK			(0x1f << 11)
-#define OMAP4_IP_REV_MAJOR_SHIFT		8
-#define OMAP4_IP_REV_MAJOR_MASK			(0x7 << 8)
-#define OMAP4_IP_REV_CUSTOM_SHIFT		6
-#define OMAP4_IP_REV_CUSTOM_MASK		(0x3 << 6)
-#define OMAP4_IP_REV_MINOR_SHIFT		0
-#define OMAP4_IP_REV_MINOR_MASK			(0x3f << 0)
-
-/* IP_HWINFO */
-#define OMAP4_IP_HWINFO_SHIFT			0
-#define OMAP4_IP_HWINFO_MASK			(0xffffffff << 0)
-
-/* IP_SYSCONFIG */
-#define OMAP4_IP_SYSCONFIG_IDLEMODE_SHIFT	2
-#define OMAP4_IP_SYSCONFIG_IDLEMODE_MASK	(0x3 << 2)
-
-/* CONF_DEBUG_SEL_TST_0 */
-#define OMAP4_WKUP_MODE_SHIFT			0
-#define OMAP4_WKUP_MODE_MASK				(1 << 0)
-
-#endif
diff --git a/arch/arm/mach-omap2/soc.h b/arch/arm/mach-omap2/soc.h
index 9e3dbb743e5c4f..b3949559edadfd 100644
--- a/arch/arm/mach-omap2/soc.h
+++ b/arch/arm/mach-omap2/soc.h
@@ -19,7 +19,7 @@
 #include "am33xx.h"
 #include "omap54xx.h"
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/bitops.h>
 #include <linux/of.h>
@@ -501,4 +501,4 @@ level(__##fn);
 #define cpu_is_ti816x()		soc_is_ti816x()
 #define cpu_is_ti81xx()		soc_is_ti81xx()
 
-#endif	/* __ASSEMBLY__ */
+#endif	/* __ASSEMBLER__ */
diff --git a/arch/arm/mach-omap2/sram.h b/arch/arm/mach-omap2/sram.h
index 030cabc39821d2..a1d72e014e62b6 100644
--- a/arch/arm/mach-omap2/sram.h
+++ b/arch/arm/mach-omap2/sram.h
@@ -3,7 +3,7 @@
  * Interface for functions that need to be run in internal SRAM
  */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 extern void omap2_sram_ddr_init(u32 *slow_dll_ctrl, u32 fast_dll_ctrl,
 				u32 base_cs, u32 force_unlock);
@@ -48,7 +48,7 @@ extern void omap_push_sram_idle(void);
 static inline void omap_push_sram_idle(void) {}
 #endif /* CONFIG_PM */
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 
 /*
  * OMAP2+: define the SRAM PA addresses.
diff --git a/arch/arm/mach-pxa/pxa3xx.c b/arch/arm/mach-pxa/pxa3xx.c
index 06c578ea658e32..ce9e418486653e 100644
--- a/arch/arm/mach-pxa/pxa3xx.c
+++ b/arch/arm/mach-pxa/pxa3xx.c
@@ -17,6 +17,7 @@
 #include <linux/gpio-pxa.h>
 #include <linux/pm.h>
 #include <linux/platform_device.h>
+#include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqchip.h>
 #include <linux/io.h>
diff --git a/arch/arm/mach-s3c/Kconfig b/arch/arm/mach-s3c/Kconfig
index b3656109f1f7e2..2e77bb4f3a8c73 100644
--- a/arch/arm/mach-s3c/Kconfig
+++ b/arch/arm/mach-s3c/Kconfig
@@ -111,11 +111,6 @@ config S3C64XX_DEV_SPI0
 	  Compile in platform device definitions for S3C64XX's type
 	  SPI controller 0
 
-config SAMSUNG_DEV_KEYPAD
-	bool
-	help
-	  Compile in platform device definitions for keypad
-
 config SAMSUNG_DEV_PWM
 	bool
 	help
diff --git a/arch/arm/mach-s3c/Kconfig.s3c64xx b/arch/arm/mach-s3c/Kconfig.s3c64xx
index cf59147dc9ca9c..145d04effdfd80 100644
--- a/arch/arm/mach-s3c/Kconfig.s3c64xx
+++ b/arch/arm/mach-s3c/Kconfig.s3c64xx
@@ -81,11 +81,6 @@ config S3C64XX_SETUP_FB_24BPP
 	help
 	  Common setup code for S3C64XX with an 24bpp RGB display helper.
 
-config S3C64XX_SETUP_KEYPAD
-	bool
-	help
-	  Common setup code for S3C64XX KEYPAD GPIO configurations
-
 config S3C64XX_SETUP_SDHCI_GPIO
 	bool
 	help
@@ -113,7 +108,6 @@ config MACH_WLF_CRAGG_6410
 	select S3C64XX_DEV_SPI0
 	select S3C64XX_SETUP_FB_24BPP
 	select S3C64XX_SETUP_I2C1
-	select S3C64XX_SETUP_KEYPAD
 	select S3C64XX_SETUP_SDHCI
 	select S3C64XX_SETUP_SPI
 	select S3C64XX_SETUP_USB_PHY
@@ -124,7 +118,6 @@ config MACH_WLF_CRAGG_6410
 	select S3C_DEV_I2C1
 	select S3C_DEV_USB_HOST
 	select S3C_DEV_USB_HSOTG
-	select SAMSUNG_DEV_KEYPAD
 	select SAMSUNG_DEV_PWM
 	help
 	  Machine support for the Wolfson Cragganmore S3C6410 variant.
diff --git a/arch/arm/mach-s3c/Makefile.s3c64xx b/arch/arm/mach-s3c/Makefile.s3c64xx
index 61287ad2ea4276..1686b1f344f8c7 100644
--- a/arch/arm/mach-s3c/Makefile.s3c64xx
+++ b/arch/arm/mach-s3c/Makefile.s3c64xx
@@ -32,7 +32,6 @@ obj-y				+= dev-audio-s3c64xx.o
 obj-$(CONFIG_S3C64XX_SETUP_FB_24BPP)	+= setup-fb-24bpp-s3c64xx.o
 obj-$(CONFIG_S3C64XX_SETUP_I2C0)	+= setup-i2c0-s3c64xx.o
 obj-$(CONFIG_S3C64XX_SETUP_I2C1)	+= setup-i2c1-s3c64xx.o
-obj-$(CONFIG_S3C64XX_SETUP_KEYPAD)	+= setup-keypad-s3c64xx.o
 obj-$(CONFIG_S3C64XX_SETUP_SDHCI_GPIO)	+= setup-sdhci-gpio-s3c64xx.o
 obj-$(CONFIG_S3C64XX_SETUP_SPI)		+= setup-spi-s3c64xx.o
 obj-$(CONFIG_S3C64XX_SETUP_USB_PHY) += setup-usb-phy-s3c64xx.o
diff --git a/arch/arm/mach-s3c/devs.c b/arch/arm/mach-s3c/devs.c
index 572b156644bd6b..35b98a9a9237cd 100644
--- a/arch/arm/mach-s3c/devs.c
+++ b/arch/arm/mach-s3c/devs.c
@@ -39,7 +39,6 @@
 #include "devs.h"
 #include "fb.h"
 #include <linux/platform_data/i2c-s3c2410.h>
-#include "keypad.h"
 #include "pwm-core.h"
 #include "sdhci.h"
 #include "usb-phy.h"
@@ -265,32 +264,6 @@ void __init s3c_i2c1_set_platdata(struct s3c2410_platform_i2c *pd)
 }
 #endif /* CONFIG_S3C_DEV_I2C1 */
 
-/* KEYPAD */
-
-#ifdef CONFIG_SAMSUNG_DEV_KEYPAD
-static struct resource samsung_keypad_resources[] = {
-	[0] = DEFINE_RES_MEM(SAMSUNG_PA_KEYPAD, SZ_32),
-	[1] = DEFINE_RES_IRQ(IRQ_KEYPAD),
-};
-
-struct platform_device samsung_device_keypad = {
-	.name		= "samsung-keypad",
-	.id		= -1,
-	.num_resources	= ARRAY_SIZE(samsung_keypad_resources),
-	.resource	= samsung_keypad_resources,
-};
-
-void __init samsung_keypad_set_platdata(struct samsung_keypad_platdata *pd)
-{
-	struct samsung_keypad_platdata *npd;
-
-	npd = s3c_set_platdata(pd, sizeof(*npd), &samsung_device_keypad);
-
-	if (!npd->cfg_gpio)
-		npd->cfg_gpio = samsung_keypad_cfg_gpio;
-}
-#endif /* CONFIG_SAMSUNG_DEV_KEYPAD */
-
 /* PWM Timer */
 
 #ifdef CONFIG_SAMSUNG_DEV_PWM
diff --git a/arch/arm/mach-s3c/devs.h b/arch/arm/mach-s3c/devs.h
index 21c00786c264e6..2737990063b18e 100644
--- a/arch/arm/mach-s3c/devs.h
+++ b/arch/arm/mach-s3c/devs.h
@@ -39,7 +39,6 @@ extern struct platform_device s3c_device_i2c1;
 extern struct platform_device s3c_device_ohci;
 extern struct platform_device s3c_device_usb_hsotg;
 
-extern struct platform_device samsung_device_keypad;
 extern struct platform_device samsung_device_pwm;
 
 /**
diff --git a/arch/arm/mach-s3c/gpio-core.h b/arch/arm/mach-s3c/gpio-core.h
index 6801c85fb9dac4..464b0d02f10269 100644
--- a/arch/arm/mach-s3c/gpio-core.h
+++ b/arch/arm/mach-s3c/gpio-core.h
@@ -14,6 +14,8 @@
 #include "gpio-samsung.h"
 #include <linux/gpio/driver.h>
 
+struct software_node;
+
 #define GPIOCON_OFF	(0x00)
 #define GPIODAT_OFF	(0x04)
 
@@ -66,6 +68,7 @@ struct samsung_gpio_cfg;
  */
 struct samsung_gpio_chip {
 	struct gpio_chip	chip;
+	const struct software_node *swnode;
 	struct samsung_gpio_cfg	*config;
 	struct samsung_gpio_pm	*pm;
 	void __iomem		*base;
diff --git a/arch/arm/mach-s3c/gpio-samsung-s3c64xx.h b/arch/arm/mach-s3c/gpio-samsung-s3c64xx.h
index 8ed144a0d47477..fc21d3f50ce967 100644
--- a/arch/arm/mach-s3c/gpio-samsung-s3c64xx.h
+++ b/arch/arm/mach-s3c/gpio-samsung-s3c64xx.h
@@ -13,6 +13,8 @@
 
 #ifdef CONFIG_GPIO_SAMSUNG
 
+#include <linux/property.h>
+
 /* GPIO bank sizes */
 #define S3C64XX_GPIO_A_NR	(8)
 #define S3C64XX_GPIO_B_NR	(7)
@@ -89,6 +91,9 @@ enum s3c_gpio_number {
 /* define the number of gpios we need to the one after the GPQ() range */
 #define GPIO_BOARD_START (S3C64XX_GPQ(S3C64XX_GPIO_Q_NR) + 1)
 
+extern const struct software_node samsung_gpiochip_nodes[];
+#define SAMSUNG_GPIO_NODE(node)	(&samsung_gpiochip_nodes[(node) - 'A'])
+
 #endif /* GPIO_SAMSUNG */
 #endif /* GPIO_SAMSUNG_S3C64XX_H */
 
diff --git a/arch/arm/mach-s3c/gpio-samsung.c b/arch/arm/mach-s3c/gpio-samsung.c
index 81e198e5a6d369..b35cf62a9157f7 100644
--- a/arch/arm/mach-s3c/gpio-samsung.c
+++ b/arch/arm/mach-s3c/gpio-samsung.c
@@ -37,6 +37,28 @@
 #include "gpio-cfg-helpers.h"
 #include "pm.h"
 
+const struct software_node samsung_gpiochip_nodes[] = {
+	SOFTWARE_NODE("GPA", NULL, NULL),
+	SOFTWARE_NODE("GPB", NULL, NULL),
+	SOFTWARE_NODE("GPC", NULL, NULL),
+	SOFTWARE_NODE("GPD", NULL, NULL),
+	SOFTWARE_NODE("GPE", NULL, NULL),
+	SOFTWARE_NODE("GPF", NULL, NULL),
+	SOFTWARE_NODE("GPG", NULL, NULL),
+	SOFTWARE_NODE("GPH", NULL, NULL),
+	SOFTWARE_NODE("GPI", NULL, NULL),
+	SOFTWARE_NODE("GPJ", NULL, NULL),
+	SOFTWARE_NODE("GPK", NULL, NULL),
+	SOFTWARE_NODE("GPL", NULL, NULL),
+	SOFTWARE_NODE("GPM", NULL, NULL),
+	SOFTWARE_NODE("GPN", NULL, NULL),
+	SOFTWARE_NODE("GPO", NULL, NULL),
+	SOFTWARE_NODE("GPP", NULL, NULL),
+	SOFTWARE_NODE("GPQ", NULL, NULL),
+};
+
+#define NUM_SAMSUNG_GPIOCHIPS ARRAY_SIZE(samsung_gpiochip_nodes)
+
 static int samsung_gpio_setpull_updown(struct samsung_gpio_chip *chip,
 				unsigned int off, samsung_gpio_pull_t pull)
 {
@@ -491,6 +513,16 @@ static __init void s3c_gpiolib_track(struct samsung_gpio_chip *chip)
 }
 #endif /* CONFIG_S3C_GPIO_TRACK */
 
+static void __init samsung_setup_gpiochip_nodes(void)
+{
+	const struct software_node *group[NUM_SAMSUNG_GPIOCHIPS + 1] = { 0 };
+
+	for (unsigned int i = 0; i < NUM_SAMSUNG_GPIOCHIPS; i++)
+		group[i] = &samsung_gpiochip_nodes[i];
+
+	software_node_register_node_group(group);
+}
+
 /*
  * samsung_gpiolib_add() - add the Samsung gpio_chip.
  * @chip: The chip to register
@@ -506,12 +538,16 @@ static void __init samsung_gpiolib_add(struct samsung_gpio_chip *chip)
 	struct gpio_chip *gc = &chip->chip;
 	int ret;
 
+	gc->label = chip->swnode->name;
+
 	BUG_ON(!chip->base);
 	BUG_ON(!gc->label);
 	BUG_ON(!gc->ngpio);
 
 	spin_lock_init(&chip->lock);
 
+	gc->fwnode = software_node_fwnode(chip->swnode);
+
 	if (!gc->direction_input)
 		gc->direction_input = samsung_gpiolib_2bit_input;
 	if (!gc->direction_output)
@@ -659,49 +695,49 @@ static struct samsung_gpio_chip s3c64xx_gpios_4bit[] = {
 		.chip	= {
 			.base	= S3C64XX_GPA(0),
 			.ngpio	= S3C64XX_GPIO_A_NR,
-			.label	= "GPA",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('A'),
 	}, {
 		.chip	= {
 			.base	= S3C64XX_GPB(0),
 			.ngpio	= S3C64XX_GPIO_B_NR,
-			.label	= "GPB",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('B'),
 	}, {
 		.chip	= {
 			.base	= S3C64XX_GPC(0),
 			.ngpio	= S3C64XX_GPIO_C_NR,
-			.label	= "GPC",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('C'),
 	}, {
 		.chip	= {
 			.base	= S3C64XX_GPD(0),
 			.ngpio	= S3C64XX_GPIO_D_NR,
-			.label	= "GPD",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('D'),
 	}, {
 		.config	= &samsung_gpio_cfgs[0],
 		.chip	= {
 			.base	= S3C64XX_GPE(0),
 			.ngpio	= S3C64XX_GPIO_E_NR,
-			.label	= "GPE",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('E'),
 	}, {
 		.base	= S3C64XX_GPG_BASE,
 		.chip	= {
 			.base	= S3C64XX_GPG(0),
 			.ngpio	= S3C64XX_GPIO_G_NR,
-			.label	= "GPG",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('G'),
 	}, {
 		.base	= S3C64XX_GPM_BASE,
 		.config	= &samsung_gpio_cfgs[1],
 		.chip	= {
 			.base	= S3C64XX_GPM(0),
 			.ngpio	= S3C64XX_GPIO_M_NR,
-			.label	= "GPM",
 			.to_irq = s3c64xx_gpiolib_mbank_to_irq,
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('M'),
 	},
 };
 
@@ -711,25 +747,25 @@ static struct samsung_gpio_chip s3c64xx_gpios_4bit2[] = {
 		.chip	= {
 			.base	= S3C64XX_GPH(0),
 			.ngpio	= S3C64XX_GPIO_H_NR,
-			.label	= "GPH",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('H'),
 	}, {
 		.base	= S3C64XX_GPK_BASE + 0x4,
 		.config	= &samsung_gpio_cfgs[0],
 		.chip	= {
 			.base	= S3C64XX_GPK(0),
 			.ngpio	= S3C64XX_GPIO_K_NR,
-			.label	= "GPK",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('K'),
 	}, {
 		.base	= S3C64XX_GPL_BASE + 0x4,
 		.config	= &samsung_gpio_cfgs[1],
 		.chip	= {
 			.base	= S3C64XX_GPL(0),
 			.ngpio	= S3C64XX_GPIO_L_NR,
-			.label	= "GPL",
 			.to_irq = s3c64xx_gpiolib_lbank_to_irq,
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('L'),
 	},
 };
 
@@ -740,43 +776,43 @@ static struct samsung_gpio_chip s3c64xx_gpios_2bit[] = {
 		.chip	= {
 			.base	= S3C64XX_GPF(0),
 			.ngpio	= S3C64XX_GPIO_F_NR,
-			.label	= "GPF",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('F'),
 	}, {
 		.config	= &samsung_gpio_cfgs[7],
 		.chip	= {
 			.base	= S3C64XX_GPI(0),
 			.ngpio	= S3C64XX_GPIO_I_NR,
-			.label	= "GPI",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('I'),
 	}, {
 		.config	= &samsung_gpio_cfgs[7],
 		.chip	= {
 			.base	= S3C64XX_GPJ(0),
 			.ngpio	= S3C64XX_GPIO_J_NR,
-			.label	= "GPJ",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('J'),
 	}, {
 		.config	= &samsung_gpio_cfgs[6],
 		.chip	= {
 			.base	= S3C64XX_GPO(0),
 			.ngpio	= S3C64XX_GPIO_O_NR,
-			.label	= "GPO",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('O'),
 	}, {
 		.config	= &samsung_gpio_cfgs[6],
 		.chip	= {
 			.base	= S3C64XX_GPP(0),
 			.ngpio	= S3C64XX_GPIO_P_NR,
-			.label	= "GPP",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('P'),
 	}, {
 		.config	= &samsung_gpio_cfgs[6],
 		.chip	= {
 			.base	= S3C64XX_GPQ(0),
 			.ngpio	= S3C64XX_GPIO_Q_NR,
-			.label	= "GPQ",
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('Q'),
 	}, {
 		.base	= S3C64XX_GPN_BASE,
 		.irq_base = IRQ_EINT(0),
@@ -784,9 +820,9 @@ static struct samsung_gpio_chip s3c64xx_gpios_2bit[] = {
 		.chip	= {
 			.base	= S3C64XX_GPN(0),
 			.ngpio	= S3C64XX_GPIO_N_NR,
-			.label	= "GPN",
 			.to_irq = samsung_gpiolib_to_irq,
 		},
+		.swnode	= SAMSUNG_GPIO_NODE('N'),
 	},
 };
 
@@ -803,6 +839,8 @@ static __init int samsung_gpiolib_init(void)
 		return 0;
 
 	if (soc_is_s3c64xx()) {
+		samsung_setup_gpiochip_nodes();
+
 		samsung_gpiolib_set_cfg(samsung_gpio_cfgs,
 				ARRAY_SIZE(samsung_gpio_cfgs));
 		samsung_gpiolib_add_2bit_chips(s3c64xx_gpios_2bit,
diff --git a/arch/arm/mach-s3c/keypad.h b/arch/arm/mach-s3c/keypad.h
deleted file mode 100644
index 9754b9a2994533..00000000000000
--- a/arch/arm/mach-s3c/keypad.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Samsung Platform - Keypad platform data definitions
- *
- * Copyright (C) 2010 Samsung Electronics Co.Ltd
- * Author: Joonyoung Shim <jy0922.shim@samsung.com>
- */
-
-#ifndef __PLAT_SAMSUNG_KEYPAD_H
-#define __PLAT_SAMSUNG_KEYPAD_H
-
-#include <linux/input/samsung-keypad.h>
-
-/**
- * samsung_keypad_set_platdata - Set platform data for Samsung Keypad device.
- * @pd: Platform data to register to device.
- *
- * Register the given platform data for use with Samsung Keypad device.
- * The call will copy the platform data, so the board definitions can
- * make the structure itself __initdata.
- */
-extern void samsung_keypad_set_platdata(struct samsung_keypad_platdata *pd);
-
-/* defined by architecture to configure gpio. */
-extern void samsung_keypad_cfg_gpio(unsigned int rows, unsigned int cols);
-
-#endif /* __PLAT_SAMSUNG_KEYPAD_H */
diff --git a/arch/arm/mach-s3c/mach-crag6410.c b/arch/arm/mach-s3c/mach-crag6410.c
index 57176719d8a6d6..2264beef6ddabb 100644
--- a/arch/arm/mach-s3c/mach-crag6410.c
+++ b/arch/arm/mach-s3c/mach-crag6410.c
@@ -15,13 +15,16 @@
 #include <linux/io.h>
 #include <linux/init.h>
 #include <linux/input-event-codes.h>
+#include <linux/input/matrix_keypad.h>
 #include <linux/gpio/legacy.h>
 #include <linux/gpio/machine.h>
+#include <linux/gpio/property.h>
 #include <linux/leds.h>
 #include <linux/delay.h>
 #include <linux/mmc/host.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/fixed.h>
+#include <linux/property.h>
 #include <linux/pwm.h>
 #include <linux/pwm_backlight.h>
 #include <linux/dm9000.h>
@@ -53,7 +56,6 @@
 #include "gpio-cfg.h"
 #include <linux/platform_data/spi-s3c64xx.h>
 
-#include "keypad.h"
 #include "devs.h"
 #include "cpu.h"
 #include <linux/platform_data/i2c-s3c2410.h>
@@ -176,7 +178,7 @@ static struct s3c_fb_platdata crag6410_lcd_pdata = {
 
 /* 2x6 keypad */
 
-static uint32_t crag6410_keymap[] = {
+static const u32 crag6410_keymap[] __initconst = {
 	/* KEY(row, col, keycode) */
 	KEY(0, 0, KEY_VOLUMEUP),
 	KEY(0, 1, KEY_HOME),
@@ -192,43 +194,103 @@ static uint32_t crag6410_keymap[] = {
 	KEY(1, 5, KEY_CAMERA),
 };
 
-static struct matrix_keymap_data crag6410_keymap_data = {
-	.keymap		= crag6410_keymap,
-	.keymap_size	= ARRAY_SIZE(crag6410_keymap),
+static const struct property_entry crag6410_keypad_props[] __initconst = {
+	PROPERTY_ENTRY_U32("keypad,num-columns", 6),
+	PROPERTY_ENTRY_U32("keypad,num-rows", 2),
+	PROPERTY_ENTRY_U32_ARRAY("linux,keymap", crag6410_keymap),
+	{ }
 };
 
-static struct samsung_keypad_platdata crag6410_keypad_data = {
-	.keymap_data	= &crag6410_keymap_data,
-	.rows		= 2,
-	.cols		= 6,
+static const struct resource crag6410_keypad_resources[] __initconst = {
+	[0] = DEFINE_RES_MEM(SAMSUNG_PA_KEYPAD, SZ_32),
+	[1] = DEFINE_RES_IRQ(IRQ_KEYPAD),
 };
 
-static struct gpio_keys_button crag6410_gpio_keys[] = {
-	[0] = {
-		.code	= KEY_SUSPEND,
-		.gpio	= S3C64XX_GPL(10),	/* EINT 18 */
-		.type	= EV_KEY,
-		.wakeup	= 1,
-		.active_low = 1,
-	},
-	[1] = {
-		.code	= SW_FRONT_PROXIMITY,
-		.gpio	= S3C64XX_GPN(11),	/* EINT 11 */
-		.type	= EV_SW,
-	},
+static const struct platform_device_info crag6410_keypad_info __initconst = {
+	.name		= "samsung-keypad",
+	.id		= PLATFORM_DEVID_NONE,
+	.res		= crag6410_keypad_resources,
+	.num_res	= ARRAY_SIZE(crag6410_keypad_resources),
+	.properties	= crag6410_keypad_props,
+};
+
+static void __init crag6410_setup_keypad(void)
+{
+	struct platform_device *pd;
+
+	/* Set all the necessary GPK pins to special-function 3: KP_ROW[x] */
+	s3c_gpio_cfgrange_nopull(S3C64XX_GPK(8), 2, S3C_GPIO_SFN(3));
+
+	/* Set all the necessary GPL pins to special-function 3: KP_COL[x] */
+	s3c_gpio_cfgrange_nopull(S3C64XX_GPL(0), 6, S3C_GPIO_SFN(3));
+
+	pd = platform_device_register_full(&crag6410_keypad_info);
+	if (IS_ERR(pd))
+		pr_err("failed to instantiate keypad device\n");
+}
+
+static const struct software_node crag6410_gpio_keys_node = {
+	.name = "crag6410-gpio-keys",
 };
 
-static struct gpio_keys_platform_data crag6410_gpio_keydata = {
-	.buttons	= crag6410_gpio_keys,
-	.nbuttons	= ARRAY_SIZE(crag6410_gpio_keys),
+static const struct property_entry crag6410_suspend_key_props[] = {
+	PROPERTY_ENTRY_U32("linux,code", KEY_SUSPEND),
+	PROPERTY_ENTRY_GPIO("gpios",
+			    SAMSUNG_GPIO_NODE('L'), 10,	/* EINT 18 */
+			    GPIO_ACTIVE_LOW),
+	PROPERTY_ENTRY_BOOL("wakeup-source"),
+	{ }
 };
 
-static struct platform_device crag6410_gpio_keydev = {
-	.name		= "gpio-keys",
-	.id		= 0,
-	.dev.platform_data = &crag6410_gpio_keydata,
+static const struct software_node crag6410_suspend_key_node = {
+	.parent = &crag6410_gpio_keys_node,
+	.properties = crag6410_suspend_key_props,
+};
+
+static const struct property_entry crag6410_prox_sw_props[] = {
+	PROPERTY_ENTRY_U32("linux,input-type", EV_SW),
+	PROPERTY_ENTRY_U32("linux,code", SW_FRONT_PROXIMITY),
+	PROPERTY_ENTRY_GPIO("gpios",
+			    SAMSUNG_GPIO_NODE('N'), 11,	/* EINT 11 */
+			    GPIO_ACTIVE_HIGH),
+	{ }
+};
+
+static const struct software_node crag6410_prox_sw_node = {
+	.parent = &crag6410_gpio_keys_node,
+	.properties = crag6410_prox_sw_props,
 };
 
+static const struct software_node *crag6410_gpio_keys_swnodes[] = {
+	&crag6410_gpio_keys_node,
+	&crag6410_suspend_key_node,
+	&crag6410_prox_sw_node,
+	NULL
+};
+
+static void __init crag6410_setup_gpio_keys(void)
+{
+	struct platform_device_info keys_info = {
+		.name	= "gpio-keys",
+		.id	= 0,
+	};
+	struct platform_device *pd;
+	int err;
+
+	err = software_node_register_node_group(crag6410_gpio_keys_swnodes);
+	if (err) {
+		pr_err("failed to register gpio-keys software nodes: %d\n", err);
+		return;
+	}
+
+	keys_info.fwnode = software_node_fwnode(&crag6410_gpio_keys_node);
+
+	pd = platform_device_register_full(&keys_info);
+	err = PTR_ERR_OR_ZERO(pd);
+	if (err)
+		pr_err("failed to create gpio-keys device: %d\n", err);
+}
+
 static struct resource crag6410_dm9k_resource[] = {
 	[0] = DEFINE_RES_MEM(S3C64XX_PA_XM0CSN5, 2),
 	[1] = DEFINE_RES_MEM(S3C64XX_PA_XM0CSN5 + (1 << 8), 2),
@@ -361,7 +423,7 @@ static struct platform_device wallvdd_device = {
 	},
 };
 
-static struct platform_device *crag6410_devices[] __initdata = {
+static struct platform_device *crag6410_devs0[] __initdata = {
 	&s3c_device_hsmmc0,
 	&s3c_device_hsmmc2,
 	&s3c_device_i2c0,
@@ -372,8 +434,9 @@ static struct platform_device *crag6410_devices[] __initdata = {
 	&samsung_device_pwm,
 	&s3c64xx_device_iis0,
 	&s3c64xx_device_iis1,
-	&samsung_device_keypad,
-	&crag6410_gpio_keydev,
+};
+
+static struct platform_device *crag6410_devs1[] __initdata = {
 	&crag6410_dm9k_device,
 	&s3c64xx_device_spi0,
 	&crag6410_lcd_powerdev,
@@ -873,16 +936,18 @@ static void __init crag6410_machine_init(void)
 	gpiod_add_lookup_table(&crag_wm1250_ev1_gpiod_table);
 	i2c_register_board_info(1, i2c_devs1, ARRAY_SIZE(i2c_devs1));
 
-	samsung_keypad_set_platdata(&crag6410_keypad_data);
-
 	gpiod_add_lookup_table(&crag_spi0_gpiod_table);
 	s3c64xx_spi0_set_platdata(0, 2);
 
 	pwm_add_table(crag6410_pwm_lookup, ARRAY_SIZE(crag6410_pwm_lookup));
-	platform_add_devices(crag6410_devices, ARRAY_SIZE(crag6410_devices));
+	platform_add_devices(crag6410_devs0, ARRAY_SIZE(crag6410_devs0));
 	platform_device_register_full(&crag6410_mmgpio_devinfo);
 
 	gpiod_add_lookup_table(&crag_leds_table);
+	crag6410_setup_keypad();
+	crag6410_setup_gpio_keys();
+
+	platform_add_devices(crag6410_devs1, ARRAY_SIZE(crag6410_devs1));
 	gpio_led_register_device(-1, &gpio_leds_pdata);
 
 	regulator_has_full_constraints();
diff --git a/arch/arm/mach-s3c/setup-keypad-s3c64xx.c b/arch/arm/mach-s3c/setup-keypad-s3c64xx.c
deleted file mode 100644
index 2e1367878fb1d0..00000000000000
--- a/arch/arm/mach-s3c/setup-keypad-s3c64xx.c
+++ /dev/null
@@ -1,20 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-//
-// Copyright (c) 2010 Samsung Electronics Co., Ltd.
-//		http://www.samsung.com/
-//
-// GPIO configuration for S3C64XX KeyPad device
-
-#include <linux/gpio/consumer.h>
-#include "gpio-cfg.h"
-#include "keypad.h"
-#include "gpio-samsung.h"
-
-void samsung_keypad_cfg_gpio(unsigned int rows, unsigned int cols)
-{
-	/* Set all the necessary GPK pins to special-function 3: KP_ROW[x] */
-	s3c_gpio_cfgrange_nopull(S3C64XX_GPK(8), rows, S3C_GPIO_SFN(3));
-
-	/* Set all the necessary GPL pins to special-function 3: KP_COL[x] */
-	s3c_gpio_cfgrange_nopull(S3C64XX_GPL(0), cols, S3C_GPIO_SFN(3));
-}
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 0cc1bf04686d83..c515faf22eebe9 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -403,7 +403,7 @@ static int __mark_rodata_ro(void *unused)
 void mark_rodata_ro(void)
 {
 	stop_machine(__mark_rodata_ro, NULL, NULL);
-	arm_debug_checkwx();
+	arm_pgtable_checkwx();
 }
 
 #else
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b5a51b0ef9440a..1ec35aada7728b 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -11,7 +11,7 @@ config ARM64
 	select ACPI_MCFG if (ACPI && PCI)
 	select ACPI_SPCR_TABLE if ACPI
 	select ACPI_PPTT if ACPI
-	select ARCH_HAS_DEBUG_WX
+	select ARCH_HAS_CHECK_WX
 	select ARCH_BINFMT_ELF_EXTRA_PHDRS
 	select ARCH_BINFMT_ELF_STATE
 	select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION
@@ -38,6 +38,7 @@ config ARM64
 	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_MEM_ENCRYPT
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
+	select ARCH_SUPPORTS_RMM
 	select ARCH_HAS_NMI_SAFE_THIS_CPU_OPS
 	select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 	select ARCH_HAS_NONLEAF_PMD_YOUNG if ARM64_HAFT
@@ -73,7 +74,7 @@ config ARM64
 	select ARCH_SUPPORTS_HUGETLBFS
 	select ARCH_SUPPORTS_MEMORY_FAILURE
 	select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
-	select ARCH_SUPPORTS_LTO_CLANG if CPU_LITTLE_ENDIAN
+	select ARCH_SUPPORTS_LTO_CLANG
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
 	select ARCH_SUPPORTS_CFI
 	select ARCH_SUPPORTS_ATOMIC_RMW
@@ -81,7 +82,6 @@ config ARM64
 	select ARCH_HAS_PTE_PROTNONE
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
-	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE
 	select ARCH_SUPPORTS_RT
 	select ARCH_SUPPORTS_SCHED_SMT
@@ -217,14 +217,12 @@ config ARM64
 	select HAVE_PERF_EVENTS_NMI if ARM64_PSEUDO_NMI
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
-	select HAVE_PREEMPT_DYNAMIC_KEY
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_FUNCTION_ARG_ACCESS_API
-	select MMU_GATHER_RCU_TABLE_FREE
 	select HAVE_RSEQ
-	select HAVE_RUST if RUSTC_SUPPORTS_ARM64
+	select HAVE_RUST
 	select HAVE_STACKPROTECTOR
 	select HAVE_STATIC_CALL if CFI
 	select HAVE_SYSCALL_TRACEPOINTS
@@ -263,10 +261,6 @@ config ARM64
 	help
 	  ARM 64-bit (AArch64) Linux support.
 
-config RUSTC_SUPPORTS_ARM64
-	def_bool y
-	depends on CPU_LITTLE_ENDIAN
-
 config CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS
 	def_bool CC_IS_CLANG
 	# https://github.com/ClangBuiltLinux/linux/issues/1507
@@ -1427,6 +1421,15 @@ config NVIDIA_OLYMPUS_1027_ERRATUM
 
 	  If unsure, say Y.
 
+config ALTERA_ERRATUM_AGILEX5_2_1_23
+	bool
+	help
+	  The Altera SoCFPGA Agilex5 GIC600 SoC integration has ACE-lite
+	  addressing limited to the first 32bit of physical address space,
+	  so the ITS cannot access memory above 4GB.
+
+	  Selected by ARCH_INTEL_SOCFPGA, as all Agilex5 devices are affected.
+
 config RENESAS_ERRATUM_GEN4GICITS1
 	bool "Renesas R-Car Gen4: GIC600 can not access physical addresses above 4 GiB"
 	default y
@@ -1596,28 +1599,6 @@ config ARM64_LPA2
 	def_bool y
 	depends on ARM64_PA_BITS_52 && !ARM64_64K_PAGES
 
-choice
-	prompt "Endianness"
-	default CPU_LITTLE_ENDIAN
-	help
-	  Select the endianness of data accesses performed by the CPU. Userspace
-	  applications will need to be compiled and linked for the endianness
-	  that is selected here.
-
-config CPU_BIG_ENDIAN
-	bool "Build big-endian kernel"
-	depends on BROKEN
-	help
-	  Say Y if you plan on running a kernel with a big-endian userspace.
-
-config CPU_LITTLE_ENDIAN
-	bool "Build little-endian kernel"
-	help
-	  Say Y if you plan on running a kernel with a little-endian userspace.
-	  This is usually the case for distributions targeting arm64.
-
-endchoice
-
 config NR_CPUS
 	int "Maximum number of CPUs (2-4096)"
 	range 2 4096
@@ -1855,7 +1836,6 @@ config KUSER_HELPERS
 
 config COMPAT_VDSO
 	bool "Enable vDSO for 32-bit applications"
-	depends on !CPU_BIG_ENDIAN
 	depends on (CC_IS_CLANG && LD_IS_LLD) || "$(CROSS_COMPILE_COMPAT)" != ""
 	default y
 	help
@@ -2319,7 +2299,7 @@ menu "ARMv9.6 architectural features"
 config ARM64_LSUI
 	bool "Support Unprivileged Load Store Instructions (LSUI)"
 	default y
-	depends on AS_HAS_LSUI && !CPU_BIG_ENDIAN
+	depends on AS_HAS_LSUI
 	help
 	  The Unprivileged Load Store Instructions (LSUI) provides
 	  variants load/store instructions that access user-space memory
@@ -2524,7 +2504,7 @@ config EFI_STUB
 
 config EFI
 	bool "UEFI runtime support"
-	depends on OF && !CPU_BIG_ENDIAN
+	depends on OF
 	depends on KERNEL_MODE_NEON
 	select ARCH_SUPPORTS_ACPI
 	select LIBFDT
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index d2acfac7300372..c52dc955df5f01 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -37,6 +37,7 @@ config ARCH_APPLE
 	bool "Apple Silicon SoC family"
 	select APPLE_AIC
 	select APPLE_PMGR_PWRSTATE if PM
+	select HAVE_SHARED_GPIOS
 	help
 	  This enables support for Apple's in-house ARM SoC family, such
 	  as the Apple M1.
@@ -153,6 +154,14 @@ config ARCH_EXYNOS
 	help
 	  This enables support for ARMv8 based Samsung Exynos SoC family.
 
+config ARCH_GOOGLE
+	bool "Google-Designed SoC family"
+	help
+	  This enables support for Google Tensor SoCs starting at the
+	  Google Tensor G5 (Laguna) found in Pixel 10 and later phones.
+	  Tensor G1 to G4 are offshoots of the Exynos family and are
+	  enabled via ARCH_EXYNOS.
+
 config ARCH_K3
 	bool "Texas Instruments Inc. K3 multicore SoC architecture"
 	select SOC_TI
@@ -361,6 +370,7 @@ config ARCH_SEATTLE
 
 config ARCH_INTEL_SOCFPGA
 	bool "Intel's SoCFPGA ARMv8 Families"
+	select ALTERA_ERRATUM_AGILEX5_2_1_23
 	help
 	  This enables support for Intel's SoCFPGA ARMv8 families:
 	  Stratix 10 (ex. Altera), Stratix10 Software Virtual Platform,
diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile
index 6b005c8fef7066..04e1a0d12f8f51 100644
--- a/arch/arm64/Makefile
+++ b/arch/arm64/Makefile
@@ -119,20 +119,12 @@ KBUILD_CFLAGS	+= -ffixed-x18
 KBUILD_RUSTFLAGS += -Zfixed-x18
 endif
 
-ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
-KBUILD_CPPFLAGS	+= -mbig-endian
-CHECKFLAGS	+= -D__AARCH64EB__
-# Prefer the baremetal ELF build target, but not all toolchains include
-# it so fall back to the standard linux version if needed.
-KBUILD_LDFLAGS	+= -EB $(call ld-option, -maarch64elfb, -maarch64linuxb -z norelro)
-UTS_MACHINE	:= aarch64_be
-else
 KBUILD_CPPFLAGS	+= -mlittle-endian
 CHECKFLAGS	+= -D__AARCH64EL__
-# Same as above, prefer ELF but fall back to linux target if needed.
+# Prefer the baremetal ELF build target, but not all toolchains include
+# it so fall back to the standard linux version if needed.
 KBUILD_LDFLAGS	+= -EL $(call ld-option, -maarch64elf, -maarch64linux -z norelro)
 UTS_MACHINE	:= aarch64
-endif
 
 ifeq ($(CONFIG_LD_IS_LLD), y)
 KBUILD_LDFLAGS	+= -z norelro
diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile
index fc726b215f125d..f2d7cd6eb5695a 100644
--- a/arch/arm64/boot/dts/Makefile
+++ b/arch/arm64/boot/dts/Makefile
@@ -19,6 +19,7 @@ subdir-y += cavium
 subdir-y += cix
 subdir-y += exynos
 subdir-y += freescale
+subdir-y += google
 subdir-y += hisilicon
 subdir-y += intel
 subdir-y += lg
diff --git a/arch/arm64/boot/dts/actions/s700.dtsi b/arch/arm64/boot/dts/actions/s700.dtsi
index 2c78caebf5152f..3d114671b90159 100644
--- a/arch/arm64/boot/dts/actions/s700.dtsi
+++ b/arch/arm64/boot/dts/actions/s700.dtsi
@@ -66,9 +66,9 @@
 	arm-pmu {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
 	};
 
diff --git a/arch/arm64/boot/dts/actions/s900.dtsi b/arch/arm64/boot/dts/actions/s900.dtsi
index eb35cf78ab73e5..267ac2dd5441ca 100644
--- a/arch/arm64/boot/dts/actions/s900.dtsi
+++ b/arch/arm64/boot/dts/actions/s900.dtsi
@@ -66,9 +66,9 @@
 	arm-pmu {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
-		             <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
 	};
 
diff --git a/arch/arm64/boot/dts/allwinner/Makefile b/arch/arm64/boot/dts/allwinner/Makefile
index aa21f58a4be1f5..d1f768dd6461d5 100644
--- a/arch/arm64/boot/dts/allwinner/Makefile
+++ b/arch/arm64/boot/dts/allwinner/Makefile
@@ -26,6 +26,7 @@ dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h64-remix-mini-pc.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a100-allwinner-perf1.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a133-helperboard.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a133-liontron-h-a133l.dtb
+dtb-$(CONFIG_ARCH_SUNXI) += sun50i-a133-teclast-p80.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-bananapi-m2-plus.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-bananapi-m2-plus-v1.2.dtb
 dtb-$(CONFIG_ARCH_SUNXI) += sun50i-h5-emlid-neutis-n5-devboard.dtb
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a133-teclast-p80.dts b/arch/arm64/boot/dts/allwinner/sun50i-a133-teclast-p80.dts
new file mode 100644
index 00000000000000..47b5f2ac824ae8
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a133-teclast-p80.dts
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2026 Arm Ltd.
+ */
+
+/dts-v1/;
+
+#include "sun50i-a100.dtsi"
+#include "sun50i-a100-cpu-opp.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+
+/{
+	model = "Teclast P80";
+	compatible = "teclast,p80", "allwinner,sun50i-a133";
+	chassis-type = "tablet";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	reg_vcc5v: vcc5v {
+		/* external power input from the USB-C port*/
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-name = "vcc-5v";
+	};
+
+	speaker_amp: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&pio 7 6 GPIO_ACTIVE_HIGH>; /* PH6 */
+		sound-name-prefix = "Speaker Amp";
+	};
+};
+
+&cpu0 {
+	cpu-supply = <&reg_dcdc2>;
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&mmc0 {
+	bus-width = <4>;
+	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
+	disable-wp;
+	vmmc-supply = <&reg_dcdc1>;
+	status = "okay";
+};
+
+&mmc2 {
+	bus-width = <8>;
+	cap-mmc-hw-reset;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	non-removable;
+	vmmc-supply = <&reg_dcdc1>;
+	vqmmc-supply = <&reg_eldo1>;
+	status = "okay";
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&r_i2c0 {
+	status = "okay";
+
+	axp803: pmic@34 {
+		compatible = "x-powers,axp803";
+		reg = <0x34>;
+		interrupt-parent = <&r_intc>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		x-powers,drive-vbus-en;
+	};
+};
+
+#include "axp803.dtsi"
+
+&ac_power_supply {
+	status = "okay";
+};
+
+&battery_power_supply {
+	status = "okay";
+};
+
+&reg_aldo1 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-codec-avcc";
+};
+
+&reg_aldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-dram-1";
+};
+
+&reg_aldo3 {
+	regulator-always-on;
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-usb-pl";
+};
+
+&reg_dcdc1 {
+	regulator-always-on;
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-io-usb-pd-mmc";
+};
+
+&reg_dcdc2 {
+	regulator-always-on;
+	regulator-min-microvolt = <810000>;
+	regulator-max-microvolt = <1200000>;
+	regulator-name = "vdd-cpux";
+};
+
+&reg_dcdc3 {
+	regulator-always-on;
+	regulator-min-microvolt = <900000>;
+	regulator-max-microvolt = <900000>;
+	regulator-name = "vdd-usb-cpus";
+};
+
+&reg_dcdc4 {
+	regulator-always-on;
+	regulator-min-microvolt = <950000>;
+	regulator-max-microvolt = <950000>;
+	regulator-name = "vdd-sys";
+};
+
+&reg_dcdc5 {
+	regulator-always-on;
+	regulator-min-microvolt = <1100000>;
+	regulator-max-microvolt = <1100000>;
+	regulator-name = "vcc-dram";
+};
+
+&reg_dc1sw {
+	regulator-name = "vcc-dc1sw";
+};
+
+&reg_drivevbus {
+	regulator-name = "usb0-vbus";
+	status = "okay";
+};
+
+&reg_dldo1 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-dldo1";
+};
+
+&reg_eldo1 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-eldo1";
+};
+
+&reg_eldo3 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-eldo3";
+};
+
+&reg_fldo1 {
+	regulator-always-on;
+	regulator-min-microvolt = <900000>;
+	regulator-max-microvolt = <900000>;
+	regulator-name = "vdd-cpus-usb";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pb_pins>;
+	status = "okay";
+};
+
+&usb_otg {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&usb_power_supply {
+	status = "okay";
+};
+
+&usbphy {
+	usb0_vbus-supply = <&reg_drivevbus>;
+	usb0_id_det-gpios = <&pio 7 8 GPIO_ACTIVE_HIGH>; /* PH8 */
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
index 0d9cad0c035134..5428b138d0acb4 100644
--- a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
@@ -1,11 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright Altera Corporation (C) 2015. All rights reserved.
+ * Copyright (C) 2025, Altera Corporation
  */
 
 /dts-v1/;
 #include <dt-bindings/reset/altr,rst-mgr-s10.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/stratix10-clock.h>
 
 / {
@@ -74,6 +76,8 @@
 			compatible = "intel,stratix10-svc";
 			method = "smc";
 			memory-region = <&service_reserved>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&intc>;
 
 			fpga_mgr: fpga-mgr {
 				compatible = "intel,stratix10-soc-fpga-mgr";
diff --git a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
index 719e111bc3dd56..8f3f1c0579a0fb 100644
--- a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
+++ b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
@@ -762,8 +762,9 @@
 				interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clkc_periphs CLKID_SYS_SD_EMMC_A>,
 					 <&clkc_periphs CLKID_SD_EMMC_A>,
-					 <&scmi_clk CLKID_FCLK_DIV2>;
-				clock-names = "core", "clkin0", "clkin1";
+					 <&scmi_clk CLKID_FCLK_DIV2>,
+					 <&clkc_periphs CLKID_SYS_AMPIPE_NAND>;
+				clock-names = "core", "clkin0", "clkin1", "pipeline";
 				resets = <&reset RESET_SD_EMMC_A>;
 				assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A_SEL>;
 				assigned-clock-parents = <&xtal>;
@@ -776,8 +777,9 @@
 				interrupts = <GIC_SPI 177 IRQ_TYPE_EDGE_RISING>;
 				clocks = <&clkc_periphs CLKID_SYS_SD_EMMC_B>,
 					 <&clkc_periphs CLKID_SD_EMMC_B>,
-					 <&scmi_clk CLKID_FCLK_DIV2>;
-				clock-names = "core", "clkin0", "clkin1";
+					 <&scmi_clk CLKID_FCLK_DIV2>,
+					 <&clkc_periphs CLKID_SYS_AMPIPE_NAND>;
+				clock-names = "core", "clkin0", "clkin1", "pipeline";
 				resets = <&reset RESET_SD_EMMC_B>;
 				assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B_SEL>;
 				assigned-clock-parents = <&xtal>;
@@ -790,8 +792,9 @@
 				interrupts = <GIC_SPI 178 IRQ_TYPE_EDGE_RISING>;
 				clocks = <&clkc_periphs CLKID_SYS_SD_EMMC_C>,
 					 <&clkc_periphs CLKID_SD_EMMC_C>,
-					 <&scmi_clk CLKID_FCLK_DIV2>;
-				clock-names = "core", "clkin0", "clkin1";
+					 <&scmi_clk CLKID_FCLK_DIV2>,
+					 <&clkc_periphs CLKID_SYS_AMPIPE_NAND>;
+				clock-names = "core", "clkin0", "clkin1", "pipeline";
 				resets = <&reset RESET_SD_EMMC_C>;
 				assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_C_SEL>;
 				assigned-clock-parents = <&xtal>;
diff --git a/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-go-ultra.dts b/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-go-ultra.dts
index cac73c59a94fde..348baa4e6d84e2 100644
--- a/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-go-ultra.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-g12b-odroid-go-ultra.dts
@@ -608,7 +608,7 @@
 	keypad_gpio_pins: keypad-gpio-state {
 		mux {
 			groups = "GPIOX_0", "GPIOX_1", "GPIOX_2", "GPIOX_3",
-			         "GPIOX_4", "GPIOX_5", "GPIOX_6", "GPIOX_7",
+				 "GPIOX_4", "GPIOX_5", "GPIOX_6", "GPIOX_7",
 				 "GPIOX_8", "GPIOX_9", "GPIOX_10", "GPIOX_11",
 				 "GPIOX_12", "GPIOX_13", "GPIOX_14",  "GPIOX_15",
 				 "GPIOX_16", "GPIOX_17", "GPIOX_18",  "GPIOX_19";
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts b/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts
index 5d9ddb814164c3..e353f0ed2b49b0 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts
@@ -70,7 +70,7 @@
 		gpios-states = <0>;
 
 		states = <3300000 0>,
-		         <1800000 1>;
+			 <1800000 1>;
 
 		regulator-settling-time-up-us = <100>;
 		regulator-settling-time-down-us = <5000>;
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
index a9c830a570cc6c..5c2fe9dc4f8075 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
@@ -781,7 +781,7 @@
 		      "dvin", "rdma", "venci", "vencp",
 		      "vdac", "vdi6", "vencl", "vid_lock";
 	clocks = <&clkc CLKID_VPU>,
-	         <&clkc CLKID_VAPB>;
+		 <&clkc CLKID_VAPB>;
 	clock-names = "vpu", "vapb";
 	/*
 	 * VPU clocking is provided by two identical clock paths
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
index d5e0f72892f824..f257fe3860f78d 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
@@ -851,7 +851,7 @@
 		      "dvin", "rdma", "venci", "vencp",
 		      "vdac", "vdi6", "vencl", "vid_lock";
 	clocks = <&clkc CLKID_VPU>,
-	         <&clkc CLKID_VAPB>;
+		 <&clkc CLKID_VAPB>;
 	clock-names = "vpu", "vapb";
 	/*
 	 * VPU clocking is provided by two identical clock paths
diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile
index 9fea43f760ec98..4f31beeb11ac47 100644
--- a/arch/arm64/boot/dts/apple/Makefile
+++ b/arch/arm64/boot/dts/apple/Makefile
@@ -103,3 +103,10 @@ dtb-$(CONFIG_ARCH_APPLE) += t8122-j434.dtb
 dtb-$(CONFIG_ARCH_APPLE) += t8122-j504.dtb
 dtb-$(CONFIG_ARCH_APPLE) += t8122-j613.dtb
 dtb-$(CONFIG_ARCH_APPLE) += t8122-j615.dtb
+dtb-$(CONFIG_ARCH_APPLE) += t8132-j604.dtb
+dtb-$(CONFIG_ARCH_APPLE) += t8132-j623.dtb
+dtb-$(CONFIG_ARCH_APPLE) += t8132-j624.dtb
+dtb-$(CONFIG_ARCH_APPLE) += t8132-j713.dtb
+dtb-$(CONFIG_ARCH_APPLE) += t8132-j715.dtb
+dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb
+dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb
diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi
index 4fb934d7e71fdc..e0acf2ba078b74 100644
--- a/arch/arm64/boot/dts/apple/t6001.dtsi
+++ b/arch/arm64/boot/dts/apple/t6001.dtsi
@@ -35,18 +35,17 @@
 
 #define DIE
 #define DIE_NO 0
+#define SOC_NODE {/soc}
 
-&{/soc} {
-	#include "t600x-die0.dtsi"
-	#include "t600x-dieX.dtsi"
-	#include "t600x-nvme.dtsi"
-};
-
+#include "t600x-die0.dtsi"
+#include "t600x-dieX.dtsi"
+#include "t600x-nvme.dtsi"
 #include "t600x-gpio-pins.dtsi"
 #include "t600x-pmgr.dtsi"
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
 
 
 &aic {
diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi
index 0f81fc612a2a17..91ce9a665cab54 100644
--- a/arch/arm64/boot/dts/apple/t6002.dtsi
+++ b/arch/arm64/boot/dts/apple/t6002.dtsi
@@ -254,30 +254,28 @@
 
 #define DIE
 #define DIE_NO 0
+#define SOC_NODE die0
 
-&die0 {
-	#include "t600x-die0.dtsi"
-	#include "t600x-dieX.dtsi"
-};
-
+#include "t600x-die0.dtsi"
+#include "t600x-dieX.dtsi"
 #include "t600x-pmgr.dtsi"
 #include "t600x-gpio-pins.dtsi"
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
 
 #define DIE _die1
 #define DIE_NO 1
+#define SOC_NODE die1
 
-&die1 {
-	#include "t600x-dieX.dtsi"
-	#include "t600x-nvme.dtsi"
-};
-
+#include "t600x-dieX.dtsi"
+#include "t600x-nvme.dtsi"
 #include "t600x-pmgr.dtsi"
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
 
 &aic {
 	affinities {
diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi
index 49b23f3da98094..122a566e75d14a 100644
--- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi
+++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi
@@ -6,7 +6,7 @@
  * Copyright The Asahi Linux Contributors
  */
 
-
+&SOC_NODE {
 	nco: clock-controller@28e03c000 {
 		compatible = "apple,t6000-nco", "apple,nco";
 		reg = <0x2 0x8e03c000 0x0 0x14000>;
@@ -537,3 +537,4 @@
 			status = "disabled";
 		};
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi
index 9676d5127039b7..7ab60873faf1eb 100644
--- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi
+++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi
@@ -6,6 +6,7 @@
  * Copyright The Asahi Linux Contributors
  */
 
+&SOC_NODE {
 	DIE_NODE(cpufreq_e): cpufreq@210e20000 {
 		compatible = "apple,t6000-cluster-cpufreq", "apple,t8103-cluster-cpufreq", "apple,cluster-cpufreq";
 		reg = <0x2 0x10e20000 0 0x1000>;
@@ -331,3 +332,4 @@
 		mode-switch;
 		power-domains = <&DIE_NODE(ps_atc3_usb)>;
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t600x-nvme.dtsi b/arch/arm64/boot/dts/apple/t600x-nvme.dtsi
index 7dff738d317edc..3a8e2299eac43d 100644
--- a/arch/arm64/boot/dts/apple/t600x-nvme.dtsi
+++ b/arch/arm64/boot/dts/apple/t600x-nvme.dtsi
@@ -5,6 +5,7 @@
  * Copyright The Asahi Linux Contributors
  */
 
+&SOC_NODE {
 	DIE_NODE(ans_mbox): mbox@38f408000 {
 		compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4";
 		reg = <0x3 0x8f408000 0x0 0x4000>;
@@ -40,3 +41,4 @@
 		power-domain-names = "ans", "apcie0", "apcie1";
 		resets = <&DIE_NODE(ps_ans2)>;
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi
index 5175797baf621e..3cddc1697e1470 100644
--- a/arch/arm64/boot/dts/apple/t6021.dtsi
+++ b/arch/arm64/boot/dts/apple/t6021.dtsi
@@ -35,13 +35,11 @@
 
 #define DIE
 #define DIE_NO 0
+#define SOC_NODE {/soc}
 
-&{/soc} {
-	#include "t602x-die0.dtsi"
-	#include "t602x-dieX.dtsi"
-	#include "t602x-nvme.dtsi"
-};
-
+#include "t602x-die0.dtsi"
+#include "t602x-dieX.dtsi"
+#include "t602x-nvme.dtsi"
 #include "t602x-gpio-pins.dtsi"
 #include "t602x-pmgr.dtsi"
 
diff --git a/arch/arm64/boot/dts/apple/t6022.dtsi b/arch/arm64/boot/dts/apple/t6022.dtsi
index 7585fd609d3461..0feda70568ee00 100644
--- a/arch/arm64/boot/dts/apple/t6022.dtsi
+++ b/arch/arm64/boot/dts/apple/t6022.dtsi
@@ -294,32 +294,30 @@
 
 #define DIE
 #define DIE_NO 0
+#define SOC_NODE die0
 
-&die0 {
-	#include "t602x-die0.dtsi"
-	#include "t602x-dieX.dtsi"
-};
-
+#include "t602x-die0.dtsi"
+#include "t602x-dieX.dtsi"
 #include "t602x-pmgr.dtsi"
 #include "t602x-gpio-pins.dtsi"
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
 
 #define DIE _die1
 #define DIE_NO 1
+#define SOC_NODE die1
 
-&die1 {
-	#include "t602x-dieX.dtsi"
-	#include "t602x-nvme.dtsi"
-};
-
+#include "t602x-dieX.dtsi"
+#include "t602x-nvme.dtsi"
 #include "t602x-pmgr.dtsi"
 
 /delete-node/ &ps_pmp_die1;
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
 
 &aic {
 	affinities {
diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi
index 9c800a391e7e87..66b6f8347d19ad 100644
--- a/arch/arm64/boot/dts/apple/t602x-common.dtsi
+++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi
@@ -254,7 +254,7 @@
 			cache-unified;
 			cache-size = <0x1000000>;
 		};
-	 };
+	};
 
 	blizzard_opp: opp-table-0 {
 		compatible = "operating-points-v2";
diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi
index c33623f0e27a41..05819755a18fe2 100644
--- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi
+++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi
@@ -6,6 +6,7 @@
  * Copyright The Asahi Linux Contributors
  */
 
+&SOC_NODE {
 	nco: clock-controller@28e03c000 {
 		compatible = "apple,t6020-nco", "apple,t8103-nco";
 		reg = <0x2 0x8e03c000 0x0 0x14000>;
@@ -590,3 +591,4 @@
 		power-domains = <&ps_apcie_gp_sys>;
 		status = "disabled";
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi
index ae3d535c5acb37..4044c4f21795ff 100644
--- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi
+++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi
@@ -5,6 +5,7 @@
  * Copyright The Asahi Linux Contributors
  */
 
+&SOC_NODE {
 	DIE_NODE(cpufreq_e): cpufreq@210e20000 {
 		compatible = "apple,t6020-cluster-cpufreq", "apple,t8112-cluster-cpufreq";
 		reg = <0x2 0x10e20000 0 0x1000>;
@@ -338,3 +339,4 @@
 		mode-switch;
 		power-domains = <&DIE_NODE(ps_atc3_usb)>;
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi
index 590cec8ac804c0..d1f4267a6cd20b 100644
--- a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi
+++ b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi
@@ -5,6 +5,7 @@
  * Copyright The Asahi Linux Contributors
  */
 
+&SOC_NODE {
 	DIE_NODE(ans_mbox): mbox@347408000 {
 		compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4";
 		reg = <0x3 0x47408000 0x0 0x4000>;
@@ -40,3 +41,4 @@
 		power-domain-names = "ans", "apcie0", "apcie1";
 		resets = <&DIE_NODE(ps_ans2)>;
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi
index 932388ea13c90a..b890d622566ece 100644
--- a/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi
+++ b/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi
@@ -1434,4 +1434,3 @@
 		power-domains = <&ps_atc3_usb_aon>, <&ps_atc3_common>;
 	};
 };
-
diff --git a/arch/arm64/boot/dts/apple/t6030.dtsi b/arch/arm64/boot/dts/apple/t6030.dtsi
index dda9568af11f92..524e532830e209 100644
--- a/arch/arm64/boot/dts/apple/t6030.dtsi
+++ b/arch/arm64/boot/dts/apple/t6030.dtsi
@@ -15,6 +15,13 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	clkref: clock-ref {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "clkref";
+	};
+
 	cpus {
 		#address-cells = <2>;
 		#size-cells = <0>;
@@ -24,18 +31,23 @@
 				core0 {
 					cpu = <&cpu_e00>;
 				};
+
 				core1 {
 					cpu = <&cpu_e01>;
 				};
+
 				core2 {
 					cpu = <&cpu_e02>;
 				};
+
 				core3 {
 					cpu = <&cpu_e03>;
 				};
+
 				core4 {
 					cpu = <&cpu_e04>;
 				};
+
 				core5 {
 					cpu = <&cpu_e05>;
 				};
@@ -45,18 +57,23 @@
 				core0 {
 					cpu = <&cpu_p00>;
 				};
+
 				core1 {
 					cpu = <&cpu_p01>;
 				};
+
 				core2 {
 					cpu = <&cpu_p02>;
 				};
+
 				core3 {
 					cpu = <&cpu_p03>;
 				};
+
 				core4 {
 					cpu = <&cpu_p04>;
 				};
+
 				core5 {
 					cpu = <&cpu_p05>;
 				};
@@ -64,300 +81,302 @@
 		};
 
 		cpu_e00: cpu@0 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x0>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e01: cpu@1 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x1>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e02: cpu@2 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x2>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e03: cpu@3 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x3>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e04: cpu@4 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x4>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e05: cpu@5 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x5>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_p00: cpu@10100 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10100>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p01: cpu@10101 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10101>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p02: cpu@10102 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10102>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p03: cpu@10103 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10103>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p04: cpu@10104 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10104>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p05: cpu@10105 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10105>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		l2_cache_0: l2-cache-0 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x400000>;
+			cache-unified;
 		};
 
 		l2_cache_1: l2-cache-1 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x1000000>;
+			cache-unified;
 		};
-	 };
-
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupt-parent = <&aic>;
-		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
-		interrupts = <AIC_FIQ AIC_TMR_GUEST_PHYS IRQ_TYPE_LEVEL_HIGH>,
-			     <AIC_FIQ AIC_TMR_GUEST_VIRT IRQ_TYPE_LEVEL_HIGH>,
-			     <AIC_FIQ AIC_TMR_HV_PHYS IRQ_TYPE_LEVEL_HIGH>,
-			     <AIC_FIQ AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>;
-	};
-
-	clkref: clock-ref {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <24000000>;
-		clock-output-names = "clkref";
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <2>;
-		ranges;
 	};
 
 	soc: soc {
 		compatible = "simple-bus";
-		#address-cells = <2>;
-		#size-cells = <2>;
-
 		ranges;
-		nonposted-mmio;
+		#address-cells = <2>;
 		/* Required to get >32-bit DMA via DARTs */
 		dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>;
+		nonposted-mmio;
+		#size-cells = <2>;
 
 		i2c1: i2c@289014000 {
 			compatible = "apple,t6030-i2c", "apple,t8103-i2c";
 			reg = <0x2 0x89014000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 890 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c1_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c1>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c2: i2c@289018000 {
 			compatible = "apple,t6030-i2c", "apple,t8103-i2c";
 			reg = <0x2 0x89018000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 891 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c2_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c2>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c3: i2c@28901c000 {
 			compatible = "apple,t6030-i2c", "apple,t8103-i2c";
 			reg = <0x2 0x8901c000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 892 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c3_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c3>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c4: i2c@289020000 {
 			compatible = "apple,t6030-i2c", "apple,t8103-i2c";
 			reg = <0x2 0x89020000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 893 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c4_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c4>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c6: i2c@289028000 {
 			compatible = "apple,t6030-i2c", "apple,t8103-i2c";
 			reg = <0x2 0x89028000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 895 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c6_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c6>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c8: i2c@289030000 {
 			compatible = "apple,t6030-i2c", "apple,t8103-i2c";
 			reg = <0x2 0x89030000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 897 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c8_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c8>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		fpwm0: pwm@289040000 {
 			compatible = "apple,t6030-fpwm", "apple,s5l-fpwm";
 			reg = <0x2 0x89040000 0x0 0x4000>;
-			power-domains = <&ps_fpwm0>;
 			clocks = <&clkref>;
+			power-domains = <&ps_fpwm0>;
 			#pwm-cells = <2>;
+
 			status = "disabled";
 		};
 
 		serial0: serial@289200000 {
 			compatible = "apple,s5l-uart";
 			reg = <0x2 0x89200000 0x0 0x1000>;
-			reg-io-width = <4>;
-			interrupt-parent = <&aic>;
-			interrupts = <AIC_IRQ 866 IRQ_TYPE_LEVEL_HIGH>;
 			/*
 			 * TODO: figure out the clocking properly, there may
 			 * be a third selectable clock.
 			 */
 			clocks = <&clkref>, <&clkref>;
 			clock-names = "uart", "clk_uart_baud0";
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 866 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&ps_uart0>;
+			reg-io-width = <4>;
+
 			status = "disabled";
 		};
 
 		pmgr_gfx: power-management@290e80000 {
-			compatible = "apple,t6030-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+			compatible = "apple,t6030-pmgr", "apple,t8103-pmgr",
+				     "syscon", "simple-mfd";
+			reg = <0x2 0x90e80000 0 0x4000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
-
-			reg = <0x2 0x90e80000 0 0x4000>;
 			/* child nodes are added in t6030-pmgr.dtsi */
 		};
 
@@ -365,6 +384,12 @@
 			compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x3 0x47100000 0x0 0x4000>;
 
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pinctrl_ap 0 0 224>;
+
+			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 273 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 274 IRQ_TYPE_LEVEL_HIGH>,
@@ -375,15 +400,8 @@
 				     <AIC_IRQ 279 IRQ_TYPE_LEVEL_HIGH>;
 
 			power-domains = <&ps_gpio>;
-
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl_ap 0 0 224>;
 			apple,npins = <224>;
 
-			interrupt-controller;
-			#interrupt-cells = <2>;
-
 			i2c1_pins: i2c1-pins {
 				pinmux = <APPLE_PINMUX(37, 1)>,
 					 <APPLE_PINMUX(36, 1)>;
@@ -416,22 +434,22 @@
 		};
 
 		pmgr: power-management@350700000 {
-			compatible = "apple,t6030-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+			compatible = "apple,t6030-pmgr", "apple,t8103-pmgr",
+				     "syscon", "simple-mfd";
+			reg = <0x3 0x50700000 0 0x14000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
-
-			reg = <0x3 0x50700000 0 0x14000>;
 			/* child nodes are added in t6030-pmgr.dtsi */
 		};
 
 		aic: interrupt-controller@351000000 {
 			compatible = "apple,t6030-aic3", "apple,t8122-aic3";
-			#interrupt-cells = <3>;
-			interrupt-controller;
-
 			reg = <0x3 0x51000000 0x0 0x184000>,
 			      <0x3 0x51040000 0x0 0x4000>;
 			reg-names = "core", "event";
+
+			#interrupt-cells = <3>;
+			interrupt-controller;
 			power-domains = <&ps_aic>;
 		};
 
@@ -439,6 +457,12 @@
 			compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x3 0x641f0000 0x0 0x4000>;
 
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pinctrl_nub 0 0 37>;
+
+			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 472 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 473 IRQ_TYPE_LEVEL_HIGH>,
@@ -448,21 +472,15 @@
 				     <AIC_IRQ 477 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 478 IRQ_TYPE_LEVEL_HIGH>;
 
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl_nub 0 0 37>;
 			apple,npins = <37>;
-
-			interrupt-controller;
-			#interrupt-cells = <2>;
 		};
 
 		pmgr_mini: power-management@364280000 {
-			compatible = "apple,t6030-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+			compatible = "apple,t6030-pmgr", "apple,t8103-pmgr",
+				     "syscon", "simple-mfd";
+			reg = <0x3 0x64280000 0 0x4000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
-
-			reg = <0x3 0x64280000 0 0x4000>;
 			/* child nodes are added in t6030-pmgr.dtsi */
 		};
 
@@ -478,13 +496,12 @@
 			compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x3 0x6c820000 0x0 0x4000>;
 
-			gpio-controller;
 			#gpio-cells = <2>;
+			gpio-controller;
 			gpio-ranges = <&pinctrl_smc 0 0 18>;
-			apple,npins = <18>;
 
-			interrupt-controller;
 			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 545 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 546 IRQ_TYPE_LEVEL_HIGH>,
@@ -493,12 +510,22 @@
 				     <AIC_IRQ 549 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 550 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 551 IRQ_TYPE_LEVEL_HIGH>;
+
+			apple,npins = <18>;
 		};
 
 		pinctrl_aop: pinctrl@374824000 {
 			compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x3 0x74824000 0x0 0x4000>;
 
+			clocks = <&clkref>;
+
+			#gpio-cells = <2>;
+			gpio-controller;
+			gpio-ranges = <&pinctrl_aop 0 0 54>;
+
+			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 394 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 395 IRQ_TYPE_LEVEL_HIGH>,
@@ -508,17 +535,19 @@
 				     <AIC_IRQ 399 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 400 IRQ_TYPE_LEVEL_HIGH>;
 
-			clocks = <&clkref>;
-
-			gpio-controller;
-			#gpio-cells = <2>;
-			gpio-ranges = <&pinctrl_aop 0 0 54>;
 			apple,npins = <54>;
-
-			interrupt-controller;
-			#interrupt-cells = <2>;
 		};
 	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupt-parent = <&aic>;
+		interrupts = <AIC_FIQ AIC_TMR_GUEST_PHYS IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_FIQ AIC_TMR_GUEST_VIRT IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_FIQ AIC_TMR_HV_PHYS IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_FIQ AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
+	};
 };
 
 #include "t6030-pmgr.dtsi"
diff --git a/arch/arm64/boot/dts/apple/t6031-base.dtsi b/arch/arm64/boot/dts/apple/t6031-base.dtsi
index 74e1fa51242d4e..f62ce0690182b5 100644
--- a/arch/arm64/boot/dts/apple/t6031-base.dtsi
+++ b/arch/arm64/boot/dts/apple/t6031-base.dtsi
@@ -11,6 +11,13 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	clkref: clock-ref {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "clkref";
+	};
+
 	cpus {
 		#address-cells = <2>;
 		#size-cells = <0>;
@@ -20,12 +27,15 @@
 				core0 {
 					cpu = <&cpu_e00>;
 				};
+
 				core1 {
 					cpu = <&cpu_e01>;
 				};
+
 				core2 {
 					cpu = <&cpu_e02>;
 				};
+
 				core3 {
 					cpu = <&cpu_e03>;
 				};
@@ -35,18 +45,23 @@
 				core0 {
 					cpu = <&cpu_p00>;
 				};
+
 				core1 {
 					cpu = <&cpu_p01>;
 				};
+
 				core2 {
 					cpu = <&cpu_p02>;
 				};
+
 				core3 {
 					cpu = <&cpu_p03>;
 				};
+
 				core4 {
 					cpu = <&cpu_p04>;
 				};
+
 				core5 {
 					cpu = <&cpu_p05>;
 				};
@@ -56,18 +71,23 @@
 				core0 {
 					cpu = <&cpu_p10>;
 				};
+
 				core1 {
 					cpu = <&cpu_p11>;
 				};
+
 				core2 {
 					cpu = <&cpu_p12>;
 				};
+
 				core3 {
 					cpu = <&cpu_p13>;
 				};
+
 				core4 {
 					cpu = <&cpu_p14>;
 				};
+
 				core5 {
 					cpu = <&cpu_p15>;
 				};
@@ -75,223 +95,232 @@
 		};
 
 		cpu_e00: cpu@0 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x0>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e01: cpu@1 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x1>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e02: cpu@2 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x2>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e03: cpu@3 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x3>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_p00: cpu@10100 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10100>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p01: cpu@10101 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10101>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p02: cpu@10102 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10102>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p03: cpu@10103 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10103>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p04: cpu@10104 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10104>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p05: cpu@10105 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10105>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p10: cpu@10200 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10200>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_2>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_2>;
 		};
 
 		cpu_p11: cpu@10201 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10201>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_2>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_2>;
 		};
 
 		cpu_p12: cpu@10202 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10202>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_2>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_2>;
 		};
 
 		cpu_p13: cpu@10203 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10203>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_2>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_2>;
 		};
 
 		cpu_p14: cpu@10204 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10204>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_2>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_2>;
 		};
 
 		cpu_p15: cpu@10205 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10205>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_2>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_2>;
 		};
 
 		l2_cache_0: l2-cache-0 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x400000>;
+			cache-unified;
 		};
 
 		l2_cache_1: l2-cache-1 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x1000000>;
+			cache-unified;
 		};
 
 		l2_cache_2: l2-cache-2 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x1000000>;
+			cache-unified;
 		};
 	};
 
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupt-parent = <&aic>;
-		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
 		interrupts = <AIC_FIQ 0 AIC_TMR_GUEST_PHYS IRQ_TYPE_LEVEL_HIGH>,
 			     <AIC_FIQ 0 AIC_TMR_GUEST_VIRT IRQ_TYPE_LEVEL_HIGH>,
 			     <AIC_FIQ 0 AIC_TMR_HV_PHYS IRQ_TYPE_LEVEL_HIGH>,
 			     <AIC_FIQ 0 AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>;
-	};
-
-	clkref: clock-ref {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <24000000>;
-		clock-output-names = "clkref";
-	};
-
-	reserved-memory {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
+		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
 	};
 };
diff --git a/arch/arm64/boot/dts/apple/t6031-die0.dtsi b/arch/arm64/boot/dts/apple/t6031-die0.dtsi
index 47d08672a04777..b14443436e26cb 100644
--- a/arch/arm64/boot/dts/apple/t6031-die0.dtsi
+++ b/arch/arm64/boot/dts/apple/t6031-die0.dtsi
@@ -6,14 +6,15 @@
  * Copyright The Asahi Linux Contributors
  */
 
+&SOC_NODE {
 	aic: interrupt-controller@292400000 {
 		compatible = "apple,t6031-aic3", "apple,t8122-aic3";
-		#interrupt-cells = <4>;
-		interrupt-controller;
-
 		reg = <0x00000002 0x92400000 0x00000000 0x1cc000>,
 		      <0x00000002 0x92440000 0x00000000 0x4000>;
 		reg-names = "core", "event";
+
+		#interrupt-cells = <4>;
+		interrupt-controller;
 		power-domains = <&ps_aic>;
 	};
 
@@ -29,169 +30,181 @@
 		compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl";
 		reg = <0x2 0xa4820000 0x0 0x4000>;
 
-		gpio-controller;
 		#gpio-cells = <2>;
+		gpio-controller;
 		gpio-ranges = <&pinctrl_smc 0 0 30>;
-		apple,npins = <30>;
 
-		interrupt-controller;
 		#interrupt-cells = <2>;
+		interrupt-controller;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 904 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ 0 905 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ 0 906 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ 0 907 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ 0 908 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ 0 909 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ 0 910 IRQ_TYPE_LEVEL_HIGH>;
+			     <AIC_IRQ 0 905 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ 0 906 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ 0 907 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ 0 908 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ 0 909 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ 0 910 IRQ_TYPE_LEVEL_HIGH>;
+
+		apple,npins = <30>;
 	};
 
 	i2c0: i2c@391010000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x91010000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1326 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c0_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c0>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
 	};
 
 	i2c1: i2c@391014000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x91014000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1327 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c1_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c1>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	i2c2: i2c@391018000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x91018000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1328 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c2_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c2>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	i2c3: i2c@39101c000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x9101c000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1329 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c3_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c3>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	i2c4: i2c@391020000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x91020000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1330 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c4_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c4>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	i2c5: i2c@391024000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x91024000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1331 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c5_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c5>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	i2c6: i2c@391028000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x91028000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1332 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c6_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c6>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	i2c7: i2c@39102c000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x9102c000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1333 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c7_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c7>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	i2c8: i2c@391030000 {
 		compatible = "apple,t6031-i2c", "apple,t8103-i2c";
 		reg = <0x3 0x91030000 0x0 0x4000>;
+		#address-cells = <0x1>;
 		clocks = <&clkref>;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ 0 1334 IRQ_TYPE_LEVEL_HIGH>;
 		pinctrl-0 = <&i2c8_pins>;
 		pinctrl-names = "default";
 		power-domains = <&ps_i2c8>;
-		#address-cells = <0x1>;
 		#size-cells = <0x0>;
+
 		status = "disabled";
 	};
 
 	fpwm0: pwm@391040000 {
 		compatible = "apple,t6031-fpwm", "apple,s5l-fpwm";
 		reg = <0x3 0x91040000 0x0 0x4000>;
-		power-domains = <&ps_fpwm0>;
 		clocks = <&clkref>;
+		power-domains = <&ps_fpwm0>;
 		#pwm-cells = <2>;
+
 		status = "disabled";
 	};
 
 	serial0: serial@391200000 {
 		compatible = "apple,s5l-uart";
 		reg = <0x3 0x91200000 0x0 0x4000>;
-		reg-io-width = <4>;
-		interrupt-parent = <&aic>;
-		interrupts = <AIC_IRQ 0 1303 IRQ_TYPE_LEVEL_HIGH>;
 		/*
 		 * TODO: figure out the clocking properly, there may
 		 * be a third selectable clock.
 		 */
 		clocks = <&clkref>, <&clkref>;
 		clock-names = "uart", "clk_uart_baud0";
+		interrupt-parent = <&aic>;
+		interrupts = <AIC_IRQ 0 1303 IRQ_TYPE_LEVEL_HIGH>;
 		power-domains = <&ps_uart0>;
+		reg-io-width = <4>;
+
 		status = "disabled";
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t6031-dieX.dtsi b/arch/arm64/boot/dts/apple/t6031-dieX.dtsi
index 2e9d3d2e6dea4b..1624651c1d92cb 100644
--- a/arch/arm64/boot/dts/apple/t6031-dieX.dtsi
+++ b/arch/arm64/boot/dts/apple/t6031-dieX.dtsi
@@ -6,102 +6,106 @@
  * Copyright The Asahi Linux Contributors
  */
 
+&SOC_NODE {
 	DIE_NODE(pmgr): power-management@292280000 {
-		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr",
+			     "syscon", "simple-mfd";
+		reg = <0x2 0x92280000 0 0xc000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-
-		reg = <0x2 0x92280000 0 0xc000>;
 	};
 
 	DIE_NODE(pmgr1): power-management@292800000 {
-		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr",
+			     "syscon", "simple-mfd";
+		reg = <0x2 0x92800000 0 0x4000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-
-		reg = <0x2 0x92800000 0 0x4000>;
 	};
 
 	DIE_NODE(pinctrl_nub): pinctrl@2a01f0000 {
 		compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl";
 		reg = <0x2 0xa01f0000 0x0 0x4000>;
-		power-domains = <&DIE_NODE(ps_nub_gpio)>;
 
-		gpio-controller;
 		#gpio-cells = <2>;
+		gpio-controller;
 		gpio-ranges = <&DIE_NODE(pinctrl_nub) 0 0 59>;
-		apple,npins = <59>;
 
-		interrupt-controller;
 		#interrupt-cells = <2>;
+		interrupt-controller;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ DIE_NO 710 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 711 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 712 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 713 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 714 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 715 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 716 IRQ_TYPE_LEVEL_HIGH>;
+			     <AIC_IRQ DIE_NO 711 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 712 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 713 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 714 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 715 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 716 IRQ_TYPE_LEVEL_HIGH>;
+
+		power-domains = <&DIE_NODE(ps_nub_gpio)>;
+		apple,npins = <59>;
 	};
 
 	DIE_NODE(pmgr_mini): power-management@2a0280000 {
-		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr",
+			     "syscon", "simple-mfd";
+		reg = <0x2 0xa0280000 0 0x4000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-
-		reg = <0x2 0xa0280000 0 0x4000>;
 	};
 
 	DIE_NODE(pinctrl_aop): pinctrl@2a8824000 {
 		compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl";
 		reg = <0x2 0xa8824000 0x0 0x4000>;
 
-		gpio-controller;
 		#gpio-cells = <2>;
+		gpio-controller;
 		gpio-ranges = <&DIE_NODE(pinctrl_aop) 0 0 102>;
-		apple,npins = <102>;
 
-		interrupt-controller;
 		#interrupt-cells = <2>;
+		interrupt-controller;
 		interrupt-parent = <&aic>;
 		interrupts = <AIC_IRQ DIE_NO 595 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 596 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 597 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 598 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 599 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 600 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 601 IRQ_TYPE_LEVEL_HIGH>;
+			     <AIC_IRQ DIE_NO 596 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 597 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 598 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 599 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 600 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 601 IRQ_TYPE_LEVEL_HIGH>;
+
+		apple,npins = <102>;
 	};
 
 	DIE_NODE(pinctrl_ap): pinctrl@2b3000000 {
 		compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl";
 		reg = <0x2 0xb3000000 0x0 0x4000>;
 
-		interrupt-parent = <&aic>;
-		interrupts = <AIC_IRQ DIE_NO 458 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 459 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 460 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 461 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 462 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 463 IRQ_TYPE_LEVEL_HIGH>,
-				<AIC_IRQ DIE_NO 464 IRQ_TYPE_LEVEL_HIGH>;
-
 		clocks = <&clkref>;
-		power-domains = <&DIE_NODE(ps_gpio)>;
 
-		gpio-controller;
 		#gpio-cells = <2>;
+		gpio-controller;
 		gpio-ranges = <&DIE_NODE(pinctrl_ap) 0 0 200>;
-		apple,npins = <200>;
 
-		interrupt-controller;
 		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&aic>;
+		interrupts = <AIC_IRQ DIE_NO 458 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 459 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 460 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 461 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 462 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 463 IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_IRQ DIE_NO 464 IRQ_TYPE_LEVEL_HIGH>;
+
+		power-domains = <&DIE_NODE(ps_gpio)>;
+		apple,npins = <200>;
 	};
 
 	DIE_NODE(pmgr_gfx): power-management@408e80000 {
-		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+		compatible = "apple,t6031-pmgr", "apple,t8103-pmgr",
+			     "syscon", "simple-mfd";
+		reg = <0x4 0x8e80000 0 0x4000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-
-		reg = <0x4 0x8e80000 0 0x4000>;
 	};
+};
diff --git a/arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi
index 5e4727e357d218..0c0d09964e6928 100644
--- a/arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi
+++ b/arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi
@@ -8,46 +8,46 @@
 &pinctrl_ap {
 	i2c0_pins: i2c0-pins {
 		pinmux = <APPLE_PINMUX(56, 1)>,
-			<APPLE_PINMUX(57, 1)>;
+			 <APPLE_PINMUX(57, 1)>;
 	};
 
 	i2c1_pins: i2c1-pins {
 		pinmux = <APPLE_PINMUX(58, 1)>,
-			<APPLE_PINMUX(59, 1)>;
+			 <APPLE_PINMUX(59, 1)>;
 	};
 
 	i2c2_pins: i2c2-pins {
 		pinmux = <APPLE_PINMUX(60, 1)>,
-			<APPLE_PINMUX(61, 1)>;
+			 <APPLE_PINMUX(61, 1)>;
 	};
 
 	i2c3_pins: i2c3-pins {
 		pinmux = <APPLE_PINMUX(62, 1)>,
-			<APPLE_PINMUX(63, 1)>;
+			 <APPLE_PINMUX(63, 1)>;
 	};
 
 	i2c4_pins: i2c4-pins {
 		pinmux = <APPLE_PINMUX(64, 1)>,
-			<APPLE_PINMUX(65, 1)>;
+			 <APPLE_PINMUX(65, 1)>;
 	};
 
 	i2c5_pins: i2c5-pins {
 		pinmux = <APPLE_PINMUX(66, 1)>,
-			<APPLE_PINMUX(67, 1)>;
+			 <APPLE_PINMUX(67, 1)>;
 	};
 
 	i2c6_pins: i2c6-pins {
 		pinmux = <APPLE_PINMUX(68, 1)>,
-			<APPLE_PINMUX(69, 1)>;
+			 <APPLE_PINMUX(69, 1)>;
 	};
 
 	i2c7_pins: i2c7-pins {
 		pinmux = <APPLE_PINMUX(70, 1)>,
-			<APPLE_PINMUX(71, 1)>;
+			 <APPLE_PINMUX(71, 1)>;
 	};
 
 	i2c8_pins: i2c8-pins {
 		pinmux = <APPLE_PINMUX(72, 1)>,
-			<APPLE_PINMUX(73, 1)>;
+			 <APPLE_PINMUX(73, 1)>;
 	};
 };
diff --git a/arch/arm64/boot/dts/apple/t6031-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6031-pmgr.dtsi
index f80ee5f7ebcf04..721ab2e9fd4b42 100644
--- a/arch/arm64/boot/dts/apple/t6031-pmgr.dtsi
+++ b/arch/arm64/boot/dts/apple/t6031-pmgr.dtsi
@@ -1383,6 +1383,7 @@
 		#reset-cells = <0>;
 		label = DIE_LABEL(afi_d2d_0);
 		apple,always-on;
+
 		status = "disabled";
 	};
 
@@ -1393,6 +1394,7 @@
 		#reset-cells = <0>;
 		label = DIE_LABEL(afi_d2d_1);
 		apple,always-on;
+
 		status = "disabled";
 	};
 
@@ -1403,6 +1405,7 @@
 		#reset-cells = <0>;
 		label = DIE_LABEL(afc_d2d_0);
 		apple,always-on;
+
 		status = "disabled";
 	};
 
@@ -1413,6 +1416,7 @@
 		#reset-cells = <0>;
 		label = DIE_LABEL(afc_d2d_1);
 		apple,always-on;
+
 		status = "disabled";
 	};
 
@@ -1423,6 +1427,7 @@
 		#reset-cells = <0>;
 		label = DIE_LABEL(afr_d2d_0);
 		apple,always-on;
+
 		status = "disabled";
 	};
 
@@ -1433,6 +1438,7 @@
 		#reset-cells = <0>;
 		label = DIE_LABEL(afr_d2d_1);
 		apple,always-on;
+
 		status = "disabled";
 	};
 
diff --git a/arch/arm64/boot/dts/apple/t6031.dtsi b/arch/arm64/boot/dts/apple/t6031.dtsi
index 0059afe9a84455..a2aa64ea9db9bd 100644
--- a/arch/arm64/boot/dts/apple/t6031.dtsi
+++ b/arch/arm64/boot/dts/apple/t6031.dtsi
@@ -21,13 +21,12 @@
 
 	soc: soc {
 		compatible = "simple-bus";
-		#address-cells = <2>;
-		#size-cells = <2>;
-
 		ranges;
-		nonposted-mmio;
+		#address-cells = <2>;
 		/* Required to get >32-bit DMA via DARTs */
 		dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>;
+		nonposted-mmio;
+		#size-cells = <2>;
 
 		// filled via templated includes at the end of the file
 	};
@@ -35,14 +34,13 @@
 
 #define DIE
 #define DIE_NO 0
+#define SOC_NODE soc
 
-&soc {
-	#include "t6031-die0.dtsi"
-	#include "t6031-dieX.dtsi"
-};
-
+#include "t6031-die0.dtsi"
+#include "t6031-dieX.dtsi"
 #include "t6031-gpio-pins.dtsi"
 #include "t6031-pmgr.dtsi"
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
diff --git a/arch/arm64/boot/dts/apple/t6032-j575d.dts b/arch/arm64/boot/dts/apple/t6032-j575d.dts
index 56edfb1139fbf7..12991546d4bb2a 100644
--- a/arch/arm64/boot/dts/apple/t6032-j575d.dts
+++ b/arch/arm64/boot/dts/apple/t6032-j575d.dts
@@ -20,18 +20,20 @@
 	};
 
 	chosen {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <2>;
-		ranges;
 
 		stdout-path = "serial0";
 
-		framebuffer0: framebuffer@0 {
-			compatible = "apple,simple-framebuffer", "simple-framebuffer";
+		framebuffer@0 {
+			compatible = "apple,simple-framebuffer",
+				     "simple-framebuffer";
 			reg = <0 0 0 0>; /* To be filled by loader */
 			/* Format properties will be added by loader */
-			status = "disabled";
 			power-domains = <&ps_dispext0_cpu>;
+
+			status = "disabled";
 		};
 	};
 
diff --git a/arch/arm64/boot/dts/apple/t6032.dtsi b/arch/arm64/boot/dts/apple/t6032.dtsi
index bbf19ecdaaec45..efe56d2a9a79d4 100644
--- a/arch/arm64/boot/dts/apple/t6032.dtsi
+++ b/arch/arm64/boot/dts/apple/t6032.dtsi
@@ -33,12 +33,15 @@
 				core0 {
 					cpu = <&cpu_e10>;
 				};
+
 				core1 {
 					cpu = <&cpu_e11>;
 				};
+
 				core2 {
 					cpu = <&cpu_e12>;
 				};
+
 				core3 {
 					cpu = <&cpu_e13>;
 				};
@@ -48,18 +51,23 @@
 				core0 {
 					cpu = <&cpu_p20>;
 				};
+
 				core1 {
 					cpu = <&cpu_p21>;
 				};
+
 				core2 {
 					cpu = <&cpu_p22>;
 				};
+
 				core3 {
 					cpu = <&cpu_p23>;
 				};
+
 				core4 {
 					cpu = <&cpu_p24>;
 				};
+
 				core5 {
 					cpu = <&cpu_p25>;
 				};
@@ -69,18 +77,23 @@
 				core0 {
 					cpu = <&cpu_p30>;
 				};
+
 				core1 {
 					cpu = <&cpu_p31>;
 				};
+
 				core2 {
 					cpu = <&cpu_p32>;
 				};
+
 				core3 {
 					cpu = <&cpu_p33>;
 				};
+
 				core4 {
 					cpu = <&cpu_p34>;
 				};
+
 				core5 {
 					cpu = <&cpu_p35>;
 				};
@@ -88,228 +101,244 @@
 		};
 
 		cpu_e10: cpu@800 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x800>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_3>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_3>;
 		};
 
 		cpu_e11: cpu@801 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x801>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_3>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_3>;
 		};
 
 		cpu_e12: cpu@802 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x802>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_3>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_3>;
 		};
 
 		cpu_e13: cpu@803 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x803>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* to be filled by loader */
-			next-level-cache = <&l2_cache_3>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_3>;
 		};
 
 		cpu_p20: cpu@10900 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10900>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_4>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_4>;
 		};
 
 		cpu_p21: cpu@10901 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10901>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_4>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_4>;
 		};
 
 		cpu_p22: cpu@10902 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10902>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_4>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_4>;
 		};
 
 		cpu_p23: cpu@10903 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10903>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_4>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_4>;
 		};
 
 		cpu_p24: cpu@10904 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10904>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_4>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_4>;
 		};
 
 		cpu_p25: cpu@10905 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10905>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_4>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_4>;
 		};
 
 		cpu_p30: cpu@10a00 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10a00>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_5>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_5>;
 		};
 
 		cpu_p31: cpu@10a01 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10a01>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_5>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_5>;
 		};
 
 		cpu_p32: cpu@10a02 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10a02>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_5>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_5>;
 		};
 
 		cpu_p33: cpu@10a03 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10a03>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_5>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_5>;
 		};
 
 		cpu_p34: cpu@10a04 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10a04>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_5>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_5>;
 		};
 
 		cpu_p35: cpu@10a05 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10a05>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_5>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_5>;
 		};
 
 		l2_cache_3: l2-cache-3 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x400000>;
+			cache-unified;
 		};
 
 		l2_cache_4: l2-cache-4 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x1000000>;
+			cache-unified;
 		};
 
 		l2_cache_5: l2-cache-5 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x1000000>;
+			cache-unified;
 		};
-	 };
+	};
 
 	die0: soc@200000000 {
 		compatible = "simple-bus";
-		#address-cells = <2>;
-		#size-cells = <2>;
 		ranges = <0x02 0x00000000 0x02 0x00000000 0x4 0x00000000>,
 			 <0x05 0x80000000 0x05 0x80000000 0x1 0x80000000>,
 			 <0x07 0x00000000 0x07 0x00000000 0xf 0x80000000>,
 			 <0x16 0x80000000 0x16 0x80000000 0x5 0x80000000>;
-		nonposted-mmio;
+		#address-cells = <2>;
 		/* Required to get >32-bit DMA via DARTs */
 		dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>;
+		nonposted-mmio;
+		#size-cells = <2>;
 
 		// filled via templated includes at the end of the file
 	};
 
 	die1: soc@2200000000 {
 		compatible = "simple-bus";
-		#address-cells = <2>;
-		#size-cells = <2>;
 		ranges = <0x02 0x00000000 0x22 0x00000000 0x4 0x00000000>,
 			 <0x07 0x00000000 0x27 0x00000000 0xf 0x80000000>,
 			 <0x16 0x80000000 0x36 0x80000000 0x5 0x80000000>;
-		nonposted-mmio;
+		#address-cells = <2>;
 		/* Required to get >32-bit DMA via DARTs */
 		dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>;
+		nonposted-mmio;
+		#size-cells = <2>;
 
 		// filled via templated includes at the end of the file
 	};
@@ -317,31 +346,29 @@
 
 #define DIE
 #define DIE_NO 0
+#define SOC_NODE die0
 
-&die0 {
-	#include "t6031-die0.dtsi"
-	#include "t6031-dieX.dtsi"
-};
-
+#include "t6031-die0.dtsi"
+#include "t6031-dieX.dtsi"
 #include "t6031-pmgr.dtsi"
 #include "t6031-gpio-pins.dtsi"
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
 
 #define DIE _die1
 #define DIE_NO 1
+#define SOC_NODE die1
 
-&die1 {
-	#include "t6031-dieX.dtsi"
-};
-
+#include "t6031-dieX.dtsi"
 #include "t6031-pmgr.dtsi"
 
 /delete-node/ &ps_pmp_die1;
 
 #undef DIE
 #undef DIE_NO
+#undef SOC_NODE
 
 /* delete non-present DISP power-states */
 /delete-node/ &ps_disp_cpu;
diff --git a/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi b/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi
index 517b84c183ca50..bbee373044f948 100644
--- a/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi
+++ b/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi
@@ -25,18 +25,20 @@
 	};
 
 	chosen {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <2>;
-		ranges;
 
 		stdout-path = "serial0";
 
 		framebuffer0: framebuffer@0 {
-			compatible = "apple,simple-framebuffer", "simple-framebuffer";
+			compatible = "apple,simple-framebuffer",
+				     "simple-framebuffer";
 			reg = <0 0 0 0>; /* To be filled by loader */
 			/* Format properties will be added by loader */
-			status = "disabled";
 			power-domains = <&ps_disp_fe>;
+
+			status = "disabled";
 		};
 	};
 
@@ -47,12 +49,13 @@
 
 	led-controller {
 		compatible = "pwm-leds";
+
 		led-0 {
-			pwms = <&fpwm0 0 40000>;
-			function = LED_FUNCTION_KBD_BACKLIGHT;
 			color = <LED_COLOR_ID_WHITE>;
-			max-brightness = <255>;
 			default-state = "keep";
+			function = LED_FUNCTION_KBD_BACKLIGHT;
+			max-brightness = <255>;
+			pwms = <&fpwm0 0 40000>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/apple/t8122-j504.dts b/arch/arm64/boot/dts/apple/t8122-j504.dts
index 464491b55b019c..e60ec3039196b8 100644
--- a/arch/arm64/boot/dts/apple/t8122-j504.dts
+++ b/arch/arm64/boot/dts/apple/t8122-j504.dts
@@ -20,13 +20,14 @@
 
 	led-controller {
 		compatible = "pwm-leds";
+
 		led-0 {
-			pwms = <&fpwm1 0 40000>;
-			label = "kbd_backlight";
-			function = LED_FUNCTION_KBD_BACKLIGHT;
 			color = <LED_COLOR_ID_WHITE>;
-			max-brightness = <255>;
 			default-state = "keep";
+			function = LED_FUNCTION_KBD_BACKLIGHT;
+			label = "kbd_backlight";
+			max-brightness = <255>;
+			pwms = <&fpwm1 0 40000>;
 		};
 	};
 };
@@ -34,4 +35,3 @@
 &fpwm1 {
 	status = "okay";
 };
-
diff --git a/arch/arm64/boot/dts/apple/t8122-j613.dts b/arch/arm64/boot/dts/apple/t8122-j613.dts
index 51894ea705e765..40274a983849af 100644
--- a/arch/arm64/boot/dts/apple/t8122-j613.dts
+++ b/arch/arm64/boot/dts/apple/t8122-j613.dts
@@ -19,13 +19,14 @@
 
 	led-controller {
 		compatible = "pwm-leds";
+
 		led-0 {
-			pwms = <&fpwm1 0 40000>;
-			label = "kbd_backlight";
-			function = LED_FUNCTION_KBD_BACKLIGHT;
 			color = <LED_COLOR_ID_WHITE>;
-			max-brightness = <255>;
 			default-state = "keep";
+			function = LED_FUNCTION_KBD_BACKLIGHT;
+			label = "kbd_backlight";
+			max-brightness = <255>;
+			pwms = <&fpwm1 0 40000>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/apple/t8122-j615.dts b/arch/arm64/boot/dts/apple/t8122-j615.dts
index 2a1970c1bc90c0..61833f6d019bbb 100644
--- a/arch/arm64/boot/dts/apple/t8122-j615.dts
+++ b/arch/arm64/boot/dts/apple/t8122-j615.dts
@@ -19,13 +19,14 @@
 
 	led-controller {
 		compatible = "pwm-leds";
+
 		led-0 {
-			pwms = <&fpwm1 0 40000>;
-			label = "kbd_backlight";
-			function = LED_FUNCTION_KBD_BACKLIGHT;
 			color = <LED_COLOR_ID_WHITE>;
-			max-brightness = <255>;
 			default-state = "keep";
+			function = LED_FUNCTION_KBD_BACKLIGHT;
+			label = "kbd_backlight";
+			max-brightness = <255>;
+			pwms = <&fpwm1 0 40000>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi
index dd85f0c9fb1e11..2dbdb77d54135a 100644
--- a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi
+++ b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi
@@ -15,9 +15,9 @@
 	};
 
 	chosen {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <2>;
-		ranges;
 
 		stdout-path = "serial0";
 
@@ -26,14 +26,15 @@
 			reg = <0 0 0 0>; /* To be filled by loader */
 			power-domains = <&ps_disp_cpu>, <&ps_dptx_ext_phy>;
 			/* Format properties will be added by loader */
+
 			status = "disabled";
 		};
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <2>;
-		ranges;
 		/* To be filled by loader */
 	};
 
diff --git a/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi
index 64093792e0adeb..3f8be74b5ea830 100644
--- a/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi
+++ b/arch/arm64/boot/dts/apple/t8122-pmgr.dtsi
@@ -1004,7 +1004,6 @@
 };
 
 &pmgr_mini {
-
 	ps_debug_gated: power-controller@0 {
 		compatible = "apple,t8122-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
 		reg = <0x0 4>;
diff --git a/arch/arm64/boot/dts/apple/t8122.dtsi b/arch/arm64/boot/dts/apple/t8122.dtsi
index 1ee61c5b3409f7..52257fa226317a 100644
--- a/arch/arm64/boot/dts/apple/t8122.dtsi
+++ b/arch/arm64/boot/dts/apple/t8122.dtsi
@@ -20,6 +20,13 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	clkref: clock-ref {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "clkref";
+	};
+
 	cpus {
 		#address-cells = <2>;
 		#size-cells = <0>;
@@ -29,12 +36,15 @@
 				core0 {
 					cpu = <&cpu_e0>;
 				};
+
 				core1 {
 					cpu = <&cpu_e1>;
 				};
+
 				core2 {
 					cpu = <&cpu_e2>;
 				};
+
 				core3 {
 					cpu = <&cpu_e3>;
 				};
@@ -44,12 +54,15 @@
 				core0 {
 					cpu = <&cpu_p0>;
 				};
+
 				core1 {
 					cpu = <&cpu_p1>;
 				};
+
 				core2 {
 					cpu = <&cpu_p2>;
 				};
+
 				core3 {
 					cpu = <&cpu_p3>;
 				};
@@ -57,272 +70,238 @@
 		};
 
 		cpu_e0: cpu@0 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x0>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e1: cpu@1 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x1>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e2: cpu@2 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x2>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_e3: cpu@3 {
-			compatible = "apple,sawtooth";
 			device_type = "cpu";
+			compatible = "apple,sawtooth";
 			reg = <0x0 0x3>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_0>;
-			i-cache-size = <0x20000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x10000>;
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
 		};
 
 		cpu_p0: cpu@10100 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10100>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p1: cpu@10101 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10101>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p2: cpu@10102 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10102>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		cpu_p3: cpu@10103 {
-			compatible = "apple,everest";
 			device_type = "cpu";
+			compatible = "apple,everest";
 			reg = <0x0 0x10103>;
-			enable-method = "spin-table";
 			cpu-release-addr = <0 0>; /* To be filled by loader */
-			next-level-cache = <&l2_cache_1>;
-			i-cache-size = <0x30000>;
+			enable-method = "spin-table";
+
 			d-cache-size = <0x20000>;
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
 		};
 
 		l2_cache_0: l2-cache-0 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x400000>;
+			cache-unified;
 		};
 
 		l2_cache_1: l2-cache-1 {
 			compatible = "cache";
 			cache-level = <2>;
-			cache-unified;
 			cache-size = <0x1000000>;
+			cache-unified;
 		};
 	};
 
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupt-parent = <&aic>;
-		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
-		interrupts = <AIC_FIQ AIC_TMR_GUEST_PHYS IRQ_TYPE_LEVEL_HIGH>,
-			     <AIC_FIQ AIC_TMR_GUEST_VIRT IRQ_TYPE_LEVEL_HIGH>,
-			     <AIC_FIQ AIC_TMR_HV_PHYS IRQ_TYPE_LEVEL_HIGH>,
-			     <AIC_FIQ AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>;
-	};
-
-	clkref: clock-ref {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <24000000>;
-		clock-output-names = "clkref";
-	};
-
 	soc {
 		compatible = "simple-bus";
+		ranges;
 		#address-cells = <2>;
-		#size-cells = <2>;
 
-		ranges;
-		nonposted-mmio;
 		/* Required to get >32-bit DMA via DARTs */
 		dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>;
+		nonposted-mmio;
+		#size-cells = <2>;
 
 		i2c0: i2c@2a1010000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
 			reg = <0x2 0xa1010000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 780 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c0_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c0>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c1: i2c@2a1014000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
 			reg = <0x2 0xa1014000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 781 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c1_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c1>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c2: i2c@2a1018000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
 			reg = <0x2 0xa1018000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 782 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c2_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c2>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c3: i2c@2a101c000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
 			reg = <0x2 0xa101c000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 783 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c3_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c3>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		i2c4: i2c@2a1020000 {
 			compatible = "apple,t8122-i2c", "apple,t8103-i2c";
 			reg = <0x2 0xa1020000 0x0 0x4000>;
+			#address-cells = <0x1>;
 			clocks = <&clkref>;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 784 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-0 = <&i2c4_pins>;
 			pinctrl-names = "default";
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
 			power-domains = <&ps_i2c4>;
+			#size-cells = <0x0>;
+
 			status = "disabled";
 		};
 
 		fpwm1: pwm@2a1044000 {
 			compatible = "apple,t8122-fpwm", "apple,s5l-fpwm";
 			reg = <0x2 0xa1044000 0x0 0x4000>;
-			power-domains = <&ps_fpwm1>;
 			clocks = <&clkref>;
+			power-domains = <&ps_fpwm1>;
 			#pwm-cells = <2>;
+
 			status = "disabled";
 		};
 
 		serial0: serial@2a1200000 {
 			compatible = "apple,s5l-uart";
 			reg = <0x2 0xa1200000 0x0 0x1000>;
-			reg-io-width = <4>;
-			interrupt-parent = <&aic>;
-			interrupts = <AIC_IRQ 757 IRQ_TYPE_LEVEL_HIGH>;
 			/*
 			 * TODO: figure out the clocking properly, there may
 			 * be a third selectable clock.
 			 */
 			clocks = <&clkref>, <&clkref>;
 			clock-names = "uart", "clk_uart_baud0";
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 757 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&ps_uart0>;
-			status = "disabled";
-		};
-
-		aic: interrupt-controller@2d1000000 {
-			compatible = "apple,t8122-aic3";
-			#interrupt-cells = <3>;
-			interrupt-controller;
-			reg = <0x2 0xd1000000 0x0 0x184000>,
-				<0x2 0xd1040000 0x0 0x4>;
-			reg-names = "core", "event";
-			power-domains = <&ps_aic>;
-
-			affinities {
-				e-core-pmu-affinity {
-					apple,fiq-index = <AIC_CPU_PMU_E>;
-					cpus = <&cpu_e0 &cpu_e1 &cpu_e2 &cpu_e3>;
-				};
-
-				p-core-pmu-affinity {
-					apple,fiq-index = <AIC_CPU_PMU_P>;
-					cpus = <&cpu_p0 &cpu_p1 &cpu_p2 &cpu_p3>;
-				};
-			};
-		};
+			reg-io-width = <4>;
 
-		pmgr: power-management@2d0700000 {
-			compatible = "apple,t8122-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			reg = <0x2 0xd0700000 0 0x14000>;
-			/* child nodes are added in t8122-pmgr.dtsi */
+			status = "disabled";
 		};
 
 		pinctrl_ap: pinctrl@2c7100000 {
 			compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x2 0xc7100000 0x0 0x100000>;
-			power-domains = <&ps_gpio>;
 
-			gpio-controller;
 			#gpio-cells = <2>;
+			gpio-controller;
 			gpio-ranges = <&pinctrl_ap 0 0 224>;
-			apple,npins = <224>;
 
-			interrupt-controller;
 			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 241 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 242 IRQ_TYPE_LEVEL_HIGH>,
@@ -332,6 +311,9 @@
 				     <AIC_IRQ 246 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 247 IRQ_TYPE_LEVEL_HIGH>;
 
+			power-domains = <&ps_gpio>;
+			apple,npins = <224>;
+
 			i2c0_pins: i2c0-pins {
 				pinmux = <APPLE_PINMUX(145, 1)>,
 					 <APPLE_PINMUX(144, 1)>;
@@ -356,21 +338,49 @@
 				pinmux = <APPLE_PINMUX(149, 1)>,
 					 <APPLE_PINMUX(148, 1)>;
 			};
+		};
 
+		pmgr: power-management@2d0700000 {
+			compatible = "apple,t8122-pmgr", "apple,t8103-pmgr",
+				     "syscon", "simple-mfd";
+			reg = <0x2 0xd0700000 0 0x14000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			/* child nodes are added in t8122-pmgr.dtsi */
+		};
+
+		aic: interrupt-controller@2d1000000 {
+			compatible = "apple,t8122-aic3";
+			reg = <0x2 0xd1000000 0x0 0x184000>,
+			      <0x2 0xd1040000 0x0 0x4>;
+			reg-names = "core", "event";
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			power-domains = <&ps_aic>;
+
+			affinities {
+				e-core-pmu-affinity {
+					cpus = <&cpu_e0 &cpu_e1 &cpu_e2 &cpu_e3>;
+					apple,fiq-index = <AIC_CPU_PMU_E>;
+				};
+
+				p-core-pmu-affinity {
+					cpus = <&cpu_p0 &cpu_p1 &cpu_p2 &cpu_p3>;
+					apple,fiq-index = <AIC_CPU_PMU_P>;
+				};
+			};
 		};
 
 		pinctrl_nub: pinctrl@2e41f0000 {
 			compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x2 0xe41f0000 0x0 0x4000>;
-			power-domains = <&ps_nub_gpio>;
 
-			gpio-controller;
 			#gpio-cells = <2>;
+			gpio-controller;
 			gpio-ranges = <&pinctrl_nub 0 0 32>;
-			apple,npins = <32>;
 
-			interrupt-controller;
 			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 424 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 425 IRQ_TYPE_LEVEL_HIGH>,
@@ -379,13 +389,17 @@
 				     <AIC_IRQ 428 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 429 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 430 IRQ_TYPE_LEVEL_HIGH>;
+
+			power-domains = <&ps_nub_gpio>;
+			apple,npins = <32>;
 		};
 
 		pmgr_mini: power-management@2e4280000 {
-			compatible = "apple,t8122-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+			compatible = "apple,t8122-pmgr", "apple,t8103-pmgr",
+				     "syscon", "simple-mfd";
+			reg = <0x2 0xe4280000 0 0x4000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
-			reg = <0x2 0xe4280000 0 0x4000>;
 			/* child nodes are added in t8122-pmgr.dtsi */
 		};
 
@@ -401,13 +415,12 @@
 			compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x2 0xec820000 0x0 0x4000>;
 
-			gpio-controller;
 			#gpio-cells = <2>;
+			gpio-controller;
 			gpio-ranges = <&pinctrl_smc 0 0 18>;
-			apple,npins = <18>;
 
-			interrupt-controller;
 			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 493 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 494 IRQ_TYPE_LEVEL_HIGH>,
@@ -416,19 +429,20 @@
 				     <AIC_IRQ 497 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 498 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 499 IRQ_TYPE_LEVEL_HIGH>;
+
+			apple,npins = <18>;
 		};
 
 		pinctrl_aop: pinctrl@2f4824000 {
 			compatible = "apple,t8122-pinctrl", "apple,t8103-pinctrl";
 			reg = <0x2 0xf4824000 0x0 0x4000>;
 
-			gpio-controller;
 			#gpio-cells = <2>;
+			gpio-controller;
 			gpio-ranges = <&pinctrl_aop 0 0 54>;
-			apple,npins = <54>;
 
-			interrupt-controller;
 			#interrupt-cells = <2>;
+			interrupt-controller;
 			interrupt-parent = <&aic>;
 			interrupts = <AIC_IRQ 346 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 347 IRQ_TYPE_LEVEL_HIGH>,
@@ -437,8 +451,20 @@
 				     <AIC_IRQ 350 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 351 IRQ_TYPE_LEVEL_HIGH>,
 				     <AIC_IRQ 352 IRQ_TYPE_LEVEL_HIGH>;
+
+			apple,npins = <54>;
 		};
 	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupt-parent = <&aic>;
+		interrupts = <AIC_FIQ AIC_TMR_GUEST_PHYS IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_FIQ AIC_TMR_GUEST_VIRT IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_FIQ AIC_TMR_HV_PHYS IRQ_TYPE_LEVEL_HIGH>,
+			     <AIC_FIQ AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
+	};
 };
 
 #include "t8122-pmgr.dtsi"
diff --git a/arch/arm64/boot/dts/apple/t8132-j604.dts b/arch/arm64/boot/dts/apple/t8132-j604.dts
new file mode 100644
index 00000000000000..c4be915cb33ab2
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-j604.dts
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple MacBook Pro (14-inch, M4, 2024)
+ *
+ * target-type: J604
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/dts-v1/;
+
+#include "t8132.dtsi"
+#include "t8132-jxxx.dtsi"
+#include <dt-bindings/leds/common.h>
+
+/ {
+	compatible = "apple,j604", "apple,t8132", "apple,arm-platform";
+	model = "Apple MacBook Pro (14-inch, M4, 2024)";
+
+	led-controller {
+		compatible = "pwm-leds";
+		led-0 {
+			pwms = <&fpwm1 0 40000>;
+			label = "kbd_backlight";
+			function = LED_FUNCTION_KBD_BACKLIGHT;
+			color = <LED_COLOR_ID_WHITE>;
+			max-brightness = <255>;
+			default-state = "keep";
+		};
+	};
+};
+
+&fpwm1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/apple/t8132-j623.dts b/arch/arm64/boot/dts/apple/t8132-j623.dts
new file mode 100644
index 00000000000000..508e26056022c1
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-j623.dts
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple iMac (24-inch, 2x USB-C, M4, 2024)
+ *
+ * target-type: J623
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/dts-v1/;
+
+#include "t8132.dtsi"
+#include "t8132-jxxx.dtsi"
+
+/ {
+	compatible = "apple,j623", "apple,t8132", "apple,arm-platform";
+	model = "Apple iMac (24-inch, 2x USB-C, M4, 2024)";
+};
diff --git a/arch/arm64/boot/dts/apple/t8132-j624.dts b/arch/arm64/boot/dts/apple/t8132-j624.dts
new file mode 100644
index 00000000000000..84bdd8e343eb16
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-j624.dts
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple iMac (24-inch, 4x USB-C, M4, 2024)
+ *
+ * target-type: J624
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/dts-v1/;
+
+#include "t8132.dtsi"
+#include "t8132-jxxx.dtsi"
+
+/ {
+	compatible = "apple,j624", "apple,t8132", "apple,arm-platform";
+	model = "Apple iMac (24-inch, 4x USB-C, M4, 2024)";
+};
diff --git a/arch/arm64/boot/dts/apple/t8132-j713.dts b/arch/arm64/boot/dts/apple/t8132-j713.dts
new file mode 100644
index 00000000000000..e1fc5d47f48766
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-j713.dts
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple MacBook Air (13-inch, M4, 2025)
+ *
+ * target-type: J713
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/dts-v1/;
+
+#include "t8132.dtsi"
+#include "t8132-jxxx.dtsi"
+#include <dt-bindings/leds/common.h>
+
+/ {
+	compatible = "apple,j713", "apple,t8132", "apple,arm-platform";
+	model = "Apple MacBook Air (13-inch, M4, 2025)";
+
+	led-controller {
+		compatible = "pwm-leds";
+		led-0 {
+			pwms = <&fpwm1 0 40000>;
+			label = "kbd_backlight";
+			function = LED_FUNCTION_KBD_BACKLIGHT;
+			color = <LED_COLOR_ID_WHITE>;
+			max-brightness = <255>;
+			default-state = "keep";
+		};
+	};
+};
+
+&fpwm1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/apple/t8132-j715.dts b/arch/arm64/boot/dts/apple/t8132-j715.dts
new file mode 100644
index 00000000000000..2672914ea76c96
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-j715.dts
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple MacBook Air (15-inch, M4, 2025)
+ *
+ * target-type: J715
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/dts-v1/;
+
+#include "t8132.dtsi"
+#include "t8132-jxxx.dtsi"
+#include <dt-bindings/leds/common.h>
+
+/ {
+	compatible = "apple,j715", "apple,t8132", "apple,arm-platform";
+	model = "Apple MacBook Air (15-inch, M4, 2025)";
+
+	led-controller {
+		compatible = "pwm-leds";
+		led-0 {
+			pwms = <&fpwm1 0 40000>;
+			label = "kbd_backlight";
+			function = LED_FUNCTION_KBD_BACKLIGHT;
+			color = <LED_COLOR_ID_WHITE>;
+			max-brightness = <255>;
+			default-state = "keep";
+		};
+	};
+};
+
+&fpwm1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/apple/t8132-j773g.dts b/arch/arm64/boot/dts/apple/t8132-j773g.dts
new file mode 100644
index 00000000000000..0583cb9d19f627
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-j773g.dts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple Mac mini (M4, 2024)
+ *
+ * target-type: J773g
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/dts-v1/;
+
+#include "t8132.dtsi"
+#include "t8132-jxxx.dtsi"
+
+/ {
+	compatible = "apple,j773g", "apple,t8132", "apple,arm-platform";
+	model = "Apple Mac mini (M4, 2024)";
+};
+
+/*
+ * Keep the power-domains used for the HDMI port on.
+ */
+&framebuffer0 {
+	power-domains = <&ps_dispext0_fe>, <&ps_dptx_phy>;
+};
diff --git a/arch/arm64/boot/dts/apple/t8132-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8132-jxxx.dtsi
new file mode 100644
index 00000000000000..9882ea3f85e612
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-jxxx.dtsi
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple M4 Mac mini, MacBook Air/Pro, iMac 24" (M4, 2024/2025)
+ *
+ * This file contains parts common to all Apple M4 devices using the t8132.
+ *
+ * target-type: J604, J623, J624, J713, J715, J773g
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/ {
+	aliases {
+		serial0 = &serial0;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		stdout-path = "serial0";
+
+		framebuffer0: framebuffer@0 {
+			compatible = "apple,simple-framebuffer", "simple-framebuffer";
+			reg = <0 0 0 0>; /* To be filled by loader */
+			power-domains = <&ps_disp_fe>, <&ps_dptx_phy>;
+			/* Format properties will be added by loader */
+			status = "disabled";
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		/* To be filled by loader */
+	};
+
+	memory@800000000 {
+		device_type = "memory";
+		reg = <0x8 0 0x2 0>; /* To be filled by loader */
+	};
+};
+
+&serial0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi
new file mode 100644
index 00000000000000..6fbf09e8f9810d
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132-pmgr.dtsi
@@ -0,0 +1,1130 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * PMGR Power domains for the Apple T8132 "M4" SoC
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+&pmgr {
+	ps_msg: power-controller@108 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x108 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "msg";
+	};
+
+	ps_aic: power-controller@110 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x110 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "aic";
+		apple,always-on; /* Core device */
+	};
+
+	ps_dwi: power-controller@118 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x118 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dwi";
+	};
+
+	ps_gpio: power-controller@120 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x120 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "gpio";
+	};
+
+	ps_pmc: power-controller@138 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x138 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pmc";
+	};
+
+	ps_pms_fpwm0: power-controller@140 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x140 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pms_fpwm0";
+	};
+
+	ps_pms_fpwm1: power-controller@148 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x148 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pms_fpwm1";
+	};
+
+	ps_pms_fpwm2: power-controller@150 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x150 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pms_fpwm2";
+	};
+
+	ps_pms_fpwm3: power-controller@158 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x158 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pms_fpwm3";
+	};
+
+	ps_pms_fpwm4: power-controller@160 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x160 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pms_fpwm4";
+	};
+
+	ps_pms_c1ppt: power-controller@168 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x168 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pms_c1ppt";
+	};
+
+	ps_soc_rc: power-controller@170 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x170 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "soc_rc";
+	};
+
+	ps_soc_dpe: power-controller@178 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x178 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "soc_dpe";
+		apple,always-on;
+	};
+
+	ps_pmgr_soc_ocla: power-controller@180 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x180 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pmgr_soc_ocla";
+	};
+
+	ps_ap_tmm: power-controller@1b8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x1b8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "ap_tmm";
+	};
+
+	ps_dispext0_sys: power-controller@1d8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x1d8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dispext0_sys";
+	};
+
+	ps_dispext0_fe: power-controller@1e0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x1e0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dispext0_fe";
+		power-domains = <&ps_dispext0_sys>;
+	};
+
+	ps_dispext0_cpu: power-controller@1e8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x1e8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dispext0_cpu";
+		power-domains = <&ps_dispext0_fe>;
+
+		status = "disabled"; /* Locked, don't touch it */
+	};
+
+	ps_scodec: power-controller@228 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x228 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "scodec";
+	};
+
+	ps_scodec_streaming: power-controller@230 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x230 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "scodec_streaming";
+		power-domains = <&ps_scodec>;
+	};
+
+	ps_disp_sys: power-controller@240 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x240 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "disp_sys";
+		apple,always-on; /* TODO: figure out if we can enable PM here */
+	};
+
+	ps_disp_fe: power-controller@248 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x248 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "disp_fe";
+		power-domains = <&ps_disp_sys>;
+		apple,always-on; /* TODO: figure out if we can enable PM here */
+	};
+
+	ps_gfx: power-controller@250 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x250 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "gfx";
+	};
+
+	ps_sio_cpu: power-controller@268 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x268 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "sio_cpu";
+	};
+
+	ps_fpwm0: power-controller@270 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x270 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "fpwm0";
+	};
+
+	ps_fpwm1: power-controller@278 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x278 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "fpwm1";
+	};
+
+	ps_fpwm2: power-controller@280 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x280 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "fpwm2";
+	};
+
+	ps_i2c0: power-controller@288 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x288 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c0";
+	};
+
+	ps_i2c1: power-controller@290 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x290 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c1";
+	};
+
+	ps_i2c2: power-controller@298 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x298 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c2";
+	};
+
+	ps_i2c3: power-controller@2a0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2a0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c3";
+	};
+
+	ps_i2c4: power-controller@2a8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2a8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c4";
+	};
+
+	ps_i2c5: power-controller@2b0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2b0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c5";
+	};
+
+	ps_i2c6: power-controller@2b8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2b8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c6";
+	};
+
+	ps_i2c7: power-controller@2c0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2c0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c7";
+	};
+
+	ps_i2c8: power-controller@2c8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2c8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "i2c8";
+	};
+
+	ps_spi_p: power-controller@2d0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2d0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "spi_p";
+		apple,always-on;
+	};
+
+	ps_uart_p: power-controller@2d8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2d8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart_p";
+		apple,always-on;
+	};
+
+	ps_dpa_p: power-controller@2e0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2e0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dpa_p";
+		apple,always-on;
+	};
+
+	ps_aes: power-controller@2e8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2e8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "aes";
+	};
+
+	ps_spi0: power-controller@2f0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2f0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "spi0";
+		power-domains = <&ps_spi_p>;
+	};
+
+	ps_afiaft: power-controller@300 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x300 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "afiaft";
+	};
+
+	ps_venc_sys: power-controller@308 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x308 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "venc_sys";
+	};
+
+	ps_spi1: power-controller@370 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x370 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "spi1";
+		power-domains = <&ps_spi_p>;
+	};
+
+	ps_atc0_common: power-controller@378 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x378 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc0_common";
+	};
+
+	ps_spi2: power-controller@380 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x380 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "spi2";
+		power-domains = <&ps_spi_p>;
+	};
+
+	ps_spi3: power-controller@388 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x388 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "spi3";
+		power-domains = <&ps_spi_p>;
+	};
+
+	ps_spi4: power-controller@390 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x390 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "spi4";
+		power-domains = <&ps_spi_p>;
+	};
+
+	ps_trace_fab: power-controller@398 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x398 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "trace_fab";
+	};
+
+	ps_atc0_pcie: power-controller@3a0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3a0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc0_pcie";
+		power-domains = <&ps_atc0_common>;
+	};
+
+	ps_atc0_cio: power-controller@3a8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3a8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc0_cio";
+		power-domains = <&ps_atc0_common>;
+	};
+
+	ps_atc0_cio_pcie: power-controller@3b0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3b0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc0_cio_pcie";
+		power-domains = <&ps_atc0_cio>;
+	};
+
+	ps_spi5: power-controller@3b8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3b8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "spi5";
+		power-domains = <&ps_spi_p>;
+	};
+
+	ps_qspi: power-controller@3c0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3c0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "qspi";
+		power-domains = <&ps_spi_p>;
+	};
+
+	ps_dptx_phy: power-controller@3c8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3c8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dptx_phy";
+	};
+
+	ps_uart_n: power-controller@3d0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3d0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart_n";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_uart0: power-controller@3d8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3d8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart0";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_uart1: power-controller@3e0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3e0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart1";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_uart2: power-controller@3e8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3e8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart2";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_uart3: power-controller@3f0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3f0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart3";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_uart4: power-controller@3f8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3f8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart4";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_uart5: power-controller@400 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x400 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart5";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_uart6: power-controller@408 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x408 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "uart6";
+		power-domains = <&ps_uart_p>;
+	};
+
+	ps_dpa0: power-controller@410 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x410 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dpa0";
+		power-domains = <&ps_dpa_p>;
+	};
+
+	ps_dpa1: power-controller@418 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x418 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dpa1";
+		power-domains = <&ps_dpa_p>;
+	};
+
+	ps_atc0_cio_usb: power-controller@420 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x420 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc0_cio_usb";
+		power-domains = <&ps_atc0_cio>;
+	};
+
+	ps_atc1_common: power-controller@428 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x428 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc1_common";
+	};
+
+	ps_atc1_pcie: power-controller@430 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x430 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc1_pcie";
+		power-domains = <&ps_atc1_common>;
+	};
+
+	ps_atc1_cio: power-controller@438 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x438 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc1_cio";
+		power-domains = <&ps_atc1_common>;
+	};
+
+	ps_atc1_cio_pcie: power-controller@440 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x440 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc1_cio_pcie";
+		power-domains = <&ps_atc1_cio>;
+	};
+
+	ps_atc1_cio_usb: power-controller@448 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x448 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc1_cio_usb";
+		power-domains = <&ps_atc1_cio>;
+	};
+
+	ps_atc2_common: power-controller@450 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x450 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc2_common";
+	};
+
+	ps_atc2_pcie: power-controller@458 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x458 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc2_pcie";
+		power-domains = <&ps_atc2_common>;
+	};
+
+	ps_atc2_cio: power-controller@460 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x460 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc2_cio";
+		power-domains = <&ps_atc2_common>;
+	};
+
+	ps_atc2_cio_pcie: power-controller@468 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x468 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc2_cio_pcie";
+		power-domains = <&ps_atc2_cio>;
+	};
+
+	ps_dpa2: power-controller@470 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x470 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dpa2";
+		power-domains = <&ps_dpa_p>;
+	};
+
+	ps_atc2_cio_usb: power-controller@478 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x478 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc2_cio_usb";
+		power-domains = <&ps_atc2_cio>;
+	};
+
+	ps_atc3_common: power-controller@480 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x480 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc3_common";
+	};
+
+	ps_pmp: power-controller@488 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x488 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pmp";
+		apple,always-on;
+	};
+
+	ps_pms_sram: power-controller@490 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x490 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "pms_sram";
+		apple,always-on;
+	};
+
+	ps_atc3_pcie: power-controller@498 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x498 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc3_pcie";
+		power-domains = <&ps_atc3_common>;
+	};
+
+	ps_atc3_cio: power-controller@4a0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4a0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc3_cio";
+		power-domains = <&ps_atc3_common>;
+	};
+
+	ps_dpa3: power-controller@4a8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4a8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dpa3";
+		power-domains = <&ps_dpa_p>;
+	};
+
+	ps_atc3_cio_pcie: power-controller@4b0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4b0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc3_cio_pcie";
+		power-domains = <&ps_atc3_cio>;
+	};
+
+	ps_atc3_cio_usb: power-controller@4b8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4b8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc3_cio_usb";
+		power-domains = <&ps_atc3_cio>;
+	};
+
+	ps_dpa4: power-controller@4c0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4c0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dpa4";
+		power-domains = <&ps_dpa_p>;
+	};
+
+	ps_apcie_gp: power-controller@4d0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4d0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "apcie_gp";
+	};
+
+	ps_apcie_sys_gp: power-controller@4d8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4d8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "apcie_sys_gp";
+		power-domains = <&ps_apcie_gp>;
+	};
+
+	ps_dispext1_sys: power-controller@4e0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4e0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dispext1_sys";
+	};
+
+	ps_dispext1_fe: power-controller@4e8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4e8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dispext1_fe";
+		power-domains = <&ps_dispext1_sys>;
+	};
+
+	ps_dispext1_cpu: power-controller@4f0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x4f0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "dispext1_cpu";
+		power-domains = <&ps_dispext1_fe>;
+
+		status = "disabled"; /* Locked, don't touch it */
+	};
+
+	ps_ans: power-controller@538 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x538 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "ans";
+		apple,always-on; /* Device reboots after 3-5 mintues if disabled */
+	};
+
+	ps_apcie_st: power-controller@540 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x540 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "apcie_st";
+		power-domains = <&ps_ans>;
+	};
+
+	ps_apcie_sys_st: power-controller@548 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x548 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "apcie_sys_st";
+		power-domains = <&ps_ans>, <&ps_apcie_st>;
+	};
+
+	ps_apcie_phy_sw: power-controller@550 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x550 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "apcie_phy_sw";
+		power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>;
+	};
+
+	ps_msr: power-controller@560 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x560 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "msr";
+	};
+
+	ps_msr_ase_core: power-controller@568 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x568 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "msr_ase_core";
+		power-domains = <&ps_msr>;
+	};
+
+	ps_ane_sys: power-controller@570 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x570 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "ane_sys";
+	};
+
+	ps_jpg: power-controller@578 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x578 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "jpg";
+	};
+
+	ps_avd_sys: power-controller@580 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x580 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "avd_sys";
+	};
+
+	ps_sep: power-controller@c00 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc00 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "sep";
+		apple,always-on;
+	};
+
+	ps_venc_dma: power-controller@8000 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8000 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "venc_dma";
+		power-domains = <&ps_venc_sys>;
+	};
+
+	ps_venc_pipe4: power-controller@8008 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8008 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "venc_pipe4";
+		power-domains = <&ps_venc_dma>;
+	};
+
+	ps_venc_pipe5: power-controller@8010 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8010 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "venc_pipe5";
+		power-domains = <&ps_venc_dma>;
+	};
+
+	ps_venc_me0: power-controller@8018 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8018 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "venc_me0";
+		power-domains = <&ps_venc_dma>;
+	};
+
+	ps_venc_me1: power-controller@8020 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8020 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "venc_me1";
+		power-domains = <&ps_venc_me0>;
+	};
+
+	ps_ane_mpm: power-controller@c000 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc000 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "ane_mpm";
+		power-domains = <&ps_ane_sys>;
+	};
+
+	ps_ane_cpu: power-controller@c008 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc008 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "ane_cpu";
+		power-domains = <&ps_ane_sys>;
+	};
+
+	ps_ane_td: power-controller@c010 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc010 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "ane_td";
+		power-domains = <&ps_ane_sys>;
+	};
+
+	ps_ane_base: power-controller@c018 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc018 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "ane_base";
+		power-domains = <&ps_ane_td>;
+	};
+
+	ps_disp_cpu: power-controller@10000 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x10000 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "disp_cpu";
+		power-domains = <&ps_disp_fe>;
+
+		status = "disabled"; /* Locked, don't touch it */
+	};
+};
+
+&pmgr_mini {
+	ps_debug_gated: power-controller@0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "debug_gated";
+		apple,always-on;
+	};
+
+	ps_nub_spmi0: power-controller@58 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x58 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_spmi0";
+		apple,always-on;
+	};
+
+	ps_nub_spmi1: power-controller@60 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x60 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_spmi1";
+		apple,always-on;
+	};
+
+	ps_nub_spmi_a0: power-controller@68 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x68 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_spmi_a0";
+		apple,always-on;
+	};
+
+	ps_nub_spmi_a1: power-controller@70 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x70 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_spmi_a1";
+		apple,always-on;
+	};
+
+	ps_nub_spi0: power-controller@80 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x80 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_spi0";
+		apple,always-on;
+	};
+
+	ps_nub_ocla: power-controller@88 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x88 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_ocla";
+		apple,always-on;
+	};
+
+	ps_nub_gpio: power-controller@90 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x90 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_gpio";
+		apple,always-on;
+	};
+
+	ps_nub_sram: power-controller@a0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xa0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "nub_sram";
+		apple,always-on;
+	};
+
+	ps_debug_switch: power-controller@a8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xa8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "debug_switch";
+		apple,always-on;
+	};
+
+	ps_atc0_usb_aon: power-controller@b0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xb0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc0_usb_aon";
+	};
+
+	ps_atc1_usb_aon: power-controller@b8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xb8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc1_usb_aon";
+	};
+
+	ps_atc0_usb: power-controller@c0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc0_usb";
+		power-domains = <&ps_atc0_common>, <&ps_atc0_usb_aon>;
+	};
+
+	ps_atc1_usb: power-controller@c8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc1_usb";
+		power-domains = <&ps_atc1_common>, <&ps_atc1_usb_aon>;
+	};
+
+	ps_atc2_usb_aon: power-controller@d0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xd0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc2_usb_aon";
+	};
+
+	ps_atc3_usb_aon: power-controller@d8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xd8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc3_usb_aon";
+	};
+
+	ps_atc2_usb: power-controller@e0 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xe0 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc2_usb";
+		power-domains = <&ps_atc2_common>, <&ps_atc2_usb_aon>;
+	};
+
+	ps_atc3_usb: power-controller@e8 {
+		compatible = "apple,t8132-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xe8 4>;
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		label = "atc3_usb";
+		power-domains = <&ps_atc3_common>, <&ps_atc3_usb_aon>;
+	};
+};
diff --git a/arch/arm64/boot/dts/apple/t8132.dtsi b/arch/arm64/boot/dts/apple/t8132.dtsi
new file mode 100644
index 00000000000000..afec92a0601562
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8132.dtsi
@@ -0,0 +1,467 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple T8132 "M4" SoC
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+#include <dt-bindings/interrupt-controller/apple-aic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pinctrl/apple.h>
+
+/ {
+	compatible = "apple,t8132", "apple,arm-platform";
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu_e0>;
+				};
+				core1 {
+					cpu = <&cpu_e1>;
+				};
+				core2 {
+					cpu = <&cpu_e2>;
+				};
+				core3 {
+					cpu = <&cpu_e3>;
+				};
+				core4 {
+					cpu = <&cpu_e4>;
+				};
+				core5 {
+					cpu = <&cpu_e5>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu_p0>;
+				};
+				core1 {
+					cpu = <&cpu_p1>;
+				};
+				core2 {
+					cpu = <&cpu_p2>;
+				};
+				core3 {
+					cpu = <&cpu_p3>;
+				};
+			};
+		};
+
+		cpu_e0: cpu@0 {
+			compatible = "apple,donan-e";
+			device_type = "cpu";
+			reg = <0x0 0x0>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_0>;
+			i-cache-size = <0x20000>;
+			d-cache-size = <0x10000>;
+		};
+
+		cpu_e1: cpu@1 {
+			compatible = "apple,donan-e";
+			device_type = "cpu";
+			reg = <0x0 0x1>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_0>;
+			i-cache-size = <0x20000>;
+			d-cache-size = <0x10000>;
+		};
+
+		cpu_e2: cpu@2 {
+			compatible = "apple,donan-e";
+			device_type = "cpu";
+			reg = <0x0 0x2>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_0>;
+			i-cache-size = <0x20000>;
+			d-cache-size = <0x10000>;
+		};
+
+		cpu_e3: cpu@3 {
+			compatible = "apple,donan-e";
+			device_type = "cpu";
+			reg = <0x0 0x3>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_0>;
+			i-cache-size = <0x20000>;
+			d-cache-size = <0x10000>;
+		};
+
+		cpu_e4: cpu@4 {
+			compatible = "apple,donan-e";
+			device_type = "cpu";
+			reg = <0x0 0x4>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_0>;
+			i-cache-size = <0x20000>;
+			d-cache-size = <0x10000>;
+		};
+
+		cpu_e5: cpu@5 {
+			compatible = "apple,donan-e";
+			device_type = "cpu";
+			reg = <0x0 0x5>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_0>;
+			i-cache-size = <0x20000>;
+			d-cache-size = <0x10000>;
+		};
+
+		cpu_p0: cpu@10100 {
+			compatible = "apple,donan-p";
+			device_type = "cpu";
+			reg = <0x0 0x10100>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_1>;
+			i-cache-size = <0x30000>;
+			d-cache-size = <0x20000>;
+		};
+
+		cpu_p1: cpu@10101 {
+			compatible = "apple,donan-p";
+			device_type = "cpu";
+			reg = <0x0 0x10101>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_1>;
+			i-cache-size = <0x30000>;
+			d-cache-size = <0x20000>;
+		};
+
+		cpu_p2: cpu@10102 {
+			compatible = "apple,donan-p";
+			device_type = "cpu";
+			reg = <0x0 0x10102>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_1>;
+			i-cache-size = <0x30000>;
+			d-cache-size = <0x20000>;
+		};
+
+		cpu_p3: cpu@10103 {
+			compatible = "apple,donan-p";
+			device_type = "cpu";
+			reg = <0x0 0x10103>;
+			enable-method = "spin-table";
+			cpu-release-addr = <0 0>;
+			next-level-cache = <&l2_cache_1>;
+			i-cache-size = <0x30000>;
+			d-cache-size = <0x20000>;
+		};
+
+		l2_cache_0: l2-cache-0 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-unified;
+			cache-size = <0x400000>;
+		};
+
+		l2_cache_1: l2-cache-1 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-unified;
+			cache-size = <0x1000000>;
+		};
+	};
+
+	clkref: clock-ref {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "clkref";
+	};
+
+	soc: soc {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		ranges;
+		nonposted-mmio;
+
+		pmgr: power-management@380700000 {
+			compatible = "apple,t8132-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = <0x3 0x80700000 0 0x14000>;
+			/* child nodes are added in t8132-pmgr.dtsi */
+		};
+
+		aic: interrupt-controller@381000000 {
+			compatible = "apple,t8132-aic3", "apple,t8122-aic3";
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			reg = <0x3 0x81000000 0x0 0x1cc000>,
+			      <0x3 0x81040000 0x0 0x4000>;
+			reg-names = "core", "event";
+			power-domains = <&ps_aic>;
+
+			affinities {
+				e-core-pmu-affinity {
+					apple,fiq-index = <AIC_CPU_PMU_E>;
+					cpus = <&cpu_e0 &cpu_e1 &cpu_e2 &cpu_e3 &cpu_e4 &cpu_e5>;
+				};
+
+				p-core-pmu-affinity {
+					apple,fiq-index = <AIC_CPU_PMU_P>;
+					cpus = <&cpu_p0 &cpu_p1 &cpu_p2 &cpu_p3>;
+				};
+			};
+		};
+
+		pinctrl_nub: pinctrl@3881f0000 {
+			compatible = "apple,t8132-pinctrl", "apple,t8103-pinctrl";
+			reg = <0x3 0x881f0000 0x0 0x4000>;
+			power-domains = <&ps_nub_gpio>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pinctrl_nub 0 0 32>;
+			apple,npins = <32>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 499 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 500 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 501 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 502 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 503 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 504 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 505 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pmgr_mini: power-management@388280000 {
+			compatible = "apple,t8132-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = <0x3 0x88280000 0 0x4000>;
+			/* child nodes are added in t8132-pmgr.dtsi */
+		};
+
+		wdt: watchdog@3882b0000 {
+			compatible = "apple,t8132-wdt", "apple,t8103-wdt";
+			reg = <0x3 0x882b0000 0x0 0x4000>;
+			clocks = <&clkref>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 507 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_smc: pinctrl@38c820000 {
+			compatible = "apple,t8132-pinctrl", "apple,t8103-pinctrl";
+			reg = <0x3 0x8c820000 0x0 0x4000>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pinctrl_smc 0 0 18>;
+			apple,npins = <18>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 552 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 553 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 554 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 555 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 556 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 557 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 558 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_aop: pinctrl@390824000 {
+			compatible = "apple,t8132-pinctrl", "apple,t8103-pinctrl";
+			reg = <0x3 0x90824000 0x0 0x4000>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pinctrl_aop 0 0 57>;
+			apple,npins = <57>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 423 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 424 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 425 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 426 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 427 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 428 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 429 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pinctrl_ap: pinctrl@39a000000 {
+			compatible = "apple,t8132-pinctrl", "apple,t8103-pinctrl";
+			reg = <0x3 0x9a000000 0x0 0x100000>;
+			power-domains = <&ps_gpio>;
+
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pinctrl_ap 0 0 224>;
+			apple,npins = <224>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 303 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 304 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 305 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 306 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 307 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 308 IRQ_TYPE_LEVEL_HIGH>,
+			             <AIC_IRQ 309 IRQ_TYPE_LEVEL_HIGH>;
+
+			i2c0_pins: i2c0-pins {
+				pinmux = <APPLE_PINMUX(104, 1)>,
+				         <APPLE_PINMUX(103, 1)>;
+			};
+
+			i2c1_pins: i2c1-pins {
+				pinmux = <APPLE_PINMUX(106, 1)>,
+				         <APPLE_PINMUX(105, 1)>;
+			};
+
+			i2c2_pins: i2c2-pins {
+				pinmux = <APPLE_PINMUX(44, 1)>,
+				         <APPLE_PINMUX(43, 1)>;
+			};
+
+			i2c3_pins: i2c3-pins {
+				pinmux = <APPLE_PINMUX(42, 1)>,
+				         <APPLE_PINMUX(41, 1)>;
+			};
+
+			i2c4_pins: i2c4-pins {
+				pinmux = <APPLE_PINMUX(108, 1)>,
+				         <APPLE_PINMUX(107, 1)>;
+			};
+		};
+
+		i2c0: i2c@3ad010000 {
+			compatible = "apple,t8132-i2c", "apple,t8103-i2c";
+			reg = <0x3 0xad010000 0x0 0x4000>;
+			clocks = <&clkref>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 1062 IRQ_TYPE_LEVEL_HIGH>;
+			pinctrl-0 = <&i2c0_pins>;
+			pinctrl-names = "default";
+			#address-cells = <0x1>;
+			#size-cells = <0x0>;
+			power-domains = <&ps_i2c0>;
+
+			status = "disabled";
+		};
+
+		i2c1: i2c@3ad014000 {
+			compatible = "apple,t8132-i2c", "apple,t8103-i2c";
+			reg = <0x3 0xad014000 0x0 0x4000>;
+			clocks = <&clkref>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 1063 IRQ_TYPE_LEVEL_HIGH>;
+			pinctrl-0 = <&i2c1_pins>;
+			pinctrl-names = "default";
+			#address-cells = <0x1>;
+			#size-cells = <0x0>;
+			power-domains = <&ps_i2c1>;
+
+			status = "disabled";
+		};
+
+		i2c2: i2c@3ad018000 {
+			compatible = "apple,t8132-i2c", "apple,t8103-i2c";
+			reg = <0x3 0xad018000 0x0 0x4000>;
+			clocks = <&clkref>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 1064 IRQ_TYPE_LEVEL_HIGH>;
+			pinctrl-0 = <&i2c2_pins>;
+			pinctrl-names = "default";
+			#address-cells = <0x1>;
+			#size-cells = <0x0>;
+			power-domains = <&ps_i2c2>;
+
+			status = "disabled";
+		};
+
+		i2c3: i2c@3ad01c000 {
+			compatible = "apple,t8132-i2c", "apple,t8103-i2c";
+			reg = <0x3 0xad01c000 0x0 0x4000>;
+			clocks = <&clkref>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 1065 IRQ_TYPE_LEVEL_HIGH>;
+			pinctrl-0 = <&i2c3_pins>;
+			pinctrl-names = "default";
+			#address-cells = <0x1>;
+			#size-cells = <0x0>;
+			power-domains = <&ps_i2c3>;
+
+			status = "disabled";
+		};
+
+		i2c4: i2c@3ad020000 {
+			compatible = "apple,t8132-i2c", "apple,t8103-i2c";
+			reg = <0x3 0xad020000 0x0 0x4000>;
+			clocks = <&clkref>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 1066 IRQ_TYPE_LEVEL_HIGH>;
+			pinctrl-0 = <&i2c4_pins>;
+			pinctrl-names = "default";
+			#address-cells = <0x1>;
+			#size-cells = <0x0>;
+			power-domains = <&ps_i2c4>;
+
+			status = "disabled";
+		};
+
+		fpwm1: pwm@3ad044000 {
+			compatible = "apple,t8132-fpwm", "apple,s5l-fpwm";
+			reg = <0x3 0xad044000 0x0 0x4000>;
+			power-domains = <&ps_fpwm1>;
+			clocks = <&clkref>;
+			#pwm-cells = <2>;
+
+			status = "disabled";
+		};
+
+		serial0: serial@3ad200000 {
+			compatible = "apple,s5l-uart";
+			reg = <0x3 0xad200000 0x0 0x1000>;
+			reg-io-width = <4>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 1046 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clkref>, <&clkref>;
+			clock-names = "uart", "clk_uart_baud0";
+			power-domains = <&ps_uart0>;
+
+			status = "disabled";
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupt-parent = <&aic>;
+		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
+		interrupts = <AIC_FIQ AIC_TMR_GUEST_PHYS IRQ_TYPE_LEVEL_HIGH>,
+		             <AIC_FIQ AIC_TMR_GUEST_VIRT IRQ_TYPE_LEVEL_HIGH>,
+		             <AIC_FIQ AIC_TMR_HV_PHYS IRQ_TYPE_LEVEL_HIGH>,
+		             <AIC_FIQ AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>;
+	};
+};
+
+#include "t8132-pmgr.dtsi"
diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts
new file mode 100644
index 00000000000000..227b58a7ce64bc
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8140-j700.dts
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple MacBook Neo (A18 Pro, 2026)
+ *
+ * target-type: J700
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+/dts-v1/;
+
+#include "t8140.dtsi"
+
+/ {
+	compatible = "apple,j700", "apple,t8140", "apple,arm-platform";
+	model = "Apple MacBook Neo (A18 Pro, 2026)";
+
+	aliases {
+		serial0 = &serial0;
+	};
+
+	chosen {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		stdout-path = "serial0";
+
+		framebuffer@0 {
+			compatible = "apple,simple-framebuffer",
+			             "simple-framebuffer";
+			reg = <0 0 0 0>; /* To be filled by loader */
+			/* Format properties will be added by loader */
+
+			status = "disabled";
+		};
+	};
+
+	memory@800000000 {
+		device_type = "memory";
+		reg = <0x8 0 0x2 0>; /* To be filled by loader */
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		/* To be filled by loader */
+	};
+};
+
+&serial0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi
new file mode 100644
index 00000000000000..dd43520394ac51
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi
@@ -0,0 +1,777 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * PMGR Power domains for the Apple T8140 "A18 Pro" SoC
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+&pmgr {
+	ps_sep_aon: power-controller@60 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x60 4>;
+		label = "sep_aon";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_msg: power-controller@108 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x108 4>;
+		label = "msg";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_aic: power-controller@110 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x110 4>;
+		label = "aic";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_dwi: power-controller@118 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x118 4>;
+		label = "dwi";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_gpio: power-controller@120 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x120 4>;
+		label = "gpio";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_pmc: power-controller@138 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x138 4>;
+		label = "pmc";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_pms_fpwm0: power-controller@140 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x140 4>;
+		label = "pms_fpwm0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_pms_fpwm1: power-controller@148 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x148 4>;
+		label = "pms_fpwm1";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_pms_fpwm2: power-controller@150 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x150 4>;
+		label = "pms_fpwm2";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_pms_fpwm3: power-controller@158 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x158 4>;
+		label = "pms_fpwm3";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_pms_fpwm4: power-controller@160 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x160 4>;
+		label = "pms_fpwm4";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_pms_c1ppt: power-controller@168 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x168 4>;
+		label = "pms_c1ppt";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_soc_rc: power-controller@170 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x170 4>;
+		label = "soc_rc";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_soc_dpe: power-controller@178 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x178 4>;
+		label = "soc_dpe";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_pmgr_soc_ocla: power-controller@180 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x180 4>;
+		label = "pmgr_soc_ocla";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_ispsens0: power-controller@188 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x188 4>;
+		label = "ispsens0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_ispsens1: power-controller@190 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x190 4>;
+		label = "ispsens1";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_ap_tmm: power-controller@1a0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x1a0 4>;
+		label = "ap_tmm";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_afggfxni0: power-controller@1b8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x1b8 4>;
+		label = "afggfxni0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_gfx: power-controller@1f0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x1f0 4>;
+		label = "gfx";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_jpg: power-controller@230 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x230 4>;
+		label = "jpg";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_msr: power-controller@238 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x238 4>;
+		label = "msr";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_venc0_sys: power-controller@240 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x240 4>;
+		label = "venc0_sys";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_prores: power-controller@250 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x250 4>;
+		label = "prores";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_apcie_gp: power-controller@258 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x258 4>;
+		label = "apcie_gp";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_isp_sys: power-controller@268 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x268 4>;
+		label = "isp_sys";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_ans: power-controller@278 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x278 4>;
+		label = "ans";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_disp_sys: power-controller@280 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x280 4>;
+		label = "disp_sys";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on; /* TODO: figure out if we can enable PM here */
+	};
+
+	ps_dispext0_sys: power-controller@288 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x288 4>;
+		label = "dispext0_sys";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_ane_sys: power-controller@290 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x290 4>;
+		label = "ane_sys";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_msr_ase_core: power-controller@298 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x298 4>;
+		label = "msr_ase_core";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_msr>;
+		#reset-cells = <0>;
+	};
+
+	ps_apcie_sys_gp: power-controller@2a0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2a0 4>;
+		label = "apcie_sys_gp";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_sio_cpu: power-controller@2a8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2a8 4>;
+		label = "sio_cpu";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_fpwm0: power-controller@2b0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2b0 4>;
+		label = "fpwm0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_fpwm1: power-controller@2b8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2b8 4>;
+		label = "fpwm1";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_fpwm2: power-controller@2c0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2c0 4>;
+		label = "fpwm2";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_i2c0: power-controller@2c8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2c8 4>;
+		label = "i2c0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_i2c1: power-controller@2d0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2d0 4>;
+		label = "i2c1";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_i2c2: power-controller@2d8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2d8 4>;
+		label = "i2c2";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_i2c3: power-controller@2e0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2e0 4>;
+		label = "i2c3";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_i2c4: power-controller@2e8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2e8 4>;
+		label = "i2c4";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_spi_p: power-controller@2f0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2f0 4>;
+		label = "spi_p";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart_p: power-controller@2f8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x2f8 4>;
+		label = "uart_p";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_dpa_p: power-controller@300 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x300 4>;
+		label = "dpa_p";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_aes: power-controller@308 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x308 4>;
+		label = "aes";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_apcie_st: power-controller@310 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x310 4>;
+		label = "apcie_st";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_ans>;
+		#reset-cells = <0>;
+	};
+
+	ps_disp_fe: power-controller@318 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x318 4>;
+		label = "disp_fe";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_disp_sys>;
+		#reset-cells = <0>;
+		apple,always-on; /* TODO: figure out if we can enable PM here */
+	};
+
+	ps_dispext0_fe: power-controller@320 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x320 4>;
+		label = "dispext0_fe";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_dispext0_sys>;
+		#reset-cells = <0>;
+	};
+
+	ps_dispext0_cpu: power-controller@328 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x328 4>;
+		label = "dispext0_cpu";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_dispext0_fe>;
+		#reset-cells = <0>;
+
+		status = "disabled"; /* Locked, don't touch it */
+	};
+
+	ps_spi0: power-controller@330 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x330 4>;
+		label = "spi0";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_spi_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_spi1: power-controller@338 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x338 4>;
+		label = "spi1";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_spi_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_spi2: power-controller@340 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x340 4>;
+		label = "spi2";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_spi_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_spi3: power-controller@348 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x348 4>;
+		label = "spi3";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_spi_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_qspi: power-controller@350 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x350 4>;
+		label = "qspi";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_spi_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart_n: power-controller@358 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x358 4>;
+		label = "uart_n";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_uart_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart0: power-controller@360 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x360 4>;
+		label = "uart0";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_uart_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart1: power-controller@368 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x368 4>;
+		label = "uart1";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_uart_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart2: power-controller@370 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x370 4>;
+		label = "uart2";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_uart_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart3: power-controller@378 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x378 4>;
+		label = "uart3";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_uart_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart4: power-controller@380 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x380 4>;
+		label = "uart4";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_uart_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_uart5: power-controller@388 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x388 4>;
+		label = "uart5";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_uart_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_dpa0: power-controller@390 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x390 4>;
+		label = "dpa0";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_dpa_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_dpa1: power-controller@398 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x398 4>;
+		label = "dpa1";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_dpa_p>;
+		#reset-cells = <0>;
+	};
+
+	ps_apcie_sys_st: power-controller@3a0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3a0 4>;
+		label = "apcie_sys_st";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_ans>, <&ps_apcie_st>;
+		#reset-cells = <0>;
+	};
+
+	ps_apcie_phy_sw: power-controller@3c0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3c0 4>;
+		label = "apcie_phy_sw";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>;
+		#reset-cells = <0>;
+	};
+
+	ps_pmp: power-controller@3d0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3d0 4>;
+		label = "pmp";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_pms_sram: power-controller@3d8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3d8 4>;
+		label = "pms_sram";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_atc0_usb_aon: power-controller@3e0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3e0 4>;
+		label = "atc0_usb_aon";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+	};
+
+	ps_atc0_common: power-controller@3e8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3e8 4>;
+		label = "atc0_common";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_atc0_usb_aon>;
+		#reset-cells = <0>;
+	};
+
+	ps_atc0_usb: power-controller@3f0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x3f0 4>;
+		label = "atc0_usb";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_atc0_common>;
+		#reset-cells = <0>;
+	};
+
+	ps_sep: power-controller@c00 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xc00 4>;
+		label = "sep";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_venc0_dma: power-controller@8000 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8000 4>;
+		label = "venc0_dma";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_venc0_sys>;
+		#reset-cells = <0>;
+	};
+
+	ps_venc0_pipe4: power-controller@8008 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8008 4>;
+		label = "venc0_pipe4";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_venc0_dma>;
+		#reset-cells = <0>;
+	};
+
+	ps_venc0_pipe5: power-controller@8010 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8010 4>;
+		label = "venc0_pipe5";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_venc0_dma>;
+		#reset-cells = <0>;
+	};
+
+	ps_venc0_me0: power-controller@8018 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8018 4>;
+		label = "venc0_me0";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_venc0_dma>;
+		#reset-cells = <0>;
+	};
+
+	ps_venc0_me1: power-controller@8020 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8020 4>;
+		label = "venc0_me1";
+		#power-domain-cells = <0>;
+		power-domains = <&ps_venc0_me0>;
+		#reset-cells = <0>;
+	};
+};
+
+&pmgr_mini {
+	ps_debug_gated: power-controller@0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x0 4>;
+		label = "debug_gated";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_spmi0: power-controller@58 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x58 4>;
+		label = "nub_spmi0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_spmi1: power-controller@60 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x60 4>;
+		label = "nub_spmi1";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_spmi_a0: power-controller@68 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x68 4>;
+		label = "nub_spmi_a0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_aon_nogate: power-controller@78 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x78 4>;
+		label = "nub_aon_nogate";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_ocla: power-controller@80 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x80 4>;
+		label = "nub_ocla";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_gpio: power-controller@88 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x88 4>;
+		label = "nub_gpio";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_fabric: power-controller@a0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xa0 4>;
+		label = "nub_fabric";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_nub_sram: power-controller@a8 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xa8 4>;
+		label = "nub_sram";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_debug_switch: power-controller@b0 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0xb0 4>;
+		label = "debug_switch";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_smc_i2cm0: power-controller@8028 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8028 4>;
+		label = "smc_i2cm0";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_smc_i2cm1: power-controller@8030 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8030 4>;
+		label = "smc_i2cm1";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+
+	ps_smc_i2cm2: power-controller@8038 {
+		compatible = "apple,t8140-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate";
+		reg = <0x8038 4>;
+		label = "smc_i2cm2";
+		#power-domain-cells = <0>;
+		#reset-cells = <0>;
+		apple,always-on;
+	};
+};
diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi
new file mode 100644
index 00000000000000..150d91bdf6fccc
--- /dev/null
+++ b/arch/arm64/boot/dts/apple/t8140.dtsi
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Apple T8140 "A18 Pro" SoC
+ *
+ * Copyright The Asahi Linux Contributors
+ */
+
+#include <dt-bindings/interrupt-controller/apple-aic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	compatible = "apple,t8140", "apple,arm-platform";
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	clkref: clock-ref {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "clkref";
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu_e0>;
+				};
+
+				core1 {
+					cpu = <&cpu_e1>;
+				};
+
+				core2 {
+					cpu = <&cpu_e2>;
+				};
+
+				core3 {
+					cpu = <&cpu_e3>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu_p0>;
+				};
+
+				core1 {
+					cpu = <&cpu_p1>;
+				};
+			};
+		};
+
+		cpu_e0: cpu@0 {
+			device_type = "cpu";
+			compatible = "apple,tahiti-e";
+			reg = <0x0 0x0>;
+			cpu-release-addr = <0 0>;
+			d-cache-size = <0x10000>;
+			enable-method = "spin-table";
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
+		};
+
+		cpu_e1: cpu@1 {
+			device_type = "cpu";
+			compatible = "apple,tahiti-e";
+			reg = <0x0 0x1>;
+			cpu-release-addr = <0 0>;
+			d-cache-size = <0x10000>;
+			enable-method = "spin-table";
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
+		};
+
+		cpu_e2: cpu@2 {
+			device_type = "cpu";
+			compatible = "apple,tahiti-e";
+			reg = <0x0 0x2>;
+			cpu-release-addr = <0 0>;
+			d-cache-size = <0x10000>;
+			enable-method = "spin-table";
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
+		};
+
+		cpu_e3: cpu@3 {
+			device_type = "cpu";
+			compatible = "apple,tahiti-e";
+			reg = <0x0 0x3>;
+			cpu-release-addr = <0 0>;
+			d-cache-size = <0x10000>;
+			enable-method = "spin-table";
+			i-cache-size = <0x20000>;
+			next-level-cache = <&l2_cache_0>;
+		};
+
+		cpu_p0: cpu@10100 {
+			device_type = "cpu";
+			compatible = "apple,tahiti-p";
+			reg = <0x0 0x10100>;
+			cpu-release-addr = <0 0>;
+			d-cache-size = <0x20000>;
+			enable-method = "spin-table";
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
+		};
+
+		cpu_p1: cpu@10101 {
+			device_type = "cpu";
+			compatible = "apple,tahiti-p";
+			reg = <0x0 0x10101>;
+			cpu-release-addr = <0 0>;
+			d-cache-size = <0x20000>;
+			enable-method = "spin-table";
+			i-cache-size = <0x30000>;
+			next-level-cache = <&l2_cache_1>;
+		};
+
+		l2_cache_0: l2-cache-0 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <0x400000>;
+			cache-unified;
+		};
+
+		l2_cache_1: l2-cache-1 {
+			compatible = "cache";
+			cache-level = <2>;
+			cache-size = <0x1000000>;
+			cache-unified;
+		};
+	};
+
+	soc: soc {
+		compatible = "simple-bus";
+		ranges;
+		#address-cells = <2>;
+		nonposted-mmio;
+		#size-cells = <2>;
+
+		pmgr: power-management@300700000 {
+			compatible = "apple,t8140-pmgr", "apple,t8103-pmgr",
+			             "syscon", "simple-mfd";
+			reg = <0x3 0x700000 0 0x14000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			/* child nodes are added in t8140-pmgr.dtsi */
+		};
+
+		aic: interrupt-controller@301000000 {
+			compatible = "apple,t8140-aic3", "apple,t8122-aic3";
+			reg = <0x3 0x01000000 0x0 0x1cc000>,
+			      <0x3 0x01040000 0x0 0x4000>;
+			reg-names = "core", "event";
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			power-domains = <&ps_aic>;
+		};
+
+		pmgr_mini: power-management@308280000 {
+			compatible = "apple,t8140-pmgr", "apple,t8103-pmgr",
+			             "syscon", "simple-mfd";
+			reg = <0x3 0x8280000 0 0xc000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			/* child nodes are added in t8140-pmgr.dtsi */
+		};
+
+		wdt: watchdog@3082b0000 {
+			compatible = "apple,t8140-wdt", "apple,t8103-wdt";
+			reg = <0x3 0x082b0000 0x0 0x4000>;
+			clocks = <&clkref>;
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 507 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		serial0: serial@385200000 {
+			compatible = "apple,s5l-uart";
+			reg = <0x3 0x85200000 0x0 0x1000>;
+			clocks = <&clkref>, <&clkref>;
+			clock-names = "uart", "clk_uart_baud0";
+			interrupt-parent = <&aic>;
+			interrupts = <AIC_IRQ 1092 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&ps_uart0>;
+			reg-io-width = <4>;
+
+			status = "disabled";
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupt-parent = <&aic>;
+		interrupts = <AIC_FIQ AIC_TMR_GUEST_PHYS IRQ_TYPE_LEVEL_HIGH>,
+		             <AIC_FIQ AIC_TMR_GUEST_VIRT IRQ_TYPE_LEVEL_HIGH>,
+		             <AIC_FIQ AIC_TMR_HV_PHYS IRQ_TYPE_LEVEL_HIGH>,
+		             <AIC_FIQ AIC_TMR_HV_VIRT IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt";
+	};
+};
+
+#include "t8140-pmgr.dtsi"
diff --git a/arch/arm64/boot/dts/arm/fvp-base-revc.dts b/arch/arm64/boot/dts/arm/fvp-base-revc.dts
index 68a69f17e93d80..d184a073c3a2a7 100644
--- a/arch/arm64/boot/dts/arm/fvp-base-revc.dts
+++ b/arch/arm64/boot/dts/arm/fvp-base-revc.dts
@@ -44,6 +44,38 @@
 		#address-cells = <2>;
 		#size-cells = <0>;
 
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+				core1 {
+					cpu = <&cpu1>;
+				};
+				core2 {
+					cpu = <&cpu2>;
+				};
+				core3 {
+					cpu = <&cpu3>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu4>;
+				};
+				core1 {
+					cpu = <&cpu5>;
+				};
+				core2 {
+					cpu = <&cpu6>;
+				};
+				core3 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
 		idle-states {
 			entry-method = "psci";
 
@@ -262,6 +294,13 @@
 		};
 	};
 
+	watchdog@2a440000 {
+		compatible = "arm,sbsa-gwdt";
+		reg = <0x0 0x2a440000 0 0x1000>,	// Watchdog control frame
+		      <0x0 0x2a450000 0 0x1000>;	// Refresh frame
+		interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
 	pmu {
 		compatible = "arm,armv8-pmuv3";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
@@ -334,7 +373,8 @@
 		device_type = "pci";
 		bus-range = <0x0 0xff>;
 		reg = <0x0 0x40000000 0x0 0x10000000>;
-		ranges = <0x2000000 0x0 0x50000000 0x0 0x50000000 0x0 0x10000000>;
+		ranges = <0x2000000 0x0 0x50000000 0x0 0x50000000 0x0 0x10000000
+				0x3000000 0x40 0x00000000 0x40 0x00000000 0x1 0x00000000>;
 		interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>,
 				<0 0 0 2 &gic 0 0 GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>,
 				<0 0 0 3 &gic 0 0 GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts b/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts
index 127be0fc27c22e..ae4b7c587dbc55 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts
+++ b/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts
@@ -3,6 +3,11 @@
 
 #include "bcm2712-rpi-5-b.dts"
 
+/ {
+	compatible = "raspberrypi,5-model-b-d0", "brcm,bcm2712-d0";
+	model = "Raspberry Pi 5 d0 Stepping";
+};
+
 &gio_aon {
 	brcm,gpio-bank-widths = <15 6>;
 
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi
index b7a6bc34ae1ab4..e5030da60f9e70 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b-base.dtsi
@@ -3,6 +3,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/power/raspberrypi-power.h>
 #include "bcm2712.dtsi"
 
 / {
@@ -104,6 +105,18 @@
 	};
 };
 
+&ddc0 {
+	status = "okay";
+};
+
+&ddc1 {
+	status = "okay";
+};
+
+&disp_intr {
+	status = "okay";
+};
+
 &pinctrl {
 	bt_shutdown_default: bt-shutdown-default-state {
 		function = "gpio";
@@ -227,19 +240,35 @@
 	};
 };
 
+&usb {
+	power-domains = <&power RPI_POWER_DOMAIN_USB>;
+	status = "okay";
+};
+
 &hvs {
 	clocks = <&firmware_clocks 4>, <&firmware_clocks 16>;
 	clock-names = "core", "disp";
+	status = "okay";
 };
 
 &hdmi0 {
 	clocks = <&firmware_clocks 13>, <&firmware_clocks 14>, <&dvp 0>, <&clk_27MHz>;
 	clock-names = "hdmi", "bvb", "audio", "cec";
+	status = "okay";
 };
 
 &hdmi1 {
 	clocks = <&firmware_clocks 13>, <&firmware_clocks 14>, <&dvp 1>, <&clk_27MHz>;
 	clock-names = "hdmi", "bvb", "audio", "cec";
+	status = "okay";
+};
+
+&mop {
+	status = "okay";
+};
+
+&moplet {
+	status = "okay";
 };
 
 &pcie1 {
@@ -250,6 +279,18 @@
 	status = "okay";
 };
 
+&pixelvalve0 {
+	status = "okay";
+};
+
+&pixelvalve1 {
+	status = "okay";
+};
+
 &v3d {
 	clocks = <&firmware_clocks 5>;
 };
+
+&vc4 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
index 45aa8ec5e2b1f5..9f83379df7aad1 100644
--- a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi
@@ -10,6 +10,15 @@
 
 	interrupt-parent = <&gicv2>;
 
+	arm-pmu {
+		compatible = "arm,cortex-a76-pmu";
+		interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>,
+			<GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>,
+			<GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>,
+			<GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	};
+
 	clocks {
 		/* The oscillator is the root of the clock tree. */
 		clk_osc: clk-osc {
@@ -46,6 +55,13 @@
 			clock-frequency = <96000000>;
 			clock-output-names = "sw-baud";
 		};
+
+		clk_usb: clk-usb {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <480000000>;
+			clock-output-names = "otg";
+		};
 	};
 
 	cpus: cpus {
@@ -359,12 +375,14 @@
 			compatible = "brcm,bcm2712-pixelvalve0";
 			reg = <0x7c410000 0x100>;
 			interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
 		};
 
 		pixelvalve1: pixelvalve@7c411000 {
 			compatible = "brcm,bcm2712-pixelvalve1";
 			reg = <0x7c411000 0x100>;
 			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
 		};
 
 		mop: mop@7c500000 {
@@ -372,6 +390,7 @@
 			reg = <0x7c500000 0x28>;
 			interrupt-parent = <&disp_intr>;
 			interrupts = <1>;
+			status = "disabled";
 		};
 
 		moplet: moplet@7c501000 {
@@ -379,6 +398,7 @@
 			reg = <0x7c501000 0x20>;
 			interrupt-parent = <&disp_intr>;
 			interrupts = <0>;
+			status = "disabled";
 		};
 
 		disp_intr: interrupt-controller@7c502000 {
@@ -387,6 +407,7 @@
 			interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-controller;
 			#interrupt-cells = <1>;
+			status = "disabled";
 		};
 
 		dvp: clock@7c700000 {
@@ -405,6 +426,7 @@
 			clock-frequency = <97500>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			status = "disabled";
 		};
 
 		ddc1: i2c@7d508280 {
@@ -415,6 +437,7 @@
 			clock-frequency = <97500>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			status = "disabled";
 		};
 
 		bsc_irq: interrupt-controller@7d508380 {
@@ -460,6 +483,7 @@
 			interrupt-names = "cec-tx", "cec-rx", "cec-low",
 					  "hpd-connected", "hpd-removed";
 			ddc = <&ddc0>;
+			status = "disabled";
 		};
 
 		hdmi1: hdmi@7c706400 {
@@ -489,6 +513,19 @@
 			interrupt-names = "cec-tx", "cec-rx", "cec-low",
 					  "hpd-connected", "hpd-removed";
 			ddc = <&ddc1>;
+			status = "disabled";
+		};
+
+		avs_monitor: avs-monitor@7d542000 {
+			compatible = "brcm,bcm2711-avs-monitor",
+						"syscon", "simple-mfd";
+			reg = <0x7d542000 0xf00>;
+			status = "okay";
+
+			thermal: thermal {
+				compatible = "brcm,bcm2711-thermal";
+				#thermal-sensor-cells = <0>;
+			};
 		};
 	};
 
@@ -663,10 +700,24 @@
 			interrupts = <GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>;
 		};
+
+		usb: usb@1000480000 {
+			compatible = "brcm,bcm2835-usb";
+			reg = <0x10 0x00480000  0x0 0x10000>;
+			interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&clk_usb>;
+			clock-names = "otg";
+			phys = <&usbphy>;
+			phy-names = "usb2-phy";
+			status = "disabled";
+		};
 	};
 
 	vc4: gpu {
 		compatible = "brcm,bcm2712-vc6";
+		status = "disabled";
 	};
 
 	timer {
@@ -701,5 +752,31 @@
 		interrupt-parent = <&disp_intr>;
 		interrupts = <2>, <9>, <16>;
 		interrupt-names = "ch0-eof", "ch1-eof", "ch2-eof";
+		status = "disabled";
+	};
+
+	thermal-zones {
+		cpu_thermal: cpu-thermal {
+			polling-delay-passive = <1000>;
+			polling-delay = <1000>;
+			coefficients = <(-550) 450000>;
+			thermal-sensors = <&thermal>;
+
+			thermal_trips: trips {
+				cpu_crit: cpu-crit {
+					temperature	= <110000>;
+					hysteresis	= <0>;
+					type		= "critical";
+				};
+			};
+
+			cooling_maps: cooling-maps {
+			};
+		};
+	};
+
+	usbphy: phy {
+		compatible = "usb-nop-xceiv";
+		#phy-cells = <0>;
 	};
 };
diff --git a/arch/arm64/boot/dts/broadcom/rp1-common.dtsi b/arch/arm64/boot/dts/broadcom/rp1-common.dtsi
index 16f5359395835f..8d9de71efdba9d 100644
--- a/arch/arm64/boot/dts/broadcom/rp1-common.dtsi
+++ b/arch/arm64/boot/dts/broadcom/rp1-common.dtsi
@@ -26,6 +26,125 @@ pci_ep_bus: pci-ep-bus@1 {
 				       <200000000>;  // RP1_CLK_SYS
 	};
 
+	rp1_spi8: spi@4004c000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x4004c000  0x0 0x130>;
+		interrupts = <56 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		num-cs = <2>;
+
+		status = "disabled";
+	};
+
+	rp1_spi0: spi@40050000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x40050000  0x0 0x130>;
+		interrupts = <19 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		num-cs = <4>;
+
+		status = "disabled";
+	};
+
+	rp1_spi1: spi@40054000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x40054000  0x0 0x130>;
+		interrupts = <20 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		num-cs = <3>;
+
+		status = "disabled";
+	};
+
+	rp1_spi2: spi@40058000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x40058000  0x0 0x130>;
+		interrupts = <21 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		num-cs = <2>;
+
+		status = "disabled";
+	};
+
+	rp1_spi3: spi@4005c000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x4005c000  0x0 0x130>;
+		interrupts = <22 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		num-cs = <2>;
+
+		status = "disabled";
+	};
+
+	rp1_spi4: spi@40060000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x40060000  0x0 0x130>;
+		interrupts = <23 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <0>;
+		#size-cells = <0>;
+		num-cs = <1>;
+		spi-slave;
+
+		status = "disabled";
+	};
+
+	rp1_spi5: spi@40064000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x40064000  0x0 0x130>;
+		interrupts = <24 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		num-cs = <2>;
+
+		status = "disabled";
+	};
+
+	rp1_spi6: spi@40068000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x40068000  0x0 0x130>;
+		interrupts = <54 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		num-cs = <3>;
+
+		status = "disabled";
+	};
+
+	rp1_spi7: spi@4006c000 {
+		compatible = "snps,dw-apb-ssi";
+		reg = <0x00 0x4006c000  0x0 0x130>;
+		interrupts = <55 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&rp1_clocks RP1_CLK_SYS>;
+		clock-names = "ssi_clk";
+		#address-cells = <0>;
+		#size-cells = <0>;
+		num-cs = <1>;
+		spi-slave;
+
+		status = "disabled";
+	};
+
 	rp1_i2c0: i2c@40070000 {
 		compatible = "snps,designware-i2c";
 		reg = <0x0 0x40070000  0x0 0x1000>;
diff --git a/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi b/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi
index 70430cb2b053a4..5bcc220e3d3830 100644
--- a/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi
+++ b/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi
@@ -355,9 +355,9 @@
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupts = <1 13 4>,
-		             <1 14 4>,
-		             <1 11 4>,
-		             <1 10 4>;
+			     <1 14 4>,
+			     <1 11 4>,
+			     <1 10 4>;
 	};
 
 	pmu {
diff --git a/arch/arm64/boot/dts/exynos/Makefile b/arch/arm64/boot/dts/exynos/Makefile
index 8c48ce2e02e512..7534b773d28f3b 100644
--- a/arch/arm64/boot/dts/exynos/Makefile
+++ b/arch/arm64/boot/dts/exynos/Makefile
@@ -13,6 +13,7 @@ dtb-$(CONFIG_ARCH_EXYNOS) += \
 	exynos7870-j7xelte.dtb		\
 	exynos7870-on7xelte.dtb		\
 	exynos7885-jackpotlte.dtb	\
+	exynos850-a217f.dtb		\
 	exynos850-e850-96.dtb		\
 	exynos8855-smdk.dtb		\
 	exynos8895-dreamlte.dtb		\
diff --git a/arch/arm64/boot/dts/exynos/axis/artpec8.dtsi b/arch/arm64/boot/dts/exynos/axis/artpec8.dtsi
index db98332979827b..9cfaf4935dd4b0 100644
--- a/arch/arm64/boot/dts/exynos/axis/artpec8.dtsi
+++ b/arch/arm64/boot/dts/exynos/axis/artpec8.dtsi
@@ -22,6 +22,22 @@
 		pinctrl1 = &pinctrl_peric;
 	};
 
+	fin_pll: clock-finpll {
+		compatible = "fixed-factor-clock";
+		clocks = <&osc_clk>;
+		#clock-cells = <0>;
+		clock-div = <2>;
+		clock-mult = <1>;
+		clock-output-names = "fin_pll";
+	};
+
+	osc_clk: clock-osc {
+		/* XXTI */
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-output-names = "osc_clk";
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -65,31 +81,15 @@
 
 			cpu_sleep: cpu-sleep {
 				compatible = "arm,idle-state";
-				arm,psci-suspend-param = <0x0010000>;
 				local-timer-stop;
 				entry-latency-us = <300>;
 				exit-latency-us = <1200>;
 				min-residency-us = <2000>;
+				arm,psci-suspend-param = <0x0010000>;
 			};
 		};
 	};
 
-	fin_pll: clock-finpll {
-		compatible = "fixed-factor-clock";
-		clocks = <&osc_clk>;
-		#clock-cells = <0>;
-		clock-div = <2>;
-		clock-mult = <1>;
-		clock-output-names = "fin_pll";
-	};
-
-	osc_clk: clock-osc {
-		/* XXTI */
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-output-names = "osc_clk";
-	};
-
 	pmu {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>,
@@ -229,8 +229,8 @@
 				 <&cmu_fsys CLK_GOUT_FSYS_UART0_SCLK_UART>;
 			clock-names = "uart", "clk_uart_baud0";
 			interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&serial0_bus>;
+			pinctrl-names = "default";
 		};
 	};
 
diff --git a/arch/arm64/boot/dts/exynos/axis/artpec9.dtsi b/arch/arm64/boot/dts/exynos/axis/artpec9.dtsi
index cd46aaf056287e..a92af2ed13f92b 100644
--- a/arch/arm64/boot/dts/exynos/axis/artpec9.dtsi
+++ b/arch/arm64/boot/dts/exynos/axis/artpec9.dtsi
@@ -23,6 +23,22 @@
 		pinctrl2 = &pinctrl_peric;
 	};
 
+	fin_pll: clock-finpll {
+		compatible = "fixed-factor-clock";
+		clocks = <&osc_clk>;
+		#clock-cells = <0>;
+		clock-div = <2>;
+		clock-mult = <1>;
+		clock-output-names = "fin_pll";
+	};
+
+	osc_clk: clock-osc {
+		/* XXTI */
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-output-names = "osc_clk";
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -82,31 +98,15 @@
 
 			cpu_sleep: cpu-sleep {
 				compatible = "arm,idle-state";
-				arm,psci-suspend-param = <0x0010000>;
 				local-timer-stop;
 				entry-latency-us = <300>;
 				exit-latency-us = <1200>;
 				min-residency-us = <2000>;
+				arm,psci-suspend-param = <0x0010000>;
 			};
 		};
 	};
 
-	fin_pll: clock-finpll {
-		compatible = "fixed-factor-clock";
-		clocks = <&osc_clk>;
-		#clock-cells = <0>;
-		clock-div = <2>;
-		clock-mult = <1>;
-		clock-output-names = "fin_pll";
-	};
-
-	osc_clk: clock-osc {
-		/* XXTI */
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-output-names = "osc_clk";
-	};
-
 	pmu {
 		compatible = "arm,cortex-a55-pmu";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
@@ -261,8 +261,8 @@
 				 <&cmu_fsys1 CLK_GOUT_FSYS1_UART0_SCLK_UART>;
 			clock-names = "uart", "clk_uart_baud0";
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&serial0_bus>;
+			pinctrl-names = "default";
 			samsung,uart-fifosize = <64>;
 		};
 	};
diff --git a/arch/arm64/boot/dts/exynos/exynos2200-g0s.dts b/arch/arm64/boot/dts/exynos/exynos2200-g0s.dts
index 458f1bc6f5d0ce..30772bf7725289 100644
--- a/arch/arm64/boot/dts/exynos/exynos2200-g0s.dts
+++ b/arch/arm64/boot/dts/exynos/exynos2200-g0s.dts
@@ -17,10 +17,6 @@
 	chassis-type = "handset";
 
 	chosen {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
 		framebuffer: framebuffer {
 			compatible = "simple-framebuffer";
 			memory-region = <&cont_splash_mem>;
@@ -52,9 +48,9 @@
 
 		volup-key {
 			label = "Volume Up";
-			linux,code = <KEY_VOLUMEUP>;
 			gpios = <&gpa3 0 GPIO_ACTIVE_LOW>;
 			wakeup-source;
+			linux,code = <KEY_VOLUMEUP>;
 		};
 	};
 
@@ -71,9 +67,9 @@
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <2>;
-		ranges;
 
 		cont_splash_mem: framebuffer@f6200000 {
 			reg = <0x0 0xf6200000 0x0 (1080 * 2340 * 4)>;
diff --git a/arch/arm64/boot/dts/exynos/exynos2200-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos2200-pinctrl.dtsi
index 5877da7baf5c40..d5aede3af15f70 100644
--- a/arch/arm64/boot/dts/exynos/exynos2200-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos2200-pinctrl.dtsi
@@ -702,7 +702,6 @@
 		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
 		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
 	};
-
 };
 
 &pinctrl_hsi1 {
@@ -1171,7 +1170,6 @@
 		samsung,pin-function = <EXYNOS_PIN_FUNC_2>;
 		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
 	};
-
 };
 
 &pinctrl_peric2 {
diff --git a/arch/arm64/boot/dts/exynos/exynos2200.dtsi b/arch/arm64/boot/dts/exynos/exynos2200.dtsi
index b1c134f2232457..3b853d66e19cee 100644
--- a/arch/arm64/boot/dts/exynos/exynos2200.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos2200.dtsi
@@ -712,7 +712,6 @@
 
 				status = "disabled";
 			};
-
 		};
 
 		cmu_hsi0: clock-controller@10a00000 {
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
index 0ad02a098765db..a17a7d06321232 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
@@ -56,32 +56,32 @@
 	gpio-keys {
 		compatible = "gpio-keys";
 
+		homepage-key {
+			gpios = <&gpa0 3 GPIO_ACTIVE_LOW>;
+			label = "homepage key";
+			debounce-interval = <10>;
+			linux,code = <KEY_MENU>;
+		};
+
 		power-key {
 			gpios = <&gpa2 7 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_POWER>;
 			label = "power key";
 			debounce-interval = <10>;
+			linux,code = <KEY_POWER>;
 		};
 
 		volume-up-key {
 			gpios = <&gpa2 0 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_VOLUMEUP>;
 			label = "volume-up key";
 			debounce-interval = <10>;
+			linux,code = <KEY_VOLUMEUP>;
 		};
 
 		volume-down-key {
 			gpios = <&gpa2 1 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_VOLUMEDOWN>;
 			label = "volume-down key";
 			debounce-interval = <10>;
-		};
-
-		homepage-key {
-			gpios = <&gpa0 3 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_MENU>;
-			label = "homepage key";
-			debounce-interval = <10>;
+			linux,code = <KEY_VOLUMEDOWN>;
 		};
 	};
 
@@ -103,13 +103,6 @@
 		};
 	};
 
-	vph_pwr_regulator: regulator-vph-pwr {
-		compatible = "regulator-fixed";
-		regulator-name = "VPH_PWR";
-		regulator-min-microvolt = <4200000>;
-		regulator-max-microvolt = <4200000>;
-	};
-
 	irda_regulator: regulator-irda {
 		compatible = "regulator-fixed";
 		enable-active-high;
@@ -117,6 +110,13 @@
 		regulator-name = "irda_regulator";
 	};
 
+	vph_pwr_regulator: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "VPH_PWR";
+		regulator-min-microvolt = <4200000>;
+		regulator-max-microvolt = <4200000>;
+	};
+
 	sound {
 		compatible = "samsung,tm2-audio";
 		audio-codec = <&wm5110>, <&hdmi>;
@@ -358,11 +358,12 @@
 &dsi {
 	vddcore-supply = <&ldo6_reg>;
 	vddio-supply = <&ldo7_reg>;
+	pinctrl-0 = <&te_irq>;
+	pinctrl-names = "default";
+
 	samsung,burst-clock-frequency = <512000000>;
 	samsung,esc-clock-frequency = <16000000>;
 	samsung,pll-clock-frequency = <24000000>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&te_irq>;
 
 	status = "okay";
 };
@@ -410,11 +411,11 @@
 
 	pmic@66 {
 		compatible = "samsung,s2mps13-pmic";
+		reg = <0x66>;
 		interrupt-parent = <&gpa0>;
 		interrupts = <7 IRQ_TYPE_LEVEL_LOW>;
-		reg = <0x66>;
-		samsung,s2mps11-wrstbi-ground;
 		wakeup-source;
+		samsung,s2mps11-wrstbi-ground;
 
 		s2mps13_osc: clocks {
 			compatible = "samsung,s2mps13-clk";
@@ -859,8 +860,8 @@
 	status = "okay";
 
 	bridge@39 {
-		reg = <0x39>;
 		compatible = "sil,sii8620";
+		reg = <0x39>;
 		cvcc10-supply = <&ldo36_reg>;
 		iovcc18-supply = <&ldo34_reg>;
 		interrupt-parent = <&gpf0>;
@@ -897,9 +898,9 @@
 
 	pmic@66 {
 		compatible = "maxim,max77843";
+		reg = <0x66>;
 		interrupt-parent = <&gpa1>;
 		interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
-		reg = <0x66>;
 
 		muic: extcon {
 			compatible = "maxim,max77843-muic";
@@ -997,19 +998,20 @@
 	cap-mmc-highspeed;
 	non-removable;
 	card-detect-delay = <200>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 4>;
-	samsung,dw-mshc-ddr-timing = <0 2>;
-	samsung,dw-mshc-hs400-timing = <0 3>;
-	samsung,read-strobe-delay = <90>;
 	fifo-depth = <0x80>;
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_qrdy &sd0_bus1 &sd0_bus4
 		     &sd0_bus8 &sd0_rdqs>;
+	pinctrl-names = "default";
 	bus-width = <8>;
 	assigned-clocks = <&cmu_top CLK_SCLK_MMC0_FSYS>;
 	assigned-clock-rates = <800000000>;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 4>;
+	samsung,dw-mshc-ddr-timing = <0 2>;
+	samsung,dw-mshc-hs400-timing = <0 3>;
+	samsung,read-strobe-delay = <90>;
+
 	status = "okay";
 };
 
@@ -1018,20 +1020,21 @@
 	disable-wp;
 	cd-gpios = <&gpa2 4 GPIO_ACTIVE_LOW>;
 	card-detect-delay = <200>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 4>;
-	samsung,dw-mshc-ddr-timing = <0 2>;
 	fifo-depth = <0x80>;
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus1 &sd2_bus4>;
+	pinctrl-names = "default";
 	bus-width = <4>;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 4>;
+	samsung,dw-mshc-ddr-timing = <0 2>;
+
 	status = "okay";
 };
 
 &pcie {
-	pinctrl-names = "default";
 	pinctrl-0 = <&pcie_bus &pcie_wlanen>;
+	pinctrl-names = "default";
 	vdd10-supply = <&ldo6_reg>;
 	vdd18-supply = <&ldo7_reg>;
 	assigned-clocks = <&cmu_fsys CLK_MOUT_SCLK_PCIE_100_USER>,
@@ -1064,14 +1067,14 @@
 
 	events {
 		ppmu_event0_d1_general: ppmu-event0-d1-general {
-		       event-name = "ppmu-event0-d1-general";
-	       };
+			event-name = "ppmu-event0-d1-general";
+		};
 	};
 };
 
 &pinctrl_alive {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_alive>;
+	pinctrl-names = "default";
 
 	initial_alive: initial-state {
 		PIN_IN(gpa0-0, DOWN, FAST_SR1);
@@ -1154,8 +1157,8 @@
 };
 
 &pinctrl_cpif {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_cpif>;
+	pinctrl-names = "default";
 
 	initial_cpif: initial-state {
 		PIN_IN(gpv6-0, DOWN, FAST_SR1);
@@ -1164,8 +1167,8 @@
 };
 
 &pinctrl_ese {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_ese>;
+	pinctrl-names = "default";
 
 	pcie_wlanen: pcie-wlanen-pins {
 		samsung,pins = "gpj2-0";
@@ -1181,8 +1184,8 @@
 };
 
 &pinctrl_fsys {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_fsys>;
+	pinctrl-names = "default";
 
 	initial_fsys: initial-state {
 		PIN_IN(gpr3-0, NONE, FAST_SR1);
@@ -1194,8 +1197,8 @@
 };
 
 &pinctrl_imem {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_imem>;
+	pinctrl-names = "default";
 
 	initial_imem: initial-state {
 		PIN_IN(gpf0-0, UP, FAST_SR1);
@@ -1210,8 +1213,8 @@
 };
 
 &pinctrl_nfc {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_nfc>;
+	pinctrl-names = "default";
 
 	initial_nfc: initial-state {
 		PIN_IN(gpj0-2, DOWN, FAST_SR1);
@@ -1219,8 +1222,8 @@
 };
 
 &pinctrl_peric {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_peric>;
+	pinctrl-names = "default";
 
 	initial_peric: initial-state {
 		PIN_IN(gpv7-0, DOWN, FAST_SR1);
@@ -1274,8 +1277,8 @@
 };
 
 &pinctrl_touch {
-	pinctrl-names = "default";
 	pinctrl-0 = <&initial_touch>;
+	pinctrl-names = "default";
 
 	initial_touch: initial-state {
 		PIN_IN(gpj1-2, DOWN, FAST_SR1);
diff --git a/arch/arm64/boot/dts/exynos/exynos5433.dtsi b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
index d318c87effaa69..a6352c015e161c 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
@@ -93,8 +93,8 @@
 		cpu0: cpu@100 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a53";
-			enable-method = "psci";
 			reg = <0x100>;
+			enable-method = "psci";
 			clocks = <&cmu_apollo CLK_SCLK_APOLLO>;
 			clock-names = "apolloclk";
 			operating-points-v2 = <&cluster_a53_opp_table>;
@@ -111,8 +111,8 @@
 		cpu1: cpu@101 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a53";
-			enable-method = "psci";
 			reg = <0x101>;
+			enable-method = "psci";
 			operating-points-v2 = <&cluster_a53_opp_table>;
 			#cooling-cells = <2>;
 			i-cache-size = <0x8000>;
@@ -127,8 +127,8 @@
 		cpu2: cpu@102 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a53";
-			enable-method = "psci";
 			reg = <0x102>;
+			enable-method = "psci";
 			operating-points-v2 = <&cluster_a53_opp_table>;
 			#cooling-cells = <2>;
 			i-cache-size = <0x8000>;
@@ -143,8 +143,8 @@
 		cpu3: cpu@103 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a53";
-			enable-method = "psci";
 			reg = <0x103>;
+			enable-method = "psci";
 			operating-points-v2 = <&cluster_a53_opp_table>;
 			#cooling-cells = <2>;
 			i-cache-size = <0x8000>;
@@ -159,8 +159,8 @@
 		cpu4: cpu@0 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
-			enable-method = "psci";
 			reg = <0x0>;
+			enable-method = "psci";
 			clocks = <&cmu_atlas CLK_SCLK_ATLAS>;
 			clock-names = "atlasclk";
 			operating-points-v2 = <&cluster_a57_opp_table>;
@@ -177,8 +177,8 @@
 		cpu5: cpu@1 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
-			enable-method = "psci";
 			reg = <0x1>;
+			enable-method = "psci";
 			operating-points-v2 = <&cluster_a57_opp_table>;
 			#cooling-cells = <2>;
 			i-cache-size = <0xc000>;
@@ -193,8 +193,8 @@
 		cpu6: cpu@2 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
-			enable-method = "psci";
 			reg = <0x2>;
+			enable-method = "psci";
 			operating-points-v2 = <&cluster_a57_opp_table>;
 			#cooling-cells = <2>;
 			i-cache-size = <0xc000>;
@@ -209,8 +209,8 @@
 		cpu7: cpu@3 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a57";
-			enable-method = "psci";
 			reg = <0x3>;
+			enable-method = "psci";
 			operating-points-v2 = <&cluster_a57_opp_table>;
 			#cooling-cells = <2>;
 			i-cache-size = <0xc000>;
@@ -385,9 +385,9 @@
 
 	soc: soc@0 {
 		compatible = "simple-bus";
+		ranges = <0x0 0x0 0x0 0x18000000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-		ranges = <0x0 0x0 0x0 0x18000000>;
 
 		chipid@10000000 {
 			compatible = "samsung,exynos5433-chipid",
@@ -1005,13 +1005,13 @@
 				      "pclk_smmu_decon1x", "sclk_decon_vclk",
 				      "sclk_decon_eclk", "dsd";
 			power-domains = <&pd_disp>;
-			interrupt-names = "fifo", "vsync", "lcd_sys";
 			interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>;
-			samsung,disp-sysreg = <&syscon_disp>;
+			interrupt-names = "fifo", "vsync", "lcd_sys";
 			iommus = <&sysmmu_decon0x>, <&sysmmu_decon1x>;
 			iommu-names = "m0", "m1";
+			samsung,disp-sysreg = <&syscon_disp>;
 
 			status = "disabled";
 
@@ -1049,14 +1049,14 @@
 				      "aclk_smmu_decon1x", "aclk_xiu_decon1x",
 				      "pclk_smmu_decon1x", "sclk_decon_vclk",
 				      "sclk_decon_eclk", "dsd";
-			samsung,disp-sysreg = <&syscon_disp>;
 			power-domains = <&pd_disp>;
-			interrupt-names = "fifo", "vsync", "lcd_sys";
 			interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "fifo", "vsync", "lcd_sys";
 			iommus = <&sysmmu_tv0x>, <&sysmmu_tv1x>;
 			iommu-names = "m0", "m1";
+			samsung,disp-sysreg = <&syscon_disp>;
 
 			status = "disabled";
 		};
@@ -1151,9 +1151,9 @@
 				      "oscclk", "i_spdif_clk";
 			phy = <&hdmiphy>;
 			ddc = <&hsi2c_11>;
+			#sound-dai-cells = <0>;
 			samsung,syscon-phandle = <&pmu_system_controller>;
 			samsung,sysreg-phandle = <&syscon_disp>;
-			#sound-dai-cells = <0>;
 
 			status = "disabled";
 		};
@@ -1479,8 +1479,8 @@
 			clocks = <&cmu_peric CLK_PCLK_UART0>,
 				 <&cmu_peric CLK_SCLK_UART0>;
 			clock-names = "uart", "clk_uart_baud0";
-			pinctrl-names = "default";
 			pinctrl-0 = <&uart0_bus>;
+			pinctrl-names = "default";
 
 			status = "disabled";
 		};
@@ -1492,8 +1492,8 @@
 			clocks = <&cmu_peric CLK_PCLK_UART1>,
 				 <&cmu_peric CLK_SCLK_UART1>;
 			clock-names = "uart", "clk_uart_baud0";
-			pinctrl-names = "default";
 			pinctrl-0 = <&uart1_bus>;
+			pinctrl-names = "default";
 
 			status = "disabled";
 		};
@@ -1505,8 +1505,8 @@
 			clocks = <&cmu_peric CLK_PCLK_UART2>,
 				 <&cmu_peric CLK_SCLK_UART2>;
 			clock-names = "uart", "clk_uart_baud0";
-			pinctrl-names = "default";
 			pinctrl-0 = <&uart2_bus>;
+			pinctrl-names = "default";
 
 			status = "disabled";
 		};
@@ -1523,11 +1523,11 @@
 				 <&cmu_peric CLK_SCLK_SPI0>,
 				 <&cmu_peric CLK_SCLK_IOCLK_SPI0>;
 			clock-names = "spi", "spi_busclk0", "spi_ioclk";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&spi0_bus>;
+			pinctrl-names = "default";
 			num-cs = <1>;
 			fifo-depth = <256>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
@@ -1544,11 +1544,11 @@
 				 <&cmu_peric CLK_SCLK_SPI1>,
 				 <&cmu_peric CLK_SCLK_IOCLK_SPI1>;
 			clock-names = "spi", "spi_busclk0", "spi_ioclk";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&spi1_bus>;
+			pinctrl-names = "default";
 			num-cs = <1>;
 			fifo-depth = <64>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
@@ -1565,11 +1565,11 @@
 				 <&cmu_peric CLK_SCLK_SPI2>,
 				 <&cmu_peric CLK_SCLK_IOCLK_SPI2>;
 			clock-names = "spi", "spi_busclk0", "spi_ioclk";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&spi2_bus>;
+			pinctrl-names = "default";
 			num-cs = <1>;
 			fifo-depth = <64>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
@@ -1586,11 +1586,11 @@
 				 <&cmu_peric CLK_SCLK_SPI3>,
 				 <&cmu_peric CLK_SCLK_IOCLK_SPI3>;
 			clock-names = "spi", "spi_busclk0", "spi_ioclk";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&spi3_bus>;
+			pinctrl-names = "default";
 			num-cs = <1>;
 			fifo-depth = <64>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
@@ -1607,11 +1607,11 @@
 				 <&cmu_peric CLK_SCLK_SPI4>,
 				 <&cmu_peric CLK_SCLK_IOCLK_SPI4>;
 			clock-names = "spi", "spi_busclk0", "spi_ioclk";
-			samsung,spi-src-clk = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&spi4_bus>;
+			pinctrl-names = "default";
 			num-cs = <1>;
 			fifo-depth = <64>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
@@ -1651,10 +1651,10 @@
 				     <GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH>;
-			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
 			clocks = <&cmu_peric CLK_PCLK_PWM>;
 			clock-names = "timers";
 			#pwm-cells = <3>;
+			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
 
 			status = "disabled";
 		};
@@ -1666,8 +1666,8 @@
 			interrupts = <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c0_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C0>;
 			clock-names = "hsi2c";
 
@@ -1681,8 +1681,8 @@
 			interrupts = <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c1_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C1>;
 			clock-names = "hsi2c";
 
@@ -1696,8 +1696,8 @@
 			interrupts = <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c2_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C2>;
 			clock-names = "hsi2c";
 
@@ -1711,8 +1711,8 @@
 			interrupts = <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c3_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C3>;
 			clock-names = "hsi2c";
 
@@ -1726,8 +1726,8 @@
 			interrupts = <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c4_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C4>;
 			clock-names = "hsi2c";
 
@@ -1741,8 +1741,8 @@
 			interrupts = <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c5_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C5>;
 			clock-names = "hsi2c";
 
@@ -1756,8 +1756,8 @@
 			interrupts = <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c6_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C6>;
 			clock-names = "hsi2c";
 
@@ -1771,8 +1771,8 @@
 			interrupts = <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c7_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C7>;
 			clock-names = "hsi2c";
 
@@ -1786,8 +1786,8 @@
 			interrupts = <GIC_SPI 443 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c8_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C8>;
 			clock-names = "hsi2c";
 
@@ -1801,8 +1801,8 @@
 			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c9_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C9>;
 			clock-names = "hsi2c";
 
@@ -1816,8 +1816,8 @@
 			interrupts = <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c10_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C10>;
 			clock-names = "hsi2c";
 
@@ -1831,8 +1831,8 @@
 			interrupts = <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&hs_i2c11_bus>;
+			pinctrl-names = "default";
 			clocks = <&cmu_peric CLK_PCLK_HSI2C11>;
 			clock-names = "hsi2c";
 
@@ -1841,6 +1841,7 @@
 
 		usbdrd30: usb@15400000 {
 			compatible = "samsung,exynos5433-dwusb3";
+			ranges = <0x0 0x15400000 0x10000>;
 			clocks = <&cmu_fsys CLK_ACLK_USBDRD30>,
 				 <&cmu_fsys CLK_SCLK_USBDRD30>,
 				 <&cmu_fsys CLK_PHYCLK_USBDRD30_UDRD30_PHYCLOCK>,
@@ -1848,17 +1849,16 @@
 			clock-names = "aclk", "susp_clk", "phyclk", "pipe_pclk";
 			#address-cells = <1>;
 			#size-cells = <1>;
-			ranges = <0x0 0x15400000 0x10000>;
 
 			status = "disabled";
 
 			usbdrd_dwc3: usb@0 {
 				compatible = "snps,dwc3";
+				reg = <0x0 0x10000>;
 				clocks = <&cmu_fsys CLK_SCLK_USBDRD30>,
 					 <&cmu_fsys CLK_ACLK_USBDRD30>,
 					 <&cmu_fsys CLK_SCLK_USBDRD30>;
 				clock-names = "ref", "bus_early", "suspend";
-				reg = <0x0 0x10000>;
 				interrupts = <GIC_SPI 231 IRQ_TYPE_LEVEL_HIGH>;
 				phys = <&usbdrd30_phy 0>, <&usbdrd30_phy 1>;
 				phy-names = "usb2-phy", "usb3-phy";
@@ -1895,6 +1895,7 @@
 
 		usbhost30: usb@15a00000 {
 			compatible = "samsung,exynos5433-dwusb3";
+			ranges = <0x0 0x15a00000 0x10000>;
 			clocks = <&cmu_fsys CLK_ACLK_USBHOST30>,
 				 <&cmu_fsys CLK_SCLK_USBHOST30>,
 				 <&cmu_fsys CLK_PHYCLK_USBHOST30_UHOST30_PHYCLOCK>,
@@ -1902,17 +1903,16 @@
 			clock-names = "aclk", "susp_clk", "phyclk", "pipe_pclk";
 			#address-cells = <1>;
 			#size-cells = <1>;
-			ranges = <0x0 0x15a00000 0x10000>;
 
 			status = "disabled";
 
 			usbhost_dwc3: usb@0 {
 				compatible = "snps,dwc3";
+				reg = <0x0 0x10000>;
 				clocks = <&cmu_fsys CLK_SCLK_USBHOST30>,
 					 <&cmu_fsys CLK_ACLK_USBHOST30>,
 					 <&cmu_fsys CLK_SCLK_USBHOST30>;
 				clock-names = "ref", "bus_early", "suspend";
-				reg = <0x0 0x10000>;
 				interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>;
 				phys = <&usbhost30_phy 0>, <&usbhost30_phy 1>;
 				phy-names = "usb2-phy", "usb3-phy";
@@ -1922,10 +1922,10 @@
 		mshc_0: mmc@15540000 {
 			compatible = "samsung,exynos5433-dw-mshc-smu",
 				     "samsung,exynos7-dw-mshc-smu";
+			reg = <0x15540000 0x2000>;
 			interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			reg = <0x15540000 0x2000>;
 			clocks = <&cmu_fsys CLK_ACLK_MMC0>,
 				 <&cmu_fsys CLK_SCLK_MMC0>;
 			clock-names = "biu", "ciu";
@@ -1937,10 +1937,10 @@
 		mshc_1: mmc@15550000 {
 			compatible = "samsung,exynos5433-dw-mshc-smu",
 				     "samsung,exynos7-dw-mshc-smu";
+			reg = <0x15550000 0x2000>;
 			interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			reg = <0x15550000 0x2000>;
 			clocks = <&cmu_fsys CLK_ACLK_MMC1>,
 				 <&cmu_fsys CLK_SCLK_MMC1>;
 			clock-names = "biu", "ciu";
@@ -1952,10 +1952,10 @@
 		mshc_2: mmc@15560000 {
 			compatible = "samsung,exynos5433-dw-mshc-smu",
 				     "samsung,exynos7-dw-mshc-smu";
+			reg = <0x15560000 0x2000>;
 			interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			reg = <0x15560000 0x2000>;
 			clocks = <&cmu_fsys CLK_ACLK_MMC2>,
 				 <&cmu_fsys CLK_SCLK_MMC2>;
 			clock-names = "biu", "ciu";
@@ -1985,12 +1985,12 @@
 		audio-subsystem@11400000 {
 			compatible = "samsung,exynos5433-lpass";
 			reg = <0x11400000 0x100>, <0x11500000 0x08>;
+			ranges;
 			clocks = <&cmu_aud CLK_PCLK_SFR0_CTRL>;
 			clock-names = "sfr0_ctrl";
 			power-domains = <&pd_aud>;
 			#address-cells = <1>;
 			#size-cells = <1>;
-			ranges;
 
 			adma: dma-controller@11420000 {
 				compatible = "arm,pl330", "arm,primecell";
@@ -2016,8 +2016,8 @@
 					 <&cmu_aud CLK_SCLK_I2S_BCLK>;
 				clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
 				#clock-cells = <1>;
-				pinctrl-names = "default";
 				pinctrl-0 = <&i2s0_bus>;
+				pinctrl-names = "default";
 				power-domains = <&pd_aud>;
 				#sound-dai-cells = <1>;
 
@@ -2031,8 +2031,8 @@
 				clocks = <&cmu_aud CLK_PCLK_AUD_UART>,
 					 <&cmu_aud CLK_SCLK_AUD_UART>;
 				clock-names = "uart", "clk_uart_baud0";
-				pinctrl-names = "default";
 				pinctrl-0 = <&uart_aud_bus>;
+				pinctrl-names = "default";
 				power-domains = <&pd_aud>;
 
 				status = "disabled";
@@ -2042,22 +2042,24 @@
 		pcie_phy: pcie-phy@15680000 {
 			compatible = "samsung,exynos5433-pcie-phy";
 			reg = <0x15680000 0x1000>;
+			#phy-cells = <0>;
 			samsung,pmu-syscon = <&pmu_system_controller>;
 			samsung,fsys-sysreg = <&syscon_fsys>;
-			#phy-cells = <0>;
 
 			status = "disabled";
 		};
 
 		pcie: pcie@15700000 {
+			device_type = "pci";
 			compatible = "samsung,exynos5433-pcie";
 			reg = <0x15700000 0x1000>, <0x156b0000 0x1000>,
 			      <0x0c000000 0x1000>;
 			reg-names = "dbi", "elbi", "config";
+			ranges = <0x81000000 0 0	  0x0c001000 0 0x00010000>,
+				 <0x82000000 0 0x0c011000 0x0c011000 0 0x03feefff>;
 			#address-cells = <3>;
 			#size-cells = <2>;
 			#interrupt-cells = <1>;
-			device_type = "pci";
 			interrupts = <GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cmu_fsys CLK_PCIE>,
 				 <&cmu_fsys CLK_PCLK_PCIE_PHY>;
@@ -2066,8 +2068,6 @@
 			num-viewport = <3>;
 			bus-range = <0x00 0xff>;
 			phys = <&pcie_phy>;
-			ranges = <0x81000000 0 0	  0x0c001000 0 0x00010000>,
-				 <0x82000000 0 0x0c011000 0x0c011000 0 0x03feefff>;
 
 			status = "disabled";
 		};
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts b/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts
index 0888cd2faca4a6..67f317afd85e6d 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-a2corelte.dts
@@ -34,8 +34,8 @@
 		compatible = "gpio-keys";
 		label = "GPIO Keys";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&key_power &key_voldown &key_volup>;
+		pinctrl-names = "default";
 
 		key-power {
 			label = "Power Key";
@@ -86,9 +86,9 @@
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		ramoops@46800000 {
 			compatible = "ramoops";
@@ -166,8 +166,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&pmic_irq>;
+		pinctrl-names = "default";
 
 		regulators {
 			vdd_buck1: buck1 {
@@ -402,8 +402,8 @@
 		interrupt-parent = <&gpa2>;
 		interrupts = <3 IRQ_TYPE_EDGE_RISING>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&accel_irq>;
+		pinctrl-names = "default";
 
 		st,drdy-int-pin = <1>;
 	};
@@ -414,8 +414,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <5 IRQ_TYPE_EDGE_BOTH>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&proxm_irq>;
+		pinctrl-names = "default";
 
 		proximity-near-level = <25>;
 	};
@@ -435,8 +435,8 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&touch_irq>;
+		pinctrl-names = "default";
 
 		vdd-supply = <&vdd_ldo35>;
 
@@ -461,18 +461,19 @@
 };
 
 &mmc0 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_rdqs &sd0_bus1 &sd0_bus4 &sd0_bus8>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_ldo26>;
 	vqmmc-supply = <&vdd_ldo27>;
 
 	bus-width = <8>;
 	fifo-depth = <64>;
+	non-removable;
+
 	samsung,dw-mshc-ciu-div = <3>;
 	samsung,dw-mshc-sdr-timing = <0 4>;
 	samsung,dw-mshc-ddr-timing = <2 4>;
-	non-removable;
 
 	status = "okay";
 };
@@ -481,36 +482,37 @@
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus1 &sd1_bus4>;
+	pinctrl-names = "default";
 
 	mmc-pwrseq = <&pwrseq_mmc1>;
 
 	bus-width = <4>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	non-removable;
 	cap-sd-highspeed;
 	cap-sdio-irq;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 
 	wifi@1 {
 		compatible = "brcm,bcm43430a1-fmac", "brcm,bcm4329-fmac";
 		reg = <0x1>;
-		interrupt-names = "host-wake";
 		interrupt-parent = <&gpa2>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-names = "host-wake";
 
 		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
 	};
 };
 
 &mmc2 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus1 &sd2_bus4 &dwmmc2_irq>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_fixed_mmc2>;
 	vqmmc-supply = <&vdd_ldo2>;
@@ -518,14 +520,15 @@
 	bus-width = <4>;
 	card-detect-delay = <200>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	sd-uhs-sdr50;
 	sd-uhs-sdr104;
 	broken-cd;
 	disable-wp;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 };
 
@@ -629,8 +632,8 @@
 	bluetooth {
 		compatible = "brcm,bcm43430a1-bt";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&bt_btwake &bt_hostwake &bt_enable>;
+		pinctrl-names = "default";
 
 		device-wakeup-gpios = <&gpa1 2 GPIO_ACTIVE_HIGH>;
 		host-wakeup-gpios = <&gpa1 6 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts b/arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts
index 36c327668db516..84421a3d47ca04 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-j5y17lte.dts
@@ -26,9 +26,9 @@
 	};
 
 	chosen {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		stdout-path = &serial2;
 
@@ -44,35 +44,35 @@
 
 	gpio-keys {
 		compatible = "gpio-keys";
-		pinctrl-names = "default";
 		pinctrl-0 = <&key_power &key_voldown &key_volup>;
+		pinctrl-names = "default";
 
 		key-home {
 			interrupt-parent = <&gpa1>;
-			linux,code = <KEY_HOMEPAGE>;
 			label = "gpio-keys: KEY_HOMEPAGE";
 			gpios = <&gpa1 7 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_HOMEPAGE>;
 		};
 
 		key-power {
 			interrupt-parent = <&gpa0>;
-			linux,code = <KEY_POWER>;
 			label = "gpio-keys: KEY_POWER";
 			gpios = <&gpa0 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_POWER>;
 		};
 
 		key-voldown {
 			interrupt-parent = <&gpa2>;
-			linux,code = <KEY_VOLUMEDOWN>;
 			label = "gpio-keys: KEY_VOLUMEDOWN";
 			gpios = <&gpa2 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
 		};
 
 		key-volup {
 			interrupt-parent = <&gpa2>;
-			linux,code = <KEY_VOLUMEUP>;
 			label = "gpio-keys: KEY_VOLUMEUP";
 			gpios = <&gpa2 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
 		};
 	};
 
@@ -98,9 +98,9 @@
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		ramoops@46e00000 {
 			compatible = "ramoops";
@@ -137,8 +137,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&pmic_irq>;
+		pinctrl-names = "default";
 
 		regulators {
 			vdd_buck1: buck1 {
@@ -368,8 +368,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&touch_irq>;
+		pinctrl-names = "default";
 
 		touchscreen-size-x = <720>;
 		touchscreen-size-y = <1280>;
@@ -379,17 +379,18 @@
 };
 
 &mmc0 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_rdqs &sd0_bus1 &sd0_bus4 &sd0_bus8>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_ldo26>;
 	vqmmc-supply = <&vdd_ldo27>;
 
 	fifo-depth = <64>;
+	non-removable;
+
 	samsung,dw-mshc-ciu-div = <3>;
 	samsung,dw-mshc-sdr-timing = <0 4>;
 	samsung,dw-mshc-ddr-timing = <2 4>;
-	non-removable;
 
 	status = "okay";
 };
@@ -398,29 +399,30 @@
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus1 &sd1_bus4>;
+	pinctrl-names = "default";
 
 	mmc-pwrseq = <&pwrseq_mmc1>;
 
 	bus-width = <4>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	non-removable;
 	cap-sd-highspeed;
 	cap-sdio-irq;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 
 	wifi@0 {
 		compatible = "brcm,bcm43455-fmac", "brcm,bcm4329-fmac";
 		reg = <0x0>;
 
-		interrupt-names = "host-wake";
 		interrupt-parent = <&gpa2>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-names = "host-wake";
 
 		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
 	};
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts b/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts
index 02bab3ab2711c1..8bd6892ae796a9 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-j6lte.dts
@@ -34,16 +34,16 @@
 		compatible = "gpio-keys";
 		label = "GPIO Hall Effect Sensor";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&hall_irq>;
+		pinctrl-names = "default";
 
 		event-hall-effect-sensor {
 			label = "Hall Effect Sensor";
 			gpios = <&gpa1 3 GPIO_ACTIVE_LOW>;
+			wakeup-source;
 			linux,input-type = <EV_SW>;
 			linux,code = <SW_LID>;
 			linux,can-disable;
-			wakeup-source;
 		};
 	};
 
@@ -51,8 +51,8 @@
 		compatible = "gpio-keys";
 		label = "GPIO Keys";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&key_power &key_voldown &key_volup>;
+		pinctrl-names = "default";
 
 		key-power {
 			label = "Power Key";
@@ -95,9 +95,9 @@
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		ramoops@46e00000 {
 			compatible = "ramoops";
@@ -159,8 +159,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&pmic_irq>;
+		pinctrl-names = "default";
 
 		regulators {
 			vdd_buck1: buck1 {
@@ -394,8 +394,8 @@
 		interrupt-parent = <&gpa2>;
 		interrupts = <3 IRQ_TYPE_EDGE_RISING>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&accel_irq>;
+		pinctrl-names = "default";
 
 		mount-matrix = "-1", "0",  "0",
 			       "0", "1",  "0",
@@ -417,8 +417,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&touch_irq>;
+		pinctrl-names = "default";
 
 		touchscreen-size-x = <720>;
 		touchscreen-size-y = <1480>;
@@ -428,18 +428,19 @@
 };
 
 &mmc0 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_rdqs &sd0_bus1 &sd0_bus4 &sd0_bus8>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_ldo26>;
 	vqmmc-supply = <&vdd_ldo27>;
 
 	bus-width = <8>;
 	fifo-depth = <64>;
+	non-removable;
+
 	samsung,dw-mshc-ciu-div = <3>;
 	samsung,dw-mshc-sdr-timing = <0 4>;
 	samsung,dw-mshc-ddr-timing = <2 4>;
-	non-removable;
 
 	status = "okay";
 };
@@ -448,36 +449,37 @@
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus1 &sd1_bus4>;
+	pinctrl-names = "default";
 
 	mmc-pwrseq = <&pwrseq_mmc1>;
 
 	bus-width = <4>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	non-removable;
 	cap-sd-highspeed;
 	cap-sdio-irq;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 
 	wifi@1 {
 		compatible = "brcm,bcm43430a1-fmac", "brcm,bcm4329-fmac";
 		reg = <0x1>;
-		interrupt-names = "host-wake";
 		interrupt-parent = <&gpa2>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-names = "host-wake";
 
 		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
 	};
 };
 
 &mmc2 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus1 &sd2_bus4 &dwmmc2_irq>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_fixed_mmc2>;
 	vqmmc-supply = <&vdd_ldo2>;
@@ -596,8 +598,8 @@
 	bluetooth {
 		compatible = "brcm,bcm43430a1-bt";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&bt_btwake &bt_hostwake &bt_enable>;
+		pinctrl-names = "default";
 
 		device-wakeup-gpios = <&gpa1 2 GPIO_ACTIVE_HIGH>;
 		host-wakeup-gpios = <&gpa1 6 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts b/arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts
index 079b2b581307c6..d258eecfbf42ca 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-j7xelte.dts
@@ -26,9 +26,9 @@
 	};
 
 	chosen {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		stdout-path = &serial2;
 
@@ -46,8 +46,8 @@
 		compatible = "gpio-keys";
 		label = "GPIO Keys";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&key_home &key_power &key_voldown &key_volup>;
+		pinctrl-names = "default";
 
 		key-home {
 			label = "Home Key";
@@ -86,9 +86,9 @@
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		ramoops@46e00000 {
 			compatible = "ramoops";
@@ -136,8 +136,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&pmic_irq>;
+		pinctrl-names = "default";
 
 		regulators {
 			vdd_buck1: buck1 {
@@ -346,23 +346,24 @@
 		interrupt-parent = <&gpc3>;
 		interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&touch_irq>;
+		pinctrl-names = "default";
 	};
 };
 
 &mmc0 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_rdqs &sd0_bus1 &sd0_bus4 &sd0_bus8>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_ldo26>;
 	vqmmc-supply = <&vdd_ldo27>;
 
 	fifo-depth = <64>;
+	non-removable;
+
 	samsung,dw-mshc-ciu-div = <3>;
 	samsung,dw-mshc-sdr-timing = <0 4>;
 	samsung,dw-mshc-ddr-timing = <2 4>;
-	non-removable;
 
 	status = "okay";
 };
@@ -371,37 +372,38 @@
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus1 &sd1_bus4>;
+	pinctrl-names = "default";
 
 	mmc-pwrseq = <&pwrseq_mmc1>;
 
 	bus-width = <4>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	non-removable;
 	cap-sd-highspeed;
 	cap-sdio-irq;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 
 	wifi@1 {
 		compatible = "brcm,bcm43430a1-fmac", "brcm,bcm4329-fmac";
 		reg = <0x1>;
 
-		interrupt-names = "host-wake";
 		interrupt-parent = <&gpa2>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-names = "host-wake";
 
 		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
 	};
 };
 
 &mmc2 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus1 &sd2_bus4 &dwmmc2_irq>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_fixed_mmc2>;
 	vqmmc-supply = <&vdd_ldo2>;
@@ -409,14 +411,15 @@
 	bus-width = <4>;
 	card-detect-delay = <200>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	sd-uhs-sdr50;
 	sd-uhs-sdr104;
 	broken-cd;
 	disable-wp;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts b/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts
index 29be4764f84d34..5bb95aab75b2df 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7870-on7xelte.dts
@@ -34,8 +34,8 @@
 		compatible = "gpio-keys";
 		label = "GPIO Keys";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&key_home &key_power &key_voldown &key_volup>;
+		pinctrl-names = "default";
 
 		key-home {
 			label = "Home Key";
@@ -84,9 +84,9 @@
 	};
 
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		ramoops@46e00000 {
 			compatible = "ramoops";
@@ -168,8 +168,8 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&pmic_irq>;
+		pinctrl-names = "default";
 
 		regulators {
 			vdd_buck1: buck1 {
@@ -411,8 +411,8 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&touch_irq>;
+		pinctrl-names = "default";
 
 		syna,reset-delay-ms = <200>;
 		syna,startup-delay-ms = <200>;
@@ -446,8 +446,8 @@
 		interrupt-parent = <&gpa2>;
 		interrupts = <3 IRQ_TYPE_EDGE_RISING>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&accel_irq>;
+		pinctrl-names = "default";
 
 		mount-matrix = "1",  "0",  "0",
 			       "0", "-1",  "0",
@@ -469,26 +469,27 @@
 		interrupt-parent = <&gpa0>;
 		interrupts = <5 IRQ_TYPE_EDGE_BOTH>;
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&proxm_irq>;
+		pinctrl-names = "default";
 
 		proximity-near-level = <25>;
 	};
 };
 
 &mmc0 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_rdqs &sd0_bus1 &sd0_bus4 &sd0_bus8>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_ldo26>;
 	vqmmc-supply = <&vdd_ldo27>;
 
 	bus-width = <8>;
 	fifo-depth = <64>;
+	non-removable;
+
 	samsung,dw-mshc-ciu-div = <3>;
 	samsung,dw-mshc-sdr-timing = <0 4>;
 	samsung,dw-mshc-ddr-timing = <2 4>;
-	non-removable;
 
 	status = "okay";
 };
@@ -497,36 +498,37 @@
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd1_clk &sd1_cmd &sd1_bus1 &sd1_bus4>;
+	pinctrl-names = "default";
 
 	mmc-pwrseq = <&pwrseq_mmc1>;
 
 	bus-width = <4>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	non-removable;
 	cap-sd-highspeed;
 	cap-sdio-irq;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 
 	wifi@1 {
 		compatible = "brcm,bcm43430a1-fmac", "brcm,bcm4329-fmac";
 		reg = <0x1>;
-		interrupt-names = "host-wake";
 		interrupt-parent = <&gpa2>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-names = "host-wake";
 
 		reset-gpios = <&gpd3 6 GPIO_ACTIVE_LOW>;
 	};
 };
 
 &mmc2 {
-	pinctrl-names = "default";
 	pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_bus1 &sd2_bus4 &dwmmc2_irq>;
+	pinctrl-names = "default";
 
 	vmmc-supply = <&vdd_fixed_mmc2>;
 	vqmmc-supply = <&vdd_ldo2>;
@@ -534,14 +536,15 @@
 	bus-width = <4>;
 	card-detect-delay = <200>;
 	fifo-depth = <64>;
-	samsung,dw-mshc-ciu-div = <3>;
-	samsung,dw-mshc-sdr-timing = <0 3>;
-	samsung,dw-mshc-ddr-timing = <1 2>;
 	sd-uhs-sdr50;
 	sd-uhs-sdr104;
 	broken-cd;
 	disable-wp;
 
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 3>;
+	samsung,dw-mshc-ddr-timing = <1 2>;
+
 	status = "okay";
 };
 
@@ -661,8 +664,8 @@
 	bluetooth {
 		compatible = "brcm,bcm43430a1-bt";
 
-		pinctrl-names = "default";
 		pinctrl-0 = <&bt_btwake &bt_hostwake &bt_enable>;
+		pinctrl-names = "default";
 
 		device-wakeup-gpios = <&gpa1 2 GPIO_ACTIVE_HIGH>;
 		host-wakeup-gpios = <&gpa1 6 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm64/boot/dts/exynos/exynos7870.dtsi b/arch/arm64/boot/dts/exynos/exynos7870.dtsi
index 9e3ec6c952f0c9..c397f577d065c5 100644
--- a/arch/arm64/boot/dts/exynos/exynos7870.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7870.dtsi
@@ -156,8 +156,6 @@
 			reg = <0x101f0000 0x1000>;
 			#clock-cells = <1>;
 
-			clock-names = "oscclk", "bus", "spi0", "spi1", "spi2",
-				      "spi3", "spi4", "uart0", "uart1", "uart2";
 			clocks = <&oscclk>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_PERI_BUS>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_PERI_SPI0>,
@@ -168,6 +166,8 @@
 				 <&cmu_mif CLK_GOUT_MIF_CMU_PERI_UART0>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_PERI_UART1>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_PERI_UART2>;
+			clock-names = "oscclk", "bus", "spi0", "spi1", "spi2",
+				      "spi3", "spi4", "uart0", "uart1", "uart2";
 		};
 
 		cmu_mif: clock-controller@10460000 {
@@ -175,8 +175,8 @@
 			reg = <0x10460000 0x1000>;
 			#clock-cells = <1>;
 
-			clock-names = "oscclk";
 			clocks = <&oscclk>;
+			clock-names = "oscclk";
 		};
 
 		pmu_system_controller: system-controller@10480000 {
@@ -219,11 +219,11 @@
 			reg = <0x10510000 0x2000>;
 			interrupts = <GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&hsi2c0_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "hsi2c";
 			clocks = <&cmu_mif CLK_GOUT_MIF_HSI2C_IPCLK>;
+			clock-names = "hsi2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -240,14 +240,14 @@
 		gpu: gpu@11400000 {
 			compatible = "samsung,exynos7870-mali", "arm,mali-t830";
 			reg = <0x11400000 0x5000>;
-			interrupt-names = "job", "mmu", "gpu";
 			interrupts = <GIC_SPI 282 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 283 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 281 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "job", "mmu", "gpu";
 
-			clock-names = "core", "bus";
 			clocks = <&cmu_g3d CLK_GOUT_G3D_CLK>,
 				 <&cmu_g3d CLK_GOUT_G3D_ASYNCS_D0_CLK>;
+			clock-names = "core", "bus";
 
 			status = "disabled";
 		};
@@ -257,9 +257,9 @@
 			reg = <0x11460000 0x1000>;
 			#clock-cells = <1>;
 
-			clock-names = "oscclk", "switch";
 			clocks = <&oscclk>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_G3D_SWITCH>;
+			clock-names = "oscclk", "switch";
 		};
 
 		cmu_mfcmscl: clock-controller@12cb0000 {
@@ -267,10 +267,10 @@
 			reg = <0x12cb0000 0x1000>;
 			#clock-cells = <1>;
 
-			clock-names = "oscclk", "mfc", "mscl";
 			clocks = <&oscclk>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_MFCMSCL_MFC>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_MFCMSCL_MSCL>;
+			clock-names = "oscclk", "mfc", "mscl";
 		};
 
 		mmc0: mmc@13540000 {
@@ -278,9 +278,9 @@
 			reg = <0x13540000 0x2000>;
 			interrupts = <GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>;
 
-			clock-names = "biu", "ciu";
 			clocks = <&cmu_fsys CLK_GOUT_FSYS_MMC0_ACLK>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_FSYS_MMC0>;
+			clock-names = "biu", "ciu";
 
 			status = "disabled";
 		};
@@ -290,9 +290,9 @@
 			reg = <0x13550000 0x2000>;
 			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
 
-			clock-names = "biu", "ciu";
 			clocks = <&cmu_fsys CLK_GOUT_FSYS_MMC1_ACLK>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_FSYS_MMC1>;
+			clock-names = "biu", "ciu";
 
 			status = "disabled";
 		};
@@ -302,9 +302,9 @@
 			reg = <0x13560000 0x2000>;
 			interrupts = <GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>;
 
-			clock-names = "biu", "ciu";
 			clocks = <&cmu_fsys CLK_GOUT_FSYS_MMC2_ACLK>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_FSYS_MMC2>;
+			clock-names = "biu", "ciu";
 
 			status = "disabled";
 		};
@@ -314,9 +314,9 @@
 			reg = <0x135c0000 0x100>;
 			#phy-cells = <1>;
 
-			clock-names = "phy", "ref";
 			clocks = <&cmu_fsys CLK_GOUT_FSYS_MUX_USB20DRD_PHYCLOCK_USER>,
 				 <&cmu_fsys CLK_GOUT_FSYS_MUX_USB_PLL>;
+			clock-names = "phy", "ref";
 
 			samsung,pmu-syscon = <&pmu_system_controller>;
 		};
@@ -327,10 +327,10 @@
 			#address-cells = <1>;
 			#size-cells = <1>;
 
-			clock-names = "bus_early", "ref", "ctrl";
 			clocks = <&cmu_fsys CLK_GOUT_FSYS_USB20DRD_ACLK_HSDRD>,
 				 <&cmu_fsys CLK_GOUT_FSYS_USB20DRD_HSDRD_REF_CLK>,
 				 <&cmu_fsys CLK_GOUT_FSYS_USB20DRD_HCLK_USB20_CTRL>;
+			clock-names = "bus_early", "ref", "ctrl";
 
 			status = "disabled";
 
@@ -339,8 +339,8 @@
 				reg = <0x0 0x10000>;
 				interrupts = <GIC_SPI 230 IRQ_TYPE_LEVEL_HIGH>;
 
-				phy-names = "usb2-phy";
 				phys = <&usbdrd_phy 0>;
+				phy-names = "usb2-phy";
 
 				usb-role-switch;
 				snps,usb2-gadget-lpm-disable;
@@ -352,10 +352,10 @@
 			reg = <0x13730000 0x1000>;
 			#clock-cells = <1>;
 
-			clock-names = "oscclk", "bus", "usb20drd";
 			clocks = <&oscclk>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_FSYS_BUS>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_FSYS_USB20DRD_REFCLK>;
+			clock-names = "oscclk", "bus", "usb20drd";
 		};
 
 		pinctrl_fsys: pinctrl@13750000 {
@@ -370,12 +370,12 @@
 			reg = <0x13800000 0x100>;
 			interrupts = <GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&uart0_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "uart", "clk_uart_baud0";
 			clocks = <&cmu_peri CLK_GOUT_PERI_UART0_PCLK>,
 				 <&cmu_peri CLK_GOUT_PERI_UART0_EXT_UCLK>;
+			clock-names = "uart", "clk_uart_baud0";
 
 			samsung,uart-fifosize = <16>;
 
@@ -388,12 +388,12 @@
 			reg = <0x13810000 0x100>;
 			interrupts = <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&uart1_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "uart", "clk_uart_baud0";
 			clocks = <&cmu_peri CLK_GOUT_PERI_UART1_PCLK>,
 				 <&cmu_peri CLK_GOUT_PERI_UART1_EXT_UCLK>;
+			clock-names = "uart", "clk_uart_baud0";
 
 			samsung,uart-fifosize = <256>;
 
@@ -406,12 +406,12 @@
 			reg = <0x13820000 0x100>;
 			interrupts = <GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&uart2_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "uart", "clk_uart_baud0";
 			clocks = <&cmu_peri CLK_GOUT_PERI_UART2_PCLK>,
 				 <&cmu_peri CLK_GOUT_PERI_UART2_EXT_UCLK>;
+			clock-names = "uart", "clk_uart_baud0";
 
 			samsung,uart-fifosize = <256>;
 
@@ -424,11 +424,11 @@
 			reg = <0x13830000 0x100>;
 			interrupts = <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c0_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C0_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -442,11 +442,11 @@
 			reg = <0x13840000 0x100>;
 			interrupts = <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c1_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C1_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -460,11 +460,11 @@
 			reg = <0x13850000 0x100>;
 			interrupts = <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c2_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C2_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -478,11 +478,11 @@
 			reg = <0x13860000 0x100>;
 			interrupts = <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c3_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C3_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -496,11 +496,11 @@
 			reg = <0x13870000 0x100>;
 			interrupts = <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c4_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C4_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -514,11 +514,11 @@
 			reg = <0x13880000 0x100>;
 			interrupts = <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c5_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C5_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -532,11 +532,11 @@
 			reg = <0x13890000 0x100>;
 			interrupts = <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c6_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C6_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -550,11 +550,11 @@
 			reg = <0x138a0000 0x1000>;
 			interrupts = <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&hsi2c1_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C1_IPCLK>;
+			clock-names = "hsi2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -568,11 +568,11 @@
 			reg = <0x138b0000 0x1000>;
 			interrupts = <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&hsi2c2_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C2_IPCLK>;
+			clock-names = "hsi2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -586,11 +586,11 @@
 			reg = <0x138c0000 0x1000>;
 			interrupts = <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&hsi2c3_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C3_IPCLK>;
+			clock-names = "hsi2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -604,11 +604,11 @@
 			reg = <0x138d0000 0x100>;
 			interrupts = <GIC_SPI 453 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c7_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C7_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -622,11 +622,11 @@
 			reg = <0x138e0000 0x100>;
 			interrupts = <GIC_SPI 454 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&i2c8_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "i2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_I2C8_PCLK>;
+			clock-names = "i2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -640,11 +640,11 @@
 			reg = <0x138f0000 0x1000>;
 			interrupts = <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&hsi2c4_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C4_IPCLK>;
+			clock-names = "hsi2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -658,11 +658,11 @@
 			reg = <0x13950000 0x1000>;
 			interrupts = <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&hsi2c5_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C5_IPCLK>;
+			clock-names = "hsi2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -676,11 +676,11 @@
 			reg = <0x13960000 0x1000>;
 			interrupts = <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>;
 
-			pinctrl-names = "default";
 			pinctrl-0 = <&hsi2c6_bus>;
+			pinctrl-names = "default";
 
-			clock-names = "hsi2c";
 			clocks = <&cmu_peri CLK_GOUT_PERI_HSI2C6_IPCLK>;
+			clock-names = "hsi2c";
 
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -729,11 +729,11 @@
 			reg = <0x144d0000 0x1000>;
 			#clock-cells = <1>;
 
-			clock-names = "oscclk", "cam", "isp", "vra";
 			clocks = <&oscclk>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_ISP_CAM>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_ISP_ISP>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_ISP_VRA>;
+			clock-names = "oscclk", "cam", "isp", "vra";
 		};
 
 		syscon_cam0: system-controller@144f1040 {
@@ -818,11 +818,11 @@
 			reg = <0x148d0000 0x1000>;
 			#clock-cells = <1>;
 
-			clock-names = "oscclk", "bus", "decon_eclk", "decon_vclk";
 			clocks = <&oscclk>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_DISPAUD_BUS>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_DISPAUD_DECON_ECLK>,
 				 <&cmu_mif CLK_GOUT_MIF_CMU_DISPAUD_DECON_VCLK>;
+			clock-names = "oscclk", "bus", "decon_eclk", "decon_vclk";
 		};
 
 		syscon_disp: system-controller@148f100c {
diff --git a/arch/arm64/boot/dts/exynos/exynos850-a217f.dts b/arch/arm64/boot/dts/exynos/exynos850-a217f.dts
new file mode 100644
index 00000000000000..b5b66c015e1e8c
--- /dev/null
+++ b/arch/arm64/boot/dts/exynos/exynos850-a217f.dts
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Samsung Galaxy A21s (a21s/SM-A217F) device tree source
+ *
+ * Copyright (c) 2026, Thomas Turner <tturner@lineageos.org>
+ */
+
+/dts-v1/;
+#include "exynos850.dtsi"
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	model = "Samsung Galaxy A21s";
+	compatible = "samsung,a217f", "samsung,exynos850";
+
+	chosen {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <1>;
+
+		framebuffer0: framebuffer@fa000000 {
+			compatible = "simple-framebuffer";
+			reg = <0x0 0xfa000000 (720 * 1600 * 4)>;
+			width = <720>;
+			height = <1600>;
+			stride = <(720 * 4)>;
+			format = "a8r8g8b8";
+			clocks = <&cmu_dpu CLK_GOUT_DPU_DECON0_ACLK>,
+				 <&cmu_dpu CLK_GOUT_DPU_DMA_ACLK>,
+				 <&cmu_dpu CLK_GOUT_DPU_DPP_ACLK>,
+				 <&cmu_dpu CLK_GOUT_DPU_PPMU_ACLK>,
+				 <&cmu_dpu CLK_GOUT_DPU_PPMU_PCLK>,
+				 <&cmu_dpu CLK_GOUT_DPU_SMMU_CLK>;
+		};
+	};
+
+	/*
+	 * RTC clock (XrtcXTI); external, must be 32.768 kHz.
+	 *
+	 * TODO: Remove this once RTC clock is implemented properly as part of
+	 *       PMIC driver.
+	 */
+	rtcclk: clock-rtcclk {
+		compatible = "fixed-clock";
+		clock-output-names = "rtcclk";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&key_voldown &key_volup>;
+		pinctrl-names = "default";
+
+		voldown-key {
+			label = "Volume Down";
+			gpios = <&gpa1 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEDOWN>;
+		};
+
+		volup-key {
+			label = "Volume Up";
+			gpios = <&gpa0 7 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x3c900000>,
+		      <0x0 0xc0000000 0x20000000>,
+		      <0x0 0xe0d00000 0x1f300000>,
+		      <0x8 0x80000000 0x20000000>,
+		      <0x8 0xa0000000 0x20000000>;
+	};
+
+	/* TODO: Remove this once PMIC is implemented  */
+	reg_dummy: regulator-0 {
+		compatible = "regulator-fixed";
+		regulator-name = "dummy_reg";
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <1>;
+
+		ramoops@f0000000 {
+			compatible = "ramoops";
+			reg = <0x0 0xf0000000 0x200000>;
+			record-size = <0x20000>;
+			console-size = <0x20000>;
+			ftrace-size = <0x100000>;
+			pmsg-size = <0x20000>;
+		};
+
+		cont_splash_mem: framebuffer@fa000000 {
+			reg = <0x0 0xfa000000 (720 * 1600 * 4)>;
+			no-map;
+		};
+	};
+};
+
+&cmu_hsi {
+	clocks = <&oscclk>, <&rtcclk>,
+		 <&cmu_top CLK_DOUT_HSI_BUS>,
+		 <&cmu_top CLK_DOUT_HSI_MMC_CARD>,
+		 <&cmu_top CLK_DOUT_HSI_USB20DRD>;
+	clock-names = "oscclk", "rtcclk", "dout_hsi_bus",
+		      "dout_hsi_mmc_card", "dout_hsi_usb20drd";
+};
+
+&gpu {
+	mali-supply = <&reg_dummy>;
+
+	status = "okay";
+};
+
+&mmc_0 {
+	compatible = "samsung,exynos7-dw-mshc";
+	mmc-hs200-1_8v;
+	mmc-hs400-1_8v;
+	cap-mmc-highspeed;
+	non-removable;
+	mmc-hs400-enhanced-strobe;
+	card-detect-delay = <200>;
+	clock-frequency = <800000000>;
+	bus-width = <8>;
+
+	pinctrl-0 = <&sd0_clk_pins &sd0_cmd_pins &sd0_rdqs_pins &sd0_nreset_pins
+		     &sd0_bus1_pins &sd0_bus4_pins &sd0_bus8_pins>;
+	pinctrl-names = "default";
+
+	samsung,dw-mshc-ciu-div = <3>;
+	samsung,dw-mshc-sdr-timing = <0 4>;
+	samsung,dw-mshc-ddr-timing = <2 4>;
+	samsung,dw-mshc-hs400-timing = <0 2>;
+
+	status = "okay";
+};
+
+&oscclk {
+	clock-frequency = <26000000>;
+};
+
+&pinctrl_alive {
+	key_voldown: key-voldown-pins {
+		samsung,pins = "gpa1-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5420_PIN_DRV_LV1>;
+	};
+
+	key_volup: key-volup-pins {
+		samsung,pins = "gpa0-7";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5420_PIN_DRV_LV1>;
+	};
+};
+
+&rtc {
+	clocks = <&cmu_apm CLK_GOUT_RTC_PCLK>, <&rtcclk>;
+	clock-names = "rtc", "rtc_src";
+
+	status = "disabled";
+};
+
+&timer {
+	/*
+	 * Non-updatable, broken stock Samsung bootloader does not
+	 * configure CNTFRQ_EL0
+	 */
+	clock-frequency = <26000000>;
+};
+
+&usbdrd {
+	vdd10-supply = <&reg_dummy>;
+	vdd33-supply = <&reg_dummy>;
+
+	status = "okay";
+};
+
+&usbdrd_dwc3 {
+	dr_mode = "otg";
+	maximum-speed = "high-speed";
+};
+
+&usbdrd_phy {
+	status = "okay";
+};
+
+&watchdog_cl0 {
+	status = "okay";
+};
+
+&watchdog_cl1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos850.dtsi b/arch/arm64/boot/dts/exynos/exynos850.dtsi
index 911643008b9f59..8a4771899a8e66 100644
--- a/arch/arm64/boot/dts/exynos/exynos850.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos850.dtsi
@@ -160,7 +160,7 @@
 		method = "smc";
 	};
 
-	timer {
+	timer: timer {
 		compatible = "arm,armv8-timer";
 		/* Hypervisor Virtual Timer interrupt is not wired to GIC */
 		interrupts =
@@ -176,6 +176,11 @@
 		#size-cells = <1>;
 		ranges = <0x0 0x0 0x0 0x20000000>;
 
+		apm_sram: sram@2039000 {
+			compatible = "samsung,exynos850-acpm-sram";
+			reg = <0x2039000 0x30000>;
+		};
+
 		chipid@10000000 {
 			compatible = "samsung,exynos850-chipid";
 			reg = <0x10000000 0x100>;
@@ -310,6 +315,19 @@
 			clock-names = "oscclk", "dout_g3d_switch";
 		};
 
+		gpu: gpu@11500000 {
+			compatible = "samsung,exynos850-mali", "arm,mali-bifrost";
+			reg = <0x11500000 0x5000>;
+			interrupts = <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "job", "mmu", "gpu";
+			clocks = <&cmu_g3d CLK_GOUT_G3D_GPU_CLK>;
+			clock-names = "core";
+
+			status = "disabled";
+		};
+
 		cmu_apm: clock-controller@11800000 {
 			compatible = "samsung,exynos850-cmu-apm";
 			reg = <0x11800000 0x8000>;
@@ -319,6 +337,15 @@
 			clock-names = "oscclk", "dout_clkcmu_apm_bus";
 		};
 
+		ap2apm_mailbox: mailbox@11900000 {
+			compatible = "samsung,exynos850-mbox";
+			reg = <0x11900000 0x1000>;
+			clocks = <&cmu_apm CLK_GOUT_MAILBOX_APM_AP_PCLK>;
+			clock-names = "pclk";
+			interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <0>;
+		};
+
 		cmu_cmgp: clock-controller@11c00000 {
 			compatible = "samsung,exynos850-cmu-cmgp";
 			reg = <0x11c00000 0x8000>;
diff --git a/arch/arm64/boot/dts/exynos/exynos8855-smdk.dts b/arch/arm64/boot/dts/exynos/exynos8855-smdk.dts
index e74ff8a1bbcf60..ec6db81536a4e9 100644
--- a/arch/arm64/boot/dts/exynos/exynos8855-smdk.dts
+++ b/arch/arm64/boot/dts/exynos/exynos8855-smdk.dts
@@ -14,7 +14,7 @@
 
 / {
 	model = "Samsung Exynos8855 SMDK board";
-	compatible = "samsung,exynos8855-smdk","samsung,exynos8855";
+	compatible = "samsung,exynos8855-smdk", "samsung,exynos8855";
 
 	chosen {
 	};
diff --git a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
index 399b56b4dbdd9e..b5e1e4494e8111 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
@@ -1429,7 +1429,7 @@
 
 		pmu_system_controller: system-controller@11860000 {
 			compatible = "samsung,exynosautov920-pmu",
-				     "samsung,exynos7-pmu","syscon";
+				     "samsung,exynos7-pmu", "syscon";
 			reg = <0x11860000 0x10000>;
 		};
 
diff --git a/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi b/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi
index a12f2b88d72d88..05bd9f422407f3 100644
--- a/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi
+++ b/arch/arm64/boot/dts/exynos/google/gs101-pixel-common.dtsi
@@ -42,25 +42,25 @@
 		pinctrl-0 = <&key_voldown>, <&key_volup>, <&key_power>;
 		pinctrl-names = "default";
 
+		button-power {
+			label = "KEY_POWER";
+			gpios = <&gpa10 1 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+			linux,code = <KEY_POWER>;
+		};
+
 		button-vol-down {
 			label = "KEY_VOLUMEDOWN";
-			linux,code = <KEY_VOLUMEDOWN>;
 			gpios = <&gpa7 3 GPIO_ACTIVE_LOW>;
 			wakeup-source;
+			linux,code = <KEY_VOLUMEDOWN>;
 		};
 
 		button-vol-up {
 			label = "KEY_VOLUMEUP";
-			linux,code = <KEY_VOLUMEUP>;
 			gpios = <&gpa8 1 GPIO_ACTIVE_LOW>;
 			wakeup-source;
-		};
-
-		button-power {
-			label = "KEY_POWER";
-			linux,code = <KEY_POWER>;
-			gpios = <&gpa10 1 GPIO_ACTIVE_LOW>;
-			wakeup-source;
+			linux,code = <KEY_VOLUMEUP>;
 		};
 	};
 
@@ -109,8 +109,8 @@
 	pmic-1 {
 		compatible = "samsung,s2mpg10-pmic";
 		interrupts-extended = <&gpa0 6 IRQ_TYPE_LEVEL_LOW>;
-		pinctrl-names = "default";
 		pinctrl-0 = <&pmic_int>;
+		pinctrl-names = "default";
 		system-power-controller;
 		wakeup-source;
 
@@ -429,8 +429,8 @@
 	pmic-2 {
 		compatible = "samsung,s2mpg11-pmic";
 		interrupts-extended = <&gpa0 7 IRQ_TYPE_LEVEL_LOW>;
-		pinctrl-names = "default";
 		pinctrl-0 = <&sub_pmic_int>;
+		pinctrl-names = "default";
 		wakeup-source;
 
 		vinl1s-supply = <&s2mpg10_buck8m>;
@@ -527,15 +527,6 @@
 				regulator-always-on;
 			};
 
-			buckd {
-				regulator-name = "vcc_ufs";
-				regulator-min-microvolt = <2400000>;
-				regulator-max-microvolt = <3300000>;
-				regulator-ramp-delay = <6250>;
-				regulator-always-on;
-				samsung,ext-control = <S2MPG11_EXTCTRL_UFS_EN>;
-			};
-
 			s2mpg11_bucka: bucka {
 				/* lots, IO */
 				regulator-name = "bucka";
@@ -551,6 +542,15 @@
 				regulator-max-microvolt = <3600000>;
 			};
 
+			buckd {
+				regulator-name = "vcc_ufs";
+				regulator-min-microvolt = <2400000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <6250>;
+				regulator-always-on;
+				samsung,ext-control = <S2MPG11_EXTCTRL_UFS_EN>;
+			};
+
 			ldo1s {
 				regulator-name = "vdd_g3d_m";
 				regulator-min-microvolt = <450000>;
@@ -646,12 +646,13 @@
 	status = "okay";
 	/* TODO: add the devices once drivers exist */
 
-	usb-typec@25 {
+	maxtcpci: usb-typec@25 {
 		compatible = "maxim,max77759-tcpci", "maxim,max33359";
 		reg = <0x25>;
 		interrupts-extended = <&gpa8 2 IRQ_TYPE_LEVEL_LOW>;
 		pinctrl-0 = <&typec_int>;
 		pinctrl-names = "default";
+		vbus-supply = <&chgin_otg_reg>;
 
 		connector {
 			compatible = "usb-c-connector";
@@ -738,9 +739,10 @@
 		interrupts-extended = <&gpa9 3 IRQ_TYPE_LEVEL_LOW>;
 
 		shunt-resistor-micro-ohms = <5000>;
+		power-supplies = <&max77759_pmic>;
 	};
 
-	pmic@66 {
+	max77759_pmic: pmic@66 {
 		compatible = "maxim,max77759";
 		reg = <0x66>;
 
@@ -750,6 +752,11 @@
 
 		interrupt-controller;
 		#interrupt-cells = <2>;
+		power-supplies = <&maxtcpci>;
+
+		chgin_otg_reg: chgin-otg-regulator {
+			regulator-name = "chgin-otg";
+		};
 
 		gpio {
 			compatible = "maxim,max77759-gpio";
diff --git a/arch/arm64/boot/dts/exynos/google/gs101.dtsi b/arch/arm64/boot/dts/exynos/google/gs101.dtsi
index bfceea5486f466..b218ff50d22898 100644
--- a/arch/arm64/boot/dts/exynos/google/gs101.dtsi
+++ b/arch/arm64/boot/dts/exynos/google/gs101.dtsi
@@ -30,6 +30,18 @@
 		pinctrl7 = &pinctrl_hsi2;
 	};
 
+	ext_24_5m: clock-1 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-output-names = "oscclk";
+	};
+
+	ext_200m: clock-2 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-output-names = "ext-200m";
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -174,37 +186,50 @@
 			entry-method = "psci";
 
 			ananke_cpu_sleep: cpu-ananke-sleep {
-				idle-state-name = "c2";
 				compatible = "arm,idle-state";
-				arm,psci-suspend-param = <0x0010000>;
+				idle-state-name = "c2";
 				local-timer-stop;
 				entry-latency-us = <70>;
 				exit-latency-us = <160>;
 				min-residency-us = <2000>;
+				arm,psci-suspend-param = <0x0010000>;
 			};
 
 			enyo_cpu_sleep: cpu-enyo-sleep {
-				idle-state-name = "c2";
 				compatible = "arm,idle-state";
-				arm,psci-suspend-param = <0x0010000>;
+				idle-state-name = "c2";
 				local-timer-stop;
 				entry-latency-us = <150>;
 				exit-latency-us = <190>;
 				min-residency-us = <2500>;
+				arm,psci-suspend-param = <0x0010000>;
 			};
 
 			hera_cpu_sleep: cpu-hera-sleep {
-				idle-state-name = "c2";
 				compatible = "arm,idle-state";
-				arm,psci-suspend-param = <0x0010000>;
+				idle-state-name = "c2";
 				local-timer-stop;
 				entry-latency-us = <235>;
 				exit-latency-us = <220>;
 				min-residency-us = <3500>;
+				arm,psci-suspend-param = <0x0010000>;
 			};
 		};
 	};
 
+	/* ect node is required to be present by bootloader */
+	ect {
+	};
+
+	firmware {
+		acpm_ipc: power-management {
+			compatible = "google,gs101-acpm-ipc";
+			#clock-cells = <1>;
+			mboxes = <&ap2apm_mailbox>;
+			shmem = <&apm_sram>;
+		};
+	};
+
 	cpucl0_opp_table: opp-table-0 {
 		compatible = "operating-points-v2";
 		opp-shared;
@@ -472,31 +497,6 @@
 		};
 	};
 
-	/* ect node is required to be present by bootloader */
-	ect {
-	};
-
-	ext_24_5m: clock-1 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-output-names = "oscclk";
-	};
-
-	ext_200m: clock-2 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-output-names = "ext-200m";
-	};
-
-	firmware {
-		acpm_ipc: power-management {
-			compatible = "google,gs101-acpm-ipc";
-			#clock-cells = <1>;
-			mboxes = <&ap2apm_mailbox>;
-			shmem = <&apm_sram>;
-		};
-	};
-
 	pmu-0 {
 		compatible = "arm,cortex-a55-pmu";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH &ppi_cluster0>;
@@ -525,9 +525,9 @@
 	};
 
 	reserved_memory: reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <1>;
-		ranges;
 
 		gsa_reserved_protected: gsa@90200000 {
 			reg = <0x0 0x90200000 0x400000>;
@@ -572,9 +572,9 @@
 
 	soc: soc@0 {
 		compatible = "simple-bus";
+		ranges = <0x0 0x0 0x0 0x40000000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-		ranges = <0x0 0x0 0x0 0x40000000>;
 
 		efuse@10000000 {
 			compatible = "google,gs101-otp";
@@ -657,11 +657,11 @@
 
 		gic: interrupt-controller@10400000 {
 			compatible = "arm,gic-v3";
+			reg = <0x10400000 0x10000>, /* GICD */
+			      <0x10440000 0x100000>;/* GICR * 8 */
 			#address-cells = <0>;
 			#interrupt-cells = <4>;
 			interrupt-controller;
-			reg = <0x10400000 0x10000>, /* GICD */
-			      <0x10440000 0x100000>;/* GICR * 8 */
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
 
 			ppi-partitions {
@@ -1778,10 +1778,10 @@
 			compatible = "google,gs101-ufs-phy";
 			reg = <0x14704000 0x3000>;
 			reg-names = "phy-pma";
-			samsung,pmu-syscon = <&pmu_system_controller>;
 			#phy-cells = <0>;
 			clocks = <&ext_24_5m>;
 			clock-names = "ref_clk";
+			samsung,pmu-syscon = <&pmu_system_controller>;
 
 			status = "disabled";
 		};
@@ -1806,17 +1806,6 @@
 			reg = <0x17460000 0x10000>;
 			google,pmu-intr-gen-syscon = <&pmu_intr_gen>;
 
-			poweroff: syscon-poweroff {
-				compatible = "syscon-poweroff";
-				offset = <0x3e9c>; /* PAD_CTRL_PWR_HOLD */
-				mask = <0x00000100>;
-				value = <0x0>;
-			};
-
-			reboot: syscon-reboot {
-				compatible = "google,gs101-reboot";
-			};
-
 			reboot-mode {
 				compatible = "syscon-reboot-mode";
 				offset = <0x0810>; /* EXYNOS_PMU_SYSIP_DAT0 */
@@ -1830,6 +1819,17 @@
 				mode-shutdown-thermal = <0x51>;
 				mode-shutdown-thermal-battery = <0x51>;
 			};
+
+			poweroff: syscon-poweroff {
+				compatible = "syscon-poweroff";
+				offset = <0x3e9c>; /* PAD_CTRL_PWR_HOLD */
+				mask = <0x00000100>;
+				value = <0x0>;
+			};
+
+			reboot: syscon-reboot {
+				compatible = "google,gs101-reboot";
+			};
 		};
 
 		pmu_intr_gen: syscon@17470000 {
@@ -1920,9 +1920,9 @@
 	apm_sram: sram@2039000 {
 		compatible = "mmio-sram";
 		reg = <0x0 0x2039000 0x40000>;
+		ranges = <0x0 0x0 0x2039000 0x40000>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-		ranges = <0x0 0x0 0x2039000 0x40000>;
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 95ae85ab4adf5f..a09672b49709e8 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -55,6 +55,8 @@ DTC_FLAGS_fsl-lx2160a-half-twins := -Wno-interrupt_map
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-half-twins.dtb
 DTC_FLAGS_fsl-lx2160a-honeycomb := -Wno-interrupt_map
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-honeycomb.dtb
+DTC_FLAGS_fsl-lx2160a-nbxv3 := -Wno-interrupt_map
+dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-nbxv3.dtb
 DTC_FLAGS_fsl-lx2160a-qds := -Wno-interrupt_map
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-lx2160a-qds.dtb
 DTC_FLAGS_fsl-lx2160a-rdb := -Wno-interrupt_map
@@ -119,7 +121,81 @@ dtb-$(CONFIG_ARCH_MXC) += imx8dxl-hummingboard-telematics.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8dxp-tqma8xdp-mba8xx.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8dxp-tqma8xdps-mb-smarc-2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-beacon-kit.dtb
-dtb-$(CONFIG_ARCH_MXC) += imx8mm-data-modul-edm-sbc.dtb
+
+imx8mm-data-modul-edm-sbc-overlay-cm4-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-cm4.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtbo
+
+imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900-dtbs := \
+	imx8mm-data-modul-edm-sbc.dtb \
+	imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtbo
+
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-data-modul-edm-sbc.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-cm4.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-cm4.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtbo \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtb \
+			  imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtbo
+
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-ddr4-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-emcon-avari.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-emtop-baseboard.dtb
@@ -245,15 +321,25 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mn-var-som-symphony.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mn-var-som-symphony-legacy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mn-venice-gw7902.dtb
 
+imx8mn-vhip4-evalboard-v1-overlay-eeprom-1000-dtbs := imx8mn-vhip4-evalboard-v1.dtb \
+						      imx8mn-vhip4-overlay-eeprom-1000.dtbo
+imx8mn-vhip4-evalboard-v1-overlay-eeprom-1100-dtbs := imx8mn-vhip4-evalboard-v1.dtb \
+						      imx8mn-vhip4-overlay-eeprom-1100.dtbo
 imx8mn-vhip4-evalboard-v1-overlay-ksz8794-dtbs := imx8mn-vhip4-evalboard-v1.dtb \
 						  imx8mn-vhip4-evalboard-v1-overlay-ksz8794.dtbo
 imx8mn-vhip4-evalboard-v1-overlay-ksz9031-dtbs := imx8mn-vhip4-evalboard-v1.dtb \
 						  imx8mn-vhip4-evalboard-v1-overlay-ksz9031.dtbo
 imx8mn-vhip4-evalboard-v2-overlay-adin1300-dtbs := imx8mn-vhip4-evalboard-v2.dtb \
 						   imx8mn-vhip4-evalboard-v2-overlay-adin1300.dtbo
+imx8mn-vhip4-evalboard-v2-overlay-eeprom-1100-dtbs := imx8mn-vhip4-evalboard-v2.dtb \
+						      imx8mn-vhip4-overlay-eeprom-1100.dtbo
+imx8mn-vhip4-evalboard-v2-overlay-eeprom-2000-dtbs := imx8mn-vhip4-evalboard-v2.dtb \
+						      imx8mn-vhip4-overlay-eeprom-2000.dtbo
 imx8mn-vhip4-evalboard-v2-overlay-ksz8794-dtbs := imx8mn-vhip4-evalboard-v2.dtb \
 						  imx8mn-vhip4-evalboard-v2-overlay-ksz8794.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx8mn-vhip4-evalboard-v1.dtb \
+			  imx8mn-vhip4-evalboard-v1-overlay-eeprom-1000.dtb \
+			  imx8mn-vhip4-evalboard-v1-overlay-eeprom-1100.dtb \
 			  imx8mn-vhip4-evalboard-v1-overlay-ksz8794.dtb \
 			  imx8mn-vhip4-evalboard-v1-overlay-ksz8794.dtbo \
 			  imx8mn-vhip4-evalboard-v1-overlay-ksz9031.dtb \
@@ -261,8 +347,13 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mn-vhip4-evalboard-v1.dtb \
 			  imx8mn-vhip4-evalboard-v2.dtb \
 			  imx8mn-vhip4-evalboard-v2-overlay-adin1300.dtb \
 			  imx8mn-vhip4-evalboard-v2-overlay-adin1300.dtbo \
+			  imx8mn-vhip4-evalboard-v2-overlay-eeprom-1100.dtb \
+			  imx8mn-vhip4-evalboard-v2-overlay-eeprom-2000.dtb \
 			  imx8mn-vhip4-evalboard-v2-overlay-ksz8794.dtb \
-			  imx8mn-vhip4-evalboard-v2-overlay-ksz8794.dtbo
+			  imx8mn-vhip4-evalboard-v2-overlay-ksz8794.dtbo \
+			  imx8mn-vhip4-overlay-eeprom-1000.dtbo \
+			  imx8mn-vhip4-overlay-eeprom-1100.dtbo \
+			  imx8mn-vhip4-overlay-eeprom-2000.dtbo
 
 imx8mn-tqma8mqnl-mba8mx-lvds-g133han01-dtbs += imx8mn-tqma8mqnl-mba8mx.dtb imx8mm-tqma8mqml-mba8mx-lvds-g133han01.dtbo
 imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33-dtbs += imx8mn-tqma8mqnl-mba8mx.dtb imx8mn-tqma8mqnl-mba8mx-lvds-tm070jvhg33.dtbo
@@ -280,7 +371,145 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-aristainetos3-proton2s.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-beacon-kit.dtb
 DTC_FLAGS_imx8mp-cubox-m := -@
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-cubox-m.dtb
-dtb-$(CONFIG_ARCH_MXC) += imx8mp-data-modul-edm-sbc.dtb
+
+imx8mp-data-modul-edm-sbc-overlay-cm7-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-cm7.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtbo
+
+DTC_FLAGS_imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01 := -@
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900-dtbs := \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtbo \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902-dtbs := \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtbo \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtbo
+
+imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902-dtbs := \
+	imx8mp-data-modul-edm-sbc.dtb \
+	imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtbo
+
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-data-modul-edm-sbc.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-cm7.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-cm7.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtbo \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtb \
+			  imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtbo
+
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-debix-model-a.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-debix-som-a-bmb-08.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-dhcom-drc02.dtb
@@ -456,6 +685,8 @@ imx8mp-phyboard-pollux-peb-av-10-etml1010g3dra-dtbs += imx8mp-phyboard-pollux-rd
 	imx8mp-phyboard-pollux-peb-av-10-etml1010g3dra.dtbo
 imx8mp-phyboard-pollux-peb-av-10-ph128800t006-dtbs += imx8mp-phyboard-pollux-rdk.dtb \
 	imx8mp-phyboard-pollux-peb-av-10-ph128800t006.dtbo
+imx8mp-phyboard-pollux-peb-wlbt-07-dtbs += imx8mp-phyboard-pollux-rdk.dtb \
+	imx8mp-phyboard-pollux-peb-wlbt-07.dtbo
 imx8mp-phyboard-pollux-ph128800t006-dtbs += imx8mp-phyboard-pollux-rdk.dtb \
 	imx8mp-phyboard-pollux-ph128800t006.dtbo
 imx8mp-phyboard-pollux-rdk-no-eth-dtbs += imx8mp-phyboard-pollux-rdk.dtb imx8mp-phycore-no-eth.dtbo
@@ -465,6 +696,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-etml1010g3dra.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-peb-av-10.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-peb-av-10-etml1010g3dra.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-peb-av-10-ph128800t006.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-peb-wlbt-07.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-ph128800t006.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-rdk-no-eth.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-wlbt.dtb
@@ -610,9 +842,17 @@ dtb-$(CONFIG_ARCH_MXC) += imx8qxp-tqma8xqps-mb-smarc-2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8ulp-9x9-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8ulp-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-9x9-qsb.dtb
+
+imx91-9x9-qsb-tianma-tm050rdh03-dtbs += imx91-9x9-qsb.dtb imx91-9x9-qsb-tianma-tm050rdh03.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx91-9x9-qsb-tianma-tm050rdh03.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-frdm.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-frdm-s.dtb
+imx91-11x11-evk-mqs-dtbs += imx91-11x11-evk.dtb imx9-mqs.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-evk-mqs.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx91-lino-verdin-dahlia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx91-lino-verdin-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-phyboard-segin.dtb
 
 imx91-9x9-qsb-can1-dtbs += imx91-9x9-qsb.dtb imx91-9x9-qsb-can1.dtbo
@@ -620,6 +860,8 @@ imx91-phyboard-segin-peb-av-18-dtbs += imx91-phyboard-segin.dtb imx91-phyboard-s
 dtb-$(CONFIG_ARCH_MXC) += imx91-9x9-qsb-can1.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-phyboard-segin-peb-av-18.dtb
 
+dtb-$(CONFIG_ARCH_MXC) += imx91-toradex-osm-dev.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx91-tqma9131-mba91xxca.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-tqma9131-mba93xxca.dtb
 
@@ -645,6 +887,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-evk.dtb
 
 imx93-11x11-evk-dy1212w-4856-dtbs += imx93-11x11-evk.dtb imx93-11x11-evk-dy1212w-4856.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-evk-dy1212w-4856.dtb
+imx93-11x11-evk-mqs-dtbs += imx93-11x11-evk.dtb imx9-mqs.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-evk-mqs.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-frdm.dtb
 
@@ -652,7 +896,11 @@ imx93-11x11-frdm-pixpaper-dtbs += imx93-11x11-frdm.dtb imx93-11x11-frdm-pixpaper
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-frdm-pixpaper.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx93-14x14-evk.dtb
+imx93-14x14-evk-mqs-dtbs += imx93-14x14-evk.dtb imx9-mqs.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx93-14x14-evk-mqs.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-kontron-bl-osm-s.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93-lino-verdin-dahlia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93-lino-verdin-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-nash.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-segin.dtb
 
@@ -673,6 +921,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-segin-peb-eval-01.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phyboard-segin-peb-wlbt-05.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-phycore-rpmsg.dtb
 
+dtb-$(CONFIG_ARCH_MXC) += imx93-toradex-osm-dev.dtb
+
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba91xxca.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxca.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxla.dtb
@@ -695,6 +945,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxla-mini-lvds-tm070jvhg33.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-var-dart-sonata.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-var-som-symphony.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93w-evk.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93w-frdm.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx943-evk.dtb
 
 imx943-evk-pcie0-ep-dtbs += imx943-evk.dtb imx-pcie0-ep.dtbo
@@ -703,6 +954,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx943-evk-pcie0-ep.dtb imx943-evk-pcie1-ep.dtb
 
 imx943-evk-sdwifi-dtbs := imx943-evk.dtb imx943-evk-sdwifi.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx943-evk-sdwifi.dtb
+imx943-evk-mqs-dtbs += imx943-evk.dtb imx9-mqs.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx943-evk-mqs.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-ab2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk.dtb
@@ -712,6 +965,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx95-19x19-evk-sof.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-19x19-frdm-pro.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-aquila-clover.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-aquila-dev.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx95-toradex-osm-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-toradex-smarc-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-tqma9596la-mba95xxca.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-tqma9596sa-mb-smarc-2.dtb
@@ -734,6 +988,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk-pcie.dtb
 
 imx95-15x15-evk-pcie0-ep-dtbs = imx95-15x15-evk-pcie.dtb imx-pcie0-ep.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk-pcie0-ep.dtb
+imx95-15x15-evk-mqs-dtbs += imx95-15x15-evk.dtb imx9-mqs.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk-mqs.dtb
 imx95-19x19-evk-pcie0-ep-dtbs += imx95-19x19-evk.dtb imx-pcie0-ep.dtbo
 imx95-19x19-evk-pcie1-ep-dtbs += imx95-19x19-evk.dtb imx-pcie1-ep.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx95-19x19-evk-pcie0-ep.dtb imx95-19x19-evk-pcie1-ep.dtb
@@ -742,6 +998,10 @@ dtb-$(CONFIG_ARCH_MXC) += imx95-libra-rdk-fpsc.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-19x19-verdin-evk.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx952-evk.dtb
+imx952-evk-mqs-dtbs += imx952-evk.dtb imx9-mqs.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx952-evk-mqs.dtb
+
+dtb-$(CONFIG_ARCH_MXC) += imx952-frdm.dtb
 
 imx8mm-kontron-dl-dtbs			:= imx8mm-kontron-bl.dtb imx8mm-kontron-dl.dtbo
 imx8mm-kontron-bl-lte-dtbs		:= imx8mm-kontron-bl.dtb imx8mm-kontron-bl-lte.dtbo
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi
index ef62968590faaa..ed810932615148 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi
@@ -105,7 +105,7 @@
 	};
 
 	firmware {
-		optee: optee  {
+		optee: optee {
 			compatible = "linaro,optee-tz";
 			method = "smc";
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dts b/arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dts
new file mode 100644
index 00000000000000..cc02204e9d1b4d
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dts
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Free Mobile Nodebox v3 CPU Module (LX2160A)
+ *
+ * Copyright 2026 Free Mobile, Vincent Jardin
+ */
+
+/dts-v1/;
+
+#include "fsl-lx2160a-rev2.dtsi"
+#include "fsl-lx2160a-nbxv3.dtsi"
+
+/ {
+	compatible = "freemobile,nbxv3", "fsl,lx2160a";
+	model = "Free Mobile Nodebox v3 CPU Module (LX2160A)";
+};
+
+&fspi {
+	pinctrl-0 = <&fspi_dqs_sck_cs10_pins>, <&fspi_data30_pins>;
+	pinctrl-names = "default";
+};
+
+&gpio1 {
+	pinctrl-0 = <&gpio1_31_28_pins>;
+	pinctrl-names = "default";
+};
+
+&pcs_mdio13 {
+	status = "okay";
+};
+
+&pcs_mdio14 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dtsi
new file mode 100644
index 00000000000000..9c5588a75cdd41
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-nbxv3.dtsi
@@ -0,0 +1,330 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Free Mobile Nodebox v3 CPU Module (LX2160A)
+ *
+ * Copyright 2026 Free Mobile, Vincent Jardin
+ *
+ * This file is shared by the Linux kernel and U-Boot.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio-restart {
+		compatible = "gpio-restart";
+		gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
+		priority = <128>;
+	};
+
+	reg_ethsw_mdio: regulator-ethsw-mdio {
+		compatible = "regulator-fixed";
+		gpio = <&gpio3 25 GPIO_ACTIVE_LOW>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-name = "ethsw-mdio-en";
+	};
+};
+
+&crypto {
+	status = "okay";
+};
+
+&dpmac13 {
+	phy-connection-type = "10gbase-r";
+	phy-handle = <&lan8023_phy_s0>;
+
+	status = "okay";
+};
+
+&dpmac14 {
+	phy-connection-type = "10gbase-r";
+	phy-handle = <&lan8023_phy_s1>;
+
+	status = "okay";
+};
+
+&dpmac17 {
+	phy-connection-type = "rgmii-id";
+	phy-handle = <&rtl8211fd>;
+
+	status = "okay";
+};
+
+&dspi0 {
+	status = "okay";
+
+	dpll@0 {
+		compatible = "microchip,zl30733";
+		reg = <0>;
+		spi-max-frequency = <12500000>;
+	};
+};
+
+&dspi2 {
+	status = "okay";
+};
+
+&emdio1 {
+	status = "okay";
+
+	/* Microchip LAN8023 dual 10G PCS/retimer, Clause 45 only, one address per slice */
+	lan8023_phy_s0: ethernet-phy@2 {
+		compatible = "ethernet-phy-ieee802.3-c45";
+		reg = <2>;
+	};
+
+	lan8023_phy_s1: ethernet-phy@3 {
+		compatible = "ethernet-phy-ieee802.3-c45";
+		reg = <3>;
+	};
+
+	rtl8211fd: ethernet-phy@6 {
+		compatible = "ethernet-phy-id001c.c916";
+		reg = <6>;
+	};
+};
+
+&emdio2 {
+	status = "okay";
+};
+
+&esdhc1 {
+	bus-width = <8>;
+	mmc-hs200-1_8v;
+	mmc-hs400-1_8v;
+	non-removable;
+
+	status = "okay";
+};
+
+&fsl_mc {
+	status = "okay";
+};
+
+&fspi {
+	status = "okay";
+
+	/* GigaDevice GD55LB02GF, 256 MiB boot NOR */
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		reset-gpios = <&gpio1 28 GPIO_ACTIVE_LOW>;
+		spi-max-frequency = <150000000>;
+		spi-rx-bus-width = <1>;
+		spi-tx-bus-width = <1>;
+	};
+};
+
+&gpio0 {
+	gpio-line-names =
+		"", "",
+		"altfn:iic1-sda", "altfn:iic1-scl",
+		"altfn:uart4-sin", "altfn:uart4-sout",
+		"altfn:uart2-sin", "altfn:uart2-sout",
+		"altfn:uart3-sin", "altfn:uart3-sout",
+		"altfn:uart1-sin", "altfn:uart1-sout",
+		"altfn:spi3-pcs3", "altfn:spi3-pcs2",
+		"altfn:spi3-pcs1", "altfn:spi3-pcs0",
+		"altfn:spi1-sck", "altfn:spi1-sin",
+		"altfn:spi1-pcs2", "altfn:spi1-pcs1",
+		"altfn:spi1-pcs0", "altfn:spi1-sout",
+		"altfn:iic6-sda", "altfn:iic6-scl",
+		"altfn:spi3-sin", "altfn:spi3-sout",
+		"altfn:iic4-sda", "altfn:iic4-scl",
+		"altfn:iic3-sda", "altfn:iic3-scl",
+		"altfn:iic2-sda", "altfn:iic2-scl";
+
+	status = "okay";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"", "", "", "", "", "",
+		"cfg-rcw-src2", "cfg-rcw-src3",
+		"reset-req-b",
+		"altfn:sdhc2-clk", "altfn:sdhc2-ds",
+		"altfn:sdhc2-dat0", "altfn:sdhc2-dat1",
+		"altfn:sdhc2-dat2", "altfn:sdhc2-dat3",
+		"altfn:sdhc2-dat4", "altfn:sdhc2-dat5",
+		"altfn:sdhc2-dat6", "altfn:sdhc2-dat7",
+		"altfn:sdhc2-cmd",
+		"cfg-svr1",
+		"altfn:xspi1-cs0", "altfn:xspi1-sck",
+		"altfn:xspi1-dqs",
+		"altfn:xspi1-data0", "altfn:xspi1-data1",
+		"", "",
+		"flash-reset#", "pltrst#", "", "ethsw-reset#";
+
+	status = "okay";
+
+	/* Platform reset, no consumer node: released at boot */
+	pltrst-hog {
+		gpio-hog;
+		gpios = <29 GPIO_ACTIVE_LOW>;
+		line-name = "pltrst#";
+		output-low;
+	};
+};
+
+&gpio2 {
+	gpio-line-names =
+		"lan8023-mdint",
+		"i2c2-alert#", "i2c4-alert#", "i2c6-alert#",
+		"u800-tca9539-int#",
+		"spi3-alert#",
+		"wake0", "wake1",
+		"carrier-hot#",
+		"u1300-rtl8211fd-int#",
+		"ethsw-int#",
+		"", "",
+		"u1700-zl30733-clk-gpio0", "u1700-zl30733-clk-gpio1",
+		"u1700-zl30733-clk-gpio2", "u1700-zl30733-clk-gpio3",
+		"", "", "", "", "", "", "", "",
+		"", "", "", "", "", "", "";
+
+	status = "okay";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"altfn:ec1-txd3", "altfn:ec1-txd2",
+		"altfn:ec1-txd1", "altfn:ec1-txd0",
+		"altfn:ec1-tx-en", "altfn:ec1-gtx-clk",
+		"altfn:ec1-rxd3", "altfn:ec1-rxd2",
+		"altfn:ec1-rxd1", "altfn:ec1-rxd0",
+		"altfn:ec1-rx-clk", "altfn:ec1-rx-dv",
+		"", "",
+		"ptp-refclk-in",
+		"", "", "", "", "", "", "", "",
+		"u1700-zl30733-pps-out",
+		"",
+		"ethsw-mdio-en",
+		"usb1-oc#",	/* USB1_PWRFAULT and USB2_PWRFAULT pads: one wired-OR net */
+		"",
+		"usb2-oc#",
+		"altfn:spi3-sck",
+		"", "";
+
+	status = "okay";
+};
+
+/* On-module management bus */
+&i2c0 {
+	status = "okay";
+
+	/* +0V8_VDD core rail, MPS MPQ8646 (MPQ8785 core) over PMBus */
+	vdd08: regulator@10 {
+		compatible = "mps,mpq8646";
+		reg = <0x10>;
+	};
+
+	/* SO-DIMM SPD temperature sensors */
+	temperature-sensor@18 {
+		compatible = "jedec,jc-42.4-temp";
+		reg = <0x18>;
+	};
+
+	temperature-sensor@19 {
+		compatible = "jedec,jc-42.4-temp";
+		reg = <0x19>;
+	};
+
+	eeprom@52 {
+		compatible = "atmel,24c32";
+		reg = <0x52>;
+		pagesize = <32>;
+	};
+
+	/* Board ID straps */
+	gpio@74 {
+		compatible = "ti,tca9539";
+		reg = <0x74>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"boardid-0x0001", "boardid-0x0002",
+			"boardid-0x0004", "boardid-0x0008",
+			"boardid-0x0010", "boardid-0x0020",
+			"boardid-0x0040", "boardid-0x0080",
+			"boardid-0x0100", "boardid-0x0200",
+			"boardid-0x0400", "boardid-0x0800",
+			"boardid-0x1000", "boardid-0x2000",
+			"boardid-0x4000", "boardid-0x8000";
+	};
+
+	u800: gpio@75 {
+		compatible = "ti,tca9539";
+		reg = <0x75>;
+		#gpio-cells = <2>;
+		#interrupt-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"carrier-gpio0", "carrier-gpio1",
+			"carrier-gpio2", "carrier-gpio3",
+			"carrier-gpio4", "carrier-gpio5",
+			"carrier-gpio6", "carrier-gpio7",
+			"carrier-gpio8", "carrier-gpio9",
+			"carrier-gpio10", "carrier-gpio11",
+			"", "", "",
+			"lan8023-rst";
+		interrupt-controller;
+		interrupt-parent = <&gpio2>;
+		interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&pcie3 {
+	status = "okay";
+};
+
+&pcie4 {
+	status = "okay";
+};
+
+&pcie5 {
+	status = "okay";
+};
+
+&pcie6 {
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-lx216x.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx216x.dtsi
index ae6e37870c5e53..bccf2d67302302 100644
--- a/arch/arm64/boot/dts/freescale/fsl-lx216x.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-lx216x.dtsi
@@ -1602,7 +1602,7 @@
 			};
 
 			dspi2_miso_mosi_pins: iic5-spi3-pins {
-				pinctrl-single,bits = <0x3 (0x2 << 9) (0x7 << 9)>;
+				pinctrl-single,bits = <0x0 (0x3 << 9) (0x7 << 9)>;
 			};
 
 			i2c5_pins: iic6-i2c-pins {
@@ -1622,7 +1622,7 @@
 			};
 
 			gpio1_31_28_pins: xspi1-data74-gpio-pins {
-				pinctrl-single,bits = <0x0 0x1 (0x7 << 15)>;
+				pinctrl-single,bits = <0x0 (0x1 << 15) (0x7 << 15)>;
 			};
 
 			fspi_data30_pins: xspi1-data30-pins {
@@ -1630,7 +1630,7 @@
 			};
 
 			gpio1_27_24_pins: xspi1-data30-gpio-pins {
-				pinctrl-single,bits = <0x0 0x1 (0x7 << 18)>;
+				pinctrl-single,bits = <0x0 (0x1 << 18) (0x7 << 18)>;
 			};
 
 			fspi_dqs_sck_cs10_pins: xspi1-base-pins {
@@ -1638,7 +1638,7 @@
 			};
 
 			gpio1_23_20_pins: xspi1-base-gpio-pins {
-				pinctrl-single,bits = <0x0 0x1 (0x7 << 21)>;
+				pinctrl-single,bits = <0x0 (0x1 << 21) (0x7 << 21)>;
 			};
 
 			esdhc0_cmd_data30_clk_vsel_pins: sdhc1-base-sdhc-vsel-pins {
@@ -1687,12 +1687,19 @@
 			};
 
 			/* RCWSR14 */
+			/*
+			 * IIC1_PMUX is a single bit at RCWSR14[10], but bit-per-mux
+			 * uses the mask for the field to clear before applying the
+			 * value.
+			 * So bits 11 and 12, which are reserved, are written as 0
+			 * on every mux switch, it is harmless.
+			 */
 			i2c0_pins: iic1-i2c-pins {
-				pinctrl-single,bits = <0x8 0x0 (0x1 << 10)>;
+				pinctrl-single,bits = <0x8 0x0 (0x7 << 10)>;
 			};
 
 			gpio0_3_2_pins: iic1-gpio-pins {
-				pinctrl-single,bits = <0x8 (0x1 << 10) (0x1 << 10)>;
+				pinctrl-single,bits = <0x8 (0x1 << 10) (0x7 << 10)>;
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi
index 6f5af37ba9af90..1d4e92f674f872 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-eval.dtsi
@@ -133,7 +133,11 @@
 	status = "okay";
 };
 
-/* TODO: Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+/* Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+&usbh1 {
+	vbus-supply = <&reg_usb_host_vbus>;
+	status = "okay";
+};
 
 /* Apalis USBO1 */
 &usbotg1 {
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi
index 93f485140b20c7..68fb2eb8c7e675 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi
@@ -222,7 +222,11 @@
 	status = "okay";
 };
 
-/* TODO: Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+/* Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+&usbh1 {
+	vbus-supply = <&reg_usb_host_vbus>;
+	status = "okay";
+};
 
 /* Apalis USBO1 */
 &usbotg1 {
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi
index 5c86bcee55fb76..baf09624fb27c6 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi
@@ -271,7 +271,11 @@
 	status = "okay";
 };
 
-/* TODO: Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+/* Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+&usbh1 {
+	vbus-supply = <&reg_usb_host_vbus>;
+	status = "okay";
+};
 
 /* Apalis USBO1 */
 &usbotg1 {
diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi
index 681f24810d52e7..64604f91d56734 100644
--- a/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-apalis-v1.1.dtsi
@@ -870,7 +870,21 @@
 	status = "okay";
 };
 
-/* TODO: Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+/* Apalis USBH2, Apalis USBH3 and on-module Wi-Fi via on-module HSIC Hub */
+&usbh1 {
+	pinctrl-names = "idle", "active";
+	pinctrl-0 = <&pinctrl_usb_hsic_idle>;
+	pinctrl-1 = <&pinctrl_usb_hsic_active>;
+	adp-disable;
+	disable-over-current;
+	hnp-disable;
+	srp-disable;
+	status = "disabled";
+};
+
+&usbphynop2 {
+	status = "okay";
+};
 
 /* Apalis USBH4 */
 &usb3_phy {
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
index 0ede0c7ed422c8..71eff3efa6d329 100644
--- a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
@@ -29,7 +29,7 @@
 		clocks = <&clk_dummy>;
 		mbox-names = "tx", "rx", "rxdb";
 		mboxes = <&lsio_mu5 0 1 &lsio_mu5 1 1 &lsio_mu5 3 1>;
-		memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+		memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 				<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
 		power-domains = <&pd IMX_SC_R_M4_0_PID0>, <&pd IMX_SC_R_M4_0_MU_1A>;
 		fsl,resource-id = <IMX_SC_R_M4_0_PID0>;
@@ -65,32 +65,32 @@
 			linux,cma-default;
 		};
 
-		vdev0vring0: memory0@90000000 {
+		vdev0vring0: vdev0vring0@90000000 {
 			reg = <0 0x90000000 0 0x8000>;
 			no-map;
 		};
 
-		vdev0vring1: memory@90008000 {
+		vdev0vring1: vdev0vring1@90008000 {
 			reg = <0 0x90008000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring0: memory@90010000 {
+		vdev1vring0: vdev1vring0@90010000 {
 			reg = <0 0x90010000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring1: memory@90018000 {
+		vdev1vring1: vdev1vring1@90018000 {
 			reg = <0 0x90018000 0 0x8000>;
 			no-map;
 		};
 
-		rsc_table: memory-rsc-table@900ff000 {
+		rsc_table: rsc-table@900ff000 {
 			reg = <0 0x900ff000 0 0x1000>;
 			no-map;
 		};
 
-		vdevbuffer: memory-vdevbuffer@90400000 {
+		vdev0buffer: vdev0buffer@90400000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0x90400000 0 0x100000>;
 			no-map;
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-sr-som.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-sr-som.dtsi
index 93a0eb4d7f7708..83d22ae373c3bb 100644
--- a/arch/arm64/boot/dts/freescale/imx8dxl-sr-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-sr-som.dtsi
@@ -26,7 +26,7 @@
 		clocks = <&clk_dummy>;
 		mboxes = <&lsio_mu5 0 1 &lsio_mu5 1 1 &lsio_mu5 3 1>;
 		mbox-names = "tx", "rx", "rxdb";
-		memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+		memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 				<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
 		power-domains = <&pd IMX_SC_R_M4_0_PID0>, <&pd IMX_SC_R_M4_0_MU_1A>;
 		fsl,entry-address = <0x34fe0000>;
@@ -103,32 +103,32 @@
 			linux,cma-default;
 		};
 
-		vdev0vring0: memory0@90000000 {
+		vdev0vring0: vdev0vring0@90000000 {
 			reg = <0 0x90000000 0 0x8000>;
 			no-map;
 		};
 
-		vdev0vring1: memory@90008000 {
+		vdev0vring1: vdev0vring1@90008000 {
 			reg = <0 0x90008000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring0: memory@90010000 {
+		vdev1vring0: vdev1vring0@90010000 {
 			reg = <0 0x90010000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring1: memory@90018000 {
+		vdev1vring1: vdev1vring1@90018000 {
 			reg = <0 0x90018000 0 0x8000>;
 			no-map;
 		};
 
-		rsc_table: memory-rsc-table@900ff000 {
+		rsc_table: rsc-table@900ff000 {
 			reg = <0 0x900ff000 0 0x1000>;
 			no-map;
 		};
 
-		vdevbuffer: memory-vdevbuffer@90400000 {
+		vdev0buffer: vdev0buffer@90400000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0x90400000 0 0x100000>;
 			no-map;
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi
index ea1d5b9c6bae04..94e18357e37fb5 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-beacon-baseboard.dtsi
@@ -49,7 +49,7 @@
 		clock-frequency = <100000000>;
 	};
 
-	pcie0_refclk_gated:  pcie0-refclk-gated {
+	pcie0_refclk_gated: pcie0-refclk-gated {
 		compatible = "gpio-gate-clock";
 		clocks = <&pcie0_refclk>;
 		#clock-cells = <0>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-cm4.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-cm4.dtso
new file mode 100644
index 00000000000000..0bd770af2a68cf
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-cm4.dtso
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2023-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/imx8mm-clock.h>
+
+&{/} {
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	reserved-memory {	/* CM4 reserved memory */
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		m_core_reserved: m_core@b7000000 {
+			reg = <0 0xb7000000 0 0x1000000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@b8000000 {
+			reg = <0 0xb8000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@b8008000 {
+			reg = <0 0xb8008000 0 0x8000>;
+			no-map;
+		};
+
+		rsc_table: rsc-table@b80ff000 {
+			reg = <0 0xb80ff000 0 0x1000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@b8400000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xb8400000 0 0x100000>;
+			no-map;
+		};
+	};
+
+	imx8mm-cm4 {
+		compatible = "fsl,imx8mm-cm4";
+		clocks = <&clk IMX8MM_CLK_M4_CORE>;
+		mbox-names = "tx", "rx", "rxdb";
+		mboxes = <&mu 0 1
+			  &mu 1 1
+			  &mu 3 1>;
+		memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>, <&rsc_table>;
+		syscon = <&src>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi
new file mode 100644
index 00000000000000..8a95dc05b1fcfd
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+&{/} {
+	reg_backlight_en_level: regulator-backlight-en-level {
+		compatible = "regulator-gpio";
+		regulator-name = "Backlight_SEL_EN";
+		regulator-type = "voltage";
+		states = <3300000 0x0>,
+			 <5000000 0x1>;
+
+		/* Default setting: lowest supported voltage. */
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	reg_backlight_pwm_level: regulator-backlight-pwm-level {
+		compatible = "regulator-gpio";
+		regulator-name = "Backlight_SEL_PWM";
+		regulator-type = "voltage";
+		gpios = <&gpio_display 2 GPIO_ACTIVE_HIGH>; /* SEL_PWM */
+		states = <3300000 0x0>,
+			 <5000000 0x1>;
+
+		/* Default setting: lowest supported voltage. */
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	reg_panel_bl: regulator-panel-bl {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_panel_backlight>;
+		regulator-name = "PANEL_BL";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio3 0 0>;
+		enable-active-high;
+		/* Used by panels which enable PWM signal before BL ON/OFF */
+		status = "disabled";
+	};
+};
+
+&reg_panel_vcc {
+	compatible = "regulator-gpio";
+	regulator-type = "voltage";
+	enable-active-high;
+	status = "okay";
+
+	/* Default setting: lowest supported voltage. */
+	gpios-states = <0 0>;	/* Default GPIO state is LOW/LOW, so 3V3 out */
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtsi
new file mode 100644
index 00000000000000..8b38b5c228e0a3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtsi
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+&{/} {
+	can_osc: can-osc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <20000000>;
+	};
+
+	sound-fio {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "SGTL5000-FIO1";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&codec_dai_fio>;
+		simple-audio-card,frame-master = <&codec_dai_fio>;
+		simple-audio-card,widgets = "Headphone", "Headphone Jack";
+		simple-audio-card,routing = "Headphone Jack", "HP_OUT";
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai2>;
+		};
+
+		codec_dai_fio: simple-audio-card,codec {
+			sound-dai = <&sgtl5000_fio>;
+		};
+	};
+};
+
+&ecspi2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	can_fio: can@0 {
+		compatible = "microchip,mcp2518fd";
+		reg = <0>;
+		clocks = <&can_osc>;
+		spi-max-frequency = <10000000>;
+	};
+};
+
+&i2c_feature {	/* Feature connector I2C */
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	sgtl5000_fio: codec@a {
+		compatible = "fsl,sgtl5000";
+		reg = <0x0a>;
+		#sound-dai-cells = <0>;
+		clocks = <&sai5clk 1>;
+		VDDA-supply = <&buck4_reg>;
+		VDDD-supply = <&buck5_reg>;
+		VDDIO-supply = <&buck4_reg>;
+	};
+
+	gpio_feature: io-expander@20 {
+		compatible = "nxp,pca9554";
+		reg = <0x20>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		gpio-line-names =
+			"GPI0", "GPI1", "GPI2", "GPI3",
+			"GPO0", "GPO1", "GPO2", "GPO3";
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c32";
+		reg = <0x50>;
+		pagesize = <32>;
+	};
+};
+
+&sai2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai2>;
+	assigned-clock-rates = <24576000>;
+	fsl,sai-asynchronous;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&uart2 {	/* RS422 J12 */
+	linux,rs485-enabled-at-boot-time;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+/* UART4 is blocked by RDC and used as CM4 console UART */
+&uart4 {	/* UART to 1-Wire J5 */
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso
new file mode 100644
index 00000000000000..88fa705ac2b67c
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/imx8mm-clock.h>
+
+#include "imx8mm-pinfunc.h"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtsi"
+
+&can_fio {
+	interrupts-extended = <&gpio4 25 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&gpio_feature {
+	interrupt-parent = <&gpio5>;
+	interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&iomuxc {
+	pinctrl_codec_mclk: codec-mclk_feature-grp {
+		fsl,pins = <
+			/* GPIO4_IO27 */
+			MX8MM_IOMUXC_SAI2_MCLK_SAI5_MCLK		0x2
+		>;
+	};
+
+	pinctrl_sai2: sai2_feature-grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI2_RXC_SAI2_RX_BCLK		0x90
+			MX8MM_IOMUXC_SAI2_TXD0_SAI2_TX_DATA0		0x96
+			MX8MM_IOMUXC_SAI2_RXD0_SAI2_RX_DATA0		0x90
+			MX8MM_IOMUXC_SAI2_TXFS_SAI2_TX_SYNC		0x96
+		>;
+	};
+};
+
+&pinctrl_hog_feature {
+	fsl,pins = <
+		/* GPIO5_IO03 */
+		MX8MM_IOMUXC_SPDIF_TX_GPIO5_IO3				0x40000006
+		/* GPIO5_IO04 */
+		MX8MM_IOMUXC_SPDIF_RX_GPIO5_IO4				0x40000006
+
+		/* CAN_INT# */
+		MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25			0x40000090
+	>;
+};
+
+&sai2 {
+	assigned-clocks = <&clk IMX8MM_CLK_SAI2>;
+	assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>;
+	fsl,sai-bit-clock-swap;
+};
+
+&spba2 {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	sai5clk: clock-controller@30050000 {	/* SAI5 */
+		compatible = "fsl,imx8mm-sai-clock", "fsl,imx8mq-sai-clock";
+		reg = <0x30050000 0x10000>;
+		#clock-cells = <1>;
+		clocks = <&clk IMX8MM_CLK_SAI5_IPG>,
+			 <&clk IMX8MM_CLK_SAI5_ROOT>;
+		clock-names = "bus", "mclk1";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_codec_mclk>;
+		assigned-clocks = <&clk IMX8MM_CLK_SAI5>,
+				  <&clk IMX8MM_CLK_CLKOUT1_SEL>,
+				  <&clk IMX8MM_CLK_CLKOUT2_SEL>;
+		assigned-clock-parents = <&clk IMX8MM_CLK_24M>,
+					 <&clk IMX8MM_CLK_24M>,
+					 <&clk IMX8MM_CLK_24M>;
+		assigned-clock-rates = <24000000>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtsi
new file mode 100644
index 00000000000000..905db21d57c215
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtsi
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+&{/} {
+	can_osc: can-osc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <20000000>;
+	};
+};
+
+&ecspi2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	can_fio: can@0 {
+		compatible = "microchip,mcp2515";
+		reg = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_can>;
+		clocks = <&can_osc>;
+		spi-max-frequency = <5000000>;
+	};
+};
+
+&i2c_feature {	/* Feature connector I2C */
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	gpio_feature: io-expander@20 {
+		compatible = "nxp,pca9554";
+		reg = <0x20>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_expander>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio4>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+		gpio-line-names =
+			"GPIO1_output", "GPIO1_input",
+			"GPIO2_output", "GPIO2_input",
+			"GPIO3_output", "GPIO3_input",
+			"PCA9511A_READY", "";
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c32";
+		reg = <0x50>;
+		pagesize = <32>;
+	};
+};
+
+&uart1 {	/* J500/J501 */
+	status = "okay";
+};
+
+&uart2 {	/* RS485 J302/J303 */
+	linux,rs485-enabled-at-boot-time;
+	uart-has-rtscts;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso
new file mode 100644
index 00000000000000..ad410db5f5b76f
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mm-pinfunc.h"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtsi"
+
+&{/} {
+	beeper {
+		compatible = "pwm-beeper";
+		beeper-hz = <1000>;
+		pwms = <&pwm3 0 250000 0>;
+	};
+};
+
+&can_fio {
+	interrupts-extended = <&gpio4 25 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&iomuxc {
+	pinctrl_can: can-feature-grp {
+		fsl,pins = <
+			/* CAN_INT# */
+			MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25		0x400000d6
+			/* CAN_RST# */
+			MX8MM_IOMUXC_SAI2_TXD0_GPIO4_IO26		0x6
+		>;
+	};
+
+	pinctrl_gpio_expander: gpio-expander-feature-grp {
+		fsl,pins = <
+			/* GPIO4_IO27 */
+			MX8MM_IOMUXC_SAI2_MCLK_GPIO4_IO27		0x6
+		>;
+	};
+
+	pinctrl_pwm3: pwm3-buzzer-feature-grp {
+		fsl,pins = <
+			/* Buzzer PWM output */
+			MX8MM_IOMUXC_SPDIF_TX_PWM3_OUT			0x100
+		>;
+	};
+};
+
+&pinctrl_hog_feature {
+	fsl,pins = <
+		/* GPIO5_IO04 */
+		MX8MM_IOMUXC_SPDIF_RX_GPIO5_IO4			0x6
+	>;
+};
+
+&pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm3>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtsi
new file mode 100644
index 00000000000000..59f15d5a33a284
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtsi
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+&{/} {
+	hdmi-out {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&lt9611_out>;
+			};
+		};
+	};
+};
+
+&i2c_display {	/* Display connector I2C */
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	lt9611_codec: hdmi-bridge@3b {
+		compatible = "lontium,lt9611";
+		reg = <0x3b>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_panel_expansion>;
+		vcc-supply = <&buck4_reg>;	/* X400 pin 55, +3V3_S0 */
+		vdd-supply = <&buck5_reg>;	/* X400 pin 51, +1V8_S0 */
+
+		/* Audio I2S not described */
+		#sound-dai-cells = <1>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lt9611_a: endpoint {
+					remote-endpoint = <&mipi_dsi_bridge1_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				lt9611_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&iomuxc {
+	/* Free &pinctrl_panel_expansion from hog for lt9611_codec above */
+	pinctrl-0 = <&pinctrl_hog_misc>, <&pinctrl_hog_feature>,
+		    <&pinctrl_hog_panel>, <&pinctrl_hog_sbc>;
+};
+
+&mipi_dsi {
+	/* HDMI 148.5 MHz x2 (DDR) x3 (24bpp / 8) */
+	samsung,burst-clock-frequency = <891000000>;
+	samsung,esc-clock-frequency = <10000000>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			mipi_dsi_bridge1_out: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				/* Clock and data lanes have DN/DP swapped */
+				lane-polarities = <1 1 1 1 1>;
+				remote-endpoint = <&lt9611_a>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso
new file mode 100644
index 00000000000000..334ba299a3541b
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtsi"
+
+&lt9611_codec {
+	interrupts-extended = <&gpio2 3 IRQ_TYPE_EDGE_FALLING>;
+	reset-gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>;
+};
+
+&lcdif {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-common.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-common.dtsi
new file mode 100644
index 00000000000000..9abd188ee271e3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-common.dtsi
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+&{/} {
+	reg_lt9211_vcc18: regulator-lt9211-vcc18 {
+		compatible = "regulator-fixed";
+		regulator-name = "LT9211_VCC18";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&buck5_reg>;	/* X400 pin 51, +1V8_S0 */
+	};
+};
+
+&i2c_display {	/* Display connector I2C */
+	#address-cells = <1>;
+	#size-cells = <0>;
+	clock-frequency = <100000>;
+
+	lt9211_codec: bridge@2d {
+		compatible = "lontium,lt9211";
+		reg = <0x2d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_panel_expansion>;
+		interrupts-extended = <&gpio2 3 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>;
+		vccio-supply = <&reg_lt9211_vcc18>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lt9211_a: endpoint {
+					data-lanes = <1 2 3 4>;
+					remote-endpoint = <&mipi_dsi_bridge1_out>;
+				};
+			};
+		};
+	};
+
+	gpio_display: io-expander@41 {
+		compatible = "nxp,pca9536";
+		reg = <0x41>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-line-names = "SEL_12V", "SEL_5V", "SEL_PWM", "SEL_EN";
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&reg_backlight_en_level {
+	gpios = <&gpio_display 3 GPIO_ACTIVE_HIGH>; /* SEL_EN */
+};
+
+&reg_backlight_pwm_level {
+	gpios = <&gpio_display 2 GPIO_ACTIVE_HIGH>; /* SEL_PWM */
+};
+
+&reg_panel_bl {
+	gpio = <&gpio3 0 0>;
+};
+
+&iomuxc {
+	/* Free &pinctrl_panel_expansion from hog for lt9211_codec above */
+	pinctrl-0 = <&pinctrl_hog_misc>, <&pinctrl_hog_feature>,
+		    <&pinctrl_hog_panel>, <&pinctrl_hog_sbc>;
+};
+
+&mipi_dsi {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			mipi_dsi_bridge1_out: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				/* Clock and data lanes have DN/DP swapped */
+				lane-polarities = <1 1 1 1 1>;
+				remote-endpoint = <&lt9211_a>;
+			};
+		};
+	};
+};
+
+&pwm1 {
+	status = "okay";
+};
+
+&reg_panel_vcc {
+	/*
+	 * AP63300 voltage divider settings:
+	 *   R1=16k2
+	 *   R2=5k23 with optional series Rs=7k68 (5V) or Rt=1k5 (12V)
+	 *
+	 * 1 / Rx = (1 / R2) [ + (1 / Rs)][ + (1 / Rt)]
+	 * Vout = 0.8 * ((R1 / Rx) + 1)
+	 */
+	gpios = <&gpio_display 1 GPIO_ACTIVE_HIGH>,	/* 5V */
+		<&gpio_display 0 GPIO_ACTIVE_HIGH>;	/* 12V */
+	states = <3300000 0x0>,
+		 <5000000 0x1>,
+		 <12000000 0x2>,
+		 <3900000 0x3>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-dual.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-dual.dtsi
new file mode 100644
index 00000000000000..abe50eb8b4b612
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-dual.dtsi
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+&lt9211_codec {
+	ports {
+		port@2 {
+			reg = <2>;
+
+			lt9211_out_a: endpoint {
+				remote-endpoint = <&panel_lvds_a>;
+			};
+		};
+
+		port@3 {
+			reg = <3>;
+
+			lt9211_out_b: endpoint {
+				remote-endpoint = <&panel_lvds_b>;
+			};
+		};
+	};
+};
+
+&panel_lvds_a {
+	remote-endpoint = <&lt9211_out_a>;
+};
+
+&panel_lvds_b {
+	remote-endpoint = <&lt9211_out_b>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtsi
new file mode 100644
index 00000000000000..cca52464a695e0
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g070y2-l01.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi"
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <216000000>;	/* RX ByteClock ~27 MHz */
+	samsung,esc-clock-frequency = <10000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso
new file mode 100644
index 00000000000000..5d1ea31f33de37
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtsi
new file mode 100644
index 00000000000000..52d216fbba432f
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g101ice-l01.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi"
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <515000000>;
+	samsung,esc-clock-frequency = <10000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso
new file mode 100644
index 00000000000000..2d1bbdd065227a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtsi
new file mode 100644
index 00000000000000..aba3c9a1a3e8a8
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g121xce-l01.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi"
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <470000000>;
+	samsung,esc-clock-frequency = <10000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso
new file mode 100644
index 00000000000000..bc2f6fef6256e3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtsi
new file mode 100644
index 00000000000000..f478f4e557ccec
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g156hce-l01.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-dual.dtsi"
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <864000000>;
+	samsung,esc-clock-frequency = <10000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso
new file mode 100644
index 00000000000000..5bd417cbac55f4
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtsi
new file mode 100644
index 00000000000000..21bec8a01287e6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g215hvn011.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-dual.dtsi"
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <864000000>;	/* RX ByteClock ~27 MHz */
+	samsung,esc-clock-frequency = <10000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso
new file mode 100644
index 00000000000000..75ae12dfd7fc61
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtsi
new file mode 100644
index 00000000000000..38a3a9a4c75b37
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-mi0700a2t-30.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi"
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <216000000>;	/* RX ByteClock ~27 MHz */
+	samsung,esc-clock-frequency = <10000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso
new file mode 100644
index 00000000000000..ee2c79664e355a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtsi
new file mode 100644
index 00000000000000..98b5c5883cfd0d
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-mi1010z1t-1cp11.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi"
+
+&mipi_dsi {
+	samsung,burst-clock-frequency = <400000000>;
+	samsung,esc-clock-frequency = <10000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso
new file mode 100644
index 00000000000000..d05f0dcc3137c6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi
new file mode 100644
index 00000000000000..9e70c1481c1eb6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-single.dtsi
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+&lt9211_codec {
+	ports {
+		port@2 {
+			reg = <2>;
+
+			lt9211_out_a: endpoint {
+				remote-endpoint = <&panel_lvds>;
+			};
+		};
+	};
+};
+
+&panel_lvds {
+	remote-endpoint = <&lt9211_out_a>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi
new file mode 100644
index 00000000000000..31598aa86b100e
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-common.dtsi"
+
+&lcdif {
+	status = "okay";
+};
+
+&lt9211_codec {
+	interrupts-extended = <&gpio2 3 IRQ_TYPE_EDGE_FALLING>;
+	reset-gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>;
+};
+
+&reg_panel_vcc {
+	enable-gpios = <&gpio3 6 0>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtso b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtso
new file mode 100644
index 00000000000000..14038215f298c7
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-edm-sbc-imx8mm-rev900.dtso
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2023-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&fec1 {
+	phy-handle = <&fec1_phy_ath>;
+};
+
+&fec1_phy_ath {
+	status = "okay";
+};
+
+&fec1_phy_bcm {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi
new file mode 100644
index 00000000000000..0955764c0ebfff
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+&reg_backlight_pwm_level {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+};
+
+&reg_backlight_en_level {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+};
+
+&reg_panel_vcc {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-5v0.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-5v0.dtsi
new file mode 100644
index 00000000000000..70ad6bb9b80ac0
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-5v0.dtsi
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+&reg_backlight_pwm_level {
+	regulator-min-microvolt = <5000000>;
+	regulator-max-microvolt = <5000000>;
+};
+
+&reg_backlight_en_level {
+	regulator-min-microvolt = <5000000>;
+	regulator-max-microvolt = <5000000>;
+};
+
+&reg_panel_vcc {
+	regulator-min-microvolt = <5000000>;
+	regulator-max-microvolt = <5000000>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-dual.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-dual.dtsi
new file mode 100644
index 00000000000000..65f9ad81cfbc7d
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-dual.dtsi
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+&panel {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+			dual-lvds-odd-pixels;
+
+			panel_lvds_b: endpoint {
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+			dual-lvds-even-pixels;
+
+			panel_lvds_a: endpoint {
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g070y2-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g070y2-l01.dtsi
new file mode 100644
index 00000000000000..80f7b74f1ea08c
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g070y2-l01.dtsi
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl>;
+	/* 6.3 POWER ON/OFF SEQUENCE, T9 >= 10 ms */
+	pwm-off-delay-ms = <10>;
+	/* 3.2 BACKLIGHT UNIT fPWM=200 Hz (Typ.), value below in ns */
+	pwms = <&pwm1 0 5000000 0>;
+	status = "okay";
+};
+
+&panel {
+	compatible = "innolux,g070y2-l01";
+};
+
+&reg_panel_bl {
+	startup-delay-us = <10000>; /* T8 */
+	off-on-delay-us = <550000>; /* T9 + T6 + T3 + T7 + T4 + T1 + T2 + T5 */
+	status = "okay";
+};
+
+&reg_panel_vcc {
+	/* 6.3 POWER ON/OFF SEQUENCE */
+	startup-delay-us = <1000>; /* 0.5ms <= T1 + T2 <= 60 ms */
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g101ice-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g101ice-l01.dtsi
new file mode 100644
index 00000000000000..1a2bd204c30b1f
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g101ice-l01.dtsi
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl>;
+	/* 6.3 POWER ON/OFF SEQUENCE, T9 >= 10 ms */
+	pwm-off-delay-ms = <10>;
+	/* 3.2 BACKLIGHT UNIT fPWM=200 Hz (Typ.), value below in ns */
+	pwms = <&pwm1 0 5000000 0>;
+	status = "okay";
+};
+
+&panel {
+	compatible = "innolux,g101ice-l01";
+};
+
+&reg_panel_bl {
+	startup-delay-us = <10000>; /* T8 */
+	off-on-delay-us = <950000>; /* T9 + T6 + T3 + T7 + T4 + T1 + T2 + T5 */
+	status = "okay";
+};
+
+&reg_panel_vcc {
+	/* 6.3 POWER ON/OFF SEQUENCE */
+	startup-delay-us = <1000>; /* 0.5ms <= T1 + T2 <= 60 ms */
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g121xce-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g121xce-l01.dtsi
new file mode 100644
index 00000000000000..f87e8c821dacb2
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g121xce-l01.dtsi
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024 Wael Karman <wkarman@data-modul.com>
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl>;
+	/* 6.2 POWER ON/OFF SEQUENCE, T9 >= 10 ms */
+	pwm-off-delay-ms = <10>;
+	/* 3.2 BACKLIGHT UNIT fPWM=200 Hz (Typ.), value below in ns */
+	pwms = <&pwm1 0 5000000 0>;
+	status = "okay";
+};
+
+&panel {
+	compatible = "innolux,g121xce-l01";
+};
+
+&reg_panel_bl {
+	startup-delay-us = <10000>; /* T8 */
+	off-on-delay-us = <1180000>; /* T9 + T6 + T3 + T7 + T4 + T1 + T2 + T5 */
+	status = "okay";
+};
+
+&reg_panel_vcc {
+	/* 6.2 POWER ON/OFF SEQUENCE */
+	startup-delay-us = <1000>; /* 0.5ms <= T1 + T2 <= 60 ms */
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g156hce-l01.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g156hce-l01.dtsi
new file mode 100644
index 00000000000000..46c8bc7021a4fe
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g156hce-l01.dtsi
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-dual.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl>;
+	/* 4.6 POWER ON/OFF SEQUENCE, T9 >= 10 ms */
+	pwm-off-delay-ms = <10>;
+	/* 4.3.2 BACKLIGHT UNIT fPWM=200 Hz (Typ.), value below in ns */
+	pwms = <&pwm1 0 5000000 0>;
+	status = "okay";
+};
+
+&panel {
+	compatible = "innolux,g156hce-l01";
+};
+
+&reg_panel_bl {
+	startup-delay-us = <10000>; /* T8 */
+	off-on-delay-us = <1170000>; /* T9 + T6 + T3 + T7 + T4 + T1 + T2 + T5 */
+	status = "okay";
+};
+
+&reg_panel_vcc {
+	/* 4.6 POWER ON/OFF SEQUENCE */
+	startup-delay-us = <1000>; /* 0.5ms <= T1 + T2 <= 60 ms */
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g215hvn011.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g215hvn011.dtsi
new file mode 100644
index 00000000000000..3585170bd59ddc
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-g215hvn011.dtsi
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-5v0.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-dual.dtsi"
+
+&backlight {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_panel_backlight>;
+	enable-gpios = <&gpio3 0 GPIO_ACTIVE_HIGH>;
+	/* 6.5 POWER ON/OFF SEQUENCE, T6 >= 10 ms */
+	post-pwm-on-delay-ms = <10>;
+	/* 6.5 POWER ON/OFF SEQUENCE, T7 >= 0 ms */
+	pwm-off-delay-ms = <10>;
+	/* 5.2 BACKLIGHT UNIT 200Hz..20kHz, value below in ns */
+	pwms = <&pwm1 0 66666 0>;	/* 15 kHz = 66666ns */
+	status = "okay";
+};
+
+&panel {
+	/* The G215HVN01 is replacement for T215HVN01, which is supported. */
+	compatible = "auo,t215hvn01";
+};
+
+&reg_panel_vcc {
+	/* 6.5 POWER ON/OFF SEQUENCE */
+	startup-delay-us = <40000>; /* 30.5ms <= T1 + T2 <= 60 ms */
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi0700a2t-30.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi0700a2t-30.dtsi
new file mode 100644
index 00000000000000..4a077e6e6b8af3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi0700a2t-30.dtsi
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl>;
+	/* 1.5 POWER ON/OFF SEQUENCE, T4 >= 200 ms */
+	pwm-off-delay-ms = <200>;
+	/* ELECTRICAL CHARACTERISTICS, BL_ADJ Frequency 20K HZ Typ., value below in ns */
+	pwms = <&pwm1 0 50000 0>;
+	status = "okay";
+};
+
+&panel {
+	compatible = "multi-inno,mi0700a2t-30";
+};
+
+&reg_panel_bl {
+	startup-delay-us = <200000>; /* T3 */
+	off-on-delay-us = <1450000>; /* T4 + T5 + T6 + T1 + T2 + T3 */
+	status = "okay";
+};
+
+&reg_panel_vcc {
+	/* 1.5 POWER ON/OFF SEQUENCE */
+	startup-delay-us = <60000>; /* T1 + T2 >= 1 ms (typ. 60ms) */
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi1010z1t-1cp11.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi1010z1t-1cp11.dtsi
new file mode 100644
index 00000000000000..e8d8cd85d04ae1
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-mi1010z1t-1cp11.dtsi
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-3v3.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl>;
+	/* 3 POWER ON/OFF SEQUENCE, T7 >= 200 ms */
+	pwm-off-delay-ms = <200>;
+	/* ELECTRICAL CHARACTERISTICS, BL_ADJ Frequency 20K HZ Typ., value below in ns */
+	pwms = <&pwm1 0 50000 0>;
+	status = "okay";
+};
+
+&panel {
+	compatible = "multi-inno,mi1010z1t-1cp11";
+};
+
+&reg_panel_bl {
+	startup-delay-us = <200000>; /* T6 */
+	off-on-delay-us = <1450000>; /* T7 + T3 + T4 + T5 + T1 + T2 + T6 */
+	status = "okay";
+};
+
+&reg_panel_vcc {
+	/* 3 POWER ON/OFF SEQUENCE */
+	startup-delay-us = <60000>; /* T1 + T2 >= 1 ms (typ. 60ms) */
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi
new file mode 100644
index 00000000000000..68c20692241a17
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc-overlay-lvds-single.dtsi
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 Marek Vasut
+ */
+
+&panel {
+	status = "okay";
+
+	port {
+		panel_lvds: endpoint {
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc.dts b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc.dts
index 472c584fb3bd29..cb764714404337 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-data-modul-edm-sbc.dts
@@ -30,11 +30,8 @@
 
 	backlight: backlight {
 		compatible = "pwm-backlight";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_panel_backlight>;
 		brightness-levels = <0 1 10 20 30 40 50 60 70 75 80 90 100>;
 		default-brightness-level = <7>;
-		enable-gpios = <&gpio3 0 GPIO_ACTIVE_HIGH>;
 		pwms = <&pwm1 0 5000000 0>;
 		/* Disabled by default, unless display board plugged in. */
 		status = "disabled";
@@ -66,7 +63,6 @@
 		regulator-name = "PANEL_VCC";
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
-		gpio = <&gpio3 6 0>;
 		enable-active-high;
 		/* Disabled by default, unless display board plugged in. */
 		status = "disabled";
@@ -435,7 +431,63 @@
 	eeprom: eeprom@50 {
 		compatible = "atmel,24c32";
 		reg = <0x50>;
+		label = "dmo-board-info";
 		pagesize = <32>;
+		wp-gpios = <&gpio2 5 GPIO_ACTIVE_LOW>;
+
+		nvmem-layout {
+			compatible = "fixed-layout";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			company-name@24 {
+				reg = <0x24 0xd>;
+			};
+
+			product-name@32 {
+				reg = <0x32 0x13>;
+			};
+
+			board-revision@4b {
+				reg = <0x4b 0x3>;
+			};
+
+			serial-number@4f {
+				reg = <0x4f 0x8>;
+			};
+
+			manufacturing-date@70 {
+				reg = <0x70 0xa>;
+			};
+
+			test-date@7b {
+				reg = <0x7b 0xa>;
+			};
+
+			product-number@86 {
+				reg = <0x86 0x8>;
+			};
+
+			board-variant@9a {
+				reg = <0x9a 0x9>;
+			};
+
+			company-acronym@a4 {
+				reg = <0xa4 0x3>;
+			};
+
+			mac-address@b0 {
+				reg = <0xb0 0x6>;
+			};
+
+			board-id@100 {
+				reg = <0x100 0x4>;
+			};
+
+			wifi-id@120 {
+				reg = <0x120 0x4>;
+			};
+		};
 	};
 
 	rtc: rtc@68 {
@@ -454,7 +506,7 @@
 	};
 };
 
-&i2c3 {	/* Display connector I2C */
+i2c_display: &i2c3 {	/* Display connector I2C */
 	/* IMX8MM ERRATA e7805 -- I2C is limited to 384 kHz due to SoC bug */
 	clock-frequency = <320000>;
 	pinctrl-names = "default", "gpio";
@@ -465,7 +517,7 @@
 	status = "okay";
 };
 
-&i2c4 {	/* Feature connector I2C */
+i2c_feature: &i2c4 {	/* Feature connector I2C */
 	/* IMX8MM ERRATA e7805 -- I2C is limited to 384 kHz due to SoC bug */
 	clock-frequency = <320000>;
 	pinctrl-names = "default", "gpio";
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
index dd59af0ebaae55..27bfad56dbe85a 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
@@ -5,6 +5,7 @@
 
 /dts-v1/;
 
+#include <dt-bindings/leds/common.h>
 #include "imx8mm-kontron-sl.dtsi"
 
 / {
@@ -96,6 +97,7 @@
 	};
 };
 
+/* CAN */
 &ecspi2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_ecspi2>;
@@ -121,8 +123,15 @@
 	pinctrl-0 = <&pinctrl_ecspi3>;
 	cs-gpios = <&gpio5 25 GPIO_ACTIVE_LOW>;
 	status = "okay";
+
+	eeram@0 {
+		compatible = "microchip,48l640";
+		reg = <0>;
+		spi-max-frequency = <20000000>;
+	};
 };
 
+/* ETH0 */
 &fec1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_enet>;
@@ -139,13 +148,47 @@
 			reset-assert-us = <1>;
 			reset-deassert-us = <15000>;
 			reset-gpios = <&gpio4 27 GPIO_ACTIVE_LOW>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@0 {
+					reg = <0>;
+					color = <LED_COLOR_ID_GREEN>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+
+				led@1 {
+					reg = <1>;
+					color = <LED_COLOR_ID_YELLOW>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+			};
 		};
 	};
 };
 
+/* DIO */
+&gpio1 {
+	gpio-line-names = "", "", "", "DOUT1",
+			  "", "", "DIN1", "DOUT2",
+			  "DIN2", "DOUT3", "DIN3", "DOUT4";
+};
+
 &gpio4 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gpio4>;
+	gpio-line-names = "", "CAN_ADR0", "", "",
+			  "CAN_ADR1", "CAN_ADR2", "CAN_ADR3", "LED6",
+			  "LED4", "LED5", "", "",
+			  "", "", "DSI_MUX_SEL", "DSI_MUX_OE",
+			  "HDMI_IRQ", "LED1", "LED3", "LED2",
+			  "", "", "", "",
+			  "", "ETH_IRQ", "LVDS_EN", "PHY_RST",
+			  "CAN_IRQ", "UART2_RTS";
 
 	dsi_mux_sel_hdmi: dsi-mux-sel-hdmi-hog {
 		gpio-hog;
@@ -170,6 +213,10 @@
 	};
 };
 
+&gpio5 {
+	gpio-line-names = "", "", "DIN4";
+};
+
 &i2c3 {
 	clock-frequency = <400000>;
 	pinctrl-names = "default";
@@ -254,6 +301,7 @@
 	status = "okay";
 };
 
+/* RS232 */
 &uart1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart1>;
@@ -261,6 +309,7 @@
 	status = "okay";
 };
 
+/* RS485 */
 &uart2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart2>;
@@ -304,6 +353,7 @@
 	};
 };
 
+/* SD-card */
 &usdhc2 {
 	pinctrl-names = "default", "state_100mhz", "state_200mhz";
 	pinctrl-0 = <&pinctrl_usdhc2>;
@@ -321,13 +371,13 @@
 
 	pinctrl_adv7535: adv7535grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SAI1_TXD4_GPIO4_IO16		0x19
+			MX8MM_IOMUXC_SAI1_TXD4_GPIO4_IO16		0x19 /* HDMI_IRQ */
 		>;
 	};
 
 	pinctrl_can: cangrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SAI3_RXFS_GPIO4_IO28		0x19
+			MX8MM_IOMUXC_SAI3_RXFS_GPIO4_IO28		0x19 /* CAN_IRQ */
 		>;
 	};
 
@@ -336,7 +386,7 @@
 			MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO		0x82
 			MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI		0x82
 			MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK		0x82
-			MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13		0x19
+			MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13		0x19 /* ECSPI2_CS1 */
 		>;
 	};
 
@@ -345,7 +395,7 @@
 			MX8MM_IOMUXC_UART2_RXD_ECSPI3_MISO		0x82
 			MX8MM_IOMUXC_UART1_TXD_ECSPI3_MOSI		0x82
 			MX8MM_IOMUXC_UART1_RXD_ECSPI3_SCLK		0x82
-			MX8MM_IOMUXC_UART2_TXD_GPIO5_IO25		0x19
+			MX8MM_IOMUXC_UART2_TXD_GPIO5_IO25		0x19 /* ECSPI3_CS1 */
 		>;
 	};
 
@@ -365,40 +415,45 @@
 			MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC		0x91
 			MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL	0x91
 			MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL	0x1f
-			MX8MM_IOMUXC_SAI2_MCLK_GPIO4_IO27		0x19 /* PHY RST */
-			MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25		0x19 /* ETH IRQ */
+			MX8MM_IOMUXC_SAI2_MCLK_GPIO4_IO27		0x19 /* PHY_RST */
+			MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25		0x19 /* ETH_IRQ */
 		>;
 	};
 
 	pinctrl_gpio_led: gpioledgrp {
 		fsl,pins = <
 			MX8MM_IOMUXC_NAND_READY_B_GPIO3_IO16		0x19
-			MX8MM_IOMUXC_SAI1_RXD5_GPIO4_IO7		0x19
-			MX8MM_IOMUXC_SAI1_RXD6_GPIO4_IO8		0x19
-			MX8MM_IOMUXC_SAI1_RXD7_GPIO4_IO9		0x19
-			MX8MM_IOMUXC_SAI1_TXD5_GPIO4_IO17		0x19
-			MX8MM_IOMUXC_SAI1_TXD6_GPIO4_IO18		0x19
-			MX8MM_IOMUXC_SAI1_TXD7_GPIO4_IO19		0x19
+			MX8MM_IOMUXC_SAI1_RXD5_GPIO4_IO7		0x19 /* LED6 */
+			MX8MM_IOMUXC_SAI1_RXD6_GPIO4_IO8		0x19 /* LED4 */
+			MX8MM_IOMUXC_SAI1_RXD7_GPIO4_IO9		0x19 /* LED5 */
+			MX8MM_IOMUXC_SAI1_TXD5_GPIO4_IO17		0x19 /* LED1 */
+			MX8MM_IOMUXC_SAI1_TXD6_GPIO4_IO18		0x19 /* LED3 */
+			MX8MM_IOMUXC_SAI1_TXD7_GPIO4_IO19		0x19 /* LED2 */
 		>;
 	};
 
 	pinctrl_gpio: gpiogrp {
 		fsl,pins = <
-			MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3		0x19
-			MX8MM_IOMUXC_GPIO1_IO07_GPIO1_IO7		0x19
-			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9		0x19
-			MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11		0x19
-			MX8MM_IOMUXC_GPIO1_IO06_GPIO1_IO6		0x19
-			MX8MM_IOMUXC_GPIO1_IO08_GPIO1_IO8		0x19
-			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10		0x19
-			MX8MM_IOMUXC_SAI3_MCLK_GPIO5_IO2		0x19
+			MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3		0x19 /* DOUT1 */
+			MX8MM_IOMUXC_GPIO1_IO06_GPIO1_IO6		0x19 /* DIN1 */
+			MX8MM_IOMUXC_GPIO1_IO07_GPIO1_IO7		0x19 /* DOUT2 */
+			MX8MM_IOMUXC_GPIO1_IO08_GPIO1_IO8		0x19 /* DIN2 */
+			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9		0x19 /* DOUT3 */
+			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10		0x19 /* DIN3 */
+			MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11		0x19 /* DOUT4 */
+			MX8MM_IOMUXC_SAI3_MCLK_GPIO5_IO2		0x19 /* DIN4 */
 		>;
 	};
 
 	pinctrl_gpio4: gpio4grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SAI1_TXD2_GPIO4_IO14		0x19
-			MX8MM_IOMUXC_SAI1_TXD3_GPIO4_IO15		0x19
+			/* CAN address DIP switches 1-4 */
+			MX8MM_IOMUXC_SAI1_RXC_GPIO4_IO1			0x19 /* CAN_ADR0 */
+			MX8MM_IOMUXC_SAI1_RXD2_GPIO4_IO4		0x19 /* CAN_ADR1 */
+			MX8MM_IOMUXC_SAI1_RXD3_GPIO4_IO5		0x19 /* CAN_ADR2 */
+			MX8MM_IOMUXC_SAI1_RXD4_GPIO4_IO6		0x19 /* CAN_ADR3 */
+			MX8MM_IOMUXC_SAI1_TXD2_GPIO4_IO14		0x19 /* DSI_MUX_SEL */
+			MX8MM_IOMUXC_SAI1_TXD3_GPIO4_IO15		0x19 /* DSI_MUX_OE */
 		>;
 	};
 
@@ -424,7 +479,7 @@
 
 	pinctrl_sn65dsi84: sn65dsi84grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_SAI2_TXD0_GPIO4_IO26		0x19
+			MX8MM_IOMUXC_SAI2_TXD0_GPIO4_IO26		0x19 /* LVDS_EN */
 			MX8MM_IOMUXC_SD2_WP_GPIO2_IO20			0x19
 		>;
 	};
@@ -443,13 +498,13 @@
 			MX8MM_IOMUXC_SAI3_TXFS_UART2_DCE_RX		0x0
 			MX8MM_IOMUXC_SAI3_TXC_UART2_DCE_TX		0x0
 			MX8MM_IOMUXC_SAI3_RXD_UART2_DCE_RTS_B		0x0
-			MX8MM_IOMUXC_SAI3_RXC_GPIO4_IO29		0x19
+			MX8MM_IOMUXC_SAI3_RXC_GPIO4_IO29		0x19 /* UART2_RTS */
 		>;
 	};
 
 	pinctrl_usb_eth2: usbeth2grp {
 		fsl,pins = <
-			MX8MM_IOMUXC_NAND_CE1_B_GPIO3_IO2		0x19
+			MX8MM_IOMUXC_NAND_CE1_B_GPIO3_IO2		0x19 /* ETH_RST */
 		>;
 	};
 
@@ -461,7 +516,7 @@
 			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d0
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d0
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d0
-			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x19
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x19 /* SD2_CD */
 			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0xd0
 		>;
 	};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi
index 2324063f8fd90e..c5f0e22f549b45 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-var-dart.dtsi
@@ -123,7 +123,7 @@
 			reg = <0>;
 			reset-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>;
 			reset-assert-us = <10000>;
-			reset-deassert-us = <100000>;
+			reset-deassert-us = <200000>;
 			vddio-supply = <&reg_phy_vddio>;
 		};
 	};
@@ -138,6 +138,16 @@
 	sda-gpios = <&gpio5 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	status = "okay";
 
+	st33ktpm2xi2c_som: tpm@2e {
+		compatible = "st,st33ktpm2xi2c", "tcg,tpm-tis-i2c";
+		reg = <0x2e>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_tpm_reset>;
+		label = "tpm";
+		/* active-low TPM reset line. */
+		reset-gpios = <&gpio4 31 GPIO_ACTIVE_LOW>;
+	};
+
 	pmic@4b {
 		compatible = "rohm,bd71847";
 		reg = <0x4b>;
@@ -305,6 +315,7 @@
 	assigned-clock-parents = <&clk IMX8MM_AUDIO_PLL1_OUT>;
 	assigned-clock-rates = <1536000>;
 	fsl,sai-mclk-direction-output;
+	fsl,sai-synchronous-rx;
 	status = "okay";
 };
 
@@ -446,13 +457,17 @@
 			MX8MM_IOMUXC_SAI3_RXFS_SAI3_RX_SYNC		0xd6
 			MX8MM_IOMUXC_SAI3_RXC_SAI3_RX_BCLK		0xd6
 			MX8MM_IOMUXC_SAI3_RXD_SAI3_RX_DATA0		0xd6
-			MX8MM_IOMUXC_SAI3_TXFS_SAI3_TX_SYNC		0xd6
-			MX8MM_IOMUXC_SAI3_TXC_SAI3_TX_BCLK		0xd6
 			MX8MM_IOMUXC_SAI3_TXD_SAI3_TX_DATA0		0xd6
 			MX8MM_IOMUXC_SAI3_MCLK_SAI3_MCLK		0xd6
 		>;
 	};
 
+	pinctrl_tpm_reset: tpm-resetgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_TXFS_GPIO4_IO31		0x19
+		>;
+	};
+
 	pinctrl_uart4: uart4grp {
 		fsl,pins = <
 			MX8MM_IOMUXC_ECSPI2_SCLK_UART4_DCE_RX		0x140
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin-ivy.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-verdin-ivy.dtsi
index 29075ff5eda63b..b64751aa31d50e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-verdin-ivy.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin-ivy.dtsi
@@ -270,8 +270,8 @@
 		"",
 		"",
 		"",
-		"DIG_1", /* SODIMM 56 */
-		"DIG_2", /* SODIMM 58 */
+		"DIGI_1", /* SODIMM 56 */
+		"DIGI_2", /* SODIMM 58 */
 		"REL1",  /* SODIMM 60 */
 		"REL2",  /* SODIMM 62 */
 		"", /* 10 */
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-solidsense-n8-compact.dts b/arch/arm64/boot/dts/freescale/imx8mn-solidsense-n8-compact.dts
index c8c6760524dbeb..9b8b971c1a5c24 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-solidsense-n8-compact.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mn-solidsense-n8-compact.dts
@@ -226,8 +226,6 @@
 			reset-assert-us = <10>;
 			reset-deassert-us = <5000>;
 			reset-gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;
-			adi,led-polarity = <GPIO_ACTIVE_LOW>;
-			adi,link-st-polarity = <GPIO_ACTIVE_LOW>;
 			status = "disabled";
 
 			leds {
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-common.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-common.dtsi
index aaf9761703aafe..d420ab398b22ec 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-common.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-common.dtsi
@@ -142,9 +142,10 @@
 	pinctrl-0 = <&pinctrl_i2c1>;
 	pinctrl-1 = <&pinctrl_i2c1_gpio>;
 
-	eeprom@51 {
+	cfg_eeprom: eeprom@51 {
 		compatible = "atmel,24c128";
 		reg = <0x51>;
+		label = "ifm-deviceinfo";
 	};
 
 	hw_rtc: rtc@52 {
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts
index a8f7c226a61f86..363f487f8c845d 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts
@@ -13,6 +13,11 @@
 		     "ifm,imx8mn-vhip4", "fsl,imx8mn";
 };
 
+&cfg_eeprom {
+	/* Shared with logging EEPROM on v1 hardware */
+	wp-gpios = <&gpio5 20 GPIO_ACTIVE_HIGH>;
+};
+
 &ifm_led {
 	pinctrl-1 = <&pinctrl_gpio_led_v1>;
 
@@ -91,6 +96,9 @@
 	eeprom@50 {
 		compatible = "atmel,24c128";
 		reg = <0x50>;
+		label = "ifm-logging";
+		/* Shared with configuration EEPROM on v1 hardware */
+		wp-gpios = <&gpio5 20 GPIO_ACTIVE_HIGH>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts
index 43fd4d0041ef30..0fcffd168f6ac4 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts
@@ -20,6 +20,12 @@
 	};
 };
 
+&cfg_eeprom {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_cfgeeprom>;
+	wp-gpios = <&gpio2 11 GPIO_ACTIVE_HIGH>;
+};
+
 &ifm_led {
 	pinctrl-1 = <&pinctrl_gpio_led_v2>;
 
@@ -88,6 +94,10 @@
 	eeprom@54 {
 		compatible = "atmel,24c128";
 		reg = <0x54>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_logeeprom>;
+		label = "ifm-logging";
+		wp-gpios = <&gpio5 3 GPIO_ACTIVE_HIGH>;
 	};
 };
 
@@ -103,6 +113,13 @@
 };
 
 &iomuxc {
+	pinctrl_cfgeeprom: cfgeeprom-grp {
+		fsl,pins = <
+			/* CFG_EEPROM_WP */
+			MX8MN_IOMUXC_SD1_STROBE_GPIO2_IO11		0x140
+		>;
+	};
+
 	pinctrl_ecspi1_cs: ecspi1-cs-grp {
 		fsl,pins = <
 			/* KS8794 nCS */
@@ -143,12 +160,8 @@
 		fsl,pins = <
 			MX8MN_IOMUXC_SD1_DATA4_GPIO2_IO6		0x56
 			MX8MN_IOMUXC_SD1_DATA5_GPIO2_IO7		0xd6
-			/* CFG_EEPROM_WP */
-			MX8MN_IOMUXC_SD1_STROBE_GPIO2_IO11		0x140
 			/* RTC_nIRQ */
 			MX8MN_IOMUXC_SPDIF_EXT_CLK_GPIO5_IO5		0x116
-			/* LOG_EE_WP */
-			MX8MN_IOMUXC_SPDIF_TX_GPIO5_IO3			0x140
 		>;
 	};
 
@@ -166,6 +179,13 @@
 		>;
 	};
 
+	pinctrl_logeeprom: logeeprom-grp {
+		fsl,pins = <
+			/* LOG_EE_WP */
+			MX8MN_IOMUXC_SPDIF_TX_GPIO5_IO3			0x140
+		>;
+	};
+
 	pinctrl_mcp2518: mcp2518-grp {
 		fsl,pins = <
 			/* CAN0_CLKO */
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1000.dtso b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1000.dtso
new file mode 100644
index 00000000000000..da3a40a310faf8
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1000.dtso
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&cfg_eeprom {
+	cfg_eeprom_layout: nvmem-layout {
+		compatible = "fixed-layout";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		vhip-signature@0 {
+			reg = <0x0 0x4>;
+		};
+
+		vhip-version@4 {
+			reg = <0x4 0x4>;
+		};
+
+		vhip-size@8 {
+			reg = <0x8 0x4>;
+		};
+
+		device-serial@c {
+			reg = <0xc 0x10>;
+		};
+
+		order-number@1c {
+			reg = <0x1c 0x20>;
+		};
+
+		article-name@3c {
+			reg = <0x3c 0x20>;
+		};
+
+		article-rev@5c {
+			reg = <0x5c 0x4>;
+		};
+
+		hardware-rev@60 {
+			reg = <0x60 0x4>;
+		};
+
+		product-type@64 {
+			reg = <0x64 0x20>;
+		};
+
+		software-compatibility@84 {
+			reg = <0x84 0x4>;
+		};
+
+		manufacturing-date@88 {
+			reg = <0x88 0x20>;
+		};
+
+		production-order-number@a8 {
+			reg = <0xa8 0x20>;
+		};
+
+		/* Each MAC address is padded to 8 Bytes */
+		cfg_eeprom_mac1: mac-address-1@c8 {
+			reg = <0xc8 0x8>;
+		};
+
+		cfg_eeprom_mac2: mac-address-2@d0 {
+			reg = <0xd0 0x8>;
+		};
+
+		cfg_eeprom_mac3: mac-address-3@d8 {
+			reg = <0xd8 0x8>;
+		};
+
+		cfg_eeprom_mac4: mac-address-4@e0 {
+			reg = <0xe0 0x8>;
+		};
+
+		cfg_eeprom_mac5: mac-address-5@e8 {
+			reg = <0xe8 0x8>;
+		};
+
+		cfg_eeprom_mac6: mac-address-6@f0 {
+			reg = <0xf0 0x8>;
+		};
+
+		cfg_eeprom_mac7: mac-address-7@f8 {
+			reg = <0xf8 0x8>;
+		};
+
+		cfg_eeprom_mac8: mac-address-8@100 {
+			reg = <0x100 0x8>;
+		};
+
+		next-block-offset@108 {
+			reg = <0x108 0x4>;
+		};
+
+		section-crc@10c {
+			reg = <0x10c 0x4>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1100.dtso b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1100.dtso
new file mode 100644
index 00000000000000..30df55621b7b4f
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-1100.dtso
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&cfg_eeprom {
+	cfg_eeprom_layout: nvmem-layout {
+		compatible = "fixed-layout";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		vhip-signature@0 {
+			reg = <0x0 0x4>;
+		};
+
+		vhip-version@4 {
+			reg = <0x4 0x4>;
+		};
+
+		vhip-size@8 {
+			reg = <0x8 0x4>;
+		};
+
+		device-serial@c {
+			reg = <0xc 0x10>;
+		};
+
+		order-number@1c {
+			reg = <0x1c 0x20>;
+		};
+
+		article-name@3c {
+			reg = <0x3c 0x40>;
+		};
+
+		article-rev@7c {
+			reg = <0x7c 0x4>;
+		};
+
+		hardware-rev@80 {
+			reg = <0x80 0x4>;
+		};
+
+		product-type@84 {
+			reg = <0x84 0x20>;
+		};
+
+		software-compatibility@a4 {
+			reg = <0xa4 0x4>;
+		};
+
+		manufacturing-date@a8 {
+			reg = <0xa8 0x20>;
+		};
+
+		production-order-number@c8 {
+			reg = <0xc8 0x20>;
+		};
+
+		/* Each MAC address is padded to 8 Bytes */
+		cfg_eeprom_mac1: mac-address-1@e8 {
+			reg = <0xe8 0x8>;
+		};
+
+		cfg_eeprom_mac2: mac-address-2@f0 {
+			reg = <0xf0 0x8>;
+		};
+
+		cfg_eeprom_mac3: mac-address-3@f8 {
+			reg = <0xf8 0x8>;
+		};
+
+		cfg_eeprom_mac4: mac-address-4@100 {
+			reg = <0x100 0x8>;
+		};
+
+		cfg_eeprom_mac5: mac-address-5@108 {
+			reg = <0x108 0x8>;
+		};
+
+		cfg_eeprom_mac6: mac-address-6@110 {
+			reg = <0x110 0x8>;
+		};
+
+		cfg_eeprom_mac7: mac-address-7@118 {
+			reg = <0x118 0x8>;
+		};
+
+		cfg_eeprom_mac8: mac-address-8@120 {
+			reg = <0x120 0x8>;
+		};
+
+		next-block-offset@128 {
+			reg = <0x128 0x4>;
+		};
+
+		section-crc@12c {
+			reg = <0x12c 0x4>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-2000.dtso b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-2000.dtso
new file mode 100644
index 00000000000000..a2ca7ee8a2d2d5
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-overlay-eeprom-2000.dtso
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&cfg_eeprom {
+	cfg_eeprom_layout: nvmem-layout {
+		compatible = "fixed-layout";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		vhip-signature@0 {
+			reg = <0x0 0x4>;
+		};
+
+		vhip-version@4 {
+			reg = <0x4 0x4>;
+		};
+
+		vhip-size@8 {
+			reg = <0x8 0x4>;
+		};
+
+		section-data-crc@c {
+			reg = <0xc 0x4>;
+		};
+
+		next-block-offset@10 {
+			reg = <0x10 0x4>;
+		};
+
+		section-header-crc@14 {
+			reg = <0x14 0x4>;
+		};
+
+		device-serial@18 {
+			reg = <0x18 0x10>;
+		};
+
+		order-number@28 {
+			reg = <0x28 0x20>;
+		};
+
+		article-name@48 {
+			reg = <0x48 0x40>;
+		};
+
+		article-rev@88 {
+			reg = <0x88 0x4>;
+		};
+
+		hardware-rev@8c {
+			reg = <0x8c 0x4>;
+		};
+
+		product-type@90 {
+			reg = <0x90 0x20>;
+		};
+
+		software-compatibility@b0 {
+			reg = <0xb0 0x4>;
+		};
+
+		manufacturing-date@b4 {
+			reg = <0xb4 0x20>;
+		};
+
+		production-order-number@d4 {
+			reg = <0xd4 0x20>;
+		};
+
+		/* Each MAC address is padded to 8 Bytes */
+		cfg_eeprom_mac1: mac-address-1@f4 {
+			reg = <0xf4 0x8>;
+		};
+
+		cfg_eeprom_mac2: mac-address-2@fc {
+			reg = <0xfc 0x8>;
+		};
+
+		cfg_eeprom_mac3: mac-address-3@104 {
+			reg = <0x104 0x8>;
+		};
+
+		cfg_eeprom_mac4: mac-address-4@10c {
+			reg = <0x10c 0x8>;
+		};
+
+		cfg_eeprom_mac5: mac-address-5@114 {
+			reg = <0x114 0x8>;
+		};
+
+		cfg_eeprom_mac6: mac-address-6@11c {
+			reg = <0x11c 0x8>;
+		};
+
+		cfg_eeprom_mac7: mac-address-7@124 {
+			reg = <0x124 0x8>;
+		};
+
+		cfg_eeprom_mac8: mac-address-8@12c {
+			reg = <0x12c 0x8>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-cm7.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-cm7.dtso
new file mode 100644
index 00000000000000..11d342712da618
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-cm7.dtso
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2023-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/imx8mp-clock.h>
+
+&{/} {
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	reserved-memory {	/* CM7 reserved memory */
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		m_core_reserved: m_core@54000000 {
+			reg = <0 0x54000000 0 0x1000000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@55000000 {
+			reg = <0 0x55000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@55008000 {
+			reg = <0 0x55008000 0 0x8000>;
+			no-map;
+		};
+
+		rsc_table: rsc-table@550ff000 {
+			reg = <0 0x550ff000 0 0x1000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@55400000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x55400000 0 0x100000>;
+			no-map;
+		};
+	};
+
+	imx8mp-cm7 {
+		compatible = "fsl,imx8mp-cm7-mmio";
+		clocks = <&clk IMX8MP_CLK_M7_CORE>;
+		fsl,iomuxc-gpr = <&gpr>;
+		mbox-names = "tx", "rx", "rxdb";
+		mboxes = <&mu 0 1
+			  &mu 1 1
+			  &mu 3 1>;
+		memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>, <&rsc_table>;
+		syscon = <&src>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso
new file mode 100644
index 00000000000000..4e2112355350fa
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtso
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/imx8mp-clock.h>
+
+#include "imx8mp-pinfunc.h"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1-audio.dtsi"
+
+&can_fio {
+	interrupts-extended = <&gpio2 10 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&gpio_feature {
+	interrupt-parent = <&gpio1>;
+	interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&iomuxc {
+	pinctrl_codec_mclk: codec-mclk_feature-grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI2_MCLK__AUDIOMIX_SAI5_MCLK	0xd6
+		>;
+	};
+
+	sai2-grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SAI2_TXFS__AUDIOMIX_SAI2_TX_SYNC	0xd6
+			MX8MP_IOMUXC_SAI2_TXD0__AUDIOMIX_SAI2_TX_DATA00	0xd6
+			MX8MP_IOMUXC_SAI2_TXC__AUDIOMIX_SAI2_TX_BCLK	0xd6
+			MX8MP_IOMUXC_SAI2_RXD0__AUDIOMIX_SAI2_RX_DATA00	0xd6
+		>;
+	};
+
+	uart1-grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__UART1_DCE_TX		0x49
+			MX8MP_IOMUXC_SD1_CMD__UART1_DCE_RX		0x49
+			MX8MP_IOMUXC_SD1_DATA1__UART1_DCE_CTS		0x49
+		>;
+	};
+};
+
+&sai2 {
+	assigned-clocks = <&clk IMX8MP_CLK_SAI2>;
+	assigned-clock-parents = <&clk IMX8MP_AUDIO_PLL2_OUT>;
+};
+
+&spba5 {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	sai5clk: clock-controller@30c50000 {
+		compatible = "fsl,imx8mp-sai-clock", "fsl,imx8mq-sai-clock";
+		reg = <0x30c50000 0x10000>;
+		#clock-cells = <1>;
+		clocks = <&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI5_IPG>,
+			 <&audio_blk_ctrl IMX8MP_CLK_AUDIOMIX_SAI5_MCLK1>;
+		clock-names = "bus", "mclk1";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_codec_mclk>;
+		status = "okay";
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso
new file mode 100644
index 00000000000000..0eccb7f7c0a8ce
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtso
@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mp-pinfunc.h"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-fio1.dtsi"
+
+&can_fio {
+	interrupts-extended = <&gpio2 10 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&iomuxc {
+	pinctrl_can: can-feature-grp {
+		fsl,pins = <
+			/* CAN_INT# */
+			MX8MP_IOMUXC_SD1_RESET_B__GPIO2_IO10		0x400000d6
+		>;
+	};
+
+	pinctrl_gpio_expander: gpio-expander-feature-grp {
+		fsl,pins = <
+			/* GPIO4_IO27 */
+			MX8MP_IOMUXC_SAI2_MCLK__GPIO4_IO27		0x6
+		>;
+	};
+};
+
+&pinctrl_sai2 {
+	fsl,pins = <
+		MX8MP_IOMUXC_SAI2_TXFS__AUDIOMIX_SAI2_TX_SYNC	0xd6
+		MX8MP_IOMUXC_SAI2_TXD0__AUDIOMIX_SAI2_TX_DATA00	0xd6
+		MX8MP_IOMUXC_SAI2_TXC__AUDIOMIX_SAI2_TX_BCLK	0xd6
+	>;
+};
+
+&pinctrl_hog_feature {
+	fsl,pins = <
+		/* GPIO5_IO03 */
+		MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07		0x40000006
+		/* GPIO5_IO04 */
+		MX8MP_IOMUXC_GPIO1_IO08__GPIO1_IO08		0x40000006
+	>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso
new file mode 100644
index 00000000000000..d8114fd869b54a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtso
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-hdmi.dtsi"
+
+&lt9611_codec {
+	interrupts-extended = <&gpio4 19 IRQ_TYPE_EDGE_FALLING>;
+	reset-gpios = <&gpio4 0 GPIO_ACTIVE_HIGH>;
+};
+
+&lcdif1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso
new file mode 100644
index 00000000000000..6b6b160c0f3a1a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g070y2-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso
new file mode 100644
index 00000000000000..549ea2ab8819aa
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g101ice-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso
new file mode 100644
index 00000000000000..c8c43282181521
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g121xce-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso
new file mode 100644
index 00000000000000..bf5b41ae61dbed
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g156hce-l01.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso
new file mode 100644
index 00000000000000..faf2c06fc50c33
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtso
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-g215hvn011.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl_supply>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso
new file mode 100644
index 00000000000000..359a8b1521b016
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi0700a2t-30.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso
new file mode 100644
index 00000000000000..525cedb64a776d
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtso
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-mi1010z1t-1cp11.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi
new file mode 100644
index 00000000000000..6c346baee903fa
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds.dtsi
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-lvds-common.dtsi"
+
+&{/} {
+	reg_panel_bl_supply: regulator-panel-bl-supply {
+		compatible = "regulator-fixed";
+		regulator-name = "BKLT0";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+};
+
+&lcdif1 {
+	status = "okay";
+};
+
+&lt9211_codec {
+	interrupts-extended = <&gpio4 19 IRQ_TYPE_EDGE_FALLING>;
+	reset-gpios = <&gpio4 0 GPIO_ACTIVE_HIGH>;
+};
+
+&reg_panel_bl {
+	vin-supply = <&reg_panel_bl_supply>;
+};
+
+&reg_panel_vcc {
+	enable-gpios = <&gpio3 6 0>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtso
new file mode 100644
index 00000000000000..7a4126214a4f61
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g070y2-l01.dtso
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g070y2-l01.dtsi"
+
+&media_blk_ctrl {
+	/*
+	 * The G070Y2-L01 panel requires 29.5 MHz LVDS clock.
+	 * Set IMX8MP_VIDEO_PLL1 to 206.5 MHz , since 206.5 MHz / 7 = 29.5 MHz .
+	 */
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>, <206500000>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_lvds>;
+};
+
+&panel_lvds {
+	remote-endpoint = <&ldb_lvds_ch1>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtso
new file mode 100644
index 00000000000000..817d4ec62d0e88
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g101ice-l01.dtso
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g101ice-l01.dtsi"
+
+&media_blk_ctrl {
+	/*
+	 * The G101ICE-L01 panel requires 71.1 MHz LVDS clock.
+	 * Set IMX8MP_VIDEO_PLL1 to 497.7 MHz , since 497.7 MHz / 7 = 71.1 MHz .
+	 */
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>, <497700000>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_lvds>;
+};
+
+&panel_lvds {
+	remote-endpoint = <&ldb_lvds_ch1>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtso
new file mode 100644
index 00000000000000..729f477038b17a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g121xce-l01.dtso
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024 Wael Karman <wkarman@data-modul.com>
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g121xce-l01.dtsi"
+
+&media_blk_ctrl {
+	/*
+	 * The G121XCE-L01 panel requires 64.9 MHz LVDS clock.
+	 * Set IMX8MP_VIDEO_PLL1 to 454.3 MHz , since 454.3 MHz / 7 = 64.9 MHz .
+	 */
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>, <454300000>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_lvds>;
+};
+
+&panel_lvds {
+	remote-endpoint = <&ldb_lvds_ch1>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtso
new file mode 100644
index 00000000000000..86163d6ddd5c69
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g156hce-l01.dtso
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g156hce-l01.dtsi"
+
+&media_blk_ctrl {
+	/*
+	 * The G156HCE-L01 panel requires 141.86 MHz LVDS clock.
+	 * Set IMX8MP_VIDEO_PLL1 to 993.2 MHz , since 993.2 MHz / 7 = 141.86 MHz .
+	 */
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>, <993020000>;
+};
+
+&ldb_lvds_ch0 {
+	remote-endpoint = <&panel_lvds_b>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_lvds_a>;
+};
+
+&panel_lvds_a {
+	remote-endpoint = <&ldb_lvds_ch1>;
+};
+
+&panel_lvds_b {
+	remote-endpoint = <&ldb_lvds_ch0>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtso
new file mode 100644
index 00000000000000..a6d13c36097966
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-g215hvn011.dtso
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-g215hvn011.dtsi"
+
+&backlight {
+	power-supply = <&reg_panel_bl_supply>;
+};
+
+&media_blk_ctrl {
+	/*
+	 * The G215HVN01 panel requires 148.8 MHz LVDS clock.
+	 * Set IMX8MP_VIDEO_PLL1 to 1041.6 MHz , since 1041.6 MHz / 7 = 148.8 MHz .
+	 */
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>, <1041600000>;
+};
+
+&ldb_lvds_ch0 {
+	remote-endpoint = <&panel_lvds_b>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_lvds_a>;
+};
+
+&panel_lvds_a {
+	remote-endpoint = <&ldb_lvds_ch1>;
+};
+
+&panel_lvds_b {
+	remote-endpoint = <&ldb_lvds_ch0>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtso
new file mode 100644
index 00000000000000..76bfbd307ba2d7
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi0700a2t-30.dtso
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-mi0700a2t-30.dtsi"
+
+&media_blk_ctrl {
+	/*
+	 * The MI0700A2T-30 panel requires 33 MHz LVDS clock.
+	 * Set IMX8MP_VIDEO_PLL1 to 231 MHz , since 231 MHz / 7 = 33 MHz .
+	 */
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>, <231000000>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_lvds>;
+};
+
+&panel_lvds {
+	remote-endpoint = <&ldb_lvds_ch1>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtso
new file mode 100644
index 00000000000000..4066bac28f3df4
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-mi1010z1t-1cp11.dtso
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024-2026 Marek Vasut
+ */
+
+#include "imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi"
+#include "imx8mm-data-modul-edm-sbc-overlay-lvds-mi1010z1t-1cp11.dtsi"
+
+&media_blk_ctrl {
+	/*
+	 * The MI1010Z1T-1CP11 panel requires 51.2 MHz LVDS clock.
+	 * Set IMX8MP_VIDEO_PLL1 to 358.4 MHz , since 358.4 MHz / 7 = 51.2 MHz .
+	 */
+	assigned-clock-rates = <500000000>, <200000000>,
+			       <0>, <0>, <500000000>, <358400000>;
+};
+
+&ldb_lvds_ch1 {
+	remote-endpoint = <&panel_lvds>;
+};
+
+&panel_lvds {
+	remote-endpoint = <&ldb_lvds_ch1>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtso
new file mode 100644
index 00000000000000..427585b78e45de
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev900.dtso
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	reg_panel_vcc_raw: regulator-panel-vcc-raw {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_panel_vcc_reg>;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-name = "PANEL_VCC";
+	};
+};
+
+&panel {
+	power-supply = <&reg_panel_vcc_raw>;
+};
+
+&reg_backlight_en_level {
+	status = "disabled";
+};
+
+&reg_backlight_pwm_level {
+	status = "disabled";
+};
+
+&reg_panel_bl_supply {
+	status = "disabled";
+};
+
+&reg_panel_bl {
+	gpio = <&gpio3 0 0>;
+};
+
+&reg_panel_vcc {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtso
new file mode 100644
index 00000000000000..a21fea27e0b41c
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds-rev902.dtso
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+&reg_panel_bl {
+	gpio = <&gpio3 0 0>;
+};
+
+&reg_panel_vcc {
+	enable-gpios = <&gpio3 6 0>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi
new file mode 100644
index 00000000000000..95f4761869d754
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-lvds.dtsi
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2023-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mm-data-modul-edm-sbc-overlay-edm-mod-imx8mm-common.dtsi"
+
+&{/} {
+	reg_panel_bl_supply: regulator-panel-bl-supply {
+		compatible = "regulator-gpio";
+		regulator-type = "voltage";
+		regulator-name = "PANEL_BL_SUPPLY";
+		enable-gpios = <&gpiolvds 0 0>;
+		enable-active-high;
+		status = "okay";
+
+		/*
+		 * MP2328 voltage divider settings:
+		 *   R1=51k1
+		 *   R2=5k62 with optional series Rs=2k21 (12V)
+		 *
+		 * 1 / Rx = (1 / R2) [ + (1 / Rs)][ + (1 / Rt)]
+		 * Vout = 0.5 + ((R1 / Rx) * 0.5)
+		 */
+		gpios = <&gpiolvds 1 GPIO_ACTIVE_HIGH>;	/* 12V */
+		states = <5000000 0x0>,
+			 <12000000 0x1>;
+
+		/* Default setting: lowest supported voltage. */
+		gpios-states = <1>;	/* Default GPIO state is HIGH, so 12V0 out */
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+};
+
+&lcdif2 {
+	status = "okay";
+};
+
+&lvds_bridge {
+	status = "okay";
+};
+
+&pwm1 {
+	status = "okay";
+};
+
+&reg_backlight_en_level {
+	gpios = <&gpiolvds 5 GPIO_ACTIVE_HIGH>; /* SEL_EN */
+};
+
+&reg_backlight_pwm_level {
+	gpios = <&gpiolvds 4 GPIO_ACTIVE_HIGH>; /* SEL_PWM */
+};
+
+&reg_panel_bl {
+	gpio = <&gpiowifi 0 0>;
+	vin-supply = <&reg_panel_bl_supply>;
+};
+
+&reg_panel_vcc {
+	enable-gpios = <&gpiowifi 4 0>;
+	/*
+	 * MP2328 voltage divider settings:
+	 *   R1=51k1
+	 *   R2=9k09 with optional series Rs=14k7 (5V) or Rt=2k94 (12V)
+	 *
+	 * 1 / Rx = (1 / R2) [ + (1 / Rs)][ + (1 / Rt)]
+	 * Vout = 0.5 + ((R1 / Rx) * 0.5)
+	 */
+	gpios = <&gpiolvds 2 GPIO_ACTIVE_HIGH>,	/* 5V */
+		<&gpiolvds 3 GPIO_ACTIVE_HIGH>;	/* 12V */
+	states = <3300000 0x0>,
+		 <5000000 0x1>,
+		 <12000000 0x2>,
+		 <14000000 0x3>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtso
new file mode 100644
index 00000000000000..ec861aa64541e1
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev900.dtso
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2023-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mp-pinfunc.h"
+
+&eeprom900 {
+	status = "okay";
+};
+
+&eeprom902 {
+	status = "disabled";
+};
+
+&eqos {	/* First ethernet */
+	phy-handle = <&phy_eqos_ath>;
+};
+
+&fec {	/* Second ethernet */
+	/* pinctrl_wifi is ENET2_INT# */
+	pinctrl-0 = <&pinctrl_fec &pinctrl_wifi>;
+	phy-handle = <&phy_fec_ath>;
+};
+
+&gpiolvds {
+	status = "disabled";
+};
+
+/*
+ * External pull ups on R242 and R243 on I2C2_SCL_3V3 and I2C2_SDA_3V3
+ * are not populated on this early board revision, activate in-SoC pull
+ * up resistors instead to work around the missing external pull ups.
+ */
+&pinctrl_i2c2 {
+	fsl,pins = <
+		MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL			0x400001c4
+		MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA			0x400001c4
+	>;
+};
+
+&pinctrl_i2c2_gpio {
+	fsl,pins = <
+		MX8MP_IOMUXC_I2C2_SCL__GPIO5_IO16		0x1c4
+		MX8MP_IOMUXC_I2C2_SDA__GPIO5_IO17		0x1c4
+	>;
+};
+
+&pcie_phy {
+	status = "disabled";
+};
+
+&pcie {
+	status = "disabled";
+};
+
+&phy_eqos_ath {
+	/*
+	 * The software support for combination of EEE capable PHY and EEE
+	 * capable MAC is so far missing from the Linux kernel. By default,
+	 * the AR8035 PHY does enable EEE functionality on the PHY side,
+	 * while the EQoS/DWMAC MAC expects to handle the EEE functionality
+	 * on the MAC side. Because the Linux kernel is currently unable to
+	 * align EEE configuration of the PHY and MAC, enabling EEE leads
+	 * to unreliable link. Disable EEE until the kernel support is in
+	 * place.
+	 */
+	eee-broken-100tx;
+	eee-broken-1000t;
+	status = "okay";
+};
+
+&phy_eqos_bcm {
+	status = "disabled";
+};
+
+&phy_fec_ath {
+	status = "okay";
+};
+
+&phy_fec_bcm {
+	status = "disabled";
+};
+
+&reg_pcie0 {
+	status = "disabled";
+};
+
+&tpm {
+	status = "disabled";
+};
+
+&uart4 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtso b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtso
new file mode 100644
index 00000000000000..0141b5d77c6bd5
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc-overlay-edm-sbc-imx8mp-rev902.dtso
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2024-2026 Marek Vasut
+ */
+/dts-v1/;
+/plugin/;
+
+#include "imx8mp-pinfunc.h"
+
+&pinctrl_hog_misc {
+	fsl,pins = <
+		/* ENET_WOL# -- shared by both PHYs */
+		MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10		0x40000090
+
+		/* PG_V_IN_VAR# */
+		MX8MP_IOMUXC_NAND_CE0_B__GPIO3_IO01		0x40000000
+		/* CSI2_PD_1V8 */
+		MX8MP_IOMUXC_NAND_DATA02__GPIO3_IO08		0x0
+		/* CSI2_RESET_1V8# */
+		MX8MP_IOMUXC_NAND_DATA03__GPIO3_IO09		0x0
+
+		/* DIS_USB_DN1 */
+		MX8MP_IOMUXC_SAI2_RXFS__GPIO4_IO21		0x0
+		/* DIS_USB_DN2 */
+		MX8MP_IOMUXC_SAI2_RXC__GPIO4_IO22		0x0
+
+		/* EEPROM_WP_1V8# */
+		MX8MP_IOMUXC_NAND_DQS__GPIO3_IO14		0x100
+		/* PCIE_CLK_GEN_CLKPWRGD_PD_1V8# */
+		MX8MP_IOMUXC_SAI5_RXD0__GPIO3_IO21		0x0
+		/* GRAPHICS_PRSNT_1V8# */
+		MX8MP_IOMUXC_SAI1_TXD6__GPIO4_IO18		0x40000000
+
+		/* CLK_CCM_CLKO1_3V3 */
+		MX8MP_IOMUXC_GPIO1_IO14__CCM_CLKO1		0x10
+	>;
+};
+
+&pinctrl_pcie0 {
+	fsl,pins = <
+		/* M2_PCIE_RST# */
+		MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05		0x2
+		/* M2_W_DISABLE1_1V8# */
+		MX8MP_IOMUXC_SAI5_RXD2__GPIO3_IO23		0x2
+		/* M2_W_DISABLE2_1V8# */
+		MX8MP_IOMUXC_SAI5_RXD3__GPIO3_IO24		0x2
+		/* CLK_M2_32K768 */
+		MX8MP_IOMUXC_GPIO1_IO00__CCM_EXT_CLK1		0x14
+		/* M2_PCIE_WAKE# */
+		MX8MP_IOMUXC_GPIO1_IO06__GPIO1_IO06		0x40000140
+		/* M2_PCIE_CLKREQ# */
+		MX8MP_IOMUXC_I2C4_SCL__PCIE_CLKREQ_B		0x61
+	>;
+};
+
+&pinctrl_uart4 {
+	fsl,pins = <
+		MX8MP_IOMUXC_UART4_RXD__UART4_DCE_RX		0x49
+		MX8MP_IOMUXC_UART4_TXD__UART4_DCE_TX		0x49
+	>;
+};
+
+&gpiowifi {
+	status = "disabled";
+};
+
+&uart4 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts
index cb28cf1cdd23fa..4614a0de079064 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts
@@ -30,11 +30,8 @@
 
 	backlight: backlight {
 		compatible = "pwm-backlight";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_panel_backlight>;
 		brightness-levels = <0 1 10 20 30 40 50 60 70 75 80 90 100>;
 		default-brightness-level = <7>;
-		enable-gpios = <&gpio3 0 GPIO_ACTIVE_HIGH>;
 		pwms = <&pwm1 0 5000000 0>;
 		/* Disabled by default, unless display board plugged in. */
 		status = "disabled";
@@ -86,9 +83,6 @@
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
 		regulator-name = "PANEL_VCC";
-		/* GPIO flags are ignored, enable-active-high applies. */
-		gpio = <&gpio3 6 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
 		/* Disabled by default, unless display board plugged in. */
 		status = "disabled";
 	};
@@ -116,7 +110,7 @@
 		enable-active-high;
 		off-on-delay-us = <12000>;
 		startup-delay-us = <100>;
-		vin-supply = <&buck4>;
+		vin-supply = <&buck4_reg>;
 	};
 
 	sound {
@@ -152,19 +146,19 @@
 };
 
 &A53_0 {
-	cpu-supply = <&buck2>;
+	cpu-supply = <&buck2_reg>;
 };
 
 &A53_1 {
-	cpu-supply = <&buck2>;
+	cpu-supply = <&buck2_reg>;
 };
 
 &A53_2 {
-	cpu-supply = <&buck2>;
+	cpu-supply = <&buck2_reg>;
 };
 
 &A53_3 {
-	cpu-supply = <&buck2>;
+	cpu-supply = <&buck2_reg>;
 };
 
 &ecspi1 {
@@ -405,8 +399,8 @@
 		reg = <0x0a>;
 		#sound-dai-cells = <0>;
 		clocks = <&clk_pwm4>;
-		VDDA-supply = <&buck4>;
-		VDDIO-supply = <&buck4>;
+		VDDA-supply = <&buck4_reg>;
+		VDDIO-supply = <&buck4_reg>;
 	};
 
 	usb-hub@2c {
@@ -427,14 +421,142 @@
 	eeprom900: eeprom@50 {	/* board rev.900 */
 		compatible = "atmel,24c32";
 		reg = <0x50>;
+		label = "dmo-board-info";
 		pagesize = <32>;
+		wp-gpios = <&gpio3 14 GPIO_ACTIVE_LOW>;
 		status = "disabled";
+
+		nvmem-layout {
+			compatible = "fixed-layout";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			company-name@24 {
+				reg = <0x24 0xd>;
+			};
+
+			product-name@32 {
+				reg = <0x32 0x18>;
+			};
+
+			board-revision@4b {
+				reg = <0x4b 0x3>;
+			};
+
+			serial-number@4f {
+				reg = <0x4f 0x8>;
+			};
+
+			manufacturing-date@70 {
+				reg = <0x70 0xa>;
+			};
+
+			test-date@7b {
+				reg = <0x7b 0xa>;
+			};
+
+			product-number@86 {
+				reg = <0x86 0x8>;
+			};
+
+			board-variant@9a {
+				reg = <0x9a 0x9>;
+			};
+
+			company-acronym@a4 {
+				reg = <0xa4 0x3>;
+			};
+
+			eeprom900mac: mac-address@b0 {
+				reg = <0xb0 0x6>;
+			};
+
+			board-id@100 {
+				reg = <0x100 0x4>;
+			};
+
+			wifi-id@120 {
+				reg = <0x120 0x4>;
+			};
+
+			board-lvds-id@140 {
+				reg = <0x140 0x4>;
+			};
+
+			camera-id@160 {
+				reg = <0x160 0x4>;
+			};
+		};
 	};
 
 	eeprom902: eeprom@51 {	/* board rev.902 */
 		compatible = "atmel,24c32";
 		reg = <0x51>;
+		label = "dmo-board-info";
 		pagesize = <32>;
+		wp-gpios = <&gpio3 14 GPIO_ACTIVE_LOW>;
+
+		nvmem-layout {
+			compatible = "fixed-layout";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			company-name@24 {
+				reg = <0x24 0xd>;
+			};
+
+			product-name@32 {
+				reg = <0x32 0x18>;
+			};
+
+			board-revision@4b {
+				reg = <0x4b 0x3>;
+			};
+
+			serial-number@4f {
+				reg = <0x4f 0x8>;
+			};
+
+			manufacturing-date@70 {
+				reg = <0x70 0xa>;
+			};
+
+			test-date@7b {
+				reg = <0x7b 0xa>;
+			};
+
+			product-number@86 {
+				reg = <0x86 0x8>;
+			};
+
+			board-variant@9a {
+				reg = <0x9a 0x9>;
+			};
+
+			company-acronym@a4 {
+				reg = <0xa4 0x3>;
+			};
+
+			eeprom902mac: mac-address@b0 {
+				reg = <0xb0 0x6>;
+			};
+
+			board-id@100 {
+				reg = <0x100 0x4>;
+			};
+
+			wifi-id@120 {
+				reg = <0x120 0x4>;
+			};
+
+			board-lvds-id@140 {
+				reg = <0x140 0x4>;
+			};
+
+			camera-id@160 {
+				reg = <0x160 0x4>;
+			};
+		};
 	};
 
 	rtc: rtc@68 {
@@ -454,7 +576,9 @@
 	};
 };
 
-&i2c2 {
+i2c_display: &i2c2 { };
+
+i2c_feature: &i2c2 {
 	clock-frequency = <100000>;
 	pinctrl-names = "default", "gpio";
 	pinctrl-0 = <&pinctrl_i2c2>;
@@ -527,7 +651,7 @@
 		 * MIMX8ML8CVNKZAB
 		 */
 		regulators {
-			buck1: BUCK1 {	/* VDD_SOC (dual-phase with BUCK3) */
+			buck1_reg: BUCK1 {	/* VDD_SOC (dual-phase with BUCK3) */
 				regulator-min-microvolt = <850000>;
 				regulator-max-microvolt = <1000000>;
 				regulator-ramp-delay = <3125>;
@@ -535,7 +659,7 @@
 				regulator-boot-on;
 			};
 
-			buck2: BUCK2 {	/* VDD_ARM */
+			buck2_reg: BUCK2 {	/* VDD_ARM */
 				nxp,dvs-run-voltage = <950000>;
 				nxp,dvs-standby-voltage = <850000>;
 				regulator-min-microvolt = <850000>;
@@ -545,47 +669,47 @@
 				regulator-boot-on;
 			};
 
-			buck4: BUCK4 {	/* VDD_3V3 */
+			buck4_reg: BUCK4 {	/* VDD_3V3 */
 				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-always-on;
 				regulator-boot-on;
 			};
 
-			buck5: BUCK5 {	/* VDD_1V8 */
+			buck5_reg: BUCK5 {	/* VDD_1V8 */
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <1800000>;
 				regulator-always-on;
 				regulator-boot-on;
 			};
 
-			buck6: BUCK6 {	/* NVCC_DRAM_1V1 */
+			buck6_reg: BUCK6 {	/* NVCC_DRAM_1V1 */
 				regulator-min-microvolt = <1100000>;
 				regulator-max-microvolt = <1100000>;
 				regulator-always-on;
 				regulator-boot-on;
 			};
 
-			ldo1: LDO1 {	/* NVCC_SNVS_1V8 */
+			ldo1_reg: LDO1 {	/* NVCC_SNVS_1V8 */
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <1800000>;
 				regulator-always-on;
 				regulator-boot-on;
 			};
 
-			ldo3: LDO3 {	/* VDDA_1V8 */
+			ldo3_reg: LDO3 {	/* VDDA_1V8 */
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <1800000>;
 				regulator-always-on;
 				regulator-boot-on;
 			};
 
-			ldo4: LDO4 {	/* PMIC_LDO4 */
+			ldo4_reg: LDO4 {	/* PMIC_LDO4 */
 				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
 			};
 
-			ldo5: LDO5 {	/* NVCC_SD2 */
+			ldo5_reg: LDO5 {	/* NVCC_SD2 */
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <3300000>;
 			};
@@ -650,7 +774,7 @@
 	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
 	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
 	vmmc-supply = <&reg_usdhc2_vmmc>;
-	vqmmc-supply = <&ldo5>;
+	vqmmc-supply = <&ldo5_reg>;
 	bus-width = <4>;
 	status = "okay";
 };
@@ -661,8 +785,8 @@
 	pinctrl-0 = <&pinctrl_usdhc3>;
 	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
 	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
-	vmmc-supply = <&buck4>;
-	vqmmc-supply = <&buck5>;
+	vmmc-supply = <&buck4_reg>;
+	vqmmc-supply = <&buck5_reg>;
 	bus-width = <8>;
 	no-sd;
 	no-sdio;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts
index a02b31c42db487..299d81a18c06ec 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts
@@ -101,7 +101,7 @@
 &usdhc2 {
 	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
 	bus-width = <4>;
-	pinctrl-names = "default" ;
+	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
 	vmmc-supply = <&reg_usdhc2_vmmc>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi
index 69558ffefa9a67..affcc20b20c7bd 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi
@@ -49,7 +49,7 @@
 				regulator-ramp-delay = <3125>;
 			};
 
-			buck2: BUCK2  {
+			buck2: BUCK2 {
 				nxp,dvs-run-voltage = <950000>;
 				nxp,dvs-standby-voltage = <850000>;
 				regulator-always-on;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso b/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso
index 41a2bb74f15652..88b30ca66adf58 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso
@@ -6,6 +6,7 @@
 /dts-v1/;
 /plugin/;
 
+#include <dt-bindings/clock/imx8mp-clock.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include "imx8mp-pinfunc.h"
@@ -99,6 +100,16 @@
 
 &lvds_bridge {
 	status = "okay";
+	/*
+	 * The video PLL clock frequency needs to be configured manually.
+	 * The panel needs a LVDS clock of 50 MHz * 7 = 350 MHz.
+	 * IMX8MP_CLK_MEDIA_LDB = IMX8MP_CLK_MEDIA_DISP2_PIX * 7
+	 * IMX8MP_VIDEO_PLL1 = IMX8MP_CLK_MEDIA_LDB * 2
+	 */
+	assigned-clocks = <&clk IMX8MP_CLK_MEDIA_LDB>,
+			  <&clk IMX8MP_VIDEO_PLL1>;
+	assigned-clock-parents = <&clk IMX8MP_VIDEO_PLL1_OUT>;
+	assigned-clock-rates = <0>, <700000000>;
 };
 
 /* redefine to remove touch controller GPIOs */
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-av-10.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-av-10.dtsi
index bb740f845855ac..8957b24a7ad2e7 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-av-10.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-av-10.dtsi
@@ -77,13 +77,12 @@
 			"MIC3R", "Microphone Jack",
 			"Microphone Jack", "Mic Bias";
 
-		simple-audio-card,cpu {
+		dailink_master: simple-audio-card,cpu {
 			sound-dai = <&sai2>;
 		};
 
-		dailink_master: simple-audio-card,codec {
+		simple-audio-card,codec {
 			sound-dai = <&codec>;
-			clocks = <&clk IMX8MP_CLK_SAI2>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-wlbt-07.dtso b/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-wlbt-07.dtso
new file mode 100644
index 00000000000000..008b6f9f46e591
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-peb-wlbt-07.dtso
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+* Copyright (C) 2026 PHYTEC Messtechnik GmbH
+*/
+
+#include <dt-bindings/clock/imx8mp-clock.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "imx8mp-pinfunc.h"
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	usdhc1_pwrseq: pwr-seq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio2 7 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&iomuxc {
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_RESET_B__UART3_DCE_RTS	0x140 /* RTS */
+			MX8MP_IOMUXC_SD1_STROBE__UART3_DCE_CTS	0x140 /* CTS */
+			MX8MP_IOMUXC_UART3_RXD__UART3_DCE_RX	0x140 /* RX */
+			MX8MP_IOMUXC_UART3_TXD__UART3_DCE_TX	0x140 /* TX */
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x190 /* SDIO_CLK */
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d0 /* SDIO_CMD */
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d0 /* SDIO_D0 */
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d0 /* SDIO_D1 */
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d0 /* SDIO_D2 */
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d0 /* SDIO_D3 */
+		>;
+	};
+
+	pinctrl_wlbt: wlbtgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SPDIF_EXT_CLK__GPIO5_IO05	0x106 /* WAKE_DEV */
+			MX8MP_IOMUXC_SD1_DATA5__GPIO2_IO07	0x106 /* PDn */
+			MX8MP_IOMUXC_SD1_DATA7__GPIO2_IO09	0x106 /* WAKE_HOST */
+		>;
+	};
+};
+
+&uart3 {
+	assigned-clocks = <&clk IMX8MP_CLK_UART3>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL1_80M>;
+	pinctrl-0 = <&pinctrl_uart3>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+		vcc-supply = <&reg_vcc_3v3_sw>;
+	};
+};
+
+&usdhc1 {
+	bus-width = <4>;
+	keep-power-in-suspend;
+	max-frequency = <50000000>;
+	mmc-pwrseq = <&usdhc1_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc1>, <&pinctrl_wlbt>;
+	pinctrl-names = "default";
+	vmmc-supply = <&reg_vcc_3v3_sw>;
+	wakeup-source;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-toradex-smarc.dtsi
index 0348da385f2399..eb8b9db7e3cd7b 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-toradex-smarc.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-toradex-smarc.dtsi
@@ -323,7 +323,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_lvds_dsi_sel>;
 	gpio-line-names = "ETH_0_INT#", /* 0 */
-			  "SLEEP#",
+			  "SMARC_SLEEP#",
 			  "",
 			  "",
 			  "",
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314-imx219.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314-imx219.dtso
index e5a2b3780215df..7b44ae0f19b23a 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314-imx219.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314-imx219.dtso
@@ -9,6 +9,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 
 #include "imx8mp-pinfunc.h"
 
@@ -47,7 +48,7 @@
 		VANA-supply = <&reg_cam>;
 		VDIG-supply = <&reg_cam>;
 		VDDL-supply = <&reg_cam>;
-		orientation = <2>;
+		orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 		rotation = <0>;
 
 		port {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso
index 2cf1de8c05b494..9ce0744c88a84e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds0-tm070jvhg33.dtso
@@ -12,7 +12,7 @@
 	status = "okay";
 };
 
-&panel_lvds0{
+&panel_lvds0 {
 	compatible = "tianma,tm070jvhg33";
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso
index cbad0cfab29257..24f5f932231f71 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpqs-mb-smarc-2-lvds1-tm070jvhg33.dtso
@@ -12,7 +12,7 @@
 	status = "okay";
 };
 
-&panel_lvds1{
+&panel_lvds1 {
 	compatible = "tianma,tm070jvhg33";
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts
index 983d67a0c1de7a..7f78026e70ffe9 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-var-dart-sonata.dts
@@ -511,6 +511,7 @@
 	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
 	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
 	vmmc-supply = <&reg_usdhc2_vmmc>;
+	vqmmc-supply = <&ldo5>;
 	bus-width = <4>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin-ivy.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-verdin-ivy.dtsi
index db1b4ee7728c07..bf021e85cfc942 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-verdin-ivy.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin-ivy.dtsi
@@ -333,8 +333,8 @@
 		"",
 		"",
 		"",
-		"DIG_1", /* SODIMM 56 */
-		"DIG_2", /* SODIMM 58 */
+		"DIGI_1", /* SODIMM 56 */
+		"DIGI_2", /* SODIMM 58 */
 		"REL1",  /* SODIMM 60 */
 		"REL2",  /* SODIMM 62 */
 		"", /* 10 */
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin-yavia.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-verdin-yavia.dtsi
index f3d28e23ba6c67..2a90f312fb2751 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-verdin-yavia.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin-yavia.dtsi
@@ -117,6 +117,11 @@
 	status = "okay";
 };
 
+&gpio3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpios_ext_yavia>;
+};
+
 &gpio4 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_ctrl_sleep_moci>;
@@ -266,7 +271,7 @@
 
 	pinctrl_gpios_ext_yavia: gpiosextyaviagrp {
 		fsl,pins = <
-			MX8MP_IOMUXC_NAND_CE1_B__GPIO3_IO02	0x106	/* SODIMM 64 */
+			MX8MP_IOMUXC_NAND_READY_B__GPIO3_IO16	0x106	/* SODIMM 64 */
 			MX8MP_IOMUXC_NAND_DQS__GPIO3_IO14	0x106	/* SODIMM 66 */
 		>;
 	};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp.dtsi
index 9cf9f6f08654d1..bec64cbb363473 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp.dtsi
@@ -1437,7 +1437,7 @@
 			#access-controller-cells = <3>;
 			ranges = <0x30c00000 0x30c00000 0x400000>;
 
-			spba-bus@30c00000 {
+			spba5: spba-bus@30c00000 {
 				compatible = "fsl,spba-bus", "simple-bus";
 				reg = <0x30c00000 0x100000>;
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-evk.dts b/arch/arm64/boot/dts/freescale/imx8mq-evk.dts
index 5ea04703ddd7ed..fc0303e4443388 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mq-evk.dts
@@ -299,6 +299,9 @@
 			vgen1_reg: vgen1 {
 				regulator-min-microvolt = <800000>;
 				regulator-max-microvolt = <1550000>;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
 			};
 
 			vgen2_reg: vgen2 {
@@ -328,6 +331,9 @@
 			vgen6_reg: vgen6 {
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <3300000>;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts b/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts
index 0c3f1b20b3d743..a82af92d608594 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5-devkit.dts
@@ -119,7 +119,7 @@
 		};
 	};
 
-	reg_22v4_p: regulator-22v4-p  {
+	reg_22v4_p: regulator-22v4-p {
 		compatible = "regulator-fixed";
 		regulator-name = "22v4_P";
 		regulator-min-microvolt = <22400000>;
@@ -507,7 +507,7 @@
 	pinctrl-0 = <&pinctrl_i2c3>;
 	status = "okay";
 
-	magnetometer@1e	{
+	magnetometer@1e {
 		compatible = "st,lsm9ds1-magn";
 		reg = <0x1e>;
 		pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
index f5d529c5baf3e6..a212c031d4c6b1 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
@@ -8,6 +8,7 @@
 #include "dt-bindings/input/input.h"
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include "dt-bindings/pwm/pwm.h"
 #include "dt-bindings/usb/pd.h"
 #include "imx8mq.dtsi"
@@ -1025,7 +1026,7 @@
 	sda-gpios = <&gpio5 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 	status = "okay";
 
-	magnetometer: magnetometer@1e	{
+	magnetometer: magnetometer@1e {
 		compatible = "st,lsm9ds1-magn";
 		reg = <0x1e>;
 		pinctrl-names = "default";
@@ -1060,7 +1061,7 @@
 		interrupts = <7 IRQ_TYPE_LEVEL_LOW>;
 	};
 
-	accel_gyro: accel-gyro@6a	{
+	accel_gyro: accel-gyro@6a {
 		compatible = "st,lsm9ds1-imu";
 		reg = <0x6a>;
 		vdd-supply = <&reg_vdd_sen>;
@@ -1116,7 +1117,7 @@
 		vddd-supply = <&reg_vcam_1v2>;
 		vddio-supply = <&reg_csi_1v8>;
 		rotation = <90>;
-		orientation = <0>;
+		orientation = <MEDIA_ORIENTATION_FRONT>;
 
 		port {
 			camera1_ep: endpoint {
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-nitrogen.dts b/arch/arm64/boot/dts/freescale/imx8mq-nitrogen.dts
index 767819cce886ce..b91509bfc9c007 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-nitrogen.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mq-nitrogen.dts
@@ -230,7 +230,7 @@
 
 			hdmi-bridge@48 {
 				compatible = "lontium,lt8912b";
-				reg = <0x48> ;
+				reg = <0x48>;
 				reset-gpios = <&max7323 0 GPIO_ACTIVE_LOW>;
 
 				ports {
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi
index c7bbba45f36852..a272c1bd75483e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq-zii-ultra.dtsi
@@ -99,7 +99,7 @@
 		regulator-max-microvolt = <1000000>;
 		gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>;
 		states = <1000000 0x1
-		           900000 0x0>;
+			   900000 0x0>;
 		regulator-always-on;
 	};
 
@@ -295,7 +295,7 @@
 		reg = <0x32>;
 		interrupt-parent = <&gpio3>;
 		interrupts = <17 IRQ_TYPE_EDGE_BOTH>,
-		             <18 IRQ_TYPE_EDGE_FALLING>;
+			     <18 IRQ_TYPE_EDGE_FALLING>;
 		interrupt-names = "a_det", "alert";
 	};
 
@@ -549,9 +549,9 @@
 	pinctrl-0 = <&pinctrl_pcie0>;
 	reset-gpio = <&gpio1 3 GPIO_ACTIVE_LOW>;
 	clocks = <&clk IMX8MQ_CLK_PCIE1_ROOT>,
-	         <&pcie0_refclk>,
-	         <&clk IMX8MQ_CLK_PCIE1_PHY>,
-	         <&clk IMX8MQ_CLK_PCIE1_AUX>;
+		 <&pcie0_refclk>,
+		 <&clk IMX8MQ_CLK_PCIE1_PHY>,
+		 <&clk IMX8MQ_CLK_PCIE1_AUX>;
 	vph-supply = <&vgen5_reg>;
 	status = "okay";
 };
@@ -561,9 +561,9 @@
 	pinctrl-0 = <&pinctrl_pcie1>;
 	reset-gpio = <&gpio1 6 GPIO_ACTIVE_LOW>;
 	clocks = <&clk IMX8MQ_CLK_PCIE2_ROOT>,
-	         <&pcie1_refclk>,
-	         <&clk IMX8MQ_CLK_PCIE2_PHY>,
-	         <&clk IMX8MQ_CLK_PCIE2_AUX>;
+		 <&pcie1_refclk>,
+		 <&clk IMX8MQ_CLK_PCIE2_PHY>,
+		 <&clk IMX8MQ_CLK_PCIE2_AUX>;
 	vph-supply = <&vgen5_reg>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
index d75710cdb5b414..5545b5b20ab2e9 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
@@ -363,9 +363,9 @@
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>, /* Physical Secure */
-		             <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>, /* Physical Non-Secure */
-		             <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>, /* Virtual */
-		             <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>; /* Hypervisor */
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>, /* Physical Non-Secure */
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>, /* Virtual */
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>; /* Hypervisor */
 		interrupt-parent = <&gic>;
 		arm,no-tick-in-suspend;
 	};
@@ -531,8 +531,8 @@
 				reg = <0x30010000 0x10000>;
 				interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_SAI1_IPG>,
-				         <&clk IMX8MQ_CLK_SAI1_ROOT>,
-				         <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
+					 <&clk IMX8MQ_CLK_SAI1_ROOT>,
+					 <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
 				clock-names = "bus", "mclk1", "mclk2", "mclk3";
 				dmas = <&sdma2 8 24 0>, <&sdma1 9 24 0>;
 				dma-names = "rx", "tx";
@@ -545,8 +545,8 @@
 				reg = <0x30030000 0x10000>;
 				interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_SAI6_IPG>,
-				         <&clk IMX8MQ_CLK_SAI6_ROOT>,
-				         <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
+					 <&clk IMX8MQ_CLK_SAI6_ROOT>,
+					 <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
 				clock-names = "bus", "mclk1", "mclk2", "mclk3";
 				dmas = <&sdma2 4 24 0>, <&sdma2 5 24 0>;
 				dma-names = "rx", "tx";
@@ -559,8 +559,8 @@
 				reg = <0x30040000 0x10000>;
 				interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_SAI5_IPG>,
-				         <&clk IMX8MQ_CLK_SAI5_ROOT>,
-				         <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
+					 <&clk IMX8MQ_CLK_SAI5_ROOT>,
+					 <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
 				clock-names = "bus", "mclk1", "mclk2", "mclk3";
 				dmas = <&sdma2 2 24 0>, <&sdma2 3 24 0>;
 				dma-names = "rx", "tx";
@@ -573,8 +573,8 @@
 				reg = <0x30050000 0x10000>;
 				interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_SAI4_IPG>,
-				         <&clk IMX8MQ_CLK_SAI4_ROOT>,
-				         <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
+					 <&clk IMX8MQ_CLK_SAI4_ROOT>,
+					 <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
 				clock-names = "bus", "mclk1", "mclk2", "mclk3";
 				dmas = <&sdma2 0 24 0>, <&sdma2 1 24 0>;
 				dma-names = "rx", "tx";
@@ -585,7 +585,7 @@
 				compatible = "fsl,imx8mq-gpio", "fsl,imx35-gpio";
 				reg = <0x30200000 0x10000>;
 				interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>,
-				             <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
+					     <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_GPIO1_ROOT>;
 				gpio-controller;
 				#gpio-cells = <2>;
@@ -598,7 +598,7 @@
 				compatible = "fsl,imx8mq-gpio", "fsl,imx35-gpio";
 				reg = <0x30210000 0x10000>;
 				interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>,
-				             <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
+					     <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_GPIO2_ROOT>;
 				gpio-controller;
 				#gpio-cells = <2>;
@@ -611,7 +611,7 @@
 				compatible = "fsl,imx8mq-gpio", "fsl,imx35-gpio";
 				reg = <0x30220000 0x10000>;
 				interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
-				             <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+					     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_GPIO3_ROOT>;
 				gpio-controller;
 				#gpio-cells = <2>;
@@ -624,7 +624,7 @@
 				compatible = "fsl,imx8mq-gpio", "fsl,imx35-gpio";
 				reg = <0x30230000 0x10000>;
 				interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>,
-				             <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
+					     <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_GPIO4_ROOT>;
 				gpio-controller;
 				#gpio-cells = <2>;
@@ -637,7 +637,7 @@
 				compatible = "fsl,imx8mq-gpio", "fsl,imx35-gpio";
 				reg = <0x30240000 0x10000>;
 				interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
-				             <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+					     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_GPIO5_ROOT>;
 				gpio-controller;
 				#gpio-cells = <2>;
@@ -845,14 +845,14 @@
 				compatible = "fsl,imx8mq-ccm";
 				reg = <0x30380000 0x10000>;
 				interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
-				             <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
+					     <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
 				#clock-cells = <1>;
 				clocks = <&ckil>, <&osc_25m>, <&osc_27m>,
-				         <&clk_ext1>, <&clk_ext2>,
-				         <&clk_ext3>, <&clk_ext4>;
+					 <&clk_ext1>, <&clk_ext2>,
+					 <&clk_ext3>, <&clk_ext4>;
 				clock-names = "ckil", "osc_25m", "osc_27m",
-				              "clk_ext1", "clk_ext2",
-				              "clk_ext3", "clk_ext4";
+					      "clk_ext1", "clk_ext2",
+					      "clk_ext3", "clk_ext4";
 				assigned-clocks = <&clk IMX8MQ_CLK_A53_SRC>,
 						  <&clk IMX8MQ_CLK_A53_CORE>,
 						  <&clk IMX8MQ_CLK_NOC>,
@@ -940,9 +940,9 @@
 						#power-domain-cells = <0>;
 						reg = <IMX8M_POWER_DOMAIN_GPU>;
 						clocks = <&clk IMX8MQ_CLK_GPU_ROOT>,
-						         <&clk IMX8MQ_CLK_GPU_SHADER_DIV>,
+							 <&clk IMX8MQ_CLK_GPU_SHADER_DIV>,
 							 <&clk IMX8MQ_CLK_GPU_AXI>,
-						         <&clk IMX8MQ_CLK_GPU_AHB>;
+							 <&clk IMX8MQ_CLK_GPU_AHB>;
 					};
 
 					pgc_vpu: power-domain@6 {
@@ -1000,7 +1000,7 @@
 				reg = <0x30660000 0x10000>;
 				interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_PWM1_ROOT>,
-				         <&clk IMX8MQ_CLK_PWM1_ROOT>;
+					 <&clk IMX8MQ_CLK_PWM1_ROOT>;
 				clock-names = "ipg", "per";
 				#pwm-cells = <3>;
 				status = "disabled";
@@ -1011,7 +1011,7 @@
 				reg = <0x30670000 0x10000>;
 				interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_PWM2_ROOT>,
-				         <&clk IMX8MQ_CLK_PWM2_ROOT>;
+					 <&clk IMX8MQ_CLK_PWM2_ROOT>;
 				clock-names = "ipg", "per";
 				#pwm-cells = <3>;
 				status = "disabled";
@@ -1022,7 +1022,7 @@
 				reg = <0x30680000 0x10000>;
 				interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_PWM3_ROOT>,
-				         <&clk IMX8MQ_CLK_PWM3_ROOT>;
+					 <&clk IMX8MQ_CLK_PWM3_ROOT>;
 				clock-names = "ipg", "per";
 				#pwm-cells = <3>;
 				status = "disabled";
@@ -1033,7 +1033,7 @@
 				reg = <0x30690000 0x10000>;
 				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_PWM4_ROOT>,
-				         <&clk IMX8MQ_CLK_PWM4_ROOT>;
+					 <&clk IMX8MQ_CLK_PWM4_ROOT>;
 				clock-names = "ipg", "per";
 				#pwm-cells = <3>;
 				status = "disabled";
@@ -1124,11 +1124,11 @@
 
 			uart1: serial@30860000 {
 				compatible = "fsl,imx8mq-uart",
-				             "fsl,imx6q-uart";
+					     "fsl,imx6q-uart";
 				reg = <0x30860000 0x10000>;
 				interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_UART1_ROOT>,
-				         <&clk IMX8MQ_CLK_UART1_ROOT>;
+					 <&clk IMX8MQ_CLK_UART1_ROOT>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma1 22 4 0>, <&sdma1 23 4 0>;
 				dma-names = "rx", "tx";
@@ -1137,11 +1137,11 @@
 
 			uart3: serial@30880000 {
 				compatible = "fsl,imx8mq-uart",
-				             "fsl,imx6q-uart";
+					     "fsl,imx6q-uart";
 				reg = <0x30880000 0x10000>;
 				interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_UART3_ROOT>,
-				         <&clk IMX8MQ_CLK_UART3_ROOT>;
+					 <&clk IMX8MQ_CLK_UART3_ROOT>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma1 26 4 0>, <&sdma1 27 4 0>;
 				dma-names = "rx", "tx";
@@ -1150,11 +1150,11 @@
 
 			uart2: serial@30890000 {
 				compatible = "fsl,imx8mq-uart",
-				             "fsl,imx6q-uart";
+					     "fsl,imx6q-uart";
 				reg = <0x30890000 0x10000>;
 				interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_UART2_ROOT>,
-				         <&clk IMX8MQ_CLK_UART2_ROOT>;
+					 <&clk IMX8MQ_CLK_UART2_ROOT>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma1 24 4 0>, <&sdma1 25 4 0>;
 				dma-names = "rx", "tx";
@@ -1205,8 +1205,8 @@
 				reg = <0x308c0000 0x10000>;
 				interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_SAI3_IPG>,
-				         <&clk IMX8MQ_CLK_SAI3_ROOT>,
-				         <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
+					 <&clk IMX8MQ_CLK_SAI3_ROOT>,
+					 <&clk IMX8MQ_CLK_DUMMY>, <&clk IMX8MQ_CLK_DUMMY>;
 				clock-names = "bus", "mclk1", "mclk2", "mclk3";
 				dmas = <&sdma1 12 24 0>, <&sdma1 13 24 0>;
 				dma-names = "rx", "tx";
@@ -1356,11 +1356,11 @@
 
 			uart4: serial@30a60000 {
 				compatible = "fsl,imx8mq-uart",
-				             "fsl,imx6q-uart";
+					     "fsl,imx6q-uart";
 				reg = <0x30a60000 0x10000>;
 				interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_UART4_ROOT>,
-				         <&clk IMX8MQ_CLK_UART4_ROOT>;
+					 <&clk IMX8MQ_CLK_UART4_ROOT>;
 				clock-names = "ipg", "per";
 				dmas = <&sdma1 28 4 0>, <&sdma1 29 4 0>;
 				dma-names = "rx", "tx";
@@ -1481,12 +1481,12 @@
 
 			usdhc1: mmc@30b40000 {
 				compatible = "fsl,imx8mq-usdhc",
-				             "fsl,imx7d-usdhc";
+					     "fsl,imx7d-usdhc";
 				reg = <0x30b40000 0x10000>;
 				interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_IPG_ROOT>,
-				         <&clk IMX8MQ_CLK_NAND_USDHC_BUS>,
-				         <&clk IMX8MQ_CLK_USDHC1_ROOT>;
+					 <&clk IMX8MQ_CLK_NAND_USDHC_BUS>,
+					 <&clk IMX8MQ_CLK_USDHC1_ROOT>;
 				clock-names = "ipg", "ahb", "per";
 				fsl,tuning-start-tap = <20>;
 				fsl,tuning-step = <2>;
@@ -1496,12 +1496,12 @@
 
 			usdhc2: mmc@30b50000 {
 				compatible = "fsl,imx8mq-usdhc",
-				             "fsl,imx7d-usdhc";
+					     "fsl,imx7d-usdhc";
 				reg = <0x30b50000 0x10000>;
 				interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_IPG_ROOT>,
-				         <&clk IMX8MQ_CLK_NAND_USDHC_BUS>,
-				         <&clk IMX8MQ_CLK_USDHC2_ROOT>;
+					 <&clk IMX8MQ_CLK_NAND_USDHC_BUS>,
+					 <&clk IMX8MQ_CLK_USDHC2_ROOT>;
 				clock-names = "ipg", "ahb", "per";
 				fsl,tuning-start-tap = <20>;
 				fsl,tuning-step = <2>;
@@ -1538,16 +1538,16 @@
 				compatible = "fsl,imx8mq-fec", "fsl,imx6sx-fec";
 				reg = <0x30be0000 0x10000>;
 				interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
-				             <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clk IMX8MQ_CLK_ENET1_ROOT>,
-				         <&clk IMX8MQ_CLK_ENET1_ROOT>,
-				         <&clk IMX8MQ_CLK_ENET_TIMER>,
-				         <&clk IMX8MQ_CLK_ENET_REF>,
-				         <&clk IMX8MQ_CLK_ENET_PHY_REF>;
+					 <&clk IMX8MQ_CLK_ENET1_ROOT>,
+					 <&clk IMX8MQ_CLK_ENET_TIMER>,
+					 <&clk IMX8MQ_CLK_ENET_REF>,
+					 <&clk IMX8MQ_CLK_ENET_PHY_REF>;
 				clock-names = "ipg", "ahb", "ptp",
-				              "enet_clk_ref", "enet_out";
+					      "enet_clk_ref", "enet_out";
 				assigned-clocks = <&clk IMX8MQ_CLK_ENET_AXI>,
 						  <&clk IMX8MQ_CLK_ENET_TIMER>,
 						  <&clk IMX8MQ_CLK_ENET_REF>,
@@ -1639,23 +1639,23 @@
 			reg = <0x38000000 0x40000>;
 			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clk IMX8MQ_CLK_GPU_ROOT>,
-			         <&clk IMX8MQ_CLK_GPU_SHADER_DIV>,
-			         <&clk IMX8MQ_CLK_GPU_AXI>,
-			         <&clk IMX8MQ_CLK_GPU_AHB>;
+				 <&clk IMX8MQ_CLK_GPU_SHADER_DIV>,
+				 <&clk IMX8MQ_CLK_GPU_AXI>,
+				 <&clk IMX8MQ_CLK_GPU_AHB>;
 			clock-names = "core", "shader", "bus", "reg";
 			#cooling-cells = <2>;
 			assigned-clocks = <&clk IMX8MQ_CLK_GPU_CORE_SRC>,
-			                  <&clk IMX8MQ_CLK_GPU_SHADER_SRC>,
-			                  <&clk IMX8MQ_CLK_GPU_AXI>,
-			                  <&clk IMX8MQ_CLK_GPU_AHB>,
-			                  <&clk IMX8MQ_GPU_PLL_BYPASS>;
+					  <&clk IMX8MQ_CLK_GPU_SHADER_SRC>,
+					  <&clk IMX8MQ_CLK_GPU_AXI>,
+					  <&clk IMX8MQ_CLK_GPU_AHB>,
+					  <&clk IMX8MQ_GPU_PLL_BYPASS>;
 			assigned-clock-parents = <&clk IMX8MQ_GPU_PLL_OUT>,
-			                         <&clk IMX8MQ_GPU_PLL_OUT>,
-			                         <&clk IMX8MQ_GPU_PLL_OUT>,
-			                         <&clk IMX8MQ_GPU_PLL_OUT>,
-			                         <&clk IMX8MQ_GPU_PLL>;
+						 <&clk IMX8MQ_GPU_PLL_OUT>,
+						 <&clk IMX8MQ_GPU_PLL_OUT>,
+						 <&clk IMX8MQ_GPU_PLL_OUT>,
+						 <&clk IMX8MQ_GPU_PLL>;
 			assigned-clock-rates = <800000000>, <800000000>,
-			                       <800000000>, <400000000>, <0>;
+					       <800000000>, <400000000>, <0>;
 			power-domains = <&pgc_gpu>;
 		};
 
@@ -1663,13 +1663,13 @@
 			compatible = "fsl,imx8mq-dwc3", "snps,dwc3";
 			reg = <0x38100000 0x10000>;
 			clocks = <&clk IMX8MQ_CLK_USB1_CTRL_ROOT>,
-			         <&clk IMX8MQ_CLK_USB_CORE_REF>,
+				 <&clk IMX8MQ_CLK_USB_CORE_REF>,
 				 <&clk IMX8MQ_CLK_32K>;
 			clock-names = "bus_early", "ref", "suspend";
 			assigned-clocks = <&clk IMX8MQ_CLK_USB_BUS>,
-			                  <&clk IMX8MQ_CLK_USB_CORE_REF>;
+					  <&clk IMX8MQ_CLK_USB_CORE_REF>;
 			assigned-clock-parents = <&clk IMX8MQ_SYS2_PLL_500M>,
-			                         <&clk IMX8MQ_SYS1_PLL_100M>;
+						 <&clk IMX8MQ_SYS1_PLL_100M>;
 			assigned-clock-rates = <500000000>, <100000000>;
 			interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
 			phys = <&usb3_phy0>, <&usb3_phy0>;
@@ -1695,13 +1695,13 @@
 			compatible = "fsl,imx8mq-dwc3", "snps,dwc3";
 			reg = <0x38200000 0x10000>;
 			clocks = <&clk IMX8MQ_CLK_USB2_CTRL_ROOT>,
-			         <&clk IMX8MQ_CLK_USB_CORE_REF>,
+				 <&clk IMX8MQ_CLK_USB_CORE_REF>,
 				 <&clk IMX8MQ_CLK_32K>;
 			clock-names = "bus_early", "ref", "suspend";
 			assigned-clocks = <&clk IMX8MQ_CLK_USB_BUS>,
-			                  <&clk IMX8MQ_CLK_USB_CORE_REF>;
+					  <&clk IMX8MQ_CLK_USB_CORE_REF>;
 			assigned-clock-parents = <&clk IMX8MQ_SYS2_PLL_500M>,
-			                         <&clk IMX8MQ_SYS1_PLL_100M>;
+						 <&clk IMX8MQ_SYS1_PLL_100M>;
 			assigned-clock-rates = <500000000>, <100000000>;
 			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			phys = <&usb3_phy1>, <&usb3_phy1>;
@@ -1767,9 +1767,9 @@
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
 			interrupt-map = <0 0 0 1 &gic GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
-			                <0 0 0 2 &gic GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>,
-			                <0 0 0 3 &gic GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
-			                <0 0 0 4 &gic GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
+					<0 0 0 2 &gic GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
 			fsl,max-link-speed = <2>;
 			linux,pci-domain = <0>;
 			clocks = <&clk IMX8MQ_CLK_PCIE1_ROOT>,
@@ -1779,17 +1779,17 @@
 			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux";
 			power-domains = <&pgc_pcie>;
 			resets = <&src IMX8MQ_RESET_PCIEPHY>,
-			         <&src IMX8MQ_RESET_PCIE_CTRL_APPS_EN>,
-			         <&src IMX8MQ_RESET_PCIE_CTRL_APPS_TURNOFF>;
+				 <&src IMX8MQ_RESET_PCIE_CTRL_APPS_EN>,
+				 <&src IMX8MQ_RESET_PCIE_CTRL_APPS_TURNOFF>;
 			reset-names = "pciephy", "apps", "turnoff";
 			assigned-clocks = <&clk IMX8MQ_CLK_PCIE1_CTRL>,
-			                  <&clk IMX8MQ_CLK_PCIE1_PHY>,
-			                  <&clk IMX8MQ_CLK_PCIE1_AUX>;
+					  <&clk IMX8MQ_CLK_PCIE1_PHY>,
+					  <&clk IMX8MQ_CLK_PCIE1_AUX>;
 			assigned-clock-parents = <&clk IMX8MQ_SYS2_PLL_250M>,
-			                         <&clk IMX8MQ_SYS2_PLL_100M>,
-			                         <&clk IMX8MQ_SYS1_PLL_80M>;
+						 <&clk IMX8MQ_SYS2_PLL_100M>,
+						 <&clk IMX8MQ_SYS1_PLL_80M>;
 			assigned-clock-rates = <250000000>, <100000000>,
-			                       <10000000>;
+					       <10000000>;
 			status = "disabled";
 
 			pcie0_port0: pcie@0 {
@@ -1868,17 +1868,17 @@
 			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux";
 			power-domains = <&pgc_pcie>;
 			resets = <&src IMX8MQ_RESET_PCIEPHY2>,
-			         <&src IMX8MQ_RESET_PCIE2_CTRL_APPS_EN>,
-			         <&src IMX8MQ_RESET_PCIE2_CTRL_APPS_TURNOFF>;
+				 <&src IMX8MQ_RESET_PCIE2_CTRL_APPS_EN>,
+				 <&src IMX8MQ_RESET_PCIE2_CTRL_APPS_TURNOFF>;
 			reset-names = "pciephy", "apps", "turnoff";
 			assigned-clocks = <&clk IMX8MQ_CLK_PCIE2_CTRL>,
-			                  <&clk IMX8MQ_CLK_PCIE2_PHY>,
-			                  <&clk IMX8MQ_CLK_PCIE2_AUX>;
+					  <&clk IMX8MQ_CLK_PCIE2_PHY>,
+					  <&clk IMX8MQ_CLK_PCIE2_AUX>;
 			assigned-clock-parents = <&clk IMX8MQ_SYS2_PLL_250M>,
-			                         <&clk IMX8MQ_SYS2_PLL_100M>,
-			                         <&clk IMX8MQ_SYS1_PLL_80M>;
+						 <&clk IMX8MQ_SYS2_PLL_100M>,
+						 <&clk IMX8MQ_SYS1_PLL_80M>;
 			assigned-clock-rates = <250000000>, <100000000>,
-			                       <10000000>;
+					       <10000000>;
 			status = "disabled";
 
 			pcie1_port0: pcie@0 {
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts
index 9e49ac377162b9..1afa65bd955ecd 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts
+++ b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts
@@ -44,57 +44,57 @@
 		#size-cells = <2>;
 		ranges;
 
-		vdev0vring0: memory@90000000 {
+		vdev0vring0: vdev0vring0@90000000 {
 			reg = <0 0x90000000 0 0x8000>;
 			no-map;
 		};
 
-		vdev0vring1: memory@90008000 {
+		vdev0vring1: vdev0vring1@90008000 {
 			reg = <0 0x90008000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring0: memory@90010000 {
+		vdev1vring0: vdev1vring0@90010000 {
 			reg = <0 0x90010000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring1: memory@90018000 {
+		vdev1vring1: vdev1vring1@90018000 {
 			reg = <0 0x90018000 0 0x8000>;
 			no-map;
 		};
 
-		rsc_table0: memory@900ff000 {
+		rsc_table0: rsc-table@900ff000 {
 			reg = <0 0x900ff000 0 0x1000>;
 			no-map;
 		};
 
-		vdev2vring0: memory@90100000 {
+		vdev2vring0: vdev2vring0@90100000 {
 			reg = <0 0x90100000 0 0x8000>;
 			no-map;
 		};
 
-		vdev2vring1: memory@90108000 {
+		vdev2vring1: vdev2vring1@90108000 {
 			reg = <0 0x90108000 0 0x8000>;
 			no-map;
 		};
 
-		vdev3vring0: memory@90110000 {
+		vdev3vring0: vdev3vring0@90110000 {
 			reg = <0 0x90110000 0 0x8000>;
 			no-map;
 		};
 
-		vdev3vring1: memory@90118000 {
+		vdev3vring1: vdev3vring1@90118000 {
 			reg = <0 0x90118000 0 0x8000>;
 			no-map;
 		};
 
-		rsc_table1: memory@901ff000 {
+		rsc_table1: rsc-table@901ff000 {
 			reg = <0 0x901ff000 0 0x1000>;
 			no-map;
 		};
 
-		vdevbuffer: memory@90400000 {
+		vdev0buffer: vdev0buffer@90400000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0x90400000 0 0x100000>;
 			no-map;
@@ -105,17 +105,17 @@
 			no-map;
 		};
 
-		dsp_vdev0vring0: memory@942f0000 {
+		dsp_vdev0vring0: vdev0vring0@942f0000 {
 			reg = <0 0x942f0000 0 0x8000>;
 			no-map;
 		};
 
-		dsp_vdev0vring1: memory@942f8000 {
+		dsp_vdev0vring1: vdev0vring1@942f8000 {
 			reg = <0 0x942f8000 0 0x8000>;
 			no-map;
 		};
 
-		dsp_vdev0buffer: memory@94300000 {
+		dsp_vdev0buffer: vdev0buffer@94300000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0x94300000 0 0x100000>;
 			no-map;
@@ -456,7 +456,7 @@
 		mboxes = <&lsio_mu5 0 1
 			  &lsio_mu5 1 1
 			  &lsio_mu5 3 1>;
-		memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+		memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 				<&vdev1vring0>, <&vdev1vring1>, <&rsc_table0>;
 		power-domains = <&pd IMX_SC_R_M4_0_PID0>, <&pd IMX_SC_R_M4_0_MU_1A>;
 
@@ -471,7 +471,7 @@
 		mboxes = <&lsio_mu6 0 1
 			  &lsio_mu6 1 1
 			  &lsio_mu6 3 1>;
-		memory-region = <&vdevbuffer>, <&vdev2vring0>, <&vdev2vring1>,
+		memory-region = <&vdev0buffer>, <&vdev2vring0>, <&vdev2vring1>,
 				<&vdev3vring0>, <&vdev3vring1>, <&rsc_table1>;
 		power-domains = <&pd IMX_SC_R_M4_1_PID0>, <&pd IMX_SC_R_M4_1_MU_1A>;
 
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-ss-conn.dtsi b/arch/arm64/boot/dts/freescale/imx8qm-ss-conn.dtsi
index ccf9f510e0f88b..f25f62d51f69ee 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm-ss-conn.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8qm-ss-conn.dtsi
@@ -4,6 +4,41 @@
  *	Dong Aisheng <aisheng.dong@nxp.com>
  */
 
+&{/} {
+	usbphynop2: usbphynop2 {
+		compatible = "usb-nop-xceiv";
+		clocks = <&usb2_lpcg IMX_LPCG_CLK_7>;
+		clock-names = "main_clk";
+		#phy-cells = <0>;
+		power-domains = <&pd IMX_SC_R_USB_1_PHY>;
+		status = "disabled";
+	};
+};
+
+&conn_subsys {
+	usbh1: usb@5b0e0000 {
+		compatible = "fsl,imx7ulp-usb", "fsl,imx6ul-usb", "fsl,imx27-usb";
+		reg = <0x5b0e0000 0x200>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>;
+		phy_type = "hsic";
+		dr_mode = "host";
+		clocks = <&usb2_lpcg IMX_LPCG_CLK_6>;
+		ahb-burst-config = <0x0>;
+		tx-burst-size-dword = <0x10>;
+		rx-burst-size-dword = <0x10>;
+		power-domains = <&pd IMX_SC_R_USB_1>;
+		fsl,usbphy = <&usbphynop2>;
+		fsl,usbmisc = <&usbmisc2 0>;
+	};
+
+	usbmisc2: usbmisc@5b0e0200 {
+		compatible = "fsl,imx7ulp-usbmisc", "fsl,imx7d-usbmisc", "fsl,imx6q-usbmisc";
+		#index-cells = <1>;
+		reg = <0x5b0e0200 0x200>;
+	};
+};
+
 &usbphy1 {
 	compatible = "fsl,imx8qm-usbphy", "fsl,imx7ulp-usbphy";
 };
@@ -32,3 +67,7 @@
 	compatible = "fsl,imx8qm-usdhc", "fsl,imx8qxp-usdhc", "fsl,imx7d-usdhc";
 	iommus = <&smmu 0x11 0x7f80>;
 };
+
+&usbotg3_cdns3 {
+	iommus = <&smmu 0x4 0x7f80>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-var-som-symphony.dts b/arch/arm64/boot/dts/freescale/imx8qm-var-som-symphony.dts
index ec55b86c220d13..4528627f58c89b 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm-var-som-symphony.dts
+++ b/arch/arm64/boot/dts/freescale/imx8qm-var-som-symphony.dts
@@ -303,7 +303,7 @@
 	};
 };
 
-&pciea{
+&pciea {
 	phys = <&hsio_phy 0 PHY_TYPE_PCIE 0>;
 	phy-names = "pcie-phy";
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-var-som.dtsi b/arch/arm64/boot/dts/freescale/imx8qm-var-som.dtsi
index b11da10c5ae39e..a44351133428e9 100644
--- a/arch/arm64/boot/dts/freescale/imx8qm-var-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8qm-var-som.dtsi
@@ -275,15 +275,15 @@
 	status = "okay";
 };
 
-&mu_m0{
+&mu_m0 {
 	status = "okay";
 };
 
-&mu1_m0{
+&mu1_m0 {
 	status = "okay";
 };
 
-&mu2_m0{
+&mu2_m0 {
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
index 857a6e51222833..eb994f3e28729f 100644
--- a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
+++ b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
@@ -27,7 +27,7 @@
 		mboxes = <&lsio_mu5 0 1
 			  &lsio_mu5 1 1
 			  &lsio_mu5 3 1>;
-		memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+		memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 				<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
 		power-domains = <&pd IMX_SC_R_M4_0_PID0>,
 				<&pd IMX_SC_R_M4_0_MU_1A>;
@@ -242,48 +242,48 @@
 		#size-cells = <2>;
 		ranges;
 
-		vdev0vring0: memory@90000000 {
+		vdev0vring0: vdev0vring0@90000000 {
 			reg = <0 0x90000000 0 0x8000>;
 			no-map;
 		};
 
-		vdev0vring1: memory@90008000 {
+		vdev0vring1: vdev0vring1@90008000 {
 			reg = <0 0x90008000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring0: memory@90010000 {
+		vdev1vring0: vdev1vring0@90010000 {
 			reg = <0 0x90010000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring1: memory@90018000 {
+		vdev1vring1: vdev1vring1@90018000 {
 			reg = <0 0x90018000 0 0x8000>;
 			no-map;
 		};
 
-		rsc_table: memory@900ff000 {
+		rsc_table: rsc-table@900ff000 {
 			reg = <0 0x900ff000 0 0x1000>;
 			no-map;
 		};
 
-		vdevbuffer: memory@90400000 {
+		vdev0buffer: vdev0buffer@90400000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0x90400000 0 0x100000>;
 			no-map;
 		};
 
-		dsp_vdev0vring0: memory@942f0000 {
+		dsp_vdev0vring0: vdev0vring0@942f0000 {
 			reg = <0 0x942f0000 0 0x8000>;
 			no-map;
 		};
 
-		dsp_vdev0vring1: memory@942f8000 {
+		dsp_vdev0vring1: vdev0vring1@942f8000 {
 			reg = <0 0x942f8000 0 0x8000>;
 			no-map;
 		};
 
-		dsp_vdev0buffer: memory@94300000 {
+		dsp_vdev0buffer: vdev0buffer@94300000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0x94300000 0 0x100000>;
 			no-map;
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
index 4b925832bb662d..fdf3aab2224b65 100644
--- a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
@@ -1,12 +1,13 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
 /*
- * Copyright 2021 NXP
+ * Copyright 2021, 2025 NXP
  */
 
 /dts-v1/;
 
 #include <dt-bindings/input/input.h>
 #include "imx8ulp.dtsi"
+#include "imx8ulp-firmware.dtsi"
 
 / {
 	model = "NXP i.MX8ULP EVK";
@@ -80,7 +81,7 @@
 			no-map;
 		};
 
-		vdevbuffer: vdevbuffer@a8400000 {
+		vdev0buffer: vdev0buffer@a8400000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0xa8400000 0 0x100000>;
 			no-map;
@@ -139,7 +140,7 @@
 	mboxes = <&mu 0 1>,
 		 <&mu 1 1>,
 		 <&mu 3 1>;
-	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+	memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-firmware.dtsi b/arch/arm64/boot/dts/freescale/imx8ulp-firmware.dtsi
new file mode 100644
index 00000000000000..e4bc352f68afb9
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8ulp-firmware.dtsi
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2025 NXP
+ *
+ * Default reserved memory and EdgeLock Enclave (ELE) enablement shared by
+ * i.MX8ULP boards. Include this from a board dts to bring up the secure
+ * enclave without having to duplicate the reserved-memory node. The ELE
+ * hardware can only access DDR in the 0x80000000 - 0xafffffff window, so the
+ * pool is constrained to that range with alloc-ranges and the kernel is left
+ * to place the 1 MiB region.
+ */
+
+/ {
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ele_reserved: ele-reserved {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0 0x80000000 0 0x30000000>;
+			size = <0 0x100000>;
+			no-map;
+		};
+	};
+};
+
+&hsm0 {
+	memory-region = <&ele_reserved>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
index 777b89ed684901..ac8293f81c6b2a 100644
--- a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
 /*
- * Copyright 2021 NXP
+ * Copyright 2021, 2025 NXP
  */
 
 #include <dt-bindings/clock/imx8ulp-clock.h>
@@ -154,7 +154,7 @@
 		#clock-cells = <0>;
 	};
 
-	sram@22010000 {
+	sram0: sram@22010000 {
 		compatible = "mmio-sram";
 		reg = <0x0 0x22010000 0x0 0x10000>;
 
@@ -186,6 +186,14 @@
 				#thermal-sensor-cells = <1>;
 			};
 		};
+
+		hsm0: secure-enclave {
+			compatible = "fsl,imx8ulp-se-ele-hsm";
+			mbox-names = "tx", "rx";
+			mboxes = <&s4muap 0 0>, <&s4muap 1 0>;
+			sram = <&sram0>;
+			status = "disabled";
+		};
 	};
 
 	cm33: remoteproc-cm33 {
diff --git a/arch/arm64/boot/dts/freescale/imx9-mqs.dtso b/arch/arm64/boot/dts/freescale/imx9-mqs.dtso
new file mode 100644
index 00000000000000..bdb251299e525e
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx9-mqs.dtso
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	sound-mqs {
+		compatible = "audio-graph-card2";
+		aux-devs = <&amp_mqs1>;
+		label = "mqs-audio";
+		links = <&sai1_port1>;
+		routing = "MQS AMP INL", "MQS_L",
+			  "MQS AMP INR", "MQS_R";
+	};
+};
+
+&mqs1_conflict_card {
+	status = "disabled";
+};
+
+&mqs1_conflict_dev {
+	status = "disabled";
+};
+
+&mqs1 {
+	status = "okay";
+
+	port {
+		mqs1_endpoint: endpoint {
+			remote-endpoint = <&sai1_endpoint1>;
+		};
+	};
+};
+
+&sai1 {
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		sai1_port1: port@1 {
+			reg = <1>;
+			playback-only;
+
+			sai1_endpoint1: endpoint {
+				dai-format = "left_j";
+				bitclock-master;
+				frame-master;
+				mclk-fs = <512>;
+				system-clock-direction-out;
+				remote-endpoint = <&mqs1_endpoint>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
index 5716febb5a50f6..66466ce001cae0 100644
--- a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
@@ -25,6 +25,7 @@
 		mmc1 = &usdhc2;
 		mmc2 = &usdhc3;
 		rtc0 = &bbnsm_rtc;
+		rtc1 = &pcf2131;
 		serial0 = &lpuart1;
 		serial1 = &lpuart2;
 		serial2 = &lpuart3;
@@ -32,6 +33,12 @@
 		serial4 = &lpuart5;
 	};
 
+	amp_mqs1: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		sound-name-prefix = "MQS AMP";
+		VCC-supply = <&reg_3p3v>;
+	};
+
 	bt_sco_codec: bt-sco-codec {
 		compatible = "linux,bt-sco";
 		#sound-dai-cells = <1>;
@@ -41,6 +48,32 @@
 		stdout-path = &lpuart1;
 	};
 
+	pdm_mqs1_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&adp5585 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		mqs1_fun: mqs1-grp {
+			mux-states = <&pdm_mqs1_mux 1>;
+		};
+
+		pdm_fun: pdm-grp {
+			mux-states = <&pdm_mqs1_mux 0>;
+		};
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "+V3.3_SW";
+	};
+
 	reg_vref_1v8: regulator-adc-vref {
 		compatible = "regulator-fixed";
 		regulator-max-microvolt = <1800000>;
@@ -126,7 +159,7 @@
 			"IN1R", "AMIC";
 	};
 
-	sound-bt-sco {
+	mqs1_conflict_card: sound-bt-sco {
 		compatible = "simple-audio-card";
 		simple-audio-card,name = "bt-sco-audio";
 		simple-audio-card,format = "dsp_a";
@@ -482,9 +515,9 @@
 	};
 };
 
-&micfil {
+mqs1_conflict_dev: &micfil {
 	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-0 = <&pinctrl_pdm>, <&pdm_fun>;
 	pinctrl-1 = <&pinctrl_pdm_sleep>;
 	assigned-clocks = <&clk IMX93_CLK_PDM>;
 	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
@@ -492,10 +525,19 @@
 	status = "okay";
 };
 
+&mqs1 {
+	pinctrl-0 = <&pinctrl_mqs1>, <&mqs1_fun>;
+	pinctrl-names = "default";
+};
+
 &sai1 {
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&pinctrl_sai1>;
 	pinctrl-1 = <&pinctrl_sai1_sleep>;
+	clocks = <&clk IMX93_CLK_SAI1_IPG>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_SAI1_GATE>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_DUMMY>, <&clk IMX93_CLK_AUDIO_PLL>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k";
 	assigned-clocks = <&clk IMX93_CLK_SAI1>;
 	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
 	assigned-clock-rates = <12288000>;
@@ -701,6 +743,13 @@
 		>;
 	};
 
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			MX91_PAD_PDM_CLK__MQS1_LEFT			0x31e
+			MX91_PAD_PDM_BIT_STREAM0__MQS1_RIGHT		0x31e
+		>;
+	};
+
 	pinctrl_pdm: pdmgrp {
 		fsl,pins = <
 			MX91_PAD_PDM_CLK__PDM_CLK			0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts b/arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts
index 62dc1dedfb0ee4..c6019d76b99818 100644
--- a/arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-11x11-frdm-s.dts
@@ -672,8 +672,6 @@
 };
 
 &mqs1 {
-	clocks = <&clk IMX93_CLK_MQS1_GATE>;
-	clock-names = "mclk";
 	pinctrl-0 = <&pinctrl_mqs1>;
 	pinctrl-names = "default";
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx91-11x11-frdm.dts b/arch/arm64/boot/dts/freescale/imx91-11x11-frdm.dts
index c25561574d3fc5..6144fa6f370672 100644
--- a/arch/arm64/boot/dts/freescale/imx91-11x11-frdm.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-11x11-frdm.dts
@@ -468,8 +468,6 @@
 };
 
 &mqs1 {
-	clocks = <&clk IMX93_CLK_MQS1_GATE>;
-	clock-names = "mclk";
 	pinctrl-0 = <&pinctrl_mqs1>;
 	pinctrl-names = "default";
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx91-9x9-qsb-tianma-tm050rdh03.dtso b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb-tianma-tm050rdh03.dtso
new file mode 100644
index 00000000000000..01c22f92e0d812
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb-tianma-tm050rdh03.dtso
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pwm/pwm.h>
+#include "imx91-pinfunc.h"
+
+&{/} {
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&tpm1 3 40000 PWM_POLARITY_INVERTED>;	/* 25KHz PWM */
+		brightness-levels = <0 100>;
+		num-interpolated-steps = <100>;
+		default-brightness-level = <80>;
+		power-supply = <&reg_exp_5v>;
+	};
+
+	panel {
+		compatible = "tianma,tm050rdh03";
+		backlight = <&backlight>;
+		enable-gpios = <&pcal6524 8 GPIO_ACTIVE_HIGH>;
+		power-supply = <&reg_exp_3v3>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dpi_to_panel>;
+			};
+		};
+	};
+};
+
+&dpi_bridge {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lcdif>, <&gpio_fun>, <&sai3_tx_fun>, <&sai3_enable>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			dpi_to_panel: endpoint {
+				remote-endpoint = <&panel_in>;
+				bus-width = <18>;
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_lcdif: lcdifgrp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO00__MEDIAMIX_DISP_CLK		0x31e
+			MX91_PAD_GPIO_IO01__MEDIAMIX_DISP_DE		0x31e
+			MX91_PAD_GPIO_IO02__MEDIAMIX_DISP_VSYNC		0x31e
+			MX91_PAD_GPIO_IO03__MEDIAMIX_DISP_HSYNC		0x31e
+			MX91_PAD_GPIO_IO04__MEDIAMIX_DISP_DATA0		0x31e
+			MX91_PAD_GPIO_IO05__MEDIAMIX_DISP_DATA1		0x31e
+			MX91_PAD_GPIO_IO06__MEDIAMIX_DISP_DATA2		0x31e
+			MX91_PAD_GPIO_IO07__MEDIAMIX_DISP_DATA3		0x31e
+			MX91_PAD_GPIO_IO08__MEDIAMIX_DISP_DATA4		0x31e
+			MX91_PAD_GPIO_IO09__MEDIAMIX_DISP_DATA5		0x31e
+			MX91_PAD_GPIO_IO10__MEDIAMIX_DISP_DATA6		0x31e
+			MX91_PAD_GPIO_IO11__MEDIAMIX_DISP_DATA7		0x31e
+			MX91_PAD_GPIO_IO12__MEDIAMIX_DISP_DATA8		0x31e
+			MX91_PAD_GPIO_IO13__MEDIAMIX_DISP_DATA9		0x31e
+			MX91_PAD_GPIO_IO14__MEDIAMIX_DISP_DATA10	0x31e
+			MX91_PAD_GPIO_IO15__MEDIAMIX_DISP_DATA11	0x31e
+			MX91_PAD_GPIO_IO16__MEDIAMIX_DISP_DATA12	0x31e
+			MX91_PAD_GPIO_IO17__MEDIAMIX_DISP_DATA13	0x31e
+			MX91_PAD_GPIO_IO18__MEDIAMIX_DISP_DATA14	0x31e
+			MX91_PAD_GPIO_IO19__MEDIAMIX_DISP_DATA15	0x31e
+			MX91_PAD_GPIO_IO20__MEDIAMIX_DISP_DATA16	0x31e
+			MX91_PAD_GPIO_IO21__MEDIAMIX_DISP_DATA17	0x31e
+		>;
+	};
+
+	pinctrl_tpm1_ch3: tpm1ch3grp {
+		fsl,pins = <
+			MX91_PAD_UART2_TXD__TPM1_CH3			0x31e
+		>;
+	};
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&media_blk_ctrl {
+	status = "okay";
+};
+
+&tpm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_tpm1_ch3>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
index 044b37fcc0ba0a..b7f0e39b867aa4 100644
--- a/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
@@ -23,6 +23,7 @@
 		mmc0 = &usdhc1;
 		mmc1 = &usdhc2;
 		rtc0 = &bbnsm_rtc;
+		rtc1 = &pcf2131;
 		serial0 = &lpuart1;
 	};
 
@@ -75,6 +76,27 @@
 		mux-gpios = <&pcal6524 9 GPIO_ACTIVE_HIGH>;
 	};
 
+	exp_mux: mux-controller-3 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524 22 GPIO_ACTIVE_HIGH>;
+	};
+
+	sai3_tx_rx_mux: mux-controller-4 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524 14 GPIO_ACTIVE_HIGH>;
+	};
+
+	sai3_en_mux: mux-controller-5 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524 23 GPIO_ACTIVE_HIGH>;
+	};
+
 	pinctrl-gpiomux {
 		compatible = "pinctrl-multiplexer";
 
@@ -82,6 +104,10 @@
 			mux-states = <&can_mux 1>;
 		};
 
+		gpio_fun: exp-grp {
+			mux-states = <&exp_mux 0>;
+		};
+
 		m2_fun: m2-grp {
 			mux-states = <&sai1_mux 1>;
 		};
@@ -101,6 +127,32 @@
 		sai1_fun: sai1-grp {
 			mux-states = <&sai1_mux 0>;
 		};
+
+		sai3_enable: sai3-enable-grp {
+			mux-states = <&sai3_en_mux 0>;
+		};
+
+		sai3_tx_fun: sai3-tx-grp {
+			mux-states = <&sai3_tx_rx_mux 0>;
+		};
+	};
+
+	reg_exp_3v3: regulator-exp-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "EXP_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pcal6524 21 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_exp_5v: regulator-exp-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "EXP_5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&pcal6524 4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
 
 	reg_vref_1v8: regulator-adc-vref {
diff --git a/arch/arm64/boot/dts/freescale/imx91-lino-verdin-dahlia.dts b/arch/arm64/boot/dts/freescale/imx91-lino-verdin-dahlia.dts
new file mode 100644
index 00000000000000..eb3a8cdcd08f8b
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-lino-verdin-dahlia.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX91 SoM on Lino-Verdin Adapter / Dahlia carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx91
+ * https://www.toradex.com/products/carrier-board/dahlia-carrier-board-kit
+ */
+
+/dts-v1/;
+
+#include "imx91.dtsi"
+#include "imx91-lino.dtsi"
+#include "imx93-imx91-lino-verdin-common.dtsi"
+#include "imx93-imx91-lino-verdin-dahlia-common.dtsi"
+
+/ {
+	model = "Toradex Lino iMX91 on Verdin Adapter plus Dahlia Board";
+	compatible = "toradex,lino-imx91-verdin-dahlia",
+		     "toradex,lino-imx91-verdin",
+		     "toradex,lino-imx91",
+		     "fsl,imx91";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-lino-verdin-dev.dts b/arch/arm64/boot/dts/freescale/imx91-lino-verdin-dev.dts
new file mode 100644
index 00000000000000..a988897f338828
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-lino-verdin-dev.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX91 SoM on Lino-Verdin Adapter / Development carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx91
+ * https://www.toradex.com/products/carrier-board/verdin-development-board-kit
+ */
+
+/dts-v1/;
+
+#include "imx91.dtsi"
+#include "imx91-lino.dtsi"
+#include "imx93-imx91-lino-verdin-common.dtsi"
+#include "imx93-imx91-lino-verdin-dev-common.dtsi"
+
+/ {
+	model = "Toradex Lino iMX91 on Verdin Adapter plus Development Board";
+	compatible = "toradex,lino-imx91-verdin-dev",
+		     "toradex,lino-imx91-verdin",
+		     "toradex,lino-imx91",
+		     "fsl,imx91";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-lino.dtsi b/arch/arm64/boot/dts/freescale/imx91-lino.dtsi
new file mode 100644
index 00000000000000..99d95694e91036
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-lino.dtsi
@@ -0,0 +1,305 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX91 SoM
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx91
+ */
+
+#include "imx91-pinfunc.h"
+#include "imx93-imx91-lino-common.dtsi"
+
+&iomuxc {
+	/* Lino CAN_1 */
+	pinctrl_can1: can1grp {
+		fsl,pins = <MX91_PAD_PDM_BIT_STREAM0__CAN1_RX	0x139e>, /* Lino P12 */
+			   <MX91_PAD_PDM_CLK__CAN1_TX		0x139e>; /* Lino P10 */
+	};
+
+	/* Lino CAN_2 */
+	pinctrl_can2: can2grp {
+		fsl,pins = <MX91_PAD_GPIO_IO25__CAN2_TX	0x139e>, /* Lino S28 */
+			   <MX91_PAD_GPIO_IO27__CAN2_RX	0x139e>; /* Lino S30 */
+	};
+
+	/* Lino ETH_1 RGMII */
+	pinctrl_eth1: eqosgrp {
+		fsl,pins = <MX91_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x1fe>, /* Lino P77 */
+			   <MX91_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x1fe>, /* Lino P79 */
+			   <MX91_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x1fe>, /* Lino P81 */
+			   <MX91_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x1fe>, /* Lino P83 */
+			   <MX91_PAD_ENET1_RXC__ENET_QOS_RGMII_RXC			0x1fe>, /* Lino P73 */
+			   <MX91_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x1fe>, /* Lino P75 */
+			   <MX91_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x1fe>, /* Lino P97 */
+			   <MX91_PAD_ENET1_TD1__ENET1_RGMII_TD1				0x1fe>, /* Lino P95 */
+			   <MX91_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x1fe>, /* Lino P93 */
+			   <MX91_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x1fe>, /* Lino P91 */
+			   <MX91_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x1fe>, /* Lino P89 */
+			   <MX91_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x1fe>; /* Lino P85 */
+	};
+
+	/* Lino ETH MDIO */
+	pinctrl_mdio: eqosmdiogrp {
+		fsl,pins = <MX91_PAD_ENET1_MDC__ENET1_MDC	0x1fe>, /* Lino P69 */
+			   <MX91_PAD_ENET1_MDIO__ENET_QOS_MDIO	0x1fe>; /* Lino P67 */
+	};
+
+	/* Lino ETH_2 RGMII */
+	pinctrl_eth2: fecgrp {
+		fsl,pins = <MX91_PAD_ENET2_RD0__ENET2_RGMII_RD0		0x1fe>, /* Lino S77 */
+			   <MX91_PAD_ENET2_RD1__ENET2_RGMII_RD1		0x1fe>, /* Lino S79 */
+			   <MX91_PAD_ENET2_RD2__ENET2_RGMII_RD2		0x1fe>, /* Lino S81 */
+			   <MX91_PAD_ENET2_RD3__ENET2_RGMII_RD3		0x1fe>, /* Lino S83 */
+			   <MX91_PAD_ENET2_RXC__ENET2_RGMII_RXC		0x1fe>, /* Lino S71 */
+			   <MX91_PAD_ENET2_RX_CTL__ENET2_RGMII_RX_CTL	0x1fe>, /* Lino S73 */
+			   <MX91_PAD_ENET2_TD0__ENET2_RGMII_TD0		0x1fe>, /* Lino S99 */
+			   <MX91_PAD_ENET2_TD1__ENET2_RGMII_TD1		0x1fe>, /* Lino S97 */
+			   <MX91_PAD_ENET2_TD2__ENET2_RGMII_TD2		0x1fe>, /* Lino S93 */
+			   <MX91_PAD_ENET2_TD3__ENET2_RGMII_TD3		0x1fe>, /* Lino S91 */
+			   <MX91_PAD_ENET2_TXC__ENET2_RGMII_TXC		0x1fe>, /* Lino S87 */
+			   <MX91_PAD_ENET2_TX_CTL__ENET2_RGMII_TX_CTL	0x1fe>; /* Lino S89 */
+	};
+
+	/* Lino GPIO_11 */
+	pinctrl_gpio_11: gpio1io10grp {
+		fsl,pins = <MX91_PAD_PDM_BIT_STREAM1__GPIO1_IO10	0x139e>; /* Lino P65 */
+	};
+
+	/* Lino SPI_2_CS as GPIO */
+	pinctrl_spi2_cs0_gpio: gpio1io11grp {
+		fsl,pins = <MX91_PAD_SAI1_TXFS__GPIO1_IO11	0x139e>; /* Lino S18 */
+	};
+
+	/* Lino GPIO_1 */
+	pinctrl_gpio1: gpio2io0grp {
+		fsl,pins = <MX91_PAD_GPIO_IO00__GPIO2_IO0	0x139e>; /* Lino P30 */
+	};
+
+	/* Lino GPIO_2 */
+	pinctrl_gpio2: gpio2io1grp {
+		fsl,pins = <MX91_PAD_GPIO_IO01__GPIO2_IO1	0x139e>; /* Lino P32 */
+	};
+
+	/* Lino SPI_1_CS as GPIO */
+	pinctrl_spi1_cs0_gpio: gpio2io12grp {
+		fsl,pins = <MX91_PAD_GPIO_IO12__GPIO2_IO12	0x139e>; /* Lino P58 */
+	};
+
+	/* Lino GPIO_3 */
+	pinctrl_gpio3: gpio2io17grp {
+		fsl,pins = <MX91_PAD_GPIO_IO17__GPIO2_IO17	0x139e>; /* Lino P34 */
+	};
+
+	/* Lino GPIO_4 */
+	pinctrl_gpio4: gpio2io22grp {
+		fsl,pins = <MX91_PAD_GPIO_IO22__GPIO2_IO22	0x139e>; /* Lino P36 */
+	};
+
+	/* Lino PWM_3 as GPIO */
+	pinctrl_pwm3_gpio: gpio2io23grp {
+		fsl,pins = <MX91_PAD_GPIO_IO23__GPIO2_IO23	0x139e>; /* Lino P5 */
+	};
+
+	/* Lino GPIO_5 */
+	pinctrl_gpio5: gpio2io24grp {
+		fsl,pins = <MX91_PAD_GPIO_IO24__GPIO2_IO24	0x139e>; /* Lino P38 */
+	};
+
+	/* Lino GPIO_6 */
+	pinctrl_gpio6: gpio3io26grp {
+		fsl,pins = <MX91_PAD_CCM_CLKO1__GPIO3_IO26	0x139e>; /* Lino P40 */
+	};
+
+	/* Lino GPIO_7 */
+	pinctrl_gpio7: gpio3io27grp {
+		fsl,pins = <MX91_PAD_CCM_CLKO2__GPIO3_IO27	0x139e>; /* Lino P42 */
+	};
+
+	/* Lino GPIO_9 */
+	pinctrl_gpio9: gpio4io14grp {
+		fsl,pins = <MX91_PAD_ENET2_MDC__GPIO4_IO14	0x139e>; /* Lino P46 */
+	};
+
+	/* Lino GPIO_10 */
+	pinctrl_gpio10: gpio4io15grp {
+		fsl,pins = <MX91_PAD_ENET2_MDIO__GPIO4_IO15	0x139e>; /* Lino P48 */
+	};
+
+	/* Lino GPIO_8 */
+	pinctrl_gpio8: gpio4io28grp {
+		fsl,pins = <MX91_PAD_CCM_CLKO3__GPIO4_IO28	0x139e>; /* Lino P44 */
+	};
+
+	/* On-module GPIO_EXP_INT# */
+	pinctrl_gpio_exp_int: gpio4io29grp {
+		fsl,pins = <MX91_PAD_CCM_CLKO4__GPIO4_IO29	0x139e>; /* GPIO_EXP_INT# */
+	};
+
+	/* Lino I2C_1 as GPIO */
+	pinctrl_i2c1_gpio: lpi2c1gpiogrp {
+		fsl,pins = <MX91_PAD_I2C1_SCL__GPIO1_IO0	0x40000b9e>, /* Lino P18 */
+			   <MX91_PAD_I2C1_SDA__GPIO1_IO1	0x40000b9e>; /* Lino P16 */
+	};
+
+	/* Lino I2C_1 */
+	pinctrl_i2c1: lpi2c1grp {
+		fsl,pins = <MX91_PAD_I2C1_SCL__LPI2C1_SCL	0x40000b9e>, /* Lino P18 */
+			   <MX91_PAD_I2C1_SDA__LPI2C1_SDA	0x40000b9e>; /* Lino P16 */
+	};
+
+	/* On-module PMIC I2C as GPIO */
+	pinctrl_pmic_i2c_gpio: lpi2c2gpiogrp {
+		fsl,pins = <MX91_PAD_I2C2_SCL__GPIO1_IO2	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX91_PAD_I2C2_SDA__GPIO1_IO3	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* On-module PMIC I2C */
+	pinctrl_pmic_i2c: lpi2c2grp {
+		fsl,pins = <MX91_PAD_I2C2_SCL__LPI2C2_SCL	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX91_PAD_I2C2_SDA__LPI2C2_SDA	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* Lino I2C_2 as GPIO */
+	pinctrl_i2c2_gpio: lpi2c3gpiogrp {
+		fsl,pins = <MX91_PAD_GPIO_IO28__GPIO2_IO28	0x40000b9e>, /* Lino P20 */
+			   <MX91_PAD_GPIO_IO29__GPIO2_IO29	0x40000b9e>; /* Lino P22 */
+	};
+
+	/* Lino I2C_2 */
+	pinctrl_i2c2: lpi2c3grp {
+		fsl,pins = <MX91_PAD_GPIO_IO28__LPI2C3_SDA	0x40000b9e>, /* Lino P20 */
+			   <MX91_PAD_GPIO_IO29__LPI2C3_SCL	0x40000b9e>; /* Lino P22 */
+	};
+
+	/* Lino I2C_3 as GPIO */
+	pinctrl_i2c3_gpio: lpi2c4gpiogrp {
+		fsl,pins = <MX91_PAD_GPIO_IO02__GPIO2_IO2	0x40000b9e>, /* Lino P24 */
+			   <MX91_PAD_GPIO_IO03__GPIO2_IO3	0x40000b9e>; /* Lino P26 */
+	};
+
+	/* Lino I2C_3 */
+	pinctrl_i2c3: lpi2c4grp {
+		fsl,pins = <MX91_PAD_GPIO_IO02__LPI2C4_SDA	0x40000b9e>, /* Lino P24 */
+			   <MX91_PAD_GPIO_IO03__LPI2C4_SCL	0x40000b9e>; /* Lino P26 */
+	};
+
+	/* Lino SPI_2 */
+	pinctrl_spi2: lpspi1grp {
+		fsl,pins = <MX91_PAD_SAI1_TXD0__LPSPI1_SCK	0x139e>, /* Lino S20 */
+			   <MX91_PAD_SAI1_RXD0__LPSPI1_SOUT	0x139e>, /* Lino S22 */
+			   <MX91_PAD_SAI1_TXC__LPSPI1_SIN	0x139e>; /* Lino S24 */
+	};
+
+	/* Lino SPI_1 */
+	pinctrl_spi1: lpspi8grp {
+		fsl,pins = <MX91_PAD_GPIO_IO13__LPSPI8_SIN	0x139e>, /* Lino P54 */
+			   <MX91_PAD_GPIO_IO14__LPSPI8_SOUT	0x139e>, /* Lino P56 */
+			   <MX91_PAD_GPIO_IO15__LPSPI8_SCK	0x139e>; /* Lino P52 */
+	};
+
+	/* Lino UART_3 */
+	pinctrl_uart3: lpuart1grp {
+		fsl,pins = <MX91_PAD_UART1_RXD__LPUART1_RX	0x39e>, /* Lino P17 */
+			   <MX91_PAD_UART1_TXD__LPUART1_TX	0x39e>; /* Lino P19 */
+	};
+
+	/* Lino UART_4 */
+	pinctrl_uart4: lpuart2grp {
+		fsl,pins = <MX91_PAD_UART2_RXD__LPUART2_RX	0x39e>, /* Lino P23 */
+			   <MX91_PAD_UART2_TXD__LPUART2_TX	0x39e>; /* Lino P25 */
+	};
+
+	/* Lino UART_1 */
+	pinctrl_uart1: lpuart6grp {
+		fsl,pins = <MX91_PAD_GPIO_IO04__LPUART6_TX	0x39e>, /* Lino P11 */
+			   <MX91_PAD_GPIO_IO05__LPUART6_RX	0x39e>, /* Lino P9 */
+			   <MX91_PAD_GPIO_IO06__LPUART6_CTS_B	0x39e>, /* Lino P8 */
+			   <MX91_PAD_GPIO_IO07__LPUART6_RTS_B	0x39e>; /* Lino P6 */
+	};
+
+	/* Lino UART_2 */
+	pinctrl_uart2: lpuart7grp {
+		fsl,pins = <MX91_PAD_GPIO_IO08__LPUART7_TX	0x39e>, /* Lino P15 */
+			   <MX91_PAD_GPIO_IO09__LPUART7_RX	0x39e>, /* Lino P13 */
+			   <MX91_PAD_GPIO_IO10__LPUART7_CTS_B	0x39e>, /* Lino S7 */
+			   <MX91_PAD_GPIO_IO11__LPUART7_RTS_B	0x39e>; /* Lino S5 */
+	};
+
+	/* Lino I2S_1 */
+	pinctrl_i2s1: sai3grp {
+		fsl,pins = <MX91_PAD_GPIO_IO16__SAI3_TX_BCLK	0x139e>, /* Lino S94 */
+			   <MX91_PAD_GPIO_IO19__SAI3_TX_DATA0	0x139e>, /* Lino S98 */
+			   <MX91_PAD_GPIO_IO20__SAI3_RX_DATA0	0x139e>, /* Lino S100 */
+			   <MX91_PAD_GPIO_IO26__SAI3_TX_SYNC	0x139e>; /* Lino S96 */
+	};
+
+	/* Lino PWM_1 */
+	pinctrl_pwm1: tpm4ch1grp {
+		fsl,pins = <MX91_PAD_GPIO_IO21__TPM4_CH1	0x139e>; /* Lino P1 */
+	};
+
+	/* Lino PWM_2 */
+	pinctrl_pwm2: tpm5ch2grp {
+		fsl,pins = <MX91_PAD_GPIO_IO18__TPM5_CH2	0x139e>; /* Lino P3 */
+	};
+
+	/* Lino PWM_3 */
+	pinctrl_pwm3: tpm6ch1grp {
+		fsl,pins = <MX91_PAD_GPIO_IO23__TPM6_CH1	0x139e>; /* Lino P5 */
+	};
+
+	/* On-module eMMC */
+	pinctrl_emmc: usdhc1grp {
+		fsl,pins = <MX91_PAD_SD1_CLK__USDHC1_CLK	0x5fe>,      /* eMMC_CLK    */
+			   <MX91_PAD_SD1_CMD__USDHC1_CMD	0x400013fe>, /* eMMC_CMD    */
+			   <MX91_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe>, /* eMMC_DATA0  */
+			   <MX91_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe>, /* eMMC_DATA1  */
+			   <MX91_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe>, /* eMMC_DATA2  */
+			   <MX91_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe>, /* eMMC_DATA3  */
+			   <MX91_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe>, /* eMMC_DATA4  */
+			   <MX91_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe>, /* eMMC_DATA5  */
+			   <MX91_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe>, /* eMMC_DATA6  */
+			   <MX91_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe>, /* eMMC_DATA7  */
+			   <MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x1580>;     /* eMMC_STROBE */
+	};
+
+	/* Lino SD_1_CD# */
+	pinctrl_sd1_cd: usdhc2cdgrp {
+		fsl,pins = <MX91_PAD_SD2_CD_B__GPIO3_IO0	0x139e>; /* Lino P78 */
+	};
+
+	/* Lino SD_1 */
+	pinctrl_sd1: usdhc2grp {
+		fsl,pins = <MX91_PAD_SD2_CLK__USDHC2_CLK	0x5fe>,      /* Lino P68 */
+			   <MX91_PAD_SD2_CMD__USDHC2_CMD	0x400013fe>, /* Lino P66 */
+			   <MX91_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe>, /* Lino P74 */
+			   <MX91_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe>, /* Lino P76 */
+			   <MX91_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe>, /* Lino P60 */
+			   <MX91_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe>; /* Lino P64 */
+	};
+
+	/* Lino SD_1_PWR_EN */
+	pinctrl_sd1_pwr_en: usdhc2pwrgrp {
+		fsl,pins = <MX91_PAD_SD2_RESET_B__GPIO3_IO7	0x139e>; /* Lino P72 */
+	};
+
+	/* On-module PMIC_SD_2_VSEL */
+	pinctrl_sd1_vsel: usdhc2vselgrp {
+		fsl,pins = <MX91_PAD_SD2_VSELECT__GPIO3_IO19	0x51e>; /* PMIC_SD_2_VSEL */
+	};
+
+	/* Lino SD_2 */
+	pinctrl_sd2: usdhc3grp {
+		fsl,pins = <MX91_PAD_SD3_CLK__USDHC3_CLK	0x5fe>,      /* Lino S10 */
+			   <MX91_PAD_SD3_CMD__USDHC3_CMD	0x400013fe>, /* Lino S8  */
+			   <MX91_PAD_SD3_DATA0__USDHC3_DATA0	0x400013fe>, /* Lino S12 */
+			   <MX91_PAD_SD3_DATA1__USDHC3_DATA1	0x400013fe>, /* Lino S16 */
+			   <MX91_PAD_SD3_DATA2__USDHC3_DATA2	0x400013fe>, /* Lino S2  */
+			   <MX91_PAD_SD3_DATA3__USDHC3_DATA3	0x400013fe>; /* Lino S4  */
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <MX91_PAD_WDOG_ANY__WDOG1_WDOG_ANY 0x31e>; /* PMIC_WDOG_B */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-toradex-osm-dev.dts b/arch/arm64/boot/dts/freescale/imx91-toradex-osm-dev.dts
new file mode 100644
index 00000000000000..ec5e701fb27566
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-toradex-osm-dev.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex OSM iMX91 SoM on OSM Development carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx91
+ * https://www.toradex.com/products/carrier-board/osm-development-board
+ */
+
+/dts-v1/;
+
+#include "imx91.dtsi"
+#include "imx91-toradex-osm.dtsi"
+#include "imx93-imx91-toradex-osm-dev-common.dtsi"
+
+/ {
+	model = "Toradex OSM iMX91 on OSM Development Board";
+	compatible = "toradex,osm-imx91-dev",
+		     "toradex,osm-imx91",
+		     "fsl,imx91";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-toradex-osm.dtsi b/arch/arm64/boot/dts/freescale/imx91-toradex-osm.dtsi
new file mode 100644
index 00000000000000..9d990937fd8d69
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-toradex-osm.dtsi
@@ -0,0 +1,303 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex OSM iMX91 SoM
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx91
+ */
+
+#include "imx91-pinfunc.h"
+#include "imx93-imx91-toradex-osm-common.dtsi"
+
+&iomuxc {
+	/* OSM CAN_A */
+	pinctrl_can_a: can1grp {
+		fsl,pins = <MX91_PAD_PDM_BIT_STREAM0__CAN1_RX	0x139e>, /* OSM AB17 */
+			   <MX91_PAD_PDM_CLK__CAN1_TX		0x139e>; /* OSM AC17 */
+	};
+
+	/* OSM CAN_B */
+	pinctrl_can_b: can2grp {
+		fsl,pins = <MX91_PAD_GPIO_IO25__CAN2_TX		0x139e>, /* OSM AC19 */
+			   <MX91_PAD_GPIO_IO27__CAN2_RX		0x139e>; /* OSM AB19 */
+	};
+
+	/* OSM CAM_MCK */
+	pinctrl_cam_mck: ccmclk1grp {
+		fsl,pins = <MX91_PAD_CCM_CLKO1__CCMSRCGPCMIX_CLKO1	0x51e>; /* OSM C2 */
+	};
+
+	/* OSM ETH_A RGMII */
+	pinctrl_eth_a: eqosgrp {
+		fsl,pins = <MX91_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x1fe>, /* OSM K15 */
+			   <MX91_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x1fe>, /* OSM L15 */
+			   <MX91_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x1fe>, /* OSM N15 */
+			   <MX91_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x1fe>, /* OSM P15 */
+			   <MX91_PAD_ENET1_RXC__ENET_QOS_RGMII_RXC			0x1fe>, /* OSM R15 */
+			   <MX91_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x1fe>, /* OSM M15 */
+			   <MX91_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x1fe>, /* OSM H15 */
+			   <MX91_PAD_ENET1_TD1__ENET1_RGMII_TD1				0x1fe>, /* OSM G15 */
+			   <MX91_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x1fe>, /* OSM H16 */
+			   <MX91_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x1fe>, /* OSM G16 */
+			   <MX91_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x1fe>, /* OSM J15 */
+			   <MX91_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x1fe>; /* OSM K16 */
+	};
+
+	/* OSM ETH_MDIO */
+	pinctrl_mdio: eqosmdiogrp {
+		fsl,pins = <MX91_PAD_ENET1_MDC__ENET1_MDC	0x1fe>, /* OSM T16 */
+			   <MX91_PAD_ENET1_MDIO__ENET_QOS_MDIO	0x1fe>; /* OSM T15 */
+	};
+
+	/* OSM ETH_B RGMII */
+	pinctrl_eth_b: fecgrp {
+		fsl,pins = <MX91_PAD_ENET2_RD0__ENET2_RGMII_RD0		0x1fe>, /* OSM J1 */
+			   <MX91_PAD_ENET2_RD1__ENET2_RGMII_RD1		0x1fe>, /* OSM K1 */
+			   <MX91_PAD_ENET2_RD2__ENET2_RGMII_RD2		0x1fe>, /* OSM M1 */
+			   <MX91_PAD_ENET2_RD3__ENET2_RGMII_RD3		0x1fe>, /* OSM N1 */
+			   <MX91_PAD_ENET2_RXC__ENET2_RGMII_RXC		0x1fe>, /* OSM P1 */
+			   <MX91_PAD_ENET2_RX_CTL__ENET2_RGMII_RX_CTL	0x1fe>, /* OSM L1 */
+			   <MX91_PAD_ENET2_TD0__ENET2_RGMII_TD0		0x1fe>, /* OSM G1 */
+			   <MX91_PAD_ENET2_TD1__ENET2_RGMII_TD1		0x1fe>, /* OSM F1 */
+			   <MX91_PAD_ENET2_TD2__ENET2_RGMII_TD2		0x1fe>, /* OSM G2 */
+			   <MX91_PAD_ENET2_TD3__ENET2_RGMII_TD3		0x1fe>, /* OSM F2 */
+			   <MX91_PAD_ENET2_TXC__ENET2_RGMII_TXC		0x1fe>, /* OSM H1 */
+			   <MX91_PAD_ENET2_TX_CTL__ENET2_RGMII_TX_CTL	0x1fe>; /* OSM J2 */
+	};
+
+	/* OSM SPI_B_CS as GPIO */
+	pinctrl_spi_b_cs0_gpio: gpio1io11grp {
+		fsl,pins = <MX91_PAD_SAI1_TXFS__GPIO1_IO11	0x139e>; /* OSM AA23 */
+	};
+
+	/* OSM GPIO_A_0 */
+	pinctrl_gpio_a0:  gpio2io0grp {
+		fsl,pins = <MX91_PAD_GPIO_IO00__GPIO2_IO0	0x139e>; /* OSM D17 */
+	};
+
+	/* OSM GPIO_A_1 */
+	pinctrl_gpio_a1: gpio2io1grp {
+		fsl,pins = <MX91_PAD_GPIO_IO01__GPIO2_IO1	0x139e>; /* OSM E17 */
+	};
+
+	/* OSM GPIO_A_2 */
+	pinctrl_gpio_a2: gpio2io2grp {
+		fsl,pins = <MX91_PAD_GPIO_IO02__GPIO2_IO2	0x139e>; /* OSM F17 */
+	};
+
+	/* OSM GPIO_A_3 */
+	pinctrl_gpio_a3: gpio2io3grp {
+		fsl,pins = <MX91_PAD_GPIO_IO03__GPIO2_IO3	0x139e>; /* OSM G17 */
+	};
+
+	/* OSM SPI_A_CS0 as GPIO */
+	pinctrl_spi_a_cs0_gpio: gpio2io12grp {
+		fsl,pins = <MX91_PAD_GPIO_IO12__GPIO2_IO12	0x139e>; /* OSM Y15 */
+	};
+
+	/* OSM PWM_0 as GPIO */
+	pinctrl_pwm0_gpio: gpio2io21grp {
+		fsl,pins = <MX91_PAD_GPIO_IO21__GPIO2_IO21	0x139e>; /* OSM E18 */
+	};
+
+	/* OSM GPIO_A_4 */
+	pinctrl_gpio_a4: gpio2io22grp {
+		fsl,pins = <MX91_PAD_GPIO_IO22__GPIO2_IO22	0x139e>; /* OSM H17 */
+	};
+
+	/* OSM GPIO_A_5 */
+	pinctrl_gpio_a5: gpio2io24grp {
+		fsl,pins = <MX91_PAD_GPIO_IO24__GPIO2_IO24	0x139e>; /* OSM J17 */
+	};
+
+	/* On-module CTRL_INT# */
+	pinctrl_ctrl_int: gpio3io27grp {
+		fsl,pins = <MX91_PAD_CCM_CLKO2__GPIO3_IO27	0x139e>; /* CTRL_INT# */
+	};
+
+	/* OSM GPIO_A_6 */
+	pinctrl_gpio_a6: gpio4io14grp {
+		fsl,pins = <MX91_PAD_ENET2_MDC__GPIO4_IO14	0x139e>; /* OSM K17 */
+	};
+
+	/* OSM GPIO_A_7 */
+	pinctrl_gpio_a7:  gpio4io15grp {
+		fsl,pins = <MX91_PAD_ENET2_MDIO__GPIO4_IO15	0x139e>; /* OSM L17 */
+	};
+
+	/* OSM I2C_B as GPIO */
+	pinctrl_i2c_b_gpio: lpi2c1gpiogrp {
+		fsl,pins = <MX91_PAD_I2C1_SCL__GPIO1_IO0	0x40000b9e>, /* OSM AA20 */
+			   <MX91_PAD_I2C1_SDA__GPIO1_IO1	0x40000b9e>; /* OSM AA21 */
+	};
+
+	/* OSM I2C_B */
+	pinctrl_i2c_b: lpi2c1grp {
+		fsl,pins = <MX91_PAD_I2C1_SCL__LPI2C1_SCL	0x40000b9e>, /* OSM AA20 */
+			   <MX91_PAD_I2C1_SDA__LPI2C1_SDA	0x40000b9e>; /* OSM AA21 */
+	};
+
+	/* On-module PMIC I2C as GPIO */
+	pinctrl_pmic_i2c_gpio: lpi2c2gpiogrp {
+		fsl,pins = <MX91_PAD_I2C2_SCL__GPIO1_IO2	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX91_PAD_I2C2_SDA__GPIO1_IO3	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* On-module PMIC I2C */
+	pinctrl_pmic_i2c: lpi2c2grp {
+		fsl,pins = <MX91_PAD_I2C2_SCL__LPI2C2_SCL	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX91_PAD_I2C2_SDA__LPI2C2_SDA	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* OSM I2C_A as GPIO */
+	pinctrl_i2c_a_gpio: lpi2c3gpiogrp {
+		fsl,pins = <MX91_PAD_GPIO_IO28__GPIO2_IO28	0x40000b9e>, /* OSM AA16 */
+			   <MX91_PAD_GPIO_IO29__GPIO2_IO29	0x40000b9e>; /* OSM AA15 */
+	};
+
+	/* OSM I2C_A */
+	pinctrl_i2c_a: lpi2c3grp {
+		fsl,pins = <MX91_PAD_GPIO_IO28__LPI2C3_SDA	0x40000b9e>, /* OSM AA16 */
+			   <MX91_PAD_GPIO_IO29__LPI2C3_SCL	0x40000b9e>; /* OSM AA15 */
+	};
+
+	/* OSM SPI_B */
+	pinctrl_spi_b: lpspi1grp {
+		fsl,pins = <MX91_PAD_SAI1_TXC__LPSPI1_SIN	0x139e>, /* OSM Y22 */
+			   <MX91_PAD_SAI1_TXD0__LPSPI1_SCK	0x139e>, /* OSM Y21 */
+			   <MX91_PAD_SAI1_RXD0__LPSPI1_SOUT	0x139e>; /* OSM Y23 */
+	};
+
+	/* OSM SPI_A */
+	pinctrl_spi_a: lpspi8grp {
+		fsl,pins = <MX91_PAD_GPIO_IO13__LPSPI8_SIN	0x139e>, /* OSM U15 */
+			   <MX91_PAD_GPIO_IO14__LPSPI8_SOUT	0x139e>, /* OSM V15 */
+			   <MX91_PAD_GPIO_IO15__LPSPI8_SCK	0x139e>; /* OSM U16 */
+	};
+
+	/* OSM UART_CON */
+	pinctrl_uart_con: lpuart1grp {
+		fsl,pins = <MX91_PAD_UART1_RXD__LPUART1_RX	0x39e>, /* OSM D22 */
+			   <MX91_PAD_UART1_TXD__LPUART1_TX	0x39e>; /* OSM D23 */
+	};
+
+	/* OSM UART_C */
+	pinctrl_uart_c: lpuart2grp {
+		fsl,pins = <MX91_PAD_UART2_RXD__LPUART2_RX	0x39e>, /* OSM A22 */
+			   <MX91_PAD_UART2_TXD__LPUART2_TX	0x39e>; /* OSM B23 */
+	};
+
+	/* OSM UART_A */
+	pinctrl_uart_a: lpuart6grp {
+		fsl,pins = <MX91_PAD_GPIO_IO04__LPUART6_TX	0x39e>, /* OSM B13 */
+			   <MX91_PAD_GPIO_IO05__LPUART6_RX	0x39e>, /* OSM A14 */
+			   <MX91_PAD_GPIO_IO06__LPUART6_CTS_B	0x39e>, /* OSM C14 */
+			   <MX91_PAD_GPIO_IO07__LPUART6_RTS_B	0x39e>; /* OSM C13 */
+	};
+
+	/* OSM UART_B */
+	pinctrl_uart_b: lpuart7grp {
+		fsl,pins = <MX91_PAD_GPIO_IO08__LPUART7_TX	0x39e>, /* OSM D13 */
+			   <MX91_PAD_GPIO_IO09__LPUART7_RX	0x39e>, /* OSM D14 */
+			   <MX91_PAD_GPIO_IO10__LPUART7_CTS_B	0x39e>, /* OSM D16 */
+			   <MX91_PAD_GPIO_IO11__LPUART7_RTS_B	0x39e>; /* OSM D15 */
+	};
+
+	/* OSM I2S_A */
+	pinctrl_i2s_a: sai3grp {
+		fsl,pins = <MX91_PAD_GPIO_IO16__SAI3_TX_BCLK	0x139e>, /* OSM W20 */
+			   <MX91_PAD_GPIO_IO19__SAI3_TX_DATA0	0x139e>, /* OSM W21 */
+			   <MX91_PAD_GPIO_IO20__SAI3_RX_DATA0	0x139e>, /* OSM V21 */
+			   <MX91_PAD_GPIO_IO26__SAI3_TX_SYNC	0x139e>; /* OSM W18 */
+	};
+
+	/* OSM I2S_MCLK */
+	pinctrl_i2s_a_mclk: sai3mclkgrp {
+		fsl,pins = <MX91_PAD_GPIO_IO17__SAI3_MCLK	0x31e>; /* OSM V18 */
+	};
+
+	/* OSM PWM_0 */
+	pinctrl_pwm0: tpm4ch1grp {
+		fsl,pins = <MX91_PAD_GPIO_IO21__TPM4_CH1	0x139e>; /* OSM E18 */
+	};
+
+	/* OSM PWM_1 */
+	pinctrl_pwm1: tpm5ch2grp {
+		fsl,pins = <MX91_PAD_GPIO_IO18__TPM5_CH2	0x139e>; /* OSM F18 */
+	};
+
+	/* OSM PWM_2 */
+	pinctrl_pwm2: tpm6ch1grp {
+		fsl,pins = <MX91_PAD_GPIO_IO23__TPM6_CH1	0x139e>; /* OSM G18 */
+	};
+
+	/* OSM USB_A_EN */
+	pinctrl_usb_a_en: usb1engrp {
+		fsl,pins = <MX91_PAD_CCM_CLKO3__GPIO4_IO28	0x139e>; /* OSM AC16 */
+	};
+
+	/* OSM USB_A_OC# */
+	pinctrl_usb_a_oc: usb1ocgrp {
+		fsl,pins = <MX91_PAD_CCM_CLKO4__GPIO4_IO29	0x139e>; /* OSM AC15 */
+	};
+
+	/* OSM USB_B_EN */
+	pinctrl_usb_b_en: usb2engrp {
+		fsl,pins = <MX91_PAD_PDM_BIT_STREAM1__GPIO1_IO10	0x139e>; /* OSM AC20 */
+	};
+
+	/* On-module eMMC */
+	pinctrl_emmc: usdhc1grp {
+		fsl,pins = <MX91_PAD_SD1_CLK__USDHC1_CLK	0x5fe>,      /* eMMC_CLK    */
+			   <MX91_PAD_SD1_CMD__USDHC1_CMD	0x400013fe>, /* eMMC_CMD    */
+			   <MX91_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe>, /* eMMC_DATA0  */
+			   <MX91_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe>, /* eMMC_DATA1  */
+			   <MX91_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe>, /* eMMC_DATA2  */
+			   <MX91_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe>, /* eMMC_DATA3  */
+			   <MX91_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe>, /* eMMC_DATA4  */
+			   <MX91_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe>, /* eMMC_DATA5  */
+			   <MX91_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe>, /* eMMC_DATA6  */
+			   <MX91_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe>, /* eMMC_DATA7  */
+			   <MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x1580>;     /* eMMC_STROBE */
+	};
+
+	/* OSM SDIO_A_CD# */
+	pinctrl_sdio_a_cd: usdhc2cdgrp {
+		fsl,pins = <MX91_PAD_SD2_CD_B__GPIO3_IO0	0x139e>; /* OSM J21 */
+	};
+
+	/* OSM SDIO_A */
+	pinctrl_sdio_a: usdhc2grp {
+		fsl,pins = <MX91_PAD_SD2_CLK__USDHC2_CLK	0x5fe>,      /* OSM F21 */
+			   <MX91_PAD_SD2_CMD__USDHC2_CMD	0x400013fe>, /* OSM E20 */
+			   <MX91_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe>, /* OSM G20 */
+			   <MX91_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe>, /* OSM G21 */
+			   <MX91_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe>, /* OSM H20 */
+			   <MX91_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe>; /* OSM H21 */
+	};
+
+	/* OSM SDIO_A_PWR_EN */
+	pinctrl_sdio_a_pwr_en: usdhc2pwrgrp {
+		fsl,pins = <MX91_PAD_SD2_RESET_B__GPIO3_IO7	0x139e>; /* OSM D21 */
+	};
+
+	/* On-module PMIC_SD_2_VSEL */
+	pinctrl_sdio_a_vsel: usdhc2vselgrp {
+		fsl,pins = <MX91_PAD_SD2_VSELECT__GPIO3_IO19	0x51e>; /* PMIC_SD_2_VSEL */
+	};
+
+	/* OSM SDIO_B */
+	pinctrl_sdio_b: usdhc3grp {
+		fsl,pins = <MX91_PAD_SD3_CLK__USDHC3_CLK	0x5fe>,      /* OSM K20 */
+			   <MX91_PAD_SD3_CMD__USDHC3_CMD	0x400013fe>, /* OSM K21 */
+			   <MX91_PAD_SD3_DATA0__USDHC3_DATA0	0x400013fe>, /* OSM L20 */
+			   <MX91_PAD_SD3_DATA1__USDHC3_DATA1	0x400013fe>, /* OSM L21 */
+			   <MX91_PAD_SD3_DATA2__USDHC3_DATA2	0x400013fe>, /* OSM M21 */
+			   <MX91_PAD_SD3_DATA3__USDHC3_DATA3	0x400013fe>; /* OSM N20 */
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <MX91_PAD_WDOG_ANY__WDOG1_WDOG_ANY 0x31e>; /* PMIC_WDOG_B */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91.dtsi b/arch/arm64/boot/dts/freescale/imx91.dtsi
index d63569b39bbc5d..a7355f533d7050 100644
--- a/arch/arm64/boot/dts/freescale/imx91.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx91.dtsi
@@ -62,6 +62,10 @@
 	compatible = "fsl,imx91-ddr-pmu", "fsl,imx93-ddr-pmu";
 };
 
+&dpi_bridge {
+	compatible = "nxp,imx91-pdfc";
+};
+
 &eqos {
 	clocks = <&clk IMX91_CLK_ENET1_QOS_TSN_GATE>,
 		 <&clk IMX91_CLK_ENET1_QOS_TSN_GATE>,
@@ -116,6 +120,13 @@
 		 <&clk IMX93_CLK_MIPI_CSI_GATE>;
 	clock-names = "apb", "axi", "nic", "disp", "cam",
 		      "lcdif", "isi", "csi";
+	assigned-clocks = <&clk IMX93_CLK_MEDIA_AXI>,
+			  <&clk IMX93_CLK_MEDIA_APB>,
+			  <&clk IMX93_CLK_MEDIA_DISP_PIX>;
+	assigned-clock-parents = <&clk IMX93_CLK_SYS_PLL_PFD0>,
+				 <&clk IMX93_CLK_SYS_PLL_PFD0_DIV2>,
+				 <&clk IMX93_CLK_VIDEO_PLL>;
+	assigned-clock-rates = <333330000>, <125000000>;
 };
 
 &ocotp {
diff --git a/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi b/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi
index a1a7e6a0571bda..a68ffd04ee0842 100644
--- a/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx91_93_common.dtsi
@@ -100,13 +100,19 @@
 
 	mqs1: mqs1 {
 		compatible = "fsl,imx93-mqs";
+		clocks = <&clk IMX93_CLK_MQS1_GATE>;
+		clock-names = "mclk";
 		gpr = <&aonmix_ns_gpr>;
+		#sound-dai-cells = <0>;
 		status = "disabled";
 	};
 
 	mqs2: mqs2 {
 		compatible = "fsl,imx93-mqs";
+		clocks = <&clk IMX93_CLK_MQS2_GATE>;
+		clock-names = "mclk";
 		gpr = <&wakeupmix_gpr>;
+		#sound-dai-cells = <0>;
 		status = "disabled";
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
index f48e7947cf6473..164b5c17bfe128 100644
--- a/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
@@ -18,6 +18,7 @@
 		mmc0 = &usdhc1;
 		mmc1 = &usdhc2;
 		rtc0 = &bbnsm_rtc;
+		rtc1 = &pcf2131;
 		serial0 = &lpuart1;
 		serial1 = &lpuart2;
 		serial2 = &lpuart3;
@@ -67,7 +68,7 @@
 			no-map;
 		};
 
-		vdevbuffer: vdevbuffer@a4020000 {
+		vdev0buffer: vdev0buffer@a4020000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0xa4020000 0 0x100000>;
 			no-map;
@@ -170,7 +171,7 @@
 	mboxes = <&mu1 0 1>,
 		 <&mu1 1 1>,
 		 <&mu1 3 1>;
-	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+	memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts b/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts
index c6db9c85f2ac50..d3312093f996e6 100644
--- a/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-evk.dts
@@ -16,6 +16,38 @@
 		mmc2 = &usdhc3;
 	};
 
+	amp_mqs1: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		sound-name-prefix = "MQS AMP";
+		VCC-supply = <&reg_3p3v>;
+	};
+
+	pdm_mqs1_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&adp5585 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		mqs1_fun: mqs1-grp {
+			mux-states = <&pdm_mqs1_mux 1>;
+		};
+
+		pdm_fun: pdm-grp {
+			mux-states = <&pdm_mqs1_mux 0>;
+		};
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "+V3.3_SW";
+	};
+
 	reg_m2_pwr: regulator-m2-pwr {
 		compatible = "regulator-fixed";
 		regulator-name = "M.2-power";
@@ -52,7 +84,7 @@
 		#sound-dai-cells = <1>;
 	};
 
-	sound-bt-sco {
+	mqs1_conflict_card: sound-bt-sco {
 		compatible = "simple-audio-card";
 		simple-audio-card,name = "bt-sco-audio";
 		simple-audio-card,format = "dsp_a";
@@ -86,9 +118,9 @@
 	};
 };
 
-&micfil {
+mqs1_conflict_dev: &micfil {
 	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-0 = <&pinctrl_pdm>, <&pdm_fun>;
 	pinctrl-1 = <&pinctrl_pdm_sleep>;
 	assigned-clocks = <&clk IMX93_CLK_PDM>;
 	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
@@ -96,6 +128,11 @@
 	status = "okay";
 };
 
+&mqs1 {
+	pinctrl-0 = <&pinctrl_mqs1>, <&mqs1_fun>;
+	pinctrl-names = "default";
+};
+
 &pcal6524 {
 	m2-pcm-level-shifter-hog {
 		gpio-hog;
@@ -108,6 +145,10 @@
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&pinctrl_sai1>;
 	pinctrl-1 = <&pinctrl_sai1_sleep>;
+	clocks = <&clk IMX93_CLK_SAI1_IPG>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_SAI1_GATE>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_DUMMY>, <&clk IMX93_CLK_AUDIO_PLL>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k";
 	assigned-clocks = <&clk IMX93_CLK_SAI1>;
 	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
 	assigned-clock-rates = <12288000>;
@@ -131,6 +172,13 @@
 };
 
 &iomuxc {
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			MX93_PAD_PDM_CLK__MQS1_LEFT			0x31e
+			MX93_PAD_PDM_BIT_STREAM0__MQS1_RIGHT		0x31e
+		>;
+	};
+
 	pinctrl_pdm: pdmgrp {
 		fsl,pins = <
 			MX93_PAD_PDM_CLK__PDM_CLK			0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-frdm-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-11x11-frdm-common.dtsi
new file mode 100644
index 00000000000000..33ed26aafde886
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-frdm-common.dtsi
@@ -0,0 +1,681 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+#include <dt-bindings/usb/pd.h>
+
+/ {
+	aliases {
+		can0 = &flexcan2;
+		ethernet0 = &fec;
+		ethernet1 = &eqos;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		i2c2 = &lpi2c3;
+		mmc0 = &usdhc1; /* EMMC */
+		mmc1 = &usdhc2; /* uSD */
+		rtc0 = &pcf2131;
+		rtc1 = &bbnsm_rtc;
+		serial0 = &lpuart1;
+		serial4 = &lpuart5;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	flexcan2_phy: can-phy {
+		compatible = "nxp,tja1051";
+		#phy-cells = <0>;
+		max-bitrate = <5000000>;
+		silent-gpios = <&pcal6524 23 GPIO_ACTIVE_HIGH>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		button-k2 {
+			label = "Button K2";
+			linux,code = <BTN_1>;
+			gpios = <&pcal6524 5 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
+			interrupt-parent = <&pcal6524>;
+			interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
+		};
+
+		button-k3 {
+			label = "Button K3";
+			linux,code = <BTN_2>;
+			gpios = <&pcal6524 6 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
+			interrupt-parent = <&pcal6524>;
+			interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+		};
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		off-on-delay-us = <12000>;
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		pinctrl-names = "default";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "VSD_3V3";
+		vin-supply = <&buck4>;
+		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0 0x80000000 0 0x30000000>;
+			reusable;
+			size = <0 0x10000000>;
+			linux,cma-default;
+		};
+
+		rsc_table: rsc-table@2021e000 {
+			reg = <0 0x2021e000 0 0x1000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@a4000000 {
+			reg = <0 0xa4000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@a4008000 {
+			reg = <0 0xa4008000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring0: vdev1vring0@a4010000 {
+			reg = <0 0xa4010000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring1: vdev1vring1@a4018000 {
+			reg = <0 0xa4018000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@a4020000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa4020000 0 0x100000>;
+			no-map;
+		};
+	};
+};
+
+&adc1 {
+	vref-supply = <&buck5>;
+	status = "okay";
+};
+
+&mu1 {
+	status = "okay";
+};
+
+&cm33 {
+	mboxes = <&mu1 0 1>,
+		 <&mu1 1 1>,
+		 <&mu1 3 1>;
+	mbox-names = "tx", "rx", "rxdb";
+	memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
+			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
+	status = "okay";
+};
+
+&eqos {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_eqos>;
+	pinctrl-1 = <&pinctrl_eqos_sleep>;
+	phy-handle = <&ethphy1>;
+	phy-mode = "rgmii-id";
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-frequency = <5000000>;
+
+		ethphy1: ethernet-phy@1 {
+			reg = <1>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <80000>;
+			reset-gpios = <&pcal6524 15 GPIO_ACTIVE_LOW>;
+			realtek,clkout-disable;
+		};
+	};
+};
+
+&fec {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_fec>;
+	pinctrl-1 = <&pinctrl_fec_sleep>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy2>;
+	fsl,magic-packet;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-frequency = <5000000>;
+
+		ethphy2: ethernet-phy@2 {
+			reg = <2>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <80000>;
+			reset-gpios = <&pcal6524 16 GPIO_ACTIVE_LOW>;
+			realtek,clkout-disable;
+		};
+	};
+};
+
+&flexcan2 {
+	phys = <&flexcan2_phy>;
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	pinctrl-1 = <&pinctrl_flexcan2_sleep>;
+	pinctrl-names = "default", "sleep";
+	status = "okay";
+};
+
+&lpi2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&lpi2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c2>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pcal6524: gpio@22 {
+		compatible = "nxp,pcal6524";
+		reg = <0x22>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		pinctrl-0 = <&pinctrl_pcal6524>;
+		pinctrl-names = "default";
+		/* does not boot with supplier set, because it is the bucks interrupt parent */
+		/* vcc-supply = <&buck4>; */
+	};
+
+	pmic@25 {
+		compatible = "nxp,pca9451a";
+		reg = <0x25>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
+
+		regulators {
+
+			buck1: BUCK1 {
+				regulator-name = "VDD_SOC_0V8";
+				regulator-min-microvolt = <610000>;
+				regulator-max-microvolt = <950000>;
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck2: BUCK2 {
+				regulator-name = "LPD4_x_VDDQ_0V6";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <670000>;
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck4: BUCK4 {
+				regulator-name = "VDD_3V3";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck5: BUCK5 {
+				regulator-name = "VDD_1V8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			buck6: BUCK6 {
+				regulator-name = "LPD4_x_VDD2_1V1";
+				regulator-min-microvolt = <1060000>;
+				regulator-max-microvolt = <1140000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo1: LDO1 {
+				regulator-name = "NVCC_BBSM_1V8";
+				regulator-min-microvolt = <1620000>;
+				regulator-max-microvolt = <1980000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo4: LDO4 {
+				regulator-name = "VDD_ANA_0V8";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <840000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+
+			ldo5: LDO5 {
+				regulator-name = "NVCC_SD";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+
+	eeprom: eeprom@50 {
+		compatible = "atmel,24c256";
+		reg = <0x50>;
+		pagesize = <64>;
+		vcc-supply = <&buck4>;
+	};
+};
+
+&lpi2c3 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c3>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ptn5110: tcpc@50 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x50>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+
+		typec1_con: connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB-C";
+			op-sink-microwatt = <15000000>;
+			power-role = "dual";
+			self-powered;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+					PDO_VAR(5000, 20000, 3000)>;
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			try-power-role = "sink";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					typec1_dr_sw: endpoint {
+						remote-endpoint = <&usb1_drd_sw>;
+					};
+				};
+			};
+		};
+	};
+
+	pcf2131: rtc@53 {
+		compatible = "nxp,pcf2131";
+		reg = <0x53>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&lpuart1 { /* console */
+	pinctrl-0 = <&pinctrl_uart1>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&lpuart5 {
+	pinctrl-0 = <&pinctrl_uart5>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	uart-has-rtscts;
+
+	bluetooth: bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&usbotg1 {
+	adp-disable;
+	disable-over-current;
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usbotg2 {
+	disable-over-current;
+	dr_mode = "host";
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	status = "okay";
+};
+
+&usdhc1 {
+	bus-width = <8>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	vmmc-supply = <&buck4>;
+	status = "okay";
+};
+
+&usdhc2 {
+	bus-width = <4>;
+	cd-gpios = <&gpio3 00 GPIO_ACTIVE_LOW>;
+	no-mmc;
+	no-sdio;
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	status = "okay";
+};
+
+&wdog3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&iomuxc {
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX93_PAD_ENET1_MDC__ENET_QOS_MDC			0x57e
+			MX93_PAD_ENET1_MDIO__ENET_QOS_MDIO			0x57e
+			MX93_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x57e
+			MX93_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x57e
+			MX93_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x57e
+			MX93_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x57e
+			MX93_PAD_ENET1_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x58e
+			MX93_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x57e
+			MX93_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x57e
+			MX93_PAD_ENET1_TD1__ENET_QOS_RGMII_TD1			0x57e
+			MX93_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x57e
+			MX93_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x57e
+			MX93_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x58e
+			MX93_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x57e
+		>;
+	};
+
+	pinctrl_eqos_sleep: eqossleepgrp {
+		fsl,pins = <
+			MX93_PAD_ENET1_MDC__GPIO4_IO00			0x31e
+			MX93_PAD_ENET1_MDIO__GPIO4_IO01			0x31e
+			MX93_PAD_ENET1_RD0__GPIO4_IO10			0x31e
+			MX93_PAD_ENET1_RD1__GPIO4_IO11			0x31e
+			MX93_PAD_ENET1_RD2__GPIO4_IO12			0x31e
+			MX93_PAD_ENET1_RD3__GPIO4_IO13			0x31e
+			MX93_PAD_ENET1_RXC__GPIO4_IO09			0x31e
+			MX93_PAD_ENET1_RX_CTL__GPIO4_IO08		0x31e
+			MX93_PAD_ENET1_TD0__GPIO4_IO05			0x31e
+			MX93_PAD_ENET1_TD1__GPIO4_IO04			0x31e
+			MX93_PAD_ENET1_TD2__GPIO4_IO03			0x31e
+			MX93_PAD_ENET1_TD3__GPIO4_IO02			0x31e
+			MX93_PAD_ENET1_TXC__GPIO4_IO07			0x31e
+			MX93_PAD_ENET1_TX_CTL__GPIO4_IO06		0x31e
+		>;
+	};
+
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_MDC__ENET1_MDC			0x57e
+			MX93_PAD_ENET2_MDIO__ENET1_MDIO			0x57e
+			MX93_PAD_ENET2_RD0__ENET1_RGMII_RD0		0x57e
+			MX93_PAD_ENET2_RD1__ENET1_RGMII_RD1		0x57e
+			MX93_PAD_ENET2_RD2__ENET1_RGMII_RD2		0x57e
+			MX93_PAD_ENET2_RD3__ENET1_RGMII_RD3		0x57e
+			MX93_PAD_ENET2_RXC__ENET1_RGMII_RXC		0x58e
+			MX93_PAD_ENET2_RX_CTL__ENET1_RGMII_RX_CTL	0x57e
+			MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x57e
+			MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x57e
+			MX93_PAD_ENET2_TD2__ENET1_RGMII_TD2		0x57e
+			MX93_PAD_ENET2_TD3__ENET1_RGMII_TD3		0x57e
+			MX93_PAD_ENET2_TXC__ENET1_RGMII_TXC		0x58e
+			MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x57e
+		>;
+	};
+
+	pinctrl_fec_sleep: fecsleepgrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_MDC__GPIO4_IO14			0x51e
+			MX93_PAD_ENET2_MDIO__GPIO4_IO15			0x51e
+			MX93_PAD_ENET2_RD0__GPIO4_IO24			0x51e
+			MX93_PAD_ENET2_RD1__GPIO4_IO25			0x51e
+			MX93_PAD_ENET2_RD2__GPIO4_IO26			0x51e
+			MX93_PAD_ENET2_RD3__GPIO4_IO27			0x51e
+			MX93_PAD_ENET2_RXC__GPIO4_IO23			0x51e
+			MX93_PAD_ENET2_RX_CTL__GPIO4_IO22		0x51e
+			MX93_PAD_ENET2_TD0__GPIO4_IO19			0x51e
+			MX93_PAD_ENET2_TD1__GPIO4_IO18			0x51e
+			MX93_PAD_ENET2_TD2__GPIO4_IO17			0x51e
+			MX93_PAD_ENET2_TD3__GPIO4_IO16			0x51e
+			MX93_PAD_ENET2_TXC__GPIO4_IO21			0x51e
+			MX93_PAD_ENET2_TX_CTL__GPIO4_IO20		0x51e
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO25__CAN2_TX			0x139e
+			MX93_PAD_GPIO_IO27__CAN2_RX			0x139e
+		>;
+	};
+
+	pinctrl_flexcan2_sleep: flexcan2sleepgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO25__GPIO2_IO25			0x31e
+			MX93_PAD_GPIO_IO27__GPIO2_IO27			0x31e
+		>;
+	};
+
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <
+			MX93_PAD_I2C1_SCL__LPI2C1_SCL			0x40000b9e
+			MX93_PAD_I2C1_SDA__LPI2C1_SDA			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c2: lpi2c2grp {
+		fsl,pins = <
+			MX93_PAD_I2C2_SCL__LPI2C2_SCL			0x40000b9e
+			MX93_PAD_I2C2_SDA__LPI2C2_SDA			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c3: lpi2c3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO28__LPI2C3_SDA			0x40000b9e
+			MX93_PAD_GPIO_IO29__LPI2C3_SCL			0x40000b9e
+		>;
+	};
+
+	pinctrl_pcal6524: pcal6524grp {
+		fsl,pins = <
+			MX93_PAD_CCM_CLKO2__GPIO3_IO27			0x31e
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_RESET_B__GPIO3_IO07		0x31e
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX93_PAD_UART1_RXD__LPUART1_RX			0x31e
+			MX93_PAD_UART1_TXD__LPUART1_TX			0x31e
+		>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			MX93_PAD_DAP_TDO_TRACESWO__LPUART5_TX		0x31e
+			MX93_PAD_DAP_TDI__LPUART5_RX			0x31e
+			MX93_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B		0x31e
+			MX93_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B		0x31e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x1582
+			MX93_PAD_SD1_CMD__USDHC1_CMD		0x40001382
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x40001382
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x40001382
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x40001382
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x40001382
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x40001382
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x40001382
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x40001382
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x40001382
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x1582
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x158e
+			MX93_PAD_SD1_CMD__USDHC1_CMD		0x4000138e
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000138e
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x4000138e
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x4000138e
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x4000138e
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x4000138e
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x4000138e
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x4000138e
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x4000138e
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x158e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x15fe
+			MX93_PAD_SD1_CMD__USDHC1_CMD		0x400013fe
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x15fe
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x31e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio_sleep: usdhc2gpiosleepgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x51e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x1582
+			MX93_PAD_SD2_CMD__USDHC2_CMD		0x40001382
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x40001382
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x40001382
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x40001382
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x40001382
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x158e
+			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000138e
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000138e
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000138e
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x4000138e
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x4000138e
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	/* need to config the SION for data and cmd pad, refer to ERR052021 */
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x15fe
+			MX93_PAD_SD2_CMD__USDHC2_CMD		0x400013fe
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_sleep: usdhc2-sleepgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__GPIO3_IO01		0x51e
+			MX93_PAD_SD2_CMD__GPIO3_IO02		0x51e
+			MX93_PAD_SD2_DATA0__GPIO3_IO03		0x51e
+			MX93_PAD_SD2_DATA1__GPIO3_IO04		0x51e
+			MX93_PAD_SD2_DATA2__GPIO3_IO05		0x51e
+			MX93_PAD_SD2_DATA3__GPIO3_IO06		0x51e
+			MX93_PAD_SD2_VSELECT__GPIO3_IO19	0x51e
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX93_PAD_WDOG_ANY__WDOG1_WDOG_ANY	0x31e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-frdm.dts b/arch/arm64/boot/dts/freescale/imx93-11x11-frdm.dts
index bd14ba28690c08..6e8351d1201605 100644
--- a/arch/arm64/boot/dts/freescale/imx93-11x11-frdm.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-frdm.dts
@@ -1,71 +1,14 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
 /dts-v1/;
 
-#include <dt-bindings/usb/pd.h>
 #include "imx93.dtsi"
+#include "imx93-11x11-frdm-common.dtsi"
 
 / {
 	compatible = "fsl,imx93-11x11-frdm", "fsl,imx93";
 	model = "NXP i.MX93 11X11 FRDM board";
 
-	aliases {
-		can0 = &flexcan2;
-		ethernet0 = &fec;
-		ethernet1 = &eqos;
-		i2c0 = &lpi2c1;
-		i2c1 = &lpi2c2;
-		i2c2 = &lpi2c3;
-		mmc0 = &usdhc1; /* EMMC */
-		mmc1 = &usdhc2; /* uSD */
-		rtc0 = &pcf2131;
-		serial0 = &lpuart1;
-		serial4 = &lpuart5;
-	};
-
-	chosen {
-		stdout-path = &lpuart1;
-	};
-
-	flexcan2_phy: can-phy {
-		compatible = "nxp,tja1051";
-		#phy-cells = <0>;
-		max-bitrate = <5000000>;
-		silent-gpios = <&pcal6524 23 GPIO_ACTIVE_HIGH>;
-	};
-
-	gpio-keys {
-		compatible = "gpio-keys";
-
-		button-k2 {
-			label = "Button K2";
-			linux,code = <BTN_1>;
-			gpios = <&pcal6524 5 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
-			interrupt-parent = <&pcal6524>;
-			interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
-		};
-
-		button-k3 {
-			label = "Button K3";
-			linux,code = <BTN_2>;
-			gpios = <&pcal6524 6 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
-			interrupt-parent = <&pcal6524>;
-			interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
-		};
-	};
-
-	reg_usdhc2_vmmc: regulator-usdhc2 {
-		compatible = "regulator-fixed";
-		off-on-delay-us = <12000>;
-		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
-		pinctrl-names = "default";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-name = "VSD_3V3";
-		vin-supply = <&buck4>;
-		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-	};
-
 	reg_usdhc3_vmmc: regulator-usdhc3 {
 		compatible = "regulator-fixed";
 		regulator-name = "VPCIe_3V3";
@@ -76,51 +19,6 @@
 		enable-active-high;
 	};
 
-	reserved-memory {
-		ranges;
-		#address-cells = <2>;
-		#size-cells = <2>;
-
-		linux,cma {
-			compatible = "shared-dma-pool";
-			alloc-ranges = <0 0x80000000 0 0x30000000>;
-			reusable;
-			size = <0 0x10000000>;
-			linux,cma-default;
-		};
-
-		rsc_table: rsc-table@2021e000 {
-			reg = <0 0x2021e000 0 0x1000>;
-			no-map;
-		};
-
-		vdev0vring0: vdev0vring0@a4000000 {
-			reg = <0 0xa4000000 0 0x8000>;
-			no-map;
-		};
-
-		vdev0vring1: vdev0vring1@a4008000 {
-			reg = <0 0xa4008000 0 0x8000>;
-			no-map;
-		};
-
-		vdev1vring0: vdev1vring0@a4010000 {
-			reg = <0 0xa4010000 0 0x8000>;
-			no-map;
-		};
-
-		vdev1vring1: vdev1vring1@a4018000 {
-			reg = <0 0xa4018000 0 0x8000>;
-			no-map;
-		};
-
-		vdevbuffer: vdevbuffer@a4020000 {
-			compatible = "shared-dma-pool";
-			reg = <0 0xa4020000 0 0x100000>;
-			no-map;
-		};
-	};
-
 	sound-mqs {
 		compatible = "fsl,imx-audio-mqs";
 		model = "mqs-audio";
@@ -134,274 +32,26 @@
 	};
 };
 
-&adc1 {
-	vref-supply = <&buck5>;
-	status = "okay";
-};
-
-&mu1 {
-	status = "okay";
-};
-
-&cm33 {
-	mboxes = <&mu1 0 1>,
-		 <&mu1 1 1>,
-		 <&mu1 3 1>;
-	mbox-names = "tx", "rx", "rxdb";
-	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
-			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
-	status = "okay";
-};
-
-&eqos {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_eqos>;
-	pinctrl-1 = <&pinctrl_eqos_sleep>;
-	phy-handle = <&ethphy1>;
-	phy-mode = "rgmii-id";
-	status = "okay";
-
-	mdio {
-		compatible = "snps,dwmac-mdio";
-		#address-cells = <1>;
-		#size-cells = <0>;
-		clock-frequency = <5000000>;
-
-		ethphy1: ethernet-phy@1 {
-			reg = <1>;
-			reset-assert-us = <10000>;
-			reset-deassert-us = <80000>;
-			reset-gpios = <&pcal6524 15 GPIO_ACTIVE_LOW>;
-			realtek,clkout-disable;
-		};
-	};
-};
-
-&fec {
-	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&pinctrl_fec>;
-	pinctrl-1 = <&pinctrl_fec_sleep>;
-	phy-mode = "rgmii-id";
-	phy-handle = <&ethphy2>;
-	fsl,magic-packet;
-	status = "okay";
-
-	mdio {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		clock-frequency = <5000000>;
-
-		ethphy2: ethernet-phy@2 {
-			reg = <2>;
-			reset-assert-us = <10000>;
-			reset-deassert-us = <80000>;
-			reset-gpios = <&pcal6524 16 GPIO_ACTIVE_LOW>;
-			realtek,clkout-disable;
-		};
-	};
-};
-
-&flexcan2 {
-	phys = <&flexcan2_phy>;
-	pinctrl-0 = <&pinctrl_flexcan2>;
-	pinctrl-1 = <&pinctrl_flexcan2_sleep>;
-	pinctrl-names = "default", "sleep";
-	status = "okay";
+&bluetooth {
+	device-wakeup-gpios = <&pcal6408 3 GPIO_ACTIVE_HIGH>;
+	reset-gpios = <&pcal6524 19 GPIO_ACTIVE_LOW>;
+	vcc-supply = <&reg_usdhc3_vmmc>;
 };
 
 &lpi2c1 {
-	clock-frequency = <400000>;
-	pinctrl-0 = <&pinctrl_lpi2c1>;
-	pinctrl-names = "default";
-	status = "okay";
-
 	pcal6408: gpio@20 {
 		compatible = "nxp,pcal6408";
 		reg = <0x20>;
 		#gpio-cells = <2>;
 		gpio-controller;
 		reset-gpios = <&pcal6524 20 GPIO_ACTIVE_LOW>;
-	};
-};
-
-&lpi2c2 {
-	clock-frequency = <400000>;
-	pinctrl-0 = <&pinctrl_lpi2c2>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	pcal6524: gpio@22 {
-		compatible = "nxp,pcal6524";
-		reg = <0x22>;
-		#interrupt-cells = <2>;
-		interrupt-controller;
-		interrupt-parent = <&gpio3>;
-		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
-		#gpio-cells = <2>;
-		gpio-controller;
-		pinctrl-0 = <&pinctrl_pcal6524>;
-		pinctrl-names = "default";
-		/* does not boot with supplier set, because it is the bucks interrupt parent */
-		/* vcc-supply = <&buck4>; */
-	};
-
-	pmic@25 {
-		compatible = "nxp,pca9451a";
-		reg = <0x25>;
-		interrupt-parent = <&pcal6524>;
-		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
-
-		regulators {
-
-			buck1: BUCK1 {
-				regulator-name = "VDD_SOC_0V8";
-				regulator-min-microvolt = <610000>;
-				regulator-max-microvolt = <950000>;
-				regulator-always-on;
-				regulator-boot-on;
-				regulator-ramp-delay = <3125>;
-			};
-
-			buck2: BUCK2 {
-				regulator-name = "LPD4_x_VDDQ_0V6";
-				regulator-min-microvolt = <600000>;
-				regulator-max-microvolt = <670000>;
-				regulator-always-on;
-				regulator-boot-on;
-				regulator-ramp-delay = <3125>;
-			};
-
-			buck4: BUCK4 {
-				regulator-name = "VDD_3V3";
-				regulator-min-microvolt = <3300000>;
-				regulator-max-microvolt = <3300000>;
-				regulator-always-on;
-				regulator-boot-on;
-			};
-
-			buck5: BUCK5 {
-				regulator-name = "VDD_1V8";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <1800000>;
-				regulator-always-on;
-				regulator-boot-on;
-			};
-
-			buck6: BUCK6 {
-				regulator-name = "LPD4_x_VDD2_1V1";
-				regulator-min-microvolt = <1060000>;
-				regulator-max-microvolt = <1140000>;
-				regulator-always-on;
-				regulator-boot-on;
-			};
-
-			ldo1: LDO1 {
-				regulator-name = "NVCC_BBSM_1V8";
-				regulator-min-microvolt = <1620000>;
-				regulator-max-microvolt = <1980000>;
-				regulator-always-on;
-				regulator-boot-on;
-			};
-
-			ldo4: LDO4 {
-				regulator-name = "VDD_ANA_0V8";
-				regulator-min-microvolt = <800000>;
-				regulator-max-microvolt = <840000>;
-				regulator-always-on;
-				regulator-boot-on;
-			};
-
-			ldo5: LDO5 {
-				regulator-name = "NVCC_SD";
-				regulator-min-microvolt = <1800000>;
-				regulator-max-microvolt = <3300000>;
-				regulator-always-on;
-				regulator-boot-on;
-			};
-		};
-	};
-
-	eeprom: eeprom@50 {
-		compatible = "atmel,24c256";
-		reg = <0x50>;
-		pagesize = <64>;
-		vcc-supply = <&buck4>;
-	};
-};
-
-&lpi2c3 {
-	clock-frequency = <400000>;
-	pinctrl-0 = <&pinctrl_lpi2c3>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	ptn5110: tcpc@50 {
-		compatible = "nxp,ptn5110", "tcpci";
-		reg = <0x50>;
-		interrupt-parent = <&gpio3>;
-		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
-
-		typec1_con: connector {
-			compatible = "usb-c-connector";
-			data-role = "dual";
-			label = "USB-C";
-			op-sink-microwatt = <15000000>;
-			power-role = "dual";
-			self-powered;
-			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
-					PDO_VAR(5000, 20000, 3000)>;
-			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
-			try-power-role = "sink";
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				port@0 {
-					reg = <0>;
-
-					typec1_dr_sw: endpoint {
-						remote-endpoint = <&usb1_drd_sw>;
-					};
-				};
-			};
-		};
-	};
-
-	pcf2131: rtc@53 {
-		compatible = "nxp,pcf2131";
-		reg = <0x53>;
-		interrupt-parent = <&pcal6524>;
-		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
-	};
-};
-
-&lpuart1 { /* console */
-	pinctrl-0 = <&pinctrl_uart1>;
-	pinctrl-names = "default";
-	status = "okay";
-};
-
-&lpuart5 {
-	pinctrl-0 = <&pinctrl_uart5>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	uart-has-rtscts;
-
-	bluetooth {
-		compatible = "nxp,88w8987-bt";
-		device-wakeup-gpios = <&pcal6408 3 GPIO_ACTIVE_HIGH>;
-		reset-gpios = <&pcal6524 19 GPIO_ACTIVE_LOW>;
-		vcc-supply = <&reg_usdhc3_vmmc>;
+		vcc-supply = <&buck5>;
 	};
 };
 
 &mqs1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_mqs1>;
-	clocks = <&clk IMX93_CLK_MQS1_GATE>;
-	clock-names = "mclk";
 	status = "okay";
 };
 
@@ -418,57 +68,6 @@
 	status = "okay";
 };
 
-&usbotg1 {
-	adp-disable;
-	disable-over-current;
-	dr_mode = "otg";
-	hnp-disable;
-	srp-disable;
-	usb-role-switch;
-	samsung,picophy-dc-vol-level-adjust = <7>;
-	samsung,picophy-pre-emp-curr-control = <3>;
-	status = "okay";
-
-	port {
-		usb1_drd_sw: endpoint {
-			remote-endpoint = <&typec1_dr_sw>;
-		};
-	};
-};
-
-&usbotg2 {
-	disable-over-current;
-	dr_mode = "host";
-	samsung,picophy-dc-vol-level-adjust = <7>;
-	samsung,picophy-pre-emp-curr-control = <3>;
-	status = "okay";
-};
-
-&usdhc1 {
-	bus-width = <8>;
-	non-removable;
-	pinctrl-0 = <&pinctrl_usdhc1>;
-	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
-	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
-	pinctrl-names = "default", "state_100mhz", "state_200mhz";
-	vmmc-supply = <&buck4>;
-	status = "okay";
-};
-
-&usdhc2 {
-	bus-width = <4>;
-	cd-gpios = <&gpio3 00 GPIO_ACTIVE_LOW>;
-	no-mmc;
-	no-sdio;
-	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
-	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
-	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
-	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>;
-	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
-	vmmc-supply = <&reg_usdhc2_vmmc>;
-	status = "okay";
-};
-
 &usdhc3 {
 	bus-width = <4>;
 	keep-power-in-suspend;
@@ -483,126 +82,7 @@
 	status = "okay";
 };
 
-&wdog3 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_wdog>;
-	fsl,ext-reset-output;
-	status = "okay";
-};
-
 &iomuxc {
-
-	pinctrl_eqos: eqosgrp {
-		fsl,pins = <
-			MX93_PAD_ENET1_MDC__ENET_QOS_MDC			0x57e
-			MX93_PAD_ENET1_MDIO__ENET_QOS_MDIO			0x57e
-			MX93_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x57e
-			MX93_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x57e
-			MX93_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x57e
-			MX93_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x57e
-			MX93_PAD_ENET1_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x58e
-			MX93_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x57e
-			MX93_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x57e
-			MX93_PAD_ENET1_TD1__ENET_QOS_RGMII_TD1			0x57e
-			MX93_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x57e
-			MX93_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x57e
-			MX93_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x58e
-			MX93_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x57e
-		>;
-	};
-
-	pinctrl_eqos_sleep: eqossleepgrp {
-		fsl,pins = <
-			MX93_PAD_ENET1_MDC__GPIO4_IO00			0x31e
-			MX93_PAD_ENET1_MDIO__GPIO4_IO01			0x31e
-			MX93_PAD_ENET1_RD0__GPIO4_IO10			0x31e
-			MX93_PAD_ENET1_RD1__GPIO4_IO11			0x31e
-			MX93_PAD_ENET1_RD2__GPIO4_IO12			0x31e
-			MX93_PAD_ENET1_RD3__GPIO4_IO13			0x31e
-			MX93_PAD_ENET1_RXC__GPIO4_IO09			0x31e
-			MX93_PAD_ENET1_RX_CTL__GPIO4_IO08		0x31e
-			MX93_PAD_ENET1_TD0__GPIO4_IO05			0x31e
-			MX93_PAD_ENET1_TD1__GPIO4_IO04			0x31e
-			MX93_PAD_ENET1_TD2__GPIO4_IO03			0x31e
-			MX93_PAD_ENET1_TD3__GPIO4_IO02			0x31e
-			MX93_PAD_ENET1_TXC__GPIO4_IO07			0x31e
-			MX93_PAD_ENET1_TX_CTL__GPIO4_IO06		0x31e
-		>;
-	};
-
-	pinctrl_fec: fecgrp {
-		fsl,pins = <
-			MX93_PAD_ENET2_MDC__ENET1_MDC			0x57e
-			MX93_PAD_ENET2_MDIO__ENET1_MDIO			0x57e
-			MX93_PAD_ENET2_RD0__ENET1_RGMII_RD0		0x57e
-			MX93_PAD_ENET2_RD1__ENET1_RGMII_RD1		0x57e
-			MX93_PAD_ENET2_RD2__ENET1_RGMII_RD2		0x57e
-			MX93_PAD_ENET2_RD3__ENET1_RGMII_RD3		0x57e
-			MX93_PAD_ENET2_RXC__ENET1_RGMII_RXC		0x58e
-			MX93_PAD_ENET2_RX_CTL__ENET1_RGMII_RX_CTL	0x57e
-			MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x57e
-			MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x57e
-			MX93_PAD_ENET2_TD2__ENET1_RGMII_TD2		0x57e
-			MX93_PAD_ENET2_TD3__ENET1_RGMII_TD3		0x57e
-			MX93_PAD_ENET2_TXC__ENET1_RGMII_TXC		0x58e
-			MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x57e
-		>;
-	};
-
-	pinctrl_fec_sleep: fecsleepgrp {
-		fsl,pins = <
-			MX93_PAD_ENET2_MDC__GPIO4_IO14			0x51e
-			MX93_PAD_ENET2_MDIO__GPIO4_IO15			0x51e
-			MX93_PAD_ENET2_RD0__GPIO4_IO24			0x51e
-			MX93_PAD_ENET2_RD1__GPIO4_IO25			0x51e
-			MX93_PAD_ENET2_RD2__GPIO4_IO26			0x51e
-			MX93_PAD_ENET2_RD3__GPIO4_IO27			0x51e
-			MX93_PAD_ENET2_RXC__GPIO4_IO23			0x51e
-			MX93_PAD_ENET2_RX_CTL__GPIO4_IO22		0x51e
-			MX93_PAD_ENET2_TD0__GPIO4_IO19			0x51e
-			MX93_PAD_ENET2_TD1__GPIO4_IO18			0x51e
-			MX93_PAD_ENET2_TD2__GPIO4_IO17			0x51e
-			MX93_PAD_ENET2_TD3__GPIO4_IO16			0x51e
-			MX93_PAD_ENET2_TXC__GPIO4_IO21			0x51e
-			MX93_PAD_ENET2_TX_CTL__GPIO4_IO20		0x51e
-		>;
-	};
-
-	pinctrl_flexcan2: flexcan2grp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO25__CAN2_TX			0x139e
-			MX93_PAD_GPIO_IO27__CAN2_RX			0x139e
-		>;
-	};
-
-	pinctrl_flexcan2_sleep: flexcan2sleepgrp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO25__GPIO2_IO25			0x31e
-			MX93_PAD_GPIO_IO27__GPIO2_IO27			0x31e
-		>;
-	};
-
-	pinctrl_lpi2c1: lpi2c1grp {
-		fsl,pins = <
-			MX93_PAD_I2C1_SCL__LPI2C1_SCL			0x40000b9e
-			MX93_PAD_I2C1_SDA__LPI2C1_SDA			0x40000b9e
-		>;
-	};
-
-	pinctrl_lpi2c2: lpi2c2grp {
-		fsl,pins = <
-			MX93_PAD_I2C2_SCL__LPI2C2_SCL			0x40000b9e
-			MX93_PAD_I2C2_SDA__LPI2C2_SDA			0x40000b9e
-		>;
-	};
-
-	pinctrl_lpi2c3: lpi2c3grp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO28__LPI2C3_SDA			0x40000b9e
-			MX93_PAD_GPIO_IO29__LPI2C3_SCL			0x40000b9e
-		>;
-	};
-
 	pinctrl_mqs1: mqs1grp {
 		fsl,pins = <
 			MX93_PAD_PDM_CLK__MQS1_LEFT		0x31e
@@ -610,149 +90,7 @@
 		>;
 	};
 
-	pinctrl_pcal6524: pcal6524grp {
-		fsl,pins = <
-			MX93_PAD_CCM_CLKO2__GPIO3_IO27			0x31e
-		>;
-	};
-
-	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_RESET_B__GPIO3_IO07		0x31e
-		>;
-	};
-
-	pinctrl_uart1: uart1grp {
-		fsl,pins = <
-			MX93_PAD_UART1_RXD__LPUART1_RX			0x31e
-			MX93_PAD_UART1_TXD__LPUART1_TX			0x31e
-		>;
-	};
-
-	pinctrl_uart5: uart5grp {
-		fsl,pins = <
-			MX93_PAD_DAP_TDO_TRACESWO__LPUART5_TX		0x31e
-			MX93_PAD_DAP_TDI__LPUART5_RX			0x31e
-			MX93_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B		0x31e
-			MX93_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B		0x31e
-		>;
-	};
-
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc1: usdhc1grp {
-		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x1582
-			MX93_PAD_SD1_CMD__USDHC1_CMD		0x40001382
-			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x40001382
-			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x40001382
-			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x40001382
-			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x40001382
-			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x40001382
-			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x40001382
-			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x40001382
-			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x40001382
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x1582
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x158e
-			MX93_PAD_SD1_CMD__USDHC1_CMD		0x4000138e
-			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000138e
-			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x4000138e
-			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x4000138e
-			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x4000138e
-			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x4000138e
-			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x4000138e
-			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x4000138e
-			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x4000138e
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x158e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x15fe
-			MX93_PAD_SD1_CMD__USDHC1_CMD		0x400013fe
-			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe
-			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe
-			MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe
-			MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe
-			MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe
-			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe
-			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe
-			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x15fe
-		>;
-	};
-
-	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x31e
-		>;
-	};
-
-	pinctrl_usdhc2_gpio_sleep: usdhc2gpiosleepgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CD_B__GPIO3_IO00		0x51e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc2: usdhc2grp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x1582
-			MX93_PAD_SD2_CMD__USDHC2_CMD		0x40001382
-			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x40001382
-			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x40001382
-			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x40001382
-			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x40001382
-			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x158e
-			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000138e
-			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000138e
-			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000138e
-			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x4000138e
-			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x4000138e
-			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
-		>;
-	};
-
-	/* need to config the SION for data and cmd pad, refer to ERR052021 */
-	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x15fe
-			MX93_PAD_SD2_CMD__USDHC2_CMD		0x400013fe
-			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe
-			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe
-			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe
-			MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe
-			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
-		>;
-	};
-
-	pinctrl_usdhc2_sleep: usdhc2-sleepgrp {
-		fsl,pins = <
-			MX93_PAD_SD2_CLK__GPIO3_IO01		0x51e
-			MX93_PAD_SD2_CMD__GPIO3_IO02		0x51e
-			MX93_PAD_SD2_DATA0__GPIO3_IO03		0x51e
-			MX93_PAD_SD2_DATA1__GPIO3_IO04		0x51e
-			MX93_PAD_SD2_DATA2__GPIO3_IO05		0x51e
-			MX93_PAD_SD2_DATA3__GPIO3_IO06		0x51e
-			MX93_PAD_SD2_VSELECT__GPIO3_IO19	0x51e
-		>;
-	};
-
-		/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc3: usdhc3grp {
 		fsl,pins = <
 			MX93_PAD_SD3_CLK__USDHC3_CLK		0x1582
@@ -798,10 +136,4 @@
 			MX93_PAD_SD3_DATA3__GPIO3_IO25		0x31e
 		>;
 	};
-
-	pinctrl_wdog: wdoggrp {
-		fsl,pins = <
-			MX93_PAD_WDOG_ANY__WDOG1_WDOG_ANY	0x31e
-		>;
-	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts b/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts
index ec78c03f4788b8..1081881028b2dc 100644
--- a/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-14x14-evk.dts
@@ -28,6 +28,13 @@
 		serial4 = &lpuart5;
 	};
 
+	amp_mqs1: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&pcal6524_2 1 GPIO_ACTIVE_HIGH>;
+		sound-name-prefix = "MQS AMP";
+		VCC-supply = <&reg_5v>;
+	};
+
 	bt_sco_codec: bt-sco-codec {
 		compatible = "linux,bt-sco";
 		#sound-dai-cells = <1>;
@@ -37,6 +44,25 @@
 		stdout-path = &lpuart1;
 	};
 
+	can1_mqs1_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524_2 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		can1_fun: can1-grp {
+			mux-states = <&can1_mqs1_mux 0>;
+		};
+
+		mqs1_fun: mqs1-grp {
+			mux-states = <&can1_mqs1_mux 1>;
+		};
+	};
+
 	reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
@@ -75,13 +101,20 @@
 			no-map;
 		};
 
-		vdevbuffer: vdevbuffer@a4020000 {
+		vdev0buffer: vdev0buffer@a4020000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0xa4020000 0 0x100000>;
 			no-map;
 		};
 	};
 
+	reg_5v: regulator-5v {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "+V5_SW";
+	};
+
 	reg_can1_stby: regulator-can1-stby {
 		compatible = "regulator-fixed";
 		regulator-name = "can1-stby";
@@ -174,7 +207,7 @@
 		regulator-max-microvolt = <1800000>;
 	};
 
-	sound-bt-sco {
+	mqs1_conflict_card: sound-bt-sco {
 		compatible = "simple-audio-card";
 		simple-audio-card,name = "bt-sco-audio";
 		simple-audio-card,format = "dsp_a";
@@ -222,7 +255,7 @@
 	mboxes = <&mu1 0 1>,
 		 <&mu1 1 1>,
 		 <&mu1 3 1>;
-	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+	memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
 	status = "okay";
 };
@@ -252,9 +285,9 @@
 	};
 };
 
-&flexcan1 {
+mqs1_conflict_dev: &flexcan1 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_flexcan1>;
+	pinctrl-0 = <&pinctrl_flexcan1>, <&can1_fun>;
 	xceiver-supply = <&reg_can1_stby>;
 	status = "okay";
 };
@@ -412,6 +445,11 @@
 	};
 };
 
+&mqs1 {
+	pinctrl-0 = <&pinctrl_mqs1>, <&mqs1_fun>;
+	pinctrl-names = "default";
+};
+
 &mu1 {
 	status = "okay";
 };
@@ -423,6 +461,10 @@
 &sai1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_sai1>;
+	clocks = <&clk IMX93_CLK_SAI1_IPG>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_SAI1_GATE>, <&clk IMX93_CLK_DUMMY>,
+		 <&clk IMX93_CLK_DUMMY>, <&clk IMX93_CLK_AUDIO_PLL>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k";
 	assigned-clocks = <&clk IMX93_CLK_SAI1>;
 	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
 	assigned-clock-rates = <12288000>;
@@ -540,6 +582,13 @@
 		>;
 	};
 
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			MX93_PAD_PDM_CLK__MQS1_LEFT			0x31e
+			MX93_PAD_PDM_BIT_STREAM0__MQS1_RIGHT		0x31e
+		>;
+	};
+
 	pinctrl_pcal6524: pcal6524grp {
 		fsl,pins = <
 			MX93_PAD_CCM_CLKO2__GPIO3_IO27			0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
index 01c11c517986a0..3be5bf4bc1015e 100644
--- a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
@@ -28,6 +28,7 @@
 		mmc0 = &usdhc1;
 		mmc1 = &usdhc2;
 		rtc0 = &bbnsm_rtc;
+		rtc1 = &pcf2131;
 		serial0 = &lpuart1;
 		serial1 = &lpuart2;
 		serial2 = &lpuart3;
@@ -76,7 +77,7 @@
 			no-map;
 		};
 
-		vdevbuffer: vdevbuffer@a4020000 {
+		vdev0buffer: vdev0buffer@a4020000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0xa4020000 0 0x100000>;
 			no-map;
@@ -212,7 +213,7 @@
 	mboxes = <&mu1 0 1>,
 		 <&mu1 1 1>,
 		 <&mu1 3 1>;
-	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+	memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
 			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
 	status = "okay";
 };
@@ -306,7 +307,7 @@
 		};
 	};
 
-	rtc@53 {
+	pcf2131: rtc@53 {
 		compatible = "nxp,pcf2131";
 		reg = <0x53>;
 		interrupt-parent = <&pcal6524>;
diff --git a/arch/arm64/boot/dts/freescale/imx93-imx91-lino-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-common.dtsi
new file mode 100644
index 00000000000000..f8f13f78586094
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-common.dtsi
@@ -0,0 +1,536 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX93/iMX91 SoM
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx93
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx91
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	aliases {
+		can0 = &flexcan1;
+		can1 = &flexcan2;
+		eeprom0 = &som_eeprom;
+		ethernet0 = &eqos;
+		ethernet1 = &fec;
+		i2c0 = &lpi2c2;
+		i2c1 = &lpi2c1;
+		i2c2 = &lpi2c3;
+		i2c3 = &lpi2c4;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		mmc2 = &usdhc3;
+		serial0 = &lpuart6;
+		serial1 = &lpuart7;
+		serial2 = &lpuart1;
+		serial3 = &lpuart2;
+		usb0 = &usbotg1;
+		usb1 = &usbotg2;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	/* Lino SD_1 Power Supply */
+	reg_sd1_vmmc: regulator-sdhci2-vmmc {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sd1_pwr_en>;
+		/* Lino SD_1_PWR_EN (Lino P72) */
+		gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		off-on-delay-us = <100000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "+V3.3_SD";
+		startup-delay-us = <20000>;
+	};
+
+	reg_sd1_vqmmc: regulator-sdhci2-vqmmc {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sd1_vsel>;
+		/* PMIC_SD_2_VSEL */
+		gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "LDO1-VSEL-SD (PMIC)";
+		states = <1800000 0x1>,
+			 <3300000 0x0>;
+		vin-supply = <&reg_sd_3v3_1v8>;
+	};
+
+	reg_usb1_vbus: regulator-usb1-vbus {
+		compatible = "regulator-fixed";
+		/* Lino USB_1_EN (P27) */
+		gpios = <&som_gpio_expander 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "USB_1_EN";
+		startup-delay-us = <20000>;
+	};
+
+	reg_usb2_vbus: regulator-usb2-vbus {
+		compatible = "regulator-fixed";
+		/* Lino USB_2_EN (P61) */
+		gpios = <&som_gpio_expander 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "USB_2_EN";
+		startup-delay-us = <20000>;
+	};
+};
+
+/* Lino ADC_1, ADC_2, ADC_3 and ADC_4 */
+&adc1 {
+	vref-supply = <&reg_1v8>;
+};
+
+/* Lino ETH_1 */
+&eqos {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_mdio>, <&pinctrl_eth1>;
+
+	mdio: mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		status = "disabled";
+	};
+};
+
+/* Lino ETH_2 */
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eth2>;
+};
+
+/* Lino CAN_1 */
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_can1>;
+};
+
+/* Lino CAN_2 */
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_can2>;
+};
+
+&gpio1 {
+	gpio-line-names =
+		"LINO_P18",
+		"LINO_P16",
+		"PMIC_I2C_SCL",
+		"PMIC_I2C_SDA",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"LINO_P65", /* 10 */
+		"LINO_S18",
+		"LINO_S24",
+		"LINO_S20",
+		"LINO_S22",
+		"";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"LINO_P30",
+		"LINO_P32",
+		"LINO_P24",
+		"LINO_P26",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"LINO_P58",
+		"",
+		"",
+		"",
+		"",
+		"LINO_P34",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"LINO_P36",
+		"",
+		"LINO_P38",
+		"",
+		"",
+		"",
+		"LINO_P20",
+		"LINO_P22";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"LINO_P78",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"LINO_P72",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"PMIC_SD_2_VSEL",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"LINO_P40",
+		"LINO_P42",
+		"",
+		"",
+		"", /* 30 */
+		"";
+};
+
+&gpio4 {
+	gpio-line-names =
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"LINO_P46",
+		"LINO_P48",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"LINO_P44",
+		"GPIO_EXP_INT#";
+};
+
+/* Lino I2C_1 */
+&lpi2c1 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	clock-frequency = <100000>;
+	scl-gpios = <&gpio1 0 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	single-master;
+};
+
+/* On-module I2C - PMIC_I2C */
+&lpi2c2 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_pmic_i2c>;
+	pinctrl-1 = <&pinctrl_pmic_i2c_gpio>;
+	clock-frequency = <400000>;
+	scl-gpios = <&gpio1 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	single-master;
+
+	status = "okay";
+
+	som_gpio_expander: gpio@21 {
+		compatible = "nxp,pcal6408";
+		reg = <0x21>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_exp_int>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio4>;
+		interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"Lino_P27", /* Lino USB_1_EN */
+			"Lino_P61", /* Lino USB_2_EN */
+			"TPM_SPI_CS#",
+			"Lino_P29", /* Lino USB_1_OC# */
+			"Lino_P63", /* Lino USB_2_OC# */
+			"PMIC_INT#",
+			"TPM_IRQ#",
+			"Lino_S66";
+	};
+
+	pmic@32 {
+		compatible = "nxp,pf9453";
+		reg = <0x32>;
+		interrupt-parent = <&som_gpio_expander>;
+		interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
+
+		regulators {
+			BUCK1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1100000>;
+				regulator-min-microvolt = <1100000>;
+				regulator-name = "+V1.1_DDR (PMIC BUCK1)";
+			};
+
+			reg_vdd_soc: BUCK2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <950000>;
+				regulator-min-microvolt = <612500>;
+				regulator-name = "+VDD_SOC (PMIC BUCK2)";
+				regulator-ramp-delay = <12500>;
+			};
+
+			reg_1v8: BUCK3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "+V1.8 / +V1.8_ANA (PMIC BUCK3)";
+			};
+
+			reg_3v3: BUCK4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <3300000>;
+				regulator-name = "+V3.3 (PMIC BUCK4)";
+			};
+
+			reg_sd_3v3_1v8: LDO1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "+V3.3_1.8_SD (PMIC LDO1)";
+			};
+
+			LDO2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <800000>;
+				regulator-min-microvolt = <800000>;
+				regulator-name = "+V0.8_ANA (PMIC LDO2)";
+			};
+
+			LDO-SNVS {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "+V1.8_BBSM (PMIC LDO-SNVS)";
+			};
+		};
+	};
+
+	temperature-sensor@48 {
+		compatible = "ti,tmp1075";
+		reg = <0x48>;
+	};
+
+	som_eeprom: eeprom@50 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+/* Lino I2C_2 */
+&lpi2c3 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	clock-frequency = <100000>;
+	scl-gpios = <&gpio2 29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 28 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	single-master;
+};
+
+/* Lino I2C_3 */
+&lpi2c4 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	pinctrl-1 = <&pinctrl_i2c3_gpio>;
+	clock-frequency = <100000>;
+	scl-gpios = <&gpio2 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	single-master;
+};
+
+/* Lino SPI_2 */
+&lpspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2>, <&pinctrl_spi2_cs0_gpio>;
+	cs-gpios = <&gpio1 11 GPIO_ACTIVE_LOW>, <&som_gpio_expander 2 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+
+	tpm@1 {
+		compatible = "infineon,slb9670", "tcg,tpm_tis-spi";
+		reg = <1>;
+		interrupt-parent = <&som_gpio_expander>;
+		interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+		spi-max-frequency = <18500000>;
+	};
+};
+
+/* Lino SPI_1 */
+&lpspi8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1>, <&pinctrl_spi1_cs0_gpio>;
+	cs-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+};
+
+/* Lino UART_3, used as the Linux Console */
+&lpuart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+};
+
+/* Lino UART_4 */
+&lpuart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>;
+};
+
+/* Lino UART_1 */
+&lpuart6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	uart-has-rtscts;
+};
+
+/* Lino UART_2 */
+&lpuart7 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	uart-has-rtscts;
+};
+
+/* Lino I2S_1 */
+&sai3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2s1>;
+	assigned-clocks = <&clk IMX93_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <24576000>;
+};
+
+/* Lino PWM_1 */
+&tpm4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+};
+
+/* Lino PWM_2 */
+&tpm5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+};
+
+/* Lino PWM_3 */
+&tpm6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm3>;
+};
+
+/* Lino USB_1 */
+&usbotg1 {
+	dr_mode = "otg";
+	adp-disable;
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	disable-over-current;
+	role-switch-default-mode = "peripheral";
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+};
+
+/* Lino USB_2 */
+&usbotg2 {
+	disable-over-current;
+	dr_mode = "host";
+	vbus-supply = <&reg_usb2_vbus>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+};
+
+/* On-module eMMC */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_emmc>;
+	pinctrl-1 = <&pinctrl_emmc>;
+	pinctrl-2 = <&pinctrl_emmc>;
+	no-sd;
+	no-sdio;
+	non-removable;
+
+	status = "okay";
+};
+
+/* Lino SD_1 */
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_sd1>, <&pinctrl_sd1_cd>;
+	pinctrl-1 = <&pinctrl_sd1>, <&pinctrl_sd1_cd>;
+	pinctrl-2 = <&pinctrl_sd1>, <&pinctrl_sd1_cd>;
+	cd-gpios = <&gpio3 0 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	vmmc-supply = <&reg_sd1_vmmc>;
+	vqmmc-supply = <&reg_sd1_vqmmc>;
+};
+
+/* Lino SD_2 */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_sd2>;
+	pinctrl-1 = <&pinctrl_sd2>;
+	pinctrl-2 = <&pinctrl_sd2>;
+};
+
+&wdog3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-common.dtsi
new file mode 100644
index 00000000000000..0023e76210cec7
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-common.dtsi
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX93/iMX91 SoM on Lino-Verdin Adapter
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx93
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx91
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/net/ti-dp83867.h>
+
+/ {
+	verdin_i2s1_mclk: clock-24576000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24576000>;
+	};
+
+	verdin_csi1_mclk: clock-25000000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+	};
+
+	connector {
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		id-gpios = <&adapter_gpio_expander 1 GPIO_ACTIVE_HIGH>;
+		self-powered;
+		type = "micro";
+		vbus-supply = <&reg_usb1_vbus>;
+
+		port {
+			usb_1_connector: endpoint {
+				remote-endpoint = <&usb1_drd_sw>;
+			};
+		};
+	};
+
+	verdin_gpio_keys: gpio-keys {
+		compatible = "gpio-keys";
+
+		status = "disabled";
+
+		key-wakeup {
+			debounce-interval = <10>;
+			/* Verdin CTRL_WAKE1_MICO# (SODIMM 252) */
+			gpios = <&adapter_gpio_expander 2 GPIO_ACTIVE_LOW>;
+			label = "Wake-Up";
+			wakeup-source;
+			linux,code = <KEY_WAKEUP>;
+		};
+	};
+};
+
+/* Lino ETH_1 */
+&eqos {
+	phy-handle = <&ethphy1>;
+	phy-mode = "rgmii-id";
+};
+
+/* Lino I2C_1 */
+&lpi2c1 {
+	status = "okay";
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9543";
+		reg = <0x70>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		verdin_i2c_1: i2c@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+		};
+
+		adapter_i2c: i2c@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			adapter_gpio_expander: gpio@21 {
+				compatible = "nxp,pcal6416";
+				reg = <0x21>;
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_gpio_11>;
+				#interrupt-cells = <2>;
+				interrupt-controller;
+				interrupt-parent = <&gpio1>;
+				interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
+				#gpio-cells = <2>;
+				gpio-controller;
+				gpio-line-names =
+					"SODIMM_244", /* Verdin PCIE_1_RESET#    */
+					"SODIMM_161", /* Verdin USB_1_ID         */
+					"SODIMM_252", /* Verdin CTRL_WAKE1_MICO# */
+					"ETH_1_INT#",
+					"CTRL_EN_WIFI",
+					"",
+					"",
+					"",
+					"SODIMM_42", /* Verdin I2S_2_BCLK  */
+					"SODIMM_48", /* Verdin I2S_2_D_IN  */
+					"SODIMM_46", /* Verdin I2S_2_D_OUT */
+					"SODIMM_44", /* Verdin I2S_2_SYNC  */
+					"SODIMM_64", /* Verdin QSPI_1_CS2# */
+					"SODIMM_66", /* Verdin QSPI_1_DQS  */
+					"SODIMM_60", /* Verdin QSPI_1_IO2  */
+					"SODIMM_62"; /* Verdin QSPI_1_IO3  */
+
+				verdin_pcie_1_reset_hog: pcie-1-reset-hog {
+					gpio-hog;
+					/* Verdin PCIE_1_RESET# (SODIMM 244) */
+					gpios = <0 GPIO_ACTIVE_LOW>;
+					line-name = "PCIE_1_RESET#";
+					output-low;
+
+					status = "disabled";
+				};
+			};
+
+			/* Current measurement into module VCC */
+			hwmon@40 {
+				compatible = "ti,ina219";
+				reg = <0x40>;
+				shunt-resistor = <10000>;
+			};
+		};
+	};
+};
+
+/* Lino UART_1 */
+&lpuart6 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+		fw-init-baudrate = <3000000>;
+	};
+};
+
+/* Lino Ethernet MDIO */
+&mdio {
+	status = "okay";
+
+	ethphy1: ethernet-phy@1 {
+		reg = <1>;
+		interrupt-parent = <&adapter_gpio_expander>;
+		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+	};
+};
+
+/* Lino USB_1 */
+&usbotg1 {
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&usb_1_connector>;
+		};
+	};
+};
+
+/* Lino SD_2 */
+&usdhc3 {
+	keep-power-in-suspend;
+	non-removable;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dahlia-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dahlia-common.dtsi
new file mode 100644
index 00000000000000..5f6136204e3074
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dahlia-common.dtsi
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX93/iMX91 SoM on Lino-Verdin Adapter / Dahlia carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx91
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx93
+ * https://www.toradex.com/products/carrier-board/dahlia-carrier-board-kit
+ */
+
+/ {
+	aliases {
+		eeprom1 = &carrier_eeprom;
+	};
+
+	reg_1v8_sw: regulator-1v8-sw {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "On-carrier +V1.8_SW";
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "verdin-wm8904";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"Microphone Jack", "MICBIAS",
+			"IN1L", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&wm8904_1a>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai3>;
+		};
+	};
+};
+
+/* Lino ADC_1, ADC_2, ADC_3 and ADC_4 */
+&adc1 {
+	status = "okay";
+};
+
+&gpio2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1>, <&pinctrl_gpio2>, <&pinctrl_gpio3>,
+		    <&pinctrl_gpio4>, <&pinctrl_gpio5>;
+};
+
+&gpio3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio6>, <&pinctrl_gpio7>;
+};
+
+&gpio4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio8>, <&pinctrl_gpio9>, <&pinctrl_gpio10>;
+};
+
+/* Lino ETH_1 */
+&eqos {
+	status = "okay";
+};
+
+/* Lino CAN_1 */
+&flexcan1 {
+	status = "okay";
+};
+
+/* Lino CAN_2 */
+&flexcan2 {
+	status = "okay";
+};
+
+/* Lino I2C_2 */
+&lpi2c3 {
+	status = "okay";
+};
+
+/* Lino I2C_3 */
+&lpi2c4 {
+	status = "okay";
+};
+
+/* Lino SPI_1 */
+&lpspi8 {
+	status = "okay";
+};
+
+/* Lino UART_3, used as the Linux Console */
+&lpuart1 {
+	status = "okay";
+};
+
+/* Lino UART_4 */
+&lpuart2 {
+	status = "okay";
+};
+
+/* Lino UART_2 */
+&lpuart7 {
+	status = "okay";
+};
+
+/* Lino I2S_1 */
+&sai3 {
+	status = "okay";
+};
+
+/* Lino PWM_1 */
+&tpm4 {
+	status = "okay";
+};
+
+/* Lino PWM_2 */
+&tpm5 {
+	status = "okay";
+};
+
+/* Lino PWM_3 */
+&tpm6 {
+	status = "okay";
+};
+
+/* Lino USB_1 */
+&usbotg1 {
+	status = "okay";
+};
+
+/* Lino USB_2 */
+&usbotg2 {
+	status = "okay";
+};
+
+/* Lino SD_1 */
+&usdhc2 {
+	status = "okay";
+};
+
+/* Verdin CTRL_WAKE1_MICO# */
+&verdin_gpio_keys {
+	status = "okay";
+};
+
+/* Verdin I2C_1 */
+&verdin_i2c_1 {
+	status = "okay";
+
+	wm8904_1a: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		clock-names = "mclk";
+		clocks = <&verdin_i2s1_mclk>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_1v8_sw>;
+		CPVDD-supply = <&reg_1v8_sw>;
+		DBVDD-supply = <&reg_1v8_sw>;
+		DCVDD-supply = <&reg_1v8_sw>;
+		MICVDD-supply = <&reg_1v8_sw>;
+	};
+
+	/* Current measurement into module VCC */
+	hwmon@40 {
+		compatible = "ti,ina219";
+		reg = <0x40>;
+		shunt-resistor = <10000>;
+	};
+
+	temperature-sensor@4f {
+		compatible = "ti,tmp75c";
+		reg = <0x4f>;
+	};
+
+	carrier_eeprom: eeprom@57 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x57>;
+		pagesize = <16>;
+	};
+};
+
+/* Verdin PCIE_1_RESET# */
+&verdin_pcie_1_reset_hog {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dev-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dev-common.dtsi
new file mode 100644
index 00000000000000..1664b380e58e7b
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-imx91-lino-verdin-dev-common.dtsi
@@ -0,0 +1,216 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX93/iMX91 SoM on Lino-Verdin Adapter / Development carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx91
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx93
+ * https://www.toradex.com/products/carrier-board/verdin-development-board-kit
+ */
+
+/ {
+	aliases {
+		eeprom1 = &carrier_eeprom;
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "verdin-nau8822";
+		simple-audio-card,routing =
+			"Headphones", "LHP",
+			"Headphones", "RHP",
+			"Speaker", "LSPK",
+			"Speaker", "RSPK",
+			"Line Out", "AUXOUT1",
+			"Line Out", "AUXOUT2",
+			"LAUX", "Line In",
+			"RAUX", "Line In",
+			"LMICP", "Mic In",
+			"RMICP", "Mic In";
+		simple-audio-card,widgets =
+			"Headphones", "Headphones",
+			"Line Out", "Line Out",
+			"Speaker", "Speaker",
+			"Microphone", "Mic In",
+			"Line", "Line In";
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&nau8822_1a>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai3>;
+		};
+	};
+};
+
+/* Lino ADC_1, ADC_2, ADC_3 and ADC_4 */
+&adc1 {
+	status = "okay";
+};
+
+&gpio2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1>, <&pinctrl_gpio2>, <&pinctrl_gpio3>,
+		    <&pinctrl_gpio4>, <&pinctrl_gpio5>;
+};
+
+&gpio3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio6>, <&pinctrl_gpio7>;
+};
+
+&gpio4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio8>, <&pinctrl_gpio9>, <&pinctrl_gpio10>;
+};
+
+/* Lino ETH_1 */
+&eqos {
+	status = "okay";
+};
+
+/* Lino ETH_2 */
+&fec {
+	phy-handle = <&ethphy2>;
+	phy-mode = "rgmii-id";
+
+	status = "okay";
+};
+
+/* Lino CAN_1 */
+&flexcan1 {
+	status = "okay";
+};
+
+/* Lino CAN_2 */
+&flexcan2 {
+	status = "okay";
+};
+
+/* Lino I2C_2 */
+&lpi2c3 {
+	status = "okay";
+};
+
+/* Lino I2C_3 */
+&lpi2c4 {
+	status = "okay";
+};
+
+/* Lino SPI_1 */
+&lpspi8 {
+	status = "okay";
+};
+
+/* Lino UART_3, used as the Linux Console */
+&lpuart1 {
+	status = "okay";
+};
+
+/* Lino UART_4 */
+&lpuart2 {
+	status = "okay";
+};
+
+/* Lino UART_2 */
+&lpuart7 {
+	status = "okay";
+};
+
+/* Lino Ethernet MDIO */
+&mdio {
+	ethphy2: ethernet-phy@7 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <7>;
+		micrel,led-mode = <0>;
+	};
+};
+
+/* Lino I2S_1 */
+&sai3 {
+	status = "okay";
+};
+
+/* Lino PWM_1 */
+&tpm4 {
+	status = "okay";
+};
+
+/* Lino PWM_2 */
+&tpm5 {
+	status = "okay";
+};
+
+/* Lino PWM_3 */
+&tpm6 {
+	status = "okay";
+};
+
+/* Lino USB_1 */
+&usbotg1 {
+	status = "okay";
+};
+
+/* Lino USB_2 */
+&usbotg2 {
+	status = "okay";
+};
+
+/* Lino SD_1 */
+&usdhc2 {
+	status = "okay";
+};
+
+/* Verdin CTRL_WAKE1_MICO# */
+&verdin_gpio_keys {
+	status = "okay";
+};
+
+/* Verdin I2C_1 */
+&verdin_i2c_1 {
+	status = "okay";
+
+	nau8822_1a: audio-codec@1a {
+		compatible = "nuvoton,nau8822";
+		reg = <0x1a>;
+		clock-names = "mclk";
+		clocks = <&verdin_i2s1_mclk>;
+		#sound-dai-cells = <0>;
+	};
+
+	carrier_gpio_expander: gpio@21 {
+		compatible = "nxp,pcal6416";
+		reg = <0x21>;
+		#gpio-cells = <2>;
+		gpio-controller;
+	};
+
+	/* Current measurement into module VCC */
+	hwmon@40 {
+		compatible = "ti,ina219";
+		reg = <0x40>;
+		shunt-resistor = <10000>;
+	};
+
+	temperature-sensor@4f {
+		compatible = "ti,tmp75c";
+		reg = <0x4f>;
+	};
+
+	carrier_eeprom: eeprom@57 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x57>;
+		pagesize = <16>;
+	};
+};
+
+/* Verdin PCIE_1_RESET# */
+&verdin_pcie_1_reset_hog {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-common.dtsi
new file mode 100644
index 00000000000000..8b0131d4e6e2d7
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-common.dtsi
@@ -0,0 +1,526 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex OSM iMX93/iMX91 SoM
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx93
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx91
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	aliases {
+		can0 = &flexcan1;
+		can1 = &flexcan2;
+		eeprom0 = &som_eeprom;
+		ethernet0 = &eqos;
+		ethernet1 = &fec;
+		i2c0 = &lpi2c2;
+		i2c1 = &lpi2c3;
+		i2c2 = &lpi2c1;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		mmc2 = &usdhc3;
+		rtc0 = &rtc_i2c;
+		rtc1 = &bbnsm_rtc;
+		serial0 = &lpuart1;
+		serial1 = &lpuart6;
+		serial2 = &lpuart7;
+		serial3 = &lpuart2;
+		usb0 = &usbotg1;
+		usb1 = &usbotg2;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	reg_sdio_a_vmmc: regulator-sdhci2-vmmc {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sdio_a_pwr_en>;
+		/* OSM SDIO_A_PWR_EN (OSM D21) */
+		gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		off-on-delay-us = <100000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "SDIO_A_PWR_EN";
+		startup-delay-us = <20000>;
+	};
+
+	reg_sdio_a_vqmmc: regulator-sdhci2-vqmmc {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sdio_a_vsel>;
+		/* PMIC_SD_2_VSEL */
+		gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "LDO1-VSEL-SD (PMIC)";
+		states = <1800000 0x1>,
+			 <3300000 0x0>;
+		vin-supply = <&reg_sd_3v3_1v8>;
+	};
+
+	reg_usb_b_vbus: regulator-usb2-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb_b_en>;
+		/* OSM USB_B_EN (OSM AC20) */
+		gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "USB_B_EN";
+		startup-delay-us = <20000>;
+	};
+};
+
+/* OSM ADC_0, ADC_1 */
+&adc1 {
+	vref-supply = <&reg_1v8>;
+};
+
+/* OSM ETH_A */
+&eqos {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_mdio>, <&pinctrl_eth_a>;
+
+	mdio: mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		status = "disabled";
+	};
+};
+
+/* OSM ETH_B */
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eth_b>;
+};
+
+/* OSM CAN_A */
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_can_a>;
+};
+
+/* OSM CAN_B */
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_can_b>;
+};
+
+&gpio1 {
+	gpio-line-names =
+		"OSM_AA20",
+		"OSM_AA21",
+		"PMIC_I2C_SCL",
+		"PMIC_I2C_SDA",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"OSM_AC20", /* 10 */
+		"OSM_AA23",
+		"",
+		"",
+		"",
+		"";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"OSM_D17",
+		"OSM_E17",
+		"OSM_F17",
+		"OSM_G17",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"OSM_Y15",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"OSM_H17",
+		"",
+		"OSM_J17",
+		"",
+		"",
+		"",
+		"OSM_AA16",
+		"OSM_AA15";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"OSM_J21",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"OSM_D21",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"PMIC_SD_2_VSEL",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"CTRL_INT#",
+		"",
+		"",
+		"", /* 30 */
+		"";
+};
+
+&gpio4 {
+	gpio-line-names =
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"OSM_K17",
+		"OSM_L17",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"OSM_AC16",
+		"OSM_AC15";
+};
+
+/* OSM I2C_B */
+&lpi2c1 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c_b>;
+	pinctrl-1 = <&pinctrl_i2c_b_gpio>;
+	clock-frequency = <100000>;
+	scl-gpios = <&gpio1 0 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	single-master;
+
+	status = "okay";
+
+	rtc_i2c: rtc@32 {
+		compatible = "epson,rx8130";
+		reg = <0x32>;
+	};
+};
+
+/* On-module I2C - PMIC_I2C */
+&lpi2c2 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_pmic_i2c>, <&pinctrl_ctrl_int>;
+	pinctrl-1 = <&pinctrl_pmic_i2c_gpio>, <&pinctrl_ctrl_int>;
+	clock-frequency = <400000>;
+	scl-gpios = <&gpio1 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	single-master;
+
+	status = "okay";
+
+	som_gpio_expander: gpio@21 {
+		compatible = "nxp,pcal6416";
+		reg = <0x21>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"OSM_D19", /* OSM GPIO_B_0 */
+			"OSM_E19", /* OSM GPIO_B_1 */
+			"OSM_F19", /* OSM GPIO_B_2 */
+			"OSM_G19", /* OSM GPIO_B_3 */
+			"OSM_H19", /* OSM GPIO_B_4 */
+			"OSM_J19", /* OSM GPIO_B_5 */
+			"OSM_K19", /* OSM GPIO_B_6 */
+			"OSM_L19", /* OSM GPIO_B_7 */
+			"OSM_D3",  /* OSM GPIO_C_0 */
+			"OSM_D4",  /* OSM GPIO_C_1 */
+			"OSM_E3",  /* OSM GPIO_C_2/CAM_B_PWR   */
+			"OSM_E4",  /* OSM GPIO_C_3/CAM_B_RST#  */
+			"OSM_F3",  /* OSM GPIO_C_4/DISP_VDD_EN */
+			"OSM_F4",  /* OSM GPIO_C_5/DISP_BL_EN  */
+			"OSM_G3",  /* OSM GPIO_C_6/CAM_A_PWR   */
+			"OSM_G4";  /* OSM GPIO_C_7/CAM_A_RST#  */
+	};
+
+	tpm@2e {
+		compatible = "st,st33ktpm2xi2c", "tcg,tpm-tis-i2c";
+		reg = <0x2e>;
+	};
+
+	pmic@32 {
+		compatible = "nxp,pf9453";
+		reg = <0x32>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
+
+		regulators {
+			BUCK1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1100000>;
+				regulator-min-microvolt = <1100000>;
+				regulator-name = "+V1.1_DDR (PMIC BUCK1)";
+			};
+
+			reg_vdd_soc: BUCK2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <950000>;
+				regulator-min-microvolt = <612500>;
+				regulator-name = "+VDD_SOC (PMIC BUCK2)";
+				regulator-ramp-delay = <12500>;
+			};
+
+			reg_1v8: BUCK3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "+V1.8 / +V1.8_ANA (PMIC BUCK3)";
+			};
+
+			reg_3v3: BUCK4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <3300000>;
+				regulator-name = "+V3.3 (PMIC BUCK4)";
+			};
+
+			reg_sd_3v3_1v8: LDO1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "+V3.3_1.8_SD (PMIC LDO1)";
+			};
+
+			LDO2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <800000>;
+				regulator-min-microvolt = <800000>;
+				regulator-name = "+V0.8_ANA (PMIC LDO2)";
+			};
+
+			LDO-SNVS {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "+V1.8_BBSM (PMIC LDO-SNVS)";
+			};
+		};
+	};
+
+	temperature-sensor@48 {
+		compatible = "ti,tmp1075";
+		reg = <0x48>;
+	};
+
+	som_eeprom: eeprom@50 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+/* OSM I2C_A */
+&lpi2c3 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c_a>;
+	pinctrl-1 = <&pinctrl_i2c_a_gpio>;
+	clock-frequency = <100000>;
+	scl-gpios = <&gpio2 29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 28 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	single-master;
+};
+
+/* OSM SPI_B */
+&lpspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi_b>, <&pinctrl_spi_b_cs0_gpio>;
+	cs-gpios = <&gpio1 11 GPIO_ACTIVE_LOW>;
+};
+
+/* OSM SPI_A */
+&lpspi8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi_a>, <&pinctrl_spi_a_cs0_gpio>;
+	cs-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+};
+
+/* OSM UART_CON, used as the Linux Console */
+&lpuart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_con>;
+};
+
+/* OSM UART_C */
+&lpuart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_c>;
+};
+
+/* OSM UART_A */
+&lpuart6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_a>;
+	uart-has-rtscts;
+};
+
+/* OSM UART_B */
+&lpuart7 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_b>;
+	uart-has-rtscts;
+};
+
+/* OSM I2S_A */
+&sai3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2s_a>;
+	assigned-clocks = <&clk IMX93_CLK_SAI3>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <12288000>;
+	fsl,sai-mclk-direction-output;
+};
+
+/* OSM PWM_0 */
+&tpm4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0>;
+};
+
+/* OSM PWM_1 */
+&tpm5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+};
+
+/* OSM PWM_2 */
+&tpm6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+};
+
+/* OSM USB_A */
+&usbotg1 {
+	dr_mode = "otg";
+	adp-disable;
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	disable-over-current;
+	role-switch-default-mode = "peripheral";
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+};
+
+/* OSM USB_B */
+&usbotg2 {
+	disable-over-current;
+	dr_mode = "host";
+	vbus-supply = <&reg_usb_b_vbus>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+};
+
+/* On-module eMMC */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_emmc>;
+	pinctrl-1 = <&pinctrl_emmc>;
+	pinctrl-2 = <&pinctrl_emmc>;
+	no-sd;
+	no-sdio;
+	non-removable;
+
+	status = "okay";
+};
+
+/* OSM SDIO_A */
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_sdio_a>, <&pinctrl_sdio_a_cd>;
+	pinctrl-1 = <&pinctrl_sdio_a>, <&pinctrl_sdio_a_cd>;
+	pinctrl-2 = <&pinctrl_sdio_a>, <&pinctrl_sdio_a_cd>;
+	cd-gpios = <&gpio3 0 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	vmmc-supply = <&reg_sdio_a_vmmc>;
+	vqmmc-supply = <&reg_sdio_a_vqmmc>;
+};
+
+/* OSM SDIO_B */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_sdio_b>;
+	pinctrl-1 = <&pinctrl_sdio_b>;
+	pinctrl-2 = <&pinctrl_sdio_b>;
+	disable-wp;
+};
+
+&wdog3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-dev-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-dev-common.dtsi
new file mode 100644
index 00000000000000..566671b32308d3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-imx91-toradex-osm-dev-common.dtsi
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex OSM iMX93/iMX91 SoM on OSM Development carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx93
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx91
+ * https://www.toradex.com/products/carrier-board/osm-development-board
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/usb/pd.h>
+
+/ {
+	aliases {
+		eeprom1 = &carrier_eeprom;
+	};
+
+	reg_carrier_1p8v: regulator-carrier-1p8v {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "On-carrier V1.8_SW";
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "verdin-wm8904";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"Microphone Jack", "MICBIAS",
+			"IN1L", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&wm8904_1a>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai3>;
+		};
+	};
+};
+
+/* OSM ADC_0, ADC_1 */
+&adc1 {
+	status = "okay";
+};
+
+/* OSM ETH_A */
+&eqos {
+	phy-handle = <&ethphy_a>;
+	phy-mode = "rgmii-id";
+
+	status = "okay";
+};
+
+/* OSM ETH_B */
+&fec {
+	phy-handle = <&ethphy_b>;
+	phy-mode = "rgmii-id";
+
+	status = "okay";
+};
+
+/* OSM CAN_A */
+&flexcan1 {
+	status = "okay";
+};
+
+/* OSM CAN_B */
+&flexcan2 {
+	status = "okay";
+};
+
+&gpio2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio_a1>, <&pinctrl_gpio_a2>, <&pinctrl_gpio_a3>,
+		    <&pinctrl_gpio_a4>, <&pinctrl_gpio_a5>;
+};
+
+&gpio4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio_a6>;
+};
+
+/* OSM I2C_B */
+&lpi2c1 {
+	i2c-mux@70 {
+		compatible = "nxp,pca9548";
+		reg = <0x70>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c_csi: i2c@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c_lvds: i2c@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		/* I2C on DSI connector pin #4 and #6 */
+		i2c_dsi_1: i2c@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		/* I2C on DSI connector pin #52 and #54 */
+		i2c_dsi_2: i2c@3 {
+			reg = <3>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	};
+};
+
+/* OSM I2C_A */
+&lpi2c3 {
+	status = "okay";
+
+	wm8904_1a: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_i2s_a_mclk>;
+		clocks = <&clk IMX93_CLK_SAI3>;
+		clock-names = "mclk";
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_carrier_1p8v>;
+		CPVDD-supply = <&reg_carrier_1p8v>;
+		DBVDD-supply = <&reg_carrier_1p8v>;
+		DCVDD-supply = <&reg_carrier_1p8v>;
+		MICVDD-supply = <&reg_carrier_1p8v>;
+	};
+
+	gpio@20 {
+		compatible = "nxp,pcal6408";
+		reg = <0x20>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"PWR_CTRL_CAN_A", /* 0 */
+			"PWR_CTRL_CAN_B",
+			"PWR_CTRL_USB-HUB",
+			"LVDS_MUX_CTRL",
+			"USB_MUX_CTRL",
+			"I2CA_IOEXP_P5",
+			"I2CA_IOEXP_P6",
+			"I2CA_IOEXP_P7";
+
+		lvds_mux_ctrl: lvds-mux-ctrl-hog {
+			gpio-hog;
+			gpios = <3 GPIO_ACTIVE_HIGH>;
+			line-name = "LVDS_MUX_CTRL";
+			output-high;
+		};
+
+		usb_mux_ctrl: usb-mux-ctrl-hog {
+			gpio-hog;
+			gpios = <4 GPIO_ACTIVE_HIGH>;
+			line-name = "USB_MUX_CTRL";
+			output-low;
+		};
+	};
+
+	hwmon@40 {
+		compatible = "ti,ina219";
+		reg = <0x40>;
+		shunt-resistor = <10000>;
+	};
+
+	temperature-sensor@4f {
+		compatible = "ti,tmp1075";
+		reg = <0x4f>;
+	};
+
+	tcpc@52 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x52>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_a0>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+
+		connector {
+			compatible = "usb-c-connector";
+			label = "USB_A";
+			data-role = "dual";
+			power-role = "dual";
+			try-power-role = "sink";
+			self-powered;
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			sink-pdos = <PDO_FIXED(5000, 0, PDO_FIXED_USB_COMM)>;
+			op-sink-microwatt = <0>;
+
+			port {
+				usb_c_a_conn_hs: endpoint {
+					remote-endpoint = <&usb_a_hs>;
+				};
+			};
+		};
+	};
+
+	carrier_eeprom: eeprom@57 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x57>;
+		pagesize = <16>;
+	};
+};
+
+/* OSM SPI_B */
+&lpspi1 {
+	status = "okay";
+};
+
+/* OSM SPI_A */
+&lpspi8 {
+	status = "okay";
+};
+
+/* OSM UART_CON, used as the Linux Console */
+&lpuart1 {
+	status = "okay";
+};
+
+/* OSM UART_C */
+&lpuart2 {
+	status = "okay";
+};
+
+/* OSM UART_A */
+&lpuart6 {
+	status = "okay";
+};
+
+/* OSM UART_B */
+&lpuart7 {
+	status = "okay";
+};
+
+/* OSM ETH_MDIO */
+&mdio {
+	status = "okay";
+
+	ethphy_a: ethernet-phy@2 {
+		reg = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_a7>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+	};
+
+	ethphy_b: ethernet-phy@3 {
+		reg = <3>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+	};
+};
+
+/* OSM I2S_A */
+&sai3 {
+	status = "okay";
+};
+
+/* OSM PWM_0 */
+&tpm4 {
+	status = "okay";
+};
+
+/* OSM PWM_1 */
+&tpm5 {
+	status = "okay";
+};
+
+/* OSM PWM_2 */
+&tpm6 {
+	status = "okay";
+};
+
+/* OSM USB_A */
+&usbotg1 {
+	status = "okay";
+
+	port {
+		usb_a_hs: endpoint {
+			remote-endpoint = <&usb_c_a_conn_hs>;
+		};
+	};
+};
+
+/* OSM USB_B */
+&usbotg2 {
+	status = "okay";
+};
+
+/* OSM SDIO_A */
+&usdhc2 {
+	status = "okay";
+};
+
+/* OSM SDIO_B */
+&usdhc3 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-kontron-bl-osm-s.dts b/arch/arm64/boot/dts/freescale/imx93-kontron-bl-osm-s.dts
index 4620c070f4d762..0277640d7c4cfa 100644
--- a/arch/arm64/boot/dts/freescale/imx93-kontron-bl-osm-s.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-kontron-bl-osm-s.dts
@@ -158,12 +158,8 @@
 };
 
 &usbotg1 {
-	adp-disable;
-	hnp-disable;
-	srp-disable;
 	disable-over-current;
-	dr_mode = "otg";
-	usb-role-switch;
+	dr_mode = "peripheral";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi
index f881912cde4605..36548d9181695d 100644
--- a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi
@@ -34,29 +34,6 @@
 		regulator-max-microvolt = <3300000>;
 		regulator-name = "VCC_SDIO_A";
 	};
-
-	reg_vdd_carrier: regulator-vdd-carrier {
-		compatible = "regulator-fixed";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_reg_vdd_carrier>;
-		gpio = <&gpio4 29 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-		regulator-always-on;
-		regulator-boot-on;
-		regulator-name = "VDD_CARRIER";
-
-		regulator-state-standby {
-			regulator-on-in-suspend;
-		};
-
-		regulator-state-mem {
-			regulator-off-in-suspend;
-		};
-
-		regulator-state-disk {
-			regulator-off-in-suspend;
-		};
-	};
 };
 
 &flexcan1 { /* OSM-S CAN_A */
@@ -113,7 +90,7 @@
 			  "ETH_A_TXD3", "ETH_A_TXD2", "ETH_A_TXD1", "ETH_A_TXD0",
 			  "ETH_A_TX_EN", "ETH_A_TX_CLK", "ETH_A_RX_CTL", "ETH_A_RX_CLK",
 			  "ETH_A_RXD0", "ETH_A_RXD1", "ETH_A_RXD2", "ETH_A_RXD3",
-			  "GPIO_B_0", "CARRIER_PWR_EN";
+			  "GPIO_B_0", "GPIO_B_2";
 };
 
 &lpi2c1 {
@@ -385,6 +362,7 @@
 	pinctrl_gpio4: gpio4grp {
 		fsl,pins = <
 			MX93_PAD_CCM_CLKO3__GPIO4_IO28			0x31e /* GPIO_B_0 */
+			MX93_PAD_CCM_CLKO4__GPIO4_IO29			0x31e /* GPIO_B_2 */
 		>;
 	};
 
@@ -465,12 +443,6 @@
 		>;
 	};
 
-	pinctrl_reg_vdd_carrier: regvddcarriergrp {
-		fsl,pins = <
-			MX93_PAD_CCM_CLKO4__GPIO4_IO29			0x31e /* CARRIER_PWR_EN */
-		>;
-	};
-
 	pinctrl_rtc: rtcgrp {
 		fsl,pins = <
 			MX93_PAD_SD2_VSELECT__GPIO3_IO19		0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx93-lino-verdin-dahlia.dts b/arch/arm64/boot/dts/freescale/imx93-lino-verdin-dahlia.dts
new file mode 100644
index 00000000000000..c708a2e18c9093
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-lino-verdin-dahlia.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX93 SoM on Lino-Verdin Adapter / Dahlia carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx93
+ * https://www.toradex.com/products/carrier-board/dahlia-carrier-board-kit
+ */
+
+/dts-v1/;
+
+#include "imx93.dtsi"
+#include "imx93-lino.dtsi"
+#include "imx93-imx91-lino-verdin-common.dtsi"
+#include "imx93-imx91-lino-verdin-dahlia-common.dtsi"
+
+/ {
+	model = "Toradex Lino iMX93 on Verdin Adapter plus Dahlia Board";
+	compatible = "toradex,lino-imx93-verdin-dahlia",
+		     "toradex,lino-imx93-verdin",
+		     "toradex,lino-imx93",
+		     "fsl,imx93";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-lino-verdin-dev.dts b/arch/arm64/boot/dts/freescale/imx93-lino-verdin-dev.dts
new file mode 100644
index 00000000000000..a79553d94649f2
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-lino-verdin-dev.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX93 SoM on Lino-Verdin Adapter / Development carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx93
+ * https://www.toradex.com/products/carrier-board/verdin-development-board-kit
+ */
+
+/dts-v1/;
+
+#include "imx93.dtsi"
+#include "imx93-lino.dtsi"
+#include "imx93-imx91-lino-verdin-common.dtsi"
+#include "imx93-imx91-lino-verdin-dev-common.dtsi"
+
+/ {
+	model = "Toradex Lino iMX93 on Verdin Adapter plus Development Board";
+	compatible = "toradex,lino-imx93-verdin-dev",
+		     "toradex,lino-imx93-verdin",
+		     "toradex,lino-imx93",
+		     "fsl,imx93";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-lino.dtsi b/arch/arm64/boot/dts/freescale/imx93-lino.dtsi
new file mode 100644
index 00000000000000..cab7f344f2fe96
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-lino.dtsi
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Lino iMX93 SoM
+ *
+ * https://www.toradex.com/computer-on-modules/lino-arm-family/nxp-imx93
+ */
+
+#include "imx93-pinfunc.h"
+#include "imx93-imx91-lino-common.dtsi"
+
+/ {
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		rsc_table: rsc-table@2021e000 {
+			reg = <0 0x2021e000 0 0x1000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@a4000000 {
+			reg = <0 0xa4000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@a4008000 {
+			reg = <0 0xa4008000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring0: vdev1vring0@a4010000 {
+			reg = <0 0xa4010000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring1: vdev1vring1@a4018000 {
+			reg = <0 0xa4018000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@a4020000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa4020000 0 0x100000>;
+			no-map;
+		};
+	};
+};
+
+&cm33 {
+	mbox-names = "tx", "rx", "rxdb";
+	mboxes = <&mu1 0 1>,
+		 <&mu1 1 1>,
+		 <&mu1 3 1>;
+	memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
+			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
+
+	status = "okay";
+};
+
+&mu1 {
+	status = "okay";
+};
+
+&iomuxc {
+	/* Lino CAN_1 */
+	pinctrl_can1: can1grp {
+		fsl,pins = <MX93_PAD_PDM_BIT_STREAM0__CAN1_RX	0x139e>, /* Lino P12 */
+			   <MX93_PAD_PDM_CLK__CAN1_TX		0x139e>; /* Lino P10 */
+	};
+
+	/* Lino CAN_2 */
+	pinctrl_can2: can2grp {
+		fsl,pins = <MX93_PAD_GPIO_IO25__CAN2_TX	0x139e>, /* Lino S28 */
+			   <MX93_PAD_GPIO_IO27__CAN2_RX	0x139e>; /* Lino S30 */
+	};
+
+	/* Lino ETH_1 RGMII */
+	pinctrl_eth1: eqosgrp {
+		fsl,pins = <MX93_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x1fe>, /* Lino P77 */
+			   <MX93_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x1fe>, /* Lino P79 */
+			   <MX93_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x1fe>, /* Lino P81 */
+			   <MX93_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x1fe>, /* Lino P83 */
+			   <MX93_PAD_ENET1_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x1fe>, /* Lino P73 */
+			   <MX93_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x1fe>, /* Lino P75 */
+			   <MX93_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x1fe>, /* Lino P97 */
+			   <MX93_PAD_ENET1_TD1__ENET_QOS_RGMII_TD1			0x1fe>, /* Lino P95 */
+			   <MX93_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x1fe>, /* Lino P93 */
+			   <MX93_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x1fe>, /* Lino P91 */
+			   <MX93_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x1fe>, /* Lino P89 */
+			   <MX93_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x1fe>; /* Lino P85 */
+	};
+
+	/* Lino ETH MDIO */
+	pinctrl_mdio: eqosmdiogrp {
+		fsl,pins = <MX93_PAD_ENET1_MDC__ENET_QOS_MDC	0x1fe>, /* Lino P69 */
+			   <MX93_PAD_ENET1_MDIO__ENET_QOS_MDIO	0x1fe>; /* Lino P67 */
+	};
+
+	/* Lino ETH_2 RGMII */
+	pinctrl_eth2: fecgrp {
+		fsl,pins = <MX93_PAD_ENET2_RD0__ENET1_RGMII_RD0		0x1fe>, /* Lino S77 */
+			   <MX93_PAD_ENET2_RD1__ENET1_RGMII_RD1		0x1fe>, /* Lino S79 */
+			   <MX93_PAD_ENET2_RD2__ENET1_RGMII_RD2		0x1fe>, /* Lino S81 */
+			   <MX93_PAD_ENET2_RD3__ENET1_RGMII_RD3		0x1fe>, /* Lino S83 */
+			   <MX93_PAD_ENET2_RXC__ENET1_RGMII_RXC		0x1fe>, /* Lino S71 */
+			   <MX93_PAD_ENET2_RX_CTL__ENET1_RGMII_RX_CTL	0x1fe>, /* Lino S73 */
+			   <MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x1fe>, /* Lino S99 */
+			   <MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x1fe>, /* Lino S97 */
+			   <MX93_PAD_ENET2_TD2__ENET1_RGMII_TD2		0x1fe>, /* Lino S93 */
+			   <MX93_PAD_ENET2_TD3__ENET1_RGMII_TD3		0x1fe>, /* Lino S91 */
+			   <MX93_PAD_ENET2_TXC__ENET1_RGMII_TXC		0x1fe>, /* Lino S87 */
+			   <MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x1fe>; /* Lino S89 */
+	};
+
+	/* Lino GPIO_11 */
+	pinctrl_gpio_11: gpio1io10grp {
+		fsl,pins = <MX93_PAD_PDM_BIT_STREAM1__GPIO1_IO10	0x139e>; /* Lino P65 */
+	};
+
+	/* Lino SPI_2_CS as GPIO */
+	pinctrl_spi2_cs0_gpio: gpio1io11grp {
+		fsl,pins = <MX93_PAD_SAI1_TXFS__GPIO1_IO11	0x139e>; /* Lino S18 */
+	};
+
+	/* Lino GPIO_1 */
+	pinctrl_gpio1: gpio2io0grp {
+		fsl,pins = <MX93_PAD_GPIO_IO00__GPIO2_IO00	0x139e>; /* Lino P30 */
+	};
+
+	/* Lino GPIO_2 */
+	pinctrl_gpio2: gpio2io1grp {
+		fsl,pins = <MX93_PAD_GPIO_IO01__GPIO2_IO01	0x139e>; /* Lino P32 */
+	};
+
+	/* Lino SPI_1_CS as GPIO */
+	pinctrl_spi1_cs0_gpio: gpio2io12grp {
+		fsl,pins = <MX93_PAD_GPIO_IO12__GPIO2_IO12	0x139e>; /* Lino P58 */
+	};
+
+	/* Lino GPIO_3 */
+	pinctrl_gpio3: gpio2io17grp {
+		fsl,pins = <MX93_PAD_GPIO_IO17__GPIO2_IO17	0x139e>; /* Lino P34 */
+	};
+
+	/* Lino GPIO_4 */
+	pinctrl_gpio4: gpio2io22grp {
+		fsl,pins = <MX93_PAD_GPIO_IO22__GPIO2_IO22	0x139e>; /* Lino P36 */
+	};
+
+	/* Lino PWM_3 as GPIO */
+	pinctrl_pwm3_gpio: gpio2io23grp {
+		fsl,pins = <MX93_PAD_GPIO_IO23__GPIO2_IO23	0x139e>; /* Lino P5 */
+	};
+
+	/* Lino GPIO_5 */
+	pinctrl_gpio5: gpio2io24grp {
+		fsl,pins = <MX93_PAD_GPIO_IO24__GPIO2_IO24	0x139e>; /* Lino P38 */
+	};
+
+	/* Lino GPIO_6 */
+	pinctrl_gpio6: gpio3io26grp {
+		fsl,pins = <MX93_PAD_CCM_CLKO1__GPIO3_IO26	0x139e>; /* Lino P40 */
+	};
+
+	/* Lino GPIO_7 */
+	pinctrl_gpio7: gpio3io27grp {
+		fsl,pins = <MX93_PAD_CCM_CLKO2__GPIO3_IO27	0x139e>; /* Lino P42 */
+	};
+
+	/* Lino GPIO_9 */
+	pinctrl_gpio9: gpio4io14grp {
+		fsl,pins = <MX93_PAD_ENET2_MDC__GPIO4_IO14	0x139e>; /* Lino P46 */
+	};
+
+	/* Lino GPIO_10 */
+	pinctrl_gpio10: gpio4io15grp {
+		fsl,pins = <MX93_PAD_ENET2_MDIO__GPIO4_IO15	0x139e>; /* Lino P48 */
+	};
+
+	/* Lino GPIO_8 */
+	pinctrl_gpio8: gpio4io28grp {
+		fsl,pins = <MX93_PAD_CCM_CLKO3__GPIO4_IO28	0x139e>; /* Lino P44 */
+	};
+
+	/* On-module GPIO_EXP_INT# */
+	pinctrl_gpio_exp_int: gpio4io29grp {
+		fsl,pins = <MX93_PAD_CCM_CLKO4__GPIO4_IO29	0x139e>; /* GPIO_EXP_INT# */
+	};
+
+	/* Lino I2C_1 as GPIO */
+	pinctrl_i2c1_gpio: lpi2c1gpiogrp {
+		fsl,pins = <MX93_PAD_I2C1_SCL__GPIO1_IO00	0x40000b9e>, /* Lino P18 */
+			   <MX93_PAD_I2C1_SDA__GPIO1_IO01	0x40000b9e>; /* Lino P16 */
+	};
+
+	/* Lino I2C_1 */
+	pinctrl_i2c1: lpi2c1grp {
+		fsl,pins = <MX93_PAD_I2C1_SCL__LPI2C1_SCL	0x40000b9e>, /* Lino P18 */
+			   <MX93_PAD_I2C1_SDA__LPI2C1_SDA	0x40000b9e>; /* Lino P16 */
+	};
+
+	/* On-module PMIC I2C as GPIO */
+	pinctrl_pmic_i2c_gpio: lpi2c2gpiogrp {
+		fsl,pins = <MX93_PAD_I2C2_SCL__GPIO1_IO02	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX93_PAD_I2C2_SDA__GPIO1_IO03	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* On-module PMIC I2C */
+	pinctrl_pmic_i2c: lpi2c2grp {
+		fsl,pins = <MX93_PAD_I2C2_SCL__LPI2C2_SCL	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX93_PAD_I2C2_SDA__LPI2C2_SDA	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* Lino I2C_2 as GPIO */
+	pinctrl_i2c2_gpio: lpi2c3gpiogrp {
+		fsl,pins = <MX93_PAD_GPIO_IO28__GPIO2_IO28	0x40000b9e>, /* Lino P20 */
+			   <MX93_PAD_GPIO_IO29__GPIO2_IO29	0x40000b9e>; /* Lino P22 */
+	};
+
+	/* Lino I2C_2 */
+	pinctrl_i2c2: lpi2c3grp {
+		fsl,pins = <MX93_PAD_GPIO_IO28__LPI2C3_SDA	0x40000b9e>, /* Lino P20 */
+			   <MX93_PAD_GPIO_IO29__LPI2C3_SCL	0x40000b9e>; /* Lino P22 */
+	};
+
+	/* Lino I2C_3 as GPIO */
+	pinctrl_i2c3_gpio: lpi2c4gpiogrp {
+		fsl,pins = <MX93_PAD_GPIO_IO02__GPIO2_IO02	0x40000b9e>, /* Lino P24 */
+			   <MX93_PAD_GPIO_IO03__GPIO2_IO03	0x40000b9e>; /* Lino P26 */
+	};
+
+	/* Lino I2C_3 */
+	pinctrl_i2c3: lpi2c4grp {
+		fsl,pins = <MX93_PAD_GPIO_IO02__LPI2C4_SDA	0x40000b9e>, /* Lino P24 */
+			   <MX93_PAD_GPIO_IO03__LPI2C4_SCL	0x40000b9e>; /* Lino P26 */
+	};
+
+	/* Lino SPI_2 */
+	pinctrl_spi2: lpspi1grp {
+		fsl,pins = <MX93_PAD_SAI1_TXD0__LPSPI1_SCK	0x139e>, /* Lino S20 */
+			   <MX93_PAD_SAI1_RXD0__LPSPI1_SOUT	0x139e>, /* Lino S22 */
+			   <MX93_PAD_SAI1_TXC__LPSPI1_SIN	0x139e>; /* Lino S24 */
+	};
+
+	/* Lino SPI_1 */
+	pinctrl_spi1: lpspi8grp {
+		fsl,pins = <MX93_PAD_GPIO_IO13__LPSPI8_SIN	0x139e>, /* Lino P54 */
+			   <MX93_PAD_GPIO_IO14__LPSPI8_SOUT	0x139e>, /* Lino P56 */
+			   <MX93_PAD_GPIO_IO15__LPSPI8_SCK	0x139e>; /* Lino P52 */
+	};
+
+	/* Lino UART_3 */
+	pinctrl_uart3: lpuart1grp {
+		fsl,pins = <MX93_PAD_UART1_RXD__LPUART1_RX	0x39e>, /* Lino P17 */
+			   <MX93_PAD_UART1_TXD__LPUART1_TX	0x39e>; /* Lino P19 */
+	};
+
+	/* Lino UART_4 */
+	pinctrl_uart4: lpuart2grp {
+		fsl,pins = <MX93_PAD_UART2_RXD__LPUART2_RX	0x39e>, /* Lino P23 */
+			   <MX93_PAD_UART2_TXD__LPUART2_TX	0x39e>; /* Lino P25 */
+	};
+
+	/* Lino UART_1 */
+	pinctrl_uart1: lpuart6grp {
+		fsl,pins = <MX93_PAD_GPIO_IO04__LPUART6_TX	0x39e>, /* Lino P11 */
+			   <MX93_PAD_GPIO_IO05__LPUART6_RX	0x39e>, /* Lino P9 */
+			   <MX93_PAD_GPIO_IO06__LPUART6_CTS_B	0x39e>, /* Lino P8 */
+			   <MX93_PAD_GPIO_IO07__LPUART6_RTS_B	0x39e>; /* Lino P6 */
+	};
+
+	/* Lino UART_2 */
+	pinctrl_uart2: lpuart7grp {
+		fsl,pins = <MX93_PAD_GPIO_IO08__LPUART7_TX	0x39e>, /* Lino P15 */
+			   <MX93_PAD_GPIO_IO09__LPUART7_RX	0x39e>, /* Lino P13 */
+			   <MX93_PAD_GPIO_IO10__LPUART7_CTS_B	0x39e>, /* Lino S7 */
+			   <MX93_PAD_GPIO_IO11__LPUART7_RTS_B	0x39e>; /* Lino S5 */
+	};
+
+	/* Lino I2S_1 */
+	pinctrl_i2s1: sai3grp {
+		fsl,pins = <MX93_PAD_GPIO_IO16__SAI3_TX_BCLK	0x139e>, /* Lino S94 */
+			   <MX93_PAD_GPIO_IO19__SAI3_TX_DATA00	0x139e>, /* Lino S98 */
+			   <MX93_PAD_GPIO_IO20__SAI3_RX_DATA00	0x139e>, /* Lino S100 */
+			   <MX93_PAD_GPIO_IO26__SAI3_TX_SYNC	0x139e>; /* Lino S96 */
+	};
+
+	/* Lino PWM_1 */
+	pinctrl_pwm1: tpm4ch1grp {
+		fsl,pins = <MX93_PAD_GPIO_IO21__TPM4_CH1	0x139e>; /* Lino P1 */
+	};
+
+	/* Lino PWM_2 */
+	pinctrl_pwm2: tpm5ch2grp {
+		fsl,pins = <MX93_PAD_GPIO_IO18__TPM5_CH2	0x139e>; /* Lino P3 */
+	};
+
+	/* Lino PWM_3 */
+	pinctrl_pwm3: tpm6ch1grp {
+		fsl,pins = <MX93_PAD_GPIO_IO23__TPM6_CH1	0x139e>; /* Lino P5 */
+	};
+
+	/* On-module eMMC */
+	pinctrl_emmc: usdhc1grp {
+		fsl,pins = <MX93_PAD_SD1_CLK__USDHC1_CLK	0x5fe>,      /* eMMC_CLK    */
+			   <MX93_PAD_SD1_CMD__USDHC1_CMD	0x400013fe>, /* eMMC_CMD    */
+			   <MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe>, /* eMMC_DATA0  */
+			   <MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe>, /* eMMC_DATA1  */
+			   <MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe>, /* eMMC_DATA2  */
+			   <MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe>, /* eMMC_DATA3  */
+			   <MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe>, /* eMMC_DATA4  */
+			   <MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe>, /* eMMC_DATA5  */
+			   <MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe>, /* eMMC_DATA6  */
+			   <MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe>, /* eMMC_DATA7  */
+			   <MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x1580>;     /* eMMC_STROBE */
+	};
+
+	/* Lino SD_1_CD# */
+	pinctrl_sd1_cd: usdhc2cdgrp {
+		fsl,pins = <MX93_PAD_SD2_CD_B__GPIO3_IO00	0x139e>; /* Lino P78 */
+	};
+
+	/* Lino SD_1 */
+	pinctrl_sd1: usdhc2grp {
+		fsl,pins = <MX93_PAD_SD2_CLK__USDHC2_CLK	0x5fe>,      /* Lino P68 */
+			   <MX93_PAD_SD2_CMD__USDHC2_CMD	0x400013fe>, /* Lino P66 */
+			   <MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe>, /* Lino P74 */
+			   <MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe>, /* Lino P76 */
+			   <MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe>, /* Lino P60 */
+			   <MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe>; /* Lino P64 */
+	};
+
+	/* Lino SD_1_PWR_EN */
+	pinctrl_sd1_pwr_en: usdhc2pwrgrp {
+		fsl,pins = <MX93_PAD_SD2_RESET_B__GPIO3_IO07	0x139e>; /* Lino P72 */
+	};
+
+	/* On-module PMIC_SD_2_VSEL */
+	pinctrl_sd1_vsel: usdhc2vselgrp {
+		fsl,pins = <MX93_PAD_SD2_VSELECT__GPIO3_IO19	0x51e>; /* PMIC_SD_2_VSEL */
+	};
+
+	/* Lino SD_2 */
+	pinctrl_sd2: usdhc3grp {
+		fsl,pins = <MX93_PAD_SD3_CLK__USDHC3_CLK	0x5fe>,      /* Lino S10 */
+			   <MX93_PAD_SD3_CMD__USDHC3_CMD	0x400013fe>, /* Lino S8  */
+			   <MX93_PAD_SD3_DATA0__USDHC3_DATA0	0x400013fe>, /* Lino S12 */
+			   <MX93_PAD_SD3_DATA1__USDHC3_DATA1	0x400013fe>, /* Lino S16 */
+			   <MX93_PAD_SD3_DATA2__USDHC3_DATA2	0x400013fe>, /* Lino S2  */
+			   <MX93_PAD_SD3_DATA3__USDHC3_DATA3	0x400013fe>; /* Lino S4  */
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <MX93_PAD_WDOG_ANY__WDOG1_WDOG_ANY 0x31e>; /* PMIC_WDOG_B */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-toradex-osm-dev.dts b/arch/arm64/boot/dts/freescale/imx93-toradex-osm-dev.dts
new file mode 100644
index 00000000000000..a6cc2ae35d04c3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-toradex-osm-dev.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex OSM iMX93 SoM on OSM Development carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx93
+ * https://www.toradex.com/products/carrier-board/osm-development-board
+ */
+
+/dts-v1/;
+
+#include "imx93.dtsi"
+#include "imx93-toradex-osm.dtsi"
+#include "imx93-imx91-toradex-osm-dev-common.dtsi"
+
+/ {
+	model = "Toradex OSM iMX93 on OSM Development Board";
+	compatible = "toradex,osm-imx93-dev",
+		     "toradex,osm-imx93",
+		     "fsl,imx93";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-toradex-osm.dtsi b/arch/arm64/boot/dts/freescale/imx93-toradex-osm.dtsi
new file mode 100644
index 00000000000000..ea5e6055e3f587
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-toradex-osm.dtsi
@@ -0,0 +1,357 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex OSM iMX93 SoM
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx93
+ */
+
+#include "imx93-pinfunc.h"
+#include "imx93-imx91-toradex-osm-common.dtsi"
+
+/ {
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		rsc_table: rsc-table@2021e000 {
+			reg = <0 0x2021e000 0 0x1000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@a4000000 {
+			reg = <0 0xa4000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@a4008000 {
+			reg = <0 0xa4008000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring0: vdev1vring0@a4010000 {
+			reg = <0 0xa4010000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring1: vdev1vring1@a4018000 {
+			reg = <0 0xa4018000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@a4020000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0xa4020000 0 0x100000>;
+			no-map;
+		};
+	};
+};
+
+&cm33 {
+	mbox-names = "tx", "rx", "rxdb";
+	mboxes = <&mu1 0 1>,
+		 <&mu1 1 1>,
+		 <&mu1 3 1>;
+	memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
+			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
+
+	status = "okay";
+};
+
+&mu1 {
+	status = "okay";
+};
+
+&iomuxc {
+	/* OSM CAN_A */
+	pinctrl_can_a: can1grp {
+		fsl,pins = <MX93_PAD_PDM_BIT_STREAM0__CAN1_RX	0x139e>, /* OSM AB17 */
+			   <MX93_PAD_PDM_CLK__CAN1_TX		0x139e>; /* OSM AC17 */
+	};
+
+	/* OSM CAN_B */
+	pinctrl_can_b: can2grp {
+		fsl,pins = <MX93_PAD_GPIO_IO25__CAN2_TX		0x139e>, /* OSM AC19 */
+			   <MX93_PAD_GPIO_IO27__CAN2_RX		0x139e>; /* OSM AB19 */
+	};
+
+	/* OSM CAM_MCK */
+	pinctrl_cam_mck: ccmclk1grp {
+		fsl,pins = <MX93_PAD_CCM_CLKO1__CCMSRCGPCMIX_CLKO1	0x51e>; /* OSM C2 */
+	};
+
+	/* OSM ETH_A RGMII */
+	pinctrl_eth_a: eqosgrp {
+		fsl,pins = <MX93_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x1fe>, /* OSM K15 */
+			   <MX93_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x1fe>, /* OSM L15 */
+			   <MX93_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x1fe>, /* OSM N15 */
+			   <MX93_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x1fe>, /* OSM P15 */
+			   <MX93_PAD_ENET1_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x1fe>, /* OSM R15 */
+			   <MX93_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x1fe>, /* OSM M15 */
+			   <MX93_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x1fe>, /* OSM H15 */
+			   <MX93_PAD_ENET1_TD1__ENET_QOS_RGMII_TD1			0x1fe>, /* OSM G15 */
+			   <MX93_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x1fe>, /* OSM H16 */
+			   <MX93_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x1fe>, /* OSM G16 */
+			   <MX93_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x1fe>, /* OSM J15 */
+			   <MX93_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x1fe>; /* OSM K16 */
+	};
+
+	/* OSM ETH_MDIO */
+	pinctrl_mdio: eqosmdiogrp {
+		fsl,pins = <MX93_PAD_ENET1_MDC__ENET_QOS_MDC	0x1fe>, /* OSM T16 */
+			   <MX93_PAD_ENET1_MDIO__ENET_QOS_MDIO	0x1fe>; /* OSM T15 */
+	};
+
+	/* OSM ETH_B RGMII */
+	pinctrl_eth_b: fecgrp {
+		fsl,pins = <MX93_PAD_ENET2_RD0__ENET1_RGMII_RD0		0x1fe>, /* OSM J1 */
+			   <MX93_PAD_ENET2_RD1__ENET1_RGMII_RD1		0x1fe>, /* OSM K1 */
+			   <MX93_PAD_ENET2_RD2__ENET1_RGMII_RD2		0x1fe>, /* OSM M1 */
+			   <MX93_PAD_ENET2_RD3__ENET1_RGMII_RD3		0x1fe>, /* OSM N1 */
+			   <MX93_PAD_ENET2_RXC__ENET1_RGMII_RXC		0x1fe>, /* OSM P1 */
+			   <MX93_PAD_ENET2_RX_CTL__ENET1_RGMII_RX_CTL	0x1fe>, /* OSM L1 */
+			   <MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x1fe>, /* OSM G1 */
+			   <MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x1fe>, /* OSM F1 */
+			   <MX93_PAD_ENET2_TD2__ENET1_RGMII_TD2		0x1fe>, /* OSM G2 */
+			   <MX93_PAD_ENET2_TD3__ENET1_RGMII_TD3		0x1fe>, /* OSM F2 */
+			   <MX93_PAD_ENET2_TXC__ENET1_RGMII_TXC		0x1fe>, /* OSM H1 */
+			   <MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x1fe>; /* OSM J2 */
+	};
+
+	/* OSM SPI_B_CS as GPIO */
+	pinctrl_spi_b_cs0_gpio: gpio1io11grp {
+		fsl,pins = <MX93_PAD_SAI1_TXFS__GPIO1_IO11	0x139e>; /* OSM AA23 */
+	};
+
+	/* OSM GPIO_A_0 */
+	pinctrl_gpio_a0:  gpio2io0grp {
+		fsl,pins = <MX93_PAD_GPIO_IO00__GPIO2_IO00	0x139e>; /* OSM D17 */
+	};
+
+	/* OSM GPIO_A_1 */
+	pinctrl_gpio_a1: gpio2io1grp {
+		fsl,pins = <MX93_PAD_GPIO_IO01__GPIO2_IO01	0x139e>; /* OSM E17 */
+	};
+
+	/* OSM GPIO_A_2 */
+	pinctrl_gpio_a2: gpio2io2grp {
+		fsl,pins = <MX93_PAD_GPIO_IO02__GPIO2_IO02	0x139e>; /* OSM F17 */
+	};
+
+	/* OSM GPIO_A_3 */
+	pinctrl_gpio_a3: gpio2io3grp {
+		fsl,pins = <MX93_PAD_GPIO_IO03__GPIO2_IO03	0x139e>; /* OSM G17 */
+	};
+
+	/* OSM SPI_A_CS0 as GPIO */
+	pinctrl_spi_a_cs0_gpio: gpio2io12grp {
+		fsl,pins = <MX93_PAD_GPIO_IO12__GPIO2_IO12	0x139e>; /* OSM Y15 */
+	};
+
+	/* OSM PWM_0 as GPIO */
+	pinctrl_pwm0_gpio: gpio2io21grp {
+		fsl,pins = <MX93_PAD_GPIO_IO21__GPIO2_IO21	0x139e>; /* OSM E18 */
+	};
+
+	/* OSM GPIO_A_4 */
+	pinctrl_gpio_a4: gpio2io22grp {
+		fsl,pins = <MX93_PAD_GPIO_IO22__GPIO2_IO22	0x139e>; /* OSM H17 */
+	};
+
+	/* OSM GPIO_A_5 */
+	pinctrl_gpio_a5: gpio2io24grp {
+		fsl,pins = <MX93_PAD_GPIO_IO24__GPIO2_IO24	0x139e>; /* OSM J17 */
+	};
+
+	/* On-module CTRL_INT# */
+	pinctrl_ctrl_int: gpio3io27grp {
+		fsl,pins = <MX93_PAD_CCM_CLKO2__GPIO3_IO27	0x139e>; /* CTRL_INT# */
+	};
+
+	/* OSM GPIO_A_6 */
+	pinctrl_gpio_a6: gpio4io14grp {
+		fsl,pins = <MX93_PAD_ENET2_MDC__GPIO4_IO14	0x139e>; /* OSM K17 */
+	};
+
+	/* OSM GPIO_A_7 */
+	pinctrl_gpio_a7:  gpio4io15grp {
+		fsl,pins = <MX93_PAD_ENET2_MDIO__GPIO4_IO15	0x139e>; /* OSM L17 */
+	};
+
+	/* OSM I2C_B as GPIO */
+	pinctrl_i2c_b_gpio: lpi2c1gpiogrp {
+		fsl,pins = <MX93_PAD_I2C1_SCL__GPIO1_IO00	0x40000b9e>, /* OSM AA20 */
+			   <MX93_PAD_I2C1_SDA__GPIO1_IO01	0x40000b9e>; /* OSM AA21 */
+	};
+
+	/* OSM I2C_B */
+	pinctrl_i2c_b: lpi2c1grp {
+		fsl,pins = <MX93_PAD_I2C1_SCL__LPI2C1_SCL	0x40000b9e>, /* OSM AA20 */
+			   <MX93_PAD_I2C1_SDA__LPI2C1_SDA	0x40000b9e>; /* OSM AA21 */
+	};
+
+	/* On-module PMIC I2C as GPIO */
+	pinctrl_pmic_i2c_gpio: lpi2c2gpiogrp {
+		fsl,pins = <MX93_PAD_I2C2_SCL__GPIO1_IO02	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX93_PAD_I2C2_SDA__GPIO1_IO03	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* On-module PMIC I2C */
+	pinctrl_pmic_i2c: lpi2c2grp {
+		fsl,pins = <MX93_PAD_I2C2_SCL__LPI2C2_SCL	0x40000b9e>, /* PMIC_I2C_SCL */
+			   <MX93_PAD_I2C2_SDA__LPI2C2_SDA	0x40000b9e>; /* PMIC_I2C_SDA */
+	};
+
+	/* OSM I2C_A as GPIO */
+	pinctrl_i2c_a_gpio: lpi2c3gpiogrp {
+		fsl,pins = <MX93_PAD_GPIO_IO28__GPIO2_IO28	0x40000b9e>, /* OSM AA16 */
+			   <MX93_PAD_GPIO_IO29__GPIO2_IO29	0x40000b9e>; /* OSM AA15 */
+	};
+
+	/* OSM I2C_A */
+	pinctrl_i2c_a: lpi2c3grp {
+		fsl,pins = <MX93_PAD_GPIO_IO28__LPI2C3_SDA	0x40000b9e>, /* OSM AA16 */
+			   <MX93_PAD_GPIO_IO29__LPI2C3_SCL	0x40000b9e>; /* OSM AA15 */
+	};
+
+	/* OSM SPI_B */
+	pinctrl_spi_b: lpspi1grp {
+		fsl,pins = <MX93_PAD_SAI1_TXC__LPSPI1_SIN	0x139e>, /* OSM Y22 */
+			   <MX93_PAD_SAI1_TXD0__LPSPI1_SCK	0x139e>, /* OSM Y21 */
+			   <MX93_PAD_SAI1_RXD0__LPSPI1_SOUT	0x139e>; /* OSM Y23 */
+	};
+
+	/* OSM SPI_A */
+	pinctrl_spi_a: lpspi8grp {
+		fsl,pins = <MX93_PAD_GPIO_IO13__LPSPI8_SIN	0x139e>, /* OSM U15 */
+			   <MX93_PAD_GPIO_IO14__LPSPI8_SOUT	0x139e>, /* OSM V15 */
+			   <MX93_PAD_GPIO_IO15__LPSPI8_SCK	0x139e>; /* OSM U16 */
+	};
+
+	/* OSM UART_CON */
+	pinctrl_uart_con: lpuart1grp {
+		fsl,pins = <MX93_PAD_UART1_RXD__LPUART1_RX	0x39e>, /* OSM D22 */
+			   <MX93_PAD_UART1_TXD__LPUART1_TX	0x39e>; /* OSM D23 */
+	};
+
+	/* OSM UART_C */
+	pinctrl_uart_c: lpuart2grp {
+		fsl,pins = <MX93_PAD_UART2_RXD__LPUART2_RX	0x39e>, /* OSM A22 */
+			   <MX93_PAD_UART2_TXD__LPUART2_TX	0x39e>; /* OSM B23 */
+	};
+
+	/* OSM UART_A */
+	pinctrl_uart_a: lpuart6grp {
+		fsl,pins = <MX93_PAD_GPIO_IO04__LPUART6_TX	0x39e>, /* OSM B13 */
+			   <MX93_PAD_GPIO_IO05__LPUART6_RX	0x39e>, /* OSM A14 */
+			   <MX93_PAD_GPIO_IO06__LPUART6_CTS_B	0x39e>, /* OSM C14 */
+			   <MX93_PAD_GPIO_IO07__LPUART6_RTS_B	0x39e>; /* OSM C13 */
+	};
+
+	/* OSM UART_B */
+	pinctrl_uart_b: lpuart7grp {
+		fsl,pins = <MX93_PAD_GPIO_IO08__LPUART7_TX	0x39e>, /* OSM D13 */
+			   <MX93_PAD_GPIO_IO09__LPUART7_RX	0x39e>, /* OSM D14 */
+			   <MX93_PAD_GPIO_IO10__LPUART7_CTS_B	0x39e>, /* OSM D16 */
+			   <MX93_PAD_GPIO_IO11__LPUART7_RTS_B	0x39e>; /* OSM D15 */
+	};
+
+	/* OSM I2S_A */
+	pinctrl_i2s_a: sai3grp {
+		fsl,pins = <MX93_PAD_GPIO_IO16__SAI3_TX_BCLK	0x139e>, /* OSM W20 */
+			   <MX93_PAD_GPIO_IO19__SAI3_TX_DATA00	0x139e>, /* OSM W21 */
+			   <MX93_PAD_GPIO_IO20__SAI3_RX_DATA00	0x139e>, /* OSM V21 */
+			   <MX93_PAD_GPIO_IO26__SAI3_TX_SYNC	0x139e>; /* OSM W18 */
+	};
+
+	/* OSM I2S_MCLK */
+	pinctrl_i2s_a_mclk: sai3mclkgrp {
+		fsl,pins = <MX93_PAD_GPIO_IO17__SAI3_MCLK	0x31e>; /* OSM V18 */
+	};
+
+	/* OSM PWM_0 */
+	pinctrl_pwm0: tpm4ch1grp {
+		fsl,pins = <MX93_PAD_GPIO_IO21__TPM4_CH1	0x139e>; /* OSM E18 */
+	};
+
+	/* OSM PWM_1 */
+	pinctrl_pwm1: tpm5ch2grp {
+		fsl,pins = <MX93_PAD_GPIO_IO18__TPM5_CH2	0x139e>; /* OSM F18 */
+	};
+
+	/* OSM PWM_2 */
+	pinctrl_pwm2: tpm6ch1grp {
+		fsl,pins = <MX93_PAD_GPIO_IO23__TPM6_CH1	0x139e>; /* OSM G18 */
+	};
+
+	/* OSM USB_A_EN */
+	pinctrl_usb_a_en: usb1engrp {
+		fsl,pins = <MX93_PAD_CCM_CLKO3__GPIO4_IO28	0x139e>; /* OSM AC16 */
+	};
+
+	/* OSM USB_A_OC# */
+	pinctrl_usb_a_oc: usb1ocgrp {
+		fsl,pins = <MX93_PAD_CCM_CLKO4__GPIO4_IO29	0x139e>; /* OSM AC15 */
+	};
+
+	/* OSM USB_B_EN */
+	pinctrl_usb_b_en: usb2engrp {
+		fsl,pins = <MX93_PAD_PDM_BIT_STREAM1__GPIO1_IO10	0x139e>; /* OSM AC20 */
+	};
+
+	/* On-module eMMC */
+	pinctrl_emmc: usdhc1grp {
+		fsl,pins = <MX93_PAD_SD1_CLK__USDHC1_CLK	0x5fe>,      /* eMMC_CLK    */
+			   <MX93_PAD_SD1_CMD__USDHC1_CMD	0x400013fe>, /* eMMC_CMD    */
+			   <MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x400013fe>, /* eMMC_DATA0  */
+			   <MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x400013fe>, /* eMMC_DATA1  */
+			   <MX93_PAD_SD1_DATA2__USDHC1_DATA2	0x400013fe>, /* eMMC_DATA2  */
+			   <MX93_PAD_SD1_DATA3__USDHC1_DATA3	0x400013fe>, /* eMMC_DATA3  */
+			   <MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x400013fe>, /* eMMC_DATA4  */
+			   <MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x400013fe>, /* eMMC_DATA5  */
+			   <MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x400013fe>, /* eMMC_DATA6  */
+			   <MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x400013fe>, /* eMMC_DATA7  */
+			   <MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x1580>;     /* eMMC_STROBE */
+	};
+
+	/* OSM SDIO_A_CD# */
+	pinctrl_sdio_a_cd: usdhc2cdgrp {
+		fsl,pins = <MX93_PAD_SD2_CD_B__GPIO3_IO00	0x139e>; /* OSM J21 */
+	};
+
+	/* OSM SDIO_A */
+	pinctrl_sdio_a: usdhc2grp {
+		fsl,pins = <MX93_PAD_SD2_CLK__USDHC2_CLK	0x5fe>,      /* OSM F21 */
+			   <MX93_PAD_SD2_CMD__USDHC2_CMD	0x400013fe>, /* OSM E20 */
+			   <MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x400013fe>, /* OSM G20 */
+			   <MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x400013fe>, /* OSM G21 */
+			   <MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x400013fe>, /* OSM H20 */
+			   <MX93_PAD_SD2_DATA3__USDHC2_DATA3	0x400013fe>; /* OSM H21 */
+	};
+
+	/* OSM SDIO_A_PWR_EN */
+	pinctrl_sdio_a_pwr_en: usdhc2pwrgrp {
+		fsl,pins = <MX93_PAD_SD2_RESET_B__GPIO3_IO07	0x139e>; /* OSM D21 */
+	};
+
+	/* On-module PMIC_SD_2_VSEL */
+	pinctrl_sdio_a_vsel: usdhc2vselgrp {
+		fsl,pins = <MX93_PAD_SD2_VSELECT__GPIO3_IO19	0x51e>; /* PMIC_SD_2_VSEL */
+	};
+
+	/* OSM SDIO_B */
+	pinctrl_sdio_b: usdhc3grp {
+		fsl,pins = <MX93_PAD_SD3_CLK__USDHC3_CLK	0x5fe>,      /* OSM K20 */
+			   <MX93_PAD_SD3_CMD__USDHC3_CMD	0x400013fe>, /* OSM K21 */
+			   <MX93_PAD_SD3_DATA0__USDHC3_DATA0	0x400013fe>, /* OSM L20 */
+			   <MX93_PAD_SD3_DATA1__USDHC3_DATA1	0x400013fe>, /* OSM L21 */
+			   <MX93_PAD_SD3_DATA2__USDHC3_DATA2	0x400013fe>, /* OSM M21 */
+			   <MX93_PAD_SD3_DATA3__USDHC3_DATA3	0x400013fe>; /* OSM N20 */
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <MX93_PAD_WDOG_ANY__WDOG1_WDOG_ANY 0x31e>; /* PMIC_WDOG_B */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93w-frdm.dts b/arch/arm64/boot/dts/freescale/imx93w-frdm.dts
new file mode 100644
index 00000000000000..766e942fe66383
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93w-frdm.dts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+
+#include "imx93w.dtsi"
+#include "imx93-11x11-frdm-common.dtsi"
+
+/ {
+	model = "NXP i.MX93W FRDM board";
+	compatible = "fsl,imx93-wireless-frdm", "fsl,imx93";
+};
+
+&iomuxc {
+	pinctrl_lpi2c3: lpi2c3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO00__LPI2C3_SDA			0x40000b9e
+			MX93_PAD_GPIO_IO01__LPI2C3_SCL			0x40000b9e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx94.dtsi b/arch/arm64/boot/dts/freescale/imx94.dtsi
index 34833e62bd7962..020aa1da33150f 100644
--- a/arch/arm64/boot/dts/freescale/imx94.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx94.dtsi
@@ -131,6 +131,9 @@
 
 	mqs1: mqs1 {
 		compatible = "fsl,imx943-aonmix-mqs";
+		clocks = <&scmi_clk IMX94_CLK_SAI1>;
+		clock-names = "mclk";
+		#sound-dai-cells = <0>;
 		status = "disabled";
 	};
 
@@ -1379,7 +1382,7 @@
 			      <0 0x4c340000 0 0x4000>;
 			reg-names = "dbi", "config", "atu", "app";
 			ranges = <0x43000000 0x9 0x00000000 0x9 0x00000000 0x0 0xe0000000>,
-				 <0x82000000 0x0 0xe0000000 0x9 0xe0000000 0x0 0x10000000>,
+				 <0x82000000 0x0 0x10000000 0x9 0xe0000000 0x0 0x10000000>,
 				 <0x81000000 0x0 0x00000000 0x9 0xf0000000 0x0 0x00100000>;
 			#address-cells = <3>;
 			#size-cells = <2>;
diff --git a/arch/arm64/boot/dts/freescale/imx943-evk-sdwifi.dtso b/arch/arm64/boot/dts/freescale/imx943-evk-sdwifi.dtso
index 59cc1c27b9b90d..b95304fd60aa1f 100644
--- a/arch/arm64/boot/dts/freescale/imx943-evk-sdwifi.dtso
+++ b/arch/arm64/boot/dts/freescale/imx943-evk-sdwifi.dtso
@@ -6,10 +6,35 @@
 /dts-v1/;
 /plugin/;
 
+#include <dt-bindings/gpio/gpio.h>
+
+&{/} {
+	reg_usdhc3_vmmc: regulator-usdhc3 {
+		compatible = "regulator-fixed";
+		regulator-name = "WLAN_EN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_m2_pwr>;
+		gpio = <&pcal6416_i2c3_u46 5 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+&lpuart6 {
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&m2_connector {
+	status = "disabled";
+};
+
 &pcie0 {
 	status = "disabled";
 };
 
 &usdhc3 {
+	vmmc-supply = <&reg_usdhc3_vmmc>;
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/imx943-evk.dts b/arch/arm64/boot/dts/freescale/imx943-evk.dts
index 64660f94f4e9ca..39e819c536cc8a 100644
--- a/arch/arm64/boot/dts/freescale/imx943-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx943-evk.dts
@@ -33,11 +33,51 @@
 		serial5 = &lpuart6;
 	};
 
+	amp_mqs1: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		sound-name-prefix = "MQS AMP";
+		VCC-supply = <&reg_3p3v>;
+	};
+
 	bt_sco_codec: bt-sco-codec {
 		compatible = "linux,bt-sco";
 		#sound-dai-cells = <1>;
 	};
 
+	can1_pdm_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6416_i2c6_u44 12 GPIO_ACTIVE_HIGH>;
+	};
+
+	mqs1_mic_mux: mux-controller-1 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6416_i2c3_u171 11 GPIO_ACTIVE_HIGH>;
+	};
+
+	pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		can1_fun: can1-grp {
+			mux-states = <&can1_pdm_mux 1>;
+		};
+
+		mic_fun: mic-grp {
+			mux-states = <&mqs1_mic_mux 0>;
+		};
+
+		mqs1_fun: mqs1-grp {
+			mux-states = <&mqs1_mic_mux 1>;
+		};
+
+		pdm_fun: pdm-grp {
+			mux-states = <&can1_pdm_mux 0>;
+		};
+	};
+
 	flexcan2_phy: can-phy0 {
 		compatible = "nxp,tjr1443";
 		#phy-cells = <0>;
@@ -77,6 +117,44 @@
 		#sound-dai-cells = <0>;
 	};
 
+	m2_connector: m2-connector {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&reg_m2_pwr>;
+		w-disable1-gpios = <&pcal6416_i2c3_u46 5 GPIO_ACTIVE_LOW>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcie0_port0_ep>;
+				};
+			};
+
+			port@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&lpuart6_ep>;
+				};
+			};
+		};
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "+V3.3_SW";
+	};
+
 	reg_m2_pwr: regulator-m2-pwr {
 		compatible = "regulator-fixed";
 		regulator-name = "M.2-power";
@@ -102,16 +180,6 @@
 		enable-active-high;
 	};
 
-	reg_m2_wlan: regulator-wlan {
-		compatible = "regulator-fixed";
-		regulator-name = "WLAN_EN";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&reg_m2_pwr>;
-		gpio = <&pcal6416_i2c3_u46 5 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-	};
-
 	reg_usdhc2_vmmc: regulator-usdhc2 {
 		compatible = "regulator-fixed";
 		off-on-delay-us = <12000>;
@@ -307,7 +375,7 @@
 		};
 	};
 
-	sound-wm8962 {
+	mqs1_conflict_card: sound-wm8962 {
 		compatible = "fsl,imx-audio-wm8962";
 		audio-codec = <&wm8962>;
 		audio-cpu = <&sai1>;
@@ -535,12 +603,6 @@
 					gpio-hog;
 					output-high;
 				};
-
-				mqs-mic-sel-hog {
-					gpios = <11 GPIO_ACTIVE_HIGH>;
-					gpio-hog;
-					output-low;
-				};
 			};
 		};
 
@@ -651,13 +713,6 @@
 				#gpio-cells = <2>;
 				gpio-controller;
 
-				/* pdm selection */
-				can-pdm-sel-hog {
-					gpios = <12 GPIO_ACTIVE_HIGH>;
-					gpio-hog;
-					output-low;
-				};
-
 				sai3-sel-hog {
 					gpios = <11 GPIO_ACTIVE_HIGH>;
 					gpio-hog;
@@ -694,12 +749,14 @@
 	pinctrl-names = "default";
 	status = "okay";
 
-	bluetooth {
-		compatible = "nxp,88w8987-bt";
+	port {
+		lpuart6_ep: endpoint {
+			remote-endpoint = <&m2_e_uart_ep>;
+		};
 	};
 };
 
-&micfil {
+mqs1_conflict_dev: &micfil {
 	assigned-clocks = <&scmi_clk IMX94_CLK_AUDIOPLL1_VCO>,
 			  <&scmi_clk IMX94_CLK_AUDIOPLL2_VCO>,
 			  <&scmi_clk IMX94_CLK_AUDIOPLL1>,
@@ -710,11 +767,16 @@
 	assigned-clock-rates = <3932160000>,
 			       <3612672000>, <393216000>,
 			       <361267200>, <49152000>;
-	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-0 = <&pinctrl_pdm>, <&mic_fun>, <&pdm_fun>;
 	pinctrl-names = "default";
 	status = "okay";
 };
 
+&mqs1 {
+	pinctrl-0 = <&pinctrl_mqs1>, <&mqs1_fun>, <&pdm_fun>;
+	pinctrl-names = "default";
+};
+
 &mu11 {
 	status = "okay";
 };
@@ -761,6 +823,11 @@
 };
 
 &sai1 {
+	clocks = <&scmi_clk IMX94_CLK_BUSAON>, <&dummy>,
+		 <&scmi_clk IMX94_CLK_SAI1>, <&dummy>,
+		 <&dummy>, <&scmi_clk IMX94_CLK_AUDIOPLL1>,
+		 <&scmi_clk IMX94_CLK_AUDIOPLL2>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
 	assigned-clocks = <&scmi_clk IMX94_CLK_AUDIOPLL1_VCO>,
 			  <&scmi_clk IMX94_CLK_AUDIOPLL2_VCO>,
 			  <&scmi_clk IMX94_CLK_AUDIOPLL1>,
@@ -883,6 +950,13 @@
 		>;
 	};
 
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			IMX94_PAD_PDM_CLK__MQS1_LEFT		0x31e
+			IMX94_PAD_PDM_BIT_STREAM0__MQS1_RIGHT	0x31e
+		>;
+	};
+
 	pinctrl_pcie0: pcie0grp {
 		fsl,pins = <
 			IMX94_PAD_GPIO_IO20__PCIE1_CLKREQ_B	0x4000031e
@@ -1175,7 +1249,12 @@
 
 &pcie0_port0 {
 	reset-gpios = <&pcal6416_i2c3_u46 3 GPIO_ACTIVE_LOW>;
-	vpcie3v3aux-supply = <&reg_m2_wlan>;
+
+	port {
+		pcie0_port0_ep: endpoint {
+			remote-endpoint = <&m2_e_pcie_ep>;
+		};
+	};
 };
 
 &pcie0_ep {
@@ -1288,7 +1367,6 @@
 	pinctrl-1 = <&pinctrl_usdhc3>;
 	pinctrl-names = "default", "sleep";
 	bus-width = <4>;
-	vmmc-supply = <&reg_m2_wlan>;
 	mmc-pwrseq = <&usdhc3_pwrseq>;
 	keep-power-in-suspend;
 	non-removable;
diff --git a/arch/arm64/boot/dts/freescale/imx943.dtsi b/arch/arm64/boot/dts/freescale/imx943.dtsi
index 01152fd0efa5e1..a564488d9cd7d4 100644
--- a/arch/arm64/boot/dts/freescale/imx943.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx943.dtsi
@@ -219,7 +219,7 @@
 			      <0 0x4c3c0000 0 0x4000>;
 			reg-names = "dbi", "config", "atu", "app";
 			ranges = <0x43000000 0xa 0x00000000 0xa 0x00000000 0x0 0xe0000000>,
-				 <0x82000000 0x0 0xe0000000 0xa 0xe0000000 0x0 0x10000000>,
+				 <0x82000000 0x0 0x10000000 0xa 0xe0000000 0x0 0x10000000>,
 				 <0x81000000 0x0 0x00000000 0xa 0xf0000000 0x0 0x00100000>;
 			#address-cells = <3>;
 			#size-cells = <2>;
diff --git a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
index 6aedcbbe915a58..1ca6850a736564 100644
--- a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
@@ -48,6 +48,12 @@
 		serial4 = &lpuart5;
 	};
 
+	amp_mqs1: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		sound-name-prefix = "MQS AMP";
+		VCC-supply = <&reg_3p3v>;
+	};
+
 	bt_sco_codec: bt-sco-codec {
 		compatible = "linux,bt-sco";
 		#sound-dai-cells = <1>;
@@ -114,6 +120,25 @@
 		silent-gpios = <&pcal6524 14 GPIO_ACTIVE_HIGH>;
 	};
 
+	pdm_mqs1_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6524 20 GPIO_ACTIVE_HIGH>;
+	};
+
+	pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		pdm_fun: pdm-grp {
+			mux-states = <&pdm_mqs1_mux 0>;
+		};
+
+		mqs1_fun: mqs1-grp {
+			mux-states = <&pdm_mqs1_mux 1>;
+		};
+	};
+
 	reg_m2_pwr: regulator-m2-pwr {
 		compatible = "regulator-fixed";
 		regulator-max-microvolt = <3300000>;
@@ -220,7 +245,7 @@
 		};
 	};
 
-	sound-bt-sco {
+	mqs1_conflict_card: sound-bt-sco {
 		compatible = "simple-audio-card";
 		simple-audio-card,bitclock-inversion;
 		simple-audio-card,bitclock-master = <&btcpu>;
@@ -479,7 +504,7 @@
 	};
 };
 
-&micfil {
+mqs1_conflict_dev: &micfil {
 	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
 			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
 			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
@@ -488,11 +513,16 @@
 	assigned-clock-parents = <0>, <0>, <0>, <0>, <&scmi_clk IMX95_CLK_AUDIOPLL1>;
 	assigned-clock-rates = <3932160000>, <3612672000>, <393216000>, <361267200>, <49152000>;
 	#sound-dai-cells = <0>;
-	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-0 = <&pinctrl_pdm>, <&pdm_fun>;
 	pinctrl-names = "default";
 	status = "okay";
 };
 
+&mqs1 {
+	pinctrl-0 = <&pinctrl_mqs1>, <&mqs1_fun>;
+	pinctrl-names = "default";
+};
+
 &mu7 {
 	status = "okay";
 };
@@ -573,6 +603,11 @@
 };
 
 &sai1 {
+	clocks = <&scmi_clk IMX95_CLK_BUSAON>, <&dummy>,
+		 <&scmi_clk IMX95_CLK_SAI1>, <&dummy>,
+		 <&dummy>, <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+		 <&scmi_clk IMX95_CLK_AUDIOPLL2>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
 	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
 			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
 			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
@@ -651,6 +686,13 @@
 		>;
 	};
 
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			IMX95_PAD_PDM_CLK__AONMIX_TOP_MQS1_LEFT			0x31e
+			IMX95_PAD_PDM_BIT_STREAM0__AONMIX_TOP_MQS1_RIGHT	0x31e
+		>;
+	};
+
 	pinctrl_hp: hpgrp {
 		fsl,pins = <
 			IMX95_PAD_GPIO_IO21__GPIO2_IO_BIT21			0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts b/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts
index 0f43e3be705895..f82e149323d34b 100644
--- a/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-15x15-frdm.dts
@@ -56,7 +56,7 @@
 		num-channels = <2>;
 	};
 
-	flexcan2_phy: can-phy {
+	flexcan2_phy: can-phy0 {
 		compatible = "nxp,tja1051";
 		#phy-cells = <0>;
 		max-bitrate = <5000000>;
@@ -67,7 +67,7 @@
 		silent-gpios = <&pcal6524 7 GPIO_ACTIVE_HIGH>;
 	};
 
-	flexcan5_phy: can-phy {
+	flexcan5_phy: can-phy1 {
 		compatible = "nxp,tja1051";
 		#phy-cells = <0>;
 		max-bitrate = <5000000>;
@@ -140,6 +140,7 @@
 		regulator-name = "VDD_SD2_3V3";
 		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
+		bootph-pre-ram;
 	};
 
 	reg_usdhc3_vmmc: regulator-usdhc3 {
@@ -188,33 +189,33 @@
 			linux,cma-default;
 		};
 
-		vdev0vring0: memory@88000000 {
+		vdev0vring0: vdev0vring0@88000000 {
 			reg = <0 0x88000000 0 0x8000>;
 			no-map;
 		};
 
-		vdev0vring1: memory@88008000 {
+		vdev0vring1: vdev0vring1@88008000 {
 			reg = <0 0x88008000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring0: memory@88010000 {
+		vdev1vring0: vdev1vring0@88010000 {
 			reg = <0 0x88010000 0 0x8000>;
 			no-map;
 		};
 
-		vdev1vring1: memory@88018000 {
+		vdev1vring1: vdev1vring1@88018000 {
 			reg = <0 0x88018000 0 0x8000>;
 			no-map;
 		};
 
-		vdevbuffer: memory@88020000 {
+		vdev0buffer: vdev0buffer@88020000 {
 			compatible = "shared-dma-pool";
 			reg = <0 0x88020000 0 0x100000>;
 			no-map;
 		};
 
-		rsc_table: memory@88220000 {
+		rsc_table: rsc-table@88220000 {
 			reg = <0 0x88220000 0 0x1000>;
 			no-map;
 		};
@@ -445,6 +446,7 @@
 &lpuart1 {
 	pinctrl-0 = <&pinctrl_uart1>;
 	pinctrl-names = "default";
+	bootph-pre-ram;
 	status = "okay";
 };
 
@@ -480,8 +482,6 @@
 };
 
 &mqs1 {
-	clocks = <&scmi_clk IMX95_CLK_SAI1>;
-	clock-names = "mclk";
 	pinctrl-0 = <&pinctrl_mqs1>;
 	pinctrl-names = "default";
 	status = "okay";
@@ -549,12 +549,15 @@
 &pcie0 {
 	pinctrl-0 = <&pinctrl_pcie0>;
 	pinctrl-names = "default";
-	reset-gpio = <&gpio5 13 GPIO_ACTIVE_LOW>;
 	supports-clkreq;
-	vpcie-supply = <&reg_m2_mkey_pwr>;
 	status = "okay";
 };
 
+&pcie0_port0 {
+	reset-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>;
+	vpcie3v3-supply = <&reg_m2_mkey_pwr>;
+};
+
 &sai1 {
 	clocks = <&scmi_clk IMX95_CLK_BUSAON>, <&dummy>,
 		 <&scmi_clk IMX95_CLK_SAI1>, <&dummy>,
@@ -721,6 +724,7 @@
 		fsl,pins = <
 			IMX95_PAD_SD2_RESET_B__GPIO3_IO_BIT7			0x31e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_uart1: uart1grp {
@@ -728,6 +732,7 @@
 			IMX95_PAD_UART1_RXD__AONMIX_TOP_LPUART1_RX		0x31e
 			IMX95_PAD_UART1_TXD__AONMIX_TOP_LPUART1_TX		0x31e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_uart5: uart5grp {
@@ -753,6 +758,7 @@
 			IMX95_PAD_SD1_DATA7__USDHC1_DATA7			0x138e
 			IMX95_PAD_SD1_STROBE__USDHC1_STROBE			0x158e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
@@ -769,6 +775,7 @@
 			IMX95_PAD_SD1_DATA7__USDHC1_DATA7			0x138e
 			IMX95_PAD_SD1_STROBE__USDHC1_STROBE			0x158e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
@@ -785,12 +792,14 @@
 			IMX95_PAD_SD1_DATA7__USDHC1_DATA7			0x13fe
 			IMX95_PAD_SD1_STROBE__USDHC1_STROBE			0x15fe
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
 		fsl,pins = <
 			IMX95_PAD_SD2_CD_B__GPIO3_IO_BIT0			0x31e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc2: usdhc2grp {
@@ -803,6 +812,7 @@
 			IMX95_PAD_SD2_DATA3__USDHC2_DATA3			0x138e
 			IMX95_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
@@ -815,6 +825,7 @@
 			IMX95_PAD_SD2_DATA3__USDHC2_DATA3			0x138e
 			IMX95_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
@@ -827,6 +838,7 @@
 			IMX95_PAD_SD2_DATA3__USDHC2_DATA3			0x138e
 			IMX95_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
 		>;
+		bootph-pre-ram;
 	};
 
 	pinctrl_usdhc3: usdhc3grp {
@@ -1001,6 +1013,7 @@
 	pinctrl-3 = <&pinctrl_usdhc1>;
 	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
 	fsl,tuning-step = <1>;
+	bootph-pre-ram;
 	status = "okay";
 };
 
@@ -1014,6 +1027,7 @@
 	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
 	vmmc-supply = <&reg_usdhc2_vmmc>;
 	fsl,tuning-step = <1>;
+	bootph-pre-ram;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
index 814da90243b211..47ed0ef67611d6 100644
--- a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
@@ -57,6 +57,37 @@
 		reg = <0x0 0x80000000 0 0x80000000>;
 	};
 
+	m2-connector {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&reg_m2_pwr>;
+		w-disable1-gpios = <&i2c7_pcal6524 6 GPIO_ACTIVE_LOW>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcie0_port0_ep>;
+				};
+			};
+
+			port@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&lpuart5_ep>;
+				};
+			};
+		};
+	};
+
 	fan0: pwm-fan {
 		compatible = "pwm-fan";
 		#cooling-cells = <2>;
@@ -181,16 +212,6 @@
 		startup-delay-us = <5000>;
 	};
 
-	reg_pcie0: regulator-pcie {
-		compatible = "regulator-fixed";
-		regulator-name = "PCIE_WLAN_EN";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&reg_m2_pwr>;
-		gpio = <&i2c7_pcal6524 6 GPIO_ACTIVE_HIGH>;
-		enable-active-high;
-	};
-
 	reg_slot_pwr: regulator-slot-pwr {
 		compatible = "regulator-fixed";
 		regulator-name = "PCIe slot-power";
@@ -523,8 +544,10 @@
 	pinctrl-0 = <&pinctrl_uart5>;
 	status = "okay";
 
-	bluetooth {
-		compatible = "nxp,88w8987-bt";
+	port {
+		lpuart5_ep: endpoint {
+			remote-endpoint = <&m2_e_uart_ep>;
+		};
 	};
 };
 
@@ -601,8 +624,12 @@
 
 &pcie0_port0 {
 	reset-gpios = <&i2c7_pcal6524 5 GPIO_ACTIVE_LOW>;
-	vpcie3v3-supply = <&reg_pcie0>;
-	vpcie3v3aux-supply = <&reg_pcie0>;
+
+	port {
+		pcie0_port0_ep: endpoint {
+			remote-endpoint = <&m2_e_pcie_ep>;
+		};
+	};
 };
 
 &pcie1 {
diff --git a/arch/arm64/boot/dts/freescale/imx95-19x19-frdm-pro.dts b/arch/arm64/boot/dts/freescale/imx95-19x19-frdm-pro.dts
index b87a26b0d7fcf5..1f62d729a5e777 100644
--- a/arch/arm64/boot/dts/freescale/imx95-19x19-frdm-pro.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-19x19-frdm-pro.dts
@@ -450,8 +450,6 @@
 };
 
 &mqs1 {
-	clocks = <&scmi_clk IMX95_CLK_SAI1>;
-	clock-names = "mclk";
 	pinctrl-0 = <&pinctrl_mqs1>;
 	pinctrl-names = "default";
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx95-aquila-clover.dts b/arch/arm64/boot/dts/freescale/imx95-aquila-clover.dts
index fd93043314466d..365bce73eddd30 100644
--- a/arch/arm64/boot/dts/freescale/imx95-aquila-clover.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-aquila-clover.dts
@@ -67,6 +67,15 @@
 	status = "okay";
 };
 
+/* Aquila ETH_2_XGMII */
+&enetc_port2 {
+	phy-handle = <&ethphy2>;
+	phy-mode = "sgmii";
+	managed = "in-band-status";
+
+	status = "okay";
+};
+
 /* Aquila CAN_1 */
 &flexcan1 {
 	status = "okay";
@@ -227,6 +236,15 @@
 	status = "okay";
 };
 
+/* Aquila ETH_2_XGMII_MDIO, shared between all ethernet ports */
+&netc_emdio {
+	ethphy2: ethernet-phy@4 {
+		reg = <4>;
+		interrupt-parent = <&som_gpio_expander_1>;
+		interrupts = <8 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
 /* Aquila PCIE_1 */
 &pcie0 {
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx95-aquila-dev.dts b/arch/arm64/boot/dts/freescale/imx95-aquila-dev.dts
index 3df17700b632f0..7698ef44095cf8 100644
--- a/arch/arm64/boot/dts/freescale/imx95-aquila-dev.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-aquila-dev.dts
@@ -104,6 +104,15 @@
 	status = "okay";
 };
 
+/* Aquila ETH_2_XGMII */
+&enetc_port2 {
+	phy-handle = <&ethphy2>;
+	phy-mode = "sgmii";
+	managed = "in-band-status";
+
+	status = "okay";
+};
+
 /* Aquila CAN_1 */
 &flexcan1 {
 	status = "okay";
@@ -326,6 +335,15 @@
 	status = "okay";
 };
 
+/* Aquila ETH_2_XGMII_MDIO, shared between all ethernet ports */
+&netc_emdio {
+	ethphy2: ethernet-phy@4 {
+		reg = <4>;
+		interrupt-parent = <&som_gpio_expander_1>;
+		interrupts = <8 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
 /* Aquila PCIE_1 */
 &pcie0 {
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx95-aquila.dtsi b/arch/arm64/boot/dts/freescale/imx95-aquila.dtsi
index 69dc962a24a1da..c5f4cf0541f2a0 100644
--- a/arch/arm64/boot/dts/freescale/imx95-aquila.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-aquila.dtsi
@@ -16,6 +16,7 @@
 		can3 = &flexcan4;
 		eeprom0 = &som_eeprom;
 		ethernet0 = &enetc_port0;
+		ethernet1 = &enetc_port2;
 		i2c0 = &lpi2c3;
 		i2c1 = &lpi2c2;
 		i2c2 = &i3c2;
@@ -209,6 +210,12 @@
 	phy-mode = "rgmii-id";
 };
 
+/* Aquila ETH_2_XGMII */
+&enetc_port2 {
+	clocks = <&clk_serdes_eth_ref>;
+	clock-names = "ref";
+};
+
 /* Aquila CAN_1 */
 &flexcan1 {
 	pinctrl-names = "default";
@@ -406,10 +413,10 @@
 		#gpio-cells = <2>;
 		gpio-controller;
 		gpio-line-names =
-			"AQUILA_C1", /* 0 */
-			"AQUILA_C2",
-			"AQUILA_C3",
-			"AQUILA_C4",
+			"AQUILA_C01", /* 0 */
+			"AQUILA_C02",
+			"AQUILA_C03",
+			"AQUILA_C04",
 			"AQUILA_C36",
 			"AQUILA_B74",
 			"AQUILA_B75",
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-osm-dev.dts b/arch/arm64/boot/dts/freescale/imx95-toradex-osm-dev.dts
new file mode 100644
index 00000000000000..b3d54be07c4341
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-osm-dev.dts
@@ -0,0 +1,404 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx95
+ * https://www.toradex.com/products/carrier-board/osm-development-board
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/usb/pd.h>
+#include "imx95-toradex-osm.dtsi"
+
+/ {
+	model = "Toradex OSM iMX95 on OSM Development Board";
+	compatible = "toradex,osm-imx95-dev",
+		     "toradex,osm-imx95",
+		     "fsl,imx95";
+
+	aliases {
+		eeprom1 = &carrier_eeprom;
+	};
+
+	reg_carrier_1p8v: regulator-carrier-1p8v {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "On-carrier 1V8";
+	};
+
+	sfp: sfp {
+		compatible = "sff,sfp";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_b0>, <&pinctrl_gpio_b1>,
+			    <&pinctrl_gpio_b2>, <&pinctrl_gpio_b3>;
+		i2c-bus = <&i2c_mux_carrier_sfp>;
+		los-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+		maximum-power-milliwatt = <3000>;
+		mod-def0-gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
+		rate-select0-gpios = <&som_gpio_expander_0 4 GPIO_ACTIVE_HIGH>;
+		rate-select1-gpios = <&som_gpio_expander_1 3 GPIO_ACTIVE_HIGH>;
+		tx-disable-gpios = <&gpio5 11 GPIO_ACTIVE_HIGH>;
+		tx-fault-gpios = <&gpio5 8 GPIO_ACTIVE_HIGH>;
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "verdin-wm8904";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"Microphone Jack", "MICBIAS",
+			"IN1L", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+
+		codec_dai: simple-audio-card,codec {
+			clocks = <&scmi_clk IMX95_CLK_SAI3>;
+			sound-dai = <&wm8904_1a>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai3>;
+		};
+	};
+};
+
+/* OSM ADC_0, ADC_1 */
+&adc1 {
+	status = "okay";
+};
+
+/* OSM ETH_A */
+&enetc_port0 {
+	phy-handle = <&ethphy_a>;
+	phy-mode = "rgmii-id";
+
+	status = "okay";
+};
+
+/* OSM ETH_B */
+&enetc_port1 {
+	phy-handle = <&ethphy_b>;
+	phy-mode = "rgmii-id";
+
+	status = "okay";
+};
+
+/* OSM ETH_C */
+&enetc_port2 {
+	managed = "in-band-status";
+	phy-mode = "10gbase-r";
+	sfp = <&sfp>;
+
+	status = "okay";
+};
+
+/* OSM CAN_A */
+&flexcan2 {
+	status = "okay";
+};
+
+/* OSM CAN_B */
+&flexcan3 {
+	status = "okay";
+};
+
+&gpio5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio_a1>, <&pinctrl_gpio_a2>, <&pinctrl_gpio_a3>,
+		    <&pinctrl_gpio_a4>, <&pinctrl_gpio_a5>, <&pinctrl_gpio_a6>;
+};
+
+/* OSM I2C_A */
+&lpi2c1 {
+	status = "okay";
+
+	fan_controller: fan-controller@19 {
+		compatible = "ti,amc6821";
+		reg = <0x19>;
+		#pwm-cells = <2>;
+
+		fan {
+			cooling-levels = <255>;
+			pwms = <&fan_controller 40000 PWM_POLARITY_INVERTED>;
+		};
+	};
+
+	wm8904_1a: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		clocks = <&scmi_clk IMX95_CLK_SAI3>;
+		clock-names = "mclk";
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_carrier_1p8v>;
+		CPVDD-supply = <&reg_carrier_1p8v>;
+		DBVDD-supply = <&reg_carrier_1p8v>;
+		DCVDD-supply = <&reg_carrier_1p8v>;
+		MICVDD-supply = <&reg_carrier_1p8v>;
+	};
+
+	gpio@20 {
+		compatible = "nxp,pcal6408";
+		reg = <0x20>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"PWR_CTRL_CAN_A", /* 0 */
+			"PWR_CTRL_CAN_B",
+			"PWR_CTRL_USB-HUB",
+			"LVDS_MUX_CTRL",
+			"USB_MUX_CTRL",
+			"I2CA_IOEXP_P5",
+			"I2CA_IOEXP_P6",
+			"I2CA_IOEXP_P7";
+
+		lvds_mux_ctrl: lvds-mux-ctrl-hog {
+			gpio-hog;
+			gpios = <3 GPIO_ACTIVE_HIGH>;
+			line-name = "LVDS_MUX_CTRL";
+			output-low;
+		};
+
+		usb_mux_ctrl: usb-mux-ctrl-hog {
+			gpio-hog;
+			gpios = <4 GPIO_ACTIVE_HIGH>;
+			line-name = "USB_MUX_CTRL";
+			output-high;
+		};
+	};
+
+	power-monitor@40 {
+		compatible = "ti,ina219";
+		reg = <0x40>;
+		shunt-resistor = <10000>;
+	};
+
+	temperature-sensor@4f {
+		compatible = "ti,tmp1075";
+		reg = <0x4f>;
+	};
+
+	tcpc@52 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x52>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_a0>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+
+		connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB_A";
+			op-sink-microwatt = <0>;
+			power-role = "dual";
+			try-power-role = "sink";
+			self-powered;
+			sink-pdos = <PDO_FIXED(5000, 0, PDO_FIXED_USB_COMM)>;
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+
+			port {
+				usb_c_a_conn_hs: endpoint {
+					remote-endpoint = <&usb_a_hs>;
+				};
+			};
+		};
+	};
+
+	carrier_eeprom: eeprom@57 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x57>;
+		pagesize = <16>;
+	};
+};
+
+/* OSM I2C_B */
+&lpi2c5 {
+	status = "okay";
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9548";
+		reg = <0x70>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c_mux_carrier_cam_a: i2c@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c_mux_carrier_lvds: i2c@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		/* I2C on DSI connector pin #4 and #6 */
+		i2c_mux_carrier_dsi_1: i2c@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		/* I2C on DSI connector pin #52 and #54 */
+		i2c_mux_carrier_dsi_2: i2c@3 {
+			reg = <3>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c_mux_carrier_cam_b: i2c@4 {
+			reg = <4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c_mux_carrier_sfp: i2c@5 {
+			reg = <5>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+	};
+};
+
+/* OSM SPI_A */
+&lpspi4 {
+	status = "okay";
+};
+
+/* OSM UART_CON, used as the Linux Console */
+&lpuart1 {
+	status = "okay";
+};
+
+/* OSM UART_D */
+&lpuart2 {
+	status = "okay";
+};
+
+/* OSM UART_C */
+&lpuart3 {
+	status = "okay";
+};
+
+/* OSM UART_B */
+&lpuart5 {
+	status = "okay";
+};
+
+/* OSM UART_A */
+&lpuart6 {
+	status = "okay";
+};
+
+&netc_blk_ctrl {
+	status = "okay";
+};
+
+/* OSM MDIO, shared between all ethernet ports */
+&netc_emdio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_mdio>;
+
+	status = "okay";
+
+	ethphy_a: ethernet-phy@2 {
+		reg = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_a7>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+	};
+
+	ethphy_b: ethernet-phy@3 {
+		reg = <3>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+	};
+};
+
+&netc_timer {
+	status = "okay";
+};
+
+&netcmix_blk_ctrl {
+	status = "okay";
+};
+
+&pcie0 {
+	status = "okay";
+};
+
+&pcie1 {
+	status = "okay";
+};
+
+/* OSM I2S_A */
+&sai3 {
+	status = "okay";
+};
+
+/* OSM PWM_0 */
+&tpm3 {
+	status = "okay";
+};
+
+/* OSM PWM_1 */
+&tpm4 {
+	status = "okay";
+};
+
+/* OSM PWM_2 */
+&tpm5 {
+	status = "okay";
+};
+
+/* OSM PWM_3 */
+&tpm6 {
+	status = "okay";
+};
+
+/* OSM USB_A */
+&usb2 {
+	status = "okay";
+
+	port {
+		usb_a_hs: endpoint {
+			remote-endpoint = <&usb_c_a_conn_hs>;
+		};
+	};
+};
+
+/* OSM USB_C */
+&usb3 {
+	status = "okay";
+};
+
+&usb3_phy {
+	status = "okay";
+};
+
+/* OSM SDIO_A */
+&usdhc2 {
+	status = "okay";
+};
+
+/* OSM SDIO_B */
+&usdhc3 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-osm.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-osm.dtsi
new file mode 100644
index 00000000000000..8eb7ea4ed9ee4f
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-osm.dtsi
@@ -0,0 +1,1032 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex OSM iMX95 SoM
+ *
+ * https://www.toradex.com/computer-on-modules/osm-arm-family/nxp-imx95
+ */
+
+#include "imx95.dtsi"
+
+/ {
+	aliases {
+		can0 = &flexcan2;
+		can1 = &flexcan3;
+		eeprom0 = &som_eeprom;
+		ethernet0 = &enetc_port0;
+		ethernet1 = &enetc_port1;
+		ethernet2 = &enetc_port2;
+		i2c0 = &lpi2c3;
+		i2c1 = &lpi2c1;
+		i2c2 = &lpi2c5;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		mmc2 = &usdhc3;
+		rtc0 = &rtc_i2c;
+		rtc1 = &scmi_bbm;
+		serial0 = &lpuart1;
+		serial1 = &lpuart6;
+		serial2 = &lpuart5;
+		serial3 = &lpuart3;
+		serial4 = &lpuart2;
+		usb0 = &usb2;
+		usb1 = &usb3;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clk_serdes_eth_ref: clock-eth-ref {
+		compatible = "gpio-gate-clock";
+		#clock-cells = <0>;
+		/* CTRL_ETH_REF_CLK_STBY# */
+		enable-gpios = <&som_gpio_expander_0 13 GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "On-module +V1.8";
+	};
+
+	reg_usb_a_vbus: regulator-usb-a-vbus {
+		compatible = "regulator-fixed";
+		/* OSM USB_A_EN (OSM_AC16) */
+		gpios = <&som_gpio_expander_0 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-name = "USB_A_EN";
+	};
+
+	reg_usb_c_vbus: regulator-usb-c-vbus {
+		compatible = "regulator-fixed";
+		/* OSM USB_C_EN (OSM_C10) */
+		gpios = <&som_gpio_expander_0 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-name = "USB_C_EN";
+	};
+
+	reg_sdio_a_vmmc: regulator-usdhc2-vmmc {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sdio_a_pwr_en>;
+		gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		off-on-delay-us = <100000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "SDIO_A_PWR_EN";
+		startup-delay-us = <20000>;
+	};
+
+	reg_sdio_a_vqmmc: regulator-usdhc2-vqmmc {
+		compatible = "regulator-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sdio_a_vsel>;
+		gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "PMIC_SDIO_A_VSEL";
+		states = <1800000 0x1>, <3300000 0x0>;
+	};
+
+	remoteproc-cm7 {
+		compatible = "fsl,imx95-cm7";
+		mbox-names = "tx", "rx", "rxdb";
+		mboxes = <&mu7 0 1 &mu7 1 1 &mu7 3 1>;
+		memory-region = <&vdev0buffer>, <&vdev0vring0>, <&vdev0vring1>,
+				<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>, <&m7_reserved>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		linux_cma: linux,cma {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0 0x80000000 0 0x7f000000>;
+			linux,cma-default;
+			reusable;
+			size = <0 0x3c000000>;
+		};
+
+		m7_reserved: memory@80000000 {
+			reg = <0 0x80000000 0 0x1000000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@88000000 {
+			reg = <0 0x88000000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@88008000 {
+			reg = <0 0x88008000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring0: vdev1vring0@88010000 {
+			reg = <0 0x88010000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring1: vdev1vring1@88018000 {
+			reg = <0 0x88018000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@88020000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x88020000 0 0x100000>;
+			no-map;
+		};
+
+		rsc_table: rsc-table@88220000 {
+			reg = <0 0x88220000 0 0x1000>;
+			no-map;
+		};
+	};
+};
+
+/* OSM ADC_0, ADC_1 */
+&adc1 {
+	vref-supply = <&reg_1p8v>;
+};
+
+/* OSM ETH_A */
+&enetc_port0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eth_a>;
+};
+
+/* OSM ETH_B */
+&enetc_port1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eth_b>;
+};
+
+/* OSM ETH_C */
+&enetc_port2 {
+	clocks = <&clk_serdes_eth_ref>;
+	clock-names = "ref";
+};
+
+/* OSM CAN_A */
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_can_a>;
+};
+
+/* OSM CAN_B */
+&flexcan3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_can_b>;
+};
+
+&gpio1 {
+	gpio-line-names =
+		"OSM_AA15", /* 0 */
+		"OSM_AA16",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"OSM_F19",
+		"OSM_AA23",
+		"OSM_G19", /* 10 */
+		"OSM_L17"; /* OSM GPIO_A_7/SPI_B_CS1# */
+
+	status = "okay";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"OSM_AA21",
+		"OSM_AA20",
+		"",
+		"",
+		"",
+		"",
+		"I2C_SOM_SDA",
+		"I2C_SOM_SCL";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"OSM_J21", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"OSM_D21",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"PMIC_SDIO_A_VSEL";
+};
+
+&gpio4 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"CTRL_GPIO_EXP_INT#",
+		"OSM_T2";
+};
+
+&gpio5 {
+	gpio-line-names =
+		"OSM_D17", /* OSM GPIO_A_0 */
+		"OSM_E17", /* OSM GPIO_A_1 */
+		"OSM_F17", /* OSM GPIO_A_2 */
+		"OSM_G17", /* OSM GPIO_A_3 */
+		"",
+		"",
+		"",
+		"",
+		"OSM_E19",
+		"OSM_H17", /* OSM GPIO_A_4 */
+		"OSM_J17", /* OSM GPIO_A_5 */
+		"OSM_D19",
+		"",
+		"OSM_K17", /* OSM GPIO_A_6/SPI_A_CS1# */
+		"OSM_Y15";
+};
+
+/* OSM I2C_A */
+&lpi2c1 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c_a>;
+	pinctrl-1 = <&pinctrl_i2c_a_gpio>;
+	clock-frequency = <100000>;
+	scl-gpios = <&gpio1 0 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+};
+
+/* I2C_SOM */
+&lpi2c3 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c_som>, <&pinctrl_ctrl_gpio_exp_int>;
+	pinctrl-1 = <&pinctrl_i2c_som_gpio>, <&pinctrl_ctrl_gpio_exp_int>;
+	clock-frequency = <400000>;
+	scl-gpios = <&gpio2 29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 28 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+
+	status = "okay";
+
+	som_gpio_expander_0: gpio@20 {
+		compatible = "nxp,pcal6416";
+		reg = <0x20>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio4>;
+		interrupts = <14 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"OSM_AC16", /* 0 */
+			"OSM_AC15",
+			"OSM_C10",
+			"OSM_C8",
+			"OSM_H19", /* OSM GPIO_B_4 */
+			"OSM_J19", /* OSM GPIO_B_5 */
+			"OSM_K19", /* OSM GPIO_B_6 */
+			"OSM_L19", /* OSM GPIO_B_7 */
+			"OSM_V2",
+			"OSM_L33",
+			"OSM_AN23", /* 10 */
+			"OSM_AN14",
+			"USB_RECOVERY_CTRL#",
+			"CTRL_ETH_REF_CLK_STBY#",
+			"CTRL_DSI_EN/CSI_EN#",
+			"SPI_B_TPM_CS#";
+
+		/*
+		 * Switch USB to default position:
+		 *   - SoC USB2 -> OSM USB_A
+		 *   - SoC USB1 -> OSM USB_C
+		 * Reset configuration:
+		 *   - SoC USB1 -> OSM USB_A (USB recovery)
+		 *   - SoC USB2 not connected
+		 */
+		usb_recov_ctrl: usb-recov-ctrl-hog {
+			gpio-hog;
+			gpios = <12 GPIO_ACTIVE_HIGH>;
+			line-name = "USB_RECOVERY_CTRL#";
+			output-high;
+		};
+	};
+
+	som_gpio_expander_1: gpio@21 {
+		compatible = "nxp,pcal6416";
+		reg = <0x21>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio4>;
+		interrupts = <14 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names =
+			"OSM_D3",  /* OSM GPIO_C_0 */
+			"OSM_D4",  /* OSM GPIO_C_1 */
+			"OSM_E3",  /* OSM GPIO_C_2/CAM_B_PWR   */
+			"OSM_E4",  /* OSM GPIO_C_3/CAM_B_RST#  */
+			"OSM_F3",  /* OSM GPIO_C_4/DISP_VDD_EN */
+			"OSM_F4",  /* OSM GPIO_C_5/DISP_BL_EN  */
+			"OSM_G3",  /* OSM GPIO_C_6/CAM_A_PWR   */
+			"OSM_G4",  /* OSM GPIO_C_7/CAM_A_RST#  */
+			"OSM_U32",
+			"OSM_U33",
+			"OSM_V32", /* 10 */
+			"OSM_V33",
+			"OSM_W32",
+			"OSM_W33",
+			"OSM_Y32",
+			"OSM_Y33";
+	};
+
+	rtc_i2c: rtc@32 {
+		compatible = "epson,rx8130";
+		reg = <0x32>;
+	};
+
+	temperature-sensor@48 {
+		compatible = "ti,tmp1075";
+		reg = <0x48>;
+	};
+
+	som_eeprom: eeprom@50 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+/* OSM I2C_B */
+&lpi2c5 {
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c_b>;
+	pinctrl-1 = <&pinctrl_i2c_b_gpio>;
+	clock-frequency = <100000>;
+	scl-gpios = <&gpio2 23 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 22 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+};
+
+/* OSM SPI_B */
+&lpspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi_b>, <&pinctrl_spi_b_cs0_gpio>;
+	cs-gpios = <&gpio1 9 GPIO_ACTIVE_LOW>,
+		   <&som_gpio_expander_0 15 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+
+	som_tpm: tpm@1 {
+		compatible = "infineon,slb9670", "tcg,tpm_tis-spi";
+		reg = <0x1>;
+		/*
+		 * Maximum TPM-supported speed is 18.5 MHz, limited to 12 MHz
+		 * here as lpspi1's per-clock (twice the max speed) is 24 MHz
+		 */
+		spi-max-frequency = <12000000>;
+	};
+};
+
+/* OSM SPI_A */
+&lpspi4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi_a>, <&pinctrl_spi_a_cs0_gpio>;
+	cs-gpios = <&gpio5 14 GPIO_ACTIVE_LOW>;
+};
+
+/* OSM UART_CON, used as the Linux Console */
+&lpuart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_con>;
+};
+
+/* OSM UART_D */
+&lpuart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_d>;
+};
+
+/* OSM UART_C */
+&lpuart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_c>;
+};
+
+/* OSM UART_B */
+&lpuart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_b>;
+	uart-has-rtscts;
+};
+
+/* OSM UART_A */
+&lpuart6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart_a>;
+	uart-has-rtscts;
+};
+
+&mu7 {
+	status = "okay";
+};
+
+/* OSM PCIE_A */
+&pcie0 {
+	/* PCIE_A_PERST# (OSM V2) */
+	reset-gpios = <&som_gpio_expander_0 8 GPIO_ACTIVE_LOW>;
+};
+
+/* OSM PCIE_B */
+&pcie1 {
+	/* PCIE_B_PERST# (OSM L33) */
+	reset-gpios = <&som_gpio_expander_0 9 GPIO_ACTIVE_LOW>;
+};
+
+/* OSM I2S_A */
+&sai3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2s_a>, <&pinctrl_i2s_mclk>;
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_SAI3>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX95_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>,
+			       <3612672000>,
+			       <393216000>,
+			       <361267200>,
+			       <12288000>;
+	#sound-dai-cells = <0>;
+	fsl,sai-mclk-direction-output;
+};
+
+/* OSM I2S_B */
+&sai5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2s_b>;
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_SAI5>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX95_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>,
+			       <3612672000>,
+			       <393216000>,
+			       <361267200>,
+			       <12288000>;
+	#sound-dai-cells = <0>;
+	fsl,sai-mclk-direction-output;
+};
+
+&scmi_bbm {
+	linux,code = <KEY_POWER>;
+};
+
+&thermal_zones {
+	/* PF09 Main PMIC */
+	pf09-thermal {
+		polling-delay = <2000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&scmi_sensor 2>;
+
+		trips {
+			trip0 {
+				hysteresis = <2000>;
+				temperature = <155000>;
+				type = "critical";
+			};
+		};
+	};
+
+	/* PF53 VDD_ARM PMIC */
+	pf53-arm-thermal {
+		polling-delay = <2000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&scmi_sensor 4>;
+
+		trips {
+			trip0 {
+				hysteresis = <2000>;
+				temperature = <155000>;
+				type = "critical";
+			};
+		};
+	};
+
+	/* PF53 VDD_SOC PMIC */
+	pf53-soc-thermal {
+		polling-delay = <2000>;
+		polling-delay-passive = <250>;
+		thermal-sensors = <&scmi_sensor 3>;
+
+		trips {
+			trip0 {
+				hysteresis = <2000>;
+				temperature = <155000>;
+				type = "critical";
+			};
+		};
+	};
+};
+
+/* OSM PWM_0 */
+&tpm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0>;
+};
+
+/* OSM PWM_1 */
+&tpm4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+};
+
+/* OSM PWM_2 */
+&tpm5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+};
+
+/* OSM PWM_3 */
+&tpm6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm3>;
+};
+
+/* OSM USB_A */
+&usb2 {
+	adp-disable;
+	disable-over-current;
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	vbus-supply = <&reg_usb_a_vbus>;
+};
+
+/* OSM USB_C */
+&usb3 {
+	fsl,disable-port-power-control;
+};
+
+&usb3_dwc3 {
+	dr_mode = "host";
+};
+
+&usb3_phy {
+	vbus-supply = <&reg_usb_c_vbus>;
+};
+
+/* On-module eMMC */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_emmc>;
+	pinctrl-1 = <&pinctrl_emmc>;
+	pinctrl-2 = <&pinctrl_emmc_200mhz>;
+	no-sd;
+	no-sdio;
+	non-removable;
+
+	status = "okay";
+};
+
+/* OSM SDIO_A */
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	pinctrl-0 = <&pinctrl_sdio_a>, <&pinctrl_sdio_a_cd>;
+	pinctrl-1 = <&pinctrl_sdio_a>, <&pinctrl_sdio_a_cd>;
+	pinctrl-2 = <&pinctrl_sdio_a_200mhz>, <&pinctrl_sdio_a_cd>;
+	pinctrl-3 = <&pinctrl_sdio_a_sleep>, <&pinctrl_sdio_a_cd>;
+	cd-gpios = <&gpio3 0 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	vmmc-supply = <&reg_sdio_a_vmmc>;
+	vqmmc-supply = <&reg_sdio_a_vqmmc>;
+};
+
+/* OSM SDIO_B */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_sdio_b>;
+	pinctrl-1 = <&pinctrl_sdio_b>;
+	pinctrl-2 = <&pinctrl_sdio_b_200mhz>;
+	disable-wp;
+};
+
+&wdog3 {
+	fsl,ext-reset-output;
+
+	status = "okay";
+};
+
+&scmi_iomuxc {
+	/* OSM MDIO, shared between all ethernet ports */
+	pinctrl_mdio: emdiogrp {
+		fsl,pins = <IMX95_PAD_ENET1_MDC__NETCMIX_TOP_NETC_MDC	0x50e>, /* OSM T16 */
+			   <IMX95_PAD_ENET1_MDIO__NETCMIX_TOP_NETC_MDIO	0x90e>; /* OSM T15 */
+	};
+
+	/* OSM ETH_A */
+	pinctrl_eth_a: enetc0grp {
+		fsl,pins = <IMX95_PAD_ENET1_TX_CTL__NETCMIX_TOP_ETH0_RGMII_TX_CTL	0x57e>, /* OSM K16 */
+			   <IMX95_PAD_ENET1_TXC__NETCMIX_TOP_ETH0_RGMII_TX_CLK		0x58e>, /* OSM J15 */
+			   <IMX95_PAD_ENET1_TD0__NETCMIX_TOP_ETH0_RGMII_TD0		0x50e>, /* OSM H15 */
+			   <IMX95_PAD_ENET1_TD1__NETCMIX_TOP_ETH0_RGMII_TD1		0x50e>, /* OSM G15 */
+			   <IMX95_PAD_ENET1_TD2__NETCMIX_TOP_ETH0_RGMII_TD2		0x50e>, /* OSM H16 */
+			   <IMX95_PAD_ENET1_TD3__NETCMIX_TOP_ETH0_RGMII_TD3		0x50e>, /* OSM G16 */
+			   <IMX95_PAD_ENET1_RX_CTL__NETCMIX_TOP_ETH0_RGMII_RX_CTL	0x57e>, /* OSM M15 */
+			   <IMX95_PAD_ENET1_RXC__NETCMIX_TOP_ETH0_RGMII_RX_CLK		0x58e>, /* OSM R15 */
+			   <IMX95_PAD_ENET1_RD0__NETCMIX_TOP_ETH0_RGMII_RD0		0x57e>, /* OSM K15 */
+			   <IMX95_PAD_ENET1_RD1__NETCMIX_TOP_ETH0_RGMII_RD1		0x57e>, /* OSM L15 */
+			   <IMX95_PAD_ENET1_RD2__NETCMIX_TOP_ETH0_RGMII_RD2		0x57e>, /* OSM N15 */
+			   <IMX95_PAD_ENET1_RD3__NETCMIX_TOP_ETH0_RGMII_RD3		0x57e>; /* OSM P15 */
+	};
+
+	/* OSM ETH_B */
+	pinctrl_eth_b: enetc1grp {
+		fsl,pins = <IMX95_PAD_ENET2_TX_CTL__NETCMIX_TOP_ETH1_RGMII_TX_CTL	0x57e>, /* OSM J2 */
+			   <IMX95_PAD_ENET2_TXC__NETCMIX_TOP_ETH1_RGMII_TX_CLK		0x58e>, /* OSM H1 */
+			   <IMX95_PAD_ENET2_TD0__NETCMIX_TOP_ETH1_RGMII_TD0		0x50e>, /* OSM G1 */
+			   <IMX95_PAD_ENET2_TD1__NETCMIX_TOP_ETH1_RGMII_TD1		0x50e>, /* OSM F1 */
+			   <IMX95_PAD_ENET2_TD2__NETCMIX_TOP_ETH1_RGMII_TD2		0x50e>, /* OSM G2 */
+			   <IMX95_PAD_ENET2_TD3__NETCMIX_TOP_ETH1_RGMII_TD3		0x50e>, /* OSM F2 */
+			   <IMX95_PAD_ENET2_RX_CTL__NETCMIX_TOP_ETH1_RGMII_RX_CTL	0x57e>, /* OSM L1 */
+			   <IMX95_PAD_ENET2_RXC__NETCMIX_TOP_ETH1_RGMII_RX_CLK		0x58e>, /* OSM P1 */
+			   <IMX95_PAD_ENET2_RD0__NETCMIX_TOP_ETH1_RGMII_RD0		0x57e>, /* OSM J1 */
+			   <IMX95_PAD_ENET2_RD1__NETCMIX_TOP_ETH1_RGMII_RD1		0x57e>, /* OSM K1 */
+			   <IMX95_PAD_ENET2_RD2__NETCMIX_TOP_ETH1_RGMII_RD2		0x57e>, /* OSM M1 */
+			   <IMX95_PAD_ENET2_RD3__NETCMIX_TOP_ETH1_RGMII_RD3		0x57e>; /* OSM N1 */
+	};
+
+	/* OSM CAN_A */
+	pinctrl_can_a: flexcan2grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO25__CAN2_TX	0x39e>, /* OSM AC17 */
+			   <IMX95_PAD_GPIO_IO27__CAN2_RX	0x39e>; /* OSM AB17 */
+	};
+
+	/* OSM CAN_B */
+	pinctrl_can_b: flexcan3grp {
+		fsl,pins = <IMX95_PAD_CCM_CLKO3__CAN3_TX	0x39e>, /* OSM AC19 */
+			   <IMX95_PAD_CCM_CLKO4__CAN3_RX	0x39e>; /* OSM AB19 */
+	};
+
+	/* OSM GPIO_B_2 */
+	pinctrl_gpio_b2: gpio1io8grp {
+		fsl,pins = <IMX95_PAD_PDM_CLK__AONMIX_TOP_GPIO1_IO_BIT8	0x31e>; /* OSM F19 */
+	};
+
+	/* OSM SPI_B_CS0# */
+	pinctrl_spi_b_cs0_gpio: gpio1io9grp {
+		fsl,pins = <IMX95_PAD_PDM_BIT_STREAM0__AONMIX_TOP_GPIO1_IO_BIT9	0x3fe>; /* OSM AA23 */
+	};
+
+	/* OSM GPIO_B_3 */
+	pinctrl_gpio_b3: gpio1io10grp {
+		fsl,pins = <IMX95_PAD_PDM_BIT_STREAM1__AONMIX_TOP_GPIO1_IO_BIT10	0x31e>; /* OSM G19 */
+	};
+
+	/* OSM GPIO_A_7/SPI_B_CS1# */
+	pinctrl_gpio_a7: gpio1io11grp {
+		fsl,pins = <IMX95_PAD_SAI1_TXFS__AONMIX_TOP_GPIO1_IO_BIT11	0x31e>; /* OSM L17 */
+	};
+
+	/* OSM PWM_0 as GPIO */
+	pinctrl_pwm0_gpio: gpio2io12grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO12__GPIO2_IO_BIT12	0x31e>; /* OSM E18 */
+	};
+
+	/* OSM GPIO_A_0 */
+	pinctrl_gpio_a0: gpio5io0grp {
+		fsl,pins = <IMX95_PAD_XSPI1_DATA0__GPIO5_IO_BIT0	0x31e>; /* OSM D17 */
+	};
+
+	/* OSM GPIO_A_1 */
+	pinctrl_gpio_a1: gpio5io1grp {
+		fsl,pins = <IMX95_PAD_XSPI1_DATA1__GPIO5_IO_BIT1	0x31e>; /* OSM E17 */
+	};
+
+	/* OSM GPIO_A_2 */
+	pinctrl_gpio_a2: gpio5io2grp {
+		fsl,pins = <IMX95_PAD_XSPI1_DATA2__GPIO5_IO_BIT2	0x31e>; /* OSM F17 */
+	};
+
+	/* OSM GPIO_A_3 */
+	pinctrl_gpio_a3: gpio5io3grp {
+		fsl,pins = <IMX95_PAD_XSPI1_DATA3__GPIO5_IO_BIT3	0x31e>; /* OSM G17 */
+	};
+
+	/* OSM GPIO_B_1 */
+	pinctrl_gpio_b1: gpio5io8grp {
+		fsl,pins = <IMX95_PAD_XSPI1_DQS__GPIO5_IO_BIT8	0x31e>; /* OSM E19 */
+	};
+
+	/* OSM GPIO_A_4 */
+	pinctrl_gpio_a4: gpio5io9grp {
+		fsl,pins = <IMX95_PAD_XSPI1_SCLK__GPIO5_IO_BIT9		0x31e>; /* OSM H17 */
+	};
+
+	/* OSM GPIO_A_5 */
+	pinctrl_gpio_a5: gpio5io10grp {
+		fsl,pins = <IMX95_PAD_XSPI1_SS0_B__GPIO5_IO_BIT10	0x31e>; /* OSM J17 */
+	};
+
+	/* OSM GPIO_B_0 */
+	pinctrl_gpio_b0: gpio5io11grp {
+		fsl,pins = <IMX95_PAD_XSPI1_SS1_B__GPIO5_IO_BIT11	0x31e>; /* OSM D19 */
+	};
+
+	/* OSM GPIO_A_6/SPI_A_CS1# */
+	pinctrl_gpio_a6: gpio5io13grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO33__GPIO5_IO_BIT13		0x31e>; /* OSM K17 */
+	};
+
+	/* OSM SPI_A_CS0# */
+	pinctrl_spi_a_cs0_gpio: gpio5io14grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO34__GPIO5_IO_BIT14	0x3fe>; /* OSM Y15 */
+	};
+
+	pinctrl_ctrl_gpio_exp_int: gpioexpintgrp {
+		fsl,pins = <IMX95_PAD_ENET2_MDC__GPIO4_IO_BIT14	0x31e>; /* CTRL_GPIO_EXP_INT# */
+	};
+
+	/* OSM I2C_A as GPIOs */
+	pinctrl_i2c_a_gpio: lpi2c1gpiogrp {
+		fsl,pins = <IMX95_PAD_I2C1_SDA__AONMIX_TOP_GPIO1_IO_BIT1	0x40000b9e>, /* OSM AA16 */
+			   <IMX95_PAD_I2C1_SCL__AONMIX_TOP_GPIO1_IO_BIT0	0x40000b9e>; /* OSM AA15 */
+	};
+
+	/* OSM I2C_A */
+	pinctrl_i2c_a: lpi2c1grp {
+		fsl,pins = <IMX95_PAD_I2C1_SDA__AONMIX_TOP_LPI2C1_SDA	0x40000b9e>, /* OSM AA16 */
+			   <IMX95_PAD_I2C1_SCL__AONMIX_TOP_LPI2C1_SCL	0x40000b9e>; /* OSM AA15 */
+	};
+
+	/* I2C_SOM (On-module I2C) as GPIOs */
+	pinctrl_i2c_som_gpio: lpi2c3gpiogrp {
+		fsl,pins = <IMX95_PAD_GPIO_IO28__GPIO2_IO_BIT28	0x40001b9e>, /* I2C_SOM_SDA */
+			   <IMX95_PAD_GPIO_IO29__GPIO2_IO_BIT29	0x40001b9e>; /* I2C_SOM_SCL */
+	};
+
+	/* I2C_SOM (On-module I2C) */
+	pinctrl_i2c_som: lpi2c3grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO28__LPI2C3_SDA	0x40001b9e>, /* I2C_SOM_SDA */
+			   <IMX95_PAD_GPIO_IO29__LPI2C3_SCL	0x40001b9e>; /* I2C_SOM_SCL */
+	};
+
+	/* OSM I2C_B as GPIOs */
+	pinctrl_i2c_b_gpio: lpi2c5gpiogrp {
+		fsl,pins = <IMX95_PAD_GPIO_IO22__GPIO2_IO_BIT22	0x40001b9e>, /* OSM AA21 */
+			   <IMX95_PAD_GPIO_IO23__GPIO2_IO_BIT23	0x40001b9e>; /* OSM AA20 */
+	};
+
+	/* OSM I2C_B */
+	pinctrl_i2c_b: lpi2c5grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO22__LPI2C5_SDA	0x40001b9e>, /* OSM AA21 */
+			   <IMX95_PAD_GPIO_IO23__LPI2C5_SCL	0x40001b9e>; /* OSM AA20 */
+	};
+
+	/* OSM SPI_B */
+	pinctrl_spi_b: lpspi1grp {
+		fsl,pins = <IMX95_PAD_SAI1_TXD0__AONMIX_TOP_LPSPI1_SCK	0x3fe>, /* OSM Y21 */
+			   <IMX95_PAD_SAI1_RXD0__AONMIX_TOP_LPSPI1_SOUT	0x3fe>, /* OSM Y23 */
+			   <IMX95_PAD_SAI1_TXC__AONMIX_TOP_LPSPI1_SIN	0x3fe>; /* OSM Y22 */
+	};
+
+	/* OSM SPI_A */
+	pinctrl_spi_a: lpspi4grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO37__LPSPI4_SCK	0x3fe>, /* OSM U16 */
+			   <IMX95_PAD_GPIO_IO36__LPSPI4_SOUT	0x3fe>, /* OSM V15 */
+			   <IMX95_PAD_GPIO_IO35__LPSPI4_SIN	0x3fe>; /* OSM U15 */
+	};
+
+	/* OSM UART_CON */
+	pinctrl_uart_con: lpuart1grp {
+		fsl,pins = <IMX95_PAD_UART1_TXD__AONMIX_TOP_LPUART1_TX	0x31e>, /* OSM D23 */
+			   <IMX95_PAD_UART1_RXD__AONMIX_TOP_LPUART1_RX	0x31e>; /* OSM D22 */
+	};
+
+	/* OSM UART_D */
+	pinctrl_uart_d: lpuart2grp {
+		fsl,pins = <IMX95_PAD_UART2_TXD__AONMIX_TOP_LPUART2_TX	0x31e>, /* OSM C23 */
+			   <IMX95_PAD_UART2_RXD__AONMIX_TOP_LPUART2_RX	0x31e>; /* OSM C22 */
+	};
+
+	/* OSM UART_C */
+	pinctrl_uart_c: lpuart3grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO14__LPUART3_TX	0x31e>, /* OSM B23 */
+			   <IMX95_PAD_GPIO_IO15__LPUART3_RX	0x31e>; /* OSM A22 */
+	};
+
+	/* OSM UART_B */
+	pinctrl_uart_b: lpuart5grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO00__LPUART5_TX	0x31e>, /* OSM D13 */
+			   <IMX95_PAD_GPIO_IO01__LPUART5_RX	0x31e>, /* OSM D14 */
+			   <IMX95_PAD_GPIO_IO02__LPUART5_CTS_B	0x31e>, /* OSM D16 */
+			   <IMX95_PAD_GPIO_IO03__LPUART5_RTS_B	0x31e>; /* OSM D15 */
+	};
+
+	/* OSM UART_A */
+	pinctrl_uart_a: lpuart6grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO04__LPUART6_TX	0x31e>, /* OSM B13 */
+			   <IMX95_PAD_GPIO_IO05__LPUART6_RX	0x31e>, /* OSM A14 */
+			   <IMX95_PAD_GPIO_IO06__LPUART6_CTS_B	0x31e>, /* OSM C14 */
+			   <IMX95_PAD_GPIO_IO07__LPUART6_RTS_B	0x31e>; /* OSM C13 */
+	};
+
+	/* OSM PCIE_WAKE# */
+	pinctrl_pcie_wake: pciewakegrp {
+		fsl,pins = <IMX95_PAD_ENET2_MDIO__GPIO4_IO_BIT15	0x31e>; /* OSM T2 */
+	};
+
+	/* OSM I2S_A */
+	pinctrl_i2s_a: sai3grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO16__SAI3_TX_BCLK		0x11e>, /* OSM W20 */
+			   <IMX95_PAD_GPIO_IO20__SAI3_RX_DATA_BIT0	0x11e>, /* OSM V21 */
+			   <IMX95_PAD_GPIO_IO21__SAI3_TX_DATA_BIT0	0x11e>, /* OSM W21 */
+			   <IMX95_PAD_GPIO_IO26__SAI3_TX_SYNC		0x11e>; /* OSM W18 */
+	};
+
+	/* OSM I2S_MCLK */
+	pinctrl_i2s_mclk: sai3mclkgrp {
+		fsl,pins = <IMX95_PAD_GPIO_IO17__SAI3_MCLK	0x31e>; /* OSM V18 */
+	};
+
+	/* OSM I2S_B */
+	pinctrl_i2s_b: sai5grp {
+		fsl,pins = <IMX95_PAD_XSPI1_DATA6__SAI5_TX_BCLK		0x11e>, /* OSM T19 */
+			   <IMX95_PAD_XSPI1_DATA7__SAI5_RX_DATA_BIT0	0x11e>, /* OSM V19 */
+			   <IMX95_PAD_XSPI1_DATA4__SAI5_TX_DATA_BIT0	0x11e>, /* OSM W19 */
+			   <IMX95_PAD_XSPI1_DATA5__SAI5_TX_SYNC		0x11e>; /* OSM T18 */
+	};
+
+	/* OSM PWM_0 */
+	pinctrl_pwm0: tpm3ch2grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO12__TPM3_CH2	0x51e>; /* OSM E18 */
+	};
+
+	/* OSM PWM_1 */
+	pinctrl_pwm1: tpm4ch2grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO13__TPM4_CH2	0x51e>; /* OSM F18 */
+	};
+
+	/* OSM PWM_2 */
+	pinctrl_pwm2: tpm5ch2grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO18__TPM5_CH2	0x51e>; /* OSM G18 */
+	};
+
+	/* OSM PWM_3 */
+	pinctrl_pwm3: tpm6ch2grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO19__TPM6_CH2	0x51e>; /* OSM H18 */
+	};
+
+	/* On-module eMMC (max 200MHz) */
+	pinctrl_emmc_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <IMX95_PAD_SD1_CLK__USDHC1_CLK	0x15fe>, /* eMMC_CLK    */
+			   <IMX95_PAD_SD1_CMD__USDHC1_CMD	0x13fe>, /* eMMC_CMD    */
+			   <IMX95_PAD_SD1_DATA0__USDHC1_DATA0	0x13fe>, /* eMMC_DATA0  */
+			   <IMX95_PAD_SD1_DATA1__USDHC1_DATA1	0x13fe>, /* eMMC_DATA1  */
+			   <IMX95_PAD_SD1_DATA2__USDHC1_DATA2	0x13fe>, /* eMMC_DATA2  */
+			   <IMX95_PAD_SD1_DATA3__USDHC1_DATA3	0x13fe>, /* eMMC_DATA3  */
+			   <IMX95_PAD_SD1_DATA4__USDHC1_DATA4	0x13fe>, /* eMMC_DATA4  */
+			   <IMX95_PAD_SD1_DATA5__USDHC1_DATA5	0x13fe>, /* eMMC_DATA5  */
+			   <IMX95_PAD_SD1_DATA6__USDHC1_DATA6	0x13fe>, /* eMMC_DATA6  */
+			   <IMX95_PAD_SD1_DATA7__USDHC1_DATA7	0x13fe>, /* eMMC_DATA7  */
+			   <IMX95_PAD_SD1_STROBE__USDHC1_STROBE	0x15fe>; /* eMMC_STROBE */
+	};
+
+	/* On-module eMMC (max 100MHz) */
+	pinctrl_emmc: usdhc1grp {
+		fsl,pins = <IMX95_PAD_SD1_CLK__USDHC1_CLK	0x158e>, /* eMMC_CLK    */
+			   <IMX95_PAD_SD1_CMD__USDHC1_CMD	0x138e>, /* eMMC_CMD    */
+			   <IMX95_PAD_SD1_DATA0__USDHC1_DATA0	0x138e>, /* eMMC_DATA0  */
+			   <IMX95_PAD_SD1_DATA1__USDHC1_DATA1	0x138e>, /* eMMC_DATA1  */
+			   <IMX95_PAD_SD1_DATA2__USDHC1_DATA2	0x138e>, /* eMMC_DATA2  */
+			   <IMX95_PAD_SD1_DATA3__USDHC1_DATA3	0x138e>, /* eMMC_DATA3  */
+			   <IMX95_PAD_SD1_DATA4__USDHC1_DATA4	0x138e>, /* eMMC_DATA4  */
+			   <IMX95_PAD_SD1_DATA5__USDHC1_DATA5	0x138e>, /* eMMC_DATA5  */
+			   <IMX95_PAD_SD1_DATA6__USDHC1_DATA6	0x138e>, /* eMMC_DATA6  */
+			   <IMX95_PAD_SD1_DATA7__USDHC1_DATA7	0x138e>, /* eMMC_DATA7  */
+			   <IMX95_PAD_SD1_STROBE__USDHC1_STROBE	0x158e>; /* eMMC_STROBE */
+	};
+
+	/* OSM SDIO_A (max 200MHz) */
+	pinctrl_sdio_a_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <IMX95_PAD_SD2_CLK__USDHC2_CLK	0x15fe>, /* OSM F21 */
+			   <IMX95_PAD_SD2_CMD__USDHC2_CMD	0x13fe>, /* OSM E20 */
+			   <IMX95_PAD_SD2_DATA0__USDHC2_DATA0	0x13fe>, /* OSM G20 */
+			   <IMX95_PAD_SD2_DATA1__USDHC2_DATA1	0x13fe>, /* OSM G21 */
+			   <IMX95_PAD_SD2_DATA2__USDHC2_DATA2	0x13fe>, /* OSM H20 */
+			   <IMX95_PAD_SD2_DATA3__USDHC2_DATA3	0x13fe>; /* OSM H21 */
+	};
+
+	/* OSM SDIO_A_CD# */
+	pinctrl_sdio_a_cd: usdhc2-cdgrp {
+		fsl,pins = <IMX95_PAD_SD2_CD_B__GPIO3_IO_BIT0	0x1100>; /* OSM J21 */
+	};
+
+	/* OSM SDIO_A_PWR_EN */
+	pinctrl_sdio_a_pwr_en: usdhc2-pwrengrp {
+		fsl,pins = <IMX95_PAD_SD2_RESET_B__GPIO3_IO_BIT7	0x11e>; /* OSM D21 */
+	};
+
+	/* OSM SDIO_A (sleep mode) */
+	pinctrl_sdio_a_sleep: usdhc2-sleepgrp {
+		fsl,pins = <IMX95_PAD_SD2_CLK__USDHC2_CLK	0x400>, /* OSM F21 */
+			   <IMX95_PAD_SD2_CMD__USDHC2_CMD	0x400>, /* OSM E20 */
+			   <IMX95_PAD_SD2_DATA0__USDHC2_DATA0	0x400>, /* OSM G20 */
+			   <IMX95_PAD_SD2_DATA1__USDHC2_DATA1	0x400>, /* OSM G21 */
+			   <IMX95_PAD_SD2_DATA2__USDHC2_DATA2	0x400>, /* OSM H20 */
+			   <IMX95_PAD_SD2_DATA3__USDHC2_DATA3	0x400>; /* OSM H21 */
+	};
+
+	/* PMIC_SDIO_A_VSEL */
+	pinctrl_sdio_a_vsel: usdhc2-vselgrp {
+		fsl,pins = <IMX95_PAD_SD2_VSELECT__GPIO3_IO_BIT19	0x4>; /* PMIC_SDIO_A_VSEL */
+	};
+
+	/* OSM SDIO_A (max 100MHz) */
+	pinctrl_sdio_a: usdhc2grp {
+		fsl,pins = <IMX95_PAD_SD2_CLK__USDHC2_CLK	0x158e>, /* OSM F21 */
+			   <IMX95_PAD_SD2_CMD__USDHC2_CMD	0x138e>, /* OSM E20 */
+			   <IMX95_PAD_SD2_DATA0__USDHC2_DATA0	0x138e>, /* OSM G20 */
+			   <IMX95_PAD_SD2_DATA1__USDHC2_DATA1	0x138e>, /* OSM G21 */
+			   <IMX95_PAD_SD2_DATA2__USDHC2_DATA2	0x138e>, /* OSM H20 */
+			   <IMX95_PAD_SD2_DATA3__USDHC2_DATA3	0x138e>; /* OSM H21 */
+	};
+
+	/* OSM SDIO_B (max 200MHz) */
+	pinctrl_sdio_b_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <IMX95_PAD_SD3_CLK__USDHC3_CLK	0x15fe>, /* OSM K20 */
+			   <IMX95_PAD_SD3_CMD__USDHC3_CMD	0x13fe>, /* OSM K21 */
+			   <IMX95_PAD_SD3_DATA0__USDHC3_DATA0	0x13fe>, /* OSM L20 */
+			   <IMX95_PAD_SD3_DATA1__USDHC3_DATA1	0x13fe>, /* OSM L21 */
+			   <IMX95_PAD_SD3_DATA2__USDHC3_DATA2	0x13fe>, /* OSM M21 */
+			   <IMX95_PAD_SD3_DATA3__USDHC3_DATA3	0x13fe>; /* OSM N20 */
+	};
+
+	/* OSM SDIO_B (max 100MHz) */
+	pinctrl_sdio_b: usdhc3grp {
+		fsl,pins = <IMX95_PAD_SD3_CLK__USDHC3_CLK	0x158e>, /* OSM K20 */
+			   <IMX95_PAD_SD3_CMD__USDHC3_CMD	0x138e>, /* OSM K21 */
+			   <IMX95_PAD_SD3_DATA0__USDHC3_DATA0	0x138e>, /* OSM L20 */
+			   <IMX95_PAD_SD3_DATA1__USDHC3_DATA1	0x138e>, /* OSM L21 */
+			   <IMX95_PAD_SD3_DATA2__USDHC3_DATA2	0x138e>, /* OSM M21 */
+			   <IMX95_PAD_SD3_DATA3__USDHC3_DATA3	0x138e>; /* OSM N20 */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts
index 7437e523ff639b..ffe537ccd5bd59 100644
--- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts
@@ -64,6 +64,14 @@
 	status = "okay";
 };
 
+/* SMARC SERDES_0 - ETH2 */
+&enetc_port2 {
+	phy-handle = <&ethphy3>;
+	phy-mode = "sgmii";
+	managed = "in-band-status";
+	status = "okay";
+};
+
 /* SMARC CAN0 */
 &flexcan1 {
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
index a6c5398a81e3c7..641573c9943e6f 100644
--- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
@@ -15,6 +15,7 @@
 		can1 = &flexcan2;
 		ethernet0 = &enetc_port0;
 		ethernet1 = &enetc_port1;
+		ethernet2 = &enetc_port2;
 		mmc0 = &usdhc1;
 		mmc1 = &usdhc2;
 		mmc2 = &usdhc3;
@@ -221,6 +222,12 @@
 	phy-mode = "rgmii-id";
 };
 
+/* SMARC SERDES_0 */
+&enetc_port2 {
+	clocks = <&clk_serdes_eth_ref>;
+	clock-names = "ref";
+};
+
 /* SMARC CAN0 */
 &flexcan1 {
 	pinctrl-names = "default";
@@ -839,6 +846,7 @@
 	keep-power-in-suspend;
 	non-removable;
 	vmmc-supply = <&reg_wifi_en>;
+	status = "okay";
 };
 
 &scmi_bbm {
diff --git a/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts
index 3479abc25df24e..9344a0d99418ba 100644
--- a/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-var-dart-sonata.dts
@@ -336,7 +336,7 @@
 	status = "okay";
 };
 
-/* Header (J12.4, J12.6) */
+/* Header (J6.4, J6.6) */
 &lpuart8 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart8>;
diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
index 38fc47024841fd..d20c2498b9aa26 100644
--- a/arch/arm64/boot/dts/freescale/imx95.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
@@ -255,19 +255,16 @@
 
 		opp-500000000 {
 			opp-hz = /bits/ 64 <500000000>;
-			opp-hz-real = /bits/ 64 <500000000>;
 			opp-microvolt = <920000>;
 		};
 
 		opp-800000000 {
 			opp-hz = /bits/ 64 <800000000>;
-			opp-hz-real = /bits/ 64 <800000000>;
 			opp-microvolt = <920000>;
 		};
 
 		opp-1000000000 {
 			opp-hz = /bits/ 64 <1000000000>;
-			opp-hz-real = /bits/ 64 <1000000000>;
 			opp-microvolt = <920000>;
 		};
 	};
@@ -439,6 +436,9 @@
 
 	mqs1: mqs-1 {
 		compatible = "fsl,imx95-aonmix-mqs";
+		clocks = <&scmi_clk IMX95_CLK_SAI1>;
+		clock-names = "mclk";
+		#sound-dai-cells = <0>;
 		status = "disabled";
 	};
 
@@ -1458,7 +1458,7 @@
 				clock-names = "per", "ipg";
 				#address-cells = <1>;
 				#size-cells = <0>;
-				dmas = <&edma1 12 0 0>, <&edma1 13 0 FSL_EDMA_RX> ;
+				dmas = <&edma1 12 0 0>, <&edma1 13 0 FSL_EDMA_RX>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
@@ -1472,7 +1472,7 @@
 				clock-names = "per", "ipg";
 				#address-cells = <1>;
 				#size-cells = <0>;
-				dmas = <&edma1 14 0 0>, <&edma1 15 0 FSL_EDMA_RX> ;
+				dmas = <&edma1 14 0 0>, <&edma1 15 0 FSL_EDMA_RX>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
@@ -1486,7 +1486,7 @@
 				clocks = <&scmi_clk IMX95_CLK_LPSPI1>,
 					 <&scmi_clk IMX95_CLK_BUSAON>;
 				clock-names = "per", "ipg";
-				dmas = <&edma1 16 0 FSL_EDMA_RX>, <&edma1 17 0 0> ;
+				dmas = <&edma1 16 0 FSL_EDMA_RX>, <&edma1 17 0 0>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
@@ -1500,7 +1500,7 @@
 				clocks = <&scmi_clk IMX95_CLK_LPSPI2>,
 					 <&scmi_clk IMX95_CLK_BUSAON>;
 				clock-names = "per", "ipg";
-				dmas = <&edma1 18 0 FSL_EDMA_RX>, <&edma1 19 0 0> ;
+				dmas = <&edma1 18 0 FSL_EDMA_RX>, <&edma1 19 0 0>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
@@ -1948,7 +1948,7 @@
 			      <0 0x4c340000 0 0x4000>;
 			reg-names = "dbi", "config", "atu", "app";
 			ranges = <0x43000000 0x9 0x00000000 0x9 0x00000000 0x0 0xe0000000>,
-				 <0x82000000 0x0 0xe0000000 0x9 0xe0000000 0x0 0x10000000>,
+				 <0x82000000 0x0 0x10000000 0x9 0xe0000000 0x0 0x10000000>,
 				 <0x81000000 0x0 0x00000000 0x9 0xf0000000 0x0 0x00100000>;
 			#address-cells = <3>;
 			#size-cells = <2>;
@@ -2039,7 +2039,7 @@
 			      <0 0x4c3c0000 0 0x4000>;
 			reg-names = "dbi", "config", "atu", "app";
 			ranges = <0x43000000 0xa 0x00000000 0xa 0x00000000 0x0 0xe0000000>,
-				 <0x82000000 0x0 0xe0000000 0xa 0xe0000000 0x0 0x10000000>,
+				 <0x82000000 0x0 0x10000000 0xa 0xe0000000 0x0 0x10000000>,
 				 <0x81000000 0x0 0x00000000 0xa 0xf0000000 0x0 0x00100000>;
 			#address-cells = <3>;
 			#size-cells = <2>;
diff --git a/arch/arm64/boot/dts/freescale/imx952-evk.dts b/arch/arm64/boot/dts/freescale/imx952-evk.dts
index 2224b90aafe93a..047dfde207181b 100644
--- a/arch/arm64/boot/dts/freescale/imx952-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx952-evk.dts
@@ -43,6 +43,12 @@
 		spi6 = &lpspi7;
 	};
 
+	amp_mqs1: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		sound-name-prefix = "MQS AMP";
+		VCC-supply = <&reg_3p3v>;
+	};
+
 	bt_sco_codec: audio-codec-bt-sco {
 		#sound-dai-cells = <1>;
 		compatible = "linux,bt-sco";
@@ -52,6 +58,40 @@
 		stdout-path = &lpuart1;
 	};
 
+	can1_pdm_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6416 10 GPIO_ACTIVE_HIGH>;
+	};
+
+	mqs1_en_mux: mux-controller-1 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		mux-gpios = <&pcal6416 15 GPIO_ACTIVE_HIGH>;
+	};
+
+	pinctrl-gpiomux {
+		compatible = "pinctrl-multiplexer";
+
+		can1_fun: can1-grp {
+			mux-states = <&can1_pdm_mux 1>;
+		};
+
+		mqs1_disable: mqs1-disable-grp {
+			mux-states = <&mqs1_en_mux 0>;
+		};
+
+		mqs1_enable: mqs1-enable-grp {
+			mux-states = <&mqs1_en_mux 1>;
+		};
+
+		pdm_fun: pdm-grp {
+			mux-states = <&can1_pdm_mux 0>;
+		};
+	};
+
 	dmic: dmic {
 		compatible = "dmic-codec";
 		#sound-dai-cells = <0>;
@@ -197,7 +237,7 @@
 				<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>, <&m7_reserved>;
 	};
 
-	sound-bt-sco {
+	mqs1_conflict_card: sound-bt-sco {
 		compatible = "simple-audio-card";
 		simple-audio-card,bitclock-inversion;
 		simple-audio-card,bitclock-master = <&btcpu>;
@@ -284,7 +324,7 @@
 /* pin conflict with PDM */
 &flexcan1 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_flexcan1>;
+	pinctrl-0 = <&pinctrl_flexcan1>, <&mqs1_disable>, <&can1_fun>;
 	phys = <&flexcan1_phy>;
 	status = "disabled";
 };
@@ -387,18 +427,6 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_pcal6416>;
 		vcc-supply = <&reg_3p3v>;
-
-		pdm-can-sel-hog {
-			gpio-hog;
-			gpios = <10 GPIO_ACTIVE_HIGH>;
-			output-low;
-		};
-
-		mqs-en-hog {
-			gpio-hog;
-			gpios = <15 GPIO_ACTIVE_HIGH>;
-			output-low;
-		};
 	};
 };
 
@@ -481,7 +509,7 @@
 	status = "okay";
 };
 
-&micfil {
+mqs1_conflict_dev: &micfil {
 	assigned-clocks = <&scmi_clk IMX952_CLK_AUDIOPLL1_VCO>,
 			  <&scmi_clk IMX952_CLK_AUDIOPLL2_VCO>,
 			  <&scmi_clk IMX952_CLK_AUDIOPLL1>,
@@ -491,17 +519,27 @@
 				<&scmi_clk IMX952_CLK_AUDIOPLL1>;
 	assigned-clock-rates = <3932160000>, <3612672000>,
 			       <393216000>, <361267200>, <49152000>;
-	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-0 = <&pinctrl_pdm>, <&mqs1_disable>, <&pdm_fun>;
 	pinctrl-1 = <&pinctrl_pdm_sleep>;
 	pinctrl-names = "default", "sleep";
 	status = "okay";
 };
 
+&mqs1 {
+	pinctrl-0 = <&pinctrl_mqs1>, <&mqs1_enable>;
+	pinctrl-names = "default";
+};
+
 &mu7 {
 	status = "okay";
 };
 
 &sai1 {
+	clocks = <&scmi_clk IMX952_CLK_BUSAON>, <&clk_dummy>,
+		 <&scmi_clk IMX952_CLK_SAI1>, <&clk_dummy>,
+		 <&clk_dummy>, <&scmi_clk IMX952_CLK_AUDIOPLL1>,
+		 <&scmi_clk IMX952_CLK_AUDIOPLL2>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
 	assigned-clocks = <&scmi_clk IMX952_CLK_AUDIOPLL1_VCO>,
 			  <&scmi_clk IMX952_CLK_AUDIOPLL2_VCO>,
 			  <&scmi_clk IMX952_CLK_AUDIOPLL1>,
@@ -682,6 +720,13 @@
 		>;
 	};
 
+	pinctrl_mqs1: mqs1grp {
+		fsl,pins = <
+			IMX952_PAD_PDM_CLK__AONMIX_TOP_MQS1_LEFT		0x31e
+			IMX952_PAD_PDM_BIT_STREAM0__AONMIX_TOP_MQS1_RIGHT	0x31e
+		>;
+	};
+
 	pinctrl_pcal6416: pcal6416grp {
 		fsl,pins = <
 			IMX952_PAD_GPIO_IO10__WAKEUPMIX_TOP_GPIO2_IO_10		0x31e
diff --git a/arch/arm64/boot/dts/freescale/imx952-frdm.dts b/arch/arm64/boot/dts/freescale/imx952-frdm.dts
new file mode 100644
index 00000000000000..0b9d74eea37668
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx952-frdm.dts
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+
+#include "imx952.dtsi"
+#include "imx952-frdm.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx952-frdm.dtsi b/arch/arm64/boot/dts/freescale/imx952-frdm.dtsi
new file mode 100644
index 00000000000000..7785eb1a68fca4
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx952-frdm.dtsi
@@ -0,0 +1,726 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+#include <dt-bindings/usb/pd.h>
+
+#define BRD_SM_CTRL_SD3_WAKE		0x8000U /*!< PCAL6408A-0 */
+#define BRD_SM_CTRL_M2M_WAKE		0x8003U /*!< PCAL6408A-6 */
+#define BRD_SM_CTRL_BUTTON		0x8004U /*!< PCAL6408A-7 */
+
+/ {
+	compatible = "fsl,imx952-frdm", "fsl,imx952";
+	model = "NXP i.MX952 FRDM board";
+
+	aliases {
+		gpio0 = &gpio1;
+		gpio1 = &gpio2;
+		gpio2 = &gpio3;
+		gpio3 = &gpio4;
+		gpio4 = &gpio5;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		i2c2 = &lpi2c3;
+		i2c3 = &lpi2c4;
+		i2c4 = &lpi2c5;
+		i2c5 = &lpi2c6;
+		i2c6 = &lpi2c7;
+		i2c7 = &lpi2c8;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		serial0 = &lpuart1;
+		serial4 = &lpuart5;
+		spi2 = &lpspi3;
+	};
+
+	bt_sco_codec: audio-codec-bt-sco {
+		compatible = "linux,bt-sco";
+		#sound-dai-cells = <1>;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	dmic: dmic {
+		compatible = "dmic-codec";
+		#sound-dai-cells = <0>;
+		num-channels = <2>;
+	};
+
+	flexcan2_phy: can-phy0 {
+		compatible = "nxp,tja1051";
+		#phy-cells = <0>;
+		max-bitrate = <8000000>;
+		silent-gpios = <&pcal6416 6 GPIO_ACTIVE_HIGH>;
+	};
+
+	flexcan3_phy: can-phy1 {
+		compatible = "nxp,tja1051";
+		#phy-cells = <0>;
+		max-bitrate = <8000000>;
+		silent-gpios = <&pcal6416 6 GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "+V1.8_SW";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "+V3.3_SW";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+	};
+
+	reg_5p0v: regulator-5p0v {
+		compatible = "regulator-fixed";
+		regulator-name = "+V5.0_SW";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+	};
+
+	reg_vref_1v8: regulator-adc-vref {
+		compatible = "regulator-fixed";
+		regulator-name = "vref_1v8";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+	};
+
+	reg_dummy: regulator-dummy {
+		compatible = "regulator-fixed";
+		regulator-name = "dummy";
+	};
+
+	reg_ext_12v: regulator-ext-12v {
+		compatible = "regulator-fixed";
+		regulator-name = "VCCEXT_12V";
+		regulator-max-microvolt = <12000000>;
+		regulator-min-microvolt = <12000000>;
+		vin-supply = <&reg_ext_5v>;
+	};
+
+	reg_ext_3v3: regulator-ext-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCCEXT_3V3";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&pcal6416 13 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	reg_ext_5v: regulator-ext-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "VCCEXT_5V";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		gpio = <&pcal6416 12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	reg_m2_pwr: regulator-m2-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "M.2-power";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&pcal6416 11 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_SD2_3V3";
+		off-on-delay-us = <12000>;
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc3_vmmc: regulator-usdhc3 {
+		compatible = "regulator-fixed";
+		regulator-name = "WLAN_EN";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		vin-supply = <&reg_m2_pwr>;
+		gpio = <&pcal6416 9 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		/*
+		 * IW612 wifi chip needs more delay than other wifi chips to complete
+		 * the host interface initialization after power up, otherwise the
+		 * internal state of IW612 may be unstable, resulting in the failure of
+		 * the SDIO3.0 switch voltage.
+		 */
+		startup-delay-us = <20000>;
+	};
+
+	reg_usb_vbus: regulator-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "USB_VBUS";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		gpio = <&pcal6416 15 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		linux_cma: linux,cma {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0 0x80000000 0 0x7f000000>;
+			reusable;
+			size = <0 0x3c000000>;
+			linux,cma-default;
+		};
+	};
+
+	sound-bt-sco {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-inversion;
+		simple-audio-card,bitclock-master = <&btcpu>;
+		simple-audio-card,format = "dsp_a";
+		simple-audio-card,frame-master = <&btcpu>;
+		simple-audio-card,name = "bt-sco-audio";
+
+		simple-audio-card,codec {
+			sound-dai = <&bt_sco_codec 1>;
+		};
+
+		btcpu: simple-audio-card,cpu {
+			dai-tdm-slot-num = <2>;
+			dai-tdm-slot-width = <16>;
+			sound-dai = <&sai5>;
+		};
+	};
+
+	sound-micfil {
+		compatible = "fsl,imx-audio-card";
+		model = "micfil-audio";
+
+		pri-dai-link {
+			format = "i2s";
+			link-name = "micfil hifi";
+
+			codec {
+				sound-dai = <&dmic>;
+			};
+
+			cpu {
+				sound-dai = <&micfil>;
+			};
+		};
+	};
+
+	usdhc3_pwrseq: usdhc3-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&pcal6416 8 GPIO_ACTIVE_LOW>;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0 0x80000000>;
+	};
+};
+
+&adc1 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+&asrc1 {
+	assigned-clocks = <&scmi_clk IMX952_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX952_CLK_ASRC1>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX952_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>, <3612672000>,
+			       <393216000>, <361267200>, <49152000>;
+	fsl,asrc-rate = <48000>;
+	status = "okay";
+};
+
+&asrc2 {
+	assigned-clocks = <&scmi_clk IMX952_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX952_CLK_ASRC2>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX952_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>, <3612672000>,
+			       <393216000>, <361267200>, <49152000>;
+	fsl,asrc-rate = <48000>;
+	status = "okay";
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	phys = <&flexcan2_phy>;
+	status = "okay";
+};
+
+&flexcan3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan3>;
+	phys = <&flexcan3_phy>;
+	status = "okay";
+};
+
+&lpi2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c2>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pcal6416: gpio@20 {
+		compatible = "nxp,pcal6416";
+		reg = <0x20>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio5>;
+		interrupts = <14 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		pinctrl-0 = <&pinctrl_pcal6416>;
+		pinctrl-names = "default";
+		vcc-supply = <&reg_3p3v>;
+
+		/* When low, select CAN; When high, select gpio. */
+		can-gpio-sel-hog {
+			gpios = <4 GPIO_ACTIVE_HIGH>;
+			gpio-hog;
+			output-low;
+		};
+	};
+};
+
+&lpi2c4 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c4>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ptn5110: tcpc@50 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x50>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pinctrl_ptn5110>;
+		pinctrl-names = "default";
+
+		typec_con: connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB-C";
+			op-sink-microwatt = <0>;
+			power-role = "dual";
+			self-powered;
+			sink-pdos = <PDO_FIXED(5000, 0, PDO_FIXED_USB_COMM)>;
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			try-power-role = "sink";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					typec1_dr_sw: endpoint {
+						remote-endpoint = <&usb1_drd_sw>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&lpspi3 {
+	cs-gpios = <&gpio2 8 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_lpspi3>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&lpuart1 {
+	/* console */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&lpuart5 {
+	/* BT */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&micfil {
+	assigned-clocks = <&scmi_clk IMX952_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX952_CLK_PDM>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				<&scmi_clk IMX952_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>, <3612672000>,
+			       <393216000>, <361267200>, <49152000>;
+	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-1 = <&pinctrl_pdm_sleep>;
+	pinctrl-names = "default", "sleep";
+	status = "okay";
+};
+
+&sai5 {
+	assigned-clocks = <&scmi_clk IMX952_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX952_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX952_CLK_SAI5>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>, <&scmi_clk IMX952_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>, <3612672000>, <393216000>, <361267200>, <12288000>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_sai5>;
+	pinctrl-names = "default";
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&scmi_iomuxc {
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			IMX952_PAD_GPIO_IO25__WAKEUPMIX_TOP_CAN2_TX		0x39e
+			IMX952_PAD_GPIO_IO27__WAKEUPMIX_TOP_CAN2_RX		0x39e
+		>;
+	};
+
+	pinctrl_flexcan3: flexcan3grp {
+		fsl,pins = <
+			IMX952_PAD_CCM_CLKO3__WAKEUPMIX_TOP_CAN3_TX		0x39e
+			IMX952_PAD_CCM_CLKO4__WAKEUPMIX_TOP_CAN3_RX		0x39e
+		>;
+	};
+
+	pinctrl_lpi2c2: lpi2c2grp {
+		fsl,pins = <
+			IMX952_PAD_I2C2_SCL__AONMIX_TOP_LPI2C2_SCL		0x40000b9e
+			IMX952_PAD_I2C2_SDA__AONMIX_TOP_LPI2C2_SDA		0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c4: lpi2c4grp {
+		fsl,pins = <
+			IMX952_PAD_GPIO_IO30__WAKEUPMIX_TOP_LPI2C4_SDA		0x40000b9e
+			IMX952_PAD_GPIO_IO31__WAKEUPMIX_TOP_LPI2C4_SCL		0x40000b9e
+		>;
+	};
+
+	pinctrl_lpspi3: lpspi3grp {
+		fsl,pins = <
+			IMX952_PAD_GPIO_IO08__WAKEUPMIX_TOP_GPIO2_IO_8		0x39e
+			IMX952_PAD_GPIO_IO09__WAKEUPMIX_TOP_LPSPI3_SIN		0x39e
+			IMX952_PAD_GPIO_IO10__WAKEUPMIX_TOP_LPSPI3_SOUT		0x39e
+			IMX952_PAD_GPIO_IO11__WAKEUPMIX_TOP_LPSPI3_SCK		0x39e
+		>;
+	};
+
+	pinctrl_pcal6416: pcal6416grp {
+		fsl,pins = <
+			IMX952_PAD_GPIO_IO34__WAKEUPMIX_TOP_GPIO5_IO_14		0x31e
+		>;
+	};
+
+	pinctrl_pdm: pdmgrp {
+		fsl,pins = <
+			IMX952_PAD_PDM_CLK__AONMIX_TOP_PDM_CLK				0x31e
+			IMX952_PAD_PDM_BIT_STREAM0__AONMIX_TOP_PDM_BIT_STREAM_0		0x31e
+		>;
+	};
+
+	pinctrl_pdm_sleep: pdmsleepgrp {
+		fsl,pins = <
+			IMX952_PAD_PDM_CLK__AONMIX_TOP_GPIO1_IO_8		0x31e
+			IMX952_PAD_PDM_BIT_STREAM0__AONMIX_TOP_GPIO1_IO_9	0x31e
+		>;
+	};
+
+	pinctrl_ptn5110: ptn5110grp {
+		fsl,pins = <
+			IMX952_PAD_XSPI1_DATA2__WAKEUPMIX_TOP_GPIO5_IO_2	0x31e
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			IMX952_PAD_SD2_RESET_B__WAKEUPMIX_TOP_GPIO3_IO_7	0x31e
+		>;
+	};
+
+	pinctrl_sai5: sai5grp {
+		fsl,pins = <
+			IMX952_PAD_XSPI1_DATA4__WAKEUPMIX_TOP_SAI5_TX_DATA_0		0x31e
+			IMX952_PAD_XSPI1_DATA5__WAKEUPMIX_TOP_SAI5_TX_SYNC		0x31e
+			IMX952_PAD_XSPI1_DATA6__WAKEUPMIX_TOP_SAI5_TX_BCLK		0x31e
+			IMX952_PAD_XSPI1_DATA7__WAKEUPMIX_TOP_SAI5_RX_DATA_0		0x31e
+		>;
+	};
+
+	pinctrl_tpm3: tpm3grp {
+		fsl,pins = <
+			IMX952_PAD_GPIO_IO04__WAKEUPMIX_TOP_TPM3_CH0		0x51e
+			IMX952_PAD_GPIO_IO12__WAKEUPMIX_TOP_TPM3_CH2		0x51e
+		>;
+	};
+
+	pinctrl_tpm4: tpm4grp {
+		fsl,pins = <
+			IMX952_PAD_GPIO_IO13__WAKEUPMIX_TOP_TPM4_CH2		0x51e
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			IMX952_PAD_UART1_RXD__AONMIX_TOP_LPUART1_RX		0x31e
+			IMX952_PAD_UART1_TXD__AONMIX_TOP_LPUART1_TX		0x31e
+		>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			IMX952_PAD_DAP_TDO_TRACESWO__WAKEUPMIX_TOP_LPUART5_TX	0x31e
+			IMX952_PAD_DAP_TDI__WAKEUPMIX_TOP_LPUART5_RX		0x31e
+			IMX952_PAD_DAP_TMS_SWDIO__WAKEUPMIX_TOP_LPUART5_RTS_B	0x31e
+			IMX952_PAD_DAP_TCLK_SWCLK__WAKEUPMIX_TOP_LPUART5_CTS_B	0x31e
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			IMX952_PAD_SD1_CLK__WAKEUPMIX_TOP_USDHC1_CLK		0x158e
+			IMX952_PAD_SD1_CMD__WAKEUPMIX_TOP_USDHC1_CMD		0x138e
+			IMX952_PAD_SD1_DATA0__WAKEUPMIX_TOP_USDHC1_DATA0	0x138e
+			IMX952_PAD_SD1_DATA1__WAKEUPMIX_TOP_USDHC1_DATA1	0x138e
+			IMX952_PAD_SD1_DATA2__WAKEUPMIX_TOP_USDHC1_DATA2	0x138e
+			IMX952_PAD_SD1_DATA3__WAKEUPMIX_TOP_USDHC1_DATA3	0x138e
+			IMX952_PAD_SD1_DATA4__WAKEUPMIX_TOP_USDHC1_DATA4	0x138e
+			IMX952_PAD_SD1_DATA5__WAKEUPMIX_TOP_USDHC1_DATA5	0x138e
+			IMX952_PAD_SD1_DATA6__WAKEUPMIX_TOP_USDHC1_DATA6	0x138e
+			IMX952_PAD_SD1_DATA7__WAKEUPMIX_TOP_USDHC1_DATA7	0x138e
+			IMX952_PAD_SD1_STROBE__WAKEUPMIX_TOP_USDHC1_STROBE	0x158e
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			IMX952_PAD_SD1_CLK__WAKEUPMIX_TOP_USDHC1_CLK		0x158e
+			IMX952_PAD_SD1_CMD__WAKEUPMIX_TOP_USDHC1_CMD		0x138e
+			IMX952_PAD_SD1_DATA0__WAKEUPMIX_TOP_USDHC1_DATA0	0x138e
+			IMX952_PAD_SD1_DATA1__WAKEUPMIX_TOP_USDHC1_DATA1	0x138e
+			IMX952_PAD_SD1_DATA2__WAKEUPMIX_TOP_USDHC1_DATA2	0x138e
+			IMX952_PAD_SD1_DATA3__WAKEUPMIX_TOP_USDHC1_DATA3	0x138e
+			IMX952_PAD_SD1_DATA4__WAKEUPMIX_TOP_USDHC1_DATA4	0x138e
+			IMX952_PAD_SD1_DATA5__WAKEUPMIX_TOP_USDHC1_DATA5	0x138e
+			IMX952_PAD_SD1_DATA6__WAKEUPMIX_TOP_USDHC1_DATA6	0x138e
+			IMX952_PAD_SD1_DATA7__WAKEUPMIX_TOP_USDHC1_DATA7	0x138e
+			IMX952_PAD_SD1_STROBE__WAKEUPMIX_TOP_USDHC1_STROBE	0x158e
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			IMX952_PAD_SD1_CLK__WAKEUPMIX_TOP_USDHC1_CLK		0x159e
+			IMX952_PAD_SD1_CMD__WAKEUPMIX_TOP_USDHC1_CMD		0x139e
+			IMX952_PAD_SD1_DATA0__WAKEUPMIX_TOP_USDHC1_DATA0	0x139e
+			IMX952_PAD_SD1_DATA1__WAKEUPMIX_TOP_USDHC1_DATA1	0x139e
+			IMX952_PAD_SD1_DATA2__WAKEUPMIX_TOP_USDHC1_DATA2	0x139e
+			IMX952_PAD_SD1_DATA3__WAKEUPMIX_TOP_USDHC1_DATA3	0x139e
+			IMX952_PAD_SD1_DATA4__WAKEUPMIX_TOP_USDHC1_DATA4	0x139e
+			IMX952_PAD_SD1_DATA5__WAKEUPMIX_TOP_USDHC1_DATA5	0x139e
+			IMX952_PAD_SD1_DATA6__WAKEUPMIX_TOP_USDHC1_DATA6	0x139e
+			IMX952_PAD_SD1_DATA7__WAKEUPMIX_TOP_USDHC1_DATA7	0x139e
+			IMX952_PAD_SD1_STROBE__WAKEUPMIX_TOP_USDHC1_STROBE	0x159e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			IMX952_PAD_SD2_CD_B__WAKEUPMIX_TOP_GPIO3_IO_0		0x31e
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			IMX952_PAD_SD2_CLK__WAKEUPMIX_TOP_USDHC2_CLK		0x158e
+			IMX952_PAD_SD2_CMD__WAKEUPMIX_TOP_USDHC2_CMD		0x138e
+			IMX952_PAD_SD2_DATA0__WAKEUPMIX_TOP_USDHC2_DATA0	0x138e
+			IMX952_PAD_SD2_DATA1__WAKEUPMIX_TOP_USDHC2_DATA1	0x138e
+			IMX952_PAD_SD2_DATA2__WAKEUPMIX_TOP_USDHC2_DATA2	0x138e
+			IMX952_PAD_SD2_DATA3__WAKEUPMIX_TOP_USDHC2_DATA3	0x138e
+			IMX952_PAD_SD2_VSELECT__WAKEUPMIX_TOP_USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			IMX952_PAD_SD2_CLK__WAKEUPMIX_TOP_USDHC2_CLK		0x158e
+			IMX952_PAD_SD2_CMD__WAKEUPMIX_TOP_USDHC2_CMD		0x138e
+			IMX952_PAD_SD2_DATA0__WAKEUPMIX_TOP_USDHC2_DATA0	0x138e
+			IMX952_PAD_SD2_DATA1__WAKEUPMIX_TOP_USDHC2_DATA1	0x138e
+			IMX952_PAD_SD2_DATA2__WAKEUPMIX_TOP_USDHC2_DATA2	0x138e
+			IMX952_PAD_SD2_DATA3__WAKEUPMIX_TOP_USDHC2_DATA3	0x138e
+			IMX952_PAD_SD2_VSELECT__WAKEUPMIX_TOP_USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			IMX952_PAD_SD2_CLK__WAKEUPMIX_TOP_USDHC2_CLK		0x158e
+			IMX952_PAD_SD2_CMD__WAKEUPMIX_TOP_USDHC2_CMD		0x138e
+			IMX952_PAD_SD2_DATA0__WAKEUPMIX_TOP_USDHC2_DATA0	0x138e
+			IMX952_PAD_SD2_DATA1__WAKEUPMIX_TOP_USDHC2_DATA1	0x138e
+			IMX952_PAD_SD2_DATA2__WAKEUPMIX_TOP_USDHC2_DATA2	0x138e
+			IMX952_PAD_SD2_DATA3__WAKEUPMIX_TOP_USDHC2_DATA3	0x138e
+			IMX952_PAD_SD2_VSELECT__WAKEUPMIX_TOP_USDHC2_VSELECT	0x51e
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			IMX952_PAD_SD3_CLK__WAKEUPMIX_TOP_USDHC3_CLK			0x158e
+			IMX952_PAD_SD3_CMD__WAKEUPMIX_TOP_USDHC3_CMD			0x138e
+			IMX952_PAD_SD3_DATA0__WAKEUPMIX_TOP_USDHC3_DATA0		0x138e
+			IMX952_PAD_SD3_DATA1__WAKEUPMIX_TOP_USDHC3_DATA1		0x138e
+			IMX952_PAD_SD3_DATA2__WAKEUPMIX_TOP_USDHC3_DATA2		0x138e
+			IMX952_PAD_SD3_DATA3__WAKEUPMIX_TOP_USDHC3_DATA3		0x138e
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			IMX952_PAD_SD3_CLK__WAKEUPMIX_TOP_USDHC3_CLK			0x158e
+			IMX952_PAD_SD3_CMD__WAKEUPMIX_TOP_USDHC3_CMD			0x138e
+			IMX952_PAD_SD3_DATA0__WAKEUPMIX_TOP_USDHC3_DATA0		0x138e
+			IMX952_PAD_SD3_DATA1__WAKEUPMIX_TOP_USDHC3_DATA1		0x138e
+			IMX952_PAD_SD3_DATA2__WAKEUPMIX_TOP_USDHC3_DATA2		0x138e
+			IMX952_PAD_SD3_DATA3__WAKEUPMIX_TOP_USDHC3_DATA3		0x138e
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			IMX952_PAD_SD3_CLK__WAKEUPMIX_TOP_USDHC3_CLK			0x158e
+			IMX952_PAD_SD3_CMD__WAKEUPMIX_TOP_USDHC3_CMD			0x138e
+			IMX952_PAD_SD3_DATA0__WAKEUPMIX_TOP_USDHC3_DATA0		0x138e
+			IMX952_PAD_SD3_DATA1__WAKEUPMIX_TOP_USDHC3_DATA1		0x138e
+			IMX952_PAD_SD3_DATA2__WAKEUPMIX_TOP_USDHC3_DATA2		0x138e
+			IMX952_PAD_SD3_DATA3__WAKEUPMIX_TOP_USDHC3_DATA3		0x138e
+		>;
+	};
+};
+
+&scmi_misc {
+	nxp,ctrl-ids = <BRD_SM_CTRL_SD3_WAKE		1
+			BRD_SM_CTRL_M2M_WAKE		1
+			BRD_SM_CTRL_BUTTON		1>;
+};
+
+&tpm3 {
+	pinctrl-0 = <&pinctrl_tpm3>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&tpm4 {
+	pinctrl-0 = <&pinctrl_tpm4>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usb1 {
+	adp-disable;
+	disable-over-current;
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usb2 {
+	disable-over-current;
+	dr_mode = "host";
+	vbus-supply = <&reg_usb_vbus>;
+	status = "okay";
+};
+
+&usdhc1 {
+	bus-width = <8>;
+	non-removable;
+	no-sd;
+	no-sdio;
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	pinctrl-3 = <&pinctrl_usdhc1>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	status = "okay";
+};
+
+&usdhc2 {
+	bus-width = <4>;
+	cd-gpios = <&gpio3 0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-3 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	status = "okay";
+};
+
+&usdhc3 {
+	bus-width = <4>;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&usdhc3_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	pinctrl-3 = <&pinctrl_usdhc3>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	vmmc-supply = <&reg_usdhc3_vmmc>;
+	wakeup-source;
+	status = "okay";
+};
+
+&wdog3 {
+	fsl,ext-reset-output;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx952.dtsi b/arch/arm64/boot/dts/freescale/imx952.dtsi
index 020de00e828593..1154b66dadb2d9 100644
--- a/arch/arm64/boot/dts/freescale/imx952.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx952.dtsi
@@ -261,6 +261,14 @@
 		};
 	};
 
+	mqs1: mqs1 {
+		compatible = "fsl,imx95-aonmix-mqs";
+		clocks = <&scmi_clk IMX952_CLK_SAI1>;
+		clock-names = "mclk";
+		#sound-dai-cells = <0>;
+		status = "disabled";
+	};
+
 	gic: interrupt-controller@48000000 {
 		compatible = "arm,gic-v3";
 		reg = <0 0x48000000 0 0x10000>,
@@ -323,19 +331,16 @@
 
 		opp-500000000 {
 			opp-hz = /bits/ 64 <500000000>;
-			opp-hz-real = /bits/ 64 <500000000>;
 			opp-microvolt = <920000>;
 		};
 
 		opp-800000000 {
 			opp-hz = /bits/ 64 <800000000>;
-			opp-hz-real = /bits/ 64 <800000000>;
 			opp-microvolt = <920000>;
 		};
 
 		opp-1000000000 {
 			opp-hz = /bits/ 64 <1000000000>;
-			opp-hz-real = /bits/ 64 <1000000000>;
 			opp-microvolt = <920000>;
 		};
 	};
@@ -1164,7 +1169,7 @@
 				clock-names = "per", "ipg";
 				#address-cells = <1>;
 				#size-cells = <0>;
-				dmas = <&edma1 12 0 0>, <&edma1 13 0 FSL_EDMA_RX> ;
+				dmas = <&edma1 12 0 0>, <&edma1 13 0 FSL_EDMA_RX>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
@@ -1178,7 +1183,7 @@
 				clock-names = "per", "ipg";
 				#address-cells = <1>;
 				#size-cells = <0>;
-				dmas = <&edma1 14 0 0>, <&edma1 15 0 FSL_EDMA_RX> ;
+				dmas = <&edma1 14 0 0>, <&edma1 15 0 FSL_EDMA_RX>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
@@ -1192,7 +1197,7 @@
 				clock-names = "per", "ipg";
 				#address-cells = <1>;
 				#size-cells = <0>;
-				dmas = <&edma1 16 0 FSL_EDMA_RX>, <&edma1 17 0 0> ;
+				dmas = <&edma1 16 0 FSL_EDMA_RX>, <&edma1 17 0 0>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
@@ -1206,7 +1211,7 @@
 				clock-names = "per", "ipg";
 				#address-cells = <1>;
 				#size-cells = <0>;
-				dmas = <&edma1 18 0 FSL_EDMA_RX>, <&edma1 19 0 0> ;
+				dmas = <&edma1 18 0 FSL_EDMA_RX>, <&edma1 19 0 0>;
 				dma-names = "tx", "rx";
 				status = "disabled";
 			};
diff --git a/arch/arm64/boot/dts/freescale/s32g2.dtsi b/arch/arm64/boot/dts/freescale/s32g2.dtsi
index 996494e34593fb..a7e5977bc146ae 100644
--- a/arch/arm64/boot/dts/freescale/s32g2.dtsi
+++ b/arch/arm64/boot/dts/freescale/s32g2.dtsi
@@ -14,7 +14,7 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	reserved-memory  {
+	reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges;
diff --git a/arch/arm64/boot/dts/freescale/s32g3.dtsi b/arch/arm64/boot/dts/freescale/s32g3.dtsi
index 31f7aad00b7645..0770af61a2c461 100644
--- a/arch/arm64/boot/dts/freescale/s32g3.dtsi
+++ b/arch/arm64/boot/dts/freescale/s32g3.dtsi
@@ -153,7 +153,7 @@
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
-	reserved-memory  {
+	reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges;
diff --git a/arch/arm64/boot/dts/google/Makefile b/arch/arm64/boot/dts/google/Makefile
new file mode 100644
index 00000000000000..4e833b2561e081
--- /dev/null
+++ b/arch/arm64/boot/dts/google/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+
+dtb-$(CONFIG_ARCH_GOOGLE) += \
+	lga-blazer.dtb \
+	lga-frankel.dtb \
+	lga-mustang.dtb
diff --git a/arch/arm64/boot/dts/google/lga-blazer.dts b/arch/arm64/boot/dts/google/lga-blazer.dts
new file mode 100644
index 00000000000000..0894606624f8ef
--- /dev/null
+++ b/arch/arm64/boot/dts/google/lga-blazer.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Blazer Device Tree (Pixel 10 Pro)
+ *
+ * Copyright 2021-2023 Google LLC
+ */
+
+/dts-v1/;
+
+#include "lga-pixel-common.dtsi"
+
+/ {
+	model = "Blazer";
+	compatible = "google,lga-blazer", "google,lga";
+};
diff --git a/arch/arm64/boot/dts/google/lga-frankel.dts b/arch/arm64/boot/dts/google/lga-frankel.dts
new file mode 100644
index 00000000000000..7504222a728d4e
--- /dev/null
+++ b/arch/arm64/boot/dts/google/lga-frankel.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Frankel Device Tree (Pixel 10)
+ *
+ * Copyright 2021-2023 Google LLC
+ */
+
+/dts-v1/;
+
+#include "lga-pixel-common.dtsi"
+
+/ {
+	model = "Frankel";
+	compatible = "google,lga-frankel", "google,lga";
+};
diff --git a/arch/arm64/boot/dts/google/lga-mustang.dts b/arch/arm64/boot/dts/google/lga-mustang.dts
new file mode 100644
index 00000000000000..6ac0b5ef22ff25
--- /dev/null
+++ b/arch/arm64/boot/dts/google/lga-mustang.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Mustang Device Tree (Pixel 10 Pro XL)
+ *
+ * Copyright 2021-2023 Google LLC
+ */
+
+/dts-v1/;
+
+#include "lga-pixel-common.dtsi"
+
+/ {
+	model = "Mustang";
+	compatible = "google,lga-mustang", "google,lga";
+};
diff --git a/arch/arm64/boot/dts/google/lga-pixel-common.dtsi b/arch/arm64/boot/dts/google/lga-pixel-common.dtsi
new file mode 100644
index 00000000000000..5aaa2e5af8ec90
--- /dev/null
+++ b/arch/arm64/boot/dts/google/lga-pixel-common.dtsi
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Google Pixel 10 family common dtsi.
+ *
+ * Copyright 2024-2025 Google LLC.
+ */
+
+#include "lga.dtsi"
+
+/ {
+	aliases {
+		serial0 = &lsion_cli16_uart;
+	};
+
+	chosen {
+		stdout-path = "serial0";
+	};
+};
+
+&osc {
+	clock-frequency = <38400000>;
+};
diff --git a/arch/arm64/boot/dts/google/lga.dtsi b/arch/arm64/boot/dts/google/lga.dtsi
new file mode 100644
index 00000000000000..3e670e524c4c9f
--- /dev/null
+++ b/arch/arm64/boot/dts/google/lga.dtsi
@@ -0,0 +1,411 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Google Tensor G5 (laguna) SoC rev B0
+ *
+ * Copyright 2024-2025 Google LLC.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	interrupt-parent = <&gic>;
+
+	osc: clock-38400000 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+	};
+
+	cpus: cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		hayes_0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x000>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <258>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd0>;
+			power-domain-names = "psci";
+		};
+
+		hayes_1: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a520";
+			reg = <0x100>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <258>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd1>;
+			power-domain-names = "psci";
+		};
+
+		hunter_0: cpu@200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a725";
+			reg = <0x200>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <891>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd2>;
+			power-domain-names = "psci";
+		};
+
+		hunter_1: cpu@300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a725";
+			reg = <0x300>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <891>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd3>;
+			power-domain-names = "psci";
+		};
+
+		hunter_2: cpu@400 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a725";
+			reg = <0x400>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <891>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd4>;
+			power-domain-names = "psci";
+		};
+
+		hunter_3: cpu@500 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a725";
+			reg = <0x500>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <891>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd5>;
+			power-domain-names = "psci";
+		};
+
+		hunter_4: cpu@600 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a725";
+			reg = <0x600>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <891>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd6>;
+			power-domain-names = "psci";
+		};
+
+		hunterelp_0: cpu@700 {
+			device_type = "cpu";
+			compatible = "arm,cortex-x4";
+			reg = <0x700>;
+
+			#cooling-cells = <2>;
+			capacity-dmips-mhz = <1024>;
+			enable-method = "psci";
+			power-domains = <&cpu_pd7>;
+			power-domain-names = "psci";
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&hayes_0>;
+				};
+
+				core1 {
+					cpu = <&hayes_1>;
+				};
+
+				core2 {
+					cpu = <&hunter_0>;
+				};
+
+				core3 {
+					cpu = <&hunter_1>;
+				};
+
+				core4 {
+					cpu = <&hunter_2>;
+				};
+
+				core5 {
+					cpu = <&hunter_3>;
+				};
+
+				core6 {
+					cpu = <&hunter_4>;
+				};
+
+				core7 {
+					cpu = <&hunterelp_0>;
+				};
+			};
+		};
+
+		domain-idle-states {
+			cluster_1_c3: cluster-1-c3 {
+				compatible = "domain-idle-state";
+
+				entry-latency-us = <325>;
+				exit-latency-us = <553>;
+				min-residency-us = <10000>;
+
+				idle-state-name = "cluster-1-c3";
+
+				arm,psci-suspend-param = <0x40010033>;
+			};
+
+			cluster_2_c3: cluster-2-c3 {
+				compatible = "domain-idle-state";
+
+				entry-latency-us = <345>;
+				exit-latency-us = <673>;
+				min-residency-us = <10000>;
+
+				idle-state-name = "cluster-2-c3";
+
+				arm,psci-suspend-param = <0x40010033>;
+			};
+
+			cluster_c4_deep: cluster-c4-deep {
+				compatible = "domain-idle-state";
+
+				entry-latency-us = <510>;
+				exit-latency-us = <1027>;
+				min-residency-us = <10000>;
+
+				idle-state-name = "cluster-c4-deep";
+
+				arm,psci-suspend-param = <0x40020333>;
+			};
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			cpu_big_c2: cpu-big-c2 {
+				compatible = "arm,idle-state";
+
+				entry-latency-us = <110>;
+				exit-latency-us = <413>;
+				min-residency-us = <3950>;
+
+				idle-state-name = "cpu-big-c2";
+				local-timer-stop;
+
+				arm,psci-suspend-param = <0x40000003>;
+			};
+
+			cpu_lit_c2: cpu-lit-c2 {
+				compatible = "arm,idle-state";
+
+				entry-latency-us = <132>;
+				exit-latency-us = <296>;
+				min-residency-us = <3610>;
+
+				idle-state-name = "cpu-lit-c2";
+				local-timer-stop;
+
+				arm,psci-suspend-param = <0x40000003>;
+			};
+
+			cpu_mid1_c2: cpu-mid1-c2 {
+				compatible = "arm,idle-state";
+
+				entry-latency-us = <130>;
+				exit-latency-us = <274>;
+				min-residency-us = <2720>;
+
+				idle-state-name = "cpu-mid1-c2";
+				local-timer-stop;
+
+				arm,psci-suspend-param = <0x40000003>;
+			};
+
+			cpu_mid2_c2: cpu-mid2-c2 {
+				compatible = "arm,idle-state";
+
+				entry-latency-us = <130>;
+				exit-latency-us = <274>;
+				min-residency-us = <2720>;
+
+				idle-state-name = "cpu-mid2-c2";
+				local-timer-stop;
+
+				arm,psci-suspend-param = <0x40000003>;
+			};
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+
+		cpucl1: power-domain-cluster-1 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cluster_1_c3>;
+			power-domains = <&cpu_top_cl>;
+		};
+
+		cpucl2: power-domain-cluster-2 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cluster_2_c3>;
+			power-domains = <&cpu_top_cl>;
+		};
+
+		cpu_pd0: power-domain-cpu0 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_lit_c2>;
+			power-domains = <&cpu_top_cl>;
+		};
+
+		cpu_pd1: power-domain-cpu1 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_lit_c2>;
+			power-domains = <&cpu_top_cl>;
+		};
+
+		cpu_pd2: power-domain-cpu2 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_mid1_c2>;
+			power-domains = <&cpucl1>;
+		};
+
+		cpu_pd3: power-domain-cpu3 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_mid1_c2>;
+			power-domains = <&cpucl1>;
+		};
+
+		cpu_pd4: power-domain-cpu4 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_mid1_c2>;
+			power-domains = <&cpucl1>;
+		};
+
+		cpu_pd5: power-domain-cpu5 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_mid2_c2>;
+			power-domains = <&cpucl2>;
+		};
+
+		cpu_pd6: power-domain-cpu6 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_mid2_c2>;
+			power-domains = <&cpucl2>;
+		};
+
+		cpu_pd7: power-domain-cpu7 {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cpu_big_c2>;
+			power-domains = <&cpucl2>;
+		};
+
+		cpu_top_cl: power-domain-top-cluster {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cluster_c4_deep>;
+		};
+	};
+
+	reserved_memory: reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		/*
+		 * Store ramoops at a known location in DRAM, for access by the
+		 * ABL following a crash-reset.
+		 */
+		ramoops@95200000 {
+			compatible = "ramoops";
+			reg = <0x0 0x95200000 0x0 0x400000>;
+			console-size = <0x200000>;
+			pmsg-size = <0x200000>;
+		};
+
+		/* bootloader logs */
+		bldr_log_reserved: bldr-log-reserved@95600000 {
+			reg = <0x0 0x95600000 0x0 0x100000>;
+			no-map;
+		};
+
+		/* gsa logs */
+		gsa_reserved_log: gsa-reserved-log@a61b0000 {
+			reg = <0x0 0xa61b0000 0x0 0x00004000>;
+			no-map;
+		};
+
+		/* Android BootLoader (ABL) for ramdump/coredump processing */
+		abl@be000000 {
+			reg = <0x0 0xbe000000 0x0 0x1000000>;
+			no-map;
+		};
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+
+		dma-ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		gic: interrupt-controller@5880000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x05880000 0x0 0x10000>,
+			      <0x0 0x05900000 0x0 0x200000>;
+
+			#address-cells = <0>;
+			#interrupt-cells = <4>;
+			interrupt-controller;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			ppi-partitions {
+				ppi_cluster0: interrupt-partition-0 {
+					affinity = <&hayes_0 &hayes_1
+						    &hunter_0 &hunter_1 &hunter_2
+						    &hunter_3 &hunter_4>;
+				};
+
+				ppi_cluster1: interrupt-partition-1 {
+					affinity = <&hunterelp_0>;
+				};
+			};
+		};
+
+		lsion_cli16_uart: serial@db62000 {
+			compatible = "google,lga-uart", "snps,dw-apb-uart";
+			reg = <0x0 0x0db62000 0x0 0x100>;
+
+			clock-frequency = <200000000>;
+			interrupts = <GIC_SPI 688 IRQ_TYPE_LEVEL_HIGH 0>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+
+			status = "disabled";
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW 0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/hisilicon/hip07.dtsi b/arch/arm64/boot/dts/hisilicon/hip07.dtsi
index 00a6bfa7478cb4..d175e242ee96c4 100644
--- a/arch/arm64/boot/dts/hisilicon/hip07.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hip07.dtsi
@@ -1113,74 +1113,6 @@
 			num-pins = <2>;
 		};
 	};
-	p0_mbigen_alg_a:interrupt-controller@d0080000 {
-		compatible = "hisilicon,mbigen-v2";
-		reg = <0x0 0xd0080000 0x0 0x10000>;
-
-		p0_mbigen_sec_a: intc_sec {
-			msi-parent = <&p0_its_dsa_a 0x40400>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <33>;
-		};
-		p0_mbigen_smmu_alg_a: intc_smmu_alg {
-			msi-parent = <&p0_its_dsa_a 0x40b1b>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <3>;
-		};
-	};
-	p0_mbigen_alg_b:interrupt-controller@8,d0080000 {
-		compatible = "hisilicon,mbigen-v2";
-		reg = <0x8 0xd0080000 0x0 0x10000>;
-
-		p0_mbigen_sec_b: intc_sec {
-			msi-parent = <&p0_its_dsa_b 0x42400>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <33>;
-		};
-		p0_mbigen_smmu_alg_b: intc_smmu_alg {
-			msi-parent = <&p0_its_dsa_b 0x42b1b>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <3>;
-		};
-	};
-	p1_mbigen_alg_a:interrupt-controller@400,d0080000 {
-		compatible = "hisilicon,mbigen-v2";
-		reg = <0x400 0xd0080000 0x0 0x10000>;
-
-		p1_mbigen_sec_a: intc_sec {
-			msi-parent = <&p1_its_dsa_a 0x44400>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <33>;
-		};
-		p1_mbigen_smmu_alg_a: intc_smmu_alg {
-			msi-parent = <&p1_its_dsa_a 0x44b1b>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <3>;
-		};
-	};
-	p1_mbigen_alg_b:interrupt-controller@408,d0080000 {
-		compatible = "hisilicon,mbigen-v2";
-		reg = <0x408 0xd0080000 0x0 0x10000>;
-
-		p1_mbigen_sec_b: intc_sec {
-			msi-parent = <&p1_its_dsa_b 0x46400>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <33>;
-		};
-		p1_mbigen_smmu_alg_b: intc_smmu_alg {
-			msi-parent = <&p1_its_dsa_b 0x46b1b>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-			num-pins = <3>;
-		};
-	};
 	p0_mbigen_dsa_a: interrupt-controller@c0080000 {
 		compatible = "hisilicon,mbigen-v2";
 		reg = <0x0 0xc0080000 0x0 0x10000>;
@@ -1237,54 +1169,6 @@
 		hisilicon,broken-prefetch-cmd;
 		status = "disabled";
 	};
-	p0_smmu_alg_a: iommu@d0040000 {
-		compatible = "arm,smmu-v3";
-		reg = <0x0 0xd0040000 0x0 0x20000>;
-		interrupt-parent = <&p0_mbigen_smmu_alg_a>;
-		interrupts = <733 1>,
-		<734 1>,
-		<735 1>;
-		interrupt-names = "eventq", "gerror", "priq";
-		#iommu-cells = <1>;
-		dma-coherent;
-		hisilicon,broken-prefetch-cmd;
-	};
-	p0_smmu_alg_b: iommu@8d0040000 {
-		compatible = "arm,smmu-v3";
-		reg = <0x8 0xd0040000 0x0 0x20000>;
-		interrupt-parent = <&p0_mbigen_smmu_alg_b>;
-		interrupts = <733 1>,
-		<734 1>,
-		<735 1>;
-		interrupt-names = "eventq", "gerror", "priq";
-		#iommu-cells = <1>;
-		dma-coherent;
-		hisilicon,broken-prefetch-cmd;
-	};
-	p1_smmu_alg_a: iommu@400d0040000 {
-		compatible = "arm,smmu-v3";
-		reg = <0x400 0xd0040000 0x0 0x20000>;
-		interrupt-parent = <&p1_mbigen_smmu_alg_a>;
-		interrupts = <733 1>,
-		<734 1>,
-		<735 1>;
-		interrupt-names = "eventq", "gerror", "priq";
-		#iommu-cells = <1>;
-		dma-coherent;
-		hisilicon,broken-prefetch-cmd;
-	};
-	p1_smmu_alg_b: iommu@408d0040000 {
-		compatible = "arm,smmu-v3";
-		reg = <0x408 0xd0040000 0x0 0x20000>;
-		interrupt-parent = <&p1_mbigen_smmu_alg_b>;
-		interrupts = <733 1>,
-		<734 1>,
-		<735 1>;
-		interrupt-names = "eventq", "gerror", "priq";
-		#iommu-cells = <1>;
-		dma-coherent;
-		hisilicon,broken-prefetch-cmd;
-	};
 
 	soc {
 		compatible = "simple-bus";
@@ -1745,170 +1629,6 @@
 					 0x0 0 0 4 &mbigen_pcie2_a 671 4>;
 			status = "disabled";
 		};
-		p0_sec_a: crypto@d0000000 {
-			compatible = "hisilicon,hip07-sec";
-			reg = <0x0 0xd0000000 0x0 0x10000>,
-			      <0x0 0xd2000000 0x0 0x10000>,
-			      <0x0 0xd2010000 0x0 0x10000>,
-			      <0x0 0xd2020000 0x0 0x10000>,
-			      <0x0 0xd2030000 0x0 0x10000>,
-			      <0x0 0xd2040000 0x0 0x10000>,
-			      <0x0 0xd2050000 0x0 0x10000>,
-			      <0x0 0xd2060000 0x0 0x10000>,
-			      <0x0 0xd2070000 0x0 0x10000>,
-			      <0x0 0xd2080000 0x0 0x10000>,
-			      <0x0 0xd2090000 0x0 0x10000>,
-			      <0x0 0xd20a0000 0x0 0x10000>,
-			      <0x0 0xd20b0000 0x0 0x10000>,
-			      <0x0 0xd20c0000 0x0 0x10000>,
-			      <0x0 0xd20d0000 0x0 0x10000>,
-			      <0x0 0xd20e0000 0x0 0x10000>,
-			      <0x0 0xd20f0000 0x0 0x10000>,
-			      <0x0 0xd2100000 0x0 0x10000>;
-			interrupt-parent = <&p0_mbigen_sec_a>;
-			iommus = <&p0_smmu_alg_a 0x600>;
-			dma-coherent;
-			interrupts = <576 4>,
-				     <577 1>, <578 4>,
-				     <579 1>, <580 4>,
-				     <581 1>, <582 4>,
-				     <583 1>, <584 4>,
-				     <585 1>, <586 4>,
-				     <587 1>, <588 4>,
-				     <589 1>, <590 4>,
-				     <591 1>, <592 4>,
-				     <593 1>, <594 4>,
-				     <595 1>, <596 4>,
-				     <597 1>, <598 4>,
-				     <599 1>, <600 4>,
-				     <601 1>, <602 4>,
-				     <603 1>, <604 4>,
-				     <605 1>, <606 4>,
-				     <607 1>, <608 4>;
-		};
-		p0_sec_b: crypto@8d0000000 {
-			compatible = "hisilicon,hip07-sec";
-			reg = <0x8 0xd0000000 0x0 0x10000>,
-			      <0x8 0xd2000000 0x0 0x10000>,
-			      <0x8 0xd2010000 0x0 0x10000>,
-			      <0x8 0xd2020000 0x0 0x10000>,
-			      <0x8 0xd2030000 0x0 0x10000>,
-			      <0x8 0xd2040000 0x0 0x10000>,
-			      <0x8 0xd2050000 0x0 0x10000>,
-			      <0x8 0xd2060000 0x0 0x10000>,
-			      <0x8 0xd2070000 0x0 0x10000>,
-			      <0x8 0xd2080000 0x0 0x10000>,
-			      <0x8 0xd2090000 0x0 0x10000>,
-			      <0x8 0xd20a0000 0x0 0x10000>,
-			      <0x8 0xd20b0000 0x0 0x10000>,
-			      <0x8 0xd20c0000 0x0 0x10000>,
-			      <0x8 0xd20d0000 0x0 0x10000>,
-			      <0x8 0xd20e0000 0x0 0x10000>,
-			      <0x8 0xd20f0000 0x0 0x10000>,
-			      <0x8 0xd2100000 0x0 0x10000>;
-			interrupt-parent = <&p0_mbigen_sec_b>;
-			iommus = <&p0_smmu_alg_b 0x600>;
-			dma-coherent;
-			interrupts = <576 4>,
-				     <577 1>, <578 4>,
-				     <579 1>, <580 4>,
-				     <581 1>, <582 4>,
-				     <583 1>, <584 4>,
-				     <585 1>, <586 4>,
-				     <587 1>, <588 4>,
-				     <589 1>, <590 4>,
-				     <591 1>, <592 4>,
-				     <593 1>, <594 4>,
-				     <595 1>, <596 4>,
-				     <597 1>, <598 4>,
-				     <599 1>, <600 4>,
-				     <601 1>, <602 4>,
-				     <603 1>, <604 4>,
-				     <605 1>, <606 4>,
-				     <607 1>, <608 4>;
-		};
-		p1_sec_a: crypto@400d0000000 {
-			compatible = "hisilicon,hip07-sec";
-			reg = <0x400 0xd0000000 0x0 0x10000>,
-			      <0x400 0xd2000000 0x0 0x10000>,
-			      <0x400 0xd2010000 0x0 0x10000>,
-			      <0x400 0xd2020000 0x0 0x10000>,
-			      <0x400 0xd2030000 0x0 0x10000>,
-			      <0x400 0xd2040000 0x0 0x10000>,
-			      <0x400 0xd2050000 0x0 0x10000>,
-			      <0x400 0xd2060000 0x0 0x10000>,
-			      <0x400 0xd2070000 0x0 0x10000>,
-			      <0x400 0xd2080000 0x0 0x10000>,
-			      <0x400 0xd2090000 0x0 0x10000>,
-			      <0x400 0xd20a0000 0x0 0x10000>,
-			      <0x400 0xd20b0000 0x0 0x10000>,
-			      <0x400 0xd20c0000 0x0 0x10000>,
-			      <0x400 0xd20d0000 0x0 0x10000>,
-			      <0x400 0xd20e0000 0x0 0x10000>,
-			      <0x400 0xd20f0000 0x0 0x10000>,
-			      <0x400 0xd2100000 0x0 0x10000>;
-			interrupt-parent = <&p1_mbigen_sec_a>;
-			iommus = <&p1_smmu_alg_a 0x600>;
-			dma-coherent;
-			interrupts = <576 4>,
-				     <577 1>, <578 4>,
-				     <579 1>, <580 4>,
-				     <581 1>, <582 4>,
-				     <583 1>, <584 4>,
-				     <585 1>, <586 4>,
-				     <587 1>, <588 4>,
-				     <589 1>, <590 4>,
-				     <591 1>, <592 4>,
-				     <593 1>, <594 4>,
-				     <595 1>, <596 4>,
-				     <597 1>, <598 4>,
-				     <599 1>, <600 4>,
-				     <601 1>, <602 4>,
-				     <603 1>, <604 4>,
-				     <605 1>, <606 4>,
-				     <607 1>, <608 4>;
-		};
-		p1_sec_b: crypto@408d0000000 {
-			compatible = "hisilicon,hip07-sec";
-			reg = <0x408 0xd0000000 0x0 0x10000>,
-			      <0x408 0xd2000000 0x0 0x10000>,
-			      <0x408 0xd2010000 0x0 0x10000>,
-			      <0x408 0xd2020000 0x0 0x10000>,
-			      <0x408 0xd2030000 0x0 0x10000>,
-			      <0x408 0xd2040000 0x0 0x10000>,
-			      <0x408 0xd2050000 0x0 0x10000>,
-			      <0x408 0xd2060000 0x0 0x10000>,
-			      <0x408 0xd2070000 0x0 0x10000>,
-			      <0x408 0xd2080000 0x0 0x10000>,
-			      <0x408 0xd2090000 0x0 0x10000>,
-			      <0x408 0xd20a0000 0x0 0x10000>,
-			      <0x408 0xd20b0000 0x0 0x10000>,
-			      <0x408 0xd20c0000 0x0 0x10000>,
-			      <0x408 0xd20d0000 0x0 0x10000>,
-			      <0x408 0xd20e0000 0x0 0x10000>,
-			      <0x408 0xd20f0000 0x0 0x10000>,
-			      <0x408 0xd2100000 0x0 0x10000>;
-			interrupt-parent = <&p1_mbigen_sec_b>;
-			iommus = <&p1_smmu_alg_b 0x600>;
-			dma-coherent;
-			interrupts = <576 4>,
-				     <577 1>, <578 4>,
-				     <579 1>, <580 4>,
-				     <581 1>, <582 4>,
-				     <583 1>, <584 4>,
-				     <585 1>, <586 4>,
-				     <587 1>, <588 4>,
-				     <589 1>, <590 4>,
-				     <591 1>, <592 4>,
-				     <593 1>, <594 4>,
-				     <595 1>, <596 4>,
-				     <597 1>, <598 4>,
-				     <599 1>, <600 4>,
-				     <601 1>, <602 4>,
-				     <603 1>, <604 4>,
-				     <605 1>, <606 4>,
-				     <607 1>, <608 4>;
-		};
 
 	};
 };
diff --git a/arch/arm64/boot/dts/intel/Makefile b/arch/arm64/boot/dts/intel/Makefile
index 270c70fdf084b1..b52cd2330cef87 100644
--- a/arch/arm64/boot/dts/intel/Makefile
+++ b/arch/arm64/boot/dts/intel/Makefile
@@ -6,8 +6,10 @@ dtb-$(CONFIG_ARCH_INTEL_SOCFPGA) += socfpga_agilex_n6000.dtb \
 				socfpga_agilex3_socdk.dtb \
 				socfpga_agilex5_socdk.dtb \
 				socfpga_agilex5_socdk_013b.dtb \
+				socfpga_agilex5_socdk_emmc.dtb \
 				socfpga_agilex5_socdk_modular.dtb \
 				socfpga_agilex5_socdk_nand.dtb \
+				socfpga_agilex5_socdk_tsn_cfg2.dtb \
 				socfpga_agilex72_socdk.dtb \
 				socfpga_agilex7m_socdk.dtb \
 				socfpga_n5x_socdk.dtb
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
index 0dfbafde882225..3168449dcd3dc4 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex.dtsi
@@ -65,6 +65,8 @@
 			compatible = "intel,agilex-svc";
 			method = "smc";
 			memory-region = <&service_reserved>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&intc>;
 
 			fpga_mgr: fpga-mgr {
 				compatible = "intel,agilex-soc-fpga-mgr";
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
index f54767d1526e11..0c501a93a8127c 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
@@ -85,9 +85,20 @@
 			method = "smc";
 			memory-region = <&service_reserved>;
 			iommus = <&smmu 10>;
+
+			fpga_mgr: fpga-mgr {
+				compatible = "intel,agilex-soc-fpga-mgr";
+			};
 		};
 	};
 
+	fpga-region {
+		compatible = "fpga-region";
+		#address-cells = <0x2>;
+		#size-cells = <0x2>;
+		fpga-mgr = <&fpga_mgr>;
+	};
+
 	psci {
 		compatible = "arm,psci-0.2";
 		method = "smc";
@@ -300,6 +311,21 @@
 			};
 		};
 
+		sdmmc: mmc@10808000 {
+			compatible = "altr,agilex5-sd6hc", "cdns,sd6hc";
+			reg = <0x10808000 0x1000>;
+			clocks = <&clkmgr AGILEX5_SDMCLK>, <&clkmgr AGILEX5_L4_MP_CLK>;
+			clock-names = "ciu", "biu";
+			dma-coherent;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			iommus = <&smmu 5>;
+			resets = <&rst SDMMC_RESET>, <&rst COMBOPHY_RESET>,
+				 <&rst SDMMC_OCP_RESET>;
+			reset-names = "sdhc-reset", "combophy", "sdmmc-ocp";
+
+			status = "disabled";
+		};
+
 		nand: nand-controller@10b80000 {
 			compatible = "cdns,hp-nfc";
 			reg = <0x10b80000 0x10000>,
@@ -493,6 +519,41 @@
 			status = "disabled";
 		};
 
+		usb31_bus: bus@11000000 {
+			compatible = "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x11000000 0x0 0x100000>;
+			/*
+			 * USB interconnect DMA is limited to 40-bit addresses
+			 * (1 TiB window from 0).
+			 */
+			dma-ranges = <0x0 0x0 0x0 0x100 0x0>;
+
+			usb31: usb@0 {
+				compatible = "altr,agilex5-dwc3";
+				reg = <0x0 0x0 0x0 0x100000>;
+				interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+				/*
+				 * ref and suspend share the same hardware
+				 * source; reuse the published suspend clock ID.
+				 */
+				clocks = <&clkmgr AGILEX5_USB31_BUS_CLK_EARLY>,
+					 <&clkmgr AGILEX5_USB31_SUSPEND_CLK>,
+					 <&clkmgr AGILEX5_USB31_SUSPEND_CLK>;
+				clock-names = "bus_early", "ref", "suspend";
+				resets = <&rst USB1_RESET>, <&rst USB1_OCP_RESET>;
+				reset-names = "core", "ecc";
+				iommus = <&smmu 7>;
+				phys = <&usbphy0>, <&usbphy0>;
+				phy-names = "usb2-phy", "usb3-phy";
+				maximum-speed = "super-speed";
+				snps,dis_u2_susphy_quirk;
+				snps,dis_u3_susphy_quirk;
+				status = "disabled";
+			};
+		};
+
 		watchdog0: watchdog@10d00200 {
 			compatible = "snps,dw-wdt";
 			reg = <0x10d00200 0x100>;
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts
index 1e9b87d96765d1..5f3eb96d038eb9 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts
@@ -36,9 +36,37 @@
 		/* We expect the bootloader to fill in the reg */
 		reg = <0x0 0x80000000 0x0 0x0>;
 	};
+
+	vqmmc_io_reg: regulator-1p8v {
+		compatible = "regulator-gpio";
+		gpios = <&portb 3 GPIO_ACTIVE_HIGH>;
+		gpios-states = <0>;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "vqmmc-io";
+		states = <1800000 0x1>,
+			 <3300000 0x0>;
+	};
+
+	vmmc_reg: regulator-fixed-3p3v {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc-sd";
+	};
 };
 
-&gpio0 {
+&sdmmc {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	disable-wp;
+	max-frequency = <200000000>;
+	no-mmc;
+	sd-uhs-sdr104;
+	vmmc-supply = <&vmmc_reg>;
+	vqmmc-supply = <&vqmmc_io_reg>;
+
 	status = "okay";
 };
 
@@ -59,6 +87,10 @@
 	};
 };
 
+&gpio0 {
+	status = "okay";
+};
+
 &gpio1 {
 	status = "okay";
 };
@@ -104,9 +136,9 @@
 	status = "okay";
 };
 
-&usb0 {
+&usb31 {
+	dr_mode = "host";
 	status = "okay";
-	disable-over-current;
 };
 
 &watchdog0 {
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_emmc.dts b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_emmc.dts
new file mode 100644
index 00000000000000..beb4f444c471fd
--- /dev/null
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_emmc.dts
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026, Altera Corporation
+ */
+#include "socfpga_agilex5.dtsi"
+
+/ {
+	model = "SoCFPGA Agilex5 SoCDK eMMC daughter board";
+	compatible = "intel,socfpga-agilex5-socdk-emmc", "intel,socfpga-agilex5";
+
+	aliases {
+		ethernet0 = &gmac0;
+		i3c0 = &i3c0;
+		i3c1 = &i3c1;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			label = "hps_led0";
+			gpios = <&porta 6 GPIO_ACTIVE_HIGH>;
+		};
+
+		led-1 {
+			label = "hps_led1";
+			gpios = <&porta 7 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* Expect the bootloader to fill in the reg */
+		reg = <0x0 0x80000000 0x0 0x0>;
+	};
+
+	vqmmc_io_reg: regulator-fixed-1p8v {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "vqmmc-io";
+	};
+
+	vmmc_reg: regulator-fixed-3p3v {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc-emmc";
+	};
+};
+
+&sdmmc {
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	disable-wp;
+	max-frequency = <200000000>;
+	mmc-hs200-1_8v;
+	mmc-hs400-1_8v;
+	no-sd;
+	no-sdio;
+	non-removable;
+	vmmc-supply = <&vmmc_reg>;
+	vqmmc-supply = <&vqmmc_io_reg>;
+
+	status = "okay";
+};
+
+&gmac0 {
+	max-frame-size = <9000>;
+	phy-handle = <&emac0_phy0>;
+	phy-mode = "rgmii"; /* TX/RX clock delays provided by Agilex5 I/O hardware */
+
+	status = "okay";
+
+	mdio0 {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		emac0_phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
+	};
+};
+
+&gpio0 {
+	status = "okay";
+};
+
+&gpio1 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+};
+
+&i3c0 {
+	status = "okay";
+};
+
+&i3c1 {
+	status = "okay";
+};
+
+&osc1 {
+	clock-frequency = <25000000>;
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&watchdog0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_tsn_cfg2.dts b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_tsn_cfg2.dts
new file mode 100644
index 00000000000000..9d39f843b42408
--- /dev/null
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_tsn_cfg2.dts
@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026, Altera Corporation
+ */
+#include "socfpga_agilex5.dtsi"
+
+/ {
+	model = "SoCFPGA Agilex5 SoCDK TSN Config2";
+	compatible = "intel,socfpga-agilex5-socdk-tsn-cfg2", "intel,socfpga-agilex5";
+
+	aliases {
+		serial0 = &uart0;
+		ethernet1 = &gmac1;
+		ethernet2 = &gmac2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			label = "hps_led0";
+			gpios = <&porta 11 GPIO_ACTIVE_HIGH>;
+		};
+
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the reg */
+		reg = <0x0 0x80000000 0x0 0x0>;
+	};
+};
+
+&gpio0 {
+	status = "okay";
+};
+
+&gpio1 {
+	status = "okay";
+};
+
+&gmac1 {
+	status = "okay";
+	compatible = "altr,socfpga-stmmac-agilex5-tsn",
+		     "snps,dwxgmac-2.10";
+	phy-mode = "rgmii-id";
+	phy-handle = <&emac1_phy0>;
+	max-frame-size = <9000>;
+
+	mdio0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		emac1_phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
+	};
+};
+
+&gmac2 {
+	status = "okay";
+	phy-mode = "rgmii-id";
+	phy-handle = <&emac2_phy0>;
+	max-frame-size = <9000>;
+
+	mdio0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		emac2_phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
+	};
+};
+
+&osc1 {
+	clock-frequency = <25000000>;
+};
+
+&qspi {
+	status = "okay";
+	flash@0 {
+		compatible = "micron,mt25qu02g", "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <100000000>;
+		m25p,fast-read;
+		cdns,read-delay = <2>;
+		cdns,tshsl-ns = <50>;
+		cdns,tsd2d-ns = <50>;
+		cdns,tchsh-ns = <4>;
+		cdns,tslch-ns = <4>;
+		spi-tx-bus-width = <4>;
+		spi-rx-bus-width = <4>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			qspi_boot: partition@0 {
+				label = "u-boot";
+				reg = <0x0 0x04200000>;
+			};
+
+			root: partition@4200000 {
+				label = "root";
+				reg = <0x04200000 0x0be00000>;
+			};
+		};
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+	disable-over-current;
+};
+
+&watchdog0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt6878-pinfunc.h b/arch/arm64/boot/dts/mediatek/mt6878-pinfunc.h
index 4e8e475a74549b..5aeceb904b18d3 100644
--- a/arch/arm64/boot/dts/mediatek/mt6878-pinfunc.h
+++ b/arch/arm64/boot/dts/mediatek/mt6878-pinfunc.h
@@ -9,7 +9,7 @@
 #ifndef __MT6878_PINFUNC_H
 #define __MT6878_PINFUNC_H
 
-#include "mt65xx.h"
+#include <dt-bindings/pinctrl/mt65xx.h>
 
 #define PINMUX_GPIO0__FUNC_GPIO0 (MTK_PIN_NO(0) | 0)
 #define PINMUX_GPIO0__FUNC_TP_GPIO0_AO (MTK_PIN_NO(0) | 1)
diff --git a/arch/arm64/boot/dts/mediatek/mt6893-pinfunc.h b/arch/arm64/boot/dts/mediatek/mt6893-pinfunc.h
index 982bc95c471ccf..7cda5c73fb6847 100644
--- a/arch/arm64/boot/dts/mediatek/mt6893-pinfunc.h
+++ b/arch/arm64/boot/dts/mediatek/mt6893-pinfunc.h
@@ -7,7 +7,7 @@
 #ifndef __MT6893_PINFUNC_H
 #define __MT6893_PINFUNC_H
 
-#include "mt65xx.h"
+#include <dt-bindings/pinctrl/mt65xx.h>
 
 #define PINMUX_GPIO0__FUNC_GPIO0 (MTK_PIN_NO(0) | 0)
 #define PINMUX_GPIO0__FUNC_SPI6_CLK (MTK_PIN_NO(0) | 1)
diff --git a/arch/arm64/boot/dts/mediatek/mt7986a-bananapi-bpi-r3.dts b/arch/arm64/boot/dts/mediatek/mt7986a-bananapi-bpi-r3.dts
index 637e5567c1e08f..70c693fd0c52ac 100644
--- a/arch/arm64/boot/dts/mediatek/mt7986a-bananapi-bpi-r3.dts
+++ b/arch/arm64/boot/dts/mediatek/mt7986a-bananapi-bpi-r3.dts
@@ -140,6 +140,19 @@
 		tx-disable-gpios = <&pio 15 GPIO_ACTIVE_HIGH>;
 		tx-fault-gpios = <&pio 48 GPIO_ACTIVE_HIGH>;
 	};
+
+	reserved-memory {
+		ramoops@42ff0000 {
+			compatible = "ramoops";
+			reg = <0 0x42ff0000 0 0x10000>;
+			no-map;
+			record-size = <0x2000>;
+			console-size = <0x8000>;
+			ftrace-size = <0x2000>;
+			pmsg-size = <0x2000>;
+			ecc-size = <16>;
+		};
+	};
 };
 
 &cpu_thermal {
diff --git a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi
index 9ebc196107e5a0..318639a7c64e95 100644
--- a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi
@@ -657,6 +657,7 @@
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupt-parent = <&gic>;
+		clock-frequency = <12986200>;
 		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
 			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
 			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso
index 973b76ba0cbfba..e9c322df15fc89 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso
@@ -16,5 +16,6 @@
 };
 
 &{/soc/pinctrl@1001f000/pcie-2-hog} {
+	line-name = "pcie2-cn13-hog";
 	output-high;
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso
index 90b2a64459c316..9d8de737ca4001 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso
@@ -16,5 +16,6 @@
 };
 
 &{/soc/pinctrl@1001f000/pcie-3-hog} {
+	line-name = "pcie3-cn14-hog";
 	output-high;
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn15.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn15.dtso
index 9750916042deb2..57ee7dacdd7801 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn15.dtso
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn15.dtso
@@ -16,5 +16,6 @@
 };
 
 &{/soc/pinctrl@1001f000/pcie-2-hog} {
+	line-name = "pcie2-cn15-hog";
 	output-low;
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn18.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn18.dtso
index 9830fb0fd97a2f..7830980ac95afa 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn18.dtso
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn18.dtso
@@ -16,5 +16,6 @@
 };
 
 &{/soc/pinctrl@1001f000/pcie-3-hog} {
+	line-name = "pcie3-cn18-hog";
 	output-low;
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi
index 1eeb72108b9b75..f54787fbbc71af 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi
@@ -315,6 +315,24 @@
 				reg = <0x20>;
 				gpio-controller;
 				#gpio-cells = <2>;
+
+				gpio-line-names =
+					"BKEY-A-PDN",	/* GPIO0 */
+					"BKEY-A-Vset",	/* GPIO1 */
+					"BKEY-B-PDN",	/* GPIO2 */
+					"BKEY-B-Vset",	/* GPIO3 */
+					"BKEY-C-PDN",	/* GPIO4 */
+					"BKEY-C-Vset",	/* GPIO5 */
+					"SIM_SEL",	/* GPIO6 */
+					"mPCIe_AB_Vset",/* GPIO7 */
+					"mPCIe_A_PDN",	/* GPIO8 */
+					"mPCIe_B_PDN",	/* GPIO9 */
+					"BKEY-A-RST",	/* GPIO10 */
+					"VBUS_AB_PDN",	/* GPIO11 */
+					"BKEY-B-RST",	/* GPIO12 */
+					"BKEY-C-RST",	/* GPIO13 */
+					"SYS_LEDB",	/* GPIO14 */
+					"SYS_LEDR";	/* GPIO15 */
 			};
 
 			rtc@51 {
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4.dtsi b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4.dtsi
index 0ff69dae45d327..75f59407794f7d 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4.dtsi
@@ -66,6 +66,19 @@
 		regulator-always-on;
 	};
 
+	reserved-memory {
+		ramoops@42f00000 {
+			compatible = "ramoops";
+			reg = <0 0x42f00000 0 0x100000>;
+			no-map;
+			record-size = <0x20000>;
+			console-size = <0x40000>;
+			ftrace-size = <0x10000>;
+			pmsg-size = <0x10000>;
+			ecc-size = <16>;
+		};
+	};
+
 	/* SFP1 cage (WAN) */
 	sfp1: sfp1 {
 		compatible = "sff,sfp";
diff --git a/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi b/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi
index a0573bc359fb65..777da2129e77fc 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8173-elm.dtsi
@@ -16,6 +16,11 @@
 		mmc2 = &mmc3;
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	memory@40000000 {
 		device_type = "memory";
 		reg = <0 0x40000000 0 0x80000000>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
index 78c2ccd5be132d..66e68702732d6b 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
@@ -505,7 +505,7 @@
 				};
 				mfg_async: power-domain@MT8173_POWER_DOMAIN_MFG_ASYNC {
 					reg = <MT8173_POWER_DOMAIN_MFG_ASYNC>;
-					clocks = <&clk26m>;
+					clocks = <&topckgen CLK_TOP_AXI_MFG_IN_SEL>;
 					clock-names = "mfg";
 					#address-cells = <1>;
 					#size-cells = <0>;
@@ -1002,6 +1002,37 @@
 			};
 		};
 
+		gpu: gpu@13000000 {
+			compatible = "mediatek,mt8173-gpu", "img,img-gx6250",
+				     "img,img-rogue";
+			reg = <0 0x13000000 0 0x10000>;
+			interrupts = <GIC_SPI 217 IRQ_TYPE_LEVEL_LOW>;
+			clocks = <&mfgtop CLK_MFG_G3D>,
+				 <&mfgtop CLK_MFG_MEM>,
+				 <&mfgtop CLK_MFG_AXI>;
+			clock-names = "core", "mem", "sys";
+			/*
+			 * Binding and GPU core HW have two power domain
+			 * inputs, but MediaTek does not have information
+			 * on how this is done internally.
+			 */
+			power-domains = <&mfgtop>, <&mfgtop>;
+			power-domain-names = "a", "b";
+		};
+
+		mfgtop: clock-controller@13fff000 {
+			compatible = "mediatek,mt8173-mfgtop";
+			reg = <0 0x13fff000 0 0x1000>;
+			clocks = <&topckgen CLK_TOP_AXI_MFG_IN_SEL>,
+				 <&topckgen CLK_TOP_MEM_MFG_IN_SEL>,
+				 <&topckgen CLK_TOP_MFG_SEL>,
+				 <&clk26m>;
+			clock-names = "sys", "mem", "core", "clk26m";
+			power-domains = <&spm MT8173_POWER_DOMAIN_MFG>;
+			#clock-cells = <1>;
+			#power-domain-cells = <0>;
+		};
+
 		mmsys: syscon@14000000 {
 			compatible = "mediatek,mt8173-mmsys", "syscon";
 			reg = <0 0x14000000 0 0x1000>;
@@ -1291,7 +1322,7 @@
 			clocks = <&mmsys CLK_MM_MUTEX_32K>;
 			mediatek,gce-client-reg = <&gce SUBSYS_1402XXXX 0 0x1000>;
 			mediatek,gce-events = <CMDQ_EVENT_MUTEX0_STREAM_EOF>,
-                                              <CMDQ_EVENT_MUTEX1_STREAM_EOF>;
+					      <CMDQ_EVENT_MUTEX1_STREAM_EOF>;
 		};
 
 		larb0: larb@14021000 {
diff --git a/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi b/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi
index 69cab9f9c53082..acd5f59b0d4c68 100644
--- a/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8183-kukui.dtsi
@@ -22,6 +22,11 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	backlight_lcd0: backlight_lcd0 {
 		compatible = "pwm-backlight";
 		pwms = <&pwm0 0 500000>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8186-corsola.dtsi b/arch/arm64/boot/dts/mediatek/mt8186-corsola.dtsi
index ff20376a44d7a5..2b327d9ef65eee 100644
--- a/arch/arm64/boot/dts/mediatek/mt8186-corsola.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8186-corsola.dtsi
@@ -26,6 +26,11 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	memory@40000000 {
 		device_type = "memory";
 		/* The size should be filled in by the bootloader. */
diff --git a/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi b/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
index f382f90c48f5e6..801eee735afe61 100644
--- a/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
@@ -41,6 +41,11 @@
 		wakeup-delay-ms = <100>;
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	memory@40000000 {
 		device_type = "memory";
 		/* The size will be filled in by the bootloader */
@@ -166,7 +171,7 @@
 
 		scp_mem_reserved: memory@50000000 {
 			compatible = "shared-dma-pool";
-			reg = <0 0x50000000 0 0x800000>;
+			reg = <0 0x50000000 0 0x900000>;
 			no-map;
 		};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi b/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi
index 40d34b16dc807f..e8ebbb74987fa6 100644
--- a/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi
@@ -25,6 +25,11 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	memory@40000000 {
 		device_type = "memory";
 		reg = <0 0x40000000 0 0x80000000>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
index ef7afc436aefc4..f06ca2bc165da8 100644
--- a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
@@ -42,6 +42,11 @@
 		wakeup-delay-ms = <50>;
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	memory@40000000 {
 		device_type = "memory";
 		reg = <0 0x40000000 0 0x80000000>;
diff --git a/arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts b/arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts
index 4012ea7d07bbdb..a7b4be16c9eeb0 100644
--- a/arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts
+++ b/arch/arm64/boot/dts/microchip/lan9696-ev23x71a.dts
@@ -463,6 +463,19 @@
 	};
 };
 
+&qspi0 {
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <100000000>;
+		spi-tx-bus-width = <1>;
+		spi-rx-bus-width = <4>;
+		m25p,fast-read;
+	};
+};
+
 &serdes {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra132.dtsi b/arch/arm64/boot/dts/nvidia/tegra132.dtsi
index 26cd11a8a4a12c..3463fa637b37a5 100644
--- a/arch/arm64/boot/dts/nvidia/tegra132.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra132.dtsi
@@ -936,13 +936,13 @@
 			    "rx3", "tx3", "rx4", "tx4", "rx5", "tx5",
 			    "rx6", "tx6", "rx7", "tx7", "rx8", "tx8",
 			    "rx9", "tx9";
-		ranges;
-		#address-cells = <2>;
-		#size-cells = <2>;
+		ranges = <0x70300000 0x0 0x70300000 0x10000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
 
 		tegra_i2s0: i2s@70301000 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301000 0x0 0x100>;
+			reg = <0x70301000 0x100>;
 			nvidia,ahub-cif-ids = <4 4>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S0>;
 			clock-names = "i2s";
@@ -953,7 +953,7 @@
 
 		tegra_i2s1: i2s@70301100 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301100 0x0 0x100>;
+			reg = <0x70301100 0x100>;
 			nvidia,ahub-cif-ids = <5 5>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S1>;
 			clock-names = "i2s";
@@ -964,7 +964,7 @@
 
 		tegra_i2s2: i2s@70301200 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301200 0x0 0x100>;
+			reg = <0x70301200 0x100>;
 			nvidia,ahub-cif-ids = <6 6>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S2>;
 			clock-names = "i2s";
@@ -975,7 +975,7 @@
 
 		tegra_i2s3: i2s@70301300 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301300 0x0 0x100>;
+			reg = <0x70301300 0x100>;
 			nvidia,ahub-cif-ids = <7 7>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S3>;
 			clock-names = "i2s";
@@ -986,7 +986,7 @@
 
 		tegra_i2s4: i2s@70301400 {
 			compatible = "nvidia,tegra124-i2s";
-			reg = <0x0 0x70301400 0x0 0x100>;
+			reg = <0x70301400 0x100>;
 			nvidia,ahub-cif-ids = <8 8>;
 			clocks = <&tegra_car TEGRA124_CLK_I2S4>;
 			clock-names = "i2s";
diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts b/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts
index d9aafe05311215..4b2fa1a49f0512 100644
--- a/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra186-p3509-0000+p3636-0001.dts
@@ -1176,22 +1176,22 @@
 			};
 
 			cooling-maps {
-				cpu-critical {
+				map-critical {
 					cooling-device = <&fan 3 3>;
 					trip = <&cpu_trip_critical>;
 				};
 
-				cpu-hot {
+				map-hot {
 					cooling-device = <&fan 2 2>;
 					trip = <&cpu_trip_hot>;
 				};
 
-				cpu-active {
+				map-active {
 					cooling-device = <&fan 1 1>;
 					trip = <&cpu_trip_active>;
 				};
 
-				cpu-passive {
+				map-passive {
 					cooling-device = <&fan 0 0>;
 					trip = <&cpu_trip_passive>;
 				};
diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
index 705af0373a0901..623d5e65432351 100644
--- a/arch/arm64/boot/dts/nvidia/tegra186.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
@@ -142,15 +142,15 @@
 			assigned-clocks = <&bpmp TEGRA186_CLK_AHUB>;
 			assigned-clock-parents = <&bpmp TEGRA186_CLK_PLLP_OUT0>;
 			assigned-clock-rates = <81600000>;
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges = <0x0 0x02900800 0x0 0x02900800 0x0 0x11800>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x02900000 0x0 0x02900000 0x20000>;
 			status = "disabled";
 
 			tegra_i2s1: i2s@2901000 {
 				compatible = "nvidia,tegra186-i2s",
 					     "nvidia,tegra210-i2s";
-				reg = <0x0 0x2901000 0x0 0x100>;
+				reg = <0x2901000 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_I2S1>,
 					 <&bpmp TEGRA186_CLK_I2S1_SYNC_INPUT>;
 				clock-names = "i2s", "sync_input";
@@ -164,7 +164,7 @@
 			tegra_i2s2: i2s@2901100 {
 				compatible = "nvidia,tegra186-i2s",
 					     "nvidia,tegra210-i2s";
-				reg = <0x0 0x2901100 0x0 0x100>;
+				reg = <0x2901100 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_I2S2>,
 					 <&bpmp TEGRA186_CLK_I2S2_SYNC_INPUT>;
 				clock-names = "i2s", "sync_input";
@@ -178,7 +178,7 @@
 			tegra_i2s3: i2s@2901200 {
 				compatible = "nvidia,tegra186-i2s",
 					     "nvidia,tegra210-i2s";
-				reg = <0x0 0x2901200 0x0 0x100>;
+				reg = <0x2901200 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_I2S3>,
 					 <&bpmp TEGRA186_CLK_I2S3_SYNC_INPUT>;
 				clock-names = "i2s", "sync_input";
@@ -192,7 +192,7 @@
 			tegra_i2s4: i2s@2901300 {
 				compatible = "nvidia,tegra186-i2s",
 					     "nvidia,tegra210-i2s";
-				reg = <0x0 0x2901300 0x0 0x100>;
+				reg = <0x2901300 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_I2S4>,
 					 <&bpmp TEGRA186_CLK_I2S4_SYNC_INPUT>;
 				clock-names = "i2s", "sync_input";
@@ -206,7 +206,7 @@
 			tegra_i2s5: i2s@2901400 {
 				compatible = "nvidia,tegra186-i2s",
 					     "nvidia,tegra210-i2s";
-				reg = <0x0 0x2901400 0x0 0x100>;
+				reg = <0x2901400 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_I2S5>,
 					 <&bpmp TEGRA186_CLK_I2S5_SYNC_INPUT>;
 				clock-names = "i2s", "sync_input";
@@ -220,7 +220,7 @@
 			tegra_i2s6: i2s@2901500 {
 				compatible = "nvidia,tegra186-i2s",
 					     "nvidia,tegra210-i2s";
-				reg = <0x0 0x2901500 0x0 0x100>;
+				reg = <0x2901500 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_I2S6>,
 					 <&bpmp TEGRA186_CLK_I2S6_SYNC_INPUT>;
 				clock-names = "i2s", "sync_input";
@@ -234,7 +234,7 @@
 			tegra_sfc1: sfc@2902000 {
 				compatible = "nvidia,tegra186-sfc",
 					     "nvidia,tegra210-sfc";
-				reg = <0x0 0x2902000 0x0 0x200>;
+				reg = <0x2902000 0x200>;
 				sound-name-prefix = "SFC1";
 				status = "disabled";
 			};
@@ -242,7 +242,7 @@
 			tegra_sfc2: sfc@2902200 {
 				compatible = "nvidia,tegra186-sfc",
 					     "nvidia,tegra210-sfc";
-				reg = <0x0 0x2902200 0x0 0x200>;
+				reg = <0x2902200 0x200>;
 				sound-name-prefix = "SFC2";
 				status = "disabled";
 			};
@@ -250,7 +250,7 @@
 			tegra_sfc3: sfc@2902400 {
 				compatible = "nvidia,tegra186-sfc",
 					     "nvidia,tegra210-sfc";
-				reg = <0x0 0x2902400 0x0 0x200>;
+				reg = <0x2902400 0x200>;
 				sound-name-prefix = "SFC3";
 				status = "disabled";
 			};
@@ -258,7 +258,7 @@
 			tegra_sfc4: sfc@2902600 {
 				compatible = "nvidia,tegra186-sfc",
 					     "nvidia,tegra210-sfc";
-				reg = <0x0 0x2902600 0x0 0x200>;
+				reg = <0x2902600 0x200>;
 				sound-name-prefix = "SFC4";
 				status = "disabled";
 			};
@@ -266,7 +266,7 @@
 			tegra_amx1: amx@2903000 {
 				compatible = "nvidia,tegra186-amx",
 					     "nvidia,tegra210-amx";
-				reg = <0x0 0x2903000 0x0 0x100>;
+				reg = <0x2903000 0x100>;
 				sound-name-prefix = "AMX1";
 				status = "disabled";
 			};
@@ -274,7 +274,7 @@
 			tegra_amx2: amx@2903100 {
 				compatible = "nvidia,tegra186-amx",
 					     "nvidia,tegra210-amx";
-				reg = <0x0 0x2903100 0x0 0x100>;
+				reg = <0x2903100 0x100>;
 				sound-name-prefix = "AMX2";
 				status = "disabled";
 			};
@@ -282,7 +282,7 @@
 			tegra_amx3: amx@2903200 {
 				compatible = "nvidia,tegra186-amx",
 					     "nvidia,tegra210-amx";
-				reg = <0x0 0x2903200 0x0 0x100>;
+				reg = <0x2903200 0x100>;
 				sound-name-prefix = "AMX3";
 				status = "disabled";
 			};
@@ -290,7 +290,7 @@
 			tegra_amx4: amx@2903300 {
 				compatible = "nvidia,tegra186-amx",
 					     "nvidia,tegra210-amx";
-				reg = <0x0 0x2903300 0x0 0x100>;
+				reg = <0x2903300 0x100>;
 				sound-name-prefix = "AMX4";
 				status = "disabled";
 			};
@@ -298,7 +298,7 @@
 			tegra_adx1: adx@2903800 {
 				compatible = "nvidia,tegra186-adx",
 					     "nvidia,tegra210-adx";
-				reg = <0x0 0x2903800 0x0 0x100>;
+				reg = <0x2903800 0x100>;
 				sound-name-prefix = "ADX1";
 				status = "disabled";
 			};
@@ -306,7 +306,7 @@
 			tegra_adx2: adx@2903900 {
 				compatible = "nvidia,tegra186-adx",
 					     "nvidia,tegra210-adx";
-				reg = <0x0 0x2903900 0x0 0x100>;
+				reg = <0x2903900 0x100>;
 				sound-name-prefix = "ADX2";
 				status = "disabled";
 			};
@@ -314,7 +314,7 @@
 			tegra_adx3: adx@2903a00 {
 				compatible = "nvidia,tegra186-adx",
 					     "nvidia,tegra210-adx";
-				reg = <0x0 0x2903a00 0x0 0x100>;
+				reg = <0x2903a00 0x100>;
 				sound-name-prefix = "ADX3";
 				status = "disabled";
 			};
@@ -322,14 +322,14 @@
 			tegra_adx4: adx@2903b00 {
 				compatible = "nvidia,tegra186-adx",
 					     "nvidia,tegra210-adx";
-				reg = <0x0 0x2903b00 0x0 0x100>;
+				reg = <0x2903b00 0x100>;
 				sound-name-prefix = "ADX4";
 				status = "disabled";
 			};
 
 			tegra_dmic1: dmic@2904000 {
 				compatible = "nvidia,tegra210-dmic";
-				reg = <0x0 0x2904000 0x0 0x100>;
+				reg = <0x2904000 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_DMIC1>;
 				clock-names = "dmic";
 				assigned-clocks = <&bpmp TEGRA186_CLK_DMIC1>;
@@ -341,7 +341,7 @@
 
 			tegra_dmic2: dmic@2904100 {
 				compatible = "nvidia,tegra210-dmic";
-				reg = <0x0 0x2904100 0x0 0x100>;
+				reg = <0x2904100 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_DMIC2>;
 				clock-names = "dmic";
 				assigned-clocks = <&bpmp TEGRA186_CLK_DMIC2>;
@@ -353,7 +353,7 @@
 
 			tegra_dmic3: dmic@2904200 {
 				compatible = "nvidia,tegra210-dmic";
-				reg = <0x0 0x2904200 0x0 0x100>;
+				reg = <0x2904200 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_DMIC3>;
 				clock-names = "dmic";
 				assigned-clocks = <&bpmp TEGRA186_CLK_DMIC3>;
@@ -365,7 +365,7 @@
 
 			tegra_dmic4: dmic@2904300 {
 				compatible = "nvidia,tegra210-dmic";
-				reg = <0x0 0x2904300 0x0 0x100>;
+				reg = <0x2904300 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_DMIC4>;
 				clock-names = "dmic";
 				assigned-clocks = <&bpmp TEGRA186_CLK_DMIC4>;
@@ -377,7 +377,7 @@
 
 			tegra_dspk1: dspk@2905000 {
 				compatible = "nvidia,tegra186-dspk";
-				reg = <0x0 0x2905000 0x0 0x100>;
+				reg = <0x2905000 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_DSPK1>;
 				clock-names = "dspk";
 				assigned-clocks = <&bpmp TEGRA186_CLK_DSPK1>;
@@ -389,7 +389,7 @@
 
 			tegra_dspk2: dspk@2905100 {
 				compatible = "nvidia,tegra186-dspk";
-				reg = <0x0 0x2905100 0x0 0x100>;
+				reg = <0x2905100 0x100>;
 				clocks = <&bpmp TEGRA186_CLK_DSPK2>;
 				clock-names = "dspk";
 				assigned-clocks = <&bpmp TEGRA186_CLK_DSPK2>;
@@ -402,9 +402,9 @@
 			tegra_ope1: processing-engine@2908000 {
 				compatible = "nvidia,tegra186-ope",
 					     "nvidia,tegra210-ope";
-				reg = <0x0 0x2908000 0x0 0x100>;
-				#address-cells = <2>;
-				#size-cells = <2>;
+				reg = <0x2908000 0x100>;
+				#address-cells = <1>;
+				#size-cells = <1>;
 				ranges;
 				sound-name-prefix = "OPE1";
 				status = "disabled";
@@ -412,20 +412,20 @@
 				equalizer@2908100 {
 					compatible = "nvidia,tegra186-peq",
 						     "nvidia,tegra210-peq";
-					reg = <0x0 0x2908100 0x0 0x100>;
+					reg = <0x2908100 0x100>;
 				};
 
 				dynamic-range-compressor@2908200 {
 					compatible = "nvidia,tegra186-mbdrc",
 						     "nvidia,tegra210-mbdrc";
-					reg = <0x0 0x2908200 0x0 0x200>;
+					reg = <0x2908200 0x200>;
 				};
 			};
 
 			tegra_mvc1: mvc@290a000 {
 				compatible = "nvidia,tegra186-mvc",
 					     "nvidia,tegra210-mvc";
-				reg = <0x0 0x290a000 0x0 0x200>;
+				reg = <0x290a000 0x200>;
 				sound-name-prefix = "MVC1";
 				status = "disabled";
 			};
@@ -433,7 +433,7 @@
 			tegra_mvc2: mvc@290a200 {
 				compatible = "nvidia,tegra186-mvc",
 					     "nvidia,tegra210-mvc";
-				reg = <0x0 0x290a200 0x0 0x200>;
+				reg = <0x290a200 0x200>;
 				sound-name-prefix = "MVC2";
 				status = "disabled";
 			};
@@ -441,14 +441,14 @@
 			tegra_amixer: amixer@290bb00 {
 				compatible = "nvidia,tegra186-amixer",
 					     "nvidia,tegra210-amixer";
-				reg = <0x0 0x290bb00 0x0 0x800>;
+				reg = <0x290bb00 0x800>;
 				sound-name-prefix = "MIXER1";
 				status = "disabled";
 			};
 
 			tegra_admaif: admaif@290f000 {
 				compatible = "nvidia,tegra186-admaif";
-				reg = <0x0 0x0290f000 0x0 0x1000>;
+				reg = <0x0290f000 0x1000>;
 				dmas = <&adma 1>, <&adma 1>,
 				       <&adma 2>, <&adma 2>,
 				       <&adma 3>, <&adma 3>,
@@ -494,7 +494,7 @@
 
 			tegra_asrc: asrc@2910000 {
 				compatible = "nvidia,tegra186-asrc";
-				reg = <0x0 0x2910000 0x0 0x2000>;
+				reg = <0x2910000 0x2000>;
 				sound-name-prefix = "ASRC1";
 				status = "disabled";
 			};
diff --git a/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts b/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts
index 5462200f117670..478464464bcbac 100644
--- a/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra194-p2972-0000.dts
@@ -2450,22 +2450,22 @@
 			};
 
 			cooling-maps {
-				cpu-critical {
+				map-critical {
 					cooling-device = <&fan 3 3>;
 					trip = <&cpu_trip_critical>;
 				};
 
-				cpu-hot {
+				map-hot {
 					cooling-device = <&fan 2 2>;
 					trip = <&cpu_trip_hot>;
 				};
 
-				cpu-active {
+				map-active {
 					cooling-device = <&fan 1 1>;
 					trip = <&cpu_trip_active>;
 				};
 
-				cpu-passive {
+				map-passive {
 					cooling-device = <&fan 0 0>;
 					trip = <&cpu_trip_passive>;
 				};
diff --git a/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi b/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi
index 6f4879c49da24b..569d181b2ad2d3 100644
--- a/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra194-p3509-0000.dtsi
@@ -2433,22 +2433,22 @@
 			};
 
 			cooling-maps {
-				cpu-critical {
+				map-critical {
 					cooling-device = <&fan 3 3>;
 					trip = <&cpu_trip_critical>;
 				};
 
-				cpu-hot {
+				map-hot {
 					cooling-device = <&fan 2 2>;
 					trip = <&cpu_trip_hot>;
 				};
 
-				cpu-active {
+				map-active {
 					cooling-device = <&fan 1 1>;
 					trip = <&cpu_trip_active>;
 				};
 
-				cpu-passive {
+				map-passive {
 					cooling-device = <&fan 0 0>;
 					trip = <&cpu_trip_passive>;
 				};
diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
index 5e510f77b53867..13a95008c18643 100644
--- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
@@ -231,14 +231,14 @@
 				assigned-clock-rates = <81600000>;
 				status = "disabled";
 
-				#address-cells = <2>;
-				#size-cells = <2>;
-				ranges = <0x0 0x02900800 0x0 0x02900800 0x0 0x11800>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0x02900000 0x0 0x02900000 0x20000>;
 
 				tegra_i2s1: i2s@2901000 {
 					compatible = "nvidia,tegra194-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901000 0x0 0x100>;
+					reg = <0x2901000 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_I2S1>,
 						 <&bpmp TEGRA194_CLK_I2S1_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -252,7 +252,7 @@
 				tegra_i2s2: i2s@2901100 {
 					compatible = "nvidia,tegra194-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901100 0x0 0x100>;
+					reg = <0x2901100 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_I2S2>,
 						 <&bpmp TEGRA194_CLK_I2S2_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -266,7 +266,7 @@
 				tegra_i2s3: i2s@2901200 {
 					compatible = "nvidia,tegra194-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901200 0x0 0x100>;
+					reg = <0x2901200 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_I2S3>,
 						 <&bpmp TEGRA194_CLK_I2S3_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -280,7 +280,7 @@
 				tegra_i2s4: i2s@2901300 {
 					compatible = "nvidia,tegra194-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901300 0x0 0x100>;
+					reg = <0x2901300 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_I2S4>,
 						 <&bpmp TEGRA194_CLK_I2S4_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -294,7 +294,7 @@
 				tegra_i2s5: i2s@2901400 {
 					compatible = "nvidia,tegra194-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901400 0x0 0x100>;
+					reg = <0x2901400 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_I2S5>,
 						 <&bpmp TEGRA194_CLK_I2S5_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -308,7 +308,7 @@
 				tegra_i2s6: i2s@2901500 {
 					compatible = "nvidia,tegra194-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901500 0x0 0x100>;
+					reg = <0x2901500 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_I2S6>,
 						 <&bpmp TEGRA194_CLK_I2S6_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -322,7 +322,7 @@
 				tegra_sfc1: sfc@2902000 {
 					compatible = "nvidia,tegra194-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902000 0x0 0x200>;
+					reg = <0x2902000 0x200>;
 					sound-name-prefix = "SFC1";
 					status = "disabled";
 				};
@@ -330,7 +330,7 @@
 				tegra_sfc2: sfc@2902200 {
 					compatible = "nvidia,tegra194-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902200 0x0 0x200>;
+					reg = <0x2902200 0x200>;
 					sound-name-prefix = "SFC2";
 					status = "disabled";
 				};
@@ -338,7 +338,7 @@
 				tegra_sfc3: sfc@2902400 {
 					compatible = "nvidia,tegra194-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902400 0x0 0x200>;
+					reg = <0x2902400 0x200>;
 					sound-name-prefix = "SFC3";
 					status = "disabled";
 				};
@@ -346,35 +346,35 @@
 				tegra_sfc4: sfc@2902600 {
 					compatible = "nvidia,tegra194-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902600 0x0 0x200>;
+					reg = <0x2902600 0x200>;
 					sound-name-prefix = "SFC4";
 					status = "disabled";
 				};
 
 				tegra_amx1: amx@2903000 {
 					compatible = "nvidia,tegra194-amx";
-					reg = <0x0 0x2903000 0x0 0x100>;
+					reg = <0x2903000 0x100>;
 					sound-name-prefix = "AMX1";
 					status = "disabled";
 				};
 
 				tegra_amx2: amx@2903100 {
 					compatible = "nvidia,tegra194-amx";
-					reg = <0x0 0x2903100 0x0 0x100>;
+					reg = <0x2903100 0x100>;
 					sound-name-prefix = "AMX2";
 					status = "disabled";
 				};
 
 				tegra_amx3: amx@2903200 {
 					compatible = "nvidia,tegra194-amx";
-					reg = <0x0 0x2903200 0x0 0x100>;
+					reg = <0x2903200 0x100>;
 					sound-name-prefix = "AMX3";
 					status = "disabled";
 				};
 
 				tegra_amx4: amx@2903300 {
 					compatible = "nvidia,tegra194-amx";
-					reg = <0x0 0x2903300 0x0 0x100>;
+					reg = <0x2903300 0x100>;
 					sound-name-prefix = "AMX4";
 					status = "disabled";
 				};
@@ -382,7 +382,7 @@
 				tegra_adx1: adx@2903800 {
 					compatible = "nvidia,tegra194-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903800 0x0 0x100>;
+					reg = <0x2903800 0x100>;
 					sound-name-prefix = "ADX1";
 					status = "disabled";
 				};
@@ -390,7 +390,7 @@
 				tegra_adx2: adx@2903900 {
 					compatible = "nvidia,tegra194-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903900 0x0 0x100>;
+					reg = <0x2903900 0x100>;
 					sound-name-prefix = "ADX2";
 					status = "disabled";
 				};
@@ -398,7 +398,7 @@
 				tegra_adx3: adx@2903a00 {
 					compatible = "nvidia,tegra194-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903a00 0x0 0x100>;
+					reg = <0x2903a00 0x100>;
 					sound-name-prefix = "ADX3";
 					status = "disabled";
 				};
@@ -406,7 +406,7 @@
 				tegra_adx4: adx@2903b00 {
 					compatible = "nvidia,tegra194-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903b00 0x0 0x100>;
+					reg = <0x2903b00 0x100>;
 					sound-name-prefix = "ADX4";
 					status = "disabled";
 				};
@@ -414,7 +414,7 @@
 				tegra_dmic1: dmic@2904000 {
 					compatible = "nvidia,tegra194-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904000 0x0 0x100>;
+					reg = <0x2904000 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_DMIC1>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA194_CLK_DMIC1>;
@@ -427,7 +427,7 @@
 				tegra_dmic2: dmic@2904100 {
 					compatible = "nvidia,tegra194-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904100 0x0 0x100>;
+					reg = <0x2904100 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_DMIC2>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA194_CLK_DMIC2>;
@@ -440,7 +440,7 @@
 				tegra_dmic3: dmic@2904200 {
 					compatible = "nvidia,tegra194-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904200 0x0 0x100>;
+					reg = <0x2904200 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_DMIC3>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA194_CLK_DMIC3>;
@@ -453,7 +453,7 @@
 				tegra_dmic4: dmic@2904300 {
 					compatible = "nvidia,tegra194-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904300 0x0 0x100>;
+					reg = <0x2904300 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_DMIC4>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA194_CLK_DMIC4>;
@@ -466,7 +466,7 @@
 				tegra_dspk1: dspk@2905000 {
 					compatible = "nvidia,tegra194-dspk",
 						     "nvidia,tegra186-dspk";
-					reg = <0x0 0x2905000 0x0 0x100>;
+					reg = <0x2905000 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_DSPK1>;
 					clock-names = "dspk";
 					assigned-clocks = <&bpmp TEGRA194_CLK_DSPK1>;
@@ -479,7 +479,7 @@
 				tegra_dspk2: dspk@2905100 {
 					compatible = "nvidia,tegra194-dspk",
 						     "nvidia,tegra186-dspk";
-					reg = <0x0 0x2905100 0x0 0x100>;
+					reg = <0x2905100 0x100>;
 					clocks = <&bpmp TEGRA194_CLK_DSPK2>;
 					clock-names = "dspk";
 					assigned-clocks = <&bpmp TEGRA194_CLK_DSPK2>;
@@ -492,31 +492,31 @@
 				tegra_ope1: processing-engine@2908000 {
 					compatible = "nvidia,tegra194-ope",
 						     "nvidia,tegra210-ope";
-					reg = <0x0 0x2908000 0x0 0x100>;
+					reg = <0x2908000 0x100>;
 					sound-name-prefix = "OPE1";
 					status = "disabled";
 
-					#address-cells = <2>;
-					#size-cells = <2>;
+					#address-cells = <1>;
+					#size-cells = <1>;
 					ranges;
 
 					equalizer@2908100 {
 						compatible = "nvidia,tegra194-peq",
 							     "nvidia,tegra210-peq";
-						reg = <0x0 0x2908100 0x0 0x100>;
+						reg = <0x2908100 0x100>;
 					};
 
 					dynamic-range-compressor@2908200 {
 						compatible = "nvidia,tegra194-mbdrc",
 							     "nvidia,tegra210-mbdrc";
-						reg = <0x0 0x2908200 0x0 0x200>;
+						reg = <0x2908200 0x200>;
 					};
 				};
 
 				tegra_mvc1: mvc@290a000 {
 					compatible = "nvidia,tegra194-mvc",
 						     "nvidia,tegra210-mvc";
-					reg = <0x0 0x290a000 0x0 0x200>;
+					reg = <0x290a000 0x200>;
 					sound-name-prefix = "MVC1";
 					status = "disabled";
 				};
@@ -524,7 +524,7 @@
 				tegra_mvc2: mvc@290a200 {
 					compatible = "nvidia,tegra194-mvc",
 						     "nvidia,tegra210-mvc";
-					reg = <0x0 0x290a200 0x0 0x200>;
+					reg = <0x290a200 0x200>;
 					sound-name-prefix = "MVC2";
 					status = "disabled";
 				};
@@ -532,7 +532,7 @@
 				tegra_amixer: amixer@290bb00 {
 					compatible = "nvidia,tegra194-amixer",
 						     "nvidia,tegra210-amixer";
-					reg = <0x0 0x290bb00 0x0 0x800>;
+					reg = <0x290bb00 0x800>;
 					sound-name-prefix = "MIXER1";
 					status = "disabled";
 				};
@@ -540,7 +540,7 @@
 				tegra_admaif: admaif@290f000 {
 					compatible = "nvidia,tegra194-admaif",
 						     "nvidia,tegra186-admaif";
-					reg = <0x0 0x0290f000 0x0 0x1000>;
+					reg = <0x0290f000 0x1000>;
 					dmas = <&adma 1>, <&adma 1>,
 					       <&adma 2>, <&adma 2>,
 					       <&adma 3>, <&adma 3>,
@@ -591,7 +591,7 @@
 				tegra_asrc: asrc@2910000 {
 					compatible = "nvidia,tegra194-asrc",
 						     "nvidia,tegra186-asrc";
-					reg = <0x0 0x2910000 0x0 0x2000>;
+					reg = <0x2910000 0x2000>;
 					sound-name-prefix = "ASRC1";
 					status = "disabled";
 				};
@@ -923,6 +923,8 @@
 				 <&bpmp TEGRA194_CLK_QSPI0_PM>;
 			clock-names = "qspi", "qspi_out";
 			resets = <&bpmp TEGRA194_RESET_QSPI0>;
+			dmas = <&gpcdma 5>, <&gpcdma 5>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -1013,6 +1015,8 @@
 				 <&bpmp TEGRA194_CLK_QSPI1_PM>;
 			clock-names = "qspi", "qspi_out";
 			resets = <&bpmp TEGRA194_RESET_QSPI1>;
+			dmas = <&gpcdma 6>, <&gpcdma 6>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts b/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts
index d78b9bd45df6cc..8ee334c1087119 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra210-p3450-0000.dts
@@ -1750,7 +1750,7 @@
 			label = "Force Recovery";
 			gpios = <&gpio TEGRA_GPIO(X, 6) GPIO_ACTIVE_LOW>;
 			linux,input-type = <EV_KEY>;
-			linux,code = <BTN_1>;
+			linux,code = <KEY_SLEEP>;
 			debounce-interval = <30>;
 		};
 
@@ -1968,22 +1968,22 @@
 			};
 
 			cooling-maps {
-				cpu-critical {
+				map-critical {
 					cooling-device = <&fan 3 3>;
 					trip = <&cpu_trip_critical>;
 				};
 
-				cpu-hot {
+				map-hot {
 					cooling-device = <&fan 2 2>;
 					trip = <&cpu_trip_hot>;
 				};
 
-				cpu-active {
+				map-active {
 					cooling-device = <&fan 1 1>;
 					trip = <&cpu_trip_active>;
 				};
 
-				cpu-passive {
+				map-passive {
 					cooling-device = <&fan 0 0>;
 					trip = <&cpu_trip_passive>;
 				};
diff --git a/arch/arm64/boot/dts/nvidia/tegra210.dtsi b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
index 5f5e5370d70966..99fefce414765a 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
@@ -1461,7 +1461,7 @@
 			assigned-clock-rates = <81600000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
-			ranges = <0x702d0000 0x702d0000 0x0000e400>;
+			ranges = <0x702d0000 0x702d0000 0x00010000>;
 			status = "disabled";
 
 			tegra_admaif: admaif@702d0000 {
@@ -2164,12 +2164,12 @@
 			};
 
 			cooling-maps {
-				dram-passive {
+				map-passive {
 					cooling-device = <&emc 0 0>;
 					trip = <&dram_nominal>;
 				};
 
-				dram-active {
+				map-active {
 					cooling-device = <&emc 1 1>;
 					trip = <&dram_throttle>;
 				};
diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701.dtsi b/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701.dtsi
index 420858a8f4a873..9b990d1cef3f89 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701.dtsi
@@ -391,7 +391,7 @@
 			label = "Force Recovery";
 			gpios = <&gpio TEGRA234_MAIN_GPIO(G, 0) GPIO_ACTIVE_LOW>;
 			linux,input-type = <EV_KEY>;
-			linux,code = <BTN_1>;
+			linux,code = <KEY_SLEEP>;
 		};
 
 		key-power {
@@ -402,13 +402,6 @@
 			wakeup-event-action = <EV_ACT_ASSERTED>;
 			wakeup-source;
 		};
-
-		key-suspend {
-			label = "Suspend";
-			gpios = <&gpio TEGRA234_MAIN_GPIO(G, 2) GPIO_ACTIVE_LOW>;
-			linux,input-type = <EV_KEY>;
-			linux,code = <KEY_SLEEP>;
-		};
 	};
 
 	fan: pwm-fan {
diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts b/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts
index 97cede1fcb70aa..8c8c71547996fa 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra234-p3740-0002+p3701-0008.dts
@@ -93,6 +93,29 @@
 				};
 			};
 
+			typec@28 {
+				compatible = "st,stusb1600";
+				reg = <0x28>;
+				interrupt-parent = <&gpio>;
+				interrupts = <TEGRA234_MAIN_GPIO(K, 6) IRQ_TYPE_LEVEL_LOW>;
+				vdd-supply = <&vdd_5v_sys>;
+				vsys-supply = <&vdd_3v3_sys>;
+
+				connector {
+					compatible = "usb-c-connector";
+					label = "USB-C";
+					data-role = "dual";
+					power-role = "dual";
+					typec-power-opmode = "default";
+
+					port {
+						typec_con_ep: endpoint {
+							remote-endpoint = <&usb_role_switch0>;
+						};
+					};
+				};
+			};
+
 			/* carrier board ID EEPROM */
 			eeprom@55 {
 				compatible = "atmel,24c02";
@@ -173,6 +196,12 @@
 					mode = "otg";
 					usb-role-switch;
 					status = "okay";
+
+					port {
+						usb_role_switch0: endpoint {
+							remote-endpoint = <&typec_con_ep>;
+						};
+					};
 				};
 
 				usb2-1 {
@@ -337,7 +366,7 @@
 			label = "Force Recovery";
 			gpios = <&gpio TEGRA234_MAIN_GPIO(G, 0) GPIO_ACTIVE_LOW>;
 			linux,input-type = <EV_KEY>;
-			linux,code = <BTN_1>;
+			linux,code = <KEY_SLEEP>;
 		};
 
 		key-power {
@@ -348,13 +377,6 @@
 			wakeup-event-action = <EV_ACT_ASSERTED>;
 			wakeup-source;
 		};
-
-		key-suspend {
-			label = "Suspend";
-			gpios = <&gpio TEGRA234_MAIN_GPIO(G, 2) GPIO_ACTIVE_LOW>;
-			linux,input-type = <EV_KEY>;
-			linux,code = <KEY_SLEEP>;
-		};
 	};
 
 	serial {
@@ -473,4 +495,11 @@
 		regulator-boot-on;
 		enable-active-high;
 	};
+
+	vdd_5v_sys: regulator-vdd-5v-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd-5v-sys";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3768-0000+p3767.dtsi b/arch/arm64/boot/dts/nvidia/tegra234-p3768-0000+p3767.dtsi
index 41821354bbdae6..9b93074e5aedee 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234-p3768-0000+p3767.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra234-p3768-0000+p3767.dtsi
@@ -216,7 +216,7 @@
 			label = "Force Recovery";
 			gpios = <&gpio TEGRA234_MAIN_GPIO(G, 0) GPIO_ACTIVE_LOW>;
 			linux,input-type = <EV_KEY>;
-			linux,code = <BTN_1>;
+			linux,code = <KEY_SLEEP>;
 		};
 
 		key-power {
diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
index 5e29316a4d7592..5311245334de74 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
@@ -245,14 +245,14 @@
 				assigned-clock-rates = <81600000>;
 				status = "disabled";
 
-				#address-cells = <2>;
-				#size-cells = <2>;
-				ranges = <0x0 0x02900800 0x0 0x02900800 0x0 0x11800>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0x02900000 0x0 0x02900000 0x20000>;
 
 				tegra_i2s1: i2s@2901000 {
 					compatible = "nvidia,tegra234-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901000 0x0 0x100>;
+					reg = <0x2901000 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_I2S1>,
 						 <&bpmp TEGRA234_CLK_I2S1_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -288,7 +288,7 @@
 				tegra_i2s2: i2s@2901100 {
 					compatible = "nvidia,tegra234-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901100 0x0 0x100>;
+					reg = <0x2901100 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_I2S2>,
 						 <&bpmp TEGRA234_CLK_I2S2_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -324,7 +324,7 @@
 				tegra_i2s3: i2s@2901200 {
 					compatible = "nvidia,tegra234-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901200 0x0 0x100>;
+					reg = <0x2901200 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_I2S3>,
 						 <&bpmp TEGRA234_CLK_I2S3_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -360,7 +360,7 @@
 				tegra_i2s4: i2s@2901300 {
 					compatible = "nvidia,tegra234-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901300 0x0 0x100>;
+					reg = <0x2901300 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_I2S4>,
 						 <&bpmp TEGRA234_CLK_I2S4_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -396,7 +396,7 @@
 				tegra_i2s5: i2s@2901400 {
 					compatible = "nvidia,tegra234-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901400 0x0 0x100>;
+					reg = <0x2901400 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_I2S5>,
 						 <&bpmp TEGRA234_CLK_I2S5_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -432,7 +432,7 @@
 				tegra_i2s6: i2s@2901500 {
 					compatible = "nvidia,tegra234-i2s",
 						     "nvidia,tegra210-i2s";
-					reg = <0x0 0x2901500 0x0 0x100>;
+					reg = <0x2901500 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_I2S6>,
 						 <&bpmp TEGRA234_CLK_I2S6_SYNC_INPUT>;
 					clock-names = "i2s", "sync_input";
@@ -468,7 +468,7 @@
 				tegra_sfc1: sfc@2902000 {
 					compatible = "nvidia,tegra234-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902000 0x0 0x200>;
+					reg = <0x2902000 0x200>;
 					sound-name-prefix = "SFC1";
 
 					ports {
@@ -496,7 +496,7 @@
 				tegra_sfc2: sfc@2902200 {
 					compatible = "nvidia,tegra234-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902200 0x0 0x200>;
+					reg = <0x2902200 0x200>;
 					sound-name-prefix = "SFC2";
 
 					ports {
@@ -524,7 +524,7 @@
 				tegra_sfc3: sfc@2902400 {
 					compatible = "nvidia,tegra234-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902400 0x0 0x200>;
+					reg = <0x2902400 0x200>;
 					sound-name-prefix = "SFC3";
 
 					ports {
@@ -552,7 +552,7 @@
 				tegra_sfc4: sfc@2902600 {
 					compatible = "nvidia,tegra234-sfc",
 						     "nvidia,tegra210-sfc";
-					reg = <0x0 0x2902600 0x0 0x200>;
+					reg = <0x2902600 0x200>;
 					sound-name-prefix = "SFC4";
 
 					ports {
@@ -580,7 +580,7 @@
 				tegra_amx1: amx@2903000 {
 					compatible = "nvidia,tegra234-amx",
 						     "nvidia,tegra194-amx";
-					reg = <0x0 0x2903000 0x0 0x100>;
+					reg = <0x2903000 0x100>;
 					sound-name-prefix = "AMX1";
 
 					ports {
@@ -632,7 +632,7 @@
 				tegra_amx2: amx@2903100 {
 					compatible = "nvidia,tegra234-amx",
 						     "nvidia,tegra194-amx";
-					reg = <0x0 0x2903100 0x0 0x100>;
+					reg = <0x2903100 0x100>;
 					sound-name-prefix = "AMX2";
 
 					ports {
@@ -684,7 +684,7 @@
 				tegra_amx3: amx@2903200 {
 					compatible = "nvidia,tegra234-amx",
 						     "nvidia,tegra194-amx";
-					reg = <0x0 0x2903200 0x0 0x100>;
+					reg = <0x2903200 0x100>;
 					sound-name-prefix = "AMX3";
 
 					ports {
@@ -736,7 +736,7 @@
 				tegra_amx4: amx@2903300 {
 					compatible = "nvidia,tegra234-amx",
 						     "nvidia,tegra194-amx";
-					reg = <0x0 0x2903300 0x0 0x100>;
+					reg = <0x2903300 0x100>;
 					sound-name-prefix = "AMX4";
 
 					ports {
@@ -788,7 +788,7 @@
 				tegra_adx1: adx@2903800 {
 					compatible = "nvidia,tegra234-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903800 0x0 0x100>;
+					reg = <0x2903800 0x100>;
 					sound-name-prefix = "ADX1";
 
 					ports {
@@ -840,7 +840,7 @@
 				tegra_adx2: adx@2903900 {
 					compatible = "nvidia,tegra234-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903900 0x0 0x100>;
+					reg = <0x2903900 0x100>;
 					sound-name-prefix = "ADX2";
 
 					ports {
@@ -892,7 +892,7 @@
 				tegra_adx3: adx@2903a00 {
 					compatible = "nvidia,tegra234-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903a00 0x0 0x100>;
+					reg = <0x2903a00 0x100>;
 					sound-name-prefix = "ADX3";
 
 					ports {
@@ -944,7 +944,7 @@
 				tegra_adx4: adx@2903b00 {
 					compatible = "nvidia,tegra234-adx",
 						     "nvidia,tegra210-adx";
-					reg = <0x0 0x2903b00 0x0 0x100>;
+					reg = <0x2903b00 0x100>;
 					sound-name-prefix = "ADX4";
 
 					ports {
@@ -997,7 +997,7 @@
 				tegra_dmic1: dmic@2904000 {
 					compatible = "nvidia,tegra234-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904000 0x0 0x100>;
+					reg = <0x2904000 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_DMIC1>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA234_CLK_DMIC1>;
@@ -1031,7 +1031,7 @@
 				tegra_dmic2: dmic@2904100 {
 					compatible = "nvidia,tegra234-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904100 0x0 0x100>;
+					reg = <0x2904100 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_DMIC2>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA234_CLK_DMIC2>;
@@ -1065,7 +1065,7 @@
 				tegra_dmic3: dmic@2904200 {
 					compatible = "nvidia,tegra234-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904200 0x0 0x100>;
+					reg = <0x2904200 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_DMIC3>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA234_CLK_DMIC3>;
@@ -1099,7 +1099,7 @@
 				tegra_dmic4: dmic@2904300 {
 					compatible = "nvidia,tegra234-dmic",
 						     "nvidia,tegra210-dmic";
-					reg = <0x0 0x2904300 0x0 0x100>;
+					reg = <0x2904300 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_DMIC4>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA234_CLK_DMIC4>;
@@ -1133,7 +1133,7 @@
 				tegra_dspk1: dspk@2905000 {
 					compatible = "nvidia,tegra234-dspk",
 						     "nvidia,tegra186-dspk";
-					reg = <0x0 0x2905000 0x0 0x100>;
+					reg = <0x2905000 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_DSPK1>;
 					clock-names = "dspk";
 					assigned-clocks = <&bpmp TEGRA234_CLK_DSPK1>;
@@ -1167,7 +1167,7 @@
 				tegra_dspk2: dspk@2905100 {
 					compatible = "nvidia,tegra234-dspk",
 						     "nvidia,tegra186-dspk";
-					reg = <0x0 0x2905100 0x0 0x100>;
+					reg = <0x2905100 0x100>;
 					clocks = <&bpmp TEGRA234_CLK_DSPK2>;
 					clock-names = "dspk";
 					assigned-clocks = <&bpmp TEGRA234_CLK_DSPK2>;
@@ -1201,23 +1201,23 @@
 				tegra_ope1: processing-engine@2908000 {
 					compatible = "nvidia,tegra234-ope",
 						     "nvidia,tegra210-ope";
-					reg = <0x0 0x2908000 0x0 0x100>;
+					reg = <0x2908000 0x100>;
 					sound-name-prefix = "OPE1";
 
-					#address-cells = <2>;
-					#size-cells = <2>;
+					#address-cells = <1>;
+					#size-cells = <1>;
 					ranges;
 
 					equalizer@2908100 {
 						compatible = "nvidia,tegra234-peq",
 							     "nvidia,tegra210-peq";
-						reg = <0x0 0x2908100 0x0 0x100>;
+						reg = <0x2908100 0x100>;
 					};
 
 					dynamic-range-compressor@2908200 {
 						compatible = "nvidia,tegra234-mbdrc",
 							     "nvidia,tegra210-mbdrc";
-						reg = <0x0 0x2908200 0x0 0x200>;
+						reg = <0x2908200 0x200>;
 					};
 
 					ports {
@@ -1247,7 +1247,7 @@
 				tegra_mvc1: mvc@290a000 {
 					compatible = "nvidia,tegra234-mvc",
 						     "nvidia,tegra210-mvc";
-					reg = <0x0 0x290a000 0x0 0x200>;
+					reg = <0x290a000 0x200>;
 					sound-name-prefix = "MVC1";
 
 					ports {
@@ -1275,7 +1275,7 @@
 				tegra_mvc2: mvc@290a200 {
 					compatible = "nvidia,tegra234-mvc",
 						     "nvidia,tegra210-mvc";
-					reg = <0x0 0x290a200 0x0 0x200>;
+					reg = <0x290a200 0x200>;
 					sound-name-prefix = "MVC2";
 
 					ports {
@@ -1303,7 +1303,7 @@
 				tegra_amixer: amixer@290bb00 {
 					compatible = "nvidia,tegra234-amixer",
 						     "nvidia,tegra210-amixer";
-					reg = <0x0 0x290bb00 0x0 0x800>;
+					reg = <0x290bb00 0x800>;
 					sound-name-prefix = "MIXER1";
 
 					ports {
@@ -1435,7 +1435,7 @@
 				tegra_admaif: admaif@290f000 {
 					compatible = "nvidia,tegra234-admaif",
 						     "nvidia,tegra186-admaif";
-					reg = <0x0 0x0290f000 0x0 0x1000>;
+					reg = <0x0290f000 0x1000>;
 					dmas = <&adma 1>, <&adma 1>,
 					       <&adma 2>, <&adma 2>,
 					       <&adma 3>, <&adma 3>,
@@ -1650,7 +1650,7 @@
 				tegra_asrc: asrc@2910000 {
 					compatible = "nvidia,tegra234-asrc",
 						     "nvidia,tegra186-asrc";
-					reg = <0x0 0x2910000 0x0 0x2000>;
+					reg = <0x2910000 0x2000>;
 					sound-name-prefix = "ASRC1";
 
 					ports {
diff --git a/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi b/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi
index 58cd81bc33d738..f39f33a66cd25e 100644
--- a/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra264-p3834.dtsi
@@ -38,5 +38,62 @@
 		cmdqv@6200000 {
 			status = "okay";
 		};
+
+		i2c@c640000 {
+			status = "okay";
+
+			eeprom@50 {
+				compatible = "atmel,24c02";
+				reg = <0x50>;
+				label = "module";
+				address-width = <8>;
+				pagesize = <8>;
+				size = <256>;
+				read-only;
+			};
+		};
+	};
+
+	bpmp {
+		i2c {
+			status = "okay";
+
+			thermal-sensor@4c {
+				compatible = "ti,tmp451";
+				reg = <0x4c>;
+			};
+		};
+	};
+
+	thermal-zones {
+		cpu-thermal {
+			polling-delay-passive = <40>;
+			polling-delay = <1000>;
+			status = "okay";
+		};
+
+		gpu-thermal {
+			polling-delay-passive = <40>;
+			polling-delay = <1000>;
+			status = "okay";
+		};
+
+		soc012-thermal {
+			polling-delay-passive = <40>;
+			polling-delay = <1000>;
+			status = "okay";
+		};
+
+		soc345-thermal {
+			polling-delay-passive = <40>;
+			polling-delay = <1000>;
+			status = "okay";
+		};
+
+		tj-thermal {
+			polling-delay-passive = <40>;
+			polling-delay = <1000>;
+			status = "okay";
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra264-p4071-0000+p3834.dtsi b/arch/arm64/boot/dts/nvidia/tegra264-p4071-0000+p3834.dtsi
index 45f8df9bbfd624..9295f9c8129620 100644
--- a/arch/arm64/boot/dts/nvidia/tegra264-p4071-0000+p3834.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra264-p4071-0000+p3834.dtsi
@@ -1,5 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
 
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/input/gpio-keys.h>
+
 / {
 	aliases {
 		serial0 = &{/bus@0/serial@c4e0000};
@@ -9,4 +12,107 @@
 	chosen {
 		stdout-path = "serial0:115200n8";
 	};
+
+	bus@0 {
+		rtc@c2c0000 {
+			status = "okay";
+		};
+
+		pwm@c6a0000 {
+			status = "okay";
+		};
+	};
+
+	bus@8100000000 {
+		i2c@c640000 {
+			eeprom@56 {
+				compatible = "atmel,24c02";
+				reg = <0x56>;
+				label = "system";
+				address-width = <8>;
+				pagesize = <8>;
+				size = <256>;
+				read-only;
+			};
+		};
+	};
+
+	fan: pwm-fan {
+		compatible = "pwm-fan";
+		cooling-levels = <77 102 140 192 255>;
+		pwms = <&pwm4 0 40000>;
+		#cooling-cells = <2>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		key-force-recovery {
+			label = "Force Recovery";
+			gpios = <&gpio_aon TEGRA264_AON_GPIO(EE, 1) GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_KEY>;
+			linux,code = <KEY_SLEEP>;
+		};
+
+		key-power {
+			label = "Power";
+			gpios = <&gpio_aon TEGRA264_AON_GPIO(AA, 5) GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_KEY>;
+			linux,code = <KEY_POWER>;
+			wakeup-event-action = <EV_ACT_ASSERTED>;
+			wakeup-source;
+		};
+	};
+
+	thermal-zones {
+		tj-thermal {
+			trips {
+				tj_trip_active0: active-0 {
+					temperature = <80000>;
+					hysteresis = <0>;
+					type = "active";
+				};
+
+				tj_trip_active1: active-1 {
+					temperature = <86000>;
+					hysteresis = <0>;
+					type = "active";
+				};
+
+				tj_trip_active2: active-2 {
+					temperature = <91000>;
+					hysteresis = <0>;
+					type = "active";
+				};
+
+				tj_trip_active3: active-3 {
+					temperature = <100000>;
+					hysteresis = <0>;
+					type = "active";
+				};
+			};
+
+			cooling-maps {
+				map-active-0 {
+					cooling-device = <&fan 0 1>;
+					trip = <&tj_trip_active0>;
+				};
+
+				map-active-1 {
+					cooling-device = <&fan 1 2>;
+					trip = <&tj_trip_active1>;
+				};
+
+				map-active-2 {
+					cooling-device = <&fan 2 3>;
+					trip = <&tj_trip_active2>;
+				};
+
+				map-active-3 {
+					cooling-device = <&fan 3 4>;
+					trip = <&tj_trip_active3>;
+				};
+			};
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi
index 013228a8172375..c04d0e5df85b97 100644
--- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi
@@ -1,18 +1,40 @@
 // SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
 
 #include <dt-bindings/clock/nvidia,tegra264.h>
+#include <dt-bindings/gpio/nvidia,tegra264-gpio.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/tegra186-hsp.h>
 #include <dt-bindings/memory/nvidia,tegra264.h>
 #include <dt-bindings/power/nvidia,tegra264-bpmp.h>
 #include <dt-bindings/reset/nvidia,tegra264.h>
 
+#include "thermal/nvidia,tegra264-bpmp.h"
+
 / {
 	compatible = "nvidia,tegra264";
 	interrupt-parent = <&gic>;
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &{/bpmp/i2c};
+		i2c6 = &i2c6;
+		i2c7 = &i2c7;
+		i2c8 = &i2c8;
+		i2c9 = &i2c9;
+		i2c10 = &i2c10;
+		i2c11 = &i2c11;
+		i2c12 = &i2c12;
+		i2c14 = &i2c14;
+		i2c15 = &i2c15;
+		i2c16 = &i2c16;
+	};
+
 	reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
@@ -40,6 +62,13 @@
 			      <0x0 0x0c140000 0x0 0x10000>;
 		};
 
+		fuse@1000000 {
+			compatible = "nvidia,tegra264-efuse";
+			reg = <0x0 0x1000000 0x0 0x20000>;
+			clocks = <&bpmp TEGRA264_CLK_FUSE>;
+			clock-names = "fuse";
+		};
+
 		timer@8000000 {
 			compatible = "nvidia,tegra234-timer";
 			reg = <0x0 0x08000000 0x0 0x140000>;
@@ -122,85 +151,85 @@
 
 		aconnect@9000000 {
 			compatible = "nvidia,tegra264-aconnect",
-					"nvidia,tegra210-aconnect";
+				     "nvidia,tegra210-aconnect";
 			clocks = <&bpmp TEGRA264_CLK_APE>,
-				<&bpmp TEGRA264_CLK_ADSP>;
+				 <&bpmp TEGRA264_CLK_ADSP>;
 			clock-names = "ape", "apb2ape";
 			power-domains = <&bpmp TEGRA264_POWER_DOMAIN_AUD>;
 			status = "disabled";
 
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges = <0x0 0x9000000 0x0 0x9000000 0x0 0x2000000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x9000000 0x0 0x9000000 0x2000000>;
 
 			adma: dma-controller@9440000 {
 				compatible = "nvidia,tegra264-adma";
-				reg = <0x0 0x9440000 0x0 0xb0000>;
+				reg = <0x9440000 0xb0000>;
 				interrupt-parent = <&agic_page0>;
 				interrupts = <GIC_SPI 0x90 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x91 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x92 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x93 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x94 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x95 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x96 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x97 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x98 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x99 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x9a IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x9b IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x9c IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x9d IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x9e IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0x9f IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa0 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa1 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa2 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa3 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa4 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa5 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa6 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa7 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa8 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xa9 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xaa IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xab IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xac IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xad IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xae IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xaf IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb0 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb1 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb2 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb3 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb4 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb5 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb6 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb7 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb8 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xb9 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xba IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xbb IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xbc IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xbd IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xbe IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xbf IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc0 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc1 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc2 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc3 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc4 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc5 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc6 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc7 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc8 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xc9 IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xca IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xcb IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xcc IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xcd IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xce IRQ_TYPE_LEVEL_HIGH>,
-					   <GIC_SPI 0xcf IRQ_TYPE_LEVEL_HIGH>;
+					     <GIC_SPI 0x91 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x92 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x93 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x94 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x95 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x96 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x97 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x98 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x99 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x9a IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x9b IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x9c IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x9d IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x9e IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0x9f IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa0 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa1 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa2 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa3 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa4 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa5 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa6 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa7 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa8 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xa9 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xaa IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xab IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xac IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xad IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xae IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xaf IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb0 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb1 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb2 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb3 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb4 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb5 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb6 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb7 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb8 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xb9 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xba IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xbb IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xbc IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xbd IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xbe IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xbf IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc0 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc1 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc2 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc3 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc4 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc5 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc6 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc7 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc8 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xc9 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xca IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xcb IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xcc IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xcd IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xce IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 0xcf IRQ_TYPE_LEVEL_HIGH>;
 				#dma-cells = <1>;
 				clocks = <&bpmp TEGRA264_CLK_AHUB>;
 				clock-names = "d_audio";
@@ -209,24 +238,24 @@
 
 			tegra_ahub: ahub@9630000 {
 				compatible = "nvidia,tegra264-ahub";
-				reg = <0x0 0x9630000 0x0 0x10000>;
+				reg = <0x9630000 0x10000>;
 				clocks = <&bpmp TEGRA264_CLK_AHUB>;
 				clock-names = "ahub";
 				assigned-clocks = <&bpmp TEGRA264_CLK_AHUB>;
 				assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLAON_APE>;
 				status = "disabled";
 
-				#address-cells = <2>;
-				#size-cells = <2>;
+				#address-cells = <1>;
+				#size-cells = <1>;
 				/* ADMA is under AHUB range, its excluded in the defined range */
-				ranges = <0x0 0x9280000 0x0 0x9280000 0x0 0x1c0000>,
-					<0x0 0x9510000 0x0 0x9510000 0x0 0x370000>;
+				ranges = <0x9280000 0x9280000 0x1c0000>,
+					 <0x9510000 0x9510000 0x370000>;
 
 				tegra_i2s1: i2s@9280000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x9280000 0x0 0x10000>;
+					reg = <0x9280000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S1>,
-					       <&bpmp TEGRA264_CLK_I2S1_SCLK_IN>;
+						 <&bpmp TEGRA264_CLK_I2S1_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
 					assigned-clocks = <&bpmp TEGRA264_CLK_I2S1>;
 					assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLA1_OUT1>;
@@ -259,7 +288,7 @@
 
 				tegra_i2s2: i2s@9290000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x9290000 0x0 0x10000>;
+					reg = <0x9290000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S2>,
 						 <&bpmp TEGRA264_CLK_I2S2_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
@@ -294,9 +323,9 @@
 
 				tegra_i2s3: i2s@92a0000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x92a0000 0x0 0x10000>;
+					reg = <0x92a0000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S3>,
-					       <&bpmp TEGRA264_CLK_I2S3_SCLK_IN>;
+						 <&bpmp TEGRA264_CLK_I2S3_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
 					assigned-clocks = <&bpmp TEGRA264_CLK_I2S3>;
 					assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLA1_OUT1>;
@@ -329,9 +358,9 @@
 
 				tegra_i2s4: i2s@92b0000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x92b0000 0x0 0x10000>;
+					reg = <0x92b0000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S4>,
-					       <&bpmp TEGRA264_CLK_I2S4_SCLK_IN>;
+						 <&bpmp TEGRA264_CLK_I2S4_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
 					assigned-clocks = <&bpmp TEGRA264_CLK_I2S4>;
 					assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLA1_OUT1>;
@@ -364,9 +393,9 @@
 
 				tegra_i2s5: i2s@92c0000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x92c0000 0x0 0x10000>;
+					reg = <0x92c0000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S5>,
-					       <&bpmp TEGRA264_CLK_I2S5_SCLK_IN>;
+						 <&bpmp TEGRA264_CLK_I2S5_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
 					assigned-clocks = <&bpmp TEGRA264_CLK_I2S5>;
 					assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLA1_OUT1>;
@@ -399,9 +428,9 @@
 
 				tegra_i2s6: i2s@92d0000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x92d0000 0x0 0x10000>;
+					reg = <0x92d0000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S6>,
-					       <&bpmp TEGRA264_CLK_I2S6_SCLK_IN>;
+						 <&bpmp TEGRA264_CLK_I2S6_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
 					assigned-clocks = <&bpmp TEGRA264_CLK_I2S6>;
 					assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLA1_OUT1>;
@@ -434,9 +463,9 @@
 
 				tegra_i2s7: i2s@92e0000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x92e0000 0x0 0x10000>;
+					reg = <0x92e0000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S7>,
-					       <&bpmp TEGRA264_CLK_I2S7_SCLK_IN>;
+						 <&bpmp TEGRA264_CLK_I2S7_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
 					assigned-clocks = <&bpmp TEGRA264_CLK_I2S7>;
 					assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLA1_OUT1>;
@@ -469,9 +498,9 @@
 
 				tegra_i2s8: i2s@92f0000 {
 					compatible = "nvidia,tegra264-i2s";
-					reg = <0x0 0x92f0000 0x0 0x10000>;
+					reg = <0x92f0000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_I2S8>,
-					       <&bpmp TEGRA264_CLK_I2S8_SCLK_IN>;
+						 <&bpmp TEGRA264_CLK_I2S8_SCLK_IN>;
 					clock-names = "i2s", "sync_input";
 					assigned-clocks = <&bpmp TEGRA264_CLK_I2S8>;
 					assigned-clock-parents = <&bpmp TEGRA264_CLK_PLLA1_OUT1>;
@@ -504,8 +533,8 @@
 
 				tegra_dmic1: dmic@9300000 {
 					compatible = "nvidia,tegra264-dmic",
-							"nvidia,tegra210-dmic";
-					reg = <0x0 0x9300000 0x0 0x10000>;
+						     "nvidia,tegra210-dmic";
+					reg = <0x9300000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_DMIC1>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA264_CLK_DMIC1>;
@@ -538,8 +567,8 @@
 
 				tegra_dmic2: dmic@9310000 {
 					compatible = "nvidia,tegra264-dmic",
-						   "nvidia,tegra210-dmic";
-					reg = <0x0 0x9310000 0x0 0x10000>;
+						     "nvidia,tegra210-dmic";
+					reg = <0x9310000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_DMIC1>;
 					clock-names = "dmic";
 					assigned-clocks = <&bpmp TEGRA264_CLK_DMIC1>;
@@ -572,8 +601,8 @@
 
 				tegra_dspk1: dspk@9380000 {
 					compatible = "nvidia,tegra264-dspk",
-							"nvidia,tegra186-dspk";
-					reg = <0x0 0x9380000 0x0 0x10000>;
+						     "nvidia,tegra186-dspk";
+					reg = <0x9380000 0x10000>;
 					clocks = <&bpmp TEGRA264_CLK_DSPK1>;
 					clock-names = "dspk";
 					assigned-clocks = <&bpmp TEGRA264_CLK_DSPK1>;
@@ -606,7 +635,7 @@
 
 				tegra_amx1: amx@9510000 {
 					compatible = "nvidia,tegra264-amx";
-					reg = <0x0 0x9510000 0x0 0x10000>;
+					reg = <0x9510000 0x10000>;
 					sound-name-prefix = "AMX1";
 
 					ports {
@@ -657,7 +686,7 @@
 
 				tegra_amx2: amx@9520000 {
 					compatible = "nvidia,tegra264-amx";
-					reg = <0x0 0x9520000 0x0 0x10000>;
+					reg = <0x9520000 0x10000>;
 					sound-name-prefix = "AMX2";
 
 					ports {
@@ -708,7 +737,7 @@
 
 				tegra_amx3: amx@9530000 {
 					compatible = "nvidia,tegra264-amx";
-					reg = <0x0 0x9530000 0x0 0x10000>;
+					reg = <0x9530000 0x10000>;
 					sound-name-prefix = "AMX3";
 
 					ports {
@@ -759,7 +788,7 @@
 
 				tegra_amx4: amx@9540000 {
 					compatible = "nvidia,tegra264-amx";
-					reg = <0x0 0x9540000 0x0 0x10000>;
+					reg = <0x9540000 0x10000>;
 					sound-name-prefix = "AMX4";
 
 					ports {
@@ -810,7 +839,7 @@
 
 				tegra_amx5: amx@9550000 {
 					compatible = "nvidia,tegra264-amx";
-					reg = <0x0 0x9550000 0x0 0x10000>;
+					reg = <0x9550000 0x10000>;
 					sound-name-prefix = "AMX5";
 
 					ports {
@@ -861,7 +890,7 @@
 
 				tegra_amx6: amx@9560000 {
 					compatible = "nvidia,tegra264-amx";
-					reg = <0x0 0x9560000 0x0 0x10000>;
+					reg = <0x9560000 0x10000>;
 					sound-name-prefix = "AMX6";
 
 					ports {
@@ -912,7 +941,7 @@
 
 				tegra_adx1: adx@9590000 {
 					compatible = "nvidia,tegra264-adx";
-					reg = <0x0 0x9590000 0x0 0x10000>;
+					reg = <0x9590000 0x10000>;
 					sound-name-prefix = "ADX1";
 
 					ports {
@@ -963,7 +992,7 @@
 
 				tegra_adx2: adx@95a0000 {
 					compatible = "nvidia,tegra264-adx";
-					reg = <0x0 0x95a0000 0x0 0x10000>;
+					reg = <0x95a0000 0x10000>;
 					sound-name-prefix = "ADX2";
 
 					ports {
@@ -1014,7 +1043,7 @@
 
 				tegra_adx3: adx@95b0000 {
 					compatible = "nvidia,tegra264-adx";
-					reg = <0x0 0x95b0000 0x0 0x10000>;
+					reg = <0x95b0000 0x10000>;
 					sound-name-prefix = "ADX3";
 
 					ports {
@@ -1065,7 +1094,7 @@
 
 				tegra_adx4: adx@95c0000 {
 					compatible = "nvidia,tegra264-adx";
-					reg = <0x0 0x95c0000 0x0 0x10000>;
+					reg = <0x95c0000 0x10000>;
 					sound-name-prefix = "ADX4";
 
 					ports {
@@ -1116,7 +1145,7 @@
 
 				tegra_adx5: adx@95d0000 {
 					compatible = "nvidia,tegra264-adx";
-					reg = <0x0 0x95d0000 0x0 0x10000>;
+					reg = <0x95d0000 0x10000>;
 					sound-name-prefix = "ADX5";
 
 					ports {
@@ -1167,7 +1196,7 @@
 
 				tegra_adx6: adx@95e0000 {
 					compatible = "nvidia,tegra264-adx";
-					reg = <0x0 0x95e0000 0x0 0x10000>;
+					reg = <0x95e0000 0x10000>;
 					sound-name-prefix = "ADX6";
 
 					ports {
@@ -1218,76 +1247,75 @@
 
 				tegra_admaif: admaif@9610000 {
 					compatible = "nvidia,tegra264-admaif";
-					reg = <0x0 0x9610000 0x0 0x10000>;
+					reg = <0x9610000 0x10000>;
 					dmas = <&adma 1>, <&adma 1>,
-					     <&adma 2>, <&adma 2>,
-					     <&adma 3>, <&adma 3>,
-					     <&adma 4>, <&adma 4>,
-					     <&adma 5>, <&adma 5>,
-					     <&adma 6>, <&adma 6>,
-					     <&adma 7>, <&adma 7>,
-					     <&adma 8>, <&adma 8>,
-					     <&adma 9>, <&adma 9>,
-					     <&adma 10>, <&adma 10>,
-					     <&adma 11>, <&adma 11>,
-					     <&adma 12>, <&adma 12>,
-					     <&adma 13>, <&adma 13>,
-					     <&adma 14>, <&adma 14>,
-					     <&adma 15>, <&adma 15>,
-					     <&adma 16>, <&adma 16>,
-					     <&adma 17>, <&adma 17>,
-					     <&adma 18>, <&adma 18>,
-					     <&adma 19>, <&adma 19>,
-					     <&adma 20>, <&adma 20>,
-					     <&adma 21>, <&adma 21>,
-					     <&adma 22>, <&adma 22>,
-					     <&adma 23>, <&adma 23>,
-					     <&adma 24>, <&adma 24>,
-					     <&adma 25>, <&adma 25>,
-					     <&adma 26>, <&adma 26>,
-					     <&adma 27>, <&adma 27>,
-					     <&adma 28>, <&adma 28>,
-					     <&adma 29>, <&adma 29>,
-					     <&adma 30>, <&adma 30>,
-					     <&adma 31>, <&adma 31>,
-					     <&adma 32>, <&adma 32>;
+					       <&adma 2>, <&adma 2>,
+					       <&adma 3>, <&adma 3>,
+					       <&adma 4>, <&adma 4>,
+					       <&adma 5>, <&adma 5>,
+					       <&adma 6>, <&adma 6>,
+					       <&adma 7>, <&adma 7>,
+					       <&adma 8>, <&adma 8>,
+					       <&adma 9>, <&adma 9>,
+					       <&adma 10>, <&adma 10>,
+					       <&adma 11>, <&adma 11>,
+					       <&adma 12>, <&adma 12>,
+					       <&adma 13>, <&adma 13>,
+					       <&adma 14>, <&adma 14>,
+					       <&adma 15>, <&adma 15>,
+					       <&adma 16>, <&adma 16>,
+					       <&adma 17>, <&adma 17>,
+					       <&adma 18>, <&adma 18>,
+					       <&adma 19>, <&adma 19>,
+					       <&adma 20>, <&adma 20>,
+					       <&adma 21>, <&adma 21>,
+					       <&adma 22>, <&adma 22>,
+					       <&adma 23>, <&adma 23>,
+					       <&adma 24>, <&adma 24>,
+					       <&adma 25>, <&adma 25>,
+					       <&adma 26>, <&adma 26>,
+					       <&adma 27>, <&adma 27>,
+					       <&adma 28>, <&adma 28>,
+					       <&adma 29>, <&adma 29>,
+					       <&adma 30>, <&adma 30>,
+					       <&adma 31>, <&adma 31>,
+					       <&adma 32>, <&adma 32>;
 					dma-names = "rx1", "tx1",
-						"rx2", "tx2",
-						"rx3", "tx3",
-						"rx4", "tx4",
-						"rx5", "tx5",
-						"rx6", "tx6",
-						"rx7", "tx7",
-						"rx8", "tx8",
-						"rx9", "tx9",
-						"rx10", "tx10",
-						"rx11", "tx11",
-						"rx12", "tx12",
-						"rx13", "tx13",
-						"rx14", "tx14",
-						"rx15", "tx15",
-						"rx16", "tx16",
-						"rx17", "tx17",
-						"rx18", "tx18",
-						"rx19", "tx19",
-						"rx20", "tx20",
-						"rx21", "tx21",
-						"rx22", "tx22",
-						"rx23", "tx23",
-						"rx24", "tx24",
-						"rx25", "tx25",
-						"rx26", "tx26",
-						"rx27", "tx27",
-						"rx28", "tx28",
-						"rx29", "tx29",
-						"rx30", "tx30",
-						"rx31", "tx31",
-						"rx32", "tx32";
-
-					 interconnects =
-						<&mc TEGRA264_MEMORY_CLIENT_APEDMAR &emc>,
-						<&mc TEGRA264_MEMORY_CLIENT_APEDMAW &emc>;
-						interconnect-names = "dma-mem", "write";
+						    "rx2", "tx2",
+						    "rx3", "tx3",
+						    "rx4", "tx4",
+						    "rx5", "tx5",
+						    "rx6", "tx6",
+						    "rx7", "tx7",
+						    "rx8", "tx8",
+						    "rx9", "tx9",
+						    "rx10", "tx10",
+						    "rx11", "tx11",
+						    "rx12", "tx12",
+						    "rx13", "tx13",
+						    "rx14", "tx14",
+						    "rx15", "tx15",
+						    "rx16", "tx16",
+						    "rx17", "tx17",
+						    "rx18", "tx18",
+						    "rx19", "tx19",
+						    "rx20", "tx20",
+						    "rx21", "tx21",
+						    "rx22", "tx22",
+						    "rx23", "tx23",
+						    "rx24", "tx24",
+						    "rx25", "tx25",
+						    "rx26", "tx26",
+						    "rx27", "tx27",
+						    "rx28", "tx28",
+						    "rx29", "tx29",
+						    "rx30", "tx30",
+						    "rx31", "tx31",
+						    "rx32", "tx32";
+
+					interconnects = <&mc TEGRA264_MEMORY_CLIENT_APEDMAR &emc>,
+							<&mc TEGRA264_MEMORY_CLIENT_APEDMAW &emc>;
+					interconnect-names = "dma-mem", "write";
 
 					iommus = <&smmu1 TEGRA264_SID_APE>;
 
@@ -1555,8 +1583,8 @@
 
 				tegra_sfc1: sfc@9700000 {
 					compatible = "nvidia,tegra264-sfc",
-							"nvidia,tegra210-sfc";
-					reg = <0x0 0x9700000 0x0 0x10000>;
+						     "nvidia,tegra210-sfc";
+					reg = <0x9700000 0x10000>;
 					sound-name-prefix = "SFC1";
 
 					ports {
@@ -1583,8 +1611,8 @@
 
 				tegra_sfc2: sfc@9710000 {
 					compatible = "nvidia,tegra264-sfc",
-							"nvidia,tegra210-sfc";
-					reg = <0x0 0x9710000 0x0 0x10000>;
+						     "nvidia,tegra210-sfc";
+					reg = <0x9710000 0x10000>;
 					sound-name-prefix = "SFC2";
 
 					ports {
@@ -1611,8 +1639,8 @@
 
 				tegra_sfc3: sfc@9720000 {
 					compatible = "nvidia,tegra264-sfc",
-							"nvidia,tegra210-sfc";
-					reg = <0x0 0x9720000 0x0 0x10000>;
+						     "nvidia,tegra210-sfc";
+					reg = <0x9720000 0x10000>;
 					sound-name-prefix = "SFC3";
 
 					ports {
@@ -1639,8 +1667,8 @@
 
 				tegra_sfc4: sfc@9730000 {
 					compatible = "nvidia,tegra264-sfc",
-							"nvidia,tegra210-sfc";
-					reg = <0x0 0x9730000 0x0 0x10000>;
+						     "nvidia,tegra210-sfc";
+					reg = <0x9730000 0x10000>;
 					sound-name-prefix = "SFC4";
 
 					ports {
@@ -1667,23 +1695,23 @@
 
 				tegra_ope1: processing-engine@9780000 {
 					compatible = "nvidia,tegra264-ope",
-							"nvidia,tegra210-ope";
-					reg = <0x0 0x9780000 0x0 0x10000>;
-					#address-cells = <2>;
-					#size-cells = <2>;
-					ranges = <0x0 0x9780000 0x0 0x9780000 0x0 0x30000>;
+						     "nvidia,tegra210-ope";
+					reg = <0x9780000 0x10000>;
+					#address-cells = <1>;
+					#size-cells = <1>;
+					ranges = <0x9780000 0x9780000 0x30000>;
 					sound-name-prefix = "OPE1";
 
 					equalizer@9790000 {
 						compatible = "nvidia,tegra264-peq",
-								"nvidia,tegra210-peq";
-						reg = <0x0 0x9790000 0x0 0x10000>;
+							     "nvidia,tegra210-peq";
+						reg = <0x9790000 0x10000>;
 					};
 
 					dynamic-range-compressor@97a0000 {
 						compatible = "nvidia,tegra264-mbdrc",
-								"nvidia,tegra210-mbdrc";
-						reg = <0x0 0x97a0000 0x0 0x10000>;
+							     "nvidia,tegra210-mbdrc";
+						reg = <0x97a0000 0x10000>;
 					};
 
 					ports {
@@ -1694,8 +1722,7 @@
 							reg = <0x0>;
 
 							ope1_cif_in_ep: endpoint {
-								remote-endpoint =
-									<&xbar_ope1_in_ep>;
+								remote-endpoint = <&xbar_ope1_in_ep>;
 							};
 						};
 
@@ -1703,8 +1730,7 @@
 							reg = <0x1>;
 
 							ope1_cif_out_ep: endpoint {
-								remote-endpoint =
-									<&xbar_ope1_out_ep>;
+								remote-endpoint = <&xbar_ope1_out_ep>;
 							};
 						};
 					};
@@ -1712,8 +1738,8 @@
 
 				tegra_mvc1: mvc@9800000 {
 					compatible = "nvidia,tegra264-mvc",
-							"nvidia,tegra210-mvc";
-					reg = <0x0 0x9800000 0x0 0x10000>;
+						     "nvidia,tegra210-mvc";
+					reg = <0x9800000 0x10000>;
 					sound-name-prefix = "MVC1";
 
 					ports {
@@ -1740,8 +1766,8 @@
 
 				tegra_mvc2: mvc@9810000 {
 					compatible = "nvidia,tegra264-mvc",
-							"nvidia,tegra210-mvc";
-					reg = <0x0 0x9810000 0x0 0x10000>;
+						     "nvidia,tegra210-mvc";
+					reg = <0x9810000 0x10000>;
 					sound-name-prefix = "MVC2";
 
 					ports {
@@ -1768,8 +1794,8 @@
 
 				tegra_amixer: amixer@9820000 {
 					compatible = "nvidia,tegra264-amixer",
-							"nvidia,tegra210-amixer";
-					reg = <0x0 0x9820000 0x0 0x10000>;
+						     "nvidia,tegra210-amixer";
+					reg = <0x9820000 0x10000>;
 					sound-name-prefix = "MIXER1";
 
 					ports {
@@ -1900,7 +1926,7 @@
 
 				tegra_asrc: asrc@9850000 {
 					compatible = "nvidia,tegra264-asrc";
-					reg = <0x0 0x9850000 0x0 0x10000>;
+					reg = <0x9850000 0x10000>;
 					sound-name-prefix = "ASRC1";
 
 					ports {
@@ -1911,8 +1937,7 @@
 							reg = <0x0>;
 
 							asrc_in1_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_in1_ep>;
+								remote-endpoint = <&xbar_asrc_in1_ep>;
 							};
 						};
 
@@ -1920,8 +1945,7 @@
 							reg = <0x1>;
 
 							asrc_in2_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_in2_ep>;
+								remote-endpoint = <&xbar_asrc_in2_ep>;
 							};
 						};
 
@@ -1929,8 +1953,7 @@
 							reg = <0x2>;
 
 							asrc_in3_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_in3_ep>;
+								remote-endpoint = <&xbar_asrc_in3_ep>;
 							};
 						};
 
@@ -1938,8 +1961,7 @@
 							reg = <0x3>;
 
 							asrc_in4_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_in4_ep>;
+								remote-endpoint = <&xbar_asrc_in4_ep>;
 							};
 						};
 
@@ -1947,8 +1969,7 @@
 							reg = <0x4>;
 
 							asrc_in5_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_in5_ep>;
+								remote-endpoint = <&xbar_asrc_in5_ep>;
 							};
 						};
 
@@ -1956,8 +1977,7 @@
 							reg = <0x5>;
 
 							asrc_in6_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_in6_ep>;
+								remote-endpoint = <&xbar_asrc_in6_ep>;
 							};
 						};
 
@@ -1965,8 +1985,7 @@
 							reg = <0x6>;
 
 							asrc_in7_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_in7_ep>;
+								remote-endpoint = <&xbar_asrc_in7_ep>;
 							};
 						};
 
@@ -1974,8 +1993,7 @@
 							reg = <0x7>;
 
 							asrc_out1_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_out1_ep>;
+								remote-endpoint = <&xbar_asrc_out1_ep>;
 							};
 						};
 
@@ -1983,8 +2001,7 @@
 							reg = <0x8>;
 
 							asrc_out2_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_out2_ep>;
+								remote-endpoint = <&xbar_asrc_out2_ep>;
 							};
 						};
 
@@ -1992,8 +2009,7 @@
 							reg = <0x9>;
 
 							asrc_out3_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_out3_ep>;
+								remote-endpoint = <&xbar_asrc_out3_ep>;
 							};
 						};
 
@@ -2001,8 +2017,7 @@
 							reg = <0xa>;
 
 							asrc_out4_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_out4_ep>;
+								remote-endpoint = <&xbar_asrc_out4_ep>;
 							};
 						};
 
@@ -2010,8 +2025,7 @@
 							reg = <0xb>;
 
 							asrc_out5_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_out5_ep>;
+								remote-endpoint = <&xbar_asrc_out5_ep>;
 							};
 						};
 
@@ -2019,8 +2033,7 @@
 							reg = <0xc>;
 
 							asrc_out6_ep: endpoint {
-								remote-endpoint =
-									<&xbar_asrc_out6_ep>;
+								remote-endpoint = <&xbar_asrc_out6_ep>;
 							};
 						};
 					};
@@ -3194,13 +3207,12 @@
 
 			agic_page0: interrupt-controller@9960000 {
 				compatible = "nvidia,tegra264-agic",
-						"nvidia,tegra210-agic";
+					     "nvidia,tegra210-agic";
 				#interrupt-cells = <3>;
 				interrupt-controller;
-				reg = <0x0 0x9961000 0x0 0x1000>,
-					<0x0 0x9962000 0x0 0x1000>;
-				interrupts = <GIC_SPI 0x230
-					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+				reg = <0x9961000 0x1000>,
+				      <0x9962000 0x1000>;
+				interrupts = <GIC_SPI 0x230 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 				clocks = <&bpmp TEGRA264_CLK_ADSP>;
 				clock-names = "clk";
 				status = "disabled";
@@ -3208,13 +3220,12 @@
 
 			agic_page1: interrupt-controller@9970000 {
 				compatible = "nvidia,tegra264-agic",
-					   "nvidia,tegra210-agic";
+					     "nvidia,tegra210-agic";
 				#interrupt-cells = <3>;
 				interrupt-controller;
-				reg = <0x0 0x9971000 0x0 0x1000>,
-					<0x0 0x9972000 0x0 0x1000>;
-				interrupts = <GIC_SPI 0x231
-					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+				reg = <0x9971000 0x1000>,
+				      <0x9972000 0x1000>;
+				interrupts = <GIC_SPI 0x231 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 				clocks = <&bpmp TEGRA264_CLK_ADSP>;
 				clock-names = "clk";
 				status = "disabled";
@@ -3222,13 +3233,12 @@
 
 			agic_page2: interrupt-controller@9980000 {
 				compatible = "nvidia,tegra264-agic",
-						"nvidia,tegra210-agic";
+					     "nvidia,tegra210-agic";
 				#interrupt-cells = <3>;
 				interrupt-controller;
-				reg = <0x0 0x9981000 0x0 0x1000>,
-					<0x0 0x9982000 0x0 0x1000>;
-				interrupts = <GIC_SPI 0x232
-					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+				reg = <0x9981000 0x1000>,
+				      <0x9982000 0x1000>;
+				interrupts = <GIC_SPI 0x232 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 				clocks = <&bpmp TEGRA264_CLK_ADSP>;
 				clock-names = "clk";
 				status = "disabled";
@@ -3236,13 +3246,12 @@
 
 			agic_page3: interrupt-controller@9990000 {
 				compatible = "nvidia,tegra264-agic",
-						"nvidia,tegra210-agic";
+					     "nvidia,tegra210-agic";
 				#interrupt-cells = <3>;
 				interrupt-controller;
-				reg = <0x0 0x9991000 0x0 0x1000>,
-					<0x0 0x9992000 0x0 0x1000>;
-				interrupts = <GIC_SPI 0x233
-					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+				reg = <0x9991000 0x1000>,
+				      <0x9992000 0x1000>;
+				interrupts = <GIC_SPI 0x233 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 				clocks = <&bpmp TEGRA264_CLK_ADSP>;
 				clock-names = "clk";
 				status = "disabled";
@@ -3250,13 +3259,12 @@
 
 			agic_page4: interrupt-controller@99a0000 {
 				compatible = "nvidia,tegra264-agic",
-						"nvidia,tegra210-agic";
+					     "nvidia,tegra210-agic";
 				#interrupt-cells = <3>;
 				interrupt-controller;
-				reg = <0x0 0x99a1000 0x0 0x1000>,
-					<0x0 0x99a2000 0x0 0x1000>;
-				interrupts = <GIC_SPI 0x234
-					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+				reg = <0x99a1000 0x1000>,
+				      <0x99a2000 0x1000>;
+				interrupts = <GIC_SPI 0x234 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 				clocks = <&bpmp TEGRA264_CLK_ADSP>;
 				clock-names = "clk";
 				status = "disabled";
@@ -3264,13 +3272,12 @@
 
 			agic_page5: interrupt-controller@99b0000 {
 				compatible = "nvidia,tegra264-agic",
-						"nvidia,tegra210-agic";
+					     "nvidia,tegra210-agic";
 				#interrupt-cells = <3>;
 				interrupt-controller;
-				reg = <0x0 0x99b1000 0x0 0x1000>,
-					<0x0 0x99b2000 0x0 0x1000>;
-				interrupts = <GIC_SPI 0x235
-					(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+				reg = <0x99b1000 0x1000>,
+				      <0x99b2000 0x1000>;
+				interrupts = <GIC_SPI 0x235 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
 				clocks = <&bpmp TEGRA264_CLK_ADSP>;
 				clock-names = "clk";
 				status = "disabled";
@@ -3330,6 +3337,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x0 0x0c600000 0x0 0x10000>;
 			interrupts = <GIC_SPI 532 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_AON_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLAON>;
@@ -3345,6 +3354,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x0 0x0c610000 0x0 0x10000>;
 			interrupts = <GIC_SPI 533 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_AON_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLAON>;
@@ -3356,6 +3367,18 @@
 			status = "disabled";
 		};
 
+		pwm4: pwm@c6a0000 {
+			compatible = "nvidia,tegra264-pwm";
+			reg = <0x0 0xc6a0000 0x0 0x10000>;
+			status = "disabled";
+
+			clocks = <&bpmp TEGRA264_CLK_PWM4>;
+			resets = <&bpmp TEGRA264_RESET_PWM4>;
+			reset-names = "pwm";
+
+			#pwm-cells = <2>;
+		};
+
 		pinmux_aon: pinmux@c7a2000 {
 			compatible = "nvidia,tegra264-pinmux-aon";
 			reg = <0x0 0x0c7a2000 0x0 0x2000>;
@@ -3537,11 +3560,11 @@
 
 		pci@c000000 {
 			compatible = "nvidia,tegra264-pcie";
-			reg = <0xd0 0xb0000000 0x0 0x10000000>,
-			      <0x00 0x0c000000 0x0 0x00004000>,
+			reg = <0x00 0x0c000000 0x0 0x00004000>,
 			      <0x00 0x0c004000 0x0 0x00001000>,
-			      <0x00 0x0c005000 0x0 0x00001000>;
-			reg-names = "ecam", "xal", "xtl", "xtl-pri";
+			      <0x00 0x0c005000 0x0 0x00001000>,
+			      <0xd0 0xb0000000 0x0 0x10000000>;
+			reg-names = "xal", "xtl", "xtl-pri", "ecam";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -3565,6 +3588,16 @@
 
 			nvidia,bpmp = <&bpmp 0>;
 			status = "disabled";
+
+			pcie@0 {
+				device_type = "pci";
+				compatible = "pciclass,0604";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pinmux_main: pinmux@c281000 {
@@ -3620,6 +3653,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c410000 0x0 0x10000>;
 			interrupts = <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3635,6 +3670,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c420000 0x0 0x10000>;
 			interrupts = <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3650,6 +3687,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c430000 0x0 0x10000>;
 			interrupts = <GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3661,10 +3700,72 @@
 			status = "disabled";
 		};
 
+		pwm2: pwm@c5e0000 {
+			compatible = "nvidia,tegra264-pwm";
+			reg = <0x0 0xc5e0000 0x0 0x10000>;
+			status = "disabled";
+
+			clocks = <&bpmp TEGRA264_CLK_PWM2>;
+			resets = <&bpmp TEGRA264_RESET_PWM2>;
+			reset-names = "pwm";
+
+			#pwm-cells = <2>;
+		};
+
+		pwm3: pwm@c5f0000 {
+			compatible = "nvidia,tegra264-pwm";
+			reg = <0x0 0xc5f0000 0x0 0x10000>;
+			status = "disabled";
+
+			clocks = <&bpmp TEGRA264_CLK_PWM3>;
+			resets = <&bpmp TEGRA264_RESET_PWM3>;
+			reset-names = "pwm";
+
+			#pwm-cells = <2>;
+		};
+
+		pwm5: pwm@c600000 {
+			compatible = "nvidia,tegra264-pwm";
+			reg = <0x0 0xc600000 0x0 0x10000>;
+			status = "disabled";
+
+			clocks = <&bpmp TEGRA264_CLK_PWM5>;
+			resets = <&bpmp TEGRA264_RESET_PWM5>;
+			reset-names = "pwm";
+
+			#pwm-cells = <2>;
+		};
+
+		pwm9: pwm@c610000 {
+			compatible = "nvidia,tegra264-pwm";
+			reg = <0x0 0xc610000 0x0 0x10000>;
+			status = "disabled";
+
+			clocks = <&bpmp TEGRA264_CLK_PWM9>;
+			resets = <&bpmp TEGRA264_RESET_PWM9>;
+			reset-names = "pwm";
+
+			#pwm-cells = <2>;
+		};
+
+		pwm10: pwm@c620000 {
+			compatible = "nvidia,tegra264-pwm";
+			reg = <0x0 0xc620000 0x0 0x10000>;
+			status = "disabled";
+
+			clocks = <&bpmp TEGRA264_CLK_PWM10>;
+			resets = <&bpmp TEGRA264_RESET_PWM10>;
+			reset-names = "pwm";
+
+			#pwm-cells = <2>;
+		};
+
 		i2c0: i2c@c630000 {
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c630000 0x0 0x10000>;
 			interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3680,6 +3781,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c640000 0x0 0x10000>;
 			interrupts = <GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3695,6 +3798,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c650000 0x0 0x10000>;
 			interrupts = <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3710,6 +3815,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c670000 0x0 0x10000>;
 			interrupts = <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3725,6 +3832,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c680000 0x0 0x10000>;
 			interrupts = <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3740,6 +3849,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c690000 0x0 0x10000>;
 			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3755,6 +3866,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c6a0000 0x0 0x10000>;
 			interrupts = <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3770,6 +3883,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c6b0000 0x0 0x10000>;
 			interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3785,6 +3900,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c6c0000 0x0 0x10000>;
 			interrupts = <GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3800,6 +3917,8 @@
 			compatible = "nvidia,tegra264-i2c";
 			reg = <0x00 0x0c6d0000 0x0 0x10000>;
 			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
 			clock-frequency = <400000>;
 			clocks = <&bpmp TEGRA264_CLK_TOP_I2C>,
 				 <&bpmp TEGRA264_CLK_PLLP_OUT0>;
@@ -3990,12 +4109,12 @@
 
 		pci@8400000 {
 			compatible = "nvidia,tegra264-pcie";
-			reg = <0xa8 0xb0000000 0x0 0x10000000>,
-			      <0x00 0x08400000 0x0 0x00004000>,
+			reg = <0x00 0x08400000 0x0 0x00004000>,
 			      <0x00 0x08404000 0x0 0x00001000>,
 			      <0x00 0x08405000 0x0 0x00001000>,
-			      <0x00 0x08410000 0x0 0x00010000>;
-			reg-names = "ecam", "xal", "xtl", "xtl-pri", "xpl";
+			      <0x00 0x08410000 0x0 0x00010000>,
+			      <0xa8 0xb0000000 0x0 0x10000000>;
+			reg-names = "xal", "xtl", "xtl-pri", "xpl", "ecam";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -4018,16 +4137,26 @@
 
 			nvidia,bpmp = <&bpmp 1>;
 			status = "disabled";
+
+			pcie@0 {
+				device_type = "pci";
+				compatible = "pciclass,0604";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pci@8420000 {
 			compatible = "nvidia,tegra264-pcie";
-			reg = <0xb0 0xb0000000 0x0 0x10000000>,
-			      <0x00 0x08420000 0x0 0x00004000>,
+			reg = <0x00 0x08420000 0x0 0x00004000>,
 			      <0x00 0x08424000 0x0 0x00001000>,
 			      <0x00 0x08425000 0x0 0x00001000>,
-			      <0x00 0x08430000 0x0 0x00010000>;
-			reg-names = "ecam", "xal", "xtl", "xtl-pri", "xpl";
+			      <0x00 0x08430000 0x0 0x00010000>,
+			      <0xb0 0xb0000000 0x0 0x10000000>;
+			reg-names = "xal", "xtl", "xtl-pri", "xpl", "ecam";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -4050,16 +4179,26 @@
 
 			nvidia,bpmp = <&bpmp 2>;
 			status = "disabled";
+
+			pcie@0 {
+				device_type = "pci";
+				compatible = "pciclass,0604";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pci@8440000 {
 			compatible = "nvidia,tegra264-pcie";
-			reg = <0xb8 0xb0000000 0x0 0x10000000>,
-			      <0x00 0x08440000 0x0 0x00004000>,
+			reg = <0x00 0x08440000 0x0 0x00004000>,
 			      <0x00 0x08444000 0x0 0x00001000>,
 			      <0x00 0x08445000 0x0 0x00001000>,
-			      <0x00 0x08450000 0x0 0x00010000>;
-			reg-names = "ecam", "xal", "xtl", "xtl-pri", "xpl";
+			      <0x00 0x08450000 0x0 0x00010000>,
+			      <0xb8 0xb0000000 0x0 0x10000000>;
+			reg-names = "xal", "xtl", "xtl-pri", "xpl", "ecam";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -4082,16 +4221,26 @@
 
 			nvidia,bpmp = <&bpmp 3>;
 			status = "disabled";
+
+			pcie@0 {
+				device_type = "pci";
+				compatible = "pciclass,0604";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pci@8460000 {
 			compatible = "nvidia,tegra264-pcie";
-			reg = <0xc0 0xb0000000 0x0 0x10000000>,
-			      <0x00 0x08460000 0x0 0x00004000>,
+			reg = <0x00 0x08460000 0x0 0x00004000>,
 			      <0x00 0x08464000 0x0 0x00001000>,
 			      <0x00 0x08465000 0x0 0x00001000>,
-			      <0x00 0x08470000 0x0 0x00010000>;
-			reg-names = "ecam", "xal", "xtl", "xtl-pri", "xpl";
+			      <0x00 0x08470000 0x0 0x00010000>,
+			      <0xc0 0xb0000000 0x0 0x10000000>;
+			reg-names = "xal", "xtl", "xtl-pri", "xpl", "ecam";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -4114,16 +4263,26 @@
 
 			nvidia,bpmp = <&bpmp 4>;
 			status = "disabled";
+
+			pcie@0 {
+				device_type = "pci";
+				compatible = "pciclass,0604";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 
 		pci@8480000 {
 			compatible = "nvidia,tegra264-pcie";
-			reg = <0xc8 0xb0000000 0x0 0x10000000>,
-			      <0x00 0x08480000 0x0 0x00004000>,
+			reg = <0x00 0x08480000 0x0 0x00004000>,
 			      <0x00 0x08484000 0x0 0x00001000>,
 			      <0x00 0x08485000 0x0 0x00001000>,
-			      <0x00 0x08490000 0x0 0x00010000>;
-			reg-names = "ecam", "xal", "xtl", "xtl-pri", "xpl";
+			      <0x00 0x08490000 0x0 0x00010000>,
+			      <0xc8 0xb0000000 0x0 0x10000000>;
+			reg-names = "xal", "xtl", "xtl-pri", "xpl", "ecam";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
@@ -4146,6 +4305,16 @@
 
 			nvidia,bpmp = <&bpmp 5>;
 			status = "disabled";
+
+			pcie@0 {
+				device_type = "pci";
+				compatible = "pciclass,0604";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
 		};
 	};
 
@@ -4553,6 +4722,33 @@
 		status = "disabled";
 	};
 
+	thermal-zones {
+		cpu-thermal {
+			thermal-sensors = <&{/bpmp/thermal} TEGRA264_THERMAL_ZONE_CPU_MAX>;
+			status = "disabled";
+		};
+
+		gpu-thermal {
+			thermal-sensors = <&{/bpmp/thermal} TEGRA264_THERMAL_ZONE_GPU_MAX>;
+			status = "disabled";
+		};
+
+		soc012-thermal {
+			thermal-sensors = <&{/bpmp/thermal} TEGRA264_THERMAL_ZONE_SOC_012_MAX>;
+			status = "disabled";
+		};
+
+		soc345-thermal {
+			thermal-sensors = <&{/bpmp/thermal} TEGRA264_THERMAL_ZONE_SOC_345_MAX>;
+			status = "disabled";
+		};
+
+		tj-thermal {
+			thermal-sensors = <&{/bpmp/thermal} TEGRA264_THERMAL_ZONE_TJ_MAX>;
+			status = "disabled";
+		};
+	};
+
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
diff --git a/arch/arm64/boot/dts/nvidia/thermal/nvidia,tegra264-bpmp.h b/arch/arm64/boot/dts/nvidia/thermal/nvidia,tegra264-bpmp.h
new file mode 100644
index 00000000000000..29695881f4cc01
--- /dev/null
+++ b/arch/arm64/boot/dts/nvidia/thermal/nvidia,tegra264-bpmp.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/* SPDX-FileCopyrightText: Copyright (c) 2023-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#ifndef DT_BINDINGS_THERMAL_NVIDIA_TEGRA264_BPMP_H
+#define DT_BINDINGS_THERMAL_NVIDIA_TEGRA264_BPMP_H
+
+#define TEGRA264_THERMAL_ZONE_TJ_MAX		0
+#define TEGRA264_THERMAL_ZONE_TJ_MIN		1
+#define TEGRA264_THERMAL_ZONE_GPU_AVG		2
+#define TEGRA264_THERMAL_ZONE_CPU_AVG		3
+#define TEGRA264_THERMAL_ZONE_SOC_012_AVG	4
+#define TEGRA264_THERMAL_ZONE_SOC_45_AVG	5
+#define TEGRA264_THERMAL_ZONE_SOC_3_AVG		6
+#define TEGRA264_THERMAL_ZONE_GPU_MAX		7
+#define TEGRA264_THERMAL_ZONE_CPU_MAX		8
+#define TEGRA264_THERMAL_ZONE_SOC_012_MAX	9
+#define TEGRA264_THERMAL_ZONE_SOC_345_MAX	10
+#define TEGRA264_THERMAL_ZONE_TJ_AVG		11
+
+#endif /* DT_BINDINGS_THERMAL_NVIDIA_TEGRA264_BPMP_H */
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index 1c86e7e98f55d5..624c9f9a86e149 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -16,14 +16,19 @@ dtb-$(CONFIG_ARCH_QCOM)	+= apq8096sg-db820c.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= apq8096-ifc6640.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= eliza-cqs-evk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= eliza-mtp.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= glymur-asus-zenbook-a14-ux3407na.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= glymur-asus-zenbook-a16-ux3607oa.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= glymur-crd.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= glymur-hp-elitebook-x-g2q.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= glymur-lenovo-yoga-slim7x.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= hamoa-advantech-som-db5830.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= hamoa-iot-evk.dtb
 
 hamoa-iot-evk-el2-dtbs	:= hamoa-iot-evk.dtb x1-el2.dtbo
 
 dtb-$(CONFIG_ARCH_QCOM)	+= hamoa-iot-evk-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= hamoa-lenovo-ideacentre-mini-01q8x10.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= hawi-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq5018-rdp432-c2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq5018-tplink-archer-ax55-v1.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq5210-rdp504.dtb
@@ -58,9 +63,11 @@ dtb-$(CONFIG_ARCH_QCOM)	+= lemans-evk-el2.dtb
 lemans-evk-ifp-mezzanine-dtbs	:= lemans-evk.dtb lemans-evk-ifp-mezzanine.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= lemans-evk-ifp-mezzanine.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= mahua-crd.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= mahua-lenovo-thinkpad-t14s-gen7-lcd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= milos-fairphone-fp6.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= milos-nothing-asteroids.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= monaco-arduino-monza.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= monaco-ac-evk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk.dtb
 
 monaco-evk-camera-imx577-dtbs	:= monaco-evk.dtb monaco-evk-camera-imx577.dtbo
@@ -68,6 +75,9 @@ dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk-camera-imx577.dtb
 
 monaco-evk-el2-dtbs := monaco-evk.dtb monaco-el2.dtbo
 
+monaco-ac-evk-ifp-mezzanine-dtbs	:= monaco-ac-evk.dtb monaco-evk-ifp-mezzanine.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= monaco-ac-evk-ifp-mezzanine.dtb
+
 dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk-el2.dtb
 monaco-evk-ifp-mezzanine-dtbs	:= monaco-evk.dtb monaco-evk-ifp-mezzanine.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk-ifp-mezzanine.dtb
@@ -160,6 +170,8 @@ dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-lilac.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-maple.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-poplar.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-xiaomi-sagit.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= nord-ride.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= nord-rrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= purwa-iot-evk.dtb
 
 purwa-iot-evk-el2-dtbs	:= purwa-iot-evk.dtb x1-el2.dtbo
@@ -186,6 +198,21 @@ dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-rb3gen2-industrial-mezzanine.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-rb3gen2-vision-mezzanine.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-minipc-g1iot.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-rubikpi3.dtb
+
+qcs6490-thundercomm-rubikpi3-cam1-imx219-dtbs := \
+	qcs6490-thundercomm-rubikpi3.dtb \
+	qcs6490-thundercomm-rubikpi3-cam1-imx219.dtbo
+qcs6490-thundercomm-rubikpi3-cam2-imx219-dtbs := \
+	qcs6490-thundercomm-rubikpi3.dtb \
+	qcs6490-thundercomm-rubikpi3-cam2-imx219.dtbo
+qcs6490-thundercomm-rubikpi3-dual-imx219-dtbs := \
+	qcs6490-thundercomm-rubikpi3.dtb \
+	qcs6490-thundercomm-rubikpi3-cam1-imx219.dtbo \
+	qcs6490-thundercomm-rubikpi3-cam2-imx219.dtbo
+
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-rubikpi3-cam1-imx219.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-rubikpi3-cam2-imx219.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-thundercomm-rubikpi3-dual-imx219.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs6490-vicharak-axon-mini.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8300-ride.dtb
 
@@ -193,6 +220,10 @@ qcs8300-ride-el2-dtbs := qcs8300-ride.dtb monaco-el2.dtbo
 
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8300-ride-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-aim300-aiot.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-ayntec-odin2mini.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-ayntec-odin2portal.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-ayntec-thor.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-imdt-sbc.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs8550-rb5gen2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs9100-ride.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcs9100-ride-r3.dtb
@@ -339,6 +370,14 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sdx75-idp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= shikra-cqm-evk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= shikra-cqs-evk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= shikra-iqs-evk.dtb
+
+shikra-cqm-evk-imx577-camera-dtbs	:= shikra-cqm-evk.dtb shikra-cqm-cqs-evk-imx577-camera.dtbo
+shikra-cqs-evk-imx577-camera-dtbs	:= shikra-cqs-evk.dtb shikra-cqm-cqs-evk-imx577-camera.dtbo
+shikra-iqs-evk-imx577-camera-dtbs	:= shikra-iqs-evk.dtb shikra-iqs-evk-imx577-camera.dtbo
+
+dtb-$(CONFIG_ARCH_QCOM)	+= shikra-cqm-evk-imx577-camera.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= shikra-cqs-evk-imx577-camera.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= shikra-iqs-evk-imx577-camera.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm4250-oneplus-billie2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm4450-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm6115-fxtec-pro1x.dtb
@@ -356,6 +395,7 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-motorola-dubai.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-motorola-dubai-csot.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-motorola-dubai-tianma.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-nothing-spacewar.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-xiaomi-lisa.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-xiaomi-taoyao.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8150-hdk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8150-microsoft-surface-duo.dtb
@@ -406,6 +446,7 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sm8650-hdk-rear-camera-card.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8650-hdk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8650-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8650-qrd.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm8650-valve-deckard.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8750-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8750-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= talos-evk.dtb
@@ -421,6 +462,9 @@ dtb-$(CONFIG_ARCH_QCOM)	+= talos-evk-camera-imx577.dtb
 talos-evk-lvds-auo,g133han01-dtbs	:= \
 	talos-evk.dtb talos-evk-lvds-auo,g133han01.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= talos-evk-lvds-auo,g133han01.dtb
+talos-evk-rpi-display-2-5in-dtbs	:= \
+	talos-evk.dtb talos-evk-rpi-display-2-5in.dtbo
+dtb-$(CONFIG_ARCH_QCOM)	+= talos-evk-rpi-display-2-5in.dtb
 x1e001de-devkit-el2-dtbs	:= x1e001de-devkit.dtb x1-el2.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= x1e001de-devkit.dtb x1e001de-devkit-el2.dtb
 x1e78100-lenovo-thinkpad-t14s-el2-dtbs	:= x1e78100-lenovo-thinkpad-t14s.dtb x1-el2.dtbo
diff --git a/arch/arm64/boot/dts/qcom/agatti.dtsi b/arch/arm64/boot/dts/qcom/agatti.dtsi
index 590bd2432d85ba..d9f4cbfd2861cd 100644
--- a/arch/arm64/boot/dts/qcom/agatti.dtsi
+++ b/arch/arm64/boot/dts/qcom/agatti.dtsi
@@ -381,6 +381,14 @@
 			qcom,client-id = <1>;
 			qcom,vmid = <QCOM_SCM_VMID_MSS_MSA QCOM_SCM_VMID_NAV>;
 		};
+
+		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0x0 0x0 0x1 0x0>;
+			reusable;
+			alignment = <0x0 0x400000>;
+			size = <0x0 0x800000>;
+		};
 	};
 
 	smp2p-adsp {
@@ -1572,12 +1580,15 @@
 			};
 		};
 
-		usb: usb@4ef8800 {
-			compatible = "qcom,qcm2290-dwc3", "qcom,dwc3";
-			reg = <0x0 0x04ef8800 0x0 0x400>;
-			interrupts-extended = <&intc GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH>,
+		usb: usb@4e00000 {
+			compatible = "qcom,qcm2290-dwc3", "qcom,snps-dwc3";
+			reg = <0x0 0x04e00000 0x0 0xfc100>;
+
+			interrupts-extended = <&intc GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>,
+					      <&intc GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH>,
 					      <&mpm 12 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "hs_phy_irq",
+			interrupt-names = "dwc_usb3",
+					  "hs_phy_irq",
 					  "ss_phy_irq";
 
 			clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
@@ -1595,7 +1606,8 @@
 
 			assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>,
 					  <&gcc GCC_USB30_PRIM_MASTER_CLK>;
-			assigned-clock-rates = <19200000>, <133333333>;
+			assigned-clock-rates = <19200000>,
+					       <133333333>;
 
 			resets = <&gcc GCC_USB30_PRIM_BCR>;
 			power-domains = <&gcc GCC_USB30_PRIM_GDSC>;
@@ -1606,48 +1618,42 @@
 					 &config_noc SLAVE_USB3 RPM_ALWAYS_TAG>;
 			interconnect-names = "usb-ddr",
 					     "apps-usb";
-			wakeup-source;
 
-			#address-cells = <2>;
-			#size-cells = <2>;
-			ranges;
+			iommus = <&apps_smmu 0x120 0x0>;
 
-			status = "disabled";
+			phys = <&usb_hsphy>,
+			       <&usb_qmpphy>;
+			phy-names = "usb2-phy",
+				    "usb3-phy";
 
-			usb_dwc3: usb@4e00000 {
-				compatible = "snps,dwc3";
-				reg = <0x0 0x04e00000 0x0 0xcd00>;
-				interrupts = <GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>;
-				phys = <&usb_hsphy>, <&usb_qmpphy>;
-				phy-names = "usb2-phy", "usb3-phy";
-				iommus = <&apps_smmu 0x120 0x0>;
-				snps,dis_u2_susphy_quirk;
-				snps,dis_enblslpm_quirk;
-				snps,has-lpm-erratum;
-				snps,hird-threshold = /bits/ 8 <0x10>;
-				snps,usb3_lpm_capable;
-				snps,parkmode-disable-ss-quirk;
-				maximum-speed = "super-speed";
-				dr_mode = "otg";
-				usb-role-switch;
+			snps,hird-threshold = /bits/ 8 <0x10>;
+			snps,parkmode-disable-ss-quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_enblslpm_quirk;
+			snps,has-lpm-erratum;
+			snps,usb3_lpm_capable;
 
-				ports {
-					#address-cells = <1>;
-					#size-cells = <0>;
+			usb-role-switch;
+			wakeup-source;
 
-					port@0 {
-						reg = <0>;
+			status = "disabled";
 
-						usb_dwc3_hs: endpoint {
-						};
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_dwc3_hs: endpoint {
 					};
+				};
 
-					port@1 {
-						reg = <1>;
+				port@1 {
+					reg = <1>;
 
-						usb_dwc3_ss: endpoint {
-							remote-endpoint = <&usb_qmpphy_usb_ss_in>;
-						};
+					usb_dwc3_ss: endpoint {
+						remote-endpoint = <&usb_qmpphy_usb_ss_in>;
 					};
 				};
 			};
@@ -1917,7 +1923,7 @@
 				 <&gcc GCC_CAMSS_TFE_0_CLK>,
 				 <&gcc GCC_CAMSS_TFE_0_CPHY_RX_CLK>,
 				 <&gcc GCC_CAMSS_TFE_1_CLK>,
-				 <&gcc GCC_CAMSS_TFE_1_CPHY_RX_CLK> ;
+				 <&gcc GCC_CAMSS_TFE_1_CPHY_RX_CLK>;
 			clock-names = "ahb",
 				      "axi",
 				      "camnoc_nrt_axi",
@@ -2348,6 +2354,9 @@
 					compatible = "qcom,fastrpc";
 					qcom,glink-channels = "fastrpcglink-apps-dsp";
 					label = "adsp";
+					memory-region = <&adsp_rpc_remote_heap_mem>;
+					qcom,vmids = <QCOM_SCM_VMID_LPASS
+						      QCOM_SCM_VMID_ADSP_HEAP>;
 
 					qcom,non-secure-domain;
 
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
index 0c076852b4946a..ad435a13ba24c1 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
@@ -458,23 +458,32 @@
 
 &pcie0 {
 	status = "okay";
-	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
 	vddpe-3v3-supply = <&wlan_en>;
 	vdda-supply = <&vreg_l28a_0p925>;
 };
 
+&pcie0_port0 {
+	reset-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+};
+
 &pcie1 {
 	status = "okay";
-	perst-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>;
 	vdda-supply = <&vreg_l28a_0p925>;
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>;
+};
+
 &pcie2 {
 	status = "okay";
-	perst-gpios = <&tlmm 114 GPIO_ACTIVE_LOW>;
 	vdda-supply = <&vreg_l28a_0p925>;
 };
 
+&pcie2_port0 {
+	reset-gpios = <&tlmm 114 GPIO_ACTIVE_LOW>;
+};
+
 &pcie_phy {
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi b/arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi
index 5ef6c51618bf91..03f9192324c73d 100644
--- a/arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/eliza-cqs-som.dtsi
@@ -374,6 +374,14 @@
 	vdd3-supply = <&vreg_l17b>;
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/eliza/gen71700_zap.mbn";
+};
+
 &remoteproc_adsp {
 	firmware-name = "qcom/eliza/adsp.mbn",
 			"qcom/eliza/adsp_dtb.mbn";
@@ -381,6 +389,13 @@
 	status = "okay";
 };
 
+&remoteproc_cdsp {
+	firmware-name = "qcom/eliza/cdsp.mbn",
+			"qcom/eliza/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
 &sdhc_1 {
 	vmmc-supply = <&vreg_l12b>;
 	vqmmc-supply = <&vreg_l1d>;
diff --git a/arch/arm64/boot/dts/qcom/eliza-evk.dtsi b/arch/arm64/boot/dts/qcom/eliza-evk.dtsi
index 2fc94565830054..6c5cc47a183eac 100644
--- a/arch/arm64/boot/dts/qcom/eliza-evk.dtsi
+++ b/arch/arm64/boot/dts/qcom/eliza-evk.dtsi
@@ -12,6 +12,19 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		key-volume-up {
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+			gpios = <&pm7550_gpios 6 GPIO_ACTIVE_LOW>;
+			debounce-interval = <10>;
+			linux,can-disable;
+			wakeup-source;
+		};
+	};
+
 	pmic-glink {
 		compatible = "qcom,eliza-pmic-glink",
 			     "qcom,sm8550-pmic-glink",
@@ -49,6 +62,121 @@
 			};
 		};
 	};
+
+	vreg_0p9: regulator-0v9 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_0P9";
+
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_1p8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_1P8";
+
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_pcie_m_3p3: regulator-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vreg_3p3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&gpio_expander1 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	connector-0 {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&vreg_pcie_m_3p3>;
+		w-disable1-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+		w-disable2-gpios = <&pm8550vs_g_gpios 4 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&m2_w_disable1>, <&m2_w_disable2>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcieport0_ep>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&uart5_ep>;
+				};
+			};
+		};
+	};
+};
+
+&i2c4 {
+	pinctrl-0 = <&qup_i2c4_data_clk>;
+	pinctrl-names = "default";
+
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	gpio_expander1: gpio@3c {
+		compatible = "ti,tca9538";
+		#gpio-cells = <2>;
+		gpio-controller;
+		reg = <0x3c>;
+	};
+};
+
+&pcie0 {
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1k>;
+	vdda-pll-supply = <&vreg_l3k>;
+
+	status = "okay";
+};
+
+&pcie0_port0 {
+	wake-gpios = <&tlmm 114 GPIO_ACTIVE_LOW>;
+	reset-gpios = <&tlmm 78 GPIO_ACTIVE_LOW>;
+
+	port {
+		pcieport0_ep: endpoint {
+			remote-endpoint = <&m2_e_pcie_ep>;
+		};
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+	status = "okay";
 };
 
 &pm7550ba_eusb2_repeater {
@@ -77,6 +205,91 @@
 	};
 };
 
+&pcie1 {
+	iommu-map = <0x0 &apps_smmu 0x1400 0x0 0x1>,
+		    <0x100 &apps_smmu 0x1401 0x0 0x1>,
+		    <0x208 &apps_smmu 0x1402 0x0 0x1>,
+		    <0x210 &apps_smmu 0x1403 0x0 0x1>,
+		    <0x218 &apps_smmu 0x1404 0x0 0x1>,
+		    <0x300 &apps_smmu 0x1405 0x0 0x1>,
+		    <0x400 &apps_smmu 0x1406 0x0 0x1>,
+		    <0x500 &apps_smmu 0x1407 0x0 0x1>,
+		    <0x501 &apps_smmu 0x1408 0x0 0x1>;
+
+	status = "okay";
+};
+
+&pcie1_phy {
+	vdda-phy-supply = <&vreg_l1k>;
+	vdda-pll-supply = <&vreg_l3k>;
+
+	status = "okay";
+};
+
+&pcie1_port0 {
+	wake-gpios = <&tlmm 53 GPIO_ACTIVE_LOW>;
+	/* Same GPIO is used for both PERST# and resx-gpio */
+
+	tc9563: pcie@0,0 {
+		compatible = "pci1179,0623";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		device_type = "pci";
+		ranges;
+		bus-range = <0x2 0xff>;
+
+		vddc-supply = <&vreg_0p9>;
+		vdd18-supply = <&vreg_1p8>;
+		vdd09-supply = <&vreg_0p9>;
+		vddio1-supply = <&vreg_1p8>;
+		vddio2-supply = <&vreg_1p8>;
+		vddio18-supply = <&vreg_1p8>;
+
+		i2c-parent = <&i2c4 0x77>;
+
+		resx-gpios = <&tlmm 54 GPIO_ACTIVE_LOW>;
+
+		pcie@1,0 {
+			reg = <0x20800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x3 0xff>;
+		};
+
+		pcie@2,0 {
+			reg = <0x21000 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			device_type = "pci";
+			ranges;
+			bus-range = <0x4 0xff>;
+		};
+
+		pcie@3,0 {
+			reg = <0x21800 0x0 0x0 0x0 0x0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			ranges;
+			bus-range = <0x5 0xff>;
+
+			ethernet@0,0 {
+				reg = <0x50000 0x0 0x0 0x0 0x0>;
+			};
+
+			ethernet@0,1 {
+				reg = <0x50100 0x0 0x0 0x0 0x0>;
+			};
+		};
+	};
+};
+
 &uart13 {
 	compatible = "qcom,geni-debug-uart";
 
@@ -110,3 +323,33 @@
 
 	status = "okay";
 };
+
+&uart5 {
+	status = "okay";
+
+	port {
+		uart5_ep: endpoint {
+			remote-endpoint = <&m2_e_uart_ep>;
+		};
+	};
+};
+
+&pm8550vs_g_gpios {
+	m2_w_disable2: m2-w-disable2-state {
+		pins = "gpio4";
+		function = "normal";
+		input-disable;
+		output-enable;
+		bias-disable;
+		power-source = <2>;
+	};
+};
+
+&tlmm {
+	m2_w_disable1: m2-w-disable1-state {
+		pins = "gpio35";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/eliza-mtp.dts b/arch/arm64/boot/dts/qcom/eliza-mtp.dts
index 868ae6c34830bb..ee05192adcfb9f 100644
--- a/arch/arm64/boot/dts/qcom/eliza-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/eliza-mtp.dts
@@ -422,6 +422,14 @@
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/eliza/gen71700_zap.mbn";
+};
+
 &mdss {
 	status = "okay";
 };
@@ -484,6 +492,26 @@
 	status = "okay";
 };
 
+&remoteproc_cdsp {
+	firmware-name = "qcom/eliza/cdsp.mbn",
+			"qcom/eliza/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&sdhc_2 {
+	vmmc-supply = <&vreg_l13b>;
+	vqmmc-supply = <&vreg_l23b>;
+
+	cd-gpios = <&tlmm 58 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&sdc2_default>, <&sd_cd>;
+	pinctrl-1 = <&sdc2_sleep>, <&sd_cd>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+};
+
 &spi8 {
 	status = "okay";
 
@@ -534,6 +562,12 @@
 		bias-pull-down;
 	};
 
+	sd_cd: sd-cd-state {
+		pins = "gpio58";
+		function = "gpio";
+		bias-pull-up;
+	};
+
 	ts_irq: ts-irq-state {
 		pins = "gpio13";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi
index 66ea4aaafd73a3..a8042291bd0758 100644
--- a/arch/arm64/boot/dts/qcom/eliza.dtsi
+++ b/arch/arm64/boot/dts/qcom/eliza.dtsi
@@ -4,9 +4,13 @@
  */
 
 #include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
+#include <dt-bindings/clock/qcom,eliza-cambistmclkcc.h>
+#include <dt-bindings/clock/qcom,eliza-camcc.h>
 #include <dt-bindings/clock/qcom,eliza-dispcc.h>
 #include <dt-bindings/clock/qcom,eliza-gcc.h>
+#include <dt-bindings/clock/qcom,eliza-gpucc.h>
 #include <dt-bindings/clock/qcom,eliza-tcsr.h>
+#include <dt-bindings/clock/qcom,eliza-videocc.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/dma/qcom-gpi.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -20,6 +24,7 @@
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+#include <dt-bindings/thermal/thermal.h>
 
 / {
 	interrupt-parent = <&intc>;
@@ -636,6 +641,30 @@
 		};
 	};
 
+	smp2p-cdsp {
+		compatible = "qcom,smp2p";
+		qcom,smem = <94>, <432>;
+		interrupts-extended = <&ipcc IPCC_CLIENT_CDSP
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+		mboxes = <&ipcc IPCC_CLIENT_CDSP
+				IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <5>;
+
+		smp2p_cdsp_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		smp2p_cdsp_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
 	soc: soc@0 {
 		compatible = "simple-bus";
 
@@ -650,12 +679,12 @@
 
 			clocks = <&bi_tcxo_div2>,
 				 <&sleep_clk>,
-				 <0>,
-				 <0>,
+				 <&pcie0_phy>,
+				 <&pcie1_phy>,
 				 <&ufs_mem_phy 0>,
 				 <&ufs_mem_phy 1>,
 				 <&ufs_mem_phy 2>,
-				 <0>;
+				 <&usb_dp_qmpphy QMP_USB43DP_USB3_PIPE_CLK>;
 
 			power-domains = <&rpmhpd RPMHPD_CX>;
 
@@ -1787,6 +1816,23 @@
 			#interconnect-cells = <2>;
 		};
 
+		cambistmclkcc: clock-controller@1760000 {
+			compatible = "qcom,eliza-cambistmclkcc";
+			reg = <0x0 0x01760000 0x0 0x6000>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&sleep_clk>,
+				 <&gcc GCC_CAM_BIST_MCLK_AHB_CLK>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
 		mmss_noc: interconnect@1780000 {
 			compatible = "qcom,eliza-mmss-noc";
 			reg = <0x0 0x01780000 0x0 0x7d800>;
@@ -1794,6 +1840,369 @@
 			#interconnect-cells = <2>;
 		};
 
+		pcie0: pcie@1c00000 {
+			device_type = "pci";
+			compatible = "qcom,eliza-pcie", "qcom,pcie-sm8550";
+			reg = <0x0 0x01c00000 0x0 0x3000>,
+			      <0x0 0x40000000 0x0 0xf1d>,
+			      <0x0 0x40000f20 0x0 0xa8>,
+			      <0x0 0x40001000 0x0 0x1000>,
+			      <0x0 0x40100000 0x0 0x100000>,
+			      <0x0 0x01c03000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x3d00000>;
+
+			interrupts = <GIC_SPI 376 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 478 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 479 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 535 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 536 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 537 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 540 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 653 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 370 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_DDRSS_PCIE_SF_QTB_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				 <&gcc GCC_CNOC_PCIE_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "ddrss_sf_tbu",
+				      "noc_aggr",
+				      "cnoc_sf_axi";
+
+			resets = <&gcc GCC_PCIE_0_BCR>,
+				 <&gcc GCC_PCIE_0_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			interconnects = <&pcie_noc MASTER_PCIE_0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc_main SLAVE_PCIE_0 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			power-domains = <&gcc GCC_PCIE_0_GDSC>;
+
+			operating-points-v2 = <&pcie0_opp_table>;
+
+			iommu-map = <0x0 &apps_smmu 0x1480 0x0 0x1>,
+				    <0x100 &apps_smmu 0x1481 0x0 0x1>;
+
+			interrupt-map = <0 0 0 1 &intc 0 0 0 564 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 0 565 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 0 566 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 0 567 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+			#interrupt-cells = <1>;
+
+			linux,pci-domain = <0>;
+			bus-range = <0x0 0xff>;
+
+			dma-coherent;
+
+			pinctrl-0 = <&pcie0_default_state>;
+			pinctrl-names = "default";
+
+			status = "disabled";
+
+			pcie0_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* 2.5 GT/s x1 */
+				opp-2500000-1 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+					opp-level = <1>;
+				};
+
+				/* 5 GT/s x1 */
+				opp-5000000-2 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <2>;
+				};
+
+				/* 8 GT/s x1 */
+				opp-8000000-3 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+					opp-level = <3>;
+				};
+
+			};
+
+			pcie0_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+				phys = <&pcie0_phy>;
+			};
+		};
+
+		pcie0_phy: phy@1c06000 {
+			compatible = "qcom,eliza-qmp-gen3x1-pcie-phy";
+			reg = <0x0 0x01c06000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_0_CLKREF_EN>,
+				 <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_0_PIPE_CLK>,
+				 <&gcc GCC_PCIE_0_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			resets = <&gcc GCC_PCIE_0_PHY_BCR>,
+				 <&gcc GCC_PCIE_0_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			power-domains = <&gcc GCC_PCIE_0_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie0_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie1: pcie@1c08000 {
+			device_type = "pci";
+			compatible = "qcom,eliza-pcie", "qcom,pcie-sm8550";
+			reg = <0x0 0x01c08000 0x0 0x3000>,
+			      <0x0 0x44000000 0x0 0xf1d>,
+			      <0x0 0x44000f20 0x0 0xa8>,
+			      <0x0 0x44001000 0x0 0x1000>,
+			      <0x0 0x44100000 0x0 0x100000>,
+			      <0x0 0x01c0b000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x44200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x44300000 0x0 0x44300000 0x0 0x3d00000>,
+				 <0x43000000 0x4 0x00000000 0x4 0x00000000 0x3 0x00000000>;
+
+			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
+				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_1_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_DDRSS_PCIE_SF_QTB_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				 <&gcc GCC_CNOC_PCIE_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "ddrss_sf_tbu",
+				      "noc_aggr",
+				      "cnoc_sf_axi";
+
+			resets = <&gcc GCC_PCIE_1_BCR>,
+				 <&gcc GCC_PCIE_1_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			interconnects = <&pcie_noc MASTER_PCIE_1 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc_main SLAVE_PCIE_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			power-domains = <&gcc GCC_PCIE_1_GDSC>;
+
+			operating-points-v2 = <&pcie1_opp_table>;
+
+			iommu-map = <0x0 &apps_smmu 0x1400 0x0 0x1>,
+				    <0x100 &apps_smmu 0x1401 0x0 0x1>;
+
+			interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 0 150 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 0 151 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 0 152 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+			#interrupt-cells = <1>;
+
+			linux,pci-domain = <1>;
+			bus-range = <0x0 0xff>;
+
+			dma-coherent;
+
+			pinctrl-0 = <&pcie1_default_state>;
+			pinctrl-names = "default";
+
+			status = "disabled";
+
+			pcie1_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* 2.5 GT/s x1 */
+				opp-2500000-1 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+					opp-level = <1>;
+				};
+
+				/* 2.5 GT/s x2 */
+				opp-5000000-1 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <1>;
+				};
+
+				/* 5 GT/s x1 */
+				opp-5000000-2 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <2>;
+				};
+
+				/* 5 GT/s x2 */
+				opp-10000000-2 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <2>;
+				};
+
+				/* 8 GT/s x1 */
+				opp-8000000-3 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+					opp-level = <3>;
+				};
+
+				/* 8 GT/s x2 */
+				opp-16000000-3 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <3>;
+				};
+
+			};
+
+			pcie1_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+				phys = <&pcie1_phy>;
+			};
+		};
+
+		pcie1_phy: phy@1c0e000 {
+			compatible = "qcom,eliza-qmp-gen3x2-pcie-phy";
+			reg = <0x0 0x01c0e000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
+				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_1_CLKREF_EN>,
+				 <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_1_PIPE_CLK>,
+				 <&gcc GCC_PCIE_1_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			resets = <&gcc GCC_PCIE_1_PHY_BCR>,
+				 <&gcc GCC_PCIE_1_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			power-domains = <&gcc GCC_PCIE_1_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie1_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
 		ufs_mem_phy: phy@1d80000 {
 			compatible = "qcom,eliza-qmp-ufs-phy",
 				     "qcom,sm8650-qmp-ufs-phy";
@@ -2893,6 +3302,98 @@
 			};
 		};
 
+		adreno_smmu: iommu@3da0000 {
+			compatible = "qcom,eliza-smmu-500", "qcom,adreno-smmu",
+				     "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x03da0000 0x0 0x40000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			interrupts = <GIC_SPI 673 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 677 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 678 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 679 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 680 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 681 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 682 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 683 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 684 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 685 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 686 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 687 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 476 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 574 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 575 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 576 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 577 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 659 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 661 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 664 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 665 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 666 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 668 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 669 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 699 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gpucc GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK>;
+			clock-names = "hlos";
+			power-domains = <&gpucc GPU_CC_CX_GDSC>;
+			dma-coherent;
+		};
+
+		gpucc: clock-controller@3d90000 {
+			compatible = "qcom,eliza-gpucc";
+			reg = <0x0 0x03d90000 0x0 0xa000>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&gcc GCC_GPU_GPLL0_CPH_CLK_SRC>,
+				 <&gcc GCC_GPU_GPLL0_DIV_CPH_CLK_SRC>;
+
+			power-domains = <&rpmhpd RPMHPD_MX>,
+					<&rpmhpd RPMHPD_CX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		videocc: clock-controller@aaf0000 {
+			compatible = "qcom,eliza-videocc";
+			reg = <0x0 0xaaf0000 0x0 0x10000>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>,
+				 <&gcc GCC_VIDEO_AHB_CLK>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		camcc: clock-controller@ade0000 {
+			compatible = "qcom,eliza-camcc";
+			reg = <0x0 0x0ade0000 0x0 0x20000>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&sleep_clk>,
+				 <&gcc GCC_CAMERA_AHB_CLK>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		dispcc: clock-controller@af00000 {
 			compatible = "qcom,eliza-dispcc";
 			reg = <0x0 0x0af00000 0x0 0x20000>;
@@ -3079,6 +3580,52 @@
 			gpio-ranges = <&tlmm 0 0 184>;
 			wakeup-parent = <&pdc>;
 
+			pcie0_default_state: pcie0-default-state {
+				perst-pins {
+					pins = "gpio78";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				clkreq-pins {
+					pins = "gpio80";
+					function = "pcie0_clk_req_n";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				wake-pins {
+					pins = "gpio114";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
+			pcie1_default_state: pcie1-default-state {
+				perst-pins {
+					pins = "gpio54";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				clkreq-pins {
+					pins = "gpio52";
+					function = "pcie1_clk_req_n";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				wake-pins {
+					pins = "gpio53";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
 			qup_i2c0_data_clk: qup-i2c0-data-clk-state {
 				pins = "gpio28", "gpio29";
 				function = "qup1_se0";
@@ -3975,6 +4522,209 @@
 			qcom,bcm-voters = <&apps_bcm_voter>;
 			#interconnect-cells = <2>;
 		};
+
+		remoteproc_cdsp: remoteproc@32300000 {
+			compatible = "qcom,eliza-cdsp-pas";
+			reg = <0x0 0x32300000 0x0 0x10000>;
+
+			interrupts-extended = <&intc GIC_SPI 578 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 7 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "shutdown-ack";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MX>,
+					<&rpmhpd RPMHPD_NSP>;
+			power-domain-names = "cx",
+					     "mx",
+					     "nsp";
+
+			interconnects = <&nsp_noc MASTER_CDSP_PROC QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+
+			memory-region = <&cdsp_mem>, <&q6_cdsp_dtb_mem>, <&global_sync_mem>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			qcom,smem-states = <&smp2p_cdsp_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			glink-edge {
+				interrupts-extended = <&ipcc IPCC_CLIENT_CDSP
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+				mboxes = <&ipcc IPCC_CLIENT_CDSP
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+				label = "cdsp";
+				qcom,remote-pid = <5>;
+			};
+		};
+
+		gpu: gpu@3d00000 {
+			compatible = "qcom,adreno-43020100", "qcom,adreno";
+			reg = <0x0 0x03d00000 0x0 0x40000>,
+			      <0x0 0x03d9e000 0x0 0x1000>,
+			      <0x0 0x03d61000 0x0 0x800>;
+			reg-names = "kgsl_3d0_reg_memory",
+				    "cx_mem",
+				    "cx_dbgc";
+
+			interrupts = <GIC_SPI 300 IRQ_TYPE_LEVEL_HIGH>;
+
+			iommus = <&adreno_smmu 0x0 0x0>;
+
+			operating-points-v2 = <&gpu_opp_table>;
+			qcom,gmu = <&gmu>;
+			#cooling-cells = <2>;
+			interconnects = <&gem_noc MASTER_GFX3D QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "gfx-mem";
+
+			status = "disabled";
+
+			gpu_zap_shader: zap-shader {
+				memory-region = <&gpu_micro_code_mem>;
+			};
+
+			gpu_opp_table: opp-table {
+				compatible = "operating-points-v2-adreno",
+					     "operating-points-v2";
+
+				opp-259000000 {
+					opp-hz = /bits/ 64 <259000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					opp-peak-kBps = <781250>;
+					qcom,opp-acd-level = <0xc82f5ffd>;
+				};
+
+				opp-345000000 {
+					opp-hz = /bits/ 64 <345000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					opp-peak-kBps = <2136718>;
+					qcom,opp-acd-level = <0xc82f5ffd>;
+				};
+
+				opp-515000000 {
+					opp-hz = /bits/ 64 <515000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					opp-peak-kBps = <5285156>;
+					qcom,opp-acd-level = <0xc02d5ffd>;
+				};
+
+				opp-645000000 {
+					opp-hz = /bits/ 64 <645000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0x882f5ffd>;
+				};
+
+				opp-724000000 {
+					opp-hz = /bits/ 64 <724000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
+					opp-peak-kBps = <6671875>;
+					qcom,opp-acd-level = <0xa82d5ffd>;
+				};
+
+				opp-796000000 {
+					opp-hz = /bits/ 64 <796000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					opp-peak-kBps = <10687500>;
+					qcom,opp-acd-level = <0x882d5ffd>;
+				};
+
+				opp-900000000 {
+					opp-hz = /bits/ 64 <900000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+					opp-peak-kBps = <10687500>;
+					qcom,opp-acd-level = <0x882d5ffd>;
+				};
+
+				opp-975000000 {
+					opp-hz = /bits/ 64 <975000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+					opp-peak-kBps = <12449218>;
+					qcom,opp-acd-level = <0x882c5ffd>;
+				};
+
+				opp-1075000000 {
+					opp-hz = /bits/ 64 <1075000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					opp-peak-kBps = <16500000>;
+					qcom,opp-acd-level = <0xa82a5ffd>;
+				};
+
+				opp-1150000000 {
+					opp-hz = /bits/ 64 <1150000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L2>;
+					opp-peak-kBps = <16500000>;
+					qcom,opp-acd-level = <0xa8295ffd>;
+				};
+			};
+		};
+
+		gmu: gmu@3d6a000 {
+			compatible = "qcom,adreno-gmu-722.0", "qcom,adreno-gmu";
+			reg = <0x0 0x03d6a000 0x0 0x35000>,
+			      <0x0 0x03d50000 0x0 0x10000>,
+			      <0x0 0x0b290000 0x0 0x10000>;
+			reg-names = "gmu", "rscc", "gmu_pdc";
+
+			interrupts = <GIC_SPI 304 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 305 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hfi", "gmu";
+
+			clocks = <&gpucc GPU_CC_AHB_CLK>,
+				 <&gpucc GPU_CC_CX_GMU_CLK>,
+				 <&gpucc GPU_CC_CXO_CLK>,
+				 <&gcc GCC_DDRSS_GPU_AXI_CLK>,
+				 <&gcc GCC_GPU_GEMNOC_GFX_CLK>,
+				 <&gpucc GPU_CC_HUB_CX_INT_CLK>,
+				 <&gpucc GPU_CC_DEMET_CLK>;
+			clock-names = "ahb",
+				      "gmu",
+				      "cxo",
+				      "axi",
+				      "memnoc",
+				      "hub",
+				      "demet";
+
+			power-domains = <&gpucc GPU_CC_CX_GDSC>,
+					<&gpucc GPU_CC_GX_GDSC>;
+			power-domain-names = "cx",
+					     "gx";
+
+			iommus = <&adreno_smmu 0x5 0x0>;
+			qcom,qmp = <&aoss_qmp>;
+			operating-points-v2 = <&gmu_opp_table>;
+
+			gmu_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-220000000 {
+					opp-hz = /bits/ 64 <220000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+				};
+
+				opp-550000000 {
+					opp-hz = /bits/ 64 <550000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+				};
+			};
+		};
 	};
 
 	thermal-zones {
@@ -4284,7 +5034,7 @@
 			thermal-sensors = <&tsens1 8>;
 
 			trips {
-				gpu-alert {
+				gpuss0_alert0: gpu-alert {
 					temperature = <95000>;
 					hysteresis = <1000>;
 					type = "passive";
@@ -4302,6 +5052,13 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss0_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss1-thermal {
@@ -4310,7 +5067,7 @@
 			thermal-sensors = <&tsens1 9>;
 
 			trips {
-				gpu-alert {
+				gpuss1_alert0: gpu-alert {
 					temperature = <95000>;
 					hysteresis = <1000>;
 					type = "passive";
@@ -4328,6 +5085,13 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss1_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		modem0-thermal {
diff --git a/arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a14-ux3407na.dts b/arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a14-ux3407na.dts
new file mode 100644
index 00000000000000..e96e65a0c5b490
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a14-ux3407na.dts
@@ -0,0 +1,1140 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+/dts-v1/;
+
+#include "glymur.dtsi"
+
+#include "pmcx0102.dtsi"
+#include "pmh0101.dtsi"
+#include "pmh0110-glymur.dtsi"
+#include "pmh0104-glymur.dtsi"
+#include "pmk8850.dtsi"
+#include "smb2370.dtsi"
+
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+/ {
+	compatible = "asus,zenbook-a14-ux3407na", "qcom,glymur";
+	model = "ASUS Zenbook A14 (UX3407NA)";
+
+	aliases {
+		i2c0 = &i2c0;
+		i2c5 = &i2c5;
+		i2c8 = &i2c8;
+		i2c10 = &i2c10;
+		i2c19 = &i2c19;
+		serial0 = &uart14;
+	};
+
+	clocks {
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			#clock-cells = <0>;
+		};
+
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&hall_int_n_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			label = "lid";
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+	};
+
+	hdmi-bridge {
+		compatible = "parade,ps185hdm";
+
+		pinctrl-0 = <&usb2_dp_hot_plug_detect>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				hdmi_bridge_in: endpoint {
+					remote-endpoint = <&usb_2_qmpphy_out_dp>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				hdmi_bridge_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&hdmi_bridge_out>;
+			};
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		pinctrl-0 = <&cam_indicator_en>,
+			    <&keyboard_cam_led>,
+			    <&keyboard_mic_led>;
+		pinctrl-names = "default";
+
+		keyboard-camera-led {
+			function = LED_FUNCTION_INDICATOR;
+			color = <LED_COLOR_ID_ORANGE>;
+			gpios = <&tlmm 120 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			panic-indicator;
+		};
+
+		keyboard-mic-led {
+			function = LED_FUNCTION_MICMUTE;
+			color = <LED_COLOR_ID_ORANGE>;
+			gpios = <&tlmm 98 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+			panic-indicator;
+		};
+
+		privacy-led {
+			function = LED_FUNCTION_INDICATOR;
+			color = <LED_COLOR_ID_WHITE>;
+			gpios = <&tlmm 111 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			panic-indicator;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,glymur-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Left side display-adjacent port */
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_0_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		/* Left side user-adjacent port */
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in1: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in1: endpoint {
+						remote-endpoint = <&usb_1_qmpphy_out>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_edp: regulator-edp {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_pwr_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_misc_3p3: regulator-misc-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_MISC_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&misc_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0101_gpios 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 94 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	reserved-memory {
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x10000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	sound {
+		compatible = "qcom,glymur-sndcard";
+		model = "GLYMUR-ASUS-Zenbook-A14-UX3407NA";
+
+		audio-routing = "SpkrLeft IN", "WSA WSA_SPK1 OUT",
+				"SpkrRight IN", "WSA WSA_SPK2 OUT",
+				"VA DMIC0", "vdd-micb",
+				"VA DMIC1", "vdd-micb";
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_speaker>, <&right_speaker>,
+					    <&swr0 0>, <&lpass_wsamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	wlan-connector {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&vreg_wcn_3p3>;
+		vpcie1v8-supply = <&vreg_l15b0>;
+
+		w-disable1-gpios = <&tlmm 117 GPIO_ACTIVE_LOW>;
+		w-disable2-gpios = <&tlmm 116 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&wcn_wlan_bt_en>;
+		pinctrl-names = "default";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcie4_port0_ep>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&uart14_ep>;
+				};
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_l1b0: ldo1 {
+			regulator-name = "vreg_l1b0";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b0: ldo7 {
+			regulator-name = "vreg_l7b0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b0: ldo8 {
+			regulator-name = "vreg_l8b0";
+			regulator-min-microvolt = <3304000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b0: ldo15 {
+			regulator-name = "vreg_l15b0";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			/* Provides reference voltage for many undescribed components */
+			regulator-always-on;
+		};
+
+		vreg_l18b0: ldo18 {
+			regulator-name = "vreg_l18b0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+
+			/*
+			 * This is also reference voltage for &pcie3b wake and
+			 * reset signals.
+			 */
+			regulator-always-on;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmcx0102-rpmh-regulators";
+		qcom,pmic-id = "C_E1";
+
+		vreg_l1c1: ldo1 {
+			regulator-name = "vreg_l1c1";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c1: ldo2 {
+			regulator-name = "vreg_l2c1";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c1: ldo3 {
+			regulator-name = "vreg_l3c1";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c1: ldo4 {
+			regulator-name = "vreg_l4c1";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_l2f0: ldo2 {
+			regulator-name = "vreg_l2f0";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f0: ldo3 {
+			regulator-name = "vreg_l3f0";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E1";
+
+		vreg_l1f1: ldo1 {
+			regulator-name = "vreg_l1f1";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f1: ldo2 {
+			regulator-name = "vreg_l2f1";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f1: ldo4 {
+			regulator-name = "vreg_l4f1";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "H_E0";
+
+		vreg_l1h0: ldo1 {
+			regulator-name = "vreg_l1h0";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2h0: ldo2 {
+			regulator-name = "vreg_l2h0";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4h0: ldo4 {
+			regulator-name = "vreg_l4h0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gmu {
+	status = "okay";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@15 {
+		compatible = "hid-over-i2c";
+		reg = <0x15>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b0>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	ptn3222_1: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+
+		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b0>;
+		vdd1v8-supply = <&vreg_l15b0>;
+
+		#phy-cells = <0>;
+	};
+
+	ptn3222_0: redriver@4f {
+		compatible = "nxp,ptn3222";
+		reg = <0x4f>;
+
+		reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b0>;
+		vdd1v8-supply = <&vreg_l15b0>;
+
+		#phy-cells = <0>;
+	};
+};
+
+&i2c8 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@15 {
+		compatible = "hid-over-i2c";
+		reg = <0x15>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&ts_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c10 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	hid@17 {
+		compatible = "hid-over-i2c";
+		reg = <0x17>;
+		interrupts-extended = <&tlmm 43 IRQ_TYPE_LEVEL_LOW>;
+		hid-descr-addr = <0x1>;
+		wakeup-source;
+
+		pinctrl-0 = <&hid_app_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c19 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	keyboard@15 {
+		compatible = "hid-over-i2c";
+		reg = <0x15>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		wakeup-source;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&lpass_vamacro {
+	vdd-micb-supply = <&vreg_l1b0>;
+
+	pinctrl-0 = <&dmic01_default>;
+	pinctrl-names = "default";
+
+	qcom,dmic-sample-rate = <2400000>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp2 {
+	status = "okay";
+};
+
+&mdss_dp2_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	pinctrl-0 = <&edp0_hpd_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "samsung,atna40ct06", "samsung,atna33xc20";
+			enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+			power-supply = <&vreg_edp>;
+
+			pinctrl-0 = <&edp_bl_en>;
+			pinctrl-names = "default";
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l2f1>;
+	vdda-pll-supply = <&vreg_l4f1>;
+
+	status = "okay";
+};
+
+&pcie3_phy {
+	vdda-phy-supply = <&vreg_l3c1>;
+	vdda-pll-supply = <&vreg_l2c1>;
+
+	vdda-refgen0p9-supply = <&vreg_l1c1>;
+	vdda-refgen1p2-supply = <&vreg_l4f1>;
+
+	status = "okay";
+};
+
+&pcie3b {
+	vddpe-3v3-supply = <&vreg_nvme>;
+
+	pinctrl-0 = <&pcie3b_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie3b_port0 {
+	reset-gpios = <&tlmm 155 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 157 GPIO_ACTIVE_LOW>;
+};
+
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l1c1>;
+	vdda-pll-supply = <&vreg_l4f1>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
+
+&pcie4_port0_ep {
+	remote-endpoint = <&m2_e_pcie_ep>;
+};
+
+&pmh0101_gpios {
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmh0110_f_e0_gpios {
+	misc_3p3_reg_en: misc-3p3-reg-en-state {
+		pins = "gpio6";
+		function = "normal";
+		bias-disable;
+		input-disable;
+		output-enable;
+		drive-push-pull;
+		power-source = <1>; /* 1.8 V */
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+	};
+};
+
+&pmk8850_rtc {
+	qcom,no-alarm;
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/glymur/ASUSTeK/UX3407NA/qcadsp8480.mbn",
+			"qcom/glymur/ASUSTeK/UX3407NA/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/glymur/ASUSTeK/UX3407NA/qccdsp8480.mbn",
+			"qcom/glymur/ASUSTeK/UX3407NA/cdsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_soccp {
+	status = "okay";
+};
+
+&smb2370_j_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b0>;
+	vdd3-supply = <&vreg_l7b0>;
+};
+
+&smb2370_k_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b0>;
+	vdd3-supply = <&vreg_l7b0>;
+};
+
+&swr0 {
+	status = "okay";
+
+	/* WSA8845, left */
+	left_speaker: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrLeft";
+		vdd-1p8-supply = <&vreg_l15b0>;
+		vdd-io-supply = <&vreg_l18b0>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, right */
+	right_speaker: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrRight";
+		vdd-1p8-supply = <&vreg_l15b0>;
+		vdd-io-supply = <&vreg_l18b0>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l2f1>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l2f1>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l2f1>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l2h0>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l2h0>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l2f1>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l2f1>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l2f1>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l2h0>;
+	vdda-qrefrx5-0p9-supply = <&vreg_l3f0>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3f0>;
+	vdda-qreftx0-1p2-supply = <&vreg_l4h0>;
+	vdda-qreftx1-0p9-supply = <&vreg_l1f1>;
+	vdda-refgen3-0p9-supply = <&vreg_l2f0>;
+	vdda-refgen3-1p2-supply = <&vreg_l4h0>;
+	vdda-refgen4-0p9-supply = <&vreg_l1f1>;
+	vdda-refgen4-1p2-supply = <&vreg_l4f1>;
+};
+
+&tlmm {
+	gpio-reserved-ranges = <4 4>, /* EC Secure */
+			       <10 2>, /* OOB UART */
+			       <44 4>, /* TPM */
+			       <65 1>, /* EC reset */
+			       <90 2>; /* TPM */
+
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	hid_app_default: hid-app-default-state {
+		pins = "gpio43";
+		function = "gpio";
+		bias-disable;
+	};
+
+	ts_default: ts-default-state {
+		reset-n-pins {
+			pins = "gpio48";
+			function = "gpio";
+			bias-disable;
+		};
+
+		int-pins {
+			pins = "gpio51";
+			function = "gpio";
+			bias-disable;
+		};
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
+	edp_pwr_en: edp-pwr-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio94";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	keyboard_mic_led: keyboard-mic-led-state {
+		pins = "gpio98";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	cam_indicator_en: cam-indicator-en-state {
+		pins = "gpio111";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
+		pins = "gpio116", "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	edp0_hpd_default: edp0-hpd-default-state {
+		pins = "gpio119";
+		function = "edp0_hot";
+		bias-disable;
+	};
+
+	keyboard_cam_led: keyboard-cam-led-state {
+		pins = "gpio120";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	usb2_dp_hot_plug_detect: usb2-dp-hot-plug-detect-state {
+		pins = "gpio126";
+		function = "usb2_dp";
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie3b_default: pcie3b-default-state {
+		clkreq-n-pins {
+			pins = "gpio156";
+			function = "pcie3b_clk";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio155";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio157";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	port {
+		uart14_ep: endpoint {
+			remote-endpoint = <&m2_e_uart_ep>;
+		};
+	};
+};
+
+&uart21 {
+	status = "disabled";
+};
+
+&usb_0 {
+	status = "okay";
+};
+
+&usb_0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_0_hsphy {
+	vdd-supply = <&vreg_l3f0>;
+	vdda12-supply = <&vreg_l4h0>;
+
+	phys = <&smb2370_j_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy {
+	vdda-phy-supply = <&vreg_l3f0>;
+	vdda-pll-supply = <&vreg_l4h0>;
+	refgen-supply = <&vreg_l2f0>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in1>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l3f0>;
+	vdda12-supply = <&vreg_l4h0>;
+
+	phys = <&smb2370_k_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l1h0>;
+	vdda-pll-supply = <&vreg_l4h0>;
+	refgen-supply = <&vreg_l2f0>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in1>;
+};
+
+&usb_2_qmpphy {
+	vdda-phy-supply = <&vreg_l4c1>;
+	vdda-pll-supply = <&vreg_l4f1>;
+	refgen-supply = <&vreg_l4f1>;
+
+	/delete-property/ mode-switch;
+	/delete-property/ orientation-switch;
+
+	status = "okay";
+
+	ports {
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/delete-node/ endpoint;
+
+			usb_2_qmpphy_out_dp: endpoint@0 {
+				reg = <0>;
+
+				data-lanes = <3 2 1 0>;
+				remote-endpoint = <&hdmi_bridge_in>;
+			};
+		};
+	};
+};
+
+&usb_mp {
+	status = "okay";
+};
+
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2h0>;
+	vdda12-supply = <&vreg_l4h0>;
+
+	phys = <&ptn3222_0>;
+
+	status = "okay";
+};
+
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2h0>;
+	vdda12-supply = <&vreg_l4h0>;
+
+	phys = <&ptn3222_1>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l2h0>;
+	vdda-pll-supply = <&vreg_l4h0>;
+	refgen-supply = <&vreg_l4f1>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l2h0>;
+	vdda-pll-supply = <&vreg_l4h0>;
+	refgen-supply = <&vreg_l4f1>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a16-ux3607oa.dts b/arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a16-ux3607oa.dts
index 939e1a927a7930..95da35125d7546 100644
--- a/arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a16-ux3607oa.dts
+++ b/arch/arm64/boot/dts/qcom/glymur-asus-zenbook-a16-ux3607oa.dts
@@ -737,7 +737,7 @@
 	pinctrl-0 = <&dmic01_default>;
 	pinctrl-names = "default";
 
-	qcom,dmic-sample-rate = <4800000>;
+	qcom,dmic-sample-rate = <2400000>;
 };
 
 &mdss {
@@ -861,7 +861,6 @@
 };
 
 &pmk8850_rtc {
-	qcom,uefi-rtc-info;
 	qcom,no-alarm;
 };
 
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts
index 44766a73b0d1a2..b8e3fedcdea812 100644
--- a/arch/arm64/boot/dts/qcom/glymur-crd.dts
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts
@@ -29,6 +29,14 @@
 	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
 };
 
+&pcie3_phy {
+	vdda-phy-supply = <&vreg_l3c_e1_0p89>;
+	vdda-pll-supply = <&vreg_l2c_e1_1p14>;
+
+	vdda-refgen0p9-supply = <&vreg_l1c_e1_0p82>;
+	vdda-refgen1p2-supply = <&vreg_l4f_e1_1p08>;
+};
+
 &tcsr {
 	vdda-qrefrpt0-0p9-supply = <&vreg_l2f_e1_0p83>;
 	vdda-qrefrpt1-0p9-supply = <&vreg_l2f_e1_0p83>;
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
index e27c697e317027..58158b51fe295f 100644
--- a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
@@ -373,7 +373,7 @@
 
 		vreg_l2b_e0_2p9: ldo2 {
 			regulator-name = "vreg_l2b_e0_2p9";
-			regulator-min-microvolt = <2904000>;
+			regulator-min-microvolt = <1804000>;
 			regulator-max-microvolt = <2904000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 		};
@@ -392,10 +392,10 @@
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 		};
 
-		vreg_l9b_e0_2p7: ldo9 {
-			regulator-name = "vreg_l9b_e0_2p7";
-			regulator-min-microvolt = <2704000>;
-			regulator-max-microvolt = <2704000>;
+		vreg_l9b_e0_2p9: ldo9 {
+			regulator-name = "vreg_l9b_e0_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 		};
 
@@ -899,6 +899,22 @@
 	vdd3-supply = <&vreg_l7b_e0_2p79>;
 };
 
+&sdhc_2 {
+	vmmc-supply = <&vreg_l9b_e0_2p9>;
+	vqmmc-supply = <&vreg_l2b_e0_2p9>;
+
+	pinctrl-0 = <&sdc2_default_state &sdc2_card_det_n>;
+	pinctrl-1 = <&sdc2_sleep_state &sdc2_card_det_n>;
+	pinctrl-names = "default", "sleep";
+
+	cd-gpios = <&tlmm 221 GPIO_ACTIVE_LOW>;
+
+	no-mmc;
+	no-sdio;
+
+	status = "okay";
+};
+
 &swr0 {
 	status = "okay";
 
@@ -963,7 +979,8 @@
 &tlmm {
 	gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */
 			       <10 2>, /* OOB UART */
-			       <44 4>; /* Security SPI (TPM) */
+			       <44 4>, /* Security SPI (TPM) */
+			       <65 1>; /* EC reset */
 
 	edp_bl_en: edp-bl-en-state {
 		pins = "gpio18";
@@ -1089,6 +1106,13 @@
 		};
 	};
 
+	sdc2_card_det_n: sd-card-det-n-state {
+		pins = "gpio221";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	tpad_default: tpad-default-state {
 		pins = "gpio3";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/glymur-hp-elitebook-x-g2q.dts b/arch/arm64/boot/dts/qcom/glymur-hp-elitebook-x-g2q.dts
new file mode 100644
index 00000000000000..096d40e49210a7
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/glymur-hp-elitebook-x-g2q.dts
@@ -0,0 +1,940 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+/dts-v1/;
+
+#include "glymur.dtsi"
+
+#include "pmcx0102.dtsi"
+#include "pmh0101.dtsi"
+#include "pmh0110-glymur.dtsi"
+#include "pmh0104-glymur.dtsi"
+#include "pmk8850.dtsi"
+#include "smb2370.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+
+/ {
+	model = "HP EliteBook X G2q 14 AI";
+	compatible = "hp,elitebook-x-g2q", "qcom,glymur";
+
+	aliases {
+		i2c0 = &i2c0;
+		i2c8 = &i2c8;
+
+		serial0 = &uart21;
+		serial1 = &uart14;
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			#clock-cells = <0>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&lid_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			label = "lid";
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,glymur-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Left side port */
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_0_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		/* Right side port */
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in1: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in1: endpoint {
+						remote-endpoint = <&usb_1_qmpphy_out>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0101_gpios 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 94 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	sound {
+		compatible = "qcom,glymur-sndcard";
+		model = "GLYMUR-HP-EliteBook-X-G2q";
+
+		audio-routing = "WooferLeft IN", "WSA WSA_SPK1 OUT",
+				"TweeterLeft IN", "WSA WSA_SPK2 OUT",
+				"WooferRight IN", "WSA2 WSA_SPK1 OUT",
+				"TweeterRight IN", "WSA2 WSA_SPK2 OUT",
+				"VA DMIC0", "vdd-micb",
+				"VA DMIC1", "vdd-micb";
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_woofer>, <&left_tweeter>,
+					    <&swr0 0>, <&lpass_wsamacro 0>,
+					    <&right_woofer>, <&right_tweeter>,
+					    <&swr3 0>, <&lpass_wsa2macro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_e0_1p8>;
+		vddaon-supply = <&vreg_l15b_e0_1p8>;
+		vdddig-supply = <&vreg_l15b_e0_1p8>;
+		vddrfa1p2-supply = <&vreg_l15b_e0_1p8>;
+		vddrfa1p8-supply = <&vreg_l15b_e0_1p8>;
+
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_wlan_bt_en>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_l1b_e0_1p8: ldo1 {
+			regulator-name = "vreg_l1b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_e0_3p1: ldo7 {
+			regulator-name = "vreg_l7b_e0_3p1";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_e0_1p8: ldo15 {
+			regulator-name = "vreg_l15b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_e0_1p2: ldo18 {
+			regulator-name = "vreg_l18b_e0_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmcx0102-rpmh-regulators";
+		qcom,pmic-id = "C_E1";
+
+		vreg_l1c_e1_0p91: ldo1 {
+			regulator-name = "vreg_l1c_e1_0p91";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_l2f_e0_0p94: ldo2 {
+			regulator-name = "vreg_l2f_e0_0p94";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_e0_0p91: ldo3 {
+			regulator-name = "vreg_l3f_e0_0p91";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E1";
+
+		vreg_l1f_e1_0p88: ldo1 {
+			regulator-name = "vreg_l1f_e1_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_e1_0p88: ldo2 {
+			regulator-name = "vreg_l2f_e1_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f_e1_1p2: ldo4 {
+			regulator-name = "vreg_l4f_e1_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "H_E0";
+
+		vreg_l1h_e0_0p94: ldo1 {
+			regulator-name = "vreg_l1h_e0_0p94";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2h_e0_0p88: ldo2 {
+			regulator-name = "vreg_l2h_e0_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4h_e0_1p2: ldo4 {
+			regulator-name = "vreg_l4h_e0_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gmu {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+
+		hid-descr-addr = <0x20>;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+
+	keyboard@3a {
+		compatible = "hid-over-i2c";
+		reg = <0x3a>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c8 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@10 {
+		compatible = "hid-over-i2c";
+		reg = <0x10>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>;
+
+		post-power-on-delay-ms = <200>;
+
+		pinctrl-0 = <&ts0_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&lpass_vamacro {
+	vdd-micb-supply = <&vreg_l1b_e0_1p8>;
+
+	pinctrl-0 = <&dmic01_default>;
+	pinctrl-names = "default";
+
+	qcom,dmic-sample-rate = <4800000>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "samsung,atna33xc20";
+			enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+			power-supply = <&vreg_edp_3p3>;
+
+			pinctrl-0 = <&edp_bl_en>;
+			pinctrl-names = "default";
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l2f_e1_0p88>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l1c_e1_0p91>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pcie5 {
+	vddpe-3v3-supply = <&vreg_nvme>;
+
+	pinctrl-0 = <&pcie5_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie5_phy {
+	vdda-phy-supply = <&vreg_l2f_e0_0p94>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+
+	status = "okay";
+};
+
+&pcie5_port0 {
+	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+};
+
+&pmh0101_gpios {
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmk8850_rtc {
+	qcom,no-alarm;
+	qcom,uefi-rtc-info;
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/glymur/hp/elitebook-x-g2q/qcadsp8480.mbn",
+			"qcom/glymur/hp/elitebook-x-g2q/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/glymur/hp/elitebook-x-g2q/qccdsp8480.mbn",
+			"qcom/glymur/hp/elitebook-x-g2q/cdsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_soccp {
+	firmware-name = "qcom/glymur/hp/elitebook-x-g2q/soccp.mbn",
+			"qcom/glymur/hp/elitebook-x-g2q/soccp_dtb.mbn";
+};
+
+&smb2370_j_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0_1p8>;
+	vdd3-supply = <&vreg_l7b_e0_3p1>;
+};
+
+&smb2370_k_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0_1p8>;
+	vdd3-supply = <&vreg_l7b_e0_3p1>;
+};
+
+&swr0 {
+	status = "okay";
+
+	/* WSA8845, left woofer */
+	left_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferLeft";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, left tweeter */
+	left_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterLeft";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
+&swr3 {
+	status = "okay";
+
+	/* WSA8845, right woofer */
+	right_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferRight";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, right tweeter */
+	right_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterRight";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l2f_e1_0p88>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l2f_e1_0p88>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l2f_e1_0p88>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l2h_e0_0p88>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l2h_e0_0p88>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l2f_e1_0p88>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l2f_e1_0p88>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l2f_e1_0p88>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l2h_e0_0p88>;
+	vdda-qrefrx5-0p9-supply = <&vreg_l3f_e0_0p91>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3f_e0_0p91>;
+	vdda-qreftx0-1p2-supply = <&vreg_l4h_e0_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l1f_e1_0p88>;
+	vdda-refgen3-0p9-supply = <&vreg_l2f_e0_0p94>;
+	vdda-refgen3-1p2-supply = <&vreg_l4h_e0_1p2>;
+	vdda-refgen4-0p9-supply = <&vreg_l1f_e1_0p88>;
+	vdda-refgen4-1p2-supply = <&vreg_l4f_e1_1p2>;
+};
+
+&tlmm {
+	gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */
+			       <10 2>, /* OOB UART */
+			       <44 4>, /* Security SPI (TPM) */
+			       <65 1>; /* EC reset */
+
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	ts0_default: ts0-default-state {
+		pins = "gpio51";
+		function = "gpio";
+		bias-disable;
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	lid_default: lid-default-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio94";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
+		pins = "gpio116", "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie5_default: pcie5-default-state {
+		clkreq-n-pins {
+			pins = "gpio153";
+			function = "pcie5_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio152";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio154";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&usb_0 {
+	status = "okay";
+};
+
+&usb_0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_0_hsphy {
+	vdd-supply = <&vreg_l3f_e0_0p91>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&smb2370_j_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy {
+	vdda-phy-supply = <&vreg_l3f_e0_0p91>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l2f_e0_0p94>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in1>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l3f_e0_0p91>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&smb2370_k_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l1h_e0_0p94>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l2f_e0_0p94>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in1>;
+};
+
+/* Internal USB2 host for the Elan fingerprint reader (usb 04f3:0ca8). */
+&usb_hs {
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_hs_phy {
+	vdd-supply = <&vreg_l2h_e0_0p88>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	status = "okay";
+};
+
+/* Multiport host: port 0 = chassis USB-A, port 1 = internal USB HID. */
+&usb_mp {
+	status = "okay";
+};
+
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2h_e0_0p88>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2h_e0_0p88>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l2h_e0_0p88>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l2h_e0_0p88>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/glymur-lenovo-yoga-slim7x.dts b/arch/arm64/boot/dts/qcom/glymur-lenovo-yoga-slim7x.dts
new file mode 100644
index 00000000000000..769f195c99d91b
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/glymur-lenovo-yoga-slim7x.dts
@@ -0,0 +1,1221 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+/dts-v1/;
+
+#include "glymur.dtsi"
+
+#include "pmcx0102.dtsi"        /* SPMI0: SID-2/3 SPMI1: SID-2/3 */
+#include "pmh0101.dtsi"         /* SPMI0: SID-1                  */
+#include "pmh0110-glymur.dtsi"  /* SPMI0: SID-5/7 SPMI1: SID-5   */
+#include "pmh0104-glymur.dtsi"  /* SPMI0: SID-8/9 SPMI1: SID-11  */
+#include "pmk8850.dtsi"         /* SPMI0: SID-0                  */
+#include "smb2370.dtsi"         /* SPMI2: SID-9/10/11            */
+
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+/ {
+	model = "Lenovo Yoga Slim 7x Gen 11";
+	compatible = "lenovo,yoga-slim7x-gen11", "qcom,glymur";
+
+	aliases {
+		serial0 = &uart21;
+		serial1 = &uart14;
+		i2c0 = &i2c0;
+		i2c5 = &i2c5;
+		i2c8 = &i2c8;
+		i2c9 = &i2c9;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			#clock-cells = <0>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&hall_int_n_default>, <&cam_privacy_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			label = "lid";
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+
+		switch-camera-privacy {
+			label = "Camera privacy switch";
+			gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_CAMERA_LENS_COVER>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		pinctrl-0 = <&cam_indicator_en>;
+		pinctrl-names = "default";
+
+		privacy_led: privacy-led {
+			function = LED_FUNCTION_INDICATOR;
+			color = <LED_COLOR_ID_WHITE>;
+			gpios = <&tlmm 111 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			panic-indicator;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,glymur-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Left side display-adjacent port */
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_0_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		/* Left side user-adjacent port */
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in1: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in1: endpoint {
+						remote-endpoint = <&usb_1_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		/* Right side port */
+		connector@2 {
+			compatible = "usb-c-connector";
+			reg = <2>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in2: endpoint {
+						remote-endpoint = <&usb_2_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in2: endpoint {
+						remote-endpoint = <&usb_2_qmpphy_out>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_misc_3p3: regulator-misc-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_MISC_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&misc_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0101_gpios 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 94 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wwan: regulator-wwan {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WWAN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 246 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wwan_reg_en>;
+		pinctrl-names = "default";
+	};
+
+	sound {
+		compatible = "qcom,glymur-sndcard";
+		model = "GLYMUR-LENOVO-Yoga-Slim7x";
+		audio-routing = "WooferLeft IN", "WSA WSA_SPK1 OUT",
+				"TweeterLeft IN", "WSA WSA_SPK2 OUT",
+				"WooferRight IN", "WSA2 WSA_SPK1 OUT",
+				"TweeterRight IN", "WSA2 WSA_SPK2 OUT",
+				"VA DMIC0", "vdd-micb",
+				"VA DMIC1", "vdd-micb",
+				"VA DMIC2", "vdd-micb",
+				"VA DMIC3", "vdd-micb";
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			codec {
+				sound-dai = <&left_woofer>, <&left_tweeter>,
+					    <&swr0 0>, <&lpass_wsamacro 0>,
+					    <&right_woofer>, <&right_tweeter>,
+					    <&swr3 0>, <&lpass_wsa2macro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_e0>;
+		vddaon-supply = <&vreg_l15b_e0>;
+		vdddig-supply = <&vreg_l15b_e0>;
+		vddrfa1p2-supply = <&vreg_l15b_e0>;
+		vddrfa1p8-supply = <&vreg_l15b_e0>;
+
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_wlan_bt_en>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_l1b_e0: ldo1 {
+			regulator-name = "vreg_l1b_e0";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_e0: ldo7 {
+			regulator-name = "vreg_l7b_e0";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_e0: ldo8 {
+			regulator-name = "vreg_l8b_e0";
+			regulator-min-microvolt = <3304000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_e0: ldo9 {
+			regulator-name = "vreg_l9b_e0";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_e0: ldo10 {
+			regulator-name = "vreg_l10b_e0";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_e0: ldo11 {
+			regulator-name = "vreg_l11b_e0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_e0: ldo12 {
+			regulator-name = "vreg_l12b_e0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_e0: ldo15 {
+			regulator-name = "vreg_l15b_e0";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_e0: ldo17 {
+			regulator-name = "vreg_l17b_e0";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_e0: ldo18 {
+			regulator-name = "vreg_l18b_e0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmcx0102-rpmh-regulators";
+		qcom,pmic-id = "C_E1";
+
+		vreg_l1c_e1: ldo1 {
+			regulator-name = "vreg_l1c_e1";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_e1: ldo2 {
+			regulator-name = "vreg_l2c_e1";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_e1: ldo3 {
+			regulator-name = "vreg_l3c_e1";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_e1: ldo4 {
+			regulator-name = "vreg_l4c_e1";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_s7f_e0: smps7 {
+			regulator-name = "vreg_s7f_e0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s8f_e0: smps8 {
+			regulator-name = "vreg_s8f_e0";
+			regulator-min-microvolt = <952000>;
+			regulator-max-microvolt = <952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_e0: ldo2 {
+			regulator-name = "vreg_l2f_e0";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_e0: ldo3 {
+			regulator-name = "vreg_l3f_e0";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E1";
+
+		vreg_l1f_e1: ldo1 {
+			regulator-name = "vreg_l1f_e1";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_e1: ldo2 {
+			regulator-name = "vreg_l2f_e1";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f_e1: ldo4 {
+			regulator-name = "vreg_l4f_e1";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "H_E0";
+
+		vreg_l1h_e0: ldo1 {
+			regulator-name = "vreg_l1h_e0";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2h_e0: ldo2 {
+			regulator-name = "vreg_l2h_e0";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3h_e0: ldo3 {
+			regulator-name = "vreg_l3h_e0";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4h_e0: ldo4 {
+			regulator-name = "vreg_l4h_e0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gmu {
+	status = "okay";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+
+		hid-descr-addr = <0x20>;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+
+	keyboard@3a {
+		compatible = "hid-over-i2c";
+		reg = <0x3a>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0>;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+
+	/* Something @ 0x5b */
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+};
+
+&i2c8 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@14 {
+		compatible = "hid-over-i2c";
+		reg = <0x14>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>;
+
+		post-power-on-delay-ms = <200>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0>;
+
+		pinctrl-0 = <&ts0_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&i2c9 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	/* EC @ 0x70, irq = TLMM 66 */
+};
+
+&lpass_vamacro {
+	pinctrl-0 = <&dmic01_default>, <&dmic23_default>;
+	pinctrl-names = "default";
+	qcom,dmic-sample-rate = <4800000>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp2 {
+	status = "okay";
+};
+
+&mdss_dp2_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "samsung,atna40hq08", "samsung,atna33xc20";
+			enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+			power-supply = <&vreg_edp_3p3>;
+
+			pinctrl-0 = <&edp_bl_en>;
+			pinctrl-names = "default";
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l2f_e1>;
+	vdda-pll-supply = <&vreg_l4f_e1>;
+
+	status = "okay";
+};
+
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l1c_e1>;
+	vdda-pll-supply = <&vreg_l4f_e1>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pcie5 {
+	vddpe-3v3-supply = <&vreg_nvme>;
+
+	pinctrl-0 = <&pcie5_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie5_phy {
+	vdda-phy-supply = <&vreg_l2f_e0>;
+	vdda-pll-supply = <&vreg_l4h_e0>;
+
+	status = "okay";
+};
+
+&pcie5_port0 {
+	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+};
+
+&pcie6 {
+	vddpe-3v3-supply = <&vreg_wwan>;
+
+	pinctrl-0 = <&pcie6_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie6_phy {
+	vdda-phy-supply = <&vreg_l1c_e1>;
+	vdda-pll-supply = <&vreg_l4f_e1>;
+
+	status = "okay";
+};
+
+&pcie6_port0 {
+	reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+};
+
+&pmh0101_gpios {
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmh0110_f_e0_gpios {
+	misc_3p3_reg_en: misc-3p3-reg-en-state {
+		pins = "gpio6";
+		function = "normal";
+		bias-disable;
+		input-disable;
+		output-enable;
+		drive-push-pull;
+		power-source = <1>; /* 1.8 V */
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+	};
+};
+
+&pmh0110_f_e1_gpios {
+	nvme_sec_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmk8850_rtc {
+	qcom,no-alarm;
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/glymur/LENOVO/83QR/qcadsp8480.mbn",
+			"qcom/glymur/LENOVO/83QR/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/glymur/LENOVO/83QR/qccdsp8480.mbn",
+			"qcom/glymur/LENOVO/83QR/cdsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_soccp {
+	status = "okay";
+};
+
+&smb2370_j_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0>;
+	vdd3-supply = <&vreg_l7b_e0>;
+};
+
+&smb2370_k_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0>;
+	vdd3-supply = <&vreg_l7b_e0>;
+};
+
+&smb2370_l_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0>;
+	vdd3-supply = <&vreg_l7b_e0>;
+
+	status = "okay";
+};
+
+&swr0 {
+	status = "okay";
+
+	/* WSA8845, Left Woofer */
+	left_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferLeft";
+		vdd-1p8-supply = <&vreg_l15b_e0>;
+		vdd-io-supply = <&vreg_l18b_e0>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, Left Tweeter */
+	left_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterLeft";
+		vdd-1p8-supply = <&vreg_l15b_e0>;
+		vdd-io-supply = <&vreg_l18b_e0>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
+&swr3 {
+	status = "okay";
+
+	/* WSA8845, Right Woofer */
+	right_woofer: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "WooferRight";
+		vdd-1p8-supply = <&vreg_l15b_e0>;
+		vdd-io-supply = <&vreg_l18b_e0>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, Right Tweeter */
+	right_tweeter: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 13 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "TweeterRight";
+		vdd-1p8-supply = <&vreg_l15b_e0>;
+		vdd-io-supply = <&vreg_l18b_e0>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l2f_e1>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l2f_e1>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l2f_e1>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l2h_e0>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l2h_e0>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l2f_e1>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l2f_e1>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l2f_e1>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l2h_e0>;
+	vdda-qrefrx5-0p9-supply = <&vreg_l3f_e0>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3f_e0>;
+	vdda-qreftx0-1p2-supply = <&vreg_l4h_e0>;
+	vdda-qreftx1-0p9-supply = <&vreg_l1f_e1>;
+	vdda-refgen3-0p9-supply = <&vreg_l2f_e0>;
+	vdda-refgen3-1p2-supply = <&vreg_l4h_e0>;
+	vdda-refgen4-0p9-supply = <&vreg_l1f_e1>;
+	vdda-refgen4-1p2-supply = <&vreg_l4f_e1>;
+};
+
+&tlmm {
+	gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */
+			       <10 2>, /* OOB UART */
+			       <44 4>, /* Security SPI (TPM) */
+			       <65 1>; /* EC reset (need to unplug the battery if asserted) */
+
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	ts0_default: ts0-default-state {
+		int-n-pins {
+			pins = "gpio51";
+			function = "gpio";
+			bias-disable;
+		};
+
+		reset-n-pins {
+			pins = "gpio48";
+			function = "gpio";
+			drive-strength = <16>;
+			bias-disable;
+		};
+	};
+
+	ec_int_n_default: ec-int-n-state {
+		pins = "gpio66";
+		function = "gpio";
+		bias-disable;
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio94";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	cam_privacy_default: cam-privacy-default-state {
+		pins = "gpio99";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	cam_indicator_en: cam-indicator-en-state {
+		pins = "gpio111";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
+		pins = "gpio116", "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie5_default: pcie5-default-state {
+		clkreq-n-pins {
+			pins = "gpio153";
+			function = "pcie5_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio152";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio154";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie6_default: pcie6-default-state {
+		clkreq-n-pins {
+			pins = "gpio150";
+			function = "pcie6_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio149";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio151";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	wwan_reg_en: wwan-reg-en-state {
+		pins = "gpio246";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&usb_0 {
+	status = "okay";
+};
+
+&usb_0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_0_hsphy {
+	vdd-supply = <&vreg_l3f_e0>;
+	vdda12-supply = <&vreg_l4h_e0>;
+
+	phys = <&smb2370_j_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy {
+	vdda-phy-supply = <&vreg_l4h_e0>;
+	vdda-pll-supply = <&vreg_l3f_e0>;
+	refgen-supply = <&vreg_l2f_e0>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in1>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l3f_e0>;
+	vdda12-supply = <&vreg_l4h_e0>;
+
+	phys = <&smb2370_k_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l4h_e0>;
+	vdda-pll-supply = <&vreg_l1h_e0>;
+	refgen-supply = <&vreg_l2f_e0>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in1>;
+};
+
+&usb_2 {
+	status = "okay";
+};
+
+&usb_2_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in2>;
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&vreg_l4c_e1>;
+	vdda12-supply = <&vreg_l4f_e1>;
+
+	phys = <&smb2370_l_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_2_qmpphy {
+	vdda-phy-supply = <&vreg_l4f_e1>;
+	vdda-pll-supply = <&vreg_l4c_e1>;
+	refgen-supply = <&vreg_l1c_e1>;
+
+	status = "okay";
+};
+
+&usb_2_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in2>;
+};
diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi
index f04058d2e08911..65a398c0a99208 100644
--- a/arch/arm64/boot/dts/qcom/glymur.dtsi
+++ b/arch/arm64/boot/dts/qcom/glymur.dtsi
@@ -21,7 +21,6 @@
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/regulator/qcom,rpmh-regulator.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
@@ -801,8 +800,8 @@
 				 <0>,				/* USB 2 Phy PCIE PIPEGMUX */
 				 <0>,				/* USB 2 Phy PIPEGMUX */
 				 <0>,				/* USB 2 Phy SYS PCIE PIPEGMUX */
-				 <0>,				/* PCIe 3a */
-				 <0>,				/* PCIe 3b */
+				 <&pcie3_phy 0>,		/* PCIe 3a pipe */
+				 <&pcie3_phy 1>,		/* PCIe 3b pipe */
 				 <&pcie4_phy>,			/* PCIe 4 */
 				 <&pcie5_phy>,			/* PCIe 5 */
 				 <&pcie6_phy>,			/* PCIe 6 */
@@ -2374,6 +2373,62 @@
 			};
 		};
 
+		pcie3_phy: phy@f00000 {
+			compatible = "qcom,glymur-qmp-gen5x8-pcie-phy";
+			reg = <0x0 0x00f00000 0x0 0x10000>,
+			      <0x0 0x00f10000 0x0 0x10000>;
+			reg-names = "port_a",
+				    "port_b";
+
+			clocks = <&gcc GCC_PCIE_PHY_3A_AUX_CLK>,
+				 <&gcc GCC_PCIE_3A_CFG_AHB_CLK>,
+				 <&tcsr TCSR_PCIE_3_CLKREF_EN>,
+				 <&gcc GCC_PCIE_3A_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_3A_PIPE_CLK>,
+				 <&gcc GCC_PCIE_PHY_3B_AUX_CLK>,
+				 <&gcc GCC_PCIE_3B_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_3B_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_3B_PIPE_CLK>,
+				 <&gcc GCC_PCIE_3B_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "phy_b_aux",
+				      "cfg_ahb_b",
+				      "rchng_b",
+				      "pipe_b",
+				      "pipediv2_b";
+
+			resets = <&gcc GCC_PCIE_3A_PHY_BCR>,
+				 <&gcc GCC_PCIE_3A_NOCSR_COM_PHY_BCR>,
+				 <&gcc GCC_PCIE_3B_PHY_BCR>,
+				 <&gcc GCC_PCIE_3B_NOCSR_COM_PHY_BCR>;
+			reset-names = "port_a",
+				      "port_a_nocsr",
+				      "port_b",
+				      "port_b_nocsr";
+
+			assigned-clocks = <&gcc GCC_PCIE_3A_PHY_RCHNG_CLK>,
+					  <&gcc GCC_PCIE_3B_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>, <100000000>;
+
+			power-domains = <&gcc GCC_PCIE_3A_PHY_GDSC>,
+					<&gcc GCC_PCIE_3B_PHY_GDSC>;
+			power-domain-names = "port_a", "port_b";
+
+			qcom,link-mode = <&tcsr 0x5000>;
+
+			#clock-cells = <1>;
+			clock-output-names = "pcie3a_pipe_clk",
+					     "pcie3b_pipe_clk";
+
+			#phy-cells = <1>;
+
+			status = "disabled";
+		};
+
 		usb_hs_phy: phy@fa0000 {
 			compatible = "qcom,glymur-m31-eusb2-phy",
 				     "qcom,sm8750-m31-eusb2-phy";
@@ -2403,7 +2458,7 @@
 			status = "disabled";
 		};
 
-		usb_mp_hsphy1: phy@fa2000  {
+		usb_mp_hsphy1: phy@fa2000 {
 			compatible = "qcom,glymur-m31-eusb2-phy",
 				     "qcom,sm8750-m31-eusb2-phy";
 
@@ -2568,7 +2623,7 @@
 			};
 		};
 
-		usb_1_hsphy: phy@fdd000  {
+		usb_1_hsphy: phy@fdd000 {
 			compatible = "qcom,glymur-m31-eusb2-phy",
 				     "qcom,sm8750-m31-eusb2-phy";
 
@@ -3755,6 +3810,8 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie3_phy 1>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -3766,6 +3823,280 @@
 			};
 		};
 
+		pcie3a: pcie@1c10000 {
+			device_type = "pci";
+			compatible = "qcom,glymur-pcie", "qcom,pcie-x1e80100";
+			reg = <0x0 0x01c10000 0x0 0x3000>,
+			      <0x0 0x70000000 0x0 0xf20>,
+			      <0x0 0x70000f40 0x0 0xa8>,
+			      <0x0 0x70001000 0x0 0x4000>,
+			      <0x0 0x70100000 0x0 0x100000>,
+			      <0x0 0x01c13000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x70200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x70300000 0x0 0x70300000 0x0 0x3d00000>,
+				 <0x03000000 0x7 0x00000000 0x7 0x00000000 0x0 0x40000000>,
+				 <0x43000000 0x70 0x00000000 0x70 0x00000000 0x10 0x00000000>;
+
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			linux,pci-domain = <3>;
+
+			operating-points-v2 = <&pcie3a_opp_table>;
+
+			msi-map = <0x0 &gic_its 0xb0000 0x10000>;
+			iommu-map = <0x0 &pcie_smmu 0x30000 0x10000>;
+
+			interrupts = <GIC_SPI 948 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 949 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 844 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 845 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 846 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 847 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 942 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 848 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_SPI 849 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_SPI 850 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_SPI 851 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_PCIE_3A_AUX_CLK>,
+				 <&gcc GCC_PCIE_3A_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_3A_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_3A_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_3A_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_3A_WEST_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "noc_aggr";
+
+			assigned-clocks = <&gcc GCC_PCIE_3A_AUX_CLK>;
+			assigned-clock-rates = <19200000>;
+
+			interconnects = <&pcie_west_anoc MASTER_PCIE_3A QCOM_ICC_TAG_ALWAYS
+					&mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+					&pcie_west_slv_noc SLAVE_PCIE_3A QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			resets = <&gcc GCC_PCIE_3A_BCR>,
+				 <&gcc GCC_PCIE_3A_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			power-domains = <&gcc GCC_PCIE_3A_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555
+						     0x5555 0x5555 0x5555 0x5555>;
+			eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55 0x55 0x55 0x55 0x55>;
+			eq-presets-32gts = /bits/ 8 <0x55 0x55 0x55 0x55 0x55 0x55 0x55 0x55>;
+
+			status = "disabled";
+
+			pcie3a_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* GEN 1 x1 */
+				opp-2500000-1 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <250000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x2 */
+				opp-5000000-1 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x4 */
+				opp-10000000-1 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 1 x8 */
+				opp-20000000-1 {
+					opp-hz = /bits/ 64 <20000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <2000000 1>;
+					opp-level = <1>;
+				};
+
+				/* GEN 2 x1 */
+				opp-5000000-2 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <500000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x2 */
+				opp-10000000-2 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x4 */
+				opp-20000000-2 {
+					opp-hz = /bits/ 64 <20000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <2000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 2 x8 */
+				opp-40000000-2 {
+					opp-hz = /bits/ 64 <40000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <4000000 1>;
+					opp-level = <2>;
+				};
+
+				/* GEN 3 x1 */
+				opp-8000000-3 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x2 */
+				opp-16000000-3 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x4 */
+				opp-32000000-3 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 3 x8 */
+				opp-64000000-3 {
+					opp-hz = /bits/ 64 <64000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <7876000 1>;
+					opp-level = <3>;
+				};
+
+				/* GEN 4 x1 */
+				opp-16000000-4 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <1969000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x2 */
+				opp-32000000-4 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x4 */
+				opp-64000000-4 {
+					opp-hz = /bits/ 64 <64000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <7876000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 4 x8 */
+				opp-128000000-4 {
+					opp-hz = /bits/ 64 <128000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <15753000 1>;
+					opp-level = <4>;
+				};
+
+				/* GEN 5 x1 */
+				opp-32000000-5 {
+					opp-hz = /bits/ 64 <32000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <3938000 1>;
+					opp-level = <5>;
+				};
+
+				/* GEN 5 x2 */
+				opp-64000000-5 {
+					opp-hz = /bits/ 64 <64000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <7876000 1>;
+					opp-level = <5>;
+				};
+
+				/* GEN 5 x4 */
+				opp-128000000-5 {
+					opp-hz = /bits/ 64 <128000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <15753000 1>;
+					opp-level = <5>;
+				};
+
+				/* GEN 5 x8 */
+				opp-256000000-5 {
+					opp-hz = /bits/ 64 <256000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <31506000 1>;
+					opp-level = <5>;
+				};
+			};
+
+			pcie3a_port0: pcie@0 {
+				device_type = "pci";
+				compatible = "pciclass,0604";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie3_phy 0>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
 		rng: rng@10c3000 {
 			compatible = "qcom,glymur-trng", "qcom,trng";
 			reg = <0x0 0x010c3000 0x0 0x1000>;
@@ -4481,7 +4812,59 @@
 			#interconnect-cells = <2>;
 		};
 
-		usb_2_hsphy: phy@88e0000  {
+		sdhc_2: mmc@8804000 {
+			compatible = "qcom,glymur-sdhci", "qcom,sdhci-msm-v5";
+
+			reg = <0x0 0x08804000 0x0 0x1000>;
+			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hc_irq",
+					  "pwr_irq";
+
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "iface",
+				      "core",
+				      "xo";
+
+			iommus = <&apps_smmu 0xd00 0x0>;
+			qcom,dll-config = <0x0007442c>;
+			qcom,ddr-config = <0x80040868>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>;
+			operating-points-v2 = <&sdhc2_opp_table>;
+
+			interconnects = <&aggre3_noc MASTER_SDCC_2 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_SDCC_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "sdhc-ddr",
+					     "cpu-sdhc";
+
+			bus-width = <4>;
+			dma-coherent;
+
+			resets = <&gcc GCC_SDCC2_BCR>;
+
+			status = "disabled";
+
+			sdhc2_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+				};
+
+				opp-202000000 {
+					opp-hz = /bits/ 64 <202000000>;
+					required-opps = <&rpmhpd_opp_nom>;
+				};
+			};
+		};
+
+		usb_2_hsphy: phy@88e0000 {
 			compatible = "qcom,glymur-m31-eusb2-phy",
 				     "qcom,sm8750-m31-eusb2-phy";
 
@@ -4572,6 +4955,10 @@
 				      "noc_aggr_north",
 				      "noc_aggr_south";
 
+			assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_PRIM_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <200000000>;
+
 			interrupts-extended = <&intc GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>,
 					      <&intc GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>,
 					      <&pdc 90 IRQ_TYPE_EDGE_BOTH>,
@@ -4584,9 +4971,18 @@
 					  "ss_phy_irq";
 
 			power-domains = <&gcc GCC_USB30_PRIM_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
 			resets = <&gcc GCC_USB30_PRIM_BCR>;
 
 			iommus = <&apps_smmu 0x1420 0x0>;
+
+			interconnects = <&aggre4_noc MASTER_USB3_0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_USB3_0 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "usb-ddr",
+					     "apps-usb";
+
 			phys = <&usb_0_hsphy>,
 			       <&usb_0_qmpphy QMP_USB43DP_USB3_PHY>;
 			phy-names = "usb2-phy",
@@ -4605,6 +5001,8 @@
 			usb-role-switch;
 			wakeup-source;
 
+			dma-coherent;
+
 			status = "disabled";
 
 			ports {
@@ -4647,6 +5045,10 @@
 				      "noc_aggr_north",
 				      "noc_aggr_south";
 
+			assigned-clocks = <&gcc GCC_USB30_SEC_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_SEC_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <200000000>;
+
 			interrupts-extended = <&intc GIC_SPI 875 IRQ_TYPE_LEVEL_HIGH>,
 					      <&intc GIC_SPI 369 IRQ_TYPE_LEVEL_HIGH>,
 					      <&pdc 88 IRQ_TYPE_EDGE_BOTH>,
@@ -4660,9 +5062,17 @@
 
 			resets = <&gcc GCC_USB30_SEC_BCR>;
 			power-domains = <&gcc GCC_USB30_SEC_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
 
 			iommus = <&apps_smmu 0x1460 0x0>;
 
+			interconnects = <&aggre4_noc MASTER_USB3_1 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_USB3_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "usb-ddr",
+					     "apps-usb";
+
 			phys = <&usb_1_hsphy>,
 			       <&usb_1_qmpphy QMP_USB43DP_USB3_PHY>;
 			phy-names = "usb2-phy",
@@ -4681,6 +5091,8 @@
 			usb-role-switch;
 			wakeup-source;
 
+			dma-coherent;
+
 			status = "disabled";
 
 			ports {
@@ -4723,6 +5135,10 @@
 				      "noc_aggr_north",
 				      "noc_aggr_south";
 
+			assigned-clocks = <&gcc GCC_USB30_TERT_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_TERT_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <200000000>;
+
 			interrupts-extended = <&intc GIC_SPI 871 IRQ_TYPE_LEVEL_HIGH>,
 					      <&intc GIC_SPI 370 IRQ_TYPE_LEVEL_HIGH>,
 					      <&pdc 89 IRQ_TYPE_EDGE_BOTH>,
@@ -4736,9 +5152,17 @@
 
 			resets = <&gcc GCC_USB30_TERT_BCR>;
 			power-domains = <&gcc GCC_USB30_TERT_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
 
 			iommus = <&apps_smmu 0x420 0x0>;
 
+			interconnects = <&aggre2_noc MASTER_USB3_2 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_USB3_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "usb-ddr",
+					     "apps-usb";
+
 			phys = <&usb_2_hsphy>,
 			       <&usb_2_qmpphy QMP_USB43DP_USB3_PHY>;
 			phy-names = "usb2-phy",
@@ -4757,6 +5181,8 @@
 			usb-role-switch;
 			wakeup-source;
 
+			dma-coherent;
+
 			status = "disabled";
 
 			ports {
@@ -4847,6 +5273,8 @@
 			usb-role-switch;
 			wakeup-source;
 
+			dma-coherent;
+
 			status = "disabled";
 		};
 
@@ -4869,6 +5297,10 @@
 				      "noc_aggr_north",
 				      "noc_aggr_south";
 
+			assigned-clocks = <&gcc GCC_USB30_MP_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_MP_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <200000000>;
+
 			interrupts-extended = <&intc GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>,
 					      <&intc GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH>,
 					      <&intc GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH>,
@@ -4894,9 +5326,17 @@
 
 			resets = <&gcc GCC_USB30_MP_BCR>;
 			power-domains = <&gcc GCC_USB30_MP_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
 
 			iommus = <&apps_smmu 0xda0 0x0>;
 
+			interconnects = <&aggre3_noc MASTER_USB3_MP QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&hsc_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_USB3_MP QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "usb-ddr",
+					     "apps-usb";
+
 			phys = <&usb_mp_hsphy0>,
 			       <&usb_mp_qmpphy0>,
 			       <&usb_mp_hsphy1>,
@@ -4919,6 +5359,8 @@
 			dr_mode = "host";
 			wakeup-source;
 
+			dma-coherent;
+
 			status = "disabled";
 		};
 
@@ -6311,6 +6753,46 @@
 					bias-disable;
 				};
 			};
+
+			sdc2_default_state: sdc2-default-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <16>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <10>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <10>;
+					bias-pull-up;
+				};
+			};
+
+			sdc2_sleep_state: sdc2-sleep-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
 		};
 
 		ctcu@10001000 {
@@ -6787,7 +7269,7 @@
 			};
 		};
 
-		itnoc@11200000  {
+		itnoc@11200000 {
 			compatible = "qcom,coresight-itnoc";
 			reg = <0x0 0x11200000 0x0 0x3c00>;
 
diff --git a/arch/arm64/boot/dts/qcom/hamoa-advantech-som-6820.dtsi b/arch/arm64/boot/dts/qcom/hamoa-advantech-som-6820.dtsi
new file mode 100644
index 00000000000000..fec6c4d0b92769
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/hamoa-advantech-som-6820.dtsi
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ * SoM: https://advdownload.advantech.com.cn/productfile/PIS/SOM-6820/file/SOM-6820_DS(052925)20250529110758.pdf
+ */
+
+#include "hamoa-iot-som.dtsi"
+
+/ {
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l3i_0p8>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 238 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <1300000>;
+	vccq-supply = <&vreg_l2i_1p2>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&usb_1_ss0 {
+	/delete-property/ usb-role-switch;
+	dr_mode = "peripheral";
+};
+
+&usb_1_ss1 {
+	/delete-property/ usb-role-switch;
+	dr_mode = "host";
+};
+
+&usb_1_ss2 {
+	/delete-property/ usb-role-switch;
+	dr_mode = "host";
+};
diff --git a/arch/arm64/boot/dts/qcom/hamoa-advantech-som-db5830.dts b/arch/arm64/boot/dts/qcom/hamoa-advantech-som-db5830.dts
new file mode 100644
index 00000000000000..9d87c98befc65e
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/hamoa-advantech-som-db5830.dts
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ * SoM: https://advdownload.advantech.com.cn/productfile/PIS/SOM-6820/file/SOM-6820_DS(052925)20250529110758.pdf
+ * carrier board: https://www.advantech.com.cn/zh-cn/products/1-fc23ff/som-db5830/mod_16b1a718-27d6-4a4c-8684-e8f2a8e9c57b
+ */
+
+/dts-v1/;
+
+#include "hamoa-advantech-som-6820.dtsi"
+
+/ {
+	model = "Advantech SOM-6820 on SOM-DB5830";
+	compatible = "advantech,som-db5830", "advantech,som-6820",
+		     "qcom,iq-x7181-evk-som", "qcom,x1e80100";
+	chassis-type = "embedded";
+
+	aliases {
+		serial0 = &uart21;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+};
+
+&pcie3_port0 {
+	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
+
+&pcie5_port0 {
+	reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+};
+
+&pcie6a_port0 {
+	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+};
+
+&thermal_zones {
+	gpuss-0-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-1-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-2-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-3-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-4-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-5-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-6-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+
+	gpuss-7-thermal {
+		trips {
+			trip-point0 {
+				temperature = <105000>;
+			};
+		};
+	};
+};
+
+&uart21 {
+	compatible = "qcom,geni-debug-uart";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
index 9fa86bb6438ecf..d0433cc3623fef 100644
--- a/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
+++ b/arch/arm64/boot/dts/qcom/hamoa-iot-evk.dts
@@ -35,6 +35,24 @@
 			clock-frequency = <40000000>;
 			#clock-cells = <0>;
 		};
+
+		clk_rtmr0_xo: clock-rtmr0-xo {
+			compatible = "fixed-clock";
+			clock-frequency = <25000000>;
+			#clock-cells = <0>;
+		};
+
+		clk_rtmr1_xo: clock-rtmr1-xo {
+			compatible = "fixed-clock";
+			clock-frequency = <25000000>;
+			#clock-cells = <0>;
+		};
+
+		clk_rtmr2_xo: clock-rtmr2-xo {
+			compatible = "fixed-clock";
+			clock-frequency = <25000000>;
+			#clock-cells = <0>;
+		};
 	};
 
 	wcd938x: audio-codec {
@@ -126,15 +144,15 @@
 					reg = <1>;
 
 					pmic_glink_ss0_ss_in: endpoint {
-						remote-endpoint = <&usb_1_ss0_qmpphy_out>;
+						remote-endpoint = <&retimer_ss0_ss_out>;
 					};
 				};
 
 				port@2 {
 					reg = <2>;
 
-					pmic_glink_ss0_sbu: endpoint {
-						remote-endpoint = <&usb_1_ss0_sbu_mux>;
+					pmic_glink_ss0_con_sbu_in: endpoint {
+						remote-endpoint = <&retimer_ss0_con_sbu_out>;
 					};
 				};
 			};
@@ -303,7 +321,6 @@
 		pinctrl-names = "default";
 	};
 
-	/* Left unused as the retimer is not used on this board. */
 	vreg_rtmr0_1p15: regulator-rtmr0-1p15 {
 		compatible = "regulator-fixed";
 
@@ -479,32 +496,6 @@
 		regulator-boot-on;
 	};
 
-	/*
-	 * TODO: These two regulators are actually part of the removable M.2
-	 * card and not the EVK mainboard. Need to describe this differently.
-	 * Functionally it works correctly, because all we need to do is to
-	 * turn on the actual 3.3V supply above.
-	 */
-	vreg_wcn_0p95: regulator-wcn-0p95 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_WCN_0P95";
-		regulator-min-microvolt = <950000>;
-		regulator-max-microvolt = <950000>;
-
-		vin-supply = <&vreg_wcn_3p3>;
-	};
-
-	vreg_wcn_1p9: regulator-wcn-1p9 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_WCN_1P9";
-		regulator-min-microvolt = <1900000>;
-		regulator-max-microvolt = <1900000>;
-
-		vin-supply = <&vreg_wcn_3p3>;
-	};
-
 	vreg_wcn_3p3: regulator-wcn-3p3 {
 		compatible = "regulator-fixed";
 
@@ -658,83 +649,44 @@
 		};
 	};
 
-	usb-1-ss0-sbu-mux {
-		compatible = "onnn,fsusb42", "gpio-sbu-mux";
-
-		enable-gpios = <&tlmm 168 GPIO_ACTIVE_LOW>;
-		select-gpios = <&tlmm 167 GPIO_ACTIVE_HIGH>;
-
-		pinctrl-0 = <&usb_1_ss0_sbu_default>;
-		pinctrl-names = "default";
-
-		mode-switch;
-		orientation-switch;
-
-		port {
-			usb_1_ss0_sbu_mux: endpoint {
-				remote-endpoint = <&pmic_glink_ss0_sbu>;
-			};
-		};
-	};
-
-	wcn7850-pmu {
-		compatible = "qcom,wcn7850-pmu";
+	wifi-bt-connector {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&vreg_wcn_3p3>;
 
-		vdd-supply = <&vreg_wcn_0p95>;
-		vddio-supply = <&vreg_l15b_1p8>;
-		vddaon-supply = <&vreg_wcn_0p95>;
-		vdddig-supply = <&vreg_wcn_0p95>;
-		vddrfa1p2-supply = <&vreg_wcn_1p9>;
-		vddrfa1p8-supply = <&vreg_wcn_1p9>;
+		w-disable1-gpios = <&tlmm 117 GPIO_ACTIVE_LOW>;
+		w-disable2-gpios = <&tlmm 116 GPIO_ACTIVE_LOW>;
 
-		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
-		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
-
-		pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>;
+		pinctrl-0 = <&wcn_wlan_en>, <&wcn_bt_en>;
 		pinctrl-names = "default";
 
-		regulators {
-			vreg_pmu_rfa_cmn: ldo0 {
-				regulator-name = "vreg_pmu_rfa_cmn";
-			};
-
-			vreg_pmu_aon_0p59: ldo1 {
-				regulator-name = "vreg_pmu_aon_0p59";
-			};
-
-			vreg_pmu_wlcx_0p8: ldo2 {
-				regulator-name = "vreg_pmu_wlcx_0p8";
-			};
-
-			vreg_pmu_wlmx_0p85: ldo3 {
-				regulator-name = "vreg_pmu_wlmx_0p85";
-			};
-
-			vreg_pmu_btcmx_0p85: ldo4 {
-				regulator-name = "vreg_pmu_btcmx_0p85";
-			};
-
-			vreg_pmu_rfa_0p8: ldo5 {
-				regulator-name = "vreg_pmu_rfa_0p8";
-			};
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
 
-			vreg_pmu_rfa_1p2: ldo6 {
-				regulator-name = "vreg_pmu_rfa_1p2";
-			};
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-			vreg_pmu_rfa_1p8: ldo7 {
-				regulator-name = "vreg_pmu_rfa_1p8";
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcie4_port0_ep>;
+				};
 			};
 
-			vreg_pmu_pcie_0p9: ldo8 {
-				regulator-name = "vreg_pmu_pcie_0p9";
-			};
+			port@3 {
+				reg = <3>;
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-			vreg_pmu_pcie_1p8: ldo9 {
-				regulator-name = "vreg_pmu_pcie_1p8";
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&uart14_ep>;
+				};
 			};
 		};
 	};
+
 };
 
 &i2c1 {
@@ -745,7 +697,7 @@
 		compatible = "parade,ps8830";
 		reg = <0x08>;
 
-		clocks = <&rpmhcc RPMH_RF_CLK5>;
+		clocks = <&clk_rtmr2_xo>;
 
 		vdd-supply = <&vreg_rtmr2_1p15>;
 		vdd33-supply = <&vreg_rtmr2_3p3>;
@@ -793,6 +745,63 @@
 	};
 };
 
+&i2c3 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	typec-mux@8 {
+		compatible = "parade,ps8830";
+		reg = <0x08>;
+
+		clocks = <&clk_rtmr0_xo>;
+
+		vdd-supply = <&vreg_rtmr0_1p15>;
+		vdd33-supply = <&vreg_rtmr0_3p3>;
+		vdd33-cap-supply = <&vreg_rtmr0_3p3>;
+		vddar-supply = <&vreg_rtmr0_1p15>;
+		vddat-supply = <&vreg_rtmr0_1p15>;
+		vddio-supply = <&vreg_rtmr0_1p8>;
+
+		reset-gpios = <&pm8550_gpios 10 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&rtmr0_default>;
+		pinctrl-names = "default";
+
+		orientation-switch;
+		retimer-switch;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				retimer_ss0_ss_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_ss_in>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				retimer_ss0_ss_in: endpoint {
+					remote-endpoint = <&usb_1_ss0_qmpphy_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				retimer_ss0_con_sbu_out: endpoint {
+					remote-endpoint = <&pmic_glink_ss0_con_sbu_in>;
+				};
+			};
+		};
+	};
+};
+
 &i2c5 {
 	clock-frequency = <400000>;
 
@@ -860,7 +869,7 @@
 		compatible = "parade,ps8830";
 		reg = <0x8>;
 
-		clocks = <&rpmhcc RPMH_RF_CLK4>;
+		clocks = <&clk_rtmr1_xo>;
 
 		vdd-supply = <&vreg_rtmr1_1p15>;
 		vdd33-supply = <&vreg_rtmr1_3p3>;
@@ -1024,21 +1033,10 @@
 &pcie4_port0 {
 	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
 	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
 
-	wifi@0 {
-		compatible = "pci17cb,1107";
-		reg = <0x10000 0x0 0x0 0x0 0x0>;
-
-		vddaon-supply = <&vreg_pmu_aon_0p59>;
-		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
-		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
-		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
-		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
-		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
-		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
-		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
-	};
+&pcie4_port0_ep {
+	remote-endpoint = <&m2_e_pcie_ep>;
 };
 
 &pcie5 {
@@ -1458,30 +1456,6 @@
 		bias-disable;
 	};
 
-	usb_1_ss0_sbu_default: usb-1-ss0-sbu-state {
-		mode-pins {
-			pins = "gpio166";
-			function = "gpio";
-			bias-disable;
-			drive-strength = <2>;
-			output-high;
-		};
-
-		oe-n-pins {
-			pins = "gpio168";
-			function = "gpio";
-			bias-disable;
-			drive-strength = <2>;
-		};
-
-		sel-pins {
-			pins = "gpio167";
-			function = "gpio";
-			bias-disable;
-			drive-strength = <2>;
-		};
-	};
-
 	wcd_default: wcd-reset-n-active-state {
 		pins = "gpio191";
 		function = "gpio";
@@ -1530,19 +1504,10 @@
 
 &uart14 {
 	status = "okay";
+};
 
-	bluetooth {
-		compatible = "qcom,wcn7850-bt";
-		max-speed = <3200000>;
-
-		vddaon-supply = <&vreg_pmu_aon_0p59>;
-		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
-		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
-		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
-		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
-		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
-	};
+&uart14_ep {
+	remote-endpoint = <&m2_e_uart_ep>;
 };
 
 &uart21 {
@@ -1578,7 +1543,7 @@
 };
 
 &usb_1_ss0_qmpphy_out {
-	remote-endpoint = <&pmic_glink_ss0_ss_in>;
+	remote-endpoint = <&retimer_ss0_ss_in>;
 };
 
 &usb_1_ss1_dwc3_hs {
diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi
index bc70fe5db96f5b..f139f3adb93353 100644
--- a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi
@@ -478,6 +478,27 @@
 	status = "okay";
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>; /* TPM LP & INT */
 
diff --git a/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts b/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts
index 28482500838913..75ce8b0c94d2a3 100644
--- a/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts
+++ b/arch/arm64/boot/dts/qcom/hamoa-lenovo-ideacentre-mini-01q8x10.dts
@@ -902,6 +902,27 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <44 4>,  /* SPI11 (TPM) */
 			       <76 4>,  /* SPI19 (TZ Protected) */
diff --git a/arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi b/arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi
index 17d754d57e1897..7fad4a9b92fa27 100644
--- a/arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi
@@ -288,6 +288,7 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			#io-channel-cells = <1>;
+			#thermal-sensor-cells = <1>;
 			pinctrl-0 = <&sys_therm_0_gpio3>, <&sys_therm_1_gpio4>;
 			pinctrl-names = "default";
 
diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi
index 8a96ea1bdb9c4c..48f47aa6ba3c7a 100644
--- a/arch/arm64/boot/dts/qcom/hamoa.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi
@@ -19,7 +19,6 @@
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
@@ -2033,6 +2032,11 @@
 				pinctrl-names = "default";
 
 				status = "disabled";
+
+				port {
+					uart14_ep: endpoint {
+					};
+				};
 			};
 
 			i2c15: i2c@a9c000 {
@@ -2108,7 +2112,7 @@
 			};
 		};
 
-		gpi_dma0: dma-controller@b00000  {
+		gpi_dma0: dma-controller@b00000 {
 			compatible = "qcom,x1e80100-gpi-dma", "qcom,sm6350-gpi-dma";
 			reg = <0 0x00b00000 0 0x60000>;
 
@@ -2833,11 +2837,13 @@
 			clocks = <&gcc GCC_USB3_PRIM_PHY_AUX_CLK>,
 				 <&rpmhcc RPMH_CXO_CLK>,
 				 <&gcc GCC_USB3_PRIM_PHY_COM_AUX_CLK>,
-				 <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>;
+				 <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>,
+				 <&gcc GCC_USB4_0_PHY_P2RR2P_PIPE_CLK>;
 			clock-names = "aux",
 				      "ref",
 				      "com_aux",
-				      "usb3_pipe";
+				      "usb3_pipe",
+				      "p2rr2p_pipe";
 
 			power-domains = <&gcc GCC_USB_0_PHY_GDSC>;
 
@@ -2904,11 +2910,13 @@
 			clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>,
 				 <&tcsr TCSR_USB4_1_CLKREF_EN>,
 				 <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>,
-				 <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>;
+				 <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>,
+				 <&gcc GCC_USB4_1_PHY_P2RR2P_PIPE_CLK>;
 			clock-names = "aux",
 				      "ref",
 				      "com_aux",
-				      "usb3_pipe";
+				      "usb3_pipe",
+				      "p2rr2p_pipe";
 
 			power-domains = <&gcc GCC_USB_1_PHY_GDSC>;
 
@@ -2975,11 +2983,13 @@
 			clocks = <&gcc GCC_USB3_TERT_PHY_AUX_CLK>,
 				 <&tcsr TCSR_USB4_2_CLKREF_EN>,
 				 <&gcc GCC_USB3_TERT_PHY_COM_AUX_CLK>,
-				 <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>;
+				 <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>,
+				 <&gcc GCC_USB4_2_PHY_P2RR2P_PIPE_CLK>;
 			clock-names = "aux",
 				      "ref",
 				      "com_aux",
-				      "usb3_pipe";
+				      "usb3_pipe",
+				      "p2rr2p_pipe";
 
 			power-domains = <&gcc GCC_USB_2_PHY_GDSC>;
 
@@ -3082,6 +3092,7 @@
 			qcom,bcm-voters = <&apps_bcm_voter>;
 
 			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>;
 		};
 
 		aggre2_noc: interconnect@1700000 {
@@ -3118,6 +3129,8 @@
 			qcom,bcm-voters = <&apps_bcm_voter>;
 
 			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_USB2_PRIM_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_MP_AXI_CLK>;
 		};
 
 		usb_south_anoc: interconnect@1770000 {
@@ -3127,6 +3140,12 @@
 			qcom,bcm-voters = <&apps_bcm_voter>;
 
 			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_SEC_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB3_TERT_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_0_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_1_AXI_CLK>,
+				 <&gcc GCC_AGGRE_USB4_2_AXI_CLK>;
 		};
 
 		mmss_noc: interconnect@1780000 {
@@ -3702,7 +3721,7 @@
 			      <0 0x7c100000 0 0x100000>,
 			      <0 0x01c0b000 0 0x1000>;
 			reg-names = "parf",
-			            "dbi",
+				    "dbi",
 				    "elbi",
 				    "atu",
 				    "config",
@@ -3784,6 +3803,7 @@
 			status = "disabled";
 
 			pcie4_port0: pcie@0 {
+				compatible = "pciclass,0604";
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
@@ -3793,6 +3813,11 @@
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
+
+				port {
+					pcie4_port0_ep: endpoint {
+					};
+				};
 			};
 		};
 
@@ -3984,7 +4009,6 @@
 			reg = <0 0x01fc0000 0 0x30000>;
 			clocks = <&rpmhcc RPMH_CXO_CLK>;
 			#clock-cells = <1>;
-			#reset-cells = <1>;
 		};
 
 		gpu: gpu@3d00000 {
diff --git a/arch/arm64/boot/dts/qcom/hawi-ipcc.h b/arch/arm64/boot/dts/qcom/hawi-ipcc.h
new file mode 100644
index 00000000000000..7659357691c1ba
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/hawi-ipcc.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DTS_HAWI_MAILBOX_IPCC_H
+#define __DTS_HAWI_MAILBOX_IPCC_H
+
+/* Physical client IDs */
+#define IPCC_MPROC_AOP			0
+#define IPCC_MPROC_TZ			1
+#define IPCC_MPROC_MPSS			2
+#define IPCC_MPROC_LPASS		3
+#define IPCC_MPROC_SDC			4
+#define IPCC_MPROC_CDSP			5
+#define IPCC_MPROC_APSS			6
+#define IPCC_MPROC_SOCCP		12
+#define IPCC_MPROC_DCP			13
+#define IPCC_MPROC_QECP			14
+#define IPCC_MPROC_SPSS			15
+#define IPCC_MPROC_TME			16
+
+#define IPCC_COMPUTE_L0_LPASS		1
+#define IPCC_COMPUTE_L0_CDSP		2
+#define IPCC_COMPUTE_L0_APSS		3
+#define IPCC_COMPUTE_L0_GPU		4
+#define IPCC_COMPUTE_L0_CVP		8
+#define IPCC_COMPUTE_L0_CAM		9
+#define IPCC_COMPUTE_L0_CAM1		10
+#define IPCC_COMPUTE_L0_DCP		11
+#define IPCC_COMPUTE_L0_VPU		12
+#define IPCC_COMPUTE_L0_SOCCP		16
+
+#define IPCC_COMPUTE_L1_LPASS		1
+#define IPCC_COMPUTE_L1_CDSP		2
+#define IPCC_COMPUTE_L1_APSS		3
+#define IPCC_COMPUTE_L1_GPU		4
+#define IPCC_COMPUTE_L1_CVP		8
+#define IPCC_COMPUTE_L1_CAM		9
+#define IPCC_COMPUTE_L1_CAM1		10
+#define IPCC_COMPUTE_L1_DCP		11
+#define IPCC_COMPUTE_L1_VPU		12
+#define IPCC_COMPUTE_L1_SOCCP		16
+
+#define IPCC_PERIPH_CDSP		2
+#define IPCC_PERIPH_APSS		3
+#define IPCC_PERIPH_PCIE0		4
+#define IPCC_PERIPH_PCIE1		15
+
+#define IPCC_FENCE_LPASS		1
+#define IPCC_FENCE_CDSP			2
+#define IPCC_FENCE_APSS			3
+#define IPCC_FENCE_GPU			4
+#define IPCC_FENCE_CVP			8
+#define IPCC_FENCE_CAM			9
+#define IPCC_FENCE_CAM1			10
+#define IPCC_FENCE_DCP			11
+#define IPCC_FENCE_VPU			20
+#define IPCC_FENCE_SOCCP		24
+
+#endif
diff --git a/arch/arm64/boot/dts/qcom/hawi-mtp.dts b/arch/arm64/boot/dts/qcom/hawi-mtp.dts
new file mode 100644
index 00000000000000..c38e5fab7a2cb2
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/hawi-mtp.dts
@@ -0,0 +1,997 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "hawi.dtsi"
+
+#include "pmd8028-kaanapali.dtsi"	/* SPMI1: SID-4 */
+#include "pmh0101.dtsi"			/* SPMI0: SID-1 */
+#include "pmh0104-hawi.dtsi"		/* SPMI0: SID-2 SPMI1: SID-9 */
+#include "pmh0110-kaanapali.dtsi"	/* SPMI0: SID-3/5/6/8 */
+#include "pmih0108-kaanapali.dtsi"	/* SPMI1: SID-7 */
+#include "pmk8850.dtsi"			/* SPMI0: SID-0 */
+#include "pmr735d-kaanapali.dtsi"	/* SPMI1: SID-A */
+#include "pm8010-kaanapali.dtsi"	/* SPMI1: SID-C/D */
+
+/ {
+	model = "Qualcomm Technologies, Inc. Hawi MTP";
+	compatible = "qcom,hawi-mtp", "qcom,hawi";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		bi_tcxo_ao_div2: bi-tcxo-ao-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK_A>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+
+		bi_tcxo_div2: bi-tcxo-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32764>;
+			#clock-cells = <0>;
+		};
+
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <76800000>;
+			#clock-cells = <0>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&key_vol_up_default>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+			gpios = <&tlmm 101 GPIO_ACTIVE_LOW>;
+			debounce-interval = <15>;
+			linux,can-disable;
+			wakeup-source;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,hawi-pmic-glink",
+			     "qcom,kaanapali-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_dp_qmpphy_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_dp_out: endpoint {
+					};
+				};
+			};
+		};
+	};
+
+	sdcard_vdd_3p0: sdcard-vdd-3p0 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "sdc-vdd123";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		regulator-enable-ramp-delay = <1500>;
+
+		gpio = <&pmh0104_c_e0_gpios 5 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&sd_card_eldo_en>;
+		pinctrl-names = "default";
+
+		vin-supply = <&vreg_bob1>;
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <4000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2704000>;
+			regulator-max-microvolt = <3552000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4b_1p8: ldo4 {
+			regulator-name = "vreg_l4b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l5b_3p1: ldo5 {
+			regulator-name = "vreg_l5b_3p1";
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l7b_1p8: ldo7 {
+			regulator-name = "vreg_l7b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l8b_1p8: ldo8 {
+			regulator-name = "vreg_l8b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l9b_2p5: ldo9 {
+			regulator-name = "vreg_l9b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l11b_1p2: ldo11 {
+			regulator-name = "vreg_l11b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l12b_1p8: ldo12 {
+			regulator-name = "vreg_l12b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l14b_3p2: ldo14 {
+			regulator-name = "vreg_l14b_3p2";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l16b_2p9: ldo16 {
+			regulator-name = "vreg_l16b_2p9";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3600000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p2: ldo18 {
+			regulator-name = "vreg_l18b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmh0104-rpmh-regulators";
+		qcom,pmic-id = "C_E0";
+
+		vreg_s1c_0p5: smps1 {
+			regulator-name = "vreg_s1c_0p5";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <572000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3c_1p2: smps3 {
+			regulator-name = "vreg_s3c_1p2";
+			regulator-min-microvolt = <1224000>;
+			regulator-max-microvolt = <1372000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4c_1p8: smps4 {
+			regulator-name = "vreg_s4c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "D_E0";
+
+		vreg_s9d_1p0: smps9 {
+			regulator-name = "vreg_s9d_1p0";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1d_0p9: ldo1 {
+			regulator-name = "vreg_l1d_0p9";
+			regulator-min-microvolt = <824000>;
+			regulator-max-microvolt = <952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2d_0p9: ldo2 {
+			regulator-name = "vreg_l2d_0p9";
+			regulator-min-microvolt = <824000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3d_1p2: ldo3 {
+			regulator-name = "vreg_l3d_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4d_0p8: ldo4 {
+			regulator-name = "vreg_l4d_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_s7f_0p9: smps7 {
+			regulator-name = "vreg_s7f_0p9";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <972000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s8f_0p8: smps8 {
+			regulator-name = "vreg_s8f_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_1p2: ldo1 {
+			regulator-name = "vreg_l1f_1p2";
+			regulator-min-microvolt = <744000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2f_1p2: ldo2 {
+			regulator-name = "vreg_l2f_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3f_0p9: ldo3 {
+			regulator-name = "vreg_l3f_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4f_0p8: ldo4 {
+			regulator-name = "vreg_l4f_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "G_E0";
+
+		vreg_l1g_1p2: ldo1 {
+			regulator-name = "vreg_l1g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2g_1p8: ldo2 {
+			regulator-name = "vreg_l2g_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3g_1p2: ldo3 {
+			regulator-name = "vreg_l3g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l4g_1p2: ldo4 {
+			regulator-name = "vreg_l4g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-5 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "I_E0";
+
+		vreg_l1i_0p8: ldo1 {
+			regulator-name = "vreg_l1i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l2i_0p8: ldo2 {
+			regulator-name = "vreg_l2i_0p8";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l3i_1p2: ldo3 {
+			regulator-name = "vreg_l3i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pmh0104-rpmh-regulators";
+		qcom,pmic-id = "J_E1";
+
+		vreg_s1j_0p8: smps1 {
+			regulator-name = "vreg_s1j_0p8";
+			regulator-min-microvolt = <400000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2j_0p8: smps2 {
+			regulator-name = "vreg_s2j_0p8";
+			regulator-min-microvolt = <400000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3j_1p2: smps3 {
+			regulator-name = "vreg_s3j_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4j_0p7: smps4 {
+			regulator-name = "vreg_s4j_0p7";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pmr735d-rpmh-regulators";
+		qcom,pmic-id = "K_E1";
+
+		vreg_l1k_0p7: ldo1 {
+			regulator-name = "vreg_l1k_0p7";
+			regulator-min-microvolt = <496000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2k_0p7: ldo2 {
+			regulator-name = "vreg_l2k_0p7";
+			regulator-min-microvolt = <696000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3k_1p2: ldo3 {
+			regulator-name = "vreg_l3k_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1256000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4k_1p0: ldo4 {
+			regulator-name = "vreg_l4k_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5k_0p9: ldo5 {
+			regulator-name = "vreg_l5k_0p9";
+			regulator-min-microvolt = <696000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6k_1p7: ldo6 {
+			regulator-name = "vreg_l6k_1p7";
+			regulator-min-microvolt = <1696000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7k_0p7: ldo7 {
+			regulator-name = "vreg_l7k_0p7";
+			regulator-min-microvolt = <696000>;
+			regulator-max-microvolt = <848000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-8 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+		qcom,pmic-id = "M_E1";
+
+		vreg_l1m_1p0: ldo1 {
+			regulator-name = "vreg_l1m_1p0";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1056000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2m_1p0: ldo2 {
+			regulator-name = "vreg_l2m_1p0";
+			regulator-min-microvolt = <1096000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3m_2p8: ldo3 {
+			regulator-name = "vreg_l3m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4m_2p8: ldo4 {
+			regulator-name = "vreg_l4m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2896000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5m_2p8: ldo5 {
+			regulator-name = "vreg_l5m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2896000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6m_2p8: ldo6 {
+			regulator-name = "vreg_l6m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2896000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7m_2p2: ldo7 {
+			regulator-name = "vreg_l7m_2p2";
+			regulator-min-microvolt = <2200000>;
+			regulator-max-microvolt = <2296000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-9 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+		qcom,pmic-id = "N_E1";
+
+		vreg_l1n_0p8: ldo1 {
+			regulator-name = "vreg_l1n_0p8";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <896000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2n_1p2: ldo2 {
+			regulator-name = "vreg_l2n_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3n_1p8: ldo3 {
+			regulator-name = "vreg_l3n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4n_1p8: ldo4 {
+			regulator-name = "vreg_l4n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5n_3p3: ldo5 {
+			regulator-name = "vreg_l5n_3p3";
+			regulator-min-microvolt = <3296000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6n_2p8: ldo6 {
+			regulator-name = "vreg_l6n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7n_3p3: ldo7 {
+			regulator-name = "vreg_l7n_3p3";
+			regulator-min-microvolt = <3296000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&pcie0 {
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1i_0p8>;
+	vdda-pll-supply = <&vreg_l3d_1p2>;
+
+	status = "okay";
+};
+
+&pcie0_port0 {
+	wake-gpios = <&tlmm 104 GPIO_ACTIVE_LOW>;
+	reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
+};
+
+&pcie1 {
+	status = "okay";
+};
+
+&pcie1_phy {
+	vdda-phy-supply = <&vreg_l1i_0p8>;
+	vdda-pll-supply = <&vreg_l3d_1p2>;
+
+	status = "okay";
+};
+
+&pcie1_port0 {
+	wake-gpios = <&tlmm 222 GPIO_ACTIVE_LOW>;
+	reset-gpios = <&tlmm 220 GPIO_ACTIVE_LOW>;
+};
+
+&pmh0101_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		function-enumerator = <0>;
+		color = <LED_COLOR_ID_YELLOW>;
+		led-sources = <1>, <4>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <2000000>;
+		flash-max-timeout-us = <1280000>;
+	};
+
+	led-1 {
+		function = LED_FUNCTION_FLASH;
+		function-enumerator = <1>;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <2>, <3>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <2000000>;
+		flash-max-timeout-us = <1280000>;
+	};
+};
+
+&pmh0101_pwm {
+	status = "okay";
+
+	multi-led {
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_STATUS;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_RED>;
+		};
+
+		led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_GREEN>;
+		};
+
+		led@3 {
+			reg = <3>;
+			color = <LED_COLOR_ID_BLUE>;
+		};
+	};
+};
+
+&pmh0104_c_e0_gpios {
+	sd_card_eldo_en: sd-card-eldo-en-state {
+		pins = "gpio5";
+		function = "normal";
+		input-disable;
+		output-low;
+		power-source = <0>;
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_HIGH>;
+	};
+};
+
+&pmih0108_e1_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_1p8>;
+	vdd3-supply = <&vreg_l5b_3p1>;
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/hawi/adsp.mbn",
+			"qcom/hawi/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/hawi/cdsp.mbn",
+			"qcom/hawi/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/hawi/modem.mbn",
+			"qcom/hawi/modem_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_soccp {
+	firmware-name = "qcom/hawi/soccp.mbn",
+			"qcom/hawi/soccp_dtb.mbn";
+};
+
+&sdhc_2 {
+	cd-gpios = <&tlmm 55 GPIO_ACTIVE_LOW>;
+
+	vmmc-supply = <&sdcard_vdd_3p0>;
+	vqmmc-supply = <&vreg_l16b_2p9>;
+
+	bus-width = <4>;
+	no-sdio;
+	no-mmc;
+
+	pinctrl-0 = <&sdc2_default>;
+	pinctrl-1 = <&sdc2_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <36 4>, /* NFC eSE SPI */
+			       <74 1>, /* eSE */
+			       <119 2>, /* SoCCP */
+			       <144 2>; /* CXM UART */
+
+	key_vol_up_default: key-vol-up-default-state {
+		pins = "gpio101";
+		function = "gpio";
+		output-disable;
+		bias-pull-up;
+	};
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 226 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l9b_2p5>;
+	vcc-max-microamp = <1200000>;
+
+	vccq-supply = <&vreg_l4g_1p2>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l3f_0p9>;
+	vdda-pll-supply = <&vreg_l3d_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	dr_mode = "otg";
+	usb-role-switch;
+
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l4f_0p8>;
+	vdda12-supply = <&vreg_l3d_1p2>;
+
+	phys = <&pmih0108_e1_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy {
+	vdda-phy-supply = <&vreg_l4f_0p8>;
+	vdda-pll-supply = <&vreg_l3d_1p2>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
diff --git a/arch/arm64/boot/dts/qcom/hawi.dtsi b/arch/arm64/boot/dts/qcom/hawi.dtsi
new file mode 100644
index 00000000000000..6b0a0ff0faa456
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/hawi.dtsi
@@ -0,0 +1,6844 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/clock/qcom,hawi-camcc.h>
+#include <dt-bindings/clock/qcom,hawi-gcc.h>
+#include <dt-bindings/clock/qcom,hawi-gpucc.h>
+#include <dt-bindings/clock/qcom,hawi-tcsrcc.h>
+#include <dt-bindings/clock/qcom,hawi-videocc.h>
+#include <dt-bindings/clock/qcom,kaanapali-cambistmclkcc.h>
+#include <dt-bindings/clock/qcom,kaanapali-gxclkctl.h>
+#include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/dma/qcom-gpi.h>
+#include <dt-bindings/interconnect/qcom,hawi-rpmh.h>
+#include <dt-bindings/interconnect/qcom,icc.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/mailbox/qcom-ipcc.h>
+#include <dt-bindings/phy/phy-qcom-qmp.h>
+#include <dt-bindings/power/qcom,rpmhpd.h>
+#include <dt-bindings/power/qcom-rpmpd.h>
+#include <dt-bindings/soc/qcom,rpmh-rsc.h>
+
+#include "hawi-ipcc.h"
+
+/ {
+	interrupt-parent = <&intc>;
+
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu6: cpu@0 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-0";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd6>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+		};
+
+		cpu7: cpu@100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-0";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd7>, <&scmi_perf 0>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+		};
+
+		cpu3: cpu@200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-1";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd3>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+		};
+
+		cpu0: cpu@300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-2";
+			reg = <0x0 0x300>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd0>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+
+			l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-1";
+			reg = <0x0 0x400>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd4>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+		};
+
+		cpu1: cpu@500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-2";
+			reg = <0x0 0x500>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd1>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+		};
+
+		cpu5: cpu@600 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-1";
+			reg = <0x0 0x600>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd5>, <&scmi_perf 2>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+		};
+
+		cpu2: cpu@700 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-3-2";
+			reg = <0x0 0x700>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+			power-domains = <&cpu_pd2>, <&scmi_perf 1>;
+			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu3>;
+				};
+
+				core1 {
+					cpu = <&cpu4>;
+				};
+
+				core2 {
+					cpu = <&cpu5>;
+				};
+			};
+
+			cluster2 {
+				core0 {
+					cpu = <&cpu6>;
+				};
+
+				core1 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
+		domain-idle-states {
+			cluster_cl5: cluster-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x01000054>;
+				entry-latency-us = <2150>;
+				exit-latency-us = <1983>;
+				min-residency-us = <17845>;
+			};
+
+			domain_ss3: domain-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x0200c354>;
+				entry-latency-us = <2800>;
+				exit-latency-us = <4400>;
+				min-residency-us = <19012>;
+			};
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			cluster0_1_c4: cpu-sleep-0 {
+				compatible = "arm,idle-state";
+				idle-state-name = "retention";
+				arm,psci-suspend-param = <0x00000004>;
+				entry-latency-us = <93>;
+				exit-latency-us = <129>;
+				min-residency-us = <1925>;
+			};
+
+			cluster2_c4: cpu-sleep-1 {
+				compatible = "arm,idle-state";
+				idle-state-name = "retention";
+				entry-latency-us = <172>;
+				exit-latency-us = <130>;
+				min-residency-us = <2356>;
+				arm,psci-suspend-param = <0x00000004>;
+			};
+		};
+	};
+
+	firmware {
+		scm {
+			compatible = "qcom,scm-hawi", "qcom,scm";
+			qcom,dload-mode = <&tcsr 0x2000>;
+			interconnects = <&aggre_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			sram = <&minidump_config>;
+		};
+
+		scmi {
+			compatible = "arm,scmi";
+			mboxes = <&pdp0_mbox 0>, <&pdp0_mbox 1>;
+			mbox-names = "tx", "rx";
+			shmem = <&pdp_tx>, <&pdp_rx>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			scmi_perf: protocol@13 {
+				reg = <0x13>;
+				#power-domain-cells = <1>;
+			};
+		};
+	};
+
+	clk_virt: interconnect-0 {
+		compatible = "qcom,hawi-clk-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	mc_virt: interconnect-1 {
+		compatible = "qcom,hawi-mc-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	memory@a0000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the size */
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	pmu {
+		compatible = "arm,armv8-pmuv3";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+
+		cpu_pd0: power-domain-cpu0 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_1_c4>;
+		};
+
+		cpu_pd1: power-domain-cpu1 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_1_c4>;
+		};
+
+		cpu_pd2: power-domain-cpu2 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_1_c4>;
+		};
+
+		cpu_pd3: power-domain-cpu3 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_1_c4>;
+		};
+
+		cpu_pd4: power-domain-cpu4 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_1_c4>;
+		};
+
+		cpu_pd5: power-domain-cpu5 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster0_1_c4>;
+		};
+
+		cpu_pd6: power-domain-cpu6 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster2_c4>;
+		};
+
+		cpu_pd7: power-domain-cpu7 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster_pd>;
+			domain-idle-states = <&cluster2_c4>;
+		};
+
+		cluster_pd: power-domain-cluster {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&cluster_cl5>,
+					     <&domain_ss3>;
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		cpucp_slc_mem_sub: cpucp@81230000 {
+			reg = <0x0 0x81230000 0x0 0x1000>;
+			no-map;
+		};
+
+		aop_cmd_db_mem: aop-cmd-db@81c60000 {
+			compatible = "qcom,cmd-db";
+			reg = <0x0 0x81c60000 0x0 0x20000>;
+			no-map;
+		};
+
+		pdp_ns_shared_mem: pdp-ns-shared@81f00000 {
+			reg = <0x0 0x81f00000 0x0 0x100000>;
+			no-map;
+		};
+
+		smem_mem: smem@82700000 {
+			compatible = "qcom,smem";
+			reg = <0x0 0x82700000 0x0 0x400000>;
+			hwlocks = <&tcsr_mutex 3>;
+			no-map;
+		};
+
+		dsm_partition_1_mem: dsm-partition-1@83500000 {
+			reg = <0x0 0x83500000 0x0 0x5180000>;
+			no-map;
+		};
+
+		dsm_partition_2_mem: dsm-partition-2@88680000 {
+			reg = <0x0 0x88680000 0x0 0x1700000>;
+			no-map;
+		};
+
+		mpss_mem: mpss@89e00000 {
+			reg = <0x0 0x89e00000 0x0 0xe600000>;
+			no-map;
+		};
+
+		q6_mpss_dtb_mem: q6-mpss-dtb@98400000 {
+			reg = <0x0 0x98400000 0x0 0x80000>;
+			no-map;
+		};
+
+		ipa_fw_mem: ipa-fw@99580000 {
+			reg = <0x0 0x99580000 0x0 0x10000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa-gsi@99590000 {
+			reg = <0x0 0x99590000 0x0 0xc000>;
+			no-map;
+		};
+
+		gpu_microcode_mem: gpu-microcode@9959c000 {
+			reg = <0x0 0x9959c000 0x0 0x4000>;
+			no-map;
+		};
+
+		camera_mem: camera@99600000 {
+			reg = <0x0 0x99600000 0x0 0x800000>;
+			no-map;
+		};
+
+		camera_2_mem: camera-2@99e00000 {
+			reg = <0x0 0x99e00000 0x0 0x800000>;
+			no-map;
+		};
+
+		video_mem: video@9a600000 {
+			reg = <0x0 0x9a600000 0x0 0x800000>;
+			no-map;
+		};
+
+		cvp_mem: cvp@9ae00000 {
+			reg = <0x0 0x9ae00000 0x0 0x800000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@9b900000 {
+			reg = <0x0 0x9b900000 0x0 0x2000000>;
+			no-map;
+		};
+
+		q6_cdsp_dtb_mem: q6-cdsp-dtb@9d900000 {
+			reg = <0x0 0x9d900000 0x0 0x80000>;
+			no-map;
+		};
+
+		dcp_dtb_mem: dcp-dtb@a0000000 {
+			reg = <0x0 0xa0000000 0x0 0x80000>;
+			no-map;
+		};
+
+		dcp_mem: dcp@a0080000 {
+			reg = <0x0 0xa0080000 0x0 0x400000>;
+			no-map;
+		};
+
+		soccp_mem: soccp@a0480000 {
+			reg = <0x0 0xa0480000 0x0 0x500000>;
+			no-map;
+		};
+
+		soccp_dtb_mem: soccp-dtb@a0980000 {
+			reg = <0x0 0xa0980000 0x0 0x40000>;
+			no-map;
+		};
+
+		q6_adsp_dtb_mem: q6-adsp-dtb@a1380000 {
+			reg = <0x0 0xa1380000 0x0 0x80000>;
+			no-map;
+		};
+
+		adspslpi_mem: adspslpi@a1400000 {
+			reg = <0x0 0xa1400000 0x0 0x5f00000>;
+			no-map;
+		};
+
+		mpss_crashdump_overlay_mem: mpss-crashdump-overlay@bd500000 {
+			reg = <0x0 0xbd500000 0x0 0xf00000>;
+			no-map;
+		};
+	};
+
+	smp2p-adsp {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_LPASS
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_LPASS
+				IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <443>, <429>;
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <2>;
+
+		smp2p_adsp_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		smp2p_adsp_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	smp2p-cdsp {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_CDSP
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_CDSP
+				IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <94>, <432>;
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <5>;
+
+		smp2p_cdsp_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		smp2p_cdsp_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	smp2p-modem {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_MPSS
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_MPSS
+				IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <435>, <428>;
+
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <1>;
+
+		smp2p_modem_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		smp2p_modem_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	smp2p-soccp {
+		compatible = "qcom,smp2p";
+
+		interrupts-extended = <&ipcc IPCC_MPROC_SOCCP
+					     IPCC_MPROC_SIGNAL_SMP2P
+					     IRQ_TYPE_EDGE_RISING>;
+
+		mboxes = <&ipcc IPCC_MPROC_SOCCP
+				IPCC_MPROC_SIGNAL_SMP2P>;
+
+		qcom,smem = <617>, <616>;
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <19>;
+
+		soccp_smp2p_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		soccp_smp2p_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
+	soc@0 {
+		compatible = "simple-bus";
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+		ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+
+		gcc: clock-controller@100000 {
+			compatible = "qcom,hawi-gcc";
+			reg = <0x0 0x00100000 0x0 0x1f4200>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>,
+				 <&pcie0_phy>,
+				 <&pcie1_phy>,
+				 <&ufs_mem_phy 0>,
+				 <&ufs_mem_phy 1>,
+				 <&ufs_mem_phy 2>,
+				 <&usb_dp_qmpphy QMP_USB43DP_USB3_PIPE_CLK>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		gpi_dma2: dma-controller@800000 {
+			compatible = "qcom,hawi-gpi-dma", "qcom,sm6350-gpi-dma";
+			reg = <0x0 0x00800000 0x0 0x60000>;
+
+			interrupts = <GIC_ESPI 343 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 344 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 345 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 346 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 347 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 912 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 913 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 914 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 915 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 916 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 917 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 918 IRQ_TYPE_LEVEL_HIGH>;
+
+			dma-channels = <12>;
+			dma-channel-mask = <0x1f>;
+			#dma-cells = <3>;
+
+			iommus = <&apps_smmu 0x36 0x0>;
+			dma-coherent;
+		};
+
+		qupv3_2: geniqup@8c0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x008c0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_WRAP_2_M_AHB_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_2_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+
+			iommus = <&apps_smmu 0x23 0x0>;
+
+			dma-coherent;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			i2c8: i2c@880000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 417 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma2 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c8_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi8: spi@880000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 417 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma2 0 0 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 0 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi8_data_clk>, <&qup_spi8_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart8: serial@880000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 417 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart8_default>, <&qup_uart8_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c9: i2c@884000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 418 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma2 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c9_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi9: spi@884000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 418 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma2 0 1 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 1 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi9_data_clk>, <&qup_spi9_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart9: serial@884000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 418 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart9_default>, <&qup_uart9_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c10: i2c@888000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 419 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma2 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c10_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi10: spi@888000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 419 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma2 0 2 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 2 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi10_data_clk>, <&qup_spi10_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart10: serial@888000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 419 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart10_default>, <&qup_uart10_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c11: i2c@88c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 420 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma2 0 3 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 3 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c11_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi11: spi@88c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 420 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma2 0 3 QCOM_GPI_SPI>,
+				       <&gpi_dma2 1 3 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi11_data_clk>, <&qup_spi11_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart11: serial@88c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 420 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart11_default>, <&qup_uart11_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c12: i2c@890000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00890000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 421 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma2 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma2 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c12_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart12: serial@890000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x00890000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 421 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP2_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart12_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+		};
+
+		i2c_master_hub: geniqup@9c0000 {
+			compatible = "qcom,geni-se-i2c-master-hub";
+			reg = <0x0 0x009c0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_I2C_S_AHB_CLK>;
+			clock-names = "s-ahb";
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			status = "disabled";
+
+			i2c_hub_0: i2c@980000 {
+				compatible = "qcom,geni-i2c-master-hub";
+				reg = <0x0 0x00980000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 528 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_I2C_S0_CLK>,
+					 <&gcc GCC_QUPV3_I2C_CORE_CLK>;
+				clock-names = "se",
+					      "core";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_I2C QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&hub_i2c0_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c_hub_1: i2c@984000 {
+				compatible = "qcom,geni-i2c-master-hub";
+				reg = <0x0 0x00984000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 529 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_I2C_S1_CLK>,
+					 <&gcc GCC_QUPV3_I2C_CORE_CLK>;
+				clock-names = "se",
+					      "core";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_I2C QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&hub_i2c1_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c_hub_2: i2c@988000 {
+				compatible = "qcom,geni-i2c-master-hub";
+				reg = <0x0 0x00988000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 530 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_I2C_S2_CLK>,
+					 <&gcc GCC_QUPV3_I2C_CORE_CLK>;
+				clock-names = "se",
+					      "core";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_I2C QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&hub_i2c2_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c_hub_3: i2c@98c000 {
+				compatible = "qcom,geni-i2c-master-hub";
+				reg = <0x0 0x0098c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 531 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_I2C_S3_CLK>,
+					 <&gcc GCC_QUPV3_I2C_CORE_CLK>;
+				clock-names = "se",
+					      "core";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_I2C QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&hub_i2c3_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			i2c_hub_4: i2c@990000 {
+				compatible = "qcom,geni-i2c-master-hub";
+				reg = <0x0 0x00990000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 532 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_I2C_S4_CLK>,
+					 <&gcc GCC_QUPV3_I2C_CORE_CLK>;
+				clock-names = "se",
+					      "core";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_I2C QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&hub_i2c4_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+		};
+
+		gpi_dma1: dma-controller@a00000 {
+			compatible = "qcom,hawi-gpi-dma", "qcom,sm6350-gpi-dma";
+			reg = <0x0 0x00a00000 0x0 0x60000>;
+
+			interrupts = <GIC_ESPI 900 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 901 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 902 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 903 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 904 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 905 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 906 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 907 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 908 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 909 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 910 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 911 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 944 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 945 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 946 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 947 IRQ_TYPE_LEVEL_HIGH>;
+
+			dma-channels = <16>;
+			dma-channel-mask = <0x7f>;
+			#dma-cells = <3>;
+
+			iommus = <&apps_smmu 0xb6 0x0>;
+			dma-coherent;
+		};
+
+		qupv3_1: geniqup@ac0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x00ac0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_WRAP_1_M_AXI_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_1_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+
+			iommus = <&apps_smmu 0xa3 0x0>;
+
+			dma-coherent;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			i2c0: i2c@a80000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 892 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c0_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi0: spi@a80000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 892 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma1 0 0 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 0 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi0_data_clk>, <&qup_spi0_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart0: serial@a80000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 892 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart0_default>, <&qup_uart0_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c1: i2c@a84000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 893 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c1_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi1: spi@a84000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 893 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma1 0 1 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 1 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi1_data_clk>, <&qup_spi1_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart1: serial@a84000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 893 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart1_default>, <&qup_uart1_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c2: i2c@a88000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a88000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 894 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c2_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi2: spi@a88000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a88000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 894 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma1 0 2 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 2 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi2_data_clk>, <&qup_spi2_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart2: serial@a88000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a88000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 894 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart2_default>, <&qup_uart2_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c3: i2c@a8c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a8c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 895 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma1 0 3 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 3 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c3_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi3: spi@a8c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a8c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 895 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma1 0 3 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 3 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi3_data_clk>, <&qup_spi3_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart3: serial@a8c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a8c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 895 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart3_default>, <&qup_uart3_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c4: i2c@a90000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 896 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c4_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi4: spi@a90000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 896 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma1 0 4 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 4 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi4_data_clk>, <&qup_spi4_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart4: serial@a90000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 896 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart4_default>, <&qup_uart4_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c5: i2c@a94000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 897 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma1 0 5 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 5 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c5_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi5: spi@a94000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 897 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma1 0 5 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 5 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi5_data_clk>, <&qup_spi5_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart5: serial@a94000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 897 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S5_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart5_default>, <&qup_uart5_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c6: i2c@a98000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 898 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma1 0 6 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 6 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c6_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi6: spi@a98000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 898 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma1 0 6 QCOM_GPI_SPI>,
+				       <&gpi_dma1 1 6 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi6_data_clk>, <&qup_spi6_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart6: serial@a98000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 898 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart6_default>, <&qup_uart6_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			uart7: serial@a9c000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x00a9c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 899 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP1_S7_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart7_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+		};
+
+		remoteproc_soccp: remoteproc@d00000 {
+			compatible = "qcom,hawi-soccp-pas", "qcom,kaanapali-soccp-pas";
+			reg = <0x0 0x00d00000 0x0 0x200000>;
+
+			interrupts-extended = <&intc GIC_ESPI 231 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&soccp_smp2p_in 9 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "pong";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MX>;
+			power-domain-names = "cx",
+					     "mx";
+
+			memory-region = <&soccp_mem>,
+					<&soccp_dtb_mem>;
+
+			qcom,smem-states = <&soccp_smp2p_out 0>,
+					   <&soccp_smp2p_out 8>;
+			qcom,smem-state-names = "stop",
+						"ping";
+
+			glink-edge {
+				interrupts-extended = <&ipcc IPCC_MPROC_SOCCP
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+
+				mboxes = <&ipcc IPCC_MPROC_SOCCP
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+				qcom,remote-pid = <19>;
+				label = "soccp";
+			};
+		};
+
+		aggre_noc: interconnect@f00000 {
+			compatible = "qcom,hawi-aggre1-noc";
+			reg = <0x0 0x00f00000 0x0 0x54400>;
+			#interconnect-cells = <2>;
+			clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
+				 <&rpmhcc RPMH_IPA_CLK>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		usb_1_hsphy: phy@fe3000 {
+			compatible = "qcom,hawi-m31-eusb2-phy", "qcom,sm8750-m31-eusb2-phy";
+			reg = <0x0 0x00fe3000 0x0 0x29c>;
+
+			clocks = <&tcsrcc TCSR_USB2_CLKREF_EN>;
+			clock-names = "ref";
+
+			resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		usb_dp_qmpphy: phy@fe8000 {
+			compatible = "qcom,hawi-qmp-usb3-dp-phy";
+			reg = <0x0 0x00fe8000 0x0 0x4000>;
+
+			clocks = <&gcc GCC_USB3_PRIM_PHY_AUX_CLK>,
+				 <&tcsrcc TCSR_USB3_CLKREF_EN>,
+				 <&gcc GCC_USB3_PRIM_PHY_COM_AUX_CLK>,
+				 <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>;
+			clock-names = "aux",
+				      "ref",
+				      "com_aux",
+				      "usb3_pipe";
+
+			resets = <&gcc GCC_USB3_PHY_PRIM_BCR>,
+				 <&gcc GCC_USB3_DP_PHY_PRIM_BCR>;
+			reset-names = "phy",
+				      "common";
+
+			power-domains = <&gcc GCC_USB3_PHY_GDSC>;
+
+			#clock-cells = <1>;
+			#phy-cells = <1>;
+
+			orientation-switch;
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_dp_qmpphy_out: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_dp_qmpphy_usb_ss_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_ss>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					usb_dp_qmpphy_dp_in: endpoint {
+					};
+				};
+			};
+		};
+
+		rng@10c3000 {
+			compatible = "qcom,hawi-trng", "qcom,trng";
+			reg = <0x0 0x010c3000 0x0 0x1000>;
+		};
+
+		ipcc: mailbox@1106000 {
+			compatible = "qcom,hawi-ipcc", "qcom,ipcc";
+			reg = <0x0 0x01106000 0x0 0x1000>;
+
+			interrupts = <GIC_ESPI 310 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+
+			#mbox-cells = <2>;
+		};
+
+		cnoc_main: interconnect@1500000 {
+			compatible = "qcom,hawi-cnoc-main";
+			reg = <0x0 0x01500000 0x0 0x20000>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		system_noc: interconnect@1680000 {
+			compatible = "qcom,hawi-system-noc";
+			reg = <0x0 0x01680000 0x0 0x20080>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		pcie_anoc: interconnect@16c0000 {
+			compatible = "qcom,hawi-pcie-anoc";
+			reg = <0x0 0x016c0000 0x0 0x12400>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+			clocks = <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				 <&gcc GCC_CFG_NOC_PCIE_ANOC_AHB_CLK>;
+		};
+
+		cambistmclkcc: clock-controller@1760000 {
+			compatible = "qcom,hawi-cambistmclkcc",
+				     "qcom,kaanapali-cambistmclkcc";
+			reg = <0x0 0x01760000 0x0 0x6000>;
+
+			clocks = <&gcc GCC_CAM_BIST_MCLK_AHB_CLK>,
+				 <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>;
+
+			power-domains = <&rpmhpd RPMHPD_MMCX>,
+					<&rpmhpd RPMHPD_MX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		mmss_noc: interconnect@1780000 {
+			compatible = "qcom,hawi-mmss-noc";
+			reg = <0x0 0x01780000 0x0 0x5f800>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		gpi_dma3: dma-controller@1900000 {
+			compatible = "qcom,hawi-gpi-dma", "qcom,sm6350-gpi-dma";
+			reg = <0x0 0x01900000 0x0 0x60000>;
+
+			interrupts = <GIC_ESPI 652 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 653 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 654 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 655 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 656 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 657 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 658 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 659 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 660 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 661 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 662 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 663 IRQ_TYPE_LEVEL_HIGH>;
+
+			dma-channels = <12>;
+			dma-channel-mask = <0x1f>;
+			#dma-cells = <3>;
+
+			iommus = <&apps_smmu 0xd6 0x0>;
+			dma-coherent;
+		};
+
+		qupv3_3: geniqup@19c0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x019c0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_WRAP_3_M_AHB_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_3_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+
+			iommus = <&apps_smmu 0xc3 0x0>;
+
+			dma-coherent;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			i2c13: i2c@1980000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01980000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 437 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_3 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma3 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma3 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c13_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart13: serial@1980000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x01980000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 437 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart13_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c14: i2c@1984000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01984000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 647 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_3 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma3 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma3 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c14_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi14: spi@1984000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x01984000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 647 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma3 0 1 QCOM_GPI_SPI>,
+				       <&gpi_dma3 1 1 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi14_data_clk>, <&qup_spi14_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart14: serial@1984000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x01984000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 647 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart14_default>, <&qup_uart14_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c15: i2c@1988000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01988000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 648 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_3 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma3 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma3 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c15_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi15: spi@1988000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x01988000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 648 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma3 0 2 QCOM_GPI_SPI>,
+				       <&gpi_dma3 1 2 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi15_data_clk>, <&qup_spi15_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart15: serial@1988000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x01988000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 648 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart15_default>, <&qup_uart15_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c16: i2c@198c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x0198c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 649 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_3 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma3 0 3 QCOM_GPI_I2C>,
+				       <&gpi_dma3 1 3 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c16_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi16: spi@198c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x0198c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 649 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma3 0 3 QCOM_GPI_SPI>,
+				       <&gpi_dma3 1 3 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi16_data_clk>, <&qup_spi16_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart16: serial@198c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x0198c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 649 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart16_default>, <&qup_uart16_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c17: i2c@1990000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01990000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 650 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_3 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma3 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma3 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c17_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi17: spi@1990000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x01990000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 650 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma3 0 4 QCOM_GPI_SPI>,
+				       <&gpi_dma3 1 4 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi17_data_clk>, <&qup_spi17_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart17: serial@1990000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x01990000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 650 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart17_default>, <&qup_uart17_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			uart18: serial@1994000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x01994000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 651 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP3_S5_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_3 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart18_default>, <&qup_uart18_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+		};
+
+		gpi_dma4: dma-controller@1a00000 {
+			compatible = "qcom,hawi-gpi-dma", "qcom,sm6350-gpi-dma";
+			reg = <0x0 0x01a00000 0x0 0x60000>;
+
+			interrupts = <GIC_ESPI 568 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 569 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 570 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 571 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 572 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 573 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 574 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 575 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 576 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 577 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 578 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 579 IRQ_TYPE_LEVEL_HIGH>;
+
+			dma-channels = <12>;
+			dma-channel-mask = <0x1f>;
+			#dma-cells = <3>;
+
+			iommus = <&apps_smmu 0x136 0x0>;
+			dma-coherent;
+		};
+
+		qupv3_4: geniqup@1ac0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x01ac0000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_QUPV3_WRAP_4_M_AHB_CLK>,
+				 <&gcc GCC_QUPV3_WRAP_4_S_AHB_CLK>;
+			clock-names = "m-ahb",
+				      "s-ahb";
+
+			iommus = <&apps_smmu 0x123 0x0>;
+
+			dma-coherent;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			i2c19: i2c@1a80000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01a80000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 920 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_4 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma4 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma4 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c19_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi19: spi@1a80000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x01a80000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 920 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma4 0 0 QCOM_GPI_SPI>,
+				       <&gpi_dma4 1 0 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi19_data_clk>, <&qup_spi19_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart19: serial@1a80000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x01a80000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 920 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S0_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart19_default>, <&qup_uart19_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c20: i2c@1a84000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01a84000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 921 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_4 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma4 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma4 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c20_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi20: spi@1a84000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x01a84000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 921 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma4 0 1 QCOM_GPI_SPI>,
+				       <&gpi_dma4 1 1 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi20_data_clk>, <&qup_spi20_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart20: serial@1a84000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x01a84000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 921 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S1_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart20_default>, <&qup_uart20_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c21: i2c@1a88000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01a88000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 922 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_4 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma4 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma4 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c21_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi21: spi@1a88000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x01a88000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 922 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				dmas = <&gpi_dma4 0 2 QCOM_GPI_SPI>,
+				       <&gpi_dma4 1 2 QCOM_GPI_SPI>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_spi21_data_clk>, <&qup_spi21_cs>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart21: serial@1a88000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x01a88000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 922 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S2_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart21_default>, <&qup_uart21_cts_rts>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c22: i2c@1a8c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01a8c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 923 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_4 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma4 0 3 QCOM_GPI_I2C>,
+				       <&gpi_dma4 1 3 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c22_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart22: serial@1a8c000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x01a8c000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 923 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S3_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart22_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+
+			i2c23: i2c@1a90000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x01a90000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 924 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>,
+						<&aggre_noc MASTER_QUP_4 QCOM_ICC_TAG_ALWAYS
+						 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+				interconnect-names = "qup-core",
+						     "qup-config",
+						     "qup-memory";
+
+				dmas = <&gpi_dma4 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma4 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx",
+					    "rx";
+
+				pinctrl-0 = <&qup_i2c23_data_clk>;
+				pinctrl-names = "default";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart23: serial@1a90000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x01a90000 0x0 0x4000>;
+
+				interrupts = <GIC_ESPI 924 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GCC_QUPV3_WRAP4_S4_CLK>;
+				clock-names = "se";
+
+				interconnects = <&clk_virt MASTER_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS
+						 &clk_virt SLAVE_QUP_CORE_4 QCOM_ICC_TAG_ALWAYS>,
+						<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+						 &config_noc SLAVE_QUP_4 QCOM_ICC_TAG_ACTIVE_ONLY>;
+				interconnect-names = "qup-core",
+						     "qup-config";
+
+				pinctrl-0 = <&qup_uart23_default>;
+				pinctrl-names = "default";
+
+				status = "disabled";
+			};
+		};
+
+		pcie0: pcie@1c00000 {
+			device_type = "pci";
+			compatible = "qcom,hawi-pcie";
+			reg = <0x0 0x01c00000 0x0 0x3000>,
+			      <0x0 0x40000000 0x0 0xf1d>,
+			      <0x0 0x40000f20 0x0 0xa8>,
+			      <0x0 0x40001000 0x0 0x1000>,
+			      <0x0 0x40100000 0x0 0x100000>,
+			      <0x0 0x01c03000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x23d00000>;
+
+			interrupts = <GIC_ESPI 205 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 206 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 207 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 208 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 209 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 210 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 211 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 212 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 204 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				 <&gcc GCC_CNOC_PCIE_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "noc_aggr",
+				      "cnoc_sf_axi";
+
+			resets = <&gcc GCC_PCIE_0_BCR>,
+				 <&gcc GCC_PCIE_0_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			interconnects = <&pcie_anoc MASTER_PCIE_0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc_main SLAVE_PCIE_0 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			power-domains = <&gcc GCC_PCIE_0_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
+
+			operating-points-v2 = <&pcie0_opp_table>;
+
+			iommu-map = <0 &apps_smmu 0x1000 0x0 0x1>,
+				    <0x100 &apps_smmu 0x1001 0x0 0x1>;
+
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_ESPI 213 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_ESPI 214 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_ESPI 215 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_ESPI 216 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			#interrupt-cells = <1>;
+
+			msi-map = <0x0 &gic_its 0x1000 0x1>,
+				  <0x100 &gic_its 0x1001 0x1>;
+			msi-map-mask = <0xff00>;
+			linux,pci-domain = <0>;
+			bus-range = <0x00 0xff>;
+
+			pinctrl-0 = <&pcie0_default_state>;
+			pinctrl-names = "default";
+
+			dma-coherent;
+
+			status = "disabled";
+
+			pcie0_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* 2.5 GT/s */
+				opp-2500000 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs_d1>;
+					opp-peak-kBps = <250000 1>;
+				};
+
+				/* 5.0 GT/s */
+				opp-5000000 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs_d1>;
+					opp-peak-kBps = <500000 1>;
+				};
+
+				/* 8.0 GT/s */
+				opp-8000000 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+				};
+
+				/* 10 GT/s */
+				opp-10000000 {
+					opp-hz = /bits/ 64 <10000000>;
+					required-opps = <&rpmhpd_opp_low_svs_d1>;
+					opp-peak-kBps = <1000000 1>;
+				};
+
+				/* 16 GT/s */
+				opp-16000000 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <1969000 1>;
+				};
+			};
+
+			pcie0_port0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+
+				phys = <&pcie0_phy>;
+			};
+		};
+
+		pcie0_phy: phy@1c06000 {
+			compatible = "qcom,hawi-qmp-gen3x2-pcie-phy";
+			reg = <0x0 0x01c06000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&tcsrcc TCSR_PCIE_0_CLKREF_EN>,
+				 <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_0_PIPE_CLK>,
+				 <&gcc GCC_PCIE_0_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			resets = <&gcc GCC_PCIE_0_PHY_BCR>,
+				 <&gcc GCC_PCIE_0_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			power-domains = <&gcc GCC_PCIE_0_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie0_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie1: pcie@1c18000 {
+			device_type = "pci";
+			compatible = "qcom,hawi-pcie";
+			reg = <0x0 0x01c18000 0x0 0x3000>,
+			      <0x0 0x64000000 0x0 0xf1d>,
+			      <0x0 0x64000f20 0x0 0xa8>,
+			      <0x0 0x64001000 0x0 0x1000>,
+			      <0x0 0x64100000 0x0 0x100000>,
+			      <0x0 0x01c1b000 0x0 0x1000>;
+			reg-names = "parf",
+				    "dbi",
+				    "elbi",
+				    "atu",
+				    "config",
+				    "mhi";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x64200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x64300000 0x0 0x64300000 0x0 0x1bd00000>;
+
+			interrupts = <GIC_ESPI 98 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 99 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 101 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 102 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 95 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
+				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_1_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_AXI_CLK>,
+				 <&gcc GCC_CNOC_PCIE_SF_AXI_CLK>;
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a",
+				      "noc_aggr",
+				      "cnoc_sf_axi";
+
+			resets = <&gcc GCC_PCIE_1_BCR>,
+				 <&gcc GCC_PCIE_1_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			interconnects = <&pcie_anoc MASTER_PCIE_1 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc_main SLAVE_PCIE_1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "pcie-mem",
+					     "cpu-pcie";
+
+			power-domains = <&gcc GCC_PCIE_1_GDSC>;
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
+
+			operating-points-v2 = <&pcie1_opp_table>;
+
+			iommu-map = <0 &apps_smmu 0x1080 0x0 0x1>,
+				    <0x100 &apps_smmu 0x1081 0x0 0x1>;
+
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_ESPI 106 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_ESPI 107 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_ESPI 108 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_ESPI 109 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			#interrupt-cells = <1>;
+
+			msi-map = <0x0 &gic_its 0x1080 0x1>,
+				  <0x100 &gic_its 0x1081 0x1>;
+			msi-map-mask = <0xff00>;
+			linux,pci-domain = <1>;
+			bus-range = <0x00 0xff>;
+
+			pinctrl-0 = <&pcie1_default_state>;
+			pinctrl-names = "default";
+
+			dma-coherent;
+
+			status = "disabled";
+
+			pcie1_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/* 2.5 GT/s */
+				opp-2500000 {
+					opp-hz = /bits/ 64 <2500000>;
+					required-opps = <&rpmhpd_opp_low_svs_d1>;
+					opp-peak-kBps = <250000 1>;
+				};
+
+				/* 5.0 GT/s */
+				opp-5000000 {
+					opp-hz = /bits/ 64 <5000000>;
+					required-opps = <&rpmhpd_opp_low_svs_d1>;
+					opp-peak-kBps = <500000 1>;
+				};
+
+				/* 8.0 GT/s */
+				opp-8000000 {
+					opp-hz = /bits/ 64 <8000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+					opp-peak-kBps = <984500 1>;
+				};
+
+				/* 16 GT/s */
+				opp-16000000 {
+					opp-hz = /bits/ 64 <16000000>;
+					required-opps = <&rpmhpd_opp_svs>;
+					opp-peak-kBps = <1969000 1>;
+				};
+			};
+
+			pcie1_port0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+
+				phys = <&pcie1_phy>;
+			};
+		};
+
+		pcie1_phy: phy@1c1e000 {
+			compatible = "qcom,hawi-qmp-gen4x1-pcie-phy";
+			reg = <0x0 0x01c1e000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>,
+				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
+				 <&tcsrcc TCSR_PCIE_1_CLKREF_EN>,
+				 <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_1_PIPE_CLK>,
+				 <&gcc GCC_PCIE_1_PIPE_DIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			resets = <&gcc GCC_PCIE_1_PHY_BCR>,
+				 <&gcc GCC_PCIE_1_NOCSR_COM_PHY_BCR>;
+			reset-names = "phy",
+				      "phy_nocsr";
+
+			power-domains = <&gcc GCC_PCIE_1_PHY_GDSC>;
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie1_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		sdhc_2: mmc@1c44000 {
+			compatible = "qcom,hawi-sdhci", "qcom,sdhci-msm-v5";
+			reg = <0x0 0x01c44000 0x0 0x1000>;
+
+			interrupts = <GIC_ESPI 271 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 287 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hc_irq", "pwr_irq";
+
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK>;
+
+			clock-names = "iface", "core", "xo";
+
+			interconnects = <&aggre_noc MASTER_SDCC_2 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_SDCC_2 QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			interconnect-names = "sdhc-ddr", "cpu-sdhc";
+
+			power-domains = <&rpmhpd RPMHPD_CX>;
+			operating-points-v2 = <&sdhc2_opp_table>;
+
+			iommus = <&apps_smmu 0x140 0x0>;
+			dma-coherent;
+
+			resets = <&gcc GCC_SDCC2_BCR>;
+
+			bus-width = <4>;
+			max-sd-hs-hz = <37500000>;
+
+			qcom,dll-config = <0x0007442c>;
+			qcom,ddr-config = <0x80040868>;
+
+			status = "disabled";
+
+			sdhc2_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>;
+					opp-peak-kBps = <160000 100000>;
+					opp-avg-kBps = <50000 0>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+				};
+
+				opp-202000000 {
+					opp-hz = /bits/ 64 <202000000>;
+					opp-peak-kBps = <200000 120000>;
+					opp-avg-kBps = <104000 0>;
+					required-opps = <&rpmhpd_opp_svs_l1>;
+				};
+			};
+		};
+
+		ufs_mem_phy: phy@1d80000 {
+			compatible = "qcom,hawi-qmp-ufs-phy";
+			reg = <0x0 0x01d80000 0x0 0x4000>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&gcc GCC_UFS_PHY_PHY_AUX_CLK>,
+				 <&tcsrcc TCSR_UFS_CLKREF_EN>;
+
+			clock-names = "ref",
+				      "ref_aux",
+				      "qref";
+
+			resets = <&ufs_mem_hc 0>;
+			reset-names = "ufsphy";
+
+			power-domains = <&gcc GCC_UFS_MEM_PHY_GDSC>;
+
+			#clock-cells = <1>;
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		ufs_mem_hc: ufshc@1d84000 {
+			compatible = "qcom,hawi-ufshc",
+				     "qcom,ufshc",
+				     "jedec,ufs-2.0";
+			reg = <0x0 0x01d84000 0x0 0x3000>;
+
+			interrupts = <GIC_ESPI 329 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
+				 <&gcc GCC_UFS_PHY_AHB_CLK>,
+				 <&gcc GCC_UFS_PHY_UNIPRO_5_CORE_CLK>,
+				 <&rpmhcc RPMH_LN_BB_CLK3>,
+				 <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>,
+				 <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>,
+				 <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>;
+
+			clock-names = "core_clk",
+				      "bus_aggr_clk",
+				      "iface_clk",
+				      "core_clk_unipro",
+				      "ref_clk",
+				      "tx_lane0_sync_clk",
+				      "rx_lane0_sync_clk",
+				      "rx_lane1_sync_clk";
+
+			operating-points-v2 = <&ufshc_opp_table>;
+
+			resets = <&gcc GCC_UFS_PHY_BCR>;
+			reset-names = "rst";
+
+			interconnects = <&aggre_noc MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "ufs-ddr",
+					     "cpu-ufs";
+
+			power-domains = <&gcc GCC_UFS_PHY_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
+
+			iommus = <&apps_smmu 0x60 0x0>;
+			dma-coherent;
+
+			lanes-per-direction = <2>;
+			limit-hs-gear = <5>;
+
+			phys = <&ufs_mem_phy>;
+			phy-names = "ufsphy";
+
+			qcom,ice = <&ice>;
+
+			#reset-cells = <1>;
+
+			status = "disabled";
+
+			ufshc_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <100000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+				};
+
+				opp-450000000 {
+					opp-hz = /bits/ 64 <403000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <403000000>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>,
+						 /bits/ 64 <0>;
+					required-opps = <&rpmhpd_opp_nom>;
+				};
+			};
+		};
+
+		ice: crypto@1d88000 {
+			compatible = "qcom,hawi-inline-crypto-engine",
+				     "qcom,inline-crypto-engine";
+			reg = <0x0 0x01d88000 0x0 0x18000>;
+
+			clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>,
+				 <&gcc GCC_UFS_PHY_AHB_CLK>;
+			clock-names = "core",
+				      "iface";
+
+			power-domains = <&gcc GCC_UFS_PHY_GDSC>;
+		};
+
+		videocc: clock-controller@20f0000 {
+			compatible = "qcom,hawi-videocc";
+			reg = <0x0 0x020f0000 0x0 0x10000>;
+			clocks = <&bi_tcxo_div2>,
+				 <&gcc GCC_VIDEO_AHB_CLK>;
+
+			power-domains = <&rpmhpd RPMHPD_MMCX>,
+					<&rpmhpd RPMHPD_MXC>;
+			required-opps = <&rpmhpd_opp_low_svs_d2>,
+					<&rpmhpd_opp_low_svs_d2>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		apps_smmu: iommu@2e00000 {
+			compatible = "qcom,hawi-smmu-500", "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x02e00000 0x0 0x100000>;
+
+			interrupts = <GIC_ESPI 128 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 161 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 162 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 163 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 164 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 165 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 166 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 167 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 168 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 169 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 170 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 171 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 172 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 173 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 175 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 176 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 177 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 178 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 179 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 180 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 182 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 245 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 246 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 247 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 248 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 249 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 250 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 251 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 252 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 253 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 254 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 255 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 256 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 379 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 380 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 381 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 382 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 383 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 384 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 385 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 386 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 387 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 388 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 389 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 390 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 391 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 392 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 393 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 394 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 395 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 396 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 397 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 398 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 399 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 400 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 401 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 402 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 403 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 404 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 405 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 406 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 408 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 409 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 459 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 460 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 461 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 462 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 463 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 464 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 465 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 466 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 467 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 468 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 469 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 470 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 471 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 472 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 473 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 482 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 483 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 476 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 485 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 771 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 487 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 488 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 489 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 754 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 755 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 756 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 757 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 758 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 759 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 760 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 761 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 474 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 552 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 553 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 554 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 555 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 556 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 557 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 558 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 559 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 560 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 561 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 562 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 563 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 564 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 565 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 566 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 567 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 928 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 929 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 930 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 931 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 932 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 933 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 934 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 935 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 936 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 937 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 938 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 939 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 940 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 941 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 942 IRQ_TYPE_LEVEL_HIGH>;
+
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+
+			dma-coherent;
+		};
+
+		stdst_noc: interconnect@3000000 {
+			compatible = "qcom,hawi-stdst-main";
+			reg = <0x0 0x03000000 0x0 0x10000>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		config_noc: interconnect@3010000 {
+			compatible = "qcom,hawi-stdst-cfg";
+			reg = <0x0 0x03010000 0x0 0xb1000>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		tcsr: syscon@3a70000 {
+			compatible = "qcom,hawi-tcsr", "syscon";
+			reg = <0x0 0x03a70000 0x0 0x10000>;
+		};
+
+		tcsr_mutex: hwlock@3ad3000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x03ad3000 0x0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsrcc: clock-controller@3b13040 {
+			compatible = "qcom,hawi-tcsrcc", "syscon";
+			reg = <0x0 0x03b13040 0x0 0x50>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		gxclkctl: clock-controller@3d64000 {
+			compatible = "qcom,hawi-gxclkctl",
+				     "qcom,kaanapali-gxclkctl";
+			reg = <0x0 0x03d64000 0x0 0x6000>;
+
+			power-domains = <&rpmhpd RPMHPD_GFX>,
+					<&rpmhpd RPMHPD_GMXC>,
+					<&gpucc GPU_CC_CX_GDSC>;
+
+			#power-domain-cells = <1>;
+		};
+
+		gpucc: clock-controller@3d90000 {
+			compatible = "qcom,hawi-gpucc";
+			reg = <0x0 0x03d90000 0x0 0x9800>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MX>,
+					<&rpmhpd RPMHPD_GMXC>;
+			required-opps = <&rpmhpd_opp_low_svs_d3>,
+					<&rpmhpd_opp_low_svs_d3>,
+					<&rpmhpd_opp_low_svs_d3>;
+			clocks = <&bi_tcxo_div2>,
+				<&gcc GCC_GPU_GPLL0_CLK_SRC>,
+				<&gcc GCC_GPU_GPLL0_DIV_CLK_SRC>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		adreno_smmu: iommu@3da0000 {
+			compatible = "qcom,hawi-smmu-500", "qcom,adreno-smmu",
+				     "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x03da0000 0x0 0x40000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			interrupts = <GIC_ESPI 738 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 742 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 743 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 744 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 745 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 746 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 747 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 748 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 749 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 750 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 751 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 752 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 486 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 540 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 638 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 639 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 640 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 641 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 724 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 726 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 729 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 730 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 731 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 733 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 734 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 764 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gpucc GPU_CC_GPU_SMMU_VOTE_CLK>;
+			clock-names = "hlos";
+			power-domains = <&gpucc GPU_CC_CX_GDSC>;
+			dma-coherent;
+		};
+
+		remoteproc_mpss: remoteproc@4000000 {
+			compatible = "qcom,hawi-mpss-pas", "qcom,sm8650-mpss-pas";
+			reg = <0x0 0x04000000 0x0 0x10000>;
+
+			interrupts-extended = <&intc GIC_ESPI 328 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_modem_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_modem_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_modem_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_modem_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_modem_in 7 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "shutdown-ack";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			interconnects = <&mc_virt MASTER_LLCC QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MSS>;
+			power-domain-names = "cx",
+					     "mss";
+
+			memory-region = <&mpss_mem>, <&q6_mpss_dtb_mem>, <&dsm_partition_1_mem>,
+					<&dsm_partition_2_mem>, <&mpss_crashdump_overlay_mem>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			qcom,smem-states = <&smp2p_modem_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			glink-edge {
+				interrupts-extended = <&ipcc IPCC_MPROC_MPSS
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+
+				mboxes = <&ipcc IPCC_MPROC_MPSS
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+				qcom,remote-pid = <1>;
+				label = "mpss";
+			};
+		};
+
+		remoteproc_adsp: remoteproc@7000000 {
+			compatible = "qcom,hawi-adsp-pas", "qcom,sm8550-adsp-pas";
+			reg = <0x0 0x07000000 0x0 0x10000>;
+
+			interrupts-extended = <&pdc 6 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 7 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "shutdown-ack";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			interconnects = <&lpass_lpicx_noc MASTER_LPASS_PROC QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+
+			power-domains = <&rpmhpd RPMHPD_LCX>,
+					<&rpmhpd RPMHPD_LMX>;
+			power-domain-names = "lcx",
+					     "lmx";
+
+			memory-region = <&adspslpi_mem>, <&q6_adsp_dtb_mem>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			qcom,smem-states = <&smp2p_adsp_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			glink-edge {
+				interrupts-extended = <&ipcc IPCC_MPROC_LPASS
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+
+				mboxes = <&ipcc IPCC_MPROC_LPASS
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+				qcom,remote-pid = <2>;
+				label = "lpass";
+
+				fastrpc {
+					compatible = "qcom,hawi-fastrpc", "qcom,kaanapali-fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "adsp";
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+
+						iommus = <&apps_smmu 0x0c03 0x80>,
+							 <&apps_smmu 0x0c43 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@4 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <4>;
+
+						iommus = <&apps_smmu 0x0c04 0x80>,
+							 <&apps_smmu 0x0c44 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@5 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <5>;
+
+						iommus = <&apps_smmu 0x0c05 0x80>,
+							 <&apps_smmu 0x0c45 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@6 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <6>;
+
+						iommus = <&apps_smmu 0x0c06 0x80>,
+							 <&apps_smmu 0x0c46 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@7 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <7>;
+
+						iommus = <&apps_smmu 0x0c07 0x80>,
+							 <&apps_smmu 0x0c47 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@8 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <8>;
+
+						iommus = <&apps_smmu 0x0c08 0x80>,
+							 <&apps_smmu 0x0c48 0x20>;
+						dma-coherent;
+					};
+				};
+			};
+		};
+
+		lpass_lpiaon_noc: interconnect@7c00000 {
+			compatible = "qcom,hawi-lpass-lpiaon-noc";
+			reg = <0x0 0x07c00000 0x0 0x19080>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		lpass_lpicx_noc: interconnect@7c20000 {
+			compatible = "qcom,hawi-lpass-lpicx-noc";
+			reg = <0x0 0x07c20000 0x0 0x46080>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		lpass_ag_noc: interconnect@8f40000 {
+			compatible = "qcom,hawi-lpass-ag-noc";
+			reg = <0x0 0x08f40000 0x0 0xc080>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		camcc: clock-controller@956d000 {
+			compatible = "qcom,hawi-camcc";
+			reg = <0x0 0x0956d000 0x0 0x80000>;
+
+			clocks = <&gcc GCC_CAMERA_AHB_CLK>,
+				 <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>;
+
+			power-domains = <&rpmhpd RPMHPD_MMCX>,
+					<&rpmhpd RPMHPD_MXC>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			interconnects = <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS
+					 &mmss_noc SLAVE_MNOC_HF_MEM_NOC QCOM_ICC_TAG_ALWAYS>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		usb_1: usb@a600000 {
+			compatible = "qcom,hawi-dwc3", "qcom,snps-dwc3";
+			reg = <0x0 0x0a600000 0x0 0xfc100>;
+
+			clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB30_PRIM_MASTER_CLK>,
+				 <&gcc GCC_AGGRE_STARDUSTNOC_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB30_PRIM_SLEEP_CLK>,
+				 <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>;
+			clock-names = "cfg_noc",
+				      "core",
+				      "iface",
+				      "sleep",
+				      "mock_utmi";
+
+			assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_PRIM_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <200000000>;
+
+			interrupts-extended = <&intc GIC_ESPI 196 IRQ_TYPE_LEVEL_HIGH>,
+					      <&intc GIC_ESPI 194 IRQ_TYPE_LEVEL_HIGH>,
+					      <&intc GIC_ESPI 195 IRQ_TYPE_LEVEL_HIGH>,
+					      <&pdc 14 IRQ_TYPE_EDGE_BOTH>,
+					      <&pdc 15 IRQ_TYPE_EDGE_BOTH>,
+					      <&pdc 17 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc_usb3",
+					  "pwr_event",
+					  "hs_phy_irq",
+					  "dp_hs_phy_irq",
+					  "dm_hs_phy_irq",
+					  "ss_phy_irq";
+
+			power-domains = <&gcc GCC_USB30_PRIM_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
+
+			resets = <&gcc GCC_USB30_PRIM_BCR>;
+
+			interconnects = <&aggre_noc MASTER_USB3 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					 <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_USB3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "usb-ddr",
+					     "apps-usb";
+			iommus = <&apps_smmu 0x40 0x0>;
+
+			phys = <&usb_1_hsphy>, <&usb_dp_qmpphy QMP_USB43DP_USB3_PHY>;
+			phy-names = "usb2-phy", "usb3-phy";
+
+			snps,hird-threshold = /bits/ 8 <0x0>;
+			snps,usb2-gadget-lpm-disable;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_enblslpm_quirk;
+			snps,dis-u1-entry-quirk;
+			snps,dis-u2-entry-quirk;
+			snps,is-utmi-l1-suspend;
+			snps,usb3_lpm_capable;
+			snps,usb2-lpm-disable;
+			snps,has-lpm-erratum;
+			tx-fifo-resize;
+			dma-coherent;
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_1_dwc3_hs: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_1_dwc3_ss: endpoint {
+						remote-endpoint = <&usb_dp_qmpphy_usb_ss_in>;
+					};
+				};
+			};
+		};
+
+		pdc: interrupt-controller@b220000 {
+			compatible = "qcom,hawi-pdc", "qcom,pdc";
+			reg = <0x0 0x0b220000 0x0 0x10000>,
+			      <0x0 0x179600f0 0x0 0xf4>;
+
+			qcom,pdc-ranges = <0 256 196>;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&intc>;
+			interrupt-controller;
+		};
+
+		tsens0: thermal-sensor@c22b000 {
+			compatible = "qcom,hawi-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22b000 0x0 0x1000>,
+			      <0x0 0x0c222000 0x0 0x1000>;
+			interrupts = <GIC_SPI 282 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 484 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <12>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens1: thermal-sensor@c22c000 {
+			compatible = "qcom,hawi-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22c000 0x0 0x1000>,
+			      <0x0 0x0c223000 0x0 0x1000>;
+			interrupts = <GIC_SPI 283 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 485 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <7>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens2: thermal-sensor@c22d000 {
+			compatible = "qcom,hawi-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22d000 0x0 0x1000>,
+			      <0x0 0x0c224000 0x0 0x1000>;
+			interrupts = <GIC_SPI 284 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 486 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <6>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens3: thermal-sensor@c22e000 {
+			compatible = "qcom,hawi-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22e000 0x0 0x1000>,
+			      <0x0 0x0c225000 0x0 0x1000>;
+			interrupts = <GIC_SPI 285 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 487 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <9>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens4: thermal-sensor@c22f000 {
+			compatible = "qcom,hawi-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c22f000 0x0 0x1000>,
+			      <0x0 0x0c226000 0x0 0x1000>;
+			interrupts = <GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 527 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <9>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens5: thermal-sensor@c230000 {
+			compatible = "qcom,hawi-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c230000 0x0 0x1000>,
+			      <0x0 0x0c227000 0x0 0x1000>;
+			interrupts = <GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 528 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <11>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens6: thermal-sensor@c231000 {
+			compatible = "qcom,hawi-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c231000 0x0 0x1000>,
+			      <0x0 0x0c228000 0x0 0x1000>;
+			interrupts = <GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 529 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <8>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		aoss_qmp: power-management@c300000 {
+			compatible = "qcom,hawi-aoss-qmp", "qcom,aoss-qmp";
+			reg = <0x0 0x0c300000 0x0 0x400>;
+
+			interrupts-extended = <&ipcc IPCC_MPROC_AOP
+						     IPCC_MPROC_SIGNAL_GLINK_QMP
+						     IRQ_TYPE_EDGE_RISING>;
+
+			mboxes = <&ipcc IPCC_MPROC_AOP
+					IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+			#clock-cells = <0>;
+		};
+
+		arbiter@c400000 {
+			compatible = "qcom,hawi-spmi-pmic-arb",
+				     "qcom,glymur-spmi-pmic-arb";
+			reg = <0x0 0x0c400000 0x0 0x3000>,
+			      <0x0 0xc900000 0x0 0x400000>,
+			      <0x0 0xc4c0000 0x0 0x400000>,
+			      <0x0 0xc403000 0x0 0x8000>;
+			reg-names = "core",
+				    "chnls",
+				    "obsrvr",
+				    "chnl_map";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+			qcom,channel = <0>;
+			qcom,ee = <0>;
+
+			spmi_bus0: spmi@c426000 {
+				reg = <0x0 0x0c426000 0x0 0x4000>,
+				      <0x0 0xc8c0000 0x0 0x10000>,
+				      <0x0 0xc42a000 0x0 0x8000>;
+				reg-names = "cnfg",
+					    "intr",
+					    "chnl_owner";
+				interrupts-extended = <&pdc 1 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "periph_irq";
+				interrupt-controller;
+				#interrupt-cells = <4>;
+				#address-cells = <2>;
+				#size-cells = <0>;
+			};
+
+			spmi_bus1: spmi@c437000 {
+				reg = <0x0 0x0c437000 0x0 0x4000>,
+				      <0x0 0xc8d0000 0x0 0x10000>,
+				      <0x0 0xc43b000 0x0 0x8000>;
+				reg-names = "cnfg",
+					    "intr",
+					    "chnl_owner";
+				interrupts-extended = <&pdc 3 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "periph_irq";
+				interrupt-controller;
+				#interrupt-cells = <4>;
+				#address-cells = <2>;
+				#size-cells = <0>;
+			};
+		};
+
+		tlmm: pinctrl@f100000 {
+			compatible = "qcom,hawi-tlmm";
+			reg = <0x0 0x0f100000 0x0 0x102000>;
+			interrupts = <GIC_ESPI 272 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&tlmm 0 0 227>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			wakeup-parent = <&pdc>;
+
+			qup_i2c8_data_clk: qup-i2c8-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio0", "gpio1";
+				function = "qup2_se0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi8_data_clk: qup-spi8-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio0", "gpio1", "gpio2";
+				function = "qup2_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart8_cts_rts: qup-uart8-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio0", "gpio1";
+				function = "qup2_se0";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart8_default: qup-uart8-default-state {
+				/* TX, RX */
+				pins = "gpio2", "gpio3";
+				function = "qup2_se0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi8_cs: qup-spi8-cs-state {
+				pins = "gpio3";
+				function = "qup2_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c9_data_clk: qup-i2c9-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio4", "gpio5";
+				function = "qup2_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi9_data_clk: qup-spi9-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio4", "gpio5", "gpio6";
+				function = "qup2_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart9_cts_rts: qup-uart9-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio4", "gpio5";
+				function = "qup2_se1";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart9_default: qup-uart9-default-state {
+				/* TX, RX */
+				pins = "gpio6", "gpio7";
+				function = "qup2_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi9_cs: qup-spi9-cs-state {
+				pins = "gpio7";
+				function = "qup2_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c14_data_clk: qup-i2c14-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio8", "gpio9";
+				function = "qup3_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi14_data_clk: qup-spi14-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio8", "gpio9", "gpio10";
+				function = "qup3_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart14_cts_rts: qup-uart14-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio8", "gpio9";
+				function = "qup3_se1";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart14_default: qup-uart14-default-state {
+				/* TX, RX */
+				pins = "gpio10", "gpio11";
+				function = "qup3_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi14_cs: qup-spi14-cs-state {
+				pins = "gpio11";
+				function = "qup3_se1";
+				drive-strength = <6>;
+				bias-pull-up;
+			};
+
+			qup_i2c15_data_clk: qup-i2c15-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio12", "gpio13";
+				function = "qup3_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi15_data_clk: qup-spi15-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio12", "gpio13", "gpio14";
+				function = "qup3_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart15_cts_rts: qup-uart15-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio12", "gpio13";
+				function = "qup3_se2";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart15_default: qup-uart15-default-state {
+				/* TX, RX */
+				pins = "gpio14", "gpio15";
+				function = "qup3_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi15_cs: qup-spi15-cs-state {
+				pins = "gpio15";
+				function = "qup3_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c16_data_clk: qup-i2c16-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio16", "gpio17";
+				function = "qup3_se3";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi16_data_clk: qup-spi16-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio16", "gpio17", "gpio18";
+				function = "qup3_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart16_cts_rts: qup-uart16-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio16", "gpio17";
+				function = "qup3_se3";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart16_default: qup-uart16-default-state {
+				/* TX, RX */
+				pins = "gpio18", "gpio19";
+				function = "qup3_se3";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi16_cs: qup-spi16-cs-state {
+				pins = "gpio19";
+				function = "qup3_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c17_data_clk: qup-i2c17-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio20", "gpio21";
+				function = "qup3_se4";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi17_data_clk: qup-spi17-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio20", "gpio21", "gpio22";
+				function = "qup3_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart17_cts_rts: qup-uart17-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio20", "gpio21";
+				function = "qup3_se4";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart17_default: qup-uart17-default-state {
+				/* TX, RX */
+				pins = "gpio22", "gpio23";
+				function = "qup3_se4";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi17_cs: qup-spi17-cs-state {
+				pins = "gpio23";
+				function = "qup3_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart18_cts_rts: qup-uart18-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio24", "gpio25";
+				function = "qup3_se5";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart18_default: qup-uart18-default-state {
+				/* TX, RX */
+				pins = "gpio26", "gpio27";
+				function = "qup3_se5";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_i2c20_data_clk: qup-i2c20-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio28", "gpio29";
+				function = "qup4_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi20_data_clk: qup-spi20-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio28", "gpio29", "gpio30";
+				function = "qup4_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart20_cts_rts: qup-uart20-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio28", "gpio29";
+				function = "qup4_se1";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart20_default: qup-uart20-default-state {
+				/* TX, RX */
+				pins = "gpio30", "gpio31";
+				function = "qup4_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi20_cs: qup-spi20-cs-state {
+				pins = "gpio31";
+				function = "qup4_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c21_data_clk: qup-i2c21-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio32", "gpio33";
+				function = "qup4_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi21_data_clk: qup-spi21-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio32", "gpio33", "gpio34";
+				function = "qup4_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart21_cts_rts: qup-uart21-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio32", "gpio33";
+				function = "qup4_se2";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart21_default: qup-uart21-default-state {
+				/* TX, RX */
+				pins = "gpio34", "gpio35";
+				function = "qup4_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi21_cs: qup-spi21-cs-state {
+				pins = "gpio35";
+				function = "qup4_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c4_data_clk: qup-i2c4-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio36", "gpio37";
+				function = "qup1_se4";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi4_data_clk: qup-spi4-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio36", "gpio37", "gpio38";
+				function = "qup1_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart4_cts_rts: qup-uart4-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio36", "gpio37";
+				function = "qup1_se4";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart4_default: qup-uart4-default-state {
+				/* TX, RX */
+				pins = "gpio38", "gpio39";
+				function = "qup1_se4";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi4_cs: qup-spi4-cs-state {
+				pins = "gpio39";
+				function = "qup1_se4";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c2_data_clk: qup-i2c2-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio40", "gpio41";
+				function = "qup1_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi2_data_clk: qup-spi2-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio40", "gpio41", "gpio42";
+				function = "qup1_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart2_cts_rts: qup-uart2-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio40", "gpio41";
+				function = "qup1_se2";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart2_default: qup-uart2-default-state {
+				/* TX, RX */
+				pins = "gpio42", "gpio43";
+				function = "qup1_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi2_cs: qup-spi2-cs-state {
+				pins = "gpio43";
+				function = "qup1_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c3_data_clk: qup-i2c3-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio44", "gpio45";
+				function = "qup1_se3";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi3_data_clk: qup-spi3-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio44", "gpio45", "gpio46";
+				function = "qup1_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart3_cts_rts: qup-uart3-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio44", "gpio45";
+				function = "qup1_se3";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart3_default: qup-uart3-default-state {
+				/* TX, RX */
+				pins = "gpio46", "gpio47";
+				function = "qup1_se3";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi3_cs: qup-spi3-cs-state {
+				pins = "gpio47";
+				function = "qup1_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c19_data_clk: qup-i2c19-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio48", "gpio49";
+				function = "qup4_se0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi19_data_clk: qup-spi19-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio48", "gpio49", "gpio50";
+				function = "qup4_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart19_cts_rts: qup-uart19-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio48", "gpio49";
+				function = "qup4_se0";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart19_default: qup-uart19-default-state {
+				/* TX, RX */
+				pins = "gpio50", "gpio51";
+				function = "qup4_se0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi19_cs: qup-spi19-cs-state {
+				pins = "gpio51";
+				function = "qup4_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c5_data_clk: qup-i2c5-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio52", "gpio53";
+				function = "qup1_se5";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi5_data_clk: qup-spi5-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio52", "gpio53", "gpio54";
+				function = "qup1_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart5_cts_rts: qup-uart5-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio52", "gpio53";
+				function = "qup1_se5";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart5_default: qup-uart5-default-state {
+				/* TX, RX */
+				pins = "gpio54", "gpio55";
+				function = "qup1_se5";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi5_cs: qup-spi5-cs-state {
+				pins = "gpio55";
+				function = "qup1_se5";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			sdc2_default: sdc2-default-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <16>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <10>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <10>;
+					bias-pull-up;
+				};
+
+				card-detect-pins {
+					pins = "gpio55";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
+			sdc2_sleep: sdc2-sleep-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				card-detect-pins {
+					pins = "gpio55";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
+			qup_i2c6_data_clk: qup-i2c6-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio56", "gpio57";
+				function = "qup1_se6";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi6_data_clk: qup-spi6-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio56", "gpio57", "gpio58";
+				function = "qup1_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart6_cts_rts: qup-uart6-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio56", "gpio57";
+				function = "qup1_se6";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart6_default: qup-uart6-default-state {
+				/* TX, RX */
+				pins = "gpio58", "gpio59";
+				function = "qup1_se6";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi6_cs: qup-spi6-cs-state {
+				pins = "gpio59";
+				function = "qup1_se6";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart7_default: qup-uart7-state {
+				/* TX, RX */
+				pins = "gpio62", "gpio63";
+				function = "qup1_se7";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			qup_i2c13_data_clk: qup-i2c13-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio64", "gpio65";
+				function = "qup3_se0_01";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_uart13_default: qup-uart13-default-state {
+				/* TX, RX */
+				pins = "gpio64", "gpio65";
+				function = "qup3_se0_23";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			hub_i2c0_data_clk: hub-i2c0-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio66", "gpio67";
+				function = "i2chub0_se0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			hub_i2c2_data_clk: hub-i2c2-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio68", "gpio69";
+				function = "i2chub0_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			hub_i2c3_data_clk: hub-i2c3-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio70", "gpio71";
+				function = "i2chub0_se3";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			hub_i2c4_data_clk: hub-i2c4-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio72", "gpio73";
+				function = "i2chub0_se4";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_i2c1_data_clk: qup-i2c1-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio74", "gpio75";
+				function = "qup1_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi1_data_clk: qup-spi1-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio74", "gpio75", "gpio76";
+				function = "qup1_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart1_cts_rts: qup-uart1-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio74", "gpio75";
+				function = "qup1_se1";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart1_default: qup-uart1-default-state {
+				/* TX, RX */
+				pins = "gpio76", "gpio77";
+				function = "qup1_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi1_cs: qup-spi1-cs-state {
+				pins = "gpio77";
+				function = "qup1_se1";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			hub_i2c1_data_clk: hub-i2c1-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio78", "gpio79";
+				function = "i2chub0_se1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_i2c0_data_clk: qup-i2c0-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio80", "gpio83";
+				function = "qup1_se0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi0_data_clk: qup-spi0-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio80", "gpio83", "gpio82";
+				function = "qup1_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart0_cts_rts: qup-uart0-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio80", "gpio83";
+				function = "qup1_se0";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_spi0_cs: qup-spi0-cs-state {
+				pins = "gpio81";
+				function = "qup1_se0";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart0_default: qup-uart0-default-state {
+				/* TX, RX */
+				pins = "gpio82", "gpio81";
+				function = "qup1_se0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_i2c22_data_clk: qup-i2c22-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio121", "gpio84";
+				function = "qup4_se3_01";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_uart22_default: qup-uart22-default-state {
+				/* TX, RX */
+				pins = "gpio121", "gpio84";
+				function = "qup4_se3_23";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			pcie0_default_state: pcie0-default-state {
+				perst-n-pins {
+					pins = "gpio102";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				clkreq-n-pins {
+					pins = "gpio103";
+					function = "pcie0_clk_req_n";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				wake-n-pins {
+					pins = "gpio104";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
+			qup_i2c10_data_clk: qup-i2c10-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio117", "gpio118";
+				function = "qup2_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi10_data_clk: qup-spi10-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio117", "gpio118", "gpio119";
+				function = "qup2_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart10_cts_rts: qup-uart10-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio117", "gpio118";
+				function = "qup2_se2";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart10_default: qup-uart10-default-state {
+				/* TX, RX */
+				pins = "gpio119", "gpio120";
+				function = "qup2_se2";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi10_cs: qup-spi10-cs-state {
+				pins = "gpio120";
+				function = "qup2_se2";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c11_data_clk: qup-i2c11-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio122", "gpio123";
+				function = "qup2_se3";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi11_data_clk: qup-spi11-data-clk-state {
+				/* MISO, MOSI, CLK */
+				pins = "gpio122", "gpio123", "gpio124";
+				function = "qup2_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_uart11_cts_rts: qup-uart11-cts-rts-state {
+				/* CTS, RTS */
+				pins = "gpio122", "gpio123";
+				function = "qup2_se3";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			qup_uart11_default: qup-uart11-default-state {
+				/* TX, RX */
+				pins = "gpio124", "gpio125";
+				function = "qup2_se3";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_spi11_cs: qup-spi11-cs-state {
+				pins = "gpio125";
+				function = "qup2_se3";
+				drive-strength = <6>;
+				bias-disable;
+			};
+
+			qup_i2c23_data_clk: qup-i2c23-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio161", "gpio162";
+				function = "qup4_se4_01";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_uart23_default: qup-uart23-default-state {
+				/* TX, RX */
+				pins = "gpio161", "gpio162";
+				function = "qup4_se4_23";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_i2c12_data_clk: qup-i2c12-data-clk-state {
+				/* SDA, SCL */
+				pins = "gpio208", "gpio209";
+				function = "qup2_se4_01";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			qup_uart12_default: qup-uart12-default-state {
+				/* TX, RX */
+				pins = "gpio208", "gpio209";
+				function = "qup2_se4_23";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			pcie1_default_state: pcie1-default-state {
+				perst-n-pins {
+					pins = "gpio220";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				clkreq-n-pins {
+					pins = "gpio221";
+					function = "pcie1_clk_req_n";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				wake-n-pins {
+					pins = "gpio222";
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+		};
+
+		sram@14680000 {
+			compatible = "qcom,hawi-imem", "mmio-sram";
+			reg = <0x0 0x14680000 0x0 0x1000>;
+			ranges = <0x0 0x0 0x14680000 0x1000>;
+
+			no-memory-wc;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			minidump_config: minidump-sram@1c {
+				reg = <0x1c 0x4>;
+			};
+
+			pil-sram@94c {
+				compatible = "qcom,pil-reloc-info";
+				reg = <0x94c 0xc8>;
+			};
+		};
+
+		intc: interrupt-controller@17000000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x17000000 0x0 0x10000>,
+			      <0x0 0x17080000 0x0 0x200000>;
+
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+			#interrupt-cells = <3>;
+			interrupt-controller;
+
+			#redistributor-regions = <1>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			gic_its: msi-controller@17040000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x17040000 0x0 0x20000>;
+
+				msi-controller;
+				#msi-cells = <1>;
+			};
+		};
+
+		watchdog@17600000 {
+			compatible = "qcom,apss-wdt-hawi", "qcom,kpss-wdt";
+			reg = <0x0 0x17600000 0x0 0x1000>;
+			clocks = <&sleep_clk>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING>;
+		};
+
+		pdp0_mbox: mailbox@17610000 {
+			compatible = "qcom,hawi-cpucp-mbox", "qcom,x1e80100-cpucp-mbox";
+			reg = <0x0 0x17610000 0x0 0x8000>,
+			      <0x0 0x19990000 0x0 0x8000>;
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <1>;
+		};
+
+		timer@17810000 {
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x0 0x17810000 0x0 0x1000>;
+
+			#address-cells = <2>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x0 0x0 0x20000000>;
+
+			frame@17811000 {
+				reg = <0x0 0x17811000 0x1000>,
+				      <0x0 0x17812000 0x1000>;
+				frame-number = <0>;
+				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+			};
+
+			frame@17813000 {
+				reg = <0x0 0x17813000 0x1000>;
+				frame-number = <1>;
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			frame@17815000 {
+				reg = <0x0 0x17815000 0x1000>;
+				frame-number = <2>;
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			frame@17817000 {
+				reg = <0x0 0x17817000 0x1000>;
+				frame-number = <3>;
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			frame@17819000 {
+				reg = <0x0 0x17819000 0x1000>;
+				frame-number = <4>;
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			frame@1781b000 {
+				reg = <0x0 0x1781b000 0x1000>;
+				frame-number = <5>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			frame@1781d000 {
+				reg = <0x0 0x1781d000 0x1000>;
+				frame-number = <6>;
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+		};
+
+		apps_rsc: rsc@18900000 {
+			compatible = "qcom,rpmh-rsc";
+			reg = <0x0 0x18900000 0x0 0x10000>,
+			      <0x0 0x18910000 0x0 0x10000>,
+			      <0x0 0x18920000 0x0 0x10000>;
+			reg-names = "drv-0",
+				    "drv-1",
+				    "drv-2";
+			interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+
+			power-domains = <&cluster_pd>;
+			label = "apps_rsc";
+
+			qcom,tcs-offset = <0xd00>;
+			qcom,drv-id = <2>;
+			qcom,tcs-config = <ACTIVE_TCS 3>,
+					  <SLEEP_TCS 2>,
+					  <WAKE_TCS 2>,
+					  <CONTROL_TCS 0>;
+
+			apps_bcm_voter: bcm-voter {
+				compatible = "qcom,bcm-voter";
+			};
+
+			rpmhcc: clock-controller {
+				compatible = "qcom,hawi-rpmh-clk";
+				#clock-cells = <1>;
+				clocks = <&xo_board>;
+				clock-names = "xo";
+			};
+
+			rpmhpd: power-controller {
+				compatible = "qcom,hawi-rpmhpd";
+
+				operating-points-v2 = <&rpmhpd_opp_table>;
+
+				#power-domain-cells = <1>;
+
+				rpmhpd_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					rpmhpd_opp_ret: opp-16 {
+						opp-level = <RPMH_REGULATOR_LEVEL_RETENTION>;
+					};
+
+					rpmhpd_opp_low_svs_d3_0: opp-49 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D3_0>;
+					};
+
+					rpmhpd_opp_low_svs_d3: opp-50 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D3>;
+					};
+
+					rpmhpd_opp_low_svs_d2_1: opp-51 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2_1>;
+					};
+
+					rpmhpd_opp_low_svs_d2: opp-52 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2>;
+					};
+
+					rpmhpd_opp_low_svs_d1_1: opp-54 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1_1>;
+					};
+
+					rpmhpd_opp_low_svs_d1_0: opp-55 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1_0>;
+					};
+
+					rpmhpd_opp_low_svs_d1: opp-56 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					};
+
+					rpmhpd_opp_low_svs_d0_0: opp-59 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0_0>;
+					};
+
+					rpmhpd_opp_low_svs_d0: opp-60 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0>;
+					};
+
+					rpmhpd_opp_low_svs: opp-64 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					};
+
+					rpmhpd_opp_low_svs_l0: opp-76 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L0>;
+					};
+
+					rpmhpd_opp_low_svs_l1: opp-80 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L1>;
+					};
+
+					rpmhpd_opp_low_svs_l2: opp-96 {
+						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L2>;
+					};
+
+					rpmhpd_opp_svs: opp-128 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					};
+
+					rpmhpd_opp_svs_l0: opp-144 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L0>;
+					};
+
+					rpmhpd_opp_svs_l1: opp-192 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					};
+
+					rpmhpd_opp_svs_l2: opp-224 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
+					};
+
+					rpmhpd_opp_svs_l2_0: opp-225 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2_0>;
+					};
+
+					rpmhpd_opp_nom: opp-256 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					};
+
+					rpmhpd_opp_nom_l1: opp-320 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+					};
+
+					rpmhpd_opp_nom_l2: opp-336 {
+						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L2>;
+					};
+
+					rpmhpd_opp_turbo: opp-384 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+					};
+
+					rpmhpd_opp_turbo_l0: opp-400 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L0>;
+					};
+
+					rpmhpd_opp_turbo_l1: opp-416 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					};
+
+					rpmhpd_opp_turbo_l1_0: opp-417 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1_0>;
+					};
+
+					rpmhpd_opp_turbo_l1_1: opp-418 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1_1>;
+					};
+
+					rpmhpd_opp_turbo_l1_2: opp-419 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1_2>;
+					};
+
+					rpmhpd_opp_turbo_l2: opp-432 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L2>;
+					};
+
+					rpmhpd_opp_turbo_l3: opp-448 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3>;
+					};
+
+					rpmhpd_opp_turbo_l3_0: opp-449 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3_0>;
+					};
+
+					rpmhpd_opp_turbo_l3_1: opp-450 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3_1>;
+					};
+
+					rpmhpd_opp_turbo_l3_2: opp-451 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3_2>;
+					};
+
+					rpmhpd_opp_turbo_l4: opp-452 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L4>;
+					};
+
+					rpmhpd_opp_turbo_l5: opp-456 {
+						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L5>;
+					};
+
+					rpmhpd_opp_super_turbo_no_cpr: opp-480 {
+						opp-level = <RPMH_REGULATOR_LEVEL_SUPER_TURBO_NO_CPR>;
+					};
+				};
+			};
+		};
+
+		efuse@221c8000 {
+			compatible = "qcom,hawi-qfprom", "qcom,qfprom";
+			reg = <0x0 0x221c8000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			gpu_speed_bin: gpu-speed-bin@142 {
+				reg = <0x142 0x2>;
+				bits = <3 9>;
+			};
+		};
+
+		nsp_noc: interconnect@260c0000 {
+			compatible = "qcom,hawi-nsp-noc";
+			reg = <0x0 0x260c0000 0x0 0x21280>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		remoteproc_cdsp: remoteproc@26300000 {
+			compatible = "qcom,hawi-cdsp-pas";
+			reg = <0x0 0x26300000 0x0 0x10000>;
+
+			interrupts-extended = <&intc GIC_ESPI 642 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 7 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "shutdown-ack";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			interconnects = <&nsp_noc MASTER_CDSP_PROC QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MXC>,
+					<&rpmhpd RPMHPD_NSP>;
+			power-domain-names = "cx",
+					     "mxc",
+					     "nsp";
+
+			memory-region = <&cdsp_mem>, <&q6_cdsp_dtb_mem>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			qcom,smem-states = <&smp2p_cdsp_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			glink-edge {
+				interrupts-extended = <&ipcc IPCC_MPROC_CDSP
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+
+				mboxes = <&ipcc IPCC_MPROC_CDSP
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+				qcom,remote-pid = <5>;
+				label = "cdsp";
+
+				fastrpc {
+					compatible = "qcom,hawi-fastrpc", "qcom,kaanapali-fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "cdsp";
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@1 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <1>;
+
+						iommus = <&apps_smmu 0x0801 0x40>,
+							 <&apps_smmu 0x15c1 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@2 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <2>;
+
+						iommus = <&apps_smmu 0x0802 0x40>,
+							 <&apps_smmu 0x15c2 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+
+						iommus = <&apps_smmu 0x0803 0x40>,
+							 <&apps_smmu 0x15c3 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@4 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <4>;
+
+						iommus = <&apps_smmu 0x0804 0x40>,
+							 <&apps_smmu 0x15c4 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@5 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <5>;
+
+						iommus = <&apps_smmu 0x0805 0x40>,
+							 <&apps_smmu 0x15c5 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@6 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <6>;
+
+						iommus = <&apps_smmu 0x0806 0x40>,
+							 <&apps_smmu 0x15c6 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@7 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <7>;
+
+						iommus = <&apps_smmu 0x0807 0x40>,
+							 <&apps_smmu 0x15c7 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@8 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <8>;
+
+						iommus = <&apps_smmu 0x0808 0x40>,
+							 <&apps_smmu 0x15c8 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@9 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <9>;
+
+						iommus = <&apps_smmu 0x0809 0x40>,
+							 <&apps_smmu 0x15c9 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@12 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <12>;
+
+						iommus = <&apps_smmu 0x080c 0x40>,
+							 <&apps_smmu 0x15cc 0x0>;
+						dma-coherent;
+					};
+				};
+			};
+		};
+
+		pmu@31081000 {
+			compatible = "qcom,hawi-llcc-bwmon", "qcom,sc7280-llcc-bwmon";
+			reg = <0x0 0x31081000 0x0 0x1000>;
+
+			interrupts = <GIC_ESPI 145 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			operating-points-v2 = <&llcc_bwmon_opp_table>;
+
+			llcc_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <13128000>;
+				};
+
+				opp-1 {
+					opp-peak-kBps = <32472000>;
+				};
+
+				opp-2 {
+					opp-peak-kBps = <37320000>;
+				};
+
+				opp-3 {
+					opp-peak-kBps = <40992000>;
+				};
+
+				opp-4 {
+					opp-peak-kBps = <50208000>;
+				};
+
+				opp-5 {
+					opp-peak-kBps = <65664000>;
+				};
+
+				opp-6 {
+					opp-peak-kBps = <76488000>;
+				};
+
+				opp-7 {
+					opp-peak-kBps = <88464000>;
+				};
+
+				opp-8 {
+					opp-peak-kBps = <101376000>;
+				};
+
+				opp-9 {
+					opp-peak-kBps = <114720000>;
+				};
+
+				opp-10 {
+					opp-peak-kBps = <127992000>;
+				};
+			};
+		};
+
+		pmu@310b7500 {
+			compatible = "qcom,hawi-cpu-bwmon", "qcom,sdm845-bwmon";
+			reg = <0x0 0x310b7500 0x0 0x600>;
+
+			interrupts = <GIC_ESPI 645 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			operating-points-v2 = <&cpu_bwmon_opp_table>;
+
+			cpu_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <12516000>;
+				};
+
+				opp-1 {
+					opp-peak-kBps = <17574000>;
+				};
+
+				opp-2 {
+					opp-peak-kBps = <35586000>;
+				};
+
+				opp-3 {
+					opp-peak-kBps = <39090000>;
+				};
+
+				opp-4 {
+					opp-peak-kBps = <47880000>;
+				};
+
+				opp-5 {
+					opp-peak-kBps = <62622000>;
+				};
+
+				opp-6 {
+					opp-peak-kBps = <73542000>;
+				};
+
+				opp-7 {
+					opp-peak-kBps = <84360000>;
+				};
+
+				opp-8 {
+					opp-peak-kBps = <96678000>;
+				};
+			};
+		};
+
+		gem_noc: interconnect@31100000 {
+			compatible = "qcom,hawi-gem-noc";
+			reg = <0x0 0x31100000 0x0 0x160200>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		llclpi_noc: interconnect@31310000 {
+			compatible = "qcom,hawi-llclpi-noc";
+			reg = <0x0 0x31310000 0x0 0x17200>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		system-cache-controller@31800000 {
+			compatible = "qcom,hawi-llcc";
+			reg = <0x0 0x31800000 0x0 0x200000>,
+			      <0x0 0x32800000 0x0 0x200000>,
+			      <0x0 0x31c00000 0x0 0x200000>,
+			      <0x0 0x32c00000 0x0 0x200000>,
+			      <0x0 0x34800000 0x0 0x200000>,
+			      <0x0 0x34c00000 0x0 0x200000>;
+			reg-names = "llcc0_base",
+				    "llcc1_base",
+				    "llcc2_base",
+				    "llcc3_base",
+				    "llcc_broadcast_base",
+				    "llcc_broadcast_and_base";
+
+			interrupts = <GIC_ESPI 330 IRQ_TYPE_LEVEL_HIGH>;
+			memory-region = <&cpucp_slc_mem_sub>;
+		};
+
+		sram: sram@81f08000 {
+			compatible = "mmio-sram";
+			reg = <0x0 0x81f08000 0x0 0x200>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x81f08000 0x200>;
+
+			pdp_rx: scp-sram-section@0 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x0 0x80>;
+			};
+
+			pdp_tx: scp-sram-section@80 {
+				compatible = "arm,scmi-shmem";
+				reg = <0x80 0x80>;
+			};
+		};
+	};
+
+	thermal-zones {
+		camera-0-thermal {
+			thermal-sensors = <&tsens6 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				camera-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		camera-1-thermal {
+			thermal-sensors = <&tsens6 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				camera-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-0-thermal {
+			thermal-sensors = <&tsens2 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-0-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-1-thermal {
+			thermal-sensors = <&tsens2 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-0-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-0-2-thermal {
+			thermal-sensors = <&tsens2 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-0-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-0-thermal {
+			thermal-sensors = <&tsens2 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-1-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-1-thermal {
+			thermal-sensors = <&tsens2 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-1-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-1-2-thermal {
+			thermal-sensors = <&tsens2 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-1-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-0-thermal {
+			thermal-sensors = <&tsens0 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-2-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-2-1-thermal {
+			thermal-sensors = <&tsens0 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-2-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-3-0-thermal {
+			thermal-sensors = <&tsens0 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-3-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-3-1-thermal {
+			thermal-sensors = <&tsens0 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-3-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-4-0-thermal {
+			thermal-sensors = <&tsens0 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-4-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-4-1-thermal {
+			thermal-sensors = <&tsens0 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-4-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-5-0-thermal {
+			thermal-sensors = <&tsens0 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-5-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-5-1-thermal {
+			thermal-sensors = <&tsens0 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-5-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-6-0-thermal {
+			thermal-sensors = <&tsens0 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-6-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-6-1-thermal {
+			thermal-sensors = <&tsens0 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-6-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-7-0-thermal {
+			thermal-sensors = <&tsens0 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-7-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu-7-1-thermal {
+			thermal-sensors = <&tsens0 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpu-7-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpullc-0-thermal {
+			thermal-sensors = <&tsens1 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpullc-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpullc-1-thermal {
+			thermal-sensors = <&tsens1 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				cpullc-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		ddr-thermal {
+			thermal-sensors = <&tsens6 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				ddr-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-0-0-thermal {
+			thermal-sensors = <&tsens5 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-0-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-0-1-thermal {
+			thermal-sensors = <&tsens5 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-0-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-0-2-thermal {
+			thermal-sensors = <&tsens5 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				gpu-0-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-1-0-thermal {
+			thermal-sensors = <&tsens5 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpu-1-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-1-1-thermal {
+			thermal-sensors = <&tsens5 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpu-1-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-1-2-thermal {
+			thermal-sensors = <&tsens5 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpu-1-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-2-0-thermal {
+			thermal-sensors = <&tsens5 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpu-2-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-2-1-thermal {
+			thermal-sensors = <&tsens5 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpu-2-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpu-2-2-thermal {
+			thermal-sensors = <&tsens5 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpu-2-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-0-thermal {
+			thermal-sensors = <&tsens5 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpuss-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-1-thermal {
+			thermal-sensors = <&tsens5 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				gpuss-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		mdmss-0-thermal {
+			thermal-sensors = <&tsens6 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				mdmss-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		mdmss-1-thermal {
+			thermal-sensors = <&tsens6 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				mdmss-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		mdmss-2-thermal {
+			thermal-sensors = <&tsens6 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				mdmss-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		mdmss-3-thermal {
+			thermal-sensors = <&tsens6 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				mdmss-3-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspllv-0-0-thermal {
+			thermal-sensors = <&tsens3 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspllv-0-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspllv-0-1-thermal {
+			thermal-sensors = <&tsens3 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspllv-0-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspllv-1-0-thermal {
+			thermal-sensors = <&tsens3 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspllv-1-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspllv-1-1-thermal {
+			thermal-sensors = <&tsens3 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspllv-1-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu0-0-0-thermal {
+			thermal-sensors = <&tsens4 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu0-0-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu0-0-1-thermal {
+			thermal-sensors = <&tsens4 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu0-0-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu0-1-0-thermal {
+			thermal-sensors = <&tsens4 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu0-1-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu0-1-1-thermal {
+			thermal-sensors = <&tsens4 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu0-1-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu1-0-0-thermal {
+			thermal-sensors = <&tsens4 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu1-0-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu1-0-1-thermal {
+			thermal-sensors = <&tsens4 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu1-0-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu1-1-0-thermal {
+			thermal-sensors = <&tsens4 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu1-1-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxu1-1-1-thermal {
+			thermal-sensors = <&tsens4 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxu1-1-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspmxutop-thermal {
+			thermal-sensors = <&tsens4 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspmxutop-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspq6-thermal {
+			thermal-sensors = <&tsens3 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspq6-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspvxu-0-thermal {
+			thermal-sensors = <&tsens3 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspvxu-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspvxu-1-thermal {
+			thermal-sensors = <&tsens3 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspvxu-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspvxu-2-thermal {
+			thermal-sensors = <&tsens3 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspvxu-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		nspvxu-3-thermal {
+			thermal-sensors = <&tsens3 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				nspvxu-3-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-0-thermal {
+			thermal-sensors = <&tsens1 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				qmx-0-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-1-thermal {
+			thermal-sensors = <&tsens1 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				qmx-1-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-2-thermal {
+			thermal-sensors = <&tsens1 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				qmx-2-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-3-thermal {
+			thermal-sensors = <&tsens1 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				qmx-3-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		qmx-4-thermal {
+			thermal-sensors = <&tsens1 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <5000>;
+					type = "hot";
+				};
+
+				qmx-4-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		video-thermal {
+			thermal-sensors = <&tsens6 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+
+				video-critical {
+					temperature = <125000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts b/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts
index 33eef92b19b140..34d4a088635604 100644
--- a/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts
+++ b/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts
@@ -30,6 +30,16 @@
 	status = "okay";
 };
 
+&mdio1 {
+	clock-frequency = <6250000>;
+
+	status = "okay";
+
+	ethernet-phy@1c {
+		reg = <0x1c>;
+	};
+};
+
 &pcie0 {
 	pinctrl-0 = <&pcie0_default>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/qcom/ipq5018.dtsi b/arch/arm64/boot/dts/qcom/ipq5018.dtsi
index 4fc627b47fe74f..fe1ed240de59ec 100644
--- a/arch/arm64/boot/dts/qcom/ipq5018.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5018.dtsi
@@ -222,8 +222,7 @@
 
 			clocks = <&gcc GCC_MDIO0_AHB_CLK>;
 			clock-names = "gcc_mdio_ahb_clk";
-
-			status = "disabled";
+			clock-frequency = <6250000>;
 
 			ge_phy: ethernet-phy@7 {
 				compatible = "ethernet-phy-id004d.d0c0";
@@ -241,6 +240,8 @@
 			reg = <0x00090000 0x64>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			pinctrl-0 = <&mdio_pins>;
+			pinctrl-names = "default";
 
 			clocks = <&gcc GCC_MDIO1_AHB_CLK>;
 			clock-names = "gcc_mdio_ahb_clk";
@@ -421,6 +422,22 @@
 			interrupt-controller;
 			#interrupt-cells = <2>;
 
+			mdio_pins: mdio-state {
+				mdc-pins {
+					pins = "gpio36";
+					function = "mdc";
+					drive-strength = <8>;
+					bias-pull-up;
+				};
+
+				mdio-pins {
+					pins = "gpio37";
+					function = "mdio";
+					drive-strength = <8>;
+					bias-pull-up;
+				};
+			};
+
 			uart1_pins: uart1-state {
 				pins = "gpio31", "gpio32", "gpio33", "gpio34";
 				function = "blsp1_uart1";
diff --git a/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi b/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi
index 3bbf748f2e5c54..6f5b193a204bfa 100644
--- a/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5332-rdp-common.dtsi
@@ -141,5 +141,7 @@
 		nand-ecc-engine = <&qpic_nand>;
 		nand-ecc-strength = <4>;
 		nand-ecc-step-size = <512>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts b/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts
index be8657239c46b3..793afc3ee82e18 100644
--- a/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts
+++ b/arch/arm64/boot/dts/qcom/ipq5424-rdp466.dts
@@ -261,6 +261,8 @@
 		nand-ecc-engine = <&qpic_nand>;
 		nand-ecc-strength = <4>;
 		nand-ecc-step-size = <512>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/ipq5424.dtsi b/arch/arm64/boot/dts/qcom/ipq5424.dtsi
index 702061e16a58d0..23f851e9241fed 100644
--- a/arch/arm64/boot/dts/qcom/ipq5424.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5424.dtsi
@@ -624,6 +624,7 @@
 			reg = <0 0x0f410000 0 0x1000>;
 			interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING>;
 			clocks = <&sleep_clk>;
+			sram = <&restart_reason>;
 		};
 
 		qusb_phy_1: phy@71000 {
@@ -730,6 +731,21 @@
 			status = "disabled";
 		};
 
+		sram@8600000 {
+			compatible = "qcom,ipq5424-imem", "mmio-sram";
+			reg = <0 0x08600000 0 0x1c000>;
+			ranges = <0 0 0x08600000 0x1c000>;
+
+			no-memory-wc;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			restart_reason: restartreason-sram@7b0 {
+				reg = <0x7b0 0x4>;
+			};
+		};
+
 		usb3: usb3@8af8800 {
 			compatible = "qcom,ipq5424-dwc3", "qcom,dwc3";
 			reg = <0 0x08af8800 0 0x400>;
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi b/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
index 62877b46f9b3fc..86138529ba77e3 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
@@ -249,6 +249,8 @@
 		nand-ecc-engine = <&qpic_nand>;
 		nand-ecc-strength = <4>;
 		nand-ecc-step-size = <512>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/ipq9574.dtsi b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
index 5a5bda5f21a114..771a5958564594 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
@@ -428,7 +428,7 @@
 				 <&pcie1_phy>,
 				 <&pcie2_phy>,
 				 <&pcie3_phy>,
-				 <0>;
+				 <&usb_0_qmpphy>;
 			#clock-cells = <1>;
 			#reset-cells = <1>;
 			#interconnect-cells = <1>;
@@ -471,6 +471,7 @@
 				 <&xo_board_clk>,
 				 <&gcc GCC_SDCC1_ICE_CORE_CLK>;
 			clock-names = "iface", "core", "xo", "ice";
+			resets = <&gcc GCC_SDCC_BCR>;
 			non-removable;
 			supports-cqe;
 			pinctrl-0 = <&sdc_default_state>;
diff --git a/arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts b/arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts
index 6871f3dc4eaf05..f6411a55c81e1c 100644
--- a/arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts
@@ -5,6 +5,8 @@
 
 /dts-v1/;
 
+#include <dt-bindings/gpio/gpio.h>
+
 #include "ipq9650.dtsi"
 
 / {
@@ -20,6 +22,54 @@
 	};
 };
 
+&pcie1 {
+	pinctrl-0 = <&pcie1_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie1_phy {
+	status = "okay";
+};
+
+&pcie1_port0 {
+	reset-gpios = <&tlmm 28 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 29 GPIO_ACTIVE_LOW>;
+};
+
+&pcie2 {
+	pinctrl-0 = <&pcie2_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie2_phy {
+	status = "okay";
+};
+
+&pcie2_port0 {
+	reset-gpios = <&tlmm 52 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 53 GPIO_ACTIVE_LOW>;
+};
+
+&pcie3 {
+	pinctrl-0 = <&pcie3_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie3_phy {
+	status = "okay";
+};
+
+&pcie3_port0 {
+	reset-gpios = <&tlmm 41 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 42 GPIO_ACTIVE_LOW>;
+};
+
 &sdhc {
 	max-frequency = <192000000>;
 	bus-width = <4>;
@@ -36,6 +86,75 @@
 };
 
 &tlmm {
+	pcie1_default_state: pcie1-default-state {
+		clkreq-n-pins {
+			pins = "gpio27";
+			function = "pcie1_clk_req_n";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio28";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-pull-down;
+		};
+
+		wake-n-pins {
+			pins = "gpio29";
+			function = "pcie1_wake";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+	};
+
+	pcie2_default_state: pcie2-default-state {
+		clkreq-n-pins {
+			pins = "gpio51";
+			function = "pcie2_clk_req_n";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio52";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-pull-down;
+		};
+
+		wake-n-pins {
+			pins = "gpio53";
+			function = "pcie2_wake";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+	};
+
+	pcie3_default_state: pcie3-default-state {
+		clkreq-n-pins {
+			pins = "gpio40";
+			function = "pcie3_clk_req_n";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio41";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-pull-down;
+		};
+
+		wake-n-pins {
+			pins = "gpio42";
+			function = "pcie3_wake";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+	};
+
 	qup_uart1_default_state: qup-uart1-default-state {
 		pins = "gpio43", "gpio44";
 		function = "qup_se6";
diff --git a/arch/arm64/boot/dts/qcom/ipq9650.dtsi b/arch/arm64/boot/dts/qcom/ipq9650.dtsi
index 3d3b317c6a3ccf..3f18438be6fc17 100644
--- a/arch/arm64/boot/dts/qcom/ipq9650.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq9650.dtsi
@@ -5,6 +5,7 @@
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/qcom,ipq9650-gcc.h>
+#include <dt-bindings/interconnect/qcom,ipq9650.h>
 #include <dt-bindings/reset/qcom,ipq9650-gcc.h>
 
 / {
@@ -200,6 +201,160 @@
 		dma-ranges = <0 0 0 0 0x10 0>;
 		ranges = <0 0 0 0 0x10 0>;
 
+		refgen_pcie: regulator@6c000 {
+			compatible = "qcom,ipq9650-refgen-regulator";
+			reg = <0x0 0x0006c000 0x0 0x84>;
+			clocks = <&gcc GCC_REFGEN_PCIE_CORE_CLK>,
+				 <&gcc GCC_REFGEN_PCIE_HCLK>;
+			clock-names = "core",
+				      "hclk";
+		};
+
+		pcie0_phy: phy@84000 {
+			compatible = "qcom,ipq9650-qmp-gen3x1-pcie-phy";
+			reg = <0x0 0x00084000 0x0 0x1000>;
+
+			clocks = <&gcc GCC_PCIE0_AUX_CLK>,
+				 <&gcc GCC_PCIE0_AHB_CLK>,
+				 <&gcc GCC_PCIE0_PIPE_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "pipe";
+
+			assigned-clocks = <&gcc GCC_PCIE0_AUX_CLK>;
+			assigned-clock-rates = <20000000>;
+
+			resets = <&gcc GCC_PCIE0_PHY_BCR>,
+				 <&gcc GCC_PCIE0PHY_PHY_BCR>;
+			reset-names = "phy",
+				      "common";
+
+			refgen-supply = <&refgen_pcie>;
+
+			clock-output-names = "pcie0_pipe_clk";
+			#clock-cells = <0>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie4_phy: phy@8c000 {
+			compatible = "qcom,ipq9650-qmp-gen3x1-pcie-phy";
+			reg = <0x0 0x0008c000 0x0 0x1000>;
+
+			clocks = <&gcc GCC_PCIE4_AUX_CLK>,
+				 <&gcc GCC_PCIE4_AHB_CLK>,
+				 <&gcc GCC_PCIE4_PIPE_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "pipe";
+
+			assigned-clocks = <&gcc GCC_PCIE4_AUX_CLK>;
+			assigned-clock-rates = <20000000>;
+
+			resets = <&gcc GCC_PCIE4_PHY_BCR>,
+				 <&gcc GCC_PCIE4PHY_PHY_BCR>;
+			reset-names = "phy",
+				      "common";
+
+			refgen-supply = <&refgen_pcie>;
+
+			clock-output-names = "pcie4_pipe_clk";
+			#clock-cells = <0>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie3_phy: phy@b4000 {
+			compatible = "qcom,ipq9650-qmp-gen3x2-pcie-phy";
+			reg = <0x0 0x000b4000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE3_AUX_CLK>,
+				 <&gcc GCC_PCIE3_AHB_CLK>,
+				 <&gcc GCC_PCIE3_PIPE_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "pipe";
+
+			assigned-clocks = <&gcc GCC_PCIE3_AUX_CLK>;
+			assigned-clock-rates = <20000000>;
+
+			resets = <&gcc GCC_PCIE3_PHY_BCR>,
+				 <&gcc GCC_PCIE3PHY_PHY_BCR>;
+			reset-names = "phy",
+				      "common";
+
+			refgen-supply = <&refgen_pcie>;
+
+			clock-output-names = "pcie3_pipe_clk";
+			#clock-cells = <0>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie1_phy: phy@f4000 {
+			compatible = "qcom,ipq9650-qmp-gen3x2-pcie-phy";
+			reg = <0x0 0x000f4000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE1_AUX_CLK>,
+				 <&gcc GCC_PCIE1_AHB_CLK>,
+				 <&gcc GCC_PCIE1_PIPE_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "pipe";
+
+			assigned-clocks = <&gcc GCC_PCIE1_AUX_CLK>;
+			assigned-clock-rates = <20000000>;
+
+			resets = <&gcc GCC_PCIE1_PHY_BCR>,
+				 <&gcc GCC_PCIE1PHY_PHY_BCR>;
+			reset-names = "phy",
+				      "common";
+
+			refgen-supply = <&refgen_pcie>;
+
+			clock-output-names = "pcie1_pipe_clk";
+			#clock-cells = <0>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie2_phy: phy@fc000 {
+			compatible = "qcom,ipq9650-qmp-gen3x2-pcie-phy";
+			reg = <0x0 0x000fc000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE2_AUX_CLK>,
+				 <&gcc GCC_PCIE2_AHB_CLK>,
+				 <&gcc GCC_PCIE2_PIPE_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "pipe";
+
+			assigned-clocks = <&gcc GCC_PCIE2_AUX_CLK>;
+			assigned-clock-rates = <20000000>;
+
+			resets = <&gcc GCC_PCIE2_PHY_BCR>,
+				 <&gcc GCC_PCIE2PHY_PHY_BCR>;
+			reset-names = "phy",
+				      "common";
+
+			refgen-supply = <&refgen_pcie>;
+
+			clock-output-names = "pcie2_pipe_clk";
+			#clock-cells = <0>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
 		tlmm: pinctrl@1000000 {
 			compatible = "qcom,ipq9650-tlmm";
 			reg = <0x0 0x01000000 0x0 0x300000>;
@@ -216,15 +371,16 @@
 			reg = <0x0 0x01800000 0x0 0x40000>;
 			clocks = <&xo_board>,
 				 <&sleep_clk>,
-				 <0>,
-				 <0>,
-				 <0>,
-				 <0>,
-				 <0>,
+				 <&pcie0_phy>,
+				 <&pcie1_phy>,
+				 <&pcie2_phy>,
+				 <&pcie3_phy>,
+				 <&pcie4_phy>,
 				 <0>,
 				 <0>;
 			#clock-cells = <1>;
 			#reset-cells = <1>;
+			#interconnect-cells = <1>;
 		};
 
 		tcsr_mutex: hwlock@1917000 {
@@ -371,6 +527,506 @@
 				status = "disabled";
 			};
 		};
+
+		pcie3: pcie@30000000 {
+			compatible = "qcom,pcie-ipq9650", "qcom,pcie-ipq9574";
+			reg =  <0x0 0x30000000 0x0 0xf1d>,
+			       <0x0 0x30000f20 0x0 0xa8>,
+			       <0x0 0x30001000 0x0 0x1000>,
+			       <0x0 0x000b0000 0x0 0x3000>,
+			       <0x0 0x30100000 0x0 0x1000>,
+			       <0x0 0x000b6000 0x0 0x1000>;
+			reg-names = "dbi",
+				    "elbi",
+				    "atu",
+				    "parf",
+				    "config",
+				    "mhi";
+			device_type = "pci";
+			linux,pci-domain = <3>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x30200000 0x0 0x00100000>,
+				 <0x02000000 0x0 0x30300000 0x0 0x30300000 0x0 0x0fd00000>;
+
+			msi-map = <0x0 &intc 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 496 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 497 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 498 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 499 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 500 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 501 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 502 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 503 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 504 IRQ_TYPE_LEVEL_HIGH 0>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+			interrupt-map = <0 0 0 1 &intc GIC_SPI 505 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 2 &intc GIC_SPI 506 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 3 &intc GIC_SPI 507 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 4 &intc GIC_SPI 508 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gcc GCC_PCIE3_AXI_M_CLK>,
+				 <&gcc GCC_PCIE3_AXI_S_CLK>,
+				 <&gcc GCC_PCIE3_AXI_S_BRIDGE_CLK>,
+				 <&gcc GCC_PCIE3_RCHNG_CLK>,
+				 <&gcc GCC_PCIE3_AHB_CLK>,
+				 <&gcc GCC_PCIE3_AUX_CLK>;
+			clock-names = "axi_m",
+				      "axi_s",
+				      "axi_bridge",
+				      "rchng",
+				      "ahb",
+				      "aux";
+
+			resets = <&gcc GCC_PCIE3_PIPE_CLK_ARES>,
+				 <&gcc GCC_PCIE3_CORE_STICKY_RESET>,
+				 <&gcc GCC_PCIE3_AXI_S_STICKY_RESET>,
+				 <&gcc GCC_PCIE3_AXI_S_CLK_ARES>,
+				 <&gcc GCC_PCIE3_AXI_M_STICKY_RESET>,
+				 <&gcc GCC_PCIE3_AXI_M_CLK_ARES>,
+				 <&gcc GCC_PCIE3_AUX_CLK_ARES>,
+				 <&gcc GCC_PCIE3_AHB_CLK_ARES>;
+			reset-names = "pipe",
+				      "sticky",
+				      "axi_s_sticky",
+				      "axi_s",
+				      "axi_m_sticky",
+				      "axi_m",
+				      "aux",
+				      "ahb";
+
+			interconnects = <&gcc MASTER_ANOC_PCIE3 &gcc SLAVE_ANOC_PCIE3>,
+					<&gcc MASTER_CNOC_PCIE3 &gcc SLAVE_CNOC_PCIE3>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			status = "disabled";
+
+			pcie3_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie3_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie2: pcie@40000000 {
+			compatible = "qcom,pcie-ipq9650", "qcom,pcie-ipq9574";
+			reg =  <0x0 0x40000000 0x0 0xf1d>,
+			       <0x0 0x40000f20 0x0 0xa8>,
+			       <0x0 0x40001000 0x0 0x1000>,
+			       <0x0 0x000f8000 0x0 0x3000>,
+			       <0x0 0x40100000 0x0 0x1000>,
+			       <0x0 0x000fe000 0x0 0x1000>;
+			reg-names = "dbi",
+				    "elbi",
+				    "atu",
+				    "parf",
+				    "config",
+				    "mhi";
+			device_type = "pci";
+			linux,pci-domain = <2>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x00100000>,
+				 <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x0fd00000>;
+
+			msi-map = <0x0 &intc 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 470 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 471 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 475 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 476 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 477 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 478 IRQ_TYPE_LEVEL_HIGH 0>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc GIC_SPI 479 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 2 &intc GIC_SPI 480 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 3 &intc GIC_SPI 481 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 4 &intc GIC_SPI 482 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gcc GCC_PCIE2_AXI_M_CLK>,
+				 <&gcc GCC_PCIE2_AXI_S_CLK>,
+				 <&gcc GCC_PCIE2_AXI_S_BRIDGE_CLK>,
+				 <&gcc GCC_PCIE2_RCHNG_CLK>,
+				 <&gcc GCC_PCIE2_AHB_CLK>,
+				 <&gcc GCC_PCIE2_AUX_CLK>;
+			clock-names = "axi_m",
+				      "axi_s",
+				      "axi_bridge",
+				      "rchng",
+				      "ahb",
+				      "aux";
+
+			resets = <&gcc GCC_PCIE2_PIPE_CLK_ARES>,
+				 <&gcc GCC_PCIE2_CORE_STICKY_RESET>,
+				 <&gcc GCC_PCIE2_AXI_S_STICKY_RESET>,
+				 <&gcc GCC_PCIE2_AXI_S_CLK_ARES>,
+				 <&gcc GCC_PCIE2_AXI_M_STICKY_RESET>,
+				 <&gcc GCC_PCIE2_AXI_M_CLK_ARES>,
+				 <&gcc GCC_PCIE2_AUX_CLK_ARES>,
+				 <&gcc GCC_PCIE2_AHB_CLK_ARES>;
+			reset-names = "pipe",
+				      "sticky",
+				      "axi_s_sticky",
+				      "axi_s",
+				      "axi_m_sticky",
+				      "axi_m",
+				      "aux",
+				      "ahb";
+
+			interconnects = <&gcc MASTER_ANOC_PCIE2 &gcc SLAVE_ANOC_PCIE2>,
+					<&gcc MASTER_CNOC_PCIE2 &gcc SLAVE_CNOC_PCIE2>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			status = "disabled";
+
+			pcie2_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie2_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie1: pcie@50000000 {
+			compatible = "qcom,pcie-ipq9650", "qcom,pcie-ipq9574";
+			reg =  <0x0 0x50000000 0x0 0xf1d>,
+			       <0x0 0x50000f20 0x0 0xa8>,
+			       <0x0 0x50001000 0x0 0x1000>,
+			       <0x0 0x000f0000 0x0 0x3000>,
+			       <0x0 0x50100000 0x0 0x1000>,
+			       <0x0 0x000f6000 0x0 0x1000>;
+			reg-names = "dbi",
+				    "elbi",
+				    "atu",
+				    "parf",
+				    "config",
+				    "mhi";
+			device_type = "pci";
+			linux,pci-domain = <1>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x50200000 0x0 0x00100000>,
+				 <0x02000000 0x0 0x50300000 0x0 0x50300000 0x0 0x0fd00000>;
+
+			msi-map = <0x0 &intc 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 455 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH 0>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 2 &intc GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 3 &intc GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 4 &intc GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gcc GCC_PCIE1_AXI_M_CLK>,
+				 <&gcc GCC_PCIE1_AXI_S_CLK>,
+				 <&gcc GCC_PCIE1_AXI_S_BRIDGE_CLK>,
+				 <&gcc GCC_PCIE1_RCHNG_CLK>,
+				 <&gcc GCC_PCIE1_AHB_CLK>,
+				 <&gcc GCC_PCIE1_AUX_CLK>;
+			clock-names = "axi_m",
+				      "axi_s",
+				      "axi_bridge",
+				      "rchng",
+				      "ahb",
+				      "aux";
+
+			resets = <&gcc GCC_PCIE1_PIPE_CLK_ARES>,
+				 <&gcc GCC_PCIE1_CORE_STICKY_RESET>,
+				 <&gcc GCC_PCIE1_AXI_S_STICKY_RESET>,
+				 <&gcc GCC_PCIE1_AXI_S_CLK_ARES>,
+				 <&gcc GCC_PCIE1_AXI_M_STICKY_RESET>,
+				 <&gcc GCC_PCIE1_AXI_M_CLK_ARES>,
+				 <&gcc GCC_PCIE1_AUX_CLK_ARES>,
+				 <&gcc GCC_PCIE1_AHB_CLK_ARES>;
+			reset-names = "pipe",
+				      "sticky",
+				      "axi_s_sticky",
+				      "axi_s",
+				      "axi_m_sticky",
+				      "axi_m",
+				      "aux",
+				      "ahb";
+
+			interconnects = <&gcc MASTER_ANOC_PCIE1 &gcc SLAVE_ANOC_PCIE1>,
+					<&gcc MASTER_CNOC_PCIE1 &gcc SLAVE_CNOC_PCIE1>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			status = "disabled";
+
+			pcie1_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie1_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie4: pcie@60000000 {
+			compatible = "qcom,pcie-ipq9650", "qcom,pcie-ipq9574";
+			reg =  <0x0 0x60000000 0x0 0xf1d>,
+			       <0x0 0x60000f20 0x0 0xa8>,
+			       <0x0 0x60001000 0x0 0x1000>,
+			       <0x0 0x00088000 0x0 0x3000>,
+			       <0x0 0x60100000 0x0 0x1000>,
+			       <0x0 0x0008e000 0x0 0x1000>;
+			reg-names = "dbi",
+				    "elbi",
+				    "atu",
+				    "parf",
+				    "config",
+				    "mhi";
+			device_type = "pci";
+			linux,pci-domain = <4>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x00100000>,
+				 <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x0fd00000>;
+
+			msi-map = <0x0 &intc 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 511 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 512 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 513 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 514 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 515 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 516 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 517 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 518 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 519 IRQ_TYPE_LEVEL_HIGH 0>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc GIC_SPI 520 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 2 &intc GIC_SPI 521 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 3 &intc GIC_SPI 522 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 4 &intc GIC_SPI 523 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gcc GCC_PCIE4_AXI_M_CLK>,
+				 <&gcc GCC_PCIE4_AXI_S_CLK>,
+				 <&gcc GCC_PCIE4_AXI_S_BRIDGE_CLK>,
+				 <&gcc GCC_PCIE4_RCHNG_CLK>,
+				 <&gcc GCC_PCIE4_AHB_CLK>,
+				 <&gcc GCC_PCIE4_AUX_CLK>;
+			clock-names = "axi_m",
+				      "axi_s",
+				      "axi_bridge",
+				      "rchng",
+				      "ahb",
+				      "aux";
+
+			resets = <&gcc GCC_PCIE4_PIPE_CLK_ARES>,
+				 <&gcc GCC_PCIE4_CORE_STICKY_RESET>,
+				 <&gcc GCC_PCIE4_AXI_S_STICKY_RESET>,
+				 <&gcc GCC_PCIE4_AXI_S_CLK_ARES>,
+				 <&gcc GCC_PCIE4_AXI_M_STICKY_RESET>,
+				 <&gcc GCC_PCIE4_AXI_M_CLK_ARES>,
+				 <&gcc GCC_PCIE4_AUX_CLK_ARES>,
+				 <&gcc GCC_PCIE4_AHB_CLK_ARES>;
+			reset-names = "pipe",
+				      "sticky",
+				      "axi_s_sticky",
+				      "axi_s",
+				      "axi_m_sticky",
+				      "axi_m",
+				      "aux",
+				      "ahb";
+
+			interconnects = <&gcc MASTER_ANOC_PCIE4 &gcc SLAVE_ANOC_PCIE4>,
+					<&gcc MASTER_CNOC_PCIE4 &gcc SLAVE_CNOC_PCIE4>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			status = "disabled";
+
+			pcie4_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie4_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie0: pcie@70000000 {
+			compatible = "qcom,pcie-ipq9650", "qcom,pcie-ipq9574";
+			reg =  <0x0 0x70000000 0x0 0xf1d>,
+			       <0x0 0x70000f20 0x0 0xa8>,
+			       <0x0 0x70001000 0x0 0x1000>,
+			       <0x0 0x00080000 0x0 0x3000>,
+			       <0x0 0x70100000 0x0 0x1000>,
+			       <0x0 0x00086000 0x0 0x1000>;
+			reg-names = "dbi",
+				    "elbi",
+				    "atu",
+				    "parf",
+				    "config",
+				    "mhi";
+			device_type = "pci";
+			linux,pci-domain = <0>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x70200000 0x0 0x00100000>,
+				 <0x02000000 0x0 0x70300000 0x0 0x70300000 0x0 0x0fd00000>;
+
+			msi-map = <0x0 &intc 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 432 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 433 IRQ_TYPE_LEVEL_HIGH 0>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 3 &intc GIC_SPI 436 IRQ_TYPE_LEVEL_HIGH 0>,
+					<0 0 0 4 &intc GIC_SPI 437 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gcc GCC_PCIE0_AXI_M_CLK>,
+				 <&gcc GCC_PCIE0_AXI_S_CLK>,
+				 <&gcc GCC_PCIE0_AXI_S_BRIDGE_CLK>,
+				 <&gcc GCC_PCIE0_RCHNG_CLK>,
+				 <&gcc GCC_PCIE0_AHB_CLK>,
+				 <&gcc GCC_PCIE0_AUX_CLK>;
+			clock-names = "axi_m",
+				      "axi_s",
+				      "axi_bridge",
+				      "rchng",
+				      "ahb",
+				      "aux";
+
+			resets = <&gcc GCC_PCIE0_PIPE_CLK_ARES>,
+				 <&gcc GCC_PCIE0_CORE_STICKY_RESET>,
+				 <&gcc GCC_PCIE0_AXI_S_STICKY_RESET>,
+				 <&gcc GCC_PCIE0_AXI_S_CLK_ARES>,
+				 <&gcc GCC_PCIE0_AXI_M_STICKY_RESET>,
+				 <&gcc GCC_PCIE0_AXI_M_CLK_ARES>,
+				 <&gcc GCC_PCIE0_AUX_CLK_ARES>,
+				 <&gcc GCC_PCIE0_AHB_CLK_ARES>;
+			reset-names = "pipe",
+				      "sticky",
+				      "axi_s_sticky",
+				      "axi_s",
+				      "axi_m_sticky",
+				      "axi_m",
+				      "aux",
+				      "ahb";
+
+			interconnects = <&gcc MASTER_ANOC_PCIE0 &gcc SLAVE_ANOC_PCIE0>,
+					<&gcc MASTER_CNOC_PCIE0 &gcc SLAVE_CNOC_PCIE0>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			status = "disabled";
+
+			pcie0_port0: pcie@0 {
+				compatible = "pciclass,0604";
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie0_phy>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
index 6d817381bb3781..9eb02fd09dab6b 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
@@ -908,6 +908,14 @@
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/kaanapali/gen80200_zap.mbn";
+};
+
 &lpass_vamacro {
 	pinctrl-0 = <&dmic01_default>, <&dmic23_default>;
 	pinctrl-names = "default";
@@ -976,7 +984,7 @@
 };
 
 &pcie_port0 {
-	wake-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 104 GPIO_ACTIVE_LOW>;
 	reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
 
 	wifi@0 {
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
index abd3e9f8a3ebcd..f6b02223f4dfb2 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
@@ -736,6 +736,14 @@
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/kaanapali/gen80200_zap.mbn";
+};
+
 &pmh0101_flash {
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
index 131fd2a16c545d..146eaf9f0b9a9e 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi
+++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
@@ -26,6 +26,7 @@
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+#include <dt-bindings/thermal/thermal.h>
 
 #include "kaanapali-ipcc.h"
 
@@ -1866,7 +1867,7 @@
 				status = "disabled";
 			};
 
-			i2c16: i2c@198c000  {
+			i2c16: i2c@198c000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0x0 0x0198c000 0x0 0x4000>;
 				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
@@ -2678,6 +2679,246 @@
 			#power-domain-cells = <1>;
 		};
 
+		gpu: gpu@3d00000 {
+			compatible = "qcom,adreno-44050a01", "qcom,adreno";
+			reg = <0x0 0x03d00000 0x0 0x6c000>,
+			      <0x0 0x03d9e000 0x0 0x2000>;
+			reg-names = "kgsl_3d0_reg_memory",
+				    "cx_mem";
+
+			interrupts = <GIC_SPI 300 IRQ_TYPE_LEVEL_HIGH>;
+
+			iommus = <&adreno_smmu 0 0x0>,
+				 <&adreno_smmu 1 0x0>;
+
+			operating-points-v2 = <&gpu_opp_table>;
+
+			qcom,gmu = <&gmu>;
+			#cooling-cells = <2>;
+
+			nvmem-cells = <&gpu_speed_bin>;
+			nvmem-cell-names = "speed_bin";
+
+			interconnects = <&gem_noc MASTER_GFX3D QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "gfx-mem";
+
+			gpu_zap_shader: zap-shader {
+				memory-region = <&gpu_microcode_mem>;
+			};
+
+			gpu_opp_table: opp-table {
+				compatible = "operating-points-v2-adreno",
+					     "operating-points-v2";
+
+				opp-160000000 {
+					opp-hz = /bits/ 64 <160000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D3>;
+					opp-peak-kBps = <2136718>;
+					opp-supported-hw = <0x0f>;
+					/* ACD is disabled */
+				};
+
+				opp-191000000 {
+					opp-hz = /bits/ 64 <191000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2_1>;
+					opp-peak-kBps = <2136718>;
+					opp-supported-hw = <0x0f>;
+					/* ACD is disabled */
+				};
+
+				opp-222000000 {
+					opp-hz = /bits/ 64 <222000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2>;
+					opp-peak-kBps = <2136718>;
+					opp-supported-hw = <0x0f>;
+					/* ACD is disabled */
+				};
+
+				opp-282000000 {
+					opp-hz = /bits/ 64 <282000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1_1>;
+					opp-peak-kBps = <5285156>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xca2e5ffd>;
+				};
+
+				opp-342000000 {
+					opp-hz = /bits/ 64 <342000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					opp-peak-kBps = <5285156>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xe22a5ffd>;
+				};
+
+				opp-382000000 {
+					opp-hz = /bits/ 64 <382000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0>;
+					opp-peak-kBps = <5285156>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xa22c5ffd>;
+				};
+
+				opp-422000000 {
+					opp-hz = /bits/ 64 <422000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					opp-peak-kBps = <6074218>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xa22c5ffd>;
+				};
+
+				opp-461000000 {
+					opp-hz = /bits/ 64 <461000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L0>;
+					opp-peak-kBps = <6074218>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xe82e5ffd>;
+				};
+
+				opp-500000000 {
+					opp-hz = /bits/ 64 <500000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L1>;
+					opp-peak-kBps = <6074218>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xe82c5ffd>;
+				};
+
+				opp-578000000 {
+					opp-hz = /bits/ 64 <578000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					opp-peak-kBps = <6074218>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xc02c5ffd>;
+				};
+
+				opp-646000000 {
+					opp-hz = /bits/ 64 <646000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L0>;
+					opp-peak-kBps = <8171875>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xc02c5ffd>;
+				};
+
+				opp-726000000 {
+					opp-hz = /bits/ 64 <726000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					opp-peak-kBps = <8171875>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0x882f5ffd>;
+				};
+
+				opp-826000000 {
+					opp-hz = /bits/ 64 <826000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
+					opp-peak-kBps = <12449218>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xa82c5ffd>;
+				};
+
+				opp-902000000 {
+					opp-hz = /bits/ 64 <902000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					opp-peak-kBps = <12449218>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0xa82b5ffd>;
+				};
+
+				opp-1000000000 {
+					opp-hz = /bits/ 64 <1000000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+					opp-peak-kBps = <12449218>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0x882a5ffd>;
+				};
+
+				opp-1050000000 {
+					opp-hz = /bits/ 64 <1050000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					opp-peak-kBps = <20832031>;
+					opp-supported-hw = <0x0f>;
+					qcom,opp-acd-level = <0x88295ffd>;
+				};
+
+				opp-1200000000 {
+					opp-hz = /bits/ 64 <1200000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3>;
+					opp-peak-kBps = <20832031>;
+					opp-supported-hw = <0x07>;
+					qcom,opp-acd-level = <0xa02e5ffd>;
+				};
+
+				opp-1300000000 {
+					opp-hz = /bits/ 64 <1300000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L4>;
+					opp-peak-kBps = <20832031>;
+					opp-supported-hw = <0x03>;
+					qcom,opp-acd-level = <0x802d5ffd>;
+				};
+			};
+		};
+
+		gmu: gmu@3d6c000 {
+			compatible = "qcom,adreno-gmu-840.1", "qcom,adreno-gmu";
+
+			reg = <0x0 0x03d6c000 0x0 0x68000>;
+			reg-names = "gmu";
+
+			interrupts = <GIC_SPI 304 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 305 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hfi", "gmu";
+
+			clocks = <&gpucc GPU_CC_AHB_CLK>,
+				 <&gpucc GPU_CC_CX_GMU_CLK>,
+				 <&gpucc GPU_CC_CXO_CLK>,
+				 <&gcc GCC_GPU_GEMNOC_GFX_CLK>,
+				 <&gpucc GPU_CC_HUB_CX_INT_CLK>;
+			clock-names = "ahb",
+				      "gmu",
+				      "cxo",
+				      "memnoc",
+				      "hub";
+
+			power-domains = <&gpucc GPU_CC_CX_GDSC>,
+					<&gxclkctl GX_CLKCTL_GX_GDSC>;
+			power-domain-names = "cx",
+					     "gx";
+
+			iommus = <&adreno_smmu 5 0x0>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			operating-points-v2 = <&gmu_opp_table>;
+
+			gmu_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-475000000 {
+					opp-hz = /bits/ 64 <475000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+				};
+
+				opp-575000000 {
+					opp-hz = /bits/ 64 <575000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+				};
+
+				opp-700000000 {
+					opp-hz = /bits/ 64 <700000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+				};
+
+				opp-725000000 {
+					opp-hz = /bits/ 64 <725000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+				};
+
+				opp-750000000 {
+					opp-hz = /bits/ 64 <750000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+				};
+			};
+		};
+
 		gxclkctl: clock-controller@3d64000 {
 			compatible = "qcom,kaanapali-gxclkctl";
 			reg = <0x0 0x03d64000 0x0 0x6000>;
@@ -2702,6 +2943,46 @@
 			#power-domain-cells = <1>;
 		};
 
+		adreno_smmu: iommu@3da0000 {
+			compatible = "qcom,kaanapali-smmu-500",
+				     "qcom,adreno-smmu",
+				     "qcom,smmu-500",
+				     "arm,mmu-500";
+			reg = <0x0 0x03da0000 0x0 0x40000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			interrupts = <GIC_SPI 674 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 678 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 679 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 680 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 681 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 682 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 683 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 684 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 685 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 686 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 687 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 688 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 476 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 574 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 575 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 576 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 577 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 660 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 662 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 665 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 666 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 667 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 669 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 670 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 700 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gpucc GPU_CC_GPU_SMMU_VOTE_CLK>;
+			clock-names = "hlos";
+			power-domains = <&gpucc GPU_CC_CX_GDSC>;
+			dma-coherent;
+		};
+
 		remoteproc_adsp: remoteproc@6800000 {
 			compatible = "qcom,kaanapali-adsp-pas", "qcom,sm8550-adsp-pas";
 			reg = <0x0 0x06800000 0x0 0x10000>;
@@ -4360,11 +4641,11 @@
 			};
 
 			qup_uart7_default: qup-uart7-state {
-				 /* TX, RX */
-				 pins = "gpio62", "gpio63";
-				 function = "qup1_se7";
-				 drive-strength = <2>;
-				 bias-disable;
+				/* TX, RX */
+				pins = "gpio62", "gpio63";
+				function = "qup1_se7";
+				drive-strength = <2>;
+				bias-disable;
 			};
 
 			qup_uart18_default: qup-uart18-default-state {
@@ -5970,6 +6251,18 @@
 			};
 		};
 
+		efuse@221c8000 {
+			compatible = "qcom,kaanapali-qfprom", "qcom,qfprom";
+			reg = <0x0 0x221c8000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			gpu_speed_bin: gpu-speed-bin@142 {
+				reg = <0x142 0x2>;
+				bits = <3 9>;
+			};
+		};
+
 		nsp_noc: interconnect@260c0000 {
 			compatible = "qcom,kaanapali-nsp-noc";
 			reg = <0x0 0x260c0000 0x0 0x21280>;
@@ -6903,9 +7196,17 @@
 		};
 
 		gpuss-0-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 0>;
 
 			trips {
+				gpuss_0_alert0: gpuss-0-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-0-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -6918,12 +7219,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_0_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-1-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 1>;
 
 			trips {
+				gpuss_1_alert0: gpuss-1-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-1-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -6936,12 +7252,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_1_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-2-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 2>;
 
 			trips {
+				gpuss_2_alert0: gpuss-2-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-2-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -6954,12 +7285,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_2_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-3-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 3>;
 
 			trips {
+				gpuss_3_alert0: gpuss-3-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-3-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -6972,12 +7318,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_3_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-4-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 4>;
 
 			trips {
+				gpuss_4_alert0: gpuss-4-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-4-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -6990,12 +7351,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_4_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-5-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 5>;
 
 			trips {
+				gpuss_5_alert0: gpuss-5-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-5-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -7008,12 +7384,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_5_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-6-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 6>;
 
 			trips {
+				gpuss_6_alert0: gpuss-6-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-6-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -7026,12 +7417,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_6_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-7-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 7>;
 
 			trips {
+				gpuss_7_alert0: gpuss-7-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-7-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -7044,12 +7450,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_7_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-8-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 8>;
 
 			trips {
+				gpuss_8_alert0: gpuss-8-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-8-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -7062,12 +7483,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_8_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-9-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 9>;
 
 			trips {
+				gpuss_9_alert0: gpuss-9-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-9-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -7080,12 +7516,27 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_9_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss-10-thermal {
+			polling-delay-passive = <10>;
+
 			thermal-sensors = <&tsens5 10>;
 
 			trips {
+				gpuss_10_alert0: gpuss-10-alert0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
 				gpuss-10-hot {
 					temperature = <120000>;
 					hysteresis = <5000>;
@@ -7098,6 +7549,13 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_10_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		ddr-thermal {
diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi
index f2da3706d5c887..8c7523602d2719 100644
--- a/arch/arm64/boot/dts/qcom/kodiak.dtsi
+++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi
@@ -1054,6 +1054,8 @@
 			qcom,dll-config = <0x0007642c>;
 			qcom,ddr-config = <0x80040868>;
 
+			qcom,ice = <&sdhc_ice>;
+
 			mmc-ddr-1_8v;
 			mmc-hs200-1_8v;
 			mmc-hs400-1_8v;
@@ -1080,6 +1082,17 @@
 			};
 		};
 
+		sdhc_ice: crypto@7c8000 {
+			compatible = "qcom,sc7280-inline-crypto-engine",
+				     "qcom,inline-crypto-engine";
+			reg = <0x0 0x007c8000 0x0 0x18000>;
+			clocks = <&gcc GCC_SDCC1_ICE_CORE_CLK>,
+				 <&gcc GCC_SDCC1_AHB_CLK>;
+			clock-names = "core",
+				      "iface";
+			power-domains = <&rpmhpd SC7280_CX>;
+		};
+
 		gpi_dma0: dma-controller@900000 {
 			#dma-cells = <3>;
 			compatible = "qcom,sc7280-gpi-dma", "qcom,sm6350-gpi-dma";
@@ -5852,142 +5865,124 @@
 			gpio-ranges = <&tlmm 0 0 175>;
 			wakeup-parent = <&pdc>;
 
-			cam_mclk3_default: cam-mclk3-default-state {
-				pins = "gpio67";
-				function = "cam_mclk";
-				drive-strength = <2>;
-				bias-disable;
+			qup_i2c0_data_clk: qup-i2c0-data-clk-state {
+				pins = "gpio0", "gpio1";
+				function = "qup00";
 			};
 
-			cci0_default: cci0-default-state {
-				pins = "gpio69", "gpio70";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-up;
+			qup_spi0_data_clk: qup-spi0-data-clk-state {
+				pins = "gpio0", "gpio1", "gpio2";
+				function = "qup00";
 			};
 
-			cci0_sleep: cci0-sleep-state {
-				pins = "gpio69", "gpio70";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-down;
+			qup_uart0_cts: qup-uart0-cts-state {
+				pins = "gpio0";
+				function = "qup00";
 			};
 
-			cci1_default: cci1-default-state {
-				pins = "gpio71", "gpio72";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-up;
+			qup_uart0_rts: qup-uart0-rts-state {
+				pins = "gpio1";
+				function = "qup00";
 			};
 
-			cci1_sleep: cci1-sleep-state {
-				pins = "gpio71", "gpio72";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-down;
+			qup_uart0_tx: qup-uart0-tx-state {
+				pins = "gpio2";
+				function = "qup00";
 			};
 
-			cci2_default: cci2-default-state {
-				pins = "gpio73", "gpio74";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-up;
+			qup_spi0_cs: qup-spi0-cs-state {
+				pins = "gpio3";
+				function = "qup00";
 			};
 
-			cci2_sleep: cci2-sleep-state {
-				pins = "gpio73", "gpio74";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-down;
+			qup_spi0_cs_gpio: qup-spi0-cs-gpio-state {
+				pins = "gpio3";
+				function = "gpio";
 			};
 
-			cci3_default: cci3-default-state {
-				pins = "gpio75", "gpio76";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-up;
+			qup_uart0_rx: qup-uart0-rx-state {
+				pins = "gpio3";
+				function = "qup00";
 			};
 
-			cci3_sleep: cci3-sleep-state {
-				pins = "gpio75", "gpio76";
-				function = "cci_i2c";
-				drive-strength = <2>;
-				bias-pull-down;
+			qup_i2c1_data_clk: qup-i2c1-data-clk-state {
+				pins = "gpio4", "gpio5";
+				function = "qup01";
 			};
 
-			dp_hot_plug_det: dp-hot-plug-det-state {
-				pins = "gpio47";
-				function = "dp_hot";
+			qup_spi1_data_clk: qup-spi1-data-clk-state {
+				pins = "gpio4", "gpio5", "gpio6";
+				function = "qup01";
 			};
 
-			edp_hot_plug_det: edp-hot-plug-det-state {
-				pins = "gpio60";
-				function = "edp_hot";
+			qup_uart1_cts: qup-uart1-cts-state {
+				pins = "gpio4";
+				function = "qup01";
 			};
 
-			mi2s0_data0: mi2s0-data0-state {
-				pins = "gpio98";
-				function = "mi2s0_data0";
+			qup_uart1_rts: qup-uart1-rts-state {
+				pins = "gpio5";
+				function = "qup01";
 			};
 
-			mi2s0_data1: mi2s0-data1-state {
-				pins = "gpio99";
-				function = "mi2s0_data1";
+			qup_uart1_tx: qup-uart1-tx-state {
+				pins = "gpio6";
+				function = "qup01";
 			};
 
-			mi2s0_mclk: mi2s0-mclk-state {
-				pins = "gpio96";
-				function = "pri_mi2s";
+			qup_spi1_cs: qup-spi1-cs-state {
+				pins = "gpio7";
+				function = "qup01";
 			};
 
-			mi2s0_sclk: mi2s0-sclk-state {
-				pins = "gpio97";
-				function = "mi2s0_sck";
+			qup_spi1_cs_gpio: qup-spi1-cs-gpio-state {
+				pins = "gpio7";
+				function = "gpio";
 			};
 
-			mi2s0_ws: mi2s0-ws-state {
-				pins = "gpio100";
-				function = "mi2s0_ws";
+			qup_uart1_rx: qup-uart1-rx-state {
+				pins = "gpio7";
+				function = "qup01";
 			};
 
-			mi2s1_data0: mi2s1-data0-state {
-				pins = "gpio107";
-				function = "mi2s1_data0";
+			qup_i2c2_data_clk: qup-i2c2-data-clk-state {
+				pins = "gpio8", "gpio9";
+				function = "qup02";
 			};
 
-			mi2s1_sclk: mi2s1-sclk-state {
-				pins = "gpio106";
-				function = "mi2s1_sck";
+			qup_spi2_data_clk: qup-spi2-data-clk-state {
+				pins = "gpio8", "gpio9", "gpio10";
+				function = "qup02";
 			};
 
-			mi2s1_ws: mi2s1-ws-state {
-				pins = "gpio108";
-				function = "mi2s1_ws";
+			qup_uart2_cts: qup-uart2-cts-state {
+				pins = "gpio8";
+				function = "qup02";
 			};
 
-			pcie0_clkreq_n: pcie0-clkreq-n-state {
-				pins = "gpio88";
-				function = "pcie0_clkreqn";
+			qup_uart2_rts: qup-uart2-rts-state {
+				pins = "gpio9";
+				function = "qup02";
 			};
 
-			pcie1_clkreq_n: pcie1-clkreq-n-state {
-				pins = "gpio79";
-				function = "pcie1_clkreqn";
+			qup_uart2_tx: qup-uart2-tx-state {
+				pins = "gpio10";
+				function = "qup02";
 			};
 
-			qspi_clk: qspi-clk-state {
-				pins = "gpio14";
-				function = "qspi_clk";
+			qup_spi2_cs: qup-spi2-cs-state {
+				pins = "gpio11";
+				function = "qup02";
 			};
 
-			qspi_cs0: qspi-cs0-state {
-				pins = "gpio15";
-				function = "qspi_cs";
+			qup_spi2_cs_gpio: qup-spi2-cs-gpio-state {
+				pins = "gpio11";
+				function = "gpio";
 			};
 
-			qspi_cs1: qspi-cs1-state {
-				pins = "gpio19";
-				function = "qspi_cs";
+			qup_uart2_rx: qup-uart2-rx-state {
+				pins = "gpio11";
+				function = "qup02";
 			};
 
 			qspi_data0: qspi-data0-state {
@@ -5995,203 +5990,203 @@
 				function = "qspi_data";
 			};
 
+			qup_i2c3_data_clk: qup-i2c3-data-clk-state {
+				pins = "gpio12", "gpio13";
+				function = "qup03";
+			};
+
+			qup_spi3_data_clk: qup-spi3-data-clk-state {
+				pins = "gpio12", "gpio13", "gpio14";
+				function = "qup03";
+			};
+
+			qup_uart3_cts: qup-uart3-cts-state {
+				pins = "gpio12";
+				function = "qup03";
+			};
+
 			qspi_data1: qspi-data1-state {
 				pins = "gpio13";
 				function = "qspi_data";
 			};
 
-			qspi_data23: qspi-data23-state {
-				pins = "gpio16", "gpio17";
-				function = "qspi_data";
+			qup_uart3_rts: qup-uart3-rts-state {
+				pins = "gpio13";
+				function = "qup03";
 			};
 
-			qup_i2c0_data_clk: qup-i2c0-data-clk-state {
-				pins = "gpio0", "gpio1";
-				function = "qup00";
+			qspi_clk: qspi-clk-state {
+				pins = "gpio14";
+				function = "qspi_clk";
 			};
 
-			qup_i2c1_data_clk: qup-i2c1-data-clk-state {
-				pins = "gpio4", "gpio5";
-				function = "qup01";
+			qup_uart3_tx: qup-uart3-tx-state {
+				pins = "gpio14";
+				function = "qup03";
 			};
 
-			qup_i2c2_data_clk: qup-i2c2-data-clk-state {
-				pins = "gpio8", "gpio9";
-				function = "qup02";
+			qspi_cs0: qspi-cs0-state {
+				pins = "gpio15";
+				function = "qspi_cs";
 			};
 
-			qup_i2c3_data_clk: qup-i2c3-data-clk-state {
-				pins = "gpio12", "gpio13";
+			qup_spi3_cs: qup-spi3-cs-state {
+				pins = "gpio15";
 				function = "qup03";
 			};
 
-			qup_i2c4_data_clk: qup-i2c4-data-clk-state {
-				pins = "gpio16", "gpio17";
-				function = "qup04";
+			qup_spi3_cs_gpio: qup-spi3-cs-gpio-state {
+				pins = "gpio15";
+				function = "gpio";
 			};
 
-			qup_i2c5_data_clk: qup-i2c5-data-clk-state {
-				pins = "gpio20", "gpio21";
-				function = "qup05";
+			qup_uart3_rx: qup-uart3-rx-state {
+				pins = "gpio15";
+				function = "qup03";
 			};
 
-			qup_i2c6_data_clk: qup-i2c6-data-clk-state {
-				pins = "gpio24", "gpio25";
-				function = "qup06";
+			qspi_data23: qspi-data23-state {
+				pins = "gpio16", "gpio17";
+				function = "qspi_data";
 			};
 
-			qup_i2c7_data_clk: qup-i2c7-data-clk-state {
-				pins = "gpio28", "gpio29";
-				function = "qup07";
+			qup_i2c4_data_clk: qup-i2c4-data-clk-state {
+				pins = "gpio16", "gpio17";
+				function = "qup04";
 			};
 
-			qup_i2c8_data_clk: qup-i2c8-data-clk-state {
-				pins = "gpio32", "gpio33";
-				function = "qup10";
+			qup_spi4_data_clk: qup-spi4-data-clk-state {
+				pins = "gpio16", "gpio17", "gpio18";
+				function = "qup04";
 			};
 
-			qup_i2c9_data_clk: qup-i2c9-data-clk-state {
-				pins = "gpio36", "gpio37";
-				function = "qup11";
+			qup_uart4_cts: qup-uart4-cts-state {
+				pins = "gpio16";
+				function = "qup04";
 			};
 
-			qup_i2c10_data_clk: qup-i2c10-data-clk-state {
-				pins = "gpio40", "gpio41";
-				function = "qup12";
+			qup_uart4_rts: qup-uart4-rts-state {
+				pins = "gpio17";
+				function = "qup04";
 			};
 
-			qup_i2c11_data_clk: qup-i2c11-data-clk-state {
-				pins = "gpio44", "gpio45";
-				function = "qup13";
-			};
-
-			qup_i2c12_data_clk: qup-i2c12-data-clk-state {
-				pins = "gpio48", "gpio49";
-				function = "qup14";
+			qup_uart4_tx: qup-uart4-tx-state {
+				pins = "gpio18";
+				function = "qup04";
 			};
 
-			qup_i2c13_data_clk: qup-i2c13-data-clk-state {
-				pins = "gpio52", "gpio53";
-				function = "qup15";
+			qspi_cs1: qspi-cs1-state {
+				pins = "gpio19";
+				function = "qspi_cs";
 			};
 
-			qup_i2c14_data_clk: qup-i2c14-data-clk-state {
-				pins = "gpio56", "gpio57";
-				function = "qup16";
+			qup_spi4_cs: qup-spi4-cs-state {
+				pins = "gpio19";
+				function = "qup04";
 			};
 
-			qup_i2c15_data_clk: qup-i2c15-data-clk-state {
-				pins = "gpio60", "gpio61";
-				function = "qup17";
+			qup_spi4_cs_gpio: qup-spi4-cs-gpio-state {
+				pins = "gpio19";
+				function = "gpio";
 			};
 
-			qup_spi0_data_clk: qup-spi0-data-clk-state {
-				pins = "gpio0", "gpio1", "gpio2";
-				function = "qup00";
+			qup_uart4_rx: qup-uart4-rx-state {
+				pins = "gpio19";
+				function = "qup04";
 			};
 
-			qup_spi0_cs: qup-spi0-cs-state {
-				pins = "gpio3";
-				function = "qup00";
+			qup_i2c5_data_clk: qup-i2c5-data-clk-state {
+				pins = "gpio20", "gpio21";
+				function = "qup05";
 			};
 
-			qup_spi0_cs_gpio: qup-spi0-cs-gpio-state {
-				pins = "gpio3";
-				function = "gpio";
+			qup_spi5_data_clk: qup-spi5-data-clk-state {
+				pins = "gpio20", "gpio21", "gpio22";
+				function = "qup05";
 			};
 
-			qup_spi1_data_clk: qup-spi1-data-clk-state {
-				pins = "gpio4", "gpio5", "gpio6";
-				function = "qup01";
+			qup_uart5_tx: qup-uart5-tx-state {
+				pins = "gpio22";
+				function = "qup05";
 			};
 
-			qup_spi1_cs: qup-spi1-cs-state {
-				pins = "gpio7";
-				function = "qup01";
+			qup_spi5_cs: qup-spi5-cs-state {
+				pins = "gpio23";
+				function = "qup05";
 			};
 
-			qup_spi1_cs_gpio: qup-spi1-cs-gpio-state {
-				pins = "gpio7";
+			qup_spi5_cs_gpio: qup-spi5-cs-gpio-state {
+				pins = "gpio23";
 				function = "gpio";
 			};
 
-			qup_spi2_data_clk: qup-spi2-data-clk-state {
-				pins = "gpio8", "gpio9", "gpio10";
-				function = "qup02";
-			};
-
-			qup_spi2_cs: qup-spi2-cs-state {
-				pins = "gpio11";
-				function = "qup02";
+			qup_uart5_rx: qup-uart5-rx-state {
+				pins = "gpio23";
+				function = "qup05";
 			};
 
-			qup_spi2_cs_gpio: qup-spi2-cs-gpio-state {
-				pins = "gpio11";
-				function = "gpio";
+			qup_i2c6_data_clk: qup-i2c6-data-clk-state {
+				pins = "gpio24", "gpio25";
+				function = "qup06";
 			};
 
-			qup_spi3_data_clk: qup-spi3-data-clk-state {
-				pins = "gpio12", "gpio13", "gpio14";
-				function = "qup03";
+			qup_spi6_data_clk: qup-spi6-data-clk-state {
+				pins = "gpio24", "gpio25", "gpio26";
+				function = "qup06";
 			};
 
-			qup_spi3_cs: qup-spi3-cs-state {
-				pins = "gpio15";
-				function = "qup03";
+			qup_uart6_cts: qup-uart6-cts-state {
+				pins = "gpio24";
+				function = "qup06";
 			};
 
-			qup_spi3_cs_gpio: qup-spi3-cs-gpio-state {
-				pins = "gpio15";
-				function = "gpio";
+			qup_uart6_rts: qup-uart6-rts-state {
+				pins = "gpio25";
+				function = "qup06";
 			};
 
-			qup_spi4_data_clk: qup-spi4-data-clk-state {
-				pins = "gpio16", "gpio17", "gpio18";
-				function = "qup04";
+			qup_uart6_tx: qup-uart6-tx-state {
+				pins = "gpio26";
+				function = "qup06";
 			};
 
-			qup_spi4_cs: qup-spi4-cs-state {
-				pins = "gpio19";
-				function = "qup04";
+			qup_spi6_cs: qup-spi6-cs-state {
+				pins = "gpio27";
+				function = "qup06";
 			};
 
-			qup_spi4_cs_gpio: qup-spi4-cs-gpio-state {
-				pins = "gpio19";
+			qup_spi6_cs_gpio: qup-spi6-cs-gpio-state {
+				pins = "gpio27";
 				function = "gpio";
 			};
 
-			qup_spi5_data_clk: qup-spi5-data-clk-state {
-				pins = "gpio20", "gpio21", "gpio22";
-				function = "qup05";
-			};
-
-			qup_spi5_cs: qup-spi5-cs-state {
-				pins = "gpio23";
-				function = "qup05";
+			qup_uart6_rx: qup-uart6-rx-state {
+				pins = "gpio27";
+				function = "qup06";
 			};
 
-			qup_spi5_cs_gpio: qup-spi5-cs-gpio-state {
-				pins = "gpio23";
-				function = "gpio";
+			qup_i2c7_data_clk: qup-i2c7-data-clk-state {
+				pins = "gpio28", "gpio29";
+				function = "qup07";
 			};
 
-			qup_spi6_data_clk: qup-spi6-data-clk-state {
-				pins = "gpio24", "gpio25", "gpio26";
-				function = "qup06";
+			qup_spi7_data_clk: qup-spi7-data-clk-state {
+				pins = "gpio28", "gpio29", "gpio30";
+				function = "qup07";
 			};
 
-			qup_spi6_cs: qup-spi6-cs-state {
-				pins = "gpio27";
-				function = "qup06";
+			qup_uart7_cts: qup-uart7-cts-state {
+				pins = "gpio28";
+				function = "qup07";
 			};
 
-			qup_spi6_cs_gpio: qup-spi6-cs-gpio-state {
-				pins = "gpio27";
-				function = "gpio";
+			qup_uart7_rts: qup-uart7-rts-state {
+				pins = "gpio29";
+				function = "qup07";
 			};
 
-			qup_spi7_data_clk: qup-spi7-data-clk-state {
-				pins = "gpio28", "gpio29", "gpio30";
+			qup_uart7_tx: qup-uart7-tx-state {
+				pins = "gpio30";
 				function = "qup07";
 			};
 
@@ -6205,11 +6200,36 @@
 				function = "gpio";
 			};
 
+			qup_uart7_rx: qup-uart7-rx-state {
+				pins = "gpio31";
+				function = "qup07";
+			};
+
+			qup_i2c8_data_clk: qup-i2c8-data-clk-state {
+				pins = "gpio32", "gpio33";
+				function = "qup10";
+			};
+
 			qup_spi8_data_clk: qup-spi8-data-clk-state {
 				pins = "gpio32", "gpio33", "gpio34";
 				function = "qup10";
 			};
 
+			qup_uart8_cts: qup-uart8-cts-state {
+				pins = "gpio32";
+				function = "qup10";
+			};
+
+			qup_uart8_rts: qup-uart8-rts-state {
+				pins = "gpio33";
+				function = "qup10";
+			};
+
+			qup_uart8_tx: qup-uart8-tx-state {
+				pins = "gpio34";
+				function = "qup10";
+			};
+
 			qup_spi8_cs: qup-spi8-cs-state {
 				pins = "gpio35";
 				function = "qup10";
@@ -6220,11 +6240,36 @@
 				function = "gpio";
 			};
 
+			qup_uart8_rx: qup-uart8-rx-state {
+				pins = "gpio35";
+				function = "qup10";
+			};
+
+			qup_i2c9_data_clk: qup-i2c9-data-clk-state {
+				pins = "gpio36", "gpio37";
+				function = "qup11";
+			};
+
 			qup_spi9_data_clk: qup-spi9-data-clk-state {
 				pins = "gpio36", "gpio37", "gpio38";
 				function = "qup11";
 			};
 
+			qup_uart9_cts: qup-uart9-cts-state {
+				pins = "gpio36";
+				function = "qup11";
+			};
+
+			qup_uart9_rts: qup-uart9-rts-state {
+				pins = "gpio37";
+				function = "qup11";
+			};
+
+			qup_uart9_tx: qup-uart9-tx-state {
+				pins = "gpio38";
+				function = "qup11";
+			};
+
 			qup_spi9_cs: qup-spi9-cs-state {
 				pins = "gpio39";
 				function = "qup11";
@@ -6235,11 +6280,36 @@
 				function = "gpio";
 			};
 
+			qup_uart9_rx: qup-uart9-rx-state {
+				pins = "gpio39";
+				function = "qup11";
+			};
+
+			qup_i2c10_data_clk: qup-i2c10-data-clk-state {
+				pins = "gpio40", "gpio41";
+				function = "qup12";
+			};
+
 			qup_spi10_data_clk: qup-spi10-data-clk-state {
 				pins = "gpio40", "gpio41", "gpio42";
 				function = "qup12";
 			};
 
+			qup_uart10_cts: qup-uart10-cts-state {
+				pins = "gpio40";
+				function = "qup12";
+			};
+
+			qup_uart10_rts: qup-uart10-rts-state {
+				pins = "gpio41";
+				function = "qup12";
+			};
+
+			qup_uart10_tx: qup-uart10-tx-state {
+				pins = "gpio42";
+				function = "qup12";
+			};
+
 			qup_spi10_cs: qup-spi10-cs-state {
 				pins = "gpio43";
 				function = "qup12";
@@ -6250,389 +6320,367 @@
 				function = "gpio";
 			};
 
+			qup_uart10_rx: qup-uart10-rx-state {
+				pins = "gpio43";
+				function = "qup12";
+			};
+
+			qup_i2c11_data_clk: qup-i2c11-data-clk-state {
+				pins = "gpio44", "gpio45";
+				function = "qup13";
+			};
+
 			qup_spi11_data_clk: qup-spi11-data-clk-state {
 				pins = "gpio44", "gpio45", "gpio46";
 				function = "qup13";
 			};
 
-			qup_spi11_cs: qup-spi11-cs-state {
-				pins = "gpio47";
+			qup_uart11_cts: qup-uart11-cts-state {
+				pins = "gpio44";
 				function = "qup13";
 			};
 
-			qup_spi11_cs_gpio: qup-spi11-cs-gpio-state {
-				pins = "gpio47";
-				function = "gpio";
+			qup_uart11_rts: qup-uart11-rts-state {
+				pins = "gpio45";
+				function = "qup13";
 			};
 
-			qup_spi12_data_clk: qup-spi12-data-clk-state {
-				pins = "gpio48", "gpio49", "gpio50";
-				function = "qup14";
+			qup_uart11_tx: qup-uart11-tx-state {
+				pins = "gpio46";
+				function = "qup13";
 			};
 
-			qup_spi12_cs: qup-spi12-cs-state {
-				pins = "gpio51";
-				function = "qup14";
-			};
-
-			qup_spi12_cs_gpio: qup-spi12-cs-gpio-state {
-				pins = "gpio51";
-				function = "gpio";
-			};
-
-			qup_spi13_data_clk: qup-spi13-data-clk-state {
-				pins = "gpio52", "gpio53", "gpio54";
-				function = "qup15";
-			};
-
-			qup_spi13_cs: qup-spi13-cs-state {
-				pins = "gpio55";
-				function = "qup15";
-			};
-
-			qup_spi13_cs_gpio: qup-spi13-cs-gpio-state {
-				pins = "gpio55";
-				function = "gpio";
-			};
-
-			qup_spi14_data_clk: qup-spi14-data-clk-state {
-				pins = "gpio56", "gpio57", "gpio58";
-				function = "qup16";
-			};
-
-			qup_spi14_cs: qup-spi14-cs-state {
-				pins = "gpio59";
-				function = "qup16";
-			};
-
-			qup_spi14_cs_gpio: qup-spi14-cs-gpio-state {
-				pins = "gpio59";
-				function = "gpio";
-			};
-
-			qup_spi15_data_clk: qup-spi15-data-clk-state {
-				pins = "gpio60", "gpio61", "gpio62";
-				function = "qup17";
+			dp_hot_plug_det: dp-hot-plug-det-state {
+				pins = "gpio47";
+				function = "dp_hot";
 			};
 
-			qup_spi15_cs: qup-spi15-cs-state {
-				pins = "gpio63";
-				function = "qup17";
+			qup_spi11_cs: qup-spi11-cs-state {
+				pins = "gpio47";
+				function = "qup13";
 			};
 
-			qup_spi15_cs_gpio: qup-spi15-cs-gpio-state {
-				pins = "gpio63";
+			qup_spi11_cs_gpio: qup-spi11-cs-gpio-state {
+				pins = "gpio47";
 				function = "gpio";
 			};
 
-			qup_uart0_cts: qup-uart0-cts-state {
-				pins = "gpio0";
-				function = "qup00";
-			};
-
-			qup_uart0_rts: qup-uart0-rts-state {
-				pins = "gpio1";
-				function = "qup00";
-			};
-
-			qup_uart0_tx: qup-uart0-tx-state {
-				pins = "gpio2";
-				function = "qup00";
-			};
-
-			qup_uart0_rx: qup-uart0-rx-state {
-				pins = "gpio3";
-				function = "qup00";
-			};
-
-			qup_uart1_cts: qup-uart1-cts-state {
-				pins = "gpio4";
-				function = "qup01";
+			qup_uart11_rx: qup-uart11-rx-state {
+				pins = "gpio47";
+				function = "qup13";
 			};
 
-			qup_uart1_rts: qup-uart1-rts-state {
-				pins = "gpio5";
-				function = "qup01";
+			qup_i2c12_data_clk: qup-i2c12-data-clk-state {
+				pins = "gpio48", "gpio49";
+				function = "qup14";
 			};
 
-			qup_uart1_tx: qup-uart1-tx-state {
-				pins = "gpio6";
-				function = "qup01";
+			qup_spi12_data_clk: qup-spi12-data-clk-state {
+				pins = "gpio48", "gpio49", "gpio50";
+				function = "qup14";
 			};
 
-			qup_uart1_rx: qup-uart1-rx-state {
-				pins = "gpio7";
-				function = "qup01";
+			qup_uart12_cts: qup-uart12-cts-state {
+				pins = "gpio48";
+				function = "qup14";
 			};
 
-			qup_uart2_cts: qup-uart2-cts-state {
-				pins = "gpio8";
-				function = "qup02";
+			qup_uart12_rts: qup-uart12-rts-state {
+				pins = "gpio49";
+				function = "qup14";
 			};
 
-			qup_uart2_rts: qup-uart2-rts-state {
-				pins = "gpio9";
-				function = "qup02";
+			qup_uart12_tx: qup-uart12-tx-state {
+				pins = "gpio50";
+				function = "qup14";
 			};
 
-			qup_uart2_tx: qup-uart2-tx-state {
-				pins = "gpio10";
-				function = "qup02";
+			qup_spi12_cs: qup-spi12-cs-state {
+				pins = "gpio51";
+				function = "qup14";
 			};
 
-			qup_uart2_rx: qup-uart2-rx-state {
-				pins = "gpio11";
-				function = "qup02";
+			qup_spi12_cs_gpio: qup-spi12-cs-gpio-state {
+				pins = "gpio51";
+				function = "gpio";
 			};
 
-			qup_uart3_cts: qup-uart3-cts-state {
-				pins = "gpio12";
-				function = "qup03";
+			qup_uart12_rx: qup-uart12-rx-state {
+				pins = "gpio51";
+				function = "qup14";
 			};
 
-			qup_uart3_rts: qup-uart3-rts-state {
-				pins = "gpio13";
-				function = "qup03";
+			qup_i2c13_data_clk: qup-i2c13-data-clk-state {
+				pins = "gpio52", "gpio53";
+				function = "qup15";
 			};
 
-			qup_uart3_tx: qup-uart3-tx-state {
-				pins = "gpio14";
-				function = "qup03";
+			qup_spi13_data_clk: qup-spi13-data-clk-state {
+				pins = "gpio52", "gpio53", "gpio54";
+				function = "qup15";
 			};
 
-			qup_uart3_rx: qup-uart3-rx-state {
-				pins = "gpio15";
-				function = "qup03";
+			qup_uart13_cts: qup-uart13-cts-state {
+				pins = "gpio52";
+				function = "qup15";
 			};
 
-			qup_uart4_cts: qup-uart4-cts-state {
-				pins = "gpio16";
-				function = "qup04";
+			qup_uart13_rts: qup-uart13-rts-state {
+				pins = "gpio53";
+				function = "qup15";
 			};
 
-			qup_uart4_rts: qup-uart4-rts-state {
-				pins = "gpio17";
-				function = "qup04";
+			qup_uart13_tx: qup-uart13-tx-state {
+				pins = "gpio54";
+				function = "qup15";
 			};
 
-			qup_uart4_tx: qup-uart4-tx-state {
-				pins = "gpio18";
-				function = "qup04";
+			qup_spi13_cs: qup-spi13-cs-state {
+				pins = "gpio55";
+				function = "qup15";
 			};
 
-			qup_uart4_rx: qup-uart4-rx-state {
-				pins = "gpio19";
-				function = "qup04";
+			qup_spi13_cs_gpio: qup-spi13-cs-gpio-state {
+				pins = "gpio55";
+				function = "gpio";
 			};
 
-			qup_uart5_tx: qup-uart5-tx-state {
-				pins = "gpio22";
-				function = "qup05";
+			qup_uart13_rx: qup-uart13-rx-state {
+				pins = "gpio55";
+				function = "qup15";
 			};
 
-			qup_uart5_rx: qup-uart5-rx-state {
-				pins = "gpio23";
-				function = "qup05";
+			qup_i2c14_data_clk: qup-i2c14-data-clk-state {
+				pins = "gpio56", "gpio57";
+				function = "qup16";
 			};
 
-			qup_uart6_cts: qup-uart6-cts-state {
-				pins = "gpio24";
-				function = "qup06";
+			qup_spi14_data_clk: qup-spi14-data-clk-state {
+				pins = "gpio56", "gpio57", "gpio58";
+				function = "qup16";
 			};
 
-			qup_uart6_rts: qup-uart6-rts-state {
-				pins = "gpio25";
-				function = "qup06";
+			qup_uart14_cts: qup-uart14-cts-state {
+				pins = "gpio56";
+				function = "qup16";
 			};
 
-			qup_uart6_tx: qup-uart6-tx-state {
-				pins = "gpio26";
-				function = "qup06";
+			qup_uart14_rts: qup-uart14-rts-state {
+				pins = "gpio57";
+				function = "qup16";
 			};
 
-			qup_uart6_rx: qup-uart6-rx-state {
-				pins = "gpio27";
-				function = "qup06";
+			qup_uart14_tx: qup-uart14-tx-state {
+				pins = "gpio58";
+				function = "qup16";
 			};
 
-			qup_uart7_cts: qup-uart7-cts-state {
-				pins = "gpio28";
-				function = "qup07";
+			qup_spi14_cs: qup-spi14-cs-state {
+				pins = "gpio59";
+				function = "qup16";
 			};
 
-			qup_uart7_rts: qup-uart7-rts-state {
-				pins = "gpio29";
-				function = "qup07";
+			qup_spi14_cs_gpio: qup-spi14-cs-gpio-state {
+				pins = "gpio59";
+				function = "gpio";
 			};
 
-			qup_uart7_tx: qup-uart7-tx-state {
-				pins = "gpio30";
-				function = "qup07";
+			qup_uart14_rx: qup-uart14-rx-state {
+				pins = "gpio59";
+				function = "qup16";
 			};
 
-			qup_uart7_rx: qup-uart7-rx-state {
-				pins = "gpio31";
-				function = "qup07";
+			edp_hot_plug_det: edp-hot-plug-det-state {
+				pins = "gpio60";
+				function = "edp_hot";
 			};
 
-			qup_uart8_cts: qup-uart8-cts-state {
-				pins = "gpio32";
-				function = "qup10";
+			qup_i2c15_data_clk: qup-i2c15-data-clk-state {
+				pins = "gpio60", "gpio61";
+				function = "qup17";
 			};
 
-			qup_uart8_rts: qup-uart8-rts-state {
-				pins = "gpio33";
-				function = "qup10";
+			qup_spi15_data_clk: qup-spi15-data-clk-state {
+				pins = "gpio60", "gpio61", "gpio62";
+				function = "qup17";
 			};
 
-			qup_uart8_tx: qup-uart8-tx-state {
-				pins = "gpio34";
-				function = "qup10";
+			qup_uart15_cts: qup-uart15-cts-state {
+				pins = "gpio60";
+				function = "qup17";
 			};
 
-			qup_uart8_rx: qup-uart8-rx-state {
-				pins = "gpio35";
-				function = "qup10";
+			qup_uart15_rts: qup-uart15-rts-state {
+				pins = "gpio61";
+				function = "qup17";
 			};
 
-			qup_uart9_cts: qup-uart9-cts-state {
-				pins = "gpio36";
-				function = "qup11";
+			qup_uart15_tx: qup-uart15-tx-state {
+				pins = "gpio62";
+				function = "qup17";
 			};
 
-			qup_uart9_rts: qup-uart9-rts-state {
-				pins = "gpio37";
-				function = "qup11";
+			qup_spi15_cs: qup-spi15-cs-state {
+				pins = "gpio63";
+				function = "qup17";
 			};
 
-			qup_uart9_tx: qup-uart9-tx-state {
-				pins = "gpio38";
-				function = "qup11";
+			qup_spi15_cs_gpio: qup-spi15-cs-gpio-state {
+				pins = "gpio63";
+				function = "gpio";
 			};
 
-			qup_uart9_rx: qup-uart9-rx-state {
-				pins = "gpio39";
-				function = "qup11";
+			qup_uart15_rx: qup-uart15-rx-state {
+				pins = "gpio63";
+				function = "qup17";
 			};
 
-			qup_uart10_cts: qup-uart10-cts-state {
-				pins = "gpio40";
-				function = "qup12";
+			cam_mclk0_default: cam-mclk0-default-state {
+				pins = "gpio64";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
 			};
 
-			qup_uart10_rts: qup-uart10-rts-state {
-				pins = "gpio41";
-				function = "qup12";
+			cam_mclk1_default: cam-mclk1-default-state {
+				pins = "gpio65";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
 			};
 
-			qup_uart10_tx: qup-uart10-tx-state {
-				pins = "gpio42";
-				function = "qup12";
+			cam_mclk2_default: cam-mclk2-default-state {
+				pins = "gpio66";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
 			};
 
-			qup_uart10_rx: qup-uart10-rx-state {
-				pins = "gpio43";
-				function = "qup12";
+			cam_mclk3_default: cam-mclk3-default-state {
+				pins = "gpio67";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
 			};
 
-			qup_uart11_cts: qup-uart11-cts-state {
-				pins = "gpio44";
-				function = "qup13";
+			cam_mclk4_default: cam-mclk4-default-state {
+				pins = "gpio68";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
 			};
 
-			qup_uart11_rts: qup-uart11-rts-state {
-				pins = "gpio45";
-				function = "qup13";
+			cci0_default: cci0-default-state {
+				pins = "gpio69", "gpio70";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
 			};
 
-			qup_uart11_tx: qup-uart11-tx-state {
-				pins = "gpio46";
-				function = "qup13";
+			cci0_sleep: cci0-sleep-state {
+				pins = "gpio69", "gpio70";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
 			};
 
-			qup_uart11_rx: qup-uart11-rx-state {
-				pins = "gpio47";
-				function = "qup13";
+			cci1_default: cci1-default-state {
+				pins = "gpio71", "gpio72";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
 			};
 
-			qup_uart12_cts: qup-uart12-cts-state {
-				pins = "gpio48";
-				function = "qup14";
+			cci1_sleep: cci1-sleep-state {
+				pins = "gpio71", "gpio72";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
 			};
 
-			qup_uart12_rts: qup-uart12-rts-state {
-				pins = "gpio49";
-				function = "qup14";
+			cci2_default: cci2-default-state {
+				pins = "gpio73", "gpio74";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
 			};
 
-			qup_uart12_tx: qup-uart12-tx-state {
-				pins = "gpio50";
-				function = "qup14";
+			cci2_sleep: cci2-sleep-state {
+				pins = "gpio73", "gpio74";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
 			};
 
-			qup_uart12_rx: qup-uart12-rx-state {
-				pins = "gpio51";
-				function = "qup14";
+			cci3_default: cci3-default-state {
+				pins = "gpio75", "gpio76";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
 			};
 
-			qup_uart13_cts: qup-uart13-cts-state {
-				pins = "gpio52";
-				function = "qup15";
+			cci3_sleep: cci3-sleep-state {
+				pins = "gpio75", "gpio76";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
 			};
 
-			qup_uart13_rts: qup-uart13-rts-state {
-				pins = "gpio53";
-				function = "qup15";
+			pcie1_clkreq_n: pcie1-clkreq-n-state {
+				pins = "gpio79";
+				function = "pcie1_clkreqn";
 			};
 
-			qup_uart13_tx: qup-uart13-tx-state {
-				pins = "gpio54";
-				function = "qup15";
+			pcie0_clkreq_n: pcie0-clkreq-n-state {
+				pins = "gpio88";
+				function = "pcie0_clkreqn";
 			};
 
-			qup_uart13_rx: qup-uart13-rx-state {
-				pins = "gpio55";
-				function = "qup15";
+			cam_mclk5_default: cam-mclk5-default-state {
+				pins = "gpio93";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
 			};
 
-			qup_uart14_cts: qup-uart14-cts-state {
-				pins = "gpio56";
-				function = "qup16";
+			mi2s0_mclk: mi2s0-mclk-state {
+				pins = "gpio96";
+				function = "pri_mi2s";
 			};
 
-			qup_uart14_rts: qup-uart14-rts-state {
-				pins = "gpio57";
-				function = "qup16";
+			mi2s0_sclk: mi2s0-sclk-state {
+				pins = "gpio97";
+				function = "mi2s0_sck";
 			};
 
-			qup_uart14_tx: qup-uart14-tx-state {
-				pins = "gpio58";
-				function = "qup16";
+			mi2s0_data0: mi2s0-data0-state {
+				pins = "gpio98";
+				function = "mi2s0_data0";
 			};
 
-			qup_uart14_rx: qup-uart14-rx-state {
-				pins = "gpio59";
-				function = "qup16";
+			mi2s0_data1: mi2s0-data1-state {
+				pins = "gpio99";
+				function = "mi2s0_data1";
 			};
 
-			qup_uart15_cts: qup-uart15-cts-state {
-				pins = "gpio60";
-				function = "qup17";
+			mi2s0_ws: mi2s0-ws-state {
+				pins = "gpio100";
+				function = "mi2s0_ws";
 			};
 
-			qup_uart15_rts: qup-uart15-rts-state {
-				pins = "gpio61";
-				function = "qup17";
+			mi2s1_sclk: mi2s1-sclk-state {
+				pins = "gpio106";
+				function = "mi2s1_sck";
 			};
 
-			qup_uart15_tx: qup-uart15-tx-state {
-				pins = "gpio62";
-				function = "qup17";
+			mi2s1_data0: mi2s1-data0-state {
+				pins = "gpio107";
+				function = "mi2s1_data0";
 			};
 
-			qup_uart15_rx: qup-uart15-rx-state {
-				pins = "gpio63";
-				function = "qup17";
+			mi2s1_ws: mi2s1-ws-state {
+				pins = "gpio108";
+				function = "mi2s1_ws";
 			};
 
 			sdc1_clk: sdc1-clk-state {
@@ -7010,39 +7058,39 @@
 				rpmhpd_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					rpmhpd_opp_ret: opp1 {
+					rpmhpd_opp_ret: opp-1 {
 						opp-level = <RPMH_REGULATOR_LEVEL_RETENTION>;
 					};
 
-					rpmhpd_opp_low_svs: opp2 {
+					rpmhpd_opp_low_svs: opp-2 {
 						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
 					};
 
-					rpmhpd_opp_svs: opp3 {
+					rpmhpd_opp_svs: opp-3 {
 						opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
 					};
 
-					rpmhpd_opp_svs_l1: opp4 {
+					rpmhpd_opp_svs_l1: opp-4 {
 						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
 					};
 
-					rpmhpd_opp_svs_l2: opp5 {
+					rpmhpd_opp_svs_l2: opp-5 {
 						opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
 					};
 
-					rpmhpd_opp_nom: opp6 {
+					rpmhpd_opp_nom: opp-6 {
 						opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
 					};
 
-					rpmhpd_opp_nom_l1: opp7 {
+					rpmhpd_opp_nom_l1: opp-7 {
 						opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
 					};
 
-					rpmhpd_opp_turbo: opp8 {
+					rpmhpd_opp_turbo: opp-8 {
 						opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
 					};
 
-					rpmhpd_opp_turbo_l1: opp9 {
+					rpmhpd_opp_turbo_l1: opp-9 {
 						opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
 					};
 				};
diff --git a/arch/arm64/boot/dts/qcom/lemans-el2.dtso b/arch/arm64/boot/dts/qcom/lemans-el2.dtso
index 621ad930cf547e..ceaab5f55fa2a2 100644
--- a/arch/arm64/boot/dts/qcom/lemans-el2.dtso
+++ b/arch/arm64/boot/dts/qcom/lemans-el2.dtso
@@ -31,9 +31,9 @@
 };
 
 &remoteproc_gpdsp0 {
-       iommus = <&apps_smmu 0x38a0 0x0>;
+	iommus = <&apps_smmu 0x38a0 0x0>;
 };
 
 &remoteproc_gpdsp1 {
-       iommus = <&apps_smmu 0x38c0 0x0>;
+	iommus = <&apps_smmu 0x38c0 0x0>;
 };
diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso
index 6cfa0bc1440e91..e34639460a966e 100644
--- a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso
@@ -271,27 +271,20 @@
 			ranges;
 			bus-range = <0x5 0xff>;
 
-			pcie@0,0 {
+			dev@0,0 {
 				reg = <0x50000 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 
-			pcie@0,1 {
+			dev@0,1 {
 				reg = <0x50100 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 		};
 	};
 };
 
 &serdes1 {
-	phy-supply = <&vreg_l5a>;
+	vdda-0p9-supply = <&vreg_l4a>;
+	vdda-1p2-supply = <&vreg_l1c>;
 
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/lemans-evk.dts b/arch/arm64/boot/dts/qcom/lemans-evk.dts
index a45485b54a5b78..23cf20f9a1783a 100644
--- a/arch/arm64/boot/dts/qcom/lemans-evk.dts
+++ b/arch/arm64/boot/dts/qcom/lemans-evk.dts
@@ -839,9 +839,6 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -856,9 +853,6 @@
 };
 
 &pcie1 {
-	perst-gpios = <&tlmm 4 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 5 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie1_default_state>;
 	pinctrl-names = "default";
 
@@ -872,7 +866,14 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 4 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 5 GPIO_ACTIVE_LOW>;
+};
+
 &pcieport0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 0 GPIO_ACTIVE_LOW>;
 	port {
 		pcieport0_ep: endpoint {
 			remote-endpoint = <&m2_e_pcie_ep>;
@@ -978,7 +979,8 @@
 };
 
 &serdes0 {
-	phy-supply = <&vreg_l5a>;
+	vdda-0p9-supply = <&vreg_l4a>;
+	vdda-1p2-supply = <&vreg_l1c>;
 
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/lemans-pmics.dtsi b/arch/arm64/boot/dts/qcom/lemans-pmics.dtsi
index 500f715931aa3a..4c853053be8584 100644
--- a/arch/arm64/boot/dts/qcom/lemans-pmics.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans-pmics.dtsi
@@ -88,6 +88,58 @@
 				};
 			};
 		};
+
+		pm8775-mbg0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmm8654au_0_mbg_temp>;
+
+			trips {
+				trip0 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pm8775-mbg1-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmm8654au_1_mbg_temp>;
+
+			trips {
+				trip0 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pm8775-mbg2-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmm8654au_2_mbg_temp>;
+
+			trips {
+				trip0 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pm8775-mbg3-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmm8654au_3_mbg_temp>;
+
+			trips {
+				trip0 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
 	};
 
 	reboot-mode {
@@ -189,6 +241,22 @@
 		};
 	};
 
+	pmm8654au_sail_0: pmic@1 {
+		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
+		reg = <0x1 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmm8654au_0_mbg_temp: temperature-sensor@d700 {
+			compatible = "qcom,pm8775-mbg-tm";
+			reg = <0xd700>;
+			io-channels = <&pmm8654au_0_adc ADC5_GEN3_DIE_TEMP(0)>;
+			io-channel-names = "thermal";
+			interrupts-extended = <&spmi_bus 0x1 0xd7 0x0 IRQ_TYPE_EDGE_RISING>;
+			#thermal-sensor-cells = <0>;
+		};
+	};
+
 	pmm8654au_1: pmic@2 {
 		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
 		reg = <0x2 SPMI_USID>;
@@ -236,6 +304,22 @@
 		};
 	};
 
+	pmm8654au_sail_1: pmic@3 {
+		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
+		reg = <0x3 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmm8654au_1_mbg_temp: temperature-sensor@d700 {
+			compatible = "qcom,pm8775-mbg-tm";
+			reg = <0xd700>;
+			io-channels = <&pmm8654au_1_adc ADC5_GEN3_DIE_TEMP(2)>;
+			io-channel-names = "thermal";
+			interrupts-extended = <&spmi_bus 0x3 0xd7 0x0 IRQ_TYPE_EDGE_RISING>;
+			#thermal-sensor-cells = <0>;
+		};
+	};
+
 	pmm8654au_2: pmic@4 {
 		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
 		reg = <0x4 SPMI_USID>;
@@ -283,6 +367,22 @@
 		};
 	};
 
+	pmm8654au_sail_2: pmic@5 {
+		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
+		reg = <0x5 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmm8654au_2_mbg_temp: temperature-sensor@d700 {
+			compatible = "qcom,pm8775-mbg-tm";
+			reg = <0xd700>;
+			io-channels = <&pmm8654au_2_adc ADC5_GEN3_DIE_TEMP(4)>;
+			io-channel-names = "thermal";
+			interrupts-extended = <&spmi_bus 0x5 0xd7 0x0 IRQ_TYPE_EDGE_RISING>;
+			#thermal-sensor-cells = <0>;
+		};
+	};
+
 	pmm8654au_3: pmic@6 {
 		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
 		reg = <0x6 SPMI_USID>;
@@ -329,4 +429,20 @@
 			#interrupt-cells = <2>;
 		};
 	};
+
+	pmm8654au_sail_3: pmic@7 {
+		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
+		reg = <0x7 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmm8654au_3_mbg_temp: temperature-sensor@d700 {
+			compatible = "qcom,pm8775-mbg-tm";
+			reg = <0xd700>;
+			io-channels = <&pmm8654au_3_adc ADC5_GEN3_DIE_TEMP(6)>;
+			io-channel-names = "thermal";
+			interrupts-extended = <&spmi_bus 0x7 0xd7 0x0 IRQ_TYPE_EDGE_RISING>;
+			#thermal-sensor-cells = <0>;
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
index 144f117ba5116e..22f764b8d721d1 100644
--- a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
@@ -856,12 +856,16 @@
 };
 
 &serdes0 {
-	phy-supply = <&vreg_l5a>;
+	vdda-0p9-supply = <&vreg_l4a>;
+	vdda-1p2-supply = <&vreg_l1c>;
+
 	status = "okay";
 };
 
 &serdes1 {
-	phy-supply = <&vreg_l5a>;
+	vdda-0p9-supply = <&vreg_l4a>;
+	vdda-1p2-supply = <&vreg_l1c>;
+
 	status = "okay";
 };
 
@@ -961,9 +965,6 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie0_default_state>;
 
@@ -971,9 +972,6 @@
 };
 
 &pcie1 {
-	perst-gpios = <&tlmm 4 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 5 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_default_state>;
 
@@ -994,7 +992,14 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 4 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 5 GPIO_ACTIVE_LOW>;
+};
+
 &pcieport0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 0 GPIO_ACTIVE_LOW>;
 	wifi@0 {
 		compatible = "pci17cb,1101";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi
index 695eae1b725691..1fdb72fa551844 100644
--- a/arch/arm64/boot/dts/qcom/lemans.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans.dtsi
@@ -1030,7 +1030,7 @@
 			};
 		};
 
-		gpi_dma2: dma-controller@800000  {
+		gpi_dma2: dma-controller@800000 {
 			compatible = "qcom,sa8775p-gpi-dma", "qcom,sm6350-gpi-dma";
 			reg = <0x0 0x00800000 0x0 0x60000>;
 			#dma-cells = <3>;
@@ -1563,7 +1563,7 @@
 
 		};
 
-		gpi_dma0: dma-controller@900000  {
+		gpi_dma0: dma-controller@900000 {
 			compatible = "qcom,sa8775p-gpi-dma", "qcom,sm6350-gpi-dma";
 			reg = <0x0 0x00900000 0x0 0x60000>;
 			#dma-cells = <3>;
@@ -2024,7 +2024,7 @@
 			};
 		};
 
-		gpi_dma1: dma-controller@a00000  {
+		gpi_dma1: dma-controller@a00000 {
 			compatible = "qcom,sa8775p-gpi-dma", "qcom,sm6350-gpi-dma";
 			reg = <0x0 0x00a00000 0x0 0x60000>;
 			#dma-cells = <3>;
@@ -2517,7 +2517,7 @@
 			};
 		};
 
-		gpi_dma3: dma-controller@b00000  {
+		gpi_dma3: dma-controller@b00000 {
 			compatible = "qcom,sa8775p-gpi-dma", "qcom,sm6350-gpi-dma";
 			reg = <0x0 0x00b00000 0x0 0x58000>;
 			#dma-cells = <3>;
@@ -2753,9 +2753,6 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
 			eq-presets-16gts = /bits/ 8 <0x55 0x55>;
 
@@ -2767,6 +2764,8 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2927,19 +2926,18 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>;
 			eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>;
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -7168,11 +7166,11 @@
 						opp-level = <RPMH_REGULATOR_LEVEL_MIN_SVS>;
 					};
 
-					rpmhpd_opp_low_svs: opp2 {
+					rpmhpd_opp_low_svs: opp-2 {
 						opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
 					};
 
-					rpmhpd_opp_svs: opp3 {
+					rpmhpd_opp_svs: opp-3 {
 						opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
 					};
 
diff --git a/arch/arm64/boot/dts/qcom/mahua-crd.dts b/arch/arm64/boot/dts/qcom/mahua-crd.dts
index fa5229064b10e7..17e40f2a6259f8 100644
--- a/arch/arm64/boot/dts/qcom/mahua-crd.dts
+++ b/arch/arm64/boot/dts/qcom/mahua-crd.dts
@@ -20,6 +20,11 @@
 	compatible = "qcom,mahua-crd", "qcom,mahua";
 };
 
+&pcie3b_phy {
+	vdda-phy-supply = <&vreg_l3c_e1_0p89>;
+	vdda-pll-supply = <&vreg_l2c_e1_1p14>;
+};
+
 &tcsr {
 	vdda-qrefrpt0-0p9-supply = <&vreg_l2f_e1_0p83>;
 	vdda-qrefrpt1-0p9-supply = <&vreg_l2f_e1_0p83>;
diff --git a/arch/arm64/boot/dts/qcom/mahua-lenovo-thinkpad-t14s-gen7-lcd.dts b/arch/arm64/boot/dts/qcom/mahua-lenovo-thinkpad-t14s-gen7-lcd.dts
new file mode 100644
index 00000000000000..26d6d8285e0409
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/mahua-lenovo-thinkpad-t14s-gen7-lcd.dts
@@ -0,0 +1,1272 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+/dts-v1/;
+
+#include "mahua.dtsi"
+
+#include "pmcx0102.dtsi"
+#include "pmh0101.dtsi"
+#include "pmh0110-glymur.dtsi"
+#include "pmh0104-glymur.dtsi"
+#include "pmk8850.dtsi"
+#include "smb2370.dtsi"
+
+#include <dt-bindings/input/gpio-keys.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+
+/* PMICs tied to Glymur's cluster2 are absent on Mahua. */
+/delete-node/ &pmcx0102_d_e0;
+/delete-node/ &pmcx0102_d0_thermal;
+/delete-node/ &pmh0104_i_e0;
+/delete-node/ &pmh0104_i0_thermal;
+/delete-node/ &pmh0104_j_e0;
+/delete-node/ &pmh0104_j0_thermal;
+
+/ {
+	model = "Lenovo ThinkPad T14s Gen 7 (LCD)";
+	compatible = "lenovo,thinkpad-t14s-gen7-lcd", "lenovo,thinkpad-t14s-gen7",
+		     "qcom,mahua";
+
+	aliases {
+		i2c0 = &i2c0;
+		i2c5 = &i2c5;
+
+		serial0 = &uart21;
+		serial1 = &uart14;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board: xo-board {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			#clock-cells = <0>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&hall_int_n_default>;
+		pinctrl-names = "default";
+
+		switch-lid {
+			label = "lid";
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,glymur-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Left-side USB-C port */
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_0_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_0_qmpphy_out>;
+					};
+				};
+			};
+		};
+
+		/* Right-side USB-C port */
+		connector@1 {
+			compatible = "usb-c-connector";
+			reg = <1>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in1: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in1: endpoint {
+						remote-endpoint = <&usb_1_qmpphy_out>;
+					};
+				};
+			};
+		};
+	};
+
+	vreg_edp_3p3: regulator-edp-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_edp_bl: regulator-edp-bl {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_EDP_BL";
+		regulator-min-microvolt = <3600000>;
+		regulator-max-microvolt = <3600000>;
+
+		gpio = <&tlmm 19 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&edp_bl_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vreg_misc_3p3: regulator-misc-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_MISC_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0110_f_e0_gpios 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&misc_3p3_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	vreg_nvme: regulator-nvme {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_NVME_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&pmh0101_gpios 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&nvme_reg_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wcn_0p95: regulator-wcn-0p95 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_wcn_3p3>;
+	};
+
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 94 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wcn_sw_en>;
+		pinctrl-names = "default";
+
+		regulator-boot-on;
+	};
+
+	vreg_wwan: regulator-wwan {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WWAN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 246 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&wwan_reg_en>;
+		pinctrl-names = "default";
+	};
+
+	reserved-memory {
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x10000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	sound {
+		compatible = "qcom,glymur-sndcard";
+		model = "Lenovo-ThinkPad-T14s-Gen-7";
+		audio-routing = "SpkrLeft IN", "WSA WSA_SPK1 OUT",
+				"SpkrRight IN", "WSA WSA_SPK2 OUT",
+				"VA DMIC0", "vdd-micb",
+				"VA DMIC1", "vdd-micb";
+
+		wsa-dai-link {
+			link-name = "WSA Playback";
+
+			cpu {
+				sound-dai = <&q6apmbedai WSA_CODEC_DMA_RX_0>;
+			};
+
+			codec {
+				sound-dai = <&left_spkr>, <&right_spkr>,
+					    <&swr0 0>, <&lpass_wsamacro 0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		va-dai-link {
+			link-name = "VA Capture";
+
+			cpu {
+				sound-dai = <&q6apmbedai VA_CODEC_DMA_TX_0>;
+			};
+
+			codec {
+				sound-dai = <&lpass_vamacro 0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		displayport-0-dai-link {
+			link-name = "DisplayPort0 Playback";
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+			};
+
+			codec {
+				sound-dai = <&mdss_dp0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		displayport-1-dai-link {
+			link-name = "DisplayPort1 Playback";
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_1>;
+			};
+
+			codec {
+				sound-dai = <&mdss_dp1>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		vdd-supply = <&vreg_wcn_0p95>;
+		vddio-supply = <&vreg_l15b_e0_1p8>;
+		vddaon-supply = <&vreg_l15b_e0_1p8>;
+		vdddig-supply = <&vreg_l15b_e0_1p8>;
+		vddrfa1p2-supply = <&vreg_l15b_e0_1p8>;
+		vddrfa1p8-supply = <&vreg_l15b_e0_1p8>;
+
+		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&wcn_wlan_bt_en>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmh0101-rpmh-regulators";
+		qcom,pmic-id = "B_E0";
+
+		vreg_bob1_e0: bob1 {
+			regulator-name = "vreg_bob1_e0";
+			regulator-min-microvolt = <2200000>;
+			regulator-max-microvolt = <4224000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_bob2_e0: bob2 {
+			regulator-name = "vreg_bob2_e0";
+			regulator-min-microvolt = <2540000>;
+			regulator-max-microvolt = <3600000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		vreg_l1b_e0_1p8: ldo1 {
+			regulator-name = "vreg_l1b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_e0_2p9: ldo2 {
+			regulator-name = "vreg_l2b_e0_2p9";
+			regulator-min-microvolt = <2904000>;
+			regulator-max-microvolt = <2904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_e0_2p79: ldo7 {
+			regulator-name = "vreg_l7b_e0_2p79";
+			regulator-min-microvolt = <2790000>;
+			regulator-max-microvolt = <2792000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_e0_3p3: ldo8 {
+			regulator-name = "vreg_l8b_e0_3p3";
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_e0_2p9: ldo9 {
+			regulator-name = "vreg_l9b_e0_2p9";
+			regulator-min-microvolt = <2704000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_e0_1p8: ldo10 {
+			regulator-name = "vreg_l10b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_e0_1p2: ldo11 {
+			regulator-name = "vreg_l11b_e0_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_e0_1p2: ldo12 {
+			regulator-name = "vreg_l12b_e0_1p2";
+			regulator-min-microvolt = <1144000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_e0_1p8: ldo15 {
+			regulator-name = "vreg_l15b_e0_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l17b_e0_2p4: ldo17 {
+			regulator-name = "vreg_l17b_e0_2p4";
+			regulator-min-microvolt = <2400000>;
+			regulator-max-microvolt = <2700000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_e0_1p2: ldo18 {
+			regulator-name = "vreg_l18b_e0_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmcx0102-rpmh-regulators";
+		qcom,pmic-id = "C_E1";
+
+		vreg_l1c_e1_0p82: ldo1 {
+			regulator-name = "vreg_l1c_e1_0p82";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_e1_1p14: ldo2 {
+			regulator-name = "vreg_l2c_e1_1p14";
+			regulator-min-microvolt = <1144000>;
+			regulator-max-microvolt = <1144000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_e1_0p89: ldo3 {
+			regulator-name = "vreg_l3c_e1_0p89";
+			regulator-min-microvolt = <890000>;
+			regulator-max-microvolt = <980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_e1_0p72: ldo4 {
+			regulator-name = "vreg_l4c_e1_0p72";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <720000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_s7f_e0_1p32: smps7 {
+			regulator-name = "vreg_s7f_e0_1p32";
+			regulator-min-microvolt = <1320000>;
+			regulator-max-microvolt = <1352000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s8f_e0_0p95: smps8 {
+			regulator-name = "vreg_s8f_e0_0p95";
+			regulator-min-microvolt = <952000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s9f_e0_1p9: smps9 {
+			regulator-name = "vreg_s9f_e0_1p9";
+			regulator-min-microvolt = <1900000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_e0_0p94: ldo2 {
+			regulator-name = "vreg_l2f_e0_0p94";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_e0_0p91: ldo3 {
+			regulator-name = "vreg_l3f_e0_0p91";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f_e0_0p3: ldo4 {
+			regulator-name = "vreg_l4f_e0_0p3";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "F_E1";
+
+		vreg_s7f_e1_0p3: smps7 {
+			regulator-name = "vreg_s7f_e1_0p3";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_e1_0p88: ldo1 {
+			regulator-name = "vreg_l1f_e1_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_e1_0p9: ldo2 {
+			regulator-name = "vreg_l2f_e1_0p9";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f_e1_1p2: ldo4 {
+			regulator-name = "vreg_l4f_e1_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pmh0110-rpmh-regulators";
+		qcom,pmic-id = "H_E0";
+
+		vreg_l1h_e0_0p94: ldo1 {
+			regulator-name = "vreg_l1h_e0_0p94";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2h_e0_0p72: ldo2 {
+			regulator-name = "vreg_l2h_e0_0p72";
+			regulator-min-microvolt = <832000>;
+			regulator-max-microvolt = <832000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3h_e0_0p32: ldo3 {
+			regulator-name = "vreg_l3h_e0_0p32";
+			regulator-min-microvolt = <320000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4h_e0_1p2: ldo4 {
+			regulator-name = "vreg_l4h_e0_1p2";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1320000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+/* GPU/GMU not fully functional yet on this platform. */
+&gmu {
+	status = "disabled";
+};
+
+&gpu {
+	status = "disabled";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchpad@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+
+		hid-descr-addr = <0x20>;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0_1p8>;
+
+		pinctrl-0 = <&tpad_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+
+	keyboard@3a {
+		compatible = "hid-over-i2c";
+		reg = <0x3a>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 67 IRQ_TYPE_LEVEL_LOW>;
+
+		vdd-supply = <&vreg_misc_3p3>;
+		vddl-supply = <&vreg_l15b_e0_1p8>;
+
+		pinctrl-0 = <&kybd_default>;
+		pinctrl-names = "default";
+
+		wakeup-source;
+	};
+};
+
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	ptn3222_0: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+
+		reset-gpios = <&tlmm 25 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b_e0_3p3>;
+		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
+
+		#phy-cells = <0>;
+	};
+
+	ptn3222_1: redriver@4b {
+		compatible = "nxp,ptn3222";
+		reg = <0x4b>;
+
+		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b_e0_3p3>;
+		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
+
+		#phy-cells = <0>;
+	};
+
+	ptn3222_cam: redriver@47 {
+		compatible = "nxp,ptn3222";
+		reg = <0x47>;
+
+		reset-gpios = <&tlmm 184 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b_e0_3p3>;
+		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
+
+		#phy-cells = <0>;
+	};
+
+	ptn3222_fpr: redriver@4f {
+		compatible = "nxp,ptn3222";
+		reg = <0x4f>;
+
+		reset-gpios = <&tlmm 40 GPIO_ACTIVE_LOW>;
+
+		vdd3v3-supply = <&vreg_l8b_e0_3p3>;
+		vdd1v8-supply = <&vreg_l15b_e0_1p8>;
+
+		#phy-cells = <0>;
+	};
+};
+
+&lpass_vamacro {
+	pinctrl-0 = <&dmic01_default>;
+	pinctrl-names = "default";
+	qcom,dmic-sample-rate = <4800000>;
+	vdd-micb-supply = <&vreg_l1b_e0_1p8>;
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp1 {
+	status = "okay";
+};
+
+&mdss_dp1_out {
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+};
+
+&mdss_dp3 {
+	/delete-property/ #sound-dai-cells;
+
+	pinctrl-0 = <&edp0_hpd_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	aux-bus {
+		panel {
+			compatible = "edp-panel";
+			enable-gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+			power-supply = <&vreg_edp_3p3>;
+
+			pinctrl-0 = <&edp_bl_en>;
+			pinctrl-names = "default";
+
+			port {
+				edp_panel_in: endpoint {
+					remote-endpoint = <&mdss_dp3_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dp3_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+
+	remote-endpoint = <&edp_panel_in>;
+};
+
+&mdss_dp3_phy {
+	vdda-phy-supply = <&vreg_l2f_e1_0p9>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+/* PCIe4: on-board WLAN (WCN7850) */
+&pcie4 {
+	pinctrl-0 = <&pcie4_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie4_phy {
+	vdda-phy-supply = <&vreg_l1c_e1_0p82>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+&pcie4_port0 {
+	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+/* PCIe5: primary M.2 2280 NVMe */
+&pcie5 {
+	vddpe-3v3-supply = <&vreg_nvme>;
+
+	pinctrl-0 = <&pcie5_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie5_phy {
+	vdda-phy-supply = <&vreg_l2f_e0_0p94>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+
+	status = "okay";
+};
+
+&pcie5_port0 {
+	reset-gpios = <&tlmm 152 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 154 GPIO_ACTIVE_LOW>;
+};
+
+/* PCIe6: optional WWAN M.2 3042 */
+&pcie6 {
+	vddpe-3v3-supply = <&vreg_wwan>;
+
+	pinctrl-0 = <&pcie6_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie6_phy {
+	vdda-phy-supply = <&vreg_l1c_e1_0p82>;
+	vdda-pll-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+&pcie6_port0 {
+	reset-gpios = <&tlmm 149 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
+};
+
+&pmh0101_gpios {
+	nvme_reg_en: nvme-reg-en-state {
+		pins = "gpio14";
+		function = "normal";
+		bias-disable;
+	};
+};
+
+&pmh0110_f_e0_gpios {
+	misc_3p3_reg_en: misc-3p3-reg-en-state {
+		pins = "gpio6";
+		function = "normal";
+		bias-disable;
+		input-disable;
+		output-enable;
+		drive-push-pull;
+		power-source = <1>; /* 1.8 V */
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+	};
+};
+
+&pmk8850_rtc {
+	qcom,no-alarm;
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/glymur/LENOVO/22A8/qcadsp8480.mbn",
+			"qcom/glymur/LENOVO/22A8/adsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/glymur/LENOVO/22A8/qccdsp8480.mbn",
+			"qcom/glymur/LENOVO/22A8/cdsp_dtbs.elf";
+
+	status = "okay";
+};
+
+&swr0 {
+	status = "okay";
+
+	/* WSA8845, Left */
+	left_spkr: speaker@0,0 {
+		compatible = "sdw20217020400";
+		reg = <0 0>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrLeft";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <1 2 3 7 12 14>;
+	};
+
+	/* WSA8845, Right */
+	right_spkr: speaker@0,1 {
+		compatible = "sdw20217020400";
+		reg = <0 1>;
+		reset-gpios = <&lpass_tlmm 12 GPIO_ACTIVE_LOW>;
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "SpkrRight";
+		vdd-1p8-supply = <&vreg_l15b_e0_1p8>;
+		vdd-io-supply = <&vreg_l18b_e0_1p2>;
+		qcom,port-mapping = <4 5 6 7 13 15>;
+	};
+};
+
+&smb2370_j_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0_1p8>;
+	vdd3-supply = <&vreg_l7b_e0_2p79>;
+};
+
+&smb2370_k_e2_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_e0_1p8>;
+	vdd3-supply = <&vreg_l7b_e0_2p79>;
+};
+
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l2f_e1_0p9>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l2f_e1_0p9>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l2f_e1_0p9>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l2h_e0_0p72>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l2h_e0_0p72>;
+	vdda-qrefrpt5-0p9-supply = <&vreg_l2h_e0_0p72>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l2f_e1_0p9>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l2f_e1_0p9>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l2f_e1_0p9>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l2h_e0_0p72>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l2h_e0_0p72>;
+	vdda-qrefrx5-0p9-supply = <&vreg_l3f_e0_0p91>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3f_e0_0p91>;
+	vdda-qreftx0-1p2-supply = <&vreg_l4h_e0_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l1f_e1_0p88>;
+	vdda-refgen3-0p9-supply = <&vreg_l2f_e0_0p94>;
+	vdda-refgen3-1p2-supply = <&vreg_l4h_e0_1p2>;
+	vdda-refgen4-0p9-supply = <&vreg_l1f_e1_0p88>;
+	vdda-refgen4-1p2-supply = <&vreg_l4f_e1_1p2>;
+};
+
+&tlmm {
+	gpio-reserved-ranges = <4 4>,   /* EC TZ Secure I3C */
+			       <10 2>,  /* OOB UART */
+			       <27 1>,  /* TZ trigger  */
+			       <44 4>,  /* TPM */
+			       <139 1>; /* TZ trigger */
+
+	edp_bl_en: edp-bl-en-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	edp_bl_reg_en: edp-bl-reg-en-state {
+		pins = "gpio19";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	edp_reg_en: edp-reg-en-state {
+		pins = "gpio70";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	edp0_hpd_default: edp0-hpd-default-state {
+		pins = "gpio119";
+		function = "edp0_hot";
+		bias-disable;
+	};
+
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
+	kybd_default: kybd-default-state {
+		pins = "gpio67";
+		function = "gpio";
+		bias-disable;
+	};
+
+	pcie4_default: pcie4-default-state {
+		clkreq-n-pins {
+			pins = "gpio147";
+			function = "pcie4_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio146";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio148";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie5_default: pcie5-default-state {
+		clkreq-n-pins {
+			pins = "gpio153";
+			function = "pcie5_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio152";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio154";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	pcie6_default: pcie6-default-state {
+		clkreq-n-pins {
+			pins = "gpio150";
+			function = "pcie6_clk_req_n";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-n-pins {
+			pins = "gpio149";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		wake-n-pins {
+			pins = "gpio151";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	tpad_default: tpad-default-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-disable;
+	};
+
+	wcn_wlan_bt_en: wcn-wlan-bt-en-state {
+		pins = "gpio116", "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wcn_sw_en: wcn-sw-en-state {
+		pins = "gpio94";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	wwan_reg_en: wwan-reg-en-state {
+		pins = "gpio246";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+		max-speed = <3200000>;
+
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+	};
+};
+
+&usb_0 {
+	status = "okay";
+};
+
+&usb_0_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_0_hsphy {
+	vdd-supply = <&vreg_l3f_e0_0p91>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&smb2370_j_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy {
+	vdda-phy-supply = <&vreg_l3f_e0_0p91>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l2f_e0_0p94>;
+
+	status = "okay";
+};
+
+&usb_0_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in1>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l3f_e0_0p91>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&smb2370_k_e2_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&vreg_l1h_e0_0p94>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l2f_e0_0p94>;
+
+	status = "okay";
+};
+
+&usb_1_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in1>;
+};
+
+&usb_mp {
+	status = "okay";
+};
+
+&usb_mp_hsphy0 {
+	vdd-supply = <&vreg_l2h_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&ptn3222_0>;
+
+	status = "okay";
+};
+
+&usb_mp_hsphy1 {
+	vdd-supply = <&vreg_l2h_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&ptn3222_1>;
+//USB-A..
+	status = "okay";
+};
+
+&usb_mp_qmpphy0 {
+	vdda-phy-supply = <&vreg_l2h_e0_0p72>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+&usb_mp_qmpphy1 {
+	vdda-phy-supply = <&vreg_l2h_e0_0p72>;
+	vdda-pll-supply = <&vreg_l4h_e0_1p2>;
+	refgen-supply = <&vreg_l4f_e1_1p2>;
+
+	status = "okay";
+};
+
+&usb_hs {
+	dr_mode = "host";
+
+	status = "okay";
+};
+
+&usb_hs_phy {
+	vdd-supply = <&vreg_l2h_e0_0p72>;
+	vdda12-supply = <&vreg_l4h_e0_1p2>;
+
+	phys = <&ptn3222_cam>;
+
+	status = "okay";
+};
+
+&usb_2 {
+	dr_mode = "host";
+	maximum-speed = "high-speed";
+
+	/delete-property/ usb-role-switch;
+	phys = <&usb_2_hsphy>;
+	phy-names = "usb2-phy";
+
+	status = "okay";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&vreg_l4c_e1_0p72>;
+	vdda12-supply = <&vreg_l4f_e1_1p2>;
+
+	phys = <&ptn3222_fpr>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/mahua.dtsi b/arch/arm64/boot/dts/qcom/mahua.dtsi
index e6c05970891229..a59c5ed7b48e77 100644
--- a/arch/arm64/boot/dts/qcom/mahua.dtsi
+++ b/arch/arm64/boot/dts/qcom/mahua.dtsi
@@ -22,6 +22,8 @@
 /delete-node/ &cpu_pd16;
 /delete-node/ &cpu_pd17;
 /delete-node/ &cti_wpss;
+/delete-node/ &pcie3_phy;
+/delete-node/ &pcie3a;
 /delete-node/ &thermal_aoss_6;
 /delete-node/ &thermal_aoss_7;
 /delete-node/ &thermal_cpu_2_0_0;
@@ -79,6 +81,52 @@
 	compatible = "qcom,mahua-cnoc-cfg";
 };
 
+&gcc {
+	clocks = <&rpmhcc RPMH_CXO_CLK>,	/* Board XO source */
+		 <&rpmhcc RPMH_CXO_CLK_A>,	/* Board XO_A source */
+		 <&sleep_clk>,			/* Sleep */
+		 <0>,				/* USB 0 Phy DP0 GMUX */
+		 <0>,				/* USB 0 Phy DP1 GMUX */
+		 <0>,				/* USB 0 Phy PCIE PIPEGMUX */
+		 <0>,				/* USB 0 Phy PIPEGMUX */
+		 <0>,				/* USB 0 Phy SYS PCIE PIPEGMUX */
+		 <0>,				/* USB 1 Phy DP0 GMUX 2 */
+		 <0>,				/* USB 1 Phy DP1 GMUX 2 */
+		 <0>,				/* USB 1 Phy PCIE PIPEGMUX */
+		 <0>,				/* USB 1 Phy PIPEGMUX */
+		 <0>,				/* USB 1 Phy SYS PCIE PIPEGMUX */
+		 <0>,				/* USB 2 Phy DP0 GMUX 2 */
+		 <0>,				/* USB 2 Phy DP1 GMUX 2 */
+		 <0>,				/* USB 2 Phy PCIE PIPEGMUX */
+		 <0>,				/* USB 2 Phy PIPEGMUX */
+		 <0>,				/* USB 2 Phy SYS PCIE PIPEGMUX */
+		 <0>,				/* PCIe 3a pipe */
+		 <&pcie3b_phy>,			/* PCIe 3b pipe */
+		 <&pcie4_phy>,			/* PCIe 4 */
+		 <&pcie5_phy>,			/* PCIe 5 */
+		 <&pcie6_phy>,			/* PCIe 6 */
+		 <0>,				/* QUSB4 0 PHY RX 0 */
+		 <0>,				/* QUSB4 0 PHY RX 1 */
+		 <0>,				/* QUSB4 1 PHY RX 0 */
+		 <0>,				/* QUSB4 1 PHY RX 1 */
+		 <0>,				/* QUSB4 2 PHY RX 0 */
+		 <0>,				/* QUSB4 2 PHY RX 1 */
+		 <0>,				/* UFS PHY RX Symbol 0 */
+		 <0>,				/* UFS PHY RX Symbol 1 */
+		 <0>,				/* UFS PHY TX Symbol 0 */
+		 <&usb_0_qmpphy QMP_USB43DP_USB3_PIPE_CLK>,
+		 <&usb_1_qmpphy QMP_USB43DP_USB3_PIPE_CLK>,
+		 <&usb_2_qmpphy QMP_USB43DP_USB3_PIPE_CLK>,
+		 <&usb_mp_qmpphy0>,		/* USB3 UNI PHY pipe 0 */
+		 <&usb_mp_qmpphy1>,		/* USB3 UNI PHY pipe 1 */
+		 <0>,				/* USB4 PHY 0 pcie pipe */
+		 <0>,				/* USB4 PHY 0 Max pipe */
+		 <0>,				/* USB4 PHY 1 pcie pipe */
+		 <0>,				/* USB4 PHY 1 Max pipe */
+		 <0>,				/* USB4 PHY 2 pcie */
+		 <0>;				/* USB4 PHY 2 Max */
+};
+
 &hsc_noc {
 	compatible = "qcom,mahua-hscnoc";
 };
@@ -115,6 +163,10 @@
 	compatible = "qcom,mahua-oobm-ss-noc", "qcom,glymur-oobm-ss-noc";
 };
 
+&pcie3b_port0 {
+	phys = <&pcie3b_phy>;
+};
+
 &pcie5_phy {
 	clocks = <&gcc GCC_PCIE_PHY_5_AUX_CLK>,
 		 <&gcc GCC_PCIE_5_CFG_AHB_CLK>,
@@ -143,6 +195,43 @@
 	compatible = "qcom,mahua-pcie-west-slv-noc";
 };
 
+&soc {
+	pcie3b_phy: phy@1b90000 {
+		compatible = "qcom,glymur-qmp-gen5x4-pcie-phy";
+		reg = <0x0 0x01b90000 0x0 0x10000>;
+
+		clocks = <&gcc GCC_PCIE_PHY_3B_AUX_CLK>,
+			 <&gcc GCC_PCIE_3B_CFG_AHB_CLK>,
+			 <&tcsr TCSR_PCIE_3_CLKREF_EN>,
+			 <&gcc GCC_PCIE_3B_PHY_RCHNG_CLK>,
+			 <&gcc GCC_PCIE_3B_PIPE_CLK>,
+			 <&gcc GCC_PCIE_3B_PIPE_DIV2_CLK>;
+		clock-names = "aux",
+			      "cfg_ahb",
+			      "ref",
+			      "rchng",
+			      "pipe",
+			      "pipediv2";
+
+		resets = <&gcc GCC_PCIE_3B_PHY_BCR>,
+			 <&gcc GCC_PCIE_3B_NOCSR_COM_PHY_BCR>;
+		reset-names = "phy",
+			      "phy_nocsr";
+
+		assigned-clocks = <&gcc GCC_PCIE_3B_PHY_RCHNG_CLK>;
+		assigned-clock-rates = <100000000>;
+
+		power-domains = <&gcc GCC_PCIE_3B_PHY_GDSC>;
+
+		#clock-cells = <0>;
+		clock-output-names = "pcie3b_pipe_clk";
+
+		#phy-cells = <0>;
+
+		status = "disabled";
+	};
+};
+
 &system_noc {
 	compatible = "qcom,mahua-system-noc", "qcom,glymur-system-noc";
 };
diff --git a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
index bb3b67e2fd0182..acdcd3fa85dc98 100644
--- a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
+++ b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
@@ -24,6 +24,7 @@
 
 	aliases {
 		serial0 = &uart5;
+		serial1 = &uart11;
 	};
 
 	gpio-keys {
@@ -225,6 +226,67 @@
 			};
 		};
 	};
+
+	wcn6750-pmu {
+		compatible = "qcom,wcn6755-pmu", "qcom,wcn6750-pmu";
+
+		vddaon-supply = <&vreg_s3b>;
+		vddasd-supply = <&vreg_l7b>;
+		vddpmu-supply = <&vreg_s3b>;
+		vddrfa0p8-supply = <&vreg_s3b>;
+		vddrfa1p2-supply = <&vreg_s2b>;
+		vddrfa1p7-supply = <&vreg_s1b>;
+		vddrfa2p2-supply = <&vreg_s1j>;
+
+		bt-enable-gpios = <&tlmm 53 GPIO_ACTIVE_HIGH>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK1>;
+
+		pinctrl-0 = <&bluetooth_enable_default>, <&pmk8550_sleep_clk_default>;
+		pinctrl-names = "default";
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p7: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p7";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
 };
 
 &apps_rsc {
@@ -723,6 +785,17 @@
 	qcom,tune-usb2-preem = /bits/ 8 <0x6>;
 };
 
+&pmk8550_gpios {
+	pmk8550_sleep_clk_default: sleep-clk-default-state {
+		pins = "gpio5";
+		function = "func1";
+		input-disable;
+		output-enable;
+		bias-disable;
+		power-source = <0>;
+	};
+};
+
 &pmr735b_gpios {
 	s1j_enable_default: s1j-enable-default-state {
 		pins = "gpio1";
@@ -771,6 +844,39 @@
 	status = "okay";
 };
 
+&qup_uart11_cts {
+	/*
+	 * Configure a bias-bus-hold on CTS to lower power
+	 * usage when Bluetooth is turned off. Bus hold will
+	 * maintain a low power state regardless of whether
+	 * the Bluetooth module drives the pin in either
+	 * direction or leaves the pin fully unpowered.
+	 */
+	bias-bus-hold;
+};
+
+&qup_uart11_rts {
+	/* We'll drive RTS, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart11_rx {
+	/*
+	 * Configure a pull-up on RX. This is needed to avoid
+	 * garbage data when the TX pin of the Bluetooth module is
+	 * in tri-state (module powered off or not driving the
+	 * signal yet).
+	 */
+	bias-pull-up;
+};
+
+&qup_uart11_tx {
+	/* We'll drive TX, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
 &qupv3_id_0 {
 	status = "okay";
 };
@@ -834,6 +940,59 @@
 			       <13 1>, /* NC */
 			       <63 2>; /* WLAN UART */
 
+	qup_uart11_sleep_cts: qup-uart11-sleep-cts-state {
+		pins = "gpio48";
+		function = "gpio";
+		/*
+		 * Configure a bias-bus-hold on CTS to lower power
+		 * usage when Bluetooth is turned off. Bus hold will
+		 * maintain a low power state regardless of whether
+		 * the Bluetooth module drives the pin in either
+		 * direction or leaves the pin fully unpowered.
+		 */
+		bias-bus-hold;
+	};
+
+	qup_uart11_sleep_rts: qup-uart11-sleep-rts-state {
+		pins = "gpio49";
+		function = "gpio";
+		/*
+		 * Configure pull-down on RTS. As RTS is active low
+		 * signal, pull it low to indicate the BT SoC that it
+		 * can wakeup the system anytime from suspend state by
+		 * pulling RX low (by sending wakeup bytes).
+		 */
+		bias-pull-down;
+	};
+
+	qup_uart11_sleep_tx: qup-uart11-sleep-tx-state {
+		pins = "gpio50";
+		function = "gpio";
+		/*
+		 * Configure pull-up on TX when it isn't actively driven
+		 * to prevent BT SoC from receiving garbage during sleep.
+		 */
+		bias-pull-up;
+	};
+
+	qup_uart11_sleep_rx: qup-uart11-sleep-rx-state {
+		pins = "gpio51";
+		function = "gpio";
+		/*
+		 * Configure a pull-up on RX. This is needed to avoid
+		 * garbage data when the TX pin of the Bluetooth module
+		 * is floating which may cause spurious wakeups.
+		 */
+		bias-pull-up;
+	};
+
+	bluetooth_enable_default: bluetooth-enable-default-state {
+		pins = "gpio53";
+		function = "gpio";
+		output-low;
+		bias-disable;
+	};
+
 	sdc2_card_det_n: sdc2-card-det-state {
 		pins = "gpio65";
 		function = "gpio";
@@ -874,6 +1033,32 @@
 	status = "okay";
 };
 
+&uart11 {
+	/delete-property/ interrupts;
+	interrupts-extended = <&intc GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH 0>,
+			      <&tlmm 51 IRQ_TYPE_EDGE_FALLING>;
+
+	pinctrl-1 =  <&qup_uart11_sleep_cts>,
+		     <&qup_uart11_sleep_rts>,
+		     <&qup_uart11_sleep_tx>,
+		     <&qup_uart11_sleep_rx>;
+	pinctrl-names = "default",
+			"sleep";
+
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn6755-bt", "qcom,wcn6750-bt";
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+	};
+};
+
 &ufs_mem_hc {
 	reset-gpios = <&tlmm 167 GPIO_ACTIVE_LOW>;
 
@@ -913,3 +1098,9 @@
 
 	status = "okay";
 };
+
+&wifi {
+	qcom,calibration-variant = "Fairphone_Gen_6";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi
index aad8bf6789162f..ed994c2170af37 100644
--- a/arch/arm64/boot/dts/qcom/milos.dtsi
+++ b/arch/arm64/boot/dts/qcom/milos.dtsi
@@ -15,10 +15,10 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,milos-rpmh.h>
+#include <dt-bindings/interconnect/qcom,osm-l3.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
@@ -66,6 +66,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 0>;
 
+			operating-points-v2 = <&cpu0_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 
 			l2_0: l2-cache {
@@ -99,6 +106,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 0>;
 
+			operating-points-v2 = <&cpu0_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 		};
 
@@ -119,6 +133,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 0>;
 
+			operating-points-v2 = <&cpu0_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 
 			l2_2: l2-cache {
@@ -146,6 +167,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 0>;
 
+			operating-points-v2 = <&cpu0_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 		};
 
@@ -166,6 +194,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 1>;
 
+			operating-points-v2 = <&cpu4_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 
 			l2_4: l2-cache {
@@ -193,6 +228,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 1>;
 
+			operating-points-v2 = <&cpu4_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 
 			l2_5: l2-cache {
@@ -220,6 +262,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 1>;
 
+			operating-points-v2 = <&cpu4_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 
 			l2_6: l2-cache {
@@ -247,6 +296,13 @@
 
 			qcom,freq-domain = <&cpufreq_hw 2>;
 
+			operating-points-v2 = <&cpu7_opp_table>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
+
 			#cooling-cells = <2>;
 
 			l2_7: l2-cache {
@@ -395,6 +451,281 @@
 		reg = <0 0 0 0>;
 	};
 
+	cpu0_opp_table: opp-table-cpu0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-441600000 {
+			opp-hz = /bits/ 64 <441600000>;
+			opp-peak-kBps = <(547000 * 4) (364800 * 32)>;
+		};
+
+		opp-595200000 {
+			opp-hz = /bits/ 64 <595200000>;
+			opp-peak-kBps = <(547000 * 4) (556800 * 32)>;
+		};
+
+		opp-787200000 {
+			opp-hz = /bits/ 64 <787200000>;
+			opp-peak-kBps = <(547000 * 4) (710400 * 32)>;
+		};
+
+		opp-902400000 {
+			opp-hz = /bits/ 64 <902400000>;
+			opp-peak-kBps = <(547000 * 4) (806400 * 32)>;
+		};
+
+		opp-1017600000 {
+			opp-hz = /bits/ 64 <1017600000>;
+			opp-peak-kBps = <(547000 * 4) (998400 * 32)>;
+		};
+
+		opp-1113600000 {
+			opp-hz = /bits/ 64 <1113600000>;
+			opp-peak-kBps = <(547000 * 4) (998400 * 32)>;
+		};
+
+		opp-1228800000 {
+			opp-hz = /bits/ 64 <1228800000>;
+			opp-peak-kBps = <(768000 * 4) (1094400 * 32)>;
+		};
+
+		opp-1344000000 {
+			opp-hz = /bits/ 64 <1344000000>;
+			opp-peak-kBps = <(768000 * 4) (1209600 * 32)>;
+		};
+
+		opp-1497600000 {
+			opp-hz = /bits/ 64 <1497600000>;
+			opp-peak-kBps = <(768000 * 4) (1363200 * 32)>;
+		};
+
+		opp-1593600000 {
+			opp-hz = /bits/ 64 <1593600000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (1363200 * 32)>;
+		};
+
+		opp-1708800000 {
+			opp-hz = /bits/ 64 <1708800000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (1497600 * 32)>;
+		};
+
+		opp-1804800000 {
+			opp-hz = /bits/ 64 <1804800000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (1516800 * 32)>;
+		};
+
+		opp-2054400000 {
+			opp-hz = /bits/ 64 <2054400000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (1804800 * 32)>;
+		};
+	};
+
+	cpu4_opp_table: opp-table-cpu4 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-480000000 {
+			opp-hz = /bits/ 64 <480000000>;
+			opp-peak-kBps = <(547000 * 4) (364800 * 32)>;
+		};
+
+		opp-633600000 {
+			opp-hz = /bits/ 64 <633600000>;
+			opp-peak-kBps = <(547000 * 4) (556800 * 32)>;
+		};
+
+		opp-787200000 {
+			opp-hz = /bits/ 64 <787200000>;
+			opp-peak-kBps = <(547000 * 4) (556800 * 32)>;
+		};
+
+		opp-940800000 {
+			opp-hz = /bits/ 64 <940800000>;
+			opp-peak-kBps = <(547000 * 4) (556800 * 32)>;
+		};
+
+		opp-1056000000 {
+			opp-hz = /bits/ 64 <1056000000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 768000 */
+			opp-peak-kBps = <(768000 * 4) (710400 * 32)>;
+		};
+
+		opp-1190400000 {
+			opp-hz = /bits/ 64 <1190400000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 768000 */
+			opp-peak-kBps = <(768000 * 4) (710400 * 32)>;
+		};
+
+		opp-1286400000 {
+			opp-hz = /bits/ 64 <1286400000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (902400 * 32)>;
+		};
+
+		opp-1401600000 {
+			opp-hz = /bits/ 64 <1401600000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (1209600 * 32)>;
+		};
+
+		opp-1497600000 {
+			opp-hz = /bits/ 64 <1497600000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (1209600 * 32)>;
+		};
+
+		opp-1612800000 {
+			opp-hz = /bits/ 64 <1612800000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 1555000 */
+			opp-peak-kBps = <(1555000 * 4) (1363200 * 32)>;
+		};
+
+		opp-1708800000 {
+			opp-hz = /bits/ 64 <1708800000>;
+			opp-peak-kBps = <(1708000 * 4) (1363200 * 32)>;
+		};
+
+		opp-1824000000 {
+			opp-hz = /bits/ 64 <1824000000>;
+			opp-peak-kBps = <(1708000 * 4) (1497600 * 32)>;
+		};
+
+		opp-1920000000 {
+			opp-hz = /bits/ 64 <1920000000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 2092000 */
+			opp-peak-kBps = <(2092000 * 4) (1497600 * 32)>;
+		};
+
+		opp-2016000000 {
+			opp-hz = /bits/ 64 <2016000000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 2092000 */
+			opp-peak-kBps = <(2092000 * 4) (1497600 * 32)>;
+		};
+
+		opp-2073600000 {
+			opp-hz = /bits/ 64 <2073600000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 2092000 */
+			opp-peak-kBps = <(2092000 * 4) (1497600 * 32)>;
+		};
+
+		opp-2208000000 {
+			opp-hz = /bits/ 64 <2208000000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 2092000 */
+			opp-peak-kBps = <(2092000 * 4) (1516800 * 32)>;
+		};
+
+		opp-2400000000 {
+			opp-hz = /bits/ 64 <2400000000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1516800 * 32)>;
+		};
+
+		opp-2707200000 {
+			opp-hz = /bits/ 64 <2707200000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1804800 * 32)>;
+		};
+	};
+
+	cpu7_opp_table: opp-table-cpu7 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-480000000 {
+			opp-hz = /bits/ 64 <480000000>;
+			opp-peak-kBps = <(547000 * 4) (364800 * 32)>;
+		};
+
+		opp-633600000 {
+			opp-hz = /bits/ 64 <633600000>;
+			opp-peak-kBps = <(547000 * 4) (556800 * 32)>;
+		};
+
+		opp-787200000 {
+			opp-hz = /bits/ 64 <787200000>;
+			opp-peak-kBps = <(547000 * 4) (806400 * 32)>;
+		};
+
+		opp-960000000 {
+			opp-hz = /bits/ 64 <960000000>;
+			opp-peak-kBps = <(547000 * 4) (806400 * 32)>;
+		};
+
+		opp-1094400000 {
+			opp-hz = /bits/ 64 <1094400000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 768000 */
+			opp-peak-kBps = <(768000 * 4) (998400 * 32)>;
+		};
+
+		opp-1209600000 {
+			opp-hz = /bits/ 64 <1209600000>;
+			/* TODO: ddr4 = 1017000, ddr5 = 768000 */
+			opp-peak-kBps = <(768000 * 4) (998400 * 32)>;
+		};
+
+		opp-1324800000 {
+			opp-hz = /bits/ 64 <1324800000>;
+			opp-peak-kBps = <(1555000 * 4) (998400 * 32)>;
+		};
+
+		opp-1459200000 {
+			opp-hz = /bits/ 64 <1459200000>;
+			opp-peak-kBps = <(1555000 * 4) (1209600 * 32)>;
+		};
+
+		opp-1651200000 {
+			opp-hz = /bits/ 64 <1651200000>;
+			opp-peak-kBps = <(1708000 * 4) (1209600 * 32)>;
+		};
+
+		opp-1766400000 {
+			opp-hz = /bits/ 64 <1766400000>;
+			/* TODO: ddr4 = 1708000, ddr5 = 2092000 */
+			opp-peak-kBps = <(2092000 * 4) (1363200 * 32)>;
+		};
+
+		opp-1900800000 {
+			opp-hz = /bits/ 64 <1900800000>;
+			/* TODO: ddr4 = 2092000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1497600 * 32)>;
+		};
+
+		opp-2208000000 {
+			opp-hz = /bits/ 64 <2208000000>;
+			/* TODO: ddr4 = 2092000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1497600 * 32)>;
+		};
+
+		opp-2304000000 {
+			opp-hz = /bits/ 64 <2304000000>;
+			/* TODO: ddr4 = 2092000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1516800 * 32)>;
+		};
+
+		opp-2496000000 {
+			opp-hz = /bits/ 64 <2496000000>;
+			/* TODO: ddr4 = 2092000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1516800 * 32)>;
+		};
+
+		opp-2707200000 {
+			opp-hz = /bits/ 64 <2707200000>;
+			/* TODO: ddr4 = 2092000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1804800 * 32)>;
+		};
+
+		opp-2918400000 {
+			opp-hz = /bits/ 64 <2918400000>;
+			/* TODO: ddr4 = 2092000, ddr5 = 3196000 */
+			opp-peak-kBps = <(3196000 * 4) (1804800 * 32)>;
+		};
+	};
+
 	pmu-a520 {
 		compatible = "arm,cortex-a520-pmu";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW &ppi_cluster0>;
@@ -1949,6 +2280,11 @@
 				 <&sleep_clk>,
 				 <&gcc GCC_VIDEO_AHB_CLK>;
 
+			power-domains = <&rpmhpd RPMHPD_CX>,
+				<&rpmhpd RPMHPD_MX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+				<&rpmhpd_opp_low_svs>;
+
 			#clock-cells = <1>;
 			#reset-cells = <1>;
 			#power-domain-cells = <1>;
@@ -2031,6 +2367,11 @@
 			interconnects = <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS
 					 &mmss_noc SLAVE_MNOC_HF_MEM_NOC QCOM_ICC_TAG_ALWAYS>;
 
+			power-domains = <&rpmhpd RPMHPD_CX>,
+				<&rpmhpd RPMHPD_MX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+				<&rpmhpd_opp_low_svs>;
+
 			#clock-cells = <1>;
 			#reset-cells = <1>;
 			#power-domain-cells = <1>;
@@ -2340,7 +2681,6 @@
 
 			qcom,ee = <0>;
 			qcom,channel = <0>;
-			qcom,bus-id = <0>;
 
 			interrupt-controller;
 			#interrupt-cells = <4>;
@@ -2413,19 +2753,27 @@
 			};
 
 			qup_uart11_cts_rts: qup-uart11-cts-rts-state {
-				/* CTS, RTS */
-				pins = "gpio48", "gpio49";
-				function = "qup1_se4";
-				drive-strength = <2>;
-				bias-pull-down;
+				qup_uart11_cts: cts-pins {
+					pins = "gpio48";
+					function = "qup1_se4";
+				};
+
+				qup_uart11_rts: rts-pins {
+					pins = "gpio49";
+					function = "qup1_se4";
+				};
 			};
 
 			qup_uart11_default: qup-uart11-default-state {
-				/* TX, RX */
-				pins = "gpio50", "gpio51";
-				function = "qup1_se4";
-				drive-strength = <2>;
-				bias-pull-up;
+				qup_uart11_tx: tx-pins {
+					pins = "gpio50";
+					function = "qup1_se4";
+				};
+
+				qup_uart11_rx: rx-pins {
+					pins = "gpio51";
+					function = "qup1_se4";
+				};
 			};
 
 			sdc2_default: sdc2-default-state {
@@ -2764,6 +3112,52 @@
 			};
 		};
 
+		wifi: wifi@17110040 {
+			compatible = "qcom,wcn6755-wifi", "qcom,wcn6750-wifi";
+			reg = <0x0 0x17110040 0x0 0x0>;
+
+			iommus = <&apps_smmu 0x1400 0x1>;
+
+			interrupts = <GIC_SPI 768 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 769 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 770 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 771 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 772 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 773 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 774 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 775 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 776 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 777 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 778 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 779 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 780 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 781 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 782 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 783 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 784 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 785 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 786 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 787 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 788 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 789 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 790 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 791 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 792 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 793 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 794 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 795 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 796 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 797 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 798 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 799 IRQ_TYPE_EDGE_RISING 0>;
+
+			qcom,rproc = <&remoteproc_wpss>;
+			qcom,smem-states = <&smp2p_wlan_out 0>;
+			qcom,smem-state-names = "wlan-smp2p-out";
+
+			status = "disabled";
+		};
+
 		timer@17420000 {
 			compatible = "arm,armv7-timer-mem";
 			reg = <0x0 0x17420000 0x0 0x1000>;
@@ -2925,6 +3319,18 @@
 			};
 		};
 
+		epss_l3: interconnect@17d90000 {
+			compatible = "qcom,milos-epss-l3", "qcom,epss-l3";
+			reg = <0x0 0x17d90000 0x0 0x1000>;
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&gcc GCC_GPLL0>;
+			clock-names = "xo",
+				      "alternate";
+
+			#interconnect-cells = <1>;
+		};
+
 		cpufreq_hw: cpufreq@17d91000 {
 			compatible = "qcom,milos-cpufreq-epss", "qcom,cpufreq-epss";
 			reg = <0x0 0x17d91000 0x0 0x1000>,
@@ -2950,6 +3356,55 @@
 			#clock-cells = <1>;
 		};
 
+		pmu@24091000 {
+			compatible = "qcom,milos-cpu-bwmon", "qcom,sc7280-llcc-bwmon";
+			reg = <0x0 0x24091000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>;
+
+			operating-points-v2 = <&llcc_bwmon_opp_table>;
+
+			llcc_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <2086000>;
+				};
+
+				opp-1 {
+					opp-peak-kBps = <2929000>;
+				};
+
+				opp-2 {
+					/* TODO: ddr4 = 3879000 */
+					opp-peak-kBps = <5931000>;
+				};
+
+				opp-3 {
+					/* TODO: ddr4 = 5163000 */
+					opp-peak-kBps = <6515000>;
+				};
+
+				opp-4 {
+					/* TODO: ddr4 = 5931000 */
+					opp-peak-kBps = <7980000>;
+				};
+
+				opp-5 {
+					/* TODO: ddr4 = 6515000 */
+					opp-peak-kBps = <10437000>;
+				};
+
+				opp-6 {
+					/* TODO: ddr4 = 7980000 */
+					opp-peak-kBps = <12191000>;
+				};
+			};
+		};
+
 		gem_noc: interconnect@24100000 {
 			compatible = "qcom,milos-gem-noc";
 			reg = <0x0 0x24100000 0x0 0xff080>;
diff --git a/arch/arm64/boot/dts/qcom/monaco-ac-evk.dts b/arch/arm64/boot/dts/qcom/monaco-ac-evk.dts
new file mode 100644
index 00000000000000..6f3c3ffefe3403
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/monaco-ac-evk.dts
@@ -0,0 +1,938 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/sound/qcom,q6afe.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "monaco.dtsi"
+#include "monaco-pmics.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. Monaco-ac EVK";
+	compatible = "qcom,monaco-evk", "qcom,qcs8300";
+
+	aliases {
+		ethernet0 = &ethernet0;
+		i2c1 = &i2c1;
+		serial0 = &uart7;
+		serial2 = &uart6;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	connector-2 {
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		label = "micro-USB";
+		type = "micro";
+
+		id-gpios = <&pmm8620au_0_gpios 9 GPIO_ACTIVE_HIGH>;
+		vbus-gpios = <&expander6 7 GPIO_ACTIVE_HIGH>;
+		vbus-supply = <&usb2_vbus>;
+
+		pinctrl-0 = <&usb2_id>;
+		pinctrl-names = "default";
+
+		port {
+			usb2_con_hs_ep: endpoint {
+				remote-endpoint = <&usb_2_dwc3_hs>;
+			};
+		};
+	};
+
+	dmic: audio-codec-0 {
+		compatible = "dmic-codec";
+		#sound-dai-cells = <0>;
+		num-channels = <1>;
+	};
+
+	max98357a: audio-codec-1 {
+		compatible = "maxim,max98357a";
+		#sound-dai-cells = <0>;
+	};
+
+	dp-connector-0 {
+		compatible = "dp-connector";
+		label = "DP0";
+		type = "mini";
+
+		port {
+			dp0_connector_in: endpoint {
+				remote-endpoint = <&lt8713sx_dp0_out>;
+			};
+		};
+	};
+
+	dp-connector-1 {
+		compatible = "dp-connector";
+		label = "DP1";
+		type = "mini";
+
+		port {
+			dp1_connector_in: endpoint {
+				remote-endpoint = <&lt8713sx_dp1_out>;
+			};
+		};
+	};
+
+	usb2_vbus: regulator-usb2-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb2_vbus";
+		gpio = <&pmm8650au_1_gpios 7 GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		enable-active-high;
+	};
+
+	sound {
+		compatible = "qcom,qcs8275-sndcard";
+		model = "MONACO-EVK";
+
+		pinctrl-0 = <&hs0_mi2s_active>, <&mi2s1_active>;
+		pinctrl-names = "default";
+
+		hs0-mi2s-playback-dai-link {
+			link-name = "HS0 MI2S Playback";
+
+			codec {
+				sound-dai = <&max98357a>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai PRIMARY_MI2S_RX>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		sec-mi2s-capture-dai-link {
+			link-name = "Secondary MI2S Capture";
+
+			codec {
+				sound-dai = <&dmic>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai SECONDARY_MI2S_TX>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	vreg_cam0_2p8: vreg-cam0-2p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_cam0_2p8";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		startup-delay-us = <10000>;
+
+		gpio = <&tlmm 73 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam0_avdd_2v8_en_default>;
+		pinctrl-names = "default";
+	};
+
+	vreg_cam1_2p8: vreg-cam1-2p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_cam1_2p8";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		startup-delay-us = <10000>;
+
+		gpio = <&tlmm 74 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam1_avdd_2v8_en_default>;
+		pinctrl-names = "default";
+	};
+
+	vreg_cam2_2p8: vreg-cam2-2p8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_cam2_2p8";
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+		startup-delay-us = <10000>;
+
+		gpio = <&tlmm 75 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam2_avdd_2v8_en_default>;
+		pinctrl-names = "default";
+	};
+
+	/* This comes from a PMIC handled within the SAIL domain */
+	vreg_s2s: vreg-s2s {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_s2s";
+
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pmm8654au-rpmh-regulators";
+		qcom,pmic-id = "a";
+
+		vreg_l3a: ldo3 {
+			regulator-name = "vreg_l3a";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4a: ldo4 {
+			regulator-name = "vreg_l4a";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5a: ldo5 {
+			regulator-name = "vreg_l5a";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6a: ldo6 {
+			regulator-name = "vreg_l6a";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7a: ldo7 {
+			regulator-name = "vreg_l7a";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8a: ldo8 {
+			regulator-name = "vreg_l8a";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9a: ldo9 {
+			regulator-name = "vreg_l9a";
+			regulator-min-microvolt = <2970000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+		vreg_s4a: smps4 {
+			regulator-name = "vreg_s4a";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s9a: smps9 {
+			regulator-name = "vreg_s9a";
+			regulator-min-microvolt = <1352000>;
+			regulator-max-microvolt = <1352000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pmm8654au-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vreg_s5c: smps5 {
+			regulator-name = "vreg_s5c";
+			regulator-min-microvolt = <1104000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1c: ldo1 {
+			regulator-name = "vreg_l1c";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <512000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c: ldo2 {
+			regulator-name = "vreg_l2c";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c: ldo4 {
+			regulator-name = "vreg_l4c";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c: ldo7 {
+			regulator-name = "vreg_l7c";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c: ldo8 {
+			regulator-name = "vreg_l8c";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c: ldo9 {
+			regulator-name = "vreg_l9c";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&ethernet0 {
+	phy-mode = "2500base-x";
+	phy-handle = <&hsgmii_phy0>;
+
+	pinctrl-0 = <&ethernet0_default>;
+	pinctrl-names = "default";
+
+	snps,mtl-rx-config = <&mtl_rx_setup>;
+	snps,mtl-tx-config = <&mtl_tx_setup>;
+	nvmem-cells = <&mac_addr0>;
+	nvmem-cell-names = "mac-address";
+
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		hsgmii_phy0: ethernet-phy@1c {
+			compatible = "ethernet-phy-id004d.d101";
+			reg = <0x1c>;
+			reset-gpios = <&tlmm 31 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <11000>;
+			reset-deassert-us = <70000>;
+		};
+	};
+
+	mtl_rx_setup: rx-queues-config {
+		snps,rx-queues-to-use = <4>;
+		snps,rx-sched-sp;
+
+		queue0 {
+			snps,dcb-algorithm;
+			snps,map-to-dma-channel = <0x0>;
+			snps,route-up;
+			snps,priority = <0x1>;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+			snps,map-to-dma-channel = <0x1>;
+			snps,route-ptp;
+		};
+
+		queue2 {
+			snps,avb-algorithm;
+			snps,map-to-dma-channel = <0x2>;
+			snps,route-avcp;
+		};
+
+		queue3 {
+			snps,avb-algorithm;
+			snps,map-to-dma-channel = <0x3>;
+			snps,priority = <0xc>;
+		};
+	};
+
+	mtl_tx_setup: tx-queues-config {
+		snps,tx-queues-to-use = <4>;
+
+		queue0 {
+			snps,dcb-algorithm;
+		};
+
+		queue1 {
+			snps,dcb-algorithm;
+		};
+
+		queue2 {
+			snps,avb-algorithm;
+			snps,send_slope = <0x1000>;
+			snps,idle_slope = <0x1000>;
+			snps,high_credit = <0x3e800>;
+			snps,low_credit = <0xffc18000>;
+		};
+
+		queue3 {
+			snps,avb-algorithm;
+			snps,send_slope = <0x1000>;
+			snps,idle_slope = <0x1000>;
+			snps,high_credit = <0x3e800>;
+			snps,low_credit = <0xffc18000>;
+		};
+	};
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/qcs8300/a623_zap.mbn";
+};
+
+&i2c0 {
+	status = "okay";
+
+	bridge@4f {
+		compatible = "lontium,lt8713sx";
+		reg = <0x4f>;
+		reset-gpios = <&expander5 6 GPIO_ACTIVE_LOW>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lt8713sx_dp_in: endpoint {
+					remote-endpoint = <&mdss_dp0_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				lt8713sx_dp0_out: endpoint {
+					remote-endpoint = <&dp0_connector_in>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				lt8713sx_dp1_out: endpoint {
+					remote-endpoint = <&dp1_connector_in>;
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-0 = <&qup_i2c1_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	fan_controller: fan@18 {
+		compatible = "ti,amc6821";
+		reg = <0x18>;
+		#pwm-cells = <2>;
+
+		fan {
+			pwms = <&fan_controller 40000 PWM_POLARITY_INVERTED>;
+		};
+	};
+
+	eeprom0: eeprom@50 {
+		compatible = "atmel,24c256";
+		reg = <0x50>;
+		pagesize = <64>;
+
+		nvmem-layout {
+			compatible = "fixed-layout";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			mac_addr0: mac-addr@0 {
+				reg = <0x0 0x6>;
+			};
+		};
+	};
+};
+
+&i2c15 {
+	pinctrl-0 = <&qup_i2c15_default>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	expander0: gpio@38 {
+		compatible = "ti,tca9538";
+		reg = <0x38>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 56 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander0_int>;
+		pinctrl-names = "default";
+	};
+
+	expander1: gpio@39 {
+		compatible = "ti,tca9538";
+		reg = <0x39>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 16 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander1_int>;
+		pinctrl-names = "default";
+	};
+
+	expander2: gpio@3a {
+		compatible = "ti,tca9538";
+		reg = <0x3a>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 95 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander2_int>;
+		pinctrl-names = "default";
+	};
+
+	expander3: gpio@3b {
+		compatible = "ti,tca9538";
+		reg = <0x3b>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 24 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander3_int>;
+		pinctrl-names = "default";
+	};
+
+	expander4: gpio@3c {
+		compatible = "ti,tca9538";
+		reg = <0x3c>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 96 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander4_int>;
+		pinctrl-names = "default";
+	};
+
+	expander5: gpio@3d {
+		compatible = "ti,tca9538";
+		reg = <0x3d>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 3 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander5_int>;
+		pinctrl-names = "default";
+	};
+
+	expander6: gpio@3e {
+		compatible = "ti,tca9538";
+		reg = <0x3e>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupts-extended = <&tlmm 52 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&expander6_int>;
+		pinctrl-names = "default";
+	};
+};
+
+&iris {
+	status = "okay";
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	pinctrl-0 = <&dp_hot_plug_det>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&mdss_dp0_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+	remote-endpoint = <&lt8713sx_dp_in>;
+};
+
+&mdss_dp0_phy {
+	vdda-phy-supply = <&vreg_l5a>;
+	vdda-pll-supply = <&vreg_l4a>;
+
+	status = "okay";
+};
+
+&pcie0 {
+	pinctrl-0 = <&pcie0_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l6a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+&pcie1 {
+	pinctrl-0 = <&pcie1_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie1_phy {
+	vdda-phy-supply = <&vreg_l6a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+&pcieport0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>;
+};
+
+&pcieport1 {
+	reset-gpios = <&tlmm 23 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>;
+};
+
+&pmm8620au_0_gpios {
+	usb2_id: usb2-id-state {
+		pins = "gpio9";
+		function = "normal";
+		input-enable;
+		bias-pull-up;
+		power-source = <0>;
+	};
+};
+
+&qup_i2c0_data_clk {
+	drive-strength = <2>;
+	bias-pull-up;
+};
+
+&qupv3_id_0 {
+	firmware-name = "qcom/qcs8300/qupv3fw.elf";
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	firmware-name = "qcom/qcs8300/qupv3fw.elf";
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/qcs8300/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/qcs8300/cdsp0.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_gpdsp {
+	firmware-name = "qcom/qcs8300/gpdsp0.mbn";
+
+	status = "okay";
+};
+
+&sdhc_1 {
+	vmmc-supply = <&vreg_l8a>;
+	vqmmc-supply = <&vreg_s2s>;
+
+	no-sd;
+	no-sdio;
+	non-removable;
+
+	status = "okay";
+};
+
+&serdes0 {
+	phy-supply = <&vreg_l4a>;
+
+	status = "okay";
+};
+
+&spi10 {
+	status = "okay";
+
+	tpm@0 {
+		compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi";
+		reg = <0>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&tlmm {
+	pcie0_default_state: pcie0-default-state {
+		wake-pins {
+			pins = "gpio0";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		clkreq-pins {
+			pins = "gpio1";
+			function = "pcie0_clkreq";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-pins {
+			pins = "gpio2";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	ethernet0_default: ethernet0-default-state {
+		ethernet0_mdc: ethernet0-mdc-pins {
+			pins = "gpio5";
+			function = "emac0_mdc";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+
+		ethernet0_mdio: ethernet0-mdio-pins {
+			pins = "gpio6";
+			function = "emac0_mdio";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+	};
+
+	expander5_int: expander5-int-state {
+		pins = "gpio3";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander1_int: expander1-int-state {
+		pins = "gpio16";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	qup_i2c1_default: qup-i2c1-state {
+		pins = "gpio19", "gpio20";
+		function = "qup0_se1";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	pcie1_default_state: pcie1-default-state {
+		wake-pins {
+			pins = "gpio21";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		clkreq-pins {
+			pins = "gpio22";
+			function = "pcie1_clkreq";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		perst-pins {
+			pins = "gpio23";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	expander3_int: expander3-int-state {
+		pins = "gpio24";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander6_int:  expander6-int-state {
+		pins = "gpio52";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander0_int: expander0-int-state {
+		pins = "gpio56";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	cam0_avdd_2v8_en_default: cam0-avdd-2v8-en-state {
+		pins = "gpio73";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	cam1_avdd_2v8_en_default: cam1-avdd-2v8-en-state {
+		pins = "gpio74";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	cam2_avdd_2v8_en_default: cam2-avdd-2v8-en-state {
+		pins = "gpio75";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_i2c15_default: qup-i2c15-state {
+		pins = "gpio91", "gpio92";
+		function = "qup1_se7";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	expander2_int: expander2-int-state {
+		pins = "gpio95";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	expander4_int: expander4-int-state {
+		pins = "gpio96";
+		function = "gpio";
+		bias-pull-up;
+	};
+};
+
+&uart6 {
+	status = "okay";
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+	vcc-supply = <&vreg_l8a>;
+	vcc-max-microamp = <1100000>;
+	vccq-supply = <&vreg_l4c>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l4a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&vreg_l7a>;
+	vdda18-supply = <&vreg_l7c>;
+	vdda33-supply = <&vreg_l9a>;
+
+	status = "okay";
+};
+
+&usb_qmpphy {
+	vdda-phy-supply = <&vreg_l7a>;
+	vdda-pll-supply = <&vreg_l5a>;
+
+	status = "okay";
+};
+
+&usb_2 {
+	status = "okay";
+};
+
+&usb_2_dwc3_hs {
+	remote-endpoint = <&usb2_con_hs_ep>;
+};
+
+&usb_2_hsphy {
+	vdda-pll-supply = <&vreg_l7a>;
+	vdda18-supply = <&vreg_l7c>;
+	vdda33-supply = <&vreg_l9a>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts b/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
index 299d0da15a1501..940a03732eefc4 100644
--- a/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
+++ b/arch/arm64/boot/dts/qcom/monaco-arduino-monza.dts
@@ -7,6 +7,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
 #include <dt-bindings/pwm/pwm.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
 
@@ -32,6 +33,14 @@
 		pinctrl-0 = <&gpio_keys_default>;
 		pinctrl-names = "default";
 
+		button-sleep {
+			label = "Sleep Key";
+			linux,code = <KEY_SLEEP>;
+			gpios = <&tlmm 69 GPIO_ACTIVE_HIGH>;
+			debounce-interval = <60>;
+			wakeup-source;
+		};
+
 		button-home {
 			label = "Home Key";
 			linux,code = <KEY_HOMEPAGE>;
@@ -176,6 +185,109 @@
 };
 
 &cci1_i2c1 {
+	led-controller@22 {
+		compatible = "nxp,pca9635";
+		reg = <0x22>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		multi-led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_RGB>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@0 {
+				reg = <0>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+
+		multi-led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_RGB>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@3 {
+				reg = <3>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@4 {
+				reg = <4>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@5 {
+				reg = <5>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+
+		multi-led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_RGB>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <3>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@6 {
+				reg = <6>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@7 {
+				reg = <7>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@8 {
+				reg = <8>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+
+		multi-led@3 {
+			reg = <3>;
+			color = <LED_COLOR_ID_RGB>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@9 {
+				reg = <9>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@a {
+				reg = <10>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@b {
+				reg = <11>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+	};
+
 	fan_controller: fan-controller@2f {
 		compatible = "microchip,emc2305";
 		reg = <0x2f>;
@@ -452,26 +564,85 @@
 
 &thermal_zones {
 	cpuss-0-0-thermal {
-		polling-delay = <2000>;
-
 		trips {
-			cpuss0_active: cpuss-active {
-				temperature = <32000>;
-				hysteresis = <2000>;
+			cpuss0_active0: cpuss-active0 {
+				temperature = <50000>;
+				hysteresis = <4000>;
+				type = "active";
+			};
+
+			cpuss0_active1: cpuss-active1 {
+				temperature = <60000>;
+				hysteresis = <4000>;
+				type = "active";
+			};
+
+			cpuss0_active2: cpuss-active2 {
+				temperature = <70000>;
+				hysteresis = <4000>;
+				type = "active";
+			};
+
+			cpuss0_active3: cpuss-active3 {
+				temperature = <80000>;
+				hysteresis = <4000>;
 				type = "active";
 			};
 		};
 
 		cooling-maps {
-			map-active {
-				trip = <&cpuss0_active>;
-				cooling-device = <&fan 0 THERMAL_NO_LIMIT>;
+			map-active0 {
+				trip = <&cpuss0_active0>;
+				cooling-device = <&fan 4 4>;
+			};
+
+			map-active1 {
+				trip = <&cpuss0_active1>;
+				cooling-device = <&fan 6 6>;
+			};
+
+			map-active2 {
+				trip = <&cpuss0_active2>;
+				cooling-device = <&fan 8 8>;
+			};
+
+			map-active3 {
+				trip = <&cpuss0_active3>;
+				cooling-device = <&fan 10 THERMAL_NO_LIMIT>;
 			};
 		};
 	};
 };
 
 &tlmm {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "JMISC_SPI_MISO", "JMISC_SPI_MOSI", "JMISC_SPI_SCK", "JMISC_SPI_CS0", "JMISC_SPI_CS1",
+			  "JMISC_CS2", "", "JHAT_P03_I2C_SDA", "JHAT_P05_I2C_SCL", "JHAT_P27_EEPROM_SDA",
+			  "JHAT_P28_EEPROM_SDL", "", "", "", "",
+			  "JHAT_P21_SPI_MISO", "JHAT_P19_SPI_MOSI", "JHAT_P23_SPI_SCLK", "JHAT_P24_SPI_CE0", "",
+			  "", "MCU_I2C1_SDA", "MCU_I2C1_SDL", "", "",
+			  "", "", "JOMEGA_SPI_SCK", "JOMEGA_SPI_CS", "JOMEGA_SPI_MISO",
+			  "JOMEGA_SPI_MOSI", "", "MCU_PG11", "JCTL_UART_DBG_TX", "JCTL_UART_DBG_RX",
+			  "", "", "MCU_BOOT0", "CYPD_SWCLK", "",
+			  "", "", "FORCE_USB_BOOT", "", "",
+			  "", "", "JMEDIA_SOC_CC0_SDA", "JMEDIA_SOC_CC0_SCL", "JMEDIA_SOC_CC1_SDA",
+			  "JMEDIA_SOC_CC1_SCL", "", "", "MCU_PG12", "",
+			  "FLAT0_P17_GPIO", "", "JMEDIA_SOC_CAM_MCLK0", "JMEDIA_SOC_CAM_MCLK1", "MCU_PG9",
+			  "MCU_PA13_SWDIO", "MCU_PA14_SWCLK", "", "JMISC_SOC_GPIO_0", "JMISC_SOC_GPIO_1",
+			  "FLAT1_P17_GPIO", "", "JHAT_P32_PWM0", "MCU_NRST", "USER_BUTTON",
+			  "JHAT_P31_GPIO", "JHAT_P33_GPIO",  "FLAT2_P17_GPIO", "JHAT_P07_GPCLK0", "JHAT_P36_UART_CTS",
+			  "JHAT_P11_UART_RFR", "JHAT_P08_UART_TX", "JHAT_P10_UART_RX", "JHAT_P26_SPI_CE1", "JHAT_P29_GPIO",
+			  "JHAT_P15_GPIO", "", "MCU_PG10", "", "CYPD_SWDIO",
+			  "MCU_PG13", "MCU_PG14", "", "", "",
+			  "", "", "", "", "CYPD_NRES",
+			  "JHAT_P16_GPIO", "JHAT_P18_GPIO", "JHAT_P22_GPIO", "JHAT_P37_GPIO", "JHAT_P13_GPIO",
+			  "", "", "", "", "",
+			  "", "JHAT_P12_PCM_CLK", "JHAT_P35_PCM_FS", "JHAT_P38_PCM_DIN", "JHAT_P40_PCM_DOUT",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "";
+
 	pcie0_default_state: pcie0-default-state {
 		wake-pins {
 			pins = "gpio0";
@@ -547,9 +718,17 @@
 	};
 
 	gpio_keys_default: gpio-keys-default-state {
-		pins = "gpio79";
-		function = "gpio";
-		bias-disable;
+		horizontal-button-pins {
+			pins = "gpio69";
+			function = "gpio";
+			bias-pull-down;
+		};
+
+		vertical-button-pins {
+			pins = "gpio79";
+			function = "gpio";
+			bias-disable;
+		};
 	};
 
 	adv7535_default: adv7535-default-state {
diff --git a/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso
index ca5bf0fe5ec6e2..4f727ff6ecb614 100644
--- a/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/monaco-evk-ifp-mezzanine.dtso
@@ -117,20 +117,12 @@
 			ranges;
 			bus-range = <0x5 0xff>;
 
-			pcie@0,0 {
+			dev@0,0 {
 				reg = <0x50000 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 
-			pcie@0,1 {
+			dev@0,1 {
 				reg = <0x50100 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/qcom/monaco-evk.dts b/arch/arm64/boot/dts/qcom/monaco-evk.dts
index 9d17ef7d2caf13..539c9c80a808d3 100644
--- a/arch/arm64/boot/dts/qcom/monaco-evk.dts
+++ b/arch/arm64/boot/dts/qcom/monaco-evk.dts
@@ -21,6 +21,7 @@
 		ethernet0 = &ethernet0;
 		i2c1 = &i2c1;
 		serial0 = &uart7;
+		serial1 = &uart2;
 		serial2 = &uart6;
 	};
 
@@ -47,6 +48,38 @@
 		};
 	};
 
+	connector-3 {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&vreg_wcn_3p3>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcieport0_ep>;
+				};
+			};
+
+			port@3 {
+				reg = <3>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&uart2_ep>;
+				};
+			};
+		};
+	};
+
 	dmic: audio-codec-0 {
 		compatible = "dmic-codec";
 		#sound-dai-cells = <0>;
@@ -91,6 +124,17 @@
 		enable-active-high;
 	};
 
+	vreg_wcn_3p3: regulator-wcn-3p3 {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_3P3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	sound {
 		compatible = "qcom,qcs8275-sndcard";
 		model = "MONACO-EVK";
@@ -643,12 +687,18 @@
 
 &pcieport0 {
 	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 0 GPIO_ACTIVE_LOW>;
+
+	port {
+		pcieport0_ep: endpoint {
+			remote-endpoint = <&m2_e_pcie_ep>;
+		};
+	};
 };
 
 &pcieport1 {
 	reset-gpios = <&tlmm 23 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 21 GPIO_ACTIVE_LOW>;
 };
 
 &pmm8620au_0_gpios {
@@ -706,7 +756,8 @@
 };
 
 &serdes0 {
-	phy-supply = <&vreg_l4a>;
+	vdda-0p9-supply = <&vreg_l4a>;
+	vdda-1p2-supply = <&vreg_l5a>;
 
 	status = "okay";
 };
@@ -809,7 +860,7 @@
 		bias-pull-up;
 	};
 
-	expander6_int:  expander6-int-state {
+	expander6_int: expander6-int-state {
 		pins = "gpio52";
 		function = "gpio";
 		bias-pull-up;
@@ -862,6 +913,16 @@
 	};
 };
 
+&uart2 {
+	status = "okay";
+
+	port {
+		uart2_ep: endpoint {
+			remote-endpoint = <&m2_e_uart_ep>;
+		};
+	};
+};
+
 &uart6 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/monaco-pmics.dtsi b/arch/arm64/boot/dts/qcom/monaco-pmics.dtsi
index 6e096b9b7546a8..361f5163b6d91e 100644
--- a/arch/arm64/boot/dts/qcom/monaco-pmics.dtsi
+++ b/arch/arm64/boot/dts/qcom/monaco-pmics.dtsi
@@ -7,6 +7,36 @@
 #include <dt-bindings/spmi/spmi.h>
 #include "qcom-adc5-gen3.h"
 
+/ {
+	thermal-zones {
+		pm8775-mbg0-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmm8620au_0_mbg_temp>;
+
+			trips {
+				trip0 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pm8775-mbg1-thermal {
+			polling-delay-passive = <100>;
+			thermal-sensors = <&pmm8620au_1_mbg_temp>;
+
+			trips {
+				trip0 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+	};
+};
+
 &spmi_bus {
 	pmm8620au_0: pmic@0 {
 		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
@@ -53,6 +83,22 @@
 		};
 	};
 
+	pmm8620au_sail_0: pmic@1 {
+		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
+		reg = <0x1 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmm8620au_0_mbg_temp: temperature-sensor@d700 {
+			compatible = "qcom,pm8775-mbg-tm";
+			reg = <0xd700>;
+			io-channels = <&pmm8620au_0_adc ADC5_GEN3_DIE_TEMP(0)>;
+			io-channel-names = "thermal";
+			interrupts-extended = <&spmi_bus 0x1 0xd7 0x0 IRQ_TYPE_EDGE_RISING>;
+			#thermal-sensor-cells = <0>;
+		};
+	};
+
 	pmm8650au_1: pmic@2 {
 		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
 		reg = <0x2 SPMI_USID>;
@@ -90,4 +136,20 @@
 			#interrupt-cells = <2>;
 		};
 	};
+
+	pmm8620au_sail_1: pmic@3 {
+		compatible = "qcom,pmm8654au", "qcom,spmi-pmic";
+		reg = <0x3 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmm8620au_1_mbg_temp: temperature-sensor@d700 {
+			compatible = "qcom,pm8775-mbg-tm";
+			reg = <0xd700>;
+			io-channels = <&pmm8650au_1_adc ADC5_GEN3_DIE_TEMP(2)>;
+			io-channel-names = "thermal";
+			interrupts-extended = <&spmi_bus 0x3 0xd7 0x0 IRQ_TYPE_EDGE_RISING>;
+			#thermal-sensor-cells = <0>;
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi
index cfe39f923224f6..75703d94fc094f 100644
--- a/arch/arm64/boot/dts/qcom/monaco.dtsi
+++ b/arch/arm64/boot/dts/qcom/monaco.dtsi
@@ -18,7 +18,6 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
@@ -2394,6 +2393,7 @@
 			};
 
 			pcieport0: pcie@0 {
+				compatible = "pciclass,0604";
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
@@ -4845,6 +4845,8 @@
 			supports-cqe;
 			dma-coherent;
 
+			qcom,ice = <&sdhc_ice>;
+
 			mmc-ddr-1_8v;
 			mmc-hs200-1_8v;
 			mmc-hs400-1_8v;
@@ -4877,6 +4879,17 @@
 			};
 		};
 
+		sdhc_ice: crypto@87c8000 {
+			compatible = "qcom,qcs8300-inline-crypto-engine",
+				     "qcom,inline-crypto-engine";
+			reg = <0x0 0x087c8000 0x0 0x18000>;
+			clocks = <&gcc GCC_SDCC1_ICE_CORE_CLK>,
+				 <&gcc GCC_SDCC1_AHB_CLK>;
+			clock-names = "core",
+				      "iface";
+			power-domains = <&rpmhpd RPMHPD_CX>;
+		};
+
 		usb_1_hsphy: phy@8904000 {
 			compatible = "qcom,qcs8300-usb-hs-phy",
 				     "qcom,usb-snps-hs-7nm-phy";
diff --git a/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts b/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts
index 4ea4fbdb5ce700..7c323214abf1dd 100644
--- a/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts
+++ b/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts
@@ -5,6 +5,7 @@
 
 /dts-v1/;
 
+#include <dt-bindings/leds/common.h>
 #include "msm8917-xiaomi-wingtech.dtsi"
 
 / {
@@ -13,6 +14,10 @@
 
 	qcom,board-id = <0x1000b 1>, <0x1000b 2>;
 
+	chosen {
+		stdout-path = &blsp1_uart2;
+	};
+
 	battery: battery {
 		compatible = "simple-battery";
 		charge-full-design-microamp-hours = <3000000>;
@@ -23,6 +28,36 @@
 		charge-term-current-microamp = <60000>;
 		voltage-min-design-microvolt = <3400000>;
 	};
+
+	vibrator {
+		compatible = "gpio-vibrator";
+		enable-gpios = <&tlmm 97 GPIO_ACTIVE_HIGH>;
+		vcc-supply = <&vph_pwr>;
+	};
+};
+
+&blsp1_i2c2 {
+	status = "okay";
+
+	led-controller@45 {
+		compatible = "awinic,aw2013";
+		reg = <0x45>;
+		vcc-supply = <&pm8937_l10>;
+		vio-supply = <&pm8937_l5>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@0 {
+			reg = <0>;
+			led-max-microamp = <5000>;
+			function = LED_FUNCTION_INDICATOR;
+			color = <LED_COLOR_ID_WHITE>;
+		};
+	};
+};
+
+&blsp1_uart2 {
+	status = "okay";
 };
 
 &blsp2_i2c1 {
diff --git a/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts b/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts
index 90e9662427201e..231a3e9c1929ce 100644
--- a/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts
+++ b/arch/arm64/boot/dts/qcom/msm8939-asus-z00t.dts
@@ -267,6 +267,14 @@
 	extcon = <&usb_id>;
 };
 
+&venus {
+	status = "okay";
+};
+
+&venus_mem {
+	status = "okay";
+};
+
 &wcnss {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts b/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts
index 13422a19c26a15..48514c3df71896 100644
--- a/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts
+++ b/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts
@@ -314,6 +314,14 @@
 	extcon = <&usb_id>;
 };
 
+&venus {
+	status = "okay";
+};
+
+&venus_mem {
+	status = "okay";
+};
+
 &wcnss {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8939.dtsi b/arch/arm64/boot/dts/qcom/msm8939.dtsi
index d4d7b0c9206c3f..013e6a87f44b1d 100644
--- a/arch/arm64/boot/dts/qcom/msm8939.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8939.dtsi
@@ -1627,6 +1627,32 @@
 			};
 		};
 
+		venus: video-codec@1d00000 {
+			compatible = "qcom,msm8939-venus";
+			reg = <0x01d00000 0xff000>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_VENUS0_VCODEC0_CLK>,
+				 <&gcc GCC_VENUS0_AHB_CLK>,
+				 <&gcc GCC_VENUS0_AXI_CLK>,
+				 <&gcc GCC_VENUS0_CORE0_VCODEC0_CLK>,
+				 <&gcc GCC_VENUS0_CORE1_VCODEC0_CLK>;
+			clock-names = "core",
+				      "iface",
+				      "bus",
+				      "vcodec0_core",
+				      "vcodec1_core";
+			power-domains = <&gcc VENUS_GDSC>,
+					<&gcc VENUS_CORE0_GDSC>,
+					<&gcc VENUS_CORE1_GDSC>;
+			power-domain-names = "venus",
+					     "vcodec0",
+					     "vcodec1";
+			iommus = <&apps_iommu 5>;
+			memory-region = <&venus_mem>;
+
+			status = "disabled";
+		};
+
 		apps_iommu: iommu@1ef0000 {
 			compatible = "qcom,msm8916-iommu", "qcom,msm-iommu-v1";
 			reg = <0x01ef0000 0x3000>;
diff --git a/arch/arm64/boot/dts/qcom/msm8976.dtsi b/arch/arm64/boot/dts/qcom/msm8976.dtsi
index 80a0a09e055d79..20a43684cec0ae 100644
--- a/arch/arm64/boot/dts/qcom/msm8976.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8976.dtsi
@@ -818,7 +818,7 @@
 				};
 			};
 
-			wcss_wlan_default: wcss-wlan-default-state  {
+			wcss_wlan_default: wcss-wlan-default-state {
 				wcss-wlan2-pins {
 					pins = "gpio40";
 					function = "wcss_wlan2";
diff --git a/arch/arm64/boot/dts/qcom/msm8996-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/msm8996-oneplus-common.dtsi
index a4dcc88bb01f0e..7a3f8afae22dd8 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-oneplus-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996-oneplus-common.dtsi
@@ -210,12 +210,15 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
 	vddpe-3v3-supply = <&wlan_en>;
 	vdda-supply = <&vreg_l28a_0p925>;
 	status = "okay";
 };
 
+&pcie0_port0 {
+	reset-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+};
+
 &pcie_phy {
 	vdda-phy-supply = <&vreg_l28a_0p925>;
 	vdda-pll-supply = <&vreg_l12a_1p8>;
diff --git a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi
index d55e4075040ffc..20b5eeef46e47f 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi
@@ -191,13 +191,16 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>;
 	vddpe-3v3-supply = <&wlan_en>;
 	vdda-supply = <&pm8994_l28>;
 	status = "okay";
 };
 
+&pcie0_port0 {
+	reset-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 37 GPIO_ACTIVE_LOW>;
+};
+
 &pcie_phy {
 	vdda-phy-supply = <&pm8994_l28>;
 	vdda-pll-supply = <&pm8994_l12>;
diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi
index 77ad613590a3ae..3c70a9be45d2fc 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi
@@ -278,9 +278,11 @@
 	/* Supplied by vdd_3v3, but choose wlan_en to drive enable pin high */
 	vddpe-3v3-supply = <&wlan_en>;
 	vdda-supply = <&vreg_l28a_0p925>;
+};
 
-	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>;
+&pcie0_port0 {
+	reset-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 37 GPIO_ACTIVE_LOW>;
 };
 
 &pcie_phy {
diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi
index 818e5660df91d9..1267c56d586bd8 100644
--- a/arch/arm64/boot/dts/qcom/msm8996.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi
@@ -1924,9 +1924,6 @@
 				      <0x0c100000 0x100000>;
 				reg-names = "parf", "dbi", "elbi","config";
 
-				phys = <&pciephy_0>;
-				phy-names = "pciephy";
-
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges = <0x01000000 0x0 0x00000000 0x0c200000 0x0 0x100000>,
@@ -1975,11 +1972,13 @@
 						"bus_master",
 						"bus_slave";
 
-				pcie@0 {
+				pcie0_port0: pcie@0 {
 					device_type = "pci";
 					reg = <0x0 0x0 0x0 0x0 0x0>;
 					bus-range = <0x01 0xff>;
 
+					phys = <&pciephy_0>;
+
 					#address-cells = <3>;
 					#size-cells = <2>;
 					ranges;
@@ -2001,9 +2000,6 @@
 
 				reg-names = "parf", "dbi", "elbi","config";
 
-				phys = <&pciephy_1>;
-				phy-names = "pciephy";
-
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges = <0x01000000 0x0 0x00000000 0x0d200000 0x0 0x100000>,
@@ -2052,11 +2048,13 @@
 						"bus_master",
 						"bus_slave";
 
-				pcie@0 {
+				pcie1_port0: pcie@0 {
 					device_type = "pci";
 					reg = <0x0 0x0 0x0 0x0 0x0>;
 					bus-range = <0x01 0xff>;
 
+					phys = <&pciephy_1>;
+
 					#address-cells = <3>;
 					#size-cells = <2>;
 					ranges;
@@ -2076,9 +2074,6 @@
 
 				reg-names = "parf", "dbi", "elbi","config";
 
-				phys = <&pciephy_2>;
-				phy-names = "pciephy";
-
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges = <0x01000000 0x0 0x00000000 0x0e200000 0x0 0x100000>,
@@ -2126,11 +2121,13 @@
 						"bus_master",
 						"bus_slave";
 
-				pcie@0 {
+				pcie2_port0: pcie@0 {
 					device_type = "pci";
 					reg = <0x0 0x0 0x0 0x0 0x0>;
 					bus-range = <0x01 0xff>;
 
+					phys = <&pciephy_2>;
+
 					#address-cells = <3>;
 					#size-cells = <2>;
 					ranges;
@@ -2504,18 +2501,18 @@
 
 			#global-interrupts = <1>;
 			interrupts = <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 393 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 394 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 396 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
-		                <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>;
+				     <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 393 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 394 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 396 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&gcc GCC_HLOS1_VOTE_LPASS_ADSP_SMMU_CLK>,
 				 <&gcc GCC_HLOS1_VOTE_LPASS_CORE_SMMU_CLK>;
diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi
index 5038d0009c1a90..bd8b69e323e125 100644
--- a/arch/arm64/boot/dts/qcom/msm8998.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi
@@ -928,8 +928,6 @@
 			#address-cells = <3>;
 			#size-cells = <2>;
 			num-lanes = <1>;
-			phys = <&pcie_phy>;
-			phy-names = "pciephy";
 			status = "disabled";
 
 			ranges = <0x01000000 0x0 0x00000000 0x1b200000 0x0 0x100000>,
@@ -969,13 +967,15 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 			iommu-map = <0x100 &anoc1_smmu 0x1480 1>;
-			perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
 
 			pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie_phy>;
+				reset-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
diff --git a/arch/arm64/boot/dts/qcom/nord-embedded.dtsi b/arch/arm64/boot/dts/qcom/nord-embedded.dtsi
new file mode 100644
index 00000000000000..1ab7d9bc3cfed0
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/nord-embedded.dtsi
@@ -0,0 +1,1928 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/clock/qcom,glymur-videocc.h>
+#include <dt-bindings/clock/qcom,nord-camcc.h>
+#include <dt-bindings/clock/qcom,nord-dispcc.h>
+#include <dt-bindings/clock/qcom,nord-gcc.h>
+#include <dt-bindings/clock/qcom,nord-gpucc.h>
+#include <dt-bindings/clock/qcom,nord-negcc.h>
+#include <dt-bindings/clock/qcom,nord-nwgcc.h>
+#include <dt-bindings/clock/qcom,nord-segcc.h>
+#include <dt-bindings/clock/qcom,nord-tcsrcc.h>
+#include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/interconnect/qcom,icc.h>
+#include <dt-bindings/interconnect/qcom,nord-rpmh.h>
+#include <dt-bindings/power/qcom,rpmhpd.h>
+#include <dt-bindings/soc/qcom,rpmh-rsc.h>
+
+#include "nord.dtsi"
+
+/ {
+	clk_virt: interconnect-clk-virt {
+		compatible = "qcom,nord-clk-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	mc_virt: interconnect-mc-virt {
+		compatible = "qcom,nord-mc-virt";
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+};
+
+&crypto {
+	interconnects = <&aggre1_noc_tile MASTER_CRYPTO_CORE0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "memory";
+
+	status = "okay";
+};
+
+&i2c0 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c0_default>;
+	pinctrl-names = "default";
+};
+
+&i2c1 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S1_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c1_default>;
+	pinctrl-names = "default";
+};
+
+&i2c2 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c2_default>;
+	pinctrl-names = "default";
+};
+
+&i2c3 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c3_default>;
+	pinctrl-names = "default";
+};
+
+&i2c4 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c4_default>;
+	pinctrl-names = "default";
+};
+
+&i2c5 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c5_default>;
+	pinctrl-names = "default";
+};
+
+&i2c7 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c7_default>;
+	pinctrl-names = "default";
+};
+
+&i2c8 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S1_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c8_default>;
+	pinctrl-names = "default";
+};
+
+&i2c9 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c9_default>;
+	pinctrl-names = "default";
+};
+
+&i2c10 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c10_default>;
+	pinctrl-names = "default";
+};
+
+&i2c11 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c11_default>;
+	pinctrl-names = "default";
+};
+
+&i2c12 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c12_default>;
+	pinctrl-names = "default";
+};
+
+&i2c13 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S6_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c13_default>;
+	pinctrl-names = "default";
+};
+
+&i2c14 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c14_default>;
+	pinctrl-names = "default";
+};
+
+&i2c16 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c16_default>;
+	pinctrl-names = "default";
+};
+
+&i2c17 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c17_default>;
+	pinctrl-names = "default";
+};
+
+&i2c18 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c18_default>;
+	pinctrl-names = "default";
+};
+
+&i2c19 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c19_default>;
+	pinctrl-names = "default";
+};
+
+&i2c20 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S6_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_i2c20_default>;
+	pinctrl-names = "default";
+};
+
+&qupv3_0 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_M_AHB_CLK>,
+		 <&segcc SE_GCC_QUPV3_WRAP0_S_AHB_CLK>;
+	clock-names = "m-ahb",
+		      "s-ahb";
+};
+
+&qupv3_1 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_M_AHB_CLK>,
+		 <&segcc SE_GCC_QUPV3_WRAP1_S_AHB_CLK>;
+	clock-names = "m-ahb",
+		      "s-ahb";
+};
+
+&qupv3_2 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_M_AHB_CLK>,
+		 <&negcc NE_GCC_QUPV3_WRAP2_S_AHB_CLK>;
+	clock-names = "m-ahb",
+		      "s-ahb";
+};
+
+&spi0 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi0_default>;
+	pinctrl-names = "default";
+};
+
+&spi1 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S1_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi1_default>;
+	pinctrl-names = "default";
+};
+
+&spi2 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi2_default>;
+	pinctrl-names = "default";
+};
+
+&spi3 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi3_default>;
+	pinctrl-names = "default";
+};
+
+&spi4 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi4_default>;
+	pinctrl-names = "default";
+};
+
+&spi5 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_0 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi5_default>;
+	pinctrl-names = "default";
+};
+
+&spi7 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi7_default>;
+	pinctrl-names = "default";
+};
+
+&spi8 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S1_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi8_default>;
+	pinctrl-names = "default";
+};
+
+&spi11 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi11_default>;
+	pinctrl-names = "default";
+};
+
+&spi12 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi12_default>;
+	pinctrl-names = "default";
+};
+
+&spi13 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S6_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre2_noc_tile MASTER_QUP_1 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi13_default>;
+	pinctrl-names = "default";
+};
+
+&spi14 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi14_default>;
+	pinctrl-names = "default";
+};
+
+&spi16 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi16_default>;
+	pinctrl-names = "default";
+};
+
+&spi17 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi17_default>;
+	pinctrl-names = "default";
+};
+
+&spi18 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi18_default>;
+	pinctrl-names = "default";
+};
+
+&spi19 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi19_default>;
+	pinctrl-names = "default";
+};
+
+&spi20 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S6_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>,
+			<&aggre1_noc_tile MASTER_QUP_2 QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config",
+			     "qup-memory";
+	pinctrl-0 = <&qup_spi20_default>;
+	pinctrl-names = "default";
+};
+
+&soc {
+	gcc: clock-controller@100000 {
+		compatible = "qcom,nord-gcc";
+		reg = <0x0 0x00100000 0x0 0x1f4200>;
+		clocks = <&bi_tcxo_div2>,
+			 <&sleep_clk>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	cnoc_main: interconnect@1500000 {
+		compatible = "qcom,nord-cnoc-main";
+		reg = <0x0 0x01500000 0x0 0x1d200>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	config_noc: interconnect@1600000 {
+		compatible = "qcom,nord-cnoc-cfg";
+		reg = <0x0 0x01600000 0x0 0xd200>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	system_noc: interconnect@1680000 {
+		compatible = "qcom,nord-system-noc";
+		reg = <0x0 0x01680000 0x0 0x1c080>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	aggre2_noc_tile: interconnect@16c0000 {
+		compatible = "qcom,nord-aggre2-noc-tile";
+		reg = <0x0 0x016c0000 0x0 0x1b400>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	aggre1_noc: interconnect@16e0000 {
+		compatible = "qcom,nord-aggre1-noc";
+		reg = <0x0 0x016e0000 0x0 0x1c400>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	aggre2_noc: interconnect@1700000 {
+		compatible = "qcom,nord-aggre2-noc";
+		reg = <0x0 0x01700000 0x0 0x1b400>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+		clocks = <&rpmhcc RPMH_IPA_CLK>;
+	};
+
+	aggre1_noc_tile: interconnect@1720000 {
+		compatible = "qcom,nord-aggre1-noc-tile";
+		reg = <0x0 0x01720000 0x0 0x23400>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+		clocks = <&negcc NE_GCC_AGGRE_NOC_USB2_AXI_CLK>,
+			 <&negcc NE_GCC_AGGRE_NOC_USB3_PRIM_AXI_CLK>,
+			 <&negcc NE_GCC_AGGRE_NOC_USB3_SEC_AXI_CLK>,
+			 <&negcc NE_GCC_AGGRE_NOC_UFS_PHY_AXI_CLK>;
+	};
+
+	mmss_noc: interconnect@1780000 {
+		compatible = "qcom,nord-mmss-noc";
+		reg = <0x0 0x01780000 0x0 0x72800>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	pcie_cfg: interconnect@1ba0000 {
+		compatible = "qcom,nord-pcie-cfg";
+		reg = <0x0 0x01ba0000 0x0 0x7200>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	pcie_data_outbound: interconnect@1bc0000 {
+		compatible = "qcom,nord-pcie-data-outbound";
+		reg = <0x0 0x01bc0000 0x0 0x17000>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	pcie_data_inbound: interconnect@1c00000 {
+		compatible = "qcom,nord-pcie-data-inbound";
+		reg = <0x0 0x01c00000 0x0 0x4b080>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	ufs_mem_phy: phy@1d40000 {
+		compatible = "qcom,nord-qmp-ufs-phy",
+			     "qcom,sm8650-qmp-ufs-phy";
+		reg = <0x0 0x01d40000 0x0 0x2000>;
+
+		clocks = <&rpmhcc RPMH_CXO_CLK>,
+			 <&negcc NE_GCC_UFS_PHY_PHY_AUX_CLK>,
+			 <&tcsrcc TCSR_UFS_CLKREF_EN>;
+		clock-names = "ref",
+			      "ref_aux",
+			      "qref";
+
+		resets = <&ufs_mem_hc 0>;
+		reset-names = "ufsphy";
+
+		power-domains = <&negcc NE_GCC_UFS_MEM_PHY_GDSC>;
+		#clock-cells = <1>;
+		#phy-cells = <0>;
+
+		status = "disabled";
+	};
+
+	ice: crypto@1d48000 {
+		compatible = "qcom,nord-inline-crypto-engine",
+			     "qcom,inline-crypto-engine";
+		reg = <0x0 0x01d48000 0x0 0x18000>;
+		clocks = <&negcc NE_GCC_UFS_PHY_ICE_CORE_CLK>,
+			 <&negcc NE_GCC_UFS_PHY_AHB_CLK>;
+		clock-names = "core",
+			      "iface";
+		power-domains = <&negcc NE_GCC_UFS_PHY_GDSC>;
+	};
+
+	hscnoc: interconnect@2000000 {
+		compatible = "qcom,nord-hscnoc";
+		reg = <0x0 0x02000000 0x0 0xb22000>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	gpucc: clock-controller@3d90000 {
+		compatible = "qcom,nord-gpucc";
+		reg = <0x0 0x03d90000 0x0 0x9800>;
+		clocks = <&rpmhcc RPMH_CXO_CLK>,
+			 <&nwgcc NW_GCC_GPU_GPLL0_CLK_SRC>,
+			 <&nwgcc NW_GCC_GPU_GPLL0_DIV_CLK_SRC>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	hpass_ag_noc: interconnect@5fc0000 {
+		compatible = "qcom,nord-hpass-ag-noc";
+		reg = <0x0  0x05fc0000 0x0 0x37080>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	negcc: clock-controller@8900000 {
+		compatible = "qcom,nord-negcc";
+		reg = <0x0 0x08900000 0x0 0xf4200>;
+		clocks = <&bi_tcxo_div2>,
+			 <&sleep_clk>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	segcc: clock-controller@8a00000 {
+		compatible = "qcom,nord-segcc";
+		reg = <0x0 0x08a00000 0x0 0xf4200>;
+		clocks = <&bi_tcxo_div2>,
+			 <&sleep_clk>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	nwgcc: clock-controller@8b00000 {
+		compatible = "qcom,nord-nwgcc";
+		reg = <0x0 0x08b00000 0x0 0xf4200>;
+		clocks = <&bi_tcxo_div2>,
+			 <&sleep_clk>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	dispcc1: clock-controller@8d00000 {
+		compatible = "qcom,nord-dispcc1";
+		reg = <0x0 0x08d00000 0x0 0x20000>;
+		clocks = <&bi_tcxo_div2>,
+			 <&bi_tcxo_ao_div2>,
+			 <&nwgcc NW_GCC_DISP_1_AHB_CLK>,
+			 <&sleep_clk>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>;
+		power-domains = <&rpmhpd RPMHPD_MMCX>;
+		required-opps = <&rpmhpd_opp_min_svs>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	gpu2cc: clock-controller@9890000 {
+		compatible = "qcom,nord-gpu2cc";
+		reg = <0x0 0x09890000 0x0 0xa000>;
+		clocks = <&bi_tcxo_div2>,
+			 <&negcc NE_GCC_GPU_2_GPLL0_CLK_SRC>,
+			 <&negcc NE_GCC_GPU_2_GPLL0_DIV_CLK_SRC>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	camcc: clock-controller@9ce0000 {
+		compatible = "qcom,nord-camcc";
+		reg = <0x0 0x09ce0000 0x0 0x20000>;
+		power-domains = <&rpmhpd RPMHPD_MMCX>,
+				<&rpmhpd RPMHPD_MXC>;
+		required-opps = <&rpmhpd_opp_low_svs>,
+				<&rpmhpd_opp_low_svs>;
+		clocks = <&nwgcc NW_GCC_CAMERA_AHB_CLK>,
+			 <&bi_tcxo_div2>,
+			 <&bi_tcxo_ao_div2>,
+			 <&sleep_clk>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	videocc: clock-controller@aaf0000 {
+		compatible = "qcom,nord-videocc";
+		reg = <0x0 0x0aaf0000 0x0 0x10000>;
+		clocks = <&bi_tcxo_div2>,
+			 <&nwgcc NW_GCC_VIDEO_AHB_CLK>;
+		power-domains = <&rpmhpd RPMHPD_MMCX>,
+				<&rpmhpd RPMHPD_MXC>;
+		required-opps = <&rpmhpd_opp_low_svs>,
+				<&rpmhpd_opp_low_svs>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	dispcc0: clock-controller@af00000 {
+		compatible = "qcom,nord-dispcc0";
+		reg = <0x0 0x0af00000 0x0 0x20000>;
+		clocks = <&bi_tcxo_div2>,
+			 <&bi_tcxo_ao_div2>,
+			 <&nwgcc NW_GCC_DISP_0_AHB_CLK>,
+			 <&sleep_clk>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>,
+			 <0>;
+		power-domains = <&rpmhpd RPMHPD_MMCX>;
+		required-opps = <&rpmhpd_opp_min_svs>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+		#power-domain-cells = <1>;
+	};
+
+	tcsrcc: clock-controller@f1d9000 {
+		compatible = "qcom,nord-tcsrcc",
+			     "syscon";
+		reg = <0x0 0x0f1d9000 0x0 0xf00c>;
+		clocks = <&rpmhcc RPMH_CXO_CLK>;
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+	};
+
+	apps_rsc: rsc@18900000 {
+		compatible = "qcom,rpmh-rsc";
+		reg = <0x0 0x18900000 0x0 0x10000>,
+		      <0x0 0x18910000 0x0 0x10000>,
+		      <0x0 0x18920000 0x0 0x10000>;
+		reg-names = "drv-0",
+			    "drv-1",
+			    "drv-2";
+		interrupts = <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+		qcom,tcs-offset = <0xd00>;
+		qcom,drv-id = <2>;
+		qcom,tcs-config = <ACTIVE_TCS 3>,
+				  <SLEEP_TCS 2>,
+				  <WAKE_TCS 2>,
+				  <CONTROL_TCS 0>;
+		label = "apps_rsc";
+		power-domains = <&system_pd>;
+
+		apps_bcm_voter: bcm-voter {
+			compatible = "qcom,bcm-voter";
+		};
+
+		rpmhcc: clock-controller {
+			compatible = "qcom,nord-rpmh-clk";
+			clocks = <&xo_board>;
+			clock-names = "xo";
+			#clock-cells = <1>;
+		};
+
+		rpmhpd: power-controller {
+			compatible = "qcom,nord-rpmhpd";
+			#power-domain-cells = <1>;
+			operating-points-v2 = <&rpmhpd_opp_table>;
+
+			rpmhpd_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				rpmhpd_opp_ret: opp-0 {
+					opp-level = <RPMH_REGULATOR_LEVEL_RETENTION>;
+				};
+
+				rpmhpd_opp_min_svs: opp-1 {
+					opp-level = <RPMH_REGULATOR_LEVEL_MIN_SVS>;
+				};
+
+				rpmhpd_opp_low_svs: opp-2 {
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+				};
+
+				rpmhpd_opp_svs: opp-3 {
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+				};
+
+				rpmhpd_opp_svs_l1: opp-4 {
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+				};
+
+				rpmhpd_opp_nom: opp-5 {
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+				};
+
+				rpmhpd_opp_nom_l1: opp-6 {
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+				};
+
+				rpmhpd_opp_nom_l2: opp-7 {
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM_L2>;
+				};
+
+				rpmhpd_opp_turbo: opp-8 {
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO>;
+				};
+
+				rpmhpd_opp_turbo_l1: opp-9 {
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+				};
+			};
+		};
+	};
+
+	nsp_data_noc_0: interconnect@1f200000 {
+		compatible = "qcom,nord-nsp-data-noc-0";
+		reg = <0x0 0x1f200000 0x0 0x2a200>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	nsp_data_noc_1: interconnect@1f600000 {
+		compatible = "qcom,nord-nsp-data-noc-1";
+		reg = <0x0 0x1f600000 0x0 0x2a200>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	nsp_data_noc_2: interconnect@1fa00000 {
+		compatible = "qcom,nord-nsp-data-noc-2";
+		reg = <0x0 0x1fa00000 0x0 0x2a200>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+
+	nsp_data_noc_3: interconnect@1fe00000 {
+		compatible = "qcom,nord-nsp-data-noc-3";
+		reg = <0x0 0x1fe00000 0x0 0x2a200>;
+		#interconnect-cells = <2>;
+		qcom,bcm-voters = <&apps_bcm_voter>;
+	};
+};
+
+&tlmm {
+	qup_i2c0_default: qup-i2c0-default-state {
+		pins = "gpio111", "gpio112";
+		function = "qup0_se0";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c1_default: qup-i2c1-default-state {
+		pins = "gpio111", "gpio112";
+		function = "qup0_se1";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c2_default: qup-i2c2-default-state {
+		pins = "gpio113", "gpio114";
+		function = "qup0_se2";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c3_default: qup-i2c3-default-state {
+		pins = "gpio115", "gpio116";
+		function = "qup0_se3";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c4_default: qup-i2c4-default-state {
+		pins = "gpio117", "gpio118";
+		function = "qup0_se4";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c5_default: qup-i2c5-default-state {
+		pins = "gpio121", "gpio122";
+		function = "qup0_se5";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c7_default: qup-i2c7-default-state {
+		pins = "gpio123", "gpio124";
+		function = "qup1_se0";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c8_default: qup-i2c8-default-state {
+		pins = "gpio125", "gpio126";
+		function = "qup1_se1";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c9_default: qup-i2c9-default-state {
+		pins = "gpio127", "gpio128";
+		function = "qup1_se2_01";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c10_default: qup-i2c10-default-state {
+		pins = "gpio129", "gpio130";
+		function = "qup1_se3_01";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c11_default: qup-i2c11-default-state {
+		pins = "gpio131", "gpio132";
+		function = "qup1_se4";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c12_default: qup-i2c12-default-state {
+		pins = "gpio133", "gpio134";
+		function = "qup1_se5";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c13_default: qup-i2c13-default-state {
+		pins = "gpio137", "gpio138";
+		function = "qup1_se6";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c14_default: qup-i2c14-default-state {
+		pins = "gpio139", "gpio140";
+		function = "qup2_se0";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c16_default: qup-i2c16-default-state {
+		pins = "gpio145", "gpio146";
+		function = "qup2_se2";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c17_default: qup-i2c17-default-state {
+		pins = "gpio150", "gpio151";
+		function = "qup2_se3";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c18_default: qup-i2c18-default-state {
+		pins = "gpio154", "gpio155";
+		function = "qup2_se4";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c19_default: qup-i2c19-default-state {
+		pins = "gpio156", "gpio157";
+		function = "qup2_se5";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_i2c20_default: qup-i2c20-default-state {
+		pins = "gpio158", "gpio159";
+		function = "qup2_se6";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	qup_spi0_default: qup-spi0-default-state {
+		data-pins {
+			pins = "gpio109", "gpio111", "gpio112";
+			function = "qup0_se0";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio110";
+			function = "qup0_se0";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi1_default: qup-spi1-default-state {
+		data-pins {
+			pins = "gpio109", "gpio111", "gpio112";
+			function = "qup0_se1";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio110";
+			function = "qup0_se1";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi2_default: qup-spi2-default-state {
+		data-pins {
+			pins = "gpio113", "gpio114", "gpio115";
+			function = "qup0_se2";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio116";
+			function = "qup0_se2";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi3_default: qup-spi3-default-state {
+		data-pins {
+			pins = "gpio113", "gpio115", "gpio116";
+			function = "qup0_se3";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio114";
+			function = "qup0_se3";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi4_default: qup-spi4-default-state {
+		data-pins {
+			pins = "gpio117", "gpio118", "gpio119";
+			function = "qup0_se4";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio120";
+			function = "qup0_se4";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi5_default: qup-spi5-default-state {
+		data-pins {
+			pins = "gpio109", "gpio121", "gpio122";
+			function = "qup0_se5";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio110";
+			function = "qup0_se5";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi7_default: qup-spi7-default-state {
+		data-pins {
+			pins = "gpio123", "gpio124", "gpio125";
+			function = "qup1_se0";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio126";
+			function = "qup1_se0";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi8_default: qup-spi8-default-state {
+		data-pins {
+			pins = "gpio123", "gpio125", "gpio126";
+			function = "qup1_se1";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio124";
+			function = "qup1_se1";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi11_default: qup-spi11-default-state {
+		data-pins {
+			pins = "gpio131", "gpio132", "gpio137";
+			function = "qup1_se4";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio138";
+			function = "qup1_se4";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi12_default: qup-spi12-default-state {
+		data-pins {
+			pins = "gpio133", "gpio134", "gpio135";
+			function = "qup1_se5";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio136";
+			function = "qup1_se5";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi13_default: qup-spi13-default-state {
+		data-pins {
+			pins = "gpio131", "gpio137", "gpio138";
+			function = "qup1_se6";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio132";
+			function = "qup1_se6";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi14_default: qup-spi14-default-state {
+		data-pins {
+			pins = "gpio139", "gpio140", "gpio141";
+			function = "qup2_se0";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio142";
+			function = "qup2_se0";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi16_default: qup-spi16-default-state {
+		data-pins {
+			pins = "gpio145", "gpio146", "gpio147";
+			function = "qup2_se2";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio148";
+			function = "qup2_se2";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi17_default: qup-spi17-default-state {
+		data-pins {
+			pins = "gpio150", "gpio151", "gpio152";
+			function = "qup2_se3";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio153";
+			function = "qup2_se3";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi18_default: qup-spi18-default-state {
+		data-pins {
+			pins = "gpio143", "gpio154", "gpio155";
+			function = "qup2_se4";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio144";
+			function = "qup2_se4";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi19_default: qup-spi19-default-state {
+		data-pins {
+			pins = "gpio156", "gpio157", "gpio158";
+			function = "qup2_se5";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio159";
+			function = "qup2_se5";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_spi20_default: qup-spi20-default-state {
+		data-pins {
+			pins = "gpio156", "gpio158", "gpio159";
+			function = "qup2_se6";
+			drive-strength = <6>;
+			bias-disable;
+		};
+
+		cs-pins {
+			pins = "gpio157";
+			function = "qup2_se6";
+			drive-strength = <2>;
+			bias-disable;
+		};
+	};
+
+	qup_uart0_default: qup-uart0-default-state {
+		pins = "gpio109", "gpio110";
+		function = "qup0_se0";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart1_default: qup-uart1-default-state {
+		pins = "gpio109", "gpio110";
+		function = "qup0_se1";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart2_default: qup-uart2-default-state {
+		pins = "gpio115", "gpio116";
+		function = "qup0_se2";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart3_default: qup-uart3-default-state {
+		pins = "gpio113", "gpio114";
+		function = "qup0_se3";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart4_default: qup-uart4-default-state {
+		pins = "gpio119", "gpio120";
+		function = "qup0_se4";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart5_default: qup-uart5-default-state {
+		pins = "gpio109", "gpio110";
+		function = "qup0_se5";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart7_default: qup-uart7-default-state {
+		pins = "gpio125", "gpio126";
+		function = "qup1_se0";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart8_default: qup-uart8-default-state {
+		pins = "gpio123", "gpio124";
+		function = "qup1_se1";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart9_default: qup-uart9-default-state {
+		pins = "gpio127", "gpio128";
+		function = "qup1_se2_23";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart10_default: qup-uart10-default-state {
+		pins = "gpio129", "gpio130";
+		function = "qup1_se3_23";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart11_default: qup-uart11-default-state {
+		pins = "gpio137", "gpio138";
+		function = "qup1_se4";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart12_default: qup-uart12-default-state {
+		pins = "gpio135", "gpio136";
+		function = "qup1_se5";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart13_default: qup-uart13-default-state {
+		pins = "gpio131", "gpio132";
+		function = "qup1_se6";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart14_default: qup-uart14-default-state {
+		pins = "gpio141", "gpio142";
+		function = "qup2_se0";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart15_default: qup-uart15-default-state {
+		pins = "gpio143", "gpio144";
+		function = "qup2_se1";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart16_default: qup-uart16-default-state {
+		pins = "gpio147", "gpio148";
+		function = "qup2_se2";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart17_default: qup-uart17-default-state {
+		pins = "gpio150", "gpio151", "gpio152", "gpio153";
+		function = "qup2_se3";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart18_default: qup-uart18-default-state {
+		pins = "gpio143", "gpio144";
+		function = "qup2_se4";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart19_default: qup-uart19-default-state {
+		pins = "gpio158", "gpio159";
+		function = "qup2_se5";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	qup_uart20_default: qup-uart20-default-state {
+		pins = "gpio156", "gpio157";
+		function = "qup2_se6";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&uart0 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart0_default>;
+	pinctrl-names = "default";
+};
+
+&uart1 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S1_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart1_default>;
+	pinctrl-names = "default";
+};
+
+&uart2 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart2_default>;
+	pinctrl-names = "default";
+};
+
+&uart3 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart3_default>;
+	pinctrl-names = "default";
+};
+
+&uart4 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart4_default>;
+	pinctrl-names = "default";
+};
+
+&uart5 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP0_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_0 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_0 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart5_default>;
+	pinctrl-names = "default";
+};
+
+&uart7 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart7_default>;
+	pinctrl-names = "default";
+};
+
+&uart8 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S1_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart8_default>;
+	pinctrl-names = "default";
+};
+
+&uart9 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart9_default>;
+	pinctrl-names = "default";
+};
+
+&uart10 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart10_default>;
+	pinctrl-names = "default";
+};
+
+&uart11 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart11_default>;
+	pinctrl-names = "default";
+};
+
+&uart12 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart12_default>;
+	pinctrl-names = "default";
+};
+
+&uart13 {
+	clocks = <&segcc SE_GCC_QUPV3_WRAP1_S6_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_1 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart13_default>;
+	pinctrl-names = "default";
+};
+
+&uart14 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S0_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart14_default>;
+	pinctrl-names = "default";
+};
+
+&uart15 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S1_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart15_default>;
+	pinctrl-names = "default";
+};
+
+&uart16 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S2_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart16_default>;
+	pinctrl-names = "default";
+};
+
+&uart17 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S3_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart17_default>;
+	pinctrl-names = "default";
+};
+
+&uart18 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S4_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart18_default>;
+	pinctrl-names = "default";
+};
+
+&uart19 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S5_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart19_default>;
+	pinctrl-names = "default";
+};
+
+&uart20 {
+	clocks = <&negcc NE_GCC_QUPV3_WRAP2_S6_CLK>;
+	clock-names = "se";
+	interconnects = <&clk_virt MASTER_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS
+			 &clk_virt SLAVE_QUP_CORE_2 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ALWAYS
+			 &config_noc SLAVE_QUP_2 QCOM_ICC_TAG_ALWAYS>;
+	interconnect-names = "qup-core",
+			     "qup-config";
+	pinctrl-0 = <&qup_uart20_default>;
+	pinctrl-names = "default";
+};
+
+&ufs_mem_hc {
+	clocks = <&negcc NE_GCC_UFS_PHY_AXI_CLK>,
+		 <&negcc NE_GCC_AGGRE_NOC_UFS_PHY_AXI_CLK>,
+		 <&negcc NE_GCC_UFS_PHY_AHB_CLK>,
+		 <&negcc NE_GCC_UFS_PHY_UNIPRO_CORE_CLK>,
+		 <&tcsrcc TCSR_UFS_CLKREF_EN>,
+		 <&negcc NE_GCC_UFS_PHY_TX_SYMBOL_0_CLK>,
+		 <&negcc NE_GCC_UFS_PHY_RX_SYMBOL_0_CLK>,
+		 <&negcc NE_GCC_UFS_PHY_RX_SYMBOL_1_CLK>;
+	clock-names = "core_clk",
+		      "bus_aggr_clk",
+		      "iface_clk",
+		      "core_clk_unipro",
+		      "ref_clk",
+		      "tx_lane0_sync_clk",
+		      "rx_lane0_sync_clk",
+		      "rx_lane1_sync_clk";
+
+	resets = <&negcc NE_GCC_UFS_PHY_BCR>;
+	reset-names = "rst";
+
+	interconnects = <&aggre1_noc_tile MASTER_UFS_MEM QCOM_ICC_TAG_ALWAYS
+			 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+			<&hscnoc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+			 &config_noc SLAVE_UFS_MEM_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+	interconnect-names = "ufs-ddr",
+			     "cpu-ufs";
+
+	phys = <&ufs_mem_phy>;
+	phy-names = "ufsphy";
+
+	power-domains = <&negcc NE_GCC_UFS_PHY_GDSC>;
+	operating-points-v2 = <&ufs_opp_table>;
+	required-opps = <&rpmhpd_opp_nom>;
+	qcom,ice = <&ice>;
+	#reset-cells = <1>;
+
+	ufs_opp_table: opp-table {
+		compatible = "operating-points-v2";
+
+		opp-100000000 {
+			opp-hz = /bits/ 64 <100000000>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <100000000>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>;
+			required-opps = <&rpmhpd_opp_low_svs>;
+		};
+
+		opp-201500000 {
+			opp-hz = /bits/ 64 <201500000>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <201500000>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>;
+			required-opps = <&rpmhpd_opp_svs>;
+		};
+
+		opp-403000000 {
+			opp-hz = /bits/ 64 <403000000>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <403000000>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>,
+				 /bits/ 64 <0>;
+			required-opps = <&rpmhpd_opp_nom>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/nord-gearvm.dtsi b/arch/arm64/boot/dts/qcom/nord-gearvm.dtsi
new file mode 100644
index 00000000000000..5514f9a6f49f54
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/nord-gearvm.dtsi
@@ -0,0 +1,2848 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "nord.dtsi"
+
+&firmware {
+	scmi0: scmi-0 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem0>;
+		/* We expect the bootloader to update the IRQ number */
+		interrupts = <GIC_SPI 963 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi0_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi0_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi0_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi1: scmi-1 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem1>;
+		interrupts = <GIC_SPI 964 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi1_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi1_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi1_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi2: scmi-2 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem2>;
+		interrupts = <GIC_SPI 965 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi2_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi2_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi2_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi3: scmi-3 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem3>;
+		interrupts = <GIC_SPI 966 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi3_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi3_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi3_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi4: scmi-4 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem4>;
+		interrupts = <GIC_SPI 967 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi4_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi4_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi4_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi5: scmi-5 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem5>;
+		interrupts = <GIC_SPI 968 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi5_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi5_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi5_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi6: scmi-6 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem6>;
+		interrupts = <GIC_SPI 969 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi6_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi6_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi6_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi7: scmi-7 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem7>;
+		interrupts = <GIC_SPI 970 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi7_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi7_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi7_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi8: scmi-8 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem8>;
+		interrupts = <GIC_SPI 971 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi8_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi8_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi8_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi9: scmi-9 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem9>;
+		interrupts = <GIC_SPI 972 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi9_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi9_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi9_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi10: scmi-10 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem10>;
+		interrupts = <GIC_SPI 973 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi10_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi10_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi10_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi11: scmi-11 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem11>;
+		interrupts = <GIC_SPI 974 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi11_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi11_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi11_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi12: scmi-12 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem12>;
+		interrupts = <GIC_SPI 975 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi12_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi12_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi12_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi13: scmi-13 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem13>;
+		interrupts = <GIC_SPI 976 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi13_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi13_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi13_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi14: scmi-14 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem14>;
+		interrupts = <GIC_SPI 977 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi14_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi14_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi14_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi15: scmi-15 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem15>;
+		interrupts = <GIC_SPI 978 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi15_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi15_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi15_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi16: scmi-16 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem16>;
+		interrupts = <GIC_SPI 979 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi16_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi16_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi16_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi17: scmi-17 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem17>;
+		interrupts = <GIC_SPI 980 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi17_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi17_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi17_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi18: scmi-18 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem18>;
+		interrupts = <GIC_SPI 981 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi18_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi18_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi18_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi19: scmi-19 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem19>;
+		interrupts = <GIC_SPI 982 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi19_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi19_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi19_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi20: scmi-20 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem20>;
+		interrupts = <GIC_SPI 983 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi20_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi20_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi20_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi21: scmi-21 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem21>;
+		interrupts = <GIC_SPI 984 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi21_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi21_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi21_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi22: scmi-22 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem22>;
+		interrupts = <GIC_SPI 985 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi22_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi22_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi22_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi23: scmi-23 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem23>;
+		interrupts = <GIC_SPI 986 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi23_sensor: protocol@15 {
+			reg = <0x15>;
+			#thermal-sensor-cells = <1>;
+		};
+	};
+
+	scmi24: scmi-24 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem24>;
+		interrupts = <GIC_SPI 987 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi24_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi24_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi24_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi25: scmi-25 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem25>;
+		interrupts = <GIC_ESPI 0 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi25_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi25_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi25_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi26: scmi-26 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem26>;
+		interrupts = <GIC_ESPI 1 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi26_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi26_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi26_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi27: scmi-27 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem27>;
+		interrupts = <GIC_ESPI 2 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi27_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi27_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi27_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi28: scmi-28 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem28>;
+		interrupts = <GIC_ESPI 3 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi28_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi28_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi28_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi29: scmi-29 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem29>;
+		interrupts = <GIC_ESPI 4 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi29_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi29_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi29_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi30: scmi-30 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem30>;
+		interrupts = <GIC_ESPI 5 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi30_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi30_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi30_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi31: scmi-31 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem31>;
+		interrupts = <GIC_ESPI 6 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi31_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi31_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi31_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi32: scmi-32 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem32>;
+		interrupts = <GIC_ESPI 7 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi32_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi32_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi32_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi33: scmi-33 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem33>;
+		interrupts = <GIC_ESPI 8 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi33_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi33_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi33_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi34: scmi-34 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem34>;
+		interrupts = <GIC_ESPI 9 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi34_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi34_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi34_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi35: scmi-35 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem35>;
+		interrupts = <GIC_ESPI 10 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi35_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi35_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi35_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi36: scmi-36 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem36>;
+		interrupts = <GIC_ESPI 11 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi36_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi36_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi36_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi37: scmi-37 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem37>;
+		interrupts = <GIC_ESPI 12 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi37_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi37_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi37_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi38: scmi-38 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem38>;
+		interrupts = <GIC_ESPI 13 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi38_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi38_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi38_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi39: scmi-39 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem39>;
+		interrupts = <GIC_ESPI 14 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi39_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi39_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi39_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi40: scmi-40 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem40>;
+		interrupts = <GIC_ESPI 15 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi40_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi40_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi40_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi41: scmi-41 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem41>;
+		interrupts = <GIC_ESPI 16 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi41_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi41_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi41_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi42: scmi-42 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem42>;
+		interrupts = <GIC_ESPI 17 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi42_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi42_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi42_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi43: scmi-43 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem43>;
+		interrupts = <GIC_ESPI 18 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi43_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi43_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi43_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi44: scmi-44 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem44>;
+		interrupts = <GIC_ESPI 19 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi44_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi44_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi44_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi45: scmi-45 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem45>;
+		interrupts = <GIC_ESPI 20 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi45_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi45_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi45_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi46: scmi-46 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem46>;
+		interrupts = <GIC_ESPI 21 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi46_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi46_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi46_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi47: scmi-47 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem47>;
+		interrupts = <GIC_ESPI 22 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi47_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi47_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi47_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi48: scmi-48 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem48>;
+		interrupts = <GIC_ESPI 23 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi48_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi48_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi48_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi49: scmi-49 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem49>;
+		interrupts = <GIC_ESPI 24 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi49_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi49_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi49_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi50: scmi-50 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem50>;
+		interrupts = <GIC_ESPI 25 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi50_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi50_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi50_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi51: scmi-51 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem51>;
+		interrupts = <GIC_ESPI 26 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi51_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi51_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi51_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi52: scmi-52 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem52>;
+		interrupts = <GIC_ESPI 27 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi52_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi52_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi52_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi53: scmi-53 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem53>;
+		interrupts = <GIC_ESPI 28 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi53_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi53_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi53_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi54: scmi-54 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem54>;
+		interrupts = <GIC_ESPI 29 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi54_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi54_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi54_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi55: scmi-55 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem55>;
+		interrupts = <GIC_ESPI 30 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi55_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi55_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi55_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi56: scmi-56 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem56>;
+		interrupts = <GIC_ESPI 31 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi56_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi56_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi56_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi57: scmi-57 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem57>;
+		interrupts = <GIC_ESPI 32 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi57_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi57_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi57_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi58: scmi-58 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem58>;
+		interrupts = <GIC_ESPI 33 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi58_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi58_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi58_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi59: scmi-59 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem59>;
+		interrupts = <GIC_ESPI 34 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi59_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi59_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi59_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi60: scmi-60 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem60>;
+		interrupts = <GIC_ESPI 35 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi60_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi60_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi60_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi61: scmi-61 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem61>;
+		interrupts = <GIC_ESPI 36 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi61_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi61_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi61_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi62: scmi-62 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem62>;
+		interrupts = <GIC_ESPI 37 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi62_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi62_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi62_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+
+	scmi63: scmi-63 {
+		compatible = "qcom,scmi-smc";
+		arm,smc-id = <0xc6008012>;
+		shmem = <&shmem63>;
+		interrupts = <GIC_ESPI 38 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "a2p";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		arm,max-msg = <10>;
+		arm,max-msg-size = <256>;
+		arm,max-rx-timeout-ms = <3000>;
+
+		status = "disabled";
+
+		scmi63_pd: protocol@11 {
+			reg = <0x11>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi63_dvfs: protocol@13 {
+			reg = <0x13>;
+			#power-domain-cells = <1>;
+		};
+
+		scmi63_rst: protocol@16 {
+			reg = <0x16>;
+			#reset-cells = <1>;
+		};
+	};
+};
+
+&i2c0 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 0>,
+			<&scmi9_dvfs 0>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c1 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 1>,
+			<&scmi9_dvfs 1>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c2 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 2>,
+			<&scmi9_dvfs 2>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c3 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 3>,
+			<&scmi9_dvfs 3>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c4 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 4>,
+			<&scmi9_dvfs 4>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c5 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 5>,
+			<&scmi9_dvfs 5>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c7 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 7>,
+			<&scmi9_dvfs 7>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c8 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 8>,
+			<&scmi9_dvfs 8>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c9 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 9>,
+			<&scmi9_dvfs 9>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c10 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 10>,
+			<&scmi9_dvfs 10>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c11 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 11>,
+			<&scmi9_dvfs 11>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c12 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 12>,
+			<&scmi9_dvfs 12>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c13 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 13>,
+			<&scmi9_dvfs 13>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c14 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 14>,
+			<&scmi9_dvfs 14>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c15 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 15>,
+			<&scmi9_dvfs 15>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c16 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 16>,
+			<&scmi9_dvfs 16>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c17 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 17>,
+			<&scmi9_dvfs 17>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c18 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 18>,
+			<&scmi9_dvfs 18>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c19 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 19>,
+			<&scmi9_dvfs 19>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&i2c20 {
+	compatible = "qcom,sa8797p-geni-i2c",
+		     "qcom,sa8255p-geni-i2c";
+	power-domains = <&scmi9_pd 20>,
+			<&scmi9_dvfs 20>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&qupv3_0 {
+	compatible = "qcom,sa8797p-geni-se-qup",
+		     "qcom,sa8255p-geni-se-qup";
+};
+
+&qupv3_1 {
+	compatible = "qcom,sa8797p-geni-se-qup",
+		     "qcom,sa8255p-geni-se-qup";
+};
+
+&qupv3_2 {
+	compatible = "qcom,sa8797p-geni-se-qup",
+		     "qcom,sa8255p-geni-se-qup";
+};
+
+&reserved_memory {
+	shmem0: scmi-shmem@d7600000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7600000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem1: scmi-shmem@d7601000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7601000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem2: scmi-shmem@d7602000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7602000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem3: scmi-shmem@d7603000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7603000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem4: scmi-shmem@d7604000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7604000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem5: scmi-shmem@d7605000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7605000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem6: scmi-shmem@d7606000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7606000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem7: scmi-shmem@d7607000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7607000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem8: scmi-shmem@d7608000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7608000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem9: scmi-shmem@d7609000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7609000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem10: scmi-shmem@d760a000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd760a000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem11: scmi-shmem@d760b000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd760b000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem12: scmi-shmem@d760c000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd760c000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem13: scmi-shmem@d760d000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd760d000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem14: scmi-shmem@d760e000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd760e000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem15: scmi-shmem@d760f000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd760f000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem16: scmi-shmem@d7610000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7610000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem17: scmi-shmem@d7611000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7611000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem18: scmi-shmem@d7612000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7612000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem19: scmi-shmem@d7613000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7613000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem20: scmi-shmem@d7614000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7614000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem21: scmi-shmem@d7615000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7615000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem22: scmi-shmem@d7616000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7616000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem23: scmi-shmem@d7617000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7617000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem24: scmi-shmem@d7618000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7618000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem25: scmi-shmem@d7619000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7619000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem26: scmi-shmem@d761a000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd761a000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem27: scmi-shmem@d761b000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd761b000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem28: scmi-shmem@d761c000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd761c000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem29: scmi-shmem@d761d000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd761d000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem30: scmi-shmem@d761e000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd761e000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem31: scmi-shmem@d761f000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd761f000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem32: scmi-shmem@d7620000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7620000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem33: scmi-shmem@d7621000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7621000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem34: scmi-shmem@d7622000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7622000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem35: scmi-shmem@d7623000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7623000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem36: scmi-shmem@d7624000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7624000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem37: scmi-shmem@d7625000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7625000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem38: scmi-shmem@d7626000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7626000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem39: scmi-shmem@d7627000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7627000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem40: scmi-shmem@d7628000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7628000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem41: scmi-shmem@d7629000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7629000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem42: scmi-shmem@d762a000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd762a000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem43: scmi-shmem@d762b000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd762b000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem44: scmi-shmem@d762c000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd762c000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem45: scmi-shmem@d762d000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd762d000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem46: scmi-shmem@d762e000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd762e000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem47: scmi-shmem@d762f000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd762f000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem48: scmi-shmem@d7630000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7630000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem49: scmi-shmem@d7631000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7631000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem50: scmi-shmem@d7632000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7632000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem51: scmi-shmem@d7633000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7633000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem52: scmi-shmem@d7634000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7634000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem53: scmi-shmem@d7635000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7635000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem54: scmi-shmem@d7636000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7636000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem55: scmi-shmem@d7637000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7637000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem56: scmi-shmem@d7638000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7638000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem57: scmi-shmem@d7639000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd7639000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem58: scmi-shmem@d763a000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd763a000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem59: scmi-shmem@d763b000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd763b000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem60: scmi-shmem@d763c000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd763c000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem61: scmi-shmem@d763d000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd763d000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem62: scmi-shmem@d763e000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd763e000 0x0 0x1000>;
+		no-map;
+	};
+
+	shmem63: scmi-shmem@d763f000 {
+		compatible = "arm,scmi-shmem";
+		reg = <0x0 0xd763f000 0x0 0x1000>;
+		no-map;
+	};
+};
+
+&spi0 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 0>,
+			<&scmi10_dvfs 0>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi1 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 1>,
+			<&scmi10_dvfs 1>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi2 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 2>,
+			<&scmi10_dvfs 2>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi3 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 3>,
+			<&scmi10_dvfs 3>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi4 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 4>,
+			<&scmi10_dvfs 4>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi5 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 5>,
+			<&scmi10_dvfs 5>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi7 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 7>,
+			<&scmi10_dvfs 7>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi8 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 8>,
+			<&scmi10_dvfs 8>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi11 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 11>,
+			<&scmi10_dvfs 11>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi12 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 12>,
+			<&scmi10_dvfs 12>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi13 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 13>,
+			<&scmi10_dvfs 13>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi14 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 14>,
+			<&scmi10_dvfs 14>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi15 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 15>,
+			<&scmi10_dvfs 15>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi16 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 16>,
+			<&scmi10_dvfs 16>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi17 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 17>,
+			<&scmi10_dvfs 17>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi18 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 18>,
+			<&scmi10_dvfs 18>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi19 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 19>,
+			<&scmi10_dvfs 19>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&spi20 {
+	compatible = "qcom,sa8797p-geni-spi",
+		     "qcom,sa8255p-geni-spi";
+	power-domains = <&scmi10_pd 20>,
+			<&scmi10_dvfs 20>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart0 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 0>,
+			<&scmi11_dvfs 0>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart1 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 1>,
+			<&scmi11_dvfs 1>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart2 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 2>,
+			<&scmi11_dvfs 2>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart3 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 3>,
+			<&scmi11_dvfs 3>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart4 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 4>,
+			<&scmi11_dvfs 4>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart5 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 5>,
+			<&scmi11_dvfs 5>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart7 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 7>,
+			<&scmi11_dvfs 7>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart8 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 8>,
+			<&scmi11_dvfs 8>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart9 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 9>,
+			<&scmi11_dvfs 9>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart10 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 10>,
+			<&scmi11_dvfs 10>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart11 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 11>,
+			<&scmi11_dvfs 11>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart12 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 12>,
+			<&scmi11_dvfs 12>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart13 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 13>,
+			<&scmi11_dvfs 13>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart14 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 14>,
+			<&scmi11_dvfs 14>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart15 {
+	compatible = "qcom,sa8797p-geni-debug-uart",
+		     "qcom,sa8255p-geni-debug-uart";
+	power-domains = <&scmi11_pd 15>,
+			<&scmi11_dvfs 15>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart16 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 16>,
+			<&scmi11_dvfs 16>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart17 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 17>,
+			<&scmi11_dvfs 17>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart18 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 18>,
+			<&scmi11_dvfs 18>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart19 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 19>,
+			<&scmi11_dvfs 19>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&uart20 {
+	compatible = "qcom,sa8797p-geni-uart",
+		     "qcom,sa8255p-geni-uart";
+	power-domains = <&scmi11_pd 20>,
+			<&scmi11_dvfs 20>;
+	power-domain-names = "power",
+			     "perf";
+};
+
+&ufs_mem_hc {
+	compatible = "qcom,sa8797p-ufshc",
+		     "qcom,sa8255p-ufshc";
+	power-domains = <&scmi3_pd 0>;
+};
diff --git a/arch/arm64/boot/dts/qcom/nord-ride.dts b/arch/arm64/boot/dts/qcom/nord-ride.dts
new file mode 100644
index 00000000000000..1af4117d24d774
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/nord-ride.dts
@@ -0,0 +1,254 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "nord-gearvm.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. SA8797P Ride";
+	compatible = "qcom,sa8797p-ride", "qcom,nord";
+
+	aliases {
+		i2c18 = &i2c18;
+		serial0 = &uart15;
+		serial1 = &uart4;
+		spi16 = &spi16;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board_clk: xo-board-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			#clock-cells = <0>;
+		};
+	};
+};
+
+&i2c18 {
+	status = "okay";
+};
+
+&scmi3 {
+	status = "okay";
+};
+
+&scmi9 {
+	status = "okay";
+};
+
+&scmi10 {
+	status = "okay";
+};
+
+&scmi11 {
+	status = "okay";
+};
+
+&scmi23 {
+	status = "okay";
+};
+
+&spi16 {
+	status = "okay";
+};
+
+&thermal_zones {
+	pmic_kobra_thermal: pmic-a-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 3>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_0_thermal: pmic-e-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 4>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_1_thermal: pmic-f-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 5>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_2_thermal: pmic-g-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 6>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_3_thermal: pmic-h-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 7>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_4_thermal: pmic-i-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 8>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_5_thermal: pmic-j-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 9>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_6_thermal: pmic-k-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 10>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_7_thermal: pmic-l-die-thermal {
+		polling-delay-passive = <100>;
+		thermal-sensors = <&scmi23_sensor 11>;
+
+		trips {
+			trip0 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+		};
+	};
+
+	pmic_kai_ufs_thermal: ufs-thermal {
+		polling-delay-passive = <0>;
+		thermal-sensors = <&scmi23_sensor 0>;
+
+		trips {
+			trip0 {
+				temperature = <105000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			trip1 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "critical";
+			};
+		};
+	};
+
+	pmic_kai_sdram0_thermal: sdram0-thermal {
+		polling-delay-passive = <0>;
+		thermal-sensors = <&scmi23_sensor 1>;
+
+		trips {
+			trip0 {
+				temperature = <105000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			trip1 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "critical";
+			};
+		};
+	};
+
+	pmic_kai_sdram1_thermal: sdram1-thermal {
+		polling-delay-passive = <0>;
+		thermal-sensors = <&scmi23_sensor 2>;
+
+		trips {
+			trip0 {
+				temperature = <105000>;
+				hysteresis = <5000>;
+				type = "passive";
+			};
+
+			trip1 {
+				temperature = <115000>;
+				hysteresis = <5000>;
+				type = "critical";
+			};
+		};
+	};
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&uart15 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/nord-rrd.dts b/arch/arm64/boot/dts/qcom/nord-rrd.dts
new file mode 100644
index 00000000000000..3673003c8b213e
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/nord-rrd.dts
@@ -0,0 +1,424 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "nord-embedded.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IQ10 RRD";
+	compatible = "qcom,iq10-rrd", "qcom,nord";
+
+	aliases {
+		serial0 = &uart15;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		xo_board: xo-board-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <38400000>;
+			#clock-cells = <0>;
+		};
+
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <32000>;
+			#clock-cells = <0>;
+		};
+
+		bi_tcxo_div2: bi-tcxo-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+
+		bi_tcxo_ao_div2: bi-tcxo-ao-div2-clk {
+			compatible = "fixed-factor-clock";
+			#clock-cells = <0>;
+			clocks = <&rpmhcc RPMH_CXO_CLK_A>;
+			clock-mult = <1>;
+			clock-div = <2>;
+		};
+	};
+};
+
+&apps_rsc {
+	/* PMIC A - Kobra_MM (PMM8650AU) - SID 0x0, Bus E0 */
+	regulators-0 {
+		compatible = "qcom,pmm8654au-rpmh-regulators";
+		qcom,pmic-id = "A_E0";
+
+		vreg_l4a_1p2: ldo4 {
+			regulator-name = "vreg_l4a_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6a_1p2: ldo6 {
+			regulator-name = "vreg_l6a_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7a_1p2: ldo7 {
+			regulator-name = "vreg_l7a_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8a_1p8: ldo8 {
+			regulator-name = "vreg_l8a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s1a_vdd2h_l: smps1 {
+			regulator-name = "vreg_s1a_vdd2h_l";
+			regulator-min-microvolt = <900000>;
+			regulator-max-microvolt = <1100000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vreg_s3a_1p8: smps3 {
+			regulator-name = "vreg_s3a_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vreg_s5a_mv: smps5 {
+			regulator-name = "vreg_s5a_mv";
+			regulator-min-microvolt = <1328000>;
+			regulator-max-microvolt = <1370000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vreg_s6a_vddq_l: smps6 {
+			regulator-name = "vreg_s6a_vddq_l";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <570000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMIC E - Kai_MV - SID 0x4, Bus E0 */
+	regulators-1 {
+		compatible = "qcom,pmau0102-rpmh-regulators";
+		qcom,pmic-id = "E_E0";
+
+		vreg_l1e_0p9: ldo1 {
+			regulator-name = "vreg_l1e_0p9";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2e_0p9: ldo2 {
+			regulator-name = "vreg_l2e_0p9";
+			regulator-min-microvolt = <936000>;
+			regulator-max-microvolt = <936000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p8: ldo3 {
+			regulator-name = "vreg_l3e_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMIC F - Kai_MV - SID 0x5, Bus E0 */
+	regulators-2 {
+		compatible = "qcom,pmau0102-rpmh-regulators";
+		qcom,pmic-id = "F_E0";
+
+		vreg_l1f_vdd2l: ldo1 {
+			regulator-name = "vreg_l1f_vdd2l";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_vdd1: ldo3 {
+			regulator-name = "vreg_l3f_vdd1";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s7f_lv_sub: smps7 {
+			regulator-name = "vreg_s7f_lv_sub";
+			regulator-min-microvolt = <1036000>;
+			regulator-max-microvolt = <1136000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vreg_s8f_vddq_h: smps8 {
+			regulator-name = "vreg_s8f_vddq_h";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <570000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMIC G - Kai_MV - SID 0x6, Bus E0 */
+	regulators-3 {
+		compatible = "qcom,pmau0102-rpmh-regulators";
+		qcom,pmic-id = "G_E0";
+
+		vreg_l2g_0p7: ldo2 {
+			regulator-name = "vreg_l2g_0p7";
+			regulator-min-microvolt = <752000>;
+			regulator-max-microvolt = <752000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5g_vdd2h_h: smps5 {
+			regulator-name = "vreg_s5g_vdd2h_h";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1150000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+	};
+
+	/* PMIC H - Kai_MV - SID 0x7, Bus E0 */
+	regulators-4 {
+		compatible = "qcom,pmau0102-rpmh-regulators";
+		qcom,pmic-id = "H_E0";
+
+		vreg_l1h_0p9: ldo1 {
+			regulator-name = "vreg_l1h_0p9";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2h_1p2: ldo2 {
+			regulator-name = "vreg_l2h_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMIC I - Kai_MV - SID 0x8, Bus E0 */
+	regulators-5 {
+		compatible = "qcom,pmau0102-rpmh-regulators";
+		qcom,pmic-id = "I_E0";
+
+		vreg_l1i_0p9: ldo1 {
+			regulator-name = "vreg_l1i_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_1p2: ldo2 {
+			regulator-name = "vreg_l2i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3i_2p5: ldo3 {
+			regulator-name = "vreg_l3i_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMIC K - Kai_MV - SID 0xA, Bus E0 */
+	regulators-6 {
+		compatible = "qcom,pmau0102-rpmh-regulators";
+		qcom,pmic-id = "K_E0";
+
+		vreg_l1k_vdd2l: ldo1 {
+			regulator-name = "vreg_l1k_vdd2l";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2k_0p9: ldo2 {
+			regulator-name = "vreg_l2k_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3k_vdd1: ldo3 {
+			regulator-name = "vreg_l3k_vdd1";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	/* PMIC L - Kai_MV - SID 0xB, Bus E0 */
+	regulators-7 {
+		compatible = "qcom,pmau0102-rpmh-regulators";
+		qcom,pmic-id = "L_E0";
+
+		vreg_l3l_1p8: ldo3 {
+			regulator-name = "vreg_l3l_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&i2c7 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	rtc@32 {
+		compatible = "epson,rx8900";
+		reg = <0x32>;
+	};
+};
+
+&qupv3_0 {
+	firmware-name = "qcom/nord/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&qupv3_1 {
+	firmware-name = "qcom/nord/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&qupv3_2 {
+	firmware-name = "qcom/nord/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&spi16 {
+	status = "okay";
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&uart15 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 181 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l3i_2p5>;
+	vcc-max-microamp = <1300000>;
+	vccq-supply = <&vreg_l2i_1p2>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l1i_0p9>;
+	vdda-pll-supply = <&vreg_l2h_1p2>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/nord.dtsi b/arch/arm64/boot/dts/qcom/nord.dtsi
new file mode 100644
index 00000000000000..fdbe290350ddd9
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/nord.dtsi
@@ -0,0 +1,4650 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	interrupt-parent = <&intc>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			power-domains = <&cpu0_pd>, <&cpu_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+
+			l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			power-domains = <&cpu1_pd>, <&cpu_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			power-domains = <&cpu2_pd>, <&cpu_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x300>;
+			enable-method = "psci";
+			power-domains = <&cpu3_pd>, <&cpu_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x400>;
+			enable-method = "psci";
+			power-domains = <&cpu4_pd>, <&cpu_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu5: cpu@500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x500>;
+			enable-method = "psci";
+			power-domains = <&cpu5_pd>, <&cpu_perf 0>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu6: cpu@10000 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x10000>;
+			power-domains = <&cpu6_pd>, <&cpu_perf 1>;
+			power-domain-names = "psci", "perf";
+			enable-method = "psci";
+			next-level-cache = <&l2_10000>;
+			#cooling-cells = <2>;
+
+			l2_10000: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu7: cpu@10100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x10100>;
+			enable-method = "psci";
+			power-domains = <&cpu7_pd>, <&cpu_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_10000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu8: cpu@10200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x10200>;
+			enable-method = "psci";
+			power-domains = <&cpu8_pd>, <&cpu_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_10000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu9: cpu@10300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x10300>;
+			enable-method = "psci";
+			power-domains = <&cpu9_pd>, <&cpu_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_10000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu10: cpu@10400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x10400>;
+			enable-method = "psci";
+			power-domains = <&cpu10_pd>, <&cpu_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_10000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu11: cpu@10500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x10500>;
+			enable-method = "psci";
+			power-domains = <&cpu11_pd>, <&cpu_perf 1>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_10000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu12: cpu@20000 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x20000>;
+			enable-method = "psci";
+			power-domains = <&cpu12_pd>, <&cpu_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_20000>;
+			#cooling-cells = <2>;
+
+			l2_20000: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+			};
+		};
+
+		cpu13: cpu@20100 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x20100>;
+			enable-method = "psci";
+			power-domains = <&cpu13_pd>, <&cpu_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_20000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu14: cpu@20200 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x20200>;
+			enable-method = "psci";
+			power-domains = <&cpu14_pd>, <&cpu_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_20000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu15: cpu@20300 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x20300>;
+			enable-method = "psci";
+			power-domains = <&cpu15_pd>, <&cpu_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_20000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu16: cpu@20400 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x20400>;
+			enable-method = "psci";
+			power-domains = <&cpu16_pd>, <&cpu_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_20000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu17: cpu@20500 {
+			device_type = "cpu";
+			compatible = "qcom,oryon-1-5";
+			reg = <0x0 0x20500>;
+			enable-method = "psci";
+			power-domains = <&cpu17_pd>, <&cpu_perf 2>;
+			power-domain-names = "psci", "perf";
+			next-level-cache = <&l2_20000>;
+			#cooling-cells = <2>;
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+
+				core4 {
+					cpu = <&cpu4>;
+				};
+
+				core5 {
+					cpu = <&cpu5>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu6>;
+				};
+
+				core1 {
+					cpu = <&cpu7>;
+				};
+
+				core2 {
+					cpu = <&cpu8>;
+				};
+
+				core3 {
+					cpu = <&cpu9>;
+				};
+
+				core4 {
+					cpu = <&cpu10>;
+				};
+
+				core5 {
+					cpu = <&cpu11>;
+				};
+			};
+
+			cluster2 {
+				core0 {
+					cpu = <&cpu12>;
+				};
+
+				core1 {
+					cpu = <&cpu13>;
+				};
+
+				core2 {
+					cpu = <&cpu14>;
+				};
+
+				core3 {
+					cpu = <&cpu15>;
+				};
+
+				core4 {
+					cpu = <&cpu16>;
+				};
+
+				core5 {
+					cpu = <&cpu17>;
+				};
+			};
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			core_off_c4: cluster-c4 {
+				compatible = "arm,idle-state";
+				idle-state-name = "retention";
+				entry-latency-us = <93>;
+				exit-latency-us = <129>;
+				min-residency-us = <560>;
+				arm,psci-suspend-param = <0x00000003>;
+			};
+		};
+
+		domain-idle-states {
+			cluster_pwr_dn: cluster-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x01000053>;
+				entry-latency-us = <2150>;
+				exit-latency-us = <1983>;
+				min-residency-us = <9144>;
+			};
+
+			domain_ss3: domain-sleep-0 {
+				compatible = "domain-idle-state";
+				arm,psci-suspend-param = <0x02000153>;
+				entry-latency-us = <2800>;
+				exit-latency-us = <4400>;
+				min-residency-us = <10150>;
+			};
+		};
+	};
+
+	firmware: firmware {
+		scm {
+			compatible = "qcom,scm-nord",
+				     "qcom,scm";
+			qcom,dload-mode = <&tcsr 0x79000>;
+		};
+
+		pdp_scmi: scmi {
+			compatible = "arm,scmi";
+			mboxes = <&pdp0_mbox 0>,
+				 <&pdp0_mbox 11>,
+				 <&pdp0_mbox 1>;
+			mbox-names = "tx",
+				     "tx_reply",
+				     "rx";
+			shmem = <&pdp0_a2p>,
+				<&pdp0_p2a>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			cpu_perf: protocol@13 {
+				reg = <0x13>;
+				#power-domain-cells = <1>;
+			};
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* Size will be updated by bootloader */
+		reg = <0x0 0x80000000 0x0 0x0>;
+	};
+
+	pmu {
+		compatible = "arm,armv8-pmuv3";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+
+		cpu0_pd: power-domain-cpu0 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu1_pd: power-domain-cpu1 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu2_pd: power-domain-cpu2 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu3_pd: power-domain-cpu3 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu4_pd: power-domain-cpu4 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu5_pd: power-domain-cpu5 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster0_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu6_pd: power-domain-cpu6 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu7_pd: power-domain-cpu7 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu8_pd: power-domain-cpu8 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu9_pd: power-domain-cpu9 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu10_pd: power-domain-cpu10 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu11_pd: power-domain-cpu11 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster1_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu12_pd: power-domain-cpu12 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu13_pd: power-domain-cpu13 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu14_pd: power-domain-cpu14 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu15_pd: power-domain-cpu15 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu16_pd: power-domain-cpu16 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cpu17_pd: power-domain-cpu17 {
+			#power-domain-cells = <0>;
+			power-domains = <&cluster2_pd>;
+			domain-idle-states = <&core_off_c4>;
+		};
+
+		cluster0_pd: power-domain-cluster0 {
+			#power-domain-cells = <0>;
+			power-domains = <&system_pd>;
+			domain-idle-states = <&cluster_pwr_dn>;
+		};
+
+		cluster1_pd: power-domain-cluster1 {
+			#power-domain-cells = <0>;
+			power-domains = <&system_pd>;
+			domain-idle-states = <&cluster_pwr_dn>;
+		};
+
+		cluster2_pd: power-domain-cluster2 {
+			#power-domain-cells = <0>;
+			power-domains = <&system_pd>;
+			domain-idle-states = <&cluster_pwr_dn>;
+		};
+
+		system_pd: power-domain-system {
+			#power-domain-cells = <0>;
+			domain-idle-states = <&domain_ss3>;
+		};
+	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		cpucp_scandump_mem: cpucp-scandump-region@80000000 {
+			reg = <0x0 0x80000000 0x0 0x800000>;
+			no-map;
+		};
+
+		tme_sail_mem: tme-sail-region@81ff0000 {
+			reg = <0x0 0x81ff0000 0x0 0x10000>;
+			no-map;
+		};
+
+		tz_sail_mailbox_mem: tz-sail-mailbox-region@82000000 {
+			reg = <0x0 0x82000000 0x0 0x8000>;
+			no-map;
+		};
+
+		sail_mailbox_mem: sail-mailbox-region@82008000 {
+			reg = <0x0 0x82008000 0x0 0x1f8000>;
+			no-map;
+		};
+
+		sail_ota_mem: sail-ota-region@82200000 {
+			reg = <0x0 0x82200000 0x0 0x5ff000>;
+			no-map;
+		};
+
+		sail_vdt_mem: sail-vdt-region@827ff000 {
+			reg = <0x0 0x827ff000 0x0 0x1000>;
+			no-map;
+		};
+
+		hyp_mem: hyp-region@82800000 {
+			reg = <0x0 0x82800000 0x0 0x2400000>;
+			no-map;
+		};
+
+		deepsleep_mem: deepsleep-region@84c00000 {
+			reg = <0x0 0x84c00000 0x0 0x800000>;
+			no-map;
+		};
+
+		deepsleep_backup_mem: deepsleep-backup-region@86a00000 {
+			reg = <0x0 0x86a00000 0x0 0x200000>;
+			no-map;
+		};
+
+		soccp_fe_vm_0: soccp-fe-vm-0-region@86c00000 {
+			reg = <0x0 0x86c00000 0x0 0xac000>;
+			no-map;
+		};
+
+		soccp_fe_vm_1: soccp-fe-vm-1-region@86cac000 {
+			reg = <0x0 0x86cac000 0x0 0x18d000>;
+			no-map;
+		};
+
+		soccp_fe_vm_2: soccp-fe-vm-2-region@86e39000 {
+			reg = <0x0 0x86e39000 0x0 0x1c7000>;
+			no-map;
+		};
+
+		tme_crash_dump_mem: tme-crash-dump-region@87000000 {
+			reg = <0x0 0x87000000 0x0 0xa0000>;
+			no-map;
+		};
+
+		tme_log_mem: tme-log-region@87140000 {
+			reg = <0x0 0x87140000 0x0 0x4000>;
+			no-map;
+		};
+
+		aop_cmd_db_p_mem: aop-cmd-db-p-region@87148000 {
+			compatible = "qcom,cmd-db";
+			reg = <0x0 0x87148000 0x0 0x20000>;
+			no-map;
+		};
+
+		nsp_sync_buffer_mem: nsp-sync-buffer-region@871ff000 {
+			reg = <0x0 0x871ff000 0x0 0x1000>;
+			no-map;
+		};
+
+		ddr_training_checksum_data_mem: ddr-training-checksum-data-region@87200000 {
+			reg = <0x0 0x87200000 0x0 0x2000>;
+			no-map;
+		};
+
+		xbl_dtlog_mem: xbl-dtlog-region@87202000 {
+			reg = <0x0 0x87202000 0x0 0x60000>;
+			no-map;
+		};
+
+		xbl_ramdump_mem: xbl-ramdump-region@87262000 {
+			reg = <0x0 0x87262000 0x0 0x1c0000>;
+			no-map;
+		};
+
+		uefi_log: uefi-log@87442000 {
+			reg = <0x0 0x87442000 0x0 0x10000>;
+			no-map;
+		};
+
+		secdata_apss_mem: secdata-apss-region@87452000 {
+			reg = <0x0 0x87452000 0x0 0x1000>;
+			no-map;
+		};
+
+		antireplay_emulation_mem: antireplay-emulation-region@87453000 {
+			reg = <0x0 0x87453000 0x0 0x1000>;
+			no-map;
+		};
+
+		soccp_sdi_mem: soccp-sdi-region@87454000 {
+			reg = <0x0 0x87454000 0x0 0x40000>;
+			no-map;
+		};
+
+		hyp_mem_database_mem: hyp-mem-database-region@87494000 {
+			reg = <0x0 0x87494000 0x0 0x60000>;
+			no-map;
+		};
+
+		pmic_mini_dump_mem: pmic-mini-dump-region@874f4000 {
+			reg = <0x0 0x874f4000 0x0 0x80000>;
+			no-map;
+		};
+
+		qup_fw_mem: qup-fw-region@87574000 {
+			reg = <0x0 0x87574000 0x0 0x20000>;
+			no-map;
+		};
+
+		softsku_mem: softsku-region@87594000 {
+			reg = <0x0 0x87594000 0x0 0x9000>;
+			no-map;
+		};
+
+		resource_scheduler_mem: resource-scheduler-region@8759d000 {
+			reg = <0x0 0x8759d000 0x0 0x20000>;
+			no-map;
+		};
+
+		pdp_ns_mem: pdp-ns-mem-region@87600000 {
+			reg = <0x0 0x87600000 0x0 0x8000>,
+			      <0x0 0x87609000 0x0 0x1f7000>;
+			no-map;
+		};
+
+		pdp0_p2a: scmi-shmem@87608000 {
+			compatible = "arm,scmi-shmem";
+			reg = <0x0 0x87608000 0x0 0x80>;
+			no-map;
+		};
+
+		pdp0_a2p: scmi-shmem@87608180 {
+			compatible = "arm,scmi-shmem";
+			reg = <0x0 0x87608180 0x0 0x80>;
+			no-map;
+		};
+
+		tz_stat_mem: tz-stat-region@87a00000 {
+			reg = <0x0 0x87a00000 0x0 0x100000>;
+			no-map;
+		};
+
+		qdss_apps_mem: qdss-apps-region@87b00000 {
+			reg = <0x0 0x87b00000 0x0 0x2000000>;
+			no-map;
+		};
+
+		global_sync_mem: global-sync-region@89f00000 {
+			reg = <0x0 0x89f00000 0x0 0x400000>;
+			no-map;
+		};
+
+		tzffi_mem: tzffi-region@8a300000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x8a300000 0x0 0x1400000>;
+			no-map;
+		};
+
+		gunyah_md_mem: gunyah-md-region@8b700000 {
+			reg = <0x0 0x8b700000 0x0 0x80000>;
+			no-map;
+		};
+
+		flashless_qntm_tool_mem: flashless-qntm-tool-region@8b780000 {
+			reg = <0x0 0x8b780000 0x0 0x182000>;
+			no-map;
+		};
+
+		ipa_fw_mem: ipa-fw-region@8bb00000 {
+			reg = <0x0 0x8bb00000 0x0 0x10000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa-gsi-region@8bb10000 {
+			reg = <0x0 0x8bb10000 0x0 0xa000>;
+			no-map;
+		};
+
+		gpu_microcode_mem: gpu-microcode-region@8bb1a000 {
+			reg = <0x0 0x8bb1a000 0x0 0x2000>;
+			no-map;
+		};
+
+		gpu_microcode_2_mem: gpu-microcode-2-region@8bb1c000 {
+			reg = <0x0 0x8bb1c000 0x0 0x2000>;
+			no-map;
+		};
+
+		soccp_mem: soccp-region@8bc00000 {
+			reg = <0x0 0x8bc00000 0x0 0x300000>;
+			no-map;
+		};
+
+		cvp_mem: cvp-region@8d100000 {
+			reg = <0x0 0x8d100000 0x0 0x800000>;
+			no-map;
+		};
+
+		cdsp0_mem: cdsp0-region@8d900000 {
+			reg = <0x0 0x8d900000 0x0 0x2300000>;
+			no-map;
+		};
+
+		cdsp1_mem: cdsp1-region@8fc00000 {
+			reg = <0x0 0x8fc00000 0x0 0x2300000>;
+			no-map;
+		};
+
+		cdsp2_mem: cdsp2-region@91f00000 {
+			reg = <0x0 0x91f00000 0x0 0x2300000>;
+			no-map;
+		};
+
+		cdsp3_mem: cdsp3-region@94200000 {
+			reg = <0x0 0x94200000 0x0 0x2300000>;
+			no-map;
+		};
+
+		hpass_dsp0_mem: hpass-dsp0-region@96500000 {
+			reg = <0x0 0x96500000 0x0 0x2800000>;
+			no-map;
+		};
+
+		hpass_dsp1_mem: hpass-dsp1-region@98d00000 {
+			reg = <0x0 0x98d00000 0x0 0x2800000>;
+			no-map;
+		};
+
+		hpass_dsp2_mem: hpass-dsp2-region@9b500000 {
+			reg = <0x0 0x9b500000 0x0 0x2800000>;
+			no-map;
+		};
+
+		q6_cdsp0_dtb_mem: q6-cdsp0-dtb-region@9dd00000 {
+			reg = <0x0 0x9dd00000 0x0 0x80000>;
+			no-map;
+		};
+
+		q6_cdsp1_dtb_mem: q6-cdsp1-dtb-region@9dd80000 {
+			reg = <0x0 0x9dd80000 0x0 0x80000>;
+			no-map;
+		};
+
+		q6_cdsp2_dtb_mem: q6-cdsp2-dtb-region@9de00000 {
+			reg = <0x0 0x9de00000 0x0 0x80000>;
+			no-map;
+		};
+
+		q6_cdsp3_dtb_mem: q6-cdsp3-dtb-region@9de80000 {
+			reg = <0x0 0x9de80000 0x0 0x80000>;
+			no-map;
+		};
+
+		hpass_dsp0_dtb_mem: hpass-dsp0-dtb-region@9df00000 {
+			reg = <0x0 0x9df00000 0x0 0x80000>;
+			no-map;
+		};
+
+		hpass_dsp1_dtb_mem: hpass-dsp1-dtb-region@9df80000 {
+			reg = <0x0 0x9df80000 0x0 0x80000>;
+			no-map;
+		};
+
+		hpass_dsp2_dtb_mem: hpass-dsp2-dtb-region@9e000000 {
+			reg = <0x0 0x9e000000 0x0 0x100000>;
+			no-map;
+		};
+
+		camera_icp_1_mem: camera-icp-1-region@9e100000 {
+			reg = <0x0 0x9e100000 0x0 0x800000>;
+			no-map;
+		};
+
+		camera_icp_2_mem: camera-icp-2-region@9e900000 {
+			reg = <0x0 0x9e900000 0x0 0x800000>;
+			no-map;
+		};
+
+		camera_qup_1_mem: camera-qup-1-region@9f100000 {
+			reg = <0x0 0x9f100000 0x0 0x200000>;
+			no-map;
+		};
+
+		camera_qup_2_mem: camera-qup-2-region@9f300000 {
+			reg = <0x0 0x9f300000 0x0 0x200000>;
+			no-map;
+		};
+
+		video_mem: video-region@9f500000 {
+			reg = <0x0 0x9f500000 0x0 0xc00000>;
+			no-map;
+		};
+
+		pil_umd_reserved: mdt-load-region@a0100000 {
+			reg = <0x0 0xa0100000 0x0 0x100000>;
+			no-map;
+		};
+
+		mm_dspq: mm-dspq-region@ba200000 {
+			reg = <0x0 0xba200000 0x0 0x200000>;
+			no-map;
+		};
+
+		display_config_reserved: display-config-region@ba400000 {
+			reg = <0x0 0xba400000 0x0 0xa00000>;
+			no-map;
+		};
+
+		mm_calibration_data_mem: mm-calibration-data-region@bae00000 {
+			reg = <0x0 0xbae00000 0x0 0x800000>;
+			no-map;
+		};
+
+		audio_config_mem: audio-config-region@bb600000 {
+			reg = <0x0 0xbb600000 0x0 0xa00000>;
+			no-map;
+		};
+
+		dare_tz_mem: dare-tz-region@bc000000 {
+			reg = <0x0 0xbc000000 0x0 0xa300000>;
+			no-map;
+		};
+
+		hpass_rpc_remote_heap_mem: hpass-rpc-remote-heap-region@d4600000 {
+			reg = <0x0 0xd4600000 0x0 0x800000>;
+			no-map;
+		};
+
+		mdf_mem: mdf-region@d4e00000 {
+			reg = <0x0 0xd4e00000 0x0 0x2000000>;
+			no-map;
+		};
+
+		firmware_mem: firmware-region@d6e00000 {
+			reg = <0x0 0xd6e00000 0x0 0x800000>;
+			no-map;
+		};
+
+		firmware_shared_mem: firmware-shared-region@d7650000 {
+			reg = <0x0 0xd7650000 0x0 0x180000>;
+			no-map;
+		};
+
+		firmware_logs_mem: firmware-logs-region@d77d0000 {
+			reg = <0x0 0xd77d0000 0x0 0x20000>;
+			no-map;
+		};
+
+		sail_p_mem: sail-p-region@8c0000000 {
+			reg = <0x8 0xc0000000 0x0 0x8000000>;
+			no-map;
+		};
+
+		reserved_mem2: reserved-region@8c8000000 {
+			reg = <0x8 0xc8000000 0x0 0x18000000>;
+			no-map;
+		};
+
+		dump_mem: mem-dump-region {
+			alloc-ranges = <0x0 0x00000000 0x0 0xffffffff>;
+			size = <0x0 0x79b0000>;
+		};
+	};
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-ranges = <0 0 0 0 0x10 0>;
+		ranges = <0 0 0 0 0x10 0>;
+
+		qupv3_2: geniqup@8c0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x008c0000 0x0 0x2000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			iommus = <&apps_smmu_0 0x15a3 0x0>;
+			ranges;
+
+			i2c14: i2c@880000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi14: spi@880000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart14: serial@880000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00880000 0x0 0x4000>;
+				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c15: i2c@884000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+				interrupts = <GIC_SPI 583 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi15: spi@884000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+				interrupts = <GIC_SPI 583 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart15: serial@884000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x00884000 0x0 0x4000>;
+				interrupts = <GIC_SPI 583 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c16: i2c@888000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi16: spi@888000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart16: serial@888000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00888000 0x0 0x4000>;
+				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c17: i2c@88c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi17: spi@88c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart17: serial@88c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x0088c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c18: i2c@890000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00890000 0x0 0x4000>;
+				interrupts = <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi18: spi@890000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00890000 0x0 0x4000>;
+				interrupts = <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart18: serial@890000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00890000 0x0 0x4000>;
+				interrupts = <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c19: i2c@894000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00894000 0x0 0x4000>;
+				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi19: spi@894000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00894000 0x0 0x4000>;
+				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart19: serial@894000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00894000 0x0 0x4000>;
+				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c20: i2c@898000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00898000 0x0 0x4000>;
+				interrupts = <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi20: spi@898000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00898000 0x0 0x4000>;
+				interrupts = <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart20: serial@898000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00898000 0x0 0x4000>;
+				interrupts = <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+		};
+
+		qupv3_0: geniqup@9c0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x009c0000 0x0 0x2000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			iommus = <&apps_smmu_2 0x1003 0x0>;
+			ranges;
+
+			i2c0: i2c@980000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00980000 0x0 0x4000>;
+				interrupts = <GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi0: spi@980000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00980000 0x0 0x4000>;
+				interrupts = <GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart0: serial@980000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00980000 0x0 0x4000>;
+				interrupts = <GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c1: i2c@984000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00984000 0x0 0x4000>;
+				interrupts = <GIC_SPI 513 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi1: spi@984000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00984000 0x0 0x4000>;
+				interrupts = <GIC_SPI 513 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart1: serial@984000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00984000 0x0 0x4000>;
+				interrupts = <GIC_SPI 513 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c2: i2c@988000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00988000 0x0 0x4000>;
+				interrupts = <GIC_SPI 514 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi2: spi@988000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00988000 0x0 0x4000>;
+				interrupts = <GIC_SPI 514 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart2: serial@988000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00988000 0x0 0x4000>;
+				interrupts = <GIC_SPI 514 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c3: i2c@98c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x0098c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 515 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi3: spi@98c000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x0098c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 515 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart3: serial@98c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x0098c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 515 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c4: i2c@990000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00990000 0x0 0x4000>;
+				interrupts = <GIC_SPI 516 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi4: spi@990000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00990000 0x0 0x4000>;
+				interrupts = <GIC_SPI 516 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart4: serial@990000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00990000 0x0 0x4000>;
+				interrupts = <GIC_SPI 516 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c5: i2c@994000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00994000 0x0 0x4000>;
+				interrupts = <GIC_SPI 517 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi5: spi@994000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00994000 0x0 0x4000>;
+				interrupts = <GIC_SPI 517 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart5: serial@994000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00994000 0x0 0x4000>;
+				interrupts = <GIC_SPI 517 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+		};
+
+		qupv3_1: geniqup@ac0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x00ac0000 0x0 0x2000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			iommus = <&apps_smmu_2 0x1043 0x0>;
+			ranges;
+
+			i2c7: i2c@a80000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi7: spi@a80000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart7: serial@a80000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a80000 0x0 0x4000>;
+				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c8: i2c@a84000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi8: spi@a84000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart8: serial@a84000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c9: i2c@a88000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a88000 0x0 0x4000>;
+				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart9: serial@a88000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a88000 0x0 0x4000>;
+				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c10: i2c@a8c000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a8c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			uart10: serial@a8c000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a8c000 0x0 0x4000>;
+				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c11: i2c@a90000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi11: spi@a90000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart11: serial@a90000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a90000 0x0 0x4000>;
+				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c12: i2c@a94000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi12: spi@a94000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart12: serial@a94000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a94000 0x0 0x4000>;
+				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			i2c13: i2c@a98000 {
+				compatible = "qcom,geni-i2c";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>;
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				status = "disabled";
+			};
+
+			spi13: spi@a98000 {
+				compatible = "qcom,geni-spi";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+
+			uart13: serial@a98000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x00a98000 0x0 0x4000>;
+				interrupts = <GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>;
+
+				status = "disabled";
+			};
+		};
+
+		rng: rng@10c3000 {
+			compatible = "qcom,nord-trng",
+				     "qcom,trng";
+			reg = <0x0 0x010c3000 0x0 0x1000>;
+		};
+
+		ufs_mem_hc: ufshc@1d44000 {
+			compatible = "qcom,nord-ufshc",
+				     "qcom,ufshc",
+				     "jedec,ufs-2.0";
+			reg = <0x0 0x01d44000 0x0 0x3000>,
+			      <0x0 0x01d60000 0x0 0x15000>;
+			reg-names = "std",
+				    "mcq";
+			interrupts = <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH>;
+			lanes-per-direction = <2>;
+			iommus = <&apps_smmu_0 0x14c0 0x0>;
+			dma-coherent;
+			msi-parent = <&gic_its 0x14c0>;
+
+			status = "disabled";
+		};
+
+		cryptobam: dma-controller@1dc4000 {
+			compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0";
+			reg = <0x0 0x01dc4000 0x0 0x28000>;
+			interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			iommus = <&apps_smmu_0 0x1689 0>;
+			qcom,ee = <0>;
+			qcom,num-ees = <4>;
+			num-channels = <20>;
+			qcom,controlled-remotely;
+		};
+
+		crypto: crypto@1dfa000 {
+			compatible = "qcom,nord-qce", "qcom,sm8150-qce", "qcom,qce";
+			reg = <0x0 0x01dfa000 0x0 0x6000>;
+			dmas = <&cryptobam 4>, <&cryptobam 5>;
+			dma-names = "rx", "tx";
+			iommus = <&apps_smmu_0 0x1689 0>;
+
+			/* It stops nord-gearvm variant from booting to console */
+			status = "disabled";
+		};
+
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x01f40000 0x0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr: syscon@1f60000 {
+			compatible = "qcom,nord-tcsr",
+				     "syscon";
+			reg = <0x0 0x01f60000 0x0 0xa0000>;
+		};
+
+		pdc: interrupt-controller@b220000 {
+			compatible = "qcom,nord-pdc",
+				     "qcom,pdc";
+			reg = <0x0 0x0b220000 0x0 0x10000>;
+			qcom,pdc-ranges = <0 745 43>,
+					  <67 543 31>,
+					  <98 609 32>,
+					  <130 717 12>,
+					  <142 251 5>,
+					  <147 796 16>;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&intc>;
+			interrupt-controller;
+		};
+
+		tsens0: thermal-sensor@c22c000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c22c000 0x0 0x1000>,
+			      <0x0 0x0c222000 0x0 0x1000>;
+			interrupts = <GIC_SPI 771 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 484 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens1: thermal-sensor@c22d000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c22d000 0x0 0x1000>,
+			      <0x0 0x0c223000 0x0 0x1000>;
+			interrupts = <GIC_SPI 772 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 485 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens2: thermal-sensor@c22e000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c22e000 0x0 0x1000>,
+			      <0x0 0x0c224000 0x0 0x1000>;
+			interrupts = <GIC_SPI 773 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 486 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens3: thermal-sensor@c22f000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c22f000 0x0 0x1000>,
+			      <0x0 0x0c225000 0x0 0x1000>;
+			interrupts = <GIC_SPI 774 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 487 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens4: thermal-sensor@c230000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c230000 0x0 0x1000>,
+			      <0x0 0x0c226000 0x0 0x1000>;
+			interrupts = <GIC_SPI 775 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens5: thermal-sensor@c231000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c231000 0x0 0x1000>,
+			      <0x0 0x0c227000 0x0 0x1000>;
+			interrupts = <GIC_SPI 776 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens6: thermal-sensor@c232000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c232000 0x0 0x1000>,
+			      <0x0 0x0c228000 0x0 0x1000>;
+			interrupts = <GIC_SPI 777 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens7: thermal-sensor@c233000 {
+			compatible = "qcom,nord-tsens",
+				     "qcom,tsens-v2";
+			reg = <0x0 0x0c233000 0x0 0x1000>,
+			      <0x0 0x0c229000 0x0 0x1000>;
+			interrupts = <GIC_SPI 778 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "uplow",
+					  "critical";
+			#qcom,sensors = <16>;
+			#thermal-sensor-cells = <1>;
+		};
+
+		tlmm: pinctrl@f100000 {
+			compatible = "qcom,nord-tlmm";
+			reg = <0x0 0x0f100000 0x0 0xc0000>;
+			interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			gpio-ranges = <&tlmm 0 0 181>;
+			wakeup-parent = <&pdc>;
+		};
+
+		apps_smmu_0: iommu@15a00000 {
+			compatible = "qcom,nord-smmu-500",
+				     "qcom,smmu-500",
+				     "arm,mmu-500";
+			reg = <0x0 0x15a00000 0x0 0x100000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			dma-coherent;
+			interrupts = <GIC_SPI 886 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 32 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 33 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 34 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 35 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 36 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 37 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 38 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 39 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 40 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 41 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 42 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 46 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 47 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 48 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 49 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 50 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 51 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 52 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 54 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 55 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 56 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 57 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 58 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 59 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 60 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 61 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 62 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 63 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 64 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 65 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 66 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 67 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 69 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 70 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 71 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 74 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 75 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 76 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 77 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 78 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 79 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 80 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 81 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 82 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 83 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 85 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 86 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 87 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 88 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 89 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 90 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 91 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 92 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 93 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 94 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 95 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 96 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 97 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 98 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 99 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 101 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 102 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 106 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 107 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 108 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 109 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 110 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 111 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 112 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 113 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 114 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 115 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 116 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 117 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 118 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 119 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 120 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 121 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 122 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 123 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 124 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 125 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 126 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 127 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		apps_smmu_1: iommu@15c00000 {
+			compatible = "qcom,nord-smmu-500",
+				     "qcom,smmu-500",
+				     "arm,mmu-500";
+			reg = <0x0 0x15c00000 0x0 0x100000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			dma-coherent;
+			interrupts = <GIC_SPI 882 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 128 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 129 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 130 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 131 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 132 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 133 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 134 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 135 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 136 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 137 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 138 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 139 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 140 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 141 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 147 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 148 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 149 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 150 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 151 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 152 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 153 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 154 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 155 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 156 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 157 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 158 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 159 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 160 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 161 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 162 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 163 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 164 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 165 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 166 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 167 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 168 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 169 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 170 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 171 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 172 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 173 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 175 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 176 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 177 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 178 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 179 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 180 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 182 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 183 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 184 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 185 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 186 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 187 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 188 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 189 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 191 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 192 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 193 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 194 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 195 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 197 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 198 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 199 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 200 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 201 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 202 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 203 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 204 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 205 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 206 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 207 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 208 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 209 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 210 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 211 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 212 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 213 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 214 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 215 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 216 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 217 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 218 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 219 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 220 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 221 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 222 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 223 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		apps_smmu_2: iommu@15e00000 {
+			compatible = "qcom,nord-smmu-500",
+				     "qcom,smmu-500",
+				     "arm,mmu-500";
+			reg = <0x0 0x15e00000 0x0 0x100000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			dma-coherent;
+			interrupts = <GIC_SPI 877 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 224 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 225 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 226 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 227 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 228 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 229 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 230 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 231 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 232 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 233 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 234 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 235 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 236 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 237 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 238 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 239 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 240 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 241 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 242 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 243 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 244 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 245 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 246 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 247 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 248 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 249 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 250 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 251 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 252 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 253 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 254 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 255 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 256 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 257 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 258 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 259 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 260 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 261 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 262 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 263 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 264 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 265 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 266 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 267 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 268 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 269 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 270 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 271 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 272 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 273 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 274 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 275 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 276 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 277 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 278 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 279 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 280 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 281 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 282 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 283 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 284 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 285 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 286 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 287 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 288 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 289 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 290 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 291 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 292 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 293 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 294 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 295 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 296 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 297 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 298 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 299 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 300 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 301 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 302 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 303 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 304 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 305 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 306 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 307 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 308 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 309 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 310 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 311 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 312 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 313 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 314 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 315 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 316 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 317 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 318 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_ESPI 319 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		intc: interrupt-controller@17000000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x17000000 0x0 0x10000>,
+			      <0x0 0x17080000 0x0 0x480000>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			#redistributor-regions = <1>;
+			redistributor-stride = <0x0 0x40000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			gic_its: msi-controller@17040000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x17040000 0x0 0x40000>;
+				msi-controller;
+				#msi-cells = <1>;
+			};
+		};
+
+		pdp0_mbox: mailbox@17610000 {
+			compatible = "qcom,nord-cpucp-mbox",
+				     "qcom,x1e80100-cpucp-mbox";
+			reg = <0x0 0x17610000 0x0 0x4c08>,
+			      <0x0 0x19980000 0x0 0x300>;
+			#mbox-cells = <1>;
+			interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		memtimer: timer@17810000 {
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x0 0x17810000 0x0 0x1000>;
+			ranges = <0x0 0x0 0x0 0x20000000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			frame@17811000 {
+				reg = <0x17811000 0x1000>,
+				      <0x17812000 0x1000>;
+				interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+				frame-number = <0>;
+			};
+
+			frame@17813000 {
+				reg = <0x17813000 0x1000>;
+				interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
+				frame-number = <1>;
+
+				status = "disabled";
+			};
+
+			frame@17815000 {
+				reg = <0x17815000 0x1000>;
+				interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
+				frame-number = <2>;
+
+				status = "disabled";
+			};
+
+			frame@17817000 {
+				reg = <0x17817000 0x1000>;
+				interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
+				frame-number = <3>;
+
+				status = "disabled";
+			};
+
+			frame@17819000 {
+				reg = <0x17819000 0x1000>;
+				interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
+				frame-number = <4>;
+
+				status = "disabled";
+			};
+
+			frame@1781b000 {
+				reg = <0x1781b000 0x1000>;
+				interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
+				frame-number = <5>;
+
+				status = "disabled";
+			};
+
+			frame@1781d000 {
+				reg = <0x1781d000 0x1000>;
+				interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
+				frame-number = <6>;
+
+				status = "disabled";
+			};
+		};
+
+		watchdog@17826000 {
+			compatible = "qcom,apss-wdt-nord",
+				     "qcom,kpss-wdt";
+			reg = <0x0 0x17826000 0x0 0x1000>;
+			clocks = <&sleep_clk>;
+			interrupts = <GIC_ESPI 848 IRQ_TYPE_EDGE_RISING>;
+		};
+
+		llcc: system-cache-controller@21800000 {
+			compatible = "qcom,nord-llcc";
+			reg = <0x0 0x21800000 0x0 0x100000>,
+			      <0x0 0x21a00000 0x0 0x100000>,
+			      <0x0 0x21c00000 0x0 0x100000>,
+			      <0x0 0x21e00000 0x0 0x100000>,
+			      <0x0 0x22800000 0x0 0x100000>,
+			      <0x0 0x22a00000 0x0 0x100000>,
+			      <0x0 0x22c00000 0x0 0x100000>,
+			      <0x0 0x22e00000 0x0 0x100000>,
+			      <0x0 0x23800000 0x0 0x100000>,
+			      <0x0 0x23a00000 0x0 0x100000>,
+			      <0x0 0x23c00000 0x0 0x100000>,
+			      <0x0 0x23e00000 0x0 0x100000>,
+			      <0x0 0x24800000 0x0 0x100000>,
+			      <0x0 0x24a00000 0x0 0x100000>,
+			      <0x0 0x24c00000 0x0 0x100000>,
+			      <0x0 0x24e00000 0x0 0x100000>,
+			      <0x0 0x20400000 0x0 0x100000>,
+			      <0x0 0x20600000 0x0 0x100000>;
+			reg-names = "llcc0_base",
+				    "llcc1_base",
+				    "llcc2_base",
+				    "llcc3_base",
+				    "llcc4_base",
+				    "llcc5_base",
+				    "llcc6_base",
+				    "llcc7_base",
+				    "llcc8_base",
+				    "llcc9_base",
+				    "llcc10_base",
+				    "llcc11_base",
+				    "llcc12_base",
+				    "llcc13_base",
+				    "llcc14_base",
+				    "llcc15_base",
+				    "llcc_broadcast_base",
+				    "llcc_broadcast_and_base";
+		};
+	};
+
+	arch_timer: timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	thermal_zones: thermal-zones {
+		ddr-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-3-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-4-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-5-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpullc-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-3-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-4-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-5-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpullc-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 14>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddr-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens0 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddr-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-3-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-4-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-5-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpullc-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-3-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-4-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-5-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpullc-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 14>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddr-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens1 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		amux-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-3-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-4-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-5-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpullc-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhvx-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhmx-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhvx-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhmx-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhvx-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhmx-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pcie-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens2 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		amux-3-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-3-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-4-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-2-5-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <125000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpullc-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhvx-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhmx-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhvx-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhmx-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhvx-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audhmx-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pcie-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens3 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-0-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-1-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-2-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-2-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-2-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-3-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-3-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-3-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 14>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-3-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens4 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-0-3-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-1-3-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-2-3-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-2-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-2-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-3-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsphvx-3-3-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-3-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 14>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-3-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens5 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		amux-6-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpu-0-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cv-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		video-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		camera-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddr-2-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddr-3-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-0-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-2-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 14>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpu-0-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens6 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		amux-7-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-0-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-2-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpu-0-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpu-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cv-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		video-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		camera-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddr-2-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddr-3-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 13>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-2-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 14>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpu-0-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&tsens7 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <10000>;
+					type = "passive";
+				};
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmh0104-hawi.dtsi b/arch/arm64/boot/dts/qcom/pmh0104-hawi.dtsi
new file mode 100644
index 00000000000000..9a45c0ddaadc33
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pmh0104-hawi.dtsi
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pmh0104-c-thermal {
+			polling-delay-passive = <100>;
+
+			thermal-sensors = <&pmh0104_c_e0_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "hot";
+				};
+			};
+		};
+
+		pmh0104-j-thermal {
+			polling-delay-passive = <100>;
+
+			thermal-sensors = <&pmh0104_j_e1_temp_alarm>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "hot";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus0 {
+	pmh0104_c_e0: pmic@2 {
+		compatible = "qcom,pmh0104", "qcom,spmi-pmic";
+		reg = <0x2 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0104_c_e0_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x2 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0104_c_e0_gpios: gpio@8800 {
+			compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0104_c_e0_gpios 0 0 8>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
+
+&spmi_bus1 {
+	pmh0104_j_e1: pmic@9 {
+		compatible = "qcom,pmh0104", "qcom,spmi-pmic";
+		reg = <0x9 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pmh0104_j_e1_temp_alarm: temp-alarm@a00 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0xa00>;
+			interrupts = <0x9 0xa 0x0 IRQ_TYPE_EDGE_BOTH>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		pmh0104_j_e1_gpios: gpio@8800 {
+			compatible = "qcom,pmh0104-gpio", "qcom,spmi-gpio";
+			reg = <0x8800>;
+			gpio-controller;
+			gpio-ranges = <&pmh0104_j_e1_gpios 0 0 8>;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pmi8996.dtsi b/arch/arm64/boot/dts/qcom/pmi8996.dtsi
index 31b47209e26194..711a01c7e62479 100644
--- a/arch/arm64/boot/dts/qcom/pmi8996.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmi8996.dtsi
@@ -3,12 +3,12 @@
  * Copyright (c) 2021, Konrad Dybcio <konrad.dybcio@somainline.org>
  */
 
- /*
-  * PMI8996 is a slight modification of PMI8994 with
-  * some notable changes, like being the first PMIC
-  * whose the bootloader has to check to continue booting
-  * and a change to a LABIBB parameter.
-  */
+/*
+ * PMI8996 is a slight modification of PMI8994 with
+ * some notable changes, like being the first PMIC
+ * whose the bootloader has to check to continue booting
+ * and a change to a LABIBB parameter.
+ */
 
 / {
 	qcom,pmic-id = <0x20009 0x10013 0 0>;
diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts b/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts
index ad503beec1d3d8..a811d8ca2dbf33 100644
--- a/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts
+++ b/arch/arm64/boot/dts/qcom/purwa-iot-evk.dts
@@ -60,6 +60,26 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	clocks {
+		clk_rtmr0_xo: clock-rtmr0-xo {
+			compatible = "fixed-clock";
+			clock-frequency = <25000000>;
+			#clock-cells = <0>;
+		};
+
+		clk_rtmr1_xo: clock-rtmr1-xo {
+			compatible = "fixed-clock";
+			clock-frequency = <25000000>;
+			#clock-cells = <0>;
+		};
+
+		clk_rtmr2_xo: clock-rtmr2-xo {
+			compatible = "fixed-clock";
+			clock-frequency = <25000000>;
+			#clock-cells = <0>;
+		};
+	};
+
 	connector3 {
 		compatible = "usb-a-connector";
 		label = "USB-3-Type-A";
@@ -295,7 +315,6 @@
 		pinctrl-names = "default";
 	};
 
-	/* Left unused as the retimer is not used on this board. */
 	vreg_rtmr0_1p15: regulator-rtmr0-1p15 {
 		compatible = "regulator-fixed";
 
@@ -471,32 +490,6 @@
 		regulator-boot-on;
 	};
 
-	/*
-	 * TODO: These two regulators are actually part of the removable M.2
-	 * card and not the EVK mainboard. Need to describe this differently.
-	 * Functionally it works correctly, because all we need to do is to
-	 * turn on the actual 3.3V supply above.
-	 */
-	vreg_wcn_0p95: regulator-wcn-0p95 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_WCN_0P95";
-		regulator-min-microvolt = <950000>;
-		regulator-max-microvolt = <950000>;
-
-		vin-supply = <&vreg_wcn_3p3>;
-	};
-
-	vreg_wcn_1p9: regulator-wcn-1p9 {
-		compatible = "regulator-fixed";
-
-		regulator-name = "VREG_WCN_1P9";
-		regulator-min-microvolt = <1900000>;
-		regulator-max-microvolt = <1900000>;
-
-		vin-supply = <&vreg_wcn_3p3>;
-	};
-
 	vreg_wcn_3p3: regulator-wcn-3p3 {
 		compatible = "regulator-fixed";
 
@@ -616,63 +609,74 @@
 				sound-dai = <&q6apm>;
 			};
 		};
-	};
-
-	wcn7850-pmu {
-		compatible = "qcom,wcn7850-pmu";
 
-		vdd-supply = <&vreg_wcn_0p95>;
-		vddio-supply = <&vreg_l15b_1p8>;
-		vddaon-supply = <&vreg_wcn_0p95>;
-		vdddig-supply = <&vreg_wcn_0p95>;
-		vddrfa1p2-supply = <&vreg_wcn_1p9>;
-		vddrfa1p8-supply = <&vreg_wcn_1p9>;
+		dp0-dai-link {
+			link-name = "DP0 Playback";
 
-		bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>;
-		wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
-
-		pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>;
-		pinctrl-names = "default";
-
-		regulators {
-			vreg_pmu_rfa_cmn: ldo0 {
-				regulator-name = "vreg_pmu_rfa_cmn";
+			codec {
+				sound-dai = <&mdss_dp0>;
 			};
 
-			vreg_pmu_aon_0p59: ldo1 {
-				regulator-name = "vreg_pmu_aon_0p59";
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
 			};
 
-			vreg_pmu_wlcx_0p8: ldo2 {
-				regulator-name = "vreg_pmu_wlcx_0p8";
+			platform {
+				sound-dai = <&q6apm>;
 			};
+		};
 
-			vreg_pmu_wlmx_0p85: ldo3 {
-				regulator-name = "vreg_pmu_wlmx_0p85";
-			};
+		dp1-dai-link {
+			link-name = "DP1 Playback";
 
-			vreg_pmu_btcmx_0p85: ldo4 {
-				regulator-name = "vreg_pmu_btcmx_0p85";
+			codec {
+				sound-dai = <&mdss_dp1>;
 			};
 
-			vreg_pmu_rfa_0p8: ldo5 {
-				regulator-name = "vreg_pmu_rfa_0p8";
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_1>;
 			};
 
-			vreg_pmu_rfa_1p2: ldo6 {
-				regulator-name = "vreg_pmu_rfa_1p2";
+			platform {
+				sound-dai = <&q6apm>;
 			};
+		};
+	};
 
-			vreg_pmu_rfa_1p8: ldo7 {
-				regulator-name = "vreg_pmu_rfa_1p8";
-			};
+	wifi-bt-connector {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&vreg_wcn_3p3>;
+
+		w-disable1-gpios = <&tlmm 117 GPIO_ACTIVE_LOW>;
+		w-disable2-gpios = <&tlmm 116 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&wcn_wlan_en>, <&wcn_bt_en>;
+		pinctrl-names = "default";
 
-			vreg_pmu_pcie_0p9: ldo8 {
-				regulator-name = "vreg_pmu_pcie_0p9";
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcie4_port0_ep>;
+				};
 			};
 
-			vreg_pmu_pcie_1p8: ldo9 {
-				regulator-name = "vreg_pmu_pcie_1p8";
+			port@3 {
+				reg = <3>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2_e_uart_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&uart14_ep>;
+				};
 			};
 		};
 	};
@@ -686,7 +690,7 @@
 		compatible = "parade,ps8830";
 		reg = <0x08>;
 
-		clocks = <&rpmhcc RPMH_RF_CLK5>;
+		clocks = <&clk_rtmr2_xo>;
 
 		vdd-supply = <&vreg_rtmr2_1p15>;
 		vdd33-supply = <&vreg_rtmr2_3p3>;
@@ -743,7 +747,7 @@
 		compatible = "parade,ps8830";
 		reg = <0x8>;
 
-		clocks = <&rpmhcc RPMH_RF_CLK4>;
+		clocks = <&clk_rtmr0_xo>;
 
 		vdd-supply = <&vreg_rtmr0_1p15>;
 		vdd33-supply = <&vreg_rtmr0_3p3>;
@@ -848,7 +852,7 @@
 		compatible = "parade,ps8830";
 		reg = <0x8>;
 
-		clocks = <&rpmhcc RPMH_RF_CLK4>;
+		clocks = <&clk_rtmr1_xo>;
 
 		vdd-supply = <&vreg_rtmr1_1p15>;
 		vdd33-supply = <&vreg_rtmr1_3p3>;
@@ -1012,21 +1016,10 @@
 &pcie4_port0 {
 	reset-gpios = <&tlmm 146 GPIO_ACTIVE_LOW>;
 	wake-gpios = <&tlmm 148 GPIO_ACTIVE_LOW>;
+};
 
-	wifi@0 {
-		compatible = "pci17cb,1107";
-		reg = <0x10000 0x0 0x0 0x0 0x0>;
-
-		vddaon-supply = <&vreg_pmu_aon_0p59>;
-		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
-		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
-		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
-		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
-		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
-		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
-		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
-	};
+&pcie4_port0_ep {
+	remote-endpoint = <&m2_e_pcie_ep>;
 };
 
 &pcie5 {
@@ -1325,6 +1318,38 @@
 	};
 };
 
+&thermal_gpuss_0 {
+	trips {
+		trip-point0 {
+			temperature = <105000>;
+		};
+	};
+};
+
+&thermal_gpuss_1 {
+	trips {
+		trip-point0 {
+			temperature = <105000>;
+		};
+	};
+};
+
+&thermal_gpuss_2 {
+	trips {
+		trip-point0 {
+			temperature = <105000>;
+		};
+	};
+};
+
+&thermal_gpuss_3 {
+	trips {
+		trip-point0 {
+			temperature = <105000>;
+		};
+	};
+};
+
 &tlmm {
 	edp_reg_en: edp-reg-en-state {
 		pins = "gpio70";
@@ -1499,19 +1524,10 @@
 
 &uart14 {
 	status = "okay";
+};
 
-	bluetooth {
-		compatible = "qcom,wcn7850-bt";
-		max-speed = <3200000>;
-
-		vddaon-supply = <&vreg_pmu_aon_0p59>;
-		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
-		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
-		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
-		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
-		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
-	};
+&uart14_ep {
+	remote-endpoint = <&m2_e_uart_ep>;
 };
 
 &uart21 {
diff --git a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
index 488e98413c3a54..4ef3a5407ce8e1 100644
--- a/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/purwa-iot-som.dtsi
@@ -476,6 +476,27 @@
 	status = "okay";
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>; /* TPM LP & INT */
 
diff --git a/arch/arm64/boot/dts/qcom/purwa.dtsi b/arch/arm64/boot/dts/qcom/purwa.dtsi
index c698e6cb2543ca..4348dd3d1dc590 100644
--- a/arch/arm64/boot/dts/qcom/purwa.dtsi
+++ b/arch/arm64/boot/dts/qcom/purwa.dtsi
@@ -224,6 +224,11 @@
 	compatible = "qcom,x1p42100-qmp-gen4x4-pcie-phy";
 };
 
+/* X1P42100 PDC is same as X1E80100, but without hardware register bug */
+&pdc {
+	compatible = "qcom,x1p42100-pdc", "qcom,pdc";
+};
+
 &qfprom {
 	gpu_speed_bin: gpu-speed-bin@119 {
 		reg = <0x119 0x2>;
diff --git a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts
index f41ad446a39e23..2758f1f57ac009 100644
--- a/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts
+++ b/arch/arm64/boot/dts/qcom/qcm6490-fairphone-fp5.dts
@@ -13,6 +13,7 @@
 #include <dt-bindings/iio/qcom,spmi-adc7-pmk8350.h>
 #include <dt-bindings/leds/common.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 #include <dt-bindings/regulator/qcom,rpmh-regulator.h>
 #include <dt-bindings/sound/qcom,q6asm.h>
@@ -196,6 +197,17 @@
 		pinctrl-names = "default";
 	};
 
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
 	vreg_vtof_ldo_2p8: regulator-vtof-ldo-2p8 {
 		compatible = "regulator-fixed";
 		regulator-name = "VTOF_LDO_2P8";
@@ -372,6 +384,18 @@
 &apps_rsc {
 	regulators-0 {
 		compatible = "qcom,pm7325-rpmh-regulators";
+
+		vdd-l1-l4-l12-l15-supply = <&vreg_s7b>;
+		vdd-l2-l7-supply = <&vreg_bob>;
+		vdd-l6-l9-l10-supply = <&vreg_s8b>;
+		vdd-l8-supply = <&vreg_s7b>;
+		vdd-l11-l17-l18-l19-supply = <&vreg_s1b>;
+		vdd-l13-supply = <&vreg_s7b>;
+		vdd-l14-l16-supply = <&vreg_s8b>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+
 		qcom,pmic-id = "b";
 
 		vreg_s1b: smps1 {
@@ -380,6 +404,12 @@
 			regulator-max-microvolt = <2040000>;
 		};
 
+		/*
+		 * s2b (smps2) is ebi.lvl
+		 * s3b-s4b-s5b (smps3-smps4-smps5) is VDD_APC1
+		 * s6b (smps6) is VDD_APC0
+		 */
+
 		vreg_s7b: smps7 {
 			regulator-name = "vreg_s7b";
 			regulator-min-microvolt = <535000>;
@@ -414,6 +444,11 @@
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 		};
 
+		/*
+		 * l4b (ldo4) is lmx.lvl
+		 * l5b (ldo5) is lcx.lvl
+		 */
+
 		vreg_l6b: ldo6 {
 			regulator-name = "vreg_l6b";
 			regulator-min-microvolt = <1140000>;
@@ -443,6 +478,8 @@
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 		};
 
+		/* l10b (ldo10) is PMK7325 VDD_ACTIVE */
+
 		vreg_l11b: ldo11 {
 			regulator-name = "vreg_l11b";
 			regulator-min-microvolt = <1504000>;
@@ -509,6 +546,16 @@
 
 	regulators-1 {
 		compatible = "qcom,pm8350c-rpmh-regulators";
+
+		vdd-bob-supply = <&vph_pwr>;
+		vdd-l1-l12-supply = <&vreg_s1b>;
+		vdd-l2-l8-supply = <&vreg_s1b>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob>;
+		vdd-l6-l9-l11-supply = <&vreg_bob>;
+		vdd-l10-supply = <&vreg_s7b>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+
 		qcom,pmic-id = "c";
 
 		vreg_s1c: smps1 {
@@ -518,6 +565,12 @@
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 		};
 
+		/*
+		 * s2c-s3c-s4c (smps2-smps3-smps4) is cx.lvl
+		 * s5c-s6c (smps5-smps6) is mss.lvl
+		 * s7c-s8c (smps7-smps8) is gfx.lvl
+		 */
+
 		vreg_s9c: smps9 {
 			regulator-name = "vreg_s9c";
 			regulator-min-microvolt = <1010000>;
@@ -525,6 +578,8 @@
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
 		};
 
+		/* s10c (smps10) is mx.lvl */
+
 		vreg_l1c: ldo1 {
 			regulator-name = "vreg_l1c";
 			regulator-min-microvolt = <1800000>;
@@ -701,7 +756,7 @@
 		pinctrl-0 = <&cam_mclk3_default>;
 		pinctrl-names = "default";
 
-		orientation = <0>; /* Front facing */
+		orientation = <MEDIA_ORIENTATION_FRONT>;
 		rotation = <270>;
 
 		port {
diff --git a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts
index 58cce89a0c1ce8..af7821e02fdbf6 100644
--- a/arch/arm64/boot/dts/qcom/qcm6490-idp.dts
+++ b/arch/arm64/boot/dts/qcom/qcm6490-idp.dts
@@ -619,6 +619,12 @@
 	firmware-name = "qcom/qcm6490/a660_zap.mbn";
 };
 
+&ipa {
+	firmware-name = "qcom/qcm6490/ipa_fws.mbn";
+
+	status = "okay";
+};
+
 &lpass_rx_macro {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts
index 5bb33a8a983047..509fc4941a0296 100644
--- a/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts
+++ b/arch/arm64/boot/dts/qcom/qcm6490-particle-tachyon.dts
@@ -558,7 +558,7 @@
 
 &pcie0_port {
 	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
 };
 
 &pcie1 {
@@ -579,6 +579,7 @@
 
 &pcie1_port0 {
 	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>;
 };
 
 &pmk8350_adc_tm {
diff --git a/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi b/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi
index a22b4501ce1ef5..55bd53f8c15995 100644
--- a/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi
+++ b/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi
@@ -101,8 +101,6 @@
 &pcie {
 	status = "okay";
 
-	perst-gpios = <&tlmm 43 GPIO_ACTIVE_LOW>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&perst_state>;
 };
@@ -114,6 +112,10 @@
 	vdda-vph-supply = <&vreg_l5_1p8>;
 };
 
+&pcie_port0 {
+	reset-gpios = <&tlmm 43 GPIO_ACTIVE_LOW>;
+};
+
 &remoteproc_adsp {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi
index 736accfc34f904..7b622660c327a0 100644
--- a/arch/arm64/boot/dts/qcom/qcs404.dtsi
+++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi
@@ -1517,16 +1517,15 @@
 				      "pwr",
 				      "ahb";
 
-			phys = <&pcie_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
-			pcie@0 {
+			pcie_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
diff --git a/arch/arm64/boot/dts/qcom/qcs615-ride.dts b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
index 047b4efb04b948..142dafb04324bf 100644
--- a/arch/arm64/boot/dts/qcom/qcs615-ride.dts
+++ b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
@@ -444,9 +444,6 @@
 };
 
 &pcie {
-	perst-gpios = <&tlmm 101 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie_default_state>;
 	pinctrl-names = "default";
 
@@ -461,6 +458,8 @@
 };
 
 &pcie_port0 {
+	reset-gpios = <&tlmm 101 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 100 GPIO_ACTIVE_LOW>;
 	wifi@0 {
 		compatible = "pci17cb,1103";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts
index 41d60c87f5bda1..ed0b81710b9a99 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-radxa-dragon-q6a.dts
@@ -541,7 +541,7 @@
 
 &pcie0_port {
 	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
 };
 
 &pcie1 {
@@ -571,7 +571,7 @@
 
 &pcie1_port0 {
 	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>;
 };
 
 &pm7325_gpios {
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso
index 46bb52c667a4d7..e8780646ab19c5 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-industrial-mezzanine.dtso
@@ -33,7 +33,7 @@
 };
 
 &remoteproc_wpss {
-       status = "disabled";
+	status = "disabled";
 };
 
 &spi11 {
@@ -78,6 +78,8 @@
 	#address-cells = <3>;
 	#size-cells = <2>;
 
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
+
 	pcie@0,0 {
 		compatible = "pci1179,0623";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -130,20 +132,12 @@
 			ranges;
 			bus-range = <0x5 0xff>;
 
-			pcie@0,0 {
+			dev@0,0 {
 				reg = <0x50000 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 
-			pcie@0,1 {
+			dev@0,1 {
 				reg = <0x50100 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 		};
 
@@ -225,20 +219,12 @@
 			ranges;
 			bus-range = <0x5 0xff>;
 
-			pcie@0,0 {
+			dev@0,0 {
 				reg = <0x50000 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 
-			pcie@0,1 {
+			dev@0,1 {
 				reg = <0x50100 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 		};
 	};
@@ -286,5 +272,5 @@
 };
 
 &wifi {
-       status = "disabled";
+	status = "disabled";
 };
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-vision-mezzanine.dtso b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-vision-mezzanine.dtso
index b9e4a5214f7074..aceafea45343c5 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-vision-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2-vision-mezzanine.dtso
@@ -51,8 +51,8 @@
 
 		reset-gpios = <&tlmm 78 GPIO_ACTIVE_LOW>;
 		pinctrl-names = "default", "suspend";
-		pinctrl-0 = <&cam2_default>;
-		pinctrl-1 = <&cam2_suspend>;
+		pinctrl-0 = <&cam_mclk3_default>;
+		pinctrl-1 = <&cam_mclk3_suspend>;
 
 		clocks = <&camcc CAM_CC_MCLK3_CLK>;
 		assigned-clocks = <&camcc CAM_CC_MCLK3_CLK>;
@@ -73,14 +73,7 @@
 };
 
 &tlmm {
-	cam2_default: cam2-default-state {
-		pins = "gpio67";
-		function = "cam_mclk";
-		drive-strength = <2>;
-		bias-disable;
-	};
-
-	cam2_suspend: cam2-suspend-state {
+	cam_mclk3_suspend: cam-mclk3-suspend-state {
 		pins = "gpio67";
 		function = "cam_mclk";
 		drive-strength = <2>;
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
index a13315bf0fb078..ed9ba2e5550e91 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
@@ -867,7 +867,7 @@
 &pcie1_port0 {
 	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
 
-	pcie@0,0 {
+	tc9563: pcie@0,0 {
 		compatible = "pci1179,0623";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
 		#address-cells = <3>;
@@ -891,6 +891,9 @@
 		pinctrl-0 = <&tc9563_resx_n>;
 		pinctrl-names = "default";
 
+		gpio-controller;
+		#gpio-cells = <2>;
+
 		pcie1_switch0_dsp1: pcie@1,0 {
 			reg = <0x20800 0x0 0x0 0x0 0x0>;
 			#address-cells = <3>;
@@ -899,6 +902,7 @@
 			device_type = "pci";
 			ranges;
 			bus-range = <0x3 0xff>;
+			reset-gpios = <&tc9563 2 GPIO_ACTIVE_LOW>;
 		};
 
 		pcie@2,0 {
@@ -909,6 +913,7 @@
 			device_type = "pci";
 			ranges;
 			bus-range = <0x4 0xff>;
+			reset-gpios = <&tc9563 3 GPIO_ACTIVE_LOW>;
 
 			/* Renesas μPD720201 PCIe USB3.0 Host Controller */
 			usb-controller@0,0 {
@@ -954,20 +959,12 @@
 			ranges;
 			bus-range = <0x5 0xff>;
 
-			pcie@0,0 {
+			dev@0,0 {
 				reg = <0x50000 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 
-			pcie@0,1 {
+			dev@0,1 {
 				reg = <0x50100 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 		};
 	};
@@ -1105,6 +1102,16 @@
 	status = "okay";
 };
 
+&qup_spi3_cs {
+	drive-strength = <8>;
+	bias-disable;
+};
+
+&qup_spi3_data_clk {
+	drive-strength = <8>;
+	bias-disable;
+};
+
 &qup_uart7_cts {
 	/*
 	 * Configure a bias-bus-hold on CTS to lower power
@@ -1233,7 +1240,7 @@
 		reg = <0>;
 		interrupts-extended = <&tlmm 7 IRQ_TYPE_LEVEL_LOW>;
 		clocks = <&mcp2518fd_osc>;
-		spi-max-frequency = <14000000>;
+		spi-max-frequency = <17000000>;
 		vdd-supply = <&vreg_l11c_2p8>;
 		microchip,xstbyen;
 	};
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
index 31bafa2cf51db1..cfcb22bfb2989e 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-minipc-g1iot.dts
@@ -705,7 +705,7 @@
 
 &pcie0_port {
 	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
 };
 
 &pcie1 {
@@ -736,7 +736,7 @@
 
 &pcie1_port0 {
 	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>;
 
 	pcie@0,0 {
 		compatible = "pci1179,0623";
@@ -790,20 +790,12 @@
 			ranges;
 			bus-range = <0x5 0xff>;
 
-			pcie@0,0 {
+			dev@0,0 {
 				reg = <0x50000 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 
-			pcie@0,1 {
+			dev@0,1 {
 				reg = <0x50100 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam1-imx219.dtso b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam1-imx219.dtso
new file mode 100644
index 00000000000000..b6d7a823572073
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam1-imx219.dtso
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026, Thundercomm All rights reserved.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
+
+&{/} {
+	cam1_imx219_clk: clock-camera1-24m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+	};
+
+	leds-camera1 {
+		compatible = "gpio-leds";
+
+		pinctrl-0 = <&cam1_privacy_led_state>;
+		pinctrl-names = "default";
+
+		cam1_privacy_led: led-camera1 {
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <1>;
+			gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+	/* Camera module power domain controlled through connector PWR_EN. */
+	vreg_cam1_pwr: regulator-camera1-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vreg_camera1_pwr";
+
+		gpio = <&tlmm 57 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam1_pwr_en>;
+		pinctrl-names = "default";
+
+		vin-supply = <&vreg_vcc3v3_output>;
+	};
+};
+
+&camss {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			csiphy1_ep: endpoint {
+				data-lanes = <0 1>;
+				remote-endpoint = <&cam1_imx219_ep>;
+			};
+		};
+	};
+};
+
+&cci1 {
+	status = "okay";
+};
+
+&cci1_i2c0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	camera@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+
+		clocks = <&cam1_imx219_clk>;
+
+		VANA-supply = <&vreg_cam1_pwr>;
+		VDIG-supply = <&vreg_cam1_pwr>;
+		VDDL-supply = <&vreg_cam1_pwr>;
+
+		leds = <&cam1_privacy_led>;
+		led-names = "privacy";
+
+		port {
+			cam1_imx219_ep: endpoint {
+				data-lanes = <1 2>;
+				link-frequencies = /bits/ 64 <456000000>;
+				remote-endpoint = <&csiphy1_ep>;
+			};
+		};
+	};
+};
+
+&tlmm {
+	cam1_privacy_led_state: cam1-privacy-led-state {
+		pins = "gpio18";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	cam1_pwr_en: cam1-pwr-en-state {
+		pins = "gpio57";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam2-imx219.dtso b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam2-imx219.dtso
new file mode 100644
index 00000000000000..ff1ccf965df8f3
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3-cam2-imx219.dtso
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026, Thundercomm All rights reserved.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
+
+&{/} {
+	cam2_imx219_clk: clock-camera2-24m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+	};
+
+	leds-camera2 {
+		compatible = "gpio-leds";
+
+		pinctrl-0 = <&cam2_privacy_led_state>;
+		pinctrl-names = "default";
+
+		cam2_privacy_led: led-camera2 {
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <2>;
+			gpios = <&tlmm 19 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+	/* Camera module power domain controlled through connector PWR_EN. */
+	vreg_cam2_pwr: regulator-camera2-pwr {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vreg_camera2_pwr";
+
+		gpio = <&tlmm 58 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam2_pwr_en>;
+		pinctrl-names = "default";
+
+		vin-supply = <&vreg_vcc3v3_output>;
+	};
+};
+
+&camss {
+	vdda-phy-supply = <&vreg_l10c_0p88>;
+	vdda-pll-supply = <&vreg_l6b_1p2>;
+
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@4 {
+			reg = <4>;
+
+			csiphy4_ep: endpoint {
+				data-lanes = <0 1>;
+				remote-endpoint = <&cam2_imx219_ep>;
+			};
+		};
+	};
+};
+
+&cci0 {
+	status = "okay";
+};
+
+&cci0_i2c0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	camera@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+
+		clocks = <&cam2_imx219_clk>;
+
+		VANA-supply = <&vreg_cam2_pwr>;
+		VDIG-supply = <&vreg_cam2_pwr>;
+		VDDL-supply = <&vreg_cam2_pwr>;
+
+		leds = <&cam2_privacy_led>;
+		led-names = "privacy";
+
+		port {
+			cam2_imx219_ep: endpoint {
+				data-lanes = <1 2>;
+				link-frequencies = /bits/ 64 <456000000>;
+				remote-endpoint = <&csiphy4_ep>;
+			};
+		};
+	};
+};
+
+&tlmm {
+	cam2_privacy_led_state: cam2-privacy-led-state {
+		pins = "gpio19";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	cam2_pwr_en: cam2-pwr-en-state {
+		pins = "gpio58";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
index 13e8e62f6205a4..4ab7771fce1b09 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts
@@ -214,6 +214,22 @@
 		regulator-boot-on;
 	};
 
+	vreg_vcc3v3_output: regulator-vcc3v3-output {
+		compatible = "regulator-fixed";
+
+		regulator-name = "vcc3v3_output";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&vcc3v3_output_en>;
+		pinctrl-names = "default";
+
+		regulator-always-on;
+	};
+
 	vph_pwr: regulator-vph-pwr {
 		compatible = "regulator-fixed";
 
@@ -833,7 +849,7 @@
 
 &pcie0_port {
 	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
 };
 
 &pcie1 {
@@ -1251,6 +1267,13 @@
 		bias-disable;
 	};
 
+	vcc3v3_output_en: vcc3v3-output-en-state {
+		pins = "gpio14";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
 	wifi_reset_active: wifi-reset-active-state {
 		pins = "gpio16";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts b/arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts
index 4f2e0109f49bfe..496b7583e4a285 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-vicharak-axon-mini.dts
@@ -652,9 +652,6 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 89 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_clkreq_n>, <&pcie0_reset_n>, <&pcie0_wake_n>;
 	pinctrl-names = "default";
 
@@ -668,10 +665,12 @@
 	status = "okay";
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 3 GPIO_ACTIVE_HIGH>;
+&pcie0_port {
+	reset-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
+};
 
+&pcie1 {
 	pinctrl-0 = <&pcie1_reset_n>, <&pcie1_wake_n>, <&pcie1_clkreq_n>;
 	pinctrl-names = "default";
 
@@ -685,6 +684,11 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 3 GPIO_ACTIVE_LOW>;
+};
+
 &pm7250b_adc {
 	channel@4d {
 		reg = <ADC5_AMUX_THM1_100K_PU>;
diff --git a/arch/arm64/boot/dts/qcom/qcs8300-ride.dts b/arch/arm64/boot/dts/qcom/qcs8300-ride.dts
index 1ff39530ea3dee..be850a23472fc7 100644
--- a/arch/arm64/boot/dts/qcom/qcs8300-ride.dts
+++ b/arch/arm64/boot/dts/qcom/qcs8300-ride.dts
@@ -615,7 +615,7 @@
 
 &pcieport0 {
 	reset-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 0 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 0 GPIO_ACTIVE_LOW>;
 
 	wifi@0 {
 		compatible = "pci17cb,1103";
@@ -651,7 +651,7 @@
 
 &pcieport1 {
 	reset-gpios = <&tlmm 23 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 21 GPIO_ACTIVE_LOW>;
 };
 
 &pcie1_phy {
@@ -702,7 +702,9 @@
 };
 
 &serdes0 {
-	phy-supply = <&vreg_l5a>;
+	vdda-0p9-supply = <&vreg_l4a>;
+	vdda-1p2-supply = <&vreg_l5a>;
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi
index ae301d78e0710a..4faedcca3c5b50 100644
--- a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi
+++ b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi
@@ -335,9 +335,6 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 };
@@ -347,10 +344,12 @@
 	vdda-pll-supply = <&vreg_l3e_1p2>;
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
+&pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+};
 
+&pcie1 {
 	pinctrl-0 = <&pcie1_default_state>;
 	pinctrl-names = "default";
 };
@@ -361,6 +360,11 @@
 	vdda-qref-supply = <&vreg_l1e_0p88>;
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+};
+
 &pm8550b_eusb2_repeater {
 	vdd18-supply = <&vreg_l15b_1p8>;
 	vdd3-supply = <&vreg_l5b_3p1>;
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-ayntec-common.dtsi b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-common.dtsi
new file mode 100644
index 00000000000000..d567708f1024b5
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-common.dtsi
@@ -0,0 +1,1819 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2025, Teguh Sobirin.
+ */
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "qcs8550.dtsi"
+#include "pm8550.dtsi"
+#include "pm8550b.dtsi"
+#define PMK8550VE_SID 5
+#include "pm8550ve.dtsi"
+#include "pm8550vs.dtsi"
+#include "pmk8550.dtsi"
+
+/delete-node/ &aop_image_mem;
+/delete-node/ &aop_config_mem;
+/delete-node/ &camera_mem;
+/delete-node/ &ipa;
+/delete-node/ &ipa_fw_mem;
+/delete-node/ &ipa_gsi_mem;
+/delete-node/ &mpss_dsm_mem;
+/delete-node/ &mpss_mem;
+/delete-node/ &q6_mpss_dtb_mem;
+/delete-node/ &remoteproc_mpss;
+
+/ {
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart7;
+		serial1 = &uart14;
+	};
+
+	/* The tzlog label is required by ABL to apply a dtbo, but it can be on any node */
+	qcom_tzlog: chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio_keys: gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&volume_up_n>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			debounce-interval = <15>;
+			gpios = <&pm8550_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+			linux,can-disable;
+			wakeup-source;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,sm8550-pmic-glink", "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		orientation-gpios = <&tlmm 11 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_ss_in: endpoint {
+						remote-endpoint = <&usb_dp_qmpphy_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu: endpoint {
+						remote-endpoint = <&usb0_sbu_mux>;
+					};
+				};
+			};
+		};
+	};
+
+	pwm_fan: pwm-fan {
+		compatible = "pwm-fan";
+
+		fan-supply = <&vdd_fan_5v0>;
+		pwms = <&pm8550_pwm 3 50000>;
+
+		pinctrl-0 = <&fan_pwm_active>, <&fan_int>;
+		pinctrl-names = "default";
+
+		pulses-per-revolution = <4>;
+		interrupts-extended = <&tlmm 13 IRQ_TYPE_EDGE_FALLING>;
+
+		cooling-levels = <0 40 65 75 90 100 120 150 175>;
+		#cooling-cells = <2>;
+	};
+
+	reserved-memory {
+		hyp_mem: hyp-region@80000000 {
+			reg = <0 0x80000000 0 0xa00000>;
+			no-map;
+		};
+
+		cpusys_vm_mem: cpusys-vm-region@80a00000 {
+			reg = <0 0x80a00000 0 0x400000>;
+			no-map;
+		};
+
+		hyp_tags_mem: hyp-tags-region@80e00000 {
+			reg = <0 0x80e00000 0 0x3d0000>;
+			no-map;
+		};
+
+		xbl_sc_mem: xbl-sc-region@d8100000 {
+			reg = <0 0xd8100000 0 0x40000>;
+			no-map;
+		};
+
+		hyp_tags_reserved_mem: hyp-tags-reserved-region@811d0000 {
+			reg = <0 0x811d0000 0 0x30000>;
+			no-map;
+		};
+
+		xbl_dt_log_merged_mem: xbl-dt-log-merged-region@81a00000 {
+			reg = <0 0x81a00000 0 0x260000>;
+			no-map;
+		};
+
+		aop_config_merged_mem: aop-config-merged-region@81c80000 {
+			reg = <0 0x81c80000 0 0x74000>;
+			no-map;
+		};
+
+		chipinfo_mem: chipinfo-region@81cf4000 {
+			reg = <0 0x81cf4000 0 0x1000>;
+			no-map;
+		};
+
+		global_sync_mem: global-sync-region@82600000 {
+			reg = <0 0x82600000 0 0x100000>;
+			no-map;
+		};
+
+		tz_stat_mem: tz-stat-region@82700000 {
+			reg = <0 0x82700000 0 0x100000>;
+			no-map;
+		};
+
+		cpucp_fw_mem: cpucp-fw-region@d8140000 {
+			reg = <0 0xd8140000 0 0x1c0000>;
+			no-map;
+		};
+
+		qtee_mem: qtee-region@d8300000 {
+			reg = <0 0xd8300000 0 0x500000>;
+			no-map;
+		};
+
+		hwfence_shbuf: hwfence-shbuf-region@e6440000 {
+			reg = <0 0xe6440000 0 0x2dd000>;
+			no-map;
+		};
+
+		hyp_ext_reserved_mem: hyp-ext-reserved-region@ff700000 {
+			reg = <0 0xff700000 0 0x100000>;
+			no-map;
+		};
+
+		llcc_lpi_mem: llcc_lpi_region@ff800000 {
+			reg = <0 0xff800000 0 0x600000>;
+			no-map;
+		};
+
+		hyp_ext_tags_mem: hyp-ext-tags-region@fce00000 {
+			reg = <0 0xfce00000 0 0x2900000>;
+			no-map;
+		};
+	};
+
+	sound {
+		compatible = "qcom,sm8550-sndcard", "qcom,sm8450-sndcard";
+		pinctrl-0 = <&i2s0_default_state>, <&audio_mclk0_default_state>, <&lpi_i2s3_active>;
+		pinctrl-names = "default";
+
+		model = "AYN-Odin2";
+		audio-routing = "IN1_HPHL", "HPHL_OUT",
+				"IN2_HPHR", "HPHR_OUT",
+				"AMIC2", "MIC BIAS2",
+				"TX SWR_INPUT1", "ADC2_OUTPUT";
+
+		speaker-i2s-dai-link {
+			link-name = "Primary MI2S Playback";
+
+			codec {
+				sound-dai = <&spk_amp_l>, <&spk_amp_r>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai PRIMARY_MI2S_RX>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wcd-playback-dai-link {
+			link-name = "WCD Playback";
+
+			codec {
+				sound-dai = <&wcd938x 0>, <&swr1 0>, <&lpass_rxmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai RX_CODEC_DMA_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		wcd-capture-dai-link {
+			link-name = "WCD Capture";
+
+			codec {
+				sound-dai = <&wcd938x 1>, <&swr2 0>, <&lpass_txmacro 0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai TX_CODEC_DMA_TX_3>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		dp0-dai-link {
+			link-name = "DP0 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
+	thermal-zones {
+		cpu0-thermal {
+			polling-delay = <200>;
+
+			trips {
+				cpuss0_active0: cpu-active0 {
+					temperature = <50000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss0_active1: cpu-active1 {
+					temperature = <55000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss0_active2: cpu-active2 {
+					temperature = <60000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss0_active3: cpu-active3 {
+					temperature = <65000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss0_active4: cpu-active4 {
+					temperature = <70000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss0_active5: cpu-active5 {
+					temperature = <75000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss0_active6: cpu-active6 {
+					temperature = <80000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss0_active7: cpu-active7 {
+					temperature = <85000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					cooling-device = <&pwm_fan 0 1>;
+					trip = <&cpuss0_active0>;
+				};
+
+				map1 {
+					cooling-device = <&pwm_fan 1 2>;
+					trip = <&cpuss0_active1>;
+				};
+
+				map2 {
+					cooling-device = <&pwm_fan 2 3>;
+					trip = <&cpuss0_active2>;
+				};
+
+				map3 {
+					cooling-device = <&pwm_fan 3 4>;
+					trip = <&cpuss0_active3>;
+				};
+
+				map4 {
+					cooling-device = <&pwm_fan 4 5>;
+					trip = <&cpuss0_active4>;
+				};
+
+				map5 {
+					cooling-device = <&pwm_fan 5 6>;
+					trip = <&cpuss0_active5>;
+				};
+
+				map6 {
+					cooling-device = <&pwm_fan 6 7>;
+					trip = <&cpuss0_active6>;
+				};
+
+				map7 {
+					cooling-device = <&pwm_fan 7 8>;
+					trip = <&cpuss0_active7>;
+				};
+			};
+		};
+
+		cpu3-bottom-thermal {
+			polling-delay = <200>;
+
+			trips {
+				cpuss3_active0: cpu-active0 {
+					temperature = <50000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss3_active1: cpu-active1 {
+					temperature = <55000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss3_active2: cpu-active2 {
+					temperature = <60000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss3_active3: cpu-active3 {
+					temperature = <65000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss3_active4: cpu-active4 {
+					temperature = <70000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss3_active5: cpu-active5 {
+					temperature = <75000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss3_active6: cpu-active6 {
+					temperature = <80000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss3_active7: cpu-active7 {
+					temperature = <85000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					cooling-device = <&pwm_fan 0 1>;
+					trip = <&cpuss3_active0>;
+				};
+
+				map1 {
+					cooling-device = <&pwm_fan 1 2>;
+					trip = <&cpuss3_active1>;
+				};
+
+				map2 {
+					cooling-device = <&pwm_fan 2 3>;
+					trip = <&cpuss3_active2>;
+				};
+
+				map3 {
+					cooling-device = <&pwm_fan 3 4>;
+					trip = <&cpuss3_active3>;
+				};
+
+				map4 {
+					cooling-device = <&pwm_fan 4 5>;
+					trip = <&cpuss3_active4>;
+				};
+
+				map5 {
+					cooling-device = <&pwm_fan 5 6>;
+					trip = <&cpuss3_active5>;
+				};
+
+				map6 {
+					cooling-device = <&pwm_fan 6 7>;
+					trip = <&cpuss3_active6>;
+				};
+
+				map7 {
+					cooling-device = <&pwm_fan 7 8>;
+					trip = <&cpuss3_active7>;
+				};
+			};
+		};
+
+		cpu7-bottom-thermal {
+			polling-delay = <200>;
+
+			trips {
+				cpuss7_active0: cpu-active0 {
+					temperature = <50000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss7_active1: cpu-active1 {
+					temperature = <55000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss7_active2: cpu-active2 {
+					temperature = <60000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss7_active3: cpu-active3 {
+					temperature = <65000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss7_active4: cpu-active4 {
+					temperature = <70000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss7_active5: cpu-active5 {
+					temperature = <75000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss7_active6: cpu-active6 {
+					temperature = <80000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				cpuss7_active7: cpu-active7 {
+					temperature = <85000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					cooling-device = <&pwm_fan 0 1>;
+					trip = <&cpuss7_active0>;
+				};
+
+				map1 {
+					cooling-device = <&pwm_fan 1 2>;
+					trip = <&cpuss7_active1>;
+				};
+
+				map2 {
+					cooling-device = <&pwm_fan 2 3>;
+					trip = <&cpuss7_active2>;
+				};
+
+				map3 {
+					cooling-device = <&pwm_fan 3 4>;
+					trip = <&cpuss7_active3>;
+				};
+
+				map4 {
+					cooling-device = <&pwm_fan 4 5>;
+					trip = <&cpuss7_active4>;
+				};
+
+				map5 {
+					cooling-device = <&pwm_fan 5 6>;
+					trip = <&cpuss7_active5>;
+				};
+
+				map6 {
+					cooling-device = <&pwm_fan 6 7>;
+					trip = <&cpuss7_active6>;
+				};
+
+				map7 {
+					cooling-device = <&pwm_fan 7 8>;
+					trip = <&cpuss7_active7>;
+				};
+			};
+		};
+
+		gpuss-0-thermal {
+			polling-delay = <200>;
+
+			trips {
+				gpuss0_active0: gpu-active0 {
+					temperature = <50000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				gpuss0_active1: gpu-active1 {
+					temperature = <55000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				gpuss0_active2: gpu-active2 {
+					temperature = <60000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				gpuss0_active3: gpu-active3 {
+					temperature = <65000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				gpuss0_active4: gpu-active4 {
+					temperature = <70000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				gpuss0_active5: gpu-active5 {
+					temperature = <75000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				gpuss0_active6: gpu-active6 {
+					temperature = <80000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+
+				gpuss0_active7: gpu-active7 {
+					temperature = <85000>;
+					hysteresis = <4000>;
+					type = "active";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					cooling-device = <&pwm_fan 0 1>;
+					trip = <&gpuss0_active0>;
+				};
+
+				map1 {
+					cooling-device = <&pwm_fan 1 2>;
+					trip = <&gpuss0_active1>;
+				};
+
+				map2 {
+					cooling-device = <&pwm_fan 2 3>;
+					trip = <&gpuss0_active2>;
+				};
+
+				map3 {
+					cooling-device = <&pwm_fan 3 4>;
+					trip = <&gpuss0_active3>;
+				};
+
+				map4 {
+					cooling-device = <&pwm_fan 4 5>;
+					trip = <&gpuss0_active4>;
+				};
+
+				map5 {
+					cooling-device = <&pwm_fan 5 6>;
+					trip = <&gpuss0_active5>;
+				};
+
+				map6 {
+					cooling-device = <&pwm_fan 6 7>;
+					trip = <&gpuss0_active6>;
+				};
+
+				map7 {
+					cooling-device = <&pwm_fan 7 8>;
+					trip = <&gpuss0_active7>;
+				};
+			};
+		};
+	};
+
+	usb0-sbu-mux {
+		compatible = "pericom,pi3usb102", "gpio-sbu-mux";
+
+		enable-gpios = <&tlmm 140 GPIO_ACTIVE_LOW>;
+		select-gpios = <&tlmm 141 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&usb0_sbu_default>;
+		pinctrl-names = "default";
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			usb0_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_sbu>;
+			};
+		};
+	};
+
+	vdd_fan_5v0: vdd-fan-5v0-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_fan_5v0";
+
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&tlmm 109 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&fan_pwr_active>;
+		pinctrl-names = "default";
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vdd_mcu_3v3: vdd-mcu-3v3-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_mcu_3v3";
+
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&tlmm 99 GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+		regulator-boot-on;
+		enable-active-high;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	wcd938x: audio-codec {
+		compatible = "qcom,wcd9385-codec";
+
+		pinctrl-0 = <&wcd_default>;
+		pinctrl-names = "default";
+
+		qcom,micbias1-microvolt = <1800000>;
+		qcom,micbias2-microvolt = <1800000>;
+		qcom,micbias3-microvolt = <1800000>;
+		qcom,micbias4-microvolt = <1800000>;
+		qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000
+							 500000 500000 500000 500000>;
+		qcom,mbhc-headset-vthreshold-microvolt = <1700000>;
+		qcom,mbhc-headphone-vthreshold-microvolt = <50000>;
+		qcom,rx-device = <&wcd_rx>;
+		qcom,tx-device = <&wcd_tx>;
+
+		reset-gpios = <&tlmm 108 GPIO_ACTIVE_LOW>;
+
+		vdd-buck-supply = <&vreg_l15b_1p8>;
+		vdd-rxtx-supply = <&vreg_l15b_1p8>;
+		vdd-io-supply = <&vreg_l15b_1p8>;
+		vdd-mic-bias-supply = <&vreg_bob1>;
+
+		#sound-dai-cells = <1>;
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		pinctrl-0 = <&wlan_en>, <&bt_default>, <&pmk8550_sleep_clk>;
+		pinctrl-names = "default";
+
+		wlan-enable-gpios = <&tlmm 80 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
+
+		vdd-supply = <&vreg_s5g_0p8>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddaon-supply = <&vreg_s2g_0p8>;
+		vdddig-supply = <&vreg_s4e_0p95>;
+		vddrfa1p2-supply = <&vreg_s4g_1p3>;
+		vddrfa1p8-supply = <&vreg_s6g_1p8>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+
+	/* The arch_timer label is unused here, but is required by ABL to apply a dtbo */
+	arch_timer: timer { };
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s6g_1p8>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l3-supply = <&vreg_s4g_1p3>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob1>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l11-supply = <&vreg_s4g_1p3>;
+		vdd-l12-supply = <&vreg_s6g_1p8>;
+		vdd-l15-supply = <&vreg_s6g_1p8>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3296000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2720000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b_3p1: ldo5 {
+			regulator-name = "vreg_l5b_3p1";
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6b_1p8: ldo6 {
+			regulator-name = "vreg_l6b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_1p8: ldo7 {
+			regulator-name = "vreg_l7b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_1p8: ldo8 {
+			regulator-name = "vreg_l8b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p2: ldo11 {
+			regulator-name = "vreg_l11b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_1p8: ldo12 {
+			regulator-name = "vreg_l12b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_3p0: ldo13 {
+			regulator-name = "vreg_l13b_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_3p2: ldo14 {
+			regulator-name = "vreg_l14b_3p2";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_2p8: ldo16 {
+			regulator-name = "vreg_l16b_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s4g_1p3>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+
+		vreg_l3c_0p9: ldo3 {
+			regulator-name = "vreg_l3c_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+
+		vreg_l1d_0p88: ldo1 {
+			regulator-name = "vreg_l1d_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4g_1p3>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+
+		vreg_s4e_0p95: smps4 {
+			regulator-name = "vreg_s4e_0p95";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <984000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5e_1p08: smps5 {
+			regulator-name = "vreg_s5e_1p08";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1120000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1e_0p88: ldo1 {
+			regulator-name = "vreg_l1e_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2e_0p9: ldo2 {
+			regulator-name = "vreg_l2e_0p9";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "f";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+		vdd-s4-supply = <&vph_pwr>;
+
+		vreg_s4f_0p5: smps4 {
+			regulator-name = "vreg_s4f_0p5";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <700000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_0p9: ldo1 {
+			regulator-name = "vreg_l1f_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_0p88: ldo2 {
+			regulator-name = "vreg_l2f_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_0p88: ldo3 {
+			regulator-name = "vreg_l3f_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-5 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "g";
+
+		vdd-l1-supply = <&vreg_s4g_1p3>;
+		vdd-l2-supply = <&vreg_s4g_1p3>;
+		vdd-l3-supply = <&vreg_s4g_1p3>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		vreg_s1g_1p2: smps1 {
+			regulator-name = "vreg_s1g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2g_0p8: smps2 {
+			regulator-name = "vreg_s2g_0p8";
+			regulator-min-microvolt = <800000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3g_0p7: smps3 {
+			regulator-name = "vreg_s3g_0p7";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1004000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4g_1p3: smps4 {
+			regulator-name = "vreg_s4g_1p3";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1352000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5g_0p8: smps5 {
+			regulator-name = "vreg_s5g_0p8";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1004000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s6g_1p8: smps6 {
+			regulator-name = "vreg_s6g_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1g_1p2: ldo1 {
+			regulator-name = "vreg_l1g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3g_1p2: ldo3 {
+			regulator-name = "vreg_l3g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpi_dma2 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm8550/a740_zap.mbn";
+};
+
+&i2c_master_hub_0 {
+	status = "okay";
+};
+
+&i2c_hub_2 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	spk_amp_l: amplifier@34 {
+		compatible = "awinic,aw88166";
+		reg = <0x34>;
+		reset-gpios = <&tlmm 103 GPIO_ACTIVE_LOW>;
+		sound-name-prefix = "Amplifier L";
+
+		awinic,audio-channel = <0>;
+		awinic,sync-flag;
+
+		#sound-dai-cells = <0>;
+	};
+
+	spk_amp_r: amplifier@35 {
+		compatible = "awinic,aw88166";
+		reg = <0x35>;
+		reset-gpios = <&tlmm 100 GPIO_ACTIVE_LOW>;
+		sound-name-prefix = "Amplifier R";
+
+		awinic,audio-channel = <1>;
+		awinic,sync-flag;
+
+		#sound-dai-cells = <0>;
+	};
+};
+
+&iris {
+	status = "okay";
+};
+
+&lpass_tlmm {
+	lpi_i2s3_active: lpi_i2s3-active-state {
+		sck-pins {
+			pins = "gpio12";
+			function = "i2s3_clk";
+			drive-strength = <8>;
+			bias-disable;
+			output-high;
+		};
+
+		ws-pins {
+			pins = "gpio13";
+			function = "i2s3_ws";
+			drive-strength = <8>;
+			bias-disable;
+			output-high;
+		};
+
+		data0-pins {
+			pins = "gpio17";
+			function = "i2s3_data";
+			drive-strength = <8>;
+			bias-disable;
+			output-high;
+		};
+
+		data1-pins {
+			pins = "gpio18";
+			function = "i2s3_data";
+			drive-strength = <8>;
+			bias-disable;
+			output-high;
+		};
+	};
+};
+
+&lpass_vamacro {
+	qcom,dmic-sample-rate = <4800000>;
+};
+
+&lpass_wsamacro {
+	status = "disabled";
+};
+
+&mdss {
+	status = "okay";
+};
+
+&mdss_dp0 {
+	status = "okay";
+};
+
+&mdss_dsi1 {
+	vdda-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+
+	display_panel: panel@0 {
+		reg = <0>;
+
+		port {
+			panel1_in: endpoint {
+				remote-endpoint = <&mdss_dsi1_out>;
+			};
+		};
+	};
+};
+
+&mdss_dsi1_out {
+	remote-endpoint = <&panel1_in>;
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi1_phy {
+	vdds-supply = <&vreg_l1e_0p88>;
+
+	status = "okay";
+};
+
+&pcie0 {
+	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
+
+	pinctrl-0 = <&pcie0_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie0_port0 {
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0x0 0x0 0x0 0x0>;
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1e_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pm8550_gpios {
+	fan_pwm_active: fan-pwm-active-state {
+		pins = "gpio8";
+		function = "func1";
+		input-disable;
+		output-enable;
+		output-low;
+		bias-disable;
+		power-source = <1>;
+	};
+
+	sdc2_card_det_n: sdc2-card-det-n-state {
+		pins = "gpio12";
+		function = "normal";
+		input-enable;
+		output-disable;
+		bias-pull-up;
+		power-source = <1>;
+	};
+
+	volume_up_n: volume-up-n-state {
+		pins = "gpio6";
+		function = "normal";
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+};
+
+&pm8550_pwm {
+	status = "okay";
+
+	pm8550_multi_led: multi-led {
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_STATUS;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_RED>;
+		};
+
+		led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_GREEN>;
+		};
+
+		led@3 {
+			reg = <3>;
+			color = <LED_COLOR_ID_BLUE>;
+		};
+	};
+};
+
+&pm8550b_eusb2_repeater {
+	qcom,tune-usb2-disc-thres = /bits/ 8 <0x6>;
+	qcom,tune-usb2-amplitude = /bits/ 8 <0xb>;
+	qcom,tune-usb2-preem = /bits/ 8 <0x3>;
+	vdd18-supply = <&vreg_l15b_1p8>;
+	vdd3-supply = <&vreg_l5b_3p1>;
+};
+
+&pmk8550_gpios {
+	pmk8550_sleep_clk: sleep-clk-state {
+		pins = "gpio3";
+		function = "func1";
+		input-disable;
+		output-enable;
+		bias-disable;
+		power-source = <0>;
+	};
+
+	pwm_backlight_default: pwm-backlight-default-state {
+		pins = "gpio5";
+		function = "func3";
+		input-disable;
+		output-low;
+		output-enable;
+		bias-disable;
+		power-source = <0>;
+		qcom,drive-strength = <2>;
+	};
+};
+
+&pmk8550_rtc {
+	nvmem-cells = <&rtc_offset>;
+	nvmem-cell-names = "offset";
+};
+
+&pmk8550_sdam_2 {
+	rtc_offset: rtc-offset@bc {
+		reg = <0xbc 0x4>;
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&q6apmbedai {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	dai@16 {
+		reg = <PRIMARY_MI2S_RX>;
+		clocks = <&q6prmcc LPASS_CLK_ID_PRI_MI2S_IBIT
+				   LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+			 <&q6prmcc LPASS_CLK_ID_MCLK_1
+				   LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+		clock-names = "bclk", "mclk";
+	};
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/sm8550/ayntec/cdsp.mbn",
+			"qcom/sm8550/ayntec/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&sdhc_2 {
+	cd-gpios = <&pm8550_gpios 12 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&sdc2_default &sdc2_card_det_n>;
+	pinctrl-1 = <&sdc2_sleep &sdc2_card_det_n>;
+	pinctrl-names = "default", "sleep";
+	vmmc-supply = <&vreg_l9b_2p9>;
+	vqmmc-supply = <&vreg_l8b_1p8>;
+	no-sdio;
+	no-mmc;
+
+	status = "okay";
+};
+
+&sleep_clk {
+	clock-frequency = <32764>;
+};
+
+&swr1 {
+	status = "okay";
+
+	wcd_rx: codec@0,4 {
+		compatible = "sdw20217010d00";
+		reg = <0 4>;
+		qcom,rx-port-mapping = <1 2 3 4 5>;
+	};
+};
+
+&swr2 {
+	status = "okay";
+
+	wcd_tx: codec@0,3 {
+		compatible = "sdw20217010d00";
+		reg = <0 3>;
+		qcom,tx-port-mapping = <2 2 3 4>;
+	};
+};
+
+&tlmm {
+	gpio-reserved-ranges = <32 8>;
+
+	mcu_en_active: mcu-en-active-state {
+		pins = "gpio12";
+		function = "gpio";
+		bias-pull-down;
+	};
+
+	fan_int: fan-int-state {
+		pins = "gpio13";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	ts_s_rst_default: ts-s-rst-default-state {
+		pins = "gpio14";
+		function = "gpio";
+		bias-pull-up;
+		drive-strength = <8>;
+	};
+
+	ts_s_rst_sleep: ts-s-rst-sleep-state {
+		pins = "gpio14";
+		function = "gpio";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
+
+	ts_s_int_default: ts-s-int-default-state {
+		pins = "gpio15";
+		function = "gpio";
+		bias-pull-up;
+		drive-strength = <8>;
+	};
+
+	ts_s_int_sleep: ts-s-int-sleep-state {
+		pins = "gpio15";
+		function = "gpio";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
+
+	ts_p_rst_default: ts-p-rst-default-state {
+		pins = "gpio24";
+		function = "gpio";
+		bias-pull-up;
+		drive-strength = <8>;
+	};
+
+	ts_p_rst_sleep: ts-p-rst-sleep-state {
+		pins = "gpio24";
+		function = "gpio";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
+
+	ts_p_int_default: ts-p-int-default-state {
+		pins = "gpio25";
+		function = "gpio";
+		bias-pull-up;
+		drive-strength = <8>;
+	};
+
+	ts_p_int_sleep: ts-p-int-sleep-state {
+		pins = "gpio25";
+		function = "gpio";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
+
+	wlan_en: wlan-en-state {
+		pins = "gpio80";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-down;
+	};
+
+	bt_default: bt-default-state {
+		bt-en-pins {
+			pins = "gpio81";
+			function = "gpio";
+			drive-strength = <16>;
+			bias-disable;
+		};
+
+		sw-ctrl-pins {
+			pins = "gpio82";
+			function = "gpio";
+			bias-pull-down;
+		};
+	};
+
+	dsi_p_te_active: dsi-p-te-active-state {
+		pins = "gpio86";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	dsi_p_te_suspend: dsi-s-te-suspend-state {
+		pins = "gpio86";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	dsi_s_te_active: dsi-s-te-active-state {
+		pins = "gpio87";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	dsi_s_te_suspend: dsi-s-te-suspend-state {
+		pins = "gpio87";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	wcd_default: wcd-reset-n-active-state {
+		pins = "gpio108";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	fan_pwr_active: fan-pwr-active-state {
+		pins = "gpio109";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	audio_mclk0_default_state: audio-mclk0-default-state {
+		pins = "gpio125";
+		function = "audio_ext_mclk0";
+		drive-strength = <8>;
+		bias-disable;
+		output-high;
+	};
+
+	i2s0_default_state: i2s0-default-state {
+		sck-pins {
+			pins = "gpio126";
+			function = "i2s0_sck";
+			drive-strength = <8>;
+			bias-disable;
+			output-high;
+		};
+
+		data0-pins {
+			pins = "gpio127";
+			function = "i2s0_data0";
+			drive-strength = <8>;
+			bias-disable;
+		};
+
+		data1-pins {
+			pins = "gpio128";
+			function = "i2s0_data1";
+			drive-strength = <8>;
+			bias-disable;
+		};
+
+		ws-pins {
+			pins = "gpio129";
+			function = "i2s0_ws";
+			drive-strength = <8>;
+			bias-disable;
+			output-high;
+		};
+	};
+
+	dsi_p_rst_active: dsi-p-rst-active-state {
+		pins = "gpio133";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	dsi_p_rst_suspend: dsi-p-rst-suspend-state {
+		pins = "gpio133";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	dsi_s_rst_active: dsi-s-rst-active-state {
+		pins = "gpio137";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	dsi_s_rst_suspend: dsi-s-rst-suspend-state {
+		pins = "gpio137";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	usb0_sbu_default: usb0-sbu-state {
+		oe-n-pins {
+			pins = "gpio140";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <8>;
+			output-high;
+		};
+
+		sel-pins {
+			pins = "gpio141";
+			function = "gpio";
+			bias-disable;
+			drive-strength = <8>;
+		};
+	};
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+
+		max-speed = <3200000>;
+	};
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 210 GPIO_ACTIVE_LOW>;
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <1300000>;
+	vccq-supply = <&vreg_l1g_1p2>;
+	vccq-max-microamp = <1200000>;
+	vdd-hba-supply = <&vreg_l3g_1p2>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l1d_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	phys = <&pm8550b_eusb2_repeater>;
+
+	vdd-supply = <&vreg_l1e_0p88>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3f_0p88>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy_out {
+	remote-endpoint = <&pmic_glink_ss_in>;
+};
+
+&xo_board {
+	clock-frequency = <76800000>;
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2mini.dts b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2mini.dts
new file mode 100644
index 00000000000000..cc894bdd0c9020
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2mini.dts
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2025, Teguh Sobirin.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "qcs8550-ayntec-common.dtsi"
+
+&{/} {
+	model = "AYN Odin 2 Mini";
+	compatible = "ayntec,odin2mini", "qcom,qcs8550", "qcom,sm8550";
+
+	vdd_disp_2v8: vdd-disp-2v8-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_disp_2v8";
+
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+
+		gpio = <&tlmm 142 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+/* There is an hdmi bridge on i2c_hub_0@48, but it is not currently supported */
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm8550/ayntec/odin2mini/adsp.mbn",
+			"qcom/sm8550/ayntec/odin2mini/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&spk_amp_l {
+	firmware-name = "qcom/sm8550/ayntec/odin2mini/aw883xx_acf.bin";
+};
+
+&spk_amp_r {
+	firmware-name = "qcom/sm8550/ayntec/odin2mini/aw883xx_acf.bin";
+};
+
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2portal.dts b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2portal.dts
new file mode 100644
index 00000000000000..2c6e63f47cceb2
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-odin2portal.dts
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2025, Teguh Sobirin.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "qcs8550-ayntec-common.dtsi"
+
+&{/} {
+	model = "AYN Odin 2 Portal";
+	compatible = "ayntec,odin2portal", "qcom,qcs8550", "qcom,sm8550";
+
+	vdd_bl_5v0: vdd-bl-5v0-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_bl_5v0";
+
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&tlmm 52 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vdd_disp_2v8: vdd-disp-2v8-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_disp_2v8";
+
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+
+		gpio = <&tlmm 142 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+&display_panel {
+	compatible = "ayntec,odin2portal-panel", "chipone,icna3512";
+
+	pinctrl-0 = <&dsi_p_rst_active>, <&dsi_p_te_active>;
+	pinctrl-1 = <&dsi_p_rst_suspend>, <&dsi_p_te_suspend>;
+	pinctrl-names = "default", "sleep";
+
+	vdd-supply = <&vreg_l11b_1p2>;
+	vddio-supply = <&vreg_l12b_1p8>;
+	vci-supply = <&vreg_l13b_3p0>;
+	disp-supply = <&vdd_disp_2v8>;
+	blvdd-supply = <&vdd_bl_5v0>;
+
+	reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+
+	rotation = <270>;
+};
+
+&i2c4 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "focaltech,ft5426";
+		reg = <0x38>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <25 IRQ_TYPE_EDGE_FALLING>;
+
+		reset-gpios = <&tlmm 24 GPIO_ACTIVE_LOW>;
+
+		vcc-supply = <&vreg_l14b_3p2>;
+		iovcc-supply = <&vreg_l12b_1p8>;
+
+		pinctrl-0 = <&ts_p_rst_default &ts_p_int_default>;
+		pinctrl-1 = <&ts_p_rst_sleep &ts_p_int_sleep>;
+		pinctrl-names = "default", "sleep";
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <1920>;
+		touchscreen-swapped-x-y;
+		touchscreen-inverted-y;
+	};
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm8550/ayntec/odin2portal/adsp.mbn",
+			"qcom/sm8550/ayntec/odin2portal/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&spk_amp_l {
+	firmware-name = "qcom/sm8550/ayntec/odin2portal/aw883xx_acf.bin";
+};
+
+&spk_amp_r {
+	firmware-name = "qcom/sm8550/ayntec/odin2portal/aw883xx_acf.bin";
+};
+
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-ayntec-thor.dts b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-thor.dts
new file mode 100644
index 00000000000000..7683911914be3b
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs8550-ayntec-thor.dts
@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2025, Teguh Sobirin.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "qcs8550-ayntec-common.dtsi"
+
+&{/} {
+	model = "AYN Thor";
+	compatible = "ayntec,thor", "qcom,qcs8550", "qcom,sm8550";
+
+	vdd_bl_5v0: vdd-bl-5v0-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_bl_5v0";
+
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&tlmm 52 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vdd_disp_1v8: vdd-disp-1v8-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_disp_1v8";
+
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&tlmm 70 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vdd_disp1_2v8: vdd-disp1-2v8-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_disp1_2v8";
+
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+
+		gpio = <&tlmm 142 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vdd_disp2_2v8: vdd-disp2-2v8-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_disp2_2v8";
+
+		regulator-min-microvolt = <2800000>;
+		regulator-max-microvolt = <2800000>;
+
+		gpio = <&tlmm 143 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vdd_ts_3v0: vdd-ts-3v0-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_ts_3v0";
+
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+
+		gpio = <&tlmm 144 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vdd_ts_1v8: vdd-ts-1v8-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_ts_1v8";
+
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		gpio = <&tlmm 102 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+&display_panel {
+	compatible = "ayntec,thor-panel-top", "chipone,icna3520";
+
+	pinctrl-0 = <&dsi_s_rst_active>, <&dsi_s_te_active>;
+	pinctrl-1 = <&dsi_s_rst_suspend>, <&dsi_s_te_suspend>;
+	pinctrl-names = "default", "sleep";
+
+	vdd-supply = <&vreg_l11b_1p2>;
+	vddio-supply = <&vreg_l12b_1p8>;
+	vci-supply = <&vreg_l14b_3p2>;
+	disp-supply = <&vdd_disp1_2v8>;
+	blvdd-supply = <&vdd_bl_5v0>;
+
+	reset-gpios = <&tlmm 137 GPIO_ACTIVE_LOW>;
+
+	rotation = <90>;
+};
+
+&gpio_keys {
+	pinctrl-0 = <&volume_up_n &key_ayn_n>;
+
+	key-ayn {
+		label = "AYN Key";
+		debounce-interval = <15>;
+		gpios = <&tlmm 41 GPIO_ACTIVE_LOW>;
+		linux,code = <KEY_F24>;
+		linux,can-disable;
+	};
+
+	switch-lid {
+		label = "Hall Lid Sensor";
+		gpios = <&tlmm 17 GPIO_ACTIVE_LOW>;
+		linux,input-type = <EV_SW>;
+		linux,code = <SW_LID>;
+		linux,can-disable;
+		wakeup-source;
+	};
+};
+
+&i2c4 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "focaltech,ft5426";
+		reg = <0x38>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <25 IRQ_TYPE_EDGE_FALLING>;
+
+		reset-gpios = <&tlmm 24 GPIO_ACTIVE_LOW>;
+
+		vcc-supply = <&vreg_l14b_3p2>;
+		iovcc-supply = <&vreg_l12b_1p8>;
+
+		pinctrl-0 = <&ts_p_rst_default &ts_p_int_default>;
+		pinctrl-1 = <&ts_p_rst_sleep &ts_p_int_sleep>;
+		pinctrl-names = "default", "sleep";
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <1920>;
+		touchscreen-swapped-x-y;
+		touchscreen-inverted-x;
+	};
+};
+
+&i2c_hub_3 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "focaltech,ft5452";
+		reg = <0x38>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <15 IRQ_TYPE_EDGE_FALLING>;
+
+		reset-gpios = <&tlmm 14 GPIO_ACTIVE_LOW>;
+
+		vcc-supply = <&vdd_ts_3v0>;
+		iovcc-supply = <&vdd_ts_1v8>;
+
+		pinctrl-0 = <&ts_s_rst_default &ts_s_int_default>;
+		pinctrl-1 = <&ts_s_rst_sleep &ts_s_int_sleep>;
+		pinctrl-names = "default", "sleep";
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <1240>;
+		touchscreen-swapped-x-y;
+		touchscreen-inverted-x;
+	};
+};
+
+&mdss_dsi0 {
+	vdda-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+
+	panel@0 {
+		reg = <0>;
+		compatible = "ayntec,thor-panel-bottom", "chipwealth,ch13726a";
+
+		pinctrl-0 = <&dsi_p_rst_active>, <&dsi_p_te_active>;
+		pinctrl-1 = <&dsi_p_rst_suspend>, <&dsi_p_te_suspend>;
+		pinctrl-names = "default", "sleep";
+
+		vdd1v2-supply = <&vreg_l11b_1p2>;
+		vddio-supply = <&vdd_disp_1v8>;
+		vdd-supply = <&vreg_l13b_3p0>;
+		avdd-supply = <&vdd_disp2_2v8>;
+
+		reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+
+		rotation = <90>;
+
+		port {
+			panel0_in: endpoint {
+				remote-endpoint = <&mdss_dsi0_out>;
+			};
+		};
+	};
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&panel0_in>;
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi0_phy {
+	vdds-supply = <&vreg_l1e_0p88>;
+
+	status = "okay";
+};
+
+&mdss_dsi1_out {
+	qcom,te-source = "mdp_vsync_s";
+};
+
+&pm8550_multi_led {
+	status = "disabled";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm8550/ayntec/thor/adsp.mbn",
+			"qcom/sm8550/ayntec/thor/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&spk_amp_l {
+	firmware-name = "qcom/sm8550/ayntec/thor/aw883xx_acf.bin";
+};
+
+&spk_amp_r {
+	firmware-name = "qcom/sm8550/ayntec/thor/aw883xx_acf.bin";
+};
+
+&tlmm {
+	key_ayn_n: key-ayn-n-state {
+		pins = "gpio41";
+		function = "gpio";
+		bias-pull-up;
+		output-disable;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-imdt-sbc.dts b/arch/arm64/boot/dts/qcom/qcs8550-imdt-sbc.dts
new file mode 100644
index 00000000000000..fa8794868cf8ec
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs8550-imdt-sbc.dts
@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 IMD Technologies Ltd
+ */
+
+/dts-v1/;
+
+#include "qcs8550-imdt-som.dtsi"
+
+/ {
+	model = "IMDT QCS8550 SBC";
+	compatible = "imdt,qcs8550-sbc", "imdt,qcs8550-som",
+		     "qcom,qcs8550", "qcom,sm8550";
+	chassis-type = "embedded";
+
+	aliases {
+		serial0 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	connector {
+		compatible = "pcie-m2-e-connector";
+		vpcie3v3-supply = <&m2e_3p3>;
+		vpcie1v8-supply = <&per_pwr>;
+		w-disable1-gpios = <&tlmm 80 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&m2e_wlan_disable>;
+		pinctrl-names = "default";
+
+		/* SDIO datapath is disabled because the SDIO lines are
+		 * routed through unpopulated resistors.
+		 * UART datapath is also disabled as the UART lines are
+		 * routed through an analogue mux which currently cannot
+		 * be described by a device tree.
+		 */
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				m2e_pcie_ep: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&pcie0_port0_ep>;
+				};
+			};
+		};
+	};
+
+	m2e_3p3: regulator-m2e-3p3 {
+		compatible = "regulator-fixed";
+		regulator-name = "m2e_3p3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		pinctrl-0 = <&m2_pd_en_default>;
+		pinctrl-names = "default";
+
+		gpio = <&tlmm 144 GPIO_ACTIVE_LOW>;
+	};
+
+	/* Enables 5V_PER, 3V3_PER and 1V8_PER rails. 5V_PER and 3V3_PER
+	 * aren't used within the DTS but are used for on board logic
+	 * level conversion and as rails for pull-ups. 1V8_PER is used
+	 * for M2 Key E.
+	 */
+	per_pwr: regulator-per-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "per_pwr";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+
+		pinctrl-0 = <&pwr_per_en_default>;
+		pinctrl-names = "default";
+
+		gpio = <&tlmm 142 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	sdhci_2_vqmmc: regulator-sdhci-2-vqmmc {
+		compatible = "regulator-gpio";
+
+		regulator-name = "sdhci_2_vqmmc";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-type = "voltage";
+
+		pinctrl-0 = <&sd_vset_default>;
+		pinctrl-names = "default";
+
+		gpios = <&tlmm 4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		gpios-states = <0>;
+		states = <3300000 0>,
+			 <1800000 1>;
+
+		startup-delay-us = <10000>;
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	usb_id: usb-id {
+		compatible = "linux,extcon-usb-gpio";
+		id-gpios = <&tlmm 53 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&usb_id_default>;
+		pinctrl-names = "default";
+	};
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm8550/a740_zap.mbn";
+};
+
+&i2c_hub_2 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	ptn3222: redriver@43 {
+		compatible = "nxp,ptn3222";
+		reg = <0x43>;
+		#phy-cells = <0>;
+
+		vdd1v8-supply = <&vreg_l15b_1p8>;
+		vdd3v3-supply = <&vreg_l5b_3p1>;
+
+		pinctrl-0 = <&eusb2_repeater_reset_default>;
+		pinctrl-names = "default";
+
+		reset-gpios = <&pm8550vs_d_gpios 4 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&i2c_master_hub_0 {
+	status = "okay";
+};
+
+&ipa {
+	qcom,gsi-loader = "self";
+	memory-region = <&ipa_fw_mem>;
+	firmware-name = "qcom/sm8550/ipa_fws.mbn";
+
+	status = "okay";
+};
+
+&iris {
+	status = "okay";
+};
+
+&pcie0 {
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
+	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&pcie0_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1e_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&pcie0_port0 {
+	/* Required for pwrctrl driver bind */
+	compatible = "pciclass,0604";
+
+	port {
+		pcie0_port0_ep: endpoint {
+			remote-endpoint = <&m2e_pcie_ep>;
+		};
+	};
+};
+
+&pm8550_gpios {
+	sd_card_detect_default: sd-card-detect-default-state {
+		pins = "gpio12";
+		function = "normal";
+		input-enable;
+		output-disable;
+		bias-disable;
+		power-source = <1>; /* 1.8 V */
+	};
+};
+
+&pm8550_rpmh_regulators {
+	vdd-bob1-supply = <&vph_pwr>;
+	vdd-bob2-supply = <&vph_pwr>;
+};
+
+&pm8550ve_rpmh_regulators {
+	vdd-s4-supply = <&vph_pwr>;
+};
+
+&pm8550vs_d_gpios {
+	eusb2_repeater_reset_default: eusb2-repeater-reset-default-state {
+		pins = "gpio4";
+		function = "normal";
+
+		input-enable;
+		output-enable;
+		bias-disable;
+
+		drive-push-pull;
+		power-source = <1>; /* 1.8V */
+		qcom,drive-strength = <3>;
+	};
+};
+
+&pm8550vs_e_rpmh_regulators {
+	vdd-s4-supply = <&vph_pwr>;
+	vdd-s5-supply = <&vph_pwr>;
+};
+
+&pm8550vs_g_rpmh_regulators {
+	vdd-s1-supply = <&vph_pwr>;
+	vdd-s2-supply = <&vph_pwr>;
+	vdd-s3-supply = <&vph_pwr>;
+	vdd-s4-supply = <&vph_pwr>;
+	vdd-s5-supply = <&vph_pwr>;
+	vdd-s6-supply = <&vph_pwr>;
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm8550/adsp.mbn",
+			"qcom/sm8550/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/sm8550/cdsp.mbn",
+			"qcom/sm8550/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&sdc2_default {
+	clk-pins {
+		drive-strength = <16>;
+	};
+
+	cmd-pins {
+		/delete-property/ bias-pull-up;
+		bias-disable;
+		drive-strength = <16>;
+	};
+
+	data-pins {
+		/delete-property/ bias-pull-up;
+		bias-disable;
+		drive-strength = <16>;
+	};
+};
+
+&sdhc_2 {
+	cd-gpios = <&pm8550_gpios 12 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&sdc2_default>, <&sd_card_detect_default>;
+	pinctrl-1 = <&sdc2_default>, <&sd_card_detect_default>;
+	pinctrl-names = "default", "sleep";
+
+	vqmmc-supply = <&sdhci_2_vqmmc>;
+
+	bus-width = <4>;
+	no-sdio;
+	no-mmc;
+
+	status = "okay";
+};
+
+&tlmm {
+	/* Reserved I/Os for NFC */
+	gpio-reserved-ranges = <32 8>;
+
+	m2_pd_en_default: m2-pd-en-default-state {
+		pins = "gpio144";
+		function = "gpio";
+		drive-strength = <16>;
+		drive-open-drain;
+	};
+
+	m2e_wlan_disable: m2e-wlan-disable-default-state {
+		pins = "gpio80";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	pwr_per_en_default: pwr-per-en-default-state {
+		pins = "gpio142";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	sd_vset_default: sd-vset-default-state {
+		pins = "gpio4";
+		function = "gpio";
+		drive-strength = <16>;
+		bias-disable;
+	};
+
+	usb_id_default: usb-id-default-state {
+		pins = "gpio53";
+		function = "gpio";
+		bias-pull-up;
+	};
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 210 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <1300000>;
+	vccq-supply = <&vreg_l1g_1p2>;
+	vccq-max-microamp = <1200000>;
+	vdd-hba-supply = <&vreg_l3g_1p2>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l1d_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	dr_mode = "otg";
+	extcon = <&usb_id>;
+
+	status = "okay";
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l1e_0p88>;
+	vdda12-supply = <&vreg_l3e_1p2>;
+
+	phys = <&ptn3222>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy {
+	vdda-phy-supply = <&vreg_l3e_1p2>;
+	vdda-pll-supply = <&vreg_l3f_0p88>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-imdt-som.dtsi b/arch/arm64/boot/dts/qcom/qcs8550-imdt-som.dtsi
new file mode 100644
index 00000000000000..3202632eef046f
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/qcs8550-imdt-som.dtsi
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026 IMD Technologies Ltd
+ */
+
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "qcs8550.dtsi"
+#include "pm8550.dtsi"
+#define PMK8550VE_SID 5
+#include "pm8550ve.dtsi"
+#include "pm8550vs.dtsi"
+#include "pmk8550.dtsi"
+
+&apps_rsc {
+	pm8550_rpmh_regulators: regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-l1-l4-l10-supply = <&vreg_s6g_1p86>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l3-supply = <&vreg_s4g_1p25>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l6-l7-supply = <&vreg_bob1>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l11-supply = <&vreg_s4g_1p25>;
+		vdd-l12-supply = <&vreg_s6g_1p86>;
+		vdd-l15-supply = <&vreg_s6g_1p86>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		/*
+		 * PMIC outputs whose testpoints lack capacitors are
+		 * omitted to prevent accidental use:
+		 *	L1B_1P8, L4B_1P8, L6B_1P8, L7B_1P8, L8B_1P8,
+		 *	L13B_3P0, L14B_3P2.
+		 * Other outputs are unused.
+		 */
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3296000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2720000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b_3p1: ldo5 {
+			regulator-name = "vreg_l5b_3p1";
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	pm8550vs_c_rpmh_regulators: regulators-1 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-l1-supply = <&vreg_s4g_1p25>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+
+		vreg_l3c_0p9: ldo3 {
+			regulator-name = "vreg_l3c_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	pm8550vs_d_rpmh_regulators: regulators-2 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "d";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+
+		vreg_l1d_0p88: ldo1 {
+			regulator-name = "vreg_l1d_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		/* ldo2 supplies SM8550 VDD_LPI_MX */
+	};
+
+	pm8550vs_e_rpmh_regulators: regulators-3 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4g_1p25>;
+
+		vreg_s4e_0p95: smps4 {
+			regulator-name = "vreg_s4e_0p95";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <984000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5e_1p08: smps5 {
+			regulator-name = "vreg_s5e_1p08";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1120000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1e_0p88: ldo1 {
+			regulator-name = "vreg_l1e_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2e_0p9: ldo2 {
+			regulator-name = "vreg_l2e_0p9";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3e_1p2: ldo3 {
+			regulator-name = "vreg_l3e_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	pm8550ve_rpmh_regulators: regulators-4 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+		qcom,pmic-id = "f";
+
+		vdd-l1-supply = <&vreg_s4e_0p95>;
+		vdd-l2-supply = <&vreg_s4e_0p95>;
+		vdd-l3-supply = <&vreg_s4e_0p95>;
+
+		vreg_s4f_0p5: smps4 {
+			regulator-name = "vreg_s4f_0p5";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <700000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1f_0p9: ldo1 {
+			regulator-name = "vreg_l1f_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f_0p88: ldo2 {
+			regulator-name = "vreg_l2f_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f_0p88: ldo3 {
+			regulator-name = "vreg_l3f_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	pm8550vs_g_rpmh_regulators: regulators-5 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+		qcom,pmic-id = "g";
+
+		vdd-l1-supply = <&vreg_s4g_1p25>;
+		vdd-l2-supply = <&vreg_s4g_1p25>;
+		vdd-l3-supply = <&vreg_s4g_1p25>;
+
+		vreg_s1g_1p25: smps1 {
+			regulator-name = "vreg_s1g_1p25";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2g_0p85: smps2 {
+			regulator-name = "vreg_s2g_0p85";
+			regulator-min-microvolt = <800000>;
+			regulator-max-microvolt = <1000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3g_0p8: smps3 {
+			regulator-name = "vreg_s3g_0p8";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1004000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4g_1p25: smps4 {
+			regulator-name = "vreg_s4g_1p25";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1352000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5g_0p85: smps5 {
+			regulator-name = "vreg_s5g_0p85";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1004000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s6g_1p86: smps6 {
+			regulator-name = "vreg_s6g_1p86";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1g_1p2: ldo1 {
+			regulator-name = "vreg_l1g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3g_1p2: ldo3 {
+			regulator-name = "vreg_l3g_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&pm8550vs_c {
+	status = "okay";
+};
+
+&pm8550vs_d {
+	status = "okay";
+};
+
+&pm8550vs_e {
+	status = "okay";
+};
+
+&pm8550vs_g {
+	status = "okay";
+};
+
+&sleep_clk {
+	clock-frequency = <32764>;
+};
+
+&xo_board {
+	clock-frequency = <76800000>;
+};
diff --git a/arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts b/arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts
index 9df8dcd6c62b68..b7ee1573cf671c 100644
--- a/arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts
+++ b/arch/arm64/boot/dts/qcom/qcs8550-rb5gen2.dts
@@ -1045,9 +1045,6 @@
 &pcie0 {
 	vddpe-3v3-supply = <&pcie_upd_3p3>;
 
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -1074,6 +1071,9 @@
 };
 
 &pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+
 	pcie@0,0 {
 		compatible = "pci1179,0623";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -1126,29 +1126,18 @@
 			ranges;
 			bus-range = <0x5 0xff>;
 
-			pci@0,0 {
+			dev@0,0 {
 				reg = <0x50000 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 
-			pci@0,1 {
+			dev@0,1 {
 				reg = <0x50100 0x0 0x0 0x0 0x0>;
-				#address-cells = <3>;
-				#size-cells = <2>;
-				device_type = "pci";
-				ranges;
 			};
 		};
 	};
 };
 
 &pcie1 {
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie1_default_state>;
 	pinctrl-names = "default";
 
@@ -1176,6 +1165,9 @@
 };
 
 &pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+
 	pcie@0,0 {
 		compatible = "pci1179,0623";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/qdu1000.dtsi b/arch/arm64/boot/dts/qcom/qdu1000.dtsi
index af106aebd30dca..3a25b51008363c 100644
--- a/arch/arm64/boot/dts/qcom/qdu1000.dtsi
+++ b/arch/arm64/boot/dts/qcom/qdu1000.dtsi
@@ -412,7 +412,7 @@
 			#reset-cells = <1>;
 		};
 
-		gpi_dma0: dma-controller@900000  {
+		gpi_dma0: dma-controller@900000 {
 			compatible = "qcom,qdu1000-gpi-dma", "qcom,sm6350-gpi-dma";
 			reg = <0x0 0x900000 0x0 0x60000>;
 			interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>,
@@ -628,7 +628,7 @@
 			};
 		};
 
-		gpi_dma1: dma-controller@a00000  {
+		gpi_dma1: dma-controller@a00000 {
 			compatible = "qcom,qdu1000-gpi-dma", "qcom,sm6350-gpi-dma";
 			reg = <0x0 0xa00000 0x0 0x60000>;
 			interrupts = <GIC_SPI 279 IRQ_TYPE_LEVEL_HIGH>,
@@ -1192,7 +1192,7 @@
 				function = "qup07";
 			};
 
-			qup_uart7_tx: qup-uart7-tx-state  {
+			qup_uart7_tx: qup-uart7-tx-state {
 				pins = "gpio134";
 				function = "qup07";
 			};
diff --git a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts
index c72faaa0ae7a29..e85d78ecac2eb4 100644
--- a/arch/arm64/boot/dts/qcom/sa8540p-ride.dts
+++ b/arch/arm64/boot/dts/qcom/sa8540p-ride.dts
@@ -374,7 +374,7 @@
 
 &pcie2a_port0 {
 	reset-gpios = <&tlmm 143 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 145 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 145 GPIO_ACTIVE_LOW>;
 };
 
 &pcie2a_phy {
@@ -397,7 +397,7 @@
 
 &pcie3a_port0 {
 	reset-gpios = <&tlmm 151 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 56 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 56 GPIO_ACTIVE_LOW>;
 };
 
 &pcie3a_phy {
diff --git a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts
index 80428c0b3b4e68..cc04eefdfcff91 100644
--- a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts
+++ b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts
@@ -357,9 +357,6 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 55 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 57 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -367,6 +364,9 @@
 };
 
 &pcieport0 {
+	reset-gpios = <&tlmm 55 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 57 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/sar2130p.dtsi b/arch/arm64/boot/dts/qcom/sar2130p.dtsi
index a2d93fd682997c..60d25bd13ed32b 100644
--- a/arch/arm64/boot/dts/qcom/sar2130p.dtsi
+++ b/arch/arm64/boot/dts/qcom/sar2130p.dtsi
@@ -15,7 +15,6 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
@@ -1337,9 +1336,6 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
 			pcieport0: pcie@0 {
@@ -1347,6 +1343,8 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -1464,16 +1462,15 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-homestar.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-homestar.dtsi
index 8fee8d7a7d4cdc..f6919e05131217 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-homestar.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-homestar.dtsi
@@ -184,7 +184,7 @@ ap_ts_pen_1v8: &i2c4 {
 };
 
 &sound_multimedia1_codec {
-	sound-dai = <&max98360a>, <&max98360a_1>, <&max98360a_2>, <&max98360a_3> ;
+	sound-dai = <&max98360a>, <&max98360a_1>, <&max98360a_2>, <&max98360a_3>;
 };
 
 &wifi {
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts
index 70d5a7aa887354..3390fd0aabce47 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-limozeen-nots-r4.dts
@@ -17,7 +17,7 @@
  */
 /delete-node/&trackpad;
 &ap_tp_i2c {
-	 trackpad: trackpad@2c {
+	trackpad: trackpad@2c {
 		compatible = "hid-over-i2c";
 		reg = <0x2c>;
 		pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts
index b60060a384267f..c70ad0398d89bf 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts
@@ -19,9 +19,9 @@
 
 
 &usb_hub_2_x {
-	 vdd-supply = <&pp3300_l7c>;
+	vdd-supply = <&pp3300_l7c>;
 };
 
 &usb_hub_3_x {
-	 vdd-supply = <&pp3300_l7c>;
+	vdd-supply = <&pp3300_l7c>;
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts
index ba222a60c4c665..9eea257197f88b 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts
@@ -35,9 +35,9 @@
 };
 
 &usb_hub_2_x {
-	 vdd-supply = <&pp3300_l7c>;
+	vdd-supply = <&pp3300_l7c>;
 };
 
 &usb_hub_3_x {
-	 vdd-supply = <&pp3300_l7c>;
+	vdd-supply = <&pp3300_l7c>;
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts
index d393a2712ce68d..1d9f3dbd9735b5 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts
@@ -53,11 +53,11 @@ ap_ts_pen_1v8: &i2c4 {
 };
 
 &usb_hub_2_x {
-	 vdd-supply = <&pp3300_l7c>;
+	vdd-supply = <&pp3300_l7c>;
 };
 
 &usb_hub_3_x {
-	 vdd-supply = <&pp3300_l7c>;
+	vdd-supply = <&pp3300_l7c>;
 };
 
 /* PINCTRL - modifications to sc7180-trogdor.dtsi */
diff --git a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts
index 565e5a9979b1ad..aa634a070283c7 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts
+++ b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts
@@ -491,8 +491,6 @@
 };
 
 &pcie3 {
-	perst-gpios = <&tlmm 178 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 180 GPIO_ACTIVE_HIGH>;
 	pinctrl-0 = <&pcie3_default_state>;
 	pinctrl-names = "default";
 
@@ -506,6 +504,11 @@
 	status = "okay";
 };
 
+&pcie3_port0 {
+	reset-gpios = <&tlmm 178 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 180 GPIO_ACTIVE_LOW>;
+};
+
 &pmc8180_2_gpios {
 	bl_pwr_en: bl-pwr-en-state {
 		pins = "gpio1";
diff --git a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts
index c10a936b320a99..2311c5fb44e4fd 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts
+++ b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts
@@ -592,8 +592,6 @@
 };
 
 &pcie1 {
-	perst-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 177 GPIO_ACTIVE_HIGH>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie2_default_state>;
 
@@ -607,6 +605,11 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 177 GPIO_ACTIVE_LOW>;
+};
+
 &pmc8180_pwrkey {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
index de1b72b64d960a..a85e3211958d17 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
@@ -1779,17 +1779,17 @@
 					<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_PCIE_0 0>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
 			dma-coherent;
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie0_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -1898,17 +1898,17 @@
 					<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_PCIE_3 0>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
-			phys = <&pcie3_phy>;
-			phy-names = "pciephy";
 			dma-coherent;
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie3_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie3_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2018,17 +2018,17 @@
 					<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_PCIE_1 0>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
 			dma-coherent;
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2138,17 +2138,17 @@
 					<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_PCIE_2 0>;
 			interconnect-names = "pcie-mem", "cpu-pcie";
 
-			phys = <&pcie2_phy>;
-			phy-names = "pciephy";
 			dma-coherent;
 
 			status = "disabled";
 
-			pcie@0 {
+			pcie2_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie2_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
index 1eea9d710d5b26..f4af4f2b6827d3 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
@@ -493,14 +493,17 @@
 };
 
 &gpi_dma0 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma1 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma2 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
index adce0302d92e36..43bd2e09119ed1 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
@@ -582,14 +582,17 @@
 };
 
 &gpi_dma0 {
+	dma-channel-mask = <0xb>;
 	status = "okay";
 };
 
 &gpi_dma1 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma2 {
+	dma-channel-mask = <0xb>;
 	status = "okay";
 };
 
@@ -1363,11 +1366,11 @@
 		};
 
 		wake-n-pins {
-		       pins = "gpio145";
-		       function = "gpio";
-		       drive-strength = <2>;
-		       bias-pull-up;
-	       };
+			pins = "gpio145";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
 	};
 
 	pcie4_default: pcie4-default-state {
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
index 218573a97785b0..ee0928721cdf70 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
@@ -11,6 +11,7 @@
 #include <dt-bindings/input/gpio-keys.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/regulator/qcom,rpmh-regulator.h>
 
 #include "sc8280xp.dtsi"
@@ -683,7 +684,7 @@
 
 		clocks = <&camcc CAMCC_MCLK3_CLK>;
 
-		orientation = <0>;	/* Front facing */
+		orientation = <MEDIA_ORIENTATION_FRONT>;
 
 		avdd-supply = <&vreg_l6q>;
 		dvdd-supply = <&vreg_l2q>;
@@ -705,14 +706,17 @@
 };
 
 &gpi_dma0 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma1 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma2 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
@@ -734,6 +738,7 @@
 
 &mdss0_dp0_out {
 	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
 	remote-endpoint = <&usb_0_qmpphy_dp_in>;
 };
 
@@ -743,6 +748,7 @@
 
 &mdss0_dp1_out {
 	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
 	remote-endpoint = <&usb_1_qmpphy_dp_in>;
 };
 
@@ -1213,6 +1219,38 @@
 		"VA DMIC2", "MIC BIAS3",
 		"TX SWR_ADC1", "ADC2_OUTPUT";
 
+	dp0-dai-link {
+		link-name = "DP0 Playback";
+
+		codec {
+			sound-dai = <&mdss0_dp0>;
+		};
+
+		cpu {
+			sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+		};
+
+		platform {
+			sound-dai = <&q6apm>;
+		};
+	};
+
+	dp1-dai-link {
+		link-name = "DP1 Playback";
+
+		codec {
+			sound-dai = <&mdss0_dp1>;
+		};
+
+		cpu {
+			sound-dai = <&q6apmbedai DISPLAY_PORT_RX_1>;
+		};
+
+		platform {
+			sound-dai = <&q6apm>;
+		};
+	};
+
 	wcd-playback-dai-link {
 		link-name = "WCD Playback";
 		cpu {
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts
index 89c4ba267b078c..cc50805b43bc5a 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts
@@ -564,14 +564,17 @@
 };
 
 &gpi_dma0 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma1 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma2 {
+	dma-channel-mask = <0x23>;
 	status = "okay";
 };
 
@@ -1114,7 +1117,7 @@
 			function = "gpio";
 			drive-strength = <2>;
 			bias-pull-up;
-	       };
+		};
 	};
 
 	pcie3a_default: pcie3a-default-state {
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
index 4f4b5f4285ec22..58e6c6ffe128af 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
@@ -561,14 +561,17 @@
 };
 
 &gpi_dma0 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma1 {
+	dma-channel-mask = <0x3>;
 	status = "okay";
 };
 
 &gpi_dma2 {
+	dma-channel-mask = <0x23>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
index fbcfd3883bbb4e..d17475c3e66c9c 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
@@ -885,7 +885,6 @@
 				     <GIC_SPI 599 IRQ_TYPE_LEVEL_HIGH>;
 
 			dma-channels = <12>;
-			dma-channel-mask = <0xfff>;
 			#dma-cells = <3>;
 
 			iommus = <&apps_smmu 0xb6 0x0>;
@@ -917,8 +916,8 @@
 				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 0 QCOM_GPI_I2C>,
@@ -939,8 +938,8 @@
 				interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 0 QCOM_GPI_SPI>,
@@ -961,8 +960,8 @@
 				interrupts = <GIC_SPI 583 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 1 QCOM_GPI_I2C>,
@@ -983,8 +982,8 @@
 				interrupts = <GIC_SPI 583 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 1 QCOM_GPI_SPI>,
@@ -1019,8 +1018,8 @@
 				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 2 QCOM_GPI_I2C>,
@@ -1041,8 +1040,8 @@
 				interrupts = <GIC_SPI 584 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 2 QCOM_GPI_SPI>,
@@ -1081,8 +1080,8 @@
 				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 3 QCOM_GPI_I2C>,
@@ -1103,8 +1102,8 @@
 				interrupts = <GIC_SPI 585 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 3 QCOM_GPI_SPI>,
@@ -1125,8 +1124,8 @@
 				interrupts = <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 4 QCOM_GPI_I2C>,
@@ -1147,8 +1146,8 @@
 				interrupts = <GIC_SPI 586 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 4 QCOM_GPI_SPI>,
@@ -1191,8 +1190,8 @@
 				interrupts = <GIC_SPI 587 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 5 QCOM_GPI_SPI>,
@@ -1235,8 +1234,8 @@
 				interrupts = <GIC_SPI 833 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 6 QCOM_GPI_SPI>,
@@ -1279,8 +1278,8 @@
 				interrupts = <GIC_SPI 834 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_2 0 &clk_virt SLAVE_QUP_CORE_2 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
-				                <&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_2 0>,
+						<&aggre1_noc MASTER_QUP_2 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma2 0 7 QCOM_GPI_SPI>,
@@ -1311,7 +1310,6 @@
 				     <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>;
 
 			dma-channels = <13>;
-			dma-channel-mask = <0x1fff>;
 			#dma-cells = <3>;
 
 			iommus = <&apps_smmu 0x576 0x0>;
@@ -1718,7 +1716,6 @@
 				     <GIC_SPI 298 IRQ_TYPE_LEVEL_HIGH>;
 
 			dma-channels = <12>;
-			dma-channel-mask = <0xfff>;
 			#dma-cells = <3>;
 
 			iommus = <&apps_smmu 0x96 0x0>;
@@ -1750,8 +1747,8 @@
 				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>,
@@ -1772,8 +1769,8 @@
 				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 0 QCOM_GPI_SPI>,
@@ -1794,8 +1791,8 @@
 				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>,
@@ -1816,8 +1813,8 @@
 				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 1 QCOM_GPI_SPI>,
@@ -1838,8 +1835,8 @@
 				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>,
@@ -1860,8 +1857,8 @@
 				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 2 QCOM_GPI_SPI>,
@@ -1882,8 +1879,8 @@
 				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 3 QCOM_GPI_I2C>,
@@ -1904,8 +1901,8 @@
 				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 3 QCOM_GPI_SPI>,
@@ -1926,8 +1923,8 @@
 				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>,
@@ -1948,8 +1945,8 @@
 				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 4 QCOM_GPI_SPI>,
@@ -1970,8 +1967,8 @@
 				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 5 QCOM_GPI_I2C>,
@@ -1992,8 +1989,8 @@
 				interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 5 QCOM_GPI_SPI>,
@@ -2014,8 +2011,8 @@
 				interrupts = <GIC_SPI 835 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 6 QCOM_GPI_I2C>,
@@ -2036,8 +2033,8 @@
 				interrupts = <GIC_SPI 835 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 6 QCOM_GPI_SPI>,
@@ -2058,8 +2055,8 @@
 				interrupts = <GIC_SPI 836 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 7 QCOM_GPI_I2C>,
@@ -2080,8 +2077,8 @@
 				interrupts = <GIC_SPI 836 IRQ_TYPE_LEVEL_HIGH>;
 				power-domains = <&rpmhpd SC8280XP_CX>;
 				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
-				                <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
-				                <&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
+						<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre1_noc MASTER_QUP_1 0 &mc_virt SLAVE_EBI1 0>;
 				interconnect-names = "qup-core", "qup-config", "qup-memory";
 
 				dmas = <&gpi_dma1 0 7 QCOM_GPI_SPI>,
@@ -2910,18 +2907,21 @@
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <1>;
 						iommus = <&apps_smmu 0x0521 0x0>;
+						dma-coherent;
 					};
 
 					compute-cb@2 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <2>;
 						iommus = <&apps_smmu 0x0522 0x0>;
+						dma-coherent;
 					};
 
 					compute-cb@3 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <3>;
 						iommus = <&apps_smmu 0x0523 0x0>;
+						dma-coherent;
 					};
 				};
 			};
@@ -2978,18 +2978,21 @@
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <3>;
 						iommus = <&apps_smmu 0x0c03 0x0>;
+						dma-coherent;
 					};
 
 					compute-cb@4 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <4>;
 						iommus = <&apps_smmu 0x0c04 0x0>;
+						dma-coherent;
 					};
 
 					compute-cb@5 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <5>;
 						iommus = <&apps_smmu 0x0c05 0x0>;
+						dma-coherent;
 					};
 				};
 
@@ -6235,84 +6238,98 @@
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <1>;
 						iommus = <&apps_smmu 0x3181 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@2 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <2>;
 						iommus = <&apps_smmu 0x3182 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@3 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <3>;
 						iommus = <&apps_smmu 0x3183 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@4 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <4>;
 						iommus = <&apps_smmu 0x3184 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@5 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <5>;
 						iommus = <&apps_smmu 0x3185 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@6 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <6>;
 						iommus = <&apps_smmu 0x3186 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@7 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <7>;
 						iommus = <&apps_smmu 0x3187 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@8 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <8>;
 						iommus = <&apps_smmu 0x3188 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@9 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <9>;
 						iommus = <&apps_smmu 0x318b 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@10 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <10>;
 						iommus = <&apps_smmu 0x318b 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@11 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <11>;
 						iommus = <&apps_smmu 0x318c 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@12 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <12>;
 						iommus = <&apps_smmu 0x318d 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@13 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <13>;
 						iommus = <&apps_smmu 0x318e 0x0420>;
+						dma-coherent;
 					};
 
 					compute-cb@14 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <14>;
 						iommus = <&apps_smmu 0x318f 0x0420>;
+						dma-coherent;
 					};
 				};
 			};
@@ -7084,7 +7101,19 @@
 	};
 
 	thermal-zones {
-		cpu0-thermal {
+		aoss-0-thermal {
+			thermal-sensors = <&tsens0 0>;
+
+			trips {
+				aoss-critical {
+					temperature = <110000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu0-0-thermal {
 			thermal-sensors = <&tsens0 1>;
 
 			trips {
@@ -7096,7 +7125,7 @@
 			};
 		};
 
-		cpu1-thermal {
+		cpu1-0-thermal {
 			thermal-sensors = <&tsens0 2>;
 
 			trips {
@@ -7108,7 +7137,7 @@
 			};
 		};
 
-		cpu2-thermal {
+		cpu2-0-thermal {
 			thermal-sensors = <&tsens0 3>;
 
 			trips {
@@ -7120,7 +7149,7 @@
 			};
 		};
 
-		cpu3-thermal {
+		cpu3-0-thermal {
 			thermal-sensors = <&tsens0 4>;
 
 			trips {
@@ -7132,7 +7161,7 @@
 			};
 		};
 
-		cpu4-thermal {
+		cpu4-0-thermal {
 			thermal-sensors = <&tsens0 5>;
 
 			trips {
@@ -7144,7 +7173,7 @@
 			};
 		};
 
-		cpu5-thermal {
+		cpu5-0-thermal {
 			thermal-sensors = <&tsens0 6>;
 
 			trips {
@@ -7156,7 +7185,7 @@
 			};
 		};
 
-		cpu6-thermal {
+		cpu6-0-thermal {
 			thermal-sensors = <&tsens0 7>;
 
 			trips {
@@ -7168,7 +7197,7 @@
 			};
 		};
 
-		cpu7-thermal {
+		cpu7-0-thermal {
 			thermal-sensors = <&tsens0 8>;
 
 			trips {
@@ -7180,7 +7209,7 @@
 			};
 		};
 
-		cluster0-thermal {
+		cpuss0-thermal {
 			thermal-sensors = <&tsens0 9>;
 
 			trips {
@@ -7192,10 +7221,262 @@
 			};
 		};
 
-		gpu-thermal {
+		nsp0-0-thermal {
+			thermal-sensors = <&tsens0 10>;
+
+			trips {
+				nsp0-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsp1-0-thermal {
+			thermal-sensors = <&tsens0 11>;
+
+			trips {
+				nsp1-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		smss-0-thermal {
+			thermal-sensors = <&tsens0 12>;
+
+			trips {
+				smss-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		camss-0-thermal {
+			thermal-sensors = <&tsens0 13>;
+
+			trips {
+				camss-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-1-thermal {
+			thermal-sensors = <&tsens1 0>;
+
+			trips {
+				aoss-critical {
+					temperature = <110000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu0-1-thermal {
+			thermal-sensors = <&tsens1 1>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu1-1-thermal {
+			thermal-sensors = <&tsens1 2>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu2-1-thermal {
+			thermal-sensors = <&tsens1 3>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu3-1-thermal {
+			thermal-sensors = <&tsens1 4>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu4-1-thermal {
+			thermal-sensors = <&tsens1 5>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu5-1-thermal {
+			thermal-sensors = <&tsens1 6>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu6-1-thermal {
+			thermal-sensors = <&tsens1 7>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpu7-1-thermal {
+			thermal-sensors = <&tsens1 8>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		cpuss1-thermal {
+			thermal-sensors = <&tsens1 9>;
+
+			trips {
+				cpu-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsp0-2-thermal {
+			thermal-sensors = <&tsens1 10>;
+
+			trips {
+				nsp0-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsp1-2-thermal {
+			thermal-sensors = <&tsens1 11>;
+
+			trips {
+				nsp1-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		smss-1-thermal {
+			thermal-sensors = <&tsens1 12>;
+
+			trips {
+				smss-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		camss-1-thermal {
+			thermal-sensors = <&tsens1 13>;
+
+			trips {
+				camss-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		pcie-0-thermal {
+			thermal-sensors = <&tsens1 14>;
+
+			trips {
+				pcie-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		ddrss-0-thermal {
+			thermal-sensors = <&tsens1 15>;
+
+			trips {
+				trip-point0 {
+					temperature = <90000>;
+					hysteresis = <2000>;
+					type = "hot";
+				};
+			};
+		};
+
+		aoss-2-thermal {
+			thermal-sensors = <&tsens2 0>;
+
+			trips {
+				aoss-critical {
+					temperature = <110000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-0-thermal {
 			polling-delay-passive = <250>;
 
-			thermal-sensors = <&tsens2 2>;
+			thermal-sensors = <&tsens2 1>;
 
 			cooling-maps {
 				map0 {
@@ -7219,8 +7500,101 @@
 			};
 		};
 
-		mem-thermal {
-			thermal-sensors = <&tsens1 15>;
+		gpuss-1-thermal {
+			polling-delay-passive = <250>;
+
+			thermal-sensors = <&tsens2 2>;
+
+			cooling-maps {
+				map0 {
+					trip = <&gpu_alert1>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
+			trips {
+				gpu_alert1: trip-point0 {
+					temperature = <85000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-2-thermal {
+			polling-delay-passive = <250>;
+
+			thermal-sensors = <&tsens2 3>;
+
+			cooling-maps {
+				map0 {
+					trip = <&gpu_alert2>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
+			trips {
+				gpu_alert2: trip-point0 {
+					temperature = <85000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-3-thermal {
+			polling-delay-passive = <250>;
+
+			thermal-sensors = <&tsens2 4>;
+
+			cooling-maps {
+				map0 {
+					trip = <&gpu_alert3>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
+			trips {
+				gpu_alert3: trip-point0 {
+					temperature = <85000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		pcie-1-thermal {
+			thermal-sensors = <&tsens2 5>;
+
+			trips {
+				pcie-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		ddrss-1-thermal {
+			thermal-sensors = <&tsens2 6>;
 
 			trips {
 				trip-point0 {
@@ -7230,6 +7604,174 @@
 				};
 			};
 		};
+
+		audio-thermal {
+			thermal-sensors = <&tsens2 7>;
+
+			trips {
+				audio-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		video-thermal {
+			thermal-sensors = <&tsens2 8>;
+
+			trips {
+				video-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsp0-1-thermal {
+			thermal-sensors = <&tsens2 9>;
+
+			trips {
+				nsp0-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		nsp1-1-thermal {
+			thermal-sensors = <&tsens2 10>;
+
+			trips {
+				nsp1-crit {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		aoss-3-thermal {
+			thermal-sensors = <&tsens3 0>;
+
+			trips {
+				aoss-critical {
+					temperature = <110000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-4-thermal {
+			polling-delay-passive = <250>;
+
+			thermal-sensors = <&tsens3 1>;
+
+			cooling-maps {
+				map0 {
+					trip = <&gpu_alert4>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
+			trips {
+				gpu_alert4: trip-point0 {
+					temperature = <85000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-5-thermal {
+			polling-delay-passive = <250>;
+
+			thermal-sensors = <&tsens3 2>;
+
+			cooling-maps {
+				map0 {
+					trip = <&gpu_alert5>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
+			trips {
+				gpu_alert5: trip-point0 {
+					temperature = <85000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-6-thermal {
+			polling-delay-passive = <250>;
+
+			thermal-sensors = <&tsens3 3>;
+
+			cooling-maps {
+				map0 {
+					trip = <&gpu_alert6>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
+			trips {
+				gpu_alert6: trip-point0 {
+					temperature = <85000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
+
+		gpuss-7-thermal {
+			polling-delay-passive = <250>;
+
+			thermal-sensors = <&tsens3 4>;
+
+			cooling-maps {
+				map0 {
+					trip = <&gpu_alert7>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+
+			trips {
+				gpu_alert7: trip-point0 {
+					temperature = <85000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <110000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+		};
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi
index a41ba0da380bb9..3a1014804fb054 100644
--- a/arch/arm64/boot/dts/qcom/sdm630.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi
@@ -1195,7 +1195,7 @@
 				 <&gcc GCC_BIMC_GFX_CLK>,
 				 <&gcc GCC_GPU_BIMC_GFX_CLK>;
 			clock-names = "iface",
-			              "mem",
+				      "mem",
 				      "mem_iface";
 			#global-interrupts = <2>;
 			#iommu-cells = <1>;
@@ -1423,26 +1423,26 @@
 			status = "disabled";
 
 			sdhc2_opp_table: opp-table {
-				 compatible = "operating-points-v2";
+				compatible = "operating-points-v2";
 
-				 opp-50000000 {
+				opp-50000000 {
 					opp-hz = /bits/ 64 <50000000>;
 					required-opps = <&rpmpd_opp_low_svs>;
 					opp-peak-kBps = <200000 140000>;
 					opp-avg-kBps = <130718 133320>;
-				 };
-				 opp-100000000 {
+				};
+				opp-100000000 {
 					opp-hz = /bits/ 64 <100000000>;
 					required-opps = <&rpmpd_opp_svs>;
 					opp-peak-kBps = <250000 160000>;
 					opp-avg-kBps = <196078 150000>;
-				 };
-				 opp-200000000 {
+				};
+				opp-200000000 {
 					opp-hz = /bits/ 64 <200000000>;
 					required-opps = <&rpmpd_opp_nom>;
 					opp-peak-kBps = <4096000 4096000>;
 					opp-avg-kBps = <1338562 1338562>;
-				 };
+				};
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi
index b4854801a5f5e9..4ce9b62abe53fd 100644
--- a/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm670-google-common.dtsi
@@ -9,6 +9,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include "sdm670.dtsi"
@@ -26,7 +27,9 @@
 /delete-node/ &gpu_mem;
 
 / {
-	aliases { };
+	aliases {
+		serial0 = &uart12;
+	};
 
 	battery: battery {
 		compatible = "simple-battery";
@@ -460,7 +463,7 @@
 		pinctrl-names = "default";
 
 		rotation = <270>;
-		orientation = <0>;
+		orientation = <MEDIA_ORIENTATION_FRONT>;
 
 		port {
 			cam_front_endpoint: endpoint {
@@ -490,6 +493,25 @@
 	firmware-name = "qcom/sdm670/sargo/a615_zap.mbn";
 };
 
+&i2c3 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	nfc@28 {
+		compatible = "nxp,pn557", "nxp,nxp-nci-i2c";
+		reg = <0x28>;
+
+		interrupts-extended = <&tlmm 44 IRQ_TYPE_EDGE_RISING>;
+
+		enable-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>;
+		firmware-gpios = <&tlmm 43 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&nfc_default>;
+		pinctrl-names = "default";
+	};
+};
+
 &i2c9 {
 	clock-frequency = <100000>;
 	status = "okay";
@@ -691,6 +713,33 @@
 			output-low;
 		};
 	};
+
+	nfc_default: nfc-default-state {
+		enable-pins {
+			pins = "gpio12";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+
+		firmware-pins {
+			pins = "gpio43";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+
+		irq-pins {
+			pins = "gpio44";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+	};
+};
+
+&uart12 {
+	status = "okay";
 };
 
 &usb_1_hsphy {
diff --git a/arch/arm64/boot/dts/qcom/sdm670.dtsi b/arch/arm64/boot/dts/qcom/sdm670.dtsi
index 5c60ea36f04a13..966ad40ea61dff 100644
--- a/arch/arm64/boot/dts/qcom/sdm670.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm670.dtsi
@@ -1060,6 +1060,21 @@
 				status = "disabled";
 			};
 
+			uart12: serial@a90000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0 0x00a90000 0 0x4000>;
+				clock-names = "se";
+				clocks = <&gcc GCC_QUPV3_WRAP1_S4_CLK>;
+				pinctrl-names = "default";
+				pinctrl-0 = <&qup_uart12_default>;
+				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+				power-domains = <&rpmhpd SDM670_CX>;
+				interconnects = <&aggre2_noc MASTER_BLSP_2 0 &config_noc SLAVE_BLSP_2 0>,
+						<&gladiator_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_BLSP_2 0>;
+				interconnect-names = "qup-core", "qup-config";
+				status = "disabled";
+			};
+
 			i2c13: i2c@a94000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0 0x00a94000 0 0x4000>;
@@ -1343,6 +1358,262 @@
 				function = "qup15";
 			};
 
+			qup_uart0_default: qup-uart0-default-state {
+				tx-pins {
+					pins = "gpio2";
+					function = "qup0";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio3";
+					function = "qup0";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart1_default: qup-uart1-default-state {
+				tx-pins {
+					pins = "gpio19";
+					function = "qup1";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio20";
+					function = "qup1";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart2_default: qup-uart2-default-state {
+				tx-pins {
+					pins = "gpio29";
+					function = "qup2";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio30";
+					function = "qup2";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart3_default: qup-uart3-default-state {
+				tx-pins {
+					pins = "gpio43";
+					function = "qup3";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio44";
+					function = "qup3";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart4_default: qup-uart4-default-state {
+				tx-pins {
+					pins = "gpio91";
+					function = "qup4";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio92";
+					function = "qup4";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart5_default: qup-uart5-default-state {
+				tx-pins {
+					pins = "gpio87";
+					function = "qup5";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio88";
+					function = "qup5";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart6_default: qup-uart6-default-state {
+				tx-pins {
+					pins = "gpio47";
+					function = "qup6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio48";
+					function = "qup6";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart7_default: qup-uart7-default-state {
+				tx-pins {
+					pins = "gpio95";
+					function = "qup7";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio96";
+					function = "qup7";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart8_default: qup-uart8-default-state {
+				tx-pins {
+					pins = "gpio67";
+					function = "qup8";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio68";
+					function = "qup8";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart9_default: qup-uart9-default-state {
+				tx-pins {
+					pins = "gpio4";
+					function = "qup9";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio5";
+					function = "qup9";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart10_default: qup-uart10-default-state {
+				tx-pins {
+					pins = "gpio53";
+					function = "qup10";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio54";
+					function = "qup10";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart11_default: qup-uart11-default-state {
+				tx-pins {
+					pins = "gpio33";
+					function = "qup11";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio34";
+					function = "qup11";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart12_default: qup-uart12-default-state {
+				tx-pins {
+					pins = "gpio51";
+					function = "qup12";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio52";
+					function = "qup12";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart13_default: qup-uart13-default-state {
+				tx-pins {
+					pins = "gpio107";
+					function = "qup13";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio108";
+					function = "qup13";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart14_default: qup-uart14-default-state {
+				tx-pins {
+					pins = "gpio31";
+					function = "qup14";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio32";
+					function = "qup14";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
+			qup_uart15_default: qup-uart15-default-state {
+				tx-pins {
+					pins = "gpio83";
+					function = "qup15";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio84";
+					function = "qup15";
+					drive-strength = <2>;
+					bias-disable;
+				};
+			};
+
 			sdc1_state_on: sdc1-on-state {
 				clk-pins {
 					pins = "sdc1_clk";
diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts
index 846138149b2cd3..02cb29afb2469e 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts
@@ -618,9 +618,6 @@
 
 &pcie0 {
 	status = "okay";
-	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 134 GPIO_ACTIVE_HIGH>;
-
 	vddpe-3v3-supply = <&pcie0_3p3v_dual>;
 
 	pinctrl-names = "default";
@@ -634,9 +631,13 @@
 	vdda-pll-supply = <&vreg_l26a_1p2>;
 };
 
+&pcie0_port0 {
+	reset-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 134 GPIO_ACTIVE_LOW>;
+};
+
 &pcie1 {
 	status = "okay";
-	perst-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
 
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_default_state>;
@@ -649,6 +650,10 @@
 	vdda-pll-supply = <&vreg_l26a_1p2>;
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
+};
+
 &pm8998_gpios {
 	gpio-line-names =
 		"NC",
diff --git a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
index 9e952f9862f24d..f1379857e27421 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
@@ -392,6 +392,8 @@
 };
 
 &i2c3 {
+	clock-frequency = <400000>;
+
 	status = "okay";
 
 	nfc@28 {
@@ -403,7 +405,7 @@
 		enable-gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>;
 		firmware-gpios = <&tlmm 79 GPIO_ACTIVE_HIGH>;
 
-		pinctrl-0 = <&nfc_int_default &nfc_enable_default>;
+		pinctrl-0 = <&nfc_default>;
 		pinctrl-names = "default";
 	};
 };
@@ -476,18 +478,28 @@
 	gpio-reserved-ranges = < 0 4>, /* SPI (Intel MNH Pixel Visual Core) */
 			       <81 4>; /* SPI (most likely Fingerprint Cards FPC1075) */
 
-	nfc_int_default: nfc-int-default-state {
-		pins = "gpio63";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-down;
-	};
+	nfc_default: nfc-default-state {
+		enable-pins {
+			pins = "gpio25";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+
+		irq-pins {
+			pins = "gpio63";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
+
+		firmware-pins {
+			pins = "gpio79";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-down;
+		};
 
-	nfc_enable_default: nfc-enable-default-state {
-		pins = "gpio25", "gpio79";
-		function = "gpio";
-		drive-strength = <2>;
-		bias-pull-down;
 	};
 
 	touchscreen_reset: ts-reset-state {
diff --git a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
index b030a10009c58c..84f0e1f0bcd323 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
@@ -581,33 +581,26 @@
 	sdc2_clk: sdc2-clk-state {
 		pins = "sdc2_clk";
 		bias-disable;
-
-		/*
-		 * It seems that mmc_test reports errors if drive
-		 * strength is not 16 on clk, cmd, and data pins.
-		 *
-		 * TODO: copy-pasted from mtp, try other values
-		 * on these devices.
-		 */
 		drive-strength = <16>;
 	};
 
 	sdc2_cmd: sdc2-cmd-state {
 		pins = "sdc2_cmd";
 		bias-pull-up;
-		drive-strength = <16>;
+		drive-strength = <10>;
 	};
 
 	sdc2_data: sdc2-data-state {
 		pins = "sdc2_data";
 		bias-pull-up;
-		drive-strength = <16>;
+		drive-strength = <10>;
 	};
 
 	sd_card_det_n: sd-card-det-n-state {
 		pins = "gpio126";
 		function = "gpio";
 		bias-pull-up;
+		drive-strength = <2>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
index c66df45eee64e3..88aa0b7de5fdf4 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
@@ -508,8 +508,6 @@
 };
 
 &pcie0 {
-	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -523,9 +521,11 @@
 	status = "okay";
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
+&pcie0_port0 {
+	reset-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+};
 
+&pcie1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_default_state>;
 
@@ -539,6 +539,10 @@
 	vdda-pll-supply = <&vreg_l26a_1p2>;
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
+};
+
 &pm8998_adc {
 	channel@4c {
 		reg = <ADC5_XO_THERM_100K_PU>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
index 439779f308e380..1df09da3c4d713 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
@@ -510,7 +510,6 @@
 
 	bq27441_fg: bq27441-battery@55 {
 		compatible = "ti,bq27411";
-		status = "okay";
 		reg = <0x55>;
 	};
 };
@@ -543,7 +542,7 @@
 		rmi4_f12: rmi4-f12@12 {
 			reg = <0x12>;
 			touchscreen-x-mm = <68>;
-			touchscreen-y-mm = <144>;
+			syna,clip-x-high = <1079>;
 			syna,sensor-type = <1>;
 			syna,rezero-wait-ms = <200>;
 		};
@@ -1099,6 +1098,10 @@
 	vdd-tx-supply = <&vreg_s4a_1p8>;
 	vdd-rx-supply = <&vreg_s4a_1p8>;
 	vdd-io-supply = <&vreg_s4a_1p8>;
+	qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000
+						  500000 500000 500000 500000>;
+	qcom,mbhc-headset-vthreshold-microvolt = <1700000>;
+	qcom,mbhc-headphone-vthreshold-microvolt = <50000>;
 };
 
 &wifi {
diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts b/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts
index 3b30d79706fe71..276217f7216b5a 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-enchilada.dts
@@ -95,6 +95,11 @@
 	};
 };
 
+&rmi4_f12 {
+	touchscreen-y-mm = <144>;
+	syna,clip-y-high = <2279>;
+};
+
 &sound {
 	model = "OnePlus 6";
 	audio-routing = "RX_BIAS", "MCLK",
diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts b/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts
index 0542333a835798..e9d1a1c6ac5963 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-fajita.dts
@@ -70,6 +70,7 @@
 
 &rmi4_f12 {
 	touchscreen-y-mm = <148>;
+	syna,clip-y-high = <2339>;
 };
 
 &wcd9340 {
diff --git a/arch/arm64/boot/dts/qcom/sdm845-sony-xperia-tama.dtsi b/arch/arm64/boot/dts/qcom/sdm845-sony-xperia-tama.dtsi
index 4c63286d8b041c..05e5a3674e5fa3 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-sony-xperia-tama.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-sony-xperia-tama.dtsi
@@ -754,6 +754,8 @@
 };
 
 &uart6 {
+	pinctrl-0 = <&qup_uart6_4pin>;
+
 	status = "okay";
 
 	bluetooth {
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
index 080f5294988d34..35babe8c5f7578 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
@@ -27,6 +27,7 @@
 /delete-node/ &slpi_mem;
 /delete-node/ &spss_mem;
 /delete-node/ &rmtfs_mem;
+/delete-node/ &xbl_mem;
 
 / {
 	chassis-type = "handset";
@@ -71,6 +72,11 @@
 
 	/* Reserved memory changes from downstream */
 	reserved-memory {
+		xbl-mem@85d00000 {
+			reg = <0 0x85d00000 0 0x200000>;
+			no-map;
+		};
+
 		tz_mem: memory@86200000 {
 			reg = <0 0x86200000 0 0x4900000>;
 			no-map;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
index d3162e4444532f..a782023d0dcb3a 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
@@ -32,6 +32,7 @@
 /delete-node/ &mba_region;
 /delete-node/ &slpi_mem;
 /delete-node/ &spss_mem;
+/delete-node/ &xbl_mem;
 
 / {
 	model = "Xiaomi Mi MIX 2S";
@@ -63,6 +64,11 @@
 	};
 
 	reserved-memory {
+		xbl-mem@85d00000 {
+			reg = <0 0x85d00000 0 0x200000>;
+			no-map;
+		};
+
 		adsp_mem: memory@8c500000 {
 			reg = <0 0x8c500000 0 0x1e00000>;
 			no-map;
diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi
index b1f6782a3ad9fa..844b840bc5468d 100644
--- a/arch/arm64/boot/dts/qcom/sdm845.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi
@@ -2395,16 +2395,15 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
-			pcie@0 {
+			pcie0_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2525,16 +2524,15 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -4626,7 +4624,7 @@
 			interconnect-names = "mdp0-mem", "mdp1-mem";
 
 			iommus = <&apps_smmu 0x880 0x8>,
-			         <&apps_smmu 0xc80 0x8>;
+				 <&apps_smmu 0xc80 0x8>;
 
 			status = "disabled";
 
@@ -5031,7 +5029,7 @@
 				     <GIC_SPI 370 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 371 IRQ_TYPE_EDGE_RISING>;
 			clocks = <&gcc GCC_GPU_MEMNOC_GFX_CLK>,
-			         <&gcc GCC_GPU_CFG_AHB_CLK>;
+				 <&gcc GCC_GPU_CFG_AHB_CLK>;
 			clock-names = "bus", "iface";
 
 			power-domains = <&gpucc GPU_CX_GDSC>;
@@ -5050,7 +5048,7 @@
 			interrupt-names = "hfi", "gmu";
 
 			clocks = <&gpucc GPU_CC_CX_GMU_CLK>,
-			         <&gpucc GPU_CC_CXO_CLK>,
+				 <&gpucc GPU_CC_CXO_CLK>,
 				 <&gcc GCC_DDRSS_GPU_AXI_CLK>,
 				 <&gcc GCC_GPU_MEMNOC_GFX_CLK>;
 			clock-names = "gmu", "cxo", "axi", "memnoc";
diff --git a/arch/arm64/boot/dts/qcom/sdx75.dtsi b/arch/arm64/boot/dts/qcom/sdx75.dtsi
index 85cc9c6e6f83f7..074079d36751f2 100644
--- a/arch/arm64/boot/dts/qcom/sdx75.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdx75.dtsi
@@ -15,7 +15,6 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 
 / {
@@ -925,7 +924,8 @@
 		};
 
 		remoteproc_mpss: remoteproc@4080000 {
-			compatible = "qcom,sdx75-mpss-pas";
+			compatible = "qcom,sdx75-mpss-pas",
+				     "qcom,sm8650-mpss-pas";
 			reg = <0 0x04080000 0 0x10000>;
 
 			interrupts-extended = <&intc GIC_SPI 250 IRQ_TYPE_EDGE_RISING>,
@@ -1137,7 +1137,6 @@
 			interrupt-names = "periph_irq";
 			qcom,ee = <0>;
 			qcom,channel = <0>;
-			qcom,bus-id = <0>;
 			#address-cells = <2>;
 			#size-cells = <0>;
 			interrupt-controller;
diff --git a/arch/arm64/boot/dts/qcom/shikra-cqm-cqs-evk-imx577-camera.dtso b/arch/arm64/boot/dts/qcom/shikra-cqm-cqs-evk-imx577-camera.dtso
new file mode 100644
index 00000000000000..791589bc7642ea
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-cqm-cqs-evk-imx577-camera.dtso
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/qcom,shikra-gcc.h>
+#include <dt-bindings/gpio/gpio.h>
+
+&camss {
+	vdd-csiphy-1p2-supply = <&pm4125_l5>;
+	vdd-csiphy-1p8-supply = <&pm4125_l13>;
+
+	status = "okay";
+};
+
+&camss_csiphy1_port {
+	camss_csiphy1_inep: endpoint {
+		data-lanes = <0 1 2 3>;
+		remote-endpoint = <&imx577_ep1>;
+	};
+};
+
+&cci {
+	status = "okay";
+};
+
+&cci_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	camera@1a {
+		compatible = "sony,imx577";
+		reg = <0x1a>;
+
+		reset-gpios = <&tlmm 33 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&cam_mclk1_default &cam1_reset_default>;
+		pinctrl-names = "default";
+
+		clocks = <&gcc GCC_CAMSS_MCLK1_CLK>;
+		assigned-clocks = <&gcc GCC_CAMSS_MCLK1_CLK>;
+		assigned-clock-rates = <24000000>;
+
+		/*
+		 * avdd and dvdd are supplied by on-board regulators on the
+		 * IMX577 module from the connector's 3.3 V rail; they are
+		 * not SoC-controlled. dovdd (1.8 V) powers the carrier board
+		 * level-shifter that translates CCI I2C and reset lines
+		 * between the SoC and the connector.
+		 */
+		dovdd-supply = <&pm4125_l15>;
+
+		port {
+			imx577_ep1: endpoint {
+				link-frequencies = /bits/ 64 <600000000>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&camss_csiphy1_inep>;
+			};
+		};
+	};
+};
+
+&tlmm {
+	cam1_reset_default: cam1-reset-default-state {
+		pins = "gpio33";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts b/arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts
index b112b21b1d79b7..6ad90df3b41563 100644
--- a/arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts
+++ b/arch/arm64/boot/dts/qcom/shikra-cqm-evk.dts
@@ -7,6 +7,7 @@
 
 #include "shikra-cqm-som.dtsi"
 #include "shikra-evk.dtsi"
+#include <dt-bindings/gpio/gpio.h>
 
 / {
 	model = "Qualcomm Technologies, Inc. Shikra CQM EVK";
@@ -15,6 +16,7 @@
 
 	aliases {
 		mmc0 = &sdhc_1;
+		mmc1 = &sdhc_2; /* SDC2 SD card slot */
 		serial0 = &uart0;
 	};
 
@@ -23,6 +25,10 @@
 	};
 };
 
+&audiocorecc {
+	status = "okay";
+};
+
 &remoteproc_cdsp {
 	firmware-name = "qcom/shikra/cdsp.mbn";
 
@@ -57,3 +63,33 @@
 
 	status = "okay";
 };
+
+&sdhc_2 {
+	vmmc-supply = <&pm4125_l21>;
+	vqmmc-supply = <&pm4125_l4>;
+
+	status = "okay";
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&pm4125_l12>;
+	vdda-pll-supply = <&pm4125_l13>;
+	vdda-phy-dpdm-supply = <&pm4125_l21>;
+
+	status = "okay";
+};
+
+&usb_qmpphy {
+	vdda-phy-supply = <&pm4125_l8>;
+	vdda-pll-supply = <&pm4125_l13>;
+
+	status = "okay";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&pm4125_l12>;
+	vdda-pll-supply = <&pm4125_l13>;
+	vdda-phy-dpdm-supply = <&pm4125_l21>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi b/arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi
index 8ac42ff625a0e9..fde7fc035bf5d2 100644
--- a/arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/shikra-cqm-som.dtsi
@@ -197,7 +197,6 @@
 
 &tlmm {
 	gpio-reserved-ranges = <6 4>, /* Fingerprint SPI */
-			       <14 4>, /* eSE SPI */
 			       <30 2>, /* NFC SPI */
 			       <138 1>, /* NFC Secure IO */
 			       <155 11>; /* eMMC Boot */
diff --git a/arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts b/arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts
index e62ba5aef71ff5..7a2f3e1518fdbd 100644
--- a/arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts
+++ b/arch/arm64/boot/dts/qcom/shikra-cqs-evk.dts
@@ -7,6 +7,7 @@
 
 #include "shikra-cqm-som.dtsi"
 #include "shikra-evk.dtsi"
+#include <dt-bindings/gpio/gpio.h>
 
 / {
 	model = "Qualcomm Technologies, Inc. Shikra CQS EVK";
@@ -15,6 +16,7 @@
 
 	aliases {
 		mmc0 = &sdhc_1;
+		mmc1 = &sdhc_2; /* SDC2 SD card slot */
 		serial0 = &uart0;
 	};
 
@@ -57,3 +59,33 @@
 
 	status = "okay";
 };
+
+&sdhc_2 {
+	vmmc-supply = <&pm4125_l21>;
+	vqmmc-supply = <&pm4125_l4>;
+
+	status = "okay";
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&pm4125_l12>;
+	vdda-pll-supply = <&pm4125_l13>;
+	vdda-phy-dpdm-supply = <&pm4125_l21>;
+
+	status = "okay";
+};
+
+&usb_qmpphy {
+	vdda-phy-supply = <&pm4125_l8>;
+	vdda-pll-supply = <&pm4125_l13>;
+
+	status = "okay";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&pm4125_l12>;
+	vdda-pll-supply = <&pm4125_l13>;
+	vdda-phy-dpdm-supply = <&pm4125_l21>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-evk.dtsi b/arch/arm64/boot/dts/qcom/shikra-evk.dtsi
index 6eb4184f76422d..7fbd6d1de19ad3 100644
--- a/arch/arm64/boot/dts/qcom/shikra-evk.dtsi
+++ b/arch/arm64/boot/dts/qcom/shikra-evk.dtsi
@@ -3,12 +3,53 @@
  * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/shikra/a704_zap.mbn";
+};
+
 &qupv3_0 {
 	firmware-name = "qcom/shikra/qupv3fw.elf";
 
 	status = "okay";
 };
 
+&sdhc_2 {
+	pinctrl-0 = <&sdc2_default &sdc2_card_det_n>;
+	pinctrl-1 = <&sdc2_sleep &sdc2_card_det_n>;
+	pinctrl-names = "default", "sleep";
+
+	no-sdio;
+	no-mmc;
+
+	cd-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
+};
+
+&spi5 {
+	status = "okay";
+
+	tpm@0 {
+		compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi";
+		reg = <0>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
 &uart0 {
 	status = "okay";
 };
+
+&usb_1 {
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
+
+&usb_2 {
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-iqs-evk-imx577-camera.dtso b/arch/arm64/boot/dts/qcom/shikra-iqs-evk-imx577-camera.dtso
new file mode 100644
index 00000000000000..e10fb33996e8e8
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/shikra-iqs-evk-imx577-camera.dtso
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/qcom,shikra-gcc.h>
+#include <dt-bindings/gpio/gpio.h>
+
+&camss {
+	vdd-csiphy-1p2-supply = <&pm8150_l11>;
+	vdd-csiphy-1p8-supply = <&pm8150_l12>;
+
+	status = "okay";
+};
+
+&camss_csiphy1_port {
+	camss_csiphy1_inep: endpoint {
+		data-lanes = <0 1 2 3>;
+		remote-endpoint = <&imx577_ep1>;
+	};
+};
+
+&cci {
+	status = "okay";
+};
+
+&cci_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	camera@1a {
+		compatible = "sony,imx577";
+		reg = <0x1a>;
+
+		reset-gpios = <&tlmm 33 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&cam_mclk1_default &cam1_reset_default>;
+		pinctrl-names = "default";
+
+		clocks = <&gcc GCC_CAMSS_MCLK1_CLK>;
+		assigned-clocks = <&gcc GCC_CAMSS_MCLK1_CLK>;
+		assigned-clock-rates = <24000000>;
+
+		/*
+		 * avdd and dvdd are supplied by on-board regulators on the
+		 * IMX577 module from the connector's 3.3 V rail; they are
+		 * not SoC-controlled. dovdd (1.8 V) powers the carrier board
+		 * level-shifter that translates CCI I2C and reset lines
+		 * between the SoC and the connector.
+		 */
+		dovdd-supply = <&pm8150_l15>;
+
+		port {
+			imx577_ep1: endpoint {
+				link-frequencies = /bits/ 64 <600000000>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&camss_csiphy1_inep>;
+			};
+		};
+	};
+};
+
+&tlmm {
+	cam1_reset_default: cam1-reset-default-state {
+		pins = "gpio33";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts b/arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts
index 727809430fd15f..5df30ad8211f8c 100644
--- a/arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts
+++ b/arch/arm64/boot/dts/qcom/shikra-iqs-evk.dts
@@ -7,6 +7,7 @@
 
 #include "shikra-iqs-som.dtsi"
 #include "shikra-evk.dtsi"
+#include <dt-bindings/gpio/gpio.h>
 
 / {
 	model = "Qualcomm Technologies, Inc. Shikra IQS EVK";
@@ -15,6 +16,7 @@
 
 	aliases {
 		mmc0 = &sdhc_1;
+		mmc1 = &sdhc_2; /* SDC2 SD card slot */
 		serial0 = &uart0;
 	};
 
@@ -57,3 +59,33 @@
 
 	status = "okay";
 };
+
+&sdhc_2 {
+	vmmc-supply = <&pm8150_l10>;
+	vqmmc-supply = <&pm8150_l2>;
+
+	status = "okay";
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&pm8150_l4>;
+	vdda-pll-supply = <&pm8150_l12>;
+	vdda-phy-dpdm-supply = <&pm8150_l13>;
+
+	status = "okay";
+};
+
+&usb_qmpphy {
+	vdda-phy-supply = <&pm8150_l6>;
+	vdda-pll-supply = <&pm8150_l12>;
+
+	status = "okay";
+};
+
+&usb_2_hsphy {
+	vdd-supply = <&pm8150_l4>;
+	vdda-pll-supply = <&pm8150_l12>;
+	vdda-phy-dpdm-supply = <&pm8150_l13>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi b/arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi
index 4ff97945274dd1..8a60f304eaa4a7 100644
--- a/arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/shikra-iqs-som.dtsi
@@ -218,8 +218,7 @@
 };
 
 &tlmm {
-	 gpio-reserved-ranges = <6 4>, /* Fingerprint SPI */
-			       <14 4>, /* eSE SPI */
+	gpio-reserved-ranges = <6 4>, /* Fingerprint SPI */
 			       <30 2>, /* NFC SPI */
 			       <138 1>, /* NFC Secure IO */
 			       <155 11>; /* eMMC Boot */
diff --git a/arch/arm64/boot/dts/qcom/shikra.dtsi b/arch/arm64/boot/dts/qcom/shikra.dtsi
index 92d22e3b6931cf..9823c591b02b19 100644
--- a/arch/arm64/boot/dts/qcom/shikra.dtsi
+++ b/arch/arm64/boot/dts/qcom/shikra.dtsi
@@ -6,6 +6,7 @@
 #include <dt-bindings/clock/qcom,dispcc-qcm2290.h>
 #include <dt-bindings/clock/qcom,qcm2290-gpucc.h>
 #include <dt-bindings/clock/qcom,rpmcc.h>
+#include <dt-bindings/clock/qcom,shikra-audiocorecc.h>
 #include <dt-bindings/clock/qcom,shikra-gcc.h>
 #include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,osm-l3.h>
@@ -13,6 +14,7 @@
 #include <dt-bindings/interconnect/qcom,rpm-icc.h>
 #include <dt-bindings/interconnect/qcom,shikra.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/thermal/thermal.h>
 
@@ -150,6 +152,88 @@
 		};
 	};
 
+	dummy-eud {
+		compatible = "arm,coresight-dummy-sink";
+
+		label = "eud";
+
+		in-ports {
+			port {
+				eud_in: endpoint {
+					remote-endpoint = <&replicator_eud_out1>;
+				};
+			};
+		};
+	};
+
+	etm0 {
+		compatible = "arm,coresight-etm4x-sysreg";
+
+		clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+		clock-names = "apb_pclk";
+		cpu = <&cpu0>;
+		qcom,skip-power-up;
+
+		out-ports {
+			port {
+				etm0_out: endpoint {
+					remote-endpoint = <&funnel_cpuss0_in0>;
+				};
+			};
+		};
+	};
+
+	etm1 {
+		compatible = "arm,coresight-etm4x-sysreg";
+
+		clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+		clock-names = "apb_pclk";
+		cpu = <&cpu1>;
+		qcom,skip-power-up;
+
+		out-ports {
+			port {
+				etm1_out: endpoint {
+					remote-endpoint = <&funnel_cpuss0_in1>;
+				};
+			};
+		};
+	};
+
+	etm2 {
+		compatible = "arm,coresight-etm4x-sysreg";
+
+		clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+		clock-names = "apb_pclk";
+		cpu = <&cpu2>;
+		qcom,skip-power-up;
+
+		out-ports {
+			port {
+				etm2_out: endpoint {
+					remote-endpoint = <&funnel_cpuss0_in2>;
+				};
+			};
+		};
+	};
+
+	etm3 {
+		compatible = "arm,coresight-etm4x-sysreg";
+
+		clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+		clock-names = "apb_pclk";
+		cpu = <&cpu3>;
+		qcom,skip-power-up;
+
+		out-ports {
+			port {
+				etm3_out: endpoint {
+					remote-endpoint = <&funnel_cpuss0_in3>;
+				};
+			};
+		};
+	};
+
 	firmware {
 		scm {
 			compatible = "qcom,scm-shikra", "qcom,scm";
@@ -248,6 +332,21 @@
 		};
 	};
 
+	modem-etm0 {
+		compatible = "arm,coresight-dummy-source";
+
+		label = "modem_etm0";
+		arm,static-trace-id = <36>;
+
+		out-ports {
+			port {
+				modem_etm0_out: endpoint {
+					remote-endpoint = <&funnel_in1_in4>;
+				};
+			};
+		};
+	};
+
 	pmu-a55 {
 		compatible = "arm,cortex-a55-pmu";
 		interrupts = <GIC_PPI 5 IRQ_TYPE_LEVEL_HIGH &ppi_cluster0>;
@@ -795,6 +894,66 @@
 				bias-disable;
 			};
 
+			cci_i2c0_default: cci-i2c0-default-state {
+				/* SDA, SCL */
+				pins = "gpio36", "gpio37";
+				function = "cci_i2c0";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			cci_i2c0_sleep: cci-i2c0-sleep-state {
+				/* SDA, SCL */
+				pins = "gpio36", "gpio37";
+				function = "cci_i2c0";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cci_i2c1_default: cci-i2c1-default-state {
+				/* SDA, SCL */
+				pins = "gpio41", "gpio42";
+				function = "cci_i2c1";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			cci_i2c1_sleep: cci-i2c1-sleep-state {
+				/* SDA, SCL */
+				pins = "gpio41", "gpio42";
+				function = "cci_i2c1";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cam_mclk0_default: cam-mclk0-default-state {
+				pins = "gpio34";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk1_default: cam-mclk1-default-state {
+				pins = "gpio35";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk2_default: cam-mclk2-default-state {
+				pins = "gpio96";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk3_default: cam-mclk3-default-state {
+				pins = "gpio98";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
 			sdc1_state_on: sdc1-on-state {
 				clk-pins {
 					pins = "sdc1_clk";
@@ -844,6 +1003,53 @@
 					bias-bus-hold;
 				};
 			};
+
+			sdc2_default: sdc2-default-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <14>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <14>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <14>;
+					bias-pull-up;
+				};
+			};
+
+			sdc2_sleep: sdc2-sleep-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
+			sdc2_card_det_n: sd-card-det-n-state {
+				pins = "gpio89";
+				function = "gpio";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
 		};
 
 		pmu@c91000 {
@@ -922,6 +1128,85 @@
 			#power-domain-cells = <1>;
 		};
 
+		usb_1_hsphy: phy@1613000 {
+			compatible = "qcom,shikra-qusb2-phy";
+			reg = <0x0 0x01613000 0x0 0x180>;
+
+			clocks = <&gcc GCC_AHB2PHY_USB_CLK>,
+				 <&rpmcc RPM_SMD_XO_CLK_SRC>;
+			clock-names = "cfg_ahb", "ref";
+
+			resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>;
+			nvmem-cells = <&qusb2_hstx_trim_1>;
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		usb_qmpphy: phy@1615000 {
+			compatible = "qcom,shikra-qmp-usb3-dp-phy";
+			reg = <0x0 0x01615000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_USB3_PRIM_PHY_COM_AUX_CLK>,
+				 <&gcc GCC_USB3_PRIM_CLKREF_EN>,
+				 <&gcc GCC_AHB2PHY_USB_CLK>,
+				 <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>;
+			clock-names = "aux",
+				      "ref",
+				      "cfg_ahb",
+				      "pipe";
+
+			resets = <&gcc GCC_USB3PHY_PHY_PRIM_SP0_BCR>,
+				 <&gcc GCC_USB3_DP_PHY_PRIM_BCR>,
+				 <&gcc GCC_USB3_PHY_PRIM_SP0_BCR>;
+			reset-names = "phy_phy",
+				      "dp_phy",
+				      "phy";
+
+			#clock-cells = <1>;
+			#phy-cells = <1>;
+			orientation-switch;
+
+			qcom,tcsr-reg = <&tcsr_regs 0xb244 0xb248>;
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_qmpphy_out: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_qmpphy_usb_ss_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_ss>;
+					};
+				};
+			};
+		};
+
+		usb_2_hsphy: phy@1617000 {
+			compatible = "qcom,shikra-qusb2-phy";
+			reg = <0x0 0x01617000 0x0 0x180>;
+
+			clocks = <&gcc GCC_AHB2PHY_USB_CLK>,
+				 <&rpmcc RPM_SMD_XO_CLK_SRC>;
+			clock-names = "cfg_ahb", "ref";
+
+			resets = <&gcc GCC_QUSB2PHY_SEC_BCR>;
+			nvmem-cells = <&qusb2_hstx_trim_2>;
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
 		system_noc: interconnect@1880000 {
 			compatible = "qcom,shikra-sys-noc";
 			reg = <0x0 0x01880000 0x0 0x6a080>;
@@ -962,12 +1247,52 @@
 			#interconnect-cells = <2>;
 		};
 
+		cryptobam: dma-controller@1b04000 {
+			compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0";
+			reg = <0x0 0x01b04000 0x0 0x24000>;
+			interrupts = <GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH 0>;
+			#dma-cells = <1>;
+			iommus = <&apps_smmu 0x84 0x0011>,
+				 <&apps_smmu 0x86 0x0011>,
+				 <&apps_smmu 0x92 0x0>,
+				 <&apps_smmu 0x94 0x0011>,
+				 <&apps_smmu 0x96 0x0011>,
+				 <&apps_smmu 0x98 0x0001>,
+				 <&apps_smmu 0x9f 0x0>;
+			qcom,ee = <0>;
+			qcom,controlled-remotely;
+			num-channels = <16>;
+			qcom,num-ees = <4>;
+		};
+
+		crypto: crypto@1b3a000 {
+			compatible = "qcom,shikra-qce", "qcom,sm8150-qce", "qcom,qce";
+			reg = <0x0 0x01b3a000 0x0 0x6000>;
+			dmas = <&cryptobam 4>, <&cryptobam 5>;
+			dma-names = "rx", "tx";
+			iommus = <&apps_smmu 0x84 0x0011>,
+				 <&apps_smmu 0x86 0x0011>,
+				 <&apps_smmu 0x92 0x0>,
+				 <&apps_smmu 0x94 0x0011>,
+				 <&apps_smmu 0x96 0x0011>,
+				 <&apps_smmu 0x98 0x0001>,
+				 <&apps_smmu 0x9f 0x0>;
+			interconnects = <&system_noc MASTER_CRYPTO_CORE0 0
+					 &mc_virt SLAVE_EBI_CH0 0>;
+			interconnect-names = "memory";
+		};
+
 		qfprom: efuse@1b44000 {
 			compatible = "qcom,shikra-qfprom", "qcom,qfprom";
 			reg = <0x0 0x01b44000 0x0 0x3000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
 
+			qusb2_hstx_trim_2: hstx-trim@25a {
+				reg = <0x25a 0x1>;
+				bits = <4 4>;
+			};
+
 			qusb2_hstx_trim_1: hstx-trim@25b {
 				reg = <0x25b 0x1>;
 				bits = <1 4>;
@@ -1013,6 +1338,11 @@
 			#thermal-sensor-cells = <1>;
 		};
 
+		rng: rng@4454000 {
+			compatible = "qcom,shikra-trng", "qcom,trng";
+			reg = <0x0 0x04454000 0x0 0x1000>;
+		};
+
 		rpm_msg_ram: sram@45f0000 {
 			compatible = "qcom,rpm-msg-ram", "mmio-sram";
 			reg = <0x0 0x045f0000 0x0 0x7000>;
@@ -1074,6 +1404,7 @@
 			mmc-hs400-enhanced-strobe;
 
 			resets = <&gcc GCC_SDCC1_BCR>;
+			qcom,ice = <&sdhc_ice>;
 
 			status = "disabled";
 
@@ -1124,6 +1455,67 @@
 			iommus = <&apps_smmu 0xf6 0x0>;
 		};
 
+		sdhc_ice: crypto@4748000 {
+			compatible = "qcom,shikra-inline-crypto-engine",
+				     "qcom,inline-crypto-engine";
+			reg = <0x0 0x04748000 0x0 0x18000>;
+			clocks = <&gcc GCC_SDCC1_ICE_CORE_CLK>,
+				 <&gcc GCC_SDCC1_AHB_CLK>;
+			clock-names = "core",
+				      "iface";
+			power-domains = <&rpmpd RPMHPD_CX>;
+		};
+
+		sdhc_2: mmc@4784000 {
+			compatible = "qcom,shikra-sdhci", "qcom,sdhci-msm-v5";
+			reg = <0x0 0x04784000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 350 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hc_irq",
+					  "pwr_irq";
+
+			bus-width = <4>;
+
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
+				 <&rpmcc RPM_SMD_XO_CLK_SRC>;
+			clock-names = "iface",
+				      "core",
+				      "xo";
+
+			qcom,dll-config = <0x0007442c>;
+			qcom,ddr-config = <0x80040868>;
+
+			iommus = <&apps_smmu 0x0a0 0x0>;
+
+			interconnects = <&system_noc MASTER_SDCC_2 RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>,
+					<&mem_noc MASTER_AMPSS_M0 RPM_ACTIVE_TAG
+					 &config_noc SLAVE_SDCC_2 RPM_ACTIVE_TAG>;
+			interconnect-names = "sdhc-ddr",
+					     "cpu-sdhc";
+
+			power-domains = <&rpmpd RPMPD_VDDCX>;
+			operating-points-v2 = <&sdhc2_opp_table>;
+
+			status = "disabled";
+
+			sdhc2_opp_table: opp-table-2 {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>;
+					required-opps = <&rpmpd_opp_low_svs>;
+				};
+
+				opp-202000000 {
+					opp-hz = /bits/ 64 <202000000>;
+					required-opps = <&rpmpd_opp_svs_plus>;
+				};
+			};
+		};
+
 		qupv3_0: geniqup@4ac0000 {
 			compatible = "qcom,geni-se-qup";
 			reg = <0x0 0x04ac0000 0x0 0x2000>;
@@ -1828,6 +2220,143 @@
 			};
 		};
 
+		camss: camss@5c11000 {
+			compatible = "qcom,shikra-camss", "qcom,qcm2290-camss";
+
+			reg = <0x0 0x05c11000 0x0 0x1000>,
+			      <0x0 0x05c6e000 0x0 0x1000>,
+			      <0x0 0x05c75000 0x0 0x1000>,
+			      <0x0 0x05c52000 0x0 0x1000>,
+			      <0x0 0x05c53000 0x0 0x1000>,
+			      <0x0 0x05c66000 0x0 0x400>,
+			      <0x0 0x05c68000 0x0 0x400>,
+			      <0x0 0x05c6f000 0x0 0x4000>,
+			      <0x0 0x05c76000 0x0 0x4000>;
+			reg-names = "top",
+				    "csid0",
+				    "csid1",
+				    "csiphy0",
+				    "csiphy1",
+				    "csitpg0",
+				    "csitpg1",
+				    "vfe0",
+				    "vfe1";
+
+			clocks = <&gcc GCC_CAMERA_AHB_CLK>,
+				 <&gcc GCC_CAMSS_AXI_CLK>,
+				 <&gcc GCC_CAMSS_NRT_AXI_CLK>,
+				 <&gcc GCC_CAMSS_RT_AXI_CLK>,
+				 <&gcc GCC_CAMSS_TFE_0_CSID_CLK>,
+				 <&gcc GCC_CAMSS_TFE_1_CSID_CLK>,
+				 <&gcc GCC_CAMSS_CPHY_0_CLK>,
+				 <&gcc GCC_CAMSS_CSI0PHYTIMER_CLK>,
+				 <&gcc GCC_CAMSS_CPHY_1_CLK>,
+				 <&gcc GCC_CAMSS_CSI1PHYTIMER_CLK>,
+				 <&gcc GCC_CAMSS_TOP_AHB_CLK>,
+				 <&gcc GCC_CAMSS_TFE_0_CLK>,
+				 <&gcc GCC_CAMSS_TFE_0_CPHY_RX_CLK>,
+				 <&gcc GCC_CAMSS_TFE_1_CLK>,
+				 <&gcc GCC_CAMSS_TFE_1_CPHY_RX_CLK>;
+			clock-names = "ahb",
+				      "axi",
+				      "camnoc_nrt_axi",
+				      "camnoc_rt_axi",
+				      "csi0",
+				      "csi1",
+				      "csiphy0",
+				      "csiphy0_timer",
+				      "csiphy1",
+				      "csiphy1_timer",
+				      "top_ahb",
+				      "vfe0",
+				      "vfe0_cphy_rx",
+				      "vfe1",
+				      "vfe1_cphy_rx";
+
+			interrupts = <GIC_SPI 210 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 212 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 72 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 73 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 309 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 310 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 211 IRQ_TYPE_EDGE_RISING 0>,
+				     <GIC_SPI 213 IRQ_TYPE_EDGE_RISING 0>;
+			interrupt-names = "csid0",
+					  "csid1",
+					  "csiphy0",
+					  "csiphy1",
+					  "csitpg0",
+					  "csitpg1",
+					  "vfe0",
+					  "vfe1";
+
+			interconnects = <&mem_noc MASTER_AMPSS_M0 RPM_ACTIVE_TAG
+					 &config_noc SLAVE_CAMERA_CFG RPM_ACTIVE_TAG>,
+					<&mmrt_virt MASTER_CAMNOC_HF RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>,
+					<&mmnrt_virt MASTER_CAMNOC_SF RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>;
+			interconnect-names = "ahb",
+					     "hf_mnoc",
+					     "sf_mnoc";
+
+			iommus = <&apps_smmu 0x400 0x0>;
+
+			power-domains = <&gcc GCC_CAMSS_TOP_GDSC>;
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				camss_csiphy0_port: port@0 {
+					reg = <0>;
+				};
+
+				camss_csiphy1_port: port@1 {
+					reg = <1>;
+				};
+			};
+		};
+
+		cci: cci@5c1b000 {
+			compatible = "qcom,shikra-cci", "qcom,msm8996-cci";
+			reg = <0x0 0x05c1b000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 206 IRQ_TYPE_EDGE_RISING 0>;
+
+			clocks = <&gcc GCC_CAMSS_TOP_AHB_CLK>,
+				 <&gcc GCC_CAMSS_CCI_0_CLK>;
+			clock-names = "ahb",
+				      "cci";
+
+			power-domains = <&gcc GCC_CAMSS_TOP_GDSC>;
+
+			pinctrl-0 = <&cci_i2c0_default &cci_i2c1_default>;
+			pinctrl-1 = <&cci_i2c0_sleep &cci_i2c1_sleep>;
+			pinctrl-names = "default", "sleep";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+
+			cci_i2c0: i2c-bus@0 {
+				reg = <0>;
+				clock-frequency = <400000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			cci_i2c1: i2c-bus@1 {
+				reg = <1>;
+				clock-frequency = <400000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
 		remoteproc_mpss: remoteproc@6080000 {
 			compatible = "qcom,shikra-mpss-pas";
 			reg = <0x0 0x06080000 0x0 0x100>;
@@ -1958,41 +2487,287 @@
 			};
 		};
 
-		remoteproc_lpaicp: remoteproc@b800000 {
-			compatible = "qcom,shikra-lpaicp-pas";
-			reg = <0x0 0x0b800000 0x0 0x200000>;
+		usb_2: usb@4c00000 {
+			compatible = "qcom,shikra-dwc3", "qcom,snps-dwc3";
+			reg = <0x0 0x04c00000 0x0 0xfc100>;
 
-			interrupts-extended = <&intc GIC_SPI 257 IRQ_TYPE_EDGE_RISING 0>,
-					      <&lmcu_smp2p_in 0 IRQ_TYPE_NONE>,
-					      <&lmcu_smp2p_in 1 IRQ_TYPE_NONE>,
-					      <&lmcu_smp2p_in 2 IRQ_TYPE_NONE>,
-					      <&lmcu_smp2p_in 3 IRQ_TYPE_NONE>;
+			clocks = <&gcc GCC_CFG_NOC_USB2_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB20_MASTER_CLK>,
+				 <&gcc GCC_SYS_NOC_USB2_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB20_SLEEP_CLK>,
+				 <&gcc GCC_USB20_MOCK_UTMI_CLK>;
+			clock-names = "cfg_noc",
+				      "core",
+				      "iface",
+				      "sleep",
+				      "mock_utmi";
 
-			interrupt-names = "wdog",
-					  "fatal",
-					  "ready",
-					  "handover",
-					  "stop-ack";
+			assigned-clocks = <&gcc GCC_USB20_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB20_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <133333333>;
 
-			clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>;
-			clock-names = "xo";
+			interrupts-extended = <&intc GIC_SPI 507 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&intc GIC_SPI 509 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&intc GIC_SPI 508 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&mpm 59 IRQ_TYPE_LEVEL_HIGH>,
+					      <&mpm 58 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc_usb3",
+					  "pwr_event",
+					  "hs_phy_irq",
+					  "dp_hs_phy_irq",
+					  "dm_hs_phy_irq";
 
-			memory-region = <&lmcu_mem &lmcu_dtb_mem>;
+			iommus = <&apps_smmu 0x140 0x0>;
 
-			qcom,smem-states = <&lmcu_smp2p_out 0>;
-			qcom,smem-state-names = "stop";
+			maximum-speed = "high-speed";
 
-			status = "disabled";
+			phys = <&usb_2_hsphy>;
+			phy-names = "usb2-phy";
 
-			glink-edge {
-				interrupts = <GIC_SPI 286 IRQ_TYPE_EDGE_RISING 0>;
-				mboxes = <&apcs_glb 9>;
-				qcom,remote-pid = <26>;
-				label = "lpaicp";
-			};
-		};
+			power-domains = <&gcc GCC_USB20_GDSC>;
 
-		gpucc: clock-controller@5990000 {
+			qcom,select-utmi-as-pipe-clk;
+			resets = <&gcc GCC_USB20_BCR>;
+
+			interconnects = <&system_noc MASTER_USB2_0 RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>,
+					<&mem_noc MASTER_AMPSS_M0 RPM_ACTIVE_TAG
+					 &config_noc SLAVE_USB2 RPM_ACTIVE_TAG>;
+			interconnect-names = "usb-ddr", "apps-usb";
+
+			snps,dis_u2_susphy_quirk;
+			snps,dis_enblslpm_quirk;
+			snps,has-lpm-erratum;
+			snps,hird-threshold = /bits/ 8 <0x10>;
+
+			usb-role-switch;
+			wakeup-source;
+
+			status = "disabled";
+
+			port {
+				usb_2_dwc3_hs: endpoint {
+				};
+			};
+		};
+
+		usb_1: usb@4e00000 {
+			compatible = "qcom,shikra-dwc3", "qcom,snps-dwc3";
+			reg = <0x0 0x04e00000 0x0 0xfc100>;
+
+			clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB30_PRIM_MASTER_CLK>,
+				 <&gcc GCC_SYS_NOC_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB30_PRIM_SLEEP_CLK>,
+				 <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>;
+			clock-names = "cfg_noc",
+				      "core",
+				      "iface",
+				      "sleep",
+				      "mock_utmi";
+
+			assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_PRIM_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <133333333>;
+
+			interrupts-extended = <&intc GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&intc GIC_SPI 302 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&intc GIC_SPI 254 IRQ_TYPE_LEVEL_HIGH 0>,
+					      <&mpm 91 IRQ_TYPE_LEVEL_HIGH>,
+					      <&mpm 90 IRQ_TYPE_LEVEL_HIGH>,
+					      <&mpm 12 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc_usb3",
+					  "pwr_event",
+					  "hs_phy_irq",
+					  "dp_hs_phy_irq",
+					  "dm_hs_phy_irq",
+					  "ss_phy_irq";
+
+			iommus = <&apps_smmu 0x120 0x0>;
+
+			phys = <&usb_1_hsphy>, <&usb_qmpphy QMP_USB43DP_USB3_PHY>;
+			phy-names = "usb2-phy", "usb3-phy";
+
+			power-domains = <&gcc GCC_USB30_PRIM_GDSC>;
+
+			resets = <&gcc GCC_USB30_PRIM_BCR>;
+
+			interconnects = <&system_noc MASTER_USB3 RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>,
+					<&mem_noc MASTER_AMPSS_M0 RPM_ACTIVE_TAG
+					 &config_noc SLAVE_USB3 RPM_ACTIVE_TAG>;
+			interconnect-names = "usb-ddr", "apps-usb";
+
+			snps,dis-u1-entry-quirk;
+			snps,dis-u2-entry-quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_u3_susphy_quirk;
+			snps,dis_enblslpm_quirk;
+			snps,has-lpm-erratum;
+			snps,hird-threshold = /bits/ 8 <0x10>;
+			snps,usb3_lpm_capable;
+			snps,parkmode-disable-ss-quirk;
+
+			usb-role-switch;
+
+			wakeup-source;
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_1_dwc3_hs: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_1_dwc3_ss: endpoint {
+						remote-endpoint = <&usb_qmpphy_usb_ss_in>;
+					};
+				};
+			};
+		};
+
+		remoteproc_lpaicp: remoteproc@b800000 {
+			compatible = "qcom,shikra-lpaicp-pas";
+			reg = <0x0 0x0b800000 0x0 0x200000>;
+
+			interrupts-extended = <&intc GIC_SPI 257 IRQ_TYPE_EDGE_RISING 0>,
+					      <&lmcu_smp2p_in 0 IRQ_TYPE_NONE>,
+					      <&lmcu_smp2p_in 1 IRQ_TYPE_NONE>,
+					      <&lmcu_smp2p_in 2 IRQ_TYPE_NONE>,
+					      <&lmcu_smp2p_in 3 IRQ_TYPE_NONE>;
+
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack";
+
+			clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>;
+			clock-names = "xo";
+
+			memory-region = <&lmcu_mem &lmcu_dtb_mem>;
+
+			qcom,smem-states = <&lmcu_smp2p_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			glink-edge {
+				interrupts = <GIC_SPI 286 IRQ_TYPE_EDGE_RISING 0>;
+				mboxes = <&apcs_glb 9>;
+				qcom,remote-pid = <26>;
+				label = "lpaicp";
+			};
+		};
+
+		gpu: gpu@5900000 {
+			compatible = "qcom,adreno-07000400", "qcom,adreno";
+			reg = <0x0 0x05900000 0x0 0x40000>,
+			      <0x0 0x0599e000 0x0 0x1000>,
+			      <0x0 0x05961000 0x0 0x800>;
+			reg-names = "kgsl_3d0_reg_memory",
+				    "cx_mem",
+				    "cx_dbgc";
+
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gpucc GPU_CC_GX_GFX3D_CLK>,
+				 <&gpucc GPU_CC_AHB_CLK>,
+				 <&gcc GCC_DDRSS_GPU_AXI_CLK>,
+				 <&gcc GCC_GPU_MEMNOC_GFX_CLK>,
+				 <&gpucc GPU_CC_CX_GMU_CLK>,
+				 <&gpucc GPU_CC_CXO_CLK>;
+			clock-names = "core",
+				      "iface",
+				      "mem_iface",
+				      "alt_mem_iface",
+				      "gmu",
+				      "xo";
+
+			interconnects = <&mem_noc MASTER_GRAPHICS_3D RPM_ALWAYS_TAG
+					 &mc_virt SLAVE_EBI_CH0 RPM_ALWAYS_TAG>;
+			interconnect-names = "gfx-mem";
+
+			iommus = <&adreno_smmu 0 1>;
+			operating-points-v2 = <&gpu_opp_table>;
+			power-domains = <&rpmpd RPMPD_VDDCX>;
+			qcom,gmu = <&gmu_wrapper>;
+
+			#cooling-cells = <2>;
+
+			status = "disabled";
+
+			gpu_zap_shader: zap-shader {
+				memory-region = <&gpu_micro_code_mem>;
+			};
+
+			gpu_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-1142400000 {
+					opp-hz = /bits/ 64 <1142400000>;
+					required-opps = <&rpmpd_opp_turbo_plus>;
+					opp-peak-kBps = <8171875>;
+				};
+
+				opp-1017600000 {
+					opp-hz = /bits/ 64 <1017600000>;
+					required-opps = <&rpmpd_opp_turbo>;
+					opp-peak-kBps = <8171875>;
+				};
+
+				opp-921600000 {
+					opp-hz = /bits/ 64 <921600000>;
+					required-opps = <&rpmpd_opp_nom_plus>;
+					opp-peak-kBps = <7046875>;
+				};
+
+				opp-844800000 {
+					opp-hz = /bits/ 64 <844800000>;
+					required-opps = <&rpmpd_opp_nom>;
+					opp-peak-kBps = <6074218>;
+				};
+
+				opp-672000000 {
+					opp-hz = /bits/ 64 <672000000>;
+					required-opps = <&rpmpd_opp_svs_plus>;
+					opp-peak-kBps = <5285156>;
+				};
+
+				opp-537600000 {
+					opp-hz = /bits/ 64 <537600000>;
+					required-opps = <&rpmpd_opp_svs>;
+					opp-peak-kBps = <3972656>;
+				};
+
+				opp-355200000 {
+					opp-hz = /bits/ 64 <355200000>;
+					required-opps = <&rpmpd_opp_low_svs>;
+					opp-peak-kBps = <2136718>;
+				};
+			};
+		};
+
+		gmu_wrapper: gmu@596a000 {
+			compatible = "qcom,adreno-gmu-wrapper";
+			reg = <0x0 0x0596a000 0x0 0x30000>;
+			reg-names = "gmu";
+			power-domains = <&gpucc GPU_CX_GDSC>,
+					<&gpucc GPU_GX_GDSC>;
+			power-domain-names = "cx",
+					     "gx";
+		};
+
+		gpucc: clock-controller@5990000 {
 			compatible = "qcom,shikra-gpucc";
 			reg = <0x0 0x05990000 0x0 0x9000>;
 			clocks = <&gcc GCC_GPU_CFG_AHB_CLK>,
@@ -2005,25 +2780,1253 @@
 			#power-domain-cells = <1>;
 		};
 
-		dispcc: clock-controller@5f00000 {
-			compatible = "qcom,shikra-dispcc", "qcom,qcm2290-dispcc";
-			reg = <0x0 0x05f00000 0x0 0x20000>;
+		adreno_smmu: iommu@59a0000 {
+			compatible = "qcom,shikra-smmu-500", "qcom,adreno-smmu",
+				     "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x059a0000 0x0 0x10000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+
+			interrupts = <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 172 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gpucc GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK>,
+				 <&gcc GCC_GPU_MEMNOC_GFX_CLK>,
+				 <&gcc GCC_GPU_SNOC_DVM_GFX_CLK>,
+				 <&gpucc GPU_CC_AHB_CLK>;
+			clock-names = "hlos",
+				      "bus",
+				      "iface",
+				      "ahb";
+
+			power-domains = <&gpucc GPU_CX_GDSC>;
+		};
+
+		dispcc: clock-controller@5f00000 {
+			compatible = "qcom,shikra-dispcc", "qcom,qcm2290-dispcc";
+			reg = <0x0 0x05f00000 0x0 0x20000>;
+			clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>,
+				 <&rpmcc RPM_SMD_XO_A_CLK_SRC>,
+				 <&gcc GCC_DISP_GPLL0_CLK_SRC>,
+				 <&gcc GCC_DISP_GPLL0_DIV_CLK_SRC>,
+				 <0>,
+				 <0>;
+			clock-names = "bi_tcxo",
+				      "bi_tcxo_ao",
+				      "gcc_disp_gpll0_clk_src",
+				      "gcc_disp_gpll0_div_clk_src",
+				      "dsi0_phy_pll_out_byteclk",
+				      "dsi0_phy_pll_out_dsiclk";
+			power-domains = <&rpmpd RPMPD_VDDCX>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		ctcu@8001000 {
+			compatible = "qcom,shikra-ctcu", "qcom,sa8775p-ctcu";
+			reg = <0x0 0x08001000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb";
+
+			in-ports {
+				port {
+					ctcu_in0: endpoint {
+						remote-endpoint = <&etr0_out>;
+					};
+				};
+			};
+		};
+
+		stm@8002000 {
+			compatible = "arm,coresight-stm", "arm,primecell";
+			reg = <0x0 0x08002000 0x0 0x1000>,
+			      <0x0 0x0e280000 0x0 0x180000>;
+			reg-names = "stm-base",
+				    "stm-stimulus-base";
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			out-ports {
+				port {
+					stm_out: endpoint {
+						remote-endpoint = <&funnel_in0_in7>;
+					};
+				};
+			};
+		};
+
+		tpdm@8003000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08003000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_dcc";
+			qcom,cmb-element-bits = <32>;
+
+			out-ports {
+				port {
+					tpdm_dcc_out: endpoint {
+						remote-endpoint = <&tpda_qdss_in0>;
+					};
+				};
+			};
+		};
+
+		tpda@8004000 {
+			compatible = "qcom,coresight-tpda", "arm,primecell";
+			reg = <0x0 0x08004000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					tpda_qdss_in0: endpoint {
+						remote-endpoint = <&tpdm_dcc_out>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					tpda_qdss_in1: endpoint {
+						remote-endpoint = <&tpdm_spdm_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					tpda_qdss_out: endpoint {
+						remote-endpoint = <&funnel_in0_in6>;
+					};
+				};
+			};
+		};
+
+		tpdm@800f000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x0800f000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_spdm";
+			qcom,cmb-element-bits = <32>;
+
+			out-ports {
+				port {
+					tpdm_spdm_out: endpoint {
+						remote-endpoint = <&tpda_qdss_in1>;
+					};
+				};
+			};
+		};
+
+		funnel@8041000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x08041000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@5 {
+					reg = <5>;
+
+					funnel_in0_in5: endpoint {
+						remote-endpoint = <&snoc_out>;
+					};
+				};
+
+				port@6 {
+					reg = <6>;
+
+					funnel_in0_in6: endpoint {
+						remote-endpoint = <&tpda_qdss_out>;
+					};
+				};
+
+				port@7 {
+					reg = <7>;
+
+					funnel_in0_in7: endpoint {
+						remote-endpoint = <&stm_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_in0_out: endpoint {
+						remote-endpoint = <&funnel_merg_in0>;
+					};
+				};
+			};
+		};
+
+		funnel@8042000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x08042000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					reg = <1>;
+
+					funnel_in1_in1: endpoint {
+						remote-endpoint = <&tpda_aodbg_out>;
+					};
+				};
+
+				port@4 {
+					reg = <4>;
+
+					funnel_in1_in4: endpoint {
+						remote-endpoint = <&modem_etm0_out>;
+					};
+				};
+
+				port@6 {
+					reg = <6>;
+
+					funnel_in1_in6: endpoint {
+						remote-endpoint = <&funnel_cpuss1_out>;
+					};
+				};
+
+				port@7 {
+					reg = <7>;
+
+					funnel_in1_in7: endpoint {
+						remote-endpoint = <&funnel_center_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_in1_out: endpoint {
+						remote-endpoint = <&funnel_merg_in1>;
+					};
+				};
+			};
+		};
+
+		funnel@8045000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x08045000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					funnel_merg_in0: endpoint {
+						remote-endpoint = <&funnel_in0_out>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					funnel_merg_in1: endpoint {
+						remote-endpoint = <&funnel_in1_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_merg_out: endpoint {
+						remote-endpoint = <&tmc_etf_in>;
+					};
+				};
+			};
+		};
+
+		replicator@8046000 {
+			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
+			reg = <0x0 0x08046000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					replicator_qdss_in: endpoint {
+						remote-endpoint = <&tmc_etf_out>;
+					};
+				};
+			};
+
+			out-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					replicator_qdss_out0: endpoint {
+						remote-endpoint = <&etr0_in>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					replicator_qdss_out1: endpoint {
+						remote-endpoint = <&replicator_eud_in>;
+					};
+				};
+			};
+		};
+
+		tmc@8047000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0x08047000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					tmc_etf_in: endpoint {
+						remote-endpoint = <&funnel_merg_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					tmc_etf_out: endpoint {
+						remote-endpoint = <&replicator_qdss_in>;
+					};
+				};
+			};
+		};
+
+		tmc@8048000 {
+			compatible = "arm,coresight-tmc", "arm,primecell";
+			reg = <0x0 0x08048000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			iommus = <&apps_smmu 0x0160 0x0>;
+			arm,scatter-gather;
+
+			in-ports {
+				port {
+					etr0_in: endpoint {
+						remote-endpoint = <&replicator_qdss_out0>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					etr0_out: endpoint {
+						remote-endpoint = <&ctcu_in0>;
+					};
+				};
+			};
+		};
+
+		replicator@804a000 {
+			compatible = "arm,coresight-dynamic-replicator", "arm,primecell";
+			reg = <0x0 0x0804a000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					replicator_eud_in: endpoint {
+						remote-endpoint = <&replicator_qdss_out1>;
+					};
+				};
+			};
+
+			out-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@1 {
+					reg = <1>;
+
+					replicator_eud_out1: endpoint {
+						remote-endpoint = <&eud_in>;
+					};
+				};
+			};
+		};
+
+		tpdm@8800000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08800000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_cdsp";
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <4>;
+
+			out-ports {
+				port {
+					tpdm_cdsp_out: endpoint {
+						remote-endpoint = <&funnel_cdsp_in0>;
+					};
+				};
+			};
+		};
+
+		funnel@8801000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x08801000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					funnel_cdsp_in0: endpoint {
+						remote-endpoint = <&tpdm_cdsp_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_cdsp_out0: endpoint {
+						remote-endpoint = <&tpda_center_in3>;
+					};
+				};
+			};
+		};
+
+		cti@8807000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08807000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_turing_q6";
+		};
+
+		cti@8833000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08833000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_mss_q6";
+		};
+
+		tpdm@8840000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08840000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_vsense";
+			qcom,cmb-element-bits = <32>;
+
+			out-ports {
+				port {
+					tpdm_vsense_out: endpoint {
+						remote-endpoint = <&tpda_center_in6>;
+					};
+				};
+			};
+		};
+
+		tpdm@8844000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08844000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_dlct_1";
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <32>;
+
+			out-ports {
+				port {
+					tpdm_dlct_1_out: endpoint {
+						remote-endpoint = <&tpda_center_in13>;
+					};
+				};
+			};
+		};
+
+		tpda@8845000 {
+			compatible = "qcom,coresight-tpda", "arm,primecell";
+			reg = <0x0 0x08845000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					tpda_center_in0: endpoint {
+						remote-endpoint = <&funnel_mcu_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					tpda_center_in2: endpoint {
+						remote-endpoint = <&tpdm_dlct_out>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+
+					tpda_center_in3: endpoint {
+						remote-endpoint = <&funnel_cdsp_out0>;
+					};
+				};
+
+				port@4 {
+					reg = <4>;
+
+					tpda_center_in4: endpoint {
+						remote-endpoint = <&funnel_ddr_out0>;
+					};
+				};
+
+				port@6 {
+					reg = <6>;
+
+					tpda_center_in6: endpoint {
+						remote-endpoint = <&tpdm_vsense_out>;
+					};
+				};
+
+				port@7 {
+					reg = <7>;
+
+					tpda_center_in7: endpoint {
+						remote-endpoint = <&tpdm_prng_out>;
+					};
+				};
+
+				port@8 {
+					reg = <8>;
+
+					tpda_center_in8: endpoint {
+						remote-endpoint = <&tpdm_west_out>;
+					};
+				};
+
+				port@9 {
+					reg = <9>;
+
+					tpda_center_in9: endpoint {
+						remote-endpoint = <&tpdm_qm_out>;
+					};
+				};
+
+				port@a {
+					reg = <0xa>;
+
+					tpda_center_in10: endpoint {
+						remote-endpoint = <&tpdm_pimem_out>;
+					};
+				};
+
+				port@d {
+					reg = <0xd>;
+
+					tpda_center_in13: endpoint {
+						remote-endpoint = <&tpdm_dlct_1_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					tpda_center_out: endpoint {
+						remote-endpoint = <&funnel_center_in0>;
+					};
+				};
+			};
+		};
+
+		funnel@8846000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x08846000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					funnel_center_in0: endpoint {
+						remote-endpoint = <&tpda_center_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_center_out: endpoint {
+						remote-endpoint = <&funnel_in1_in7>;
+					};
+				};
+			};
+		};
+
+		tpdm@884c000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x0884c000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_prng";
+			qcom,cmb-element-bits = <32>;
+
+			out-ports {
+				port {
+					tpdm_prng_out: endpoint {
+						remote-endpoint = <&tpda_center_in7>;
+					};
+				};
+			};
+		};
+
+		tpdm@8850000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08850000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_pimem";
+			qcom,cmb-element-bits = <64>;
+			qcom,dsb-element-bits = <32>;
+
+			out-ports {
+				port {
+					tpdm_pimem_out: endpoint {
+						remote-endpoint = <&tpda_center_in10>;
+					};
+				};
+			};
+		};
+
+		tpdm@8980000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08980000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_mcu";
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <16>;
+
+			out-ports {
+				port {
+					tpdm_mcu_out: endpoint {
+						remote-endpoint = <&funnel_mcu_in0>;
+					};
+				};
+			};
+		};
+
+		funnel@8982000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x08982000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					funnel_mcu_in0: endpoint {
+						remote-endpoint = <&tpdm_mcu_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_mcu_out: endpoint {
+						remote-endpoint = <&tpda_center_in0>;
+					};
+				};
+			};
+		};
+
+		tpdm@89d0000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x089d0000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_qm";
+			qcom,dsb-element-bits = <32>;
+
+			out-ports {
+				port {
+					tpdm_qm_out: endpoint {
+						remote-endpoint = <&tpda_center_in9>;
+					};
+				};
+			};
+		};
+
+		tpdm@8a01000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08a01000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_mapss";
+			qcom,cmb-element-bits = <32>;
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <16>;
+
+			out-ports {
+				port {
+					tpdm_mapss_out: endpoint {
+						remote-endpoint = <&tpda_aodbg_in>;
+					};
+				};
+			};
+		};
+
+		cti@8a02000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08a02000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_mapss";
+		};
+
+		tpda@8a04000 {
+			compatible = "qcom,coresight-tpda", "arm,primecell";
+			reg = <0x0 0x08a04000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				port {
+					tpda_aodbg_in: endpoint {
+						remote-endpoint = <&tpdm_mapss_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					tpda_aodbg_out: endpoint {
+						remote-endpoint = <&funnel_in1_in1>;
+					};
+				};
+			};
+		};
+
+		tpdm@8a58000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08a58000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_west";
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <16>;
+
+			out-ports {
+				port {
+					tpdm_west_out: endpoint {
+						remote-endpoint = <&tpda_center_in8>;
+					};
+				};
+			};
+		};
+
+		cti@8b30000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b30000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_cortex_m3";
+		};
+
+		tpdm@8b58000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08b58000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_dlct";
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <16>;
+
+			out-ports {
+				port {
+					tpdm_dlct_out: endpoint {
+						remote-endpoint = <&tpda_center_in2>;
+					};
+				};
+			};
+		};
+
+		cti@8b59000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b59000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_dlct_0";
+		};
+
+		cti@8b5a000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b5a000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_dlct_1";
+		};
+
+		cti@8b5b000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b5b000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_dlct_2";
+		};
+
+		cti@8b5c000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b5c000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_dlct_3";
+		};
+
+		tpdm@8b60000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x08b60000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_ddr";
+			qcom,cmb-element-bits = <32>;
+			qcom,cmb-msrs-num = <1>;
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <32>;
+
+			out-ports {
+				port {
+					tpdm_ddr_out: endpoint {
+						remote-endpoint = <&funnel_ddr_in0>;
+					};
+				};
+			};
+		};
+
+		cti@8b62000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b62000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_ddr_dl_0";
+		};
+
+		funnel@8b65000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x08b65000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					funnel_ddr_in0: endpoint {
+						remote-endpoint = <&tpdm_ddr_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_ddr_out0: endpoint {
+						remote-endpoint = <&tpda_center_in4>;
+					};
+				};
+			};
+		};
+
+		cti@8b70000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b70000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_ddr_dl_0_1";
+		};
+
+		cti@8b71000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x08b71000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_ddr_dl_1_1";
+		};
+
+		cti@9020000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x09020000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_pe0";
+		};
+
+		cti@90e0000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x090e0000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_cluster";
+		};
+
+		cti@9120000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x09120000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_pe1";
+		};
+
+		cti@9220000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x09220000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_pe2";
+		};
+
+		cti@9320000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x09320000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_pe3";
+		};
+
+		funnel@9800000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x09800000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					funnel_cpuss0_in0: endpoint {
+						remote-endpoint = <&etm0_out>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					funnel_cpuss0_in1: endpoint {
+						remote-endpoint = <&etm1_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					funnel_cpuss0_in2: endpoint {
+						remote-endpoint = <&etm2_out>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+
+					funnel_cpuss0_in3: endpoint {
+						remote-endpoint = <&etm3_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_cpuss0_out: endpoint {
+						remote-endpoint = <&funnel_cpuss1_in0>;
+					};
+				};
+			};
+		};
+
+		funnel@9810000 {
+			compatible = "arm,coresight-dynamic-funnel", "arm,primecell";
+			reg = <0x0 0x09810000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					funnel_cpuss1_in0: endpoint {
+						remote-endpoint = <&funnel_cpuss0_out>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+
+					funnel_cpuss1_in3: endpoint {
+						remote-endpoint = <&tpda_apss_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					funnel_cpuss1_out: endpoint {
+						remote-endpoint = <&funnel_in1_in6>;
+					};
+				};
+			};
+		};
+
+		cti@982b000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x0982b000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_riscv";
+		};
+
+		tpdm@9860000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x09860000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_actpm";
+			qcom,cmb-element-bits = <64>;
+			qcom,cmb-msrs-num = <1>;
+
+			out-ports {
+				port {
+					tpdm_actpm_out: endpoint {
+						remote-endpoint = <&tpda_apss_in2>;
+					};
+				};
+			};
+		};
+
+		tpdm@9861000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x09861000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_hwe";
+			qcom,dsb-element-bits = <32>;
+			qcom,dsb-msrs-num = <32>;
+
+			out-ports {
+				port {
+					tpdm_hwe_out: endpoint {
+						remote-endpoint = <&tpda_apss_in3>;
+					};
+				};
+			};
+		};
+
+		tpda@9863000 {
+			compatible = "qcom,coresight-tpda", "arm,primecell";
+			reg = <0x0 0x09863000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			in-ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					tpda_apss_in0: endpoint {
+						remote-endpoint = <&tpdm_llm_silver_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					tpda_apss_in2: endpoint {
+						remote-endpoint = <&tpdm_actpm_out>;
+					};
+				};
+
+				port@3 {
+					reg = <3>;
+
+					tpda_apss_in3: endpoint {
+						remote-endpoint = <&tpdm_hwe_out>;
+					};
+				};
+			};
+
+			out-ports {
+				port {
+					tpda_apss_out: endpoint {
+						remote-endpoint = <&funnel_cpuss1_in3>;
+					};
+				};
+			};
+		};
+
+		tpdm@98a0000 {
+			compatible = "qcom,coresight-tpdm", "arm,primecell";
+			reg = <0x0 0x098a0000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+
+			label = "tpdm_llm_silver";
+			qcom,cmb-element-bits = <32>;
+			qcom,cmb-msrs-num = <12>;
+
+			out-ports {
+				port {
+					tpdm_llm_silver_out: endpoint {
+						remote-endpoint = <&tpda_apss_in0>;
+					};
+				};
+			};
+		};
+
+		cti@98e0000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x098e0000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_0";
+		};
+
+		cti@98f0000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x098f0000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_1";
+		};
+
+		cti@9900000 {
+			compatible = "arm,coresight-cti", "arm,primecell";
+			reg = <0x0 0x09900000 0x0 0x1000>;
+
+			clocks = <&rpmcc RPM_SMD_QDSS_CLK>;
+			clock-names = "apb_pclk";
+			label = "cti_apss_2";
+		};
+
+		audiocorecc: clock-controller@a0a0000 {
+			compatible = "qcom,shikra-audiocorecc";
+			reg = <0x0 0x0a0a0000 0x0 0x10000>;
 			clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>,
-				 <&rpmcc RPM_SMD_XO_A_CLK_SRC>,
-				 <&gcc GCC_DISP_GPLL0_CLK_SRC>,
-				 <&gcc GCC_DISP_GPLL0_DIV_CLK_SRC>,
-				 <0>,
+				 <&sleep_clk>,
 				 <0>;
-			clock-names = "bi_tcxo",
-				      "bi_tcxo_ao",
-				      "gcc_disp_gpll0_clk_src",
-				      "gcc_disp_gpll0_div_clk_src",
-				      "dsi0_phy_pll_out_byteclk",
-				      "dsi0_phy_pll_out_dsiclk";
-			power-domains = <&rpmpd RPMPD_VDDCX>;
 			#clock-cells = <1>;
+			status = "reserved"; /* Owned by Modem firmware */
+		};
+
+		audiocore_csr: reset-controller@a0b4000 {
+			compatible = "qcom,shikra-audiocore-csr";
+			reg = <0x0 0x0a0b4000 0x0 0x1000>;
 			#reset-cells = <1>;
-			#power-domain-cells = <1>;
 		};
 
 		sram@c11e000 {
@@ -2377,6 +4380,7 @@
 		};
 
 		gpuss-thermal {
+			polling-delay-passive = <10>;
 			thermal-sensors = <&tsens0 6>;
 
 			trips {
@@ -2386,12 +4390,25 @@
 					type = "hot";
 				};
 
-				gpuss-critical {
+				gpuss_alert0: gpuss-alert0 {
 					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				gpuss-critical {
+					temperature = <120000>;
 					hysteresis = <0>;
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		nsp-thermal {
@@ -2521,6 +4538,21 @@
 		};
 	};
 
+	snoc {
+		compatible = "arm,coresight-dummy-source";
+
+		label = "snoc";
+		arm,static-trace-id = <18>;
+
+		out-ports {
+			port {
+				snoc_out: endpoint {
+					remote-endpoint = <&funnel_in0_in5>;
+				};
+			};
+		};
+	};
+
 	timer {
 		compatible = "arm,armv8-timer";
 
diff --git a/arch/arm64/boot/dts/qcom/sm4450.dtsi b/arch/arm64/boot/dts/qcom/sm4450.dtsi
index b7dc40f3778c4f..bc9c788b92b1f5 100644
--- a/arch/arm64/boot/dts/qcom/sm4450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm4450.dtsi
@@ -11,7 +11,6 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 
 / {
diff --git a/arch/arm64/boot/dts/qcom/sm6115.dtsi b/arch/arm64/boot/dts/qcom/sm6115.dtsi
index 3520802d1fed57..0f2af8abacc3a0 100644
--- a/arch/arm64/boot/dts/qcom/sm6115.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6115.dtsi
@@ -1665,7 +1665,7 @@
 
 			resets = <&gcc GCC_USB30_PRIM_BCR>;
 			power-domains = <&gcc GCC_USB30_PRIM_GDSC>;
-			 /* TODO: USB<->IPA path */
+			/* TODO: USB<->IPA path */
 			interconnects = <&system_noc MASTER_USB3 RPM_ALWAYS_TAG
 					 &bimc SLAVE_EBI_CH0 RPM_ALWAYS_TAG>,
 					<&bimc MASTER_AMPSS_M0 RPM_ALWAYS_TAG
diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi
index 15d38b7e803c57..57773c181f16f8 100644
--- a/arch/arm64/boot/dts/qcom/sm6350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi
@@ -2242,6 +2242,239 @@
 			/* SM6350 seems to have cci1_i2c1 on gpio2 & gpio3 but unused downstream */
 		};
 
+		camss: isp@acb3000 {
+			compatible = "qcom,sm6350-camss";
+
+			reg = <0x0 0x0acb3000 0x0 0x1000>,
+			      <0x0 0x0acba000 0x0 0x1000>,
+			      <0x0 0x0acc1000 0x0 0x1000>,
+			      <0x0 0x0acc8000 0x0 0x1000>,
+			      <0x0 0x0ac65000 0x0 0x1000>,
+			      <0x0 0x0ac66000 0x0 0x1000>,
+			      <0x0 0x0ac67000 0x0 0x1000>,
+			      <0x0 0x0ac68000 0x0 0x1000>,
+			      <0x0 0x0acaf000 0x0 0x4000>,
+			      <0x0 0x0acb6000 0x0 0x4000>,
+			      <0x0 0x0acbd000 0x0 0x4000>,
+			      <0x0 0x0acc4000 0x0 0x4000>,
+			      <0x0 0x0ac18000 0x0 0x3000>,
+			      <0x0 0x0ac00000 0x0 0x6000>,
+			      <0x0 0x0ac10000 0x0 0x8000>,
+			      <0x0 0x0ac6f000 0x0 0x8000>,
+			      <0x0 0x0ac42000 0x0 0x4600>,
+			      <0x0 0x01fc0000 0x0 0x40000>,
+			      <0x0 0x0ac48000 0x0 0x1000>,
+			      <0x0 0x0ac40000 0x0 0x1000>,
+			      <0x0 0x0ac87000 0x0 0xa000>,
+			      <0x0 0x0ac52000 0x0 0x4000>,
+			      <0x0 0x0ac4e000 0x0 0x4000>,
+			      <0x0 0x0ac6b000 0x0 0xa00>;
+			reg-names = "csid0",
+				    "csid1",
+				    "csid2",
+				    "csid_lite",
+				    "csiphy0",
+				    "csiphy1",
+				    "csiphy2",
+				    "csiphy3",
+				    "vfe0",
+				    "vfe1",
+				    "vfe2",
+				    "vfe_lite",
+				    "a5_csr",
+				    "a5_qgic",
+				    "a5_sierra",
+				    "bps",
+				    "camnoc",
+				    "core_top_csr_tcsr",
+				    "cpas_cdm",
+				    "cpas_top",
+				    "ipe",
+				    "jpeg_dma",
+				    "jpeg_enc",
+				    "lrme";
+
+			clocks = <&gcc GCC_CAMERA_AXI_CLK>,
+				 <&camcc CAMCC_SOC_AHB_CLK>,
+				 <&camcc CAMCC_CAMNOC_AXI_CLK>,
+				 <&camcc CAMCC_CORE_AHB_CLK>,
+				 <&camcc CAMCC_CPAS_AHB_CLK>,
+				 <&camcc CAMCC_CSIPHY0_CLK>,
+				 <&camcc CAMCC_CSI0PHYTIMER_CLK>,
+				 <&camcc CAMCC_CSIPHY1_CLK>,
+				 <&camcc CAMCC_CSI1PHYTIMER_CLK>,
+				 <&camcc CAMCC_CSIPHY2_CLK>,
+				 <&camcc CAMCC_CSI2PHYTIMER_CLK>,
+				 <&camcc CAMCC_CSIPHY3_CLK>,
+				 <&camcc CAMCC_CSI3PHYTIMER_CLK>,
+				 <&camcc CAMCC_IFE_0_AXI_CLK>,
+				 <&camcc CAMCC_IFE_0_CLK>,
+				 <&camcc CAMCC_IFE_0_CPHY_RX_CLK>,
+				 <&camcc CAMCC_IFE_0_CSID_CLK>,
+				 <&camcc CAMCC_IFE_1_AXI_CLK>,
+				 <&camcc CAMCC_IFE_1_CLK>,
+				 <&camcc CAMCC_IFE_1_CPHY_RX_CLK>,
+				 <&camcc CAMCC_IFE_1_CSID_CLK>,
+				 <&camcc CAMCC_IFE_2_AXI_CLK>,
+				 <&camcc CAMCC_IFE_2_CLK>,
+				 <&camcc CAMCC_IFE_2_CPHY_RX_CLK>,
+				 <&camcc CAMCC_IFE_2_CSID_CLK>,
+				 <&camcc CAMCC_IFE_LITE_CLK>,
+				 <&camcc CAMCC_IFE_LITE_CPHY_RX_CLK>,
+				 <&camcc CAMCC_IFE_LITE_CSID_CLK>,
+				 <&camcc CAMCC_BPS_CLK>,
+				 <&camcc CAMCC_BPS_AHB_CLK>,
+				 <&camcc CAMCC_BPS_AREG_CLK>,
+				 <&camcc CAMCC_BPS_AXI_CLK>,
+				 <&camcc CAMCC_ICP_CLK>,
+				 <&camcc CAMCC_IPE_0_CLK>,
+				 <&camcc CAMCC_IPE_0_AHB_CLK>,
+				 <&camcc CAMCC_IPE_0_AREG_CLK>,
+				 <&camcc CAMCC_IPE_0_AXI_CLK>,
+				 <&camcc CAMCC_JPEG_CLK>,
+				 <&camcc CAMCC_LRME_CLK>;
+			clock-names = "cam_axi",
+				      "soc_ahb",
+				      "camnoc_axi",
+				      "core_ahb",
+				      "cpas_ahb",
+				      "csiphy0",
+				      "csiphy0_timer",
+				      "csiphy1",
+				      "csiphy1_timer",
+				      "csiphy2",
+				      "csiphy2_timer",
+				      "csiphy3",
+				      "csiphy3_timer",
+				      "vfe0_axi",
+				      "vfe0",
+				      "vfe0_cphy_rx",
+				      "vfe0_csid",
+				      "vfe1_axi",
+				      "vfe1",
+				      "vfe1_cphy_rx",
+				      "vfe1_csid",
+				      "vfe2_axi",
+				      "vfe2",
+				      "vfe2_cphy_rx",
+				      "vfe2_csid",
+				      "vfe_lite",
+				      "vfe_lite_cphy_rx",
+				      "vfe_lite_csid",
+				      "bps",
+				      "bps_ahb",
+				      "bps_areg",
+				      "bps_axi",
+				      "icp",
+				      "ipe0",
+				      "ipe0_ahb",
+				      "ipe0_areg",
+				      "ipe0_axi",
+				      "jpeg",
+				      "lrme";
+
+			interrupts = <GIC_SPI 464 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 466 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 717 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 473 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 477 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 478 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 479 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 461 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 465 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 467 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 718 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 472 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 463 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 459 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 469 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 475 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 474 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 476 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "csid0",
+					  "csid1",
+					  "csid2",
+					  "csid_lite",
+					  "csiphy0",
+					  "csiphy1",
+					  "csiphy2",
+					  "csiphy3",
+					  "vfe0",
+					  "vfe1",
+					  "vfe2",
+					  "vfe_lite",
+					  "a5",
+					  "cpas",
+					  "cpas_cdm",
+					  "jpeg_dma",
+					  "jpeg_enc",
+					  "lrme";
+
+			interconnects = <&gem_noc MASTER_AMPSS_M0 QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS
+					 &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>,
+					<&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS
+					 &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>,
+					<&mmss_noc MASTER_CAMNOC_ICP QCOM_ICC_TAG_ALWAYS
+					 &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "ahb",
+					     "hf_mnoc",
+					     "sf_mnoc",
+					     "sf_icp_mnoc";
+
+			iommus = <&apps_smmu 0x820 0xc0>,
+				 <&apps_smmu 0x840 0x0>,
+				 <&apps_smmu 0x860 0xc0>,
+				 <&apps_smmu 0x880 0x0>,
+				 <&apps_smmu 0xc40 0x20>,
+				 <&apps_smmu 0xc60 0x20>,
+				 <&apps_smmu 0xc80 0x0>,
+				 <&apps_smmu 0xca2 0x0>,
+				 <&apps_smmu 0xcc0 0x20>,
+				 <&apps_smmu 0xce0 0x20>,
+				 <&apps_smmu 0xd00 0x20>,
+				 <&apps_smmu 0xd20 0x20>,
+				 <&apps_smmu 0xd40 0x20>,
+				 <&apps_smmu 0xd60 0x20>;
+
+			power-domains = <&camcc IFE_0_GDSC>,
+					<&camcc IFE_1_GDSC>,
+					<&camcc IFE_2_GDSC>,
+					<&camcc TITAN_TOP_GDSC>,
+					<&camcc BPS_GDSC>,
+					<&camcc IPE_0_GDSC>;
+			power-domain-names = "ife0",
+					     "ife1",
+					     "ife2",
+					     "top",
+					     "bps",
+					     "ipe";
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+				};
+
+				port@1 {
+					reg = <1>;
+				};
+
+				port@2 {
+					reg = <2>;
+				};
+
+				port@3 {
+					reg = <3>;
+				};
+			};
+		};
+
 		camcc: clock-controller@ad00000 {
 			compatible = "qcom,sm6350-camcc";
 			reg = <0x0 0x0ad00000 0x0 0x16000>;
diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
index 5d20ad93b0c544..773304b4759e68 100644
--- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
+++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
@@ -992,6 +992,10 @@
 	};
 };
 
+&pmk8350_pon {
+	status = "disabled";
+};
+
 &pmk8350_rtc {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts b/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts
index 7db9ae1c3cef9b..8d8fef94026454 100644
--- a/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts
+++ b/arch/arm64/boot/dts/qcom/sm7325-nothing-spacewar.dts
@@ -1455,5 +1455,7 @@
 };
 
 &wifi {
+	qcom,calibration-variant = "Nothing_Spacewar";
+
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm7325-xiaomi-lisa.dts b/arch/arm64/boot/dts/qcom/sm7325-xiaomi-lisa.dts
new file mode 100644
index 00000000000000..28a19c96c8c94a
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm7325-xiaomi-lisa.dts
@@ -0,0 +1,1108 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2024, Jens Reidel <adrian@travitia.xyz>
+ * Copyright (c) 2026, Oleksii Onchul <oleksiionchul@gmail.com>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pm7325.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pm8350b.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pmk8350.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "sm7325.dtsi"
+#include "pm7325.dtsi"
+#include "pm8350b.dtsi" /* PM7325B */
+#include "pm8350c.dtsi" /* PM7350C */
+#include "pmk8350.dtsi" /* PMK7325 */
+
+/* The following reserved memory regions have different addresses or sizes */
+/delete-node/ &adsp_mem;
+/delete-node/ &adsp_rpc_remote_heap_mem;
+/delete-node/ &cdsp_mem;
+/delete-node/ &rmtfs_mem;
+
+/ {
+	model = "Xiaomi 11 Lite 5G NE";
+	compatible = "xiaomi,lisa", "qcom,sm7325";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart5;
+		serial1 = &uart7;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		stdout-path = "serial0:115200n8";
+
+		framebuffer0: framebuffer {
+			compatible = "simple-framebuffer";
+			memory-region = <&framebuffer_mem>;
+			width = <1080>;
+			height = <2400>;
+			stride = <(1080 * 4)>;
+			format = "a8r8g8b8";
+
+			clocks = <&gcc GCC_DISP_HF_AXI_CLK>,
+				 <&dispcc DISP_CC_MDSS_MDP_CLK>,
+				 <&dispcc DISP_CC_MDSS_BYTE0_CLK>,
+				 <&dispcc DISP_CC_MDSS_BYTE0_INTF_CLK>,
+				 <&dispcc DISP_CC_MDSS_PCLK0_CLK>,
+				 <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+			power-domains = <&dispcc DISP_CC_MDSS_CORE_GDSC>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&volume_up_default>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&pm7325_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,sm7325-pmic-glink",
+			     "qcom,qcm6490-pmic-glink",
+			     "qcom,pmic-glink";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		orientation-gpios = <&tlmm 140 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu: endpoint {
+						remote-endpoint = <&fsa4480_sbu_mux>;
+					};
+				};
+			};
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		cdsp_secure_heap_mem: cdsp-secure-heap@81800000 {
+			reg = <0x0 0x81800000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		adsp_mem: adsp@86700000 {
+			reg = <0x0 0x86700000 0x0 0x4000000>;
+			no-map;
+		};
+
+		/* Mainline video_mem is downstream cvp_mem */
+		real_video_mem: video@8ad00000 {
+			reg = <0x0 0x8ad00000 0x0 0x500000>;
+			no-map;
+		};
+
+		ipa_gsi_mem: ipa-gsi@8b710000 {
+			reg = <0x0 0x8b710000 0x0 0xa000>;
+			no-map;
+		};
+
+		cdsp_mem: cdsp@9c700000 {
+			reg = <0x0 0x9c700000 0x0 0x1e00000>;
+			no-map;
+		};
+
+		adsp_rpc_remote_heap_mem: adsp-rpc-remote-heap@9e500000 {
+			reg = <0x0 0x9e500000 0x0 0x800000>;
+			no-map;
+		};
+
+		ramoops@a9000000 {
+			compatible = "ramoops";
+			reg = <0x0 0xa9000000 0x0 0x200000>;
+			pmsg-size = <0x200000>;
+			ecc-size = <16>;
+			mem-type = <2>;
+		};
+
+		removed_mem: removed@c0000000 {
+			reg = <0x0 0xc0000000 0x0 0x5100000>;
+			no-map;
+		};
+
+		pil_trustedvm_mem: pil-trustedvm-region@d0800000 {
+			reg = <0x0 0xd0800000 0x0 0x76f7000>;
+			no-map;
+		};
+
+		qrtr_shmem: qrtr-shmem@d7ef7000 {
+			reg = <0x0 0xd7ef7000 0x0 0x9000>;
+			no-map;
+		};
+
+		neuron_block_0_mem: neuron-block@d7f00000 {
+			reg = <0x0 0xd7f00000 0x0 0x80000>;
+			no-map;
+		};
+
+		neuron_block_1_mem: neuron-block@d7f80000 {
+			reg = <0x0 0xd7f80000 0x0 0x80000>;
+			no-map;
+		};
+
+		framebuffer_mem: framebuffer@e1000000 {
+			reg = <0x0 0xe1000000 0x0 (1080 * 2400 * 4)>;
+			no-map;
+		};
+
+		rmtfs_mem: rmtfs@ef500000 {
+			compatible = "qcom,rmtfs-mem";
+			reg = <0x0 0xef500000 0x0 0x280000>;
+			no-map;
+
+			qcom,client-id = <1>;
+			qcom,vmid = <QCOM_SCM_VMID_MSS_MSA>,
+				    <QCOM_SCM_VMID_NAV>;
+		};
+	};
+
+	vph_pwr: vph-pwr-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
+	/* S2B is really ebi.lvl but it's there for supply map completeness sake. */
+	vreg_s2b_0p7: smpa3-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_s2b_0p7";
+
+		regulator-min-microvolt = <700000>;
+		regulator-max-microvolt = <700000>;
+		regulator-always-on;
+		vin-supply = <&vph_pwr>;
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7325-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+
+		vdd-l1-l4-l12-l15-supply = <&vreg_s7b_0p952>;
+		vdd-l2-l7-supply = <&vreg_bob>;
+		vdd-l3-supply = <&vreg_s2b_0p7>;
+		vdd-l5-supply = <&vreg_s2b_0p7>;
+		vdd-l6-l9-l10-supply = <&vreg_s8b_1p256>;
+		vdd-l8-supply = <&vreg_s7b_0p952>;
+		vdd-l11-l17-l18-l19-supply = <&vreg_s1b_1p856>;
+		vdd-l13-supply = <&vreg_s7b_0p952>;
+		vdd-l14-l16-supply = <&vreg_s8b_1p256>;
+
+		/*
+		 * S2, L4-L5 are ARCs:
+		 * S2 - ebi.lvl,
+		 * L4 - lmx.lvl,
+		 * l5 - lcx.lvl.
+		 *
+		 * L10 are unused.
+		 */
+
+		vdd19_pmu_rfa_i:
+		vreg_s1b_1p856: smps1 {
+			regulator-name = "vreg_s1b_1p856";
+			regulator-min-microvolt = <1840000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		vreg_s7b_0p952: smps7 {
+			regulator-name = "vreg_s7b_0p952";
+			regulator-min-microvolt = <535000>;
+			regulator-max-microvolt = <1120000>;
+		};
+
+		vdd13_pmu_rfa_i:
+		vreg_s8b_1p256: smps8 {
+			regulator-name = "vreg_s8b_1p256";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1500000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_RET>;
+		};
+
+		vreg_l1b_0p912: ldo1 {
+			regulator-name = "vreg_l1b_0p912";
+			regulator-min-microvolt = <825000>;
+			regulator-max-microvolt = <925000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_usbhs_3p1:
+		vreg_l2b_3p072: ldo2 {
+			regulator-name = "vreg_l2b_3p072";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b_0p6: ldo3 {
+			regulator-name = "vreg_l3b_0p6";
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <910000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_dsi_0_1p2:
+		vdd_a_ufs_0_1p2:
+		vreg_l6b_1p2: ldo6 {
+			regulator-name = "vreg_l6b_1p2";
+			regulator-min-microvolt = <1140000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p96: ldo7 {
+			regulator-name = "vreg_l7b_2p96";
+			/* Constrained for UFS VCC, at least until UFS driver scales voltage */
+			regulator-min-microvolt = <2952000>;
+			regulator-max-microvolt = <2952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_0p904: ldo8 {
+			regulator-name = "vreg_l8b_0p904";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_1p2: ldo9 {
+			regulator-name = "vreg_l9b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p776: ldo11 {
+			regulator-name = "vreg_l11b_1p776";
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_0p8: ldo12 {
+			regulator-name = "vreg_l12b_0p8";
+			regulator-min-microvolt = <751000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_0p8: ldo13 {
+			regulator-name = "vreg_l13b_0p8";
+			regulator-min-microvolt = <530000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_1p2: ldo14 {
+			regulator-name = "vreg_l14b_1p2";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_0p88: ldo15 {
+			regulator-name = "vreg_l15b_0p88";
+			regulator-min-microvolt = <765000>;
+			regulator-max-microvolt = <1020000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_1p2: ldo16 {
+			regulator-name = "vreg_l16b_1p2";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_1p8: ldo17 {
+			regulator-name = "vreg_l17b_1p8";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p8: ldo18 {
+			regulator-name = "vreg_l18b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19b_1p8: ldo19 {
+			regulator-name = "vreg_l19b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8350c-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+
+		vdd-l1-l12-supply = <&vreg_s1b_1p856>;
+		vdd-l2-l8-supply = <&vreg_s1b_1p856>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob>;
+		vdd-l6-l9-l11-supply = <&vreg_bob>;
+		vdd-l10-supply = <&vreg_s7b_0p952>;
+
+		vdd-bob-supply = <&vph_pwr>;
+
+		/*
+		 * S2, S5, S7, S10 are ARCs:
+		 * S2 - cx.lvl,
+		 * S5 - mss.lvl,
+		 * S7 - gfx.lvl,
+		 * S10 - mx.lvl.
+		 */
+
+		vreg_s1c_2p2: smps1 {
+			regulator-name = "vreg_s1c_2p2";
+			regulator-min-microvolt = <2190000>;
+			regulator-max-microvolt = <2210000>;
+		};
+
+		vreg_s9c_0p676: smps9 {
+			regulator-name = "vreg_s9c_0p676";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1170000>;
+		};
+
+		vdd_a_usbhs_1p8:
+		vdd_qfprom:
+		vreg_l1c_1p8: ldo1 {
+			regulator-name = "vreg_l1c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c_1p8: ldo2 {
+			regulator-name = "vreg_l2c_1p8";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_1p8_3p0: ldo4 {
+			regulator-name = "vreg_l4c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5c_1p8_3p0: ldo5 {
+			regulator-name = "vreg_l5c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6c_2p96: ldo6 {
+			regulator-name = "vreg_l6c_2p96";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c_3p0: ldo7 {
+			regulator-name = "vreg_l7c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c_1p8: ldo8 {
+			regulator-name = "vreg_l8c_1p8";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+		};
+
+		vreg_l9c_2p96: ldo9 {
+			regulator-name = "vreg_l9c_2p96";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_dsi_0_0p9:
+		vdd_a_dsi_0_pll_0p9:
+		vdd_a_ufs_0_core:
+		vdd_a_usbhs_core:
+		vreg_l10c_0p88: ldo10 {
+			regulator-name = "vreg_l10c_0p88";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <1050000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11c_2p8: ldo11 {
+			regulator-name = "vreg_l11c_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12c_1p8: ldo12 {
+			regulator-name = "vreg_l12c_1p8";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13c_3p0: ldo13 {
+			regulator-name = "vreg_l13c_3p0";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_flash:
+		vdd_iris_rgb:
+		vdd_mic_bias:
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+};
+
+&gcc {
+	protected-clocks = <GCC_CFG_NOC_LPASS_CLK>,
+			   <GCC_MSS_CFG_AHB_CLK>,
+			   <GCC_MSS_OFFLINE_AXI_CLK>,
+			   <GCC_MSS_Q6SS_BOOT_CLK_SRC>,
+			   <GCC_MSS_Q6_MEMNOC_AXI_CLK>,
+			   <GCC_MSS_SNOC_AXI_CLK>,
+			   <GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
+			   <GCC_QSPI_CORE_CLK>,
+			   <GCC_QSPI_CORE_CLK_SRC>,
+			   <GCC_SEC_CTRL_CLK_SRC>,
+			   <GCC_WPSS_AHB_BDG_MST_CLK>,
+			   <GCC_WPSS_AHB_CLK>,
+			   <GCC_WPSS_RSCP_CLK>;
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm7325/xiaomi/lisa/a660_zap.mbn";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	pm8008: pmic@8 {
+		compatible = "qcom,pm8008";
+		reg = <0x8>;
+
+		interrupts-extended = <&tlmm 25 IRQ_TYPE_EDGE_RISING>;
+		reset-gpios = <&pm8350c_gpios 3 GPIO_ACTIVE_LOW>;
+
+		vdd-l1-l2-supply = <&vreg_s8b_1p256>;
+		vdd-l3-l4-supply = <&vreg_bob>;
+		vdd-l5-supply = <&vreg_s1b_1p856>;
+		vdd-l6-supply = <&vreg_bob>;
+		vdd-l7-supply = <&vreg_bob>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pm8008_int_default>, <&pm8008_reset_n_default>;
+
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-ranges = <&pm8008 0 0 2>;
+
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		#thermal-sensor-cells = <0>;
+
+		regulators {
+			vreg_l1p: ldo1 {
+				regulator-name = "vreg_l1p";
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1200000>;
+			};
+
+			vreg_l2p: ldo2 {
+				regulator-name = "vreg_l2p";
+				regulator-min-microvolt = <950000>;
+				regulator-max-microvolt = <1200000>;
+			};
+
+			vreg_l3p: ldo3 {
+				regulator-name = "vreg_l3p";
+				regulator-min-microvolt = <2700000>;
+				regulator-max-microvolt = <3000000>;
+			};
+
+			vreg_l4p: ldo4 {
+				regulator-name = "vreg_l4p";
+				regulator-min-microvolt = <2700000>;
+				regulator-max-microvolt = <2900000>;
+			};
+
+			vreg_l5p: ldo5 {
+				regulator-name = "vreg_l5p";
+				regulator-min-microvolt = <1700000>;
+				regulator-max-microvolt = <1900000>;
+			};
+
+			vreg_l6p: ldo6 {
+				regulator-name = "vreg_l6p";
+				regulator-min-microvolt = <2700000>;
+				regulator-max-microvolt = <2900000>;
+			};
+
+			vreg_l7p: ldo7 {
+				regulator-name = "vreg_l7p";
+				regulator-min-microvolt = <2700000>;
+				regulator-max-microvolt = <3000000>;
+			};
+		};
+	};
+
+	fsa4480: typec-mux@42 {
+		compatible = "fcs,fsa4480";
+		reg = <0x42>;
+
+		vcc-supply = <&vreg_bob>;
+		mode-switch;
+		orientation-switch;
+
+		port {
+			fsa4480_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_sbu>;
+			};
+		};
+	};
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	/* awinic,aw8624_haptic @ 5a */
+};
+
+&ipa {
+	firmware-name = "qcom/sm7325/xiaomi/lisa/ipa_fws.mbn";
+
+	status = "okay";
+};
+
+&pm7325_gpios {
+	volume_up_default: volume-up-default-state {
+		pins = "gpio6";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+};
+
+&pm8350c_gpios {
+	pm8008_reset_n_default: pm8008-reset-n-default-state {
+		pins = "gpio3";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		bias-pull-down;
+	};
+};
+
+&pm8350c_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <1>, <4>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <1280000>;
+	};
+};
+
+&pmk8350_adc_tm {
+	status = "okay";
+
+	/* PMK8350 */
+	xo-therm@0 {
+		reg = <0>;
+		io-channels = <&pmk8350_vadc PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	/* PM7325 */
+	quiet-therm@1 {
+		reg = <1>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	cam-flash-therm@2 {
+		reg = <2>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	sdm-skin-therm@3 {
+		reg = <3>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	wide-rfc-therm@4 {
+		reg = <4>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	5g-pa-therm@5 {
+		reg = <5>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM5_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	4g-pa-therm@6 {
+		reg = <6>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_GPIO3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	/* PM8350B */
+	chg-skin-therm@7 {
+		reg = <7>;
+		io-channels = <&pmk8350_vadc PM8350B_ADC7_GPIO2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pmk8350_rtc {
+	status = "okay";
+};
+
+&pmk8350_vadc {
+	/* PMK8350 */
+	channel@44 {
+		reg = <PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pmk8350_xo_therm";
+	};
+
+	/* PM7325 */
+	channel@144 {
+		reg = <PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_quiet_therm";
+	};
+
+	channel@145 {
+		reg = <PM7325_ADC7_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_cam_flash_therm";
+	};
+
+	channel@146 {
+		reg = <PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_sdm_skin_therm";
+	};
+
+	channel@147 {
+		reg = <PM7325_ADC7_AMUX_THM4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_wide_rfc_therm";
+	};
+
+	channel@148 {
+		reg = <PM7325_ADC7_AMUX_THM5_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_5g_pa_therm";
+	};
+
+	channel@14c {
+		reg = <PM7325_ADC7_GPIO3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_4g_pa_therm";
+	};
+
+	/* PM8350B */
+	channel@34b {
+		reg = <PM8350B_ADC7_GPIO2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm8350b_chg_skin_therm";
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qfprom {
+	vcc-supply = <&vdd_qfprom>;
+};
+
+&qup_spi13_cs {
+	drive-strength = <6>;
+	bias-disable;
+};
+
+&qup_spi13_data_clk {
+	drive-strength = <6>;
+	bias-disable;
+};
+
+&qup_uart5_rx {
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart5_tx {
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm7325/xiaomi/lisa/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/sm7325/xiaomi/lisa/cdsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/sm7325/xiaomi/lisa/modem.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_wpss {
+	firmware-name = "qcom/sm7325/xiaomi/lisa/wpss.mbn";
+
+	status = "okay";
+};
+
+&spi13 {
+	status = "okay";
+
+	touchscreen@0 {
+		compatible = "goodix,gt9897";
+		reg = <0>;
+		spi-max-frequency = <6000000>;
+
+		interrupts-extended = <&tlmm 81 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&tlmm 105 GPIO_ACTIVE_LOW>;
+
+		avdd-supply = <&vreg_l7c_3p0>;
+		vddio-supply = <&vreg_l2c_1p8>;
+
+		pinctrl-0 = <&touchscreen_default>;
+		pinctrl-names = "default";
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <2400>;
+	};
+};
+
+&tlmm {
+	gpio-reserved-ranges = <32 2>, /* SMB1394 (SPMI) */
+			       <48 4>, /* Presumably NFC SE */
+			       <56 4>; /* Presumably fingerprint reader */
+
+	qup_uart7_sleep_cts: qup-uart7-sleep-cts-state {
+		pins = "gpio28";
+		function = "gpio";
+		/*
+		 * Configure a bias-bus-hold on CTS to lower power
+		 * usage when Bluetooth is turned off. Bus hold will
+		 * maintain a low power state regardless of whether
+		 * the Bluetooth module drives the pin in either
+		 * direction or leaves the pin fully unpowered.
+		 */
+		bias-bus-hold;
+	};
+
+	qup_uart7_sleep_rts: qup-uart7-sleep-rts-state {
+		pins = "gpio29";
+		function = "gpio";
+		/*
+		 * Configure pull-down on RTS. As RTS is active low
+		 * signal, pull it low to indicate the BT SoC that it
+		 * can wakeup the system anytime from suspend state by
+		 * pulling RX low (by sending wakeup bytes).
+		 */
+		bias-pull-down;
+	};
+
+	qup_uart7_sleep_tx: qup-uart7-sleep-tx-state {
+		pins = "gpio30";
+		function = "gpio";
+		/*
+		 * Configure pull-up on TX when it isn't actively driven
+		 * to prevent BT SoC from receiving garbage during sleep.
+		 */
+		bias-pull-up;
+	};
+
+	qup_uart7_sleep_rx: qup-uart7-sleep-rx-state {
+		pins = "gpio31";
+		function = "gpio";
+		/*
+		 * Configure a pull-up on RX. This is needed to avoid
+		 * garbage data when the TX pin of the Bluetooth module
+		 * is floating which may cause spurious wakeups.
+		 */
+		bias-pull-up;
+	};
+
+	bluetooth_enable_default: bluetooth-enable-default-state {
+		pins = "gpio85";
+		function = "gpio";
+		output-low;
+		bias-disable;
+	};
+
+	sw_ctrl_default: sw-ctrl-default-state {
+		pins = "gpio86";
+		function = "gpio";
+		bias-pull-down;
+	};
+
+	touchscreen_default: touchscreen-default-state {
+		pins = "gpio81", "gpio105";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+
+	pm8008_int_default: pm8008-int-default-state {
+		pins = "gpio25";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&uart5 {
+	compatible = "qcom,geni-debug-uart";
+
+	status = "okay";
+};
+
+&uart7 {
+	/delete-property/interrupts;
+	interrupts-extended = <&intc GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>,
+			      <&tlmm 31 IRQ_TYPE_EDGE_FALLING>;
+
+	pinctrl-1 = <&qup_uart7_sleep_cts>,
+		    <&qup_uart7_sleep_rts>,
+		    <&qup_uart7_sleep_tx>,
+		    <&qup_uart7_sleep_rx>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+
+	bluetooth: bluetooth {
+		compatible = "qcom,wcn6750-bt";
+
+		pinctrl-0 = <&bluetooth_enable_default>, <&sw_ctrl_default>;
+		pinctrl-names = "default";
+
+		enable-gpios = <&tlmm 85 GPIO_ACTIVE_HIGH>;
+		swctrl-gpios = <&tlmm 86 GPIO_ACTIVE_HIGH>;
+
+		vddio-supply = <&vreg_l19b_1p8>;
+		vddaon-supply = <&vreg_s7b_0p952>;
+		vddbtcxmx-supply = <&vreg_s7b_0p952>;
+		vddrfacmn-supply = <&vreg_s7b_0p952>;
+		vddrfa0p8-supply = <&vreg_s7b_0p952>;
+		vddrfa1p2-supply = <&vdd13_pmu_rfa_i>;
+		vddrfa1p7-supply = <&vdd19_pmu_rfa_i>;
+		vddrfa2p2-supply = <&vreg_s1c_2p2>;
+		vddasd-supply = <&vreg_l11c_2p8>;
+		max-speed = <3200000>;
+
+		qcom,local-bd-address-broken;
+	};
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l7b_2p96>;
+	vcc-max-microamp = <800000>;
+	/*
+	 * Technically l9b enables an eLDO (supplied by s1b) which then powers
+	 * VCCQ2 of the UFS.
+	 */
+	vccq-supply = <&vreg_l9b_1p2>;
+	vccq-max-microamp = <900000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vdd_a_ufs_0_core>;
+	vdda-pll-supply = <&vdd_a_ufs_0_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	qcom,select-utmi-as-pipe-clk;
+
+	maximum-speed = "high-speed";
+	phys = <&usb_1_hsphy>;
+	phy-names = "usb2-phy";
+
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&vdd_a_usbhs_core>;
+	vdda18-supply = <&vdd_a_usbhs_1p8>;
+	vdda33-supply = <&vdd_a_usbhs_3p1>;
+
+	status = "okay";
+};
+
+&venus {
+	firmware-name = "qcom/sm7325/xiaomi/lisa/vpu20_1v.mbn";
+
+	status = "okay";
+};
+
+&wifi {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi
index 94a3b02cb28977..e823091457c8b7 100644
--- a/arch/arm64/boot/dts/qcom/sm8150.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi
@@ -610,6 +610,7 @@
 	firmware {
 		scm: scm {
 			compatible = "qcom,scm-sm8150", "qcom,scm";
+			qcom,dload-mode = <&tcsr 0x13000>;
 			#reset-cells = <1>;
 		};
 	};
@@ -1901,12 +1902,6 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
-			perst-gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>;
-			wake-gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>;
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie0_default_state>;
 
@@ -1917,6 +1912,10 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+				reset-gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>;
+				wake-gpios = <&tlmm 37 GPIO_ACTIVE_LOW>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2019,10 +2018,6 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
-			perst-gpios = <&tlmm 102 GPIO_ACTIVE_HIGH>;
 			enable-gpio = <&tlmm 104 GPIO_ACTIVE_HIGH>;
 
 			pinctrl-names = "default";
@@ -2035,6 +2030,9 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+				reset-gpios = <&tlmm 102 GPIO_ACTIVE_HIGH>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2181,6 +2179,11 @@
 			reg = <0x0 0x01f60000 0x0 0x20000>;
 		};
 
+		tcsr: syscon@1fc0000 {
+			compatible = "qcom,sm8150-tcsr", "syscon";
+			reg = <0x0 0x01fc0000 0x0 0x30000>;
+		};
+
 		remoteproc_slpi: remoteproc@2400000 {
 			compatible = "qcom,sm8150-slpi-pas";
 			reg = <0x0 0x02400000 0x0 0x4040>;
@@ -3574,7 +3577,7 @@
 			iommus = <&apps_smmu 0x6a0 0x0>;
 			qcom,dll-config = <0x0007642c>;
 			qcom,ddr-config = <0x80040868>;
-			power-domains = <&rpmhpd 0>;
+			power-domains = <&rpmhpd SM8150_CX>;
 			operating-points-v2 = <&sdhc2_opp_table>;
 
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi
index 90f8b091634e9f..b8452a4845affa 100644
--- a/arch/arm64/boot/dts/qcom/sm8250.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi
@@ -16,7 +16,6 @@
 #include <dt-bindings/interconnect/qcom,sm8250.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
 #include <dt-bindings/soc/qcom,apr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
@@ -2207,12 +2206,6 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
-			perst-gpios = <&tlmm 79 GPIO_ACTIVE_LOW>;
-			wake-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie0_default_state>;
 			dma-coherent;
@@ -2224,6 +2217,10 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+				reset-gpios = <&tlmm 79 GPIO_ACTIVE_LOW>;
+				wake-gpios = <&tlmm 81 GPIO_ACTIVE_LOW>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2334,12 +2331,6 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
-			perst-gpios = <&tlmm 82 GPIO_ACTIVE_LOW>;
-			wake-gpios = <&tlmm 84 GPIO_ACTIVE_HIGH>;
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie1_default_state>;
 			dma-coherent;
@@ -2351,6 +2342,10 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+				reset-gpios = <&tlmm 82 GPIO_ACTIVE_LOW>;
+				wake-gpios = <&tlmm 84 GPIO_ACTIVE_LOW>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2461,12 +2456,6 @@
 
 			power-domains = <&gcc PCIE_2_GDSC>;
 
-			phys = <&pcie2_phy>;
-			phy-names = "pciephy";
-
-			perst-gpios = <&tlmm 85 GPIO_ACTIVE_LOW>;
-			wake-gpios = <&tlmm 87 GPIO_ACTIVE_HIGH>;
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie2_default_state>;
 			dma-coherent;
@@ -2478,6 +2467,10 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie2_phy>;
+				reset-gpios = <&tlmm 85 GPIO_ACTIVE_LOW>;
+				wake-gpios = <&tlmm 87 GPIO_ACTIVE_LOW>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -3290,7 +3283,7 @@
 				port@4 {
 					reg = <4>;
 					funnel_in1_in_funnel_apss_merg: endpoint {
-					remote-endpoint = <&funnel_apss_merg_out_funnel_in1>;
+						remote-endpoint = <&funnel_apss_merg_out_funnel_in1>;
 					};
 				};
 			};
@@ -3306,7 +3299,7 @@
 			out-ports {
 				port {
 					funnel_merg_out_funnel_swao: endpoint {
-					remote-endpoint = <&funnel_swao_in_funnel_merg>;
+						remote-endpoint = <&funnel_swao_in_funnel_merg>;
 					};
 				};
 			};
@@ -3318,14 +3311,14 @@
 				port@0 {
 					reg = <0>;
 					funnel_merg_in_funnel_in0: endpoint {
-					remote-endpoint = <&funnel_in0_out_funnel_merg>;
+						remote-endpoint = <&funnel_in0_out_funnel_merg>;
 					};
 				};
 
 				port@1 {
 					reg = <1>;
 					funnel_merg_in_funnel_in1: endpoint {
-					remote-endpoint = <&funnel_in1_out_funnel_merg>;
+						remote-endpoint = <&funnel_in1_out_funnel_merg>;
 					};
 				};
 			};
@@ -3493,7 +3486,7 @@
 			out-ports {
 				port {
 					funnel_dl_mm_out_funnel_dl_center: endpoint {
-					remote-endpoint = <&funnel_dl_center_in_funnel_dl_mm>;
+						remote-endpoint = <&funnel_dl_center_in_funnel_dl_mm>;
 					};
 				};
 			};
@@ -3533,7 +3526,7 @@
 				port@2 {
 					reg = <2>;
 					funnel_dl_center_in_funnel_dl_mm: endpoint {
-					remote-endpoint = <&funnel_dl_mm_out_funnel_dl_center>;
+						remote-endpoint = <&funnel_dl_mm_out_funnel_dl_center>;
 					};
 				};
 			};
@@ -3701,7 +3694,7 @@
 			out-ports {
 				port {
 					funnel_apss_out_funnel_apss_merg: endpoint {
-					remote-endpoint = <&funnel_apss_merg_in_funnel_apss>;
+						remote-endpoint = <&funnel_apss_merg_in_funnel_apss>;
 					};
 				};
 			};
@@ -3778,7 +3771,7 @@
 			out-ports {
 				port {
 					funnel_apss_merg_out_funnel_in1: endpoint {
-					remote-endpoint = <&funnel_in1_in_funnel_apss_merg>;
+						remote-endpoint = <&funnel_in1_in_funnel_apss_merg>;
 					};
 				};
 			};
@@ -3786,7 +3779,7 @@
 			in-ports {
 				port {
 					funnel_apss_merg_in_funnel_apss: endpoint {
-					remote-endpoint = <&funnel_apss_out_funnel_apss_merg>;
+						remote-endpoint = <&funnel_apss_out_funnel_apss_merg>;
 					};
 				};
 			};
diff --git a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts
index bb93c3e9417b41..24ad073b414300 100644
--- a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts
+++ b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts
@@ -392,7 +392,7 @@
 	data-lanes = <0 1 2 3>;
 };
 
-&mdss_dsi0_phy  {
+&mdss_dsi0_phy {
 	vdds-supply = <&vreg_l5b_0p88>;
 	status = "okay";
 };
@@ -499,9 +499,6 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie0_default_state>;
 
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-
 	status = "okay";
 };
 
@@ -512,10 +509,12 @@
 	status = "okay";
 };
 
-&pcie1 {
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
+&pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+};
 
+&pcie1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_default_state>;
 
@@ -528,6 +527,11 @@
 	vdda-pll-supply = <&vreg_l6b_1p2>;
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+};
+
 &qupv3_id_0 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
index 4b62ee50da7bec..969c8cb31df9f1 100644
--- a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
@@ -893,9 +893,20 @@
 	};
 };
 
-/* BIG WARNING! DO NOT TOUCH UFS, YOUR DEVICE WILL DIE! */
-&ufs_mem_hc { status = "disabled"; };
-&ufs_mem_phy { status = "disabled"; };
+&ufs_mem_hc {
+	status = "okay";
+	reset-gpios = <&tlmm 203 GPIO_ACTIVE_LOW>;
+	vcc-supply = <&pm8350_l7>;
+	vcc-max-microamp = <800000>;
+	vccq-supply = <&pm8350_l9>;
+	vcc-max-microamp = <900000>;
+};
+
+&ufs_mem_phy {
+	status = "okay";
+	vdda-phy-supply = <&pm8350_l5>;
+	vdda-pll-supply = <&pm8350_l6>;
+};
 
 &usb_1 {
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi
index 32b47f5ec1f675..9045557c0b344a 100644
--- a/arch/arm64/boot/dts/qcom/sm8350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi
@@ -18,7 +18,6 @@
 #include <dt-bindings/interconnect/qcom,sm8350.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
 #include <dt-bindings/reset/qcom,sm8350-videocc.h>
 #include <dt-bindings/soc/qcom,apr.h>
@@ -1586,16 +1585,15 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
-			pcie@0 {
+			pcie0_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -1695,16 +1693,15 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			status = "disabled";
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi
index b8793482b1815b..d8a5f6d66625ff 100644
--- a/arch/arm64/boot/dts/qcom/sm8450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi
@@ -17,7 +17,6 @@
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,sm8450.h>
 #include <dt-bindings/reset/qcom,sm8450-gpucc.h>
@@ -2017,12 +2016,6 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
-			perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-			wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie0_default_state>;
 
@@ -2063,6 +2056,10 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+				reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+				wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2182,12 +2179,6 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
-			perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
-			wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
-
 			pinctrl-names = "default";
 			pinctrl-0 = <&pcie1_default_state>;
 
@@ -2263,11 +2254,15 @@
 				};
 			};
 
-			pcie@0 {
+			pcie1_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+				reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+				wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -3489,7 +3484,7 @@
 						mdss_dp0_out: endpoint {
 							remote-endpoint = <&usb_1_qmpphy_dp_in>;
 						};
-		};
+					};
 				};
 
 				dp_opp_table: opp-table {
diff --git a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts
index 82fb10ed6b6ba6..e75e3b878e89e0 100644
--- a/arch/arm64/boot/dts/qcom/sm8550-hdk.dts
+++ b/arch/arm64/boot/dts/qcom/sm8550-hdk.dts
@@ -1087,16 +1087,23 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
 	status = "okay";
 };
 
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1e_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
 &pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -1113,17 +1120,7 @@
 	};
 };
 
-&pcie0_phy {
-	vdda-phy-supply = <&vreg_l1e_0p88>;
-	vdda-pll-supply = <&vreg_l3e_1p2>;
-
-	status = "okay";
-};
-
 &pcie1 {
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie1_default_state>;
 	pinctrl-names = "default";
 
@@ -1138,6 +1135,11 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+};
+
 &pm8550_gpios {
 	sdc2_card_det_n: sdc2-card-det-state {
 		pins = "gpio12";
diff --git a/arch/arm64/boot/dts/qcom/sm8550-mtp.dts b/arch/arm64/boot/dts/qcom/sm8550-mtp.dts
index 7c634572c6b715..7b191c9553f062 100644
--- a/arch/arm64/boot/dts/qcom/sm8550-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sm8550-mtp.dts
@@ -739,9 +739,6 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie0_default_state>;
 
@@ -755,10 +752,12 @@
 	status = "okay";
 };
 
-&pcie1 {
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+&pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+};
 
+&pcie1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_default_state>;
 
@@ -773,6 +772,11 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+};
+
 &pm8550_gpios {
 	sdc2_card_det_n: sdc2-card-det-state {
 		pins = "gpio12";
diff --git a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts
index 473ec4e27dafd9..a3ab0bd3fb0118 100644
--- a/arch/arm64/boot/dts/qcom/sm8550-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/sm8550-qrd.dts
@@ -987,16 +987,23 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
 	status = "okay";
 };
 
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1e_0p88>;
+	vdda-pll-supply = <&vreg_l3e_1p2>;
+
+	status = "okay";
+};
+
 &pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -1013,13 +1020,6 @@
 	};
 };
 
-&pcie0_phy {
-	vdda-phy-supply = <&vreg_l1e_0p88>;
-	vdda-pll-supply = <&vreg_l3e_1p2>;
-
-	status = "okay";
-};
-
 &pm8550_flash {
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/qcom/sm8550-samsung-q5q.dts b/arch/arm64/boot/dts/qcom/sm8550-samsung-q5q.dts
index 81c02ee27fe998..bf0b16d690e6b5 100644
--- a/arch/arm64/boot/dts/qcom/sm8550-samsung-q5q.dts
+++ b/arch/arm64/boot/dts/qcom/sm8550-samsung-q5q.dts
@@ -510,8 +510,6 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 	status = "okay";
@@ -523,6 +521,11 @@
 	status = "okay";
 };
 
+&pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+};
+
 &pm8550_gpios {
 	volume_up_n: volume-up-n-state {
 		pins = "gpio6";
diff --git a/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts b/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts
index 367f6eae5e6529..a23ebd3cd06573 100644
--- a/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts
+++ b/arch/arm64/boot/dts/qcom/sm8550-sony-xperia-yodo-pdx234.dts
@@ -584,9 +584,6 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -600,6 +597,11 @@
 	status = "okay";
 };
 
+&pcie0_port0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+};
+
 &pm8550_flash {
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi
index 0e701e709235c6..8e26f832d34779 100644
--- a/arch/arm64/boot/dts/qcom/sm8550.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi
@@ -19,7 +19,6 @@
 #include <dt-bindings/interconnect/qcom,sm8550-rpmh.h>
 #include <dt-bindings/interconnect/qcom,osm-l3.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
@@ -2397,9 +2396,6 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			operating-points-v2 = <&pcie0_opp_table>;
 
 			status = "disabled";
@@ -2461,6 +2457,8 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -2583,9 +2581,6 @@
 
 			power-domains = <&gcc PCIE_1_GDSC>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			operating-points-v2 = <&pcie1_opp_table>;
 
 			status = "disabled";
@@ -2663,6 +2658,8 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -4692,7 +4689,6 @@
 			interrupts-extended = <&pdc 1 IRQ_TYPE_LEVEL_HIGH>;
 			qcom,ee = <0>;
 			qcom,channel = <0>;
-			qcom,bus-id = <0>;
 			#address-cells = <2>;
 			#size-cells = <0>;
 			interrupt-controller;
diff --git a/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts
index 6ddf30ecabfef3..48fbf0650f0eae 100644
--- a/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts
+++ b/arch/arm64/boot/dts/qcom/sm8650-ayaneo-pocket-s2.dts
@@ -221,7 +221,7 @@
 	};
 
 	sound {
-		compatible = "qcom,sm8650-sndcard", "qcom,sm8450-sndcard";
+		compatible = "ayaneo,pocket-s2-sndcard";
 		model = "SM8650-APS2";
 		audio-routing = "SpkrLeft IN", "WSA_SPK1 OUT",
 				"SpkrRight IN", "WSA_SPK2 OUT",
@@ -325,6 +325,23 @@
 		};
 	};
 
+	sy7758_vdd33_reg: sy7758-vdd33-regulator {
+		compatible = "regulator-fixed";
+
+		regulator-name = "sy7758_vdd33";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+
+		gpios = <&tlmm 163 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		vin-supply = <&vph_pwr>;
+
+		pinctrl-0 = <&sy7758_vdd33>;
+		pinctrl-names = "default";
+	};
+
 	vph_pwr: vph-pwr-regulator {
 		compatible = "regulator-fixed";
 
@@ -986,6 +1003,10 @@
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
 &i2c3 {
 	clock-frequency = <100000>;
 
@@ -1016,6 +1037,30 @@
 	};
 };
 
+&i2c4 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	touchscreen@5d {
+		compatible = "goodix,gt911";
+		reg = <0x5d>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <162 IRQ_TYPE_EDGE_FALLING>;
+
+		reset-gpios = <&tlmm 161 GPIO_ACTIVE_HIGH>;
+		VDDIO-supply = <&vreg_l14b_3p2>;
+		AVDD28-supply = <&vreg_l14b_3p2>;
+
+		touchscreen-size-x = <1440>;
+		touchscreen-size-y = <2560>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&ts_reset_default>, <&ts_irq_default>;
+	};
+};
+
 &i2c6 {
 	clock-frequency = <100000>;
 
@@ -1053,6 +1098,51 @@
 	};
 };
 
+&i2c9 {
+	status = "okay";
+
+	/* Screen power */
+	regulator@3e {
+		compatible = "sgmicro,sgm3804";
+		reg = <0x3e>;
+
+		pinctrl-0 = <&sgm3804_default>;
+		pinctrl-names = "default";
+
+		vin-supply = <&vph_pwr>;
+
+		sgm3804_pos: pos {
+			regulator-name = "panel-avdd-pos";
+			regulator-min-microvolt = <5200000>;
+			regulator-max-microvolt = <5200000>;
+			regulator-active-discharge = <1>;
+			regulator-boot-on;
+			enable-gpios = <&tlmm 59 GPIO_ACTIVE_HIGH>;
+		};
+
+		sgm3804_neg: neg {
+			regulator-name = "panel-avdd-neg";
+			regulator-min-microvolt = <5200000>;
+			regulator-max-microvolt = <5200000>;
+			regulator-active-discharge = <1>;
+			regulator-boot-on;
+			enable-gpios = <&tlmm 58 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	/* Backlight */
+	backlight: backlight@2e {
+		compatible = "silergy,sy7758";
+		reg = <0x2e>;
+
+		pinctrl-0 = <&sy7758_default>;
+		pinctrl-names = "default";
+
+		vdd-supply = <&sy7758_vdd33_reg>;
+		enable-gpios = <&tlmm 164 GPIO_ACTIVE_HIGH>;
+	};
+};
+
 &iris {
 	status = "okay";
 };
@@ -1073,10 +1163,94 @@
 	status = "okay";
 };
 
-&pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+&mdss_dsi0 {
+	vdda-supply = <&vreg_l3i_1p2>;
+
+	qcom,master-dsi;
+	qcom,dual-dsi-mode;
+	qcom,sync-dual-dsi;
+
+	status = "okay";
+
+	panel@0 {
+		status = "okay";
+		compatible = "ayaneo,wt0630-2k", "renesas,r63419";
+		reg = <0>;
+
+		pinctrl-0 = <&disp0_reset_n_active>;
+		pinctrl-1 = <&disp0_reset_n_suspend>;
+		pinctrl-names = "default", "sleep";
+
+		vddio-supply = <&vreg_l12b_1p8>;
+		vdd-supply = <&vreg_l11b_1p2>;
+		vsp-supply = <&sgm3804_pos>;
+		vsn-supply = <&sgm3804_neg>;
+		vci-supply = <&vreg_l13b_3p0>;
+
+		backlight = <&backlight>;
+
+		reset-gpios = <&tlmm 133 GPIO_ACTIVE_LOW>;
+
+		rotation = <90>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				panel0_in_0: endpoint {
+					remote-endpoint = <&mdss_dsi0_out>;
+				};
+			};
 
+			port@1{
+				reg = <1>;
+				panel0_in_1: endpoint {
+					remote-endpoint = <&mdss_dsi1_out>;
+				};
+			};
+		};
+	};
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&panel0_in_0>;
+
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi0_phy {
+	vdds-supply = <&vreg_l1i_0p88>;
+
+	status = "okay";
+};
+
+&mdss_dsi1 {
+	vdda-supply = <&vreg_l3i_1p2>;
+
+	assigned-clock-parents = <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+				 <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>;
+
+	qcom,dual-dsi-mode;
+	qcom,sync-dual-dsi;
+
+	status = "okay";
+};
+
+&mdss_dsi1_out {
+	remote-endpoint = <&panel0_in_1>;
+
+	data-lanes = <0 1 2 3>;
+};
+
+&mdss_dsi1_phy {
+	vdds-supply = <&vreg_l1i_0p88>;
+
+	status = "okay";
+};
+
+&pcie0 {
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -1084,6 +1258,9 @@
 };
 
 &pcieport0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -1108,9 +1285,6 @@
 };
 
 &pcie1 {
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie1_default_state>;
 	pinctrl-names = "default";
 
@@ -1118,6 +1292,8 @@
 };
 
 &pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
 	/* Renesas μPD720201 PCIe USB3.0 HOST CONTROLLER */
 	usb-controller@0 {
 		compatible = "pci1912,0014";
@@ -1390,6 +1566,20 @@
 		};
 	};
 
+	disp0_reset_n_active: disp0-reset-n-active-state {
+		pins = "gpio133";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	disp0_reset_n_suspend: disp0-reset-n-suspend-state {
+		pins = "gpio133";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
 	fan_pwr_pins: fan-pwr-state {
 		pins = "gpio125";
 		function = "gpio";
@@ -1440,6 +1630,13 @@
 		output-high;
 	};
 
+	sgm3804_default: sgm3804-default-state {
+		pins = "gpio58", "gpio59";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
 	spkr_23_sd_n_active: spkr-23-sd-n-active-state {
 		pins = "gpio77";
 		function = "gpio";
@@ -1454,6 +1651,34 @@
 		bias-disable;
 	};
 
+	sy7758_default: sy7758-default-state {
+		pins = "gpio164";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	sy7758_vdd33: sy7758-vdd33-state {
+		pins = "gpio163";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	ts_irq_default: ts-irq-active-state {
+		pins = "gpio162";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	ts_reset_default: ts-reset-active-state {
+		pins = "gpio161";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-down;
+	};
+
 	wcd_default: wcd-reset-n-active-state {
 		pins = "gpio107";
 		function = "gpio";
@@ -1549,3 +1774,7 @@
 &xo_board {
 	clock-frequency = <76800000>;
 };
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm8650/ayaneo/ps2/gen70900_zap.mbn";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts
index 41d0c7edc43399..6cb0dc089b17a0 100644
--- a/arch/arm64/boot/dts/qcom/sm8650-hdk.dts
+++ b/arch/arm64/boot/dts/qcom/sm8650-hdk.dts
@@ -125,7 +125,7 @@
 
 					pmic_glink_sbu: endpoint {
 						remote-endpoint = <&wcd_usbss_sbu_mux>;
-				    };
+					};
 				};
 			};
 		};
@@ -905,9 +905,9 @@
 };
 
 &i2c3 {
-       status = "okay";
+	status = "okay";
 
-       wcd_usbss: typec-mux@e {
+	wcd_usbss: typec-mux@e {
 		compatible = "qcom,wcd9395-usbss", "qcom,wcd9390-usbss";
 		reg = <0xe>;
 
@@ -929,7 +929,7 @@
 				};
 			};
 		};
-       };
+	};
 };
 
 &i2c6 {
@@ -1027,9 +1027,6 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -1037,6 +1034,9 @@
 };
 
 &pcieport0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -1061,9 +1061,6 @@
 };
 
 &pcie1 {
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie1_default_state>;
 	pinctrl-names = "default";
 
@@ -1078,6 +1075,11 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+};
+
 &pm8550_gpios {
 	sdc2_card_det_n: sdc2-card-det-state {
 		pins = "gpio12";
diff --git a/arch/arm64/boot/dts/qcom/sm8650-mtp.dts b/arch/arm64/boot/dts/qcom/sm8650-mtp.dts
index 26045430d34ada..f3ed42f81b0b62 100644
--- a/arch/arm64/boot/dts/qcom/sm8650-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sm8650-mtp.dts
@@ -642,9 +642,6 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -658,10 +655,12 @@
 	status = "okay";
 };
 
-&pcie1 {
-	wake-gpios = <&tlmm 99 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+&pcieport0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+};
 
+&pcie1 {
 	pinctrl-0 = <&pcie1_default_state>;
 	pinctrl-names = "default";
 
@@ -676,6 +675,11 @@
 	status = "okay";
 };
 
+&pcie1_port0 {
+	reset-gpios = <&tlmm 97 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 99 GPIO_ACTIVE_LOW>;
+};
+
 &pm8550_gpios {
 	sdc2_card_det_n: sdc2-card-det-state {
 		pins = "gpio12";
diff --git a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts
index 1fb78588a5048c..f693ea6a5461ea 100644
--- a/arch/arm64/boot/dts/qcom/sm8650-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/sm8650-qrd.dts
@@ -86,7 +86,7 @@
 
 					pmic_glink_sbu: endpoint {
 						remote-endpoint = <&wcd_usbss_sbu_mux>;
-				    };
+					};
 				};
 			};
 		};
@@ -875,9 +875,9 @@
 };
 
 &i2c3 {
-       status = "okay";
+	status = "okay";
 
-       wcd_usbss: typec-mux@e {
+	wcd_usbss: typec-mux@e {
 		compatible = "qcom,wcd9395-usbss", "qcom,wcd9390-usbss";
 		reg = <0xe>;
 
@@ -907,7 +907,7 @@
 				};
 			};
 		};
-       };
+	};
 };
 
 &i2c6 {
@@ -1021,9 +1021,6 @@
 };
 
 &pcie0 {
-	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
-	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
-
 	pinctrl-0 = <&pcie0_default_state>;
 	pinctrl-names = "default";
 
@@ -1031,6 +1028,9 @@
 };
 
 &pcieport0 {
+	reset-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_LOW>;
+
 	wifi@0 {
 		compatible = "pci17cb,1107";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
diff --git a/arch/arm64/boot/dts/qcom/sm8650-valve-deckard.dts b/arch/arm64/boot/dts/qcom/sm8650-valve-deckard.dts
new file mode 100644
index 00000000000000..d25a09e162dd3b
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm8650-valve-deckard.dts
@@ -0,0 +1,1288 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2026, Valve Software
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include <dt-bindings/phy/phy.h>
+#include "sm8650.dtsi"
+#include "pm8010.dtsi"
+#include "pm8550.dtsi"
+#include "pm8550b.dtsi"
+#define PMK8550VE_SID 8
+#include "pm8550ve.dtsi"
+#include "pm8550vs.dtsi"
+#include "pmk8550.dtsi"
+
+/* Remove nodes */
+/delete-node/ &ipa;
+/delete-node/ &gpu_zap_shader;
+/delete-node/ &lpass_wsa2macro;
+/delete-node/ &lpass_rxmacro;
+/delete-node/ &lpass_txmacro;
+/delete-node/ &lpass_wsamacro;
+/delete-node/ &lpass_vamacro;
+/delete-node/ &lpass_tlmm;
+/delete-node/ &remoteproc_adsp;
+/delete-node/ &remoteproc_cdsp;
+/delete-node/ &remoteproc_mpss;
+/delete-node/ &reserved_memory;
+/delete-node/ &smp2p_adsp;
+/delete-node/ &smp2p_cdsp;
+/delete-node/ &smp2p_modem;
+/delete-node/ &swr0;
+/delete-node/ &swr1;
+/delete-node/ &swr2;
+/delete-node/ &swr3;
+
+/ {
+	model = "Valve Deckard";
+	compatible = "valve,deckard", "qcom,sm8650";
+
+	aliases {
+		serial0 = &uart15;
+		serial1 = &uart14;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+
+		framebuffer {
+			compatible = "simple-framebuffer";
+			width = <4320>;
+			height = <2160>;
+			stride = <(4320 * 4)>;
+			format = "a8r8g8b8";
+			memory-region = <&framebuffer_mem>;
+
+			/* Bootloader will enable the node */
+			status = "disabled";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&volume_up_n>, <&select_n>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+			gpios = <&pm8550_gpios 6 GPIO_ACTIVE_LOW>;
+			debounce-interval = <15>;
+			linux,can-disable;
+		};
+
+		key-select {
+			label = "Select";
+			linux,code = <KEY_SELECT>;
+			gpios = <&pm8550b_gpios 5 GPIO_ACTIVE_LOW>;
+			debounce-interval = <15>;
+			linux,can-disable;
+		};
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		aop_mem: aop-region@81c00000 {
+			no-map;
+			reg = <0 0x81c00000 0 0x60000>;
+		};
+
+		ramoops: ramoops-region@84a00000 {
+			compatible = "ramoops";
+			reg = <0 0x84a00000 0 0x100000>;
+			record-size = <0x40000>;
+			console-size = <0x40000>;
+			pmsg-size = <0x1000>;
+			ecc-size = <16>;
+		};
+
+		aop_cmd_db_mem: aop-cmd-db-region@81c60000 {
+			compatible = "qcom,cmd-db";
+			reg = <0 0x81c60000 0 0x20000>;
+			no-map;
+		};
+
+		/* merged aop_config, tme_crash_dump, tme_log */
+		aop_config_merged_mem: aop-config-merged-region@81c80000 {
+			reg = <0 0x81c80000 0 0x75000>;
+			no-map;
+		};
+
+		smem: smem-region@81d00000 {
+			compatible = "qcom,smem";
+			reg = <0 0x81d00000 0 0x200000>;
+			hwlocks = <&tcsr_mutex 3>;
+			no-map;
+		};
+
+		cpucp_fw_mem: cpucp-fw-region@d8040000 {
+			reg = <0 0xd8040000 0 0x1b2000>;
+			no-map;
+		};
+
+		video_mem: video-region@9bb00000 {
+			reg = <0 0x9bb00000 0 0x800000>;
+			no-map;
+		};
+
+		tz_stat_mem: tz-stat-region@82700000 {
+			reg = <0 0x82700000 0 0x100000>;
+			no-map;
+		};
+
+		/* 2MB, 8 clocks to power on each 256KB block */
+		lpass_lpi_tcm: lpm-region@6000000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x06000000 0 0x00200000>;
+			no-map;
+		};
+
+		/* 64KB, 4 clocks to power on each 16KB block */
+		lpass_audio_lpm: lpm-region@6c80000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x06c80000 0 0x10000>;
+			no-map;
+		};
+
+		/* 12KB, 1 clock */
+		lpass_va_lpm: lpm-region@6d40000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x06d40000 0 0x3000>;
+			no-map;
+		};
+
+		lpass_lpm: lpm-region@7c10000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x07c10000 0 0x8000>;
+			no-map;
+		};
+
+		framebuffer_mem: framebuffer-region@af400000 {
+			reg = <0 0xaf400000 0 0x03000000>;
+			no-map;
+		};
+	};
+
+	wcn7850-pmu {
+		compatible = "qcom,wcn7850-pmu";
+
+		pinctrl-0 = <&wlan_en>, <&bt_default>;
+		pinctrl-names = "default";
+
+		wlan-enable-gpios = <&tlmm 16 GPIO_ACTIVE_HIGH>;
+		bt-enable-gpios = <&tlmm 17 GPIO_ACTIVE_HIGH>;
+
+		vdd-supply = <&vreg_s4i_0p85>;
+		vddio-supply = <&vreg_l15b_1p8>;
+		vddio1p2-supply = <&vreg_l3c_1p2>;
+		vddaon-supply = <&vreg_s2c_0p8>;
+		vdddig-supply = <&vreg_s3c_0p9>;
+		vddrfa1p2-supply = <&vreg_s1c_1p2>;
+		vddrfa1p8-supply = <&vreg_s6c_1p8>;
+
+		clocks = <&rpmhcc RPMH_RF_CLK1>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p8: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p8";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm8550-rpmh-regulators";
+
+		vdd-bob1-supply = <&vph_pwr>;
+		vdd-bob2-supply = <&vph_pwr>;
+		vdd-l1-l4-l10-supply = <&vreg_s6c_1p8>;
+		vdd-l2-l13-l14-supply = <&vreg_bob1>;
+		vdd-l3-supply = <&vreg_s1c_1p2>;
+		vdd-l5-l16-supply = <&vreg_bob1>;
+		vdd-l8-l9-supply = <&vreg_bob1>;
+		vdd-l12-supply = <&vreg_s6c_1p8>;
+		vdd-l15-supply = <&vreg_s6c_1p8>;
+		vdd-l17-supply = <&vreg_bob2>;
+
+		qcom,pmic-id = "b";
+
+		vreg_bob1: bob1 {
+			regulator-name = "vreg_bob1";
+			regulator-min-microvolt = <3296000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob2: bob2 {
+			regulator-name = "vreg_bob2";
+			regulator-min-microvolt = <2720000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b_1p8: ldo1 {
+			regulator-name = "vreg_l1b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b_3p0: ldo2 {
+			regulator-name = "vreg_l2b_3p0";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b_3p1: ldo5 {
+			regulator-name = "vreg_l5b_3p1";
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_1p8: ldo8 {
+			regulator-name = "vreg_l8b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_2p9: ldo9 {
+			regulator-name = "vreg_l9b_2p9";
+			regulator-min-microvolt = <2960000>;
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b_1p8: ldo10 {
+			regulator-name = "vreg_l10b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_1p8: ldo12 {
+			regulator-name = "vreg_l12b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_1p8: ldo15 {
+			regulator-name = "vreg_l15b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_2p5: ldo17 {
+			regulator-name = "vreg_l17b_2p5";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s1c_1p2>;
+		vdd-l3-supply = <&vreg_s1c_1p2>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		qcom,pmic-id = "c";
+
+		vreg_s1c_1p2: smps1 {
+			regulator-name = "vreg_s1c_1p2";
+			regulator-min-microvolt = <1224000>;
+			regulator-max-microvolt = <1348000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2c_0p8: smps2 {
+			regulator-name = "vreg_s2c_0p8";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1036000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3c_0p9: smps3 {
+			regulator-name = "vreg_s3c_0p9";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <1064000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4c_1p2: smps4 {
+			regulator-name = "vreg_s4c_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1280000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5c_0p7: smps5 {
+			regulator-name = "vreg_s5c_0p7";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s6c_1p8: smps6 {
+			regulator-name = "vreg_s6c_1p8";
+			regulator-min-microvolt = <1856000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1c_1p2: ldo1 {
+			regulator-name = "vreg_l1c_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c_1p2: ldo3 {
+			regulator-name = "vreg_l3c_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s3c_0p9>;
+
+		qcom,pmic-id = "d";
+
+		vreg_l1d_0p9: ldo1 {
+			regulator-name = "vreg_l1d_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-3 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l3-supply = <&vreg_s3c_0p9>;
+
+		qcom,pmic-id = "e";
+
+		vreg_l3e_0p9: ldo3 {
+			regulator-name = "vreg_l3e_0p9";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-4 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s3c_0p9>;
+		vdd-l2-supply = <&vreg_s1c_1p2>;
+		vdd-l3-supply = <&vreg_s3c_0p9>;
+
+		qcom,pmic-id = "g";
+
+		vreg_l1g_0p9: ldo1 {
+			regulator-name = "vreg_l1g_0p9";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2g_1p1: ldo2 {
+			regulator-name = "vreg_l2g_1p1";
+			regulator-min-microvolt = <1104000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3g_0p91: ldo3 {
+			regulator-name = "vreg_l3g_0p91";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-5 {
+		compatible = "qcom,pm8550ve-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s3c_0p9>;
+		vdd-l2-supply = <&vreg_s3c_0p9>;
+		vdd-l3-supply = <&vreg_s1c_1p2>;
+		vdd-s4-supply = <&vph_pwr>;
+
+		qcom,pmic-id = "i";
+
+		vreg_s4i_0p85: smps4 {
+			regulator-name = "vreg_s4i_0p85";
+			regulator-min-microvolt = <852000>;
+			regulator-max-microvolt = <1004000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1i_0p88: ldo1 {
+			regulator-name = "vreg_l1i_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2i_0p88: ldo2 {
+			regulator-name = "vreg_l2i_0p88";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3i_1p2: ldo3 {
+			regulator-name = "vreg_l3i_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-6 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+
+		vdd-l1-l2-supply = <&vreg_s1c_1p2>;
+		vdd-l3-l4-supply = <&vreg_bob2>;
+		vdd-l5-supply = <&vreg_s6c_1p8>;
+		vdd-l6-supply = <&vreg_bob2>;
+		vdd-l7-supply = <&vreg_bob1>;
+
+		qcom,pmic-id = "m";
+
+		vreg_l1m_1p1: ldo1 {
+			regulator-name = "vreg_l1m_1p1";
+			regulator-min-microvolt = <1104000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2m_1p2: ldo2 {
+			regulator-name = "vreg_l2m_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3m_2p8: ldo3 {
+			regulator-name = "vreg_l3m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4m_2p8: ldo4 {
+			regulator-name = "vreg_l4m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5m_1p8: ldo5 {
+			regulator-name = "vreg_l5m_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6m_2p8: ldo6 {
+			regulator-name = "vreg_l6m_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7m_3p3: ldo7 {
+			regulator-name = "vreg_l7m_3p3";
+			regulator-min-microvolt = <3304000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+
+			regulator-always-on;
+		};
+	};
+
+	regulators-7 {
+		compatible = "qcom,pm8010-rpmh-regulators";
+
+		vdd-l1-l2-supply = <&vreg_s1c_1p2>;
+		vdd-l3-l4-supply = <&vreg_bob2>;
+		vdd-l5-supply = <&vreg_s6c_1p8>;
+		vdd-l6-supply = <&vreg_bob2>;
+
+		qcom,pmic-id = "n";
+
+		vreg_l1n_1p1: ldo1 {
+			regulator-name = "vreg_l1n_1p1";
+			regulator-min-microvolt = <1104000>;
+			regulator-max-microvolt = <1104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2n_1p2: ldo2 {
+			regulator-name = "vreg_l2n_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3n_2p8: ldo3 {
+			regulator-name = "vreg_l3n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4n_2p8: ldo4 {
+			regulator-name = "vreg_l4n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5n_1p8: ldo5 {
+			regulator-name = "vreg_l5n_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6n_2p8: ldo6 {
+			regulator-name = "vreg_l6n_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&dispcc {
+	status = "disabled";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpi_dma2 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&i2c_hub_8 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	right_spkr_i2s: speaker@38 {
+		compatible = "maxim,max98390";
+		reg = <0x38>;
+
+		reset-gpios = <&tlmm 73 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&r_spkr_irq_default>;
+		pinctrl-names = "default";
+
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "Speaker Right";
+
+		maxim,temperature_calib = <1024>;
+		maxim,r0_calib = <100232>;
+	};
+
+	left_spkr_i2s: speaker@3d {
+		compatible = "maxim,max98390";
+		reg = <0x3d>;
+
+		reset-gpios = <&tlmm 161 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&l_spkr_irq_default>;
+		pinctrl-names = "default";
+
+		#sound-dai-cells = <0>;
+		sound-name-prefix = "Speaker Left";
+
+		maxim,temperature_calib = <1024>;
+		maxim,r0_calib = <100232>;
+	};
+
+	light-sensor@60 {
+		compatible = "vishay,vcnl4040";
+		reg = <0x60>;
+
+		vdd-supply = <&vreg_l2b_3p0>;
+		vio-supply = <&vreg_l1b_1p8>;
+
+		proximity-near-level = <220>;
+	};
+};
+
+&i2c_master_hub_0 {
+	firmware-name = "qcom/sm8650/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	eeprom@50 {
+		compatible  = "atmel,24c32";
+		reg = <0x50>;
+
+		label = "strap_eeprom";
+		vcc-supply  = <&vreg_l15b_1p8>;
+	};
+
+	eeprom@51 {
+		compatible  = "atmel,24c04";
+		reg = <0x51>;
+
+		label = "strap_v2_eeprom";
+		vcc-supply = <&vreg_l15b_1p8>;
+	};
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	magnetometer@6c {
+		compatible = "allegromicro,als31300-500";
+		reg = <0x6c>;
+
+		vcc-supply = <&vreg_l2b_3p0>;
+	};
+};
+
+&i2c5 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	eeprom@50 {
+		compatible  = "atmel,24c64";
+		reg = <0x50>;
+		label = "core_v2_eeprom";
+		vcc-supply  = <&vreg_l15b_1p8>;
+		pagesize = <32>;
+		size = <8192>;
+		address-width = <16>;
+		wp-gpios = <&tlmm 78 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&pcie0 {
+	wake-gpios = <&tlmm 96 GPIO_ACTIVE_HIGH>;
+	perst-gpios = <&tlmm 94 GPIO_ACTIVE_LOW>;
+
+	pinctrl-0 = <&pcie0_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&pcieport0 {
+	wifi@0 {
+		compatible = "pci17cb,1107";
+		reg = <0x10000 0 0 0 0>;
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+		vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>;
+		vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>;
+	};
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&vreg_l1i_0p88>;
+	vdda-pll-supply = <&vreg_l3i_1p2>;
+
+	status = "okay";
+};
+
+&pm8550_gpios {
+	sdc2_card_det_n: sdc2-card-det-state {
+		pins = "gpio12";
+		function = "normal";
+		bias-pull-up;
+		input-enable;
+		output-disable;
+		power-source = <1>; /* 1.8 V */
+	};
+
+	volume_up_n: volume-up-n-state {
+		pins = "gpio6";
+		function = "normal";
+		bias-pull-up;
+		input-enable;
+		power-source = <1>;
+	};
+};
+
+&pm8550_pwm {
+	status = "okay";
+
+	multi-led {
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_STATUS;
+		default-state = "keep";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_RED>;
+		};
+
+		led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_GREEN>;
+		};
+
+		led@3 {
+			reg = <3>;
+			color = <LED_COLOR_ID_BLUE>;
+		};
+	};
+};
+
+&pm8550_temp_alarm {
+	io-channels = <&pmk8550_vadc ADC5_GEN3_DIE_TEMP(1)>;
+	io-channel-names = "thermal";
+};
+
+&pm8550b_gpios {
+	detect_n: detect-n-state {
+		pins = "gpio6";
+		function = "normal";
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+
+	select_n: select-n-state {
+		pins = "gpio5";
+		function = "normal";
+		power-source = <1>;
+		bias-pull-up;
+		input-enable;
+	};
+};
+
+&pm8550b_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_1p8>;
+	vdd3-supply = <&vreg_l5b_3p1>;
+};
+
+&pm8550b_temp_alarm {
+	io-channels = <&pmk8550_vadc ADC5_GEN3_DIE_TEMP(7)>;
+	io-channel-names = "thermal";
+};
+
+&pm8550vs_c {
+	status = "okay";
+};
+
+&pm8550vs_c_temp_alarm {
+	io-channels = <&pmk8550_vadc ADC5_GEN3_DIE_TEMP(2)>;
+	io-channel-names = "thermal";
+};
+
+&pm8550vs_d {
+	status = "okay";
+};
+
+&pm8550vs_d_temp_alarm {
+	io-channels = <&pmk8550_vadc ADC5_GEN3_DIE_TEMP(3)>;
+	io-channel-names = "thermal";
+};
+
+&pm8550vs_e {
+	status = "okay";
+};
+
+&pm8550vs_e_temp_alarm {
+	io-channels = <&pmk8550_vadc ADC5_GEN3_DIE_TEMP(4)>;
+	io-channel-names = "thermal";
+};
+
+&pm8550vs_g {
+	status = "okay";
+};
+
+&pm8550vs_g_temp_alarm {
+	io-channels = <&pmk8550_vadc ADC5_GEN3_DIE_TEMP(6)>;
+	io-channel-names = "thermal";
+};
+
+&pm8550ve_temp_alarm {
+	io-channels = <&pmk8550_vadc ADC5_GEN3_DIE_TEMP(PMK8550VE_SID)>;
+	io-channel-names = "thermal";
+};
+
+&pmk8550_rtc {
+	allow-set-time;
+
+	status = "okay";
+};
+
+&pmk8550_vadc {
+	/* PMK8550 Channel nodes */
+	channel@44 {
+		reg = <ADC5_GEN3_AMUX1_THM_100K_PU(0)>;
+		label = "pmk8550_xo_therm";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@49 {
+		reg = <ADC5_GEN3_AMUX6_THM_100K_PU(0)>;
+		label = "display_left_therm";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <15>;
+	};
+
+	/* PM8550 Channel nodes */
+	channel@100 {
+		reg = <ADC5_GEN3_REF_GND(1)>;
+		label = "pm8550_offset_ref";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@101 {
+		reg = <ADC5_GEN3_1P25VREF(1)>;
+		label = "pm8550_vref_1p25";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@103 {
+		reg = <ADC5_GEN3_DIE_TEMP(1)>;
+		label = "pm8550_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@144 {
+		reg = <ADC5_GEN3_AMUX1_THM_100K_PU(1)>;
+		label = "pcba_at_fan_exhaust";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+	};
+
+	channel@145 {
+		reg = <ADC5_GEN3_AMUX2_THM_100K_PU(1)>;
+		label = "pcba_at_40pin_connector";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+	};
+
+	channel@146 {
+		reg = <ADC5_GEN3_AMUX3_THM_100K_PU(1)>;
+		label = "pcba_at_wifi";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+	};
+
+	channel@147 {
+		reg = <ADC5_GEN3_AMUX4_THM_100K_PU(1)>;
+		label = "pcba_at_flash";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+	};
+
+	channel@148 {
+		reg = <ADC5_GEN3_AMUX5_THM_100K_PU(1)>;
+		label = "pcba_at_nrf";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+	};
+
+	channel@18e {
+		reg = <ADC5_GEN3_VPH_PWR(1)>;
+		label = "pm8550_vph_pwr";
+		qcom,pre-scaling = <1 3>;
+	};
+
+	/* PM8550VS_C Channel nodes */
+	channel@203 {
+		reg = <ADC5_GEN3_DIE_TEMP(2)>;
+		label = "pm8550vs_c_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	/* PM8550VS_D Channel nodes */
+	channel@303 {
+		reg = <ADC5_GEN3_DIE_TEMP(3)>;
+		label = "pm8550vs_d_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	/* PM8550VS_E Channel nodes */
+	channel@403 {
+		reg = <ADC5_GEN3_DIE_TEMP(4)>;
+		label = "pm8550vs_e_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	/* PM8550VE Channel nodes */
+	channel@503 {
+		reg = <ADC5_GEN3_DIE_TEMP(PMK8550VE_SID)>;
+		label = "pm8550ve_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	/* PM8550VS_G Channel nodes */
+	channel@603 {
+		reg = <ADC5_GEN3_DIE_TEMP(6)>;
+		label = "pm8550vs_g_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	/* PM8550B Channel nodes */
+	channel@700 {
+		reg = <ADC5_GEN3_REF_GND(7)>;
+		label = "pm8550b_offset_ref";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@701 {
+		reg = <ADC5_GEN3_1P25VREF(7)>;
+		label = "pm8550b_vref_1p25";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@703 {
+		reg = <ADC5_GEN3_DIE_TEMP(7)>;
+		label = "pm8550b_die_temp";
+		qcom,pre-scaling = <1 1>;
+	};
+
+	channel@711 {
+		reg = <ADC5_GEN3_USB_SNS_V_16(7)>;
+		label = "pm8550b_usb_sns_v_div_16";
+		qcom,pre-scaling = <1 16>;
+	};
+
+	channel@712 {
+		reg = <ADC5_GEN3_VIN_DIV16_MUX(7)>;
+		label = "pm8550b_wls_vin_div_16";
+		qcom,pre-scaling = <1 16>;
+	};
+
+	channel@744 {
+		reg = <ADC5_GEN3_AMUX1_THM_100K_PU(7)>;
+		label = "battery_temp";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+	};
+
+	channel@747 {
+		reg = <ADC5_GEN3_AMUX4_THM_100K_PU(7)>;
+		label = "pcba_at_heat_sink_fins";
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+	};
+
+	channel@78e {
+		reg = <ADC5_GEN3_VPH_PWR(7)>;
+		label = "pm8550b_vph_pwr";
+		qcom,pre-scaling = <1 3>;
+	};
+
+	channel@78f {
+		reg = <ADC5_GEN3_VBAT_SNS_QBG(7)>;
+		label = "pm8550b_vbat_sns_qbg";
+		qcom,pre-scaling = <1 6>;
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	firmware-name = "qcom/sm8650/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	firmware-name = "qcom/sm8650/qupv3fw.elf";
+
+	status = "okay";
+};
+
+&sdhc_2 {
+	cd-gpios = <&pm8550_gpios 12 GPIO_ACTIVE_LOW>;
+
+	vmmc-supply = <&vreg_l9b_2p9>;
+	vqmmc-supply = <&vreg_l8b_1p8>;
+	bus-width = <4>;
+	no-sdio;
+	no-mmc;
+
+	pinctrl-0 = <&sdc2_default>, <&sdc2_card_det_n>;
+	pinctrl-1 = <&sdc2_sleep>, <&sdc2_card_det_n>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+};
+
+&sleep_clk {
+	clock-frequency = <32764>;
+};
+
+&tlmm {
+	bt_default: bt-default-state {
+		bt-en-pins {
+			pins = "gpio17";
+			function = "gpio";
+			drive-strength = <16>;
+			bias-disable;
+		};
+
+		sw-ctrl-pins {
+			pins = "gpio18";
+			function = "gpio";
+			bias-pull-down;
+		};
+	};
+
+	i2c4_default: i2c4-default-state {
+		pins = "gpio48", "gpio49";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	i2c5_default: i2c5-default-state {
+		pins = "gpio52", "gpio53";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	l_spkr_irq_default: l-spkr-irq-state-state {
+		pins = "gpio77";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	r_spkr_irq_default: r-spkr-irq-default-state {
+		pins = "gpio186";
+		function = "gpio";
+		bias-pull-up;
+	};
+
+	wlan_en: wlan-en-state {
+		pins = "gpio16";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-down;
+	};
+};
+
+&uart14 {
+	status = "okay";
+
+	bluetooth {
+		compatible = "qcom,wcn7850-bt";
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+		vddrfa1p8-supply = <&vreg_pmu_rfa_1p8>;
+
+		max-speed = <3200000>;
+	};
+};
+
+&uart15 {
+	status = "okay";
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 210 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l17b_2p5>;
+	vcc-max-microamp = <1300000>;
+	vccq-supply = <&vreg_l1c_1p2>;
+	vccq-max-microamp = <1200000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vreg_l1d_0p9>;
+	vdda-pll-supply = <&vreg_l3i_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	dr_mode = "peripheral";
+
+	/* USB 2.0 only */
+	qcom,select-utmi-as-pipe-clk;
+	maximum-speed = "high-speed";
+
+	/* Remove USB3 phy */
+	phys = <&usb_1_hsphy>;
+	phy-names = "usb2-phy";
+
+	status = "okay";
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l1i_0p88>;
+	vdda12-supply = <&vreg_l3i_1p2>;
+
+	phys = <&pm8550b_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&xo_board {
+	clock-frequency = <76800000>;
+};
diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi
index b679fd8afb7d53..ac3896448f18d5 100644
--- a/arch/arm64/boot/dts/qcom/sm8650.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi
@@ -21,7 +21,6 @@
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/reset/qcom,sm8650-gpucc.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
@@ -1907,7 +1906,7 @@
 		};
 	};
 
-	smp2p-adsp {
+	smp2p_adsp: smp2p-adsp {
 		compatible = "qcom,smp2p";
 
 		interrupts-extended = <&ipcc IPCC_CLIENT_LPASS
@@ -1933,7 +1932,7 @@
 		};
 	};
 
-	smp2p-cdsp {
+	smp2p_cdsp: smp2p-cdsp {
 		compatible = "qcom,smp2p";
 
 		interrupts-extended = <&ipcc IPCC_CLIENT_CDSP
@@ -1959,7 +1958,7 @@
 		};
 	};
 
-	smp2p-modem {
+	smp2p_modem: smp2p-modem {
 		compatible = "qcom,smp2p";
 
 		interrupts-extended = <&ipcc IPCC_CLIENT_MPSS
@@ -3700,9 +3699,6 @@
 			num-lanes = <2>;
 			bus-range = <0 0xff>;
 
-			phys = <&pcie0_phy>;
-			phy-names = "pciephy";
-
 			#address-cells = <3>;
 			#size-cells = <2>;
 			ranges = <0x01000000 0 0x00000000 0 0x60200000 0 0x100000>,
@@ -3769,6 +3765,8 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie0_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -3893,9 +3891,6 @@
 			num-lanes = <2>;
 			bus-range = <0 0xff>;
 
-			phys = <&pcie1_phy>;
-			phy-names = "pciephy";
-
 			dma-coherent;
 
 			#address-cells = <3>;
@@ -3978,6 +3973,8 @@
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				bus-range = <0x01 0xff>;
 
+				phys = <&pcie1_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
@@ -6154,7 +6151,6 @@
 
 			qcom,ee = <0>;
 			qcom,channel = <0>;
-			qcom,bus-id = <0>;
 
 			interrupt-controller;
 			#interrupt-cells = <4>;
diff --git a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts
index 9b5fbc004df4ec..64bc1cec612efa 100644
--- a/arch/arm64/boot/dts/qcom/sm8750-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sm8750-mtp.dts
@@ -935,6 +935,14 @@
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm8750/gen80000_zap.mbn";
+};
+
 &i2c3 {
 	status = "okay";
 
@@ -1119,7 +1127,7 @@
 };
 
 &pcieport0 {
-	wake-gpios = <&tlmm 104 GPIO_ACTIVE_HIGH>;
+	wake-gpios = <&tlmm 104 GPIO_ACTIVE_LOW>;
 	reset-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
 
 	wifi@0 {
diff --git a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts
index b56317cfa9fc7f..75927e1f7b9ab0 100644
--- a/arch/arm64/boot/dts/qcom/sm8750-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/sm8750-qrd.dts
@@ -858,6 +858,14 @@
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm8750/gen80000_zap.mbn";
+};
+
 &iris {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi
index dd738d13df8e04..8ed5fa4d24fcd8 100644
--- a/arch/arm64/boot/dts/qcom/sm8750.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi
@@ -5,10 +5,12 @@
 
 #include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/clock/qcom,kaanapali-gxclkctl.h>
 #include <dt-bindings/clock/qcom,sm8750-cambistmclkcc.h>
 #include <dt-bindings/clock/qcom,sm8750-camcc.h>
 #include <dt-bindings/clock/qcom,sm8750-dispcc.h>
 #include <dt-bindings/clock/qcom,sm8750-gcc.h>
+#include <dt-bindings/clock/qcom,sm8750-gpucc.h>
 #include <dt-bindings/clock/qcom,sm8750-tcsr.h>
 #include <dt-bindings/clock/qcom,sm8750-videocc.h>
 #include <dt-bindings/dma/qcom-gpi.h>
@@ -20,7 +22,6 @@
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
@@ -462,7 +463,7 @@
 			no-map;
 		};
 
-		spss_region_mem: spss@9b0a0000  {
+		spss_region_mem: spss@9b0a0000 {
 			reg = <0x0 0x9b0a0000 0x0 0x1e0000>;
 			no-map;
 		};
@@ -2180,6 +2181,291 @@
 			#hwlock-cells = <1>;
 		};
 
+		gpu: gpu@3d00000 {
+			compatible = "qcom,adreno-44050001", "qcom,adreno";
+			reg = <0x0 0x03d00000 0x0 0x6d000>,
+			      <0x0 0x03d9e000 0x0 0x2000>;
+			reg-names = "kgsl_3d0_reg_memory",
+				    "cx_mem";
+
+			interrupts = <GIC_SPI 300 IRQ_TYPE_LEVEL_HIGH>;
+
+			iommus = <&adreno_smmu 0 0x0>,
+				 <&adreno_smmu 1 0x0>;
+
+			operating-points-v2 = <&gpu_opp_table>;
+
+			qcom,gmu = <&gmu>;
+			#cooling-cells = <2>;
+
+			nvmem-cells = <&gpu_speed_bin>;
+			nvmem-cell-names = "speed_bin";
+
+			interconnects = <&gem_noc MASTER_GFX3D QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "gfx-mem";
+
+			status = "disabled";
+
+			gpu_zap_shader: zap-shader {
+				memory-region = <&gpu_micro_code_mem>;
+			};
+
+			gpu_opp_table: opp-table {
+				compatible = "operating-points-v2-adreno",
+					     "operating-points-v2";
+
+				opp-160000000 {
+					opp-hz = /bits/ 64 <160000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D3>;
+					opp-peak-kBps = <2136718>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-222000000 {
+					opp-hz = /bits/ 64 <222000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D2>;
+					opp-peak-kBps = <2136718>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-342000000 {
+					opp-hz = /bits/ 64 <342000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D1>;
+					opp-peak-kBps = <5285156>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-389000000 {
+					opp-hz = /bits/ 64 <389000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_D0>;
+					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0xc8285ffd>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-443000000 {
+					opp-hz = /bits/ 64 <443000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0xc02e5ffd>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-525000000 {
+					opp-hz = /bits/ 64 <525000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS_L1>;
+					opp-peak-kBps = <6074218>;
+					qcom,opp-acd-level = <0xc02e5ffd>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-607000000 {
+					opp-hz = /bits/ 64 <607000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+					opp-peak-kBps = <8171875>;
+					qcom,opp-acd-level = <0xa82e5ffd>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-660000000 {
+					opp-hz = /bits/ 64 <660000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L0>;
+					opp-peak-kBps = <8171875>;
+					qcom,opp-acd-level = <0x882c5ffd>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-734000000 {
+					opp-hz = /bits/ 64 <734000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					opp-peak-kBps = <12449218>;
+					qcom,opp-acd-level = <0xa82a5ffd>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-832000000 {
+					opp-hz = /bits/ 64 <832000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L2>;
+					opp-peak-kBps = <16500000>;
+					qcom,opp-acd-level = <0x882a5ffd>;
+					opp-supported-hw = <0x1f>;
+				};
+
+				opp-900000000-0 {
+					opp-hz = /bits/ 64 <900000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					opp-peak-kBps = <16500000>;
+					qcom,opp-acd-level = <0x882a5ffd>;
+					opp-supported-hw = <0x0f>;
+				};
+
+				/* Only applicable for SKUs which has 900Mhz as Fmax */
+				opp-900000000-1 {
+					opp-hz = /bits/ 64 <900000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM>;
+					opp-peak-kBps = <18597656>;
+					qcom,opp-acd-level = <0x882a5ffd>;
+					opp-supported-hw = <0x10>;
+				};
+
+				opp-967000000 {
+					opp-hz = /bits/ 64 <967000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_NOM_L1>;
+					opp-peak-kBps = <18597656>;
+					qcom,opp-acd-level = <0x882a5ffd>;
+					opp-supported-hw = <0x0f>;
+				};
+
+				opp-1050000000 {
+					opp-hz = /bits/ 64 <1050000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L1>;
+					opp-peak-kBps = <18597656>;
+					qcom,opp-acd-level = <0xa8295ffd>;
+					opp-supported-hw = <0x0f>;
+				};
+
+				opp-1100000000 {
+					opp-hz = /bits/ 64 <1100000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3>;
+					opp-peak-kBps = <18597656>;
+					qcom,opp-acd-level = <0x88295ffd>;
+					opp-supported-hw = <0x05>;
+				};
+
+				opp-1150000000 {
+					opp-hz = /bits/ 64 <1150000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L3>;
+					opp-peak-kBps = <18597656>;
+					qcom,opp-acd-level = <0x88295ffd>;
+					opp-supported-hw = <0x02>;
+				};
+
+				opp-1200000000 {
+					opp-hz = /bits/ 64 <1200000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_TURBO_L4>;
+					opp-peak-kBps = <18597656>;
+					qcom,opp-acd-level = <0x88295ffd>;
+					opp-supported-hw = <0x01>;
+				};
+			};
+		};
+
+		gxclkctl: clock-controller@3d64000 {
+			compatible = "qcom,sm8750-gxclkctl";
+			reg = <0x0 0x03d64000 0x0 0x6000>;
+
+			power-domains = <&rpmhpd RPMHPD_GFX>,
+					<&rpmhpd RPMHPD_GMXC>,
+					<&gpucc GPU_CC_CX_GDSC>;
+
+			#power-domain-cells = <1>;
+		};
+
+		gmu: gmu@3d6c000 {
+			compatible = "qcom,adreno-gmu-830.1", "qcom,adreno-gmu";
+
+			reg = <0x0 0x03d6c000 0x0 0x32000>;
+			reg-names = "gmu";
+
+			interrupts = <GIC_SPI 304 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 305 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hfi", "gmu";
+
+			clocks = <&gpucc GPU_CC_AHB_CLK>,
+				 <&gpucc GPU_CC_CX_GMU_CLK>,
+				 <&gpucc GPU_CC_CXO_CLK>,
+				 <&gcc GCC_DDRSS_GPU_AXI_CLK>,
+				 <&gcc GCC_GPU_GEMNOC_GFX_CLK>,
+				 <&gpucc GPU_CC_HUB_CX_INT_CLK>;
+			clock-names = "ahb",
+				      "gmu",
+				      "cxo",
+				      "axi",
+				      "memnoc",
+				      "hub";
+
+			power-domains = <&gpucc GPU_CC_CX_GDSC>,
+					<&gxclkctl GX_CLKCTL_GX_GDSC>;
+			power-domain-names = "cx",
+					     "gx";
+
+			iommus = <&adreno_smmu 5 0x0>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			operating-points-v2 = <&gmu_opp_table>;
+
+			gmu_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-500000000 {
+					opp-hz = /bits/ 64 <500000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_LOW_SVS>;
+				};
+
+				opp-650000000 {
+					opp-hz = /bits/ 64 <650000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS>;
+				};
+			};
+		};
+
+		gpucc: clock-controller@3d90000 {
+			compatible = "qcom,sm8750-gpucc";
+			reg = <0x0 0x03d90000 0x0 0x9800>;
+
+			clocks = <&bi_tcxo_div2>,
+				 <&gcc GCC_GPU_GPLL0_CLK_SRC>,
+				 <&gcc GCC_GPU_GPLL0_DIV_CLK_SRC>;
+
+			power-domains = <&rpmhpd RPMHPD_MX>,
+					<&rpmhpd RPMHPD_CX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
+		adreno_smmu: iommu@3da0000 {
+			compatible = "qcom,sm8750-smmu-500", "qcom,adreno-smmu",
+				     "qcom,smmu-500", "arm,mmu-500";
+			reg = <0x0 0x03da0000 0x0 0x40000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <1>;
+			interrupts = <GIC_SPI 674 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 678 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 679 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 680 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 681 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 682 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 683 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 684 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 685 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 686 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 687 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 688 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 476 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 574 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 575 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 576 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 577 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 660 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 662 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 665 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 666 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 667 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 669 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 670 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 700 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gpucc GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK>;
+			clock-names = "hlos";
+			power-domains = <&gpucc GPU_CC_CX_GDSC>;
+			dma-coherent;
+		};
+
 		remoteproc_mpss: remoteproc@4080000 {
 			compatible = "qcom,sm8750-mpss-pas";
 			reg = <0x0 0x04080000 0x0 0x10000>;
@@ -3609,7 +3895,6 @@
 
 			qcom,ee = <0>;
 			qcom,channel = <0>;
-			qcom,bus-id = <0>;
 
 			interrupt-controller;
 			#interrupt-cells = <4>;
@@ -5951,6 +6236,18 @@
 			};
 		};
 
+		qfprom: efuse@221c8000 {
+			compatible = "qcom,sm8750-qfprom", "qcom,qfprom";
+			reg = <0x0 0x221c8000 0x0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			gpu_speed_bin: gpu-speed-bin@138 {
+				reg = <0x138 0x2>;
+				bits = <0 9>;
+			};
+		};
+
 		/* cluster0 */
 		pmu@240b3400 {
 			compatible = "qcom,sm8750-cpu-bwmon", "qcom,sdm845-bwmon";
@@ -6647,10 +6944,10 @@
 			thermal-sensors = <&tsens2 1>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss0_alert0: gpuss0-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss0-critical {
@@ -6659,16 +6956,23 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss0_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss1-thermal {
 			thermal-sensors = <&tsens2 2>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss1_alert0: gpuss1-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss1-critical {
@@ -6677,16 +6981,23 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss1_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss2-thermal {
 			thermal-sensors = <&tsens2 3>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss2_alert0: gpuss2-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss2-critical {
@@ -6695,16 +7006,23 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss2_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss3-thermal {
 			thermal-sensors = <&tsens2 4>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss3_alert0: gpuss3-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss3-critical {
@@ -6713,16 +7031,23 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss3_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss4-thermal {
 			thermal-sensors = <&tsens2 5>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss4_alert0: gpuss4-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss4-critical {
@@ -6731,16 +7056,23 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss4_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss5-thermal {
 			thermal-sensors = <&tsens2 6>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss5_alert0: gpuss5-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss5-critical {
@@ -6749,16 +7081,23 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss5_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss6-thermal {
 			thermal-sensors = <&tsens2 7>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss6_alert0: gpuss6-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss6-critical {
@@ -6767,16 +7106,23 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss6_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		gpuss7-thermal {
 			thermal-sensors = <&tsens2 8>;
 
 			trips {
-				trip-point0 {
-					temperature = <120000>;
+				gpuss7_alert0: gpuss7-alert0 {
+					temperature = <105000>;
 					hysteresis = <5000>;
-					type = "hot";
+					type = "passive";
 				};
 
 				gpuss7-critical {
@@ -6785,6 +7131,13 @@
 					type = "critical";
 				};
 			};
+
+			cooling-maps {
+				map0 {
+					trip = <&gpuss7_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 
 		modem0-thermal {
diff --git a/arch/arm64/boot/dts/qcom/talos-evk-rpi-display-2-5in.dtso b/arch/arm64/boot/dts/qcom/talos-evk-rpi-display-2-5in.dtso
new file mode 100644
index 00000000000000..e49da096d9e384
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/talos-evk-rpi-display-2-5in.dtso
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+
+&{/} {
+	display_bl: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&mcu 0 255 0>;
+	};
+
+	reg_display: regulator-display {
+		compatible = "regulator-fixed";
+		regulator-name = "rpi-display";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	reg_dsi_touch: regulator-dsi-touch {
+		compatible = "regulator-fixed";
+		gpio = <&mcu 1 GPIO_ACTIVE_HIGH>;
+		regulator-name = "rpi-touch";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		startup-delay-us = <50000>;
+		enable-active-high;
+	};
+};
+
+&hdmi_bridge {
+	status = "disabled";
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	mcu: gpio@45 {
+		compatible = "raspberrypi,touchscreen-panel-regulator-v2";
+		reg = <0x45>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		#pwm-cells = <3>;
+	};
+
+	touchscreen@5d {
+		compatible = "goodix,gt911";
+		reg = <0x5d>;
+		AVDD28-supply = <&reg_dsi_touch>;
+		touchscreen-size-x = <720>;
+		touchscreen-size-y = <1280>;
+	};
+};
+
+&mdss_dsi0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	status = "okay";
+
+	panel: panel@0 {
+		compatible = "raspberrypi,dsi-5inch", "ilitek,ili9881c";
+		reg = <0>;
+
+		backlight = <&display_bl>;
+		power-supply = <&reg_display>;
+		reset-gpios = <&mcu 0 GPIO_ACTIVE_LOW>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mdss_dsi0_out>;
+			};
+		};
+	};
+};
+
+&mdss_dsi0_out {
+	remote-endpoint = <&panel_in>;
+	data-lanes = <0 1>;
+};
diff --git a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi
index a31bed7fad07f2..b76df9d86cc0dd 100644
--- a/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi
+++ b/arch/arm64/boot/dts/qcom/talos-evk-som.dtsi
@@ -366,9 +366,6 @@
 };
 
 &pcie {
-	perst-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
-	wake-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>;
-
 	pinctrl-0 = <&pcie_default_state>;
 	pinctrl-names = "default";
 
@@ -383,6 +380,8 @@
 };
 
 &pcie_port0 {
+	reset-gpios = <&tlmm 89 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 100 GPIO_ACTIVE_LOW>;
 	wifi@0 {
 		compatible = "pci17cb,1103";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
@@ -411,10 +410,14 @@
 };
 
 &qupv3_id_0 {
+	firmware-name = "qcom/qcs615/qupv3fw.elf";
+
 	status = "okay";
 };
 
 &qupv3_id_1 {
+	firmware-name = "qcom/qcs615/qupv3fw.elf";
+
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/talos-evk.dts b/arch/arm64/boot/dts/qcom/talos-evk.dts
index b7f514fbc7b2d5..2ec3710a227af3 100644
--- a/arch/arm64/boot/dts/qcom/talos-evk.dts
+++ b/arch/arm64/boot/dts/qcom/talos-evk.dts
@@ -36,7 +36,7 @@
 
 		port {
 			hdmi_con_out: endpoint {
-			remote-endpoint = <&adv7535_out>;
+				remote-endpoint = <&adv7535_out>;
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi
index eedfdce3c479e7..4938efa44137e7 100644
--- a/arch/arm64/boot/dts/qcom/talos.dtsi
+++ b/arch/arm64/boot/dts/qcom/talos.dtsi
@@ -17,7 +17,6 @@
 #include <dt-bindings/interconnect/qcom,qcs615-rpmh.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/phy/phy-qcom-qmp.h>
-#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
@@ -795,7 +794,7 @@
 			};
 		};
 
-		gpi_dma0: dma-controller@800000  {
+		gpi_dma0: dma-controller@800000 {
 			compatible = "qcom,qcs615-gpi-dma", "qcom,sdm845-gpi-dma";
 			reg = <0x0 0x800000 0x0 0x60000>;
 			#dma-cells = <3>;
@@ -1367,9 +1366,6 @@
 
 			power-domains = <&gcc PCIE_0_GDSC>;
 
-			phys = <&pcie_phy>;
-			phy-names = "pciephy";
-
 			max-link-speed = <2>;
 
 			operating-points-v2 = <&pcie_opp_table>;
@@ -1397,10 +1393,13 @@
 			pcie_port0: pcie@0 {
 				device_type = "pci";
 				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				phys = <&pcie_phy>;
+
 				#address-cells = <3>;
 				#size-cells = <2>;
 				ranges;
-				bus-range = <0x01 0xff>;
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi
index 0dc7c1b5a14a0b..7ef682e928f9c0 100644
--- a/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-asus-vivobook-s15.dtsi
@@ -1036,9 +1036,31 @@
 	vdd3-supply = <&vreg_l14b_3p0>;
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
 			       <44 4>, /* SPI (TPM) */
+			       <65 1>, /* EC reset */
 			       <238 1>; /* UFS Reset */
 
 	edp_reg_en: edp-reg-en-state {
diff --git a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi
index 59cbbfbce4905d..ceda30b1f2545f 100644
--- a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi
@@ -1272,8 +1272,30 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p9>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p9>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <44 4>,  /* SPI11, TZ Protected */
+			       <65 1>,  /* EC reset */
 			       <90 1>;	/* Unknown, TZ Protected */
 
 	cam_indicator_en: cam-indicator-en-state {
diff --git a/arch/arm64/boot/dts/qcom/x1-crd.dtsi b/arch/arm64/boot/dts/qcom/x1-crd.dtsi
index 60db83b8985fec..fcd7cef6660fd9 100644
--- a/arch/arm64/boot/dts/qcom/x1-crd.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-crd.dtsi
@@ -1528,9 +1528,31 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
 			       <44 4>, /* SPI (TPM) */
+			       <65 1>, /* EC reset */
 			       <238 1>; /* UFS Reset */
 
 	ec_int_n_default: ec-int-n-state {
diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
index 4cda708455d842..f00feb3ca85c66 100644
--- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
@@ -1039,7 +1039,7 @@
 	status = "okay";
 
 	aux-bus {
-		panel {
+		panel: panel {
 			compatible = "edp-panel";
 			enable-gpios = <&tlmm 74 GPIO_ACTIVE_HIGH>;
 			power-supply = <&vreg_edp_3p3>;
@@ -1301,8 +1301,30 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <44 4>,  /* SPI11 (TPM) */
+			       <65 1>,  /* EC reset */
 			       <76 4>,  /* SPI19 (TZ Protected) */
 			       <80 2>,  /* I2C20 (Battery SMBus) */
 			       <238 1>; /* UFS Reset */
diff --git a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi
index dba8cee43e2947..443901a09d4c94 100644
--- a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi
@@ -1284,9 +1284,31 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
 			       <44 4>, /* SPI (TPM) */
+			       <65 1>, /* EC reset */
 			       <72 2>, /* Secure EC I2C connection (?) */
 			       <238 1>; /* UFS Reset */
 
diff --git a/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi b/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi
index ce7d958ca68798..a71c297627fe21 100644
--- a/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-microsoft-denali.dtsi
@@ -20,9 +20,27 @@
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		pinctrl-0 = <&hall_int_n_default>;
+		pinctrl-0 = <&vol_down_n>, <&vol_up_n>, <&hall_int_n_default>;
 		pinctrl-names = "default";
 
+		key-vol-down {
+			debounce-interval = <15>;
+			gpios = <&pm8550_gpios 8 GPIO_ACTIVE_LOW>;
+			label = "Volume Down";
+			linux,can-disable;
+			linux,code = <KEY_VOLUMEDOWN>;
+			wakeup-source;
+		};
+
+		key-vol-up {
+			debounce-interval = <15>;
+			gpios = <&pm8550_gpios 6 GPIO_ACTIVE_LOW>;
+			label = "Volume Up";
+			linux,can-disable;
+			linux,code = <KEY_VOLUMEUP>;
+			wakeup-source;
+		};
+
 		switch-lid {
 			gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
 			linux,input-type = <EV_SW>;
@@ -314,6 +332,38 @@
 				"VA DMIC0", "vdd-micb",
 				"VA DMIC1", "vdd-micb";
 
+		displayport-0-dai-link {
+			link-name = "DisplayPort0 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp0>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_0>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		displayport-1-dai-link {
+			link-name = "DisplayPort1 Playback";
+
+			codec {
+				sound-dai = <&mdss_dp1>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai DISPLAY_PORT_RX_1>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
 		wsa-dai-link {
 			link-name = "WSA Playback";
 
@@ -863,11 +913,11 @@
 };
 
 &lpass_vamacro {
-	pinctrl-0 = <&dmic01_default>, <&dmic23_default>;
+	pinctrl-0 = <&dmic01_default>;
 	pinctrl-names = "default";
 
 	vdd-micb-supply = <&vreg_l1b_1p8>;
-	qcom,dmic-sample-rate = <4800000>;
+	qcom,dmic-sample-rate = <2400000>;
 };
 
 &mdss {
@@ -984,6 +1034,22 @@
 };
 
 &pm8550_gpios {
+	vol_up_n: vol-up-n-state {
+		pins = "gpio6";
+		function = "normal";
+		bias-pull-up;
+		input-enable;
+		power-source = <1>; /* 1.8V */
+	};
+
+	vol_down_n: vol-down-n-state {
+		pins = "gpio8";
+		function = "normal";
+		bias-pull-up;
+		input-enable;
+		power-source = <1>; /* 1.8V */
+	};
+
 	rtmr0_default: rtmr0-reset-n-active-state {
 		pins = "gpio10";
 		function = "normal";
@@ -1107,6 +1173,27 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <44 4>, /* SPI (TPM) */
 			       <238 1>; /* UFS Reset */
diff --git a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts
index be1a8ac8c699ee..e105db4cbfb0ae 100644
--- a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts
+++ b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts
@@ -1193,8 +1193,30 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
-	gpio-reserved-ranges = <44 4>; /* SPI (TPM) */
+	gpio-reserved-ranges = <44 4>, /* SPI (TPM) */
+			       <65 1>; /* EC reset */
 
 	eusb3_reset_n: eusb3-reset-n-state {
 		pins = "gpio6";
diff --git a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi
index 0bee2a3d6e7a9c..93c7e3a02041c3 100644
--- a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi
@@ -1382,9 +1382,31 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
 			       <44 4>, /* SPI (TPM) */
+			       <65 1>, /* EC reset */
 			       <72 2>, /* Secure EC I2C connection (?) */
 			       <238 1>; /* UFS Reset */
 
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-crd.dts b/arch/arm64/boot/dts/qcom/x1e80100-crd.dts
index 429deffcf3e9df..29150fe912509d 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-crd.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-crd.dts
@@ -11,6 +11,95 @@
 / {
 	model = "Qualcomm Technologies, Inc. X1E80100 CRD";
 	compatible = "qcom,x1e80100-crd", "qcom,x1e80100";
+
+	thermal-zones {
+		soc-thermal {
+			thermal-sensors = <&pmk8550_vadc ADC5_GEN3_AMUX1_GPIO_100K_PU(1)>;
+		};
+
+		keyboard-thermal {
+			critical-action = "shutdown";
+			polling-delay-passive = <1000>;
+
+			thermal-sensors = <&pmk8550_vadc ADC5_GEN3_AMUX2_GPIO_100K_PU(1)>;
+
+			trips {
+				keyboard_alert0: trip-point0 {
+					temperature = <53000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				keyboard-critical {
+					temperature = <73000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					trip = <&keyboard_alert0>;
+					cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu4 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu5 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu6 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu7 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu8 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu9 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu10 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							 <&cpu11 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+		};
+
+		backcover-thermal {
+			critical-action = "shutdown";
+			polling-delay-passive = <400>;
+
+			thermal-sensors = <&pmk8550_vadc ADC5_GEN3_AMUX1_THM_100K_PU(1)>;
+
+			trips {
+				backcover_alert0: trip-point0 {
+					temperature = <64000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+
+				backcover-critical {
+					temperature = <73000>;
+					hysteresis = <1000>;
+					type = "critical";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					trip = <&backcover_alert0>;
+					cooling-device = <&gpu THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+		};
+
+		smb-thermal {
+			thermal-sensors = <&pmk8550_vadc ADC5_GEN3_AMUX2_THM_100K_PU(1)>;
+		};
+
+		sdx-thermal {
+			thermal-sensors = <&pmk8550_vadc ADC5_GEN3_AMUX3_THM_100K_PU(1)>;
+		};
+
+		east-thermal {
+			thermal-sensors = <&pmk8550_vadc ADC5_GEN3_AMUX4_THM_100K_PU(1)>;
+		};
+
+		west-thermal {
+			thermal-sensors = <&pmk8550_vadc ADC5_GEN3_AMUX5_THM_100K_PU(1)>;
+		};
+	};
 };
 
 &gpu_zap_shader {
@@ -20,3 +109,33 @@
 &iris {
 	status = "okay";
 };
+
+&pmk8550_vadc {
+	channel@144 {
+		qcom,adc-tm;
+	};
+
+	channel@145 {
+		qcom,adc-tm;
+	};
+
+	channel@146 {
+		qcom,adc-tm;
+	};
+
+	channel@147 {
+		qcom,adc-tm;
+	};
+
+	channel@148 {
+		qcom,adc-tm;
+	};
+
+	channel@14a {
+		qcom,adc-tm;
+	};
+
+	channel@14b {
+		qcom,adc-tm;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts
index 75e10d97c38674..edbba5101b822f 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts
@@ -10,6 +10,18 @@
 / {
 	model = "Dell Inspiron 14 Plus 7441";
 	compatible = "dell,inspiron-14-plus-7441", "qcom,x1e80100";
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pmk8550_pwm 0 5000000>;
+
+		brightness-levels = <0 2048 4096 8192 16384 65535>;
+		num-interpolated-steps = <20>;
+		default-brightness-level = <80>;
+
+		pinctrl-0 = <&edp_bl_pwm>;
+		pinctrl-names = "default";
+	};
 };
 
 &sound {
@@ -25,9 +37,9 @@
 };
 
 &i2c8 {
-	touchscreen@10 {
+	touchscreen@9 {
 		compatible = "hid-over-i2c";
-		reg = <0x10>;
+		reg = <0x9>;
 
 		hid-descr-addr = <0x1>;
 		interrupts-extended = <&tlmm 51 IRQ_TYPE_LEVEL_LOW>;
@@ -42,6 +54,21 @@
 	status = "okay";
 };
 
+&panel {
+	backlight = <&backlight>;
+};
+
+&pmk8550_gpios {
+	edp_bl_pwm: edp-bl-pwm-state {
+		pins = "gpio5";
+		function = "func3";
+	};
+};
+
+&pmk8550_pwm {
+	status = "okay";
+};
+
 &remoteproc_adsp {
 	firmware-name = "qcom/x1e80100/dell/inspiron-14-plus-7441/qcadsp8380.mbn",
 			"qcom/x1e80100/dell/inspiron-14-plus-7441/adsp_dtbs.elf";
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts
index 2be5a37c00ccce..f48e6bc350063d 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts
@@ -1069,9 +1069,31 @@
 	vdd3-supply = <&vreg_l14b_3p0>;
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p9>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p9>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <44 4>,  /* SPI11 (TPM) */
 			       <76 4>,  /* SPI19 (TZ Protected) */
+			       <65 1>, /* EC reset */
 			       <238 1>; /* UFS Reset */
 
 	cam_indicator_en: cam-indicator-en-state {
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts b/arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts
index b70c1e094bbf2e..e5b285b04f5364 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-honor-magicbook-art-14.dts
@@ -1066,9 +1066,31 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
-			       <44 4>; /* SPI (TPM) */
+			       <44 4>, /* SPI (TPM) */
+			       <65 1>; /* EC reset */
 
 	tpad_default: tpad-default-state {
 		pins = "gpio3";
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts
index 9eb2dfd99f20d4..a807744d1750b1 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts
@@ -951,6 +951,22 @@
 	};
 };
 
+&i2c5 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	embedded-controller@76 {
+		compatible = "lenovo,yoga-slim7x-ec", "qcom,hamoa-crd-ec";
+		reg = <0x76>;
+
+		interrupts-extended = <&tlmm 66 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-0 = <&ec_int_n_default>;
+		pinctrl-names = "default";
+	};
+};
+
 &i2c7 {
 	clock-frequency = <400000>;
 
@@ -1347,11 +1363,39 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
 			       <44 4>, /* SPI (TPM) */
+			       <65 1>, /* EC reset */
 			       <238 1>; /* UFS Reset */
 
+	ec_int_n_default: ec-int-n-state {
+		pins = "gpio66";
+		function = "gpio";
+		bias-disable;
+	};
+
 	edp_reg_en: edp-reg-en-state {
 		pins = "gpio70";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts
index e4a0b86847cdde..9b599917c15a6f 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-medion-sprchrgd-14-s1.dts
@@ -971,8 +971,6 @@
 };
 
 &mdss_dp0 {
-	sound-name-prefix = "DisplayPort0";
-
 	status = "okay";
 };
 
@@ -981,8 +979,6 @@
 };
 
 &mdss_dp2 {
-	sound-name-prefix = "DisplayPort2";
-
 	status = "okay";
 };
 
@@ -1248,9 +1244,31 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p9>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p9>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <28 4>, /* Unused */
 			       <44 4>, /* SPI (TPM) */
+			       <65 1>, /* EC reset */
 			       <238 1>; /* UFS Reset */
 
 	edp_reg_en: edp-reg-en-state {
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi
index 13d0f09e0c2a1b..9f45ad9305e888 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi
@@ -1322,6 +1322,27 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <44 4>, /* SPI (TPM) */
 			       <238 1>; /* UFS Reset */
diff --git a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts
index 78380fec8839eb..65f0d3b1441953 100644
--- a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts
+++ b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts
@@ -1177,9 +1177,31 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <33 3>, /* Unused */
 			       <44 4>, /* SPI (TPM) */
+			       <65 1>, /* EC reset */
 			       <238 1>; /* UFS Reset */
 
 	edp_reg_en: edp-reg-en-state {
diff --git a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts
index c2f21fcb1b09fb..951888845c9083 100644
--- a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts
+++ b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts
@@ -1325,8 +1325,30 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
+			       <65 1>, /* EC reset */
 			       <72 2>; /* Secure EC I2C connection (?) */
 
 	edp_hpd_default: edp-hpd-default-state {
diff --git a/arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts b/arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts
index 0314b2246cdb71..edcb6b99a384bc 100644
--- a/arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts
+++ b/arch/arm64/boot/dts/qcom/x1p42100-microsoft-sp12in.dts
@@ -968,6 +968,27 @@
 	};
 };
 
+&tcsr {
+	vdda-qrefrpt0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrpt1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrpt3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrpt4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qrefrx0-0p9-supply = <&vreg_l3c_0p8>;
+	vdda-qrefrx1-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx2-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-qrefrx3-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qrefrx4-0p9-supply = <&vreg_l1d_0p8>;
+	vdda-qreftx0-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx0-1p2-supply = <&vreg_l2j_1p2>;
+	vdda-qreftx1-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-qreftx1-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen0-0p9-supply = <&vreg_l3i_0p8>;
+	vdda-refgen0-1p2-supply = <&vreg_l3e_1p2>;
+	vdda-refgen2-0p9-supply = <&vreg_l3j_0p8>;
+	vdda-refgen2-1p2-supply = <&vreg_l2j_1p2>;
+};
+
 &tlmm {
 	gpio-reserved-ranges = <34 2>, /* Unused */
 			       <44 4>; /* SPI (TPM) */
diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile
index 8bf155badd111d..c6f3718cd19fad 100644
--- a/arch/arm64/boot/dts/renesas/Makefile
+++ b/arch/arm64/boot/dts/renesas/Makefile
@@ -8,6 +8,9 @@ dtb-$(CONFIG_ARCH_R8A774A1) += r8a774a1-hihope-rzg2m-rev2.dtb
 dtb-$(CONFIG_ARCH_R8A774A1) += r8a774a1-hihope-rzg2m-rev2-ex.dtb
 dtb-$(CONFIG_ARCH_R8A774A1) += r8a774a1-hihope-rzg2m-rev2-ex-idk-1110wr.dtb
 
+dtb-$(CONFIG_ARCH_R8A774A3) += r8a774a3-hihope-rzg2m.dtb
+dtb-$(CONFIG_ARCH_R8A774A3) += r8a774a3-hihope-rzg2m-ex.dtb
+
 dtb-$(CONFIG_ARCH_R8A774B1) += r8a774b1-beacon-rzg2n-kit.dtb
 dtb-$(CONFIG_ARCH_R8A774B1) += r8a774b1-hihope-rzg2n.dtb
 dtb-$(CONFIG_ARCH_R8A774B1) += r8a774b1-hihope-rzg2n-ex.dtb
@@ -124,6 +127,9 @@ dtb-$(CONFIG_ARCH_R8A779G0) += r8a779g3-sparrow-hawk-rpi-display-2-5in.dtb
 dtb-$(CONFIG_ARCH_R8A779G0) += r8a779g3-sparrow-hawk-rpi-display-2-7in.dtbo
 r8a779g3-sparrow-hawk-rpi-display-2-7in-dtbs := r8a779g3-sparrow-hawk.dtb r8a779g3-sparrow-hawk-rpi-display-2-7in.dtbo
 dtb-$(CONFIG_ARCH_R8A779G0) += r8a779g3-sparrow-hawk-rpi-display-2-7in.dtb
+dtb-$(CONFIG_ARCH_R8A779G0) += r8a779g3-sparrow-hawk-ws-2ch-canfd.dtbo
+r8a779g3-sparrow-hawk-ws-2ch-canfd-dtbs := r8a779g3-sparrow-hawk.dtb r8a779g3-sparrow-hawk-ws-2ch-canfd.dtbo
+dtb-$(CONFIG_ARCH_R8A779G0) += r8a779g3-sparrow-hawk-ws-2ch-canfd.dtb
 dtb-$(CONFIG_ARCH_R8A779G0) += r8a779g3-sparrow-hawk-ws-display-13in.dtbo
 r8a779g3-sparrow-hawk-ws-display-13in-dtbs := r8a779g3-sparrow-hawk.dtb r8a779g3-sparrow-hawk-ws-display-13in.dtbo
 dtb-$(CONFIG_ARCH_R8A779G0) += r8a779g3-sparrow-hawk-ws-display-13in.dtb
@@ -164,18 +170,89 @@ r9a07g043u11-smarc-du-adv7513-dtbs := r9a07g043u11-smarc.dtb r9a07g043u11-smarc-
 dtb-$(CONFIG_ARCH_R9A07G043) += r9a07g043u11-smarc-du-adv7513.dtb
 r9a07g043u11-smarc-pmod-dtbs := r9a07g043u11-smarc.dtb r9a07g043-smarc-pmod.dtbo
 dtb-$(CONFIG_ARCH_R9A07G043) += r9a07g043u11-smarc-pmod.dtb
-
+dtb-$(CONFIG_ARCH_R9A07G043) += r9a07g043u12-hummingboard-ripple.dtb
+dtb-$(CONFIG_ARCH_R9A07G043) += rzg2l-sr-som-emmc.dtbo
+r9a07g043u12-hummingboard-ripple-emmc-dtbs := r9a07g043u12-hummingboard-ripple.dtb rzg2l-sr-som-emmc.dtbo
+dtb-$(CONFIG_ARCH_R9A07G043) += r9a07g043u12-hummingboard-ripple-emmc.dtb
+dtb-$(CONFIG_ARCH_R9A07G043) += rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G043) += rzg2ul-hummingboard-pulse-leds.dtbo
+r9a07g043u12-hummingboard-ripple-leds-dtbs := r9a07g043u12-hummingboard-ripple.dtb rzg2ul-hummingboard-pulse-leds.dtbo
+dtb-$(CONFIG_ARCH_R9A07G043) += r9a07g043u12-hummingboard-ripple-leds.dtb
+r9a07g043u12-hummingboard-ripple-microsd-dtbs := r9a07g043u12-hummingboard-ripple.dtb rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G043) += r9a07g043u12-hummingboard-ripple-microsd.dtb
+
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-iiot.dtb
+r9a07g044c2-hummingboard-iiot-emmc-dtbs := r9a07g044c2-hummingboard-iiot.dtb rzg2l-sr-som-emmc.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-iiot-emmc.dtb
+r9a07g044c2-hummingboard-iiot-microsd-dtbs := r9a07g044c2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-iiot-microsd.dtb
+r9a07g044c2-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0-dtbs := r9a07g044c2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtb
+r9a07g044c2-hummingboard-iiot-rs485-a-dtbs := r9a07g044c2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-rs485-a.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-iiot-rs485-a.dtb
+r9a07g044c2-hummingboard-iiot-rs485-b-dtbs := r9a07g044c2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-rs485-b.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-iiot-rs485-b.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-ripple.dtb
+r9a07g044c2-hummingboard-ripple-emmc-dtbs := r9a07g044c2-hummingboard-ripple.dtb rzg2l-sr-som-emmc.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-ripple-emmc.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += rzg2lc-hummingboard-pulse-leds.dtbo
+r9a07g044c2-hummingboard-ripple-leds-dtbs := r9a07g044c2-hummingboard-ripple.dtb rzg2lc-hummingboard-pulse-leds.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-ripple-leds.dtb
+r9a07g044c2-hummingboard-ripple-microsd-dtbs := r9a07g044c2-hummingboard-ripple.dtb rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-hummingboard-ripple-microsd.dtb
 dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-smarc.dtb
 dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-smarc-cru-csi-ov5645.dtbo
 r9a07g044c2-smarc-cru-csi-ov5645-dtbs := r9a07g044c2-smarc.dtb r9a07g044c2-smarc-cru-csi-ov5645.dtbo
 dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044c2-smarc-cru-csi-ov5645.dtb
 
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-iiot.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += rzg2l-sr-som-emmc.dtbo
+r9a07g044l2-hummingboard-iiot-emmc-dtbs := r9a07g044l2-hummingboard-iiot.dtb rzg2l-sr-som-emmc.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-iiot-emmc.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += rzg2l-hummingboard-iiot-microsd.dtbo
+r9a07g044l2-hummingboard-iiot-microsd-dtbs := r9a07g044l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-iiot-microsd.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtbo
+r9a07g044l2-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0-dtbs := r9a07g044l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += rzg2l-hummingboard-iiot-rs485-a.dtbo
+r9a07g044l2-hummingboard-iiot-rs485-a-dtbs := r9a07g044l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-rs485-a.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-iiot-rs485-a.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += rzg2l-hummingboard-iiot-rs485-b.dtbo
+r9a07g044l2-hummingboard-iiot-rs485-b-dtbs := r9a07g044l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-rs485-b.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-iiot-rs485-b.dtb
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-ripple.dtb
+r9a07g044l2-hummingboard-ripple-emmc-dtbs := r9a07g044l2-hummingboard-ripple.dtb rzg2l-sr-som-emmc.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-ripple-emmc.dtb
+r9a07g044l2-hummingboard-ripple-microsd-dtbs := r9a07g044l2-hummingboard-ripple.dtb rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-hummingboard-ripple-microsd.dtb
 dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-remi-pi.dtb
 dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-smarc.dtb
 dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-smarc-cru-csi-ov5645.dtbo
 r9a07g044l2-smarc-cru-csi-ov5645-dtbs := r9a07g044l2-smarc.dtb r9a07g044l2-smarc-cru-csi-ov5645.dtbo
 dtb-$(CONFIG_ARCH_R9A07G044) += r9a07g044l2-smarc-cru-csi-ov5645.dtb
 
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-iiot.dtb
+dtb-$(CONFIG_ARCH_R9A07G054) += rzg2l-sr-som-emmc.dtbo
+r9a07g054l2-hummingboard-iiot-emmc-dtbs := r9a07g054l2-hummingboard-iiot.dtb rzg2l-sr-som-emmc.dtbo
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-iiot-emmc.dtb
+dtb-$(CONFIG_ARCH_R9A07G054) += rzg2l-hummingboard-iiot-microsd.dtbo
+r9a07g054l2-hummingboard-iiot-microsd-dtbs := r9a07g054l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-iiot-microsd.dtb
+dtb-$(CONFIG_ARCH_R9A07G054) += rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtbo
+r9a07g054l2-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0-dtbs := r9a07g054l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtbo
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtb
+dtb-$(CONFIG_ARCH_R9A07G054) += rzg2l-hummingboard-iiot-rs485-a.dtbo
+r9a07g054l2-hummingboard-iiot-rs485-a-dtbs := r9a07g054l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-rs485-a.dtbo
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-iiot-rs485-a.dtb
+dtb-$(CONFIG_ARCH_R9A07G054) += rzg2l-hummingboard-iiot-rs485-b.dtbo
+r9a07g054l2-hummingboard-iiot-rs485-b-dtbs := r9a07g054l2-hummingboard-iiot.dtb rzg2l-hummingboard-iiot-rs485-b.dtbo
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-iiot-rs485-b.dtb
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-ripple.dtb
+r9a07g054l2-hummingboard-ripple-emmc-dtbs := r9a07g054l2-hummingboard-ripple.dtb rzg2l-sr-som-emmc.dtbo
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-ripple-emmc.dtb
+r9a07g054l2-hummingboard-ripple-microsd-dtbs := r9a07g054l2-hummingboard-ripple.dtb rzg2l-hummingboard-iiot-microsd.dtbo
+dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-hummingboard-ripple-microsd.dtb
 dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-smarc.dtb
 dtb-$(CONFIG_ARCH_R9A07G054) += r9a07g054l2-smarc-cru-csi-ov5645.dtbo
 r9a07g054l2-smarc-cru-csi-ov5645-dtbs := r9a07g054l2-smarc.dtb r9a07g054l2-smarc-cru-csi-ov5645.dtbo
@@ -216,8 +293,14 @@ r9a09g057h48-kakip-pixpaper-dtbs := r9a09g057h48-kakip.dtb r9a09g057h48-kakip-pi
 dtb-$(CONFIG_ARCH_R9A09G057) += r9a09g057h48-kakip-pixpaper.dtb
 
 dtb-$(CONFIG_ARCH_R9A09G077) += r9a09g077m44-rzt2h-evk.dtb
+dtb-$(CONFIG_ARCH_R9A09G077) += r9a09g077m44-evk-cn15-lcdc.dtbo
+r9a09g077m44-rzt2h-evk-cn15-lcdc-dtbs := r9a09g077m44-rzt2h-evk.dtb r9a09g077m44-evk-cn15-lcdc.dtbo
+dtb-$(CONFIG_ARCH_R9A09G077) += r9a09g077m44-rzt2h-evk-cn15-lcdc.dtb
 
 dtb-$(CONFIG_ARCH_R9A09G087) += r9a09g087m44-rzn2h-evk.dtb
+dtb-$(CONFIG_ARCH_R9A09G087) += r9a09g087m44-evk-cn20-lcdc.dtbo
+r9a09g087m44-rzn2h-evk-cn20-lcdc-dtbs := r9a09g087m44-rzn2h-evk.dtb r9a09g087m44-evk-cn20-lcdc.dtbo
+dtb-$(CONFIG_ARCH_R9A09G087) += r9a09g087m44-rzn2h-evk-cn20-lcdc.dtb
 
 dtb-$(CONFIG_ARCH_RCAR_GEN3) += draak-ebisu-panel-aa104xd12.dtbo
 dtb-$(CONFIG_ARCH_RCAR_GEN3) += salvator-panel-aa104xd12.dtbo
diff --git a/arch/arm64/boot/dts/renesas/aistarvision-mipi-adapter-2.1.dtsi b/arch/arm64/boot/dts/renesas/aistarvision-mipi-adapter-2.1.dtsi
index 529388f6bf2b70..c5db0516ae83f6 100644
--- a/arch/arm64/boot/dts/renesas/aistarvision-mipi-adapter-2.1.dtsi
+++ b/arch/arm64/boot/dts/renesas/aistarvision-mipi-adapter-2.1.dtsi
@@ -54,7 +54,7 @@
 		regulator-always-on;
 	};
 
-	osc25250_clk: osc25250_clk {
+	osc25250_clk: clock-24000000 {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <24000000>;
diff --git a/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi b/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi
index 62ab0a3776e75a..963253866d717d 100644
--- a/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi
+++ b/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi
@@ -142,7 +142,7 @@
 		startup-delay-us = <100000>;
 	};
 
-	sound_card {
+	sound-card {
 		compatible = "audio-graph-card";
 		label = "rcar-sound";
 		dais = <&rsnd_port0>, <&rsnd_port1>;
@@ -505,7 +505,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
@@ -516,7 +516,7 @@
 		function = "ssi";
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clk_a_a", "audio_clk_b_a";
 		function = "audio_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi
index f8442b6a85a756..88251314904209 100644
--- a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi
@@ -13,7 +13,7 @@
 		reg = <0x0 0x48000000 0x0 0x78000000>;
 	};
 
-	osc_32k: osc_32k {
+	osc_32k: clock-32768 {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <32768>;
@@ -38,7 +38,7 @@
 		regulator-always-on;
 	};
 
-	wlan_pwrseq: wlan_pwrseq {
+	wlan_pwrseq: wlan-pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		reset-gpios = <&pca9654 1 GPIO_ACTIVE_LOW>;
 		clocks = <&osc_32k>;
@@ -63,6 +63,8 @@
 		reg = <0>;
 		interrupts-extended = <&gpio2 11 IRQ_TYPE_LEVEL_LOW>;
 		reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <300>;
 	};
 };
 
@@ -209,12 +211,12 @@
 			function = "avb";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb_mdio";
 			drive-strength = <24>;
 		};
 
-		pins_mii_tx {
+		pins-mii-tx {
 			pins = "PIN_AVB_TX_CTL", "PIN_AVB_TXC", "PIN_AVB_TD0",
 			       "PIN_AVB_TD1", "PIN_AVB_TD2", "PIN_AVB_TD3";
 			drive-strength = <12>;
@@ -251,7 +253,7 @@
 		function = "scif5";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_a";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/cat875.dtsi b/arch/arm64/boot/dts/renesas/cat875.dtsi
index 5815e9d2d8a933..196cf2e6007ea3 100644
--- a/arch/arm64/boot/dts/renesas/cat875.dtsi
+++ b/arch/arm64/boot/dts/renesas/cat875.dtsi
@@ -26,6 +26,8 @@
 		reg = <0>;
 		interrupts-extended = <&gpio2 21 IRQ_TYPE_LEVEL_LOW>;
 		reset-gpios = <&gpio1 20 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <15000>;
+		reset-deassert-us = <35000>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/renesas/condor-common.dtsi b/arch/arm64/boot/dts/renesas/condor-common.dtsi
index 9d55509b00b150..953c5bb491ecd1 100644
--- a/arch/arm64/boot/dts/renesas/condor-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/condor-common.dtsi
@@ -472,7 +472,7 @@
 		function = "scif0";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_b";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/draak.dtsi b/arch/arm64/boot/dts/renesas/draak.dtsi
index f2f25fe5d77853..33046295f2d66f 100644
--- a/arch/arm64/boot/dts/renesas/draak.dtsi
+++ b/arch/arm64/boot/dts/renesas/draak.dtsi
@@ -561,7 +561,7 @@
 		power-source = <1800>;
 	};
 
-	sdhi2_pins_uhs: sd2_uhs {
+	sdhi2_pins_uhs: sd2-uhs {
 		groups = "mmc_data8", "mmc_ctrl";
 		function = "mmc";
 		power-source = <1800>;
diff --git a/arch/arm64/boot/dts/renesas/ebisu.dtsi b/arch/arm64/boot/dts/renesas/ebisu.dtsi
index 4b3775afcb0178..842e1a57e22828 100644
--- a/arch/arm64/boot/dts/renesas/ebisu.dtsi
+++ b/arch/arm64/boot/dts/renesas/ebisu.dtsi
@@ -674,7 +674,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
@@ -686,7 +686,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi1_pins_uhs: sd1_uhs {
+	sdhi1_pins_uhs: sd1-uhs {
 		groups = "sdhi1_data4", "sdhi1_ctrl";
 		function = "sdhi1";
 		power-source = <1800>;
@@ -698,7 +698,7 @@
 		power-source = <1800>;
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clk_a", "audio_clk_b_a", "audio_clk_c_a",
 			 "audio_clkout_a", "audio_clkout1_a";
 		function = "audio_clk";
diff --git a/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi b/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi
index 274493720b14e6..da6133375727ca 100644
--- a/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi
+++ b/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi
@@ -595,12 +595,12 @@
 			function = "avb0";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb0_mdio";
 			drive-strength = <21>;
 		};
 
-		pins_mii {
+		pins-mii {
 			groups = "avb0_rgmii";
 			drive-strength = <21>;
 		};
@@ -696,7 +696,7 @@
 		function = "i2c3";
 	};
 
-	irq0_pins: irq0_pins {
+	irq0_pins: irq0 {
 		groups = "intc_ex_irq0_a";
 		function = "intc_ex";
 	};
@@ -727,7 +727,7 @@
 		function = "scif_clk2";
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clkin", "audio_clkout";
 		function = "audio_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/hihope-common.dtsi b/arch/arm64/boot/dts/renesas/hihope-common.dtsi
index 4e78139d52f6c8..1999f97e0e606f 100644
--- a/arch/arm64/boot/dts/renesas/hihope-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/hihope-common.dtsi
@@ -229,7 +229,7 @@
 		function = "scif2";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_a";
 		function = "scif_clk";
 	};
@@ -240,7 +240,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
diff --git a/arch/arm64/boot/dts/renesas/hihope-rev2.dtsi b/arch/arm64/boot/dts/renesas/hihope-rev2.dtsi
index 25c55b32aafe5a..99ade271844a29 100644
--- a/arch/arm64/boot/dts/renesas/hihope-rev2.dtsi
+++ b/arch/arm64/boot/dts/renesas/hihope-rev2.dtsi
@@ -13,14 +13,14 @@
 	leds {
 		compatible = "gpio-leds";
 
-		bt_active_led {
+		bt-active-led {
 			label = "blue:bt";
 			gpios = <&gpio7  0 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "hci0-power";
 			default-state = "off";
 		};
 
-		wlan_active_led {
+		wlan-active-led {
 			label = "yellow:wlan";
 			gpios = <&gpio7  1 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "phy0tx";
@@ -28,7 +28,7 @@
 		};
 	};
 
-	wlan_en_reg: regulator-wlan_en {
+	wlan_en_reg: regulator-wlan-en {
 		compatible = "regulator-fixed";
 		regulator-name = "wlan-en-regulator";
 		regulator-min-microvolt = <1800000>;
@@ -57,7 +57,7 @@
 };
 
 &pfc {
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clk_a_a";
 		function = "audio_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/hihope-rev4.dtsi b/arch/arm64/boot/dts/renesas/hihope-rev4.dtsi
index acce3c0452f4a4..a26a985ec48b63 100644
--- a/arch/arm64/boot/dts/renesas/hihope-rev4.dtsi
+++ b/arch/arm64/boot/dts/renesas/hihope-rev4.dtsi
@@ -20,7 +20,7 @@
 		clock-frequency = <12288000>;
 	};
 
-	wlan_en_reg: regulator-wlan_en {
+	wlan_en_reg: regulator-wlan-en {
 		compatible = "regulator-fixed";
 		regulator-name = "wlan-en-regulator";
 		regulator-min-microvolt = <1800000>;
@@ -68,7 +68,7 @@
 		function = "i2c2";
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clk_a_a", "audio_clk_b_a", "audio_clkout_a";
 		function = "audio_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi b/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi
index 83b6c04274ac93..0878526506561c 100644
--- a/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi
+++ b/arch/arm64/boot/dts/renesas/hihope-rzg2-ex.dtsi
@@ -28,6 +28,8 @@
 		reg = <0>;
 		interrupts-extended = <&gpio2 11 IRQ_TYPE_LEVEL_LOW>;
 		reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <15000>;
+		reset-deassert-us = <35000>;
 	};
 };
 
@@ -57,12 +59,12 @@
 			function = "avb";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb_mdio";
 			drive-strength = <24>;
 		};
 
-		pins_mii_tx {
+		pins-mii-tx {
 			pins = "PIN_AVB_TX_CTL", "PIN_AVB_TXC", "PIN_AVB_TD0",
 			       "PIN_AVB_TD1", "PIN_AVB_TD2", "PIN_AVB_TD3";
 			drive-strength = <12>;
diff --git a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
index e66d86db6e6c44..5d500505b3ff1f 100644
--- a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
@@ -21,19 +21,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -228,13 +228,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
@@ -243,7 +243,7 @@
 		interrupt-affinity = <&a53_0>, <&a53_1>, <&a53_2>, <&a53_3>;
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts= <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			    <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
@@ -2877,7 +2877,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m-ex.dts b/arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m-ex.dts
new file mode 100644
index 00000000000000..d346a4009ec81a
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m-ex.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/*
+ * Device Tree Source for the HiHope RZ/G2M v3.0 SoC sub board
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+#include "r8a774a3-hihope-rzg2m.dts"
+
+/ {
+	model = "HopeRun HiHope RZ/G2M v3.0 with sub board";
+	compatible = "hoperun,hihope-rzg2-ex", "hoperun,hihope-rzg2m",
+		     "renesas,r8a774a3";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m.dts b/arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m.dts
new file mode 100644
index 00000000000000..4a4a09bad404ef
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a774a3-hihope-rzg2m.dts
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/*
+ * Device Tree Source for the HiHope RZ/G2M v3.0 SoC main board
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+/dts-v1/;
+#include "r8a774a3.dtsi"
+
+/ {
+	model = "HopeRun HiHope RZ/G2M v3.0 main board based on r8a774a3";
+	compatible = "hoperun,hihope-rzg2m", "renesas,r8a774a3";
+
+	memory@48000000 {
+		device_type = "memory";
+		/* First 128MiB is reserved for secure area */
+		reg = <0x0 0x48000000 0x0 0x78000000>;
+	};
+
+	memory@600000000 {
+		device_type = "memory";
+		reg = <0x6 0x00000000 0x0 0x80000000>;
+	};
+
+	aliases {
+		serial0 = &scif2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+};
+
+&extal_clk {
+	clock-frequency = <16666666>;
+};
+
+&extalr_clk {
+	clock-frequency = <32768>;
+};
+
+&scif2 {
+	status = "okay";
+};
+
+&scif_clk {
+	clock-frequency = <14745600>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a774a3.dtsi b/arch/arm64/boot/dts/renesas/r8a774a3.dtsi
new file mode 100644
index 00000000000000..4a3af85974ffcc
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a774a3.dtsi
@@ -0,0 +1,302 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/*
+ * Device Tree Source for the r8a774a3 SoC
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+#include <dt-bindings/clock/renesas,r8a774a3-cpg-mssr.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/power/renesas,r8a774a3-sysc.h>
+
+/ {
+	compatible = "renesas,r8a774a3";
+	#address-cells = <2>;
+	#size-cells = <2>;
+	interrupt-parent = <&gic>;
+
+	cluster0_opp: opp-table-0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1500000000 {
+			opp-hz = /bits/ 64 <1500000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+			opp-suspend;
+		};
+	};
+
+	cluster1_opp: opp-table-1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <820000>;
+			clock-latency-ns = <300000>;
+		};
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&a57_0>;
+				};
+				core1 {
+					cpu = <&a57_1>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&a53_0>;
+				};
+				core1 {
+					cpu = <&a53_1>;
+				};
+				core2 {
+					cpu = <&a53_2>;
+				};
+				core3 {
+					cpu = <&a53_3>;
+				};
+			};
+		};
+
+		a57_0: cpu@0 {
+			compatible = "arm,cortex-a57";
+			reg = <0x0>;
+			device_type = "cpu";
+			power-domains = <&sysc R8A774A3_PD_CA57_CPU0>;
+			next-level-cache = <&L2_CA57>;
+			enable-method = "psci";
+			dynamic-power-coefficient = <854>;
+			clocks = <&cpg CPG_CORE R8A774A3_CLK_Z>;
+			operating-points-v2 = <&cluster0_opp>;
+			capacity-dmips-mhz = <1024>;
+		};
+
+		a57_1: cpu@1 {
+			compatible = "arm,cortex-a57";
+			reg = <0x1>;
+			device_type = "cpu";
+			power-domains = <&sysc R8A774A3_PD_CA57_CPU1>;
+			next-level-cache = <&L2_CA57>;
+			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R8A774A3_CLK_Z>;
+			operating-points-v2 = <&cluster0_opp>;
+			capacity-dmips-mhz = <1024>;
+		};
+
+		a53_0: cpu@100 {
+			compatible = "arm,cortex-a53";
+			reg = <0x100>;
+			device_type = "cpu";
+			power-domains = <&sysc R8A774A3_PD_CA53_CPU0>;
+			next-level-cache = <&L2_CA53>;
+			enable-method = "psci";
+			dynamic-power-coefficient = <277>;
+			clocks = <&cpg CPG_CORE R8A774A3_CLK_Z2>;
+			operating-points-v2 = <&cluster1_opp>;
+			capacity-dmips-mhz = <560>;
+		};
+
+		a53_1: cpu@101 {
+			compatible = "arm,cortex-a53";
+			reg = <0x101>;
+			device_type = "cpu";
+			power-domains = <&sysc R8A774A3_PD_CA53_CPU1>;
+			next-level-cache = <&L2_CA53>;
+			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R8A774A3_CLK_Z2>;
+			operating-points-v2 = <&cluster1_opp>;
+			capacity-dmips-mhz = <560>;
+		};
+
+		a53_2: cpu@102 {
+			compatible = "arm,cortex-a53";
+			reg = <0x102>;
+			device_type = "cpu";
+			power-domains = <&sysc R8A774A3_PD_CA53_CPU2>;
+			next-level-cache = <&L2_CA53>;
+			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R8A774A3_CLK_Z2>;
+			operating-points-v2 = <&cluster1_opp>;
+			capacity-dmips-mhz = <560>;
+		};
+
+		a53_3: cpu@103 {
+			compatible = "arm,cortex-a53";
+			reg = <0x103>;
+			device_type = "cpu";
+			power-domains = <&sysc R8A774A3_PD_CA53_CPU3>;
+			next-level-cache = <&L2_CA53>;
+			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R8A774A3_CLK_Z2>;
+			operating-points-v2 = <&cluster1_opp>;
+			capacity-dmips-mhz = <560>;
+		};
+
+		L2_CA57: cache-controller-0 {
+			compatible = "cache";
+			power-domains = <&sysc R8A774A3_PD_CA57_SCU>;
+			cache-unified;
+			cache-level = <2>;
+		};
+
+		L2_CA53: cache-controller-1 {
+			compatible = "cache";
+			power-domains = <&sysc R8A774A3_PD_CA53_SCU>;
+			cache-unified;
+			cache-level = <2>;
+		};
+	};
+
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+		bootph-all;
+	};
+
+	extalr_clk: extalr {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board */
+		clock-frequency = <0>;
+		bootph-all;
+	};
+
+	pmu-a53 {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a53_0>, <&a53_1>, <&a53_2>, <&a53_3>;
+	};
+
+	pmu-a57 {
+		compatible = "arm,cortex-a57-pmu";
+		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+			    <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a57_0>, <&a57_1>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
+	};
+
+	/* External SCIF clock - to be overridden by boards that provide it */
+	scif_clk: scif {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	soc: soc {
+		compatible = "simple-bus";
+		bootph-all;
+
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		cpg: clock-controller@e6150000 {
+			compatible = "renesas,r8a774a3-cpg-mssr";
+			reg = <0 0xe6150000 0 0x1000>;
+			clocks = <&extal_clk>, <&extalr_clk>;
+			clock-names = "extal", "extalr";
+			#clock-cells = <2>;
+			#power-domain-cells = <0>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		rst: reset-controller@e6160000 {
+			compatible = "renesas,r8a774a3-rst";
+			reg = <0 0xe6160000 0 0x200>;
+			bootph-all;
+		};
+
+		sysc: system-controller@e6180000 {
+			compatible = "renesas,r8a774a3-sysc";
+			reg = <0 0xe6180000 0 0x400>;
+			#power-domain-cells = <1>;
+		};
+
+		scif2: serial@e6e88000 {
+			compatible = "renesas,scif-r8a774a3",
+				     "renesas,rcar-gen3-scif", "renesas,scif";
+			reg = <0 0xe6e88000 0 0x40>;
+			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 310>,
+				 <&cpg CPG_CORE R8A774A3_CLK_S3D1>,
+				 <&scif_clk>;
+			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&sysc R8A774A3_PD_ALWAYS_ON>;
+			resets = <&cpg 310>;
+			status = "disabled";
+		};
+
+		gic: interrupt-controller@f1010000 {
+			compatible = "arm,gic-400";
+			#interrupt-cells = <3>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0x0 0xf1010000 0 0x1000>,
+			      <0x0 0xf1020000 0 0x20000>,
+			      <0x0 0xf1040000 0 0x20000>,
+			      <0x0 0xf1060000 0 0x20000>;
+			interrupts = <GIC_PPI 9
+					(GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_HIGH)>;
+			clocks = <&cpg CPG_MOD 408>;
+			clock-names = "clk";
+			power-domains = <&sysc R8A774A3_PD_ALWAYS_ON>;
+			resets = <&cpg 408>;
+		};
+
+		prr: chipid@fff00044 {
+			compatible = "renesas,prr";
+			reg = <0 0xfff00044 0 4>;
+			bootph-all;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>;
+		interrupt-names = "sec-phys", "phys", "virt", "hyp-phys";
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
index 62c6703917db4e..e0d593c49a86a8 100644
--- a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
@@ -21,19 +21,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -121,13 +121,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
@@ -2748,7 +2748,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts b/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts
index 57a281fc49775d..4be3f82a0460b5 100644
--- a/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts
+++ b/arch/arm64/boot/dts/renesas/r8a774c0-cat874.dts
@@ -322,7 +322,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
@@ -334,7 +334,7 @@
 		power-source = <1800>;
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clkout1_a";
 		function = "audio_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi
index 3858f4328e9619..37313f525a9323 100644
--- a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi
@@ -20,19 +20,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -112,13 +112,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts= <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			    <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
@@ -2014,7 +2014,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
index 0ae9bb72d2dda5..335b94d3fec7db 100644
--- a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
@@ -21,19 +21,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -290,13 +290,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
@@ -305,7 +305,7 @@
 		interrupt-affinity = <&a53_0>, <&a53_1>, <&a53_2>, <&a53_3>;
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
@@ -3011,7 +3011,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77951.dtsi b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
index 59a0f2e1479d08..a08b1609bbe685 100644
--- a/arch/arm64/boot/dts/renesas/r8a77951.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
@@ -25,19 +25,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -305,13 +305,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
@@ -323,7 +323,7 @@
 				     <&a53_3>;
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
@@ -3520,7 +3520,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77960.dtsi b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
index 4f9989b5e77a89..cdbb7f9556fb69 100644
--- a/arch/arm64/boot/dts/renesas/r8a77960.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
@@ -20,19 +20,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -277,13 +277,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
@@ -292,7 +292,7 @@
 		interrupt-affinity = <&a53_0>, <&a53_1>, <&a53_2>, <&a53_3>;
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
@@ -3135,7 +3135,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77961.dtsi b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
index ad4491ba948f2f..38f34c5c9f65ea 100644
--- a/arch/arm64/boot/dts/renesas/r8a77961.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77961.dtsi
@@ -20,19 +20,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -277,13 +277,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
@@ -292,7 +292,7 @@
 		interrupt-affinity = <&a53_0>, <&a53_1>, <&a53_2>, <&a53_3>;
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
@@ -2956,7 +2956,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
index 70708f5cf74671..52376c6de208ed 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
@@ -25,19 +25,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -156,13 +156,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a57 {
+	pmu-a57 {
 		compatible = "arm,cortex-a57-pmu";
 		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
@@ -2964,7 +2964,7 @@
 		clock-frequency = <0>;
 	};
 
-	usb_extal_clk: usb_extal {
+	usb_extal_clk: usb-extal {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle-function-expansion.dtso b/arch/arm64/boot/dts/renesas/r8a77970-eagle-function-expansion.dtso
index ecb35257b9ae18..a9ad009ff7b04b 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-eagle-function-expansion.dtso
+++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle-function-expansion.dtso
@@ -112,7 +112,7 @@
 		reg = <0x70 0x71 0x72 0x73 0x74 0x75
 		       0x60 0x61 0x62 0x63 0x64 0x65>;
 		reg-names = "main", "dpll", "cp", "hdmi", "edid", "repeater",
-			    "infoframe", "cbus", "cec", "sdp", "txa", "txb" ;
+			    "infoframe", "cbus", "cec", "sdp", "txa", "txb";
 		interrupts-extended = <&gpio3 3 IRQ_TYPE_LEVEL_LOW>,
 				      <&gpio3 4 IRQ_TYPE_LEVEL_LOW>;
 		interrupt-names = "intrq1", "intrq2";
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
index a2ad79ddf73db1..be0ee054fc5f7a 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
@@ -335,7 +335,7 @@
 		function = "scif0";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_b";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
index 10c9a2e9ed18d3..f184fa93d2a43b 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
@@ -215,7 +215,7 @@
 		function = "i2c0";
 	};
 
-	mmc_pins: mmc_3_3v {
+	mmc_pins: mmc-3-3v {
 		groups = "mmc_data8", "mmc_ctrl";
 		function = "mmc";
 		power-source = <3300>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77970.dtsi b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
index f7f1f280fa0b63..3eb519b61fca7b 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
@@ -72,7 +72,7 @@
 		bootph-all;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
index 52462e61b7194c..32b5ea7d1dc563 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
@@ -207,7 +207,7 @@
 		function = "scif0";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_b";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
index 514dafe3447869..1df6d3db046cc9 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
@@ -93,13 +93,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/renesas/r8a77990.dtsi b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
index fadb5f4effcf0a..0e9b837a6e5bd2 100644
--- a/arch/arm64/boot/dts/renesas/r8a77990.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
@@ -20,19 +20,19 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_c: audio_clk_c {
+	audio_clk_c: audio-clk-c {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -127,13 +127,13 @@
 	};
 
 	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
+	pcie_bus_clk: pcie-bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77995.dtsi b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
index 522a49db025877..a20fff95407b6a 100644
--- a/arch/arm64/boot/dts/renesas/r8a77995.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
@@ -21,13 +21,13 @@
 	 * clocks by default.
 	 * Boards that provide audio clocks should override them.
 	 */
-	audio_clk_a: audio_clk_a {
+	audio_clk_a: audio-clk-a {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
+	audio_clk_b: audio-clk-b {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -69,7 +69,7 @@
 		bootph-all;
 	};
 
-	pmu_a53 {
+	pmu-a53 {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi
index 0916fd57d1f1a0..2675f48d8fd025 100644
--- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi
@@ -306,7 +306,7 @@
 		function = "scif0";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts b/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts
index ea5dcee73658ad..5708e395a226df 100644
--- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon.dts
@@ -73,12 +73,12 @@
 			function = "avb0";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb0_mdio";
 			drive-strength = <21>;
 		};
 
-		pins_mii {
+		pins-mii {
 			groups = "avb0_rgmii";
 			drive-strength = <21>;
 		};
diff --git a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
index 0483a5d0714af7..38f35c81981bc4 100644
--- a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
@@ -59,7 +59,7 @@
 		bootph-all;
 	};
 
-	pmu_a76 {
+	pmu-a76 {
 		compatible = "arm,cortex-a76-pmu";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi
index 1781bb79a6196f..0cfbff532f42d7 100644
--- a/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi
@@ -206,7 +206,7 @@
 		function = "scif0";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi
index a5118c2f27429b..17a632770e6531 100644
--- a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi
@@ -279,7 +279,7 @@
 		clock-frequency = <0>;
 	};
 
-	pmu_a55 {
+	pmu-a55 {
 		compatible = "arm,cortex-a55-pmu";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
 	};
@@ -790,9 +790,11 @@
 			clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
-			resets = <&cpg 624>;
-			reset-names = "pwr";
+			resets = <&cpg 624>, <&cpg 1121>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
+			msi-parent = <&its 0>;
+			msi-map = <0x0 &its 0x0 0x10000>;
 			num-lanes = <2>;
 			#address-cells = <3>;
 			#size-cells = <2>;
@@ -803,10 +805,10 @@
 			dma-ranges = <0x42000000 0 0x00000000 0 0x00000000 1 0x00000000>;
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 7>;
-			interrupt-map = <0 0 0 1 &gic GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &gic GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &gic GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &gic GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic 0 0 GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic 0 0 GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic 0 0 GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>;
 			snps,enable-cdm-check;
 			status = "disabled";
 		};
@@ -827,9 +829,11 @@
 			clocks = <&cpg CPG_MOD 625>, <&pcie1_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
-			resets = <&cpg 625>;
-			reset-names = "pwr";
+			resets = <&cpg 625>, <&cpg 1122>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
+			msi-parent = <&its 0>;
+			msi-map = <0x0 &its 0x0 0x10000>;
 			num-lanes = <2>;
 			#address-cells = <3>;
 			#size-cells = <2>;
@@ -840,10 +844,10 @@
 			dma-ranges = <0x42000000 0 0x00000000 0 0x00000000 1 0x00000000>;
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 7>;
-			interrupt-map = <0 0 0 1 &gic GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &gic GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &gic GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &gic GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic 0 0 GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic 0 0 GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic 0 0 GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH>;
 			snps,enable-cdm-check;
 			status = "disabled";
 		};
@@ -863,8 +867,8 @@
 			clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
-			resets = <&cpg 624>;
-			reset-names = "pwr";
+			resets = <&cpg 624>, <&cpg 1121>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
 			num-lanes = <2>;
 			max-functions = /bits/ 8 <2>;
@@ -886,8 +890,8 @@
 			clocks = <&cpg CPG_MOD 625>, <&pcie1_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
-			resets = <&cpg 625>;
-			reset-names = "pwr";
+			resets = <&cpg 625>, <&cpg 1122>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
 			num-lanes = <2>;
 			max-functions = /bits/ 8 <2>;
@@ -1291,11 +1295,23 @@
 		gic: interrupt-controller@f1000000 {
 			compatible = "arm,gic-v3";
 			#interrupt-cells = <3>;
-			#address-cells = <0>;
+			#address-cells = <2>;
+			#size-cells = <2>;
 			interrupt-controller;
 			reg = <0x0 0xf1000000 0 0x20000>,
 			      <0x0 0xf1060000 0 0x110000>;
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			dma-noncoherent;
+
+			ranges = <0x0 0x0 0x0 0xf1000000 0x0 0x200000>;
+
+			its: msi-controller@40000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x40000 0x0 0x20000>;
+				dma-noncoherent;
+				msi-controller;
+				#msi-cells = <1>;
+			};
 		};
 
 		wwdt0: watchdog@ffc90000 {
diff --git a/arch/arm64/boot/dts/renesas/r8a779f4-s4sk.dts b/arch/arm64/boot/dts/renesas/r8a779f4-s4sk.dts
index 67b18f2bffbd06..a0fd7f3e1ba99f 100644
--- a/arch/arm64/boot/dts/renesas/r8a779f4-s4sk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779f4-s4sk.dts
@@ -151,7 +151,7 @@
 		function = "i2c5";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts
index 784d4e8b204ce8..89d60b83ac4f72 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts
@@ -12,4 +12,98 @@
 / {
 	model = "Renesas White Hawk CPU and Breakout boards based on r8a779g0";
 	compatible = "renesas,white-hawk-breakout", "renesas,white-hawk-cpu", "renesas,r8a779g0";
+
+	sn65dsi86_refclk2: clk-x16 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <38400000>;
+	};
+
+	mini-dp-con2 {
+		compatible = "dp-connector";
+		label = "CN15";
+		type = "mini";
+
+		port {
+			mini_dp_con_in2: endpoint {
+				remote-endpoint = <&sn65dsi86_out2>;
+			};
+		};
+	};
+};
+
+&dsc {
+	status = "okay";
+};
+
+&dsi1 {
+	status = "okay";
+
+	ports {
+		port@1 {
+			dsi1_out: endpoint {
+				remote-endpoint = <&sn65dsi86_in2>;
+				data-lanes = <1 2 3 4>;
+			};
+		};
+	};
+};
+
+&i2c4 {
+	pinctrl-0 = <&i2c4_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	bridge@2c {
+		pinctrl-0 = <&irq1_pins>;
+		pinctrl-names = "default";
+
+		compatible = "ti,sn65dsi86";
+		reg = <0x2c>;
+
+		clocks = <&sn65dsi86_refclk2>;
+		clock-names = "refclk";
+
+		interrupts-extended = <&intc_ex 1 IRQ_TYPE_LEVEL_HIGH>;
+
+		enable-gpios = <&gpio1 27 GPIO_ACTIVE_HIGH>;
+
+		vccio-supply = <&reg_1p8v>;
+		vpll-supply = <&reg_1p8v>;
+		vcca-supply = <&reg_1p2v>;
+		vcc-supply = <&reg_1p2v>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				sn65dsi86_in2: endpoint {
+					remote-endpoint = <&dsi1_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				sn65dsi86_out2: endpoint {
+					remote-endpoint = <&mini_dp_con_in2>;
+				};
+			};
+		};
+	};
+};
+
+&pfc {
+	i2c4_pins: i2c4 {
+		groups = "i2c4";
+		function = "i2c4";
+	};
+
+	irq1_pins: irq1 {
+		groups = "intc_ex_irq1_a";
+		function = "intc_ex";
+	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
index 8a291447b90450..5284923b6cef79 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
@@ -16,7 +16,7 @@
 	interrupt-parent = <&gic>;
 
 	/* External Audio clock - to be overridden by boards that provide it */
-	audio_clkin: audio_clkin {
+	audio_clkin: audio-clkin {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -192,7 +192,7 @@
 		clock-frequency = <0>;
 	};
 
-	pmu_a76 {
+	pmu-a76 {
 		compatible = "arm,cortex-a76-pmu";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW>;
 	};
@@ -806,9 +806,11 @@
 			clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
-			resets = <&cpg 624>;
-			reset-names = "pwr";
+			resets = <&cpg 624>, <&cpg 1121>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
+			msi-parent = <&its 0>;
+			msi-map = <0x0 &its 0x0 0x10000>;
 			num-lanes = <2>;
 			#address-cells = <3>;
 			#size-cells = <2>;
@@ -819,10 +821,10 @@
 			dma-ranges = <0x42000000 0 0x00000000 0 0x00000000 1 0x00000000>;
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 7>;
-			interrupt-map = <0 0 0 1 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>;
 			snps,enable-cdm-check;
 			status = "disabled";
 
@@ -853,9 +855,11 @@
 			clocks = <&cpg CPG_MOD 625>, <&pcie1_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
-			resets = <&cpg 625>;
-			reset-names = "pwr";
+			resets = <&cpg 625>, <&cpg 1122>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
+			msi-parent = <&its 0>;
+			msi-map = <0x0 &its 0x0 0x10000>;
 			num-lanes = <2>;
 			#address-cells = <3>;
 			#size-cells = <2>;
@@ -866,10 +870,10 @@
 			dma-ranges = <0x42000000 0 0x00000000 0 0x00000000 1 0x00000000>;
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 7>;
-			interrupt-map = <0 0 0 1 &gic GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &gic GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &gic GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &gic GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic 0 0 GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic 0 0 GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic 0 0 GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>;
 			snps,enable-cdm-check;
 			status = "disabled";
 
@@ -899,8 +903,8 @@
 			clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
-			resets = <&cpg 624>;
-			reset-names = "pwr";
+			resets = <&cpg 624>, <&cpg 1121>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
 			num-lanes = <2>;
 			max-functions = /bits/ 8 <2>;
@@ -922,8 +926,8 @@
 			clocks = <&cpg CPG_MOD 625>, <&pcie1_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
-			resets = <&cpg 625>;
-			reset-names = "pwr";
+			resets = <&cpg 625>, <&cpg 1122>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
 			num-lanes = <2>;
 			max-functions = /bits/ 8 <2>;
@@ -2148,11 +2152,23 @@
 		gic: interrupt-controller@f1000000 {
 			compatible = "arm,gic-v3";
 			#interrupt-cells = <3>;
-			#address-cells = <0>;
+			#address-cells = <2>;
+			#size-cells = <2>;
 			interrupt-controller;
 			reg = <0x0 0xf1000000 0 0x20000>,
 			      <0x0 0xf1060000 0 0x110000>;
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			dma-noncoherent;
+
+			ranges = <0x0 0x0 0x0 0xf1000000 0x0 0x200000>;
+
+			its: msi-controller@40000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x40000 0x0 0x20000>;
+				dma-noncoherent;
+				msi-controller;
+				#msi-cells = <1>;
+			};
 		};
 
 		csi40: csi2@fe500000 {
@@ -2315,6 +2331,35 @@
 				port@1 {
 					reg = <1>;
 					du_out_dsi1: endpoint {
+						remote-endpoint = <&dsc_in_dsi1>;
+					};
+				};
+			};
+		};
+
+		dsc: bridge@feb8d000 {
+			compatible = "renesas,r8a779g0-dsc";
+			reg = <0 0xfeb8d000 0 0x400>;
+			interrupts = <GIC_SPI 559 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 2819>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 2819>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					dsc_in_dsi1: endpoint {
+						remote-endpoint = <&du_out_dsi1>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					dsc_out_dsi1: endpoint {
 						remote-endpoint = <&dsi1_in>;
 					};
 				};
@@ -2551,7 +2596,7 @@
 				port@0 {
 					reg = <0>;
 					dsi1_in: endpoint {
-						remote-endpoint = <&du_out_dsi1>;
+						remote-endpoint = <&dsc_out_dsi1>;
 					};
 				};
 
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx219.dtso b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx219.dtso
index 3acaf714cf248a..b816382bba0aca 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx219.dtso
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx219.dtso
@@ -12,6 +12,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 
 &{/} {
 	clk_cam_j1: clk-cam-j1 {
@@ -44,7 +45,7 @@
 		VDIG-supply = <&reg_cam_j1>;
 		VDDL-supply = <&reg_cam_j1>;
 
-		orientation = <2>;
+		orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 		rotation = <0>;
 
 		port {
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx462.dtso b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx462.dtso
index a19bc084039228..4019b80a88b757 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx462.dtso
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j1-imx462.dtso
@@ -12,6 +12,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 
 &{/} {
 	clk_cam_j1: clk-cam-j1 {
@@ -46,7 +47,7 @@
 		vdda-supply = <&reg_cam_j1>;
 		vddd-supply = <&reg_cam_j1>;
 
-		orientation = <2>;
+		orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 		rotation = <0>;
 
 		port {
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx219.dtso b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx219.dtso
index 512810b861aac1..fea1ef4a1178e5 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx219.dtso
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx219.dtso
@@ -12,6 +12,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 
 &{/} {
 	clk_cam_j2: clk-cam-j2 {
@@ -44,7 +45,7 @@
 		VDIG-supply = <&reg_cam_j2>;
 		VDDL-supply = <&reg_cam_j2>;
 
-		orientation = <2>;
+		orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 		rotation = <0>;
 
 		port {
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx462.dtso b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx462.dtso
index a31524b598348d..177201a8a6d20c 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx462.dtso
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-camera-j2-imx462.dtso
@@ -12,6 +12,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 
 &{/} {
 	clk_cam_j2: clk-cam-j2 {
@@ -46,7 +47,7 @@
 		vdda-supply = <&reg_cam_j2>;
 		vddd-supply = <&reg_cam_j2>;
 
-		orientation = <2>;
+		orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 		rotation = <0>;
 
 		port {
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-fan-argon40.dtso b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-fan-argon40.dtso
index c730ef39c7d7d1..6f10310140b9b2 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-fan-argon40.dtso
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-fan-argon40.dtso
@@ -41,7 +41,6 @@
 	#address-cells = <1>;
 	#size-cells = <0>;
 	clock-frequency = <400000>;
-	status = "okay";
 
 	pwmhat: pwm@1a {
 		compatible = "argon40,fan-hat";
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-ws-2ch-canfd.dtso b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-ws-2ch-canfd.dtso
new file mode 100644
index 00000000000000..94405a3e80b8dd
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk-ws-2ch-canfd.dtso
@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Overlay for the Waveshare 2-Channel Isolated CAN FD HAT
+ * on connector CN7 on R-Car V4H ES3.0 Sparrow Hawk board
+ *
+ * Copyright (C) 2026 Marek Vasut <marek.vasut+renesas@mailbox.org>
+ *
+ * Example usage:
+ * - Connect HAT signals to Sparrow Hawk board CAN BUS connector CONN2:
+ *   HAT CAN_0 H -- CONN2 pin 6
+ *   HAT CAN_0 L -- CONN2 pin 2
+ *   HAT CAN_1 H -- CONN2 pin 5
+ *   HAT CAN_1 L -- CONN2 pin 1
+ *
+ * - Determine which interface is which:
+ *   $ ip -d link show
+ *   2: can0: <NOARP,ECHO> ...
+ *      ...
+ *        clock 40000000 ... e6660000.can <------ Native Renesas CANFD
+ *   3: can1: <NOARP,ECHO> ...
+ *      ...
+ *   4: can2: <NOARP,ECHO> ...
+ *      ...
+ *        clock 40000000 ... spi1.0 <------------ SPI CANFD on this HAT
+ *   5: can3: <NOARP,ECHO> ...
+ *      ...
+ *
+ * - Bring connected interfaces up:
+ *   # CAN_0 link
+ *   $ ip link set can0 up type can bitrate 125000 dbitrate 250000 fd on
+ *   $ ip link set can3 up type can bitrate 125000 dbitrate 250000 fd on
+ *
+ *   # CAN_1 link
+ *   $ ip link set can1 up type can bitrate 125000 dbitrate 250000 fd on
+ *   $ ip link set can2 up type can bitrate 125000 dbitrate 250000 fd on
+ *
+ * - Perform CAN test, run each of the pair commands in separate terminals:
+ *   # CAN_0 link
+ *   $ canfdtest -v can0
+ *   $ canfdtest -g -v can3
+ *
+ *   # CAN_1 link
+ *   $ canfdtest -v can1
+ *   $ canfdtest -g -v can2
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/*
+ * PWM1 uses PWM1_B pin configuration for pin GP2_13. This conflicts
+ * with SPI_1 CE_1 signal which is also on pin GP2_13. Disable PWM1
+ * in case this hat is in use.
+ *
+ * In case of a dynamic overlay, pwm1 must be disabled before SPI_1
+ * is probed. Otherwise the latter will fail with -EBUSY due to the
+ * conflicting pin being still in use. Hence place pwm1 node here.
+ */
+&pwm1 {
+	status = "disabled";
+};
+
+&{/} {
+	/* Fixed clock dedicated to SPI CAN on 2-CH CAN FD HAT */
+	clk_can0: clk-ws-2ch-canfd0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <40000000>;
+	};
+
+	clk_can1: clk-ws-2ch-canfd1 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <40000000>;
+	};
+
+	/*
+	 * SPI_1 is operated as GPIO SPI, because MSIOF5 can not be
+	 * muxed on the pins allocated for SPI_1 on this CANFD HAT.
+	 */
+	spi-ws-2ch-canfd {	/* SPI_1 */
+		compatible = "spi-gpio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		sck-gpios = <&gpio0 10 0>;		/* CN7 pin 40 */
+		miso-gpios = <&gpio0 9 0>;		/* CN7 pin 35 */
+		mosi-gpios = <&gpio0 12 0>;		/* CN7 pin 38 */
+		cs-gpios = <&gpio2 13 GPIO_ACTIVE_LOW>;	/* CN7 pin 12 */
+		num-chipselects = <1>;
+
+		can@0 {
+			compatible = "microchip,mcp2518fd";
+			reg = <0>;
+			clocks = <&clk_can1>;
+			/* CN7 pin 18 */
+			interrupts-extended = <&gpio1 25 IRQ_TYPE_LEVEL_LOW>;
+			spi-max-frequency = <20000000>;
+		};
+	};
+};
+
+&msiof0 {	/* SPI_0 */
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-0 = <&msiof0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	/*
+	 * Native MSIOF chipselect does not work with MCP2518FD
+	 * because MCP251xFD driver needs to keep the chipselect
+	 * line asserted between transfers. This requirement is
+	 * fulfilled only by using GPIO chipselect.
+	 */
+	cs-gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
+
+	can@0 {
+		compatible = "microchip,mcp2518fd";
+		reg = <0>;
+		clocks = <&clk_can0>;
+		/* CN7 pin 22 */
+		interrupts-extended = <&gpio1 26 IRQ_TYPE_LEVEL_LOW>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&pfc {
+	msiof0_pins: msiof0 {	/* SPI_0 */
+		groups = "msiof0_clk", "msiof0_txd", "msiof0_rxd";
+		function = "msiof0";
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts
index af680290ce8170..3d6df106ff6736 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779g3-sparrow-hawk.dts
@@ -288,6 +288,11 @@
 	};
 };
 
+/* Page 27 / DSI to Display */
+&dsc {
+	status = "okay";
+};
+
 /* Page 27 / DSI to Display */
 &dsi1 {
 	status = "okay";
@@ -498,6 +503,7 @@
 	#size-cells = <0>;
 	pinctrl-0 = <&i2c3_pins>;
 	pinctrl-names = "default";
+	status = "okay";
 };
 
 /* Page 31 / IO_CN */
@@ -506,6 +512,7 @@
 	#size-cells = <0>;
 	pinctrl-0 = <&i2c4_pins>;
 	pinctrl-names = "default";
+	status = "okay";
 };
 
 /* Page 18 / POWER_CORE and Page 19 / POWER_PMIC */
@@ -698,6 +705,12 @@
 		function = "intc_ex";
 	};
 
+	/* Page 30 / Audio_Codec */
+	msiof1_pins: msiof1 {
+		groups = "msiof1_clk", "msiof1_sync", "msiof1_txd", "msiof1_rxd";
+		function = "msiof1";
+	};
+
 	/* Page 31 / FAN */
 	pwm0_pins: pwm0 {
 		groups = "pwm0";
@@ -749,12 +762,6 @@
 		power-source = <1800>;
 	};
 
-	/* Page 30 / Audio_Codec */
-	msiof1_pins: sound {
-		groups = "msiof1_clk", "msiof1_sync", "msiof1_txd", "msiof1_rxd";
-		function = "msiof1";
-	};
-
 	/* Page 30 / Audio_Codec */
 	sound_clk_pins: sound-clk {
 		groups = "audio_clkin", "audio_clkout";
diff --git a/arch/arm64/boot/dts/renesas/r8a779h0.dtsi b/arch/arm64/boot/dts/renesas/r8a779h0.dtsi
index 74bc4c4854ecae..5f9223a0f44687 100644
--- a/arch/arm64/boot/dts/renesas/r8a779h0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779h0.dtsi
@@ -16,7 +16,7 @@
 	interrupt-parent = <&gic>;
 
 	/* External Audio clock - to be overridden by boards that provide it */
-	audio_clkin: audio_clkin {
+	audio_clkin: audio-clkin {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
@@ -688,9 +688,11 @@
 			clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779H0_PD_A2PCIPHY>;
-			resets = <&cpg 624>;
-			reset-names = "pwr";
+			resets = <&cpg 624>, <&cpg 1121>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
+			msi-parent = <&its 0>;
+			msi-map = <0x0 &its 0x0 0x10000>;
 			num-lanes = <2>;
 			#address-cells = <3>;
 			#size-cells = <2>;
@@ -701,10 +703,10 @@
 			dma-ranges = <0x42000000 0 0x00000000 0 0x00000000 1 0x00000000>;
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 7>;
-			interrupt-map = <0 0 0 1 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &gic GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic 0 0 GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>;
 			snps,enable-cdm-check;
 			status = "disabled";
 		};
@@ -724,8 +726,8 @@
 			clocks = <&cpg CPG_MOD 624>, <&pcie0_clkref>;
 			clock-names = "core", "ref";
 			power-domains = <&sysc R8A779H0_PD_A2PCIPHY>;
-			resets = <&cpg 624>;
-			reset-names = "pwr";
+			resets = <&cpg 624>, <&cpg 1121>;
+			reset-names = "pwr", "core";
 			max-link-speed = <4>;
 			num-lanes = <2>;
 			max-functions = /bits/ 8 <2>;
@@ -1852,11 +1854,23 @@
 		gic: interrupt-controller@f1000000 {
 			compatible = "arm,gic-v3";
 			#interrupt-cells = <3>;
-			#address-cells = <0>;
+			#address-cells = <2>;
+			#size-cells = <2>;
 			interrupt-controller;
 			reg = <0x0 0xf1000000 0 0x20000>,
 			      <0x0 0xf1060000 0 0x110000>;
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+			dma-noncoherent;
+
+			ranges = <0x0 0x0 0x0 0xf1000000 0x0 0x200000>;
+
+			its: msi-controller@40000 {
+				compatible = "arm,gic-v3-its";
+				reg = <0x0 0x40000 0x0 0x20000>;
+				dma-noncoherent;
+				msi-controller;
+				#msi-cells = <1>;
+			};
 		};
 
 		csi40: csi2@fe500000 {
diff --git a/arch/arm64/boot/dts/renesas/r8a779md-geist.dts b/arch/arm64/boot/dts/renesas/r8a779md-geist.dts
index 0e4724336e7346..252d9e9d3940db 100644
--- a/arch/arm64/boot/dts/renesas/r8a779md-geist.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779md-geist.dts
@@ -367,7 +367,7 @@
 		reg = <0x70 0x71 0x72 0x73 0x74 0x75
 		       0x60 0x61 0x62 0x63 0x64 0x65>;
 		reg-names = "main", "dpll", "cp", "hdmi", "edid", "repeater",
-			    "infoframe", "cbus", "cec", "sdp", "txa", "txb" ;
+			    "infoframe", "cbus", "cec", "sdp", "txa", "txb";
 
 		interrupts-extended = <&gpio6 30 IRQ_TYPE_LEVEL_LOW>,
 				      <&gpio6 31 IRQ_TYPE_LEVEL_LOW>;
@@ -460,12 +460,12 @@
 			function = "avb";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb_mdio";
 			drive-strength = <24>;
 		};
 
-		pins_mii_tx {
+		pins-mii-tx {
 			pins = "PIN_AVB_TX_CTL", "PIN_AVB_TXC", "PIN_AVB_TD0",
 			       "PIN_AVB_TD1", "PIN_AVB_TD2", "PIN_AVB_TD3";
 			drive-strength = <12>;
@@ -507,7 +507,7 @@
 		function = "scif2";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_a";
 		function = "scif_clk";
 	};
@@ -518,7 +518,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
@@ -535,7 +535,7 @@
 		function = "ssi";
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clk_a_a", "audio_clk_b_a", "audio_clk_c_a",
 			 "audio_clkout_a", "audio_clkout3_a";
 		function = "audio_clk";
diff --git a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts
index 0ab303863155e1..b2d6b06c638818 100644
--- a/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts
+++ b/arch/arm64/boot/dts/renesas/r8a78000-ironhide.dts
@@ -134,18 +134,18 @@
 			no-map;
 		};
 
-		ecc@1a66660000 {
-			reg = <0x1a 0x66660000 0x0 0x999a0000>;
+		ecc@1acccc0000 {
+			reg = <0x1a 0xcccc0000 0x0 0x33340000>;
 			no-map;
 		};
 
-		ecc@1c66660000 {
-			reg = <0x1c 0x66660000 0x0 0x999a0000>;
+		ecc@1ccccc0000 {
+			reg = <0x1c 0xcccc0000 0x0 0x33340000>;
 			no-map;
 		};
 
-		ecc@1e66660000 {
-			reg = <0x1e 0x66660000 0x0 0x999a0000>;
+		ecc@1ecccc0000 {
+			reg = <0x1e 0xcccc0000 0x0 0x33340000>;
 			no-map;
 		};
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a78000.dtsi b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
index fb71974ef39050..28555fa7566fb6 100644
--- a/arch/arm64/boot/dts/renesas/r8a78000.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
@@ -5,6 +5,8 @@
  * Copyright (C) 2025 Renesas Electronics Corp.
  */
 
+#include <dt-bindings/clock/renesas,r8a78000-cpg.h>
+#include <dt-bindings/power/renesas,r8a78000-mdlc.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
 / {
@@ -668,23 +670,6 @@
 		};
 	};
 
-	/*
-	 * In the early phase, there is no clock control support,
-	 * so assume that the clocks are enabled by default.
-	 * Therefore, dummy clocks are used.
-	 */
-	dummy_clk_sgasyncd16: dummy-clk-sgasyncd16 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <66666000>;
-	};
-
-	dummy_clk_sgasyncd4: dummy-clk-sgasyncd4 {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <266660000>;
-	};
-
 	extal_clk: extal-clk {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -877,13 +862,129 @@
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		i2c1: i2c@c06c0000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06c0000 0 0x40>;
+			interrupts = <GIC_SPI 956 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x44>;
+			resets = <&mdlc_perw 0x44>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@c06c8000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06c8000 0 0x40>;
+			interrupts = <GIC_SPI 957 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x45>;
+			resets = <&mdlc_perw 0x45>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@c06d0000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06d0000 0 0x40>;
+			interrupts = <GIC_SPI 958 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x46>;
+			resets = <&mdlc_perw 0x46>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@c06d8000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06d8000 0 0x40>;
+			interrupts = <GIC_SPI 959 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x47>;
+			resets = <&mdlc_perw 0x47>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c5: i2c@c06e0000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06e0000 0 0x40>;
+			interrupts = <GIC_ESPI 0 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x48>;
+			resets = <&mdlc_perw 0x48>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c6: i2c@c06e8000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06e8000 0 0x40>;
+			interrupts = <GIC_ESPI 1 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x49>;
+			resets = <&mdlc_perw 0x49>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c7: i2c@c06f0000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06f0000 0 0x40>;
+			interrupts = <GIC_ESPI 2 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x4a>;
+			resets = <&mdlc_perw 0x4a>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c8: i2c@c06f8000 {
+			compatible = "renesas,i2c-r8a78000",
+				     "renesas,rcar-gen5-i2c";
+			reg = <0 0xc06f8000 0 0x40>;
+			interrupts = <GIC_ESPI 3 IRQ_TYPE_LEVEL_HIGH>;
+			i2c-scl-internal-delay-ns = <110>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD8_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x4b>;
+			resets = <&mdlc_perw 0x4b>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		scif0: serial@c0700000 {
 			compatible = "renesas,scif-r8a78000",
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0700000 0 0x40>;
 			interrupts = <GIC_ESPI 10 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD16_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x40>;
+			resets = <&mdlc_perw 0x40>;
 			status = "disabled";
 		};
 
@@ -892,8 +993,12 @@
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0704000 0 0x40>;
 			interrupts = <GIC_ESPI 11 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD16_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x41>;
+			resets = <&mdlc_perw 0x41>;
 			status = "disabled";
 		};
 
@@ -902,8 +1007,12 @@
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0708000 0 0x40>;
 			interrupts = <GIC_ESPI 12 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD16_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x42>;
+			resets = <&mdlc_perw 0x42>;
 			status = "disabled";
 		};
 
@@ -912,8 +1021,12 @@
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc070c000 0 0x40>;
 			interrupts = <GIC_ESPI 13 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD16_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x43>;
+			resets = <&mdlc_perw 0x43>;
 			status = "disabled";
 		};
 
@@ -922,8 +1035,12 @@
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc0710000 0 0x60>;
 			interrupts = <GIC_ESPI 14 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x54>;
+			resets = <&mdlc_perw 0x54>;
 			status = "disabled";
 		};
 
@@ -932,8 +1049,12 @@
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc0714000 0 0x60>;
 			interrupts = <GIC_ESPI 15 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x55>;
+			resets = <&mdlc_perw 0x55>;
 			status = "disabled";
 		};
 
@@ -942,8 +1063,12 @@
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc0718000 0 0x60>;
 			interrupts = <GIC_ESPI 16 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x56>;
+			resets = <&mdlc_perw 0x56>;
 			status = "disabled";
 		};
 
@@ -952,8 +1077,12 @@
 				     "renesas,rcar-gen5-hscif", "renesas,hscif";
 			reg = <0 0xc071c000 0 0x60>;
 			interrupts = <GIC_ESPI 17 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd4>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
+			clocks = <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&cpg R8A78000_CPG_SGASYNCD4_PERW_BUS>,
+				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x57>;
+			resets = <&mdlc_perw 0x57>;
 			status = "disabled";
 		};
 
@@ -965,6 +1094,343 @@
 			ranges = <0 0x0 0xc1060000 0x1c00>;
 			/* actual transport nodes must be set per board file */
 		};
+
+		cpg: clock-controller@c1320000 {
+			compatible = "renesas,r8a78000-cpg";
+			reg = <0 0xc1320000 0 0x10000>;
+			clocks = <&extal_clk>, <&extalr_clk>;
+			clock-names = "extal", "extalr";
+			#clock-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_aon: system-controller@c1338000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc1338000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_cmnn: system-controller@ca410000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xca410000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_cmns: system-controller@ca510000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xca510000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr0: system-controller@e8000000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8000000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr1: system-controller@e8080000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8080000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr2: system-controller@e8100000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8100000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr3: system-controller@e8180000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8180000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr4: system-controller@e8200000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8200000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr5: system-controller@e8280000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8280000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr6: system-controller@e8300000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8300000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ddr7: system-controller@e8380000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe8380000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_dsp: system-controller@cbe90000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xcbe90000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_gpc: system-controller@cb510000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xcb510000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_hscn: system-controller@c9c90000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc9c90000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_hscs: system-controller@de200000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xde200000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_imn: system-controller@c1990000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc1990000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_ims: system-controller@c1d90000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc1d90000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_mm: system-controller@e9980000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xe9980000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_npu0: system-controller@d2c30000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xd2c30000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_npu1: system-controller@d6c30000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xd6c30000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_pere: system-controller@c08f0000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc08f0000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_perw: system-controller@c05d0000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc05d0000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_rt: system-controller@19440000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0x19440000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_scp: system-controller@c1330000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc1330000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_top: system-controller@c6480000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc6480000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_vio: system-controller@c5000000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc5000000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_vipn: system-controller@c3060000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc3060000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		mdlc_vips: system-controller@c3460000 {
+			compatible = "renesas,r8a78000-mdlc";
+			reg = <0 0xc3460000 0 0x1000>;
+			#power-domain-cells = <2>;
+			#reset-cells = <1>;
+			bootph-all;
+		};
+
+		msiof0: spi@c1700000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc1700000 0 0x100>;
+			interrupts = <GIC_ESPI 744 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSO_SCP_MAIN>;
+			power-domains = <&mdlc_scp R8A78000_MDLC_PD_SCP 0x50>;
+			resets = <&mdlc_scp 0x50>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof1: spi@c1710000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc1710000 0 0x100>;
+			interrupts = <GIC_ESPI 745 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSO_SCP_MAIN>;
+			power-domains = <&mdlc_scp R8A78000_MDLC_PD_SCP 0x51>;
+			resets = <&mdlc_scp 0x51>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof2: spi@c1720000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc1720000 0 0x100>;
+			interrupts = <GIC_ESPI 746 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSO_SCP_MAIN>;
+			power-domains = <&mdlc_scp R8A78000_MDLC_PD_SCP 0x52>;
+			resets = <&mdlc_scp 0x52>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof3: spi@c1730000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc1730000 0 0x100>;
+			interrupts = <GIC_ESPI 747 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSO_SCP_MAIN>;
+			power-domains = <&mdlc_scp R8A78000_MDLC_PD_SCP 0x53>;
+			resets = <&mdlc_scp 0x53>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof4: spi@c0600000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc0600000 0 0x100>;
+			interrupts = <GIC_ESPI 18 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSOCK_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x4f>;
+			resets = <&mdlc_perw 0x4f>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof5: spi@c0608000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc0608000 0 0x100>;
+			interrupts = <GIC_ESPI 19 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSOCK_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x50>;
+			resets = <&mdlc_perw 0x50>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof6: spi@c0610000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc0610000 0 0x100>;
+			interrupts = <GIC_ESPI 20 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSOCK_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x51>;
+			resets = <&mdlc_perw 0x51>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof7: spi@c0618000 {
+			compatible = "renesas,msiof-r8a78000",
+				     "renesas,rcar-gen5-msiof";
+			reg = <0 0xc0618000 0 0x100>;
+			interrupts = <GIC_ESPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg R8A78000_CPG_MSOCK_PERW_BUS>;
+			power-domains = <&mdlc_perw R8A78000_MDLC_PD_APL 0x52>;
+			resets = <&mdlc_perw 0x52>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/renesas/r9a07g043u12-hummingboard-ripple.dts b/arch/arm64/boot/dts/renesas/r9a07g043u12-hummingboard-ripple.dts
new file mode 100644
index 00000000000000..8ffa51dab56b22
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a07g043u12-hummingboard-ripple.dts
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Description for SolidRun RZ/G2UL Type-2 HummingBoard Ripple.
+ *
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+
+#include "r9a07g043u.dtsi"
+#include "rzg2ul-sr-som.dtsi"
+#include "rzg2l-hummingboard-pulse-common.dtsi"
+
+/ {
+	compatible = "solidrun,rzg2ul-hummingboard-ripple", "solidrun,rzg2ul-sr-som",
+		     "renesas,r9a07g043u12", "renesas,r9a07g043";
+	model = "SolidRun RZ/G2UL HummingBoard Ripple";
+
+	leds: leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			gpios = <&pinctrl RZG2L_GPIO(12, 1) GPIO_ACTIVE_LOW>;
+			label = "D30";
+		};
+	};
+
+	vbus1: regulator-vbus-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vbus1";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		gpios = <&pinctrl RZG2L_GPIO(5, 2) GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+&hub_2_0 {
+	reset-gpios = <&pinctrl RZG2L_GPIO(13, 3) GPIO_ACTIVE_LOW>;
+};
+
+&pinctrl {
+	hdmi_pd_hog: hdmi-pd-hog {
+		/*
+		 * RZ/G2UL does not have DSI output to feed DSI-to-HDMI
+		 * bridge. Keep its shutdown signal asserted.
+		 */
+		gpio-hog;
+		gpios = <RZG2L_GPIO(14, 2) GPIO_ACTIVE_LOW>;
+		line-name = "hdmi-pd";
+		output-high;
+	};
+
+	mpcie_rst_hog: mpcie-reset-hog {
+		gpio-hog;
+		gpios = <RZG2L_GPIO(0, 3) GPIO_ACTIVE_LOW>;
+		line-name = "mpcie-reset";
+		output-low;
+	};
+
+	mikro_uart_pins: pinctrl-mikro-uart-grp {
+		pinmux = <RZG2L_PORT_PINMUX(14, 1, 1)>, /* SCIF1_RXD */
+			 <RZG2L_PORT_PINMUX(14, 0, 1)>; /* SCIF1_TXD */
+	};
+
+	usb0_vbus_pins: usb0-vbus {
+		pinmux = <RZG2L_PORT_PINMUX(5, 0, 1)>; /* USB0_VBUSEN */
+	};
+};
+
+&usb2_phy1 {
+	vbus-supply = <&vbus1>;
+};
+
+&vmmc {
+	gpios = <&pinctrl RZG2L_GPIO(0, 1) GPIO_ACTIVE_LOW>;
+};
+
+&vmpcie {
+	/*
+	 * G2UL SoM v1.2 moved this IO from J7-28 (CON4-12)
+	 * to J7-18 (this regulator).
+	 * There are no known users of CON4-12 and thus no need
+	 * to differentiate here between SoM v1.1 and v1.2.
+	 */
+	gpios = <&pinctrl RZG2L_GPIO(13, 4) GPIO_ACTIVE_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-iiot.dts b/arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-iiot.dts
new file mode 100644
index 00000000000000..9cc21ae32ed4db
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-iiot.dts
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+
+#include "r9a07g044c2.dtsi"
+#include "rzg2lc-sr-som.dtsi"
+#include "rzg2l-hummingboard-iiot-common.dtsi"
+
+/ {
+	compatible = "solidrun,rzg2lc-hummingboard-iiot", "solidrun,rzg2lc-sr-som",
+		     "renesas,r9a07g044c2", "renesas,r9a07g044";
+	model = "SolidRun RZ/G2LC HummingBoard IIoT";
+};
+
+&vmmc {
+	gpios = <&pinctrl RZG2L_GPIO(18, 1) GPIO_ACTIVE_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-ripple.dts b/arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-ripple.dts
new file mode 100644
index 00000000000000..ef8e2f62b6827f
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a07g044c2-hummingboard-ripple.dts
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+
+#include "r9a07g044c2.dtsi"
+#include "rzg2lc-sr-som.dtsi"
+#include "rzg2l-hummingboard-pulse-common.dtsi"
+#include "rzg2l-hummingboard-pulse-micro-hdmi.dtsi"
+
+/ {
+	compatible = "solidrun,rzg2lc-hummingboard-ripple", "solidrun,rzg2lc-sr-som",
+		     "renesas,r9a07g044c2", "renesas,r9a07g044";
+	model = "SolidRun RZ/G2LC HummingBoard Ripple";
+
+	leds: leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			gpios = <&pinctrl RZG2L_GPIO(0, 1) GPIO_ACTIVE_LOW>;
+			label = "D30";
+		};
+	};
+
+	vbus1: regulator-vbus-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vbus1";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		gpios = <&pinctrl RZG2L_GPIO(5, 0) GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+};
+
+&gpt {
+	/* mikrobus header pwm pin is pwm channel 6 */
+	pinctrl-0 = <&mikro_pwm_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdmi {
+	interrupts-extended = <&pinctrl RZG2L_GPIO(42, 2) IRQ_TYPE_EDGE_FALLING>;
+	pd-gpios = <&pinctrl RZG2L_GPIO(40, 2) GPIO_ACTIVE_LOW>;
+};
+
+&hub_2_0 {
+	reset-gpios = <&pinctrl RZG2L_GPIO(39, 1) GPIO_ACTIVE_LOW>;
+};
+
+&pinctrl {
+	mpcie_rst_hog: mpcie-reset-hog {
+		gpios = <RZG2L_GPIO(19, 1) GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		line-name = "mpcie-reset";
+		output-low;
+	};
+
+	mikro_pwm_pins: pinctrl-mikro-pwm-grp {
+		pinmux = <RZG2L_PORT_PINMUX(41, 0, 4)>; /* GTIOC3A */
+	};
+
+	mikro_uart_pins: pinctrl-mikro-uart-grp {
+		pinmux = <RZG2L_PORT_PINMUX(40, 1, 1)>, /* SCIF1_RXD */
+			 <RZG2L_PORT_PINMUX(40, 0, 1)>; /* SCIF1_TXD */
+	};
+
+	usb0_vbus_pins: usb0-vbus {
+		pinmux = <RZG2L_PORT_PINMUX(4, 0, 1)>; /* USB0_VBUSEN */
+	};
+};
+
+&usb2_phy1 {
+	vbus-supply = <&vbus1>;
+};
+
+&vmmc {
+	gpios = <&pinctrl RZG2L_GPIO(18, 1) GPIO_ACTIVE_LOW>;
+};
+
+&vmpcie {
+	/*
+	 * G2LC SoM v1.2 moved this IO from J7-28 (CON4-12)
+	 * to J7-18 (this regulator).
+	 * There are no known users of CON4-12 and thus no need
+	 * to differentiate here between SoM v1.1 and v1.2.
+	 */
+	gpios = <&pinctrl RZG2L_GPIO(39, 2) GPIO_ACTIVE_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-iiot.dts b/arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-iiot.dts
new file mode 100644
index 00000000000000..eba4f423c8f050
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-iiot.dts
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+
+#include "r9a07g044l2.dtsi"
+#include "rzg2l-sr-som.dtsi"
+#include "rzg2l-hummingboard-iiot.dtsi"
+
+/ {
+	compatible = "solidrun,rzg2l-hummingboard-iiot", "solidrun,rzg2l-sr-som",
+		     "renesas,r9a07g044l2", "renesas,r9a07g044";
+	model = "SolidRun RZ/G2L HummingBoard IIoT";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-ripple.dts b/arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-ripple.dts
new file mode 100644
index 00000000000000..95a36f8804b558
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a07g044l2-hummingboard-ripple.dts
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+
+#include "r9a07g044l2.dtsi"
+#include "rzg2l-sr-som.dtsi"
+#include "rzg2l-hummingboard-pulse-common.dtsi"
+#include "rzg2l-hummingboard-pulse-micro-hdmi.dtsi"
+#include "rzg2l-hummingboard-ripple.dtsi"
+
+/ {
+	compatible = "solidrun,rzg2l-hummingboard-ripple", "solidrun,rzg2l-sr-som",
+		     "renesas,r9a07g044l2", "renesas,r9a07g044";
+	model = "SolidRun RZ/G2L HummingBoard Ripple";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g044l2-remi-pi.dts b/arch/arm64/boot/dts/renesas/r9a07g044l2-remi-pi.dts
index 3267e7b75b58ff..dea386f3c836e9 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g044l2-remi-pi.dts
+++ b/arch/arm64/boot/dts/renesas/r9a07g044l2-remi-pi.dts
@@ -156,7 +156,7 @@
 
 	hdmi-bridge@48 {
 		compatible = "lontium,lt8912b";
-		reg = <0x48> ;
+		reg = <0x48>;
 		reset-gpios = <&pinctrl RZG2L_GPIO(42, 2) GPIO_ACTIVE_LOW>;
 
 		ports {
diff --git a/arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-iiot.dts b/arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-iiot.dts
new file mode 100644
index 00000000000000..d77a6ff163bea1
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-iiot.dts
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+
+#include "r9a07g054l2.dtsi"
+#include "rzg2l-sr-som.dtsi"
+#include "rzg2l-hummingboard-iiot.dtsi"
+
+/ {
+	compatible = "solidrun,rzv2l-hummingboard-iiot", "solidrun,rzv2l-sr-som",
+		     "renesas,r9a07g054l2", "renesas,r9a07g054";
+	model = "SolidRun RZ/V2L HummingBoard IIoT";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-ripple.dts b/arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-ripple.dts
new file mode 100644
index 00000000000000..0c738715245414
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a07g054l2-hummingboard-ripple.dts
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+
+#include "r9a07g054l2.dtsi"
+#include "rzg2l-sr-som.dtsi"
+#include "rzg2l-hummingboard-pulse-common.dtsi"
+#include "rzg2l-hummingboard-pulse-micro-hdmi.dtsi"
+#include "rzg2l-hummingboard-ripple.dtsi"
+
+/ {
+	compatible = "solidrun,rzv2l-hummingboard-ripple", "solidrun,rzv2l-sr-som",
+		     "renesas,r9a07g054l2", "renesas,r9a07g054";
+	model = "SolidRun RZ/V2L HummingBoard Ripple";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a08g045.dtsi b/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
index 3a69bb246babd0..b98ad375a3e9c1 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
@@ -645,7 +645,7 @@
 			dma-channels = <16>;
 		};
 
-		sdhi0: mmc@11c00000  {
+		sdhi0: mmc@11c00000 {
 			compatible = "renesas,sdhi-r9a08g045", "renesas,rzg2l-sdhi";
 			reg = <0x0 0x11c00000 0 0x10000>;
 			interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
@@ -655,6 +655,7 @@
 				 <&cpg CPG_MOD R9A08G045_SDHI0_IMCLK2>,
 				 <&cpg CPG_MOD R9A08G045_SDHI0_ACLK>;
 			clock-names = "core", "clkh", "cd", "aclk";
+			max-frequency = <125000000>;
 			resets = <&cpg R9A08G045_SDHI0_IXRST>;
 			power-domains = <&cpg>;
 			status = "disabled";
@@ -670,6 +671,7 @@
 				 <&cpg CPG_MOD R9A08G045_SDHI1_IMCLK2>,
 				 <&cpg CPG_MOD R9A08G045_SDHI1_ACLK>;
 			clock-names = "core", "clkh", "cd", "aclk";
+			max-frequency = <125000000>;
 			resets = <&cpg R9A08G045_SDHI1_IXRST>;
 			power-domains = <&cpg>;
 			status = "disabled";
@@ -685,6 +687,7 @@
 				 <&cpg CPG_MOD R9A08G045_SDHI2_IMCLK2>,
 				 <&cpg CPG_MOD R9A08G045_SDHI2_ACLK>;
 			clock-names = "core", "clkh", "cd", "aclk";
+			max-frequency = <50000000>;
 			resets = <&cpg R9A08G045_SDHI2_IXRST>;
 			power-domains = <&cpg>;
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/renesas/r9a08g046.dtsi b/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
index c63a857f0e5b01..f33d5d7468ffa4 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
@@ -64,6 +64,110 @@
 		};
 	};
 
+	gpu_opp_table: opp-table-1 {
+		compatible = "operating-points-v2";
+
+		opp-600000000 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-533330000 {
+			opp-hz = /bits/ 64 <533330000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-400000000 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-300000000 {
+			opp-hz = /bits/ 64 <300000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-266667000 {
+			opp-hz = /bits/ 64 <266667000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-250000000 {
+			opp-hz = /bits/ 64 <250000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-200000000 {
+			opp-hz = /bits/ 64 <200000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-150000000 {
+			opp-hz = /bits/ 64 <150000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-133333000 {
+			opp-hz = /bits/ 64 <133333000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-125000000 {
+			opp-hz = /bits/ 64 <125000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-100000000 {
+			opp-hz = /bits/ 64 <100000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-75000000 {
+			opp-hz = /bits/ 64 <75000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-66667000 {
+			opp-hz = /bits/ 64 <66667000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-62500000 {
+			opp-hz = /bits/ 64 <62500000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-50000000 {
+			opp-hz = /bits/ 64 <50000000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-18750000 {
+			opp-hz = /bits/ 64 <18750000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-16667000 {
+			opp-hz = /bits/ 64 <16667000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-15625000 {
+			opp-hz = /bits/ 64 <15625000>;
+			opp-microvolt = <1000000>;
+		};
+
+		opp-12500000 {
+			opp-hz = /bits/ 64 <12500000>;
+			opp-microvolt = <1000000>;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -592,6 +696,167 @@
 			status = "disabled";
 		};
 
+		dsi: dsi@10850000 {
+			compatible = "renesas,r9a08g046-mipi-dsi";
+			reg = <0 0x10850000 0 0x20000>;
+			interrupts = <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 172 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "seq0", "seq1", "vin1", "rcv",
+					  "ferr", "ppi", "debug";
+			clocks = <&cpg CPG_CORE R9A08G046_MIPI_DSI_PLLCLK>,
+				 <&cpg CPG_MOD R9A08G046_MIPI_DSI_SYSCLK>,
+				 <&cpg CPG_MOD R9A08G046_MIPI_DSI_ACLK>,
+				 <&cpg CPG_MOD R9A08G046_MIPI_DSI_PCLK>,
+				 <&cpg CPG_MOD R9A08G046_MIPI_DSI_VCLK>,
+				 <&cpg CPG_MOD R9A08G046_MIPI_DSI_LPCLK>;
+			clock-names = "pllclk", "sysclk", "aclk", "pclk", "vclk", "lpclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_MIPI_DSI_CMN_RSTB>,
+				 <&cpg R9A08G046_MIPI_DSI_ARESET_N>,
+				 <&cpg R9A08G046_MIPI_DSI_PRESET_N>;
+			reset-names = "rst", "arst", "prst";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					dsi_in: endpoint {
+						remote-endpoint = <&du_out_dsi>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+				};
+			};
+		};
+
+		vspd: vsp@10870000 {
+			compatible = "renesas,r9a08g046-vsp2",
+				     "renesas,r9a07g044-vsp2";
+			reg = <0 0x10870000 0 0x10000>;
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD R9A08G046_LCDC_CLK_A>,
+				 <&cpg CPG_MOD R9A08G046_LCDC_CLK_P>,
+				 <&cpg CPG_MOD R9A08G046_LCDC_CLK_D>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_LCDC_RESET_N>;
+			renesas,fcp = <&fcpvd>;
+		};
+
+		fcpvd: fcp@10880000 {
+			compatible = "renesas,r9a08g046-fcpvd", "renesas,fcpv";
+			reg = <0 0x10880000 0 0x10000>;
+			clocks = <&cpg CPG_MOD R9A08G046_LCDC_CLK_A>,
+				 <&cpg CPG_MOD R9A08G046_LCDC_CLK_P>,
+				 <&cpg CPG_MOD R9A08G046_LCDC_CLK_D>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_LCDC_RESET_N>;
+		};
+
+		du: display@10890000 {
+			compatible = "renesas,r9a08g046-du";
+			reg = <0 0x10890000 0 0x10000>;
+			interrupts = <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD R9A08G046_LCDC_CLK_A>,
+				 <&cpg CPG_MOD R9A08G046_LCDC_CLK_P>,
+				 <&cpg CPG_MOD R9A08G046_LCDC_CLK_D>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_LCDC_RESET_N>;
+			renesas,vsps = <&vspd 0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					du_out_dsi: endpoint {
+						remote-endpoint = <&dsi_in>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+				};
+
+				port@2 {
+					reg = <2>;
+					du_out_lvds: endpoint {
+						remote-endpoint = <&lvds_in>;
+					};
+				};
+			};
+		};
+
+		lvds: lvds@108a0000 {
+			compatible = "renesas,r9a08g046-lvds";
+			reg = <0 0x108a0000 0 0x10000>;
+			clocks = <&cpg CPG_MOD R9A08G046_MIPI_DSI_PCLK>,
+				 <&cpg CPG_MOD R9A08G046_LVDS_PLLCLK>,
+				 <&cpg CPG_MOD R9A08G046_LVDS_CLK_DOT0>;
+			clock-names = "pclk", "phyclk", "dotclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_LVDS_RESET_N>,
+				 <&cpg R9A08G046_MIPI_DSI_PRESET_N>,
+				 <&cpg R9A08G046_MIPI_DSI_CMN_RSTB>,
+				 <&cpg R9A08G046_MIPI_DSI_ARESET_N>;
+			reset-names = "lvdrst", "prst", "rst", "arst";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					lvds_in: endpoint {
+						remote-endpoint = <&du_out_lvds>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+					lvds_out: endpoint {
+					};
+				};
+			};
+		};
+
+		gpu: gpu@108b0000 {
+			compatible = "renesas,r9a08g046-mali",
+				     "arm,mali-bifrost";
+			reg = <0x0 0x108b0000 0x0 0x10000>;
+			interrupts = <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "job", "mmu", "gpu", "event";
+			clocks = <&cpg CPG_MOD R9A08G046_GE3D_CLK>,
+				 <&cpg CPG_MOD R9A08G046_GE3D_AXI_CLK>,
+				 <&cpg CPG_MOD R9A08G046_GE3D_ACE_CLK>;
+			clock-names = "gpu", "bus", "bus_ace";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_GE3D_RESETN>,
+				 <&cpg R9A08G046_GE3D_AXI_RESETN>,
+				 <&cpg R9A08G046_GE3D_ACE_RESETN>;
+			reset-names = "rst", "axi_rst", "ace_rst";
+			operating-points-v2 = <&gpu_opp_table>;
+			status = "disabled";
+		};
+
 		cpg: clock-controller@11010000 {
 			compatible = "renesas,r9a08g046-cpg";
 			reg = <0 0x11010000 0 0x10000>;
@@ -1001,21 +1266,87 @@
 			};
 		};
 
+		ohci0: usb@11e10000 {
+			reg = <0 0x11e10000 0 0x100>;
+			/* placeholder */
+		};
+
+		ohci1: usb@11e90000 {
+			reg = <0 0x11e90000 0 0x100>;
+			/* placeholder */
+		};
+
+		ehci0: usb@11e10100 {
+			reg = <0 0x11e10100 0 0x100>;
+			/* placeholder */
+		};
+
+		ehci1: usb@11e90100 {
+			reg = <0 0x11e90100 0 0x100>;
+			/* placeholder */
+		};
+
+		hsusb0: usb@11e30000 {
+			reg = <0 0x11e30000 0 0x10000>;
+			/* placeholder */
+		};
+
 		pcie: pcie@11e40000 {
+			compatible = "renesas,r9a08g046-pcie";
 			reg = <0 0x11e40000 0 0x10000>;
 			ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x08000000>;
+			/* Map all possible DRAM ranges (4 GB). */
+			dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 1 0x00000000>;
+			bus-range = <0x0 0xff>;
+			interrupts = <GIC_SPI 483 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 484 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 485 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 486 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 487 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 488 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 489 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 490 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 491 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 492 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 493 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 494 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 495 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 496 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 497 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 498 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "serr", "serr_cor", "serr_nonfatal",
+					  "serr_fatal", "axi_err", "inta",
+					  "intb", "intc", "intd", "msi",
+					  "link_bandwidth", "pm_pme", "dma",
+					  "pcie_evt", "msg", "all";
+			#interrupt-cells = <1>;
+			interrupt-controller;
+			interrupt-map-mask = <0 0 0 7>;
+			interrupt-map = <0 0 0 1 &pcie 0 0 0 0>, /* INTA */
+					<0 0 0 2 &pcie 0 0 0 1>, /* INTB */
+					<0 0 0 3 &pcie 0 0 0 2>, /* INTC */
+					<0 0 0 4 &pcie 0 0 0 3>; /* INTD */
+			clocks = <&cpg CPG_MOD R9A08G046_PCI_ACLK>,
+				 <&cpg CPG_MOD R9A08G046_PCI_CLKL1PM>,
+				 <&cpg CPG_MOD R9A08G046_PCI_CLK_PMU>;
+			clock-names = "aclk", "pm", "pmu";
+			resets = <&cpg R9A08G046_PCI_ARESETN>;
+			reset-names = "aresetn";
+			power-domains = <&cpg>;
 			device_type = "pci";
 			#address-cells = <3>;
 			#size-cells = <2>;
-			/* placeholder */
+			renesas,sysc = <&sysc>;
+			status = "disabled";
 
 			pcie_port0: pcie@0,0 {
 				reg = <0x0 0x0 0x0 0x0 0x0>;
 				ranges;
 				device_type = "pci";
+				vendor-id = <0x1912>;
+				device-id = <0x0033>;
 				#address-cells = <3>;
 				#size-cells = <2>;
-				/* placeholder */
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts b/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
index 5289efd1a430a5..68d7b579e35217 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
+++ b/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
@@ -71,6 +71,18 @@
 #endif
 };
 
+&ehci0 {
+	status = "disabled";
+};
+
+&ehci1 {
+	status = "disabled";
+};
+
+&hsusb0 {
+	status = "disabled";
+};
+
 &i2c2 {
 	pinctrl-0 = <&i2c2_pins>;
 	pinctrl-names = "default";
@@ -122,6 +134,19 @@
 #endif
 };
 
+&ohci0 {
+	status = "disabled";
+};
+
+&ohci1 {
+	status = "disabled";
+};
+
+&pcie {
+	pinctrl-0 = <&pcie_pins>;
+	pinctrl-names = "default";
+};
+
 &pinctrl {
 	audio_clk_pins: audio-clock {
 		pinmux = <RZG3L_PORT_PINMUX(H, 4, 6)>, /* AUDIO_CLK_B */
@@ -138,6 +163,17 @@
 			 <RZG3L_PORT_PINMUX(2, 1, 4)>; /* RIIC3_SDA */
 	};
 
+	pcie-clkreq-n-hog {
+		gpio-hog;
+		gpios = <RZG3L_GPIO(G, 6) GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "PCIE_A_CKREQ";
+	};
+
+	pcie_pins: pcie {
+		pinmux = <RZG3L_PORT_PINMUX(G, 7, 1)>; /* PCIE_RST_OUT# */
+	};
+
 	rsci1_pins: rsci1 {
 		pinmux = <RZG3L_PORT_PINMUX(D, 4, 5)>, /* RSCI1_RXD_MISO_SCL */
 			 <RZG3L_PORT_PINMUX(D, 5, 5)>, /* RSCI1_TXD_MOSI_SDA */
diff --git a/arch/arm64/boot/dts/renesas/r9a09g011.dtsi b/arch/arm64/boot/dts/renesas/r9a09g011.dtsi
index 42462c138dd236..fbe1e1c24f57a6 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g011.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g011.dtsi
@@ -85,7 +85,7 @@
 			status = "disabled";
 		};
 
-		sdhi1: mmc@85010000  {
+		sdhi1: mmc@85010000 {
 			compatible = "renesas,sdhi-r9a09g011",
 				     "renesas,rzg2l-sdhi";
 			reg = <0x0 0x85010000 0 0x2000>;
@@ -101,7 +101,7 @@
 			status = "disabled";
 		};
 
-		emmc: mmc@85020000  {
+		emmc: mmc@85020000 {
 			compatible = "renesas,sdhi-r9a09g011",
 				     "renesas,rzg2l-sdhi";
 			reg = <0x0 0x85020000 0 0x2000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
index 060405d4a2ddea..737bf1c60451ff 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
@@ -1743,6 +1743,130 @@
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_LOW>;
 		};
 
+		ohci0: usb@15800000 {
+			compatible = "generic-ohci";
+			reg = <0 0x15800000 0 0x100>;
+			interrupts = <GIC_SPI 742 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 0xb3>, <&cpg CPG_MOD 0xb6>;
+			resets = <&usb20phyrst>, <&cpg 0xac>;
+			phys = <&usb2_phy0 1>;
+			phy-names = "usb";
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
+		ohci1: usb@15810000 {
+			compatible = "generic-ohci";
+			reg = <0 0x15810000 0 0x100>;
+			interrupts = <GIC_SPI 747 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 0xb4>, <&cpg CPG_MOD 0xb7>;
+			resets = <&usb21phyrst>, <&cpg 0xad>;
+			phys = <&usb2_phy1 1>;
+			phy-names = "usb";
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
+		ehci0: usb@15800100 {
+			compatible = "generic-ehci";
+			reg = <0 0x15800100 0 0x100>;
+			interrupts = <GIC_SPI 743 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 0xb3>, <&cpg CPG_MOD 0xb6>;
+			resets = <&usb20phyrst>, <&cpg 0xac>;
+			phys = <&usb2_phy0 2>;
+			phy-names = "usb";
+			companion = <&ohci0>;
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
+		ehci1: usb@15810100 {
+			compatible = "generic-ehci";
+			reg = <0 0x15810100 0 0x100>;
+			interrupts = <GIC_SPI 748 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 0xb4>, <&cpg CPG_MOD 0xb7>;
+			resets = <&usb21phyrst>, <&cpg 0xad>;
+			phys = <&usb2_phy1 2>;
+			phy-names = "usb";
+			companion = <&ohci1>;
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
+		usb2_phy0: usb-phy@15800200 {
+			compatible = "renesas,usb2-phy-r9a09g047",
+				     "renesas,usb2-phy-r9a09g057";
+			reg = <0 0x15800200 0 0x700>;
+			interrupts = <GIC_SPI 745 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 0xb3>,
+				 <&cpg CPG_CORE R9A09G047_USB2_0_CLK_CORE0>;
+			clock-names = "fck", "usb_x1";
+			resets = <&usb20phyrst>, <&cpg 0xac>;
+			#phy-cells = <1>;
+			power-domains = <&cpg>;
+			mux-states = <&usb20phyrst 1>;
+			status = "disabled";
+
+			usb2_phy0_vbus_otg: vbus-regulator {
+				regulator-name = "USB2PHY0-VBUS-OTG";
+			};
+		};
+
+		usb2_phy1: usb-phy@15810200 {
+			compatible = "renesas,usb2-phy-r9a09g047",
+				     "renesas,usb2-phy-r9a09g057";
+			reg = <0 0x15810200 0 0x700>;
+			interrupts = <GIC_SPI 750 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 0xb4>,
+				 <&cpg CPG_CORE R9A09G047_USB2_0_CLK_CORE1>;
+			clock-names = "fck", "usb_x1";
+			resets = <&usb21phyrst>, <&cpg 0xad>;
+			#phy-cells = <1>;
+			power-domains = <&cpg>;
+			mux-states = <&usb21phyrst 0>;
+			status = "disabled";
+		};
+
+		hsusb0: usb@15820000 {
+			compatible = "renesas,usbhs-r9a09g047",
+				     "renesas,rzg2l-usbhs";
+			reg = <0 0x15820000 0 0x10000>;
+			interrupts = <GIC_SPI 751 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 752 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 753 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 754 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 0xb3>, <&cpg CPG_MOD 0xb5>;
+			resets = <&usb20phyrst>, <&cpg 0xae>;
+			phys = <&usb2_phy0 3>;
+			phy-names = "usb";
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
+		usb20phyrst: reset-controller@15830000 {
+			compatible = "renesas,r9a09g047-usb2phy-reset",
+				     "renesas,r9a09g057-usb2phy-reset";
+			reg = <0 0x15830000 0 0x10000>;
+			clocks = <&cpg CPG_MOD 0xb6>;
+			resets = <&cpg 0xaf>;
+			power-domains = <&cpg>;
+			#reset-cells = <0>;
+			#mux-state-cells = <1>;
+			status = "disabled";
+		};
+
+		usb21phyrst: reset-controller@15840000 {
+			compatible = "renesas,r9a09g047-usb2phy-reset",
+				     "renesas,r9a09g057-usb2phy-reset";
+			reg = <0 0x15840000 0 0x10000>;
+			clocks = <&cpg CPG_MOD 0xb7>;
+			resets = <&cpg 0xaf>;
+			power-domains = <&cpg>;
+			#reset-cells = <0>;
+			#mux-state-cells = <1>;
+			status = "disabled";
+		};
+
 		xhci: usb@15850000 {
 			compatible = "renesas,r9a09g047-xhci";
 			reg = <0 0x15850000 0 0x10000>;
@@ -1773,7 +1897,7 @@
 			status = "disabled";
 		};
 
-		sdhi0: mmc@15c00000  {
+		sdhi0: mmc@15c00000 {
 			compatible = "renesas,sdhi-r9a09g047", "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x15c00000 0 0x10000>;
 			interrupts = <GIC_SPI 735 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
index 4eed095b683b31..095acb87c20c6c 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g047e57-smarc.dts
@@ -112,7 +112,7 @@
 	channel1 {
 		status = "okay";
 #if (!SW_LCD_EN) && (SW_GPIO9_CAN1_STB)
-		phys = <&can_transceiver1>;
+		phys = <&can_transceiver 1>;
 #endif
 	};
 #endif
@@ -121,22 +121,25 @@
 	channel4 {
 		status = "okay";
 #if (SW_GPIO8_CAN0_STB)
-		phys = <&can_transceiver0>;
+		phys = <&can_transceiver 0>;
 #endif
 	};
 #endif
 };
 
-#if (!SW_LCD_EN) && (SW_GPIO8_CAN0_STB)
-&can_transceiver0 {
-	standby-gpios = <&pinctrl RZG3E_GPIO(5, 4) GPIO_ACTIVE_HIGH>;
-	status = "okay";
-};
+#if (!SW_LCD_EN) && (SW_GPIO8_CAN0_STB || SW_GPIO9_CAN1_STB)
+&can_transceiver {
+	standby-gpios =
+#if (SW_GPIO8_CAN0_STB)
+			<&pinctrl RZG3E_GPIO(5, 4) GPIO_ACTIVE_HIGH>,
+#else
+			<0>,
+#endif
+#if (SW_GPIO9_CAN1_STB)
+			<&pinctrl RZG3E_GPIO(5, 5) GPIO_ACTIVE_HIGH>;
+#else
+			<0>;
 #endif
-
-#if (!SW_LCD_EN) && (SW_GPIO9_CAN1_STB)
-&can_transceiver1 {
-	standby-gpios = <&pinctrl RZG3E_GPIO(5, 5) GPIO_ACTIVE_HIGH>;
 	status = "okay";
 };
 #endif
@@ -147,6 +150,8 @@
 };
 
 &i2c1 {
+	clock-frequency = <1000000>;
+
 	da7212: codec@1a {
 		compatible = "dlg,da7212";
 		#sound-dai-cells = <0>;
@@ -296,6 +301,17 @@
 		};
 	};
 
+	usb20_pins: usb20 {
+		ovc {
+			pinmux = <RZG3E_PORT_PINMUX(0, 0, 12)>; /* USB20_OVRCURN */
+			bias-pull-up;
+		};
+
+		vbus {
+			pinmux = <RZG3E_PORT_PINMUX(0, 1, 12)>; /* USB20_VBUSEN */
+		};
+	};
+
 	usb3_pins: usb3 {
 		pinmux = <RZG3E_PORT_PINMUX(4, 1, 12)>, /* USB30_VBUSEN */
 			 <RZG3E_PORT_PINMUX(4, 0, 12)>; /* USB30_OVRCURN */
@@ -392,6 +408,26 @@
 	shared-pin;
 };
 
+&usb20phyrst {
+	status = "okay";
+};
+
+&usb21phyrst {
+	status = "okay";
+};
+
+&usb2_phy0 {
+	pinctrl-0 = <&usb20_pins>;
+	pinctrl-names = "default";
+
+	vbus-supply = <&usb2_phy0_vbus_otg>;
+	status = "okay";
+};
+
+&usb2_phy1 {
+	status = "okay";
+};
+
 &usb3_phy {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
index 77c2221a9e2a3c..175e24c98e11d2 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
@@ -1447,7 +1447,7 @@
 			status = "disabled";
 		};
 
-		sdhi0: mmc@15c00000  {
+		sdhi0: mmc@15c00000 {
 			compatible = "renesas,sdhi-r9a09g056", "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x15c00000 0 0x10000>;
 			interrupts = <GIC_SPI 735 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
index 188ce9f9c7c21d..bbaab75681c299 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
@@ -1577,7 +1577,7 @@
 			status = "disabled";
 		};
 
-		sdhi0: mmc@15c00000  {
+		sdhi0: mmc@15c00000 {
 			compatible = "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x15c00000 0 0x10000>;
 			interrupts = <GIC_SPI 735 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
index 637fc92dcc2632..0926ab891fa5cf 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g057h44-rzv2h-evk.dts
@@ -461,20 +461,20 @@
 	};
 
 	sdhi1_pins: sd1 {
-		sd1_dat_cmd {
+		sd1-dat-cmd {
 			pins = "SD1DAT0", "SD1DAT1", "SD1DAT2", "SD1DAT3", "SD1CMD";
 			input-enable;
 			renesas,output-impedance = <3>;
 			slew-rate = <0>;
 		};
 
-		sd1_clk {
+		sd1-clk {
 			pins = "SD1CLK";
 			renesas,output-impedance = <3>;
 			slew-rate = <0>;
 		};
 
-		sd1_cd {
+		sd1-cd {
 			pinmux = <RZV2H_PORT_PINMUX(9, 4, 14)>; /* SD1_CD */
 		};
 	};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
index bac39390ead74d..674e35e4f24254 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g077.dtsi
@@ -1075,8 +1075,8 @@
 
 		cpg: clock-controller@80280000 {
 			compatible = "renesas,r9a09g077-cpg-mssr";
-			reg = <0 0x80280000 0 0x10000>,
-			      <0 0x81280000 0 0x10000>;
+			reg = <0 0x80280000 0 0x20000>,
+			      <0 0x81280000 0 0x20000>;
 			clocks = <&extal_clk>;
 			clock-names = "extal";
 			#clock-cells = <2>;
@@ -1173,6 +1173,19 @@
 			power-domains = <&cpg>;
 		};
 
+		rtc0: rtc@81009000 {
+			compatible = "renesas,r9a09g077-rtc";
+			reg = <0 0x81009000 0 0x1000>;
+			interrupts = <GIC_SPI 495 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 497 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 496 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "alarm", "timer", "pps";
+			clocks = <&cpg CPG_MOD 605>, <&cpg CPG_CORE R9A09G077_PCLKRTC>;
+			clock-names = "hclk", "xtal";
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
 		gic: interrupt-controller@83000000 {
 			compatible = "arm,gic-v3";
 			reg = <0x0 0x83000000 0 0x40000>,
@@ -1258,14 +1271,14 @@
 				     <GIC_SPI 699 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 700 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 701 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 702 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 851 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 852 IRQ_TYPE_EDGE_RISING>;
+				     <GIC_SPI 702 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "adi", "gbadi", "gcadi",
-					  "cmpai", "cmpbi", "wcmpm", "wcmpum";
+					  "cmpai", "cmpbi";
 			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKL>,
 				 <&cpg CPG_MOD 206>;
 			clock-names = "adclk", "pclk";
+			dmas = <&dmac0 0x26ba>, <&dmac1 0x26ba>, <&dmac2 0x26ba>;
+			dma-names = "rx", "rx", "rx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1280,14 +1293,14 @@
 				     <GIC_SPI 704 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 705 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 706 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 853 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 854 IRQ_TYPE_EDGE_RISING>;
+				     <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "adi", "gbadi", "gcadi",
-					  "cmpai", "cmpbi", "wcmpm", "wcmpum";
+					  "cmpai", "cmpbi";
 			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKL>,
 				 <&cpg CPG_MOD 207>;
 			clock-names = "adclk", "pclk";
+			dmas = <&dmac0 0x26bf>, <&dmac1 0x26bf>, <&dmac2 0x26bf>;
+			dma-names = "rx", "rx", "rx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1302,14 +1315,14 @@
 				     <GIC_SPI 709 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 710 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 711 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 712 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 855 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 856 IRQ_TYPE_EDGE_RISING>;
+				     <GIC_SPI 712 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "adi", "gbadi", "gcadi",
-					  "cmpai", "cmpbi", "wcmpm", "wcmpum";
+					  "cmpai", "cmpbi";
 			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKL>,
 				 <&cpg CPG_MOD 225>;
 			clock-names = "adclk", "pclk";
+			dmas = <&dmac0 0x26c4>, <&dmac1 0x26c4>, <&dmac2 0x26c4>;
+			dma-names = "rx", "rx", "rx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1364,7 +1377,7 @@
 			status = "disabled";
 		};
 
-		sdhi0: mmc@92080000  {
+		sdhi0: mmc@92080000 {
 			compatible = "renesas,sdhi-r9a09g077",
 				     "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x92080000 0 0x10000>;
@@ -1403,6 +1416,52 @@
 				status = "disabled";
 			};
 		};
+
+		du: display@920c0000 {
+			compatible = "renesas,r9a09g077-du";
+			reg = <0 0x920c0000 0 0x10000>;
+			interrupts = <GIC_SPI 781 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKAH>,
+				 <&cpg CPG_MOD 1204>,
+				 <&cpg CPG_CORE R9A09G077_LCDC_CLKD>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+			renesas,vsps = <&vspd 0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					du_out_rgb: endpoint {
+					};
+				};
+			};
+		};
+
+		fcpvd: fcp@920d0000 {
+			compatible = "renesas,r9a09g077-fcpvd", "renesas,fcpv";
+			reg = <0 0x920d0000 0 0x10000>;
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKAH>,
+				 <&cpg CPG_MOD 1204>,
+				 <&cpg CPG_CORE R9A09G077_LCDC_CLKD>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+		};
+
+		vspd: vsp@920e0000 {
+			compatible = "renesas,r9a09g077-vsp2", "renesas,r9a07g044-vsp2";
+			reg = <0 0x920e0000 0 0x8000>;
+			interrupts = <GIC_SPI 780 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_CORE R9A09G077_CLK_PCLKAH>,
+				 <&cpg CPG_MOD 1204>,
+				 <&cpg CPG_CORE R9A09G077_LCDC_CLKD>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+			renesas,fcp = <&fcpvd>;
+		};
 	};
 
 	stmmac_axi_setup: stmmac-axi-config {
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077m44-evk-cn15-lcdc.dtso b/arch/arm64/boot/dts/renesas/r9a09g077m44-evk-cn15-lcdc.dtso
new file mode 100644
index 00000000000000..aaddfc1154d54e
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a09g077m44-evk-cn15-lcdc.dtso
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DT overlay for the RZ/T2H EVK with ADV7513 transmitter
+ * connected to DU enabled.
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/interrupt-controller/irq.h>
+
+#define RZT2H_IRQ8	24
+
+/*
+ * RZ/T2H LCDC configuration:
+ * ----------------------------------------------------------
+ * Function        Pin     SW Setting
+ * ----------------------------------------------------------
+ * LCDC_DATG0     P11_0, SW6[3]: OFF, SW6[4]: ON, SW6[5]: OFF
+ * LCDC_DATB1     P18_0, SW8[3]: OFF, SW8[4]: ON
+ * LCDC_DATB2     P18_1, SW8[1]: OFF, SW8[2]: ON
+ * IRQ8           P22_6, SW2[1]: ON, SW2[2]: OFF
+ */
+
+&{/leds/led-4} {
+	/* P18_0 is used for DU function LCDC_DATB1. */
+	status = "disabled";
+};
+
+&{/leds/led-5} {
+	/* P18_1 is used for DU function LCDC_DATB2. */
+	status = "disabled";
+};
+
+/*
+ * Disable SDHI0 as SW2 settings for eMMC/SD card conflict with DU pin
+ * settings.
+ */
+&sdhi0 {
+	status = "disabled";
+};
+
+#include "rzt2h-n2h-evk-du-adv7513.dtsi"
+
+&adv7513 {
+	interrupts-extended = <&icu RZT2H_IRQ8 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&adv7513_irq_pins {
+	pinmux = <RZT2H_PORT_PINMUX(22, 6, 0x0)>; /* IRQ8 */
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
index 572be8aa6d7ef5..d21367cef534e2 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g077m44-rzt2h-evk.dts
@@ -69,7 +69,7 @@
 		compatible = "gpio-keys";
 
 #if (!SD1_MICRO_SD)
-		/* SW2-3: OFF */
+		/* SW2[3]: OFF */
 		key-1 {
 			interrupts-extended = <&pinctrl RZT2H_GPIO(8, 6) IRQ_TYPE_EDGE_FALLING>;
 			linux,code = <KEY_1>;
@@ -100,7 +100,7 @@
 		compatible = "gpio-leds";
 
 		led-0 {
-			/* SW8-9: ON, SW8-10: OFF */
+			/* SW8[9]: ON, SW8[10]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(23, 1) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -108,7 +108,7 @@
 		};
 
 		led-1 {
-			/* SW5-1: OFF, SW5-2: ON */
+			/* SW5[1]: OFF, SW5[2]: ON */
 			gpios = <&pinctrl RZT2H_GPIO(32, 2) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -124,7 +124,7 @@
 
 #if (!SD1_MICRO_SD)
 		led-3 {
-			/* SW2-3: OFF */
+			/* SW2[3]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(8, 5) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_RED>;
 			function = LED_FUNCTION_DEBUG;
@@ -133,7 +133,7 @@
 #endif
 
 		led-4 {
-			/* SW8-3: ON, SW8-4: OFF */
+			/* SW8[3]: ON, SW8[4]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(18, 0) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -141,7 +141,7 @@
 		};
 
 		led-5 {
-			/* SW8-1: ON, SW8-2: OFF */
+			/* SW8[1]: ON, SW8[2]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(18, 1) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_RED>;
 			function = LED_FUNCTION_DEBUG;
@@ -149,7 +149,7 @@
 		};
 
 		led-6 {
-			/* SW5-9: OFF, SW5-10: ON */
+			/* SW5[9]: OFF, SW5[10]: ON */
 			gpios = <&pinctrl RZT2H_GPIO(22, 7) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -157,7 +157,7 @@
 		};
 
 		led-7 {
-			/* SW5-7: OFF, SW5-8: ON */
+			/* SW5[7]: OFF, SW5[8]: ON */
 			gpios = <&pinctrl RZT2H_GPIO(23, 0) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -165,7 +165,7 @@
 		};
 
 		led-8 {
-			/* SW7-5: OFF, SW7-6: ON */
+			/* SW7[5]: OFF, SW7[6]: ON */
 			gpios = <&pinctrl RZT2H_GPIO(23, 5) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
index 03b976d93e1058..68c8daed557bdc 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g087.dtsi
@@ -1078,8 +1078,8 @@
 
 		cpg: clock-controller@80280000 {
 			compatible = "renesas,r9a09g087-cpg-mssr";
-			reg = <0 0x80280000 0 0x10000>,
-			      <0 0x81280000 0 0x10000>;
+			reg = <0 0x80280000 0 0x20000>,
+			      <0 0x81280000 0 0x20000>;
 			clocks = <&extal_clk>;
 			clock-names = "extal";
 			#clock-cells = <2>;
@@ -1176,6 +1176,19 @@
 			power-domains = <&cpg>;
 		};
 
+		rtc0: rtc@81009000 {
+			compatible = "renesas,r9a09g087-rtc", "renesas,r9a09g077-rtc";
+			reg = <0 0x81009000 0 0x1000>;
+			interrupts = <GIC_SPI 495 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 497 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 496 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "alarm", "timer", "pps";
+			clocks = <&cpg CPG_MOD 605>, <&cpg CPG_CORE R9A09G087_PCLKRTC>;
+			clock-names = "hclk", "xtal";
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
 		gic: interrupt-controller@83000000 {
 			compatible = "arm,gic-v3";
 			reg = <0x0 0x83000000 0 0x40000>,
@@ -1261,14 +1274,14 @@
 				     <GIC_SPI 699 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 700 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 701 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 702 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 851 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 852 IRQ_TYPE_EDGE_RISING>;
+				     <GIC_SPI 702 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "adi", "gbadi", "gcadi",
-					  "cmpai", "cmpbi", "wcmpm", "wcmpum";
+					  "cmpai", "cmpbi";
 			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKL>,
 				 <&cpg CPG_MOD 206>;
 			clock-names = "adclk", "pclk";
+			dmas = <&dmac0 0x26ba>, <&dmac1 0x26ba>, <&dmac2 0x26ba>;
+			dma-names = "rx", "rx", "rx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1283,14 +1296,14 @@
 				     <GIC_SPI 704 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 705 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 706 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 853 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 854 IRQ_TYPE_EDGE_RISING>;
+				     <GIC_SPI 707 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "adi", "gbadi", "gcadi",
-					  "cmpai", "cmpbi", "wcmpm", "wcmpum";
+					  "cmpai", "cmpbi";
 			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKL>,
 				 <&cpg CPG_MOD 207>;
 			clock-names = "adclk", "pclk";
+			dmas = <&dmac0 0x26bf>, <&dmac1 0x26bf>, <&dmac2 0x26bf>;
+			dma-names = "rx", "rx", "rx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1305,14 +1318,14 @@
 				     <GIC_SPI 709 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 710 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 711 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 712 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 855 IRQ_TYPE_EDGE_RISING>,
-				     <GIC_SPI 856 IRQ_TYPE_EDGE_RISING>;
+				     <GIC_SPI 712 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "adi", "gbadi", "gcadi",
-					  "cmpai", "cmpbi", "wcmpm", "wcmpum";
+					  "cmpai", "cmpbi";
 			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKL>,
 				 <&cpg CPG_MOD 225>;
 			clock-names = "adclk", "pclk";
+			dmas = <&dmac0 0x26c4>, <&dmac1 0x26c4>, <&dmac2 0x26c4>;
+			dma-names = "rx", "rx", "rx";
 			power-domains = <&cpg>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1367,7 +1380,7 @@
 			status = "disabled";
 		};
 
-		sdhi0: mmc@92080000  {
+		sdhi0: mmc@92080000 {
 			compatible = "renesas,sdhi-r9a09g087",
 				     "renesas,sdhi-r9a09g057";
 			reg = <0x0 0x92080000 0 0x10000>;
@@ -1406,6 +1419,52 @@
 				status = "disabled";
 			};
 		};
+
+		du: display@920c0000 {
+			compatible = "renesas,r9a09g087-du", "renesas,r9a09g077-du";
+			reg = <0 0x920c0000 0 0x10000>;
+			interrupts = <GIC_SPI 781 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKAH>,
+				 <&cpg CPG_MOD 1204>,
+				 <&cpg CPG_CORE R9A09G087_LCDC_CLKD>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+			renesas,vsps = <&vspd 0>;
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					du_out_rgb: endpoint {
+					};
+				};
+			};
+		};
+
+		fcpvd: fcp@920d0000 {
+			compatible = "renesas,r9a09g087-fcpvd", "renesas,fcpv";
+			reg = <0 0x920d0000 0 0x10000>;
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKAH>,
+				 <&cpg CPG_MOD 1204>,
+				 <&cpg CPG_CORE R9A09G087_LCDC_CLKD>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+		};
+
+		vspd: vsp@920e0000 {
+			compatible = "renesas,r9a09g087-vsp2", "renesas,r9a07g044-vsp2";
+			reg = <0 0x920e0000 0 0x8000>;
+			interrupts = <GIC_SPI 780 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_CORE R9A09G087_CLK_PCLKAH>,
+				 <&cpg CPG_MOD 1204>,
+				 <&cpg CPG_CORE R9A09G087_LCDC_CLKD>;
+			clock-names = "aclk", "pclk", "vclk";
+			power-domains = <&cpg>;
+			renesas,fcp = <&fcpvd>;
+		};
 	};
 
 	stmmac_axi_setup: stmmac-axi-config {
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087m44-evk-cn20-lcdc.dtso b/arch/arm64/boot/dts/renesas/r9a09g087m44-evk-cn20-lcdc.dtso
new file mode 100644
index 00000000000000..9c9e4fc80363e6
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r9a09g087m44-evk-cn20-lcdc.dtso
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DT overlay for the RZ/N2H EVK with ADV7513 transmitter
+ * connected to DU enabled.
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/interrupt-controller/irq.h>
+
+#define RZN2H_IRQ5	21
+
+/*
+ * RZ/N2H LCDC configuration:
+ * ----------------------------------------------------------
+ * Function        Pin     SW Setting
+ * ----------------------------------------------------------
+ * LCDC_DATG0     P11_0, DSW12[3]: ON, DSW12[4]: OFF
+ * LCDC_DATG3     P14_3, DSW18[5]: OFF, DSW18[6]: ON, DSW19[3]: OFF, DSW19[4]: ON
+ * LCDC_DATG6     P14_6, DSW15[8]: ON, DSW15[9]: OFF, DSW15[10]: OFF
+ * LCDC_DATB2     P18_1, DSW18[9]: OFF, DSW18[10]: ON
+ * I2C_SCL1       P03_3, DSW7[1]: ON, DSW7[2]: OFF
+ * I2C_SDA1       P03_4, DSW7[3]: ON, DSW7[4]: OFF
+ * ------------------------------------------------
+ */
+
+&{/keys/key-1} {
+	/* P18_2 is used for DU function LCDC_DATB3. */
+	status = "disabled";
+};
+
+&{/leds/led-4} {
+	/* P18_1 is used for DU function LCDC_DATB2. */
+	status = "disabled";
+};
+
+&{/leds/led-7} {
+	/* P14_3 is used for DU function LCDC_DATG3. */
+	status = "disabled";
+};
+
+&{/leds/led-8} {
+	/* P14_6 is used for DU function LCDC_DATG6. */
+	status = "disabled";
+};
+
+&i2c0 {
+	/* P14_6 is used for DU function LCDC_DATG6. */
+	status = "disabled";
+};
+
+#include "rzt2h-n2h-evk-du-adv7513.dtsi"
+
+&adv7513 {
+	interrupts-extended = <&icu RZN2H_IRQ5 IRQ_TYPE_LEVEL_LOW>;
+};
+
+&adv7513_irq_pins {
+	pinmux = <RZT2H_PORT_PINMUX(14, 0, 0x0)>; /* IRQ5 */
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
index 4e57d4fe195ca0..9d54e765095771 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g087m44-rzn2h-evk.dts
@@ -120,7 +120,7 @@
 		compatible = "gpio-leds";
 
 		led-3 {
-			/* DSW18-7: ON, DSW18-8: OFF */
+			/* DSW18[7]: ON, DSW18[8]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(31, 6) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -128,7 +128,7 @@
 		};
 
 		led-4 {
-			/* DSW18-9: ON, DSW18-10: OFF */
+			/* DSW18[9]: ON, DSW18[10]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(18, 1) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_RED>;
 			function = LED_FUNCTION_DEBUG;
@@ -136,7 +136,7 @@
 		};
 
 		led-5 {
-			/* DSW18-1: ON, DSW18-2: OFF */
+			/* DSW18[1]: ON, DSW18[2]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(22, 7) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -144,7 +144,7 @@
 		};
 
 		led-6 {
-			/* DSW18-3: ON, DSW18-4: OFF */
+			/* DSW18[3]: ON, DSW18[4]: OFF */
 			gpios = <&pinctrl RZT2H_GPIO(23, 0) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
@@ -153,8 +153,8 @@
 
 		led-7 {
 			/*
-			 * DSW18-5: ON, DSW18-6: OFF
-			 * DSW19-3: OFF, DSW19-4: ON
+			 * DSW18[5]: ON, DSW18[6]: OFF
+			 * DSW19[3]: OFF, DSW19[4]: ON
 			 */
 			gpios = <&pinctrl RZT2H_GPIO(14, 3) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
@@ -162,36 +162,40 @@
 			function-enumerator = <8>;
 		};
 
-#if LED8
 		led-8 {
+#if LED8
 			/*
 			 * USER_LED0
-			 * DSW15-8: OFF, DSW15-9: OFF, DSW15-10: ON
+			 * DSW15[8]: OFF, DSW15[9]: OFF, DSW15[10]: ON
 			 */
 			gpios = <&pinctrl RZT2H_GPIO(14, 6) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
 			function-enumerator = <0>;
-		};
+#else
+			status = "disabled";
 #endif
+		};
 
-#if LED9
 		led-9 {
+#if LED9
 			/*
 			 * USER_LED1
-			 * DSW15-5: OFF, DSW15-6: ON
+			 * DSW15[5]: OFF, DSW15[6]: ON
 			 */
 			gpios = <&pinctrl RZT2H_GPIO(14, 7) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_GREEN>;
 			function = LED_FUNCTION_DEBUG;
 			function-enumerator = <1>;
-		};
+#else
+			status = "disabled";
 #endif
+		};
 
 		led-10 {
 			/*
 			 * USER_LED2
-			 * DSW17-3: OFF, DSW17-4: ON
+			 * DSW17[3]: OFF, DSW17[4]: ON
 			 */
 			gpios = <&pinctrl RZT2H_GPIO(2, 7) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_YELLOW>;
@@ -202,7 +206,7 @@
 		led-11 {
 			/*
 			 * USER_LED3
-			 * DSW17-1: OFF, DSW17-2: ON
+			 * DSW17[1]: OFF, DSW17[2]: ON
 			 */
 			gpios = <&pinctrl RZT2H_GPIO(3, 0) GPIO_ACTIVE_HIGH>;
 			color = <LED_COLOR_ID_RED>;
diff --git a/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi b/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi
index 696a933af808a8..234ca287ed2890 100644
--- a/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi
+++ b/arch/arm64/boot/dts/renesas/renesas-smarc2.dtsi
@@ -44,16 +44,9 @@
 		stdout-path = "serial3:115200n8";
 	};
 
-	can_transceiver0: can-phy0 {
-		compatible = "ti,tcan1042";
-		#phy-cells = <0>;
-		max-bitrate = <8000000>;
-		status = "disabled";
-	};
-
-	can_transceiver1: can-phy1 {
-		compatible = "ti,tcan1042";
-		#phy-cells = <0>;
+	can_transceiver: can-phy {
+		compatible = "ti,tcan1046", "nxp,tja1048";
+		#phy-cells = <1>;
 		max-bitrate = <8000000>;
 		status = "disabled";
 	};
@@ -91,11 +84,36 @@
 	status = "okay";
 };
 
+&ehci0 {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&ehci1 {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&hsusb0 {
+	dr_mode = "otg";
+	status = "okay";
+};
+
 &i2c0 {
 	status = "okay";
 	clock-frequency = <400000>;
 };
 
+&ohci0 {
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&ohci1 {
+	dr_mode = "host";
+	status = "okay";
+};
+
 &pcie {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-common.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-common.dtsi
new file mode 100644
index 00000000000000..9f46c73ba3b5ac
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-common.dtsi
@@ -0,0 +1,560 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+/ {
+	/* power for M.2 B-Key connector (J6) */
+	regulator-m2-b {
+		compatible = "regulator-fixed";
+		regulator-name = "m2-b";
+		gpios = <&tca6416_u20 5 GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		enable-active-high;
+	};
+
+	/* power for M.2 M-Key connector (J4) */
+	regulator-m2-m {
+		compatible = "regulator-fixed";
+		regulator-name = "m2-m";
+		gpios = <&tca6416_u20 6 GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		enable-active-high;
+	};
+
+	/* power for USB-A J27 behind USB Hub Port 3 */
+	regulator-vbus-2 {
+		compatible = "regulator-fixed";
+		regulator-name = "vbus2";
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		gpios = <&tca6416_u20 12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	/* power for USB-A J27 behind USB Hub Port 4 */
+	regulator-vbus-3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vbus3";
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		gpios = <&tca6416_u20 13 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	aliases {
+		i2c3 = &i2c_exp;
+		i2c4 = &i2c_csi;
+		i2c5 = &i2c_dsi;
+		i2c6 = &i2c_lvds;
+		rtc0 = &carrier_rtc;
+		rtc1 = &pmic;
+		serial3 = &scif3;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		wakeup-event {
+			interrupts-extended = <&tca6416_u21 11 IRQ_TYPE_EDGE_FALLING>;
+			label = "m2-m-wakeup";
+			wakeup-source;
+			linux,code = <KEY_WAKEUP>;
+		};
+	};
+
+	can_mux: mux-controller-1 {
+		compatible = "gpio-mux";
+		/* default J9-55/57/59/61 to on-board transceivers */
+		idle-state = <0>;
+		#mux-control-cells = <0>;
+		/*
+		 * Mux routes CAN bus signals between SoM connector pins,
+		 * expansion connector (J22) and on-board transceivers using
+		 * two GPIO:
+		 * - IO3: 0 = on-board transceivers, 1 = expansion connector
+		 * - IO4: 0 = J9-55/57/59/61, 1 = J7-12/16 & J9-54/56
+		 */
+		mux-gpios = <&tca6416_u20 3 GPIO_ACTIVE_HIGH>,
+			    <&tca6416_u20 4 GPIO_ACTIVE_HIGH>;
+	};
+
+	spi_mux: mux-controller-2 {
+		compatible = "gpio-mux";
+		/* default on-board */
+		idle-state = <0>;
+		/*
+		 * Mux switches spi bus between on-board tpm
+		 * and expansion connector (J22).
+		 */
+		mux-gpios = <&tca6416_u21 0 GPIO_ACTIVE_HIGH>;
+		#mux-control-cells = <0>;
+	};
+
+	scif1_scif3_b2b_mux: mux-controller-3 {
+		compatible = "gpio-mux";
+		/* default on-board */
+		idle-state = <0>;
+		#mux-control-cells = <0>;
+		/*
+		 * Mux switches both scif1 and scif3 tx/rx between expansion
+		 * connector (J22) and on-board rs232/rs485 transceivers
+		 * using one GPIO: 0 = on-board, 1 = connector.
+		 */
+		mux-gpios = <&tca6416_u20 0 GPIO_ACTIVE_HIGH>;
+	};
+
+	scif1_rs_232_485_mux: mux-controller-4 {
+		compatible = "gpio-mux";
+		/* default rs232 */
+		idle-state = <0>;
+		#mux-control-cells = <0>;
+		/*
+		 * Mux switches scif1 tx/rx between rs232 and rs485
+		 * transceivers. using one GPIO: 0 = rs232, 1 = rs485.
+		 */
+		mux-gpios = <&tca6416_u20 1 GPIO_ACTIVE_HIGH>;
+	};
+
+	scif3_rs_232_485_mux: mux-controller-5 {
+		compatible = "gpio-mux";
+		/* default rs232 */
+		idle-state = <0>;
+		#mux-control-cells = <0>;
+		/*
+		 * Mux switches scif3 tx/rx between rs232 and rs485
+		 * transceivers. using one GPIO: 0 = rs232, 1 = rs485.
+		 */
+		mux-gpios = <&tca6416_u20 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	v_1_2: regulator-1v2 {
+		compatible = "regulator-fixed";
+		regulator-name = "1v2";
+		regulator-max-microvolt = <1200000>;
+		regulator-min-microvolt = <1200000>;
+	};
+
+	v_3_3: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "3v3";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+	};
+
+	reg_dsi_panel: regulator-dsi-panel {
+		compatible = "regulator-fixed";
+		regulator-name = "dsi-panel";
+		gpios = <&tca6416_u20 15 GPIO_ACTIVE_HIGH>;
+		regulator-max-microvolt = <11200000>;
+		regulator-min-microvolt = <11200000>;
+		enable-active-high;
+	};
+
+	vmmc: regulator-mmc {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmc";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		startup-delay-us = <250>;
+		vin-supply = <&v_3_3>;
+		gpios = <&pinctrl RZG2L_GPIO(4, 1) GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	/* power for USB-A J5003 */
+	vbus1: regulator-vbus-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vbus1";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		gpios = <&tca6416_u20 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	rfkill-m2-b-gnss {
+		compatible = "rfkill-gpio";
+		/* rfkill-gpio inverts internally */
+		shutdown-gpios = <&tca6416_u20 10 GPIO_ACTIVE_LOW>;
+		label = "m2-b gnss";
+		radio-type = "gps";
+	};
+
+	rfkill-m2-b-wwan {
+		compatible = "rfkill-gpio";
+		/* rfkill-gpio inverts internally */
+		shutdown-gpios = <&tca6416_u20 9 GPIO_ACTIVE_HIGH>;
+		label = "m2-b radio";
+		radio-type = "wwan";
+	};
+};
+
+&ehci1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	hub_2_0: hub@1 {
+		compatible = "usb4b4,6502", "usb4b4,6506";
+		reg = <1>;
+		reset-gpios = <&tca6416_u20 11 GPIO_ACTIVE_LOW>;
+		vdd2-supply = <&v_3_3>;
+		vdd-supply = <&v_1_2>;
+	};
+};
+
+&i2c0 {
+	/* highest i2c clock supported by all peripherals is 400kHz */
+
+	tca6416_u20: gpio@20 {
+		compatible = "ti,tcal6416";
+		reg = <0x20>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names = "TCA_INT/EXT_UART", "TCA_UARTA_232/485",
+				  "TCA_UARTB_232/485", "TCA_INT/EXT_CAN",
+				  "TCA_NXP/REN", "TCA_M.2B_3V3_EN",
+				  "TCA_M.2M_3V3_EN", "TCA_M.2M_RESET#",
+				  "TCA_M.2B_RESET#", "TCA_M.2B_W_DIS#",
+				  "TCA_M.2B_GPS_EN#", "TCA_USB-HUB_RST#",
+				  "TCA_USB_HUB3_PWR_EN", "TCA_USB_HUB4_PWR_EN",
+				  "TCA_USB1_PWR_EN", "TCA_VIDEO_PWR_EN";
+
+		m2-b-reset-hog {
+			gpios = <8 GPIO_ACTIVE_LOW>;
+			gpio-hog;
+			line-name = "m2-b-reset";
+			output-low;
+		};
+
+		m2-m-reset-hog {
+			gpios = <7 GPIO_ACTIVE_LOW>;
+			gpio-hog;
+			line-name = "m2-m-reset";
+			/*
+			 * M.2 Key-M connector only supports PCI,
+			 * but RZ/G2L(C) has no pci controller.
+			 * Keep any card in reset.
+			 */
+			output-high;
+		};
+	};
+
+	tca6416_u21: gpio@21 {
+		compatible = "ti,tcal6416";
+		reg = <0x21>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		#gpio-cells = <2>;
+		gpio-controller;
+		gpio-line-names = "TCA_SPI_TPM/EXT", "TCA_TPM_RST#",
+				  "TCA_I2C_RST", "TCA_RS232_SHTD#",
+				  "TCA_LCD_I2C_RST", "TCA_DIG_OUT1",
+				  "TCA_bDIG_IN1", "TCA_SENS_INT",
+				  "TCA_ALERT#", "TCA_TPM_PIRQ#",
+				  "TCA_RTC_INT", "TCA_M.2M_WAKW_ON_LAN",
+				  "TCA_M.2M_CLKREQ#", "TCA_LVDS_INT#",
+				  "", "TCA_POE_AT";
+		/*
+		 * Level interrupts are typical for gpio expander but not supported on rz/g2l
+		 * gpios. However since TCAL6416 latches interrupts and clears only on reading
+		 * port register, falling edge trigger doesn't lose events.
+		 */
+		interrupts-extended = <&pinctrl RZG2L_GPIO(4, 0) IRQ_TYPE_EDGE_FALLING>;
+
+		lcd-i2c-reset-hog {
+			gpios = <4 (GPIO_ACTIVE_LOW|GPIO_PULL_UP|GPIO_OPEN_DRAIN)>;
+			line-name = "lcd-i2c-reset";
+			output-low;
+			/*
+			 * reset shared between U37 and U48, to be
+			 * supported once gpio-pca953x switches to
+			 * reset framework.
+			 */
+			gpio-hog;
+		};
+
+		lvds-irq-hog {
+			gpios = <13 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
+			gpio-hog;
+			input;
+			line-name = "lvds-irq";
+		};
+
+		m2-m-clkreq-hog {
+			gpios = <12 GPIO_ACTIVE_LOW>;
+			gpio-hog;
+			input;
+			line-name = "m2-m-clkreq";
+		};
+
+		rs232_shutdown: rs232-shutdown-hog {
+			gpios = <3 GPIO_ACTIVE_LOW>;
+			gpio-hog;
+			line-name = "rs232-shutdown";
+			output-low;
+		};
+
+		sensor-irq-hog {
+			gpios = <7 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
+			gpio-hog;
+			input;
+			line-name = "sensor-irq";
+		};
+
+		tpm-irq-hog {
+			gpios = <9 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
+			gpio-hog;
+			input;
+			line-name = "tpm-irq";
+		};
+	};
+
+	led-controller@30 {
+		compatible = "ti,lp5562";
+		reg = <0x30>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		/* use internal clock, could use external generated by rtc */
+		clock-mode = /bits/ 8 <1>;
+
+		multi-led@0 {
+			reg = <0x0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			color = <LED_COLOR_ID_RGB>;
+			label = "D7";
+
+			led@0 {
+				reg = <0x0>;
+				color = <LED_COLOR_ID_RED>;
+				led-cur = /bits/ 8 <0x32>;
+				max-cur = /bits/ 8 <0x64>;
+			};
+
+			led@1 {
+				reg = <0x1>;
+				color = <LED_COLOR_ID_GREEN>;
+				led-cur = /bits/ 8 <0x19>;
+				max-cur = /bits/ 8 <0x32>;
+			};
+
+			led@2 {
+				reg = <0x2>;
+				color = <LED_COLOR_ID_BLUE>;
+				led-cur = /bits/ 8 <0x19>;
+				max-cur = /bits/ 8 <0x32>;
+			};
+		};
+
+		led@3 {
+			reg = <0x3>;
+			chan-name = "D8";
+			color = <LED_COLOR_ID_GREEN>;
+			label = "D8";
+			led-cur = /bits/ 8 <0x19>;
+			max-cur = /bits/ 8 <0x64>;
+		};
+	};
+
+	light-sensor@44 {
+		compatible = "isil,isl29023";
+		reg = <0x44>;
+		/* IRQ shared between accelerometer, light-sensor and Tamper input (J5007) */
+		interrupts-extended = <&tca6416_u21 7 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	accelerometer@53 {
+		compatible = "adi,adxl345";
+		reg = <0x53>;
+		interrupts-extended = <&tca6416_u21 7 IRQ_TYPE_LEVEL_LOW>;
+		/* IRQ shared between accelerometer, light-sensor and Tamper input (J5007) */
+		interrupt-names = "INT1";
+	};
+
+	carrier_eeprom: eeprom@57 {
+		compatible = "atmel,24c02";
+		reg = <0x57>;
+		pagesize = <8>;
+	};
+
+	carrier_rtc: rtc@69 {
+		compatible = "abracon,ab1805";
+		reg = <0x69>;
+		/*
+		 * AM1805 RTC used on this board has only nTIRQ pins wired,
+		 * which is for countdown timer irqs only.
+		 * Driver does not support this, disable for now.
+		 *
+		 * interrupts-extended = <&tca6416_u21 10 IRQ_TYPE_LEVEL_LOW>;
+		 */
+		abracon,tc-diode = "schottky";
+		abracon,tc-resistor = <3>;
+	};
+};
+
+&i2c1 {
+	/* highest i2c clock supported by all peripherals is 400kHz */
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9546";
+		reg = <0x70>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		/*
+		 * This reset is open drain with HW 10k pull-up resistor,
+		 * but reset core does not support GPIO_OPEN_DRAIN flag.
+		 * The pull-up and GPIO voltages match, push-pull is safe.
+		 */
+		reset-gpios = <&tca6416_u21 2 GPIO_ACTIVE_LOW>;
+
+		/* channel 0 routed to expansion connector (J22) */
+		i2c_exp: i2c@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		/* channel 1 routed to mipi-csi connector (J23) */
+		i2c_csi: i2c@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		/* channel 2 routed to mipi-dsi connector (J25) */
+		i2c_dsi: i2c@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			tca6408_u48: gpio@21 {
+				compatible = "ti,tca6408";
+				reg = <0x21>;
+				#gpio-cells = <2>;
+				gpio-line-names = "CAM_RST#", "DSI_RESET",
+						  "DSI_STBYB", "DSI_PWM_BL",
+						  "DSI_L/R", "DSI_U/D",
+						  "DSI_CTP_/RST", "CAM_TRIG";
+				/*
+				 * reset shared between U37 and U48, to be
+				 * supported once gpio-pca953x switches to
+				 * reset framework.
+				 *
+				 * reset-gpios = <&tca6416_u21 4
+				 *                (GPIO_ACTIVE_LOW|GPIO_PULL_UP|GPIO_OPEN_DRAIN)>;
+				 */
+				gpio-controller;
+			};
+		};
+
+		/* channel 3 routed to lvds connector (J24) */
+		i2c_lvds: i2c@3 {
+			reg = <3>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			tca6408_u37: gpio@20 {
+				compatible = "ti,tca6408";
+				reg = <0x20>;
+				#gpio-cells = <2>;
+				gpio-line-names = "SELB", "LVDS_RESET",
+						  "LVDS_STBYB", "LVDS_PWM_BL",
+						  "LVDS_L/R", "LVDS_U/D",
+						  "LVDS_CTP_/RST", "";
+				/*
+				 * reset shared between U37 and U48, to be
+				 * supported once gpio-pca953x switches to
+				 * reset framework.
+				 *
+				 * reset-gpios = <&tca6416_u21 4
+				 *                (GPIO_ACTIVE_LOW|GPIO_PULL_UP|GPIO_OPEN_DRAIN)>;
+				 */
+				gpio-controller;
+			};
+		};
+	};
+};
+
+&ohci0 {
+	dr_mode = "host";
+};
+
+&phy0 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* LED_0 pin */
+		led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "keep";
+			function = LED_FUNCTION_LAN;
+		};
+	};
+};
+
+&pinctrl {
+	/* UARTA */
+	scif1_pins: scif1 {
+		pinmux = <RZG2L_PORT_PINMUX(40, 1, 1)>, /* SCIF1_RXD */
+			 <RZG2L_PORT_PINMUX(40, 0, 1)>; /* SCIF1_TXD */
+	};
+
+	/* UARTB */
+	scif3_pins: scif3 {
+		pinmux = <RZG2L_PORT_PINMUX(0, 1, 5)>, /* SCIF3_RXD */
+			 <RZG2L_PORT_PINMUX(0, 0, 5)>; /* SCIF3_TXD */
+	};
+};
+
+&scif1 {
+	pinctrl-0 = <&scif1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&scif3 {
+	pinctrl-0 = <&scif3_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&spi1 {
+	/* native cs does not support cs persistence required for tpm */
+	cs-gpios = <&pinctrl RZG2L_GPIO(44, 3) GPIO_ACTIVE_LOW>;
+	num-cs = <1>;
+	pinctrl-0 = <&spi1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	spi1_muxed: spi@0 {
+		compatible = "spi-mux";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		mux-controls = <&spi_mux>;
+		/* mux bandwidth is 2GHz, soc max. spi clock is P0/2 = 50MHz */
+		spi-max-frequency = <50000000>;
+
+		tpm@0 {
+			compatible = "infineon,slb9670", "tcg,tpm_tis-spi";
+			reg = <0>;
+			interrupts-extended = <&tca6416_u21 9 IRQ_TYPE_LEVEL_LOW>;
+			reset-gpios = <&tca6416_u21 1 (GPIO_ACTIVE_LOW | GPIO_OPEN_DRAIN)>;
+			spi-max-frequency = <43000000>;
+		};
+	};
+};
+
+&usb2_phy0 {
+	vbus-supply = <&vbus1>;
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-microsd.dtso b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-microsd.dtso
new file mode 100644
index 00000000000000..0f92a9d2ccd98f
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-microsd.dtso
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Overlay for the RZ/G2L(C) Solidrun SOM SD
+ *
+ * Copyright (C) 2024 SolidRun Ltd.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+&sdhi0 {
+	bus-width = <4>;
+	full-pwr-cycle;
+	mux-states = <&sdhi0_mux 1>;
+	pinctrl-0 = <&sdhi0_4bit_pins>, <&sdhi0_cd_pins>;
+	pinctrl-1 = <&sdhi0_4bit_uhs_pins>, <&sdhi0_cd_pins>;
+	pinctrl-names = "default", "state_uhs";
+	sd-uhs-sdr104;
+	sd-uhs-sdr50;
+	vmmc-supply = <&vmmc>;
+	vqmmc-supply = <&reg_pmic_ldo1>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtso b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtso
new file mode 100644
index 00000000000000..f8584c473fafed
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-panel-dsi-WJ70N3TYJHMNG0.dtso
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ *
+ * Overlay for enabling HummingBoard IIoT MIPI-DSI connector
+ * with Winstar WJ70N3TYJHMNG0 panel.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&{/} {
+	dsi_backlight: dsi-backlight {
+		compatible = "gpio-backlight";
+		gpios = <&tca6408_u48 3 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&i2c_dsi {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	touchscreen@41 {
+		compatible = "ilitek,ili2130";
+		reg = <0x41>;
+		/*
+		 * Touchscreen fires short active-low pulses which level trigger
+		 * on i2c gpio expander can miss. Trigger on falling edge instead.
+		 */
+		interrupts-extended = <&tca6416_u21 13 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&tca6408_u48 6 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&dsi {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	panel@0 {
+		/* This is a Winstar panel, but the ronbo panel uses same controls. */
+		compatible = "ronbo,rb070d30";
+		reg = <0>;
+		/* reset is active-low but driver inverts it internally */
+		reset-gpios = <&tca6408_u48 1 GPIO_ACTIVE_HIGH>;
+		backlight = <&dsi_backlight>;
+		power-gpios = <&tca6408_u48 2 GPIO_ACTIVE_HIGH>;
+		shlr-gpios = <&tca6408_u48 4 GPIO_ACTIVE_LOW>;
+		updn-gpios = <&tca6408_u48 5 GPIO_ACTIVE_HIGH>;
+		vcc-lcd-supply = <&reg_dsi_panel>;
+
+		port {
+			dsi_in_panel: endpoint {
+				remote-endpoint = <&mipi_dsi_out>;
+			};
+		};
+	};
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			mipi_dsi_out: endpoint {
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&dsi_in_panel>;
+			};
+		};
+	};
+};
+
+&du {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-a.dtso b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-a.dtso
new file mode 100644
index 00000000000000..4bcb22d518f053
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-a.dtso
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ *
+ * Overlay for enabling HummingBoard IIoT on-board RS485 Port A on connector J5004.
+ *
+ * Because Renesas uart driver does not support rs485,
+ * users must manually toggle P41_1 between RX & TX.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&scif1_rs_232_485_mux {
+	/* select rs485 */
+	idle-state = <1>;
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-b.dtso b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-b.dtso
new file mode 100644
index 00000000000000..6f460b3e0b256f
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot-rs485-b.dtso
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ *
+ * Overlay for enabling HummingBoard IIoT on-board RS485 Port B on connector J5004.
+ *
+ * Because Renesas uart driver does not support rs485,
+ * users must manually toggle P41_0 between RX & TX.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&scif3_rs_232_485_mux {
+	/* select rs485 */
+	idle-state = <1>;
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot.dtsi
new file mode 100644
index 00000000000000..22f066079e69ab
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-iiot.dtsi
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+#include "rzg2l-hummingboard-iiot-common.dtsi"
+
+&canfd {
+	pinctrl-names = "default";
+	pinctrl-0 = <&can0_pins>, <&can1_pins>;
+	status = "okay";
+
+	channel0 {
+		status = "okay";
+	};
+
+	channel1 {
+		status = "okay";
+	};
+};
+
+&phy1 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* LED_0 pin */
+		led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			default-state = "keep";
+		};
+	};
+};
+
+&pinctrl {
+	/* CANA */
+	can0_pins: can0 {
+		pinmux = <RZG2L_PORT_PINMUX(10, 1, 2)>, /* CAN0_TX */
+			 <RZG2L_PORT_PINMUX(11, 0, 2)>; /* CAN0_RX */
+	};
+
+	/* CANB */
+	can1_pins: can1 {
+		pinmux = <RZG2L_PORT_PINMUX(12, 1, 2)>, /* CAN1_TX */
+			 <RZG2L_PORT_PINMUX(13, 0, 2)>; /* CAN1_RX */
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-common.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-common.dtsi
new file mode 100644
index 00000000000000..3fd49333ff1953
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-common.dtsi
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+#include <dt-bindings/leds/common.h>
+
+/ {
+	aliases {
+		rtc0 = &carrier_rtc;
+		rtc1 = &pmic;
+	};
+
+	v_1_2: regulator-1v2 {
+		compatible = "regulator-fixed";
+		regulator-name = "1v2";
+		regulator-max-microvolt = <1200000>;
+		regulator-min-microvolt = <1200000>;
+		vin-supply = <&reg_pmic_buck4>;
+	};
+
+	vmmc: regulator-mmc {
+		compatible = "regulator-fixed";
+		regulator-name = "vmmc";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		startup-delay-us = <250>;
+		vin-supply = <&reg_pmic_buck4>;
+	};
+
+	vmpcie: regulator-mpcie {
+		/* supplies mpcie and m2 connectors */
+		regulator-name = "vmpcie";
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		enable-active-high;
+	};
+};
+
+&ehci1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	hub_2_0: hub@1 {
+		compatible = "usb4b4,6502", "usb4b4,6506";
+		reg = <1>;
+		vdd2-supply = <&reg_pmic_buck4>;
+		vdd-supply = <&v_1_2>;
+	};
+};
+
+&i2c0 {
+	carrier_eeprom: eeprom@57 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x57>;
+		pagesize = <16>;
+	};
+
+	carrier_rtc: rtc@69 {
+		compatible = "abracon,ab1805";
+		reg = <0x69>;
+		abracon,tc-diode = "schottky";
+		abracon,tc-resistor = <3>;
+	};
+};
+
+&ohci0 {
+	dr_mode = "host";
+};
+
+&ohci1 {
+	dr_mode = "host";
+};
+
+&phy0 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_ORANGE>;
+			default-state = "keep";
+			function = LED_FUNCTION_LAN;
+		};
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "keep";
+			function = LED_FUNCTION_LAN;
+		};
+	};
+};
+
+/* mikrobus uart */
+&scif1 {
+	pinctrl-0 = <&mikro_uart_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+/* mikrobus spi */
+&spi1 {
+	num-cs = <1>;
+	pinctrl-0 = <&spi1_pins>, <&spi1_cs_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usb2_phy0 {
+	pinctrl-0 = <&usb0_vbus_pins>;
+	pinctrl-names = "default";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-micro-hdmi.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-micro-hdmi.dtsi
new file mode 100644
index 00000000000000..80e43510142ab5
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-pulse-micro-hdmi.dtsi
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/ {
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+		type = "d";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&adv7535_out>;
+			};
+		};
+	};
+
+	vmipi: regulator-mipi-1-8 {
+		compatible = "regulator-fixed";
+		regulator-name = "vmipi";
+		regulator-always-on;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		vin-supply = <&reg_pmic_buck4>;
+	};
+};
+
+&dsi {
+	status = "okay";
+
+	ports {
+		port@1 {
+			mipi_dsi_out: endpoint {
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&adv7535_from_dsim>;
+			};
+		};
+	};
+};
+
+&du {
+	status = "okay";
+};
+
+&i2c0 {
+	hdmi: hdmi@3d {
+		compatible = "adi,adv7535";
+		reg = <0x3d>, <0x3f>, <0x3c>, <0x38>;
+		reg-names = "main", "edid", "cec", "packet";
+		a2vdd-supply = <&vmipi>;
+		avdd-supply = <&vmipi>;
+		dvdd-supply = <&vmipi>;
+		pvdd-supply = <&vmipi>;
+		v3p3-supply = <&reg_pmic_buck4>;
+		adi,dsi-lanes = <4>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				adv7535_from_dsim: endpoint {
+					remote-endpoint = <&mipi_dsi_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				adv7535_out: endpoint {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-ripple.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-ripple.dtsi
new file mode 100644
index 00000000000000..aeb92630f4c6b4
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-hummingboard-ripple.dtsi
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/ {
+	aliases {
+		/* this board does not use second phy / ethernet on SoM */
+		/delete-property/ ethernet1;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			gpios = <&pinctrl RZG2L_GPIO(0, 1) GPIO_ACTIVE_LOW>;
+			label = "D30";
+		};
+
+		led-1 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			gpios = <&pinctrl RZG2L_GPIO(47, 2) GPIO_ACTIVE_LOW>;
+			label = "D31";
+		};
+
+		led-2 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			gpios = <&pinctrl RZG2L_GPIO(47, 1) GPIO_ACTIVE_LOW>;
+			label = "D32";
+		};
+
+		led-3 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			gpios = <&pinctrl RZG2L_GPIO(48, 2) GPIO_ACTIVE_LOW>;
+			label = "D33";
+		};
+
+		led-4 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			gpios = <&pinctrl RZG2L_GPIO(7, 2) GPIO_ACTIVE_LOW>;
+			label = "D34";
+		};
+	};
+
+	rfkill-mpcie {
+		/* rfkill-gpio inverts internally */
+		shutdown-gpios = <&pinctrl RZG2L_GPIO(14, 1) GPIO_ACTIVE_HIGH>;
+		label = "mpcie radio";
+		/*
+		 * The mpcie connector only has USB,
+		 * therefore this rfkill is for cellular radios only.
+		 */
+		radio-type = "wwan";
+		compatible = "rfkill-gpio";
+	};
+};
+
+&eth1 {
+	/* this board does not use second phy / ethernet on SoM */
+	status = "disabled";
+};
+
+&gpt {
+	/* mikrobus header pwm pin is pwm channel 6 */
+	pinctrl-0 = <&mikro_pwm_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdmi {
+	interrupts-extended = <&pinctrl RZG2L_GPIO(9, 1) IRQ_TYPE_EDGE_FALLING>;
+	pd-gpios = <&pinctrl RZG2L_GPIO(3, 0) GPIO_ACTIVE_LOW>;
+};
+
+&hub_2_0 {
+	reset-gpios = <&pinctrl RZG2L_GPIO(39, 1) GPIO_ACTIVE_LOW>;
+};
+
+&pinctrl {
+	mpcie_rst_hog: mpcie-reset-hog {
+		gpios = <RZG2L_GPIO(42, 2) GPIO_ACTIVE_LOW>;
+		gpio-hog;
+		line-name = "mpcie-reset";
+		output-low;
+	};
+
+	mikro_pwm_pins: pinctrl-mikro-pwm-grp {
+		pinmux = <RZG2L_PORT_PINMUX(41, 0, 4)>; /* GTIOC3A */
+	};
+
+	mikro_uart_pins: pinctrl-mikro-uart-grp {
+		pinmux = <RZG2L_PORT_PINMUX(40, 1, 1)>, /* SCIF1_RXD */
+			 <RZG2L_PORT_PINMUX(40, 0, 1)>; /* SCIF1_TXD */
+	};
+
+	usb0_vbus_pins: usb0-vbus {
+		pinmux = <RZG2L_PORT_PINMUX(4, 0, 1)>; /* USB0_VBUSEN */
+	};
+
+	usb1_vbus_pins: usb1-vbus {
+		pinmux = <RZG2L_PORT_PINMUX(42, 0, 1)>; /* USB1_VBUSEN */
+	};
+};
+
+&usb2_phy1 {
+	pinctrl-0 = <&usb1_vbus_pins>;
+	pinctrl-names = "default";
+};
+
+&vmmc {
+	gpios = <&pinctrl RZG2L_GPIO(4, 1) GPIO_ACTIVE_LOW>;
+};
+
+&vmpcie {
+	gpios = <&pinctrl RZG2L_GPIO(17, 0) GPIO_ACTIVE_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-smarc-pinfunction.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-smarc-pinfunction.dtsi
index 2616dbde4dd597..ee141993d93ada 100644
--- a/arch/arm64/boot/dts/renesas/rzg2l-smarc-pinfunction.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2l-smarc-pinfunction.dtsi
@@ -98,38 +98,38 @@
 	};
 
 	sdhi1_pins: sd1 {
-		sd1_data {
+		sd1-data {
 			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
 			power-source = <3300>;
 		};
 
-		sd1_ctrl {
+		sd1-ctrl {
 			pins = "SD1_CLK", "SD1_CMD";
 			power-source = <3300>;
 		};
 
-		sd1_mux {
+		sd1-mux {
 			pinmux = <RZG2L_PORT_PINMUX(19, 0, 1)>; /* SD1_CD */
 		};
 	};
 
-	sdhi1_pins_uhs: sd1_uhs {
-		sd1_data_uhs {
+	sdhi1_pins_uhs: sd1-uhs {
+		sd1-data-uhs {
 			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
 			power-source = <1800>;
 		};
 
-		sd1_ctrl_uhs {
+		sd1-ctrl-uhs {
 			pins = "SD1_CLK", "SD1_CMD";
 			power-source = <1800>;
 		};
 
-		sd1_mux_uhs {
+		sd1-mux-uhs {
 			pinmux = <RZG2L_PORT_PINMUX(19, 0, 1)>; /* SD1_CD */
 		};
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		pins = "AUDIO_CLK1", "AUDIO_CLK2";
 		input-enable;
 	};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
index 7eccdaffb221f4..b5f20ff1710968 100644
--- a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
@@ -269,51 +269,51 @@
 	};
 
 	sdhi0_emmc_pins: sd0emmc {
-		sd0_emmc_data {
+		sd0-emmc-data {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
 			       "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
 			power-source = <1800>;
 		};
 
-		sd0_emmc_ctrl {
+		sd0-emmc-ctrl {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <1800>;
 		};
 
-		sd0_emmc_rst {
+		sd0-emmc-rst {
 			pins = "SD0_RST#";
 			power-source = <1800>;
 		};
 	};
 
 	sdhi0_pins: sd0 {
-		sd0_data {
+		sd0-data {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3";
 			power-source = <3300>;
 		};
 
-		sd0_ctrl {
+		sd0-ctrl {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <3300>;
 		};
 
-		sd0_mux {
+		sd0-mux {
 			pinmux = <RZG2L_PORT_PINMUX(47, 0, 2)>; /* SD0_CD */
 		};
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
-		sd0_data_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
+		sd0-data-uhs {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3";
 			power-source = <1800>;
 		};
 
-		sd0_ctrl_uhs {
+		sd0-ctrl-uhs {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <1800>;
 		};
 
-		sd0_mux_uhs {
+		sd0-mux-uhs {
 			pinmux = <RZG2L_PORT_PINMUX(47, 0, 2)>; /* SD0_CD */
 		};
 	};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi
index b76b55e7f09dfb..1411e61a53000c 100644
--- a/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2l-smarc.dtsi
@@ -31,7 +31,7 @@
 		};
 	};
 
-	sound_card {
+	sound-card {
 		compatible = "audio-graph-card";
 		label = "HDMI-Audio";
 		dais = <&i2s2_port>;
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-sr-som-emmc.dtso b/arch/arm64/boot/dts/renesas/rzg2l-sr-som-emmc.dtso
new file mode 100644
index 00000000000000..92e3406be7dfc4
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-sr-som-emmc.dtso
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Overlay for the RZ/G2L(C) Solidrun SOM eMMC
+ *
+ * Copyright (C) 2024 SolidRun Ltd.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+&reg_pmic_ldo1 {
+	/*
+	 * This ldo can switch mmc host controller io voltage between
+	 * 1.8V and 3.3V. The eMMC IO voltage however is supplied from
+	 * reg_pmic_buck3 which is fixed at 1.8V.
+	 * Lower this ldo maximum voltage to 1.8V to prevent setting 3.3V.
+	 */
+	regulator-max-microvolt = <1800000>;
+};
+
+&sdhi0 {
+	/*
+	 * Host controller and eMMC have separate io voltage regulators:
+	 * reg_pmic_ldo1 (1.8V/3.3V); reg_pmic_buck3 (1.8V only).
+	 * Link to the switchable regulator ensuring that it gets configured.
+	 */
+	vqmmc-supply = <&reg_pmic_ldo1>;
+	bus-width = <8>;
+	cap-mmc-hw-reset;
+	mmc-hs200-1_8v;
+	mux-states = <&sdhi0_mux 0>;
+	non-removable;
+	no-sdio;
+	pinctrl-0 = <&sdhi0_8bit_uhs_pins>, <&sdhi0_rst_pins>;
+	pinctrl-1 = <&sdhi0_8bit_uhs_pins>, <&sdhi0_rst_pins>;
+	pinctrl-names = "default", "state_uhs";
+	vmmc-supply = <&reg_pmic_buck4>;
+	/* emmc io voltage is hard-wired for 1.8V, disable sd modes */
+	no-sd;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-sr-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-sr-som.dtsi
new file mode 100644
index 00000000000000..c1dcc5ea430aa0
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2l-sr-som.dtsi
@@ -0,0 +1,473 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Source for the RZ/G2L Solidrun SoM
+ *
+ * Copyright 2023 SolidRun Ltd.
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+/ {
+	aliases {
+		ethernet0 = &eth0;
+		ethernet1 = &eth1;
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c3;
+		mmc0 = &sdhi0;
+		mmc1 = &sdhi1;
+		rtc0 = &pmic;
+		serial0 = &scif0;
+		serial1 = &scif1;
+		serial2 = &scif2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@48000000 {
+		/*
+		 * Physical RAM starts at 0x40000000, but Renesas RZ/G2 BSPs
+		 * reserve and hide the first 128MB for TrustZone by default.
+		 *
+		 * Minimum size for this SoM is 1GB, and the bootloader patches
+		 * the dtb according to the actual RAM size and reservations.
+		 */
+		reg = <0x0 0x48000000 0x0 0x38000000>;
+		device_type = "memory";
+	};
+
+	sdhi0_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		/*
+		 * Mux switches SD0_DATA[0-3], SD0_CMD & SD0_CLK between
+		 * on-SoM eMMC and board-to-board connector using one gpio:
+		 * Low = connector, High = eMMC.
+		 * Define active-low for logical mux state 0 = eMMC, 1 = connector.
+		 */
+		mux-gpios = <&pinctrl RZG2L_GPIO(22, 1) GPIO_ACTIVE_LOW>;
+	};
+
+	reg_pmic_buck1: regulator-pmic-buck1 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck1";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1100000>;
+		regulator-min-microvolt = <1100000>;
+	};
+
+	reg_pmic_buck3: regulator-pmic-buck3 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+	};
+
+	reg_pmic_buck4: regulator-pmic-buck4 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck4";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+	};
+
+	reg_pmic_ldo1: regulator-pmic-ldo1 {
+		compatible = "regulator-gpio";
+		regulator-name = "pmic-ldo1";
+		gpios = <&pinctrl RZG2L_GPIO(39, 0) GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <1800000>;
+		states = <3300000 1>, <1800000 0>;
+	};
+
+	reg_pmic_ldo2: regulator-pmic-ldo2 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-ldo2";
+		/*
+		 * This ldo can switch mmc host controller io voltage between
+		 * 1.8V and 3.3V by assembly option of pull-up / pull-dow.
+		 * Default assembly is 3.3V.
+		 */
+		regulator-min-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		global_cma: linux,cma@58000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x58000000 0x0 0x10000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	sdhi1_pwrseq: sdhi1-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&pinctrl RZG2L_GPIO(23, 1) GPIO_ACTIVE_LOW>;
+	};
+
+	/* 32.768kHz crystal */
+	x2: x2-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&eth0 {
+	phy-handle = <&phy0>;
+	pinctrl-0 = <&eth0_pins>;
+	pinctrl-names = "default";
+	/*
+	 * ravb driver does not configure mac internal delays for RZ/G2L(C),
+	 * instead delays are added by the MxL86110 phy driver.
+	 */
+	phy-mode = "rgmii-id";
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+		/* Level interrupts are typical for phy but not supported on RZ/G2L gpios. */
+		interrupts-extended = <&pinctrl RZG2L_GPIO(27, 0) IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&eth1 {
+	phy-handle = <&phy1>;
+	pinctrl-0 = <&eth1_pins>;
+	pinctrl-names = "default";
+	/*
+	 * ravb driver does not configure mac internal delays for RZ/G2L(C),
+	 * instead delays are added by the MxL86110 phy driver.
+	 */
+	phy-mode = "rgmii-id";
+	status = "okay";
+
+	phy1: ethernet-phy@4 {
+		reg = <4>;
+		/* Level interrupts are typical for phy but not supported on RZ/G2L gpios. */
+		interrupts-extended = <&pinctrl RZG2L_GPIO(42, 4) IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <24000000>;
+};
+
+&gpu {
+	mali-supply = <&reg_pmic_buck1>;
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	eeprom: eeprom@50 {
+		compatible = "atmel,24c01";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c3_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pmic: pmic@12 {
+		compatible = "renesas,raa215300";
+		reg = <0x12>, <0x6f>;
+		reg-names = "main", "rtc";
+		clocks = <&x2>;
+		clock-names = "xin";
+	};
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&ostm1 {
+	status = "okay";
+};
+
+&ostm2 {
+	status = "okay";
+};
+
+&phyrst {
+	status = "okay";
+};
+
+&pinctrl {
+	eth0_pins: eth0 {
+		pinmux = <RZG2L_PORT_PINMUX(28, 1, 1)>, /* ET0_LINKSTA */
+			 <RZG2L_PORT_PINMUX(27, 1, 1)>, /* ET0_MDC */
+			 <RZG2L_PORT_PINMUX(28, 0, 1)>, /* ET0_MDIO */
+			 <RZG2L_PORT_PINMUX(20, 0, 1)>, /* ET0_TXC */
+			 <RZG2L_PORT_PINMUX(20, 1, 1)>, /* ET0_TX_CTL */
+			 <RZG2L_PORT_PINMUX(20, 2, 1)>, /* ET0_TXD0 */
+			 <RZG2L_PORT_PINMUX(21, 0, 1)>, /* ET0_TXD1 */
+			 <RZG2L_PORT_PINMUX(21, 1, 1)>, /* ET0_TXD2 */
+			 <RZG2L_PORT_PINMUX(22, 0, 1)>, /* ET0_TXD3 */
+			 <RZG2L_PORT_PINMUX(24, 0, 1)>, /* ET0_RXC */
+			 <RZG2L_PORT_PINMUX(24, 1, 1)>, /* ET0_RX_CTL */
+			 <RZG2L_PORT_PINMUX(25, 0, 1)>, /* ET0_RXD0 */
+			 <RZG2L_PORT_PINMUX(25, 1, 1)>, /* ET0_RXD1 */
+			 <RZG2L_PORT_PINMUX(26, 0, 1)>, /* ET0_RXD2 */
+			 <RZG2L_PORT_PINMUX(26, 1, 1)>; /* ET0_RXD3 */
+	};
+
+	eth1_pins: eth1 {
+		pinmux = <RZG2L_PORT_PINMUX(37, 2, 1)>, /* ET1_LINKSTA */
+			 <RZG2L_PORT_PINMUX(37, 0, 1)>, /* ET1_MDC */
+			 <RZG2L_PORT_PINMUX(37, 1, 1)>, /* ET1_MDIO */
+			 <RZG2L_PORT_PINMUX(29, 0, 1)>, /* ET1_TXC */
+			 <RZG2L_PORT_PINMUX(29, 1, 1)>, /* ET1_TX_CTL */
+			 <RZG2L_PORT_PINMUX(30, 0, 1)>, /* ET1_TXD0 */
+			 <RZG2L_PORT_PINMUX(30, 1, 1)>, /* ET1_TXD1 */
+			 <RZG2L_PORT_PINMUX(31, 0, 1)>, /* ET1_TXD2 */
+			 <RZG2L_PORT_PINMUX(31, 1, 1)>, /* ET1_TXD3 */
+			 <RZG2L_PORT_PINMUX(33, 1, 1)>, /* ET1_RXC */
+			 <RZG2L_PORT_PINMUX(34, 0, 1)>, /* ET1_RX_CTL */
+			 <RZG2L_PORT_PINMUX(34, 1, 1)>, /* ET1_RXD0 */
+			 <RZG2L_PORT_PINMUX(35, 0, 1)>, /* ET1_RXD1 */
+			 <RZG2L_PORT_PINMUX(35, 1, 1)>, /* ET1_RXD2 */
+			 <RZG2L_PORT_PINMUX(36, 0, 1)>; /* ET1_RXD3 */
+	};
+
+	i2c0_pins: i2c0 {
+		input-enable;
+		pins = "RIIC0_SDA", "RIIC0_SCL";
+	};
+
+	i2c1_pins: i2c1 {
+		input-enable;
+		pins = "RIIC1_SDA", "RIIC1_SCL";
+	};
+
+	i2c3_pins: i2c3 {
+		pinmux = <RZG2L_PORT_PINMUX(18, 0, 3)>, /* RIIC3_SDA */
+			 <RZG2L_PORT_PINMUX(18, 1, 3)>; /* RIIC3_SCL */
+	};
+
+	qspi0_pins: qspi0 {
+		pins = "QSPI0_IO0", "QSPI0_IO1", "QSPI0_IO2", "QSPI0_IO3",
+		       "QSPI0_SPCLK", "QSPI0_SSL";
+		power-source = <1800>;
+	};
+
+	scif0_pins: scif0 {
+		pinmux = <RZG2L_PORT_PINMUX(38, 0, 1)>, /* SCIF0_TXD */
+			 <RZG2L_PORT_PINMUX(38, 1, 1)>; /* SCIF0_RXD */
+	};
+
+	scif2_pins: scif2 {
+		pinmux = <RZG2L_PORT_PINMUX(48, 0, 1)>, /* SCIF2_TXD */
+			 <RZG2L_PORT_PINMUX(48, 1, 1)>, /* SCIF2_RXD */
+			 <RZG2L_PORT_PINMUX(48, 3, 1)>, /* SCIF2_CTS# */
+			 <RZG2L_PORT_PINMUX(48, 4, 1)>; /* SCIF2_RTS# */
+	};
+
+	sdhi0_4bit_pins: sdhi0-4bit {
+		sd0_ctrl_dat03 {
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_CLK", "SD0_CMD";
+			power-source = <3300>;
+		};
+
+		/*
+		 * Pins 4-7 are hard-wired to eMMC with 1.8V IO voltage,
+		 * with all pins sharing a single voltage domain.
+		 *
+		 * These pins are fixed-function lacking independently configurable
+		 * pull-up, pull-down or output-disable.
+		 *
+		 * While unused, disable the input buffers to avoid sampling the still
+		 * physically connected but unused eMMC lines.
+		 * For output direction rely on correct controller bus-width configuration
+		 * to avoid over-voltage to eMMC e.g. when operating at 3.3V for a 4-bit microSD.
+		 */
+		sd0_dat47 {
+			input-disable;
+			pins = "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_4bit_uhs_pins: sdhi0-4bit-uhs {
+		sd0_ctrl_dat03 {
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_CLK", "SD0_CMD";
+			power-source = <1800>;
+		};
+
+		/*
+		 * Pins 4-7 are hard-wired to eMMC with 1.8V IO voltage,
+		 * with all pins sharing a single voltage domain.
+		 *
+		 * These pins are fixed-function, lacking configurable pull-up, pull-down or
+		 * output-disable, while sharing the voltage domain with all SD0_* pins.
+		 *
+		 * While unused, disable the input buffers to avoid sampling the still
+		 * physically connected but unused eMMC lines.
+		 * For output direction rely on correct controller bus-width configuration
+		 * to avoid over-voltage to eMMC e.g. when operating at 3.3V for a 4-bit microSD.
+		 */
+		sd0_dat47 {
+			input-disable;
+			pins = "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_8bit_uhs_pins: sdhi0-8bit-uhs {
+		sd0_ctrl {
+			pins = "SD0_CLK", "SD0_CMD";
+			power-source = <1800>;
+		};
+
+		sd0_dat {
+			input-enable;
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_cd_pins: sdhi0-cd {
+		pinmux = <RZG2L_PORT_PINMUX(47, 0, 2)>; /* SD0_CD */
+	};
+
+	/* SD0_RST is only routed to eMMC which uses fixed 1.8V IO voltage */
+	sdhi0_rst_pins: sdhi0-rst {
+		pins = "SD0_RST#";
+		power-source = <1800>;
+	};
+
+	sdhi1_pins: sdhi1 {
+		pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3",
+		       "SD1_CLK", "SD1_CMD";
+		power-source = <3300>;
+	};
+
+	spi1_pins: spi1 {
+		pinmux = <RZG2L_PORT_PINMUX(44, 2, 1)>, /* RSPI1_MISO */
+			 <RZG2L_PORT_PINMUX(44, 1, 1)>, /* RSPI1_MOSI */
+			 <RZG2L_PORT_PINMUX(44, 0, 1)>; /* RSPI1_CK */
+	};
+
+	spi1_cs_pins: spi1-cs {
+		pinmux = <RZG2L_PORT_PINMUX(44, 3, 1)>; /* RSPI1_SSL */
+	};
+};
+
+&sbc {
+	pinctrl-0 = <&qspi0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	flash@0 {
+		compatible = "winbond,w25q80bl", "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <50000000>;
+		m25p,fast-read;
+	};
+};
+
+&scif0 {
+	pinctrl-0 = <&scif0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&scif2 {
+	pinctrl-0 = <&scif2_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		/*
+		 * The BT is based on BCM4345C0. Murata 1MW variant seems to
+		 * require driving RTS on open.
+		 * Use compatible bcm43438 to accomplish this.
+		 */
+		compatible = "brcm,bcm43438-bt";
+		max-speed = <115200>;
+		shutdown-gpios = <&pinctrl RZG2L_GPIO(23, 0) GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&reg_pmic_buck4>;
+		vddio-supply = <&reg_pmic_buck4>;
+	};
+};
+
+/* WiFi */
+&sdhi1 {
+	/* Murata 1MW max rate is 50MHz */
+	max-frequency = <50000000>;
+	bus-width = <4>;
+	cap-sdio-irq;
+	mmc-pwrseq = <&sdhi1_pwrseq>;
+	non-removable;
+	no-1-8-v;
+	no-sd;
+	pinctrl-0 = <&sdhi1_pins>;
+	pinctrl-names = "default";
+	vmmc-supply = <&reg_pmic_buck4>;
+	/*
+	 * Host controller IO voltage is provided from reg_pmic_ldo2,
+	 * WiFi module IO voltage from reg_pmic_buck4.
+	 * Neither is configurable at run-time so either can be set here.
+	 */
+	vqmmc-supply = <&reg_pmic_ldo2>;
+	status = "okay";
+};
+
+&usb2_phy0 {
+	vbus-supply = <&usb0_vbus_otg>;
+	status = "okay";
+};
+
+&usb2_phy1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2lc-hummingboard-pulse-leds.dtso b/arch/arm64/boot/dts/renesas/rzg2lc-hummingboard-pulse-leds.dtso
new file mode 100644
index 00000000000000..a2572718fa87c5
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2lc-hummingboard-pulse-leds.dtso
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Overlay for SolidRun RZ/G2LC HummingBoard Pulse
+ * enabling LEDs D31-34 in exchange for on-SoM Bluetooth.
+ *
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+&leds {
+	led-1 {
+		label = "D31";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(5, 2) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+
+	led-2 {
+		label = "D32";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(5, 1) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+
+	led-3 {
+		label = "D33";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(42, 0) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+
+	led-4 {
+		label = "D34";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(42, 1) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+};
+
+&scif2 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2lc-smarc-pinfunction.dtsi b/arch/arm64/boot/dts/renesas/rzg2lc-smarc-pinfunction.dtsi
index 92c64d58349f36..92ec7935025cb0 100644
--- a/arch/arm64/boot/dts/renesas/rzg2lc-smarc-pinfunction.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2lc-smarc-pinfunction.dtsi
@@ -79,38 +79,38 @@
 	};
 
 	sdhi1_pins: sd1 {
-		sd1_data {
+		sd1-data {
 			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
 			power-source = <3300>;
 		};
 
-		sd1_ctrl {
+		sd1-ctrl {
 			pins = "SD1_CLK", "SD1_CMD";
 			power-source = <3300>;
 		};
 
-		sd1_mux {
+		sd1-mux {
 			pinmux = <RZG2L_PORT_PINMUX(19, 0, 1)>; /* SD1_CD */
 		};
 	};
 
-	sdhi1_pins_uhs: sd1_uhs {
-		sd1_data_uhs {
+	sdhi1_pins_uhs: sd1-uhs {
+		sd1-data-uhs {
 			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
 			power-source = <1800>;
 		};
 
-		sd1_ctrl_uhs {
+		sd1-ctrl-uhs {
 			pins = "SD1_CLK", "SD1_CMD";
 			power-source = <1800>;
 		};
 
-		sd1_mux_uhs {
+		sd1-mux-uhs {
 			pinmux = <RZG2L_PORT_PINMUX(19, 0, 1)>; /* SD1_CD */
 		};
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		pins = "AUDIO_CLK1", "AUDIO_CLK2";
 		input-enable;
 	};
diff --git a/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi
index 15f2e9eaaf0b62..a043f2f5703632 100644
--- a/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi
@@ -189,51 +189,51 @@
 	};
 
 	sdhi0_emmc_pins: sd0emmc {
-		sd0_emmc_data {
+		sd0-emmc-data {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
 			       "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
 			power-source = <1800>;
 		};
 
-		sd0_emmc_ctrl {
+		sd0-emmc-ctrl {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <1800>;
 		};
 
-		sd0_emmc_rst {
+		sd0-emmc-rst {
 			pins = "SD0_RST#";
 			power-source = <1800>;
 		};
 	};
 
 	sdhi0_pins: sd0 {
-		sd0_data {
+		sd0-data {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3";
 			power-source = <3300>;
 		};
 
-		sd0_ctrl {
+		sd0-ctrl {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <3300>;
 		};
 
-		sd0_mux {
+		sd0-mux {
 			pinmux = <RZG2L_PORT_PINMUX(18, 0, 1)>; /* SD0_CD */
 		};
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
-		sd0_data_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
+		sd0-data-uhs {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3";
 			power-source = <1800>;
 		};
 
-		sd0_ctrl_uhs {
+		sd0-ctrl-uhs {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <1800>;
 		};
 
-		sd0_mux_uhs {
+		sd0-mux-uhs {
 			pinmux = <RZG2L_PORT_PINMUX(18, 0, 1)>; /* SD0_CD */
 		};
 	};
diff --git a/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi
index f3d7eff0d2f2a0..0d4c6b37d7a70f 100644
--- a/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2lc-smarc.dtsi
@@ -37,7 +37,7 @@
 #if (SW_I2S0_I2S1)
 	/delete-node/ sound;
 
-	sound_card {
+	sound-card {
 		compatible = "audio-graph-card";
 		label = "HDMI-Audio";
 		dais = <&i2s2_port>;
diff --git a/arch/arm64/boot/dts/renesas/rzg2lc-sr-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2lc-sr-som.dtsi
new file mode 100644
index 00000000000000..bf3d9e8ed109cd
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2lc-sr-som.dtsi
@@ -0,0 +1,436 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Source for the RZ/G2LC Solidrun SOM
+ *
+ * Copyright 2023 SolidRun Ltd.
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+/ {
+	aliases {
+		ethernet0 = &eth0;
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		mmc0 = &sdhi0;
+		mmc1 = &sdhi1;
+		rtc0 = &pmic;
+		serial0 = &scif0;
+		serial1 = &scif1;
+		serial2 = &scif2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@48000000 {
+		/*
+		 * Physical RAM starts at 0x40000000, but Renesas RZ/G2 BSPs
+		 * reserve and hide the first 128MB for TrustZone by default.
+		 *
+		 * Minimum size for this SoM is 1GB, and the bootloader patches
+		 * the dtb according to the actual RAM size and reservations.
+		 */
+		reg = <0x0 0x48000000 0x0 0x38000000>;
+		device_type = "memory";
+	};
+
+	sdhi0_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		/*
+		 * Mux switches SD0_DATA[0-3], SD0_CMD & SD0_CLK between
+		 * on-SoM eMMC and board-to-board connector using one gpio:
+		 * High = connector, Low = eMMC.
+		 * Define active-high for logical mux state 0 = eMMC, 1 = connector.
+		 */
+		mux-gpios = <&pinctrl RZG2L_GPIO(22, 1) GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_pmic_buck1: regulator-pmic-buck1 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck1";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1100000>;
+		regulator-min-microvolt = <1100000>;
+	};
+
+	reg_pmic_buck3: regulator-pmic-buck3 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+	};
+
+	reg_pmic_buck4: regulator-pmic-buck4 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck4";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+	};
+
+	reg_pmic_ldo1: regulator-pmic-ldo1 {
+		compatible = "regulator-gpio";
+		regulator-name = "pmic-ldo1";
+		gpios = <&pinctrl RZG2L_GPIO(39, 0) GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <1800000>;
+		states = <3300000 1>, <1800000 0>;
+	};
+
+	reg_pmic_ldo2: regulator-pmic-ldo2 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-ldo2";
+		/*
+		 * This ldo can switch mmc host controller io voltage between
+		 * 1.8V and 3.3V by assembly option of pull-up / pull-dow.
+		 * Default assembly is 3.3V.
+		 */
+		regulator-min-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		global_cma: linux,cma@58000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x58000000 0x0 0x10000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	sdhi1_pwrseq: sdhi1-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&pinctrl RZG2L_GPIO(23, 0) GPIO_ACTIVE_LOW>;
+	};
+
+	/* 32.768kHz crystal */
+	x2: x2-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&eth0 {
+	phy-handle = <&phy0>;
+	pinctrl-0 = <&eth0_pins>;
+	pinctrl-names = "default";
+	/*
+	 * ravb driver does not configure mac internal delays for RZ/G2L(C),
+	 * instead delays are added by the ADIN1200 phy driver.
+	 */
+	phy-mode = "rgmii-id";
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+		/* Level interrupts are typical for phy but not supported on RZ/G2L gpios. */
+		interrupts-extended = <&pinctrl RZG2L_GPIO(27, 0) IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <24000000>;
+};
+
+&gpu {
+	mali-supply = <&reg_pmic_buck1>;
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	eeprom: eeprom@50 {
+		compatible = "atmel,24c01";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pmic: pmic@12 {
+		compatible = "renesas,raa215300";
+		reg = <0x12>, <0x6f>;
+		reg-names = "main", "rtc";
+		clocks = <&x2>;
+		clock-names = "xin";
+	};
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&ostm1 {
+	status = "okay";
+};
+
+&ostm2 {
+	status = "okay";
+};
+
+&phyrst {
+	status = "okay";
+};
+
+&pinctrl {
+	eth0_pins: eth0 {
+		pinmux = <RZG2L_PORT_PINMUX(28, 1, 1)>, /* ET0_LINKSTA */
+			 <RZG2L_PORT_PINMUX(27, 1, 1)>, /* ET0_MDC */
+			 <RZG2L_PORT_PINMUX(28, 0, 1)>, /* ET0_MDIO */
+			 <RZG2L_PORT_PINMUX(20, 0, 1)>, /* ET0_TXC */
+			 <RZG2L_PORT_PINMUX(20, 1, 1)>, /* ET0_TX_CTL */
+			 <RZG2L_PORT_PINMUX(20, 2, 1)>, /* ET0_TXD0 */
+			 <RZG2L_PORT_PINMUX(21, 0, 1)>, /* ET0_TXD1 */
+			 <RZG2L_PORT_PINMUX(21, 1, 1)>, /* ET0_TXD2 */
+			 <RZG2L_PORT_PINMUX(22, 0, 1)>, /* ET0_TXD3 */
+			 <RZG2L_PORT_PINMUX(24, 0, 1)>, /* ET0_RXC */
+			 <RZG2L_PORT_PINMUX(24, 1, 1)>, /* ET0_RX_CTL */
+			 <RZG2L_PORT_PINMUX(25, 0, 1)>, /* ET0_RXD0 */
+			 <RZG2L_PORT_PINMUX(25, 1, 1)>, /* ET0_RXD1 */
+			 <RZG2L_PORT_PINMUX(26, 0, 1)>, /* ET0_RXD2 */
+			 <RZG2L_PORT_PINMUX(26, 1, 1)>; /* ET0_RXD3 */
+	};
+
+	i2c0_pins: i2c0 {
+		input-enable;
+		pins = "RIIC0_SDA", "RIIC0_SCL";
+	};
+
+	i2c1_pins: i2c1 {
+		input-enable;
+		pins = "RIIC1_SDA", "RIIC1_SCL";
+	};
+
+	i2c2_pins: i2c2 {
+		pinmux = <RZG2L_PORT_PINMUX(42, 3, 1)>, /* RIIC2_SDA */
+			 <RZG2L_PORT_PINMUX(42, 4, 1)>; /* RIIC2_SCL */
+	};
+
+	qspi0_pins: qspi0 {
+		pins = "QSPI0_IO0", "QSPI0_IO1", "QSPI0_IO2", "QSPI0_IO3",
+		       "QSPI0_SPCLK", "QSPI0_SSL";
+		power-source = <1800>;
+	};
+
+	scif0_pins: scif0 {
+		pinmux = <RZG2L_PORT_PINMUX(38, 0, 1)>, /* SCIF0_TXD */
+			 <RZG2L_PORT_PINMUX(38, 1, 1)>; /* SCIF0_RXD */
+	};
+
+	scif2_pins: scif2 {
+		pinmux = <RZG2L_PORT_PINMUX(42, 0, 4)>, /* SCIF2_TXD */
+			 <RZG2L_PORT_PINMUX(42, 1, 4)>, /* SCIF2_RXD */
+			 <RZG2L_PORT_PINMUX(5, 1, 2)>, /* SCIF2_CTS# */
+			 <RZG2L_PORT_PINMUX(5, 2, 2)>; /* SCIF2_RTS# */
+	};
+
+	sdhi0_4bit_pins: sdhi0-4bit {
+		sd0_ctrl_dat03 {
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_CLK", "SD0_CMD";
+			power-source = <3300>;
+		};
+
+		/*
+		 * Pins 4-7 are hard-wired to eMMC with 1.8V IO voltage,
+		 * with all pins sharing a single voltage domain.
+		 *
+		 * These pins are fixed-function lacking independently configurable
+		 * pull-up, pull-down or output-disable.
+		 *
+		 * While unused, disable the input buffers to avoid sampling the still
+		 * physically connected but unused eMMC lines.
+		 * For output direction rely on correct controller bus-width configuration
+		 * to avoid over-voltage to eMMC e.g. when operating at 3.3V for a 4-bit microSD.
+		 */
+		sd0_dat47 {
+			input-disable;
+			pins = "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_4bit_uhs_pins: sdhi0-4bit-uhs {
+		sd0_ctrl_dat03 {
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_CLK", "SD0_CMD";
+			power-source = <1800>;
+		};
+
+		/*
+		 * Pins 4-7 are hard-wired to eMMC with 1.8V IO voltage,
+		 * with all pins sharing a single voltage domain.
+		 *
+		 * These pins are fixed-function, lacking configurable pull-up, pull-down or
+		 * output-disable, while sharing the voltage domain with all SD0_* pins.
+		 *
+		 * While unused, disable the input buffers to avoid sampling the still
+		 * physically connected but unused eMMC lines.
+		 * For output direction rely on correct controller bus-width configuration
+		 * to avoid over-voltage to eMMC e.g. when operating at 3.3V for a 4-bit microSD.
+		 */
+		sd0_dat47 {
+			input-disable;
+			pins = "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_8bit_uhs_pins: sdhi0-8bit-uhs {
+		sd0_ctrl {
+			pins = "SD0_CLK", "SD0_CMD";
+			power-source = <1800>;
+		};
+
+		sd0_dat {
+			input-enable;
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_cd_pins: sdhi0-cd {
+		pinmux = <RZG2L_PORT_PINMUX(18, 0, 1)>; /* SD0_CD */
+	};
+
+	/* SD0_RST is only routed to eMMC which uses fixed 1.8V IO voltage */
+	sdhi0_rst_pins: sdhi0-rst {
+		pins = "SD0_RST#";
+		power-source = <1800>;
+	};
+
+	sdhi1_pins: sdhi1 {
+		pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3",
+		       "SD1_CLK", "SD1_CMD";
+		power-source = <3300>;
+	};
+
+	spi1_pins: spi1 {
+		pinmux = <RZG2L_PORT_PINMUX(44, 2, 1)>, /* RSPI1_MISO */
+			 <RZG2L_PORT_PINMUX(44, 1, 1)>, /* RSPI1_MOSI */
+			 <RZG2L_PORT_PINMUX(44, 0, 1)>; /* RSPI1_CK */
+	};
+
+	spi1_cs_pins: spi1-cs {
+		pinmux = <RZG2L_PORT_PINMUX(44, 3, 1)>; /* RSPI1_SSL */
+	};
+};
+
+&sbc {
+	pinctrl-0 = <&qspi0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	flash@0 {
+		compatible = "winbond,w25q80bl", "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <50000000>;
+		m25p,fast-read;
+	};
+};
+
+&scif0 {
+	pinctrl-0 = <&scif0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&scif2 {
+	pinctrl-0 = <&scif2_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		/*
+		 * The BT is based on BCM4343A2. Murata 1YN variant seems to
+		 * require driving RTS on open.
+		 * Use compatible bcm43438 to accomplish this.
+		 */
+		compatible = "brcm,bcm43438-bt";
+		max-speed = <115200>;
+		shutdown-gpios = <&pinctrl RZG2L_GPIO(23, 1) GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&reg_pmic_buck4>;
+		vddio-supply = <&reg_pmic_buck4>;
+	};
+};
+
+/* WiFi */
+&sdhi1 {
+	/* Murata 1YN max rate is 50MHz */
+	max-frequency = <50000000>;
+	bus-width = <4>;
+	cap-sdio-irq;
+	mmc-pwrseq = <&sdhi1_pwrseq>;
+	non-removable;
+	no-1-8-v;
+	no-sd;
+	pinctrl-0 = <&sdhi1_pins>;
+	pinctrl-names = "default";
+	vmmc-supply = <&reg_pmic_buck4>;
+	/*
+	 * Host controller IO voltage is provided from reg_pmic_ldo2,
+	 * WiFi module IO voltage from reg_pmic_buck4.
+	 * Neither is configurable at run-time so either can be set here.
+	 */
+	vqmmc-supply = <&reg_pmic_ldo2>;
+	status = "okay";
+};
+
+&usb2_phy0 {
+	vbus-supply = <&usb0_vbus_otg>;
+	status = "okay";
+};
+
+&usb2_phy1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-hummingboard-pulse-leds.dtso b/arch/arm64/boot/dts/renesas/rzg2ul-hummingboard-pulse-leds.dtso
new file mode 100644
index 00000000000000..ab7f826c6e0398
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2ul-hummingboard-pulse-leds.dtso
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Overlay for SolidRun RZ/G2UL HummingBoard Pulse
+ * enabling LEDs D31-34 in exchange for on-SoM Bluetooth.
+ *
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+&leds {
+	led-1 {
+		label = "D31";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(5, 4) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+
+	led-2 {
+		label = "D32";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(5, 3) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+
+	led-3 {
+		label = "D33";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(6, 0) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+
+	led-4 {
+		label = "D34";
+		color = <LED_COLOR_ID_GREEN>;
+		gpios = <&pinctrl RZG2L_GPIO(6, 1) GPIO_ACTIVE_LOW>;
+		default-state = "on";
+	};
+};
+
+&scif2 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-pinfunction.dtsi b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-pinfunction.dtsi
index 355694fe4af686..0be666c9049fb5 100644
--- a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-pinfunction.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-pinfunction.dtsi
@@ -69,38 +69,38 @@
 	};
 
 	sdhi1_pins: sd1 {
-		sd1_data {
+		sd1-data {
 			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
 			power-source = <3300>;
 		};
 
-		sd1_ctrl {
+		sd1-ctrl {
 			pins = "SD1_CLK", "SD1_CMD";
 			power-source = <3300>;
 		};
 
-		sd1_mux {
+		sd1-mux {
 			pinmux = <RZG2L_PORT_PINMUX(0, 2, 1)>; /* SD1_CD */
 		};
 	};
 
-	sdhi1_pins_uhs: sd1_uhs {
-		sd1_data_uhs {
+	sdhi1_pins_uhs: sd1-uhs {
+		sd1-data-uhs {
 			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
 			power-source = <1800>;
 		};
 
-		sd1_ctrl_uhs {
+		sd1-ctrl-uhs {
 			pins = "SD1_CLK", "SD1_CMD";
 			power-source = <1800>;
 		};
 
-		sd1_mux_uhs {
+		sd1-mux-uhs {
 			pinmux = <RZG2L_PORT_PINMUX(0, 2, 1)>; /* SD1_CD */
 		};
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		pins = "AUDIO_CLK1", "AUDIO_CLK2";
 		input-enable;
 	};
diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi
index 0f917d7c99398f..99d30fa9a1a83d 100644
--- a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi
@@ -204,51 +204,51 @@
 	};
 
 	sdhi0_emmc_pins: sd0emmc {
-		sd0_emmc_data {
+		sd0-emmc-data {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
 			       "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
 			power-source = <1800>;
 		};
 
-		sd0_emmc_ctrl {
+		sd0-emmc-ctrl {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <1800>;
 		};
 
-		sd0_emmc_rst {
+		sd0-emmc-rst {
 			pins = "SD0_RST#";
 			power-source = <1800>;
 		};
 	};
 
 	sdhi0_pins: sd0 {
-		sd0_data {
+		sd0-data {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3";
 			power-source = <3300>;
 		};
 
-		sd0_ctrl {
+		sd0-ctrl {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <3300>;
 		};
 
-		sd0_mux {
+		sd0-mux {
 			pinmux = <RZG2L_PORT_PINMUX(0, 0, 1)>; /* SD0_CD */
 		};
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
-		sd0_data_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
+		sd0-data-uhs {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3";
 			power-source = <1800>;
 		};
 
-		sd0_ctrl_uhs {
+		sd0-ctrl-uhs {
 			pins = "SD0_CLK", "SD0_CMD";
 			power-source = <1800>;
 		};
 
-		sd0_mux_uhs {
+		sd0-mux-uhs {
 			pinmux = <RZG2L_PORT_PINMUX(0, 0, 1)>; /* SD0_CD */
 		};
 	};
diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-sr-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2ul-sr-som.dtsi
new file mode 100644
index 00000000000000..5e4d7861066dd5
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzg2ul-sr-som.dtsi
@@ -0,0 +1,418 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Source for the RZ/G2UL Solidrun SOM
+ *
+ * Copyright 2025 Josua Mayer <josua@solid-run.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+/ {
+	aliases {
+		ethernet0 = &eth0;
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		mmc0 = &sdhi0;
+		mmc1 = &sdhi1;
+		rtc0 = &pmic;
+		serial0 = &scif0;
+		serial1 = &scif1;
+		serial2 = &scif2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@48000000 {
+		/*
+		 * Physical RAM starts at 0x40000000, but Renesas RZ/G2 BSPs
+		 * reserve and hide the first 128MB for TrustZone by default.
+		 *
+		 * Minimum size for this SoM is 512MB, and the bootloader patches
+		 * the dtb according to the actual RAM size and reservations.
+		 */
+		reg = <0x0 0x48000000 0x0 0x18000000>;
+		device_type = "memory";
+	};
+
+	sdhi0_mux: mux-controller-0 {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		#mux-state-cells = <1>;
+		/*
+		 * Mux switches SD0_DATA[0-3], SD0_CMD & SD0_CLK between
+		 * on-SoM eMMC and board-to-board connector using one gpio:
+		 * High = connector, Low = eMMC.
+		 * Define active-high for logical mux state 0 = eMMC, 1 = connector.
+		 */
+		mux-gpios = <&pinctrl RZG2L_GPIO(2, 1) GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_pmic_buck1: regulator-pmic-buck1 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck1";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1100000>;
+		regulator-min-microvolt = <1100000>;
+	};
+
+	reg_pmic_buck3: regulator-pmic-buck3 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+	};
+
+	reg_pmic_buck4: regulator-pmic-buck4 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-buck4";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+	};
+
+	reg_pmic_ldo1: regulator-pmic-ldo1 {
+		compatible = "regulator-gpio";
+		regulator-name = "pmic-ldo1";
+		gpios = <&pinctrl RZG2L_GPIO(13, 2) GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <1800000>;
+		states = <3300000 1>, <1800000 0>;
+	};
+
+	reg_pmic_ldo2: regulator-pmic-ldo2 {
+		compatible = "regulator-fixed";
+		regulator-name = "pmic-ldo2";
+		/*
+		 * This ldo can switch mmc host controller io voltage between
+		 * 1.8V and 3.3V by assembly option of pull-up / pull-dow.
+		 * Default assembly is 3.3V.
+		 */
+		regulator-min-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	sdhi1_pwrseq: sdhi1-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&pinctrl RZG2L_GPIO(2, 2) GPIO_ACTIVE_LOW>;
+	};
+
+	/* 32.768kHz crystal */
+	x2: x2-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&eth0 {
+	phy-handle = <&phy0>;
+	pinctrl-0 = <&eth0_pins>;
+	pinctrl-names = "default";
+	/*
+	 * ravb driver does not configure mac internal delays for RZ/G2(UL/L/C),
+	 * instead delays are added by the ADIN1200 phy driver.
+	 */
+	phy-mode = "rgmii-id";
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		reg = <0>;
+		/* Level interrupts are typical for phy but not supported on rz/g2l gpios. */
+		interrupts-extended = <&pinctrl RZG2L_GPIO(4, 2) IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <24000000>;
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	eeprom: eeprom@50 {
+		compatible = "atmel,24c01";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&i2c2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pmic: pmic@12 {
+		compatible = "renesas,raa215300";
+		reg = <0x12>, <0x6f>;
+		reg-names = "main", "rtc";
+		clocks = <&x2>;
+		clock-names = "xin";
+	};
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&ostm1 {
+	status = "okay";
+};
+
+&ostm2 {
+	status = "okay";
+};
+
+&phyrst {
+	status = "okay";
+};
+
+&pinctrl {
+	eth0_pins: eth0 {
+		pinmux = <RZG2L_PORT_PINMUX(4, 5, 1)>, /* ET0_LINKSTA */
+			 <RZG2L_PORT_PINMUX(4, 3, 1)>, /* ET0_MDC */
+			 <RZG2L_PORT_PINMUX(4, 4, 1)>, /* ET0_MDIO */
+			 <RZG2L_PORT_PINMUX(1, 0, 1)>, /* ET0_TXC */
+			 <RZG2L_PORT_PINMUX(1, 1, 1)>, /* ET0_TX_CTL */
+			 <RZG2L_PORT_PINMUX(1, 2, 1)>, /* ET0_TXD0 */
+			 <RZG2L_PORT_PINMUX(1, 3, 1)>, /* ET0_TXD1 */
+			 <RZG2L_PORT_PINMUX(1, 4, 1)>, /* ET0_TXD2 */
+			 <RZG2L_PORT_PINMUX(2, 0, 1)>, /* ET0_TXD3 */
+			 <RZG2L_PORT_PINMUX(3, 0, 1)>, /* ET0_RXC */
+			 <RZG2L_PORT_PINMUX(3, 1, 1)>, /* ET0_RX_CTL */
+			 <RZG2L_PORT_PINMUX(3, 2, 1)>, /* ET0_RXD0 */
+			 <RZG2L_PORT_PINMUX(3, 3, 1)>, /* ET0_RXD1 */
+			 <RZG2L_PORT_PINMUX(4, 0, 1)>, /* ET0_RXD2 */
+			 <RZG2L_PORT_PINMUX(4, 1, 1)>; /* ET0_RXD3 */
+	};
+
+	i2c0_pins: i2c0 {
+		input-enable;
+		pins = "RIIC0_SDA", "RIIC0_SCL";
+	};
+
+	i2c1_pins: i2c1 {
+		input-enable;
+		pins = "RIIC1_SDA", "RIIC1_SCL";
+	};
+
+	i2c2_pins: i2c2 {
+		pinmux = <RZG2L_PORT_PINMUX(6, 3, 1)>, /* RIIC2_SDA */
+			 <RZG2L_PORT_PINMUX(6, 4, 1)>; /* RIIC2_SCL */
+	};
+
+	qspi0_pins: qspi0 {
+		pins = "QSPI0_IO0", "QSPI0_IO1", "QSPI0_IO2", "QSPI0_IO3",
+		       "QSPI0_SPCLK", "QSPI0_SSL";
+		power-source = <1800>;
+	};
+
+	scif0_pins: scif0 {
+		pinmux = <RZG2L_PORT_PINMUX(13, 0, 1)>, /* SCIF0_TXD */
+			 <RZG2L_PORT_PINMUX(13, 1, 1)>; /* SCIF0_RXD */
+	};
+
+	scif2_pins: scif2 {
+		pinmux = <RZG2L_PORT_PINMUX(6, 0, 4)>, /* SCIF2_TXD */
+			 <RZG2L_PORT_PINMUX(6, 1, 4)>, /* SCIF2_RXD */
+			 <RZG2L_PORT_PINMUX(5, 3, 2)>, /* SCIF2_CTS# */
+			 <RZG2L_PORT_PINMUX(5, 4, 2)>; /* SCIF2_RTS# */
+	};
+
+	sdhi0_4bit_pins: sdhi0-4bit {
+		sd0_ctrl_dat03 {
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_CLK", "SD0_CMD";
+			power-source = <3300>;
+		};
+
+		/*
+		 * Pins 4-7 are hard-wired to eMMC with 1.8V IO voltage,
+		 * with all pins sharing a single voltage domain.
+		 *
+		 * These pins are fixed-function lacking independently configurable
+		 * pull-up, pull-down or output-disable.
+		 *
+		 * While unused, disable the input buffers to avoid sampling the still
+		 * physically connected but unused eMMC lines.
+		 * For output direction rely on correct controller bus-width configuration
+		 * to avoid over-voltage to eMMC e.g. when operating at 3.3V for a 4-bit microSD.
+		 */
+		sd0_dat47 {
+			input-disable;
+			pins = "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_4bit_uhs_pins: sdhi0-4bit-uhs {
+		sd0_ctrl_dat03 {
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_CLK", "SD0_CMD";
+			power-source = <1800>;
+		};
+
+		/*
+		 * Pins 4-7 are hard-wired to eMMC with 1.8V IO voltage,
+		 * with all pins sharing a single voltage domain.
+		 *
+		 * These pins are fixed-function, lacking configurable pull-up, pull-down or
+		 * output-disable, while sharing the voltage domain with all SD0_* pins.
+		 *
+		 * While unused, disable the input buffers to avoid sampling the still
+		 * physically connected but unused eMMC lines.
+		 * For output direction rely on correct controller bus-width configuration
+		 * to avoid over-voltage to eMMC e.g. when operating at 3.3V for a 4-bit microSD.
+		 */
+		sd0_dat47 {
+			input-disable;
+			pins = "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_8bit_uhs_pins: sdhi0-8bit-uhs {
+		sd0_ctrl {
+			pins = "SD0_CLK", "SD0_CMD";
+			power-source = <1800>;
+		};
+
+		sd0_dat {
+			input-enable;
+			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3",
+			       "SD0_DATA4", "SD0_DATA5", "SD0_DATA6", "SD0_DATA7";
+		};
+	};
+
+	sdhi0_cd_pins: sdhi0-cd {
+		pinmux = <RZG2L_PORT_PINMUX(0, 0, 1)>; /* SD0_CD */
+	};
+
+	/* SD0_RST is only routed to eMMC which uses fixed 1.8V IO voltage */
+	sdhi0_rst_pins: sdhi0-rst {
+		pins = "SD0_RST#";
+		power-source = <1800>;
+	};
+
+	sdhi1_pins: sdhi1 {
+		pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3",
+		       "SD1_CLK", "SD1_CMD";
+		power-source = <3300>;
+	};
+
+	spi1_pins: spi1 {
+		pinmux = <RZG2L_PORT_PINMUX(17, 2, 1)>, /* RSPI1_MISO */
+			 <RZG2L_PORT_PINMUX(17, 1, 1)>, /* RSPI1_MOSI */
+			 <RZG2L_PORT_PINMUX(17, 0, 1)>; /* RSPI1_CK */
+	};
+
+	spi1_cs_pins: spi1-cs {
+		pinmux = <RZG2L_PORT_PINMUX(17, 3, 1)>; /* RSPI1_SSL */
+	};
+};
+
+&sbc {
+	pinctrl-0 = <&qspi0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	flash@0 {
+		compatible = "winbond,w25q80bl", "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <50000000>;
+		m25p,fast-read;
+	};
+};
+
+&scif0 {
+	pinctrl-0 = <&scif0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&scif2 {
+	pinctrl-0 = <&scif2_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		/*
+		 * The BT is based on BCM4343A2. Murata 1YN variant seems to
+		 * require driving RTS on open.
+		 * Use compatible bcm43438 to accomplish this.
+		 */
+		compatible = "brcm,bcm43438-bt";
+		max-speed = <115200>;
+		shutdown-gpios = <&pinctrl RZG2L_GPIO(2, 3) GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&reg_pmic_buck4>;
+		vddio-supply = <&reg_pmic_buck4>;
+	};
+};
+
+/* WiFi */
+&sdhi1 {
+	/* Murata 1YN max rate is 50MHz */
+	max-frequency = <50000000>;
+	bus-width = <4>;
+	cap-sdio-irq;
+	mmc-pwrseq = <&sdhi1_pwrseq>;
+	non-removable;
+	no-1-8-v;
+	no-sd;
+	pinctrl-0 = <&sdhi1_pins>;
+	pinctrl-names = "default";
+	vmmc-supply = <&reg_pmic_buck4>;
+	/*
+	 * Host controller IO voltage is provided from reg_pmic_ldo2,
+	 * WiFi module IO voltage from reg_pmic_buck4.
+	 * Neither is configurable at run-time so either can be set here.
+	 */
+	vqmmc-supply = <&reg_pmic_ldo2>;
+	status = "okay";
+};
+
+&usb2_phy0 {
+	vbus-supply = <&usb0_vbus_otg>;
+	status = "okay";
+};
+
+&usb2_phy1 {
+	status = "okay";
+};
+
+&wdt0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi
index 091a227233cbaa..91b2d0d431045f 100644
--- a/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi
@@ -45,6 +45,21 @@
 		reg = <0x0 0x48000000 0x0 0x78000000>;
 	};
 
+	pcie_refclk: pcie-ref-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+	};
+
+	reg_1p0v: regulator-1p0v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.0V";
+		regulator-min-microvolt = <1000000>;
+		regulator-max-microvolt = <1000000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	reg_1p8v: regulator-1p8v {
 		compatible = "regulator-fixed";
 		regulator-name = "fixed-1.8V";
@@ -100,6 +115,11 @@
 	clock-frequency = <24000000>;
 };
 
+&gpu {
+	status = "okay";
+	mali-supply = <&reg_1p0v>;
+};
+
 &i2c0 {
 	pinctrl-0 = <&i2c0_pins>;
 	pinctrl-names = "default";
@@ -157,6 +177,11 @@
 	};
 };
 
+&pcie_port0 {
+	clocks = <&pcie_refclk>;
+	clock-names = "ref";
+};
+
 &pinctrl {
 	eth0_pins: eth0 {
 		txc {
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
index b45acfe6288a7c..462102a620257b 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
@@ -168,6 +168,14 @@
 	};
 };
 
+&i3c {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i3c_pins>;
+	i2c-scl-hz = <400000>;
+	i3c-scl-hz = <12500000>;
+	status = "okay";
+};
+
 &pcie_port0 {
 	clocks = <&versa3 5>;
 	clock-names = "ref";
@@ -184,7 +192,6 @@
 	bus-width = <4>;
 	sd-uhs-sdr50;
 	sd-uhs-sdr104;
-	max-frequency = <125000000>;
 	status = "okay";
 };
 #else
@@ -199,7 +206,6 @@
 	mmc-hs200-1_8v;
 	non-removable;
 	fixed-emmc-driver-type = <1>;
-	max-frequency = <125000000>;
 	status = "okay";
 };
 #endif
@@ -210,7 +216,6 @@
 	pinctrl-names = "default";
 	vmmc-supply = <&vcc_sdhi2>;
 	bus-width = <4>;
-	max-frequency = <50000000>;
 	status = "okay";
 };
 #endif
@@ -234,7 +239,7 @@
 			drive-strength-microamp = <5200>;
 		};
 
-		tx_ctl {
+		tx-ctl {
 			pinmux = <RZG2L_PORT_PINMUX(1, 1, 1)>;  /* ET0_TX_CTL */
 			power-source = <1800>;
 			output-enable;
@@ -277,7 +282,7 @@
 			drive-strength-microamp = <5200>;
 		};
 
-		tx_ctl {
+		tx-ctl {
 			pinmux = <RZG2L_PORT_PINMUX(7, 1, 1)>;	/* ET1_TX_CTL */
 			power-source = <1800>;
 			output-enable;
@@ -302,6 +307,16 @@
 		};
 	};
 
+	i3c_pins: i3c {
+		pins = "I3C_SDA", "I3C_SCL";
+#if SW_CONFIG4 == SW_ON
+		power-source = <1200>;
+#else
+		power-source = <1800>;
+#endif
+		input-enable;
+	};
+
 	sdhi0_pins: sd0 {
 		data {
 			pins = "SD0_DATA0", "SD0_DATA1", "SD0_DATA2", "SD0_DATA3";
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc-switches.h b/arch/arm64/boot/dts/renesas/rzg3s-smarc-switches.h
index bbf908a5322c91..9cccc87da05750 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc-switches.h
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc-switches.h
@@ -25,9 +25,13 @@
  * @SW_CONFIG3:
  *	SW_OFF - SD2 is connected to SoC
  *	SW_ON  - SCIF1, SSI0, IRQ0, IRQ1 connected to SoC
+ * @SW_CONFIG4:
+ *	SW_OFF - I3C voltage is 1.8V
+ *	SW_ON  - I3C voltage is 1.2V
  */
 #define SW_CONFIG2	SW_OFF
 #define SW_CONFIG3	SW_ON
+#define SW_CONFIG4	SW_OFF
 
 /*
  * SW_OPT_MUX[x] switches' states:
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
index 70af605168b07c..e3821d8c01e30a 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
@@ -285,7 +285,6 @@
 	bus-width = <4>;
 	sd-uhs-sdr50;
 	sd-uhs-sdr104;
-	max-frequency = <125000000>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
index ffcc0960903bf4..224d133e3a0f7b 100644
--- a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-common.dtsi
@@ -20,6 +20,7 @@
 		i2c1 = &i2c1;
 		mmc0 = &sdhi0;
 		mmc1 = &sdhi1;
+		rtc0 = &rtc0;
 		serial0 = &sci0;
 		spi0 = &xspi0;
 		spi1 = &xspi1;
@@ -514,6 +515,10 @@
 	};
 };
 
+&rtc0 {
+	status = "okay";
+};
+
 &sci0 {
 	pinctrl-0 = <&sci0_pins>;
 	pinctrl-names = "default";
@@ -647,11 +652,13 @@
 			partition@0 {
 				label = "bl2-1";
 				reg = <0x00000000 0x00060000>;
+				read-only;
 			};
 
 			partition@60000 {
 				label = "fip-1";
 				reg = <0x00060000 0x007a0000>;
+				read-only;
 			};
 
 			partition@800000 {
diff --git a/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-du-adv7513.dtsi b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-du-adv7513.dtsi
new file mode 100644
index 00000000000000..d0242dc5516100
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/rzt2h-n2h-evk-du-adv7513.dtsi
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DT overlay common parts for the RZ/{T2H/N2H} EVKs with ADV7513
+ * transmitter connected to DU enabled.
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+#include <dt-bindings/pinctrl/renesas,r9a09g077-pinctrl.h>
+
+#define ADV7513_PARENT_I2C i2c1
+#include "rz-smarc-du-adv7513.dtsi"
+
+&pinctrl {
+	du_pins: du-group {
+		du-data-pins {
+			pinmux = <RZT2H_PORT_PINMUX(10, 0, 0x28)>, /* LCDC_DATR0 */
+				 <RZT2H_PORT_PINMUX(10, 1, 0x28)>, /* LCDC_DATR1 */
+				 <RZT2H_PORT_PINMUX(10, 2, 0x28)>, /* LCDC_DATR2 */
+				 <RZT2H_PORT_PINMUX(10, 3, 0x28)>, /* LCDC_DATR3 */
+				 <RZT2H_PORT_PINMUX(10, 4, 0x28)>, /* LCDC_DATR4 */
+				 <RZT2H_PORT_PINMUX(10, 5, 0x28)>, /* LCDC_DATR5 */
+				 <RZT2H_PORT_PINMUX(10, 6, 0x28)>, /* LCDC_DATR6 */
+				 <RZT2H_PORT_PINMUX(10, 7, 0x28)>, /* LCDC_DATR7 */
+				 <RZT2H_PORT_PINMUX(11, 0, 0x28)>, /* LCDC_DATG0 */
+				 <RZT2H_PORT_PINMUX(14, 1, 0x28)>, /* LCDC_DATG1 */
+				 <RZT2H_PORT_PINMUX(14, 2, 0x28)>, /* LCDC_DATG2 */
+				 <RZT2H_PORT_PINMUX(14, 3, 0x28)>, /* LCDC_DATG3 */
+				 <RZT2H_PORT_PINMUX(14, 4, 0x28)>, /* LCDC_DATG4 */
+				 <RZT2H_PORT_PINMUX(14, 5, 0x28)>, /* LCDC_DATG5 */
+				 <RZT2H_PORT_PINMUX(14, 6, 0x28)>, /* LCDC_DATG6 */
+				 <RZT2H_PORT_PINMUX(17, 6, 0x28)>, /* LCDC_DATG7 */
+				 <RZT2H_PORT_PINMUX(17, 7, 0x28)>, /* LCDC_DATB0 */
+				 <RZT2H_PORT_PINMUX(18, 0, 0x28)>, /* LCDC_DATB1 */
+				 <RZT2H_PORT_PINMUX(18, 1, 0x28)>, /* LCDC_DATB2 */
+				 <RZT2H_PORT_PINMUX(18, 2, 0x28)>, /* LCDC_DATB3 */
+				 <RZT2H_PORT_PINMUX(18, 3, 0x28)>, /* LCDC_DATB4 */
+				 <RZT2H_PORT_PINMUX(18, 4, 0x28)>, /* LCDC_DATB5 */
+				 <RZT2H_PORT_PINMUX(18, 5, 0x28)>, /* LCDC_DATB6 */
+				 <RZT2H_PORT_PINMUX(18, 6, 0x28)>; /* LCDC_DATB7 */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+		};
+
+		du-clk-pins {
+			pinmux = <RZT2H_PORT_PINMUX(9, 4, 0x28)>; /* LCDC_CLK */
+			drive-strength-microamp = <11800>;
+			slew-rate = <1>;
+		};
+
+		du-sync-pins {
+			pinmux = <RZT2H_PORT_PINMUX(9, 5, 0x28)>, /* LCDC_HSYNC */
+				 <RZT2H_PORT_PINMUX(9, 6, 0x28)>; /* LCDC_VSYNC */
+			drive-strength-microamp = <11800>;
+			slew-rate = <0>;
+		};
+
+		du-de-pins {
+			pinmux = <RZT2H_PORT_PINMUX(9, 7, 0x28)>; /* LCDC_DE */
+			drive-strength-microamp = <11800>;
+			slew-rate = <0>;
+		};
+	};
+
+	adv7513_irq_pins: adv7513-irq-pins {
+	};
+};
+
+&adv7513 {
+	pinctrl-0 = <&adv7513_irq_pins>;
+	pinctrl-names = "default";
+};
diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
index 1317ede2f719be..b9bdc366088eea 100644
--- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
@@ -543,7 +543,7 @@
 		reg = <0x70 0x71 0x72 0x73 0x74 0x75
 		       0x60 0x61 0x62 0x63 0x64 0x65>;
 		reg-names = "main", "dpll", "cp", "hdmi", "edid", "repeater",
-			    "infoframe", "cbus", "cec", "sdp", "txa", "txb" ;
+			    "infoframe", "cbus", "cec", "sdp", "txa", "txb";
 
 		interrupts-extended = <&gpio6 30 IRQ_TYPE_LEVEL_LOW>,
 				      <&gpio6 31 IRQ_TYPE_LEVEL_LOW>;
@@ -686,12 +686,12 @@
 			function = "avb";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb_mdio";
 			drive-strength = <24>;
 		};
 
-		pins_mii_tx {
+		pins-mii-tx {
 			pins = "PIN_AVB_TX_CTL", "PIN_AVB_TXC", "PIN_AVB_TD0",
 			       "PIN_AVB_TD1", "PIN_AVB_TD2", "PIN_AVB_TD3";
 			drive-strength = <12>;
@@ -738,7 +738,7 @@
 		function = "scif2";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_a";
 		function = "scif_clk";
 	};
@@ -749,7 +749,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
@@ -767,7 +767,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi3_pins_uhs: sd3_uhs {
+	sdhi3_pins_uhs: sd3-uhs {
 		groups = "sdhi3_data4", "sdhi3_ctrl";
 		function = "sdhi3";
 		power-source = <1800>;
@@ -778,7 +778,7 @@
 		function = "ssi";
 	};
 
-	sound_clk_pins: sound_clk {
+	sound_clk_pins: sound-clk {
 		groups = "audio_clk_a_a", "audio_clk_b_a", "audio_clk_c_a",
 			 "audio_clkout_a", "audio_clkout3_a";
 		function = "audio_clk";
diff --git a/arch/arm64/boot/dts/renesas/salvator-xs.dtsi b/arch/arm64/boot/dts/renesas/salvator-xs.dtsi
index 1d18dedb1ff039..f5157940bd60cc 100644
--- a/arch/arm64/boot/dts/renesas/salvator-xs.dtsi
+++ b/arch/arm64/boot/dts/renesas/salvator-xs.dtsi
@@ -72,7 +72,7 @@
 	 *	- Connect GP6_3[01] to BD082065 (USB2.0 ch3's host power).
 	 *	- Connect GP6_{04,21} to ADV7842.
 	 */
-	usb2_ch3_pins: usb2_ch3 {
+	usb2_ch3_pins: usb2-ch3 {
 		groups = "usb2_ch3";
 		function = "usb2_ch3";
 	};
diff --git a/arch/arm64/boot/dts/renesas/ulcb-kf-audio-graph-card2-mix+split.dtsi b/arch/arm64/boot/dts/renesas/ulcb-kf-audio-graph-card2-mix+split.dtsi
index 67a0057a3383da..2460ad849b9399 100644
--- a/arch/arm64/boot/dts/renesas/ulcb-kf-audio-graph-card2-mix+split.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb-kf-audio-graph-card2-mix+split.dtsi
@@ -76,14 +76,14 @@
 				 * (H) CPU7
 				 * (I) CPU8
 				 */
-			fe_c:	port@2 { reg = <2>; fe_c_ep: endpoint { remote-endpoint = <&rsnd_c_ep>; }; };
-			fe_d:	port@3 { reg = <3>; fe_d_ep: endpoint { remote-endpoint = <&rsnd_d_ep>; }; };
-			fe_e:	port@4 { reg = <4>; fe_e_ep: endpoint { remote-endpoint = <&rsnd_e_ep>; }; };
-			fe_f:	port@5 { reg = <5>; fe_f_ep: endpoint { remote-endpoint = <&rsnd_f_ep>; }; };
+			fe_c: port@2 { reg = <2>; fe_c_ep: endpoint { remote-endpoint = <&rsnd_c_ep>; }; };
+			fe_d: port@3 { reg = <3>; fe_d_ep: endpoint { remote-endpoint = <&rsnd_d_ep>; }; };
+			fe_e: port@4 { reg = <4>; fe_e_ep: endpoint { remote-endpoint = <&rsnd_e_ep>; }; };
+			fe_f: port@5 { reg = <5>; fe_f_ep: endpoint { remote-endpoint = <&rsnd_f_ep>; }; };
 
-			fe_g:	port@6 { reg = <6>; fe_g_ep: endpoint { remote-endpoint = <&rsnd_g_ep>; }; };
-			fe_h:	port@7 { reg = <7>; fe_h_ep: endpoint { remote-endpoint = <&rsnd_h_ep>; }; };
-			fe_i:	port@8 { reg = <8>; fe_i_ep: endpoint { remote-endpoint = <&rsnd_i_ep>; }; };
+			fe_g: port@6 { reg = <6>; fe_g_ep: endpoint { remote-endpoint = <&rsnd_g_ep>; }; };
+			fe_h: port@7 { reg = <7>; fe_h_ep: endpoint { remote-endpoint = <&rsnd_h_ep>; }; };
+			fe_i: port@8 { reg = <8>; fe_i_ep: endpoint { remote-endpoint = <&rsnd_i_ep>; }; };
 			};
 
 			ports@1 {
@@ -96,8 +96,8 @@
 				 * (Y) PCM3168A-p
 				 * (Z) PCM3168A-c
 				 */
-			be_y:	port@0 { reg = <0>; be_y_ep: endpoint { remote-endpoint = <&pcm3168a_y_ep>; }; };
-			be_z:	port@1 { reg = <1>; be_z_ep: endpoint { remote-endpoint = <&pcm3168a_z_ep>; }; };
+			be_y: port@0 { reg = <0>; be_y_ep: endpoint { remote-endpoint = <&pcm3168a_y_ep>; }; };
+			be_z: port@1 { reg = <1>; be_z_ep: endpoint { remote-endpoint = <&pcm3168a_z_ep>; }; };
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
index 97014bcfbb1d22..603c791e438db6 100644
--- a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
@@ -84,7 +84,7 @@
 		regulator-always-on;
 	};
 
-	wlan_en: regulator-wlan_en {
+	wlan_en: regulator-wlan-en {
 		compatible = "regulator-fixed";
 		regulator-name = "wlan-en-regulator";
 
diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi
index 119f2b5024b3d9..cedff8e0157edf 100644
--- a/arch/arm64/boot/dts/renesas/ulcb.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi
@@ -311,12 +311,12 @@
 			function = "avb";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb_mdio";
 			drive-strength = <24>;
 		};
 
-		pins_mii_tx {
+		pins-mii-tx {
 			pins = "PIN_AVB_TX_CTL", "PIN_AVB_TXC", "PIN_AVB_TD0",
 			       "PIN_AVB_TD1", "PIN_AVB_TD2", "PIN_AVB_TD3";
 			drive-strength = <12>;
@@ -338,7 +338,7 @@
 		function = "scif2";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk_a";
 		function = "scif_clk";
 	};
@@ -349,7 +349,7 @@
 		power-source = <3300>;
 	};
 
-	sdhi0_pins_uhs: sd0_uhs {
+	sdhi0_pins_uhs: sd0-uhs {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <1800>;
diff --git a/arch/arm64/boot/dts/renesas/white-hawk-cpu-common.dtsi b/arch/arm64/boot/dts/renesas/white-hawk-cpu-common.dtsi
index c5045bda45c338..2bcd486b03e93b 100644
--- a/arch/arm64/boot/dts/renesas/white-hawk-cpu-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/white-hawk-cpu-common.dtsi
@@ -320,12 +320,12 @@
 			function = "avb0";
 		};
 
-		pins_mdio {
+		pins-mdio {
 			groups = "avb0_mdio";
 			drive-strength = <21>;
 		};
 
-		pins_mii {
+		pins-mii {
 			groups = "avb0_rgmii";
 			drive-strength = <21>;
 		};
@@ -368,7 +368,7 @@
 		function = "qspi0";
 	};
 
-	scif_clk_pins: scif_clk {
+	scif_clk_pins: scif-clk {
 		groups = "scif_clk";
 		function = "scif_clk";
 	};
diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile
index e7936a21c11851..b677914cd3f161 100644
--- a/arch/arm64/boot/dts/rockchip/Makefile
+++ b/arch/arm64/boot/dts/rockchip/Makefile
@@ -197,8 +197,10 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-firefly-itx-3588j.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-friendlyelec-cm3588-nas.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-h96-max-v58.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-jaguar.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-jaguar-can1-can2-uart4.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-jaguar-ethernet-switch.dtbo
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-jaguar-pre-ict-tester.dtbo
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-lubancat-5io.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-mnt-reform2.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-nanopc-t6.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-nanopc-t6-lts.dtb
@@ -319,6 +321,10 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-edgeble-neu6b-wifi.dtb
 rk3588-edgeble-neu6b-wifi-dtbs := rk3588-edgeble-neu6b-io.dtb \
 	rk3588-edgeble-neu6a-wifi.dtbo
 
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-jaguar-can1-can2-uart4.dtb
+rk3588-jaguar-can1-can2-uart4-dtbs := rk3588-jaguar.dtb \
+	rk3588-jaguar-can1-can2-uart4.dtbo
+
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3588-jaguar-ethernet-switch.dtb
 rk3588-jaguar-ethernet-switch-dtbs := rk3588-jaguar.dtb \
 	rk3588-jaguar-ethernet-switch.dtbo
diff --git a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi
index 02200de695d31b..3ae65dbcfffe7f 100644
--- a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi
+++ b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi
@@ -6,6 +6,7 @@
 /dts-v1/;
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 #include "px30.dtsi"
 
@@ -413,7 +414,7 @@
 		dvdd-supply = <&vcc_cam_dvdd>;
 		dovdd-supply = <&vcc_cam_dovdd>;
 		lens-focus = <&focus>;
-		orientation = <0>;
+		orientation = <MEDIA_ORIENTATION_FRONT>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&cif_clkout_m0 &cam_pwdn>;
 		reset-gpios = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso b/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso
index d0725595ade073..1813f06bbff530 100644
--- a/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso
+++ b/arch/arm64/boot/dts/rockchip/px30-ringneck-haikou-video-demo.dtso
@@ -16,6 +16,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 
 &{/} {
@@ -185,7 +186,7 @@
 		dvdd-supply = <&cam_dvdd_1v2>;
 		dovdd-supply = <&cam_dovdd_1v8>;
 		lens-focus = <&focus>;
-		orientation = <0>;
+		orientation = <MEDIA_ORIENTATION_FRONT>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&cif_clkout_m0>;
 		reset-gpios = <&pca9670 6 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
index d46cdfe3f7849a..1a36d54ddfa271 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
@@ -13,6 +13,7 @@
 #include <dt-bindings/input/gpio-keys.h>
 #include <dt-bindings/input/linux-event-codes.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include "rk3399-s.dtsi"
 
 / {
@@ -455,7 +456,7 @@
 		reg = <0x1a>;
 		clocks = <&cru SCLK_CIF_OUT>; /* MIPI_MCLK0, derived from CIF_CLKO */
 		lens-focus = <&wcam_lens>;
-		orientation = <1>; /* V4L2_CAMERA_ORIENTATION_BACK */
+		orientation = <MEDIA_ORIENTATION_BACK>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&camera_rst_l>;
 		reset-gpios = <&gpio1 RK_PA0 GPIO_ACTIVE_LOW>;
@@ -487,7 +488,7 @@
 		clocks = <&cru SCLK_CIF_OUT>; /* MIPI_MCLK1, derived from CIF_CLK0 */
 		clock-names = "xvclk";
 		dovdd-supply = <&vcc1v8_dvp>;
-		orientation = <0>; /* V4L2_CAMERA_ORIENTATION_FRONT */
+		orientation = <MEDIA_ORIENTATION_FRONT>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&camera2_rst_l &dvp_pdn0_h>;
 		powerdown-gpios = <&gpio2 RK_PB4 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi b/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi
index 64e6ba3457397a..074f09c6d11c8a 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi
@@ -296,6 +296,8 @@
 
 &i2c2 {
 	clock-frequency = <400000>;
+	i2c-scl-falling-time-ns = <15>;
+	i2c-scl-rising-time-ns = <450>;
 	status = "okay";
 
 	hym8563: rtc@51 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-armsom-cm5-io.dts b/arch/arm64/boot/dts/rockchip/rk3576-armsom-cm5-io.dts
index ebb231271e457f..33596322784865 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-armsom-cm5-io.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-armsom-cm5-io.dts
@@ -5,8 +5,8 @@
  * Carrier provides:
  *   - Full-size HDMI 2.1 (HDMI TX via HDPTX PHY)
  *   - Gigabit Ethernet RJ45 (GMAC0 + on-module YT8531)
- *   - 2x USB-A behind a USB3 hub (combphy1 + u2phy1)
- *   - USB-C with PD/altmode (FUSB302 on i2c0, combphy0 + USBDP PHY)
+ *   - 4x USB-A behind a USB3 hub (combphy1 + u2phy1)
+ *   - USB-C with PD/altmode (FUSB302 on i2c0, USBDP PHY)
  *   - M.2 PCIe 2.0 x1 slot (combphy0 as PCIe lane)
  *   - microSD card slot
  *   - Green/red status LEDs, 40-pin RPi-compatible header
diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
index b0c0d3c8b1b142..e87ace5d05e60f 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
@@ -21,6 +21,11 @@
 	#size-cells = <2>;
 
 	aliases {
+		gpio0 = &gpio0;
+		gpio1 = &gpio1;
+		gpio2 = &gpio2;
+		gpio3 = &gpio3;
+		gpio4 = &gpio4;
 		i2c0 = &i2c0;
 		i2c1 = &i2c1;
 		i2c2 = &i2c2;
@@ -970,7 +975,7 @@
 
 		sdgmac_grf: syscon@26038000 {
 			compatible = "rockchip,rk3576-sdgmac-grf", "syscon";
-			reg = <0x0 0x26038000 0x0 0x1000>;
+			reg = <0x0 0x26038000 0x0 0x2000>;
 		};
 
 		ioc_grf: syscon@26040000 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
index 376ad04e07869c..165245f5091dee 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
@@ -2648,6 +2648,45 @@
 		#dma-cells = <1>;
 	};
 
+	can0: can@fea50000 {
+		compatible = "rockchip,rk3588-canfd";
+		reg = <0x0 0xfea50000 0x0 0x1000>;
+		interrupts = <GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH 0>;
+		clocks = <&cru CLK_CAN0>, <&cru PCLK_CAN0>;
+		clock-names = "baud", "pclk";
+		resets = <&cru SRST_CAN0>, <&cru SRST_P_CAN0>;
+		reset-names = "core", "apb";
+		pinctrl-names = "default";
+		pinctrl-0 = <&can0m0_pins>;
+		status = "disabled";
+	};
+
+	can1: can@fea60000 {
+		compatible = "rockchip,rk3588-canfd";
+		reg = <0x0 0xfea60000 0x0 0x1000>;
+		interrupts = <GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH 0>;
+		clocks = <&cru CLK_CAN1>, <&cru PCLK_CAN1>;
+		clock-names = "baud", "pclk";
+		resets = <&cru SRST_CAN1>, <&cru SRST_P_CAN1>;
+		reset-names = "core", "apb";
+		pinctrl-names = "default";
+		pinctrl-0 = <&can1m0_pins>;
+		status = "disabled";
+	};
+
+	can2: can@fea70000 {
+		compatible = "rockchip,rk3588-canfd";
+		reg = <0x0 0xfea70000 0x0 0x1000>;
+		interrupts = <GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH 0>;
+		clocks = <&cru CLK_CAN2>, <&cru PCLK_CAN2>;
+		clock-names = "baud", "pclk";
+		resets = <&cru SRST_CAN2>, <&cru SRST_P_CAN2>;
+		reset-names = "core", "apb";
+		pinctrl-names = "default";
+		pinctrl-0 = <&can2m0_pins>;
+		status = "disabled";
+	};
+
 	i2c1: i2c@fea90000 {
 		compatible = "rockchip,rk3588-i2c", "rockchip,rk3399-i2c";
 		reg = <0x0 0xfea90000 0x0 0x1000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-jaguar-can1-can2-uart4.dtso b/arch/arm64/boot/dts/rockchip/rk3588-jaguar-can1-can2-uart4.dtso
new file mode 100644
index 00000000000000..6c21bf78591b3f
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3588-jaguar-can1-can2-uart4.dtso
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2026 Cherry Embedded Solutions GmbH
+ *
+ * Device Tree Overlay for CAN1-CAN2-UART4 adapter for the Mezzanine connector
+ * on RK3588 Jaguar.
+ *
+ * This adapter has two CAN transceivers and one RS232 transceiver.
+ */
+
+/dts-v1/;
+/plugin/;
+
+&can1 {
+	status = "okay";
+};
+
+&can2 {
+	status = "okay";
+};
+
+/* RS232 */
+&uart4 {
+	/* ST3232B only guarantees 250Kbps */
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
index 41758fe7d36899..0446834e84d5db 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
@@ -195,6 +195,22 @@
 	};
 };
 
+&can0 {
+	assigned-clocks = <&cru CLK_CAN0>;
+	assigned-clock-rates = <300000000>;
+	status = "okay";
+};
+
+&can1 {
+	assigned-clocks = <&cru CLK_CAN1>;
+	assigned-clock-rates = <300000000>;
+};
+
+&can2 {
+	assigned-clocks = <&cru CLK_CAN2>;
+	assigned-clock-rates = <300000000>;
+};
+
 &combphy1_ps {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-lubancat-5-btb.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-lubancat-5-btb.dtsi
new file mode 100644
index 00000000000000..106d0c495f4354
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3588-lubancat-5-btb.dtsi
@@ -0,0 +1,534 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2025 Cunhao Lu <1579567540@qq.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	compatible = "embedfire,lubancat-5-btb", "rockchip,rk3588";
+
+	aliases {
+		mmc0 = &sdhci;
+	};
+
+	combophy_avdd0v85: regulator-combophy-avdd0v85 {
+		compatible = "regulator-fixed";
+		regulator-boot-on;
+		regulator-max-microvolt = <850000>;
+		regulator-min-microvolt = <850000>;
+		regulator-name = "combophy_avdd0v85";
+		vin-supply = <&avdd_0v85_s0>;
+	};
+
+	combophy_avdd1v8: regulator-combophy-avdd1v8 {
+		compatible = "regulator-fixed";
+		regulator-boot-on;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "combophy_avdd1v8";
+		vin-supply = <&avcc_1v8_s0>;
+	};
+
+	vcc_1v1_nldo_s3: regulator-vcc-1v1-nldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1100000>;
+		regulator-min-microvolt = <1100000>;
+		regulator-name = "vcc_1v1_nldo_s3";
+		vin-supply = <&vcc4v0_sys>;
+	};
+};
+
+&can1 {
+	assigned-clock-rates = <300000000>;
+	assigned-clocks = <&cru CLK_CAN1>;
+};
+
+&can2 {
+	assigned-clock-rates = <300000000>;
+	assigned-clocks = <&cru CLK_CAN2>;
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_big0_s0>;
+};
+
+&cpu_b1 {
+	cpu-supply = <&vdd_cpu_big0_s0>;
+};
+
+&cpu_b2 {
+	cpu-supply = <&vdd_cpu_big1_s0>;
+};
+
+&cpu_b3 {
+	cpu-supply = <&vdd_cpu_big1_s0>;
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l1 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l2 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_l3 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu_s0>;
+	status = "okay";
+};
+
+&i2c0 {
+	pinctrl-0 = <&i2c0m2_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	vdd_cpu_big0_s0: regulator@42 {
+		compatible = "rockchip,rk8602";
+		reg = <0x42>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1050000>;
+		regulator-min-microvolt = <550000>;
+		regulator-name = "vdd_cpu_big0_s0";
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc4v0_sys>;
+		fcs,suspend-voltage-selector = <1>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vdd_cpu_big1_s0: regulator@43 {
+		compatible = "rockchip,rk8603", "rockchip,rk8602";
+		reg = <0x43>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <1050000>;
+		regulator-min-microvolt = <550000>;
+		regulator-name = "vdd_cpu_big1_s0";
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc4v0_sys>;
+		fcs,suspend-voltage-selector = <1>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-0 = <&i2c1m2_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	vdd_npu_s0: regulator@42 {
+		compatible = "rockchip,rk8602";
+		reg = <0x42>;
+		regulator-boot-on;
+		regulator-max-microvolt = <950000>;
+		regulator-min-microvolt = <550000>;
+		regulator-name = "vdd_npu_s0";
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc4v0_sys>;
+		fcs,suspend-voltage-selector = <1>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&pd_gpu {
+	domain-supply = <&vdd_gpu_s0>;
+};
+
+&pd_npu {
+	domain-supply = <&vdd_npu_s0>;
+};
+
+&rknn_core_0 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_core_1 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_core_2 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_mmu_0 {
+	status = "okay";
+};
+
+&rknn_mmu_1 {
+	status = "okay";
+};
+
+&rknn_mmu_2 {
+	status = "okay";
+};
+
+&sdhci {
+	bus-width = <8>;
+	max-frequency = <200000000>;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	non-removable;
+	status = "okay";
+};
+
+&sfc {
+	status = "disabled";
+};
+
+&spi2 {
+	assigned-clock-rates = <200000000>;
+	assigned-clocks = <&cru CLK_SPI2>;
+	num-cs = <1>;
+	pinctrl-0 = <&spi2m2_cs0 &spi2m2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pmic@0 {
+		compatible = "rockchip,rk806";
+		reg = <0x0>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA7 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&pmic_pins>, <&rk806_dvs1_null>,
+			    <&rk806_dvs2_null>, <&rk806_dvs3_null>;
+		pinctrl-names = "default";
+		spi-max-frequency = <1000000>;
+		system-power-controller;
+
+		vcc1-supply = <&vcc4v0_sys>;
+		vcc2-supply = <&vcc4v0_sys>;
+		vcc3-supply = <&vcc4v0_sys>;
+		vcc4-supply = <&vcc4v0_sys>;
+		vcc5-supply = <&vcc4v0_sys>;
+		vcc6-supply = <&vcc4v0_sys>;
+		vcc7-supply = <&vcc4v0_sys>;
+		vcc8-supply = <&vcc4v0_sys>;
+		vcc9-supply = <&vcc4v0_sys>;
+		vcc10-supply = <&vcc4v0_sys>;
+		vcc11-supply = <&vcc_2v0_pldo_s3>;
+		vcc12-supply = <&vcc4v0_sys>;
+		vcc13-supply = <&vcc_1v1_nldo_s3>;
+		vcc14-supply = <&vcc_1v1_nldo_s3>;
+		vcca-supply = <&vcc4v0_sys>;
+
+		rk806_dvs1_null: dvs1-null-pins {
+			function = "pin_fun0";
+			pins = "gpio_pwrctrl1";
+		};
+
+		rk806_dvs2_null: dvs2-null-pins {
+			function = "pin_fun0";
+			pins = "gpio_pwrctrl2";
+		};
+
+		rk806_dvs3_null: dvs3-null-pins {
+			function = "pin_fun0";
+			pins = "gpio_pwrctrl3";
+		};
+
+		regulators {
+			vdd_gpu_s0: dcdc-reg1 {
+				regulator-boot-on;
+				regulator-enable-ramp-delay = <400>;
+				regulator-max-microvolt = <950000>;
+				regulator-min-microvolt = <550000>;
+				regulator-name = "vdd_gpu_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_cpu_lit_s0: dcdc-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <950000>;
+				regulator-min-microvolt = <550000>;
+				regulator-name = "vdd_cpu_lit_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_log_s0: dcdc-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <800000>;
+				regulator-min-microvolt = <675000>;
+				regulator-name = "vdd_log_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vdd_vdenc_s0: dcdc-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <950000>;
+				regulator-min-microvolt = <550000>;
+				regulator-name = "vdd_vdenc_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_ddr_s0: dcdc-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <900000>;
+				regulator-min-microvolt = <675000>;
+				regulator-name = "vdd_ddr_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <850000>;
+				};
+			};
+
+			vdd2_ddr_s3: dcdc-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vdd2_ddr_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_2v0_pldo_s3: dcdc-reg7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <2000000>;
+				regulator-min-microvolt = <2000000>;
+				regulator-name = "vcc_2v0_pldo_s3";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <2000000>;
+				};
+			};
+
+			vcc_3v3_s3: dcdc-reg8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <3300000>;
+				regulator-name = "vcc_3v3_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vddq_ddr_s0: dcdc-reg9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vddq_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v8_s3: dcdc-reg10 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd_0v75_s3: nldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <750000>;
+				regulator-min-microvolt = <750000>;
+				regulator-name = "vdd_0v75_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			avdd_ddr_pll_s0: nldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <850000>;
+				regulator-min-microvolt = <850000>;
+				regulator-name = "avdd_ddr_pll_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <850000>;
+				};
+			};
+
+			avdd_0v75_s0: nldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <837500>;
+				regulator-min-microvolt = <837500>;
+				regulator-name = "avdd_0v75_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			avdd_0v85_s0: nldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <850000>;
+				regulator-min-microvolt = <850000>;
+				regulator-name = "avdd_0v85_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_0v75_s0: nldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <750000>;
+				regulator-min-microvolt = <750000>;
+				regulator-name = "vdd_0v75_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			avcc_1v8_s0: pldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "avcc_1v8_s0";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vcc_1v8_s0: pldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			avdd_1v2_s0: pldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1200000>;
+				regulator-min-microvolt = <1200000>;
+				regulator-name = "avdd_1v2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			avcc_3v3_s0: pldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <3300000>;
+				regulator-name = "avcc_3v3_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd_s0: pldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "vccio_sd_s0";
+				regulator-ramp-delay = <12500>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			pldo6_s3: pldo-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1800000>;
+				regulator-min-microvolt = <1800000>;
+				regulator-name = "pldo6_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+		};
+	};
+};
+
+&tsadc {
+	status = "okay";
+};
+
+&wdt {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-lubancat-5io.dts b/arch/arm64/boot/dts/rockchip/rk3588-lubancat-5io.dts
new file mode 100644
index 00000000000000..19598be2cfad1c
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3588-lubancat-5io.dts
@@ -0,0 +1,1032 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2025 Cunhao Lu <1579567540@qq.com>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
+#include <dt-bindings/usb/pd.h>
+#include "rk3588.dtsi"
+#include "rk3588-lubancat-5-btb.dtsi"
+
+/ {
+	compatible = "embedfire,lubancat-5io", "embedfire,lubancat-5-btb",
+		     "rockchip,rk3588";
+	model = "EmbedFire LubanCat 5IO";
+
+	aliases {
+		ethernet0 = &gmac0;
+		ethernet1 = &gmac1;
+		mmc1 = &sdmmc;
+		rtc0 = &hym8563;
+	};
+
+	chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 1>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <100>;
+
+		button-recovery {
+			label = "Recovery";
+			press-threshold-microvolt = <1750>;
+			linux,code = <KEY_VENDOR>;
+		};
+	};
+
+	analog-sound {
+		compatible = "simple-audio-card";
+		pinctrl-0 = <&hp_detect>;
+		pinctrl-names = "default";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,hp-det-gpios =
+			<&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "rk3588-es8388";
+		simple-audio-card,pin-switches = "Headphones", "Speaker";
+		simple-audio-card,routing =
+			"Headphones", "LOUT1",
+			"Headphones", "ROUT1",
+			"Speaker", "LOUT2",
+			"Speaker", "ROUT2",
+			"LINPUT1", "Main Mic",
+			"LINPUT2", "Main Mic",
+			"RINPUT1", "Headset Mic",
+			"RINPUT2", "Headset Mic";
+		simple-audio-card,widgets =
+			"Headphone", "Headphones",
+			"Speaker", "Speaker",
+			"Microphone", "Main Mic",
+			"Microphone", "Headset Mic";
+
+		simple-audio-card,codec {
+			sound-dai = <&es8388>;
+			system-clock-frequency = <12288000>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s0_8ch>;
+		};
+	};
+
+	hdmi0-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi0_con_in: endpoint {
+				remote-endpoint = <&hdmi0_out_con>;
+			};
+		};
+	};
+
+	hdmi1-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi1_con_in: endpoint {
+				remote-endpoint = <&hdmi1_out_con>;
+			};
+		};
+	};
+
+	ir-receiver {
+		compatible = "gpio-ir-receiver";
+		gpios = <&gpio0 RK_PD0 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&ir_receiver_pin>;
+		pinctrl-names = "default";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-0 = <&sys_led_pin>;
+		pinctrl-names = "default";
+
+		led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio1 RK_PC6 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	fan: pwm-fan {
+		compatible = "pwm-fan";
+		#cooling-cells = <2>;
+		cooling-levels = <0 100 150 200 255>;
+		fan-supply = <&vcc_5v0_s0>;
+		pwms = <&pwm11 0 5000 0>;
+	};
+
+	regulator-mipi-dsi0-power {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio1 RK_PC0 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&mipi_dsi0_power_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-name = "mipi_dsi0_power";
+		vin-supply = <&vcc_5v0_s0>;
+	};
+
+	regulator-mipi-dsi1-power {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&mipi_dsi1_power_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-name = "mipi_dsi1_power";
+		vin-supply = <&vcc_5v0_s0>;
+	};
+
+	sata_power: regulator-sata-power {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio1 RK_PD3 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&sata_power_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-name = "sata_power";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vbus5v0_typec: regulator-vbus5v0-typec {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio4 RK_PA3 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&typec_vbus_en>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vbus5v0_typec";
+		vin-supply = <&vcc5v0_usb_s0>;
+	};
+
+	vcc3v3_ekey_pcie: regulator-vcc3v3-ekey-pcie {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc3v3_ekey_pcie_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc3v3_ekey_pcie";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc3v3_m2_pcie: regulator-vcc3v3-m2-pcie {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc3v3_m2_pcie_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc3v3_m2_pcie";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc3v3_mini_pcie: regulator-vcc3v3-mini-pcie {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio1 RK_PD5 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc3v3_mini_pcie_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc3v3_mini_pcie";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc3v3_pciex4: regulator-vcc3v3-pciex4 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc3v3_pcie_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc3v3_pciex4";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc4v0_sys: regulator-vcc4v0-sys {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <4000000>;
+		regulator-min-microvolt = <4000000>;
+		regulator-name = "vcc4v0_sys";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc5v0_usb20_host: regulator-vcc5v0-usb20-host {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio2 RK_PC3 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc5v0_usb20_host_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc5v0_usb20_host";
+		vin-supply = <&vcc5v0_usb_s0>;
+	};
+
+	vcc5v0_usb20_hub: regulator-vcc5v0-usb20-hub {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio4 RK_PA5 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc5v0_usb20_hub_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc5v0_usb20_hub";
+		vin-supply = <&vcc5v0_usb_s0>;
+	};
+
+	vcc5v0_usb30_otg: regulator-vcc5v0-usb30-otg {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio4 RK_PA4 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc5v0_usb30_otg_en>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc5v0_usb30_otg";
+		vin-supply = <&vcc5v0_usb_s0>;
+	};
+
+	vcc5v0_usb_s0: regulator-vcc5v0-usb-s0 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc5v0_usb_s0";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc12v_dcin: regulator-vcc12v-dcin {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <12000000>;
+		regulator-min-microvolt = <12000000>;
+		regulator-name = "vcc12v_dcin";
+	};
+
+	vcc_3v3_s0: regulator-vcc-3v3-s0 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc_3v3_s0";
+		vin-supply = <&vcc12v_dcin>;
+	};
+
+	vcc_3v3_sd_s3: regulator-vcc-3v3-sd-s3 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "vcc_3v3_sd_s3";
+		vin-supply = <&vcc_3v3_s3>;
+	};
+
+	vcc_5v0_s0: regulator-vcc-5v0-s0 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "vcc_5v0_s0";
+		vin-supply = <&vcc12v_dcin>;
+	};
+};
+
+&can1 {
+	pinctrl-0 = <&can1m1_pins>;
+	status = "okay";
+};
+
+&can2 {
+	pinctrl-0 = <&can2m0_pins>;
+	status = "okay";
+};
+
+&combphy0_ps {
+	status = "okay";
+};
+
+&combphy1_ps {
+	status = "okay";
+};
+
+&combphy2_psu {
+	status = "okay";
+};
+
+&dp0 {
+	status = "okay";
+};
+
+&dp0_in {
+	dp0_in_vp2: endpoint {
+		remote-endpoint = <&vp2_out_dp0>;
+	};
+};
+
+&dp0_out {
+	dp0_out_con: endpoint {
+		remote-endpoint = <&usbdp_phy0_dp_in>;
+	};
+};
+
+&gmac0 {
+	clock_in_out = "output";
+	phy-handle = <&rgmii_phy0>;
+	phy-mode = "rgmii-id";
+	phy-supply = <&vcc_3v3_s0>;
+	pinctrl-0 = <&gmac0_miim
+		    &gmac0_tx_bus2
+		    &gmac0_rx_bus2
+		    &gmac0_rgmii_clk
+		    &gmac0_rgmii_bus>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&gmac1 {
+	clock_in_out = "output";
+	phy-handle = <&rgmii_phy1>;
+	phy-mode = "rgmii-id";
+	phy-supply = <&vcc_3v3_s0>;
+	pinctrl-0 = <&gmac1_miim
+		    &gmac1_tx_bus2
+		    &gmac1_rx_bus2
+		    &gmac1_rgmii_clk
+		    &gmac1_rgmii_bus>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdmi0 {
+	frl-enable-gpios = <&gpio1 RK_PA4 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		    &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdmi0_in {
+	hdmi0_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi0>;
+	};
+};
+
+&hdmi0_out {
+	hdmi0_out_con: endpoint {
+		remote-endpoint = <&hdmi0_con_in>;
+	};
+};
+
+&hdmi0_sound {
+	status = "okay";
+};
+
+&hdmi1 {
+	frl-enable-gpios = <&gpio1 RK_PA7 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx1_cec &hdmim0_tx1_hpd
+		    &hdmim0_tx1_scl &hdmim0_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdmi1_in {
+	hdmi1_in_vp1: endpoint {
+		remote-endpoint = <&vp1_out_hdmi1>;
+	};
+};
+
+&hdmi1_out {
+	hdmi1_out_con: endpoint {
+		remote-endpoint = <&hdmi1_con_in>;
+	};
+};
+
+&hdmi1_sound {
+	status = "okay";
+};
+
+&hdmi_receiver_cma {
+	status = "okay";
+};
+
+&hdmi_receiver {
+	hpd-gpios = <&gpio1 RK_PD2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim1_rx_cec &hdmim1_rx_hpdin
+		     &hdmim1_rx_scl &hdmim1_rx_sda
+		     &hdmirx_det>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdptxphy0 {
+	status = "okay";
+};
+
+&hdptxphy1 {
+	status = "okay";
+};
+
+&i2c0 {
+	gpio@21 {
+		compatible = "nxp,pca9535";
+		reg = <0x21>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		vcc-supply = <&vcc_3v3_s0>;
+	};
+
+	hym8563: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+		clock-output-names = "hym8563";
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PB0 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&hym8563_int>;
+		pinctrl-names = "default";
+		wakeup-source;
+	};
+};
+
+&i2c1 {
+	usb-typec@22 {
+		compatible = "fcs,fusb302";
+		reg = <0x22>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PD3 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&usbc0_int>;
+		pinctrl-names = "default";
+		vbus-supply = <&vbus5v0_typec>;
+		status = "okay";
+
+		connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB-C";
+			op-sink-microwatt = <1000000>;
+			power-role = "dual";
+			sink-pdos = <PDO_FIXED(5000, 1000, PDO_FIXED_USB_COMM)>;
+			source-pdos =
+				<PDO_FIXED(5000, 2000, PDO_FIXED_USB_COMM)>;
+			try-power-role = "source";
+
+			altmodes {
+				displayport {
+					svid = /bits/ 16 <0xff01>;
+					vdo = <0xffffffff>;
+				};
+			};
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usbc0_hs: endpoint {
+						remote-endpoint = <&usb_host0_xhci_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usbc0_ss: endpoint {
+						remote-endpoint = <&usbdp_phy0_ss_out>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					usbc0_sbu: endpoint {
+						remote-endpoint = <&usbdp_phy0_dp_out>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2c7 {
+	pinctrl-0 = <&i2c7m0_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	es8388: audio-codec@11 {
+		compatible = "everest,es8388", "everest,es8328";
+		reg = <0x11>;
+		AVDD-supply = <&avcc_3v3_s0>;
+		DVDD-supply = <&vcc_1v8_s3>;
+		HPVDD-supply = <&avcc_3v3_s0>;
+		PVDD-supply = <&vcc_1v8_s3>;
+		assigned-clock-rates = <12288000>;
+		assigned-clocks = <&cru I2S0_8CH_MCLKOUT>;
+		clocks = <&cru I2S0_8CH_MCLKOUT>;
+		#sound-dai-cells = <0>;
+	};
+};
+
+&i2s0_8ch {
+	pinctrl-0 = <&i2s0_lrck
+		    &i2s0_mclk
+		    &i2s0_sclk
+		    &i2s0_sdi0
+		    &i2s0_sdo0>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&i2s5_8ch {
+	status = "okay";
+};
+
+&i2s6_8ch {
+	status = "okay";
+};
+
+&mdio0 {
+	rgmii_phy0: ethernet-phy@0 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x0>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio3 RK_PC0 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&mdio1 {
+	rgmii_phy1: ethernet-phy@0 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x0>;
+		reset-assert-us = <20000>;
+		reset-deassert-us = <100000>;
+		reset-gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&package_thermal {
+	polling-delay = <1000>;
+
+	cooling-maps {
+		map0 {
+			cooling-device = <&fan THERMAL_NO_LIMIT 1>;
+			trip = <&package_fan0>;
+		};
+
+		map1 {
+			cooling-device = <&fan 2 THERMAL_NO_LIMIT>;
+			trip = <&package_fan1>;
+		};
+	};
+
+	trips {
+		package_fan0: package-fan0 {
+			hysteresis = <2000>;
+			temperature = <55000>;
+			type = "active";
+		};
+
+		package_fan1: package-fan1 {
+			hysteresis = <2000>;
+			temperature = <65000>;
+			type = "active";
+		};
+	};
+};
+
+&pcie30phy {
+	data-lanes = <1 1 2 2>;
+	status = "okay";
+};
+
+/* M.2 M-Key (NVMe) */
+&pcie3x4 {
+	num-lanes = <2>;
+	pinctrl-0 = <&pcie3x4_rst>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_m2_pcie>;
+	status = "okay";
+};
+
+/* PCIe x4 slot */
+&pcie3x2 {
+	pinctrl-0 = <&pcie3x2_rst>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio4 RK_PB0 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_pciex4>;
+	status = "okay";
+};
+
+/* M.2 E-Key (WiFi/BT) */
+&pcie2x1l0 {
+	pinctrl-0 = <&pcie2x1l0_rst &ekey_disable>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_ekey_pcie>;
+	status = "okay";
+};
+
+/* Mini-PCIe */
+&pcie2x1l1 {
+	pinctrl-0 = <&pcie2x1l1_rst &minipcie_disable>;
+	pinctrl-names = "default";
+	reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_mini_pcie>;
+	status = "okay";
+};
+
+&pinctrl {
+	hdmi-tx {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <1 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <1 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	hdmirx {
+		hdmirx_det: hdmirx-det {
+			rockchip,pins = <1 RK_PD2 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	hym8563 {
+		hym8563_int: hym8563-int {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	ir-receiver {
+		ir_receiver_pin: ir-receiver-pin {
+			rockchip,pins = <0 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	leds {
+		sys_led_pin: sys-led-pin {
+			rockchip,pins = <1 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	mipi {
+		mipi_dsi0_power_en: mipi-dsi0-power-en {
+			rockchip,pins = <1 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		mipi_dsi1_power_en: mipi-dsi1-power-en {
+			rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pcie2 {
+		ekey_disable: ekey-disable {
+			rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		minipcie_disable: minipcie-disable {
+			rockchip,pins = <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pcie2x1l0_rst: pcie2x1l0-rst {
+			rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pcie2x1l1_rst: pcie2x1l1-rst {
+			rockchip,pins = <4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pcie3 {
+		pcie3x2_rst: pcie3x2-rst {
+			rockchip,pins = <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pcie3x4_rst: pcie3x4-rst {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	sata {
+		sata_power_en: sata-power-en {
+			rockchip,pins = <1 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	sound {
+		hp_detect: hp-detect {
+			rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb-typec {
+		typec_vbus_en: typec-vbus-en {
+			rockchip,pins = <4 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		usbc0_int: usbc0-int {
+			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		vcc3v3_ekey_pcie_en: vcc3v3-ekey-pcie-en {
+			rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		vcc3v3_m2_pcie_en: vcc3v3-m2-pcie-en {
+			rockchip,pins = <2 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		vcc3v3_mini_pcie_en: vcc3v3-mini-pcie-en {
+			rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		vcc3v3_pcie_en: vcc3v3-pcie-en {
+			rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		vcc5v0_usb20_host_en: vcc5v0-usb20-host-en {
+			rockchip,pins = <2 RK_PC3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		vcc5v0_usb20_hub_en: vcc5v0-usb20-hub-en {
+			rockchip,pins = <4 RK_PA5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		vcc5v0_usb30_otg_en: vcc5v0-usb30-otg-en {
+			rockchip,pins = <4 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pwm0 {
+	pinctrl-0 = <&pwm0m2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&pwm1 {
+	pinctrl-0 = <&pwm1m2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&pwm7 {
+	status = "disabled";
+};
+
+&pwm11 {
+	pinctrl-0 = <&pwm11m2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&avcc_1v8_s0>;
+	status = "okay";
+};
+
+&sata0 {
+	status = "okay";
+
+	sata-port@0 {
+		target-supply = <&sata_power>;
+	};
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	max-frequency = <200000000>;
+	no-mmc;
+	no-sdio;
+	pinctrl-0 = <&sdmmc_bus4 &sdmmc_clk &sdmmc_cmd &sdmmc_det>;
+	pinctrl-names = "default";
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc_3v3_sd_s3>;
+	vqmmc-supply = <&vccio_sd_s0>;
+	status = "okay";
+};
+
+&u2phy0 {
+	status = "okay";
+};
+
+&u2phy0_otg {
+	status = "okay";
+};
+
+&u2phy1 {
+	status = "okay";
+};
+
+&u2phy1_otg {
+	phy-supply = <&vcc5v0_usb30_otg>;
+	status = "okay";
+};
+
+&u2phy2 {
+	status = "okay";
+};
+
+&u2phy2_host {
+	phy-supply = <&vcc5v0_usb20_host>;
+	status = "okay";
+};
+
+&u2phy3 {
+	status = "okay";
+};
+
+&u2phy3_host {
+	phy-supply = <&vcc5v0_usb20_hub>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-0 = <&uart4m2_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&uart7 {
+	pinctrl-0 = <&uart7m2_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usbdp_phy0 {
+	mode-switch;
+	orientation-switch;
+	sbu1-dc-gpios = <&gpio1 RK_PB0 GPIO_ACTIVE_HIGH>;
+	sbu2-dc-gpios = <&gpio1 RK_PB1 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		usbdp_phy0_ss_out: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&usbc0_ss>;
+		};
+
+		usbdp_phy0_ss_in: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&usb_host0_xhci_ss>;
+		};
+
+		usbdp_phy0_dp_in: endpoint@2 {
+			reg = <2>;
+			remote-endpoint = <&dp0_out_con>;
+		};
+
+		usbdp_phy0_dp_out: endpoint@3 {
+			reg = <3>;
+			remote-endpoint = <&usbc0_sbu>;
+		};
+	};
+};
+
+&usbdp_phy1 {
+	rockchip,dp-lane-mux = <2 3>;
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_host0_ohci {
+	status = "okay";
+};
+
+&usb_host0_xhci {
+	usb-role-switch;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			usb_host0_xhci_hs: endpoint {
+				remote-endpoint = <&usbc0_hs>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			usb_host0_xhci_ss: endpoint {
+				remote-endpoint = <&usbdp_phy0_ss_in>;
+			};
+		};
+	};
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usb_host1_xhci {
+	dr_mode = "host";
+	status = "okay";
+};
+
+/*
+ * combphy2_psu is shared between usb_host2_xhci (USB3) and pcie2x1l1 (PCIe).
+ * Disable USB3 host2 so the PHY can be used for the Mini-PCIe slot.
+ */
+&usb_host2_xhci {
+	status = "disabled";
+};
+
+&vop {
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi0: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi0_in_vp0>;
+	};
+};
+
+&vp1 {
+	vp1_out_hdmi1: endpoint@ROCKCHIP_VOP2_EP_HDMI1 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI1>;
+		remote-endpoint = <&hdmi1_in_vp1>;
+	};
+};
+
+&vp2 {
+	vp2_out_dp0: endpoint@a {
+		reg = <ROCKCHIP_VOP2_EP_DP0>;
+		remote-endpoint = <&dp0_in_vp2>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6-lts.dts b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6-lts.dts
index ff855064be0818..0ee67ee24f3ce4 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6-lts.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6-lts.dts
@@ -38,23 +38,6 @@
 	};
 };
 
-&u2phy1 {
-	status = "okay";
-};
-
-&u2phy1_otg {
-	status = "okay";
-};
-
 &u2phy2_host {
 	phy-supply = <&vcc5v0_usb20_host>;
 };
-
-&usbdp_phy1 {
-	status = "okay";
-};
-
-&usb_host1_xhci {
-	dr_mode = "host";
-	status = "okay";
-};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
index cfdb5c13f86061..eb0eb124c4cd4c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
@@ -100,14 +100,14 @@
 
 	sound {
 		compatible = "simple-audio-card";
-		pinctrl-names = "default";
+		label = "Onboard Analog Realtek ALC5616";
 		pinctrl-0 = <&hp_det>;
+		pinctrl-names = "default";
 
-		simple-audio-card,name = "realtek,rt5616-codec";
 		simple-audio-card,format = "i2s";
-		simple-audio-card,mclk-fs = <256>;
-
 		simple-audio-card,hp-det-gpios = <&gpio1 RK_PC4 GPIO_ACTIVE_LOW>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "realtek,rt5616-codec";
 
 		simple-audio-card,widgets =
 			"Headphone", "Headphones",
@@ -121,6 +121,7 @@
 		simple-audio-card,cpu {
 			sound-dai = <&i2s0_8ch>;
 		};
+
 		simple-audio-card,codec {
 			sound-dai = <&rt5616>;
 		};
@@ -167,16 +168,6 @@
 		vin-supply = <&vcc4v0_sys>;
 	};
 
-	vcc_3v3_pcie20: regulator-vcc3v3-pcie20 {
-		compatible = "regulator-fixed";
-		regulator-name = "vcc_3v3_pcie20";
-		regulator-always-on;
-		regulator-boot-on;
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vcc_3v3_s3>;
-	};
-
 	vbus5v0_typec: regulator-vbus5v0-typec {
 		compatible = "regulator-fixed";
 		enable-active-high;
@@ -205,8 +196,8 @@
 		compatible = "regulator-fixed";
 		enable-active-high;
 		gpio = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&pcie_m2_1_pwren_pin>;
 		pinctrl-names = "default";
-		pinctrl-0 = <&pcie_m2_1_pwren>;
 		regulator-name = "vdd_mpcie_3v3";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
@@ -217,8 +208,8 @@
 		compatible = "regulator-fixed";
 		enable-active-high;
 		gpios = <&gpio2 RK_PC5 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&pcie_m2_0_pwren_pin>;
 		pinctrl-names = "default";
-		pinctrl-0 = <&pcie_m2_0_pwren>;
 		regulator-name = "vcc3v3_pcie_m2_0";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
@@ -546,10 +537,10 @@
 		reg = <0x51>;
 		#clock-cells = <0>;
 		clock-output-names = "hym8563";
-		pinctrl-names = "default";
-		pinctrl-0 = <&hym8563_int>;
 		interrupt-parent = <&gpio0>;
 		interrupts = <RK_PB0 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-0 = <&rtc_int_l_pin>;
+		pinctrl-names = "default";
 		wakeup-source;
 	};
 };
@@ -561,17 +552,11 @@
 	rt5616: codec@1b {
 		compatible = "realtek,rt5616";
 		reg = <0x1b>;
-		clocks = <&cru I2S0_8CH_MCLKOUT>;
+		assigned-clocks = <&cru I2S0_8CH_MCLKOUT_TO_IO>;
+		assigned-clock-rates = <12288000>;
+		clocks = <&cru I2S0_8CH_MCLKOUT_TO_IO>;
 		clock-names = "mclk";
 		#sound-dai-cells = <0>;
-		assigned-clocks = <&cru I2S0_8CH_MCLKOUT>;
-		assigned-clock-rates = <12288000>;
-
-		port {
-			rt5616_p0_0: endpoint {
-				remote-endpoint = <&i2s0_8ch_p0_0>;
-			};
-		};
 	};
 
 	/* connected with MIPI-CSI1 */
@@ -583,21 +568,9 @@
 };
 
 &i2s0_8ch {
+	pinctrl-0 = <&i2s0_lrck &i2s0_mclk &i2s0_sclk &i2s0_sdi0 &i2s0_sdo0>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&i2s0_lrck
-		     &i2s0_mclk
-		     &i2s0_sclk
-		     &i2s0_sdi0
-		     &i2s0_sdo0>;
 	status = "okay";
-
-	i2s0_8ch_p0: port {
-		i2s0_8ch_p0_0: endpoint {
-			dai-format = "i2s";
-			mclk-fs = <256>;
-			remote-endpoint = <&rt5616_p0_0>;
-		};
-	};
 };
 
 &i2s5_8ch {
@@ -639,26 +612,26 @@
 };
 
 &pcie2x1l0 {
-	reset-gpios = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>;
-	vpcie3v3-supply = <&vcc_3v3_pcie20>;
+	pinctrl-0 = <&pcie_25glan_perstb_b_pin>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&pcie2_0_rst>;
+	reset-gpios = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc_3v3_s3>;
 	status = "okay";
 };
 
 &pcie2x1l1 {
+	pinctrl-0 = <&pcie_wifi_perstn_pin>;
+	pinctrl-names = "default";
 	reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>;
 	vpcie3v3-supply = <&vdd_mpcie_3v3>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pcie2_1_rst>;
 	status = "okay";
 };
 
 &pcie2x1l2 {
-	reset-gpios = <&gpio4 RK_PA4 GPIO_ACTIVE_HIGH>;
-	vpcie3v3-supply = <&vcc_3v3_pcie20>;
+	pinctrl-0 = <&pcie_25glan_perstb_pin>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&pcie2_2_rst>;
+	reset-gpios = <&gpio4 RK_PA4 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc_3v3_s3>;
 	status = "okay";
 };
 
@@ -714,8 +687,8 @@
 	};
 
 	hym8563 {
-		hym8563_int: hym8563-int {
-			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
+		rtc_int_l_pin: rtc-int-l-pin {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 
@@ -726,25 +699,25 @@
 	};
 
 	pcie {
-		pcie2_0_rst: pcie2-0-rst {
-			rockchip,pins = <4 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
-		};
-
-		pcie2_1_rst: pcie2-1-rst {
-			rockchip,pins = <4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		pcie_25glan_perstb_pin: pcie-25glan-perstb-pin {
+			rockchip,pins = <4 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 
-		pcie2_2_rst: pcie2-2-rst {
-			rockchip,pins = <4 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>;
+		pcie_25glan_perstb_b_pin: pcie-25glan-perstb-b-pin {
+			rockchip,pins = <4 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 
-		pcie_m2_0_pwren: pcie-m20-pwren {
+		pcie_m2_0_pwren_pin: pcie-m2-0-pwren-pin {
 			rockchip,pins = <2 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 
-		pcie_m2_1_pwren: pcie-m21-pwren {
+		pcie_m2_1_pwren_pin: pcie-m2-1-pwren-pin {
 			rockchip,pins = <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
+
+		pcie_wifi_perstn_pin: pcie-wifi-perstn-pin {
+			rockchip,pins = <4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
 	};
 
 	usb {
@@ -1174,12 +1147,6 @@
 	status = "okay";
 };
 
-&uart2 {
-	pinctrl-0 = <&uart2m0_xfer>;
-	pinctrl-names = "default";
-	status = "okay";
-};
-
 &u2phy0 {
 	status = "okay";
 };
@@ -1196,15 +1163,11 @@
 	status = "okay";
 };
 
-&u2phy2_host {
-	status = "okay";
-};
-
-&u2phy3_host {
+&u2phy2 {
 	status = "okay";
 };
 
-&u2phy2 {
+&u2phy2_host {
 	status = "okay";
 };
 
@@ -1212,31 +1175,13 @@
 	status = "okay";
 };
 
-&usbdp_phy0 {
-	mode-switch;
-	orientation-switch;
-	sbu1-dc-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>;
-	sbu2-dc-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>;
+&u2phy3_host {
 	status = "okay";
-
-	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		usbdp_phy0_typec_ss: endpoint@0 {
-			reg = <0>;
-			remote-endpoint = <&usbc0_ss>;
-		};
-
-		usbdp_phy0_typec_sbu: endpoint@1 {
-			reg = <1>;
-			remote-endpoint = <&usbc0_sbu>;
-		};
-	};
 };
 
-&usbdp_phy1 {
-	phy-supply = <&vbus5v0_usb>;
+&uart2 {
+	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -1259,16 +1204,44 @@
 	};
 };
 
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
 &usb_host1_xhci {
 	dr_mode = "host";
 	status = "okay";
 };
 
-&usb_host1_ehci {
+&usbdp_phy0 {
+	mode-switch;
+	orientation-switch;
+	sbu1-dc-gpios = <&gpio4 RK_PA6 GPIO_ACTIVE_HIGH>;
+	sbu2-dc-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>;
 	status = "okay";
+
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		usbdp_phy0_typec_ss: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&usbc0_ss>;
+		};
+
+		usbdp_phy0_typec_sbu: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&usbc0_sbu>;
+		};
+	};
 };
 
-&usb_host1_ohci {
+&usbdp_phy1 {
+	phy-supply = <&vbus5v0_usb>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam0.dtso b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam0.dtso
index 5c7fb960b09947..5420ec3fe2ed01 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam0.dtso
+++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam0.dtso
@@ -10,6 +10,7 @@
 #include <dt-bindings/clock/rockchip,rk3588-cru.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 
 &{/} {
@@ -51,7 +52,7 @@
 		avdd-supply = <&savdd_cam0>;
 		clocks = <&cru CLK_MIPI_CAMARAOUT_M3>;
 		dvdd-supply = <&sdvdd_cam0>;
-		orientation = <2>; /* External */
+		orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 		ovdd-supply = <&siovdd_cam0>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&cam0_rstn &mipim0_camera3_clk>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam1.dtso b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam1.dtso
index 1ee9036b9095bd..8ef764aab73a44 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam1.dtso
+++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-plus-radxa-cam4k-cam1.dtso
@@ -10,6 +10,7 @@
 #include <dt-bindings/clock/rockchip,rk3588-cru.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/media/video-interfaces.h>
+#include <dt-bindings/media/video-interface-devices.h>
 #include <dt-bindings/pinctrl/rockchip.h>
 
 &{/} {
@@ -51,7 +52,7 @@
 		avdd-supply = <&savdd_cam1>;
 		clocks = <&cru CLK_MIPI_CAMARAOUT_M4>;
 		dvdd-supply = <&sdvdd_cam1>;
-		orientation = <2>; /* External */
+		orientation = <MEDIA_ORIENTATION_EXTERNAL>;
 		ovdd-supply = <&siovdd_cam1>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&cam1_rstn &mipim0_camera4_clk>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts b/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts
index 935c40c5a9ba27..ea63bf2f7233c0 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts
@@ -155,6 +155,10 @@
 	};
 };
 
+&can0 {
+	status = "okay";
+};
+
 &combphy2_psu {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
index 139f70e7bbd906..161c89c62bcdab 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
@@ -90,6 +90,11 @@
 	};
 };
 
+&can0 {
+	assigned-clocks = <&cru CLK_CAN0>;
+	assigned-clock-rates = <300000000>;
+};
+
 &cpu_b0 {
 	cpu-supply = <&vdd_cpu_big0_s0>;
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
index 230aac005e8f67..d43eb48152e8ad 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
@@ -596,6 +596,7 @@
 &i2c3 {
 	status = "okay";
 
+	/* Touchscreen is rotated 90 degrees with 0,0 position near home button. */
 	touchscreen@14 {
 		compatible = "goodix,gt911";
 		reg = <0x14>;
@@ -605,10 +606,8 @@
 		pinctrl-0 = <&touch_int>, <&touch_rst>;
 		pinctrl-names = "default";
 		reset-gpios = <&gpio1 RK_PA7 GPIO_ACTIVE_HIGH>;
-		touchscreen-inverted-x;
 		touchscreen-size-x = <1080>;
 		touchscreen-size-y = <1920>;
-		touchscreen-swapped-x-y;
 	};
 };
 
@@ -641,11 +640,15 @@
 		};
 	};
 
+	/* IMU oriented relative to panel, which is rotated 90 degrees. */
 	imu@68 {
 		compatible = "invensense,mpu6880";
 		reg = <0x68>;
 		interrupt-parent = <&gpio0>;
 		interrupts = <RK_PD3 IRQ_TYPE_EDGE_RISING>;
+		mount-matrix = "-1", "0", "0",
+			       "0", "-1", "0",
+			       "0", "0", "1";
 	};
 };
 
diff --git a/arch/arm64/boot/dts/st/Makefile b/arch/arm64/boot/dts/st/Makefile
index 63908113ae36ba..9829a5b104fef1 100644
--- a/arch/arm64/boot/dts/st/Makefile
+++ b/arch/arm64/boot/dts/st/Makefile
@@ -1,6 +1,29 @@
 # SPDX-License-Identifier: GPL-2.0-only
+
+stm32mp255c-dhcos-dhsbc-overlay-imx219-x10-dtbs := \
+	stm32mp255c-dhcos-dhsbc.dtb \
+	stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtbo
+
+stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h-dtbs := \
+	stm32mp257d-engicam-microgea-rmm.dtb \
+	stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtbo
+
+stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a-dtbs := \
+	stm32mp257d-engicam-microgea-rmm.dtb \
+	stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a.dtbo
+
 dtb-$(CONFIG_ARCH_STM32) += \
 	stm32mp215f-dk.dtb \
 	stm32mp235f-dk.dtb \
+	stm32mp23xx-dhcos-bb.dtb \
+	stm32mp25xx-dhcos-bb.dtb \
+	stm32mp255c-dhcos-dhsbc.dtb \
+	stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtb \
+	stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtbo \
+	stm32mp257d-engicam-microgea-rmm.dtb \
+	stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtb \
+	stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtbo \
+	stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a.dtb \
+	stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a.dtbo \
 	stm32mp257f-dk.dtb \
 	stm32mp257f-ev1.dtb
diff --git a/arch/arm64/boot/dts/st/stm32mp231.dtsi b/arch/arm64/boot/dts/st/stm32mp231.dtsi
index 9e1d240888ff2f..fefbf6f4f15377 100644
--- a/arch/arm64/boot/dts/st/stm32mp231.dtsi
+++ b/arch/arm64/boot/dts/st/stm32mp231.dtsi
@@ -35,7 +35,7 @@
 
 	arm_wdt: watchdog {
 		compatible = "arm,smc-wdt";
-		arm,smc-id = <0xb200005a>;
+		arm,smc-id = <0xbc000000>;
 		status = "disabled";
 	};
 
@@ -222,6 +222,60 @@
 			#dma-cells = <3>;
 		};
 
+		ommanager: ommanager@40500000 {
+			compatible = "st,stm32mp25-omm";
+			reg = <0x40500000 0x400>, <0x60000000 0x10000000>;
+			reg-names = "regs", "memory_map";
+			ranges = <0 0 0x40430000 0x400>,
+				 <1 0 0x40440000 0x400>;
+			clocks = <&rcc CK_BUS_OSPIIOM>,
+				 <&scmi_clk CK_SCMI_OSPI1>,
+				 <&scmi_clk CK_SCMI_OSPI2>;
+			clock-names = "omm", "ospi1", "ospi2";
+			resets = <&rcc OSPIIOM_R>,
+				 <&scmi_reset RST_SCMI_OSPI1>,
+				 <&scmi_reset RST_SCMI_OSPI2>;
+			reset-names = "omm", "ospi1", "ospi2";
+			access-controllers = <&rifsc 111>;
+			power-domains = <&cluster_pd>;
+			#address-cells = <2>;
+			#size-cells = <1>;
+			st,syscfg-amcr = <&syscfg 0x2c00 0x7>;
+			status = "disabled";
+
+			ospi1: spi@0 {
+				compatible = "st,stm32mp25-ospi";
+				reg = <0 0 0x400>;
+				interrupts = <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&hpdma 2 0x62 0x3121>,
+				       <&hpdma 2 0x42 0x3112>;
+				dma-names = "tx", "rx";
+				clocks = <&scmi_clk CK_SCMI_OSPI1>;
+				resets = <&scmi_reset RST_SCMI_OSPI1>,
+					 <&scmi_reset RST_SCMI_OSPI1DLL>;
+				access-controllers = <&rifsc 74>;
+				power-domains = <&cluster_pd>;
+				st,syscfg-dlyb = <&syscfg 0x1000>;
+				status = "disabled";
+			};
+
+			ospi2: spi@1 {
+				compatible = "st,stm32mp25-ospi";
+				reg = <1 0 0x400>;
+				interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&hpdma 3 0x62 0x3121>,
+				       <&hpdma 3 0x42 0x3112>;
+				dma-names = "tx", "rx";
+				clocks = <&scmi_clk CK_SCMI_OSPI2>;
+				resets = <&scmi_reset RST_SCMI_OSPI2>,
+					 <&scmi_reset RST_SCMI_OSPI2DLL>;
+				access-controllers = <&rifsc 75>;
+				power-domains = <&cluster_pd>;
+				st,syscfg-dlyb = <&syscfg 0x1400>;
+				status = "disabled";
+			};
+		};
+
 		rifsc: bus@42080000 {
 			compatible = "st,stm32mp25-rifsc", "simple-bus";
 			reg = <0x42080000 0x1000>;
@@ -577,7 +631,7 @@
 					status = "disabled";
 				};
 
-				sai3b: audio-controller@502b0024 {
+				sai3b: audio-controller@402b0024 {
 					compatible = "st,stm32-sai-sub-b";
 					reg = <0x24 0x20>;
 					#sound-dai-cells = <0>;
@@ -646,6 +700,20 @@
 				status = "disabled";
 			};
 
+			dcmi: dcmi@404a0000 {
+				compatible = "st,stm32-dcmi";
+				reg = <0x404a0000 0x400>;
+				interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>;
+				resets = <&rcc CCI_R>;
+				clocks = <&rcc CK_BUS_CCI>;
+				clock-names = "mclk";
+				dmas = <&hpdma 137 0x60 0x00003012>;
+				dma-names = "tx";
+				access-controllers = <&rifsc 88>;
+				power-domains = <&cluster_pd>;
+				status = "disabled";
+			};
+
 			rng: rng@42020000 {
 				compatible = "st,stm32mp25-rng";
 				reg = <0x42020000 0x400>;
@@ -720,13 +788,44 @@
 				reg = <0x48220000 0x400>, <0x44230400 0x8>;
 				arm,primecell-periphid = <0x00353180>;
 				interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&rcc CK_KER_SDMMC1 >;
+				clocks = <&rcc CK_KER_SDMMC1>;
 				clock-names = "apb_pclk";
 				resets = <&rcc SDMMC1_R>;
 				cap-sd-highspeed;
 				cap-mmc-highspeed;
 				max-frequency = <120000000>;
 				access-controllers = <&rifsc 76>;
+				power-domains = <&cluster_pd>;
+				status = "disabled";
+			};
+
+			sdmmc2: mmc@48230000 {
+				compatible = "st,stm32mp25-sdmmc2", "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00353180>;
+				reg = <0x48230000 0x400>, <0x44230800 0x8>;
+				interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&rcc CK_KER_SDMMC2>;
+				clock-names = "apb_pclk";
+				resets = <&rcc SDMMC2_R>;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				max-frequency = <166000000>;
+				access-controllers = <&rifsc 77>;
+				status = "disabled";
+			};
+
+			sdmmc3: mmc@48240000 {
+				compatible = "st,stm32mp25-sdmmc2", "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00353180>;
+				reg = <0x48240000 0x400>, <0x44230c00 0x8>;
+				interrupts = <GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&rcc CK_KER_SDMMC3>;
+				clock-names = "apb_pclk";
+				resets = <&rcc SDMMC3_R>;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				max-frequency = <166000000>;
+				access-controllers = <&rifsc 78>;
 				status = "disabled";
 			};
 
@@ -778,6 +877,48 @@
 					snps,wr_osr_lmt = <0x7>;
 				};
 			};
+
+			usbh_ohci: usb@482e0000 {
+				compatible = "st,stm32mp23-ohci", "generic-ohci";
+				reg = <0x482e0000 0x1000>;
+				access-controllers = <&rifsc 63>;
+				clocks = <&usb2_phy1 1>, <&rcc CK_BUS_USB2OHCI>;
+				interrupts = <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&usb2_phy1>;
+				phy-names = "usb";
+				resets = <&rcc USBH_R>;
+				status = "disabled";
+			};
+
+			usbh_ehci: usb@482f0000 {
+				compatible = "st,stm32mp23-ehci", "generic-ehci";
+				reg = <0x482f0000 0x1000>;
+				access-controllers = <&rifsc 63>;
+				clocks = <&usb2_phy1 0>, <&rcc CK_BUS_USB2EHCI>;
+				companion = <&usbh_ohci>;
+				interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&usb2_phy1>;
+				phy-names = "usb";
+				resets = <&rcc USBH_R>;
+				status = "disabled";
+			};
+
+			usb3dr: usb@48300000 {
+				compatible = "st,stm32mp25-dwc3";
+				reg = <0x48300000 0x100000>;
+				access-controllers = <&rifsc 66>;
+				clocks = <&rcc CK_KER_USB2PHY2>,
+					 <&rcc CK_BUS_USB3DR>,
+					 <&rcc CK_KER_USB2PHY2>;
+				clock-names = "ref", "bus_early", "suspend";
+				interrupts = <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH>;
+				maximum-speed = "high-speed";
+				phys = <&usb2_phy2>;
+				phy-names = "usb2-phy";
+				resets = <&rcc USB3DR_R>;
+				st,syscfg = <&syscfg 0x4800>;
+				status = "disabled";
+			};
 		};
 
 		bsec: efuse@44000000 {
@@ -981,6 +1122,28 @@
 			compatible = "st,stm32mp23-syscfg", "syscon";
 			reg = <0x44230000 0x10000>;
 			#clock-cells = <0>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			usb2_phy1: phy@2400 {
+				compatible = "st,stm32mp25-usb2phy1";
+				reg = <0x2400 0x24>;
+				#phy-cells = <0>;
+				#clock-cells = <1>;
+				clocks = <&rcc CK_KER_USB2PHY1>;
+				resets = <&rcc USB2PHY1_R>;
+				status = "disabled";
+			};
+
+			usb2_phy2: phy@2800 {
+				compatible = "st,stm32mp25-usb2phy2";
+				reg = <0x2800 0x10>;
+				#phy-cells = <0>;
+				#clock-cells = <1>;
+				clocks = <&rcc CK_KER_USB2PHY2EN>;
+				resets = <&rcc USB2PHY2_R>;
+				status = "disabled";
+			};
 		};
 
 		pinctrl: pinctrl@44240000 {
diff --git a/arch/arm64/boot/dts/st/stm32mp235f-dk.dts b/arch/arm64/boot/dts/st/stm32mp235f-dk.dts
index dd4efbe5a46e86..f06b334dc9ce1d 100644
--- a/arch/arm64/boot/dts/st/stm32mp235f-dk.dts
+++ b/arch/arm64/boot/dts/st/stm32mp235f-dk.dts
@@ -9,6 +9,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interfaces.h>
 #include "stm32mp235.dtsi"
 #include "stm32mp23xf.dtsi"
 #include "stm32mp25-pinctrl.dtsi"
@@ -27,6 +28,14 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	clocks {
+		clk_ext_camera: clk-ext-camera {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <24000000>;
+		};
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 
@@ -131,6 +140,42 @@
 	status = "okay";
 };
 
+&csi {
+	vdd-supply =  <&scmi_vddcore>;
+	vdda18-supply = <&scmi_v1v8>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			csi_sink: endpoint {
+				remote-endpoint = <&imx335_ep>;
+				data-lanes = <1 2>;
+				bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+			};
+		};
+		port@1 {
+			reg = <1>;
+			csi_source: endpoint {
+				remote-endpoint = <&dcmipp_0>;
+			};
+		};
+	};
+};
+
+&dcmipp {
+	status = "okay";
+
+	port {
+		dcmipp_0: endpoint {
+			remote-endpoint = <&csi_source>;
+			bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+		};
+	};
+};
+
 &ethernet1 {
 	pinctrl-0 = <&eth1_rgmii_pins_b>;
 	pinctrl-1 = <&eth1_rgmii_sleep_pins_b>;
@@ -153,6 +198,16 @@
 	};
 };
 
+&gpiob {
+	/* Enable the IMX335 power line */
+	imx335-en-hog {
+		gpio-hog;
+		gpios = <11 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "imx335_en";
+	};
+};
+
 &i2c2 {
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&i2c2_pins_b>;
@@ -165,6 +220,26 @@
 	/delete-property/dmas;
 	/delete-property/dma-names;
 
+	imx335: camera@1a {
+		compatible = "sony,imx335";
+		reg = <0x1a>;
+		clocks = <&clk_ext_camera>;
+		avdd-supply = <&scmi_v3v3>;
+		ovdd-supply = <&scmi_v3v3>;
+		dvdd-supply = <&scmi_v3v3>;
+		reset-gpios = <&gpiob 1 (GPIO_ACTIVE_LOW | GPIO_PUSH_PULL)>;
+		status = "okay";
+
+		port {
+			imx335_ep: endpoint {
+				remote-endpoint = <&csi_sink>;
+				clock-lanes = <0>;
+				data-lanes = <1 2>;
+				link-frequencies = /bits/ 64 <594000000>;
+			};
+		};
+	};
+
 	ili2511: ili2511@41 {
 		compatible = "ilitek,ili251x";
 		reg = <0x41>;
diff --git a/arch/arm64/boot/dts/st/stm32mp23xc.dtsi b/arch/arm64/boot/dts/st/stm32mp23xc.dtsi
new file mode 100644
index 00000000000000..56872fb1deeb0f
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp23xc.dtsi
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-3-Clause)
+/*
+ * Copyright (C) 2026 Marek Vasut <marex@nabladev.com>
+ */
+
+/ {
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp23xx-dhcos-bb.dts b/arch/arm64/boot/dts/st/stm32mp23xx-dhcos-bb.dts
new file mode 100644
index 00000000000000..125c76fe3e7bed
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp23xx-dhcos-bb.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-3-Clause)
+/*
+ * Copyright (C) 2026 Marek Vasut <marex@nabladev.com>
+ */
+
+/dts-v1/;
+
+#include "stm32mp235.dtsi"
+#include "stm32mp23xc.dtsi"
+#include "stm32mp23xx-dhcos-som.dtsi"
+
+/ {
+	model = "DH electronics STM32MP23xx DHCOS Breakout Board";
+	compatible = "dh,stm32mp231a-dhcos-bb", "dh,stm32mp231a-dhcos-som", "st,stm32mp231";
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp23xx-dhcos-som.dtsi b/arch/arm64/boot/dts/st/stm32mp23xx-dhcos-som.dtsi
new file mode 100644
index 00000000000000..ffdcceb2fa237b
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp23xx-dhcos-som.dtsi
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+/*
+ * Copyright (C) 2025-2026 Marek Vasut <marex@nabladev.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include "stm32mp25-pinctrl.dtsi"
+#include "stm32mp25xxak-pinctrl.dtsi"
+#include "stm32mp2xxx-dhcos-som.dtsi"
+
+/ {
+	model = "DH electronics STM32MP23xx DHCOS SoM";
+	compatible = "dh,stm32mp231a-dhcos-som", "st,stm32mp231";
+
+	aliases {
+		serial1 = &usart1;
+		serial2 = &usart2;
+	};
+};
+
+&rv3032 {
+	interrupts-extended = <&gpiod 10 IRQ_TYPE_EDGE_FALLING>;
+};
+
+/* Bluetooth */
+&usart1 {
+	pinctrl-names = "default", "sleep", "idle";
+	pinctrl-0 = <&usart1_pins_a>;
+	pinctrl-1 = <&usart1_sleep_pins_a>;
+	pinctrl-2 = <&usart1_idle_pins_a>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "infineon,cyw55572-bt";
+		brcm,requires-autobaud-mode;
+		max-speed = <3000000>;
+		shutdown-gpios = <&ioexp 2 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&usart2 {
+	pinctrl-names = "default", "sleep", "idle";
+	pinctrl-0 = <&usart2_pins_b>;
+	pinctrl-1 = <&usart2_sleep_pins_b>;
+	pinctrl-2 = <&usart2_idle_pins_b>;
+	uart-has-rtscts;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi b/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi
index 456ece7f8ebc31..757cdd24153947 100644
--- a/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/st/stm32mp25-pinctrl.dtsi
@@ -95,6 +95,7 @@
 			bias-disable;
 			drive-push-pull;
 			slew-rate = <3>;
+			st,io-sync = "data on both edges";
 		};
 		pins2 {
 			pinmux = <STM32_PINMUX('H', 9, AF10)>, /* ETH_RGMII_CLK125 */
@@ -112,6 +113,7 @@
 				 <STM32_PINMUX('H', 13, AF10)>, /* ETH_RGMII_RXD3 */
 				 <STM32_PINMUX('A', 11, AF10)>; /* ETH_RGMII_RX_CTL */
 			bias-disable;
+			st,io-sync = "data on both edges";
 		};
 		pins4 {
 			pinmux = <STM32_PINMUX('A', 14, AF10)>; /* ETH_RGMII_RX_CLK */
@@ -182,6 +184,30 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	eth2_mdio_pins_a: eth2-mdio-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('G', 4, AF11)>; /* ETH_MDC */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <3>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('C', 5, AF10)>; /* ETH_MDIO */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+	};
+
+	/omit-if-no-ref/
+	eth2_mdio_sleep_pins_a: eth2-mdio-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('G', 4, ANALOG)>, /* ETH_MDC */
+				 <STM32_PINMUX('C', 5, ANALOG)>; /* ETH_MDIO */
+		};
+	};
+
 	/omit-if-no-ref/
 	eth2_rgmii_sleep_pins_a: eth2-rgmii-sleep-0 {
 		pins {
@@ -203,6 +229,77 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	eth2_rgmii_pins_b: eth2-rgmii-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('C', 7, AF10)>, /* ETH_RGMII_TXD0 */
+				 <STM32_PINMUX('C', 8, AF10)>, /* ETH_RGMII_TXD1 */
+				 <STM32_PINMUX('C', 9, AF10)>, /* ETH_RGMII_TXD2 */
+				 <STM32_PINMUX('C', 10, AF10)>, /* ETH_RGMII_TXD3 */
+				 <STM32_PINMUX('C', 4, AF10)>; /* ETH_RGMII_TX_CTL */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <3>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('F', 8, AF10)>, /* ETH_RGMII_CLK125 */
+				 <STM32_PINMUX('F', 7, AF10)>; /* ETH_MDC */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <3>;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('G', 0, AF10)>, /* ETH_RGMII_RXD0 */
+				 <STM32_PINMUX('C', 12, AF10)>, /* ETH_RGMII_RXD1 */
+				 <STM32_PINMUX('F', 9, AF10)>, /* ETH_RGMII_RXD2 */
+				 <STM32_PINMUX('C', 11, AF10)>, /* ETH_RGMII_RXD3 */
+				 <STM32_PINMUX('C', 3, AF10)>; /* ETH_RGMII_RX_CTL */
+			bias-disable;
+		};
+		pins4 {
+			pinmux = <STM32_PINMUX('F', 6, AF10)>; /* ETH_RGMII_RX_CLK */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	eth2_rgmii_sleep_pins_b: eth2-rgmii-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('C', 7, ANALOG)>, /* ETH_RGMII_TXD0 */
+				 <STM32_PINMUX('C', 8, ANALOG)>, /* ETH_RGMII_TXD1 */
+				 <STM32_PINMUX('C', 9, ANALOG)>, /* ETH_RGMII_TXD2 */
+				 <STM32_PINMUX('C', 10, ANALOG)>, /* ETH_RGMII_TXD3 */
+				 <STM32_PINMUX('C', 4, ANALOG)>, /* ETH_RGMII_TX_CTL */
+				 <STM32_PINMUX('F', 8, ANALOG)>, /* ETH_RGMII_CLK125 */
+				 <STM32_PINMUX('F', 7, ANALOG)>, /* ETH_RGMII_GTX_CLK */
+				 <STM32_PINMUX('G', 0, ANALOG)>, /* ETH_RGMII_RXD0 */
+				 <STM32_PINMUX('C', 12, ANALOG)>, /* ETH_RGMII_RXD1 */
+				 <STM32_PINMUX('F', 9, ANALOG)>, /* ETH_RGMII_RXD2 */
+				 <STM32_PINMUX('C', 11, ANALOG)>, /* ETH_RGMII_RXD3 */
+				 <STM32_PINMUX('C', 3, ANALOG)>, /* ETH_RGMII_RX_CTL */
+				 <STM32_PINMUX('F', 6, ANALOG)>; /* ETH_RGMII_RX_CLK */
+		};
+	};
+
+	/omit-if-no-ref/
+	i2c1_pins_a: i2c1-0 {
+		pins {
+			pinmux = <STM32_PINMUX('G', 13, AF9)>, /* I2C1_SCL */
+				 <STM32_PINMUX('A', 2, AF10)>; /* I2C1_SDA */
+			bias-disable;
+			drive-open-drain;
+			slew-rate = <0>;
+		};
+	};
+
+	/omit-if-no-ref/
+	i2c1_sleep_pins_a: i2c1-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('G', 13, ANALOG)>, /* I2C1_SCL */
+				 <STM32_PINMUX('A', 2, ANALOG)>; /* I2C1_SDA */
+		};
+	};
+
 	/omit-if-no-ref/
 	i2c2_pins_a: i2c2-0 {
 		pins {
@@ -241,6 +338,109 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	ltdc_pins_a: ltdc-0 {
+		pins {
+			pinmux = <STM32_PINMUX('C',  6, AF14)>, /* LCD_CLK */
+				 <STM32_PINMUX('G',  2, AF13)>, /* LCD_HSYNC */
+				 <STM32_PINMUX('G',  1, AF13)>, /* LCD_VSYNC */
+				 <STM32_PINMUX('C',  5, AF14)>, /* LCD_DE */
+				 <STM32_PINMUX('H',  4, AF10)>, /* LCD_R0 */
+				 <STM32_PINMUX('F',  7, AF13)>, /* LCD_R1 */
+				 <STM32_PINMUX('C', 11, AF13)>, /* LCD_R2 */
+				 <STM32_PINMUX('A',  1, AF11)>, /* LCD_R3 */
+				 <STM32_PINMUX('B', 15, AF13)>, /* LCD_R4 */
+				 <STM32_PINMUX('G',  3, AF13)>, /* LCD_R5 */
+				 <STM32_PINMUX('A', 10, AF12)>, /* LCD_R6 */
+				 <STM32_PINMUX('G',  7, AF13)>, /* LCD_R7 */
+				 <STM32_PINMUX('F',  8, AF13)>, /* LCD_G0 */
+				 <STM32_PINMUX('H',  5, AF10)>, /* LCD_G1 */
+				 <STM32_PINMUX('C',  9, AF13)>, /* LCD_G2 */
+				 <STM32_PINMUX('C', 10, AF13)>, /* LCD_G3 */
+				 <STM32_PINMUX('A',  6, AF10)>, /* LCD_G4 */
+				 <STM32_PINMUX('G', 11, AF13)>, /* LCD_G5 */
+				 <STM32_PINMUX('G', 12, AF13)>, /* LCD_G6 */
+				 <STM32_PINMUX('A',  9, AF12)>, /* LCD_G7 */
+				 <STM32_PINMUX('F',  6, AF13)>, /* LCD_B0 */
+				 <STM32_PINMUX('A',  3, AF11)>, /* LCD_B1 */
+				 <STM32_PINMUX('G', 15, AF13)>, /* LCD_B2 */
+				 <STM32_PINMUX('I',  0, AF13)>, /* LCD_B3 */
+				 <STM32_PINMUX('I',  1, AF13)>, /* LCD_B4 */
+				 <STM32_PINMUX('A',  7, AF10)>, /* LCD_B5 */
+				 <STM32_PINMUX('F',  5, AF13)>, /* LCD_B6 */
+				 <STM32_PINMUX('I',  4, AF13)>; /* LCD_B7 */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <1>;
+		};
+	};
+
+	/omit-if-no-ref/
+	ltdc_sleep_pins_a: ltdc-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('C',  6, ANALOG)>, /* LCD_CLK */
+				 <STM32_PINMUX('G',  2, ANALOG)>, /* LCD_HSYNC */
+				 <STM32_PINMUX('G',  1, ANALOG)>, /* LCD_VSYNC */
+				 <STM32_PINMUX('C',  5, ANALOG)>, /* LCD_DE */
+				 <STM32_PINMUX('H',  4, ANALOG)>, /* LCD_R0 */
+				 <STM32_PINMUX('F',  7, ANALOG)>, /* LCD_R1 */
+				 <STM32_PINMUX('C', 11, ANALOG)>, /* LCD_R2 */
+				 <STM32_PINMUX('A',  1, ANALOG)>, /* LCD_R3 */
+				 <STM32_PINMUX('B', 15, ANALOG)>, /* LCD_R4 */
+				 <STM32_PINMUX('G',  3, ANALOG)>, /* LCD_R5 */
+				 <STM32_PINMUX('A', 10, ANALOG)>, /* LCD_R6 */
+				 <STM32_PINMUX('G',  7, ANALOG)>, /* LCD_R7 */
+				 <STM32_PINMUX('F',  8, ANALOG)>, /* LCD_G0 */
+				 <STM32_PINMUX('H',  5, ANALOG)>, /* LCD_G1 */
+				 <STM32_PINMUX('C',  9, ANALOG)>, /* LCD_G2 */
+				 <STM32_PINMUX('C', 10, ANALOG)>, /* LCD_G3 */
+				 <STM32_PINMUX('A',  6, ANALOG)>, /* LCD_G4 */
+				 <STM32_PINMUX('G', 11, ANALOG)>, /* LCD_G5 */
+				 <STM32_PINMUX('G', 12, ANALOG)>, /* LCD_G6 */
+				 <STM32_PINMUX('A',  9, ANALOG)>, /* LCD_G7 */
+				 <STM32_PINMUX('F',  6, ANALOG)>, /* LCD_B0 */
+				 <STM32_PINMUX('A',  3, ANALOG)>, /* LCD_B1 */
+				 <STM32_PINMUX('G', 15, ANALOG)>, /* LCD_B2 */
+				 <STM32_PINMUX('I',  0, ANALOG)>, /* LCD_B3 */
+				 <STM32_PINMUX('I',  1, ANALOG)>, /* LCD_B4 */
+				 <STM32_PINMUX('A',  7, ANALOG)>, /* LCD_B5 */
+				 <STM32_PINMUX('F',  5, ANALOG)>, /* LCD_B6 */
+				 <STM32_PINMUX('I',  4, ANALOG)>; /* LCD_B7 */
+		};
+	};
+
+	/omit-if-no-ref/
+	m_can1_pins_a: m-can1-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 9, AF7)>; /* CAN1_TX */
+			slew-rate = <1>;
+			drive-push-pull;
+			bias-disable;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 11, AF7)>; /* CAN1_RX */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	m_can1_sleep_pins_a: m-can1-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 9, ANALOG)>, /* CAN1_TX */
+				 <STM32_PINMUX('B', 11, ANALOG)>; /* CAN1_RX */
+		};
+	};
+
+	/omit-if-no-ref/
+	mco1_pins_a: mco1-0 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 6, AF1)>; /* MCO1 */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <2>;
+		};
+	};
+
 	/omit-if-no-ref/
 	ospi_port1_clk_pins_a: ospi-port1-clk-0 {
 		pins {
@@ -321,6 +521,23 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	pwm2_pins_a: pwm2-0 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 7, AF7)>; /* TIM2_CH1 */
+			bias-pull-down;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+	};
+
+	/omit-if-no-ref/
+	pwm2_sleep_pins_a: pwm2-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 7, ANALOG)>; /* TIM2_CH1 */
+		};
+	};
+
 	/omit-if-no-ref/
 	pwm3_pins_a: pwm3-0 {
 		pins {
@@ -338,6 +555,23 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	pwm4_pins_a: pwm4-0 {
+		pins {
+			pinmux = <STM32_PINMUX('A', 12, AF7)>; /* TIM4_CH1 */
+			bias-pull-down;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+	};
+
+	/omit-if-no-ref/
+	pwm4_sleep_pins_a: pwm4-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('A', 12, ANALOG)>; /* TIM4_CH1 */
+		};
+	};
+
 	/omit-if-no-ref/
 	pwm8_pins_a: pwm8-0 {
 		pins {
@@ -374,6 +608,51 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	sai1a_pins_a: sai1a-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('D', 9, AF3)>, /* SAI1_SD_A */
+				 <STM32_PINMUX('D', 8, AF3)>, /* SAI1_FS_A */
+				 <STM32_PINMUX('D', 10, AF3)>; /* SAI1_SCK_A */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <1>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('D', 11, AF3)>; /* SAI1_MCLK_A */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <2>;
+		};
+	};
+
+	/omit-if-no-ref/
+	sai1a_sleep_pins_a: sai1a-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('D', 9, ANALOG)>, /* SAI1_SD_A */
+				 <STM32_PINMUX('D', 8, ANALOG)>, /* SAI1_FS_A */
+				 <STM32_PINMUX('D', 10, ANALOG)>, /* SAI1_SCK_A */
+				 <STM32_PINMUX('D', 11, ANALOG)>; /* SAI1_MCLK_A */
+		};
+	};
+
+	/omit-if-no-ref/
+	sai1b_pins_a: sai1b-0 {
+		pins {
+			pinmux = <STM32_PINMUX('D', 4, AF4)>; /* SAI1_SD_B */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+	};
+
+	/omit-if-no-ref/
+	sai1b_sleep_pins_a: sai1b-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('D', 4, ANALOG)>; /* SAI1_SD_B */
+		};
+	};
+
 	/omit-if-no-ref/
 	sdmmc1_b4_pins_a: sdmmc1-b4-0 {
 		pins1 {
@@ -394,6 +673,26 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	sdmmc1_b4_pins_b: sdmmc1-b4-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('E', 4, AF10)>, /* SDMMC1_D0 */
+				 <STM32_PINMUX('E', 5, AF10)>, /* SDMMC1_D1 */
+				 <STM32_PINMUX('E', 0, AF10)>, /* SDMMC1_D2 */
+				 <STM32_PINMUX('E', 1, AF10)>, /* SDMMC1_D3 */
+				 <STM32_PINMUX('E', 2, AF10)>; /* SDMMC1_CMD */
+			slew-rate = <1>;
+			drive-push-pull;
+			bias-disable;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('E', 3, AF10)>; /* SDMMC1_CK */
+			slew-rate = <2>;
+			drive-push-pull;
+			bias-disable;
+		};
+	};
+
 	/omit-if-no-ref/
 	sdmmc1_b4_od_pins_a: sdmmc1-b4-od-0 {
 		pins1 {
@@ -419,6 +718,31 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	sdmmc1_b4_od_pins_b: sdmmc1-b4-od-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('E', 4, AF10)>, /* SDMMC1_D0 */
+				 <STM32_PINMUX('E', 5, AF10)>, /* SDMMC1_D1 */
+				 <STM32_PINMUX('E', 0, AF10)>, /* SDMMC1_D2 */
+				 <STM32_PINMUX('E', 1, AF10)>; /* SDMMC1_D3 */
+			slew-rate = <1>;
+			drive-push-pull;
+			bias-disable;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('E', 3, AF10)>; /* SDMMC1_CK */
+			slew-rate = <2>;
+			drive-push-pull;
+			bias-disable;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('E', 2, AF10)>; /* SDMMC1_CMD */
+			slew-rate = <1>;
+			drive-open-drain;
+			bias-disable;
+		};
+	};
+
 	/omit-if-no-ref/
 	sdmmc1_b4_sleep_pins_a: sdmmc1-b4-sleep-0 {
 		pins {
@@ -432,43 +756,383 @@
 	};
 
 	/omit-if-no-ref/
-	spi3_pins_a: spi3-0 {
+	sdmmc2_b4_pins_a: sdmmc2-b4-0 {
 		pins1 {
-			pinmux = <STM32_PINMUX('B', 7, AF1)>, /* SPI3_SCK */
-				 <STM32_PINMUX('B', 8, AF1)>; /* SPI3_MOSI */
-			drive-push-pull;
-			bias-disable;
+			pinmux = <STM32_PINMUX('E', 13, AF12)>, /* SDMMC2_D0 */
+				 <STM32_PINMUX('E', 11, AF12)>, /* SDMMC2_D1 */
+				 <STM32_PINMUX('E', 8, AF12)>, /* SDMMC2_D2 */
+				 <STM32_PINMUX('E', 12, AF12)>, /* SDMMC2_D3 */
+				 <STM32_PINMUX('E', 15, AF12)>; /* SDMMC2_CMD */
 			slew-rate = <1>;
+			drive-push-pull;
+			bias-pull-up;
 		};
 		pins2 {
-			pinmux = <STM32_PINMUX('B', 10, AF1)>; /* SPI3_MISO */
+			pinmux = <STM32_PINMUX('E', 14, AF12)>; /* SDMMC2_CK */
+			slew-rate = <2>;
+			drive-push-pull;
 			bias-disable;
 		};
 	};
 
 	/omit-if-no-ref/
-	spi3_sleep_pins_a: spi3-sleep-0 {
+	sdmmc2_b4_od_pins_a: sdmmc2-b4-od-0 {
 		pins1 {
-			pinmux = <STM32_PINMUX('B', 7, ANALOG)>, /* SPI3_SCK */
-				 <STM32_PINMUX('B', 8, ANALOG)>, /* SPI3_MOSI */
-				 <STM32_PINMUX('B', 10, ANALOG)>; /* SPI3_MISO */
+			pinmux = <STM32_PINMUX('E', 13, AF12)>, /* SDMMC2_D0 */
+				 <STM32_PINMUX('E', 11, AF12)>, /* SDMMC2_D1 */
+				 <STM32_PINMUX('E', 8, AF12)>, /* SDMMC2_D2 */
+				 <STM32_PINMUX('E', 12, AF12)>; /* SDMMC2_D3 */
+			slew-rate = <1>;
+			drive-push-pull;
+			bias-pull-up;
 		};
-	};
-
-	/omit-if-no-ref/
-	tim10_counter_pins_a: tim10-counter-0 {
-		pins {
-			pinmux = <STM32_PINMUX('B', 9, AF9)>; /* TIM10_CH1 */
+		pins2 {
+			pinmux = <STM32_PINMUX('E', 14, AF12)>; /* SDMMC2_CK */
+			slew-rate = <2>;
+			drive-push-pull;
 			bias-disable;
 		};
+		pins3 {
+			pinmux = <STM32_PINMUX('E', 15, AF12)>; /* SDMMC2_CMD */
+			slew-rate = <1>;
+			drive-open-drain;
+			bias-pull-up;
+		};
 	};
 
 	/omit-if-no-ref/
-	tim10_counter_sleep_pins_a: tim10-counter-sleep-0 {
-		pins {
-			pinmux = <STM32_PINMUX('B', 9, ANALOG)>; /* TIM10_CH1 */
-			bias-disable;
-		};
+	sdmmc2_b4_pins_b: sdmmc2-b4-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('E', 13, AF12)>, /* SDMMC2_D0 */
+				 <STM32_PINMUX('E', 11, AF12)>, /* SDMMC2_D1 */
+				 <STM32_PINMUX('E', 8, AF12)>, /* SDMMC2_D2 */
+				 <STM32_PINMUX('E', 12, AF12)>, /* SDMMC2_D3 */
+				 <STM32_PINMUX('E', 15, AF12)>; /* SDMMC2_CMD */
+			slew-rate = <2>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('E', 14, AF12)>; /* SDMMC2_CK */
+			slew-rate = <3>;
+			drive-push-pull;
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc2_b4_od_pins_b: sdmmc2-b4-od-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('E', 13, AF12)>, /* SDMMC2_D0 */
+				 <STM32_PINMUX('E', 11, AF12)>, /* SDMMC2_D1 */
+				 <STM32_PINMUX('E', 8, AF12)>, /* SDMMC2_D2 */
+				 <STM32_PINMUX('E', 12, AF12)>; /* SDMMC2_D3 */
+			slew-rate = <2>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('E', 14, AF12)>; /* SDMMC2_CK */
+			slew-rate = <3>;
+			drive-push-pull;
+			bias-disable;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('E', 15, AF12)>; /* SDMMC2_CMD */
+			slew-rate = <2>;
+			drive-open-drain;
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc2_d47_pins_a: sdmmc2-d47-0 {
+		pins {
+			pinmux = <STM32_PINMUX('E', 10, AF12)>, /* SDMMC2_D4 */
+				 <STM32_PINMUX('E', 9, AF12)>, /* SDMMC2_D5 */
+				 <STM32_PINMUX('E', 6, AF12)>, /* SDMMC2_D6 */
+				 <STM32_PINMUX('E', 7, AF12)>; /* SDMMC2_D7 */
+			slew-rate = <1>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc2_b4_sleep_pins_a: sdmmc2-b4-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('E', 13, ANALOG)>, /* SDMMC2_D0 */
+				 <STM32_PINMUX('E', 11, ANALOG)>, /* SDMMC2_D1 */
+				 <STM32_PINMUX('E', 8, ANALOG)>, /* SDMMC2_D2 */
+				 <STM32_PINMUX('E', 12, ANALOG)>, /* SDMMC2_D3 */
+				 <STM32_PINMUX('E', 14, ANALOG)>, /* SDMMC2_CK */
+				 <STM32_PINMUX('E', 15, ANALOG)>; /* SDMMC2_CMD */
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc2_d47_sleep_pins_a: sdmmc2-d47-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('E', 10, ANALOG)>, /* SDMMC2_D4 */
+				 <STM32_PINMUX('E', 9, ANALOG)>, /* SDMMC2_D5 */
+				 <STM32_PINMUX('E', 6, ANALOG)>, /* SDMMC2_D6 */
+				 <STM32_PINMUX('E', 7, ANALOG)>; /* SDMMC2_D7 */
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc3_b4_pins_a: sdmmc3-b4-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 14, AF10)>, /* SDMMC3_D0 */
+				 <STM32_PINMUX('D', 13, AF10)>, /* SDMMC3_D1 */
+				 <STM32_PINMUX('B', 12, AF10)>, /* SDMMC3_D2 */
+				 <STM32_PINMUX('I', 11, AF10)>, /* SDMMC3_D3 */
+				 <STM32_PINMUX('D', 12, AF10)>; /* SDMMC3_CMD */
+			slew-rate = <0>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 13, AF10)>; /* SDMMC3_CK */
+			slew-rate = <1>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc3_b4_pins_b: sdmmc3-b4-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 14, AF10)>, /* SDMMC3_D0 */
+				 <STM32_PINMUX('D', 13, AF10)>, /* SDMMC3_D1 */
+				 <STM32_PINMUX('B', 12, AF10)>, /* SDMMC3_D2 */
+				 <STM32_PINMUX('D', 14, AF12)>, /* SDMMC3_D3 */
+				 <STM32_PINMUX('D', 12, AF10)>; /* SDMMC3_CMD */
+			slew-rate = <0>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 13, AF10)>; /* SDMMC3_CK */
+			slew-rate = <1>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc3_b4_od_pins_b: sdmmc3-b4-od-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 14, AF10)>, /* SDMMC3_D0 */
+				 <STM32_PINMUX('D', 13, AF10)>, /* SDMMC3_D1 */
+				 <STM32_PINMUX('B', 12, AF10)>, /* SDMMC3_D2 */
+				 <STM32_PINMUX('D', 14, AF10)>; /* SDMMC3_D3 */
+			slew-rate = <2>;
+			drive-push-pull;
+			bias-disable;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 13, AF10)>; /* SDMMC3_CK */
+			slew-rate = <3>;
+			drive-push-pull;
+			bias-disable;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('D', 12, AF10)>; /* SDMMC3_CMD */
+			slew-rate = <2>;
+			drive-open-drain;
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc3_b4_sleep_pins_b: sdmmc3-b4-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 14, ANALOG)>, /* SDMMC3_D0 */
+				 <STM32_PINMUX('D', 13, ANALOG)>, /* SDMMC3_D1 */
+				 <STM32_PINMUX('B', 12, ANALOG)>, /* SDMMC3_D2 */
+				 <STM32_PINMUX('D', 14, ANALOG)>, /* SDMMC3_D3 */
+				 <STM32_PINMUX('B', 13, ANALOG)>, /* SDMMC3_CK */
+				 <STM32_PINMUX('D', 12, ANALOG)>; /* SDMMC3_CMD */
+		};
+	};
+
+	/omit-if-no-ref/
+	sdmmc2_d47_pins_b: sdmmc2-d47-1 {
+		pins {
+			pinmux = <STM32_PINMUX('E', 10, AF12)>, /* SDMMC2_D4 */
+				 <STM32_PINMUX('E', 9, AF12)>, /* SDMMC2_D5 */
+				 <STM32_PINMUX('E', 6, AF12)>, /* SDMMC2_D6 */
+				 <STM32_PINMUX('E', 7, AF12)>; /* SDMMC2_D7 */
+			slew-rate = <2>;
+			drive-push-pull;
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	spi1_pins_a: spi1-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('G', 6, AF3)>, /* SPI1_SCK */
+				 <STM32_PINMUX('I', 5, AF3)>; /* SPI1_MOSI */
+			drive-push-pull;
+			bias-disable;
+			slew-rate = <1>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('F', 12, AF3)>; /* SPI1_MISO */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	spi1_sleep_pins_a: spi1-sleep-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('G', 6, ANALOG)>, /* SPI1_SCK */
+				 <STM32_PINMUX('I', 5, ANALOG)>, /* SPI1_MOSI */
+				 <STM32_PINMUX('F', 12, ANALOG)>; /* SPI1_MISO */
+		};
+	};
+
+	/omit-if-no-ref/
+	spi1_pins_b: spi1-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('G', 6, AF3)>, /* SPI1_SCK */
+				 <STM32_PINMUX('H', 7, AF2)>; /* SPI1_MOSI */
+			drive-push-pull;
+			bias-disable;
+			slew-rate = <1>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('H', 8, AF2)>; /* SPI1_MISO */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	spi1_sleep_pins_b: spi1-sleep-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('G', 6, ANALOG)>, /* SPI1_SCK */
+				 <STM32_PINMUX('H', 7, ANALOG)>, /* SPI1_MOSI */
+				 <STM32_PINMUX('H', 8, ANALOG)>; /* SPI1_MISO */
+		};
+	};
+
+	/omit-if-no-ref/
+	spi3_pins_a: spi3-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 7, AF1)>, /* SPI3_SCK */
+				 <STM32_PINMUX('B', 8, AF1)>; /* SPI3_MOSI */
+			drive-push-pull;
+			bias-disable;
+			slew-rate = <1>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 10, AF1)>; /* SPI3_MISO */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	spi3_sleep_pins_a: spi3-sleep-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 7, ANALOG)>, /* SPI3_SCK */
+				 <STM32_PINMUX('B', 8, ANALOG)>, /* SPI3_MOSI */
+				 <STM32_PINMUX('B', 10, ANALOG)>; /* SPI3_MISO */
+		};
+	};
+
+	/omit-if-no-ref/
+	tim10_counter_pins_a: tim10-counter-0 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 9, AF9)>; /* TIM10_CH1 */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	tim10_counter_sleep_pins_a: tim10-counter-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 9, ANALOG)>; /* TIM10_CH1 */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart1_pins_b: usart1-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 8, AF6)>; /* USART1_TX */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 10, AF6)>; /* USART1_RX */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart1_idle_pins_b: usart1-idle-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('B', 8, ANALOG)>; /* USART1_TX */
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 10, AF6)>; /* USART1_RX */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart1_sleep_pins_b: usart1-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 8, ANALOG)>, /* USART1_TX */
+				 <STM32_PINMUX('B', 10, ANALOG)>; /* USART1_RX */
+		};
+	};
+
+	/omit-if-no-ref/
+	usart1_pins_a: usart1-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('A', 3, AF6)>, /* USART1_TX */
+				 <STM32_PINMUX('A', 1, AF5)>;  /* USART1_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('A', 2, AF6)>, /* USART1_RX */
+				 <STM32_PINMUX('A', 7, AF6)>; /* USART1_CTS_NSS */
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart1_idle_pins_a: usart1-idle-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('A', 3, ANALOG)>, /* USART1_TX */
+				 <STM32_PINMUX('A', 7, ANALOG)>; /* USART1_CTS_NSS */
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('A', 1, AF5)>; /* USART1_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('A', 2, AF6)>; /* USART1_RX */
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart1_sleep_pins_a: usart1-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('A', 3, ANALOG)>, /* USART1_TX */
+				 <STM32_PINMUX('A', 1, ANALOG)>,  /* USART1_RTS */
+				 <STM32_PINMUX('A', 7, ANALOG)>, /* USART1_CTS_NSS */
+				 <STM32_PINMUX('A', 2, ANALOG)>; /* USART1_RX */
+		};
 	};
 
 	/omit-if-no-ref/
@@ -504,6 +1168,50 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	usart2_pins_b: usart2-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('A', 4, AF6)>, /* USART2_TX */
+				 <STM32_PINMUX('A', 5, AF6)>; /* USART2_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('A', 8, AF8)>, /* USART2_RX */
+				 <STM32_PINMUX('A', 9, AF6)>; /* USART2_CTS_NSS */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart2_idle_pins_b: usart2-idle-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('A', 4, ANALOG)>, /* USART2_TX */
+				 <STM32_PINMUX('A', 9, ANALOG)>; /* USART2_CTS_NSS */
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('A', 5, AF6)>; /* USART2_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('A', 8, AF8)>; /* USART2_RX */
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart2_sleep_pins_b: usart2-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('A', 4, ANALOG)>, /* USART2_TX */
+				 <STM32_PINMUX('A', 5, ANALOG)>, /* USART2_RTS */
+				 <STM32_PINMUX('A', 9, ANALOG)>, /* USART2_CTS_NSS */
+				 <STM32_PINMUX('A', 8, ANALOG)>; /* USART2_RX */
+		};
+	};
+
 	/omit-if-no-ref/
 	usart6_pins_a: usart6-0 {
 		pins1 {
@@ -547,6 +1255,127 @@
 				 <STM32_PINMUX('F', 14, ANALOG)>; /* USART6_RX */
 		};
 	};
+
+	/omit-if-no-ref/
+	usart6_pins_b: usart6-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('F', 5, AF6)>; /* USART6_TX */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('F', 4, AF6)>; /* USART6_RX */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart6_idle_pins_b: usart6-idle-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('F', 5, ANALOG)>; /* USART6_TX */
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('F', 4, AF6)>; /* USART6_RX */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	usart6_sleep_pins_b: usart6-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('F', 5, ANALOG)>, /* USART6_TX */
+				 <STM32_PINMUX('F', 4, ANALOG)>; /* USART6_RX */
+		};
+	};
+
+	/omit-if-no-ref/
+	uart8_pins_a: uart8-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('I', 11, AF5)>, /* UART8_TX */
+				 <STM32_PINMUX('B', 15, AF3)>;  /* UART8_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('F', 11, AF6)>, /* UART8_RX */
+				 <STM32_PINMUX('H', 5, AF6)>; /* UART8_CTS_NSS */
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	uart8_idle_pins_a: uart8-idle-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('I', 11, ANALOG)>, /* UART8_TX */
+				 <STM32_PINMUX('H', 5, ANALOG)>; /* UART8_CTS_NSS */
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('B', 15, AF3)>; /* UART8_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('F', 11, AF6)>; /* UART8_RX */
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	uart8_sleep_pins_a: uart8-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('I', 11, ANALOG)>, /* UART8_TX */
+				 <STM32_PINMUX('B', 15, ANALOG)>,  /* UART8_RTS */
+				 <STM32_PINMUX('H', 5, ANALOG)>, /* UART8_CTS_NSS */
+				 <STM32_PINMUX('F', 11, ANALOG)>; /* UART8_RX */
+		};
+	};
+
+	/omit-if-no-ref/
+	uart9_pins_a: uart9-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('G', 8, AF6)>, /* UART9_TX */
+				 <STM32_PINMUX('F', 12, AF6)>;  /* UART9_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('I', 5, AF6)>, /* UART9_RX */
+				 <STM32_PINMUX('G', 7, AF6)>; /* UART9_CTS_NSS */
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	uart9_idle_pins_a: uart9-idle-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('G', 8, ANALOG)>, /* UART9_TX */
+				 <STM32_PINMUX('G', 7, ANALOG)>; /* UART9_CTS_NSS */
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('F', 12, AF6)>; /* UART9_RTS */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <0>;
+		};
+		pins3 {
+			pinmux = <STM32_PINMUX('I', 5, AF6)>; /* UART9_RX */
+			bias-pull-up;
+		};
+	};
+
+	/omit-if-no-ref/
+	uart9_sleep_pins_a: uart9-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('G', 8, ANALOG)>, /* UART9_TX */
+				 <STM32_PINMUX('F', 12, ANALOG)>,  /* UART9_RTS */
+				 <STM32_PINMUX('G', 7, ANALOG)>, /* UART9_CTS_NSS */
+				 <STM32_PINMUX('I', 5, ANALOG)>; /* UART9_RX */
+		};
+	};
 };
 
 &pinctrl_z {
@@ -569,6 +1398,25 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	i2c8_pins_b: i2c8-1 {
+		pins {
+			pinmux = <STM32_PINMUX('Z', 4, AF8)>, /* I2C1_SCL */
+				 <STM32_PINMUX('Z', 9, AF8)>; /* I2C1_SDA */
+			bias-disable;
+			drive-open-drain;
+			slew-rate = <0>;
+		};
+	};
+
+	/omit-if-no-ref/
+	i2c8_sleep_pins_b: i2c8-sleep-1 {
+		pins {
+			pinmux = <STM32_PINMUX('Z', 4, ANALOG)>, /* I2C1_SCL */
+				 <STM32_PINMUX('Z', 9, ANALOG)>; /* I2C1_SDA */
+		};
+	};
+
 	/omit-if-no-ref/
 	spi8_pins_a: spi8-0 {
 		pins1 {
@@ -592,4 +1440,28 @@
 				 <STM32_PINMUX('Z', 1, ANALOG)>; /* SPI8_MISO */
 		};
 	};
+
+	/omit-if-no-ref/
+	spi8_pins_b: spi8-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('Z', 5, AF3)>, /* SPI8_SCK */
+				 <STM32_PINMUX('Z', 7, AF3)>; /* SPI8_MOSI */
+			drive-push-pull;
+			bias-disable;
+			slew-rate = <1>;
+		};
+		pins2 {
+			pinmux = <STM32_PINMUX('Z', 8, AF3)>; /* SPI8_MISO */
+			bias-disable;
+		};
+	};
+
+	/omit-if-no-ref/
+	spi8_sleep_pins_b: spi8-sleep-1 {
+		pins1 {
+			pinmux = <STM32_PINMUX('Z', 5, ANALOG)>, /* SPI8_SCK */
+				 <STM32_PINMUX('Z', 7, ANALOG)>, /* SPI8_MOSI */
+				 <STM32_PINMUX('Z', 8, ANALOG)>; /* SPI8_MISO */
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/st/stm32mp251.dtsi b/arch/arm64/boot/dts/st/stm32mp251.dtsi
index 9c63fdb5a885af..dac3d2d23ff008 100644
--- a/arch/arm64/boot/dts/st/stm32mp251.dtsi
+++ b/arch/arm64/boot/dts/st/stm32mp251.dtsi
@@ -36,7 +36,7 @@
 
 	arm_wdt: watchdog {
 		compatible = "arm,smc-wdt";
-		arm,smc-id = <0xb200005a>;
+		arm,smc-id = <0xbc000000>;
 		status = "disabled";
 	};
 
@@ -1291,7 +1291,7 @@
 					status = "disabled";
 				};
 
-				sai3b: audio-controller@502b0024 {
+				sai3b: audio-controller@402b0024 {
 					compatible = "st,stm32-sai-sub-b";
 					reg = <0x24 0x20>;
 					#sound-dai-cells = <0>;
@@ -1450,6 +1450,20 @@
 				status = "disabled";
 			};
 
+			dcmi: dcmi@404a0000 {
+				compatible = "st,stm32-dcmi";
+				reg = <0x404a0000 0x400>;
+				interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>;
+				resets = <&rcc CCI_R>;
+				clocks = <&rcc CK_BUS_CCI>;
+				clock-names = "mclk";
+				dmas = <&hpdma 137 0x60 0x00003012>;
+				dma-names = "tx";
+				access-controllers = <&rifsc 88>;
+				power-domains = <&CLUSTER_PD>;
+				status = "disabled";
+			};
+
 			rng: rng@42020000 {
 				compatible = "st,stm32mp25-rng";
 				reg = <0x42020000 0x400>;
@@ -1657,13 +1671,45 @@
 				arm,primecell-periphid = <0x00353180>;
 				reg = <0x48220000 0x400>, <0x44230400 0x8>;
 				interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&rcc CK_KER_SDMMC1 >;
+				clocks = <&rcc CK_KER_SDMMC1>;
 				clock-names = "apb_pclk";
 				resets = <&rcc SDMMC1_R>;
 				cap-sd-highspeed;
 				cap-mmc-highspeed;
 				max-frequency = <120000000>;
 				access-controllers = <&rifsc 76>;
+				power-domains = <&CLUSTER_PD>;
+				status = "disabled";
+			};
+
+			sdmmc2: mmc@48230000 {
+				compatible = "st,stm32mp25-sdmmc2", "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00353180>;
+				reg = <0x48230000 0x400>, <0x44230800 0x8>;
+				interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&rcc CK_KER_SDMMC2>;
+				clock-names = "apb_pclk";
+				resets = <&rcc SDMMC2_R>;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				max-frequency = <166000000>;
+				access-controllers = <&rifsc 77>;
+				power-domains = <&CLUSTER_PD>;
+				status = "disabled";
+			};
+
+			sdmmc3: mmc@48240000 {
+				compatible = "st,stm32mp25-sdmmc2", "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00353180>;
+				reg = <0x48240000 0x400>, <0x44230c00 0x8>;
+				interrupts = <GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&rcc CK_KER_SDMMC3>;
+				clock-names = "apb_pclk";
+				resets = <&rcc SDMMC3_R>;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				max-frequency = <166000000>;
+				access-controllers = <&rifsc 78>;
 				status = "disabled";
 			};
 
@@ -1716,6 +1762,47 @@
 				};
 			};
 
+			usbh_ohci: usb@482e0000 {
+				compatible = "st,stm32mp25-ohci", "generic-ohci";
+				reg = <0x482e0000 0x1000>;
+				access-controllers = <&rifsc 63>;
+				clocks = <&usb2_phy1 1>, <&rcc CK_BUS_USB2OHCI>;
+				interrupts = <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&usb2_phy1>;
+				phy-names = "usb";
+				resets = <&rcc USBH_R>;
+				status = "disabled";
+			};
+
+			usbh_ehci: usb@482f0000 {
+				compatible = "st,stm32mp25-ehci", "generic-ehci";
+				reg = <0x482f0000 0x1000>;
+				access-controllers = <&rifsc 63>;
+				clocks = <&usb2_phy1 0>, <&rcc CK_BUS_USB2EHCI>;
+				companion = <&usbh_ohci>;
+				interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&usb2_phy1>;
+				phy-names = "usb";
+				resets = <&rcc USBH_R>;
+				status = "disabled";
+			};
+
+			usb3dr: usb@48300000 {
+				compatible = "st,stm32mp25-dwc3";
+				reg = <0x48300000 0x100000>;
+				access-controllers = <&rifsc 66>;
+				clocks = <&rcc CK_KER_USB2PHY2>,
+					 <&rcc CK_BUS_USB3DR>,
+					 <&rcc CK_KER_USB2PHY2>;
+				clock-names = "ref", "bus_early", "suspend";
+				interrupts = <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&usb2_phy2>, <&combophy PHY_TYPE_USB3>;
+				phy-names = "usb2-phy", "usb3-phy";
+				resets = <&rcc USB3DR_R>;
+				st,syscfg = <&syscfg 0x4800>;
+				status = "disabled";
+			};
+
 			pcie_ep: pcie-ep@48400000 {
 				compatible = "st,stm32mp25-pcie-ep";
 				reg = <0x48400000 0x100000>,
@@ -1783,6 +1870,21 @@
 			};
 		};
 
+		dts: thermal-sensor@44070000 {
+			compatible = "moortec,mr75203";
+			reg = <0x44070000 0x80>,
+			      <0x44070080 0xc0>,
+			      <0x44070180 0x80>,
+			      <0x44070400 0x400>;
+			reg-names = "common", "ts", "pd", "vm";
+			clocks = <&rcc CK_KER_DTS>;
+			resets = <&rcc DTS_R>;
+			#thermal-sensor-cells = <1>;
+			moortec,ts-coeff-g = <58500>;
+			moortec,ts-coeff-h = <201200>;
+			moortec,ts-coeff-j = <0>;
+		};
+
 		hdp: pinctrl@44090000 {
 			compatible = "st,stm32mp251-hdp";
 			reg = <0x44090000 0x400>;
@@ -1975,6 +2077,28 @@
 			compatible = "st,stm32mp25-syscfg", "syscon";
 			reg = <0x44230000 0x10000>;
 			#clock-cells = <0>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			usb2_phy1: phy@2400 {
+				compatible = "st,stm32mp25-usb2phy1";
+				reg = <0x2400 0x24>;
+				#phy-cells = <0>;
+				#clock-cells = <1>;
+				clocks = <&rcc CK_KER_USB2PHY1>;
+				resets = <&rcc USB2PHY1_R>;
+				status = "disabled";
+			};
+
+			usb2_phy2: phy@2800 {
+				compatible = "st,stm32mp25-usb2phy2";
+				reg = <0x2800 0x10>;
+				#phy-cells = <0>;
+				#clock-cells = <1>;
+				clocks = <&rcc CK_KER_USB2PHY2EN>;
+				resets = <&rcc USB2PHY2_R>;
+				status = "disabled";
+			};
 		};
 
 		pinctrl: pinctrl@44240000 {
diff --git a/arch/arm64/boot/dts/st/stm32mp253.dtsi b/arch/arm64/boot/dts/st/stm32mp253.dtsi
index eeceb086252bfb..7e82f01fdc10f8 100644
--- a/arch/arm64/boot/dts/st/stm32mp253.dtsi
+++ b/arch/arm64/boot/dts/st/stm32mp253.dtsi
@@ -43,6 +43,22 @@
 };
 
 &rifsc {
+	m_can1: can@402d0000 {
+		compatible = "bosch,m_can";
+		reg = <0x402d0000 0x400>, <0x40310000 0xd50>;
+		reg-names = "m_can", "message_ram";
+		interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "int0", "int1";
+		clocks = <&rcc CK_BUS_FDCAN>, <&rcc CK_KER_FDCAN>;
+		clock-names = "hclk", "cclk";
+		resets = <&rcc FDCAN_R>;
+		bosch,mram-cfg = <0x0 0 0 32 0 0 2 2>;
+		access-controllers = <&rifsc 56>;
+		power-domains = <&CLUSTER_PD>;
+		status = "disabled";
+	};
+
 	ethernet2: ethernet@482d0000 {
 		compatible = "st,stm32mp25-dwmac", "snps,dwmac-5.20";
 		reg = <0x482d0000 0x4000>;
diff --git a/arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtso b/arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtso
new file mode 100644
index 00000000000000..fbec84eb7720f1
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc-overlay-imx219-x10.dtso
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-3-Clause)
+/*
+ * Copyright (C) 2026 Marek Vasut <marex@nabladev.com>
+ *
+ * Set up pipeline for raw bayer capture:
+ * $ media-ctl -d platform:48030000.dcmipp -r
+ * $ media-ctl -d platform:48030000.dcmipp -l '"48020000.csi":1->"dcmipp_input":0[1]'
+ * $ media-ctl -d platform:48030000.dcmipp -l "'dcmipp_input':1->'dcmipp_dump_postproc':0[1]"
+ * $ media-ctl -d platform:48030000.dcmipp --set-v4l2 "'imx219 0-0010':0[fmt:SRGGB8_1X8/1920x1080]"
+ * $ media-ctl -d platform:48030000.dcmipp --set-v4l2 "'48020000.csi':1[fmt:SRGGB8_1X8/1920x1080]"
+ * $ media-ctl -d platform:48030000.dcmipp --set-v4l2 "'dcmipp_input':1[fmt:SRGGB8_1X8/1920x1080 field:none]"
+ * $ media-ctl -d platform:48030000.dcmipp --set-v4l2 "'dcmipp_dump_postproc':0[compose:(0,0)/1920x1080]"
+ * $ media-ctl -d platform:48030000.dcmipp --set-v4l2 "'dcmipp_dump_postproc':1[fmt:SRGGB8_1X8/1920x1080]"
+ * $ v4l2-ctl -d /dev/video0 --set-fmt-video=width=1920,height=1080,pixelformat=RGGB
+ *
+ * Capture frame using v4l2-ctl:
+ * $ v4l2-ctl -d /dev/video0 --stream-mmap --stream-count=1 --stream-to=/frame.raw
+ *
+ * Capture frame using gstreamer:
+ * $ gst-launch-1.0 v4l2src device=/dev/video0 num-buffers=1 ! \
+ *                  video/x-bayer,width=1920,height=1080,format=rggb ! \
+ *                  bayer2rgb ! jpegenc ! filesink location=/test.jpg
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+
+&{/} {
+	clk_cam_x10: clk-cam-j1 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+	};
+
+	/* Page 29 / CSI_IF_CN / J1 */
+	reg_cam_x10: reg-cam-j1 {
+		compatible = "regulator-fixed";
+		regulator-name = "cam-X10";
+		enable-active-high;
+		gpios = <&ioexp 13 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&csi {
+	vdd-supply = <&scmi_vddcore>;
+	vdda18-supply = <&scmi_v1v8>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			csi_sink: endpoint {
+				remote-endpoint = <&imx219_x10_out>;
+				data-lanes = <1 2>;
+				bus-type = <4>;
+			};
+		};
+		port@1 {
+			reg = <1>;
+
+			csi_source: endpoint {
+				remote-endpoint = <&dcmipp_0>;
+			};
+		};
+	};
+};
+
+&dcmipp {
+	status = "okay";
+
+	port {
+		dcmipp_0: endpoint {
+			remote-endpoint = <&csi_source>;
+			bus-type = <4>;
+		};
+	};
+};
+
+&i2c2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	cam@10 {
+		compatible = "sony,imx219";
+		reg = <0x10>;
+
+		clocks = <&clk_cam_x10>;
+
+		VANA-supply = <&reg_cam_x10>;
+		VDIG-supply = <&reg_cam_x10>;
+		VDDL-supply = <&reg_cam_x10>;
+
+		orientation = <2>;
+		rotation = <0>;
+
+		port {
+			imx219_x10_out: endpoint {
+				clock-noncontinuous;
+				link-frequencies = /bits/ 64 <456000000>;
+				data-lanes = <1 2>;
+				remote-endpoint = <&csi_sink>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc.dts b/arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc.dts
new file mode 100644
index 00000000000000..332a8034cc3724
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp255c-dhcos-dhsbc.dts
@@ -0,0 +1,189 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-3-Clause)
+/*
+ * Copyright (C) 2025-2026 Marek Vasut <marex@nabladev.com>
+ */
+
+/dts-v1/;
+
+#include "stm32mp255.dtsi"
+#include "stm32mp25xc.dtsi"
+#include "stm32mp25xx-dhcos-som.dtsi"
+
+/ {
+	model = "DH electronics STM32MP255C DHCOS DHSBC";
+	compatible = "dh,stm32mp255c-dhcos-dhsbc", "dh,stm32mp255c-dhcos-som", "st,stm32mp255";
+
+	aliases {
+		ethernet0 = &ethernet1;
+		ethernet1 = &ethernet2;
+	};
+};
+
+&ethernet1 {
+	phy-handle = <&ethphy1>;
+	phy-mode = "rgmii-id";
+	pinctrl-0 = <&eth1_mdio_pins_a &eth1_rgmii_pins_a>;
+	pinctrl-1 = <&eth1_mdio_sleep_pins_a &eth1_rgmii_sleep_pins_a>;
+	pinctrl-names = "default", "sleep";
+	st,ext-phyclk;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		ethphy1: ethernet-phy@1 {
+			/* RTL8211F */
+			compatible = "ethernet-phy-id001c.c916";
+			interrupt-parent = <&gpioc>;
+			interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+			reg = <1>;
+			realtek,clkout-disable;
+			realtek,clkout-ssc-enable;
+			realtek,rxc-ssc-enable;
+			realtek,sysclk-ssc-enable;
+			reset-assert-us = <15000>;
+			reset-deassert-us = <55000>;
+			reset-gpios = <&ioexp 1 GPIO_ACTIVE_LOW>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@0 {
+					reg = <0>;
+					color = <LED_COLOR_ID_GREEN>;
+					function = LED_FUNCTION_WAN;
+					linux,default-trigger = "netdev";
+				};
+
+				led@1 {
+					reg = <1>;
+					color = <LED_COLOR_ID_YELLOW>;
+					function = LED_FUNCTION_WAN;
+					linux,default-trigger = "netdev";
+				};
+			};
+		};
+	};
+};
+
+&ethernet2 {
+	phy-handle = <&ethphy2>;
+	phy-mode = "rgmii-id";
+	pinctrl-0 = <&eth2_mdio_pins_a &eth2_rgmii_pins_b>;
+	pinctrl-1 = <&eth2_mdio_sleep_pins_a &eth2_rgmii_sleep_pins_b>;
+	pinctrl-names = "default", "sleep";
+	st,ext-phyclk;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+
+		ethphy2: ethernet-phy@1 {
+			/* RTL8211F */
+			compatible = "ethernet-phy-id001c.c916";
+			interrupt-parent = <&gpiog>;
+			interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+			reg = <1>;
+			realtek,clkout-disable;
+			realtek,clkout-ssc-enable;
+			realtek,rxc-ssc-enable;
+			realtek,sysclk-ssc-enable;
+			reset-assert-us = <15000>;
+			reset-deassert-us = <55000>;
+			reset-gpios = <&ioexp 0 GPIO_ACTIVE_LOW>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@0 {
+					reg = <0>;
+					color = <LED_COLOR_ID_GREEN>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+
+				led@1 {
+					reg = <1>;
+					color = <LED_COLOR_ID_YELLOW>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+			};
+		};
+	};
+};
+
+&gpioa {
+	gpio-line-names = "DHSBC_HW-CODE_0", "DHSBC_HW-CODE_1", "DHSBC_HW-CODE_2", "",
+			  "DHCOS-E", "DHCOS-J", "", "",
+			  "DHCOS-D", "", "", "",
+			  "", "", "", "";
+};
+
+&i2c2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c2_pins_a>;
+	pinctrl-1 = <&i2c2_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <57>;
+	i2c-scl-falling-time-ns = <7>;
+	clock-frequency = <400000>;
+	status = "okay";
+};
+
+&scmi_vddio1 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-settling-time-up-us = <1000>;
+	regulator-settling-time-down-us = <100000>;
+};
+
+&sdmmc1 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc1_b4_pins_b>;
+	pinctrl-1 = <&sdmmc1_b4_od_pins_b>;
+	pinctrl-2 = <&sdmmc1_b4_sleep_pins_a>;
+	cd-gpios = <&ioexp 8 GPIO_ACTIVE_HIGH>;
+	disable-wp;
+	st,neg-edge;
+	bus-width = <4>;
+	vmmc-supply = <&scmi_v3v3>;
+	vqmmc-supply = <&scmi_vddio1>;
+	sd-uhs-sdr12;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-ddr50;
+	sd-uhs-sdr104;
+	status = "okay";
+};
+
+&spi1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi1_pins_b>;
+	pinctrl-1 = <&spi1_sleep_pins_b>;
+	cs-gpios = <&gpioh 3 0>;
+	status = "okay";
+
+	st33htph: tpm@0 {
+		compatible = "st,st33htpm-spi", "tcg,tpm_tis-spi";
+		reg = <0>;
+		interrupt-parent = <&gpioa>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&gpioh 2 GPIO_ACTIVE_LOW>;
+		spi-max-frequency = <24000000>;
+	};
+};
+
+&spi8 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi8_pins_b>;
+	pinctrl-1 = <&spi8_sleep_pins_b>;
+	cs-gpios = <&gpioz 6 0>;
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp257-engicam-microgea.dtsi b/arch/arm64/boot/dts/st/stm32mp257-engicam-microgea.dtsi
new file mode 100644
index 00000000000000..5b4287e86deff9
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp257-engicam-microgea.dtsi
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 Amarula Solutions, Dario Binacchi <dario.binacchi@amarulasolutions.com>
+ * Copyright (C) 2026 Engicam srl
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/regulator/st,stm32mp25-regulator.h>
+#include "stm32mp257.dtsi"
+#include "stm32mp25-pinctrl.dtsi"
+#include "stm32mp25xxai-pinctrl.dtsi"
+
+/ {
+	model = "Engicam MicroGEA STM32MP257 SoM";
+	compatible = "engicam,microgea-stm32mp257", "st,stm32mp257";
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x80000000>;
+	};
+};
+
+&scmi_regu {
+	scmi_vddio1: regulator@0 {
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+	scmi_vddcore: regulator@b {
+		reg = <VOLTD_SCMI_STPMIC2_BUCK2>;
+		regulator-name = "vddcore";
+	};
+	scmi_v1v8: regulator@e {
+		reg = <VOLTD_SCMI_STPMIC2_BUCK5>;
+		regulator-name = "v1v8";
+	};
+	scmi_v3v3: regulator@10 {
+		reg = <VOLTD_SCMI_STPMIC2_BUCK7>;
+		regulator-name = "v3v3";
+	};
+	scmi_vdd3v3_usb: regulator@14 {
+		reg = <VOLTD_SCMI_STPMIC2_LDO4>;
+		regulator-name = "vdd3v3_usb";
+	};
+};
+
+/* eMMC */
+&sdmmc2 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc2_b4_pins_a &sdmmc2_d47_pins_a>;
+	pinctrl-1 = <&sdmmc2_b4_od_pins_a &sdmmc2_d47_pins_a>;
+	pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_a>;
+	non-removable;
+	no-sd;
+	no-sdio;
+	st,neg-edge;
+	bus-width = <8>;
+	vmmc-supply = <&scmi_v3v3>;
+	vqmmc-supply = <&scmi_vddio2>;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtso b/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtso
new file mode 100644
index 00000000000000..7655c1bf56b1ef
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-am-1280800w8tzqw-t00h.dtso
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 Amarula Solutions, Dario Binacchi <dario.binacchi@amarulasolutions.com>
+ * Copyright (C) 2026 Engicam srl
+ *
+ * Ampire AM-1280800W8TZQW-T00H 10.1" 1280x800 LVDS panel. Its touchscreen
+ * controller is connected over USB.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/clock/st,stm32mp25-rcc.h>
+
+&{/} {
+	panel-lvds {
+		compatible = "ampire,am-1280800w8tzqw-t00h";
+		backlight = <&backlight>;
+		power-supply = <&reg_3v3>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&lvds_out>;
+			};
+		};
+	};
+};
+
+&framebuffer {
+	clocks = <&rcc CK_BUS_LTDC>, <&rcc CK_KER_LTDC>,
+		 <&rcc CK_BUS_LVDS>, <&rcc CK_KER_LVDSPHY>;
+};
+
+&ltdc {
+	status = "okay";
+
+	port {
+		ltdc_out: endpoint {
+			remote-endpoint = <&lvds_in>;
+		};
+	};
+};
+
+&lvds {
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			reg = <0>;
+
+			lvds_in: endpoint {
+				remote-endpoint = <&ltdc_out>;
+			};
+		};
+
+		port@1 {
+			reg = <1>;
+
+			lvds_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a.dtso b/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a.dtso
new file mode 100644
index 00000000000000..f89641bdb23b6f
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm-overlay-rk050hr345-ct106a.dtso
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 Amarula Solutions, Dario Binacchi <dario.binacchi@amarulasolutions.com>
+ * Copyright (C) 2026 Engicam srl
+ *
+ * Rocktech RK050HR345-CT106A 5" 480x854 RGB panel with EDT FT5306
+ * I2C touchscreen controller.
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&i2c1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c1_pins_a>;
+	pinctrl-1 = <&i2c1_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "edt,edt-ft5306";
+		reg = <0x38>;
+		interrupt-parent = <&gpiob>;
+		interrupts = <0 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&gpiod 1 GPIO_ACTIVE_LOW>;
+		touchscreen-size-x = <480>;
+		touchscreen-size-y = <854>;
+	};
+};
+
+&ltdc {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&ltdc_pins_a>;
+	pinctrl-1 = <&ltdc_sleep_pins_a>;
+	status = "okay";
+
+	port {
+		ltdc_out: endpoint {
+			remote-endpoint = <&panel_in>;
+		};
+	};
+};
+
+&spi1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	display@0 {
+		compatible = "rocktech,rk050hr345-ct106a", "ilitek,ili9806e";
+		reg = <0>;
+		vdd-supply = <&reg_3v3>;
+		spi-max-frequency = <10000000>;
+		reset-gpios = <&gpiob 6 GPIO_ACTIVE_LOW>;
+		backlight = <&backlight>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&ltdc_out>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm.dts b/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm.dts
new file mode 100644
index 00000000000000..96b7cb6b5bc104
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp257d-engicam-microgea-rmm.dts
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 Amarula Solutions, Dario Binacchi <dario.binacchi@amarulasolutions.com>
+ * Copyright (C) 2026 Engicam srl
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+
+#include "stm32mp257-engicam-microgea.dtsi"
+
+/ {
+	model = "Engicam MicroGEA STM32MP257D RMM Board";
+	compatible = "engicam,microgea-stm32mp257d-rmm",
+		     "engicam,microgea-stm32mp257", "st,stm32mp257";
+
+	aliases {
+		mmc0 = &sdmmc1;
+		mmc1 = &sdmmc2;
+		serial0 = &usart2;
+		serial1 = &usart1;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		brightness-levels = <0 100>;
+		num-interpolated-steps = <100>;
+		default-brightness-level = <85>;
+		pwms = <&pwm2 0 100000 0>;
+	};
+
+	buzzer {
+		compatible = "pwm-beeper";
+		pwms = <&pwm4 0 1000000 0>;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		framebuffer: framebuffer {
+			compatible = "simple-framebuffer";
+			clocks = <&rcc CK_BUS_LTDC>, <&rcc CK_KER_LTDC>;
+			lcd-supply = <&reg_3v3>;
+			status = "disabled";
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			gpios = <&gpioh 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			status = "okay";
+		};
+
+		led-1 {
+			gpios = <&gpioh 6 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			status = "okay";
+		};
+	};
+
+	mclk: clock-mclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+	};
+
+	reg_1v8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	reg_3v3: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	reg_ext_pwr: regulator-ext-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "ext-pwr";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpiog 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	sound {
+		compatible = "audio-graph-card";
+		label = "STM32MP25-RMM";
+		widgets = "Headphone", "Headphone Jack",
+			  "Microphone", "Microphone Jack";
+		routing = "Headphone Jack", "HP_OUT",
+			  "MIC_IN", "Microphone Jack",
+			  "Microphone Jack", "Mic Bias";
+		dais = <&sai1a_port &sai1b_port>;
+		status = "okay";
+	};
+};
+
+&arm_wdt {
+	timeout-sec = <32>;
+	status = "okay";
+};
+
+&i2c1 {
+	/* dt overlays don't support /delete-property/ */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+};
+
+&i2c2 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c2_pins_a>;
+	pinctrl-1 = <&i2c2_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <185>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+
+	sgtl5000: codec@a {
+		compatible = "fsl,sgtl5000";
+		reg = <0x0a>;
+		#sound-dai-cells = <0>;
+		clocks = <&mclk>;
+
+		VDDA-supply = <&reg_3v3>;
+		VDDIO-supply = <&reg_3v3>;
+		VDDD-supply = <&reg_1v8>;
+
+		sgtl5000_port: port {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			sgtl5000_tx_endpoint: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&sai1a_endpoint>;
+				frame-master = <&sgtl5000_tx_endpoint>;
+				bitclock-master = <&sgtl5000_tx_endpoint>;
+			};
+
+			sgtl5000_rx_endpoint: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&sai1b_endpoint>;
+				frame-master = <&sgtl5000_rx_endpoint>;
+				bitclock-master = <&sgtl5000_rx_endpoint>;
+			};
+		};
+	};
+};
+
+&m_can1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&m_can1_pins_a>;
+	pinctrl-1 = <&m_can1_sleep_pins_a>;
+	status = "okay";
+};
+
+&sai1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&sai1a_pins_a>, <&sai1b_pins_a>;
+	pinctrl-1 = <&sai1a_sleep_pins_a>, <&sai1b_sleep_pins_a>;
+	status = "okay";
+};
+
+&sai1a {
+	dma-names = "tx";
+	status = "okay";
+
+	sai1a_port: port {
+		sai1a_endpoint: endpoint {
+			remote-endpoint = <&sgtl5000_tx_endpoint>;
+			dai-format = "i2s";
+			mclk-fs = <512>;
+		};
+	};
+};
+
+&sai1b {
+	dma-names = "rx";
+	st,sync = <&sai1a 2>;
+	clocks = <&rcc CK_KER_SAI1>;
+	clock-names = "sai_ck";
+	status = "okay";
+
+	sai1b_port: port {
+		sai1b_endpoint: endpoint {
+			remote-endpoint = <&sgtl5000_rx_endpoint>;
+			dai-format = "i2s";
+			mclk-fs = <512>;
+		};
+	};
+};
+
+/* MicroSD */
+&sdmmc1 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc1_b4_pins_a>;
+	pinctrl-1 = <&sdmmc1_b4_od_pins_a>;
+	pinctrl-2 = <&sdmmc1_b4_sleep_pins_a>;
+	broken-cd;
+	disable-wp;
+	st,neg-edge;
+	bus-width = <4>;
+	vmmc-supply = <&scmi_v3v3>;
+	vqmmc-supply = <&scmi_vddio1>;
+	no-1-8-v;
+	status = "okay";
+};
+
+&spi1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi1_pins_a>;
+	pinctrl-1 = <&spi1_sleep_pins_a>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	cs-gpios = <&gpioh 8 GPIO_ACTIVE_LOW>, <&gpioh 3 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&timers2 {
+	status = "okay";
+
+	pwm2: pwm {
+		pinctrl-0 = <&pwm2_pins_a>;
+		pinctrl-1 = <&pwm2_sleep_pins_a>;
+		pinctrl-names = "default", "sleep";
+		status = "okay";
+	};
+};
+
+&timers4 {
+	status = "okay";
+
+	pwm4: pwm {
+		pinctrl-names = "default", "sleep";
+		pinctrl-0 = <&pwm4_pins_a>;
+		pinctrl-1 = <&pwm4_sleep_pins_a>;
+		status = "okay";
+	};
+};
+
+&usart1 {
+	pinctrl-names = "default", "idle", "sleep";
+	pinctrl-0 = <&usart1_pins_b>;
+	pinctrl-1 = <&usart1_idle_pins_b>;
+	pinctrl-2 = <&usart1_sleep_pins_b>;
+	/delete-property/ dmas;
+	/delete-property/ dma-names;
+	status = "okay";
+};
+
+&usart2 {
+	pinctrl-names = "default", "idle", "sleep";
+	pinctrl-0 = <&usart2_pins_a>;
+	pinctrl-1 = <&usart2_idle_pins_a>;
+	pinctrl-2 = <&usart2_sleep_pins_a>;
+	/delete-property/ dmas;
+	/delete-property/ dma-names;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp257f-dk.dts b/arch/arm64/boot/dts/st/stm32mp257f-dk.dts
index 8daf3dfd513393..a8f18508db3e45 100644
--- a/arch/arm64/boot/dts/st/stm32mp257f-dk.dts
+++ b/arch/arm64/boot/dts/st/stm32mp257f-dk.dts
@@ -9,6 +9,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interfaces.h>
 #include "stm32mp257.dtsi"
 #include "stm32mp25xf.dtsi"
 #include "stm32mp25-pinctrl.dtsi"
@@ -21,12 +22,22 @@
 	aliases {
 		ethernet0 = &ethernet1;
 		serial0 = &usart2;
+		mmc0 = &sdmmc1;
+		mmc1 = &sdmmc2;
 	};
 
 	chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	clocks {
+		clk_ext_camera: clk-ext-camera {
+			#clock-cells = <0>;
+			compatible = "fixed-clock";
+			clock-frequency = <24000000>;
+		};
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 
@@ -138,6 +149,42 @@
 	status = "okay";
 };
 
+&csi {
+	vdd-supply =  <&scmi_vddcore>;
+	vdda18-supply = <&scmi_v1v8>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+			csi_sink: endpoint {
+				remote-endpoint = <&imx335_ep>;
+				data-lanes = <1 2>;
+				bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+			};
+		};
+		port@1 {
+			reg = <1>;
+			csi_source: endpoint {
+				remote-endpoint = <&dcmipp_0>;
+			};
+		};
+	};
+};
+
+&dcmipp {
+	status = "okay";
+
+	port {
+		dcmipp_0: endpoint {
+			remote-endpoint = <&csi_source>;
+			bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+		};
+	};
+};
+
 &ethernet1 {
 	pinctrl-0 = <&eth1_rgmii_pins_b>;
 	pinctrl-1 = <&eth1_rgmii_sleep_pins_b>;
@@ -160,6 +207,16 @@
 	};
 };
 
+&gpiob {
+	/* Enable the IMX335 power line */
+	imx335-en-hog {
+		gpio-hog;
+		gpios = <11 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "imx335_en";
+	};
+};
+
 &i2c2 {
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&i2c2_pins_b>;
@@ -172,6 +229,25 @@
 	/delete-property/dmas;
 	/delete-property/dma-names;
 
+	imx335: camera@1a {
+		compatible = "sony,imx335";
+		reg = <0x1a>;
+		clocks = <&clk_ext_camera>;
+		avdd-supply = <&scmi_v3v3>;
+		ovdd-supply = <&scmi_v3v3>;
+		dvdd-supply = <&scmi_v3v3>;
+		reset-gpios = <&gpiob 1 (GPIO_ACTIVE_LOW | GPIO_PUSH_PULL)>;
+
+		port {
+			imx335_ep: endpoint {
+				remote-endpoint = <&csi_sink>;
+				clock-lanes = <0>;
+				data-lanes = <1 2>;
+				link-frequencies = /bits/ 64 <594000000>;
+			};
+		};
+	};
+
 	ili2511: ili2511@41 {
 		compatible = "ilitek,ili251x";
 		reg = <0x41>;
@@ -279,6 +355,23 @@
 	status = "okay";
 };
 
+&sdmmc2 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc2_b4_pins_b &sdmmc2_d47_pins_b>;
+	pinctrl-1 = <&sdmmc2_b4_od_pins_b &sdmmc2_d47_pins_b>;
+	pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_a>;
+	non-removable;
+	no-sd;
+	no-sdio;
+	st,neg-edge;
+	bus-width = <8>;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	vmmc-supply = <&scmi_vdd_emmc>;
+	vqmmc-supply = <&scmi_vddio2>;
+	status = "okay";
+};
+
 &usart2 {
 	pinctrl-names = "default", "idle", "sleep";
 	pinctrl-0 = <&usart2_pins_a>;
diff --git a/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts b/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts
index b5db62a0a5504b..782b4bb843eeb1 100644
--- a/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts
+++ b/arch/arm64/boot/dts/st/stm32mp257f-ev1.dts
@@ -8,6 +8,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/media/video-interfaces.h>
 #include <dt-bindings/regulator/st,stm32mp25-regulator.h>
 #include "stm32mp257.dtsi"
 #include "stm32mp25xf.dtsi"
@@ -72,30 +73,6 @@
 		io-width = <32>;
 	};
 
-	imx335_2v9: regulator-2v9 {
-		compatible = "regulator-fixed";
-		regulator-name = "imx335-avdd";
-		regulator-min-microvolt = <2900000>;
-		regulator-max-microvolt = <2900000>;
-		regulator-always-on;
-	};
-
-	imx335_1v8: regulator-1v8 {
-		compatible = "regulator-fixed";
-		regulator-name = "imx335-ovdd";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		regulator-always-on;
-	};
-
-	imx335_1v2: regulator-1v2 {
-		compatible = "regulator-fixed";
-		regulator-name = "imx335-dvdd";
-		regulator-min-microvolt = <1200000>;
-		regulator-max-microvolt = <1200000>;
-		regulator-always-on;
-	};
-
 	memory@80000000 {
 		device_type = "memory";
 		reg = <0x0 0x80000000 0x1 0x0>;
@@ -185,7 +162,7 @@
 			csi_sink: endpoint {
 				remote-endpoint = <&imx335_ep>;
 				data-lanes = <1 2>;
-				bus-type = <4>;
+				bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
 			};
 		};
 		port@1 {
@@ -202,7 +179,7 @@
 	port {
 		dcmipp_0: endpoint {
 			remote-endpoint = <&csi_source>;
-			bus-type = <4>;
+			bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
 		};
 	};
 };
@@ -253,6 +230,16 @@
 	};
 };
 
+&gpioi {
+	/* Enable the IMX335 power line */
+	imx335-en-hog {
+		gpio-hog;
+		gpios = <0 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "imx335_en";
+	};
+};
+
 &i2c2 {
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&i2c2_pins_a>;
@@ -269,9 +256,9 @@
 		compatible = "sony,imx335";
 		reg = <0x1a>;
 		clocks = <&clk_ext_camera>;
-		avdd-supply = <&imx335_2v9>;
-		ovdd-supply = <&imx335_1v8>;
-		dvdd-supply = <&imx335_1v2>;
+		avdd-supply = <&scmi_v3v3>;
+		ovdd-supply = <&scmi_v3v3>;
+		dvdd-supply = <&scmi_v3v3>;
 		reset-gpios = <&gpioi 7 (GPIO_ACTIVE_LOW | GPIO_PUSH_PULL)>;
 
 		port {
@@ -307,34 +294,6 @@
 	/delete-property/dma-names;
 };
 
-&ommanager {
-	memory-region = <&mm_ospi1>;
-	memory-region-names = "ospi1";
-	pinctrl-0 = <&ospi_port1_clk_pins_a
-		     &ospi_port1_io03_pins_a
-		     &ospi_port1_cs0_pins_a>;
-	pinctrl-1 = <&ospi_port1_clk_sleep_pins_a
-		     &ospi_port1_io03_sleep_pins_a
-		     &ospi_port1_cs0_sleep_pins_a>;
-	pinctrl-names = "default", "sleep";
-	status = "okay";
-
-	spi@0 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		memory-region = <&mm_ospi1>;
-		status = "okay";
-
-		flash0: flash@0 {
-			compatible = "jedec,spi-nor";
-			reg = <0>;
-			spi-rx-bus-width = <4>;
-			spi-tx-bus-width = <4>;
-			spi-max-frequency = <50000000>;
-		};
-	};
-};
-
 /* use LPTIMER with tick broadcast for suspend mode */
 &lptimer3 {
 	status = "okay";
@@ -374,6 +333,34 @@
 	};
 };
 
+&ommanager {
+	memory-region = <&mm_ospi1>;
+	memory-region-names = "ospi1";
+	pinctrl-0 = <&ospi_port1_clk_pins_a
+		     &ospi_port1_io03_pins_a
+		     &ospi_port1_cs0_pins_a>;
+	pinctrl-1 = <&ospi_port1_clk_sleep_pins_a
+		     &ospi_port1_io03_sleep_pins_a
+		     &ospi_port1_cs0_sleep_pins_a>;
+	pinctrl-names = "default", "sleep";
+	status = "okay";
+
+	spi@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		memory-region = <&mm_ospi1>;
+		status = "okay";
+
+		flash0: flash@0 {
+			compatible = "jedec,spi-nor";
+			reg = <0>;
+			spi-rx-bus-width = <4>;
+			spi-tx-bus-width = <4>;
+			spi-max-frequency = <50000000>;
+		};
+	};
+};
+
 &pcie_ep {
 	pinctrl-names = "default", "init";
 	pinctrl-0 = <&pcie_pins_a>;
diff --git a/arch/arm64/boot/dts/st/stm32mp25xc.dtsi b/arch/arm64/boot/dts/st/stm32mp25xc.dtsi
new file mode 100644
index 00000000000000..56872fb1deeb0f
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp25xc.dtsi
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-3-Clause)
+/*
+ * Copyright (C) 2026 Marek Vasut <marex@nabladev.com>
+ */
+
+/ {
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp25xx-dhcos-bb.dts b/arch/arm64/boot/dts/st/stm32mp25xx-dhcos-bb.dts
new file mode 100644
index 00000000000000..cf66e8e48c99a3
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp25xx-dhcos-bb.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-3-Clause)
+/*
+ * Copyright (C) 2025-2026 Marek Vasut <marex@nabladev.com>
+ */
+
+/dts-v1/;
+
+#include "stm32mp255.dtsi"
+#include "stm32mp25xc.dtsi"
+#include "stm32mp25xx-dhcos-som.dtsi"
+
+/ {
+	model = "DH electronics STM32MP25xx DHCOS Breakout Board";
+	compatible = "dh,stm32mp251a-dhcos-bb", "dh,stm32mp251a-dhcos-som", "st,stm32mp251";
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp25xx-dhcos-som.dtsi b/arch/arm64/boot/dts/st/stm32mp25xx-dhcos-som.dtsi
new file mode 100644
index 00000000000000..23c25ea086445a
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp25xx-dhcos-som.dtsi
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+/*
+ * Copyright (C) 2025-2026 Marek Vasut <marex@nabladev.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include "stm32mp25-pinctrl.dtsi"
+#include "stm32mp25xxak-pinctrl.dtsi"
+#include "stm32mp2xxx-dhcos-som.dtsi"
+
+/ {
+	model = "DH electronics STM32MP25xx DHCOS SoM";
+	compatible = "dh,stm32mp251a-dhcos-som", "st,stm32mp251";
+
+	aliases {
+		serial1 = &uart8;
+		serial2 = &uart9;
+	};
+};
+
+&rv3032 {
+	interrupts-extended = <&gpioi 10 IRQ_TYPE_EDGE_FALLING>;
+};
+
+/* Bluetooth */
+&uart8 {
+	pinctrl-names = "default", "sleep", "idle";
+	pinctrl-0 = <&uart8_pins_a>;
+	pinctrl-1 = <&uart8_sleep_pins_a>;
+	pinctrl-2 = <&uart8_idle_pins_a>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "infineon,cyw55572-bt";
+		brcm,requires-autobaud-mode;
+		max-speed = <3000000>;
+		shutdown-gpios = <&ioexp 2 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&uart9 {
+	pinctrl-names = "default", "sleep", "idle";
+	pinctrl-0 = <&uart9_pins_a>;
+	pinctrl-1 = <&uart9_sleep_pins_a>;
+	pinctrl-2 = <&uart9_idle_pins_a>;
+	uart-has-rtscts;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/st/stm32mp2xxx-dhcos-som.dtsi b/arch/arm64/boot/dts/st/stm32mp2xxx-dhcos-som.dtsi
new file mode 100644
index 00000000000000..80d829c55d9d87
--- /dev/null
+++ b/arch/arm64/boot/dts/st/stm32mp2xxx-dhcos-som.dtsi
@@ -0,0 +1,452 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+/*
+ * Copyright (C) 2025-2026 Marek Vasut <marex@nabladev.com>
+ */
+
+/ {
+	aliases {
+		mmc0 = &sdmmc2;
+		mmc1 = &sdmmc3;
+		serial0 = &usart6;
+		serial1 = &uart8;
+		eeprom0 = &eeprom0;
+		eeprom0wl = &eeprom0wl;
+		rtc0 = &rv3032;
+		rtc1 = &rtc;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x1 0x0>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		/* Internal RAM reserved memory declaration */
+		tfa_bl31: tfa-bl31@a000000 {
+			reg = <0x0 0xa000000 0x0 0x20000>;
+			no-map;
+		};
+
+		hpdma1_lli: hpdma1-lli@a020000 {
+			reg = <0x0 0xa020000 0x0 0xf0f0>;
+			no-map;
+		};
+
+		hpdma2_lli: hpdma2-lli@a02f0f0 {
+			reg = <0x0 0xa02f0f0 0x0 0xf0f0>;
+			no-map;
+		};
+
+		hpdma3_lli: hpdma3-lli@a03e1e0 {
+			reg = <0x0 0xa03e1e0 0x0 0x1e20>;
+			no-map;
+		};
+
+		bsec_mirror: bsec-mirror@a040000 {
+			reg = <0x0 0xa040000 0x0 0x1000>;
+			no-map;
+		};
+
+		scmi_cid2_s: scmi-cid2-s@a041000 {
+			reg = <0x0 0xa041000 0x0 0x1000>;
+			no-map;
+		};
+
+		scmi_cid2_ns: scmi-cid2-ns@a042000 {
+			reg = <0x0 0xa042000 0x0 0x1000>;
+			no-map;
+		};
+
+		cm33_sram1: cm33-sram1@a043000 {
+			reg = <0x0 0xa043000 0x0 0x1d000>;
+			no-map;
+		};
+
+		cm33_sram2: cm33-sram2@a060000 {
+			reg = <0x0 0xa060000 0x0 0x20000>;
+			no-map;
+		};
+
+		cm33_retram: cm33-retram@a080000 {
+			reg = <0x0 0xa080000 0x0 0x1f000>;
+			no-map;
+		};
+
+		ddr_param: ddr-param@a09f000 {
+			reg = <0x0 0xa09f000 0x0 0x1000>;
+			no-map;
+		};
+
+		cm0_cube_fw: cm0-cube-fw@200C0000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x200C0000 0x0 0x4000>;
+			no-map;
+		};
+
+		cm0_cube_data: cm0-cube-data@200C4000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x200C4000 0x0 0x2000>;
+			no-map;
+		};
+
+		ipc_shmem_2: ipc-shmem-2@200C6000{
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x200C6000 0x0 0x2000>;
+			no-map;
+		};
+
+		/* Backup RAM reserved memory declaration */
+		bl31_lowpower: bl31-lowpower@42000000 {
+			reg = <0x0 0x42000000 0x0 0x1000>;
+			no-map;
+		};
+
+		tfm_its: tfm-its@42001000 {
+			reg = <0x0 0x42001000 0x0 0x1000>;
+			no-map;
+		};
+
+		/* Octo Memory Manager reserved memory declaration */
+		mm_ospi1: mm-ospi@60000000 {
+			reg = <0x0 0x60000000 0x0 0x10000000>;
+			no-map;
+		};
+
+		/* DDR reserved memory declaration */
+		tfm_code: tfm-code@80000000 {
+			reg = <0x0 0x80000000 0x0 0x100000>;
+			no-map;
+		};
+
+		cm33_cube_fw: cm33-cube-fw@80100000 {
+			reg = <0x0 0x80100000 0x0 0x800000>;
+			no-map;
+		};
+
+		tfm_data: tfm-data@80900000 {
+			reg = <0x0 0x80900000 0x0 0x100000>;
+			no-map;
+		};
+
+		cm33_cube_data: cm33-cube-data@80a00000 {
+			reg = <0x0 0x80a00000 0x0 0x800000>;
+			no-map;
+		};
+
+		ipc_shmem_1: ipc-shmem-1@81200000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x81200000 0x0 0xf8000>;
+			no-map;
+		};
+
+		vdev0vring0: vdev0vring0@812f8000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x812f8000 0x0 0x1000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@812f9000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x812f9000 0x0 0x1000>;
+			no-map;
+		};
+
+		vdev0buffer: vdev0buffer@812fa000 {
+			compatible = "shared-dma-pool";
+			reg = <0x0 0x812fa000 0x0 0x6000>;
+			no-map;
+		};
+
+		spare1: spare1@81300000 {
+			reg = <0x0 0x81300000 0x0 0xcc0000>;
+			no-map;
+		};
+
+		bl31_context: bl31-context@81fc0000 {
+			reg = <0x0 0x81fc0000 0x0 0x40000>;
+			no-map;
+		};
+
+		op_tee: op-tee@82000000 {
+			reg = <0x0 0x82000000 0x0 0x2000000>;
+			no-map;
+		};
+
+		gpu_reserved: gpu-reserved@fa800000 {
+			reg = <0x0 0xfa800000 0x0 0x4000000>;
+			no-map;
+		};
+
+		ltdc_sec_layer: ltdc-sec-layer@fe800000 {
+			reg = <0x0 0xfe800000 0x0 0x800000>;
+			no-map;
+		};
+
+		ltdc_sec_rotation: ltdc-sec-rotation@ff000000 {
+			reg = <0x0 0xff000000 0x0 0x1000000>;
+			no-map;
+		};
+
+		/* global autoconfigured region for contiguous allocations */
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			alloc-ranges = <0 0x80000000 0 0x80000000>;
+			size = <0x0 0x8000000>;
+			alignment = <0x0 0x2000>;
+			linux,cma-default;
+		};
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		post-power-on-delay-ms = <50>;
+		reset-gpios = <&ioexp 3 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&arm_wdt {
+	timeout-sec = <32>;
+	status = "okay";
+};
+
+&gpioa {
+	gpio-line-names = "", "DHCOS-M", "DHCOS-K", "DHCOS-L",
+			  "DHCOS-E", "DHCOS-J", "", "",
+			  "DHCOS-D", "", "", "",
+			  "", "", "", "";
+};
+
+&gpiof {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "DHCOS_RAM-CODE_2";
+};
+
+&gpiog {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "DHCOS_RAM-CODE_0", "DHCOS_RAM-CODE_1", "",
+			  "", "", "DHCOS-F", "DHCOS_HW-CODE_0";
+};
+
+&gpioh {
+	gpio-line-names = "", "", "DHCOS-I", "DHCOS-N",
+			  "", "", "DHCOS-O", "DHCOS-H",
+			  "DHCOS-G", "", "", "",
+			  "", "", "", "";
+};
+
+&gpioi {
+	gpio-line-names = "DHCOS_HW-CODE_1", "DHCOS_HW-CODE_2", "", "",
+			  "", "", "", "",
+			  "", "DHCOS-C", "", "",
+			  "", "", "", "";
+};
+
+&gpioz {
+	gpio-line-names = "", "", "DHCOS-A", "DHCOS-B",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+};
+
+&i2c8 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c8_pins_b>;
+	pinctrl-1 = <&i2c8_sleep_pins_b>;
+	i2c-scl-rising-time-ns = <57>;
+	i2c-scl-falling-time-ns = <7>;
+	clock-frequency = <400000>;
+	status = "okay";
+
+	ioexp: gpio@20 {
+		compatible = "kinetic,kts1622", "ti,tcal6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupts-extended = <&gpiod 11 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		wakeup-source;
+
+		gpio-line-names =
+			"#ETH1_RST_P0_0", "#ETH0_RST_P0_1",
+			"BT_REG_ON_P0_2", "WL_REG_ON_P0_3",
+			"USB1_PWR_EN_P0_4", "USB1_PWR_STAT_P0_5",
+			"USB0_PWR_EN_P0_6", "USB0_PWR_STAT_P0_7",
+			"SDIO0_CD_P1_0", "USB0_SS_SEL_P1_1",
+			"#PCIe_RST_1_2", "#PCIe_Wake_P1_3",
+			"#QSPI_RST_P1_4", "#CSI0_PWDN_P1_5",
+			"#CSI0_RST_P1_6", "#ETH3_RST_P1_7";
+	};
+
+	eeprom0: eeprom@50 {
+		compatible = "atmel,24c256";	/* ST M24256 */
+		reg = <0x50>;
+		pagesize = <64>;
+	};
+
+	rv3032: rtc@51 {
+		compatible = "microcrystal,rv3032";
+		reg = <0x51>;
+		wakeup-source;
+	};
+
+	eeprom0wl: eeprom@58 {
+		compatible = "st,24256e-wl";	/* ST M24256E WL page of 0x50 */
+		pagesize = <64>;
+		reg = <0x58>;
+	};
+};
+
+&ommanager {
+	memory-region = <&mm_ospi1>;
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&ospi_port1_clk_pins_a
+		     &ospi_port1_io03_pins_a
+		     &ospi_port1_cs0_pins_a>;
+	pinctrl-1 = <&ospi_port1_clk_sleep_pins_a
+		     &ospi_port1_io03_sleep_pins_a
+		     &ospi_port1_cs0_sleep_pins_a>;
+	pinctrl-names = "default", "sleep";
+	status = "okay";
+};
+
+&ospi1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	memory-region = <&mm_ospi1>;
+	status = "okay";
+
+	flash0: flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+		spi-max-frequency = <100000000>;
+	};
+};
+
+&rtc {
+	status = "okay";
+};
+
+&scmi_regu {
+	scmi_vddcore: regulator@b {
+		reg = <VOLTD_SCMI_STPMIC2_BUCK2>;
+		regulator-name = "vddcore";
+	};
+
+	regulator@c {
+		reg = <VOLTD_SCMI_STPMIC2_BUCK3>;
+		regulator-name = "vddgpu";
+		regulator-always-on;
+	};
+
+	scmi_v1v8: regulator@e {
+		reg = <VOLTD_SCMI_STPMIC2_BUCK5>;
+		regulator-name = "v1v8";
+		regulator-always-on;
+	};
+
+	scmi_v3v3: regulator@10 {
+		reg = <VOLTD_SCMI_STPMIC2_BUCK7>;
+		regulator-name = "v3v3";
+		regulator-always-on;
+	};
+
+	scmi_vdd_emmc: regulator@12 {
+		reg = <VOLTD_SCMI_STPMIC2_LDO2>;
+		regulator-name = "vdd_emmc";
+	};
+
+	scmi_vdd3v3_usb: regulator@14 {
+		reg = <VOLTD_SCMI_STPMIC2_LDO4>;
+		regulator-name = "vdd3v3_usb";
+	};
+
+	scmi_vdd_sdcard: regulator@17 {
+		reg = <VOLTD_SCMI_STPMIC2_LDO7>;
+		regulator-name = "vdd_sdcard";
+	};
+};
+
+&scmi_vddio3 {
+	regulator-always-on;
+};
+
+&scmi_vddio4 {
+	regulator-always-on;
+};
+
+&{sdmmc2_b4_pins_a/pins2} {
+	/delete-property/ bias-disable;
+	bias-pull-up;
+};
+
+&{sdmmc2_b4_od_pins_a/pins2} {
+	/delete-property/ bias-disable;
+	bias-pull-up;
+};
+
+&sdmmc2 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc2_b4_pins_a &sdmmc2_d47_pins_a>;
+	pinctrl-1 = <&sdmmc2_b4_od_pins_a &sdmmc2_d47_pins_a>;
+	pinctrl-2 = <&sdmmc2_b4_sleep_pins_a &sdmmc2_d47_sleep_pins_a>;
+	non-removable;
+	no-sd;
+	no-sdio;
+	st,neg-edge;
+	bus-width = <8>;
+	vmmc-supply = <&scmi_vdd_emmc>;
+	vqmmc-supply = <&scmi_vddio2>;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	status = "okay";
+};
+
+&sdmmc3 {
+	pinctrl-names = "default", "opendrain", "sleep";
+	pinctrl-0 = <&sdmmc3_b4_pins_b>;
+	pinctrl-1 = <&sdmmc3_b4_od_pins_b>;
+	pinctrl-2 = <&sdmmc3_b4_sleep_pins_b>;
+	bus-width = <4>;
+	keep-power-in-suspend;
+	non-removable;
+	no-sd;
+	no-mmc;
+	st,neg-edge;
+	vmmc-supply = <&scmi_v3v3>;
+	vqmmc-supply = <&scmi_v1v8>;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	status = "okay";
+
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	brcmf: wifi@1 {	/* muRata 2FY */
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
+};
+
+&usart6 {
+	pinctrl-names = "default", "idle", "sleep";
+	pinctrl-0 = <&usart6_pins_b>;
+	pinctrl-1 = <&usart6_idle_pins_b>;
+	pinctrl-2 = <&usart6_sleep_pins_b>;
+	/delete-property/dmas;
+	/delete-property/dma-names;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/tesla/fsd-evb.dts b/arch/arm64/boot/dts/tesla/fsd-evb.dts
index 503abe52a81cf1..a1f3d3fbbf8bea 100644
--- a/arch/arm64/boot/dts/tesla/fsd-evb.dts
+++ b/arch/arm64/boot/dts/tesla/fsd-evb.dts
@@ -50,6 +50,11 @@
 			bitclock-master = <&tlv320aic3x>;
 			frame-master = <&tlv320aic3x>;
 
+			codec {
+				sound-dai = <&tlv320aic3x>;
+				system-clock-frequency = <33000000>;
+			};
+
 			cpu-0 {
 				sound-dai = <&i2s_0 0>;
 			};
@@ -57,11 +62,6 @@
 			cpu-1 {
 				sound-dai = <&i2s_0 1>;
 			};
-
-			codec {
-				sound-dai = <&tlv320aic3x>;
-				system-clock-frequency = <33000000>;
-			};
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/tesla/fsd-pinctrl.dtsi b/arch/arm64/boot/dts/tesla/fsd-pinctrl.dtsi
index 6f4658f57453ad..ab25433e9b3253 100644
--- a/arch/arm64/boot/dts/tesla/fsd-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/tesla/fsd-pinctrl.dtsi
@@ -27,7 +27,7 @@
 		#interrupt-cells = <2>;
 	};
 
-	gpf6: gpf6-gpio-bank {
+	gpf4: gpf4-gpio-bank {
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -35,7 +35,7 @@
 		#interrupt-cells = <2>;
 	};
 
-	gpf4: gpf4-gpio-bank {
+	gpf5: gpf5-gpio-bank {
 		gpio-controller;
 		#gpio-cells = <2>;
 
@@ -43,7 +43,7 @@
 		#interrupt-cells = <2>;
 	};
 
-	gpf5: gpf5-gpio-bank {
+	gpf6: gpf6-gpio-bank {
 		gpio-controller;
 		#gpio-cells = <2>;
 
diff --git a/arch/arm64/boot/dts/tesla/fsd.dtsi b/arch/arm64/boot/dts/tesla/fsd.dtsi
index 6766c78243a3f5..36925b2749b59f 100644
--- a/arch/arm64/boot/dts/tesla/fsd.dtsi
+++ b/arch/arm64/boot/dts/tesla/fsd.dtsi
@@ -34,6 +34,12 @@
 		spi2 = &spi_2;
 	};
 
+	fin_pll: clock {
+		compatible = "fixed-clock";
+		clock-output-names = "fin_pll";
+		#clock-cells = <0>;
+	};
+
 	cpus {
 		#address-cells = <2>;
 		#size-cells = <0>;
@@ -96,196 +102,196 @@
 
 		/* Cluster 0 */
 		cpucl0_0: cpu@0 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x000>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x000>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl0_1: cpu@1 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x001>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x001>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl0_2: cpu@2 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x002>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x002>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl0_3: cpu@3 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x003>;
-				enable-method = "psci";
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x003>;
+			enable-method = "psci";
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		/* Cluster 1 */
 		cpucl1_0: cpu@100 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x100>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl1_1: cpu@101 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x101>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x101>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl1_2: cpu@102 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x102>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x102>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl1_3: cpu@103 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x103>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x103>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		/* Cluster 2 */
 		cpucl2_0: cpu@200 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x200>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl2_1: cpu@201 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x201>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x201>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl2_2: cpu@202 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x202>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x202>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl2_3: cpu@203 {
-				device_type = "cpu";
-				compatible = "arm,cortex-a72";
-				reg = <0x0 0x203>;
-				enable-method = "psci";
-				clock-frequency = <2400000000>;
-				cpu-idle-states = <&cpu_sleep>;
-				i-cache-size = <0xc000>;
-				i-cache-line-size = <64>;
-				i-cache-sets = <256>;
-				d-cache-size = <0x8000>;
-				d-cache-line-size = <64>;
-				d-cache-sets = <256>;
-				next-level-cache = <&cpucl_l2>;
+			device_type = "cpu";
+			compatible = "arm,cortex-a72";
+			reg = <0x0 0x203>;
+			enable-method = "psci";
+			clock-frequency = <2400000000>;
+			cpu-idle-states = <&cpu_sleep>;
+			i-cache-size = <0xc000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <256>;
+			next-level-cache = <&cpucl_l2>;
 		};
 
 		cpucl_l2: l2-cache0 {
@@ -296,23 +302,23 @@
 			cache-line-size = <64>;
 			cache-sets = <4096>;
 		};
+	};
 
-		idle-states {
-			entry-method = "psci";
+	idle-states {
+		entry-method = "psci";
 
-			cpu_sleep: cpu-sleep {
-				idle-state-name = "c2";
-				compatible = "arm,idle-state";
-				local-timer-stop;
-				arm,psci-suspend-param = <0x0010000>;
-				entry-latency-us = <30>;
-				exit-latency-us = <75>;
-				min-residency-us = <300>;
-			};
+		cpu_sleep: cpu-sleep {
+			compatible = "arm,idle-state";
+			idle-state-name = "c2";
+			local-timer-stop;
+			entry-latency-us = <30>;
+			exit-latency-us = <75>;
+			min-residency-us = <300>;
+			arm,psci-suspend-param = <0x0010000>;
 		};
 	};
 
-	arm-pmu {
+	pmu {
 		compatible = "arm,cortex-a72-pmu";
 		interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>,
@@ -337,47 +343,107 @@
 		method = "smc";
 	};
 
-	timer {
-		compatible = "arm,armv8-timer";
-		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
-			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
-			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
-			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
-	};
-
-	fin_pll: clock {
-		compatible = "fixed-clock";
-		clock-output-names = "fin_pll";
-		#clock-cells = <0>;
-	};
-
 	reserved-memory {
+		ranges;
 		#address-cells = <2>;
 		#size-cells = <2>;
-		ranges;
 
 		mfc_left: region@84000000 {
 			compatible = "shared-dma-pool";
-			no-map;
 			reg = <0 0x84000000 0 0x8000000>;
+			no-map;
 		};
 	};
 
 	soc: soc@0 {
 		compatible = "simple-bus";
-		#address-cells = <2>;
-		#size-cells = <2>;
 		ranges = <0x0 0x0 0x0 0x0 0x0 0x18000000>;
 		dma-ranges = <0x0 0x0 0x0 0x0 0x10 0x0>;
+		#address-cells = <2>;
+		#size-cells = <2>;
 
-		gic: interrupt-controller@10400000 {
-			compatible = "arm,gic-v3";
-			#address-cells = <0>;
-			#interrupt-cells = <3>;
-			interrupt-controller;
-			reg = <0x0 0x10400000 0x0 0x10000>, /* GICD */
-			      <0x0 0x10600000 0x0 0x200000>; /* GICR_RD+GICR_SGI */
-			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+		clock_imem: clock-controller@10010000 {
+			compatible = "tesla,fsd-clock-imem";
+			reg = <0x0 0x10010000 0x0 0x3000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&clock_cmu DOUT_CMU_IMEM_TCUCLK>,
+				 <&clock_cmu DOUT_CMU_IMEM_ACLK>,
+				 <&clock_cmu DOUT_CMU_IMEM_DMACLK>;
+			clock-names = "fin_pll",
+				      "dout_cmu_imem_tcuclk",
+				      "dout_cmu_imem_aclk",
+				      "dout_cmu_imem_dmaclk";
+		};
+
+		timer@10040000 {
+			compatible = "tesla,fsd-mct", "samsung,exynos4210-mct";
+			reg = <0x0 0x10040000 0x0 0x800>;
+			interrupts = <GIC_SPI 455 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 468 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 469 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 470 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&fin_pll>, <&clock_imem IMEM_MCT_PCLK>;
+			clock-names = "fin_pll", "mct";
+		};
+
+		watchdog_0: watchdog@100a0000 {
+			compatible = "tesla,fsd-wdt", "samsung,exynos7-wdt";
+			reg = <0x0 0x100a0000 0x0 0x100>;
+			interrupts = <GIC_SPI 471 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&fin_pll>;
+			clock-names = "watchdog";
+			samsung,syscon-phandle = <&pmu_system_controller>;
+		};
+
+		watchdog_1: watchdog@100b0000 {
+			compatible = "tesla,fsd-wdt", "samsung,exynos7-wdt";
+			reg = <0x0 0x100b0000 0x0 0x100>;
+			interrupts = <GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&fin_pll>;
+			clock-names = "watchdog";
+			samsung,syscon-phandle = <&pmu_system_controller>;
+		};
+
+		watchdog_2: watchdog@100c0000 {
+			compatible = "tesla,fsd-wdt", "samsung,exynos7-wdt";
+			reg = <0x0 0x100c0000 0x0 0x100>;
+			interrupts = <GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&fin_pll>;
+			clock-names = "watchdog";
+			samsung,syscon-phandle = <&pmu_system_controller>;
+		};
+
+		mdma0: dma-controller@10100000 {
+			compatible = "arm,pl330", "arm,primecell";
+			reg = <0x0 0x10100000 0x0 0x1000>;
+			interrupts = <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			clocks = <&clock_imem IMEM_DMA0_IPCLKPORT_ACLK>;
+			clock-names = "apb_pclk";
+			iommus = <&smmu_imem 0x800 0x0>;
+		};
+
+		mdma1: dma-controller@10110000 {
+			compatible = "arm,pl330", "arm,primecell";
+			reg = <0x0 0x10110000 0x0 0x1000>;
+			interrupts = <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			clocks = <&clock_imem IMEM_DMA1_IPCLKPORT_ACLK>;
+			clock-names = "apb_pclk";
+			iommus = <&smmu_imem 0x801 0x0>;
 		};
 
 		smmu_imem: iommu@10200000 {
@@ -400,6 +466,34 @@
 				     <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>; /* for CONTEXT_3 */
 		};
 
+		gic: interrupt-controller@10400000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x10400000 0x0 0x10000>, /* GICD */
+			      <0x0 0x10600000 0x0 0x200000>; /* GICR_RD+GICR_SGI */
+			#address-cells = <0>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		pmu_system_controller: system-controller@11400000 {
+			compatible = "tesla,fsd-pmu", "samsung,exynos7-pmu", "syscon";
+			reg = <0x0 0x11400000 0x0 0x5000>;
+		};
+
+		pinctrl_pmu: pinctrl@114f0000 {
+			compatible = "tesla,fsd-pinctrl";
+			reg = <0x0 0x114f0000 0x0 0x1000>;
+		};
+
+		clock_cmu: clock-controller@11c10000 {
+			compatible = "tesla,fsd-clock-cmu";
+			reg = <0x0 0x11c10000 0x0 0x3000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>;
+			clock-names = "fin_pll";
+		};
+
 		smmu_isp: iommu@12100000 {
 			compatible = "arm,mmu-500";
 			reg = <0x0 0x12100000 0x0 0x10000>;
@@ -428,60 +522,6 @@
 				     <GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH>; /* for CONTEXT_7 */
 		};
 
-		smmu_peric: iommu@14900000 {
-			compatible = "arm,mmu-500";
-			reg = <0x0 0x14900000 0x0 0x10000>;
-			#iommu-cells = <2>;
-			#global-interrupts = <5>;
-			interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>, /* Global secure fault */
-				     <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>, /* Global non-secure fault */
-				     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>, /* Combined secure interrupt */
-				     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>, /* Combined non-secure interrupt */
-				     /* Performance counter interrupts */
-				     <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>, /* for PERIC */
-				     /* Per context non-secure context interrupts, 0-1 interrupts */
-				     <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>, /* for CONTEXT_0 */
-				     <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>; /* for CONTEXT_1 */
-		};
-
-		smmu_fsys0: iommu@15450000 {
-			compatible = "arm,mmu-500";
-			reg = <0x0 0x15450000 0x0 0x10000>;
-			#iommu-cells = <2>;
-			#global-interrupts = <5>;
-			interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>, /* Global secure fault */
-				     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>, /* Global non-secure fault */
-				     <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>, /* Combined secure interrupt */
-				     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>, /* Combined non-secure interrupt */
-				     /* Performance counter interrupts */
-				     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>, /* for FSYS0   */
-				     /* Per context non-secure context interrupts, 0-1 interrupts */
-				     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>, /* for CONTEXT_0 */
-				     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>; /* for CONTEXT_1 */
-		};
-
-		clock_imem: clock-controller@10010000 {
-			compatible = "tesla,fsd-clock-imem";
-			reg = <0x0 0x10010000 0x0 0x3000>;
-			#clock-cells = <1>;
-			clocks = <&fin_pll>,
-				 <&clock_cmu DOUT_CMU_IMEM_TCUCLK>,
-				 <&clock_cmu DOUT_CMU_IMEM_ACLK>,
-				 <&clock_cmu DOUT_CMU_IMEM_DMACLK>;
-			clock-names = "fin_pll",
-				      "dout_cmu_imem_tcuclk",
-				      "dout_cmu_imem_aclk",
-				      "dout_cmu_imem_dmaclk";
-		};
-
-		clock_cmu: clock-controller@11c10000 {
-			compatible = "tesla,fsd-clock-cmu";
-			reg = <0x0 0x11c10000 0x0 0x3000>;
-			#clock-cells = <1>;
-			clocks = <&fin_pll>;
-			clock-names = "fin_pll";
-		};
-
 		clock_csi: clock-controller@12610000 {
 			compatible = "tesla,fsd-clock-cam_csi";
 			reg = <0x0 0x12610000 0x0 0x3000>;
@@ -503,6 +543,15 @@
 			clock-names = "fin_pll";
 		};
 
+		mfc: mfc@12880000 {
+			compatible = "tesla,fsd-mfc";
+			reg = <0x0 0x12880000 0x0 0x10000>;
+			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clock_mfc MFC_MFC_IPCLKPORT_ACLK>;
+			clock-names = "mfc";
+			memory-region = <&mfc_left>;
+		};
+
 		clock_peric: clock-controller@14010000 {
 			compatible = "tesla,fsd-clock-peric";
 			reg = <0x0 0x14010000 0x0 0x3000>;
@@ -526,149 +575,121 @@
 			reg = <0x0 0x14030000 0x0 0x1000>;
 		};
 
-		clock_fsys0: clock-controller@15010000 {
-			compatible = "tesla,fsd-clock-fsys0";
-			reg = <0x0 0x15010000 0x0 0x3000>;
-			#clock-cells = <1>;
-			clocks = <&fin_pll>,
-				 <&clock_cmu DOUT_CMU_PLL_SHARED0_DIV6>,
-				 <&clock_cmu DOUT_CMU_FSYS0_SHARED1DIV4>,
-				 <&clock_cmu DOUT_CMU_FSYS0_SHARED0DIV4>;
-			clock-names = "fin_pll",
-				      "dout_cmu_pll_shared0_div6",
-				      "dout_cmu_fsys0_shared1div4",
-				      "dout_cmu_fsys0_shared0div4";
-		};
-
-		sysreg_fsys0: system-controller@15030000 {
-			compatible = "tesla,fsd-fsys0-sysreg", "syscon";
-			reg = <0x0 0x15030000 0x0 0x1000>;
-		};
+		m_can0: can@14088000 {
+			compatible = "bosch,m_can";
+			reg = <0x0 0x14088000 0x0 0x0200>,
+			      <0x0 0x14080000 0x0 0x8000>;
+			reg-names = "m_can", "message_ram";
+			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "int0", "int1";
+			pinctrl-0 = <&m_can0_bus>;
+			pinctrl-names = "default";
+			clocks = <&clock_peric PERIC_MCAN0_IPCLKPORT_PCLK>,
+				 <&clock_peric PERIC_MCAN0_IPCLKPORT_CCLK>;
+			clock-names = "hclk", "cclk";
+			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
 
-		clock_fsys1: clock-controller@16810000 {
-			compatible = "tesla,fsd-clock-fsys1";
-			reg = <0x0 0x16810000 0x0 0x3000>;
-			#clock-cells = <1>;
-			clocks = <&fin_pll>,
-				 <&clock_cmu DOUT_CMU_FSYS1_SHARED0DIV8>,
-				 <&clock_cmu DOUT_CMU_FSYS1_SHARED0DIV4>;
-			clock-names = "fin_pll",
-				      "dout_cmu_fsys1_shared0div8",
-				      "dout_cmu_fsys1_shared0div4";
+			status = "disabled";
 		};
 
-		sysreg_fsys1: system-controller@16830000 {
-			compatible = "tesla,fsd-fsys1-sysreg", "syscon";
-			reg = <0x0 0x16830000 0x0 0x1000>;
-		};
-
-		mdma0: dma-controller@10100000 {
-			compatible = "arm,pl330", "arm,primecell";
-			reg = <0x0 0x10100000 0x0 0x1000>;
-			interrupts = <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>;
-			#dma-cells = <1>;
-			clocks = <&clock_imem IMEM_DMA0_IPCLKPORT_ACLK>;
-			clock-names = "apb_pclk";
-			iommus = <&smmu_imem 0x800 0x0>;
-		};
-
-		mdma1: dma-controller@10110000 {
-			compatible = "arm,pl330", "arm,primecell";
-			reg = <0x0 0x10110000 0x0 0x1000>;
-			interrupts = <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>;
-			#dma-cells = <1>;
-			clocks = <&clock_imem IMEM_DMA1_IPCLKPORT_ACLK>;
-			clock-names = "apb_pclk";
-			iommus = <&smmu_imem 0x801 0x0>;
-		};
-
-		pdma0: dma-controller@14280000 {
-			compatible = "arm,pl330", "arm,primecell";
-			reg = <0x0 0x14280000 0x0 0x1000>;
-			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
-			#dma-cells = <1>;
-			clocks = <&clock_peric PERIC_DMA0_IPCLKPORT_ACLK>;
-			clock-names = "apb_pclk";
-			iommus = <&smmu_peric 0x2 0x0>;
-		};
+		m_can1: can@14098000 {
+			compatible = "bosch,m_can";
+			reg = <0x0 0x14098000 0x0 0x0200>,
+			      <0x0 0x14090000 0x0 0x8000>;
+			reg-names = "m_can", "message_ram";
+			interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "int0", "int1";
+			pinctrl-0 = <&m_can1_bus>;
+			pinctrl-names = "default";
+			clocks = <&clock_peric PERIC_MCAN1_IPCLKPORT_PCLK>,
+				 <&clock_peric PERIC_MCAN1_IPCLKPORT_CCLK>;
+			clock-names = "hclk", "cclk";
+			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
 
-		pdma1: dma-controller@14290000 {
-			compatible = "arm,pl330", "arm,primecell";
-			reg = <0x0 0x14290000 0x0 0x1000>;
-			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
-			#dma-cells = <1>;
-			clocks = <&clock_peric PERIC_DMA1_IPCLKPORT_ACLK>;
-			clock-names = "apb_pclk";
-			iommus = <&smmu_peric 0x1 0x0>;
+			status = "disabled";
 		};
 
-		serial_0: serial@14180000 {
-			compatible = "tesla,fsd-uart", "samsung,exynos4210-uart";
-			reg = <0x0 0x14180000 0x0 0x100>;
-			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
-			dmas = <&pdma1 1>, <&pdma1 0>;
-			dma-names = "rx", "tx";
-			clocks = <&clock_peric PERIC_PCLK_UART0>,
-				 <&clock_peric PERIC_SCLK_UART0>;
-			clock-names = "uart", "clk_uart_baud0";
-			samsung,uart-fifosize = <64>;
+		m_can2: can@140a8000 {
+			compatible = "bosch,m_can";
+			reg = <0x0 0x140a8000 0x0 0x0200>,
+			      <0x0 0x140a0000 0x0 0x8000>;
+			reg-names = "m_can", "message_ram";
+			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "int0", "int1";
+			pinctrl-0 = <&m_can2_bus>;
+			pinctrl-names = "default";
+			clocks = <&clock_peric PERIC_MCAN2_IPCLKPORT_PCLK>,
+				 <&clock_peric PERIC_MCAN2_IPCLKPORT_CCLK>;
+			clock-names = "hclk", "cclk";
+			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
 
 			status = "disabled";
 		};
 
-		serial_1: serial@14190000 {
-			compatible = "tesla,fsd-uart", "samsung,exynos4210-uart";
-			reg = <0x0 0x14190000 0x0 0x100>;
-			interrupts = <GIC_SPI 172 IRQ_TYPE_LEVEL_HIGH>;
-			dmas = <&pdma1 3>, <&pdma1 2>;
-			dma-names = "rx", "tx";
-			clocks = <&clock_peric PERIC_PCLK_UART1>,
-				 <&clock_peric PERIC_SCLK_UART1>;
-			clock-names = "uart", "clk_uart_baud0";
-			samsung,uart-fifosize = <64>;
+		m_can3: can@140b8000 {
+			compatible = "bosch,m_can";
+			reg = <0x0 0x140b8000 0x0 0x0200>,
+			      <0x0 0x140b0000 0x0 0x8000>;
+			reg-names = "m_can", "message_ram";
+			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "int0", "int1";
+			pinctrl-0 = <&m_can3_bus>;
+			pinctrl-names = "default";
+			clocks = <&clock_peric PERIC_MCAN3_IPCLKPORT_PCLK>,
+				 <&clock_peric PERIC_MCAN3_IPCLKPORT_CCLK>;
+			clock-names = "hclk", "cclk";
+			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
 
 			status = "disabled";
 		};
 
-		pmu_system_controller: system-controller@11400000 {
-			compatible = "tesla,fsd-pmu", "samsung,exynos7-pmu", "syscon";
-			reg = <0x0 0x11400000 0x0 0x5000>;
-		};
+		i2s_0: i2s@140e0000 {
+			compatible = "tesla,fsd-i2s";
+			reg = <0x0 0x140e0000 0x0 0x100>;
+			interrupts = <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma1 14>, <&pdma1 13>, <&pdma1 12>;
+			dma-names = "tx", "rx", "tx-sec";
+			#clock-cells = <1>;
+			clocks = <&clock_peric PERIC_PCLK_TDM0>,
+				 <&clock_peric PERIC_HCLK_TDM0>,
+				 <&clock_peric PERIC_HCLK_TDM0>;
+			clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
+			pinctrl-0 = <&i2s0_bus>;
+			pinctrl-names = "default";
+			#sound-dai-cells = <1>;
 
-		watchdog_0: watchdog@100a0000 {
-			compatible = "tesla,fsd-wdt", "samsung,exynos7-wdt";
-			reg = <0x0 0x100a0000 0x0 0x100>;
-			interrupts = <GIC_SPI 471 IRQ_TYPE_LEVEL_HIGH>;
-			samsung,syscon-phandle = <&pmu_system_controller>;
-			clocks = <&fin_pll>;
-			clock-names = "watchdog";
+			status = "disabled";
 		};
 
-		watchdog_1: watchdog@100b0000 {
-			compatible = "tesla,fsd-wdt", "samsung,exynos7-wdt";
-			reg = <0x0 0x100b0000 0x0 0x100>;
-			interrupts = <GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH>;
-			samsung,syscon-phandle = <&pmu_system_controller>;
-			clocks = <&fin_pll>;
-			clock-names = "watchdog";
-		};
+		i2s_1: i2s@140f0000 {
+			compatible = "tesla,fsd-i2s";
+			reg = <0x0 0x140f0000 0x0 0x100>;
+			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma1 17>, <&pdma1 16>, <&pdma1 15>;
+			dma-names = "tx", "rx", "tx-sec";
+			#clock-cells = <1>;
+			clocks = <&clock_peric PERIC_PCLK_TDM1>,
+				 <&clock_peric PERIC_HCLK_TDM1>,
+				 <&clock_peric PERIC_HCLK_TDM1>;
+			clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
+			pinctrl-0 = <&i2s1_bus>;
+			pinctrl-names = "default";
+			#sound-dai-cells = <1>;
 
-		watchdog_2: watchdog@100c0000 {
-			compatible = "tesla,fsd-wdt", "samsung,exynos7-wdt";
-			reg = <0x0 0x100c0000 0x0 0x100>;
-			interrupts = <GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH>;
-			samsung,syscon-phandle = <&pmu_system_controller>;
-			clocks = <&fin_pll>;
-			clock-names = "watchdog";
+			status = "disabled";
 		};
 
 		pwm_0: pwm@14100000 {
 			compatible = "tesla,fsd-pwm", "samsung,exynos4210-pwm";
 			reg = <0x0 0x14100000 0x0 0x100>;
-			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
 			#pwm-cells = <3>;
 			clocks = <&clock_peric PERIC_PWM0_IPCLKPORT_I_PCLK_S0>;
 			clock-names = "timers";
+			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
 
 			status = "disabled";
 		};
@@ -676,338 +697,235 @@
 		pwm_1: pwm@14110000 {
 			compatible = "tesla,fsd-pwm", "samsung,exynos4210-pwm";
 			reg = <0x0 0x14110000 0x0 0x100>;
-			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
 			#pwm-cells = <3>;
 			clocks = <&clock_peric PERIC_PWM1_IPCLKPORT_I_PCLK_S0>;
 			clock-names = "timers";
+			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
 
 			status = "disabled";
 		};
 
-		hsi2c_0: i2c@14200000 {
-			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14200000 0x0 0x1000>;
-			interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
+		spi_0: spi@14140000 {
+			compatible = "tesla,fsd-spi";
+			reg = <0x0 0x14140000 0x0 0x100>;
+			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma1 4>, <&pdma1 5>;
+			dma-names = "tx", "rx";
 			#address-cells = <1>;
 			#size-cells = <0>;
+			clocks = <&clock_peric PERIC_PCLK_SPI0>,
+				 <&clock_peric PERIC_SCLK_SPI0>;
+			clock-names = "spi", "spi_busclk0";
+			pinctrl-0 = <&spi0_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c0_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C0>;
-			clock-names = "hsi2c";
+			num-cs = <1>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
 
-		hsi2c_1: i2c@14210000 {
-			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14210000 0x0 0x1000>;
-			interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
+		spi_1: spi@14150000 {
+			compatible = "tesla,fsd-spi";
+			reg = <0x0 0x14150000 0x0 0x100>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma1 6>, <&pdma1 7>;
+			dma-names = "tx", "rx";
 			#address-cells = <1>;
 			#size-cells = <0>;
+			clocks = <&clock_peric PERIC_PCLK_SPI1>,
+				 <&clock_peric PERIC_SCLK_SPI1>;
+			clock-names = "spi", "spi_busclk0";
+			pinctrl-0 = <&spi1_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c1_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C1>;
-			clock-names = "hsi2c";
+			num-cs = <1>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
 
-		hsi2c_2: i2c@14220000 {
-			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14220000 0x0 0x1000>;
-			interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
+		spi_2: spi@14160000 {
+			compatible = "tesla,fsd-spi";
+			reg = <0x0 0x14160000 0x0 0x100>;
+			interrupts = <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma1 8>, <&pdma1 9>;
+			dma-names = "tx", "rx";
 			#address-cells = <1>;
 			#size-cells = <0>;
+			clocks = <&clock_peric PERIC_PCLK_SPI2>,
+				 <&clock_peric PERIC_SCLK_SPI2>;
+			clock-names = "spi", "spi_busclk0";
+			pinctrl-0 = <&spi2_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c2_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C2>;
-			clock-names = "hsi2c";
+			num-cs = <1>;
+			samsung,spi-src-clk = <0>;
 
 			status = "disabled";
 		};
 
-		hsi2c_3: i2c@14230000 {
-			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14230000 0x0 0x1000>;
-			interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c3_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C3>;
-			clock-names = "hsi2c";
+		serial_0: serial@14180000 {
+			compatible = "tesla,fsd-uart", "samsung,exynos4210-uart";
+			reg = <0x0 0x14180000 0x0 0x100>;
+			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma1 1>, <&pdma1 0>;
+			dma-names = "rx", "tx";
+			clocks = <&clock_peric PERIC_PCLK_UART0>,
+				 <&clock_peric PERIC_SCLK_UART0>;
+			clock-names = "uart", "clk_uart_baud0";
+			samsung,uart-fifosize = <64>;
 
 			status = "disabled";
 		};
 
-		hsi2c_4: i2c@14240000 {
-			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14240000 0x0 0x1000>;
-			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c4_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C4>;
-			clock-names = "hsi2c";
+		serial_1: serial@14190000 {
+			compatible = "tesla,fsd-uart", "samsung,exynos4210-uart";
+			reg = <0x0 0x14190000 0x0 0x100>;
+			interrupts = <GIC_SPI 172 IRQ_TYPE_LEVEL_HIGH>;
+			dmas = <&pdma1 3>, <&pdma1 2>;
+			dma-names = "rx", "tx";
+			clocks = <&clock_peric PERIC_PCLK_UART1>,
+				 <&clock_peric PERIC_SCLK_UART1>;
+			clock-names = "uart", "clk_uart_baud0";
+			samsung,uart-fifosize = <64>;
 
 			status = "disabled";
 		};
 
-		hsi2c_5: i2c@14250000 {
-			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14250000 0x0 0x1000>;
-			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
-			#address-cells = <1>;
-			#size-cells = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c5_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C5>;
-			clock-names = "hsi2c";
-
-			status = "disabled";
+		pinctrl_peric: pinctrl@141f0000 {
+			compatible = "tesla,fsd-pinctrl";
+			reg = <0x0 0x141f0000 0x0 0x1000>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		hsi2c_6: i2c@14260000 {
+		hsi2c_0: i2c@14200000 {
 			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14260000 0x0 0x1000>;
-			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+			reg = <0x0 0x14200000 0x0 0x1000>;
+			interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			pinctrl-0 = <&hs_i2c0_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c6_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C6>;
+			clocks = <&clock_peric PERIC_PCLK_HSI2C0>;
 			clock-names = "hsi2c";
 
 			status = "disabled";
 		};
 
-		hsi2c_7: i2c@14270000 {
+		hsi2c_1: i2c@14210000 {
 			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
-			reg = <0x0 0x14270000 0x0 0x1000>;
-			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+			reg = <0x0 0x14210000 0x0 0x1000>;
+			interrupts = <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
+			pinctrl-0 = <&hs_i2c1_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&hs_i2c7_bus>;
-			clocks = <&clock_peric PERIC_PCLK_HSI2C7>;
-			clock-names = "hsi2c";
-
-			status = "disabled";
-		};
-
-		i2s_0: i2s@140e0000 {
-			compatible = "tesla,fsd-i2s";
-			reg = <0x0 0x140e0000 0x0 0x100>;
-			interrupts = <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>;
-			dmas = <&pdma1 14>, <&pdma1 13>, <&pdma1 12>;
-			dma-names = "tx", "rx", "tx-sec";
-			#clock-cells = <1>;
-			clocks = <&clock_peric PERIC_PCLK_TDM0>,
-				 <&clock_peric PERIC_HCLK_TDM0>,
-				 <&clock_peric PERIC_HCLK_TDM0>;
-			clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2s0_bus>;
-			#sound-dai-cells = <1>;
-
-			status = "disabled";
-		};
-
-		i2s_1: i2s@140f0000 {
-			compatible = "tesla,fsd-i2s";
-			reg = <0x0 0x140f0000 0x0 0x100>;
-			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>;
-			dmas = <&pdma1 17>, <&pdma1 16>, <&pdma1 15>;
-			dma-names = "tx", "rx", "tx-sec";
-			#clock-cells = <1>;
-			clocks = <&clock_peric PERIC_PCLK_TDM1>,
-				 <&clock_peric PERIC_HCLK_TDM1>,
-				 <&clock_peric PERIC_HCLK_TDM1>;
-			clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
-			pinctrl-names = "default";
-			pinctrl-0 = <&i2s1_bus>;
-			#sound-dai-cells = <1>;
-
-			status = "disabled";
-		};
-
-		pinctrl_pmu: pinctrl@114f0000 {
-			compatible = "tesla,fsd-pinctrl";
-			reg = <0x0 0x114f0000 0x0 0x1000>;
-		};
-
-		pinctrl_peric: pinctrl@141f0000 {
-			compatible = "tesla,fsd-pinctrl";
-			reg = <0x0 0x141f0000 0x0 0x1000>;
-			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		pinctrl_fsys0: pinctrl@15020000 {
-			compatible = "tesla,fsd-pinctrl";
-			reg = <0x0 0x15020000 0x0 0x1000>;
-			interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
-		m_can0: can@14088000 {
-			compatible = "bosch,m_can";
-			reg = <0x0 0x14088000 0x0 0x0200>,
-			      <0x0 0x14080000 0x0 0x8000>;
-			reg-names = "m_can", "message_ram";
-			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "int0", "int1";
-			pinctrl-names = "default";
-			pinctrl-0 = <&m_can0_bus>;
-			clocks = <&clock_peric PERIC_MCAN0_IPCLKPORT_PCLK>,
-				 <&clock_peric PERIC_MCAN0_IPCLKPORT_CCLK>;
-			clock-names = "hclk", "cclk";
-			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
-
-			status = "disabled";
-		};
-
-		m_can1: can@14098000 {
-			compatible = "bosch,m_can";
-			reg = <0x0 0x14098000 0x0 0x0200>,
-			      <0x0 0x14090000 0x0 0x8000>;
-			reg-names = "m_can", "message_ram";
-			interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "int0", "int1";
-			pinctrl-names = "default";
-			pinctrl-0 = <&m_can1_bus>;
-			clocks = <&clock_peric PERIC_MCAN1_IPCLKPORT_PCLK>,
-				 <&clock_peric PERIC_MCAN1_IPCLKPORT_CCLK>;
-			clock-names = "hclk", "cclk";
-			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
+			clocks = <&clock_peric PERIC_PCLK_HSI2C1>;
+			clock-names = "hsi2c";
 
 			status = "disabled";
 		};
 
-		m_can2: can@140a8000 {
-			compatible = "bosch,m_can";
-			reg = <0x0 0x140a8000 0x0 0x0200>,
-			      <0x0 0x140a0000 0x0 0x8000>;
-			reg-names = "m_can", "message_ram";
-			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "int0", "int1";
+		hsi2c_2: i2c@14220000 {
+			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
+			reg = <0x0 0x14220000 0x0 0x1000>;
+			interrupts = <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-0 = <&hs_i2c2_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&m_can2_bus>;
-			clocks = <&clock_peric PERIC_MCAN2_IPCLKPORT_PCLK>,
-				 <&clock_peric PERIC_MCAN2_IPCLKPORT_CCLK>;
-			clock-names = "hclk", "cclk";
-			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
+			clocks = <&clock_peric PERIC_PCLK_HSI2C2>;
+			clock-names = "hsi2c";
 
 			status = "disabled";
 		};
 
-		m_can3: can@140b8000 {
-			compatible = "bosch,m_can";
-			reg = <0x0 0x140b8000 0x0 0x0200>,
-			      <0x0 0x140b0000 0x0 0x8000>;
-			reg-names = "m_can", "message_ram";
-			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "int0", "int1";
+		hsi2c_3: i2c@14230000 {
+			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
+			reg = <0x0 0x14230000 0x0 0x1000>;
+			interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-0 = <&hs_i2c3_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&m_can3_bus>;
-			clocks = <&clock_peric PERIC_MCAN3_IPCLKPORT_PCLK>,
-				 <&clock_peric PERIC_MCAN3_IPCLKPORT_CCLK>;
-			clock-names = "hclk", "cclk";
-			bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
+			clocks = <&clock_peric PERIC_PCLK_HSI2C3>;
+			clock-names = "hsi2c";
 
 			status = "disabled";
 		};
 
-		spi_0: spi@14140000 {
-			compatible = "tesla,fsd-spi";
-			reg = <0x0 0x14140000 0x0 0x100>;
-			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
-			dmas = <&pdma1 4>, <&pdma1 5>;
-			dma-names = "tx", "rx";
+		hsi2c_4: i2c@14240000 {
+			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
+			reg = <0x0 0x14240000 0x0 0x1000>;
+			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			clocks = <&clock_peric PERIC_PCLK_SPI0>,
-				 <&clock_peric PERIC_SCLK_SPI0>;
-			clock-names = "spi", "spi_busclk0";
-			samsung,spi-src-clk = <0>;
+			pinctrl-0 = <&hs_i2c4_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&spi0_bus>;
-			num-cs = <1>;
+			clocks = <&clock_peric PERIC_PCLK_HSI2C4>;
+			clock-names = "hsi2c";
 
 			status = "disabled";
 		};
 
-		spi_1: spi@14150000 {
-			compatible = "tesla,fsd-spi";
-			reg = <0x0 0x14150000 0x0 0x100>;
-			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
-			dmas = <&pdma1 6>, <&pdma1 7>;
-			dma-names = "tx", "rx";
+		hsi2c_5: i2c@14250000 {
+			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
+			reg = <0x0 0x14250000 0x0 0x1000>;
+			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			clocks = <&clock_peric PERIC_PCLK_SPI1>,
-				 <&clock_peric PERIC_SCLK_SPI1>;
-			clock-names = "spi", "spi_busclk0";
-			samsung,spi-src-clk = <0>;
+			pinctrl-0 = <&hs_i2c5_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&spi1_bus>;
-			num-cs = <1>;
+			clocks = <&clock_peric PERIC_PCLK_HSI2C5>;
+			clock-names = "hsi2c";
 
 			status = "disabled";
 		};
 
-		spi_2: spi@14160000 {
-			compatible = "tesla,fsd-spi";
-			reg = <0x0 0x14160000 0x0 0x100>;
-			interrupts = <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>;
-			dmas = <&pdma1 8>, <&pdma1 9>;
-			dma-names = "tx", "rx";
+		hsi2c_6: i2c@14260000 {
+			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
+			reg = <0x0 0x14260000 0x0 0x1000>;
+			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			clocks = <&clock_peric PERIC_PCLK_SPI2>,
-				 <&clock_peric PERIC_SCLK_SPI2>;
-			clock-names = "spi", "spi_busclk0";
-			samsung,spi-src-clk = <0>;
+			pinctrl-0 = <&hs_i2c6_bus>;
 			pinctrl-names = "default";
-			pinctrl-0 = <&spi2_bus>;
-			num-cs = <1>;
+			clocks = <&clock_peric PERIC_PCLK_HSI2C6>;
+			clock-names = "hsi2c";
 
 			status = "disabled";
 		};
 
-		timer@10040000 {
-			compatible = "tesla,fsd-mct", "samsung,exynos4210-mct";
-			reg = <0x0 0x10040000 0x0 0x800>;
-			interrupts = <GIC_SPI 455 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 468 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 469 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 470 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&fin_pll>, <&clock_imem IMEM_MCT_PCLK>;
-			clock-names = "fin_pll", "mct";
+		hsi2c_7: i2c@14270000 {
+			compatible = "tesla,fsd-hsi2c", "samsung,exynos7-hsi2c";
+			reg = <0x0 0x14270000 0x0 0x1000>;
+			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			pinctrl-0 = <&hs_i2c7_bus>;
+			pinctrl-names = "default";
+			clocks = <&clock_peric PERIC_PCLK_HSI2C7>;
+			clock-names = "hsi2c";
+
+			status = "disabled";
 		};
 
-		mfc: mfc@12880000 {
-			compatible = "tesla,fsd-mfc";
-			reg = <0x0 0x12880000 0x0 0x10000>;
-			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
-			clock-names = "mfc";
-			clocks = <&clock_mfc MFC_MFC_IPCLKPORT_ACLK>;
-			memory-region = <&mfc_left>;
+		pdma0: dma-controller@14280000 {
+			compatible = "arm,pl330", "arm,primecell";
+			reg = <0x0 0x14280000 0x0 0x1000>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			clocks = <&clock_peric PERIC_DMA0_IPCLKPORT_ACLK>;
+			clock-names = "apb_pclk";
+			iommus = <&smmu_peric 0x2 0x0>;
+		};
+
+		pdma1: dma-controller@14290000 {
+			compatible = "arm,pl330", "arm,primecell";
+			reg = <0x0 0x14290000 0x0 0x1000>;
+			interrupts = <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			clocks = <&clock_peric PERIC_DMA1_IPCLKPORT_ACLK>;
+			clock-names = "apb_pclk";
+			iommus = <&smmu_peric 0x1 0x0>;
 		};
 
 		ethernet1: ethernet@14300000 {
@@ -1031,16 +949,57 @@
 			assigned-clocks = <&clock_peric PERIC_EQOS_PHYRXCLK_MUX>,
 					  <&clock_peric PERIC_EQOS_PHYRXCLK>;
 			assigned-clock-parents = <&clock_peric PERIC_EQOS_PHYRXCLK>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&eth1_tx_clk>, <&eth1_tx_data>, <&eth1_tx_ctrl>,
 				    <&eth1_phy_intr>, <&eth1_rx_clk>, <&eth1_rx_data>,
 				    <&eth1_rx_ctrl>, <&eth1_mdio>;
+			pinctrl-names = "default";
 			local-mac-address = [00 00 00 00 00 00];
 			iommus = <&smmu_peric 0x0 0x1>;
 
 			status = "disabled";
 		};
 
+		smmu_peric: iommu@14900000 {
+			compatible = "arm,mmu-500";
+			reg = <0x0 0x14900000 0x0 0x10000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <5>;
+			interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>, /* Global secure fault */
+				     <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>, /* Global non-secure fault */
+				     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>, /* Combined secure interrupt */
+				     <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>, /* Combined non-secure interrupt */
+				     /* Performance counter interrupts */
+				     <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>, /* for PERIC */
+				     /* Per context non-secure context interrupts, 0-1 interrupts */
+				     <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>, /* for CONTEXT_0 */
+				     <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>; /* for CONTEXT_1 */
+		};
+
+		clock_fsys0: clock-controller@15010000 {
+			compatible = "tesla,fsd-clock-fsys0";
+			reg = <0x0 0x15010000 0x0 0x3000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&clock_cmu DOUT_CMU_PLL_SHARED0_DIV6>,
+				 <&clock_cmu DOUT_CMU_FSYS0_SHARED1DIV4>,
+				 <&clock_cmu DOUT_CMU_FSYS0_SHARED0DIV4>;
+			clock-names = "fin_pll",
+				      "dout_cmu_pll_shared0_div6",
+				      "dout_cmu_fsys0_shared1div4",
+				      "dout_cmu_fsys0_shared0div4";
+		};
+
+		pinctrl_fsys0: pinctrl@15020000 {
+			compatible = "tesla,fsd-pinctrl";
+			reg = <0x0 0x15020000 0x0 0x1000>;
+			interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		sysreg_fsys0: system-controller@15030000 {
+			compatible = "tesla,fsd-fsys0-sysreg", "syscon";
+			reg = <0x0 0x15030000 0x0 0x1000>;
+		};
+
 		ufs: ufs@15120000 {
 			compatible = "tesla,fsd-ufs";
 			reg = <0x0 0x15120000 0x0 0x200>,  /* 0: HCI standard */
@@ -1053,8 +1012,8 @@
 				 <&clock_fsys0 UFS0_TOP0_CLK_UNIPRO>;
 			clock-names = "core_clk", "sclk_unipro_main";
 			freq-table-hz = <0 0>, <0 0>;
-			pinctrl-names = "default";
 			pinctrl-0 = <&ufs_rst_n &ufs_refclk_out>;
+			pinctrl-names = "default";
 			phys = <&ufs_phy>;
 			phy-names = "ufs-phy";
 
@@ -1065,10 +1024,10 @@
 			compatible = "tesla,fsd-ufs-phy";
 			reg = <0x0 0x15124000 0x0 0x800>;
 			reg-names = "phy-pma";
-			samsung,pmu-syscon = <&pmu_system_controller>;
 			#phy-cells = <0>;
 			clocks = <&clock_fsys0 UFS0_MPHY_REFCLK_IXTAL26>;
 			clock-names = "ref_clk";
+			samsung,pmu-syscon = <&pmu_system_controller>;
 		};
 
 		ethernet0: ethernet@15300000 {
@@ -1082,15 +1041,56 @@
 				 <&clock_fsys0 FSYS0_EQOS_TOP0_IPCLKPORT_RGMII_CLK_I>,
 				 <&clock_fsys0 FSYS0_EQOS_TOP0_IPCLKPORT_CLK_RX_I>;
 			clock-names = "ptp_ref", "master_bus", "slave_bus", "tx", "rx";
-			pinctrl-names = "default";
 			pinctrl-0 = <&eth0_tx_clk>, <&eth0_tx_data>, <&eth0_tx_ctrl>,
 				    <&eth0_phy_intr>, <&eth0_rx_clk>, <&eth0_rx_data>,
 				    <&eth0_rx_ctrl>, <&eth0_mdio>;
+			pinctrl-names = "default";
 			local-mac-address = [00 00 00 00 00 00];
 			iommus = <&smmu_fsys0 0x0 0x1>;
 
 			status = "disabled";
 		};
+
+		smmu_fsys0: iommu@15450000 {
+			compatible = "arm,mmu-500";
+			reg = <0x0 0x15450000 0x0 0x10000>;
+			#iommu-cells = <2>;
+			#global-interrupts = <5>;
+			interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>, /* Global secure fault */
+				     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>, /* Global non-secure fault */
+				     <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>, /* Combined secure interrupt */
+				     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>, /* Combined non-secure interrupt */
+				     /* Performance counter interrupts */
+				     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>, /* for FSYS0   */
+				     /* Per context non-secure context interrupts, 0-1 interrupts */
+				     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>, /* for CONTEXT_0 */
+				     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>; /* for CONTEXT_1 */
+		};
+
+		clock_fsys1: clock-controller@16810000 {
+			compatible = "tesla,fsd-clock-fsys1";
+			reg = <0x0 0x16810000 0x0 0x3000>;
+			#clock-cells = <1>;
+			clocks = <&fin_pll>,
+				 <&clock_cmu DOUT_CMU_FSYS1_SHARED0DIV8>,
+				 <&clock_cmu DOUT_CMU_FSYS1_SHARED0DIV4>;
+			clock-names = "fin_pll",
+				      "dout_cmu_fsys1_shared0div8",
+				      "dout_cmu_fsys1_shared0div4";
+		};
+
+		sysreg_fsys1: system-controller@16830000 {
+			compatible = "tesla,fsd-fsys1-sysreg", "syscon";
+			reg = <0x0 0x16830000 0x0 0x1000>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/ti/Makefile b/arch/arm64/boot/dts/ti/Makefile
index 795978e962ee8b..da654153a35bad 100644
--- a/arch/arm64/boot/dts/ti/Makefile
+++ b/arch/arm64/boot/dts/ti/Makefile
@@ -91,6 +91,7 @@ dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-nonwifi-dev.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-nonwifi-ivy.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-nonwifi-mallow.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-nonwifi-yavia.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-nonwifi-zinnia.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-ov5640-24mhz.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-ov5640.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-wifi-dahlia.dtb
@@ -98,6 +99,7 @@ dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-wifi-dev.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-wifi-ivy.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-wifi-mallow.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-wifi-yavia.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-am62p5-verdin-wifi-zinnia.dtb
 
 # Common overlays for SK-AM62* family of boards
 dtb-$(CONFIG_ARCH_K3) += k3-am62x-sk-csi2-ov5640.dtbo
@@ -196,10 +198,12 @@ dtb-$(CONFIG_ARCH_K3) += k3-j721s2-common-proc-board.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-j721s2-evm-gesi-exp-board.dtbo
 k3-j721s2-evm-dtbs := k3-j721s2-common-proc-board.dtb k3-j721s2-evm-gesi-exp-board.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-j721s2-evm.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-j721s2-evm-audio.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-j721s2-evm-pcie1-ep.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-j721s2-evm-usb0-type-a.dtbo
 
 # Boards with J722s SoC
+dtb-$(CONFIG_ARCH_K3) += k3-am6754-phyboard-rigel.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am67a-beagley-ai.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am67a-beagley-ai-csi0-imx219.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am67a-beagley-ai-csi1-imx219.dtbo
@@ -225,6 +229,7 @@ dtb-$(CONFIG_ARCH_K3) += k3-am69-aquila-dev.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am69-aquila-dev-dsi-to-hdmi.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am69-aquila-dsi-to-hdmi.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am69-aquila-dsi-to-lvds-panel-cap-touch-10inch.dtbo
+dtb-$(CONFIG_ARCH_K3) += k3-am69-aquila-dsi-to-lvds-v2-panel-cap-touch-10inch.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am69-aquila-panel-cap-touch-10inch-dsi.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am69-aquila-panel-cap-touch-7inch-dsi.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am69-sk.dtb
@@ -389,6 +394,9 @@ k3-am69-aquila-dev-csi-ov5640-dtbs := \
 k3-am69-aquila-dev-dsi-to-lvds-panel-cap-touch-10inch-dtbs := \
 	k3-am69-aquila-dev.dtb \
 	k3-am69-aquila-dsi-to-lvds-panel-cap-touch-10inch.dtbo
+k3-am69-aquila-dev-dsi-to-lvds-v2-panel-cap-touch-10inch-dtbs := \
+	k3-am69-aquila-dev.dtb \
+	k3-am69-aquila-dsi-to-lvds-v2-panel-cap-touch-10inch.dtbo
 k3-am69-sk-csi2-dual-imx219-dtbs := k3-am69-sk.dtb \
 	k3-j721e-sk-csi2-dual-imx219.dtbo
 k3-am69-sk-pcie0-ep-dtbs := k3-am69-sk.dtb \
@@ -403,6 +411,8 @@ k3-j721e-evm-pcie1-ep-dtbs := k3-j721e-common-proc-board.dtb \
 	k3-j721e-evm-pcie1-ep.dtbo
 k3-j721e-sk-csi2-dual-imx219-dtbs := k3-j721e-sk.dtb \
 	k3-j721e-sk-csi2-dual-imx219.dtbo
+k3-j721s2-evm-audio-dtbs := k3-j721s2-common-proc-board.dtb \
+	k3-j721s2-evm-audio.dtbo
 k3-j721s2-evm-pcie1-ep-dtbs := k3-j721s2-common-proc-board.dtb \
 	k3-j721s2-evm-pcie1-ep.dtbo
 k3-j721s2-evm-usb0-type-a-dtbs := k3-j721s2-common-proc-board.dtb \
@@ -491,6 +501,7 @@ dtb- += k3-am625-beagleplay-csi2-ov5640.dtb \
 	k3-am69-aquila-dev-csi-ov5640-24mhz.dtb \
 	k3-am69-aquila-dev-csi-ov5640.dtb \
 	k3-am69-aquila-dev-dsi-to-lvds-panel-cap-touch-10inch.dtb \
+	k3-am69-aquila-dev-dsi-to-lvds-v2-panel-cap-touch-10inch.dtb \
 	k3-am69-sk-csi2-dual-imx219.dtb \
 	k3-am69-sk-pcie0-ep.dtb \
 	k3-j7200-evm-pcie1-ep.dtb \
@@ -498,6 +509,7 @@ dtb- += k3-am625-beagleplay-csi2-ov5640.dtb \
 	k3-j721e-evm-pcie0-ep.dtb \
 	k3-j721e-evm-pcie1-ep.dtb \
 	k3-j721e-sk-csi2-dual-imx219.dtb \
+	k3-j721s2-evm-audio.dtb \
 	k3-j721s2-evm-pcie1-ep.dtb \
 	k3-j721s2-evm-usb0-type-a.dtb \
 	k3-j722s-evm-csi2-quad-rpi-cam-imx219.dtb \
diff --git a/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts b/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts
index dfd65c00f5b95b..7fd508f4eddf1c 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts
@@ -86,7 +86,7 @@
 		bootph-all;
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x220, PIN_INPUT, 0) /* (V3) MMC0_CMD */
-			AM62X_IOPAD(0x218, PIN_INPUT, 0) /* (Y1) MMC0_CLK */
+			AM62X_IOPAD(0x218, PIN_OUTPUT, 0) /* (Y1) MMC0_CLK */
 			AM62X_IOPAD(0x214, PIN_INPUT, 0) /* (V2) MMC0_DAT0 */
 			AM62X_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (V1) MMC0_DAT1 */
 			AM62X_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (W2) MMC0_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi b/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
index 3baa653257bb6b..ce3e5165bd3107 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62-phycore-som.dtsi
@@ -153,7 +153,7 @@
 	main_mmc0_pins_default: main-mmc0-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x220, PIN_INPUT_PULLUP, 0) /* (Y3) MMC0_CMD */
-			AM62X_IOPAD(0x218, PIN_INPUT_PULLDOWN, 0) /* (AB1) MMC0_CLK */
+			AM62X_IOPAD(0x218, PIN_OUTPUT_PULLDOWN, 0) /* (AB1) MMC0_CLK */
 			AM62X_IOPAD(0x214, PIN_INPUT_PULLUP, 0) /* (AA2) MMC0_DAT0 */
 			AM62X_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (AA1) MMC0_DAT1 */
 			AM62X_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (AA3) MMC0_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62-pocketbeagle2.dts b/arch/arm64/boot/dts/ti/k3-am62-pocketbeagle2.dts
index 7a4cffc27bda31..1c691629d4ad07 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-pocketbeagle2.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62-pocketbeagle2.dts
@@ -235,7 +235,7 @@
 	main_mmc1_pins_default: main-mmc1-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x23c, PIN_INPUT, 0) /* (A21/C18) MMC1_CMD */
-			AM62X_IOPAD(0x234, PIN_INPUT, 0) /* (B22/A20) MMC1_CLK */
+			AM62X_IOPAD(0x234, PIN_OUTPUT, 0) /* (B22/A20) MMC1_CLK */
 			AM62X_IOPAD(0x230, PIN_INPUT, 0) /* (A22/A19) MMC1_DAT0 */
 			AM62X_IOPAD(0x22c, PIN_INPUT, 0) /* (B21/B19) MMC1_DAT1 */
 			AM62X_IOPAD(0x228, PIN_INPUT, 0) /* (C21/B20) MMC1_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62-verdin-ivy.dtsi b/arch/arm64/boot/dts/ti/k3-am62-verdin-ivy.dtsi
index 844f59f772e18d..c99076109a9b66 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-verdin-ivy.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62-verdin-ivy.dtsi
@@ -367,6 +367,11 @@
 		"",
 		"",
 		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 90 */
 		"";
 };
 
@@ -425,42 +430,6 @@
 		"",
 		"",
 		"", /* 50 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 60 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 70 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 80 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
 		"";
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-am62-verdin-zinnia.dtsi b/arch/arm64/boot/dts/ti/k3-am62-verdin-zinnia.dtsi
index 0ce8357797ed7e..f399764b05cf16 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-verdin-zinnia.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62-verdin-zinnia.dtsi
@@ -274,6 +274,11 @@
 		"",
 		"",
 		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 90 */
 		"";
 };
 
@@ -330,42 +335,6 @@
 		"",
 		"",
 		"", /* 50 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 60 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 70 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 80 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
 		"";
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi b/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi
index e97b2b047d1044..77e6885555fb16 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi
@@ -573,7 +573,7 @@
 	pinctrl_sdhci0: main-mmc0-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x220, PIN_INPUT,        0) /*  (Y3) MMC0_CMD  */
-			AM62X_IOPAD(0x218, PIN_INPUT,        0) /* (AB1) MMC0_CLK  */
+			AM62X_IOPAD(0x218, PIN_OUTPUT,       0) /* (AB1) MMC0_CLK  */
 			AM62X_IOPAD(0x214, PIN_INPUT,        0) /* (AA2) MMC0_DAT0 */
 			AM62X_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (AA1) MMC0_DAT1 */
 			AM62X_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (AA3) MMC0_DAT2 */
@@ -589,7 +589,7 @@
 	pinctrl_sdhci1: main-mmc1-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x23c, PIN_INPUT,        0) /* (A21) MMC1_CMD  */ /* SODIMM 74 */
-			AM62X_IOPAD(0x234, PIN_INPUT,        0) /* (B22) MMC1_CLK  */ /* SODIMM 78 */
+			AM62X_IOPAD(0x234, PIN_OUTPUT,       0) /* (B22) MMC1_CLK  */ /* SODIMM 78 */
 			AM62X_IOPAD(0x230, PIN_INPUT,        0) /* (A22) MMC1_DAT0 */ /* SODIMM 80 */
 			AM62X_IOPAD(0x22c, PIN_INPUT,        0) /* (B21) MMC1_DAT1 */ /* SODIMM 82 */
 			AM62X_IOPAD(0x228, PIN_INPUT,        0) /* (C21) MMC1_DAT2 */ /* SODIMM 70 */
@@ -601,7 +601,7 @@
 	pinctrl_sdhci2: main-mmc2-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x120, PIN_INPUT, 0) /* (C24) MMC2_CMD   */ /* WiFi_SDIO_CMD   */
-			AM62X_IOPAD(0x118, PIN_INPUT, 0) /* (D25) MMC2_CLK   */ /* WiFi_SDIO_CLK   */
+			AM62X_IOPAD(0x118, PIN_OUTPUT,0) /* (D25) MMC2_CLK   */ /* WiFi_SDIO_CLK   */
 			AM62X_IOPAD(0x114, PIN_INPUT, 0) /* (B24) MMC2_DAT0  */ /* WiFi_SDIO_DATA0 */
 			AM62X_IOPAD(0x110, PIN_INPUT, 0) /* (C25) MMC2_DAT1  */ /* WiFi_SDIO_DATA1 */
 			AM62X_IOPAD(0x10c, PIN_INPUT, 0) /* (E23) MMC2_DAT2  */ /* WiFi_SDIO_DATA2 */
@@ -1012,6 +1012,11 @@
 		"",
 		"",
 		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 90 */
 		"";
 };
 
@@ -1065,45 +1070,9 @@
 		"",
 		"",
 		"",
-		"",
+		"SODIMM_84",
 		"SODIMM_17",
 		"SODIMM_155", /* 50 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 60 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 70 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
-		"", /* 80 */
-		"",
-		"",
-		"",
-		"",
-		"",
-		"",
 		"";
 };
 
diff --git a/arch/arm64/boot/dts/ti/k3-am625-beagleplay.dts b/arch/arm64/boot/dts/ti/k3-am625-beagleplay.dts
index c468b9c5fc09ad..898e4ca3fd87e8 100644
--- a/arch/arm64/boot/dts/ti/k3-am625-beagleplay.dts
+++ b/arch/arm64/boot/dts/ti/k3-am625-beagleplay.dts
@@ -323,7 +323,7 @@
 		bootph-all;
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x0220, PIN_INPUT, 0) /* (Y3) MMC0_CMD */
-			AM62X_IOPAD(0x0218, PIN_INPUT, 0) /* (AB1) MMC0_CLK */
+			AM62X_IOPAD(0x0218, PIN_OUTPUT, 0) /* (AB1) MMC0_CLK */
 			AM62X_IOPAD(0x0214, PIN_INPUT, 0) /* (AA2) MMC0_DAT0 */
 			AM62X_IOPAD(0x0210, PIN_INPUT, 0) /* (AA1) MMC0_DAT1 */
 			AM62X_IOPAD(0x020c, PIN_INPUT, 0) /* (AA3) MMC0_DAT2 */
@@ -346,7 +346,7 @@
 		bootph-all;
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x023c, PIN_INPUT, 0) /* (A21) MMC1_CMD */
-			AM62X_IOPAD(0x0234, PIN_INPUT, 0) /* (B22) MMC1_CLK */
+			AM62X_IOPAD(0x0234, PIN_OUTPUT, 0) /* (B22) MMC1_CLK */
 			AM62X_IOPAD(0x0230, PIN_INPUT, 0) /* (A22) MMC1_DAT0 */
 			AM62X_IOPAD(0x022c, PIN_INPUT, 0) /* (B21) MMC1_DAT1 */
 			AM62X_IOPAD(0x0228, PIN_INPUT, 0) /* (C21) MMC1_DAT2 */
@@ -358,7 +358,7 @@
 	wifi_pins_default: wifi-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x0120, PIN_INPUT, 0) /* (C24) MMC2_CMD */
-			AM62X_IOPAD(0x0118, PIN_INPUT, 0) /* (D25) MMC2_CLK */
+			AM62X_IOPAD(0x0118, PIN_OUTPUT, 0) /* (D25) MMC2_CLK */
 			AM62X_IOPAD(0x0114, PIN_INPUT, 0) /* (B24) MMC2_DAT0 */
 			AM62X_IOPAD(0x0110, PIN_INPUT, 0) /* (C25) MMC2_DAT1 */
 			AM62X_IOPAD(0x010c, PIN_INPUT, 0) /* (E23) MMC2_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am625-sk-common.dtsi b/arch/arm64/boot/dts/ti/k3-am625-sk-common.dtsi
index 9c836268264532..8e07d2c2e62fd5 100644
--- a/arch/arm64/boot/dts/ti/k3-am625-sk-common.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am625-sk-common.dtsi
@@ -98,7 +98,7 @@
 		bootph-all;
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x220, PIN_INPUT, 0) /* (Y3) MMC0_CMD */
-			AM62X_IOPAD(0x218, PIN_INPUT, 0) /* (AB1) MMC0_CLK */
+			AM62X_IOPAD(0x218, PIN_OUTPUT, 0) /* (AB1) MMC0_CLK */
 			AM62X_IOPAD(0x214, PIN_INPUT, 0) /* (AA2) MMC0_DAT0 */
 			AM62X_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (AA1) MMC0_DAT1 */
 			AM62X_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (AA3) MMC0_DAT2 */
@@ -198,6 +198,7 @@
 
 &sdhci0 {
 	bootph-all;
+	non-removable;
 	status = "okay";
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_mmc0_pins_default>;
diff --git a/arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts
index efc00d2a860a77..3859512c426288 100644
--- a/arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts
+++ b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts
@@ -735,7 +735,7 @@
 			/* (A21) MMC1_CMD */
 			AM62X_IOPAD(0x23c, PIN_INPUT, 0)
 			/* (B22) MMC1_CLK */
-			AM62X_IOPAD(0x234, PIN_INPUT, 0)
+			AM62X_IOPAD(0x234, PIN_OUTPUT, 0)
 			/* (A22) MMC1_DAT0 */
 			AM62X_IOPAD(0x230, PIN_INPUT, 0)
 			/* (B21) MMC1_DAT1 */
@@ -755,7 +755,7 @@
 			/* (C24) MMC2_CMD */
 			AM62X_IOPAD(0x120, PIN_INPUT, 0)
 			/* (D25) MMC2_CLK */
-			AM62X_IOPAD(0x118, PIN_INPUT, 0)
+			AM62X_IOPAD(0x118, PIN_OUTPUT, 0)
 			/* (B24) MMC2_DAT0 */
 			AM62X_IOPAD(0x114, PIN_INPUT, 0)
 			/* (C25) MMC2_DAT1 */
diff --git a/arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi
index 72288678cd01fa..a200a2afe16fa9 100644
--- a/arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi
@@ -299,7 +299,7 @@
 			/* (Y3) MMC0_CMD */
 			AM62X_IOPAD(0x220, PIN_INPUT, 0)
 			/* (AB1) MMC0_CLK */
-			AM62X_IOPAD(0x218, PIN_INPUT, 0)
+			AM62X_IOPAD(0x218, PIN_OUTPUT, 0)
 			/* (AA2) MMC0_DAT0 */
 			AM62X_IOPAD(0x214, PIN_INPUT, 0)
 			/* (AA1) MMC0_DAT1 */
diff --git a/arch/arm64/boot/dts/ti/k3-am625-var-som-symphony.dts b/arch/arm64/boot/dts/ti/k3-am625-var-som-symphony.dts
index 7951d7ec952b53..082bf81143dcb4 100644
--- a/arch/arm64/boot/dts/ti/k3-am625-var-som-symphony.dts
+++ b/arch/arm64/boot/dts/ti/k3-am625-var-som-symphony.dts
@@ -391,7 +391,7 @@
 		bootph-all;
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x23c, PIN_INPUT, 0) /* (A21) MMC1_CMD */
-			AM62X_IOPAD(0x234, PIN_INPUT, 0) /* (B22) MMC1_CLK */
+			AM62X_IOPAD(0x234, PIN_OUTPUT, 0) /* (B22) MMC1_CLK */
 			AM62X_IOPAD(0x230, PIN_INPUT, 0) /* (A22) MMC1_DAT0 */
 			AM62X_IOPAD(0x22c, PIN_INPUT, 0) /* (B21) MMC1_DAT1 */
 			AM62X_IOPAD(0x228, PIN_INPUT, 0) /* (C21) MMC1_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am625-var-som.dtsi b/arch/arm64/boot/dts/ti/k3-am625-var-som.dtsi
index c2daa56c35e177..c23b6a90b8b13d 100644
--- a/arch/arm64/boot/dts/ti/k3-am625-var-som.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am625-var-som.dtsi
@@ -277,7 +277,7 @@
 		bootph-all;
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x220, PIN_INPUT, 0) /* (Y3) MMC0_CMD */
-			AM62X_IOPAD(0x218, PIN_INPUT, 0) /* (AB1) MMC0_CLK */
+			AM62X_IOPAD(0x218, PIN_OUTPUT, 0) /* (AB1) MMC0_CLK */
 			AM62X_IOPAD(0x214, PIN_INPUT, 0) /* (AA2) MMC0_DAT0 */
 			AM62X_IOPAD(0x210, PIN_INPUT, 0) /* (AA1) MMC0_DAT1 */
 			AM62X_IOPAD(0x20c, PIN_INPUT, 0) /* (AA3) MMC0_DAT2 */
@@ -292,7 +292,7 @@
 	pinctrl_mmc2: main-mmc2-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x120, PIN_INPUT_PULLUP, 0) /* (C24) MMC2_CMD */
-			AM62X_IOPAD(0x118, PIN_INPUT_PULLDOWN, 0) /* (D25) MMC2_CLK */
+			AM62X_IOPAD(0x118, PIN_OUTPUT_PULLDOWN, 0) /* (D25) MMC2_CLK */
 			AM62X_IOPAD(0x114, PIN_INPUT_PULLUP, 0) /* (B24) MMC2_DAT0 */
 			AM62X_IOPAD(0x110, PIN_INPUT_PULLUP, 0) /* (C25) MMC2_DAT1 */
 			AM62X_IOPAD(0x10c, PIN_INPUT_PULLUP, 0) /* (E23) MMC2_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi
index 9e5b75a4e88e23..92dd1b96cef4d3 100644
--- a/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi
@@ -1149,6 +1149,7 @@
 		reg = <0x00 0x30210000 0x00 0x10000>;
 		clocks = <&k3_clks 204 2>;
 		power-domains = <&k3_pds 204 TI_SCI_PD_EXCLUSIVE>;
+		sram = <&oc_sram>;
 	};
 
 	c7x_0: dsp@7e000000 {
diff --git a/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi
index d22caa7c346b35..5b903207787478 100644
--- a/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi
@@ -147,7 +147,7 @@
 		power-domains = <&k3_pds 131 TI_SCI_PD_EXCLUSIVE>;
 		assigned-clocks = <&k3_clks 131 0>;
 		assigned-clock-parents = <&k3_clks 131 2>;
-		/* Tightly coupled to M4F */
+		/* Tightly coupled to R5F */
 		status = "reserved";
 	};
 
diff --git a/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi
index de4048a3564bca..5ad93be3d33639 100644
--- a/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62a-phycore-som.dtsi
@@ -120,7 +120,7 @@
 	main_mmc0_pins_default: main-mmc0-default-pins {
 		pinctrl-single,pins = <
 			AM62AX_IOPAD(0x220, PIN_INPUT_PULLUP, 0) /* (Y6) MMC0_CMD */
-			AM62AX_IOPAD(0x218, PIN_INPUT_PULLDOWN, 0) /* (AB7) MMC0_CLK */
+			AM62AX_IOPAD(0x218, PIN_OUTPUT_PULLDOWN, 0) /* (AB7) MMC0_CLK */
 			AM62AX_IOPAD(0x214, PIN_INPUT_PULLUP, 0) /* (AA6) MMC0_DAT0 */
 			AM62AX_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (AB6) MMC0_DAT1 */
 			AM62AX_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (Y7) MMC0_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
index d3b3675e7a8fce..95e61ed9a635cb 100644
--- a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
@@ -25,6 +25,7 @@
 		mmc1 = &sdhci1;
 		rtc0 = &wkup_rtc0;
 		rtc1 = &tps659312;
+		ethernet0 = &cpsw_port1;
 	};
 
 	chosen {
@@ -328,8 +329,7 @@
 	main_mmc0_pins_default: main-mmc0-default-pins {
 		pinctrl-single,pins = <
 			AM62AX_IOPAD(0x220, PIN_INPUT, 0) /* (Y3) MMC0_CMD */
-			AM62AX_IOPAD(0x218, PIN_INPUT, 0) /* (AB1) MMC0_CLKLB */
-			AM62AX_IOPAD(0x21c, PIN_INPUT, 0) /* (AB1) MMC0_CLK */
+			AM62AX_IOPAD(0x218, PIN_OUTPUT, 0) /* (AB1) MMC0_CLK */
 			AM62AX_IOPAD(0x214, PIN_INPUT, 0) /* (AA2) MMC0_DAT0 */
 			AM62AX_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (AA1) MMC0_DAT1 */
 			AM62AX_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (AA3) MMC0_DAT2 */
@@ -345,7 +345,7 @@
 	main_mmc1_pins_default: main-mmc1-default-pins {
 		pinctrl-single,pins = <
 			AM62AX_IOPAD(0x23c, PIN_INPUT, 0) /* (A21) MMC1_CMD */
-			AM62AX_IOPAD(0x234, PIN_INPUT, 0) /* (B22) MMC1_CLK */
+			AM62AX_IOPAD(0x234, PIN_OUTPUT, 0) /* (B22) MMC1_CLK */
 			AM62AX_IOPAD(0x230, PIN_INPUT, 0) /* (A22) MMC1_DAT0 */
 			AM62AX_IOPAD(0x22c, PIN_INPUT, 0) /* (B21) MMC1_DAT1 */
 			AM62AX_IOPAD(0x228, PIN_INPUT, 0) /* (C21) MMC1_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts b/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts
index 84a0cfaaf196ad..964387eb517888 100644
--- a/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts
@@ -253,9 +253,9 @@
 
 	mmc0_pins_default: mmc0-default-pins {
 		pinctrl-single,pins = <
-			AM62LX_IOPAD(0x0214, PIN_INPUT_PULLUP, 0) /* (D2) MMC0_CMD */
+			AM62LX_IOPAD(0x0214, PIN_INPUT, 0) /* (D2) MMC0_CMD */
 			AM62LX_IOPAD(0x020c, PIN_OUTPUT, 0) /* (B2) MMC0_CLK */
-			AM62LX_IOPAD(0x0208, PIN_INPUT_PULLUP, 0) /* (D3) MMC0_DAT0 */
+			AM62LX_IOPAD(0x0208, PIN_INPUT, 0) /* (D3) MMC0_DAT0 */
 			AM62LX_IOPAD(0x0204, PIN_INPUT_PULLUP, 0) /* (D4) MMC0_DAT1 */
 			AM62LX_IOPAD(0x0200, PIN_INPUT_PULLUP, 0) /* (C1) MMC0_DAT2 */
 			AM62LX_IOPAD(0x01fc, PIN_INPUT_PULLUP, 0) /* (C2) MMC0_DAT3 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-main.dtsi
index f130c7cb998d79..e93f4ddbb2ddce 100644
--- a/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-main.dtsi
@@ -1111,12 +1111,24 @@
 		status = "disabled";
 	};
 
+	gpu: gpu@fd80000 {
+		compatible = "ti,am62p-gpu", "img,img-bxs-4-64", "img,img-rogue";
+		reg = <0x00 0x0fd80000 0x00 0x80000>;
+		clocks = <&k3_clks 237 3>;
+		clock-names = "core";
+		interrupts = <GIC_SPI 241 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 237 TI_SCI_PD_EXCLUSIVE>,
+				<&k3_pds 242 TI_SCI_PD_EXCLUSIVE>;
+		power-domain-names = "a", "b";
+	};
+
 	vpu: video-codec@30210000 {
 		compatible = "ti,j721s2-wave521c", "cnm,wave521c";
 		reg = <0x00 0x30210000 0x00 0x10000>;
 		interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&k3_clks 204 2>;
 		power-domains = <&k3_pds 204 TI_SCI_PD_EXCLUSIVE>;
+		sram = <&oc_sram>;
 	};
 
 	hsm: remoteproc@43c00000 {
diff --git a/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-mcu.dtsi
index 5288c959f3c1ae..2394a4fb5a94c8 100644
--- a/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-mcu.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62p-j722s-common-mcu.dtsi
@@ -149,7 +149,7 @@
 		power-domains = <&k3_pds 131 TI_SCI_PD_EXCLUSIVE>;
 		assigned-clocks = <&k3_clks 131 0>;
 		assigned-clock-parents = <&k3_clks 131 2>;
-		/* Tightly coupled to M4F */
+		/* Tightly coupled to R5F */
 		status = "reserved";
 	};
 
diff --git a/arch/arm64/boot/dts/ti/k3-am62x-phyboard-lyra.dtsi b/arch/arm64/boot/dts/ti/k3-am62x-phyboard-lyra.dtsi
index d6e70ee15938e8..c9b3e0518faf5e 100644
--- a/arch/arm64/boot/dts/ti/k3-am62x-phyboard-lyra.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62x-phyboard-lyra.dtsi
@@ -227,7 +227,7 @@
 	main_mmc1_pins_default: main-mmc1-default-pins {
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x23c, PIN_INPUT_PULLUP, 0) /* (A21) MMC1_CMD */
-			AM62X_IOPAD(0x234, PIN_INPUT_PULLDOWN, 0) /* (B22) MMC1_CLK */
+			AM62X_IOPAD(0x234, PIN_OUTPUT_PULLDOWN, 0) /* (B22) MMC1_CLK */
 			AM62X_IOPAD(0x230, PIN_INPUT_PULLUP, 0) /* (A22) MMC1_DAT0 */
 			AM62X_IOPAD(0x22c, PIN_INPUT_PULLUP, 0) /* (B21) MMC1_DAT1 */
 			AM62X_IOPAD(0x228, PIN_INPUT_PULLUP, 0) /* (C21) MMC1_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am62x-sk-common.dtsi b/arch/arm64/boot/dts/ti/k3-am62x-sk-common.dtsi
index ab9e58c2d225eb..d2f41e2ed62396 100644
--- a/arch/arm64/boot/dts/ti/k3-am62x-sk-common.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62x-sk-common.dtsi
@@ -194,7 +194,7 @@
 		bootph-all;
 		pinctrl-single,pins = <
 			AM62X_IOPAD(0x23c, PIN_INPUT, 0) /* (A21/C18) MMC1_CMD */
-			AM62X_IOPAD(0x234, PIN_INPUT, 0) /* (B22/A20) MMC1_CLK */
+			AM62X_IOPAD(0x234, PIN_OUTPUT, 0) /* (B22/A20) MMC1_CLK */
 			AM62X_IOPAD(0x230, PIN_INPUT, 0) /* (A22/A19) MMC1_DAT0 */
 			AM62X_IOPAD(0x22c, PIN_INPUT, 0) /* (B21/B19) MMC1_DAT1 */
 			AM62X_IOPAD(0x228, PIN_INPUT, 0) /* (C21/B20) MMC1_DAT2 */
diff --git a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts
index ca1300e1309365..edcf039ccfbd0e 100644
--- a/arch/arm64/boot/dts/ti/k3-am654-base-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-am654-base-board.dts
@@ -460,6 +460,7 @@
 &sdhci1 {
 	status = "okay";
 	vmmc-supply = <&vdd_mmc1_sd>;
+	power-off-delay-us = <650000>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_mmc1_pins_default>;
 	ti,driver-strength-ohm = <50>;
diff --git a/arch/arm64/boot/dts/ti/k3-am67-phycore-som.dtsi b/arch/arm64/boot/dts/ti/k3-am67-phycore-som.dtsi
new file mode 100644
index 00000000000000..bc74c4eef1932b
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am67-phycore-som.dtsi
@@ -0,0 +1,324 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (C) 2026 PHYTEC America LLC
+ * Author: Nathan Morrisson <nmorrisson@phytec.com>
+ */
+
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	compatible = "phytec,am67-phycore-som", "ti,j722s";
+	model = "PHYTEC phyCORE-AM67";
+
+	aliases {
+		ethernet0 = &cpsw_port1;
+		gpio0 = &main_gpio0;
+		mmc0 = &sdhci0;
+		rtc0 = &i2c_som_rtc;
+		rtc1 = &wkup_rtc0;
+		spi0 = &ospi0;
+	};
+
+	memory@80000000 {
+		/* 4G RAM */
+		reg = <0x00000000 0x80000000 0x00000000 0x80000000>,
+		      <0x00000008 0x80000000 0x00000000 0x80000000>;
+		device_type = "memory";
+		bootph-all;
+	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		secure_tfa_ddr: tfa@9e780000 {
+			reg = <0x00 0x9e780000 0x00 0x80000>;
+			no-map;
+		};
+
+		secure_ddr: optee@9e800000 {
+			reg = <0x00 0x9e800000 0x00 0x01800000>;
+			no-map;
+		};
+
+		wkup_r5fss0_core0_dma_memory_region: memory@a0000000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa0000000 0x00 0x100000>;
+			no-map;
+		};
+
+		wkup_r5fss0_core0_memory_region: memory@a0100000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0xa0100000 0x00 0xf00000>;
+			no-map;
+		};
+	};
+
+	vcc_5v0_som: regulator-vcc-5v0-som {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_5V0_SOM";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&leds_pins_default>;
+
+		led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			gpios = <&main_gpio0 13 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			function = LED_FUNCTION_HEARTBEAT;
+		};
+	};
+};
+
+&main_pmx0 {
+	leds_pins_default: leds-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x034, PIN_OUTPUT, 7)	/* (K22) OSPI0_CSN2.GPIO0_13 */
+		>;
+	};
+
+	mdio_pins_default: mdio-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x0160, PIN_OUTPUT, 0)	/* (AC24) MDIO0_MDC */
+			J722S_IOPAD(0x015c, PIN_INPUT, 0)	/* (AD25) MDIO0_MDIO */
+		>;
+		bootph-all;
+	};
+
+	ospi0_pins_default: ospi0-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x000, PIN_OUTPUT, 0)	/* (L24) OSPI0_CLK */
+			J722S_IOPAD(0x02c, PIN_OUTPUT, 0)	/* (K26) OSPI0_CSn0 */
+			J722S_IOPAD(0x00c, PIN_INPUT, 0)	/* (K27) OSPI0_D0 */
+			J722S_IOPAD(0x010, PIN_INPUT, 0)	/* (L27) OSPI0_D1 */
+			J722S_IOPAD(0x014, PIN_INPUT, 0)	/* (L26) OSPI0_D2 */
+			J722S_IOPAD(0x018, PIN_INPUT, 0)	/* (L25) OSPI0_D3 */
+			J722S_IOPAD(0x01c, PIN_INPUT, 0)	/* (L21) OSPI0_D4 */
+			J722S_IOPAD(0x020, PIN_INPUT, 0)	/* (M26) OSPI0_D5 */
+			J722S_IOPAD(0x024, PIN_INPUT, 0)	/* (N27) OSPI0_D6 */
+			J722S_IOPAD(0x028, PIN_INPUT, 0)	/* (M27) OSPI0_D7 */
+			J722S_IOPAD(0x008, PIN_INPUT, 0)	/* (L22) OSPI0_DQS */
+			J722S_IOPAD(0x038, PIN_INPUT, 7)	/* (J22) OSPI0_CSn3.GPIO0_14 */
+		>;
+		bootph-all;
+	};
+
+	pmic_irq_pins_default: pmic-irq-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x030, PIN_INPUT, 7)	/* (K23) OSPI0_CSN1.GPIO0_12 */
+		>;
+	};
+
+	rgmii1_pins_default: rgmii1-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x014c, PIN_INPUT, 0)	/* (AC25) RGMII1_RD0 */
+			J722S_IOPAD(0x0150, PIN_INPUT, 0)	/* (AD27) RGMII1_RD1 */
+			J722S_IOPAD(0x0154, PIN_INPUT, 0)	/* (AE24) RGMII1_RD2 */
+			J722S_IOPAD(0x0158, PIN_INPUT, 0)	/* (AE26) RGMII1_RD3 */
+			J722S_IOPAD(0x0148, PIN_INPUT, 0)	/* (AE27) RGMII1_RXC */
+			J722S_IOPAD(0x0144, PIN_INPUT, 0)	/* (AD23) RGMII1_RX_CTL */
+			J722S_IOPAD(0x0134, PIN_OUTPUT, 0)	/* (AF27) RGMII1_TD0 */
+			J722S_IOPAD(0x0138, PIN_OUTPUT, 0)	/* (AE23) RGMII1_TD1 */
+			J722S_IOPAD(0x013c, PIN_OUTPUT, 0)	/* (AG25) RGMII1_TD2 */
+			J722S_IOPAD(0x0140, PIN_OUTPUT, 0)	/* (AF24) RGMII1_TD3 */
+			J722S_IOPAD(0x0130, PIN_OUTPUT, 0)	/* (AG26) RGMII1_TXC */
+			J722S_IOPAD(0x012c, PIN_OUTPUT, 0)	/* (AF25) RGMII1_TX_CTL */
+		>;
+		bootph-all;
+	};
+};
+
+&mcu_pmx0 {
+	wkup_i2c0_pins_default: wkup-i2c0-default-pins {
+		pinctrl-single,pins = <
+			J722S_MCU_IOPAD(0x04c, PIN_INPUT_PULLUP, 0)	/* (B9) WKUP_I2C0_SCL */
+			J722S_MCU_IOPAD(0x050, PIN_INPUT_PULLUP, 0)	/* (D11) WKUP_I2C0_SDA */
+		>;
+		bootph-all;
+	};
+};
+
+&cpsw3g {
+	pinctrl-names = "default";
+	pinctrl-0 = <&rgmii1_pins_default>;
+	bootph-all;
+	status = "okay";
+};
+
+&cpsw3g_mdio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mdio_pins_default>;
+	status = "okay";
+
+	cpsw3g_phy1: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		tx-fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,min-output-impedance;
+	};
+};
+
+&cpsw_port1 {
+	phy-mode = "rgmii-id";
+	phy-handle = <&cpsw3g_phy1>;
+	status = "okay";
+};
+
+&ospi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ospi0_pins_default>;
+	bootph-all;
+	status = "okay";
+
+	serial_flash: flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		spi-tx-bus-width = <8>;
+		spi-rx-bus-width = <8>;
+		spi-max-frequency = <25000000>;
+		vcc-supply = <&vdd_1v8>;
+		cdns,tshsl-ns = <60>;
+		cdns,tsd2d-ns = <60>;
+		cdns,tchsh-ns = <60>;
+		cdns,tslch-ns = <60>;
+		cdns,read-delay = <0>;
+	};
+};
+
+&sdhci0 {
+	non-removable;
+	bootph-all;
+	ti,driver-strength-ohm = <50>;
+	status = "okay";
+};
+
+&wkup_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&wkup_i2c0_pins_default>;
+	clock-frequency = <400000>;
+	bootph-all;
+	status = "okay";
+
+	pmic@30 {
+		compatible = "ti,tps65219";
+		reg = <0x30>;
+		buck1-supply = <&vcc_5v0_som>;
+		buck2-supply = <&vcc_5v0_som>;
+		buck3-supply = <&vcc_5v0_som>;
+		ldo1-supply = <&vdd_3v3>;
+		ldo2-supply = <&vdd_1v8>;
+		ldo3-supply = <&vdd_3v3>;
+		ldo4-supply = <&vdd_3v3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_irq_pins_default>;
+		interrupt-parent = <&main_gpio0>;
+		interrupts = <12 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-controller;
+		#interrupt-cells = <1>;
+
+		system-power-controller;
+		ti,power-button;
+
+		regulators {
+			vdd_3v3: buck1 {
+				regulator-name = "VDD_3V3";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_1v8: buck2 {
+				regulator-name = "VDD_1V8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_lpddr4: buck3 {
+				regulator-name = "VDD_LPDDR4";
+				regulator-min-microvolt = <1100000>;
+				regulator-max-microvolt = <1100000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vddshv_sdio: ldo1 {
+				regulator-name = "VDDSHV_SDIO";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-allow-bypass;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_1v2: ldo2 {
+				regulator-name = "VDD_1V2";
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdda_1v8_phy: ldo3 {
+				regulator-name = "VDDA_1V8_PHY";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vdd_1v8_pll: ldo4 {
+				regulator-name = "VDD_1V8_PLL";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+
+	vdd_core: regulator-vdd-core@44 {
+		compatible = "ti,tps62873";
+		reg = <0x44>;
+		bootph-pre-ram;
+		regulator-name = "VDD_CORE";
+		regulator-min-microvolt = <850000>;
+		regulator-max-microvolt = <850000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c32";
+		reg = <0x50>;
+		pagesize = <32>;
+	};
+
+	som_eeprom_opt: eeprom@51 {
+		compatible = "atmel,24c32";
+		reg = <0x51>;
+		pagesize = <32>;
+	};
+
+	i2c_som_rtc: rtc@52 {
+		compatible = "microcrystal,rv3028";
+		reg = <0x52>;
+	};
+};
+
+#include "k3-j722s-ti-ipc-firmware.dtsi"
diff --git a/arch/arm64/boot/dts/ti/k3-am6754-phyboard-rigel.dts b/arch/arm64/boot/dts/ti/k3-am6754-phyboard-rigel.dts
new file mode 100644
index 00000000000000..5fd4a8ceca160f
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am6754-phyboard-rigel.dts
@@ -0,0 +1,431 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (C) 2026 PHYTEC America LLC
+ * Author: Nathan Morrisson <nmorrisson@phytec.com>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/phy/phy.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "k3-serdes.h"
+#include "k3-j722s.dtsi"
+#include "k3-am67-phycore-som.dtsi"
+
+/ {
+	compatible = "phytec,am6754-phyboard-rigel",
+		     "phytec,am67-phycore-som", "ti,j722s";
+	model = "PHYTEC phyBOARD-Rigel AM67";
+
+	aliases {
+		gpio1 = &main_gpio1;
+		mmc1 = &sdhci1;
+		serial2 = &main_uart0;
+		usb0 = &usb0;
+		usb1 = &usb1;
+	};
+
+	can_tc0: can-phy0 {
+		compatible = "ti,tcan1042";
+		#phy-cells = <0>;
+		max-bitrate = <8000000>;
+		standby-gpios = <&gpio_exp1 1 GPIO_ACTIVE_HIGH>;
+	};
+
+	usb0_connector: connector {
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		label = "USB-C";
+		data-role = "dual";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&main_usbc_power_pins_default>;
+
+		id-gpios = <&main_gpio1 15 GPIO_ACTIVE_HIGH>;
+
+		port {
+			usb0_con: endpoint {
+				remote-endpoint = <&usb0_ep>;
+			};
+		};
+	};
+
+	keys {
+		compatible = "gpio-keys";
+		autorepeat;
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_keys_pins_default>;
+
+		key-home {
+			label = "home";
+			linux,code = <KEY_HOME>;
+			gpios = <&main_gpio1 23 GPIO_ACTIVE_HIGH>;
+		};
+
+		key-menu {
+			label = "menu";
+			linux,code = <KEY_MENU>;
+			gpios = <&gpio_exp1 4 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	vcc_1v8: regulator-vcc-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_3v3_aud: regulator-vcc-3v3-aud {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3_AUD";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_3v3_mmc: regulator-vcc-3v3-mmc {
+		/* TPS22963C OUTPUT */
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3_MMC";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_3v3_sw: regulator-vcc-3v3-sw {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3_SW";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_speaker: regulator-vcc-speaker {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_SPEAKER";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,widgets =
+			"Microphone", "Mic Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Stereo Jack",
+			"Speaker", "L SPKR",
+			"Speaker", "R SPKR";
+		simple-audio-card,routing =
+			"MIC1RP", "Mic Jack",
+			"Mic Jack", "MICBIAS",
+			"Headphone Jack", "HPL",
+			"Headphone Jack", "HPR",
+			"MIC1LM", "Stereo Jack",
+			"MIC1LP", "Stereo Jack",
+			"L SPKR", "SPL",
+			"R SPKR", "SPR";
+		simple-audio-card,name = "phyBOARD-Rigel";
+		simple-audio-card,format = "dsp_b";
+		simple-audio-card,bitclock-master = <&sound_master>;
+		simple-audio-card,frame-master = <&sound_master>;
+		simple-audio-card,bitclock-inversion;
+
+		simple-audio-card,cpu {
+			sound-dai = <&mcasp0>;
+		};
+
+		sound_master: simple-audio-card,codec {
+			sound-dai = <&audio_codec>;
+			clocks = <&audio_refclk1>;
+		};
+	};
+};
+
+&main_pmx0 {
+	audio_ext_refclk1_pins_default: audio-ext-refclk1-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x0a0, PIN_OUTPUT, 1)	/* (N24) GPMC0_WPn.AUDIO_EXT_REFCLK1 */
+		>;
+	};
+
+	gpio_exp0_int_pins_default: gpio-exp0-int-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x0054, PIN_INPUT, 7)	/* (T21) GPMC0_AD6.GPIO0_21 */
+		>;
+	};
+
+	gpio_exp1_int_pins_default: gpio-exp1-int-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x0244, PIN_INPUT, 7)	/* (A24) MMC1_SDWP.GPIO1_49 */
+		>;
+	};
+
+	gpio_exp2_int_pins_default: gpio-exp2-int-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x0050, PIN_INPUT, 7)	/* (T24) GPMC0_AD5.GPIO0_20 */
+		>;
+	};
+
+	gpio_keys_pins_default: gpio-keys-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x01d4, PIN_INPUT, 7)	/* (B21) UART0_RTSn.GPIO1_23 */
+		>;
+	};
+
+	main_i2c0_pins_default: main-i2c0-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x01e0, PIN_INPUT_PULLUP, 0)	/* (D23) I2C0_SCL */
+			J722S_IOPAD(0x01e4, PIN_INPUT_PULLUP, 0)	/* (B22) I2C0_SDA */
+		>;
+		bootph-all;
+	};
+
+	main_i2c1_pins_default: main-i2c1-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x01e8, PIN_INPUT_PULLUP, 0)	/* (C24) I2C1_SCL */
+			J722S_IOPAD(0x01ec, PIN_INPUT_PULLUP, 0)	/* (A22) I2C1_SDA */
+		>;
+		bootph-all;
+	};
+
+	main_mcan0_pins_default: main-mcan0-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x1dc, PIN_INPUT, 0)	/* (C22) MCAN0_RX */
+			J722S_IOPAD(0x1d8, PIN_OUTPUT, 0)	/* (D22) MCAN0_TX */
+		>;
+	};
+
+	main_mcasp0_pins_default: main-mcasp0-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x1a8, PIN_INPUT, 0)	/* (C26) MCASP0_AFSX */
+			J722S_IOPAD(0x1a4, PIN_INPUT, 0)	/* (D25) MCASP0_ACLKX */
+			J722S_IOPAD(0x198, PIN_OUTPUT, 0)	/* (A26) MCASP0_AXR2 */
+			J722S_IOPAD(0x194, PIN_INPUT, 0)	/* (A25) MCASP0_AXR3 */
+		>;
+	};
+
+	main_mmc1_pins_default: main-mmc1-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x023c, PIN_INPUT, 0)	/* (H22) MMC1_CMD */
+			J722S_IOPAD(0x0234, PIN_INPUT, 0)	/* (H24) MMC1_CLK */
+			J722S_IOPAD(0x0230, PIN_INPUT, 0)	/* (H23) MMC1_DAT0 */
+			J722S_IOPAD(0x022c, PIN_INPUT, 0)	/* (H20) MMC1_DAT1 */
+			J722S_IOPAD(0x0228, PIN_INPUT, 0)	/* (J23) MMC1_DAT2 */
+			J722S_IOPAD(0x0224, PIN_INPUT, 0)	/* (H25) MMC1_DAT3 */
+			J722S_IOPAD(0x0240, PIN_INPUT, 0)	/* (B24) MMC1_SDCD */
+		>;
+		bootph-all;
+	};
+
+	main_uart0_pins_default: main-uart0-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x01c8, PIN_INPUT, 0)	/* (F19) UART0_RXD */
+			J722S_IOPAD(0x01cc, PIN_OUTPUT, 0)	/* (F20) UART0_TXD */
+		>;
+		bootph-all;
+	};
+
+	main_usbc_power_pins_default: main-usbc-power-default-pins {
+		pinctrl-single,pins = <
+			J722S_IOPAD(0x1b4, PIN_INPUT, 7)	/* (B20) SPI0_CS0.GPIO1_15 */
+		>;
+	};
+};
+
+&audio_refclk1 {
+	assigned-clock-rates = <25000000>;
+};
+
+&main_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c0_pins_default>;
+	clock-frequency = <400000>;
+	status = "okay";
+
+	veml6030: light-sensor@10 {
+		compatible = "vishay,veml6030";
+		reg = <0x10>;
+		vdd-supply = <&vcc_3v3_sw>;
+	};
+};
+
+&main_i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c1_pins_default>;
+	clock-frequency = <100000>;
+	status = "okay";
+
+	audio_codec: audio-codec@18 {
+		compatible = "ti,tlv320aic3110";
+		reg = <0x18>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&audio_ext_refclk1_pins_default>;
+		#sound-dai-cells = <0>;
+		ai31xx-micbias-vg = <2>;
+		reset-gpios = <&gpio_exp1 7 GPIO_ACTIVE_LOW>;
+
+		HPVDD-supply = <&vcc_3v3_aud>;
+		SPRVDD-supply = <&vcc_speaker>;
+		SPLVDD-supply = <&vcc_speaker>;
+		AVDD-supply = <&vcc_3v3_aud>;
+		IOVDD-supply = <&vcc_3v3_aud>;
+		DVDD-supply = <&vcc_1v8>;
+	};
+
+	gpio_exp0: gpio@20 {
+		compatible = "nxp,pcf8574";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_exp0_int_pins_default>;
+		interrupt-parent = <&main_gpio0>;
+		interrupts = <21 IRQ_TYPE_EDGE_FALLING>;
+		gpio-line-names = "CSI3_STROBE", "CSI3_TRIGGER",
+				  "CSI3_SHUTTER", "CSI3_OE",
+				  "CSI2_STROBE", "CSI2_TRIGGER",
+				  "CSI2_SHUTTER", "CSI2_OE";
+	};
+
+	gpio_exp1: gpio@21 {
+		compatible = "nxp,pcf8574";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_exp1_int_pins_default>;
+		interrupt-parent = <&main_gpio1>;
+		interrupts = <49 IRQ_TYPE_EDGE_FALLING>;
+		gpio-line-names = "GPIO0_HDMI_RST", "GPIO1_CAN_nEN",
+				  "GPIO2_LED", "GPIO3_MCU_CAN0_nEN",
+				  "GPIO4_BUT2", "GPIO5_MCU_CAN1_nEN",
+				  "GPIO6_AUDIO_GPIO", "GPIO7_AUDIO_USER_RESET";
+	};
+
+	gpio_exp2: gpio@23 {
+		compatible = "nxp,pcf8574";
+		reg = <0x23>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_exp2_int_pins_default>;
+		interrupt-parent = <&main_gpio0>;
+		interrupts = <20 IRQ_TYPE_EDGE_FALLING>;
+		gpio-line-names = "CSI1_STROBE", "CSI1_TRIGGER",
+				  "CSI1_SHUTTER", "CSI1_OE",
+				  "CSI0_STROBE", "CSI0_TRIGGER",
+				  "CSI0_SHUTTER", "CSI0_OE";
+	};
+
+	current-sensor@40 {
+		compatible = "ti,ina233";
+		reg = <0x40>;
+		shunt-resistor = <18000>;
+	};
+
+	eeprom@51 {
+		compatible = "atmel,24c02";
+		reg = <0x51>;
+		pagesize = <16>;
+	};
+};
+
+&main_mcan0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mcan0_pins_default>;
+	phys = <&can_tc0>;
+	status = "okay";
+};
+
+&main_uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_uart0_pins_default>;
+	bootph-all;
+	status = "okay";
+};
+
+&mcasp0 {
+	#sound-dai-cells = <0>;
+	op-mode = <0>; /* MCASP_IIS_MODE */
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mcasp0_pins_default>;
+	tdm-slots = <2>;
+	serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */
+	       0 0 1 2
+	       0 0 0 0
+	       0 0 0 0
+	       0 0 0 0
+	>;
+	status = "okay";
+};
+
+&sdhci1 {
+	/* SD/MMC */
+	vmmc-supply = <&vcc_3v3_mmc>;
+	vqmmc-supply = <&vddshv_sdio>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc1_pins_default>;
+	disable-wp;
+	no-1-8-v;
+	bootph-all;
+	status = "okay";
+};
+
+&serdes_ln_ctrl {
+	idle-states = <J722S_SERDES0_LANE0_USB>,
+		      <J722S_SERDES1_LANE0_PCIE0_LANE0>;
+};
+
+&serdes0 {
+	status = "okay";
+
+	serdes0_usb_link: phy@0 {
+		reg = <0>;
+		cdns,num-lanes = <1>;
+		#phy-cells = <0>;
+		cdns,phy-type = <PHY_TYPE_USB3>;
+		resets = <&serdes_wiz0 1>;
+	};
+};
+
+&serdes_wiz0 {
+	status = "okay";
+};
+
+&usbss0 {
+	ti,vbus-divider;
+	status = "okay";
+};
+
+&usb0 {
+	dr_mode = "otg";
+	usb-role-switch;
+	maximum-speed = "high-speed";
+
+	port {
+		usb0_ep: endpoint {
+			remote-endpoint = <&usb0_con>;
+		};
+	};
+};
+
+&usbss1 {
+	ti,vbus-divider;
+	status = "okay";
+};
+
+&usb1 {
+	dr_mode = "host";
+	phys = <&serdes0_usb_link>;
+	phy-names = "cdns3,usb3-phy";
+	maximum-speed = "super-speed";
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-dsi-to-lvds-v2-panel-cap-touch-10inch.dtso b/arch/arm64/boot/dts/ti/k3-am69-aquila-dsi-to-lvds-v2-panel-cap-touch-10inch.dtso
new file mode 100644
index 00000000000000..92e270611f05e9
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-dsi-to-lvds-v2-panel-cap-touch-10inch.dtso
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex DSI to LVDS Adapter on Aquila DSI_1 with Capacitive Touch Display
+ * 10.1" V2 Used on Development Board (J44). DSI_1 is exposed via a Samtec
+ * LSS-130 connector.
+ *
+ * https://developer.toradex.com/hardware/accessories/displays/capacitive-touch-display-101inch-lvds
+ * https://www.toradex.com/accessories/capacitive-touch-display-10.1-inch-lvds
+ * https://developer.toradex.com/hardware/accessories/add-ons/dsi-lvds-adapter
+ * https://www.toradex.com/accessories/verdin-dsi-to-lvds-adapter
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+&{/} {
+	backlight_pwm3: backlight-pwm3 {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_19_dsi_1>;
+		brightness-levels = <0 45 63 88 119 158 203 255>;
+		default-brightness-level = <4>;
+		/* Aquila GPIO_19_DSI_1 (AQUILA B44) - DSI_1_BKL_EN_LVDS */
+		enable-gpios = <&main_gpio0 13 GPIO_ACTIVE_HIGH>;
+		/* Aquila PWM_3_DSI (AQUILA B46) - PWM_3_DSI_LVDS */
+		pwms = <&main_ehrpwm5 0 6666667 0>;
+	};
+
+	panel-lvds-bridge {
+		compatible = "optologic,scx1001511ggc49", "panel-lvds";
+		backlight = <&backlight_pwm3>;
+		data-mapping = "vesa-24";
+		height-mm = <136>;
+		width-mm = <217>;
+
+		/*
+		 * The timings here are different from the nominal ones for the
+		 * SCX1001511GGC49 display. The clock-frequency, along with the
+		 * other timings, are adjusted from experimentation to work
+		 * consistently with the DSI to LVDS bridge (TI SN65DSI84)
+		 * across Toradex modules that use it.
+		 */
+		panel-timing {
+			clock-frequency = <70000000>;
+			hactive = <1280>;
+			hback-porch = <55>;
+			hfront-porch = <55>;
+			hsync-len = <45>;
+			vactive = <800>;
+			vback-porch = <5>;
+			vfront-porch = <5>;
+			vsync-len = <3>;
+			de-active = <1>;
+			hsync-active = <0>;
+			vsync-active = <0>;
+			pixelclk-active = <0>;
+		};
+
+		port {
+			panel_lvds_bridge_in: endpoint {
+				remote-endpoint = <&dsi_lvds_bridge_out>;
+			};
+		};
+	};
+};
+
+&dphy_tx0 {
+	status = "okay";
+};
+
+&dsi0 {
+	status = "okay";
+};
+
+&dsi0_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@0 {
+		reg = <0>;
+
+		dsi0_out: endpoint {
+			remote-endpoint = <&dsi_lvds_bridge_in>;
+		};
+	};
+
+	port@1 {
+		reg = <1>;
+
+		dsi0_in: endpoint {
+			remote-endpoint = <&dpi2_out>;
+		};
+	};
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@2 {
+		reg = <2>;
+
+		dpi2_out: endpoint {
+			remote-endpoint = <&dsi0_in>;
+		};
+	};
+};
+
+&i2c_dsi_0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bridge@2c {
+		compatible = "ti,sn65dsi84";
+		reg = <0x2c>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_20_dsi_1>;
+		/* Aquila GPIO_20_DSI_1 (AQUILA B45) - LVDS_EN */
+		enable-gpios = <&main_gpio0 18 GPIO_ACTIVE_HIGH>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				dsi_lvds_bridge_in: endpoint {
+					remote-endpoint = <&dsi0_out>;
+					data-lanes = <1 2 3 4>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+
+				dsi_lvds_bridge_out: endpoint {
+					remote-endpoint = <&panel_lvds_bridge_in>;
+				};
+			};
+		};
+	};
+
+	touchscreen@41 {
+		compatible = "ilitek,ili251x";
+		reg = <0x41>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_17_dsi_1>, <&pinctrl_gpio_18_dsi_1>;
+		/* Aquila GPIO_17_DSI_1 (AQUILA B42) - TOUCH_INT# */
+		/* Rising edge avoids spurious interrupts from the noisy falling edge. */
+		interrupt-parent = <&main_gpio0>;
+		interrupts = <12 IRQ_TYPE_EDGE_RISING>;
+		/* Aquila GPIO_18_DSI_1 (AQUILA B43) - TOUCH_RESET# */
+		reset-gpios = <&main_gpio0 31 GPIO_ACTIVE_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi b/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi
index 5119baf62a4c23..b6854acee8fecd 100644
--- a/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am69-aquila.dtsi
@@ -1573,7 +1573,7 @@
 		"AQUILA_B70",
 		"AQUILA_B71",
 		"AQUILA_B72",
-		"AQUILC_B73",
+		"AQUILA_B73",
 		"AQUILA_B66",
 		"AQUILA_B62",
 		"AQUILA_C21",
diff --git a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
index 54dea36b153b18..ac1c1ffe40b7d6 100644
--- a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
@@ -24,6 +24,7 @@
 		serial5 = &main_uart3;
 		mmc0 = &main_sdhci0;
 		mmc1 = &main_sdhci1;
+		ethernet0 = &cpsw_port1;
 	};
 
 	chosen {
diff --git a/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts
index 19c5157510b261..a4a180bbca1285 100644
--- a/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-j721s2-common-proc-board.dts
@@ -32,6 +32,7 @@
 		can2 = &mcu_mcan1;
 		can3 = &main_mcan3;
 		can4 = &main_mcan5;
+		ethernet0 = &cpsw_port1;
 	};
 
 	evm_12v0: fixedregulator-evm12v0 {
diff --git a/arch/arm64/boot/dts/ti/k3-j721s2-evm-audio.dtso b/arch/arm64/boot/dts/ti/k3-j721s2-evm-audio.dtso
new file mode 100644
index 00000000000000..ac5a827e1b7509
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-j721s2-evm-audio.dtso
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Device Tree Overlay for J721S2 Audio Support
+ *
+ * Copyright (C) 2026 Texas Instruments Incorporated - https://www.ti.com/
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+
+#include "k3-pinctrl.h"
+
+&{/} {
+	codec_audio: sound {
+		compatible = "ti,j7200-cpb-audio";
+		model = "j721s2-cpb";
+
+		ti,cpb-mcasp = <&mcasp4>;
+		ti,cpb-codec = <&pcm3168a_1>;
+
+		clocks = <&k3_clks 213 0>, <&k3_clks 213 1>,
+			 <&k3_clks 157 299>, <&k3_clks 157 328>;
+		clock-names = "cpb-mcasp-auxclk", "cpb-mcasp-auxclk-48000",
+			      "cpb-codec-scki", "cpb-codec-scki-48000";
+	};
+
+	i2c_mux: mux-controller-2 {
+		compatible = "gpio-mux";
+		#mux-state-cells = <1>;
+		mux-gpios = <&wkup_gpio0 54 GPIO_ACTIVE_HIGH>;
+		idle-state = <1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&main_i2c3_mux_pins_default>;
+	};
+};
+
+&main_pmx0 {
+	mcasp4_pins_default: mcasp4-default-pins {
+		pinctrl-single,pins = <
+			J721S2_IOPAD(0x0c8, PIN_OUTPUT_PULLDOWN, 1) /* (AD28) MCASP4_ACLKX */
+			J721S2_IOPAD(0x06c, PIN_OUTPUT_PULLDOWN, 1) /* (V26) MCASP4_AFSX */
+			J721S2_IOPAD(0x068, PIN_INPUT_PULLDOWN, 1) /* (U28) MCASP4_AXR1 */
+			J721S2_IOPAD(0x0c4, PIN_OUTPUT_PULLDOWN, 1) /* (AB26) MCASP4_AXR2 */
+			J721S2_IOPAD(0x070, PIN_OUTPUT_PULLDOWN, 1) /* (R27) MCASP4_AXR3 */
+		>;
+	};
+
+	audio_ext_refclk1_pins_default: audio-ext-refclk1-default-pins {
+		pinctrl-single,pins = <
+			J721S2_IOPAD(0x078, PIN_OUTPUT, 1) /* (Y25) MCAN2_RX.AUDIO_EXT_REFCLK1 */
+		>;
+	};
+};
+
+&wkup_pmx2 {
+	main_i2c3_mux_pins_default: main-i2c3-mux-default-pins {
+		pinctrl-single,pins = <
+			J721S2_WKUP_IOPAD(0x038, PIN_OUTPUT, 7) /* (B27) WKUP_GPIO0_54 */
+		>;
+	};
+};
+
+&exp2 {
+	p09-hog {
+		/* P09 - MCASP/TRACE_MUX_S0 */
+		gpio-hog;
+		gpios = <9 GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "MCASP/TRACE_MUX_S0";
+	};
+
+	p10-hog {
+		/* P10 - MCASP/TRACE_MUX_S1 */
+		gpio-hog;
+		gpios = <10 GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "MCASP/TRACE_MUX_S1";
+	};
+};
+
+&mux0 {
+	idle-state = <0>;
+};
+
+&mux1 {
+	idle-state = <0>;
+};
+
+&main_mcan3 {
+	/* Conflicts with McASP4 signal routing via mux0 */
+	status = "disabled";
+};
+
+&main_mcan5 {
+	/* Conflicts with McASP4 signal routing via mux1 */
+	status = "disabled";
+};
+
+&k3_clks {
+	/* Configure AUDIO_EXT_REFCLK1 pin as output */
+	pinctrl-names = "default";
+	pinctrl-0 = <&audio_ext_refclk1_pins_default>;
+};
+
+&main_i2c3 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c3_pins_default>;
+	clock-frequency = <400000>;
+	mux-states = <&i2c_mux 1>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	exp3: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-line-names = "CODEC_RSTZ", "CODEC_SPARE1",
+				  "UB926_RESETN", "UB926_LOCK",
+				  "UB926_PWR_SW_CNTRL", "UB926_TUNER_RESET",
+				  "UB926_GPIO_SPARE";
+	};
+
+	pcm3168a_1: audio-codec@44 {
+		compatible = "ti,pcm3168a";
+		reg = <0x44>;
+		#sound-dai-cells = <1>;
+		reset-gpios = <&exp3 0 GPIO_ACTIVE_LOW>;
+		clocks = <&audio_refclk1>;
+		clock-names = "scki";
+		VDD1-supply = <&vsys_3v3>;
+		VDD2-supply = <&vsys_3v3>;
+		VCCAD1-supply = <&vsys_5v0>;
+		VCCAD2-supply = <&vsys_5v0>;
+		VCCDA1-supply = <&vsys_5v0>;
+		VCCDA2-supply = <&vsys_5v0>;
+	};
+};
+
+&mcasp4 {
+	status = "okay";
+	#sound-dai-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcasp4_pins_default>;
+	op-mode = <0>;          /* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	auxclk-fs-ratio = <256>;
+	serial-dir = <	/* 0: INACTIVE, 1: TX, 2: RX */
+		0 2 1 1
+		0 0 0 0
+		0 0 0 0
+		0 0 0 0
+	>;
+};
diff --git a/arch/arm64/boot/dts/ti/k3-j721s2-main.dtsi b/arch/arm64/boot/dts/ti/k3-j721s2-main.dtsi
index 1228ac5711bf07..1bb25fb3cfdf02 100644
--- a/arch/arm64/boot/dts/ti/k3-j721s2-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j721s2-main.dtsi
@@ -74,6 +74,15 @@
 			reg = <0x140 0x18>;
 			#clock-cells = <1>;
 		};
+
+		audio_refclk1: clock-controller@42e4 {
+			compatible = "ti,j721s2-audio-refclk", "ti,am62-audio-refclk";
+			reg = <0x42e4 0x4>;
+			clocks = <&k3_clks 157 299>;
+			assigned-clocks = <&k3_clks 157 299>;
+			assigned-clock-parents = <&k3_clks 157 328>;
+			#clock-cells = <0>;
+		};
 	};
 
 	main_ehrpwm0: pwm@3000000 {
@@ -716,6 +725,7 @@
 		interrupts = <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&k3_clks 179 2>;
 		power-domains = <&k3_pds 179 TI_SCI_PD_EXCLUSIVE>;
+		sram = <&main_navss_sram>;
 	};
 
 	main_sdhci0: mmc@4f80000 {
@@ -784,6 +794,14 @@
 		dma-coherent;
 		dma-ranges;
 
+		main_navss_sram: sram@30000000 {
+			compatible = "mmio-sram";
+			reg = <0x00 0x30000000 0x00 0x10000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x00 0x30000000 0x10000>;
+		};
+
 		main_navss_intr: interrupt-controller@310e0000 {
 			compatible = "ti,sci-intr";
 			reg = <0x00 0x310e0000 0x00 0x4000>;
diff --git a/arch/arm64/boot/dts/ti/k3-j722s-evm.dts b/arch/arm64/boot/dts/ti/k3-j722s-evm.dts
index e66330c71593ad..c6486cf5a05cfa 100644
--- a/arch/arm64/boot/dts/ti/k3-j722s-evm.dts
+++ b/arch/arm64/boot/dts/ti/k3-j722s-evm.dts
@@ -23,6 +23,7 @@
 		serial3 = &main_uart5;
 		mmc0 = &sdhci0;
 		mmc1 = &sdhci1;
+		ethernet0 = &cpsw_port1;
 	};
 
 	chosen {
@@ -162,6 +163,12 @@
 		regulator-boot-on;
 	};
 
+	tlv320_mclk: clock-12288000 {
+		compatible = "fixed-clock";
+		clock-frequency = <12288000>;
+		#clock-cells = <0>;
+	};
+
 	codec_audio: sound {
 		compatible = "simple-audio-card";
 		simple-audio-card,name = "J722S-EVM";
@@ -187,7 +194,7 @@
 
 		sound_master: simple-audio-card,codec {
 			sound-dai = <&tlv320aic3106>;
-			clocks = <&audio_refclk1>;
+			clocks = <&tlv320_mclk>;
 		};
 	};
 
@@ -228,11 +235,6 @@
 	};
 };
 
-&audio_refclk1 {
-	assigned-clocks = <&k3_clks 157 0>;
-	assigned-clock-parents = <&k3_clks 157 15>;
-};
-
 &cpsw_mac_syscon {
 	bootph-all;
 };
@@ -358,12 +360,6 @@
 		>;
 	};
 
-	audio_ext_refclk1_pins_default: audio-ext-refclk1-default-pins {
-		pinctrl-single,pins = <
-			J722S_IOPAD(0x00a0, PIN_OUTPUT, 1) /* (N24) GPMC0_WPn.AUDIO_EXT_REFCLK1 */
-		>;
-	};
-
 	pmic_irq_pins_default: pmic-irq-default-pins {
 		pinctrl-single,pins = <
 			J722S_IOPAD(0x030, PIN_INPUT, 7) /* (K23) GPIO0_12 */
@@ -562,12 +558,6 @@
 	};
 };
 
-&k3_clks {
-	/* Configure AUDIO_EXT_REFCLK1 pin as output */
-	pinctrl-names = "default";
-	pinctrl-0 = <&audio_ext_refclk1_pins_default>;
-};
-
 &main_i2c0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&main_i2c0_pins_default>;
diff --git a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi
index ff3a85cbc524a9..7c085663c8880e 100644
--- a/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j784s4-j742s2-evm-common.dtsi
@@ -1206,6 +1206,8 @@
 	status = "okay";
 	clocks = <&k3_clks 333 0>, <&serdes0 CDNS_TORRENT_REFCLK_DRIVER>;
 	clock-names = "fck", "pcie_refclk";
+	assigned-clocks = <&k3_clks 415 0>;
+	assigned-clock-parents = <&k3_clks 415 2>;
 	num-lanes = <2>;
 	reset-gpios = <&exp1 2 GPIO_ACTIVE_HIGH>;
 	phys = <&serdes0_pcie1_link>;
diff --git a/arch/arm64/boot/dts/ti/k3-pinctrl.h b/arch/arm64/boot/dts/ti/k3-pinctrl.h
index 4491898d8294ca..e8ca69a97b653b 100644
--- a/arch/arm64/boot/dts/ti/k3-pinctrl.h
+++ b/arch/arm64/boot/dts/ti/k3-pinctrl.h
@@ -8,6 +8,7 @@
 #ifndef DTS_ARM64_TI_K3_PINCTRL_H
 #define DTS_ARM64_TI_K3_PINCTRL_H
 
+#define VGPIO_SEL_SHIFT		(4)
 #define WKUP_LVL_EN_SHIFT	(7)
 #define WKUP_LVL_POL_SHIFT	(8)
 #define DEBOUNCE_SHIFT		(11)
@@ -111,6 +112,15 @@
 #define PIN_WKUP_EN_LEVEL_HIGH		(WKUP_ENABLE | WKUP_ON_LEVEL | WKUP_LEVEL_HIGH)
 #define PIN_WKUP_EN			(WKUP_ENABLE | WKUP_ON_EDGE)
 
+/*
+ * SoC-specific field - Refer to the SoC Technical Reference Manual to
+ * determine which GPIO controller each value selects.
+ */
+#define PIN_VGPIO_SEL0		(0 << VGPIO_SEL_SHIFT)
+#define PIN_VGPIO_SEL1		(1 << VGPIO_SEL_SHIFT)
+#define PIN_VGPIO_SEL2		(2 << VGPIO_SEL_SHIFT)
+#define PIN_VGPIO_SEL3		(3 << VGPIO_SEL_SHIFT)
+
 /* Default mux configuration for gpio-ranges to use with pinctrl */
 #define PIN_GPIO_RANGE_IOPAD	(PIN_INPUT | 7)
 
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 6ec8d3a69c98e0..fce418fe6ff654 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -51,6 +51,7 @@ CONFIG_ARCH_BLAIZE=y
 CONFIG_ARCH_BST=y
 CONFIG_ARCH_CIX=y
 CONFIG_ARCH_EXYNOS=y
+CONFIG_ARCH_GOOGLE=y
 CONFIG_ARCH_K3=y
 CONFIG_ARCH_LG1K=y
 CONFIG_ARCH_HISI=y
@@ -267,6 +268,7 @@ CONFIG_RASPBERRYPI_FIRMWARE=y
 CONFIG_INTEL_STRATIX10_SERVICE=y
 CONFIG_INTEL_STRATIX10_RSU=m
 CONFIG_MTK_ADSP_IPC=m
+CONFIG_TI_SCI_PROTOCOL=m
 CONFIG_GOOGLE_FIRMWARE=y
 CONFIG_GOOGLE_CBMEM=m
 CONFIG_GOOGLE_COREBOOT_TABLE=m
@@ -636,6 +638,7 @@ CONFIG_I2C_RK3X=y
 CONFIG_I2C_RZV2M=m
 CONFIG_I2C_S3C2410=y
 CONFIG_I2C_SH_MOBILE=y
+CONFIG_I2C_STM32F7=m
 CONFIG_I2C_TEGRA=y
 CONFIG_I2C_UNIPHIER_F=y
 CONFIG_I2C_XILINX=m
@@ -673,6 +676,7 @@ CONFIG_SPI_QUP=y
 CONFIG_SPI_QCOM_GENI=m
 CONFIG_SPI_S3C64XX=y
 CONFIG_SPI_SH_MSIOF=m
+CONFIG_SPI_STM32=m
 CONFIG_SPI_STM32_OSPI=m
 CONFIG_SPI_SUN6I=y
 CONFIG_SPI_TEGRA210_QUAD=m
@@ -763,12 +767,14 @@ CONFIG_SENSORS_ARM_SCPI=y
 CONFIG_SENSORS_GPIO_FAN=m
 CONFIG_SENSORS_JC42=m
 CONFIG_SENSORS_MACSMC_HWMON=m
+CONFIG_SENSORS_MR75203=m
 CONFIG_SENSORS_LM75=m
 CONFIG_SENSORS_LM90=m
 CONFIG_SENSORS_PWM_FAN=m
 CONFIG_SENSORS_RASPBERRYPI_HWMON=m
 CONFIG_SENSORS_SL28CPLD=m
 CONFIG_SENSORS_SHT4x=m
+CONFIG_SENSORS_EMC2305=m
 CONFIG_SENSORS_AMC6821=m
 CONFIG_SENSORS_INA2XX=m
 CONFIG_SENSORS_INA3221=m
@@ -800,6 +806,7 @@ CONFIG_TEGRA_BPMP_THERMAL=m
 CONFIG_GENERIC_ADC_THERMAL=m
 CONFIG_QCOM_TSENS=y
 CONFIG_QCOM_SPMI_ADC_TM5=m
+CONFIG_QCOM_SPMI_ADC_TM5_GEN3=m
 CONFIG_QCOM_SPMI_TEMP_ALARM=m
 CONFIG_QCOM_LMH=m
 CONFIG_UNIPHIER_THERMAL=y
@@ -895,6 +902,7 @@ CONFIG_REGULATOR_QCOM_USB_VBUS=m
 CONFIG_REGULATOR_RAA215300=y
 CONFIG_REGULATOR_RK808=y
 CONFIG_REGULATOR_S2MPS11=y
+CONFIG_REGULATOR_SGM3804=y
 CONFIG_REGULATOR_TPS65132=m
 CONFIG_REGULATOR_TPS65219=y
 CONFIG_REGULATOR_VCTRL=m
@@ -1018,6 +1026,7 @@ CONFIG_DRM_PANEL_LVDS=m
 CONFIG_DRM_PANEL_HIMAX_HX8279=m
 CONFIG_DRM_PANEL_HIMAX_HX83112A=m
 CONFIG_DRM_PANEL_HIMAX_HX83112B=m
+CONFIG_DRM_PANEL_ILITEK_ILI9806E_SPI=m
 CONFIG_DRM_PANEL_ILITEK_ILI9882T=m
 CONFIG_DRM_PANEL_KHADAS_TS050=m
 CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m
@@ -1026,6 +1035,7 @@ CONFIG_DRM_PANEL_NOVATEK_NT36672E=m
 CONFIG_DRM_PANEL_NOVATEK_NT37801=m
 CONFIG_DRM_PANEL_RAYDIUM_RM67191=m
 CONFIG_DRM_PANEL_RAYDIUM_RM692E5=m
+CONFIG_DRM_PANEL_RENESAS_R63419=m
 CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20=m
 CONFIG_DRM_PANEL_SITRONIX_ST7703=m
 CONFIG_DRM_PANEL_STARTEK_KD070FHFID015=m
@@ -1062,6 +1072,7 @@ CONFIG_DRM_ZYNQMP_DPSUB_AUDIO=y
 CONFIG_FB=y
 CONFIG_FB_EFI=y
 CONFIG_FB_MODE_HELPERS=y
+CONFIG_BACKLIGHT_SY7758=m
 CONFIG_BACKLIGHT_PWM=m
 CONFIG_BACKLIGHT_APPLE_DWI=m
 CONFIG_BACKLIGHT_QCOM_WLED=m
@@ -1120,6 +1131,7 @@ CONFIG_SND_SOC_ROCKCHIP_SPDIF=m
 CONFIG_SND_SOC_ROCKCHIP_RT5645=m
 CONFIG_SND_SOC_RK3399_GRU_SOUND=m
 CONFIG_SND_SOC_SAMSUNG=m
+CONFIG_SND_SOC_STM32_SAI=m
 CONFIG_SND_SUN8I_CODEC=m
 CONFIG_SND_SUN8I_CODEC_ANALOG=m
 CONFIG_SND_SUN50I_CODEC_ANALOG=m
@@ -1323,6 +1335,7 @@ CONFIG_LEDS_CLASS_MULTICOLOR=m
 CONFIG_LEDS_LM3692X=m
 CONFIG_LEDS_PCA9532=m
 CONFIG_LEDS_GPIO=y
+CONFIG_LEDS_PCA963X=m
 CONFIG_LEDS_PWM=y
 CONFIG_LEDS_SYSCON=y
 CONFIG_LEDS_QCOM_FLASH=m
@@ -1403,8 +1416,8 @@ CONFIG_DW_EDMA=m
 CONFIG_RCAR_DMAC=y
 CONFIG_RENESAS_USB_DMAC=m
 CONFIG_RZ_DMAC=y
-CONFIG_TI_K3_UDMA=y
-CONFIG_TI_K3_UDMA_GLUE_LAYER=y
+CONFIG_TI_K3_UDMA=m
+CONFIG_TI_K3_UDMA_GLUE_LAYER=m
 CONFIG_STM32_DMA3=m
 CONFIG_DMABUF_HEAPS=y
 CONFIG_DMABUF_HEAPS_SYSTEM=m
@@ -1456,7 +1469,7 @@ CONFIG_CLK_IMX8QXP=y
 CONFIG_CLK_IMX8ULP=y
 CONFIG_CLK_IMX93=y
 CONFIG_CLK_IMX95_BLK_CTL=y
-CONFIG_TI_SCI_CLK=y
+CONFIG_TI_SCI_CLK=m
 CONFIG_COMMON_CLK_MT8192_AUDSYS=y
 CONFIG_COMMON_CLK_MT8192_CAMSYS=y
 CONFIG_COMMON_CLK_MT8192_IMGSYS=y
@@ -1470,126 +1483,7 @@ CONFIG_COMMON_CLK_MT8192_SCP_ADSP=y
 CONFIG_COMMON_CLK_MT8192_VDECSYS=y
 CONFIG_COMMON_CLK_MT8192_VENCSYS=y
 CONFIG_COMMON_CLK_QCOM=y
-CONFIG_CLK_ELIZA_DISPCC=m
-CONFIG_CLK_ELIZA_GCC=y
-CONFIG_CLK_ELIZA_TCSRCC=m
-CONFIG_CLK_GLYMUR_DISPCC=m
-CONFIG_CLK_GLYMUR_GCC=y
-CONFIG_CLK_GLYMUR_TCSRCC=m
-CONFIG_CLK_KAANAPALI_CAMCC=m
-CONFIG_CLK_KAANAPALI_DISPCC=m
-CONFIG_CLK_KAANAPALI_GCC=y
-CONFIG_CLK_KAANAPALI_GPUCC=m
-CONFIG_CLK_KAANAPALI_TCSRCC=m
-CONFIG_CLK_KAANAPALI_VIDEOCC=m
-CONFIG_CLK_X1E80100_CAMCC=m
-CONFIG_CLK_X1E80100_DISPCC=m
-CONFIG_CLK_X1E80100_GCC=y
-CONFIG_CLK_X1E80100_GPUCC=m
-CONFIG_CLK_X1E80100_TCSRCC=y
-CONFIG_CLK_X1P42100_GPUCC=m
-CONFIG_CLK_QCM2290_GPUCC=m
-CONFIG_QCOM_A53PLL=y
-CONFIG_QCOM_CLK_APCS_MSM8916=y
-CONFIG_QCOM_CLK_APCC_MSM8996=y
-CONFIG_QCOM_CLK_SMD_RPM=y
-CONFIG_QCOM_CLK_RPMH=y
-CONFIG_IPQ_APSS_6018=y
-CONFIG_IPQ_CMN_PLL=m
-CONFIG_IPQ_GCC_5018=y
-CONFIG_IPQ_GCC_5210=y
-CONFIG_IPQ_GCC_5332=y
-CONFIG_IPQ_GCC_5424=y
-CONFIG_IPQ_GCC_6018=y
-CONFIG_IPQ_GCC_8074=y
-CONFIG_IPQ_GCC_9574=y
-CONFIG_IPQ_NSSCC_5424=m
-CONFIG_IPQ_NSSCC_9574=m
-CONFIG_MSM_GCC_8916=y
-CONFIG_MSM_GCC_8953=y
-CONFIG_MSM_MMCC_8994=m
-CONFIG_MSM_GCC_8994=y
-CONFIG_MSM_GCC_8996=y
-CONFIG_MSM_MMCC_8996=m
-CONFIG_MSM_GCC_8998=y
-CONFIG_MSM_MMCC_8998=m
-CONFIG_QCM_GCC_2290=y
-CONFIG_QCM_DISPCC_2290=m
-CONFIG_QCS_DISPCC_615=m
-CONFIG_QCS_CAMCC_615=m
-CONFIG_QCS_GCC_404=y
-CONFIG_SA_CAMCC_8775P=m
-CONFIG_QCS_GCC_8300=y
-CONFIG_QCS_GCC_615=y
-CONFIG_QCS_GPUCC_615=m
-CONFIG_QCS_VIDEOCC_615=m
-CONFIG_SC_CAMCC_7280=m
-CONFIG_SC_CAMCC_8280XP=m
-CONFIG_SA_DISPCC_8775P=m
-CONFIG_SC_DISPCC_7280=m
-CONFIG_SC_DISPCC_8280XP=m
-CONFIG_SA_GCC_8775P=y
-CONFIG_SA_GPUCC_8775P=m
-CONFIG_SC_GCC_7180=y
-CONFIG_SC_GCC_7280=y
-CONFIG_SC_GCC_8180X=y
-CONFIG_SC_GCC_8280XP=y
-CONFIG_SC_GPUCC_7280=m
-CONFIG_SC_GPUCC_8280XP=m
-CONFIG_SC_LPASSCC_8280XP=m
-CONFIG_SC_LPASS_CORECC_7280=m
-CONFIG_SC_VIDEOCC_7280=m
-CONFIG_SDM_CAMCC_845=m
-CONFIG_QDU_GCC_1000=y
-CONFIG_SDM_GPUCC_845=y
-CONFIG_SDM_VIDEOCC_845=y
-CONFIG_SDM_DISPCC_845=y
-CONFIG_SDM_LPASSCC_845=m
-CONFIG_SDX_GCC_75=y
-CONFIG_SM_CAMCC_6350=m
-CONFIG_SM_CAMCC_MILOS=m
-CONFIG_SM_CAMCC_8250=m
-CONFIG_SM_CAMCC_8550=m
-CONFIG_SM_CAMCC_8650=m
-CONFIG_SM_CAMCC_8750=m
-CONFIG_SM_DISPCC_6115=m
-CONFIG_SM_DISPCC_8250=y
-CONFIG_SM_DISPCC_6350=m
-CONFIG_SM_DISPCC_MILOS=m
-CONFIG_SM_DISPCC_8450=m
-CONFIG_SM_DISPCC_8550=m
-CONFIG_SM_DISPCC_8750=m
-CONFIG_SM_GCC_4450=y
-CONFIG_SM_GCC_6115=y
-CONFIG_SM_GCC_6350=y
-CONFIG_SM_GCC_MILOS=y
-CONFIG_SM_GCC_8350=y
-CONFIG_SM_GCC_8450=y
-CONFIG_SM_GCC_8550=y
-CONFIG_SM_GCC_8650=y
-CONFIG_SM_GCC_8750=y
-CONFIG_SM_GPUCC_6115=m
-CONFIG_SM_GPUCC_6350=m
-CONFIG_SM_GPUCC_MILOS=m
-CONFIG_SM_GPUCC_8150=y
-CONFIG_SM_GPUCC_8250=y
-CONFIG_SM_GPUCC_8350=m
-CONFIG_SM_GPUCC_8450=m
-CONFIG_SM_GPUCC_8550=m
-CONFIG_SM_GPUCC_8650=m
-CONFIG_SM_GPUCC_8750=m
-CONFIG_SM_TCSRCC_8550=y
-CONFIG_SM_TCSRCC_8650=y
-CONFIG_SM_TCSRCC_8750=m
-CONFIG_SA_VIDEOCC_8775P=m
-CONFIG_SM_VIDEOCC_6350=m
-CONFIG_SM_VIDEOCC_MILOS=m
-CONFIG_SM_VIDEOCC_8250=y
-CONFIG_SM_VIDEOCC_8550=m
-CONFIG_SM_VIDEOCC_8750=m
 CONFIG_QCOM_HFPLL=y
-CONFIG_CLK_GFM_LPASS_SM8250=m
-CONFIG_SM_VIDEOCC_8450=m
 CONFIG_CLK_RCAR_USB2_CLOCK_SEL=y
 CONFIG_CLK_RENESAS_VBATTB=m
 CONFIG_EXYNOS_ACPM_CLK=m
@@ -1678,7 +1572,7 @@ CONFIG_QCOM_CPR=y
 CONFIG_QCOM_RPMHPD=y
 CONFIG_QCOM_RPMPD=y
 CONFIG_ROCKCHIP_PM_DOMAINS=y
-CONFIG_TI_SCI_PM_DOMAINS=y
+CONFIG_TI_SCI_PM_DOMAINS=m
 CONFIG_ARM_IMX_BUS_DEVFREQ=y
 CONFIG_ARM_IMX8M_DDRC_DEVFREQ=m
 CONFIG_ARM_MEDIATEK_CCI_DEVFREQ=m
@@ -1699,6 +1593,7 @@ CONFIG_MEDIATEK_MT6359_AUXADC=m
 CONFIG_MEDIATEK_MT6577_AUXADC=m
 CONFIG_QCOM_SPMI_VADC=m
 CONFIG_QCOM_SPMI_ADC5=m
+CONFIG_QCOM_SPMI_ADC5_GEN3=m
 CONFIG_ROCKCHIP_SARADC=m
 CONFIG_RZG2L_ADC=m
 CONFIG_RZT2H_ADC=m
@@ -1743,17 +1638,15 @@ CONFIG_PWM_VISCONTI=m
 CONFIG_PWM_XILINX=m
 CONFIG_SL28CPLD_INTC=y
 CONFIG_XILINX_INTC=y
-CONFIG_QCOM_PDC=y
-CONFIG_QCOM_MPM=y
-CONFIG_TI_SCI_INTR_IRQCHIP=y
-CONFIG_TI_SCI_INTA_IRQCHIP=y
+CONFIG_TI_SCI_INTR_IRQCHIP=m
+CONFIG_TI_SCI_INTA_IRQCHIP=m
 CONFIG_RESET_GPIO=m
 CONFIG_RESET_IMX7=y
 CONFIG_RESET_QCOM_AOSS=y
 CONFIG_RESET_QCOM_PDC=m
 CONFIG_RESET_RZG2L_USBPHY_CTRL=y
 CONFIG_RESET_RZV2H_USB2PHY=m
-CONFIG_RESET_TI_SCI=y
+CONFIG_RESET_TI_SCI=m
 CONFIG_PHY_CAN_TRANSCEIVER=m
 CONFIG_PHY_NXP_PTN3222=m
 CONFIG_PHY_SNPS_EUSB2=m
@@ -1778,6 +1671,7 @@ CONFIG_PHY_MTK_DP=m
 CONFIG_PHY_QCOM_EDP=m
 CONFIG_PHY_QCOM_PCIE2=m
 CONFIG_PHY_QCOM_QMP=m
+CONFIG_PHY_QCOM_QMP_PCIE_MULTIPHY=m
 CONFIG_PHY_QCOM_QUSB2=m
 CONFIG_PHY_QCOM_EUSB2_REPEATER=m
 CONFIG_PHY_QCOM_M31_USB=m
diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S
index f2624238fd9543..c2e8f500da1446 100644
--- a/arch/arm64/crypto/aes-ce-ccm-core.S
+++ b/arch/arm64/crypto/aes-ce-ccm-core.S
@@ -50,7 +50,7 @@
 	ld1	{v0.16b}, [x5]			/* load mac */
 	cbz	x2, ce_aes_ccm_final
 	ldr	x8, [x6, #8]			/* load lower ctr */
-CPU_LE(	rev	x8, x8			)	/* keep swabbed ctr in reg */
+	rev	x8, x8				/* keep swabbed ctr in reg */
 0:	/* outer loop */
 	ld1	{v1.8b}, [x6]			/* load upper ctr */
 	prfm	pldl1strm, [x1]
@@ -73,7 +73,7 @@ CPU_LE(	rev	x8, x8			)	/* keep swabbed ctr in reg */
 	eor	v0.16b, v0.16b, v2.16b		/* xor mac with pt ^ rk[last] */
 	st1	{v6.16b}, [x0], #16		/* write output block */
 	bne	0b
-CPU_LE(	rev	x8, x8			)
+	rev	x8, x8
 	str	x8, [x6, #8]			/* store lsb end of ctr (BE) */
 	cbnz	x7, ce_aes_ccm_final
 	st1	{v0.16b}, [x5]			/* store mac */
diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S
index baf450717b24ba..ba1f981a5464d1 100644
--- a/arch/arm64/crypto/aes-neonbs-core.S
+++ b/arch/arm64/crypto/aes-neonbs-core.S
@@ -824,8 +824,8 @@ SYM_FUNC_START(aesbs_ctr_encrypt)
 	frame_push	0
 	ldp		x7, x8, [x5]
 	ld1		{v0.16b}, [x5]
-CPU_LE(	rev		x7, x7		)
-CPU_LE(	rev		x8, x8		)
+	rev		x7, x7
+	rev		x8, x8
 	adds		x8, x8, #1
 	adc		x7, x7, xzr
 
diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
index 7cb1aede9a484e..20b2b23ffa1f20 100644
--- a/arch/arm64/crypto/aes-neonbs-glue.c
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -247,7 +247,7 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 			    unsigned int key_len)
 {
 	struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct crypto_aes_ctx rk;
+	struct crypto_aes_ctx rk __cleanup(aes_zeroize_ctx);
 	int err;
 
 	err = xts_verify_key(tfm, in_key, key_len);
diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S
index 33772d8fe6b5e0..ba350d57fc766d 100644
--- a/arch/arm64/crypto/ghash-ce-core.S
+++ b/arch/arm64/crypto/ghash-ce-core.S
@@ -148,7 +148,7 @@ SYM_FUNC_START(pmull_ghash_update_p64)
 	sub		w0, w0, #1
 
 3:	/* multiply XL by SHASH in GF(2^128) */
-CPU_LE(	rev64		T1.16b, T1.16b	)
+	rev64		T1.16b, T1.16b
 
 	ext		T2.16b, XL.16b, XL.16b, #8
 	ext		IN1.16b, T1.16b, T1.16b, #8
@@ -272,7 +272,7 @@ SYM_FUNC_END(pmull_ghash_update_p64)
 	cbz		x0, 3f				// tag only?
 
 	ldr		w8, [x5, #12]			// load lower counter
-CPU_LE(	rev		w8, w8		)
+	rev		w8, w8
 
 0:	mov		w9, #4				// max blocks per round
 	add		x10, x0, #0xf
@@ -380,7 +380,7 @@ CPU_LE(	rev		w8, w8		)
 	ret
 
 5:
-CPU_LE(	rev		w8, w8		)
+	rev		w8, w8
 	str		w8, [x5, #12]			// store lower counter
 	st1		{XL.2d}, [x4]
 	b		4b
diff --git a/arch/arm64/crypto/sm4-ce-cipher-core.S b/arch/arm64/crypto/sm4-ce-cipher-core.S
index 4ac6cfbc57970e..1682d727e0970d 100644
--- a/arch/arm64/crypto/sm4-ce-cipher-core.S
+++ b/arch/arm64/crypto/sm4-ce-cipher-core.S
@@ -18,7 +18,7 @@
 SYM_FUNC_START(sm4_ce_do_crypt)
 	ld1		{v8.4s}, [x2]
 	ld1		{v0.4s-v3.4s}, [x0], #64
-CPU_LE(	rev32		v8.16b, v8.16b		)
+	rev32		v8.16b, v8.16b
 	ld1		{v4.4s-v7.4s}, [x0]
 	sm4e		v8.4s, v0.4s
 	sm4e		v8.4s, v1.4s
@@ -30,7 +30,7 @@ CPU_LE(	rev32		v8.16b, v8.16b		)
 	sm4e		v8.4s, v7.4s
 	rev64		v8.4s, v8.4s
 	ext		v8.16b, v8.16b, v8.16b, #8
-CPU_LE(	rev32		v8.16b, v8.16b		)
+	rev32		v8.16b, v8.16b
 	st1		{v8.4s}, [x1]
 	ret
 SYM_FUNC_END(sm4_ce_do_crypt)
diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h
index 86241662485286..ee75a2f1a575fe 100644
--- a/arch/arm64/include/asm/alternative-macros.h
+++ b/arch/arm64/include/asm/alternative-macros.h
@@ -53,8 +53,7 @@
  *
  * Alternatives with callbacks do not generate replacement instructions.
  */
-#define __ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, cfg_enabled)	\
-	".if "__stringify(cfg_enabled)" == 1\n"				\
+#define __ALTERNATIVE(oldinstr, newinstr, cpucap)			\
 	"661:\n\t"							\
 	oldinstr "\n"							\
 	"662:\n"							\
@@ -68,10 +67,8 @@
 	".org	. - (664b-663b) + (662b-661b)\n\t"			\
 	".org	. - (662b-661b) + (664b-663b)\n\t"			\
 	".previous\n"							\
-	".endif\n"
 
-#define __ALTERNATIVE_CFG_CB(oldinstr, cpucap, cfg_enabled, cb)	\
-	".if "__stringify(cfg_enabled)" == 1\n"				\
+#define __ALTERNATIVE_CB(oldinstr, cpucap, cb)				\
 	"661:\n\t"							\
 	oldinstr "\n"							\
 	"662:\n"							\
@@ -80,13 +77,12 @@
 	".popsection\n"							\
 	"663:\n\t"							\
 	"664:\n\t"							\
-	".endif\n"
 
-#define _ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, cfg, ...)	\
-	__ALTERNATIVE_CFG(oldinstr, newinstr, cpucap, IS_ENABLED(cfg))
+#define _ALTERNATIVE(oldinstr, newinstr, cpucap)			\
+	__ALTERNATIVE(oldinstr, newinstr, cpucap)
 
 #define ALTERNATIVE_CB(oldinstr, cpucap, cb) \
-	__ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (cpucap), 1, cb)
+	__ALTERNATIVE_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (cpucap), cb)
 #else
 
 #include <asm/assembler.h>
@@ -99,8 +95,7 @@
 	.byte \alt_len
 .endm
 
-.macro alternative_insn insn1, insn2, cap, enable = 1
-	.if \enable
+.macro alternative_insn insn1, insn2, cap
 661:	\insn1
 662:	.pushsection .altinstructions, "a"
 	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
@@ -110,7 +105,6 @@
 664:	.org	. - (664b-663b) + (662b-661b)
 	.org	. - (662b-661b) + (664b-663b)
 	.previous
-	.endif
 .endm
 
 /*
@@ -204,20 +198,16 @@ alternative_else
 alternative_endif
 .endm
 
-#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)	\
-	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
+#define _ALTERNATIVE(insn1, insn2, cap)	\
+	alternative_insn insn1, insn2, cap
 
 #endif  /*  __ASSEMBLER__  */
 
 /*
  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, cpucap));
- *
- * Usage: asm(ALTERNATIVE(oldinstr, newinstr, cpucap, CONFIG_FOO));
- * N.B. If CONFIG_FOO is specified, but not selected, the whole block
- *      will be omitted, including oldinstr.
  */
-#define ALTERNATIVE(oldinstr, newinstr, ...)   \
-	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
+#define ALTERNATIVE(oldinstr, newinstr, cpucap)   \
+	_ALTERNATIVE(oldinstr, newinstr, cpucap)
 
 #ifndef __ASSEMBLER__
 
diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h
index 8babfbe31f9541..7605dd81bd1e1f 100644
--- a/arch/arm64/include/asm/archrandom.h
+++ b/arch/arm64/include/asm/archrandom.h
@@ -12,7 +12,7 @@
 
 extern bool smccc_trng_available;
 
-static inline bool __init smccc_probe_trng(void)
+static inline bool smccc_probe_trng(void)
 {
 	struct arm_smccc_res res;
 
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 0b58b550e8dce7..42a167a75486ea 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -129,37 +129,6 @@ lr	.req	x30		// link register
 	b	\label
 	.endm
 
-/*
- * Select code when configured for BE.
- */
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define CPU_BE(code...) code
-#else
-#define CPU_BE(code...)
-#endif
-
-/*
- * Select code when configured for LE.
- */
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define CPU_LE(code...)
-#else
-#define CPU_LE(code...) code
-#endif
-
-/*
- * Define a macro that constructs a 64-bit value by concatenating two
- * 32-bit registers. Note that on big endian systems the order of the
- * registers is swapped.
- */
-#ifndef CONFIG_CPU_BIG_ENDIAN
-	.macro	regs_to_64, rd, lbits, hbits
-#else
-	.macro	regs_to_64, rd, hbits, lbits
-#endif
-	orr	\rd, \lbits, \hbits, lsl #32
-	.endm
-
 /*
  * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where
  * <symbol> is within the range +/- 4 GB of the PC.
diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
index 89d2ba2723590d..160c1251f0ec91 100644
--- a/arch/arm64/include/asm/atomic_ll_sc.h
+++ b/arch/arm64/include/asm/atomic_ll_sc.h
@@ -11,6 +11,7 @@
 #define __ASM_ATOMIC_LL_SC_H
 
 #include <linux/stringify.h>
+#include <asm/xwreg.h>
 
 #ifndef CONFIG_CC_HAS_K_CONSTRAINT
 #define K
@@ -239,20 +240,17 @@ __ll_sc_atomic64_dec_if_positive(atomic64_t *v)
 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)	\
 static __always_inline u##sz						\
 __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\
-					 unsigned long old,		\
+					 u##sz old,			\
 					 u##sz new)			\
 {									\
+	/*                                                              \
+	 * Sub-word sizes require zero extension so that EOR+CBNZ won't	\
+	 * consume non-zero upper bits of the register containing "old".\
+	 */								\
+	xwreg_t(w) cmpval = xwreg_zero_extend(old, w, sz);		\
 	unsigned long tmp;						\
 	u##sz oldval;							\
 									\
-	/*								\
-	 * Sub-word sizes require explicit casting so that the compare  \
-	 * part of the cmpxchg doesn't end up interpreting non-zero	\
-	 * upper bits of the register containing "old".			\
-	 */								\
-	if (sz < 32)							\
-		old = (u##sz)old;					\
-									\
 	asm volatile(							\
 	"	prfm	pstl1strm, %[v]\n"				\
 	"1:	ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"		\
@@ -264,7 +262,8 @@ __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\
 	"2:"								\
 	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),			\
 	  [v] "+Q" (*(u##sz *)ptr)					\
-	: [old] __stringify(constraint) "r" (old), [new] "r" (new)	\
+	: [old] __stringify(constraint) "r" (cmpval),			\
+	  [new] "r" (new)						\
 	: cl);								\
 									\
 	return oldval;							\
@@ -278,19 +277,19 @@ __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\
 __CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         , K)
 __CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         , K)
 __CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         , K)
-__CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         , L)
+__CMPXCHG_CASE(x,  ,     , 64,        ,  ,  ,         , L)
 __CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory", K)
 __CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory", K)
 __CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory", K)
-__CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory", L)
+__CMPXCHG_CASE(x,  , acq_, 64,        , a,  , "memory", L)
 __CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory", K)
 __CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory", K)
 __CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory", K)
-__CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory", L)
+__CMPXCHG_CASE(x,  , rel_, 64,        ,  , l, "memory", L)
 __CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory", K)
 __CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory", K)
 __CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory", K)
-__CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)
+__CMPXCHG_CASE(x,  ,  mb_, 64, dmb ish,  , l, "memory", L)
 
 #undef __CMPXCHG_CASE
 
diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h
index afad1849c4cf5b..d588af05653319 100644
--- a/arch/arm64/include/asm/atomic_lse.h
+++ b/arch/arm64/include/asm/atomic_lse.h
@@ -291,15 +291,13 @@ __lse__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new)		\
 	register unsigned long x1 asm ("x1") = o.high;			\
 	register unsigned long x2 asm ("x2") = n.low;			\
 	register unsigned long x3 asm ("x3") = n.high;			\
-	register unsigned long x4 asm ("x4") = (unsigned long)ptr;	\
 									\
 	asm volatile(							\
 	__LSE_PREAMBLE							\
 	"	casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
 	: [old1] "+&r" (x0), [old2] "+&r" (x1),				\
 	  [v] "+Q" (*(u128 *)ptr)					\
-	: [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4),		\
-	  [oldval1] "r" (o.low), [oldval2] "r" (o.high)			\
+	: [new1] "r" (x2), [new2] "r" (x3)				\
 	: cl);								\
 									\
 	r.low = x0; r.high = x1;					\
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h
index ae904a1ad5293b..5e4cc4d2f872dd 100644
--- a/arch/arm64/include/asm/compat.h
+++ b/arch/arm64/include/asm/compat.h
@@ -28,34 +28,20 @@ typedef u16		compat_ipc_pid_t;
 #include <linux/sched.h>
 #include <linux/sched/task_stack.h>
 
-#ifdef __AARCH64EB__
-#define COMPAT_UTS_MACHINE	"armv8b\0\0"
-#else
 #define COMPAT_UTS_MACHINE	"armv8l\0\0"
-#endif
 
 typedef u16		__compat_uid16_t;
 typedef u16		__compat_gid16_t;
 typedef s32		compat_nlink_t;
 
 struct compat_stat {
-#ifdef __AARCH64EB__
-	short		st_dev;
-	short		__pad1;
-#else
 	compat_dev_t	st_dev;
-#endif
 	compat_ino_t	st_ino;
 	compat_mode_t	st_mode;
 	compat_ushort_t	st_nlink;
 	__compat_uid16_t	st_uid;
 	__compat_gid16_t	st_gid;
-#ifdef __AARCH64EB__
-	short		st_rdev;
-	short		__pad2;
-#else
 	compat_dev_t	st_rdev;
-#endif
 	compat_off_t	st_size;
 	compat_off_t	st_blksize;
 	compat_off_t	st_blocks;
diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
index aa8ec9df802436..87560d8b254e67 100644
--- a/arch/arm64/include/asm/el2_setup.h
+++ b/arch/arm64/include/asm/el2_setup.h
@@ -418,6 +418,7 @@
 	b.lt	.Lskip_fgt2_\@
 
 	mov	x0, xzr
+	mov	x2, xzr
 	mrs	x1, id_aa64dfr0_el1
 	ubfx	x1, x1, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4
 	cmp	x1, #ID_AA64DFR0_EL1_PMUVer_V3P9
@@ -426,6 +427,11 @@
 	orr	x0, x0, #HDFGRTR2_EL2_nPMICNTR_EL0
 	orr	x0, x0, #HDFGRTR2_EL2_nPMICFILTR_EL0
 	orr	x0, x0, #HDFGRTR2_EL2_nPMUACR_EL1
+	orr	x2, x2, #HDFGWTR2_EL2_nPMICNTR_EL0
+	orr	x2, x2, #HDFGWTR2_EL2_nPMICFILTR_EL0
+	orr	x2, x2, #HDFGWTR2_EL2_nPMUACR_EL1
+	/* PMZR_EL0 is write-only, so it has no read trap to disable */
+	orr	x2, x2, #HDFGWTR2_EL2_nPMZR_EL0
 .Lskip_pmuv3p9_\@:
 	/* If SPE is implemented, */
 	__spe_vers_imp .Lskip_spefds_\@, ID_AA64DFR0_EL1_PMSVer_IMP, x1
@@ -436,10 +442,11 @@
 	cbz	x1, .Lskip_spefds_\@
 	/* disable traps of PMSDSFR to EL2. */
 	orr	x0, x0, #HDFGRTR2_EL2_nPMSDSFR_EL1
+	orr	x2, x2, #HDFGWTR2_EL2_nPMSDSFR_EL1
 
 .Lskip_spefds_\@:
 	msr_s   SYS_HDFGRTR2_EL2, x0
-	msr_s   SYS_HDFGWTR2_EL2, x0
+	msr_s   SYS_HDFGWTR2_EL2, x2
 	msr_s   SYS_HFGRTR2_EL2, xzr
 	msr_s   SYS_HFGWTR2_EL2, xzr
 	msr_s   SYS_HFGITR2_EL2, xzr
diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h
index d2779d604c7be2..5a0bd3d38b1a79 100644
--- a/arch/arm64/include/asm/elf.h
+++ b/arch/arm64/include/asm/elf.h
@@ -72,11 +72,7 @@
  * These are used to set parameters in the core dumps.
  */
 #define ELF_CLASS	ELFCLASS64
-#ifdef __AARCH64EB__
-#define ELF_DATA	ELFDATA2MSB
-#else
 #define ELF_DATA	ELFDATA2LSB
-#endif
 #define ELF_ARCH	EM_AARCH64
 
 /*
@@ -85,11 +81,7 @@
  * intent than poking at uname or /proc/cpuinfo.
  */
 #define ELF_PLATFORM_SIZE	16
-#ifdef __AARCH64EB__
-#define ELF_PLATFORM		("aarch64_be")
-#else
 #define ELF_PLATFORM		("aarch64")
-#endif
 
 /*
  * This is used to ensure we don't load something for the wrong architecture.
@@ -195,11 +187,7 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm,
 #define STACK_RND_MASK			(0x3ffff >> (PAGE_SHIFT - 12))
 #endif
 
-#ifdef __AARCH64EB__
-#define COMPAT_ELF_PLATFORM		("v8b")
-#else
 #define COMPAT_ELF_PLATFORM		("v8l")
-#endif
 
 /* AArch32 registers. */
 #define COMPAT_ELF_NGREG		18
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index f816f5d77f1a54..86756fc9bb5a24 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -437,6 +437,35 @@
 #ifndef __ASSEMBLER__
 #include <asm/types.h>
 
+static __always_inline bool esr_trap_is_iabt(unsigned long esr)
+{
+	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
+}
+
+/* Always check for S1PTW *before* using this. */
+static __always_inline bool esr_dabt_is_write(unsigned long esr)
+{
+	return esr & ESR_ELx_WNR;
+}
+
+static __always_inline bool esr_dabt_is_cm(unsigned long esr)
+{
+	return esr & ESR_ELx_CM;
+}
+
+static __always_inline bool esr_abt_is_sea(unsigned long esr)
+{
+	switch (esr & ESR_ELx_FSC) {
+	case ESR_ELx_FSC_EXTABT:
+	case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
+	case ESR_ELx_FSC_SECC:
+	case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
+		return true;
+	default:
+		return false;
+	}
+}
+
 static inline unsigned long esr_brk_comment(unsigned long esr)
 {
 	return esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
diff --git a/arch/arm64/include/asm/gpr-num.h b/arch/arm64/include/asm/gpr-num.h
index a114e4f8209b09..30fb8a67b19f00 100644
--- a/arch/arm64/include/asm/gpr-num.h
+++ b/arch/arm64/include/asm/gpr-num.h
@@ -11,6 +11,8 @@
 	.equ	.L__gpr_num_xzr, 31
 	.equ	.L__gpr_num_wzr, 31
 
+#define __GPR_NUM(gpr)		(.L__gpr_num_##gpr)
+
 #else /* __ASSEMBLER__ */
 
 #define __DEFINE_ASM_GPR_NUMS					\
@@ -21,6 +23,13 @@
 "	.equ	.L__gpr_num_xzr, 31\n"				\
 "	.equ	.L__gpr_num_wzr, 31\n"
 
+#define __DEFINE_ASM_GPR_ALIASES									\
+"	.irp	n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30\n"	\
+"	wx\\n	.req	w\\n\n"										\
+"	.endr\n"
+
+#define __GPR_NUM(gpr)		"(.L__gpr_num_" gpr ")"
+
 #endif /* __ASSEMBLER__ */
 
 #endif /* __ASM_GPR_NUM_H */
diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 49a7002661a928..076224734c539c 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -8,6 +8,7 @@
 #ifndef __ASM_IO_H
 #define __ASM_IO_H
 
+#include <linux/arm-rsi-cmds.h>
 #include <linux/types.h>
 #include <linux/pgtable.h>
 
@@ -17,7 +18,6 @@
 #include <asm/early_ioremap.h>
 #include <asm/alternative.h>
 #include <asm/cpufeature.h>
-#include <asm/rsi.h>
 
 /*
  * Generic IO read/write.  These perform native-endian accesses.
@@ -280,7 +280,8 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size,
 	pgprot_t prot;
 	ptval_t user_prot_val = pgprot_val(user_prot);
 
-	if (WARN_ON_ONCE(!(user_prot_val & PTE_USER)))
+	/* Reject PROT_NONE and exec-only */
+	if (!(user_prot_val & PTE_USER))
 		return NULL;
 
 	prot = __pgprot_modify(PAGE_KERNEL, PTE_ATTRINDX_MASK,
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index e5b92ac09e69e7..bffdeb36da1d21 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -87,6 +87,9 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___tracing_write_event,
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
 	__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident,
+	__KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend,
 	__KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr,
 	__KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr,
 
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index a3c1928bdf7437..708a4b6d882309 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -336,6 +336,16 @@ static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu)
 	return vcpu->arch.fault.esr_el2;
 }
 
+static __always_inline bool esr_abt_is_s1ptw(unsigned long esr)
+{
+	return esr & ESR_ELx_S1PTW;
+}
+
+static __always_inline bool esr_abt_is_exec_fault(unsigned long esr)
+{
+	return esr_trap_is_iabt(esr) && !esr_abt_is_s1ptw(esr);
+}
+
 static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu)
 {
 	u64 esr = kvm_vcpu_get_esr(vcpu);
@@ -411,18 +421,13 @@ static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
 
 static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
 {
-	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW);
+	return esr_abt_is_s1ptw(kvm_vcpu_get_esr(vcpu));
 }
 
 /* Always check for S1PTW *before* using this. */
 static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
 {
-	return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR;
-}
-
-static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
-{
-	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM);
+	return esr_dabt_is_write(kvm_vcpu_get_esr(vcpu));
 }
 
 static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
@@ -443,12 +448,7 @@ static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
 
 static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
 {
-	return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
-}
-
-static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
-{
-	return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
+	return esr_trap_is_iabt(kvm_vcpu_get_esr(vcpu));
 }
 
 static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
@@ -468,26 +468,9 @@ bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu)
 	return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu));
 }
 
-static inline
-u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu)
-{
-	unsigned long esr = kvm_vcpu_get_esr(vcpu);
-
-	BUG_ON(!esr_fsc_is_permission_fault(esr));
-	return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL));
-}
-
 static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu)
 {
-	switch (kvm_vcpu_trap_get_fault(vcpu)) {
-	case ESR_ELx_FSC_EXTABT:
-	case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
-	case ESR_ELx_FSC_SECC:
-	case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
-		return true;
-	default:
-		return false;
-	}
+	return esr_abt_is_sea(kvm_vcpu_get_esr(vcpu));
 }
 
 static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
@@ -496,9 +479,9 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
 	return ESR_ELx_SYS64_ISS_RT(esr);
 }
 
-static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
+static inline bool esr_abt_is_write_fault(unsigned long esr)
 {
-	if (kvm_vcpu_abt_iss1tw(vcpu)) {
+	if (esr_abt_is_s1ptw(esr)) {
 		/*
 		 * Only a permission fault on a S1PTW should be
 		 * considered as a write. Otherwise, page tables baked
@@ -511,13 +494,18 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
 		 * first), then a permission fault to allow the flags
 		 * to be set.
 		 */
-		return kvm_vcpu_trap_is_permission_fault(vcpu);
+		return esr_fsc_is_permission_fault(esr);
 	}
 
-	if (kvm_vcpu_trap_is_iabt(vcpu))
+	if (esr_trap_is_iabt(esr))
 		return false;
 
-	return kvm_vcpu_dabt_iswrite(vcpu);
+	return esr_dabt_is_write(esr);
+}
+
+static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
+{
+	return esr_abt_is_write_fault(kvm_vcpu_get_esr(vcpu));
 }
 
 static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h
new file mode 100644
index 00000000000000..29aad153a7c85d
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_hcall.h
@@ -0,0 +1,247 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * The host<->hyp hypercall interface.
+ *
+ * Copyright (C) 2026 Google LLC
+ * Author: Fuad Tabba <fuad.tabba@linux.dev>
+ */
+
+#ifndef __ARM64_KVM_HCALL_H__
+#define __ARM64_KVM_HCALL_H__
+
+#include <linux/args.h>
+#include <linux/arm-smccc.h>
+#include <linux/bug.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+
+#include <asm/barrier.h>
+#include <asm/kvm_asm.h>
+#include <asm/spectre.h>
+#include <asm/virt.h>
+
+typedef u16 pkvm_handle_t;
+
+struct kvm;
+struct kvm_s2_mmu;
+struct kvm_vcpu;
+struct vgic_v3_cpu_if;
+struct vgic_v5_cpu_if;
+
+/*
+ * A host VA carried by a hypercall argument. At EL2 such a pointer must not
+ * be dereferenced until it is translated with kern_hyp_va_host(); sparse
+ * flags any use that skips the translation. The tag describes the EL2 view
+ * only: the host dereferences its own VAs freely.
+ */
+#if defined(__KVM_NVHE_HYPERVISOR__) && defined(__CHECKER__)
+#define __kern	__attribute__((noderef, address_space(__kern)))
+#else
+#define __kern
+#endif
+
+/*
+ * Hypercall signatures are declared as (type, name) argument pairs.
+ * __KVM_HCALL_MAP() applies a macro to each pair, in the mold of __MAP()
+ * in <linux/syscalls.h>. The ladder is indexed by list entries, two per
+ * argument; __KVM_HCALL_MAP_N() takes that count explicitly.
+ */
+#define __KVM_HCALL_MAP2(m, t, a, ...) m(t, a)
+#define __KVM_HCALL_MAP4(m, t, a, ...) m(t, a), __KVM_HCALL_MAP2(m, __VA_ARGS__)
+#define __KVM_HCALL_MAP6(m, t, a, ...) m(t, a), __KVM_HCALL_MAP4(m, __VA_ARGS__)
+#define __KVM_HCALL_MAP8(m, t, a, ...) m(t, a), __KVM_HCALL_MAP6(m, __VA_ARGS__)
+#define __KVM_HCALL_MAP10(m, t, a, ...) m(t, a), __KVM_HCALL_MAP8(m, __VA_ARGS__)
+#define __KVM_HCALL_MAP12(m, t, a, ...) m(t, a), __KVM_HCALL_MAP10(m, __VA_ARGS__)
+#define __KVM_HCALL_MAP_N(n, m, ...) CONCATENATE(__KVM_HCALL_MAP, n)(m, __VA_ARGS__)
+#define __KVM_HCALL_MAP(m, ...) __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), m, __VA_ARGS__)
+
+#define __KVM_HCALL_DECL(t, a)	t a
+#define __KVM_HCALL_LONG(t, a)	unsigned long a
+#define __KVM_HCALL_CAST(t, a)	(__force t) a
+#define __KVM_HCALL_ARGS(t, a)	a
+
+#ifndef __KVM_NVHE_HYPERVISOR__
+#define __kvm_call_hyp_nvhe(f, ...)					\
+	({								\
+		struct arm_smccc_res res;				\
+									\
+		arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f),		\
+				  ##__VA_ARGS__, &res);			\
+		if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS))		\
+			res.a1 = -EOPNOTSUPP;				\
+									\
+		res.a1;							\
+	})
+
+/*
+ * Generate a typed stub for each declared hypercall. kvm_call_hyp_nvhe()
+ * resolves to the stub, so a call with the wrong argument count or types
+ * fails to compile instead of being silently truncated to an SMCCC function
+ * number and a pile of registers. The stub inlines to the same SMCCC call
+ * the untyped macro used to make.
+ */
+#define DECLARE_KVM_HOST_HCALL(ret, name, ...)				\
+	static __always_inline						\
+	ret nvhe_hvc_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) \
+	{								\
+		return (ret)__kvm_call_hyp_nvhe(name,			\
+			__KVM_HCALL_MAP(__KVM_HCALL_ARGS, __VA_ARGS__));\
+	}
+
+#define DECLARE_KVM_HOST_HCALL0(ret, name)				\
+	static __always_inline ret nvhe_hvc_##name(void)		\
+	{								\
+		return (ret)__kvm_call_hyp_nvhe(name);			\
+	}
+
+#define kvm_call_hyp_nvhe(f, ...)	nvhe_hvc_##f(__VA_ARGS__)
+
+/*
+ * The isb() below is there to guarantee the same behaviour on VHE as on !VHE,
+ * where the eret to EL1 acts as a context synchronization event.
+ */
+#define kvm_call_hyp(f, ...)						\
+	do {								\
+		if (has_vhe()) {					\
+			f(__VA_ARGS__);					\
+			isb();						\
+		} else {						\
+			kvm_call_hyp_nvhe(f, ##__VA_ARGS__);		\
+		}							\
+	} while (0)
+
+#define kvm_call_hyp_ret(f, ...)					\
+	({								\
+		typeof(f(__VA_ARGS__)) ret;				\
+									\
+		if (has_vhe()) {					\
+			ret = f(__VA_ARGS__);				\
+		} else {						\
+			ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__);	\
+		}							\
+									\
+		ret;							\
+	})
+#else /* __KVM_NVHE_HYPERVISOR__ */
+#define kvm_call_hyp(f, ...) f(__VA_ARGS__)
+#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__)
+#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__)
+
+/*
+ * At EL2 each declaration emits the canonical signature of the hypercall,
+ * which DEFINE_KVM_HOST_HCALL() in hyp-main.c checks the handler
+ * definition against.
+ */
+#define DECLARE_KVM_HOST_HCALL(ret, name, ...)				\
+	typedef ret kvm_host_hcall_sig_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__));
+#define DECLARE_KVM_HOST_HCALL0(ret, name)				\
+	typedef ret kvm_host_hcall_sig_##name(void);
+#endif /* __KVM_NVHE_HYPERVISOR__ */
+
+/* Hypercalls that are unavailable once pKVM has finalised. */
+DECLARE_KVM_HOST_HCALL(int, __pkvm_init,
+	phys_addr_t, phys, unsigned long, size,
+	unsigned long *, per_cpu_base, u32, hyp_va_bits)
+DECLARE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping,
+	phys_addr_t, phys, size_t, size, u64, prot)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector,
+	enum arm64_hyp_spectre_vector, slot)
+DECLARE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs)
+DECLARE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs)
+DECLARE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config)
+
+DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize)
+
+/* Hypercalls that are always available and common to [nh]VHE/pKVM. */
+DECLARE_KVM_HOST_HCALL(void, __kvm_adjust_pc,
+	struct kvm_vcpu __kern *, vcpu)
+DECLARE_KVM_HOST_HCALL(int, __kvm_vcpu_run,
+	struct kvm_vcpu __kern *, vcpu)
+DECLARE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context)
+DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa,
+	struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level)
+DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh,
+	struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level)
+DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid,
+	struct kvm_s2_mmu __kern *, mmu)
+DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range,
+	struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages)
+DECLARE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context,
+	struct kvm_s2_mmu __kern *, mmu)
+DECLARE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff,
+	u64, cntvoff)
+DECLARE_KVM_HOST_HCALL(int, __tracing_load,
+	void __kern *, desc_hva, size_t, desc_size)
+DECLARE_KVM_HOST_HCALL0(void, __tracing_unload)
+DECLARE_KVM_HOST_HCALL(int, __tracing_enable,
+	bool, enable)
+DECLARE_KVM_HOST_HCALL(int, __tracing_swap_reader,
+	unsigned int, cpu)
+DECLARE_KVM_HOST_HCALL(void, __tracing_update_clock,
+	u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc)
+DECLARE_KVM_HOST_HCALL(int, __tracing_reset,
+	unsigned int, cpu)
+DECLARE_KVM_HOST_HCALL(int, __tracing_enable_event,
+	unsigned short, id, bool, enable)
+DECLARE_KVM_HOST_HCALL(void, __tracing_write_event,
+	u64, id)
+DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_resident,
+		       struct vgic_v5_cpu_if __kern *, cpu_if);
+DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident,
+		       struct vgic_v5_cpu_if __kern *, cpu_if);
+DECLARE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs,
+	struct vgic_v3_cpu_if __kern *, cpu_if)
+DECLARE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs,
+	struct vgic_v3_cpu_if __kern *, cpu_if)
+DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr,
+	struct vgic_v5_cpu_if __kern *, cpu_if)
+DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr,
+	struct vgic_v5_cpu_if __kern *, cpu_if)
+DECLARE_KVM_HOST_HCALL(void, __vgic_v5_vdpend,
+		       u32, intid, bool, pending, u16, vm);
+
+/* Hypercalls that are available only when pKVM has finalised. */
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp,
+	u64, pfn)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp,
+	u64, pfn)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest,
+	u64, pfn, u64, gfn)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_guest,
+	u64, pfn, u64, gfn, u64, nr_pages, u64, prot)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest,
+	pkvm_handle_t, handle, u64, gfn, u64, nr_pages)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest,
+	u64, gfn, u64, prot)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest,
+	pkvm_handle_t, handle, u64, gfn, u64, nr_pages)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest,
+	pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest,
+	u64, gfn)
+DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm)
+DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm,
+	pkvm_handle_t, handle)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm,
+	struct kvm __kern *, host_kvm, void __kern *, vm_hva,
+	void __kern *, pgd_hva)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu,
+	pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu,
+	void __kern *, vcpu_hva)
+DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page,
+	phys_addr_t, phys)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page,
+	pkvm_handle_t, handle, u64, gfn)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm,
+	pkvm_handle_t, handle)
+DECLARE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm,
+	pkvm_handle_t, handle)
+DECLARE_KVM_HOST_HCALL(void, __pkvm_vcpu_load,
+	pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2)
+DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put)
+DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state)
+DECLARE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid,
+	pkvm_handle_t, handle)
+
+#endif /* __ARM64_KVM_HCALL_H__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 27fe0cd5b2d7a1..3ec41444257860 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -14,6 +14,7 @@
 #include <linux/arm-smccc.h>
 #include <linux/bitmap.h>
 #include <linux/types.h>
+#include <linux/interval_tree.h>
 #include <linux/jump_label.h>
 #include <linux/kvm_types.h>
 #include <linux/maple_tree.h>
@@ -27,6 +28,7 @@
 #include <asm/fpsimd.h>
 #include <asm/kvm.h>
 #include <asm/kvm_asm.h>
+#include <asm/kvm_hcall.h>
 #include <asm/vncr_mapping.h>
 
 #define __KVM_HAVE_ARCH_INTC_INITIALIZED
@@ -55,6 +57,7 @@
 #define KVM_REQ_GUEST_HYP_IRQ_PENDING	KVM_ARCH_REQ(9)
 #define KVM_REQ_MAP_L1_VNCR_EL2		KVM_ARCH_REQ(10)
 #define KVM_REQ_VGIC_PROCESS_UPDATE	KVM_ARCH_REQ(11)
+#define KVM_REQ_RELOAD_GICv5		KVM_ARCH_REQ(12)
 
 #define KVM_DIRTY_LOG_MANUAL_CAPS   (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
 				     KVM_DIRTY_LOG_INITIALLY_SET)
@@ -150,6 +153,16 @@ struct kvm_vmid {
 	atomic64_t id;
 };
 
+/*
+ * Record of a guest stage-2 mapping, storing canonical and nested IPA
+ * ranges. Both ranges have the same size. The lower bits of nested.start
+ * store the index of the nested mmu this mapping belongs to.
+ */
+struct kvm_guest_s2_mapping {
+	struct interval_tree_node canonical;
+	struct interval_tree_node nested;
+};
+
 struct kvm_s2_mmu {
 	struct kvm_vmid vmid;
 
@@ -211,22 +224,24 @@ struct kvm_s2_mmu {
 	u64	tlb_vttbr;
 	u64	tlb_vtcr;
 
+	/* Guest s2 mapping records indexed in this MMU's IPA space. */
+	struct rb_root_cached guest_s2_mappings;
+
 	/*
 	 * true when this represents a nested context where virtual
 	 * HCR_EL2.VM == 1
 	 */
 	bool	nested_stage2_enabled;
 
-#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
-	struct dentry *shadow_pt_debugfs_dentry;
-#endif
-
 	/*
 	 * true when this MMU needs to be unmapped before being used for a new
 	 * purpose.
 	 */
 	bool	pending_unmap;
 
+	/* Index in the S2 MMU array, only valid for a shadow S2 */
+	u16	s2_mmu_idx;
+
 	/*
 	 *  0: Nobody is currently using this, check vttbr for validity
 	 * >0: Somebody is actively using this.
@@ -251,8 +266,6 @@ struct kvm_smccc_features {
 	unsigned long vendor_hyp_bmap_2; /* Function numbers 64-127 */
 };
 
-typedef u16 pkvm_handle_t;
-
 struct kvm_protected_vm {
 	pkvm_handle_t handle;
 	struct kvm_hyp_memcache teardown_mc;
@@ -322,10 +335,16 @@ struct kvm_arch {
 	 * Stage 2 paging state for VMs with nested S2 using a virtual
 	 * VMID.
 	 */
-	struct kvm_s2_mmu *nested_mmus;
+	struct kvm_s2_mmu **nested_mmus;
 	size_t nested_mmus_size;
 	int nested_mmus_next;
 
+	/*
+	 * Serializes guest s2 tracking trees access when the mmu_lock
+	 * is only held for read.
+	 */
+	spinlock_t guest_s2_tracking_lock;
+
 	/* Interrupt controller */
 	struct vgic_dist	vgic;
 
@@ -367,6 +386,8 @@ struct kvm_arch {
 #define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS		10
 	/* Unhandled SEAs are taken to userspace */
 #define KVM_ARCH_FLAG_EXIT_SEA				11
+	/* No further ID register changes possible */
+#define KVM_ARCH_FLAG_ID_REGS_FINAL			12
 	unsigned long flags;
 
 	/* VM-wide vCPU feature set */
@@ -1149,6 +1170,9 @@ struct kvm_vcpu_arch {
 #define vcpu_has_ptrauth(vcpu)		false
 #endif
 
+#define kvm_id_regs_final(kvm)						\
+	test_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &(kvm)->arch.flags)
+
 #define vcpu_on_unsupported_cpu(vcpu)					\
 	vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU)
 
@@ -1258,51 +1282,6 @@ void kvm_arm_resume_guest(struct kvm *kvm);
 
 #define vcpu_has_run_once(vcpu)	(!!READ_ONCE((vcpu)->pid))
 
-#ifndef __KVM_NVHE_HYPERVISOR__
-#define kvm_call_hyp_nvhe(f, ...)					\
-	({								\
-		struct arm_smccc_res res;				\
-									\
-		arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f),		\
-				  ##__VA_ARGS__, &res);			\
-		if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS))		\
-			res.a1 = -EOPNOTSUPP;				\
-									\
-		res.a1;							\
-	})
-
-/*
- * The isb() below is there to guarantee the same behaviour on VHE as on !VHE,
- * where the eret to EL1 acts as a context synchronization event.
- */
-#define kvm_call_hyp(f, ...)						\
-	do {								\
-		if (has_vhe()) {					\
-			f(__VA_ARGS__);					\
-			isb();						\
-		} else {						\
-			kvm_call_hyp_nvhe(f, ##__VA_ARGS__);		\
-		}							\
-	} while(0)
-
-#define kvm_call_hyp_ret(f, ...)					\
-	({								\
-		typeof(f(__VA_ARGS__)) ret;				\
-									\
-		if (has_vhe()) {					\
-			ret = f(__VA_ARGS__);				\
-		} else {						\
-			ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__);	\
-		}							\
-									\
-		ret;							\
-	})
-#else /* __KVM_NVHE_HYPERVISOR__ */
-#define kvm_call_hyp(f, ...) f(__VA_ARGS__)
-#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__)
-#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__)
-#endif /* __KVM_NVHE_HYPERVISOR__ */
-
 int handle_exit(struct kvm_vcpu *vcpu, int exception_index);
 void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index);
 
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 4974492744cc8d..8e2f0280c0cc58 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -88,6 +88,8 @@ void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if);
 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);
 
 /* GICv5 */
+void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if);
+void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if);
 void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if);
 void __vgic_v5_restore_vmcr_apr(struct vgic_v5_cpu_if *cpu_if);
 /* No hypercalls for the following */
@@ -95,6 +97,7 @@ void __vgic_v5_save_ppi_state(struct vgic_v5_cpu_if *cpu_if);
 void __vgic_v5_restore_ppi_state(struct vgic_v5_cpu_if *cpu_if);
 void __vgic_v5_save_state(struct vgic_v5_cpu_if *cpu_if);
 void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if);
+void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm);
 
 #ifdef __KVM_NVHE_HYPERVISOR__
 void __timer_enable_traps(struct kvm_vcpu *vcpu);
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 6eae7e7e2a6845..49ecc863e131de 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -7,6 +7,8 @@
 #ifndef __ARM64_KVM_MMU_H__
 #define __ARM64_KVM_MMU_H__
 
+#include <linux/compiler.h>
+
 #include <asm/page.h>
 #include <asm/memory.h>
 #include <asm/mmu.h>
@@ -140,6 +142,14 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v)
 
 #define kern_hyp_va(v) 	((typeof(v))(__kern_hyp_va((unsigned long)(v))))
 
+/*
+ * Translate a __kern-tagged host VA, dropping the tag: the only sanctioned
+ * unwrap. Translation only, no ownership or bounds validation; the result
+ * carries the pointee type stripped of the tag and of any cv-qualifiers.
+ */
+#define kern_hyp_va_host(v)						\
+	((TYPEOF_UNQUAL(*(v)) *)__kern_hyp_va((unsigned long)(__force void *)(v)))
+
 extern u32 __hyp_va_bits;
 
 /*
@@ -392,12 +402,10 @@ static inline bool kvm_supports_cacheable_pfnmap(void)
 
 #ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
 void kvm_s2_ptdump_create_debugfs(struct kvm *kvm);
-void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu);
-void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu);
+void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx);
 #else
 static inline void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) {}
-static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) {}
-static inline void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) {}
+static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) {}
 #endif /* CONFIG_PTDUMP_STAGE2_DEBUGFS */
 
 #endif /* __ASSEMBLER__ */
diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h
index 1ed7083358096d..0f899addafa8b4 100644
--- a/arch/arm64/include/asm/kvm_nested.h
+++ b/arch/arm64/include/asm/kvm_nested.h
@@ -66,7 +66,8 @@ static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)
 
 extern bool forward_smc_trap(struct kvm_vcpu *vcpu);
 extern bool forward_debug_exception(struct kvm_vcpu *vcpu);
-extern void kvm_init_nested(struct kvm *kvm);
+extern int kvm_init_nested(struct kvm *kvm);
+extern void kvm_destroy_nested(struct kvm *kvm);
 extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu);
 extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);
 extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu);
@@ -77,6 +78,11 @@ extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
 				       const union tlbi_info *info,
 				       void (*)(struct kvm_s2_mmu *,
 						const union tlbi_info *));
+extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa,
+					gpa_t nested_ipa, size_t map_size,
+					struct kvm_guest_s2_mapping *mapping);
+extern void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu,
+					 gpa_t nipa, size_t size);
 extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
 extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
 
@@ -165,6 +171,8 @@ extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
 				    struct kvm_s2_trans *trans);
 extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);
 extern void kvm_nested_s2_wp(struct kvm *kvm);
+extern void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa,
+					size_t unmap_size, bool may_block);
 extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
 extern void kvm_nested_s2_flush(struct kvm *kvm);
 
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 41a8687938eb6b..589db1d6405fa1 100644
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -532,7 +532,7 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
  * kvm_get_vtcr() - Helper to construct VTCR_EL2
  * @mmfr0:	Sanitized value of SYS_ID_AA64MMFR0_EL1 register.
  * @mmfr1:	Sanitized value of SYS_ID_AA64MMFR1_EL1 register.
- * @phys_shfit:	Value to set in VTCR_EL2.T0SZ.
+ * @phys_shift:	IPA size, in bits.
  *
  * The VTCR value is common across all the physical CPUs on the system.
  * We use system wide sanitised values to fill in different fields,
@@ -859,6 +859,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
  * @addr:	Input address for the start of the walk.
  * @ptep:	Pointer to storage for the retrieved PTE.
  * @level:	Pointer to storage for the level of the retrieved PTE.
+ * @flags:	Flags to control the page-table walk
+ *		(see struct kvm_pgtable_visit_ctx).
  *
  * The offset of @addr within a page is ignored.
  *
@@ -869,7 +871,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
  * Return: 0 on success, negative error code on failure.
  */
 int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
-			 kvm_pte_t *ptep, s8 *level);
+			 kvm_pte_t *ptep, s8 *level,
+			 enum kvm_pgtable_walk_flags flags);
 
 /**
  * kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a
diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h
index beea00e693a0a8..c1831c8e421d43 100644
--- a/arch/arm64/include/asm/kvm_pkvm.h
+++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -44,9 +44,9 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext)
 	case KVM_CAP_ARM_PTRAUTH_GENERIC:
 		return true;
 	case KVM_CAP_ARM_MTE:
-		return false;
 	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
 	case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
+	case KVM_CAP_PRE_FAULT_MEMORY:
 		return false;
 	default:
 		return !kvm || !kvm_vm_is_protected(kvm);
@@ -65,8 +65,7 @@ static inline bool kvm_pkvm_ioctl_allowed(struct kvm *kvm, unsigned int ioctl)
 	int r;
 
 	r = kvm_get_cap_for_kvm_ioctl(ioctl, &ext);
-
-	if (WARN_ON_ONCE(r < 0))
+	if (r < 0)
 		return false;
 
 	return kvm_pkvm_ext_allowed(kvm, ext);
diff --git a/arch/arm64/include/asm/mem_encrypt.h b/arch/arm64/include/asm/mem_encrypt.h
index 636f45b4d8afc7..ef8b8463e52b19 100644
--- a/arch/arm64/include/asm/mem_encrypt.h
+++ b/arch/arm64/include/asm/mem_encrypt.h
@@ -2,8 +2,8 @@
 #ifndef __ASM_MEM_ENCRYPT_H
 #define __ASM_MEM_ENCRYPT_H
 
+#include <linux/arm-rsi-cmds.h>
 #include <asm/hypervisor.h>
-#include <asm/rsi.h>
 
 struct device;
 
@@ -16,8 +16,7 @@ int arm64_mem_crypt_ops_register(const struct arm64_mem_crypt_ops *ops);
 
 int set_memory_encrypted(unsigned long addr, int numpages);
 int set_memory_decrypted(unsigned long addr, int numpages);
-
-int realm_register_memory_enc_ops(void);
+int __set_memory_enc_dec(unsigned long addr, int numpages, bool encrypt);
 
 static inline bool force_dma_unencrypted(struct device *dev)
 {
diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h
index 35213927191892..de910f1c19e1b2 100644
--- a/arch/arm64/include/asm/mte-kasan.h
+++ b/arch/arm64/include/asm/mte-kasan.h
@@ -53,14 +53,16 @@ static inline bool system_uses_mte_async_or_asymm_mode(void)
  */
 static inline void mte_disable_tco(void)
 {
-	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0),
-				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
+	if (!IS_ENABLED(CONFIG_KASAN_HW_TAGS))
+		return;
+	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(0), ARM64_MTE));
 }
 
 static inline void mte_enable_tco(void)
 {
-	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1),
-				 ARM64_MTE, CONFIG_KASAN_HW_TAGS));
+	if (!IS_ENABLED(CONFIG_KASAN_HW_TAGS))
+		return;
+	asm volatile(ALTERNATIVE("nop", SET_PSTATE_TCO(1), ARM64_MTE));
 }
 
 /*
diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index 7f7b97e099968e..83f3b05fc78490 100644
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -23,9 +23,7 @@ unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
 				      unsigned long n);
 unsigned long mte_copy_tags_to_user(void __user *to, void *from,
 				    unsigned long n);
-int mte_save_tags(struct page *page);
 void mte_save_page_tags(const void *page_addr, void *tag_storage);
-void mte_restore_tags(swp_entry_t entry, struct page *page);
 void mte_restore_page_tags(void *page_addr, const void *tag_storage);
 void mte_invalidate_tags(int type, pgoff_t offset);
 void mte_invalidate_tags_area(int type);
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 8cf4068ce1b562..3cc316091d6655 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -5,10 +5,13 @@
 #ifndef __ASM_PERCPU_H
 #define __ASM_PERCPU_H
 
+#include <linux/bits.h>
 #include <linux/preempt.h>
+#include <linux/stringify.h>
 
 #include <asm/alternative.h>
 #include <asm/cmpxchg.h>
+#include <asm/gpr-num.h>
 #include <asm/stack_pointer.h>
 #include <asm/sysreg.h>
 
@@ -29,6 +32,11 @@ static inline unsigned long __hyp_my_cpu_offset(void)
 	return read_sysreg(tpidr_el2);
 }
 
+#define __KERN_ASM_CPU_OFFSET(dst)					\
+	ALTERNATIVE("mrs " dst ", tpidr_el1",				\
+		    "mrs " dst ", tpidr_el2",				\
+		       ARM64_HAS_VIRT_HOST_EXTN)
+
 static inline unsigned long __kern_my_cpu_offset(void)
 {
 	unsigned long off;
@@ -37,73 +45,192 @@ static inline unsigned long __kern_my_cpu_offset(void)
 	 * We want to allow caching the value, so avoid using volatile and
 	 * instead use a fake stack read to hazard against barrier().
 	 */
-	asm(ALTERNATIVE("mrs %0, tpidr_el1",
-			"mrs %0, tpidr_el2",
-			ARM64_HAS_VIRT_HOST_EXTN)
-		: "=r" (off) :
-		"Q" (*(const unsigned long *)current_stack_pointer));
+	asm(
+	__KERN_ASM_CPU_OFFSET("%0")
+	: "=r" (off)
+	: "Q" (*(const unsigned long *)current_stack_pointer)
+	);
 
 	return off;
 }
 
+#define PCPU_GPR_PCP_SHIFT		0
+#define PCPU_GPR_PCP			GENMASK(4, 0)
+#define PCPU_GPR_OFF_SHIFT		5
+#define PCPU_GPR_OFF			GENMASK(9, 5)
+#define PCPU_GPR_ADDR_SHIFT		10
+#define PCPU_GPR_ADDR			GENMASK(14, 10)
+
+#define __VAL_PCPU_GPRS(pcp, off, addr)							\
+	"("										\
+		"(" __GPR_NUM(pcp)  " << " __stringify(PCPU_GPR_PCP_SHIFT) ") | "	\
+		"(" __GPR_NUM(off)  " << " __stringify(PCPU_GPR_OFF_SHIFT) ") | "	\
+		"(" __GPR_NUM(addr) " << " __stringify(PCPU_GPR_ADDR_SHIFT) ")"		\
+	")"
+
+#define __ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr)			\
+	".if ("								\
+		"(" __GPR_NUM(pcp) " == " __GPR_NUM(off) ") || "	\
+		"(" __GPR_NUM(pcp) " == " __GPR_NUM(addr) ") || "	\
+		"(" __GPR_NUM(off) " == " __GPR_NUM(addr) ")"		\
+	"    )\n"							\
+	".error \"PCPU GPRS overlap: {" pcp "," off "," addr "}\"\n"	\
+	".endif\n"
+
+#define ____PCPU_GPRS_BEGIN(gprs, pcp, off, addr)			\
+	"// ____PCPU_GPRS_BEGIN(" gprs ", " pcp ", " off ", " addr")\n"	\
+	__DEFINE_ASM_GPR_NUMS						\
+	__DEFINE_ASM_GPR_ALIASES					\
+	__ASSERT_PCPU_GPRS_DISTINCT(pcp, off, addr)			\
+	"	mov w" off ", #" __VAL_PCPU_GPRS(pcp, off, addr) "\n"	\
+	"	strh	w" off ", " gprs "\n"				\
+	__KERN_ASM_CPU_OFFSET(off) "\n"
+
+/*
+ * Begin a PCPU GPR critical section which requires <addr> (and <off>).
+ *
+ * At the start of the critical section, and upon any (preemptible) exception
+ * until __PCPU_GPRS_END():
+ * - <off>  will be set to the current CPU's percpu offset.
+ * - <addr> will be set to <pcp> + <off>.
+ *
+ * The <pcp>, <off>, and <addr> registers must be distinct GPRs.
+ *
+ * <gprs> must be '&current_thread_info()->pcpu_gprs', as a memory operand
+ * which can be written both at the start and end of the critical section
+ * (e.g. using "=Qo" constraints).
+ */
+#define __PCPU_GPRS_BEGIN(gprs, pcp, off, addr)				\
+	____PCPU_GPRS_BEGIN(gprs, pcp, off, addr)			\
+	"	add	" addr ", " pcp ", " off "\n"
+
+/*
+ * Begin a PCPU GPR critical section which only requires <off> and does not
+ * require <addr>.
+ *
+ * This is only for operations that can use register-offset addressing,
+ * e.g. STR <Xt>, [<Xn>, <Xm>].
+ *
+ * All other details are the same as __PCPU_GPRS_BEGIN().
+ */
+#define __PCPU_GPRS_BEGIN_OFFSET(gprs, pcp, off)			\
+	____PCPU_GPRS_BEGIN(gprs, pcp, off, "xzr")
+
+/*
+ * End a PCPU GPR critical section.
+ */
+#define __PCPU_GPRS_END(gprs)						\
+	"	strh	wzr, " gprs "\n"
+
 #ifdef __KVM_NVHE_HYPERVISOR__
 #define __my_cpu_offset __hyp_my_cpu_offset()
 #else
 #define __my_cpu_offset __kern_my_cpu_offset()
 #endif
 
-#define PERCPU_RW_OPS(sz)						\
-static inline unsigned long __percpu_read_##sz(void *ptr)		\
-{									\
-	return READ_ONCE(*(u##sz *)ptr);				\
-}									\
-									\
-static inline void __percpu_write_##sz(void *ptr, unsigned long val)	\
-{									\
-	WRITE_ONCE(*(u##sz *)ptr, (u##sz)val);				\
+#define PERCPU_RW_OPS(w, sfx, sz)						\
+static inline unsigned long __percpu_read_##sz(void __percpu *pcp)		\
+{										\
+	u16 *gprs = &current_thread_info()->pcpu_gprs;				\
+	unsigned long off;							\
+	unsigned long val;							\
+										\
+	asm volatile(								\
+	__PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]")			\
+	"	ldr" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n"			\
+	__PCPU_GPRS_END("%[gprs]")						\
+	: [gprs] "=Qo" (*gprs),							\
+	  [off] "=&r" (off),							\
+	  [val] "=&r" (val)							\
+	: [pcp] "r" (pcp)							\
+	: "memory"								\
+	);									\
+										\
+	return val;								\
+}										\
+										\
+static inline void __percpu_write_##sz(void __percpu *pcp, unsigned long val)	\
+{										\
+	u16 *gprs = &current_thread_info()->pcpu_gprs;				\
+	unsigned long off;							\
+										\
+	asm volatile(								\
+	__PCPU_GPRS_BEGIN_OFFSET("%[gprs]", "%[pcp]", "%[off]")			\
+	"	str" #sfx "\t%" #w "[val], [%[pcp], %[off]]\n"			\
+	__PCPU_GPRS_END("%[gprs]")						\
+	: [gprs] "=Qo" (*gprs),							\
+	  [off] "=&r" (off)							\
+	: [pcp] "r" (pcp),							\
+	  [val] "r" ((u##sz)val)						\
+	: "memory"								\
+	);									\
 }
 
 #define __PERCPU_OP_CASE(w, sfx, name, sz, op_llsc, op_lse)		\
 static inline void							\
-__percpu_##name##_case_##sz(void *ptr, unsigned long val)		\
+__percpu_##name##_case_##sz(void __percpu *pcp, unsigned long val)	\
 {									\
+	u16 *gprs = &current_thread_info()->pcpu_gprs;			\
+	unsigned long addr;						\
+	unsigned long off;						\
 	unsigned int loop;						\
 	u##sz tmp;							\
 									\
-	asm volatile (ARM64_LSE_ATOMIC_INSN(				\
+	asm volatile (							\
+	__PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")	\
+	ARM64_LSE_ATOMIC_INSN(						\
 	/* LL/SC */							\
-	"1:	ldxr" #sfx "\t%" #w "[tmp], %[ptr]\n"			\
+	"1:	ldxr" #sfx "\t%" #w "[tmp], [%[addr]]\n"		\
 		#op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n"	\
-	"	stxr" #sfx "\t%w[loop], %" #w "[tmp], %[ptr]\n"		\
+	"	stxr" #sfx "\t%w[loop], %" #w "[tmp], [%[addr]]\n"	\
 	"	cbnz	%w[loop], 1b",					\
 	/* LSE atomics */						\
-		#op_lse #sfx "\t%" #w "[val], %" #w "[tmp], %[ptr]\n"	\
+		#op_lse #sfx "\t%" #w "[val], %" #w "[tmp], [%[addr]]\n"\
 		__nops(3))						\
-	: [loop] "=&r" (loop), [tmp] "=&r" (tmp),			\
-	  [ptr] "+Q"(*(u##sz *)ptr)					\
-	: [val] "r" ((u##sz)(val)));					\
+	__PCPU_GPRS_END("%[gprs]")					\
+	: [gprs] "=Qo" (*gprs),						\
+	  [addr] "=&r" (addr),						\
+	  [off] "=&r" (off),						\
+	  [loop] "=&r" (loop),						\
+	  [tmp] "=&r" (tmp)						\
+	: [pcp] "r" (pcp),						\
+	  [val] "r" ((u##sz)(val))					\
+	: "memory"							\
+	);								\
 }
 
 #define __PERCPU_RET_OP_CASE(w, sfx, name, sz, op_llsc, op_lse)		\
 static inline u##sz							\
-__percpu_##name##_return_case_##sz(void *ptr, unsigned long val)	\
+__percpu_##name##_return_case_##sz(void __percpu *pcp, unsigned long val)	\
 {									\
+	u16 *gprs = &current_thread_info()->pcpu_gprs;			\
+	unsigned long addr;						\
+	unsigned long off;						\
 	unsigned int loop;						\
 	u##sz ret;							\
 									\
-	asm volatile (ARM64_LSE_ATOMIC_INSN(				\
+	asm volatile (							\
+	__PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")	\
+	ARM64_LSE_ATOMIC_INSN(						\
 	/* LL/SC */							\
-	"1:	ldxr" #sfx "\t%" #w "[ret], %[ptr]\n"			\
+	"1:	ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n"		\
 		#op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n"	\
-	"	stxr" #sfx "\t%w[loop], %" #w "[ret], %[ptr]\n"		\
+	"	stxr" #sfx "\t%w[loop], %" #w "[ret], [%[addr]]\n"	\
 	"	cbnz	%w[loop], 1b",					\
 	/* LSE atomics */						\
-		#op_lse #sfx "\t%" #w "[val], %" #w "[ret], %[ptr]\n"	\
+		#op_lse #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n"\
 		#op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n"	\
 		__nops(2))						\
-	: [loop] "=&r" (loop), [ret] "=&r" (ret),			\
-	  [ptr] "+Q"(*(u##sz *)ptr)					\
-	: [val] "r" ((u##sz)(val)));					\
+	__PCPU_GPRS_END("%[gprs]")					\
+	: [gprs] "=Qo" (*gprs),						\
+	  [addr] "=&r" (addr),						\
+	  [off] "=&r" (off),						\
+	  [loop] "=&r" (loop),						\
+	  [ret] "=&r" (ret)						\
+	: [pcp] "r" (pcp),						\
+	  [val] "r" ((u##sz)(val))					\
+	: "memory"							\
+	);								\
 									\
 	return ret;							\
 }
@@ -120,10 +247,10 @@ __percpu_##name##_return_case_##sz(void *ptr, unsigned long val)	\
 	__PERCPU_RET_OP_CASE(w,  , name, 32, op_llsc, op_lse)		\
 	__PERCPU_RET_OP_CASE( ,  , name, 64, op_llsc, op_lse)
 
-PERCPU_RW_OPS(8)
-PERCPU_RW_OPS(16)
-PERCPU_RW_OPS(32)
-PERCPU_RW_OPS(64)
+PERCPU_RW_OPS(w, b, 8)
+PERCPU_RW_OPS(w, h, 16)
+PERCPU_RW_OPS(w,  , 32)
+PERCPU_RW_OPS( ,  , 64)
 
 /*
  * Use value-returning atomics for CPU-local ops as they are more likely
@@ -142,120 +269,257 @@ PERCPU_RET_OP(add, add, ldadd)
 #undef PERCPU_OP
 #undef PERCPU_RET_OP
 
-/*
- * It would be nice to avoid the conditional call into the scheduler when
- * re-enabling preemption for preemptible kernels, but doing that in a way
- * which builds inside a module would mean messing directly with the preempt
- * count. If you do this, peterz and tglx will hunt you down.
- *
- * Not to mention it'll break the actual preemption model for missing a
- * preemption point when TIF_NEED_RESCHED gets set while preemption is
- * disabled.
- */
+#define PERCPU_XCHG_OP(w, sfx, sz)					\
+static inline unsigned long						\
+__percpu_xchg_case_##sz(void __percpu *pcp, u##sz val)			\
+{									\
+	u16 *gprs = &current_thread_info()->pcpu_gprs;			\
+	unsigned long addr;						\
+	unsigned long off;						\
+	unsigned int loop;						\
+	unsigned long ret;						\
+									\
+	asm volatile (							\
+	__PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")	\
+	ARM64_LSE_ATOMIC_INSN(						\
+	/* LL/SC */							\
+	"	prfm	pstl1strm, [%[addr]]\n"				\
+	"1:	ldxr" #sfx "\t%" #w "[ret], [%[addr]]\n"		\
+	"	stxr" #sfx "\t%w[loop], %" #w "[val], [%[addr]]\n"	\
+	"	cbnz	%w[loop], 1b\n"					\
+	,								\
+	/* LSE atomics */						\
+	"	swp" #sfx "\t%" #w "[val], %" #w "[ret], [%[addr]]\n"	\
+		__nops(3)						\
+	)								\
+	__PCPU_GPRS_END("%[gprs]")					\
+	: [gprs] "=Qo" (*gprs),						\
+	  [addr] "=&r" (addr),						\
+	  [off] "=&r" (off),						\
+	  [loop] "=&r" (loop),						\
+	  [ret] "=&r" (ret)						\
+	: [pcp] "r" (pcp),						\
+	  [val] "r" (val)						\
+	: "memory"							\
+	);								\
+									\
+	return ret;							\
+}
+
+#define PERCPU_CMPXCHG_OP(w, sfx, sz)					\
+static inline unsigned long						\
+__percpu_cmpxchg_case_##sz(void __percpu *pcp,				\
+			   u##sz old,					\
+			   u##sz new)					\
+{									\
+	/*                                                              \
+	 * Sub-word sizes require zero extension so that EOR+CBNZ don't	\
+	 * consume non-zero upper bits of the register containing "old".\
+	 */								\
+	xwreg_t(w) cmpval = xwreg_zero_extend(old, w, sz);		\
+	u16 *gprs = &current_thread_info()->pcpu_gprs;			\
+	unsigned long addr;						\
+	unsigned long off;						\
+	unsigned long tmp;						\
+	unsigned long oldval;						\
+									\
+	asm volatile (							\
+	__PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")	\
+	ARM64_LSE_ATOMIC_INSN(						\
+	/* LL/SC */							\
+	"	prfm	pstl1strm, [%[addr]]\n"				\
+	"1:	ldxr" #sfx "\t%" #w "[oldval], [%[addr]]\n"		\
+	"	eor	%" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"	\
+	"	cbnz	%" #w "[tmp], 2f\n"				\
+	"	stxr" #sfx "\t%w[tmp], %" #w "[new], [%[addr]]\n"	\
+	"	cbnz	%w[tmp], 1b\n"					\
+	"2:\n"								\
+	,								\
+	/* LSE atomics */						\
+	"	mov	%" #w "[oldval], %" #w "[old]\n"		\
+	"	cas" #sfx "\t%" #w "[oldval], %" #w "[new], [%[addr]]\n"\
+		__nops(4)						\
+	)								\
+	__PCPU_GPRS_END("%[gprs]")					\
+	: [gprs] "=Qo" (*gprs),						\
+	  [addr] "=&r" (addr),						\
+	  [off] "=&r" (off),						\
+	  [tmp] "=&r" (tmp),						\
+	  [oldval] "=&r" (oldval)					\
+	: [pcp] "r" (pcp),						\
+	  [old] "r" (cmpval),						\
+	  [new] "r" (new)						\
+	: "memory"							\
+	);								\
+									\
+	return oldval;							\
+}
+
+PERCPU_XCHG_OP(w, b, 8)
+PERCPU_XCHG_OP(w, h, 16)
+PERCPU_XCHG_OP(w,  , 32)
+PERCPU_XCHG_OP(x,  , 64)
+
+PERCPU_CMPXCHG_OP(w, b, 8)
+PERCPU_CMPXCHG_OP(w, h, 16)
+PERCPU_CMPXCHG_OP(w,  , 32)
+PERCPU_CMPXCHG_OP(x,  , 64)
 
-#define _pcp_protect(op, pcp, ...)					\
+#undef PERCPU_XCHG_OP
+#undef PERCPU_CMPXCHG_OP
+
+#define _pcp_wrap(op, pcp, ...)						\
+({									\
+	op(&(pcp), __VA_ARGS__);					\
+})
+
+#define _pcp_wrap_return(op, pcp, args...)				\
+({									\
+	(typeof(pcp))op(&(pcp), ##args);				\
+})
+
+#define _pcp_wrap_xchg(op, pcp, val)					\
 ({									\
-	preempt_disable_notrace();					\
-	op(raw_cpu_ptr(&(pcp)), __VA_ARGS__);				\
-	preempt_enable_notrace();					\
+	(typeof(pcp))op(&(pcp), (unsigned long)(val));			\
 })
 
-#define _pcp_protect_return(op, pcp, args...)				\
+#define _pcp_wrap_cmpxchg(op, pcp, old, new)				\
 ({									\
-	typeof(pcp) __retval;						\
-	preempt_disable_notrace();					\
-	__retval = (typeof(pcp))op(raw_cpu_ptr(&(pcp)), ##args);	\
-	preempt_enable_notrace();					\
-	__retval;							\
+	(typeof(pcp))op(&(pcp), (unsigned long)(old),			\
+			(unsigned long)(new));				\
 })
 
 #define this_cpu_read_1(pcp)		\
-	_pcp_protect_return(__percpu_read_8, pcp)
+	_pcp_wrap_return(__percpu_read_8, pcp)
 #define this_cpu_read_2(pcp)		\
-	_pcp_protect_return(__percpu_read_16, pcp)
+	_pcp_wrap_return(__percpu_read_16, pcp)
 #define this_cpu_read_4(pcp)		\
-	_pcp_protect_return(__percpu_read_32, pcp)
+	_pcp_wrap_return(__percpu_read_32, pcp)
 #define this_cpu_read_8(pcp)		\
-	_pcp_protect_return(__percpu_read_64, pcp)
+	_pcp_wrap_return(__percpu_read_64, pcp)
 
 #define this_cpu_write_1(pcp, val)	\
-	_pcp_protect(__percpu_write_8, pcp, (unsigned long)(val))
+	_pcp_wrap(__percpu_write_8, pcp, (unsigned long)(val))
 #define this_cpu_write_2(pcp, val)	\
-	_pcp_protect(__percpu_write_16, pcp, (unsigned long)(val))
+	_pcp_wrap(__percpu_write_16, pcp, (unsigned long)(val))
 #define this_cpu_write_4(pcp, val)	\
-	_pcp_protect(__percpu_write_32, pcp, (unsigned long)(val))
+	_pcp_wrap(__percpu_write_32, pcp, (unsigned long)(val))
 #define this_cpu_write_8(pcp, val)	\
-	_pcp_protect(__percpu_write_64, pcp, (unsigned long)(val))
+	_pcp_wrap(__percpu_write_64, pcp, (unsigned long)(val))
 
 #define this_cpu_add_1(pcp, val)	\
-	_pcp_protect(__percpu_add_case_8, pcp, val)
+	_pcp_wrap(__percpu_add_case_8, pcp, val)
 #define this_cpu_add_2(pcp, val)	\
-	_pcp_protect(__percpu_add_case_16, pcp, val)
+	_pcp_wrap(__percpu_add_case_16, pcp, val)
 #define this_cpu_add_4(pcp, val)	\
-	_pcp_protect(__percpu_add_case_32, pcp, val)
+	_pcp_wrap(__percpu_add_case_32, pcp, val)
 #define this_cpu_add_8(pcp, val)	\
-	_pcp_protect(__percpu_add_case_64, pcp, val)
+	_pcp_wrap(__percpu_add_case_64, pcp, val)
 
 #define this_cpu_add_return_1(pcp, val)	\
-	_pcp_protect_return(__percpu_add_return_case_8, pcp, val)
+	_pcp_wrap_return(__percpu_add_return_case_8, pcp, val)
 #define this_cpu_add_return_2(pcp, val)	\
-	_pcp_protect_return(__percpu_add_return_case_16, pcp, val)
+	_pcp_wrap_return(__percpu_add_return_case_16, pcp, val)
 #define this_cpu_add_return_4(pcp, val)	\
-	_pcp_protect_return(__percpu_add_return_case_32, pcp, val)
+	_pcp_wrap_return(__percpu_add_return_case_32, pcp, val)
 #define this_cpu_add_return_8(pcp, val)	\
-	_pcp_protect_return(__percpu_add_return_case_64, pcp, val)
+	_pcp_wrap_return(__percpu_add_return_case_64, pcp, val)
 
 #define this_cpu_and_1(pcp, val)	\
-	_pcp_protect(__percpu_andnot_case_8, pcp, ~(u8)(val))
+	_pcp_wrap(__percpu_andnot_case_8, pcp, ~(u8)(val))
 #define this_cpu_and_2(pcp, val)	\
-	_pcp_protect(__percpu_andnot_case_16, pcp, ~(u16)(val))
+	_pcp_wrap(__percpu_andnot_case_16, pcp, ~(u16)(val))
 #define this_cpu_and_4(pcp, val)	\
-	_pcp_protect(__percpu_andnot_case_32, pcp, ~(u32)(val))
+	_pcp_wrap(__percpu_andnot_case_32, pcp, ~(u32)(val))
 #define this_cpu_and_8(pcp, val)	\
-	_pcp_protect(__percpu_andnot_case_64, pcp, ~(u64)(val))
+	_pcp_wrap(__percpu_andnot_case_64, pcp, ~(u64)(val))
 
 #define this_cpu_or_1(pcp, val)		\
-	_pcp_protect(__percpu_or_case_8, pcp, val)
+	_pcp_wrap(__percpu_or_case_8, pcp, val)
 #define this_cpu_or_2(pcp, val)		\
-	_pcp_protect(__percpu_or_case_16, pcp, val)
+	_pcp_wrap(__percpu_or_case_16, pcp, val)
 #define this_cpu_or_4(pcp, val)		\
-	_pcp_protect(__percpu_or_case_32, pcp, val)
+	_pcp_wrap(__percpu_or_case_32, pcp, val)
 #define this_cpu_or_8(pcp, val)		\
-	_pcp_protect(__percpu_or_case_64, pcp, val)
+	_pcp_wrap(__percpu_or_case_64, pcp, val)
 
 #define this_cpu_xchg_1(pcp, val)	\
-	_pcp_protect_return(xchg_relaxed, pcp, val)
+	_pcp_wrap_xchg(__percpu_xchg_case_8, pcp, val)
 #define this_cpu_xchg_2(pcp, val)	\
-	_pcp_protect_return(xchg_relaxed, pcp, val)
+	_pcp_wrap_xchg(__percpu_xchg_case_16, pcp, val)
 #define this_cpu_xchg_4(pcp, val)	\
-	_pcp_protect_return(xchg_relaxed, pcp, val)
+	_pcp_wrap_xchg(__percpu_xchg_case_32, pcp, val)
 #define this_cpu_xchg_8(pcp, val)	\
-	_pcp_protect_return(xchg_relaxed, pcp, val)
+	_pcp_wrap_xchg(__percpu_xchg_case_64, pcp, val)
 
 #define this_cpu_cmpxchg_1(pcp, o, n)	\
-	_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
+	_pcp_wrap_cmpxchg(__percpu_cmpxchg_case_8, pcp, o, n)
 #define this_cpu_cmpxchg_2(pcp, o, n)	\
-	_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
+	_pcp_wrap_cmpxchg(__percpu_cmpxchg_case_16, pcp, o, n)
 #define this_cpu_cmpxchg_4(pcp, o, n)	\
-	_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
+	_pcp_wrap_cmpxchg(__percpu_cmpxchg_case_32, pcp, o, n)
 #define this_cpu_cmpxchg_8(pcp, o, n)	\
-	_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
+	_pcp_wrap_cmpxchg(__percpu_cmpxchg_case_64, pcp, o, n)
 
 #define this_cpu_cmpxchg64(pcp, o, n)	this_cpu_cmpxchg_8(pcp, o, n)
 
-#define this_cpu_cmpxchg128(pcp, o, n)					\
-({									\
-	typedef typeof(pcp) pcp_op_T__;					\
-	u128 old__, new__, ret__;					\
-	pcp_op_T__ *ptr__;						\
-	old__ = o;							\
-	new__ = n;							\
-	preempt_disable_notrace();					\
-	ptr__ = raw_cpu_ptr(&(pcp));					\
-	ret__ = cmpxchg128_local((void *)ptr__, old__, new__);		\
-	preempt_enable_notrace();					\
-	ret__;								\
-})
+static inline u128
+__percpu_cmpxchg_128(void __percpu *pcp, u128 old, u128 new)
+{
+	u16 *gprs = &current_thread_info()->pcpu_gprs;
+	unsigned long addr;
+	unsigned long off;
+	union __u128_halves r, o = { .full = (old) },
+			       n = { .full = (new) };
+	register unsigned long ol asm ("x0") = o.low;
+	register unsigned long oh asm ("x1") = o.high;
+	register unsigned long nl asm ("x2") = n.low;
+	register unsigned long nh asm ("x3") = n.high;
+	unsigned long rl, rh;
+	unsigned long tmp;
+
+	asm volatile (
+	__PCPU_GPRS_BEGIN("%[gprs]", "%[pcp]", "%[off]", "%[addr]")
+	ARM64_LSE_ATOMIC_INSN(
+	/* LL/SC */
+       "       prfm    pstl1strm, [%[addr]]\n"
+       "1:     ldxp    %[rl], %[rh], [%[addr]]\n"
+       "       cmp     %[rl], %[ol]\n"
+       "       ccmp    %[rh], %[oh], 0, eq\n"
+       "       b.ne    2f\n"
+       "       stxp    %w[tmp], %[nl], %[nh], [%[addr]]\n"
+       "       cbnz    %w[tmp], 1b\n"
+       "2:\n"
+	,
+	/* LSE atomics */
+	"	casp	%[ol], %[oh], %[nl], %[nh], [%[addr]]\n"
+	"	mov	%[rl], %[ol]\n"
+	"	mov	%[rh], %[oh]\n"
+	__nops(4)
+	)
+	__PCPU_GPRS_END("%[gprs]")
+	: [gprs] "=Qo" (*gprs),
+	  [addr] "=&r" (addr),
+	  [off] "=&r" (off),
+	  [tmp] "=&r" (tmp),
+	  [ol] "+&r" (ol),
+	  [oh] "+&r" (oh),
+	  [rl] "=&r" (rl),
+	  [rh] "=&r" (rh)
+	: [pcp] "r" (pcp),
+	  [nl] "r" (nl),
+	  [nh] "r" (nh)
+	: "memory", "cc"
+	);
+
+	r.low = rl;
+	r.high = rh;
+
+	return r.full;
+}
+
+#define this_cpu_cmpxchg128(pcp, o, n)	\
+	_pcp_wrap_return(__percpu_cmpxchg_128, pcp, o, n)
 
 #ifdef __KVM_NVHE_HYPERVISOR__
 extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h
index 09d7c00cf40541..590ef4328f12f5 100644
--- a/arch/arm64/include/asm/pgtable-prot.h
+++ b/arch/arm64/include/asm/pgtable-prot.h
@@ -66,9 +66,9 @@
 
 #ifndef __ASSEMBLER__
 
+#include <linux/arm-rsi-cmds.h>
 #include <asm/cpufeature.h>
 #include <asm/pgtable-types.h>
-#include <asm/rsi.h>
 
 extern bool arm64_use_ng_mappings;
 extern unsigned long prot_ns_shared;
@@ -139,7 +139,7 @@ static inline bool __pure lpa2_is_enabled(void)
 
 /*
  * Page types used via Permission Indirection Extension (PIE). PIE uses
- * the USER, DBM, PXN and UXN bits to to generate an index which is used
+ * the USER, DBM, PXN and UXN bits to generate an index which is used
  * to look up the actual permission in PIR_ELx and PIRE0_EL1. We define
  * combinations we use on non-PIE systems with the same encoding, for
  * convenience these are listed here as comments as are the unallocated
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 6000905a2e865e..763c5a411d64e3 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -107,9 +107,6 @@ static inline void arch_leave_lazy_mmu_mode(void)
 	__flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2,	\
 			  TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE)
 
-#define pte_ERROR(e)	\
-	pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e))
-
 #ifdef CONFIG_ARM64_PA_BITS_52
 static inline phys_addr_t __pte_to_phys(pte_t pte)
 {
@@ -390,52 +387,13 @@ static inline pte_t __ptep_get(pte_t *ptep)
 extern void __sync_icache_dcache(pte_t pteval);
 bool pgattr_change_is_safe(pteval_t old, pteval_t new);
 
-/*
- * PTE bits configuration in the presence of hardware Dirty Bit Management
- * (PTE_WRITE == PTE_DBM):
- *
- * Dirty  Writable | PTE_RDONLY  PTE_WRITE  PTE_DIRTY (sw)
- *   0      0      |   1           0          0
- *   0      1      |   1           1          0
- *   1      0      |   1           0          1
- *   1      1      |   0           1          x
- *
- * When hardware DBM is not present, the software PTE_DIRTY bit is updated via
- * the page fault mechanism. Checking the dirty status of a pte becomes:
- *
- *   PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
- */
-
-static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep,
-					   pte_t pte)
+#ifdef CONFIG_DEBUG_VM
+void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, pte_t pte);
+#else
+static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, pte_t pte)
 {
-	pte_t old_pte;
-
-	if (!IS_ENABLED(CONFIG_DEBUG_VM))
-		return;
-
-	old_pte = __ptep_get(ptep);
-
-	if (!pte_valid(old_pte) || !pte_valid(pte))
-		return;
-	if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
-		return;
-
-	/*
-	 * Check for potential race with hardware updates of the pte
-	 * (__ptep_set_access_flags safely changes valid ptes without going
-	 * through an invalid entry).
-	 */
-	VM_WARN_ONCE(!pte_young(pte),
-		     "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
-		     __func__, pte_val(old_pte), pte_val(pte));
-	VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
-		     "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
-		     __func__, pte_val(old_pte), pte_val(pte));
-	VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)),
-		     "%s: unsafe attribute change: 0x%016llx -> 0x%016llx",
-		     __func__, pte_val(old_pte), pte_val(pte));
 }
+#endif /* CONFIG_DEBUG_VM */
 
 static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages)
 {
@@ -866,9 +824,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 
 #if CONFIG_PGTABLE_LEVELS > 2
 
-#define pmd_ERROR(e)	\
-	pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
-
 #define pud_none(pud)		(!pud_val(pud))
 #define pud_bad(pud)		((pud_val(pud) & PUD_TYPE_MASK) != \
 				 PUD_TYPE_TABLE)
@@ -960,9 +915,6 @@ static inline bool mm_pud_folded(const struct mm_struct *mm)
 }
 #define mm_pud_folded  mm_pud_folded
 
-#define pud_ERROR(e)	\
-	pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e))
-
 #define p4d_none(p4d)		(pgtable_l4_enabled() && !p4d_val(p4d))
 #define p4d_bad(p4d)		(pgtable_l4_enabled() && \
 				((p4d_val(p4d) & P4D_TYPE_MASK) != \
@@ -1088,9 +1040,6 @@ static inline bool mm_p4d_folded(const struct mm_struct *mm)
 }
 #define mm_p4d_folded  mm_p4d_folded
 
-#define p4d_ERROR(e)	\
-	pr_err("%s:%d: bad p4d %016llx.\n", __FILE__, __LINE__, p4d_val(e))
-
 #define pgd_none(pgd)		(pgtable_l5_enabled() && !pgd_val(pgd))
 #define pgd_bad(pgd)		(pgtable_l5_enabled() && \
 				((pgd_val(pgd) & PGD_TYPE_MASK) != \
@@ -1217,9 +1166,6 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
 
 #endif  /* CONFIG_PGTABLE_LEVELS > 4 */
 
-#define pgd_ERROR(e)	\
-	pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
-
 #define pgd_set_fixmap(addr)	((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
 #define pgd_clear_fixmap()	clear_fixmap(FIX_PGD)
 
diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h
index 9ecc2766a9f218..75a592b7f47f7a 100644
--- a/arch/arm64/include/asm/preempt.h
+++ b/arch/arm64/include/asm/preempt.h
@@ -75,28 +75,21 @@ static inline int __preempt_count_sub_return(int val)
 	return pc;
 }
 
-static inline bool __preempt_count_dec_and_test(void)
+static inline bool should_resched(int preempt_offset)
 {
-	struct thread_info *ti = current_thread_info();
-	u64 pc = READ_ONCE(ti->preempt_count);
-
-	/* Update only the count field, leaving need_resched unchanged */
-	WRITE_ONCE(ti->preempt.count, --pc);
-
-	/*
-	 * If we wrote back all zeroes, then we're preemptible and in
-	 * need of a reschedule. Otherwise, we need to reload the
-	 * preempt_count in case the need_resched flag was cleared by an
-	 * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE
-	 * pair.
-	 */
-	return !pc || !READ_ONCE(ti->preempt_count);
+	u64 pc = READ_ONCE(current_thread_info()->preempt_count);
+	return unlikely(pc == preempt_offset);
 }
 
-static inline bool should_resched(int preempt_offset)
+static inline bool __preempt_count_dec_and_test(void)
 {
-	u64 pc = READ_ONCE(current_thread_info()->preempt_count);
-	return pc == preempt_offset;
+	/*
+	 * We must load the combined 'preempt_count' after decrementing
+	 * 'preempt.count' as an interrupt could modify 'need_resched' before
+	 * __preempt_count_sub() writes back to 'preempt.count'.
+	 */
+	__preempt_count_sub(1);
+	return should_resched(0);
 }
 
 #ifdef CONFIG_PREEMPTION
@@ -104,19 +97,9 @@ static inline bool should_resched(int preempt_offset)
 void preempt_schedule(void);
 void preempt_schedule_notrace(void);
 
-#ifdef CONFIG_PREEMPT_DYNAMIC
-
-void dynamic_preempt_schedule(void);
-#define __preempt_schedule()		dynamic_preempt_schedule()
-void dynamic_preempt_schedule_notrace(void);
-#define __preempt_schedule_notrace()	dynamic_preempt_schedule_notrace()
-
-#else /* CONFIG_PREEMPT_DYNAMIC */
-
 #define __preempt_schedule()		preempt_schedule()
 #define __preempt_schedule_notrace()	preempt_schedule_notrace()
 
-#endif /* CONFIG_PREEMPT_DYNAMIC */
 #endif /* CONFIG_PREEMPTION */
 
 #endif /* __ASM_PREEMPT_H */
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 6dfbcacd9ba090..481f1999f7c142 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -351,8 +351,6 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
 	unsigned long pstate = PSR_AA32_MODE_USR;
 	if (pc & 1)
 		pstate |= PSR_AA32_T_BIT;
-	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
-		pstate |= PSR_AA32_E_BIT;
 
 	start_thread_common(regs, pc, pstate);
 	spectre_v4_enable_task_mitigation(current);
diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index f7dc5fb9427d63..bd291bd6cfac7c 100644
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -66,12 +66,6 @@
 #define PSR_AA32_IT_MASK	0x0600fc00	/* If-Then execution state mask */
 #define PSR_AA32_GE_MASK	0x000f0000
 
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define PSR_AA32_ENDSTATE	PSR_AA32_E_BIT
-#else
-#define PSR_AA32_ENDSTATE	0
-#endif
-
 /* AArch32 CPSR bits, as seen in AArch32 */
 #define COMPAT_PSR_DIT_BIT	0x00200000
 
@@ -167,6 +161,11 @@ struct pt_regs {
 
 	u64 sdei_ttbr1;
 	struct frame_record_meta stackframe;
+
+	u16	pcpu_gprs;
+	u16	__unused1;
+	u32	__unused2;
+	u64	__unused3;
 };
 
 /* For correct stack alignment, pt_regs has to be a multiple of 16 bytes. */
@@ -266,7 +265,8 @@ static inline u64 regs_get_register(struct pt_regs *regs, unsigned int offset)
  * Read a register given an architectural register index r.
  * This handles the common case where 31 means XZR, not SP.
  */
-static inline unsigned long pt_regs_read_reg(const struct pt_regs *regs, int r)
+static __always_inline unsigned long
+pt_regs_read_reg(const struct pt_regs *regs, int r)
 {
 	return (r == 31) ? 0 : regs->regs[r];
 }
@@ -275,8 +275,8 @@ static inline unsigned long pt_regs_read_reg(const struct pt_regs *regs, int r)
  * Write a register given an architectural register index r.
  * This handles the common case where 31 means XZR, not SP.
  */
-static inline void pt_regs_write_reg(struct pt_regs *regs, int r,
-				     unsigned long val)
+static __always_inline void
+pt_regs_write_reg(struct pt_regs *regs, int r, unsigned long val)
 {
 	if (r != 31)
 		regs->regs[r] = val;
diff --git a/arch/arm64/include/asm/rsi.h b/arch/arm64/include/asm/rsi.h
deleted file mode 100644
index 88b50d660e85a0..00000000000000
--- a/arch/arm64/include/asm/rsi.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2024 ARM Ltd.
- */
-
-#ifndef __ASM_RSI_H_
-#define __ASM_RSI_H_
-
-#include <linux/errno.h>
-#include <linux/jump_label.h>
-#include <asm/rsi_cmds.h>
-
-#define RSI_PDEV_NAME "arm-cca-dev"
-
-DECLARE_STATIC_KEY_FALSE(rsi_present);
-
-void __init arm64_rsi_init(void);
-
-bool arm64_rsi_is_protected(phys_addr_t base, size_t size);
-
-static inline bool is_realm_world(void)
-{
-	return static_branch_unlikely(&rsi_present);
-}
-
-static inline int rsi_set_memory_range(phys_addr_t start, phys_addr_t end,
-				       enum ripas state, unsigned long flags)
-{
-	unsigned long ret;
-	phys_addr_t top;
-
-	while (start != end) {
-		ret = rsi_set_addr_range_state(start, end, state, flags, &top);
-		if (ret || top < start || top > end)
-			return -EINVAL;
-		start = top;
-	}
-
-	return 0;
-}
-
-/*
- * Convert the specified range to RAM. Do not use this if you rely on the
- * contents of a page that may already be in RAM state.
- */
-static inline int rsi_set_memory_range_protected(phys_addr_t start,
-						 phys_addr_t end)
-{
-	return rsi_set_memory_range(start, end, RSI_RIPAS_RAM,
-				    RSI_CHANGE_DESTROYED);
-}
-
-/*
- * Convert the specified range to RAM. Do not convert any pages that may have
- * been DESTROYED, without our permission.
- */
-static inline int rsi_set_memory_range_protected_safe(phys_addr_t start,
-						      phys_addr_t end)
-{
-	return rsi_set_memory_range(start, end, RSI_RIPAS_RAM,
-				    RSI_NO_CHANGE_DESTROYED);
-}
-
-static inline int rsi_set_memory_range_shared(phys_addr_t start,
-					      phys_addr_t end)
-{
-	return rsi_set_memory_range(start, end, RSI_RIPAS_EMPTY,
-				    RSI_CHANGE_DESTROYED);
-}
-#endif /* __ASM_RSI_H_ */
diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h
deleted file mode 100644
index c1fab41f671ec9..00000000000000
--- a/arch/arm64/include/asm/rsi_cmds.h
+++ /dev/null
@@ -1,173 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2023 ARM Ltd.
- */
-
-#ifndef __ASM_RSI_CMDS_H
-#define __ASM_RSI_CMDS_H
-
-#include <linux/arm-smccc.h>
-#include <linux/string.h>
-#include <asm/memory.h>
-
-#include <asm/rsi_smc.h>
-
-#define RSI_GRANULE_SHIFT		12
-#define RSI_GRANULE_SIZE		(_AC(1, UL) << RSI_GRANULE_SHIFT)
-
-enum ripas {
-	RSI_RIPAS_EMPTY = 0,
-	RSI_RIPAS_RAM = 1,
-	RSI_RIPAS_DESTROYED = 2,
-	RSI_RIPAS_DEV = 3,
-};
-
-static inline unsigned long rsi_request_version(unsigned long req,
-						unsigned long *out_lower,
-						unsigned long *out_higher)
-{
-	struct arm_smccc_res res;
-
-	arm_smccc_smc(SMC_RSI_ABI_VERSION, req, 0, 0, 0, 0, 0, 0, &res);
-
-	if (out_lower)
-		*out_lower = res.a1;
-	if (out_higher)
-		*out_higher = res.a2;
-
-	return res.a0;
-}
-
-static inline unsigned long rsi_get_realm_config(struct realm_config *cfg)
-{
-	struct arm_smccc_res res;
-
-	arm_smccc_smc(SMC_RSI_REALM_CONFIG, virt_to_phys(cfg),
-		      0, 0, 0, 0, 0, 0, &res);
-	return res.a0;
-}
-
-static inline unsigned long rsi_ipa_state_get(phys_addr_t start,
-					      phys_addr_t end,
-					      enum ripas *state,
-					      phys_addr_t *top)
-{
-	struct arm_smccc_res res;
-
-	arm_smccc_smc(SMC_RSI_IPA_STATE_GET,
-		      start, end, 0, 0, 0, 0, 0,
-		      &res);
-
-	if (res.a0 == RSI_SUCCESS) {
-		if (top)
-			*top = res.a1;
-		if (state)
-			*state = res.a2;
-	}
-
-	return res.a0;
-}
-
-static inline long rsi_set_addr_range_state(phys_addr_t start,
-					    phys_addr_t end,
-					    enum ripas state,
-					    unsigned long flags,
-					    phys_addr_t *top)
-{
-	struct arm_smccc_res res;
-
-	arm_smccc_smc(SMC_RSI_IPA_STATE_SET, start, end, state,
-		      flags, 0, 0, 0, &res);
-
-	if (top)
-		*top = res.a1;
-
-	if (res.a2 != RSI_ACCEPT)
-		return -EPERM;
-
-	return res.a0;
-}
-
-#define RSI_ATTEST_CHALLENGE_MIN_SIZE	32
-#define RSI_ATTEST_CHALLENGE_MAX_SIZE	64
-
-struct rsi_attestation_token_init_args {
-	unsigned long fid;
-	u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE];
-};
-
-/**
- * rsi_attestation_token_init - Initialise the operation to retrieve an
- * attestation token.
- *
- * @challenge:	The challenge data to be used in the attestation token
- *		generation.
- * @size:	Size of the challenge data in bytes.
- *
- * Initialises the attestation token generation and returns an upper bound
- * on the attestation token size that can be used to allocate an adequate
- * buffer. The caller is expected to subsequently call
- * rsi_attestation_token_continue() to retrieve the attestation token data on
- * the same CPU.
- *
- * Returns:
- *  On success, returns the upper limit of the attestation report size.
- *  Otherwise, -EINVAL
- */
-static inline long
-rsi_attestation_token_init(const u8 *challenge, unsigned long size)
-{
-	union {
-		struct arm_smccc_1_2_regs regs;
-		struct rsi_attestation_token_init_args init;
-	} args = { 0 };
-
-	if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE ||
-	    size > RSI_ATTEST_CHALLENGE_MAX_SIZE)
-		return -EINVAL;
-
-	args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT;
-	memcpy(args.init.challenge, challenge, size);
-	arm_smccc_1_2_smc(&args.regs, &args.regs);
-
-	if (args.regs.a0 == RSI_SUCCESS)
-		return args.regs.a1;
-
-	return -EINVAL;
-}
-
-/**
- * rsi_attestation_token_continue - Continue the operation to retrieve an
- * attestation token.
- *
- * @granule: {I}PA of the Granule to which the token will be written.
- * @offset:  Offset within Granule to start of buffer in bytes.
- * @size:    The size of the buffer.
- * @len:     The number of bytes written to the buffer.
- *
- * Retrieves up to a RSI_GRANULE_SIZE worth of token data per call. The caller
- * is expected to call rsi_attestation_token_init() before calling this
- * function to retrieve the attestation token.
- *
- * Return:
- * * %RSI_SUCCESS     - Attestation token retrieved successfully.
- * * %RSI_INCOMPLETE  - Token generation is not complete.
- * * %RSI_ERROR_INPUT - A parameter was not valid.
- * * %RSI_ERROR_STATE - Attestation not in progress.
- */
-static inline unsigned long rsi_attestation_token_continue(phys_addr_t granule,
-							   unsigned long offset,
-							   unsigned long size,
-							   unsigned long *len)
-{
-	struct arm_smccc_res res;
-
-	arm_smccc_1_1_invoke(SMC_RSI_ATTESTATION_TOKEN_CONTINUE,
-			     granule, offset, size, 0, &res);
-
-	if (len)
-		*len = res.a1;
-	return res.a0;
-}
-
-#endif /* __ASM_RSI_CMDS_H */
diff --git a/arch/arm64/include/asm/rsi_smc.h b/arch/arm64/include/asm/rsi_smc.h
deleted file mode 100644
index e19253f96c9407..00000000000000
--- a/arch/arm64/include/asm/rsi_smc.h
+++ /dev/null
@@ -1,193 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2023 ARM Ltd.
- */
-
-#ifndef __ASM_RSI_SMC_H_
-#define __ASM_RSI_SMC_H_
-
-#include <linux/arm-smccc.h>
-
-/*
- * This file describes the Realm Services Interface (RSI) Application Binary
- * Interface (ABI) for SMC calls made from within the Realm to the RMM and
- * serviced by the RMM.
- */
-
-/*
- * The major version number of the RSI implementation.  This is increased when
- * the binary format or semantics of the SMC calls change.
- */
-#define RSI_ABI_VERSION_MAJOR		UL(1)
-
-/*
- * The minor version number of the RSI implementation.  This is increased when
- * a bug is fixed, or a feature is added without breaking binary compatibility.
- */
-#define RSI_ABI_VERSION_MINOR		UL(0)
-
-#define RSI_ABI_VERSION			((RSI_ABI_VERSION_MAJOR << 16) | \
-					 RSI_ABI_VERSION_MINOR)
-
-#define RSI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16)
-#define RSI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF)
-
-#define RSI_SUCCESS		UL(0)
-#define RSI_ERROR_INPUT		UL(1)
-#define RSI_ERROR_STATE		UL(2)
-#define RSI_INCOMPLETE		UL(3)
-#define RSI_ERROR_UNKNOWN	UL(4)
-
-#define SMC_RSI_FID(n)		ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,      \
-						   ARM_SMCCC_SMC_64,         \
-						   ARM_SMCCC_OWNER_STANDARD, \
-						   n)
-
-/*
- * Returns RSI version.
- *
- * arg1 == Requested interface revision
- * ret0 == Status / error
- * ret1 == Lower implemented interface revision
- * ret2 == Higher implemented interface revision
- */
-#define SMC_RSI_ABI_VERSION	SMC_RSI_FID(0x190)
-
-/*
- * Read feature register.
- *
- * arg1 == Feature register index
- * ret0 == Status / error
- * ret1 == Feature register value
- */
-#define SMC_RSI_FEATURES			SMC_RSI_FID(0x191)
-
-/*
- * Read measurement for the current Realm.
- *
- * arg1 == Index, which measurements slot to read
- * ret0 == Status / error
- * ret1 == Measurement value, bytes:  0 -  7
- * ret2 == Measurement value, bytes:  8 - 15
- * ret3 == Measurement value, bytes: 16 - 23
- * ret4 == Measurement value, bytes: 24 - 31
- * ret5 == Measurement value, bytes: 32 - 39
- * ret6 == Measurement value, bytes: 40 - 47
- * ret7 == Measurement value, bytes: 48 - 55
- * ret8 == Measurement value, bytes: 56 - 63
- */
-#define SMC_RSI_MEASUREMENT_READ		SMC_RSI_FID(0x192)
-
-/*
- * Extend Realm Extensible Measurement (REM) value.
- *
- * arg1  == Index, which measurements slot to extend
- * arg2  == Size of realm measurement in bytes, max 64 bytes
- * arg3  == Measurement value, bytes:  0 -  7
- * arg4  == Measurement value, bytes:  8 - 15
- * arg5  == Measurement value, bytes: 16 - 23
- * arg6  == Measurement value, bytes: 24 - 31
- * arg7  == Measurement value, bytes: 32 - 39
- * arg8  == Measurement value, bytes: 40 - 47
- * arg9  == Measurement value, bytes: 48 - 55
- * arg10 == Measurement value, bytes: 56 - 63
- * ret0  == Status / error
- */
-#define SMC_RSI_MEASUREMENT_EXTEND		SMC_RSI_FID(0x193)
-
-/*
- * Initialize the operation to retrieve an attestation token.
- *
- * arg1 == Challenge value, bytes:  0 -  7
- * arg2 == Challenge value, bytes:  8 - 15
- * arg3 == Challenge value, bytes: 16 - 23
- * arg4 == Challenge value, bytes: 24 - 31
- * arg5 == Challenge value, bytes: 32 - 39
- * arg6 == Challenge value, bytes: 40 - 47
- * arg7 == Challenge value, bytes: 48 - 55
- * arg8 == Challenge value, bytes: 56 - 63
- * ret0 == Status / error
- * ret1 == Upper bound of token size in bytes
- */
-#define SMC_RSI_ATTESTATION_TOKEN_INIT		SMC_RSI_FID(0x194)
-
-/*
- * Continue the operation to retrieve an attestation token.
- *
- * arg1 == The IPA of token buffer
- * arg2 == Offset within the granule of the token buffer
- * arg3 == Size of the granule buffer
- * ret0 == Status / error
- * ret1 == Length of token bytes copied to the granule buffer
- */
-#define SMC_RSI_ATTESTATION_TOKEN_CONTINUE	SMC_RSI_FID(0x195)
-
-#ifndef __ASSEMBLER__
-
-struct realm_config {
-	union {
-		struct {
-			unsigned long ipa_bits; /* Width of IPA in bits */
-			unsigned long hash_algo; /* Hash algorithm */
-		};
-		u8 pad[0x200];
-	};
-	union {
-		u8 rpv[64]; /* Realm Personalization Value */
-		u8 pad2[0xe00];
-	};
-	/*
-	 * The RMM requires the configuration structure to be aligned to a 4k
-	 * boundary, ensure this happens by aligning this structure.
-	 */
-} __aligned(0x1000);
-
-#endif /* __ASSEMBLER__ */
-
-/*
- * Read configuration for the current Realm.
- *
- * arg1 == struct realm_config addr
- * ret0 == Status / error
- */
-#define SMC_RSI_REALM_CONFIG			SMC_RSI_FID(0x196)
-
-/*
- * Request RIPAS of a target IPA range to be changed to a specified value.
- *
- * arg1 == Base IPA address of target region
- * arg2 == Top of the region
- * arg3 == RIPAS value
- * arg4 == flags
- * ret0 == Status / error
- * ret1 == Top of modified IPA range
- * ret2 == Whether the Host accepted or rejected the request
- */
-#define SMC_RSI_IPA_STATE_SET			SMC_RSI_FID(0x197)
-
-#define RSI_NO_CHANGE_DESTROYED			UL(0)
-#define RSI_CHANGE_DESTROYED			UL(1)
-
-#define RSI_ACCEPT				UL(0)
-#define RSI_REJECT				UL(1)
-
-/*
- * Get RIPAS of a target IPA range.
- *
- * arg1 == Base IPA of target region
- * arg2 == End of target IPA region
- * ret0 == Status / error
- * ret1 == Top of IPA region which has the reported RIPAS value
- * ret2 == RIPAS value
- */
-#define SMC_RSI_IPA_STATE_GET			SMC_RSI_FID(0x198)
-
-/*
- * Make a Host call.
- *
- * arg1 == IPA of host call structure
- * ret0 == Status / error
- */
-#define SMC_RSI_HOST_CALL			SMC_RSI_FID(0x199)
-
-#endif /* __ASM_RSI_SMC_H_ */
diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h
index 90f61b17275e1b..0091ba12200e68 100644
--- a/arch/arm64/include/asm/set_memory.h
+++ b/arch/arm64/include/asm/set_memory.h
@@ -11,9 +11,8 @@ bool can_set_direct_map(void);
 
 int set_memory_valid(unsigned long addr, int numpages, int enable);
 
-int set_direct_map_invalid_noflush(struct page *page);
-int set_direct_map_default_noflush(struct page *page);
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);
+int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages);
+int set_direct_map_default_noflush(struct page *page, unsigned int numpages);
 bool kernel_page_present(struct page *page);
 
 int set_memory_encrypted(unsigned long addr, int numpages);
diff --git a/arch/arm64/include/asm/stacktrace/nvhe.h b/arch/arm64/include/asm/stacktrace/nvhe.h
index 171f9edef49fc4..a631a577cbe5dd 100644
--- a/arch/arm64/include/asm/stacktrace/nvhe.h
+++ b/arch/arm64/include/asm/stacktrace/nvhe.h
@@ -37,6 +37,14 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state,
 	state->pc = pc;
 }
 
+DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
+DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info);
+
+#ifdef CONFIG_PKVM_STACKTRACE
+DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
+			 pkvm_stacktrace);
+#endif
+
 #ifndef __KVM_NVHE_HYPERVISOR__
 /*
  * Conventional (non-protected) nVHE HYP stack unwinder
@@ -45,8 +53,6 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state,
  * (by the host in EL1).
  */
 
-DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
-DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info);
 DECLARE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
 
 void kvm_nvhe_dump_backtrace(unsigned long hyp_offset);
diff --git a/arch/arm64/include/asm/stage2_pgtable.h b/arch/arm64/include/asm/stage2_pgtable.h
index 23d27623e478b8..cf57da90b5d152 100644
--- a/arch/arm64/include/asm/stage2_pgtable.h
+++ b/arch/arm64/include/asm/stage2_pgtable.h
@@ -17,7 +17,7 @@
  *
  * This implies, the total number of page table levels required for
  * IPA_SHIFT at stage2 expected by the hardware can be calculated using
- * the same logic used for the (non-collapsable) stage1 page tables but for
+ * the same logic used for the (non-collapsible) stage1 page tables but for
  * (IPA_SHIFT - 4).
  */
 #define stage2_pgtable_levels(ipa)	ARM64_HW_PGTABLE_LEVELS((ipa) - 4)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 7aa08d59d49448..3cd7063df2bf4d 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -62,19 +62,10 @@
 
 #else  /* CONFIG_BROKEN_GAS_INST */
 
-#ifndef CONFIG_CPU_BIG_ENDIAN
-#define __INSTR_BSWAP(x)		(x)
-#else  /* CONFIG_CPU_BIG_ENDIAN */
-#define __INSTR_BSWAP(x)		((((x) << 24) & 0xff000000)	| \
-					 (((x) <<  8) & 0x00ff0000)	| \
-					 (((x) >>  8) & 0x0000ff00)	| \
-					 (((x) >> 24) & 0x000000ff))
-#endif	/* CONFIG_CPU_BIG_ENDIAN */
-
 #ifdef __ASSEMBLER__
-#define __emit_inst(x)			.long __INSTR_BSWAP(x)
+#define __emit_inst(x)			.long (x)
 #else  /* __ASSEMBLER__ */
-#define __emit_inst(x)			".long " __stringify(__INSTR_BSWAP(x)) "\n\t"
+#define __emit_inst(x)			".long " __stringify(x) "\n\t"
 #endif	/* __ASSEMBLER__ */
 
 #endif	/* CONFIG_BROKEN_GAS_INST */
@@ -173,7 +164,7 @@
  * come from here. The header relies on the definition of sys_reg()
  * earlier in this file.
  */
-#include "asm/sysreg-defs.h"
+#include <asm/sysreg-defs.h>
 
 /*
  * System registers, organised loosely by encoding but grouped together
@@ -835,39 +826,26 @@
 #define SCTLR_ELx_A	 (BIT(1))
 #define SCTLR_ELx_M	 (BIT(0))
 
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define ENDIAN_SET_EL2		SCTLR_ELx_EE
-#else
-#define ENDIAN_SET_EL2		0
-#endif
-
 #define INIT_SCTLR_EL2_MMU_ON						\
 	(SCTLR_ELx_M  | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I |	\
-	 SCTLR_ELx_IESB | SCTLR_ELx_WXN | ENDIAN_SET_EL2 |		\
-	 SCTLR_ELx_ITFSB | SCTLR_ELx_EIS | SCTLR_ELx_EOS | SCTLR_EL2_RES1)
+	 SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_ITFSB |		\
+	 SCTLR_ELx_EIS | SCTLR_ELx_EOS | SCTLR_EL2_RES1)
 
-#define INIT_SCTLR_EL2_MMU_OFF \
-	(SCTLR_EL2_RES1 | ENDIAN_SET_EL2)
+#define INIT_SCTLR_EL2_MMU_OFF	(SCTLR_EL2_RES1)
 
 /* SCTLR_EL1 specific flags. */
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define ENDIAN_SET_EL1		(SCTLR_EL1_E0E | SCTLR_ELx_EE)
-#else
-#define ENDIAN_SET_EL1		0
-#endif
-
 #define INIT_SCTLR_EL1_MMU_OFF \
-	(ENDIAN_SET_EL1 | SCTLR_EL1_LSMAOE | SCTLR_EL1_nTLSMD | \
-	 SCTLR_EL1_EIS  | SCTLR_EL1_TSCXT  | SCTLR_EL1_EOS)
+	(SCTLR_EL1_LSMAOE | SCTLR_EL1_nTLSMD | SCTLR_EL1_EIS  |		\
+	 SCTLR_EL1_TSCXT  | SCTLR_EL1_EOS)
 
 #define INIT_SCTLR_EL1_MMU_ON \
-	(SCTLR_ELx_M      | SCTLR_ELx_C      | SCTLR_ELx_SA    | \
-	 SCTLR_EL1_SA0    | SCTLR_EL1_SED    | SCTLR_ELx_I     | \
-	 SCTLR_EL1_DZE    | SCTLR_EL1_UCT    | SCTLR_EL1_nTWE  | \
-	 SCTLR_ELx_IESB   | SCTLR_EL1_SPAN   | SCTLR_ELx_ITFSB | \
-	 ENDIAN_SET_EL1   | SCTLR_EL1_UCI    | SCTLR_EL1_EPAN  | \
-	 SCTLR_EL1_LSMAOE | SCTLR_EL1_nTLSMD | SCTLR_EL1_EIS   | \
-	 SCTLR_EL1_TSCXT  | SCTLR_EL1_EOS)
+	(SCTLR_ELx_M      | SCTLR_ELx_C      | SCTLR_ELx_SA     | \
+	 SCTLR_EL1_SA0    | SCTLR_EL1_SED    | SCTLR_ELx_I      | \
+	 SCTLR_EL1_DZE    | SCTLR_EL1_UCT    | SCTLR_EL1_nTWE   | \
+	 SCTLR_ELx_IESB   | SCTLR_EL1_SPAN   | SCTLR_ELx_ITFSB  | \
+	 SCTLR_EL1_UCI    | SCTLR_EL1_EPAN   | SCTLR_EL1_LSMAOE | \
+	 SCTLR_EL1_nTLSMD | SCTLR_EL1_EIS    | SCTLR_EL1_TSCXT  | \
+	 SCTLR_EL1_EOS)
 
 /* MAIR_ELx memory attributes (used by Linux) */
 #define MAIR_ATTR_DEVICE_nGnRnE		UL(0x00)
@@ -1040,7 +1018,7 @@
 #define GCS_CAP(x)	((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \
 					       GCS_CAP_VALID_TOKEN)
 /*
- * Definitions for GICv5 instructions
+ * Definitions for GICv5 instructions for the Current Domain
  */
 #define GICV5_OP_GIC_CDAFF		sys_insn(1, 0, 12, 1, 3)
 #define GICV5_OP_GIC_CDDI		sys_insn(1, 0, 12, 2, 0)
@@ -1105,6 +1083,17 @@
 #define GICV5_GICR_CDNMIA_TYPE_MASK	GENMASK_ULL(31, 29)
 #define GICV5_GICR_CDNMIA_ID_MASK	GENMASK_ULL(23, 0)
 
+/*
+ * Definitions for GICv5 instructions for the Virtual Domain
+ */
+#define GICV5_OP_GIC_VDPEND		sys_insn(1, 4, 12, 1, 4)
+
+/* Shift and mask definitions for GIC VDPEND */
+#define GICV5_GIC_VDPEND_PENDING_MASK	BIT_ULL(63)
+#define GICV5_GIC_VDPEND_VM_MASK	GENMASK_ULL(47, 32)
+#define GICV5_GIC_VDPEND_TYPE_MASK	GENMASK_ULL(31, 29)
+#define GICV5_GIC_VDPEND_ID_MASK	GENMASK_ULL(23, 0)
+
 #define gicr_insn(insn)			read_sysreg_s(GICV5_OP_GICR_##insn)
 #define gic_insn(v, insn)		write_sysreg_s(v, GICV5_OP_GIC_##insn)
 
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 5d7fe3e153c850..6cb976c1c69313 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -29,13 +29,8 @@ struct thread_info {
 	union {
 		u64		preempt_count;	/* 0 => preemptible, <0 => bug */
 		struct {
-#ifdef CONFIG_CPU_BIG_ENDIAN
-			u32	need_resched;
-			u32	count;
-#else
 			u32	count;
 			u32	need_resched;
-#endif
 		} preempt;
 	};
 #ifdef CONFIG_SHADOW_CALL_STACK
@@ -46,6 +41,7 @@ struct thread_info {
 	u64			mpam_partid_pmg;
 #endif
 	u32			cpu;
+	u16			pcpu_gprs;
 };
 
 #define thread_saved_pc(tsk)	\
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 751bd57bc3bac5..2bb9ede20261d4 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -8,9 +8,6 @@
 #ifndef __ASM_TLB_H
 #define __ASM_TLB_H
 
-#include <linux/pagemap.h>
-
-
 #define tlb_flush tlb_flush
 static void tlb_flush(struct mmu_gather *tlb);
 
diff --git a/arch/arm64/include/asm/word-at-a-time.h b/arch/arm64/include/asm/word-at-a-time.h
index 824ca6987a516d..d7a3b69dc8c6f6 100644
--- a/arch/arm64/include/asm/word-at-a-time.h
+++ b/arch/arm64/include/asm/word-at-a-time.h
@@ -7,8 +7,6 @@
 
 #include <linux/uaccess.h>
 
-#ifndef __AARCH64EB__
-
 #include <linux/bitops.h>
 #include <linux/wordpart.h>
 
@@ -36,10 +34,6 @@ static inline unsigned long zero_bytemask(unsigned long bits)
 	return bits >> 7;
 }
 
-#else	/* __AARCH64EB__ */
-#include <asm-generic/word-at-a-time.h>
-#endif
-
 /*
  * Load an unaligned word from kernel space.
  *
diff --git a/arch/arm64/include/asm/xwreg.h b/arch/arm64/include/asm/xwreg.h
new file mode 100644
index 00000000000000..d55e3ac68f7ebf
--- /dev/null
+++ b/arch/arm64/include/asm/xwreg.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __ASM_XWREG_H
+#define __ASM_XWREG_H
+
+#include <asm/types.h>
+
+#define __xwreg_t_x		u64
+#define __xwreg_t_w		u32
+#define xwreg_t(xw)		__xwreg_t_##xw
+
+/*
+ * Zero extend 'v' from 'sz' bits (8/16/32/64) to fill an X or W register.
+ */
+#define xwreg_zero_extend(v, xw, sz)	((xwreg_t(xw))(u##sz)(v))
+
+#endif /* __ASM_XWREG_H */
diff --git a/arch/arm64/include/uapi/asm/byteorder.h b/arch/arm64/include/uapi/asm/byteorder.h
index ca9cfdab33fc59..b078e60f050656 100644
--- a/arch/arm64/include/uapi/asm/byteorder.h
+++ b/arch/arm64/include/uapi/asm/byteorder.h
@@ -17,10 +17,6 @@
 #ifndef __ASM_BYTEORDER_H
 #define __ASM_BYTEORDER_H
 
-#ifdef __AARCH64EB__
-#include <linux/byteorder/big_endian.h>
-#else
 #include <linux/byteorder/little_endian.h>
-#endif
 
 #endif	/* __ASM_BYTEORDER_H */
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 019e5e3d892e6d..65ee09bd14636d 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -97,6 +97,11 @@ struct kvm_regs {
 #define KVM_VGIC_V3_REDIST_SIZE		(2 * SZ_64K)
 #define KVM_VGIC_V3_ITS_SIZE		(2 * SZ_64K)
 
+/* Supported VGICv5 address types  */
+#define KVM_VGIC_V5_ADDR_TYPE_IRS	6
+
+#define KVM_VGIC_V5_IRS_SIZE		(2 * SZ_64K)
+
 #define KVM_ARM_VCPU_POWER_OFF		0 /* CPU is started in OFF state */
 #define KVM_ARM_VCPU_EL1_32BIT		1 /* CPU running a 32bit VM */
 #define KVM_ARM_VCPU_PSCI_0_2		2 /* CPU uses PSCI v0.2 */
@@ -418,6 +423,16 @@ enum {
 #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO  7
 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8
 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ  9
+#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS	10
+#define KVM_DEV_ARM_VGIC_GRP_IST	11
+
+struct kvm_vgic_v5_ist {
+	__u64	spi_ist_addr;
+	__u64	spi_ist_size;
+	__u64	lpi_ist_addr;
+	__u64	lpi_ist_size;
+};
+
 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT	10
 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \
 			(0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h
index 15649a253a5796..0c8c6a126c3b5b 100644
--- a/arch/arm64/include/uapi/asm/ptrace.h
+++ b/arch/arm64/include/uapi/asm/ptrace.h
@@ -113,7 +113,7 @@ struct user_hwdebug_state {
 
 struct user_sve_header {
 	__u32 size; /* total meaningful regset content in bytes */
-	__u32 max_size; /* maxmium possible size for this thread */
+	__u32 max_size; /* maximum possible size for this thread */
 	__u16 vl; /* current vector length */
 	__u16 max_vl; /* maximum possible vector length */
 	__u16 flags;
@@ -272,7 +272,7 @@ struct user_pac_generic_keys {
 
 struct user_za_header {
 	__u32 size; /* total meaningful regset content in bytes */
-	__u32 max_size; /* maxmium possible size for this thread */
+	__u32 max_size; /* maximum possible size for this thread */
 	__u16 vl; /* current vector length */
 	__u16 max_vl; /* maximum possible vector length */
 	__u16 flags;
diff --git a/arch/arm64/include/uapi/asm/setup.h b/arch/arm64/include/uapi/asm/setup.h
index 5d703888f35110..2236890175a5ab 100644
--- a/arch/arm64/include/uapi/asm/setup.h
+++ b/arch/arm64/include/uapi/asm/setup.h
@@ -22,6 +22,10 @@
 
 #include <linux/types.h>
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	2048
+#endif
 
 #endif
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index d2690c3ec52885..21fc3a3d25c91f 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -34,7 +34,7 @@ obj-y			:= debug-monitors.o entry.o irq.o fpsimd.o		\
 			   cpufeature.o alternative.o cacheinfo.o		\
 			   smp.o smp_spin_table.o topology.o smccc-call.o	\
 			   syscall.o proton-pack.o idle.o patching.o pi/	\
-			   rsi.o jump_label.o
+			   jump_label.o
 
 obj-$(CONFIG_COMPAT)			+= sys32.o signal32.o			\
 					   sys_compat.o
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 9c853ed3ceab86..9d603317eeb996 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -36,6 +36,7 @@ int main(void)
   DEFINE(TSK_TI_SCS_BASE,	offsetof(struct task_struct, thread_info.scs_base));
   DEFINE(TSK_TI_SCS_SP,		offsetof(struct task_struct, thread_info.scs_sp));
 #endif
+  DEFINE(TSK_TI_PCPU_GPRS,	offsetof(struct task_struct, thread_info.pcpu_gprs));
   DEFINE(TSK_STACK,		offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR
   DEFINE(TSK_STACK_CANARY,	offsetof(struct task_struct, stack_canary));
@@ -78,6 +79,7 @@ int main(void)
   DEFINE(S_PMR,			offsetof(struct pt_regs, pmr));
   DEFINE(S_STACKFRAME,		offsetof(struct pt_regs, stackframe));
   DEFINE(S_STACKFRAME_TYPE,	offsetof(struct pt_regs, stackframe.type));
+  DEFINE(S_PCPU_GPRS,		offsetof(struct pt_regs, pcpu_gprs));
   DEFINE(PT_REGS_SIZE,		sizeof(struct pt_regs));
   BLANK();
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index b33dccfafaf8c3..8ec47d89b45bb3 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -28,18 +28,21 @@ bool cpu_errata_set_target_impl(u64 num, void *impl_cpus)
 	return true;
 }
 
+static inline bool __is_midr_in_range(u32 midr, struct midr_range const *range)
+{
+	return midr_is_cpu_model_range(midr, range->model,
+				       range->rv_min, range->rv_max);
+}
+
 static inline bool is_midr_in_range(struct midr_range const *range)
 {
 	int i;
 
 	if (!target_impl_cpu_num)
-		return midr_is_cpu_model_range(read_cpuid_id(), range->model,
-					       range->rv_min, range->rv_max);
+		return __is_midr_in_range(read_cpuid_id(), range);
 
 	for (i = 0; i < target_impl_cpu_num; i++) {
-		if (midr_is_cpu_model_range(target_impl_cpus[i].midr,
-					    range->model,
-					    range->rv_min, range->rv_max))
+		if (__is_midr_in_range(target_impl_cpus[i].midr, range))
 			return true;
 	}
 	return false;
@@ -59,7 +62,7 @@ __is_affected_midr_range(const struct arm64_cpu_capabilities *entry,
 			 u32 midr, u32 revidr)
 {
 	const struct arm64_midr_revidr *fix;
-	if (!is_midr_in_range(&entry->midr_range))
+	if (!__is_midr_in_range(midr, &entry->midr_range))
 		return false;
 
 	midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 32102c3912fa7b..1f694a61d3b5db 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -888,7 +888,7 @@ static int search_cmp_ftr_reg(const void *id, const void *regp)
  * entry.
  *
  * returns - Upon success,  matching ftr_reg entry for id.
- *         - NULL on failure. It is upto the caller to decide
+ *         - NULL on failure. It is up to the caller to decide
  *	     the impact of a failure.
  */
 static struct arm64_ftr_reg *get_arm64_ftr_reg_nowarn(u32 sys_id)
@@ -3103,7 +3103,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_bbml3,
 	},
 	{
-		.desc = "52-bit Virtual Addressing for KVM (LPA2)",
+		.desc = "52-bit Virtual-Physical Addressing for S1 and S2 (LPA2)",
 		.capability = ARM64_HAS_LPA2,
 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
 		.matches = has_lpa2,
@@ -3707,7 +3707,7 @@ static void verify_local_cpu_caps(u16 scope_mask)
 			/*
 			 * We have to issue cpu_enable() irrespective of
 			 * whether the CPU has it or not, as it is enabeld
-			 * system wide. It is upto the call back to take
+			 * system wide. It is up to the call back to take
 			 * appropriate action on this CPU.
 			 */
 			if (caps->cpu_enable)
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 72c03ccea59fec..11d3f888fcd873 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -25,12 +25,49 @@
 #include <asm/irq_regs.h>
 #include <asm/kprobes.h>
 #include <asm/mmu.h>
+#include <asm/percpu.h>
 #include <asm/processor.h>
 #include <asm/sdei.h>
 #include <asm/stacktrace.h>
 #include <asm/sysreg.h>
 #include <asm/system_misc.h>
 
+/*
+ * Where the context being returned to had an active percpu GPR critical
+ * section, ensure that the offset and address GPRs are updated to match the
+ * current CPU.
+ *
+ * For simplicity we always update the GPRs when a critical section is active
+ * and preemption was *possible*, regardless of whether preemption actually
+ * occurred. Where preemption did not occur, the updates are redundant but not
+ * harmful.
+ */
+static __always_inline void irqentry_exit_pcpu_adjust(struct pt_regs *regs)
+{
+	int reg_pcp, reg_off, reg_addr;
+	unsigned long pcp, off, addr;
+	u16 gprs = regs->pcpu_gprs;
+
+	/*
+	 * Zero means no active PCPU GPRs. As the PCPU GPRs must be distinct,
+	 * a PCPU critical section cannot possibly use {x0,x0,x0}.
+	 */
+	if (likely(!gprs))
+		return;
+
+	reg_pcp  = FIELD_GET(PCPU_GPR_PCP,  gprs);
+	reg_off  = FIELD_GET(PCPU_GPR_OFF,  gprs);
+	reg_addr = FIELD_GET(PCPU_GPR_ADDR, gprs);
+
+	pcp = pt_regs_read_reg(regs, reg_pcp);
+
+	off = __kern_my_cpu_offset();
+	pt_regs_write_reg(regs, reg_off, off);
+
+	addr = pcp + off;
+	pt_regs_write_reg(regs, reg_addr, addr);
+}
+
 /*
  * Handle IRQ/context state management when entering from kernel mode.
  * Before this function is called it is not safe to call regular kernel code,
@@ -56,6 +93,7 @@ static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs,
 						irqentry_state_t state)
 {
 	local_daif_mask();
+	irqentry_exit_pcpu_adjust(regs);
 	mte_check_tfsr_exit();
 	irqentry_exit_to_kernel_mode_after_preempt(regs, state);
 }
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index f63049ac32dc6c..8d17f61d1b5b2c 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -194,6 +194,17 @@ alternative_cb_end
 #endif
 	.endm
 
+	.macro pcpu_gprs_entry, tsk:req, regs:req, tmp:req
+	ldrh	w\tmp, [\tsk, #TSK_TI_PCPU_GPRS]
+	strh	w\tmp, [\regs, #S_PCPU_GPRS]
+	strh	wzr, [\tsk, #TSK_TI_PCPU_GPRS]
+	.endm
+
+	.macro pcpu_gprs_exit, tsk:req, regs:req, tmp:req
+	ldrh	w\tmp, [\regs, #S_PCPU_GPRS]
+	strh	w\tmp, [\tsk, #TSK_TI_PCPU_GPRS]
+	.endm
+
 	.macro	kernel_entry, el, regsize = 64
 	.if	\el == 0
 	alternative_insn nop, SET_PSTATE_DIT(1), ARM64_HAS_DIT
@@ -277,6 +288,7 @@ alternative_else_nop_endif
 	.else
 	add	x21, sp, #PT_REGS_SIZE
 	get_current_task tsk
+	pcpu_gprs_entry	tsk, sp, x0
 	.endif /* \el == 0 */
 	mrs	x22, elr_el1
 	mrs	x23, spsr_el1
@@ -333,6 +345,10 @@ alternative_else_nop_endif
 	.endm
 
 	.macro	kernel_exit, el
+	.if	\el != 0
+	pcpu_gprs_exit	tsk, sp, x0
+	.endif
+
 #ifdef CONFIG_ARM64_PSEUDO_NMI
 alternative_if_not ARM64_HAS_GIC_PRIO_MASKING
 	b	.Lskip_pmr_restore\@
@@ -1023,11 +1039,12 @@ SYM_CODE_START(__sdei_asm_handler)
 
 	/*
 	 * We may have interrupted userspace, or a guest, or exit-from or
-	 * return-to either of these. We can't trust sp_el0, restore it.
+	 * return-to either of these. Preserve the interrupted sp_el0, and
+	 * initialize sp_el0 to the current task.
 	 */
-	mrs	x28, sp_el0
-	ldr_this_cpu	dst=x0, sym=__entry_task, tmp=x1
-	msr	sp_el0, x0
+	mrs	x20, sp_el0
+	ldr_this_cpu    dst=tsk, sym=__entry_task, tmp=x1
+	msr	sp_el0, tsk
 
 	/* If we interrupted the kernel point to the previous stack/frame. */
 	and     x0, x3, #0xc
@@ -1039,15 +1056,25 @@ SYM_CODE_START(__sdei_asm_handler)
 	stp	x29, x4, [sp, #-16]!
 	mov	x29, sp
 
+	add	x16, x19, #SDEI_EVENT_INTREGS
+	pcpu_gprs_entry tsk, x16, x17
+
 	add	x0, x19, #SDEI_EVENT_INTREGS
 	mov	x1, x19
 	bl	__sdei_handler
 
-	msr	sp_el0, x28
-	/* restore regs >x17 that we clobbered */
+	add	x16, x19, #SDEI_EVENT_INTREGS
+	pcpu_gprs_exit tsk, x16, x17
+
+	msr	sp_el0, x20
+	/* restore regs >x17 that firmware won't restore */
 	mov	x4, x19         // keep x4 for __sdei_asm_exit_trampoline
-	ldp	x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14]
 	ldp	x18, x19, [x4, #SDEI_EVENT_INTREGS + 16 * 9]
+	ldp	x20, x21, [x4, #SDEI_EVENT_INTREGS + 16 * 10]
+	ldp	x22, x23, [x4, #SDEI_EVENT_INTREGS + 16 * 11]
+	ldp	x24, x25, [x4, #SDEI_EVENT_INTREGS + 16 * 12]
+	ldp	x26, x27, [x4, #SDEI_EVENT_INTREGS + 16 * 13]
+	ldp	x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14]
 	ldp	lr, x1, [x4, #SDEI_EVENT_INTREGS + S_LR]
 	mov	sp, x1
 
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index e7f1682a3059b5..65e7147067aa3c 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -621,23 +621,6 @@ static int __init sme_sysctl_init(void) { return 0; }
 #define ZREG(sve_state, vq, n) ((char *)(sve_state) +		\
 	(SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
 
-#ifdef CONFIG_CPU_BIG_ENDIAN
-static __uint128_t arm64_cpu_to_le128(__uint128_t x)
-{
-	u64 a = swab64(x);
-	u64 b = swab64(x >> 64);
-
-	return ((__uint128_t)a << 64) | b;
-}
-#else
-static __uint128_t arm64_cpu_to_le128(__uint128_t x)
-{
-	return x;
-}
-#endif
-
-#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
-
 static void __fpsimd_to_sve(struct arm64_sve_state *sst,
 			    struct user_fpsimd_state const *fst,
 			    unsigned int vq)
@@ -647,7 +630,7 @@ static void __fpsimd_to_sve(struct arm64_sve_state *sst,
 
 	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
 		p = (__uint128_t *)ZREG(sst, vq, i);
-		*p = arm64_cpu_to_le128(fst->vregs[i]);
+		*p = fst->vregs[i];
 	}
 }
 
@@ -702,7 +685,7 @@ static inline void sve_to_fpsimd(struct task_struct *task)
 	vq = sve_vq_from_vl(vl);
 	for (i = 0; i < SVE_NUM_ZREGS; ++i) {
 		p = (__uint128_t const *)ZREG(sst, vq, i);
-		fst->vregs[i] = arm64_le128_to_cpu(*p);
+		fst->vregs[i] = *p;
 	}
 }
 
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 87a822e5c4ca83..8951ce69355206 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -138,8 +138,7 @@ SYM_CODE_START_LOCAL(record_mmu_state)
 	b.ne	0f
 	mrs	x19, sctlr_el2
 0:
-CPU_LE( tbnz	x19, #SCTLR_ELx_EE_SHIFT, 1f	)
-CPU_BE( tbz	x19, #SCTLR_ELx_EE_SHIFT, 1f	)
+	tbnz	x19, #SCTLR_ELx_EE_SHIFT, 1f
 	tst	x19, #SCTLR_ELx_C		// Z := (C == 0)
 	and	x19, x19, #SCTLR_ELx_M		// isolate M bit
 	csel	x19, xzr, x19, eq		// clear x19 if Z
diff --git a/arch/arm64/kernel/image.h b/arch/arm64/kernel/image.h
index 7bc3ba89790191..5c10ca77336388 100644
--- a/arch/arm64/kernel/image.h
+++ b/arch/arm64/kernel/image.h
@@ -24,28 +24,14 @@
  * when PIE is in effect. So we need to split them up in 32-bit high and low
  * words.
  */
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define DATA_LE32(data)				\
-	((((data) & 0x000000ff) << 24) |	\
-	 (((data) & 0x0000ff00) << 8)  |	\
-	 (((data) & 0x00ff0000) >> 8)  |	\
-	 (((data) & 0xff000000) >> 24))
-#else
-#define DATA_LE32(data) ((data) & 0xffffffff)
-#endif
-
 #define DEFINE_IMAGE_LE64(sym, data)				\
-	sym##_lo32 = DATA_LE32((data) & 0xffffffff);		\
-	sym##_hi32 = DATA_LE32((data) >> 32)
+	sym##_lo32 = (data) & 0xffffffff;			\
+	sym##_hi32 = (data) >> 32
 
 #define __HEAD_FLAG(field)	(__HEAD_FLAG_##field << \
 					ARM64_IMAGE_FLAG_##field##_SHIFT)
 
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define __HEAD_FLAG_BE		ARM64_IMAGE_FLAG_BE
-#else
 #define __HEAD_FLAG_BE		ARM64_IMAGE_FLAG_LE
-#endif
 
 #define __HEAD_FLAG_PAGE_SIZE	((PAGE_SHIFT - 10) / 2)
 
diff --git a/arch/arm64/kernel/kexec_image.c b/arch/arm64/kernel/kexec_image.c
index b70f4df15a1ae5..78b1f956d8d724 100644
--- a/arch/arm64/kernel/kexec_image.c
+++ b/arch/arm64/kernel/kexec_image.c
@@ -25,10 +25,10 @@ static int image_probe(const char *kernel_buf, unsigned long kernel_len)
 		(const struct arm64_image_header *)(kernel_buf);
 
 	if (!h || (kernel_len < sizeof(*h)))
-		return -EINVAL;
+		return -ENOEXEC;
 
 	if (memcmp(&h->magic, ARM64_IMAGE_MAGIC, sizeof(h->magic)))
-		return -EINVAL;
+		return -ENOEXEC;
 
 	return 0;
 }
@@ -40,7 +40,7 @@ static void *image_load(struct kimage *image,
 {
 	struct arm64_image_header *h;
 	u64 flags, value;
-	bool be_image, be_kernel;
+	bool be_image;
 	struct kexec_buf kbuf = {};
 	unsigned long text_offset, kernel_segment_number;
 	struct kexec_segment *kernel_segment;
@@ -58,8 +58,7 @@ static void *image_load(struct kimage *image,
 	/* Check cpu features */
 	flags = le64_to_cpu(h->flags);
 	be_image = arm64_image_flag_field(flags, ARM64_IMAGE_FLAG_BE);
-	be_kernel = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
-	if ((be_image != be_kernel) && !system_supports_mixed_endian())
+	if (be_image && !system_supports_mixed_endian())
 		return ERR_PTR(-EINVAL);
 
 	value = arm64_image_flag_field(flags, ARM64_IMAGE_FLAG_PAGE_SIZE);
diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c
index 968324a79a8972..3426d0908973ab 100644
--- a/arch/arm64/kernel/kgdb.c
+++ b/arch/arm64/kernel/kgdb.c
@@ -60,11 +60,7 @@ struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = {
 	 * 32-bits. Look for the big comment in asm/kgdb.h for more
 	 * detail.
 	 */
-	{ "pstate", 4, offsetof(struct pt_regs, pstate)
-#ifdef CONFIG_CPU_BIG_ENDIAN
-							+ 4
-#endif
-	},
+	{ "pstate", 4, offsetof(struct pt_regs, pstate)},
 	{ "v0", 16, -1 },
 	{ "v1", 16, -1 },
 	{ "v2", 16, -1 },
diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c
index 572efb96b23fec..30bf61d031eb7e 100644
--- a/arch/arm64/kernel/paravirt.c
+++ b/arch/arm64/kernel/paravirt.c
@@ -157,9 +157,9 @@ int __init pv_time_init(void)
 
 	static_call_update(pv_steal_clock, para_steal_clock);
 
-	static_key_slow_inc(&paravirt_steal_enabled);
+	static_branch_inc(&paravirt_steal_enabled);
 	if (steal_acc)
-		static_key_slow_inc(&paravirt_steal_rq_enabled);
+		static_branch_inc(&paravirt_steal_rq_enabled);
 
 	pr_info("using stolen time PV\n");
 
diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c
index b4eece3eb17d03..71a3e0238de419 100644
--- a/arch/arm64/kernel/perf_regs.c
+++ b/arch/arm64/kernel/perf_regs.c
@@ -77,7 +77,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 
 #define REG_RESERVED (~((1ULL << PERF_REG_ARM64_MAX) - 1))
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	u64 reserved_mask = REG_RESERVED;
 
@@ -98,9 +98,3 @@ u64 perf_reg_abi(struct task_struct *task)
 		return PERF_SAMPLE_REGS_ABI_64;
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
diff --git a/arch/arm64/kernel/pi/Makefile b/arch/arm64/kernel/pi/Makefile
index be92d73c25b219..a273ab5af805c6 100644
--- a/arch/arm64/kernel/pi/Makefile
+++ b/arch/arm64/kernel/pi/Makefile
@@ -9,7 +9,7 @@ KBUILD_CFLAGS	:= $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) -fpie \
 		   -include $(srctree)/include/linux/hidden.h \
 		   -D__DISABLE_EXPORTS -ffreestanding -D__NO_FORTIFY \
 		   -fno-asynchronous-unwind-tables -fno-unwind-tables \
-		   $(call cc-option,-fno-addrsig)
+		   $(CONFIG_CC_OPT_NO_ADDRSIG)
 
 # this code may run with the MMU off so disable unaligned accesses
 CFLAGS_map_range.o += -mstrict-align
@@ -34,6 +34,7 @@ $(obj)/lib-%.pi.o: OBJCOPYFLAGS += --prefix-alloc-sections=.init
 
 $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE
 	$(call if_changed_rule,cc_o_c)
+	$(call cmd,force_checksrc)
 
 obj-y					:= idreg-override.pi.o \
 					   map_kernel.pi.o map_range.pi.o \
diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c
index 274bc72ba98d0f..acaf56234973b9 100644
--- a/arch/arm64/kernel/pi/idreg-override.c
+++ b/arch/arm64/kernel/pi/idreg-override.c
@@ -186,6 +186,7 @@ static const struct ftr_set_desc isar2 __prel64_initconst = {
 	.name		= "id_aa64isar2",
 	.override	= &id_aa64isar2_override,
 	.fields		= {
+		FIELD("wfxt", ID_AA64ISAR2_EL1_WFxT_SHIFT, NULL),
 		FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL),
 		FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL),
 		FIELD("mops", ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL),
@@ -254,6 +255,7 @@ static const struct {
 	  "id_aa64isar1.api=0 id_aa64isar1.apa=0 "
 	  "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0"	   },
 	{ "arm64.nomops",		"id_aa64isar2.mops=0" },
+	{ "arm64.nowfxt",		"id_aa64isar2.wfxt=0" },
 	{ "arm64.nomte",		"id_aa64pfr1.mte=0" },
 	{ "nokaslr",			"arm64_sw.nokaslr=1" },
 	{ "rodata=off",			"arm64_sw.rodataoff=1" },
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index f743cbec1c3ad3..7405c76724bfec 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -2517,12 +2517,8 @@ static int valid_compat_regs(struct user_pt_regs *regs)
 {
 	regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS;
 
-	if (!system_supports_mixed_endian_el0()) {
-		if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
-			regs->pstate |= PSR_AA32_E_BIT;
-		else
-			regs->pstate &= ~PSR_AA32_E_BIT;
-	}
+	if (!system_supports_mixed_endian_el0())
+		regs->pstate &= ~PSR_AA32_E_BIT;
 
 	if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
 	    (regs->pstate & PSR_AA32_A_BIT) == 0 &&
diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c
deleted file mode 100644
index 25ca75ce1a4dff..00000000000000
--- a/arch/arm64/kernel/rsi.c
+++ /dev/null
@@ -1,164 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2023 ARM Ltd.
- */
-
-#include <linux/jump_label.h>
-#include <linux/memblock.h>
-#include <linux/psci.h>
-#include <linux/swiotlb.h>
-#include <linux/platform_device.h>
-
-#include <asm/io.h>
-#include <asm/mem_encrypt.h>
-#include <asm/pgtable.h>
-#include <asm/rsi.h>
-
-static struct realm_config config;
-
-unsigned long prot_ns_shared;
-EXPORT_SYMBOL(prot_ns_shared);
-
-DEFINE_STATIC_KEY_FALSE_RO(rsi_present);
-EXPORT_SYMBOL(rsi_present);
-
-static bool rsi_version_matches(void)
-{
-	unsigned long ver_lower, ver_higher;
-	unsigned long ret = rsi_request_version(RSI_ABI_VERSION,
-						&ver_lower,
-						&ver_higher);
-
-	if (ret == SMCCC_RET_NOT_SUPPORTED)
-		return false;
-
-	if (ret != RSI_SUCCESS) {
-		pr_err("RME: RMM doesn't support RSI version %lu.%lu. Supported range: %lu.%lu-%lu.%lu\n",
-		       RSI_ABI_VERSION_MAJOR, RSI_ABI_VERSION_MINOR,
-		       RSI_ABI_VERSION_GET_MAJOR(ver_lower),
-		       RSI_ABI_VERSION_GET_MINOR(ver_lower),
-		       RSI_ABI_VERSION_GET_MAJOR(ver_higher),
-		       RSI_ABI_VERSION_GET_MINOR(ver_higher));
-		return false;
-	}
-
-	pr_info("RME: Using RSI version %lu.%lu\n",
-		RSI_ABI_VERSION_GET_MAJOR(ver_lower),
-		RSI_ABI_VERSION_GET_MINOR(ver_lower));
-
-	return true;
-}
-
-static void __init arm64_rsi_setup_memory(void)
-{
-	u64 i;
-	phys_addr_t start, end;
-
-	/*
-	 * Iterate over the available memory ranges and convert the state to
-	 * protected memory. We should take extra care to ensure that we DO NOT
-	 * permit any "DESTROYED" pages to be converted to "RAM".
-	 *
-	 * panic() is used because if the attempt to switch the memory to
-	 * protected has failed here, then future accesses to the memory are
-	 * simply going to be reflected as a SEA (Synchronous External Abort)
-	 * which we can't handle.  Bailing out early prevents the guest limping
-	 * on and dying later.
-	 */
-	for_each_mem_range(i, &start, &end) {
-		if (rsi_set_memory_range_protected_safe(start, end)) {
-			panic("Failed to set memory range to protected: %pa-%pa",
-			      &start, &end);
-		}
-	}
-}
-
-/*
- * Check if a given PA range is Trusted (e.g., Protected memory, a Trusted Device
- * mapping, or an MMIO emulated in the Realm world).
- *
- * We can rely on the RIPAS value of the region to detect if a given region is
- * protected.
- *
- *  RIPAS_DEV - A trusted device memory or a trusted emulated MMIO (in the Realm
- *		world
- *  RIPAS_RAM - Memory (RAM), protected by the RMM guarantees. (e.g., Firmware
- *		reserved regions for data sharing).
- *
- *  RIPAS_DESTROYED is a special case of one of the above, where the host did
- *  something without our permission and as such we can't do anything about it.
- *
- * The only case where something is emulated by the untrusted hypervisor or is
- * backed by shared memory is indicated by RSI_RIPAS_EMPTY.
- */
-bool arm64_rsi_is_protected(phys_addr_t base, size_t size)
-{
-	enum ripas ripas;
-	phys_addr_t end, top;
-
-	/* Overflow ? */
-	if (WARN_ON(base + size <= base))
-		return false;
-
-	end = ALIGN(base + size, RSI_GRANULE_SIZE);
-	base = ALIGN_DOWN(base, RSI_GRANULE_SIZE);
-
-	while (base < end) {
-		if (WARN_ON(rsi_ipa_state_get(base, end, &ripas, &top)))
-			break;
-		if (WARN_ON(top <= base))
-			break;
-		if (ripas == RSI_RIPAS_EMPTY)
-			break;
-		base = top;
-	}
-
-	return base >= end;
-}
-EXPORT_SYMBOL(arm64_rsi_is_protected);
-
-static int realm_ioremap_hook(phys_addr_t phys, size_t size, pgprot_t *prot)
-{
-	if (arm64_rsi_is_protected(phys, size))
-		*prot = pgprot_encrypted(*prot);
-	else
-		*prot = pgprot_decrypted(*prot);
-
-	return 0;
-}
-
-void __init arm64_rsi_init(void)
-{
-	if (arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_SMC)
-		return;
-	if (!rsi_version_matches())
-		return;
-	if (WARN_ON(rsi_get_realm_config(lm_alias(&config))))
-		return;
-	prot_ns_shared = __phys_to_pte_val(BIT(config.ipa_bits - 1));
-
-	if (arm64_ioremap_prot_hook_register(realm_ioremap_hook))
-		return;
-
-	if (realm_register_memory_enc_ops())
-		return;
-
-	arm64_rsi_setup_memory();
-
-	static_branch_enable(&rsi_present);
-}
-
-static struct platform_device rsi_dev = {
-	.name = RSI_PDEV_NAME,
-	.id = PLATFORM_DEVID_NONE
-};
-
-static int __init arm64_create_dummy_rsi_dev(void)
-{
-	if (is_realm_world() &&
-	    platform_device_register(&rsi_dev))
-		pr_err("failed to register rsi platform device\n");
-	return 0;
-}
-
-arch_initcall(arm64_create_dummy_rsi_dev)
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 29c6100f0c50b2..a707deba6ae701 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/arm-rsi-cmds.h>
 #include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/stddef.h>
@@ -43,7 +44,6 @@
 #include <asm/cpu_ops.h>
 #include <asm/kasan.h>
 #include <asm/numa.h>
-#include <asm/rsi.h>
 #include <asm/scs.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index bb3b526ff43f71..3e1ce0b00a6802 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -70,22 +70,12 @@ static inline int get_sigset_t(sigset_t *set,
 
 /*
  * VFP save/restore code.
- *
- * We have to be careful with endianness, since the fpsimd context-switch
- * code operates on 128-bit (Q) register values whereas the compat ABI
- * uses an array of 64-bit (D) registers. Consequently, we need to swap
- * the two halves of each Q register when running on a big-endian CPU.
  */
 union __fpsimd_vreg {
 	__uint128_t	raw;
 	struct {
-#ifdef __AARCH64EB__
-		u64	hi;
-		u64	lo;
-#else
 		u64	lo;
 		u64	hi;
-#endif
 	};
 };
 
@@ -334,9 +324,6 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
 	/* The IT state must be cleared for both ARM and Thumb-2 */
 	spsr &= ~PSR_AA32_IT_MASK;
 
-	/* Restore the original endianness */
-	spsr |= PSR_AA32_ENDSTATE;
-
 	if (ka->sa.sa_flags & SA_RESTORER) {
 		retcode = ptr_to_compat(ka->sa.sa_restorer);
 	} else {
diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c
index 12a948f3a50437..8ac07aeb018df5 100644
--- a/arch/arm64/kernel/sys32.c
+++ b/arch/arm64/kernel/sys32.c
@@ -59,12 +59,7 @@ COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len,
 	return ksys_mmap_pgoff(addr, len, prot, flags, fd, off_4k);
 }
 
-#ifdef CONFIG_CPU_BIG_ENDIAN
-#define arg_u32p(name)	u32, name##_hi, u32, name##_lo
-#else
 #define arg_u32p(name)	u32, name##_lo, u32, name##_hi
-#endif
-
 #define arg_u64(name)	(((u64)name##_hi << 32) | name##_lo)
 
 COMPAT_SYSCALL_DEFINE6(aarch32_pread64, unsigned int, fd, char __user *, buf,
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index d28438f8b83f12..928079332fe622 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -186,7 +186,6 @@ int arch_freq_get_on_cpu(int cpu)
 	struct amu_cntr_sample *amu_sample;
 	unsigned int start_cpu = cpu;
 	unsigned long last_update;
-	unsigned int freq = 0;
 	u64 scale;
 
 	if (!amu_fie_cpu_supported(cpu) || !arch_scale_freq_ref(cpu))
@@ -245,9 +244,8 @@ int arch_freq_get_on_cpu(int cpu)
 	 * (see amu_scale_freq_tick for details)
 	 */
 	scale = arch_scale_freq_capacity(cpu);
-	freq = scale * arch_scale_freq_ref(cpu);
-	freq >>= SCHED_CAPACITY_SHIFT;
-	return freq;
+
+	return (scale * arch_scale_freq_ref(cpu)) >> SCHED_CAPACITY_SHIFT;
 }
 
 static void amu_fie_setup(const struct cpumask *cpus)
@@ -427,7 +425,7 @@ int counters_read_on_cpu(int cpu, smp_call_func_t func, void *val)
 			return -EPERM;
 		func(val);
 	} else {
-		smp_call_function_single(cpu, func, val, 1);
+		return smp_call_function_single(cpu, func, val, 1);
 	}
 
 	return 0;
@@ -470,6 +468,12 @@ static void amu_read_core_const_ctrs(void *val)
 	cpu_read_corecnt(&ctrs->corecnt);
 }
 
+static bool cpc_ffh_reg_valid(const struct cpc_reg *reg)
+{
+	return reg->bit_width && reg->bit_width <= 64 &&
+	       reg->bit_offset <= 64 - reg->bit_width;
+}
+
 static u64 cpc_ffh_extract_bits(const struct cpc_reg *reg, u64 val)
 {
 	val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
@@ -506,7 +510,8 @@ int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1,
 	struct amu_ffh_ctrs ctrs;
 	int ret;
 
-	if (!is_amu_ctr_reg(reg1) || !is_amu_ctr_reg(reg2))
+	if (!is_amu_ctr_reg(reg1) || !is_amu_ctr_reg(reg2) ||
+	    !cpc_ffh_reg_valid(reg1) || !cpc_ffh_reg_valid(reg2))
 		return -EINVAL;
 
 	ret = counters_read_on_cpu(cpu, amu_read_core_const_ctrs, &ctrs);
@@ -530,6 +535,9 @@ int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)
 {
 	int ret = -EOPNOTSUPP;
 
+	if (!cpc_ffh_reg_valid(reg))
+		return -EINVAL;
+
 	switch ((u64)reg->address) {
 	case CPC_FFH_CTR_CORE:
 		ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val);
diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile
index bea3675fa668bd..00efdc5d23543d 100644
--- a/arch/arm64/kernel/vdso32/Makefile
+++ b/arch/arm64/kernel/vdso32/Makefile
@@ -45,11 +45,7 @@ endif
 # From arm Makefile
 VDSO_CAFLAGS += $(call cc32-option,-fno-dwarf2-cfi-asm)
 VDSO_CAFLAGS += -mabi=aapcs-linux -mfloat-abi=soft
-ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
-VDSO_CAFLAGS += -mbig-endian
-else
 VDSO_CAFLAGS += -mlittle-endian
-endif
 
 # From arm vDSO Makefile
 VDSO_CAFLAGS += -fPIC -fno-builtin -fno-stack-protector
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index af1d720209764f..c0b090db8be43b 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -15,7 +15,7 @@
 
 #ifdef CONFIG_NVHE_EL2_TRACING
 #define HYPERVISOR_EVENT_IDS 					\
-	. = ALIGN(PAGE_SIZE);					\
+	. = ALIGN(SZ_8);					\
 	__hyp_event_ids_start = .;				\
 	*(HYP_SECTION_NAME(.event_ids))				\
 	__hyp_event_ids_end = .;
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 449154f9a4852d..71233068b7cb68 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -37,6 +37,7 @@ menuconfig KVM
 	select SCHED_INFO
 	select GUEST_PERF_EVENTS if PERF_EVENTS
 	select KVM_GUEST_MEMFD
+	select KVM_GENERIC_PRE_FAULT_MEMORY
 	help
 	  Support hosting virtualized guest machines.
 
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 59612d2f277c1d..7aaeeb84e788ec 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -24,7 +24,8 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
 	 vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \
 	 vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \
 	 vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \
-	 vgic/vgic-v5.o
+	 vgic/vgic-v5.o vgic/vgic-v5-tables.o vgic/vgic-irs-v5.o \
+	 vgic-sys-reg-v5.o
 
 kvm-$(CONFIG_HW_PERF_EVENTS)  += pmu-emul.o pmu.o
 kvm-$(CONFIG_ARM64_PTR_AUTH)  += pauth.o
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 8b080804bc90b2..ae1b87c9195da7 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -236,8 +236,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	mutex_unlock(&kvm->lock);
 #endif
 
-	kvm_init_nested(kvm);
-
 	ret = kvm_share_hyp(kvm, kvm + 1);
 	if (ret)
 		return ret;
@@ -252,6 +250,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	if (ret)
 		goto err_free_cpumask;
 
+	ret = kvm_init_nested(kvm);
+	if (ret)
+		goto err_uninit_mmu;
+
 	if (is_protected_kvm_enabled()) {
 		/*
 		 * If any failures occur after this is successful, make sure to
@@ -280,6 +282,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
 err_uninit_mmu:
 	kvm_uninit_stage2_mmu(kvm);
+	kvm_destroy_nested(kvm);
 err_free_cpumask:
 	free_cpumask_var(kvm->arch.supported_cpus);
 err_unshare_kvm:
@@ -337,6 +340,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 
 	kvm_unshare_hyp(kvm, kvm + 1);
 
+	kvm_destroy_nested(kvm);
 	kvm_arm_teardown_hypercalls(kvm);
 }
 
@@ -406,6 +410,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_COUNTER_OFFSET:
 	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
 	case KVM_CAP_ARM_SEA_TO_USER:
+	case KVM_CAP_PRE_FAULT_MEMORY:
 		r = 1;
 		break;
 	case KVM_CAP_SET_GUEST_DEBUG2:
@@ -949,7 +954,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
 			return ret;
 	}
 
-	ret = kvm_finalize_sys_regs(vcpu);
+	ret = kvm_vcpu_finalize_sys_regs(vcpu);
 	if (ret)
 		return ret;
 
@@ -979,7 +984,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
 			return ret;
 	}
 
-	ret = vgic_v5_finalize_ppi_state(kvm);
+	ret = vgic_v5_finalize_ppi_state(vcpu);
 	if (ret)
 		return ret;
 
@@ -1129,7 +1134,7 @@ static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
 static int check_vcpu_requests(struct kvm_vcpu *vcpu)
 {
 	if (kvm_request_pending(vcpu)) {
-		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
+		if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu))
 			return -EIO;
 
 		if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
@@ -1159,6 +1164,14 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu)
 			preempt_enable();
 		}
 
+		if (kvm_check_request(KVM_REQ_RELOAD_GICv5, vcpu)) {
+			/* The IRS enable bit was changed */
+			preempt_disable();
+			vgic_v5_put(vcpu);
+			vgic_v5_load(vcpu);
+			preempt_enable();
+		}
+
 		if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
 			kvm_vcpu_reload_pmu(vcpu);
 
diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
index 625604019fb321..f59d89d04ecaf6 100644
--- a/arch/arm64/kvm/emulate-nested.c
+++ b/arch/arm64/kvm/emulate-nested.c
@@ -1445,7 +1445,7 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = {
 	SR_FGT(OP_AT_S1E1A, 		HFGITR, ATS1E1A, 1),
 	SR_FGT(OP_COSP_RCTX, 		HFGITR, COSPRCTX, 1),
 	SR_FGT(OP_GCSPUSHX, 		HFGITR, nGCSEPP, 0),
-	SR_FGT(OP_GCSPOPX, 		HFGITR, nGCSEPP, 0),
+	SR_FGT(OP_GCSPOPCX, 		HFGITR, nGCSEPP, 0),
 	SR_FGT(OP_GCSPUSHM, 		HFGITR, nGCSPUSHM_EL1, 0),
 	SR_FGT(OP_BRB_IALL, 		HFGITR, nBRBIALL, 0),
 	SR_FGT(OP_BRB_INJ, 		HFGITR, nBRBINJ, 0),
@@ -2386,9 +2386,6 @@ int __init populate_nv_trap_config(void)
 				print_nv_trap_error(fgt, "FGT bit is reserved", ret);
 			}
 
-			if (!cpus_have_final_cap(ARM64_HAS_FGT))
-				continue;
-
 			prev = xa_store(&sr_forward_xa, enc,
 					xa_mk_value(tc.val), GFP_KERNEL);
 
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index 308100ed25de9d..9709905f21b5e6 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -46,7 +46,7 @@ alternative_else_nop_endif
 
 	// Ensure that __guest_enter() always provides a context
 	// synchronization event so that callers don't need ISBs for anything
-	// that would usually be synchonized by the ERET.
+	// that would usually be synchronized by the ERET.
 	isb
 	mov	x0, #ARM_EXCEPTION_IRQ
 	ret
diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
index 29935c7da1dec2..ec85a95471207b 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
@@ -52,8 +52,10 @@ int __pkvm_host_test_clear_young_guest(u64 gfn, u64 nr_pages, bool mkold, struct
 int __pkvm_host_mkyoung_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu);
 
 bool addr_is_memory(phys_addr_t phys);
+bool addr_is_hyp_text(phys_addr_t phys);
 int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot);
 int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id);
+bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte);
 int kvm_host_prepare_stage2(void *pgt_pool_base);
 int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd);
 void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm);
diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h
index 6e83ce35c2f2e0..31cae95ddb7165 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/mm.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h
@@ -29,6 +29,7 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
 				  enum kvm_pgtable_prot prot,
 				  unsigned long *haddr);
 int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr);
+int pkvm_check_host_ownership(void);
 int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr);
 
 #endif /* __KVM_HYP_MM_H */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
index c904647d2f7606..78525f16c809d5 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
@@ -7,11 +7,21 @@
 #ifndef __ARM64_KVM_NVHE_PKVM_H__
 #define __ARM64_KVM_NVHE_PKVM_H__
 
+#include <asm/kvm_hcall.h>
 #include <asm/kvm_pkvm.h>
 
 #include <nvhe/gfp.h>
 #include <nvhe/spinlock.h>
 
+/*
+ * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. The rest
+ * are EL2's own and nothing the host sets there reaches the guest.
+ */
+#define PKVM_HCR_EL2_HOST_PVM	(HCR_EL2_TWI | HCR_EL2_TWE | HCR_EL2_VSE)
+#define PKVM_HCR_EL2_HOST_NPVM	(PKVM_HCR_EL2_HOST_PVM | HCR_EL2_VI | HCR_EL2_VF |	\
+				 HCR_EL2_TVM | HCR_EL2_TID2 | HCR_EL2_TID4 |		\
+				 HCR_EL2_TID5 | HCR_EL2_TTLBOS)
+
 /*
  * Holds the relevant data for maintaining the vcpu state completely at hyp.
  */
@@ -69,10 +79,10 @@ void pkvm_hyp_vm_table_init(void *tbl);
 
 int __pkvm_reserve_vm(void);
 void __pkvm_unreserve_vm(pkvm_handle_t handle);
-int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
-		   unsigned long pgd_hva);
+int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva,
+		   void __kern *pgd_hva);
 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
-		     unsigned long vcpu_hva);
+		     void __kern *vcpu_hva);
 
 int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn);
 int __pkvm_start_teardown_vm(pkvm_handle_t handle);
diff --git a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h
index 7c7ea8c55405f5..b6978f03e86aa1 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h
@@ -20,11 +20,7 @@
 typedef union hyp_spinlock {
 	u32	__val;
 	struct {
-#ifdef __AARCH64EB__
-		u16 next, owner;
-#else
 		u16 owner, next;
-#endif
 	};
 } hyp_spinlock_t;
 
diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h
index 8813ff250f8e07..c2db9f70ea265a 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/trace.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h
@@ -4,6 +4,7 @@
 
 #include <linux/trace_remote_event.h>
 
+#include <asm/kvm_hcall.h>
 #include <asm/kvm_hyptrace.h>
 
 static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt)
@@ -46,7 +47,7 @@ static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt)
 void *tracing_reserve_entry(unsigned long length);
 void tracing_commit_entry(void);
 
-int __tracing_load(unsigned long desc_va, size_t desc_size);
+int __tracing_load(void __kern *desc_va, size_t desc_size);
 void __tracing_unload(void);
 int __tracing_enable(bool enable);
 int __tracing_swap_reader(unsigned int cpu);
@@ -59,7 +60,7 @@ static inline void tracing_commit_entry(void) { }
 #define HYP_EVENT(__name, __proto, __struct, __assign, __printk)      \
 	static inline void trace_##__name(__proto) {}
 
-static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; }
+static inline int __tracing_load(void __kern *desc_va, size_t desc_size) { return -ENODEV; }
 static inline void __tracing_unload(void) { }
 static inline int __tracing_enable(bool enable) { return -ENODEV; }
 static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; }
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index f57450ebcb4989..ccc1fe8394094c 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -49,6 +49,7 @@ targets += $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S hyp-relo
 #    avoids file name clashes for files shared with VHE.
 $(obj)/%.nvhe.o: $(src)/%.c FORCE
 	$(call if_changed_rule,cc_o_c)
+	$(call cmd,force_checksrc)
 $(obj)/%.nvhe.o: $(src)/%.S FORCE
 	$(call if_changed_rule,as_o_S)
 
diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c
index add9383aadb5ac..b845be0acd1171 100644
--- a/arch/arm64/kvm/hyp/nvhe/events.c
+++ b/arch/arm64/kvm/hyp/nvhe/events.c
@@ -7,6 +7,8 @@
 #include <nvhe/mm.h>
 #include <nvhe/trace.h>
 
+#include <asm/kvm_hypevents.h>
+
 #include <nvhe/define_events.h>
 
 int __tracing_enable_event(unsigned short id, bool enable)
diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c
index a327c2bbb6b64d..5ad74ab265b578 100644
--- a/arch/arm64/kvm/hyp/nvhe/ffa.c
+++ b/arch/arm64/kvm/hyp/nvhe/ffa.c
@@ -42,6 +42,9 @@
  */
 #define HOST_FFA_ID	0
 
+#define FFA_NOTIF_RECEIVER_ENDP_MASK	GENMASK(15, 0)
+#define FFA_NOTIF_SENDER_ENDP_MASK	GENMASK(31, 16)
+
 /*
  * A buffer to hold the maximum descriptor size we can see from the host,
  * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP
@@ -71,6 +74,21 @@ static u32 hyp_ffa_version;
 static bool has_version_negotiated;
 static hyp_spinlock_t version_lock;
 
+static bool ffa_check_unused_args_sbz(struct kvm_cpu_context *ctxt, int first_reg)
+{
+	DECLARE_REG(u32, func_id, ctxt, 0);
+	int reg, end_reg = 7;
+
+	if (hyp_ffa_version >= FFA_VERSION_1_2 && ARM_SMCCC_IS_64(func_id))
+		end_reg = 17;
+	for (reg = first_reg; reg <= end_reg; reg++) {
+		if (cpu_reg(ctxt, reg))
+			return true;
+	}
+
+	return false;
+}
+
 static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno)
 {
 	*res = (struct arm_smccc_1_2_regs) {
@@ -239,6 +257,11 @@ static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res,
 	int ret = 0;
 	void *rx_virt, *tx_virt;
 
+	if (ffa_check_unused_args_sbz(ctxt, 4)) {
+		ret = FFA_RET_INVALID_PARAMETERS;
+		goto out;
+	}
+
 	if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) {
 		ret = FFA_RET_INVALID_PARAMETERS;
 		goto out;
@@ -315,6 +338,11 @@ static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res,
 	DECLARE_REG(u32, id, ctxt, 1);
 	int ret = 0;
 
+	if (ffa_check_unused_args_sbz(ctxt, 2)) {
+		ret = FFA_RET_INVALID_PARAMETERS;
+		goto out;
+	}
+
 	if (id != HOST_FFA_ID) {
 		ret = FFA_RET_INVALID_PARAMETERS;
 		goto out;
@@ -421,6 +449,11 @@ static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res,
 	int ret = FFA_RET_INVALID_PARAMETERS;
 	u32 nr_ranges;
 
+	if (ffa_check_unused_args_sbz(ctxt, 5)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
 	if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)
 		goto out;
 
@@ -483,6 +516,11 @@ static void __do_ffa_mem_xfer(const u64 func_id,
 	int ret = 0;
 	size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version);
 
+	if (ffa_check_unused_args_sbz(ctxt, 5)) {
+		ret = FFA_RET_INVALID_PARAMETERS;
+		goto out;
+	}
+
 	if (addr_mbz || npages_mbz || fraglen > len ||
 	    fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
 		ret = FFA_RET_INVALID_PARAMETERS;
@@ -586,6 +624,11 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
 	int ret = 0;
 	u64 handle;
 
+	if (ffa_check_unused_args_sbz(ctxt, 4)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
 	handle = PACK_HANDLE(handle_lo, handle_hi);
 
 	hyp_spin_lock(&host_buffers.lock);
@@ -691,14 +734,6 @@ static bool ffa_call_supported(u64 func_id)
 	case FFA_RXTX_MAP:
 	case FFA_MEM_DONATE:
 	case FFA_MEM_RETRIEVE_REQ:
-       /* Optional notification interfaces added in FF-A 1.1 */
-	case FFA_NOTIFICATION_BITMAP_CREATE:
-	case FFA_NOTIFICATION_BITMAP_DESTROY:
-	case FFA_NOTIFICATION_BIND:
-	case FFA_NOTIFICATION_UNBIND:
-	case FFA_NOTIFICATION_SET:
-	case FFA_NOTIFICATION_GET:
-	case FFA_NOTIFICATION_INFO_GET:
 	/* Optional interfaces added in FF-A 1.2 */
 	case FFA_MSG_SEND_DIRECT_REQ2:		/* Optional per 7.5.1 */
 	case FFA_MSG_SEND_DIRECT_RESP2:		/* Optional per 7.5.1 */
@@ -785,6 +820,11 @@ static void do_ffa_version(struct arm_smccc_1_2_regs *res,
 {
 	DECLARE_REG(u32, ffa_req_version, ctxt, 1);
 
+	if (ffa_check_unused_args_sbz(ctxt, 2)) {
+		res->a0 = FFA_RET_NOT_SUPPORTED;
+		return;
+	}
+
 	if (FFA_MAJOR_VERSION(ffa_req_version) != 1) {
 		res->a0 = FFA_RET_NOT_SUPPORTED;
 		return;
@@ -834,6 +874,11 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
 	DECLARE_REG(u32, flags, ctxt, 5);
 	u32 count, partition_sz, copy_sz;
 
+	if (ffa_check_unused_args_sbz(ctxt, 6)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
 	hyp_spin_lock(&host_buffers.lock);
 	if (!host_buffers.rx) {
 		ffa_to_smccc_res(res, FFA_RET_BUSY);
@@ -878,6 +923,140 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
 	hyp_spin_unlock(&host_buffers.lock);
 }
 
+static void do_ffa_notif_bitmap(struct arm_smccc_1_2_regs *res,
+				struct kvm_cpu_context *ctxt)
+{
+	DECLARE_REG(u32, func_id, ctxt, 0);
+	DECLARE_REG(u32, vmid, ctxt, 1);
+	struct arm_smccc_1_2_regs *args;
+
+	if (ffa_check_unused_args_sbz(ctxt, func_id == FFA_NOTIFICATION_BITMAP_CREATE ? 3 : 2)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (vmid != HOST_FFA_ID) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	args = (void *)&ctxt->regs.regs[0];
+	hyp_smccc_1_2_smc(args, res);
+}
+
+static void do_ffa_notif_bind(struct arm_smccc_1_2_regs *res,
+			      struct kvm_cpu_context *ctxt)
+{
+	DECLARE_REG(u32, endp_id, ctxt, 1);
+	DECLARE_REG(u32, flags, ctxt, 2);
+	struct arm_smccc_1_2_regs *args;
+
+	if (ffa_check_unused_args_sbz(ctxt, 5)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (flags > 1) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	args = (void *)&ctxt->regs.regs[0];
+	hyp_smccc_1_2_smc(args, res);
+}
+
+static void do_ffa_notif_unbind(struct arm_smccc_1_2_regs *res,
+				struct kvm_cpu_context *ctxt)
+{
+	DECLARE_REG(u32, endp_id, ctxt, 1);
+	DECLARE_REG(u32, reserved, ctxt, 2);
+	struct arm_smccc_1_2_regs *args;
+
+	if (ffa_check_unused_args_sbz(ctxt, 5) || reserved) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	args = (void *)&ctxt->regs.regs[0];
+	hyp_smccc_1_2_smc(args, res);
+}
+
+static void do_ffa_notif_set(struct arm_smccc_1_2_regs *res,
+			     struct kvm_cpu_context *ctxt)
+{
+	DECLARE_REG(u32, endp_id, ctxt, 1);
+	DECLARE_REG(u32, flags, ctxt, 2);
+	struct arm_smccc_1_2_regs *args;
+
+	if (FIELD_GET(FFA_NOTIF_SENDER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (ffa_check_unused_args_sbz(ctxt, 5)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (flags & GENMASK(15, 2)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	args = (void *)&ctxt->regs.regs[0];
+	hyp_smccc_1_2_smc(args, res);
+}
+
+static void do_ffa_notif_get(struct arm_smccc_1_2_regs *res,
+			     struct kvm_cpu_context *ctxt)
+{
+	DECLARE_REG(u32, endp_id, ctxt, 1);
+	DECLARE_REG(u32, flags, ctxt, 2);
+	struct arm_smccc_1_2_regs *args;
+
+	if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (ffa_check_unused_args_sbz(ctxt, 3)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	if (flags & GENMASK(31, 4)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	args = (void *)&ctxt->regs.regs[0];
+	hyp_smccc_1_2_smc(args, res);
+}
+
+static void do_ffa_notif_info_get(struct arm_smccc_1_2_regs *res,
+				  struct kvm_cpu_context *ctxt)
+{
+	struct arm_smccc_1_2_regs *args;
+
+	if (ffa_check_unused_args_sbz(ctxt, 1)) {
+		ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
+		return;
+	}
+
+	args = (void *)&ctxt->regs.regs[0];
+	hyp_smccc_1_2_smc(args, res);
+}
+
 bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
 {
 	struct arm_smccc_1_2_regs res = {0};
@@ -906,6 +1085,11 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
 
 	switch (func_id) {
 	case FFA_FEATURES:
+		if (ffa_check_unused_args_sbz(host_ctxt, 3)) {
+			ffa_to_smccc_res(&res, FFA_RET_INVALID_PARAMETERS);
+			goto out_handled;
+		}
+
 		if (!do_ffa_features(&res, host_ctxt))
 			return false;
 		goto out_handled;
@@ -936,6 +1120,26 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
 	case FFA_PARTITION_INFO_GET:
 		do_ffa_part_get(&res, host_ctxt);
 		goto out_handled;
+	case FFA_NOTIFICATION_BITMAP_CREATE:
+	case FFA_NOTIFICATION_BITMAP_DESTROY:
+		do_ffa_notif_bitmap(&res, host_ctxt);
+		goto out_handled;
+	case FFA_NOTIFICATION_BIND:
+		do_ffa_notif_bind(&res, host_ctxt);
+		goto out_handled;
+	case FFA_NOTIFICATION_UNBIND:
+		do_ffa_notif_unbind(&res, host_ctxt);
+		goto out_handled;
+	case FFA_NOTIFICATION_SET:
+		do_ffa_notif_set(&res, host_ctxt);
+		goto out_handled;
+	case FFA_NOTIFICATION_GET:
+		do_ffa_notif_get(&res, host_ctxt);
+		goto out_handled;
+	case FFA_NOTIFICATION_INFO_GET:
+	case FFA_FN64_NOTIFICATION_INFO_GET:
+		do_ffa_notif_info_get(&res, host_ctxt);
+		goto out_handled;
 	}
 
 	if (ffa_call_supported(func_id))
diff --git a/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c b/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
index b63f4e1c10330d..99243e0fefb0f2 100644
--- a/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
+++ b/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
@@ -139,28 +139,12 @@ static struct {
 	const char	*sh_string;
 } elf;
 
-#if defined(CONFIG_CPU_LITTLE_ENDIAN)
-
 #define elf16toh(x)	le16toh(x)
 #define elf32toh(x)	le32toh(x)
 #define elf64toh(x)	le64toh(x)
 
 #define ELFENDIAN	ELFDATA2LSB
 
-#elif defined(CONFIG_CPU_BIG_ENDIAN)
-
-#define elf16toh(x)	be16toh(x)
-#define elf32toh(x)	be32toh(x)
-#define elf64toh(x)	be64toh(x)
-
-#define ELFENDIAN	ELFDATA2MSB
-
-#else
-
-#error PDP-endian sadly unsupported...
-
-#endif
-
 #define fatal_error(fmt, ...)						\
 	({								\
 		fprintf(stderr, "error: %s: " fmt "\n",			\
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 9a3b92e626adba..a776d5f4cd545d 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -26,6 +26,48 @@
 
 DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
 
+/*
+ * Define a hypercall handler: handle_<name> unmarshals the arguments from
+ * the host context and hands them, correctly typed, to the body that
+ * follows the macro. The parameter list is type-checked against the
+ * signature declared in <asm/kvm_hcall.h>, so the handler cannot drift
+ * from what the typed caller stubs marshal in. Modelled on the syscall
+ * wrappers.
+ */
+/* Truncate the fixed list of argument registers to the declared signature. */
+#define KVM_HOST_HCALL_REGS(...)					\
+	__KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), __KVM_HCALL_ARGS	\
+		,, cpu_reg(host_ctxt, 1),, cpu_reg(host_ctxt, 2)	\
+		,, cpu_reg(host_ctxt, 3),, cpu_reg(host_ctxt, 4)	\
+		,, cpu_reg(host_ctxt, 5),, cpu_reg(host_ctxt, 6))
+
+#define set_cpu_reg_ulong(ctxt, r, v)	{ cpu_reg(ctxt, r) = v; }
+#define set_cpu_reg_u64(ctxt, r, v)	{ cpu_reg(ctxt, r) = v; }
+#define set_cpu_reg_int(ctxt, r, v)	{ cpu_reg(ctxt, r) = v; }
+#define set_cpu_reg_void(ctxt, r, v)	{ v; }
+#define set_cpu_reg(ctxt, r, t, v)	set_cpu_reg_##t(ctxt, r, v)
+
+#define DEFINE_KVM_HOST_HCALL(ret, name, ...)				\
+	static kvm_host_hcall_sig_##name __do_##name;			\
+	static __always_inline						\
+	ret __se_##name(__KVM_HCALL_MAP(__KVM_HCALL_LONG, __VA_ARGS__))	\
+	{								\
+		return __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_CAST, __VA_ARGS__)); \
+	}								\
+	static void handle_##name(struct kvm_cpu_context *host_ctxt)	\
+	{								\
+		set_cpu_reg(host_ctxt, 1, ret, __se_##name(KVM_HOST_HCALL_REGS(__VA_ARGS__))); \
+	}								\
+	static ret __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__))
+
+#define DEFINE_KVM_HOST_HCALL0(ret, name)				\
+	static kvm_host_hcall_sig_##name __do_##name;			\
+	static void handle_##name(struct kvm_cpu_context *host_ctxt)	\
+	{								\
+		set_cpu_reg(host_ctxt, 1, ret, __do_##name());		\
+	}								\
+	static ret __do_##name(void)
+
 /* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */
 unsigned int hyp_gicv3_nr_lr;
 
@@ -216,6 +258,7 @@ static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu)
 static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 {
 	struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
+	u64 host_hcr_mask = PKVM_HCR_EL2_HOST_PVM;
 
 	fpsimd_sve_flush();
 	flush_debug_state(hyp_vcpu);
@@ -228,6 +271,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 	if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) {
 		if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY))
 			flush_hyp_vcpu_state(hyp_vcpu);
+		host_hcr_mask = PKVM_HCR_EL2_HOST_NPVM;
 	} else {
 		hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt;
 	}
@@ -241,9 +285,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 	 * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE
 	 * for the vSError to be delivered. sync_hyp_vcpu() reflects it back.
 	 */
-	hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE | HCR_VSE);
-	hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) &
-						 (HCR_TWI | HCR_TWE | HCR_VSE);
+	hyp_vcpu->vcpu.arch.hcr_el2 &= ~host_hcr_mask;
+	hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & host_hcr_mask;
 
 	hyp_vcpu->vcpu.arch.iflags	= host_vcpu->arch.iflags;
 
@@ -279,11 +322,9 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
 	sync_hyp_vgic_state(hyp_vcpu);
 }
 
-static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load,
+	pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-	DECLARE_REG(unsigned int, vcpu_idx, host_ctxt, 2);
-	DECLARE_REG(u64, hcr_el2, host_ctxt, 3);
 	struct pkvm_hyp_vcpu *hyp_vcpu;
 
 	hyp_vcpu = pkvm_load_hyp_vcpu(handle, vcpu_idx);
@@ -300,7 +341,7 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt)
 	}
 }
 
-static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put)
 {
 	struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
 
@@ -316,7 +357,7 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt)
 	}
 }
 
-static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state)
 {
 	struct pkvm_hyp_vcpu *hyp_vcpu;
 
@@ -327,10 +368,9 @@ static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt)
 	sync_hyp_vcpu_state(hyp_vcpu);
 }
 
-static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg,
+static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *host_vcpu,
 					     struct pkvm_hyp_vcpu **hyp_vcpup)
 {
-	struct kvm_vcpu *host_vcpu = kern_hyp_va(arg);
 	struct pkvm_hyp_vcpu *hyp_vcpu = NULL;
 
 	if (unlikely(is_protected_kvm_enabled())) {
@@ -346,34 +386,27 @@ static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg,
 	return host_vcpu;
 }
 
-#define get_host_hyp_vcpus(ctxt, regnr, hyp_vcpup)			\
-	({								\
-		DECLARE_REG(struct kvm_vcpu *, __vcpu, ctxt, regnr);	\
-		__get_host_hyp_vcpus(__vcpu, hyp_vcpup);		\
-	})
+static struct kvm_vcpu *
+__get_host_hyp_vcpus_from_vgic_v3_cpu_if(struct vgic_v3_cpu_if __kern *cpu_if,
+					 struct pkvm_hyp_vcpu **hyp_vcpup)
+{
+	struct vgic_v3_cpu_if *host_cpu_if = kern_hyp_va_host(cpu_if);
+	struct kvm_vcpu *host_vcpu = container_of(host_cpu_if, struct kvm_vcpu,
+						  arch.vgic_cpu.vgic_v3);
 
-#define get_host_hyp_vcpus_from_vgic_v3_cpu_if(ctxt, regnr, hyp_vcpup)		\
-	({									\
-		DECLARE_REG(struct vgic_v3_cpu_if *, cif, ctxt, regnr);\
-		struct kvm_vcpu *__vcpu = container_of(cif,			\
-						       struct kvm_vcpu,		\
-						       arch.vgic_cpu.vgic_v3);	\
-										\
-		__get_host_hyp_vcpus(__vcpu, hyp_vcpup);			\
-	})
+	return __get_host_hyp_vcpus(host_vcpu, hyp_vcpup);
+}
 
-static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __kvm_vcpu_run,
+	struct kvm_vcpu __kern *, vcpu)
 {
 	struct pkvm_hyp_vcpu *hyp_vcpu;
 	struct kvm_vcpu *host_vcpu;
 	int ret;
 
-	host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu);
-
-	if (!host_vcpu) {
-		ret = -EINVAL;
-		goto out;
-	}
+	host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu);
+	if (!host_vcpu)
+		return -EINVAL;
 
 	if (unlikely(hyp_vcpu)) {
 		/*
@@ -382,10 +415,8 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
 		 * loading a vcpu. Therefore, if SME features enabled the host
 		 * is misbehaving.
 		 */
-		if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) {
-			ret = -EINVAL;
-			goto out;
-		}
+		if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR)))
+			return -EINVAL;
 
 		flush_hyp_vcpu(hyp_vcpu);
 
@@ -398,8 +429,8 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
 		ret = __kvm_vcpu_run(host_vcpu);
 		fpsimd_lazy_switch_to_host(host_vcpu);
 	}
-out:
-	cpu_reg(host_ctxt, 1) =  ret;
+
+	return ret;
 }
 
 static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu)
@@ -411,184 +442,150 @@ static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu)
 			       &host_vcpu->arch.pkvm_memcache);
 }
 
-static void handle___pkvm_host_donate_guest(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest,
+	u64, pfn, u64, gfn)
 {
-	DECLARE_REG(u64, pfn, host_ctxt, 1);
-	DECLARE_REG(u64, gfn, host_ctxt, 2);
 	struct pkvm_hyp_vcpu *hyp_vcpu;
-	int ret = -EINVAL;
+	int ret;
 
 	hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
 	if (!hyp_vcpu || !pkvm_hyp_vcpu_is_protected(hyp_vcpu))
-		goto out;
+		return -EINVAL;
 
 	ret = pkvm_refill_memcache(hyp_vcpu);
 	if (ret)
-		goto out;
+		return ret;
 
-	ret = __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu);
-out:
-	cpu_reg(host_ctxt, 1) =  ret;
+	return __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu);
 }
 
-static void handle___pkvm_host_share_guest(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_guest,
+	u64, pfn, u64, gfn, u64, nr_pages, u64, prot)
 {
-	DECLARE_REG(u64, pfn, host_ctxt, 1);
-	DECLARE_REG(u64, gfn, host_ctxt, 2);
-	DECLARE_REG(u64, nr_pages, host_ctxt, 3);
-	DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4);
 	struct pkvm_hyp_vcpu *hyp_vcpu;
-	int ret = -EINVAL;
+	int ret;
 
 	hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
 	if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu))
-		goto out;
+		return -EINVAL;
 
 	ret = pkvm_refill_memcache(hyp_vcpu);
 	if (ret)
-		goto out;
+		return ret;
 
-	ret = __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot);
-out:
-	cpu_reg(host_ctxt, 1) =  ret;
+	return __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot);
 }
 
-static void handle___pkvm_host_unshare_guest(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest,
+	pkvm_handle_t, handle, u64, gfn, u64, nr_pages)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-	DECLARE_REG(u64, gfn, host_ctxt, 2);
-	DECLARE_REG(u64, nr_pages, host_ctxt, 3);
 	struct pkvm_hyp_vm *hyp_vm;
-	int ret = -EINVAL;
+	int ret;
 
 	hyp_vm = get_np_pkvm_hyp_vm(handle);
 	if (!hyp_vm)
-		goto out;
+		return -EINVAL;
 
 	ret = __pkvm_host_unshare_guest(gfn, nr_pages, hyp_vm);
 	put_pkvm_hyp_vm(hyp_vm);
-out:
-	cpu_reg(host_ctxt, 1) =  ret;
+
+	return ret;
 }
 
-static void handle___pkvm_host_relax_perms_guest(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest,
+	u64, gfn, u64, prot)
 {
-	DECLARE_REG(u64, gfn, host_ctxt, 1);
-	DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 2);
 	struct pkvm_hyp_vcpu *hyp_vcpu;
-	int ret = -EINVAL;
 
 	hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
 	if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu))
-		goto out;
+		return -EINVAL;
 
-	ret = __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot);
-out:
-	cpu_reg(host_ctxt, 1) = ret;
+	return __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot);
 }
 
-static void handle___pkvm_host_wrprotect_guest(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest,
+	pkvm_handle_t, handle, u64, gfn, u64, nr_pages)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-	DECLARE_REG(u64, gfn, host_ctxt, 2);
-	DECLARE_REG(u64, nr_pages, host_ctxt, 3);
 	struct pkvm_hyp_vm *hyp_vm;
-	int ret = -EINVAL;
+	int ret;
 
 	hyp_vm = get_np_pkvm_hyp_vm(handle);
 	if (!hyp_vm)
-		goto out;
+		return -EINVAL;
 
 	ret = __pkvm_host_wrprotect_guest(gfn, nr_pages, hyp_vm);
 	put_pkvm_hyp_vm(hyp_vm);
-out:
-	cpu_reg(host_ctxt, 1) = ret;
+
+	return ret;
 }
 
-static void handle___pkvm_host_test_clear_young_guest(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest,
+	pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-	DECLARE_REG(u64, gfn, host_ctxt, 2);
-	DECLARE_REG(u64, nr_pages, host_ctxt, 3);
-	DECLARE_REG(bool, mkold, host_ctxt, 4);
 	struct pkvm_hyp_vm *hyp_vm;
-	int ret = -EINVAL;
+	int ret;
 
 	hyp_vm = get_np_pkvm_hyp_vm(handle);
 	if (!hyp_vm)
-		goto out;
+		return -EINVAL;
 
 	ret = __pkvm_host_test_clear_young_guest(gfn, nr_pages, mkold, hyp_vm);
 	put_pkvm_hyp_vm(hyp_vm);
-out:
-	cpu_reg(host_ctxt, 1) = ret;
+
+	return ret;
 }
 
-static void handle___pkvm_host_mkyoung_guest(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest,
+	u64, gfn)
 {
-	DECLARE_REG(u64, gfn, host_ctxt, 1);
 	struct pkvm_hyp_vcpu *hyp_vcpu;
-	int ret = -EINVAL;
 
 	hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
 	if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu))
-		goto out;
+		return -EINVAL;
 
-	ret = __pkvm_host_mkyoung_guest(gfn, hyp_vcpu);
-out:
-	cpu_reg(host_ctxt, 1) =  ret;
+	return __pkvm_host_mkyoung_guest(gfn, hyp_vcpu);
 }
 
-static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __kvm_adjust_pc,
+	struct kvm_vcpu __kern *, vcpu)
 {
-	DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
-
-	__kvm_adjust_pc(kern_hyp_va(vcpu));
+	__kvm_adjust_pc(kern_hyp_va_host(vcpu));
 }
 
-static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context)
 {
 	__kvm_flush_vm_context();
 }
 
-static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa,
+	struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level)
 {
-	DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
-	DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2);
-	DECLARE_REG(int, level, host_ctxt, 3);
-
-	__kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level);
+	__kvm_tlb_flush_vmid_ipa(kern_hyp_va_host(mmu), ipa, level);
 }
 
-static void handle___kvm_tlb_flush_vmid_ipa_nsh(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh,
+	struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level)
 {
-	DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
-	DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2);
-	DECLARE_REG(int, level, host_ctxt, 3);
-
-	__kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level);
+	__kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va_host(mmu), ipa, level);
 }
 
-static void
-handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range,
+	struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages)
 {
-	DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
-	DECLARE_REG(phys_addr_t, start, host_ctxt, 2);
-	DECLARE_REG(unsigned long, pages, host_ctxt, 3);
-
-	__kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, pages);
+	__kvm_tlb_flush_vmid_range(kern_hyp_va_host(mmu), start, pages);
 }
 
-static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid,
+	struct kvm_s2_mmu __kern *, mmu)
 {
-	DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
-
-	__kvm_tlb_flush_vmid(kern_hyp_va(mmu));
+	__kvm_tlb_flush_vmid(kern_hyp_va_host(mmu));
 }
 
-static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid,
+	pkvm_handle_t, handle)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
 	struct pkvm_hyp_vm *hyp_vm = get_np_pkvm_hyp_vm(handle);
 
 	if (!hyp_vm)
@@ -598,19 +595,19 @@ static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt)
 	put_pkvm_hyp_vm(hyp_vm);
 }
 
-static void handle___kvm_flush_cpu_context(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context,
+	struct kvm_s2_mmu __kern *, mmu)
 {
-	DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1);
-
-	__kvm_flush_cpu_context(kern_hyp_va(mmu));
+	__kvm_flush_cpu_context(kern_hyp_va_host(mmu));
 }
 
-static void handle___kvm_timer_set_cntvoff(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff,
+	u64, cntvoff)
 {
-	__kvm_timer_set_cntvoff(cpu_reg(host_ctxt, 1));
+	__kvm_timer_set_cntvoff(cntvoff);
 }
 
-static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs)
 {
 	u64 tmp;
 
@@ -619,23 +616,23 @@ static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt)
 	write_sysreg_el2(tmp, SYS_SCTLR);
 }
 
-static void handle___vgic_v3_get_gic_config(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config)
 {
-	cpu_reg(host_ctxt, 1) = __vgic_v3_get_gic_config();
+	return __vgic_v3_get_gic_config();
 }
 
-static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs)
 {
 	__vgic_v3_init_lrs();
 }
 
-static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs,
+	struct vgic_v3_cpu_if __kern *, cpu_if)
 {
 	struct pkvm_hyp_vcpu *hyp_vcpu;
 	struct kvm_vcpu *host_vcpu;
 
-	host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1,
-							   &hyp_vcpu);
+	host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu);
 	if (!host_vcpu)
 		return;
 
@@ -657,13 +654,13 @@ static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt)
 	}
 }
 
-static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs,
+	struct vgic_v3_cpu_if __kern *, cpu_if)
 {
 	struct pkvm_hyp_vcpu *hyp_vcpu;
 	struct kvm_vcpu *host_vcpu;
 
-	host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1,
-							   &hyp_vcpu);
+	host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu);
 	if (!host_vcpu)
 		return;
 
@@ -690,48 +687,39 @@ static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt
 	}
 }
 
-static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_init,
+	phys_addr_t, phys, unsigned long, size,
+	unsigned long *, per_cpu_base, u32, hyp_va_bits)
 {
-	DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
-	DECLARE_REG(unsigned long, size, host_ctxt, 2);
-	DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 3);
-	DECLARE_REG(u32, hyp_va_bits, host_ctxt, 4);
-
 	/*
 	 * __pkvm_init() will return only if an error occurred, otherwise it
 	 * will tail-call in __pkvm_init_finalise() which will have to deal
 	 * with the host context directly.
 	 */
-	cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, per_cpu_base, hyp_va_bits);
+	return __pkvm_init(phys, size, per_cpu_base, hyp_va_bits);
 }
 
-static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector,
+	enum arm64_hyp_spectre_vector, slot)
 {
-	DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot);
+	return pkvm_cpu_set_vector(slot);
 }
 
-static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp,
+	u64, pfn)
 {
-	DECLARE_REG(u64, pfn, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn);
+	return __pkvm_host_share_hyp(pfn);
 }
 
-static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp,
+	u64, pfn)
 {
-	DECLARE_REG(u64, pfn, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn);
+	return __pkvm_host_unshare_hyp(pfn);
 }
 
-static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping,
+	phys_addr_t, phys, size_t, size, u64, prot)
 {
-	DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
-	DECLARE_REG(size_t, size, host_ctxt, 2);
-	DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3);
-
 	/*
 	 * __pkvm_create_private_mapping() populates a pointer with the
 	 * hypervisor start address of the allocation.
@@ -742,161 +730,160 @@ static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ct
 	 * Instead pass the allocation address as the return value (or return
 	 * ERR_PTR() on failure).
 	 */
-	unsigned long haddr;
+	ulong haddr;
 	int err = __pkvm_create_private_mapping(phys, size, prot, &haddr);
 
 	if (err)
-		haddr = (unsigned long)ERR_PTR(err);
+		haddr = (ulong)ERR_PTR(err);
 
-	cpu_reg(host_ctxt, 1) = haddr;
+	return haddr;
 }
 
-static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize)
 {
-	cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize();
+	return __pkvm_prot_finalize();
 }
 
-static void handle___pkvm_reserve_vm(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm)
 {
-	cpu_reg(host_ctxt, 1) = __pkvm_reserve_vm();
+	return __pkvm_reserve_vm();
 }
 
-static void handle___pkvm_unreserve_vm(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm,
+	pkvm_handle_t, handle)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-
 	__pkvm_unreserve_vm(handle);
 }
 
-static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm,
+	struct kvm __kern *, host_kvm, void __kern *, vm_hva,
+	void __kern *, pgd_hva)
 {
-	DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1);
-	DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2);
-	DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3);
-
-	host_kvm = kern_hyp_va(host_kvm);
-	cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva);
+	return __pkvm_init_vm(kern_hyp_va_host(host_kvm), vm_hva, pgd_hva);
 }
 
-static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu,
+	pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu,
+	void __kern *, vcpu_hva)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-	DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2);
-	DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3);
-
-	host_vcpu = kern_hyp_va(host_vcpu);
-	cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva);
+	return __pkvm_init_vcpu(handle, kern_hyp_va_host(host_vcpu), vcpu_hva);
 }
 
-static void handle___pkvm_vcpu_in_poison_fault(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault)
 {
-	int ret;
 	struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
 
-	ret = hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL;
-	cpu_reg(host_ctxt, 1) = ret;
+	return hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL;
 }
 
-static void handle___pkvm_force_reclaim_guest_page(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page,
+	phys_addr_t, phys)
 {
-	DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = __pkvm_host_force_reclaim_page_guest(phys);
+	return __pkvm_host_force_reclaim_page_guest(phys);
 }
 
-static void handle___pkvm_reclaim_dying_guest_page(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page,
+	pkvm_handle_t, handle, u64, gfn)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-	DECLARE_REG(u64, gfn, host_ctxt, 2);
-
-	cpu_reg(host_ctxt, 1) = __pkvm_reclaim_dying_guest_page(handle, gfn);
+	return __pkvm_reclaim_dying_guest_page(handle, gfn);
 }
 
-static void handle___pkvm_start_teardown_vm(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm,
+	pkvm_handle_t, handle)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = __pkvm_start_teardown_vm(handle);
+	return __pkvm_start_teardown_vm(handle);
 }
 
-static void handle___pkvm_finalize_teardown_vm(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm,
+	pkvm_handle_t, handle)
 {
-	DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = __pkvm_finalize_teardown_vm(handle);
+	return __pkvm_finalize_teardown_vm(handle);
 }
 
-static void handle___tracing_load(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __tracing_load,
+	void __kern *, desc_hva, size_t, desc_size)
 {
-	DECLARE_REG(unsigned long, desc_hva, host_ctxt, 1);
-	DECLARE_REG(size_t, desc_size, host_ctxt, 2);
-
-	cpu_reg(host_ctxt, 1) = __tracing_load(desc_hva, desc_size);
+	return __tracing_load(desc_hva, desc_size);
 }
 
-static void handle___tracing_unload(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL0(void, __tracing_unload)
 {
 	__tracing_unload();
 }
 
-static void handle___tracing_enable(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __tracing_enable,
+	bool, enable)
 {
-	DECLARE_REG(bool, enable, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = __tracing_enable(enable);
+	return __tracing_enable(enable);
 }
 
-static void handle___tracing_swap_reader(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __tracing_swap_reader,
+	unsigned int, cpu)
 {
-	DECLARE_REG(unsigned int, cpu, host_ctxt, 1);
-
-	cpu_reg(host_ctxt, 1) = __tracing_swap_reader(cpu);
+	return __tracing_swap_reader(cpu);
 }
 
-static void handle___tracing_update_clock(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __tracing_update_clock,
+	u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc)
 {
-	DECLARE_REG(u32, mult, host_ctxt, 1);
-	DECLARE_REG(u32, shift, host_ctxt, 2);
-	DECLARE_REG(u64, epoch_ns, host_ctxt, 3);
-	DECLARE_REG(u64, epoch_cyc, host_ctxt, 4);
-
 	__tracing_update_clock(mult, shift, epoch_ns, epoch_cyc);
 }
 
-static void handle___tracing_reset(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(int, __tracing_reset,
+	unsigned int, cpu)
 {
-	DECLARE_REG(unsigned int, cpu, host_ctxt, 1);
+	return __tracing_reset(cpu);
+}
 
-	cpu_reg(host_ctxt, 1) = __tracing_reset(cpu);
+DEFINE_KVM_HOST_HCALL(int, __tracing_enable_event,
+	unsigned short, id, bool, enable)
+{
+	return __tracing_enable_event(id, enable);
 }
 
-static void handle___tracing_enable_event(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __tracing_write_event,
+	u64, id)
 {
-	DECLARE_REG(unsigned short, id, host_ctxt, 1);
-	DECLARE_REG(bool, enable, host_ctxt, 2);
+	trace_selftest(id);
+}
 
-	cpu_reg(host_ctxt, 1) = __tracing_enable_event(id, enable);
+DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_resident,
+		      struct vgic_v5_cpu_if __kern *, cpu_if)
+{
+	if (unlikely(is_protected_kvm_enabled()))
+		return;
+
+	__vgic_v5_make_resident(kern_hyp_va_host(cpu_if));
 }
 
-static void handle___tracing_write_event(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident,
+		      struct vgic_v5_cpu_if __kern *, cpu_if)
 {
-	DECLARE_REG(u64, id, host_ctxt, 1);
+	if (unlikely(is_protected_kvm_enabled()))
+		return;
 
-	trace_selftest(id);
+	__vgic_v5_make_non_resident(kern_hyp_va_host(cpu_if));
 }
 
-static void handle___vgic_v5_save_apr(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __vgic_v5_save_apr,
+	struct vgic_v5_cpu_if __kern *, cpu_if)
 {
-	DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1);
+	__vgic_v5_save_apr(kern_hyp_va_host(cpu_if));
+}
 
-	__vgic_v5_save_apr(kern_hyp_va(cpu_if));
+DEFINE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr,
+	struct vgic_v5_cpu_if __kern *, cpu_if)
+{
+	__vgic_v5_restore_vmcr_apr(kern_hyp_va_host(cpu_if));
 }
 
-static void handle___vgic_v5_restore_vmcr_apr(struct kvm_cpu_context *host_ctxt)
+DEFINE_KVM_HOST_HCALL(void, __vgic_v5_vdpend,
+		      u32, intid, bool, pending, u16, vm)
 {
-	DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1);
+	if (unlikely(is_protected_kvm_enabled()))
+		return;
 
-	__vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if));
+	__vgic_v5_vdpend(intid, pending, vm);
 }
 
 typedef void (*hcall_t)(struct kvm_cpu_context *);
@@ -932,6 +919,9 @@ static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__tracing_write_event),
 	HANDLE_FUNC(__vgic_v3_save_aprs),
 	HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs),
+	HANDLE_FUNC(__vgic_v5_make_resident),
+	HANDLE_FUNC(__vgic_v5_make_non_resident),
+	HANDLE_FUNC(__vgic_v5_vdpend),
 	HANDLE_FUNC(__vgic_v5_save_apr),
 	HANDLE_FUNC(__vgic_v5_restore_vmcr_apr),
 
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index 7a02837203d15d..89222425359a73 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -17,7 +17,6 @@ SECTIONS {
 	HYP_SECTION(.data..ro_after_init)
 	HYP_SECTION(.rodata)
 #ifdef CONFIG_NVHE_EL2_TRACING
-	. = ALIGN(PAGE_SIZE);
 	BEGIN_HYP_SECTION(.event_ids)
 		*(SORT(.hyp.event_ids.*))
 	END_HYP_SECTION
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 39aa8911f62c1b..443ee433119ef8 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -450,6 +450,14 @@ bool addr_is_memory(phys_addr_t phys)
 	return !!find_mem_range(phys, &range);
 }
 
+bool addr_is_hyp_text(phys_addr_t phys)
+{
+	phys_addr_t start = ALIGN_DOWN(__hyp_pa(__hyp_text_start), PAGE_SIZE);
+	phys_addr_t end = PAGE_ALIGN(__hyp_pa(__hyp_text_end));
+
+	return phys >= start && phys < end;
+}
+
 static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range)
 {
 	return range->start <= addr && addr < range->end;
@@ -532,7 +540,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range)
 	int ret;
 
 	hyp_assert_lock_held(&host_mmu.lock);
-	ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);
+	ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level, 0);
 	if (ret)
 		return ret;
 
@@ -633,6 +641,18 @@ int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id)
 	return ret;
 }
 
+bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte)
+{
+	if (kvm_pte_valid(pte))
+		return false;
+
+	if (FIELD_GET(KVM_INVALID_PTE_TYPE_MASK, pte) !=
+	    KVM_HOST_INVALID_PTE_TYPE_DONATION)
+		return false;
+
+	return FIELD_GET(KVM_HOST_DONATION_PTE_OWNER_MASK, pte) == PKVM_ID_HYP;
+}
+
 #define KVM_HOST_PTE_OWNER_GUEST_HANDLE_MASK	GENMASK(15, 0)
 /* We need 40 bits for the GFN to cover a 52-bit IPA with 4k pages and LPA2 */
 #define KVM_HOST_PTE_OWNER_GUEST_GFN_MASK	GENMASK(55, 16)
@@ -910,7 +930,7 @@ static int get_valid_guest_pte(struct pkvm_hyp_vm *vm, u64 ipa, kvm_pte_t *ptep,
 	s8 level;
 	int ret;
 
-	ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
+	ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
 	if (ret)
 		return ret;
 	if (guest_pte_is_poisoned(pte))
@@ -959,7 +979,7 @@ int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu)
 	ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(&hyp_vcpu->vcpu));
 
 	guest_lock_component(vm);
-	ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
+	ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
 	if (ret)
 		goto unlock;
 
@@ -1315,7 +1335,7 @@ static int host_stage2_get_guest_info(phys_addr_t phys, struct pkvm_hyp_vm **vm,
 		return -EPERM;
 	}
 
-	ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level);
+	ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level, 0);
 	if (ret)
 		return ret;
 
@@ -1544,7 +1564,7 @@ static int __check_host_shared_guest(struct pkvm_hyp_vm *vm, u64 *__phys, u64 ip
 	s8 level;
 	int ret;
 
-	ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
+	ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
 	if (ret)
 		return ret;
 	if (!kvm_pte_valid(pte))
diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c
index 3b0bee496bffb6..bcf2fd1ddd75d4 100644
--- a/arch/arm64/kvm/hyp/nvhe/mm.c
+++ b/arch/arm64/kvm/hyp/nvhe/mm.c
@@ -25,6 +25,7 @@ struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
 unsigned int hyp_memblock_nr;
 
 static u64 __io_map_base;
+static u64 __io_map_next;
 
 struct hyp_fixmap_slot {
 	u64 addr;
@@ -50,7 +51,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size)
 
 	hyp_assert_lock_held(&pkvm_pgd_lock);
 
-	if (!start || start < __io_map_base)
+	if (!start || start < __io_map_next)
 		return -EINVAL;
 
 	/* The allocated size is always a multiple of PAGE_SIZE */
@@ -60,7 +61,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size)
 	if (cur > __hyp_vmemmap)
 		return -ENOMEM;
 
-	__io_map_base = cur;
+	__io_map_next = cur;
 
 	return 0;
 }
@@ -70,7 +71,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size)
  * @size:	The size of the VA range to reserve.
  * @haddr:	The hypervisor virtual start address of the allocation.
  *
- * The private virtual address (VA) range is allocated above __io_map_base
+ * The private virtual address (VA) range is allocated above __io_map_next
  * and aligned based on the order of @size.
  *
  * Return: 0 on success or negative error code on failure.
@@ -81,7 +82,7 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr)
 	int ret;
 
 	hyp_spin_lock(&pkvm_pgd_lock);
-	addr = __io_map_base;
+	addr = __io_map_next;
 	ret = __pkvm_alloc_private_va_range(addr, size);
 	hyp_spin_unlock(&pkvm_pgd_lock);
 
@@ -341,7 +342,7 @@ static int create_fixblock(void)
 		return -EINVAL;
 
 	hyp_spin_lock(&pkvm_pgd_lock);
-	addr = ALIGN(__io_map_base, PMD_SIZE);
+	addr = ALIGN(__io_map_next, PMD_SIZE);
 	ret = __pkvm_alloc_private_va_range(addr, PMD_SIZE);
 	if (ret)
 		goto unlock;
@@ -426,6 +427,7 @@ int hyp_create_idmap(u32 hyp_va_bits)
 	 */
 	__io_map_base = start & BIT(hyp_va_bits - 2);
 	__io_map_base ^= BIT(hyp_va_bits - 2);
+	__io_map_next = __io_map_base;
 	__hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3);
 
 	return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);
@@ -433,19 +435,19 @@ int hyp_create_idmap(u32 hyp_va_bits)
 
 int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr)
 {
-	unsigned long addr, prev_base;
+	unsigned long addr, prev_next;
 	size_t size;
 	int ret;
 
 	hyp_spin_lock(&pkvm_pgd_lock);
 
-	prev_base = __io_map_base;
+	prev_next = __io_map_next;
 	/*
 	 * Efficient stack verification using the NVHE_STACK_SHIFT bit implies
 	 * an alignment of our allocation on the order of the size.
 	 */
 	size = NVHE_STACK_SIZE * 2;
-	addr = ALIGN(__io_map_base, size);
+	addr = ALIGN(__io_map_next, size);
 
 	ret = __pkvm_alloc_private_va_range(addr, size);
 	if (!ret) {
@@ -461,7 +463,7 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr)
 		ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr + NVHE_STACK_SIZE,
 					  NVHE_STACK_SIZE, phys, PAGE_HYP);
 		if (ret)
-			__io_map_base = prev_base;
+			__io_map_next = prev_next;
 	}
 	hyp_spin_unlock(&pkvm_pgd_lock);
 
@@ -470,6 +472,65 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr)
 	return ret;
 }
 
+static int check_page_ownership(phys_addr_t phys)
+{
+	kvm_pte_t pte;
+	bool host_ok;
+	int ret;
+
+	if (addr_is_memory(phys)) {
+		struct hyp_page *page = hyp_phys_to_page(phys);
+
+		if (get_hyp_state(page) != PKVM_PAGE_OWNED ||
+		    get_host_state(page) != PKVM_NOPAGE)
+			return -EPERM;
+	}
+
+	ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL, 0);
+	if (ret)
+		return ret;
+
+	/* Hyp text may stay host-readable, see fix_host_ownership_walker(). */
+	if (kvm_pte_valid(pte) && addr_is_hyp_text(phys))
+		host_ok = !(kvm_pgtable_stage2_pte_prot(pte) & KVM_PGTABLE_PROT_W);
+	else
+		host_ok = host_stage2_pte_is_hyp_owned(pte);
+
+	return host_ok ? 0 : -EPERM;
+}
+
+static int check_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx,
+				       enum kvm_pgtable_walk_flags visit)
+{
+	phys_addr_t phys, end;
+	int ret;
+
+	if (!kvm_pte_valid(ctx->old))
+		return 0;
+
+	phys = kvm_pte_to_phys(ctx->old);
+	end = phys + kvm_granule_size(ctx->level);
+	for (; phys < end; phys += PAGE_SIZE) {
+		ret = check_page_ownership(phys);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int pkvm_check_host_ownership(void)
+{
+	struct kvm_pgtable_walker walker = {
+		.cb	= check_host_ownership_walker,
+		.flags	= KVM_PGTABLE_WALK_LEAF,
+	};
+
+	/* The private range and the vmemmap share one quarter of the VA space. */
+	return kvm_pgtable_walk(&pkvm_pgtable, __io_map_base,
+				BIT(pkvm_pgtable.ia_bits - 2), &walker);
+}
+
 static void *admit_host_page(void *arg)
 {
 	struct kvm_hyp_memcache *host_mc = arg;
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 459bd9eb7e4bc7..c6068795360d0d 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -30,6 +30,7 @@ unsigned int kvm_host_sve_max_vl;
  */
 static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu);
 
+/* The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on each entry. */
 static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
@@ -47,18 +48,17 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 	if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
 		vcpu->arch.hcr_el2 |= HCR_FWB;
 
-	if (cpus_have_final_cap(ARM64_HAS_EVT) &&
-	    !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) &&
-	    kvm_read_vm_id_reg(vcpu->kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0))
-		vcpu->arch.hcr_el2 |= HCR_TID4;
-	else
-		vcpu->arch.hcr_el2 |= HCR_TID2;
+	/*
+	 * Without AArch32 EL1, leave RW set and let the entry fail with an
+	 * illegal exception return: the *32_EL2 registers EL2 would otherwise
+	 * switch are UNDEFINED there.
+	 */
+	if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT) &&
+	    cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
+		vcpu->arch.hcr_el2 &= ~HCR_EL2_RW;
 
 	if (vcpu_has_ptrauth(vcpu))
 		vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
-
-	if (kvm_has_mte(vcpu->kvm))
-		vcpu->arch.hcr_el2 |= HCR_ATA;
 }
 
 static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu)
@@ -76,6 +76,13 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu)
 	 */
 	val |= HCR_TACR | HCR_TIDCP | HCR_TID3 | HCR_TID1;
 
+	if (cpus_have_final_cap(ARM64_HAS_EVT) &&
+	    !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) &&
+	    kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0))
+		val |= HCR_EL2_TID4;
+	else
+		val |= HCR_EL2_TID2;
+
 	if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) {
 		val |= HCR_TERR | HCR_TEA;
 		val &= ~(HCR_FIEN);
@@ -360,7 +367,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc
 		if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags))
 			hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1;
 
-		return;
+		goto out;
 	}
 
 	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE))
@@ -379,13 +386,14 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc
 	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC))
 		set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features);
 
-	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) {
+	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE))
 		set_bit(KVM_ARM_VCPU_SVE, allowed_features);
-		kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_GUEST_HAS_SVE);
-	}
 
 	bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features,
 		   allowed_features, KVM_VCPU_MAX_FEATURES);
+out:
+	__assign_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &kvm->arch.flags,
+		     kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE));
 }
 
 static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu)
@@ -398,10 +406,10 @@ static void unpin_host_sve_state(struct pkvm_hyp_vcpu *hyp_vcpu)
 {
 	void *sve_state;
 
-	if (!vcpu_has_feature(&hyp_vcpu->vcpu, KVM_ARM_VCPU_SVE))
+	sve_state = hyp_vcpu->vcpu.arch.sve_state;
+	if (!sve_state)
 		return;
 
-	sve_state = hyp_vcpu->vcpu.arch.sve_state;
 	hyp_unpin_shared_mem(sve_state,
 			     sve_state + vcpu_sve_state_size(&hyp_vcpu->vcpu));
 }
@@ -450,7 +458,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h
 	unsigned int sve_max_vl;
 	size_t sve_state_size;
 	void *sve_state;
-	int ret = 0;
+	int ret;
 
 	if (!vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE)) {
 		vcpu_clear_flag(vcpu, VCPU_SVE_FINALIZED);
@@ -459,25 +467,21 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h
 
 	/* Limit guest vector length to the maximum supported by the host. */
 	sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl);
-	sve_state_size = sve_state_size_from_vl(sve_max_vl);
 	sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state));
 
-	if (!sve_state || !sve_state_size) {
-		ret = -EINVAL;
-		goto err;
-	}
+	if (!sve_vl_valid(sve_max_vl) || !sve_state)
+		return -EINVAL;
+
+	sve_state_size = sve_state_size_from_vl(sve_max_vl);
 
 	ret = hyp_pin_shared_mem(sve_state, sve_state + sve_state_size);
 	if (ret)
-		goto err;
+		return ret;
 
 	vcpu->arch.sve_state = sve_state;
 	vcpu->arch.sve_max_vl = sve_max_vl;
 
 	return 0;
-err:
-	clear_bit(KVM_ARM_VCPU_SVE, vcpu->kvm->arch.vcpu_features);
-	return ret;
 }
 
 static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu)
@@ -657,9 +661,9 @@ static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus)
 		size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus));
 }
 
-static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
+static void *map_donated_memory_noclear(void __kern *host_va, size_t size)
 {
-	void *va = (void *)kern_hyp_va(host_va);
+	void *va = kern_hyp_va_host(host_va);
 
 	if (!PAGE_ALIGNED(va))
 		return NULL;
@@ -671,7 +675,7 @@ static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
 	return va;
 }
 
-static void *map_donated_memory(unsigned long host_va, size_t size)
+static void *map_donated_memory(void __kern *host_va, size_t size)
 {
 	void *va = map_donated_memory_noclear(host_va, size);
 
@@ -818,8 +822,8 @@ void teardown_selftest_vm(void)
  *
  * Return 0 success, negative error code on failure.
  */
-int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
-		   unsigned long pgd_hva)
+int __pkvm_init_vm(struct kvm *host_kvm, void __kern *vm_hva,
+		   void __kern *pgd_hva)
 {
 	struct pkvm_hyp_vm *hyp_vm = NULL;
 	size_t vm_size, pgd_size;
@@ -910,7 +914,7 @@ static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm,
 }
 
 int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
-		     unsigned long vcpu_hva)
+		     void __kern *vcpu_hva)
 {
 	struct pkvm_hyp_vcpu *hyp_vcpu;
 	struct pkvm_hyp_vm *hyp_vm;
diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index 75b00c32331025..45ac5f2ba4f7ae 100644
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -217,7 +217,7 @@ static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx,
 	case PKVM_PAGE_OWNED:
 		set_hyp_state(page, PKVM_PAGE_OWNED);
 		/* hyp text is RO in the host stage-2 to be inspected on panic. */
-		if (prot == PAGE_HYP_EXEC) {
+		if (addr_is_hyp_text(phys)) {
 			set_host_state(page, PKVM_NOPAGE);
 			return host_stage2_idmap_locked(phys, PAGE_SIZE, KVM_PGTABLE_PROT_R);
 		} else {
@@ -269,6 +269,16 @@ static int fix_host_ownership(void)
 			return ret;
 	}
 
+	/* The stacks sit in the private VA range, not the linear map. */
+	for (i = 0; i < hyp_nr_cpus; i++) {
+		struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i);
+		u64 start = params->stack_hyp_va - NVHE_STACK_SIZE;
+
+		ret = kvm_pgtable_walk(&pkvm_pgtable, start, NVHE_STACK_SIZE, &walker);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
 
@@ -324,6 +334,10 @@ void __noreturn __pkvm_init_finalise(void)
 	if (ret)
 		goto out;
 
+	ret = pkvm_check_host_ownership();
+	if (ret)
+		goto out;
+
 	ret = hyp_ffa_init(ffa_proxy_pages);
 	if (ret)
 		goto out;
diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe/stacktrace.c
index 7c832d60d22bb0..11fadbebbf1d6f 100644
--- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c
+++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c
@@ -8,6 +8,7 @@
 #include <asm/kvm_hyp.h>
 #include <asm/memory.h>
 #include <asm/percpu.h>
+#include <asm/stacktrace/nvhe.h>
 
 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
 	__aligned(16);
@@ -35,8 +36,6 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc)
 }
 
 #ifdef CONFIG_PKVM_STACKTRACE
-#include <asm/stacktrace/nvhe.h>
-
 DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace);
 
 static struct stack_info stackinfo_get_overflow(void)
diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c
index 65be6c9fb379df..47680a949f61f5 100644
--- a/arch/arm64/kvm/hyp/nvhe/trace.c
+++ b/arch/arm64/kvm/hyp/nvhe/trace.c
@@ -93,7 +93,7 @@ static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace
 
 	__release_host_mem(start, size);
 
-	trace_buffer->bpages_backing_start = 0;
+	trace_buffer->bpages_backing_start = NULL;
 	trace_buffer->bpages_backing_size = 0;
 }
 
@@ -206,9 +206,9 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz
 	return true;
 }
 
-int __tracing_load(unsigned long desc_hva, size_t desc_size)
+int __tracing_load(void __kern *desc_hva, size_t desc_size)
 {
-	struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva);
+	struct hyp_trace_desc *desc = kern_hyp_va_host(desc_hva);
 	int ret;
 
 	ret = __admit_host_mem(desc, desc_size);
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index b74dd5ce1efd37..347eec3957d6a9 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -298,12 +298,13 @@ static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx,
 }
 
 int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
-			 kvm_pte_t *ptep, s8 *level)
+			 kvm_pte_t *ptep, s8 *level,
+			 enum kvm_pgtable_walk_flags flags)
 {
 	struct leaf_walk_data data;
 	struct kvm_pgtable_walker walker = {
 		.cb	= leaf_walker,
-		.flags	= KVM_PGTABLE_WALK_LEAF,
+		.flags	= flags | KVM_PGTABLE_WALK_LEAF,
 		.arg	= &data,
 	};
 	int ret;
diff --git a/arch/arm64/kvm/hyp/vgic-v5-sr.c b/arch/arm64/kvm/hyp/vgic-v5-sr.c
index 6d69dfe89a96cf..94746317ca79fc 100644
--- a/arch/arm64/kvm/hyp/vgic-v5-sr.c
+++ b/arch/arm64/kvm/hyp/vgic-v5-sr.c
@@ -7,6 +7,41 @@
 
 #include <asm/kvm_hyp.h>
 
+void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if)
+{
+	write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2);
+	isb();
+
+	/* Catch any faults */
+	cpu_if->vgic_contextr = read_sysreg_s(SYS_ICH_CONTEXTR_EL2);
+	if (!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr))
+		return;
+
+	cpu_if->gicv5_vpe.resident = true;
+}
+
+void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if)
+{
+	/*
+	 * Clear the db_fired state to ensure that we're ready for the next
+	 * doorbell when it is requested. If a doorbell firing caused us to
+	 * enter the guest, then we've already consumed that state at this
+	 * point, so this is safe to clear. Use WRITE_ONCE() to ensure we're not
+	 * racing with the doorbell firing and setting the state true again.
+	 */
+	WRITE_ONCE(cpu_if->gicv5_vpe.db_fired, false);
+
+	/*
+	 * Make as non-resident before actually making non-resident. Avoids race
+	 * with doorbell arriving.
+	 */
+	cpu_if->gicv5_vpe.resident = false;
+	dsb(st);
+
+	write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2);
+	isb();
+}
+
 void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if)
 {
 	cpu_if->vgic_apr = read_sysreg_s(SYS_ICH_APR_EL2);
@@ -114,3 +149,13 @@ void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if)
 {
 	write_sysreg_s(cpu_if->vgic_icsr, SYS_ICC_ICSR_EL1);
 }
+
+void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm)
+{
+	u64 value;
+
+	value = intid & (GICV5_GIC_VDPEND_ID_MASK | GICV5_GIC_VDPEND_TYPE_MASK);
+	value |= FIELD_PREP(GICV5_GIC_VDPEND_PENDING_MASK, pending);
+	value |= FIELD_PREP(GICV5_GIC_VDPEND_VM_MASK, vm);
+	gic_insn(value, VDPEND);
+}
diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c
index 9644c424819b76..44937e257375a2 100644
--- a/arch/arm64/kvm/hyp_trace.c
+++ b/arch/arm64/kvm/hyp_trace.c
@@ -269,7 +269,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv)
 	if (ret)
 		goto err_free_buffer;
 
-	ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size);
+	ret = kvm_call_hyp_nvhe(__tracing_load, desc, desc_size);
 	if (ret)
 		goto err_unload_pages;
 
diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c
index b11b8821c9fbc6..b393ba9af5b6fa 100644
--- a/arch/arm64/kvm/hypercalls.c
+++ b/arch/arm64/kvm/hypercalls.c
@@ -185,7 +185,8 @@ static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user
 	start = filter.base;
 	end = start + filter.nr_functions - 1;
 
-	if (end < start || filter.action >= NR_SMCCC_FILTER_ACTIONS)
+	if (!filter.nr_functions || end < start ||
+	    filter.action >= NR_SMCCC_FILTER_ACTIONS)
 		return -EINVAL;
 
 	mutex_lock(&kvm->arch.config_lock);
@@ -293,6 +294,10 @@ int kvm_smccc_call_handler(struct kvm_vcpu *vcpu)
 	case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
 		feature = smccc_get_arg1(vcpu);
 		switch (feature) {
+		case ARM_SMCCC_VERSION_FUNC_ID:
+		case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
+			val[0] = SMCCC_RET_SUCCESS;
+			break;
 		case ARM_SMCCC_ARCH_WORKAROUND_1:
 			switch (arm64_get_spectre_v2_state()) {
 			case SPECTRE_VULNERABLE:
diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 9ba86450fe4af6..cd609dc070cc5f 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -5,8 +5,10 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/cleanup.h>
 #include <linux/mman.h>
 #include <linux/kvm_host.h>
+#include <linux/interval_tree.h>
 #include <linux/io.h>
 #include <linux/hugetlb.h>
 #include <linux/sched/signal.h>
@@ -59,27 +61,36 @@ static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end)
  * long will also starve other vCPUs. We have to also make sure that the page
  * tables are not freed while we released the lock.
  */
-static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t addr,
+static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
 			      phys_addr_t end,
 			      int (*fn)(struct kvm_pgtable *, u64, u64),
 			      bool resched)
 {
 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
+	bool lock_dropped = false;
+	phys_addr_t addr = start;
 	int ret;
 	u64 next;
 
 	do {
 		struct kvm_pgtable *pgt = mmu->pgt;
+		/*
+		 * We may be raced on PGT teardown when we release the
+		 * kvm->mmu_lock. That's fine as the PGT is legitimately no
+		 * longer present.
+		 */
 		if (!pgt)
-			return -EINVAL;
+			return lock_dropped ? 0 : -EINVAL;
 
 		next = stage2_range_addr_end(addr, end);
 		ret = fn(pgt, addr, next - addr);
 		if (ret)
 			break;
 
-		if (resched && next != end)
+		if (resched && next != end) {
 			cond_resched_rwlock_write(&kvm->mmu_lock);
+			lock_dropped = true;
+		}
 	} while (addr = next, addr != end);
 
 	return ret;
@@ -313,6 +324,19 @@ static void invalidate_icache_guest_page(void *va, size_t size)
  * we then fully enforce cacheability of RAM, no matter what the guest
  * does.
  */
+
+static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt, u64 addr, u64 size)
+{
+	int ret;
+
+	ret = kvm_pgtable_stage2_unmap(pgt, addr, size);
+	if (ret)
+		return ret;
+
+	kvm_remove_guest_s2_mappings(pgt->mmu, addr, size);
+	return 0;
+}
+
 /**
  * __unmap_stage2_range -- Clear stage2 page table entries to unmap a range
  * @mmu:   The KVM stage-2 MMU pointer
@@ -330,11 +354,17 @@ static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64
 {
 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
 	phys_addr_t end = start + size;
+	int (*fn)(struct kvm_pgtable *, u64, u64);
 
 	lockdep_assert_held_write(&kvm->mmu_lock);
 	WARN_ON(size & ~PAGE_MASK);
-	WARN_ON(stage2_apply_range(mmu, start, end, KVM_PGT_FN(kvm_pgtable_stage2_unmap),
-				   may_block));
+
+	if (kvm_is_nested_s2_mmu(kvm, mmu))
+		fn = kvm_pgtable_stage2_unmap_tracked;
+	else
+		fn = KVM_PGT_FN(kvm_pgtable_stage2_unmap);
+
+	WARN_ON(stage2_apply_range(mmu, start, end, fn, may_block));
 }
 
 void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start,
@@ -866,7 +896,7 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr)
 	 * IPI-ing threads).
 	 */
 	local_irq_save(flags);
-	ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level);
+	ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level, 0);
 	local_irq_restore(flags);
 
 	if (ret)
@@ -1033,6 +1063,8 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t
 
 	mmu->pgd_phys = __pa(pgt->pgd);
 
+	mmu->guest_s2_mappings = RB_ROOT_CACHED;
+
 	if (kvm_is_nested_s2_mmu(kvm, mmu))
 		kvm_init_nested_s2_mmu(mmu);
 
@@ -1122,10 +1154,25 @@ void stage2_unmap_vm(struct kvm *kvm)
 	srcu_read_unlock(&kvm->srcu, idx);
 }
 
+static void guest_s2_tracking_destroy(struct rb_root_cached *tree)
+{
+	struct kvm_guest_s2_mapping *mapping;
+	struct interval_tree_node *node;
+
+	while ((node = interval_tree_iter_first(tree, 0, ULONG_MAX))) {
+		interval_tree_remove(node, tree);
+		mapping = container_of(node, struct kvm_guest_s2_mapping,
+				       canonical);
+		kfree(mapping);
+		cond_resched();
+	}
+}
+
 void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
 {
 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
 	struct kvm_pgtable *pgt = NULL;
+	struct rb_root_cached mappings_tree;
 
 	write_lock(&kvm->mmu_lock);
 	pgt = mmu->pgt;
@@ -1138,12 +1185,18 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
 	if (kvm_is_nested_s2_mmu(kvm, mmu))
 		kvm_init_nested_s2_mmu(mmu);
 
+	mappings_tree = mmu->guest_s2_mappings;
+	mmu->guest_s2_mappings = RB_ROOT_CACHED;
+
 	write_unlock(&kvm->mmu_lock);
 
 	if (pgt) {
 		kvm_stage2_destroy(pgt);
 		kfree(pgt);
 	}
+
+	if (!kvm_is_nested_s2_mmu(kvm, mmu))
+		guest_s2_tracking_destroy(&mappings_tree);
 }
 
 static void hyp_mc_free_fn(void *addr, void *mc)
@@ -1463,14 +1516,12 @@ static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva)
 {
 	unsigned long pa;
 
-	if (is_vm_hugetlb_page(vma) && !(vma->vm_flags & VM_PFNMAP))
+	if (vma_is_hugetlb(vma))
 		return huge_page_shift(hstate_vma(vma));
 
 	if (!(vma->vm_flags & VM_PFNMAP))
 		return PAGE_SHIFT;
 
-	VM_BUG_ON(is_vm_hugetlb_page(vma));
-
 	pa = (vma->vm_pgoff << PAGE_SHIFT) + (hva - vma->vm_start);
 
 #ifndef __PAGETABLE_PMD_FOLDED
@@ -1554,9 +1605,9 @@ static void *get_mmu_memcache(struct kvm_vcpu *vcpu)
 		return &vcpu->arch.pkvm_memcache;
 }
 
-static int topup_mmu_memcache(struct kvm_vcpu *vcpu, void *memcache)
+static int topup_mmu_memcache(struct kvm_s2_mmu *mmu, void *memcache)
 {
-	int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu);
+	int min_pages = kvm_mmu_cache_min_pages(mmu);
 
 	if (!is_protected_kvm_enabled())
 		return kvm_mmu_topup_memory_cache(memcache, min_pages);
@@ -1603,17 +1654,49 @@ struct kvm_s2_fault_desc {
 	struct kvm_s2_trans	*nested;
 	struct kvm_memory_slot	*memslot;
 	unsigned long		hva;
+	unsigned long		esr;
+	struct kvm_s2_mmu	*mmu;
 };
 
-static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
+struct kvm_s2_fault_result {
+	unsigned long mapping_size;
+};
+
+static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd)
+{
+	return esr_fsc_is_permission_fault(s2fd->esr);
+}
+
+static bool kvm_s2_fault_is_exec(const struct kvm_s2_fault_desc *s2fd)
+{
+	return esr_abt_is_exec_fault(s2fd->esr);
+}
+
+static bool kvm_s2_fault_is_write(const struct kvm_s2_fault_desc *s2fd)
+{
+	return esr_abt_is_write_fault(s2fd->esr);
+}
+
+static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd)
+{
+	u64 level;
+
+	if (!kvm_s2_fault_is_perm(s2fd))
+		return 0;
+	level = s2fd->esr & ESR_ELx_FSC_LEVEL;
+	return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level));
+}
+
+static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
+		      struct kvm_s2_fault_result *result)
 {
 	bool write_fault, exec_fault;
-	bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu);
+	bool perm_fault = kvm_s2_fault_is_perm(s2fd);
 	enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
 	enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
-	struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt;
+	struct kvm_pgtable *pgt = s2fd->mmu->pgt;
+	struct kvm_guest_s2_mapping *mapping = NULL;
 	unsigned long mmu_seq;
-	struct page *page;
 	struct kvm *kvm = s2fd->vcpu->kvm;
 	void *memcache = NULL;
 	kvm_pfn_t pfn;
@@ -1622,9 +1705,14 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
 
 	if (!perm_fault) {
 		memcache = get_mmu_memcache(s2fd->vcpu);
-		ret = topup_mmu_memcache(s2fd->vcpu, memcache);
+		ret = topup_mmu_memcache(s2fd->mmu, memcache);
 		if (ret)
 			return ret;
+		if (kvm_is_nested_s2_mmu(kvm, pgt->mmu)) {
+			mapping = kmalloc_obj(struct kvm_guest_s2_mapping, GFP_KERNEL_ACCOUNT);
+			if (!mapping)
+				return -ENOMEM;
+		}
 	}
 
 	if (s2fd->nested)
@@ -1632,8 +1720,8 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
 	else
 		gfn = s2fd->fault_ipa >> PAGE_SHIFT;
 
-	write_fault = kvm_is_write_fault(s2fd->vcpu);
-	exec_fault = kvm_vcpu_trap_is_exec_fault(s2fd->vcpu);
+	write_fault = kvm_s2_fault_is_write(s2fd);
+	exec_fault = kvm_s2_fault_is_exec(s2fd);
 
 	VM_WARN_ON_ONCE(write_fault && exec_fault);
 
@@ -1641,10 +1729,13 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
 	/* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */
 	smp_rmb();
 
-	ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL);
+	ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, NULL);
 	if (ret) {
-		kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE,
-					      write_fault, exec_fault, false);
+		/* If result is non-NULL this is a synthetic fault. */
+		if (!result)
+			kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE,
+						      write_fault, exec_fault, false);
+		kfree(mapping);
 		return ret;
 	}
 
@@ -1678,16 +1769,32 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd)
 		ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE,
 							 __pfn_to_phys(pfn), prot,
 							 memcache, flags);
+		/*
+		 * -EAGAIN from kvm_pgtable_stage2_map() can install mappings.
+		 * We don't know which subrange is installed, track the whole
+		 * thing.
+		 */
+		if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) {
+			kvm_record_guest_s2_mapping(pgt->mmu, gfn << PAGE_SHIFT,
+						    s2fd->fault_ipa, PAGE_SIZE, mapping);
+			mapping = NULL;
+		}
 	}
 
 out_unlock:
-	kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W);
 	kvm_fault_unlock(kvm);
+	kfree(mapping);
 
 	if ((prot & KVM_PGTABLE_PROT_W) && !ret)
 		mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn);
 
-	return ret != -EAGAIN ? ret : 0;
+	if (ret == -EAGAIN)
+		return result ? ret : 0;
+
+	if (result && !ret)
+		result->mapping_size = PAGE_SIZE;
+
+	return ret;
 }
 
 struct kvm_s2_fault_vma_info {
@@ -1709,7 +1816,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
 {
 	unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE;
 	struct kvm_vcpu *vcpu = s2fd->vcpu;
-	struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt;
+	struct kvm_pgtable *pgt = s2fd->mmu->pgt;
 	struct mm_struct *mm = current->mm;
 	struct kvm *kvm = vcpu->kvm;
 	void *hyp_memcache;
@@ -1717,7 +1824,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd)
 	int ret;
 
 	hyp_memcache = get_mmu_memcache(vcpu);
-	ret = topup_mmu_memcache(vcpu, hyp_memcache);
+	ret = topup_mmu_memcache(s2fd->mmu, hyp_memcache);
 	if (ret)
 		return -ENOMEM;
 
@@ -1840,11 +1947,6 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd,
 	return vma_shift;
 }
 
-static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd)
-{
-	return kvm_vcpu_trap_is_permission_fault(s2fd->vcpu);
-}
-
 static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd,
 				     struct kvm_s2_fault_vma_info *s2vi)
 {
@@ -1910,13 +2012,11 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd,
 		return ret;
 
 	s2vi->pfn = __kvm_faultin_pfn(s2fd->memslot, get_canonical_gfn(s2fd, s2vi),
-				      kvm_is_write_fault(s2fd->vcpu) ? FOLL_WRITE : 0,
+				      kvm_s2_fault_is_write(s2fd) ? FOLL_WRITE : 0,
 				      &s2vi->map_writable, &s2vi->page);
 	if (unlikely(is_error_noslot_pfn(s2vi->pfn))) {
-		if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) {
-			kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi->vma_pagesize));
-			return 0;
-		}
+		if (s2vi->pfn == KVM_PFN_ERR_HWPOISON)
+			return -EHWPOISON;
 		return -EFAULT;
 	}
 
@@ -1968,7 +2068,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
 {
 	struct kvm *kvm = s2fd->vcpu->kvm;
 
-	if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu) && s2vi->map_non_cacheable)
+	if (kvm_s2_fault_is_exec(s2fd) && s2vi->map_non_cacheable)
 		return -ENOEXEC;
 
 	/*
@@ -1977,7 +2077,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
 	 * and trigger the exception here. Since the memslot is valid, inject
 	 * the fault back to the guest.
 	 */
-	if (esr_fsc_is_excl_atomic_fault(kvm_vcpu_get_esr(s2fd->vcpu))) {
+	if (esr_fsc_is_excl_atomic_fault(s2fd->esr)) {
 		kvm_inject_dabt_excl_atomic(s2fd->vcpu, kvm_vcpu_get_hfar(s2fd->vcpu));
 		return 1;
 	}
@@ -1986,13 +2086,13 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
 
 	if (s2vi->map_writable && (s2vi->device ||
 				   !memslot_is_logging(s2fd->memslot) ||
-				   kvm_is_write_fault(s2fd->vcpu)))
+				   kvm_s2_fault_is_write(s2fd)))
 		*prot |= KVM_PGTABLE_PROT_W;
 
 	if (s2fd->nested)
 		*prot = adjust_nested_fault_perms(s2fd->nested, *prot);
 
-	if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu))
+	if (kvm_s2_fault_is_exec(s2fd))
 		*prot |= KVM_PGTABLE_PROT_X;
 
 	if (s2vi->map_non_cacheable)
@@ -2016,11 +2116,14 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd,
 static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
 			    const struct kvm_s2_fault_vma_info *s2vi,
 			    enum kvm_pgtable_prot prot,
-			    void *memcache)
+			    void *memcache,
+			    struct kvm_s2_fault_result *result)
 {
 	enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
+	struct kvm_guest_s2_mapping *mapping = NULL;
 	bool writable = prot & KVM_PGTABLE_PROT_W;
 	struct kvm *kvm = s2fd->vcpu->kvm;
+	phys_addr_t canonical_ipa;
 	struct kvm_pgtable *pgt;
 	long perm_fault_granule;
 	long mapping_size;
@@ -2028,17 +2131,26 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
 	gfn_t gfn;
 	int ret;
 
+	if (kvm_is_nested_s2_mmu(kvm, s2fd->mmu)) {
+		mapping = kmalloc_obj(struct kvm_guest_s2_mapping,
+				      GFP_KERNEL_ACCOUNT);
+		if (!mapping) {
+			kvm_release_page_unused(s2vi->page);
+			return -ENOMEM;
+		}
+	}
+
 	kvm_fault_lock(kvm);
-	pgt = s2fd->vcpu->arch.hw_mmu->pgt;
+	pgt = s2fd->mmu->pgt;
 	ret = -EAGAIN;
 	if (mmu_invalidate_retry(kvm, s2vi->mmu_seq))
 		goto out_unlock;
 
-	perm_fault_granule = (kvm_s2_fault_is_perm(s2fd) ?
-			      kvm_vcpu_trap_get_perm_fault_granule(s2fd->vcpu) : 0);
+	perm_fault_granule = kvm_s2_perm_fault_granule(s2fd);
 	mapping_size = s2vi->vma_pagesize;
 	pfn = s2vi->pfn;
 	gfn = s2vi->gfn;
+	canonical_ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi));
 
 	/*
 	 * If we are not forced to use page mapping, check if we are
@@ -2057,6 +2169,7 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
 				goto out_unlock;
 			}
 		}
+		canonical_ipa = ALIGN_DOWN(canonical_ipa, mapping_size);
 	}
 
 	if (!perm_fault_granule && !s2vi->map_non_cacheable && kvm_has_mte(kvm))
@@ -2079,31 +2192,45 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd,
 		ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(gfn), mapping_size,
 							 __pfn_to_phys(pfn), prot,
 							 memcache, flags);
+		/*
+		 * -EAGAIN from kvm_pgtable_stage2_map() can install mappings.
+		 * We don't know which subrange is installed, track the whole
+		 * thing.
+		 */
+		if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) {
+			kvm_record_guest_s2_mapping(pgt->mmu, canonical_ipa,
+						    gfn_to_gpa(gfn), mapping_size, mapping);
+			mapping = NULL;
+		}
 	}
 
 out_unlock:
 	kvm_release_faultin_page(kvm, s2vi->page, !!ret, writable);
 	kvm_fault_unlock(kvm);
+	kfree(mapping);
 
 	/*
 	 * Mark the page dirty only if the fault is handled successfully,
 	 * making sure we adjust the canonical IPA if the mapping size has
 	 * been updated (via a THP upgrade, for example).
 	 */
-	if (writable && !ret) {
-		phys_addr_t ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi));
-		ipa &= ~(mapping_size - 1);
-		mark_page_dirty_in_slot(kvm, s2fd->memslot, gpa_to_gfn(ipa));
-	}
+	if (writable && !ret)
+		mark_page_dirty_in_slot(kvm, s2fd->memslot,
+					gpa_to_gfn(canonical_ipa));
 
-	if (ret != -EAGAIN)
-		return ret;
-	return 0;
+	if (ret == -EAGAIN)
+		return result ? ret : 0;
+
+	if (result && !ret)
+		result->mapping_size = mapping_size;
+
+	return ret;
 }
 
-static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
+static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd,
+			  struct kvm_s2_fault_result *result)
 {
-	bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu);
+	bool perm_fault = kvm_s2_fault_is_perm(s2fd);
 	struct kvm_s2_fault_vma_info s2vi = {};
 	enum kvm_pgtable_prot prot;
 	void *memcache;
@@ -2121,7 +2248,7 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
 	memcache = get_mmu_memcache(s2fd->vcpu);
 	if (!perm_fault || memslot_is_logging(s2fd->memslot) ||
 	    is_protected_kvm_enabled()) {
-		ret = topup_mmu_memcache(s2fd->vcpu, memcache);
+		ret = topup_mmu_memcache(s2fd->mmu, memcache);
 		if (ret)
 			return ret;
 	}
@@ -2131,6 +2258,13 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
 	 * get block mapping for device MMIO region.
 	 */
 	ret = kvm_s2_fault_pin_pfn(s2fd, &s2vi);
+	if (ret == -EHWPOISON) {
+		/* If result is specified, let the caller handle this. */
+		if (result)
+			return -EHWPOISON;
+		kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi.vma_pagesize));
+		return 0;
+	}
 	if (ret != 1)
 		return ret;
 
@@ -2140,7 +2274,7 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd)
 		return ret;
 	}
 
-	return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache);
+	return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache, result);
 }
 
 /* Resolve the access fault by making the page young again. */
@@ -2250,7 +2384,8 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu)
 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 {
 	struct kvm_s2_trans nested_trans, *nested = NULL;
-	unsigned long esr;
+	unsigned long esr = kvm_vcpu_get_esr(vcpu);
+	struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu;
 	phys_addr_t fault_ipa; /* The address we faulted on */
 	phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */
 	struct kvm_memory_slot *memslot;
@@ -2259,11 +2394,9 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 	gfn_t gfn;
 	int ret, idx;
 
-	if (kvm_vcpu_abt_issea(vcpu))
+	if (esr_abt_is_sea(esr))
 		return kvm_handle_guest_sea(vcpu);
 
-	esr = kvm_vcpu_get_esr(vcpu);
-
 	/*
 	 * The fault IPA should be reliable at this point as we're not dealing
 	 * with an SEA.
@@ -2272,7 +2405,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 	if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm))
 		return -EFAULT;
 
-	is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
+	is_iabt = esr_trap_is_iabt(esr);
 
 	if (esr_fsc_is_translation_fault(esr)) {
 		/* Beyond sanitised PARange (which is the IPA limit) */
@@ -2282,14 +2415,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 		}
 
 		/* Falls between the IPA range and the PARange? */
-		if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) {
+		if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(mmu->vtcr))) {
 			fault_ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu));
 
 			return kvm_inject_sea(vcpu, is_iabt, fault_ipa);
 		}
 	}
 
-	trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
+	trace_kvm_guest_fault(*vcpu_pc(vcpu), esr,
 			      kvm_vcpu_get_hfar(vcpu), fault_ipa);
 
 	/* Check the stage-2 fault is trans. fault or write fault */
@@ -2297,10 +2430,10 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 	    !esr_fsc_is_permission_fault(esr) &&
 	    !esr_fsc_is_access_flag_fault(esr) &&
 	    !esr_fsc_is_excl_atomic_fault(esr)) {
-		kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
-			kvm_vcpu_trap_get_class(vcpu),
-			(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
-			(unsigned long)kvm_vcpu_get_esr(vcpu));
+		kvm_err("Unsupported FSC: EC=%#lx xFSC=%#lx ESR_EL2=%#lx\n",
+			ESR_ELx_EC(esr),
+			(unsigned long)(esr & ESR_ELx_FSC),
+			(unsigned long)esr);
 		return -EFAULT;
 	}
 
@@ -2319,8 +2452,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 	 * nothing to walk and we treat it as a 1:1 before going through the
 	 * canonical translation.
 	 */
-	if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) &&
-	    vcpu->arch.hw_mmu->nested_stage2_enabled) {
+	if (kvm_is_nested_s2_mmu(vcpu->kvm, mmu) &&
+	    mmu->nested_stage2_enabled) {
 		u32 esr;
 
 		ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans);
@@ -2349,7 +2482,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 	gfn = ipa >> PAGE_SHIFT;
 	memslot = gfn_to_memslot(vcpu->kvm, gfn);
 	hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
-	write_fault = kvm_is_write_fault(vcpu);
+	write_fault = esr_abt_is_write_fault(esr);
 	if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
 		/*
 		 * The guest has put either its instructions or its page-tables
@@ -2362,7 +2495,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 			goto out;
 		}
 
-		if (kvm_vcpu_abt_iss1tw(vcpu)) {
+		if (esr_abt_is_s1ptw(esr)) {
 			ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
 			goto out_unlock;
 		}
@@ -2377,7 +2510,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 		 * So let's assume that the guest is just being
 		 * cautious, and skip the instruction.
 		 */
-		if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) {
+		if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) {
 			kvm_incr_pc(vcpu);
 			ret = 1;
 			goto out_unlock;
@@ -2395,7 +2528,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 	}
 
 	/* Userspace should not be able to register out-of-bounds IPAs */
-	VM_BUG_ON(ipa >= kvm_phys_size(vcpu->arch.hw_mmu));
+	VM_BUG_ON(ipa >= kvm_phys_size(mmu));
 
 	if (esr_fsc_is_access_flag_fault(esr)) {
 		handle_access_fault(vcpu, fault_ipa);
@@ -2409,19 +2542,20 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 		.nested		= nested,
 		.memslot	= memslot,
 		.hva		= hva,
+		.esr		= esr,
+		.mmu		= mmu,
 	};
 
 	if (kvm_vm_is_protected(vcpu->kvm)) {
 		ret = pkvm_mem_abort(&s2fd);
 	} else {
-		VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) &&
-				!write_fault &&
-				!kvm_vcpu_trap_is_exec_fault(vcpu));
+		VM_WARN_ON_ONCE(kvm_s2_fault_is_perm(&s2fd) && !write_fault &&
+				!kvm_s2_fault_is_exec(&s2fd));
 
 		if (kvm_slot_has_gmem(memslot))
-			ret = gmem_abort(&s2fd);
+			ret = gmem_abort(&s2fd, NULL);
 		else
-			ret = user_mem_abort(&s2fd);
+			ret = user_mem_abort(&s2fd, NULL);
 	}
 
 	if (ret == 0)
@@ -2436,14 +2570,16 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
 
 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
 {
+	gpa_t gpa = range->start << PAGE_SHIFT;
+	size_t size = (range->end - range->start) << PAGE_SHIFT;
+	bool may_block = range->may_block;
+
 	if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm))
 		return false;
 
-	__unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT,
-			     (range->end - range->start) << PAGE_SHIFT,
-			     range->may_block);
+	__unmap_stage2_range(&kvm->arch.mmu, gpa, size, may_block);
+	kvm_nested_unmap_cipa_range(kvm, gpa, size, may_block);
 
-	kvm_nested_s2_unmap(kvm, range->may_block);
 	return false;
 }
 
@@ -2725,7 +2861,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 
 	write_lock(&kvm->mmu_lock);
 	kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, true);
-	kvm_nested_s2_unmap(kvm, true);
+	kvm_nested_unmap_cipa_range(kvm, gpa, size, true);
 	write_unlock(&kvm->mmu_lock);
 }
 
@@ -2796,3 +2932,146 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
 
 	trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
 }
+
+/*
+ * Try to walk to the specified GPA in canonical mmu - if unmapped returns 0, if
+ * mapped returns the granule size, otherwise returns an error.
+ */
+static long kvm_walk_s2(struct kvm_pgtable *pgt,
+			gpa_t gpa, s8 *level)
+{
+	struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu);
+	kvm_pte_t pte;
+	long ret;
+
+	guard(read_lock)(&kvm->mmu_lock);
+
+	ret = kvm_pgtable_get_leaf(pgt, gpa, &pte, level,
+				   KVM_PGTABLE_WALK_SHARED);
+	if (ret)
+		return ret;
+	/* Unpopulated, must fault. */
+	if (!kvm_pte_valid(pte))
+		return 0;
+	return kvm_granule_size(*level);
+}
+
+/* Synthesised data abort at specified page table level. */
+#define PRE_FAULT_ESR(level)				\
+	 ((ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) |	\
+	  ESR_ELx_IL | ESR_ELx_FSC_FAULT_L(level))
+
+/* Retrieve either a read-only or a read/write hva. */
+static hva_t gfn_to_hva_memslot_read(struct kvm_memory_slot *slot, gfn_t gfn)
+{
+	return gfn_to_hva_memslot_prot(slot, gfn, /*writable=*/NULL);
+}
+
+static long __pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu,
+			   gpa_t gpa, struct kvm_memory_slot *memslot, s8 level)
+{
+	const bool is_gmem = kvm_slot_has_gmem(memslot);
+	const gfn_t gfn = gpa_to_gfn(gpa);
+	const hva_t hva = is_gmem ? 0 : gfn_to_hva_memslot_read(memslot, gfn);
+	const struct kvm_s2_fault_desc s2fd = {
+		.vcpu		= vcpu,
+		.fault_ipa	= gpa,
+		.nested		= NULL,
+		.memslot	= memslot,
+		.hva		= hva,
+		.esr		= PRE_FAULT_ESR(level),
+		.mmu		= mmu,
+	};
+	struct kvm_s2_fault_result result = {};
+	long ret;
+
+	if (kvm_is_error_hva(hva))
+		return -EFAULT;
+
+	if (is_gmem)
+		ret = gmem_abort(&s2fd, &result);
+	else
+		ret = user_mem_abort(&s2fd, &result);
+	if (IS_ERR_VALUE(ret))
+		return ret;
+	return result.mapping_size;
+}
+
+static long pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu,
+			 gpa_t gpa, struct kvm_memory_slot *memslot)
+{
+	s8 level;
+	long ret;
+
+	/* Try a walk first. */
+	ret = kvm_walk_s2(mmu->pgt, gpa, &level);
+	if (ret)
+		return ret;
+	/* OK, have to fault page in. */
+	return __pre_fault_s2(mmu, vcpu, gpa, memslot, level);
+}
+
+static unsigned long
+pre_fault_bytes_consumed(gpa_t gpa, unsigned long granule_size,
+			 unsigned long bytes_remaining)
+{
+	/* Granules are always a power-of-2. */
+	const unsigned long granule_bytes_remaining =
+		granule_size - (gpa % granule_size);
+
+	return min(granule_bytes_remaining, bytes_remaining);
+}
+
+/* If you lose the race this many times, time to give up. */
+#define MAX_PRE_FAULT_RETRIES 3
+
+int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
+{
+	if (is_protected_kvm_enabled())
+		return -EOPNOTSUPP;
+	if (!kvm_vcpu_initialized(vcpu))
+		return -ENOEXEC;
+
+	return 0;
+}
+
+/**
+ * kvm_arch_vcpu_pre_fault_memory - pre-fault stage-2 page tables for the
+ * specified GPA.
+ * @vcpu:	The VCPU pointer
+ * @range:	{gpa, size, flags} tuple
+ *
+ * The mapping performed is always best-effort - faulting in is necessarily
+ * racey. The ranges faulted in are canonical, nested page tables are ignored.
+ *
+ * @range->gpa specifies the GPA to pre-fault, @range->size specifies how many
+ * bytes remain to be pre-faulted and @range->flags is reserved and must be 0.
+ *
+ * Returns: the number of bytes the pre-fault consumed, or an error.
+ */
+long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
+				    struct kvm_pre_fault_memory *range)
+{
+	struct kvm *kvm = vcpu->kvm;
+	const u64 bytes_remaining = range->size;
+	struct kvm_s2_mmu *mmu = &kvm->arch.mmu; /* Canonical. */
+	struct kvm_memory_slot *memslot;
+	const gpa_t gpa = range->gpa;
+	int num_retries = 0;
+	long ret;
+
+	memslot = gfn_to_memslot(kvm, gpa_to_gfn(gpa));
+	if (!memslot)
+		return -ENOENT;
+	/* SRCU must be released for progress and only userland can do that. */
+	if (memslot->flags & KVM_MEMSLOT_INVALID)
+		return -EAGAIN;
+
+	do {
+		ret = pre_fault_s2(mmu, vcpu, gpa, memslot);
+	} while (ret == -EAGAIN && num_retries++ < MAX_PRE_FAULT_RETRIES);
+
+	if (IS_ERR_VALUE(ret))
+		return ret;
+	return pre_fault_bytes_consumed(gpa, ret, bytes_remaining);
+}
diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index 3c4fc566eafc63..f1c414f1c3fafe 100644
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/interval_tree.h>
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
 
@@ -44,12 +45,27 @@ struct vncr_tlb {
  * will invalidate them more often).
  */
 #define S2_MMU_PER_VCPU		2
+#define S2_MMU_PER_VM		(KVM_MAX_VCPUS * S2_MMU_PER_VCPU)
 
-void kvm_init_nested(struct kvm *kvm)
+int kvm_init_nested(struct kvm *kvm)
 {
-	kvm->arch.nested_mmus = NULL;
+	kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *,
+					      S2_MMU_PER_VM,
+					      GFP_KERNEL_ACCOUNT);
 	kvm->arch.nested_mmus_size = 0;
 	atomic_set(&kvm->arch.vncr_tlb_count, 0);
+	spin_lock_init(&kvm->arch.guest_s2_tracking_lock);
+
+	return kvm->arch.nested_mmus ? 0 : -ENOMEM;
+}
+
+void kvm_destroy_nested(struct kvm *kvm)
+{
+	for (int i = 0; i < kvm->arch.nested_mmus_size; i+= S2_MMU_PER_VCPU)
+		kvfree(kvm->arch.nested_mmus[i]);
+
+	kvm->arch.nested_mmus_size = 0;
+	kvfree(kvm->arch.nested_mmus);
 }
 
 static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
@@ -70,8 +86,9 @@ static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
 int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
 {
 	struct kvm *kvm = vcpu->kvm;
-	struct kvm_s2_mmu *tmp;
-	int num_mmus, ret = 0;
+	int num_mmus;
+
+	lockdep_assert_held(&kvm->arch.config_lock);
 
 	if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) &&
 	    !cpus_have_final_cap(ARM64_HAS_HCR_NV1))
@@ -84,51 +101,45 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
 	if (!vcpu->arch.ctxt.vncr_array)
 		return -ENOMEM;
 
-	/*
-	 * Let's treat memory allocation failures as benign: If we fail to
-	 * allocate anything, return an error and keep the allocated array
-	 * alive. Userspace may try to recover by initializing the vcpu
-	 * again, and there is no reason to affect the whole VM for this.
-	 */
 	num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU;
 
 	if (num_mmus > kvm->arch.nested_mmus_size) {
-		tmp = kvzalloc_objs(*tmp, num_mmus, GFP_KERNEL_ACCOUNT);
-		if (!tmp)
-			return -ENOMEM;
+		struct kvm_s2_mmu *tmp;
+		int i, ret = 0;
 
-		write_lock(&kvm->mmu_lock);
-
-		if (kvm->arch.nested_mmus_size) {
-			memcpy(tmp, kvm->arch.nested_mmus,
-			       size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size));
+		tmp = kvzalloc_objs(*tmp, S2_MMU_PER_VCPU, GFP_KERNEL_ACCOUNT);
+		if (!tmp)
+			ret = -ENOMEM;
 
-			for (int i = 0; i < kvm->arch.nested_mmus_size; i++)
-				tmp[i].pgt->mmu = &tmp[i];
+		for (i = 0; !ret && i < S2_MMU_PER_VCPU; i++) {
+			ret = init_nested_s2_mmu(kvm, &tmp[i]);
+			if (ret)
+				break;
 		}
 
-		swap(kvm->arch.nested_mmus, tmp);
-
-		write_unlock(&kvm->mmu_lock);
+		if (ret) {
+			while (--i >= 0)
+				kvm_free_stage2_pgd(&tmp[i]);
 
-		kvfree(tmp);
-	}
+			kvfree(tmp);
+			free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
+			vcpu->arch.ctxt.vncr_array = NULL;
+			return ret;
+		}
 
-	for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++)
-		ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]);
+		for (i = 0; i < S2_MMU_PER_VCPU; i++)
+			kvm_nested_s2_ptdump_create_debugfs(&tmp[i], i + kvm->arch.nested_mmus_size);
 
-	if (ret) {
-		for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++)
-			kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]);
+		guard(write_lock)(&kvm->mmu_lock);
 
-		free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
-		vcpu->arch.ctxt.vncr_array = NULL;
+		for (i = 0; i < S2_MMU_PER_VCPU; i++) {
+			tmp[i].s2_mmu_idx = i + kvm->arch.nested_mmus_size;
+			kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i];
+		}
 
-		return ret;
+		kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU;
 	}
 
-	kvm->arch.nested_mmus_size = num_mmus;
-
 	return 0;
 }
 
@@ -654,7 +665,7 @@ static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
 		return 0;
 
 	tmp &= ~(sz - 1);
-	if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
+	if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL, 0))
 		goto again;
 	if (!(pte & PTE_VALID))
 		goto again;
@@ -742,7 +753,7 @@ void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
 	write_lock(&kvm->mmu_lock);
 
 	for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
-		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
+		struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i];
 
 		if (!kvm_s2_mmu_valid(mmu))
 			continue;
@@ -784,7 +795,7 @@ struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu)
 	 *   if S2 translation is disabled.
 	 */
 	for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
-		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
+		struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i];
 
 		if (!kvm_s2_mmu_valid(mmu))
 			continue;
@@ -823,7 +834,7 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
 	for (i = kvm->arch.nested_mmus_next;
 	     i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next);
 	     i++) {
-		s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
+		s2_mmu = kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size];
 
 		if (atomic_read(&s2_mmu->refcnt) == 0)
 			break;
@@ -834,10 +845,8 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
 	kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
 
 	/* Make sure we don't forget to do the laundry */
-	if (kvm_s2_mmu_valid(s2_mmu)) {
-		kvm_nested_s2_ptdump_remove_debugfs(s2_mmu);
+	if (kvm_s2_mmu_valid(s2_mmu))
 		s2_mmu->pending_unmap = true;
-	}
 
 	/*
 	 * The virtual VMID (modulo CnP) will be used as a key when matching
@@ -851,8 +860,6 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
 	s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
 	s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
 
-	kvm_nested_s2_ptdump_create_debugfs(s2_mmu);
-
 out:
 	atomic_inc(&s2_mmu->refcnt);
 
@@ -868,6 +875,87 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
 	return s2_mmu;
 }
 
+#define S2_MMU_IDX_MASK	GENMASK_ULL(11, 0)
+
+static void tag_s2_mapping_mmu(struct kvm_guest_s2_mapping *mapping,
+			       struct kvm_s2_mmu *mmu)
+{
+	BUILD_BUG_ON(S2_MMU_PER_VM > SZ_4K);
+	mapping->nested.start &= ~S2_MMU_IDX_MASK;
+	mapping->nested.start |= mmu->s2_mmu_idx;
+}
+
+static struct kvm_s2_mmu *s2_mapping_to_mmu(struct kvm *kvm,
+					    struct kvm_guest_s2_mapping *mapping)
+{
+	return kvm->arch.nested_mmus[mapping->nested.start & S2_MMU_IDX_MASK];
+}
+
+static unsigned long s2_mapping_to_nested_start(struct kvm_guest_s2_mapping *mapping)
+{
+	return mapping->nested.start & ~S2_MMU_IDX_MASK;
+}
+
+void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa,
+				 gpa_t nested_ipa, size_t map_size,
+				 struct kvm_guest_s2_mapping *mapping)
+{
+	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
+
+	lockdep_assert_held_read(&kvm->mmu_lock);
+
+	canonical_ipa = ALIGN_DOWN(canonical_ipa, map_size);
+	nested_ipa = ALIGN_DOWN(nested_ipa, map_size);
+
+	mapping->canonical.start = canonical_ipa;
+	mapping->canonical.last  = canonical_ipa + map_size - 1;
+
+	mapping->nested.start    = nested_ipa;
+	mapping->nested.last     = nested_ipa + map_size - 1;
+
+	tag_s2_mapping_mmu(mapping, mmu);
+
+	guard(spinlock)(&kvm->arch.guest_s2_tracking_lock);
+	interval_tree_insert(&mapping->nested, &mmu->guest_s2_mappings);
+	interval_tree_insert(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings);
+}
+
+void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa,
+				  size_t size)
+{
+	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
+	struct interval_tree_node *node, *next;
+	struct kvm_guest_s2_mapping *mapping;
+	gpa_t nipa_end = nipa + size - 1;
+
+	/*
+	 * See kvm_nested_unmap_cipa_range() for why guest_s2_tracking_lock
+	 * isn't taken here.
+	 */
+	lockdep_assert_held_write(&kvm->mmu_lock);
+
+	node = interval_tree_iter_first(&mmu->guest_s2_mappings, nipa, nipa_end);
+	while (node) {
+		unsigned long nested_start;
+
+		next = interval_tree_iter_next(node, nipa, nipa_end);
+		mapping = container_of(node, struct kvm_guest_s2_mapping,
+				       nested);
+		/*
+		 * Tracking must be conservative on removal, only remove
+		 * mappings that are within the unmap range.
+		 */
+		nested_start = s2_mapping_to_nested_start(mapping);
+		if (nipa <= nested_start && nipa_end >= mapping->nested.last) {
+			interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings);
+			interval_tree_remove(&mapping->canonical,
+					     &kvm->arch.mmu.guest_s2_mappings);
+			kfree(mapping);
+		}
+		node = next;
+	}
+}
+
 void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
 {
 	/* CnP being set denotes an invalid entry */
@@ -1260,6 +1348,17 @@ void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val)
 	invalidate_vncr_va(vcpu->kvm, &scope);
 }
 
+static void kvm_invalidate_vncr_ipa_all(struct kvm *kvm)
+{
+	struct kvm_pgtable *pgt = kvm->arch.mmu.pgt;
+
+	lockdep_assert_held_write(&kvm->mmu_lock);
+
+	/* if the mmu lock was dropped, pgt teardown may have raced. */
+	if (pgt)
+		kvm_invalidate_vncr_ipa(kvm, 0, BIT(pgt->ia_bits));
+}
+
 void kvm_nested_s2_wp(struct kvm *kvm)
 {
 	int i;
@@ -1270,13 +1369,63 @@ void kvm_nested_s2_wp(struct kvm *kvm)
 		return;
 
 	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
-		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
+		struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i];
 
 		if (kvm_s2_mmu_valid(mmu))
 			kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu));
 	}
 
-	kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
+	kvm_invalidate_vncr_ipa_all(kvm);
+}
+
+void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, size_t unmap_size,
+				 bool may_block)
+{
+	gpa_t cipa_end = cipa + unmap_size - 1;
+	struct kvm_guest_s2_mapping *mapping;
+	struct interval_tree_node *node;
+	size_t mapping_size;
+
+	/*
+	 * Guest s2 tracking interval trees are only accessed while holding the
+	 * mmu_lock, hence we don't have to take guest_s2_tracking_lock if the
+	 * mmu_lock is held for write. This saves us from having to manually
+	 * lock/unlock guest_s2_tracking_lock below around
+	 * cond_resched_rwlock_write().
+	 */
+	lockdep_assert_held_write(&kvm->mmu_lock);
+
+	if (!kvm->arch.nested_mmus_size)
+		return;
+
+	while ((node = interval_tree_iter_first(&kvm->arch.mmu.guest_s2_mappings,
+						cipa, cipa_end))) {
+		unsigned long nested_start;
+		struct kvm_s2_mmu *mmu;
+
+		mapping = container_of(node, struct kvm_guest_s2_mapping,
+				       canonical);
+		nested_start = s2_mapping_to_nested_start(mapping);
+		mmu = s2_mapping_to_mmu(kvm, mapping);
+
+		/* We could race against MMU teardown, which frees mmu->pgt. */
+		if (mmu->pgt) {
+			mapping_size = mapping->nested.last - nested_start + 1;
+
+			if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mmu->pgt, nested_start,
+								  mapping_size)))
+				return;
+
+			interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings);
+		}
+		interval_tree_remove(node, &kvm->arch.mmu.guest_s2_mappings);
+		kfree(mapping);
+
+		if (may_block)
+			cond_resched_rwlock_write(&kvm->mmu_lock);
+	}
+
+	kvm_invalidate_vncr_ipa(kvm, cipa, cipa + unmap_size);
 }
 
 void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
@@ -1289,13 +1438,13 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
 		return;
 
 	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
-		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
+		struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i];
 
 		if (kvm_s2_mmu_valid(mmu))
 			kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block);
 	}
 
-	kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits));
+	kvm_invalidate_vncr_ipa_all(kvm);
 }
 
 void kvm_nested_s2_flush(struct kvm *kvm)
@@ -1308,7 +1457,7 @@ void kvm_nested_s2_flush(struct kvm *kvm)
 		return;
 
 	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
-		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
+		struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i];
 
 		if (kvm_s2_mmu_valid(mmu))
 			kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu));
@@ -1317,17 +1466,12 @@ void kvm_nested_s2_flush(struct kvm *kvm)
 
 void kvm_arch_flush_shadow_all(struct kvm *kvm)
 {
-	int i;
-
-	for (i = 0; i < kvm->arch.nested_mmus_size; i++) {
-		struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i];
+	for (int i = 0; i < kvm->arch.nested_mmus_size; i++) {
+		struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i];
 
 		if (!WARN_ON(atomic_read(&mmu->refcnt)))
 			kvm_free_stage2_pgd(mmu);
 	}
-	kvfree(kvm->arch.nested_mmus);
-	kvm->arch.nested_mmus = NULL;
-	kvm->arch.nested_mmus_size = 0;
 	kvm_uninit_stage2_mmu(kvm);
 }
 
@@ -1415,7 +1559,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
 	bool write_fault, writable;
 	unsigned long mmu_seq;
 	struct vncr_tlb *vt;
-	struct page *page;
+	struct page *page = NULL;
 	u64 va, pfn, gfn;
 	int ret;
 
@@ -1471,7 +1615,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
 			return -EFAULT;
 		}
 	} else {
-		ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
+		ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, NULL);
 		if (ret) {
 			kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE,
 					      write_fault, false, false);
diff --git a/arch/arm64/kvm/pauth.c b/arch/arm64/kvm/pauth.c
index d5eb3ae876be4f..bb6c42e7805834 100644
--- a/arch/arm64/kvm/pauth.c
+++ b/arch/arm64/kvm/pauth.c
@@ -5,7 +5,7 @@
  *
  * Primitive PAuth emulation for ERETAA/ERETAB.
  *
- * This code assumes that is is run from EL2, and that it is part of
+ * This code assumes that it is run from EL2, and that it is part of
  * the emulation of ERETAx for a guest hypervisor. That's a lot of
  * baked-in assumptions and shortcuts.
  *
diff --git a/arch/arm64/kvm/ptdump.c b/arch/arm64/kvm/ptdump.c
index 69899797dbad8d..9f0aebacea5bbe 100644
--- a/arch/arm64/kvm/ptdump.c
+++ b/arch/arm64/kvm/ptdump.c
@@ -17,7 +17,7 @@
 
 #define MARKERS_LEN		2
 #define KVM_PGTABLE_MAX_LEVELS	(KVM_PGTABLE_LAST_LEVEL + 1)
-#define S2FNAMESZ		sizeof("0x0123456789abcdef-0x0123456789abcdef-s2-disabled")
+#define S2FNAMESZ		sizeof("nested_mmu0000")
 
 struct kvm_ptdump_guest_state {
 	struct kvm_s2_mmu	*mmu;
@@ -115,15 +115,28 @@ static int kvm_ptdump_build_levels(struct ptdump_pg_level *level, u32 start_lvl)
 
 static struct kvm_ptdump_guest_state *kvm_ptdump_parser_create(struct kvm_s2_mmu *mmu)
 {
+	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
 	struct kvm_ptdump_guest_state *st;
-	struct kvm_pgtable *pgtable = mmu->pgt;
+	s8 start_level;
 	int ret;
 
+	/*
+	 * We only need the pgt start level to initialize the ptdump, get it
+	 * while holding the mmu_lock. It's fine if the pgt gets freed
+	 * afterwards, we'll check again when doing the actual dump.
+	 */
+	scoped_guard(read_lock, &kvm->mmu_lock) {
+		if (mmu->pgt)
+			start_level = mmu->pgt->start_level;
+		else
+			return ERR_PTR(-ENOENT);
+	}
+
 	st = kzalloc_obj(struct kvm_ptdump_guest_state, GFP_KERNEL_ACCOUNT);
 	if (!st)
 		return ERR_PTR(-ENOMEM);
 
-	ret = kvm_ptdump_build_levels(&st->level[0], pgtable->start_level);
+	ret = kvm_ptdump_build_levels(&st->level[0], start_level);
 	if (ret) {
 		kfree(st);
 		return ERR_PTR(ret);
@@ -149,6 +162,9 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused)
 	};
 
 	guard(write_lock)(&kvm->mmu_lock);
+	if (!mmu->pgt)
+		return 0;
+
 	st->parser_state = (struct ptdump_pg_state) {
 		.marker		= &st->ipa_marker[0],
 		.end_address	= BIT(mmu->pgt->ia_bits),
@@ -157,6 +173,15 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused)
 		.seq		= m,
 	};
 
+	if (kvm_is_nested_s2_mmu(kvm, mmu)) {
+		if (kvm_s2_mmu_valid(mmu))
+			seq_printf(m, "VTCR: 0x%016llx VTTBR: 0x%016llx s2: %s\n",
+				   mmu->tlb_vtcr, mmu->tlb_vttbr,
+				   mmu->nested_stage2_enabled ? "enabled" : "disabled");
+		else
+			return 0;
+	}
+
 	ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker);
 	if (ret)
 		return ret;
@@ -211,17 +236,27 @@ static const struct file_operations kvm_ptdump_guest_fops = {
 
 static int kvm_pgtable_range_show(struct seq_file *m, void *unused)
 {
-	struct kvm_pgtable *pgtable = m->private;
+	struct kvm_s2_mmu *mmu = m->private;
+	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
+
+	guard(read_lock)(&kvm->mmu_lock);
+
+	if (mmu->pgt)
+		seq_printf(m, "%2u\n", mmu->pgt->ia_bits);
 
-	seq_printf(m, "%2u\n", pgtable->ia_bits);
 	return 0;
 }
 
 static int kvm_pgtable_levels_show(struct seq_file *m, void *unused)
 {
-	struct kvm_pgtable *pgtable = m->private;
+	struct kvm_s2_mmu *mmu = m->private;
+	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
+
+	guard(read_lock)(&kvm->mmu_lock);
+
+	if (mmu->pgt)
+		seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - mmu->pgt->start_level);
 
-	seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - pgtable->start_level);
 	return 0;
 }
 
@@ -230,15 +265,12 @@ static int kvm_pgtable_debugfs_open(struct inode *m, struct file *file,
 {
 	struct kvm_s2_mmu *mmu = m->i_private;
 	struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
-	struct kvm_pgtable *pgtable;
 	int ret;
 
 	if (!kvm_get_kvm_safe(kvm))
 		return -ENOENT;
 
-	pgtable = mmu->pgt;
-
-	ret = single_open(file, show, pgtable);
+	ret = single_open(file, show, mmu);
 	if (ret < 0)
 		kvm_put_kvm(kvm);
 	return ret;
@@ -276,30 +308,23 @@ static const struct file_operations kvm_pgtable_levels_fops = {
 	.release	= kvm_pgtable_debugfs_close,
 };
 
-void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu)
+void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx)
 {
-	struct dentry *dent;
 	char file_name[S2FNAMESZ];
 
-	snprintf(file_name, sizeof(file_name), "0x%016llx-0x%016llx-s2-%sabled",
-		 mmu->tlb_vttbr,
-		 mmu->tlb_vtcr,
-		 mmu->nested_stage2_enabled ? "en" : "dis");
-
-	dent = debugfs_create_file(file_name, 0400,
-				   mmu->arch->debugfs_nv_dentry, mmu,
-				   &kvm_ptdump_guest_fops);
+	snprintf(file_name, sizeof(file_name), "nested_mmu%d", idx);
 
-	mmu->shadow_pt_debugfs_dentry = dent;
-}
-
-void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu)
-{
-	debugfs_remove(mmu->shadow_pt_debugfs_dentry);
+	debugfs_create_file(file_name, 0400,
+			    mmu->arch->debugfs_nv_dentry, mmu,
+			    &kvm_ptdump_guest_fops);
 }
 
 void kvm_s2_ptdump_create_debugfs(struct kvm *kvm)
 {
+	/* pKVM handles the page-tables in the hypervisor */
+	if (is_protected_kvm_enabled())
+		return;
+
 	debugfs_create_file("stage2_page_tables", 0400, kvm->debugfs_dentry,
 			    &kvm->arch.mmu, &kvm_ptdump_guest_fops);
 	debugfs_create_file("ipa_range", 0400, kvm->debugfs_dentry,
diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c
index 9724c320126b7b..69377195e18b70 100644
--- a/arch/arm64/kvm/stacktrace.c
+++ b/arch/arm64/kvm/stacktrace.c
@@ -198,9 +198,6 @@ static void hyp_dump_backtrace(unsigned long hyp_offset)
 }
 
 #ifdef CONFIG_PKVM_STACKTRACE
-DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
-			 pkvm_stacktrace);
-
 /*
  * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
  *
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 44aae52c473d7b..a59d530db11920 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -926,6 +926,7 @@ static u64 *demux_wb_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd)
 	struct kvm_guest_debug_arch *dbg = &vcpu->arch.vcpu_debug_state;
 
 	switch (rd->Op2) {
+	case 0b001:
 	case 0b100:
 		return &dbg->dbg_bvr[rd->CRm];
 	case 0b101:
@@ -2511,9 +2512,10 @@ static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
 
 	/*
 	 * Once the VM has started the ID registers are immutable. Reject any
-	 * write that does not match the final register value.
+	 * write that does not match the final register value once we have
+	 * got far enough into first running the VM to use the values.
 	 */
-	if (kvm_vm_has_ran_once(vcpu->kvm)) {
+	if (kvm_id_regs_final(vcpu->kvm)) {
 		if (val != read_id_reg(vcpu, rd))
 			ret = -EBUSY;
 		else
@@ -2547,7 +2549,7 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val)
 
 	lockdep_assert_held(&kvm->arch.config_lock);
 
-	if (KVM_BUG_ON(kvm_vm_has_ran_once(kvm) || !p, kvm))
+	if (KVM_BUG_ON(kvm_id_regs_final(kvm) || !p, kvm))
 		return;
 
 	*p = val;
@@ -3243,10 +3245,10 @@ static int set_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
 		return -EINVAL;
 
 	/*
-	 * Once the VM has started the ID registers are immutable. Reject the
-	 * write if userspace tries to change it.
+	 * Once we have been far enough into starting the VM the ID registers
+	 * are immutable. Reject the write if userspace tries to change it.
 	 */
-	if (kvm_vm_has_ran_once(kvm))
+	if (kvm_id_regs_final(kvm))
 		return -EBUSY;
 
 	/*
@@ -5861,26 +5863,17 @@ void kvm_calculate_traps(struct kvm_vcpu *vcpu)
 }
 
 /*
- * Perform last adjustments to the ID registers that are implied by the
+ * Do system register finalization that is shared by the whole guest. This
+ * includes last adjustments to the ID registers that are implied by the
  * configuration outside of the ID regs themselves, as well as any
  * initialisation that directly depend on these ID registers (such as
  * RES0/RES1 behaviours). This is not the place to configure traps though.
- *
- * Because this can be called once per CPU, changes must be idempotent.
  */
-int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu)
+static int kvm_vm_finalize_sys_regs(struct kvm *kvm)
 {
-	struct kvm *kvm = vcpu->kvm;
-
-	guard(mutex)(&kvm->arch.config_lock);
-
-	if (vcpu_has_nv(vcpu)) {
-		int ret = kvm_init_nv_sysregs(vcpu);
-		if (ret)
-			return ret;
-	}
+	lockdep_assert_held(&kvm->arch.config_lock);
 
-	if (kvm_vm_has_ran_once(kvm))
+	if (kvm_id_regs_final(kvm))
 		return 0;
 
 	/*
@@ -5928,12 +5921,37 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu)
 		kvm_vgic_finalize_idregs(kvm);
 	}
 
+	set_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &kvm->arch.flags);
+
+	return 0;
+}
+
+/*
+ * Because this can be called once per CPU, changes must be idempotent.
+ */
+int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu)
+{
+	struct kvm *kvm = vcpu->kvm;
+	int ret;
+
+	guard(mutex)(&kvm->arch.config_lock);
+
+	ret = kvm_vm_finalize_sys_regs(kvm);
+	if (ret)
+		return ret;
+
+	if (vcpu_has_nv(vcpu)) {
+		ret = kvm_init_nv_sysregs(vcpu);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
 
 int __init kvm_sys_reg_table_init(void)
 {
-	const struct sys_reg_desc *gicv3_regs;
+	const struct sys_reg_desc *gicv3_regs, *gicv5_regs;
 	bool valid = true;
 	unsigned int i, sz;
 	int ret = 0;
@@ -5946,8 +5964,12 @@ int __init kvm_sys_reg_table_init(void)
 	valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false);
 	valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false);
 
+	/* The GICv3 system registers... */
 	gicv3_regs = vgic_v3_get_sysreg_table(&sz);
 	valid &= check_sysreg_table(gicv3_regs, sz, false);
+	/* ...and the GICv5 system registers. */
+	gicv5_regs = vgic_v5_get_sysreg_table(&sz);
+	valid &= check_sysreg_table(gicv5_regs, sz, false);
 
 	if (!valid)
 		return -EINVAL;
diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h
index bd56a45abbf9ca..a3cccad2766f27 100644
--- a/arch/arm64/kvm/sys_regs.h
+++ b/arch/arm64/kvm/sys_regs.h
@@ -254,7 +254,7 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
 
 bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index);
 
-int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu);
+int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu);
 
 #define AA32(_x)	.aarch32_map = AA32_##_x
 #define Op0(_x) 	.Op0 = _x
diff --git a/arch/arm64/kvm/vgic-sys-reg-v5.c b/arch/arm64/kvm/vgic-sys-reg-v5.c
new file mode 100644
index 00000000000000..6b255eacbf60d8
--- /dev/null
+++ b/arch/arm64/kvm/vgic-sys-reg-v5.c
@@ -0,0 +1,519 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025, 2026 Arm Ltd.
+ */
+
+/*
+ * VGICv5 system registers handling functions for AArch64 mode
+ */
+
+#include <linux/irqchip/arm-gic-v5.h>
+
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <linux/wordpart.h>
+
+#include <asm/kvm_emulate.h>
+
+#include "vgic/vgic.h"
+#include "sys_regs.h"
+
+#define ICC_PPI_PRIORITYR_PRIORITY_MASK		REPEAT_BYTE(0x1f)
+
+static int set_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+		       u64 val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	/* The upper 32 bits are RES0 */
+	cpu_if->vgic_apr = val & ~ICC_APR_EL1_RES0;
+
+	return 0;
+}
+
+static int get_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+		       u64 *val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	*val = cpu_if->vgic_apr;
+
+	return 0;
+}
+
+static int set_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+		       u64 val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	/*
+	 * We only support setting the ICC_CR0_EL1.En bit, which is actually
+	 * stored in the VMCR.
+	 */
+	FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_EN, &cpu_if->vgic_vmcr,
+		     FIELD_GET(ICC_CR0_EL1_EN, val));
+
+	return 0;
+}
+
+static int get_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+		       u64 *val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	/*
+	 * PID only applies if EL3 is present. Same applies to IPPT. Hence,
+	 * those fields are always presented as 0.
+	 *
+	 * We always present the link as connected and idle:
+	 *     (LINK = 1, LINK_IDLE = 1).
+	 */
+	*val = FIELD_PREP(ICC_CR0_EL1_EN,
+			  FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, cpu_if->vgic_vmcr));
+	*val |= ICC_CR0_EL1_LINK_MASK;
+	*val |= ICC_CR0_EL1_LINK_IDLE_MASK;
+
+	return 0;
+}
+
+static int set_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+		       u64 val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	/* Set the VPMR field in the VMCR */
+	FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_VPMR, &cpu_if->vgic_vmcr,
+		     FIELD_GET(ICC_PCR_EL1_PRIORITY, val));
+
+	return 0;
+}
+
+static int get_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+		       u64 *val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	*val = FIELD_PREP(ICC_PCR_EL1_PRIORITY,
+			  FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, cpu_if->vgic_vmcr));
+
+	return 0;
+}
+
+static int set_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+			u64 val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	cpu_if->vgic_icsr = val & ~ICC_ICSR_EL1_RES0;
+
+	return 0;
+}
+
+static int get_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
+			u64 *val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+
+	*val = cpu_if->vgic_icsr;
+
+	return 0;
+}
+
+/*
+ * Helper macro to iterate over a range of PPIs and execute some code (to either
+ * extract or set the vgic_irq state). This is used when `get`-ing the PPI
+ * ENABLER, ACTIVER, PENDR and when setting the PRIORITYR state.
+ *
+ * vcpu: Pointer to struct kvm_vcpu (to which these PPIs belong)
+ * r: The register index. 0 or 1 for all except PRIORITYR (which is 0-15)
+ * nr: The number of PPIs iterated over. 64 for all but PRIORITYR (which is 8)
+ * code: The code snippet to execute for each vgic_irq
+ */
+#define for_ppi_state(vcpu, r, nr, code)				\
+	do {								\
+		struct kvm_vcpu *__vcpu = (vcpu);			\
+		int __r = (r);						\
+		int __nr = (nr);					\
+									\
+		for (int i = 0; i < __nr; i++) {			\
+			u32 id = vgic_v5_make_ppi(__r * __nr + i);	\
+			struct vgic_irq *irq;				\
+									\
+			irq = vgic_get_vcpu_irq(__vcpu, id);		\
+			scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { \
+				code;					\
+			}						\
+			vgic_put_irq(__vcpu->kvm, irq);			\
+		}							\
+	} while (0)
+
+static int set_gic_ppi_enabler(struct kvm_vcpu *vcpu,
+			       const struct sys_reg_desc *r, u64 val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	int i, start, end, reg = r->Op2 % 2;
+
+	/*
+	 * If we're only handling architected PPIs and the guest writes to the
+	 * enable for the non-architected PPIs, we just return as there's
+	 * nothing to do at all. We don't even allocate the storage for them in
+	 * this case.
+	 */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1)
+		return 0;
+
+	/*
+	 * Merge the raw guest write into our bitmap at an offset of either 0 or
+	 * 64.
+	 *
+	 * Note that there is *NO* masking applied - the enable state is written
+	 * unfiltered. The assumption is that userspace uses this interface to
+	 * set initial state before the guest runs, and then the exposed PPI
+	 * mask is applied later, when vgic_v5_finalize_ppi_state() runs on
+	 * first entry to each vCPU. If userspace chooses to set the enabler
+	 * state later, it is fully capable of breaking the illusion we provided
+	 * to the guest by exposing register state (and PPIs) to the guest that
+	 * were not initially exposed. Good luck!
+	 */
+	bitmap_write(cpu_if->vgic_ppi_enabler, val, 64 * reg, 64);
+
+	/*
+	 * Sync the change in enable states to the vgic_irqs for the written
+	 * register slice.
+	 */
+	start = reg * 64;
+	end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS);
+	for (i = start; i < end; i++) {
+		u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+			irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler);
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	return 0;
+}
+
+static int get_gic_ppi_enabler(struct kvm_vcpu *vcpu,
+			       const struct sys_reg_desc *r, u64 *val)
+{
+	unsigned long enabler = 0;
+	int reg = r->Op2 % 2;
+
+	/* If we only support architected PPIs, return 0 */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) {
+		*val = 0;
+		return 0;
+	}
+
+	/* Iterate over each struct vgic_irq to build the ENABLER value. */
+	for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &enabler, irq->enabled));
+
+	*val = enabler;
+
+	return 0;
+}
+
+static int set_gic_ppi_activer(struct kvm_vcpu *vcpu,
+			       const struct sys_reg_desc *r, u64 val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	int i, start, end, reg = r->Op2 % 2;
+
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1)
+		return 0;
+
+	/*
+	 * Store the raw guest write. The exposed PPI mask is applied later,
+	 * when vgic_v5_finalize_ppi_state() runs on first entry to each
+	 * vCPU. See comment on set_gic_ppi_enabler() for details.
+	 */
+	bitmap_write(cpu_if->vgic_ppi_activer, val, 64 * reg, 64);
+
+	start = reg * 64;
+	end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS);
+	for (i = start; i < end; i++) {
+		u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+			irq->active = test_bit(i, cpu_if->vgic_ppi_activer);
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	return 0;
+}
+
+static int get_gic_ppi_activer(struct kvm_vcpu *vcpu,
+			       const struct sys_reg_desc *r, u64 *val)
+{
+	unsigned long activer = 0;
+	int reg = r->Op2 % 2;
+
+	/* If we only support architected PPIs, return 0 */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) {
+		*val = 0;
+		return 0;
+	}
+
+	/* Iterate over each struct vgic_irq to build the ACTIVER value. */
+	for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &activer, irq->active));
+
+	*val = activer;
+
+	return 0;
+}
+
+static int set_gic_ppi_pendr(struct kvm_vcpu *vcpu,
+			     const struct sys_reg_desc *r, u64 val)
+{
+	int i, start, end, reg = r->Op2 % 2;
+
+	/* If we only support architected PPIs, return */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1)
+		return 0;
+
+	/*
+	 * Update each struct vgic_irq with the pending state, treating Level
+	 * and Edge interrupts differently. The exposed PPI mask is applied
+	 * later, when vgic_v5_finalize_ppi_state() runs on first entry to each
+	 * vCPU. See comment on set_gic_ppi_enabler() for details.
+	 */
+	start = reg * 64;
+	end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS);
+	for (i = start; i < end; i++) {
+		u32 intid = vgic_v5_make_ppi(i);
+		struct vgic_irq *irq;
+
+		irq = vgic_get_vcpu_irq(vcpu, intid);
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+			bool level = !!(val & BIT_ULL(i - start));
+
+			if (irq->config == VGIC_CONFIG_LEVEL)
+				irq->line_level = level;
+			else
+				irq->pending_latch = level;
+		}
+
+		vgic_put_irq(vcpu->kvm, irq);
+	}
+
+	/*
+	 * The pending state is generated from the vgic_irqs on each guest
+	 * entry. Therefore, we don't store the raw value written anywhere in
+	 * the case of userspace PPI_PENDRx_EL1 writes.
+	 */
+
+	return 0;
+}
+
+static int get_gic_ppi_pendr(struct kvm_vcpu *vcpu,
+			     const struct sys_reg_desc *r, u64 *val)
+{
+	unsigned long pendr = 0;
+	int reg = r->Op2 % 2;
+
+	/* If we only support architected PPIs, return 0 */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) {
+		*val = 0;
+		return 0;
+	}
+
+	/* Iterate over each struct vgic_irq to build the PENDR value. */
+	for_ppi_state(vcpu, reg, 64, {
+		if (irq_is_pending(irq))
+			__assign_bit(i % 64, &pendr, 1);
+	});
+
+	*val = pendr;
+
+	return 0;
+}
+
+static int set_gic_ppi_priorityr(struct kvm_vcpu *vcpu,
+				 const struct sys_reg_desc *r, u64 val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	int reg = ((r->CRm & 0x1) << 3) + r->Op2;
+
+	/* If we only support architected PPIs, return */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7)
+		return 0;
+
+	val &= ICC_PPI_PRIORITYR_PRIORITY_MASK;
+
+	/*
+	 * Although priorities are not regularly synced back to the vgic_irq
+	 * state, they are explicitly synced back here. This is to ensure that
+	 * any pending PPIs are evaluated correctly when first running the guest
+	 * after setting the state.
+	 */
+	for_ppi_state(vcpu, reg, 8,
+		      irq->priority = (u8)(val >> (8 * i));
+		);
+
+	/*
+	 * Update the state that will be written to the ICH_PPI_PRIORITYRx_EL2
+	 * on next guest entry.
+	 */
+	cpu_if->vgic_ppi_priorityr[reg] = val;
+
+	return 0;
+}
+
+static int get_gic_ppi_priorityr(struct kvm_vcpu *vcpu,
+				 const struct sys_reg_desc *r, u64 *val)
+{
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	int reg = ((r->CRm & 0x1) << 3) + r->Op2;
+
+	/* If we only support architected PPIs, return 0 */
+	if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) {
+		*val = 0;
+		return 0;
+	}
+
+	/*
+	 * The priorities are only synced back to the vgic_irq state when the
+	 * vcpu is entering WFI (KVM only needs to know the priorities when
+	 * evaluating if there are pending PPI interrupts for a vcpu). The raw
+	 * register ICH_PPI_PRIORITYRx_EL1 state is simply saved and restored
+	 * blindly. This state is just returned as it contains the most recent
+	 * priorities written by the guest.
+	 */
+	*val = cpu_if->vgic_ppi_priorityr[reg];
+
+	return 0;
+}
+
+/*
+ * The following registers are NOT supported:
+ *
+ * - ICC_HAPR_EL1
+ *	The value of this is directly generated by the GICv5 hardware based on
+ *	the ICC_APR_EL1 when the guest is running.
+ * - ICC_IAFFIDR_EL1
+ *	The IAFFID for a GICv5 VPE is the same as the VPE ID, which is the index
+ *	into the in-memory VPE Table. This is not configurable, and instead we
+ *	rely on userspace recreating the VPEs in the same order prior to
+ *	restoring guest state.
+ * - ICC_PPI_CACTIVER<n>_EL1
+ *	Only raw state writes are supported via the S(et) variant.
+ * - ICC_PPI_CPENDR<n>_EL1
+ *	Only raw state writes are supported via the S(et) variant.
+ */
+static const struct sys_reg_desc gic_v5_icc_reg_descs[] = {
+	{ SYS_DESC(SYS_ICC_ICSR_EL1),
+	  .set_user = set_gic_icsr, .get_user = get_gic_icsr, },
+	{ SYS_DESC(SYS_ICC_PPI_ENABLER0_EL1),
+	  .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, },
+	{ SYS_DESC(SYS_ICC_PPI_ENABLER1_EL1),
+	  .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, },
+	/*
+	 * Only ICC_SACTIVER<n>_EL1 is exposed to the guest. This is treated as
+	 * a *RAW* write of register state for writes.
+	 */
+	{ SYS_DESC(SYS_ICC_PPI_SACTIVER0_EL1),
+	  .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, },
+	{ SYS_DESC(SYS_ICC_PPI_SACTIVER1_EL1),
+	  .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, },
+	/*
+	 * Only ICC_SPENDR<n>_EL1 is exposed to the guest. This is treated as
+	 * a *RAW* write of register state for writes.
+	 */
+	{ SYS_DESC(SYS_ICC_PPI_SPENDR0_EL1),
+	  .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, },
+	{ SYS_DESC(SYS_ICC_PPI_SPENDR1_EL1),
+	  .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR0_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR1_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR2_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR3_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR4_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR5_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR6_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR7_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR8_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR9_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR10_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR11_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR12_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR13_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR14_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_PPI_PRIORITYR15_EL1),
+	  .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
+	{ SYS_DESC(SYS_ICC_APR_EL1),
+	  .set_user = set_gic_apr, .get_user = get_gic_apr, },
+	{ SYS_DESC(SYS_ICC_CR0_EL1),
+	  .set_user = set_gic_cr0, .get_user = get_gic_cr0, },
+	{ SYS_DESC(SYS_ICC_PCR_EL1),
+	  .set_user = set_gic_pcr, .get_user = get_gic_pcr, },
+};
+
+const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz)
+{
+	*sz = ARRAY_SIZE(gic_v5_icc_reg_descs);
+	return gic_v5_icc_reg_descs;
+}
+
+static u64 attr_to_id(u64 attr)
+{
+	return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr),
+			     FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP1_MASK, attr),
+			     FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRN_MASK, attr),
+			     FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRM_MASK, attr),
+			     FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP2_MASK, attr));
+}
+
+int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
+{
+	const struct sys_reg_desc *r;
+
+	r = get_reg_by_id(attr_to_id(attr->attr), gic_v5_icc_reg_descs,
+			  ARRAY_SIZE(gic_v5_icc_reg_descs));
+
+	if (r && !sysreg_hidden(vcpu, r))
+		return 0;
+
+	return -ENXIO;
+}
+
+int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr,
+				bool is_write)
+{
+	struct kvm_one_reg reg = {
+		.id	= attr_to_id(attr->attr),
+		.addr	= attr->addr,
+	};
+
+	if (is_write)
+		return kvm_sys_reg_set_user(vcpu, &reg, gic_v5_icc_reg_descs,
+					    ARRAY_SIZE(gic_v5_icc_reg_descs));
+	else
+		return kvm_sys_reg_get_user(vcpu, &reg, gic_v5_icc_reg_descs,
+					    ARRAY_SIZE(gic_v5_icc_reg_descs));
+}
diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c
index 4012df6002ea66..17f16e95ac6a15 100644
--- a/arch/arm64/kvm/vgic/vgic-init.c
+++ b/arch/arm64/kvm/vgic/vgic-init.c
@@ -97,6 +97,9 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 	/*
 	 *  - Acquiring the vCPU mutex for every *online* vCPU to prevent
 	 *    concurrent vCPU ioctls for vCPUs already visible to userspace.
+	 *    This also ensures KVM isn't in the middle of creating a vCPU,
+	 *    i.e. that there are no vCPUs that have been created but aren't
+	 *    yet fully online.
 	 */
 	ret = -EBUSY;
 	if (kvm_trylock_all_vcpus(kvm))
@@ -105,37 +108,45 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 	/*
 	 *  - Taking the config_lock which protects VGIC data structures such
 	 *    as the per-vCPU arrays of private IRQs (SGIs, PPIs).
-	 */
-	mutex_lock(&kvm->arch.config_lock);
-
-	/*
-	 * - Bailing on the entire thing if a vCPU is in the middle of creation,
-	 *   dropped the kvm->lock, but hasn't reached kvm_arch_vcpu_create().
 	 *
 	 * The whole combination of this guarantees that no vCPU can get into
 	 * KVM with a VGIC configuration inconsistent with the VM's VGIC.
 	 */
-	if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
-		goto out_unlock;
+	mutex_lock(&kvm->arch.config_lock);
 
 	if (irqchip_in_kernel(kvm)) {
 		ret = -EEXIST;
 		goto out_unlock;
 	}
 
-	kvm_for_each_vcpu(i, vcpu, kvm) {
-		if (vcpu_has_run_once(vcpu))
-			goto out_unlock;
-	}
+	if (kvm_id_regs_final(kvm))
+		goto out_unlock;
 	ret = 0;
 
-	if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
+	switch (type) {
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
 		kvm->max_vcpus = VGIC_V2_MAX_CPUS;
-	else if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		kvm->max_vcpus = VGIC_V3_MAX_CPUS;
-	else if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
-		kvm->max_vcpus = min(VGIC_V5_MAX_CPUS,
-				     kvm_vgic_global_state.max_gic_vcpus);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		kvm->max_vcpus = kvm_vgic_global_state.max_gicv5_vcpus;
+		break;
+	}
+
+	/*
+	 * KVM_CREATE_VCPU enforces the model-specific limit if the VGIC has
+	 * already been created. Apply the same limit to any existing vCPUs so
+	 * that the result does not depend on the order in which userspace
+	 * creates them.
+	 */
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		if (vcpu->vcpu_id >= kvm->max_vcpus) {
+			ret = -E2BIG;
+			goto out_unlock;
+		}
+	}
 
 	if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) {
 		ret = -E2BIG;
@@ -154,6 +165,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
 		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL;
 	}
 
 	/*
@@ -168,29 +181,49 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
 			break;
 	}
 
-	if (ret) {
-		kvm_for_each_vcpu(i, vcpu, kvm) {
-			struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
-			kfree(vgic_cpu->private_irqs);
-			vgic_cpu->private_irqs = NULL;
-		}
-
-		kvm->arch.vgic.vgic_model = 0;
-		kvm->arch.vgic.in_kernel = false;
-		goto out_unlock;
-	}
+	if (ret)
+		goto out_free_private_irqs;
 
 	if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
 		kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
 
-	/*
-	 * We now know that we have a GICv5. The Arch Timer PPI interrupts may
-	 * have been initialised at this stage, but will have done so assuming
-	 * that we have an older GIC, meaning that the IntIDs won't be
-	 * correct. We init them again, and this time they will be correct.
-	 */
-	if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
+	if (type == KVM_DEV_TYPE_ARM_VGIC_V5) {
+		/* Allocate a vIRS for GICv5 systems */
+		kvm->arch.vgic.vgic_v5_irs_data = kzalloc_obj(struct vgic_v5_irs,
+							      GFP_KERNEL_ACCOUNT);
+		if (!kvm->arch.vgic.vgic_v5_irs_data) {
+			ret = -ENOMEM;
+			goto out_free_private_irqs;
+		}
+
+		/*
+		 * Initialization happens later, for now just explicitly
+		 * disable the device and undef its base address.
+		 */
+		kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base = VGIC_ADDR_UNDEF;
+
+		/*
+		 * We now know that we have a GICv5. The Arch Timer PPI
+		 * interrupts may have been initialised at this stage, but will
+		 * have done so assuming that we have an older GIC, meaning that
+		 * the IntIDs won't be correct. We init them again, and this
+		 * time they will be correct.
+		 */
 		kvm_timer_init_vm(kvm);
+	}
+
+	goto out_unlock;
+
+out_free_private_irqs:
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+
+		kfree(vgic_cpu->private_irqs);
+		vgic_cpu->private_irqs = NULL;
+	}
+
+	kvm->arch.vgic.in_kernel = false;
+	kvm->arch.vgic.vgic_model = 0;
 
 out_unlock:
 	mutex_unlock(&kvm->arch.config_lock);
@@ -397,15 +430,27 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
 	if (ret)
 		return ret;
 
-	/*
-	 * If we are creating a VCPU with a GICv3 we must also register the
-	 * KVM io device for the redistributor that belongs to this VCPU.
-	 */
-	if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
+	switch (dist->vgic_model) {
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
+		/*
+		 * If we are creating a VCPU with a GICv3 we must also register
+		 * the KVM io device for the redistributor that belongs to this
+		 * VCPU.
+		 */
 		mutex_lock(&vcpu->kvm->slots_lock);
 		ret = vgic_register_redist_iodev(vcpu);
 		mutex_unlock(&vcpu->kvm->slots_lock);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		/*
+		 * Ensure that it is possible to represent the
+		 * userspace-allocated vcpu_id in the hardware-limited (or
+		 * KVM-capped) VPE table used by GICv5.
+		 */
+		if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus)
+			return -EINVAL;
 	}
+
 	return ret;
 }
 
@@ -465,6 +510,9 @@ int vgic_init(struct kvm *kvm)
 				return ret;
 		}
 	} else {
+		if (!dist->nr_spis)
+			dist->nr_spis = VGIC_V5_DEFAULT_NR_SPIS;
+
 		ret = vgic_v5_init(kvm);
 		if (ret)
 			return ret;
@@ -474,8 +522,12 @@ int vgic_init(struct kvm *kvm)
 		kvm_vgic_vcpu_reset(vcpu);
 
 	ret = kvm_vgic_setup_default_irq_routing(kvm);
-	if (ret)
+	if (ret) {
+		if (vgic_is_v5(kvm))
+			vgic_v5_teardown(kvm);
+
 		return ret;
+	}
 
 	vgic_debug_init(kvm);
 	dist->initialized = true;
@@ -496,16 +548,24 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
 	dist->nr_spis = 0;
 	dist->vgic_dist_base = VGIC_ADDR_UNDEF;
 
-	if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
+	switch (dist->vgic_model) {
+	case KVM_DEV_TYPE_ARM_VGIC_V2:
+		dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V3:
 		list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list)
 			vgic_v3_free_redist_region(kvm, rdreg);
 		INIT_LIST_HEAD(&dist->rd_regions);
-	} else {
-		dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
-	}
 
-	if (vgic_supports_direct_irqs(kvm))
-		vgic_v4_teardown(kvm);
+		if (vgic_supports_direct_irqs(kvm))
+			vgic_v4_teardown(kvm);
+		break;
+	case KVM_DEV_TYPE_ARM_VGIC_V5:
+		vgic_v5_teardown(kvm);
+		kfree(dist->vgic_v5_irs_data);
+		dist->vgic_v5_irs_data = NULL;
+		break;
+	}
 
 	xa_destroy(&dist->lpi_xa);
 }
@@ -624,9 +684,8 @@ int vgic_lazy_init(struct kvm *kvm)
 int kvm_vgic_map_resources(struct kvm *kvm)
 {
 	struct vgic_dist *dist = &kvm->arch.vgic;
-	bool needs_dist = true;
 	enum vgic_type type;
-	gpa_t dist_base;
+	gpa_t dist_base, irs_base;
 	int ret = 0;
 
 	if (likely(smp_load_acquire(&dist->ready)))
@@ -649,13 +708,12 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	} else {
 		ret = vgic_v5_map_resources(kvm);
 		type = VGIC_V5;
-		needs_dist = false;
 	}
 
 	if (ret)
 		goto out;
 
-	if (needs_dist) {
+	if (type != VGIC_V5) {
 		dist_base = dist->vgic_dist_base;
 		mutex_unlock(&kvm->arch.config_lock);
 
@@ -665,7 +723,19 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 			goto out_slots;
 		}
 	} else {
+		irs_base = dist->vgic_v5_irs_data->vgic_v5_irs_base;
 		mutex_unlock(&kvm->arch.config_lock);
+
+		if (IS_VGIC_ADDR_UNDEF(irs_base)) {
+			ret = -ENXIO;
+			goto out_slots;
+		}
+
+		ret = vgic_v5_register_irs_iodev(kvm, irs_base);
+		if (ret) {
+			kvm_err("Unable to register VGIC IRS MMIO regions\n");
+			goto out_slots;
+		}
 	}
 
 	smp_store_release(&dist->ready, true);
diff --git a/arch/arm64/kvm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c
index 19a1094536e6a8..ee1a1b214f03cf 100644
--- a/arch/arm64/kvm/vgic/vgic-irqfd.c
+++ b/arch/arm64/kvm/vgic/vgic-irqfd.c
@@ -19,9 +19,14 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e,
 			struct kvm *kvm, int irq_source_id,
 			int level, bool line_status)
 {
-	unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS;
+	unsigned int spi_id;
 	int ret;
 
+	if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5)
+		spi_id = vgic_v5_make_spi(e->irqchip.pin);
+	else
+		spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS;
+
 	if (!vgic_valid_spi(kvm, spi_id))
 		return -EINVAL;
 
@@ -45,6 +50,13 @@ int kvm_set_routing_entry(struct kvm *kvm,
 			  struct kvm_kernel_irq_routing_entry *e,
 			  const struct kvm_irq_routing_entry *ue)
 {
+	/*
+	 * The common routing table reserves enough entries for all GICv5 SPI
+	 * pins. KVM limits GICv5 to 1024 SPI pins, while GICv2 and GICv3
+	 * only support SPIs 32..1019, leaving 988 pins.
+	 */
+	unsigned int max_irqchip_pins = vgic_is_v5(kvm) ?
+		VGIC_V5_MAX_NR_SPIS : VGIC_MAX_SPI - VGIC_NR_PRIVATE_IRQS + 1;
 	int r = -EINVAL;
 
 	switch (ue->type) {
@@ -52,8 +64,8 @@ int kvm_set_routing_entry(struct kvm *kvm,
 		e->set = vgic_irqfd_set_irq;
 		e->irqchip.irqchip = ue->u.irqchip.irqchip;
 		e->irqchip.pin = ue->u.irqchip.pin;
-		if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) ||
-		    (e->irqchip.irqchip >= KVM_NR_IRQCHIPS))
+		if (e->irqchip.pin >= max_irqchip_pins ||
+		    e->irqchip.irqchip >= KVM_NR_IRQCHIPS)
 			goto out;
 		break;
 	case KVM_IRQ_ROUTING_MSI:
diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c
new file mode 100644
index 00000000000000..1874c3c289a6ee
--- /dev/null
+++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c
@@ -0,0 +1,1217 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 ARM Limited, All Rights Reserved.
+ */
+#include <linux/bitops.h>
+#include <linux/bsearch.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/kvm.h>
+#include <linux/kvm_host.h>
+#include <kvm/iodev.h>
+#include <kvm/arm_arch_timer.h>
+#include <kvm/arm_vgic.h>
+
+#include "vgic.h"
+#include "vgic-mmio.h"
+#include "vgic-v5-tables.h"
+
+#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps
+
+static struct vgic_dist *vgic_v5_get_vgic(struct kvm_vcpu *vcpu)
+{
+	return &vcpu->kvm->arch.vgic;
+}
+
+static struct vgic_v5_irs *vgic_v5_get_irs(struct kvm_vcpu *vcpu)
+{
+	return vcpu->kvm->arch.vgic.vgic_v5_irs_data;
+}
+
+static unsigned long vgic_v5_mmio_read_irs_misc(struct kvm_vcpu *vcpu,
+						gpa_t addr, unsigned int len)
+{
+	struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu);
+	const size_t offset = addr & (SZ_64K - 1);
+	struct kvm_vcpu *target_vcpu;
+	u8 vpe_id_bits;
+	u64 value = 0;
+
+	switch (offset) {
+	case GICV5_IRS_IDR0:
+		value = FIELD_PREP(GICV5_IRS_IDR0_INT_DOM, irs->idr0.domain);
+		value |= FIELD_PREP(GICV5_IRS_IDR0_PA_RANGE, irs->idr0.pa_range);
+		if (irs->idr0.virt)
+			value |= GICV5_IRS_IDR0_VIRT;
+		if (irs->idr0.setlpi)
+			value |= GICV5_IRS_IDR0_SETLPI;
+		if (irs->idr0.mec)
+			value |= GICV5_IRS_IDR0_MEC;
+		if (irs->idr0.mpam)
+			value |= GICV5_IRS_IDR0_MPAM;
+		if (irs->idr0.swe)
+			value |= GICV5_IRS_IDR0_SWE;
+		value |= FIELD_PREP(GICV5_IRS_IDR0_IRSID, irs->idr0.irs_id);
+		break;
+	case GICV5_IRS_IDR1:
+		value = FIELD_PREP(GICV5_IRS_IDR1_PE_CNT,
+				   atomic_read(&vcpu->kvm->online_vcpus));
+		/*
+		 * IRS_IDR1 encodes IAFFID_BITS as N - 1.
+		 */
+		vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu);
+		value |= FIELD_PREP(GICV5_IRS_IDR1_IAFFID_BITS, vpe_id_bits - 1);
+		value |= FIELD_PREP(GICV5_IRS_IDR1_PRIORITY_BITS, irs->idr1.priority_bits);
+		break;
+	case GICV5_IRS_IDR2:
+		value = FIELD_PREP(GICV5_IRS_IDR2_ISTMD_SZ, irs->idr2.istmd_sz);
+		if (irs->idr2.istmd)
+			value |= GICV5_IRS_IDR2_ISTMD;
+		value |= FIELD_PREP(GICV5_IRS_IDR2_IST_L2SZ, irs->idr2.ist_l2sz);
+		if (irs->idr2.ist_levels)
+			value |= GICV5_IRS_IDR2_IST_LEVELS;
+		value |= FIELD_PREP(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, irs->idr2.min_lpi_id_bits);
+		value |= GICV5_IRS_IDR2_LPI;
+		value |= FIELD_PREP(GICV5_IRS_IDR2_ID_BITS, irs->idr2.id_bits);
+		break;
+	case GICV5_IRS_IDR5:
+		value = FIELD_PREP(GICV5_IRS_IDR5_SPI_RANGE, irs->idr5.spi_range);
+		break;
+	case GICV5_IRS_IDR6:
+		value = FIELD_PREP(GICV5_IRS_IDR6_SPI_IRS_RANGE, irs->idr6.spi_irs_range);
+		break;
+	case GICV5_IRS_IDR7:
+		value = FIELD_PREP(GICV5_IRS_IDR7_SPI_BASE, irs->idr7.spi_base);
+		break;
+	case GICV5_IRS_IIDR:
+		/* Revision, Variant, ProductID are implementation defined */
+		value = FIELD_PREP(GICV5_IRS_IIDR_PRODUCT_ID, PRODUCT_ID_KVM);
+		value |= FIELD_PREP(GICV5_IRS_IIDR_VARIANT, 0);
+		value |= FIELD_PREP(GICV5_IRS_IIDR_REVISION, 0);
+		value |= FIELD_PREP(GICV5_IRS_IIDR_IMPLEMENTER, IMPLEMENTER_ARM);
+		break;
+	case GICV5_IRS_AIDR:
+		value = FIELD_PREP(GICV5_IRS_AIDR_COMPONENT,
+				   GICV5_AIDR_COMPONENT_IRS);
+		value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMAJORREV,
+				    GICV5_AIDR_ARCH_MAJ_REV_V5);
+		value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMINORREV,
+				    GICV5_AIDR_ARCH_MIN_REV_V0);
+		break;
+	case GICV5_IRS_CR0:
+		/*
+		 * The IRS is ALWAYS idle as we handle things instantaneously
+		 * from a guest's viewpoint.
+		 */
+		value = GICV5_IRS_CR0_IDLE;
+		if (READ_ONCE(vcpu->kvm->arch.vgic.enabled))
+			value |= GICV5_IRS_CR0_IRSEN;
+		break;
+	case GICV5_IRS_CR1:
+		if (irs->cr1.vped_wa)
+			value |= GICV5_IRS_CR1_VPED_WA;
+		if (irs->cr1.vped_ra)
+			value |= GICV5_IRS_CR1_VPED_RA;
+		if (irs->cr1.vmd_wa)
+			value |= GICV5_IRS_CR1_VMD_WA;
+		if (irs->cr1.vmd_ra)
+			value |= GICV5_IRS_CR1_VMD_RA;
+		if (irs->cr1.vpet_ra)
+			value |= GICV5_IRS_CR1_VPET_RA;
+		if (irs->cr1.vmt_ra)
+			value |= GICV5_IRS_CR1_VMT_RA;
+		if (irs->cr1.ist_wa)
+			value |= GICV5_IRS_CR1_IST_WA;
+		if (irs->cr1.ist_ra)
+			value |= GICV5_IRS_CR1_IST_RA;
+		value |= FIELD_PREP(GICV5_IRS_CR1_IC, irs->cr1.ic);
+		value |= FIELD_PREP(GICV5_IRS_CR1_OC, irs->cr1.oc);
+		value |= FIELD_PREP(GICV5_IRS_CR1_SH, irs->cr1.sh);
+		break;
+	case GICV5_IRS_SYNC_STATUSR:
+		value = GICV5_IRS_SYNC_STATUSR_IDLE;
+		break;
+	case GICV5_IRS_PE_SELR:
+		value = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID,
+				   READ_ONCE(irs->pe_selr.iaffid));
+		break;
+	case GICV5_IRS_PE_STATUSR:
+		/* We assume that the PE is Online if present. Always IDLE too */
+		value = GICV5_IRS_PE_STATUSR_IDLE;
+
+		/* Set ONLINE and V if IAFFID selects a present PE */
+		if (kvm_get_vcpu_by_id(vcpu->kvm,
+				       READ_ONCE(irs->pe_selr.iaffid))) {
+			value |= GICV5_IRS_PE_STATUSR_ONLINE;
+			value |= GICV5_IRS_PE_STATUSR_V;
+		}
+		break;
+	case GICV5_IRS_PE_CR0: {
+		u32 iaffid = READ_ONCE(irs->pe_selr.iaffid);
+
+		/*
+		 * Make sure that we are doing something reasonable first.
+		 * Remember, the IAFFID is the same as the VPE_ID
+		 */
+		target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, iaffid);
+		if (!target_vcpu) {
+			kvm_debug("Guest programmed invalid IAFFID (0x%x) into the IRS_PE_SELR\n",
+				  iaffid);
+			break;
+		}
+
+		value = GICV5_IRS_PE_CR0_DPS;
+		break;
+	}
+	default:
+		return 0;
+	}
+
+	return value;
+}
+
+static void vgic_v5_mmio_write_irs_misc(struct kvm_vcpu *vcpu, gpa_t addr,
+					unsigned int len, unsigned long val)
+{
+	struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu);
+	struct vgic_dist *vgic = vgic_v5_get_vgic(vcpu);
+	const size_t offset = addr & (SZ_64K - 1);
+
+	switch (offset) {
+	case GICV5_IRS_CR0:
+		scoped_guard(mutex, &vcpu->kvm->arch.config_lock) {
+			bool was_enabled = vgic->enabled;
+
+			WRITE_ONCE(vgic->enabled, !!(val & GICV5_IRS_CR0_IRSEN));
+
+			/* Reload other vCPUs on an edge */
+			if (was_enabled != !!(val & GICV5_IRS_CR0_IRSEN))
+				kvm_make_all_cpus_request(vcpu->kvm,
+							  KVM_REQ_RELOAD_GICv5);
+		}
+		return;
+	case GICV5_IRS_CR1:
+		irs->cr1.sh = FIELD_GET(GICV5_IRS_CR1_SH, val);
+		irs->cr1.oc = FIELD_GET(GICV5_IRS_CR1_OC, val);
+		irs->cr1.ic = FIELD_GET(GICV5_IRS_CR1_IC, val);
+		irs->cr1.ist_ra = !!(val & GICV5_IRS_CR1_IST_RA);
+		irs->cr1.ist_wa = !!(val & GICV5_IRS_CR1_IST_WA);
+		irs->cr1.vmt_ra = !!(val & GICV5_IRS_CR1_VMT_RA);
+		irs->cr1.vpet_ra = !!(val & GICV5_IRS_CR1_VPET_RA);
+		irs->cr1.vmd_ra = !!(val & GICV5_IRS_CR1_VMD_RA);
+		irs->cr1.vmd_wa = !!(val & GICV5_IRS_CR1_VMD_WA);
+		irs->cr1.vped_ra = !!(val & GICV5_IRS_CR1_VPED_RA);
+		irs->cr1.vped_wa = !!(val & GICV5_IRS_CR1_VPED_WA);
+		return;
+	case GICV5_IRS_PE_SELR:
+		WRITE_ONCE(irs->pe_selr.iaffid,
+			   FIELD_GET(GICV5_IRS_PE_SELR_IAFFID, val));
+		return;
+	case GICV5_IRS_PE_CR0:
+		/*
+		 * We actually have nothing to do here as we don't support
+		 * 1-of-N routing. The only thing that the guest can correctly
+		 * write here is 0x1. However, there's no way to fault if it
+		 * writes something else. This is effectively a WI in our case,
+		 * but we keep it here for the purposes of documenting it.
+		 */
+		return;
+	default:
+		return;
+	}
+}
+
+static bool vgic_v5_is_spi_selr_valid(struct vgic_v5_irs *irs, u32 id)
+{
+	/* Invalid - we don't have any SPIs at all */
+	if (irs->idr5.spi_range == 0)
+		return false;
+
+	/* Invalid - we don't have any on this IRS */
+	if (irs->idr6.spi_irs_range == 0)
+		return false;
+
+	/* Invalid - ID is less than min */
+	if (id < irs->idr7.spi_base)
+		return false;
+
+	/* Invalid - ID is greater than max */
+	if (id >= (irs->idr7.spi_base + irs->idr6.spi_irs_range))
+		return false;
+
+	return true;
+}
+
+static unsigned long vgic_v5_mmio_read_irs_spi(struct kvm_vcpu *vcpu,
+					       gpa_t addr, unsigned int len)
+{
+	struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu);
+	const size_t offset = addr & (SZ_64K - 1);
+	struct vgic_irq *irq;
+	u64 value = 0;
+
+	switch (offset) {
+	case GICV5_IRS_SPI_SELR:
+		/* Return whatever was last written */
+		value = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, READ_ONCE(irs->spi_selr.id));
+		break;
+	case GICV5_IRS_SPI_STATUSR:
+		/* We assume that we can always claim to be idle */
+		value = GICV5_IRS_SPI_STATUSR_IDLE;
+		if (vgic_v5_is_spi_selr_valid(irs, READ_ONCE(irs->spi_selr.id)))
+			value |= GICV5_IRS_SPI_STATUSR_V;
+		break;
+	case GICV5_IRS_SPI_DOMAINR:
+		value = FIELD_PREP(GICV5_IRS_SPI_DOMAINR_DOMAIN,
+				   GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE);
+		break;
+	case GICV5_IRS_SPI_CFGR: {
+		u32 id = READ_ONCE(irs->spi_selr.id);
+
+		if (!vgic_v5_is_spi_selr_valid(irs, id)) {
+			/* Fault with IRS_SPI_SELR; return 0*/
+			value = 0;
+			break;
+		}
+
+		irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id));
+		if (!irq) {
+			kvm_debug_ratelimited("Guest trying to access SPI not backed by KVM\n");
+			value = 0;
+			break;
+		}
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+			if (irq->config == VGIC_CONFIG_LEVEL)
+				value = GICV5_IRS_SPI_CFGR_TM;
+		}
+
+		vgic_put_irq(vcpu->kvm, irq);
+
+		break;
+	}
+	default:
+		return 0;
+	}
+
+	return value;
+}
+
+static void vgic_v5_mmio_write_irs_spi(struct kvm_vcpu *vcpu, gpa_t addr,
+				       unsigned int len, unsigned long val)
+{
+	struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu);
+	const size_t offset = addr & (SZ_64K - 1);
+	struct vgic_irq *irq;
+
+	switch (offset) {
+	case GICV5_IRS_SPI_SELR:
+		WRITE_ONCE(irs->spi_selr.id,
+			   FIELD_GET(GICV5_IRS_SPI_SELR_ID, val));
+		return;
+	case GICV5_IRS_SPI_CFGR: {
+		u32 id = READ_ONCE(irs->spi_selr.id);
+
+		if (!vgic_v5_is_spi_selr_valid(irs, id))
+			return;
+
+		/*
+		 * Find KVM's representation of the interrupt - we need to make
+		 * sure that KVM's view agrees with the guest's, else interrupt
+		 * injection won't work properly for level-triggered interrupts
+		 * (we fail to handle the clearing of the pending state if KVM
+		 * thinks that the interrupt is edge-triggered, which is the
+		 * default.)
+		 */
+		irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id));
+		if (!irq)
+			return;
+
+		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+			if (val & GICV5_IRS_SPI_CFGR_TM)
+				irq->config = VGIC_CONFIG_LEVEL;
+			else
+				irq->config = VGIC_CONFIG_EDGE;
+		}
+
+		vgic_put_irq(vcpu->kvm, irq);
+
+		return;
+	}
+	default:
+		return;
+	}
+}
+
+static bool vgic_v5_ist_cfgr_valid(struct vgic_v5_irs *irs)
+{
+	unsigned int expected_istsz;
+
+	if (irs->ist_cfgr.lpi_id_bits < irs->idr2.min_lpi_id_bits ||
+	    irs->ist_cfgr.lpi_id_bits > irs->idr2.id_bits)
+		return false;
+
+	if (!irs->idr2.istmd)
+		expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_4;
+	else if (irs->ist_cfgr.lpi_id_bits >= irs->idr2.istmd_sz)
+		expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_16;
+	else
+		expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_8;
+
+	if (irs->ist_cfgr.istsz != expected_istsz)
+		return false;
+
+	if (irs->ist_cfgr.structure && !irs->idr2.ist_levels)
+		return false;
+
+	if (!irs->ist_cfgr.structure)
+		return true;
+
+	return irs->ist_cfgr.l2sz == irs->idr2.ist_l2sz;
+}
+
+static u64 vgic_v5_read_irs_ist_baser(struct vgic_v5_irs *irs)
+{
+	u64 value;
+
+	value = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK,
+			   irs->ist_baser.addr >> GICV5_IRS_IST_BASER_ADDR_SHIFT);
+	if (irs->ist_baser.valid)
+		value |= GICV5_IRS_IST_BASER_VALID;
+
+	return value;
+}
+
+static bool vgic_v5_ist_cfgr_matches(const struct vgic_v5_irs *irs,
+				     unsigned long val)
+{
+	u8 lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val);
+	u8 l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val);
+	u8 istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val);
+	bool structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE);
+
+	return irs->ist_cfgr.lpi_id_bits == lpi_id_bits &&
+	       irs->ist_cfgr.l2sz == l2sz &&
+	       irs->ist_cfgr.istsz == istsz &&
+	       irs->ist_cfgr.structure == structure;
+}
+
+static unsigned long vgic_v5_mmio_read_irs_ist(struct kvm_vcpu *vcpu,
+					       gpa_t addr, unsigned int len)
+{
+	struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu);
+	const size_t offset = addr & (SZ_64K - 1);
+	u64 value = 0;
+
+	switch (offset) {
+	case GICV5_IRS_IST_STATUSR:
+		return GICV5_IRS_IST_STATUSR_IDLE;
+	case GICV5_IRS_IST_CFGR:
+		if (irs->ist_cfgr.structure)
+			value |= GICV5_IRS_IST_CFGR_STRUCTURE;
+		value |= FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, irs->ist_cfgr.istsz);
+		value |= FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, irs->ist_cfgr.l2sz);
+		value |= FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, irs->ist_cfgr.lpi_id_bits);
+		break;
+	case GICV5_IRS_IST_BASER:
+	case GICV5_IRS_IST_BASER + 4:
+		value = vgic_v5_read_irs_ist_baser(irs);
+		return extract_bytes(value, offset & 7, len);
+	default:
+		return 0;
+	}
+
+	return value;
+}
+
+static bool vgic_v5_ist_baser_matches(const struct vgic_v5_irs *irs,
+				      unsigned long val)
+{
+	u64 addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val)
+		   << GICV5_IRS_IST_BASER_ADDR_SHIFT;
+	bool valid = !!(val & GICV5_IRS_IST_BASER_VALID);
+
+	return irs->ist_baser.addr == addr && irs->ist_baser.valid == valid;
+}
+
+static void vgic_v5_update_irs_ist_baser(struct vgic_v5_irs *irs,
+					 unsigned long val)
+{
+	irs->ist_baser.valid = !!(val & GICV5_IRS_IST_BASER_VALID);
+	irs->ist_baser.addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val)
+		<< GICV5_IRS_IST_BASER_ADDR_SHIFT;
+}
+
+static int vgic_v5_write_irs_ist_baser(struct kvm_vcpu *vcpu, unsigned long val)
+{
+	struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu);
+	enum gicv5_vcpu_cmd cmd = LPI_VIST_MAKE_INVALID;
+	bool valid = !!(val & GICV5_IRS_IST_BASER_VALID);
+	int rc;
+
+	/* The address cannot be changed while the IST is valid. */
+	if (irs->ist_baser.valid && valid)
+		return 0;
+
+	/* Valid -> Invalid */
+	if (irs->ist_baser.valid && !valid) {
+		/* Make the LPI IST invalid and then ... */
+		rc = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu), &cmd);
+		if (rc)
+			return rc;
+
+		vgic_v5_update_irs_ist_baser(irs, val);
+
+		/*
+		 * ... reflect that in the emulated BASER before freeing the
+		 * host IST. If the free fails, the guest-visible valid bit
+		 * still matches the hardware state.
+		 */
+		return vgic_v5_lpi_ist_free(vcpu->kvm);
+	} else if (!irs->ist_baser.valid && valid) { /* Invalid -> Valid */
+		if (!vgic_v5_ist_cfgr_valid(irs))
+			return -EINVAL;
+
+		rc = vgic_v5_lpi_ist_alloc(vcpu->kvm, irs->ist_cfgr.lpi_id_bits);
+		if (rc)
+			return rc;
+	}
+
+	/* Now that we've handled the edges, update the valid bit and addr */
+	vgic_v5_update_irs_ist_baser(irs, val);
+
+	return 0;
+}
+
+static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr,
+				       unsigned int len, unsigned long val)
+{
+	struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu);
+	const size_t offset = addr & (SZ_64K - 1);
+
+	switch (offset) {
+	case GICV5_IRS_IST_CFGR:
+		scoped_guard(mutex, &vcpu->kvm->arch.config_lock) {
+			if (irs->ist_baser.valid)
+				return;
+
+			irs->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val);
+			irs->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val);
+			irs->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val);
+			irs->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE);
+		}
+		return;
+	case GICV5_IRS_IST_BASER:
+	case GICV5_IRS_IST_BASER + 4: {
+		scoped_guard(mutex, &vcpu->kvm->arch.config_lock) {
+			u64 baser;
+
+			baser = update_64bit_reg(vgic_v5_read_irs_ist_baser(irs),
+						 offset & 4, len, val);
+			vgic_v5_write_irs_ist_baser(vcpu, baser);
+		}
+		return;
+	}
+	default:
+		return;
+	}
+}
+
+static unsigned long vgic_v5_coresight_read(struct kvm_vcpu *vcpu,
+					    gpa_t addr, unsigned int len)
+{
+	const size_t offset = addr & (SZ_64K - 1);
+
+	switch (offset) {
+	case GICV5_CORESIGHT_DEVARCH:
+		return GICV5_CORESIGHT_DEVARCH_VAL;
+	case GICV5_CORESIGHT_PIDR4:
+		return GICV5_CORESIGHT_PIDR4_JEP106_CONT;
+	case GICV5_CORESIGHT_PIDR5:
+		return GICV5_CORESIGHT_PIDR5_RES0;
+	case GICV5_CORESIGHT_PIDR6:
+		return GICV5_CORESIGHT_PIDR6_RES0;
+	case GICV5_CORESIGHT_PIDR7:
+		return GICV5_CORESIGHT_PIDR7_RES0;
+	case GICV5_CORESIGHT_PIDR0:
+		return GICV5_CORESIGHT_PIDR0_PART_0;
+	case GICV5_CORESIGHT_PIDR1:
+		return GICV5_CORESIGHT_PIDR1_DES_0_PART_1;
+	case GICV5_CORESIGHT_PIDR2:
+		return GICV5_CORESIGHT_PIDR2_DES_1;
+	case GICV5_CORESIGHT_PIDR3:
+		return GICV5_CORESIGHT_PIDR3_REVAND_CMOD;
+	case GICV5_CORESIGHT_CIDR0:
+		return GICV5_CORESIGHT_CIDR0_VAL;
+	case GICV5_CORESIGHT_CIDR1:
+		return GICV5_CORESIGHT_CIDR1_VAL;
+	case GICV5_CORESIGHT_CIDR2:
+		return GICV5_CORESIGHT_CIDR2_VAL;
+	case GICV5_CORESIGHT_CIDR3:
+		return GICV5_CORESIGHT_CIDR3_VAL;
+	default:
+		return 0;
+	}
+}
+
+static unsigned long vgic_v5_mmio_uaccess_read_irs_status(struct kvm_vcpu *vcpu,
+							  gpa_t addr,
+							  unsigned int len)
+{
+	const size_t offset = addr & (SZ_64K - 1);
+
+	switch (offset) {
+	case GICV5_IRS_SYNC_STATUSR:
+		return GICV5_IRS_SYNC_STATUSR_IDLE;
+	case GICV5_IRS_SPI_STATUSR:
+		return GICV5_IRS_SPI_STATUSR_IDLE;
+	case GICV5_IRS_PE_STATUSR:
+		return GICV5_IRS_PE_STATUSR_IDLE;
+	case GICV5_IRS_IST_STATUSR:
+		return GICV5_IRS_IST_STATUSR_IDLE;
+	default:
+		return 0;
+	}
+}
+
+static int vgic_v5_mmio_uaccess_write_irs(struct kvm_vcpu *vcpu, gpa_t addr,
+					  unsigned int len, unsigned long val)
+{
+	struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
+	struct vgic_v5_irs *irs_data = vgic->vgic_v5_irs_data;
+	size_t offset = addr & (SZ_64K - 1);
+	int ret;
+
+	/*
+	 * The following registers are ONLY settable via uaccesses. The guest
+	 * cannot write them!
+	 */
+
+	switch (offset) {
+	case GICV5_IRS_IDR0:
+		if (FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val) !=
+		    GICV5_IRS_IDR0_INT_DOM_NON_SECURE)
+			return -EINVAL;
+
+		if ((val & GICV5_IRS_IDR0_VIRT) ||
+		    (val & GICV5_IRS_IDR0_ONE_N) ||
+		    (val & GICV5_IRS_IDR0_VIRT_ONE_N) ||
+		    (val & GICV5_IRS_IDR0_SETLPI) ||
+		    (val & GICV5_IRS_IDR0_MEC) ||
+		    (val & GICV5_IRS_IDR0_MPAM) ||
+		    (val & GICV5_IRS_IDR0_SWE))
+			return -EINVAL;
+
+		irs_data->idr0.domain = FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val);
+		irs_data->idr0.pa_range = FIELD_GET(GICV5_IRS_IDR0_PA_RANGE, val);
+		irs_data->idr0.virt = !!(val & GICV5_IRS_IDR0_VIRT);
+		irs_data->idr0.setlpi = !!(val & GICV5_IRS_IDR0_SETLPI);
+		irs_data->idr0.mec = !!(val & GICV5_IRS_IDR0_MEC);
+		irs_data->idr0.mpam = !!(val & GICV5_IRS_IDR0_MPAM);
+		irs_data->idr0.swe = !!(val & GICV5_IRS_IDR0_SWE);
+		irs_data->idr0.irs_id = FIELD_GET(GICV5_IRS_IDR0_IRSID, val);
+		break;
+	case GICV5_IRS_IDR1: {
+		unsigned int iaffid_bits, priority_bits;
+		u8 vpe_id_bits;
+
+		/* Ignore writes to PE_CNT as this is populated from num vcpus */
+		iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, val);
+		priority_bits = FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, val);
+
+		/* IAFFID_BITS is derived from the VMTE and encoded as N - 1. */
+		vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu);
+		if (iaffid_bits != vpe_id_bits - 1)
+			return -EINVAL;
+
+		if (priority_bits > gicv5_global_data.irs_pri_bits - 1)
+			return -EINVAL;
+
+		irs_data->idr1.priority_bits = priority_bits;
+		break;
+	}
+	case GICV5_IRS_IDR2:
+		/* We always support LPIs */
+		if (!(val & GICV5_IRS_IDR2_LPI))
+			return -EINVAL;
+
+		/* We only support LPIs with linear, non-metadata guest ISTs */
+		if (val & GICV5_IRS_IDR2_IST_LEVELS)
+			return -EINVAL;
+
+		if ((val & GICV5_IRS_IDR2_ISTMD) ||
+		    FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val))
+			return -EINVAL;
+
+		/* We can't present more bits than we have support for in HW */
+		if (FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val) >
+		    vgic_v5_irs_ist_id_bits(&irs_caps))
+			return -EINVAL;
+
+		/* Min LPI ID bits must be greater than or equal to the HW */
+		if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) <
+		    vgic_v5_irs_min_lpi_id_bits(&irs_caps))
+			return -EINVAL;
+
+		if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) >
+		    FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val))
+			return -EINVAL;
+
+		irs_data->idr2.istmd_sz = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val);
+		irs_data->idr2.istmd = !!(val & GICV5_IRS_IDR2_ISTMD);
+		irs_data->idr2.ist_l2sz = FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, val);
+		irs_data->idr2.ist_levels = !!(val & GICV5_IRS_IDR2_IST_LEVELS);
+		irs_data->idr2.min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val);
+		irs_data->idr2.id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val);
+		break;
+	case GICV5_IRS_IDR5:
+		if (FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, val) != irs_data->idr5.spi_range)
+			return -EINVAL;
+		break;
+	case GICV5_IRS_IDR6:
+		if (FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, val) != irs_data->idr6.spi_irs_range)
+			return -EINVAL;
+		break;
+	case GICV5_IRS_IDR7:
+		if (FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, val) != irs_data->idr7.spi_base)
+			return -EINVAL;
+		break;
+	case GICV5_IRS_IST_BASER:
+		ret = vgic_v5_lpi_ist_exists(vcpu->kvm);
+		if (ret < 0)
+			return ret;
+
+		if (ret && !vgic_v5_ist_baser_matches(irs_data, val))
+			return -EINVAL;
+
+		if (!irs_data->ist_baser.valid &&
+		    (val & GICV5_IRS_IST_BASER_VALID))
+			irs_data->lpi_ist_restore_pending = true;
+
+		vgic_v5_update_irs_ist_baser(irs_data, val);
+		break;
+	case GICV5_IRS_IST_CFGR:
+		ret = vgic_v5_lpi_ist_exists(vcpu->kvm);
+		if (ret < 0)
+			return ret;
+
+		if (ret && !vgic_v5_ist_cfgr_matches(irs_data, val))
+			return -EINVAL;
+
+		irs_data->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val);
+		irs_data->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val);
+		irs_data->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val);
+		irs_data->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE);
+		break;
+	case GICV5_IRS_CR0:
+		vgic->enabled = !!(val & GICV5_IRS_CR0_IRSEN);
+		break;
+	case GICV5_IRS_SPI_CFGR:
+		break;
+	case GICV5_IRS_IIDR:
+		fallthrough;
+	case GICV5_IRS_AIDR:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct vgic_register_region vgic_v5_irs_registers[] = {
+	/*
+	 * This is the IRS_CONFIG_FRAME.
+	 */
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR0, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR1, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR2, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR3, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR4, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR5, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR6, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR7, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IIDR, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_AIDR, vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_CR0,
+					  vgic_v5_mmio_read_irs_misc,
+					  vgic_v5_mmio_write_irs_misc, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_CR1, vgic_v5_mmio_read_irs_misc,
+				  vgic_v5_mmio_write_irs_misc, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SYNCR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SYNC_STATUSR,
+					  vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi,
+					  vgic_v5_mmio_uaccess_read_irs_status,
+					  vgic_mmio_uaccess_write_wi, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_VMR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_SELR, vgic_v5_mmio_read_irs_spi,
+				  vgic_v5_mmio_write_irs_spi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_DOMAINR, vgic_v5_mmio_read_irs_spi,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_RESAMPLER, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_CFGR,
+					  vgic_v5_mmio_read_irs_spi,
+					  vgic_v5_mmio_write_irs_spi, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_STATUSR,
+					  vgic_v5_mmio_read_irs_spi,
+					  vgic_mmio_write_wi,
+					  vgic_v5_mmio_uaccess_read_irs_status,
+					  vgic_mmio_uaccess_write_wi, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_SELR,
+				  vgic_v5_mmio_read_irs_misc,
+				  vgic_v5_mmio_write_irs_misc, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_PE_STATUSR,
+					  vgic_v5_mmio_read_irs_misc,
+					  vgic_mmio_write_wi,
+					  vgic_v5_mmio_uaccess_read_irs_status,
+					  vgic_mmio_uaccess_write_wi, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_CR0, vgic_v5_mmio_read_irs_misc,
+				  vgic_v5_mmio_write_irs_misc, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_BASER,
+					  vgic_v5_mmio_read_irs_ist,
+					  vgic_v5_mmio_write_irs_ist, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 8,
+					  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_CFGR,
+					  vgic_v5_mmio_read_irs_ist,
+					  vgic_v5_mmio_write_irs_ist, NULL,
+					  vgic_v5_mmio_uaccess_write_irs, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_STATUSR,
+					  vgic_v5_mmio_read_irs_ist,
+					  vgic_mmio_write_wi,
+					  vgic_v5_mmio_uaccess_read_irs_status,
+					  vgic_mmio_uaccess_write_wi, 4,
+					  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MAP_L2_ISTR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+
+	/*
+	 * The following registers are only for running VMs. They are not yet
+	 * supported as we don't currently support nested, so expose them as
+	 * read-as-zero/write-ignored.
+	 */
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_BASER, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_CFGR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_STATUSR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_SELR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_DBR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_HPPIR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_CR0, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_STATUSR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_DBR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_SELR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_STATUSR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VMTR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VMR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VISTR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VISTR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VPER, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VMR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VM_STATUSR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+
+	/* MEC, MPAM, SWERR - all unimplemented */
+
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_IDR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_MECID_R, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_IDR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_PARTID_R, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_STATUSR, vgic_mmio_read_raz,
+				  vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER0,
+				  vgic_mmio_read_raz, vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER1,
+				  vgic_mmio_read_raz, vgic_mmio_write_wi, 8,
+				  VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
+
+	/* CoreSight identification registers */
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_DEVARCH,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR4,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR5,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR6,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR7,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR0,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR1,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR2,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR3,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR0,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR1,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR2,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+	REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR3,
+				  vgic_v5_coresight_read, vgic_mmio_write_wi, 4,
+				  VGIC_ACCESS_32bit),
+};
+
+unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev)
+{
+	dev->regions = vgic_v5_irs_registers;
+	dev->nr_regions = ARRAY_SIZE(vgic_v5_irs_registers);
+
+	kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops);
+
+	/* We represent both of the IRS frames back to back, so this is 128K */
+	return KVM_VGIC_V5_IRS_SIZE;
+}
+
+int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address)
+{
+	struct vgic_io_device *io_device = &kvm->arch.vgic.vgic_v5_irs_data->iodev;
+	unsigned int len;
+
+	/*
+	 * Design choice: Force MMIO region to be 64k aligned. Simplifies
+	 * pulling out registers.
+	 */
+	if (!IS_ALIGNED(irs_base_address, SZ_64K))
+		return -EINVAL;
+
+	len = vgic_v5_init_irs_iodev(io_device);
+
+	io_device->base_addr = irs_base_address;
+	io_device->iodev_type = IODEV_GICV5_IRS;
+	io_device->redist_vcpu = NULL;
+
+	return kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, irs_base_address, len,
+				       &io_device->dev);
+}
+
+/**
+ * kvm_vgic_v5_irs_init: initialize the IRS data structures
+ * @kvm: kvm struct pointer
+ * @nr_spis: number of spis, frozen by caller
+ */
+int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis)
+{
+	struct vgic_dist *dist = &kvm->arch.vgic;
+	struct vgic_v5_irs *irs = dist->vgic_v5_irs_data;
+	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
+	size_t nr_spi_bits;
+	u64 mmfr0;
+	int ret, i;
+
+	/*
+	 * We (KVM) allocate an Interrupt State Table (IST) for SPIs. The
+	 * hardware mandates that lower 6 bits of the address are 0. Each ISTE
+	 * is 4 bytes in size (or larger if metadata storage is required), so 16
+	 * entries would be enough for alignment. We require userspace to
+	 * provide at least 32 SPIs, so we always meet alignment requirements.
+	 */
+	if (nr_spis) {
+		dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq),
+				     GFP_KERNEL_ACCOUNT);
+		if (!dist->spis)
+			return -ENOMEM;
+
+		/*
+		 * In the following code we do not take the irq struct lock since
+		 * no other action on irq structs can happen while the VGIC is
+		 * not initialized yet.
+		 */
+		for (i = 0; i < nr_spis; i++) {
+			struct vgic_irq *irq = &dist->spis[i];
+
+			irq->intid = vgic_v5_make_spi(i);
+			INIT_LIST_HEAD(&irq->ap_list);
+			raw_spin_lock_init(&irq->irq_lock);
+			irq->vcpu = NULL;
+			irq->target_vcpu = vcpu0;
+			refcount_set(&irq->refcount, 0);
+			/*
+			 * The guest controls the enable state, and again it is
+			 * directly handled by the hardware. From our point of
+			 * view it is always enabled.
+			 */
+			irq->enabled = 1;
+			vgic_v5_set_spi_ops(irq);
+		}
+
+		nr_spi_bits = fls(roundup_pow_of_two(nr_spis)) - 1;
+
+		ret = vgic_v5_spi_ist_alloc(kvm, nr_spi_bits);
+		if (ret) {
+			kfree(dist->spis);
+			dist->spis = NULL;
+			return ret;
+		}
+	}
+
+	/* Set sane initial state for the IRS MMIO registers */
+
+	irs->idr0.domain = GICV5_IRS_IDR0_INT_DOM_NON_SECURE;
+
+	mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
+	irs->idr0.pa_range = cpuid_feature_extract_unsigned_field(mmfr0,
+								  ID_AA64MMFR0_EL1_PARANGE_SHIFT);
+
+	irs->idr0.virt = 0;
+	irs->idr0.setlpi = 0;
+	irs->idr0.mec = 0;
+	irs->idr0.mpam = 0;
+	irs->idr0.swe = 0;
+	irs->idr0.irs_id = 0;
+
+	irs->idr1.priority_bits = gicv5_global_data.irs_pri_bits - 1;
+
+	/*
+	 * Support 16-bits of ID space for the IRS. This should be sufficient
+	 * for most applications, and the CPUIF is guaranteed to have at least
+	 * 16-bits of ID space support (we actually present 16-bits there, even
+	 * if the hardware supports more). Warn if the hardware doesn't support
+	 * 16 bits, and use the smaller value. YMMV!
+	 *
+	 * As for the minimum number of ID bits, we match the hardware's
+	 * capability.
+	 */
+	if (vgic_v5_irs_ist_id_bits(&irs_caps) < 16)
+		pr_warn_ratelimited("Host IRS supports fewer than 16 ID bits for ISTs (%u)\n",
+				    vgic_v5_irs_ist_id_bits(&irs_caps));
+
+	irs->idr2.id_bits = min(16, vgic_v5_irs_ist_id_bits(&irs_caps));
+	irs->idr2.min_lpi_id_bits = vgic_v5_irs_min_lpi_id_bits(&irs_caps);
+
+	/* Only allow the guest to create Linear ISTs - simplifies Save/Restore */
+	irs->idr2.ist_levels = 0;
+	irs->idr2.ist_l2sz = GICV5_IRS_IST_CFGR_L2SZ_4K;
+	irs->idr2.istmd = 0;
+	irs->idr2.istmd_sz = 0;
+
+	/* We have a single IRS, only. All SPIs reside here! */
+	irs->idr5.spi_range = nr_spis;
+	irs->idr6.spi_irs_range = nr_spis;
+	irs->idr7.spi_base = 0;
+
+	irs->cr1.sh = 0;
+	irs->cr1.oc = 0;
+	irs->cr1.ic = 0;
+	irs->cr1.ist_ra = 0;
+	irs->cr1.ist_wa = 0;
+	irs->cr1.vmt_ra = 0;
+	irs->cr1.vpet_ra = 0;
+	irs->cr1.vmd_ra = 0;
+	irs->cr1.vmd_wa = 0;
+	irs->cr1.vped_ra = 0;
+	irs->cr1.vped_wa = 0;
+
+	irs->spi_selr.id = -1;
+
+	irs->pe_selr.iaffid = -1;
+
+	irs->ist_cfgr.lpi_id_bits = 0;
+	irs->ist_cfgr.l2sz = 0;
+	irs->ist_cfgr.istsz = 0;
+	irs->ist_cfgr.structure = 0;
+
+	irs->ist_baser.valid = 0;
+	irs->ist_baser.addr = 0;
+	irs->lpi_ist_restore_pending = false;
+
+	return 0;
+}
+
+int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits)
+{
+	struct vgic_v5_irs *irs = kvm->arch.vgic.vgic_v5_irs_data;
+
+	if (!irs)
+		return -ENXIO;
+
+	if (!irs->ist_baser.valid)
+		return 0;
+
+	if (!vgic_v5_ist_cfgr_valid(irs))
+		return -EINVAL;
+
+	*id_bits = irs->ist_cfgr.lpi_id_bits;
+
+	return 1;
+}
+
+int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
+{
+	const struct vgic_register_region *region;
+	struct vgic_reg_attr reg_attr;
+	struct kvm_vcpu *vcpu;
+	gpa_t addr, offset;
+	int ret, align;
+
+	ret = vgic_v5_parse_attr(dev, attr, &reg_attr);
+	if (ret)
+		return ret;
+
+	vcpu = reg_attr.vcpu;
+	addr = reg_attr.addr;
+
+	if (attr->group == KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS)
+		return vgic_v5_has_cpu_sysregs_attr(vcpu, attr);
+
+	offset = attr->attr;
+
+	region = vgic_find_mmio_region(vgic_v5_irs_registers,
+				       ARRAY_SIZE(vgic_v5_irs_registers),
+				       offset);
+	if (!region)
+		return -ENXIO;
+
+	align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3;
+	if (offset & align)
+		return -EINVAL;
+
+	return 0;
+}
+
+/*
+ * Access the IRS MMIO Regs. Relevant locks have been taken by the calling code.
+ */
+int vgic_v5_irs_attr_regs_access(struct kvm_device *dev,
+				 struct kvm_device_attr *attr,
+				 u64 *reg, bool is_write)
+{
+	const struct vgic_register_region *region;
+	gpa_t addr, offset;
+	unsigned int len;
+	int align, ret = 0;
+
+	offset = attr->attr;
+
+	if (IS_VGIC_ADDR_UNDEF(dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base))
+		return -ENXIO;
+
+	region = vgic_find_mmio_region(vgic_v5_irs_registers,
+				       ARRAY_SIZE(vgic_v5_irs_registers),
+				       offset);
+	if (!region)
+		return -ENXIO;
+
+	/*
+	 * Although the spec supports upper/lower 32-bit accesses to
+	 * 64-bit IRS registers, the userspace ABI requires 64-bit
+	 * accesses to all 64-bit wide registers. We therefore only
+	 * support 32-bit accesses to 32-bit-wide registers.
+	 */
+	align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3;
+	len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
+
+	if (offset & align)
+		return -EINVAL;
+
+	addr = dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base + offset;
+
+	if (is_write) {
+		if (region->uaccess_write)
+			ret = region->uaccess_write(kvm_get_vcpu(dev->kvm, 0),
+						    addr, len, *reg);
+		else
+			region->write(kvm_get_vcpu(dev->kvm, 0), addr, len, *reg);
+	} else {
+		if (region->uaccess_read)
+			*reg = region->uaccess_read(kvm_get_vcpu(dev->kvm, 0),
+						    addr, len);
+		else
+			*reg = region->read(kvm_get_vcpu(dev->kvm, 0), addr, len);
+	}
+
+	return ret;
+}
diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c
index 9e782a4fea7e5c..e5657cb6035670 100644
--- a/arch/arm64/kvm/vgic/vgic-its.c
+++ b/arch/arm64/kvm/vgic/vgic-its.c
@@ -319,12 +319,16 @@ static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 	return ret;
 }
 
-static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
+static int update_affinity(struct vgic_irq *irq,
+			   struct kvm_vcpu *from_vcpu, struct kvm_vcpu *vcpu)
 {
 	struct its_vlpi_map map;
 	int ret;
 
 	guard(raw_spinlock_irqsave)(&irq->irq_lock);
+	if (from_vcpu && irq->target_vcpu != from_vcpu)
+		return 0;
+
 	irq->target_vcpu = vcpu;
 
 	if (!irq->hw)
@@ -362,7 +366,7 @@ static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
 		return;
 
 	vcpu = collection_to_vcpu(kvm, ite->collection);
-	update_affinity(ite->irq, vcpu);
+	update_affinity(ite->irq, NULL, vcpu);
 }
 
 /*
@@ -856,7 +860,7 @@ static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
 
 	vgic_its_invalidate_cache(its);
 
-	return update_affinity(ite->irq, vcpu);
+	return update_affinity(ite->irq, NULL, vcpu);
 }
 
 static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)
@@ -1383,7 +1387,7 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
 		if (!irq)
 			continue;
 
-		update_affinity(irq, vcpu2);
+		update_affinity(irq, vcpu1, vcpu2);
 
 		vgic_put_irq(kvm, irq);
 	}
@@ -1658,7 +1662,7 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm,
 				      unsigned long val)
 {
 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);
-	u64 entry_size, table_type;
+	u64 old, entry_size, table_type;
 	u64 reg, *regptr, clearbits = 0;
 
 	/* When GITS_CTLR.Enable is 1, we ignore write accesses. */
@@ -1681,7 +1685,9 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm,
 		return;
 	}
 
-	reg = update_64bit_reg(*regptr, addr & 7, len, val);
+	old = *regptr;
+
+	reg = update_64bit_reg(old, addr & 7, len, val);
 	reg &= ~GITS_BASER_RO_MASK;
 	reg &= ~clearbits;
 
@@ -1691,7 +1697,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm,
 
 	*regptr = reg;
 
-	if (!(reg & GITS_BASER_VALID)) {
+	/* The ITS driver rewrites an unchanged GITS_BASER<n> on resume. */
+	if (reg != old) {
 		/* Take the its_lock to prevent a race with a save/restore */
 		mutex_lock(&its->its_lock);
 		switch (table_type) {
@@ -1702,6 +1709,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm,
 			vgic_its_free_collection_list(kvm, its);
 			break;
 		}
+		/* A concurrent injection may have cached a translation. */
+		vgic_its_invalidate_cache(its);
 		mutex_unlock(&its->its_lock);
 	}
 }
@@ -2019,18 +2028,22 @@ static int vgic_its_attr_regs_access(struct kvm_device *dev,
 	return ret;
 }
 
-static u32 compute_next_devid_offset(struct list_head *h,
+static u32 compute_next_devid_offset(struct vgic_its *its, u64 baser,
 				     struct its_device *dev)
 {
-	struct its_device *next;
-	u32 next_offset;
+	struct its_device *next = dev;
 
-	if (list_is_last(&dev->dev_list, h))
-		return 0;
-	next = list_next_entry(dev, dev_list);
-	next_offset = next->device_id - dev->device_id;
+	/*
+	 * Point at the next device vgic_its_save_device_tables() saves. It
+	 * sorts device_list first, so the subtraction cannot underflow.
+	 */
+	list_for_each_entry_continue(next, &its->device_list, dev_list) {
+		if (vgic_its_check_id(its, baser, next->device_id, NULL))
+			return min_t(u32, next->device_id - dev->device_id,
+				     VITS_DTE_MAX_DEVID_OFFSET);
+	}
 
-	return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
+	return 0;
 }
 
 static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
@@ -2121,9 +2134,9 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
 	u64 val;
 
 	/*
-	 * MAPC with V=0 keeps the ITEs mapped but drops their collection,
-	 * and with it the ICID. Save a zeroed entry, which the restore path
-	 * reads back as invalid.
+	 * MAPC with V=0 and clearing GITS_BASER<coll>.Valid both keep the
+	 * ITEs mapped while dropping their collection, and with it the ICID.
+	 * Save a zeroed entry, which the restore path reads back as invalid.
 	 */
 	if (!ite->collection)
 		return vgic_its_write_entry_lock(its, gpa, 0ULL, ite);
@@ -2271,17 +2284,18 @@ static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
  * vgic_its_save_dte - Save a device table entry at a given GPA
  *
  * @its: ITS handle
+ * @baser: GITS_BASER<dev> the caller is saving against
  * @dev: ITS device
  * @ptr: GPA
  */
-static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
-			     gpa_t ptr)
+static int vgic_its_save_dte(struct vgic_its *its, u64 baser,
+			     struct its_device *dev, gpa_t ptr)
 {
 	u64 val, itt_addr_field;
 	u32 next_offset;
 
 	itt_addr_field = dev->itt_addr >> 8;
-	next_offset = compute_next_devid_offset(&its->device_list, dev);
+	next_offset = compute_next_devid_offset(its, baser, dev);
 	val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
 	       ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
 	       (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
@@ -2380,15 +2394,16 @@ static int vgic_its_save_device_tables(struct vgic_its *its)
 		int ret;
 		gpa_t eaddr;
 
+		/* Don't fail a save that userspace must be able to issue. */
 		if (!vgic_its_check_id(its, baser,
 				       dev->device_id, &eaddr))
-			return -EINVAL;
+			continue;
 
 		ret = vgic_its_save_itt(its, dev);
 		if (ret)
 			return ret;
 
-		ret = vgic_its_save_dte(its, dev, eaddr);
+		ret = vgic_its_save_dte(its, baser, dev, eaddr);
 		if (ret)
 			return ret;
 	}
@@ -2541,9 +2556,6 @@ static int vgic_its_save_collection_table(struct vgic_its *its)
 	max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
 
 	list_for_each_entry(collection, &its->collection_list, coll_list) {
-		if (!vgic_its_check_id(its, baser, collection->collection_id, NULL))
-			return -EINVAL;
-
 		ret = vgic_its_save_cte(its, collection, gpa);
 		if (ret)
 			return ret;
diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c
index 90be99443df3bc..1c205cb1361fd5 100644
--- a/arch/arm64/kvm/vgic/vgic-kvm-device.c
+++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c
@@ -181,6 +181,14 @@ static int kvm_vgic_addr(struct kvm *kvm, struct kvm_device_attr *attr, bool wri
 		addr |= (u64)rdreg->count << KVM_VGIC_V3_RDIST_COUNT_SHIFT;
 		goto out;
 	}
+	case KVM_VGIC_V5_ADDR_TYPE_IRS:
+		r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V5);
+		if (r)
+			break;
+		addr_ptr = &vgic->vgic_v5_irs_data->vgic_v5_irs_base;
+		alignment = SZ_64K;
+		size = KVM_VGIC_V5_IRS_SIZE;
+		break;
 	default:
 		r = -ENODEV;
 	}
@@ -224,31 +232,48 @@ static int vgic_set_common_attr(struct kvm_device *dev,
 		if (get_user(val, uaddr))
 			return -EFAULT;
 
-		/*
-		 * We require:
-		 * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
-		 * - at most 1024 interrupts
-		 * - a multiple of 32 interrupts
-		 */
-		if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
-		    val > VGIC_MAX_RESERVED ||
-		    (val & 31))
-			return -EINVAL;
+		if (!vgic_is_v5(dev->kvm)) {
+			/*
+			 * We require:
+			 * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
+			 * - at most 1024 interrupts
+			 * - a multiple of 32 interrupts
+			 */
+			if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
+			    val > VGIC_MAX_RESERVED || (val & 31))
+				return -EINVAL;
+
+			mutex_lock(&dev->kvm->arch.config_lock);
 
-		mutex_lock(&dev->kvm->arch.config_lock);
+			/*
+			 * Either userspace has already configured NR_IRQS or
+			 * the vgic has already been initialized and vgic_init()
+			 * supplied a default amount of SPIs.
+			 */
+			if (dev->kvm->arch.vgic.nr_spis)
+				ret = -EBUSY;
+			else
+				dev->kvm->arch.vgic.nr_spis =
+					val - VGIC_NR_PRIVATE_IRQS;
 
-		/*
-		 * Either userspace has already configured NR_IRQS or
-		 * the vgic has already been initialized and vgic_init()
-		 * supplied a default amount of SPIs.
-		 */
-		if (dev->kvm->arch.vgic.nr_spis)
-			ret = -EBUSY;
-		else
-			dev->kvm->arch.vgic.nr_spis =
-				val - VGIC_NR_PRIVATE_IRQS;
+			mutex_unlock(&dev->kvm->arch.config_lock);
+		} else {
+			/*
+			 * GICv5 reports a number of SPIs, not a total number of
+			 * interrupts. Require a multiple of 32 SPIs.
+			 */
+			if (val < VGIC_V5_DEFAULT_NR_SPIS ||
+			    val > VGIC_V5_MAX_NR_SPIS ||
+			    (val & 31))
+				return -EINVAL;
 
-		mutex_unlock(&dev->kvm->arch.config_lock);
+			mutex_lock(&dev->kvm->arch.config_lock);
+			if (vgic_initialized(dev->kvm) || dev->kvm->arch.vgic.nr_spis)
+				ret = -EBUSY;
+			else
+				dev->kvm->arch.vgic.nr_spis = val;
+			mutex_unlock(&dev->kvm->arch.config_lock);
+		}
 
 		return ret;
 	}
@@ -299,9 +324,14 @@ static int vgic_get_common_attr(struct kvm_device *dev,
 		return (r == -ENODEV) ? -ENXIO : r;
 	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
 		u32 __user *uaddr = (u32 __user *)(long)attr->addr;
-
-		r = put_user(dev->kvm->arch.vgic.nr_spis +
-			     VGIC_NR_PRIVATE_IRQS, uaddr);
+		/* Older GICs */
+		if (!vgic_is_v5(dev->kvm)) {
+			r = put_user(dev->kvm->arch.vgic.nr_spis +
+					     VGIC_NR_PRIVATE_IRQS,
+				     uaddr);
+		} else {
+			r = put_user(dev->kvm->arch.vgic.nr_spis, uaddr);
+		}
 		break;
 	}
 	}
@@ -512,7 +542,7 @@ int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
  * Allow access to certain ID-like registers prior to VGIC initialization,
  * thereby allowing the VMM to provision the features / sizing of the VGIC.
  */
-static bool reg_allowed_pre_init(struct kvm_device_attr *attr)
+static bool v3_reg_allowed_pre_init(struct kvm_device_attr *attr)
 {
 	if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS)
 		return false;
@@ -575,7 +605,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
 
 	mutex_lock(&dev->kvm->arch.config_lock);
 
-	if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) {
+	if (!(vgic_initialized(dev->kvm) || v3_reg_allowed_pre_init(attr))) {
 		ret = -EBUSY;
 		goto out;
 	}
@@ -743,26 +773,178 @@ static int vgic_v5_get_userspace_ppis(struct kvm_device *dev,
 	return ret;
 }
 
+int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
+		       struct vgic_reg_attr *reg_attr)
+{
+	unsigned long vgic_mpidr, mpidr_reg;
+
+	switch (attr->group) {
+	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+		vgic_mpidr = (attr->attr & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) >>
+			KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT;
+
+		mpidr_reg = VGIC_TO_MPIDR(vgic_mpidr);
+		reg_attr->vcpu = kvm_mpidr_to_vcpu(dev->kvm, mpidr_reg);
+		break;
+	case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
+		reg_attr->vcpu = kvm_get_vcpu(dev->kvm, 0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (!reg_attr->vcpu)
+		return -EINVAL;
+
+	reg_attr->addr = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
+
+	return 0;
+}
+
+/*
+ * Some registers can potentially be read before the core GIC & IRS has been
+ * initialised. Right now, everything is required to be post-init.
+ */
+static bool v5_reg_allowed_pre_init(struct kvm_device_attr *attr)
+{
+	return false;
+}
+
+static bool vgic_v5_vm_has_run_once(struct kvm *kvm)
+{
+	struct kvm_vcpu *vcpu;
+	unsigned long i;
+
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		if (vcpu_has_run_once(vcpu))
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * vgic_v5_attr_regs_access - allows user space to access VGIC v5 state
+ *
+ * @dev:      kvm device handle
+ * @attr:     kvm device attribute
+ * @is_write: true if userspace is writing a register
+ */
+static int vgic_v5_attr_regs_access(struct kvm_device *dev,
+				    struct kvm_device_attr *attr,
+				    bool is_write)
+{
+	u64 __user *uaddr = (u64 __user *)(unsigned long)attr->addr;
+	struct vgic_reg_attr reg_attr;
+	struct kvm_vcpu *vcpu;
+	bool uaccess;
+	u64 val;
+	int ret;
+
+	ret = vgic_v5_parse_attr(dev, attr, &reg_attr);
+	if (ret)
+		return ret;
+
+	vcpu = reg_attr.vcpu;
+
+	switch (attr->group) {
+	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+		/* Sysregs uaccess is performed by the sysreg handling code */
+		uaccess = false;
+		break;
+	case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
+		fallthrough;
+	default:
+		uaccess = true;
+	}
+
+	if (uaccess && is_write) {
+		if (get_user(val, uaddr))
+			return -EFAULT;
+	}
+
+	mutex_lock(&dev->kvm->lock);
+
+	if (kvm_trylock_all_vcpus(dev->kvm)) {
+		mutex_unlock(&dev->kvm->lock);
+		return -EBUSY;
+	}
+
+	mutex_lock(&dev->kvm->arch.config_lock);
+
+	if (!(vgic_initialized(dev->kvm) || v5_reg_allowed_pre_init(attr))) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	if (is_write && vgic_v5_vm_has_run_once(dev->kvm)) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	switch (attr->group) {
+	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+		ret = vgic_v5_cpu_sysregs_uaccess(vcpu, attr, is_write);
+		break;
+	case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
+		/*
+		 * The IRS registers are a mixture of 32-bit and 64-bit
+		 * registers. Internally, we always perform the correctly sized
+		 * access, but the UAPI is defined in such a way that we are
+		 * always provided a __u64 by userspace. When userspace writes,
+		 * the upper 32-bits are ignored for 32-bit accesses, and on a
+		 * read any 32-bit accesses are written back to user memory
+		 * using the full 64-bits.
+		 */
+		ret = vgic_v5_irs_attr_regs_access(dev, attr, &val, is_write);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+out:
+	mutex_unlock(&dev->kvm->arch.config_lock);
+	kvm_unlock_all_vcpus(dev->kvm);
+	mutex_unlock(&dev->kvm->lock);
+
+	if (!ret && uaccess && !is_write)
+		ret = put_user(val, uaddr);
+
+	return ret;
+}
+
 static int vgic_v5_set_attr(struct kvm_device *dev,
 			    struct kvm_device_attr *attr)
 {
 	switch (attr->group) {
 	case KVM_DEV_ARM_VGIC_GRP_ADDR:
+		break;
+	case KVM_DEV_ARM_VGIC_GRP_IST:
+		if (attr->attr)
+			return -ENXIO;
+
+		return vgic_v5_irs_restore_ists(dev->kvm, attr);
+	case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
+		fallthrough;
 	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+		return vgic_v5_attr_regs_access(dev, attr, true);
 	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
-		return -ENXIO;
+		break;
 	case KVM_DEV_ARM_VGIC_GRP_CTRL:
 		switch (attr->attr) {
 		case KVM_DEV_ARM_VGIC_CTRL_INIT:
-			return vgic_set_common_attr(dev, attr);
+			break;
 		case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
 		default:
 			return -ENXIO;
 		}
+		break;
 	default:
 		return -ENXIO;
 	}
 
+	return vgic_set_common_attr(dev, attr);
 }
 
 static int vgic_v5_get_attr(struct kvm_device *dev,
@@ -770,21 +952,33 @@ static int vgic_v5_get_attr(struct kvm_device *dev,
 {
 	switch (attr->group) {
 	case KVM_DEV_ARM_VGIC_GRP_ADDR:
+		break;
+	case KVM_DEV_ARM_VGIC_GRP_IST:
+		if (attr->attr)
+			return -ENXIO;
+
+		return vgic_v5_irs_save_ists(dev->kvm, attr);
+	case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
+		fallthrough;
 	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+		return vgic_v5_attr_regs_access(dev, attr, false);
 	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
-		return -ENXIO;
+		break;
 	case KVM_DEV_ARM_VGIC_GRP_CTRL:
 		switch (attr->attr) {
 		case KVM_DEV_ARM_VGIC_CTRL_INIT:
-			return vgic_get_common_attr(dev, attr);
+			break;
 		case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
 			return vgic_v5_get_userspace_ppis(dev, attr);
 		default:
 			return -ENXIO;
 		}
+		break;
 	default:
 		return -ENXIO;
 	}
+
+	return vgic_get_common_attr(dev, attr);
 }
 
 static int vgic_v5_has_attr(struct kvm_device *dev,
@@ -792,18 +986,30 @@ static int vgic_v5_has_attr(struct kvm_device *dev,
 {
 	switch (attr->group) {
 	case KVM_DEV_ARM_VGIC_GRP_ADDR:
+		switch (attr->attr) {
+		case KVM_VGIC_V5_ADDR_TYPE_IRS:
+			return 0;
+		}
+		return -ENXIO;
+	case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
+		fallthrough;
 	case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
+		return vgic_v5_has_attr_regs(dev, attr);
 	case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
-		return -ENXIO;
+		return 0;
 	case KVM_DEV_ARM_VGIC_GRP_CTRL:
 		switch (attr->attr) {
 		case KVM_DEV_ARM_VGIC_CTRL_INIT:
 			return 0;
 		case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
 			return 0;
+		case KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES:
 		default:
 			return -ENXIO;
 		}
+		break;
+	case KVM_DEV_ARM_VGIC_GRP_IST:
+		return attr->attr ? -ENXIO : 0;
 	default:
 		return -ENXIO;
 	}
diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c
index 74d76dec97304b..fddb9da0403d55 100644
--- a/arch/arm64/kvm/vgic/vgic-mmio.c
+++ b/arch/arm64/kvm/vgic/vgic-mmio.c
@@ -1065,6 +1065,9 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 	case IODEV_ITS:
 		data = region->its_read(vcpu->kvm, iodev->its, addr, len);
 		break;
+	case IODEV_GICV5_IRS:
+		data = region->read(vcpu, addr, len);
+		break;
 	}
 
 	vgic_data_host_to_mmio_bus(val, len, data);
@@ -1095,6 +1098,9 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
 	case IODEV_ITS:
 		region->its_write(vcpu->kvm, iodev->its, addr, len, data);
 		break;
+	case IODEV_GICV5_IRS:
+		region->write(vcpu, addr, len, data);
+		break;
 	}
 
 	return 0;
diff --git a/arch/arm64/kvm/vgic/vgic-mmio.h b/arch/arm64/kvm/vgic/vgic-mmio.h
index 50dc80220b0f37..38ed730d68ac38 100644
--- a/arch/arm64/kvm/vgic/vgic-mmio.h
+++ b/arch/arm64/kvm/vgic/vgic-mmio.h
@@ -217,6 +217,8 @@ unsigned int vgic_v2_init_cpuif_iodev(struct vgic_io_device *dev);
 
 unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev);
 
+unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev);
+
 u64 vgic_sanitise_outer_cacheability(u64 reg);
 u64 vgic_sanitise_inner_cacheability(u64 reg);
 u64 vgic_sanitise_shareability(u64 reg);
diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c
index 7182f63fc9382c..70cc53ba810ade 100644
--- a/arch/arm64/kvm/vgic/vgic-v2.c
+++ b/arch/arm64/kvm/vgic/vgic-v2.c
@@ -122,6 +122,10 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
 	for (int lr = 0; lr < vgic_cpu->vgic_v2.used_lrs; lr++)
 		vgic_v2_fold_lr(vcpu, cpuif->vgic_lr[lr]);
 
+	cpuif->used_lrs = 0;
+	if (!irq)
+		return;
+
 	/* See the GICv3 equivalent for the EOIcount handling rationale */
 	list_for_each_entry_continue(irq, &vgic_cpu->ap_list_head, ap_list) {
 		u32 lr;
@@ -144,8 +148,6 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
 		vgic_v2_fold_lr(vcpu, lr);
 		eoicount--;
 	}
-
-	cpuif->used_lrs = 0;
 }
 
 void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val)
diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c
index 726e20a1da6e72..346bacb3198f22 100644
--- a/arch/arm64/kvm/vgic/vgic-v3.c
+++ b/arch/arm64/kvm/vgic/vgic-v3.c
@@ -155,6 +155,10 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
 	for (int lr = 0; lr < cpuif->used_lrs; lr++)
 		vgic_v3_fold_lr(vcpu, cpuif->vgic_lr[lr]);
 
+	cpuif->used_lrs = 0;
+	if (!irq)
+		return;
+
 	/*
 	 * EOIMode=0: use EOIcount to emulate deactivation. We are
 	 * guaranteed to deactivate in reverse order of the activation, so
@@ -188,8 +192,6 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
 		vgic_v3_fold_lr(vcpu, lr);
 		eoicount--;
 	}
-
-	cpuif->used_lrs = 0;
 }
 
 void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val)
diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c
new file mode 100644
index 00000000000000..d577e485ebe9e8
--- /dev/null
+++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c
@@ -0,0 +1,1872 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025, 2026 Arm Ltd.
+ */
+
+#include <kvm/arm_vgic.h>
+#include <linux/dma-mapping.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/xarray.h>
+#include <asm/kvm_mmu.h>
+
+#include "vgic.h"
+#include "vgic-v5-tables.h"
+
+#define irs_caps	kvm_vgic_global_state.vgic_v5_irs_caps
+
+static struct vgic_v5_vmt *vmt_info;
+
+/* Serialises IRS MMIO interaction (commands) and accesses to IRS tables. */
+DEFINE_RAW_SPINLOCK(vgic_v5_irs_lock);
+
+/* Serialises lazy installation of shared second-level VMTs. */
+static DEFINE_MUTEX(vmt_l2_lock);
+
+static DEFINE_XARRAY(vm_info);
+
+/* Level 1 Virtual Machine Table Entry */
+#define GICV5_VMTEL1E_VALID		BIT_ULL(0)
+/* Note that there is no shift for the address by design */
+#define GICV5_VMTEL1E_L2_ADDR		GENMASK(51, 12)
+
+#define GICV5_VMTEL2E_SIZE		32ULL
+/* An L2 table (two-level VMT) is ALWAYS 4kB! */
+#define GICV5_VMT_L2_TABLE_SIZE		4096ULL
+#define GICV5_VMT_L2_TABLE_ENTRIES	(GICV5_VMT_L2_TABLE_SIZE / GICV5_VMTEL2E_SIZE)
+
+/*
+ * As the L2 VMTE is a large data structure, we are splitting it into 4 parts.
+ * We only mask and shift WITHIN each part for simplicity.
+ */
+/* First 64-bit chunk */
+#define GICV5_VMTEL2E_VALID		BIT_ULL(0)
+#define GICV5_VMTEL2E_VMD_ADDR_SHIFT	3ULL
+#define GICV5_VMTEL2E_VMD_ADDR		GENMASK_ULL(55, 3)
+/* Second 64-bit chunk */
+#define GICV5_VMTEL2E_VPET_ADDR_SHIFT	3ULL
+#define GICV5_VMTEL2E_VPET_ADDR		GENMASK_ULL(55, 3)
+#define GICV5_VMTEL2E_VPE_ID_BITS	GENMASK_ULL(63, 59)
+/* Third & fourth 64-bit chunks (the encodings are the same for each) */
+#define GICV5_VMTEL2E_IST_VALID		BIT_ULL(0)
+#define GICV5_VMTEL2E_IST_L2SZ		GENMASK_ULL(2, 1)
+#define GICV5_VMTEL2E_IST_ADDR_SHIFT	6ULL
+#define GICV5_VMTEL2E_IST_ADDR		GENMASK_ULL(55, 6)
+#define GICV5_VMTEL2E_IST_ISTSZ		GENMASK_ULL(57, 56)
+#define GICV5_VMTEL2E_IST_STRUCTURE	BIT_ULL(58)
+#define GICV5_VMTEL2E_IST_ID_BITS	GENMASK_ULL(63, 59)
+
+/* Virtual PE Table Entry */
+#define GICV5_VPE_VALID			BIT_ULL(0)
+/* Note that there is no shift for the address by design. */
+#define GICV5_VPED_ADDR			GENMASK_ULL(55, 3)
+
+/* L2 Interrupt State Table Entry */
+#define GICV5_ISTL2E_PENDING		BIT(0)
+#define GICV5_ISTL2E_ACTIVE		BIT(1)
+#define GICV5_ISTL2E_HM			BIT(2)
+#define GICV5_ISTL2E_ENABLE		BIT(3)
+#define GICV5_ISTL2E_IRM		BIT(4)
+#define GICV5_ISTL2E_HWU		GENMASK(10, 9)
+#define GICV5_ISTL2E_PRIORITY		GENMASK(15, 11)
+#define GICV5_ISTL2E_IAFFID		GENMASK(31, 16)
+
+#define GICV5_ISTE_SIZE(istsz)		BIT((istsz) + 2)
+#define GICV5_LINEAR_IST_SIZE(id_bits, istsz)	\
+	(BIT(id_bits) * GICV5_ISTE_SIZE(istsz))
+
+/*
+ * The LPI and SPI configuration is stored in the 2nd and 3rd 64-bit chunks of
+ * the VMTE (0-based). We call this a section here in an attempt to simplify the
+ * code.
+ */
+#define GICV5_VMTEL2_LPI_SECTION	2
+#define GICV5_VMTEL2_SPI_SECTION	3
+
+struct vgic_v5_ist_desc {
+	struct vgic_v5_vm_info	*vmi;
+	void			*base;
+	unsigned int		id_bits;
+	unsigned int		istsz;
+	unsigned int		l2sz;
+	size_t			iste_size;
+	bool			present;
+};
+
+struct vgic_v5_two_level_ist_shape {
+	size_t	l1_entries;
+	size_t	l2_entries;
+};
+
+struct vgic_v5_pending_irq {
+	u32			irq;
+	struct list_head	next;
+};
+
+static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist,
+				    unsigned int id_bits,
+				    unsigned int istsz);
+static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits,
+				unsigned int istsz, unsigned int l2_split);
+static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits,
+				 unsigned int istsz, unsigned int l2_split);
+static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm,
+					   unsigned int id_bits,
+					   unsigned int istsz,
+					   unsigned int l2_split);
+static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi);
+static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi);
+static int vgic_v5_spi_ist_free(struct kvm *kvm);
+
+/*
+ * Our IRS might be coherent or non-coherent. If coherent, we can just emit a
+ * DSB to ensure that we're in sync. However, when non-coherent, we need to
+ * manage our cached data explicitly.
+ *
+ * This helper is used to handle both coherent and non-coherent IRSes, and
+ * handles all combinations of cleaning and invalidating to the PoC.
+ */
+static void vgic_v5_clean_inval(void *va, size_t size)
+{
+	unsigned long base = (unsigned long)va;
+
+	dsb(ishst);
+
+	if (kvm_vgic_global_state.vgic_v5_irs_caps.non_coherent)
+		dcache_clean_inval_poc(base, base + size);
+}
+
+static void vgic_v5_drain_pending_irqs(struct kvm *kvm,
+				       struct vgic_v5_vm_info *vmi,
+				       bool reinject)
+{
+	struct vgic_v5_pending_irq *pirq, *tmp;
+
+	list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) {
+		if (reinject)
+			kvm_call_hyp(__vgic_v5_vdpend, pirq->irq, true,
+				     vgic_v5_vm_id(kvm));
+
+		list_del(&pirq->next);
+		kfree(pirq);
+	}
+}
+
+/*
+ * Create a linear VM Table. Directly using the number of entries supplied as
+ * the size of an L2 VMTE (32 bytes) guarantees that our allocation is aligned per
+ * the GICv5 requirements for the IRS_VMT_BASER.
+ */
+static int vgic_v5_alloc_vmt_linear(unsigned int num_entries)
+{
+	vmt_info->linear.vmt_base = kzalloc_objs(*vmt_info->linear.vmt_base,
+						 num_entries);
+	if (!vmt_info->linear.vmt_base)
+		return -ENOMEM;
+
+	vgic_v5_clean_inval(vmt_info->linear.vmt_base,
+			    num_entries * sizeof(struct vmtl2_entry));
+
+	return 0;
+}
+
+/*
+ * Allocate the first level of a two-level VM table. The second-level VM tables
+ * are allocated on demand (by vgic_v5_alloc_l2_vmt()).
+ */
+static int vgic_v5_alloc_vmt_two_level(unsigned int num_entries)
+{
+	/*
+	 * Each L2 VMT array is always 4k-sized (covering 128 VMs). This is
+	 * mandated by the GICv5 specification (GICv5 EAC0 Specification rule
+	 * D_LSPBK). Hence, round up the number of entries to be at least 128
+	 * (or the next highest power of two as we give the HW the number of VM
+	 * ID bits).
+	 */
+	if (num_entries < GICV5_VMT_L2_TABLE_ENTRIES)
+		num_entries = GICV5_VMT_L2_TABLE_ENTRIES;
+	num_entries = roundup_pow_of_two(num_entries);
+
+	vmt_info->l2.num_l1_ents = (num_entries / GICV5_VMT_L2_TABLE_ENTRIES);
+	vmt_info->l2.vmt_base = kzalloc_objs(*vmt_info->l2.vmt_base,
+					     vmt_info->l2.num_l1_ents);
+	if (!vmt_info->l2.vmt_base)
+		return -ENOMEM;
+
+	vmt_info->l2.l2ptrs = kzalloc_objs(*vmt_info->l2.l2ptrs,
+					   vmt_info->l2.num_l1_ents,
+					   GFP_KERNEL);
+	if (!vmt_info->l2.l2ptrs) {
+		kfree(vmt_info->l2.vmt_base);
+		return -ENOMEM;
+	}
+
+	vgic_v5_clean_inval(vmt_info->l2.vmt_base,
+			    vmt_info->l2.num_l1_ents * sizeof(vmtl1_entry));
+
+	return 0;
+}
+
+/*
+ * Allocate a second level VMT, if required. This can be called eagerly, and
+ * will only perform the allocation if required.
+ */
+static int vgic_v5_alloc_l2_vmt(struct kvm *kvm)
+{
+	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	enum gicv5_vcpu_cmd cmd = VMT_L2_MAP;
+	struct vmtl2_entry *l2_table;
+	unsigned int l1_index;
+	int ret;
+
+	/* Nothing to do if we have linear tables! */
+	if (!vmt_info->two_level)
+		return 0;
+
+	if (vm_id == VGIC_V5_VM_ID_INVAL)
+		return -EINVAL;
+
+	/*
+	 * We have 4k-sized L2 tables - this is mandated by the spec for
+	 * two-level VMTs (GICv5 EAC0 Specification rule D_LSPBK). This means
+	 * that we have 128 entries per L1 VMTE.
+	 */
+	l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES;
+
+	guard(mutex)(&vmt_l2_lock);
+
+	/* Already valid? Great! */
+	if (vmt_info->l2.l2ptrs[l1_index])
+		return 0;
+
+	l2_table = kzalloc_objs(*l2_table, GICV5_VMT_L2_TABLE_ENTRIES);
+	if (!l2_table)
+		return -ENOMEM;
+
+	/* The VMT is shared between all VMs. */
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		vgic_v5_clean_inval(l2_table, GICV5_VMT_L2_TABLE_SIZE);
+		vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index,
+				    sizeof(vmtl1_entry));
+
+		WRITE_ONCE(vmt_info->l2.vmt_base[l1_index],
+			   cpu_to_le64(virt_to_phys(l2_table)));
+
+		vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index,
+				    sizeof(vmtl1_entry));
+
+	}
+
+	/*
+	 * VMAP in the L2 VMT via the IRS. We use any of the VM's CPUs as a
+	 * conduit for interacting with the host's IRS. In the current case,
+	 * this lets us resolve the VM ID to pass to the hardware.
+	 */
+	ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd);
+
+	/* We've failed to make the L2 VMT valid - things are very broken! */
+	if (ret) {
+		scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+			/* Remove the pointer from L1 table */
+			WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], 0);
+
+			vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index,
+					    sizeof(vmtl1_entry));
+		}
+
+		kfree(l2_table);
+		return ret;
+	}
+
+	vmt_info->l2.l2ptrs[l1_index] = l2_table;
+
+	return 0;
+}
+
+/*
+ * Allocate the top-level VMT. This can either be linear or two-level.
+ */
+int vgic_v5_vmt_allocate(unsigned int max_vpes)
+{
+	int ret;
+
+	/* Allocate the tracking structure */
+	vmt_info = kzalloc_obj(*vmt_info, GFP_KERNEL);
+	if (!vmt_info)
+		return -ENOMEM;
+
+	ida_init(&vmt_info->vm_id_ida);
+	vmt_info->max_vpes = max_vpes;
+	vmt_info->vmd_size = vgic_v5_irs_vmd_size(&irs_caps);
+	vmt_info->vped_size = vgic_v5_irs_vped_size(&irs_caps);
+	vmt_info->two_level = vgic_v5_irs_two_level_vmt_support(&irs_caps);
+	vmt_info->num_entries = vgic_v5_irs_max_vms(&irs_caps);
+
+	if (vmt_info->two_level)
+		ret = vgic_v5_alloc_vmt_two_level(vmt_info->num_entries);
+	else
+		ret = vgic_v5_alloc_vmt_linear(vmt_info->num_entries);
+
+	/* If anything failed, free our tracking structure before returning */
+	if (ret) {
+		kfree(vmt_info);
+		vmt_info = NULL;
+	}
+
+	return ret;
+}
+
+/*
+ * Free the VMT and associated tracking structures. This isn't strictly expected
+ * to be called in general operation, but instead exists for completeness.
+ */
+int vgic_v5_vmt_free(void)
+{
+	if (!vmt_info)
+		return 0;
+
+	if (!vmt_info->two_level) {
+		kfree(vmt_info->linear.vmt_base);
+	} else {
+		/* Free the L2 tables; kfree(NULL) is safe */
+		for (int i = 0; i < vmt_info->l2.num_l1_ents; ++i)
+			kfree(vmt_info->l2.l2ptrs[i]);
+		kfree(vmt_info->l2.l2ptrs);
+
+		/* And now free the L1 table */
+		kfree(vmt_info->l2.vmt_base);
+	}
+
+	ida_destroy(&vmt_info->vm_id_ida);
+	kfree(vmt_info);
+	vmt_info = NULL;
+
+	return 0;
+}
+
+/*
+ * Look up a VMT Entry by VM ID.
+ */
+static struct vmtl2_entry *vgic_v5_get_l2_vmte(u32 vm_id)
+{
+	unsigned int l1_index, l2_index;
+	struct vmtl2_entry *l2_table;
+
+	if (vm_id == VGIC_V5_VM_ID_INVAL)
+		return ERR_PTR(-EINVAL);
+
+	if (!vmt_info->two_level)
+		return &vmt_info->linear.vmt_base[vm_id];
+
+	l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES;
+	l2_index = vm_id % GICV5_VMT_L2_TABLE_ENTRIES;
+
+	if (l1_index >= vmt_info->l2.num_l1_ents)
+		return ERR_PTR(-E2BIG);
+
+	if (!vmt_info->l2.l2ptrs[l1_index])
+		return ERR_PTR(-EINVAL);
+
+	l2_table = vmt_info->l2.l2ptrs[l1_index];
+	return &l2_table[l2_index];
+}
+
+/*
+ * Zero a VMT Entry, and flush & invalidate to the PoC, if required.
+ */
+static int vgic_v5_reset_vmte(struct kvm *kvm)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vmtl2_entry *vmte;
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		/*
+		 * The VMT is normal memory shared with the IRS. Invalidate before
+		 * rewriting the entry so that cacheline-granular maintenance cannot
+		 * later push stale data for neighbouring IRS-visible state back to
+		 * memory.
+		 */
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+
+		/*
+		 * Prevent the compiler from eliding the individual VMTE
+		 * stores. Ordering and visibility to the IRS are provided by the
+		 * surrounding cache maintenance and command protocol, not by
+		 * WRITE_ONCE().
+		 *
+		 * The same compiler-access constraint applies to READ_ONCE() users in
+		 * this file: when inspecting IRS-visible table entries, read the field
+		 * exactly once and prevent the compiler from reusing, merging or
+		 * tearing the access. Coherency and freshness for non-coherent IRSes
+		 * still come from the surrounding cache maintenance.
+		 */
+		WRITE_ONCE(vmte->val[0], cpu_to_le64(0ULL));
+		WRITE_ONCE(vmte->val[1], cpu_to_le64(0ULL));
+		WRITE_ONCE(vmte->val[2], cpu_to_le64(0ULL));
+		WRITE_ONCE(vmte->val[3], cpu_to_le64(0ULL));
+
+		/* And make our write visible to the IRS (if non-coherent) */
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+	}
+
+	return 0;
+}
+
+/*
+ * Use the IDA to allocate a new VM ID, and track it in the gicv5_vm data
+ * structure. If we're out of VM IDs, the IDA catches that, and we return the
+ * error (-ENOSPC). If we've previously allocated a VM ID, we catch that too and
+ * return -EBUSY.
+ */
+int vgic_v5_allocate_vm_id(struct kvm *kvm)
+{
+	int id;
+
+	if (kvm->arch.vgic.gicv5_vm.vm_id != VGIC_V5_VM_ID_INVAL)
+		return -EBUSY;
+
+	id = ida_alloc_max(&vmt_info->vm_id_ida, vmt_info->num_entries - 1u,
+			   GFP_KERNEL);
+	if (id < 0)
+		return id;
+
+	kvm->arch.vgic.gicv5_vm.vm_id = id;
+
+	return 0;
+}
+
+/*
+ * Release the VM ID to allow it to be reallocated in the future.
+ */
+void vgic_v5_release_vm_id(struct kvm *kvm)
+{
+	if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL)
+		return;
+
+	ida_free(&vmt_info->vm_id_ida, kvm->arch.vgic.gicv5_vm.vm_id);
+	kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL;
+}
+
+/*
+ * Initialise an entry in the VMT based on the index of the VM.
+ *
+ * Note: We don't mark the VMTE as valid as this needs to be done by
+ * the hardware.
+ */
+int vgic_v5_vmte_init(struct kvm *kvm)
+{
+	size_t vmd_alloc_size, vpet_alloc_size, vped_alloc_size;
+	void *vped_base = NULL, *vmd = NULL;
+	struct vgic_v5_vm_info *vmi = NULL;
+	u64 tmp, vmte_val0 = 0, vmte_val1;
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	int ret, nr_cpus, nr_vcpus;
+	bool vmi_inserted = false;
+	struct vmtl2_entry *vmte;
+	vpe_entry *vpet = NULL;
+	struct kvm_vcpu *vcpu;
+	u16 max_vpe_id = 0;
+	unsigned long i;
+
+	nr_vcpus = atomic_read(&kvm->online_vcpus);
+	if (nr_vcpus > vmt_info->max_vpes)
+		return -E2BIG;
+
+	/*
+	 * If we're using two-level VMTs, L2 is allocated on demand. For linear
+	 * VMTs, this is a NOP.
+	 */
+	ret = vgic_v5_alloc_l2_vmt(kvm);
+	if (ret)
+		return ret;
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	/* If the entry is already valid, something went wrong */
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+		if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID)
+			return -EINVAL;
+	}
+
+	ret = vgic_v5_reset_vmte(kvm);
+	if (ret)
+		return ret;
+
+	vmi = kzalloc_obj(*vmi);
+	if (!vmi) {
+		ret = -ENOMEM;
+		goto out_fail;
+	}
+
+	ret = xa_insert(&vm_info, vm_id, vmi, GFP_KERNEL);
+	if (ret)
+		goto out_fail;
+	vmi_inserted = true;
+
+	/*
+	 * If we are restoring the state of a guest, we need to re-inject any
+	 * IRQs that were pending when the state of the guest was originally
+	 * saved. We use the pending_irqs list for this.
+	 */
+	INIT_LIST_HEAD(&vmi->pending_irqs);
+
+	/* Allocate and assign the VM Descriptor, if required. */
+	if (vmt_info->vmd_size != 0) {
+		vmd_alloc_size = round_up(vmt_info->vmd_size,
+					  dma_get_cache_alignment());
+		vmd = kzalloc(vmd_alloc_size, GFP_KERNEL);
+		if (!vmd) {
+			ret = -ENOMEM;
+			goto out_fail;
+		}
+
+		/* Stash the VA so we can free it later */
+		vmi->vmd_base = vmd;
+
+		tmp = FIELD_PREP(GICV5_VMTEL2E_VMD_ADDR,
+				 virt_to_phys(vmd) >> GICV5_VMTEL2E_VMD_ADDR_SHIFT);
+		vmte_val0 = tmp;
+	}
+
+	/*
+	 * Allocate and assign the VPE Table.
+	 *
+	 * First of all, iterate over all vcpus to find the highest VPE ID we
+	 * require - we need to ensure that we have enough storage for all
+	 * vcpu_id values that userspace has picked and not just the total
+	 * number of vcpus. This gives us the number of VPEs required for the
+	 * VM.
+	 *
+	 * Round up the number of VPEs to a whole power of two as we cannot
+	 * describe non-powers-of-two in the VMTE field as it conveys the number
+	 * of ID bits used and not the number of vPEs. IRS_IDR1.IAFFID_BITS is
+	 * encoded as N - 1, so expose at least one VPE ID bit even for a
+	 * single-vCPU VM to keep the views consistent.
+	 */
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		u16 vpe_id = vgic_v5_vpe_id(vcpu);
+
+		if (vpe_id > max_vpe_id)
+			max_vpe_id = vpe_id;
+	}
+
+	nr_cpus = max(2UL, roundup_pow_of_two(max_vpe_id + 1));
+	vmi->vpe_id_bits = fls(nr_cpus) - 1;
+
+	vpet_alloc_size = round_up((size_t)nr_cpus * sizeof(*vpet),
+				   dma_get_cache_alignment());
+	vpet = kzalloc(vpet_alloc_size, GFP_KERNEL);
+	if (!vpet) {
+		ret = -ENOMEM;
+		goto out_fail;
+	}
+
+	/* Stash the VA so we can free it later */
+	vmi->vpet_base = vpet;
+
+	tmp = FIELD_PREP(GICV5_VMTEL2E_VPET_ADDR,
+			 virt_to_phys(vpet) >> GICV5_VMTEL2E_VPET_ADDR_SHIFT);
+	tmp |= FIELD_PREP(GICV5_VMTEL2E_VPE_ID_BITS, vmi->vpe_id_bits);
+	vmte_val1 = tmp;
+
+	/*
+	 * Allocate a dense VPED array indexed by vcpu_idx. The VPET is indexed
+	 * by the potentially sparse vcpu_id, but using that ID here would waste
+	 * memory. Given that this is not userspace visible, we can cheat a bit
+	 * and use the dense index instead. This is the ONLY place that we do
+	 * this.
+	 *
+	 * Round the requested size up to a whole cacheline. kzalloc() gurantees
+	 * natural alignment, so we ensure that the cachelines cannot be shared
+	 * with unrelated slab objects. VPED and cacheline sizes are powers of
+	 * two, so this also preserves the required VPED alignment when a VPED
+	 * is larger than a cacheline.
+	 */
+	vped_alloc_size = round_up((size_t)nr_vcpus * vmt_info->vped_size,
+				   dma_get_cache_alignment());
+	vped_base = kzalloc(vped_alloc_size, GFP_KERNEL);
+	if (!vped_base) {
+		ret = -ENOMEM;
+		goto out_fail;
+	}
+	vmi->vped_base = vped_base;
+
+	if (vmd)
+		vgic_v5_clean_inval(vmd, vmd_alloc_size);
+	vgic_v5_clean_inval(vpet, vpet_alloc_size);
+	vgic_v5_clean_inval(vped_base, vped_alloc_size);
+
+	/* Publish the VMTE while serialising access to the shared VMT. */
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		WRITE_ONCE(vmte->val[0], cpu_to_le64(vmte_val0));
+		WRITE_ONCE(vmte->val[1], cpu_to_le64(vmte_val1));
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+	}
+
+	kvm->arch.vgic.gicv5_vm.vmte_allocated = true;
+
+	return 0;
+
+out_fail:
+	/* kfree(NULL) is safe so we can just kfree() at leisure */
+	kfree(vmd);
+	kfree(vpet);
+	kfree(vped_base);
+	if (vmi_inserted)
+		xa_erase(&vm_info, vm_id);
+	kfree(vmi);
+
+	vgic_v5_reset_vmte(kvm);
+
+	return ret;
+}
+
+/*
+ * Release the VMT Entry, freeing up any allocated data structures before
+ * zeroing the VMTE.
+ *
+ * The VMTE must be marked as invalid before it is released.
+ */
+int vgic_v5_vmte_release(struct kvm *kvm)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+	struct vmtl2_entry *vmte;
+	int ret;
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	/* Reject if the VMTE has not been marked as invalid! */
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+		if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID)
+			return -EINVAL;
+	}
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		goto no_vmi;
+
+	/* Unlikely, but possible. Avoid leaking the memory. */
+	vgic_v5_drain_pending_irqs(kvm, vmi, false);
+
+	/* If we have an LPI IST, free it */
+	if (vmi->h_lpi_ist) {
+		ret = vgic_v5_lpi_ist_free(kvm);
+		if (ret)
+			return ret;
+	}
+	vmi->h_lpi_ist = NULL;
+
+	/* If we have an SPI IST, free it */
+	if (vmi->h_spi_ist) {
+		ret = vgic_v5_spi_ist_free(kvm);
+		if (ret)
+			return ret;
+	}
+	vmi->h_spi_ist = NULL;
+
+	kfree(vmi->vped_base);
+	kfree(vmi->vpet_base);
+	kfree(vmi->vmd_base);
+
+	xa_erase(&vm_info, vm_id);
+	kfree(vmi);
+
+no_vmi:
+	/*
+	 * If we didn't get far enough into allocating a VMTE to create the VM
+	 * info structure, then we just zero the VMTE and move on. There's
+	 * nothing else we can realistically do here.
+	 */
+	ret = vgic_v5_reset_vmte(kvm);
+	if (ret)
+		return ret;
+
+	kvm->arch.vgic.gicv5_vm.vmte_allocated = false;
+
+	return 0;
+}
+
+/*
+ * Provide a way for the IRS MMIO emulation to correctly populate the number of
+ * IAFFID bits (which correspond to our vpe_id_bits.
+ */
+u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu)
+{
+	u32 vm_id = vgic_v5_vm_id(vcpu->kvm);
+	struct vgic_v5_vm_info *vmi;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return 0;
+
+	return vmi->vpe_id_bits;
+}
+
+/* Provide the preallocated VPE descriptor to the hardware via the VPE Table. */
+int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu)
+{
+	u32 vm_id = vgic_v5_vm_id(vcpu->kvm);
+	u16 vpe_id = vgic_v5_vpe_id(vcpu);
+	struct vgic_v5_vm_info *vmi;
+	vpe_entry *vpet_base;
+	void *vped;
+
+	/* Make sure we're not over what the hardware supports */
+	if (vpe_id >= vmt_info->max_vpes)
+		return -E2BIG;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	if (vpe_id >= 1 << vmi->vpe_id_bits)
+		return -E2BIG;
+
+	vpet_base = vmi->vpet_base;
+
+	/* If the VPETE for this CPU is already valid we've gone wrong */
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		vgic_v5_clean_inval(&vpet_base[vpe_id], sizeof(*vpet_base));
+		if (le64_to_cpu(READ_ONCE(vpet_base[vpe_id])) & GICV5_VPE_VALID)
+			return -EBUSY;
+	}
+
+	vped = (u8 *)vmi->vped_base +
+		(size_t)vcpu->vcpu_idx * vmt_info->vped_size;
+
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		phys_addr_t paddr = virt_to_phys(vped) & GICV5_VPED_ADDR;
+		WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(paddr));
+		vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry));
+	}
+
+	return 0;
+}
+
+/* Clear the VPE's table entry after the VMTE has been made invalid. */
+int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu)
+{
+	u32 vm_id = vgic_v5_vm_id(vcpu->kvm);
+	u16 vpe_id = vgic_v5_vpe_id(vcpu);
+	struct vgic_v5_vm_info *vmi;
+	struct vmtl2_entry *vmte;
+	vpe_entry *vpet_base;
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+		if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID)
+			return -EBUSY;
+	}
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	if (vpe_id >= 1 << vmi->vpe_id_bits)
+		return -E2BIG;
+
+	vpet_base = vmi->vpet_base;
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		WRITE_ONCE(vpet_base[vpe_id], 0ULL);
+		vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry));
+	}
+
+	return 0;
+}
+
+phys_addr_t vgic_v5_get_vmt_base(void)
+{
+	phys_addr_t vmt_base;
+
+	if (!vmt_info->two_level)
+		vmt_base = virt_to_phys(vmt_info->linear.vmt_base);
+	else
+		vmt_base = virt_to_phys(vmt_info->l2.vmt_base);
+
+	return vmt_base;
+}
+
+/*
+ * Assign an already allocated IST to the VM by populating the fields in the
+ * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even
+ * if the paths to reach it might be vastly different.
+ */
+static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base,
+				   bool two_level, unsigned int id_bits,
+				   unsigned int l2sz, unsigned int istsz,
+				   bool spi_ist)
+{
+	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	enum gicv5_vcpu_cmd cmd;
+	struct vmtl2_entry *vmte;
+	unsigned int section;
+	u64 tmp;
+	int ret;
+
+	/*
+	 * The L2 VMTE comprises four 64-bit "sections", where sections 2 & 3
+	 * describe the LPI and SPI ISTs, respectively. Both the LPI and SPI
+	 * sections have the same layout, and as we are either operating on SPIs
+	 * or LPIs we pick a section of the VMTE to modify up-front.
+	 *
+	 * See the GICv5 EAC0 Specification 11.2.2 for more details about the
+	 * VMTE layout.
+	 */
+	section = spi_ist ? GICV5_VMTEL2_SPI_SECTION : GICV5_VMTEL2_LPI_SECTION;
+
+	if (ist_base & ~GICV5_VMTEL2E_IST_ADDR) {
+		pr_err_ratelimited("kvm [%i]: IST misaligned: address 0x%llx, mask 0x%llx\n",
+				   task_pid_nr(current), ist_base,
+				   GICV5_VMTEL2E_IST_ADDR);
+		return -EINVAL;
+	}
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	tmp = FIELD_PREP(GICV5_VMTEL2E_IST_L2SZ, l2sz);
+	tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ADDR,
+			ist_base >> GICV5_VMTEL2E_IST_ADDR_SHIFT);
+	tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ISTSZ, istsz);
+	tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ID_BITS, id_bits);
+	if (two_level)
+		tmp |= GICV5_VMTEL2E_IST_STRUCTURE;
+
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		/* Bail if already allocated */
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+		if (le64_to_cpu(READ_ONCE(vmte->val[section])) &
+		    GICV5_VMTEL2E_IST_VALID)
+			return -EINVAL;
+
+		WRITE_ONCE(vmte->val[section], cpu_to_le64(tmp));
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+	}
+
+	/* Finally, mark the entry as valid */
+	cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID;
+	ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd);
+	if (ret) {
+		/*
+		 * A timeout does not tell us whether the IRS consumed the
+		 * command, so we're in some indeterminate and potentially
+		 * transient state.  It may still make the VIST valid and access
+		 * the backing memory, so leave both the VMTE and allocation
+		 * intact. Killing the VM ensures that teardown is the only path
+		 * that can reclaim them, after it has successfully made the
+		 * complete VMTE invalid.
+		 */
+		if (ret == -ETIMEDOUT) {
+			kvm_vm_dead(kvm);
+			return ret;
+		}
+
+		scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+			WRITE_ONCE(vmte->val[section], 0ULL);
+			vgic_v5_clean_inval(vmte, sizeof(*vmte));
+		}
+
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Allocate a Linear IST - always used for SPIs and potentially LPIs.
+ *
+ * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0
+ * spec.
+ *
+ * NOTE: istsz is the FIELD used by GICv5, not the actual size (or log2() of the
+ * size).
+ */
+static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist,
+				    unsigned int id_bits, unsigned int istsz)
+{
+	const size_t n = max(5, id_bits + 1 + istsz);
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+	__le64 *ist;
+	u32 l1sz;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	/*
+	 * Allocate the IST. We only have one level, so we just use the L2 ISTE.
+	 */
+	l1sz = BIT(n + 1);
+	ist = kzalloc(l1sz, GFP_KERNEL_ACCOUNT);
+	if (!ist)
+		return -ENOMEM;
+
+	if (spi_ist) {
+		vmi->h_spi_ist = ist;
+	} else {
+		vmi->h_lpi_ist_structure = false;
+		vmi->h_lpi_ist = ist;
+	}
+
+	vgic_v5_clean_inval(ist, l1sz);
+
+	return 0;
+}
+
+/*
+ * Allocate the first level of a two-level IST - LPI, only.
+ *
+ * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0
+ * spec.
+ *
+ * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or
+ * log2() of the sizes).
+ */
+static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits,
+				unsigned int istsz, unsigned int l2sz)
+{
+	const size_t n =  max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1);
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	const u32 l1_size = BIT(n + 1);
+	struct vgic_v5_vm_info *vmi;
+	__le64 *ist;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	ist = kzalloc(l1_size, GFP_KERNEL_ACCOUNT);
+	if (!ist)
+		return -ENOMEM;
+
+	vmi->h_lpi_ist_structure = true;
+	vmi->h_lpi_ist = ist;
+
+	vgic_v5_clean_inval(ist, l1_size);
+
+	return 0;
+}
+
+/*
+ * Allocate ALL of the second level ISTs for a two-level IST - LPI, only.
+ *
+ * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0
+ * spec. The l2_size calculation is from section 11.2.3 of the same document.
+ *
+ * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or
+ * log2() of the sizes).
+ */
+static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits,
+				 unsigned int istsz, unsigned int l2sz)
+{
+	const size_t n =  max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1);
+	const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
+	const size_t l2_size = BIT(11 + (2 * l2sz) + 1);
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+	__le64 *l2ist;
+	__le64 *l1ist;
+	int index;
+	u64 val;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	l1ist = vmi->h_lpi_ist;
+
+	/*
+	 * Allocate the storage for the pointers to the L2 ISTs (used when
+	 * freeing later).
+	 */
+	vmi->h_lpi_l2_ists = kzalloc_objs(*vmi->h_lpi_l2_ists, l1_entries,
+					  GFP_KERNEL_ACCOUNT);
+	if (!vmi->h_lpi_l2_ists)
+		return -ENOMEM;
+
+	/* Allocate the L2 IST for each L1 IST entry */
+	for (index = 0; index < l1_entries; ++index) {
+		l2ist = kzalloc(l2_size, GFP_KERNEL_ACCOUNT);
+		if (!l2ist) {
+			while (--index >= 0)
+				kfree(vmi->h_lpi_l2_ists[index]);
+
+			kfree(vmi->h_lpi_l2_ists);
+			vmi->h_lpi_l2_ists = NULL;
+
+			return -ENOMEM;
+		}
+
+		/*
+		 * We are not doing on-demand allocation of the L2 ISTs, and are
+		 * instead provisioning the whole IST up front. This means that
+		 * we are able to mark the L2 ISTs as valid in the L1 ISTEs as
+		 * the overall IST is not yet valid.
+		 */
+		val = (virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK) |
+		      GICV5_ISTL1E_VALID;
+		l1ist[index] = cpu_to_le64(val);
+
+		vmi->h_lpi_l2_ists[index] = l2ist;
+
+		vgic_v5_clean_inval(l2ist, l2_size);
+	}
+
+	/* Handle CMOs for the whole L1 IST in one go */
+	vgic_v5_clean_inval(l1ist, l1_entries * sizeof(*l1ist));
+
+	return 0;
+}
+
+/* Allocate a two-level IST - LPIs, only */
+static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, unsigned int id_bits,
+					   unsigned int istsz, unsigned int l2sz)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+	int ret;
+
+	/*
+	 * Allocate the L1 IST first, then all of the L2s. Everything
+	 * is preallocated and we do no on-demand IST allocation. This
+	 * is to avoid needing to track if and when the guest is doing
+	 * on-demand IST allocation.
+	 */
+	ret = vgic_v5_alloc_l1_ist(kvm, id_bits, istsz, l2sz);
+	if (ret)
+		return ret;
+
+	ret = vgic_v5_alloc_l2_ists(kvm, id_bits, istsz, l2sz);
+	if (ret) {
+		/* Free the L1 IST again */
+		vmi = xa_load(&vm_info, vm_id);
+		kfree(vmi->h_lpi_ist);
+		vmi->h_lpi_ist = NULL;
+
+		return ret;
+	}
+
+	return 0;
+}
+
+static void vgic_v5_free_allocated_lpi_ist(struct vgic_v5_vm_info *vmi,
+					   unsigned int id_bits,
+					   unsigned int istsz,
+					   unsigned int l2sz)
+{
+	if (!vmi->h_lpi_ist_structure) {
+		kfree(vmi->h_lpi_ist);
+		vmi->h_lpi_ist = NULL;
+		return;
+	}
+
+	if (vmi->h_lpi_l2_ists) {
+		const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1);
+		const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
+		int index;
+
+		for (index = 0; index < l1_entries; ++index)
+			kfree(vmi->h_lpi_l2_ists[index]);
+
+		kfree(vmi->h_lpi_l2_ists);
+		vmi->h_lpi_l2_ists = NULL;
+	}
+
+	kfree(vmi->h_lpi_ist);
+	vmi->h_lpi_ist = NULL;
+}
+
+static void vgic_v5_free_allocated_spi_ist(struct kvm *kvm)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return;
+
+	kfree(vmi->h_spi_ist);
+	vmi->h_spi_ist = NULL;
+}
+
+/*
+ * Free a Linear IST. Can only happen once the VM is dead.
+ */
+static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vmtl2_entry *vmte;
+	struct vgic_v5_vm_info *vmi;
+	int section;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	if (spi) {
+		section = GICV5_VMTEL2_SPI_SECTION;
+		vgic_v5_free_allocated_spi_ist(kvm);
+	} else {
+		section = GICV5_VMTEL2_LPI_SECTION;
+		vgic_v5_free_allocated_lpi_ist(vmi, 0, 0, 0);
+	}
+
+	/* The VM should be dead here, so we can just zero the VMT section */
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		WRITE_ONCE(vmte->val[section], cpu_to_le64(0));
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+	}
+
+	return 0;
+}
+
+/*
+ * Free a Two-Level IST. Can only happen once the VM is dead.
+ */
+static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi)
+{
+	unsigned int id_bits, istsz, l2sz;
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+	struct vmtl2_entry *vmte;
+	u64 tmp;
+	int section;
+
+	/* We don't create two-level SPI ISTs, so freeing is a bad idea! */
+	if (spi)
+		return -EINVAL;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	section = GICV5_VMTEL2_LPI_SECTION;
+
+	if (!vmi->h_lpi_ist_structure)
+		return -EINVAL;
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+		tmp = le64_to_cpu(READ_ONCE(vmte->val[section]));
+	}
+
+	id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp);
+	istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp);
+	l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp);
+
+	vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz);
+
+	/* The VM must be dead, so we can just zero the VMT section */
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		WRITE_ONCE(vmte->val[section], cpu_to_le64(0));
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+	}
+
+	return 0;
+}
+
+/* Helper to determine ISTE size based on metadata requirements */
+static unsigned int vgic_v5_ist_istsz(unsigned int id_bits)
+{
+	if (!vgic_v5_irs_istmd(&irs_caps))
+		return GICV5_IRS_IST_CFGR_ISTSZ_4;
+
+	if (id_bits >= vgic_v5_irs_istmd_sz(&irs_caps))
+		return GICV5_IRS_IST_CFGR_ISTSZ_16;
+
+	return GICV5_IRS_IST_CFGR_ISTSZ_8;
+}
+
+/*
+ * Allocate an IST for SPIs.
+ *
+ * We don't anticipate a large number of SPIs being allocated. Therefore, we
+ * always allocate a Linear IST for SPIs. This will need to be revisited should
+ * that assumption no longer hold.
+ */
+int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+	phys_addr_t base_addr;
+	unsigned int istsz;
+	int ret;
+
+	lockdep_assert_held(&kvm->arch.config_lock);
+
+	istsz = vgic_v5_ist_istsz(id_bits);
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	if (vmi->h_spi_ist)
+		return -EBUSY;
+
+	ret = vgic_v5_alloc_linear_ist(kvm, true, id_bits, istsz);
+	if (ret)
+		return ret;
+	base_addr = virt_to_phys(vmi->h_spi_ist);
+
+	ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz,
+				      true);
+	if (ret) {
+		if (ret != -ETIMEDOUT)
+			vgic_v5_free_allocated_spi_ist(kvm);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Free the IST for SPIs. Should only happen once the VM is dead.
+ */
+static int vgic_v5_spi_ist_free(struct kvm *kvm)
+{
+	lockdep_assert_held(&kvm->arch.config_lock);
+
+	return vgic_v5_linear_ist_free(kvm, true);
+}
+
+int vgic_v5_lpi_ist_exists(struct kvm *kvm)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -ENXIO;
+
+	return !!vmi->h_lpi_ist;
+}
+
+/*
+ * Allocate an IST for LPIs.
+ *
+ * Unlike with SPIs, we anticipate that the guest will allocate a relatively
+ * large number of LPIs. Therefore, while we support doing a linear LPI IST, it
+ * is expected that LPI ISTs will be two-level.
+ */
+int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+	unsigned int istsz, l2sz;
+	phys_addr_t phys_addr;
+	bool two_level;
+	int ret;
+
+	lockdep_assert_held(&kvm->arch.config_lock);
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -EINVAL;
+
+	if (vmi->h_lpi_ist)
+		return -EBUSY;
+
+	istsz = vgic_v5_ist_istsz(id_bits);
+	l2sz = gicv5_irs_l2_sz(vgic_v5_irs_ist_l2sz(&irs_caps));
+
+	/*
+	 * Determine if we want to create a Linear or a Two-Level IST.
+	 *
+	 * A two-level IST is only required when a single L2 IST cannot cover
+	 * the requested ID space. This depends on the L2 IST size selected for
+	 * the IRS, not PAGE_SIZE. Using PAGE_SIZE here would switch to
+	 * two-level too early when the selected L2 IST is larger than a page,
+	 * and the allocation sizing arithmetic would underflow.
+	 */
+	two_level = vgic_v5_irs_ist_levels(&irs_caps) &&
+		id_bits > ((10 - istsz) + (2 * l2sz));
+
+	if (!two_level)
+		ret = vgic_v5_alloc_linear_ist(kvm, false /* LPIs, not SPIs */,
+					       id_bits, istsz);
+	else
+		ret = vgic_v5_alloc_two_level_lpi_ist(kvm, id_bits, istsz,
+						      l2sz);
+
+	if (ret)
+		return ret;
+
+	phys_addr = virt_to_phys(vmi->h_lpi_ist);
+	ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz,
+				      istsz, false);
+	if (ret && ret != -ETIMEDOUT)
+		vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz);
+
+	return ret;
+}
+
+/* Free the LPI IST again */
+int vgic_v5_lpi_ist_free(struct kvm *kvm)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+
+	lockdep_assert_held(&kvm->arch.config_lock);
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -ENXIO;
+
+	if (!vmi->h_lpi_ist_structure)
+		return vgic_v5_linear_ist_free(kvm, false);
+	else
+		return vgic_v5_two_level_ist_free(kvm, false);
+}
+
+static struct vgic_v5_two_level_ist_shape
+vgic_v5_two_level_ist_shape(const struct vgic_v5_ist_desc *ist)
+{
+	struct vgic_v5_two_level_ist_shape shape;
+	size_t l2bits, n;
+
+	l2bits = (10 - ist->istsz) + (2 * ist->l2sz);
+	n = max(2, ist->id_bits - l2bits + 3 - 1);
+
+	shape.l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE;
+	shape.l2_entries = BIT(l2bits);
+
+	return shape;
+}
+
+static int vgic_v5_read_vm_ist_desc(struct kvm *kvm, unsigned int section,
+				    struct vgic_v5_ist_desc *ist)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vmtl2_entry *vmte;
+	u64 vmte_ist_section;
+
+	vmte = vgic_v5_get_l2_vmte(vm_id);
+	if (IS_ERR(vmte))
+		return PTR_ERR(vmte);
+
+	scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) {
+		vgic_v5_clean_inval(vmte, sizeof(*vmte));
+		vmte_ist_section = le64_to_cpu(READ_ONCE(vmte->val[section]));
+	}
+
+	ist->id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, vmte_ist_section);
+	ist->istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, vmte_ist_section);
+	ist->l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, vmte_ist_section);
+	ist->iste_size = GICV5_ISTE_SIZE(ist->istsz);
+
+	return !!(vmte_ist_section & GICV5_VMTEL2E_IST_VALID);
+}
+
+static int vgic_v5_get_spi_ist_desc(struct kvm *kvm,
+				    struct vgic_v5_ist_desc *ist)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	int ret;
+
+	memset(ist, 0, sizeof(*ist));
+
+	ist->vmi = xa_load(&vm_info, vm_id);
+	if (!ist->vmi)
+		return -ENXIO;
+
+	ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_SPI_SECTION, ist);
+	if (ret < 0)
+		return ret;
+
+	ist->base = ist->vmi->h_spi_ist;
+	if (!ret || !ist->base)
+		return -ENXIO;
+
+	ist->present = true;
+	return 0;
+}
+
+static int vgic_v5_get_lpi_ist_desc(struct kvm *kvm,
+				    struct vgic_v5_ist_desc *ist)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	bool guest_valid, host_valid;
+	int ret;
+
+	memset(ist, 0, sizeof(*ist));
+
+	ist->vmi = xa_load(&vm_info, vm_id);
+	if (WARN_ON_ONCE(!ist->vmi))
+		return -ENXIO;
+
+	ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_LPI_SECTION, ist);
+	if (ret < 0)
+		return ret;
+
+	host_valid = ret;
+	guest_valid = kvm->arch.vgic.vgic_v5_irs_data->ist_baser.valid;
+	ist->base = ist->vmi->h_lpi_ist;
+
+	/* If there is no IST to save/restore, return without error. */
+	if (!guest_valid && !host_valid && !ist->base)
+		return 0;
+
+	/* Mismatched combination of valid state */
+	if (!guest_valid || !host_valid || !ist->base)
+		return -ENXIO;
+
+	if (ist->vmi->h_lpi_ist_structure && !ist->vmi->h_lpi_l2_ists)
+		return -ENXIO;
+
+	ist->present = true;
+	return 0;
+}
+
+/*
+ * Save a linear host IST to userspace memory.
+ *
+ * Only the architected 32-bit ISTE state is stored. Metadata is skipped when
+ * striding through the host IST.
+ */
+static int vgic_v5_save_linear_ist(const struct vgic_v5_ist_desc *ist,
+				   u32 __user *uaddr, size_t nr_entries)
+{
+	size_t index;
+	int ret;
+
+	vgic_v5_clean_inval(ist->base,
+			    GICV5_LINEAR_IST_SIZE(ist->id_bits, ist->istsz));
+
+	for (index = 0; index < nr_entries; index++) {
+		__le32 *h_iste_addr = ist->base + index * ist->iste_size;
+		u32 h_iste;
+
+		h_iste = le32_to_cpu(READ_ONCE(*h_iste_addr));
+		ret = put_user(h_iste, uaddr);
+		if (ret)
+			return ret;
+
+		uaddr++;
+	}
+
+	return 0;
+}
+
+/*
+ * Save a two-level host IST to userspace memory.
+ *
+ * Only the architected 32-bit ISTE state is stored. Metadata is skipped when
+ * striding through the host IST.
+ */
+static int vgic_v5_save_two_level_ist(const struct vgic_v5_ist_desc *ist,
+				      u32 __user *uaddr)
+{
+	struct vgic_v5_two_level_ist_shape shape;
+	size_t h_l1_index, h_l2_index;
+	void *h_l2_ist_base;
+	int ret;
+
+	shape = vgic_v5_two_level_ist_shape(ist);
+
+	vgic_v5_clean_inval(ist->base,
+			    shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist));
+
+	for (h_l1_index = 0; h_l1_index < shape.l1_entries; h_l1_index++) {
+		u64 l1_iste;
+
+		/*
+		 * Host L2 ISTs are preallocated. Any invalid L1 entry means the
+		 * host IST state is inconsistent.
+		 */
+		l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index]));
+		if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste))
+			return -ENXIO;
+
+		h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index];
+		if (!h_l2_ist_base)
+			return -ENXIO;
+
+		vgic_v5_clean_inval(h_l2_ist_base,
+				    shape.l2_entries * ist->iste_size);
+
+		for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) {
+			void *h_iste_addr = h_l2_ist_base + h_l2_index * ist->iste_size;
+			u32 h_iste = le32_to_cpu(*(__le32 *)h_iste_addr);
+
+			ret = put_user(h_iste, uaddr);
+			if (ret)
+				return ret;
+
+			uaddr++;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Save the SPI IST to userspace-provided memory.
+ */
+int vgic_v5_save_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
+{
+	struct vgic_v5_ist_desc ist;
+	u32 __user *uaddr;
+	int ret;
+
+	ret = vgic_v5_get_spi_ist_desc(kvm, &ist);
+	if (ret)
+		return ret;
+
+	uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr;
+
+	/* The host SPI IST is always linear. */
+	return vgic_v5_save_linear_ist(&ist, uaddr,
+				       kvm->arch.vgic.nr_spis);
+}
+
+/*
+ * Save the LPI IST to userspace memory.
+ *
+ * The LPI IST may be linear or two-level, so host iteration depends on the
+ * allocated host shape.
+ */
+int vgic_v5_save_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
+{
+	struct vgic_v5_ist_desc ist;
+	u32 __user *uaddr;
+	int ret;
+
+	ret = vgic_v5_get_lpi_ist_desc(kvm, &ist);
+	if (ret)
+		return ret;
+
+	if (!ist.present)
+		return 0;
+
+	/*
+	 * Userspace sized the buffer from the guest-visible configuration.
+	 * Refuse to walk a host IST with a different number of entries.
+	 */
+	if (ist.id_bits !=
+	    kvm->arch.vgic.vgic_v5_irs_data->ist_cfgr.lpi_id_bits)
+		return -EINVAL;
+
+	uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr;
+
+	if (!ist.vmi->h_lpi_ist_structure)
+		return vgic_v5_save_linear_ist(&ist, uaddr,
+					       BIT(ist.id_bits));
+
+	return vgic_v5_save_two_level_ist(&ist, uaddr);
+}
+
+/*
+ * Track any SPIs and LPIs that were marked as pending at the point where the
+ * IST was restored.
+ *
+ * Restored pending state is cleared from the host ISTE and replayed with VDPEND
+ * before the VM first runs.
+ */
+static int vgic_v5_track_pending_irq(struct list_head *pending_irqs, u32 intid,
+				     u32 type)
+{
+	struct vgic_v5_pending_irq *pirq;
+
+	pirq = kzalloc_obj(*pirq, GFP_KERNEL);
+	if (!pirq)
+		return -ENOMEM;
+
+	/* Encode the interrupt as a GICv5 IntID. */
+	pirq->irq = FIELD_PREP(GICV5_HWIRQ_TYPE, type) |
+		    FIELD_PREP(GICV5_HWIRQ_ID, intid);
+
+	INIT_LIST_HEAD(&pirq->next);
+	list_add_tail(&pirq->next, pending_irqs);
+
+	return 0;
+}
+
+/*
+ * Process and sanitise each restored ISTE.
+ *
+ * HWU is for hardware use and must not survive migration. Pending state is
+ * tracked, cleared from the ISTE, and replayed before the VM first runs.
+ */
+static int vgic_v5_process_iste(__le32 *iste, struct list_head *pending_irqs,
+				u32 intid, u32 type)
+{
+	u32 iste_data = le32_to_cpu(READ_ONCE(*iste));
+	int ret;
+
+	/* Pending state is replayed later with VDPEND. */
+	if (iste_data & GICV5_ISTL2E_PENDING) {
+		ret = vgic_v5_track_pending_irq(pending_irqs, intid, type);
+		if (ret)
+			return ret;
+	}
+
+	iste_data &= ~GICV5_ISTL2E_PENDING;
+	iste_data &= ~GICV5_ISTL2E_HWU;
+
+	WRITE_ONCE(*iste, cpu_to_le32(iste_data));
+
+	return 0;
+}
+
+static void vgic_v5_restore_spi_config(struct kvm *kvm, __le32 iste, u32 spi)
+{
+	u32 iste_data = le32_to_cpu(iste);
+	bool pending = iste_data & GICV5_ISTL2E_PENDING;
+	struct vgic_irq *irq;
+	unsigned long flags;
+
+	irq = vgic_get_irq(kvm, vgic_v5_make_spi(spi));
+	if (WARN_ON_ONCE(!irq))
+		return;
+
+	raw_spin_lock_irqsave(&irq->irq_lock, flags);
+
+	if (iste_data & GICV5_ISTL2E_HM)
+		irq->config = VGIC_CONFIG_LEVEL;
+	else
+		irq->config = VGIC_CONFIG_EDGE;
+
+	if (irq->config == VGIC_CONFIG_EDGE)
+		irq->pending_latch = pending;
+	else if (pending)
+		irq->pending_latch = true;
+	else if (!irq->active)
+		irq->pending_latch = false;
+
+	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
+	vgic_put_irq(kvm, irq);
+}
+
+static int vgic_v5_restore_ist_entry(struct kvm *kvm,
+				     const struct vgic_v5_ist_desc *ist,
+				     void *h_iste_addr, __le32 h_iste,
+				     u32 intid, u32 intid_type)
+{
+	__le32 raw_iste = h_iste;
+	int ret;
+
+	/*
+	 * Sanitise the IST, clearing HWU & pending fields. Pending state is
+	 * later replayed via GIC VDPEND.
+	 */
+	ret = vgic_v5_process_iste(&h_iste, &ist->vmi->pending_irqs,
+				   intid, intid_type);
+	if (ret)
+		return ret;
+
+	if (intid_type == GICV5_HWIRQ_TYPE_SPI)
+		vgic_v5_restore_spi_config(kvm, raw_iste, intid);
+
+	/*
+	 * Zero the full ISTE (incl metadata), and write back the non-metadata
+	 * region, only.
+	 */
+	memset(h_iste_addr, 0, ist->iste_size);
+	WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste);
+	vgic_v5_clean_inval(h_iste_addr, ist->iste_size);
+
+	return 0;
+}
+
+/*
+ * Restore a userspace IST image to a linear host IST.
+ *
+ * The userspace IST image is a linear array of 32-bit ISTEs.
+ */
+static int vgic_v5_restore_linear_ist(struct kvm *kvm,
+				      const struct vgic_v5_ist_desc *ist,
+				      u32 __user *uaddr, size_t nr_entries,
+				      u32 intid_type)
+{
+	size_t index;
+	int ret;
+
+	for (index = 0; index < nr_entries; index++) {
+		void *h_iste_addr = ist->base + index * ist->iste_size;
+		u32 h_iste;
+
+		ret = get_user(h_iste, uaddr);
+		if (ret)
+			return ret;
+
+		ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr,
+						cpu_to_le32(h_iste), index, intid_type);
+		if (ret)
+			return ret;
+
+		uaddr++;
+	}
+
+	return 0;
+}
+
+/*
+ * Restore a userspace IST image to a two-level host IST.
+ *
+ * The userspace IST image is a linear array of 32-bit ISTEs.
+ */
+static int vgic_v5_restore_two_level_ist(struct kvm *kvm,
+					 const struct vgic_v5_ist_desc *ist,
+					 u32 __user *uaddr, u32 intid_type)
+{
+	struct vgic_v5_two_level_ist_shape shape;
+	size_t h_l1_index, h_l2_index;
+	void *h_l2_ist_base;
+	int ret;
+
+	shape = vgic_v5_two_level_ist_shape(ist);
+
+	vgic_v5_clean_inval(ist->vmi->h_lpi_ist,
+			    shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist));
+
+	for (h_l1_index = 0; h_l1_index < shape.l1_entries; ++h_l1_index) {
+		u64 l1_iste;
+
+		/*
+		 * Host L2 ISTs are preallocated. Any invalid L1 entry means the
+		 * host IST state is inconsistent.
+		 */
+		l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index]));
+		if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste))
+			return -ENXIO;
+
+		h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index];
+		if (!h_l2_ist_base)
+			return -ENXIO;
+
+		for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) {
+			void *h_iste_addr = h_l2_ist_base +
+					    h_l2_index * ist->iste_size;
+			u32 intid = h_l1_index * shape.l2_entries + h_l2_index;
+			u32 h_iste;
+
+			ret = get_user(h_iste, uaddr);
+			if (ret)
+				return ret;
+
+			ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr,
+							cpu_to_le32(h_iste), intid,
+							intid_type);
+			if (ret)
+				return ret;
+
+			uaddr++;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Restore the SPI IST from userspace-provided buffer to the host-allocated IST.
+ */
+int vgic_v5_restore_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
+{
+	u32 __user *uaddr;
+	struct vgic_v5_ist_desc ist;
+	int ret;
+
+	ret = vgic_v5_get_spi_ist_desc(kvm, &ist);
+	if (ret)
+		return ret;
+
+	uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr;
+
+	/* The host SPI IST is always linear. */
+	return vgic_v5_restore_linear_ist(kvm, &ist, uaddr,
+					  kvm->arch.vgic.nr_spis,
+					  GICV5_HWIRQ_TYPE_SPI);
+}
+
+/*
+ * Restore the LPI IST from userspace memory to the host-allocated LPI IST.
+ *
+ * The host LPI IST may be linear or two-level, so host iteration depends on the
+ * host IST's shape.
+ */
+int vgic_v5_restore_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr)
+{
+	u32 __user *uaddr;
+	struct vgic_v5_ist_desc ist;
+	int ret;
+
+	ret = vgic_v5_get_lpi_ist_desc(kvm, &ist);
+	if (ret)
+		return ret;
+
+	if (!ist.present)
+		return 0;
+
+	uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr;
+
+	if (!ist.vmi->h_lpi_ist_structure)
+		return vgic_v5_restore_linear_ist(kvm, &ist, uaddr,
+						  BIT(ist.id_bits),
+						  GICV5_HWIRQ_TYPE_LPI);
+
+	return vgic_v5_restore_two_level_ist(kvm, &ist, uaddr,
+					     GICV5_HWIRQ_TYPE_LPI);
+}
+
+/*
+ * Process the pending IRQs removing them from the list and optionally injecting
+ * them.
+ */
+static int vgic_v5_process_pending_irqs(struct kvm *kvm, bool inject)
+{
+	u32 vm_id = vgic_v5_vm_id(kvm);
+	struct vgic_v5_vm_info *vmi;
+
+	lockdep_assert_held(&kvm->arch.config_lock);
+
+	vmi = xa_load(&vm_info, vm_id);
+	if (!vmi)
+		return -ENXIO;
+
+	vgic_v5_drain_pending_irqs(kvm, vmi, inject);
+
+	return 0;
+}
+
+/* Replay pending state that was cleared while restoring guest IST state. */
+int vgic_v5_restore_pending_irqs(struct kvm *kvm)
+{
+	return vgic_v5_process_pending_irqs(kvm, true);
+}
+
+/* Drop pending state collected by a failed IST restore. */
+void vgic_v5_discard_pending_irqs(struct kvm *kvm)
+{
+	vgic_v5_process_pending_irqs(kvm, false);
+}
diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h
new file mode 100644
index 00000000000000..89096926563f78
--- /dev/null
+++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h
@@ -0,0 +1,140 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025, 2026 Arm Ltd.
+ */
+
+#ifndef __KVM_ARM_VGICV5_TABLES_H__
+#define __KVM_ARM_VGICV5_TABLES_H__
+
+#include <linux/spinlock.h>
+#include <linux/idr.h>
+#include <linux/irqchip/arm-gic-v5.h>
+#include <linux/list.h>
+
+/* Level 1 Virtual Machine Table Entry */
+typedef __le64 vmtl1_entry;
+
+/* Level 2 Virtual Machine Table Entry */
+struct vmtl2_entry {
+	__le64 val[4];
+};
+
+/* Virtual PE Table Entry */
+typedef __le64 vpe_entry;
+
+struct vgic_v5_vm_info {
+	void			*vmd_base;
+	vpe_entry		*vpet_base;
+	void			*vped_base;
+	u8			vpe_id_bits;
+
+	/*
+	 * Both the LPI and SPI ISTs are allocated by the hypervisor. While it
+	 * would be possible to track and access them by iterating over the ISTs
+	 * themselves, it makes more sense to store pointers to the ISTs.
+	 *
+	 * The LPI IST can either be two-level or linear. Hence, we keep track
+	 * of the structure. If it is two-level, we retain pointers to the L1
+	 * IST and to each L2 IST array. If it is linear, we just store the base
+	 * address of the IST array.
+	 *
+	 * The SPI IST is linear, and therefore we just store the base address
+	 * of the SPI IST array.
+	 */
+	bool			h_lpi_ist_structure;
+	__le64			*h_lpi_ist;
+	__le64			**h_lpi_l2_ists;
+	__le64			*h_spi_ist;
+
+	/* Tracking of pending interrupts as part of IST restore */
+	struct list_head	pending_irqs;
+};
+
+struct vgic_v5_vmt {
+	union {
+		struct {
+			struct vmtl2_entry *vmt_base;
+			unsigned int num_ents;
+		} linear;
+		struct {
+			vmtl1_entry *vmt_base;
+			struct vmtl2_entry **l2ptrs;
+			unsigned int num_l1_ents;
+		} l2;
+	};
+	bool		two_level;
+	unsigned int	num_entries;
+	unsigned int	max_vpes;
+	size_t		vmd_size;
+	size_t		vped_size;
+	struct ida	vm_id_ida;
+};
+
+static inline u32 vgic_v5_vm_id(struct kvm *kvm)
+{
+	return kvm->arch.vgic.gicv5_vm.vm_id;
+}
+
+/*
+ * For vGICv5, we need to consolidate two views:
+ *   - The vcpu ID assigned by userspace, which it can assign however it
+ *     sees fit.
+ *   - The index into the VPET, which is also presented to the guest as
+ *     the IAFFID for the VPE in question.
+ *
+ * Ideally, we don't want to intercept the guest setting an interrupt's affinity
+ * - this would somewhat defeat the point of allowing the hardware to manage the
+ * interrupt directly. Therefore, we need to use vcpu->vcpu_id as our VPE ID,
+ * which ensures that both userspace and the GICv5 hardware have a consistent
+ * view.
+ *
+ * This means two things:
+ *   - Our VPET might be sparse, depending on the IDs picked by userspace.
+ *   - We need to check the vcpu ID provided by userspace, and reject anything
+ *     we cannot handle.
+ *
+ * In order to at least ensure consistent usage in the places that we need to
+ * use the VPE ID, we provide this helper.
+ */
+static inline u16 vgic_v5_vpe_id(struct kvm_vcpu *vcpu)
+{
+	return vcpu->vcpu_id;
+}
+
+static inline int vgic_v5_vpe_db(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db;
+}
+
+extern raw_spinlock_t vgic_v5_irs_lock;
+
+int vgic_v5_vmt_allocate(unsigned int max_vpes);
+int vgic_v5_vmt_free(void);
+phys_addr_t vgic_v5_get_vmt_base(void);
+
+int vgic_v5_allocate_vm_id(struct kvm *kvm);
+void vgic_v5_release_vm_id(struct kvm *kvm);
+
+int vgic_v5_vmte_init(struct kvm *kvm);
+int vgic_v5_vmte_release(struct kvm *kvm);
+u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu);
+int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu);
+int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu);
+
+int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits);
+int vgic_v5_lpi_ist_exists(struct kvm *kvm);
+int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits);
+int vgic_v5_lpi_ist_free(struct kvm *kvm);
+
+int vgic_v5_save_spi_ist(struct kvm *kvm,
+			 struct kvm_vgic_v5_ist *ist_attr);
+int vgic_v5_save_lpi_ist(struct kvm *kvm,
+			 struct kvm_vgic_v5_ist *ist_attr);
+int vgic_v5_restore_spi_ist(struct kvm *kvm,
+			    struct kvm_vgic_v5_ist *ist_attr);
+int vgic_v5_restore_lpi_ist(struct kvm *kvm,
+			    struct kvm_vgic_v5_ist *ist_attr);
+int vgic_v5_restore_pending_irqs(struct kvm *kvm);
+void vgic_v5_discard_pending_irqs(struct kvm *kvm);
+
+#endif
diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c
index 4e3e4db24e7737..91810fbd99afa1 100644
--- a/arch/arm64/kvm/vgic/vgic-v5.c
+++ b/arch/arm64/kvm/vgic/vgic-v5.c
@@ -7,10 +7,18 @@
 
 #include <linux/bitops.h>
 #include <linux/irqchip/arm-vgic-info.h>
+#include <linux/irqdomain.h>
+#include <linux/kvm_host.h>
+#include <linux/uaccess.h>
 
+#include "vgic-v5-tables.h"
 #include "vgic.h"
 
 #define ppi_caps	kvm_vgic_global_state.vgic_v5_ppi_caps
+#define irs_caps	kvm_vgic_global_state.vgic_v5_irs_caps
+
+static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, phys_addr_t vmt_base);
+static int vgic_v5_irs_clear_vmt(void);
 
 /*
  * Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which
@@ -34,6 +42,36 @@ static void vgic_v5_get_implemented_ppis(void)
 	__assign_bit(GICV5_ARCH_PPI_PMUIRQ, ppi_caps.impl_ppi_mask, system_supports_pmuv3());
 }
 
+static u32 irs_readl_relaxed(const u32 reg_offset)
+{
+	return readl_relaxed(irs_caps.irs_base + reg_offset);
+}
+
+static void irs_writel_relaxed(const u32 val, const u32 reg_offset)
+{
+	writel_relaxed(val, irs_caps.irs_base + reg_offset);
+}
+
+static u64 irs_readq_relaxed(const u32 reg_offset)
+{
+	return readq_relaxed(irs_caps.irs_base + reg_offset);
+}
+
+static void irs_writeq_relaxed(const u64 val, const u32 reg_offset)
+{
+	writeq_relaxed(val, irs_caps.irs_base + reg_offset);
+}
+
+static void vgic_v5_irs_cache_id_regs(const struct gic_kvm_info *info)
+{
+	irs_caps.irs_base = info->gicv5_irs.base;
+	irs_caps.non_coherent = info->gicv5_irs.non_coherent;
+
+	irs_caps.idr2 = irs_readl_relaxed(GICV5_IRS_IDR2);
+	irs_caps.idr3 = irs_readl_relaxed(GICV5_IRS_IDR3);
+	irs_caps.idr4 = irs_readl_relaxed(GICV5_IRS_IDR4);
+}
+
 /*
  * Probe for a vGICv5 compatible interrupt controller, returning 0 on success.
  */
@@ -43,6 +81,8 @@ int vgic_v5_probe(const struct gic_kvm_info *info)
 	int ret;
 
 	kvm_vgic_global_state.type = VGIC_V5;
+	kvm_vgic_global_state.max_gic_vcpus = 0;
+	kvm_vgic_global_state.max_gicv5_vcpus = 0;
 
 	kvm_vgic_global_state.vcpu_base = 0;
 	kvm_vgic_global_state.vctrl_base = NULL;
@@ -60,17 +100,65 @@ int vgic_v5_probe(const struct gic_kvm_info *info)
 		goto skip_v5;
 	}
 
-	kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS;
+	if (!gicv5_global_data.lpi_domain) {
+		kvm_err("GICv5 LPI domain unavailable\n");
+		return -ENODEV;
+	}
 
+	vgic_v5_irs_cache_id_regs(info);
 	vgic_v5_get_implemented_ppis();
 
+	/*
+	 * Even if the HW supports more per-VM vCPUs, artificially cap as we
+	 * can't use them all.
+	 */
+	kvm_vgic_global_state.max_gicv5_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps),
+						    VGIC_V5_MAX_CPUS);
+
+	/*
+	 * GICv5 requires a set of tables to be allocated in order to manage
+	 * VMs. We allocate them in advance here, which alas means that we
+	 * already have to make a decisions regarding the maximum number of VMs
+	 * we want to run. For now, we match the maximum number offered by the
+	 * hardware, but this might not be a wise choice in the long term.
+	 */
+	ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gicv5_vcpus);
+	if (ret) {
+		kvm_err("Failed to allocate the GICv5 VM tables; no GICv5 support\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * We've now allocated the VM table, but the host's IRS doesn't know
+	 * about it yet. Provide the base address of the VMT to the IRS, as well
+	 * as the number of ID bits that it covers and the structure used
+	 * (linear/two-level).
+	 */
+	ret = vgic_v5_irs_assign_vmt(vgic_v5_irs_two_level_vmt_support(&irs_caps),
+				     ilog2(vgic_v5_irs_max_vms(&irs_caps)),
+				     vgic_v5_get_vmt_base());
+	if (ret) {
+		kvm_err("Failed to assign the GICv5 VM tables to the IRS; no GICv5 support\n");
+		if (!vgic_v5_irs_clear_vmt())
+			vgic_v5_vmt_free();
+		return -ENODEV;
+	}
+
 	ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5);
 	if (ret) {
 		kvm_err("Cannot register GICv5 KVM device.\n");
-		goto skip_v5;
+		/*
+		 * Don't free the VMT itself if the hardware still has a valid
+		 * pointer to it.
+		 */
+		if (!vgic_v5_irs_clear_vmt())
+			vgic_v5_vmt_free();
+		return -ENODEV;
 	}
 
 	v5_registered = true;
+	kvm_vgic_global_state.max_gic_vcpus =
+		kvm_vgic_global_state.max_gicv5_vcpus;
 	kvm_info("GCIE system register CPU interface\n");
 
 skip_v5:
@@ -92,12 +180,12 @@ int vgic_v5_probe(const struct gic_kvm_info *info)
 	ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3);
 	if (ret) {
 		kvm_err("Cannot register GICv3-legacy KVM device.\n");
-		return ret;
+		/* vGICv5 should still work */
+		return v5_registered ? 0 : ret;
 	}
 
-	/* We potentially limit the max VCPUs further than we need to here */
-	kvm_vgic_global_state.max_gic_vcpus = min(VGIC_V3_MAX_CPUS,
-						  VGIC_V5_MAX_CPUS);
+	kvm_vgic_global_state.max_gic_vcpus = max(kvm_vgic_global_state.max_gic_vcpus,
+						  VGIC_V3_MAX_CPUS);
 
 	static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
 	kvm_info("GCIE legacy system register CPU interface\n");
@@ -107,6 +195,411 @@ int vgic_v5_probe(const struct gic_kvm_info *info)
 	return 0;
 }
 
+/*
+ * Wait for completion of a change in any of IRS_VMT_BASER, IRS_VMAP_L2_VMTR,
+ * IRS_VMAP_VMR, IRS_VMAP_VPER, IRS_VMAP_VISTR, IRS_VMAP_L2_VISTR.
+ */
+static int vgic_v5_irs_wait_for_vm_op(void)
+{
+	return gicv5_wait_for_op_atomic(irs_caps.irs_base,
+					GICV5_IRS_VMT_STATUSR,
+					GICV5_IRS_VMT_STATUSR_IDLE,
+					NULL);
+}
+
+/*
+ * Wait for completion of a change in any of IRS_VPE_SELR, IRS_VPE_DBR,
+ * IRS_VPE_CR0.
+ */
+static int vgic_v5_irs_wait_for_vpe_op(void)
+{
+	return gicv5_wait_for_op_atomic(irs_caps.irs_base,
+					GICV5_IRS_VPE_STATUSR,
+					GICV5_IRS_VPE_STATUSR_IDLE,
+					NULL);
+}
+
+/*
+ * Wait for a write to IRS_SAVE_VMR to complete.
+ */
+static int vgic_v5_irs_wait_for_save_vm_op(u32 *statusr)
+{
+	return gicv5_wait_for_op_atomic(irs_caps.irs_base,
+					GICV5_IRS_SAVE_VM_STATUSR,
+					GICV5_IRS_SAVE_VM_STATUSR_IDLE,
+					statusr);
+}
+
+static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset)
+{
+	int ret;
+
+	lockdep_assert_held(&vgic_v5_irs_lock);
+
+	/* Make sure that we are idle to begin with */
+	ret = vgic_v5_irs_wait_for_vm_op();
+	if (ret)
+		return ret;
+
+	irs_writeq_relaxed(val, offset);
+
+	return vgic_v5_irs_wait_for_vm_op();
+}
+
+static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits,
+				  phys_addr_t vmt_base)
+{
+	u64 vmt_baser;
+	u32 vmt_cfgr;
+	int ret;
+
+	guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
+
+	ret = vgic_v5_irs_wait_for_vm_op();
+	if (ret)
+		return ret;
+
+	vmt_baser = irs_readq_relaxed(GICV5_IRS_VMT_BASER);
+	if (!!FIELD_GET(GICV5_IRS_VMT_BASER_VALID, vmt_baser))
+		return -EBUSY;
+
+	vmt_cfgr = FIELD_PREP(GICV5_IRS_VMT_CFGR_VM_ID_BITS, vm_id_bits);
+	if (two_level)
+		vmt_cfgr |= FIELD_PREP(GICV5_IRS_VMT_CFGR_STRUCTURE,
+				       GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL);
+
+	irs_writel_relaxed(vmt_cfgr, GICV5_IRS_VMT_CFGR);
+
+	/* The base address is intentionally only masked and not shifted */
+	vmt_baser = FIELD_PREP(GICV5_IRS_VMT_BASER_VALID, true) |
+		    (vmt_base & GICV5_IRS_VMT_BASER_ADDR);
+	irs_writeq_relaxed(vmt_baser, GICV5_IRS_VMT_BASER);
+
+	return vgic_v5_irs_wait_for_vm_op();
+}
+
+static int vgic_v5_irs_clear_vmt(void)
+{
+	guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
+
+	return vgic_v5_irs_write_vm_mmio_reg(0, GICV5_IRS_VMT_BASER);
+}
+
+static int vgic_v5_irs_vmap_l2_vmt(u16 vm_id)
+{
+	u64 val = FIELD_PREP(GICV5_IRS_VMAP_L2_VMTR_VM_ID, vm_id) |
+		GICV5_IRS_VMAP_L2_VMTR_M;
+
+	return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_L2_VMTR);
+}
+
+static int __vgic_v5_irs_vmap_vm(u16 vm_id, bool unmap)
+{
+	u64 val = FIELD_PREP(GICV5_IRS_VMAP_VMR_VM_ID, vm_id) |
+		FIELD_PREP(GICV5_IRS_VMAP_VMR_U, unmap) |
+		GICV5_IRS_VMAP_VMR_M;
+
+	return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VMR);
+}
+
+static int vgic_v5_irs_set_vm_valid(u16 vm_id)
+{
+	return __vgic_v5_irs_vmap_vm(vm_id, false);
+}
+
+static int vgic_v5_irs_set_vm_invalid(u16 vm_id)
+{
+	return __vgic_v5_irs_vmap_vm(vm_id, true);
+}
+
+static int __vgic_v5_irs_update_vist_validity(u16 vm_id, bool spi_ist, bool unmap)
+{
+	u8 type = spi_ist ? 0b011 : 0b010;
+	u64 val = FIELD_PREP(GICV5_IRS_VMAP_VISTR_TYPE, type) |
+		FIELD_PREP(GICV5_IRS_VMAP_VISTR_VM_ID, vm_id) |
+		FIELD_PREP(GICV5_IRS_VMAP_VISTR_U, unmap) |
+		GICV5_IRS_VMAP_VISTR_M;
+
+	return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VISTR);
+}
+
+static int vgic_v5_irs_set_vist_valid(u16 vm_id, bool spi_ist)
+{
+	return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, false);
+}
+
+/*
+ * LPI ISTs can be invalidated explicitly. SPI ISTs are invalidated by making
+ * the VMTE invalid during teardown.
+ */
+static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist)
+{
+	return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true);
+}
+
+static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id,
+				  irq_hw_number_t db_hwirq)
+{
+	u64 vmap_vper, dbr, selr;
+	u32 statusr, cr0;
+	int ret;
+
+	lockdep_assert_held(&vgic_v5_irs_lock);
+
+	/* Make sure that we are idle to begin with */
+	ret = vgic_v5_irs_wait_for_vm_op();
+	if (ret)
+		return ret;
+
+	/* Mark the VPE as valid */
+	vmap_vper = FIELD_PREP(GICV5_IRS_VMAP_VPER_VPE_ID, vpe_id) |
+		    FIELD_PREP(GICV5_IRS_VMAP_VPER_VM_ID, vm_id) |
+		    GICV5_IRS_VMAP_VPER_M;
+	irs_writeq_relaxed(vmap_vper, GICV5_IRS_VMAP_VPER);
+
+	/* Wait for the VPE to be marked valid in the VPET */
+	ret = vgic_v5_irs_wait_for_vm_op();
+	if (ret)
+		return ret;
+
+	selr = FIELD_PREP(GICV5_IRS_VPE_SELR_VPE_ID, vpe_id) |
+	       FIELD_PREP(GICV5_IRS_VPE_SELR_VM_ID, vm_id) |
+	       GICV5_IRS_VPE_SELR_S;
+	irs_writeq_relaxed(selr, GICV5_IRS_VPE_SELR);
+
+	ret = vgic_v5_irs_wait_for_vpe_op();
+	if (ret)
+		return ret;
+
+	statusr = irs_readl_relaxed(GICV5_IRS_VPE_STATUSR);
+	if (!FIELD_GET(GICV5_IRS_VPE_STATUSR_V, statusr))
+		return -EINVAL;
+
+	/* Set targeted only routing (disable 1ofN vPE selection) */
+	cr0 = GICV5_IRS_VPE_CR0_DPS;
+	irs_writel_relaxed(cr0, GICV5_IRS_VPE_CR0);
+
+	ret = vgic_v5_irs_wait_for_vpe_op();
+	if (ret)
+		return ret;
+
+	/*
+	 * The VPE has not yet run. Therefore, make sure that all interrupts
+	 * will generate a doorbell.
+	 */
+	dbr = FIELD_PREP(GICV5_IRS_VPE_DBR_INTID, db_hwirq) |
+	      GICV5_IRS_VPE_DBR_DBV;
+	irs_writeq_relaxed(dbr, GICV5_IRS_VPE_DBR);
+
+	ret = vgic_v5_irs_wait_for_vpe_op();
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int vgic_v5_irs_save_vm_op(u16 vm_id, bool save, u32 *statusr)
+{
+	u64 save_vmr;
+	int ret;
+
+	save_vmr = FIELD_PREP(GICV5_IRS_SAVE_VMR_VM_ID, vm_id);
+	save_vmr |= GICV5_IRS_SAVE_VMR_Q;
+	save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_S, save);
+
+	guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
+
+	/* Make sure that we are idle to begin with. */
+	ret = vgic_v5_irs_wait_for_save_vm_op(NULL);
+	if (ret)
+		return ret;
+
+	irs_writeq_relaxed(save_vmr, GICV5_IRS_SAVE_VMR);
+
+	return vgic_v5_irs_wait_for_save_vm_op(statusr);
+}
+
+static irqreturn_t db_handler(int irq, void *data)
+{
+	struct kvm_vcpu *vcpu = data;
+
+	WRITE_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired, true);
+
+	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
+	kvm_vcpu_kick(vcpu);
+
+	return IRQ_HANDLED;
+}
+
+static int vgic_v5_send_command(struct kvm_vcpu *vcpu, enum gicv5_vcpu_cmd cmd)
+{
+	int irq = vgic_v5_vpe_db(vcpu);
+
+	if (!irq)
+		return -ENXIO;
+
+	return irq_set_vcpu_affinity(irq, &cmd);
+}
+
+static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
+{
+	struct vgic_v5_vm *vm = data->domain->host_data;
+	enum gicv5_vcpu_cmd *cmd = vcpu_info;
+	unsigned int vcpu_idx = data->hwirq;
+	struct kvm_vcpu *vcpu;
+	u16 vpe_id;
+
+	guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock);
+
+	switch (*cmd) {
+	case VMT_L2_MAP:
+		return vgic_v5_irs_vmap_l2_vmt(vm->vm_id);
+	case VMTE_MAKE_VALID:
+		return vgic_v5_irs_set_vm_valid(vm->vm_id);
+	case VMTE_MAKE_INVALID:
+		return vgic_v5_irs_set_vm_invalid(vm->vm_id);
+	case VPE_MAKE_VALID:
+		/*
+		 * The index in the doorbell domain aligns with our flat
+		 * vcpu_idx index. However, we need the actual VPE ID, which
+		 * means we first need to resolve the actual vcpu.
+		 */
+		vcpu = kvm_get_vcpu(vm->kvm, vcpu_idx);
+		if (!vcpu)
+			return -EINVAL;
+
+		vpe_id = vgic_v5_vpe_id(vcpu);
+
+		/*
+		 * We need the actual LPI ID which lives in the top-most parent
+		 * domain. This hwirq won't include the type (LPI) but that's
+		 * not required for the IRS_VPE_DBR.
+		 */
+		while (data->parent_data)
+			data = data->parent_data;
+		return vgic_v5_irs_set_up_vpe(vm->vm_id, vpe_id, data->hwirq);
+	case SPI_VIST_MAKE_VALID:
+		return vgic_v5_irs_set_vist_valid(vm->vm_id, true);
+	case LPI_VIST_MAKE_VALID:
+		return vgic_v5_irs_set_vist_valid(vm->vm_id, false);
+	case LPI_VIST_MAKE_INVALID:
+		return vgic_v5_irs_set_vist_invalid(vm->vm_id, false);
+	default:
+		return -EINVAL;
+	}
+}
+
+/*
+ * This set of irq_chip functions is specific for doorbells.
+ */
+static const struct irq_chip vgic_v5_db_irq_chip = {
+	.name = "GICv5-DB",
+	.irq_mask = irq_chip_mask_parent,
+	.irq_unmask = irq_chip_unmask_parent,
+	.irq_eoi = irq_chip_eoi_parent,
+	.irq_set_affinity = irq_chip_set_affinity_parent,
+	.irq_get_irqchip_state = irq_chip_get_parent_state,
+	.irq_set_irqchip_state = irq_chip_set_parent_state,
+	.irq_set_vcpu_affinity = vgic_v5_db_set_vcpu_affinity,
+	.flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE |
+		 IRQCHIP_MASK_ON_SUSPEND,
+};
+
+static void vgic_v5_irq_db_domain_free(struct irq_domain *domain,
+				       unsigned int virq, unsigned int nr_irqs)
+{
+	int i;
+
+	for (i = 0; i < nr_irqs; i++) {
+		struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
+
+		irq_set_handler(virq + i, NULL);
+		irq_domain_reset_irq_data(d);
+	}
+
+	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+}
+
+static int vgic_v5_irq_db_domain_alloc(struct irq_domain *domain,
+				       unsigned int virq, unsigned int nr_irqs,
+				       void *arg)
+{
+	const struct irq_chip *chip = &vgic_v5_db_irq_chip;
+	struct irq_data *irqd;
+	int ret;
+
+	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL);
+	if (ret)
+		return ret;
+
+	for (int i = 0; i < nr_irqs; i++) {
+		irq_domain_set_hwirq_and_chip(domain, virq + i, i, chip,
+					      domain->host_data);
+		irqd = irq_desc_get_irq_data(irq_to_desc(virq + i));
+		irqd_set_single_target(irqd);
+	}
+
+	return 0;
+}
+
+static const struct irq_domain_ops vgic_v5_irq_db_domain_ops = {
+	.alloc = vgic_v5_irq_db_domain_alloc,
+	.free = vgic_v5_irq_db_domain_free,
+};
+
+static int vgic_v5_create_per_vm_domain(struct kvm *kvm)
+{
+	struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm;
+	int nr_vcpus = atomic_read(&kvm->online_vcpus);
+	int id = task_pid_nr(current);
+	int ret, db_virq = 0;
+
+	vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv5-vpe-db", id);
+	if (!vm->fwnode)
+		return -ENOMEM;
+
+	vm->domain = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain,
+						 0, nr_vcpus, vm->fwnode,
+						 &vgic_v5_irq_db_domain_ops, vm);
+	if (!vm->domain) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	db_virq = irq_domain_alloc_irqs(vm->domain, nr_vcpus, NUMA_NO_NODE, vm);
+	if (db_virq <= 0) {
+		ret = db_virq;
+		goto err;
+	}
+
+	kvm->arch.vgic.gicv5_vm.vpe_db_base = db_virq;
+
+	return 0;
+
+err:
+	if (vm->domain)
+		irq_domain_remove(vm->domain);
+	if (vm->fwnode)
+		irq_domain_free_fwnode(vm->fwnode);
+
+	kvm->arch.vgic.gicv5_vm.vpe_db_base = 0;
+	vm->domain = NULL;
+	vm->fwnode = NULL;
+
+	return ret;
+}
+
+static void vgic_v5_teardown_per_vm_domain(struct vgic_v5_vm *vm)
+{
+	if (!vm->domain)
+		return;
+
+	irq_domain_remove(vm->domain);
+	irq_domain_free_fwnode(vm->fwnode);
+	vm->domain = NULL;
+	vm->fwnode = NULL;
+}
+
 void vgic_v5_reset(struct kvm_vcpu *vcpu)
 {
 	/*
@@ -120,17 +613,117 @@ void vgic_v5_reset(struct kvm_vcpu *vcpu)
 	 * CPUIF (but potentially fewer in the IRS).
 	 */
 	vcpu->arch.vgic_cpu.num_pri_bits = 5;
+
+	/* Make the VPE valid in the VPET */
+	if (WARN_ON(vgic_v5_send_command(vcpu, VPE_MAKE_VALID)))
+		return;
 }
 
-int vgic_v5_init(struct kvm *kvm)
+static void vgic_v5_free_doorbells(struct kvm *kvm, unsigned int nr_dbs)
 {
+	struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm;
 	struct kvm_vcpu *vcpu;
-	unsigned long idx;
+	unsigned long i;
+	int db;
 
-	if (vgic_initialized(kvm))
-		return 0;
+	for (i = 0; i < nr_dbs; i++) {
+		vcpu = kvm_get_vcpu(kvm, i);
+		db = vgic_v5_vpe_db(vcpu);
+		if (!db)
+			continue;
+
+		free_irq(db, vcpu);
+		vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = 0;
+	}
+
+	if (vm->vpe_db_base) {
+		irq_domain_free_irqs(vm->vpe_db_base,
+				     atomic_read(&kvm->online_vcpus));
+		vm->vpe_db_base = 0;
+	}
+}
+
+void vgic_v5_teardown(struct kvm *kvm)
+{
+	struct vgic_dist *dist = &kvm->arch.vgic;
+	struct kvm_vcpu *vcpu, *vcpu0;
+	bool release_vm_id = true;
+	unsigned long i;
+	int rc;
+
+	lockdep_assert_held(&kvm->arch.config_lock);
 
-	kvm_for_each_vcpu(idx, vcpu, kvm) {
+	/*
+	 * If the VM's ID isn't valid, then we either failed init very early or
+	 * we've been called a second time. Nothing to do here in either case.
+	 */
+	if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL)
+		return;
+
+	if (kvm->arch.vgic.gicv5_vm.vmte_allocated) {
+		/* Make the VM invalid  */
+		vcpu0 = kvm_get_vcpu(kvm, 0);
+		rc = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID);
+		if (rc) {
+			kvm_err("could not make VMTE invalid\n");
+			release_vm_id = false;
+			goto out_free_doorbells;
+		}
+
+		kvm_for_each_vcpu(i, vcpu, kvm) {
+			if (vgic_v5_vmte_free_vpe(vcpu)) {
+				kvm_err("Failed to free VPE\n");
+				release_vm_id = false;
+				goto out_free_doorbells;
+			}
+		}
+
+		if (vgic_v5_vmte_release(kvm)) {
+			kvm_err("Failed to release VM 0x%x\n", dist->gicv5_vm.vm_id);
+			release_vm_id = false;
+		}
+	}
+
+out_free_doorbells:
+	vgic_v5_free_doorbells(kvm, atomic_read(&kvm->online_vcpus));
+	vgic_v5_teardown_per_vm_domain(&kvm->arch.vgic.gicv5_vm);
+
+	/*
+	 * We only release the VM ID itself if we didn't fail earlier. It does
+	 * mean that we might lose the VM ID (and associated VMTE, etc), but
+	 * given that we've failed to tear them down correctly there's no way to
+	 * safely reuse them. The VM ID allocating IDA will make sure we don't
+	 * accidentally reuse this partially torn down state.
+	 */
+	if (release_vm_id)
+		vgic_v5_release_vm_id(kvm);
+}
+
+/*
+ * Claim and populate a VMTE (optionally making a new L2 VMT valid), create VPE
+ * doorbells, allocate VPET and populate for each VPE. Finally, we also init the
+ * vIRS, which means allocating and making the virtual SPI IST valid.
+ *
+ * Note: We do need to put the cart before the horse here. The VPE doorbells are
+ * our conduit for communication with the IRS, which means we need to have those
+ * before making the VMTE valid.
+ *
+ * On failure, we clean up in the teardown path (vgic_v5_teardown()).
+ */
+int vgic_v5_init(struct kvm *kvm)
+{
+	struct kvm_vcpu *vcpu, *vcpu0;
+	int nr_vcpus, ret = 0;
+	unsigned int db_virq;
+	unsigned long i;
+
+	lockdep_assert_held(&kvm->arch.config_lock);
+
+	nr_vcpus = atomic_read(&kvm->online_vcpus);
+	if (nr_vcpus == 0)
+		return -ENODEV;
+
+	kvm_for_each_vcpu(i, vcpu, kvm) {
 		if (vcpu_has_nv(vcpu)) {
 			kvm_err("Nested GICv5 VMs are currently unsupported\n");
 			return -EINVAL;
@@ -145,20 +738,84 @@ int vgic_v5_init(struct kvm *kvm)
 		   kvm->arch.vgic.gicv5_vm.userspace_ppis,
 		   ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
 
+	ret = vgic_v5_allocate_vm_id(kvm);
+	if (ret)
+		return ret;
+
+	/*
+	 * Stash a backpointer to struct kvm. It is required to resolve the VPE
+	 * ID from the doorbell index, which matches the flat vcpu_idx and not
+	 * the vcpu_id.
+	 */
+	kvm->arch.vgic.gicv5_vm.kvm = kvm;
+
+	ret = vgic_v5_create_per_vm_domain(kvm);
+	if (ret)
+		goto err;
+
+	db_virq = kvm->arch.vgic.gicv5_vm.vpe_db_base;
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		ret = request_irq(db_virq + i, db_handler, 0, "vcpu", vcpu);
+		if (ret)
+			goto err;
+
+		/* Stash it with the VCPU for easy retrieval */
+		vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = db_virq + i;
+	}
+
+	/* Populate VMTE (with VPET and VM descriptor) */
+	ret = vgic_v5_vmte_init(kvm);
+	if (ret)
+		goto err;
+
+	/* We pick the first vcpu to make the VMTE valid - any would do */
+	vcpu0 = kvm_get_vcpu(kvm, 0);
+	ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID);
+	if (ret)
+		goto err;
+
+	/* Populate the VPETE for each VPE. */
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		ret = vgic_v5_vmte_alloc_vpe(vcpu);
+		if (ret)
+			goto err;
+	}
+
+	/* Init IRS (and alloc SPI IST) */
+	ret = kvm_vgic_v5_irs_init(kvm, kvm->arch.vgic.nr_spis);
+	if (ret)
+		goto err;
+
 	return 0;
+
+err:
+	/*
+	 * Explicitly tear everything down on failure. The teardown function is
+	 * written to handle any partial state we might have, so we don't need
+	 * to do any clean-up first. Teardown will be called a second time on VM
+	 * destruction, but that's fine - it is better to leave things in a
+	 * clean state now, and doubly so because userspace could actually go
+	 * and retry init.
+	 */
+	vgic_v5_teardown(kvm);
+
+	return ret;
 }
 
 int vgic_v5_map_resources(struct kvm *kvm)
 {
 	if (!vgic_initialized(kvm))
 		return -EBUSY;
+	if (kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending)
+		return -EINVAL;
 
 	return 0;
 }
 
-int vgic_v5_finalize_ppi_state(struct kvm *kvm)
+int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu)
 {
-	struct kvm_vcpu *vcpu0;
+	struct kvm *kvm	= vcpu->kvm;
+	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
 	int i;
 
 	if (!vgic_is_v5(kvm))
@@ -167,35 +824,65 @@ int vgic_v5_finalize_ppi_state(struct kvm *kvm)
 	guard(mutex)(&kvm->arch.config_lock);
 
 	/*
-	 * If SW_PPI has been advertised, then we know we already
-	 * initialised the whole thing, and we can return early. Yes,
-	 * this is pretty hackish as far as state tracking goes...
+	 * Discover the set of PPIs that are exposed to the guest once per VM.
+	 * Once known, apply that mask to each VCPU's restored PPI state as the
+	 * VCPUs are first run.
 	 */
-	if (test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask))
-		return 0;
-
-	/* The PPI state for all VCPUs should be the same. Pick the first. */
-	vcpu0 = kvm_get_vcpu(kvm, 0);
+	if (!test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) {
+		bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask,
+			    VGIC_V5_NR_PRIVATE_IRQS);
+		bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr,
+			    VGIC_V5_NR_PRIVATE_IRQS);
+
+		for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) {
+			const u32 intid = vgic_v5_make_ppi(i);
+			struct vgic_irq *irq;
+
+			irq = vgic_get_vcpu_irq(vcpu, intid);
+
+			/* Expose PPIs with an owner or the SW_PPI, only */
+			scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+				if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) {
+					__set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask);
+					__assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr,
+						     irq->config == VGIC_CONFIG_LEVEL);
+				}
+			}
 
-	bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
-	bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS);
+			vgic_put_irq(kvm, irq);
+		}
+	}
 
-	for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) {
+	/*
+	 * Apply the mask to Enable, Active. Skip pending as that's calculated
+	 * on guest entry.
+	 */
+	bitmap_and(cpu_if->vgic_ppi_enabler, cpu_if->vgic_ppi_enabler,
+		   kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
+	bitmap_and(cpu_if->vgic_ppi_activer, cpu_if->vgic_ppi_activer,
+		   kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
+
+	/* Also update the vgic_irqs */
+	for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) {
+		bool visible = test_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask);
 		const u32 intid = vgic_v5_make_ppi(i);
 		struct vgic_irq *irq;
 
-		irq = vgic_get_vcpu_irq(vcpu0, intid);
+		irq = vgic_get_vcpu_irq(vcpu, intid);
 
-		/* Expose PPIs with an owner or the SW_PPI, only */
 		scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
-			if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) {
-				__set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask);
-				__assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr,
-					     irq->config == VGIC_CONFIG_LEVEL);
+			if (!visible) {
+				irq->enabled = false;
+				irq->active = false;
+				irq->pending_latch = false;
+				irq->line_level = false;
+			} else {
+				irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler);
+				irq->active = test_bit(i, cpu_if->vgic_ppi_activer);
 			}
 		}
 
-		vgic_put_irq(vcpu0->kvm, irq);
+		vgic_put_irq(kvm, irq);
 	}
 
 	return 0;
@@ -436,9 +1123,53 @@ void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu)
 		    VGIC_V5_NR_PRIVATE_IRQS);
 }
 
+static bool vgic_v5_set_spi_pending_state(struct kvm_vcpu *vcpu,
+					  struct vgic_irq *irq)
+{
+	vgic_v5_set_irq_pend(irq->target_vcpu, irq);
+	return true;
+}
+
+bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm,
+				  struct vgic_irq *irq,
+				  unsigned long flags)
+	__releases(&irq->irq_lock)
+{
+	lockdep_assert_held(&irq->irq_lock);
+
+	raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
+	return true;
+}
+
+static const struct irq_ops vgic_v5_spi_irq_ops = {
+	.set_pending_state = vgic_v5_set_spi_pending_state,
+	.queue_irq_unlock = vgic_v5_spi_queue_irq_unlock,
+};
+
+void vgic_v5_set_spi_ops(struct vgic_irq *irq)
+{
+	if (WARN_ON(!irq) || WARN_ON(irq->ops))
+		return;
+
+	irq->ops = &vgic_v5_spi_irq_ops;
+}
+
+/* Set the pending state for GICv5 SPIs and LPIs */
+void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq)
+{
+	if (WARN_ON(__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, irq->intid)))
+		return;
+
+	kvm_call_hyp(__vgic_v5_vdpend, irq->intid, irq_is_pending(irq),
+		     vcpu->kvm->arch.vgic.gicv5_vm.vm_id);
+}
+
 void vgic_v5_load(struct kvm_vcpu *vcpu)
 {
+	bool irichppidis = !READ_ONCE(vcpu->kvm->arch.vgic.enabled);
 	struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+	u16 vm = vgic_v5_vm_id(vcpu->kvm);
+	u16 vpe = vgic_v5_vpe_id(vcpu);
 
 	/*
 	 * On the WFI path, vgic_load is called a second time. The first is when
@@ -451,7 +1182,16 @@ void vgic_v5_load(struct kvm_vcpu *vcpu)
 
 	kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if);
 
-	cpu_if->gicv5_vpe.resident = true;
+	cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_V, true) |
+				FIELD_PREP(ICH_CONTEXTR_EL2_IRICHPPIDIS, irichppidis) |
+				FIELD_PREP(ICH_CONTEXTR_EL2_VPE, vpe) |
+				FIELD_PREP(ICH_CONTEXTR_EL2_VM, vm);
+
+	kvm_call_hyp(__vgic_v5_make_resident, cpu_if);
+
+	/* Failed to make the VPE resident? Bang! */
+	if (WARN_ON(!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr)))
+		kvm_vm_dead(vcpu->kvm);
 }
 
 void vgic_v5_put(struct kvm_vcpu *vcpu)
@@ -469,7 +1209,58 @@ void vgic_v5_put(struct kvm_vcpu *vcpu)
 
 	kvm_call_hyp(__vgic_v5_save_apr, cpu_if);
 
-	cpu_if->gicv5_vpe.resident = false;
+	cpu_if->vgic_contextr = 0;
+
+	/* Request a doorbell if entering WFI, unless the IRS is disabled */
+	if (vcpu_get_flag(vcpu, IN_WFI) &&
+	    READ_ONCE(vcpu->kvm->arch.vgic.enabled)) {
+		u32 priority_mask;
+		int dbpm;
+
+		/*
+		 * Find the virtual running priority and use this to calculate
+		 * the doorbell priority mask. We combine the highest active
+		 * priority and the CPU's priority mask. The guest can't handle
+		 * interrupts with priorities less than or equal to the virtual
+		 * running priority, so there's literally no point in waking the
+		 * guest for these.
+		 *
+		 * The priority needs to be higher than the mask to signal, so
+		 * pick the next higher priority (subtract 1).
+		 */
+		priority_mask = vgic_v5_get_effective_priority_mask(vcpu);
+
+		/*
+		 * Request a doorbell *unless* the priority is 0, indicating
+		 * that no interrupt can wake the CPU up.
+		 */
+		if (priority_mask) {
+			int db_irq = vgic_v5_vpe_db(vcpu);
+			struct irq_data *d = irq_get_irq_data(db_irq);
+			const struct cpumask *aff = irq_data_get_effective_affinity_mask(d);
+			int cpu = smp_processor_id();
+
+			dbpm = priority_mask - 1;
+			cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_DB, 1) |
+						FIELD_PREP(ICH_CONTEXTR_EL2_DBPM, dbpm);
+
+			/*
+			 * Make the doorbell affine to this CPU, if it isn't
+			 * already. Actively check the cpumask first as it is
+			 * cheaper than changing the affinity every time.
+			 */
+			if (!cpumask_test_cpu(cpu, aff)) {
+				int ret;
+
+				ret = irq_set_affinity(db_irq, cpumask_of(cpu));
+				if (ret)
+					kvm_debug_ratelimited("Failed to move doorbell IRQ %d to CPU %d: %d\n",
+							      db_irq, cpu, ret);
+			}
+		}
+	}
+
+	kvm_call_hyp(__vgic_v5_make_non_resident, cpu_if);
 
 	/* The shadow priority is only updated on entering WFI */
 	if (vcpu_get_flag(vcpu, IN_WFI))
@@ -513,3 +1304,300 @@ void vgic_v5_save_state(struct kvm_vcpu *vcpu)
 	__vgic_v5_save_ppi_state(cpu_if);
 	dsb(sy);
 }
+
+static int vgic_v5_irs_status_is_quiesced(u32 statusr)
+{
+	if (statusr & GICV5_IRS_SAVE_VM_STATUSR_Q)
+		return 0;
+
+	return -EBUSY;
+}
+
+static int vgic_v5_irs_is_quiesced(u16 vm_id)
+{
+	u32 statusr;
+	int ret;
+
+	/*
+	 * Ensure that the IST snapshot is complete before asking the IRS
+	 * whether the VM remained quiescent while it was copied.
+	 */
+	mb();
+
+	ret = vgic_v5_irs_save_vm_op(vm_id, false, &statusr);
+	if (ret)
+		return ret;
+
+	return vgic_v5_irs_status_is_quiesced(statusr);
+}
+
+static int vgic_v5_copy_ist_attr(struct kvm_device_attr *attr,
+				 struct kvm_vgic_v5_ist *ist_attr)
+{
+	void __user *uaddr = (void __user *)(unsigned long)attr->addr;
+
+	if (!uaddr)
+		return -EINVAL;
+
+	if (copy_from_user(ist_attr, uaddr, sizeof(*ist_attr)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int vgic_v5_validate_ist_user_buffer(__u64 addr, __u64 size,
+					    size_t expected)
+{
+	if (!addr || size != expected)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int vgic_v5_validate_ist_attr(struct kvm *kvm,
+				     const struct kvm_vgic_v5_ist *ist_attr)
+{
+	unsigned int id_bits;
+	int ret;
+
+	/* We always have SPIs to save */
+	ret = vgic_v5_validate_ist_user_buffer(ist_attr->spi_ist_addr,
+					ist_attr->spi_ist_size,
+					kvm->arch.vgic.nr_spis * sizeof(__u32));
+	if (ret)
+		return ret;
+
+	/* We don't always have LPIs to save */
+	ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits);
+	if (ret < 0)
+		return ret;
+
+	/* No LPI IST */
+	if (!ret) {
+		if (ist_attr->lpi_ist_addr || ist_attr->lpi_ist_size)
+			return -EINVAL;
+
+		return 0;
+	}
+
+	return vgic_v5_validate_ist_user_buffer(ist_attr->lpi_ist_addr,
+					       ist_attr->lpi_ist_size,
+					       BIT(id_bits) * sizeof(__u32));
+}
+
+int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+	struct kvm_vgic_v5_ist ist_attr;
+	u16 vm_id = vgic_v5_vm_id(kvm);
+	u32 statusr;
+	int ret = 0;
+
+	mutex_lock(&kvm->lock);
+
+	if (kvm_trylock_all_vcpus(kvm)) {
+		mutex_unlock(&kvm->lock);
+		return -EBUSY;
+	}
+
+	mutex_lock(&kvm->arch.config_lock);
+
+	if (!vgic_initialized(kvm)) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	ret = vgic_v5_copy_ist_attr(attr, &ist_attr);
+	if (ret)
+		goto out_unlock;
+
+	ret = vgic_v5_validate_ist_attr(kvm, &ist_attr);
+	if (ret)
+		goto out_unlock;
+
+	ret = vgic_v5_irs_save_vm_op(vm_id, true, &statusr);
+	if (ret) {
+		kvm_err("Failed to save GICv5 IRS VM state: %d\n", ret);
+		goto out_unlock;
+	}
+
+	ret = vgic_v5_irs_status_is_quiesced(statusr);
+	if (ret)
+		goto out_unlock;
+
+	/* Save the SPI IST to the userspace buffer. */
+	ret = vgic_v5_save_spi_ist(kvm, &ist_attr);
+	if (ret)
+		goto out_unlock;
+
+	ret = vgic_v5_irs_is_quiesced(vm_id);
+	if (ret)
+		goto out_unlock;
+
+	/* Save the LPI IST to the userspace buffer. */
+	ret = vgic_v5_save_lpi_ist(kvm, &ist_attr);
+	if (ret)
+		goto out_unlock;
+
+	ret = vgic_v5_irs_is_quiesced(vm_id);
+	if (ret)
+		goto out_unlock;
+
+out_unlock:
+	mutex_unlock(&kvm->arch.config_lock);
+	kvm_unlock_all_vcpus(kvm);
+	mutex_unlock(&kvm->lock);
+
+	return ret;
+}
+
+/* Allocate the LPI IST to restore into */
+static int vgic_v5_restore_lpi_ist_alloc(struct kvm *kvm, bool *allocated)
+{
+	unsigned int id_bits;
+	int ret;
+
+	*allocated = false;
+
+	ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits);
+	if (ret <= 0)
+		return ret;
+
+	ret = vgic_v5_lpi_ist_alloc(kvm, id_bits);
+	if (ret)
+		return ret;
+
+	*allocated = true;
+
+	return 0;
+}
+
+/*
+ * Clean up the LPI IST if we allocated it, and restore the VMTE to the
+ * original, valid state.
+ */
+static void vgic_v5_restore_cleanup(struct kvm *kvm,
+				    struct kvm_vcpu *vcpu,
+				    bool lpi_ist_allocated)
+{
+	/*
+	 * We are on the restore failure path, so we do a best-effort
+	 * cleanup. These commands might fail, but at this stage this is the
+	 * best we can realistically do.
+	 */
+	if (lpi_ist_allocated) {
+		if (!vgic_v5_send_command(vcpu, VMTE_MAKE_INVALID))
+			vgic_v5_lpi_ist_free(kvm);
+	}
+
+	vgic_v5_send_command(vcpu, VMTE_MAKE_VALID);
+}
+
+int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr)
+{
+	bool lpi_ist_allocated = false, vmte_invalid = false;
+	struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
+	struct kvm_vgic_v5_ist ist_attr;
+	int ret = 0;
+
+	mutex_lock(&kvm->lock);
+
+	if (kvm_trylock_all_vcpus(kvm)) {
+		mutex_unlock(&kvm->lock);
+		return -EBUSY;
+	}
+
+	mutex_lock(&kvm->arch.config_lock);
+
+	if (!vgic_initialized(kvm)) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	if (kvm_vm_has_ran_once(kvm)) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	ret = vgic_v5_copy_ist_attr(attr, &ist_attr);
+	if (ret)
+		goto out_unlock;
+
+	ret = vgic_v5_validate_ist_attr(kvm, &ist_attr);
+	if (ret)
+		goto out_unlock;
+
+	ret = vgic_v5_lpi_ist_exists(kvm);
+	if (ret) {
+		if (ret > 0)
+			ret = -EBUSY;
+		goto out_unlock;
+	}
+
+	/*
+	 * If the guest has previously allocated an IST (which we check based on
+	 * the IRS_IST_BASER), extract the number of LPI ID bits from the
+	 * IRS_IST_CFGR. Else, do nothing.
+	 *
+	 * We do this before making the VMTE invalid as we rely on
+	 * IRS_VMAP_VISTR to mark the IST as valid in the VMTE. This can only
+	 * happen while the VMTE is valid.
+	 */
+	ret = vgic_v5_restore_lpi_ist_alloc(kvm, &lpi_ist_allocated);
+	if (ret)
+		goto out_unlock;
+
+	/*
+	 * Host ISTs are updated while the VMTE is invalid, so the GIC cannot
+	 * observe partially restored state.
+	 */
+	ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID);
+	if (ret) {
+		/*
+		 * If invalidation fails, the restore cannot safely update host
+		 * IST state.
+		 */
+		goto out_unlock;
+	}
+	vmte_invalid = true;
+
+	/* Restore the SPI IST from the userspace buffer. */
+	ret = vgic_v5_restore_spi_ist(kvm, &ist_attr);
+	if (ret)
+		goto out_unlock;
+
+	/* Restore the LPI IST from the userspace buffer. */
+	if (lpi_ist_allocated) {
+		ret = vgic_v5_restore_lpi_ist(kvm, &ist_attr);
+		if (ret)
+			goto out_unlock;
+	}
+
+	/* And make the VM Valid again */
+	ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID);
+	if (ret)
+		goto out_unlock;
+	vmte_invalid = false;
+
+	/*
+	 * As part of restoring the ISTs, and previously pending interrupts have
+	 * been tracked and made non-pending. Now that the ISTs have been
+	 * restored, and the VM is valid again, restore the pending interrupts.
+	 */
+	ret = vgic_v5_restore_pending_irqs(kvm);
+	if (ret)
+		goto out_unlock;
+
+	kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending = false;
+
+out_unlock:
+	if (ret && (vmte_invalid || lpi_ist_allocated)) {
+		vgic_v5_discard_pending_irqs(kvm);
+		vgic_v5_restore_cleanup(kvm, vcpu0, lpi_ist_allocated);
+	}
+
+	mutex_unlock(&kvm->arch.config_lock);
+	kvm_unlock_all_vcpus(kvm);
+	mutex_unlock(&kvm->lock);
+
+	return ret;
+}
diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c
index b25303d9919fd2..c2726d4c06e6e8 100644
--- a/arch/arm64/kvm/vgic/vgic.c
+++ b/arch/arm64/kvm/vgic/vgic.c
@@ -86,20 +86,32 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
  */
 struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid)
 {
-	/* Non-private IRQs are not yet implemented for GICv5 */
-	if (vgic_is_v5(kvm))
-		return NULL;
+	enum kvm_device_type type = kvm->arch.vgic.vgic_model;
 
 	/* SPIs */
-	if (intid >= VGIC_NR_PRIVATE_IRQS &&
-	    intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) {
-		intid -= VGIC_NR_PRIVATE_IRQS;
-		intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis);
-		return &kvm->arch.vgic.spis[intid];
+	if (__irq_is_spi(type, intid)) {
+		switch (type) {
+		case KVM_DEV_TYPE_ARM_VGIC_V5:
+			intid = vgic_v5_get_hwirq_id(intid);
+
+			if (intid >= kvm->arch.vgic.nr_spis)
+				return NULL;
+
+			intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis);
+			return &kvm->arch.vgic.spis[intid];
+		default: {
+			u32 max_intid = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
+
+			if (intid < max_intid) {
+				intid -= VGIC_NR_PRIVATE_IRQS;
+				intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis);
+				return &kvm->arch.vgic.spis[intid];
+			}
+		}}
 	}
 
 	/* LPIs */
-	if (irq_is_lpi(kvm, intid))
+	if (__irq_is_lpi(type, intid))
 		return vgic_get_lpi(kvm, intid);
 
 	return NULL;
@@ -565,6 +577,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
 	else
 		irq->pending_latch = true;
 
+	if (irq->ops && irq->ops->set_pending_state)
+		WARN_ON_ONCE(!irq->ops->set_pending_state(vcpu, irq));
+
 	vgic_queue_irq_unlock(kvm, irq, flags);
 	vgic_put_irq(kvm, irq);
 
@@ -853,6 +868,17 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
 		goto retry;
 	}
 
+	/*
+	 * Fix the last_lr_irq refcount which was obtained while
+	 * populating the LRs. This can also result in the LPI being
+	 * deleted.
+	 */
+	irq = *host_data_ptr(last_lr_irq);
+	if (irq) {
+		deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq);
+		*host_data_ptr(last_lr_irq) = NULL;
+	}
+
 	raw_spin_unlock(&vgic_cpu->ap_list_lock);
 
 	if (unlikely(deleted_lpis))
@@ -866,9 +892,6 @@ static void vgic_fold_state(struct kvm_vcpu *vcpu)
 		return;
 	}
 
-	if (!*host_data_ptr(last_lr_irq))
-		return;
-
 	if (kvm_vgic_global_state.type == VGIC_V2)
 		vgic_v2_fold_lr_state(vcpu);
 	else
@@ -1021,11 +1044,14 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
 		scoped_guard(raw_spinlock,  &irq->irq_lock) {
 			if (likely(vgic_target_oracle(irq) == vcpu)) {
 				vgic_populate_lr(vcpu, irq, count++);
-				*host_data_ptr(last_lr_irq) = irq;
+				if (count == kvm_vgic_global_state.nr_lr) {
+					vgic_get_irq_ref(irq);
+					*host_data_ptr(last_lr_irq) = irq;
+				}
 			}
 		}
 
-		if (count == kvm_vgic_global_state.nr_lr)
+		if (*host_data_ptr(last_lr_irq))
 			break;
 	}
 
@@ -1236,8 +1262,12 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
 	unsigned long flags;
 	struct vgic_vmcr vmcr;
 
-	if (vgic_is_v5(vcpu->kvm))
+	if (vgic_is_v5(vcpu->kvm)) {
+		if (READ_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired))
+			return true;
+
 		return vgic_v5_has_pending_ppi(vcpu);
+	}
 
 	if (!vcpu->kvm->arch.vgic.enabled)
 		return false;
diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h
index b71d486ae51459..97fc3bdf520e5b 100644
--- a/arch/arm64/kvm/vgic/vgic.h
+++ b/arch/arm64/kvm/vgic/vgic.h
@@ -269,6 +269,8 @@ struct ap_list_summary {
 #define irqs_active_outside_lrs(s)		\
 	((s)->nr_act &&	irqs_outside_lrs(s))
 
+int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
+		       struct vgic_reg_attr *reg_attr);
 int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
 		       struct vgic_reg_attr *reg_attr);
 int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
@@ -383,8 +385,14 @@ void vgic_debug_destroy(struct kvm *kvm);
 int vgic_v5_probe(const struct gic_kvm_info *info);
 void vgic_v5_reset(struct kvm_vcpu *vcpu);
 int vgic_v5_init(struct kvm *kvm);
+int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis);
+void vgic_v5_teardown(struct kvm *kvm);
 int vgic_v5_map_resources(struct kvm *kvm);
 void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid);
+void vgic_v5_set_spi_ops(struct vgic_irq *irq);
+bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
+				  unsigned long flags);
+void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq);
 bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu);
 void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu);
 void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu);
@@ -394,6 +402,19 @@ void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
 void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
 void vgic_v5_restore_state(struct kvm_vcpu *vcpu);
 void vgic_v5_save_state(struct kvm_vcpu *vcpu);
+int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address);
+int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits);
+
+int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
+				struct kvm_device_attr *attr, bool is_write);
+int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
+const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz);
+int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr);
+int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr);
+int vgic_v5_irs_attr_regs_access(struct kvm_device *dev,
+				 struct kvm_device_attr *attr,
+				 u64 *reg, bool is_write);
+int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr);
 
 #define for_each_visible_v5_ppi(__i, __k)		\
 	for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS)
diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c
index 2432683e48a61f..75db718e5b6a71 100644
--- a/arch/arm64/lib/csum.c
+++ b/arch/arm64/lib/csum.c
@@ -47,11 +47,7 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
 	 */
 	shift = offset * 8;
 	data = *ptr++;
-#ifdef __LITTLE_ENDIAN
 	data = (data >> shift) << shift;
-#else
-	data = (data << shift) >> shift;
-#endif
 
 	/*
 	 * Body: straightforward aligned loads from here on (the paired loads
@@ -94,13 +90,8 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
 		len -= 16;
 		ptr += 2;
 
-#ifdef __LITTLE_ENDIAN
 		data = tmp >> 64;
 		sum64 = accumulate(sum64, tmp);
-#else
-		data = tmp;
-		sum64 = accumulate(sum64, tmp >> 64);
-#endif
 	}
 	if (len > 0) {
 		sum64 = accumulate(sum64, data);
@@ -112,11 +103,7 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
 	 * preserving odd/even alignment.
 	 */
 	shift = len * -8;
-#ifdef __LITTLE_ENDIAN
 	data = (data << shift) >> shift;
-#else
-	data = (data >> shift) << shift;
-#endif
 	sum64 = accumulate(sum64, data);
 
 	/* Finally, folding */
@@ -140,11 +127,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
 	dst = *(const __uint128_t *)daddr->s6_addr;
 
 	sum += (__force u32)htonl(len);
-#ifdef __LITTLE_ENDIAN
 	sum += (u32)proto << 24;
-#else
-	sum += proto;
-#endif
 	src += (src >> 64) | (src << 64);
 	dst += (dst >> 64) | (dst << 64);
 
diff --git a/arch/arm64/lib/memchr.S b/arch/arm64/lib/memchr.S
index 37a9f2a4f7f4b5..b14b27cb841953 100644
--- a/arch/arm64/lib/memchr.S
+++ b/arch/arm64/lib/memchr.S
@@ -63,7 +63,7 @@ L(byte_loop):
 	sub	srcin, srcin, #1
 	ret
 L(found_word):
-CPU_LE(	rev	tmp, tmp)
+	rev	tmp, tmp
 	clz	tmp, tmp
 	sub	tmp, tmp, #64
 	add	result, srcin, tmp, asr #3
diff --git a/arch/arm64/lib/memcmp.S b/arch/arm64/lib/memcmp.S
index a5ccf2c55f9119..1f528ce046e495 100644
--- a/arch/arm64/lib/memcmp.S
+++ b/arch/arm64/lib/memcmp.S
@@ -103,10 +103,8 @@ L(last_bytes):
 
 	/* Compare data bytes and set return value to 0, -1 or 1.  */
 L(return):
-#ifndef __AARCH64EB__
 	rev	data1, data1
 	rev	data2, data2
-#endif
 	cmp	data1, data2
 L(ret_eq):
 	cset	result, ne
diff --git a/arch/arm64/lib/strcmp.S b/arch/arm64/lib/strcmp.S
index 9b89b453360746..2389b47dca2277 100644
--- a/arch/arm64/lib/strcmp.S
+++ b/arch/arm64/lib/strcmp.S
@@ -38,14 +38,6 @@
 #define shift		x9
 #define off2		x10
 
-/* On big-endian early bytes are at MSB and on little-endian LSB.
-   LS_FW means shifting towards early bytes.  */
-#ifdef __AARCH64EB__
-# define LS_FW lsl
-#else
-# define LS_FW lsr
-#endif
-
 /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
    (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
    can be done in parallel across the entire word.
@@ -67,28 +59,17 @@ L(loop_aligned):
 	ldr	data2, [src1, off2]
 	ldr	data1, [src1], 8
 L(start_realigned):
-#ifdef __AARCH64EB__
-	rev	tmp, data1
-	sub	has_nul, tmp, zeroones
-	orr	tmp, tmp, REP8_7f
-#else
 	sub	has_nul, data1, zeroones
 	orr	tmp, data1, REP8_7f
-#endif
 	bics	has_nul, has_nul, tmp	/* Non-zero if NUL terminator.  */
 	ccmp	data1, data2, 0, eq
 	b.eq	L(loop_aligned)
-#ifdef __AARCH64EB__
-	rev	has_nul, has_nul
-#endif
 	eor	diff, data1, data2
 	orr	syndrome, diff, has_nul
 L(end):
-#ifndef __AARCH64EB__
 	rev	syndrome, syndrome
 	rev	data1, data1
 	rev	data2, data2
-#endif
 	clz	shift, syndrome
 	/* The most-significant-non-zero bit of the syndrome marks either the
 	   first bit that is different, or the top bit of the first zero byte.
@@ -113,7 +94,7 @@ L(mutual_align):
 	ldr	data1, [src1], 8
 	neg	shift, src2, lsl 3	/* Bits to alignment -64.  */
 	mov	tmp, -1
-	LS_FW	tmp, tmp, shift
+	lsr	tmp, tmp, shift
 	orr	data1, data1, tmp
 	orr	data2, data2, tmp
 	b	L(start_realigned)
@@ -135,9 +116,6 @@ L(src1_aligned):
 	neg	shift, src2, lsl 3
 	bic	src2, src2, 7
 	ldr	data3, [src2], 8
-#ifdef __AARCH64EB__
-	rev	data3, data3
-#endif
 	lsr	tmp, zeroones, shift
 	orr	data3, data3, tmp
 	sub	has_nul, data3, zeroones
@@ -152,9 +130,6 @@ L(src1_aligned):
 L(loop_unaligned):
 	ldr	data3, [src1, off1]
 	ldr	data2, [src1, off2]
-#ifdef __AARCH64EB__
-	rev	data3, data3
-#endif
 	sub	has_nul, data3, zeroones
 	orr	tmp, data3, REP8_7f
 	ldr	data1, [src1], 8
@@ -163,9 +138,6 @@ L(loop_unaligned):
 	b.eq	L(loop_unaligned)
 
 	lsl	tmp, has_nul, shift
-#ifdef __AARCH64EB__
-	rev	tmp, tmp
-#endif
 	eor	diff, data1, data2
 	orr	syndrome, diff, tmp
 	cbnz	syndrome, L(end)
@@ -174,10 +146,6 @@ L(tail):
 	neg	shift, shift
 	lsr	data2, data3, shift
 	lsr	has_nul, has_nul, shift
-#ifdef __AARCH64EB__
-	rev     data2, data2
-	rev	has_nul, has_nul
-#endif
 	eor	diff, data1, data2
 	orr	syndrome, diff, has_nul
 	b	L(end)
diff --git a/arch/arm64/lib/strlen.S b/arch/arm64/lib/strlen.S
index 4919fe81ae540e..a8c49a3886f638 100644
--- a/arch/arm64/lib/strlen.S
+++ b/arch/arm64/lib/strlen.S
@@ -85,14 +85,6 @@ SYM_FUNC_START(__pi_strlen)
 	cmp	tmp1, MIN_PAGE_SIZE - 16
 	b.gt	L(page_cross)
 	ldp	data1, data2, [srcin]
-#ifdef __AARCH64EB__
-	/* For big-endian, carry propagation (if the final byte in the
-	   string is 0x01) means we cannot use has_nul1/2 directly.
-	   Since we expect strings to be small and early-exit,
-	   byte-swap the data now so has_null1/2 will be correct.  */
-	rev	data1, data1
-	rev	data2, data2
-#endif
 	sub	tmp1, data1, zeroones
 	orr	tmp2, data1, REP8_7f
 	sub	tmp3, data2, zeroones
@@ -144,19 +136,7 @@ L(page_cross_entry):
 
 	/* Enter with C = has_nul1 == 0.  */
 L(tail):
-#ifdef __AARCH64EB__
-	/* For big-endian, carry propagation (if the final byte in the
-	   string is 0x01) means we cannot use has_nul1/2 directly.  The
-	   easiest way to get the correct byte is to byte-swap the data
-	   and calculate the syndrome a second time.  */
-	csel	data1, data1, data2, cc
-	rev	data1, data1
-	sub	tmp1, data1, zeroones
-	orr	tmp2, data1, REP8_7f
-	bic	has_nul1, tmp1, tmp2
-#else
 	csel	has_nul1, has_nul1, has_nul2, cc
-#endif
 	sub	len, src, srcin
 	rev	has_nul1, has_nul1
 	add	tmp2, len, 8
@@ -194,13 +174,7 @@ L(page_cross):
 	ldp	data1, data2, [src]
 	lsl	tmp1, srcin, 3
 	mov	tmp4, -1
-#ifdef __AARCH64EB__
-	/* Big-endian.	Early bytes are at MSB.	 */
-	lsr	tmp1, tmp4, tmp1	/* Shift (tmp1 & 63).  */
-#else
-	/* Little-endian.  Early bytes are at LSB.  */
 	lsl	tmp1, tmp4, tmp1	/* Shift (tmp1 & 63).  */
-#endif
 	orr	tmp1, tmp1, REP8_80
 	orn	data1, data1, tmp1
 	orn	tmp2, data2, tmp1
diff --git a/arch/arm64/lib/strncmp.S b/arch/arm64/lib/strncmp.S
index fe7bbc0b42a787..e4fde8586089a6 100644
--- a/arch/arm64/lib/strncmp.S
+++ b/arch/arm64/lib/strncmp.S
@@ -45,19 +45,6 @@
 #define offset		pos
 #define neg_offset	x15
 
-/* Define endian dependent shift operations.
-   On big-endian early bytes are at MSB and on little-endian LSB.
-   LS_FW means shifting towards early bytes.
-   LS_BK means shifting towards later bytes.
-   */
-#ifdef __AARCH64EB__
-#define LS_FW lsl
-#define LS_BK lsr
-#else
-#define LS_FW lsr
-#define LS_BK lsl
-#endif
-
 SYM_FUNC_START(__pi_strncmp)
 	cbz	limit, L(ret0)
 	eor	tmp1, src1, src2
@@ -86,7 +73,6 @@ L(start_realigned):
 	/* End of main loop */
 
 L(full_check):
-#ifndef __AARCH64EB__
 	orr	syndrome, diff, has_nul
 	add	limit, limit, 8	/* Rewind limit to before last subs. */
 L(syndrome_check):
@@ -105,55 +91,6 @@ L(syndrome_check):
 	sub	result, data1, data2, lsr #56
 	csel result, result, xzr, hi
 	ret
-#else
-	/* Not reached the limit, must have found the end or a diff.  */
-	tbz	limit, #63, L(not_limit)
-	add	tmp1, limit, 8
-	cbz	limit, L(not_limit)
-
-	lsl	limit, tmp1, #3	/* Bits -> bytes.  */
-	mov	mask, #~0
-	lsr	mask, mask, limit
-	bic	data1, data1, mask
-	bic	data2, data2, mask
-
-	/* Make sure that the NUL byte is marked in the syndrome.  */
-	orr	has_nul, has_nul, mask
-
-L(not_limit):
-	/* For big-endian we cannot use the trick with the syndrome value
-	   as carry-propagation can corrupt the upper bits if the trailing
-	   bytes in the string contain 0x01.  */
-	/* However, if there is no NUL byte in the dword, we can generate
-	   the result directly.  We can't just subtract the bytes as the
-	   MSB might be significant.  */
-	cbnz	has_nul, 1f
-	cmp	data1, data2
-	cset	result, ne
-	cneg	result, result, lo
-	ret
-1:
-	/* Re-compute the NUL-byte detection, using a byte-reversed value.  */
-	rev	tmp3, data1
-	sub	tmp1, tmp3, zeroones
-	orr	tmp2, tmp3, #REP8_7f
-	bic	has_nul, tmp1, tmp2
-	rev	has_nul, has_nul
-	orr	syndrome, diff, has_nul
-	clz	pos, syndrome
-	/* The most-significant-non-zero bit of the syndrome marks either the
-	   first bit that is different, or the top bit of the first zero byte.
-	   Shifting left now will bring the critical information into the
-	   top bits.  */
-L(end_quick):
-	lsl	data1, data1, pos
-	lsl	data2, data2, pos
-	/* But we need to zero-extend (char is unsigned) the value and then
-	   perform a signed 32-bit subtraction.  */
-	lsr	data1, data1, #56
-	sub	result, data1, data2, lsr #56
-	ret
-#endif
 
 L(mutual_align):
 	/* Sources are mutually aligned, but are not currently at an
@@ -167,7 +104,7 @@ L(mutual_align):
 	neg	tmp3, count, lsl #3	/* 64 - bits(bytes beyond align). */
 	ldr	data2, [src2], #8
 	mov	tmp2, #~0
-	LS_FW	tmp2, tmp2, tmp3	/* Shift (count & 63).  */
+	lsr	tmp2, tmp2, tmp3	/* Shift (count & 63).  */
 	/* Adjust the limit and ensure it doesn't overflow.  */
 	adds	limit, limit, count
 	csinv	limit, limit, xzr, lo
@@ -236,15 +173,15 @@ L(src1_aligned):
 	neg	neg_offset, offset
 	ldr	data1, [src1], #8
 	ldp	tmp1, tmp2, [src2], #16
-	LS_BK	mask, mask, neg_offset
+	lsl	mask, mask, neg_offset
 	and	neg_offset, neg_offset, #63	/* Need actual value for cmp later. */
 	/* Skip the first compare if data in tmp1 is irrelevant. */
 	tbnz	offset, 6, L(misaligned_mid_loop)
 
 L(loop_misaligned):
 	/* STEP_A: Compare full 8 bytes when there is enough data from SRC2.*/
-	LS_FW	data2, tmp1, offset
-	LS_BK	tmp1, tmp2, neg_offset
+	lsr	data2, tmp1, offset
+	lsl	tmp1, tmp2, neg_offset
 	subs	limit, limit, #8
 	orr	data2, data2, tmp1	/* 8 bytes from SRC2 combined from two regs.*/
 	sub	has_nul, data1, zeroones
@@ -258,23 +195,12 @@ L(loop_misaligned):
 	ldr	data1, [src1], #8
 L(misaligned_mid_loop):
 	/* STEP_B: Compare first part of data1 to second part of tmp2. */
-	LS_FW	data2, tmp2, offset
-#ifdef __AARCH64EB__
-	/* For big-endian we do a byte reverse to avoid carry-propagation
-	problem described above. This way we can reuse the has_nul in the
-	next step and also use syndrome value trick at the end. */
-	rev	tmp3, data1
-	#define data1_fixed tmp3
-#else
+	lsr	data2, tmp2, offset
 	#define data1_fixed data1
-#endif
 	sub	has_nul, data1_fixed, zeroones
 	orr	tmp3, data1_fixed, #REP8_7f
 	eor	diff, data2, data1	/* Non-zero if differences found.  */
 	bic	has_nul, has_nul, tmp3	/* Non-zero if NUL terminator.  */
-#ifdef __AARCH64EB__
-	rev	has_nul, has_nul
-#endif
 	cmp	limit, neg_offset, lsr #3
 	orr	syndrome, diff, has_nul
 	bic	syndrome, syndrome, mask	/* Ignore later bytes. */
@@ -284,7 +210,7 @@ L(misaligned_mid_loop):
 	/* STEP_C: Compare second part of data1 to first part of tmp1. */
 	ldp	tmp1, tmp2, [src2], #16
 	cmp	limit, #8
-	LS_BK	data2, tmp1, neg_offset
+	lsl	data2, tmp1, neg_offset
 	eor	diff, data2, data1	/* Non-zero if differences found.  */
 	orr	syndrome, diff, has_nul
 	and	syndrome, syndrome, mask	/* Ignore earlier bytes. */
@@ -295,13 +221,6 @@ L(misaligned_mid_loop):
 	sub	limit, limit, #8
 	b	L(loop_misaligned)
 
-#ifdef	__AARCH64EB__
-L(syndrome_check):
-	clz	pos, syndrome
-	cmp	pos, limit, lsl #3
-	b.lo	L(end_quick)
-#endif
-
 L(ret0):
 	mov	result, #0
 	ret
diff --git a/arch/arm64/lib/strnlen.S b/arch/arm64/lib/strnlen.S
index d5ac0e10a01db7..b20011d559f197 100644
--- a/arch/arm64/lib/strnlen.S
+++ b/arch/arm64/lib/strnlen.S
@@ -91,22 +91,10 @@ SYM_FUNC_START(__pi_strnlen)
 	*/
 	sub	len, src, srcin
 	cbz	has_nul1, .Lnul_in_data2
-CPU_BE( mov	data2, data1 )	/*perpare data to re-calculate the syndrome*/
 
 	sub	len, len, #8
 	mov	has_nul2, has_nul1
 .Lnul_in_data2:
-	/*
-	* For big-endian, carry propagation (if the final byte in the
-	* string is 0x01) means we cannot use has_nul directly.  The
-	* easiest way to get the correct byte is to byte-swap the data
-	* and calculate the syndrome a second time.
-	*/
-CPU_BE( rev	data2, data2 )
-CPU_BE( sub	tmp1, data2, zeroones )
-CPU_BE( orr	tmp2, data2, #REP8_7f )
-CPU_BE( bic	has_nul2, tmp1, tmp2 )
-
 	sub	len, len, #8
 	rev	has_nul2, has_nul2
 	clz	pos, has_nul2
@@ -139,10 +127,8 @@ CPU_BE( bic	has_nul2, tmp1, tmp2 )
 	lsl	tmp4, tmp4, #3  /* Bytes beyond alignment -> bits.  */
 
 	mov	tmp2, #~0
-	/* Big-endian.  Early bytes are at MSB.  */
-CPU_BE( lsl	tmp2, tmp2, tmp4 )	/* Shift (tmp1 & 63).  */
 	/* Little-endian.  Early bytes are at LSB.  */
-CPU_LE( lsr	tmp2, tmp2, tmp4 )	/* Shift (tmp1 & 63).  */
+	lsr	tmp2, tmp2, tmp4	/* Shift (tmp1 & 63).  */
 
 	cmp	tmp1, #8
 
diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index 6e0528831cd3a6..299a244f68e608 100644
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -79,12 +79,7 @@ ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex,
 	addr &= ~0x7UL;
 
 	data = *(unsigned long*)addr;
-
-#ifndef __AARCH64EB__
 	data >>= 8 * offset;
-#else
-	data <<= 8 * offset;
-#endif
 
 	pt_regs_write_reg(regs, reg_data, data);
 
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 75c3e463df2ef2..2cecf6ba6df7cf 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -131,6 +131,7 @@ static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
  */
 static void show_pte(unsigned long addr)
 {
+	char pxd_str[PTVAL_STR_MAX];
 	struct mm_struct *mm;
 	pgd_t *pgdp;
 	pgd_t pgd;
@@ -160,7 +161,8 @@ static void show_pte(unsigned long addr)
 
 	pgdp = pgd_offset(mm, addr);
 	pgd = READ_ONCE(*pgdp);
-	pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
+	ptval_to_str(pxd_str, pgd_val(pgd));
+	pr_alert("[%016lx] pgd=%s", addr, pxd_str);
 
 	do {
 		p4d_t *p4dp, p4d;
@@ -173,19 +175,22 @@ static void show_pte(unsigned long addr)
 
 		p4dp = p4d_offset_lockless(pgdp, pgd, addr);
 		p4d = READ_ONCE(*p4dp);
-		pr_cont(", p4d=%016llx", p4d_val(p4d));
+		ptval_to_str(pxd_str, p4d_val(p4d));
+		pr_cont(", p4d=%s", pxd_str);
 		if (p4d_none(p4d) || p4d_bad(p4d))
 			break;
 
 		pudp = pud_offset_lockless(p4dp, p4d, addr);
 		pud = READ_ONCE(*pudp);
-		pr_cont(", pud=%016llx", pud_val(pud));
+		ptval_to_str(pxd_str, pud_val(pud));
+		pr_cont(", pud=%s", pxd_str);
 		if (pud_none(pud) || pud_bad(pud))
 			break;
 
 		pmdp = pmd_offset_lockless(pudp, pud, addr);
 		pmd = READ_ONCE(*pmdp);
-		pr_cont(", pmd=%016llx", pmd_val(pmd));
+		ptval_to_str(pxd_str, pmd_val(pmd));
+		pr_cont(", pmd=%s", pxd_str);
 		if (pmd_none(pmd) || pmd_bad(pmd))
 			break;
 
@@ -194,7 +199,8 @@ static void show_pte(unsigned long addr)
 			break;
 
 		pte = __ptep_get(ptep);
-		pr_cont(", pte=%016llx", pte_val(pte));
+		ptval_to_str(pxd_str, pte_val(pte));
+		pr_cont(", pte=%s", pxd_str);
 		pte_unmap(ptep);
 	} while(0);
 
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index fbf215ecc7d061..2a8de20d2b447b 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -34,6 +34,8 @@
 #include <linux/acpi_iort.h>
 #include <linux/kmemleak.h>
 #include <linux/execmem.h>
+#include <linux/arm-rsi-cmds.h>
+
 
 #include <asm/boot.h>
 #include <asm/fixmap.h>
@@ -43,7 +45,6 @@
 #include <asm/kvm_host.h>
 #include <asm/memory.h>
 #include <asm/numa.h>
-#include <asm/rsi.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <linux/sizes.h>
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 79d90226fd5dc9..140520b351e282 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -2295,7 +2295,7 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
 		 * in cases where cpu is affected with errata #2645198.
 		 */
 		if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte))
-			__flush_tlb_range(vma, addr, nr * PAGE_SIZE,
+			__flush_tlb_range(vma, addr, addr + nr * PAGE_SIZE,
 					  PAGE_SIZE, 3, TLBF_NOWALKCACHE);
 	}
 
@@ -2393,3 +2393,51 @@ int arch_set_user_pkey_access(int pkey, unsigned long init_val)
 	return 0;
 }
 #endif
+
+/*
+ * PTE bits configuration in the presence of hardware Dirty Bit Management
+ * (PTE_WRITE == PTE_DBM):
+ *
+ * Dirty  Writable | PTE_RDONLY  PTE_WRITE  PTE_DIRTY (sw)
+ *   0      0      |   1           0          0
+ *   0      1      |   1           1          0
+ *   1      0      |   1           0          1
+ *   1      1      |   0           1          x
+ *
+ * When hardware DBM is not present, the software PTE_DIRTY bit is updated via
+ * the page fault mechanism. Checking the dirty status of a pte becomes:
+ *
+ *   PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
+ */
+#ifdef CONFIG_DEBUG_VM
+void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep, pte_t pte)
+{
+	char pte_str_old[PTVAL_STR_MAX];
+	char pte_str[PTVAL_STR_MAX];
+	pte_t old_pte;
+
+	old_pte = __ptep_get(ptep);
+
+	if (!pte_valid(old_pte) || !pte_valid(pte))
+		return;
+	if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
+		return;
+
+	/*
+	 * Check for potential race with hardware updates of the pte
+	 * (__ptep_set_access_flags safely changes valid ptes without going
+	 * through an invalid entry).
+	 */
+	ptval_to_str(pte_str, pte_val(pte));
+	ptval_to_str(pte_str_old, pte_val(old_pte));
+	VM_WARN_ONCE(!pte_young(pte),
+		     "%s: racy access flag clearing: %s -> %s",
+		     __func__, pte_str_old, pte_str);
+	VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
+		     "%s: racy dirty state clearing: %s -> %s",
+		     __func__, pte_str_old, pte_str);
+	VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)),
+		     "%s: unsafe attribute change: %s -> %s",
+		     __func__, pte_str_old, pte_str);
+}
+#endif /* CONFIG_DEBUG_VM */
diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c
index 63e8d72f202a3b..f64202b69309a7 100644
--- a/arch/arm64/mm/mteswap.c
+++ b/arch/arm64/mm/mteswap.c
@@ -20,22 +20,17 @@ void mte_free_tag_storage(char *storage)
 	kfree(storage);
 }
 
-int mte_save_tags(struct page *page)
+static int mte_save_tags(swp_entry_t entry, struct page *page)
 {
 	void *tag_storage, *ret;
 
-	if (!page_mte_tagged(page))
-		return 0;
-
 	tag_storage = mte_allocate_tag_storage();
 	if (!tag_storage)
 		return -ENOMEM;
 
 	mte_save_page_tags(page_address(page), tag_storage);
 
-	/* lookup the swap entry.val from the page */
-	ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage,
-		       GFP_KERNEL);
+	ret = xa_store(&mte_pages, entry.val, tag_storage, GFP_KERNEL);
 	if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
 		mte_free_tag_storage(tag_storage);
 		return xa_err(ret);
@@ -47,7 +42,7 @@ int mte_save_tags(struct page *page)
 	return 0;
 }
 
-void mte_restore_tags(swp_entry_t entry, struct page *page)
+static void mte_restore_tags(swp_entry_t entry, struct page *page)
 {
 	void *tags = xa_load(&mte_pages, entry.val);
 
@@ -68,13 +63,6 @@ void mte_invalidate_tags(int type, pgoff_t offset)
 	mte_free_tag_storage(tags);
 }
 
-static inline void __mte_invalidate_tags(struct page *page)
-{
-	swp_entry_t entry = page_swap_entry(page);
-
-	mte_invalidate_tags(swp_type(entry), swp_offset(entry));
-}
-
 void mte_invalidate_tags_area(int type)
 {
 	swp_entry_t entry = swp_entry(type, 0);
@@ -102,15 +90,23 @@ int arch_prepare_to_swap(struct folio *folio)
 	nr = folio_nr_pages(folio);
 
 	for (i = 0; i < nr; i++) {
-		err = mte_save_tags(folio_page(folio, i));
+		struct page *page = folio_page(folio, i);
+
+		if (!page_mte_tagged(page))
+			continue;
+
+		err = mte_save_tags(folio_swap_entry(folio, i), page);
 		if (err)
 			goto out;
 	}
 	return 0;
 
 out:
-	while (i--)
-		__mte_invalidate_tags(folio_page(folio, i));
+	while (i--) {
+		swp_entry_t swap = folio_swap_entry(folio, i);
+
+		mte_invalidate_tags(swp_type(swap), swp_offset(swap));
+	}
 	return err;
 }
 
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index bbe98ac9ad8c67..de7bf95003eaaa 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -251,7 +251,7 @@ int set_memory_valid(unsigned long addr, int numpages, int enable)
 					__pgprot(PTE_PRESENT_VALID_KERNEL));
 }
 
-int set_direct_map_invalid_noflush(struct page *page)
+int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages)
 {
 	pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL);
 	pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID);
@@ -260,10 +260,10 @@ int set_direct_map_invalid_noflush(struct page *page)
 		return 0;
 
 	return update_range_prot((unsigned long)page_address(page),
-				 PAGE_SIZE, set_mask, clear_mask);
+				 PAGE_SIZE * numpages, set_mask, clear_mask);
 }
 
-int set_direct_map_default_noflush(struct page *page)
+int set_direct_map_default_noflush(struct page *page, unsigned int numpages)
 {
 	pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE);
 	pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY);
@@ -272,12 +272,10 @@ int set_direct_map_default_noflush(struct page *page)
 		return 0;
 
 	return update_range_prot((unsigned long)page_address(page),
-				 PAGE_SIZE, set_mask, clear_mask);
+				 PAGE_SIZE * numpages, set_mask, clear_mask);
 }
 
-static int __set_memory_enc_dec(unsigned long addr,
-				int numpages,
-				bool encrypt)
+int __set_memory_enc_dec(unsigned long addr, int numpages, bool encrypt)
 {
 	unsigned long set_prot = 0, clear_prot = 0;
 	phys_addr_t start, end;
@@ -321,57 +319,7 @@ static int __set_memory_enc_dec(unsigned long addr,
 				      __pgprot(PTE_PRESENT_INVALID));
 }
 
-static int realm_set_memory_encrypted(unsigned long addr, int numpages)
-{
-	int ret = __set_memory_enc_dec(addr, numpages, true);
-
-	/*
-	 * If the request to change state fails, then the only sensible cause
-	 * of action for the caller is to leak the memory
-	 */
-	WARN(ret, "Failed to encrypt memory, %d pages will be leaked",
-	     numpages);
-
-	return ret;
-}
-
-static int realm_set_memory_decrypted(unsigned long addr, int numpages)
-{
-	int ret = __set_memory_enc_dec(addr, numpages, false);
-
-	WARN(ret, "Failed to decrypt memory, %d pages will be leaked",
-	     numpages);
-
-	return ret;
-}
-
-static const struct arm64_mem_crypt_ops realm_crypt_ops = {
-	.encrypt = realm_set_memory_encrypted,
-	.decrypt = realm_set_memory_decrypted,
-};
-
-int realm_register_memory_enc_ops(void)
-{
-	return arm64_mem_crypt_ops_register(&realm_crypt_ops);
-}
-
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)
-{
-	unsigned long addr = (unsigned long)page_address(page);
-
-	if (!can_set_direct_map())
-		return 0;
-
-	return set_memory_valid(addr, nr, valid);
-}
-
 #ifdef CONFIG_DEBUG_PAGEALLOC
-/*
- * This is - apart from the return value - doing the same
- * thing as the new set_direct_map_valid_noflush() function.
- *
- * Unify? Explain the conceptual differences?
- */
 void __kernel_map_pages(struct page *page, int numpages, int enable)
 {
 	if (!can_set_direct_map())
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index c18e005a41dbeb..7a6c6edae0887d 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -600,6 +600,8 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 		 * 12 registers are on the stack
 		 */
 		emit(A64_SUB_I(1, A64_SP, A64_FP, 96), ctx);
+		/* The callback may use its own BPF stack, set up fp for it. */
+		ctx->fp_used = true;
 	}
 
 	/* Stack must be multiples of 16B */
@@ -1220,6 +1222,12 @@ static int add_exception_handler(const struct bpf_insn *insn,
 	return 0;
 }
 
+static const struct bpf_jit_arg_abi arm64_arg_abi = {
+	.nr_arg_regs		= 8,
+	.even_reg_align		= true,
+	.even_stack_align	= true,
+};
+
 static const u8 stack_arg_reg[] = { A64_R(5), A64_R(6), A64_R(7) };
 
 #define NR_STACK_ARG_REGS	ARRAY_SIZE(stack_arg_reg)
@@ -1262,19 +1270,20 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct
  * kern_vm_start. A nullable arg preserves NULL by skipping the add, tested
  * on the truncated value as arena NULL is offset 0.
  */
-static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *insn)
+static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct btf_func_model *fm)
 {
 	const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
-	const struct btf_func_model *fm;
-	int i;
+	int i, slot;
 
-	fm = bpf_jit_find_kfunc_model(ctx->prog, insn);
-	if (!fm)
-		return -EINVAL;
-
-	for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) {
-		const u8 reg = bpf2a64[BPF_REG_1 + i];
+	for (i = 0, slot = 0; i < fm->nr_args; i++) {
+		u32 arg_regs = (fm->arg_size[i] + 7) / 8;
 		u8 flags = fm->arg_flags[i];
+		u8 reg;
+
+		if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS)
+			break;
+		reg = bpf2a64[BPF_REG_1 + slot];
+		slot += arg_regs;
 
 		if (!(flags & BTF_FMODEL_ARENA_ARG))
 			continue;
@@ -1293,6 +1302,52 @@ static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *ins
 	return 0;
 }
 
+static bool a64_arg_on_stack(u8 slot)
+{
+	return slot >= arm64_arg_abi.nr_arg_regs;
+}
+
+static s32 a64_arg_stack_off(u8 slot)
+{
+	return (slot - arm64_arg_abi.nr_arg_regs) * sizeof(u64);
+}
+
+/*
+ * Move the arguments AAPCS64 places somewhere other than the argument slot the
+ * BPF calling convention gave them. Slot N is X(N) up to the eighth, and the
+ * outgoing stack argument area from SP beyond it, both for the slot an
+ * argument comes from and for the one it goes to.
+ *
+ * AAPCS64 only ever moves an argument to a higher slot, so no move here ever
+ * takes BPF_JIT_ARG_TMP: bpf_jit_plan_arg_moves() hands out the scratch only
+ * for a convention that moves one down, which needs a register to carry the
+ * value past its own destination.
+ */
+static void emit_kfunc_arg_moves(struct jit_ctx *ctx, const struct btf_func_model *fm)
+{
+	struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES];
+	const u8 tmp = bpf2a64[TMP_REG_1];
+	u32 i, n;
+
+	n = bpf_jit_plan_arg_moves(&arm64_arg_abi, fm, moves);
+
+	for (i = 0; i < n; i++) {
+		u8 dst = moves[i].dst, src = moves[i].src, reg;
+
+		if (a64_arg_on_stack(src)) {
+			reg = tmp;
+			emit(A64_LDR64I(reg, A64_SP, a64_arg_stack_off(src)), ctx);
+		} else {
+			reg = src;
+		}
+
+		if (a64_arg_on_stack(dst))
+			emit(A64_STR64I(reg, A64_SP, a64_arg_stack_off(dst)), ctx);
+		else if (reg != dst)
+			emit(A64_MOV(1, dst, reg), ctx);
+	}
+}
+
 /* JITs an eBPF instruction.
  * Returns:
  * 0  - successfully JITed an 8-byte eBPF instruction.
@@ -1439,13 +1494,8 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 	case BPF_ALU | BPF_END | BPF_FROM_LE:
 	case BPF_ALU | BPF_END | BPF_FROM_BE:
 	case BPF_ALU64 | BPF_END | BPF_FROM_LE:
-#ifdef CONFIG_CPU_BIG_ENDIAN
-		if (BPF_CLASS(code) == BPF_ALU && BPF_SRC(code) == BPF_FROM_BE)
-			goto emit_bswap_uxt;
-#else /* !CONFIG_CPU_BIG_ENDIAN */
 		if (BPF_CLASS(code) == BPF_ALU && BPF_SRC(code) == BPF_FROM_LE)
 			goto emit_bswap_uxt;
-#endif
 		switch (imm) {
 		case 16:
 			emit(A64_REV16(is64, dst, dst), ctx);
@@ -1716,9 +1766,15 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 		if (ret < 0)
 			return ret;
 		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
-			ret = emit_kfunc_arena_args(ctx, insn);
+			const struct btf_func_model *fm;
+
+			fm = bpf_jit_find_kfunc_model(ctx->prog, insn);
+			if (!fm)
+				return -EINVAL;
+			ret = emit_kfunc_arena_args(ctx, fm);
 			if (ret < 0)
 				return ret;
+			emit_kfunc_arg_moves(ctx, fm);
 		}
 		emit_call(func_addr, ctx);
 		/*
@@ -1730,6 +1786,17 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 			emit(A64_MOV(1, r0, A64_R(0)), ctx);
 		break;
 	}
+	/* indirect call of a bpf subprog, dst holds its address */
+	case BPF_JMP | BPF_CALL | BPF_X:
+		/*
+		 * It's the same as a direct call of a subprog that is out of
+		 * range of BL: the subprog starts with BTI JC, the arguments
+		 * are in place, and the registers that hold the tail call
+		 * counter and the private stack are callee saved.
+		 */
+		emit(A64_BLR(dst), ctx);
+		emit(A64_MOV(1, bpf2a64[BPF_REG_0], A64_R(0)), ctx);
+		break;
 	/* tail call */
 	case BPF_JMP | BPF_TAIL_CALL:
 		if (emit_bpf_tail_call(ctx))
@@ -2388,6 +2455,16 @@ bool bpf_jit_supports_kfunc_call(void)
 	return true;
 }
 
+bool bpf_jit_supports_kfunc_ret_reg_pair(void)
+{
+	return true;
+}
+
+const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void)
+{
+	return &arm64_arg_abi;
+}
+
 bool bpf_jit_supports_stack_args(void)
 {
 	return true;
@@ -2416,10 +2493,21 @@ bool bpf_jit_supports_subprog_tailcalls(void)
 	return true;
 }
 
-static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node,
-			    int bargs_off, int retval_off, int run_ctx_off,
-			    bool save_ret)
+bool bpf_jit_supports_callx(void)
 {
+	return true;
+}
+
+bool bpf_jit_supports_large_stack(void)
+{
+	return true;
+}
+
+static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im,
+			    struct bpf_tramp_node *node, int bargs_off,
+			    int retval_off, int run_ctx_off, bool save_ret)
+{
+	void *skip;
 	__le32 *branch;
 	u64 enter_prog;
 	u64 exit_prog;
@@ -2429,6 +2517,10 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node,
 	enter_prog = (u64)bpf_trampoline_enter(p);
 	exit_prog = (u64)bpf_trampoline_exit(p);
 
+	/* nop, patched to skip this prog when it is detached */
+	skip = ctx->ro_image + ctx->idx;
+	emit(A64_NOP, ctx);
+
 	if (node->cookie == 0) {
 		/* if cookie is zero, one instruction is enough to store it */
 		emit(A64_STR64I(A64_ZR, A64_SP, run_ctx_off + cookie_off), ctx);
@@ -2481,11 +2573,13 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node,
 	emit(A64_ADD_I(1, A64_R(2), A64_SP, run_ctx_off), ctx);
 
 	emit_call(exit_prog, ctx);
+
+	bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx);
 }
 
-static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn,
-			       int bargs_off, int retval_off, int run_ctx_off,
-			       __le32 **branches)
+static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_image *im,
+			       struct bpf_tramp_nodes *tn, int bargs_off,
+			       int retval_off, int run_ctx_off, __le32 **branches)
 {
 	int i;
 
@@ -2494,7 +2588,7 @@ static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn,
 	 */
 	emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx);
 	for (i = 0; i < tn->nr_nodes; i++) {
-		invoke_bpf_prog(ctx, tn->nodes[i], bargs_off, retval_off,
+		invoke_bpf_prog(ctx, im, tn->nodes[i], bargs_off, retval_off,
 				run_ctx_off, true);
 		/* if (*(u64 *)(sp + retval_off) !=  0)
 		 *	goto do_fexit;
@@ -2524,33 +2618,41 @@ struct arg_aux {
 	 * arguments to be properly aligned)
 	 */
 	int ostack_for_args;
+	/* where AAPCS64 puts each argument slot: an argument register below
+	 * the eighth, an on-stack argument slot from it up
+	 */
+	u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS];
 };
 
 static int calc_arg_aux(const struct btf_func_model *m,
 			 struct arg_aux *a)
 {
-	int stack_slots, nregs, slots, i;
+	int slots, i, slot, total;
+
+	total = bpf_jit_place_args(&arm64_arg_abi, m, a->pos_of_slot);
+	if (total > MAX_BPF_FUNC_ARGS)
+		return -ENOTSUPP;
 
 	/* verifier ensures m->nr_args <= MAX_BPF_FUNC_ARGS */
-	for (i = 0, nregs = 0; i < m->nr_args; i++) {
+	for (i = 0, slot = 0; i < m->nr_args; i++) {
 		slots = (m->arg_size[i] + 7) / 8;
-		if (nregs + slots <= 8) /* passed through register ? */
-			nregs += slots;
-		else
+		if (a64_arg_on_stack(a->pos_of_slot[slot])) /* passed through register ? */
 			break;
+		slot += slots;
 	}
 
 	a->args_in_regs = i;
-	a->regs_for_args = nregs;
+	a->regs_for_args = slot;
 	a->ostack_for_args = 0;
 	a->bstack_for_args = 0;
 
 	/* the rest arguments are passed through stack */
-	for (; i < m->nr_args; i++) {
-		stack_slots = (m->arg_size[i] + 7) / 8;
-		a->bstack_for_args += stack_slots * 8;
-		a->ostack_for_args = a->ostack_for_args + stack_slots * 8;
-	}
+	for (; i < m->nr_args; i++)
+		a->bstack_for_args += ((m->arg_size[i] + 7) / 8) * 8;
+
+	/* the outgoing area reaches the last slot, over any alignment hole */
+	if (a->bstack_for_args)
+		a->ostack_for_args = a64_arg_stack_off(a->pos_of_slot[total - 1]) + 8;
 
 	return 0;
 }
@@ -2559,15 +2661,9 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
 {
 	if (effective_bytes) {
 		int garbage_bits = 64 - 8 * effective_bytes;
-#ifdef CONFIG_CPU_BIG_ENDIAN
-		/* garbage bits are at the right end */
-		emit(A64_LSR(1, reg, reg, garbage_bits), ctx);
-		emit(A64_LSL(1, reg, reg, garbage_bits), ctx);
-#else
 		/* garbage bits are at the left end */
 		emit(A64_LSL(1, reg, reg, garbage_bits), ctx);
 		emit(A64_LSR(1, reg, reg, garbage_bits), ctx);
-#endif
 	}
 }
 
@@ -2597,7 +2693,7 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 {
 	u8 tmp = bpf2a64[TMP_REG_1];
 	u8 base_lo = bpf2a64[TMP_REG_2];
-	int i, reg, doff, soff, slots;
+	int i, reg, slot, soff, slots;
 
 	/* only the low 32 bits of the base take part in the subtraction */
 	if (arena_base)
@@ -2606,12 +2702,13 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 	/* store arguments to the stack for the bpf program, or restore
 	 * arguments from stack for the original function
 	 */
-	for (i = 0, reg = 0; i < a->args_in_regs; i++) {
+	for (i = 0, slot = 0; i < a->args_in_regs; i++) {
 		bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG);
 		bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG;
 
 		slots = (m->arg_size[i] + 7) / 8;
 		while (slots-- > 0) {
+			reg = a->pos_of_slot[slot++];
 			if (for_call_origin) {
 				emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx);
 			} else if (arena_arg) {
@@ -2620,7 +2717,6 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 			} else {
 				emit(A64_STR64I(reg, A64_SP, bargs_off), ctx);
 			}
-			reg++;
 			bargs_off += 8;
 		}
 	}
@@ -2632,9 +2728,11 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 	 * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops
 	 * callback is called indirectly and only the FP/LR frame is saved, so
 	 * they start at FP + 16.
+	 *
+	 * The outgoing area mirrors the incoming one, hole and all; only the
+	 * bpf program takes the arguments packed.
 	 */
 	soff = is_struct_ops ? 16 : 32;
-	doff = (for_call_origin ? oargs_off : bargs_off);
 
 	/* save on stack arguments */
 	for (i = a->args_in_regs; i < m->nr_args; i++) {
@@ -2644,7 +2742,9 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 		slots = (m->arg_size[i] + 7) / 8;
 		/* verifier ensures arg_size <= 16, so slots equals 1 or 2 */
 		while (slots-- > 0) {
-			emit(A64_LDR64I(tmp, A64_FP, soff), ctx);
+			int off = a64_arg_stack_off(a->pos_of_slot[slot++]);
+
+			emit(A64_LDR64I(tmp, A64_FP, soff + off), ctx);
 			/* if there is unused space in the last slot, clear
 			 * the garbage contained in the space.
 			 */
@@ -2659,19 +2759,21 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
 			 */
 			if (arena_arg)
 				emit_arena_arg_conv(ctx, tmp, tmp, nullable, base_lo);
-			emit(A64_STR64I(tmp, A64_SP, doff), ctx);
-			soff += 8;
-			doff += 8;
+			if (for_call_origin)
+				emit(A64_STR64I(tmp, A64_SP, oargs_off + off), ctx);
+			else
+				emit(A64_STR64I(tmp, A64_SP, bargs_off), ctx);
+			bargs_off += 8;
 		}
 	}
 }
 
-static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs)
+static void restore_args(struct jit_ctx *ctx, int bargs_off, const struct arg_aux *a)
 {
-	int reg;
+	int slot;
 
-	for (reg = 0; reg < nregs; reg++) {
-		emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx);
+	for (slot = 0; slot < a->regs_for_args; slot++) {
+		emit(A64_LDR64I(a->pos_of_slot[slot], A64_SP, bargs_off), ctx);
 		bargs_off += 8;
 	}
 }
@@ -2880,7 +2982,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 			store_func_meta(ctx, meta, func_meta_off);
 			cookie_bargs_off--;
 		}
-		invoke_bpf_prog(ctx, fentry->nodes[i], bargs_off,
+		invoke_bpf_prog(ctx, im, fentry->nodes[i], bargs_off,
 				retval_off, run_ctx_off,
 				flags & BPF_TRAMP_F_RET_FENTRY_RET);
 	}
@@ -2891,7 +2993,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 		if (!branches)
 			return -ENOMEM;
 
-		invoke_bpf_mod_ret(ctx, fmod_ret, bargs_off, retval_off,
+		invoke_bpf_mod_ret(ctx, im, fmod_ret, bargs_off, retval_off,
 				   run_ctx_off, branches);
 	}
 
@@ -2904,9 +3006,6 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 		emit(A64_RET(A64_R(10)), ctx);
 		/* store return value */
 		emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx);
-		/* reserve a nop for bpf_tramp_image_put */
-		im->ip_after_call = ctx->ro_image + ctx->idx;
-		emit(A64_NOP, ctx);
 	}
 
 	/* update the branches saved in invoke_bpf_mod_ret with cbnz */
@@ -2928,12 +3027,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 			store_func_meta(ctx, meta, func_meta_off);
 			cookie_bargs_off--;
 		}
-		invoke_bpf_prog(ctx, fexit->nodes[i], bargs_off, retval_off,
+		invoke_bpf_prog(ctx, im, fexit->nodes[i], bargs_off, retval_off,
 				run_ctx_off, false);
 	}
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
-		im->ip_epilogue = ctx->ro_image + ctx->idx;
 		/* for the first pass, assume the worst case */
 		if (!ctx->image)
 			ctx->idx += 4;
@@ -2943,7 +3041,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
 	}
 
 	if (flags & BPF_TRAMP_F_RESTORE_REGS)
-		restore_args(ctx, bargs_off, a->regs_for_args);
+		restore_args(ctx, bargs_off, a);
 
 	/* restore callee saved register x19 and x20 */
 	emit(A64_LDR64I(A64_R(19), A64_SP, regs_off), ctx);
@@ -2992,7 +3090,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 		.image = NULL,
 		.idx = 0,
 	};
-	struct bpf_tramp_image im;
+	struct bpf_tramp_image im = {};
 	struct arg_aux aaux;
 	int ret;
 
@@ -3279,6 +3377,10 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 	 *    longer reachable, since bpf_tramp_image_put() function already
 	 *    uses percpu_ref and task-based rcu to do the sync, no need to call
 	 *    the sync version here, see bpf_tramp_image_put() for details.
+	 *
+	 * 3. when a detached prog is patched out of a trampoline, a CPU that
+	 *    still executes the old nop calls the prog before it went through
+	 *    a quiescent state, and the prog is freed after grace periods.
 	 */
 	ret = aarch64_insn_patch_text_nosync(ip, new_insn);
 out:
diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h
index bafcd5823531a5..5ca77ff89ef939 100644
--- a/arch/csky/include/asm/pgtable.h
+++ b/arch/csky/include/asm/pgtable.h
@@ -23,10 +23,6 @@
 #define PTRS_PER_PMD	1
 #define PTRS_PER_PTE	(PAGE_SIZE / sizeof(pte_t))
 
-#define pte_ERROR(e) \
-	pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, (e).pte_low)
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
 
 #define PFN_PTE_SHIFT	PAGE_SHIFT
 #define pmd_pfn(pmd)	(pmd_phys(pmd) >> PAGE_SHIFT)
diff --git a/arch/csky/kernel/perf_regs.c b/arch/csky/kernel/perf_regs.c
index 09b7f88a2d6ad4..c932a96afc5642 100644
--- a/arch/csky/kernel/perf_regs.c
+++ b/arch/csky/kernel/perf_regs.c
@@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 
 #define REG_RESERVED (~((1ULL << PERF_REG_CSKY_MAX) - 1))
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask || mask & REG_RESERVED)
 		return -EINVAL;
@@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task)
 	return PERF_SAMPLE_REGS_ABI_32;
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h
index 27b269e2870d37..2fdb27afe70329 100644
--- a/arch/hexagon/include/asm/pgtable.h
+++ b/arch/hexagon/include/asm/pgtable.h
@@ -94,9 +94,6 @@
 #endif
 
 /*  Any bigger and the PTE disappears.  */
-#define pgd_ERROR(e) \
-	printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__,\
-		pgd_val(e))
 
 /*
  * Page Protection Constants. Includes (in this variant) cache attributes.
diff --git a/arch/hexagon/include/asm/tlb.h b/arch/hexagon/include/asm/tlb.h
index 2c9390e7ca8dde..39a182b61f6c86 100644
--- a/arch/hexagon/include/asm/tlb.h
+++ b/arch/hexagon/include/asm/tlb.h
@@ -6,7 +6,6 @@
 #ifndef _ASM_TLB_H
 #define _ASM_TLB_H
 
-#include <linux/pagemap.h>
 #include <asm/tlbflush.h>
 
 #include <asm-generic/tlb.h>
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index 2067d1f2ad7acb..0aad0004b01077 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -69,7 +69,6 @@ config LOONGARCH
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
 	select ARCH_HAS_PTE_PROTNONE if 64BIT
 	select ARCH_SUPPORTS_NUMA_BALANCING if NUMA
-	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_SUPPORTS_RT
 	select ARCH_SUPPORTS_SCHED_SMT if SMP
 	select ARCH_SUPPORTS_SCHED_MC  if SMP
@@ -170,7 +169,6 @@ config LOONGARCH
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
-	select HAVE_PREEMPT_DYNAMIC_KEY
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC
 	select HAVE_RETHOOK
@@ -189,7 +187,6 @@ config LOONGARCH
 	select IRQ_LOONGARCH_CPU
 	select LOCK_MM_AND_FIND_VMA
 	select MMU_GATHER_MERGE_VMAS if MMU
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_RELA if MODULES
 	select NEED_PER_CPU_EMBED_FIRST_CHUNK
 	select NEED_PER_CPU_PAGE_FIRST_CHUNK
diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index a05f6a4928dc68..f876031351319a 100644
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -72,6 +72,7 @@
 
 #include <linux/mm_types.h>
 #include <linux/mmzone.h>
+#include <linux/vmemmap-optimization.h>
 #include <asm/fixmap.h>
 #include <asm/sparsemem.h>
 
@@ -135,19 +136,6 @@ struct vm_area_struct;
 #define ptep_get(ptep) READ_ONCE(*(ptep))
 #define pmdp_get(pmdp) READ_ONCE(*(pmdp))
 
-#define pte_ERROR(e) \
-	pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
-#ifndef __PAGETABLE_PMD_FOLDED
-#define pmd_ERROR(e) \
-	pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
-#endif
-#ifndef __PAGETABLE_PUD_FOLDED
-#define pud_ERROR(e) \
-	pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
-#endif
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
-
 extern pte_t invalid_pte_table[PTRS_PER_PTE];
 
 #ifndef __PAGETABLE_PUD_FOLDED
diff --git a/arch/loongarch/include/asm/set_memory.h b/arch/loongarch/include/asm/set_memory.h
index 55dfaefd02c8a6..4bb01172fbc245 100644
--- a/arch/loongarch/include/asm/set_memory.h
+++ b/arch/loongarch/include/asm/set_memory.h
@@ -15,8 +15,7 @@ int set_memory_ro(unsigned long addr, int numpages);
 int set_memory_rw(unsigned long addr, int numpages);
 
 bool kernel_page_present(struct page *page);
-int set_direct_map_default_noflush(struct page *page);
-int set_direct_map_invalid_noflush(struct page *page);
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);
+int set_direct_map_default_noflush(struct page *page, unsigned int nr);
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr);
 
 #endif /* _ASM_LOONGARCH_SET_MEMORY_H */
diff --git a/arch/loongarch/include/uapi/asm/setup.h b/arch/loongarch/include/uapi/asm/setup.h
index d46363ce3e024c..03c7bfa1e5d9f2 100644
--- a/arch/loongarch/include/uapi/asm/setup.h
+++ b/arch/loongarch/include/uapi/asm/setup.h
@@ -3,6 +3,10 @@
 #ifndef _UAPI_ASM_LOONGARCH_SETUP_H
 #define _UAPI_ASM_LOONGARCH_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	4096
+#endif
 
 #endif /* _UAPI_ASM_LOONGARCH_SETUP_H */
diff --git a/arch/loongarch/kernel/kexec_efi.c b/arch/loongarch/kernel/kexec_efi.c
index 5ee78ebb1546e2..28a1d0420ba3d5 100644
--- a/arch/loongarch/kernel/kexec_efi.c
+++ b/arch/loongarch/kernel/kexec_efi.c
@@ -24,12 +24,12 @@ static int efi_kexec_probe(const char *kernel_buf, unsigned long kernel_len)
 
 	if (!h || (kernel_len < sizeof(*h))) {
 		kexec_dprintk("No LoongArch image header.\n");
-		return -EINVAL;
+		return -ENOEXEC;
 	}
 
 	if (!loongarch_header_check_dos_sig(h)) {
 		kexec_dprintk("No LoongArch PE image header.\n");
-		return -EINVAL;
+		return -ENOEXEC;
 	}
 
 	return 0;
diff --git a/arch/loongarch/kernel/paravirt.c b/arch/loongarch/kernel/paravirt.c
index 10821cce554c0f..e8965a3f808241 100644
--- a/arch/loongarch/kernel/paravirt.c
+++ b/arch/loongarch/kernel/paravirt.c
@@ -308,10 +308,10 @@ int __init pv_time_init(void)
 
 	static_call_update(pv_steal_clock, paravt_steal_clock);
 
-	static_key_slow_inc(&paravirt_steal_enabled);
+	static_branch_inc(&paravirt_steal_enabled);
 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
 	if (steal_acc)
-		static_key_slow_inc(&paravirt_steal_rq_enabled);
+		static_branch_inc(&paravirt_steal_rq_enabled);
 #endif
 
 	if (static_key_enabled(&virt_preempt_key))
diff --git a/arch/loongarch/kernel/perf_regs.c b/arch/loongarch/kernel/perf_regs.c
index 263ac4ab5af684..164514f40ae00a 100644
--- a/arch/loongarch/kernel/perf_regs.c
+++ b/arch/loongarch/kernel/perf_regs.c
@@ -25,7 +25,7 @@ u64 perf_reg_abi(struct task_struct *tsk)
 }
 #endif /* CONFIG_32BIT */
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask)
 		return -EINVAL;
@@ -45,9 +45,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 	return regs->regs[idx];
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 4b46c5d30708d8..f801f7097f0379 100644
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -197,13 +197,15 @@ void __init __set_fixmap(enum fixed_addresses idx,
 			       phys_addr_t phys, pgprot_t flags)
 {
 	unsigned long addr = __fix_to_virt(idx);
+	char str[PTVAL_STR_MAX];
 	pte_t *ptep;
 
 	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
 
 	ptep = populate_kernel_pte(addr);
 	if (!pte_none(ptep_get(ptep))) {
-		pte_ERROR(*ptep);
+		ptval_to_str(str, pte_val(*ptep));
+		pr_err("unexpected set PTE at %lx in %s: %s\n", addr, __func__, str);
 		return;
 	}
 
diff --git a/arch/loongarch/mm/pageattr.c b/arch/loongarch/mm/pageattr.c
index 614ccc7afccbea..a7dcff40f75982 100644
--- a/arch/loongarch/mm/pageattr.c
+++ b/arch/loongarch/mm/pageattr.c
@@ -198,41 +198,22 @@ bool kernel_page_present(struct page *page)
 	return pte_present(ptep_get(pte));
 }
 
-int set_direct_map_default_noflush(struct page *page)
+int set_direct_map_default_noflush(struct page *page, unsigned int nr)
 {
 	unsigned long addr = (unsigned long)page_address(page);
 
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, 1, PAGE_KERNEL, __pgprot(0));
+	return __set_memory(addr, nr, PAGE_KERNEL, __pgprot(0));
 }
 
-int set_direct_map_invalid_noflush(struct page *page)
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr)
 {
 	unsigned long addr = (unsigned long)page_address(page);
 
 	if (addr < vm_map_base)
 		return 0;
 
-	return __set_memory(addr, 1, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID));
-}
-
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)
-{
-	unsigned long addr = (unsigned long)page_address(page);
-	pgprot_t set, clear;
-
-	if (addr < vm_map_base)
-		return 0;
-
-	if (valid) {
-		set = PAGE_KERNEL;
-		clear = __pgprot(0);
-	} else {
-		set = __pgprot(0);
-		clear = __pgprot(_PAGE_PRESENT | _PAGE_VALID);
-	}
-
-	return __set_memory(addr, nr, set, clear);
+	return __set_memory(addr, nr, __pgprot(0), __pgprot(_PAGE_PRESENT | _PAGE_VALID));
 }
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 4da2789009383b..e78eb582c3005e 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1578,6 +1578,24 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 	return ret ? ERR_PTR(-EINVAL) : dst;
 }
 
+int arch_bpf_trampoline_skip(void *nop, void *target)
+{
+	u32 old_insn = INSN_NOP;
+	u32 new_insn = larch_insn_gen_b((unsigned long)nop, (unsigned long)target);
+	int ret;
+
+	if (memcmp(nop, &old_insn, LOONGARCH_INSN_SIZE))
+		return -EFAULT;
+
+	cpus_read_lock();
+	mutex_lock(&text_mutex);
+	ret = larch_insn_text_copy(nop, &new_insn, LOONGARCH_INSN_SIZE);
+	mutex_unlock(&text_mutex);
+	cpus_read_unlock();
+
+	return ret;
+}
+
 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 		       enum bpf_text_poke_type new_t, void *old_addr,
 		       void *new_addr)
@@ -1696,13 +1714,18 @@ static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off,
 	}
 }
 
-static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n,
-			   int args_off, int retval_off, int run_ctx_off, bool save_ret)
+static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im,
+			   struct bpf_tramp_node *n, int args_off, int retval_off,
+			   int run_ctx_off, bool save_ret)
 {
 	int ret;
 	u32 *branch;
 	struct bpf_prog *p = n->link->prog;
 	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
+	void *skip = ctx->ro_image + ctx->idx;
+
+	/* nop, patched to a b over this prog when it is detached */
+	emit_insn(ctx, nop);
 
 	if (n->cookie)
 		emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1,
@@ -1755,13 +1778,17 @@ static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n,
 	/* arg3: &run_ctx */
 	emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
 	ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
+	if (ret)
+		return ret;
 
-	return ret;
+	bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx);
+	return 0;
 }
 
-static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn,
-		      int args_off, int retval_off, int run_ctx_off,
-		      int func_meta_off, bool save_ret, u64 func_meta, int cookie_off)
+static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_image *im,
+		      struct bpf_tramp_nodes *tn, int args_off, int retval_off,
+		      int run_ctx_off, int func_meta_off, bool save_ret,
+		      u64 func_meta, int cookie_off)
 {
 	int i, cur_cookie = (cookie_off - args_off) / 8;
 
@@ -1774,7 +1801,8 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn,
 			emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta);
 			cur_cookie--;
 		}
-		err = invoke_bpf_prog(ctx, tn->nodes[i], args_off, retval_off, run_ctx_off, save_ret);
+		err = invoke_bpf_prog(ctx, im, tn->nodes[i], args_off, retval_off,
+				      run_ctx_off, save_ret);
 		if (err)
 			return err;
 	}
@@ -2017,7 +2045,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i
 	}
 
 	if (fentry->nr_nodes) {
-		ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off,
+		ret = invoke_bpf(ctx, im, fentry, args_off, retval_off, run_ctx_off, func_meta_off,
 				 flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off);
 		if (ret)
 			return ret;
@@ -2029,7 +2057,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i
 
 		emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
 		for (i = 0; i < fmod_ret->nr_nodes; i++) {
-			ret = invoke_bpf_prog(ctx, fmod_ret->nodes[i],
+			ret = invoke_bpf_prog(ctx, im, fmod_ret->nodes[i],
 					      args_off, retval_off, run_ctx_off, true);
 			if (ret)
 				goto out;
@@ -2051,10 +2079,6 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i
 			goto out;
 		emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
 		emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
-		im->ip_after_call = ctx->ro_image + ctx->idx;
-		/* Reserve space for the move_imm + jirl instruction */
-		for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
-			emit_insn(ctx, nop);
 	}
 
 	for (i = 0; ctx->image && i < fmod_ret->nr_nodes; i++) {
@@ -2068,14 +2092,13 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i
 		emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta);
 
 	if (fexit->nr_nodes) {
-		ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off,
+		ret = invoke_bpf(ctx, im, fexit, args_off, retval_off, run_ctx_off,
 				 func_meta_off, false, func_meta, cookie_off);
 		if (ret)
 			goto out;
 	}
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
-		im->ip_epilogue = ctx->ro_image + ctx->idx;
 		move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im);
 		ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
 		if (ret)
@@ -2177,11 +2200,8 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 			     struct bpf_tramp_nodes *tnodes, void *func_addr)
 {
 	int ret;
-	struct jit_ctx ctx;
-	struct bpf_tramp_image im;
-
-	ctx.image = NULL;
-	ctx.idx = 0;
+	struct jit_ctx ctx = {};
+	struct bpf_tramp_image im = {};
 
 	ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tnodes, func_addr, flags);
 
diff --git a/arch/m68k/68000/entry.S b/arch/m68k/68000/entry.S
index c257cc415c4780..0d3cbededc43c3 100644
--- a/arch/m68k/68000/entry.S
+++ b/arch/m68k/68000/entry.S
@@ -79,7 +79,8 @@ ENTRY(system_call)
 
 	/* Doing a trace ? */
 	getthreadinfo
-	btst	#(TIF_SYSCALL_TRACE%8),%a2@(TINFO_FLAGS+(31-TIF_SYSCALL_TRACE)/8)
+	moveb	%a2@(TINFO_FLAGS+2),%d1
+	andl	#((_TIF_SYSCALL_TRACE + _TIF_SECCOMP) >> 8),%d1
 	jne	do_trace
 	cmpl	#NR_syscalls,%d0
 	jcc	badsys
diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig
index 11835eb59d94db..eb84c3af92c02c 100644
--- a/arch/m68k/Kconfig
+++ b/arch/m68k/Kconfig
@@ -29,6 +29,7 @@ config M68K
 	select HAVE_ARCH_LIBGCC_H
 	select HAVE_ARCH_SECCOMP
 	select HAVE_ARCH_SECCOMP_FILTER
+	select HAVE_ARCH_TLB_REMOVE_TABLE if MMU_MOTOROLA
 	select HAVE_ASM_MODVERSIONS
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS if !CPU_HAS_NO_UNALIGNED
diff --git a/arch/m68k/Kconfig.devices b/arch/m68k/Kconfig.devices
index e6e3efac184074..3f809a02438b26 100644
--- a/arch/m68k/Kconfig.devices
+++ b/arch/m68k/Kconfig.devices
@@ -67,6 +67,18 @@ config ATARI_ETHERNAT
 	  To compile the actual ethernet driver, choose Y or M for the SMC91X
 	  option in the network device section; the module will be called smc91x.
 
+config ATARI_SVETHLANA
+	bool "Atari SuperVidel SVEthLANa Ethernet support"
+	depends on ATARI
+	help
+	  Say Y to include support for the SVEthLANa network adapter built
+	  into the SuperVidel FPGA. SuperVidel firmware version 10 or newer
+	  is required (older firmware lacks the Ethernet DMA).
+
+	  To compile the actual ethernet driver, choose Y or M for the ETHOC
+	  option in the network device section; the module will be called
+	  ethoc.
+
 config ATARI_ETHERNEC
 	bool "Atari EtherNEC Ethernet support"
 	depends on ATARI_ROM_ISA
diff --git a/arch/m68k/apollo/dn_ints.c b/arch/m68k/apollo/dn_ints.c
index ba96a92f8f18f0..89bfb5aaad9df8 100644
--- a/arch/m68k/apollo/dn_ints.c
+++ b/arch/m68k/apollo/dn_ints.c
@@ -2,6 +2,7 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 
+#include <asm/irq.h>
 #include <asm/traps.h>
 #include <asm/apollohw.h>
 
@@ -11,10 +12,13 @@ static unsigned int apollo_irq_startup(struct irq_data *data)
 {
 	unsigned int irq = data->irq;
 
+	m68k_irq_startup(data);
+
 	if (irq < 8)
 		*(volatile unsigned char *)(pica+1) &= ~(1 << irq);
 	else
 		*(volatile unsigned char *)(picb+1) &= ~(1 << (irq - 8));
+
 	return 0;
 }
 
@@ -26,6 +30,8 @@ static void apollo_irq_shutdown(struct irq_data *data)
 		*(volatile unsigned char *)(pica+1) |= (1 << irq);
 	else
 		*(volatile unsigned char *)(picb+1) |= (1 << (irq - 8));
+
+	m68k_irq_shutdown(data);
 }
 
 static void apollo_irq_eoi(struct irq_data *data)
diff --git a/arch/m68k/atari/config.c b/arch/m68k/atari/config.c
index ee2d061efb2a85..bdafc86241b760 100644
--- a/arch/m68k/atari/config.c
+++ b/arch/m68k/atari/config.c
@@ -34,6 +34,7 @@
 #include <linux/platform_device.h>
 #include <linux/usb/isp116x.h>
 #include <linux/module.h>
+#include <net/ethoc.h>
 
 #include <asm/bootinfo.h>
 #include <asm/bootinfo-atari.h>
@@ -748,6 +749,62 @@ static struct platform_device *atari_ethernat_devices[] __initdata = {
 };
 #endif /* CONFIG_ATARI_ETHERNAT */
 
+#ifdef CONFIG_ATARI_SVETHLANA
+/*
+ * SVEthLANa: OpenCores 10/100 Mbps Ethernet MAC in the SuperVidel FPGA,
+ * handled by the ethoc driver. Requires SuperVidel firmware version 10
+ * or newer (Ethernet DMA).
+ *
+ * The MAC can only DMA within the SuperVidel DDR RAM, so the packet
+ * buffers are carved out of the top of the DDR; the framebuffer lives
+ * at the bottom.
+ */
+
+#define ATARI_SVETHLANA_PHYS_ADDR	0x80012000
+#define ATARI_SVETHLANA_IRQ		141	/* vector 0xc5 */
+
+#define ATARI_SV_VERSION_PHYS_ADDR	0x8001007c
+#define ATARI_SV_DDR_END		0xa8000000
+/* 128 packet buffers of 1536 bytes each, the maximum ethoc supports */
+#define SVETHLANA_BUF_SIZE		(128 * 1536)
+
+static struct resource svethlana_resources[] = {
+	[0] = {
+		.name	= "ethoc-regs",
+		.start	= ATARI_SVETHLANA_PHYS_ADDR,
+		.end	= ATARI_SVETHLANA_PHYS_ADDR + 0x7ff,
+		.flags	= IORESOURCE_MEM,
+	},
+	[1] = {
+		.name	= "ethoc-buf",
+		.start	= ATARI_SV_DDR_END - SVETHLANA_BUF_SIZE,
+		.end	= ATARI_SV_DDR_END - 1,
+		.flags	= IORESOURCE_MEM,
+	},
+	[2] = {
+		.name	= "ethoc-irq",
+		.start	= ATARI_SVETHLANA_IRQ,
+		.end	= ATARI_SVETHLANA_IRQ,
+		.flags	= IORESOURCE_IRQ,
+	},
+};
+
+static struct ethoc_platform_data svethlana_platform_data = {
+	.phy_id		= -1,
+	.big_endian	= true,
+};
+
+static struct platform_device svethlana_device = {
+	.name		= "ethoc",
+	.id		= -1,
+	.num_resources	= ARRAY_SIZE(svethlana_resources),
+	.resource	= svethlana_resources,
+	.dev		= {
+		.platform_data	= &svethlana_platform_data,
+	},
+};
+#endif /* CONFIG_ATARI_SVETHLANA */
+
 #ifdef CONFIG_ATARI_ETHERNEC
 /*
  * EtherNEC: RTL8019 (NE2000 compatible) Ethernet chipset,
@@ -892,6 +949,24 @@ static int __init atari_platform_init(void)
 	}
 #endif
 
+#ifdef CONFIG_ATARI_SVETHLANA
+	{
+		int error;
+		void __iomem *sv_version;
+
+		sv_version = ioremap(ATARI_SV_VERSION_PHYS_ADDR, 4);
+		if (sv_version) {
+			if (hwreg_present(sv_version) &&
+			    (__raw_readl(sv_version) & 0x3ff) >= 10) {
+				error = platform_device_register(&svethlana_device);
+				if (error && !rv)
+					rv = error;
+			}
+			iounmap(sv_version);
+		}
+	}
+#endif
+
 #ifdef CONFIG_ATARI_ETHERNEC
 	{
 		int error;
diff --git a/arch/m68k/coldfire/entry.S b/arch/m68k/coldfire/entry.S
index 4ea08336e2fb0a..d658abdc520753 100644
--- a/arch/m68k/coldfire/entry.S
+++ b/arch/m68k/coldfire/entry.S
@@ -72,7 +72,8 @@ ENTRY(system_call)
 	movel	%d2,%a0
 	movel	%a0@,%a1		/* save top of frame */
 	movel	%sp,%a1@(TASK_THREAD+THREAD_ESP0)
-	btst	#(TIF_SYSCALL_TRACE%8),%a0@(TINFO_FLAGS+(31-TIF_SYSCALL_TRACE)/8)
+	moveb	%a0@(TINFO_FLAGS+2),%d2
+	andl	#((_TIF_SYSCALL_TRACE + _TIF_SECCOMP) >> 8),%d2
 	bnes	1f
 
 	movel	%d3,%a0
diff --git a/arch/m68k/coldfire/m5407.c b/arch/m68k/coldfire/m5407.c
index 23a14416241c86..1f3e30b3339826 100644
--- a/arch/m68k/coldfire/m5407.c
+++ b/arch/m68k/coldfire/m5407.c
@@ -22,6 +22,11 @@
 
 /***************************************************************************/
 
+/*
+ *	Some platforms need software versions of the GPIO data registers.
+ */
+unsigned short ppdata;
+
 DEFINE_CLK(pll, "pll.0", MCF_CLK);
 DEFINE_CLK(sys, "sys.0", MCF_BUSCLK);
 
diff --git a/arch/m68k/coldfire/reset.c b/arch/m68k/coldfire/reset.c
index 6e5f8ab39f324a..079c92593eff0c 100644
--- a/arch/m68k/coldfire/reset.c
+++ b/arch/m68k/coldfire/reset.c
@@ -16,7 +16,7 @@
 #include <asm/mcfsim.h>
 
 /*
- *	There are 2 common methods amongst the ColdFure parts for reseting
+ *	There are 2 common methods amongst the ColdFire parts for resetting
  *	the CPU. But there are couple of exceptions, the 5272 and the 547x
  *	have something completely special to them, and we let their specific
  *	subarch code handle them.
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index 8b7efe20dd1285..fefcaa5ad44c32 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -284,6 +284,7 @@ CONFIG_ATA=y
 # CONFIG_ATA_VERBOSE_ERROR is not set
 # CONFIG_ATA_BMDMA is not set
 CONFIG_PATA_GAYLE=y
+CONFIG_PATA_CSWARP=y
 CONFIG_PATA_BUDDHA=y
 CONFIG_MD=y
 CONFIG_MD_LINEAR=m
@@ -551,6 +552,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -578,6 +581,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index c7f7705806f97c..84d4a6ed1707c6 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -506,6 +506,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -533,6 +535,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index 2fb213161baec9..22d8a85816015a 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -25,6 +25,7 @@ CONFIG_NFBLOCK=y
 CONFIG_NFCON=y
 CONFIG_NFETH=y
 CONFIG_ATARI_ETHERNAT=y
+CONFIG_ATARI_SVETHLANA=y
 CONFIG_ATARI_ETHERNEC=y
 CONFIG_ATARI_DSP56K=m
 CONFIG_MODULES=y
@@ -328,6 +329,7 @@ CONFIG_TUN=m
 CONFIG_VETH=m
 CONFIG_ATARILANCE=y
 CONFIG_NE2000=y
+CONFIG_ETHOC=y
 CONFIG_SMC91X=y
 CONFIG_PPP=m
 CONFIG_PPP_BSDCOMP=m
@@ -529,6 +531,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -556,6 +560,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index 6730242d4d4861..0f79c6382d3548 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -498,6 +498,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -525,6 +527,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 2b3ff26ac8fdff..2bf412a1713c75 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -508,6 +508,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -535,6 +537,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index 61f32ef2547e05..5496f1b6053191 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -527,6 +527,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -554,6 +556,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index 2cbd6bf20a6873..a8f593b4fc6a73 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -38,6 +38,7 @@ CONFIG_NFBLOCK=y
 CONFIG_NFCON=y
 CONFIG_NFETH=y
 CONFIG_ATARI_ETHERNAT=y
+CONFIG_ATARI_SVETHLANA=y
 CONFIG_ATARI_ETHERNEC=y
 CONFIG_ATARI_DSP56K=m
 CONFIG_AMIGA_BUILTIN_SERIAL=y
@@ -310,6 +311,7 @@ CONFIG_ATA=y
 # CONFIG_ATA_BMDMA is not set
 CONFIG_PATA_FALCON=y
 CONFIG_PATA_GAYLE=y
+CONFIG_PATA_CSWARP=y
 CONFIG_PATA_BUDDHA=y
 CONFIG_PATA_PLATFORM=y
 CONFIG_MD=y
@@ -382,6 +384,7 @@ CONFIG_MAC8390=y
 CONFIG_NE2000=y
 CONFIG_APNE=y
 CONFIG_ZORRO8390=y
+CONFIG_ETHOC=y
 CONFIG_SMC91X=y
 CONFIG_PLIP=m
 CONFIG_PPP=m
@@ -614,6 +617,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -641,6 +646,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index 019f8b8070be8d..bbce43b8f7aa4c 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -498,6 +498,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -525,6 +527,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 38916328742582..a63d09dc8dbd39 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -499,6 +499,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -526,6 +528,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index cbc41443cffa6e..d6b7cbab170048 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -517,6 +517,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -544,6 +546,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/stmark2_defconfig b/arch/m68k/configs/stmark2_defconfig
index cd454369bfe8df..d303b21949a771 100644
--- a/arch/m68k/configs/stmark2_defconfig
+++ b/arch/m68k/configs/stmark2_defconfig
@@ -76,6 +76,8 @@ CONFIG_DMADEVICES=y
 CONFIG_MCF_EDMA=y
 # CONFIG_VIRTIO_MENU is not set
 # CONFIG_VHOST_MENU is not set
+CONFIG_IIO=y
+CONFIG_MCF54415_DAC=y
 CONFIG_EXT2_FS=y
 CONFIG_EXT2_FS_XATTR=y
 CONFIG_EXT2_FS_POSIX_ACL=y
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index 273e840dc8d6a0..bee5cdd2063757 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -496,6 +496,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -522,6 +524,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index 843d72924bcb20..ef2b76a064391d 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -496,6 +496,8 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_PRIME_NUMBERS=m
 CONFIG_CRC_ENABLE_ALL_FOR_KUNIT=m
 CONFIG_CRC_BENCHMARK=y
+CONFIG_XOR_BENCHMARK=y
+CONFIG_RAID6_PQ_KUNIT_BENCHMARK=y
 CONFIG_XZ_DEC_TEST=m
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
 CONFIG_MAGIC_SYSRQ=y
@@ -523,6 +525,7 @@ CONFIG_TEST_VMALLOC=m
 CONFIG_TEST_WORKQUEUE=m
 CONFIG_TEST_BPF=m
 CONFIG_FIND_BIT_BENCHMARK=m
+CONFIG_REGION_ALLOC_BENCHMARK=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_LINEAR_RANGES_TEST=m
diff --git a/arch/m68k/include/asm/atariints.h b/arch/m68k/include/asm/atariints.h
index 6321c44956206f..cb15361d14e91e 100644
--- a/arch/m68k/include/asm/atariints.h
+++ b/arch/m68k/include/asm/atariints.h
@@ -32,7 +32,7 @@
 #define VME_SOURCE_BASE    56
 #define VME_MAX_SOURCES    16
 
-#define NUM_ATARI_SOURCES  141
+#define NUM_ATARI_SOURCES  142
 
 /* convert vector number to int source number */
 #define IRQ_VECTOR_TO_SOURCE(v)	((v) - ((v) < 0x20 ? 0x18 : (0x40-8)))
diff --git a/arch/m68k/include/asm/irq.h b/arch/m68k/include/asm/irq.h
index 2263e92d418ab7..53cbe6db46919a 100644
--- a/arch/m68k/include/asm/irq.h
+++ b/arch/m68k/include/asm/irq.h
@@ -10,8 +10,8 @@
  * different m68k hosts compiled into the kernel.
  * Currently the Atari has 72 and the Amiga 24, but if both are
  * supported in the kernel it is better to make room for 72.
- * With EtherNAT add-on card on Atari, the highest interrupt
- * number is 140 so NR_IRQS needs to be 141.
+ * With the SVEthLANa adapter in the SuperVidel on Atari, the highest
+ * interrupt number is 141 so NR_IRQS needs to be 142.
  */
 #if defined(CONFIG_COLDFIRE)
 #define NR_IRQS 256
@@ -19,7 +19,7 @@
       defined(CONFIG_SUN3X) || defined(CONFIG_VIRT)
 #define NR_IRQS 200
 #elif defined(CONFIG_ATARI)
-#define NR_IRQS 141
+#define NR_IRQS 142
 #elif defined(CONFIG_MAC)
 #define NR_IRQS 72
 #elif defined(CONFIG_Q40)
diff --git a/arch/m68k/include/asm/m53xxacr.h b/arch/m68k/include/asm/m53xxacr.h
index 692f90e7fecc13..a29ad33aaf0555 100644
--- a/arch/m68k/include/asm/m53xxacr.h
+++ b/arch/m68k/include/asm/m53xxacr.h
@@ -32,7 +32,7 @@
 #define CACR_DCM_PRE	0x00000200	/* Cache inhibited, precise */
 #define CACR_DCM_IMPRE	0x00000300	/* Cache inhibited, imprecise */
 #define CACR_WPROTECT	0x00000020	/* Write protect*/
-#define CACR_EUSP	0x00000010	/* Eanble separate user a7 */
+#define CACR_EUSP	0x00000010	/* Enable separate user a7 */
 
 /*
  * Define the Access Control register flags.
diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h
index fc5454d37da318..b53ff0950db2e3 100644
--- a/arch/m68k/include/asm/mcf_pgalloc.h
+++ b/arch/m68k/include/asm/mcf_pgalloc.h
@@ -39,10 +39,7 @@ extern inline pmd_t *pmd_alloc_kernel(pgd_t *pgd, unsigned long address)
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable,
 				  unsigned long address)
 {
-	struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
-
-	pagetable_dtor(ptdesc);
-	pagetable_free(ptdesc);
+	tlb_remove_ptdesc(tlb, virt_to_ptdesc(pgtable));
 }
 
 static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
diff --git a/arch/m68k/include/asm/mcf_pgtable.h b/arch/m68k/include/asm/mcf_pgtable.h
index 189bb7b1e6630f..f45a882238dbb6 100644
--- a/arch/m68k/include/asm/mcf_pgtable.h
+++ b/arch/m68k/include/asm/mcf_pgtable.h
@@ -137,12 +137,6 @@ static inline int pmd_bad2(pmd_t *pmd) { return 0; }
 #define pmd_present(pmd) (!pmd_none2(&(pmd)))
 static inline void pmd_clear(pmd_t *pmdp) { pmd_val(*pmdp) = 0; }
 
-#define pte_ERROR(e) \
-	printk(KERN_ERR "%s:%d: bad pte %08lx.\n",	\
-	__FILE__, __LINE__, pte_val(e))
-#define pgd_ERROR(e) \
-	printk(KERN_ERR "%s:%d: bad pgd %08lx.\n",	\
-	__FILE__, __LINE__, pgd_val(e))
 
 /*
  * The following only work if pte_present() is true.
diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
index 1091fb0affbee4..dcde40e8b5c6a1 100644
--- a/arch/m68k/include/asm/motorola_pgalloc.h
+++ b/arch/m68k/include/asm/motorola_pgalloc.h
@@ -17,6 +17,7 @@ enum m68k_table_types {
 extern void init_pointer_table(void *table, int type);
 extern void *get_pointer_table(struct mm_struct *mm, int type);
 extern int free_pointer_table(void *table, int type);
+extern void __tlb_remove_table(void *table);
 
 /*
  * Allocate and free page tables. The xxx_kernel() versions are
@@ -47,7 +48,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable)
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable,
 				  unsigned long address)
 {
-	free_pointer_table(pgtable, TABLE_PTE);
+	tlb_remove_table(tlb, (void *)((unsigned long)pgtable | TABLE_PTE));
 }
 
 
@@ -61,10 +62,10 @@ static inline int pmd_free(struct mm_struct *mm, pmd_t *pmd)
 	return free_pointer_table(pmd, TABLE_PMD);
 }
 
-static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
-				 unsigned long address)
+static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
+				  unsigned long address)
 {
-	return free_pointer_table(pmd, TABLE_PMD);
+	tlb_remove_table(tlb, (void *)((unsigned long)pmd | TABLE_PMD));
 }
 
 
diff --git a/arch/m68k/include/asm/motorola_pgtable.h b/arch/m68k/include/asm/motorola_pgtable.h
index dcf6829b3eab97..d9393b310add6a 100644
--- a/arch/m68k/include/asm/motorola_pgtable.h
+++ b/arch/m68k/include/asm/motorola_pgtable.h
@@ -131,14 +131,6 @@ static inline void pud_set(pud_t *pudp, pmd_t *pmdp)
 #define pud_clear(pudp)		({ pud_val(*pudp) = 0; })
 #define pud_page(pud)		(mem_map + ((unsigned long)(__va(pud_val(pud)) - PAGE_OFFSET) >> PAGE_SHIFT))
 
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
-#define pmd_ERROR(e) \
-	printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
-
-
 /*
  * The following only work if pte_present() is true.
  * Undefined behaviour if not..
diff --git a/arch/m68k/include/asm/quicc_simple.h b/arch/m68k/include/asm/quicc_simple.h
deleted file mode 100644
index b9e2808b44ac9b..00000000000000
--- a/arch/m68k/include/asm/quicc_simple.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/***********************************
- * $Id: quicc_simple.h,v 1.1 2002/03/02 15:01:10 gerg Exp $
- ***********************************
- *
- ***************************************
- * Simple drivers common header
- ***************************************
- */
-
-#ifndef __SIMPLE_H
-#define __SIMPLE_H
-
-/* #include "quicc.h" */
-
-#define GLB_SCC_0   0
-#define GLB_SCC_1   1
-#define GLB_SCC_2   2
-#define GLB_SCC_3   3
-
-typedef void (int_routine)(unsigned short interrupt_event);
-typedef int_routine *int_routine_ptr; 
-typedef void *(alloc_routine)(int length);
-typedef void (free_routine)(int scc_num, int channel_num, void *buf);
-typedef void (store_rx_buffer_routine)(int scc_num, int channel_num, void *buff, int length);
-typedef int  (handle_tx_error_routine)(int scc_num, int channel_num, QUICC_BD *tbd);
-typedef void (handle_rx_error_routine)(int scc_num, int channel_num, QUICC_BD *rbd);
-typedef void (handle_lost_error_routine)(int scc_num, int channel_num);
-
-/* user defined functions for global errors */
-typedef void (handle_glob_overrun_routine)(int scc_number);
-typedef void (handle_glob_underrun_routine)(int scc_number);
-typedef void (glob_intr_q_overflow_routine)(int scc_number);
-
-/*
- * General initialization and command routines
- */
-void quicc_issue_cmd (unsigned short cmd, int scc_num);
-void quicc_init(void);
-void quicc_scc_init(int scc_number, int number_of_rx_buf, int number_of_tx_buf);
-void quicc_smc_init(int smc_number, int number_of_rx_buf, int number_of_tx_buf);
-void quicc_scc_start(int scc_num);
-void quicc_scc_loopback(int scc_num);
-
-/* Interrupt enable/disable routines for critical pieces of code*/
-unsigned short  IntrDis(void);
-void            IntrEna(unsigned short old_sr);
-
-/* For debugging */
-void print_rbd(int scc_num);
-void print_tbd(int scc_num);
-
-#endif
diff --git a/arch/m68k/include/asm/serial.h b/arch/m68k/include/asm/serial.h
index 3f76bc7b9edb2f..866ef25a74a207 100644
--- a/arch/m68k/include/asm/serial.h
+++ b/arch/m68k/include/asm/serial.h
@@ -1,21 +1,8 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/*
- * include/asm-m68k/serial.h
- *
- * currently this seems useful only for a Q40,
- * it's an almost exact copy of ../asm-alpha/serial.h
- *
- */
+#ifndef _ASM_M68K_SERIAL_H
+#define _ASM_M68K_SERIAL_H
 
-
-/*
- * This assumes you have a 1.8432 MHz clock for your UART.
- *
- * It'd be nice if someone built a serial card with a 24.576 MHz
- * clock, since the 16550A is capable of handling a top speed of 1.5
- * megabits/second; but this requires the faster clock.
- */
-#define BASE_BAUD ( 1843200 / 16 )
+#include <asm-generic/serial.h>
 
 /* Standard COM flags (except for COM4, because of the 8514 problem) */
 #ifdef CONFIG_SERIAL_8250_DETECT_IRQ
@@ -34,3 +21,5 @@
 	{ 0, BASE_BAUD, 0x3E8, 4, STD_COM_FLAGS },	/* ttyS2 */	\
 	{ 0, BASE_BAUD, 0x2E8, 3, STD_COM4_FLAGS },	/* ttyS3 */
 #endif
+
+#endif /* _ASM_M68K_SERIAL_H */
diff --git a/arch/m68k/include/asm/sun3_pgtable.h b/arch/m68k/include/asm/sun3_pgtable.h
index 80ca185a18a193..704442a391fd74 100644
--- a/arch/m68k/include/asm/sun3_pgtable.h
+++ b/arch/m68k/include/asm/sun3_pgtable.h
@@ -119,13 +119,6 @@ static inline int pmd_present2 (pmd_t *pmd) { return pmd_val (*pmd) & SUN3_PMD_V
 #define pmd_present(pmd) (!pmd_none2(&(pmd)))
 static inline void pmd_clear (pmd_t *pmdp) { pmd_val (*pmdp) = 0; }
 
-
-#define pte_ERROR(e) \
-	pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
-
-
 /*
  * The following only work if pte_present() is true.
  * Undefined behaviour if not...
diff --git a/arch/m68k/include/uapi/asm/setup.h b/arch/m68k/include/uapi/asm/setup.h
index 25fe26d5597cc6..2d5b24a5345f92 100644
--- a/arch/m68k/include/uapi/asm/setup.h
+++ b/arch/m68k/include/uapi/asm/setup.h
@@ -12,6 +12,10 @@
 #ifndef _UAPI_M68K_SETUP_H
 #define _UAPI_M68K_SETUP_H
 
-#define COMMAND_LINE_SIZE 256
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
+#define COMMAND_LINE_SIZE	256
+#endif
 
 #endif /* _UAPI_M68K_SETUP_H */
diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c
index c0033f885ed4f0..b745cf20a71f9b 100644
--- a/arch/m68k/mac/config.c
+++ b/arch/m68k/mac/config.c
@@ -984,8 +984,37 @@ static int __init mac_platform_init(void)
 			.start = swim_base,
 			.end   = swim_base + 0x1FFF,
 		};
+		bool fast_fclk = false;
+
+		/* Models in this list are supposed to run FCLK at 32 MHz */
+		switch (macintosh_config->ident) {
+		case MAC_MODEL_CCL:
+		case MAC_MODEL_CCLII:
+		case MAC_MODEL_LCIII:
+		case MAC_MODEL_Q605:
+		case MAC_MODEL_Q605_ACC:
+		case MAC_MODEL_Q610:
+		case MAC_MODEL_Q630:
+		case MAC_MODEL_Q650:
+		case MAC_MODEL_Q800:
+		case MAC_MODEL_P460:
+		case MAC_MODEL_P475:
+		case MAC_MODEL_P475F:
+		case MAC_MODEL_P520:
+		case MAC_MODEL_P550:
+		case MAC_MODEL_P575:
+		case MAC_MODEL_P588:
+		case MAC_MODEL_TV:
+		case MAC_MODEL_C610:
+		case MAC_MODEL_C650:
+		case MAC_MODEL_PB190:
+		case MAC_MODEL_PB520:
+			fast_fclk = true;
+			break;
+		}
 
-		platform_device_register_simple("swim", -1, &swim_rsrc, 1);
+		platform_device_register_resndata(NULL, "swim", -1, &swim_rsrc, 1,
+						  &fast_fclk, sizeof(fast_fclk));
 	}
 
 	/*
diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c
index b30aa69a73a6ad..f3efa0d139634c 100644
--- a/arch/m68k/mm/motorola.c
+++ b/arch/m68k/mm/motorola.c
@@ -20,6 +20,7 @@
 #include <linux/init.h>
 #include <linux/memblock.h>
 #include <linux/gfp.h>
+#include <linux/cleanup.h>
 
 #include <asm/setup.h>
 #include <linux/uaccess.h>
@@ -103,6 +104,8 @@ static struct list_head ptable_list[3] = {
 	LIST_HEAD_INIT(ptable_list[2]),
 };
 
+static DEFINE_SPINLOCK(ptable_lock);
+
 #define PD_PTABLE(ptdesc) ((ptable_desc *)&(virt_to_ptdesc((void *)(ptdesc))->pt_list))
 #define PD_PTDESC(ptable) (list_entry(ptable, struct ptdesc, pt_list))
 #define PD_MARKBITS(dp) (*(unsigned int *)&PD_PTDESC(dp)->pt_index)
@@ -139,52 +142,65 @@ void __init init_pointer_table(void *table, int type)
 	return;
 }
 
-void *get_pointer_table(struct mm_struct *mm, int type)
+/*
+ * For a pointer table for a user process address space, a
+ * table is taken from a ptdesc allocated for the purpose.  Each
+ * ptdesc can hold 8 pointer tables.  The ptdesc is remapped in
+ * virtual address space to be noncacheable.
+ */
+static void *add_pointer_table(struct mm_struct *mm, int type)
 {
-	ptable_desc *dp = ptable_list[type].next;
-	unsigned int mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp);
-	unsigned int tmp, off;
+	struct ptdesc *ptdesc;
+	ptable_desc *new;
+	void *pt_addr;
 
-	/*
-	 * For a pointer table for a user process address space, a
-	 * table is taken from a ptdesc allocated for the purpose.  Each
-	 * ptdesc can hold 8 pointer tables.  The ptdesc is remapped in
-	 * virtual address space to be noncacheable.
-	 */
-	if (mask == 0) {
-		struct ptdesc *ptdesc;
-		ptable_desc *new;
-		void *pt_addr;
-
-		ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0);
-		if (!ptdesc)
-			return NULL;
-
-		pt_addr = ptdesc_address(ptdesc);
-
-		switch (type) {
-		case TABLE_PTE:
-			/*
-			 * m68k doesn't have SPLIT_PTE_PTLOCKS for not having
-			 * SMP.
-			 */
-			pagetable_pte_ctor(mm, ptdesc);
-			break;
-		case TABLE_PMD:
-			pagetable_pmd_ctor(mm, ptdesc);
-			break;
-		case TABLE_PGD:
-			pagetable_pgd_ctor(ptdesc);
-			break;
-		}
+	ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0);
+	if (!ptdesc)
+		return NULL;
+
+	pt_addr = ptdesc_address(ptdesc);
+
+	switch (type) {
+	case TABLE_PTE:
+		/*
+		 * m68k doesn't have SPLIT_PTE_PTLOCKS for not having
+		 * SMP.
+		 */
+		pagetable_pte_ctor(mm, ptdesc);
+		break;
+	case TABLE_PMD:
+		pagetable_pmd_ctor(mm, ptdesc);
+		break;
+	case TABLE_PGD:
+		pagetable_pgd_ctor(ptdesc);
+		break;
+	}
+
+	mmu_page_ctor(pt_addr);
+
+	new = PD_PTABLE(pt_addr);
 
-		mmu_page_ctor(pt_addr);
+	PD_MARKBITS(new) = ptable_mask(type) - 1;
+	scoped_guard(spinlock_bh, &ptable_lock)
+		list_add(new, &ptable_list[type]);
 
-		new = PD_PTABLE(pt_addr);
-		PD_MARKBITS(new) = ptable_mask(type) - 1;
-		list_add_tail(new, dp);
+	return (pmd_t *)pt_addr;
+}
+
+void *get_pointer_table(struct mm_struct *mm, int type)
+{
+	unsigned int tmp, off;
+	unsigned long mask;
+	ptable_desc *dp;
+	void *ret;
 
-		return (pmd_t *)pt_addr;
+	spin_lock_bh(&ptable_lock);
+	dp = ptable_list[type].next;
+	mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp);
+
+	if (mask == 0) {
+		spin_unlock_bh(&ptable_lock);
+		return add_pointer_table(mm, type);
 	}
 
 	for (tmp = 1, off = 0; (mask & tmp) == 0; tmp <<= 1, off += ptable_size(type))
@@ -194,7 +210,10 @@ void *get_pointer_table(struct mm_struct *mm, int type)
 		/* move to end of list */
 		list_move_tail(dp, &ptable_list[type]);
 	}
-	return ptdesc_address(PD_PTDESC(dp)) + off;
+
+	ret = ptdesc_address(PD_PTDESC(dp)) + off;
+	spin_unlock_bh(&ptable_lock);
+	return ret;
 }
 
 int free_pointer_table(void *table, int type)
@@ -204,6 +223,8 @@ int free_pointer_table(void *table, int type)
 	unsigned long pt_addr = ptable & PAGE_MASK;
 	unsigned int mask = 1U << ((ptable - pt_addr)/ptable_size(type));
 
+	spin_lock_bh(&ptable_lock);
+
 	dp = PD_PTABLE(pt_addr);
 	if (PD_MARKBITS (dp) & mask)
 		panic ("table already free!");
@@ -213,6 +234,8 @@ int free_pointer_table(void *table, int type)
 	if (PD_MARKBITS(dp) == ptable_mask(type)) {
 		/* all tables in ptdesc are free, free ptdesc */
 		list_del(dp);
+		spin_unlock_bh(&ptable_lock);
+
 		mmu_page_dtor((void *)pt_addr);
 		pagetable_dtor_free(virt_to_ptdesc((void *)pt_addr));
 		return 1;
@@ -223,9 +246,21 @@ int free_pointer_table(void *table, int type)
 		 */
 		list_move(dp, &ptable_list[type]);
 	}
+
+	spin_unlock_bh(&ptable_lock);
 	return 0;
 }
 
+void __tlb_remove_table(void *table)
+{
+	/* The bottom 2 bits are used to encode page table type. */
+	const unsigned long encoded = (unsigned long)table;
+	void *addr = (void *)(encoded & ~3UL);
+	const int type = encoded & 3;
+
+	free_pointer_table(addr, type);
+}
+
 /* size of memory already mapped in head.S */
 extern __initdata unsigned long m68k_init_mapped_size;
 
diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig
index 484ebb3baedf15..0ca8999dd7709d 100644
--- a/arch/microblaze/Kconfig
+++ b/arch/microblaze/Kconfig
@@ -216,3 +216,12 @@ config MB_MANAGER
 	  Say N here unless you know what you are doing.
 
 endmenu
+
+config MB_POWER_OFF_THROUGH_UNALIGNED_PC
+	bool "Power off through unaligned PC"
+	help
+	  This options adds a power off handler which executes an unaligned PC
+	  so the machine resets in a generic way. This works for the
+	  petalogix-s3adsp1800 QEMU machine.
+
+	  Say N here unless you know what you are doing.
diff --git a/arch/microblaze/Makefile b/arch/microblaze/Makefile
index 02e6be9c5b0dca..01085dd5e6f871 100644
--- a/arch/microblaze/Makefile
+++ b/arch/microblaze/Makefile
@@ -16,7 +16,7 @@ export CPU_VER CPU_MAJOR CPU_MINOR CPU_REV
 # The various CONFIG_XILINX cpu features options are integers 0/1/2...
 # rather than bools y/n
 
-# Work out HW multipler support. This is tricky.
+# Work out HW multiplier support. This is tricky.
 # 1. Spartan2 has no HW multipliers.
 # 2. MicroBlaze v3.x always uses them, except in Spartan 2
 # 3. All other FPGa/CPU ver combos, we can trust the CONFIG_ settings
@@ -34,12 +34,10 @@ CPUFLAGS-$(CONFIG_XILINX_MICROBLAZE0_USE_BARREL) += -mxl-barrel-shift
 CPUFLAGS-$(CONFIG_XILINX_MICROBLAZE0_USE_PCMP_INSTR) += -mxl-pattern-compare
 
 ifdef CONFIG_CPU_BIG_ENDIAN
-KBUILD_CFLAGS += -mbig-endian
-KBUILD_AFLAGS += -mbig-endian
+KBUILD_CPPFLAGS += -mbig-endian
 KBUILD_LDFLAGS += -EB
 else
-KBUILD_CFLAGS += -mlittle-endian
-KBUILD_AFLAGS += -mlittle-endian
+KBUILD_CPPFLAGS += -mlittle-endian
 KBUILD_LDFLAGS += -EL
 endif
 
diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig
index fbbdcb394ca24e..7f9b19d0d9dbf5 100644
--- a/arch/microblaze/configs/mmu_defconfig
+++ b/arch/microblaze/configs/mmu_defconfig
@@ -70,6 +70,8 @@ CONFIG_XILINX_WATCHDOG=y
 CONFIG_FB=y
 CONFIG_FB_XILINX=y
 # CONFIG_USB_SUPPORT is not set
+CONFIG_DMADEVICES=y
+CONFIG_XILINX_DMA=y
 CONFIG_UIO=y
 CONFIG_UIO_PDRV_GENIRQ=y
 CONFIG_UIO_DMEM_GENIRQ=y
diff --git a/arch/microblaze/include/asm/entry.h b/arch/microblaze/include/asm/entry.h
index 9efadf12397ca8..05acf4d7bc4b18 100644
--- a/arch/microblaze/include/asm/entry.h
+++ b/arch/microblaze/include/asm/entry.h
@@ -21,6 +21,19 @@
 
 #define PER_CPU(var) var
 
+/*
+ * The MicroBlaze ABI has the caller reserve an argument home area:
+ * REG_PARM_STACK_SPACE is 24 and OUTGOING_REG_PARM_STACK_SPACE is 1, so a
+ * callee may write to [caller_sp + 4, caller_sp + 28).  The kernel calls C
+ * with r1 at the frame base, so reserve that area below pt_regs and reach
+ * the saved registers through PTO.  This restores what 6e83557c38b4
+ * ("microblaze: Remove r0_ram pointer and PTO alignment") removed; the old
+ * value of 24 was one word short and left the last argument slot
+ * overlapping pt_regs' r0.
+ */
+#define PTO			28
+#define STATE_SAVE_SIZE		(PT_SIZE + PTO)
+
 # ifndef __ASSEMBLER__
 DECLARE_PER_CPU(unsigned int, KSP); /* Saved kernel stack pointer */
 DECLARE_PER_CPU(unsigned int, KM); /* Kernel/user mode */
diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h
index 084a8a0dc23952..ffee6a009219ac 100644
--- a/arch/microblaze/include/asm/pgalloc.h
+++ b/arch/microblaze/include/asm/pgalloc.h
@@ -25,7 +25,7 @@ extern void __bad_pte(pmd_t *pmd);
 
 extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm);
 
-#define __pte_free_tlb(tlb, pte, addr)	pte_free((tlb)->mm, (pte))
+#define __pte_free_tlb(tlb, pte, addr)	tlb_remove_ptdesc((tlb), page_ptdesc(pte))
 
 #define pmd_populate(mm, pmd, pte) \
 			(pmd_val(*(pmd)) = (unsigned long)page_address(pte))
diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h
index 7678c040a2fd36..b10273a3d0a4cf 100644
--- a/arch/microblaze/include/asm/pgtable.h
+++ b/arch/microblaze/include/asm/pgtable.h
@@ -14,6 +14,7 @@
 extern int mem_init_done;
 #endif
 
+#include <asm/page.h>
 #include <asm-generic/pgtable-nopmd.h>
 
 #ifdef __KERNEL__
@@ -23,7 +24,6 @@ extern int mem_init_done;
 #include <linux/threads.h>
 #include <asm/processor.h>		/* For TASK_SIZE */
 #include <asm/mmu.h>
-#include <asm/page.h>
 
 extern unsigned long va_to_phys(unsigned long address);
 extern pte_t *va_to_pte(unsigned long address);
@@ -103,13 +103,6 @@ extern pte_t *va_to_pte(unsigned long address);
 #define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT)
 #define KERNEL_PGD_PTRS (PTRS_PER_PGD-USER_PGD_PTRS)
 
-#define pte_ERROR(e) \
-	printk(KERN_ERR "%s:%d: bad pte "PTE_FMT".\n", \
-		__FILE__, __LINE__, pte_val(e))
-#define pgd_ERROR(e) \
-	printk(KERN_ERR "%s:%d: bad pgd %08lx.\n", \
-		__FILE__, __LINE__, pgd_val(e))
-
 /*
  * Bits in a linux-style PTE.  These match the bits in the
  * (hardware-defined) PTE as closely as possible.
diff --git a/arch/microblaze/include/asm/processor.h b/arch/microblaze/include/asm/processor.h
index d59bdfffca7cc0..595c24db272838 100644
--- a/arch/microblaze/include/asm/processor.h
+++ b/arch/microblaze/include/asm/processor.h
@@ -73,7 +73,7 @@ unsigned long __get_wchan(struct task_struct *p);
 #  define task_regs(task) ((struct pt_regs *)task_tos(task) - 1)
 
 #  define task_pt_regs_plus_args(tsk) \
-	((void *)task_pt_regs(tsk))
+	(((void *)task_pt_regs(tsk)) - PTO)
 
 #  define task_sp(task)	(task_regs(task)->r1)
 #  define task_pc(task)	(task_regs(task)->pc)
diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h
index afa0dd8d013fbc..77203af255e5f9 100644
--- a/arch/microblaze/include/asm/uaccess.h
+++ b/arch/microblaze/include/asm/uaccess.h
@@ -95,7 +95,8 @@ extern long __user_bad(void);
 #define get_user(x, ptr) ({				\
 	const typeof(*(ptr)) __user *__gu_ptr = (ptr);	\
 	access_ok(__gu_ptr, sizeof(*__gu_ptr)) ?	\
-		__get_user(x, __gu_ptr) : -EFAULT;	\
+		__get_user(x, __gu_ptr) :		\
+		((x) = 0, -EFAULT);			\
 })
 
 #define __get_user(x, ptr)						\
diff --git a/arch/microblaze/include/uapi/asm/setup.h b/arch/microblaze/include/uapi/asm/setup.h
index 16c56807f86a2d..e4b253064c7d99 100644
--- a/arch/microblaze/include/uapi/asm/setup.h
+++ b/arch/microblaze/include/uapi/asm/setup.h
@@ -12,6 +12,10 @@
 #ifndef _UAPI_ASM_MICROBLAZE_SETUP_H
 #define _UAPI_ASM_MICROBLAZE_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	256
+#endif
 
 #endif /* _UAPI_ASM_MICROBLAZE_SETUP_H */
diff --git a/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c b/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c
index c7ee51b0900e69..1cc3fd9846ad1e 100644
--- a/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c
+++ b/arch/microblaze/kernel/cpu/cpuinfo-pvr-full.c
@@ -23,7 +23,7 @@
 #define CI(c, p) { ci->c = PVR_##p(pvr); }
 
 #define err_printk(x) \
-	pr_err("ERROR: Microblaze " x "-different for PVR and DTS\n");
+	pr_err("ERROR: Microblaze " x "-different for PVR and DTS\n")
 
 void set_cpuinfo_pvr_full(struct cpuinfo *ci, struct device_node *cpu)
 {
diff --git a/arch/microblaze/kernel/cpu/cpuinfo-static.c b/arch/microblaze/kernel/cpu/cpuinfo-static.c
index 03da36dc6d9c92..1e4d30b6cb01a8 100644
--- a/arch/microblaze/kernel/cpu/cpuinfo-static.c
+++ b/arch/microblaze/kernel/cpu/cpuinfo-static.c
@@ -18,7 +18,7 @@ static const char family_string[] = CONFIG_XILINX_MICROBLAZE0_FAMILY;
 static const char cpu_ver_string[] = CONFIG_XILINX_MICROBLAZE0_HW_VER;
 
 #define err_printk(x) \
-	pr_err("ERROR: Microblaze " x "-different for kernel and DTS\n");
+	pr_err("ERROR: Microblaze " x "-different for kernel and DTS\n")
 
 void __init set_cpuinfo_static(struct cpuinfo *ci, struct device_node *cpu)
 {
diff --git a/arch/microblaze/kernel/cpu/cpuinfo.c b/arch/microblaze/kernel/cpu/cpuinfo.c
index cd9b4450763b0b..32c508e85408ac 100644
--- a/arch/microblaze/kernel/cpu/cpuinfo.c
+++ b/arch/microblaze/kernel/cpu/cpuinfo.c
@@ -133,6 +133,7 @@ void __init setup_cpuinfo_clk(void)
 		cpuinfo.cpu_clock_freq = fcpu(cpu, "timebase-frequency");
 	} else {
 		cpuinfo.cpu_clock_freq = clk_get_rate(clk);
+		clk_put(clk);
 	}
 
 	if (!cpuinfo.cpu_clock_freq) {
diff --git a/arch/microblaze/kernel/entry.S b/arch/microblaze/kernel/entry.S
index 582d7256d81547..ebc8bcb98c5220 100644
--- a/arch/microblaze/kernel/entry.S
+++ b/arch/microblaze/kernel/entry.S
@@ -177,78 +177,78 @@ syscall_debug_table:
 1:
 
 #define SAVE_REGS \
-	swi	r2, r1, PT_R2;	/* Save SDA */			\
-	swi	r3, r1, PT_R3;					\
-	swi	r4, r1, PT_R4;					\
-	swi	r5, r1, PT_R5;					\
-	swi	r6, r1, PT_R6;					\
-	swi	r7, r1, PT_R7;					\
-	swi	r8, r1, PT_R8;					\
-	swi	r9, r1, PT_R9;					\
-	swi	r10, r1, PT_R10;					\
-	swi	r11, r1, PT_R11;	/* save clobbered regs after rval */\
-	swi	r12, r1, PT_R12;					\
-	swi	r13, r1, PT_R13;	/* Save SDA2 */			\
-	swi	r14, r1, PT_PC;	/* PC, before IRQ/trap */	\
-	swi	r15, r1, PT_R15;	/* Save LP */			\
-	swi	r16, r1, PT_R16;					\
-	swi	r17, r1, PT_R17;					\
-	swi	r18, r1, PT_R18;	/* Save asm scratch reg */	\
-	swi	r19, r1, PT_R19;					\
-	swi	r20, r1, PT_R20;					\
-	swi	r21, r1, PT_R21;					\
-	swi	r22, r1, PT_R22;					\
-	swi	r23, r1, PT_R23;					\
-	swi	r24, r1, PT_R24;					\
-	swi	r25, r1, PT_R25;					\
-	swi	r26, r1, PT_R26;					\
-	swi	r27, r1, PT_R27;					\
-	swi	r28, r1, PT_R28;					\
-	swi	r29, r1, PT_R29;					\
-	swi	r30, r1, PT_R30;					\
-	swi	r31, r1, PT_R31;	/* Save current task reg */	\
+	swi	r2, r1, PTO+PT_R2;	/* Save SDA */			\
+	swi	r3, r1, PTO+PT_R3;					\
+	swi	r4, r1, PTO+PT_R4;					\
+	swi	r5, r1, PTO+PT_R5;					\
+	swi	r6, r1, PTO+PT_R6;					\
+	swi	r7, r1, PTO+PT_R7;					\
+	swi	r8, r1, PTO+PT_R8;					\
+	swi	r9, r1, PTO+PT_R9;					\
+	swi	r10, r1, PTO+PT_R10;					\
+	swi	r11, r1, PTO+PT_R11;	/* save clobbered regs after rval */\
+	swi	r12, r1, PTO+PT_R12;					\
+	swi	r13, r1, PTO+PT_R13;	/* Save SDA2 */			\
+	swi	r14, r1, PTO+PT_PC;	/* PC, before IRQ/trap */	\
+	swi	r15, r1, PTO+PT_R15;	/* Save LP */			\
+	swi	r16, r1, PTO+PT_R16;					\
+	swi	r17, r1, PTO+PT_R17;					\
+	swi	r18, r1, PTO+PT_R18;	/* Save asm scratch reg */	\
+	swi	r19, r1, PTO+PT_R19;					\
+	swi	r20, r1, PTO+PT_R20;					\
+	swi	r21, r1, PTO+PT_R21;					\
+	swi	r22, r1, PTO+PT_R22;					\
+	swi	r23, r1, PTO+PT_R23;					\
+	swi	r24, r1, PTO+PT_R24;					\
+	swi	r25, r1, PTO+PT_R25;					\
+	swi	r26, r1, PTO+PT_R26;					\
+	swi	r27, r1, PTO+PT_R27;					\
+	swi	r28, r1, PTO+PT_R28;					\
+	swi	r29, r1, PTO+PT_R29;					\
+	swi	r30, r1, PTO+PT_R30;					\
+	swi	r31, r1, PTO+PT_R31;	/* Save current task reg */	\
 	mfs	r11, rmsr;		/* save MSR */			\
-	swi	r11, r1, PT_MSR;
+	swi	r11, r1, PTO+PT_MSR;
 
 #define RESTORE_REGS_GP \
-	lwi	r2, r1, PT_R2;	/* restore SDA */		\
-	lwi	r3, r1, PT_R3;					\
-	lwi	r4, r1, PT_R4;					\
-	lwi	r5, r1, PT_R5;					\
-	lwi	r6, r1, PT_R6;					\
-	lwi	r7, r1, PT_R7;					\
-	lwi	r8, r1, PT_R8;					\
-	lwi	r9, r1, PT_R9;					\
-	lwi	r10, r1, PT_R10;					\
-	lwi	r11, r1, PT_R11;	/* restore clobbered regs after rval */\
-	lwi	r12, r1, PT_R12;					\
-	lwi	r13, r1, PT_R13;	/* restore SDA2 */		\
-	lwi	r14, r1, PT_PC;	/* RESTORE_LINK PC, before IRQ/trap */\
-	lwi	r15, r1, PT_R15;	/* restore LP */		\
-	lwi	r16, r1, PT_R16;					\
-	lwi	r17, r1, PT_R17;					\
-	lwi	r18, r1, PT_R18;	/* restore asm scratch reg */	\
-	lwi	r19, r1, PT_R19;					\
-	lwi	r20, r1, PT_R20;					\
-	lwi	r21, r1, PT_R21;					\
-	lwi	r22, r1, PT_R22;					\
-	lwi	r23, r1, PT_R23;					\
-	lwi	r24, r1, PT_R24;					\
-	lwi	r25, r1, PT_R25;					\
-	lwi	r26, r1, PT_R26;					\
-	lwi	r27, r1, PT_R27;					\
-	lwi	r28, r1, PT_R28;					\
-	lwi	r29, r1, PT_R29;					\
-	lwi	r30, r1, PT_R30;					\
-	lwi	r31, r1, PT_R31;	/* Restore cur task reg */
+	lwi	r2, r1, PTO+PT_R2;	/* restore SDA */		\
+	lwi	r3, r1, PTO+PT_R3;					\
+	lwi	r4, r1, PTO+PT_R4;					\
+	lwi	r5, r1, PTO+PT_R5;					\
+	lwi	r6, r1, PTO+PT_R6;					\
+	lwi	r7, r1, PTO+PT_R7;					\
+	lwi	r8, r1, PTO+PT_R8;					\
+	lwi	r9, r1, PTO+PT_R9;					\
+	lwi	r10, r1, PTO+PT_R10;					\
+	lwi	r11, r1, PTO+PT_R11;	/* restore clobbered regs after rval */\
+	lwi	r12, r1, PTO+PT_R12;					\
+	lwi	r13, r1, PTO+PT_R13;	/* restore SDA2 */		\
+	lwi	r14, r1, PTO+PT_PC;	/* RESTORE_LINK PC, before IRQ/trap */\
+	lwi	r15, r1, PTO+PT_R15;	/* restore LP */		\
+	lwi	r16, r1, PTO+PT_R16;					\
+	lwi	r17, r1, PTO+PT_R17;					\
+	lwi	r18, r1, PTO+PT_R18;	/* restore asm scratch reg */	\
+	lwi	r19, r1, PTO+PT_R19;					\
+	lwi	r20, r1, PTO+PT_R20;					\
+	lwi	r21, r1, PTO+PT_R21;					\
+	lwi	r22, r1, PTO+PT_R22;					\
+	lwi	r23, r1, PTO+PT_R23;					\
+	lwi	r24, r1, PTO+PT_R24;					\
+	lwi	r25, r1, PTO+PT_R25;					\
+	lwi	r26, r1, PTO+PT_R26;					\
+	lwi	r27, r1, PTO+PT_R27;					\
+	lwi	r28, r1, PTO+PT_R28;					\
+	lwi	r29, r1, PTO+PT_R29;					\
+	lwi	r30, r1, PTO+PT_R30;					\
+	lwi	r31, r1, PTO+PT_R31;	/* Restore cur task reg */
 
 #define RESTORE_REGS \
-	lwi	r11, r1, PT_MSR;					\
+	lwi	r11, r1, PTO+PT_MSR;					\
 	mts	rmsr , r11;						\
 	RESTORE_REGS_GP
 
 #define RESTORE_REGS_RTBD \
-	lwi	r11, r1, PT_MSR;					\
+	lwi	r11, r1, PTO+PT_MSR;					\
 	andni	r11, r11, MSR_EIP;          /* clear EIP */             \
 	ori	r11, r11, MSR_EE | MSR_BIP; /* set EE and BIP */        \
 	mts	rmsr , r11;						\
@@ -265,11 +265,11 @@ syscall_debug_table:
 	lwi	r1, r0, TOPHYS(PER_CPU(ENTRY_SP));			\
 	/* FIXME: I can add these two lines to one */			\
 	/* tophys(r1,r1); */						\
-	/* addik	r1, r1, -PT_SIZE; */				\
-	addik	r1, r1, CONFIG_KERNEL_BASE_ADDR - CONFIG_KERNEL_START - PT_SIZE; \
+	/* addik	r1, r1, -STATE_SAVE_SIZE; */				\
+	addik	r1, r1, CONFIG_KERNEL_BASE_ADDR - CONFIG_KERNEL_START - STATE_SAVE_SIZE; \
 	SAVE_REGS							\
 	brid	2f;							\
-	swi	r1, r1, PT_MODE; 	 				\
+	swi	r1, r1, PTO+PT_MODE; 	 				\
 1:	/* User-mode state save.  */					\
 	lwi	r1, r0, TOPHYS(PER_CPU(CURRENT_SAVE)); /* get saved current */\
 	tophys(r1,r1);							\
@@ -277,12 +277,13 @@ syscall_debug_table:
 	/* MS these three instructions can be added to one */		\
 	/* addik	r1, r1, THREAD_SIZE; */				\
 	/* tophys(r1,r1); */						\
-	/* addik	r1, r1, -PT_SIZE; */			\
-	addik r1, r1, THREAD_SIZE + CONFIG_KERNEL_BASE_ADDR - CONFIG_KERNEL_START - PT_SIZE; \
+	/* addik	r1, r1, -STATE_SAVE_SIZE; */			\
+	addik r1, r1, THREAD_SIZE + CONFIG_KERNEL_BASE_ADDR		\
+		- CONFIG_KERNEL_START - STATE_SAVE_SIZE;		\
 	SAVE_REGS							\
 	lwi	r11, r0, TOPHYS(PER_CPU(ENTRY_SP));			\
-	swi	r11, r1, PT_R1; /* Store user SP.  */		\
-	swi	r0, r1, PT_MODE; /* Was in user-mode.  */		\
+	swi	r11, r1, PTO+PT_R1; /* Store user SP.  */		\
+	swi	r0, r1, PTO+PT_MODE; /* Was in user-mode.  */		\
 	/* MS: I am clearing UMS even in case when I come from kernel space */ \
 	clear_ums; 							\
 2:	lwi	CURRENT_TASK, r0, TOPHYS(PER_CPU(CURRENT_SAVE));
@@ -292,7 +293,7 @@ syscall_debug_table:
 .extern cpuinfo
 
 C_ENTRY(mb_flush_dcache):
-	addik	r1, r1, -PT_SIZE
+	addik	r1, r1, -STATE_SAVE_SIZE
 	SAVE_REGS
 
 	addik	r3, r0, cpuinfo
@@ -305,12 +306,12 @@ C_ENTRY(mb_flush_dcache):
 	addk	r9, r9, r8
 
 	RESTORE_REGS
-	addik	r1, r1, PT_SIZE
+	addik	r1, r1, STATE_SAVE_SIZE
 	rtsd	r15, 8
 	nop
 
 C_ENTRY(mb_invalidate_icache):
-	addik	r1, r1, -PT_SIZE
+	addik	r1, r1, -STATE_SAVE_SIZE
 	SAVE_REGS
 
 	addik	r3, r0, cpuinfo
@@ -323,7 +324,7 @@ C_ENTRY(mb_invalidate_icache):
 	addk	r9, r9, r8
 
 	RESTORE_REGS
-	addik	r1, r1, PT_SIZE
+	addik	r1, r1, STATE_SAVE_SIZE
 	rtsd	r15, 8
 	nop
 
@@ -350,18 +351,18 @@ C_ENTRY(_user_exception):
 	addik	r1, r1, THREAD_SIZE;
 	tophys(r1,r1);
 
-	addik	r1, r1, -PT_SIZE; /* Make room on the stack.  */
+	addik	r1, r1, -STATE_SAVE_SIZE; /* Make room on the stack.  */
 	SAVE_REGS
-	swi	r0, r1, PT_R3
-	swi	r0, r1, PT_R4
+	swi	r0, r1, PTO+PT_R3
+	swi	r0, r1, PTO+PT_R4
 
-	swi	r0, r1, PT_MODE;			/* Was in user-mode. */
+	swi	r0, r1, PTO+PT_MODE;			/* Was in user-mode. */
 	lwi	r11, r0, TOPHYS(PER_CPU(ENTRY_SP));
-	swi	r11, r1, PT_R1;		/* Store user SP.  */
+	swi	r11, r1, PTO+PT_R1;		/* Store user SP.  */
 	clear_ums;
 2:	lwi	CURRENT_TASK, r0, TOPHYS(PER_CPU(CURRENT_SAVE));
 	/* Save away the syscall number.  */
-	swi	r12, r1, PT_R0;
+	swi	r12, r1, PTO+PT_R0;
 	tovirt(r1,r1)
 
 /* where the trap should return need -8 to adjust for rtsd r15, 8*/
@@ -380,18 +381,18 @@ C_ENTRY(_user_exception):
 	beqi	r11, 4f
 
 	addik	r3, r0, -ENOSYS
-	swi	r3, r1, PT_R3
+	swi	r3, r1, PTO+PT_R3
 	brlid	r15, do_syscall_trace_enter
-	addik	r5, r1, PT_R0
+	addik	r5, r1, PTO+PT_R0
 
 	# do_syscall_trace_enter returns the new syscall nr.
 	addk	r12, r0, r3
-	lwi	r5, r1, PT_R5;
-	lwi	r6, r1, PT_R6;
-	lwi	r7, r1, PT_R7;
-	lwi	r8, r1, PT_R8;
-	lwi	r9, r1, PT_R9;
-	lwi	r10, r1, PT_R10;
+	lwi	r5, r1, PTO+PT_R5;
+	lwi	r6, r1, PTO+PT_R6;
+	lwi	r7, r1, PTO+PT_R7;
+	lwi	r8, r1, PTO+PT_R8;
+	lwi	r9, r1, PTO+PT_R9;
+	lwi	r10, r1, PTO+PT_R10;
 4:
 /* Jump to the appropriate function for the system call number in r12
  * (r12 is not preserved), or return an error if r12 is not valid.
@@ -432,10 +433,16 @@ C_ENTRY(_user_exception):
 /* Entry point used to return from a syscall/trap */
 /* We re-enable BIP bit before state restore */
 C_ENTRY(ret_from_trap):
-	swi	r3, r1, PT_R3
-	swi	r4, r1, PT_R4
-
-	lwi	r11, r1, PT_MODE;
+	swi	r3, r1, PTO+PT_R3
+	swi	r4, r1, PTO+PT_R4
+/*
+ * Entry point for returns that must not store r3/r4 back into pt_regs,
+ * i.e. rt_sigreturn, which has already restored them from the signal
+ * context.  Reached as "rtsd r15, 8" with r15 set to this label minus 8,
+ * so it stays correct if the number of stores above ever changes.
+ */
+ret_from_trap_no_rval:
+	lwi	r11, r1, PTO+PT_MODE;
 /* See if returning to kernel mode, if so, skip resched &c.  */
 	bnei	r11, 2f;
 	/* We're returning to user mode, so check for various conditions that
@@ -447,7 +454,7 @@ C_ENTRY(ret_from_trap):
 	beqi	r11, 1f
 
 	brlid	r15, do_syscall_trace_leave
-	addik	r5, r1, PT_R0
+	addik	r5, r1, PTO+PT_R0
 1:
 	/* We're returning to user mode, so check for various conditions that
 	 * trigger rescheduling. */
@@ -466,7 +473,7 @@ C_ENTRY(ret_from_trap):
 	andi	r11, r19, _TIF_SIGPENDING | _TIF_NOTIFY_RESUME;
 	beqi	r11, 4f;		/* Signals to handle, handle them */
 
-	addik	r5, r1, 0;		/* Arg 1: struct pt_regs *regs */
+	addik	r5, r1, PTO;		/* Arg 1: struct pt_regs *regs */
 	bralid	r15, do_notify_resume;	/* Handle any signals */
 	add	r6, r30, r0;		/* Arg 2: int in_syscall */
 	add	r30, r0, r0		/* no more restarts */
@@ -478,7 +485,11 @@ C_ENTRY(ret_from_trap):
 	VM_OFF;
 	tophys(r1,r1);
 	RESTORE_REGS_RTBD;
-	addik	r1, r1, PT_SIZE		/* Clean up stack space.  */
+	addik	r1, r1, STATE_SAVE_SIZE		/* Clean up stack space.  */
+	/*
+	 * Deliberately PT_SIZE, not STATE_SAVE_SIZE: r1 is back at the stack
+	 * top, so PTO cancels and the saved SP sits at PT_R1 - PT_SIZE.
+	 */
 	lwi	r1, r1, PT_R1 - PT_SIZE;/* Restore user stack pointer. */
 	bri	6f;
 
@@ -487,7 +498,7 @@ C_ENTRY(ret_from_trap):
 	VM_OFF;
 	tophys(r1,r1);
 	RESTORE_REGS_RTBD;
-	addik	r1, r1, PT_SIZE		/* Clean up stack space.  */
+	addik	r1, r1, STATE_SAVE_SIZE		/* Clean up stack space.  */
 	tovirt(r1,r1);
 6:
 TRAP_return:		/* Make global symbol for debugging */
@@ -518,8 +529,17 @@ C_ENTRY(ret_from_kernel_thread):
 
 C_ENTRY(sys_rt_sigreturn_wrapper):
 	addik	r30, r0, 0		/* no restarts */
+	/*
+	 * rt_sigreturn restores the full register set from the signal
+	 * context, so it must skip the r3/r4 syscall-return stores at the
+	 * head of ret_from_trap which would otherwise overwrite the
+	 * just-restored r3/r4.  Every C function returns with "rtsd r15, 8"
+	 * -- the ABI return, where the 8 skips the caller's branch and its
+	 * delay slot -- so bias r15 by -8 to land on ret_from_trap_no_rval.
+	 */
+	addik	r15, r0, ret_from_trap_no_rval - 8
 	brid	sys_rt_sigreturn	/* Do real work */
-	addik	r5, r1, 0;		/* add user context as 1st arg */
+	addik	r5, r1, PTO;		/* add user context as 1st arg */
 
 /*
  * HW EXCEPTION rutine start
@@ -530,7 +550,7 @@ C_ENTRY(full_exception_trap):
 	addik	r17, r17, -4
 	SAVE_STATE /* Save registers */
 	/* PC, before IRQ/trap - this is one instruction above */
-	swi	r17, r1, PT_PC;
+	swi	r17, r1, PTO+PT_PC;
 	tovirt(r1,r1)
 	/* FIXME this can be store directly in PT_ESR reg.
 	 * I tested it but there is a fault */
@@ -540,7 +560,7 @@ C_ENTRY(full_exception_trap):
 	mfs	r7, rfsr;		/* save FSR */
 	mts	rfsr, r0;	/* Clear sticky fsr */
 	rted	r0, full_exception
-	addik	r5, r1, 0		 /* parameter struct pt_regs * regs */
+	addik	r5, r1, PTO		 /* parameter struct pt_regs * regs */
 
 /*
  * Unaligned data trap.
@@ -566,14 +586,14 @@ C_ENTRY(unaligned_data_trap):
 	lwi	r11, r0, TOPHYS(PER_CPU(ENTRY_SP));
 	SAVE_STATE		/* Save registers.*/
 	/* PC, before IRQ/trap - this is one instruction above */
-	swi	r17, r1, PT_PC;
+	swi	r17, r1, PTO+PT_PC;
 	tovirt(r1,r1)
 	/* where the trap should return need -8 to adjust for rtsd r15, 8 */
 	addik	r15, r0, ret_from_exc-8
 	mfs	r3, resr		/* ESR */
 	mfs	r4, rear		/* EAR */
 	rtbd	r0, _unaligned_data_exception
-	addik	r7, r1, 0		/* parameter struct pt_regs * regs */
+	addik	r7, r1, PTO		/* parameter struct pt_regs * regs */
 
 /*
  * Page fault traps.
@@ -596,30 +616,30 @@ C_ENTRY(unaligned_data_trap):
 C_ENTRY(page_fault_data_trap):
 	SAVE_STATE		/* Save registers.*/
 	/* PC, before IRQ/trap - this is one instruction above */
-	swi	r17, r1, PT_PC;
+	swi	r17, r1, PTO+PT_PC;
 	tovirt(r1,r1)
 	/* where the trap should return need -8 to adjust for rtsd r15, 8 */
 	addik	r15, r0, ret_from_exc-8
 	mfs	r6, rear		/* parameter unsigned long address */
 	mfs	r7, resr		/* parameter unsigned long error_code */
 	rted	r0, do_page_fault
-	addik	r5, r1, 0		/* parameter struct pt_regs * regs */
+	addik	r5, r1, PTO		/* parameter struct pt_regs * regs */
 
 C_ENTRY(page_fault_instr_trap):
 	SAVE_STATE		/* Save registers.*/
 	/* PC, before IRQ/trap - this is one instruction above */
-	swi	r17, r1, PT_PC;
+	swi	r17, r1, PTO+PT_PC;
 	tovirt(r1,r1)
 	/* where the trap should return need -8 to adjust for rtsd r15, 8 */
 	addik	r15, r0, ret_from_exc-8
 	mfs	r6, rear		/* parameter unsigned long address */
 	ori	r7, r0, 0		/* parameter unsigned long error_code */
 	rted	r0, do_page_fault
-	addik	r5, r1, 0		/* parameter struct pt_regs * regs */
+	addik	r5, r1, PTO		/* parameter struct pt_regs * regs */
 
 /* Entry point used to return from an exception.  */
 C_ENTRY(ret_from_exc):
-	lwi	r11, r1, PT_MODE;
+	lwi	r11, r1, PTO+PT_MODE;
 	bnei	r11, 2f;		/* See if returning to kernel mode, */
 					/* ... if so, skip resched &c.  */
 
@@ -651,7 +671,7 @@ C_ENTRY(ret_from_exc):
 	 * complete register state.  Here we save anything not saved by
 	 * the normal entry sequence, so that it may be safely restored
 	 * (in a possibly modified form) after do_notify_resume returns. */
-	addik	r5, r1, 0;		/* Arg 1: struct pt_regs *regs */
+	addik	r5, r1, PTO;		/* Arg 1: struct pt_regs *regs */
 	bralid	r15, do_notify_resume;	/* Handle any signals */
 	addi	r6, r0, 0;		/* Arg 2: int in_syscall */
 	bri	1b
@@ -663,7 +683,7 @@ C_ENTRY(ret_from_exc):
 	tophys(r1,r1);
 
 	RESTORE_REGS_RTBD;
-	addik	r1, r1, PT_SIZE		/* Clean up stack space.  */
+	addik	r1, r1, STATE_SAVE_SIZE		/* Clean up stack space.  */
 
 	lwi	r1, r1, PT_R1 - PT_SIZE; /* Restore user stack pointer. */
 	bri	6f;
@@ -672,7 +692,7 @@ C_ENTRY(ret_from_exc):
 	VM_OFF;
 	tophys(r1,r1);
 	RESTORE_REGS_RTBD;
-	addik	r1, r1, PT_SIZE		/* Clean up stack space.  */
+	addik	r1, r1, STATE_SAVE_SIZE		/* Clean up stack space.  */
 
 	tovirt(r1,r1);
 6:
@@ -705,10 +725,10 @@ C_ENTRY(_interrupt):
 	tophys(r1,r1); /* MS: I have in r1 physical address where stack is */
 	/* save registers */
 /* MS: Make room on the stack -> activation record */
-	addik	r1, r1, -PT_SIZE;
+	addik	r1, r1, -STATE_SAVE_SIZE;
 	SAVE_REGS
 	brid	2f;
-	swi	r1, r1, PT_MODE; /* 0 - user mode, 1 - kernel mode */
+	swi	r1, r1, PTO+PT_MODE; /* 0 - user mode, 1 - kernel mode */
 1:
 /* User-mode state save. */
  /* MS: get the saved current */
@@ -718,23 +738,23 @@ C_ENTRY(_interrupt):
 	addik	r1, r1, THREAD_SIZE;
 	tophys(r1,r1);
 	/* save registers */
-	addik	r1, r1, -PT_SIZE;
+	addik	r1, r1, -STATE_SAVE_SIZE;
 	SAVE_REGS
 	/* calculate mode */
-	swi	r0, r1, PT_MODE;
+	swi	r0, r1, PTO+PT_MODE;
 	lwi	r11, r0, TOPHYS(PER_CPU(ENTRY_SP));
-	swi	r11, r1, PT_R1;
+	swi	r11, r1, PTO+PT_R1;
 	clear_ums;
 2:
 	lwi	CURRENT_TASK, r0, TOPHYS(PER_CPU(CURRENT_SAVE));
 	tovirt(r1,r1)
 	addik	r15, r0, irq_call;
 irq_call:rtbd	r0, do_IRQ;
-	addik	r5, r1, 0;
+	addik	r5, r1, PTO;
 
 /* MS: we are in virtual mode */
 ret_from_irq:
-	lwi	r11, r1, PT_MODE;
+	lwi	r11, r1, PTO+PT_MODE;
 	bnei	r11, 2f;
 
 1:
@@ -750,7 +770,7 @@ ret_from_irq:
 5:	andi	r11, r19, _TIF_SIGPENDING | _TIF_NOTIFY_RESUME;
 	beqid	r11, no_intr_resched
 /* Handle a signal return; Pending signals should be in r18. */
-	addik	r5, r1, 0; /* Arg 1: struct pt_regs *regs */
+	addik	r5, r1, PTO; /* Arg 1: struct pt_regs *regs */
 	bralid	r15, do_notify_resume;	/* Handle any signals */
 	addi	r6, r0, 0; /* Arg 2: int in_syscall */
 	bri	1b
@@ -763,7 +783,7 @@ no_intr_resched:
 	VM_OFF;
 	tophys(r1,r1);
 	RESTORE_REGS
-	addik	r1, r1, PT_SIZE /* MS: Clean up stack space. */
+	addik	r1, r1, STATE_SAVE_SIZE /* MS: Clean up stack space. */
 	lwi	r1, r1, PT_R1 - PT_SIZE;
 	bri	6f;
 /* MS: Return to kernel state. */
@@ -786,7 +806,7 @@ restore:
 	VM_OFF /* MS: turn off MMU */
 	tophys(r1,r1)
 	RESTORE_REGS
-	addik	r1, r1, PT_SIZE	/* MS: Clean up stack space. */
+	addik	r1, r1, STATE_SAVE_SIZE	/* MS: Clean up stack space. */
 	tovirt(r1,r1);
 6:
 IRQ_return: /* MS: Make global symbol for debugging */
@@ -805,29 +825,29 @@ C_ENTRY(_xtmr_manager_reset):
 	lwi	r1, r0, xmb_manager_stackpointer
 
 	/* Restore MSR */
-	lwi	r2, r1, PT_MSR
+	lwi	r2, r1, PTO+PT_MSR
 	mts	rmsr, r2
 	bri	4
 
 	/* restore Special purpose registers */
-	lwi	r2, r1, PT_PID
+	lwi	r2, r1, PTO+PT_PID
 	mts	rpid, r2
 
-	lwi	r2, r1, PT_TLBI
+	lwi	r2, r1, PTO+PT_TLBI
 	mts	rtlbx, r2
 
-	lwi	r2, r1, PT_ZPR
+	lwi	r2, r1, PTO+PT_ZPR
 	mts	rzpr, r2
 
 #if CONFIG_XILINX_MICROBLAZE0_USE_FPU
-	lwi	r2, r1, PT_FSR
+	lwi	r2, r1, PTO+PT_FSR
 	mts	rfsr, r2
 #endif
 
 	/* restore all the tlb's */
 	addik	r3, r0, TOPHYS(tlb_skip)
-	addik	r6, r0, PT_TLBL0
-	addik	r7, r0, PT_TLBH0
+	addik	r6, r0, PTO+PT_TLBL0
+	addik	r7, r0, PTO+PT_TLBH0
 restore_tlb:
 	add	r6, r6, r1
 	add	r7, r7, r1
@@ -853,9 +873,9 @@ ret_from_reset:
 	VM_OFF
 	/* MS: Restore all regs */
 	RESTORE_REGS
-	lwi	r14, r1, PT_R14
-	lwi	r16, r1, PT_PC
-	addik	r1, r1, PT_SIZE + 36
+	lwi	r14, r1, PTO+PT_R14
+	lwi	r16, r1, PTO+PT_PC
+	addik	r1, r1, STATE_SAVE_SIZE + 36
 	rtbd	r16, 0
 	nop
 
@@ -870,11 +890,11 @@ C_ENTRY(_xmb_manager_break):
 	 * Reserve memory in the stack for context store/restore
 	 * (which includes memory for storing tlbs (max two tlbs))
 	 */
-	addik	r1, r1, -PT_SIZE - 36
+	addik	r1, r1, -STATE_SAVE_SIZE - 36
 	swi	r1, r0, xmb_manager_stackpointer
 	SAVE_REGS
-	swi	r14, r1, PT_R14	/* rewrite saved R14 value */
-	swi	r16, r1, PT_PC; /* PC and r16 are the same */
+	swi	r14, r1, PTO+PT_R14	/* rewrite saved R14 value */
+	swi	r16, r1, PTO+PT_PC; /* PC and r16 are the same */
 
 	lwi	r6, r0, TOPHYS(xmb_manager_baseaddr)
 	lwi	r7, r0, TOPHYS(xmb_manager_crval)
@@ -888,25 +908,25 @@ C_ENTRY(_xmb_manager_break):
 
 	/* Save the special purpose registers  */
 	mfs	r2, rpid
-	swi	r2, r1, PT_PID
+	swi	r2, r1, PTO+PT_PID
 
 	mfs	r2, rtlbx
-	swi	r2, r1, PT_TLBI
+	swi	r2, r1, PTO+PT_TLBI
 
 	mfs	r2, rzpr
-	swi	r2, r1, PT_ZPR
+	swi	r2, r1, PTO+PT_ZPR
 
 #if CONFIG_XILINX_MICROBLAZE0_USE_FPU
 	mfs	r2, rfsr
-	swi	r2, r1, PT_FSR
+	swi	r2, r1, PTO+PT_FSR
 #endif
 	mfs	r2, rmsr
-	swi	r2, r1, PT_MSR
+	swi	r2, r1, PTO+PT_MSR
 
 	/* Save all the tlb's */
 	addik	r3, r0, TOPHYS(tlb_skip)
-	addik	r6, r0, PT_TLBL0
-	addik	r7, r0, PT_TLBH0
+	addik	r6, r0, PTO+PT_TLBL0
+	addik	r7, r0, PTO+PT_TLBH0
 save_tlb:
 	add	r6, r6, r1
 	add	r7, r7, r1
@@ -943,7 +963,7 @@ ret_from_break:
 	mbar	1
 	mbar	2
 	bri	4
-	suspend
+	mbar	24 /* suspend */
 	nop
 #endif
 
@@ -963,28 +983,28 @@ C_ENTRY(_debug_exception):
 	lwi	r1, r0, TOPHYS(PER_CPU(ENTRY_SP)); /* Reload kernel stack-ptr*/
 
 	/* BIP bit is set on entry, no interrupts can occur */
-	addik   r1, r1, CONFIG_KERNEL_BASE_ADDR - CONFIG_KERNEL_START - PT_SIZE;
+	addik   r1, r1, CONFIG_KERNEL_BASE_ADDR - CONFIG_KERNEL_START - STATE_SAVE_SIZE;
 	SAVE_REGS;
 	/* save all regs to pt_reg structure */
-	swi	r0, r1, PT_R0;	/* R0 must be saved too */
-	swi	r14, r1, PT_R14	/* rewrite saved R14 value */
-	swi	r16, r1, PT_PC; /* PC and r16 are the same */
+	swi	r0, r1, PTO+PT_R0;	/* R0 must be saved too */
+	swi	r14, r1, PTO+PT_R14	/* rewrite saved R14 value */
+	swi	r16, r1, PTO+PT_PC; /* PC and r16 are the same */
 	/* save special purpose registers to pt_regs */
 	mfs	r11, rear;
-	swi	r11, r1, PT_EAR;
+	swi	r11, r1, PTO+PT_EAR;
 	mfs	r11, resr;
-	swi	r11, r1, PT_ESR;
+	swi	r11, r1, PTO+PT_ESR;
 	mfs	r11, rfsr;
-	swi	r11, r1, PT_FSR;
+	swi	r11, r1, PTO+PT_FSR;
 
 	/* stack pointer is in physical address at it is decrease
-	 * by PT_SIZE but we need to get correct R1 value */
-	addik   r11, r1, CONFIG_KERNEL_START - CONFIG_KERNEL_BASE_ADDR + PT_SIZE;
-	swi	r11, r1, PT_R1
+	 * by STATE_SAVE_SIZE but we need to get correct R1 value */
+	addik   r11, r1, CONFIG_KERNEL_START - CONFIG_KERNEL_BASE_ADDR + STATE_SAVE_SIZE;
+	swi	r11, r1, PTO+PT_R1
 	/* MS: r31 - current pointer isn't changed */
 	tovirt(r1,r1)
 #ifdef CONFIG_KGDB
-	addi	r5, r1, 0 /* pass pt_reg address as the first arg */
+	addi	r5, r1, PTO /* pass pt_reg address as the first arg */
 	addik	r15, r0, dbtrap_call; /* return address */
 	rtbd	r0, microblaze_kgdb_break
 	nop;
@@ -1000,16 +1020,16 @@ C_ENTRY(_debug_exception):
 	addik	r1, r1, THREAD_SIZE;	/* calculate kernel stack pointer */
 	tophys(r1,r1);
 
-	addik	r1, r1, -PT_SIZE; /* Make room on the stack.  */
+	addik	r1, r1, -STATE_SAVE_SIZE; /* Make room on the stack.  */
 	SAVE_REGS;
-	swi	r16, r1, PT_PC;	/* Save LP */
-	swi	r0, r1, PT_MODE; /* Was in user-mode.  */
+	swi	r16, r1, PTO+PT_PC;	/* Save LP */
+	swi	r0, r1, PTO+PT_MODE; /* Was in user-mode.  */
 	lwi	r11, r0, TOPHYS(PER_CPU(ENTRY_SP));
-	swi	r11, r1, PT_R1; /* Store user SP.  */
+	swi	r11, r1, PTO+PT_R1; /* Store user SP.  */
 	lwi	CURRENT_TASK, r0, TOPHYS(PER_CPU(CURRENT_SAVE));
 	tovirt(r1,r1)
 	set_vms;
-	addik	r5, r1, 0;
+	addik	r5, r1, PTO;
 	addik	r15, r0, dbtrap_call;
 dbtrap_call: /* Return point for kernel/user entry + 8 because of rtsd r15, 8 */
 	rtbd	r0, sw_exception
@@ -1017,7 +1037,7 @@ dbtrap_call: /* Return point for kernel/user entry + 8 because of rtsd r15, 8 */
 
 	/* MS: The first instruction for the second part of the gdb/kgdb */
 	set_bip; /* Ints masked for state restore */
-	lwi	r11, r1, PT_MODE;
+	lwi	r11, r1, PTO+PT_MODE;
 	bnei	r11, 2f;
 /* MS: Return to user space - gdb */
 1:
@@ -1036,7 +1056,7 @@ dbtrap_call: /* Return point for kernel/user entry + 8 because of rtsd r15, 8 */
 5:	andi	r11, r19, _TIF_SIGPENDING | _TIF_NOTIFY_RESUME;
 	beqi	r11, 4f;		/* Signals to handle, handle them */
 
-	addik	r5, r1, 0;		/* Arg 1: struct pt_regs *regs */
+	addik	r5, r1, PTO;		/* Arg 1: struct pt_regs *regs */
 	bralid	r15, do_notify_resume;	/* Handle any signals */
 	addi  r6, r0, 0;	/* Arg 2: int in_syscall */
 	bri	1b
@@ -1047,7 +1067,7 @@ dbtrap_call: /* Return point for kernel/user entry + 8 because of rtsd r15, 8 */
 	tophys(r1,r1);
 	/* MS: Restore all regs */
 	RESTORE_REGS_RTBD
-	addik	r1, r1, PT_SIZE	 /* Clean up stack space */
+	addik	r1, r1, STATE_SAVE_SIZE	 /* Clean up stack space */
 	lwi	r1, r1, PT_R1 - PT_SIZE; /* Restore user stack pointer */
 DBTRAP_return_user: /* MS: Make global symbol for debugging */
 	rtbd	r16, 0; /* MS: Instructions to return from a debug trap */
@@ -1058,9 +1078,9 @@ DBTRAP_return_user: /* MS: Make global symbol for debugging */
 	tophys(r1,r1);
 	/* MS: Restore all regs */
 	RESTORE_REGS_RTBD
-	lwi	r14, r1, PT_R14;
-	lwi	r16, r1, PT_PC;
-	addik	r1, r1, PT_SIZE; /* MS: Clean up stack space */
+	lwi	r14, r1, PTO+PT_R14;
+	lwi	r16, r1, PTO+PT_PC;
+	addik	r1, r1, STATE_SAVE_SIZE; /* MS: Clean up stack space */
 	tovirt(r1,r1);
 DBTRAP_return_kernel: /* MS: Make global symbol for debugging */
 	rtbd	r16, 0; /* MS: Instructions to return from a debug trap */
@@ -1158,7 +1178,7 @@ ENTRY(_switch_to)
 .ent xmb_inject_err
 .type xmb_inject_err, @function
 xmb_inject_err:
-	addik	r1, r1, -PT_SIZE
+	addik	r1, r1, -STATE_SAVE_SIZE
 	SAVE_REGS
 
 	/* Switch to real mode */
@@ -1182,7 +1202,7 @@ xmb_inject_err:
 	nop;
 1:
 	RESTORE_REGS
-	addik	r1, r1, PT_SIZE
+	addik	r1, r1, STATE_SAVE_SIZE
 	rtsd	r15, 8;
 	nop;
 .end xmb_inject_err
diff --git a/arch/microblaze/kernel/hw_exception_handler.S b/arch/microblaze/kernel/hw_exception_handler.S
index 07ea23965f8175..4693916be52672 100644
--- a/arch/microblaze/kernel/hw_exception_handler.S
+++ b/arch/microblaze/kernel/hw_exception_handler.S
@@ -313,6 +313,11 @@ _MB_HW_ExceptionVectorTable:
 .align 4
 .ent _hw_exception_handler
 _hw_exception_handler:
+	/*
+	 * This handler saves into the static pt_pool_space buffer, not into a
+	 * stack frame, so the plain PT_* offsets are right here -- do not add
+	 * the PTO of entry.S's frames.
+	 */
 	swi	r1, r0, TOPHYS(pt_pool_space + PT_R1); /* GET_SP */
 	/* Save date to kernel memory. Here is the problem
 	 * when you came from user space */
diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c
index 6cbf642d7b801d..090e1697ea7aee 100644
--- a/arch/microblaze/kernel/process.c
+++ b/arch/microblaze/kernel/process.c
@@ -19,6 +19,7 @@
 #include <linux/bitops.h>
 #include <linux/ptrace.h>
 #include <asm/cacheflush.h>
+#include <asm/entry.h>
 
 void show_regs(struct pt_regs *regs)
 {
@@ -65,7 +66,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 		 * the registers. That's OK for a brand new thread.*/
 		memset(childregs, 0, sizeof(struct pt_regs));
 		memset(&ti->cpu_context, 0, sizeof(struct cpu_context));
-		ti->cpu_context.r1  = (unsigned long)childregs;
+		ti->cpu_context.r1  = (unsigned long)childregs - PTO;
 		ti->cpu_context.r20 = (unsigned long)args->fn;
 		ti->cpu_context.r19 = (unsigned long)args->fn_arg;
 		childregs->pt_mode = 1;
@@ -79,7 +80,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
 		childregs->r1 = usp;
 
 	memset(&ti->cpu_context, 0, sizeof(struct cpu_context));
-	ti->cpu_context.r1 = (unsigned long)childregs;
+	ti->cpu_context.r1 = (unsigned long)childregs - PTO;
 	childregs->msr |= MSR_UMS;
 
 	/* we should consider the fact that childregs is a copy of the parent
diff --git a/arch/microblaze/kernel/ptrace.c b/arch/microblaze/kernel/ptrace.c
index 236264e932d64f..ac1f0503216b42 100644
--- a/arch/microblaze/kernel/ptrace.c
+++ b/arch/microblaze/kernel/ptrace.c
@@ -101,6 +101,9 @@ long arch_ptrace(struct task_struct *child, long request,
 			} else {
 				rval = -EIO;
 			}
+		} else if (request == PTRACE_POKEUSR &&
+			   (addr == PT_MSR || addr == PT_MODE)) {
+			rval = -EIO;
 		} else if (addr < PT_SIZE && (addr & 0x3) == 0) {
 			microblaze_reg_t *reg_addr = reg_save_addr(addr, child);
 			if (request == PTRACE_PEEKUSR)
diff --git a/arch/microblaze/kernel/reset.c b/arch/microblaze/kernel/reset.c
index 2f66c7963084dd..42031c535d48d2 100644
--- a/arch/microblaze/kernel/reset.c
+++ b/arch/microblaze/kernel/reset.c
@@ -28,6 +28,7 @@ void machine_halt(void)
 void machine_power_off(void)
 {
 	pr_notice("Machine power off...\n");
+	do_kernel_power_off();
 	while (1)
 		;
 }
@@ -40,3 +41,24 @@ void machine_restart(char *cmd)
 	pr_emerg("Reboot failed -- System halted\n");
 	while (1);
 }
+
+#ifdef CONFIG_MB_POWER_OFF_THROUGH_UNALIGNED_PC
+static int unaligned_pc_sys_off(struct sys_off_data *data)
+{
+	__asm__(
+		"bri 1\n"
+	);
+
+	return NOTIFY_DONE;
+}
+
+static int __init register_unaligned_pc_sys_off(void)
+{
+	struct sys_off_handler *sys_off;
+
+	sys_off = register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_LOW,
+					   unaligned_pc_sys_off, NULL);
+	return PTR_ERR_OR_ZERO(sys_off);
+}
+device_initcall(register_unaligned_pc_sys_off);
+#endif /* CONFIG_MB_POWER_OFF_THROUGH_UNALIGNED_PC */
diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c
index c78a0ff480660d..6bbc16f98d5ebb 100644
--- a/arch/microblaze/kernel/signal.c
+++ b/arch/microblaze/kernel/signal.c
@@ -33,6 +33,7 @@
 #include <linux/linkage.h>
 #include <linux/resume_user_mode.h>
 #include <asm/entry.h>
+#include <asm/registers.h>
 #include <asm/ucontext.h>
 #include <linux/uaccess.h>
 #include <linux/syscalls.h>
@@ -49,6 +50,14 @@ struct sigframe {
 };
 
 struct rt_sigframe {
+	/*
+	 * Home area for the handler's register arguments: the MicroBlaze
+	 * ABI reserves [r1+0] for the return address and lets the callee
+	 * store r5..r10 at [r1+4]..[r1+24], and r1 points at this frame
+	 * when the handler is entered.  Without the gap those stores
+	 * corrupt frame->info.
+	 */
+	unsigned long abi_gap[7];
 	struct siginfo info;
 	struct ucontext uc;
 	unsigned long tramp[2];	/* signal trampoline */
@@ -73,6 +82,22 @@ static int restore_sigcontext(struct pt_regs *regs,
 	COPY(pc);	COPY(ear);	COPY(esr);	COPY(fsr);
 #undef COPY
 
+	/*
+	 * The frame is user-writable, so restore only the user-writable
+	 * status flags (carry) and keep the kernel-controlled MSR bits
+	 * (UMS/VMS/IE/EE/...) from regs->msr: rtbd derives the resumed mode
+	 * from UMS/VMS, so a verbatim restore would hand userspace the
+	 * privileged return state.  Same idea as x86 masking the restored
+	 * EFLAGS to FIX_EFLAGS.
+	 */
+	{
+		unsigned long msr;
+
+		err |= __get_user(msr, &sc->regs.msr);
+		regs->msr = (regs->msr & ~(MSR_C | MSR_CC)) |
+			    (msr & (MSR_C | MSR_CC));
+	}
+
 	*rval_p = regs->r3;
 
 	return err;
@@ -132,6 +157,7 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
 	COPY(r26);	COPY(r27);	COPY(r28);	COPY(r29);
 	COPY(r30);	COPY(r31);
 	COPY(pc);	COPY(ear);	COPY(esr);	COPY(fsr);
+	COPY(msr);	/* restore_sigcontext() accepts only carry state */
 #undef COPY
 
 	err |= __put_user(mask, &sc->oldmask);
diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl
index c55a5c96f49b82..76d7cbe2a133c4 100644
--- a/arch/microblaze/kernel/syscalls/syscall.tbl
+++ b/arch/microblaze/kernel/syscalls/syscall.tbl
@@ -193,7 +193,7 @@
 183	common	getcwd				sys_getcwd
 184	common	capget				sys_capget
 185	common	capset				sys_capset
-186	common	sigaltstack			sys_ni_syscall
+186	common	sigaltstack			sys_sigaltstack
 187	common	sendfile			sys_sendfile
 188	common	getpmsg				sys_ni_syscall
 189	common	putpmsg				sys_ni_syscall
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index d7c67cebe065f9..ed17c9b1624cac 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -97,7 +97,6 @@ config MIPS
 	select IRQ_FORCED_THREADING
 	select ISA if EISA
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_REL if MODULES
 	select MODULES_USE_ELF_RELA if MODULES && 64BIT
 	select PERF_USE_VMALLOC
diff --git a/arch/mips/Makefile b/arch/mips/Makefile
index cff1a9a43b897d..c84e5dbcb25c5f 100644
--- a/arch/mips/Makefile
+++ b/arch/mips/Makefile
@@ -285,23 +285,27 @@ cflags-y			+= -I$(srctree)/arch/mips/include/asm/mach-generic
 drivers-$(CONFIG_PCI)		+= arch/mips/pci/
 
 #
-# Automatically detect the build format. By default we choose
-# the elf format according to the load address.
-# We can always force a build with a 64-bits symbol format by
-# passing 'KBUILD_SYM32=no' option to the make's command line.
+# Automatically detect whether to build a 64-bit kernel using symbols
+# limited to 32-bit values, which produces more compact code.  This
+# relies on the kernel image being linked at a 32-bit load address,
+# which we use to infer whether to use this arrangement for symbols.
+# Use 'make KBUILD_SYM32=n' to override this default choice and force
+# 64-bit symbols to be used instead.
 #
-ifdef CONFIG_64BIT
-  ifndef KBUILD_SYM32
-    ifeq ($(shell expr $(load-y) \< 0xffffffff80000000), 0)
-      KBUILD_SYM32 = $(call cc-option-yn, -msym32)
+ifdef need-compiler
+  ifdef CONFIG_64BIT
+    ifndef KBUILD_SYM32
+      ifeq ($(shell expr $(load-y) \< 0xffffffff80000000), 0)
+        KBUILD_SYM32 = $(call cc-option-yn, -msym32)
+      endif
     endif
-  endif
 
-  ifeq ($(KBUILD_SYM32), y)
-    cflags-$(KBUILD_SYM32) += -msym32 -DKBUILD_64BIT_SYM32
-  else
-    ifeq ($(CONFIG_CPU_DADDI_WORKAROUNDS), y)
-      $(error CONFIG_CPU_DADDI_WORKAROUNDS unsupported without -msym32)
+    ifeq ($(KBUILD_SYM32), y)
+      cflags-y += -msym32 -DKBUILD_64BIT_SYM32
+    else
+      ifeq ($(CONFIG_CPU_DADDI_WORKAROUNDS), y)
+        $(error CONFIG_CPU_DADDI_WORKAROUNDS unsupported without -msym32)
+      endif
     endif
   endif
 endif
diff --git a/arch/mips/alchemy/common/clock.c b/arch/mips/alchemy/common/clock.c
index 62a7304ff35f61..b5d2cbb7c69ec4 100644
--- a/arch/mips/alchemy/common/clock.c
+++ b/arch/mips/alchemy/common/clock.c
@@ -150,7 +150,7 @@ static const struct clk_ops alchemy_clkops_cpu = {
 static struct clk __init *alchemy_clk_setup_cpu(const char *parent_name,
 						int ctype)
 {
-	struct clk_init_data id;
+	struct clk_init_data id = {};
 	struct clk_hw *h;
 	struct clk *clk;
 
@@ -245,7 +245,7 @@ static struct clk __init *alchemy_clk_setup_aux(const char *parent_name,
 						char *name, int maxmult,
 						unsigned long reg)
 {
-	struct clk_init_data id;
+	struct clk_init_data id = {};
 	struct clk *c;
 	struct alchemy_auxpll_clk *a;
 
@@ -754,7 +754,7 @@ static const char * const alchemy_clk_fgen_names[] = {
 static int __init alchemy_clk_init_fgens(int ctype)
 {
 	struct clk *c;
-	struct clk_init_data id;
+	struct clk_init_data id = {};
 	struct alchemy_fgcs_clk *a;
 	unsigned long v;
 	int i, ret;
@@ -959,9 +959,9 @@ static int alchemy_csrc_dt2[] = { 1, 4, 3, 2 };	/* Au1300 */
 
 static int __init alchemy_clk_setup_imux(int ctype)
 {
+	struct clk_init_data id = {};
 	struct alchemy_fgcs_clk *a;
 	const char * const *names;
-	struct clk_init_data id;
 	unsigned long v;
 	int i, ret, *dt;
 	struct clk *c;
diff --git a/arch/mips/boot/dts/pic32/pic32mzda.dtsi b/arch/mips/boot/dts/pic32/pic32mzda.dtsi
index feca35ba56a47a..433dff3cf14f3d 100644
--- a/arch/mips/boot/dts/pic32/pic32mzda.dtsi
+++ b/arch/mips/boot/dts/pic32/pic32mzda.dtsi
@@ -76,8 +76,6 @@
 	};
 
 	pic32_pinctrl: pinctrl@1f801400 {
-		#address-cells = <1>;
-		#size-cells = <1>;
 		compatible = "microchip,pic32mzda-pinctrl";
 		reg = <0x1f801400 0x400>;
 		clocks = <&rootclk PB1CLK>;
diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c
index 9fbba6a8fa4c5c..c049632fbcc338 100644
--- a/arch/mips/cavium-octeon/dma-octeon.c
+++ b/arch/mips/cavium-octeon/dma-octeon.c
@@ -228,7 +228,7 @@ void __init plat_swiotlb_setup(void)
 		swiotlbsize = 64 * (1<<20);
 	}
 #endif
-#ifdef CONFIG_USB_OHCI_HCD_PLATFORM
+#if IS_ENABLED(CONFIG_USB_OHCI_HCD_PLATFORM)
 	/* OCTEON II ohci is only 32-bit. */
 	if (OCTEON_IS_OCTEON2() && max_addr >= 0x100000000ul)
 		swiotlbsize = 64 * (1<<20);
diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper.c b/arch/mips/cavium-octeon/executive/cvmx-helper.c
index 1985cd66806a84..107b1be4a45ee9 100644
--- a/arch/mips/cavium-octeon/executive/cvmx-helper.c
+++ b/arch/mips/cavium-octeon/executive/cvmx-helper.c
@@ -1014,7 +1014,6 @@ int cvmx_helper_initialize_packet_io_global(void)
 {
 	int result = 0;
 	int interface;
-	union cvmx_l2c_cfg l2c_cfg;
 	const int num_interfaces = cvmx_helper_get_number_of_interfaces();
 
 	/*
@@ -1029,10 +1028,21 @@ int cvmx_helper_initialize_packet_io_global(void)
 	 * to the cores. This avoids conditions where IO blocks might
 	 * be starved under very high L2 loads.
 	 */
-	l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG);
-	l2c_cfg.s.lrf_arb_mode = 0;
-	l2c_cfg.s.rfb_arb_mode = 0;
-	cvmx_write_csr(CVMX_L2C_CFG, l2c_cfg.u64);
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
+		union cvmx_l2c_ctl l2c_ctl;
+
+		l2c_ctl.u64 = cvmx_read_csr(CVMX_L2C_CTL);
+		l2c_ctl.s.rsp_arb_mode = 1;
+		l2c_ctl.s.xmc_arb_mode = 0;
+		cvmx_write_csr(CVMX_L2C_CTL, l2c_ctl.u64);
+	} else {
+		union cvmx_l2c_cfg l2c_cfg;
+
+		l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG);
+		l2c_cfg.s.lrf_arb_mode = 0;
+		l2c_cfg.s.rfb_arb_mode = 0;
+		cvmx_write_csr(CVMX_L2C_CFG, l2c_cfg.u64);
+	}
 
 	cvmx_pko_initialize_global();
 	for (interface = 0; interface < num_interfaces; interface++) {
diff --git a/arch/mips/cavium-octeon/executive/cvmx-l2c.c b/arch/mips/cavium-octeon/executive/cvmx-l2c.c
index 33b303691bc2cd..373907e1e66860 100644
--- a/arch/mips/cavium-octeon/executive/cvmx-l2c.c
+++ b/arch/mips/cavium-octeon/executive/cvmx-l2c.c
@@ -54,7 +54,7 @@ int cvmx_l2c_get_core_way_partition(uint32_t core)
 	if (core >= cvmx_octeon_num_cores())
 		return -1;
 
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX))
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
 		return cvmx_read_csr(CVMX_L2C_WPAR_PPX(core)) & 0xffff;
 
 	/*
@@ -91,14 +91,14 @@ int cvmx_l2c_set_core_way_partition(uint32_t core, uint32_t mask)
 	mask &= valid_mask;
 
 	/* A UMSK setting which blocks all L2C Ways is an error on some chips */
-	if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX))
+	if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN6XXX))
 		return -1;
 
 	/* Validate the core number */
 	if (core >= cvmx_octeon_num_cores())
 		return -1;
 
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 		cvmx_write_csr(CVMX_L2C_WPAR_PPX(core), mask);
 		return 0;
 	}
@@ -147,10 +147,10 @@ int cvmx_l2c_set_hw_way_partition(uint32_t mask)
 	mask &= valid_mask;
 
 	/* A UMSK setting which blocks all L2C Ways is an error on some chips */
-	if (mask == valid_mask	&& !OCTEON_IS_MODEL(OCTEON_CN63XX))
+	if (mask == valid_mask	&& !OCTEON_IS_MODEL(OCTEON_CN6XXX))
 		return -1;
 
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX))
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
 		cvmx_write_csr(CVMX_L2C_WPAR_IOBX(0), mask);
 	else
 		cvmx_write_csr(CVMX_L2C_SPAR4,
@@ -160,7 +160,7 @@ int cvmx_l2c_set_hw_way_partition(uint32_t mask)
 
 int cvmx_l2c_get_hw_way_partition(void)
 {
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX))
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX))
 		return cvmx_read_csr(CVMX_L2C_WPAR_IOBX(0)) & 0xffff;
 	else
 		return cvmx_read_csr(CVMX_L2C_SPAR4) & (0xFF);
@@ -313,7 +313,7 @@ static void fault_in(uint64_t addr, int len)
 
 int cvmx_l2c_lock_line(uint64_t addr)
 {
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 		int shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
 		uint64_t assoc = cvmx_l2c_get_num_assoc();
 		uint64_t tag = addr >> shift;
@@ -460,7 +460,7 @@ void cvmx_l2c_flush(void)
 int cvmx_l2c_unlock_line(uint64_t address)
 {
 
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 		int assoc;
 		union cvmx_l2c_tag tag;
 		uint32_t tag_addr;
@@ -661,7 +661,7 @@ union cvmx_l2c_tag cvmx_l2c_get_tag(uint32_t association, uint32_t index)
 			     (int)index, cvmx_l2c_get_num_sets());
 		return tag;
 	}
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 		union cvmx_l2c_tadx_tag l2c_tadx_tag;
 		uint64_t address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
 						(association << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
@@ -745,7 +745,7 @@ uint32_t cvmx_l2c_address_to_index(uint64_t addr)
 	}
 
 	if (indxalias) {
-		if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
+		if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 			uint32_t a_14_12 = (idx / (CVMX_L2C_MEMBANK_SELECT_SIZE/(1<<CVMX_L2C_IDX_ADDR_SHIFT))) & 0x7;
 
 			idx ^= idx / cvmx_l2c_get_num_sets();
@@ -773,7 +773,9 @@ int cvmx_l2c_get_set_bits(void)
 
 	if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN58XX))
 		l2_set_bits = 11;	/* 2048 sets */
-	else if (OCTEON_IS_MODEL(OCTEON_CN38XX) || OCTEON_IS_MODEL(OCTEON_CN63XX))
+	else if (OCTEON_IS_MODEL(OCTEON_CN38XX) ||
+		 OCTEON_IS_MODEL(OCTEON_CN63XX) ||
+		 OCTEON_IS_MODEL(OCTEON_CN66XX))
 		l2_set_bits = 10;	/* 1024 sets */
 	else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX))
 		l2_set_bits = 9;	/* 512 sets */
@@ -805,7 +807,8 @@ int cvmx_l2c_get_num_assoc(void)
 	    OCTEON_IS_MODEL(OCTEON_CN50XX) ||
 	    OCTEON_IS_MODEL(OCTEON_CN38XX))
 		l2_assoc = 8;
-	else if (OCTEON_IS_MODEL(OCTEON_CN63XX))
+	else if (OCTEON_IS_MODEL(OCTEON_CN63XX) ||
+		 OCTEON_IS_MODEL(OCTEON_CN66XX))
 		l2_assoc = 16;
 	else if (OCTEON_IS_MODEL(OCTEON_CN31XX) ||
 		 OCTEON_IS_MODEL(OCTEON_CN30XX))
@@ -816,7 +819,7 @@ int cvmx_l2c_get_num_assoc(void)
 	}
 
 	/* Check to see if part of the cache is disabled */
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
+	if (OCTEON_IS_MODEL(OCTEON_CN63XX) || OCTEON_IS_MODEL(OCTEON_CN66XX)) {
 		union cvmx_mio_fus_dat3 mio_fus_dat3;
 
 		mio_fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3);
@@ -878,7 +881,7 @@ void cvmx_l2c_flush_line(uint32_t assoc, uint32_t index)
 		return;
 	}
 
-	if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
 		uint64_t address;
 		/* Create the address based on index and association.
 		 * Bits<20:17> select the way of the cache block involved in
diff --git a/arch/mips/cavium-octeon/flash_setup.c b/arch/mips/cavium-octeon/flash_setup.c
index 3395acde4d60ac..66d173b57c3ff3 100644
--- a/arch/mips/cavium-octeon/flash_setup.c
+++ b/arch/mips/cavium-octeon/flash_setup.c
@@ -22,7 +22,7 @@ static struct map_info flash_map;
 static struct mtd_info *mymtd;
 static const char *part_probe_types[] = {
 	"cmdlinepart",
-#ifdef CONFIG_MTD_REDBOOT_PARTS
+#if IS_ENABLED(CONFIG_MTD_REDBOOT_PARTS)
 	"RedBoot",
 #endif
 	NULL
diff --git a/arch/mips/cavium-octeon/octeon-memcpy.S b/arch/mips/cavium-octeon/octeon-memcpy.S
index fef0c6de3fa107..fb27126afc5647 100644
--- a/arch/mips/cavium-octeon/octeon-memcpy.S
+++ b/arch/mips/cavium-octeon/octeon-memcpy.S
@@ -60,7 +60,7 @@
  *  3- (dst - src) == (dst_entry - src_entry),
  * The _entry suffix denotes values when __copy_user was called.
  *
- * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user
+ * (1) is set up by uaccess.h and maintained by not writing AT in copy_user
  * (2) is met by incrementing src by the number of bytes copied
  * (3) is met by not doing loads between a pair of increments of dst and src
  *
diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c
index 1ad2602a0383e9..d2bba2fe636d2c 100644
--- a/arch/mips/cavium-octeon/setup.c
+++ b/arch/mips/cavium-octeon/setup.c
@@ -653,6 +653,24 @@ void octeon_user_io_init(void)
 	write_c0_derraddr1(0);
 }
 
+#ifdef CONFIG_CAVIUM_OCTEON_LOCK_L2
+static bool __init octeon_l2_is_crippled(void)
+{
+	/*
+	 * L2D_FUS3 exists on CN3XXX/CN5XXX only. On OCTEON II the
+	 * crippled-L2 fuses live in MIO_FUS_DAT3[l2c_crip].
+	 */
+	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
+		union cvmx_mio_fus_dat3 fus_dat3;
+
+		fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3);
+		return fus_dat3.s.l2c_crip != 0;
+	}
+
+	return cvmx_read_csr(CVMX_L2D_FUS3) & (3ull << 34);
+}
+#endif
+
 /**
  * prom_init - Early entry point for arch setup
  */
@@ -801,7 +819,7 @@ void __init prom_init(void)
 	}
 
 #ifdef CONFIG_CAVIUM_OCTEON_LOCK_L2
-	if (cvmx_read_csr(CVMX_L2D_FUS3) & (3ull << 34)) {
+	if (octeon_l2_is_crippled()) {
 		pr_info("Skipping L2 locking due to reduced L2 cache size\n");
 	} else {
 		uint32_t __maybe_unused ebase = read_c0_ebase() & 0x3ffff000;
@@ -1109,7 +1127,7 @@ EXPORT_SYMBOL(prom_putchar);
 
 void __init prom_free_prom_memory(void)
 {
-	if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
+	if (OCTEON_IS_MODEL(OCTEON_CN63XX_PASS1_X)) {
 		/* Check for presence of Core-14449 fix.  */
 		u32 insn;
 		u32 *foo;
diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig
index 8afb8ac3ed78bd..62efc81b394f7e 100644
--- a/arch/mips/configs/decstation_64_defconfig
+++ b/arch/mips/configs/decstation_64_defconfig
@@ -1,27 +1,27 @@
+CONFIG_WERROR=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_HIGH_RES_TIMERS=y
+CONFIG_BPF_SYSCALL=y
 CONFIG_BSD_PROCESS_ACCT=y
 CONFIG_BSD_PROCESS_ACCT_V3=y
 CONFIG_LOG_BUF_SHIFT=15
 CONFIG_EXPERT=y
 # CONFIG_SGETMASK_SYSCALL is not set
-# CONFIG_SYSFS_SYSCALL is not set
-CONFIG_BPF_SYSCALL=y
-# CONFIG_COMPAT_BRK is not set
 CONFIG_MACH_DECSTATION=y
 CONFIG_64BIT=y
-CONFIG_PAGE_SIZE_16KB=y
 CONFIG_TC=y
 CONFIG_MIPS32_O32=y
 CONFIG_MIPS32_N32=y
 # CONFIG_SUSPEND is not set
+CONFIG_PAGE_SIZE_16KB=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_SRCVERSION_ALL=y
 CONFIG_PARTITION_ADVANCED=y
 CONFIG_OSF_PARTITION=y
 # CONFIG_EFI_PARTITION is not set
+# CONFIG_COMPAT_BRK is not set
 CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_NET=y
 CONFIG_PACKET=y
@@ -49,7 +49,6 @@ CONFIG_NETWORK_SECMARK=y
 CONFIG_IP_SCTP=m
 CONFIG_VLAN_8021Q=m
 # CONFIG_WIRELESS is not set
-# CONFIG_UEVENT_HELPER is not set
 # CONFIG_FW_LOADER is not set
 # CONFIG_ALLOW_DEV_COREDUMP is not set
 CONFIG_MTD=m
@@ -83,9 +82,9 @@ CONFIG_DECLANCE=y
 # CONFIG_NET_VENDOR_MICREL is not set
 # CONFIG_NET_VENDOR_MICROCHIP is not set
 # CONFIG_NET_VENDOR_MICROSEMI is not set
+# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_NATSEMI is not set
 # CONFIG_NET_VENDOR_NETRONOME is not set
-# CONFIG_NET_VENDOR_NI is not set
 # CONFIG_NET_VENDOR_QUALCOMM is not set
 # CONFIG_NET_VENDOR_RENESAS is not set
 # CONFIG_NET_VENDOR_ROCKER is not set
@@ -114,13 +113,11 @@ CONFIG_FB_TGA=y
 CONFIG_FB_PMAG_AA=y
 CONFIG_FB_PMAG_BA=y
 CONFIG_FB_PMAGB_B=y
-# CONFIG_VGA_CONSOLE is not set
 CONFIG_DUMMY_CONSOLE_COLUMNS=160
 CONFIG_DUMMY_CONSOLE_ROWS=64
 CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
 CONFIG_LOGO=y
-# CONFIG_LOGO_LINUX_VGA16 is not set
 # CONFIG_LOGO_LINUX_CLUT224 is not set
 # CONFIG_HID is not set
 # CONFIG_USB_SUPPORT is not set
@@ -168,31 +165,24 @@ CONFIG_NLS_ISO8859_14=m
 CONFIG_NLS_ISO8859_15=m
 CONFIG_NLS_UTF8=m
 CONFIG_CRYPTO_RSA=m
-CONFIG_CRYPTO_CCM=m
-CONFIG_CRYPTO_GCM=m
-CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_BLOWFISH=m
+CONFIG_CRYPTO_CAMELLIA=m
+CONFIG_CRYPTO_CAST5=m
+CONFIG_CRYPTO_CAST6=m
+CONFIG_CRYPTO_SERPENT=m
+CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_CTS=m
 CONFIG_CRYPTO_LRW=m
-CONFIG_CRYPTO_OFB=m
 CONFIG_CRYPTO_XTS=m
+CONFIG_CRYPTO_CHACHA20POLY1305=m
+CONFIG_CRYPTO_CCM=m
 CONFIG_CRYPTO_CMAC=m
-CONFIG_CRYPTO_XCBC=m
-CONFIG_CRYPTO_CRC32=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_RMD160=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_WP512=m
-CONFIG_CRYPTO_ANUBIS=m
-CONFIG_CRYPTO_ARC4=m
-CONFIG_CRYPTO_BLOWFISH=m
-CONFIG_CRYPTO_CAMELLIA=m
-CONFIG_CRYPTO_CAST5=m
-CONFIG_CRYPTO_CAST6=m
-CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SEED=m
-CONFIG_CRYPTO_SERPENT=m
-CONFIG_CRYPTO_TEA=m
-CONFIG_CRYPTO_TWOFISH=m
+CONFIG_CRYPTO_XCBC=m
+CONFIG_CRYPTO_CRC32=m
 CONFIG_CRYPTO_LZO=m
 CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig
index 56a8f76dab4140..4fb4d45f3cdce1 100644
--- a/arch/mips/configs/malta_defconfig
+++ b/arch/mips/configs/malta_defconfig
@@ -333,7 +333,6 @@ CONFIG_AFFS_FS=m
 CONFIG_HFS_FS=m
 CONFIG_HFSPLUS_FS=m
 CONFIG_BEFS_FS=m
-CONFIG_BFS_FS=m
 CONFIG_EFS_FS=m
 CONFIG_JFFS2_FS=m
 CONFIG_JFFS2_FS_XATTR=y
diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig
index 85e95c0ae4107b..3d1c32d3475e78 100644
--- a/arch/mips/configs/malta_kvm_defconfig
+++ b/arch/mips/configs/malta_kvm_defconfig
@@ -340,7 +340,6 @@ CONFIG_AFFS_FS=m
 CONFIG_HFS_FS=m
 CONFIG_HFSPLUS_FS=m
 CONFIG_BEFS_FS=m
-CONFIG_BFS_FS=m
 CONFIG_EFS_FS=m
 CONFIG_JFFS2_FS=m
 CONFIG_JFFS2_FS_XATTR=y
diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig
index 0498a011534907..2545168b75bbce 100644
--- a/arch/mips/configs/maltaup_xpa_defconfig
+++ b/arch/mips/configs/maltaup_xpa_defconfig
@@ -339,7 +339,6 @@ CONFIG_AFFS_FS=m
 CONFIG_HFS_FS=m
 CONFIG_HFSPLUS_FS=m
 CONFIG_BEFS_FS=m
-CONFIG_BFS_FS=m
 CONFIG_EFS_FS=m
 CONFIG_JFFS2_FS=m
 CONFIG_JFFS2_FS_XATTR=y
diff --git a/arch/mips/configs/rm200_defconfig b/arch/mips/configs/rm200_defconfig
index 09e006c7cd7c62..cd2a831614c89e 100644
--- a/arch/mips/configs/rm200_defconfig
+++ b/arch/mips/configs/rm200_defconfig
@@ -318,7 +318,6 @@ CONFIG_ADFS_FS=m
 CONFIG_AFFS_FS=m
 CONFIG_HFS_FS=m
 CONFIG_BEFS_FS=m
-CONFIG_BFS_FS=m
 CONFIG_EFS_FS=m
 CONFIG_CRAMFS=m
 CONFIG_VXFS_FS=m
diff --git a/arch/mips/dec/setup.c b/arch/mips/dec/setup.c
index 78ad28ac16db5b..645473f4e098ea 100644
--- a/arch/mips/dec/setup.c
+++ b/arch/mips/dec/setup.c
@@ -18,16 +18,16 @@
 #include <linux/memblock.h>
 #include <linux/param.h>
 #include <linux/percpu-defs.h>
+#include <linux/pm.h>
 #include <linux/sched.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
-#include <linux/pm.h>
 
 #include <asm/addrspace.h>
 #include <asm/bootinfo.h>
-#include <asm/cpu.h>
 #include <asm/cpu-features.h>
 #include <asm/cpu-type.h>
+#include <asm/cpu.h>
 #include <asm/irq.h>
 #include <asm/irq_cpu.h>
 #include <asm/mipsregs.h>
diff --git a/arch/mips/generic/board-jaguar2.its.S b/arch/mips/generic/board-jaguar2.its.S
index c2b8d479b26cd2..a91a06f1254faa 100644
--- a/arch/mips/generic/board-jaguar2.its.S
+++ b/arch/mips/generic/board-jaguar2.its.S
@@ -28,13 +28,11 @@
 			description = "Jaguar2 Linux kernel";
 			kernel = "kernel";
 			fdt = "fdt-jaguar2_pcb110";
-			ramdisk = "ramdisk";
 		};
 		pcb111 {
 			description = "Jaguar2 Linux kernel";
 			kernel = "kernel";
 			fdt = "fdt-jaguar2_pcb111";
-			ramdisk = "ramdisk";
 		};
 	};
 };
diff --git a/arch/mips/generic/board-serval.its.S b/arch/mips/generic/board-serval.its.S
index dde833efe980aa..3b89e208d29aaa 100644
--- a/arch/mips/generic/board-serval.its.S
+++ b/arch/mips/generic/board-serval.its.S
@@ -18,7 +18,6 @@
 			description = "Serval Linux kernel";
 			kernel = "kernel";
 			fdt = "fdt-serval_pcb105";
-			ramdisk = "ramdisk";
 		};
 	};
 };
diff --git a/arch/mips/include/asm/amon.h b/arch/mips/include/asm/amon.h
deleted file mode 100644
index 3cc03c64a9c790..00000000000000
--- a/arch/mips/include/asm/amon.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2013 Imagination Technologies Ltd.
- *
- * Arbitrary Monitor Support (AMON)
- */
-int amon_cpu_avail(int cpu);
-int amon_cpu_start(int cpu, unsigned long pc, unsigned long sp,
-		   unsigned long gp, unsigned long a0);
diff --git a/arch/mips/include/asm/cmp.h b/arch/mips/include/asm/cmp.h
deleted file mode 100644
index 71e20e6cd38d87..00000000000000
--- a/arch/mips/include/asm/cmp.h
+++ /dev/null
@@ -1,10 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_CMP_H
-#define _ASM_CMP_H
-
-/*
- * Definitions for CMP multitasking on MIPS cores
- */
-struct task_struct;
-
-#endif /*  _ASM_CMP_H */
diff --git a/arch/mips/include/asm/dec/reset.h b/arch/mips/include/asm/dec/reset.h
index 6af49f3f24da83..caf60ef0edd0ce 100644
--- a/arch/mips/include/asm/dec/reset.h
+++ b/arch/mips/include/asm/dec/reset.h
@@ -10,6 +10,7 @@
 #define __ASM_DEC_RESET_H
 
 #include <linux/compiler_attributes.h>
+#include <linux/interrupt.h>
 
 void __noreturn dec_machine_restart(char *command);
 void __noreturn dec_machine_halt(void);
diff --git a/arch/mips/include/asm/mach-loongson64/boot_param.h b/arch/mips/include/asm/mach-loongson64/boot_param.h
index 3a11ce85762be6..3e5c2b3a7f9bea 100644
--- a/arch/mips/include/asm/mach-loongson64/boot_param.h
+++ b/arch/mips/include/asm/mach-loongson64/boot_param.h
@@ -24,7 +24,7 @@
 struct efi_memory_map_loongson {
 	u16 vers;	/* version of efi_memory_map */
 	u32 nr_map;	/* number of memory_maps */
-	u32 mem_freq;	/* memory frequence */
+	u32 mem_freq;	/* memory frequency */
 	struct mem_map {
 		u32 node_id;	/* node_id which memory attached to */
 		u32 mem_type;	/* system memory, pci memory, pci io, etc. */
diff --git a/arch/mips/include/asm/mips-boards/sead3-addr.h b/arch/mips/include/asm/mips-boards/sead3-addr.h
deleted file mode 100644
index c0db57802f7c35..00000000000000
--- a/arch/mips/include/asm/mips-boards/sead3-addr.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 2015 Imagination Technologies, Inc.
- *   written by Ralf Baechle <ralf@linux-mips.org>
- */
-#ifndef __ASM_MIPS_BOARDS_SEAD3_ADDR_H
-#define __ASM_MIPS_BOARDS_SEAD3_ADDR_H
-
-/*
- * Target #0 Register Decode
- */
-#define SEAD3_SD_SPDCNF				0xbb000040
-#define SEAD3_SD_SPADDR				0xbb000048
-#define SEAD3_SD_DATA				0xbb000050
-
-/*
- * Target #1 Register Decode
- */
-#define SEAD3_CFG				0xbb100110
-#define SEAD3_GIC_BASE_ADDRESS			0xbb1c0000
-#define SEAD3_SHARED_SECTION			0xbb1c0000
-#define SEAD3_VPE_LOCAL_SECTION			0xbb1c8000
-#define SEAD3_VPE_OTHER_SECTION			0xbb1cc000
-#define SEAD3_USER_MODE_VISIBLE_SECTION		0xbb1d0000
-
-/*
- * Target #3 Register Decode
- */
-#define SEAD3_USB_HS_BASE			0xbb200000
-#define SEAD3_USB_HS_IDENTIFICATION_REGS	0xbb200000
-#define SEAD3_USB_HS_CAPABILITY_REGS		0xbb200100
-#define SEAD3_USB_HS_OPERATIONAL_REGS		0xbb200140
-#define SEAD3_RESERVED				0xbe800000
-
-/*
- * Target #3 Register Decode
- */
-#define SEAD3_SRAM				0xbe000000
-#define SEAD3_OPTIONAL_SRAM			0xbe400000
-#define SEAD3_FPGA				0xbf000000
-
-#define SEAD3_PI_PIC32_USB_STATUS		0xbf000060
-#define   SEAD3_PI_PIC32_USB_STATUS_IO_RDY	(1 << 0)
-#define   SEAD3_PI_PIC32_USB_STATUS_SPL_INT	(1 << 1)
-#define   SEAD3_PI_PIC32_USB_STATUS_GPIOA_INT	(1 << 2)
-#define   SEAD3_PI_PIC32_USB_STATUS_GPIOB_INT	(1 << 3)
-
-#define SEAD3_PI_SOFT_ENDIAN			0xbf000070
-
-#define SEAD3_CPLD_P_SWITCH			0xbf000200
-#define SEAD3_CPLD_F_SWITCH			0xbf000208
-#define SEAD3_CPLD_P_LED			0xbf000210
-#define SEAD3_CPLD_F_LED			0xbf000218
-#define SEAD3_NEWSC_LIVE			0xbf000220
-#define SEAD3_NEWSC_REG				0xbf000228
-#define SEAD3_NEWSC_CTRL			0xbf000230
-
-#define SEAD3_LCD_CONTROL			0xbf000400
-#define SEAD3_LCD_DATA				0xbf000408
-#define SEAD3_CPLD_LCD_STATUS			0xbf000410
-#define SEAD3_CPLD_LCD_DATA			0xbf000418
-
-#define SEAD3_CPLD_PI_DEVRST			0xbf000480
-#define SEAD3_CPLD_PI_DEVRST_IC32_RST		(1 << 0)
-#define SEAD3_RESERVED_0			0xbf000500
-
-#define SEAD3_PIC32_REGISTERS			0xbf000600
-#define SEAD3_RESERVED_1			0xbf000700
-#define SEAD3_UART_CH_0				0xbf000800
-#define SEAD3_UART_CH_1				0xbf000900
-#define SEAD3_RESERVED_2			0xbf000a00
-#define SEAD3_ETHERNET				0xbf010000
-#define SEAD3_RESERVED_3			0xbf020000
-#define SEAD3_USER_EXPANSION			0xbf400000
-#define SEAD3_RESERVED_4			0xbf800000
-#define SEAD3_BOOT_FLASH_EXTENSION		0xbfa00000
-#define SEAD3_BOOT_FLASH			0xbfc00000
-#define SEAD3_REVISION_REGISTER			0xbfc00010
-
-#endif /* __ASM_MIPS_BOARDS_SEAD3_ADDR_H  */
diff --git a/arch/mips/include/asm/pgtable-32.h b/arch/mips/include/asm/pgtable-32.h
index 92b7591aac2acd..ef1001ab09c5be 100644
--- a/arch/mips/include/asm/pgtable-32.h
+++ b/arch/mips/include/asm/pgtable-32.h
@@ -104,16 +104,6 @@ extern int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
 # define VMALLOC_END	(FIXADDR_START-2*PAGE_SIZE)
 #endif
 
-#ifdef CONFIG_PHYS_ADDR_T_64BIT
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %016Lx.\n", __FILE__, __LINE__, pte_val(e))
-#else
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
-#endif
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
-
 extern void load_pgd(unsigned long pg_dir);
 
 extern pte_t invalid_pte_table[PTRS_PER_PTE];
diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h
index 6e854bb11f37de..785fc37bab9417 100644
--- a/arch/mips/include/asm/pgtable-64.h
+++ b/arch/mips/include/asm/pgtable-64.h
@@ -151,19 +151,6 @@
 #define MODULES_END	(FIXADDR_START-2*PAGE_SIZE)
 #endif
 
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
-#ifndef __PAGETABLE_PMD_FOLDED
-#define pmd_ERROR(e) \
-	printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
-#endif
-#ifndef __PAGETABLE_PUD_FOLDED
-#define pud_ERROR(e) \
-	printk("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
-#endif
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
-
 extern pte_t invalid_pte_table[PTRS_PER_PTE];
 
 #ifndef __PAGETABLE_PUD_FOLDED
diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h
index b43bfd44525213..adea35df0f8dc4 100644
--- a/arch/mips/include/asm/uasm.h
+++ b/arch/mips/include/asm/uasm.h
@@ -85,15 +85,20 @@ Ip_u1u2(_ctc1);
 Ip_u2u1(_ctcmsa);
 Ip_u2u1s3(_daddiu);
 Ip_u3u1u2(_daddu);
+Ip_u1u2(_ddiv);
+Ip_u3u1u2(_ddiv_r6);
 Ip_u1u2(_ddivu);
 Ip_u3u1u2(_ddivu_r6);
 Ip_u1(_di);
 Ip_u2u1msbu3(_dins);
 Ip_u2u1msbu3(_dinsm);
 Ip_u2u1msbu3(_dinsu);
+Ip_u1u2(_div);
+Ip_u3u1u2(_div_r6);
 Ip_u1u2(_divu);
 Ip_u3u1u2(_divu_r6);
 Ip_u1u2u3(_dmfc0);
+Ip_u3u1u2(_dmod);
 Ip_u3u1u2(_dmodu);
 Ip_u1u2u3(_dmtc0);
 Ip_u1u2(_dmultu);
@@ -135,6 +140,7 @@ Ip_u1u2u3(_mfc0);
 Ip_u1u2u3(_mfhc0);
 Ip_u1(_mfhi);
 Ip_u1(_mflo);
+Ip_u3u1u2(_mod);
 Ip_u3u1u2(_modu);
 Ip_u3u1u2(_movn);
 Ip_u3u1u2(_movz);
@@ -156,6 +162,8 @@ Ip_u2s3u1(_sb);
 Ip_u2s3u1(_sc);
 Ip_u2s3u1(_scd);
 Ip_u2s3u1(_sd);
+Ip_u2u1(_seb);
+Ip_u2u1(_seh);
 Ip_u3u1u2(_seleqz);
 Ip_u3u1u2(_selnez);
 Ip_u2s3u1(_sh);
diff --git a/arch/mips/include/uapi/asm/setup.h b/arch/mips/include/uapi/asm/setup.h
index 7d48c433b0c27d..8d6c474835aece 100644
--- a/arch/mips/include/uapi/asm/setup.h
+++ b/arch/mips/include/uapi/asm/setup.h
@@ -2,7 +2,11 @@
 #ifndef _UAPI_MIPS_SETUP_H
 #define _UAPI_MIPS_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	4096
+#endif
 
 
 #endif /* _UAPI_MIPS_SETUP_H */
diff --git a/arch/mips/kernel/perf_regs.c b/arch/mips/kernel/perf_regs.c
index e686780d1647af..00a5201dbd5db0 100644
--- a/arch/mips/kernel/perf_regs.c
+++ b/arch/mips/kernel/perf_regs.c
@@ -28,7 +28,7 @@ u64 perf_reg_abi(struct task_struct *tsk)
 }
 #endif /* CONFIG_32BIT */
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask)
 		return -EINVAL;
@@ -60,9 +60,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 	return (s64)v; /* Sign extend if 32-bit. */
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
diff --git a/arch/mips/lib/memcpy.S b/arch/mips/lib/memcpy.S
index 84f85aba6f4b86..49408c0049f2f4 100644
--- a/arch/mips/lib/memcpy.S
+++ b/arch/mips/lib/memcpy.S
@@ -79,7 +79,7 @@
  *  3- (dst - src) == (dst_entry - src_entry),
  * The _entry suffix denotes values when __copy_user was called.
  *
- * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user
+ * (1) is set up by uaccess.h and maintained by not writing AT in copy_user
  * (2) is met by incrementing src by the number of bytes copied
  * (3) is met by not doing loads between a pair of increments of dst and src
  *
diff --git a/arch/mips/mm/mmap.c b/arch/mips/mm/mmap.c
index 5d2a1225785b81..119edaa928f336 100644
--- a/arch/mips/mm/mmap.c
+++ b/arch/mips/mm/mmap.c
@@ -9,6 +9,7 @@
 #include <linux/compiler.h>
 #include <linux/elf-randomize.h>
 #include <linux/errno.h>
+#include <linux/hugetlb.h>
 #include <linux/mm.h>
 #include <linux/mman.h>
 #include <linux/export.h>
@@ -72,8 +73,11 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp,
 	}
 
 	info.length = len;
-	info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
 	info.align_offset = pgoff << PAGE_SHIFT;
+	if (filp && is_file_hugepages(filp))
+		info.align_mask = huge_page_mask_align(filp);
+	else
+		info.align_mask = do_color_align ? (PAGE_MASK & shm_align_mask) : 0;
 
 	if (dir == DOWN) {
 		info.flags = VM_UNMAPPED_AREA_TOPDOWN;
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index 24fe85fa169d16..78bafa317fa6ff 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -332,6 +332,19 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 	/* this could be a huge page  */
 	if (pmd_leaf(*pmdp)) {
 		unsigned long lo;
+
+		/*
+		 * The probe above only covers the 8K pair at @address, and
+		 * a huge mapping is installed with the huge-aligned address
+		 * while the refill that started the fault left a 4K entry
+		 * for the faulting page elsewhere in the range. Writing the
+		 * huge entry to a random index would leave two entries
+		 * matching the faulting address; leave it to the refill and
+		 * TLBL/TLBS handlers, which probe the faulting address.
+		 */
+		if (idx < 0)
+			goto out;
+
 		write_c0_pagemask(PM_HUGE_MASK);
 		ptep = (pte_t *)pmdp;
 		lo = pte_to_entrylo(pte_val(*ptep));
@@ -339,10 +352,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 		write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
 
 		mtc0_tlbw_hazard();
-		if (idx < 0)
-			tlb_write_random();
-		else
-			tlb_write_indexed();
+		tlb_write_indexed();
 		tlbw_use_hazard();
 		write_c0_pagemask(PM_DEFAULT_MASK);
 	} else
@@ -380,6 +390,9 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 			tlb_write_indexed();
 	}
 	tlbw_use_hazard();
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
+out:
+#endif
 	htw_start();
 	flush_micro_tlb_vm(vma);
 
diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c
index e15c6700cd0884..2d1668b58ddde2 100644
--- a/arch/mips/mm/uasm-mips.c
+++ b/arch/mips/mm/uasm-mips.c
@@ -75,6 +75,9 @@ static const struct insn insn_table[insn_invalid] = {
 	[insn_ctcmsa]	= {M(msa_op, 0, msa_ctc_op, 0, 0, msa_elm_op), RD | RE},
 	[insn_daddiu]	= {M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
 	[insn_daddu]	= {M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD},
+	[insn_ddiv]	= {M(spec_op, 0, 0, 0, 0, ddiv_op), RS | RT},
+	[insn_ddiv_r6]	= {M(spec_op, 0, 0, 0, ddiv_ddiv6_op, ddiv_op),
+				RS | RT | RD},
 	[insn_ddivu]	= {M(spec_op, 0, 0, 0, 0, ddivu_op), RS | RT},
 	[insn_ddivu_r6]	= {M(spec_op, 0, 0, 0, ddivu_ddivu6_op, ddivu_op),
 				RS | RT | RD},
@@ -82,10 +85,15 @@ static const struct insn insn_table[insn_invalid] = {
 	[insn_dins]	= {M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE},
 	[insn_dinsm]	= {M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE},
 	[insn_dinsu]	= {M(spec3_op, 0, 0, 0, 0, dinsu_op), RS | RT | RD | RE},
+	[insn_div]	= {M(spec_op, 0, 0, 0, 0, div_op), RS | RT},
+	[insn_div_r6]	= {M(spec_op, 0, 0, 0, div_div6_op, div_op),
+				RS | RT | RD},
 	[insn_divu]	= {M(spec_op, 0, 0, 0, 0, divu_op), RS | RT},
 	[insn_divu_r6]	= {M(spec_op, 0, 0, 0, divu_divu6_op, divu_op),
 				RS | RT | RD},
 	[insn_dmfc0]	= {M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
+	[insn_dmod]	= {M(spec_op, 0, 0, 0, ddiv_dmod_op, ddiv_op),
+				RS | RT | RD},
 	[insn_dmodu]	= {M(spec_op, 0, 0, 0, ddivu_dmodu_op, ddivu_op),
 				RS | RT | RD},
 	[insn_dmtc0]	= {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
@@ -140,6 +148,8 @@ static const struct insn insn_table[insn_invalid] = {
 	[insn_mfhc0]	= {M(cop0_op, mfhc0_op, 0, 0, 0, 0),  RT | RD | SET},
 	[insn_mfhi]	= {M(spec_op, 0, 0, 0, 0, mfhi_op), RD},
 	[insn_mflo]	= {M(spec_op, 0, 0, 0, 0, mflo_op), RD},
+	[insn_mod]	= {M(spec_op, 0, 0, 0, div_mod_op, div_op),
+				RS | RT | RD},
 	[insn_modu]	= {M(spec_op, 0, 0, 0, divu_modu_op, divu_op),
 				RS | RT | RD},
 	[insn_movn]	= {M(spec_op, 0, 0, 0, 0, movn_op), RS | RT | RD},
@@ -177,6 +187,8 @@ static const struct insn insn_table[insn_invalid] = {
 	[insn_scd]	= {M6(spec3_op, 0, 0, 0, scd6_op),  RS | RT | SIMM9},
 #endif
 	[insn_sd]	= {M(sd_op, 0, 0, 0, 0, 0),  RS | RT | SIMM},
+	[insn_seb]	= {M(spec3_op, 0, 0, 0, seb_op, bshfl_op), RT | RD},
+	[insn_seh]	= {M(spec3_op, 0, 0, 0, seh_op, bshfl_op), RT | RD},
 	[insn_seleqz]	= {M(spec_op, 0, 0, 0, 0, seleqz_op), RS | RT | RD},
 	[insn_selnez]	= {M(spec_op, 0, 0, 0, 0, selnez_op), RS | RT | RD},
 	[insn_sh]	= {M(sh_op, 0, 0, 0, 0, 0),  RS | RT | SIMM},
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
index 125140979d62c5..139afb0e14cdfb 100644
--- a/arch/mips/mm/uasm.c
+++ b/arch/mips/mm/uasm.c
@@ -49,19 +49,21 @@ enum opcode {
 	insn_addiu, insn_addu, insn_and, insn_andi, insn_bbit0, insn_bbit1,
 	insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bgtz, insn_blez,
 	insn_bltz, insn_bltzl, insn_bne, insn_break, insn_cache, insn_cfc1,
-	insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddivu,
-	insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu,
-	insn_divu_r6, insn_dmfc0, insn_dmodu, insn_dmtc0, insn_dmultu,
+	insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddiv,
+	insn_ddiv_r6, insn_ddivu, insn_ddivu_r6, insn_di, insn_dins, insn_dinsm,
+	insn_dinsu, insn_div, insn_div_r6, insn_divu, insn_divu_r6, insn_dmfc0,
+	insn_dmod, insn_dmodu, insn_dmtc0, insn_dmultu,
 	insn_dmulu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd, insn_dsll,
 	insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav, insn_dsrl,
 	insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext, insn_ins,
 	insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu, insn_ld,
 	insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld,
 	insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi,
-	insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0,
-	insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor,
-	insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc,
-	insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll,
+	insn_mflo, insn_mod, insn_modu, insn_movn, insn_movz, insn_mtc0,
+	insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu,
+	insn_muhu, insn_nor, insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr,
+	insn_sb, insn_sc, insn_scd, insn_seb, insn_seh,
+	insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll,
 	insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra,
 	insn_srav, insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync,
 	insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait,
@@ -287,9 +289,12 @@ I_u1u2(_cfc1)
 I_u2u1(_cfcmsa)
 I_u1u2(_ctc1)
 I_u2u1(_ctcmsa)
+I_u1u2(_ddiv)
+I_u3u1u2(_ddiv_r6)
 I_u1u2(_ddivu)
 I_u3u1u2(_ddivu_r6)
 I_u1u2u3(_dmfc0)
+I_u3u1u2(_dmod)
 I_u3u1u2(_dmodu)
 I_u1u2u3(_dmtc0)
 I_u1u2(_dmultu)
@@ -297,6 +302,8 @@ I_u3u1u2(_dmulu)
 I_u2u1s3(_daddiu)
 I_u3u1u2(_daddu)
 I_u1(_di);
+I_u1u2(_div)
+I_u3u1u2(_div_r6)
 I_u1u2(_divu)
 I_u3u1u2(_divu_r6)
 I_u2u1(_dsbh);
@@ -332,6 +339,7 @@ I_u2s3u1(_lw)
 I_u2s3u1(_lwu)
 I_u1u2u3(_mfc0)
 I_u1u2u3(_mfhc0)
+I_u3u1u2(_mod)
 I_u3u1u2(_modu)
 I_u3u1u2(_movn)
 I_u3u1u2(_movz)
@@ -353,6 +361,8 @@ I_u2s3u1(_sb)
 I_u2s3u1(_sc)
 I_u2s3u1(_scd)
 I_u2s3u1(_sd)
+I_u2u1(_seb)
+I_u2u1(_seh)
 I_u3u1u2(_seleqz)
 I_u3u1u2(_selnez)
 I_u2s3u1(_sh)
diff --git a/arch/mips/n64/init.c b/arch/mips/n64/init.c
index 66ec28ab41f321..ddbd7d1224d311 100644
--- a/arch/mips/n64/init.c
+++ b/arch/mips/n64/init.c
@@ -20,7 +20,7 @@
 #define IO_MEM_RESOURCE_END	0x1fffffffUL
 
 /*
- * System-specifc irq names for clarity
+ * System-specific irq names for clarity
  */
 #define MIPS_CPU_IRQ(x)		(MIPS_CPU_IRQ_BASE + (x))
 #define MIPS_SOFTINT0_IRQ	MIPS_CPU_IRQ(0)
diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index 6ee4abe6a1f705..4682dbb3a05920 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -208,7 +208,7 @@ void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src)
 }
 
 /* Validate ALU immediate range */
-bool valid_alu_i(u8 op, s32 imm)
+bool valid_alu_i(u8 op, s32 imm, s16 off)
 {
 	switch (BPF_OP(op)) {
 	case BPF_NEG:
@@ -237,6 +237,9 @@ bool valid_alu_i(u8 op, s32 imm)
 		return imm == 0 || (imm > 0 && is_power_of_2(imm));
 	case BPF_DIV:
 	case BPF_MOD:
+		/* Do not use unsigned shift/mask rewrites for signed div/mod. */
+		if (off == 1)
+			return false;
 		/* imm must be an 17-bit power of two */
 		return (u32)imm <= 0x10000 && is_power_of_2((u32)imm);
 	}
@@ -338,8 +341,42 @@ void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
 	clobber_reg(ctx, dst);
 }
 
+/* ALU division operation (32-bit) */
+static void emit_div(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	if (cpu_has_mips32r6) {
+		if (off == 1)
+			emit(ctx, div_r6, dst, dst, src);
+		else
+			emit(ctx, divu_r6, dst, dst, src);
+	} else {
+		if (off == 1)
+			emit(ctx, div, dst, src);
+		else
+			emit(ctx, divu, dst, src);
+		emit(ctx, mflo, dst);
+	}
+}
+
+/* ALU modulo operation (32-bit) */
+static void emit_mod(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	if (cpu_has_mips32r6) {
+		if (off == 1)
+			emit(ctx, mod, dst, dst, src);
+		else
+			emit(ctx, modu, dst, dst, src);
+	} else {
+		if (off == 1)
+			emit(ctx, div, dst, src);
+		else
+			emit(ctx, divu, dst, src);
+		emit(ctx, mfhi, dst);
+	}
+}
+
 /* ALU register operation (32-bit) */
-void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
+void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off)
 {
 	switch (BPF_OP(op)) {
 	/* dst = dst & src */
@@ -385,21 +422,11 @@ void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op)
 		break;
 	/* dst = dst / src */
 	case BPF_DIV:
-		if (cpu_has_mips32r6) {
-			emit(ctx, divu_r6, dst, dst, src);
-		} else {
-			emit(ctx, divu, dst, src);
-			emit(ctx, mflo, dst);
-		}
+		emit_div(ctx, dst, src, off);
 		break;
 	/* dst = dst % src */
 	case BPF_MOD:
-		if (cpu_has_mips32r6) {
-			emit(ctx, modu, dst, dst, src);
-		} else {
-			emit(ctx, divu, dst, src);
-			emit(ctx, mfhi, dst);
-		}
+		emit_mod(ctx, dst, src, off);
 		break;
 	}
 	clobber_reg(ctx, dst);
@@ -823,7 +850,7 @@ void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op)
 }
 
 /* Jump always */
-int emit_ja(struct jit_context *ctx, s16 off)
+int emit_ja(struct jit_context *ctx, s32 off)
 {
 	int target = get_target(ctx, ctx->bpf_index + off + 1);
 
diff --git a/arch/mips/net/bpf_jit_comp.h b/arch/mips/net/bpf_jit_comp.h
index a37fe20818eb9c..d0ab5c9a0c46f7 100644
--- a/arch/mips/net/bpf_jit_comp.h
+++ b/arch/mips/net/bpf_jit_comp.h
@@ -163,7 +163,7 @@ void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm);
 void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src);
 
 /* Validate ALU/ALU64 immediate range */
-bool valid_alu_i(u8 op, s32 imm);
+bool valid_alu_i(u8 op, s32 imm, s16 off);
 
 /* Rewrite ALU/ALU64 immediate operation */
 bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val);
@@ -172,7 +172,7 @@ bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val);
 void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op);
 
 /* ALU register operation (32-bit) */
-void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op);
+void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op, s16 off);
 
 /* Atomic read-modify-write (32-bit) */
 void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code);
@@ -204,7 +204,7 @@ void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op);
 void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op);
 
 /* Jump always */
-int emit_ja(struct jit_context *ctx, s16 off);
+int emit_ja(struct jit_context *ctx, s32 off);
 
 /* Jump to epilogue */
 int emit_exit(struct jit_context *ctx);
diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c
index 40a878b672f5d3..84d9c7036e4f09 100644
--- a/arch/mips/net/bpf_jit_comp32.c
+++ b/arch/mips/net/bpf_jit_comp32.c
@@ -190,6 +190,69 @@ static void emit_zext_ver(struct jit_context *ctx, const u8 dst[])
 	}
 }
 
+/* Sign-extend an 8- or 16-bit field into a native register. */
+static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	int shift;
+
+	if (cpu_has_mips32r2 || cpu_has_mips32r6) {
+		if (off == 8)
+			emit(ctx, seb, dst, src);
+		else
+			emit(ctx, seh, dst, src);
+	} else {
+		shift = 32 - off;
+		emit(ctx, sll, dst, src, shift);
+		emit(ctx, sra, dst, dst, shift);
+	}
+}
+
+/* Register move operation (32-bit), optionally with sign extension */
+static void emit_mov_r32(struct jit_context *ctx, const u8 dst[],
+			 const u8 src[], s16 off)
+{
+	switch (off) {
+	case 8:
+	case 16:
+		emit_movsx_r(ctx, lo(dst), lo(src), off);
+		break;
+	default:
+		/* off == 0 is MOV; the verifier rejects other offsets. */
+		emit_mov_r(ctx, lo(dst), lo(src));
+		break;
+	}
+	clobber_reg(ctx, lo(dst));
+	emit_zext_ver(ctx, dst);
+}
+
+/* Register move operation (64-bit), optionally with sign extension */
+static void emit_mov_r64(struct jit_context *ctx, const u8 dst[],
+			 const u8 src[], s16 off)
+{
+	switch (off) {
+	case 8:
+	case 16:
+		emit_movsx_r(ctx, lo(dst), lo(src), off);
+		emit(ctx, sra, hi(dst), lo(dst), 31);
+		break;
+	case 32:
+		emit(ctx, move, lo(dst), lo(src));
+		emit(ctx, sra, hi(dst), lo(dst), 31);
+		break;
+	default:
+		/*
+		 * off == 0 is ordinary MOV. The verifier rejects other
+		 * offsets; defined exceptions require
+		 * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(),
+		 * neither implemented by MIPS.
+		 */
+		emit_mov_r(ctx, lo(dst), lo(src));
+		emit_mov_r(ctx, hi(dst), hi(src));
+		break;
+	}
+	clobber_reg64(ctx, dst);
+}
+
 /* Load delay slot, if ISA mandates it */
 static void emit_load_delay(struct jit_context *ctx)
 {
@@ -512,7 +575,7 @@ static void emit_mul_r64(struct jit_context *ctx,
 	clobber_reg64(ctx, dst);
 }
 
-/* Helper function for 64-bit modulo */
+/* Helper function for unsigned 64-bit modulo */
 static u64 jit_mod64(u64 a, u64 b)
 {
 	u64 rem;
@@ -521,13 +584,22 @@ static u64 jit_mod64(u64 a, u64 b)
 	return rem;
 }
 
+/* Helper function for signed 64-bit modulo */
+static s64 jit_smod64(s64 a, s64 b)
+{
+	s64 quot = div64_s64(a, b);
+
+	return a - quot * b;
+}
+
 /* ALU div/mod register (64-bit) */
-static void emit_divmod_r64(struct jit_context *ctx,
-			    const u8 dst[], const u8 src[], u8 op)
+static void emit_divmod_r64(struct jit_context *ctx, const u8 dst[],
+			    const u8 src[], u8 op, s16 off)
 {
 	const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */
 	const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */
 	const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */
+	bool is_signed = off == 1;
 	int exclude, k;
 	u32 addr = 0;
 
@@ -546,11 +618,11 @@ static void emit_divmod_r64(struct jit_context *ctx,
 	switch (BPF_OP(op)) {
 	/* dst = dst / src */
 	case BPF_DIV:
-		addr = (u32)&div64_u64;
+		addr = is_signed ? (u32)&div64_s64 : (u32)&div64_u64;
 		break;
 	/* dst = dst % src */
 	case BPF_MOD:
-		addr = (u32)&jit_mod64;
+		addr = is_signed ? (u32)&jit_smod64 : (u32)&jit_mod64;
 		break;
 	}
 	emit_mov_i(ctx, MIPS_R_T9, addr);
@@ -1111,7 +1183,7 @@ static void emit_jmp_i64(struct jit_context *ctx,
 			emit(ctx, xor, tmp, lo(dst), tmp);
 		}
 		if (imm < 0) { /* Compare sign extension */
-			emit(ctx, addu, MIPS_R_T9, hi(dst), 1);
+			emit(ctx, addiu, MIPS_R_T9, hi(dst), 1);
 			emit(ctx, or, tmp, tmp, MIPS_R_T9);
 		} else {       /* Compare zero extension */
 			emit(ctx, or, tmp, tmp, hi(dst));
@@ -1485,8 +1557,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 			/* Special mov32 for zext */
 			emit_mov_i(ctx, hi(dst), 0);
 		} else {
-			emit_mov_r(ctx, lo(dst), lo(src));
-			emit_zext_ver(ctx, dst);
+			emit_mov_r32(ctx, dst, src, off);
 		}
 		break;
 	/* dst = -dst */
@@ -1516,9 +1587,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU | BPF_MUL | BPF_K:
 	case BPF_ALU | BPF_DIV | BPF_K:
 	case BPF_ALU | BPF_MOD | BPF_K:
-		if (!valid_alu_i(BPF_OP(code), imm)) {
+		if (!valid_alu_i(BPF_OP(code), imm, off)) {
 			emit_mov_i(ctx, MIPS_R_T6, imm);
-			emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code));
+			emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code), off);
 		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
 			emit_alu_i(ctx, lo(dst), val, alu);
 		}
@@ -1546,7 +1617,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU | BPF_MUL | BPF_X:
 	case BPF_ALU | BPF_DIV | BPF_X:
 	case BPF_ALU | BPF_MOD | BPF_X:
-		emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code));
+		emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code), off);
 		emit_zext_ver(ctx, dst);
 		break;
 	/* dst = imm (64-bit) */
@@ -1555,8 +1626,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 		break;
 	/* dst = src (64-bit) */
 	case BPF_ALU64 | BPF_MOV | BPF_X:
-		emit_mov_r(ctx, lo(dst), lo(src));
-		emit_mov_r(ctx, hi(dst), hi(src));
+		emit_mov_r64(ctx, dst, src, off);
 		break;
 	/* dst = -dst (64-bit) */
 	case BPF_ALU64 | BPF_NEG:
@@ -1599,7 +1669,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 		 * and then do the operation on this register.
 		 */
 		emit_mov_se_i64(ctx, tmp, imm);
-		emit_divmod_r64(ctx, dst, tmp, BPF_OP(code));
+		emit_divmod_r64(ctx, dst, tmp, BPF_OP(code), off);
 		break;
 	/* dst = dst & src (64-bit) */
 	/* dst = dst | src (64-bit) */
@@ -1629,7 +1699,11 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	/* dst = dst % src (64-bit) */
 	case BPF_ALU64 | BPF_DIV | BPF_X:
 	case BPF_ALU64 | BPF_MOD | BPF_X:
-		emit_divmod_r64(ctx, dst, src, BPF_OP(code));
+		emit_divmod_r64(ctx, dst, src, BPF_OP(code), off);
+		break;
+	/* dst = bswap(dst) */
+	case BPF_ALU64 | BPF_END:
+		emit_bswap_r64(ctx, dst, imm);
 		break;
 	/* dst = htole(dst) */
 	/* dst = htobe(dst) */
@@ -1803,6 +1877,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 		if (finish_jmp(ctx, jmp, off) < 0)
 			goto toofar;
 		break;
+	/* PC += imm */
+	case BPF_JMP32 | BPF_JA:
+		if (imm == 0)
+			break;
+		if (emit_ja(ctx, imm) < 0)
+			goto toofar;
+		break;
 	/* PC += off if dst == src */
 	/* PC += off if dst != src */
 	/* PC += off if dst & src */
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
index fa7e9aa37f498d..994589c3de46c8 100644
--- a/arch/mips/net/bpf_jit_comp64.c
+++ b/arch/mips/net/bpf_jit_comp64.c
@@ -120,6 +120,58 @@ static void emit_zext_ver(struct jit_context *ctx, u8 dst)
 		emit_zext(ctx, dst);
 }
 
+/* Sign-extend an 8- or 16-bit field into a native register. */
+static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	/* seb/seh would need a sign-extended 32-bit source first. */
+	int shift = 32 - off;
+
+	/* dsll32 and dsra32 add 32 to the shift argument. */
+	emit(ctx, dsll32, dst, src, shift);
+	emit(ctx, dsra32, dst, dst, shift);
+}
+
+/* Register move operation (32-bit), optionally with sign extension */
+static void emit_mov_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	switch (off) {
+	case 8:
+	case 16:
+		emit_movsx_r(ctx, dst, src, off);
+		break;
+	default:
+		/* off == 0 is MOV; the verifier rejects other offsets. */
+		emit_mov_r(ctx, dst, src);
+		break;
+	}
+	clobber_reg(ctx, dst);
+	emit_zext_ver(ctx, dst);
+}
+
+/* Register move operation (64-bit), optionally with sign extension */
+static void emit_mov_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	switch (off) {
+	case 8:
+	case 16:
+		emit_movsx_r(ctx, dst, src, off);
+		break;
+	case 32:
+		emit_sext(ctx, dst, src);
+		break;
+	default:
+		/*
+		 * off == 0 is ordinary MOV. The verifier rejects other
+		 * offsets; defined exceptions require
+		 * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(),
+		 * neither implemented by MIPS.
+		 */
+		emit_mov_r(ctx, dst, src);
+		break;
+	}
+	clobber_reg(ctx, dst);
+}
+
 /* dst = imm (64-bit) */
 static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64)
 {
@@ -197,8 +249,43 @@ static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op)
 	clobber_reg(ctx, dst);
 }
 
+/* ALU division operation (64-bit) */
+static void emit_div64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	if (cpu_has_mips64r6) {
+		if (off == 1)
+			emit(ctx, ddiv_r6, dst, dst, src);
+		else
+			emit(ctx, ddivu_r6, dst, dst, src);
+	} else {
+		if (off == 1)
+			emit(ctx, ddiv, dst, src);
+		else
+			emit(ctx, ddivu, dst, src);
+		emit(ctx, mflo, dst);
+	}
+}
+
+/* ALU modulo operation (64-bit) */
+static void emit_mod64(struct jit_context *ctx, u8 dst, u8 src, s16 off)
+{
+	if (cpu_has_mips64r6) {
+		if (off == 1)
+			emit(ctx, dmod, dst, dst, src);
+		else
+			emit(ctx, dmodu, dst, dst, src);
+	} else {
+		if (off == 1)
+			emit(ctx, ddiv, dst, src);
+		else
+			emit(ctx, ddivu, dst, src);
+		emit(ctx, mfhi, dst);
+	}
+}
+
 /* ALU register operation (64-bit) */
-static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op)
+static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op,
+			 s16 off)
 {
 	switch (BPF_OP(op)) {
 	/* dst = dst << src */
@@ -235,25 +322,15 @@ static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op)
 		break;
 	/* dst = dst / src */
 	case BPF_DIV:
-		if (cpu_has_mips64r6) {
-			emit(ctx, ddivu_r6, dst, dst, src);
-		} else {
-			emit(ctx, ddivu, dst, src);
-			emit(ctx, mflo, dst);
-		}
+		emit_div64(ctx, dst, src, off);
 		break;
 	/* dst = dst % src */
 	case BPF_MOD:
-		if (cpu_has_mips64r6) {
-			emit(ctx, dmodu, dst, dst, src);
-		} else {
-			emit(ctx, ddivu, dst, src);
-			emit(ctx, mfhi, dst);
-		}
+		emit_mod64(ctx, dst, src, off);
 		break;
 	default:
 		/* Width-generic operations */
-		emit_alu_r(ctx, dst, src, op);
+		emit_alu_r(ctx, dst, src, op, off);
 	}
 	clobber_reg(ctx, dst);
 }
@@ -305,8 +382,7 @@ static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width)
 	case 16:
 		emit_sext(ctx, dst, dst);
 		emit_bswap_r(ctx, dst, width);
-		if (cpu_has_mips64r2 || cpu_has_mips64r6)
-			emit_zext(ctx, dst);
+		emit_zext(ctx, dst);
 		break;
 	}
 	clobber_reg(ctx, dst);
@@ -656,8 +732,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 			/* Special mov32 for zext */
 			emit_zext(ctx, dst);
 		} else {
-			emit_mov_r(ctx, dst, src);
-			emit_zext_ver(ctx, dst);
+			emit_mov_r32(ctx, dst, src, off);
 		}
 		break;
 	/* dst = -dst */
@@ -674,9 +749,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU | BPF_AND | BPF_K:
 	case BPF_ALU | BPF_XOR | BPF_K:
 	case BPF_ALU | BPF_LSH | BPF_K:
-		if (!valid_alu_i(BPF_OP(code), imm)) {
+		if (!valid_alu_i(BPF_OP(code), imm, off)) {
 			emit_mov_i(ctx, MIPS_R_T4, imm);
-			emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
+			emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
 		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
 			emit_alu_i(ctx, dst, val, alu);
 		}
@@ -696,10 +771,10 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU | BPF_MUL | BPF_K:
 	case BPF_ALU | BPF_DIV | BPF_K:
 	case BPF_ALU | BPF_MOD | BPF_K:
-		if (!valid_alu_i(BPF_OP(code), imm)) {
+		if (!valid_alu_i(BPF_OP(code), imm, off)) {
 			emit_sext(ctx, dst, dst);
 			emit_mov_i(ctx, MIPS_R_T4, imm);
-			emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
+			emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
 		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
 			emit_sext(ctx, dst, dst);
 			emit_alu_i(ctx, dst, val, alu);
@@ -714,7 +789,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU | BPF_OR | BPF_X:
 	case BPF_ALU | BPF_XOR | BPF_X:
 	case BPF_ALU | BPF_LSH | BPF_X:
-		emit_alu_r(ctx, dst, src, BPF_OP(code));
+		emit_alu_r(ctx, dst, src, BPF_OP(code), off);
 		emit_zext_ver(ctx, dst);
 		break;
 	/* dst = dst >> src */
@@ -733,7 +808,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU | BPF_MOD | BPF_X:
 		emit_sext(ctx, dst, dst);
 		emit_sext(ctx, MIPS_R_T4, src);
-		emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code));
+		emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
 		emit_zext_ver(ctx, dst);
 		break;
 	/* dst = imm (64-bit) */
@@ -742,7 +817,7 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 		break;
 	/* dst = src (64-bit) */
 	case BPF_ALU64 | BPF_MOV | BPF_X:
-		emit_mov_r(ctx, dst, src);
+		emit_mov_r64(ctx, dst, src, off);
 		break;
 	/* dst = -dst (64-bit) */
 	case BPF_ALU64 | BPF_NEG:
@@ -770,9 +845,9 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU64 | BPF_MUL | BPF_K:
 	case BPF_ALU64 | BPF_DIV | BPF_K:
 	case BPF_ALU64 | BPF_MOD | BPF_K:
-		if (!valid_alu_i(BPF_OP(code), imm)) {
+		if (!valid_alu_i(BPF_OP(code), imm, off)) {
 			emit_mov_i(ctx, MIPS_R_T4, imm);
-			emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code));
+			emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code), off);
 		} else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) {
 			emit_alu_i64(ctx, dst, val, alu);
 		}
@@ -799,7 +874,11 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 	case BPF_ALU64 | BPF_MUL | BPF_X:
 	case BPF_ALU64 | BPF_DIV | BPF_X:
 	case BPF_ALU64 | BPF_MOD | BPF_X:
-		emit_alu_r64(ctx, dst, src, BPF_OP(code));
+		emit_alu_r64(ctx, dst, src, BPF_OP(code), off);
+		break;
+	/* dst = bswap(dst) */
+	case BPF_ALU64 | BPF_END:
+		emit_bswap_r64(ctx, dst, imm);
 		break;
 	/* dst = htole(dst) */
 	/* dst = htobe(dst) */
@@ -962,6 +1041,13 @@ int build_insn(const struct bpf_insn *insn, struct jit_context *ctx)
 		if (finish_jmp(ctx, jmp, off) < 0)
 			goto toofar;
 		break;
+	/* PC += imm */
+	case BPF_JMP32 | BPF_JA:
+		if (imm == 0)
+			break;
+		if (emit_ja(ctx, imm) < 0)
+			goto toofar;
+		break;
 	/* PC += off if dst == src */
 	/* PC += off if dst != src */
 	/* PC += off if dst & src */
diff --git a/arch/mips/rb532/gpio.c b/arch/mips/rb532/gpio.c
index 49e1a15a3827ea..458895a9fb2526 100644
--- a/arch/mips/rb532/gpio.c
+++ b/arch/mips/rb532/gpio.c
@@ -192,13 +192,8 @@ EXPORT_SYMBOL(rb532_gpio_set_func);
 static int rb532_gpio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct resource *res;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -EINVAL;
-
-	rb532_gpio_chip->regbase = devm_ioremap_resource(dev, res);
+	rb532_gpio_chip->regbase = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(rb532_gpio_chip->regbase))
 		return PTR_ERR(rb532_gpio_chip->regbase);
 
diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h
index d389aa9ca57ce0..272707d48f1bb1 100644
--- a/arch/nios2/include/asm/pgtable.h
+++ b/arch/nios2/include/asm/pgtable.h
@@ -223,13 +223,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 	return pmd_val(pmd);
 }
 
-#define pte_ERROR(e) \
-	pr_err("%s:%d: bad pte %08lx.\n", \
-		__FILE__, __LINE__, pte_val(e))
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08lx.\n", \
-		__FILE__, __LINE__, pgd_val(e))
-
 /*
  * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
  * are !pte_none() && !pte_present().
diff --git a/arch/nios2/include/asm/tlb.h b/arch/nios2/include/asm/tlb.h
index f9f2e27e32dd5e..5cca73a5d1fae2 100644
--- a/arch/nios2/include/asm/tlb.h
+++ b/arch/nios2/include/asm/tlb.h
@@ -18,7 +18,6 @@ extern void set_mmu_pid(unsigned long pid);
  * full mm invalidation. So use flush_tlb_mm() for everything.
  */
 
-#include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
 
 #endif /* _ASM_NIOS2_TLB_H */
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index 8aa34621702dea..91d8432ed9918a 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -9,4 +9,5 @@ generic-y += spinlock.h
 generic-y += qrwlock_types.h
 generic-y += qrwlock.h
 generic-y += ring_buffer.h
+generic-y += tlb.h
 generic-y += user.h
diff --git a/arch/openrisc/include/asm/pgtable.h b/arch/openrisc/include/asm/pgtable.h
index 6b89996d0b628e..13afcc0bd8631b 100644
--- a/arch/openrisc/include/asm/pgtable.h
+++ b/arch/openrisc/include/asm/pgtable.h
@@ -338,13 +338,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 #define pte_pfn(x)		((unsigned long)(((x).pte)) >> PAGE_SHIFT)
 #define pfn_pte(pfn, prot)  __pte((((pfn) << PAGE_SHIFT)) | pgprot_val(prot))
 
-#define pte_ERROR(e) \
-	printk(KERN_ERR "%s:%d: bad pte %p(%08lx).\n", \
-	       __FILE__, __LINE__, &(e), pte_val(e))
-#define pgd_ERROR(e) \
-	printk(KERN_ERR "%s:%d: bad pgd %p(%08lx).\n", \
-	       __FILE__, __LINE__, &(e), pgd_val(e))
-
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; /* defined in head.S */
 
 struct vm_area_struct;
diff --git a/arch/openrisc/include/asm/tlb.h b/arch/openrisc/include/asm/tlb.h
deleted file mode 100644
index 1936d0494b2a7a..00000000000000
--- a/arch/openrisc/include/asm/tlb.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * OpenRISC Linux
- *
- * Linux architectural port borrowing liberally from similar works of
- * others.  All original copyrights apply as per the original source
- * declaration.
- *
- * OpenRISC implementation:
- * Copyright (C) 2003 Matjaz Breskvar <phoenix@bsemi.com>
- * Copyright (C) 2010-2011 Jonas Bonn <jonas@southpole.se>
- * et al.
- */
-
-#ifndef __ASM_OPENRISC_TLB_H__
-#define __ASM_OPENRISC_TLB_H__
-
-/*
- * OpenRISC doesn't have an efficient flush_tlb_range() so use flush_tlb_mm()
- * for everything.
- */
-
-#include <linux/pagemap.h>
-#include <asm-generic/tlb.h>
-
-#endif /* __ASM_OPENRISC_TLB_H__ */
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index d3afac2f0d9be9..77f67028ad89ce 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -80,7 +80,6 @@ config PARISC
 	select GENERIC_CLOCKEVENTS
 	select CPU_NO_EFFICIENT_FFS
 	select THREAD_INFO_IN_TASK
-	select MMU_GATHER_RCU_TABLE_FREE
 	select NEED_DMA_MAP_STATE
 	select NEED_SG_DMA_LENGTH
 	select HAVE_ARCH_KGDB
diff --git a/arch/parisc/include/asm/compat_ucontext.h b/arch/parisc/include/asm/compat_ucontext.h
deleted file mode 100644
index c606f1bc891df7..00000000000000
--- a/arch/parisc/include/asm/compat_ucontext.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_PARISC_COMPAT_UCONTEXT_H
-#define _ASM_PARISC_COMPAT_UCONTEXT_H
-
-#include <linux/compat.h>
-
-/* 32-bit ucontext as seen from an 64-bit kernel */
-struct compat_ucontext {
-	compat_uint_t uc_flags;
-	compat_uptr_t uc_link;
-	compat_stack_t uc_stack;	/* struct compat_sigaltstack (12 bytes)*/	
-	/* FIXME: Pad out to get uc_mcontext to start at an 8-byte aligned boundary */
-	compat_uint_t pad[1];
-	struct compat_sigcontext uc_mcontext;
-	compat_sigset_t uc_sigmask;	/* mask last for extensibility */
-};
-
-#endif /* !_ASM_PARISC_COMPAT_UCONTEXT_H */
diff --git a/arch/parisc/include/asm/pdcpat.h b/arch/parisc/include/asm/pdcpat.h
index ec97eefa4ef6fa..6014fa004b8923 100644
--- a/arch/parisc/include/asm/pdcpat.h
+++ b/arch/parisc/include/asm/pdcpat.h
@@ -54,7 +54,7 @@
 #define PAT_GMMIO           2         /* > 4G MMIO Space */
 #define PAT_NPIOP           3         /* Non Postable I/O Port Space */
 #define PAT_PIOP            4         /* Postable I/O Port Space */
-#define PAT_AHPA            5         /* Addional HPA Space */
+#define PAT_AHPA            5         /* Additional HPA Space */
 #define PAT_UFO             6         /* HPA Space (UFO for Mariposa) */
 #define PAT_GNIP            7         /* GNI Reserved Space */
 
@@ -292,7 +292,7 @@ struct pdc_pat_pd_addr_map_entry {
 * ----------------------------------------------------------
 * Bit  0 to 51 - conf_base_addr
 * Bit 52 to 62 - reserved
-* Bit       63 - endianess bit
+* Bit       63 - endianness bit
 ********************************************************************/
 #define PAT_GET_CBA(value) ((value) & 0xfffffffffffff000UL)
 
diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h
index 467b8547ac8bfa..f6899375cb4393 100644
--- a/arch/parisc/include/asm/pgtable.h
+++ b/arch/parisc/include/asm/pgtable.h
@@ -75,15 +75,6 @@ extern void __update_cache(pte_t pte);
 
 #endif /* !__ASSEMBLER__ */
 
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
-#if CONFIG_PGTABLE_LEVELS == 3
-#define pmd_ERROR(e) \
-	printk("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, (unsigned long)pmd_val(e))
-#endif
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, (unsigned long)pgd_val(e))
-
 /* This is the size of the initially mapped kernel memory */
 #if defined(CONFIG_64BIT) || defined(CONFIG_KALLSYMS)
 #define KERNEL_INITIAL_ORDER	26	/* 1<<26 = 64MB */
diff --git a/arch/parisc/include/asm/special_insns.h b/arch/parisc/include/asm/special_insns.h
index 1013eeba31e5bb..9a54df5eb4f4a1 100644
--- a/arch/parisc/include/asm/special_insns.h
+++ b/arch/parisc/include/asm/special_insns.h
@@ -34,7 +34,7 @@
 
 /**
  * prober_user() - Probe user read access
- * @sr:		Space regster.
+ * @sr:		Space register.
  * @va:		Virtual address.
  *
  * Return: Non-zero if address is accessible.
diff --git a/arch/parisc/include/asm/thread_info.h b/arch/parisc/include/asm/thread_info.h
index b283738bb6dab3..249370706242c5 100644
--- a/arch/parisc/include/asm/thread_info.h
+++ b/arch/parisc/include/asm/thread_info.h
@@ -24,7 +24,7 @@ struct thread_info {
 
 /* thread information allocation */
 
-#ifdef CONFIG_IRQSTACKS
+#if defined(CONFIG_IRQSTACKS) && !defined(CONFIG_64BIT)
 #define THREAD_SIZE_ORDER	2 /* PA-RISC requires at least 16k stack */
 #else
 #define THREAD_SIZE_ORDER	3 /* PA-RISC requires at least 32k stack */
diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h
index b6a709506987e0..aca942f1386ce0 100644
--- a/arch/parisc/include/uapi/asm/mman.h
+++ b/arch/parisc/include/uapi/asm/mman.h
@@ -55,7 +55,7 @@
 #define MADV_HUGEPAGE	14		/* Worth backing with hugepages */
 #define MADV_NOHUGEPAGE 15		/* Not worth backing with hugepages */
 
-#define MADV_DONTDUMP   16		/* Explicity exclude from the core dump,
+#define MADV_DONTDUMP   16		/* Explicitly exclude from the core dump,
 					   overrides the coredump filter bits */
 #define MADV_DODUMP	17		/* Clear the MADV_NODUMP flag */
 
diff --git a/arch/parisc/include/uapi/asm/setup.h b/arch/parisc/include/uapi/asm/setup.h
index 78b2f4ec7d6522..cfa77e84205dc6 100644
--- a/arch/parisc/include/uapi/asm/setup.h
+++ b/arch/parisc/include/uapi/asm/setup.h
@@ -2,6 +2,10 @@
 #ifndef _PARISC_SETUP_H
 #define _PARISC_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	1024
+#endif
 
 #endif /* _PARISC_SETUP_H */
diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c
index bf9f192c826ebe..84e7309a826696 100644
--- a/arch/parisc/kernel/pci-dma.c
+++ b/arch/parisc/kernel/pci-dma.c
@@ -160,8 +160,7 @@ static inline void unmap_uncached_pte(pmd_t * pmd, unsigned long vaddr,
 	if (pmd_none(*pmd))
 		return;
 	if (pmd_bad(*pmd)) {
-		pmd_ERROR(*pmd);
-		pmd_clear(pmd);
+		pmd_clear_bad(pmd);
 		return;
 	}
 	pte = pte_offset_kernel(pmd, vaddr);
@@ -196,8 +195,7 @@ static inline void unmap_uncached_pmd(pgd_t * dir, unsigned long vaddr,
 	if (pgd_none(*dir))
 		return;
 	if (pgd_bad(*dir)) {
-		pgd_ERROR(*dir);
-		pgd_clear(dir);
+		pgd_clear_bad(dir);
 		return;
 	}
 	pmd = pmd_offset(pud_offset(p4d_offset(dir, vaddr), vaddr), vaddr);
diff --git a/arch/parisc/kernel/perf_regs.c b/arch/parisc/kernel/perf_regs.c
index 10a1a5f06a185b..4f21aab5405ccc 100644
--- a/arch/parisc/kernel/perf_regs.c
+++ b/arch/parisc/kernel/perf_regs.c
@@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 
 #define REG_RESERVED (~((1ULL << PERF_REG_PARISC_MAX) - 1))
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask || mask & REG_RESERVED)
 		return -EINVAL;
@@ -53,9 +53,3 @@ u64 perf_reg_abi(struct task_struct *task)
 	return PERF_SAMPLE_REGS_ABI_64;
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c
index d3e17a7a89016a..4d3015a411d6f2 100644
--- a/arch/parisc/kernel/setup.c
+++ b/arch/parisc/kernel/setup.c
@@ -132,6 +132,18 @@ void __init setup_arch(char **cmdline_p)
 	parisc_cache_init();
 	paging_init();
 
+	/*
+	 * Parse early parameters before mm_core_init_early() runs.
+	 * Several early_param() handlers only record data that is consumed
+	 * from there - for example hugepages=, hugepagesz=,
+	 * default_hugepagesz=, hugetlb_cma= and hugetlb_free_vmemmap= - so
+	 * the generic parse_early_param() call in start_kernel() is too late
+	 * for them.  jump_label_init() must come first, since early param
+	 * handlers may enable or disable static keys.
+	 */
+	jump_label_init();
+	parse_early_param();
+
 #ifdef CONFIG_PA11
 	dma_ops_init();
 #endif
diff --git a/arch/parisc/kernel/unwind.c b/arch/parisc/kernel/unwind.c
index 32103a270a8ea1..fab9ae22191af6 100644
--- a/arch/parisc/kernel/unwind.c
+++ b/arch/parisc/kernel/unwind.c
@@ -14,6 +14,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/sort.h>
+#include <linux/bsearch.h>
 #include <linux/sched/task_stack.h>
 
 #include <linux/uaccess.h>
@@ -49,27 +50,23 @@ static DEFINE_SPINLOCK(unwind_lock);
 static struct unwind_table kernel_unwind_table __ro_after_init;
 static LIST_HEAD(unwind_tables);
 
-static inline const struct unwind_table_entry *
-find_unwind_entry_in_table(const struct unwind_table *table, unsigned long addr)
+static int cmp_unwind_entry(const void *key, const void *elt)
 {
-	const struct unwind_table_entry *e = NULL;
-	unsigned long lo, hi, mid;
+	unsigned long addr = (unsigned long)key;
+	const struct unwind_table_entry *e = elt;
 
-	lo = 0; 
-	hi = table->length - 1; 
-	
-	while (lo <= hi) {
-		mid = (hi - lo) / 2 + lo;
-		e = &table->table[mid];
-		if (addr < e->region_start)
-			hi = mid - 1;
-		else if (addr > e->region_end)
-			lo = mid + 1;
-		else
-			return e;
-	}
+	if (addr < e->region_start)
+		return -1;
+	if (addr > e->region_end)
+		return 1;
+	return 0;
+}
 
-	return NULL;
+static inline const struct unwind_table_entry *
+find_unwind_entry_in_table(const struct unwind_table *table, unsigned long addr)
+{
+	return bsearch((void *)addr, table->table, table->length,
+		       sizeof(*table->table), cmp_unwind_entry);
 }
 
 static const struct unwind_table_entry *
diff --git a/arch/parisc/lib/memcpy.c b/arch/parisc/lib/memcpy.c
index 03165c82dfdbd9..93d4e17d7e6890 100644
--- a/arch/parisc/lib/memcpy.c
+++ b/arch/parisc/lib/memcpy.c
@@ -47,7 +47,7 @@ unsigned long raw_copy_from_user(void *dst, const void __user *src,
 			break;
 		}
 		start += PAGE_SIZE;
-		/* align to page boundry which may have different permission */
+		/* align to page boundary which may have different permission */
 		start = PAGE_ALIGN_DOWN(start);
 	}
 	return len - newlen + pa_memcpy(dst, (void __force *)src, newlen);
diff --git a/arch/parisc/net/bpf_jit.h b/arch/parisc/net/bpf_jit.h
index 8b8896959f0449..074115c93c909c 100644
--- a/arch/parisc/net/bpf_jit.h
+++ b/arch/parisc/net/bpf_jit.h
@@ -467,6 +467,8 @@ static inline u32 hppa_t21_insn(u8 opcode, u8 r2, u8 r1, u8 ext8, u8 t)
 
 u64 hppa_div64(u64 div, u64 divisor);
 u64 hppa_div64_rem(u64 div, u64 divisor);
+u64 hppa_sdiv64(u64 div, u64 divisor);
+u64 hppa_sdiv64_rem(u64 div, u64 divisor);
 
 /* Helper functions that emit HPPA instructions when possible. */
 
diff --git a/arch/parisc/net/bpf_jit_comp32.c b/arch/parisc/net/bpf_jit_comp32.c
index 5ff0cf925fe953..eea0a34a3bfee7 100644
--- a/arch/parisc/net/bpf_jit_comp32.c
+++ b/arch/parisc/net/bpf_jit_comp32.c
@@ -321,6 +321,8 @@ static void bpf_put_reg32(const s8 *reg, const s8 *src,
 extern void $$mulI(void);
 extern void $$divU(void);
 extern void $$remU(void);
+extern void $$divI(void);
+extern void $$remI(void);
 
 static void emit_call_millicode(void *func, const s8 arg0,
 		const s8 arg1, u8 opcode, struct hppa_jit_context *ctx)
@@ -562,12 +564,14 @@ static void emit_alu_i32(const s8 *dst, s32 imm,
 }
 
 static void emit_alu_r64(const s8 *dst, const s8 *src,
-			 struct hppa_jit_context *ctx, const u8 op)
+			 struct hppa_jit_context *ctx, const u8 op,
+			 bool is_signed)
 {
 	const s8 *tmp1 = regmap[TMP_REG_1];
 	const s8 *tmp2 = regmap[TMP_REG_2];
 	const s8 *rd;
 	const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
+	void *func;
 
 	if (op == BPF_MOV)
 		rd = bpf_get_reg64_ref(dst, tmp1, false, ctx);
@@ -604,10 +608,12 @@ static void emit_alu_r64(const s8 *dst, const s8 *src,
 		emit_call_libgcc_ll(__muldi3, rd, rs, op, ctx);
 		break;
 	case BPF_DIV:
-		emit_call_libgcc_ll(&hppa_div64, rd, rs, op, ctx);
+		func = is_signed ? &hppa_sdiv64 : &hppa_div64;
+		emit_call_libgcc_ll(func, rd, rs, op, ctx);
 		break;
 	case BPF_MOD:
-		emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, op, ctx);
+		func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem;
+		emit_call_libgcc_ll(func, rd, rs, op, ctx);
 		break;
 	case BPF_LSH:
 		emit_call_libgcc_ll(__ashldi3, rd, rs, op, ctx);
@@ -630,7 +636,8 @@ static void emit_alu_r64(const s8 *dst, const s8 *src,
 }
 
 static void emit_alu_r32(const s8 *dst, const s8 *src,
-			 struct hppa_jit_context *ctx, const u8 op)
+			 struct hppa_jit_context *ctx, const u8 op,
+			 bool is_signed)
 {
 	const s8 *tmp1 = regmap[TMP_REG_1];
 	const s8 *tmp2 = regmap[TMP_REG_2];
@@ -666,10 +673,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src,
 		emit_call_millicode($$mulI, lo(rd), lo(rs), op, ctx);
 		break;
 	case BPF_DIV:
-		emit_call_millicode($$divU, lo(rd), lo(rs), op, ctx);
+		emit_call_millicode(is_signed ? $$divI : $$divU,
+				    lo(rd), lo(rs), op, ctx);
 		break;
 	case BPF_MOD:
-		emit_call_millicode($$remU, lo(rd), lo(rs), op, ctx);
+		emit_call_millicode(is_signed ? $$remI : $$remU,
+				    lo(rd), lo(rs), op, ctx);
 		break;
 	case BPF_LSH:
 		emit(hppa_subi(0x1f, lo(rs), HPPA_REG_T0), ctx);
@@ -1163,12 +1172,12 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx,
 			emit_imm32(tmp2, imm, ctx);
 			src = tmp2;
 		}
-		emit_alu_r64(dst, src, ctx, BPF_OP(code));
+		emit_alu_r64(dst, src, ctx, BPF_OP(code), off == 1);
 		break;
 
 	/* dst = -dst */
 	case BPF_ALU64 | BPF_NEG:
-		emit_alu_r64(dst, tmp2, ctx, BPF_OP(code));
+		emit_alu_r64(dst, tmp2, ctx, BPF_OP(code), false);
 		break;
 
 	case BPF_ALU64 | BPF_MOV | BPF_K:
@@ -1211,7 +1220,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx,
 			emit_imm32(tmp2, imm, ctx);
 			src = tmp2;
 		}
-		emit_alu_r32(dst, src, ctx, BPF_OP(code));
+		emit_alu_r32(dst, src, ctx, BPF_OP(code), off == 1);
 		break;
 
 	/* dst = dst OP imm */
@@ -1236,7 +1245,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx,
 		 * src is ignored---choose tmp2 as a dummy register since it
 		 * is not on the stack.
 		 */
-		emit_alu_r32(dst, tmp2, ctx, BPF_OP(code));
+		emit_alu_r32(dst, tmp2, ctx, BPF_OP(code), false);
 		break;
 
 	/* dst = BSWAP##imm(dst) */
diff --git a/arch/parisc/net/bpf_jit_comp64.c b/arch/parisc/net/bpf_jit_comp64.c
index 54b0d5e25e0246..c326fa737ec8c5 100644
--- a/arch/parisc/net/bpf_jit_comp64.c
+++ b/arch/parisc/net/bpf_jit_comp64.c
@@ -502,14 +502,19 @@ static void emit_call(u64 addr, bool fixed, struct hppa_jit_context *ctx)
 	emit_hppa_copy(HPPA_REG_RET0, regmap[BPF_REG_0], ctx);
 }
 
-static void emit_call_libgcc_ll(void *func, const s8 arg0,
-		const s8 arg1, u8 opcode, struct hppa_jit_context *ctx)
+static void emit_call_libgcc_ll(void *func, const s8 arg0, const s8 arg1,
+		u8 opcode, bool is_signed, struct hppa_jit_context *ctx)
 {
 	u64 func_addr;
 
 	if (BPF_CLASS(opcode) == BPF_ALU) {
-		emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx);
-		emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx);
+		if (is_signed) {
+			emit_hppa64_sext32(arg0, HPPA_REG_ARG0, ctx);
+			emit_hppa64_sext32(arg1, HPPA_REG_ARG1, ctx);
+		} else {
+			emit_hppa64_zext32(arg0, HPPA_REG_ARG0, ctx);
+			emit_hppa64_zext32(arg1, HPPA_REG_ARG1, ctx);
+		}
 	} else {
 		emit_hppa_copy(arg0, HPPA_REG_ARG0, ctx);
 		emit_hppa_copy(arg1, HPPA_REG_ARG1, ctx);
@@ -600,6 +605,8 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx,
 	u8 rd = -1, rs = -1, code = insn->code;
 	s16 off = insn->off;
 	s32 imm = insn->imm;
+	bool is_signed;
+	void *func;
 
 	init_regs(&rd, &rs, insn, ctx);
 
@@ -656,29 +663,39 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct hppa_jit_context *ctx,
 		fallthrough;
 	case BPF_ALU | BPF_MUL | BPF_X:
 	case BPF_ALU64 | BPF_MUL | BPF_X:
-		emit_call_libgcc_ll(__muldi3, rd, rs, code, ctx);
+		emit_call_libgcc_ll(__muldi3, rd, rs, code, false, ctx);
 		if (!is64 && !aux->verifier_zext)
 			emit_zext_32(rd, ctx);
 		break;
 	case BPF_ALU | BPF_DIV | BPF_K:
 	case BPF_ALU64 | BPF_DIV | BPF_K:
-		emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx);
+		is_signed = (off == 1);
+		emit_imm(HPPA_REG_T1,
+			 is64 || is_signed ? (s64)(s32)imm : (u32)imm,
+			 HPPA_REG_T2, ctx);
 		rs = HPPA_REG_T1;
 		fallthrough;
 	case BPF_ALU | BPF_DIV | BPF_X:
 	case BPF_ALU64 | BPF_DIV | BPF_X:
-		emit_call_libgcc_ll(&hppa_div64, rd, rs, code, ctx);
+		is_signed = (off == 1);
+		func = is_signed ? &hppa_sdiv64 : &hppa_div64;
+		emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx);
 		if (!is64 && !aux->verifier_zext)
 			emit_zext_32(rd, ctx);
 		break;
 	case BPF_ALU | BPF_MOD | BPF_K:
 	case BPF_ALU64 | BPF_MOD | BPF_K:
-		emit_imm(HPPA_REG_T1, is64 ? (s64)(s32)imm : (u32)imm, HPPA_REG_T2, ctx);
+		is_signed = (off == 1);
+		emit_imm(HPPA_REG_T1,
+			 is64 || is_signed ? (s64)(s32)imm : (u32)imm,
+			 HPPA_REG_T2, ctx);
 		rs = HPPA_REG_T1;
 		fallthrough;
 	case BPF_ALU | BPF_MOD | BPF_X:
 	case BPF_ALU64 | BPF_MOD | BPF_X:
-		emit_call_libgcc_ll(&hppa_div64_rem, rd, rs, code, ctx);
+		is_signed = (off == 1);
+		func = is_signed ? &hppa_sdiv64_rem : &hppa_div64_rem;
+		emit_call_libgcc_ll(func, rd, rs, code, is_signed, ctx);
 		if (!is64 && !aux->verifier_zext)
 			emit_zext_32(rd, ctx);
 		break;
diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c
index 172770132440db..a2f93202a2ef9c 100644
--- a/arch/parisc/net/bpf_jit_core.c
+++ b/arch/parisc/net/bpf_jit_core.c
@@ -190,3 +190,17 @@ u64 hppa_div64_rem(u64 div, u64 divisor)
 	div64_u64_rem(div, divisor, &rem);
 	return rem;
 }
+
+u64 hppa_sdiv64(u64 div, u64 divisor)
+{
+	s64 sdiv = div64_s64((s64)div, (s64)divisor);
+
+	return (u64)sdiv;
+}
+
+u64 hppa_sdiv64_rem(u64 div, u64 divisor)
+{
+	s64 sdiv = div64_s64((s64)div, (s64)divisor);
+
+	return (u64)((s64)div - sdiv * (s64)divisor);
+}
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 2580e27e432874..c3b6cd65560697 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -130,7 +130,7 @@ config PPC
 	select ARCH_HAS_CURRENT_STACK_POINTER
 	select ARCH_HAS_DEBUG_VIRTUAL
 	select ARCH_HAS_DEBUG_VM_PGTABLE
-	select ARCH_HAS_DEBUG_WX		if STRICT_KERNEL_RWX
+	select ARCH_HAS_CHECK_WX		if STRICT_KERNEL_RWX
 	select ARCH_HAS_DEVMEM_IS_ALLOWED
 	select ARCH_HAS_DMA_MAP_DIRECT 		if PPC_PSERIES
 	select ARCH_HAS_DMA_OPS			if PPC64
@@ -160,7 +160,6 @@ config PPC
 	select ARCH_HAS_UBSAN
 	select ARCH_HAS_VDSO_ARCH_DATA
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
-	select ARCH_HAVE_EXTRA_ELF_NOTES        if SPU_BASE
 	select ARCH_KEEP_MEMBLOCK
 	select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE	if PPC_RADIX_MMU
 	select ARCH_MIGHT_HAVE_PC_PARPORT
@@ -279,7 +278,6 @@ config PPC
 	select HAVE_PERF_EVENTS_NMI		if PPC64
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
-	select HAVE_PREEMPT_DYNAMIC_KEY
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_RETHOOK			if KPROBES
 	select HAVE_REGS_AND_STACK_ACCESS_API
@@ -307,7 +305,6 @@ config PPC
 	select KASAN_VMALLOC			if KASAN && EXECMEM
 	select LOCK_MM_AND_FIND_VMA
 	select MMU_GATHER_PAGE_SIZE
-	select MMU_GATHER_RCU_TABLE_FREE
 	select HAVE_ARCH_TLB_REMOVE_TABLE
 	select MMU_GATHER_MERGE_VMAS
 	select MMU_LAZY_TLB_SHOOTDOWN		if PPC_BOOK3S_64
diff --git a/arch/powerpc/boot/dts/ac14xx.dts b/arch/powerpc/boot/dts/ac14xx.dts
index 5d8877e1f4ad9a..05fcf08df1ca6f 100644
--- a/arch/powerpc/boot/dts/ac14xx.dts
+++ b/arch/powerpc/boot/dts/ac14xx.dts
@@ -148,19 +148,6 @@
 			compatible = "fsl,mpc5121rev2-clock", "fsl,mpc5121-clock";
 		};
 
-		/*
-		 * GPIO PIC:
-		 * interrupts cell = <pin nr, sense>
-		 * sense == 8: Level, low assertion
-		 * sense == 2: Edge, high-to-low change
-		 */
-		gpio_pic: gpio@1100 {
-			gpio-controller;
-			#gpio-cells = <2>;
-			interrupt-controller;
-			#interrupt-cells = <2>;
-		};
-
 		sdhc@1500 {
 			cd-gpios = <&gpio_pic 23 0>;	/* card detect */
 			wp-gpios = <&gpio_pic 24 0>;	/* write protect */
diff --git a/arch/powerpc/boot/dts/mpc5121.dtsi b/arch/powerpc/boot/dts/mpc5121.dtsi
index a278fb7b9e7118..9c5dacb94b9249 100644
--- a/arch/powerpc/boot/dts/mpc5121.dtsi
+++ b/arch/powerpc/boot/dts/mpc5121.dtsi
@@ -146,10 +146,14 @@
 			interrupts = <83 0x8>;
 		};
 
-		gpio@1100 {
+		gpio_pic: gpio@1100 {
 			compatible = "fsl,mpc5121-gpio";
 			reg = <0x1100 0x100>;
 			interrupts = <78 0x8>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
 		};
 
 		can@1300 {
diff --git a/arch/powerpc/boot/dts/pdm360ng.dts b/arch/powerpc/boot/dts/pdm360ng.dts
index 67c3b9db75d76a..05e93ca294ee32 100644
--- a/arch/powerpc/boot/dts/pdm360ng.dts
+++ b/arch/powerpc/boot/dts/pdm360ng.dts
@@ -9,6 +9,7 @@
  * Copyright 2008 Freescale Semiconductor Inc.
  */
 
+#include <dt-bindings/gpio/gpio.h>
 #include "mpc5121.dtsi"
 
 / {
@@ -176,11 +177,12 @@
 
 			/* ADS7845 touch screen controller */
 			ts@0 {
-				compatible = "ti,ads7846";
+				compatible = "ti,ads7845";
 				reg = <0x0>;
 				spi-max-frequency = <3000000>;
-				/* pen irq is GPIO25 */
-				interrupts = <78 0x8>;
+				interrupt-parent = <&gpio_pic>;
+				interrupts = <25 0x8>;
+				pendown-gpio = <&gpio_pic 25 GPIO_ACTIVE_LOW>;
 			};
 		};
 
diff --git a/arch/powerpc/configs/fsl-emb-nonhw.config b/arch/powerpc/configs/fsl-emb-nonhw.config
index 391c99117ee06b..688051b11ce146 100644
--- a/arch/powerpc/configs/fsl-emb-nonhw.config
+++ b/arch/powerpc/configs/fsl-emb-nonhw.config
@@ -2,7 +2,6 @@ CONFIG_ADFS_FS=m
 CONFIG_AFFS_FS=m
 CONFIG_AUDIT=y
 CONFIG_BEFS_FS=m
-CONFIG_BFS_FS=m
 CONFIG_BINFMT_MISC=m
 # CONFIG_BLK_DEV_BSG is not set
 CONFIG_BLK_DEV_INITRD=y
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index 1eb8e3457e8bff..5c33f0bba0e361 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -393,7 +393,7 @@ CONFIG_MAGIC_SYSRQ=y
 CONFIG_PAGE_OWNER=y
 CONFIG_PAGE_POISONING=y
 CONFIG_DEBUG_RODATA_TEST=y
-CONFIG_DEBUG_WX=y
+CONFIG_CHECK_WX=y
 CONFIG_DEBUG_STACK_USAGE=y
 CONFIG_DEBUG_VM=y
 # CONFIG_DEBUG_VM_PGTABLE is not set
diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
index acffc5c17f92d8..8aab229fb90977 100644
--- a/arch/powerpc/configs/ppc6xx_defconfig
+++ b/arch/powerpc/configs/ppc6xx_defconfig
@@ -945,7 +945,6 @@ CONFIG_ECRYPT_FS=m
 CONFIG_HFS_FS=m
 CONFIG_HFSPLUS_FS=m
 CONFIG_BEFS_FS=m
-CONFIG_BFS_FS=m
 CONFIG_EFS_FS=m
 CONFIG_CRAMFS=m
 CONFIG_VXFS_FS=m
diff --git a/arch/powerpc/crypto/vmx.c b/arch/powerpc/crypto/vmx.c
index 08da5311dfdf20..8abc4db8ee4ebd 100644
--- a/arch/powerpc/crypto/vmx.c
+++ b/arch/powerpc/crypto/vmx.c
@@ -8,12 +8,7 @@
  */
 
 #include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/types.h>
-#include <linux/err.h>
 #include <linux/cpufeature.h>
-#include <linux/crypto.h>
-#include <asm/cputable.h>
 #include <crypto/internal/skcipher.h>
 
 #include "aesp8-ppc.h"
@@ -24,7 +19,7 @@ static int __init p8_init(void)
 
 	ret = crypto_register_skcipher(&p8_aes_cbc_alg);
 	if (ret)
-		goto err;
+		return ret;
 
 	ret = crypto_register_skcipher(&p8_aes_ctr_alg);
 	if (ret)
@@ -40,7 +35,6 @@ static int __init p8_init(void)
 	crypto_unregister_skcipher(&p8_aes_ctr_alg);
 err_unregister_aes_cbc:
 	crypto_unregister_skcipher(&p8_aes_cbc_alg);
-err:
 	return ret;
 }
 
@@ -54,8 +48,6 @@ static void __exit p8_exit(void)
 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, p8_init);
 module_exit(p8_exit);
 
-MODULE_AUTHOR("Marcelo Cerri<mhcerri@br.ibm.com>");
-MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions "
-		   "support on Power 8");
+MODULE_AUTHOR("Marcelo Cerri <mhcerri@br.ibm.com>");
+MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions support on Power 8");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("1.0.0");
diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h
index e18a4fa282a1b6..835e84caee13fa 100644
--- a/arch/powerpc/include/asm/book3s/32/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/32/pgtable.h
@@ -203,8 +203,6 @@ void unmap_kernel_page(unsigned long va);
 /* Bits to mask out from a PGD to get to the PUD page */
 #define PGD_MASKED_BITS		0
 
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
 /*
  * Bits in a linux-style PTE.  These match the bits in the
  * (hardware-defined) PowerPC PTE as closely as possible.
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index f4db7d7fbd5c62..dff8790a047db5 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -991,13 +991,6 @@ static inline pmd_t *pud_pgtable(pud_t pud)
 	return (pmd_t *)__va(pud_val(pud) & ~PUD_MASKED_BITS);
 }
 
-#define pmd_ERROR(e) \
-	pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
-#define pud_ERROR(e) \
-	pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
-
 static inline int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot)
 {
 	if (radix_enabled()) {
diff --git a/arch/powerpc/include/asm/elf.h b/arch/powerpc/include/asm/elf.h
index bb4b94444d3e8a..5dc8c4923eb1df 100644
--- a/arch/powerpc/include/asm/elf.h
+++ b/arch/powerpc/include/asm/elf.h
@@ -123,12 +123,6 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm,
 	(0x7ff >> (PAGE_SHIFT - 12)) : \
 	(0x3ffff >> (PAGE_SHIFT - 12)))
 
-#ifdef CONFIG_SPU_BASE
-/* Notes used in ET_CORE. Note name is "SPU/<fd>/<filename>". */
-#define NT_SPU		1
-
-#endif /* CONFIG_SPU_BASE */
-
 #ifdef CONFIG_PPC64
 
 #define get_cache_geometry(level) \
diff --git a/arch/powerpc/include/asm/hardirq.h b/arch/powerpc/include/asm/hardirq.h
index f133b5930ae15e..be6cd5aab01626 100644
--- a/arch/powerpc/include/asm/hardirq.h
+++ b/arch/powerpc/include/asm/hardirq.h
@@ -5,24 +5,33 @@
 #include <linux/threads.h>
 #include <linux/irq.h>
 
-typedef struct {
-	unsigned int __softirq_pending;
-	unsigned int timer_irqs_event;
-	unsigned int broadcast_irqs_event;
-	unsigned int timer_irqs_others;
-	unsigned int pmu_irqs;
-	unsigned int mce_exceptions;
-	unsigned int spurious_irqs;
-	unsigned int sreset_irqs;
+enum irq_stat_counts {
+	IRQ_COUNT_LOC_TIMER,
+	IRQ_COUNT_BCT_TIMER,
+	IRQ_COUNT_OTHER_TIMER,
+	IRQ_COUNT_SPURIOUS,
+	IRQ_COUNT_PMI,
+	IRQ_COUNT_MCE,
+	IRQ_COUNT_NMI_SRESET,
 #ifdef CONFIG_PPC_WATCHDOG
-	unsigned int soft_nmi_irqs;
+	IRQ_COUNT_WATCHDOG,
 #endif
 #ifdef CONFIG_PPC_DOORBELL
-	unsigned int doorbell_irqs;
+	IRQ_COUNT_DOORBELL,
 #endif
+	IRQ_COUNT_MAX,
+};
+
+typedef struct {
+	unsigned int counts[IRQ_COUNT_MAX];
 } ____cacheline_aligned irq_cpustat_t;
 
 DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
+DECLARE_PER_CPU(unsigned int, __softirq_pending);
+#define local_softirq_pending_ref       __softirq_pending
+
+#define inc_irq_stat(index)	__this_cpu_inc(irq_stat.counts[IRQ_COUNT_##index])
+void inc_irq_stat_and_enable(enum irq_stat_counts which);
 
 #define __ARCH_IRQ_STAT
 #define __ARCH_IRQ_EXIT_IRQS_DISABLED
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index 496ecc65ac255a..f17afde89fa13d 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -51,8 +51,6 @@
 
 #define USER_PTRS_PER_PGD	(TASK_SIZE / PGDIR_SIZE)
 
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08llx.\n", __FILE__, __LINE__, (unsigned long long)pgd_val(e))
 
 /*
  * This is the bottom of the PKMAP area with HIGHMEM or an arbitrary
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h
index fb6fa1d4e0749a..75cf3c331b9292 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h
@@ -82,9 +82,6 @@ extern struct page *p4d_page(p4d_t p4d);
 
 #endif /* !__ASSEMBLER__ */
 
-#define pud_ERROR(e) \
-	pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
-
 /*
  * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() */
 #define remap_4k_pfn(vma, addr, pfn, prot)	\
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h
index 661eb3820d1291..446dde8b6ead54 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
@@ -159,11 +159,6 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
 	__young;							\
 })
 
-#define pmd_ERROR(e) \
-	pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
-
 /*
  * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
  * are !pte_none() && !pte_present().
diff --git a/arch/powerpc/include/asm/spu.h b/arch/powerpc/include/asm/spu.h
index 96ad4510c89542..7152285b6268fd 100644
--- a/arch/powerpc/include/asm/spu.h
+++ b/arch/powerpc/include/asm/spu.h
@@ -210,15 +210,12 @@ extern long spu_sys_callback(struct spu_syscall_block *s);
 
 /* syscalls implemented in spufs */
 struct file;
-struct coredump_params;
 struct spufs_calls {
 	long (*create_thread)(const char __user *name,
 					unsigned int flags, umode_t mode,
 					struct file *neighbor);
 	long (*spu_run)(struct file *filp, __u32 __user *unpc,
 						__u32 __user *ustatus);
-	int (*coredump_extra_notes_size)(void);
-	int (*coredump_extra_notes_write)(struct coredump_params *cprm);
 	void (*notify_spus_active)(void);
 	struct module *owner;
 };
diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h
index 1ca7d4c4b90dbf..7ee22249148bec 100644
--- a/arch/powerpc/include/asm/tlb.h
+++ b/arch/powerpc/include/asm/tlb.h
@@ -17,8 +17,6 @@
 #include <asm/mmu.h>
 #endif
 
-#include <linux/pagemap.h>
-
 static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep,
 					  unsigned long address);
 #define __tlb_remove_tlb_entry	__tlb_remove_tlb_entry
diff --git a/arch/powerpc/include/uapi/asm/setup.h b/arch/powerpc/include/uapi/asm/setup.h
index c54940b09d065c..daa15ac4e94c6e 100644
--- a/arch/powerpc/include/uapi/asm/setup.h
+++ b/arch/powerpc/include/uapi/asm/setup.h
@@ -2,6 +2,10 @@
 #ifndef _UAPI_ASM_POWERPC_SETUP_H
 #define _UAPI_ASM_POWERPC_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	2048
+#endif
 
 #endif /* _UAPI_ASM_POWERPC_SETUP_H */
diff --git a/arch/powerpc/kernel/dbell.c b/arch/powerpc/kernel/dbell.c
index 5712dd846263c5..f5e298a4c4c03b 100644
--- a/arch/powerpc/kernel/dbell.c
+++ b/arch/powerpc/kernel/dbell.c
@@ -31,7 +31,7 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception)
 		do_hard_irq_enable();
 
 	kvmppc_clear_host_ipi(smp_processor_id());
-	__this_cpu_inc(irq_stat.doorbell_irqs);
+	inc_irq_stat(DOORBELL);
 
 	smp_ipi_demux_relaxed(); /* already performed the barrier */
 
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 15a3c3fd8e7051..bce546931a13bb 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -73,6 +73,7 @@
 
 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 EXPORT_PER_CPU_SYMBOL(irq_stat);
+DEFINE_PER_CPU(unsigned int, __softirq_pending);
 
 #ifdef CONFIG_PPC32
 atomic_t ppc_n_lost_interrupts;
@@ -83,105 +84,105 @@ u32 tau_interrupts(unsigned long cpu);
 #endif
 #endif /* CONFIG_PPC32 */
 
+struct irq_stat_info {
+	const char	*symbol;
+	const char	*text;
+	int		optional;
+};
+
+/* ISE - IRQ STAT ENABLED, ISC - IRQ STAT CONDITIONAL */
+#define ISE(idx, sym, txt)[IRQ_COUNT_##idx] = { .symbol = sym, .text = txt, .optional = 0}
+#define ISC(idx, sym, txt)[IRQ_COUNT_##idx] = { .symbol = sym, .text = txt, .optional = 1}
+
+
+static struct irq_stat_info irq_stat_info[IRQ_COUNT_MAX] __ro_after_init = {
+	ISE(LOC_TIMER,		"LOC", "  Local timer interrupts for timer event device\n"),
+	ISE(BCT_TIMER,		"BCT", "  Broadcast timer interrupts for timer event device\n"),
+	ISE(OTHER_TIMER,	"LOC", "  Local timer interrupts for others\n"),
+	ISE(SPURIOUS,		"SPU", "  Spurious interrupts\n"),
+	ISE(PMI,		"PMI", "  Performance monitoring interrupts\n"),
+	ISC(MCE,		"MCE", "  Machine check exceptions\n"),
+	ISC(NMI_SRESET,		"NMI", "  System Reset interrupts\n"),
+#ifdef CONFIG_PPC_WATCHDOG
+	ISE(WATCHDOG,		"WDG", "  Watchdog soft-NMI interrupts\n"),
+#endif
+#ifdef CONFIG_PPC_DOORBELL
+	ISE(DOORBELL,		"DBL", "  Doorbell interrupts\n"),
+#endif
+};
+
+/*
+ * Used for default disabled counters to increment the stats and to enable the
+ * entry for /proc/interrupts output.
+ */
+static DECLARE_BITMAP(irq_stat_count_show, IRQ_COUNT_MAX) __read_mostly;
+void inc_irq_stat_and_enable(enum irq_stat_counts which)
+{
+	__this_cpu_inc(irq_stat.counts[which]);
+	set_bit(which, irq_stat_count_show);
+}
+
 int arch_show_interrupts(struct seq_file *p, int prec)
 {
-	int j;
+	const struct irq_stat_info *info = irq_stat_info;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++, info++) {
+		if (info->optional && !test_bit(i, irq_stat_count_show))
+			continue;
+
+		seq_printf(p, "%*s:", prec, info->symbol);
+		irq_proc_emit_counts(p, &irq_stat.counts[i]);
+		seq_puts(p, info->text);
+	}
 
 #if defined(CONFIG_PPC32) && defined(CONFIG_TAU_INT)
 	if (tau_initialized) {
+		int j;
 		seq_printf(p, "%*s:", prec, "TAU");
 		for_each_online_cpu(j)
 			seq_put_decimal_ull_width(p, " ", tau_interrupts(j), 10);
 		seq_puts(p, "  PowerPC             Thermal Assist (cpu temp)\n");
 	}
 #endif /* CONFIG_PPC32 && CONFIG_TAU_INT */
-
-	seq_printf(p, "%*s:", prec, "LOC");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).timer_irqs_event, 10);
-        seq_printf(p, "  Local timer interrupts for timer event device\n");
-
-	seq_printf(p, "%*s:", prec, "BCT");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).broadcast_irqs_event, 10);
-	seq_printf(p, "  Broadcast timer interrupts for timer event device\n");
-
-	seq_printf(p, "%*s:", prec, "LOC");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).timer_irqs_others, 10);
-        seq_printf(p, "  Local timer interrupts for others\n");
-
-	seq_printf(p, "%*s:", prec, "SPU");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).spurious_irqs, 10);
-	seq_printf(p, "  Spurious interrupts\n");
-
-	seq_printf(p, "%*s:", prec, "PMI");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).pmu_irqs, 10);
-	seq_printf(p, "  Performance monitoring interrupts\n");
-
-	seq_printf(p, "%*s:", prec, "MCE");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).mce_exceptions, 10);
-	seq_printf(p, "  Machine check exceptions\n");
-
 #ifdef CONFIG_PPC_BOOK3S_64
 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
+		int j;
 		seq_printf(p, "%*s:", prec, "HMI");
 		for_each_online_cpu(j)
 			seq_put_decimal_ull_width(p, " ", paca_ptrs[j]->hmi_irqs, 10);
 		seq_printf(p, "  Hypervisor Maintenance Interrupts\n");
 	}
 #endif
+	return 0;
+}
 
-	seq_printf(p, "%*s:", prec, "NMI");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).sreset_irqs, 10);
-	seq_printf(p, "  System Reset interrupts\n");
-
-#ifdef CONFIG_PPC_WATCHDOG
-	seq_printf(p, "%*s:", prec, "WDG");
-	for_each_online_cpu(j)
-		seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).soft_nmi_irqs, 10);
-	seq_printf(p, "  Watchdog soft-NMI interrupts\n");
-#endif
+static int __init irq_init_stats(void)
+{
+	struct irq_stat_info *info = irq_stat_info;
 
-#ifdef CONFIG_PPC_DOORBELL
-	if (cpu_has_feature(CPU_FTR_DBELL)) {
-		seq_printf(p, "%*s:", prec, "DBL");
-		for_each_online_cpu(j)
-			seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, j).doorbell_irqs, 10);
-		seq_printf(p, "  Doorbell interrupts\n");
+	for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++, info++) {
+		if (info->optional == 0)
+			set_bit(i, irq_stat_count_show);
 	}
-#endif
 
 	return 0;
 }
+late_initcall(irq_init_stats);
 
 /*
  * /proc/stat helpers
  */
 u64 arch_irq_stat_cpu(unsigned int cpu)
 {
-	u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event;
+	irq_cpustat_t *p = per_cpu_ptr(&irq_stat, cpu);
+	u64 sum = 0;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(irq_stat_info); i++)
+		sum += p->counts[i];
 
-	sum += per_cpu(irq_stat, cpu).broadcast_irqs_event;
-	sum += per_cpu(irq_stat, cpu).pmu_irqs;
-	sum += per_cpu(irq_stat, cpu).mce_exceptions;
-	sum += per_cpu(irq_stat, cpu).spurious_irqs;
-	sum += per_cpu(irq_stat, cpu).timer_irqs_others;
 #ifdef CONFIG_PPC_BOOK3S_64
 	sum += paca_ptrs[cpu]->hmi_irqs;
 #endif
-	sum += per_cpu(irq_stat, cpu).sreset_irqs;
-#ifdef CONFIG_PPC_WATCHDOG
-	sum += per_cpu(irq_stat, cpu).soft_nmi_irqs;
-#endif
-#ifdef CONFIG_PPC_DOORBELL
-	sum += per_cpu(irq_stat, cpu).doorbell_irqs;
-#endif
-
 	return sum;
 }
 
@@ -251,7 +252,7 @@ static void __do_irq(struct pt_regs *regs, unsigned long oldsp)
 
 	/* And finally process it */
 	if (unlikely(!irq))
-		__this_cpu_inc(irq_stat.spurious_irqs);
+		inc_irq_stat(SPURIOUS);
 	else
 		generic_handle_irq(irq);
 
diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c
index 329c1690b5ed35..2100c5e2c82151 100644
--- a/arch/powerpc/kernel/sysfs.c
+++ b/arch/powerpc/kernel/sysfs.c
@@ -338,7 +338,7 @@ static ssize_t show_pw20_wait_time(struct device *dev,
 		time = pw20_wt;
 	}
 
-	return sysfs_emit(buf, "%llu\n", time > 0 ? time : 0);
+	return sysfs_emit(buf, "%llu\n", time);
 }
 
 static void set_pw20_wait_entry_bit(void *val)
@@ -460,7 +460,7 @@ static ssize_t show_altivec_idle_wait_time(struct device *dev,
 		time = altivec_idle_wt;
 	}
 
-	return sysfs_emit(buf, "%llu\n", time > 0 ? time : 0);
+	return sysfs_emit(buf, "%llu\n", time);
 }
 
 static void set_altivec_idle_wait_entry_bit(void *val)
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 11145c40183dd1..e00dcda8240b5e 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -615,13 +615,13 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt)
 	now = get_tb();
 	if (now >= *next_tb) {
 		evt->event_handler(evt);
-		__this_cpu_inc(irq_stat.timer_irqs_event);
+		inc_irq_stat(LOC_TIMER);
 	} else {
 		now = *next_tb - now;
 		if (now > decrementer_max)
 			now = decrementer_max;
 		set_dec_or_work(now);
-		__this_cpu_inc(irq_stat.timer_irqs_others);
+		inc_irq_stat(OTHER_TIMER);
 	}
 
 	trace_timer_interrupt_exit(regs);
@@ -634,7 +634,7 @@ EXPORT_SYMBOL(timer_interrupt);
 void timer_broadcast_interrupt(void)
 {
 	tick_receive_broadcast();
-	__this_cpu_inc(irq_stat.broadcast_irqs_event);
+	inc_irq_stat(BCT_TIMER);
 }
 #endif
 
diff --git a/arch/powerpc/kernel/trace/ftrace_entry.S b/arch/powerpc/kernel/trace/ftrace_entry.S
index 6599fe3c62347f..5eb8eee32549d5 100644
--- a/arch/powerpc/kernel/trace/ftrace_entry.S
+++ b/arch/powerpc/kernel/trace/ftrace_entry.S
@@ -220,7 +220,13 @@
 
 	/* Restore gprs */
 	.if \allregs == 1
+#ifdef CONFIG_PPC64
+	REST_GPRS(2, 12, r1)
+	/* Do not restore a stale PACA pointer if the task migrated */
+	REST_GPRS(14, 31, r1)
+#else
 	REST_GPRS(2, 31, r1)
+#endif
 	.else
 	REST_GPRS(3, 10, r1)
 #if defined(CONFIG_LIVEPATCH_64) || defined(CONFIG_PPC_FTRACE_OUT_OF_LINE)
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 629f2a2d4780e3..3a8f5416e3f6e1 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -459,8 +459,7 @@ DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception)
 	}
 
 	hv_nmi_check_nonrecoverable(regs);
-
-	__this_cpu_inc(irq_stat.sreset_irqs);
+	inc_irq_stat_and_enable(IRQ_COUNT_NMI_SRESET);
 
 	/* See if any machine dependent calls */
 	if (ppc_md.system_reset_exception) {
@@ -817,7 +816,7 @@ static void __machine_check_exception(struct pt_regs *regs)
 {
 	int recover = 0;
 
-	__this_cpu_inc(irq_stat.mce_exceptions);
+	inc_irq_stat_and_enable(IRQ_COUNT_MCE);
 
 	add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
 
@@ -1932,8 +1931,7 @@ DEFINE_INTERRUPT_HANDLER(vsx_unavailable_tm)
 DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi);
 DEFINE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi)
 {
-	__this_cpu_inc(irq_stat.pmu_irqs);
-
+	inc_irq_stat(PMI);
 	perf_irq(regs);
 
 	return 0;
@@ -1943,8 +1941,7 @@ DEFINE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi)
 DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async);
 DEFINE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async)
 {
-	__this_cpu_inc(irq_stat.pmu_irqs);
-
+	inc_irq_stat(PMI);
 	perf_irq(regs);
 }
 
diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c
index c40c693684764c..582ee3fd71c1ba 100644
--- a/arch/powerpc/kernel/watchdog.c
+++ b/arch/powerpc/kernel/watchdog.c
@@ -381,7 +381,7 @@ DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt)
 	if (!cpumask_test_cpu(cpu, &wd_cpus_enabled))
 		return 0;
 
-	__this_cpu_inc(irq_stat.soft_nmi_irqs);
+	inc_irq_stat(WATCHDOG);
 
 	tb = get_tb();
 	if (tb - per_cpu(wd_timer_tb, cpu) >= wd_panic_timeout_tb) {
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index dbac3573b2c899..de05d721edc4ef 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -3058,7 +3058,6 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu)
 
 	init_waitqueue_head(&vcpu->arch.cpu_run);
 
-	mutex_lock(&kvm->lock);
 	vcore = NULL;
 	err = -EINVAL;
 	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
@@ -3091,7 +3090,6 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu)
 			mutex_unlock(&kvm->arch.mmu_setup_lock);
 		}
 	}
-	mutex_unlock(&kvm->lock);
 
 	if (!vcore)
 		return err;
diff --git a/arch/powerpc/kvm/guest-state-buffer.c b/arch/powerpc/kvm/guest-state-buffer.c
index 42843eca672793..aeb586ba452310 100644
--- a/arch/powerpc/kvm/guest-state-buffer.c
+++ b/arch/powerpc/kvm/guest-state-buffer.c
@@ -598,7 +598,7 @@ EXPORT_SYMBOL_GPL(kvmppc_gsm_fill_info);
 int kvmppc_gsm_refresh_info(struct kvmppc_gs_msg *gsm,
 			    struct kvmppc_gs_buff *gsb)
 {
-	if (!gsm->ops->fill_info)
+	if (!gsm->ops->refresh_info)
 		return -EINVAL;
 
 	return gsm->ops->refresh_info(gsm, gsb);
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index f0d6aa657c1acc..d49b2ef5d763aa 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -47,7 +47,7 @@ extern void conv_sp_to_dp(const float *sp, double *dp);
 extern void conv_dp_to_sp(const double *dp, float *sp);
 #endif
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 /*
  * Functions in quad.S
  */
@@ -134,7 +134,7 @@ static nokprobe_inline unsigned long dform_ea(unsigned int instr,
 	return ea;
 }
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 /*
  * Calculate effective address for a DS-form instruction
  */
@@ -168,7 +168,7 @@ static nokprobe_inline unsigned long dqform_ea(unsigned int instr,
 
 	return ea;
 }
-#endif /* __powerpc64 */
+#endif /* CONFIG_PPC64 */
 
 /*
  * Calculate effective address for an X-form instruction
@@ -250,7 +250,7 @@ static nokprobe_inline unsigned long byterev_4(unsigned long x)
 		((x & 0xff00) << 8) | ((x & 0xff) << 24);
 }
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 static nokprobe_inline unsigned long byterev_8(unsigned long x)
 {
 	return (byterev_4(x) << 32) | byterev_4(x >> 32);
@@ -266,7 +266,7 @@ static nokprobe_inline void do_byte_reverse(void *ptr, int nb)
 	case 4:
 		*(u32 *)ptr = byterev_4(*(u32 *)ptr);
 		break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 8:
 		*(unsigned long *)ptr = byterev_8(*(unsigned long *)ptr);
 		break;
@@ -312,7 +312,7 @@ __read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs
 	case 4:
 		unsafe_get_user(x, (unsigned int __user *)ea, Efault);
 		break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 8:
 		unsafe_get_user(x, (unsigned long __user *)ea, Efault);
 		break;
@@ -364,7 +364,7 @@ static __always_inline int __copy_mem_in(u8 *dest, unsigned long ea, int nb, str
 		case 4:
 			unsafe_get_user(*(u32 *)dest, (u32 __user *)ea, Efault);
 			break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 8:
 			unsafe_get_user(*(u64 *)dest, (u64 __user *)ea, Efault);
 			break;
@@ -443,7 +443,7 @@ __write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs
 	case 4:
 		unsafe_put_user(val, (unsigned int __user *)ea, Efault);
 		break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 8:
 		unsafe_put_user(val, (unsigned long __user *)ea, Efault);
 		break;
@@ -494,7 +494,7 @@ static __always_inline int __copy_mem_out(u8 *dest, unsigned long ea, int nb, st
 		case 4:
 			unsafe_put_user(*(u32 *)dest, (u32 __user *)ea, Efault);
 			break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 8:
 			unsafe_put_user(*(u64 *)dest, (u64 __user *)ea, Efault);
 			break;
@@ -722,7 +722,7 @@ static nokprobe_inline int do_vec_store(int rn, unsigned long ea,
 }
 #endif /* CONFIG_ALTIVEC */
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 static nokprobe_inline int emulate_lq(struct pt_regs *regs, unsigned long ea,
 				      int reg, bool cross_endian)
 {
@@ -765,7 +765,7 @@ static nokprobe_inline int emulate_stq(struct pt_regs *regs, unsigned long ea,
 		err = write_mem(vals[IS_BE], ea + 8, 8, regs);
 	return err;
 }
-#endif /* __powerpc64 */
+#endif /* CONFIG_PPC64 */
 
 #ifdef CONFIG_VSX
 static nokprobe_inline void emulate_vsx_load(struct instruction_op *op, union vsx_reg *reg,
@@ -1316,7 +1316,7 @@ static nokprobe_inline int trap_compare(long v1, long v2)
  */
 #define MASK32(mb, me)	((0xffffffffUL >> (mb)) + \
 			 ((signed long)-0x80000000L >> (me)) + ((me) >= (mb)))
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 #define MASK64_L(mb)	(~0UL >> (mb))
 #define MASK64_R(me)	((signed long)-0x8000000000000000L >> (me))
 #define MASK64(mb, me)	(MASK64_L(mb) + MASK64_R(me) + ((me) >= (mb)))
@@ -1342,16 +1342,19 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 {
 #ifdef CONFIG_PPC64
 	unsigned int suffixopcode, prefixtype, prefix_r;
+	unsigned int rc, suffix;
 #endif
-	unsigned int opcode, ra, rb, rc, rd, spr, u;
+	unsigned int opcode, ra, rb, rd, spr, u;
 	unsigned long int imm;
 	unsigned long int val, val2;
 	unsigned int mb, me, sh;
-	unsigned int word, suffix;
+	unsigned int word;
 	long ival;
 
 	word = ppc_inst_val(instr);
+#ifdef CONFIG_PPC64
 	suffix = ppc_inst_suffix(instr);
+#endif
 
 	op->type = COMPUTE;
 
@@ -1447,7 +1450,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		switch ((word >> 1) & 0x3ff) {
 		case 598:	/* sync */
 			op->type = BARRIER + BARRIER_SYNC;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 			switch ((word >> 21) & 3) {
 			case 1:		/* lwsync */
 				op->type = BARRIER + BARRIER_LWSYNC;
@@ -1469,10 +1472,12 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 	rd = (word >> 21) & 0x1f;
 	ra = (word >> 16) & 0x1f;
 	rb = (word >> 11) & 0x1f;
+#ifdef CONFIG_PPC64
 	rc = (word >> 6) & 0x1f;
+#endif
 
 	switch (opcode) {
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 1:
 		if (!cpu_has_feature(CPU_FTR_ARCH_31))
 			goto unknown_opcode;
@@ -1506,7 +1511,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			goto trap;
 		return 1;
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 4:
 		/*
 		 * There are very many instructions with this primary opcode
@@ -1555,7 +1560,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 	case 10:	/* cmpli */
 		imm = (unsigned short) word;
 		val = regs->gpr[ra];
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		if ((rd & 1) == 0)
 			val = (unsigned int) val;
 #endif
@@ -1565,7 +1570,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 	case 11:	/* cmpi */
 		imm = (short) word;
 		val = regs->gpr[ra];
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		if ((rd & 1) == 0)
 			val = (int) val;
 #endif
@@ -1662,7 +1667,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		set_cr0(regs, op);
 		goto logical_done_nocc;
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 30:	/* rld* */
 		mb = ((word >> 6) & 0x1f) | (word & 0x20);
 		val = regs->gpr[rd];
@@ -1720,7 +1725,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 					       (int)regs->gpr[rb])))
 				goto trap;
 			return 1;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 68:	/* td */
 			if (rd & trap_compare(regs->gpr[ra], regs->gpr[rb]))
 				goto trap;
@@ -1826,7 +1831,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		case 0:	/* cmp */
 			val = regs->gpr[ra];
 			val2 = regs->gpr[rb];
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 			if ((rd & 1) == 0) {
 				/* word (32-bit) compare */
 				val = (int) val;
@@ -1839,7 +1844,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		case 32:	/* cmpl */
 			val = regs->gpr[ra];
 			val2 = regs->gpr[rb];
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 			if ((rd & 1) == 0) {
 				/* word (32-bit) compare */
 				val = (unsigned int) val;
@@ -1860,7 +1865,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			add_with_carry(regs, op, rd, ~regs->gpr[ra],
 				       regs->gpr[rb], 1);
 			goto arith_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 9:	/* mulhdu */
 			asm("mulhdu %0,%1,%2" : "=r" (op->val) :
 			    "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
@@ -1879,7 +1884,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		case 40:	/* subf */
 			op->val = regs->gpr[rb] - regs->gpr[ra];
 			goto arith_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 73:	/* mulhd */
 			asm("mulhd %0,%1,%2" : "=r" (op->val) :
 			    "r" (regs->gpr[ra]), "r" (regs->gpr[rb]));
@@ -1918,7 +1923,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			add_with_carry(regs, op, rd, ~regs->gpr[ra], -1L,
 				       regs->xer & XER_CA);
 			goto arith_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 233:	/* mulld */
 			op->val = regs->gpr[ra] * regs->gpr[rb];
 			goto arith_done;
@@ -1933,7 +1938,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 				(int) regs->gpr[rb];
 
 			goto arith_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 265:	/* modud */
 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
 				goto unknown_opcode;
@@ -1950,7 +1955,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			op->val = (unsigned int) regs->gpr[ra] %
 				(unsigned int) regs->gpr[rb];
 			goto compute_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 457:	/* divdu */
 			op->val = regs->gpr[ra] / regs->gpr[rb];
 			goto arith_done;
@@ -1959,7 +1964,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			op->val = (unsigned int) regs->gpr[ra] /
 				(unsigned int) regs->gpr[rb];
 			goto arith_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 489:	/* divd */
 			op->val = (long int) regs->gpr[ra] /
 				(long int) regs->gpr[rb];
@@ -1969,7 +1974,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			op->val = (int) regs->gpr[ra] /
 				(int) regs->gpr[rb];
 			goto arith_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 425:	/* divde[.] */
 			asm volatile(PPC_DIVDE(%0, %1, %2) :
 				"=r" (op->val) : "r" (regs->gpr[ra]),
@@ -2002,7 +2007,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			}
 
 			goto unknown_opcode;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 777:	/* modsd */
 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
 				goto unknown_opcode;
@@ -2025,7 +2030,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			val = (unsigned int) regs->gpr[rd];
 			op->val = ( val ? __builtin_clz(val) : 32 );
 			goto logical_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 58:	/* cntlzd */
 			val = regs->gpr[rd];
 			op->val = ( val ? __builtin_clzl(val) : 64 );
@@ -2093,7 +2098,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			val = (unsigned int) regs->gpr[rd];
 			op->val = (val ? __builtin_ctz(val) : 32);
 			goto logical_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 570:	/* cnttzd */
 			if (!cpu_has_feature(CPU_FTR_ARCH_300))
 				goto unknown_opcode;
@@ -2108,7 +2113,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		case 954:	/* extsb */
 			op->val = (signed char) regs->gpr[rd];
 			goto logical_done;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 986:	/* extsw */
 			op->val = (signed int) regs->gpr[rd];
 			goto logical_done;
@@ -2159,7 +2164,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			set_ca32(op, op->xerval & XER_CA);
 			goto logical_done;
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 27:	/* sld */
 			sh = regs->gpr[rb] & 0x7f;
 			if (sh < 64)
@@ -2216,7 +2221,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 				op->val = val;
 			goto logical_done;
 
-#endif /* __powerpc64__ */
+#endif /* CONFIG_PPC64 */
 
 /*
  * Cache instructions
@@ -2296,7 +2301,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			op->type = MKOP(STCX, 0, 2);
 			break;
 #endif
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 84:	/* ldarx */
 			op->type = MKOP(LARX, 0, 8);
 			break;
@@ -2373,7 +2378,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			break;
 #endif /* CONFIG_ALTIVEC */
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 21:	/* ldx */
 		case 53:	/* ldux */
 			op->type = MKOP(LOAD, u, 8);
@@ -2400,7 +2405,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			op->type = MKOP(LOAD, u, 2);
 			break;
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 341:	/* lwax */
 		case 373:	/* lwaux */
 			op->type = MKOP(LOAD, SIGNEXT | u, 4);
@@ -2417,7 +2422,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			op->type = MKOP(STORE, u, 2);
 			break;
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 532:	/* ldbrx */
 			op->type = MKOP(LOAD, BYTEREV, 8);
 			break;
@@ -2459,7 +2464,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 			op->type = MKOP(STORE_FP, u, 8);
 			break;
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 791:	/* lfdpx */
 			op->type = MKOP(LOAD_FP, 0, 16);
 			break;
@@ -2479,10 +2484,10 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		case 983:	/* stfiwx */
 			op->type = MKOP(STORE_FP, 0, 4);
 			break;
-#endif /* __powerpc64 */
+#endif /* CONFIG_PPC64 */
 #endif /* CONFIG_PPC_FPU */
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 660:	/* stdbrx */
 			op->type = MKOP(STORE, BYTEREV, 8);
 			op->val = byterev_8(regs->gpr[rd]);
@@ -2820,7 +2825,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		break;
 #endif
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 56:	/* lq */
 		if (!((rd & 1) || (rd == ra)))
 			op->type = MKOP(LOAD, 0, 16);
@@ -2857,7 +2862,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		break;
 #endif /* CONFIG_VSX */
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 58:	/* ld[u], lwa */
 		op->ea = dsform_ea(word, regs);
 		switch (word & 3) {
@@ -2946,7 +2951,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		break;
 #endif /* CONFIG_VSX */
 
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 62:	/* std[u] */
 		op->ea = dsform_ea(word, regs);
 		switch (word & 3) {
@@ -3099,7 +3104,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
 		case 3: /* Type 11 Modified Register-to-Register */
 			break;
 		}
-#endif /* __powerpc64__ */
+#endif /* CONFIG_PPC64 */
 
 	}
 
@@ -3200,7 +3205,7 @@ static nokprobe_inline void do_byterev(unsigned long *valp, int size)
 	case 4:
 		*valp = byterev_4(*valp);
 		break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 	case 8:
 		*valp = byterev_8(*valp);
 		break;
@@ -3341,7 +3346,7 @@ int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op)
 		case 4:
 			__get_user_asmx(val, ea, err, "lwarx");
 			break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 8:
 			__get_user_asmx(val, ea, err, "ldarx");
 			break;
@@ -3367,7 +3372,7 @@ int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op)
 			return -EFAULT;
 		err = 0;
 		switch (size) {
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 1:
 			__put_user_asmx(op->val, ea, err, "stbcx.", cr);
 			break;
@@ -3378,7 +3383,7 @@ int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op)
 		case 4:
 			__put_user_asmx(op->val, ea, err, "stwcx.", cr);
 			break;
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		case 8:
 			__put_user_asmx(op->val, ea, err, "stdcx.", cr);
 			break;
@@ -3399,7 +3404,7 @@ int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op)
 		break;
 
 	case LOAD:
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		if (size == 16) {
 			err = emulate_lq(regs, ea, op->reg, cross_endian);
 			break;
@@ -3473,7 +3478,7 @@ int emulate_loadstore(struct pt_regs *regs, struct instruction_op *op)
 		break;
 
 	case STORE:
-#ifdef __powerpc64__
+#ifdef CONFIG_PPC64
 		if (size == 16) {
 			err = emulate_stq(regs, ea, op->reg, cross_endian);
 			break;
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index cf692b2b5f7bc5..9ca28e4a610a26 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -19,6 +19,7 @@
 #include <linux/string_helpers.h>
 #include <linux/memory.h>
 #include <linux/kfence.h>
+#include <linux/vmemmap-optimization.h>
 
 #include <asm/pgalloc.h>
 #include <asm/mmu_context.h>
@@ -1217,92 +1218,6 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in
 	return 0;
 }
 
-static pte_t * __meminit radix__vmemmap_populate_address(unsigned long addr, int node,
-							 struct vmem_altmap *altmap,
-							 struct page *reuse)
-{
-	pgd_t *pgd;
-	p4d_t *p4d;
-	pud_t *pud;
-	pmd_t *pmd;
-	pte_t *pte;
-
-	pgd = pgd_offset_k(addr);
-	p4d = p4d_offset(pgd, addr);
-	pud = vmemmap_pud_alloc(p4d, node, addr);
-	if (!pud)
-		return NULL;
-	pmd = vmemmap_pmd_alloc(pud, node, addr);
-	if (!pmd)
-		return NULL;
-	if (pmd_leaf(*pmd))
-		/*
-		 * The second page is mapped as a hugepage due to a nearby request.
-		 * Force our mapping to page size without deduplication
-		 */
-		return NULL;
-	pte = vmemmap_pte_alloc(pmd, node, addr);
-	if (!pte)
-		return NULL;
-	radix__vmemmap_pte_populate(pmd, addr, node, NULL, NULL);
-	vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
-
-	return pte;
-}
-
-static pte_t * __meminit vmemmap_compound_tail_page(unsigned long addr,
-						    unsigned long pfn_offset, int node)
-{
-	pgd_t *pgd;
-	p4d_t *p4d;
-	pud_t *pud;
-	pmd_t *pmd;
-	pte_t *pte;
-	unsigned long map_addr;
-
-	/* the second vmemmap page which we use for duplication */
-	map_addr = addr - pfn_offset * sizeof(struct page) + PAGE_SIZE;
-	pgd = pgd_offset_k(map_addr);
-	p4d = p4d_offset(pgd, map_addr);
-	pud = vmemmap_pud_alloc(p4d, node, map_addr);
-	if (!pud)
-		return NULL;
-	pmd = vmemmap_pmd_alloc(pud, node, map_addr);
-	if (!pmd)
-		return NULL;
-	if (pmd_leaf(*pmd))
-		/*
-		 * The second page is mapped as a hugepage due to a nearby request.
-		 * Force our mapping to page size without deduplication
-		 */
-		return NULL;
-	pte = vmemmap_pte_alloc(pmd, node, map_addr);
-	if (!pte)
-		return NULL;
-	/*
-	 * Check if there exist a mapping to the left
-	 */
-	if (pte_none(*pte)) {
-		/*
-		 * Populate the head page vmemmap page.
-		 * It can fall in different pmd, hence
-		 * vmemmap_populate_address()
-		 */
-		pte = radix__vmemmap_populate_address(map_addr - PAGE_SIZE, node, NULL, NULL);
-		if (!pte)
-			return NULL;
-		/*
-		 * Populate the tail pages vmemmap page
-		 */
-		pte = radix__vmemmap_pte_populate(pmd, map_addr, node, NULL, NULL);
-		if (!pte)
-			return NULL;
-		vmemmap_verify(pte, node, map_addr, map_addr + PAGE_SIZE);
-		return pte;
-	}
-	return pte;
-}
-
 int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 					      unsigned long start,
 					      unsigned long end, int node,
@@ -1320,6 +1235,12 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
+	struct page *tail_page;
+	unsigned int order = pfn_to_section_compound_order(start_pfn);
+
+	tail_page = vmemmap_shared_tail_page(order, device_zone(node));
+	if (!tail_page)
+		return -ENOMEM;
 
 	for (addr = start; addr < end; addr = next) {
 
@@ -1343,16 +1264,15 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 		if (!pte_none(*pte)) {
 			/*
 			 * This could be because we already have a compound
-			 * page whose VMEMMAP_RESERVE_NR pages were mapped and
+			 * page whose retained vmemmap page was mapped and
 			 * this request fall in those pages.
 			 */
 			next = addr + PAGE_SIZE;
 			continue;
 		} else {
-			unsigned long nr_pages = pgmap_vmemmap_nr(pgmap);
+			unsigned long nr_pages = 1UL << order;
 			unsigned long addr_pfn = page_to_pfn((struct page *)addr);
 			unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages);
-			pte_t *tail_page_pte;
 
 			/*
 			 * if the address is aligned to huge page size it is the
@@ -1365,35 +1285,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 					return -ENOMEM;
 				vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
 
-				/*
-				 * Populate the tail pages vmemmap page
-				 * It can fall in different pmd, hence
-				 * vmemmap_populate_address()
-				 */
-				pte = radix__vmemmap_populate_address(addr + PAGE_SIZE, node, NULL, NULL);
-				if (!pte)
-					return -ENOMEM;
-
-				next = addr + 2 * PAGE_SIZE;
-				continue;
-			}
-			/*
-			 * get the 2nd mapping details
-			 * Also create it if that doesn't exist
-			 */
-			tail_page_pte = vmemmap_compound_tail_page(addr, pfn_offset, node);
-			if (!tail_page_pte) {
-
-				pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, NULL);
-				if (!pte)
-					return -ENOMEM;
-				vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
-
 				next = addr + PAGE_SIZE;
 				continue;
 			}
 
-			pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, pte_page(*tail_page_pte));
+			pte = radix__vmemmap_pte_populate(pmd, addr, node, NULL, tail_page);
 			if (!pte)
 				return -ENOMEM;
 			vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 7de5760164a90f..b4603a98224b32 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -627,7 +627,7 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
 {
 #ifdef CONFIG_HUGETLB_PAGE
 	/* need the return fix for nohash.c */
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		return radix__local_flush_hugetlb_page(vma, vmaddr);
 #endif
 	radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
@@ -945,7 +945,7 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 {
 #ifdef CONFIG_HUGETLB_PAGE
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		return radix__flush_hugetlb_page(vma, vmaddr);
 #endif
 	radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, mmu_virtual_psize);
@@ -1113,7 +1113,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 
 {
 #ifdef CONFIG_HUGETLB_PAGE
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		return radix__flush_hugetlb_tlb_range(vma, start, end);
 #endif
 
diff --git a/arch/powerpc/mm/nohash/e500_hugetlbpage.c b/arch/powerpc/mm/nohash/e500_hugetlbpage.c
index a134d28a0e4d39..b87623f04be53c 100644
--- a/arch/powerpc/mm/nohash/e500_hugetlbpage.c
+++ b/arch/powerpc/mm/nohash/e500_hugetlbpage.c
@@ -180,7 +180,7 @@ book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea, pte_t pte)
  */
 void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
 {
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		book3e_hugetlb_preload(vma, address, *ptep);
 }
 
diff --git a/arch/powerpc/mm/nohash/tlb.c b/arch/powerpc/mm/nohash/tlb.c
index 0a650742f3a008..07a2db16c2b153 100644
--- a/arch/powerpc/mm/nohash/tlb.c
+++ b/arch/powerpc/mm/nohash/tlb.c
@@ -278,7 +278,7 @@ void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 {
 #ifdef CONFIG_HUGETLB_PAGE
-	if (vma && is_vm_hugetlb_page(vma))
+	if (vma && vma_is_hugetlb(vma))
 		flush_hugetlb_page(vma, vmaddr);
 #endif
 
diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c
index 0d499aebee72ff..3451351b756b4d 100644
--- a/arch/powerpc/mm/ptdump/ptdump.c
+++ b/arch/powerpc/mm/ptdump/ptdump.c
@@ -191,7 +191,7 @@ static void note_prot_wx(struct pg_state *st, unsigned long addr)
 	if (!pte_write(pte) || !pte_exec(pte))
 		return;
 
-	WARN_ONCE(IS_ENABLED(CONFIG_DEBUG_WX),
+	WARN_ONCE(IS_ENABLED(CONFIG_CHECK_WX),
 		  "powerpc/mm: Found insecure W+X mapping at address %p/%pS\n",
 		  (void *)st->start_address, (void *)st->start_address);
 
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 7b07b43575f111..7a612688e7a20b 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -602,14 +602,18 @@ int arch_protect_bpf_trampoline(void *image, unsigned int size)
 }
 
 static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx,
-			   struct bpf_tramp_node *n, int regs_off, int retval_off,
-			   int run_ctx_off, bool save_ret)
+			   struct bpf_tramp_image *im, struct bpf_tramp_node *n,
+			   int regs_off, int retval_off, int run_ctx_off, bool save_ret)
 {
 	struct bpf_prog *p = n->link->prog;
 	ppc_inst_t branch_insn;
-	u32 jmp_idx;
+	u32 jmp_idx, skip_idx;
 	int ret = 0;
 
+	/* nop, patched to skip this prog when it is detached */
+	skip_idx = ctx->idx;
+	EMIT(PPC_RAW_NOP());
+
 	/* Save cookie */
 	if (IS_ENABLED(CONFIG_PPC64)) {
 		PPC_LI64(_R3, n->cookie);
@@ -679,13 +683,17 @@ static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ct
 	EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off));
 	ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
 					 (unsigned long)bpf_trampoline_exit(p));
+	if (ret)
+		return ret;
 
-	return ret;
+	if (ro_image) /* image is NULL for dummy pass */
+		bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]);
+	return 0;
 }
 
 static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx,
-			      struct bpf_tramp_nodes *tn, int regs_off, int retval_off,
-			      int run_ctx_off, u32 *branches)
+			      struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn,
+			      int regs_off, int retval_off, int run_ctx_off, u32 *branches)
 {
 	int i;
 
@@ -696,8 +704,8 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context
 	EMIT(PPC_RAW_LI(_R3, 0));
 	EMIT(PPC_RAW_STL(_R3, _R1, retval_off));
 	for (i = 0; i < tn->nr_nodes; i++) {
-		if (invoke_bpf_prog(image, ro_image, ctx, tn->nodes[i], regs_off, retval_off,
-				    run_ctx_off, true))
+		if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off,
+				    retval_off, run_ctx_off, true))
 			return -EINVAL;
 
 		/*
@@ -1043,8 +1051,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 			cookie_ctx_off--;
 		}
 
-		if (invoke_bpf_prog(image, ro_image, ctx, fentry->nodes[i], regs_off, retval_off,
-				    run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))
+		if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off,
+				    retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))
 			return -EINVAL;
 	}
 
@@ -1053,7 +1061,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		if (!branches)
 			return -ENOMEM;
 
-		if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off,
+		if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off,
 				       run_ctx_off, branches)) {
 			ret = -EINVAL;
 			goto cleanup;
@@ -1090,11 +1098,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		/* Restore updated tail_call_cnt */
 		if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
 			bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off);
-
-		/* Reserve space to patch branch instruction to skip fexit progs */
-		if (ro_image) /* image is NULL for dummy pass */
-			im->ip_after_call = &((u32 *)ro_image)[ctx->idx];
-		EMIT(PPC_RAW_NOP());
 	}
 
 	/* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */
@@ -1123,16 +1126,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 			cookie_ctx_off--;
 		}
 
-		if (invoke_bpf_prog(image, ro_image, ctx, fexit->nodes[i], regs_off, retval_off,
-				    run_ctx_off, false)) {
+		if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off,
+				    retval_off, run_ctx_off, false)) {
 			ret = -EINVAL;
 			goto cleanup;
 		}
 	}
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
-		if (ro_image) /* image is NULL for dummy pass */
-			im->ip_epilogue = &((u32 *)ro_image)[ctx->idx];
 		PPC_LI_ADDR(_R3, im);
 		ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx,
 						 (unsigned long)__bpf_tramp_exit);
@@ -1192,7 +1193,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 			     struct bpf_tramp_nodes *tnodes, void *func_addr)
 {
-	struct bpf_tramp_image im;
+	struct bpf_tramp_image im = {};
 	int ret;
 
 	ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr);
@@ -1320,7 +1321,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 
 	/*
 	 * If we are not poking at bpf prog entry, then we are simply patching in/out
-	 * an unconditional branch instruction at im->ip_after_call
+	 * an unconditional branch instruction in a trampoline image
 	 */
 	if (offset) {
 		if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) {
diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c
index b0eedf18596060..3e094dc7d76b29 100644
--- a/arch/powerpc/perf/hv-24x7.c
+++ b/arch/powerpc/perf/hv-24x7.c
@@ -462,7 +462,7 @@ static struct attribute *device_str_attr_create_(char *name, char *str)
 	attr->var = str;
 	attr->attr.attr.name = name;
 	attr->attr.attr.mode = 0444;
-	attr->attr.show = device_show_string;
+	attr->attr.show_const = device_show_string;
 
 	return &attr->attr.attr;
 }
diff --git a/arch/powerpc/perf/perf_regs.c b/arch/powerpc/perf/perf_regs.c
index 350dccb0143c77..a01d8a903640d4 100644
--- a/arch/powerpc/perf/perf_regs.c
+++ b/arch/powerpc/perf/perf_regs.c
@@ -125,7 +125,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 	return regs_get_register(regs, pt_regs_offset[idx]);
 }
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask || mask & REG_RESERVED)
 		return -EINVAL;
diff --git a/arch/powerpc/platforms/512x/Kconfig b/arch/powerpc/platforms/512x/Kconfig
index deecede78776c2..9d46d791af05c5 100644
--- a/arch/powerpc/platforms/512x/Kconfig
+++ b/arch/powerpc/platforms/512x/Kconfig
@@ -32,11 +32,12 @@ config MPC512x_GENERIC
 	  which do not need custom platform specific setup.
 
 	  Compatible boards include:  Protonic LVT base boards (ZANMCU
-	  and VICVT2), Freescale MPC5125 Tower system.
+	  and VICVT2), Freescale MPC5125 Tower system, IFM PDM360NG.
 
 config PDM360NG
 	bool "ifm PDM360NG board"
 	depends on PPC_MPC512x
+	select MPC512x_GENERIC
 	select DEFAULT_UIMAGE
 	help
 	  This option enables support for the PDM360NG board.
diff --git a/arch/powerpc/platforms/512x/Makefile b/arch/powerpc/platforms/512x/Makefile
index 2daf22ee26a04b..4da6f6923d8fe1 100644
--- a/arch/powerpc/platforms/512x/Makefile
+++ b/arch/powerpc/platforms/512x/Makefile
@@ -7,4 +7,3 @@ obj-y				+= mpc512x_shared.o
 obj-$(CONFIG_MPC5121_ADS)	+= mpc5121_ads.o mpc5121_ads_cpld.o
 obj-$(CONFIG_MPC512x_GENERIC)	+= mpc512x_generic.o
 obj-$(CONFIG_MPC512x_LPBFIFO)	+= mpc512x_lpbfifo.o
-obj-$(CONFIG_PDM360NG)		+= pdm360ng.o
diff --git a/arch/powerpc/platforms/512x/mpc512x_generic.c b/arch/powerpc/platforms/512x/mpc512x_generic.c
index d4fa6c302ccfb0..d330d3d8588723 100644
--- a/arch/powerpc/platforms/512x/mpc512x_generic.c
+++ b/arch/powerpc/platforms/512x/mpc512x_generic.c
@@ -24,6 +24,7 @@ static const char * const board[] __initconst = {
 	"prt,prtlvt",
 	"fsl,mpc5125ads",
 	"ifm,ac14xx",
+	"ifm,pdm360ng",
 	NULL
 };
 
diff --git a/arch/powerpc/platforms/512x/pdm360ng.c b/arch/powerpc/platforms/512x/pdm360ng.c
deleted file mode 100644
index 8bbbf78bb42bdd..00000000000000
--- a/arch/powerpc/platforms/512x/pdm360ng.c
+++ /dev/null
@@ -1,126 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2010 DENX Software Engineering
- *
- * Anatolij Gustschin, <agust@denx.de>
- *
- * PDM360NG board setup
- */
-
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/io.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/of_fdt.h>
-
-#include <asm/machdep.h>
-#include <asm/ipic.h>
-
-#include "mpc512x.h"
-
-#if defined(CONFIG_TOUCHSCREEN_ADS7846) || \
-    defined(CONFIG_TOUCHSCREEN_ADS7846_MODULE)
-#include <linux/interrupt.h>
-#include <linux/spi/ads7846.h>
-#include <linux/spi/spi.h>
-#include <linux/notifier.h>
-
-static void *pdm360ng_gpio_base;
-
-static int pdm360ng_get_pendown_state(void)
-{
-	u32 reg;
-
-	reg = in_be32(pdm360ng_gpio_base + 0xc);
-	if (reg & 0x40)
-		setbits32(pdm360ng_gpio_base + 0xc, 0x40);
-
-	reg = in_be32(pdm360ng_gpio_base + 0x8);
-
-	/* return 1 if pen is down */
-	return (reg & 0x40) == 0;
-}
-
-static struct ads7846_platform_data pdm360ng_ads7846_pdata = {
-	.model			= 7845,
-	.get_pendown_state	= pdm360ng_get_pendown_state,
-	.irq_flags		= IRQF_TRIGGER_LOW,
-};
-
-static int __init pdm360ng_penirq_init(void)
-{
-	struct device_node *np;
-
-	np = of_find_compatible_node(NULL, NULL, "fsl,mpc5121-gpio");
-	if (!np) {
-		pr_err("%s: Can't find 'mpc5121-gpio' node\n", __func__);
-		return -ENODEV;
-	}
-
-	pdm360ng_gpio_base = of_iomap(np, 0);
-	of_node_put(np);
-	if (!pdm360ng_gpio_base) {
-		pr_err("%s: Can't map gpio regs.\n", __func__);
-		return -ENODEV;
-	}
-	out_be32(pdm360ng_gpio_base + 0xc, 0xffffffff);
-	setbits32(pdm360ng_gpio_base + 0x18, 0x2000);
-	setbits32(pdm360ng_gpio_base + 0x10, 0x40);
-
-	return 0;
-}
-
-static int pdm360ng_touchscreen_notifier_call(struct notifier_block *nb,
-					unsigned long event, void *__dev)
-{
-	struct device *dev = __dev;
-
-	if ((event == BUS_NOTIFY_ADD_DEVICE) &&
-	    of_device_is_compatible(dev->of_node, "ti,ads7846")) {
-		dev->platform_data = &pdm360ng_ads7846_pdata;
-		return NOTIFY_OK;
-	}
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block pdm360ng_touchscreen_nb = {
-	.notifier_call = pdm360ng_touchscreen_notifier_call,
-};
-
-static void __init pdm360ng_touchscreen_init(void)
-{
-	if (pdm360ng_penirq_init())
-		return;
-
-	bus_register_notifier(&spi_bus_type, &pdm360ng_touchscreen_nb);
-}
-#else
-static inline void __init pdm360ng_touchscreen_init(void)
-{
-}
-#endif /* CONFIG_TOUCHSCREEN_ADS7846 */
-
-static void __init pdm360ng_init(void)
-{
-	mpc512x_init();
-	pdm360ng_touchscreen_init();
-}
-
-static int __init pdm360ng_probe(void)
-{
-	mpc512x_init_early();
-
-	return 1;
-}
-
-define_machine(pdm360ng) {
-	.name			= "PDM360NG",
-	.compatible		= "ifm,pdm360ng",
-	.probe			= pdm360ng_probe,
-	.setup_arch		= mpc512x_setup_arch,
-	.init			= pdm360ng_init,
-	.init_IRQ		= mpc512x_init_IRQ,
-	.get_irq		= ipic_get_irq,
-	.restart		= mpc512x_restart,
-};
diff --git a/arch/powerpc/platforms/cell/Kconfig b/arch/powerpc/platforms/cell/Kconfig
index db65bfcd1e7498..6bd26815c33103 100644
--- a/arch/powerpc/platforms/cell/Kconfig
+++ b/arch/powerpc/platforms/cell/Kconfig
@@ -10,7 +10,6 @@ config SPU_FS
 	tristate "SPU file system"
 	default m
 	depends on PPC_CELL
-	depends on COREDUMP
 	select SPU_BASE
 	help
 	  The SPU file system is used to access Synergistic Processing
diff --git a/arch/powerpc/platforms/cell/spu_syscalls.c b/arch/powerpc/platforms/cell/spu_syscalls.c
index 000894e07b027d..8be81207e886a1 100644
--- a/arch/powerpc/platforms/cell/spu_syscalls.c
+++ b/arch/powerpc/platforms/cell/spu_syscalls.c
@@ -88,26 +88,6 @@ SYSCALL_DEFINE3(spu_run,int, fd, __u32 __user *, unpc, __u32 __user *, ustatus)
 	return calls->spu_run(fd_file(arg), unpc, ustatus);
 }
 
-#ifdef CONFIG_COREDUMP
-int elf_coredump_extra_notes_size(void)
-{
-	CLASS(spufs_calls, calls)();
-	if (!calls)
-		return 0;
-
-	return calls->coredump_extra_notes_size();
-}
-
-int elf_coredump_extra_notes_write(struct coredump_params *cprm)
-{
-	CLASS(spufs_calls, calls)();
-	if (!calls)
-		return 0;
-
-	return calls->coredump_extra_notes_write(cprm);
-}
-#endif
-
 void notify_spus_active(void)
 {
 	struct spufs_calls *calls;
diff --git a/arch/powerpc/platforms/cell/spufs/Makefile b/arch/powerpc/platforms/cell/spufs/Makefile
index 52e4c80ec8d031..60319d4ff25a6a 100644
--- a/arch/powerpc/platforms/cell/spufs/Makefile
+++ b/arch/powerpc/platforms/cell/spufs/Makefile
@@ -4,7 +4,6 @@ obj-$(CONFIG_SPU_FS) += spufs.o
 spufs-y += inode.o file.o context.o syscalls.o
 spufs-y += sched.o backing_ops.o hw_ops.o run.o gang.o
 spufs-y += switch.o fault.o lscsa_alloc.o
-spufs-$(CONFIG_COREDUMP) += coredump.o
 
 # magic for the trace events
 CFLAGS_sched.o := -I$(src)
diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c
deleted file mode 100644
index 301ee7d8b7df02..00000000000000
--- a/arch/powerpc/platforms/cell/spufs/coredump.c
+++ /dev/null
@@ -1,183 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * SPU core dump code
- *
- * (C) Copyright 2006 IBM Corp.
- *
- * Author: Dwayne Grant McConnell <decimal@us.ibm.com>
- */
-
-#include <linux/elf.h>
-#include <linux/file.h>
-#include <linux/fdtable.h>
-#include <linux/fs.h>
-#include <linux/gfp.h>
-#include <linux/list.h>
-#include <linux/syscalls.h>
-#include <linux/coredump.h>
-#include <linux/binfmts.h>
-
-#include <linux/uaccess.h>
-
-#include "spufs.h"
-
-static int spufs_ctx_note_size(struct spu_context *ctx, int dfd)
-{
-	int i, sz, total = 0;
-	char *name;
-	char fullname[80];
-
-	for (i = 0; spufs_coredump_read[i].name != NULL; i++) {
-		name = spufs_coredump_read[i].name;
-		sz = spufs_coredump_read[i].size;
-
-		sprintf(fullname, "SPU/%d/%s", dfd, name);
-
-		total += sizeof(struct elf_note);
-		total += roundup(strlen(fullname) + 1, 4);
-		total += roundup(sz, 4);
-	}
-
-	return total;
-}
-
-static int match_context(const void *v, struct file *file, unsigned fd)
-{
-	struct spu_context *ctx;
-	if (file->f_op != &spufs_context_fops)
-		return 0;
-	ctx = SPUFS_I(file_inode(file))->i_ctx;
-	if (ctx->flags & SPU_CREATE_NOSCHED)
-		return 0;
-	return fd + 1;
-}
-
-/*
- * The additional architecture-specific notes for Cell are various
- * context files in the spu context.
- *
- * This function iterates over all open file descriptors and sees
- * if they are a directory in spufs.  In that case we use spufs
- * internal functionality to dump them without needing to actually
- * open the files.
- */
-/*
- * descriptor table is not shared, so files can't change or go away.
- */
-static struct spu_context *coredump_next_context(int *fd)
-{
-	struct spu_context *ctx = NULL;
-	struct file *file;
-	int n = iterate_fd(current->files, *fd, match_context, NULL);
-	if (!n)
-		return NULL;
-	*fd = n - 1;
-
-	file = fget_raw(*fd);
-	if (file) {
-		ctx = SPUFS_I(file_inode(file))->i_ctx;
-		get_spu_context(ctx);
-		fput(file);
-	}
-
-	return ctx;
-}
-
-int spufs_coredump_extra_notes_size(void)
-{
-	struct spu_context *ctx;
-	int size = 0, rc, fd;
-
-	fd = 0;
-	while ((ctx = coredump_next_context(&fd)) != NULL) {
-		rc = spu_acquire_saved(ctx);
-		if (rc) {
-			put_spu_context(ctx);
-			break;
-		}
-
-		rc = spufs_ctx_note_size(ctx, fd);
-		spu_release_saved(ctx);
-		if (rc < 0) {
-			put_spu_context(ctx);
-			break;
-		}
-
-		size += rc;
-
-		/* start searching the next fd next time */
-		fd++;
-		put_spu_context(ctx);
-	}
-
-	return size;
-}
-
-static int spufs_arch_write_note(struct spu_context *ctx, int i,
-				  struct coredump_params *cprm, int dfd)
-{
-	size_t sz = spufs_coredump_read[i].size;
-	char fullname[80];
-	struct elf_note en;
-	int ret;
-
-	sprintf(fullname, "SPU/%d/%s", dfd, spufs_coredump_read[i].name);
-	en.n_namesz = strlen(fullname) + 1;
-	en.n_descsz = sz;
-	en.n_type = NT_SPU;
-
-	if (!dump_emit(cprm, &en, sizeof(en)))
-		return -EIO;
-	if (!dump_emit(cprm, fullname, en.n_namesz))
-		return -EIO;
-	if (!dump_align(cprm, 4))
-		return -EIO;
-
-	if (spufs_coredump_read[i].dump) {
-		ret = spufs_coredump_read[i].dump(ctx, cprm);
-		if (ret < 0)
-			return ret;
-	} else {
-		char buf[32];
-
-		ret = snprintf(buf, sizeof(buf), "0x%.16llx",
-			       spufs_coredump_read[i].get(ctx));
-		if (ret >= sizeof(buf))
-			return sizeof(buf);
-
-		/* count trailing the NULL: */
-		if (!dump_emit(cprm, buf, ret + 1))
-			return -EIO;
-	}
-
-	dump_skip_to(cprm, roundup(cprm->pos - ret + sz, 4));
-	return 0;
-}
-
-int spufs_coredump_extra_notes_write(struct coredump_params *cprm)
-{
-	struct spu_context *ctx;
-	int fd, j, rc;
-
-	fd = 0;
-	while ((ctx = coredump_next_context(&fd)) != NULL) {
-		rc = spu_acquire_saved(ctx);
-		if (rc)
-			return rc;
-
-		for (j = 0; spufs_coredump_read[j].name != NULL; j++) {
-			rc = spufs_arch_write_note(ctx, j, cprm, fd);
-			if (rc) {
-				spu_release_saved(ctx);
-				return rc;
-			}
-		}
-
-		spu_release_saved(ctx);
-
-		/* start searching the next fd next time */
-		fd++;
-	}
-
-	return 0;
-}
diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c
index de7494748fecd6..98c47bafaf6781 100644
--- a/arch/powerpc/platforms/cell/spufs/file.c
+++ b/arch/powerpc/platforms/cell/spufs/file.c
@@ -9,7 +9,6 @@
 
 #undef DEBUG
 
-#include <linux/coredump.h>
 #include <linux/fs.h>
 #include <linux/ioctl.h>
 #include <linux/export.h>
@@ -130,14 +129,6 @@ static ssize_t spufs_attr_write(struct file *file, const char __user *buf,
 	return ret;
 }
 
-static ssize_t spufs_dump_emit(struct coredump_params *cprm, void *buf,
-		size_t size)
-{
-	if (!dump_emit(cprm, buf, size))
-		return -EIO;
-	return size;
-}
-
 #define DEFINE_SPUFS_SIMPLE_ATTRIBUTE(__fops, __get, __set, __fmt)	\
 static int __fops ## _open(struct inode *inode, struct file *file)	\
 {									\
@@ -180,12 +171,6 @@ spufs_mem_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static ssize_t
-spufs_mem_dump(struct spu_context *ctx, struct coredump_params *cprm)
-{
-	return spufs_dump_emit(cprm, ctx->ops->get_ls(ctx), LS_SIZE);
-}
-
 static ssize_t
 spufs_mem_read(struct file *file, char __user *buffer,
 				size_t size, loff_t *pos)
@@ -466,13 +451,6 @@ spufs_regs_open(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static ssize_t
-spufs_regs_dump(struct spu_context *ctx, struct coredump_params *cprm)
-{
-	return spufs_dump_emit(cprm, ctx->csa.lscsa->gprs,
-			       sizeof(ctx->csa.lscsa->gprs));
-}
-
 static ssize_t
 spufs_regs_read(struct file *file, char __user *buffer,
 		size_t size, loff_t *pos)
@@ -523,13 +501,6 @@ static const struct file_operations spufs_regs_fops = {
 	.llseek  = generic_file_llseek,
 };
 
-static ssize_t
-spufs_fpcr_dump(struct spu_context *ctx, struct coredump_params *cprm)
-{
-	return spufs_dump_emit(cprm, &ctx->csa.lscsa->fpcr,
-			       sizeof(ctx->csa.lscsa->fpcr));
-}
-
 static ssize_t
 spufs_fpcr_read(struct file *file, char __user * buffer,
 		size_t size, loff_t * pos)
@@ -953,15 +924,6 @@ spufs_signal1_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static ssize_t spufs_signal1_dump(struct spu_context *ctx,
-		struct coredump_params *cprm)
-{
-	if (!ctx->csa.spu_chnlcnt_RW[3])
-		return 0;
-	return spufs_dump_emit(cprm, &ctx->csa.spu_chnldata_RW[3],
-			       sizeof(ctx->csa.spu_chnldata_RW[3]));
-}
-
 static ssize_t __spufs_signal1_read(struct spu_context *ctx, char __user *buf,
 			size_t len)
 {
@@ -1086,15 +1048,6 @@ spufs_signal2_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static ssize_t spufs_signal2_dump(struct spu_context *ctx,
-		struct coredump_params *cprm)
-{
-	if (!ctx->csa.spu_chnlcnt_RW[4])
-		return 0;
-	return spufs_dump_emit(cprm, &ctx->csa.spu_chnldata_RW[4],
-			       sizeof(ctx->csa.spu_chnldata_RW[4]));
-}
-
 static ssize_t __spufs_signal2_read(struct spu_context *ctx, char __user *buf,
 			size_t len)
 {
@@ -1924,15 +1877,6 @@ static const struct file_operations spufs_caps_fops = {
 	.release	= single_release,
 };
 
-static ssize_t spufs_mbox_info_dump(struct spu_context *ctx,
-		struct coredump_params *cprm)
-{
-	if (!(ctx->csa.prob.mb_stat_R & 0x0000ff))
-		return 0;
-	return spufs_dump_emit(cprm, &ctx->csa.prob.pu_mb_R,
-			       sizeof(ctx->csa.prob.pu_mb_R));
-}
-
 static ssize_t spufs_mbox_info_read(struct file *file, char __user *buf,
 				   size_t len, loff_t *pos)
 {
@@ -1962,15 +1906,6 @@ static const struct file_operations spufs_mbox_info_fops = {
 	.llseek  = generic_file_llseek,
 };
 
-static ssize_t spufs_ibox_info_dump(struct spu_context *ctx,
-		struct coredump_params *cprm)
-{
-	if (!(ctx->csa.prob.mb_stat_R & 0xff0000))
-		return 0;
-	return spufs_dump_emit(cprm, &ctx->csa.priv2.puint_mb_R,
-			       sizeof(ctx->csa.priv2.puint_mb_R));
-}
-
 static ssize_t spufs_ibox_info_read(struct file *file, char __user *buf,
 				   size_t len, loff_t *pos)
 {
@@ -2005,13 +1940,6 @@ static size_t spufs_wbox_info_cnt(struct spu_context *ctx)
 	return (4 - ((ctx->csa.prob.mb_stat_R & 0x00ff00) >> 8)) * sizeof(u32);
 }
 
-static ssize_t spufs_wbox_info_dump(struct spu_context *ctx,
-		struct coredump_params *cprm)
-{
-	return spufs_dump_emit(cprm, &ctx->csa.spu_mailbox_data,
-			spufs_wbox_info_cnt(ctx));
-}
-
 static ssize_t spufs_wbox_info_read(struct file *file, char __user *buf,
 				   size_t len, loff_t *pos)
 {
@@ -2059,15 +1987,6 @@ static void spufs_get_dma_info(struct spu_context *ctx,
 	}
 }
 
-static ssize_t spufs_dma_info_dump(struct spu_context *ctx,
-		struct coredump_params *cprm)
-{
-	struct spu_dma_info info;
-
-	spufs_get_dma_info(ctx, &info);
-	return spufs_dump_emit(cprm, &info, sizeof(info));
-}
-
 static ssize_t spufs_dma_info_read(struct file *file, char __user *buf,
 			      size_t len, loff_t *pos)
 {
@@ -2112,15 +2031,6 @@ static void spufs_get_proxydma_info(struct spu_context *ctx,
 	}
 }
 
-static ssize_t spufs_proxydma_info_dump(struct spu_context *ctx,
-		struct coredump_params *cprm)
-{
-	struct spu_proxydma_info info;
-
-	spufs_get_proxydma_info(ctx, &info);
-	return spufs_dump_emit(cprm, &info, sizeof(info));
-}
-
 static ssize_t spufs_proxydma_info_read(struct file *file, char __user *buf,
 				   size_t len, loff_t *pos)
 {
@@ -2580,27 +2490,3 @@ const struct spufs_tree_descr spufs_dir_debug_contents[] = {
 	{ ".ctx", &spufs_ctx_fops, 0444, },
 	{},
 };
-
-const struct spufs_coredump_reader spufs_coredump_read[] = {
-	{ "regs", spufs_regs_dump, NULL, sizeof(struct spu_reg128[128])},
-	{ "fpcr", spufs_fpcr_dump, NULL, sizeof(struct spu_reg128) },
-	{ "lslr", NULL, spufs_lslr_get, 19 },
-	{ "decr", NULL, spufs_decr_get, 19 },
-	{ "decr_status", NULL, spufs_decr_status_get, 19 },
-	{ "mem", spufs_mem_dump, NULL, LS_SIZE, },
-	{ "signal1", spufs_signal1_dump, NULL, sizeof(u32) },
-	{ "signal1_type", NULL, spufs_signal1_type_get, 19 },
-	{ "signal2", spufs_signal2_dump, NULL, sizeof(u32) },
-	{ "signal2_type", NULL, spufs_signal2_type_get, 19 },
-	{ "event_mask", NULL, spufs_event_mask_get, 19 },
-	{ "event_status", NULL, spufs_event_status_get, 19 },
-	{ "mbox_info", spufs_mbox_info_dump, NULL, sizeof(u32) },
-	{ "ibox_info", spufs_ibox_info_dump, NULL, sizeof(u32) },
-	{ "wbox_info", spufs_wbox_info_dump, NULL, 4 * sizeof(u32)},
-	{ "dma_info", spufs_dma_info_dump, NULL, sizeof(struct spu_dma_info)},
-	{ "proxydma_info", spufs_proxydma_info_dump,
-			   NULL, sizeof(struct spu_proxydma_info)},
-	{ "object-id", NULL, spufs_object_id_get, 19 },
-	{ "npc", NULL, spufs_npc_get, 19 },
-	{ NULL },
-};
diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c
index 2b54afb31529f1..f066d9c7344d37 100644
--- a/arch/powerpc/platforms/cell/spufs/inode.c
+++ b/arch/powerpc/platforms/cell/spufs/inode.c
@@ -92,7 +92,7 @@ spufs_new_inode(struct super_block *sb, umode_t mode)
 }
 
 static int
-spufs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+spufs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 	      struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -266,9 +266,9 @@ spufs_mkdir(struct inode *dir, struct dentry *dentry, unsigned int flags,
 static int spufs_context_open(const struct path *path)
 {
 	FD_PREPARE(fdf, 0, dentry_open(path, O_RDONLY, current_cred()));
-	if (fdf.err)
-		return fdf.err;
-	fd_prepare_file(fdf)->f_op = &spufs_context_fops;
+	if (fdf->fd < 0)
+		return fdf->fd;
+	fdf->file->f_op = &spufs_context_fops;
 	return fd_publish(fdf);
 }
 
@@ -499,9 +499,9 @@ static int spufs_gang_open(const struct path *path)
 	 * in error path of *_open().
 	 */
 	FD_PREPARE(fdf, 0, dentry_open(path, O_RDONLY, current_cred()));
-	if (fdf.err)
-		return fdf.err;
-	fd_prepare_file(fdf)->f_op = &spufs_gang_fops;
+	if (fdf->fd < 0)
+		return fdf->fd;
+	fdf->file->f_op = &spufs_gang_fops;
 	return fd_publish(fdf);
 }
 
diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h
index d33787c57c39a2..612b5075d0ecb8 100644
--- a/arch/powerpc/platforms/cell/spufs/spufs.h
+++ b/arch/powerpc/platforms/cell/spufs/spufs.h
@@ -232,13 +232,9 @@ extern const struct spufs_tree_descr spufs_dir_debug_contents[];
 
 /* system call implementation */
 extern struct spufs_calls spufs_calls;
-struct coredump_params;
 long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *status);
 long spufs_create(const struct path *nd, struct dentry *dentry, unsigned int flags,
 			umode_t mode, struct file *filp);
-/* ELF coredump callbacks for writing SPU ELF notes */
-extern int spufs_coredump_extra_notes_size(void);
-extern int spufs_coredump_extra_notes_write(struct coredump_params *cprm);
 
 extern const struct file_operations spufs_context_fops;
 
@@ -335,14 +331,6 @@ void spufs_stop_callback(struct spu *spu, int irq);
 void spufs_mfc_callback(struct spu *spu);
 void spufs_dma_callback(struct spu *spu, int type);
 
-struct spufs_coredump_reader {
-	char *name;
-	ssize_t (*dump)(struct spu_context *ctx, struct coredump_params *cprm);
-	u64 (*get)(struct spu_context *ctx);
-	size_t size;
-};
-extern const struct spufs_coredump_reader spufs_coredump_read[];
-
 extern int spu_init_csa(struct spu_state *csa);
 extern void spu_fini_csa(struct spu_state *csa);
 extern int spu_save(struct spu_state *prev, struct spu *spu);
diff --git a/arch/powerpc/platforms/cell/spufs/syscalls.c b/arch/powerpc/platforms/cell/spufs/syscalls.c
index ea4ba1b6ce6a96..b6de37150e734d 100644
--- a/arch/powerpc/platforms/cell/spufs/syscalls.c
+++ b/arch/powerpc/platforms/cell/spufs/syscalls.c
@@ -82,8 +82,4 @@ struct spufs_calls spufs_calls = {
 	.spu_run = do_spu_run,
 	.notify_spus_active = do_notify_spus_active,
 	.owner = THIS_MODULE,
-#ifdef CONFIG_COREDUMP
-	.coredump_extra_notes_size = spufs_coredump_extra_notes_size,
-	.coredump_extra_notes_write = spufs_coredump_extra_notes_write,
-#endif
 };
diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig
index b5ad7c173ef0c1..dd8f6060fb7a2e 100644
--- a/arch/powerpc/platforms/powernv/Kconfig
+++ b/arch/powerpc/platforms/powernv/Kconfig
@@ -17,7 +17,6 @@ config PPC_POWERNV
 	select PPC_DOORBELL
 	select MMU_NOTIFIER
 	select FORCE_SMP
-	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU
 	default y
 
diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c
index a4ad4b49eef7ea..3586116382586a 100644
--- a/arch/powerpc/platforms/ps3/interrupt.c
+++ b/arch/powerpc/platforms/ps3/interrupt.c
@@ -184,8 +184,7 @@ static int ps3_virq_setup(enum ps3_cpu_binding cpu, unsigned long outlet,
 	if (!*virq) {
 		FAIL("%s:%d: irq_create_mapping failed: outlet %lu\n",
 			__func__, __LINE__, outlet);
-		result = -ENOMEM;
-		goto fail_create;
+		return -ENOMEM;
 	}
 
 	DBG("%s:%d: outlet %lu => cpu %u, virq %u\n", __func__, __LINE__,
@@ -196,17 +195,13 @@ static int ps3_virq_setup(enum ps3_cpu_binding cpu, unsigned long outlet,
 	if (result) {
 		FAIL("%s:%d: irq_set_chip_data failed\n",
 			__func__, __LINE__);
-		goto fail_set;
+		irq_dispose_mapping(*virq);
+		return result;
 	}
 
 	ps3_chip_mask(irq_get_irq_data(*virq));
 
 	return result;
-
-fail_set:
-	irq_dispose_mapping(*virq);
-fail_create:
-	return result;
 }
 
 /**
@@ -250,7 +245,7 @@ int ps3_irq_plug_setup(enum ps3_cpu_binding cpu, unsigned long outlet,
 
 	if (result) {
 		FAIL("%s:%d: ps3_virq_setup failed\n", __func__, __LINE__);
-		goto fail_setup;
+		return result;
 	}
 
 	pd = irq_get_chip_data(*virq);
@@ -263,16 +258,11 @@ int ps3_irq_plug_setup(enum ps3_cpu_binding cpu, unsigned long outlet,
 	if (result) {
 		FAIL("%s:%d: lv1_connect_irq_plug_ext failed: %s\n",
 		__func__, __LINE__, ps3_result(result));
-		result = -EPERM;
-		goto fail_connect;
+		ps3_virq_destroy(*virq);
+		return -EPERM;
 	}
 
 	return result;
-
-fail_connect:
-	ps3_virq_destroy(*virq);
-fail_setup:
-	return result;
 }
 EXPORT_SYMBOL_GPL(ps3_irq_plug_setup);
 
diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig
index 74910ce3a541c3..7d125e288f6ef7 100644
--- a/arch/powerpc/platforms/pseries/Kconfig
+++ b/arch/powerpc/platforms/pseries/Kconfig
@@ -23,7 +23,6 @@ config PPC_PSERIES
 	select HOTPLUG_CPU
 	select FORCE_SMP
 	select SWIOTLB
-	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU
 	default y
 
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index f29e7547c99566..c181df4dec7003 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -876,9 +876,9 @@ static void __init pSeries_setup_arch(void)
 			static_branch_enable(&shared_processor);
 			pv_spinlocks_init();
 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
-			static_key_slow_inc(&paravirt_steal_enabled);
+			static_branch_inc(&paravirt_steal_enabled);
 			if (steal_acc)
-				static_key_slow_inc(&paravirt_steal_rq_enabled);
+				static_branch_inc(&paravirt_steal_rq_enabled);
 #endif
 		}
 
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index d6c2dbf8455ced..0ccb72101d28b2 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -29,7 +29,7 @@ config RISCV
 	select ARCH_HAS_CURRENT_STACK_POINTER
 	select ARCH_HAS_DEBUG_VIRTUAL if MMU
 	select ARCH_HAS_DEBUG_VM_PGTABLE
-	select ARCH_HAS_DEBUG_WX
+	select ARCH_HAS_CHECK_WX
 	select ARCH_HAS_DELAY_TIMER
 	select ARCH_HAS_ELF_CORE_EFLAGS if BINFMT_ELF && ELF_CORE
 	select ARCH_HAS_FAST_MULTIPLIER
@@ -72,7 +72,6 @@ config RISCV
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS if 64BIT && MMU
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
-	select ARCH_SUPPORTS_PER_VMA_LOCK if MMU
 	select ARCH_HAS_PTE_PROTNONE if MMU
 	select ARCH_SUPPORTS_RT
 	select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK
@@ -195,7 +194,6 @@ config RISCV
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
-	select HAVE_PREEMPT_DYNAMIC_KEY
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RETHOOK
 	select HAVE_RSEQ
@@ -209,7 +207,6 @@ config RISCV
 	select IRQ_FORCED_THREADING
 	select KASAN_VMALLOC if KASAN
 	select LOCK_MM_AND_FIND_VMA
-	select MMU_GATHER_RCU_TABLE_FREE if SMP && MMU
 	select MODULES_USE_ELF_RELA if MODULES
 	select OF
 	select OF_EARLY_FLATTREE
diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata
index 3c945d086c7d02..1762ce7ae921c8 100644
--- a/arch/riscv/Kconfig.errata
+++ b/arch/riscv/Kconfig.errata
@@ -24,6 +24,7 @@ config ERRATA_ANDES_CMO
 config ERRATA_MIPS
 	bool "MIPS errata"
 	depends on RISCV_ALTERNATIVE
+	select RISCV_ALTERNATIVE_EARLY
 	help
 	  All MIPS errata Kconfig depend on this Kconfig. Disabling
 	  this Kconfig will disable all MIPS errata. Please say "Y"
@@ -46,7 +47,7 @@ config ERRATA_MIPS_P8700_PAUSE_OPCODE
 
 config ERRATA_SIFIVE
 	bool "SiFive errata"
-	depends on RISCV_ALTERNATIVE
+	depends on RISCV_ALTERNATIVE && 64BIT
 	help
 	  All SiFive errata Kconfig depend on this Kconfig. Disabling
 	  this Kconfig will disable all SiFive errata. Please say "Y"
diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs
index 429e0758930687..cecbce0c498384 100644
--- a/arch/riscv/Kconfig.socs
+++ b/arch/riscv/Kconfig.socs
@@ -34,6 +34,7 @@ config ARCH_RENESAS
 config ARCH_SIFIVE
 	bool "SiFive SoCs"
 	select ERRATA_SIFIVE
+	depends on 64BIT
 	help
 	  This enables support for SiFive SoC platform hardware.
 
diff --git a/arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts b/arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts
index 597407655efd2e..af78f1a4eeccb1 100644
--- a/arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts
+++ b/arch/riscv/boot/dts/anlogic/dr1v90-mlkpai-fs01.dts
@@ -23,6 +23,10 @@
 	};
 };
 
+&osc {
+	clock-frequency = <33333333>;
+};
+
 &uart1 {
 	status = "okay";
 };
diff --git a/arch/riscv/boot/dts/anlogic/dr1v90.dtsi b/arch/riscv/boot/dts/anlogic/dr1v90.dtsi
index 9fe183f5f5c8d3..574c6608aef014 100644
--- a/arch/riscv/boot/dts/anlogic/dr1v90.dtsi
+++ b/arch/riscv/boot/dts/anlogic/dr1v90.dtsi
@@ -3,6 +3,9 @@
  * Copyright (C) 2025 Junhui Liu <junhui.liu@pigmoral.tech>
  */
 
+#include <dt-bindings/clock/anlogic,dr1v90-cru.h>
+#include <dt-bindings/reset/anlogic,dr1v90-cru.h>
+
 /dts-v1/;
 / {
 	#address-cells = <2>;
@@ -40,6 +43,26 @@
 		};
 	};
 
+	clocks {
+		can_ext: clock-ext-can {
+			compatible = "fixed-clock";
+			clock-output-names = "can_ext";
+			#clock-cells = <0>;
+		};
+
+		osc: clock-osc {
+			compatible = "fixed-clock";
+			clock-output-names = "osc";
+			#clock-cells = <0>;
+		};
+
+		wdt_ext: clock-ext-wdt {
+			compatible = "fixed-clock";
+			clock-output-names = "wdt_ext";
+			#clock-cells = <0>;
+		};
+	};
+
 	soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&plic>;
@@ -81,21 +104,34 @@
 		uart0: serial@f8400000 {
 			compatible = "anlogic,dr1v90-uart", "snps,dw-apb-uart";
 			reg = <0x0 0xf8400000 0x0 0x1000>;
-			clock-frequency = <50000000>;
+			clocks = <&cru CLK_IO_400M_DIV8>, <&cru CLK_CPU_1X>;
+			clock-names = "baudclk", "apb_pclk";
 			interrupts = <71>;
 			reg-io-width = <4>;
 			reg-shift = <2>;
+			resets = <&cru RESET_UART0>;
 			status = "disabled";
 		};
 
 		uart1: serial@f8401000 {
 			compatible = "anlogic,dr1v90-uart", "snps,dw-apb-uart";
 			reg = <0x0 0xf8401000 0x0 0x1000>;
-			clock-frequency = <50000000>;
+			clocks = <&cru CLK_IO_400M_DIV8>, <&cru CLK_CPU_1X>;
+			clock-names = "baudclk", "apb_pclk";
 			interrupts = <72>;
 			reg-io-width = <4>;
 			reg-shift = <2>;
+			resets = <&cru RESET_UART1>;
 			status = "disabled";
 		};
+
+		cru: clock-controller@f8801000 {
+			compatible = "anlogic,dr1v90-cru";
+			reg = <0x0 0xf8801000 0 0x400>;
+			clocks = <&osc>, <&can_ext>, <&wdt_ext>;
+			clock-names = "osc", "can_ext", "wdt_ext";
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
 	};
 };
diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
index 6d7037c9528dd8..a09371d9d06406 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
@@ -90,10 +90,6 @@
 };
 
 &gpio0 {
-	interrupts = <13>, <14>, <15>, <16>,
-		     <17>, <18>, <19>, <20>,
-		     <21>, <22>, <23>, <24>,
-		     <25>, <26>;
 	ngpios = <14>;
 	gpio-line-names = "", "", "", "", "", "", "",
 			  "", "", "", "", "", "SD_CARD_CS", "USER_BUTTON";
@@ -101,12 +97,6 @@
 };
 
 &gpio1 {
-	interrupts = <27>, <28>, <29>, <30>,
-		     <31>, <32>, <33>, <34>,
-		     <35>, <36>, <37>, <38>,
-		     <39>, <40>, <41>, <42>,
-		     <43>, <44>, <45>, <46>,
-		     <47>, <48>, <49>, <50>;
 	ngpios = <24>;
 	gpio-line-names = "", "", "", "", "", "", "", "", "", "",
 			  "", "", "", "", "", "", "", "", "", "",
@@ -115,14 +105,6 @@
 };
 
 &gpio2 {
-	interrupts = <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>;
 	ngpios = <32>;
 	gpio-line-names = "P8_PIN3_USER_LED_0", "P8_PIN4_USER_LED_1", "P8_PIN5_USER_LED_2",
 			  "P8_PIN6_USER_LED_3", "P8_PIN7_USER_LED_4", "P8_PIN8_USER_LED_5",
diff --git a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts
index 246f8f2ab62b98..23f32a8a3d4593 100644
--- a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts
@@ -8,6 +8,7 @@
 /dts-v1/;
 
 #include "k1.dtsi"
+#include "k1-opp.dtsi"
 #include "k1-pinctrl.dtsi"
 
 / {
@@ -99,6 +100,38 @@
 	status = "okay";
 };
 
+&cpu_0 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_1 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_2 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_3 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_4 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_5 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_6 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_7 {
+	cpu-supply = <&buck1_0v9>;
+};
+
 &eth0 {
 	phy-handle = <&rgmii0>;
 	phy-mode = "rgmii-id";
@@ -178,7 +211,7 @@
 		dldoin2-supply = <&buck5>;
 
 		regulators {
-			buck1 {
+			buck1_0v9: buck1 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <1050000>;
 				regulator-ramp-delay = <5000>;
diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts
index 2d1ba34cb00e2b..b789319f9f98a9 100644
--- a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts
@@ -6,6 +6,7 @@
 /dts-v1/;
 
 #include "k1.dtsi"
+#include "k1-opp.dtsi"
 #include "k1-pinctrl.dtsi"
 
 / {
@@ -13,6 +14,7 @@
 	compatible = "xunlong,orangepi-r2s", "spacemit,k1";
 
 	aliases {
+		i2c2 = &i2c2;
 		i2c8 = &i2c8;
 		serial0 = &uart0;
 		ethernet0 = &eth0;
@@ -50,6 +52,38 @@
 	};
 };
 
+&cpu_0 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_1 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_2 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_3 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_4 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_5 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_6 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_7 {
+	cpu-supply = <&buck1_0v9>;
+};
+
 &combo_phy {
 	status = "okay";
 };
@@ -112,6 +146,35 @@
 	};
 };
 
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_0_cfg>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+		size = <256>;
+		read-only;
+		vcc-supply = <&buck3_1v8>;
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			mac-address {
+				#nvmem-cell-cells = <1>;
+			};
+
+			num-macs {
+			};
+
+			product-name {
+			};
+		};
+	};
+};
+
 &i2c8 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c8_cfg>;
@@ -132,7 +195,7 @@
 		dldoin2-supply = <&buck5>;
 
 		regulators {
-			buck1 {
+			buck1_0v9: buck1 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <1050000>;
 				regulator-ramp-delay = <5000>;
diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts
index b907a1396b21ff..0cf60d1985604a 100644
--- a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts
@@ -7,6 +7,7 @@
 /dts-v1/;
 
 #include "k1.dtsi"
+#include "k1-opp.dtsi"
 #include "k1-pinctrl.dtsi"
 
 / {
@@ -17,6 +18,8 @@
 		serial0 = &uart0;
 		ethernet0 = &eth0;
 		ethernet1 = &eth1;
+		i2c2 = &i2c2;
+		i2c8 = &i2c8;
 		mmc0 = &emmc;
 		mmc1 = &sdhci0;
 	};
@@ -25,6 +28,17 @@
 		stdout-path = "serial0";
 	};
 
+	leds {
+		compatible = "gpio-leds";
+
+		led-1 {
+			label = "sys-led";
+			gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "heartbeat";
+			default-state = "on";
+		};
+	};
+
 	pcie_vcc3v3: regulator-pcie-vcc3v3 {
 		compatible = "regulator-fixed";
 		enable-active-high;
@@ -63,23 +77,44 @@
 		regulator-max-microvolt = <5000000>;
 		vin-supply = <&vcc_5v0>;
 	};
-
-	leds {
-		compatible = "gpio-leds";
-
-		led1 {
-			label = "sys-led";
-			gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_LOW>;
-			linux,default-trigger = "heartbeat";
-			default-state = "on";
-		};
-	};
 };
 
 &combo_phy {
 	status = "okay";
 };
 
+&cpu_0 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_1 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_2 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_3 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_4 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_5 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_6 {
+	cpu-supply = <&buck1_0v9>;
+};
+
+&cpu_7 {
+	cpu-supply = <&buck1_0v9>;
+};
+
 &emmc {
 	bus-width = <8>;
 	mmc-hs400-1_8v;
@@ -138,8 +173,33 @@
 	};
 };
 
-&pdma {
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_0_cfg>;
 	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+		size = <256>;
+		read-only;
+		vcc-supply = <&buck3_1v8>;
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			mac-address {
+				#nvmem-cell-cells = <1>;
+			};
+
+			num-macs {
+			};
+
+			product-name {
+			};
+		};
+	};
 };
 
 &i2c8 {
@@ -162,7 +222,7 @@
 		dldoin2-supply = <&buck5>;
 
 		regulators {
-			buck1 {
+			buck1_0v9: buck1 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <1050000>;
 				regulator-ramp-delay = <5000>;
@@ -290,6 +350,10 @@
 	status = "okay";
 };
 
+&pdma {
+	status = "okay";
+};
+
 &qspi {
 	pinctrl-names = "default";
 	pinctrl-0 = <&qspi_cfg>;
@@ -335,6 +399,24 @@
 	};
 };
 
+&sdhci0 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio K1_GPIO(80) GPIO_ACTIVE_LOW>;
+	disable-wp;
+	no-mmc;
+	no-sdio;
+	pinctrl-names = "default", "uhs";
+	pinctrl-0 = <&mmc1_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg>;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	vmmc-supply = <&buck4_3v3>;
+	vqmmc-supply = <&aldo1>;
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_2_cfg>;
@@ -366,21 +448,3 @@
 		vdd-supply = <&vcc_5v0>;
 	};
 };
-
-&sdhci0 {
-	pinctrl-names = "default", "uhs";
-	pinctrl-0 = <&mmc1_cfg>;
-	pinctrl-1 = <&mmc1_uhs_cfg>;
-	bus-width = <4>;
-	cd-gpios = <&gpio K1_GPIO(80) GPIO_ACTIVE_LOW>;
-	no-mmc;
-	no-sdio;
-	disable-wp;
-	cap-sd-highspeed;
-	vmmc-supply = <&buck4_3v3>;
-	vqmmc-supply = <&aldo1>;
-	sd-uhs-sdr25;
-	sd-uhs-sdr50;
-	sd-uhs-sdr104;
-	status = "okay";
-};
diff --git a/arch/riscv/boot/dts/spacemit/k3-com260.dtsi b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi
index 37627963a64923..92dea86397798a 100644
--- a/arch/riscv/boot/dts/spacemit/k3-com260.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k3-com260.dtsi
@@ -58,6 +58,23 @@
 	};
 };
 
+&i2c6_1_cfg {
+	i2c6-1-pins {
+		power-source = <3300>;
+	};
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_1_cfg>;
+	status = "okay";
+
+	temperature-sensor@4c {
+		compatible = "sensylink,ctf2301";
+		reg = <0x4c>;
+	};
+};
+
 &i2c8 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c8_cfg>;
diff --git a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
index 6482d24be13946..cb4f4f4e33afff 100644
--- a/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k3-pinctrl.dtsi
@@ -101,6 +101,17 @@
 		};
 	};
 
+	/omit-if-no-ref/
+	i2c6_1_cfg: i2c6-1-cfg {
+		i2c6-1-pins {
+			pinmux = <K3_PADCONF(7, 6)>,	/* i2c6 scl */
+				 <K3_PADCONF(8, 6)>;	/* i2c6 sda */
+
+			bias-pull-up = <0>;
+			drive-strength = <25>;
+		};
+	};
+
 	/omit-if-no-ref/
 	i2c8_cfg: i2c8-cfg {
 		i2c8-pins {
diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi
index c3f2dce0969c0c..b9977e78e096a7 100644
--- a/arch/riscv/boot/dts/spacemit/k3.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k3.dtsi
@@ -1140,6 +1140,85 @@
 			status = "disabled";
 		};
 
+		i2s0: i2s@d4026000 {
+			compatible = "spacemit,k3-i2s";
+			reg = <0x0 0xd4026000 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S0_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S0_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S0_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S0>,
+				 <&syscon_mpmu CLK_MPMU_I2S0_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk", "bclk", "bus", "func",
+				      "sysclk_div", "c_sysclk", "c_bclk";
+			dmas = <&pdma K3_PDMA_SSPA0_TX>, <&pdma K3_PDMA_SSPA0_RX>;
+			dma-names = "tx", "rx";
+			resets = <&syscon_apbc RESET_APBC_I2S0>;
+			#sound-dai-cells = <0>;
+
+			status = "disabled";
+		};
+
+		i2s1: i2s@d4026800 {
+			compatible = "spacemit,k3-i2s";
+			reg = <0x0 0xd4026800 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S1_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S1_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S1_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S1>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk", "bclk", "bus", "func",
+				      "c_sysclk", "c_bclk";
+			dmas = <&pdma K3_PDMA_SSPA1_TX>, <&pdma K3_PDMA_SSPA1_RX>;
+			dma-names = "tx", "rx";
+			resets = <&syscon_apbc RESET_APBC_I2S1>;
+			#sound-dai-cells = <0>;
+
+			status = "disabled";
+		};
+
+		i2s2: i2s@d4027000 {
+			compatible = "spacemit,k3-i2s";
+			reg = <0x0 0xd4027000 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S2_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S2_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S2_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S2>,
+				 <&syscon_mpmu CLK_MPMU_I2S2_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk", "bclk", "bus", "func",
+				      "sysclk_div", "c_sysclk", "c_bclk";
+			dmas = <&pdma K3_PDMA_SSPA2_TX>, <&pdma K3_PDMA_SSPA2_RX>;
+			dma-names = "tx", "rx";
+			resets = <&syscon_apbc RESET_APBC_I2S2>;
+			#sound-dai-cells = <0>;
+
+			status = "disabled";
+		};
+
+		i2s3: i2s@d4027800 {
+			compatible = "spacemit,k3-i2s";
+			reg = <0x0 0xd4027800 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S3_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S3_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S3_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S3>,
+				 <&syscon_mpmu CLK_MPMU_I2S3_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk", "bclk", "bus", "func",
+				      "sysclk_div", "c_sysclk", "c_bclk";
+			dmas = <&pdma K3_PDMA_SSPA3_TX>, <&pdma K3_PDMA_SSPA3_RX>;
+			dma-names = "tx", "rx";
+			resets = <&syscon_apbc RESET_APBC_I2S3>;
+			#sound-dai-cells = <0>;
+
+			status = "disabled";
+		};
+
 		spi0: spi@d4040000 {
 			compatible = "spacemit,k3-spi", "spacemit,k1-spi";
 			reg = <0x0 0xd4040000 0x0 0x30>;
@@ -1170,6 +1249,46 @@
 			status = "disabled";
 		};
 
+		i2s4: i2s@d4041000 {
+			compatible = "spacemit,k3-i2s";
+			reg = <0x0 0xd4041000 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S4_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S4_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S4_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S4>,
+				 <&syscon_mpmu CLK_MPMU_I2S4_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk", "bclk", "bus", "func",
+				      "sysclk_div", "c_sysclk", "c_bclk";
+			dmas = <&pdma K3_PDMA_SSPA4_TX>, <&pdma K3_PDMA_SSPA4_RX>;
+			dma-names = "tx", "rx";
+			resets = <&syscon_apbc RESET_APBC_I2S4>;
+			#sound-dai-cells = <0>;
+
+			status = "disabled";
+		};
+
+		i2s5: i2s@d4041800 {
+			compatible = "spacemit,k3-i2s";
+			reg = <0x0 0xd4041800 0x0 0x30>;
+			clocks = <&syscon_mpmu CLK_MPMU_I2S5_SYSCLK>,
+				 <&syscon_apbc CLK_APBC_I2S5_BCLK>,
+				 <&syscon_apbc CLK_APBC_I2S5_BUS>,
+				 <&syscon_apbc CLK_APBC_I2S5>,
+				 <&syscon_mpmu CLK_MPMU_I2S5_SYSCLK_DIV>,
+				 <&syscon_mpmu CLK_MPMU_I2S_SYSCLK>,
+				 <&syscon_mpmu CLK_MPMU_I2S_BCLK>;
+			clock-names = "sysclk", "bclk", "bus", "func",
+				      "sysclk_div", "c_sysclk", "c_bclk";
+			dmas = <&pdma K3_PDMA_SSPA5_TX>, <&pdma K3_PDMA_SSPA5_RX>;
+			dma-names = "tx", "rx";
+			resets = <&syscon_apbc RESET_APBC_I2S5>;
+			#sound-dai-cells = <0>;
+
+			status = "disabled";
+		};
+
 		syscon_mpmu: system-controller@d4050000 {
 			compatible = "spacemit,k3-syscon-mpmu";
 			reg = <0x0 0xd4050000 0x0 0x10000>;
@@ -1248,7 +1367,7 @@
 			msi-controller;
 			riscv,guest-index-bits = <6>;
 			riscv,hart-index-bits = <4>;
-			riscv,num-guest-ids = <511>;
+			riscv,num-guest-ids = <63>;
 			riscv,num-ids = <511>;
 		};
 
@@ -1287,9 +1406,7 @@
 					      <&cpu4_intc 11>, <&cpu5_intc 11>,
 					      <&cpu6_intc 11>, <&cpu7_intc 11>;
 			msi-controller;
-			riscv,guest-index-bits = <6>;
 			riscv,hart-index-bits = <4>;
-			riscv,num-guest-ids = <511>;
 			riscv,num-ids = <511>;
 			status = "reserved";
 		};
diff --git a/arch/riscv/errata/mips/Makefile b/arch/riscv/errata/mips/Makefile
index 6278c389b801ee..137e700d9d3f8e 100644
--- a/arch/riscv/errata/mips/Makefile
+++ b/arch/riscv/errata/mips/Makefile
@@ -1,5 +1,11 @@
 ifdef CONFIG_RISCV_ALTERNATIVE_EARLY
 CFLAGS_errata.o := -mcmodel=medany
+ifdef CONFIG_FTRACE
+CFLAGS_REMOVE_errata.o = $(CC_FLAGS_FTRACE)
+endif
+ifdef CONFIG_KASAN
+KASAN_SANITIZE_errata.o := n
+endif
 endif
 
 obj-y += errata.o
diff --git a/arch/riscv/errata/mips/errata.c b/arch/riscv/errata/mips/errata.c
index 2c3dc2259e93e9..ac9a12d0a30c9d 100644
--- a/arch/riscv/errata/mips/errata.c
+++ b/arch/riscv/errata/mips/errata.c
@@ -7,12 +7,13 @@
 #include <linux/module.h>
 #include <asm/text-patching.h>
 #include <asm/alternative.h>
+#include <asm/cacheflush.h>
 #include <asm/errata_list.h>
 #include <asm/vendorid_list.h>
 #include <asm/vendor_extensions.h>
 #include <asm/vendor_extensions/mips.h>
 
-static inline bool errata_probe_pause(void)
+static inline bool errata_probe_pause(unsigned int stage)
 {
 	if (!IS_ENABLED(CONFIG_ERRATA_MIPS_P8700_PAUSE_OPCODE))
 		return false;
@@ -20,14 +21,17 @@ static inline bool errata_probe_pause(void)
 	if (!riscv_isa_vendor_extension_available(MIPS_VENDOR_ID, XMIPSEXECTL))
 		return false;
 
+	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
+		return false;
+
 	return true;
 }
 
-static u32 mips_errata_probe(void)
+static u32 mips_errata_probe(unsigned int stage)
 {
 	u32 cpu_req_errata = 0;
 
-	if (errata_probe_pause())
+	if (errata_probe_pause(stage))
 		cpu_req_errata |= BIT(ERRATA_MIPS_P8700_PAUSE_OPCODE);
 
 	return cpu_req_errata;
@@ -38,30 +42,34 @@ void mips_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
 			    unsigned int stage)
 {
 	struct alt_entry *alt;
-	u32 cpu_req_errata = mips_errata_probe();
+	u32 cpu_req_errata = mips_errata_probe(stage);
 	u32 tmp;
+	void *oldptr, *altptr;
 
 	BUILD_BUG_ON(ERRATA_MIPS_NUMBER >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE);
 
-	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
-		return;
-
 	for (alt = begin; alt < end; alt++) {
 		if (alt->vendor_id != MIPS_VENDOR_ID)
 			continue;
 
-		if (alt->patch_id >= ERRATA_MIPS_NUMBER) {
-			WARN(1, "MIPS errata id:%d not in kernel errata list\n",
-			     alt->patch_id);
+		if (alt->patch_id >= ERRATA_MIPS_NUMBER)
 			continue;
-		}
 
 		tmp = (1U << alt->patch_id);
 		if (cpu_req_errata & tmp) {
-			mutex_lock(&text_mutex);
-			patch_text_nosync(ALT_OLD_PTR(alt), ALT_ALT_PTR(alt),
-					  alt->alt_len);
-			mutex_unlock(&text_mutex);
+			oldptr = ALT_OLD_PTR(alt);
+			altptr = ALT_ALT_PTR(alt);
+
+			if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) {
+				memcpy(oldptr, altptr, alt->alt_len);
+			} else {
+				mutex_lock(&text_mutex);
+				patch_text_nosync(oldptr, altptr, alt->alt_len);
+				mutex_unlock(&text_mutex);
+			}
 		}
 	}
+
+	if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
+		local_flush_icache_all();
 }
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 6c823361be8658..621f84d1898f43 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -188,12 +188,24 @@
 #define HGATP_MODE_SHIFT	HGATP32_MODE_SHIFT
 #endif
 
-/* VSIP & HVIP relation */
+/*
+ * VSIP & HVIP relation
+ *
+ * The bit positions are same between VSIP and HVIP for interrupt
+ * numbers 13-63, where there's a shift for the SSI, STI and SEI.
+ */
 #define VSIP_TO_HVIP_SHIFT	(IRQ_VS_SOFT - IRQ_S_SOFT)
-#define VSIP_VALID_MASK		((_AC(1, UL) << IRQ_S_SOFT) | \
+#define VSIP_BIAS_MASK		((_AC(1, UL) << IRQ_S_SOFT) | \
 				 (_AC(1, UL) << IRQ_S_TIMER) | \
-				 (_AC(1, UL) << IRQ_S_EXT) | \
-				 (_AC(1, UL) << IRQ_PMU_OVF))
+				 (_AC(1, UL) << IRQ_S_EXT))
+#define VSIP_NO_BIAS_MASK	(_AC(1, UL) << IRQ_PMU_OVF)
+#define VSIP_VALID_MASK		(VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK)
+#define vsip_to_hvip(_vsip)	((((_vsip) & VSIP_BIAS_MASK) << \
+				  VSIP_TO_HVIP_SHIFT) | \
+				 ((_vsip) & VSIP_NO_BIAS_MASK))
+#define hvip_to_vsip(_hvip)	((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \
+				  VSIP_BIAS_MASK) | \
+				 ((_hvip) & VSIP_NO_BIAS_MASK))
 
 /* AIA CSR bits */
 #define TOPI_IID_SHIFT		16
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index 709a36fb432343..b4bbae55e93111 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -76,12 +76,6 @@ typedef struct page *pgtable_t;
 #define __pgd(x)	((pgd_t) { (x) })
 #define __pgprot(x)	((pgprot_t) { (x) })
 
-#ifdef CONFIG_64BIT
-#define PTE_FMT "%016lx"
-#else
-#define PTE_FMT "%08lx"
-#endif
-
 #if defined(CONFIG_64BIT) && defined(CONFIG_MMU)
 /*
  * We override this value as its generic definition uses __pa too early in
diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h
index 6e789fa58514c7..ae23182b572cdd 100644
--- a/arch/riscv/include/asm/pgtable-64.h
+++ b/arch/riscv/include/asm/pgtable-64.h
@@ -264,15 +264,6 @@ static inline unsigned long _pmd_pfn(pmd_t pmd)
 	return __page_val_to_pfn(pmd_val(pmd));
 }
 
-#define pmd_ERROR(e) \
-	pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
-
-#define pud_ERROR(e)   \
-	pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
-
-#define p4d_ERROR(e)   \
-	pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
-
 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
 {
 	if (pgtable_l4_enabled)
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 40b1ed4f3ea893..4c8fc684550311 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -556,10 +556,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 	return __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val);
 }
 
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd " PTE_FMT ".\n", __FILE__, __LINE__, pgd_val(e))
-
-
 /* Commit new configuration to MMU hardware */
 static inline void update_mmu_cache_range(struct vm_fault *vmf,
 		struct vm_area_struct *vma, unsigned long address,
diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h
index ef59e1716a2cfd..e9f9960c194772 100644
--- a/arch/riscv/include/asm/set_memory.h
+++ b/arch/riscv/include/asm/set_memory.h
@@ -40,9 +40,8 @@ static inline int set_kernel_memory(char *startp, char *endp,
 }
 #endif
 
-int set_direct_map_invalid_noflush(struct page *page);
-int set_direct_map_default_noflush(struct page *page);
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr);
+int set_direct_map_default_noflush(struct page *page, unsigned int nr);
 bool kernel_page_present(struct page *page);
 
 #endif /* __ASSEMBLER__ */
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index 0ecc67641b09d6..bed39fff1f8a78 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -15,6 +15,18 @@
 struct seq_file;
 extern unsigned long boot_cpu_hartid;
 
+enum ipi_message_type {
+	IPI_RESCHEDULE,
+	IPI_CALL_FUNC,
+	IPI_CPU_STOP,
+	IPI_CPU_CRASH_STOP,
+	IPI_IRQ_WORK,
+	IPI_TIMER,
+	IPI_CPU_BACKTRACE,
+	IPI_KGDB_ROUNDUP,
+	IPI_MAX
+};
+
 #ifdef CONFIG_SMP
 
 #include <linux/jump_label.h>
diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h
index fffe72a7720804..1f8dcae3af517e 100644
--- a/arch/riscv/include/asm/vector.h
+++ b/arch/riscv/include/asm/vector.h
@@ -137,8 +137,8 @@ static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest)
 		"csrr	%0, " __stringify(CSR_VSTART) "\n\t"
 		"csrr	%1, " __stringify(CSR_VTYPE) "\n\t"
 		"csrr	%2, " __stringify(CSR_VL) "\n\t"
-		: "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl),
-		"=r" (dest->vcsr) : :);
+		: "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl)
+		: :);
 
 	if (has_xtheadvector()) {
 		unsigned long status;
@@ -230,7 +230,7 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to,
 			"add		%1, %1, %0\n\t"
 			"vse8.v		v24, (%1)\n\t"
 			".option pop\n\t"
-			: "=&r" (vl) : "r" (datap) : "memory");
+			: "=&r" (vl), "+r" (datap) : : "memory");
 	}
 	riscv_v_disable();
 }
@@ -266,7 +266,7 @@ static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_
 			"add		%1, %1, %0\n\t"
 			"vle8.v		v24, (%1)\n\t"
 			".option pop\n\t"
-			: "=&r" (vl) : "r" (datap) : "memory");
+			: "=&r" (vl), "+r" (datap) : : "memory");
 	}
 	__vstate_csr_restore(restore_from);
 	riscv_v_disable();
diff --git a/arch/riscv/include/uapi/asm/setup.h b/arch/riscv/include/uapi/asm/setup.h
index eb4f0209c6960e..a6c1f4b0987e35 100644
--- a/arch/riscv/include/uapi/asm/setup.h
+++ b/arch/riscv/include/uapi/asm/setup.h
@@ -3,6 +3,10 @@
 #ifndef _UAPI_ASM_RISCV_SETUP_H
 #define _UAPI_ASM_RISCV_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	2048
+#endif
 
 #endif /* _UAPI_ASM_RISCV_SETUP_H */
diff --git a/arch/riscv/include/uapi/asm/vendor/mips.h b/arch/riscv/include/uapi/asm/vendor/mips.h
index e65ab268b26551..a31c23afe74fb2 100644
--- a/arch/riscv/include/uapi/asm/vendor/mips.h
+++ b/arch/riscv/include/uapi/asm/vendor/mips.h
@@ -1,3 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
 
-#define RISCV_HWPROBE_VENDOR_EXT_XMIPSEXECTL	BIT(0)
+#include <linux/const.h>
+
+#define RISCV_HWPROBE_VENDOR_EXT_XMIPSEXECTL	_BITULL(0)
diff --git a/arch/riscv/kernel/kexec_elf.c b/arch/riscv/kernel/kexec_elf.c
index 3e9a32acb8f219..ca5abbad59e541 100644
--- a/arch/riscv/kernel/kexec_elf.c
+++ b/arch/riscv/kernel/kexec_elf.c
@@ -91,7 +91,7 @@ static int elf_find_pbase(struct kimage *image, unsigned long kernel_len,
 	kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE);
 	kbuf.cma = NULL;
 	kbuf.top_down = false;
-	ret = arch_kexec_locate_mem_hole(&kbuf);
+	ret = kexec_locate_mem_hole(&kbuf);
 	if (!ret) {
 		*old_pbase = lowest_paddr;
 		*new_pbase = kbuf.mem;
diff --git a/arch/riscv/kernel/kexec_image.c b/arch/riscv/kernel/kexec_image.c
index 51dc89259f1697..963a25f5b55bc7 100644
--- a/arch/riscv/kernel/kexec_image.c
+++ b/arch/riscv/kernel/kexec_image.c
@@ -20,7 +20,7 @@ static int image_probe(const char *kernel_buf, unsigned long kernel_len)
 	const struct riscv_image_header *h = (const struct riscv_image_header *)kernel_buf;
 
 	if (!h || kernel_len < sizeof(*h))
-		return -EINVAL;
+		return -ENOEXEC;
 
 	/* According to Documentation/arch/riscv/boot-image-header.rst,
 	 * use "magic2" field to check when version >= 0.2.
@@ -28,7 +28,7 @@ static int image_probe(const char *kernel_buf, unsigned long kernel_len)
 
 	if (h->version >= RISCV_HEADER_VERSION &&
 	    memcmp(&h->magic2, RISCV_IMAGE_MAGIC2, sizeof(h->magic2)))
-		return -EINVAL;
+		return -ENOEXEC;
 
 	return 0;
 }
diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c
index 5f56be79cd062a..9c13a6f1ea2afe 100644
--- a/arch/riscv/kernel/paravirt.c
+++ b/arch/riscv/kernel/paravirt.c
@@ -116,9 +116,9 @@ int __init pv_time_init(void)
 
 	static_call_update(pv_steal_clock, pv_time_steal_clock);
 
-	static_key_slow_inc(&paravirt_steal_enabled);
+	static_branch_inc(&paravirt_steal_enabled);
 	if (steal_acc)
-		static_key_slow_inc(&paravirt_steal_rq_enabled);
+		static_branch_inc(&paravirt_steal_rq_enabled);
 
 	pr_info("Computing paravirt steal-time\n");
 
diff --git a/arch/riscv/kernel/perf_regs.c b/arch/riscv/kernel/perf_regs.c
index fd304a248de68a..1ecc8760b88b51 100644
--- a/arch/riscv/kernel/perf_regs.c
+++ b/arch/riscv/kernel/perf_regs.c
@@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 
 #define REG_RESERVED (~((1ULL << PERF_REG_RISCV_MAX) - 1))
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask || mask & REG_RESERVED)
 		return -EINVAL;
@@ -35,9 +35,3 @@ u64 perf_reg_abi(struct task_struct *task)
 #endif
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
diff --git a/arch/riscv/kernel/pi/Makefile b/arch/riscv/kernel/pi/Makefile
index bc098edac89813..4c8e52b3f17602 100644
--- a/arch/riscv/kernel/pi/Makefile
+++ b/arch/riscv/kernel/pi/Makefile
@@ -8,7 +8,7 @@ KBUILD_CFLAGS	:= $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) -fpie \
 		   -include $(srctree)/include/linux/hidden.h \
 		   -D__DISABLE_EXPORTS -ffreestanding \
 		   -fno-asynchronous-unwind-tables -fno-unwind-tables \
-		   $(call cc-option,-fno-addrsig)
+		   $(CONFIG_CC_OPT_NO_ADDRSIG)
 
 # Disable LTO
 KBUILD_CFLAGS	:= $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS))
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 7cc5a6a5c02062..55294ee6896bf2 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -13,6 +13,7 @@
 #include <linux/sched.h>
 #include <linux/sched/debug.h>
 #include <linux/sched/task_stack.h>
+#include <linux/string_choices.h>
 #include <linux/tick.h>
 #include <linux/ptrace.h>
 #include <linux/uaccess.h>
@@ -134,7 +135,7 @@ static int __init compat_mode_detect(void)
 	csr_write(CSR_STATUS, tmp);
 
 	pr_info("riscv: ELF compat mode %s",
-			compat_mode_supported ? "supported" : "unsupported");
+		str_supported_unsupported(compat_mode_supported));
 
 	return 0;
 }
diff --git a/arch/riscv/kernel/sbi-ipi.c b/arch/riscv/kernel/sbi-ipi.c
index 0cc5559c08d8ff..eeec178a9b95bb 100644
--- a/arch/riscv/kernel/sbi-ipi.c
+++ b/arch/riscv/kernel/sbi-ipi.c
@@ -57,7 +57,7 @@ void __init sbi_ipi_init(void)
 		return;
 	}
 
-	virq = ipi_mux_create(BITS_PER_BYTE, sbi_send_ipi);
+	virq = ipi_mux_create(IPI_MAX, sbi_send_ipi);
 	if (virq <= 0) {
 		pr_err("unable to create muxed IPIs\n");
 		irq_dispose_mapping(sbi_ipi_virq);
@@ -75,7 +75,7 @@ void __init sbi_ipi_init(void)
 			  "irqchip/sbi-ipi:starting",
 			  sbi_ipi_starting_cpu, NULL);
 
-	riscv_ipi_set_virq_range(virq, BITS_PER_BYTE);
+	riscv_ipi_set_virq_range(virq, IPI_MAX);
 	pr_info("providing IPIs using SBI IPI extension\n");
 
 	/*
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index fa66f9c97d748d..8930b62b15e742 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -28,18 +28,6 @@
 #include <asm/cacheflush.h>
 #include <asm/cpu_ops.h>
 
-enum ipi_message_type {
-	IPI_RESCHEDULE,
-	IPI_CALL_FUNC,
-	IPI_CPU_STOP,
-	IPI_CPU_CRASH_STOP,
-	IPI_IRQ_WORK,
-	IPI_TIMER,
-	IPI_CPU_BACKTRACE,
-	IPI_KGDB_ROUNDUP,
-	IPI_MAX
-};
-
 static const char * const ipi_names[] = {
 	[IPI_RESCHEDULE]	= "Rescheduling interrupts",
 	[IPI_CALL_FUNC]		= "Function call interrupts",
diff --git a/arch/riscv/kvm/aia_device.c b/arch/riscv/kvm/aia_device.c
index efc7c0bcfba921..97833a04268a9c 100644
--- a/arch/riscv/kvm/aia_device.c
+++ b/arch/riscv/kvm/aia_device.c
@@ -237,7 +237,7 @@ static int aia_init(struct kvm *kvm)
 		return -EBUSY;
 
 	/* We might be in the middle of creating a VCPU? */
-	if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
+	if (kvm_is_vcpu_creation_in_progress(kvm))
 		return -EBUSY;
 
 	/* Number of sources should be less than or equals number of IDs */
diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c
index c1af23e79ae063..f17dc559a70921 100644
--- a/arch/riscv/kvm/aia_imsic.c
+++ b/arch/riscv/kvm/aia_imsic.c
@@ -965,9 +965,14 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
 	if (!vcpu)
 		return -ENODEV;
 
+	if (mutex_lock_killable(&vcpu->mutex))
+		return -EINTR;
+
 	imsic = vcpu->arch.aia_context.imsic_state;
-	if (!imsic)
-		return -ENODEV;
+	if (!imsic) {
+		rc = -ENODEV;
+		goto out_unlock;
+	}
 	isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type);
 
 	read_lock_irqsave(&imsic->vsfile_lock, flags);
@@ -991,6 +996,8 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type,
 		rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix,
 				     isel, write, val);
 
+out_unlock:
+	mutex_unlock(&vcpu->mutex);
 	return rc;
 }
 
diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c
index 6035b5ec950398..371fccaf9df08a 100644
--- a/arch/riscv/kvm/mmu.c
+++ b/arch/riscv/kvm/mmu.c
@@ -627,6 +627,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 	int ret;
 	kvm_pfn_t hfn;
 	bool is_hugetlb;
+	bool unused = false;
 	bool writable;
 	unsigned int vma_pageshift;
 	gfn_t gfn = gpa >> PAGE_SHIFT;
@@ -664,7 +665,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 		return -EFAULT;
 	}
 
-	is_hugetlb = is_vm_hugetlb_page(vma);
+	is_hugetlb = vma_is_hugetlb(vma);
 	if (is_hugetlb)
 		vma_pageshift = huge_page_shift(hstate_vma(vma));
 	else
@@ -707,6 +708,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 				vma_pageshift, current);
 		return 0;
 	}
+	if (is_sigpending_pfn(hfn))
+		return -EINTR;
 	if (is_error_noslot_pfn(hfn))
 		return -EFAULT;
 
@@ -719,8 +722,10 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 
 	write_lock(&kvm->mmu_lock);
 
-	if (mmu_invalidate_retry(kvm, mmu_seq))
+	if (mmu_invalidate_retry(kvm, mmu_seq)) {
+		unused = true;
 		goto out_unlock;
+	}
 
 	/*
 	 * Check if we are backed by a THP and thus use block mapping if
@@ -743,7 +748,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 		kvm_err("Failed to map in G-stage\n");
 
 out_unlock:
-	kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable);
+	kvm_release_faultin_page(kvm, page,
+				 unused || (ret && ret != -EEXIST), writable);
 	write_unlock(&kvm->mmu_lock);
 	return ret;
 }
diff --git a/arch/riscv/kvm/nacl.c b/arch/riscv/kvm/nacl.c
index 9aff03c4f667ae..a5cda9a65156d4 100644
--- a/arch/riscv/kvm/nacl.c
+++ b/arch/riscv/kvm/nacl.c
@@ -42,12 +42,24 @@ void __kvm_riscv_nacl_hfence(void *shmem,
 		}
 	}
 
-	entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i);
-	*entp = cpu_to_lelong(control);
+	/*
+	 * Per SBI v3.0 section 15.1.2, the Page_Number and Page_Count
+	 * words must be updated before the Config word with its Pending
+	 * bit set. WRITE_ONCE() stops the compiler from reordering the
+	 * stores and smp_wmb() makes the parameter words globally
+	 * visible to the SBI implementation (or NACL hardware) before
+	 * the Pending bit is set.
+	 */
 	entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PNUM(i);
-	*entp = cpu_to_lelong(page_num);
+	WRITE_ONCE(*entp, cpu_to_lelong(page_num));
 	entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PCOUNT(i);
-	*entp = cpu_to_lelong(page_count);
+	WRITE_ONCE(*entp, cpu_to_lelong(page_count));
+
+	/* Ensure the parameter words are visible before the Pending bit */
+	smp_wmb();
+
+	entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i);
+	WRITE_ONCE(*entp, cpu_to_lelong(control));
 }
 
 int kvm_riscv_nacl_enable(void)
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index e062ca19f9d8f3..7f1636c44d413c 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -491,8 +491,7 @@ bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
 	bool ret;
 
 	raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags);
-	ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
-		<< VSIP_TO_HVIP_SHIFT) & (unsigned long)mask;
+	ie = vsip_to_hvip(vcpu->arch.guest_csr.vsie) & (unsigned long)mask;
 	ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &
 		(unsigned long)mask;
 	ret = vcpu->arch.irqs_pending[0] & ie;
diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c
index 88e0c369b3544b..8d36eb8abce1eb 100644
--- a/arch/riscv/kvm/vcpu_exit.c
+++ b/arch/riscv/kvm/vcpu_exit.c
@@ -283,7 +283,7 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 	}
 
 	/* Print details in-case of error */
-	if (ret < 0) {
+	if (ret < 0 && ret != -EINTR) {
 		kvm_err("VCPU exit error %d\n", ret);
 		kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n",
 			vcpu->arch.guest_context.sepc,
diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c
index 841f2cf8716836..f16e25098af118 100644
--- a/arch/riscv/kvm/vcpu_onereg.c
+++ b/arch/riscv/kvm/vcpu_onereg.c
@@ -272,7 +272,7 @@ static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu,
 
 	if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
 		kvm_riscv_vcpu_flush_interrupts(vcpu);
-		*out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK;
+		*out_val = hvip_to_vsip(csr->hvip);
 		*out_val |= csr->hvip & ~IRQ_LOCAL_MASK;
 	} else
 		*out_val = ((unsigned long *)csr)[reg_num];
@@ -293,10 +293,8 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu,
 
 	reg_num = array_index_nospec(reg_num, regs_max);
 
-	if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
-		reg_val &= VSIP_VALID_MASK;
-		reg_val <<= VSIP_TO_HVIP_SHIFT;
-	}
+	if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
+		reg_val = vsip_to_hvip(reg_val);
 
 	((unsigned long *)csr)[reg_num] = reg_val;
 
diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c
index 6ff741ee78036d..e095d1ff439f19 100644
--- a/arch/riscv/kvm/vcpu_pmu.c
+++ b/arch/riscv/kvm/vcpu_pmu.c
@@ -270,12 +270,13 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx,
 			return -EINVAL;
 
 		pmc->counter_val = kvpmu->fw_event[fevent_code].value;
+		*out_val = pmc->counter_val;
 	} else if (pmc->perf_event) {
-		pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running);
+		*out_val = pmc->counter_val +
+			   perf_event_read_value(pmc->perf_event, &enabled, &running);
 	} else {
 		return -EINVAL;
 	}
-	*out_val = pmc->counter_val;
 
 	return 0;
 }
@@ -454,6 +455,14 @@ int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long s
 		}
 	}
 
+	/*
+	 * Clear any previously installed snapshot area to avoid leaking
+	 * the old sdata and to keep sdata/snapshot_addr consistent if
+	 * the re-install fails below.
+	 */
+	if (kvpmu->snapshot_addr != INVALID_GPA)
+		kvm_pmu_clear_snapshot_area(vcpu);
+
 	kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC | __GFP_ACCOUNT);
 	if (!kvpmu->sdata) {
 		sbiret = SBI_ERR_FAILURE;
@@ -645,7 +654,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
 {
 	struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu);
 	int i, pmc_index, sbiret = 0;
-	u64 enabled, running;
 	struct kvm_pmc *pmc;
 	int fevent_code;
 	bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT;
@@ -675,14 +683,19 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
 				goto out;
 			}
 
-			if (!kvpmu->fw_event[fevent_code].started)
+			if (!kvpmu->fw_event[fevent_code].started) {
 				sbiret = SBI_ERR_ALREADY_STOPPED;
-
-			kvpmu->fw_event[fevent_code].started = false;
+			} else {
+				kvpmu->fw_event[fevent_code].started = false;
+				pmc->counter_val = kvpmu->fw_event[fevent_code].value;
+			}
 		} else if (pmc->perf_event) {
 			if (pmc->started) {
-				/* Stop counting the counter */
-				perf_event_disable(pmc->perf_event);
+				/*
+				 * Stop the counter and fold the live count into counter_val.
+				 * Reset the event value to avoid redundant accumulation.
+				 */
+				pmc->counter_val += perf_event_pause(pmc->perf_event, true);
 				pmc->started = false;
 			} else {
 				sbiret = SBI_ERR_ALREADY_STOPPED;
@@ -696,11 +709,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
 		}
 
 		if (snap_flag_set && !sbiret) {
-			if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW)
-				pmc->counter_val = kvpmu->fw_event[fevent_code].value;
-			else if (pmc->perf_event)
-				pmc->counter_val += perf_event_read_value(pmc->perf_event,
-									  &enabled, &running);
 			/*
 			 * The counter and overflow indices in the snapshot region are w.r.to
 			 * cbase. Modify the set bit in the counter mask instead of the pmc_index
@@ -727,9 +735,10 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base,
 		}
 	}
 
-	if (shmem_needs_update)
-		kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
-					     sizeof(struct riscv_pmu_snapshot_data));
+	if (shmem_needs_update &&
+	    kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata,
+				 sizeof(struct riscv_pmu_snapshot_data)))
+		sbiret = SBI_ERR_FAILURE;
 
 out:
 	retdata->err_val = sbiret;
diff --git a/arch/riscv/kvm/vcpu_sbi_hsm.c b/arch/riscv/kvm/vcpu_sbi_hsm.c
index f26207f84bab65..06a15629c26bbc 100644
--- a/arch/riscv/kvm/vcpu_sbi_hsm.c
+++ b/arch/riscv/kvm/vcpu_sbi_hsm.c
@@ -95,9 +95,9 @@ static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run,
 		ret = kvm_sbi_hsm_vcpu_get_status(vcpu);
 		if (ret >= 0) {
 			retdata->out_val = ret;
-			retdata->err_val = 0;
+			ret = 0;
 		}
-		return 0;
+		break;
 	case SBI_EXT_HSM_HART_SUSPEND:
 		switch (lower_32_bits(cp->a0)) {
 		case SBI_HSM_SUSPEND_RET_DEFAULT:
diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c
index ae53133c7ab035..a2cd277a40596d 100644
--- a/arch/riscv/kvm/vcpu_timer.c
+++ b/arch/riscv/kvm/vcpu_timer.c
@@ -61,10 +61,13 @@ static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h)
 
 static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t)
 {
-	if (!t->init_done || !t->next_set)
+	if (!t->init_done)
 		return -EINVAL;
 
 	hrtimer_cancel(&t->hrt);
+
+	if (!t->next_set)
+		return -EINVAL;
 	t->next_set = false;
 
 	return 0;
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index fb37b0b67efec6..857f9a55039ce7 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -22,6 +22,7 @@
 #include <linux/hugetlb.h>
 #include <linux/kfence.h>
 #include <linux/execmem.h>
+#include <linux/vmemmap-optimization.h>
 
 #include <asm/alternative.h>
 #include <asm/fixmap.h>
diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c
index 3f76db3d276992..5b3cf326455db1 100644
--- a/arch/riscv/mm/pageattr.c
+++ b/arch/riscv/mm/pageattr.c
@@ -374,33 +374,18 @@ int set_memory_nx(unsigned long addr, int numpages)
 	return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));
 }
 
-int set_direct_map_invalid_noflush(struct page *page)
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr)
 {
-	return __set_memory((unsigned long)page_address(page), 1,
+	return __set_memory((unsigned long)page_address(page), nr,
 			    __pgprot(0), __pgprot(_PAGE_PRESENT));
 }
 
-int set_direct_map_default_noflush(struct page *page)
+int set_direct_map_default_noflush(struct page *page, unsigned int nr)
 {
-	return __set_memory((unsigned long)page_address(page), 1,
+	return __set_memory((unsigned long)page_address(page), nr,
 			    PAGE_KERNEL, __pgprot(_PAGE_EXEC));
 }
 
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)
-{
-	pgprot_t set, clear;
-
-	if (valid) {
-		set = PAGE_KERNEL;
-		clear = __pgprot(_PAGE_EXEC);
-	} else {
-		set = __pgprot(0);
-		clear = __pgprot(_PAGE_PRESENT);
-	}
-
-	return __set_memory((unsigned long)page_address(page), nr, set, clear);
-}
-
 #ifdef CONFIG_DEBUG_PAGEALLOC
 static int debug_pagealloc_set_page(pte_t *pte, unsigned long addr, void *data)
 {
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 962db300a16659..a74a7d5258aa1d 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -149,7 +149,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 {
 	unsigned long stride_size;
 
-	if (!is_vm_hugetlb_page(vma)) {
+	if (!vma_is_hugetlb(vma)) {
 		stride_size = PAGE_SIZE;
 	} else {
 		stride_size = huge_page_size(hstate_vma(vma));
diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h
index 419b9d795f2a7a..5c4f53060c9188 100644
--- a/arch/riscv/net/bpf_jit.h
+++ b/arch/riscv/net/bpf_jit.h
@@ -12,6 +12,9 @@
 #include <linux/bpf.h>
 #include <linux/filter.h>
 
+#define RV_MAX_REG_ARGS		8
+#define RV_EXTRA_STK_ARGS	(RV_MAX_REG_ARGS - MAX_BPF_FUNC_REG_ARGS)
+
 /* verify runtime detection extension status */
 #define rv_ext_enabled(ext) \
 	(IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext))
@@ -82,6 +85,7 @@ struct rv_jit_context {
 	unsigned long flags;
 	int stack_size;
 	int tcc_offset;
+	int stack_arg_sz;
 	u64 arena_vm_start;
 	u64 user_vm_start;
 };
diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index 151031e97a243c..26da34387e035c 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -15,7 +15,6 @@
 #include <asm/percpu.h>
 #include "bpf_jit.h"
 
-#define RV_MAX_REG_ARGS 8
 #define RV_FENTRY_NINSNS 2
 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4)
 /* imm that allows emit_imm to emit max count insns */
@@ -23,11 +22,11 @@
 /* fentry and TCC init insns will be skipped on tailcall */
 #define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4)
 
-#define RV_REG_TCC RV_REG_A6
+#define RV_REG_TCC RV_REG_T5
 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */
 
 static const int regmap[] = {
-	[BPF_REG_0] =	RV_REG_A5,
+	[BPF_REG_0] =	RV_REG_T6,
 	[BPF_REG_1] =	RV_REG_A0,
 	[BPF_REG_2] =	RV_REG_A1,
 	[BPF_REG_3] =	RV_REG_A2,
@@ -47,13 +46,13 @@ static const int pt_regmap[] = {
 	[RV_REG_A2] = offsetof(struct pt_regs, a2),
 	[RV_REG_A3] = offsetof(struct pt_regs, a3),
 	[RV_REG_A4] = offsetof(struct pt_regs, a4),
-	[RV_REG_A5] = offsetof(struct pt_regs, a5),
 	[RV_REG_S1] = offsetof(struct pt_regs, s1),
 	[RV_REG_S2] = offsetof(struct pt_regs, s2),
 	[RV_REG_S3] = offsetof(struct pt_regs, s3),
 	[RV_REG_S4] = offsetof(struct pt_regs, s4),
 	[RV_REG_S5] = offsetof(struct pt_regs, s5),
 	[RV_REG_T0] = offsetof(struct pt_regs, t0),
+	[RV_REG_T6] = offsetof(struct pt_regs, t6),
 };
 
 enum {
@@ -239,7 +238,7 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
 	emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx);
 	/* Set return value. */
 	if (!is_tail_call)
-		emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx);
+		emit_addiw(RV_REG_A0, regmap[BPF_REG_0], 0, ctx);
 	emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
 		  is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx);
 }
@@ -498,6 +497,18 @@ static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext,
 	ctx->ex_jmp_off = ctx->ninsns;
 }
 
+static void emit_stack_arg_ldx(u8 rd, s16 off, struct rv_jit_context *ctx)
+{
+	int idx = off / 8 - 1;
+
+	if (idx < RV_EXTRA_STK_ARGS) {
+		emit_mv(rd, RV_REG_A5 + idx, ctx);
+		return;
+	}
+
+	emit_ldx_insn(rd, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_FP, BPF_DW, false, ctx);
+}
+
 static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx)
 {
 	emit_imm(RV_REG_T1, imm, ctx);
@@ -515,6 +526,19 @@ static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx
 	ctx->ex_jmp_off = ctx->ninsns;
 }
 
+static void emit_stack_arg_st(s16 off, s32 imm, struct rv_jit_context *ctx)
+{
+	int idx = -off / 8 - 1;
+
+	if (idx < RV_EXTRA_STK_ARGS) {
+		emit_imm(RV_REG_A5 + idx, imm, ctx);
+		return;
+	}
+
+	emit_imm(RV_REG_T1, imm, ctx);
+	emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_T1, BPF_DW, ctx);
+}
+
 static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
 {
 	if (is_12b_int(off)) {
@@ -531,6 +555,18 @@ static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
 	ctx->ex_jmp_off = ctx->ninsns;
 }
 
+static void emit_stack_arg_stx(s16 off, u8 rs, struct rv_jit_context *ctx)
+{
+	int idx = -off / 8 - 1;
+
+	if (idx < RV_EXTRA_STK_ARGS) {
+		emit_mv(RV_REG_A5 + idx, rs, ctx);
+		return;
+	}
+
+	emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, rs, BPF_DW, ctx);
+}
+
 static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
 			     struct rv_jit_context *ctx)
 {
@@ -747,6 +783,8 @@ static int add_exception_handler(const struct bpf_insn *insn, int dst_reg,
 	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
 	    BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
 	    BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
+	    !(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX &&
+	      BPF_CLASS(insn->code) == BPF_LDX) &&
 	    BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
 		return 0;
 
@@ -822,6 +860,24 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
 	return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx);
 }
 
+int arch_bpf_trampoline_skip(void *nop, void *target)
+{
+	u32 old_insn = rv_nop();
+	u32 new_insn = rv_jal(RV_REG_ZERO, (target - nop) >> 1);
+	int ret;
+
+	if (memcmp(nop, &old_insn, sizeof(old_insn)))
+		return -EFAULT;
+
+	cpus_read_lock();
+	mutex_lock(&text_mutex);
+	ret = patch_text(nop, &new_insn, sizeof(new_insn));
+	mutex_unlock(&text_mutex);
+	cpus_read_unlock();
+
+	return ret;
+}
+
 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
 		       enum bpf_text_poke_type new_t, void *old_addr,
 		       void *new_addr)
@@ -904,12 +960,17 @@ static void emit_store_stack_imm64(u8 reg, int stack_off, u64 imm64,
 	emit_sd(RV_REG_FP, stack_off, reg, ctx);
 }
 
-static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval_off,
-			   int run_ctx_off, bool save_ret, struct rv_jit_context *ctx)
+static int invoke_bpf_prog(struct bpf_tramp_image *im, struct bpf_tramp_node *node,
+			   int args_off, int retval_off, int run_ctx_off, bool save_ret,
+			   struct rv_jit_context *ctx)
 {
 	int ret, branch_off;
 	struct bpf_prog *p = node->link->prog;
 	int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
+	void *skip = ctx->ro_insns + ctx->ninsns;
+
+	/* nop, patched to a jal over this prog when it is detached */
+	emit(rv_nop(), ctx);
 
 	if (node->cookie)
 		emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, node->cookie, ctx);
@@ -962,13 +1023,17 @@ static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval
 	/* arg3: &run_ctx */
 	emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx);
 	ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx);
+	if (ret)
+		return ret;
 
-	return ret;
+	bpf_tramp_image_add_skip(im, p, skip, ctx->ro_insns + ctx->ninsns);
+	return 0;
 }
 
-static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off,
-		      int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta,
-		      int cookie_off, struct rv_jit_context *ctx)
+static int invoke_bpf(struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn,
+		      int args_off, int retval_off, int run_ctx_off, int func_meta_off,
+		      bool save_ret, u64 func_meta, int cookie_off,
+		      struct rv_jit_context *ctx)
 {
 	int i, cur_cookie = (cookie_off - args_off) / 8;
 
@@ -981,8 +1046,8 @@ static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off,
 			emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx);
 			cur_cookie--;
 		}
-		err = invoke_bpf_prog(tn->nodes[i], args_off, retval_off, run_ctx_off,
-				      save_ret, ctx);
+		err = invoke_bpf_prog(im, tn->nodes[i], args_off, retval_off,
+				      run_ctx_off, save_ret, ctx);
 		if (err)
 			return err;
 	}
@@ -1073,7 +1138,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 
 	save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
 	if (save_ret)
-		stack_size += 16; /* Save both A5 (BPF R0) and A0 */
+		stack_size += 16; /* Save both RV_REG_T6 (BPF R0) and RV_REG_A0 */
 	retval_off = stack_size;
 
 	stack_size += nr_arg_slots * 8;
@@ -1170,7 +1235,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 	}
 
 	if (fentry->nr_nodes) {
-		ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off,
+		ret = invoke_bpf(im, fentry, args_off, retval_off, run_ctx_off, func_meta_off,
 				 flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx);
 		if (ret)
 			return ret;
@@ -1184,7 +1249,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 		/* cleanup to avoid garbage return value confusion */
 		emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx);
 		for (i = 0; i < fmod_ret->nr_nodes; i++) {
-			ret = invoke_bpf_prog(fmod_ret->nodes[i], args_off, retval_off,
+			ret = invoke_bpf_prog(im, fmod_ret->nodes[i], args_off, retval_off,
 					      run_ctx_off, true, ctx);
 			if (ret)
 				goto out;
@@ -1211,10 +1276,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 			emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx);
 		emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx);
 		emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx);
-		im->ip_after_call = ctx->ro_insns + ctx->ninsns;
-		/* 2 nops reserved for auipc+jalr pair */
-		emit(rv_nop(), ctx);
-		emit(rv_nop(), ctx);
 	}
 
 	/* update branches saved in invoke_bpf_mod_ret with bnez */
@@ -1230,14 +1291,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 		emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx);
 
 	if (fexit->nr_nodes) {
-		ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off,
+		ret = invoke_bpf(im, fexit, args_off, retval_off, run_ctx_off, func_meta_off,
 				 false, func_meta, cookie_off, ctx);
 		if (ret)
 			goto out;
 	}
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
-		im->ip_epilogue = ctx->ro_insns + ctx->ninsns;
 		emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx);
 		ret = emit_call((const u64)__bpf_tramp_exit, true, ctx);
 		if (ret)
@@ -1299,7 +1359,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 			     struct bpf_tramp_nodes *tnodes, void *func_addr)
 {
-	struct bpf_tramp_image im;
+	struct bpf_tramp_image im = {};
 	struct rv_jit_context ctx;
 	int ret;
 
@@ -1822,11 +1882,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 				return -EINVAL;
 
 			for (idx = 0; idx < fm->nr_args; idx++) {
-				u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx);
 				bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG;
-
-				if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx))
-					return -EINVAL;
+				u8 arg_sz = fm->arg_size[idx];
+
+				if (arg_sz == 8 || (arg_sz != 4 && !sign))
+					continue;
+
+				if (idx < RV_MAX_REG_ARGS) {
+					if (sign_extend(RV_REG_A0 + idx, RV_REG_A0 + idx, arg_sz, sign, ctx))
+						return -EINVAL;
+				} else {
+					emit_ld(RV_REG_T1, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_SP, ctx);
+					if (sign_extend(RV_REG_T1, RV_REG_T1, arg_sz, sign, ctx))
+						return -EINVAL;
+					emit_sd(RV_REG_SP, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_T1, ctx);
+				}
 			}
 		}
 
@@ -1908,18 +1978,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 	case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
 	case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
 	case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
+	/* LDX | PROBE_MEM32SX: dst = *(signed size *)(src + RV_REG_ARENA + off) */
+	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
+	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
+	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
 	{
 		bool sign_ext;
 
 		sign_ext = BPF_MODE(insn->code) == BPF_MEMSX ||
-			   BPF_MODE(insn->code) == BPF_PROBE_MEMSX;
+			   BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
+			   BPF_MODE(insn->code) == BPF_PROBE_MEM32SX;
 
-		if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
+		if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
+		    BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) {
 			emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx);
 			rs = RV_REG_T2;
 		}
 
-		emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx);
+		if (is_stack_arg_ldx(insn))
+			emit_stack_arg_ldx(rd, off, ctx);
+		else
+			emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx);
 
 		ret = add_exception_handler(insn, rd, ctx);
 		if (ret)
@@ -1949,7 +2028,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 			rd = RV_REG_T3;
 		}
 
-		emit_st(rd, off, imm, BPF_SIZE(code), ctx);
+		if (is_stack_arg_st(insn))
+			emit_stack_arg_st(off, imm, ctx);
+		else
+			emit_st(rd, off, imm, BPF_SIZE(code), ctx);
 
 		ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
 		if (ret)
@@ -1971,7 +2053,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 			rd = RV_REG_T2;
 		}
 
-		emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
+		if (is_stack_arg_stx(insn))
+			emit_stack_arg_stx(off, rs, ctx);
+		else
+			emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
 
 		ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
 		if (ret)
@@ -2021,9 +2106,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 
 void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
 {
-	int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
+	int i, stack_adjust = 0, store_offset, bpf_stack_adjust = ctx->stack_arg_sz;
 
-	bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
+	bpf_stack_adjust += round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
 	if (bpf_stack_adjust)
 		mark_fp(ctx);
 
@@ -2121,6 +2206,11 @@ bool bpf_jit_supports_kfunc_call(void)
 	return true;
 }
 
+bool bpf_jit_supports_kfunc_ret_reg_pair(void)
+{
+	return true;
+}
+
 bool bpf_jit_supports_ptr_xchg(void)
 {
 	return true;
@@ -2148,10 +2238,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
 			if (insn->imm == BPF_CMPXCHG)
 				return rv_ext_enabled(ZACAS);
 			break;
-		case BPF_LDX | BPF_MEMSX | BPF_B:
-		case BPF_LDX | BPF_MEMSX | BPF_H:
-		case BPF_LDX | BPF_MEMSX | BPF_W:
-			return false;
 		}
 	}
 
@@ -2189,3 +2275,8 @@ bool bpf_jit_supports_timed_may_goto(void)
 {
 	return true;
 }
+
+bool bpf_jit_supports_stack_args(void)
+{
+	return true;
+}
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index cbfcd287ea1668..470a6ace566208 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -48,6 +48,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	int pass = 0, prev_ninsns = 0, i;
 	struct rv_jit_data *jit_data;
 	struct rv_jit_context *ctx;
+	u16 stack_arg_cnt;
 
 	if (!prog->jit_requested)
 		return prog;
@@ -71,6 +72,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 
 	ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
 	ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
+
+	stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog);
+	/* First 3 stack args in regs, rest on stack */
+	ctx->stack_arg_sz = stack_arg_cnt > RV_EXTRA_STK_ARGS ?
+			    round_up((stack_arg_cnt - RV_EXTRA_STK_ARGS) * 8, STACK_ALIGN) : 0;
+
 	ctx->prog = prog;
 	ctx->offset = kvzalloc_objs(int, prog->len);
 	if (!ctx->offset)
@@ -237,6 +244,7 @@ void bpf_jit_free(struct bpf_prog *prog)
 			kvfree(jit_data->ctx.offset);
 			kfree(jit_data);
 		}
+		prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset();
 		hdr = bpf_jit_binary_pack_hdr(prog);
 		bpf_jit_binary_pack_free(hdr, NULL);
 		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S
index 02c637d874205a..fcf850f244c903 100644
--- a/arch/riscv/net/bpf_timed_may_goto.S
+++ b/arch/riscv/net/bpf_timed_may_goto.S
@@ -10,7 +10,7 @@
  *	- output: updated count in BPF_REG_AX (t0)
  *
  * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where
- * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved
+ * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved
  * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving.
  */
 
@@ -20,8 +20,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
 	REG_S	s0, 6*SZREG(sp)
 	addi	s0, sp, 8*SZREG
 
-	/* Save BPF registers R0-R5 (a5, a0-a4) */
-	REG_S	a5, 5*SZREG(sp)
+	/* Save BPF registers R0-R5 (t6, a0-a4) */
+	REG_S	t6, 5*SZREG(sp)
 	REG_S	a0, 4*SZREG(sp)
 	REG_S	a1, 3*SZREG(sp)
 	REG_S	a2, 2*SZREG(sp)
@@ -38,7 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
 	REG_L	a2, 2*SZREG(sp)
 	REG_L	a1, 3*SZREG(sp)
 	REG_L	a0, 4*SZREG(sp)
-	REG_L	a5, 5*SZREG(sp)
+	REG_L	t6, 5*SZREG(sp)
 
 	REG_L	s0, 6*SZREG(sp)
 	REG_L	ra, 7*SZREG(sp)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 4b51bc6e8948d7..2b2b51224d5de3 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -92,7 +92,7 @@ config S390
 	select ARCH_HAS_CURRENT_STACK_POINTER
 	select ARCH_HAS_DEBUG_VIRTUAL
 	select ARCH_HAS_DEBUG_VM_PGTABLE
-	select ARCH_HAS_DEBUG_WX
+	select ARCH_HAS_CHECK_WX
 	select ARCH_HAS_DEVMEM_IS_ALLOWED
 	select ARCH_HAS_DMA_OPS if PCI
 	select ARCH_HAS_ELF_RANDOMIZE
@@ -156,7 +156,6 @@ config S390
 	select ARCH_HAS_PTE_PROTNONE
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
-	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_USES_CFI_GENERIC_LLVM_PASS if CC_IS_CLANG
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_CMPXCHG_LOCKREF
@@ -246,7 +245,6 @@ config S390
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
-	select HAVE_PREEMPT_DYNAMIC_KEY
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RELIABLE_STACKTRACE
 	select HAVE_RETHOOK
@@ -268,7 +266,6 @@ config S390
 	select LOCK_MM_AND_FIND_VMA
 	select MMU_GATHER_MERGE_VMAS
 	select MMU_GATHER_NO_GATHER
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MODULES_USE_ELF_RELA
 	select NEED_DMA_MAP_STATE	if PCI
 	select NEED_PER_CPU_EMBED_FIRST_CHUNK
@@ -521,14 +518,6 @@ endchoice
 config 64BIT
 	def_bool y
 
-config COMMAND_LINE_SIZE
-	int "Maximum size of kernel command line"
-	default 4096
-	range 896 1048576
-	help
-	  This allows you to specify the maximum length of the kernel command
-	  line.
-
 config SMP
 	def_bool y
 
diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c
index c1b43e5e688a06..637cb7b6343f95 100644
--- a/arch/s390/boot/ipl_parm.c
+++ b/arch/s390/boot/ipl_parm.c
@@ -172,12 +172,23 @@ static inline int has_ebcdic_char(const char *str)
 	return 0;
 }
 
+static inline bool has_nonprintable_ebcdic_char(const char *str)
+{
+	for (; *str; str++)
+		if (!isprint_ebc_asc(*str) && !isspace_ebc_asc(*str))
+			return true;
+	return false;
+}
+
 void setup_boot_command_line(void)
 {
 	parmarea.command_line[COMMAND_LINE_SIZE - 1] = 0;
 	/* convert arch command line to ascii if necessary */
-	if (has_ebcdic_char(parmarea.command_line))
+	if (has_ebcdic_char(parmarea.command_line)) {
+		if (has_nonprintable_ebcdic_char(parmarea.command_line))
+			boot_warn("Kernel command line was treated as EBCDIC, but contains non-printable characters\n");
 		EBCASC(parmarea.command_line, COMMAND_LINE_SIZE);
+	}
 	/* copy arch command line */
 	strscpy(early_command_line, strim(parmarea.command_line));
 
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 3dae7147433305..68d53c0bc8dbe5 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -841,7 +841,7 @@ CONFIG_DEBUG_PAGEALLOC=y
 CONFIG_SLUB_DEBUG_ON=y
 CONFIG_PAGE_OWNER=y
 CONFIG_DEBUG_RODATA_TEST=y
-CONFIG_DEBUG_WX=y
+CONFIG_CHECK_WX=y
 CONFIG_PTDUMP_DEBUGFS=y
 CONFIG_DEBUG_OBJECTS=y
 CONFIG_DEBUG_OBJECTS_SELFTEST=y
diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig
index 6f5722634b4d39..8e5cfc69512119 100644
--- a/arch/s390/configs/defconfig
+++ b/arch/s390/configs/defconfig
@@ -820,7 +820,7 @@ CONFIG_DEBUG_INFO_DWARF4=y
 CONFIG_GDB_SCRIPTS=y
 CONFIG_DEBUG_SECTION_MISMATCH=y
 CONFIG_MAGIC_SYSRQ=y
-CONFIG_DEBUG_WX=y
+CONFIG_CHECK_WX=y
 CONFIG_PTDUMP_DEBUGFS=y
 CONFIG_DEBUG_MEMORY_INIT=y
 CONFIG_PANIC_ON_OOPS=y
diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c
index f8cd09f341d4b0..445fa7bbd95860 100644
--- a/arch/s390/crypto/hmac_s390.c
+++ b/arch/s390/crypto/hmac_s390.c
@@ -150,7 +150,10 @@ static int hash_data(const u8 *in, unsigned int inlen,
 
 #undef PARAM_INIT
 
-	cpacf_klmd(func, &param, in, inlen);
+	if (final)
+		cpacf_klmd(func, &param, in, inlen);
+	else
+		cpacf_kimd(func, &param, in, inlen);
 
 	memcpy(digest, &param, digestsize);
 
diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h
index 39d484c597748c..ad438d6352c8e6 100644
--- a/arch/s390/include/asm/debug.h
+++ b/arch/s390/include/asm/debug.h
@@ -460,7 +460,11 @@ static int VNAME(var, active_entries)[EARLY_AREAS] __initdata
 #define __REGISTER_STATIC_DEBUG_INFO(var, name, pages, areas, view)	\
 static int __init VNAME(var, reg)(void)					\
 {									\
-	debug_register_static(&var, (pages), (areas));			\
+	int rc;								\
+									\
+	rc = debug_register_static(&var, (pages), (areas));		\
+	if (rc)								\
+		return rc;						\
 	debug_register_view(&var, (view));				\
 	return 0;							\
 }									\
@@ -493,7 +497,7 @@ static debug_info_t __refdata var =					\
 static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \
 __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view)
 
-void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas);
+int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas);
 
 #endif /* MODULE */
 
diff --git a/arch/s390/include/asm/ebcdic.h b/arch/s390/include/asm/ebcdic.h
index 7164cb6584355b..78d2e37e985d35 100644
--- a/arch/s390/include/asm/ebcdic.h
+++ b/arch/s390/include/asm/ebcdic.h
@@ -10,6 +10,7 @@
 #ifndef _EBCDIC_H
 #define _EBCDIC_H
 
+#include <linux/ctype.h>
 #include <linux/types.h>
 
 extern __u8 _ascebc_500[256];   /* ASCII -> EBCDIC 500 conversion table */
@@ -18,6 +19,11 @@ extern __u8 _ascebc[256];   /* ASCII -> EBCDIC conversion table */
 extern __u8 _ebcasc[256];   /* EBCDIC -> ASCII conversion table */
 extern __u8 _ebc_tolower[256]; /* EBCDIC -> lowercase */
 extern __u8 _ebc_toupper[256]; /* EBCDIC -> uppercase */
+extern const __u8 _ebctype_asc[256]; /* EBCDIC -> character classes */
+
+#define __ismask_ebc_asc(x)	(_ebctype_asc[(int)(unsigned char)(x)])
+#define isprint_ebc_asc(c)	((__ismask_ebc_asc(c)&(_P|_U|_L|_D|_SP)) != 0)
+#define isspace_ebc_asc(c)	((__ismask_ebc_asc(c)&(_S)) != 0)
 
 static inline void
 codepage_convert(const __u8 *codepage, volatile char *addr, unsigned long nr)
diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h
index 4ef47265cfce84..f6f79ec4bee3bc 100644
--- a/arch/s390/include/asm/entry-percpu.h
+++ b/arch/s390/include/asm/entry-percpu.h
@@ -2,79 +2,42 @@
 #ifndef ARCH_S390_ENTRY_PERCPU_H
 #define ARCH_S390_ENTRY_PERCPU_H
 
-#include <linux/kprobes.h>
+#include <linux/bitfield.h>
 #include <linux/percpu.h>
 #include <asm/lowcore.h>
 #include <asm/ptrace.h>
-#include <asm/asm-offsets.h>
 
 static __always_inline void percpu_entry(struct pt_regs *regs)
 {
 	struct lowcore *lc = get_lowcore();
 
-	if (user_mode(regs))
-		return;
 	regs->cpu = lc->cpu_nr;
 	regs->percpu_register = lc->percpu_register;
 	lc->percpu_register = 0;
 }
 
-static __always_inline bool percpu_code_check(struct pt_regs *regs)
-{
-	unsigned int insn, disp;
-	struct kprobe *p;
-
-	if (likely(user_mode(regs) || !regs->percpu_register))
-		return false;
-	/*
-	 * Within a percpu code section - check if the percpu base register
-	 * needs to be updated. This is the case if the PSW does not point to
-	 * the ADD instruction within the section.
-	 * - AG %rx,percpu_offset_in_lowcore(%r0,%r0)
-	 * which adds the percpu offset to the percpu base register.
-	 */
-	lockdep_assert_preemption_disabled();
-again:
-	insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia);
-	if (unlikely(insn == BREAKPOINT_INSTRUCTION)) {
-		p = get_kprobe((void *)psw_bits(regs->psw).ia);
-		/*
-		 * If the kprobe is concurrently removed on a different CPU
-		 * it might not be found anymore. However text must have
-		 * been restored - try again.
-		 */
-		if (!p)
-			goto again;
-		insn = p->opcode;
-	}
-	if ((insn & 0xff0f) != 0xe300)
-		return true;
-	disp = offsetof(struct lowcore, percpu_offset);
-	if (machine_has_relocated_lowcore())
-		disp += LOWCORE_ALT_ADDRESS;
-	insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8;
-	if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn)
-		return true;
-	return false;
-}
-
-static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup)
+static __always_inline void percpu_exit(struct pt_regs *regs)
 {
+	unsigned char regval, regpcp, regoff, regptr;
 	struct lowcore *lc = get_lowcore();
-	unsigned char reg;
 
-	if (user_mode(regs))
-		return;
-	reg = regs->percpu_register;
-	lc->percpu_register = reg;
-	if (likely(!needs_fixup))
+	if (!regs->percpu_register)
 		return;
-	/* Check if process has been migrated to a different CPU. */
+	regval = regs->percpu_register;
+	lc->percpu_register = regval;
+	/* Migrated to a different CPU? */
 	if (regs->cpu == lc->cpu_nr)
 		return;
-	/* Fixup percpu base register */
-	regs->gprs[reg] -= __per_cpu_offset[regs->cpu];
-	regs->gprs[reg] += lc->percpu_offset;
+	regpcp = FIELD_GET(PCPU_REG_PCP, regval);
+	regoff = FIELD_GET(PCPU_REG_OFF, regval);
+	regptr = regoff + 1;
+	/*
+	 * Update register 'regoff' with the current CPU's percpu offset,
+	 * and recalculate and update current CPU's percpu variable address
+	 * contained in register 'regptr'.
+	 */
+	regs->gprs[regoff] = lc->percpu_offset;
+	regs->gprs[regptr] = regs->gprs[regpcp] + regs->gprs[regoff];
 }
 
 #endif
diff --git a/arch/s390/include/asm/idals.h b/arch/s390/include/asm/idals.h
index 06e1ec2afd5afd..213f430a8d33a2 100644
--- a/arch/s390/include/asm/idals.h
+++ b/arch/s390/include/asm/idals.h
@@ -18,6 +18,7 @@
 #include <linux/types.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
+#include <linux/kmemleak.h>
 #include <asm/dma-types.h>
 #include <asm/cio.h>
 
@@ -147,9 +148,11 @@ static inline struct idal_buffer *idal_buffer_alloc(size_t size, int page_order)
 			ib->data[i] = dma64_add(ib->data[i - 1], IDA_BLOCK_SIZE);
 			continue;
 		}
-		vaddr = (void *)__get_free_pages(GFP_KERNEL, page_order);
+		vaddr = kmalloc(PAGE_SIZE << page_order, GFP_KERNEL);
 		if (!vaddr)
 			goto error;
+		/* Only DMA addresses are retained in ib->data. */
+		kmemleak_ignore(vaddr);
 		ib->data[i] = virt_to_dma64(vaddr);
 	}
 	return ib;
@@ -157,7 +160,7 @@ static inline struct idal_buffer *idal_buffer_alloc(size_t size, int page_order)
 	while (i >= nr_chunks) {
 		i -= nr_chunks;
 		vaddr = dma64_to_virt(ib->data[i]);
-		free_pages((unsigned long)vaddr, ib->page_order);
+		kfree(vaddr);
 	}
 	kfree(ib);
 	return ERR_PTR(-ENOMEM);
@@ -175,7 +178,7 @@ static inline void idal_buffer_free(struct idal_buffer *ib)
 	nr_chunks = (PAGE_SIZE << ib->page_order) >> IDA_SIZE_SHIFT;
 	for (i = 0; i < nr_ptrs; i += nr_chunks) {
 		vaddr = dma64_to_virt(ib->data[i]);
-		free_pages((unsigned long)vaddr, ib->page_order);
+		kfree(vaddr);
 	}
 	kfree(ib);
 }
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 5cef215d30e7da..f4517bbe405be9 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -21,7 +21,7 @@
 #define LC_ORDER 1
 #define LC_PAGES 2
 
-#define LOWCORE_ALT_ADDRESS	_AC(0x70000, UL)
+#define LOWCORE_ALT_ADDRESS	0x70000
 
 #ifndef __ASSEMBLER__
 
@@ -222,6 +222,9 @@ struct lowcore {
 	__u8	pad_0x1900[0x2000-0x1900];	/* 0x1900 */
 } __packed __aligned(8192);
 
+#define LC_PERCPU_OFFSET	offsetof(struct lowcore, percpu_offset)
+#define LC_PERCPU_REGISTER	offsetof(struct lowcore, percpu_register)
+
 static __always_inline struct lowcore *get_lowcore(void)
 {
 	struct lowcore *lc;
diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h
index feb58acf87d69f..c3995bd8d1de81 100644
--- a/arch/s390/include/asm/percpu.h
+++ b/arch/s390/include/asm/percpu.h
@@ -14,17 +14,16 @@
 
 #define arch_raw_cpu_ptr(_ptr)						\
 ({									\
-	unsigned long lc_percpu, tcp_ptr__;				\
+	unsigned long tcp_ptr__;					\
 									\
 	tcp_ptr__ = (__force unsigned long)(_ptr);			\
-	lc_percpu = offsetof(struct lowcore, percpu_offset);		\
 	asm_inline volatile(						\
 	ALTERNATIVE("ag		%[__ptr__],%[offzero](%%r0)\n",		\
 		    "ag		%[__ptr__],%[offalt](%%r0)\n",		\
 		    ALT_FEATURE(MFEATURE_LOWCORE))			\
 	: [__ptr__] "+d" (tcp_ptr__)					\
-	: [offzero] "i" (lc_percpu),					\
-	  [offalt] "i" (lc_percpu + LOWCORE_ALT_ADDRESS),		\
+	: [offzero] "i" (LC_PERCPU_OFFSET),				\
+	  [offalt] "i" (LC_PERCPU_OFFSET + LOWCORE_ALT_ADDRESS),	\
 	  "m" (((struct lowcore *)0)->percpu_offset)			\
 	: "cc");							\
 	(TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__;		\
@@ -63,65 +62,114 @@
 /*
  * Macros to be used for percpu code section based on atomic instructions.
  *
- * Avoid the need to use preempt_disable() / preempt_disable() pairs and the
+ * Avoid the need to use preempt_disable() / preempt_enable() pairs and the
  * conditional preempt_schedule_notrace() function calls which come with
  * this. The idea is that this_cpu operations based on atomic instructions are
  * guarded with mviy instructions:
  *
- * - The first mviy instruction writes the register number, which contains the
- *   percpu address variable to lowcore. This also indicates that a percpu
- *   code section is executed.
+ * - The first mviy instruction writes the register number of the percpu address
+ *   variable and the even register number of a register pair (which encodes two
+ *   registers: the even register for the percpu offset and the odd register for
+ *   the percpu pointer) to lowcore. This also indicates that a percpu code
+ *   section is executed.
  *
- * - The first mviy instruction following the mviy instruction must be the ag
- *   instruction which adds the percpu offset to the percpu address register.
+ * - The mviy instruction is followed by the lg instruction which loads the
+ *   percpu offset into the even register of the pair and the la instruction
+ *   which adds the percpu offset and the percpu address into the odd register
+ *   of the pair (the percpu pointer register).
  *
  * - Afterwards the atomic percpu operation follows.
  *
  * - Then a second mviy instruction writes a zero to lowcore, which indicates
  *   the end of the percpu code section.
  *
- * - In case of an interrupt/exception/nmi the register number which was
- *   written to lowcore is copied to the exception frame (pt_regs), and a zero
- *   is written to lowcore.
+ * - In case of an interrupt/exception/nmi the encoded register numbers which
+ *   were written to lowcore are copied to the exception frame (pt_regs), and a
+ *   zero is written to lowcore.
  *
  * - On return to the previous context it is checked if a percpu code section
- *   was executed (saved register number not zero), and if the process was
- *   migrated to a different cpu. If the percpu offset was already added to
- *   the percpu address register (instruction address does _not_ point to the
- *   ag instruction) the content of the percpu address register is adjusted so
- *   it points to percpu variable of the new cpu.
+ *   was executed (saved register value not zero), and if the process was
+ *   migrated to a different cpu. The content of the percpu offset register
+ *   (even register of the pair) is reloaded with the current cpu's percpu
+ *   offset and the percpu pointer register (odd register of the pair) is
+ *   recalculated so it points to the percpu variable of the new cpu.
  *
  * Inline assemblies making use of this typically have a code sequence like:
  *
- *   MVIY_PERCPU(...) <- start of percpu code section
- *   AG_ALT(...)      <- add percpu offset; must be the second instruction
- *   atomic_op	      <- atomic op
- *   MVIY_ALT(...)    <- end of percpu code section
+ *   __PCPU_BEGIN(...) <- start of percpu code section
+ *   atomic_op	       <- atomic op
+ *   __PCPU_END(...)   <- end of percpu code section
  */
 
-#define MVIY_PERCPU(disp, dispalt, reg)						\
-	".macro GEN_MVIY disp reg\n"						\
-	".irp	rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n"			\
-	"	.ifc \\reg,%%r\\rs\n"						\
-	"	mviy	\\disp(%%r0),\\rs\n"					\
+#define LC_ALT_ADDR	__stringify(LOWCORE_ALT_ADDRESS)
+
+#define DEFINE_GR_NUM								\
+	".macro _GR_NUM opd, gr\n"						\
+	".set	\\opd,255\n"							\
+	".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n"			\
+	"	.ifc \\gr,%%r\\rs\n"						\
+	"		\\opd = \\rs\n"						\
 	"	.endif\n"							\
 	".endr\n"								\
-	".endm\n"								\
-	ALTERNATIVE("GEN_MVIY " disp    ", " reg "\n",				\
-		    "GEN_MVIY " dispalt ", " reg "\n",				\
-		    ALT_FEATURE(MFEATURE_LOWCORE))				\
-	".purgem GEN_MVIY\n"
-
-#define MVIY_ALT(disp, dispalt)							\
-	ALTERNATIVE("	mviy	" disp	  "(%%r0),0\n",				\
-		    "	mviy	" dispalt "(%%r0),0\n",				\
+	".if \\opd == 255\n"							\
+	"	.error \"Illegal register number\"\n"				\
+	".endif\n"								\
+	".endm\n"
+
+#define UNDEF_GR_NUM								\
+	".purgem _GR_NUM\n"
+
+#define PCPU_REG_PCP_SHIFT		0
+#define PCPU_REG_PCP			GENMASK(3, 0)
+#define PCPU_REG_OFF_SHIFT		4
+#define PCPU_REG_OFF			GENMASK(7, 4)
+
+#define __PCPU_MVIY(lcreg, imm)							\
+	ALTERNATIVE("	mviy	" lcreg			"(%%r0)," imm "\n",	\
+		    "	mviy	" lcreg "+" LC_ALT_ADDR "(%%r0)," imm "\n",	\
 		    ALT_FEATURE(MFEATURE_LOWCORE))
 
-#define AG_ALT(disp, dispalt, reg)						\
-	ALTERNATIVE("	ag	" reg ", " disp	   "(%%r0)\n",			\
-		    "	ag	" reg ", " dispalt "(%%r0)\n",			\
+#define __PCPU_LG(regoff, lcoff)						\
+	ALTERNATIVE("	lg	" regoff ", " lcoff		    "(%%r0)\n",	\
+		    "	lg	" regoff ", " lcoff "+" LC_ALT_ADDR "(%%r0)\n",	\
 		    ALT_FEATURE(MFEATURE_LOWCORE))
 
+#define __PCPU_LA(regptr, regoff, regpcp)					\
+	"	la	" regptr ",0(" regoff "," regpcp ")\n"
+
+#define __PCPU_CALC_REGVAL(regpcp, regoff)					\
+	"(" regpcp " << " __stringify(PCPU_REG_PCP_SHIFT) ") |"			\
+	"(" regoff " << " __stringify(PCPU_REG_OFF_SHIFT) ")"
+
+#define __PCPU_CHECK_REGS(regpcp, regoff, regptr)				\
+	".if " regoff " & 1\n"							\
+	"	.error \"Percpu offset register must be even\"\n"		\
+	".endif\n"								\
+	".if (" regpcp " == 0) || (" regoff " == 0)\n"				\
+	"	.error \"Percpu address and offset register must be non-zero\"\n"\
+	".endif\n"								\
+	".if " regptr " != " regoff " + 1\n"					\
+	"	.error \"Percpu pointer and offset register must be a pair\"\n" \
+	".endif\n"								\
+	".if (" regpcp " == " regoff ") || (" regpcp " == " regptr ")\n"	\
+	"	.error \"Percpu registers must be distinct\"\n"			\
+	".endif\n"
+
+#define __PCPU_BEGIN(lcreg, lcoff, regpcp, regoff, regptr)			\
+	DEFINE_GR_NUM								\
+	"_GR_NUM .Lregpcp, " regpcp "\n"					\
+	"_GR_NUM .Lregoff, " regoff "\n"					\
+	"_GR_NUM .Lregptr, " regptr "\n"					\
+	UNDEF_GR_NUM								\
+	__PCPU_CHECK_REGS(".Lregpcp", ".Lregoff", ".Lregptr")			\
+	".set .Lregval, " __PCPU_CALC_REGVAL(".Lregpcp", ".Lregoff") "\n"	\
+	__PCPU_MVIY(lcreg, ".Lregval")						\
+	__PCPU_LG(regoff, lcoff)						\
+	__PCPU_LA(regptr, regoff, regpcp)
+
+#define __PCPU_END(lcreg)							\
+	__PCPU_MVIY(lcreg, "0")
+
 #ifndef MARCH_HAS_Z196_FEATURES
 
 #define this_cpu_add_4(pcp, val)	arch_this_cpu_to_op_simple(pcp, val, +)
@@ -137,44 +185,38 @@
 
 #define arch_this_cpu_add(pcp, val, op1, op2, szcast)				\
 do {										\
-	unsigned long lc_pcpr, lc_pcpo;						\
 	typedef typeof(pcp) pcp_op_T__;						\
+	union register_pair rp__;						\
 	pcp_op_T__ val__ = (val);						\
 	pcp_op_T__ old__, *ptr__;						\
 										\
-	lc_pcpr = offsetof(struct lowcore, percpu_register);			\
-	lc_pcpo = offsetof(struct lowcore, percpu_offset);			\
 	ptr__ = PERCPU_PTR(&(pcp));						\
 	if (__builtin_constant_p(val__) &&					\
 	    ((szcast)val__ > -129) && ((szcast)val__ < 128)) {			\
 		asm volatile(							\
-			MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
-			AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]")	\
-			op2 "   0(%[ptr__]),%[val__]\n"				\
-			MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]")		\
-			: [ptr__] "+&a" (ptr__), "+m" (*ptr__),			\
+			__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+				      "%[pair__]","%N[pair__]")			\
+			op2 "   0(%N[pair__]),%[val__]\n"			\
+			__PCPU_END("%[lcreg]")					\
+			: [pair__] "=&a" (rp__.pair), "+m" (*ptr__),		\
 			  "=m" (((struct lowcore *)0)->percpu_register)		\
-			: [val__] "i" ((szcast)val__),				\
-			  [disppcpr] "i" (lc_pcpr),				\
-			  [disppcpo] "i" (lc_pcpo),				\
-			  [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS),	\
-			  [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS),	\
+			: [val__] "i" ((szcast)val__), [ptr__] "a" (ptr__),	\
+			  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+			  [lcoff] "i" (LC_PERCPU_OFFSET),			\
 			  "m" (((struct lowcore *)0)->percpu_offset)		\
 			: "cc");						\
 	} else {								\
 		asm volatile(							\
-			MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
-			AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]")	\
-			op1 "   %[old__],%[val__],0(%[ptr__])\n"		\
-			MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]")		\
-			: [old__] "=&d" (old__),				\
-			  [ptr__] "+&a" (ptr__),  "+m" (*ptr__),		\
+			__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+				     "%[pair__]","%N[pair__]")			\
+			op1 "   %[old__],%[val__],0(%N[pair__])\n"		\
+			__PCPU_END("%[lcreg]")					\
+			: [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair),	\
+			  "+m" (*ptr__),					\
 			  "=m" (((struct lowcore *)0)->percpu_register)		\
-			: [val__] "d" (val__),					\
-			  [disppcpr] "i" (lc_pcpr),				\
-			  [disppcpo] "i" (lc_pcpo),				\
-			  [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS),	\
-			  [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS),	\
+			: [val__] "d" (val__), [ptr__] "a" (ptr__),		\
+			  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+			  [lcoff] "i" (LC_PERCPU_OFFSET),			\
 			  "m" (((struct lowcore *)0)->percpu_offset)		\
 			: "cc");						\
 	}									\
@@ -185,27 +227,23 @@ do {										\
 
 #define arch_this_cpu_add_return(pcp, val, op)				\
 ({									\
-	unsigned long lc_pcpr, lc_pcpo;					\
 	typedef typeof(pcp) pcp_op_T__; 				\
+	union register_pair rp__;					\
 	pcp_op_T__ val__ = (val);					\
 	pcp_op_T__ old__, *ptr__;					\
 									\
-	lc_pcpr = offsetof(struct lowcore, percpu_register);		\
-	lc_pcpo = offsetof(struct lowcore, percpu_offset);		\
 	ptr__ = PERCPU_PTR(&(pcp));					\
 	asm_inline volatile(						\
-		MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
-		AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]")	\
-		op "	%[old__],%[val__],0(%[ptr__])\n"		\
-		MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]")		\
-		: [old__] "=&d" (old__),				\
-		  [ptr__] "+&a" (ptr__), "+m" (*ptr__),			\
+		__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+			     "%[pair__]","%N[pair__]")			\
+		op "	%[old__],%[val__],0(%N[pair__])\n"		\
+		__PCPU_END("%[lcreg]")					\
+		: [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair),	\
+		  "+m" (*ptr__),					\
 		  "=m" (((struct lowcore *)0)->percpu_register)		\
-		: [val__] "d" (val__),					\
-		  [disppcpr] "i" (lc_pcpr),				\
-		  [disppcpo] "i" (lc_pcpo),				\
-		  [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS),	\
-		  [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS),	\
+		: [val__] "d" (val__), [ptr__] "a" (ptr__),		\
+		  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+		  [lcoff] "i" (LC_PERCPU_OFFSET),			\
 		  "m" (((struct lowcore *)0)->percpu_offset)		\
 		: "cc");						\
 	old__ + val__;							\
@@ -216,27 +254,23 @@ do {										\
 
 #define arch_this_cpu_to_op(pcp, val, op)				\
 do {									\
-	unsigned long lc_pcpr, lc_pcpo;					\
 	typedef typeof(pcp) pcp_op_T__; 				\
+	union register_pair rp__;					\
 	pcp_op_T__ val__ = (val);					\
 	pcp_op_T__ old__, *ptr__;					\
 									\
-	lc_pcpr = offsetof(struct lowcore, percpu_register);		\
-	lc_pcpo = offsetof(struct lowcore, percpu_offset);		\
 	ptr__ = PERCPU_PTR(&(pcp));					\
 	asm_inline volatile(						\
-		MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
-		AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]")	\
-		op "    %[old__],%[val__],0(%[ptr__])\n"		\
-		MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]")		\
-		: [old__] "=&d" (old__),				\
-		  [ptr__] "+&a" (ptr__), "+m" (*ptr__),			\
+		__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+			     "%[pair__]","%N[pair__]")			\
+		op "    %[old__],%[val__],0(%N[pair__])\n"		\
+		__PCPU_END("%[lcreg]")					\
+		: [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair),	\
+		  "+m" (*ptr__),					\
 		  "=m" (((struct lowcore *)0)->percpu_register)		\
-		: [val__] "d" (val__),					\
-		  [disppcpr] "i" (lc_pcpr),				\
-		  [disppcpo] "i" (lc_pcpo),				\
-		  [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS),	\
-		  [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS),	\
+		: [val__] "d" (val__), [ptr__] "a" (ptr__),		\
+		  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+		  [lcoff] "i" (LC_PERCPU_OFFSET),			\
 		  "m" (((struct lowcore *)0)->percpu_offset)		\
 		: "cc");						\
 } while (0)
@@ -250,24 +284,22 @@ do {									\
 
 #define arch_this_cpu_read(pcp, op)					\
 ({									\
-	unsigned long lc_pcpr, lc_pcpo, res__;				\
 	typedef typeof(pcp) pcp_op_T__;					\
+	union register_pair rp__;					\
+	unsigned long res__;						\
 	pcp_op_T__ *ptr__;						\
 									\
-	lc_pcpr = offsetof(struct lowcore, percpu_register);		\
-	lc_pcpo = offsetof(struct lowcore, percpu_offset);		\
 	ptr__ = PERCPU_PTR(&(pcp));					\
 	asm_inline volatile(						\
-		MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
-		AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]")	\
-		op "	%[res__],0(%[ptr__])\n"				\
-		MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]")		\
-		: [res__] "=&d" (res__), [ptr__] "+&a" (ptr__),		\
+		__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+			      "%[pair__]","%N[pair__]")			\
+		op "	%[res__],0(%N[pair__])\n"			\
+		__PCPU_END("%[lcreg]")					\
+		: [res__] "=&d" (res__), [pair__] "=&a" (rp__.pair),	\
 		  "=m" (((struct lowcore *)0)->percpu_register)		\
-		: [disppcpr] "i" (lc_pcpr),				\
-		  [disppcpo] "i" (lc_pcpo),				\
-		  [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS),	\
-		  [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS),	\
+		: [ptr__] "a" (ptr__),					\
+		  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+		  [lcoff] "i" (LC_PERCPU_OFFSET),			\
 		  "m" (*ptr__),						\
 		  "m" (((struct lowcore *)0)->percpu_offset)		\
 		: "cc");						\
@@ -281,25 +313,21 @@ do {									\
 
 #define arch_this_cpu_write(pcp, val, op)				\
 do {									\
-	unsigned long lc_pcpr, lc_pcpo;					\
 	typedef typeof(pcp) pcp_op_T__;					\
+	union register_pair rp__;					\
 	pcp_op_T__ *ptr__, val__ = (val);				\
 									\
-	lc_pcpr = offsetof(struct lowcore, percpu_register);		\
-	lc_pcpo = offsetof(struct lowcore, percpu_offset);		\
 	ptr__ = PERCPU_PTR(&(pcp));					\
 	asm_inline volatile(						\
-		MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
-		AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]")	\
-		op "    %[val__],0(%[ptr__])\n"				\
-		MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]")		\
-		: [ptr__] "+&a" (ptr__), "=m" (*ptr__),			\
+		__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+			      "%[pair__]","%N[pair__]")			\
+		op "    %[val__],0(%N[pair__])\n"			\
+		__PCPU_END("%[lcreg]")					\
+		: [pair__] "=&a" (rp__.pair), "=m" (*ptr__),		\
 		  "=m" (((struct lowcore *)0)->percpu_register)		\
-		: [val__] "d" (val__),					\
-		  [disppcpr] "i" (lc_pcpr),				\
-		  [disppcpo] "i" (lc_pcpo),				\
-		  [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS),	\
-		  [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS),	\
+		: [val__] "d" (val__), [ptr__] "a" (ptr__),		\
+		  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+		  [lcoff] "i" (LC_PERCPU_OFFSET),			\
 		  "m" (((struct lowcore *)0)->percpu_offset)		\
 		: "cc");						\
 } while (0)
@@ -309,7 +337,7 @@ do {									\
 #define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st")
 #define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg")
 
-#define arch_this_cpu_cmpxchg(pcp, oval, nval)				\
+#define arch_this_cpu_cmpxchg_simple(pcp, oval, nval)			\
 ({									\
 	typedef typeof(pcp) pcp_op_T__;					\
 	pcp_op_T__ ret__;						\
@@ -321,28 +349,62 @@ do {									\
 	ret__;								\
 })
 
-#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
-#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
-#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
-#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
+#define arch_this_cpu_cmpxchg(pcp, oval, nval, op)			\
+({									\
+	typedef typeof(pcp) pcp_op_T__;					\
+	pcp_op_T__ old__ = (oval), new__ = (nval);			\
+	union register_pair rp__;					\
+	pcp_op_T__ *ptr__;						\
+									\
+	ptr__ = PERCPU_PTR(&(pcp));					\
+	asm_inline volatile(						\
+		__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+			     "%[pair__]","%N[pair__]")			\
+		op "	%[old__],%[new__],0(%N[pair__])\n"		\
+		__PCPU_END("%[lcreg]")					\
+		: [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair),	\
+		  "+m" (*ptr__),					\
+		  "=m" (((struct lowcore *)0)->percpu_register)		\
+		: [new__] "d" (new__), [ptr__] "a" (ptr__),		\
+		  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+		  [lcoff] "i" (LC_PERCPU_OFFSET),			\
+		  "m" (((struct lowcore *)0)->percpu_offset)		\
+		: "memory", "cc");					\
+	old__;								\
+})
+
+#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval)
+#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg_simple(pcp, oval, nval)
+#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "cs")
+#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval, "csg")
 
 #define this_cpu_cmpxchg64(pcp, o, n)	this_cpu_cmpxchg_8(pcp, o, n)
 
 #define this_cpu_cmpxchg128(pcp, oval, nval)				\
 ({									\
 	typedef typeof(pcp) pcp_op_T__;					\
-	u128 old__, new__, ret__;					\
+	u128 old__ = (oval), new__ = (nval);				\
+	union register_pair rp__;					\
 	pcp_op_T__ *ptr__;						\
-	old__ = oval;							\
-	new__ = nval;							\
-	preempt_disable_notrace();					\
-	ptr__ = raw_cpu_ptr(&(pcp));					\
-	ret__ = cmpxchg128((void *)ptr__, old__, new__);		\
-	preempt_enable_notrace();					\
-	ret__;								\
+									\
+	ptr__ = PERCPU_PTR(&(pcp));					\
+	asm_inline volatile(						\
+		__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+			     "%[pair__]","%N[pair__]")			\
+		"	cdsg	%[old__],%[new__],0(%N[pair__])\n"	\
+		__PCPU_END("%[lcreg]")					\
+		: [old__] "+&d" (old__), [pair__] "=&a" (rp__.pair),	\
+		  "+m" (*ptr__),					\
+		  "=m" (((struct lowcore *)0)->percpu_register)		\
+		: [new__] "d" (new__), [ptr__] "a" (ptr__),		\
+		  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+		  [lcoff] "i" (LC_PERCPU_OFFSET),			\
+		  "m" (((struct lowcore *)0)->percpu_offset)		\
+		: "memory", "cc");					\
+	old__;								\
 })
 
-#define arch_this_cpu_xchg(pcp, nval)					\
+#define arch_this_cpu_xchg_simple(pcp, nval)				\
 ({									\
 	typeof(pcp) *ptr__;						\
 	typeof(pcp) ret__;						\
@@ -353,10 +415,36 @@ do {									\
 	ret__;								\
 })
 
-#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
-#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
-#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
-#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
+#define arch_this_cpu_xchg(pcp, nval, ldop, csop)			\
+({									\
+	typedef typeof(pcp) pcp_op_T__;					\
+	pcp_op_T__ old__, new__ = (nval);				\
+	union register_pair rp__;					\
+	pcp_op_T__ *ptr__;						\
+									\
+	ptr__ = PERCPU_PTR(&(pcp));					\
+	asm_inline volatile(						\
+		__PCPU_BEGIN("%[lcreg]","%[lcoff]","%[ptr__]",		\
+			     "%[pair__]","%N[pair__]")			\
+		"	" ldop "	%[old__],0(%N[pair__])\n"	\
+		"0:	" csop "	%[old__],%[new__],0(%N[pair__])\n"\
+		"	jnz	0b\n"					\
+		__PCPU_END("%[lcreg]")					\
+		: [old__] "=&d" (old__), [pair__] "=&a" (rp__.pair),	\
+		  "+m" (*ptr__),					\
+		  "=m" (((struct lowcore *)0)->percpu_register)		\
+		: [new__] "d" (new__), [ptr__] "a" (ptr__),		\
+		  [lcreg] "i" (LC_PERCPU_REGISTER),			\
+		  [lcoff] "i" (LC_PERCPU_OFFSET),			\
+		  "m" (((struct lowcore *)0)->percpu_offset)		\
+		: "memory", "cc");					\
+	old__;								\
+})
+
+#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval)
+#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg_simple(pcp, nval)
+#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval, "l",  "cs")
+#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval, "lg", "csg")
 
 #include <asm-generic/percpu.h>
 
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index e882663a58e776..2d5c2ab06de988 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -68,17 +68,6 @@ extern unsigned long zero_page_mask;
 
 /* TODO: s390 cannot support io_remap_pfn_range... */
 
-#define pte_ERROR(e) \
-	pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
-#define pmd_ERROR(e) \
-	pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
-#define pud_ERROR(e) \
-	pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
-#define p4d_ERROR(e) \
-	pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
-
 /*
  * The vmalloc and module area will always be on the topmost area of the
  * kernel mapping. 512GB are reserved for vmalloc by default.
diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h
index 5560d5fca2a30c..60c6f019ec4573 100644
--- a/arch/s390/include/asm/preempt.h
+++ b/arch/s390/include/asm/preempt.h
@@ -155,20 +155,9 @@ static __always_inline int __preempt_count_sub_return(int val)
 void preempt_schedule(void);
 void preempt_schedule_notrace(void);
 
-#ifdef CONFIG_PREEMPT_DYNAMIC
-
-void dynamic_preempt_schedule(void);
-void dynamic_preempt_schedule_notrace(void);
-#define __preempt_schedule()		dynamic_preempt_schedule()
-#define __preempt_schedule_notrace()	dynamic_preempt_schedule_notrace()
-
-#else /* CONFIG_PREEMPT_DYNAMIC */
-
 #define __preempt_schedule()		preempt_schedule()
 #define __preempt_schedule_notrace()	preempt_schedule_notrace()
 
-#endif /* CONFIG_PREEMPT_DYNAMIC */
-
 #endif /* CONFIG_PREEMPTION */
 
 #endif /* __ASM_PREEMPT_H */
diff --git a/arch/s390/include/asm/set_memory.h b/arch/s390/include/asm/set_memory.h
index 94092f4ae76499..e3562bf0c1aa5e 100644
--- a/arch/s390/include/asm/set_memory.h
+++ b/arch/s390/include/asm/set_memory.h
@@ -60,9 +60,8 @@ __SET_MEMORY_FUNC(set_memory_rox, SET_MEMORY_RO | SET_MEMORY_X)
 __SET_MEMORY_FUNC(set_memory_rwnx, SET_MEMORY_RW | SET_MEMORY_NX)
 __SET_MEMORY_FUNC(set_memory_4k, SET_MEMORY_4K)
 
-int set_direct_map_invalid_noflush(struct page *page);
-int set_direct_map_default_noflush(struct page *page);
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr);
+int set_direct_map_default_noflush(struct page *page, unsigned int nr);
 bool kernel_page_present(struct page *page);
 
 #endif
diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c
index b5bf8284dbfc8c..3b68f5c58333e0 100644
--- a/arch/s390/kernel/debug.c
+++ b/arch/s390/kernel/debug.c
@@ -434,7 +434,8 @@ static debug_info_t *debug_info_copy(debug_info_t *in, int mode)
 		debug_info_free(rc);
 	} while (1);
 
-	if (mode == NO_AREAS)
+	/* debug_register_static() failure leaves areas NULL, bounds intact */
+	if (mode == NO_AREAS || !in->areas)
 		goto out;
 
 	for (i = 0; i < in->nr_areas; i++) {
@@ -823,6 +824,9 @@ ssize_t debug_dump(debug_info_t *id, struct debug_view *view,
 	file_private_info_t *p_info;
 	size_t size, offset = 0;
 
+	if (!id)
+		return -EINVAL;
+
 	/* Need space for '\0' byte */
 	if (buf_size < 1)
 		return 0;
@@ -950,8 +954,12 @@ EXPORT_SYMBOL(debug_register);
  *
  * Note: This function is called automatically via an initcall generated by
  *	 DEFINE_STATIC_DEBUG_INFO.
+ *
+ * Return:
+ * - 0 on success
+ * - negative error code on failure
  */
-void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas)
+int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas)
 {
 	unsigned long flags;
 	debug_info_t *copy;
@@ -959,7 +967,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas)
 	if (!initialized) {
 		pr_err("Tried to register debug feature %s too early\n",
 		       id->name);
-		return;
+		return -EINVAL;
 	}
 
 	debug_get_param(id->name, &id->level, &pages_per_area, false);
@@ -975,7 +983,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas)
 		id->active_entries = NULL;
 		raw_spin_unlock_irqrestore(&id->lock, flags);
 
-		return;
+		return -ENOMEM;
 	}
 
 	/* Replace static trace area with dynamic copy. */
@@ -993,6 +1001,8 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas)
 	mutex_lock(&debug_mutex);
 	_debug_register(id);
 	mutex_unlock(&debug_mutex);
+
+	return 0;
 }
 
 /* Remove debugfs entries. */
diff --git a/arch/s390/kernel/ebcdic.c b/arch/s390/kernel/ebcdic.c
index 0e51fa537262b1..970aace8719d70 100644
--- a/arch/s390/kernel/ebcdic.c
+++ b/arch/s390/kernel/ebcdic.c
@@ -392,10 +392,48 @@ __u8 _ebc_toupper[256] =
 	0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
 };
 
+/*
+ * ASCII subset of EBCDIC -> character classes
+ */
+const __u8 _ebctype_asc[256] = {
+      _C,     _C,     _C,     _C,     _C,  _C|_S,     _C,     _C, /* 0x00-0x07 */
+      _C,     _C,     _C,  _C|_S,  _C|_S,  _C|_S,     _C,     _C, /* 0x08-0x0f */
+      _C,     _C,     _C,     _C,     _C,  _C|_S,     _C,     _C, /* 0x10-0x17 */
+      _C,     _C,     _C,     _C,     _C,     _C,     _C,     _C, /* 0x18-0x1f */
+      _C,     _C,     _C,     _C,     _C,  _C|_S,     _C,     _C, /* 0x20-0x27 */
+      _C,     _C,     _C,     _C,     _C,     _C,     _C,     _C, /* 0x28-0x2f */
+       0,      0,     _C,     _C,     _C,     _C,     _C,     _C, /* 0x30-0x37 */
+      _C,     _C,     _C,     _C,     _C,     _C,      0,     _C, /* 0x38-0x3f */
+  _S|_SP,      0,      0,      0,      0,      0,      0,      0, /* 0x40-0x47 */
+       0,      0,      0,     _P,     _P,     _P,     _P,     _P, /* 0x48-0x4f */
+      _P,      0,      0,      0,      0,      0,      0,      0, /* 0x50-0x57 */
+       0,      0,     _P,     _P,     _P,     _P,     _P,      0, /* 0x58-0x5f */
+      _P,     _P,      0,      0,      0,      0,      0,      0, /* 0x60-0x67 */
+       0,      0,      0,     _P,     _P,     _P,     _P,     _P, /* 0x68-0x6f */
+       0,      0,      0,      0,      0,      0,      0,      0, /* 0x70-0x77 */
+       0,     _P,     _P,     _P,     _P,     _P,     _P,     _P, /* 0x78-0x7f */
+       0,  _L|_X,  _L|_X,  _L|_X,  _L|_X,  _L|_X,  _L|_X,     _L, /* 0x80-0x87 */
+      _L,     _L,      0,      0,      0,      0,      0,      0, /* 0x88-0x8f */
+       0,     _L,     _L,     _L,     _L,     _L,     _L,     _L, /* 0x90-0x97 */
+      _L,     _L,      0,      0,      0,      0,      0,      0, /* 0x98-0x9f */
+       0,     _P,     _L,     _L,     _L,     _L,     _L,     _L, /* 0xa0-0xa7 */
+      _L,     _L,      0,      0,      0,      0,      0,      0, /* 0xa8-0xaf */
+      _P,      0,      0,      0,      0,      0,      0,      0, /* 0xb0-0xb7 */
+       0,      0,     _P,     _P,      0,      0,      0,      0, /* 0xb8-0xbf */
+      _P,  _U|_X,  _U|_X,  _U|_X,  _U|_X,  _U|_X,  _U|_X,     _U, /* 0xc0-0xc7 */
+      _U,     _U,      0,      0,      0,      0,      0,      0, /* 0xc8-0xcf */
+      _P,     _U,     _U,     _U,     _U,     _U,     _U,     _U, /* 0xd0-0xd7 */
+      _U,     _U,      0,      0,      0,      0,      0,      0, /* 0xd8-0xdf */
+      _P,      0,     _U,     _U,     _U,     _U,     _U,     _U, /* 0xe0-0xe7 */
+      _U,     _U,      0,      0,      0,      0,      0,      0, /* 0xe8-0xef */
+      _D,     _D,     _D,     _D,     _D,     _D,     _D,     _D, /* 0xf0-0xf7 */
+      _D,     _D,      0,      0,      0,      0,      0,     _C, /* 0xf8-0xff */
+};
+
 EXPORT_SYMBOL(_ascebc_500);
 EXPORT_SYMBOL(_ebcasc_500);
 EXPORT_SYMBOL(_ascebc);
 EXPORT_SYMBOL(_ebcasc);
 EXPORT_SYMBOL(_ebc_tolower);
 EXPORT_SYMBOL(_ebc_toupper);
-
+EXPORT_SYMBOL(_ebctype_asc);
diff --git a/arch/s390/kernel/hiperdispatch.c b/arch/s390/kernel/hiperdispatch.c
index 2172065222661b..1397dab8a677a7 100644
--- a/arch/s390/kernel/hiperdispatch.c
+++ b/arch/s390/kernel/hiperdispatch.c
@@ -207,16 +207,12 @@ static unsigned long hd_calculate_steal_percentage(void)
 {
 	unsigned long time_delta, steal_delta, steal, percentage;
 	static ktime_t prev;
-	int cpus, cpu;
+	int cpus;
 	ktime_t now;
 
-	cpus = 0;
-	steal = 0;
 	percentage = 0;
-	for_each_cpu(cpu, &hd_vmvl_cpumask) {
-		steal += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
-		cpus++;
-	}
+	steal = kcpustat_field_total(CPUTIME_STEAL, &hd_vmvl_cpumask);
+	cpus = cpumask_weight(&hd_vmvl_cpumask);
 	/*
 	 * If there is no vertical medium and low CPUs steal time
 	 * is 0 as vertical high CPUs shouldn't experience steal time.
@@ -265,7 +261,7 @@ static void hd_capacity_work_fn(struct work_struct *work)
 	}
 	trace_s390_hd_work_fn(steal_percentage, hd_entitled_cores, hd_high_capacity_cores);
 	mutex_unlock(&smp_cpu_state_mutex);
-	schedule_delayed_work(&hd_capacity_work, HD_DELAY_INTERVAL);
+	queue_delayed_work(system_dfl_wq, &hd_capacity_work, HD_DELAY_INTERVAL);
 }
 
 static int hiperdispatch_ctl_handler(const struct ctl_table *ctl, int write,
diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c
index c923496aa7b480..8f21bea349a659 100644
--- a/arch/s390/kernel/irq.c
+++ b/arch/s390/kernel/irq.c
@@ -144,9 +144,9 @@ static int irq_pending(struct pt_regs *regs)
 
 void noinstr do_io_irq(struct pt_regs *regs)
 {
-	bool from_idle, percpu_needs_fixup;
 	struct pt_regs *old_regs;
 	irqentry_state_t state;
+	bool from_idle;
 
 	percpu_entry(regs);
 	state = irqentry_enter(regs);
@@ -174,21 +174,20 @@ void noinstr do_io_irq(struct pt_regs *regs)
 			do_irq_async(regs, IO_INTERRUPT);
 	} while (machine_is_lpar() && irq_pending(regs));
 
-	percpu_needs_fixup = percpu_code_check(regs);
 	irq_exit_rcu();
 	set_irq_regs(old_regs);
 	irqentry_exit(regs, state);
 
 	if (from_idle)
 		regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT);
-	percpu_exit(regs, percpu_needs_fixup);
+	percpu_exit(regs);
 }
 
 void noinstr do_ext_irq(struct pt_regs *regs)
 {
-	bool from_idle, percpu_needs_fixup;
 	struct pt_regs *old_regs;
 	irqentry_state_t state;
+	bool from_idle;
 
 	percpu_entry(regs);
 	state = irqentry_enter(regs);
@@ -214,14 +213,13 @@ void noinstr do_ext_irq(struct pt_regs *regs)
 
 	do_irq_async(regs, EXT_INTERRUPT);
 
-	percpu_needs_fixup = percpu_code_check(regs);
 	irq_exit_rcu();
 	set_irq_regs(old_regs);
 	irqentry_exit(regs, state);
 
 	if (from_idle)
 		regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT);
-	percpu_exit(regs, percpu_needs_fixup);
+	percpu_exit(regs);
 }
 
 static void show_msi_interrupt(struct seq_file *p, int irq)
diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c
index 17297a8b63d951..56a15edc65d71c 100644
--- a/arch/s390/kernel/nmi.c
+++ b/arch/s390/kernel/nmi.c
@@ -361,7 +361,6 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info);
  */
 void notrace s390_do_machine_check(struct pt_regs *regs)
 {
-	bool percpu_needs_fixup;
 	static int ipd_count;
 	static DEFINE_SPINLOCK(ipd_lock);
 	static unsigned long long last_ipd;
@@ -495,9 +494,8 @@ void notrace s390_do_machine_check(struct pt_regs *regs)
 	if (mcck_pending)
 		schedule_mcck_handler();
 
-	percpu_needs_fixup = percpu_code_check(regs);
 	irqentry_nmi_exit(regs, irq_state);
-	percpu_exit(regs, percpu_needs_fixup);
+	percpu_exit(regs);
 }
 NOKPROBE_SYMBOL(s390_do_machine_check);
 
diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c
index 7b305f1456f825..6496fd23c54077 100644
--- a/arch/s390/kernel/perf_regs.c
+++ b/arch/s390/kernel/perf_regs.c
@@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 
 #define REG_RESERVED (~((1UL << PERF_REG_S390_MAX) - 1))
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
 	if (!mask || mask & REG_RESERVED)
 		return -EINVAL;
diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
index b6ba4465f59dea..d747b616bc814b 100644
--- a/arch/s390/kernel/traps.c
+++ b/arch/s390/kernel/traps.c
@@ -26,6 +26,7 @@
 #include <linux/entry-common.h>
 #include <linux/kmsan.h>
 #include <linux/bug.h>
+#include <linux/panic.h>
 #include <asm/entry-percpu.h>
 #include <asm/asm-extable.h>
 #include <asm/irqflags.h>
@@ -33,6 +34,7 @@
 #include <asm/vtime.h>
 #include <asm/fpu.h>
 #include <asm/fault.h>
+#include <asm/processor.h>
 #include "entry.h"
 
 struct pgm_stat {
@@ -283,6 +285,11 @@ static void monitor_event_exception(struct pt_regs *regs)
 	}
 }
 
+void arch_do_panic(void)
+{
+	disabled_wait();
+}
+
 void kernel_stack_invalid(struct pt_regs *regs)
 {
 	/*
@@ -338,7 +345,6 @@ static void (*pgm_check_table[128])(struct pt_regs *regs);
 void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags)
 {
 	struct lowcore *lc = get_lowcore();
-	bool percpu_needs_fixup;
 	irqentry_state_t state;
 	struct pgm_stat *stat;
 	unsigned int trapnr;
@@ -400,9 +406,8 @@ void noinstr __do_pgm_check(struct pt_regs *regs, unsigned long flags)
 		pgm_check_table[trapnr](regs);
 out:
 	local_irq_disable();
-	percpu_needs_fixup = percpu_code_check(regs);
 	irqentry_exit(regs, state);
-	percpu_exit(regs, percpu_needs_fixup);
+	percpu_exit(regs);
 }
 
 static int pgm_check_stat_show(struct seq_file *p, void *v)
diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c
index dc14ebc0105b3c..abb94720c79b79 100644
--- a/arch/s390/kernel/uv.c
+++ b/arch/s390/kernel/uv.c
@@ -16,6 +16,7 @@
 #include <linux/swap.h>
 #include <linux/pagewalk.h>
 #include <linux/backing-dev.h>
+#include <linux/slab.h>
 #include <linux/vmalloc.h>
 #include <asm/facility.h>
 #include <asm/sections.h>
@@ -242,34 +243,38 @@ void uv_free_stor_var(void *stor_var)
 }
 EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm");
 
-static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data)
-{
-	struct page *page;
-	pte_t pte;
-
-	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
-	if (!page)
-		return -ENOMEM;
-	pte = __pte(page_to_phys(page) | pgprot_val(PAGE_KERNEL));
-	set_pte(ptep, pte);
-	return 0;
-}
-
 void *uv_alloc_stor_var(unsigned long size)
 {
+	unsigned long i, nr_pages, addr;
 	struct vm_struct *area;
-	unsigned long addr;
+	struct page **pages;
 
 	size = PAGE_ALIGN(size);
+	nr_pages = size >> PAGE_SHIFT;
 	area = get_vm_area(size, VM_SPARSE);
 	if (!area)
 		return NULL;
+	pages = kvcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL_ACCOUNT);
+	if (!pages)
+		goto free_area;
+	for (i = 0; i < nr_pages; i++) {
+		pages[i] = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+		if (!pages[i])
+			goto free_pages;
+	}
 	addr = (unsigned long)area->addr;
-	if (apply_to_page_range(&init_mm, addr, size, uv_alloc_range_cb, NULL))
-		goto out;
+	if (vm_area_map_pages(area, addr, addr + size, pages))
+		goto free_pages;
+	kvfree(pages);
 	return area->addr;
-out:
-	uv_free_stor_var(area->addr);
+free_pages:
+	for (i = 0; i < nr_pages; i++) {
+		if (pages[i])
+			__free_page(pages[i]);
+	}
+	kvfree(pages);
+free_area:
+	free_vm_area(area);
 	return NULL;
 }
 EXPORT_SYMBOL_FOR_MODULES(uv_alloc_stor_var, "kvm");
@@ -825,7 +830,7 @@ static int find_secret_in_page(const u8 secret_id[UV_SECRET_ID_LEN],
 {
 	u16 i;
 
-	for (i = 0; i < list->total_num_secrets; i++) {
+	for (i = 0; i < list->num_secr_stored; i++) {
 		if (memcmp(secret_id, list->secrets[i].id, UV_SECRET_ID_LEN) == 0) {
 			*secret = list->secrets[i].hdr;
 			return 0;
@@ -846,11 +851,14 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN],
 		   struct uv_secret_list *list,
 		   struct uv_secret_list_item_hdr *secret)
 {
-	u16 start_idx = 0;
+	u16 start_idx;
 	u16 list_rc;
 	int ret;
 
+	list->next_secret_idx = 0;
+
 	do {
+		start_idx = list->next_secret_idx;
 		uv_list_secrets(list, start_idx, &list_rc, NULL);
 		if (list_rc != UVC_RC_EXECUTED && list_rc != UVC_RC_MORE_DATA) {
 			if (list_rc == UVC_RC_INV_CMD)
@@ -861,7 +869,6 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN],
 		ret = find_secret_in_page(secret_id, list, secret);
 		if (ret == 0)
 			return ret;
-		start_idx = list->next_secret_idx;
 	} while (list_rc == UVC_RC_MORE_DATA && start_idx < list->next_secret_idx);
 
 	return -ENOENT;
diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c
index 24547e39fab244..ff80d02c9f5643 100644
--- a/arch/s390/kvm/gmap/dat.c
+++ b/arch/s390/kvm/gmap/dat.c
@@ -621,17 +621,20 @@ int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey)
 	union pte *ptep;
 	int rc;
 
+again:
 	skey->skey = 0;
 	rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep);
 	if (rc)
 		return rc;
 
 	if (!ptep) {
-		union crste crste;
+		union crste crste = READ_ONCE(*crstep);
 
-		crste = READ_ONCE(*crstep);
-		if (!crste.h.fc || !crste.s.fc1.pr)
+		if (!crste_leaf(crste) && !crste.h.i)
+			goto again;
+		if (!crste.s.fc1.pr)
 			return 0;
+
 		skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn));
 		return 0;
 	}
@@ -662,13 +665,20 @@ int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gf
 	union pte *ptep;
 	int rc;
 
+again:
 	rc = dat_entry_walk(mc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE,
 			    &crstep, &ptep);
 	if (rc)
 		return rc;
 
 	if (!ptep) {
-		page_set_storage_key(large_crste_to_phys(*crstep, gfn), skey.skey, !nq);
+		union crste crste = READ_ONCE(*crstep);
+
+		/* A large page has been split concurrently, try again */
+		if (!crste_leaf(crste))
+			goto again;
+
+		page_set_storage_key(large_crste_to_phys(crste, gfn), skey.skey, !nq);
 		return 0;
 	}
 
@@ -718,15 +728,22 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf
 	union pte *ptep;
 	int rc;
 
+again:
 	rc = dat_entry_walk(mmc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE,
 			    &crstep, &ptep);
 	if (rc)
 		return rc;
 
 	if (!ptep) {
+		union crste crste = READ_ONCE(*crstep);
+
+		/* A large page has been split concurrently, try again */
+		if (!crste_leaf(crste))
+			goto again;
 		if (!oldkey)
 			oldkey = &prev;
-		return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey,
+
+		return page_cond_set_storage_key(large_crste_to_phys(crste, gfn), skey, oldkey,
 						 nq, mr, mc);
 	}
 
@@ -768,7 +785,7 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey)
 	int rc;
 
 	skey->skey = 0;
-
+again:
 	rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep);
 	if (rc)
 		return rc;
@@ -776,9 +793,12 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey)
 	if (!ptep) {
 		union crste crste = READ_ONCE(*crstep);
 
-		if (!crste.h.fc || !crste.s.fc1.pr)
+		/* A large page has been split concurrently, try again */
+		if (!crste_leaf(crste) && !crste.h.i)
+			goto again;
+		if (!crste.s.fc1.pr)
 			return 0;
-		skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1;
+		skey->skey = page_reset_referenced(large_crste_to_phys(crste, gfn)) << 1;
 		return 0;
 	}
 	old = pgste_get_lock(ptep);
@@ -846,19 +866,12 @@ long dat_reset_skeys(union asce asce, gfn_t start)
 }
 #endif /* KVM_S390_MANAGES_S390_GUEST */
 
-struct slot_priv {
-	unsigned long token;
-	struct kvm_s390_mmu_cache *mc;
-};
-
 static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
 {
-	struct slot_priv *p = walk->priv;
-	union crste dummy = { .val = p->token };
 	union pte new_pte, pte = READ_ONCE(*ptep);
 	union pgste pgste;
 
-	new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par);
+	new_pte = walk->priv ? _PTE_EMPTY : _PTE_TOK(_DAT_TOKEN_PIC, PGM_ADDRESSING);
 
 	/* Table entry already in the desired state. */
 	if (pte.val == new_pte.val)
@@ -875,10 +888,9 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal
 static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
 {
 	union crste new_crste, crste = READ_ONCE(*crstep);
-	struct slot_priv *p = walk->priv;
+	struct kvm_s390_mmu_cache *mc = walk->priv;
 
-	new_crste.val = p->token;
-	new_crste.h.tt = crste.h.tt;
+	new_crste = mc ? _CRSTE_EMPTY(crste.h.tt) : _CRSTE_HOLE(crste.h.tt);
 
 	/* Table entry already in the desired state. */
 	if (crste.val == new_crste.val)
@@ -902,7 +914,10 @@ static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct d
 	if (!crste.h.fc && !crste.h.i)
 		return 0;
 	/* Split (install a lower level table), and handle things there. */
-	return dat_split_crste(p->mc, crstep, gfn, walk->asce, false);
+	if (mc)
+		return dat_split_crste(mc, crstep, gfn, walk->asce, false);
+	/* A large page should never cross memslots boundaries */
+	return -EINVAL;
 }
 
 static const struct dat_walk_ops dat_slot_ops = {
@@ -910,16 +925,10 @@ static const struct dat_walk_ops dat_slot_ops = {
 	.crste_ops = { _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, },
 };
 
-int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end,
-		 u16 type, u16 param)
+int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end)
 {
-	struct slot_priv priv = {
-		.token = _CRSTE_TOK(0, type, param).val,
-		.mc = mc,
-	};
-
 	return _dat_walk_gfn_range(start, end, asce, &dat_slot_ops,
-				   DAT_WALK_IGN_HOLES | DAT_WALK_ANY, &priv);
+				   DAT_WALK_IGN_HOLES | DAT_WALK_ANY, mc);
 }
 
 static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgstes)
diff --git a/arch/s390/kvm/gmap/dat.h b/arch/s390/kvm/gmap/dat.h
index e452c141b84138..90389d47ba4e04 100644
--- a/arch/s390/kvm/gmap/dat.h
+++ b/arch/s390/kvm/gmap/dat.h
@@ -547,8 +547,7 @@ long dat_reset_skeys(union asce asce, gfn_t start);
 unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param);
 void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val);
 
-int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end,
-		 u16 type, u16 param);
+int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end);
 
 #if KVM_S390_MANAGES_S390_GUEST
 int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn);
@@ -973,16 +972,15 @@ static inline int get_level(union crste *crstep, union pte *ptep)
 	return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt;
 }
 
-static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
-				  unsigned long npages)
+static inline int dat_delete_slot(union asce asce, gfn_t start, unsigned long npages)
 {
-	return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING);
+	return dat_set_slot(NULL, asce, start, start + npages);
 }
 
 static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
 				  unsigned long npages)
 {
-	return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0);
+	return dat_set_slot(mc, asce, start, start + npages);
 }
 
 static inline bool crste_is_ucas(union crste crste)
diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c
index 4968330e9553bd..3f3fa864cc36b9 100644
--- a/arch/s390/kvm/gmap/gmap.c
+++ b/arch/s390/kvm/gmap/gmap.c
@@ -994,11 +994,13 @@ static long _destroy_pages_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct da
 static long _destroy_pages_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
 {
 	phys_addr_t origin, cur, end;
+	union crste crste;
 
-	if (!crstep->h.fc || !crstep->s.fc1.pr)
+	crste = READ_ONCE(*crstep);
+	if (!crste.h.fc || !crste.s.fc1.pr)
 		return 0;
 
-	origin = crste_origin_large(*crstep);
+	origin = crste_origin_large(crste);
 	cur = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin;
 	end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin;
 	for ( ; cur < end; cur += PAGE_SIZE)
diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c
index b08b8229bb6fff..c2ffb5e59ec637 100644
--- a/arch/s390/kvm/gmap/kvm_mmu.c
+++ b/arch/s390/kvm/gmap/kvm_mmu.c
@@ -47,6 +47,9 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm,
 				       struct kvm_memory_slot *new,
 				       enum kvm_mr_change change)
 {
+	struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
+	int rc = 0;
+
 	if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS)
 		return -EINVAL;
 
@@ -61,6 +64,10 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm,
 		 * and munmap() stuff in this slot after doing this call at any
 		 * time.
 		 */
+		if (change != KVM_MR_MOVE && change != KVM_MR_CREATE) {
+			WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
+			return -EINVAL;
+		}
 		if (new->userspace_addr & ~PAGE_MASK)
 			return -EINVAL;
 		if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit)
@@ -69,65 +76,41 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm,
 			return -EINVAL;
 	}
 
-	if (!kvm_s390_is_migration_mode(kvm))
-		return 0;
-
-	/*
-	 * Turn off migration mode when:
-	 * - userspace creates a new memslot with dirty logging off,
-	 * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and
-	 *   dirty logging is turned off.
-	 * Migration mode expects dirty page logging being enabled to store
-	 * its dirty bitmap.
-	 */
-	if (change != KVM_MR_DELETE &&
-	    !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
-		WARN(kvm_s390_vm_stop_migration(kvm),
-		     "Failed to stop migration mode");
-
-	return 0;
-}
-
-void s390_kvm_mmu_commit_memory_region(struct kvm *kvm,
-				       struct kvm_memory_slot *old,
-				       const struct kvm_memory_slot *new,
-				       enum kvm_mr_change change)
-{
-	struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
-	int rc = 0;
-
-	guard(mutex)(&kvm->slots_arch_lock);
+	if (kvm->arch.migration_mode) {
+		/*
+		 * Turn off migration mode when:
+		 * - userspace creates a new memslot with dirty logging off,
+		 * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY)
+		 *   and dirty logging is turned off.
+		 * Migration mode expects dirty page logging being enabled to
+		 * store its dirty bitmap.
+		 */
+		if (change != KVM_MR_DELETE &&
+		    !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
+			WARN(kvm_s390_vm_stop_migration(kvm),
+			     "Failed to stop migration mode");
+	}
 
 	if (change == KVM_MR_FLAGS_ONLY)
-		return;
-
-	mc = kvm_s390_new_mmu_cache();
-	if (!mc) {
-		rc = -ENOMEM;
-		goto out;
+		return 0;
+	if (change != KVM_MR_DELETE) {
+		/* Enough capacity to add a new memslot */
+		mc = kvm_s390_new_mmu_cache();
+		if (!mc)
+			return -ENOMEM;
 	}
-
 	scoped_guard(write_lock, &kvm->mmu_lock) {
 		kvm_s390_update_cmma_dirty(kvm, old);
-		switch (change) {
-		case KVM_MR_DELETE:
-			rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
-			break;
-		case KVM_MR_MOVE:
-			rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
-			if (rc)
-				break;
-			fallthrough;
-		case KVM_MR_CREATE:
+		if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
+			rc = dat_delete_slot(kvm->arch.gmap->asce, old->base_gfn, old->npages);
+		if (!rc && (change == KVM_MR_MOVE || change == KVM_MR_CREATE))
 			rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages);
-			break;
-		case KVM_MR_FLAGS_ONLY:
-			break;
-		default:
-			WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
-		}
 	}
-out:
-	if (rc)
-		pr_warn("failed to commit memory region\n");
+	/*
+	 * Can only be triggered if dat_{create,delete}_slot() found an
+	 * internal inconsistency or if the mmu cache ran out of memory;
+	 * both should be impossible.
+	 */
+	KVM_BUG_ON(rc, kvm);
+	return rc;
 }
diff --git a/arch/s390/kvm/gmap/kvm_mmu.h b/arch/s390/kvm/gmap/kvm_mmu.h
index cdbd390bd33cd2..43cde61bae0326 100644
--- a/arch/s390/kvm/gmap/kvm_mmu.h
+++ b/arch/s390/kvm/gmap/kvm_mmu.h
@@ -10,9 +10,5 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm,
 				       const struct kvm_memory_slot *old,
 				       struct kvm_memory_slot *new,
 				       enum kvm_mr_change change);
-void s390_kvm_mmu_commit_memory_region(struct kvm *kvm,
-				       struct kvm_memory_slot *old,
-				       const struct kvm_memory_slot *new,
-				       enum kvm_mr_change change);
 
 #endif /* ARCH_KVM_GMAP_KVM_MMU_H */
diff --git a/arch/s390/kvm/s390/gaccess.c b/arch/s390/kvm/s390/gaccess.c
index e5c064f263df8d..405345ccc4f1bb 100644
--- a/arch/s390/kvm/s390/gaccess.c
+++ b/arch/s390/kvm/s390/gaccess.c
@@ -1589,12 +1589,25 @@ static inline int ___gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg
 	parent = READ_ONCE(sg->parent);
 	if (!parent)
 		return -EAGAIN;
+retry:
 	scoped_guard(spinlock, &parent->children_lock) {
 		if (READ_ONCE(sg->parent) != parent)
 			return -EAGAIN;
 		sg->invalidated = false;
 		rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk);
 	}
+	if (rc == -ENOENT) {
+		struct kvm_memory_slot *slot;
+		struct guest_fault *entries;
+
+		entries = get_entries(walk);
+		slot = kvm_vcpu_gfn_to_memslot(vcpu, entries[LEVEL_MEM].gfn);
+		if (!slot)
+			return PGM_ADDRESSING;
+		rc = gmap_link(vcpu->arch.mc, parent, entries + LEVEL_MEM, slot);
+		if (!rc)
+			goto retry;
+	}
 	if (!rc)
 		kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false);
 	return rc;
diff --git a/arch/s390/kvm/s390/intercept.c b/arch/s390/kvm/s390/intercept.c
index ca1205dfac8b07..adfde2a656e747 100644
--- a/arch/s390/kvm/s390/intercept.c
+++ b/arch/s390/kvm/s390/intercept.c
@@ -11,6 +11,8 @@
 #include <linux/kvm_host.h>
 #include <linux/errno.h>
 #include <linux/pagemap.h>
+#include <linux/cleanup.h>
+#include <linux/slab.h>
 
 #include <asm/asm-offsets.h>
 #include <asm/irq.h>
@@ -411,9 +413,9 @@ static int handle_partial_execution(struct kvm_vcpu *vcpu)
  */
 int handle_sthyi(struct kvm_vcpu *vcpu)
 {
+	struct sthyi_sctns *sctns __free(kfree) = NULL;
 	int reg1, reg2, cc = 0, r = 0;
 	u64 code, addr, rc = 0;
-	struct sthyi_sctns *sctns = NULL;
 
 	if (!test_kvm_facility(vcpu->kvm, 74))
 		return kvm_s390_inject_program_int(vcpu, PGM_OPERATION);
@@ -430,37 +432,32 @@ int handle_sthyi(struct kvm_vcpu *vcpu)
 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 
 	if (code & 0xffff) {
-		cc = 3;
-		rc = 4;
-		goto out;
+		vcpu->run->s.regs.gprs[reg2 + 1] = 4;
+		kvm_s390_set_psw_cc(vcpu, 3);
+		return 0;
 	}
 
 	if (!kvm_s390_pv_cpu_is_protected(vcpu) && (addr & ~PAGE_MASK))
 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 
-	sctns = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
+	sctns = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT);
 	if (!sctns)
 		return -ENOMEM;
 
 	cc = sthyi_fill(sctns, &rc);
-	if (cc < 0) {
-		free_page((unsigned long)sctns);
+	if (cc < 0)
 		return cc;
-	}
-out:
+
 	if (!cc) {
 		if (kvm_s390_pv_cpu_is_protected(vcpu)) {
 			memcpy(sida_addr(vcpu->arch.sie_block), sctns, PAGE_SIZE);
 		} else {
 			r = write_guest(vcpu, addr, reg2, sctns, PAGE_SIZE);
-			if (r) {
-				free_page((unsigned long)sctns);
+			if (r)
 				return kvm_s390_inject_prog_cond(vcpu, r);
-			}
 		}
 	}
 
-	free_page((unsigned long)sctns);
 	vcpu->run->s.regs.gprs[reg2 + 1] = rc;
 	kvm_s390_set_psw_cc(vcpu, cc);
 	return r;
@@ -536,6 +533,15 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu)
 	set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
 	clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
 	spin_unlock_irqrestore(&fi->lock, flags);
+
+	/*
+	 * We missed the floating IRQ kick since we can only inject
+	 * when we end up here and not when the irq was injected via
+	 * the FLIC.
+	 *
+	 * Now that we have cleared the masking we can kick cpus.
+	 */
+	kvm_s390_pv_sclp_kick(vcpu);
 	return 0;
 }
 
diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c
index 0381ae98170354..31a6074feffcb1 100644
--- a/arch/s390/kvm/s390/interrupt.c
+++ b/arch/s390/kvm/s390/interrupt.c
@@ -1918,49 +1918,126 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 	return 0;
 }
 
+static u64 inti_to_irq_pend_mask(u64 type, int isc)
+{
+	switch (type) {
+	case KVM_S390_MCHK:
+		/* Only repressible machine checks are floating */
+		return BIT(IRQ_PEND_MCHK_REP);
+	case KVM_S390_INT_VIRTIO:
+		return BIT(IRQ_PEND_VIRTIO);
+	case KVM_S390_INT_SERVICE:
+		return BIT(IRQ_PEND_EXT_SERVICE) |
+		       BIT(IRQ_PEND_EXT_SERVICE_EV);
+	case KVM_S390_INT_PFAULT_DONE:
+		return BIT(IRQ_PEND_PFAULT_DONE);
+	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
+		return BIT(isc_to_irq_type(isc));
+	default:
+		return 0;
+	}
+}
+
 /*
- * Find a destination VCPU for a floating irq and kick it.
+ * Setup intervention masks to catch running vcpus that hopefully open
+ * their masks soonish and kick sleeping vcpus to motivate them to
+ * take IRQs.
  */
-static void __floating_irq_kick(struct kvm *kvm, u64 type)
+static void vcpu_intervention_kick(struct kvm_vcpu *vcpu, u64 type)
+{
+	/* make the VCPU drop out of the SIE, or wake it up if sleeping */
+	switch (type) {
+	case KVM_S390_MCHK:
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
+		break;
+	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
+		if (!(type & KVM_S390_INT_IO_AI_MASK &&
+		      vcpu->kvm->arch.gisa_int.origin) ||
+		      kvm_s390_pv_cpu_get_handle(vcpu))
+			kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
+		break;
+	default:
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
+		break;
+	}
+	kvm_s390_vcpu_wakeup(vcpu);
+}
+
+static void kick_cpu_irq(struct kvm *kvm, u64 type, u64 parm)
 {
 	struct kvm_vcpu *dst_vcpu;
 	int sigcpu, online_vcpus, nr_tries = 0;
+	u64 irq_pend_mask;
+	unsigned long i;
 
 	online_vcpus = atomic_read(&kvm->online_vcpus);
 	if (!online_vcpus)
 		return;
 
+	irq_pend_mask = inti_to_irq_pend_mask(type, parm);
 	for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) {
 		sigcpu %= online_vcpus;
 		dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
-		if (!is_vcpu_stopped(dst_vcpu))
+		if (!is_vcpu_stopped(dst_vcpu) &&
+		    deliverable_irqs(dst_vcpu) & irq_pend_mask)
 			break;
 		/* avoid endless loops if all vcpus are stopped */
-		if (nr_tries++ >= online_vcpus)
-			return;
+		if (nr_tries++ >= online_vcpus) {
+			dst_vcpu = NULL;
+			break;
+		}
 	}
 
-	/* make the VCPU drop out of the SIE, or wake it up if sleeping */
-	switch (type) {
-	case KVM_S390_MCHK:
-		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
-		break;
-	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
-		if (!(type & KVM_S390_INT_IO_AI_MASK &&
-		      kvm->arch.gisa_int.origin) ||
-		      kvm_s390_pv_cpu_get_handle(dst_vcpu))
-			kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
-		break;
-	default:
-		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
-		break;
+	/* Nobody was enabled, time to wake all of them */
+	if (!dst_vcpu) {
+		kvm_for_each_vcpu(i, dst_vcpu, kvm)
+			vcpu_intervention_kick(dst_vcpu, type);
+		return;
 	}
-	kvm_s390_vcpu_wakeup(dst_vcpu);
+
+	vcpu_intervention_kick(dst_vcpu, type);
+}
+
+void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * The cpu that called sclp likely will also take the IRQ, no
+	 * need to kick anyone.
+	 */
+	if (deliverable_irqs(vcpu) & BIT(IRQ_PEND_EXT_SERVICE))
+		return;
+
+	/*
+	 * For the other cases we might have sleeping cpus with open
+	 * masks. Time to find and kick them.
+	 */
+	kick_cpu_irq(vcpu->kvm, KVM_S390_INT_SERVICE, -1);
+}
+
+/*
+ * Find a destination VCPU for a floating irq and kick it.
+ */
+static void __floating_irq_kick(struct kvm *kvm, u64 type, u64 parm)
+{
+	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
+
+	/*
+	 * No need to kick on non-ev service IRQs for PV VMs, we're
+	 * not allowed to inject anyway. We need to wait for the sclp
+	 * instruction notification AFTER re-entry of the vcpu that
+	 * handled the instruction intercept.
+	 */
+	if (type == KVM_S390_INT_SERVICE && !(parm & SCCB_EVENT_PENDING) &&
+	    test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs))
+		return;
+
+	kick_cpu_irq(kvm, type, parm);
 }
 
 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 {
 	u64 type = READ_ONCE(inti->type);
+	u64 parm = -1;
 	int rc;
 
 	switch (type) {
@@ -1971,12 +2048,15 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 		rc = __inject_virtio(kvm, inti);
 		break;
 	case KVM_S390_INT_SERVICE:
+		parm = inti->ext.ext_params & SCCB_EVENT_PENDING;
 		rc = __inject_service(kvm, inti);
 		break;
 	case KVM_S390_INT_PFAULT_DONE:
 		rc = __inject_pfault_done(kvm, inti);
 		break;
 	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
+		/* Grab isc here since __inject_io() might free inti */
+		parm = int_word_to_isc(inti->io.io_int_word);
 		rc = __inject_io(kvm, inti);
 		break;
 	default:
@@ -1985,7 +2065,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 	if (rc)
 		return rc;
 
-	__floating_irq_kick(kvm, type);
+	__floating_irq_kick(kvm, type, parm);
 	return 0;
 }
 
@@ -2984,61 +3064,58 @@ static int adapter_indicators_set(struct kvm *kvm,
 				  struct s390_io_adapter *adapter,
 				  struct kvm_s390_adapter_int *adapter_int)
 {
-	unsigned long bit;
-	int summary_set, idx;
 	struct s390_map_info *ind_info, *summary_info;
-	void *map;
 	struct page *ind_page, *summary_page;
-	unsigned long flags;
+	unsigned long bit;
+	int summary_set;
+	void *map;
 
 	ind_page = NULL;
 
-	spin_lock_irqsave(&adapter->maps_lock, flags);
-	ind_info = get_map_info(adapter, adapter_int->ind_addr);
+	scoped_guard(spinlock_irqsave, &adapter->maps_lock) {
+		ind_info = get_map_info(adapter, adapter_int->ind_addr);
+		if (ind_info) {
+			map = page_address(ind_info->page);
+			bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap);
+			set_bit(bit, map);
+		}
+	}
 	if (!ind_info) {
-		spin_unlock_irqrestore(&adapter->maps_lock, flags);
 		ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0);
 		if (!ind_page)
 			return -1;
-		idx = srcu_read_lock(&kvm->srcu);
 		map = page_address(ind_page);
 		bit = get_ind_bit(adapter_int->ind_addr,
 				  adapter_int->ind_offset, adapter->swap);
 		set_bit(bit, map);
-		mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT);
 		set_page_dirty_lock(ind_page);
-		srcu_read_unlock(&kvm->srcu, idx);
 		unpin_user_page(ind_page);
-	} else {
-		map = page_address(ind_info->page);
-		bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap);
-		set_bit(bit, map);
-		spin_unlock_irqrestore(&adapter->maps_lock, flags);
 	}
+	scoped_guard(srcu, &kvm->srcu)
+		mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr));
 
-	spin_lock_irqsave(&adapter->maps_lock, flags);
-	summary_info = get_map_info(adapter, adapter_int->summary_addr);
+	scoped_guard(spinlock_irqsave, &adapter->maps_lock) {
+		summary_info = get_map_info(adapter, adapter_int->summary_addr);
+		if (summary_info) {
+			map = page_address(summary_info->page);
+			bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset,
+					  adapter->swap);
+			summary_set = test_and_set_bit(bit, map);
+		}
+	}
 	if (!summary_info) {
-		spin_unlock_irqrestore(&adapter->maps_lock, flags);
 		summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0);
 		if (!summary_page)
 			return -1;
-		idx = srcu_read_lock(&kvm->srcu);
 		map = page_address(summary_page);
 		bit = get_ind_bit(adapter_int->summary_addr,
 				  adapter_int->summary_offset, adapter->swap);
 		summary_set = test_and_set_bit(bit, map);
-		mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT);
 		set_page_dirty_lock(summary_page);
-		srcu_read_unlock(&kvm->srcu, idx);
 		unpin_user_page(summary_page);
-	} else {
-		map = page_address(summary_info->page);
-		bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset,
-				  adapter->swap);
-		summary_set = test_and_set_bit(bit, map);
-		spin_unlock_irqrestore(&adapter->maps_lock, flags);
 	}
+	scoped_guard(srcu, &kvm->srcu)
+		mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr));
 
 	return summary_set ? 0 : 1;
 }
@@ -3048,26 +3125,29 @@ static int adapter_indicators_set_fast(struct kvm *kvm,
 				       struct kvm_s390_adapter_int *adapter_int,
 				       int setbit)
 {
+	struct s390_map_info *ind_info, *summary_info;
 	unsigned long bit;
 	int summary_set;
-	struct s390_map_info *ind_info, *summary_info;
 	void *map;
 
-	spin_lock(&adapter->maps_lock);
+	guard(srcu)(&kvm->srcu);
+	guard(spinlock)(&adapter->maps_lock);
+
 	ind_info = get_map_info(adapter, adapter_int->ind_addr);
-	if (!ind_info) {
-		spin_unlock(&adapter->maps_lock);
+	if (!ind_info)
 		return -EWOULDBLOCK;
-	}
+
 	map = page_address(ind_info->page);
 	bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap);
-	if (setbit)
+	if (setbit) {
 		set_bit(bit, map);
+		mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr));
+	}
+
 	summary_info = get_map_info(adapter, adapter_int->summary_addr);
-	if (!summary_info) {
-		spin_unlock(&adapter->maps_lock);
+	if (!summary_info)
 		return -EWOULDBLOCK;
-	}
+
 	map = page_address(summary_info->page);
 	bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset,
 			  adapter->swap);
@@ -3077,7 +3157,8 @@ static int adapter_indicators_set_fast(struct kvm *kvm,
 		summary_set = test_and_set_bit(bit, map);
 	else
 		summary_set = test_and_clear_bit(bit, map);
-	spin_unlock(&adapter->maps_lock);
+	mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr));
+
 	return summary_set ? 0 : 1;
 }
 
@@ -3228,9 +3309,9 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len
 				break;
 		}
 	}
-
 	if (storestatus) {
-		n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR);
+		scoped_guard(srcu, &vcpu->kvm->srcu)
+			n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR);
 		return r ? r : n;
 	}
 
@@ -3689,7 +3770,7 @@ void kvm_s390_gib_destroy(void)
 	}
 	chsc_sgib(0);
 	unregister_adapter_interrupt(&gib_alert_irq);
-	free_page((unsigned long)gib);
+	kfree(gib);
 	gib = NULL;
 }
 
@@ -3703,7 +3784,7 @@ int __init kvm_s390_gib_init(u8 nisc)
 		goto out;
 	}
 
-	gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
+	gib = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT | GFP_DMA);
 	if (!gib) {
 		rc = -ENOMEM;
 		goto out;
@@ -3722,7 +3803,7 @@ int __init kvm_s390_gib_init(u8 nisc)
 	gib_origin = virt_to_phys(gib);
 	if (chsc_sgib(gib_origin)) {
 		pr_err("Associating the GIB with the AIV facility failed\n");
-		free_page((unsigned long)gib);
+		kfree(gib);
 		gib = NULL;
 		rc = -EIO;
 		goto out_unreg_gal;
@@ -3742,7 +3823,7 @@ int __init kvm_s390_gib_init(u8 nisc)
 out_unreg_gal:
 	unregister_adapter_interrupt(&gib_alert_irq);
 out_free_gib:
-	free_page((unsigned long)gib);
+	kfree(gib);
 	gib = NULL;
 out:
 	return rc;
diff --git a/arch/s390/kvm/s390/priv.c b/arch/s390/kvm/s390/priv.c
index b3cb2c2c3aa71d..856939acf012fd 100644
--- a/arch/s390/kvm/s390/priv.c
+++ b/arch/s390/kvm/s390/priv.c
@@ -14,6 +14,7 @@
 #include <linux/mm_types.h>
 #include <linux/pgtable.h>
 #include <linux/io.h>
+#include <linux/slab.h>
 #include <asm/asm-offsets.h>
 #include <asm/facility.h>
 #include <asm/current.h>
@@ -869,7 +870,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
 	int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28;
 	int sel1 = vcpu->run->s.regs.gprs[0] & 0xff;
 	int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff;
-	unsigned long mem = 0;
+	void *mem = NULL;
 	u64 operand2;
 	int rc = 0;
 	u8 ar;
@@ -911,19 +912,19 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
 	switch (fc) {
 	case 1: /* same handling for 1 and 2 */
 	case 2:
-		mem = get_zeroed_page(GFP_KERNEL_ACCOUNT);
+		mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT);
 		if (!mem)
 			goto out_no_data;
-		if (stsi((void *) mem, fc, sel1, sel2))
+		if (stsi(mem, fc, sel1, sel2))
 			goto out_no_data;
 		break;
 	case 3:
 		if (sel1 != 2 || sel2 != 2)
 			goto out_no_data;
-		mem = get_zeroed_page(GFP_KERNEL_ACCOUNT);
+		mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT);
 		if (!mem)
 			goto out_no_data;
-		handle_stsi_3_2_2(vcpu, (void *) mem);
+		handle_stsi_3_2_2(vcpu, mem);
 		break;
 	case 15: /* fc 15 is fully handled in userspace */
 		insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2);
@@ -931,10 +932,10 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
 		return -EREMOTE;
 	}
 	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
-		memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE);
+		memcpy(sida_addr(vcpu->arch.sie_block), mem, PAGE_SIZE);
 		rc = 0;
 	} else {
-		rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE);
+		rc = write_guest(vcpu, operand2, ar, mem, PAGE_SIZE);
 	}
 	if (rc) {
 		rc = kvm_s390_inject_prog_cond(vcpu, rc);
@@ -945,14 +946,14 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
 		rc = -EREMOTE;
 	}
 	trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2);
-	free_page(mem);
+	kfree(mem);
 	kvm_s390_set_psw_cc(vcpu, 0);
 	vcpu->run->s.regs.gprs[0] = 0;
 	return rc;
 out_no_data:
 	kvm_s390_set_psw_cc(vcpu, 3);
 out:
-	free_page(mem);
+	kfree(mem);
 	return rc;
 }
 
diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c
index 5c73f43782a741..18fb200f854a84 100644
--- a/arch/s390/kvm/s390/s390.c
+++ b/arch/s390/kvm/s390/s390.c
@@ -3273,8 +3273,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 		goto out_err;
 
 	BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
-	kvm->arch.sie_page2 =
-	     (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
+	kvm->arch.sie_page2 = kzalloc_obj(*kvm->arch.sie_page2,
+					  GFP_KERNEL_ACCOUNT | GFP_DMA);
 	if (!kvm->arch.sie_page2)
 		goto out_err;
 
@@ -3369,7 +3369,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
 	return 0;
 out_err:
-	free_page((unsigned long)kvm->arch.sie_page2);
+	kfree(kvm->arch.sie_page2);
 	debug_unregister(kvm->arch.dbf);
 	sca_dispose(kvm);
 	KVM_EVENT(3, "creation of vm failed: %d", rc);
@@ -3427,7 +3427,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 		mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
 
 	debug_unregister(kvm->arch.dbf);
-	free_page((unsigned long)kvm->arch.sie_page2);
+	kfree(kvm->arch.sie_page2);
 	kvm_s390_destroy_adapters(kvm);
 	kvm_s390_clear_float_irqs(kvm);
 	kvm_s390_vsie_destroy(kvm);
@@ -3579,12 +3579,11 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 {
-	mutex_lock(&vcpu->kvm->lock);
 	preempt_disable();
 	vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
 	vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
 	preempt_enable();
-	mutex_unlock(&vcpu->kvm->lock);
+
 	if (!kvm_is_ucontrol(vcpu->kvm)) {
 		vcpu->arch.gmap = vcpu->kvm->arch.gmap;
 		sca_add_vcpu(vcpu);
@@ -3657,13 +3656,13 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
 {
 	if (vcpu->arch.sie_block->cbrlo)
-		free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
+		kfree(phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK));
 	vcpu->arch.sie_block->cbrlo = 0;
 }
 
 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
 {
-	void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
+	void *cbrlo_page = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT);
 
 	if (!cbrlo_page)
 		return -ENOMEM;
@@ -3757,13 +3756,11 @@ static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
 
 	kvm_s390_vcpu_pci_setup(vcpu);
 
-	mutex_lock(&vcpu->kvm->lock);
 	if (kvm_s390_pv_is_protected(vcpu->kvm)) {
 		rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
 		if (rc)
 			kvm_s390_vcpu_unsetup_cmma(vcpu);
 	}
-	mutex_unlock(&vcpu->kvm->lock);
 
 	return rc;
 }
@@ -5766,7 +5763,7 @@ static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_
 	return 0;
 }
 
-void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old)
+void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old)
 {
 	const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, };
 
@@ -5781,7 +5778,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old,
 				   const struct kvm_memory_slot *new,
 				   enum kvm_mr_change change)
 {
-	s390_kvm_mmu_commit_memory_region(kvm, old, new, change);
+}
+
+int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
+{
+	if (kvm_is_ucontrol(vcpu->kvm))
+		return -EINVAL;
+
+	return 0;
 }
 
 /**
@@ -5810,9 +5814,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_
 	gpa_t end;
 	int rc;
 
-	if (kvm_is_ucontrol(vcpu->kvm))
-		return -EINVAL;
-
 	rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
 	if (rc == PGM_ADDRESSING)
 		return -ENOENT;
diff --git a/arch/s390/kvm/s390/s390.h b/arch/s390/kvm/s390/s390.h
index d284a263ba7042..9ed1af5583e3ff 100644
--- a/arch/s390/kvm/s390/s390.h
+++ b/arch/s390/kvm/s390/s390.h
@@ -375,6 +375,7 @@ enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer);
 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu);
 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu);
 void kvm_s390_clear_float_irqs(struct kvm *kvm);
+void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu);
 int __must_check kvm_s390_inject_vm(struct kvm *kvm,
 				    struct kvm_s390_interrupt *s390int,
 				    struct kvm_s390_interrupt_info *inti);
@@ -472,7 +473,7 @@ int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned i
 			     unsigned long bits);
 
 bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu);
-void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old);
+void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old);
 int kvm_s390_vm_stop_migration(struct kvm *kvm);
 
 
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index dbabca35c008ca..c84991a1be200d 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -318,6 +318,7 @@ void arch_read_lock_wait(arch_rwlock_t *rw)
 
 	/* Remove this reader again to allow recursive read locking */
 	__atomic_add_const(-1, &rw->cnts);
+	trace_contention_begin(rw, LCB_F_SPIN | LCB_F_READ);
 	/* Put the reader into the wait queue */
 	arch_spin_lock(&rw->wait);
 	/* Now add this reader to the count value again */
@@ -326,6 +327,7 @@ void arch_read_lock_wait(arch_rwlock_t *rw)
 	while (READ_ONCE(rw->cnts) & 0x10000)
 		barrier();
 	arch_spin_unlock(&rw->wait);
+	trace_contention_end(rw, 0);
 }
 EXPORT_SYMBOL(arch_read_lock_wait);
 
@@ -333,6 +335,7 @@ void arch_write_lock_wait(arch_rwlock_t *rw)
 {
 	int old;
 
+	trace_contention_begin(rw, LCB_F_SPIN | LCB_F_WRITE);
 	/* Add this CPU to the write waiters */
 	__atomic_add(0x20000, &rw->cnts);
 
@@ -349,6 +352,7 @@ void arch_write_lock_wait(arch_rwlock_t *rw)
 	}
 
 	arch_spin_unlock(&rw->wait);
+	trace_contention_end(rw, 0);
 }
 EXPORT_SYMBOL(arch_write_lock_wait);
 
diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c
index 89badbe72ae706..a23a0bd4d8a885 100644
--- a/arch/s390/mm/dump_pagetables.c
+++ b/arch/s390/mm/dump_pagetables.c
@@ -86,7 +86,7 @@ static void note_prot_wx(struct pg_state *st, unsigned long addr)
 	 */
 	if (addr == PAGE_SIZE && (nospec_uses_trampoline() || !cpu_has_bear()))
 		return;
-	WARN_ONCE(IS_ENABLED(CONFIG_DEBUG_WX),
+	WARN_ONCE(IS_ENABLED(CONFIG_CHECK_WX),
 		  "s390/mm: Found insecure W+X mapping at address %pS\n",
 		  (void *)st->start_address);
 	st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
diff --git a/arch/s390/mm/gmap_helpers.c b/arch/s390/mm/gmap_helpers.c
index ff63ffb1dbd29c..3f6783b93e679f 100644
--- a/arch/s390/mm/gmap_helpers.c
+++ b/arch/s390/mm/gmap_helpers.c
@@ -102,7 +102,7 @@ __context_unsafe(/* pte_unmap_unlock() not instrumented */)
 
 	/* Find the vm address for the guest address */
 	vma = vma_lookup(mm, vmaddr);
-	if (!vma || is_vm_hugetlb_page(vma))
+	if (!vma || vma_is_hugetlb(vma))
 		return;
 
 	/* Get pointer to the page table entry */
@@ -139,7 +139,7 @@ void gmap_helper_discard(struct mm_struct *mm, unsigned long vmaddr, unsigned lo
 		vma = find_vma_intersection(mm, vmaddr, end);
 		if (!vma)
 			return;
-		if (!is_vm_hugetlb_page(vma))
+		if (!vma_is_hugetlb(vma))
 			zap_vma_range(vma, vmaddr, min(end, vma->vm_end) - vmaddr);
 		vmaddr = vma->vm_end;
 	}
@@ -247,7 +247,7 @@ static int __gmap_helper_unshare_zeropages(struct mm_struct *mm)
 		 * proof to catch unexpected zeropages in other mappings and
 		 * fail.
 		 */
-		if ((vma->vm_flags & VM_PFNMAP) || is_vm_hugetlb_page(vma))
+		if ((vma->vm_flags & VM_PFNMAP) || vma_is_hugetlb(vma))
 			continue;
 		addr = vma->vm_start;
 
diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c
index 1e202e3d08e75f..80e834e8b8e1b5 100644
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -382,26 +382,14 @@ int __set_memory(unsigned long addr, unsigned long numpages, unsigned long flags
 	return rc;
 }
 
-int set_direct_map_invalid_noflush(struct page *page)
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr)
 {
-	return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_INV);
+	return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_INV);
 }
 
-int set_direct_map_default_noflush(struct page *page)
+int set_direct_map_default_noflush(struct page *page, unsigned int nr)
 {
-	return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_DEF);
-}
-
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)
-{
-	unsigned long flags;
-
-	if (valid)
-		flags = SET_MEMORY_DEF;
-	else
-		flags = SET_MEMORY_INV;
-
-	return __set_memory((unsigned long)page_to_virt(page), nr, flags);
+	return __set_memory((unsigned long)page_to_virt(page), nr, SET_MEMORY_DEF);
 }
 
 bool kernel_page_present(struct page *page)
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index c4b47070bb590c..1b2566012b63af 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -2566,6 +2566,8 @@ struct bpf_tramp_jit {
 	int r14_off;		/* Offset of saved %r14, has to be at the
 				 * bottom */
 	int do_fexit;		/* do_fexit: label */
+	int skip[BPF_MAX_TRAMP_LINKS];	/* skip: labels after each prog */
+	int nr_progs;
 };
 
 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val)
@@ -2584,6 +2586,7 @@ static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_o
 }
 
 static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
+			   struct bpf_tramp_image *im,
 			   const struct btf_func_model *m,
 			   struct bpf_tramp_node *node, bool save_ret)
 {
@@ -2591,8 +2594,20 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
 	int cookie_off = tjit->run_ctx_off +
 			 offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 	struct bpf_prog *p = node->link->prog;
+	void *skip = jit->prg_buf + jit->prg;
+	int idx = tjit->nr_progs++;
 	int patch;
 
+	if (idx >= ARRAY_SIZE(tjit->skip))
+		return -E2BIG;
+
+	/*
+	 * nop, patched to skip this prog when it is detached
+	 */
+
+	/* brcl 0,skip */
+	EMIT6_PCREL_RILC(0xc0040000, 0, tjit->skip[idx]);
+
 	/*
 	 * run_ctx.cookie = node->cookie;
 	 */
@@ -2652,10 +2667,15 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
 	/* brasl %r14,__bpf_prog_exit */
 	EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p));
 
+	/* skip: */
+	tjit->skip[idx] = jit->prg;
+	bpf_tramp_image_add_skip(im, p, skip, jit->prg_buf + jit->prg);
+
 	return 0;
 }
 
 static int invoke_bpf(struct bpf_tramp_jit *tjit,
+		      struct bpf_tramp_image *im,
 		      const struct btf_func_model *m,
 		      struct bpf_tramp_nodes *tn, bool save_ret,
 		      u64 func_meta, int cookie_off)
@@ -2670,7 +2690,7 @@ static int invoke_bpf(struct bpf_tramp_jit *tjit,
 			emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta);
 			cur_cookie--;
 		}
-		if (invoke_bpf_prog(tjit, m, tn->nodes[i], save_ret))
+		if (invoke_bpf_prog(tjit, im, m, tn->nodes[i], save_ret))
 			return -EINVAL;
 	}
 
@@ -2712,6 +2732,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 	u64 func_meta;
 	int i, j;
 
+	/* The skip labels are taken from the previous pass. */
+	tjit->nr_progs = 0;
+	if (im)
+		im->nr_skips = 0;
+
 	/* Support as many stack arguments as "mvc" instruction can handle. */
 	nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS);
 	nr_stack_args = m->nr_args - nr_reg_args;
@@ -2875,7 +2900,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 		emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0);
 	}
 
-	if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET,
+	if (invoke_bpf(tjit, im, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET,
 		       func_meta, cookie_off))
 		return -EINVAL;
 
@@ -2889,7 +2914,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 		       0xf000 | tjit->retval_off);
 
 		for (i = 0; i < fmod_ret->nr_nodes; i++) {
-			if (invoke_bpf_prog(tjit, m, fmod_ret->nodes[i], true))
+			if (invoke_bpf_prog(tjit, im, m, fmod_ret->nodes[i], true))
 				return -EINVAL;
 
 			/*
@@ -2943,15 +2968,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 		/* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */
 		_EMIT6(0xd203f000 | tjit->tccnt_off,
 		       0xf000 | offsetof(struct prog_frame, tail_call_cnt));
-
-		im->ip_after_call = jit->prg_buf + jit->prg;
-
-		/*
-		 * The following nop will be patched by bpf_tramp_image_put().
-		 */
-
-		/* brcl 0,im->ip_epilogue */
-		EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue);
 	}
 
 	/* Set the "is_return" flag for fsession. */
@@ -2962,12 +2978,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 
 	/* do_fexit: */
 	tjit->do_fexit = jit->prg;
-	if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off))
+	if (invoke_bpf(tjit, im, m, fexit, false, func_meta, cookie_off))
 		return -EINVAL;
 
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
-		im->ip_epilogue = jit->prg_buf + jit->prg;
-
 		/*
 		 * __bpf_tramp_exit(im);
 		 */
@@ -3016,7 +3030,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 			     struct bpf_tramp_nodes *tnodes, void *orig_call)
 {
-	struct bpf_tramp_image im;
+	struct bpf_tramp_image im = {};
 	struct bpf_tramp_jit tjit;
 	int ret;
 
diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c
index f317a1465dad8c..d6af4015223e79 100644
--- a/arch/s390/pci/pci_event.c
+++ b/arch/s390/pci/pci_event.c
@@ -226,6 +226,7 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev,
 	device_lock(&pdev->dev);
 	if (pdev->error_state == pci_channel_io_perm_failure) {
 		ers_res = PCI_ERS_RESULT_DISCONNECT;
+		status_str = "skipped (permanent failure)";
 		goto out_unlock;
 	}
 	pdev->error_state = pci_channel_io_frozen;
@@ -297,8 +298,8 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev,
 		driver->err_handler->resume(pdev);
 	pci_uevent_ers(pdev, PCI_ERS_RESULT_RECOVERED);
 out_unlock:
+	zpci_report_status(zdev, pdev, "recovery", status_str);
 	device_unlock(&pdev->dev);
-	zpci_report_status(zdev, "recovery", status_str);
 
 	return ers_res;
 }
@@ -361,8 +362,10 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf)
 
 	__zpci_event_print_error(pdev, ccdf);
 
-	if (!pdev)
+	if (!pdev) {
+		zpci_report_status(zdev, NULL, "error event", "no pdev bound");
 		goto no_pdev;
+	}
 
 	switch (ccdf->pec) {
 	case 0x002a: /* Error event concerns FMB */
diff --git a/arch/s390/pci/pci_report.c b/arch/s390/pci/pci_report.c
index 7030f7052926ae..72ecabf7003c4c 100644
--- a/arch/s390/pci/pci_report.c
+++ b/arch/s390/pci/pci_report.c
@@ -87,9 +87,23 @@ static struct debug_view debug_log_view = {
 	NULL
 };
 
+static ssize_t zpci_report_pdev(struct pci_dev *pdev, char *buf, size_t size)
+{
+	struct pci_driver *driver;
+	const char *start = buf;
+	char *end = buf + size;
+
+	device_lock_assert(&pdev->dev);
+	buf += scnprintf(buf, end - buf, "state: %s\n", zpci_state_str(pdev->error_state));
+	driver = to_pci_driver(pdev->dev.driver);
+	buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a");
+	return buf - start;
+}
+
 /**
  * zpci_report_status - Report the status of operations on a PCI device
- * @zdev:	The PCI device for which to report status
+ * @zdev:	The zPCI device for which to report status
+ * @pdev:	The PCI device associated with the zdev if any, NULL otherwise
  * @operation:	A string representing the operation reported
  * @status:	A string representing the status of the operation
  *
@@ -103,15 +117,14 @@ static struct debug_view debug_log_view = {
  *
  * Return: 0 on success an error code < 0 otherwise.
  */
-int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status)
+int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev,
+		       const char *operation, const char *status)
 {
 	struct zpci_report_error *report;
-	struct pci_driver *driver = NULL;
-	struct pci_dev *pdev = NULL;
 	char *buf, *end;
 	int ret;
 
-	if (!zdev || !zdev->zbus)
+	if (!zdev)
 		return -ENODEV;
 
 	/* Protected virtualization hosts get nothing from us */
@@ -121,18 +134,13 @@ int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char
 	report = (void *)get_zeroed_page(GFP_KERNEL);
 	if (!report)
 		return -ENOMEM;
-	if (zdev->zbus->bus)
-		pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn);
-	if (pdev)
-		driver = to_pci_driver(pdev->dev.driver);
 
 	buf = report->data.log_data;
 	end = report->data.log_data + ZPCI_REPORT_DATA_SIZE;
 	buf += scnprintf(buf, end - buf, "report: %s\n", operation);
 	buf += scnprintf(buf, end - buf, "status: %s\n", status);
-	buf += scnprintf(buf, end - buf, "state: %s\n",
-			 (pdev) ? zpci_state_str(pdev->error_state) : "n/a");
-	buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a");
+	if (pdev)
+		buf += zpci_report_pdev(pdev, buf, end - buf);
 	ret = debug_dump(pci_debug_msg_id, &debug_log_view, buf, end - buf, true);
 	if (ret < 0)
 		pr_err("Reading PCI debug messages failed with code %d\n", ret);
diff --git a/arch/s390/pci/pci_report.h b/arch/s390/pci/pci_report.h
index e08003d51a9727..dd7b0b05001c19 100644
--- a/arch/s390/pci/pci_report.h
+++ b/arch/s390/pci/pci_report.h
@@ -8,9 +8,11 @@
  */
 #ifndef __S390_PCI_REPORT_H
 #define __S390_PCI_REPORT_H
+#include <linux/pci.h>
 
 struct zpci_dev;
 
-int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status);
+int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev,
+		       const char *operation, const char *status);
 
 #endif /* __S390_PCI_REPORT_H */
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index d60f1d5a94c0f4..fe859def918cc3 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -33,6 +33,7 @@ config SUPERH
 	select HAVE_ARCH_AUDITSYSCALL
 	select HAVE_ARCH_KGDB
 	select HAVE_ARCH_SECCOMP_FILTER
+	select HAVE_ARCH_TLB_REMOVE_TABLE if X2TLB
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c
index ca4b4dd1ddefa6..f662a296a54c1a 100644
--- a/arch/sh/boards/mach-ecovec24/setup.c
+++ b/arch/sh/boards/mach-ecovec24/setup.c
@@ -26,6 +26,7 @@
 #include <linux/platform_data/tmio.h>
 #include <linux/platform_data/tsc2007.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/regulator/fixed.h>
 #include <linux/regulator/machine.h>
 #include <linux/sh_eth.h>
@@ -589,39 +590,21 @@ static struct platform_device keysc_device = {
 /* TouchScreen */
 #define IRQ0 evt2irq(0x600)
 
-static int ts_get_pendown_state(struct device *dev)
-{
-	int val = 0;
-	gpio_free(GPIO_FN_INTC_IRQ0);
-	gpio_request(GPIO_PTZ0, NULL);
-	gpio_direction_input(GPIO_PTZ0);
-
-	val = gpio_get_value(GPIO_PTZ0);
-
-	gpio_free(GPIO_PTZ0);
-	gpio_request(GPIO_FN_INTC_IRQ0, NULL);
-
-	return val ? 0 : 1;
-}
-
-static int ts_init(void)
-{
-	gpio_request(GPIO_FN_INTC_IRQ0, NULL);
-	return 0;
-}
+static const struct property_entry tsc2007_properties[] = {
+	PROPERTY_ENTRY_U32("ti,x-plate-ohms", 180),
+	{ }
+};
 
-static struct tsc2007_platform_data tsc2007_info = {
-	.model			= 2007,
-	.x_plate_ohms		= 180,
-	.get_pendown_state	= ts_get_pendown_state,
-	.init_platform_hw	= ts_init,
+static const struct software_node tsc2007_swnode = {
+	.name = "tsc2007",
+	.properties = tsc2007_properties,
 };
 
 static struct i2c_board_info ts_i2c_clients = {
 	I2C_BOARD_INFO("tsc2007", 0x48),
 	.type		= "tsc2007",
-	.platform_data	= &tsc2007_info,
 	.irq		= IRQ0,
+	.swnode		= &tsc2007_swnode,
 };
 
 static struct regulator_consumer_supply cn12_power_consumers[] =
@@ -1242,8 +1225,9 @@ static int __init arch_setup(void)
 		gpio_direction_output(GPIO_PTF4, 1);
 
 		/* enable TouchScreen */
-		i2c_register_board_info(0, &ts_i2c_clients, 1);
+		gpio_request(GPIO_FN_INTC_IRQ0, NULL);
 		irq_set_irq_type(IRQ0, IRQ_TYPE_LEVEL_LOW);
+		i2c_register_board_info(0, &ts_i2c_clients, 1);
 	}
 
 	/* enable CEU0 */
diff --git a/arch/sh/boards/mach-x3proto/gpio.c b/arch/sh/boards/mach-x3proto/gpio.c
index c13d51b29702d0..a09cafc782b454 100644
--- a/arch/sh/boards/mach-x3proto/gpio.c
+++ b/arch/sh/boards/mach-x3proto/gpio.c
@@ -18,6 +18,7 @@
 #include <linux/io.h>
 #include <mach/ilsel.h>
 #include <mach/hardware.h>
+#include <linux/property.h>
 
 #define KEYCTLR	0xb81c0000
 #define KEYOUTR	0xb81c0002
@@ -73,6 +74,10 @@ static void x3proto_gpio_irq_handler(struct irq_desc *desc)
 	chip->irq_unmask(data);
 }
 
+const struct software_node x3proto_gpiochip_node = {
+	.name = "x3proto-gpio",
+};
+
 struct gpio_chip x3proto_gpio_chip = {
 	.label			= "x3proto-gpio",
 	.direction_input	= x3proto_gpio_direction_input,
@@ -104,9 +109,15 @@ int __init x3proto_gpio_setup(void)
 	if (unlikely(ilsel < 0))
 		return ilsel;
 
+	ret = software_node_register(&x3proto_gpiochip_node);
+	if (ret)
+		goto err_gpio;
+
+	x3proto_gpio_chip.fwnode = software_node_fwnode(&x3proto_gpiochip_node);
+
 	ret = gpiochip_add_data(&x3proto_gpio_chip, NULL);
 	if (unlikely(ret))
-		goto err_gpio;
+		goto err_swnode;
 
 	x3proto_irq_domain = irq_domain_create_linear(NULL, NR_BASEBOARD_GPIOS,
 						   &x3proto_gpio_irq_ops, NULL);
@@ -127,6 +138,8 @@ int __init x3proto_gpio_setup(void)
 err_irq:
 	gpiochip_remove(&x3proto_gpio_chip);
 	ret = 0;
+err_swnode:
+	software_node_unregister(&x3proto_gpiochip_node);
 err_gpio:
 	synchronize_irq(ilsel);
 
diff --git a/arch/sh/boards/mach-x3proto/setup.c b/arch/sh/boards/mach-x3proto/setup.c
index ca2802d3056503..0d9d5982cf0c6c 100644
--- a/arch/sh/boards/mach-x3proto/setup.c
+++ b/arch/sh/boards/mach-x3proto/setup.c
@@ -16,8 +16,8 @@
 #include <linux/input.h>
 #include <linux/usb/r8a66597.h>
 #include <linux/usb/m66592.h>
-#include <linux/gpio/driver.h>
-#include <linux/gpio_keys.h>
+#include <linux/gpio/machine.h>
+#include <linux/gpio/property.h>
 #include <mach/ilsel.h>
 #include <mach/hardware.h>
 #include <asm/smp-ops.h>
@@ -123,87 +123,74 @@ static struct platform_device m66592_usb_peripheral_device = {
 	.resource	= m66592_usb_peripheral_resources,
 };
 
-static struct gpio_keys_button baseboard_buttons[NR_BASEBOARD_GPIOS] = {
-	{
-		.desc		= "key44",
-		.code		= KEY_POWER,
-		.active_low	= 1,
-		.wakeup		= 1,
-	}, {
-		.desc		= "key43",
-		.code		= KEY_SUSPEND,
-		.active_low	= 1,
-		.wakeup		= 1,
-	}, {
-		.desc		= "key42",
-		.code		= KEY_KATAKANAHIRAGANA,
-		.active_low	= 1,
-	}, {
-		.desc		= "key41",
-		.code		= KEY_SWITCHVIDEOMODE,
-		.active_low	= 1,
-	}, {
-		.desc		= "key34",
-		.code		= KEY_F12,
-		.active_low	= 1,
-	}, {
-		.desc		= "key33",
-		.code		= KEY_F11,
-		.active_low	= 1,
-	}, {
-		.desc		= "key32",
-		.code		= KEY_F10,
-		.active_low	= 1,
-	}, {
-		.desc		= "key31",
-		.code		= KEY_F9,
-		.active_low	= 1,
-	}, {
-		.desc		= "key24",
-		.code		= KEY_F8,
-		.active_low	= 1,
-	}, {
-		.desc		= "key23",
-		.code		= KEY_F7,
-		.active_low	= 1,
-	}, {
-		.desc		= "key22",
-		.code		= KEY_F6,
-		.active_low	= 1,
-	}, {
-		.desc		= "key21",
-		.code		= KEY_F5,
-		.active_low	= 1,
-	}, {
-		.desc		= "key14",
-		.code		= KEY_F4,
-		.active_low	= 1,
-	}, {
-		.desc		= "key13",
-		.code		= KEY_F3,
-		.active_low	= 1,
-	}, {
-		.desc		= "key12",
-		.code		= KEY_F2,
-		.active_low	= 1,
-	}, {
-		.desc		= "key11",
-		.code		= KEY_F1,
-		.active_low	= 1,
-	},
+static const struct software_node x3proto_gpio_keys_node = {
+	.name = "x3proto-gpio-keys",
 };
 
-static struct gpio_keys_platform_data baseboard_buttons_data = {
-	.buttons	= baseboard_buttons,
-	.nbuttons	= ARRAY_SIZE(baseboard_buttons),
+#define __X3PROTO_KEY(_id, _code, _gpio, ...)				\
+static const struct property_entry x3proto_key##_id##_props[] = {	\
+	PROPERTY_ENTRY_STRING("label", "key" #_id),			\
+	PROPERTY_ENTRY_U32("linux,code", _code),			\
+	PROPERTY_ENTRY_GPIO("gpios", &x3proto_gpiochip_node,		\
+			    _gpio, GPIO_ACTIVE_LOW),			\
+	__VA_ARGS__							\
+	{ }								\
+};									\
+static const struct software_node x3proto_key##_id##_node = {		\
+	.parent = &x3proto_gpio_keys_node,				\
+	.properties = x3proto_key##_id##_props,				\
+}
+
+#define X3PROTO_KEY(_id, _code, _gpio)					\
+	__X3PROTO_KEY(_id, _code, _gpio)
+
+#define X3PROTO_KEY_WAKEUP(_id, _code, _gpio)				\
+	__X3PROTO_KEY(_id, _code, _gpio,				\
+		      PROPERTY_ENTRY_BOOL("wakeup-source"),		\
+	)
+
+X3PROTO_KEY_WAKEUP(44, KEY_POWER, 0);
+X3PROTO_KEY_WAKEUP(43, KEY_SUSPEND, 1);
+X3PROTO_KEY(42, KEY_KATAKANAHIRAGANA, 2);
+X3PROTO_KEY(41, KEY_SWITCHVIDEOMODE, 3);
+X3PROTO_KEY(34, KEY_F12, 4);
+X3PROTO_KEY(33, KEY_F11, 5);
+X3PROTO_KEY(32, KEY_F10, 6);
+X3PROTO_KEY(31, KEY_F9, 7);
+X3PROTO_KEY(24, KEY_F8, 8);
+X3PROTO_KEY(23, KEY_F7, 9);
+X3PROTO_KEY(22, KEY_F6, 10);
+X3PROTO_KEY(21, KEY_F5, 11);
+X3PROTO_KEY(14, KEY_F4, 12);
+X3PROTO_KEY(13, KEY_F3, 13);
+X3PROTO_KEY(12, KEY_F2, 14);
+X3PROTO_KEY(11, KEY_F1, 15);
+
+static const struct software_node *const x3proto_swnodes[] __initconst = {
+	&x3proto_gpio_keys_node,
+	&x3proto_key44_node,
+	&x3proto_key43_node,
+	&x3proto_key42_node,
+	&x3proto_key41_node,
+	&x3proto_key34_node,
+	&x3proto_key33_node,
+	&x3proto_key32_node,
+	&x3proto_key31_node,
+	&x3proto_key24_node,
+	&x3proto_key23_node,
+	&x3proto_key22_node,
+	&x3proto_key21_node,
+	&x3proto_key14_node,
+	&x3proto_key13_node,
+	&x3proto_key12_node,
+	&x3proto_key11_node,
+	NULL
 };
 
-static struct platform_device baseboard_buttons_device = {
+static const struct platform_device_info x3proto_gpio_keys_device_info __initconst = {
 	.name		= "gpio-keys",
-	.id		= -1,
-	.dev		= {
-		.platform_data	= &baseboard_buttons_data,
-	},
+	.id		= PLATFORM_DEVID_NONE,
+	.swnode		= &x3proto_gpio_keys_node,
 };
 
 static struct platform_device *x3proto_devices[] __initdata = {
@@ -211,7 +198,6 @@ static struct platform_device *x3proto_devices[] __initdata = {
 	&smc91x_device,
 	&r8a66597_usb_host_device,
 	&m66592_usb_peripheral_device,
-	&baseboard_buttons_device,
 };
 
 static void __init x3proto_init_irq(void)
@@ -224,7 +210,8 @@ static void __init x3proto_init_irq(void)
 
 static int __init x3proto_devices_setup(void)
 {
-	int ret, i;
+	struct platform_device *pd;
+	int ret;
 
 	/*
 	 * IRLs are only needed for ILSEL mappings, so flip over the INTC
@@ -239,11 +226,11 @@ static int __init x3proto_devices_setup(void)
 	if (unlikely(ret))
 		return ret;
 
-	/*
-	 * Propagate dynamic GPIOs for the baseboard button device.
-	 */
-	for (i = 0; i < ARRAY_SIZE(baseboard_buttons); i++)
-		baseboard_buttons[i].gpio = x3proto_gpio_chip.base + i;
+	ret = software_node_register_node_group(x3proto_swnodes);
+	if (ret) {
+		pr_err("Failed to register software nodes: %d\n", ret);
+		return ret;
+	}
 
 	r8a66597_usb_host_resources[1].start =
 		r8a66597_usb_host_resources[1].end = ilsel_enable(ILSEL_USBH_I);
@@ -254,8 +241,12 @@ static int __init x3proto_devices_setup(void)
 	smc91x_resources[1].start =
 		smc91x_resources[1].end = ilsel_enable(ILSEL_LAN);
 
-	return platform_add_devices(x3proto_devices,
-				    ARRAY_SIZE(x3proto_devices));
+	ret = platform_add_devices(x3proto_devices, ARRAY_SIZE(x3proto_devices));
+	if (ret)
+		return ret;
+
+	pd = platform_device_register_full(&x3proto_gpio_keys_device_info);
+	return PTR_ERR_OR_ZERO(pd);
 }
 device_initcall(x3proto_devices_setup);
 
diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h
index 6fe7123d38fa9e..67ce7fa23fa128 100644
--- a/arch/sh/include/asm/pgalloc.h
+++ b/arch/sh/include/asm/pgalloc.h
@@ -17,7 +17,11 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
 extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
 extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
 extern void pmd_free(struct mm_struct *mm, pmd_t *pmd);
-#define __pmd_free_tlb(tlb, pmdp, addr)		pmd_free((tlb)->mm, (pmdp))
+extern void __tlb_remove_table(void *table);
+
+/* PMDs are slab-allocated, tag so they are freed correctly. */
+#define __pmd_free_tlb(tlb, pmdp, addr)					\
+	tlb_remove_table((tlb), (void *)((unsigned long)(pmdp) | 1))
 #endif
 
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
diff --git a/arch/sh/include/asm/pgtable-3level.h b/arch/sh/include/asm/pgtable-3level.h
index d1ce73f3bd85ef..3f4d747f30f40b 100644
--- a/arch/sh/include/asm/pgtable-3level.h
+++ b/arch/sh/include/asm/pgtable-3level.h
@@ -25,9 +25,6 @@
 
 #define PTRS_PER_PMD	((1 << PGDIR_SHIFT) / PMD_SIZE)
 
-#define pmd_ERROR(e) \
-	printk("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
-
 typedef union {
 	struct {
 		unsigned long pmd_low;
diff --git a/arch/sh/include/asm/pgtable.h b/arch/sh/include/asm/pgtable.h
index d5ce0950a32319..59244af775fdc3 100644
--- a/arch/sh/include/asm/pgtable.h
+++ b/arch/sh/include/asm/pgtable.h
@@ -9,12 +9,12 @@
 #ifndef __ASM_SH_PGTABLE_H
 #define __ASM_SH_PGTABLE_H
 
+#include <asm/page.h>
 #ifdef CONFIG_X2TLB
 #include <asm/pgtable-3level.h>
 #else
 #include <asm/pgtable-2level.h>
 #endif
-#include <asm/page.h>
 #include <asm/mmu.h>
 
 #ifndef __ASSEMBLER__
diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h
index 5f51af18997b57..cde1bf0c67342b 100644
--- a/arch/sh/include/asm/pgtable_32.h
+++ b/arch/sh/include/asm/pgtable_32.h
@@ -400,19 +400,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 #define pmd_pfn(pmd)		(__pa(pmd_val(pmd)) >> PAGE_SHIFT)
 #define pmd_page(pmd)		(virt_to_page(pmd_val(pmd)))
 
-#ifdef CONFIG_X2TLB
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %p(%08lx%08lx).\n", __FILE__, __LINE__, \
-	       &(e), (e).pte_high, (e).pte_low)
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
-#else
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
-#endif
-
 /*
  * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
  * are !pte_none() && !pte_present().
diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index 39df40d0ebc294..6d07dae196f55a 100644
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -3,7 +3,6 @@
 #define __ASM_SH_TLB_H
 
 #ifndef __ASSEMBLER__
-#include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
 
 #ifdef CONFIG_MMU
diff --git a/arch/sh/include/mach-x3proto/mach/hardware.h b/arch/sh/include/mach-x3proto/mach/hardware.h
index 57ea4a5cca359d..1ee5a404973c32 100644
--- a/arch/sh/include/mach-x3proto/mach/hardware.h
+++ b/arch/sh/include/mach-x3proto/mach/hardware.h
@@ -3,10 +3,12 @@
 #define __MACH_X3PROTO_HARDWARE_H
 
 struct gpio_chip;
+struct software_node;
 
 /* arch/sh/boards/mach-x3proto/gpio.c */
 int x3proto_gpio_setup(void);
 extern struct gpio_chip x3proto_gpio_chip;
+extern const struct software_node x3proto_gpiochip_node;
 
 #define NR_BASEBOARD_GPIOS	16
 
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index 110308bdef01d0..93921109f4e64b 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -52,26 +52,24 @@ static pte_t *__get_pte_phys(unsigned long addr)
 	pmd_t *pmd;
 
 	pgd = pgd_offset_k(addr);
-	if (pgd_none(*pgd)) {
-		pgd_ERROR(*pgd);
+	if (pgd_none(*pgd))
 		return NULL;
-	}
 
 	p4d = p4d_alloc(NULL, pgd, addr);
 	if (unlikely(!p4d)) {
-		p4d_ERROR(*p4d);
+		pr_err("allocating p4d table failed\n");
 		return NULL;
 	}
 
 	pud = pud_alloc(NULL, p4d, addr);
 	if (unlikely(!pud)) {
-		pud_ERROR(*pud);
+		pr_err("allocating pud table failed\n");
 		return NULL;
 	}
 
 	pmd = pmd_alloc(NULL, pud, addr);
 	if (unlikely(!pmd)) {
-		pmd_ERROR(*pmd);
+		pr_err("allocating pmd table failed\n");
 		return NULL;
 	}
 
@@ -84,7 +82,11 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot)
 
 	pte = __get_pte_phys(addr);
 	if (!pte_none(*pte)) {
-		pte_ERROR(*pte);
+		char str[PTVAL_STR_MAX];
+
+		ptval_to_str(str, pte_val(*pte));
+		pr_err("unexpected set PTE at %lx in %s: bad pte %p(%s).\n",
+			addr, __func__, pte, str);
 		return;
 	}
 
diff --git a/arch/sh/mm/pgtable.c b/arch/sh/mm/pgtable.c
index 3a4085ea0161fe..f6184b86b89c6c 100644
--- a/arch/sh/mm/pgtable.c
+++ b/arch/sh/mm/pgtable.c
@@ -56,4 +56,24 @@ void pmd_free(struct mm_struct *mm, pmd_t *pmd)
 {
 	kmem_cache_free(pmd_cachep, pmd);
 }
+
+static void __tlb_remove_table_slab(void *table)
+{
+	kmem_cache_free(pmd_cachep, table);
+}
+
+static void __tlb_remove_table_pgtable(void *table)
+{
+	pagetable_dtor_free(table);
+}
+
+void __tlb_remove_table(void *table)
+{
+	const unsigned long addr = (unsigned long)table;
+
+	if (addr & 1)
+		__tlb_remove_table_slab((void *)(addr & ~1UL));
+	else
+		__tlb_remove_table_pgtable(table);
+}
 #endif /* PAGETABLE_LEVELS > 2 */
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index ab77d3f2536e1a..742ffff8c37f21 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -64,6 +64,7 @@ config SPARC32
 	select HAVE_UID16
 	select HAVE_PAGE_SIZE_4KB
 	select LOCK_MM_AND_FIND_VMA
+	select HAVE_ARCH_TLB_REMOVE_TABLE
 	select OLD_SIGACTION
 	select ZONE_DMA
 
@@ -75,8 +76,7 @@ config SPARC64
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_KRETPROBES
 	select HAVE_KPROBES
-	select MMU_GATHER_RCU_TABLE_FREE if SMP
-	select HAVE_ARCH_TLB_REMOVE_TABLE if SMP
+	select HAVE_ARCH_TLB_REMOVE_TABLE
 	select MMU_GATHER_MERGE_VMAS
 	select MMU_GATHER_NO_FLUSH_CACHE
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h
index 4f73e87b22a32b..36010852ba0c04 100644
--- a/arch/sparc/include/asm/pgalloc_32.h
+++ b/arch/sparc/include/asm/pgalloc_32.h
@@ -48,7 +48,9 @@ static inline void free_pmd_fast(pmd_t * pmd)
 }
 
 #define pmd_free(mm, pmd)		free_pmd_fast(pmd)
-#define __pmd_free_tlb(tlb, pmd, addr)	pmd_free((tlb)->mm, pmd)
+
+#define __pmd_free_tlb(tlb, pmd, addr)					\
+	tlb_remove_table((tlb), (void *)((unsigned long)(pmd) | 1UL))
 
 #define pmd_populate(mm, pmd, pte)	pmd_set(pmd, pte)
 
@@ -72,6 +74,7 @@ static inline void free_pte_fast(pte_t *pte)
 #define pte_free_kernel(mm, pte)	free_pte_fast(pte)
 
 void pte_free(struct mm_struct * mm, pgtable_t pte);
-#define __pte_free_tlb(tlb, pte, addr)	pte_free((tlb)->mm, pte)
+void __tlb_remove_table(void *table);
+#define __pte_free_tlb(tlb, pte, addr)	tlb_remove_table((tlb), (void *)(pte))
 
 #endif /* _SPARC_PGALLOC_H */
diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index caa7632be4c2ae..b5055d259b74d6 100644
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -74,8 +74,6 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable);
 
 void pgtable_free(void *table, bool is_page);
 
-#ifdef CONFIG_SMP
-
 struct mmu_gather;
 void tlb_remove_table(struct mmu_gather *, void *);
 
@@ -96,12 +94,6 @@ static inline void __tlb_remove_table(void *_table)
 		is_page = true;
 	pgtable_free(table, is_page);
 }
-#else /* CONFIG_SMP */
-static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page)
-{
-	pgtable_free(table, is_page);
-}
-#endif /* !CONFIG_SMP */
 
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte,
 				  unsigned long address)
diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h
index f89b1250661dff..5a5f54a090f5b2 100644
--- a/arch/sparc/include/asm/pgtable_32.h
+++ b/arch/sparc/include/asm/pgtable_32.h
@@ -40,9 +40,6 @@ void load_mmu(void);
 unsigned long calc_highpages(void);
 unsigned long __init bootmem_init(unsigned long *pages_avail);
 
-#define pte_ERROR(e)   __builtin_trap()
-#define pmd_ERROR(e)   __builtin_trap()
-#define pgd_ERROR(e)   __builtin_trap()
 
 #define PTRS_PER_PTE    	64
 #define PTRS_PER_PMD    	64
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 0837ebbc5dce63..35a217bf6e5c6d 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -13,6 +13,7 @@
  * the SpitFire page tables.
  */
 
+#include <asm/page.h>
 #include <asm-generic/pgtable-nop4d.h>
 #include <linux/compiler.h>
 #include <linux/const.h>
@@ -20,7 +21,6 @@
 #include <asm/spitfire.h>
 #include <asm/asi.h>
 #include <asm/adi.h>
-#include <asm/page.h>
 #include <asm/processor.h>
 
 /* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
@@ -96,16 +96,6 @@ bool kern_addr_valid(unsigned long addr);
 #define PTRS_PER_PUD	(1UL << PUD_BITS)
 #define PTRS_PER_PGD	(1UL << PGDIR_BITS)
 
-#define pmd_ERROR(e)							\
-	pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n",		\
-	       __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))
-#define pud_ERROR(e)							\
-	pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n",		\
-	       __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
-#define pgd_ERROR(e)							\
-	pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n",		\
-	       __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
-
 #endif /* !(__ASSEMBLER__) */
 
 /* PTE bits which are the same in SUN4U and SUN4V format.  */
diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h
index 3037187482db7e..dc83d92786a98c 100644
--- a/arch/sparc/include/asm/tlb_64.h
+++ b/arch/sparc/include/asm/tlb_64.h
@@ -3,7 +3,6 @@
 #define _SPARC64_TLB_H
 
 #include <linux/swap.h>
-#include <linux/pagemap.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
 
@@ -29,9 +28,7 @@ void flush_tlb_pending(void);
  * and therefore we don't need a TLBI when freeing page-table pages.
  */
 
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 #define tlb_needs_table_invalidate()	(false)
-#endif
 
 #include <asm-generic/tlb.h>
 
diff --git a/arch/sparc/include/uapi/asm/setup.h b/arch/sparc/include/uapi/asm/setup.h
index 3c208a4dd46405..7054f5249a3a68 100644
--- a/arch/sparc/include/uapi/asm/setup.h
+++ b/arch/sparc/include/uapi/asm/setup.h
@@ -6,10 +6,14 @@
 #ifndef _UAPI_SPARC_SETUP_H
 #define _UAPI_SPARC_SETUP_H
 
-#if defined(__sparc__) && defined(__arch64__)
-# define COMMAND_LINE_SIZE 2048
+#ifdef __KERNEL__
+# define COMMAND_LINE_SIZE CONFIG_COMMAND_LINE_SIZE
 #else
-# define COMMAND_LINE_SIZE 256
+# if defined(__sparc__) && defined(__arch64__)
+#  define COMMAND_LINE_SIZE 2048
+# else
+#  define COMMAND_LINE_SIZE 256
+# endif
 #endif
 
 
diff --git a/arch/sparc/kernel/setup.c b/arch/sparc/kernel/setup.c
index 4975867d9001b6..5f43cef8063825 100644
--- a/arch/sparc/kernel/setup.c
+++ b/arch/sparc/kernel/setup.c
@@ -2,6 +2,8 @@
 
 #include <asm/setup.h>
 #include <linux/sysctl.h>
+#include <linux/panic.h>
+#include <linux/printk.h>
 
 static const struct ctl_table sparc_sysctl_table[] = {
 	{
@@ -36,6 +38,13 @@ static const struct ctl_table sparc_sysctl_table[] = {
 #endif
 };
 
+void arch_do_panic(void)
+{
+	/* Make sure the user can actually press Stop-A (L1-A) */
+	stop_a_enabled = 1;
+	pr_emerg("Press Stop-A (L1-A) from sun keyboard or send break\n"
+		 "twice on console to return to the boot prom\n");
+}
 
 static int __init init_sparc_sysctls(void)
 {
diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c
index 32a5c1d9459cde..c309e27973ce6f 100644
--- a/arch/sparc/lib/bitext.c
+++ b/arch/sparc/lib/bitext.c
@@ -22,8 +22,6 @@
  * @align: requested alignment
  *
  * Returns offset in the map or -1 if out of space.
- *
- * Not safe to call from an interrupt (uses spin_lock).
  */
 int bit_map_string_get(struct bit_map *t, int len, int align)
 {
@@ -31,6 +29,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
 	int off_new;
 	int align1;
 	int i, color;
+	unsigned long flags;
 
 	if (t->num_colors) {
 		/* align is overloaded to be the page color */
@@ -50,7 +49,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
 		BUG();
 	color &= align1;
 
-	spin_lock(&t->lock);
+	spin_lock_irqsave(&t->lock, flags);
 	if (len < t->last_size)
 		offset = t->first_free;
 	else
@@ -64,7 +63,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
 		if (offset >= t->size)
 			offset = 0;
 		if (count + len > t->size) {
-			spin_unlock(&t->lock);
+			spin_unlock_irqrestore(&t->lock, flags);
 /* P3 */ printk(KERN_ERR
   "bitmap out: size %d used %d off %d len %d align %d count %d\n",
   t->size, t->used, offset, len, align, count);
@@ -90,7 +89,7 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
 					t->last_off = 0;
 				t->used += len;
 				t->last_size = len;
-				spin_unlock(&t->lock);
+				spin_unlock_irqrestore(&t->lock, flags);
 				return offset;
 			}
 		}
@@ -103,10 +102,11 @@ int bit_map_string_get(struct bit_map *t, int len, int align)
 void bit_map_clear(struct bit_map *t, int offset, int len)
 {
 	int i;
+	unsigned long flags;
 
 	if (t->used < len)
 		BUG();		/* Much too late to do any good, but alas... */
-	spin_lock(&t->lock);
+	spin_lock_irqsave(&t->lock, flags);
 	for (i = 0; i < len; i++) {
 		if (test_bit(offset + i, t->map) == 0)
 			BUG();
@@ -115,7 +115,7 @@ void bit_map_clear(struct bit_map *t, int offset, int len)
 	if (offset < t->first_free)
 		t->first_free = offset;
 	t->used -= len;
-	spin_unlock(&t->lock);
+	spin_unlock_irqrestore(&t->lock, flags);
 }
 
 void bit_map_init(struct bit_map *t, unsigned long *map, int size)
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 103db4683b165e..9bbccb5d23a8f1 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -413,7 +413,7 @@ void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma,
 	if (mm->context.hugetlb_pte_count || mm->context.thp_pte_count) {
 		unsigned long hugepage_size = PAGE_SIZE;
 
-		if (is_vm_hugetlb_page(vma))
+		if (vma_is_hugetlb(vma))
 			hugepage_size = huge_page_size(hstate_vma(vma));
 
 		if (hugepage_size >= PUD_SIZE) {
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 9a74902ad18147..1c277ab3cdb848 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -340,38 +340,60 @@ pgd_t *get_pgd_fast(void)
  * Alignments up to the page size are the same for physical and virtual
  * addresses of the nocache area.
  */
+
+static DEFINE_SPINLOCK(pte_page_lock);
+
 pgtable_t pte_alloc_one(struct mm_struct *mm)
 {
+	unsigned long flags;
 	pte_t *ptep;
 	struct page *page;
 
 	if (!(ptep = pte_alloc_one_kernel(mm)))
 		return NULL;
 	page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
-	spin_lock(&mm->page_table_lock);
+	spin_lock_irqsave(&pte_page_lock, flags);
 	if (page_ref_inc_return(page) == 2 &&
 			!pagetable_pte_ctor(mm, page_ptdesc(page))) {
 		page_ref_dec(page);
 		ptep = NULL;
 	}
-	spin_unlock(&mm->page_table_lock);
+	spin_unlock_irqrestore(&pte_page_lock, flags);
 
 	return ptep;
 }
 
-void pte_free(struct mm_struct *mm, pgtable_t ptep)
+static void __pte_free(pgtable_t ptep)
 {
 	struct page *page;
+	unsigned long flags;
 
 	page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
-	spin_lock(&mm->page_table_lock);
+	spin_lock_irqsave(&pte_page_lock, flags);
 	if (page_ref_dec_return(page) == 1)
 		pagetable_dtor(page_ptdesc(page));
-	spin_unlock(&mm->page_table_lock);
+	spin_unlock_irqrestore(&pte_page_lock, flags);
 
 	srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE);
 }
 
+void pte_free(struct mm_struct *mm, pgtable_t ptep)
+{
+	__pte_free(ptep);
+}
+
+void __tlb_remove_table(void *table)
+{
+	const unsigned long encoded = (unsigned long)table;
+	const unsigned long addr = encoded & ~1UL;
+	const bool is_pmd = encoded & 1;
+
+	if (is_pmd)
+		free_pmd_fast((pmd_t *)addr);
+	else
+		__pte_free((pgtable_t)addr);
+}
+
 /* context handling - a dynamically sized pool is used */
 #define NO_CONTEXT	-1
 
diff --git a/arch/um/Kconfig b/arch/um/Kconfig
index d9541d13d9eb06..7245e3ada1c5a7 100644
--- a/arch/um/Kconfig
+++ b/arch/um/Kconfig
@@ -17,9 +17,9 @@ config UML
 	select ARCH_HAS_KCOV
 	select ARCH_HAS_STRNCPY_FROM_USER
 	select ARCH_HAS_STRNLEN_USER
-	select ARCH_HAS_STRICT_KERNEL_RWX
+	select ARCH_HAS_STRICT_KERNEL_RWX if MMU
 	select HAVE_ARCH_AUDITSYSCALL
-	select HAVE_ARCH_KASAN if X86_64
+	select HAVE_ARCH_KASAN if X86_64 && MMU
 	select HAVE_ARCH_KASAN_VMALLOC if HAVE_ARCH_KASAN
 	select HAVE_ARCH_SECCOMP_FILTER
 	select HAVE_ASM_MODVERSIONS
@@ -27,6 +27,7 @@ config UML
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_PAGE_SIZE_4KB
+	select HAVE_PERF_EVENTS
 	select NO_DMA if !UML_DMA_EMULATION
 	select OF_EARLY_FLATTREE if OF
 	select GENERIC_IRQ_SHOW
@@ -37,18 +38,22 @@ config UML
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
 	select TRACE_IRQFLAGS_SUPPORT
 	select TTY # Needed for line.c
-	select HAVE_ARCH_VMAP_STACK
+	select HAVE_ARCH_VMAP_STACK if MMU
 	select HAVE_RUST
 	select ARCH_HAS_UBSAN
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_SYSCALL_TRACEPOINTS
 	select THREAD_INFO_IN_TASK
 	select SPARSE_IRQ
-	select MMU_GATHER_RCU_TABLE_FREE
+	select UACCESS_MEMCPY if !MMU
+	select GENERIC_STRNLEN_USER if !MMU
+	select GENERIC_STRNCPY_FROM_USER if !MMU
 
 config MMU
-	bool
+	bool "MMU support"
 	default y
+	help
+	  Turning this off allows a nommu build, say Y.
 
 config UML_DMA_EMULATION
 	bool
@@ -210,27 +215,15 @@ config MCONSOLE
 
 	  It is safe to say 'Y' here.
 
-config MAGIC_SYSRQ
-	bool "Magic SysRq key"
-	depends on MCONSOLE
-	help
-	  If you say Y here, you will have some control over the system even
-	  if the system crashes for example during kernel debugging (e.g., you
-	  will be able to flush the buffer cache to disk, reboot the system
-	  immediately or dump some status information). A key for each of the
-	  possible requests is provided.
-
-	  This is the feature normally accomplished by pressing a key
-	  while holding SysRq (Alt+PrintScreen).
-
-	  On UML, this is accomplished by sending a "sysrq" command with
-	  mconsole, followed by the letter for the requested command.
-
-	  The keys are documented in <file:Documentation/admin-guide/sysrq.rst>. Don't say Y
-	  unless you really know what this hack does.
+config ARCH_FORCE_MAX_ORDER
+	int "Order of maximal physically contiguous allocations" if EXPERT
+	default "10" if MMU
+	default "16" if !MMU
 
 config KERNEL_STACK_ORDER
 	int "Kernel stack size order"
+	default 3 if !MMU
+	range 3 10 if !MMU
 	default 2 if 64BIT
 	range 2 10 if 64BIT
 	default 1 if !64BIT
diff --git a/arch/um/Makefile b/arch/um/Makefile
index 937639edc295eb..d930c5f25fb95c 100644
--- a/arch/um/Makefile
+++ b/arch/um/Makefile
@@ -134,9 +134,10 @@ LINK_WRAPS = -Wl,--wrap,malloc -Wl,--wrap,free -Wl,--wrap,calloc
 # Avoid binutils 2.39+ warnings by marking the stack non-executable and
 # ignorning warnings for the kallsyms sections.
 LDFLAGS_EXECSTACK = -z noexecstack
-ifeq ($(CONFIG_LD_IS_BFD),y)
-LDFLAGS_EXECSTACK += $(call ld-option,--no-warn-rwx-segments)
-endif
+# check --no-warn-rwx-segments with CC-as-linker since that gets used
+# for vmlinux and is where we need the option, in particular with LLVM=1
+# it might be different links (ld.bfd vs. ld.lld)
+LDFLAGS_EXECSTACK += $(call try-run,$(CC) -Wl$(comma)--no-warn-rwx-segments -shared -x c /dev/null -o "$$TMP",--no-warn-rwx-segments)
 
 LD_FLAGS_CMDLINE = $(foreach opt,$(KBUILD_LDFLAGS) $(LDFLAGS_EXECSTACK),-Wl,$(opt))
 
diff --git a/arch/um/drivers/Kconfig b/arch/um/drivers/Kconfig
index 6a0354ca032fb0..655e5a270c97d2 100644
--- a/arch/um/drivers/Kconfig
+++ b/arch/um/drivers/Kconfig
@@ -159,6 +159,7 @@ config UML_RTC
 
 config UML_PCI
 	bool
+	depends on MMU
 	select FORCE_PCI
 	select IRQ_MSI_LIB
 	select UML_IOMEM_EMULATION
@@ -169,6 +170,7 @@ config UML_PCI
 config UML_PCI_OVER_VIRTIO
 	bool "Enable PCI over VIRTIO device simulation"
 	# in theory, just VIRTIO is enough, but that causes recursion
+	depends on MMU
 	depends on VIRTIO_UML
 	select UML_PCI
 
@@ -183,6 +185,7 @@ config UML_PCI_OVER_VIRTIO_DEVICE_ID
 
 config UML_PCI_OVER_VFIO
 	bool "Enable VFIO-based PCI passthrough"
+	depends on MMU
 	select UML_PCI
 	help
 	  This driver provides support for VFIO-based PCI passthrough.
diff --git a/arch/um/drivers/cow_user.c b/arch/um/drivers/cow_user.c
index dc1d1bcd85ec24..bb9f05ac70cf41 100644
--- a/arch/um/drivers/cow_user.c
+++ b/arch/um/drivers/cow_user.c
@@ -279,7 +279,7 @@ int file_reader(__u64 offset, char *buf, int len, void *arg)
 	return pread(fd, buf, len, offset);
 }
 
-/* XXX Need to sanity-check the values read from the header */
+/* Sanity checks are performed after reading each header version below. */
 
 int read_cow_header(int (*reader)(__u64, char *, int, void *), void *arg,
 		    __u32 *version_out, char **backing_file_out,
@@ -325,6 +325,12 @@ int read_cow_header(int (*reader)(__u64, char *, int, void *), void *arg,
 		*sectorsize_out = header->v1.sectorsize;
 		*bitmap_offset_out = sizeof(header->v1);
 		*align_out = *sectorsize_out;
+		if (!memchr(header->v1.backing_file, '\0',
+			    sizeof(header->v1.backing_file))) {
+			cow_printf("%s - V1 backing_file not terminated\n",
+				   __func__);
+			goto out;
+		}
 		file = header->v1.backing_file;
 	}
 	else if (version == 2) {
@@ -338,6 +344,12 @@ int read_cow_header(int (*reader)(__u64, char *, int, void *), void *arg,
 		*sectorsize_out = be32toh(header->v2.sectorsize);
 		*bitmap_offset_out = sizeof(header->v2);
 		*align_out = *sectorsize_out;
+		if (!memchr(header->v2.backing_file, '\0',
+			    sizeof(header->v2.backing_file))) {
+			cow_printf("%s - V2 backing_file not terminated\n",
+				   __func__);
+			goto out;
+		}
 		file = header->v2.backing_file;
 	}
 	/* This is very subtle - see above at union cow_header definition */
@@ -354,8 +366,15 @@ int read_cow_header(int (*reader)(__u64, char *, int, void *), void *arg,
 		if (*align_out == 0) {
 			cow_printf("read_cow_header - invalid COW header, "
 				   "align == 0\n");
+			goto out;
 		}
 		*bitmap_offset_out = ROUND_UP(sizeof(header->v3), *align_out);
+		if (!memchr(header->v3.backing_file, '\0',
+			    sizeof(header->v3.backing_file))) {
+			cow_printf("%s - V3 backing_file not terminated\n",
+				   __func__);
+			goto out;
+		}
 		file = header->v3.backing_file;
 	}
 	else if (version == 3) {
@@ -385,14 +404,27 @@ int read_cow_header(int (*reader)(__u64, char *, int, void *), void *arg,
 		if (*align_out == 0) {
 			cow_printf("read_cow_header - invalid COW header, "
 				   "align == 0\n");
+			goto out;
 		}
 		*bitmap_offset_out = ROUND_UP(sizeof(header->v3_b), *align_out);
+		if (!memchr(header->v3_b.backing_file, '\0',
+			    sizeof(header->v3_b.backing_file))) {
+			cow_printf("%s - V3 backing_file not terminated\n",
+				   __func__);
+			goto out;
+		}
 		file = header->v3_b.backing_file;
 	}
 	else {
 		cow_printf("read_cow_header - invalid COW version\n");
 		goto out;
 	}
+
+	if (*sectorsize_out <= 0) {
+		cow_printf("%s - invalid sectorsize %d\n", __func__,
+			   *sectorsize_out);
+		goto out;
+	}
 	err = -ENOMEM;
 	*backing_file_out = cow_strdup(file);
 	if (*backing_file_out == NULL) {
diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c
index e2a9e8879f5847..f01b7d8a07bcaf 100644
--- a/arch/um/drivers/mconsole_kern.c
+++ b/arch/um/drivers/mconsole_kern.c
@@ -106,9 +106,9 @@ void mconsole_version(struct mc_request *req)
 {
 	char version[256];
 
-	sprintf(version, "%s %s %s %s %s", utsname()->sysname,
-		utsname()->nodename, utsname()->release, utsname()->version,
-		utsname()->machine);
+	snprintf(version, sizeof(version), "%s %s %s %s %s",
+		 utsname()->sysname, utsname()->nodename, utsname()->release,
+		 utsname()->version, utsname()->machine);
 	mconsole_reply(req, version, 0, 0);
 }
 
@@ -117,10 +117,12 @@ void mconsole_log(struct mc_request *req)
 	int len;
 	char *ptr = req->request.data;
 
-	ptr += strlen("log ");
+	ptr += strlen(req->cmd->command);
+	ptr = skip_spaces(ptr);
 
-	len = req->len - (ptr - req->request.data);
-	printk(KERN_WARNING "%.*s", len, ptr);
+	len = req->request.len - (ptr - req->request.data);
+	if (len > 0)
+		printk(KERN_WARNING "%.*s", len, ptr);
 	mconsole_reply(req, "", 0, 0);
 }
 
diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c
index 8a70b3a625537c..e902e78c5ff674 100644
--- a/arch/um/drivers/vector_kern.c
+++ b/arch/um/drivers/vector_kern.c
@@ -111,19 +111,26 @@ static void vector_reset_stats(struct vector_private *vp)
 	 * in vector_poll.
 	 */
 
-	spin_lock(&vp->rx_queue->head_lock);
+	if (vp->rx_queue)
+		spin_lock(&vp->rx_queue->head_lock);
+
 	vp->estats.rx_queue_max = 0;
 	vp->estats.rx_queue_running_average = 0;
 	vp->estats.rx_encaps_errors = 0;
 	vp->estats.sg_ok = 0;
 	vp->estats.sg_linearized = 0;
-	spin_unlock(&vp->rx_queue->head_lock);
+
+	if (vp->rx_queue)
+		spin_unlock(&vp->rx_queue->head_lock);
+
 
 	/* TX stats are modified with TX head_lock held
 	 * in vector_send.
 	 */
 
-	spin_lock(&vp->tx_queue->head_lock);
+	if (vp->tx_queue)
+		spin_lock(&vp->tx_queue->head_lock);
+
 	vp->estats.tx_timeout_count = 0;
 	vp->estats.tx_restart_queue = 0;
 	vp->estats.tx_kicks = 0;
@@ -131,7 +138,10 @@ static void vector_reset_stats(struct vector_private *vp)
 	vp->estats.tx_flow_control_xoff = 0;
 	vp->estats.tx_queue_max = 0;
 	vp->estats.tx_queue_running_average = 0;
-	spin_unlock(&vp->tx_queue->head_lock);
+
+	if (vp->tx_queue)
+		spin_unlock(&vp->tx_queue->head_lock);
+
 }
 
 static int get_mtu(struct arglist *def)
@@ -471,8 +481,6 @@ static void destroy_queue(struct vector_queue *qi)
 	struct vector_private *vp = netdev_priv(qi->dev);
 	struct mmsghdr *mmsg_vector;
 
-	if (qi == NULL)
-		return;
 	/* deallocate any skbuffs - we rely on any unused to be
 	 * set to NULL.
 	 */
@@ -1166,15 +1174,15 @@ static int vector_poll(struct napi_struct *napi, int budget)
 
 	if ((vp->options & VECTOR_TX) != 0)
 		tx_enqueued = (vector_send(vp->tx_queue) > 0);
-	spin_lock(&vp->rx_queue->head_lock);
-	if ((vp->options & VECTOR_RX) > 0)
+	if ((vp->options & VECTOR_RX) > 0) {
+		spin_lock(&vp->rx_queue->head_lock);
 		err = vector_mmsg_rx(vp, budget);
-	else {
+		spin_unlock(&vp->rx_queue->head_lock);
+	} else {
 		err = vector_legacy_rx(vp);
 		if (err > 0)
 			err = 1;
 	}
-	spin_unlock(&vp->rx_queue->head_lock);
 	if (err > 0)
 		work_done += err;
 
@@ -1560,13 +1568,16 @@ static void vector_setup_etheraddr(struct net_device *dev, char *str)
 			"Attempt to assign an invalid ethernet address to a device disallowed\n");
 		goto random;
 	}
+
+	eth_hw_addr_set(dev, addr);
+
 	if (!is_local_ether_addr(addr)) {
+		addr[0] |= 0x02;
 		netdev_warn(dev, "Warning: Assigning a globally valid ethernet address to a device\n");
 		netdev_warn(dev, "You should set the 2nd rightmost bit in the first byte of the MAC,\n");
-		netdev_warn(dev, "i.e. %02x:%02x:%02x:%02x:%02x:%02x\n",
-			addr[0] | 0x02, addr[1], addr[2], addr[3], addr[4], addr[5]);
+		netdev_warn(dev, "i.e. %pM\n", addr);
 	}
-	eth_hw_addr_set(dev, addr);
+
 	return;
 
 random:
diff --git a/arch/um/drivers/vector_user.c b/arch/um/drivers/vector_user.c
index 2ea67e6fd067db..3bee634f210279 100644
--- a/arch/um/drivers/vector_user.c
+++ b/arch/um/drivers/vector_user.c
@@ -93,6 +93,9 @@ struct arglist *uml_parse_vector_ifspec(char *arg)
 	len = strlen(arg);
 	for (pos = 0; pos < len; pos++) {
 		if (next_starts) {
+			if (result->numargs >= MAXVARGS)
+				goto cleanup;
+
 			if (parsing_token) {
 				result->tokens[result->numargs] = arg + pos;
 			} else {
diff --git a/arch/um/drivers/virtio_pcidev.c b/arch/um/drivers/virtio_pcidev.c
index 2bf5a645db45fd..53cffd373890b6 100644
--- a/arch/um/drivers/virtio_pcidev.c
+++ b/arch/um/drivers/virtio_pcidev.c
@@ -490,13 +490,24 @@ static int virtio_pcidev_init_vqs(struct virtio_pcidev_device *dev)
 	return 0;
 }
 
+static void virtio_pcidev_del_vqs(struct virtio_device *vdev,
+				  struct virtio_pcidev_device *dev)
+{
+	void *msg;
+
+	while ((msg = virtqueue_detach_unused_buf(dev->irq_vq)))
+		kfree(msg);
+
+	vdev->config->del_vqs(vdev);
+}
+
 static void __virtio_pcidev_virtio_platform_remove(struct virtio_device *vdev,
 						   struct virtio_pcidev_device *dev)
 {
 	um_pci_platform_device_unregister(&dev->pdev);
 
 	virtio_reset_device(vdev);
-	vdev->config->del_vqs(vdev);
+	virtio_pcidev_del_vqs(vdev, dev);
 
 	kfree(dev);
 }
@@ -526,7 +537,7 @@ static int virtio_pcidev_virtio_platform_probe(struct virtio_device *vdev,
 	um_pci_platform_device_unregister(&dev->pdev);
 err_reset:
 	virtio_reset_device(vdev);
-	vdev->config->del_vqs(vdev);
+	virtio_pcidev_del_vqs(vdev, dev);
 err_free:
 	kfree(dev);
 	return err;
@@ -569,7 +580,7 @@ static int virtio_pcidev_virtio_probe(struct virtio_device *vdev)
 
 err_reset:
 	virtio_reset_device(vdev);
-	vdev->config->del_vqs(vdev);
+	virtio_pcidev_del_vqs(vdev, dev);
 err_free:
 	kfree(dev);
 	return err;
@@ -591,9 +602,9 @@ static void virtio_pcidev_virtio_remove(struct virtio_device *vdev)
 
 	/* Stop all virtqueues */
 	virtio_reset_device(vdev);
+	virtio_pcidev_del_vqs(vdev, dev);
 	dev->cmd_vq = NULL;
 	dev->irq_vq = NULL;
-	vdev->config->del_vqs(vdev);
 
 	kfree(dev);
 }
diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c
index 7425a8548141a1..881c34098a7408 100644
--- a/arch/um/drivers/virtio_uml.c
+++ b/arch/um/drivers/virtio_uml.c
@@ -20,6 +20,7 @@
  *
  * Based on Virtio MMIO driver by Pawel Moll, copyright 2011-2014, ARM Ltd.
  */
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -568,7 +569,7 @@ static void vhost_user_get_config(struct virtio_uml_device *vu_dev,
 		goto free;
 	}
 
-	rc = vhost_user_recv_resp(vu_dev, msg, msg_size);
+	rc = vhost_user_recv_resp(vu_dev, msg, payload_size);
 	if (rc) {
 		vu_err(vu_dev,
 		       "receiving VHOST_USER_GET_CONFIG response failed: %d\n",
@@ -869,6 +870,14 @@ static void vu_reset(struct virtio_device *vdev)
 	vu_dev->status = 0;
 }
 
+static void vu_synchronize_cbs(struct virtio_device *vdev)
+{
+	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
+
+	if (vu_dev->irq != UM_IRQ_ALLOC)
+		synchronize_irq(vu_dev->irq);
+}
+
 static void vu_del_vq(struct virtqueue *vq)
 {
 	struct virtio_uml_vq_info *info = vq->priv;
@@ -1121,6 +1130,7 @@ static const struct virtio_config_ops virtio_uml_config_ops = {
 	.reset = vu_reset,
 	.find_vqs = vu_find_vqs,
 	.del_vqs = vu_del_vqs,
+	.synchronize_cbs = vu_synchronize_cbs,
 	.get_features = vu_get_features,
 	.finalize_features = vu_finalize_features,
 	.bus_name = vu_bus_name,
@@ -1274,14 +1284,7 @@ static void virtio_uml_remove(struct platform_device *pdev)
 
 /* Command line device list */
 
-static void vu_cmdline_release_dev(struct device *d)
-{
-}
-
-static struct device vu_cmdline_parent = {
-	.init_name = "virtio-uml-cmdline",
-	.release = vu_cmdline_release_dev,
-};
+static struct device *vu_cmdline_parent;
 
 static DEFINE_MUTEX(vu_cmdline_lock);
 static bool vu_cmdline_parent_registered;
@@ -1333,11 +1336,10 @@ static int vu_cmdline_set_device(const char *device)
 		return -EINVAL;
 
 	if (!vu_cmdline_parent_registered) {
-		err = device_register(&vu_cmdline_parent);
-		if (err) {
+		vu_cmdline_parent = __root_device_register("virtio-uml-cmdline", NULL);
+		if (IS_ERR(vu_cmdline_parent)) {
 			pr_err("Failed to register parent device!\n");
-			put_device(&vu_cmdline_parent);
-			return err;
+			return PTR_ERR(vu_cmdline_parent);
 		}
 		vu_cmdline_parent_registered = true;
 	}
@@ -1352,7 +1354,7 @@ static int vu_cmdline_set_device(const char *device)
 	pr_info("Registering device virtio-uml.%d id=%d at %s\n",
 		vu_cmdline_id, virtio_device_id, socket_path);
 
-	pdev = platform_device_register_data(&vu_cmdline_parent, "virtio-uml",
+	pdev = platform_device_register_data(vu_cmdline_parent, "virtio-uml",
 					     vu_cmdline_id++, &pdata,
 					     sizeof(pdata));
 	err = PTR_ERR_OR_ZERO(pdev);
@@ -1393,7 +1395,7 @@ static int vu_cmdline_get(char *buffer, const struct kernel_param *kp)
 
 	buffer[0] = '\0';
 	if (vu_cmdline_parent_registered)
-		device_for_each_child(&vu_cmdline_parent, buffer,
+		device_for_each_child(vu_cmdline_parent, buffer,
 				      vu_cmdline_get_device);
 	return strlen(buffer) + 1;
 }
@@ -1417,9 +1419,9 @@ static void vu_unregister_cmdline_devices(void)
 	guard(mutex)(&vu_cmdline_lock);
 
 	if (vu_cmdline_parent_registered) {
-		device_for_each_child(&vu_cmdline_parent, NULL,
+		device_for_each_child(vu_cmdline_parent, NULL,
 				      vu_unregister_cmdline_device);
-		device_unregister(&vu_cmdline_parent);
+		root_device_unregister(vu_cmdline_parent);
 		vu_cmdline_parent_registered = false;
 	}
 }
diff --git a/arch/um/include/asm/common.lds.S b/arch/um/include/asm/common.lds.S
index fd481ac371de0b..7b0c6f8a9d4f6d 100644
--- a/arch/um/include/asm/common.lds.S
+++ b/arch/um/include/asm/common.lds.S
@@ -11,6 +11,8 @@
 
   RO_DATA(4096)
 
+  .lrodata : { *(.lrodata .lrodata.[0-9a-zA-Z_]* .lrodata..L*) }
+
   .unprotected : { *(.unprotected) }
   . = ALIGN(4096);
   PROVIDE (_unprotected_end = .);
@@ -91,6 +93,7 @@
   .fini_array : {
 	__fini_array_start = .;
 	*(.fini_array)
+	*(.fini_array.*)
 	__fini_array_end = .;
   }
 
diff --git a/arch/um/include/asm/futex.h b/arch/um/include/asm/futex.h
index 780aa6bfc050c2..07ab81f09281da 100644
--- a/arch/um/include/asm/futex.h
+++ b/arch/um/include/asm/futex.h
@@ -7,8 +7,61 @@
 #include <asm/errno.h>
 
 
+#ifdef CONFIG_MMU
 int arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr);
 int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 			      u32 oldval, u32 newval);
+#else
+#include <linux/atomic.h>
+#include <asm-generic/futex.h>
+
+#ifdef CONFIG_SMP
+static inline int
+arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr)
+{
+	u32 oldval, newval;
+	u32 *p = (u32 __force *)uaddr;
+
+	do {
+		oldval = READ_ONCE(*p);
+
+		switch (op) {
+		case FUTEX_OP_SET:
+			newval = oparg;
+			break;
+		case FUTEX_OP_ADD:
+			newval = oldval + oparg;
+			break;
+		case FUTEX_OP_OR:
+			newval = oldval | oparg;
+			break;
+		case FUTEX_OP_ANDN:
+			newval = oldval & ~oparg;
+			break;
+		case FUTEX_OP_XOR:
+			newval = oldval ^ oparg;
+			break;
+		default:
+			return -ENOSYS;
+		}
+	} while (!try_cmpxchg(p, &oldval, newval));
+
+	*oval = oldval;
+	return 0;
+}
+
+static inline int
+futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
+			      u32 oldval, u32 newval)
+{
+	u32 *p = (u32 __force *)uaddr;
+
+	*uval = cmpxchg(p, oldval, newval);
+
+	return 0;
+}
+#endif /* CONFIG_SMP */
+
+#endif /* CONFIG_MMU */
 
 #endif
diff --git a/arch/um/include/asm/mmu.h b/arch/um/include/asm/mmu.h
index 07d48738b402b4..c0b9ce3215c431 100644
--- a/arch/um/include/asm/mmu.h
+++ b/arch/um/include/asm/mmu.h
@@ -21,6 +21,14 @@ typedef struct mm_context {
 	spinlock_t sync_tlb_lock;
 	unsigned long sync_tlb_range_from;
 	unsigned long sync_tlb_range_to;
+
+#ifndef CONFIG_MMU
+	unsigned long   end_brk;
+#ifdef CONFIG_BINFMT_ELF_FDPIC
+	unsigned long   exec_fdpic_loadmap;
+	unsigned long   interp_fdpic_loadmap;
+#endif
+#endif /* !CONFIG_MMU */
 } mm_context_t;
 
 #define INIT_MM_CONTEXT(mm)						\
diff --git a/arch/um/include/asm/mmu_context.h b/arch/um/include/asm/mmu_context.h
index c727e56ba116ce..528b217da285b0 100644
--- a/arch/um/include/asm/mmu_context.h
+++ b/arch/um/include/asm/mmu_context.h
@@ -18,11 +18,13 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 {
 }
 
+#ifdef CONFIG_MMU
 #define init_new_context init_new_context
 extern int init_new_context(struct task_struct *task, struct mm_struct *mm);
 
 #define destroy_context destroy_context
 extern void destroy_context(struct mm_struct *mm);
+#endif
 
 #include <asm-generic/mmu_context.h>
 
diff --git a/arch/um/include/asm/perf_event.h b/arch/um/include/asm/perf_event.h
new file mode 100644
index 00000000000000..9f774e2db1989a
--- /dev/null
+++ b/arch/um/include/asm/perf_event.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __UM_PERF_EVENT_H
+#define __UM_PERF_EVENT_H
+
+/* nothing here - clearly no real hardware support */
+
+#endif /* __UM_PERF_EVENT_H */
diff --git a/arch/um/include/asm/pgtable-2level.h b/arch/um/include/asm/pgtable-2level.h
index 14ec16f92ce408..fa625f5b5ef750 100644
--- a/arch/um/include/asm/pgtable-2level.h
+++ b/arch/um/include/asm/pgtable-2level.h
@@ -24,13 +24,6 @@
 #define USER_PTRS_PER_PGD ((TASK_SIZE + (PGDIR_SIZE - 1)) / PGDIR_SIZE)
 #define PTRS_PER_PGD	1024
 
-#define pte_ERROR(e) \
-        printk("%s:%d: bad pte %p(%08lx).\n", __FILE__, __LINE__, &(e), \
-	       pte_val(e))
-#define pgd_ERROR(e) \
-        printk("%s:%d: bad pgd %p(%08lx).\n", __FILE__, __LINE__, &(e), \
-	       pgd_val(e))
-
 static inline int pgd_needsync(pgd_t pgd)	{ return 0; }
 static inline void pgd_mkuptodate(pgd_t pgd)	{ }
 
diff --git a/arch/um/include/asm/pgtable-4level.h b/arch/um/include/asm/pgtable-4level.h
index 7a271b7b83d2bd..ff82f99c80fa10 100644
--- a/arch/um/include/asm/pgtable-4level.h
+++ b/arch/um/include/asm/pgtable-4level.h
@@ -42,19 +42,6 @@
 
 #define USER_PTRS_PER_PGD ((TASK_SIZE + (PGDIR_SIZE - 1)) / PGDIR_SIZE)
 
-#define pte_ERROR(e) \
-        printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), \
-	       pte_val(e))
-#define pmd_ERROR(e) \
-        printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), \
-	       pmd_val(e))
-#define pud_ERROR(e) \
-        printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), \
-	       pud_val(e))
-#define pgd_ERROR(e) \
-        printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), \
-	       pgd_val(e))
-
 #define pud_none(x)	(!(pud_val(x) & ~_PAGE_NEEDSYNC))
 #define	pud_bad(x)	((pud_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE)
 #define pud_present(x)	(pud_val(x) & _PAGE_PRESENT)
diff --git a/arch/um/include/asm/pgtable.h b/arch/um/include/asm/pgtable.h
index 88ea8434364dc1..6304026a2b2ecd 100644
--- a/arch/um/include/asm/pgtable.h
+++ b/arch/um/include/asm/pgtable.h
@@ -13,6 +13,7 @@
 
 #define _PAGE_PRESENT	0x001
 #define _PAGE_NEEDSYNC	0x002
+#define _PAGE_EXEC	0x004
 #define _PAGE_RW	0x020
 #define _PAGE_USER	0x040
 #define _PAGE_ACCESSED	0x080
@@ -55,14 +56,19 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 #define _PAGE_TABLE	(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED | _PAGE_DIRTY)
 #define _KERNPG_TABLE	(_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)
 #define _PAGE_CHG_MASK	(PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY)
-#define __PAGE_KERNEL_EXEC                                              \
-	 (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED)
-#define PAGE_NONE	__pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED)
-#define PAGE_SHARED	__pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED)
-#define PAGE_COPY	__pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED)
-#define PAGE_READONLY	__pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED)
-#define PAGE_KERNEL	__pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED)
-#define PAGE_KERNEL_EXEC	__pgprot(__PAGE_KERNEL_EXEC)
+
+#define _PAGE_BASE		(_PAGE_PRESENT | _PAGE_USER | _PAGE_ACCESSED)
+#define PAGE_NONE		__pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED)
+#define PAGE_SHARED		__pgprot(_PAGE_BASE | _PAGE_RW)
+#define PAGE_SHARED_EXEC	__pgprot(_PAGE_BASE | _PAGE_RW | _PAGE_EXEC)
+#define PAGE_COPY		__pgprot(_PAGE_BASE)
+#define PAGE_COPY_EXEC		__pgprot(_PAGE_BASE | _PAGE_EXEC)
+#define PAGE_READONLY		__pgprot(_PAGE_BASE)
+#define PAGE_READONLY_EXEC	__pgprot(_PAGE_BASE | _PAGE_EXEC)
+
+#define _PAGE_KERNEL		(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED)
+#define PAGE_KERNEL		__pgprot(_PAGE_KERNEL)
+#define PAGE_KERNEL_EXEC	__pgprot(_PAGE_KERNEL | _PAGE_EXEC)
 
 /*
  * The i386 can't do page protection for execute, and considers that the same
@@ -117,7 +123,8 @@ static inline int pte_read(pte_t pte)
 
 static inline int pte_exec(pte_t pte)
 {
-	return !pte_get_bits(pte, _PAGE_PROTNONE);
+	return pte_get_bits(pte, _PAGE_EXEC) &&
+	       !pte_get_bits(pte, _PAGE_PROTNONE);
 }
 
 static inline int pte_write(pte_t pte)
diff --git a/arch/um/include/asm/ptrace-generic.h b/arch/um/include/asm/ptrace-generic.h
index 86d74f9d33cf2d..62e9916078ec6e 100644
--- a/arch/um/include/asm/ptrace-generic.h
+++ b/arch/um/include/asm/ptrace-generic.h
@@ -29,6 +29,12 @@ struct pt_regs {
 
 #define PTRACE_OLDSETOPTIONS 21
 
+#ifdef CONFIG_BINFMT_ELF_FDPIC
+#define PTRACE_GETFDPIC		31
+#define PTRACE_GETFDPIC_EXEC	0
+#define PTRACE_GETFDPIC_INTERP	1
+#endif
+
 struct task_struct;
 
 extern long subarch_ptrace(struct task_struct *child, long request,
diff --git a/arch/um/include/asm/setup.h b/arch/um/include/asm/setup.h
index 80ada899f25426..bc83dc4d467d30 100644
--- a/arch/um/include/asm/setup.h
+++ b/arch/um/include/asm/setup.h
@@ -6,6 +6,6 @@
  * command line, so this choice is ok.
  */
 
-#define COMMAND_LINE_SIZE 4096
+#define COMMAND_LINE_SIZE CONFIG_COMMAND_LINE_SIZE
 
 #endif		/* SETUP_H_INCLUDED */
diff --git a/arch/um/include/asm/tlbflush.h b/arch/um/include/asm/tlbflush.h
index 13a3009942be6b..f73b568a08595f 100644
--- a/arch/um/include/asm/tlbflush.h
+++ b/arch/um/include/asm/tlbflush.h
@@ -30,6 +30,8 @@
  *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
  */
 
+#ifdef CONFIG_MMU
+
 extern int um_tlb_sync(struct mm_struct *mm);
 
 extern void flush_tlb_all(void);
@@ -56,4 +58,22 @@ static inline void flush_tlb_kernel_range(unsigned long start,
 	um_tlb_sync(&init_mm);
 }
 
+#else /* !CONFIG_MMU */
+
+/*
+ * With NOMMU the kernel and userspace share a single host address space,
+ * so there is nothing to synchronise and all TLB flushes are no-ops.
+ */
+static inline int um_tlb_sync(struct mm_struct *mm) { return 0; }
+static inline void flush_tlb_all(void) { }
+static inline void flush_tlb_mm(struct mm_struct *mm) { }
+static inline void flush_tlb_page(struct vm_area_struct *vma,
+				  unsigned long address) { }
+static inline void flush_tlb_range(struct vm_area_struct *vma,
+				   unsigned long start, unsigned long end) { }
+static inline void flush_tlb_kernel_range(unsigned long start,
+					  unsigned long end) { }
+
+#endif /* CONFIG_MMU */
+
 #endif
diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h
index 4417c8b1d37a6d..e493596f6663c8 100644
--- a/arch/um/include/asm/uaccess.h
+++ b/arch/um/include/asm/uaccess.h
@@ -18,6 +18,7 @@
 #define __addr_range_nowrap(addr, size) \
 	((unsigned long) (addr) <= ((unsigned long) (addr) + (size)))
 
+#ifdef CONFIG_MMU
 extern unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n);
 extern unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n);
 extern unsigned long __clear_user(void __user *mem, unsigned long len);
@@ -29,8 +30,6 @@ static inline int __access_ok(const void __user *ptr, unsigned long size);
 
 #define INLINE_COPY_USER
 
-#include <asm-generic/uaccess.h>
-
 static inline int __access_ok(const void __user *ptr, unsigned long size)
 {
 	unsigned long addr = (unsigned long)ptr;
@@ -62,5 +61,8 @@ do {									\
 	barrier();							\
 	current->thread.segv_continue = NULL;				\
 } while (0)
+#endif
+
+#include <asm-generic/uaccess.h>
 
 #endif
diff --git a/arch/um/include/shared/as-layout.h b/arch/um/include/shared/as-layout.h
index 02ef258e3395e0..7c4ccb6d9d37f7 100644
--- a/arch/um/include/shared/as-layout.h
+++ b/arch/um/include/shared/as-layout.h
@@ -29,6 +29,7 @@
 
 #ifndef __ASSEMBLER__
 
+#include <linux/compiler_types.h>
 #include <sysdep/ptrace.h>
 
 struct task_struct;
@@ -46,7 +47,7 @@ extern unsigned long brk_start;
 
 extern unsigned long stub_start;
 
-extern int linux_main(int argc, char **argv, char **envp);
+extern void __noreturn linux_main(int argc, char **argv, char **envp);
 extern void uml_finishsetup(void);
 
 struct siginfo;
diff --git a/arch/um/include/shared/kern_util.h b/arch/um/include/shared/kern_util.h
index 9812efd14ec0ee..7950d58a4ae851 100644
--- a/arch/um/include/shared/kern_util.h
+++ b/arch/um/include/shared/kern_util.h
@@ -6,6 +6,7 @@
 #ifndef __KERN_UTIL_H__
 #define __KERN_UTIL_H__
 
+#include <linux/compiler_types.h>
 #include <sysdep/ptrace.h>
 #include <sysdep/faultinfo.h>
 
@@ -37,7 +38,7 @@ extern void timer_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs
 
 extern void uml_pm_wake(void);
 
-extern int start_uml(void);
+extern void __noreturn start_uml(void);
 
 extern void uml_cleanup(void);
 extern void do_uml_exitcalls(void);
diff --git a/arch/um/include/shared/longjmp.h b/arch/um/include/shared/longjmp.h
index c53e43d980c85f..b35ed756e3e4e1 100644
--- a/arch/um/include/shared/longjmp.h
+++ b/arch/um/include/shared/longjmp.h
@@ -2,11 +2,12 @@
 #ifndef __UML_LONGJMP_H
 #define __UML_LONGJMP_H
 
+#include <linux/compiler_types.h>
 #include <sysdep/archsetjmp.h>
 #include <os.h>
 
 extern int setjmp(jmp_buf);
-extern void longjmp(jmp_buf, int);
+extern void __noreturn longjmp(jmp_buf, int);
 
 #define UML_LONGJMP(buf, val) do { \
 	longjmp(*buf, val);	\
diff --git a/arch/um/include/shared/os.h b/arch/um/include/shared/os.h
index b26e94292fc14d..0067c5208c4ccd 100644
--- a/arch/um/include/shared/os.h
+++ b/arch/um/include/shared/os.h
@@ -15,6 +15,7 @@
 #ifndef __UM_HOST__
 #include <linux/types.h>
 #else
+#include <stdbool.h>
 #include <sys/types.h>
 #endif
 
@@ -196,6 +197,9 @@ extern int create_mem_file(unsigned long long len);
 /* tlb.c */
 extern void report_enomem(void);
 
+/* main.c */
+extern void __noreturn os_exit(bool reboot);
+
 /* process.c */
 pid_t os_reap_child(void);
 extern void os_alarm_process(int pid);
@@ -295,7 +299,7 @@ extern int start_userspace(struct mm_id *mm_id);
 extern void userspace(struct uml_pt_regs *regs);
 extern void new_thread(void *stack, jmp_buf *buf, void (*handler)(void));
 extern void switch_threads(jmp_buf *me, jmp_buf *you);
-extern int start_idle_thread(void *stack, jmp_buf *switch_buf);
+extern void __noreturn start_idle_thread(void *stack, jmp_buf *switch_buf);
 extern void initial_thread_cb_skas(void (*proc)(void *),
 				 void *arg);
 extern void halt_skas(void);
diff --git a/arch/um/include/shared/skas/stub-data.h b/arch/um/include/shared/skas/stub-data.h
index 27db38e95df916..1230a325debcd9 100644
--- a/arch/um/include/shared/skas/stub-data.h
+++ b/arch/um/include/shared/skas/stub-data.h
@@ -9,11 +9,20 @@
 #define __STUB_DATA_H
 
 #include <linux/compiler_types.h>
+#include <asm/unistd.h>
 #include <as-layout.h>
 #include <sysdep/tls.h>
 #include <sysdep/stub-data.h>
 #include <mm_id.h>
 
+#ifndef __NR_close_range
+# if defined(__alpha__)
+#  define __NR_close_range 546
+# else
+#  define __NR_close_range 436
+# endif
+#endif
+
 #define FUTEX_IN_CHILD 0
 #define FUTEX_IN_KERN 1
 
diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile
index be60bc451b3f9b..b087b4278173fc 100644
--- a/arch/um/kernel/Makefile
+++ b/arch/um/kernel/Makefile
@@ -16,9 +16,13 @@ always-$(KBUILD_BUILTIN) := vmlinux.lds
 
 obj-y = config.o exec.o exitcode.o irq.o ksyms.o mem.o \
 	physmem.o process.o ptrace.o reboot.o sigio.o \
-	signal.o sysrq.o time.o tlb.o trap.o \
+	signal.o sysrq.o time.o trap.o \
 	um_arch.o umid.o kmsg_dump.o capflags.o skas/
 obj-y += load_file.o
+obj-$(CONFIG_MMU) += mem-pgtable.o tlb.o trap-mmu.o
+ifneq ($(CONFIG_MMU),y)
+obj-y += trap-nommu.o
+endif
 
 obj-$(CONFIG_BLK_DEV_INITRD) += initrd.o
 obj-$(CONFIG_GPROF)	+= gprof_syms.o
diff --git a/arch/um/kernel/dyn.lds.S b/arch/um/kernel/dyn.lds.S
index ad3cefeff2acb1..31adf945150740 100644
--- a/arch/um/kernel/dyn.lds.S
+++ b/arch/um/kernel/dyn.lds.S
@@ -20,9 +20,12 @@ SECTIONS
   __binary_start = .;
   . = ALIGN(4096);		/* Init code and data */
   _text = .;
+  __init_begin = .;
   INIT_TEXT_SECTION(PAGE_SIZE)
 
+  init.data : { INIT_DATA }
   . = ALIGN(PAGE_SIZE);
+  __init_end = .;
 
   /* Read-only sections, merged into text segment: */
   .hash           : { *(.hash) }
@@ -98,10 +101,6 @@ SECTIONS
 
   #include <asm/common.lds.S>
 
-  __init_begin = .;
-  init.data : { INIT_DATA }
-  __init_end = .;
-
   /* Ensure the __preinit_array_start label is properly aligned.  We
      could instead move the label definition inside the section, but
      the linker would then create the section even if it turns out to
@@ -117,6 +116,7 @@ SECTIONS
   .data           : {
     INIT_TASK_DATA(KERNEL_STACK_SIZE)
     DATA_DATA
+    *(.ldata .ldata.[0-9a-zA-Z_]* .ldata..L*)
     *(.data.* .gnu.linkonce.d.*)
     SORT(CONSTRUCTORS)
   }
@@ -159,6 +159,7 @@ SECTIONS
    __bss_start = .;
    *(.dynbss)
    *(.bss .bss.* .gnu.linkonce.b.*)
+   *(.lbss .lbss.[0-9a-zA-Z_]* .lbss..L*)
    *(COMMON)
    /* Align here to ensure that the .bss section occupies space up to
       _end.  Align after .bss to ensure correct alignment even if the
diff --git a/arch/um/kernel/mem-pgtable.c b/arch/um/kernel/mem-pgtable.c
new file mode 100644
index 00000000000000..e33894431acde3
--- /dev/null
+++ b/arch/um/kernel/mem-pgtable.c
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
+ */
+
+#include <linux/stddef.h>
+#include <linux/module.h>
+#include <linux/memblock.h>
+#include <linux/swap.h>
+#include <linux/slab.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <as-layout.h>
+#include <init.h>
+#include <kern.h>
+#include <kern_util.h>
+#include <mem_user.h>
+#include <os.h>
+#include <um_malloc.h>
+
+
+/* Allocate and free page tables. */
+
+pgd_t *pgd_alloc(struct mm_struct *mm)
+{
+	pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
+
+	if (pgd) {
+		memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
+		memcpy(pgd + USER_PTRS_PER_PGD,
+		       swapper_pg_dir + USER_PTRS_PER_PGD,
+		       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
+	}
+	return pgd;
+}
+
+static const pgprot_t protection_map[16] = {
+	[VM_NONE]					= PAGE_NONE,
+	[VM_READ]					= PAGE_READONLY,
+	[VM_WRITE]					= PAGE_COPY,
+	[VM_WRITE | VM_READ]				= PAGE_COPY,
+	[VM_EXEC]					= PAGE_READONLY_EXEC,
+	[VM_EXEC | VM_READ]				= PAGE_READONLY_EXEC,
+	[VM_EXEC | VM_WRITE]				= PAGE_COPY_EXEC,
+	[VM_EXEC | VM_WRITE | VM_READ]			= PAGE_COPY_EXEC,
+	[VM_SHARED]					= PAGE_NONE,
+	[VM_SHARED | VM_READ]				= PAGE_READONLY,
+	[VM_SHARED | VM_WRITE]				= PAGE_SHARED,
+	[VM_SHARED | VM_WRITE | VM_READ]		= PAGE_SHARED,
+	[VM_SHARED | VM_EXEC]				= PAGE_READONLY_EXEC,
+	[VM_SHARED | VM_EXEC | VM_READ]			= PAGE_READONLY_EXEC,
+	[VM_SHARED | VM_EXEC | VM_WRITE]		= PAGE_SHARED_EXEC,
+	[VM_SHARED | VM_EXEC | VM_WRITE | VM_READ]	= PAGE_SHARED_EXEC
+};
+DECLARE_VM_GET_PAGE_PROT
diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index 1eef0e42ef5dba..2219827ac3adaf 100644
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -6,7 +6,6 @@
 #include <linux/stddef.h>
 #include <linux/module.h>
 #include <linux/memblock.h>
-#include <linux/mm.h>
 #include <linux/swap.h>
 #include <linux/slab.h>
 #include <linux/init.h>
@@ -65,7 +64,8 @@ void __init arch_mm_preinit(void)
 	 * to be turned on.
 	 */
 	brk_end = PAGE_ALIGN((unsigned long) sbrk(0));
-	map_memory(brk_end, __pa(brk_end), uml_reserved - brk_end, 1, 1, 0);
+	map_memory(brk_end, __pa(brk_end), uml_reserved - brk_end, 1, 1,
+		   !IS_ENABLED(CONFIG_MMU));
 	memblock_free((void *)brk_end, uml_reserved - brk_end);
 	uml_reserved = brk_end;
 	min_low_pfn = PFN_UP(__pa(uml_reserved));
@@ -83,26 +83,17 @@ void __init arch_zone_limits_init(unsigned long *max_zone_pfns)
 }
 
 /*
- * This can't do anything because nothing in the kernel image can be freed
- * since it's not in kernel physical memory.
+ * We could munmap() this instead, but then libc allocations could
+ * land in this area and stray initdata access could erroneously
+ * succeed - just mprotect() it to reliably catch bad accesses.
  */
-
 void free_initmem(void)
 {
-}
+	unsigned long start = PAGE_ALIGN((unsigned long)__init_begin);
+	unsigned long end = round_down((unsigned long)__init_end, PAGE_SIZE);
 
-/* Allocate and free page tables. */
-
-pgd_t *pgd_alloc(struct mm_struct *mm)
-{
-	pgd_t *pgd = __pgd_alloc(mm, 0);
-
-	if (pgd)
-		memcpy(pgd + USER_PTRS_PER_PGD,
-		       swapper_pg_dir + USER_PTRS_PER_PGD,
-		       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
-
-	return pgd;
+	if (end > start)
+		os_protect_memory((void *)start, end - start, 0, 0, 0);
 }
 
 void *uml_kmalloc(int size, int flags)
@@ -110,26 +101,6 @@ void *uml_kmalloc(int size, int flags)
 	return kmalloc(size, flags);
 }
 
-static const pgprot_t protection_map[16] = {
-	[VM_NONE]					= PAGE_NONE,
-	[VM_READ]					= PAGE_READONLY,
-	[VM_WRITE]					= PAGE_COPY,
-	[VM_WRITE | VM_READ]				= PAGE_COPY,
-	[VM_EXEC]					= PAGE_READONLY,
-	[VM_EXEC | VM_READ]				= PAGE_READONLY,
-	[VM_EXEC | VM_WRITE]				= PAGE_COPY,
-	[VM_EXEC | VM_WRITE | VM_READ]			= PAGE_COPY,
-	[VM_SHARED]					= PAGE_NONE,
-	[VM_SHARED | VM_READ]				= PAGE_READONLY,
-	[VM_SHARED | VM_WRITE]				= PAGE_SHARED,
-	[VM_SHARED | VM_WRITE | VM_READ]		= PAGE_SHARED,
-	[VM_SHARED | VM_EXEC]				= PAGE_READONLY,
-	[VM_SHARED | VM_EXEC | VM_READ]			= PAGE_READONLY,
-	[VM_SHARED | VM_EXEC | VM_WRITE]		= PAGE_SHARED,
-	[VM_SHARED | VM_EXEC | VM_WRITE | VM_READ]	= PAGE_SHARED
-};
-DECLARE_VM_GET_PAGE_PROT
-
 void mark_rodata_ro(void)
 {
 	unsigned long rodata_start = PFN_ALIGN(__start_rodata);
diff --git a/arch/um/kernel/physmem.c b/arch/um/kernel/physmem.c
index ae6ca373c26109..ae9b8f3144b895 100644
--- a/arch/um/kernel/physmem.c
+++ b/arch/um/kernel/physmem.c
@@ -110,6 +110,13 @@ int phys_mapping(unsigned long phys, unsigned long long *offset_out)
 }
 EXPORT_SYMBOL(phys_mapping);
 
+void report_enomem(void)
+{
+	printk(KERN_ERR "UML ran out of memory on the host side! "
+			"This can happen due to a memory limitation or "
+			"vm.max_map_count has been reached.\n");
+}
+
 static int __init uml_mem_setup(char *line, int *add)
 {
 	char *retptr;
diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c
index 63b38a3f73f78d..dfca9e0aeec9c0 100644
--- a/arch/um/kernel/process.c
+++ b/arch/um/kernel/process.c
@@ -276,16 +276,12 @@ unsigned long arch_align_stack(unsigned long sp)
 }
 #endif
 
-unsigned long __get_wchan(struct task_struct *p)
+static unsigned long ___get_wchan(struct task_struct *p)
 {
 	unsigned long stack_page, sp, ip;
 	bool seen_sched = 0;
 
 	stack_page = (unsigned long) task_stack_page(p);
-	/* Bail if the process has no kernel stack for some reason */
-	if (stack_page == 0)
-		return 0;
-
 	sp = p->thread.switch_buf->JB_SP;
 	/*
 	 * Bail if the stack pointer is below the bottom of the kernel
@@ -306,4 +302,21 @@ unsigned long __get_wchan(struct task_struct *p)
 	}
 
 	return 0;
+
+}
+
+unsigned long __get_wchan(struct task_struct *p)
+{
+	unsigned long ret;
+
+	/* Bail if the process has no kernel stack for some reason */
+	if (!try_get_task_stack(p))
+		return 0;
+
+	ret = ___get_wchan(p);
+
+	put_task_stack(p);
+
+	return ret;
+
 }
diff --git a/arch/um/kernel/skas/Makefile b/arch/um/kernel/skas/Makefile
index 3384be42691f89..2cf82eba52f85d 100644
--- a/arch/um/kernel/skas/Makefile
+++ b/arch/um/kernel/skas/Makefile
@@ -3,8 +3,12 @@
 # Copyright (C) 2002 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
 #
 
-obj-y := stub.o mmu.o process.o syscall.o uaccess.o \
+obj-y := stub.o process.o syscall.o \
 	 stub_exe_embed.o
+obj-$(CONFIG_MMU) += mmu.o uaccess.o
+ifneq ($(CONFIG_MMU),y)
+obj-y += nommu.o
+endif
 
 # Stub executable
 
@@ -40,8 +44,14 @@ targets += stub_exe.dbg stub_exe $(stub_exe_objs-y)
 CFLAGS_stub.o := $(CFLAGS_NO_HARDENING)
 CFLAGS_stub_exe.o := $(CFLAGS_NO_HARDENING)
 
-# Clang will call memset() from __builtin_alloca() when stack variable
-# initialization is enabled, which is used in stub_exe.c.
+# stub_signal_restorer() must not touch stack pointer, clang doesn't
+# always omit the frame pointer automatically
+CFLAGS_stub.o += -fomit-frame-pointer
+CFLAGS_stub_exe.o += -fomit-frame-pointer
+
+# Clang's stack var initialisation can use memset() or similar and that
+# cannot be done for this code since we embed/move the stub
+CFLAGS_stub.o += $(call cc-option, -ftrivial-auto-var-init=uninitialized)
 CFLAGS_stub_exe.o += $(call cc-option, -ftrivial-auto-var-init=uninitialized)
 
 UNPROFILE_OBJS := stub.o stub_exe.o
diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c
index b5017096028ba1..facb88d1e393a6 100644
--- a/arch/um/kernel/skas/mmu.c
+++ b/arch/um/kernel/skas/mmu.c
@@ -12,6 +12,7 @@
 #include <asm/pgalloc.h>
 #include <asm/sections.h>
 #include <asm/mmu_context.h>
+#include <asm/tlbflush.h>
 #include <as-layout.h>
 #include <os.h>
 #include <skas.h>
@@ -40,6 +41,30 @@ void exit_turnstile(struct mm_id *mm_id)
 	mutex_unlock(__get_turnstile(mm_id));
 }
 
+unsigned long current_stub_stack(void)
+{
+	if (current->mm == NULL)
+		return 0;
+
+	return current->mm->context.id.stack;
+}
+
+struct mm_id *current_mm_id(void)
+{
+	if (current->mm == NULL)
+		return NULL;
+
+	return &current->mm->context.id;
+}
+
+void current_mm_sync(void)
+{
+	if (current->mm == NULL)
+		return;
+
+	um_tlb_sync(current->mm);
+}
+
 int init_new_context(struct task_struct *task, struct mm_struct *mm)
 {
 	struct mm_id *new_id = &mm->context.id;
diff --git a/arch/um/kernel/skas/nommu.c b/arch/um/kernel/skas/nommu.c
new file mode 100644
index 00000000000000..80d5c4eaa96bd0
--- /dev/null
+++ b/arch/um/kernel/skas/nommu.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/cpumask.h>
+#include <linux/gfp.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/log2.h>
+#include <linux/mutex.h>
+#include <linux/panic.h>
+#include <linux/smp.h>
+#include <linux/threads.h>
+#include <as-layout.h>
+#include <mem.h>
+#include <os.h>
+#include <skas.h>
+#include <mm_id.h>
+
+struct nommu_runner {
+	struct mm_id mm_id;
+	struct mutex turnstile;
+};
+
+static struct nommu_runner nommu_runners[NR_CPUS];
+
+static struct nommu_runner *this_runner(void)
+{
+	return &nommu_runners[raw_smp_processor_id()];
+}
+
+struct mutex *__get_turnstile(struct mm_id *mm_id)
+{
+	return &container_of(mm_id, struct nommu_runner, mm_id)->turnstile;
+}
+
+void enter_turnstile(struct mm_id *mm_id)
+{
+	mutex_lock(__get_turnstile(mm_id));
+}
+
+void exit_turnstile(struct mm_id *mm_id)
+{
+	mutex_unlock(__get_turnstile(mm_id));
+}
+
+unsigned long current_stub_stack(void)
+{
+	return this_runner()->mm_id.stack;
+}
+
+struct mm_id *current_mm_id(void)
+{
+	return &this_runner()->mm_id;
+}
+
+void current_mm_sync(void)
+{
+}
+
+static int __init nommu_start_runners(void)
+{
+	unsigned long long offset;
+	struct nommu_runner *r;
+	int cpu, err, fd;
+
+	fd = phys_mapping(uml_reserved - uml_physmem, &offset);
+
+	for_each_possible_cpu(cpu) {
+		r = &nommu_runners[cpu];
+		mutex_init(&r->turnstile);
+
+		r->mm_id.stack = __get_free_pages(GFP_KERNEL | __GFP_ZERO,
+						  ilog2(STUB_DATA_PAGES));
+		if (!r->mm_id.stack)
+			panic("OOM allocating userspace stack for CPU %d", cpu);
+
+		err = start_userspace(&r->mm_id);
+		if (err < 0)
+			panic("userspace startup for CPU %d failed: %d",
+			      cpu, err);
+
+		map(&r->mm_id, uml_reserved, high_physmem - uml_reserved,
+		    UM_PROT_READ | UM_PROT_WRITE | UM_PROT_EXEC, fd, offset);
+
+		err = syscall_stub_flush(&r->mm_id);
+		if (err < 0)
+			panic("physmem map/flush failed for CPU%d: %d",
+			      cpu, err);
+	}
+
+	return 0;
+}
+arch_initcall(nommu_start_runners);
diff --git a/arch/um/kernel/skas/process.c b/arch/um/kernel/skas/process.c
index 4a7673b0261a88..f60541df209fe5 100644
--- a/arch/um/kernel/skas/process.c
+++ b/arch/um/kernel/skas/process.c
@@ -9,8 +9,6 @@
 #include <linux/sched/task.h>
 #include <linux/smp-internal.h>
 
-#include <asm/tlbflush.h>
-
 #include <as-layout.h>
 #include <kern.h>
 #include <os.h>
@@ -29,7 +27,7 @@ static int __init start_kernel_proc(void *unused)
 
 char cpu_irqstacks[NR_CPUS][THREAD_SIZE] __aligned(THREAD_SIZE);
 
-int __init start_uml(void)
+void __init start_uml(void)
 {
 	stack_protections((unsigned long) &cpu_irqstacks[0]);
 	set_sigstack(cpu_irqstacks[0], THREAD_SIZE);
@@ -38,32 +36,8 @@ int __init start_uml(void)
 
 	init_task.thread.request.thread.proc = start_kernel_proc;
 	init_task.thread.request.thread.arg = NULL;
-	return start_idle_thread(task_stack_page(&init_task),
-				 &init_task.thread.switch_buf);
-}
-
-unsigned long current_stub_stack(void)
-{
-	if (current->mm == NULL)
-		return 0;
-
-	return current->mm->context.id.stack;
-}
-
-struct mm_id *current_mm_id(void)
-{
-	if (current->mm == NULL)
-		return NULL;
-
-	return &current->mm->context.id;
-}
-
-void current_mm_sync(void)
-{
-	if (current->mm == NULL)
-		return;
-
-	um_tlb_sync(current->mm);
+	start_idle_thread(task_stack_page(&init_task),
+			  &init_task.thread.switch_buf);
 }
 
 static DEFINE_SPINLOCK(initial_jmpbuf_spinlock);
diff --git a/arch/um/kernel/skas/syscall.c b/arch/um/kernel/skas/syscall.c
index bc117733e8effe..914b90af42ed2e 100644
--- a/arch/um/kernel/skas/syscall.c
+++ b/arch/um/kernel/skas/syscall.c
@@ -35,12 +35,13 @@ void handle_syscall(struct uml_pt_regs *r)
 	/*
 	 * If no time passes, then sched_yield may not actually yield, causing
 	 * broken spinlock implementations in userspace (ASAN) to hang for long
-	 * periods of time.
+	 * periods of time. Advance the time-travel clock a bit so sched_yield()
+	 * in a loop (like ping does for small intervals) also works correctly.
 	 */
 	if ((time_travel_mode == TT_MODE_INFCPU ||
 	     time_travel_mode == TT_MODE_EXTERNAL) &&
 	    syscall == __NR_sched_yield)
-		tt_extra_sched_jiffies += 1;
+		time_travel_ndelay(1000);
 
 	if (syscall >= 0 && syscall < __NR_syscalls) {
 		unsigned long ret;
diff --git a/arch/um/kernel/skas/uaccess.c b/arch/um/kernel/skas/uaccess.c
index caef1deef795c0..f48f0e7c2497ca 100644
--- a/arch/um/kernel/skas/uaccess.c
+++ b/arch/um/kernel/skas/uaccess.c
@@ -231,116 +231,90 @@ long strnlen_user(const char __user *str, long len)
 }
 EXPORT_SYMBOL(strnlen_user);
 
-/**
- * arch_futex_atomic_op_inuser() - Atomic arithmetic operation with constant
- *			  argument and comparison of the previous
- *			  futex value with another constant.
- *
- * @op:		operation to execute
- * @oparg:	argument to operation
- * @oval:	old value at uaddr
- * @uaddr:	pointer to user space address
- *
- * Return:
- * 0 - On success
- * -EFAULT - User access resulted in a page fault
- * -EAGAIN - Atomic operation was unable to complete due to contention
- * -ENOSYS - Operation not supported
- */
-
 int arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr)
 {
-	int oldval, ret;
-	struct page *page;
 	unsigned long addr = (unsigned long) uaddr;
+	u32 *p, oldval, newval;
+	struct page *page;
 	pte_t *pte;
 
-	ret = -EFAULT;
 	if (!access_ok(uaddr, sizeof(*uaddr)))
 		return -EFAULT;
-	preempt_disable();
-	pte = maybe_map(addr, 1);
-	if (pte == NULL)
-		goto out_inuser;
-
-	page = pte_page(*pte);
-#ifdef CONFIG_64BIT
-	pagefault_disable();
-	addr = (unsigned long) page_address(page) +
-			(((unsigned long) addr) & ~PAGE_MASK);
-#else
-	addr = (unsigned long) kmap_atomic(page) +
-		((unsigned long) addr & ~PAGE_MASK);
-#endif
-	uaddr = (u32 *) addr;
-	oldval = *uaddr;
-
-	ret = 0;
 
 	switch (op) {
 	case FUTEX_OP_SET:
-		*uaddr = oparg;
-		break;
 	case FUTEX_OP_ADD:
-		*uaddr += oparg;
-		break;
 	case FUTEX_OP_OR:
-		*uaddr |= oparg;
-		break;
 	case FUTEX_OP_ANDN:
-		*uaddr &= ~oparg;
-		break;
 	case FUTEX_OP_XOR:
-		*uaddr ^= oparg;
 		break;
 	default:
-		ret = -ENOSYS;
+		return -ENOSYS;
+	}
+
+	preempt_disable();
+	pte = maybe_map(addr, 1);
+	if (pte == NULL) {
+		preempt_enable();
+		return -EFAULT;
 	}
+
+	page = pte_page(*pte);
 #ifdef CONFIG_64BIT
-	pagefault_enable();
+	pagefault_disable();
+	p = page_address(page) + (addr & ~PAGE_MASK);
 #else
-	kunmap_atomic((void *)addr);
+	p = kmap_atomic(page) + (addr & ~PAGE_MASK);
 #endif
 
-out_inuser:
-	preempt_enable();
+	do {
+		oldval = READ_ONCE(*p);
+
+		switch (op) {
+		case FUTEX_OP_SET:
+			newval = oparg;
+			break;
+		case FUTEX_OP_ADD:
+			newval = oldval + oparg;
+			break;
+		case FUTEX_OP_OR:
+			newval = oldval | oparg;
+			break;
+		case FUTEX_OP_ANDN:
+			newval = oldval & ~oparg;
+			break;
+		case FUTEX_OP_XOR:
+			newval = oldval ^ oparg;
+			break;
+		}
+	} while (!try_cmpxchg(p, &oldval, newval));
 
-	if (ret == 0)
-		*oval = oldval;
+	*oval = oldval;
 
-	return ret;
+#ifdef CONFIG_64BIT
+	pagefault_enable();
+#else
+	kunmap_atomic((void *)addr);
+#endif
+	preempt_enable();
+	return 0;
 }
-EXPORT_SYMBOL(arch_futex_atomic_op_inuser);
-
-/**
- * futex_atomic_cmpxchg_inatomic() - Compare and exchange the content of the
- *				uaddr with newval if the current value is
- *				oldval.
- * @uval:	pointer to store content of @uaddr
- * @uaddr:	pointer to user space address
- * @oldval:	old value
- * @newval:	new value to store to @uaddr
- *
- * Return:
- * 0 - On success
- * -EFAULT - User access resulted in a page fault
- * -EAGAIN - Atomic operation was unable to complete due to contention
- */
 
 int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
-			      u32 oldval, u32 newval)
+				  u32 oldval, u32 newval)
 {
 	struct page *page;
 	pte_t *pte;
-	int ret = -EFAULT;
 
 	if (!access_ok(uaddr, sizeof(*uaddr)))
 		return -EFAULT;
 
 	preempt_disable();
 	pte = maybe_map((unsigned long) uaddr, 1);
-	if (pte == NULL)
-		goto out_inatomic;
+	if (pte == NULL) {
+		preempt_enable();
+		return -EFAULT;
+	}
 
 	page = pte_page(*pte);
 #ifdef CONFIG_64BIT
@@ -350,19 +324,13 @@ int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 	uaddr = kmap_atomic(page) + ((unsigned long) uaddr & ~PAGE_MASK);
 #endif
 
-	*uval = *uaddr;
-
-	ret = cmpxchg(uaddr, oldval, newval);
+	*uval = cmpxchg(uaddr, oldval, newval);
 
 #ifdef CONFIG_64BIT
 	pagefault_enable();
 #else
 	kunmap_atomic(uaddr);
 #endif
-	ret = 0;
-
-out_inatomic:
 	preempt_enable();
-	return ret;
+	return 0;
 }
-EXPORT_SYMBOL(futex_atomic_cmpxchg_inatomic);
diff --git a/arch/um/kernel/smp.c b/arch/um/kernel/smp.c
index f1e52b7348fb8a..92ea1beaed8b21 100644
--- a/arch/um/kernel/smp.c
+++ b/arch/um/kernel/smp.c
@@ -20,6 +20,7 @@
 #include <init.h>
 #include <kern.h>
 #include <os.h>
+#include <skas.h>
 #include <smp.h>
 
 enum {
@@ -66,7 +67,7 @@ static void ipi_handler(int vector, struct uml_pt_regs *regs)
 	irq_enter();
 
 	if (current->mm)
-		os_alarm_process(current->mm->context.id.pid);
+		os_alarm_process(current_mm_id()->pid);
 
 	switch (vector) {
 	case UML_IPI_RES:
diff --git a/arch/um/kernel/sysrq.c b/arch/um/kernel/sysrq.c
index 13ee5666668dd4..5a2c28e0929492 100644
--- a/arch/um/kernel/sysrq.c
+++ b/arch/um/kernel/sysrq.c
@@ -18,7 +18,7 @@ static void _print_addr(void *data, unsigned long address, int reliable)
 {
 	const char *loglvl = data;
 
-	printk("%s [<%08lx>] %s%pS\n", loglvl, address, reliable ? "" : "? ",
+	printk("%s [<%08lx>] %s%pB\n", loglvl, address, reliable ? "" : "? ",
 		(void *)address);
 }
 
diff --git a/arch/um/kernel/time.c b/arch/um/kernel/time.c
index b344a36b44eb16..d620b6d6310a54 100644
--- a/arch/um/kernel/time.c
+++ b/arch/um/kernel/time.c
@@ -19,6 +19,7 @@
 #include <asm/param.h>
 #include <kern_util.h>
 #include <os.h>
+#include <skas.h>
 #include <linux/delay.h>
 #include <linux/time-internal.h>
 #include <linux/um_timetravel.h>
@@ -873,7 +874,7 @@ static irqreturn_t um_timer(int irq, void *dev)
 	if (time_travel_mode != TT_MODE_INFCPU &&
 	    time_travel_mode != TT_MODE_EXTERNAL &&
 	    get_current()->mm)
-		os_alarm_process(get_current()->mm->context.id.pid);
+		os_alarm_process(current_mm_id()->pid);
 
 	evt->event_handler(evt);
 
diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c
index 1f175716b47478..012bcc76168df5 100644
--- a/arch/um/kernel/tlb.c
+++ b/arch/um/kernel/tlb.c
@@ -40,13 +40,6 @@ static int kern_unmap(struct mm_id *mm_idp,
 	return os_unmap_memory((void *)virt, len);
 }
 
-void report_enomem(void)
-{
-	printk(KERN_ERR "UML ran out of memory on the host side! "
-			"This can happen due to a memory limitation or "
-			"vm.max_map_count has been reached.\n");
-}
-
 static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
 				   unsigned long end,
 				   struct vm_ops *ops)
diff --git a/arch/um/kernel/trap-mmu.c b/arch/um/kernel/trap-mmu.c
new file mode 100644
index 00000000000000..467c100aafdefd
--- /dev/null
+++ b/arch/um/kernel/trap-mmu.c
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
+ */
+
+#include <linux/mm.h>
+#include <linux/sched/signal.h>
+#include <linux/hardirq.h>
+#include <linux/module.h>
+#include <linux/uaccess.h>
+#include <linux/sched/debug.h>
+#include <asm/current.h>
+#include <asm/tlbflush.h>
+#include <arch.h>
+#include <as-layout.h>
+#include <kern_util.h>
+#include <os.h>
+#include <skas.h>
+
+/*
+ * NOTE: UML does not have exception tables. As such, this is almost a copy
+ * of the code in mm/memory.c, only adjusting the logic to simply check whether
+ * we are coming from the kernel instead of doing an additional lookup in the
+ * exception table.
+ * We can do this simplification because we never get here if the exception was
+ * fixable.
+ */
+static inline bool get_mmap_lock_carefully(struct mm_struct *mm, bool is_user)
+{
+	if (likely(mmap_read_trylock(mm)))
+		return true;
+
+	if (!is_user)
+		return false;
+
+	return !mmap_read_lock_killable(mm);
+}
+
+static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
+{
+	/*
+	 * We don't have this operation yet.
+	 *
+	 * It should be easy enough to do: it's basically a
+	 *    atomic_long_try_cmpxchg_acquire()
+	 * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
+	 * it also needs the proper lockdep magic etc.
+	 */
+	return false;
+}
+
+static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, bool is_user)
+{
+	mmap_read_unlock(mm);
+	if (!is_user)
+		return false;
+
+	return !mmap_write_lock_killable(mm);
+}
+
+/*
+ * Helper for page fault handling.
+ *
+ * This is kind of equivalend to "mmap_read_lock()" followed
+ * by "find_extend_vma()", except it's a lot more careful about
+ * the locking (and will drop the lock on failure).
+ *
+ * For example, if we have a kernel bug that causes a page
+ * fault, we don't want to just use mmap_read_lock() to get
+ * the mm lock, because that would deadlock if the bug were
+ * to happen while we're holding the mm lock for writing.
+ *
+ * So this checks the exception tables on kernel faults in
+ * order to only do this all for instructions that are actually
+ * expected to fault.
+ *
+ * We can also actually take the mm lock for writing if we
+ * need to extend the vma, which helps the VM layer a lot.
+ */
+static struct vm_area_struct *
+um_lock_mm_and_find_vma(struct mm_struct *mm,
+			unsigned long addr, bool is_user)
+{
+	struct vm_area_struct *vma;
+
+	if (!get_mmap_lock_carefully(mm, is_user))
+		return NULL;
+
+	vma = find_vma(mm, addr);
+	if (likely(vma && (vma->vm_start <= addr)))
+		return vma;
+
+	/*
+	 * Well, dang. We might still be successful, but only
+	 * if we can extend a vma to do so.
+	 */
+	if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) {
+		mmap_read_unlock(mm);
+		return NULL;
+	}
+
+	/*
+	 * We can try to upgrade the mmap lock atomically,
+	 * in which case we can continue to use the vma
+	 * we already looked up.
+	 *
+	 * Otherwise we'll have to drop the mmap lock and
+	 * re-take it, and also look up the vma again,
+	 * re-checking it.
+	 */
+	if (!mmap_upgrade_trylock(mm)) {
+		if (!upgrade_mmap_lock_carefully(mm, is_user))
+			return NULL;
+
+		vma = find_vma(mm, addr);
+		if (!vma)
+			goto fail;
+		if (vma->vm_start <= addr)
+			goto success;
+		if (!(vma->vm_flags & VM_GROWSDOWN))
+			goto fail;
+	}
+
+	if (expand_stack_locked(vma, addr))
+		goto fail;
+
+success:
+	mmap_write_downgrade(mm);
+	return vma;
+
+fail:
+	mmap_write_unlock(mm);
+	return NULL;
+}
+
+/*
+ * Note this is constrained to return 0, -EFAULT, -EACCES, -ENOMEM by
+ * segv().
+ */
+int handle_page_fault(unsigned long address, unsigned long ip,
+		      int is_write, int is_user, int *code_out)
+{
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *vma;
+	pmd_t *pmd;
+	pte_t *pte;
+	int err = -EFAULT;
+	unsigned int flags = FAULT_FLAG_DEFAULT;
+
+	*code_out = SEGV_MAPERR;
+
+	/*
+	 * If the fault was with pagefaults disabled, don't take the fault, just
+	 * fail.
+	 */
+	if (faulthandler_disabled())
+		goto out_nosemaphore;
+
+	if (is_user)
+		flags |= FAULT_FLAG_USER;
+retry:
+	vma = um_lock_mm_and_find_vma(mm, address, is_user);
+	if (!vma)
+		goto out_nosemaphore;
+
+	*code_out = SEGV_ACCERR;
+	if (is_write) {
+		if (!(vma->vm_flags & VM_WRITE))
+			goto out;
+		flags |= FAULT_FLAG_WRITE;
+	} else {
+		/* Don't require VM_READ|VM_EXEC for write faults! */
+		if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
+			goto out;
+	}
+
+	do {
+		vm_fault_t fault;
+
+		fault = handle_mm_fault(vma, address, flags, NULL);
+
+		if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
+			goto out_nosemaphore;
+
+		/* The fault is fully completed (including releasing mmap lock) */
+		if (fault & VM_FAULT_COMPLETED)
+			return 0;
+
+		if (unlikely(fault & VM_FAULT_ERROR)) {
+			if (fault & VM_FAULT_OOM) {
+				goto out_of_memory;
+			} else if (fault & VM_FAULT_SIGSEGV) {
+				goto out;
+			} else if (fault & VM_FAULT_SIGBUS) {
+				err = -EACCES;
+				goto out;
+			}
+			BUG();
+		}
+		if (fault & VM_FAULT_RETRY) {
+			flags |= FAULT_FLAG_TRIED;
+
+			goto retry;
+		}
+
+		pmd = pmd_off(mm, address);
+		pte = pte_offset_kernel(pmd, address);
+	} while (!pte_present(*pte));
+	err = 0;
+	/*
+	 * The below warning was added in place of
+	 *	pte_mkyoung(); if (is_write) pte_mkdirty();
+	 * If it's triggered, we'd see normally a hang here (a clean pte is
+	 * marked read-only to emulate the dirty bit).
+	 * However, the generic code can mark a PTE writable but clean on a
+	 * concurrent read fault, triggering this harmlessly. So comment it out.
+	 */
+#if 0
+	WARN_ON(!pte_young(*pte) || (is_write && !pte_dirty(*pte)));
+#endif
+
+out:
+	mmap_read_unlock(mm);
+out_nosemaphore:
+	return err;
+
+out_of_memory:
+	/*
+	 * We ran out of memory, call the OOM killer, and return the userspace
+	 * (which will retry the fault, or kill us if we got oom-killed).
+	 */
+	mmap_read_unlock(mm);
+	if (!is_user)
+		goto out_nosemaphore;
+	pagefault_out_of_memory();
+	return 0;
+}
diff --git a/arch/um/kernel/trap-nommu.c b/arch/um/kernel/trap-nommu.c
new file mode 100644
index 00000000000000..891a9cefa9357e
--- /dev/null
+++ b/arch/um/kernel/trap-nommu.c
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/signal.h>
+#include <shared/kern_util.h>
+
+int handle_page_fault(unsigned long address, unsigned long ip,
+		      int is_write, int is_user, int *code_out)
+{
+	/* everything that's valid is already mapped in NOMMU */
+	*code_out = SEGV_MAPERR;
+	return -EFAULT;
+}
diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
index 177615820a4cc5..ab60bab8d987a4 100644
--- a/arch/um/kernel/trap.c
+++ b/arch/um/kernel/trap.c
@@ -17,225 +17,6 @@
 #include <os.h>
 #include <skas.h>
 
-/*
- * NOTE: UML does not have exception tables. As such, this is almost a copy
- * of the code in mm/memory.c, only adjusting the logic to simply check whether
- * we are coming from the kernel instead of doing an additional lookup in the
- * exception table.
- * We can do this simplification because we never get here if the exception was
- * fixable.
- */
-static inline bool get_mmap_lock_carefully(struct mm_struct *mm, bool is_user)
-{
-	if (likely(mmap_read_trylock(mm)))
-		return true;
-
-	if (!is_user)
-		return false;
-
-	return !mmap_read_lock_killable(mm);
-}
-
-static inline bool mmap_upgrade_trylock(struct mm_struct *mm)
-{
-	/*
-	 * We don't have this operation yet.
-	 *
-	 * It should be easy enough to do: it's basically a
-	 *    atomic_long_try_cmpxchg_acquire()
-	 * from RWSEM_READER_BIAS -> RWSEM_WRITER_LOCKED, but
-	 * it also needs the proper lockdep magic etc.
-	 */
-	return false;
-}
-
-static inline bool upgrade_mmap_lock_carefully(struct mm_struct *mm, bool is_user)
-{
-	mmap_read_unlock(mm);
-	if (!is_user)
-		return false;
-
-	return !mmap_write_lock_killable(mm);
-}
-
-/*
- * Helper for page fault handling.
- *
- * This is kind of equivalend to "mmap_read_lock()" followed
- * by "find_extend_vma()", except it's a lot more careful about
- * the locking (and will drop the lock on failure).
- *
- * For example, if we have a kernel bug that causes a page
- * fault, we don't want to just use mmap_read_lock() to get
- * the mm lock, because that would deadlock if the bug were
- * to happen while we're holding the mm lock for writing.
- *
- * So this checks the exception tables on kernel faults in
- * order to only do this all for instructions that are actually
- * expected to fault.
- *
- * We can also actually take the mm lock for writing if we
- * need to extend the vma, which helps the VM layer a lot.
- */
-static struct vm_area_struct *
-um_lock_mm_and_find_vma(struct mm_struct *mm,
-			unsigned long addr, bool is_user)
-{
-	struct vm_area_struct *vma;
-
-	if (!get_mmap_lock_carefully(mm, is_user))
-		return NULL;
-
-	vma = find_vma(mm, addr);
-	if (likely(vma && (vma->vm_start <= addr)))
-		return vma;
-
-	/*
-	 * Well, dang. We might still be successful, but only
-	 * if we can extend a vma to do so.
-	 */
-	if (!vma || !(vma->vm_flags & VM_GROWSDOWN)) {
-		mmap_read_unlock(mm);
-		return NULL;
-	}
-
-	/*
-	 * We can try to upgrade the mmap lock atomically,
-	 * in which case we can continue to use the vma
-	 * we already looked up.
-	 *
-	 * Otherwise we'll have to drop the mmap lock and
-	 * re-take it, and also look up the vma again,
-	 * re-checking it.
-	 */
-	if (!mmap_upgrade_trylock(mm)) {
-		if (!upgrade_mmap_lock_carefully(mm, is_user))
-			return NULL;
-
-		vma = find_vma(mm, addr);
-		if (!vma)
-			goto fail;
-		if (vma->vm_start <= addr)
-			goto success;
-		if (!(vma->vm_flags & VM_GROWSDOWN))
-			goto fail;
-	}
-
-	if (expand_stack_locked(vma, addr))
-		goto fail;
-
-success:
-	mmap_write_downgrade(mm);
-	return vma;
-
-fail:
-	mmap_write_unlock(mm);
-	return NULL;
-}
-
-/*
- * Note this is constrained to return 0, -EFAULT, -EACCES, -ENOMEM by
- * segv().
- */
-int handle_page_fault(unsigned long address, unsigned long ip,
-		      int is_write, int is_user, int *code_out)
-{
-	struct mm_struct *mm = current->mm;
-	struct vm_area_struct *vma;
-	pmd_t *pmd;
-	pte_t *pte;
-	int err = -EFAULT;
-	unsigned int flags = FAULT_FLAG_DEFAULT;
-
-	*code_out = SEGV_MAPERR;
-
-	/*
-	 * If the fault was with pagefaults disabled, don't take the fault, just
-	 * fail.
-	 */
-	if (faulthandler_disabled())
-		goto out_nosemaphore;
-
-	if (is_user)
-		flags |= FAULT_FLAG_USER;
-retry:
-	vma = um_lock_mm_and_find_vma(mm, address, is_user);
-	if (!vma)
-		goto out_nosemaphore;
-
-	*code_out = SEGV_ACCERR;
-	if (is_write) {
-		if (!(vma->vm_flags & VM_WRITE))
-			goto out;
-		flags |= FAULT_FLAG_WRITE;
-	} else {
-		/* Don't require VM_READ|VM_EXEC for write faults! */
-		if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
-			goto out;
-	}
-
-	do {
-		vm_fault_t fault;
-
-		fault = handle_mm_fault(vma, address, flags, NULL);
-
-		if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
-			goto out_nosemaphore;
-
-		/* The fault is fully completed (including releasing mmap lock) */
-		if (fault & VM_FAULT_COMPLETED)
-			return 0;
-
-		if (unlikely(fault & VM_FAULT_ERROR)) {
-			if (fault & VM_FAULT_OOM) {
-				goto out_of_memory;
-			} else if (fault & VM_FAULT_SIGSEGV) {
-				goto out;
-			} else if (fault & VM_FAULT_SIGBUS) {
-				err = -EACCES;
-				goto out;
-			}
-			BUG();
-		}
-		if (fault & VM_FAULT_RETRY) {
-			flags |= FAULT_FLAG_TRIED;
-
-			goto retry;
-		}
-
-		pmd = pmd_off(mm, address);
-		pte = pte_offset_kernel(pmd, address);
-	} while (!pte_present(*pte));
-	err = 0;
-	/*
-	 * The below warning was added in place of
-	 *	pte_mkyoung(); if (is_write) pte_mkdirty();
-	 * If it's triggered, we'd see normally a hang here (a clean pte is
-	 * marked read-only to emulate the dirty bit).
-	 * However, the generic code can mark a PTE writable but clean on a
-	 * concurrent read fault, triggering this harmlessly. So comment it out.
-	 */
-#if 0
-	WARN_ON(!pte_young(*pte) || (is_write && !pte_dirty(*pte)));
-#endif
-
-out:
-	mmap_read_unlock(mm);
-out_nosemaphore:
-	return err;
-
-out_of_memory:
-	/*
-	 * We ran out of memory, call the OOM killer, and return the userspace
-	 * (which will retry the fault, or kill us if we got oom-killed).
-	 */
-	mmap_read_unlock(mm);
-	if (!is_user)
-		goto out_nosemaphore;
-	pagefault_out_of_memory();
-	return 0;
-}
-
 static void show_segv_info(struct uml_pt_regs *regs)
 {
 	struct task_struct *tsk = current;
diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c
index e4ee693961e4e7..c3f83e8053923f 100644
--- a/arch/um/kernel/um_arch.c
+++ b/arch/um/kernel/um_arch.c
@@ -44,7 +44,12 @@ static char __initdata command_line[COMMAND_LINE_SIZE] = { 0 };
 
 static void __init add_arg(char *arg)
 {
-	if (strlen(command_line) + strlen(arg) + 1 > COMMAND_LINE_SIZE) {
+	size_t len = strlen(command_line) + strlen(arg) + 1;
+
+	if (command_line[0])
+		len++;
+
+	if (len > COMMAND_LINE_SIZE) {
 		os_warn("add_arg: Too many command line arguments!\n");
 		exit(1);
 	}
@@ -264,12 +269,88 @@ unsigned long brk_start;
 
 #define MIN_VMALLOC (32 * 1024 * 1024)
 
+static u64 __init read_xcr0(void)
+{
+	u32 a, b, c, d;
+
+	asm volatile("cpuid"
+		     : "=a"(a), "=b"(b), "=c"(c), "=d"(d)
+		     : "a"(0), "c"(0));
+	if (a >= 1) { /* max_leaf >= 1 */
+		asm volatile("cpuid"
+			     : "=a"(a), "=b"(b), "=c"(c), "=d"(d)
+			     : "a"(1), "c"(0));
+		if (c & (1 << 27)) { /* XSAVE enabled by OS */
+			asm volatile("xgetbv" : "=d"(d), "=a"(a) : "c"(0));
+			return ((u64)d << 32) | a;
+		}
+	}
+	return 0;
+}
+
+static void __init validate_and_set_cpu_cap(int cap, u64 xcr0)
+{
+	/*
+	 * Check for missing xstate features right away, so that there's no
+	 * perceived need for all optimized code in the kernel to do so.
+	 */
+	switch (cap) {
+	case X86_FEATURE_AVX:
+	case X86_FEATURE_AVX2:
+	case X86_FEATURE_AVX_VNNI:
+	case X86_FEATURE_FMA:
+	case X86_FEATURE_VAES:
+	case X86_FEATURE_VPCLMULQDQ:
+		if ((xcr0 & 0x7) != 0x7) {
+			static bool warned;
+
+			if (!warned) {
+				os_warn("Disabling AVX support due to missing xstate features\n");
+				warned = true;
+			}
+			return;
+		}
+		break;
+	case X86_FEATURE_AVX512F:
+	case X86_FEATURE_AVX512BW:
+	case X86_FEATURE_AVX512CD:
+	case X86_FEATURE_AVX512DQ:
+	case X86_FEATURE_AVX512ER:
+	case X86_FEATURE_AVX512IFMA:
+	case X86_FEATURE_AVX512PF:
+	case X86_FEATURE_AVX512VBMI:
+	case X86_FEATURE_AVX512VL:
+	case X86_FEATURE_AVX512_4FMAPS:
+	case X86_FEATURE_AVX512_4VNNIW:
+	case X86_FEATURE_AVX512_BF16:
+	case X86_FEATURE_AVX512_BITALG:
+	case X86_FEATURE_AVX512_FP16:
+	case X86_FEATURE_AVX512_VBMI2:
+	case X86_FEATURE_AVX512_VNNI:
+	case X86_FEATURE_AVX512_VP2INTERSECT:
+	case X86_FEATURE_AVX512_VPOPCNTDQ:
+		if ((xcr0 & 0xe7) != 0xe7) {
+			static bool warned;
+
+			if (!warned) {
+				os_warn("Disabling AVX-512 support due to missing xstate features\n");
+				warned = true;
+			}
+			return;
+		}
+		break;
+	}
+	set_cpu_cap(&boot_cpu_data, cap);
+}
+
 static void __init parse_host_cpu_flags(char *line)
 {
+	u64 xcr0 = read_xcr0();
 	int i;
+
 	for (i = 0; i < 32*NCAPINTS; i++) {
 		if ((x86_cap_flags[i] != NULL) && strstr(line, x86_cap_flags[i]))
-			set_cpu_cap(&boot_cpu_data, i);
+			validate_and_set_cpu_cap(i, xcr0);
 	}
 }
 
@@ -303,7 +384,7 @@ static unsigned long __init get_top_address(char **envp)
 	return PAGE_ALIGN(top_addr + 1);
 }
 
-int __init linux_main(int argc, char **argv, char **envp)
+void __init linux_main(int argc, char **argv, char **envp)
 {
 	unsigned long avail, diff;
 	unsigned long virtmem_size, max_physmem;
@@ -396,7 +477,7 @@ int __init linux_main(int argc, char **argv, char **envp)
 
 	os_flush_stdout();
 
-	return start_uml();
+	start_uml();
 }
 
 int __init __weak read_initrd(void)
diff --git a/arch/um/kernel/uml.lds.S b/arch/um/kernel/uml.lds.S
index 30aa24348d60cd..4aabcb7faa38ad 100644
--- a/arch/um/kernel/uml.lds.S
+++ b/arch/um/kernel/uml.lds.S
@@ -28,8 +28,13 @@ SECTIONS
   . = ALIGN(PAGE_SIZE);
 
   _text = .;
+  __init_begin = .;
   INIT_TEXT_SECTION(0)
 
+  init.data : { INIT_DATA }
+  . = ALIGN(PAGE_SIZE);
+  __init_end = .;
+
   .text      :
   {
     _stext = .;
@@ -70,14 +75,11 @@ SECTIONS
 
   #include <asm/common.lds.S>
 
-  __init_begin = .;
-  init.data : { INIT_DATA }
-  __init_end = .;
-
   .data    :
   {
     INIT_TASK_DATA(KERNEL_STACK_SIZE)
     DATA_DATA
+    *(.ldata .ldata.[0-9a-zA-Z_]* .ldata..L*)
     *(.gnu.linkonce.d*)
     CONSTRUCTORS
   }
@@ -107,6 +109,7 @@ SECTIONS
   PROVIDE(_bss_start = .);
   SBSS(0)
   BSS(0)
+  *(.lbss .lbss.[0-9a-zA-Z_]* .lbss..L*)
    __bss_stop = .;
   _end = .;
   PROVIDE (end = .);
diff --git a/arch/um/os-Linux/main.c b/arch/um/os-Linux/main.c
index 7e114862a7235c..ea41188ef2d0e4 100644
--- a/arch/um/os-Linux/main.c
+++ b/arch/um/os-Linux/main.c
@@ -100,10 +100,11 @@ static void __init setup_env_path(void)
 	}
 }
 
+static char **new_argv;
+
 int __init main(int argc, char **argv, char **envp)
 {
-	char **new_argv;
-	int ret, i, err;
+	int ret, i;
 
 	/* Disable randomization and re-exec if it was changed successfully */
 	ret = personality(PER_LINUX | ADDR_NO_RANDOMIZE);
@@ -150,7 +151,12 @@ int __init main(int argc, char **argv, char **envp)
 	scan_elf_aux(envp);
 
 	change_sig(SIGPIPE, 0);
-	ret = linux_main(argc, argv, envp);
+	linux_main(argc, argv, envp);
+}
+
+void os_exit(bool reboot)
+{
+	int err;
 
 	/*
 	 * Disable SIGPROF - I have no idea why libc doesn't do this or turn
@@ -183,12 +189,11 @@ int __init main(int argc, char **argv, char **envp)
 
 	os_info("\n");
 	/* Reboot */
-	if (ret) {
+	if (reboot) {
 		execvp(new_argv[0], new_argv);
 		perror("Failed to exec kernel");
-		ret = 1;
 	}
-	return uml_exitcode;
+	exit(uml_exitcode);
 }
 
 extern void *__real_malloc(int);
diff --git a/arch/um/os-Linux/signal.c b/arch/um/os-Linux/signal.c
index 6c993bc8c78eb2..57f0a5e46afb40 100644
--- a/arch/um/os-Linux/signal.c
+++ b/arch/um/os-Linux/signal.c
@@ -453,7 +453,7 @@ void unblock_signals_hard(void)
 	 * call _within_ e.g. vu_req_read_message(), where we observed this
 	 * issue, which loops. Thus, if the inner call handles the recorded
 	 * pending signals, we can get out of the inner call with the real
-	 * signal hander no longer blocked, and still have a race. Thus don't
+	 * signal handler no longer blocked, and still have a race. Thus don't
 	 * handle unblocking in the inner call, if it happens, but only in
 	 * the outermost call - 'unblocking' serves as an ownership for the
 	 * signals_blocked_pending decrement.
diff --git a/arch/um/os-Linux/skas/process.c b/arch/um/os-Linux/skas/process.c
index d6c22f8aa06d1c..bca05a5a7a83a3 100644
--- a/arch/um/os-Linux/skas/process.c
+++ b/arch/um/os-Linux/skas/process.c
@@ -816,7 +816,7 @@ static __thread void (*cb_proc)(void *arg);
 static __thread void *cb_arg;
 static __thread jmp_buf *cb_back;
 
-int start_idle_thread(void *stack, jmp_buf *switch_buf)
+void start_idle_thread(void *stack, jmp_buf *switch_buf)
 {
 	int n;
 
@@ -843,21 +843,16 @@ int start_idle_thread(void *stack, jmp_buf *switch_buf)
 		break;
 	case INIT_JMP_HALT:
 		kmalloc_ok = 0;
-		return 0;
+		os_exit(false);
 	case INIT_JMP_REBOOT:
 		kmalloc_ok = 0;
-		return 1;
+		os_exit(true);
 	default:
 		printk(UM_KERN_ERR "Bad sigsetjmp return in %s - %d\n",
 		       __func__, n);
 		fatal_sigsegv();
 	}
 	longjmp(*switch_buf, 1);
-
-	/* unreachable */
-	printk(UM_KERN_ERR "impossible long jump!");
-	fatal_sigsegv();
-	return 0;
 }
 
 void initial_thread_cb_skas(void (*proc)(void *), void *arg)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 15fd9ec5ecacb7..58dce0b66179b6 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -27,7 +27,6 @@ config X86_64
 	select ARCH_HAS_GIGANTIC_PAGE
 	select ARCH_SUPPORTS_MSEAL_SYSTEM_MAPPINGS
 	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
-	select ARCH_SUPPORTS_PER_VMA_LOCK
 	select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE
 	select HAVE_ARCH_SOFT_DIRTY
 	select MODULES_USE_ELF_RELA
@@ -110,7 +109,7 @@ config X86
 	select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE
 	select ARCH_HAS_SYSCALL_WRAPPER
 	select ARCH_HAS_UBSAN
-	select ARCH_HAS_DEBUG_WX
+	select ARCH_HAS_CHECK_WX
 	select ARCH_HAS_ZONE_DMA_SET if EXPERT
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select ARCH_HAVE_EXTRA_ELF_NOTES
@@ -150,7 +149,6 @@ config X86
 	select ARCH_WANT_LD_ORPHAN_WARN
 	select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP	if X86_64
 	select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP	if X86_64
-	select ARCH_WANT_HUGETLB_VMEMMAP_PREINIT if X86_64
 	select ARCH_WANTS_THP_SWAP		if X86_64
 	select ARCH_HAS_PARANOID_L1D_FLUSH
 	select ARCH_WANT_IRQS_OFF_ACTIVATE_MM
@@ -236,6 +234,7 @@ config X86
 	select HAVE_SAMPLE_FTRACE_DIRECT	if X86_64
 	select HAVE_SAMPLE_FTRACE_DIRECT_MULTI	if X86_64
 	select HAVE_EBPF_JIT
+	select HAVE_EBPF_JIT_KASAN		if X86_64
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS
 	select HAVE_EISA			if X86_32
 	select HAVE_EXIT_THREAD
@@ -285,7 +284,6 @@ config X86
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select ASYNC_KERNEL_PGTABLE_FREE	if IOMMU_SVA
-	select MMU_GATHER_RCU_TABLE_FREE
 	select MMU_GATHER_MERGE_VMAS
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_REGS_AND_STACK_ACCESS_API
@@ -297,7 +295,6 @@ config X86
 	select HAVE_STACK_VALIDATION		if HAVE_OBJTOOL
 	select HAVE_STATIC_CALL
 	select HAVE_STATIC_CALL_INLINE		if HAVE_OBJTOOL
-	select HAVE_PREEMPT_DYNAMIC_CALL
 	select HAVE_RSEQ
 	select HAVE_RUST			if X86_64
 	select HAVE_SYSCALL_TRACEPOINTS
@@ -1849,7 +1846,6 @@ config X86_USER_SHADOW_STACK
 	bool "X86 userspace shadow stack"
 	depends on AS_WRUSS
 	depends on X86_64
-	depends on PER_VMA_LOCK
 	select ARCH_USES_HIGH_VMA_FLAGS
 	select ARCH_HAS_USER_SHADOW_STACK
 	select X86_CET
@@ -2364,6 +2360,23 @@ config CC_HAS_KCFI_ARITY
 	def_bool $(cc-option,-fsanitize=kcfi -fsanitize-kcfi-arity)
 	depends on CC_IS_CLANG && !RUST
 
+config CC_OPT_INDIRECT_BRANCH_CS_PREFIX
+	def_string "$(cc-option-str,-mindirect-branch-cs-prefix)"
+
+config CC_OPT_CF_PROTECTION_NONE
+	def_string "$(cc-option-str,-fcf-protection=none)"
+
+# with jump tables off: the compilers use NOTRACK for them, which kernel IBT
+# does not allow
+config CC_OPT_CF_PROTECTION_BRANCH
+	def_string "$(cc-option-str,-fcf-protection=branch -fno-jump-tables)"
+
+config CC_OPT_FALIGN_JUMPS
+	def_string "$(cc-option-str,-falign-jumps=1)"
+
+config CC_OPT_FALIGN_LOOPS
+	def_string "$(cc-option-str,-falign-loops=1)"
+
 config FUNCTION_PADDING_CFI
 	int
 	default 59 if FUNCTION_ALIGNMENT_64B
@@ -3059,15 +3072,6 @@ config GEOS
 	help
 	  This option enables system support for the Traverse Technologies GEOS.
 
-config TS5500
-	bool "Technologic Systems TS-5500 platform support"
-	depends on MELAN
-	select CHECK_SIGNATURE
-	select NEW_LEDS
-	select LEDS_CLASS
-	help
-	  This option enables system support for the Technologic Systems TS-5500.
-
 endif # X86_32
 
 config AMD_NB
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index e4654388d794ba..6e7a366f079841 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -204,10 +204,21 @@ config CC_HAS_MARCH_NATIVE
 	# usage warnings that only appear wth '-march=native'.
 	depends on CC_IS_GCC || CLANG_VERSION >= 190100
 
+config RUSTC_HAS_APXF
+	# The kernel isn't ready for in-kernel APX instructions. Without
+	# explicit frontend gating of APX, the backend may emit those
+	# instructions in native builds.
+	#
+	# Rust 1.88 added the `apxf` feature option, but versions before 1.93
+	# emit an `apxf` target attribute that only LLVM 23+ can interpret.
+	def_bool (RUSTC_VERSION >= 108800 && RUSTC_LLVM_MAJOR_VERSION >= 23) || \
+		 RUSTC_VERSION >= 109300
+
 config X86_NATIVE_CPU
 	bool "Build and optimize for local/native CPU"
 	depends on X86_64
 	depends on CC_HAS_MARCH_NATIVE
+	depends on !RUST || RUSTC_HAS_APXF
 	help
 	  Optimize for the current CPU used to compile the kernel.
 	  Use this option if you intend to build the kernel for your
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 598f178102ee4a..b7e08519b8c1e2 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -20,7 +20,7 @@ ifdef CONFIG_CC_IS_CLANG
 RETPOLINE_CFLAGS	:= -mretpoline-external-thunk
 RETPOLINE_VDSO_CFLAGS	:= -mretpoline
 endif
-RETPOLINE_CFLAGS	+= $(call cc-option,-mindirect-branch-cs-prefix)
+RETPOLINE_CFLAGS	+= $(CONFIG_CC_OPT_INDIRECT_BRANCH_CS_PREFIX)
 
 ifdef CONFIG_MITIGATION_RETHUNK
 RETHUNK_CFLAGS		:= -mfunction-return=thunk-extern
@@ -52,7 +52,7 @@ REALMODE_CFLAGS	:= $(CC_FLAGS_DIALECT) -m16 -g -Os \
 		   -DDISABLE_BRANCH_PROFILING -D__DISABLE_EXPORTS \
 		   -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \
 		   -fno-strict-aliasing -fomit-frame-pointer -fno-pic \
-		   -mno-mmx -mno-sse $(call cc-option,-fcf-protection=none)
+		   -mno-mmx -mno-sse $(CONFIG_CC_OPT_CF_PROTECTION_NONE)
 
 REALMODE_CFLAGS += -ffreestanding
 REALMODE_CFLAGS += -fno-stack-protector
@@ -99,10 +99,10 @@ ifeq ($(CONFIG_X86_KERNEL_IBT),y)
 #
 #   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104816
 #
-KBUILD_CFLAGS += $(call cc-option,-fcf-protection=branch -fno-jump-tables)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_CF_PROTECTION_BRANCH)
 KBUILD_RUSTFLAGS += -Zcf-protection=branch $(if $(call rustc-min-version,109300),-Cjump-tables=n,-Zno-jump-tables)
 else
-KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_CF_PROTECTION_NONE)
 endif
 
 ifeq ($(CONFIG_X86_32),y)
@@ -138,10 +138,10 @@ else
         KBUILD_CFLAGS += -m64
 
         # Align jump targets to 1 byte, not the default 16 bytes:
-        KBUILD_CFLAGS += $(call cc-option,-falign-jumps=1)
+        KBUILD_CFLAGS += $(CONFIG_CC_OPT_FALIGN_JUMPS)
 
         # Pack loops tightly as well:
-        KBUILD_CFLAGS += $(call cc-option,-falign-loops=1)
+        KBUILD_CFLAGS += $(CONFIG_CC_OPT_FALIGN_LOOPS)
 
         # Don't autogenerate traditional x87 instructions
         KBUILD_CFLAGS += -mno-80387
@@ -161,6 +161,11 @@ else
 
 ifdef CONFIG_X86_NATIVE_CPU
         KBUILD_CFLAGS += -march=native
+        # Prevent the compiler from generating EGPR use. The kernel is
+        # not yet prepared for general in-kernel use.
+        KBUILD_CFLAGS += $(call cc-option,-mno-apx-features=egpr)
+
+        # generate_rust_target.rs handles Rust APX gating.
         KBUILD_RUSTFLAGS += -Ctarget-cpu=native
 else
         KBUILD_CFLAGS += -march=x86-64 -mtune=generic
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index 3f9fb3698d6690..4a9bce32586d3b 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -74,7 +74,7 @@ SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
 sed-zoffset := -e 's/^\([0-9a-fA-F]*\) [a-zA-Z] \(startup_32\|efi.._stub_entry\|efi\(32\)\?_pe_entry\|input_data\|kernel_info\|_end\|_ehead\|_text\|_e\?data\|_e\?sbat\|z_.*\)$$/\#define ZO_\2 0x\1/p'
 
 quiet_cmd_zoffset = ZOFFSET $@
-      cmd_zoffset = $(NM) $< | sed -n $(sed-zoffset) > $@
+      cmd_zoffset = $(NM) -p $< | sed -n $(sed-zoffset) > $@
 
 targets += zoffset.h
 $(obj)/zoffset.h: $(obj)/compressed/vmlinux FORCE
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index 06934f9691d6a9..6ec5e031db1d27 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -76,7 +76,7 @@ HOST_EXTRACFLAGS += -I$(srctree)/tools/include
 sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABbCDGRSTtVW] \(_text\|__start_rodata\|_sinittext\|__inittext_end\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p'
 
 quiet_cmd_voffset = VOFFSET $@
-      cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
+      cmd_voffset = $(NM) -p $< | sed -n $(sed-voffset) > $@
 
 targets += ../voffset.h
 
diff --git a/arch/x86/boot/compressed/error.c b/arch/x86/boot/compressed/error.c
index 19a8251de506a5..ce5ed7d8265e6d 100644
--- a/arch/x86/boot/compressed/error.c
+++ b/arch/x86/boot/compressed/error.c
@@ -22,22 +22,3 @@ void error(char *m)
 	while (1)
 		asm("hlt");
 }
-
-/* EFI libstub  provides vsnprintf() */
-#ifdef CONFIG_EFI_STUB
-void panic(const char *fmt, ...)
-{
-	static char buf[1024];
-	va_list args;
-	int len;
-
-	va_start(args, fmt);
-	len = vsnprintf(buf, sizeof(buf), fmt, args);
-	va_end(args);
-
-	if (len && buf[len - 1] == '\n')
-		buf[len - 1] = '\0';
-
-	error(buf);
-}
-#endif
diff --git a/arch/x86/boot/compressed/error.h b/arch/x86/boot/compressed/error.h
index 31f9e080d61a80..87062dea9a20da 100644
--- a/arch/x86/boot/compressed/error.h
+++ b/arch/x86/boot/compressed/error.h
@@ -6,6 +6,5 @@
 
 void warn(const char *m);
 void error(char *m) __noreturn;
-void panic(const char *fmt, ...) __noreturn __cold;
 
 #endif /* BOOT_COMPRESSED_ERROR_H */
diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
index 22267a83e064ae..290153818d0f3e 100644
--- a/arch/x86/boot/compressed/kaslr.c
+++ b/arch/x86/boot/compressed/kaslr.c
@@ -764,12 +764,12 @@ static void process_e820_entries(unsigned long minimum,
 }
 
 /*
- * If KHO is active, only process its scratch areas to ensure we are not
+ * If KHO is active, only process its bootmem areas to ensure we are not
  * stepping onto preserved memory.
  */
 static bool process_kho_entries(unsigned long minimum, unsigned long image_size)
 {
-	struct kho_scratch *kho_scratch;
+	struct kho_bootmem *kho_bootmem;
 	struct setup_data *ptr;
 	struct kho_data *kho;
 	int i, nr_areas = 0;
@@ -781,8 +781,8 @@ static bool process_kho_entries(unsigned long minimum, unsigned long image_size)
 	while (ptr) {
 		if (ptr->type == SETUP_KEXEC_KHO) {
 			kho = (struct kho_data *)(unsigned long)ptr->data;
-			kho_scratch = (void *)(unsigned long)kho->scratch_addr;
-			nr_areas = kho->scratch_size / sizeof(*kho_scratch);
+			kho_bootmem = (void *)(unsigned long)kho->bootmem_addr;
+			nr_areas = kho->bootmem_size / sizeof(*kho_bootmem);
 			break;
 		}
 
@@ -793,7 +793,7 @@ static bool process_kho_entries(unsigned long minimum, unsigned long image_size)
 		return false;
 
 	for (i = 0; i < nr_areas; i++) {
-		struct kho_scratch *area = &kho_scratch[i];
+		struct kho_bootmem *area = &kho_bootmem[i];
 		struct mem_vector region = {
 			.start = area->addr,
 			.size = area->size,
@@ -822,7 +822,7 @@ static unsigned long find_random_phys_addr(unsigned long minimum,
 	}
 
 	/*
-	 * During kexec handover only process KHO scratch areas that are known
+	 * During kexec handover only process KHO bootmem areas that are known
 	 * not to contain any data that must be preserved.
 	 */
 	if (!process_kho_entries(minimum, image_size) &&
diff --git a/arch/x86/boot/compressed/mem.c b/arch/x86/boot/compressed/mem.c
index 0e9f84ab4bdcd1..1721af3a80399b 100644
--- a/arch/x86/boot/compressed/mem.c
+++ b/arch/x86/boot/compressed/mem.c
@@ -2,48 +2,6 @@
 
 #include "error.h"
 #include "misc.h"
-#include "tdx.h"
-#include "sev.h"
-#include <asm/shared/tdx.h>
-
-/*
- * accept_memory() and process_unaccepted_memory() called from EFI stub which
- * runs before decompressor and its early_tdx_detect().
- *
- * Enumerate TDX directly from the early users.
- */
-static bool early_is_tdx_guest(void)
-{
-	static bool once;
-	static bool is_tdx;
-
-	if (!IS_ENABLED(CONFIG_INTEL_TDX_GUEST))
-		return false;
-
-	if (!once) {
-		u32 eax, sig[3];
-
-		cpuid_count(TDX_CPUID_LEAF_ID, 0, &eax,
-			    &sig[0], &sig[2],  &sig[1]);
-		is_tdx = !memcmp(TDX_IDENT, sig, sizeof(sig));
-		once = true;
-	}
-
-	return is_tdx;
-}
-
-void arch_accept_memory(phys_addr_t start, phys_addr_t end)
-{
-	/* Platform-specific memory-acceptance call goes here */
-	if (early_is_tdx_guest()) {
-		if (!tdx_accept_memory(start, end))
-			panic("TDX: Failed to accept memory\n");
-	} else if (early_is_sevsnp_guest()) {
-		snp_accept_memory(start, end);
-	} else {
-		error("Cannot accept memory: unknown platform\n");
-	}
-}
 
 bool init_unaccepted_memory(void)
 {
diff --git a/arch/x86/boot/compressed/sev.h b/arch/x86/boot/compressed/sev.h
index 22637b416b46fc..62e50c2e71edb1 100644
--- a/arch/x86/boot/compressed/sev.h
+++ b/arch/x86/boot/compressed/sev.h
@@ -14,7 +14,6 @@
 
 void snp_accept_memory(phys_addr_t start, phys_addr_t end);
 u64 sev_get_status(void);
-bool early_is_sevsnp_guest(void);
 
 static inline u64 sev_es_rd_ghcb_msr(void)
 {
@@ -37,7 +36,6 @@ static inline void sev_es_wr_ghcb_msr(u64 val)
 
 static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
 static inline u64 sev_get_status(void) { return 0; }
-static inline bool early_is_sevsnp_guest(void) { return false; }
 
 #endif
 
diff --git a/arch/x86/boot/compressed/tdx-shared.c b/arch/x86/boot/compressed/tdx-shared.c
index 5ac43762fe13c1..dc38047647ccec 100644
--- a/arch/x86/boot/compressed/tdx-shared.c
+++ b/arch/x86/boot/compressed/tdx-shared.c
@@ -1,2 +1,4 @@
+#define __NO_FORTIFY
+
 #include "error.h"
 #include "../../coco/tdx/tdx-shared.c"
diff --git a/arch/x86/boot/early_serial_console.c b/arch/x86/boot/early_serial_console.c
index 5b83beab89e150..39fcd551fc811d 100644
--- a/arch/x86/boot/early_serial_console.c
+++ b/arch/x86/boot/early_serial_console.c
@@ -22,6 +22,7 @@
 #define DLH             1       /*  Divisor latch High        */
 
 #define DEFAULT_BAUD 9600
+#define BASE_BAUD (1843200 / 16)
 
 static void early_serial_init(int port, int baud)
 {
@@ -33,7 +34,7 @@ static void early_serial_init(int port, int baud)
 	outb(0, port + FCR);	/* no fifo */
 	outb(0x3, port + MCR);	/* DTR + RTS */
 
-	divisor	= 115200 / baud;
+	divisor	= BASE_BAUD / baud;
 	c = inb(port + LCR);
 	outb(c | DLAB, port + LCR);
 	outb(divisor & 0xff, port + DLL);
@@ -74,16 +75,13 @@ static void parse_earlyprintk(void)
 			else
 				pos = e - arg;
 		} else if (!strncmp(arg + pos, "ttyS", 4)) {
-			static const int bases[] = { 0x3f8, 0x2f8 };
-			int idx = 0;
-
 			/* += strlen("ttyS"); */
 			pos += 4;
 
 			if (arg[pos++] == '1')
-				idx = 1;
-
-			port = bases[idx];
+				port = 0x2f8; /* ttyS1 */
+			else
+				port = DEFAULT_SERIAL_PORT;
 		}
 
 		if (arg[pos] == ',')
@@ -98,7 +96,6 @@ static void parse_earlyprintk(void)
 		early_serial_init(port, baud);
 }
 
-#define BASE_BAUD (1843200/16)
 static unsigned int probe_baud(int port)
 {
 	unsigned char lcr, dll, dlh;
diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c
index 1632d40e1f545a..be454a6864225f 100644
--- a/arch/x86/boot/string.c
+++ b/arch/x86/boot/string.c
@@ -15,6 +15,7 @@
 #include <linux/errno.h>
 #include <linux/limits.h>
 #include <asm/asm.h>
+#include <asm/shared/string.h>
 #include "ctype.h"
 #include "string.h"
 
@@ -31,17 +32,7 @@
 
 int memcmp(const void *s1, const void *s2, size_t len)
 {
-	bool diff;
-
-	/*
-	 * Make sure ZF is properly set in the len==0 case because in it,
-	 * RCX==0 and the REPE; CMPSB won't get executed.
-	 */
-	asm volatile("test %3, %3\n\t"
-		     "repe cmpsb"
-		     : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len)
-		     : : "cc", "memory");
-	return diff;
+	return __inline_memcmp(s1, s2, len);
 }
 
 /*
diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index cc292d7c6fd1e0..eb2e853e950aaa 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -1431,15 +1431,22 @@ static ssize_t vmpl_show(struct kobject *kobj,
 	return sysfs_emit(buf, "%d\n", snp_vmpl);
 }
 
+static ssize_t sev_status_show(struct kobject *kobj,
+			       struct kobj_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "0x%llx\n", sev_status);
+}
+
 static struct kobj_attribute vmpl_attr = __ATTR_RO(vmpl);
+static struct kobj_attribute sev_status_attr = __ATTR_RO(sev_status);
 
-static struct attribute *vmpl_attrs[] = {
-	&vmpl_attr.attr,
+static struct attribute *sev_status_attrs[] = {
+	&sev_status_attr.attr,
 	NULL
 };
 
 static struct attribute_group sev_attr_group = {
-	.attrs = vmpl_attrs,
+	.attrs = sev_status_attrs,
 };
 
 static int __init sev_sysfs_init(void)
@@ -1448,7 +1455,7 @@ static int __init sev_sysfs_init(void)
 	struct device *dev_root;
 	int ret;
 
-	if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP))
+	if (!(sev_status & MSR_AMD64_SEV_ENABLED))
 		return -ENODEV;
 
 	dev_root = bus_get_dev_root(&cpu_subsys);
@@ -1463,7 +1470,20 @@ static int __init sev_sysfs_init(void)
 
 	ret = sysfs_create_group(sev_kobj, &sev_attr_group);
 	if (ret)
-		kobject_put(sev_kobj);
+		goto drop_kobj;
+
+	if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
+		ret = sysfs_add_file_to_group(sev_kobj, &vmpl_attr.attr, NULL);
+		if (ret)
+			goto drop_sysfs;
+	}
+
+	return 0;
+
+drop_sysfs:
+	sysfs_remove_group(sev_kobj, &sev_attr_group);
+drop_kobj:
+	kobject_put(sev_kobj);
 
 	return ret;
 }
diff --git a/arch/x86/coco/sev/svsm.c b/arch/x86/coco/sev/svsm.c
index 916d62cd17dc70..2d493d55ff2e86 100644
--- a/arch/x86/coco/sev/svsm.c
+++ b/arch/x86/coco/sev/svsm.c
@@ -74,6 +74,14 @@ int svsm_perform_call_protocol(struct svsm_call *call)
 
 	flags = native_local_irq_save();
 
+	/*
+	 * 'caa' is a per-CPU variable. To avoid using a stale or incorrect
+	 * 'caa' if the task is preempted or migrated to another CPU after it
+	 * is fetched, always fetch 'caa' and then issue the SVSM call with
+	 * interrupts disabled. This ensures the correct 'caa' is used.
+	 */
+	call->caa = svsm_get_caa();
+
 	ghcb = __sev_get_ghcb(&state);
 
 	do {
@@ -321,7 +329,6 @@ int snp_svsm_vtpm_send_command(u8 *buffer)
 {
 	struct svsm_call call = {};
 
-	call.caa = svsm_get_caa();
 	call.rax = SVSM_VTPM_CALL(SVSM_VTPM_CMD);
 	call.rcx = __pa(buffer);
 
@@ -345,7 +352,6 @@ bool snp_svsm_vtpm_probe(void)
 	if (!snp_vmpl)
 		return false;
 
-	call.caa = svsm_get_caa();
 	call.rax = SVSM_VTPM_CALL(SVSM_VTPM_QUERY);
 
 	if (svsm_perform_call_protocol(&call))
diff --git a/arch/x86/coco/tdx/tdx-shared.c b/arch/x86/coco/tdx/tdx-shared.c
index 1655aa56a0a512..29661d8dfe8eaf 100644
--- a/arch/x86/coco/tdx/tdx-shared.c
+++ b/arch/x86/coco/tdx/tdx-shared.c
@@ -89,3 +89,34 @@ noinstr u64 __tdx_hypercall(struct tdx_module_args *args)
 	/* TDVMCALL leaf return code is in R10 */
 	return args->r10;
 }
+
+void __noreturn tdx_panic(const char *msg)
+{
+	struct tdx_module_args args = {
+		.r10 = TDX_HYPERCALL_STANDARD,
+		.r11 = TDVMCALL_REPORT_FATAL_ERROR,
+		.r12 = 0, /* Error code: 0 is Panic */
+	};
+	/* Define register order according to the GHCI */
+	struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; } message = {};
+
+	/* VMM assumes '\0' in byte 65, if the message took all 64 bytes */
+	memcpy(&message, msg, strnlen(msg, sizeof(message)));
+
+	args.r8  = message.r8;
+	args.r9  = message.r9;
+	args.r14 = message.r14;
+	args.r15 = message.r15;
+	args.rdi = message.rdi;
+	args.rsi = message.rsi;
+	args.rbx = message.rbx;
+	args.rdx = message.rdx;
+
+	/*
+	 * This hypercall should never return and it is not safe
+	 * to keep the guest running. Call it forever if it
+	 * happens to return.
+	 */
+	while (1)
+		__tdx_hypercall(&args);
+}
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index f904a636d449ba..ad0131813a3091 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -139,7 +139,7 @@ int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(tdx_mcall_get_report0);
+EXPORT_SYMBOL_FOR_MODULES(tdx_mcall_get_report0, "tdx-guest");
 
 /**
  * tdx_mcall_extend_rtmr() - Wrapper to extend RTMR registers using
@@ -175,7 +175,7 @@ int tdx_mcall_extend_rtmr(u8 index, u8 *data)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(tdx_mcall_extend_rtmr);
+EXPORT_SYMBOL_FOR_MODULES(tdx_mcall_extend_rtmr, "tdx-guest");
 
 /**
  * tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote
@@ -196,42 +196,7 @@ u64 tdx_hcall_get_quote(u8 *buf, size_t size)
 	/* Since buf is a shared memory, set the shared (decrypted) bits */
 	return _tdx_hypercall(TDVMCALL_GET_QUOTE, cc_mkdec(virt_to_phys(buf)), size, 0, 0);
 }
-EXPORT_SYMBOL_GPL(tdx_hcall_get_quote);
-
-static void __noreturn tdx_panic(const char *msg)
-{
-	struct tdx_module_args args = {
-		.r10 = TDX_HYPERCALL_STANDARD,
-		.r11 = TDVMCALL_REPORT_FATAL_ERROR,
-		.r12 = 0, /* Error code: 0 is Panic */
-	};
-	union {
-		/* Define register order according to the GHCI */
-		struct { u64 r14, r15, rbx, rdi, rsi, r8, r9, rdx; };
-
-		char bytes[64] __nonstring;
-	} message;
-
-	/* VMM assumes '\0' in byte 65, if the message took all 64 bytes */
-	strtomem_pad(message.bytes, msg, '\0');
-
-	args.r8  = message.r8;
-	args.r9  = message.r9;
-	args.r14 = message.r14;
-	args.r15 = message.r15;
-	args.rdi = message.rdi;
-	args.rsi = message.rsi;
-	args.rbx = message.rbx;
-	args.rdx = message.rdx;
-
-	/*
-	 * This hypercall should never return and it is not safe
-	 * to keep the guest running. Call it forever if it
-	 * happens to return.
-	 */
-	while (1)
-		__tdx_hypercall(&args);
-}
+EXPORT_SYMBOL_FOR_MODULES(tdx_hcall_get_quote, "tdx-guest");
 
 /*
  * The kernel cannot handle #VEs when accessing normal kernel memory. Ensure
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index 269f7d808be4ec..e6896aeb77d81d 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -263,7 +263,7 @@ CONFIG_SECURITY_SELINUX_BOOTPARAM=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_MAGIC_SYSRQ=y
-CONFIG_DEBUG_WX=y
+CONFIG_CHECK_WX=y
 CONFIG_DEBUG_STACK_USAGE=y
 CONFIG_SCHEDSTATS=y
 CONFIG_BLK_DEV_IO_TRACE=y
diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
index f1adfba1a76ea9..09fc0b15b0e998 100644
--- a/arch/x86/crypto/aegis128-aesni-glue.c
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -265,8 +265,7 @@ static struct aead_alg crypto_aegis128_aesni_alg = {
 static int __init crypto_aegis128_aesni_module_init(void)
 {
 	if (!boot_cpu_has(X86_FEATURE_XMM4_1) ||
-	    !boot_cpu_has(X86_FEATURE_AES) ||
-	    !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
+	    !boot_cpu_has(X86_FEATURE_AES))
 		return -ENODEV;
 
 	return crypto_register_aead(&crypto_aegis128_aesni_alg);
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index f522fff9231e8e..5135bd4e6ea620 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -1293,6 +1293,7 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
 		unsigned int len;
 		const u8 *src = walk.addr;
 
+		assoclen -= orig_len_this_step;
 		if (unlikely(pos)) {
 			len = min(len_this_step, 16 - pos);
 			memcpy(&buf[pos], src, len);
@@ -1320,7 +1321,6 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
 			kernel_fpu_end();
 			kernel_fpu_begin();
 		}
-		assoclen -= orig_len_this_step;
 	}
 	if (unlikely(pos))
 		aes_gcm_aad_update(key, ghash_acc, buf, pos, flags);
@@ -1548,8 +1548,7 @@ static int __init register_avx_algs(void)
 	if (!boot_cpu_has(X86_FEATURE_AVX2) ||
 	    !boot_cpu_has(X86_FEATURE_VAES) ||
 	    !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) ||
-	    !boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
-	    !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
+	    !boot_cpu_has(X86_FEATURE_PCLMULQDQ))
 		return 0;
 	err = crypto_register_skciphers(skcipher_algs_vaes_avx2,
 					ARRAY_SIZE(skcipher_algs_vaes_avx2));
@@ -1562,9 +1561,7 @@ static int __init register_avx_algs(void)
 
 	if (!boot_cpu_has(X86_FEATURE_AVX512BW) ||
 	    !boot_cpu_has(X86_FEATURE_AVX512VL) ||
-	    !boot_cpu_has(X86_FEATURE_BMI2) ||
-	    !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
-			       XFEATURE_MASK_AVX512, NULL))
+	    !boot_cpu_has(X86_FEATURE_BMI2))
 		return 0;
 
 	if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
diff --git a/arch/x86/crypto/aria_aesni_avx2_glue.c b/arch/x86/crypto/aria_aesni_avx2_glue.c
index 1487a49bfbaccf..371be2fb64695e 100644
--- a/arch/x86/crypto/aria_aesni_avx2_glue.c
+++ b/arch/x86/crypto/aria_aesni_avx2_glue.c
@@ -195,22 +195,13 @@ static struct skcipher_alg aria_algs[] = {
 
 static int __init aria_avx2_init(void)
 {
-	const char *feature_name;
-
 	if (!boot_cpu_has(X86_FEATURE_AVX) ||
 	    !boot_cpu_has(X86_FEATURE_AVX2) ||
-	    !boot_cpu_has(X86_FEATURE_AES) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	    !boot_cpu_has(X86_FEATURE_AES)) {
 		pr_info("AVX2 or AES-NI instructions are not detected.\n");
 		return -ENODEV;
 	}
 
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
-
 	if (boot_cpu_has(X86_FEATURE_GFNI)) {
 		aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way;
 		aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way;
diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c
index e4e3d78915a5fd..d23fc91c0ebdd1 100644
--- a/arch/x86/crypto/aria_aesni_avx_glue.c
+++ b/arch/x86/crypto/aria_aesni_avx_glue.c
@@ -182,21 +182,12 @@ static struct skcipher_alg aria_algs[] = {
 
 static int __init aria_avx_init(void)
 {
-	const char *feature_name;
-
 	if (!boot_cpu_has(X86_FEATURE_AVX) ||
-	    !boot_cpu_has(X86_FEATURE_AES) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	    !boot_cpu_has(X86_FEATURE_AES)) {
 		pr_info("AVX or AES-NI instructions are not detected.\n");
 		return -ENODEV;
 	}
 
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
-
 	if (boot_cpu_has(X86_FEATURE_GFNI)) {
 		aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way;
 		aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way;
diff --git a/arch/x86/crypto/aria_gfni_avx512_glue.c b/arch/x86/crypto/aria_gfni_avx512_glue.c
index 363cbf4399cca0..e05bbeb22d4a90 100644
--- a/arch/x86/crypto/aria_gfni_avx512_glue.c
+++ b/arch/x86/crypto/aria_gfni_avx512_glue.c
@@ -196,24 +196,15 @@ static struct skcipher_alg aria_algs[] = {
 
 static int __init aria_avx512_init(void)
 {
-	const char *feature_name;
-
 	if (!boot_cpu_has(X86_FEATURE_AVX) ||
 	    !boot_cpu_has(X86_FEATURE_AVX2) ||
 	    !boot_cpu_has(X86_FEATURE_AVX512F) ||
 	    !boot_cpu_has(X86_FEATURE_AVX512VL) ||
-	    !boot_cpu_has(X86_FEATURE_GFNI) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	    !boot_cpu_has(X86_FEATURE_GFNI)) {
 		pr_info("AVX512/GFNI instructions are not detected.\n");
 		return -ENODEV;
 	}
 
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
-			       XFEATURE_MASK_AVX512, &feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
-
 	aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way;
 	aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way;
 	aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way;
diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c
index 2d2f4e16537c4f..073fa3bb838889 100644
--- a/arch/x86/crypto/camellia_aesni_avx2_glue.c
+++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c
@@ -97,22 +97,13 @@ static struct skcipher_alg camellia_algs[] = {
 
 static int __init camellia_aesni_init(void)
 {
-	const char *feature_name;
-
 	if (!boot_cpu_has(X86_FEATURE_AVX) ||
 	    !boot_cpu_has(X86_FEATURE_AVX2) ||
-	    !boot_cpu_has(X86_FEATURE_AES) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	    !boot_cpu_has(X86_FEATURE_AES)) {
 		pr_info("AVX2 or AES-NI instructions are not detected.\n");
 		return -ENODEV;
 	}
 
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
-
 	return crypto_register_skciphers(camellia_algs,
 					 ARRAY_SIZE(camellia_algs));
 }
diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c
index 5c321f255eb7bd..872e5e07220fa8 100644
--- a/arch/x86/crypto/camellia_aesni_avx_glue.c
+++ b/arch/x86/crypto/camellia_aesni_avx_glue.c
@@ -98,21 +98,12 @@ static struct skcipher_alg camellia_algs[] = {
 
 static int __init camellia_aesni_init(void)
 {
-	const char *feature_name;
-
 	if (!boot_cpu_has(X86_FEATURE_AVX) ||
-	    !boot_cpu_has(X86_FEATURE_AES) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	    !boot_cpu_has(X86_FEATURE_AES)) {
 		pr_info("AVX or AES-NI instructions are not detected.\n");
 		return -ENODEV;
 	}
 
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
-
 	return crypto_register_skciphers(camellia_algs,
 					 ARRAY_SIZE(camellia_algs));
 }
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
index 3aca04d43b34a4..5de35e863370cb 100644
--- a/arch/x86/crypto/cast5_avx_glue.c
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -92,11 +92,8 @@ static struct skcipher_alg cast5_algs[] = {
 
 static int __init cast5_init(void)
 {
-	const char *feature_name;
-
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
+	if (!boot_cpu_has(X86_FEATURE_AVX)) {
+		pr_info("AVX instructions are not detected.\n");
 		return -ENODEV;
 	}
 
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c
index c4dd28c3030364..3d7ea48007bc09 100644
--- a/arch/x86/crypto/cast6_avx_glue.c
+++ b/arch/x86/crypto/cast6_avx_glue.c
@@ -92,11 +92,8 @@ static struct skcipher_alg cast6_algs[] = {
 
 static int __init cast6_init(void)
 {
-	const char *feature_name;
-
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
+	if (!boot_cpu_has(X86_FEATURE_AVX)) {
+		pr_info("AVX instructions are not detected.\n");
 		return -ENODEV;
 	}
 
diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c
index f5f2121b79567c..72a9e2b306d639 100644
--- a/arch/x86/crypto/serpent_avx2_glue.c
+++ b/arch/x86/crypto/serpent_avx2_glue.c
@@ -93,17 +93,10 @@ static struct skcipher_alg serpent_algs[] = {
 
 static int __init serpent_avx2_init(void)
 {
-	const char *feature_name;
-
-	if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	if (!boot_cpu_has(X86_FEATURE_AVX2)) {
 		pr_info("AVX2 instructions are not detected.\n");
 		return -ENODEV;
 	}
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
 
 	return crypto_register_skciphers(serpent_algs,
 					 ARRAY_SIZE(serpent_algs));
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
index 9c8b3a335d5ca3..42c4e156967438 100644
--- a/arch/x86/crypto/serpent_avx_glue.c
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -100,11 +100,8 @@ static struct skcipher_alg serpent_algs[] = {
 
 static int __init serpent_init(void)
 {
-	const char *feature_name;
-
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
+	if (!boot_cpu_has(X86_FEATURE_AVX)) {
+		pr_info("AVX instructions are not detected.\n");
 		return -ENODEV;
 	}
 
diff --git a/arch/x86/crypto/sm4_aesni_avx2_glue.c b/arch/x86/crypto/sm4_aesni_avx2_glue.c
index fec0ab7a63dd4b..eef73894e7776a 100644
--- a/arch/x86/crypto/sm4_aesni_avx2_glue.c
+++ b/arch/x86/crypto/sm4_aesni_avx2_glue.c
@@ -98,22 +98,13 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
 
 static int __init sm4_init(void)
 {
-	const char *feature_name;
-
 	if (!boot_cpu_has(X86_FEATURE_AVX) ||
 	    !boot_cpu_has(X86_FEATURE_AVX2) ||
-	    !boot_cpu_has(X86_FEATURE_AES) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	    !boot_cpu_has(X86_FEATURE_AES)) {
 		pr_info("AVX2 or AES-NI instructions are not detected.\n");
 		return -ENODEV;
 	}
 
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
-
 	return crypto_register_skciphers(sm4_aesni_avx2_skciphers,
 					 ARRAY_SIZE(sm4_aesni_avx2_skciphers));
 }
diff --git a/arch/x86/crypto/sm4_aesni_avx_glue.c b/arch/x86/crypto/sm4_aesni_avx_glue.c
index 88caf418a06f16..ed383da5ff4604 100644
--- a/arch/x86/crypto/sm4_aesni_avx_glue.c
+++ b/arch/x86/crypto/sm4_aesni_avx_glue.c
@@ -314,21 +314,12 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = {
 
 static int __init sm4_init(void)
 {
-	const char *feature_name;
-
 	if (!boot_cpu_has(X86_FEATURE_AVX) ||
-	    !boot_cpu_has(X86_FEATURE_AES) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	    !boot_cpu_has(X86_FEATURE_AES)) {
 		pr_info("AVX or AES-NI instructions are not detected.\n");
 		return -ENODEV;
 	}
 
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				&feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
-		return -ENODEV;
-	}
-
 	return crypto_register_skciphers(sm4_aesni_avx_skciphers,
 					 ARRAY_SIZE(sm4_aesni_avx_skciphers));
 }
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
index 9e20db01375011..985bc54a23404e 100644
--- a/arch/x86/crypto/twofish_avx_glue.c
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -102,10 +102,8 @@ static struct skcipher_alg twofish_algs[] = {
 
 static int __init twofish_init(void)
 {
-	const char *feature_name;
-
-	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
-		pr_info("CPU feature '%s' is not supported.\n", feature_name);
+	if (!boot_cpu_has(X86_FEATURE_AVX)) {
+		pr_info("AVX instructions are not detected.\n");
 		return -ENODEV;
 	}
 
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c
index fb3594ddf731f5..854899bfec5d9d 100644
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -10,6 +10,7 @@
 #include <asm/desc.h>
 #include <asm/fred.h>
 #include <asm/idtentry.h>
+#include <asm/processor-flags.h>
 #include <asm/syscall.h>
 #include <asm/trapnr.h>
 #include <asm/traps.h>
@@ -71,7 +72,15 @@ static noinstr void fred_intx(struct pt_regs *regs)
 #endif
 
 	default:
-		return exc_general_protection(regs, 0);
+		/*
+		 * Reconstruct the #GP fault state that IDT delivery would produce.
+		 * Clear the software event flag so ERETU with TF set does not trap
+		 * before the resumed instruction. See prevent_single_step_upon_eretu().
+		 */
+		regs->ip -= regs->fred_ss.insnlen;
+		regs->flags |= X86_EFLAGS_RF;
+		regs->fred_ss.swevent = 0;
+		return exc_general_protection(regs, (regs->fred_ss.vector << 3) | 2);
 	}
 }
 
diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h
index bc3e549795c3f3..72a92cb9b53d3b 100644
--- a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h
+++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h
@@ -11,7 +11,7 @@
 #undef CONFIG_PGTABLE_LEVELS
 #undef CONFIG_ILLEGAL_POINTER_VALUE
 #undef CONFIG_SPARSEMEM_VMEMMAP
-#undef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
+#undef CONFIG_VMEMMAP_OPTIMIZATION
 #undef CONFIG_NR_CPUS
 #undef CONFIG_PARAVIRT_XXL
 
diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c
index dc564688f3d73a..54b13faba116c8 100644
--- a/arch/x86/events/amd/brs.c
+++ b/arch/x86/events/amd/brs.c
@@ -343,11 +343,10 @@ void amd_brs_drain(void)
 		if (!amd_brs_match_plm(event, from, to))
 			continue;
 
-		perf_clear_branch_entry_bitfields(br+nr);
-
-		br[nr].from = from;
-		br[nr].to   = to;
-
+		br[nr] = (struct perf_branch_entry){
+			.from	= from,
+			.to	= to,
+		};
 		nr++;
 	}
 empty:
diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index 9d9c961989d51c..a55646fcb8465c 100644
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -184,13 +184,6 @@ void amd_pmu_lbr_read(void)
 		    entry.to.split.reserved)
 			continue;
 
-		perf_clear_branch_entry_bitfields(br + out);
-
-		br[out].from	= sign_ext_branch_ip(entry.from.split.ip);
-		br[out].to	= sign_ext_branch_ip(entry.to.split.ip);
-		br[out].mispred	= entry.from.split.mispredict;
-		br[out].predicted = !br[out].mispred;
-
 		/*
 		 * Set branch speculation information using the status of
 		 * the valid and spec bits.
@@ -208,7 +201,14 @@ void amd_pmu_lbr_read(void)
 		 * speculative and took the correct path
 		 */
 		idx = (entry.to.split.valid << 1) | entry.to.split.spec;
-		br[out].spec = lbr_spec_map[idx];
+
+		br[out] = (struct perf_branch_entry){
+			.from		= sign_ext_branch_ip(entry.from.split.ip),
+			.to		= sign_ext_branch_ip(entry.to.split.ip),
+			.mispred	= entry.from.split.mispredict,
+			.predicted	= !entry.from.split.mispredict,
+			.spec		= lbr_spec_map[idx],
+		};
 		out++;
 	}
 
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c
index 7181973b5b1273..7aa5a5b652a2bd 100644
--- a/arch/x86/events/amd/uncore.c
+++ b/arch/x86/events/amd/uncore.c
@@ -39,11 +39,11 @@ static int pmu_version;
 struct amd_uncore_ctx {
 	int refcnt;
 	int cpu;
-	struct perf_event **events;
 	unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)];
 	int nr_active;
 	struct hrtimer hrtimer;
 	u64 hrtimer_duration;
+	struct perf_event *events[];
 };
 
 struct amd_uncore_pmu {
@@ -206,17 +206,15 @@ static int amd_uncore_add(struct perf_event *event, int flags)
 	struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
 	struct hw_perf_event *hwc = &event->hw;
 
-	/* are we already assigned? */
+	/*
+	 * Perf serializes ->add() and ->del() for an event. A successful
+	 * ->add() records the claimed slot in hwc->idx before returning, and
+	 * ->del() clears that slot before resetting hwc->idx. Therefore, an
+	 * existing assignment must be at hwc->idx.
+	 */
 	if (hwc->idx != -1 && ctx->events[hwc->idx] == event)
 		goto out;
 
-	for (i = 0; i < pmu->num_counters; i++) {
-		if (ctx->events[i] == event) {
-			hwc->idx = i;
-			goto out;
-		}
-	}
-
 	/* if not, take the first available counter */
 	hwc->idx = -1;
 	for (i = 0; i < pmu->num_counters; i++) {
@@ -248,19 +246,15 @@ static int amd_uncore_add(struct perf_event *event, int flags)
 
 static void amd_uncore_del(struct perf_event *event, int flags)
 {
-	int i;
 	struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
 	struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
 	struct hw_perf_event *hwc = &event->hw;
+	struct perf_event *old = event;
 
 	event->pmu->stop(event, PERF_EF_UPDATE);
 
-	for (i = 0; i < pmu->num_counters; i++) {
-		struct perf_event *tmp = event;
-
-		if (try_cmpxchg(&ctx->events[i], &tmp, NULL))
-			break;
-	}
+	/* ->del() follows a successful ->add(), so hwc->idx owns this slot. */
+	WARN_ON_ONCE(!try_cmpxchg(&ctx->events[hwc->idx], &old, NULL));
 
 	hwc->idx = -1;
 }
@@ -519,10 +513,8 @@ static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu)
 		if (cpu == ctx->cpu)
 			cpumask_clear_cpu(cpu, &pmu->active_mask);
 
-		if (!--ctx->refcnt) {
-			kfree(ctx->events);
+		if (!--ctx->refcnt)
 			kfree(ctx);
-		}
 
 		*per_cpu_ptr(pmu->ctx, cpu) = NULL;
 	}
@@ -567,18 +559,11 @@ static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
 		/* Allocate context if sibling does not exist */
 		if (!curr) {
 			node = cpu_to_node(cpu);
-			curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node);
+			curr = kzalloc_node(struct_size(curr, events, pmu->num_counters), GFP_KERNEL, node);
 			if (!curr)
 				goto fail;
 
 			curr->cpu = cpu;
-			curr->events = kzalloc_node(sizeof(*curr->events) *
-						    pmu->num_counters,
-						    GFP_KERNEL, node);
-			if (!curr->events) {
-				kfree(curr);
-				goto fail;
-			}
 
 			amd_uncore_init_hrtimer(curr);
 			curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC;
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 8b3ea0adb9655c..9b1a9032e36d50 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -93,7 +93,7 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling,   *x86_pmu.stop_scheduling);
 
 DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task,    *x86_pmu.sched_task);
 
-DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs,   *x86_pmu.drain_pebs);
+DEFINE_STATIC_CALL_RET0(x86_pmu_drain_pebs,   *x86_pmu.drain_pebs);
 DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);
 
 DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter);
@@ -408,6 +408,53 @@ set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
 	return x86_pmu_extra_regs(val, event);
 }
 
+static DEFINE_PER_CPU(struct xregs_state *, ext_regs_buf);
+
+static void release_ext_regs_buffers(void)
+{
+	int cpu;
+
+	if (!x86_pmu.ext_regs_mask)
+		return;
+
+	for_each_possible_cpu(cpu) {
+		kfree(per_cpu(ext_regs_buf, cpu));
+		per_cpu(ext_regs_buf, cpu) = NULL;
+	}
+}
+
+static void reserve_ext_regs_buffers(void)
+{
+	bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED);
+	unsigned int size;
+	int cpu;
+
+	if (!x86_pmu.ext_regs_mask)
+		return;
+
+	/* Add 64 bytes to satisfy the XSAVE area's 64-byte alignment. */
+	size = xstate_calculate_size(x86_pmu.ext_regs_mask, compacted) + 64;
+
+	for_each_possible_cpu(cpu) {
+		per_cpu(ext_regs_buf, cpu) = kzalloc_node(size, GFP_KERNEL,
+							  cpu_to_node(cpu));
+		if (WARN_ON_ONCE(!per_cpu(ext_regs_buf, cpu)))
+			goto err;
+	}
+
+	return;
+
+err:
+	release_ext_regs_buffers();
+}
+
+static inline struct xregs_state *get_ext_regs_buf(int cpu)
+{
+	void *buf = per_cpu(ext_regs_buf, cpu);
+
+	return buf ? PTR_ALIGN(buf, 64) : NULL;
+}
+
 int x86_reserve_hardware(void)
 {
 	int err = 0;
@@ -420,6 +467,7 @@ int x86_reserve_hardware(void)
 			} else {
 				reserve_ds_buffers();
 				reserve_lbr_buffers();
+				reserve_ext_regs_buffers();
 			}
 		}
 		if (!err)
@@ -436,6 +484,7 @@ void x86_release_hardware(void)
 		release_pmc_hardware();
 		release_ds_buffers();
 		release_lbr_buffers();
+		release_ext_regs_buffers();
 		mutex_unlock(&pmc_reserve_mutex);
 	}
 }
@@ -583,6 +632,79 @@ int x86_pmu_max_precise(struct pmu *pmu)
 	return precise;
 }
 
+static int pebs_simd_regs_validate(struct perf_event *event)
+{
+	u64 caps = hybrid(event->pmu, arch_pebs_cap).caps;
+
+	if (event_needs_xmm(event) &&
+	    !x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline)
+		return -EINVAL;
+	if (event_needs_xmm(event) &&
+	    x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM))
+		return -EINVAL;
+
+	if (event_needs_ssp(event) &&
+	    !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_GPR)))
+		return -EINVAL;
+	if (event_needs_ymm(event) &&
+	    !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_YMMH)))
+		return -EINVAL;
+	if (event_needs_egprs(event) &&
+	    !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_EGPRS)))
+		return -EINVAL;
+	if (event_needs_opmask(event) &&
+	    !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_OPMASK)))
+		return -EINVAL;
+	if (event_needs_low16_zmm(event) &&
+	    !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_ZMMH)))
+		return -EINVAL;
+	if (event_needs_high16_zmm(event) &&
+	    !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_H16ZMM)))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int event_simd_regs_validate(struct perf_event *event)
+{
+	u64 reserved = ~GENMASK_ULL(PERF_REG_MISC_MAX - 1, 0);
+
+	if (!get_ext_regs_buf(raw_smp_processor_id()))
+		return -ENOMEM;
+	/*
+	 * The XMM space in the perf_event_x86_regs is reclaimed
+	 * for eGPRs and other general registers.
+	 */
+	if (((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	     (event->attr.sample_regs_intr & reserved)) ||
+	    ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	     (event->attr.sample_regs_user & reserved)))
+		return -EINVAL;
+	if (event_needs_egprs(event) &&
+	    !(x86_pmu.ext_regs_mask & XFEATURE_MASK_APX))
+		return -EINVAL;
+	if (event_needs_ssp(event) &&
+	    !(x86_pmu.ext_regs_mask & XFEATURE_MASK_CET_USER))
+		return -EINVAL;
+	if (event_needs_xmm(event) &&
+	    !(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE))
+		return -EINVAL;
+	if (event_needs_ymm(event) &&
+	   !(x86_pmu.ext_regs_mask & XFEATURE_MASK_YMM))
+		return -EINVAL;
+	if (event_needs_low16_zmm(event) &&
+	    !(x86_pmu.ext_regs_mask & XFEATURE_MASK_ZMM_Hi256))
+		return -EINVAL;
+	if (event_needs_high16_zmm(event) &&
+	    !(x86_pmu.ext_regs_mask & XFEATURE_MASK_Hi16_ZMM))
+		return -EINVAL;
+	if (event_needs_opmask(event) &&
+	    !(x86_pmu.ext_regs_mask & XFEATURE_MASK_OPMASK))
+		return -EINVAL;
+
+	return 0;
+}
+
 int x86_pmu_hw_config(struct perf_event *event)
 {
 	if (event->attr.precise_ip) {
@@ -653,19 +775,38 @@ int x86_pmu_hw_config(struct perf_event *event)
 			return -EINVAL;
 	}
 
-	/* sample_regs_user never support XMM registers */
-	if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
-		return -EINVAL;
-	/*
-	 * Besides the general purpose registers, XMM registers may
-	 * be collected in PEBS on some platforms, e.g. Icelake
-	 */
-	if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) {
-		if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
-			return -EINVAL;
+	if (event->attr.sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) {
+		int ret;
 
-		if (!event->attr.precise_ip)
-			return -EINVAL;
+		if (event->attr.sample_simd_regs_enabled) {
+			if (!(event->pmu->capabilities & PERF_PMU_CAP_SIMD_REGS))
+				return -EINVAL;
+
+			if (event->attr.precise_ip) {
+				ret = pebs_simd_regs_validate(event);
+				if (ret)
+					return ret;
+			}
+			ret = event_simd_regs_validate(event);
+			if (ret)
+				return ret;
+		} else if (event_has_extended_regs(event)) {
+			if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
+				return -EINVAL;
+
+			if (event->attr.precise_ip) {
+				u64 caps = hybrid(event->pmu, arch_pebs_cap).caps;
+
+				if (x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM))
+					return -EINVAL;
+				if (!x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline)
+					return -EINVAL;
+			}
+			if (!get_ext_regs_buf(raw_smp_processor_id()))
+				return -ENOMEM;
+			if (!(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE))
+				return -EINVAL;
+		}
 	}
 
 	return x86_setup_perfctr(event);
@@ -721,9 +862,10 @@ void x86_pmu_disable_all(void)
 	}
 }
 
-struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data)
+struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr,
+						  struct x86_guest_pebs *guest_pebs)
 {
-	return static_call(x86_pmu_guest_get_msrs)(nr, data);
+	return static_call(x86_pmu_guest_get_msrs)(nr, guest_pebs);
 }
 EXPORT_SYMBOL_FOR_KVM(perf_guest_get_msrs);
 
@@ -1717,6 +1859,302 @@ static void x86_pmu_del(struct perf_event *event, int flags)
 	static_call_cond(x86_pmu_del)(event);
 }
 
+void x86_pmu_clear_perf_regs(struct pt_regs *regs)
+{
+	struct x86_perf_regs *perf_regs = container_of(regs, struct x86_perf_regs, regs);
+
+	perf_regs->abi = PERF_SAMPLE_REGS_ABI_NONE;
+	perf_regs->xmm_regs = NULL;
+	perf_regs->ymmh_regs = NULL;
+	perf_regs->zmmh_regs = NULL;
+	perf_regs->h16zmm_regs = NULL;
+	perf_regs->opmask_regs = NULL;
+	perf_regs->egpr_regs = NULL;
+	perf_regs->ssp = NULL;
+}
+
+static void update_perf_regs(struct x86_perf_regs *perf_regs,
+			     struct xregs_state *xsave, u64 bitmap)
+{
+	struct cet_user_state *cet;
+	u64 mask;
+
+	if (!xsave)
+		return;
+
+	/* Restrict to features actually saved by XSAVES */
+	mask = bitmap & xsave->header.xfeatures;
+
+	if (mask & XFEATURE_MASK_SSE)
+		perf_regs->xmm_space = xsave->i387.xmm_space;
+	if (mask & XFEATURE_MASK_YMM)
+		perf_regs->ymmh = get_xsave_addr(xsave, XFEATURE_YMM);
+	if (mask & XFEATURE_MASK_ZMM_Hi256)
+		perf_regs->zmmh = get_xsave_addr(xsave, XFEATURE_ZMM_Hi256);
+	if (mask & XFEATURE_MASK_Hi16_ZMM)
+		perf_regs->h16zmm = get_xsave_addr(xsave, XFEATURE_Hi16_ZMM);
+	if (mask & XFEATURE_MASK_OPMASK)
+		perf_regs->opmask = get_xsave_addr(xsave, XFEATURE_OPMASK);
+	if (mask & XFEATURE_MASK_APX)
+		perf_regs->egpr = get_xsave_addr(xsave, XFEATURE_APX);
+	if (mask & XFEATURE_MASK_CET_USER) {
+		cet = get_xsave_addr(xsave, XFEATURE_CET_USER);
+		perf_regs->ssp = cet ? &cet->user_ssp : NULL;
+	}
+}
+
+/*
+ * The x86 specific variant of perf_sample_regs_intr().
+ * Update data->regs_intr fields for extended registers (e.g., SIMD).
+ */
+static void x86_pmu_update_regs_intr(struct perf_event *event,
+				     struct perf_sample_data *data,
+				     struct pt_regs *regs,
+				     bool exclude_kernel)
+{
+	struct x86_perf_regs *perf_regs;
+
+	if (exclude_kernel && !user_mode(regs)) {
+		data->regs_intr.regs = NULL;
+		data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE;
+	} else {
+		data->regs_intr.regs = regs;
+		data->regs_intr.abi = perf_reg_abi(current);
+	}
+
+	data->dyn_size += sizeof(u64);
+	if (data->regs_intr.regs) {
+		data->dyn_size += hweight64(event->attr.sample_regs_intr) *
+				  sizeof(u64);
+		if (event_has_simd_regs(event)) {
+			data->dyn_size += perf_update_xregs_size(event, true);
+			data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
+		}
+
+		perf_regs = container_of(data->regs_intr.regs,
+					 struct x86_perf_regs, regs);
+		perf_regs->abi = data->regs_intr.abi;
+	}
+
+	/*
+	 * Set PERF_SAMPLE_REGS_INTR to bypass perf_sample_regs_intr() call
+	 * in perf_prepare_sample() function.
+	 */
+	data->sample_flags |= PERF_SAMPLE_REGS_INTR;
+}
+
+static DEFINE_PER_CPU(struct x86_perf_regs, x86_user_regs);
+
+static void x86_pmu_get_regs_user(struct perf_sample_data *data,
+				  struct pt_regs *regs)
+{
+	struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs);
+	struct perf_regs regs_user;
+
+	x86_pmu_clear_perf_regs(&x86_regs_user->regs);
+
+	perf_get_regs_user(&regs_user, regs);
+	data->regs_user.abi = regs_user.abi;
+	if (regs_user.regs) {
+		x86_regs_user->regs = *regs_user.regs;
+		data->regs_user.regs = &x86_regs_user->regs;
+	} else {
+		data->regs_user.regs = NULL;
+	}
+}
+
+/*
+ * The x86 specific variant of perf_sample_regs_user().
+ * Update data->regs_user fields for extended registers (e.g., SIMD).
+ */
+static void x86_pmu_update_regs_user(struct perf_event *event,
+				     struct perf_sample_data *data,
+				     struct pt_regs *regs)
+{
+	struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs);
+	struct perf_event_attr *attr = &event->attr;
+	struct x86_perf_regs *perf_regs;
+
+	/*
+	 * PERF_SAMPLE_REGS_INTR and PERF_SAMPLE_REGS_USER can both be
+	 * requested for one event. Keep user regs in a separate x86_perf_regs
+	 * instance, so intr-reg collection does not overwrite user-reg data.
+	 */
+	if (user_mode(regs)) {
+		x86_pmu_clear_perf_regs(&x86_regs_user->regs);
+		perf_regs = container_of(regs, struct x86_perf_regs, regs);
+		/* Copy all sampled regs data to x86_regs_user. */
+		*x86_regs_user = *perf_regs;
+		data->regs_user.regs = &x86_regs_user->regs;
+		data->regs_user.abi = perf_reg_abi(current);
+	} else if (is_user_task(current)) {
+		x86_pmu_get_regs_user(data, regs);
+	} else {
+		data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE;
+		data->regs_user.regs = NULL;
+	}
+
+	data->dyn_size += sizeof(u64);
+	if (data->regs_user.regs) {
+		data->dyn_size += hweight64(attr->sample_regs_user) * sizeof(u64);
+		if (event_has_simd_regs(event)) {
+			data->dyn_size += perf_update_xregs_size(event, false);
+			data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
+		}
+
+		x86_regs_user->abi = data->regs_user.abi;
+	}
+
+	/*
+	 * Set PERF_SAMPLE_REGS_USER to bypass perf_sample_regs_user() call
+	 * in perf_prepare_sample() function.
+	 */
+	data->sample_flags |= PERF_SAMPLE_REGS_USER;
+}
+
+/*
+ * This function retrieves cached user-space fpu registers (XMM/YMM/ZMM).
+ * If TIF_NEED_FPU_LOAD is set or PMI hits into guest, it indicates that
+ * the user-space FPU state is cached. Otherwise, the data should be read
+ * directly from the hardware registers.
+ */
+static inline u64 x86_pmu_update_user_xregs(struct perf_sample_data *data,
+					    struct pt_regs *regs,
+					    u64 mask, bool from_pebs)
+{
+	struct x86_perf_regs *perf_regs;
+	struct xregs_state *xsave;
+	struct fpu *fpu;
+	struct fpstate *fps;
+	u64 user_mask = mask;
+
+	if (!is_user_task(current))
+		return 0;
+
+	if (data->regs_user.abi == PERF_SAMPLE_REGS_ABI_NONE)
+		return 0;
+
+	/*
+	 * If PEBS hits kernel space, need to re-sample extended
+	 * registers for user space.
+	 */
+	if (user_mode(regs))
+		user_mask = from_pebs ? 0 : mask;
+
+	fpu = x86_task_fpu(current);
+	fps = READ_ONCE(fpu->__task_fpstate);
+	/*
+	 * If fpu->__task_fpstate is set, it points to the cached user
+	 * FPU state (e.g. with KVM guest-state swapping). Otherwise,
+	 * when TIF_NEED_FPU_LOAD is set, fpu->fpstate holds the cached
+	 * user state. If neither is true, the user state is live in hardware.
+	 */
+	if (user_mask && (test_thread_flag(TIF_NEED_FPU_LOAD) || fps)) {
+		perf_regs = container_of(data->regs_user.regs,
+					 struct x86_perf_regs, regs);
+		if (!fps)
+			fps = fpu->fpstate;
+		xsave = &fps->regs.xsave;
+
+		update_perf_regs(perf_regs, xsave, user_mask);
+		return 0;
+	}
+
+	return user_mask;
+}
+
+static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type)
+{
+	u64 mask = 0;
+
+	if (__event_needs_xmm(event, sample_type))
+		mask |= XFEATURE_MASK_SSE;
+	if (__event_needs_ymm(event, sample_type))
+		mask |= XFEATURE_MASK_YMM;
+	if (__event_needs_low16_zmm(event, sample_type))
+		mask |= XFEATURE_MASK_ZMM_Hi256;
+	if (__event_needs_high16_zmm(event, sample_type))
+		mask |= XFEATURE_MASK_Hi16_ZMM;
+	if (__event_needs_opmask(event, sample_type))
+		mask |= XFEATURE_MASK_OPMASK;
+	if (__event_needs_egprs(event, sample_type))
+		mask |= XFEATURE_MASK_APX;
+	if (__event_needs_ssp(event, sample_type))
+		mask |= XFEATURE_MASK_CET_USER;
+
+	return mask;
+}
+
+static void x86_pmu_sample_xregs(struct perf_event *event,
+				 struct perf_sample_data *data,
+				 struct pt_regs *regs,
+				 bool from_pebs)
+{
+	struct xregs_state *xsave = get_ext_regs_buf(smp_processor_id());
+	u64 sample_type = event->attr.sample_type;
+	struct x86_perf_regs *perf_regs;
+	u64 intr_mask = 0;
+	u64 user_mask = 0;
+
+	if (WARN_ON_ONCE(!xsave) || !in_nmi())
+		return;
+
+	if ((sample_type & PERF_SAMPLE_REGS_INTR) && data->regs_intr.regs) {
+		intr_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_INTR);
+		intr_mask &= x86_pmu.ext_regs_mask;
+		intr_mask = from_pebs ? 0 : intr_mask;
+	}
+
+	if ((sample_type & PERF_SAMPLE_REGS_USER) && data->regs_user.regs) {
+		user_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_USER);
+		user_mask &= x86_pmu.ext_regs_mask;
+		user_mask = x86_pmu_update_user_xregs(data, regs,
+						      user_mask, from_pebs);
+	}
+
+	if (user_mask | intr_mask) {
+		xsave->header.xfeatures = 0;
+		xsaves_nmi(xsave, user_mask | intr_mask);
+	}
+
+	if (intr_mask) {
+		perf_regs = container_of(data->regs_intr.regs,
+					 struct x86_perf_regs, regs);
+		update_perf_regs(perf_regs, xsave, intr_mask);
+	}
+
+	if (user_mask) {
+		perf_regs = container_of(data->regs_user.regs,
+					 struct x86_perf_regs, regs);
+		update_perf_regs(perf_regs, xsave, user_mask);
+	}
+}
+
+void x86_pmu_update_perf_regs(struct perf_event *event,
+			      struct perf_sample_data *data,
+			      struct pt_regs *regs,
+			      bool from_pebs)
+{
+	u64 sample_type = event->attr.sample_type;
+
+	if (!(sample_type &
+	      (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)))
+		return;
+
+	if (!event_needs_xmm(event) &&
+	    !event_has_simd_regs(event))
+		return;
+
+	if (sample_type & PERF_SAMPLE_REGS_INTR) {
+		x86_pmu_update_regs_intr(event, data, regs,
+					 event->attr.exclude_kernel);
+	}
+	if (sample_type & PERF_SAMPLE_REGS_USER)
+		x86_pmu_update_regs_user(event, data, regs);
+
+	x86_pmu_sample_xregs(event, data, regs, from_pebs);
+}
+
 int x86_pmu_handle_irq(struct pt_regs *regs)
 {
 	struct perf_sample_data data;
@@ -1800,9 +2238,11 @@ void perf_put_guest_lvtpc(void)
 EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc);
 #endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */
 
+static DEFINE_PER_CPU(struct x86_perf_regs, x86_intr_regs);
 static int
 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
 {
+	struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_intr_regs);
 	u64 start_clock;
 	u64 finish_clock;
 	int ret;
@@ -1826,7 +2266,8 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
 		return NMI_DONE;
 
 	start_clock = sched_clock();
-	ret = static_call(x86_pmu_handle_irq)(regs);
+	x86_regs->regs = *regs;
+	ret = static_call(x86_pmu_handle_irq)(&x86_regs->regs);
 	finish_clock = sched_clock();
 
 	perf_sample_event_took(finish_clock - start_clock);
@@ -2218,8 +2659,20 @@ static int __init init_hw_perf_events(void)
 
 	pmu.attr_update = x86_pmu.attr_update;
 
-	if (!is_hybrid())
+	if (!is_hybrid()) {
 		x86_pmu_show_pmu_cap(NULL);
+	} else {
+		int i;
+
+		/*
+		 * Init default ops.
+		 * Must be called before registering x86_pmu_starting_cpu(),
+		 * otherwise some key PMU fields, e.g., capabilities
+		 * initialized in x86_pmu_starting_cpu(), would be overwritten.
+		 */
+		for (i = 0; i < x86_pmu.num_hybrid_pmus; i++)
+			x86_pmu.hybrid_pmu[i].pmu = pmu;
+	}
 
 	if (!x86_pmu.read)
 		x86_pmu.read = _x86_pmu_read;
@@ -2266,7 +2719,6 @@ static int __init init_hw_perf_events(void)
 		for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
 			hybrid_pmu = &x86_pmu.hybrid_pmu[i];
 
-			hybrid_pmu->pmu = pmu;
 			hybrid_pmu->pmu.type = -1;
 			hybrid_pmu->pmu.attr_update = x86_pmu.attr_update;
 			hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE;
diff --git a/arch/x86/events/intel/bts.c b/arch/x86/events/intel/bts.c
index cbac54cb3a9ec5..5849392cf26d5b 100644
--- a/arch/x86/events/intel/bts.c
+++ b/arch/x86/events/intel/bts.c
@@ -66,9 +66,6 @@ static struct pmu bts_pmu;
 
 static int buf_nr_pages(struct page *page)
 {
-	if (!PagePrivate(page))
-		return 1;
-
 	return 1 << page_private(page);
 }
 
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 1ac2ca35db5375..0a34d674df590c 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -14,7 +14,6 @@
 #include <linux/slab.h>
 #include <linux/export.h>
 #include <linux/nmi.h>
-#include <linux/kvm_host.h>
 
 #include <asm/cpufeature.h>
 #include <asm/cpuid/api.h>
@@ -506,6 +505,8 @@ static struct event_constraint intel_pnc_event_constraints[] = {
 	INTEL_EVENT_CONSTRAINT(0xce, 0x1),
 
 	INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8),
+	INTEL_UEVENT_CONSTRAINT(0x01b2, 0xf),
+	INTEL_UEVENT_CONSTRAINT(0x02b2, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x0847, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x0446, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x0846, 0xf),
@@ -520,6 +521,14 @@ static struct extra_reg intel_pnc_extra_regs[] __read_mostly = {
 	INTEL_UEVENT_EXTRA_REG(0x022a, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1),
 	INTEL_UEVENT_EXTRA_REG(0x042a, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2),
 	INTEL_UEVENT_EXTRA_REG(0x082a, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3),
+	INTEL_UEVENT_EXTRA_REG(0x014f, MSR_OMR_0, 0x40ffffff0000ffffull, OMR_0),
+	INTEL_UEVENT_EXTRA_REG(0x024f, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1),
+	INTEL_UEVENT_EXTRA_REG(0x044f, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2),
+	INTEL_UEVENT_EXTRA_REG(0x084f, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3),
+	INTEL_UEVENT_EXTRA_REG(0x01d6, MSR_OMR_0, 0x40ffffff0000ffffull, OMR_0),
+	INTEL_UEVENT_EXTRA_REG(0x02d6, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1),
+	INTEL_UEVENT_EXTRA_REG(0x04d6, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2),
+	INTEL_UEVENT_EXTRA_REG(0x08d6, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3),
 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
 	INTEL_UEVENT_EXTRA_REG(0x02c6, MSR_PEBS_FRONTEND, 0x9, FE),
 	INTEL_UEVENT_EXTRA_REG(0x03c6, MSR_PEBS_FRONTEND, 0x7fff1f, FE),
@@ -2787,7 +2796,7 @@ static void __intel_pmu_enable_all(int added, bool pmi)
 	}
 
 	wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL,
-	       intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
+	       intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask);
 
 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
 		struct perf_event *event =
@@ -2985,9 +2994,9 @@ static inline void intel_set_masks(struct perf_event *event, int idx)
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
 	if (event->attr.exclude_host)
-		__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
+		__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask);
 	if (event->attr.exclude_guest)
-		__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
+		__set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask);
 	if (event_is_checkpointed(event))
 		__set_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
 }
@@ -2996,8 +3005,8 @@ static inline void intel_clear_masks(struct perf_event *event, int idx)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
-	__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
-	__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
+	__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask);
+	__clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask);
 	__clear_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
 }
 
@@ -3489,6 +3498,21 @@ static void intel_pmu_enable_event_ext(struct perf_event *event)
 			if (pebs_data_cfg & PEBS_DATACFG_XMMS)
 				ext |= ARCH_PEBS_VECR_XMM & cap.caps;
 
+			if (pebs_data_cfg & PEBS_DATACFG_YMMHS)
+				ext |= ARCH_PEBS_VECR_YMMH & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_EGPRS)
+				ext |= ARCH_PEBS_VECR_EGPRS & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_OPMASKS)
+				ext |= ARCH_PEBS_VECR_OPMASK & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_ZMMHS)
+				ext |= ARCH_PEBS_VECR_ZMMH & cap.caps;
+
+			if (pebs_data_cfg & PEBS_DATACFG_H16ZMMS)
+				ext |= ARCH_PEBS_VECR_H16ZMM & cap.caps;
+
 			if (pebs_data_cfg & PEBS_DATACFG_LBRS)
 				ext |= ARCH_PEBS_LBR & cap.caps;
 
@@ -3760,20 +3784,20 @@ static void intel_pmu_reset(void)
  *
  * The contents and other behavior of the guest event do not matter.
  */
-static void x86_pmu_handle_guest_pebs(struct pt_regs *regs,
-				      struct perf_sample_data *data)
+static int x86_pmu_handle_guest_pebs(struct pt_regs *regs,
+				     struct perf_sample_data *data)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_host_mask;
+	u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_exclude_guest_mask;
 	struct perf_event *event = NULL;
 	int bit;
 
 	if (!unlikely(perf_guest_state()))
-		return;
+		return 0;
 
 	if (!x86_pmu.pebs_ept || !x86_pmu.pebs_active ||
 	    !guest_pebs_idxs)
-		return;
+		return 0;
 
 	for_each_set_bit(bit, (unsigned long *)&guest_pebs_idxs, X86_PMC_IDX_MAX) {
 		event = cpuc->events[bit];
@@ -3783,9 +3807,14 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs,
 		perf_sample_data_init(data, 0, event->hw.last_period);
 		perf_event_overflow(event, data, regs);
 
-		/* Inject one fake event is enough. */
-		break;
+		/*
+		 * Inject one fake event is enough.
+		 * Returning 1 to inform PMI is handled.
+		 */
+		return 1;
 	}
+
+	return 0;
 }
 
 static int handle_pmi_common(struct pt_regs *regs, u64 status)
@@ -3834,9 +3863,11 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 	if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) {
 		u64 pebs_enabled = cpuc->pebs_enabled;
 
-		handled++;
-		x86_pmu_handle_guest_pebs(regs, &data);
-		static_call(x86_pmu_drain_pebs)(regs, &data);
+		handled += x86_pmu_handle_guest_pebs(regs, &data);
+		handled += static_call(x86_pmu_drain_pebs)(regs, &data);
+		/* Ensure no "suspicious NMI" warning for empty PEBS buffer. */
+		if (!handled)
+			handled++;
 
 		/*
 		 * PMI throttle may be triggered, which stops the PEBS event.
@@ -3863,8 +3894,10 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 	 */
 	if (__test_and_clear_bit(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT,
 				 (unsigned long *)&status)) {
-		handled++;
-		static_call(x86_pmu_drain_pebs)(regs, &data);
+		handled += static_call(x86_pmu_drain_pebs)(regs, &data);
+		/* Ensure no "suspicious NMI" warning for empty PEBS buffer. */
+		if (!handled)
+			handled++;
 
 		if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] &&
 		    is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS]))
@@ -3940,6 +3973,9 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 		if (has_branch_stack(event))
 			intel_pmu_lbr_save_brstack(&data, cpuc, event);
 
+		x86_pmu_clear_perf_regs(regs);
+		x86_pmu_update_perf_regs(event, &data, regs, false);
+
 		perf_event_overflow(event, &data, regs);
 	}
 
@@ -4707,14 +4743,20 @@ static void intel_pebs_aliases_skl(struct perf_event *event)
 static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event)
 {
 	unsigned long flags = x86_pmu.large_pebs_flags;
+	u64 gprs_mask = event->attr.sample_simd_regs_enabled ?
+			PEBS_GP_REGS | PERF_X86_EGPRS_MASK |
+			BIT_ULL(PERF_REG_X86_SSP) :
+			PEBS_GP_REGS | PERF_REG_EXTENDED_MASK;
 
 	if (event->attr.use_clockid)
 		flags &= ~PERF_SAMPLE_TIME;
 	if (!event->attr.exclude_kernel)
 		flags &= ~PERF_SAMPLE_REGS_USER;
-	if (event->attr.sample_regs_user & ~PEBS_GP_REGS)
+	if ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_regs_user & ~gprs_mask))
 		flags &= ~PERF_SAMPLE_REGS_USER;
-	if (event->attr.sample_regs_intr & ~PEBS_GP_REGS)
+	if ((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_regs_intr & ~gprs_mask))
 		flags &= ~PERF_SAMPLE_REGS_INTR;
 	return flags;
 }
@@ -5285,26 +5327,29 @@ static int intel_pmu_hw_config(struct perf_event *event)
  * when it uses {RD,WR}MSR, which should be handled by the KVM context,
  * specifically in the intel_pmu_{get,set}_msr().
  */
-static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
+static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr,
+							  struct x86_guest_pebs *guest_pebs)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
-	struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data;
 	u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl);
-	u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable;
-	int global_ctrl, pebs_enable;
+	u64 pebs_mask = intel_ctrl & cpuc->pebs_enabled & x86_pmu.pebs_capable;
+	u64 guest_pebs_mask;
+	int global_ctrl;
 
 	/*
 	 * In addition to obeying exclude_guest/exclude_host, remove bits being
 	 * used for PEBS when running a guest, because PEBS writes to virtual
-	 * addresses (not physical addresses).
+	 * addresses (not physical addresses).  If the guest wants to utilize
+	 * PEBS, and PEBS can be safely enabled in the guest, bits for the guest's
+	 * PEBS-enabled counters will be OR'd back in as appropriate.
 	 */
 	*nr = 0;
 	global_ctrl = (*nr)++;
 	arr[global_ctrl] = (struct perf_guest_switch_msr){
 		.msr = MSR_CORE_PERF_GLOBAL_CTRL,
-		.host = intel_ctrl & ~cpuc->intel_ctrl_guest_mask,
-		.guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask,
+		.host = intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask,
+		.guest = intel_ctrl & ~cpuc->intel_ctrl_exclude_guest_mask & ~pebs_mask,
 	};
 
 	if (!x86_pmu.ds_pebs)
@@ -5327,45 +5372,65 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data)
 		return arr;
 	}
 
-	if (!kvm_pmu || !x86_pmu.pebs_ept)
+	/*
+	 * If the CPU doesn't support PEBS in the guest, then there's nothing
+	 * more to do as disabling PMCs via PERF_GLOBAL_CTRL is sufficient on
+	 * CPUs with guest/host isolation.
+	 */
+	if (!x86_pmu.pebs_ept)
 		return arr;
 
+	/*
+	 * Restrict guest PEBS events to counters that (a) perf supports, (b)
+	 * the guest wants to use for PEBS, (c) are not excluded from counting
+	 * in the guest, and (d) _are_ excluded from counting in the host.
+	 */
+	guest_pebs_mask = pebs_mask & guest_pebs->enable &
+			  ~cpuc->intel_ctrl_exclude_guest_mask &
+			  cpuc->intel_ctrl_exclude_host_mask;
+
+	/*
+	 * FIXME: Allow guest and host usage of PEBS events to co-exist instead
+	 *        of disabling guest PEBS entirely if the host is using PEBS.
+	 *        What exactly goes wrong if guest and host are using PEBS is
+	 *        unknown.
+	 */
+	if (pebs_mask & ~cpuc->intel_ctrl_exclude_host_mask)
+		guest_pebs_mask = 0;
+
+	/*
+	 * Context switch DS_AREA and PEBS_DATA_CFG if and only if PEBS will be
+	 * active in the guest; if no records will be generated while the guest
+	 * is running, then simply keep the host values resident in hardware.
+	 */
 	arr[(*nr)++] = (struct perf_guest_switch_msr){
 		.msr = MSR_IA32_DS_AREA,
 		.host = (unsigned long)cpuc->ds,
-		.guest = kvm_pmu->ds_area,
+		.guest = guest_pebs_mask ? guest_pebs->ds_area : (unsigned long)cpuc->ds,
 	};
 
 	if (x86_pmu.intel_cap.pebs_baseline) {
 		arr[(*nr)++] = (struct perf_guest_switch_msr){
 			.msr = MSR_PEBS_DATA_CFG,
 			.host = cpuc->active_pebs_data_cfg,
-			.guest = kvm_pmu->pebs_data_cfg,
+			.guest = guest_pebs_mask ? guest_pebs->data_cfg :
+						   cpuc->active_pebs_data_cfg,
 		};
 	}
 
-	pebs_enable = (*nr)++;
-	arr[pebs_enable] = (struct perf_guest_switch_msr){
-		.msr = MSR_IA32_PEBS_ENABLE,
-		.host = cpuc->pebs_enabled & ~cpuc->intel_ctrl_guest_mask,
-		.guest = pebs_mask & ~cpuc->intel_ctrl_host_mask & kvm_pmu->pebs_enable,
-	};
-
-	if (arr[pebs_enable].host) {
-		/* Disable guest PEBS if host PEBS is enabled. */
-		arr[pebs_enable].guest = 0;
-	} else {
-		/* Disable guest PEBS thoroughly for cross-mapped PEBS counters. */
-		arr[pebs_enable].guest &= ~kvm_pmu->host_cross_mapped_mask;
-		arr[global_ctrl].guest &= ~kvm_pmu->host_cross_mapped_mask;
-		/* Set hw GLOBAL_CTRL bits for PEBS counter when it runs for guest */
-		arr[global_ctrl].guest |= arr[pebs_enable].guest;
-	}
-
+	/*
+	 * Do NOT mess with PEBS_ENABLED.  As above, disabling counters via
+	 * PERF_GLOBAL_CTRL is sufficient, and loading a stale PEBS_ENABLED,
+	 * e.g. on VM-Exit, can put the system in a bad state.  Simply enable
+	 * counters in PERF_GLOBAL_CTRL, as perf load PEBS_ENABLED with the
+	 * full value, i.e. perf *also* relies on PERF_GLOBAL_CTRL.
+	 */
+	arr[global_ctrl].guest |= guest_pebs_mask;
 	return arr;
 }
 
-static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr, void *data)
+static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr,
+							 struct x86_guest_pebs *guest_pebs)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
@@ -6176,12 +6241,46 @@ static inline bool intel_pmu_broken_perf_cap(void)
 	return false;
 }
 
-static inline void __intel_update_pmu_caps(struct pmu *pmu)
+static inline void __intel_update_pmu_xregs_caps(struct pmu *pmu)
 {
 	struct pmu *dest_pmu = pmu ? pmu : x86_get_pmu(smp_processor_id());
 
-	if (hybrid(pmu, arch_pebs_cap).caps & ARCH_PEBS_VECR_XMM)
-		dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
+	/* Only support the extension when XSAVES is available. */
+	if (!boot_cpu_has(X86_FEATURE_XSAVES))
+		return;
+
+	if (!boot_cpu_has(X86_FEATURE_XMM) ||
+	    !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
+		return;
+
+	/*
+	 * On current hybrid platforms, P-cores and E-cores expose the same
+	 * XSAVE feature set. Therefore, using the global x86_pmu.ext_regs_mask
+	 * is sufficient to represent the hardware-supported XSAVE features.
+	 */
+	x86_pmu.ext_regs_mask |= XFEATURE_MASK_SSE;
+
+	if (boot_cpu_has(X86_FEATURE_AVX) &&
+	    cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL))
+		x86_pmu.ext_regs_mask |= XFEATURE_MASK_YMM;
+	if (boot_cpu_has(X86_FEATURE_APX) &&
+	    cpu_has_xfeatures(XFEATURE_MASK_APX, NULL))
+		x86_pmu.ext_regs_mask |= XFEATURE_MASK_APX;
+	if (boot_cpu_has(X86_FEATURE_AVX512F)) {
+		if (cpu_has_xfeatures(XFEATURE_MASK_OPMASK, NULL))
+			x86_pmu.ext_regs_mask |= XFEATURE_MASK_OPMASK;
+		if (cpu_has_xfeatures(XFEATURE_MASK_ZMM_Hi256, NULL))
+			x86_pmu.ext_regs_mask |= XFEATURE_MASK_ZMM_Hi256;
+		if (cpu_has_xfeatures(XFEATURE_MASK_Hi16_ZMM, NULL))
+			x86_pmu.ext_regs_mask |= XFEATURE_MASK_Hi16_ZMM;
+	}
+	if (cpu_feature_enabled(X86_FEATURE_USER_SHSTK) &&
+	    cpu_has_xfeatures(XFEATURE_MASK_CET_USER, NULL))
+		x86_pmu.ext_regs_mask |= XFEATURE_MASK_CET_USER;
+
+	dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
+	if (x86_pmu.ext_regs_mask > XFEATURE_MASK_SSE)
+		dest_pmu->capabilities |= PERF_PMU_CAP_SIMD_REGS;
 }
 
 static inline void __intel_update_large_pebs_flags(struct pmu *pmu)
@@ -6252,12 +6351,10 @@ static void update_pmu_cap_from_perfmonext(struct pmu *pmu)
 		hybrid(pmu, arch_pebs_cap).counters = pebs_mask;
 		hybrid(pmu, arch_pebs_cap).pdists = pdists_mask;
 
-		if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask)) {
+		if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask))
 			x86_pmu.arch_pebs = 0;
-		} else {
-			__intel_update_pmu_caps(pmu);
+		else
 			__intel_update_large_pebs_flags(pmu);
-		}
 	} else {
 		WARN_ON(x86_pmu.arch_pebs == 1);
 		x86_pmu.arch_pebs = 0;
@@ -6281,6 +6378,7 @@ static void intel_update_pmu_caps(struct pmu *pmu)
 		    hybrid_pmu(pmu)->pmu_type == hybrid_big)
 			hybrid(pmu, intel_cap).perf_metrics = 1;
 	}
+	__intel_update_pmu_xregs_caps(pmu);
 }
 
 static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu)
@@ -6434,8 +6532,6 @@ static void intel_pmu_cpu_starting(int cpu)
 		}
 	}
 
-	__intel_update_pmu_caps(cpuc->pmu);
-
 	if (!cpuc->shared_regs)
 		return;
 
@@ -6561,6 +6657,8 @@ static void intel_pmu_filter(struct pmu *pmu, int cpu, bool *ret)
 
 PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
 
+PMU_FORMAT_ATTR(offmodule_rsp, "config1:0-63");
+
 PMU_FORMAT_ATTR(ldlat, "config1:0-15");
 
 PMU_FORMAT_ATTR(frontend, "config1:0-23");
@@ -6609,6 +6707,20 @@ static struct attribute *skl_format_attr[] = {
 	NULL,
 };
 
+static struct attribute *pnc_format_attr_rtm[] = {
+	&format_attr_in_tx.attr,
+	&format_attr_in_tx_cp.attr,
+	&format_attr_offmodule_rsp.attr,
+	&format_attr_ldlat.attr,
+	NULL
+};
+
+static struct attribute *pnc_format_attr[] = {
+	&format_attr_offmodule_rsp.attr,
+	&format_attr_ldlat.attr,
+	NULL
+};
+
 static __initconst const struct x86_pmu core_pmu = {
 	.name			= "core",
 	.handle_irq		= x86_pmu_handle_irq,
@@ -7474,6 +7586,7 @@ static struct attribute *adl_hybrid_tsx_attrs[] = {
 FORMAT_ATTR_HYBRID(in_tx,       hybrid_big);
 FORMAT_ATTR_HYBRID(in_tx_cp,    hybrid_big);
 FORMAT_ATTR_HYBRID(offcore_rsp, hybrid_big_small_tiny);
+FORMAT_ATTR_HYBRID(offmodule_rsp, hybrid_big_small_tiny);
 FORMAT_ATTR_HYBRID(ldlat,       hybrid_big_small_tiny);
 FORMAT_ATTR_HYBRID(frontend,    hybrid_big);
 
@@ -7512,6 +7625,23 @@ static struct attribute *mtl_hybrid_extra_attr[] = {
 	NULL
 };
 
+static struct attribute *nvl_hybrid_extra_attr_rtm[] = {
+	ADL_HYBRID_RTM_FORMAT_ATTR,
+	FORMAT_HYBRID_PTR(offmodule_rsp),
+	FORMAT_HYBRID_PTR(ldlat),
+	FORMAT_HYBRID_PTR(frontend),
+	FORMAT_HYBRID_PTR(snoop_rsp),
+	NULL
+};
+
+static struct attribute *nvl_hybrid_extra_attr[] = {
+	FORMAT_HYBRID_PTR(offmodule_rsp),
+	FORMAT_HYBRID_PTR(ldlat),
+	FORMAT_HYBRID_PTR(frontend),
+	FORMAT_HYBRID_PTR(snoop_rsp),
+	NULL
+};
+
 static bool is_attr_for_this_pmu(struct kobject *kobj, struct attribute *attr)
 {
 	struct device *dev = kobj_to_dev(kobj);
@@ -8549,6 +8679,8 @@ __init int intel_pmu_init(void)
 	case INTEL_DIAMONDRAPIDS_X:
 		intel_pmu_init_pnc(NULL);
 		x86_pmu.pebs_latency_data = pnc_latency_data;
+		extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
+			     pnc_format_attr_rtm : pnc_format_attr;
 
 		pr_cont("Panthercove events, ");
 		name = "panthercove";
@@ -8557,13 +8689,12 @@ __init int intel_pmu_init(void)
 	glc_common:
 		intel_pmu_init_glc(NULL);
 		intel_pmu_pebs_data_source_skl(true);
-
+		extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
+			hsw_format_attr : nhm_format_attr;
 	glc_base:
 		x86_pmu.pebs_ept = 1;
 		x86_pmu.hw_config = hsw_hw_config;
 		x86_pmu.get_event_constraints = glc_get_event_constraints;
-		extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
-			hsw_format_attr : nhm_format_attr;
 		extra_skl_attr = skl_format_attr;
 		mem_attr = glc_events_attrs;
 		td_attr = glc_td_events_attrs;
@@ -8785,7 +8916,7 @@ __init int intel_pmu_init(void)
 		mem_attr = mtl_hybrid_mem_attrs;
 		tsx_attr = adl_hybrid_tsx_attrs;
 		extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
-			mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr;
+			nvl_hybrid_extra_attr_rtm : nvl_hybrid_extra_attr;
 
 		/* Initialize big core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
@@ -8794,8 +8925,6 @@ __init int intel_pmu_init(void)
 		/* Initialize Atom core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
 		intel_pmu_init_arw(&pmu->pmu);
-
-		intel_pmu_pebs_data_source_lnl();
 		break;
 
 	default:
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index b98029b4405218..8444670cee4aea 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -277,8 +277,8 @@ static u64 pnc_pebs_l2_hit_data_source[PNC_PEBS_DATA_SOURCE_MAX] = {
 	0,							/* 0x06: Reserved */
 	OP_LH | P(LVL, L2)  | LEVEL(L2) | P(SNOOP, HIT),	/* 0x07: L2 Hit Snoop HIT */
 	OP_LH | P(LVL, L2)  | LEVEL(L2) | P(SNOOP, HITM),	/* 0x08: L2 Hit Snoop Hit Modified */
-	OP_LH | P(LVL, L2)  | LEVEL(L2) | P(SNOOP, MISS),	/* 0x09: Prefetch Promotion */
-	OP_LH | P(LVL, L2)  | LEVEL(L2) | P(SNOOP, MISS),	/* 0x0a: Cross Core Prefetch Promotion */
+	OP_LH | P(LVL, L2)  | LEVEL(L2) | P(SNOOP, NONE),	/* 0x09: Prefetch Promotion */
+	OP_LH | P(LVL, L2)  | LEVEL(L2) | P(SNOOP, NONE),	/* 0x0a: Cross Core Prefetch Promotion */
 	0,							/* 0x0b: Reserved */
 	0,							/* 0x0c: Reserved */
 	0,							/* 0x0d: Reserved */
@@ -455,6 +455,7 @@ static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock)
 static u64 __grt_latency_data(struct perf_event *event, u64 status,
 			       u8 dse, bool tlb, bool lock, bool blk)
 {
+	union perf_mem_data_src src;
 	u64 val;
 
 	WARN_ON_ONCE(is_hybrid() &&
@@ -470,7 +471,16 @@ static u64 __grt_latency_data(struct perf_event *event, u64 status,
 	else
 		val |= P(BLK, NA);
 
-	return val;
+	src.val = val;
+
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW))
+		src.mem_op = P(OP, LOAD);
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW))
+		src.mem_op = P(OP, STORE);
+
+	return src.val;
 }
 
 u64 grt_latency_data(struct perf_event *event, u64 status)
@@ -528,7 +538,11 @@ static u64 arw_latency_data(struct perf_event *event, u64 status)
 		val |= P(BLK, NA);
 
 	src.val = val;
-	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW))
+		src.mem_op = P(OP, LOAD);
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW))
 		src.mem_op = P(OP, STORE);
 
 	return src.val;
@@ -563,7 +577,11 @@ static u64 lnc_latency_data(struct perf_event *event, u64 status)
 		val |= P(BLK, NA);
 
 	src.val = val;
-	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW))
+		src.mem_op = P(OP, LOAD);
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW))
 		src.mem_op = P(OP, STORE);
 
 	return src.val;
@@ -621,7 +639,11 @@ u64 pnc_latency_data(struct perf_event *event, u64 status)
 		val |= P(BLK, NA);
 
 	src.val = val;
-	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW))
+		src.mem_op = P(OP, LOAD);
+	if (event->hw.flags &
+	    (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW))
 		src.mem_op = P(OP, STORE);
 
 	return src.val;
@@ -1291,22 +1313,22 @@ struct event_constraint intel_glm_pebs_event_constraints[] = {
 
 struct event_constraint intel_grt_pebs_event_constraints[] = {
 	/* Allow all events as PEBS with no flags */
-	INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3),
-	INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0x3f),
+	INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3),
+	INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0x3f),
 	EVENT_CONSTRAINT_END
 };
 
 struct event_constraint intel_cmt_pebs_event_constraints[] = {
 	/* Allow all events as PEBS with no flags */
-	INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3),
-	INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff),
+	INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3),
+	INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff),
 	EVENT_CONSTRAINT_END
 };
 
 struct event_constraint intel_dkt_pebs_event_constraints[] = {
 	/* Allow all events as PEBS with no flags */
-	INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xff),
-	INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff),
+	INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0xff),
+	INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff),
 	EVENT_CONSTRAINT_END
 };
 
@@ -1496,24 +1518,8 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1),		/* OCR.* events */
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1),		/* OCR.* events */
 
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x04a4, 0x1),		/* TOPDOWN.BAD_SPEC_SLOTS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x08a4, 0x1),		/* TOPDOWN.BR_MISPREDICT_SLOTS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x10a4, 0x8),		/* TOPDOWN.MEMORY_BOUND_SLOTS */
-
 	INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc),
 	INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3),
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_STORES */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf),	/* MEM_INST_RETIRED.LOCK_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf),	/* MEM_INST_RETIRED.SPLIT_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf),	/* MEM_INST_RETIRED.SPLIT_STORES */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf),	/* MEM_INST_RETIRED.ALL_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf),	/* MEM_INST_RETIRED.ALL_STORES */
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x87d0, 0x3ff),		/* MEM_INST_RETIRED.ANY */
-
-	INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf),
-
-	INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf),
 
 	/*
 	 * Everything else is handled by PMU_FL_PEBS_ALL, because we
@@ -1526,18 +1532,6 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
 struct event_constraint intel_pnc_pebs_event_constraints[] = {
 	INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0xfc),
 	INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3),
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_STORES */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf),	/* MEM_INST_RETIRED.LOCK_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf),	/* MEM_INST_RETIRED.SPLIT_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf),	/* MEM_INST_RETIRED.SPLIT_STORES */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf),	/* MEM_INST_RETIRED.ALL_LOADS */
-	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf),	/* MEM_INST_RETIRED.ALL_STORES */
-
-	INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf),
-
-	INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf),
-	INTEL_FLAGS_EVENT_CONSTRAINT(0xd6, 0xf),
 
 	/*
 	 * Everything else is handled by PMU_FL_PEBS_ALL, because we
@@ -1710,6 +1704,7 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)
 	u64 sample_type = attr->sample_type;
 	u64 pebs_data_cfg = 0;
 	bool gprs, tsx_weight;
+	u64 xgprs_mask;
 
 	if (!(sample_type & ~(PERF_SAMPLE_IP|PERF_SAMPLE_TIME)) &&
 	    attr->precise_ip > 1)
@@ -1724,10 +1719,13 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)
 	 * + precise_ip < 2 for the non event IP
 	 * + For RTM TSX weight we need GPRs for the abort code.
 	 */
+	xgprs_mask = event->attr.sample_simd_regs_enabled ?
+		     PEBS_GP_REGS | BIT_ULL(PERF_REG_X86_SSP) :
+		     PEBS_GP_REGS;
 	gprs = ((sample_type & PERF_SAMPLE_REGS_INTR) &&
-		(attr->sample_regs_intr & PEBS_GP_REGS)) ||
+		(attr->sample_regs_intr & xgprs_mask)) ||
 	       ((sample_type & PERF_SAMPLE_REGS_USER) &&
-		(attr->sample_regs_user & PEBS_GP_REGS));
+		(attr->sample_regs_user & xgprs_mask));
 
 	tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) &&
 		     ((attr->config & INTEL_ARCH_EVENT_MASK) ==
@@ -1736,9 +1734,20 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event)
 	if (gprs || (attr->precise_ip < 2) || tsx_weight)
 		pebs_data_cfg |= PEBS_DATACFG_GP;
 
-	if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
-	    (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK))
-		pebs_data_cfg |= PEBS_DATACFG_XMMS;
+	if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) {
+		if (event_needs_xmm(event))
+			pebs_data_cfg |= PEBS_DATACFG_XMMS;
+		if (x86_pmu.arch_pebs && event_needs_ymm(event))
+			pebs_data_cfg |= PEBS_DATACFG_YMMHS;
+		if (x86_pmu.arch_pebs && event_needs_low16_zmm(event))
+			pebs_data_cfg |= PEBS_DATACFG_ZMMHS;
+		if (x86_pmu.arch_pebs && event_needs_high16_zmm(event))
+			pebs_data_cfg |= PEBS_DATACFG_H16ZMMS;
+		if (x86_pmu.arch_pebs && event_needs_opmask(event))
+			pebs_data_cfg |= PEBS_DATACFG_OPMASKS;
+		if (x86_pmu.arch_pebs && event_needs_egprs(event))
+			pebs_data_cfg |= PEBS_DATACFG_EGPRS;
+	}
 
 	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
 		/*
@@ -2529,7 +2538,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
 		return;
 
 	perf_regs = container_of(regs, struct x86_perf_regs, regs);
-	perf_regs->xmm_regs = NULL;
+	x86_pmu_clear_perf_regs(regs);
 
 	format_group = basic->format_group;
 
@@ -2614,6 +2623,8 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
 		next_record += nr * sizeof(u64);
 	}
 
+	x86_pmu_update_perf_regs(event, data, regs, true);
+
 	WARN_ONCE(next_record != __pebs + basic->format_size,
 			"PEBS record size %u, expected %llu, config %llx\n",
 			basic->format_size,
@@ -2646,7 +2657,7 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
 		return;
 
 	perf_regs = container_of(regs, struct x86_perf_regs, regs);
-	perf_regs->xmm_regs = NULL;
+	x86_pmu_clear_perf_regs(regs);
 
 	__setup_perf_sample_data(event, iregs, data);
 
@@ -2654,6 +2665,9 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
 
 again:
 	header = at;
+	if (!header->size)
+		return;
+
 	next_record = at + sizeof(struct arch_pebs_header);
 	if (header->basic) {
 		struct arch_pebs_basic *basic = next_record;
@@ -2684,6 +2698,11 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
 		__setup_pebs_gpr_group(event, regs,
 				       (struct pebs_gprs *)gprs,
 				       sample_type);
+
+		/* Currently only user space mode enables SSP. */
+		if (user_mode(regs) && (sample_type &
+		    (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)))
+			perf_regs->ssp = &gprs->ssp;
 	}
 
 	if (header->aux) {
@@ -2696,14 +2715,63 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
 					   meminfo->tsx_tuning, ax);
 	}
 
-	if (header->xmm) {
+	if (header->xmm || header->ymmh || header->egpr ||
+	    header->opmask || header->zmmh || header->h16zmm) {
+		struct arch_pebs_xer_header *xer_header = next_record;
 		struct pebs_xmm *xmm;
+		struct ymmh_struct *ymmh;
+		struct avx_512_zmm_uppers_state *zmmh;
+		struct avx_512_hi16_state *h16zmm;
+		struct avx_512_opmask_state *opmask;
+		struct apx_state *egpr;
 
 		next_record += sizeof(struct arch_pebs_xer_header);
 
-		xmm = next_record;
-		perf_regs->xmm_regs = xmm->xmm;
-		next_record = xmm + 1;
+		if (header->xmm) {
+			xmm = next_record;
+			/*
+			 * Only output XMM regs to user space when arch-PEBS
+			 * really writes data into xstate area.
+			 */
+			if (xer_header->xstate & XFEATURE_MASK_SSE)
+				perf_regs->xmm_regs = xmm->xmm;
+			next_record = xmm + 1;
+		}
+
+		if (header->ymmh) {
+			ymmh = next_record;
+			if (xer_header->xstate & XFEATURE_MASK_YMM)
+				perf_regs->ymmh = ymmh;
+			next_record = ymmh + 1;
+		}
+
+		if (header->egpr) {
+			egpr = next_record;
+			if (xer_header->xstate & XFEATURE_MASK_APX)
+				perf_regs->egpr = egpr;
+			next_record = egpr + 1;
+		}
+
+		if (header->opmask) {
+			opmask = next_record;
+			if (xer_header->xstate & XFEATURE_MASK_OPMASK)
+				perf_regs->opmask = opmask;
+			next_record = opmask + 1;
+		}
+
+		if (header->zmmh) {
+			zmmh = next_record;
+			if (xer_header->xstate & XFEATURE_MASK_ZMM_Hi256)
+				perf_regs->zmmh = zmmh;
+			next_record = zmmh + 1;
+		}
+
+		if (header->h16zmm) {
+			h16zmm = next_record;
+			if (xer_header->xstate & XFEATURE_MASK_Hi16_ZMM)
+				perf_regs->h16zmm = h16zmm;
+			next_record = h16zmm + 1;
+		}
 	}
 
 	if (header->lbr) {
@@ -2748,6 +2816,8 @@ static void setup_arch_pebs_sample_data(struct perf_event *event,
 		at = at + header->size;
 		goto again;
 	}
+
+	x86_pmu_update_perf_regs(event, data, regs, true);
 }
 
 static inline void *
@@ -2871,13 +2941,21 @@ __intel_pmu_pebs_last_event(struct perf_event *event,
 			    struct pt_regs *iregs,
 			    struct pt_regs *regs,
 			    struct perf_sample_data *data,
-			    void *at,
-			    int count,
+			    void *at, int count, bool corrupted,
 			    setup_fn setup_sample)
 {
 	struct hw_perf_event *hwc = &event->hw;
 
-	setup_sample(event, iregs, at, data, regs);
+	/* Skip parsing corrupted PEBS record. */
+	if (corrupted) {
+		/* Clear stale register states in previous records. */
+		memset(regs, 0, sizeof(*regs));
+		x86_pmu_clear_perf_regs(regs);
+		perf_sample_data_init(data, 0, event->hw.last_period);
+	} else {
+		setup_sample(event, iregs, at, data, regs);
+	}
+
 	if (iregs == &dummy_iregs) {
 		/*
 		 * The PEBS records may be drained in the non-overflow context,
@@ -2895,12 +2973,16 @@ __intel_pmu_pebs_last_event(struct perf_event *event,
 	}
 
 	if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
-		if ((is_pebs_counter_event_group(event))) {
-			/*
-			 * The value of each sample has been updated when setup
-			 * the corresponding sample data.
-			 */
-			perf_event_update_userpage(event);
+		if (is_pebs_counter_event_group(event)) {
+			if (corrupted) {
+				intel_pmu_save_and_restart_reload(event, 1);
+			} else {
+				/*
+				 * The value of each sample has been updated
+				 * when setup the corresponding sample data.
+				 */
+				perf_event_update_userpage(event);
+			}
 		} else {
 			/*
 			 * Now, auto-reload is only enabled in fixed period mode.
@@ -2924,13 +3006,15 @@ __intel_pmu_pebs_last_event(struct perf_event *event,
 		 * counters-snapshotting record, only needs to set the new
 		 * period for the counter.
 		 */
-		if (is_pebs_counter_event_group(event))
+		if (is_pebs_counter_event_group(event) && !corrupted)
 			static_call(x86_pmu_set_period)(event);
 		else
 			intel_pmu_save_and_restart(event);
 	}
 }
 
+static DEFINE_PER_CPU(struct x86_perf_regs, x86_pebs_regs);
+
 static __always_inline void
 __intel_pmu_pebs_events(struct perf_event *event,
 			struct pt_regs *iregs,
@@ -2940,25 +3024,29 @@ __intel_pmu_pebs_events(struct perf_event *event,
 			setup_fn setup_sample)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
-	struct x86_perf_regs perf_regs;
-	struct pt_regs *regs = &perf_regs.regs;
+	struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs);
+	struct pt_regs *regs = &perf_regs->regs;
 	void *at = get_next_pebs_record_by_bit(base, top, bit);
 	int cnt = count;
 
+	x86_pmu_clear_perf_regs(regs);
+
 	if (!iregs)
 		iregs = &dummy_iregs;
 
 	while (cnt > 1) {
-		__intel_pmu_pebs_event(event, iregs, regs, data, at, setup_sample);
+		__intel_pmu_pebs_event(event, iregs, regs, data,
+				       at, setup_sample);
 		at += cpuc->pebs_record_size;
 		at = get_next_pebs_record_by_bit(at, top, bit);
 		cnt--;
 	}
 
-	__intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample);
+	__intel_pmu_pebs_last_event(event, iregs, regs, data, at,
+				    count, false, setup_sample);
 }
 
-static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
+static int intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct debug_store *ds = cpuc->ds;
@@ -2967,7 +3055,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_
 	int n;
 
 	if (!x86_pmu.pebs_active)
-		return;
+		return 0;
 
 	at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
 	top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
@@ -2978,22 +3066,25 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_
 	ds->pebs_index = ds->pebs_buffer_base;
 
 	if (!test_bit(0, cpuc->active_mask))
-		return;
+		return 0;
 
 	WARN_ON_ONCE(!event);
 
 	if (!event->attr.precise_ip)
-		return;
+		return 0;
 
 	n = top - at;
 	if (n <= 0) {
 		if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
 			intel_pmu_save_and_restart_reload(event, 0);
-		return;
+		return 0;
 	}
 
 	__intel_pmu_pebs_events(event, iregs, data, at, top, 0, n,
 				setup_pebs_fixed_sample_data);
+
+	/* PMC0 only */
+	return 1;
 }
 
 static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 mask)
@@ -3016,7 +3107,7 @@ static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64
 	}
 }
 
-static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data)
+static int intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct debug_store *ds = cpuc->ds;
@@ -3025,11 +3116,12 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
 	short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
 	short error[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
 	int max_pebs_events = intel_pmu_max_num_pebs(NULL);
+	u64 events_bitmap = 0;
 	int bit, i, size;
 	u64 mask;
 
 	if (!x86_pmu.pebs_active)
-		return;
+		return 0;
 
 	base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
 	top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
@@ -3045,7 +3137,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
 
 	if (unlikely(base >= top)) {
 		intel_pmu_pebs_event_update_no_drain(cpuc, mask);
-		return;
+		return 0;
 	}
 
 	for (at = base; at < top; at += x86_pmu.pebs_record_size) {
@@ -3109,6 +3201,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
 		if ((counts[bit] == 0) && (error[bit] == 0))
 			continue;
 
+		events_bitmap |= BIT_ULL(bit);
 		event = cpuc->events[bit];
 		if (WARN_ON_ONCE(!event))
 			continue;
@@ -3130,6 +3223,8 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d
 						setup_pebs_fixed_sample_data);
 		}
 	}
+
+	return hweight64(events_bitmap);
 }
 
 static __always_inline void
@@ -3164,39 +3259,46 @@ static __always_inline void
 __intel_pmu_handle_last_pebs_record(struct pt_regs *iregs,
 				    struct pt_regs *regs,
 				    struct perf_sample_data *data,
-				    u64 mask, short *counts, void **last,
+				    u64 mask, short *counts,
+				    void **last, bool corrupted,
 				    setup_fn setup_sample)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct perf_event *event;
+	bool handled = false;
 	int bit;
 
 	for_each_set_bit(bit, (unsigned long *)&mask, X86_PMC_IDX_MAX) {
 		if (!counts[bit])
 			continue;
 
+		handled = true;
 		event = cpuc->events[bit];
-
 		__intel_pmu_pebs_last_event(event, iregs, regs, data, last[bit],
-					    counts[bit], setup_sample);
+					    counts[bit], corrupted, setup_sample);
 	}
 
+	/* All records are corrupted, reset sampling period. */
+	if (!handled)
+		intel_pmu_pebs_event_update_no_drain(cpuc, mask);
 }
 
-static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)
+static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)
 {
 	short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
 	void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct debug_store *ds = cpuc->ds;
-	struct x86_perf_regs perf_regs;
-	struct pt_regs *regs = &perf_regs.regs;
+	struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs);
+	struct pt_regs *regs = &perf_regs->regs;
 	struct pebs_basic *basic;
 	void *base, *at, *top;
+	u64 events_bitmap = 0;
+	bool corrupted = false;
 	u64 mask;
 
 	if (!x86_pmu.pebs_active)
-		return;
+		return 0;
 
 	base = (struct pebs_basic *)(unsigned long)ds->pebs_buffer_base;
 	top = (struct pebs_basic *)(unsigned long)ds->pebs_index;
@@ -3209,7 +3311,7 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
 
 	if (unlikely(base >= top)) {
 		intel_pmu_pebs_event_update_no_drain(cpuc, mask);
-		return;
+		return 0;
 	}
 
 	if (!iregs)
@@ -3220,36 +3322,45 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
 		u64 pebs_status;
 
 		basic = at;
+		if (WARN_ON_ONCE(!basic->format_size)) {
+			corrupted = true;
+			break;
+		}
 		if (basic->format_size != cpuc->pebs_record_size)
 			continue;
 
 		pebs_status = mask & basic->applicable_counters;
+		events_bitmap |= pebs_status;
 		__intel_pmu_handle_pebs_record(iregs, regs, data, at,
 					       pebs_status, counts, last,
 					       setup_pebs_adaptive_sample_data);
 	}
 
 	__intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last,
-					    setup_pebs_adaptive_sample_data);
+					    corrupted, setup_pebs_adaptive_sample_data);
+
+	return hweight64(events_bitmap);
 }
 
-static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
-				      struct perf_sample_data *data)
+static int intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
+				     struct perf_sample_data *data)
 {
 	short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
 	void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	union arch_pebs_index index;
-	struct x86_perf_regs perf_regs;
-	struct pt_regs *regs = &perf_regs.regs;
+	struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs);
+	struct pt_regs *regs = &perf_regs->regs;
 	void *base, *at, *top;
+	u64 events_bitmap = 0;
+	bool corrupted = false;
 	u64 mask;
 
 	rdmsrq(MSR_IA32_PEBS_INDEX, index.whole);
 
 	if (unlikely(!index.wr)) {
 		intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX);
-		return;
+		return 0;
 	}
 
 	base = cpuc->pebs_vaddr;
@@ -3277,8 +3388,10 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
 
 		header = at;
 
-		if (WARN_ON_ONCE(!header->size))
-			break;
+		if (WARN_ON_ONCE(!header->size)) {
+			corrupted = true;
+			goto done;
+		}
 
 		/* 1st fragment or single record must have basic group */
 		if (!header->basic) {
@@ -3288,6 +3401,7 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
 
 		basic = at + sizeof(struct arch_pebs_header);
 		pebs_status = mask & basic->applicable_counters;
+		events_bitmap |= pebs_status;
 		__intel_pmu_handle_pebs_record(iregs, regs, data, at,
 					       pebs_status, counts, last,
 					       setup_arch_pebs_sample_data);
@@ -3297,16 +3411,27 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
 			if (!header->size)
 				break;
 			at += header->size;
+			if (WARN_ON_ONCE(at >= top)) {
+				corrupted = true;
+				goto done;
+			}
 			header = at;
 		}
 
 		/* Skip last fragment or the single record */
 		at += header->size;
+		if (WARN_ON_ONCE(at > top)) {
+			corrupted = true;
+			goto done;
+		}
 	}
 
+done:
 	__intel_pmu_handle_last_pebs_record(iregs, regs, data, mask,
-					    counts, last,
+					    counts, last, corrupted,
 					    setup_arch_pebs_sample_data);
+
+	return hweight64(events_bitmap);
 }
 
 static void __init intel_arch_pebs_init(void)
@@ -3408,7 +3533,6 @@ static void __init intel_ds_pebs_init(void)
 				x86_pmu.flags |= PMU_FL_PEBS_ALL;
 				x86_pmu.pebs_capable = ~0ULL;
 				pebs_qual = "-baseline";
-				x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
 			} else {
 				/* Only basic record supported */
 				x86_pmu.large_pebs_flags &=
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index cbe5c762008d2d..0b446d8912e4eb 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -714,7 +714,7 @@ static inline bool vlbr_exclude_host(void)
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
 	return test_bit(INTEL_PMC_IDX_FIXED_VLBR,
-		(unsigned long *)&cpuc->intel_ctrl_guest_mask);
+		(unsigned long *)&cpuc->intel_ctrl_exclude_host_mask);
 }
 
 void intel_pmu_lbr_enable_all(bool pmi)
@@ -756,10 +756,10 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 
 		rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 
-		perf_clear_branch_entry_bitfields(br);
-
-		br->from	= msr_lastbranch.from;
-		br->to		= msr_lastbranch.to;
+		*br = (struct perf_branch_entry){
+			.from	= msr_lastbranch.from,
+			.to	= msr_lastbranch.to,
+		};
 		br++;
 	}
 	cpuc->lbr_stack.nr = i;
@@ -847,14 +847,15 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 		if (abort && x86_pmu.lbr_double_abort && out > 0)
 			out--;
 
-		perf_clear_branch_entry_bitfields(br+out);
-		br[out].from	 = from;
-		br[out].to	 = to;
-		br[out].mispred	 = mis;
-		br[out].predicted = pred;
-		br[out].in_tx	 = in_tx;
-		br[out].abort	 = abort;
-		br[out].cycles	 = cycles;
+		br[out] = (struct perf_branch_entry){
+			.from		= from,
+			.to		= to,
+			.mispred	= mis,
+			.predicted	= pred,
+			.in_tx		= in_tx,
+			.abort		= abort,
+			.cycles		= cycles,
+		};
 		out++;
 	}
 	cpuc->lbr_stack.nr = out;
@@ -905,6 +906,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
 	struct perf_branch_entry *e;
 	struct lbr_entry *lbr;
 	u64 from, to, info;
+	bool mispred;
 	int i;
 
 	for (i = 0; i < x86_pmu.lbr_nr; i++) {
@@ -921,24 +923,27 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
 		to = rdlbr_to(i, lbr);
 		info = rdlbr_info(i, lbr);
 
-		perf_clear_branch_entry_bitfields(e);
-
-		e->from		= from;
-		e->to		= to;
-		e->mispred	= get_lbr_mispred(info);
-		e->predicted	= !e->mispred;
-		e->in_tx	= !!(info & LBR_INFO_IN_TX);
-		e->abort	= !!(info & LBR_INFO_ABORT);
-		e->cycles	= get_lbr_cycles(info);
-		e->type		= get_lbr_br_type(info);
-
-		/*
-		 * Leverage the reserved field of cpuc->lbr_entries[i] to
-		 * temporarily store the branch counters information.
-		 * The later code will decide what content can be disclosed
-		 * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder().
-		 */
-		e->reserved	= (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK;
+		mispred = get_lbr_mispred(info);
+
+		*e = (struct perf_branch_entry){
+			.from		= from,
+			.to		= to,
+			.mispred	= mispred,
+			.predicted	= !mispred,
+			.in_tx		= !!(info & LBR_INFO_IN_TX),
+			.abort		= !!(info & LBR_INFO_ABORT),
+			.cycles		= get_lbr_cycles(info),
+			.type		= get_lbr_br_type(info),
+			/*
+			 * Leverage the reserved field of
+			 * cpuc->lbr_entries[i] to temporarily store the
+			 * branch counters information. The later code will
+			 * decide what content can be disclosed to the perf
+			 * tool. Pleae see intel_pmu_lbr_counters_reorder().
+			 */
+			.reserved	= (info >> LBR_INFO_BR_CNTR_OFFSET) &
+					  LBR_INFO_BR_CNTR_FULL_MASK,
+		};
 	}
 
 	cpuc->lbr_stack.nr = i;
diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c
index 5754cd40556281..49349afee6119f 100644
--- a/arch/x86/events/intel/pt.c
+++ b/arch/x86/events/intel/pt.c
@@ -781,8 +781,7 @@ static int topa_insert_pages(struct pt_buffer *buf, int cpu, gfp_t gfp)
 	struct page *p;
 
 	p = virt_to_page(buf->data_pages[buf->nr_pages]);
-	if (PagePrivate(p))
-		order = page_private(p);
+	order = page_private(p);
 
 	if (topa_table_full(topa)) {
 		topa = topa_alloc(cpu, gfp);
@@ -1296,8 +1295,7 @@ static int pt_buffer_try_single(struct pt_buffer *buf, int nr_pages)
 	if (!intel_pt_validate_hw_cap(PT_CAP_single_range_output))
 		goto out;
 
-	if (PagePrivate(p))
-		order = page_private(p);
+	order = page_private(p);
 
 	if (1 << order != nr_pages)
 		goto out;
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index 4680cba9134062..e274802ef06250 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -147,6 +147,199 @@ static inline bool is_acr_self_reload_event(struct perf_event *event)
 	return test_bit(hwc->idx, (unsigned long *)&hwc->config1);
 }
 
+static inline bool __event_needs_xmm(struct perf_event *event, u64 sample_type)
+{
+	if (event->attr.sample_simd_regs_enabled) {
+		if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_XMM_QWORDS)
+			return false;
+
+		if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+		    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+		    (event->attr.sample_simd_vec_reg_user > 0))
+			return true;
+
+		if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+		    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+		    (event->attr.sample_simd_vec_reg_intr > 0))
+			return true;
+	} else {
+		if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+		    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+		    (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
+			return true;
+
+		if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+		    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+		    (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK))
+			return true;
+	}
+
+	return false;
+}
+
+static inline bool event_needs_xmm(struct perf_event *event)
+{
+	return __event_needs_xmm(event,
+			PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+}
+
+static inline bool __event_needs_ymm(struct perf_event *event, u64 sample_type)
+{
+	if (!event->attr.sample_simd_regs_enabled)
+		return false;
+	if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_YMM_QWORDS)
+		return false;
+
+	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_simd_vec_reg_user > 0))
+		return true;
+
+	if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_simd_vec_reg_intr > 0))
+		return true;
+
+	return false;
+}
+
+static inline bool event_needs_ymm(struct perf_event *event)
+{
+	return __event_needs_ymm(event,
+			PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+}
+
+static inline bool __event_needs_low16_zmm(struct perf_event *event,
+					   u64 sample_type)
+{
+	if (!event->attr.sample_simd_regs_enabled)
+		return false;
+	if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS)
+		return false;
+
+	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_simd_vec_reg_user > 0))
+		return true;
+
+	if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_simd_vec_reg_intr > 0))
+		return true;
+
+	return false;
+}
+
+static inline bool event_needs_low16_zmm(struct perf_event *event)
+{
+	return __event_needs_low16_zmm(event,
+			PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+}
+
+static inline bool __event_needs_high16_zmm(struct perf_event *event,
+					    u64 sample_type)
+{
+	if (!event->attr.sample_simd_regs_enabled)
+		return false;
+	if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS)
+		return false;
+
+	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (fls64(event->attr.sample_simd_vec_reg_user) > PERF_X86_H16ZMM_BASE))
+		return true;
+
+	if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (fls64(event->attr.sample_simd_vec_reg_intr) > PERF_X86_H16ZMM_BASE))
+		return true;
+
+	return false;
+}
+
+static inline bool event_needs_high16_zmm(struct perf_event *event)
+{
+	return __event_needs_high16_zmm(event,
+			PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+}
+
+static inline bool __event_needs_opmask(struct perf_event *event,
+					u64 sample_type)
+{
+	if (!event->attr.sample_simd_regs_enabled)
+		return false;
+	if (event->attr.sample_simd_pred_reg_qwords != PERF_X86_OPMASK_QWORDS)
+		return false;
+
+	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_simd_pred_reg_user > 0))
+		return true;
+
+	if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_simd_pred_reg_intr > 0))
+		return true;
+
+	return false;
+}
+
+static inline bool event_needs_opmask(struct perf_event *event)
+{
+	return __event_needs_opmask(event,
+			PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+}
+
+static inline bool __event_needs_egprs(struct perf_event *event,
+				       u64 sample_type)
+{
+	if (!event->attr.sample_simd_regs_enabled)
+		return false;
+
+	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_regs_user & PERF_X86_EGPRS_MASK))
+		return true;
+
+	if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_regs_intr & PERF_X86_EGPRS_MASK))
+		return true;
+
+	return false;
+}
+
+static inline bool event_needs_egprs(struct perf_event *event)
+{
+	return __event_needs_egprs(event,
+			PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+}
+
+static inline bool __event_needs_ssp(struct perf_event *event,
+				     u64 sample_type)
+{
+	if (!event->attr.sample_simd_regs_enabled)
+		return false;
+
+	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_USER) &&
+	    (event->attr.sample_regs_user & BIT_ULL(PERF_REG_X86_SSP)))
+		return true;
+
+	if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) &&
+	    (event->attr.sample_regs_intr & BIT_ULL(PERF_REG_X86_SSP)))
+		return true;
+
+	return false;
+}
+
+static inline bool event_needs_ssp(struct perf_event *event)
+{
+	return __event_needs_ssp(event,
+			PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER);
+}
+
 struct amd_nb {
 	int nb_id;  /* NorthBridge id */
 	int refcnt; /* reference count */
@@ -347,8 +540,8 @@ struct cpu_hw_events {
 	/*
 	 * Intel host/guest exclude bits
 	 */
-	u64				intel_ctrl_guest_mask;
-	u64				intel_ctrl_host_mask;
+	u64				intel_ctrl_exclude_host_mask;
+	u64				intel_ctrl_exclude_guest_mask;
 	struct perf_guest_switch_msr	guest_switch_msrs[X86_PMC_IDX_MAX];
 
 	/*
@@ -517,10 +710,6 @@ struct cpu_hw_events {
 	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
 
-#define INTEL_HYBRID_LAT_CONSTRAINT(c, n)	\
-	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
-			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID)
-
 #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n)	\
 	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW)
@@ -952,7 +1141,7 @@ struct x86_pmu {
 	int		pebs_record_size;
 	int		pebs_buffer_size;
 	u64		pebs_events_mask;
-	void		(*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
+	int		(*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
 	struct event_constraint *pebs_constraints;
 	void		(*pebs_aliases)(struct perf_event *event);
 	u64		(*pebs_latency_data)(struct perf_event *event, u64 status);
@@ -1028,10 +1217,17 @@ struct x86_pmu {
 	struct extra_reg *extra_regs;
 	unsigned int flags;
 
+	/*
+	 * Extended regs, e.g., vector registers
+	 * Utilize the same format as the XFEATURE_MASK_*
+	 */
+	u64		ext_regs_mask;
+
 	/*
 	 * Intel host/guest support (KVM)
 	 */
-	struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data);
+	struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr,
+							struct x86_guest_pebs *guest_pebs);
 
 	/*
 	 * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
@@ -1050,7 +1246,7 @@ struct x86_pmu {
 	 * unique capabilities.
 	 */
 	int				num_hybrid_pmus;
-	struct x86_hybrid_pmu		*hybrid_pmu;
+	struct x86_hybrid_pmu		*hybrid_pmu __counted_by_ptr(num_hybrid_pmus);
 	enum intel_cpu_type (*get_hybrid_cpu_type)	(void);
 };
 
@@ -1315,6 +1511,13 @@ void x86_pmu_enable_event(struct perf_event *event);
 
 int x86_pmu_handle_irq(struct pt_regs *regs);
 
+void x86_pmu_clear_perf_regs(struct pt_regs *regs);
+
+void x86_pmu_update_perf_regs(struct perf_event *event,
+			      struct perf_sample_data *data,
+			      struct pt_regs *regs,
+			      bool from_pebs);
+
 void x86_pmu_show_pmu_cap(struct pmu *pmu);
 
 static inline int x86_pmu_num_counters(struct pmu *pmu)
diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
index 0b4a1c0b0b16fb..9900a5c59fd3b7 100644
--- a/arch/x86/hyperv/hv_init.c
+++ b/arch/x86/hyperv/hv_init.c
@@ -426,7 +426,7 @@ static void __init hv_stimer_setup_percpu_clockev(void)
 	int ret;
 
 	/*
-	 * Continue afters errors in setting up stimer clockevents
+	 * Continue after errors in setting up stimer clockevents
 	 * as we can run with the LAPIC timer as a fallback.
 	 */
 	ret = hv_stimer_alloc(false);
@@ -434,7 +434,7 @@ static void __init hv_stimer_setup_percpu_clockev(void)
 		pr_warn("stimer setup failed with error %d\n", ret);
 
 	/*
-	 * Still register the LAPIC timer to allows users
+	 * Still register the LAPIC timer to allow users
 	 * to switch to LAPIC timer via /sys, if they want to.
 	 */
 	if (old_setup_percpu_clockev)
@@ -727,7 +727,8 @@ int hv_apicid_to_vp_index(u32 apic_id)
 	input->partition_id = HV_PARTITION_ID_SELF;
 	input->apic_ids[0] = apic_id;
 
-	output = *this_cpu_ptr(hyperv_pcpu_output_arg);
+	/* Treat input as having 2 APIC IDs so output is 64-bit aligned */
+	output = (void *)input + struct_size(input, apic_ids, 2);
 
 	control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_INDEX_FROM_APIC_ID;
 	status = hv_do_hypercall(control, input, output);
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index f70ee74b5f9217..3bed8ccd0ae7d5 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -76,7 +76,7 @@
 #define X86_FEATURE_K8			( 3*32+ 4) /* Opteron, Athlon64 */
 #define X86_FEATURE_ZEN5		( 3*32+ 5) /* CPU based on Zen5 microarchitecture */
 #define X86_FEATURE_ZEN6		( 3*32+ 6) /* CPU based on Zen6 microarchitecture */
-/* Free                                 ( 3*32+ 7) */
+#define X86_FEATURE_RMPOPT		( 3*32+ 7) /* Support for AMD RMPOPT instruction */
 #define X86_FEATURE_CONSTANT_TSC	( 3*32+ 8) /* "constant_tsc" TSC ticks at a constant rate */
 /* free: was #define X86_FEATURE_UP	( 3*32+ 9) * "up" SMP kernel running on UP */
 #define X86_FEATURE_ART			( 3*32+10) /* "art" Always running timer (ART) */
@@ -238,6 +238,7 @@
 // free: was #define X86_FEATURE_PVUNLOCK               ( 8*32+20) /* PV unlock function */
 #define X86_FEATURE_VCPUPREEMPT		( 8*32+21) /* PV vcpu_is_preempted function */
 #define X86_FEATURE_TDX_GUEST		( 8*32+22) /* "tdx_guest" Intel Trust Domain Extensions Guest */
+#define X86_FEATURE_PML			( 8*32+23) /* AMD Page Modification logging */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */
 #define X86_FEATURE_FSGSBASE		( 9*32+ 0) /* "fsgsbase" RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/
@@ -430,6 +431,7 @@
 #define X86_FEATURE_SUCCOR		(17*32+ 1) /* "succor" Uncorrectable error containment and recovery */
 #define X86_FEATURE_CPPC_PERF_PRIO	(17*32+ 2) /* CPPC Floor Perf support */
 #define X86_FEATURE_SMCA		(17*32+ 3) /* "smca" Scalable MCA */
+#define X86_FEATURE_BTB_CTX_ISOLATION	(17*32+ 4) /* AMD: Branch predictions contexts isolated */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:0 (EDX), word 18 */
 #define X86_FEATURE_AVX512_4VNNIW	(18*32+ 2) /* "avx512_4vnniw" AVX-512 Neural Network Instructions */
@@ -483,6 +485,8 @@
 #define X86_FEATURE_AUTOIBRS		(20*32+ 8) /* Automatic IBRS */
 #define X86_FEATURE_NO_SMM_CTL_MSR	(20*32+ 9) /* SMM_CTL MSR is not present */
 
+#define X86_FEATURE_L2_TLB_SIZE_X32	(20*32+14) /* L2 TLB sizes are encoded as multiples of 32 */
+
 #define X86_FEATURE_GP_ON_USER_CPUID	(20*32+17) /* User CPUID faulting */
 
 #define X86_FEATURE_PREFETCHI		(20*32+20) /* Prefetch Data/Instruction to Cache Level */
diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h
index 82eddfa2347b32..2d9f3d4d63de6e 100644
--- a/arch/x86/include/asm/cpuid/api.h
+++ b/arch/x86/include/asm/cpuid/api.h
@@ -204,7 +204,7 @@ static inline u32 cpuid_base_hypervisor(const char *sig, u32 leaves)
 		 * from PVH early boot code before instrumentation is set up
 		 * and memcmp() itself may be instrumented.
 		 */
-		if (!__builtin_memcmp(sig, signature, 12) &&
+		if (!__inline_memcmp(sig, signature, 12) &&
 		    (leaves == 0 || ((eax - base) >= leaves)))
 			return base;
 	}
diff --git a/arch/x86/include/asm/fpu/regset.h b/arch/x86/include/asm/fpu/regset.h
index 697b77e96025a7..433720990f5de9 100644
--- a/arch/x86/include/asm/fpu/regset.h
+++ b/arch/x86/include/asm/fpu/regset.h
@@ -9,10 +9,8 @@
 
 extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active,
 				ssp_active;
-extern user_regset_get2_fn fpregs_get, xfpregs_get, fpregs_soft_get,
-				 xstateregs_get, ssp_get;
-extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
-				 xstateregs_set, ssp_set;
+extern user_regset_get2_fn fpregs_get, xfpregs_get, xstateregs_get, ssp_get;
+extern user_regset_set_fn fpregs_set, xfpregs_set, xstateregs_set, ssp_set;
 
 /*
  * xstateregs_active == regset_fpregs_active. Please refer to the comment
diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h
index 89004f4ca208da..67b0dfa3530ff2 100644
--- a/arch/x86/include/asm/fpu/sched.h
+++ b/arch/x86/include/asm/fpu/sched.h
@@ -10,6 +10,8 @@
 #include <asm/trace/fpu.h>
 
 extern void save_fpregs_to_fpstate(struct fpu *fpu);
+extern void update_fpu_state_and_flag(struct fpu *fpu,
+				      struct task_struct *task);
 extern void fpu__drop(struct task_struct *tsk);
 extern int  fpu_clone(struct task_struct *dst, u64 clone_flags, bool minimal,
 		      unsigned long shstk_addr);
@@ -32,12 +34,10 @@ extern void fpu_flush_thread(void);
 static inline void switch_fpu(struct task_struct *old, int cpu)
 {
 	if (!test_tsk_thread_flag(old, TIF_NEED_FPU_LOAD) &&
-	    cpu_feature_enabled(X86_FEATURE_FPU) &&
 	    !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) {
 		struct fpu *old_fpu = x86_task_fpu(old);
 
-		set_tsk_thread_flag(old, TIF_NEED_FPU_LOAD);
-		save_fpregs_to_fpstate(old_fpu);
+		update_fpu_state_and_flag(old_fpu, old);
 		/*
 		 * The save operation preserved register state, so the
 		 * fpu_fpregs_owner_ctx is still @old_fpu. Store the
diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h
index 93e99d2583d6f6..ea02832f7043c7 100644
--- a/arch/x86/include/asm/fpu/types.h
+++ b/arch/x86/include/asm/fpu/types.h
@@ -74,30 +74,6 @@ struct fxregs_state {
 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
 #define MXCSR_AND_FLAGS_SIZE sizeof(u64)
 
-/*
- * Software based FPU emulation state. This is arbitrary really,
- * it matches the x87 format to make it easier to understand:
- */
-struct swregs_state {
-	u32			cwd;
-	u32			swd;
-	u32			twd;
-	u32			fip;
-	u32			fcs;
-	u32			foo;
-	u32			fos;
-	/* 8*10 bytes for each FP-reg = 80 bytes: */
-	u32			st_space[20];
-	u8			ftop;
-	u8			changed;
-	u8			lookahead;
-	u8			no_update;
-	u8			rm;
-	u8			alimit;
-	struct math_emu_info	*info;
-	u32			entry_eip;
-};
-
 /*
  * List of XSAVE features Linux knows about:
  */
@@ -369,7 +345,6 @@ struct xregs_state {
 union fpregs_state {
 	struct fregs_state		fsave;
 	struct fxregs_state		fxsave;
-	struct swregs_state		soft;
 	struct xregs_state		xsave;
 	u8 __padding[PAGE_SIZE];
 };
diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h
index 7a7dc9d5602750..19dec5f0b1c711 100644
--- a/arch/x86/include/asm/fpu/xstate.h
+++ b/arch/x86/include/asm/fpu/xstate.h
@@ -110,6 +110,9 @@ int xfeature_size(int xfeature_nr);
 
 void xsaves(struct xregs_state *xsave, u64 mask);
 void xrstors(struct xregs_state *xsave, u64 mask);
+void xsaves_nmi(struct xregs_state *xsave, u64 mask);
+
+unsigned int xstate_calculate_size(u64 xfeatures, bool compacted);
 
 int xfd_enable_feature(u64 xfd_err);
 
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index e213c9ae3e301d..4dde4e3e608a32 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -82,6 +82,7 @@ KVM_X86_OP(interrupt_allowed)
 KVM_X86_OP(nmi_allowed)
 KVM_X86_OP(get_nmi_mask)
 KVM_X86_OP(set_nmi_mask)
+KVM_X86_OP_OPTIONAL(clear_hlt)
 KVM_X86_OP(enable_nmi_window)
 KVM_X86_OP(enable_irq_window)
 KVM_X86_OP_OPTIONAL(update_cr8_intercept)
@@ -99,6 +100,7 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep)
 KVM_X86_OP(load_mmu_pgd)
 KVM_X86_OP_OPTIONAL_RET0(set_external_spte)
 KVM_X86_OP_OPTIONAL(free_external_spt)
+KVM_X86_OP_OPTIONAL_RET0(topup_external_cache)
 KVM_X86_OP(has_wbinvd_exit)
 KVM_X86_OP(get_l2_tsc_offset)
 KVM_X86_OP(get_l2_tsc_multiplier)
@@ -150,7 +152,7 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page)
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
 KVM_X86_OP_OPTIONAL_RET0(gmem_make_private)
 #endif
-#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
+#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM)
 KVM_X86_OP_OPTIONAL(gmem_make_shared)
 #endif
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 683bb8bf43a945..db104e0a3b04a9 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -16,6 +16,7 @@
 #include <linux/irq_work.h>
 #include <linux/irq.h>
 #include <linux/workqueue.h>
+#include <linux/xarray.h>
 
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
@@ -608,15 +609,6 @@ struct kvm_pmu {
 	u64 pebs_data_cfg;
 	u64 pebs_data_cfg_rsvd;
 
-	/*
-	 * If a guest counter is cross-mapped to host counter with different
-	 * index, its PEBS capability will be temporarily disabled.
-	 *
-	 * The user should make sure that this mask is updated
-	 * after disabling interrupts and before perf_guest_get_msrs();
-	 */
-	u64 host_cross_mapped_mask;
-
 	/*
 	 * The gate to release perf_events not marked in
 	 * pmc_in_use only once in a vcpu time slice.
@@ -788,6 +780,8 @@ struct kvm_vcpu_arch {
 	 */
 	struct kvm_mmu_memory_cache mmu_external_spt_cache;
 
+	struct page *pml_page;
+
 	/*
 	 * QEMU userspace and the guest each have their own FPU state.
 	 * In vcpu_run, we switch between the user and guest FPU contexts.
@@ -864,7 +858,7 @@ struct kvm_vcpu_arch {
 	gpa_t time;
 	s8  pvclock_tsc_shift;
 	u32 pvclock_tsc_mul;
-	unsigned int hw_tsc_khz;
+	u64 hw_tsc_hz;
 	struct gfn_to_pfn_cache pv_time;
 	/* set guest stopped flag in pvclock flags field */
 	bool pvclock_set_guest_stopped_request;
@@ -973,7 +967,7 @@ struct kvm_vcpu_arch {
 		bool pv_unhalted;
 	} pv;
 
-	int pending_ioapic_eoi;
+	u8 pending_ioapic_eoi;
 	int pending_external_vector;
 	int highest_stale_pending_ioapic_eoi;
 
@@ -1113,7 +1107,7 @@ struct kvm_xen {
 	bool runstate_update_flag;
 	u8 upcall_vector;
 	struct gfn_to_pfn_cache shinfo_cache;
-	struct idr evtchn_ports;
+	struct xarray evtchn_ports;
 	unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)];
 
 	struct kvm_xen_hvm_config hvm_config;
@@ -1165,6 +1159,7 @@ struct kvm_arch {
 	u8 mmu_valid_gen;
 	u8 vm_type;
 	bool has_private_mem;
+	bool has_protected_page_tables;
 	bool has_protected_state;
 	bool has_protected_eoi;
 	bool has_protected_pmu;
@@ -1236,6 +1231,7 @@ struct kvm_arch {
 	u64 last_tsc_write;
 	u32 last_tsc_khz;
 	u64 last_tsc_offset;
+	u64 last_tsc_scaling_ratio;
 	u64 cur_tsc_nsec;
 	u64 cur_tsc_write;
 	u64 cur_tsc_offset;
@@ -1251,6 +1247,9 @@ struct kvm_arch {
 	u64 master_kernel_ns;
 	u64 master_cycle_now;
 	struct ratelimit_state kvmclock_update_rs;
+	u64 master_tsc_scaling_ratio;
+	s8  master_tsc_shift;
+	u32 master_tsc_mul;
 
 #ifdef CONFIG_KVM_HYPERV
 	struct kvm_hv hyperv;
@@ -1406,6 +1405,7 @@ struct kvm_arch {
 	struct kvm_mmu_memory_cache split_desc_cache;
 
 	gfn_t gfn_direct_bits;
+	int mirror_root_level;
 
 	/*
 	 * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero
@@ -1578,7 +1578,7 @@ struct kvm_x86_ops {
 	 * Can potentially get non-canonical addresses through INVLPGs, which
 	 * the implementation may choose to ignore if appropriate.
 	 */
-	void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full);
+	void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr);
 
 	/*
 	 * Flush any TLB entries created by the guest.  Like tlb_flush_gva(),
@@ -1607,6 +1607,13 @@ struct kvm_x86_ops {
 	int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection);
 	bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
 	void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked);
+	/*
+	 * Force the vCPU out of any hardware-tracked halted/inactive state so
+	 * that it will fetch and execute instructions on the next VM-Enter.
+	 * Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across
+	 * VM-Exit/VM-Enter; SVM has no equivalent VMCB field.
+	 */
+	void (*clear_hlt)(struct kvm_vcpu *vcpu);
 	/* Whether or not a virtual NMI is pending in hardware. */
 	bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu);
 	/*
@@ -1644,6 +1651,7 @@ struct kvm_x86_ops {
 	/* Update external page tables for page table about to be freed. */
 	void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp);
 
+	int (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts);
 
 	bool (*has_wbinvd_exit)(void);
 
@@ -1669,7 +1677,7 @@ struct kvm_x86_ops {
 			       struct x86_exception *exception);
 	void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu);
 
-	void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu);
+	void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable);
 
 	void (*vcpu_blocking)(struct kvm_vcpu *vcpu);
 	void (*vcpu_unblocking)(struct kvm_vcpu *vcpu);
@@ -1732,7 +1740,7 @@ struct kvm_x86_ops {
 	int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 				 kvm_pfn_t nr_pages);
 #endif
-#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
+#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM)
 	void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
 #endif
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
@@ -1851,9 +1859,14 @@ enum kvm_intr_type {
 	((vcpu) && (vcpu)->arch.handling_intr_from_guest && \
 	 (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI)))
 
-#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+#if defined(CONFIG_KVM_SW_PROTECTED_VM) ||	\
+    defined(CONFIG_KVM_INTEL_TDX) ||		\
+    defined(CONFIG_KVM_AMD_SEV)
 #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem)
 #endif
+#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
+#define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private)
+#endif
 
 #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)
 
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 4957018fef3ede..68af7b74450af7 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -32,14 +32,14 @@ static inline void local_add(long i, local_t *l)
 {
 	asm volatile(_ASM_ADD "%1,%0"
 		     : "+m" (l->a.counter)
-		     : "ir" (i));
+		     : "er" (i));
 }
 
 static inline void local_sub(long i, local_t *l)
 {
 	asm volatile(_ASM_SUB "%1,%0"
 		     : "+m" (l->a.counter)
-		     : "ir" (i));
+		     : "er" (i));
 }
 
 /**
diff --git a/arch/x86/include/asm/math_emu.h b/arch/x86/include/asm/math_emu.h
deleted file mode 100644
index 3c42743083ed9f..00000000000000
--- a/arch/x86/include/asm/math_emu.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_X86_MATH_EMU_H
-#define _ASM_X86_MATH_EMU_H
-
-#include <asm/ptrace.h>
-
-/* This structure matches the layout of the data saved to the stack
-   following a device-not-present interrupt, part of it saved
-   automatically by the 80386/80486.
-   */
-struct math_emu_info {
-	long ___orig_eip;
-	struct pt_regs *regs;
-};
-#endif /* _ASM_X86_MATH_EMU_H */
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index e575b702063d0d..9995176d687905 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -13,6 +13,7 @@
 #define MCG_CTL_P		BIT_ULL(8)   /* MCG_CTL register available */
 #define MCG_EXT_P		BIT_ULL(9)   /* Extended registers available */
 #define MCG_CMCI_P		BIT_ULL(10)  /* CMCI supported */
+#define MCG_TES_P		BIT_ULL(11)  /* Threshold-based error status supported */
 #define MCG_SEAM_NR		BIT_ULL(12)  /* MCG_STATUS_SEAM_NR supported */
 #define MCG_EXT_CNT_MASK	0xff0000     /* Number of Extended registers */
 #define MCG_EXT_CNT_SHIFT	16
@@ -41,6 +42,10 @@
 #define MCI_STATUS_PCC		BIT_ULL(57)  /* processor context corrupt */
 #define MCI_STATUS_S		BIT_ULL(56)  /* Signaled machine check */
 #define MCI_STATUS_AR		BIT_ULL(55)  /* Action required */
+#define MCI_STATUS_TES_MASK	GENMASK_ULL(54, 53)
+#define MCI_STATUS_TES(s)	FIELD_GET(MCI_STATUS_TES_MASK, s)
+#define  MCI_STATUS_TES_GREEN	1	     /* Threshold-based errors below threshold */
+#define  MCI_STATUS_TES_YELLOW	2	     /* Threshold-based errors above threshold */
 #define MCI_STATUS_CEC_SHIFT	38           /* Corrected Error Count */
 #define MCI_STATUS_CEC_MASK	GENMASK_ULL(52,38)
 #define MCI_STATUS_CEC(c)	(((c) & MCI_STATUS_CEC_MASK) >> MCI_STATUS_CEC_SHIFT)
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 3a8e51a0c9e88c..c0914829a925d5 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -114,6 +114,8 @@
 #define MSR_IA32_CORE_CAPS_INTEGRITY_CAPS	  BIT(MSR_IA32_CORE_CAPS_INTEGRITY_CAPS_BIT)
 #define MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT_BIT  5
 #define MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT	  BIT(MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT_BIT)
+#define MSR_IA32_CORE_CAPS_BFF_RESET_BIT	  9
+#define MSR_IA32_CORE_CAPS_BFF_RESET		  BIT(MSR_IA32_CORE_CAPS_BFF_RESET_BIT)
 
 #define MSR_PKG_CST_CONFIG_CONTROL	0x000000e2
 #define NHM_C3_AUTO_DEMOTE		(1UL << 25)
@@ -350,6 +352,13 @@
 #define ARCH_PEBS_LBR_SHIFT		40
 #define ARCH_PEBS_LBR			(0x3ull << ARCH_PEBS_LBR_SHIFT)
 #define ARCH_PEBS_VECR_XMM		BIT_ULL(49)
+#define ARCH_PEBS_VECR_YMMH		BIT_ULL(50)
+#define ARCH_PEBS_VECR_EGPRS		BIT_ULL(51)
+#define ARCH_PEBS_VECR_OPMASK		BIT_ULL(53)
+#define ARCH_PEBS_VECR_ZMMH		BIT_ULL(54)
+#define ARCH_PEBS_VECR_H16ZMM		BIT_ULL(55)
+#define ARCH_PEBS_VECR_EXT_SHIFT	49
+#define ARCH_PEBS_VECR_EXT		(0x7full << ARCH_PEBS_VECR_EXT_SHIFT)
 #define ARCH_PEBS_GPR			BIT_ULL(61)
 #define ARCH_PEBS_AUX			BIT_ULL(62)
 #define ARCH_PEBS_EN			BIT_ULL(63)
@@ -761,6 +770,9 @@
 #define MSR_AMD64_SEG_RMP_ENABLED_BIT	0
 #define MSR_AMD64_SEG_RMP_ENABLED	BIT_ULL(MSR_AMD64_SEG_RMP_ENABLED_BIT)
 #define MSR_AMD64_RMP_SEGMENT_SHIFT(x)	(((x) & GENMASK_ULL(13, 8)) >> 8)
+#define MSR_AMD64_RMPOPT_BASE		0xc0010139
+#define MSR_AMD64_RMPOPT_ENABLE_BIT	0
+#define MSR_AMD64_RMPOPT_ENABLE		BIT_ULL(MSR_AMD64_RMPOPT_ENABLE_BIT)
 
 #define MSR_SVSM_CAA			0xc001f000
 
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 1eb13673e889f9..5c92d43bbef831 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -150,6 +150,11 @@
 #define PEBS_DATACFG_LBRS	BIT_ULL(3)
 #define PEBS_DATACFG_CNTR	BIT_ULL(4)
 #define PEBS_DATACFG_METRICS	BIT_ULL(5)
+#define PEBS_DATACFG_YMMHS	BIT_ULL(6)
+#define PEBS_DATACFG_OPMASKS	BIT_ULL(7)
+#define PEBS_DATACFG_ZMMHS	BIT_ULL(8)
+#define PEBS_DATACFG_H16ZMMS	BIT_ULL(9)
+#define PEBS_DATACFG_EGPRS	BIT_ULL(10)
 #define PEBS_DATACFG_LBR_SHIFT	24
 #define PEBS_DATACFG_CNTR_SHIFT	32
 #define PEBS_DATACFG_CNTR_MASK	GENMASK_ULL(15, 0)
@@ -547,7 +552,8 @@ struct arch_pebs_header {
 			    rsvd3:7,
 			    xmm:1,
 			    ymmh:1,
-			    rsvd4:2,
+			    egpr:1,
+			    rsvd4:1,
 			    opmask:1,
 			    zmmh:1,
 			    h16zmm:1,
@@ -728,7 +734,32 @@ extern void perf_events_lapic_init(void);
 struct pt_regs;
 struct x86_perf_regs {
 	struct pt_regs	regs;
-	u64		*xmm_regs;
+	u64		abi;
+	union {
+		u64	*xmm_regs;
+		u32	*xmm_space;	/* for xsaves */
+	};
+	union {
+		u64	*ymmh_regs;
+		struct ymmh_struct *ymmh;
+	};
+	union {
+		u64	*zmmh_regs;
+		struct avx_512_zmm_uppers_state *zmmh;
+	};
+	union {
+		u64	*h16zmm_regs;
+		struct avx_512_hi16_state *h16zmm;
+	};
+	union {
+		u64	*opmask_regs;
+		struct avx_512_opmask_state *opmask;
+	};
+	union {
+		u64	*egpr_regs;
+		struct apx_state *egpr;
+	};
+	u64	*ssp;
 };
 
 extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs);
@@ -788,11 +819,18 @@ extern void perf_load_guest_lvtpc(u32 guest_lvtpc);
 extern void perf_put_guest_lvtpc(void);
 #endif
 
+struct x86_guest_pebs {
+	u64	enable;
+	u64	ds_area;
+	u64	data_cfg;
+};
 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
-extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data);
+extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr,
+							 struct x86_guest_pebs *guest_pebs);
 extern void x86_perf_get_lbr(struct x86_pmu_lbr *lbr);
 #else
-struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data);
+struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr,
+						  struct x86_guest_pebs *guest_pebs);
 static inline void x86_perf_get_lbr(struct x86_pmu_lbr *lbr)
 {
 	memset(lbr, 0, sizeof(*lbr));
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index c88691b15f3c67..2d03b8dd95d477 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -4,7 +4,6 @@
 
 #include <linux/threads.h>
 #include <linux/mm.h>		/* for struct page */
-#include <linux/pagemap.h>
 
 #include <asm/cpufeature.h>
 
diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h
index e9482a11ac52d6..83427765cfbfd2 100644
--- a/arch/x86/include/asm/pgtable-2level.h
+++ b/arch/x86/include/asm/pgtable-2level.h
@@ -2,11 +2,6 @@
 #ifndef _ASM_X86_PGTABLE_2LEVEL_H
 #define _ASM_X86_PGTABLE_2LEVEL_H
 
-#define pte_ERROR(e) \
-	pr_err("%s:%d: bad pte %08lx\n", __FILE__, __LINE__, (e).pte_low)
-#define pgd_ERROR(e) \
-	pr_err("%s:%d: bad pgd %08lx\n", __FILE__, __LINE__, pgd_val(e))
-
 /*
  * Certain architectures need to do special things when PTEs
  * within a page table are directly modified.  Thus, the following
diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
index dabafba957ea6f..d6729911e09a28 100644
--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -8,17 +8,6 @@
  *
  * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
  */
-
-#define pte_ERROR(e)							\
-	pr_err("%s:%d: bad pte %p(%08lx%08lx)\n",			\
-	       __FILE__, __LINE__, &(e), (e).pte_high, (e).pte_low)
-#define pmd_ERROR(e)							\
-	pr_err("%s:%d: bad pmd %p(%016Lx)\n",				\
-	       __FILE__, __LINE__, &(e), pmd_val(e))
-#define pgd_ERROR(e)							\
-	pr_err("%s:%d: bad pgd %p(%016Lx)\n",				\
-	       __FILE__, __LINE__, &(e), pgd_val(e))
-
 #define pxx_xchg64(_pxx, _ptr, _val) ({					\
 	_pxx##val_t *_p = (_pxx##val_t *)_ptr;				\
 	_pxx##val_t _o = *_p;						\
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index d551120a7c889b..000fc6c1ca37c4 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -41,10 +41,10 @@ void ptdump_walk_user_pgd_level_checkwx(void);
 #define pgprot_encrypted(prot)	__pgprot(cc_mkenc(pgprot_val(prot)))
 #define pgprot_decrypted(prot)	__pgprot(cc_mkdec(pgprot_val(prot)))
 
-#ifdef CONFIG_DEBUG_WX
-#define debug_checkwx_user()	ptdump_walk_user_pgd_level_checkwx()
+#ifdef CONFIG_CHECK_WX
+#define pgtable_checkwx_user()	ptdump_walk_user_pgd_level_checkwx()
 #else
-#define debug_checkwx_user()	do { } while (0)
+#define pgtable_checkwx_user()	do { } while (0)
 #endif
 
 extern spinlock_t pgd_lock;
@@ -66,7 +66,11 @@ extern pmdval_t early_pmd_flags;
 
 #ifndef __PAGETABLE_P4D_FOLDED
 #define set_pgd(pgdp, pgd)		native_set_pgd(pgdp, pgd)
-#define pgd_clear(pgd)			(pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
+#define pgd_clear(pgdp)			\
+do {					\
+	if (pgtable_l5_enabled())	\
+		native_pgd_clear(pgdp);	\
+} while (0)
 #endif
 
 #ifndef set_p4d
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index ce45882ccd071b..c861f3832bed2e 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -29,24 +29,6 @@ extern pgd_t init_top_pgt[];
 extern void paging_init(void);
 static inline void sync_initial_page_table(void) { }
 
-#define pte_ERROR(e)					\
-	pr_err("%s:%d: bad pte %p(%016lx)\n",		\
-	       __FILE__, __LINE__, &(e), pte_val(e))
-#define pmd_ERROR(e)					\
-	pr_err("%s:%d: bad pmd %p(%016lx)\n",		\
-	       __FILE__, __LINE__, &(e), pmd_val(e))
-#define pud_ERROR(e)					\
-	pr_err("%s:%d: bad pud %p(%016lx)\n",		\
-	       __FILE__, __LINE__, &(e), pud_val(e))
-
-#define p4d_ERROR(e)					\
-	pr_err("%s:%d: bad p4d %p(%016lx)\n",		\
-	       __FILE__, __LINE__, &(e), p4d_val(e))
-
-#define pgd_ERROR(e)					\
-	pr_err("%s:%d: bad pgd %p(%016lx)\n",		\
-	       __FILE__, __LINE__, &(e), pgd_val(e))
-
 struct mm_struct;
 
 #define mm_p4d_folded mm_p4d_folded
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index fafb6f8cdac3f1..d16d9c2f7b0866 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -137,43 +137,15 @@ static __always_inline bool should_resched(int preempt_offset)
 extern asmlinkage void preempt_schedule(void);
 extern asmlinkage void preempt_schedule_thunk(void);
 
-#define preempt_schedule_dynamic_enabled	preempt_schedule_thunk
-#define preempt_schedule_dynamic_disabled	NULL
-
 extern asmlinkage void preempt_schedule_notrace(void);
 extern asmlinkage void preempt_schedule_notrace_thunk(void);
 
-#define preempt_schedule_notrace_dynamic_enabled	preempt_schedule_notrace_thunk
-#define preempt_schedule_notrace_dynamic_disabled	NULL
-
-#ifdef CONFIG_PREEMPT_DYNAMIC
-
-DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
-
-#define __preempt_schedule() \
-do { \
-	__STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule); \
-	asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \
-} while (0)
-
-DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
-
-#define __preempt_schedule_notrace() \
-do { \
-	__STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule_notrace); \
-	asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule_notrace) : ASM_CALL_CONSTRAINT); \
-} while (0)
-
-#else /* PREEMPT_DYNAMIC */
-
 #define __preempt_schedule() \
 	asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT);
 
 #define __preempt_schedule_notrace() \
 	asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT);
 
-#endif /* PREEMPT_DYNAMIC */
-
 #endif /* PREEMPTION */
 
 #undef __pc_op
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index ec9db0dfa0dffc..77488ff5eb8c7c 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -10,7 +10,7 @@ struct mm_struct;
 struct io_bitmap;
 struct vm86;
 
-#include <asm/math_emu.h>
+#include <asm/ptrace.h>
 #include <asm/segment.h>
 #include <asm/types.h>
 #include <uapi/asm/sigcontext.h>
diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h
index 4362c26aa992db..39271a5ea92527 100644
--- a/arch/x86/include/asm/set_memory.h
+++ b/arch/x86/include/asm/set_memory.h
@@ -86,9 +86,8 @@ int set_pages_wb(struct page *page, int numpages);
 int set_pages_ro(struct page *page, int numpages);
 int set_pages_rw(struct page *page, int numpages);
 
-int set_direct_map_invalid_noflush(struct page *page);
-int set_direct_map_default_noflush(struct page *page);
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid);
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr);
+int set_direct_map_default_noflush(struct page *page, unsigned int nr);
 bool kernel_page_present(struct page *page);
 
 extern int kernel_set_to_readonly;
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 895d09faaf832e..1b333bb091d7f2 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -4,7 +4,7 @@
 
 #include <uapi/asm/setup.h>
 
-#define COMMAND_LINE_SIZE 2048
+#define COMMAND_LINE_SIZE CONFIG_COMMAND_LINE_SIZE
 
 #include <linux/linkage.h>
 #include <asm/page_types.h>
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index 9e7a077c445dcc..c7ea9845351e33 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -517,6 +517,7 @@ void snp_accept_memory(phys_addr_t start, phys_addr_t end);
 u64 snp_get_unsupported_features(u64 status);
 u64 sev_get_status(void);
 void sev_show_status(void);
+bool early_is_sevsnp_guest(void);
 int prepare_pte_enc(struct pte_enc_desc *d);
 void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot);
 void snp_kexec_finish(void);
@@ -626,6 +627,7 @@ static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { }
 static inline u64 snp_get_unsupported_features(u64 status) { return 0; }
 static inline u64 sev_get_status(void) { return 0; }
 static inline void sev_show_status(void) { }
+static inline bool early_is_sevsnp_guest(void) { return false; }
 static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; }
 static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { }
 static inline void snp_kexec_finish(void) { }
@@ -662,6 +664,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int pages)
 	__snp_leak_pages(pfn, pages, true);
 }
 int snp_prepare(void);
+void snp_enable_rmpopt(void);
 void snp_shutdown(void);
 #else
 static inline bool snp_probe_rmptable_info(void) { return false; }
@@ -680,6 +683,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int npages) {}
 static inline void kdump_sev_callback(void) { }
 static inline void snp_fixup_e820_tables(void) {}
 static inline int snp_prepare(void) { return -ENODEV; }
+static inline void snp_enable_rmpopt(void) {}
 static inline void snp_shutdown(void) {}
 #endif
 
diff --git a/arch/x86/include/asm/shared/string.h b/arch/x86/include/asm/shared/string.h
new file mode 100644
index 00000000000000..6bef90d62a2113
--- /dev/null
+++ b/arch/x86/include/asm/shared/string.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_SHARED_STRING_H
+#define _ASM_X86_SHARED_STRING_H
+
+static __always_inline void *__inline_memcpy(void *to, const void *from, size_t len)
+{
+	void *ret = to;
+
+	asm volatile("rep movsb"
+		     : "+D" (to), "+S" (from), "+c" (len)
+		     : : "memory");
+	return ret;
+}
+
+static __always_inline void *__inline_memset(void *s, int v, size_t n)
+{
+	void *ret = s;
+
+	asm volatile("rep stosb"
+		     : "+D" (s), "+c" (n)
+		     : "a" ((uint8_t)v)
+		     : "memory");
+	return ret;
+}
+
+/*
+ * Returns:	0 (equal)
+ * 		1 (not equal)
+ *
+ * In contrast, the regular memcmp() follows glibc return value semantics.
+ */
+static __always_inline int __inline_memcmp(const void *s1, const void *s2, size_t len)
+{
+	bool diff;
+
+	/*
+	 * Make sure ZF is properly set in the len==0 case because in it,
+	 * RCX==0 and the REPE; CMPSB won't get executed.
+	 *
+	 * The "cc" clobber has no meaning anymore, just source compatibility.
+	 * On x86 the flag status bits are automatically added to the clobber
+	 * set when there are no =@ccXY constraints. Keep it as documentation.
+	 */
+	asm volatile("test %3, %3\n\t"
+		     "repe cmpsb"
+		     : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len)
+		     : : /* "cc", */ "memory");
+
+	return diff;
+}
+
+#endif /* _ASM_X86_SHARED_STRING_H */
diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h
index f20e91d7ac35be..bf000f1fb42e4b 100644
--- a/arch/x86/include/asm/shared/tdx.h
+++ b/arch/x86/include/asm/shared/tdx.h
@@ -143,6 +143,11 @@ struct tdx_module_args {
 	u64 rbx;
 	u64 rdi;
 	u64 rsi;
+	/*
+	 * Leaf ABI version. Note that it gets encoded into RAX along with the
+	 * leaf number.
+	 */
+	u8  version;
 };
 
 /* Used to communicate with the TDX module */
@@ -171,6 +176,7 @@ static inline u64 _tdx_hypercall(u64 fn, u64 r12, u64 r13, u64 r14, u64 r15)
 	return __tdx_hypercall(&args);
 }
 
+void __noreturn tdx_panic(const char *msg);
 
 /* Called from __tdx_hypercall() for unrecoverable failure */
 void __noreturn __tdx_hypercall_failed(void);
diff --git a/arch/x86/include/asm/string.h b/arch/x86/include/asm/string.h
index 9cb5aae7fba9ff..dbf59f0d4cca71 100644
--- a/arch/x86/include/asm/string.h
+++ b/arch/x86/include/asm/string.h
@@ -8,25 +8,6 @@
 # include <asm/string_64.h>
 #endif
 
-static __always_inline void *__inline_memcpy(void *to, const void *from, size_t len)
-{
-	void *ret = to;
-
-	asm volatile("rep movsb"
-		     : "+D" (to), "+S" (from), "+c" (len)
-		     : : "memory");
-	return ret;
-}
-
-static __always_inline void *__inline_memset(void *s, int v, size_t n)
-{
-	void *ret = s;
-
-	asm volatile("rep stosb"
-		     : "+D" (s), "+c" (n)
-		     : "a" ((uint8_t)v)
-		     : "memory");
-	return ret;
-}
+#include <asm/shared/string.h>
 
 #endif /* _ASM_X86_STRING_H */
diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h
index 4635616863f53d..2dae70c2b01465 100644
--- a/arch/x86/include/asm/string_64.h
+++ b/arch/x86/include/asm/string_64.h
@@ -3,7 +3,6 @@
 #define _ASM_X86_STRING_64_H
 
 #ifdef __KERNEL__
-#include <linux/jump_label.h>
 
 /* Written 2002 by Andi Kleen */
 
@@ -82,24 +81,77 @@ int strcmp(const char *cs, const char *ct);
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
 void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
-static __always_inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
+
+static __always_inline void movnti_4(void *dst, const void *src)
+{
+	asm volatile("movntil %1, %0"
+		     : "=m"(*(u32 *)dst)
+		     : "r"(*(const u32 *)src)
+		     : "memory");
+}
+
+static __always_inline void movnti_8(void *dst, const void *src)
+{
+	asm volatile("movntiq %1, %0"
+		     : "=m"(*(u64 *)dst)
+		     : "r"(*(const u64 *)src)
+		     : "memory");
+}
+
+static __always_inline void movnti_16(void *dst, const void *src)
+{
+	movnti_8(dst, src);
+	movnti_8(dst + 8, src + 8);
+}
+
+static __always_inline void movnti_32(void *dst, const void *src)
 {
-	if (__builtin_constant_p(cnt)) {
-		switch (cnt) {
-			case 4:
-				asm ("movntil %1, %0" : "=m"(*(u32 *)dst) : "r"(*(u32 *)src));
-				return;
-			case 8:
-				asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
-				return;
-			case 16:
-				asm ("movntiq %1, %0" : "=m"(*(u64 *)dst) : "r"(*(u64 *)src));
-				asm ("movntiq %1, %0" : "=m"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src + 8)));
-				return;
-		}
+	movnti_16(dst, src);
+	movnti_16(dst + 16, src + 16);
+}
+
+static __always_inline void movnti_64(void *dst, const void *src)
+{
+	movnti_32(dst, src);
+	movnti_32(dst + 32, src + 32);
+}
+
+static __always_inline void memcpy_flushcache(void *dst, const void *src,
+					      size_t cnt)
+{
+	if (!__builtin_constant_p(cnt))
+		return __memcpy_flushcache(dst, src, cnt);
+
+	/*
+	 * The relevant fixed-size copies here are the x86_64 struct page sizes:
+	 * 64, 80, and 96 bytes. Keep 32-byte and 48-byte copies inline as well
+	 * instead of sending those nearby fixed-size cases back to
+	 * __memcpy_flushcache().
+	 */
+	switch (cnt) {
+	case 4:  movnti_4(dst, src); break;
+	case 8:  movnti_8(dst, src); break;
+	case 16: movnti_16(dst, src); break;
+	case 32: movnti_32(dst, src); break;
+	case 48: movnti_32(dst, src); movnti_16(dst + 32, src + 32); break;
+	case 64: movnti_64(dst, src); break;
+	case 80: movnti_64(dst, src); movnti_16(dst + 64, src + 64); break;
+	case 96: movnti_64(dst, src); movnti_32(dst + 64, src + 64); break;
+	default: __memcpy_flushcache(dst, src, cnt); break;
 	}
-	__memcpy_flushcache(dst, src, cnt);
 }
+
+#define memcpy_nontemporal memcpy_nontemporal
+/*
+ * Reuse the existing x86 flushcache backend as the non-temporal copy
+ * primitive.
+ */
+static __always_inline void memcpy_nontemporal(void *dst, const void *src,
+		size_t cnt)
+{
+	memcpy_flushcache(dst, src, cnt);
+}
+
 #endif
 
 #endif /* __KERNEL__ */
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index aa63431ba92c31..83fd49b26a4b17 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -165,7 +165,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 	u8 reserved_9[22];
 	u64 allowed_sev_features;	/* Offset 0x138 */
 	u64 guest_sev_features;		/* Offset 0x140 */
-	u8 reserved_10[664];
+	u8 reserved_10[128];
+	u64 pml_addr;			/* Offset 0x1c8 */
+	u16 pml_index;			/* Offset 0x1d0 */
+	u8 reserved_11[526];
 	/*
 	 * Offset 0x3e0, 32 bytes reserved
 	 * for use by hypervisor/software.
@@ -240,10 +243,11 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT)
 #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT)
 
-#define SVM_MISC_ENABLE_NP		BIT(0)
-#define SVM_MISC_ENABLE_SEV		BIT(1)
-#define SVM_MISC_ENABLE_SEV_ES	BIT(2)
-#define SVM_MISC_ENABLE_GMET	BIT(3)
+#define SVM_MISC_ENABLE_NP		BIT_ULL(0)
+#define SVM_MISC_ENABLE_SEV		BIT_ULL(1)
+#define SVM_MISC_ENABLE_SEV_ES		BIT_ULL(2)
+#define SVM_MISC_ENABLE_GMET		BIT_ULL(3)
+#define SVM_MISC_ENABLE_PML		BIT_ULL(11)
 
 #define SVM_MISC2_ENABLE_V_LBR	BIT_ULL(0)
 #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE	BIT_ULL(1)
@@ -637,8 +641,8 @@ static inline void __unused_size_checks(void)
 #define SVM_EVTINJ_TYPE_EXEPT (3 << SVM_EVTINJ_TYPE_SHIFT)
 #define SVM_EVTINJ_TYPE_SOFT (4 << SVM_EVTINJ_TYPE_SHIFT)
 
-#define SVM_EVTINJ_VALID (1 << 31)
-#define SVM_EVTINJ_VALID_ERR (1 << 11)
+#define SVM_EVTINJ_VALID	BIT(31)
+#define SVM_EVTINJ_VALID_ERR	BIT(11)
 
 #define SVM_EVTINJ_RESERVED_BITS ~(SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | \
 				   SVM_EVTINJ_VALID_ERR | SVM_EVTINJ_VALID)
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 89e97d5761d89e..e186dfe5bf8855 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -36,6 +36,7 @@
 /* Bit definitions of TDX_FEATURES0 metadata field */
 #define TDX_FEATURES0_TD_PRESERVING	BIT_ULL(1)
 #define TDX_FEATURES0_NO_RBP_MOD	BIT_ULL(18)
+#define TDX_FEATURES0_DYNAMIC_PAMT	BIT_ULL(36)
 
 #ifndef __ASSEMBLER__
 
@@ -118,12 +119,34 @@ static inline bool tdx_supports_runtime_update(const struct tdx_sys_info *sysinf
 	return sysinfo->features.tdx_features0 & TDX_FEATURES0_TD_PRESERVING;
 }
 
+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo);
+
+/* Simple structure for pre-allocating DPAMT pages outside of spinlocks. */
+struct tdx_pamt_cache {
+	struct list_head page_list;
+	int cnt;
+};
+
+static inline void tdx_init_pamt_cache(struct tdx_pamt_cache *cache)
+{
+	INIT_LIST_HEAD(&cache->page_list);
+	cache->cnt = 0;
+}
+
+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache);
+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages);
+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache);
+void tdx_pamt_put(kvm_pfn_t pfn);
+
 int tdx_guest_keyid_alloc(void);
 u32 tdx_get_nr_guest_keyids(void);
 void tdx_guest_keyid_free(unsigned int keyid);
 
 void tdx_quirk_reset_paddr(unsigned long base, unsigned long size);
 
+struct page *tdx_alloc_control_page(void);
+void tdx_free_control_page(struct page *page);
+
 struct tdx_td {
 	/* TD root structure: */
 	struct page *tdr_page;
diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h
index 41150d546589c1..8a3cc1a2a41e2f 100644
--- a/arch/x86/include/asm/tdx_global_metadata.h
+++ b/arch/x86/include/asm/tdx_global_metadata.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-/* Automatically generated TDX global metadata structures. */
-#ifndef _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H
-#define _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H
+/* TDX global metadata structures. */
+#ifndef _X86_VIRT_TDX_TDX_GLOBAL_METADATA_H
+#define _X86_VIRT_TDX_TDX_GLOBAL_METADATA_H
 
 #include <linux/types.h>
 
@@ -21,6 +21,9 @@ struct tdx_sys_info_tdmr {
 	u16 pamt_4k_entry_size;
 	u16 pamt_2m_entry_size;
 	u16 pamt_1g_entry_size;
+
+	/* Optional metadata, if DPAMT is supported */
+	u8  pamt_page_bitmap_entry_bits;
 };
 
 struct tdx_sys_info_td_ctrl {
diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h
index f2d142a0a862eb..ea09381070e82d 100644
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -164,9 +164,9 @@ unsigned long int3_emulate_pop(struct pt_regs *regs)
 }
 
 static __always_inline
-void int3_emulate_call(struct pt_regs *regs, unsigned long func)
+void int3_emulate_call(struct pt_regs *regs, unsigned long ip, unsigned long func)
 {
-	int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE);
+	int3_emulate_push(regs, ip);
 	int3_emulate_jmp(regs, func);
 }
 
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 3f24cc472ce9be..e13f1025c62e0a 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -37,8 +37,6 @@ static inline int get_si_code(unsigned long condition)
 		return TRAP_BRKPT;
 }
 
-void math_emulate(struct math_emu_info *);
-
 bool fault_in_kernel_space(unsigned long address);
 
 #ifdef CONFIG_VMAP_STACK
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 3f1b3096ff0401..e49b836c7313fb 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -20,6 +20,14 @@
 #include <asm/trapnr.h>
 #include <asm/vmxfeatures.h>
 
+/*
+ * UNDEFINED is a KVM-defined value used to indicate that the basic exit reason
+ * is undefined/invalid, e.g. on VM-Fail or if TDX never attempted VM-Enter.
+ * The value itself is arbitrary, it just needs to not collide with a real exit
+ * reason.
+ */
+#define EXIT_REASON_UNDEFINED			0xdead
+
 struct vmcs_hdr {
 	u32 revision_id:31;
 	u32 shadow_vmcs:1;
diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h
index 7c9d2bb3833bd2..faaa82df688d8a 100644
--- a/arch/x86/include/uapi/asm/perf_regs.h
+++ b/arch/x86/include/uapi/asm/perf_regs.h
@@ -2,6 +2,8 @@
 #ifndef _ASM_X86_PERF_REGS_H
 #define _ASM_X86_PERF_REGS_H
 
+#include <linux/bits.h>
+
 enum perf_event_x86_regs {
 	PERF_REG_X86_AX,
 	PERF_REG_X86_BX,
@@ -27,9 +29,35 @@ enum perf_event_x86_regs {
 	PERF_REG_X86_R13,
 	PERF_REG_X86_R14,
 	PERF_REG_X86_R15,
+	/*
+	 * The eGPRs/SSP and XMM have overlaps. Only one can be used
+	 * at a time. The ABI PERF_SAMPLE_REGS_ABI_SIMD is used to
+	 * distinguish which one is used. If PERF_SAMPLE_REGS_ABI_SIMD
+	 * is set, then eGPRs/SSP is used, otherwise, XMM is used.
+	 *
+	 * Extended GPRs (eGPRs)
+	 */
+	PERF_REG_X86_R16,
+	PERF_REG_X86_R17,
+	PERF_REG_X86_R18,
+	PERF_REG_X86_R19,
+	PERF_REG_X86_R20,
+	PERF_REG_X86_R21,
+	PERF_REG_X86_R22,
+	PERF_REG_X86_R23,
+	PERF_REG_X86_R24,
+	PERF_REG_X86_R25,
+	PERF_REG_X86_R26,
+	PERF_REG_X86_R27,
+	PERF_REG_X86_R28,
+	PERF_REG_X86_R29,
+	PERF_REG_X86_R30,
+	PERF_REG_X86_R31,
+	PERF_REG_X86_SSP,
 	/* These are the limits for the GPRs. */
 	PERF_REG_X86_32_MAX = PERF_REG_X86_GS + 1,
 	PERF_REG_X86_64_MAX = PERF_REG_X86_R15 + 1,
+	PERF_REG_MISC_MAX = PERF_REG_X86_SSP + 1,
 
 	/* These all need two bits set because they are 128bit */
 	PERF_REG_X86_XMM0  = 32,
@@ -54,5 +82,30 @@ enum perf_event_x86_regs {
 };
 
 #define PERF_REG_EXTENDED_MASK	(~((1ULL << PERF_REG_X86_XMM0) - 1))
+#define PERF_X86_EGPRS_MASK	__GENMASK_ULL(PERF_REG_X86_R31, PERF_REG_X86_R16)
+
+enum {
+	PERF_X86_SIMD_XMM_REGS      = 16,
+	PERF_X86_SIMD_YMM_REGS      = 16,
+	PERF_X86_SIMD_ZMM_REGS      = 32,
+	PERF_X86_SIMD_VEC_REGS_MAX  = PERF_X86_SIMD_ZMM_REGS,
+
+	PERF_X86_SIMD_OPMASK_REGS   = 8,
+	PERF_X86_SIMD_PRED_REGS_MAX = PERF_X86_SIMD_OPMASK_REGS,
+};
+
+#define PERF_X86_SIMD_PRED_MASK	__GENMASK(PERF_X86_SIMD_PRED_REGS_MAX - 1, 0)
+#define PERF_X86_SIMD_VEC_MASK	__GENMASK_ULL(PERF_X86_SIMD_VEC_REGS_MAX - 1, 0)
+
+#define PERF_X86_H16ZMM_BASE		16
+
+enum {
+	/* 1 qword = 8 bytes */
+	PERF_X86_OPMASK_QWORDS   = 1,
+	PERF_X86_XMM_QWORDS      = 2,
+	PERF_X86_YMM_QWORDS      = 4,
+	PERF_X86_ZMM_QWORDS      = 8,
+	PERF_X86_SIMD_QWORDS_MAX = PERF_X86_ZMM_QWORDS,
+};
 
 #endif /* _ASM_X86_PERF_REGS_H */
diff --git a/arch/x86/include/uapi/asm/ptrace-abi.h b/arch/x86/include/uapi/asm/ptrace-abi.h
index 5823584dea132b..3656955c6faad3 100644
--- a/arch/x86/include/uapi/asm/ptrace-abi.h
+++ b/arch/x86/include/uapi/asm/ptrace-abi.h
@@ -2,6 +2,7 @@
 #ifndef _ASM_X86_PTRACE_ABI_H
 #define _ASM_X86_PTRACE_ABI_H
 
+#if defined(__ASSEMBLER__) || defined(__FRAME_OFFSETS)
 #ifdef __i386__
 
 #define EBX 0
@@ -25,7 +26,6 @@
 
 #else /* __i386__ */
 
-#if defined(__ASSEMBLER__) || defined(__FRAME_OFFSETS)
 /*
  * C ABI says these regs are callee-preserved. They aren't saved on kernel entry
  * unless syscall needs a complete, fully filled "struct pt_regs".
@@ -57,12 +57,12 @@
 #define EFLAGS 144
 #define RSP 152
 #define SS 160
-#endif /* __ASSEMBLER__ */
 
 /* top of stack page */
 #define FRAME_SIZE 168
 
 #endif /* !__i386__ */
+#endif /* defined(__ASSEMBLER__) || defined(__FRAME_OFFSETS) */
 
 /* Arbitrarily choose the same ptrace numbers as used by the Sparc code. */
 #define PTRACE_GETREGS            12
diff --git a/arch/x86/include/uapi/asm/setup_data.h b/arch/x86/include/uapi/asm/setup_data.h
index 2671c4e1b3a0b7..61090db5065c4c 100644
--- a/arch/x86/include/uapi/asm/setup_data.h
+++ b/arch/x86/include/uapi/asm/setup_data.h
@@ -85,8 +85,8 @@ struct ima_setup_data {
 struct kho_data {
 	__u64 fdt_addr;
 	__u64 fdt_size;
-	__u64 scratch_addr;
-	__u64 scratch_size;
+	__u64 bootmem_addr;
+	__u64 bootmem_size;
 } __attribute__((packed));
 
 #endif /* __ASSEMBLER__ */
diff --git a/arch/x86/include/uapi/asm/sigcontext.h b/arch/x86/include/uapi/asm/sigcontext.h
index d0d9b331d3a138..cff01406c0f453 100644
--- a/arch/x86/include/uapi/asm/sigcontext.h
+++ b/arch/x86/include/uapi/asm/sigcontext.h
@@ -34,6 +34,21 @@
  * fpstate+extended_size-FP_XSTATE_MAGIC2_SIZE address) is set to
  * FP_XSTATE_MAGIC2 so that you can sanity check your size calculations.)
  *
+ * The xstate_size field indicates the actual size of the xstate context
+ * (including the 512-byte FXSAVE area and the 64-byte XSAVE header struct
+ * _header). This size is used in conjunction with the pointer to the xstate
+ * context to locate FP_XSTATE_MAGIC2.
+ *
+ * In 64-bit signal frames, the fpstate pointer points directly to the xstate
+ * context. In 32-bit signal frames (including 32-bit compat tasks on 64-bit
+ * kernels), the fpstate pointer points to struct _fpstate_32, which contains
+ * the 112-byte legacy FPU state followed by the 512-byte FXSR state (and any
+ * extended xstate), so the xstate context starts at fpstate + 112.
+ *
+ * This makes the signal frame self-describing and portable across machines
+ * with different xstate features. See Documentation/arch/x86/xstate.rst
+ * for details on signal frame portability and its architectural constraints.
+ *
  * This extended area typically grows with newer CPUs that have larger and
  * larger XSAVE areas.
  */
diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h
index 010a45c9f6147f..e8067618509215 100644
--- a/arch/x86/include/uapi/asm/svm.h
+++ b/arch/x86/include/uapi/asm/svm.h
@@ -101,6 +101,7 @@
 #define SVM_EXIT_AVIC_INCOMPLETE_IPI		0x401
 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS	0x402
 #define SVM_EXIT_VMGEXIT       0x403
+#define SVM_EXIT_PML_FULL	0x407
 
 /* SEV-ES software-defined VMGEXIT events */
 #define SVM_VMGEXIT_MMIO_READ			0x80000001ull
@@ -236,6 +237,7 @@
 	{ SVM_EXIT_AVIC_INCOMPLETE_IPI,		"avic_incomplete_ipi" }, \
 	{ SVM_EXIT_AVIC_UNACCELERATED_ACCESS,   "avic_unaccelerated_access" }, \
 	{ SVM_EXIT_VMGEXIT,		"vmgexit" }, \
+	{ SVM_EXIT_PML_FULL,		"pml_full" }, \
 	{ SVM_VMGEXIT_MMIO_READ,	"vmgexit_mmio_read" }, \
 	{ SVM_VMGEXIT_MMIO_WRITE,	"vmgexit_mmio_write" }, \
 	{ SVM_VMGEXIT_NMI_COMPLETE,	"vmgexit_nmi_complete" }, \
diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c
index bbade0da5130f9..d2185b7a203061 100644
--- a/arch/x86/kernel/acpi/cppc.c
+++ b/arch/x86/kernel/acpi/cppc.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/limits.h>
 
 #include <acpi/cppc_acpi.h>
 #include <asm/msr.h>
@@ -45,10 +46,20 @@ bool cpc_ffh_supported(void)
 	return true;
 }
 
+static bool cpc_ffh_reg_valid(const struct cpc_reg *reg)
+{
+	return reg->address <= U32_MAX && reg->bit_width &&
+	       reg->bit_width <= 64 &&
+	       reg->bit_offset <= 64 - reg->bit_width;
+}
+
 int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val)
 {
 	int err;
 
+	if (!cpc_ffh_reg_valid(reg))
+		return -EINVAL;
+
 	err = rdmsrq_safe_on_cpu(cpunum, reg->address, val);
 	if (!err) {
 		u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
@@ -65,6 +76,9 @@ int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
 	u64 rd_val;
 	int err;
 
+	if (!cpc_ffh_reg_valid(reg))
+		return -EINVAL;
+
 	err = rdmsrq_safe_on_cpu(cpunum, reg->address, &rd_val);
 	if (!err) {
 		u64 mask = GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 741d8767ddf895..582c6d8307d15d 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -2176,6 +2176,7 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data
 	unsigned long selftest = (unsigned long)&int3_selftest_asm;
 	struct die_args *args = data;
 	struct pt_regs *regs = args->regs;
+	unsigned long ip;
 
 	OPTIMIZER_HIDE_VAR(selftest);
 
@@ -2188,7 +2189,8 @@ int3_exception_notify(struct notifier_block *self, unsigned long val, void *data
 	if (regs->ip - INT3_INSN_SIZE != selftest)
 		return NOTIFY_DONE;
 
-	int3_emulate_call(regs, (unsigned long)&int3_selftest_callee);
+	ip = regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE;
+	int3_emulate_call(regs, ip, (unsigned long)&int3_selftest_callee);
 	return NOTIFY_STOP;
 }
 
@@ -2758,7 +2760,7 @@ noinstr int smp_text_poke_int3_handler(struct pt_regs *regs)
 		break;
 
 	case CALL_INSN_OPCODE:
-		int3_emulate_call(regs, (long)ip + tpl->disp);
+		int3_emulate_call(regs, (long)ip, (long)ip + tpl->disp);
 		break;
 
 	case JMP32_INSN_OPCODE:
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 081816888f7a05..b3c00ff4d81919 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -95,6 +95,7 @@ static void __used common(void)
 	OFFSET(TDX_MODULE_rbx, tdx_module_args, rbx);
 	OFFSET(TDX_MODULE_rdi, tdx_module_args, rdi);
 	OFFSET(TDX_MODULE_rsi, tdx_module_args, rsi);
+	OFFSET(TDX_MODULE_version, tdx_module_args, version);
 
 	BLANK();
 	OFFSET(BP_scratch, boot_params, scratch);
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 54e14ed276b521..e5279bc648d3ab 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -1192,7 +1192,7 @@ static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
 
 static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
 {
-	u32 ebx, eax, ecx, edx;
+	u32 ebx, eax, ecx, edx, shift, tmp;
 	u16 mask = 0xfff;
 
 	if (c->x86 < 0xf)
@@ -1201,10 +1201,12 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
 	if (c->extended_cpuid_level < 0x80000006)
 		return;
 
+	shift = !!cpu_has(c, X86_FEATURE_L2_TLB_SIZE_X32) * 5;
+
 	cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
 
-	tlb_lld_4k = (ebx >> 16) & mask;
-	tlb_lli_4k = ebx & mask;
+	tlb_lld_4k = ((ebx >> 16) & mask) << shift;
+	tlb_lli_4k = (ebx & mask) << shift;
 
 	/*
 	 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
@@ -1216,16 +1218,18 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
 	}
 
 	/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
-	if (!((eax >> 16) & mask))
+	tmp = ((eax >> 16) & mask) << shift;
+	if (!tmp)
 		tlb_lld_2m = (cpuid_eax(0x80000005) >> 16) & 0xff;
 	else
-		tlb_lld_2m = (eax >> 16) & mask;
+		tlb_lld_2m = tmp;
 
 	/* a 4M entry uses two 2M entries */
 	tlb_lld_4m = tlb_lld_2m >> 1;
 
 	/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
-	if (!(eax & mask)) {
+	tmp = (eax & mask) << shift;
+	if (!tmp) {
 		/* Erratum 658 */
 		if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
 			tlb_lli_2m = 1024;
@@ -1233,8 +1237,9 @@ static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
 			cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
 			tlb_lli_2m = eax & 0xff;
 		}
-	} else
-		tlb_lli_2m = eax & mask;
+	} else {
+		tlb_lli_2m = tmp;
+	}
 
 	tlb_lli_4m = tlb_lli_2m >> 1;
 
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 56eac5611c3189..1b2381da4d8319 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -1175,6 +1175,7 @@ enum srso_mitigation {
 	SRSO_MITIGATION_IBPB,
 	SRSO_MITIGATION_IBPB_ON_VMEXIT,
 	SRSO_MITIGATION_BP_SPEC_REDUCE,
+	SRSO_MITIGATION_USER_IBPB,
 };
 
 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_AUTO;
@@ -2908,7 +2909,8 @@ static const char * const srso_strings[] = {
 	[SRSO_MITIGATION_SAFE_RET]		= "Mitigation: Safe RET",
 	[SRSO_MITIGATION_IBPB]			= "Mitigation: IBPB",
 	[SRSO_MITIGATION_IBPB_ON_VMEXIT]	= "Mitigation: IBPB on VMEXIT only",
-	[SRSO_MITIGATION_BP_SPEC_REDUCE]	= "Mitigation: Reduced Speculation"
+	[SRSO_MITIGATION_BP_SPEC_REDUCE]	= "Mitigation: Reduced Speculation",
+	[SRSO_MITIGATION_USER_IBPB]		= "Mitigation: IBPB on context switch",
 };
 
 static int __init srso_parse_cmdline(char *str)
@@ -2948,7 +2950,9 @@ static void __init srso_select_mitigation(void)
 		 * required.  Otherwise the 'microcode' mitigation is sufficient
 		 * to protect the user->user and guest->guest vectors.
 		 */
-		if (cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) ||
+		if ((cpu_attack_vector_mitigated(CPU_MITIGATE_GUEST_HOST) &&
+		    !boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION))
+					||
 		    (cpu_attack_vector_mitigated(CPU_MITIGATE_USER_KERNEL) &&
 		     !boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO))) {
 			srso_mitigation = SRSO_MITIGATION_SAFE_RET;
@@ -3024,6 +3028,16 @@ static void __init srso_update_mitigation(void)
 	    boot_cpu_has(X86_FEATURE_IBPB_BRTYPE))
 		srso_mitigation = SRSO_MITIGATION_IBPB;
 
+	/*
+	 * See if IBPB on context switch is the only thing needed to address
+	 * GUEST/GUEST and USER/USER vectors.
+	 */
+	if (srso_mitigation == SRSO_MITIGATION_MICROCODE  &&
+	    boot_cpu_has(X86_FEATURE_SRSO_USER_KERNEL_NO) &&
+	    boot_cpu_has(X86_FEATURE_BTB_CTX_ISOLATION) &&
+	    spectre_v2_user_ibpb != SPECTRE_V2_USER_NONE)
+		srso_mitigation = SRSO_MITIGATION_USER_IBPB;
+
 	pr_info("%s\n", srso_strings[srso_mitigation]);
 }
 
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index c7352827f491d5..7d0b9bdc64cc7c 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -857,7 +857,7 @@ static void get_model_name(struct cpuinfo_x86 *c)
 
 void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 {
-	unsigned int n, dummy, ebx, ecx, edx, l2size;
+	unsigned int n, dummy, ebx, ecx, edx, l2size, shift __maybe_unused;
 
 	n = c->extended_cpuid_level;
 
@@ -877,7 +877,9 @@ void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
 	l2size = ecx >> 16;
 
 #ifdef CONFIG_X86_64
+	shift = !!cpu_has(c, X86_FEATURE_L2_TLB_SIZE_X32) * 5;
 	c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
+	c->x86_tlbsize <<= shift;
 #else
 	/* do processor-specific cache resizing */
 	if (this_cpu->legacy_cache_size)
@@ -1427,7 +1429,7 @@ static bool __init vulnerable_to_its(u64 x86_arch_cap_msr)
 	return false;
 }
 
-static struct x86_cpu_id cpu_latest_microcode[] = {
+static const struct x86_cpu_id cpu_latest_microcode[] __initconst = {
 #include "microcode/intel-ucode-defs.h"
 	{}
 };
@@ -1782,25 +1784,50 @@ static void __init cpu_parse_early_param(void)
 	}
 }
 
+static void init_cpu_info(struct cpuinfo_x86 *c)
+{
+	c->x86_cache_size = 0;
+	c->x86_vendor = X86_VENDOR_UNKNOWN;
+	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
+	c->x86_vendor_id[0] = '\0'; /* Unset */
+	c->x86_model_id[0] = '\0';  /* Unset */
+#ifdef CONFIG_X86_64
+	c->x86_clflush_size = 64;
+	c->x86_phys_bits = 36;
+	c->x86_virt_bits = 48;
+#else
+	c->cpuid_level = -1;	/* CPUID not detected */
+	c->x86_clflush_size = 32;
+	c->x86_phys_bits = 32;
+	c->x86_virt_bits = 32;
+#endif
+	c->x86_cache_alignment = c->x86_clflush_size;
+	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
+	memset(&c->cpuid, 0, sizeof(c->cpuid));
+#ifdef CONFIG_X86_VMX_FEATURE_NAMES
+	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
+#endif
+	c->extended_cpuid_level = 0;
+}
+
 /*
  * Do minimum CPU detection early.
  * Fields really needed: vendor, cpuid_level, family, model, mask,
  * cache alignment.
- * The others are not touched to avoid unwanted side effects.
+ * The others are reset to their defaults here and only filled in later,
+ * by identify_cpu().
  *
  * WARNING: this function is only called on the boot CPU.  Don't add code
  * here that is supposed to run on all CPUs.
  */
 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 {
-	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
-	memset(&c->cpuid, 0, sizeof(c->cpuid));
-	c->extended_cpuid_level = 0;
+	init_cpu_info(c);
 
 	if (!cpuid_feature())
 		identify_cpu_without_cpuid(c);
 
-	/* cyrix could have cpuid enabled via c_identify()*/
+	/* Cyrix could have CPUID enabled via c_identify(). */
 	if (cpuid_feature()) {
 		cpuid_scan_cpu(c);
 		cpu_detect(c);
@@ -1964,16 +1991,21 @@ void check_null_seg_clears_base(struct cpuinfo_x86 *c)
 	set_cpu_bug(c, X86_BUG_NULL_SEG);
 }
 
-static void generic_identify(struct cpuinfo_x86 *c)
+/*
+ * This does the hard work of actually picking apart the CPU stuff...
+ */
+static void identify_cpu(struct cpuinfo_x86 *c)
 {
-	c->extended_cpuid_level = 0;
+	int i;
+
+	c->loops_per_jiffy = loops_per_jiffy;
 
 	if (!cpuid_feature())
 		identify_cpu_without_cpuid(c);
 
-	/* cyrix could have cpuid enabled via c_identify()*/
+	/* Cyrix could have CPUID enabled via c_identify(). */
 	if (!cpuid_feature())
-		return;
+		goto no_cpuid;
 
 	cpuid_scan_cpu(c);
 	cpu_detect(c);
@@ -2001,40 +2033,8 @@ static void generic_identify(struct cpuinfo_x86 *c)
 #ifdef CONFIG_X86_32
 	set_cpu_bug(c, X86_BUG_ESPFIX);
 #endif
-}
-
-/*
- * This does the hard work of actually picking apart the CPU stuff...
- */
-static void identify_cpu(struct cpuinfo_x86 *c)
-{
-	int i;
-
-	c->loops_per_jiffy = loops_per_jiffy;
-	c->x86_cache_size = 0;
-	c->x86_vendor = X86_VENDOR_UNKNOWN;
-	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
-	c->x86_vendor_id[0] = '\0'; /* Unset */
-	c->x86_model_id[0] = '\0';  /* Unset */
-#ifdef CONFIG_X86_64
-	c->x86_clflush_size = 64;
-	c->x86_phys_bits = 36;
-	c->x86_virt_bits = 48;
-#else
-	c->cpuid_level = -1;	/* CPUID not detected */
-	c->x86_clflush_size = 32;
-	c->x86_phys_bits = 32;
-	c->x86_virt_bits = 32;
-#endif
-	c->x86_cache_alignment = c->x86_clflush_size;
-	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
-	memset(&c->cpuid, 0, sizeof(c->cpuid));
-#ifdef CONFIG_X86_VMX_FEATURE_NAMES
-	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
-#endif
-
-	generic_identify(c);
 
+no_cpuid:
 	cpu_parse_topology(c);
 
 	if (this_cpu->c_identify)
@@ -2126,6 +2126,9 @@ static void identify_cpu(struct cpuinfo_x86 *c)
 	mcheck_cpu_init(c);
 
 	numa_add_cpu(smp_processor_id());
+
+	if (IS_ENABLED(CONFIG_X86_32))
+		enable_sep_cpu();
 }
 
 /*
@@ -2163,9 +2166,6 @@ static __init void identify_boot_cpu(void)
 	identify_cpu(&boot_cpu_data);
 	if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT))
 		pr_info("CET detected: Indirect Branch Tracking enabled\n");
-#ifdef CONFIG_X86_32
-	enable_sep_cpu();
-#endif
 	cpu_detect_tlb(&boot_cpu_data);
 	setup_cr_pinning();
 
@@ -2184,10 +2184,8 @@ void identify_secondary_cpu(unsigned int cpu)
 		*c = boot_cpu_data;
 	c->cpu_index = cpu;
 
+	init_cpu_info(c);
 	identify_cpu(c);
-#ifdef CONFIG_X86_32
-	enable_sep_cpu();
-#endif
 	x86_spec_ctrl_setup_ap();
 	update_srbds_msr();
 	if (boot_cpu_has_bug(X86_BUG_GDS))
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index ab469605fc893a..50b1614968be6a 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -2108,6 +2108,9 @@ bool filter_mce(struct mce *m)
 static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
 {
 	irqentry_state_t irq_state;
+	unsigned long dr7;
+
+	dr7 = local_db_save();
 
 	WARN_ON_ONCE(user_mode(regs));
 
@@ -2116,20 +2119,26 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
 	 * mce_check_crashing_cpu() for details.
 	 */
 	if (mca_cfg.initialized && mce_check_crashing_cpu())
-		return;
+		goto out;
 
 	irq_state = irqentry_nmi_enter(regs);
 
 	do_machine_check(regs);
 
 	irqentry_nmi_exit(regs, irq_state);
+out:
+	local_db_restore(dr7);
 }
 
 static __always_inline void exc_machine_check_user(struct pt_regs *regs)
 {
+	unsigned long dr7;
+
 	irqentry_enter_from_user_mode(regs);
 
+	dr7 = local_db_save();
 	do_machine_check(regs);
+	local_db_restore(dr7);
 
 	irqentry_exit_to_user_mode(regs);
 }
@@ -2138,21 +2147,13 @@ static __always_inline void exc_machine_check_user(struct pt_regs *regs)
 /* MCE hit kernel mode */
 DEFINE_IDTENTRY_MCE(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	exc_machine_check_kernel(regs);
-	local_db_restore(dr7);
 }
 
 /* The user mode variant. */
 DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	exc_machine_check_user(regs);
-	local_db_restore(dr7);
 }
 
 #ifdef CONFIG_X86_FRED
@@ -2169,28 +2170,20 @@ DEFINE_IDTENTRY_MCE_USER(exc_machine_check)
  */
 DEFINE_FREDENTRY_MCE(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	if (user_mode(regs))
 		exc_machine_check_user(regs);
 	else
 		exc_machine_check_kernel(regs);
-	local_db_restore(dr7);
 }
 #endif
 #else
 /* 32bit unified entry point */
 DEFINE_IDTENTRY_RAW(exc_machine_check)
 {
-	unsigned long dr7;
-
-	dr7 = local_db_save();
 	if (user_mode(regs))
 		exc_machine_check_user(regs);
 	else
 		exc_machine_check_kernel(regs);
-	local_db_restore(dr7);
 }
 #endif
 
@@ -2621,17 +2614,17 @@ static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
 static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all);
 
 static struct dev_ext_attribute dev_attr_check_interval = {
-	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
+	__DEVICE_ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
 	&check_interval
 };
 
 static struct dev_ext_attribute dev_attr_ignore_ce = {
-	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
+	__DEVICE_ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
 	&mca_cfg.ignore_ce
 };
 
 static struct dev_ext_attribute dev_attr_cmci_disabled = {
-	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
+	__DEVICE_ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
 	&mca_cfg.cmci_disabled
 };
 
diff --git a/arch/x86/kernel/cpu/mce/dev-mcelog.c b/arch/x86/kernel/cpu/mce/dev-mcelog.c
index 053555206d816d..af4e76babe7ab5 100644
--- a/arch/x86/kernel/cpu/mce/dev-mcelog.c
+++ b/arch/x86/kernel/cpu/mce/dev-mcelog.c
@@ -11,7 +11,7 @@
 
 #include <linux/miscdevice.h>
 #include <linux/slab.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/poll.h>
 
 #include "internal.h"
diff --git a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
index af8b1d8b26f689..7f463c85c5f84c 100644
--- a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
+++ b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
@@ -136,35 +136,35 @@
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000410 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002e0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000421 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002f1 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xcc },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbe },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x58 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c000410 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000650 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c000410 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000650 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c000410 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000650 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000650 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c000410 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c000421 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000670 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c000421 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000670 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c000421 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000670 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000670 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c000421 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000670 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x43a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xb },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x43a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x43b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x43b },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa },
@@ -176,30 +176,33 @@
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x25 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa000124 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003f0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xae, .steppings = 0x0002, .platform_mask = 0x97, .driver_data = 0x1000273 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000382 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x132 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x132 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x6133 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x6133 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x6133 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x125 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x11a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002c0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002c0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x65 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x28 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa000142 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x1000423 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xae, .steppings = 0x0002, .platform_mask = 0x97, .driver_data = 0x1000307 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x30003a3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x6134 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x6134 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x6134 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x126 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x21 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x121 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x121 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x121 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x121 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcc, .steppings = 0x0002, .platform_mask = 0x90, .driver_data = 0x11b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcc, .steppings = 0x0004, .platform_mask = 0x90, .driver_data = 0x11b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcc, .steppings = 0x0008, .platform_mask = 0x90, .driver_data = 0x11b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002e0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002e0 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 },
 { .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 },
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 1142183c950c0c..9f09d388ed2056 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -309,6 +309,26 @@ static void save_microcode_patch(struct microcode_intel *patch)
 		pr_err("Unable to allocate microcode memory size: %u\n", size);
 }
 
+static bool revision_is_safe(struct cpu_signature *sig, u32 rev)
+{
+	u32 vfm = IFM(x86_family(sig->sig), x86_model(sig->sig));
+
+	/*
+	 * Erratum GNR98 can cause #MCs if "jumping over" revision 0x1000405.
+	 * Avoid the jumps.
+	 */
+	if (vfm == INTEL_GRANITERAPIDS_X &&
+	    x86_stepping(sig->sig) == 1 &&
+	    sig->pf & 0x95 &&
+	    sig->rev < 0x1000405 &&
+	    rev > 0x1000405) {
+		pr_err_once("Erratum GNR98: skipping revision 0x%x.\n", rev);
+		return false;
+	}
+
+	return true;
+}
+
 /* Scan blob for microcode matching the boot CPUs family, model, stepping */
 static __init struct microcode_intel *scan_microcode(void *data, size_t size,
 						     struct ucode_cpu_info *uci,
@@ -330,6 +350,9 @@ static __init struct microcode_intel *scan_microcode(void *data, size_t size,
 		if (!intel_find_matching_signature(data, &uci->cpu_sig))
 			continue;
 
+		if (!revision_is_safe(&uci->cpu_sig, mc_header->rev))
+			continue;
+
 		/*
 		 * For saving the early microcode, find the matching revision which
 		 * was loaded on the BSP.
@@ -878,6 +901,9 @@ static enum ucode_state parse_microcode_blobs(int cpu, struct iov_iter *iter)
 		if (!intel_find_matching_signature(mc, &uci->cpu_sig))
 			continue;
 
+		if (!revision_is_safe(&uci->cpu_sig, mc_header.rev))
+			continue;
+
 		is_safe = ucode_validate_minrev(&mc_header);
 		if (force_minrev && !is_safe)
 			continue;
diff --git a/arch/x86/kernel/cpu/mtrr/amd.c b/arch/x86/kernel/cpu/mtrr/amd.c
index a73715d6f05c12..9e440e30c1796b 100644
--- a/arch/x86/kernel/cpu/mtrr/amd.c
+++ b/arch/x86/kernel/cpu/mtrr/amd.c
@@ -51,11 +51,10 @@ amd_get_mtrr(unsigned int reg, unsigned long *base,
 
 /**
  * amd_set_mtrr - Set variable MTRR register on the local CPU.
- *
- * @reg The register to set.
- * @base The base address of the region.
- * @size The size of the region. If this is 0 the region is disabled.
- * @type The type of the region.
+ * @reg: The register to set.
+ * @base: The base address of the region.
+ * @size: The size of the region. If this is 0 the region is disabled.
+ * @type: The type of the region.
  *
  * Returns nothing.
  */
diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
index e74f1ed54b86c4..62044489b05297 100644
--- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
+++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
@@ -16,9 +16,15 @@
 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
 
 #include <linux/cpu.h>
+#include <linux/math.h>
 
 #include "internal.h"
 
+u32 resctrl_arch_preconvert_bw(const struct rdt_resource *r, u32 val)
+{
+	return roundup(val, (unsigned long)r->membw.bw_gran);
+}
+
 int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d,
 			    u32 closid, enum resctrl_conf_type t, u32 cfg_val)
 {
diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index 8665a6474806c9..d867ab90d42a54 100644
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -55,6 +55,7 @@ static const struct cpuid_bit cpuid_bits[] = {
 	{ X86_FEATURE_AMD_FAST_CPPC,		CPUID_EDX, 15, 0x80000007, 0 },
 	{ X86_FEATURE_CPPC_PERF_PRIO,		CPUID_EDX, 16, 0x80000007, 0 },
 	{ X86_FEATURE_MBA,			CPUID_EBX,  6, 0x80000008, 0 },
+	{ X86_FEATURE_PML,			CPUID_ECX,  4, 0x8000000a, 0 },
 	{ X86_FEATURE_X2AVIC_EXT,		CPUID_ECX,  6, 0x8000000a, 0 },
 	{ X86_FEATURE_COHERENCY_SFW_NO,		CPUID_EBX, 31, 0x8000001f, 0 },
 	{ X86_FEATURE_SMBA,			CPUID_EBX,  2, 0x80000020, 0 },
@@ -64,9 +65,11 @@ static const struct cpuid_bit cpuid_bits[] = {
 	{ X86_FEATURE_AMD_WORKLOAD_CLASS,	CPUID_EAX, 22, 0x80000021, 0 },
 	{ X86_FEATURE_TSA_SQ_NO,		CPUID_ECX,  1, 0x80000021, 0 },
 	{ X86_FEATURE_TSA_L1_NO,		CPUID_ECX,  2, 0x80000021, 0 },
+	{ X86_FEATURE_BTB_CTX_ISOLATION,	CPUID_ECX,  8, 0x80000021, 0 },
 	{ X86_FEATURE_PERFMON_V2,		CPUID_EAX,  0, 0x80000022, 0 },
 	{ X86_FEATURE_AMD_LBR_V2,		CPUID_EAX,  1, 0x80000022, 0 },
 	{ X86_FEATURE_AMD_LBR_PMC_FREEZE,	CPUID_EAX,  2, 0x80000022, 0 },
+	{ X86_FEATURE_RMPOPT,			CPUID_EDX,  0, 0x80000025, 0 },
 	{ X86_FEATURE_AMD_HTR_CORES,		CPUID_EAX, 30, 0x80000026, 0 },
 	{ 0, 0, 0, 0, 0 }
 };
diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c
index 4505f808af5e5f..a5f2aabb2da137 100644
--- a/arch/x86/kernel/cpu/sgx/main.c
+++ b/arch/x86/kernel/cpu/sgx/main.c
@@ -106,7 +106,13 @@ static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list)
 			left_dirty++;
 		}
 
-		cond_resched();
+		/*
+		 * cond_resched() only schedules when TIF_NEED_RESCHED is set.
+		 * During this boot-time loop that condition may not happen for a
+		 * long time, so report an RCU-Tasks quiescent state explicitly.
+		 * Therefore, change cond_resched() to cond_resched_tasks_rcu_qs().
+		 */
+		cond_resched_tasks_rcu_qs();
 	}
 
 	list_splice(&dirty, dirty_page_list);
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index 34b73573b108d7..f7ab9e7902cfe5 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -328,9 +328,9 @@ static int vmware_cpu_down_prepare(unsigned int cpu)
 static __init int activate_jump_labels(void)
 {
 	if (has_steal_clock) {
-		static_key_slow_inc(&paravirt_steal_enabled);
+		static_branch_inc(&paravirt_steal_enabled);
 		if (steal_acc)
-			static_key_slow_inc(&paravirt_steal_rq_enabled);
+			static_branch_inc(&paravirt_steal_rq_enabled);
 	}
 
 	return 0;
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index e681ec9cf1dc85..e6f23933a6df2f 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -369,9 +369,9 @@ int crash_load_segments(struct kimage *image)
 	 * maximum CPUs and maximum memory ranges.
 	 */
 	if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
-		pnum = 2 + CONFIG_NR_CPUS_DEFAULT + CONFIG_CRASH_MAX_MEMORY_RANGES;
+		pnum = 2 + CONFIG_NR_CPUS + CONFIG_CRASH_MAX_MEMORY_RANGES;
 	else
-		pnum += 2 + CONFIG_NR_CPUS_DEFAULT;
+		pnum += 2 + CONFIG_NR_CPUS;
 
 	if (pnum < (unsigned long)PN_XNUM) {
 		kbuf.memsz = pnum * sizeof(Elf64_Phdr);
@@ -430,7 +430,7 @@ unsigned int arch_crash_get_elfcorehdr_size(void)
 	unsigned int sz;
 
 	/* kernel_map, VMCOREINFO and maximum CPUs */
-	sz = 2 + CONFIG_NR_CPUS_DEFAULT;
+	sz = 2 + CONFIG_NR_CPUS;
 	if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
 		sz += CONFIG_CRASH_MAX_MEMORY_RANGES;
 	sz *= sizeof(Elf64_Phdr);
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index eb72537bc0b195..0967b485f38b9a 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -1352,18 +1352,17 @@ __init void e820__memblock_setup(void)
 	 * below 1M (aka ISA_END_ADDRESS) up until direct map is completely set
 	 * up in init_mem_mapping().
 	 *
-	 * KHO kernels are special and use only scratch memory for memblock
+	 * KHO kernels are special and use only NOPRSRV memory for memblock
 	 * allocations, but memory below 1M is ignored by kernel after early
-	 * boot and cannot be naturally marked as scratch.
+	 * boot and cannot be naturally marked as NOPRSRV.
 	 *
 	 * To allow allocation of the real-mode trampoline and a few (if any)
 	 * other very early allocations from below 1M forcibly mark the memory
-	 * below 1M as scratch.
+	 * below 1M as NOPRSRV.
 	 *
-	 * After real mode trampoline is allocated, we clear that scratch
-	 * marking.
+	 * After real mode trampoline is allocated, we clear that marking.
 	 */
-	memblock_mark_kho_scratch(0, SZ_1M);
+	memblock_mark_kho_noprsrv(0, SZ_1M);
 
 	/*
 	 * 32-bit systems are limited to 4BG of memory even with HIGHMEM and
diff --git a/arch/x86/kernel/fpu/bugs.c b/arch/x86/kernel/fpu/bugs.c
index edbafc5940e33b..4b84cd0a9d24f0 100644
--- a/arch/x86/kernel/fpu/bugs.c
+++ b/arch/x86/kernel/fpu/bugs.c
@@ -29,10 +29,6 @@ void __init fpu__init_check_bugs(void)
 {
 	s32 fdiv_bug;
 
-	/* kernel_fpu_begin/end() relies on patched alternative instructions. */
-	if (!boot_cpu_has(X86_FEATURE_FPU))
-		return;
-
 	kernel_fpu_begin();
 
 	/*
diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index d1aeecd57f5edb..80a7f41cf3ad90 100644
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -213,6 +213,19 @@ void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask)
 	}
 }
 
+/*
+ * Save the FPU register state in fpu->fpstate->regs and set
+ * TIF_NEED_FPU_LOAD subsequently.
+ *
+ * Must be called with fpregs_lock() held, ensuring flag
+ * TIF_NEED_FPU_LOAD is set last.
+ */
+void update_fpu_state_and_flag(struct fpu *fpu, struct task_struct *task)
+{
+	save_fpregs_to_fpstate(fpu);
+	set_tsk_thread_flag(task, TIF_NEED_FPU_LOAD);
+}
+
 void fpu_reset_from_exception_fixup(void)
 {
 	restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE);
@@ -383,13 +396,13 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)
 
 	/* Swap fpstate */
 	if (enter_guest) {
-		fpu->__task_fpstate = cur_fps;
+		WRITE_ONCE(fpu->__task_fpstate, cur_fps);
+		barrier();
 		fpu->fpstate = guest_fps;
 		guest_fps->in_use = true;
 	} else {
 		guest_fps->in_use = false;
 		fpu->fpstate = fpu->__task_fpstate;
-		fpu->__task_fpstate = NULL;
 	}
 
 	cur_fps = fpu->fpstate;
@@ -406,6 +419,16 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest)
 		xfd_update_state(cur_fps);
 	}
 
+	/*
+	 * Clear fpu->__task_fpstate after switching back to host state.
+	 * A non-NULL __task_fpstate means guest state is still resident in
+	 * hardware; reset it only once host state has been restored.
+	 */
+	if (!enter_guest) {
+		barrier();
+		WRITE_ONCE(fpu->__task_fpstate, NULL);
+	}
+
 	fpregs_mark_activate();
 	fpregs_unlock();
 	return 0;
@@ -481,17 +504,15 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask)
 	this_cpu_write(kernel_fpu_allowed, false);
 
 	if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) &&
-	    !test_thread_flag(TIF_NEED_FPU_LOAD)) {
-		set_thread_flag(TIF_NEED_FPU_LOAD);
-		save_fpregs_to_fpstate(x86_task_fpu(current));
-	}
+	    !test_thread_flag(TIF_NEED_FPU_LOAD))
+		update_fpu_state_and_flag(x86_task_fpu(current), current);
 	__cpu_invalidate_fpregs_state();
 
 	/* Put sane initial values into the control registers. */
 	if (likely(kfpu_mask & KFPU_MXCSR) && boot_cpu_has(X86_FEATURE_XMM))
 		ldmxcsr(MXCSR_DEFAULT);
 
-	if (unlikely(kfpu_mask & KFPU_387) && boot_cpu_has(X86_FEATURE_FPU))
+	if (unlikely(kfpu_mask & KFPU_387))
 		asm volatile ("fninit");
 }
 EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask);
@@ -672,9 +693,6 @@ int fpu_clone(struct task_struct *dst, u64 clone_flags, bool minimal,
 
 	fpstate_reset(dst_fpu);
 
-	if (!cpu_feature_enabled(X86_FEATURE_FPU))
-		return 0;
-
 	/*
 	 * Enforce reload for user space tasks and prevent kernel threads
 	 * from trying to save the FPU registers on context switch.
@@ -833,11 +851,6 @@ void fpu__clear_user_states(struct fpu *fpu)
 	WARN_ON_FPU(fpu != x86_task_fpu(current));
 
 	fpregs_lock();
-	if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
-		fpu_reset_fpstate_regs();
-		fpregs_unlock();
-		return;
-	}
 
 	/*
 	 * Ensure that current's supervisor states are loaded into their
@@ -874,9 +887,6 @@ void fpu_flush_thread(void)
  */
 void switch_fpu_return(void)
 {
-	if (!cpu_feature_enabled(X86_FEATURE_FPU))
-		return;
-
 	fpregs_restore_userregs();
 }
 EXPORT_SYMBOL_FOR_KVM(switch_fpu_return);
diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c
index 0d33c217b71c1a..1a92ec433f9d34 100644
--- a/arch/x86/kernel/fpu/init.c
+++ b/arch/x86/kernel/fpu/init.c
@@ -31,8 +31,6 @@ static void fpu__init_cpu_generic(void)
 
 	cr0 = read_cr0();
 	cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
-	if (!boot_cpu_has(X86_FEATURE_FPU))
-		cr0 |= X86_CR0_EM;
 	write_cr0(cr0);
 
 	/* Flush out any pending x87 state: */
@@ -184,9 +182,7 @@ static void __init fpu__init_system_xstate_size_legacy(void)
 	 * Note that the size configuration might be overwritten later
 	 * during fpu__init_system_xstate().
 	 */
-	if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
-		size = sizeof(struct swregs_state);
-	} else if (cpu_feature_enabled(X86_FEATURE_FXSR)) {
+	if (cpu_feature_enabled(X86_FEATURE_FXSR)) {
 		size = sizeof(struct fxregs_state);
 		fpu_user_cfg.legacy_features = XFEATURE_MASK_FPSSE;
 	} else {
diff --git a/arch/x86/kernel/fpu/regset.c b/arch/x86/kernel/fpu/regset.c
index 0986c2200adc56..f96affd834a146 100644
--- a/arch/x86/kernel/fpu/regset.c
+++ b/arch/x86/kernel/fpu/regset.c
@@ -407,9 +407,6 @@ int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 
 	sync_fpstate(fpu);
 
-	if (!cpu_feature_enabled(X86_FEATURE_FPU))
-		return fpregs_soft_get(target, regset, to);
-
 	if (!cpu_feature_enabled(X86_FEATURE_FXSR)) {
 		return membuf_write(&to, &fpu->fpstate->regs.fsave,
 				    sizeof(struct fregs_state));
@@ -441,9 +438,6 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 	if (pos != 0 || count != sizeof(struct user_i387_ia32_struct))
 		return -EINVAL;
 
-	if (!cpu_feature_enabled(X86_FEATURE_FPU))
-		return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
-
 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 	if (ret)
 		return ret;
diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c
index 33e1284bf3e436..1f721ac842833a 100644
--- a/arch/x86/kernel/fpu/signal.c
+++ b/arch/x86/kernel/fpu/signal.c
@@ -24,23 +24,24 @@
  * Check for the presence of extended state information in the
  * user fpstate pointer in the sigcontext.
  */
-static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
+static inline bool check_xstate_in_sigframe(struct fxregs_state __user *buf_fx,
 					    struct _fpx_sw_bytes *fx_sw)
 {
+	struct fpstate *fpstate = x86_task_fpu(current)->fpstate;
 	int min_xstate_size = sizeof(struct fxregs_state) +
 			      sizeof(struct xstate_header);
-	void __user *fpstate = fxbuf;
+	void __user *buf = buf_fx;
 	unsigned int magic2;
 
-	if (__copy_from_user(fx_sw, &fxbuf->sw_reserved[0], sizeof(*fx_sw)))
+	if (__copy_from_user(fx_sw, &buf_fx->sw_reserved[0], sizeof(*fx_sw)))
 		return false;
 
 	/* Check for the first magic field and other error scenarios. */
 	if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
 	    fx_sw->xstate_size < min_xstate_size ||
-	    fx_sw->xstate_size > x86_task_fpu(current)->fpstate->user_size ||
-	    fx_sw->xstate_size > fx_sw->extended_size)
-		goto setfx;
+	    fx_sw->xstate_size > fpstate->user_size ||
+	    fx_sw->extended_size < fx_sw->xstate_size + FP_XSTATE_MAGIC2_SIZE)
+		goto err_setfx;
 
 	/*
 	 * Check for the presence of second magic word at the end of memory
@@ -48,12 +49,36 @@ static inline bool check_xstate_in_sigframe(struct fxregs_state __user *fxbuf,
 	 * fpstate layout with out copying the extended state information
 	 * in the memory layout.
 	 */
-	if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size)))
+	if (__get_user(magic2, (__u32 __user *)(buf + fx_sw->xstate_size)))
 		return false;
+	if (unlikely(magic2 != FP_XSTATE_MAGIC2))
+		goto err_setfx;
 
-	if (likely(magic2 == FP_XSTATE_MAGIC2))
-		return true;
-setfx:
+	if (fx_sw->xstate_size != fpstate->user_size ||
+	    fx_sw->xfeatures != fpstate->user_xfeatures) {
+		unsigned int xsize;
+		u64 xfeatures;
+
+		/* Calculate size of enabled features only. */
+		xfeatures = fx_sw->xfeatures & fpstate->user_xfeatures;
+
+		xsize = xstate_calculate_size(xfeatures, false);
+		if (fx_sw->xstate_size < xsize)
+			return false;
+
+		fx_sw->xstate_size = xsize;
+	}
+
+	return true;
+err_setfx:
+	/*
+	 * The fallback to FX-only state is used to preserve backward
+	 * compatibility with user-space processes that are not aware of xsave
+	 * states.
+	 *
+	 * In all other cases, returning false (to trigger SIGSEGV) is
+	 * preferred to avoid silent user-space state corruption.
+	 */
 	trace_x86_fpu_xstate_check_failed(x86_task_fpu(current));
 
 	/* Set the parameters for fx only state */
@@ -187,14 +212,6 @@ bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size, u
 	ia32_fxstate &= (IS_ENABLED(CONFIG_X86_32) ||
 			 IS_ENABLED(CONFIG_IA32_EMULATION));
 
-	if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
-		struct user_i387_ia32_struct fp;
-
-		fpregs_soft_get(current, NULL, (struct membuf){.p = &fp,
-						.left = sizeof(fp)});
-		return !copy_to_user(buf, &fp, sizeof(fp));
-	}
-
 	if (!access_ok(buf, size))
 		return false;
 
@@ -240,15 +257,18 @@ bool copy_fpstate_to_sigframe(void __user *buf, void __user *buf_fx, int size, u
 	return true;
 }
 
-static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures,
-				      u64 xrestore, bool fx_only)
+static int __restore_fpregs_from_user(void __user *buf, u64 task_xfeatures,
+				      u64 xrestore_mask, bool fx_only)
 {
 	if (use_xsave()) {
-		u64 init_bv = ufeatures & ~xrestore;
+		u64 init_bv;
 		int ret;
 
+		/* Restore enabled features only. */
+		xrestore_mask &= task_xfeatures;
+		init_bv = task_xfeatures & ~xrestore_mask;
 		if (likely(!fx_only))
-			ret = xrstor_from_user_sigframe(buf, xrestore);
+			ret = xrstor_from_user_sigframe(buf, xrestore_mask);
 		else
 			ret = fxrstor_from_user_sigframe(buf);
 
@@ -266,20 +286,19 @@ static int __restore_fpregs_from_user(void __user *buf, u64 ufeatures,
  * Attempt to restore the FPU registers directly from user memory.
  * Pagefaults are handled and any errors returned are fatal.
  */
-static bool restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_only)
+static bool restore_fpregs_from_user(void __user *buf, u64 xrestore_mask,
+				     bool fx_only, size_t xstate_size)
 {
 	struct fpu *fpu = x86_task_fpu(current);
 	int ret;
 
-	/* Restore enabled features only. */
-	xrestore &= fpu->fpstate->user_xfeatures;
 retry:
 	fpregs_lock();
 	/* Ensure that XFD is up to date */
 	xfd_update_state(fpu->fpstate);
 	pagefault_disable();
 	ret = __restore_fpregs_from_user(buf, fpu->fpstate->user_xfeatures,
-					 xrestore, fx_only);
+					 xrestore_mask, fx_only);
 	pagefault_enable();
 
 	if (unlikely(ret)) {
@@ -302,7 +321,7 @@ static bool restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_onl
 		if (ret != X86_TRAP_PF)
 			return false;
 
-		if (!fault_in_readable(buf, fpu->fpstate->user_size))
+		if (!fault_in_readable(buf, xstate_size))
 			goto retry;
 		return false;
 	}
@@ -324,39 +343,33 @@ static bool restore_fpregs_from_user(void __user *buf, u64 xrestore, bool fx_onl
 	return true;
 }
 
-static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx,
-			      bool ia32_fxstate)
+/*
+ * Restore FPU state from a signal frame when a legacy 32-bit FP frame
+ * (buf_f) is present.
+ *
+ * The legacy FP frame duplicates the FP state portion of the FX/XSAVE
+ * frame (buf_fx). For backward compatibility, the legacy FP frame is
+ * treated as the source of truth, and its state is folded into the
+ * FX/XSAVE state before restoring the registers.
+ */
+static bool restore_from_ia32_fxstate(void __user *buf_f, void __user *buf_fx,
+				      u64 xrestore_mask, bool fx_only)
 {
 	struct task_struct *tsk = current;
 	struct fpu *fpu = x86_task_fpu(tsk);
 	struct user_i387_ia32_struct env;
-	bool success, fx_only = false;
 	union fpregs_state *fpregs;
-	u64 user_xfeatures = 0;
-
-	if (use_xsave()) {
-		struct _fpx_sw_bytes fx_sw_user;
-
-		if (!check_xstate_in_sigframe(buf_fx, &fx_sw_user))
-			return false;
-
-		fx_only = !fx_sw_user.magic1;
-		user_xfeatures = fx_sw_user.xfeatures;
-	} else {
-		user_xfeatures = XFEATURE_MASK_FPSSE;
-	}
+	bool success;
 
-	if (likely(!ia32_fxstate)) {
-		/* Restore the FPU registers directly from user memory. */
-		return restore_fpregs_from_user(buf_fx, user_xfeatures, fx_only);
-	}
+	if (!IS_ENABLED(CONFIG_X86_32) && !IS_ENABLED(CONFIG_IA32_EMULATION))
+		return false;
 
 	/*
 	 * Copy the legacy state because the FP portion of the FX frame has
 	 * to be ignored for histerical raisins. The legacy state is folded
 	 * in once the larger state has been copied.
 	 */
-	if (__copy_from_user(&env, buf, sizeof(env)))
+	if (__copy_from_user(&env, buf_f, sizeof(env)))
 		return false;
 
 	/*
@@ -420,7 +433,7 @@ static bool __fpu_restore_sig(void __user *buf, void __user *buf_fx,
 		 *
 		 * Preserve supervisor states!
 		 */
-		u64 mask = user_xfeatures | xfeatures_mask_supervisor();
+		u64 mask = xrestore_mask | xfeatures_mask_supervisor();
 
 		fpregs->xsave.header.xfeatures &= mask;
 		success = !os_xrstor_safe(fpu->fpstate,
@@ -449,10 +462,11 @@ static inline unsigned int xstate_sigframe_size(struct fpstate *fpstate)
 bool fpu__restore_sig(void __user *buf, int ia32_frame)
 {
 	struct fpu *fpu = x86_task_fpu(current);
-	void __user *buf_fx = buf;
+	bool success = false, fx_only = false;
 	bool ia32_fxstate = false;
-	bool success = false;
+	void __user *buf_fx = buf;
 	unsigned int size;
+	u64 xrestore_mask;
 
 	if (unlikely(!buf)) {
 		fpu__clear_user_states(fpu);
@@ -477,14 +491,24 @@ bool fpu__restore_sig(void __user *buf, int ia32_frame)
 	if (!access_ok(buf, size))
 		goto out;
 
-	if (!IS_ENABLED(CONFIG_X86_64) && !cpu_feature_enabled(X86_FEATURE_FPU)) {
-		success = !fpregs_soft_set(current, NULL, 0,
-					   sizeof(struct user_i387_ia32_struct),
-					   NULL, buf);
+	if (use_xsave()) {
+		struct _fpx_sw_bytes fx_sw_user;
+
+		if (!check_xstate_in_sigframe(buf_fx, &fx_sw_user))
+			goto out;
+
+		fx_only = !fx_sw_user.magic1;
+		xrestore_mask = fx_sw_user.xfeatures;
+		size = fx_sw_user.xstate_size;
 	} else {
-		success = __fpu_restore_sig(buf, buf_fx, ia32_fxstate);
+		xrestore_mask = XFEATURE_MASK_FPSSE;
+		size = fpu->fpstate->user_size;
 	}
 
+	if (ia32_fxstate)
+		success = restore_from_ia32_fxstate(buf, buf_fx, xrestore_mask, fx_only);
+	else
+		success = restore_fpregs_from_user(buf_fx, xrestore_mask, fx_only, size);
 out:
 	if (unlikely(!success))
 		fpu__clear_user_states(fpu);
diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
index a7b6524a9dea22..da303714379cd1 100644
--- a/arch/x86/kernel/fpu/xstate.c
+++ b/arch/x86/kernel/fpu/xstate.c
@@ -587,14 +587,15 @@ static bool __init check_xstate_against_struct(int nr)
 	return true;
 }
 
-static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted)
+unsigned int xstate_calculate_size(u64 xfeatures, bool compacted)
 {
-	unsigned int topmost = fls64(xfeatures) -  1;
-	unsigned int offset, i;
+	unsigned int topmost, offset, i;
 
-	if (topmost <= XFEATURE_SSE)
+	if (!(xfeatures & ~XFEATURE_MASK_FPSSE))
 		return sizeof(struct xregs_state);
 
+	topmost = fls64(xfeatures) -  1;
+
 	if (compacted) {
 		offset = xfeature_get_offset(xfeatures, topmost);
 	} else {
@@ -806,18 +807,15 @@ static u64 __init guest_default_mask(void)
 void __init fpu__init_system_xstate(unsigned int legacy_size)
 {
 	unsigned int eax, ebx, ecx, edx;
-	u64 xfeatures;
+	u64 xfeatures, mask;
 	int err;
 	int i;
 
-	if (!boot_cpu_has(X86_FEATURE_FPU)) {
-		pr_info("x86/fpu: No FPU detected\n");
-		return;
-	}
-
 	if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
 		pr_info("x86/fpu: x87 FPU will use %s\n",
 			boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
+		/* Disable all dependent flags too */
+		setup_clear_cpu_cap(X86_FEATURE_XSAVE);
 		return;
 	}
 
@@ -833,7 +831,8 @@ void __init fpu__init_system_xstate(unsigned int legacy_size)
 	cpuid_count(CPUID_LEAF_XSTATE, 1, &eax, &ebx, &ecx, &edx);
 	fpu_kernel_cfg.max_features |= ecx + ((u64)edx << 32);
 
-	if ((fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
+	mask = XFEATURE_MASK_FPSSE;
+	if ((fpu_kernel_cfg.max_features & mask) != mask) {
 		/*
 		 * This indicates that something really unexpected happened
 		 * with the enumeration.  Disable XSAVE and try to continue
@@ -844,6 +843,24 @@ void __init fpu__init_system_xstate(unsigned int legacy_size)
 		goto out_disable;
 	}
 
+	mask |= XFEATURE_MASK_YMM;
+	if (boot_cpu_has(X86_FEATURE_AVX)) {
+		if ((fpu_kernel_cfg.max_features & mask) != mask) {
+			pr_err(FW_BUG
+			       "x86/fpu: Disabling AVX support due to missing xstate features\n");
+			setup_clear_cpu_cap(X86_FEATURE_AVX);
+		}
+	}
+
+	mask |= XFEATURE_MASK_AVX512;
+	if (boot_cpu_has(X86_FEATURE_AVX512F)) {
+		if ((fpu_kernel_cfg.max_features & mask) != mask) {
+			pr_err(FW_BUG
+			       "x86/fpu: Disabling AVX-512 support due to missing xstate features\n");
+			setup_clear_cpu_cap(X86_FEATURE_AVX512F);
+		}
+	}
+
 	if (fpu_kernel_cfg.max_features & XFEATURE_MASK_APX &&
 	    fpu_kernel_cfg.max_features & (XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR)) {
 		/*
@@ -1474,6 +1491,29 @@ void xrstors(struct xregs_state *xstate, u64 mask)
 	WARN_ON_ONCE(err);
 }
 
+/**
+ * xsaves_nmi - Save selected components to a kernel xstate buffer in NMI
+ * @xstate:	Pointer to the buffer
+ * @mask:	Feature mask to select the components to save
+ *
+ * This function is similar to xsaves(), but should only be called within
+ * the NMI handler. This function returns the actual register contents at
+ * the moment the NMI occurs.
+ *
+ * Currently, the perf subsystem is the sole user of this helper. It uses
+ * the function to snapshot SIMD (XMM/YMM/ZMM) and APX eGPRs registers.
+ */
+void xsaves_nmi(struct xregs_state *xstate, u64 mask)
+{
+	int err;
+
+	if (!in_nmi())
+		return;
+
+	XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err);
+	WARN_ON_ONCE(err);
+}
+
 #if IS_ENABLED(CONFIG_KVM)
 void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature)
 {
diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c
index 7e980ea49d8d62..3aa77006adab20 100644
--- a/arch/x86/kernel/kexec-bzimage64.c
+++ b/arch/x86/kernel/kexec-bzimage64.c
@@ -285,14 +285,14 @@ static void setup_kho(const struct kimage *image, struct boot_params *params,
 	sd->len = sizeof(struct kho_data);
 
 	/* Only add if we have all KHO images in place */
-	if (!image->kho.fdt || !image->kho.scratch)
+	if (!image->kho.fdt || !image->kho.bootmem)
 		return;
 
 	/* Add setup data */
 	kho->fdt_addr = image->kho.fdt;
 	kho->fdt_size = PAGE_SIZE;
-	kho->scratch_addr = image->kho.scratch->mem;
-	kho->scratch_size = image->kho.scratch->bufsz;
+	kho->bootmem_addr = image->kho.bootmem->mem;
+	kho->bootmem_size = image->kho.bootmem->bufsz;
 	sd->next = params->hdr.setup_data;
 	params->hdr.setup_data = params_load_addr + setup_data_offset;
 }
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index 4e5f8c1736ec14..133ff20caccd45 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -510,10 +510,9 @@ NOKPROBE_SYMBOL(kprobe_emulate_ret);
 
 static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs)
 {
-	unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size;
+	unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;
 
-	func += p->ainsn.rel32;
-	int3_emulate_call(regs, func);
+	int3_emulate_call(regs, ip, ip + p->ainsn.rel32);
 }
 NOKPROBE_SYMBOL(kprobe_emulate_call);
 
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 6b0a5861ccb8eb..2ce01fdb8b4b9c 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -1054,9 +1054,9 @@ const __initconst struct hypervisor_x86 x86_hyper_kvm = {
 static __init int activate_jump_labels(void)
 {
 	if (has_steal_clock) {
-		static_key_slow_inc(&paravirt_steal_enabled);
+		static_branch_inc(&paravirt_steal_enabled);
 		if (steal_acc)
-			static_key_slow_inc(&paravirt_steal_rq_enabled);
+			static_branch_inc(&paravirt_steal_rq_enabled);
 	}
 
 	return 0;
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 3c9f60d6ca5a76..aae72547746802 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -85,6 +85,7 @@ static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
 static int ignore_nmis __read_mostly;
 
 int unknown_nmi_panic;
+static unsigned long toolong_nmi_panic;
 int panic_on_unrecovered_nmi;
 int panic_on_io_nmi;
 
@@ -101,6 +102,20 @@ static int __init setup_unknown_nmi_panic(char *str)
 }
 __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
 
+static int __init setup_toolong_nmi_panic(char *str)
+{
+	int ret;
+
+	ret = kstrtoul(str, 0, &toolong_nmi_panic);
+	if (ret) {
+		pr_alert("%s: %s on '%s', disabling to zero.\n",
+			 __func__, ret == -ERANGE ? "Overflow" : "Parse error", str);
+		toolong_nmi_panic = 0;
+	}
+	return 1;
+}
+__setup("toolong_nmi_panic=", setup_toolong_nmi_panic);
+
 static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
 
 static int __init nmi_warning_debugfs(void)
@@ -111,7 +126,7 @@ static int __init nmi_warning_debugfs(void)
 }
 fs_initcall(nmi_warning_debugfs);
 
-static void nmi_check_duration(struct nmiaction *action, u64 duration)
+static void nmi_check_duration(struct nmiaction *action, u64 duration, struct pt_regs *regs)
 {
 	int remainder_ns, decimal_msecs;
 
@@ -126,6 +141,8 @@ static void nmi_check_duration(struct nmiaction *action, u64 duration)
 
 	pr_info_ratelimited("INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
 			    action->handler, duration, decimal_msecs);
+	if (toolong_nmi_panic && duration >= toolong_nmi_panic)
+		nmi_panic(regs, "Panic per toolong_nmi_panic");
 }
 
 static int nmi_handle(unsigned int type, struct pt_regs *regs)
@@ -165,7 +182,7 @@ static int nmi_handle(unsigned int type, struct pt_regs *regs)
 		delta = sched_clock() - delta;
 		trace_nmi_handler(a->handler, (int)delta, thishandled);
 
-		nmi_check_duration(a, delta);
+		nmi_check_duration(a, delta, regs);
 	}
 
 	rcu_read_unlock();
diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c
index 624703af80a199..f8952f7c36cce5 100644
--- a/arch/x86/kernel/perf_regs.c
+++ b/arch/x86/kernel/perf_regs.c
@@ -61,11 +61,29 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 {
 	struct x86_perf_regs *perf_regs;
 
-	if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) {
+	if (idx > PERF_REG_X86_R15) {
 		perf_regs = container_of(regs, struct x86_perf_regs, regs);
-		if (!perf_regs->xmm_regs)
+		if (perf_regs->abi == PERF_SAMPLE_REGS_ABI_NONE)
 			return 0;
-		return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0];
+
+		if (perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) {
+			if (idx <= PERF_REG_X86_R31) {
+				if (!perf_regs->egpr_regs)
+					return 0;
+				return perf_regs->egpr_regs[idx - PERF_REG_X86_R16];
+			}
+			if (idx == PERF_REG_X86_SSP) {
+				if (!perf_regs->ssp)
+					return 0;
+				return *perf_regs->ssp;
+			}
+		} else {
+			if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) {
+				if (!perf_regs->xmm_regs)
+					return 0;
+				return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0];
+			}
+		}
 	}
 
 	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset)))
@@ -74,22 +92,130 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 	return regs_get_register(regs, pt_regs_offset[idx]);
 }
 
-#define PERF_REG_X86_RESERVED	(((1ULL << PERF_REG_X86_XMM0) - 1) & \
-				 ~((1ULL << PERF_REG_X86_MAX) - 1))
+#define PERF_X86_YMMH_QWORDS	(PERF_X86_YMM_QWORDS / 2)
+#define PERF_X86_ZMMH_QWORDS	(PERF_X86_ZMM_QWORDS / 2)
+
+u64 perf_simd_reg_value(struct pt_regs *regs, int idx,
+			u16 qwords_idx, bool pred)
+{
+	struct x86_perf_regs *perf_regs =
+			container_of(regs, struct x86_perf_regs, regs);
+
+	if (!(perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD))
+		return 0;
+
+	if (pred) {
+		if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_PRED_REGS_MAX ||
+				 qwords_idx >= PERF_X86_OPMASK_QWORDS))
+			return 0;
+		if (!perf_regs->opmask_regs)
+			return 0;
+		return perf_regs->opmask_regs[idx];
+	}
+
+	if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_VEC_REGS_MAX ||
+			 qwords_idx >= PERF_X86_SIMD_QWORDS_MAX))
+		return 0;
+
+	if (idx >= PERF_X86_H16ZMM_BASE) {
+		if (!perf_regs->h16zmm_regs)
+			return 0;
+		return perf_regs->h16zmm_regs[(idx - PERF_X86_H16ZMM_BASE) *
+					PERF_X86_ZMM_QWORDS + qwords_idx];
+	}
+
+	if (qwords_idx < PERF_X86_XMM_QWORDS) {
+		if (!perf_regs->xmm_regs)
+			return 0;
+		return perf_regs->xmm_regs[idx * PERF_X86_XMM_QWORDS +
+					   qwords_idx];
+	} else if (qwords_idx < PERF_X86_YMM_QWORDS) {
+		if (!perf_regs->ymmh_regs)
+			return 0;
+		return perf_regs->ymmh_regs[idx * PERF_X86_YMMH_QWORDS +
+					    qwords_idx - PERF_X86_XMM_QWORDS];
+	} else if (qwords_idx < PERF_X86_ZMM_QWORDS) {
+		if (!perf_regs->zmmh_regs)
+			return 0;
+		return perf_regs->zmmh_regs[idx * PERF_X86_ZMMH_QWORDS +
+					    qwords_idx - PERF_X86_YMM_QWORDS];
+	}
+
+	return 0;
+}
+
+int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask,
+			   u16 pred_qwords, u32 pred_mask)
+{
+	unsigned long mask;
+	u64 size;
+
+	if (!vec_qwords && !pred_qwords) {
+		if (vec_mask || pred_mask)
+			return -EINVAL;
+	}
+
+	if (vec_qwords) {
+		if (vec_qwords != PERF_X86_XMM_QWORDS &&
+		    vec_qwords != PERF_X86_YMM_QWORDS &&
+		    vec_qwords != PERF_X86_ZMM_QWORDS)
+			return -EINVAL;
+		if (vec_mask & ~PERF_X86_SIMD_VEC_MASK)
+			return -EINVAL;
+		/* Only full-register sampling is allowed. */
+		mask = vec_mask;
+		if (vec_qwords == PERF_X86_XMM_QWORDS && mask &&
+		    !bitmap_full(&mask, PERF_X86_SIMD_XMM_REGS))
+			return -EINVAL;
+		if (vec_qwords == PERF_X86_YMM_QWORDS && mask &&
+		    !bitmap_full(&mask, PERF_X86_SIMD_YMM_REGS))
+			return -EINVAL;
+		if (vec_qwords == PERF_X86_ZMM_QWORDS && mask &&
+		    !bitmap_full(&mask, PERF_X86_SIMD_ZMM_REGS))
+			return -EINVAL;
+	}
+
+	if (pred_qwords) {
+		if (pred_qwords != PERF_X86_OPMASK_QWORDS)
+			return -EINVAL;
+		if (pred_mask & ~PERF_X86_SIMD_PRED_MASK)
+			return -EINVAL;
+		/* Only full-register sampling is allowed. */
+		mask = pred_mask;
+		if (pred_qwords == PERF_X86_OPMASK_QWORDS && mask &&
+		    !bitmap_full(&mask, PERF_X86_SIMD_OPMASK_REGS))
+			return -EINVAL;
+	}
+
+	size = sizeof(u64) * 4;
+	size += (hweight64(vec_mask) * vec_qwords +
+		 hweight32(pred_mask) * pred_qwords) * sizeof(u64);
+	/*
+	 * INTR_REGS and USR_REGS could be sampled simultaneously,
+	 * so roughly restrict the size to half of U16_MAX.
+	 */
+	if (size >= U16_MAX / 2)
+		return -EINVAL;
+
+	return 0;
+}
+
+#define PERF_REG_X86_RESERVED	(GENMASK_ULL(PERF_REG_X86_XMM0 - 1, PERF_REG_X86_AX) & \
+				 ~GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_AX))
+#define PERF_REG_X86_EXT_RESERVED	(~GENMASK_ULL(PERF_REG_MISC_MAX - 1, PERF_REG_X86_AX))
 
 #ifdef CONFIG_X86_32
-#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_R8) | \
-		       (1ULL << PERF_REG_X86_R9) | \
-		       (1ULL << PERF_REG_X86_R10) | \
-		       (1ULL << PERF_REG_X86_R11) | \
-		       (1ULL << PERF_REG_X86_R12) | \
-		       (1ULL << PERF_REG_X86_R13) | \
-		       (1ULL << PERF_REG_X86_R14) | \
-		       (1ULL << PERF_REG_X86_R15))
-
-int perf_reg_validate(u64 mask)
+#define REG_NOSUPPORT GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_R8)
+
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
-	if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))
+	if (!simd_enabled &&
+	    (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))))
+		return -EINVAL;
+
+	/* The mask could be 0 if only the SIMD registers are interested */
+	if (simd_enabled &&
+	    (mask & ~GENMASK_ULL(PERF_REG_X86_GS, PERF_REG_X86_AX)))
 		return -EINVAL;
 
 	return 0;
@@ -100,21 +226,21 @@ u64 perf_reg_abi(struct task_struct *task)
 	return PERF_SAMPLE_REGS_ABI_32;
 }
 
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
 #else /* CONFIG_X86_64 */
 #define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \
 		       (1ULL << PERF_REG_X86_ES) | \
 		       (1ULL << PERF_REG_X86_FS) | \
 		       (1ULL << PERF_REG_X86_GS))
 
-int perf_reg_validate(u64 mask)
+int perf_reg_validate(u64 mask, bool simd_enabled)
 {
-	if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))
+	if (!simd_enabled &&
+	    (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))))
+		return -EINVAL;
+
+	/* The mask could be 0 if only the SIMD registers are interested */
+	if (simd_enabled &&
+	    (mask & (REG_NOSUPPORT | PERF_REG_X86_EXT_RESERVED)))
 		return -EINVAL;
 
 	return 0;
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index cda6adb9f69c42..19fc8e1714e645 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -472,7 +472,7 @@ static void __init add_kho(u64 phys_addr, u32 data_len)
 		return;
 	}
 
-	kho_populate(kho->fdt_addr, kho->fdt_size, kho->scratch_addr, kho->scratch_size);
+	kho_populate(kho->fdt_addr, kho->fdt_size, kho->bootmem_addr, kho->bootmem_size);
 
 	early_memunmap(kho, size);
 }
diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c
index 0ca64900192f83..eb690ba90180c7 100644
--- a/arch/x86/kernel/shstk.c
+++ b/arch/x86/kernel/shstk.c
@@ -490,7 +490,7 @@ static int wrss_control(bool enable)
 	 * when disabling.
 	 */
 	if (!features_enabled(ARCH_SHSTK_SHSTK))
-		return -EPERM;
+		return -EINVAL;
 
 	/* Already enabled/disabled? */
 	if (features_enabled(ARCH_SHSTK_WRSS) == enable)
diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c
index 776ae6fa7f2d67..4c078827f26a15 100644
--- a/arch/x86/kernel/sys_x86_64.c
+++ b/arch/x86/kernel/sys_x86_64.c
@@ -50,9 +50,16 @@ static unsigned long get_align_mask(struct file *filp)
  * value before calling vm_unmapped_area() or ORed directly to the
  * address.
  */
-static unsigned long get_align_bits(void)
+static unsigned long get_align_bits(struct file *filp)
 {
-	return va_align.bits & get_align_mask(NULL);
+	/*
+	 * va_align.bits is smaller than the huge page size and will
+	 * lead to misaligned huge pages. Ignore it for huge mappings.
+	 */
+	if (is_file_hugepages(filp))
+		return 0;
+
+	return va_align.bits & get_align_mask(filp);
 }
 
 static int __init control_va_addr_alignment(char *str)
@@ -157,7 +164,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len,
 	}
 	if (filp) {
 		info.align_mask = get_align_mask(filp);
-		info.align_offset += get_align_bits();
+		info.align_offset += get_align_bits(filp);
 	}
 
 	return vm_unmapped_area(&info);
@@ -222,7 +229,7 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr0,
 
 	if (filp) {
 		info.align_mask = get_align_mask(filp);
-		info.align_offset += get_align_bits();
+		info.align_offset += get_align_bits(filp);
 	}
 	addr = vm_unmapped_area(&info);
 	if (!(addr & ~PAGE_MASK))
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index 65a2de82ecd292..0f60c0d076b620 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -715,7 +715,7 @@ static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsign
 
 	*new_mapping = true;
 	return _install_special_mapping(mm, vaddr, PAGE_SIZE,
-				VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO,
+				VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_MIXEDMAP,
 				&tramp_mapping);
 }
 
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index 538ed1e8033268..2c3c22aeafa542 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -80,11 +80,22 @@ config KVM_WERROR
 
 	  If in doubt, say "N".
 
+config KVM_VM_MEMORY_ATTRIBUTES
+	bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)"
+	depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV
+	help
+	  Enable support for tracking PRIVATE vs. SHARED memory using per-VM
+	  memory attributes.  Using per-VM attributes is deprecated in favor of
+	  tracking PRIVATE state in guest_memfd.  Select this if you need to run
+	  CoCo VMs using a VMM that doesn't support guest_memfd memory
+	  attributes.
+
+	  If unsure, say N.
+
 config KVM_SW_PROTECTED_VM
 	bool "Enable support for KVM software-protected VMs"
 	depends on EXPERT
 	depends on KVM_X86 && X86_64
-	select KVM_GENERIC_MEMORY_ATTRIBUTES
 	help
 	  Enable support for KVM software-protected VMs.  Currently, software-
 	  protected VMs are purely a development and testing vehicle for
@@ -135,7 +146,6 @@ config KVM_INTEL_TDX
 	bool "Intel Trust Domain Extensions (TDX) support"
 	default y
 	depends on INTEL_TDX_HOST
-	select KVM_GENERIC_MEMORY_ATTRIBUTES
 	select HAVE_KVM_ARCH_GMEM_POPULATE
 	help
 	  Provides support for launching Intel Trust Domain Extensions (TDX)
@@ -159,7 +169,6 @@ config KVM_AMD_SEV
 	depends on KVM_AMD && X86_64
 	depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m)
 	select ARCH_HAS_CC_PLATFORM
-	select KVM_GENERIC_MEMORY_ATTRIBUTES
 	select HAVE_KVM_ARCH_GMEM_CONVERT
 	select HAVE_KVM_ARCH_GMEM_RECLAIM
 	select HAVE_KVM_ARCH_GMEM_INVALIDATE
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 851f151efb35b5..7c63ef90fc60ad 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -16,6 +16,7 @@
 #include <linux/vmalloc.h>
 #include <linux/uaccess.h>
 #include <linux/sched/stat.h>
+#include <linux/units.h>
 
 #include <asm/processor.h>
 #include <asm/user.h>
@@ -483,20 +484,6 @@ int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu)
 	return 36;
 }
 
-int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu)
-{
-	struct kvm_cpuid_entry2 *best;
-
-	best = kvm_find_cpuid_entry(vcpu, 0x80000000);
-	if (!best || best->eax < 0x80000008)
-		goto not_found;
-	best = kvm_find_cpuid_entry(vcpu, 0x80000008);
-	if (best)
-		return (best->eax >> 16) & 0xff;
-not_found:
-	return 0;
-}
-
 /*
  * This "raw" version returns the reserved GPA bits without any adjustments for
  * encryption technologies that usurp bits.  The raw mask should be used if and
@@ -1095,7 +1082,7 @@ void kvm_initialize_cpu_caps(void)
 	);
 
 	kvm_cpu_cap_init(CPUID_24_1_ECX,
-		F(AVX10_VNNI_INT),
+		F(AVX10_V1_AUX),
 	);
 
 	kvm_cpu_cap_init(CPUID_8000_0001_ECX,
diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h
index 8d863f45585de0..46bfe8699e67f1 100644
--- a/arch/x86/kvm/cpuid.h
+++ b/arch/x86/kvm/cpuid.h
@@ -68,7 +68,6 @@ void __init kvm_init_xstate_sizes(void);
 u32 xstate_required_size(u64 xstate_bv, bool compacted);
 
 int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu);
-int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu);
 u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu);
 
 static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index 604651cb273993..5d2e3617c95200 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -802,7 +802,9 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint,
 		if (no_retry)
 			return 0;
 
+		hv_hdr.message_flags.asu8 = 0;
 		hv_hdr.message_flags.msg_pending = 1;
+
 		r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn,
 					      &hv_hdr.message_flags,
 					      msg_off +
@@ -1995,7 +1997,6 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
 	u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE];
 	int i, j, count;
 	gva_t gva;
-	bool full = false;
 
 	if (!tdp_enabled)
 		return -EINVAL;
@@ -2006,7 +2007,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
 
 	count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE);
 
-	for (i = 0; i < count && !full; i++) {
+	for (i = 0; i < count; i++) {
 		if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY)
 			goto out_flush_all;
 
@@ -2015,11 +2016,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
 		 * pages to flush.
 		 */
 		gva = entries[i] & PAGE_MASK;
-		for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) {
+		for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) {
 			if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu))
 				continue;
 
-			kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full);
+			kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE);
 		}
 
 		++vcpu->stat.tlb_flush;
diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h
index 3e375af15c035f..10886bea82c91f 100644
--- a/arch/x86/kvm/kvm_emulate.h
+++ b/arch/x86/kvm/kvm_emulate.h
@@ -354,8 +354,8 @@ struct x86_emulate_ctxt {
 	bool have_exception;
 	struct x86_exception exception;
 
-	/* GPA available */
-	bool gpa_available;
+	/* Saved GPA and permitted emulated accesses. */
+	u64 gpa_access;
 	gpa_t gpa_val;
 
 	/*
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index e1f3cea147659c..ecf611845fb12d 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1552,7 +1552,7 @@ static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
 	return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
 }
 
-static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
+static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, u8 vector)
 {
 	int __maybe_unused trigger_mode;
 
@@ -1622,7 +1622,7 @@ static int apic_set_eoi(struct kvm_lapic *apic)
  * this interface assumes a trap-like exit, which has already finished
  * desired side effect including vISR and vPPR update.
  */
-void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
+void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index bd1098c89d9980..db071d0b172d05 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -177,7 +177,7 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu);
 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data);
 
 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset);
-void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector);
+void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector);
 
 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr);
 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 064ecc33b92670..dc2aef72ffddce 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -617,6 +617,10 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
 					       PT64_ROOT_MAX_LEVEL);
 		if (r)
 			return r;
+
+		r = kvm_x86_call(topup_external_cache)(vcpu, PT64_ROOT_MAX_LEVEL);
+		if (r)
+			return r;
 	}
 	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
 				       PT64_ROOT_MAX_LEVEL);
@@ -864,11 +868,12 @@ void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
 	list_del_init(&sp->possible_nx_huge_page_link);
 }
 
-static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp)
+static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp,
+				    enum kvm_mmu_type mmu_type)
 {
 	sp->nx_huge_page_disallowed = false;
 
-	untrack_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU);
+	untrack_possible_nx_huge_page(kvm, sp, mmu_type);
 }
 
 static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu,
@@ -895,6 +900,7 @@ static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu
  */
 #define KVM_RMAP_MANY	BIT(0)
 
+#ifndef CONFIG_PREEMPT_RT
 /*
  * rmaps and PTE lists are mostly protected by mmu_lock (the shadow MMU always
  * operates with mmu_lock held for write), but rmaps can be walked without
@@ -1008,7 +1014,8 @@ static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head)
  * actual locking is the same, but the caller is disallowed from modifying the
  * rmap, and so the unlock flow is a nop if the rmap is/was empty.
  */
-static unsigned long kvm_rmap_lock_readonly(struct kvm_rmap_head *rmap_head)
+static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm,
+					    struct kvm_rmap_head *rmap_head)
 {
 	unsigned long rmap_val;
 
@@ -1032,6 +1039,35 @@ static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head,
 	__kvm_rmap_unlock(rmap_head, old_val);
 	preempt_enable();
 }
+#else
+static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head)
+{
+	return atomic_long_read(&rmap_head->val);
+}
+static unsigned long kvm_rmap_lock(struct kvm *kvm,
+				   struct kvm_rmap_head *rmap_head)
+{
+	lockdep_assert_held_write(&kvm->mmu_lock);
+	return kvm_rmap_get(rmap_head);
+}
+
+static void kvm_rmap_unlock(struct kvm *kvm,
+			    struct kvm_rmap_head *rmap_head,
+			    unsigned long new_val)
+{
+	atomic_long_set_release(&rmap_head->val, new_val);
+}
+
+static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm,
+					    struct kvm_rmap_head *rmap_head)
+{
+	lockdep_assert_held_read(&kvm->mmu_lock);
+	return kvm_rmap_get(rmap_head);
+}
+
+static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head,
+				     unsigned long old_val) { }
+#endif
 
 /*
  * Returns the number of pointers in the rmap chain, not counting the new one.
@@ -1745,11 +1781,24 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm,
 	gfn_t gfn;
 	int level;
 
+	BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING));
+
+	/*
+	 * For realtime kernels, do aging under mmu_lock (per-rmap locking is
+	 * compiled out), as realtime deployments are unlikely to benefit from
+	 * increased aging throughput and reduced jitter for memory-overcommitted
+	 * nested VMs, whereas keeping preemption enabled is extremely valuable
+	 * (mmu_lock becomes a sleepable lock on realtime kernels).
+	 */
+#ifdef CONFIG_PREEMPT_RT
+	guard(read_lock)(&kvm->mmu_lock);
+#endif
+
 	for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
 		for (gfn = range->start; gfn < range->end;
 		     gfn += KVM_PAGES_PER_HPAGE(level)) {
 			rmap_head = gfn_to_rmap(gfn, level, range->slot);
-			rmap_val = kvm_rmap_lock_readonly(rmap_head);
+			rmap_val = kvm_rmap_lock_readonly(kvm, rmap_head);
 
 			for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) {
 				if (!is_accessed_spte(old_spte))
@@ -2512,6 +2561,12 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato
 
 	iterator->addr = addr;
 	iterator->shadow_addr = root;
+
+	if (WARN_ON_ONCE(!VALID_PAGE(root)) || kvm_mmu_is_dummy_root(root)) {
+		iterator->level = 0;
+		return;
+	}
+
 	iterator->level = vcpu->arch.mmu->root_role.level;
 
 	if (iterator->level >= PT64_ROOT_4LEVEL &&
@@ -2785,7 +2840,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
 	}
 
 	if (sp->nx_huge_page_disallowed)
-		unaccount_nx_huge_page(kvm, sp);
+		unaccount_nx_huge_page(kvm, sp, KVM_SHADOW_MMU);
 
 	sp->role.invalid = 1;
 
@@ -3021,6 +3076,9 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
 		if (prefetch)
 			return -EEXIST;
 
+		if (KVM_BUG_ON(sp->role.level != PG_LEVEL_4K, kvm))
+			continue;
+
 		/*
 		 * TDP MMU page faults require an additional spinlock as they
 		 * run with mmu_lock held for read, not write, and the unsync
@@ -3044,7 +3102,6 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
 				continue;
 		}
 
-		WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K);
 		kvm_unsync_page(kvm, sp);
 	}
 	if (locked)
@@ -3421,7 +3478,7 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault,
 		is_private = fault->is_private;
 	} else {
 		max_level = PG_LEVEL_NUM;
-		is_private = kvm_mem_is_private(kvm, gfn);
+		is_private = kvm_is_private_gfn(kvm, gfn);
 	}
 
 	max_level = min(max_level, max_huge_page_level);
@@ -3531,8 +3588,8 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 					     fault->req_level >= it.level);
 	}
 
-	if (WARN_ON_ONCE(it.level != fault->goal_level))
-		return -EFAULT;
+	if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm))
+		return -EIO;
 
 	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, access,
 			   base_gfn, fault->pfn, fault);
@@ -3570,6 +3627,7 @@ static int kvm_handle_error_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa
 		return RET_PF_RETRY;
 	}
 
+	kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
 	return -EFAULT;
 }
 
@@ -3634,13 +3692,13 @@ static bool page_fault_can_be_fast(struct kvm *kvm, struct kvm_page_fault *fault
 	 * guest spinning on a #PF indefinitely, so don't attempt the fast path
 	 * in this case.
 	 *
-	 * Note that the kvm_mem_is_private() check might race with an
+	 * Note that the kvm_is_private_gfn() check might race with an
 	 * attribute update, but this will either result in the guest spinning
 	 * on RET_PF_SPURIOUS until the update completes, or an actual spurious
 	 * case might go down the slow path. Either case will resolve itself.
 	 */
 	if (kvm->arch.has_private_mem &&
-	    fault->is_private != kvm_mem_is_private(kvm, fault->gfn))
+	    fault->is_private != kvm_is_private_gfn(kvm, fault->gfn))
 		return false;
 
 	/*
@@ -4628,7 +4686,7 @@ static int kvm_mmu_faultin_pfn_gmem(struct kvm_vcpu *vcpu,
 	}
 
 	r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn,
-			     &fault->refcounted_page, &max_order);
+			     &max_order);
 	if (r) {
 		kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
 		return r;
@@ -4708,7 +4766,7 @@ static int kvm_mmu_faultin_pfn(struct kvm_vcpu *vcpu,
 	 * Now that we have a snapshot of mmu_invalidate_seq we can check for a
 	 * private vs. shared mismatch.
 	 */
-	if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) {
+	if (fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) {
 		kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
 		return -EFAULT;
 	}
@@ -4895,8 +4953,8 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 
 #ifndef CONFIG_X86_64
 	/* A 64-bit CR2 should be impossible on 32-bit KVM. */
-	if (WARN_ON_ONCE(fault_address >> 32))
-		return -EFAULT;
+	if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm))
+		return -EIO;
 #endif
 	/*
 	 * Legacy #PF exception only have a 32-bit error code.  Simply drop the
@@ -5058,9 +5116,13 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa,
 		if (signal_pending(current))
 			return -EINTR;
 
-		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
+		if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu))
 			return -EIO;
 
+		r = kvm_mmu_reload(vcpu);
+		if (r)
+			return r;
+
 		cond_resched();
 		r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, true, NULL, level);
 	} while (r == RET_PF_RETRY);
@@ -5086,6 +5148,14 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa,
 	}
 }
 
+int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
+{
+	if (!vcpu->kvm->arch.pre_fault_allowed)
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
 long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
 				    struct kvm_pre_fault_memory *range)
 {
@@ -5095,23 +5165,12 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
 	u64 end;
 	int r;
 
-	if (!vcpu->kvm->arch.pre_fault_allowed)
-		return -EOPNOTSUPP;
-
 	if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa)))
 		return -EINVAL;
 
-	/*
-	 * reload is efficient when called repeatedly, so we can do it on
-	 * every iteration.
-	 */
-	r = kvm_mmu_reload(vcpu);
-	if (r)
-		return r;
-
 	direct_bits = 0;
 	if (kvm_arch_has_private_mem(vcpu->kvm) &&
-	    kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa)))
+	    kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(range->gpa)))
 		error_code |= PFERR_PRIVATE_ACCESS;
 	else
 		direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
@@ -5213,14 +5272,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
 	if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn))
 		return -EPERM;
 
-	r = kvm_mmu_reload(vcpu);
-	if (r)
-		return r;
-
-	r = mmu_topup_memory_caches(vcpu, false);
-	if (r)
-		return r;
-
 	do {
 		if (signal_pending(current))
 			return -EINTR;
@@ -5228,10 +5279,38 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
 		if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu))
 			return -EIO;
 
+		r = kvm_mmu_reload(vcpu);
+		if (r)
+			return r;
+
+		r = mmu_topup_memory_caches(vcpu, false);
+		if (r)
+			return r;
+
 		cond_resched();
 
 		guard(read_lock)(&kvm->mmu_lock);
 
+		/*
+		 * Because slots_lock is held, it should be impossible for *any*
+		 * roots to be invalidated after the initial MMU reload.  WARN,
+		 * but continue on; the above MMU reload will do the right thing
+		 * if the current root is actually invalid.
+		 */
+		WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu));
+
+		/*
+		 * Snapshot the invalidation sequence counter after acquiring
+		 * mmu_lock, as guest_memfd guarantees the validity of the pfn,
+		 * i.e. any concurrent invalidations are guaranteed to be
+		 * irrelevant.
+		 */
+		fault.mmu_seq = vcpu->kvm->mmu_invalidate_seq;
+		if (is_page_fault_stale(vcpu, &fault)) {
+			r = RET_PF_RETRY;
+			continue;
+		}
+
 		r = kvm_tdp_mmu_map(vcpu, &fault);
 	} while (r == RET_PF_RETRY);
 
@@ -5910,6 +5989,9 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu,
 		return role;
 	}
 
+	if (KVM_BUG_ON(____is_efer_lma(regs) && !____is_cr4_pae(regs), vcpu->kvm))
+		*(u64 *)&regs->efer &= ~EFER_LMA;
+
 	role.base.efer_nx = ____is_efer_nx(regs);
 	role.base.cr0_wp = ____is_cr0_wp(regs);
 	role.base.cr4_smep = ____is_cr4_smep(regs);
@@ -5953,19 +6035,22 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
 
 static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
 {
-	int maxpa;
-
-	if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM)
-		maxpa = cpuid_query_maxguestphyaddr(vcpu);
-	else
-		maxpa = cpuid_maxphyaddr(vcpu);
-
 	/* tdp_root_level is architecture forced level, use it if nonzero */
 	if (tdp_root_level)
 		return tdp_root_level;
 
+	/*
+	 * If the VM has mirror roots, then the root level is predefined as the
+	 * mirror root (and by extension the normal root) needs to match the
+	 * root level that was configured for the external page tables that are
+	 * being mirrored by KVM.
+	 */
+	if (kvm_has_mirrored_tdp(vcpu->kvm) &&
+	    !WARN_ON_ONCE(!vcpu->kvm->arch.mirror_root_level))
+		return vcpu->kvm->arch.mirror_root_level;
+
 	/* Use 5-level TDP if and only if it's useful/necessary. */
-	if (max_tdp_level == 5 && maxpa <= 48)
+	if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48)
 		return 4;
 
 	return max_tdp_level;
@@ -6584,13 +6669,13 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
 	if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) &&
 	    !(error_code & PFERR_RSVD_MASK) &&
 	    vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM &&
-	    kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)))
+	    kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)))
 		error_code |= PFERR_PRIVATE_ACCESS;
 
 	r = RET_PF_INVALID;
 	if (unlikely(error_code & PFERR_RSVD_MASK)) {
-		if (WARN_ON_ONCE(error_code & PFERR_PRIVATE_ACCESS))
-			return -EFAULT;
+		if (KVM_BUG_ON(error_code & PFERR_PRIVATE_ACCESS, vcpu->kvm))
+			return -EIO;
 
 		r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct);
 		if (r == RET_PF_EMULATE)
@@ -6632,6 +6717,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
 		return r;
 
 emulate:
+	if (error_code & PFERR_WRITE_MASK)
+		emulation_type |= EMULTYPE_PF_WRITE;
+
 	return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn,
 				       insn_len);
 }
@@ -6704,7 +6792,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
 		if (is_noncanonical_invlpg_address(addr, vcpu))
 			return;
 
-		kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL);
+		kvm_x86_call(flush_tlb_gva)(vcpu, addr);
 
 		if (tdp_enabled)
 			return;
@@ -6834,7 +6922,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct k
 	 * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit
 	 * KVM; that horror is handled on-demand by mmu_alloc_special_roots().
 	 */
-	if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL)
+	if (tdp_enabled && kvm_mmu_get_max_tdp_level() > PT32E_ROOT_LEVEL)
 		return 0;
 
 	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32);
@@ -7117,6 +7205,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
 	if (WARN_ON_ONCE(gfn_end <= gfn_start))
 		return;
 
+	guard(srcu)(&kvm->srcu);
+
 	write_lock(&kvm->mmu_lock);
 
 	kvm_mmu_invalidate_start(kvm);
@@ -7985,7 +8075,7 @@ static void kvm_recover_nx_huge_pages(struct kvm *kvm,
 		WARN_ON_ONCE(!sp->nx_huge_page_disallowed);
 		WARN_ON_ONCE(!sp->role.direct);
 
-		unaccount_nx_huge_page(kvm, sp);
+		unaccount_nx_huge_page(kvm, sp, mmu_type);
 
 		if (is_tdp_mmu)
 			spin_unlock(tdp_mmu_pages_lock);
@@ -8097,7 +8187,7 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
 		vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread);
 }
 
-#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
 static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
 				int level)
 {
@@ -8116,8 +8206,8 @@ static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
 	lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;
 }
 
-bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
-					struct kvm_gfn_range *range)
+bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm,
+					   struct kvm_gfn_range *range)
 {
 	struct kvm_memory_slot *slot = range->slot;
 	int level;
@@ -8186,18 +8276,18 @@ static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,
 	const unsigned long end = start + KVM_PAGES_PER_HPAGE(level);
 
 	if (level == PG_LEVEL_2M)
-		return kvm_range_has_memory_attributes(kvm, start, end, ~0, attrs);
+		return kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attrs);
 
 	for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) {
 		if (hugepage_test_mixed(slot, gfn, level - 1) ||
-		    attrs != kvm_get_memory_attributes(kvm, gfn))
+		    attrs != kvm_get_vm_memory_attributes(kvm, gfn))
 			return false;
 	}
 	return true;
 }
 
-bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
-					 struct kvm_gfn_range *range)
+bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm,
+					    struct kvm_gfn_range *range)
 {
 	unsigned long attrs = range->arg.attributes;
 	struct kvm_memory_slot *slot = range->slot;
@@ -8290,7 +8380,7 @@ void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
 		 * be manually checked as the attributes may already be mixed.
 		 */
 		for (gfn = start; gfn < end; gfn += nr_pages) {
-			unsigned long attrs = kvm_get_memory_attributes(kvm, gfn);
+			unsigned long attrs = kvm_get_vm_memory_attributes(kvm, gfn);
 
 			if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
 				hugepage_clear_mixed(slot, gfn, level);
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 27427e7f22fa0f..8e350095508c5e 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -368,13 +368,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	pte_access = ~0;
 
 	/*
-	 * Queue a page fault for injection if this assertion fails, as callers
-	 * assume that walker.fault contains sane info on a walk failure.  I.e.
-	 * avoid making the situation worse by inducing even worse badness
-	 * between when the assertion fails and when KVM kicks the vCPU out to
-	 * userspace (because the VM is bugged).
+	 * Queue a page fault for injection if any of the below assertions fail,
+	 * as callers assume that walker.fault contains sane info on a walk
+	 * failure.  I.e. avoid making the situation worse by inducing even
+	 * worse badness between when the assertion fails and when KVM kicks
+	 * the vCPU out to userspace (because the VM is bugged).
 	 */
-	if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm))
+	if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm) ||
+	    KVM_BUG_ON(walker->max_level > PT_MAX_FULL_LEVELS, vcpu->kvm))
 		goto error;
 
 	++walker->level;
@@ -391,7 +392,9 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		offset    = index * sizeof(pt_element_t);
 		pte_gpa   = gfn_to_gpa(table_gfn) + offset;
 
-		BUG_ON(walker->level < 1);
+		if (KVM_BUG_ON(walker->level < 1, vcpu->kvm))
+			goto error;
+
 		walker->table_gfn[walker->level - 1] = table_gfn;
 		walker->pte_gpa[walker->level - 1] = pte_gpa;
 
@@ -775,8 +778,8 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 					     fault->req_level >= it.level);
 	}
 
-	if (WARN_ON_ONCE(it.level != fault->goal_level))
-		return -EFAULT;
+	if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm))
+		return -EIO;
 
 	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access,
 			   base_gfn, fault->pfn, fault);
diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c
index dd3bb04878ca6f..37d5bed3529f55 100644
--- a/arch/x86/kvm/msrs.c
+++ b/arch/x86/kvm/msrs.c
@@ -1631,7 +1631,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		if (msr_info->host_initiated) {
 			kvm_synchronize_tsc(vcpu, &data);
 		} else if (!vcpu->arch.guest_tsc_protected) {
-			u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
+			u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) -
+				  vcpu->arch.l1_tsc_offset;
 			adjust_tsc_offset_guest(vcpu, adj);
 			vcpu->arch.ia32_tsc_adjust_msr += adj;
 		}
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index a7d60c8785cd46..eef2f792dc9e80 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -662,7 +662,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
 	DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX);
 	struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 	struct kvm_pmc *pmc;
-	int bit;
+	int bit, r;
 
 	bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX);
 
@@ -676,12 +676,14 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
 
 	kvm_for_each_pmc(pmu, pmc, bit, bitmap) {
 		/*
-		 * If reprogramming fails, e.g. due to contention, re-set the
-		 * reprogram bit, i.e. opportunistically try again on the next
-		 * PMU refresh.  Don't make a new request as doing so can stall
-		 * the guest if reprogramming repeatedly fails.
+		 * If reprogramming fails on a transient condition, e.g. due to
+		 * contention, re-set the reprogram bit, i.e. opportunistically
+		 * try again on the next PMU refresh.  Don't make a new request
+		 * as doing so can stall the guest if reprogramming repeatedly
+		 * fails.
 		 */
-		if (reprogram_counter(pmc))
+		r = reprogram_counter(pmc);
+		if (r == -EBUSY || r == -ENOMEM)
 			set_bit(pmc->idx, pmu->reprogram_pmi);
 	}
 
@@ -823,14 +825,6 @@ void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
 
 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 {
-	switch (msr) {
-	case MSR_CORE_PERF_GLOBAL_STATUS:
-	case MSR_CORE_PERF_GLOBAL_CTRL:
-	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
-		return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu));
-	default:
-		break;
-	}
 	return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) ||
 	       kvm_pmu_call(is_valid_msr)(vcpu, msr);
 }
diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c
index 8f66438989e474..fad31b59c622fa 100644
--- a/arch/x86/kvm/regs.c
+++ b/arch/x86/kvm/regs.c
@@ -440,9 +440,14 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4);
 
+static bool kvm_is_valid_cr8(unsigned long cr8)
+{
+	return !(cr8 & CR8_RESERVED_BITS);
+}
+
 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 {
-	if (cr8 & CR8_RESERVED_BITS)
+	if (!kvm_is_valid_cr8(cr8))
 		return 1;
 	if (lapic_in_kernel(vcpu))
 		kvm_lapic_set_tpr(vcpu, cr8);
@@ -565,6 +570,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 
 	return kvm_is_valid_cr4(vcpu, sregs->cr4) &&
 	       kvm_is_valid_cr0(vcpu, sregs->cr0) &&
+	       kvm_is_valid_cr8(sregs->cr8) &&
 	       kvm_valid_efer(vcpu, sregs->efer);
 }
 
diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h
index 2ad25781cefb0d..89345af0f7ad67 100644
--- a/arch/x86/kvm/reverse_cpuid.h
+++ b/arch/x86/kvm/reverse_cpuid.h
@@ -66,7 +66,7 @@
 #define X86_FEATURE_AVX10_512		KVM_X86_FEATURE(CPUID_24_0_EBX, 18)
 
 /* Intel-defined sub-features, CPUID level 0x00000024:1 (ECX) */
-#define X86_FEATURE_AVX10_VNNI_INT	KVM_X86_FEATURE(CPUID_24_1_ECX, 2)
+#define X86_FEATURE_AVX10_V1_AUX	KVM_X86_FEATURE(CPUID_24_1_ECX, 2)
 
 /* CPUID level 0x80000007 (EDX). */
 #define KVM_X86_FEATURE_CONSTANT_TSC	KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8)
diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c
index 3b037e3855234e..96ca39c0045f10 100644
--- a/arch/x86/kvm/svm/avic.c
+++ b/arch/x86/kvm/svm/avic.c
@@ -395,6 +395,11 @@ static phys_addr_t avic_get_backing_page_address(struct vcpu_svm *svm)
 	return __sme_set(__pa(svm->vcpu.arch.apic->regs));
 }
 
+static bool avic_is_addressable_vcpu(struct kvm_vcpu *vcpu)
+{
+	return vcpu->vcpu_id <= __avic_get_max_physical_id(vcpu->kvm, NULL);
+}
+
 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb)
 {
 	struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
@@ -412,7 +417,6 @@ void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb)
 
 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
 {
-	u32 max_id = x2avic_enabled ? x2avic_max_physical_id : AVIC_MAX_PHYSICAL_ID;
 	struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
 	struct vcpu_svm *svm = to_svm(vcpu);
 	u32 id = vcpu->vcpu_id;
@@ -425,7 +429,7 @@ static int avic_init_backing_page(struct kvm_vcpu *vcpu)
 	 * avic_vcpu_load() expects to be called if and only if the vCPU has
 	 * fully initialized AVIC.
 	 */
-	if (id > max_id) {
+	if (!avic_is_addressable_vcpu(vcpu)) {
 		kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_PHYSICAL_ID_TOO_BIG);
 		vcpu->arch.apic->apicv_active = false;
 		return 0;
@@ -885,6 +889,14 @@ int avic_init_vcpu(struct vcpu_svm *svm)
 	return ret;
 }
 
+void avic_vcpu_free(struct kvm_vcpu *vcpu)
+{
+	struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
+
+	if (kvm_svm->avic_physical_id_table && avic_is_addressable_vcpu(vcpu))
+		WRITE_ONCE(kvm_svm->avic_physical_id_table[vcpu->vcpu_id], 0);
+}
+
 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu)
 {
 	avic_handle_dfr_update(vcpu);
@@ -1045,8 +1057,7 @@ static void __avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu,
 	if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
 		return;
 
-	if (WARN_ON_ONCE(vcpu->vcpu_id * sizeof(entry) >=
-			 PAGE_SIZE << avic_get_physical_id_table_order(vcpu->kvm)))
+	if (WARN_ON_ONCE(!avic_is_addressable_vcpu(vcpu)))
 		return;
 
 	/*
@@ -1108,8 +1119,7 @@ static void __avic_vcpu_put(struct kvm_vcpu *vcpu, enum avic_vcpu_action action)
 
 	lockdep_assert_preemption_disabled();
 
-	if (WARN_ON_ONCE(vcpu->vcpu_id * sizeof(entry) >=
-			 PAGE_SIZE << avic_get_physical_id_table_order(vcpu->kvm)))
+	if (WARN_ON_ONCE(!avic_is_addressable_vcpu(vcpu)))
 		return;
 
 	/*
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 73f37b050d0a0b..b61149814acce1 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -789,6 +789,10 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm)
 
 	kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED);
 
+	/* SVM ignores EFER.LMA if EFER.LME=0 (instead of failing VMRUN). */
+	if (!(svm->nested.save.efer & EFER_LME))
+		svm->nested.save.efer &= ~EFER_LMA;
+
 	svm_set_efer(vcpu, svm->nested.save.efer);
 
 	svm_set_cr0(vcpu, svm->nested.save.cr0);
@@ -901,6 +905,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
 	vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa;
 	vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP);
 
+	/*
+	 * PML is never enabled in hardware for L2.  Make sure that an
+	 * unexpected PML write would trigger a PML_FULL VM-Exit.
+	 */
+	if (pml)
+		vmcb02->control.pml_index = -1;
+
 	/*
 	 * Stash vmcb02's counter if the guest hasn't moved past the guilty
 	 * instruction; otherwise, reset the counter to '0'.
@@ -1494,6 +1505,7 @@ int svm_allocate_nested(struct vcpu_svm *svm)
 	if (!svm->nested.msrpm)
 		goto err_free_vmcb02;
 
+	svm->nested.vmcb02.cpu = -1;
 	svm->nested.initialized = true;
 	return 0;
 
@@ -1820,6 +1832,13 @@ int nested_svm_exit_special(struct vcpu_svm *svm)
 		if (nested_svm_is_l2_tlb_flush_hcall(vcpu))
 			return NESTED_EXIT_HOST;
 		break;
+	case SVM_EXIT_PML_FULL:
+		/*
+		 * All PML full exits are handled by KVM.  KVM emulates PML in
+		 * software for L1, but never enables PML in hardware on behalf
+		 * of L1.
+		 */
+		return NESTED_EXIT_HOST;
 	default:
 		break;
 	}
@@ -2028,6 +2047,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 	if (!(save->cr0 & X86_CR0_PG) ||
 	    !(save->cr0 & X86_CR0_PE) ||
 	    (save->rflags & X86_EFLAGS_VM) ||
+	    ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) ||
 	    !nested_vmcb_check_save(vcpu, &save_cached, false))
 		goto out_free;
 
@@ -2125,13 +2145,8 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu)
 			return false;
 	}
 
-	if (!nested_svm_merge_msrpm(vcpu)) {
-		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
-		vcpu->run->internal.suberror =
-			KVM_INTERNAL_ERROR_EMULATION;
-		vcpu->run->internal.ndata = 0;
+	if (!nested_svm_merge_msrpm(vcpu))
 		return false;
-	}
 
 	if (kvm_hv_verify_vp_assist(vcpu))
 		return false;
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index 5705723f1f4121..5dc7357432ba69 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -488,6 +488,20 @@ static void snp_guest_req_cleanup(struct kvm *kvm)
 	sev->guest_resp_buf = NULL;
 }
 
+static int sev_alloc_have_run_cpus(struct kvm_sev_info *sev)
+{
+	if (!zalloc_cpumask_var(&sev->have_run_cpus, GFP_KERNEL_ACCOUNT))
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void sev_free_have_run_cpus(struct kvm_sev_info *sev)
+{
+	free_cpumask_var(sev->have_run_cpus);
+	memset(&sev->have_run_cpus, 0, sizeof(sev->have_run_cpus));
+}
+
 static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp,
 			    struct kvm_sev_init *data,
 			    unsigned long vm_type)
@@ -545,10 +559,9 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp,
 	if (ret)
 		goto e_free_asid;
 
-	if (!zalloc_cpumask_var(&sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) {
-		ret = -ENOMEM;
+	ret = sev_alloc_have_run_cpus(sev);
+	if (ret)
 		goto e_free_asid;
-	}
 
 	/* This needs to happen after SEV/SNP firmware initialization. */
 	if (snp_active) {
@@ -560,13 +573,14 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp,
 	INIT_LIST_HEAD(&sev->regions_list);
 	INIT_LIST_HEAD(&sev->mirror_vms);
 	sev->need_init = false;
+	kvm->arch.has_protected_page_tables = true;
 
 	kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV);
 
 	return 0;
 
 e_free:
-	free_cpumask_var(sev->have_run_cpus);
+	sev_free_have_run_cpus(sev);
 e_free_asid:
 	argp->error = init_args.error;
 	sev_asid_free(sev);
@@ -1125,9 +1139,6 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	if (!sev_es_guest(kvm))
 		return -ENOTTY;
 
-	if (kvm_is_vcpu_creation_in_progress(kvm))
-		return -EBUSY;
-
 	ret = kvm_lock_all_vcpus(kvm);
 	if (ret)
 		return ret;
@@ -2035,18 +2046,27 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm)
 	struct kvm_sev_info *mirror;
 	unsigned long i;
 
+	/*
+	 * Do cache maintenance on the source VM *before* clearing "SEV active",
+	 * as memory reclaim flows won't trigger cache maintenance on the VM
+	 * once it's no longer an SEV VM.
+	 */
+	sev_writeback_caches(src_kvm);
+
 	dst->active = true;
 	dst->asid = src->asid;
 	dst->handle = src->handle;
 	dst->pages_locked = src->pages_locked;
 	dst->es_active = src->es_active;
 	dst->vmsa_features = src->vmsa_features;
+	dst_kvm->arch.has_protected_page_tables = true;
 
 	src->asid = 0;
 	src->active = false;
 	src->handle = 0;
 	src->pages_locked = 0;
 	src->es_active = false;
+	src_kvm->arch.has_protected_page_tables = false;
 
 	list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list);
 
@@ -2115,10 +2135,6 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src)
 	struct kvm_vcpu *src_vcpu;
 	unsigned long i;
 
-	if (kvm_is_vcpu_creation_in_progress(src) ||
-	    kvm_is_vcpu_creation_in_progress(dst))
-		return -EBUSY;
-
 	if (!sev_es_guest(src))
 		return 0;
 
@@ -2187,11 +2203,14 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd)
 	 * the set of CPUs from the source.  If a CPU was used to run a vCPU in
 	 * the source VM but is never used for the destination VM, then the CPU
 	 * can only have cached memory that was accessible to the source VM.
+	 * Furthermore, KVM *must* perform cache maintenance on the source VM,
+	 * as the source VM may have access to memory that the destination VM
+	 * does not, i.e. KVM could skip flushes if memory is reclaimed from
+	 * the old VM but not the new VM.
 	 */
-	if (!zalloc_cpumask_var(&dst_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) {
-		ret = -ENOMEM;
+	ret = sev_alloc_have_run_cpus(dst_sev);
+	if (ret)
 		goto out_source_vcpu;
-	}
 
 	sev_migrate_from(kvm, source_kvm);
 	kvm_vm_dead(source_kvm);
@@ -2400,12 +2419,12 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 	 */
 	if (ret && !snp_page_reclaim(kvm, pfn) &&
 	    sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID &&
-	    sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) {
+	    sev_populate_args->fw_error == SEV_RET_INVALID_PARAM && src_page) {
 		void *src_vaddr = kmap_local_page(src_page);
 		void *dst_vaddr = kmap_local_pfn(pfn);
 
 		memcpy(src_vaddr, dst_vaddr, PAGE_SIZE);
-		set_page_dirty(src_page);
+		set_page_dirty_lock(src_page);
 
 		kunmap_local(dst_vaddr);
 		kunmap_local(src_vaddr);
@@ -2433,8 +2452,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	if (copy_from_user(&params, u64_to_user_ptr(argp->data), sizeof(params)))
 		return -EFAULT;
 
-	pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d\n", __func__,
-		 params.gfn_start, params.len, params.type, params.flags);
+	pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d src %llx\n", __func__,
+		 params.gfn_start, params.len, params.type, params.flags, params.uaddr);
 
 	if (!params.len || !PAGE_ALIGNED(params.len) || params.flags ||
 	    (params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL &&
@@ -2446,7 +2465,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
 
 	if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO)
 		src = NULL;
-	else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr))
+	else if ((!gmem_in_place_conversion && !params.uaddr) ||
+		 !PAGE_ALIGNED(params.uaddr))
 		return -EINVAL;
 	else
 		src = u64_to_user_ptr(params.uaddr);
@@ -2493,7 +2513,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
 
 	params.gfn_start += count;
 	params.len -= count * PAGE_SIZE;
-	if (params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO)
+	if (src && params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO)
 		params.uaddr += count * PAGE_SIZE;
 
 	if (copy_to_user(u64_to_user_ptr(argp->data), &params, sizeof(params)))
@@ -2510,9 +2530,6 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	unsigned long i;
 	int ret;
 
-	if (kvm_is_vcpu_creation_in_progress(kvm))
-		return -EBUSY;
-
 	ret = kvm_lock_all_vcpus(kvm);
 	if (ret)
 		return ret;
@@ -2888,10 +2905,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd)
 	}
 
 	mirror_sev = to_kvm_sev_info(kvm);
-	if (!zalloc_cpumask_var(&mirror_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) {
-		ret = -ENOMEM;
+	ret = sev_alloc_have_run_cpus(mirror_sev);
+	if (ret)
 		goto e_unlock;
-	}
 
 	/*
 	 * The mirror kvm holds an enc_context_owner ref so its asid can't
@@ -2907,6 +2923,7 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd)
 	mutex_unlock(&sev_mirror_lock);
 
 	mirror_sev->active = true;
+	kvm->arch.has_protected_page_tables = true;
 	mirror_sev->asid = source_sev->asid;
 	mirror_sev->fd = source_sev->fd;
 	mirror_sev->es_active = source_sev->es_active;
@@ -2965,6 +2982,7 @@ void sev_vm_init(struct kvm *kvm)
 		kvm->arch.has_protected_state = true;
 		fallthrough;
 	case KVM_X86_SEV_VM:
+		kvm->arch.has_protected_page_tables = true;
 		kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem;
 		to_kvm_sev_info(kvm)->need_init = true;
 		break;
@@ -2980,13 +2998,17 @@ void sev_vm_destroy(struct kvm *kvm)
 	struct list_head *head = &sev->regions_list;
 	struct list_head *pos, *q;
 
+	/*
+	 * Free the mask even if the VM is not *currently* an SEV VM, as it may
+	 * have been an SEV VM prior to intra-host migration.
+	 */
+	sev_free_have_run_cpus(sev);
+
 	if (!sev_guest(kvm))
 		return;
 
 	WARN_ON(!list_empty(&sev->mirror_vms));
 
-	free_cpumask_var(sev->have_run_cpus);
-
 	/*
 	 * If this is a mirror VM, remove it from the owner's list of a mirrors
 	 * and skip ASID cleanup (the ASID is tied to the lifetime of the owner).
@@ -3032,6 +3054,8 @@ void sev_vm_destroy(struct kvm *kvm)
 		 */
 		if (snp_decommission_context(kvm))
 			return;
+
+		snp_enable_rmpopt();
 	} else {
 		sev_unbind_asid(kvm, sev->handle);
 	}
@@ -3616,7 +3640,6 @@ int pre_sev_run(struct vcpu_svm *svm, int cpu)
 
 	sd->sev_vmcbs[asid] = svm->vmcb;
 	svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
-	vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
 	return 0;
 }
 
@@ -4032,7 +4055,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
 	struct kvm *kvm = vcpu->kvm;
 	gfn_t gfn = gpa_to_gfn(gpa);
 	unsigned long mmu_seq;
-	struct page *page;
 	kvm_pfn_t pfn;
 
 	lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex);
@@ -4076,7 +4098,7 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
 	 * The new VMSA will be private memory guest memory, so retrieve the
 	 * PFN from the gmem backend.
 	 */
-	if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL))
+	if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, NULL))
 		return;
 
 	read_lock(&kvm->mmu_lock);
@@ -4092,8 +4114,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
 	else
 		svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
 	read_unlock(&kvm->mmu_lock);
-
-	kvm_release_page_clean(page);
 }
 
 /*
@@ -4221,6 +4241,7 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_
 	struct kvm *kvm = svm->vcpu.kvm;
 	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
 	sev_ret_code fw_err = 0;
+	u8 tickle = 0;
 	int ret;
 
 	if (!is_sev_snp_guest(&svm->vcpu))
@@ -4228,8 +4249,21 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_
 
 	guard(mutex)(&sev->guest_req_mutex);
 
-	if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE))
-		return -EIO;
+	if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) {
+		svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
+		return 1;
+	}
+
+	/*
+	 * "Tickle" the response buffer to verify it's writable before sending
+	 * the request to firmware, which will modify the VMPCK sequence number.
+	 * Note, this is a best effort check and doesn't guard against TOCTOU
+	 * issues.  See below for more information.
+	 */
+	if (kvm_write_guest(kvm, resp_gpa, &tickle, sizeof(tickle))) {
+		svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
+		return 1;
+	}
 
 	data.gctx_paddr = __psp_pa(sev->snp_context);
 	data.req_paddr = __psp_pa(sev->guest_req_buf);
@@ -4244,6 +4278,13 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_
 	if (ret && !fw_err)
 		return ret;
 
+	/*
+	 * Exit to userspace if writing the response fails, e.g. if the mapping
+	 * changed between the initial tickle and the actual write, to avoid
+	 * creating an ambiguous failure ABI with the guest.  Don't return an
+	 * error to the guest because firmware already incremented the VMPCK
+	 * sequence number, but there's no way to communicate that to the guest.
+	 */
 	if (kvm_write_guest(kvm, resp_gpa, sev->guest_resp_buf, PAGE_SIZE))
 		return -EIO;
 
@@ -4295,8 +4336,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r
 		return -EINVAL;
 
 	if (kvm_read_guest(kvm, req_gpa + offsetof(struct snp_guest_msg_hdr, msg_type),
-			   &msg_type, 1))
-		return -EIO;
+			   &msg_type, 1)) {
+		svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
+		return 1;
+	}
 
 	/*
 	 * As per GHCB spec, requests of type MSG_REPORT_REQ also allow for
@@ -4335,8 +4378,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r
 		 * As per GHCB spec (see "SNP Extended Guest Request"), the
 		 * certificate table is terminated by 24-bytes of zeroes.
 		 */
-		if (data_npages && kvm_clear_guest(kvm, data_gpa, 24))
-			return -EIO;
+		if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) {
+			svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT);
+			return 1;
+		}
 	}
 
 	return snp_handle_guest_req(svm, req_gpa, resp_gpa);
@@ -4994,7 +5039,7 @@ struct page *snp_safe_alloc_page_node(int node, gfp_t gfp)
 	 * Allocate an SNP-safe page to workaround the SNP erratum where
 	 * the CPU will incorrectly signal an RMP violation #PF if a
 	 * hugepage (2MB or 1GB) collides with the RMP entry of a
-	 * 2MB-aligned VMCB, VMSA, or AVIC backing page.
+	 * 2MB-aligned VMCB, VMSA, PML or AVIC backing page.
 	 *
 	 * Allocate one extra page, choose a page which is not
 	 * 2MB-aligned, and free the other.
@@ -5019,7 +5064,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
 	struct kvm_memory_slot *slot;
 	struct kvm *kvm = vcpu->kvm;
 	int order, rmp_level, ret;
-	struct page *page;
+	unsigned long mmu_seq;
 	bool assigned;
 	kvm_pfn_t pfn;
 	gfn_t gfn;
@@ -5033,7 +5078,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
 	 * userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults
 	 * for shared pages should not end up here.
 	 */
-	if (!kvm_mem_is_private(kvm, gfn)) {
+	if (!kvm_is_private_gfn(kvm, gfn)) {
 		pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n",
 				    gpa);
 		return;
@@ -5046,7 +5091,10 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
 		return;
 	}
 
-	ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order);
+	mmu_seq = kvm->mmu_invalidate_seq;
+	smp_rmb();
+
+	ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order);
 	if (ret) {
 		pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n",
 				    gpa);
@@ -5055,9 +5103,13 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
 
 	ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level);
 	if (ret || !assigned) {
-		pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n",
-				    gpa, pfn, ret);
-		goto out_no_trace;
+		guard(read_lock)(&kvm->mmu_lock);
+
+		if (!mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
+			pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n",
+					    gpa, pfn, ret);
+
+		return;
 	}
 
 	/*
@@ -5085,26 +5137,31 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
 	if (rmp_level == PG_LEVEL_4K)
 		goto out;
 
-	ret = snp_rmptable_psmash(pfn);
-	if (ret) {
-		/*
-		 * Look it up again. If it's 4K now then the PSMASH may have
-		 * raced with another process and the issue has already resolved
-		 * itself.
-		 */
-		if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) &&
-		    assigned && rmp_level == PG_LEVEL_4K)
+	scoped_guard(read_lock, &kvm->mmu_lock) {
+		if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
 			goto out;
 
-		pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n",
-				    gpa, pfn, ret);
+		ret = snp_rmptable_psmash(pfn);
+		if (ret) {
+			/*
+			 * Look it up again. If it's 4K now then the PSMASH may
+			 * have raced with another process and the issue has
+			 * already resolved itself. If it's not assigned, then
+			 * this must have raced with another process that made
+			 * this page shared.
+			 */
+			if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) &&
+			    ((assigned && rmp_level == PG_LEVEL_4K) || !assigned))
+				goto out;
+
+			pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n",
+					    gpa, pfn, ret);
+		}
 	}
 
 	kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD);
 out:
 	trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret);
-out_no_trace:
-	kvm_release_page_unused(page);
 }
 
 static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end)
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 7d59d301e1e543..0eb1623052c1f4 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -179,6 +179,9 @@ module_param(vnmi, bool, 0444);
 
 module_param(enable_mediated_pmu, bool, 0444);
 
+bool __ro_after_init pml = true;
+module_param(pml, bool, 0444);
+
 static bool __ro_after_init svm_gp_erratum_intercept = true;
 
 static u8 rsm_ins_bytes[] = "\x0f\xaa";
@@ -673,16 +676,7 @@ static void clr_dr_intercepts(struct vcpu_svm *svm)
 
 static bool msr_write_intercepted(struct vcpu_svm *svm, u32 msr)
 {
-	/*
-	 * For non-nested case:
-	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
-	 * save it.
-	 *
-	 * For nested case:
-	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
-	 * save it.
-	 */
-	void *msrpm = is_guest_mode(&svm->vcpu) ? svm->nested.msrpm : svm->msrpm;
+	void *msrpm = __va(__sme_clr(svm->vmcb->control.msrpm_base_pa));
 
 	return svm_test_msr_bitmap_write(msrpm, msr);
 }
@@ -1267,6 +1261,23 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event)
 	if (vcpu->kvm->arch.bus_lock_detection_enabled)
 		svm_set_intercept(svm, INTERCEPT_BUSLOCK);
 
+	if (pml) {
+		/*
+		 * Populate the page address and index here, PML is enabled
+		 * when dirty logging is enabled on the memslot through
+		 * svm_update_cpu_dirty_logging()
+		 */
+		control->pml_addr = (u64)__sme_set(page_to_phys(vcpu->arch.pml_page));
+
+		/*
+		 * The VMCB control area remains valid after an intercepted SHUTDOWN
+		 * (AMD APM Rev 3.45+), so only initialize PML index on reset to avoid
+		 * discarding already-logged entries that haven't been flushed.
+		 */
+		if (!init_event)
+			control->pml_index = PML_HEAD_INDEX;
+	}
+
 	if (is_sev_guest(vcpu))
 		sev_init_vmcb(svm, init_event);
 
@@ -1326,9 +1337,15 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu)
 	if (!vmcb01_page)
 		goto out;
 
+	if (pml) {
+		vcpu->arch.pml_page = snp_safe_alloc_page();
+		if (!vcpu->arch.pml_page)
+			goto error_free_vmcb_page;
+	}
+
 	err = sev_vcpu_create(vcpu);
 	if (err)
-		goto error_free_vmcb_page;
+		goto error_free_pml_page;
 
 	err = avic_init_vcpu(svm);
 	if (err)
@@ -1337,7 +1354,7 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu)
 	svm->msrpm = svm_vcpu_alloc_msrpm();
 	if (!svm->msrpm) {
 		err = -ENOMEM;
-		goto error_free_sev;
+		goto error_free_avic;
 	}
 
 	svm->x2avic_msrs_intercepted = true;
@@ -1351,8 +1368,13 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu)
 
 	return 0;
 
+error_free_avic:
+	avic_vcpu_free(vcpu);
 error_free_sev:
 	sev_free_vcpu(vcpu);
+error_free_pml_page:
+	if (vcpu->arch.pml_page)
+		__free_page(vcpu->arch.pml_page);
 error_free_vmcb_page:
 	__free_page(vmcb01_page);
 out:
@@ -1365,11 +1387,16 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu)
 
 	WARN_ON_ONCE(!list_empty(&svm->ir_list));
 
+	avic_vcpu_free(vcpu);
+
 	svm_leave_nested(vcpu);
 	svm_free_nested(svm);
 
 	sev_free_vcpu(vcpu);
 
+	if (pml && vcpu->arch.pml_page)
+		__free_page(vcpu->arch.pml_page);
+
 	__free_page(__sme_pa_to_page(svm->vmcb01.pa));
 	svm_vcpu_free_msrpm(svm->msrpm);
 }
@@ -1902,8 +1929,7 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
 	if (sd->next_asid > sd->max_asid) {
 		++sd->asid_generation;
 		sd->next_asid = sd->min_asid;
-		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
-		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
+		sd->flush_all_asids = true;
 	}
 
 	svm->current_vmcb->asid_generation = sd->asid_generation;
@@ -3344,6 +3370,45 @@ static int vmmcall_interception(struct kvm_vcpu *vcpu)
 	return kvm_emulate_hypercall(vcpu);
 }
 
+static void svm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	struct vmcb *vmcb01 = svm->vmcb01.ptr;
+
+	if (enable)
+		vmcb01->control.misc_ctl |= SVM_MISC_ENABLE_PML;
+	else
+		vmcb01->control.misc_ctl &= ~SVM_MISC_ENABLE_PML;
+
+	vmcb_mark_dirty(vmcb01, VMCB_NPT);
+}
+
+static void svm_flush_pml_buffer(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	struct vmcb_control_area *control = &svm->vmcb->control;
+
+	/* Do nothing if PML buffer is empty */
+	if (control->pml_index == PML_HEAD_INDEX)
+		return;
+
+	kvm_flush_pml_buffer(vcpu, control->pml_index);
+
+	/* Reset the PML index */
+	control->pml_index = PML_HEAD_INDEX;
+}
+
+static int pml_full_interception(struct kvm_vcpu *vcpu)
+{
+	trace_kvm_pml_full(vcpu->vcpu_id);
+
+	/*
+	 * PML buffer is already flushed at the beginning of svm_handle_exit().
+	 * Nothing to do here.
+	 */
+	return 1;
+}
+
 static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[SVM_EXIT_READ_CR0]			= cr_interception,
 	[SVM_EXIT_READ_CR3]			= cr_interception,
@@ -3421,6 +3486,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 #ifdef CONFIG_KVM_AMD_SEV
 	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
 #endif
+	[SVM_EXIT_PML_FULL]			= pml_full_interception,
 };
 
 static void dump_vmcb(struct kvm_vcpu *vcpu)
@@ -3470,8 +3536,14 @@ static void dump_vmcb(struct kvm_vcpu *vcpu)
 	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
 	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
 	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
-	pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl);
+	pr_err("%-20s%llx\n", "misc_ctl:", control->misc_ctl);
 	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
+
+	if (pml) {
+		pr_err("%-20s%016llx\n", "pml_addr:", control->pml_addr);
+		pr_err("%-20s%04x\n", "pml_index:", control->pml_index);
+	}
+
 	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
 	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
 	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
@@ -3649,6 +3721,13 @@ int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code)
 	    (u64)exit_code != __exit_code)
 		goto unexpected_vmexit;
 
+	/*
+	 * PML is never enabled when running L2, bail immediately if a PML full
+	 * exit occurs as something is horribly wrong.
+	 */
+	if (unlikely(is_guest_mode(vcpu) && exit_code == SVM_EXIT_PML_FULL))
+		goto unexpected_vmexit;
+
 #ifdef CONFIG_MITIGATION_RETPOLINE
 	if (exit_code == SVM_EXIT_MSR)
 		return msr_interception(vcpu);
@@ -3717,6 +3796,14 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct kvm_run *kvm_run = vcpu->run;
 
+	/*
+	 * Opportunistically flush the PML buffer on VM exit. This keeps the
+	 * dirty bitmap current by processing logged GPAs rather than waiting for
+	 * PML_FULL exit.
+	 */
+	if (vcpu->kvm->arch.cpu_dirty_log_size && !is_guest_mode(vcpu))
+		svm_flush_pml_buffer(vcpu);
+
 	if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
 		return 0;
 
@@ -3765,14 +3852,25 @@ static int pre_svm_run(struct kvm_vcpu *vcpu)
 	struct vcpu_svm *svm = to_svm(vcpu);
 
 	/*
-	 * If the previous vmrun of the vmcb occurred on a different physical
-	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
-	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
+	 * If the previous VMRUN of the VMCB occurred on a different physical
+	 * cpu, then mark the VMCB dirty as hardware's clean bits are per pCPU.
+	 *
+	 * Reset the ASID generation in both VMCBs. This will lead to assigning
+	 * a new ASID now, and then again when switching to the other VMCB.
+	 * However, this is needed as the ASID is shared between the VMCBs, and
+	 * otherwise it would be possible to use an ASID allocated on one pCPU
+	 * on another, for example:
+	 * - vCPU migrates from pCPU A to pCPU B, allocates a new ASID.
+	 * - vCPU migrates back to pCPU A, and then switches the VMCB.
+	 * - The new VMCB does not detect a pCPU change and runs on pCPU A with
+	 *   the new ASID allocated on pCPU B, which is potentially used by
+	 *   another vCPU/VM.
 	 */
 	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
-		svm->current_vmcb->asid_generation = 0;
 		vmcb_mark_all_dirty(svm->vmcb);
 		svm->current_vmcb->cpu = vcpu->cpu;
+		svm->vmcb01.asid_generation = 0;
+		svm->nested.vmcb02.asid_generation = 0;
         }
 
 	if (is_sev_guest(vcpu))
@@ -4245,7 +4343,7 @@ static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu)
 	svm_flush_tlb_asid(vcpu);
 }
 
-static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full)
+static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -4261,8 +4359,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full)
 	}
 
 	svm_flush_tlb_guest(vcpu);
-	if (full)
-		*full = true;
 }
 
 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
@@ -4528,6 +4624,9 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 		svm->vmcb->control.asid = svm->asid;
 		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
 	}
+	if (this_cpu_ptr(&svm_data)->flush_all_asids)
+		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
+
 	svm->vmcb->save.cr2 = vcpu->arch.cr2;
 
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS) &&
@@ -4618,16 +4717,23 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 		vcpu->arch.nested_run_pending = 0;
 	}
 
-	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
+	if (!svm_is_vmrun_failure(svm->vmcb->control.exit_code)) {
+		this_cpu_ptr(&svm_data)->flush_all_asids = false;
+		svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
 
-	/*
-	 * Unconditionally mask off the CLEAR_RAP bit, the AND is just as cheap
-	 * as the TEST+Jcc to avoid it.
-	 */
-	if (cpu_feature_enabled(X86_FEATURE_ERAPS))
-		svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP;
+		/*
+		 * Unconditionally mask off the CLEAR_RAP bit, the AND is just
+		 * as cheap as the TEST+Jcc to avoid it.
+		 */
+		if (cpu_feature_enabled(X86_FEATURE_ERAPS))
+			svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP;
 
-	vmcb_mark_all_clean(svm->vmcb);
+		vmcb_mark_all_clean(svm->vmcb);
+
+		if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL))
+			rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
+			       vcpu_to_pmu(vcpu)->global_ctrl);
+	}
 
 	/* if exit due to PF check for async PF */
 	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
@@ -4636,9 +4742,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 	kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET);
 
-	if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL))
-		rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, vcpu_to_pmu(vcpu)->global_ctrl);
-
 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
 
 	svm_complete_interrupts(vcpu);
@@ -5311,6 +5414,9 @@ static int svm_vm_init(struct kvm *kvm)
 	if (!pause_filter_count || !pause_filter_thresh)
 		kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE);
 
+	if (pml)
+		kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES;
+
 	svm_srso_vm_init();
 	return 0;
 }
@@ -5429,6 +5535,8 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
 	.check_intercept = svm_check_intercept,
 	.handle_exit_irqoff = svm_handle_exit_irqoff,
 
+	.update_cpu_dirty_logging = svm_update_cpu_dirty_logging,
+
 	.deliver_interrupt = svm_deliver_interrupt,
 	.pi_update_irte = avic_pi_update_irte,
 	.setup_mce = svm_setup_mce,
@@ -5620,7 +5728,8 @@ static __init int svm_hardware_setup(void)
 				     XFEATURE_MASK_BNDCSR);
 
 	if (tsc_scaling) {
-		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
+		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR) ||
+		    !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
 			tsc_scaling = false;
 		} else {
 			pr_info("TSC scaling supported\n");
@@ -5689,6 +5798,10 @@ static __init int svm_hardware_setup(void)
 
 	nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
 
+	pml = pml && npt_enabled && cpu_feature_enabled(X86_FEATURE_PML);
+	if (pml)
+		pr_info("Page modification logging supported\n");
+
 	if (lbrv) {
 		if (!boot_cpu_has(X86_FEATURE_LBRV))
 			lbrv = false;
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index e958943b81627e..ac6160d78e15a2 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -53,6 +53,7 @@ extern int vgif;
 extern bool intercept_smi;
 extern bool vnmi;
 extern int lbrv;
+extern bool pml;
 
 extern int tsc_aux_uret_slot __ro_after_init;
 
@@ -376,6 +377,7 @@ struct svm_cpu_data {
 	u32 next_asid;
 	u32 min_asid;
 
+	bool flush_all_asids;
 	bool bp_spec_reduce_set;
 
 	struct vmcb *save_area;
@@ -954,6 +956,7 @@ void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb);
 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu);
 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu);
 int avic_init_vcpu(struct vcpu_svm *svm);
+void avic_vcpu_free(struct kvm_vcpu *vcpu);
 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
 void avic_vcpu_put(struct kvm_vcpu *vcpu);
 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h
index 08005676702c2b..b2905335a2d21e 100644
--- a/arch/x86/kvm/vmx/common.h
+++ b/arch/x86/kvm/vmx/common.h
@@ -2,10 +2,13 @@
 #ifndef __KVM_X86_VMX_COMMON_H
 #define __KVM_X86_VMX_COMMON_H
 
+#include <linux/container_of.h>
 #include <linux/kvm_host.h>
 #include <asm/posted_intr.h>
+#include <asm/vmx.h>
 
 #include "mmu.h"
+#include "vmx_ops.h"
 
 union vmx_exit_reason {
 	struct {
@@ -55,6 +58,8 @@ struct vcpu_vt {
 #endif
 };
 
+noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu);
+
 #ifdef CONFIG_KVM_INTEL_TDX
 
 static __always_inline bool is_td(struct kvm *kvm)
@@ -74,14 +79,62 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; }
 
 #endif
 
+struct vcpu_vmx_tdx {
+	struct kvm_vcpu vcpu;
+	struct vcpu_vt vt;
+};
+
+static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu)
+{
+	return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt);
+}
+
+static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt)
+{
+	return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu);
+}
+
+static __always_inline union vmx_exit_reason vt_get_exit_reason(struct kvm_vcpu *vcpu)
+{
+	return to_vt(vcpu)->exit_reason;
+}
+
+static __always_inline unsigned long vt_get_exit_qual(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_vt *vt = to_vt(vcpu);
+
+	if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) &&
+	    !WARN_ON_ONCE(is_td_vcpu(vcpu)))
+		vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
+
+	return vt->exit_qualification;
+}
+
+static __always_inline u32 vt_get_intr_info(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_vt *vt = to_vt(vcpu);
+
+	if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) &&
+	    !WARN_ON_ONCE(is_td_vcpu(vcpu)))
+		vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
+
+	return vt->exit_intr_info;
+}
+
+static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.guest_fpu.fpstate->xfd &&
+	       !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS);
+}
+
 static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa)
 {
 	/* For TDX the direct mask is the shared mask. */
 	return !kvm_is_addr_direct(kvm, gpa);
 }
 
-static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa,
-					     unsigned long exit_qualification)
+static inline int __vt_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa,
+					    unsigned long exit_qualification)
 {
 	u64 error_code;
 
@@ -164,8 +217,8 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
  * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the
  * interrupt if necessary.
  */
-static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
-						  struct pi_desc *pi_desc, int vector)
+static inline void __vt_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
+						 struct pi_desc *pi_desc, int vector)
 {
 	if (pi_test_and_set_pir(vector, pi_desc))
 		return;
@@ -183,6 +236,33 @@ static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu,
 	kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR);
 }
 
-noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu);
+static inline int __vt_handle_notify(struct kvm_vcpu *vcpu,
+				     unsigned long exit_qual)
+{
+	bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID;
+
+	++vcpu->stat.notify_window_exits;
+
+	if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER ||
+	    context_invalid) {
+		vcpu->run->exit_reason = KVM_EXIT_NOTIFY;
+		vcpu->run->notify.flags = context_invalid ?
+					  KVM_NOTIFY_CONTEXT_INVALID : 0;
+		return 0;
+	}
+
+	return 1;
+}
+
+static inline int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK
+	 * VM-Exits. Unconditionally set the flag here and leave the handling
+	 * to .handle_exit() callback.
+	 */
+	to_vt(vcpu)->exit_reason.bus_lock_detected = true;
+	return 1;
+}
 
 #endif /* __KVM_X86_VMX_COMMON_H */
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index 4c52ab8d07869e..d75170a6199eba 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+#include <linux/entry-common.h>
 #include <linux/moduleparam.h>
 
 #include "x86_ops.h"
@@ -10,8 +11,12 @@
 #include "tdx.h"
 #include "tdx_arch.h"
 
+#pragma GCC poison to_vmx
+
+static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_vmx, vt));
+
 #ifdef CONFIG_KVM_INTEL_TDX
-static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt));
+static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_tdx, vt));
 
 static void vt_disable_virtualization_cpu(void)
 {
@@ -108,7 +113,7 @@ static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 	vmx_vcpu_load(vcpu, cpu);
 }
 
-static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
+static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable)
 {
 	/*
 	 * Basic TDX does not support feature PML. KVM does not enable PML in
@@ -117,7 +122,7 @@ static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
 	if (WARN_ON_ONCE(is_td_vcpu(vcpu)))
 		return;
 
-	vmx_update_cpu_dirty_logging(vcpu);
+	vmx_update_cpu_dirty_logging(vcpu, enable);
 }
 
 static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
@@ -154,15 +159,6 @@ static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	return vmx_vcpu_run(vcpu, run_flags);
 }
 
-static int vt_handle_exit(struct kvm_vcpu *vcpu,
-			  enum exit_fastpath_completion fastpath)
-{
-	if (is_td_vcpu(vcpu))
-		return tdx_handle_exit(vcpu, fastpath);
-
-	return vmx_handle_exit(vcpu, fastpath);
-}
-
 static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu)
 {
 	if (is_td_vcpu(vcpu)) {
@@ -543,12 +539,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu)
 	vmx_flush_tlb_current(vcpu);
 }
 
-static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full)
+static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
 {
 	if (is_td_vcpu(vcpu))
 		return;
 
-	vmx_flush_tlb_gva(vcpu, addr, full);
+	vmx_flush_tlb_gva(vcpu, addr);
 }
 
 static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu)
@@ -601,6 +597,18 @@ static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
 	vmx_set_nmi_mask(vcpu, masked);
 }
 
+static void vt_clear_hlt(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * TDX doesn't support disabling HLT-exiting, and KVM can't access a
+	 * TD's VMCS, so there is never any hardware halted state to clear.
+	 */
+	if (is_td_vcpu(vcpu))
+		return;
+
+	vmx_clear_hlt(vcpu);
+}
+
 static void vt_enable_nmi_window(struct kvm_vcpu *vcpu)
 {
 	/* Refer to the comments in tdx_inject_nmi(). */
@@ -807,7 +815,6 @@ static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu)
 	vmx_write_tsc_multiplier(vcpu);
 }
 
-#ifdef CONFIG_X86_64
 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
 			      bool *expired)
 {
@@ -826,7 +833,6 @@ static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu)
 
 	vmx_cancel_hv_timer(vcpu);
 }
-#endif
 
 static void vt_setup_mce(struct kvm_vcpu *vcpu)
 {
@@ -876,6 +882,112 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn,
 #define vt_op_tdx_only(name) NULL
 #endif /* CONFIG_KVM_INTEL_TDX */
 
+noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu)
+{
+	if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI ||
+	    !is_nmi(vt_get_intr_info(vcpu)))
+		return;
+
+	kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
+	x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
+	kvm_after_interrupt(vcpu);
+}
+
+static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * Save xfd_err to guest_fpu before interrupt is enabled, so the
+	 * MSR value is not clobbered by the host activity before the guest
+	 * has chance to consume it.
+	 *
+	 * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM
+	 * interception may have been caused by L1 interception.  Per the SDM,
+	 * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1.
+	 *
+	 * Note, XFD_ERR is updated _before_ the #NM interception check, i.e.
+	 * unlike CR2 and DR6, the value is not a payload that is attached to
+	 * the #NM exception.
+	 */
+	if (is_xfd_nm_fault(vcpu))
+		rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
+}
+
+static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
+{
+	/* if exit due to PF check for async PF */
+	if (is_page_fault(intr_info))
+		vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
+	/* if exit due to NM, handle before interrupts are enabled */
+	else if (is_nm_fault(intr_info))
+		handle_nm_fault_irqoff(vcpu);
+	/* Handle machine checks before interrupts are enabled */
+	else if (is_machine_check(intr_info))
+		kvm_machine_check();
+}
+
+static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
+					     u32 intr_info)
+{
+	unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK;
+
+	if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm,
+	    "unexpected VM-Exit interrupt info: 0x%x", intr_info))
+		return;
+
+	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
+	x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
+	kvm_after_interrupt(vcpu);
+
+	vcpu->arch.at_instruction_boundary = true;
+}
+
+static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu)
+{
+	if (to_vt(vcpu)->emulation_required)
+		return;
+
+	switch (vt_get_exit_reason(vcpu).basic) {
+	case EXIT_REASON_EXTERNAL_INTERRUPT:
+		handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu));
+		break;
+	case EXIT_REASON_EXCEPTION_NMI:
+		handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu));
+		break;
+	case EXIT_REASON_MCE_DURING_VMENTRY:
+		kvm_machine_check();
+		break;
+	default:
+		break;
+	}
+}
+
+static int vt_handle_exit(struct kvm_vcpu *vcpu,
+			  enum exit_fastpath_completion fastpath)
+{
+	int ret;
+
+#ifdef CONFIG_KVM_INTEL_TDX
+	if (is_td_vcpu(vcpu))
+		ret = tdx_handle_exit(vcpu, fastpath);
+	else
+#endif
+		ret = vmx_handle_exit(vcpu, fastpath);
+
+	/*
+	 * Exit to user space when bus lock detected to inform that there is
+	 * a bus lock in guest.
+	 */
+	if (vt_get_exit_reason(vcpu).bus_lock_detected) {
+		if (ret > 0) {
+			vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
+			ret = 0;
+		}
+
+		vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK;
+	}
+	return ret;
+}
+
 #define VMX_REQUIRED_APICV_INHIBITS				\
 	(BIT(APICV_INHIBIT_REASON_DISABLED) |			\
 	 BIT(APICV_INHIBIT_REASON_ABSENT) |			\
@@ -949,7 +1061,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 
 	.vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization),
 	.vcpu_run = vt_op(vcpu_run),
-	.handle_exit = vt_op(handle_exit),
+	.handle_exit = vt_handle_exit,
 	.skip_emulated_instruction = vmx_skip_emulated_instruction,
 	.update_emulated_instruction = vmx_update_emulated_instruction,
 	.unhandleable_emulation_required = vt_op(unhandleable_emulation_required),
@@ -964,6 +1076,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 	.nmi_allowed = vt_op(nmi_allowed),
 	.get_nmi_mask = vt_op(get_nmi_mask),
 	.set_nmi_mask = vt_op(set_nmi_mask),
+	.clear_hlt = vt_op(clear_hlt),
 	.enable_nmi_window = vt_op(enable_nmi_window),
 	.enable_irq_window = vt_op(enable_irq_window),
 	.update_cr8_intercept = vt_op(update_cr8_intercept),
@@ -1000,12 +1113,12 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 	.load_mmu_pgd = vt_op(load_mmu_pgd),
 
 	.check_intercept = vmx_check_intercept,
-	.handle_exit_irqoff = vmx_handle_exit_irqoff,
+	.handle_exit_irqoff = vt_handle_exit_irqoff,
 
 	.update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging),
 
-	.pi_update_irte = vmx_pi_update_irte,
-	.pi_start_bypass = vmx_pi_start_bypass,
+	.pi_update_irte = vt_pi_update_irte,
+	.pi_start_bypass = vt_pi_start_bypass,
 
 #ifdef CONFIG_X86_64
 	.set_hv_timer = vt_op(set_hv_timer),
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 151873407abd36..9b0bfa2f854cf9 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -446,7 +446,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
 		 * "NMI unblocking due to IRET", i.e. the bit can be propagated
 		 * as-is from the original EXIT_QUALIFICATION.
 		 */
-		exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI;
+		exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI;
 	} else {
 		if (fault->error_code & PFERR_RSVD_MASK) {
 			vm_exit_reason = EXIT_REASON_EPT_MISCONFIG;
@@ -472,7 +472,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
 			 * and walks the faulting GPA.
 			 */
 			if (from_hardware)
-				exit_qualification |= vmx_get_exit_qual(vcpu) & mask;
+				exit_qualification |= vt_get_exit_qual(vcpu) & mask;
 			else
 				exit_qualification |= fault->exit_qualification & mask;
 
@@ -1754,6 +1754,9 @@ static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
 static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields)
 {
 #ifdef CONFIG_KVM_HYPERV
+	const u64 runtime_controls = HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1 |
+				     HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2 |
+				     HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC;
 	struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
 	struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx);
 	struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu);
@@ -1762,6 +1765,9 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields
 	vmcs12->tpr_threshold = evmcs->tpr_threshold;
 	vmcs12->guest_rip = evmcs->guest_rip;
 
+	if ((hv_clean_fields & runtime_controls) != runtime_controls)
+		vmx->nested.force_msr_bitmap_recalc = true;
+
 	if (unlikely(!(hv_clean_fields &
 		       HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) {
 		hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page;
@@ -3465,10 +3471,6 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu)
 		} else {
 			pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n",
 					     __func__);
-			vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
-			vcpu->run->internal.suberror =
-				KVM_INTERNAL_ERROR_EMULATION;
-			vcpu->run->internal.ndata = 0;
 			return false;
 		}
 	}
@@ -3539,11 +3541,6 @@ static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu)
 	if (!nested_get_evmcs_page(vcpu)) {
 		pr_debug_ratelimited("%s: enlightened vmptrld failed\n",
 				     __func__);
-		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
-		vcpu->run->internal.suberror =
-			KVM_INTERNAL_ERROR_EMULATION;
-		vcpu->run->internal.ndata = 0;
-
 		return false;
 	}
 #endif
@@ -3915,8 +3912,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 
 vmentry_failed:
 	vcpu->arch.nested_run_pending = 0;
-	if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR)
+	if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) {
+		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
+		vcpu->run->internal.ndata = 0;
 		return 0;
+	}
 	if (status == NVMX_VMENTRY_VMEXIT)
 		return 1;
 	WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL);
@@ -4747,7 +4748,7 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 {
 	/* update exit information fields: */
 	vmcs12->vm_exit_reason = vm_exit_reason;
-	if (vmx_get_exit_reason(vcpu).enclave_mode)
+	if (vt_get_exit_reason(vcpu).enclave_mode)
 		vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE;
 	vmcs12->exit_qualification = exit_qualification;
 
@@ -5373,7 +5374,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer,
 	struct x86_exception e;
 	int r;
 
-	if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
+	if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
 				vmcs_read32(VMX_INSTRUCTION_INFO), false,
 				sizeof(*vmpointer), &gva)) {
 		*ret = 1;
@@ -5665,7 +5666,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu)
 {
 	struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
 						    : get_vmcs12(vcpu);
-	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
+	unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 	u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct x86_exception e;
@@ -5771,7 +5772,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
 {
 	struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
 						    : get_vmcs12(vcpu);
-	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
+	unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 	u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct x86_exception e;
@@ -5897,6 +5898,12 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 
+	/* Forbid normal VMPTRLD if Enlightened version was used */
+	if (nested_vmx_is_evmptr12_valid(vmx)) {
+		kvm_queue_exception(vcpu, UD_VECTOR);
+		return 1;
+	}
+
 	if (nested_vmx_get_vmptr(vcpu, &vmptr, &r))
 		return r;
 
@@ -5906,10 +5913,6 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 	if (vmptr == vmx->nested.vmxon_ptr)
 		return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER);
 
-	/* Forbid normal VMPTRLD if Enlightened version was used */
-	if (nested_vmx_is_evmptr12_valid(vmx))
-		return 1;
-
 	if (vmx->nested.current_vmptr != vmptr) {
 		struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache;
 		struct vmcs_hdr hdr;
@@ -5960,9 +5963,9 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
 /* Emulate the VMPTRST instruction */
 static int handle_vmptrst(struct kvm_vcpu *vcpu)
 {
-	unsigned long exit_qual = vmx_get_exit_qual(vcpu);
+	unsigned long exit_qual = vt_get_exit_qual(vcpu);
 	u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
-	gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
+	gpa_t current_vmptr;
 	struct x86_exception e;
 	gva_t gva;
 	int r;
@@ -5970,8 +5973,16 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu)
 	if (!nested_vmx_check_permission(vcpu))
 		return 1;
 
-	if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu))))
-		return 1;
+	/*
+	 * Hyper-V TLFS does not specify the behavior of VMPTRST when eVMCS is used
+	 * but genuine Hyper-V seems to be returning eVMCS GPA.
+	 */
+#ifdef CONFIG_KVM_HYPERV
+	if (nested_vmx_is_evmptr12_valid(to_vmx(vcpu)))
+		current_vmptr = to_vmx(vcpu)->nested.hv_evmcs_vmptr;
+	else
+#endif
+		current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
 
 	if (get_vmx_mem_address(vcpu, exit_qual, instr_info,
 				true, sizeof(gpa_t), &gva))
@@ -6021,7 +6032,7 @@ static int handle_invept(struct kvm_vcpu *vcpu)
 	/* According to the Intel VMX instruction reference, the memory
 	 * operand is read even if it isn't needed (e.g., for type==global)
 	 */
-	if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
+	if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
 			vmx_instruction_info, false, sizeof(operand), &gva))
 		return 1;
 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
@@ -6104,7 +6115,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 	/* according to the intel vmx instruction reference, the memory
 	 * operand is read even if it isn't needed (e.g., for type==global)
 	 */
-	if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
+	if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
 			vmx_instruction_info, false, sizeof(operand), &gva))
 		return 1;
 	r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
@@ -6236,8 +6247,8 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu)
 	 * EXIT_REASON_VMFUNC as the exit reason.
 	 */
 	nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full,
-			  vmx_get_intr_info(vcpu),
-			  vmx_get_exit_qual(vcpu));
+			  vt_get_intr_info(vcpu),
+			  vt_get_exit_qual(vcpu));
 	return 1;
 }
 
@@ -6288,7 +6299,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
 	if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
 		return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
 
-	exit_qualification = vmx_get_exit_qual(vcpu);
+	exit_qualification = vt_get_exit_qual(vcpu);
 
 	port = exit_qualification >> 16;
 	size = (exit_qualification & 7) + 1;
@@ -6314,7 +6325,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
 
 	if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM ||
 	    exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM)
-		msr_index = vmx_get_exit_qual(vcpu);
+		msr_index = vt_get_exit_qual(vcpu);
 	else
 		msr_index = kvm_ecx_read(vcpu);
 
@@ -6350,7 +6361,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
 	struct vmcs12 *vmcs12)
 {
-	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
+	unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 	int cr = exit_qualification & 15;
 	int reg;
 	unsigned long val;
@@ -6484,7 +6495,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
 
 	switch ((u16)exit_reason.basic) {
 	case EXIT_REASON_EXCEPTION_NMI:
-		intr_info = vmx_get_intr_info(vcpu);
+		intr_info = vt_get_intr_info(vcpu);
 		if (is_nmi(intr_info))
 			return true;
 		else if (is_page_fault(intr_info))
@@ -6567,7 +6578,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu,
 
 	switch ((u16)exit_reason.basic) {
 	case EXIT_REASON_EXCEPTION_NMI:
-		intr_info = vmx_get_intr_info(vcpu);
+		intr_info = vt_get_intr_info(vcpu);
 		if (is_nmi(intr_info))
 			return true;
 		else if (is_page_fault(intr_info))
@@ -6737,14 +6748,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)
 	 * need to be synthesized by querying the in-kernel LAPIC, but external
 	 * interrupts are never reflected to L1 so it's a non-issue.
 	 */
-	exit_intr_info = vmx_get_intr_info(vcpu);
+	exit_intr_info = vt_get_intr_info(vcpu);
 	if (is_exception_with_error_code(exit_intr_info)) {
 		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 
 		vmcs12->vm_exit_intr_error_code =
 			vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
 	}
-	exit_qual = vmx_get_exit_qual(vcpu);
+	exit_qual = vt_get_exit_qual(vcpu);
 
 reflect_vmexit:
 	nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual);
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index bfa8612fb4508f..fb7ad53855f1db 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -187,6 +187,9 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
 	int ret;
 
 	switch (msr) {
+	case MSR_CORE_PERF_GLOBAL_STATUS:
+	case MSR_CORE_PERF_GLOBAL_CTRL:
+	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
 	case MSR_CORE_PERF_FIXED_CTR_CTRL:
 		return kvm_pmu_has_perf_global_ctrl(pmu);
 	case MSR_IA32_PEBS_ENABLE:
@@ -747,24 +750,36 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu)
 		intel_pmu_release_guest_lbr_event(vcpu);
 }
 
-void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu)
+u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu)
 {
-	struct kvm_pmc *pmc = NULL;
+	u64 guest_pebs_enable = pmu->pebs_enable & pmu->global_ctrl;
+	u64 pebs_enable = 0;
+	struct kvm_pmc *pmc;
 	int bit, hw_idx;
 
-	kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) {
-		if (!pmc_is_locally_enabled(pmc) ||
-		    !pmc_is_globally_enabled(pmc) || !pmc->perf_event)
+	/*
+	 * Omit counters that are locally disabled, don't have a perf event, or
+	 * ended up with a perf event that is using a different counter than
+	 * the guest, i.e. where the guest PMC is different than the host PMC
+	 * being used on behalf of the guest.  PEBS records include
+	 * PERF_GLOBAL_STATUS, and so using a counter with a different index
+	 * means the guest will see overflow status for the wrong counter(s).
+	 */
+	kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&guest_pebs_enable) {
+		if (!pmc_is_locally_enabled(pmc) || !pmc->perf_event)
 			continue;
 
 		/*
-		 * A negative index indicates the event isn't mapped to a
+		 * Note, a negative index indicates the event isn't mapped to a
 		 * physical counter in the host, e.g. due to contention.
 		 */
 		hw_idx = pmc->perf_event->hw.idx;
-		if (hw_idx != pmc->idx && hw_idx > -1)
-			pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx);
+		if (hw_idx != pmc->idx)
+			continue;
+
+		pebs_enable |= BIT_ULL(pmc->idx);
 	}
+	return pebs_enable;
 }
 
 static bool intel_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pmu)
diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c
index 4a6d9a17da238c..2644add54b76bc 100644
--- a/arch/x86/kvm/vmx/posted_intr.c
+++ b/arch/x86/kvm/vmx/posted_intr.c
@@ -14,6 +14,8 @@
 #include "vmx.h"
 #include "tdx.h"
 
+#pragma GCC poison to_vmx
+
 /*
  * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler()
  * when a WAKEUP_VECTOR interrupted is posted.  vCPUs are added to the list when
@@ -54,7 +56,7 @@ static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new)
 	return 0;
 }
 
-void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
+void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
 	struct vcpu_vt *vt = to_vt(vcpu);
@@ -144,11 +146,11 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
 		pi_set_on(pi_desc);
 }
 
-static bool vmx_can_use_vtd_pi(struct kvm *kvm)
+static bool vt_can_use_vtd_pi(struct kvm *kvm)
 {
 	/*
 	 * Note, reading the number of possible bypass IRQs can race with a
-	 * bypass IRQ being attached to the VM.  vmx_pi_start_bypass() ensures
+	 * bypass IRQ being attached to the VM.  vt_pi_start_bypass() ensures
 	 * blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK.
 	 */
 	return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() &&
@@ -206,7 +208,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu)
 		__apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR);
 }
 
-static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu)
+static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu)
 {
 	/*
 	 * The default posted interrupt vector does nothing when
@@ -217,14 +219,14 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu)
 	 * back to the pi_wakeup_handler() function.
 	 */
 	return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) ||
-		vmx_can_use_vtd_pi(vcpu->kvm);
+		vt_can_use_vtd_pi(vcpu->kvm);
 }
 
-void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
+void vt_vcpu_pi_put(struct kvm_vcpu *vcpu)
 {
 	struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
 
-	if (!vmx_needs_pi_wakeup(vcpu))
+	if (!vt_needs_pi_wakeup(vcpu))
 		return;
 
 	/*
@@ -292,17 +294,17 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu)
 /*
  * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as
  * blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup
- * vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put().
+ * vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put().
  */
-void vmx_pi_start_bypass(struct kvm *kvm)
+void vt_pi_start_bypass(struct kvm *kvm)
 {
-	if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm)))
+	if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm)))
 		return;
 
 	kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK);
 }
 
-int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
+int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
 		       unsigned int host_irq, uint32_t guest_irq,
 		       struct kvm_vcpu *vcpu, u32 vector)
 {
diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h
index a4af39948cf040..c5860ea828ae97 100644
--- a/arch/x86/kvm/vmx/posted_intr.h
+++ b/arch/x86/kvm/vmx/posted_intr.h
@@ -8,16 +8,16 @@
 
 #include <asm/posted_intr.h>
 
-void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
-void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu);
+void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu);
+void vt_vcpu_pi_put(struct kvm_vcpu *vcpu);
 void pi_wakeup_handler(void);
 void __init pi_init_cpu(int cpu);
 void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
 bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu);
-int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
+int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm,
 		       unsigned int host_irq, uint32_t guest_irq,
 		       struct kvm_vcpu *vcpu, u32 vector);
-void vmx_pi_start_bypass(struct kvm *kvm);
+void vt_pi_start_bypass(struct kvm *kvm);
 
 static inline int pi_find_highest_vector(struct pi_desc *pi_desc)
 {
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index b272c20586a744..8a593159c9b7cc 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -56,9 +56,6 @@
 bool enable_tdx __ro_after_init;
 module_param_named(tdx, enable_tdx, bool, 0444);
 
-#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51))
-#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47))
-
 static const struct tdx_sys_info *tdx_sysinfo;
 
 void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err)
@@ -362,7 +359,7 @@ static void tdx_reclaim_control_page(struct page *ctrl_page)
 	if (tdx_reclaim_page(ctrl_page))
 		return;
 
-	__free_page(ctrl_page);
+	tdx_free_control_page(ctrl_page);
 }
 
 struct tdx_flush_vp_arg {
@@ -589,7 +586,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm)
 
 	tdx_quirk_reset_paddr(page_to_phys(kvm_tdx->td.tdr_page), PAGE_SIZE);
 
-	__free_page(kvm_tdx->td.tdr_page);
+	tdx_free_control_page(kvm_tdx->td.tdr_page);
 	kvm_tdx->td.tdr_page = NULL;
 }
 
@@ -619,6 +616,7 @@ int tdx_vm_init(struct kvm *kvm)
 {
 	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
 
+	kvm->arch.has_protected_page_tables = true;
 	kvm->arch.has_protected_state = true;
 	/*
 	 * TDX Module doesn't allow the hypervisor to modify the EOI-bitmap,
@@ -681,6 +679,8 @@ int tdx_vcpu_create(struct kvm_vcpu *vcpu)
 	if (!irqchip_split(vcpu->kvm))
 		return -EINVAL;
 
+	tdx_init_pamt_cache(&tdx->pamt_cache);
+
 	fpstate_set_confidential(&vcpu->arch.guest_fpu);
 	vcpu->arch.apic->guest_apic_protected = true;
 	INIT_LIST_HEAD(&tdx->vt.pi_wakeup_list);
@@ -716,7 +716,7 @@ void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	struct vcpu_tdx *tdx = to_tdx(vcpu);
 
-	vmx_vcpu_pi_load(vcpu, cpu);
+	vt_vcpu_pi_load(vcpu, cpu);
 	if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm)))
 		return;
 
@@ -741,7 +741,7 @@ bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu)
 	 * interrupt is always allowed unless TDX guest calls TDVMCALL with HLT,
 	 * which passes the interrupt blocked flag.
 	 */
-	return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
+	return vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
 	       !to_tdx(vcpu)->vp_enter_args.r12;
 }
 
@@ -759,7 +759,7 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu)
 	 * otherwise the interrupt would have been serviced at the instruction
 	 * boundary.
 	 */
-	if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
+	if (vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT ||
 	    to_tdx(vcpu)->vp_enter_args.r12)
 		return false;
 
@@ -826,7 +826,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu)
 
 void tdx_vcpu_put(struct kvm_vcpu *vcpu)
 {
-	vmx_vcpu_pi_put(vcpu);
+	vt_vcpu_pi_put(vcpu);
 	tdx_prepare_switch_to_host(vcpu);
 }
 
@@ -866,6 +866,8 @@ void tdx_vcpu_free(struct kvm_vcpu *vcpu)
 	struct vcpu_tdx *tdx = to_tdx(vcpu);
 	int i;
 
+	tdx_free_pamt_cache(&tdx->pamt_cache);
+
 	if (vcpu->cpu != -1) {
 		KVM_BUG_ON(tdx->state == VCPU_TD_STATE_INITIALIZED, vcpu->kvm);
 		tdx_flush_vp_on_cpu(vcpu);
@@ -921,36 +923,51 @@ static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
 	return EXIT_REASON_TDCALL;
 }
 
-static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
+static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret)
 {
-	struct vcpu_tdx *tdx = to_tdx(vcpu);
-	u32 exit_reason;
-
-	switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) {
+	switch (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) {
 	case TDX_SUCCESS:
 	case TDX_NON_RECOVERABLE_VCPU:
 	case TDX_NON_RECOVERABLE_TD:
 	case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE:
 	case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE:
-		break;
+		return true;
 	default:
-		return -1u;
+		return false;
 	}
+}
+
+static __always_inline union vmx_exit_reason tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_tdx *tdx = to_tdx(vcpu);
+	union vmx_exit_reason exit_reason;
+
+	/*
+	 * Return the synthesized invalid Exit Reason, as the TDX module
+	 * never attempted to run the vCPU, i.e. the Exit Reason is undefined,
+	 * but this is NOT a failed VM-Enter.
+	 */
+	if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret))
+		return (union vmx_exit_reason) {
+			.basic = EXIT_REASON_UNDEFINED,
+		};
 
-	exit_reason = tdx->vp_enter_ret;
+	exit_reason.full = (u32)tdx->vp_enter_ret;
 
-	switch (exit_reason) {
+	switch (exit_reason.basic) {
 	case EXIT_REASON_TDCALL:
 		if (tdvmcall_exit_type(vcpu))
-			return EXIT_REASON_VMCALL;
-
-		return tdcall_to_vmx_exit_reason(vcpu);
+			exit_reason.basic = EXIT_REASON_VMCALL;
+		else
+			exit_reason.basic = tdcall_to_vmx_exit_reason(vcpu);
+		break;
 	case EXIT_REASON_EPT_MISCONFIG:
 		/*
 		 * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in
 		 * non-instrumentable code with interrupts disabled.
 		 */
-		return -1u;
+		exit_reason.basic = EXIT_REASON_UNDEFINED;
+		break;
 	default:
 		break;
 	}
@@ -967,22 +984,21 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu)
 
 	tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args);
 
-	vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu);
+	vt->exit_reason = tdx_to_vmx_exit_reason(vcpu);
 
 	vt->exit_qualification = tdx->vp_enter_args.rcx;
 	tdx->ext_exit_qualification = tdx->vp_enter_args.rdx;
 	tdx->exit_gpa = tdx->vp_enter_args.r8;
 	vt->exit_intr_info = tdx->vp_enter_args.r9;
 
-	vmx_handle_nmi(vcpu);
+	vt_handle_nmi(vcpu);
 
 	guest_state_exit_irqoff();
 }
 
 static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu)
 {
-	return vmx_get_exit_reason(vcpu).failed_vmentry &&
-	       vmx_get_exit_reason(vcpu).full != -1u;
+	return vt_get_exit_reason(vcpu).failed_vmentry;
 }
 
 static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
@@ -1071,8 +1087,16 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	 * allowing vCPU entry to avoid contention with tdh_vp_enter() and
 	 * TDCALLs.
 	 */
-	if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap)))
+	if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) {
+		/*
+		 * The vCPU never entered the guest, but this looks like a
+		 * handled exit to the caller.  Synthesize an invalid exit
+		 * reason so the previous exit's stale value isn't consumed
+		 * a second time.
+		 */
+		vt->exit_reason.full = EXIT_REASON_UNDEFINED;
 		return EXIT_FASTPATH_EXIT_HANDLED;
+	}
 
 	trace_kvm_entry(vcpu, run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT);
 
@@ -1131,7 +1155,7 @@ void tdx_inject_nmi(struct kvm_vcpu *vcpu)
 
 static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu)
 {
-	u32 intr_info = vmx_get_intr_info(vcpu);
+	u32 intr_info = vt_get_intr_info(vcpu);
 
 	/*
 	 * Machine checks are handled by handle_exception_irqoff(), or by
@@ -1609,15 +1633,23 @@ static int handle_tdvmcall(struct kvm_vcpu *vcpu)
 
 void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)
 {
-	u64 shared_bit = (pgd_level == 5) ? TDX_SHARED_BIT_PWL_5 :
-			  TDX_SHARED_BIT_PWL_4;
-
-	if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm))
+	if (KVM_BUG_ON(pgd_level != vcpu->kvm->arch.mirror_root_level, vcpu->kvm))
 		return;
 
 	td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);
 }
 
+static int tdx_topup_external_pamt_cache(struct kvm_vcpu *vcpu, int min_nr_spts)
+{
+	/*
+	 * Minus one page to exclude the root SPT, but plus one page for a
+	 * possible 4KB private mapping.
+	 */
+	min_nr_spts += -1 + 1;
+
+	return tdx_topup_pamt_cache(&to_tdx(vcpu)->pamt_cache, min_nr_spts);
+}
+
 static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,
 			    kvm_pfn_t pfn)
 {
@@ -1676,16 +1708,28 @@ static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn,
 static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,
 				     enum pg_level level, u64 new_spte)
 {
+	struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
 	gpa_t gpa = gfn_to_gpa(gfn);
 	u64 err, entry, level_state;
 	struct page *sept_pt;
+	int ret;
+
+	if (KVM_BUG_ON(!vcpu, kvm))
+		return -EIO;
 
 	sept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level);
 	if (!sept_pt)
 		return -EIO;
 
+	ret = tdx_pamt_get(page_to_pfn(sept_pt), &to_tdx(vcpu)->pamt_cache);
+	if (KVM_BUG_ON(ret, kvm))
+		return ret;
+
 	err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, sept_pt,
 			       &entry, &level_state);
+	if (err)
+		tdx_pamt_put(page_to_pfn(sept_pt));
+
 	if (unlikely(tdx_operand_busy(err)))
 		return -EBUSY;
 
@@ -1698,8 +1742,13 @@ static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,
 static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level,
 				  u64 new_spte)
 {
+	struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
 	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
 	kvm_pfn_t pfn = spte_to_pfn(new_spte);
+	int ret;
+
+	if (KVM_BUG_ON(!vcpu, kvm))
+		return -EIO;
 
 	/* TODO: handle large pages. */
 	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
@@ -1707,8 +1756,12 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
 
 	WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);
 
+	ret = tdx_pamt_get(pfn, &to_tdx(vcpu)->pamt_cache);
+	if (KVM_BUG_ON(ret, kvm))
+		return ret;
+
 	/*
-	 * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()
+	 * Ensure pre_fault_allowed is read by kvm_arch_pre_fault_allowed()
 	 * before kvm_tdx->state.  Userspace must not be allowed to pre-fault
 	 * arbitrary memory until the initial memory image is finalized.  Pairs
 	 * with the smp_wmb() in tdx_td_finalize().
@@ -1719,10 +1772,15 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
 	 * If the TD isn't finalized/runnable, then userspace is initializing
 	 * the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD.
 	 */
-	if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE))
-		return tdx_mem_page_add(kvm, gfn, level, pfn);
+	if (likely(kvm_tdx->state == TD_STATE_RUNNABLE))
+		ret = tdx_mem_page_aug(kvm, gfn, level, pfn);
+	else
+		ret = tdx_mem_page_add(kvm, gfn, level, pfn);
+
+	if (ret)
+		tdx_pamt_put(pfn);
 
-	return tdx_mem_page_aug(kvm, gfn, level, pfn);
+	return ret;
 }
 
 /*
@@ -1819,6 +1877,7 @@ static int tdx_sept_remove_leaf_spte(struct kvm *kvm, gfn_t gfn,
 		return -EIO;
 
 	tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE);
+	tdx_pamt_put(pfn);
 	return 0;
 }
 
@@ -1862,6 +1921,8 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,
  */
 static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
+	struct page *sept_pt = virt_to_page(sp->external_spt);
+
 	/*
 	 * KVM doesn't (yet) zap page table pages in mirror page table while
 	 * TD is active, though guest pages mapped in mirror page table could be
@@ -1875,15 +1936,15 @@ static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)
 	 * the page to prevent the kernel from accessing the encrypted page.
 	 */
 	if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) ||
-	    tdx_reclaim_page(virt_to_page(sp->external_spt)))
+	    tdx_reclaim_page(sept_pt))
 		goto out;
 
 	/*
-	 * Immediately free the S-EPT page because RCU-time free is unnecessary
-	 * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding
-	 * readers.
+	 * Immediately free the S-EPT page as the TDX subsystem doesn't support
+	 * freeing pages from RCU callbacks, and more importantly because
+	 * TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding readers.
 	 */
-	free_page((unsigned long)sp->external_spt);
+	tdx_free_control_page(sept_pt);
 out:
 	sp->external_spt = NULL;
 }
@@ -1895,7 +1956,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
 	struct vcpu_tdx *tdx = to_tdx(vcpu);
 
 	/* TDX supports only posted interrupt.  No lapic emulation. */
-	__vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector);
+	__vt_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector);
 
 	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
 }
@@ -1903,7 +1964,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
 static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu)
 {
 	u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK;
-	u64 eq = vmx_get_exit_qual(vcpu);
+	u64 eq = vt_get_exit_qual(vcpu);
 
 	if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION)
 		return false;
@@ -1939,7 +2000,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu)
 		/* Only private GPA triggers zero-step mitigation */
 		local_retry = true;
 	} else {
-		exit_qual = vmx_get_exit_qual(vcpu);
+		exit_qual = vt_get_exit_qual(vcpu);
 		/*
 		 * EPT violation due to instruction fetch should never be
 		 * triggered from shared memory in TDX guest.  If such EPT
@@ -1987,7 +2048,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu)
 	while (1) {
 		struct kvm_memory_slot *slot;
 
-		ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual);
+		ret = __vt_handle_ept_violation(vcpu, gpa, exit_qual);
 
 		if (ret != RET_PF_RETRY || !local_retry)
 			break;
@@ -1995,7 +2056,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu)
 		if (kvm_vcpu_has_events(vcpu) || signal_pending(current))
 			break;
 
-		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
+		if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) {
 			ret = -EIO;
 			break;
 		}
@@ -2021,7 +2082,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
 		return 1;
 	}
 
-	if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ)
+	if (vt_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ)
 		tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu));
 
 	return 1;
@@ -2032,7 +2093,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath)
 {
 	struct vcpu_tdx *tdx = to_tdx(vcpu);
 	u64 vp_enter_ret = tdx->vp_enter_ret;
-	union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
+	union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu);
 
 	if (fastpath != EXIT_FASTPATH_NONE)
 		return 1;
@@ -2126,6 +2187,11 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath)
 		 * - If it's not an MSMI, no need to do anything here.
 		 */
 		return 1;
+	case EXIT_REASON_NOTIFY:
+		/* NMI blocking state is handled by TDX module */
+		return __vt_handle_notify(vcpu, vt_get_exit_qual(vcpu));
+	case EXIT_REASON_BUS_LOCK:
+		return vt_handle_bus_lock_vmexit(vcpu);
 	default:
 		break;
 	}
@@ -2141,10 +2207,10 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
 	struct vcpu_tdx *tdx = to_tdx(vcpu);
 
 	*reason = tdx->vt.exit_reason.full;
-	if (*reason != -1u) {
-		*info1 = vmx_get_exit_qual(vcpu);
+	if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) {
+		*info1 = vt_get_exit_qual(vcpu);
 		*info2 = tdx->ext_exit_qualification;
-		*intr_info = vmx_get_intr_info(vcpu);
+		*intr_info = vt_get_intr_info(vcpu);
 	} else {
 		*info1 = 0;
 		*info2 = 0;
@@ -2456,7 +2522,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 
 	ret = -ENOMEM;
 
-	tdr_page = alloc_page(GFP_KERNEL_ACCOUNT);
+	tdr_page = tdx_alloc_control_page();
 	if (!tdr_page)
 		goto free_hkid;
 
@@ -2469,7 +2535,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 		goto free_tdr;
 
 	for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
-		tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT);
+		tdcs_pages[i] = tdx_alloc_control_page();
 		if (!tdcs_pages[i])
 			goto free_tdcs;
 	}
@@ -2587,10 +2653,8 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 teardown:
 	/* Only free pages not yet added, so start at 'i' */
 	for (; i < kvm_tdx->td.tdcs_nr_pages; i++) {
-		if (tdcs_pages[i]) {
-			__free_page(tdcs_pages[i]);
-			tdcs_pages[i] = NULL;
-		}
+		tdx_free_control_page(tdcs_pages[i]);
+		tdcs_pages[i] = NULL;
 	}
 	if (!kvm_tdx->td.tdcs_pages)
 		kfree(tdcs_pages);
@@ -2605,16 +2669,13 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 	free_cpumask_var(packages);
 
 free_tdcs:
-	for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
-		if (tdcs_pages[i])
-			__free_page(tdcs_pages[i]);
-	}
+	for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++)
+		tdx_free_control_page(tdcs_pages[i]);
 	kfree(tdcs_pages);
 	kvm_tdx->td.tdcs_pages = NULL;
 
 free_tdr:
-	if (tdr_page)
-		__free_page(tdr_page);
+	tdx_free_control_page(tdr_page);
 	kvm_tdx->td.tdr_page = NULL;
 
 free_hkid:
@@ -2728,11 +2789,6 @@ static tdx_vm_state_guard_t tdx_acquire_vm_state_locks(struct kvm *kvm)
 
 	mutex_lock(&kvm->lock);
 
-	if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) {
-		r = -EBUSY;
-		goto out_err;
-	}
-
 	r = kvm_lock_all_vcpus(kvm);
 	if (r)
 		goto out_err;
@@ -2760,6 +2816,14 @@ DEFINE_CLASS(tdx_vm_state_guard, tdx_vm_state_guard_t,
 	     if (!IS_ERR(_T)) tdx_release_vm_state_locks(_T),
 	     tdx_acquire_vm_state_locks(kvm), struct kvm *kvm);
 
+static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level)
+{
+	BUILD_BUG_ON(level != 4 && level != 5);
+
+	kvm->arch.mirror_root_level = level;
+	kvm->arch.gfn_direct_bits = gpa_to_gfn(BIT_ULL(level == 4 ? 47 : 51));
+}
+
 static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd)
 {
 	struct kvm_tdx_init_vm __user *user_data = u64_to_user_ptr(cmd->data);
@@ -2823,9 +2887,9 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd)
 	kvm_tdx->xfam = td_params->xfam;
 
 	if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW)
-		kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5;
+		tdx_set_mirror_root_level(kvm, 5);
 	else
-		kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4;
+		tdx_set_mirror_root_level(kvm, 4);
 
 	kvm_tdx->state = TD_STATE_INITIALIZED;
 out:
@@ -2948,7 +3012,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 	int ret, i;
 	u64 err;
 
-	page = alloc_page(GFP_KERNEL_ACCOUNT);
+	page = tdx_alloc_control_page();
 	if (!page)
 		return -ENOMEM;
 	tdx->vp.tdvpr_page = page;
@@ -2968,7 +3032,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 	}
 
 	for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
-		page = alloc_page(GFP_KERNEL_ACCOUNT);
+		page = tdx_alloc_control_page();
 		if (!page) {
 			ret = -ENOMEM;
 			goto free_tdcx;
@@ -2990,7 +3054,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 			 * method, but the rest are freed here.
 			 */
 			for (; i < kvm_tdx->td.tdcx_nr_pages; i++) {
-				__free_page(tdx->vp.tdcx_pages[i]);
+				tdx_free_control_page(tdx->vp.tdcx_pages[i]);
 				tdx->vp.tdcx_pages[i] = NULL;
 			}
 			return -EIO;
@@ -3018,16 +3082,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 
 free_tdcx:
 	for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
-		if (tdx->vp.tdcx_pages[i])
-			__free_page(tdx->vp.tdcx_pages[i]);
+		tdx_free_control_page(tdx->vp.tdcx_pages[i]);
 		tdx->vp.tdcx_pages[i] = NULL;
 	}
 	kfree(tdx->vp.tdcx_pages);
 	tdx->vp.tdcx_pages = NULL;
 
 free_tdvpr:
-	if (tdx->vp.tdvpr_page)
-		__free_page(tdx->vp.tdvpr_page);
+	tdx_free_control_page(tdx->vp.tdvpr_page);
 	tdx->vp.tdvpr_page = NULL;
 	tdx->vp.tdvpr_pa = 0;
 
@@ -3154,6 +3216,17 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd)
 	td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc));
 	td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR);
 
+	if (kvm_notify_vmexit_enabled(vcpu->kvm)) {
+		td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL,
+				 SECONDARY_EXEC_NOTIFY_VM_EXITING);
+		td_vmcs_write32(tdx, NOTIFY_WINDOW,
+				vcpu->kvm->arch.notify_window);
+	}
+
+	if (vcpu->kvm->arch.bus_lock_detection_enabled)
+		td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL,
+				 SECONDARY_EXEC_BUS_LOCK_DETECTION);
+
 	tdx->state = VCPU_TD_STATE_INITIALIZED;
 
 	return 0;
@@ -3189,7 +3262,7 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 	if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm))
 		return -EIO;
 
-	kvm_tdx->page_add_src = src_page;
+	kvm_tdx->page_add_src = src_page ?: pfn_to_page(pfn);
 	ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn);
 	kvm_tdx->page_add_src = NULL;
 
@@ -3219,6 +3292,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
 	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
 	struct kvm_tdx_init_mem_region region;
 	struct tdx_gmem_post_populate_arg arg;
+	gpa_t nr_bytes, end_gpa;
 	long gmem_ret;
 	int ret;
 
@@ -3235,11 +3309,15 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
 	if (copy_from_user(&region, u64_to_user_ptr(cmd->data), sizeof(region)))
 		return -EFAULT;
 
-	if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr ||
-	    !PAGE_ALIGNED(region.gpa) || !region.nr_pages ||
-	    region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa ||
+	if (!PAGE_ALIGNED(region.source_addr) ||
+	    (!gmem_in_place_conversion && !region.source_addr) ||
+	    !PAGE_ALIGNED(region.gpa) || !region.nr_pages)
+		return -EINVAL;
+
+	if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) ||
+	    check_add_overflow(region.gpa, nr_bytes - 1, &end_gpa) ||
 	    !vt_is_tdx_private_gpa(kvm, region.gpa) ||
-	    !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1))
+	    !vt_is_tdx_private_gpa(kvm, end_gpa))
 		return -EINVAL;
 
 	ret = 0;
@@ -3266,7 +3344,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
 			break;
 		}
 
-		region.source_addr += PAGE_SIZE;
+		if (region.source_addr)
+			region.source_addr += PAGE_SIZE;
 		region.gpa += PAGE_SIZE;
 		region.nr_pages--;
 
@@ -3487,6 +3566,10 @@ int __init tdx_hardware_setup(void)
 
 	vt_x86_ops.set_external_spte = tdx_sept_set_private_spte;
 	vt_x86_ops.free_external_spt = tdx_sept_free_private_spt;
+
+	if (tdx_supports_dynamic_pamt(tdx_sysinfo))
+		vt_x86_ops.topup_external_cache = tdx_topup_external_pamt_cache;
+
 	vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt;
 	return 0;
 
diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index ac8323a68b163a..fd368e3ee060b6 100644
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -72,6 +72,8 @@ struct vcpu_tdx {
 
 	u64 map_gpa_next;
 	u64 map_gpa_end;
+
+	struct tdx_pamt_cache pamt_cache;
 };
 
 void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err);
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 612ab07d4100fe..57f73e3d0f9887 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -1577,12 +1577,12 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
 	vmx_vcpu_load_vmcs(vcpu, cpu);
 
-	vmx_vcpu_pi_load(vcpu, cpu);
+	vt_vcpu_pi_load(vcpu, cpu);
 }
 
 void vmx_vcpu_put(struct kvm_vcpu *vcpu)
 {
-	vmx_vcpu_pi_put(vcpu);
+	vt_vcpu_pi_put(vcpu);
 
 	vmx_prepare_switch_to_host(to_vmx(vcpu));
 }
@@ -1791,7 +1791,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
 	 * so that guest userspace can't DoS the guest simply by triggering
 	 * emulation (enclaves are CPL3 only).
 	 */
-	if (vmx_get_exit_reason(vcpu).enclave_mode) {
+	if (vt_get_exit_reason(vcpu).enclave_mode) {
 		kvm_queue_exception(vcpu, UD_VECTOR);
 		return X86EMUL_PROPAGATE_FAULT;
 	}
@@ -1806,7 +1806,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
 
 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
 {
-	union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
+	union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu);
 	unsigned long rip, orig_rip;
 	u32 instr_len;
 
@@ -1909,7 +1909,7 @@ int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
 	return skip_emulated_instruction(vcpu);
 }
 
-static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
+void vmx_clear_hlt(struct kvm_vcpu *vcpu)
 {
 	/*
 	 * Ensure that we clear the HLT state in the VMCS.  We don't need to
@@ -2843,6 +2843,16 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf,
 	if (!cpu_has_sgx())
 		_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING;
 
+	/*
+	 * KVM doesn't re-derive the TSC scaling ratio when the host TSC
+	 * frequency changes, so TSC scaling is only usable with a constant
+	 * TSC.  Clear the control here rather than in vmx_hardware_setup() so
+	 * that the per-CPU configs recomputed by vmx_check_processor_compat()
+	 * stay consistent with the golden vmcs_config.
+	 */
+	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
+		_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_TSC_SCALING;
+
 	if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS)
 		_cpu_based_3rd_exec_control =
 			adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL,
@@ -3379,7 +3389,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
 		vpid_sync_context(vmx_get_current_vpid(vcpu));
 }
 
-void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full)
+void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
 {
 	/*
 	 * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in
@@ -4426,7 +4436,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
 	if (!vcpu->arch.apic->apicv_active)
 		return -1;
 
-	__vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector);
+	__vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector);
 	return 0;
 }
 
@@ -4883,10 +4893,11 @@ int vmx_vcpu_precreate(struct kvm *kvm)
 
 #define VMX_XSS_EXIT_BITMAP 0
 
-static void init_vmcs(struct vcpu_vmx *vmx)
+static void init_vmcs(struct kvm_vcpu *vcpu)
 {
-	struct kvm *kvm = vmx->vcpu.kvm;
+	struct kvm *kvm = vcpu->kvm;
 	struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
 	if (nested)
 		nested_vmx_set_vmcs_shadowing_bitmap();
@@ -4911,7 +4922,7 @@ static void init_vmcs(struct vcpu_vmx *vmx)
 	if (cpu_has_tertiary_exec_ctrls())
 		tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx));
 
-	if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) {
+	if (enable_apicv && lapic_in_kernel(vcpu)) {
 		vmcs_write64(EOI_EXIT_BITMAP0, 0);
 		vmcs_write64(EOI_EXIT_BITMAP1, 0);
 		vmcs_write64(EOI_EXIT_BITMAP2, 0);
@@ -4923,7 +4934,7 @@ static void init_vmcs(struct vcpu_vmx *vmx)
 		vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc)));
 	}
 
-	if (vmx_can_use_ipiv(&vmx->vcpu)) {
+	if (vmx_can_use_ipiv(vcpu)) {
 		vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table));
 		vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1);
 	}
@@ -4958,15 +4969,15 @@ static void init_vmcs(struct vcpu_vmx *vmx)
 	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
 
 	if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
-		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
+		vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat);
 
 	vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl());
 
 	/* 22.2.1, 20.8.1 */
 	vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl());
 
-	vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits();
-	vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits);
+	vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits();
+	vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
 
 	set_cr4_guest_host_mask(vmx);
 
@@ -4977,11 +4988,11 @@ static void init_vmcs(struct vcpu_vmx *vmx)
 		vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
 
 	if (enable_pml) {
-		vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
+		vmcs_write64(PML_ADDRESS, page_to_phys(vcpu->arch.pml_page));
 		vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX);
 	}
 
-	vmx_write_encls_bitmap(&vmx->vcpu, NULL);
+	vmx_write_encls_bitmap(vcpu, NULL);
 
 	if (vmx_pt_mode_is_host_guest()) {
 		memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc));
@@ -4994,13 +5005,13 @@ static void init_vmcs(struct vcpu_vmx *vmx)
 	vmcs_writel(GUEST_SYSENTER_ESP, 0);
 	vmcs_writel(GUEST_SYSENTER_EIP, 0);
 
-	vmx_guest_debugctl_write(&vmx->vcpu, 0);
+	vmx_guest_debugctl_write(vcpu, 0);
 
 	if (cpu_has_vmx_tpr_shadow()) {
 		vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
-		if (cpu_need_tpr_shadow(&vmx->vcpu))
+		if (cpu_need_tpr_shadow(vcpu))
 			vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
-				     __pa(vmx->vcpu.arch.apic->regs));
+				     __pa(vcpu->arch.apic->regs));
 		vmcs_write32(TPR_THRESHOLD, 0);
 	}
 
@@ -5011,7 +5022,7 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
-	init_vmcs(vmx);
+	init_vmcs(vcpu);
 
 	if (nested &&
 	    kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS))
@@ -5381,15 +5392,9 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu)
 	       (kvm_get_rflags(vcpu) & X86_EFLAGS_AC);
 }
 
-static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu)
-{
-	return vcpu->arch.guest_fpu.fpstate->xfd &&
-	       !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS);
-}
-
 static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code)
 {
-	unsigned long cr2 = vmx_get_exit_qual(vcpu);
+	unsigned long cr2 = vt_get_exit_qual(vcpu);
 
 	if (vcpu->arch.apf.host_apf_flags)
 		goto handle_pf;
@@ -5437,7 +5442,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu)
 	u32 vect_info;
 
 	vect_info = vmx->idt_vectoring_info;
-	intr_info = vmx_get_intr_info(vcpu);
+	intr_info = vt_get_intr_info(vcpu);
 
 	/*
 	 * Machine checks are handled by handle_exception_irqoff(), or by
@@ -5518,7 +5523,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu)
 
 	switch (ex_no) {
 	case DB_VECTOR:
-		dr6 = vmx_get_exit_qual(vcpu);
+		dr6 = vt_get_exit_qual(vcpu);
 		if (!(vcpu->guest_debug &
 		      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
 			/*
@@ -5594,7 +5599,7 @@ static int handle_io(struct kvm_vcpu *vcpu)
 	int size, in, string;
 	unsigned port;
 
-	exit_qualification = vmx_get_exit_qual(vcpu);
+	exit_qualification = vt_get_exit_qual(vcpu);
 	string = (exit_qualification & 16) != 0;
 
 	++vcpu->stat.io_exits;
@@ -5683,7 +5688,7 @@ static int handle_cr(struct kvm_vcpu *vcpu)
 	int err;
 	int ret;
 
-	exit_qualification = vmx_get_exit_qual(vcpu);
+	exit_qualification = vt_get_exit_qual(vcpu);
 	cr = exit_qualification & 15;
 	reg = (exit_qualification >> 8) & 15;
 	switch ((exit_qualification >> 4) & 3) {
@@ -5761,7 +5766,7 @@ static int handle_dr(struct kvm_vcpu *vcpu)
 	int dr, dr7, reg;
 	int err = 1;
 
-	exit_qualification = vmx_get_exit_qual(vcpu);
+	exit_qualification = vt_get_exit_qual(vcpu);
 	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
 
 	/* First, if DR does not exist, trigger UD */
@@ -5857,7 +5862,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu)
 
 static int handle_invlpg(struct kvm_vcpu *vcpu)
 {
-	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
+	unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 
 	kvm_mmu_invlpg(vcpu, exit_qualification);
 	return kvm_skip_emulated_instruction(vcpu);
@@ -5866,7 +5871,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu)
 static int handle_apic_access(struct kvm_vcpu *vcpu)
 {
 	if (likely(fasteoi)) {
-		unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
+		unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 		int access_type, offset;
 
 		access_type = exit_qualification & APIC_ACCESS_TYPE;
@@ -5887,17 +5892,16 @@ static int handle_apic_access(struct kvm_vcpu *vcpu)
 
 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
 {
-	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
-	int vector = exit_qualification & 0xff;
+	unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 
 	/* EOI-induced VM exit is trap-like and thus no need to adjust IP */
-	kvm_apic_set_eoi_accelerated(vcpu, vector);
+	kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff);
 	return 1;
 }
 
 static int handle_apic_write(struct kvm_vcpu *vcpu)
 {
-	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
+	unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 
 	/*
 	 * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and
@@ -5925,7 +5929,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu)
 	idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
 	type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
 
-	exit_qualification = vmx_get_exit_qual(vcpu);
+	exit_qualification = vt_get_exit_qual(vcpu);
 
 	reason = (u32)exit_qualification >> 30;
 	if (reason == TASK_SWITCH_GATE && idt_v) {
@@ -5971,7 +5975,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu)
 
 static int handle_ept_violation(struct kvm_vcpu *vcpu)
 {
-	unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
+	unsigned long exit_qualification = vt_get_exit_qual(vcpu);
 	gpa_t gpa;
 
 	/*
@@ -5999,7 +6003,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu)
 	if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa)))
 		return kvm_emulate_instruction(vcpu, 0);
 
-	return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification);
+	return __vt_handle_ept_violation(vcpu, gpa, exit_qualification);
 }
 
 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
@@ -6162,7 +6166,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu)
 	/* According to the Intel instruction reference, the memory operand
 	 * is read even if it isn't needed (e.g., for type==all)
 	 */
-	if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
+	if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu),
 				vmx_instruction_info, false,
 				sizeof(operand), &gva))
 		return 1;
@@ -6176,7 +6180,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu)
 
 	trace_kvm_pml_full(vcpu->vcpu_id);
 
-	exit_qualification = vmx_get_exit_qual(vcpu);
+	exit_qualification = vt_get_exit_qual(vcpu);
 
 	/*
 	 * PML buffer FULL happened while executing iret from NMI,
@@ -6267,23 +6271,9 @@ static int handle_encls(struct kvm_vcpu *vcpu)
 }
 #endif /* CONFIG_X86_SGX_KVM */
 
-static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu)
-{
-	/*
-	 * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK
-	 * VM-Exits. Unconditionally set the flag here and leave the handling to
-	 * vmx_handle_exit().
-	 */
-	to_vt(vcpu)->exit_reason.bus_lock_detected = true;
-	return 1;
-}
-
 static int handle_notify(struct kvm_vcpu *vcpu)
 {
-	unsigned long exit_qual = vmx_get_exit_qual(vcpu);
-	bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID;
-
-	++vcpu->stat.notify_window_exits;
+	unsigned long exit_qual = vt_get_exit_qual(vcpu);
 
 	/*
 	 * Notify VM exit happened while executing iret from NMI,
@@ -6293,15 +6283,7 @@ static int handle_notify(struct kvm_vcpu *vcpu)
 		vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
 			      GUEST_INTR_STATE_NMI);
 
-	if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER ||
-	    context_invalid) {
-		vcpu->run->exit_reason = KVM_EXIT_NOTIFY;
-		vcpu->run->notify.flags = context_invalid ?
-					  KVM_NOTIFY_CONTEXT_INVALID : 0;
-		return 0;
-	}
-
-	return 1;
+	return __vt_handle_notify(vcpu, exit_qual);
 }
 
 static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu)
@@ -6311,13 +6293,13 @@ static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu)
 
 static int handle_rdmsr_imm(struct kvm_vcpu *vcpu)
 {
-	return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu),
+	return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu),
 				     vmx_get_msr_imm_reg(vcpu));
 }
 
 static int handle_wrmsr_imm(struct kvm_vcpu *vcpu)
 {
-	return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu),
+	return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu),
 				     vmx_get_msr_imm_reg(vcpu));
 }
 
@@ -6377,7 +6359,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[EXIT_REASON_VMFUNC]		      = handle_vmx_instruction,
 	[EXIT_REASON_PREEMPTION_TIMER]	      = handle_preemption_timer,
 	[EXIT_REASON_ENCLS]		      = handle_encls,
-	[EXIT_REASON_BUS_LOCK]                = handle_bus_lock_vmexit,
+	[EXIT_REASON_BUS_LOCK]                = vt_handle_bus_lock_vmexit,
 	[EXIT_REASON_NOTIFY]		      = handle_notify,
 	[EXIT_REASON_SEAMCALL]		      = handle_tdx_instruction,
 	[EXIT_REASON_TDCALL]		      = handle_tdx_instruction,
@@ -6388,16 +6370,18 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 static const int kvm_vmx_max_exit_handlers =
 	ARRAY_SIZE(kvm_vmx_exit_handlers);
 
+static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers));
+
 void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
 		       u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
 	*reason = vmx->vt.exit_reason.full;
-	*info1 = vmx_get_exit_qual(vcpu);
+	*info1 = vt_get_exit_qual(vcpu);
 	if (!(vmx->vt.exit_reason.failed_vmentry)) {
 		*info2 = vmx->idt_vectoring_info;
-		*intr_info = vmx_get_intr_info(vcpu);
+		*intr_info = vt_get_intr_info(vcpu);
 		if (is_exception_with_error_code(*intr_info))
 			*error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
 		else
@@ -6418,48 +6402,25 @@ void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code)
 		*error_code = 0;
 }
 
-static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
+static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu)
 {
-	if (vmx->pml_pg) {
-		__free_page(vmx->pml_pg);
-		vmx->pml_pg = NULL;
+	if (vcpu->arch.pml_page) {
+		__free_page(vcpu->arch.pml_page);
+		vcpu->arch.pml_page = NULL;
 	}
 }
 
 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
 {
-	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	u16 pml_idx, pml_tail_index;
-	u64 *pml_buf;
-	int i;
+	u16 pml_idx;
 
 	pml_idx = vmcs_read16(GUEST_PML_INDEX);
 
 	/* Do nothing if PML buffer is empty */
 	if (pml_idx == PML_HEAD_INDEX)
 		return;
-	/*
-	 * PML index always points to the next available PML buffer entity
-	 * unless PML log has just overflowed.
-	 */
-	pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1;
 
-	/*
-	 * PML log is written backwards: the CPU first writes the entry 511
-	 * then the entry 510, and so on.
-	 *
-	 * Read the entries in the same order they were written, to ensure that
-	 * the dirty ring is filled in the same order the CPU wrote them.
-	 */
-	pml_buf = page_address(vmx->pml_pg);
-
-	for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) {
-		u64 gpa;
-
-		gpa = pml_buf[i];
-		WARN_ON(gpa & (PAGE_SIZE - 1));
-		kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
-	}
+	kvm_flush_pml_buffer(vcpu, pml_idx);
 
 	/* reset PML index */
 	vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX);
@@ -6705,10 +6666,10 @@ void dump_vmcs(struct kvm_vcpu *vcpu)
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
  */
-static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
+int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu);
+	union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu);
 	u32 vectoring_info = vmx->idt_vectoring_info;
 	u16 exit_handler_index;
 
@@ -6864,24 +6825,6 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
 	return 0;
 }
 
-int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
-{
-	int ret = __vmx_handle_exit(vcpu, exit_fastpath);
-
-	/*
-	 * Exit to user space when bus lock detected to inform that there is
-	 * a bus lock in guest.
-	 */
-	if (vmx_get_exit_reason(vcpu).bus_lock_detected) {
-		if (ret > 0)
-			vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
-
-		vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK;
-		return 0;
-	}
-	return ret;
-}
-
 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 {
 	int tpr_threshold;
@@ -7145,74 +7088,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
 	vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
 }
 
-static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
-{
-	/*
-	 * Save xfd_err to guest_fpu before interrupt is enabled, so the
-	 * MSR value is not clobbered by the host activity before the guest
-	 * has chance to consume it.
-	 *
-	 * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM
-	 * interception may have been caused by L1 interception.  Per the SDM,
-	 * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1.
-	 *
-	 * Note, XFD_ERR is updated _before_ the #NM interception check, i.e.
-	 * unlike CR2 and DR6, the value is not a payload that is attached to
-	 * the #NM exception.
-	 */
-	if (is_xfd_nm_fault(vcpu))
-		rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
-}
-
-static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info)
-{
-	/* if exit due to PF check for async PF */
-	if (is_page_fault(intr_info))
-		vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
-	/* if exit due to NM, handle before interrupts are enabled */
-	else if (is_nm_fault(intr_info))
-		handle_nm_fault_irqoff(vcpu);
-	/* Handle machine checks before interrupts are enabled */
-	else if (is_machine_check(intr_info))
-		kvm_machine_check();
-}
-
-static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
-					     u32 intr_info)
-{
-	unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK;
-
-	if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm,
-	    "unexpected VM-Exit interrupt info: 0x%x", intr_info))
-		return;
-
-	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
-	x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
-	kvm_after_interrupt(vcpu);
-
-	vcpu->arch.at_instruction_boundary = true;
-}
-
-void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
-{
-	if (to_vt(vcpu)->emulation_required)
-		return;
-
-	switch (vmx_get_exit_reason(vcpu).basic) {
-	case EXIT_REASON_EXTERNAL_INTERRUPT:
-		handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu));
-		break;
-	case EXIT_REASON_EXCEPTION_NMI:
-		handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu));
-		break;
-	case EXIT_REASON_MCE_DURING_VMENTRY:
-		kvm_machine_check();
-		break;
-	default:
-		break;
-	}
-}
-
 /*
  * The kvm parameter can be NULL (module initialization, or invocation before
  * VM creation). Be sure to check the kvm parameter before using it.
@@ -7252,7 +7127,7 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
 		if (vmx->loaded_vmcs->nmi_known_unmasked)
 			return;
 
-		exit_intr_info = vmx_get_intr_info(&vmx->vcpu);
+		exit_intr_info = vt_get_intr_info(&vmx->vcpu);
 		unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
 		vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
 		/*
@@ -7363,12 +7238,14 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
 	if (kvm_vcpu_has_mediated_pmu(&vmx->vcpu))
 		return;
 
-	pmu->host_cross_mapped_mask = 0;
-	if (pmu->pebs_enable & pmu->global_ctrl)
-		intel_pmu_cross_mapped_check(pmu);
+	struct x86_guest_pebs guest_pebs = {
+		.enable = intel_pmu_compute_pebs_enable(pmu),
+		.ds_area = pmu->ds_area,
+		.data_cfg = pmu->pebs_data_cfg,
+	};
 
 	/* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */
-	msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu);
+	msrs = perf_guest_get_msrs(&nr_msrs, &guest_pebs);
 	if (!msrs)
 		return;
 
@@ -7419,14 +7296,14 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu,
 	 * the fastpath even, all other exits must use the slow path.
 	 */
 	if (is_guest_mode(vcpu) &&
-	    vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER)
+	    vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER)
 		return EXIT_FASTPATH_NONE;
 
-	switch (vmx_get_exit_reason(vcpu).basic) {
+	switch (vt_get_exit_reason(vcpu).basic) {
 	case EXIT_REASON_MSR_WRITE:
 		return handle_fastpath_wrmsr(vcpu);
 	case EXIT_REASON_MSR_WRITE_IMM:
-		return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu),
+		return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu),
 						 vmx_get_msr_imm_reg(vcpu));
 	case EXIT_REASON_PREEMPTION_TIMER:
 		return handle_fastpath_preemption_timer(vcpu, force_immediate_exit);
@@ -7439,17 +7316,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu,
 	}
 }
 
-noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu)
-{
-	if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI ||
-	    !is_nmi(vmx_get_intr_info(vcpu)))
-		return;
-
-	kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
-	x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
-	kvm_after_interrupt(vcpu);
-}
-
 static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
 					unsigned int flags)
 {
@@ -7474,15 +7340,15 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
 	vmx_enable_fb_clear(vmx);
 
 	if (unlikely(vmx->fail)) {
-		vmx->vt.exit_reason.full = 0xdead;
+		vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED;
 		goto out;
 	}
 
 	vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON);
-	if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry))
+	if (likely(!vt_get_exit_reason(vcpu).failed_vmentry))
 		vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
 
-	vmx_handle_nmi(vcpu);
+	vt_handle_nmi(vcpu);
 
 out:
 	guest_state_exit_irqoff();
@@ -7619,7 +7485,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 		 * checking.
 		 */
 		if (vcpu->arch.nested_run_pending &&
-		    !vmx_get_exit_reason(vcpu).failed_vmentry)
+		    !vt_get_exit_reason(vcpu).failed_vmentry)
 			++vcpu->stat.nested_run;
 
 		vcpu->arch.nested_run_pending = 0;
@@ -7630,7 +7496,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 	trace_kvm_exit(vcpu, KVM_ISA_VMX);
 
-	if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry))
+	if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry))
 		return EXIT_FASTPATH_NONE;
 
 	vmx->loaded_vmcs->launched = 1;
@@ -7648,7 +7514,7 @@ void vmx_vcpu_free(struct kvm_vcpu *vcpu)
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
 	if (enable_pml)
-		vmx_destroy_pml_buffer(vmx);
+		vmx_destroy_pml_buffer(vcpu);
 	free_vpid(vmx->vpid);
 	nested_vmx_free_vcpu(vcpu);
 	free_loaded_vmcs(vmx->loaded_vmcs);
@@ -7680,8 +7546,8 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu)
 	 * for the guest), etc.
 	 */
 	if (enable_pml) {
-		vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
-		if (!vmx->pml_pg)
+		vcpu->arch.pml_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+		if (!vcpu->arch.pml_page)
 			goto free_vpid;
 	}
 
@@ -7752,7 +7618,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu)
 free_vmcs:
 	free_loaded_vmcs(vmx->loaded_vmcs);
 free_pml:
-	vmx_destroy_pml_buffer(vmx);
+	vmx_destroy_pml_buffer(vcpu);
 free_vpid:
 	free_vpid(vmx->vpid);
 	return err;
@@ -8440,21 +8306,13 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit
 }
 #endif
 
-void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
+void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 
-	if (WARN_ON_ONCE(!enable_pml))
-		return;
-
 	guard(vmx_vmcs01)(vcpu);
 
-	/*
-	 * Note, nr_memslots_dirty_logging can be changed concurrent with this
-	 * code, but in that case another update request will be made and so
-	 * the guest will never run with a stale PML value.
-	 */
-	if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging))
+	if (enable)
 		secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML);
 	else
 		secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML);
diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index dc8517f15bc463..74cd50f29fa478 100644
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -265,13 +265,6 @@ struct vcpu_vmx {
 	unsigned int ple_window;
 	bool ple_window_dirty;
 
-	/* Support for PML */
-#define PML_LOG_NR_ENTRIES	512
-	/* PML is written backwards: this is the first entry written by the CPU */
-#define PML_HEAD_INDEX		(PML_LOG_NR_ENTRIES-1)
-
-	struct page *pml_pg;
-
 	/* apic deadline value in host tsc */
 	u64 hv_deadline_tsc;
 
@@ -304,43 +297,6 @@ struct kvm_vmx {
 	u64 *pid_table;
 };
 
-static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu)
-{
-	return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt);
-}
-
-static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt)
-{
-	return &(container_of(vt, struct vcpu_vmx, vt)->vcpu);
-}
-
-static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu)
-{
-	return to_vt(vcpu)->exit_reason;
-}
-
-static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu)
-{
-	struct vcpu_vt *vt = to_vt(vcpu);
-
-	if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) &&
-	    !WARN_ON_ONCE(is_td_vcpu(vcpu)))
-		vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
-
-	return vt->exit_qualification;
-}
-
-static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu)
-{
-	struct vcpu_vt *vt = to_vt(vcpu);
-
-	if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) &&
-	    !WARN_ON_ONCE(is_td_vcpu(vcpu)))
-		vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
-
-	return vt->exit_intr_info;
-}
-
 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu);
 int allocate_vpid(void);
 void free_vpid(int vpid);
@@ -397,7 +353,7 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu);
 
 gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
 
-void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu);
+void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable);
 
 u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated);
 bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated);
@@ -664,7 +620,20 @@ static __always_inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
 	return container_of(vcpu, struct vcpu_vmx, vcpu);
 }
 
-void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu);
+u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu);
+
+static inline u64 intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu)
+{
+	/*
+	 * Avoid the function call overhead in the common case that the guest
+	 * isn't using PEBS.
+	 */
+	if (!(pmu->pebs_enable & pmu->global_ctrl))
+		return 0;
+
+	return __intel_pmu_compute_pebs_enable(pmu);
+}
+
 int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu);
 void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu);
 
diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h
index 054fd14bb2e16b..cd72684d075d36 100644
--- a/arch/x86/kvm/vmx/x86_ops.h
+++ b/arch/x86/kvm/vmx/x86_ops.h
@@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
 void vmx_vcpu_put(struct kvm_vcpu *vcpu);
 int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath);
-void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu);
 int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu);
 void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu);
 bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu);
@@ -82,7 +81,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
 bool vmx_get_if_flag(struct kvm_vcpu *vcpu);
 void vmx_flush_tlb_all(struct kvm_vcpu *vcpu);
 void vmx_flush_tlb_current(struct kvm_vcpu *vcpu);
-void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full);
+void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr);
 void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu);
 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask);
 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu);
@@ -95,6 +94,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection);
 int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection);
 bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
 void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
+void vmx_clear_hlt(struct kvm_vcpu *vcpu);
 void vmx_enable_nmi_window(struct kvm_vcpu *vcpu);
 void vmx_enable_irq_window(struct kvm_vcpu *vcpu);
 void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr);
@@ -118,7 +118,7 @@ u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu);
 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu);
 void vmx_write_tsc_offset(struct kvm_vcpu *vcpu);
 void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu);
-void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu);
+void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable);
 #ifdef CONFIG_X86_64
 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
 		     bool *expired);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 79468ddfe47367..cf3fcdfd8ad2ee 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -35,6 +35,7 @@
 #include "smm.h"
 
 #include <linux/clocksource.h>
+#include <linux/timekeeping.h>
 #include <linux/interrupt.h>
 #include <linux/kvm.h>
 #include <linux/fs.h>
@@ -61,6 +62,7 @@
 #include <linux/mem_encrypt.h>
 #include <linux/suspend.h>
 #include <linux/smp.h>
+#include <linux/units.h>
 
 #include <trace/events/ipi.h>
 #include <trace/events/kvm.h>
@@ -925,19 +927,13 @@ static void update_pvclock_gtod(struct timekeeper *tk)
 
 	write_seqcount_end(&vdata->seq);
 }
+#endif
 
 static s64 get_kvmclock_base_ns(void)
 {
 	/* Count up from boot time, but with the frequency of the raw clock.  */
-	return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
-}
-#else
-static s64 get_kvmclock_base_ns(void)
-{
-	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
-	return ktime_get_boottime_ns();
+	return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT));
 }
-#endif
 
 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
 {
@@ -945,32 +941,57 @@ static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
 	return dividend;
 }
 
-static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
+static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz,
 			       s8 *pshift, u32 *pmultiplier)
 {
-	uint64_t scaled64;
-	int32_t  shift = 0;
-	uint64_t tps64;
-	uint32_t tps32;
+	u64 scaled_hz_u64 = scaled_hz;
+	s32 shift = 0;
+	u64 base_hz_u64;
+	u32 base32;
+
+	/*
+	 * This function calculates a fixed-point multiplier and shift such
+	 * that:
+	 *   time_ns = (tsc_cycles << shift) * multiplier >> 32
+	 *
+	 * Where tsc_cycles tick at base_hz, and time_ns should count at
+	 * scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion).
+	 *
+	 * The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift
+	 * to keep everything in range.
+	 */
+
+	base_hz_u64 = base_hz;
 
-	tps64 = base_hz;
-	scaled64 = scaled_hz;
-	while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
-		tps64 >>= 1;
+	/*
+	 * Start by shifting base_hz right until it fits in 32 bits, and
+	 * is lower than double the target rate. This introduces a negative
+	 * shift value which would result in pvclock_scale_delta() shifting
+	 * the actual tick count right before performing the multiplication.
+	 */
+	while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) {
+		base_hz_u64 >>= 1;
 		shift--;
 	}
 
-	tps32 = (uint32_t)tps64;
-	while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
-		if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
-			scaled64 >>= 1;
+	/* Now the shifted base_hz fits in 32 bits. */
+	base32 = (u32)base_hz_u64;
+
+	/*
+	 * Next, shift scaled_hz right until it fits in 32 bits, and ensure
+	 * that the shifted base_hz is strictly larger (so that the result of the
+	 * final division also fits in 32 bits).
+	 */
+	while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) {
+		if (scaled_hz_u64 >> 32 || base32 & BIT(31))
+			scaled_hz_u64 >>= 1;
 		else
-			tps32 <<= 1;
+			base32 <<= 1;
 		shift++;
 	}
 
 	*pshift = shift;
-	*pmultiplier = div_frac(scaled64, tps32);
+	*pmultiplier = div_frac(scaled_hz_u64, base32);
 }
 
 #ifdef CONFIG_X86_64
@@ -1061,11 +1082,15 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
 
 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
 {
-	u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
-				      vcpu->arch.virtual_tsc_mult,
-				      vcpu->arch.virtual_tsc_shift);
-	tsc += vcpu->arch.this_tsc_write;
-	return tsc;
+	s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec;
+	u64 tsc;
+
+	/* Handle negative deltas gracefully (master clock ref may be earlier) */
+	tsc = pvclock_scale_delta(abs(delta_ns),
+				  vcpu->arch.virtual_tsc_mult,
+				  vcpu->arch.virtual_tsc_shift);
+
+	return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc);
 }
 
 #ifdef CONFIG_X86_64
@@ -1131,11 +1156,12 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio)
 	return _tsc;
 }
 
-u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
+u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc,
+			      u64 target_tsc)
 {
 	u64 tsc;
 
-	tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
+	tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
 
 	return target_tsc - tsc;
 }
@@ -1254,6 +1280,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
 	kvm->arch.last_tsc_write = tsc;
 	kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
 	kvm->arch.last_tsc_offset = offset;
+	kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
 
 	vcpu->arch.last_guest_tsc = tsc;
 
@@ -1296,7 +1323,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
 	bool synchronizing = false;
 
 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
-	offset = kvm_compute_l1_tsc_offset(vcpu, data);
+	offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data);
 	ns = get_kvmclock_base_ns();
 	elapsed = ns - kvm->arch.last_tsc_nsec;
 
@@ -1345,7 +1372,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
 		} else {
 			u64 delta = nsec_to_cycles(vcpu, elapsed);
 			data += delta;
-			offset = kvm_compute_l1_tsc_offset(vcpu, data);
+			offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data);
 		}
 		matched = true;
 	}
@@ -1356,126 +1383,27 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
 
 #ifdef CONFIG_X86_64
 
-static u64 read_tsc(void)
+static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap,
+				 u64 *tsc_timestamp)
 {
-	u64 ret = (u64)rdtsc_ordered();
-	u64 last = pvclock_gtod_data.clock.cycle_last;
-
-	if (likely(ret >= last))
-		return ret;
-
 	/*
-	 * GCC likes to generate cmov here, but this branch is extremely
-	 * predictable (it's just a function of time and the likely is
-	 * very likely) and there's a data dependence, so force GCC
-	 * to generate a branch instead.  I don't barrier() because
-	 * we don't actually need a barrier, and if this function
-	 * ever gets inlined it will generate worse code.
+	 * ktime_get_snapshot_id() cannot fail for standard clock IDs
+	 * (only for invalid/aux clocks or during suspend, with a WARN).
 	 */
-	asm volatile ("");
-	return last;
-}
-
-static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
-			  int *mode)
-{
-	u64 tsc_pg_val;
-	long v;
+	if (!snap->valid)
+		return false;
 
-	switch (clock->vclock_mode) {
-	case VDSO_CLOCKMODE_HVCLOCK:
-		if (hv_read_tsc_page_tsc(hv_get_tsc_page(),
-					 tsc_timestamp, &tsc_pg_val)) {
-			/* TSC page valid */
-			*mode = VDSO_CLOCKMODE_HVCLOCK;
-			v = (tsc_pg_val - clock->cycle_last) &
-				clock->mask;
-		} else {
-			/* TSC page invalid */
-			*mode = VDSO_CLOCKMODE_NONE;
-		}
-		break;
-	case VDSO_CLOCKMODE_TSC:
-		*mode = VDSO_CLOCKMODE_TSC;
-		*tsc_timestamp = read_tsc();
-		v = (*tsc_timestamp - clock->cycle_last) &
-			clock->mask;
-		break;
-	default:
-		*mode = VDSO_CLOCKMODE_NONE;
+	if (snap->cs_id == CSID_X86_TSC) {
+		*tsc_timestamp = snap->cycles;
+		return true;
 	}
 
-	if (*mode == VDSO_CLOCKMODE_NONE)
-		*tsc_timestamp = v = 0;
-
-	return v * clock->mult;
-}
-
-/*
- * As with get_kvmclock_base_ns(), this counts from boot time, at the
- * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot).
- */
-static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp)
-{
-	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
-	unsigned long seq;
-	int mode;
-	u64 ns;
-
-	do {
-		seq = read_seqcount_begin(&gtod->seq);
-		ns = gtod->raw_clock.base_cycles;
-		ns += vgettsc(&gtod->raw_clock, tsc_timestamp, &mode);
-		ns >>= gtod->raw_clock.shift;
-		ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
-	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
-	*t = ns;
-
-	return mode;
-}
-
-/*
- * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with
- * no boot time offset.
- */
-static int do_monotonic(s64 *t, u64 *tsc_timestamp)
-{
-	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
-	unsigned long seq;
-	int mode;
-	u64 ns;
-
-	do {
-		seq = read_seqcount_begin(&gtod->seq);
-		ns = gtod->clock.base_cycles;
-		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
-		ns >>= gtod->clock.shift;
-		ns += ktime_to_ns(gtod->clock.offset);
-	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
-	*t = ns;
-
-	return mode;
-}
-
-static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
-{
-	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
-	unsigned long seq;
-	int mode;
-	u64 ns;
-
-	do {
-		seq = read_seqcount_begin(&gtod->seq);
-		ts->tv_sec = gtod->wall_time_sec;
-		ns = gtod->clock.base_cycles;
-		ns += vgettsc(&gtod->clock, tsc_timestamp, &mode);
-		ns >>= gtod->clock.shift;
-	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
-
-	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
-	ts->tv_nsec = ns;
+	if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) {
+		*tsc_timestamp = snap->hw_cycles;
+		return true;
+	}
 
-	return mode;
+	return false;
 }
 
 /*
@@ -1485,12 +1413,14 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
  */
 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
 {
-	/* checked again under seqlock below */
-	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+	struct system_time_snapshot snap = {};
+
+	ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap);
+	if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
 		return false;
 
-	return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns,
-						     tsc_timestamp));
+	*kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT));
+	return true;
 }
 
 /*
@@ -1499,12 +1429,14 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
  */
 bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
 {
-	/* checked again under seqlock below */
-	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+	struct system_time_snapshot snap = {};
+
+	ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap);
+	if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
 		return false;
 
-	return gtod_is_based_on_tsc(do_monotonic(kernel_ns,
-						 tsc_timestamp));
+	*kernel_ns = ktime_to_ns(snap.systime);
+	return true;
 }
 
 /*
@@ -1517,11 +1449,14 @@ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
 					   u64 *tsc_timestamp)
 {
-	/* checked again under seqlock below */
-	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
+	struct system_time_snapshot snap = {};
+
+	ktime_get_snapshot_id(CLOCK_REALTIME, &snap);
+	if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp))
 		return false;
 
-	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
+	*ts = ktime_to_timespec64(snap.systime);
+	return true;
 }
 #endif
 
@@ -1566,6 +1501,8 @@ static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
  *
  */
 
+static unsigned long get_cpu_tsc_khz(void);
+
 static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
 {
 #ifdef CONFIG_X86_64
@@ -1589,9 +1526,30 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
 				&& !ka->backwards_tsc_observed
 				&& !ka->boot_vcpu_runs_old_kvmclock;
 
-	if (ka->use_master_clock)
+	if (ka->use_master_clock) {
+		u64 tsc_hz;
+
 		atomic_set(&kvm_guest_has_master_clock, 1);
 
+		/*
+		 * Copy the scaling ratio and precompute the mul/shift for
+		 * converting guest TSC to nanoseconds. These are used by
+		 * get_kvmclock() to compute kvmclock from the host TSC
+		 * without needing a vCPU reference.
+		 */
+		ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio;
+		tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ;
+		if (tsc_hz && kvm_caps.has_tsc_control)
+			tsc_hz = kvm_scale_tsc(tsc_hz,
+					       ka->master_tsc_scaling_ratio);
+		if (tsc_hz)
+			kvm_get_time_scale(NSEC_PER_SEC, tsc_hz,
+					   &ka->master_tsc_shift,
+					   &ka->master_tsc_mul);
+		else
+			ka->use_master_clock = false;
+	}
+
 	vclock_mode = pvclock_gtod_data.clock.vclock_mode;
 	trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
 					vcpus_matched);
@@ -1658,39 +1616,49 @@ static unsigned long get_cpu_tsc_khz(void)
 }
 
 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc.  */
-static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
+static bool __get_kvmclock_master_clock(struct kvm *kvm,
+					struct kvm_clock_data *data)
 {
+#ifdef CONFIG_X86_64
 	struct kvm_arch *ka = &kvm->arch;
 	struct pvclock_vcpu_time_info hv_clock;
+	struct timespec64 ts;
 
-	/* both __this_cpu_read() and rdtsc() should be on the same cpu */
-	get_cpu();
+	if (!ka->use_master_clock)
+		return false;
 
-	data->flags = 0;
-	if (ka->use_master_clock &&
-	    (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) {
-#ifdef CONFIG_X86_64
-		struct timespec64 ts;
+	if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc))
+		return false;
 
-		if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
-			data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
-			data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
-		} else
-#endif
-		data->host_tsc = rdtsc();
-
-		data->flags |= KVM_CLOCK_TSC_STABLE;
-		hv_clock.tsc_timestamp = ka->master_cycle_now;
-		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
-		kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL,
-				   &hv_clock.tsc_shift,
-				   &hv_clock.tsc_to_system_mul);
-		data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
+	data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
+	data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC |
+		       KVM_CLOCK_TSC_STABLE;
+
+	hv_clock.tsc_timestamp = ka->master_cycle_now;
+	hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
+
+	/*
+	 * Use the precomputed guest-TSC-based mul/shift so that the kvmclock
+	 * value matches what the guest computes from its own TSC.
+	 */
+	hv_clock.tsc_shift = ka->master_tsc_shift;
+	hv_clock.tsc_to_system_mul = ka->master_tsc_mul;
+
+	if (kvm_caps.has_tsc_control) {
+		u64 tsc_delta = data->host_tsc - ka->master_cycle_now;
+
+		tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio);
+		data->clock = hv_clock.system_time +
+			      pvclock_scale_delta(tsc_delta,
+						  hv_clock.tsc_to_system_mul,
+						  hv_clock.tsc_shift);
 	} else {
-		data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
+		data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
 	}
-
-	put_cpu();
+	return true;
+#else
+	return false;
+#endif
 }
 
 static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
@@ -1699,8 +1667,11 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
 	unsigned seq;
 
 	do {
+		data->flags = 0;
+
 		seq = read_seqcount_begin(&ka->pvclock_sc);
-		__get_kvmclock(kvm, data);
+		if (!__get_kvmclock_master_clock(kvm, data))
+			data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
 	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
 }
 
@@ -1762,36 +1733,45 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock,
 
 int kvm_guest_time_update(struct kvm_vcpu *v)
 {
+	u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns;
+	struct kvm_arch *ka __maybe_unused = &v->kvm->arch;
 	struct pvclock_vcpu_time_info hv_clock = {};
-	unsigned long flags, tgt_tsc_khz;
-	unsigned seq;
 	struct kvm_vcpu_arch *vcpu = &v->arch;
-	struct kvm_arch *ka = &v->kvm->arch;
 	s64 kernel_ns;
-	u64 tsc_timestamp, host_tsc;
-	bool use_master_clock;
-
-	kernel_ns = 0;
-	host_tsc = 0;
 
 	/*
 	 * If the host uses TSC clock, then passthrough TSC as stable
 	 * to the guest.
 	 */
+#ifdef CONFIG_X86_64
+	bool use_master_clock;
+	unsigned int seq;
+
 	do {
 		seq = read_seqcount_begin(&ka->pvclock_sc);
 		use_master_clock = ka->use_master_clock;
-		if (use_master_clock) {
-			host_tsc = ka->master_cycle_now;
-			kernel_ns = ka->master_kernel_ns;
+		if (!use_master_clock)
+			continue;
+
+		if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) {
+			use_master_clock = false;
+			continue;
 		}
-	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
 
-	/* Keep irq disabled to prevent changes to the clock */
-	local_irq_save(flags);
-	tgt_tsc_khz = get_cpu_tsc_khz();
-	if (unlikely(tgt_tsc_khz == 0)) {
-		local_irq_restore(flags);
+		master_tsc = ka->master_cycle_now;
+		master_ns = ka->master_kernel_ns;
+	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
+#else
+	const bool use_master_clock = false;
+#endif
+	/*
+	 * Ensure reading the TSC+frequency pair is done on the same CPU.  When
+	 * NOT using the master clock, the TSC frequency may vary between CPUs.
+	 */
+	preempt_disable();
+	tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ;
+	if (unlikely(tgt_tsc_hz == 0)) {
+		preempt_enable();
 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
 		return 1;
 	}
@@ -1820,28 +1800,43 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
 		}
 	}
 
-	local_irq_restore(flags);
+	/*
+	 * Refresh L1's last "observed" TSC to match the PV clock's timestamp,
+	 * e.g. so that the guest can't see a TSC that's behind the reference.
+	 */
+	vcpu->last_guest_tsc = tsc_timestamp;
+
+	preempt_enable();
 
 	/* With all the info we got, fill in the values */
 
 	if (kvm_caps.has_tsc_control) {
-		tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
+		tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz,
 					    v->arch.l1_tsc_scaling_ratio);
-		tgt_tsc_khz = tgt_tsc_khz ? : 1;
+		tgt_tsc_hz = tgt_tsc_hz ? : 1;
 	}
 
-	if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
-		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
+	if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) {
+		kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz,
 				   &vcpu->pvclock_tsc_shift,
 				   &vcpu->pvclock_tsc_mul);
-		vcpu->hw_tsc_khz = tgt_tsc_khz;
+		vcpu->hw_tsc_hz = tgt_tsc_hz;
 	}
 
 	hv_clock.tsc_shift = vcpu->pvclock_tsc_shift;
 	hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul;
-	hv_clock.tsc_timestamp = tsc_timestamp;
-	hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
-	vcpu->last_guest_tsc = tsc_timestamp;
+	/*
+	 * If the master clock is NOT in use, the reference time placed in the
+	 * hv_clock is "now".  If master clock is in use, the reference time is
+	 * the master clock's snapshot from some time in the past, not "now".
+	 */
+	if (use_master_clock) {
+		hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc);
+		hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset;
+	} else {
+		hv_clock.tsc_timestamp = tsc_timestamp;
+		hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
+	}
 
 	/* If the host uses TSC clocksource, then it is stable */
 	hv_clock.flags = 0;
@@ -1903,63 +1898,22 @@ int kvm_guest_time_update(struct kvm_vcpu *v)
  * wallclock and kvmclock times, and subtracting one from the other.
  *
  * Fall back to using their values at slightly different moments by
- * calling ktime_get_real_ns() and get_kvmclock_ns() separately.
+ * calling ktime_get_real_ns() and get_kvmclock() separately.
  */
 uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm)
 {
-#ifdef CONFIG_X86_64
-	struct pvclock_vcpu_time_info hv_clock;
-	struct kvm_arch *ka = &kvm->arch;
-	unsigned long seq, local_tsc_khz;
-	struct timespec64 ts;
-	uint64_t host_tsc;
-
-	do {
-		seq = read_seqcount_begin(&ka->pvclock_sc);
-
-		local_tsc_khz = 0;
-		if (!ka->use_master_clock)
-			break;
-
-		/*
-		 * The TSC read and the call to get_cpu_tsc_khz() must happen
-		 * on the same CPU.
-		 */
-		get_cpu();
-
-		local_tsc_khz = get_cpu_tsc_khz();
-
-		if (local_tsc_khz &&
-		    !kvm_get_walltime_and_clockread(&ts, &host_tsc))
-			local_tsc_khz = 0; /* Fall back to old method */
-
-		put_cpu();
-
-		/*
-		 * These values must be snapshotted within the seqcount loop.
-		 * After that, it's just mathematics which can happen on any
-		 * CPU at any time.
-		 */
-		hv_clock.tsc_timestamp = ka->master_cycle_now;
-		hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
+	struct kvm_clock_data data;
 
-	} while (read_seqcount_retry(&ka->pvclock_sc, seq));
+	get_kvmclock(kvm, &data);
 
 	/*
-	 * If the conditions were right, and obtaining the wallclock+TSC was
-	 * successful, calculate the KVM clock at the corresponding time and
-	 * subtract one from the other to get the guest's epoch in nanoseconds
-	 * since 1970-01-01.
+	 * If get_kvmclock() captured both wallclock and kvmclock from the
+	 * same TSC reading, use them for a precise epoch calculation.
 	 */
-	if (local_tsc_khz) {
-		kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC,
-				   &hv_clock.tsc_shift,
-				   &hv_clock.tsc_to_system_mul);
-		return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec -
-			__pvclock_read_cycles(&hv_clock, host_tsc);
-	}
-#endif
-	return ktime_get_real_ns() - get_kvmclock_ns(kvm);
+	if (data.flags & KVM_CLOCK_REALTIME)
+		return data.realtime - data.clock;
+
+	return ktime_get_real_ns() - data.clock;
 }
 
 /*
@@ -2583,7 +2537,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 			mark_tsc_unstable("KVM discovered backwards TSC");
 
 		if (kvm_check_tsc_unstable()) {
-			u64 offset = kvm_compute_l1_tsc_offset(vcpu,
+			u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(),
 						vcpu->arch.last_guest_tsc);
 			kvm_vcpu_write_tsc_offset(vcpu, offset);
 			if (!vcpu->arch.guest_tsc_protected)
@@ -3941,6 +3895,37 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
 		kvm_vcpu_kick(vcpu);
 }
 
+void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx)
+{
+	u16 pml_tail_index;
+	u64 *pml_buf;
+	int i;
+
+	/*
+	 * PML index always points to the next available PML buffer entity
+	 * unless PML log has just overflowed.
+	 */
+	pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1;
+
+	/*
+	 * PML log is written backwards: the CPU first writes the entry 511
+	 * then the entry 510, and so on.
+	 *
+	 * Read the entries in the same order they were written, to ensure that
+	 * the dirty ring is filled in the same order the CPU wrote them.
+	 */
+	pml_buf = page_address(vcpu->arch.pml_page);
+
+	for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) {
+		u64 gpa;
+
+		gpa = pml_buf[i];
+		WARN_ON_ONCE(gpa & (PAGE_SIZE - 1));
+		kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
+	}
+}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer);
+
 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
 			    struct kvm_enable_cap *cap)
 {
@@ -5089,7 +5074,8 @@ static int emulator_read_write_onepage(unsigned long addr, void *val,
 	 * operation using rep will only have the initial GPA from the NPF
 	 * occurred.
 	 */
-	if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&
+	if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) &&
+	    emulator_can_use_gpa(ctxt) &&
 	    (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {
 		gpa = ctxt->gpa_val;
 		ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
@@ -5938,7 +5924,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
 
 	kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
 
-	ctxt->gpa_available = false;
+	ctxt->gpa_access = 0;
 	ctxt->eflags = kvm_get_rflags(vcpu);
 	ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
 
@@ -6454,7 +6440,17 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 
 		/* With shadow page tables, cr2 contains a GVA or nGPA. */
 		if (vcpu->arch.mmu->root_role.direct) {
-			ctxt->gpa_available = true;
+			ctxt->gpa_access = ACC_READ_MASK;
+			/*
+			 * Always allow writes for guests with protected page
+			 * tables, as KVM can't walk the guest's page tables,
+			 * i.e. KVM can't get the RMW protections for a given
+			 * GVA to see if the write side of a RMW operation
+			 * should be allowed.
+			 */
+			if ((emulation_type & EMULTYPE_PF_WRITE) ||
+			    vcpu->kvm->arch.has_protected_page_tables)
+				ctxt->gpa_access |= ACC_WRITE_MASK;
 			ctxt->gpa_val = cr2_or_gpa;
 		}
 	} else {
@@ -7925,10 +7921,8 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
 		kvm->arch.apicv_inhibit_reasons = new;
 		if (new) {
 			unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE);
-			int idx = srcu_read_lock(&kvm->srcu);
 
 			kvm_zap_gfn_range(kvm, gfn, gfn+1);
-			srcu_read_unlock(&kvm->srcu, idx);
 		}
 	} else {
 		kvm->arch.apicv_inhibit_reasons = new;
@@ -8042,6 +8036,21 @@ static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
 	kvm_x86_call(set_apic_access_page_addr)(vcpu);
 }
 
+static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * Note, nr_memslots_dirty_logging can be changed concurrent with this
+	 * code, but in that case another update request will be made and so
+	 * the guest will never run with a stale PML value.
+	 */
+	bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging);
+
+	if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size))
+		return;
+
+	kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable);
+}
+
 /*
  * Called within kvm->srcu read side.
  * Returns 1 to let vcpu_run() continue the guest execution loop without
@@ -8060,7 +8069,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	bool req_immediate_exit = false;
 
 	if (kvm_request_pending(vcpu)) {
-		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
+		if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) {
 			r = -EIO;
 			goto out;
 		}
@@ -8072,6 +8081,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 
 		if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
 			if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) {
+				vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+				vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
+				vcpu->run->internal.ndata = 0;
+				kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
 				r = 0;
 				goto out;
 			}
@@ -8151,7 +8164,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		if (kvm_check_request(KVM_REQ_NMI, vcpu))
 			process_nmi(vcpu);
 		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
-			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
 			if (test_bit(vcpu->arch.pending_ioapic_eoi,
 				     vcpu->arch.ioapic_handled_vectors)) {
 				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
@@ -8208,7 +8220,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 			kvm_x86_call(recalc_intercepts)(vcpu);
 
 		if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
-			kvm_x86_call(update_cpu_dirty_logging)(vcpu);
+			kvm_update_cpu_dirty_logging(vcpu);
 
 		if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) {
 			kvm_vcpu_reset(vcpu, true);
@@ -9058,6 +9070,19 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 	}
 
 	kvm_set_mp_state(vcpu, mp_state->mp_state);
+
+	/*
+	 * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's
+	 * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state
+	 * other than HALTED.  The hardware state is sticky across VM-Exit and
+	 * VM-Enter and is not touched by any other ioctl, so a vCPU that halted
+	 * with HLT-exiting disabled stays wedged even after userspace rewrites
+	 * its registers, e.g. when a VMM emulates a machine reset.
+	 */
+	if (kvm_hlt_in_guest(vcpu->kvm) &&
+	    mp_state->mp_state != KVM_MP_STATE_HALTED)
+		kvm_x86_call(clear_hlt)(vcpu);
+
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 
 	ret = 0;
@@ -9440,6 +9465,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 		return;
 	vcpu_load(vcpu);
 	kvm_synchronize_tsc(vcpu, NULL);
+	if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
+		kvm_update_masterclock(vcpu->kvm);
 	vcpu_put(vcpu);
 
 	/* poll control enabled by default */
@@ -10140,7 +10167,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm,
 		}
 	}
 
-#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
 	kvm_mmu_init_memslot_memory_attributes(kvm, slot);
 #endif
 
@@ -10637,14 +10664,15 @@ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
 }
 
 #ifdef CONFIG_KVM_GUEST_MEMFD
-/*
- * KVM doesn't yet support initializing guest_memfd memory as shared for VMs
- * with private memory (the private vs. shared tracking needs to be moved into
- * guest_memfd).
- */
 bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
 {
-	return !kvm_arch_has_private_mem(kvm);
+	/*
+	 * INIT_SHARED is supported if in-place conversion is enabled, or if
+	 * the VM doesn't support private memory.  If the VM has private memory
+	 * and in-place conversion is disabled, then guest_memfd can _only_ be
+	 * used for private memory.
+	 */
+	return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm);
 }
 
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
@@ -10653,6 +10681,11 @@ int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 {
 	return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages);
 }
+
+void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages)
+{
+	kvm_x86_call(gmem_make_shared)(pfn, nr_pages);
+}
 #endif
 
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 0f5919b092e475..436e2fb1396c46 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -250,7 +250,7 @@ static inline bool is_noncanonical_invlpg_address(u64 la, struct kvm_vcpu *vcpu)
 static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu,
 					gva_t gva, gfn_t gfn, unsigned access)
 {
-	u64 gen = kvm_memslots(vcpu->kvm)->generation;
+	u64 gen = kvm_vcpu_memslots(vcpu)->generation;
 
 	if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
 		return;
@@ -267,7 +267,7 @@ static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu,
 
 static inline bool vcpu_match_mmio_gen(struct kvm_vcpu *vcpu)
 {
-	return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation;
+	return vcpu->arch.mmio_gen == kvm_vcpu_memslots(vcpu)->generation;
 }
 
 /*
@@ -335,7 +335,8 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio);
 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc);
 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier);
 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier);
-u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc);
+u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc,
+			      u64 target_tsc);
 void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset);
 
 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
@@ -407,6 +408,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
  * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case
  *		 the CR2/GPA value pass on the stack is valid.
  *
+ * EMULTYPE_PF_WRITE - Set with EMULTYPE_PF when hardware reports a write access.
+ *
  * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility
  *				 state and inject single-step #DBs after skipping
  *				 an instruction (after completing userspace I/O).
@@ -445,6 +448,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7)
 #define EMULTYPE_WRITE_PF_TO_SP	    (1 << 8)
 #define EMULTYPE_SKIP_SOFT_INT	    (1 << 9)
+#define EMULTYPE_PF_WRITE	    (1 << 10)
 
 #define EMULTYPE_SET_SOFT_INT_VECTOR(v)	((u32)((v) & 0xff) << 16)
 #define EMULTYPE_GET_SOFT_INT_VECTOR(e)	(((e) >> 16) & 0xff)
@@ -900,4 +904,11 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl,
 
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
 
+/* Support for PML */
+#define PML_LOG_NR_ENTRIES	512
+/* PML is written backwards: this is the first entry written by the CPU */
+#define PML_HEAD_INDEX		(PML_LOG_NR_ENTRIES-1)
+
+void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx);
+
 #endif
diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c
index d113bd0f1c3d23..6aa00f746cba39 100644
--- a/arch/x86/kvm/xen.c
+++ b/arch/x86/kvm/xen.c
@@ -73,7 +73,7 @@ static int kvm_xen_shared_info_init(struct kvm *kvm)
 	BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00);
 	BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c);
 
-	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
+	if (kvm_xen_has_64bit_shinfo(kvm)) {
 		struct shared_info *shinfo = gpc->khva;
 
 		wc_sec_hi = &shinfo->wc_sec_hi;
@@ -389,7 +389,7 @@ static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic)
 	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) !=
 		     sizeof(vx->runstate_times));
 
-	if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
+	if (kvm_xen_has_64bit_shinfo(v->kvm)) {
 		user_len = sizeof(struct vcpu_runstate_info);
 		times_ofs = offsetof(struct vcpu_runstate_info,
 				     state_entry_time);
@@ -676,28 +676,32 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v)
 	}
 
 	/* Now gpc->khva is a valid kernel address for the vcpu_info */
-	if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) {
+	if (kvm_xen_has_64bit_shinfo(v->kvm)) {
 		struct vcpu_info *vi = gpc->khva;
+		void *vi_pending_sel = &vi->evtchn_pending_sel;
 
-		asm volatile(LOCK_PREFIX "orq %0, %1\n"
-			     "notq %0\n"
-			     LOCK_PREFIX "andq %0, %2\n"
-			     : "=r" (evtchn_pending_sel),
-			       "+m" (vi->evtchn_pending_sel),
-			       "+m" (v->arch.xen.evtchn_pending_sel)
-			     : "0" (evtchn_pending_sel));
+		if (IS_ALIGNED((unsigned long)vi_pending_sel, sizeof(u64))) {
+			atomic64_or(evtchn_pending_sel, vi_pending_sel);
+		} else {
+			atomic_or(evtchn_pending_sel, vi_pending_sel);
+			/*
+			 * The cast keeps the shift well-defined on 32-bit,
+			 * where evtchn_pending_sel is 32 bits wide and this
+			 * branch is unreachable anyway (this is inside
+			 * kvm_xen_has_64bit_shinfo(), which is gated on
+			 * IS_ENABLED(CONFIG_64BIT)).
+			 */
+			atomic_or((u64)evtchn_pending_sel >> 32,
+				  vi_pending_sel + 4);
+		}
+
+		atomic64_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel);
 		WRITE_ONCE(vi->evtchn_upcall_pending, 1);
 	} else {
-		u32 evtchn_pending_sel32 = evtchn_pending_sel;
 		struct compat_vcpu_info *vi = gpc->khva;
 
-		asm volatile(LOCK_PREFIX "orl %0, %1\n"
-			     "notl %0\n"
-			     LOCK_PREFIX "andl %0, %2\n"
-			     : "=r" (evtchn_pending_sel32),
-			       "+m" (vi->evtchn_pending_sel),
-			       "+m" (v->arch.xen.evtchn_pending_sel)
-			     : "0" (evtchn_pending_sel32));
+		atomic_or(evtchn_pending_sel, (void *)&vi->evtchn_pending_sel);
+		atomic_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel);
 		WRITE_ONCE(vi->evtchn_upcall_pending, 1);
 	}
 
@@ -728,6 +732,16 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v)
 	BUILD_BUG_ON(sizeof(rc) !=
 		     sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending));
 
+	/*
+	 * kvm_gpc_check() checks the memslot generation, so kvm->srcu must be
+	 * held. Most callers hold it already, but this is also reached from
+	 * kvm_emulate_halt() on the VM-Exit path and from kvm_vcpu_block(),
+	 * where vcpu_enter_guest() has already dropped the vCPU's SRCU lock.
+	 * Taking SRCU does not sleep, so it is safe even in the atomic case
+	 * which is handled below.
+	 */
+	guard(srcu)(&v->kvm->srcu);
+
 	read_lock_irqsave(&gpc->lock, flags);
 	while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) {
 		read_unlock_irqrestore(&gpc->lock, flags);
@@ -941,6 +955,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
 				break;
 			}
 
+			r = -ENXIO;
+			if (!IS_ALIGNED(data->u.gpa, sizeof(u32)))
+				break;
+
 			r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_info_cache,
 					     data->u.gpa, sizeof(struct vcpu_info));
 		} else {
@@ -950,6 +968,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
 				break;
 			}
 
+			r = -ENXIO;
+			if (!IS_ALIGNED(data->u.hva, sizeof(u32)))
+				break;
+
 			r = kvm_gpc_activate_hva(&vcpu->arch.xen.vcpu_info_cache,
 						 data->u.hva, sizeof(struct vcpu_info));
 		}
@@ -993,7 +1015,7 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data)
 		 * address, that's actually OK. kvm_xen_update_runstate_guest()
 		 * will cope.
 		 */
-		if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode)
+		if (kvm_xen_has_64bit_shinfo(vcpu->kvm))
 			sz = sizeof(struct vcpu_runstate_info);
 		else
 			sz = sizeof(struct compat_vcpu_runstate_info);
@@ -1439,16 +1461,21 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu)
 	return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result);
 }
 
-static inline int max_evtchn_port(struct kvm *kvm)
+static inline int max_evtchn_port(bool has_64bit_shinfo)
 {
-	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode)
+	if (has_64bit_shinfo)
 		return EVTCHN_2L_NR_CHANNELS;
 	else
 		return COMPAT_EVTCHN_2L_NR_CHANNELS;
 }
 
-static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports,
-			       evtchn_port_t *ports)
+static inline int kvm_max_evtchn_port(struct kvm *kvm)
+{
+	return max_evtchn_port(kvm_xen_has_64bit_shinfo(kvm));
+}
+
+static bool wait_pending_event(struct kvm_vcpu *vcpu, bool has_64bit_shinfo,
+			       int nr_ports, evtchn_port_t *ports)
 {
 	struct kvm *kvm = vcpu->kvm;
 	struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
@@ -1463,7 +1490,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports,
 		goto out_rcu;
 
 	ret = false;
-	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
+	if (has_64bit_shinfo) {
 		struct shared_info *shinfo = gpc->khva;
 		pending_bits = (unsigned long *)&shinfo->evtchn_pending;
 	} else {
@@ -1485,9 +1512,10 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports,
 	return ret;
 }
 
-static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode,
+static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit,
 				 u64 param, u64 *r)
 {
+	bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(vcpu->kvm);
 	struct sched_poll sched_poll;
 	evtchn_port_t port, *ports;
 	struct x86_exception e;
@@ -1497,7 +1525,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode,
 	    !(vcpu->kvm->arch.xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND))
 		return false;
 
-	if (IS_ENABLED(CONFIG_64BIT) && !longmode) {
+	if (IS_ENABLED(CONFIG_64BIT) && !is_64bit) {
 		struct compat_sched_poll sp32;
 
 		/* Sanity check that the compat struct definition is correct */
@@ -1546,20 +1574,20 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode,
 	}
 
 	for (i = 0; i < sched_poll.nr_ports; i++) {
-		if (ports[i] >= max_evtchn_port(vcpu->kvm)) {
+		if (ports[i] >= max_evtchn_port(has_64bit_shinfo)) {
 			*r = -EINVAL;
 			goto out;
 		}
 	}
 
 	if (sched_poll.nr_ports == 1)
-		vcpu->arch.xen.poll_evtchn = port;
+		WRITE_ONCE(vcpu->arch.xen.poll_evtchn, port);
 	else
-		vcpu->arch.xen.poll_evtchn = -1;
+		WRITE_ONCE(vcpu->arch.xen.poll_evtchn, -1);
 
 	set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask);
 
-	if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) {
+	if (!wait_pending_event(vcpu, has_64bit_shinfo, sched_poll.nr_ports, ports)) {
 		kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED);
 
 		if (sched_poll.timeout)
@@ -1574,7 +1602,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode,
 		kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
 	}
 
-	vcpu->arch.xen.poll_evtchn = 0;
+	WRITE_ONCE(vcpu->arch.xen.poll_evtchn, 0);
 	*r = 0;
 out:
 	/* Really, this is only needed in case of timeout */
@@ -1594,12 +1622,12 @@ static void cancel_evtchn_poll(struct timer_list *t)
 	kvm_vcpu_kick(vcpu);
 }
 
-static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode,
+static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool is_64bit,
 				   int cmd, u64 param, u64 *r)
 {
 	switch (cmd) {
 	case SCHEDOP_poll:
-		if (kvm_xen_schedop_poll(vcpu, longmode, param, r))
+		if (kvm_xen_schedop_poll(vcpu, is_64bit, param, r))
 			return true;
 		fallthrough;
 	case SCHEDOP_yield:
@@ -1618,7 +1646,7 @@ struct compat_vcpu_set_singleshot_timer {
     uint32_t flags;
 } __attribute__((packed));
 
-static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd,
+static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd,
 				  int vcpu_id, u64 param, u64 *r)
 {
 	struct vcpu_set_singleshot_timer oneshot;
@@ -1662,7 +1690,7 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd,
 		BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) !=
 			     sizeof_field(struct vcpu_set_singleshot_timer, flags));
 
-		if (kvm_read_guest_virt(vcpu, param, &oneshot, longmode ? sizeof(oneshot) :
+		if (kvm_read_guest_virt(vcpu, param, &oneshot, is_64bit ? sizeof(oneshot) :
 					sizeof(struct compat_vcpu_set_singleshot_timer), &e)) {
 			*r = -EFAULT;
 			return true;
@@ -1694,7 +1722,7 @@ static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout,
 
 int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
 {
-	bool longmode;
+	bool is_64bit;
 	u64 input, params[6], r = -ENOSYS;
 	bool handled = false;
 	u8 cpl;
@@ -1704,8 +1732,8 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
 	    kvm_hv_hypercall_enabled(vcpu))
 		return kvm_hv_hypercall(vcpu);
 
-	longmode = is_64_bit_hypercall(vcpu);
-	if (!longmode) {
+	is_64bit = is_64_bit_hypercall(vcpu);
+	if (!is_64bit) {
 		input = kvm_eax_read(vcpu);
 		params[0] = kvm_ebx_read(vcpu);
 		params[1] = kvm_ecx_read(vcpu);
@@ -1751,17 +1779,17 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
 			handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r);
 		break;
 	case __HYPERVISOR_sched_op:
-		handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0],
+		handled = kvm_xen_hcall_sched_op(vcpu, is_64bit, params[0],
 						 params[1], &r);
 		break;
 	case __HYPERVISOR_vcpu_op:
-		handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1],
+		handled = kvm_xen_hcall_vcpu_op(vcpu, is_64bit, params[0], params[1],
 						params[2], &r);
 		break;
 	case __HYPERVISOR_set_timer_op: {
 		u64 timeout = params[0];
 		/* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */
-		if (!longmode)
+		if (!is_64bit)
 			timeout |= params[1] << 32;
 		handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r);
 		break;
@@ -1776,7 +1804,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
 handle_in_userspace:
 	vcpu->run->exit_reason = KVM_EXIT_XEN;
 	vcpu->run->xen.type = KVM_EXIT_XEN_HCALL;
-	vcpu->run->xen.u.hcall.longmode = longmode;
+	vcpu->run->xen.u.hcall.longmode = is_64bit;
 	vcpu->run->xen.u.hcall.cpl = cpl;
 	vcpu->run->xen.u.hcall.input = input;
 	vcpu->run->xen.u.hcall.params[0] = params[0];
@@ -1794,7 +1822,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu)
 
 static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port)
 {
-	int poll_evtchn = vcpu->arch.xen.poll_evtchn;
+	int poll_evtchn = READ_ONCE(vcpu->arch.xen.poll_evtchn);
 
 	if ((poll_evtchn == port || poll_evtchn == -1) &&
 	    test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) {
@@ -1816,8 +1844,9 @@ static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port)
 int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
 {
 	struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache;
+	bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(kvm);
+	unsigned long *pending_bits, *mask_bits, vi_pending_sel_ofs;
 	struct kvm_vcpu *vcpu;
-	unsigned long *pending_bits, *mask_bits;
 	unsigned long flags;
 	int port_word_bit;
 	bool kick_vcpu = false;
@@ -1833,7 +1862,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
 		WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx);
 	}
 
-	if (xe->port >= max_evtchn_port(kvm))
+	if (xe->port >= max_evtchn_port(has_64bit_shinfo))
 		return -EINVAL;
 
 	rc = -EWOULDBLOCK;
@@ -1844,16 +1873,23 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
 	if (!kvm_gpc_check(gpc, PAGE_SIZE))
 		goto out_rcu;
 
-	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
+	if (has_64bit_shinfo) {
 		struct shared_info *shinfo = gpc->khva;
 		pending_bits = (unsigned long *)&shinfo->evtchn_pending;
 		mask_bits = (unsigned long *)&shinfo->evtchn_mask;
 		port_word_bit = xe->port / 64;
+
+		vi_pending_sel_ofs = offsetof(struct vcpu_info, evtchn_pending_sel);
 	} else {
 		struct compat_shared_info *shinfo = gpc->khva;
 		pending_bits = (unsigned long *)&shinfo->evtchn_pending;
 		mask_bits = (unsigned long *)&shinfo->evtchn_mask;
 		port_word_bit = xe->port / 32;
+
+		vi_pending_sel_ofs = offsetof(struct compat_vcpu_info, evtchn_pending_sel);
+
+		/* test_and_set_bit() needs 64-bit alignment, but that's OK */
+		BUILD_BUG_ON(offsetof(struct compat_shared_info, evtchn_pending) & 7);
 	}
 
 	/*
@@ -1869,6 +1905,8 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
 		rc = -ENOTCONN; /* Masked */
 		kvm_xen_check_poller(vcpu, xe->port);
 	} else {
+		bool old;
+
 		rc = 1; /* Delivered to the bitmap in shared_info. */
 		/* Now switch to the vCPU's vcpu_info to set the index and pending_sel */
 		read_unlock_irqrestore(&gpc->lock, flags);
@@ -1885,19 +1923,29 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm)
 			goto out_rcu;
 		}
 
-		if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) {
-			struct vcpu_info *vcpu_info = gpc->khva;
-			if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) {
-				WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1);
-				kick_vcpu = true;
-			}
-		} else {
-			struct compat_vcpu_info *vcpu_info = gpc->khva;
-			if (!test_and_set_bit(port_word_bit,
-					      (unsigned long *)&vcpu_info->evtchn_pending_sel)) {
-				WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1);
-				kick_vcpu = true;
-			}
+		/*
+		 * Explicitly use a 32-bit btsl instead of test_and_set_bit(),
+		 * which would use btsq on x86-64. The vcpu_info is guest-
+		 * controlled and only required to be 32-bit aligned, so a
+		 * 64-bit access could generate a split-lock #AC.
+		 *
+		 * Note, this does not apply to the test_and_set_bit() on
+		 * pending_bits above: that is in the per-VM shared_info, which
+		 * is page aligned, so the access is guaranteed to be 64-bit
+		 * aligned.
+		 */
+		old = GEN_BINARY_RMWcc(LOCK_PREFIX "btsl",
+				       *(u32 *)(gpc->khva + vi_pending_sel_ofs),
+				       c, "Ir", port_word_bit);
+		if (!old) {
+			struct vcpu_info *vi = gpc->khva;
+
+			/* No need for compat handling */
+			BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) !=
+				     offsetof(struct compat_vcpu_info, evtchn_upcall_pending));
+
+			WRITE_ONCE(vi->evtchn_upcall_pending, 1);
+			kick_vcpu = true;
 		}
 
 		/* For the per-vCPU lapic vector, deliver it as MSI. */
@@ -1995,7 +2043,7 @@ int kvm_xen_setup_evtchn(struct kvm *kvm,
 	struct kvm_vcpu *vcpu;
 
 	/*
-	 * Don't check for the port being within range of max_evtchn_port().
+	 * Don't check for the port being within range of kvm_max_evtchn_port().
 	 * Userspace can configure what ever targets it likes; events just won't
 	 * be delivered if/while the target is invalid, just like userspace can
 	 * configure MSIs which target non-existent APICs.
@@ -2004,8 +2052,8 @@ int kvm_xen_setup_evtchn(struct kvm *kvm,
 	 * can be restored *independently* of other things like creating vCPUs,
 	 * without imposing an ordering dependency on userspace.  In this
 	 * particular case, the problematic ordering would be with setting the
-	 * Xen 'long mode' flag, which changes max_evtchn_port() to allow 4096
-	 * instead of 1024 event channels.
+	 * Xen 'long mode' flag, which changes kvm_max_evtchn_port() to allow
+	 * 4096 instead of 1024 event channels.
 	 */
 
 	/* We only support 2 level event channels for now */
@@ -2042,7 +2090,7 @@ int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn *
 	struct kvm_xen_evtchn e;
 	int ret;
 
-	if (!uxe->port || uxe->port >= max_evtchn_port(kvm))
+	if (!uxe->port || uxe->port >= kvm_max_evtchn_port(kvm))
 		return -EINVAL;
 
 	/* We only support 2 level event channels for now */
@@ -2093,7 +2141,7 @@ static int kvm_xen_eventfd_update(struct kvm *kvm,
 
 	/* Protect writes to evtchnfd as well as the idr lookup.  */
 	mutex_lock(&kvm->arch.xen.xen_lock);
-	evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port);
+	evtchnfd = xa_load(&kvm->arch.xen.evtchn_ports, port);
 
 	ret = -ENOENT;
 	if (!evtchnfd)
@@ -2152,7 +2200,7 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm,
 
 	case EVTCHNSTAT_interdomain:
 		if (data->u.evtchn.deliver.port.port) {
-			if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm))
+			if (data->u.evtchn.deliver.port.port >= kvm_max_evtchn_port(kvm))
 				goto out_noeventfd; /* -EINVAL */
 		} else {
 			eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd);
@@ -2187,13 +2235,13 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm,
 	}
 
 	mutex_lock(&kvm->arch.xen.xen_lock);
-	ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1,
+	ret = xa_insert(&kvm->arch.xen.evtchn_ports, port, evtchnfd,
 			GFP_KERNEL);
 	mutex_unlock(&kvm->arch.xen.xen_lock);
-	if (ret >= 0)
+	if (!ret)
 		return 0;
 
-	if (ret == -ENOSPC)
+	if (ret == -EBUSY)
 		ret = -EEXIST;
 out:
 	if (eventfd)
@@ -2208,7 +2256,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port)
 	struct evtchnfd *evtchnfd;
 
 	mutex_lock(&kvm->arch.xen.xen_lock);
-	evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port);
+	evtchnfd = xa_erase(&kvm->arch.xen.evtchn_ports, port);
 	mutex_unlock(&kvm->arch.xen.xen_lock);
 
 	if (!evtchnfd)
@@ -2224,7 +2272,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port)
 static int kvm_xen_eventfd_reset(struct kvm *kvm)
 {
 	struct evtchnfd *evtchnfd, **all_evtchnfds;
-	int i;
+	unsigned long i;
 	int n = 0;
 
 	mutex_lock(&kvm->arch.xen.xen_lock);
@@ -2234,7 +2282,7 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm)
 	 * critical section, first collect all the evtchnfd objects
 	 * in an array as they are removed from evtchn_ports.
 	 */
-	idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i)
+	xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd)
 		n++;
 
 	all_evtchnfds = kmalloc_objs(struct evtchnfd *, n);
@@ -2244,9 +2292,9 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm)
 	}
 
 	n = 0;
-	idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
+	xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) {
 		all_evtchnfds[n++] = evtchnfd;
-		idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port);
+		xa_erase(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port);
 	}
 	mutex_unlock(&kvm->arch.xen.xen_lock);
 
@@ -2270,7 +2318,7 @@ static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data
 	if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET)
 		return kvm_xen_eventfd_reset(kvm);
 
-	if (!port || port >= max_evtchn_port(kvm))
+	if (!port || port >= kvm_max_evtchn_port(kvm))
 		return -EINVAL;
 
 	if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN)
@@ -2297,12 +2345,10 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r)
 	}
 
 	/*
-	 * evtchnfd is protected by kvm->srcu; the idr lookup instead
-	 * is protected by RCU.
+	 * evtchnfd is protected by kvm->srcu; the xa_load is RCU-safe
+	 * internally, no explicit rcu_read_lock() needed.
 	 */
-	rcu_read_lock();
-	evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port);
-	rcu_read_unlock();
+	evtchnfd = xa_load(&vcpu->kvm->arch.xen.evtchn_ports, send.port);
 	if (!evtchnfd)
 		return false;
 
@@ -2349,23 +2395,23 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu)
 void kvm_xen_init_vm(struct kvm *kvm)
 {
 	mutex_init(&kvm->arch.xen.xen_lock);
-	idr_init(&kvm->arch.xen.evtchn_ports);
+	xa_init(&kvm->arch.xen.evtchn_ports);
 	kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm);
 }
 
 void kvm_xen_destroy_vm(struct kvm *kvm)
 {
 	struct evtchnfd *evtchnfd;
-	int i;
+	unsigned long i;
 
 	kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache);
 
-	idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) {
+	xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) {
 		if (!evtchnfd->deliver.port.port)
 			eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx);
 		kfree(evtchnfd);
 	}
-	idr_destroy(&kvm->arch.xen.evtchn_ports);
+	xa_destroy(&kvm->arch.xen.evtchn_ports);
 
 	if (kvm->arch.xen.hvm_config.msr)
 		static_branch_slow_dec_deferred(&kvm_xen_enabled);
diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h
index f372855857a889..9d04e350bdb1f6 100644
--- a/arch/x86/kvm/xen.h
+++ b/arch/x86/kvm/xen.h
@@ -235,6 +235,11 @@ struct compat_shared_info {
 #define COMPAT_EVTCHN_2L_NR_CHANNELS (8 *				\
 				      sizeof_field(struct compat_shared_info, \
 						   evtchn_pending))
+
+/* Latched VM-wide mode; the KVM equivalent of Xen's !has_32bit_shinfo(). */
+#define kvm_xen_has_64bit_shinfo(kvm) \
+	(IS_ENABLED(CONFIG_64BIT) && READ_ONCE((kvm)->arch.xen.long_mode))
+
 struct compat_vcpu_runstate_info {
     int state;
     uint64_t state_entry_time;
diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index 4652487b5572b0..b1d5c5a0ffb41d 100644
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -719,6 +719,21 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star
 		return new;
 
 	end = start + npg * PAGE_SIZE - 1;
+
+	/*
+	 * If the kernel text is still RWX, gently complain, this could be a
+	 * false positive.
+	 * Once debug_checkwx() becomes mandatory for CONFIG_STRICT_KERNEL_RWX,
+	 * the warning can be removed completely.
+	 */
+	if (!kernel_set_to_readonly) {
+		pr_warn_once("CPA detected W^X violation: %016llx -> %016llx range: 0x%016lx - 0x%016lx PFN %lx\n",
+		  (unsigned long long)pgprot_val(old),
+		  (unsigned long long)pgprot_val(new),
+		  start, end, pfn);
+		return new;
+	}
+
 	WARN_ONCE(1, "CPA detected W^X violation: %016llx -> %016llx range: 0x%016lx - 0x%016lx PFN %lx\n",
 		  (unsigned long long)pgprot_val(old),
 		  (unsigned long long)pgprot_val(new),
@@ -754,7 +769,7 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
 
 	*level = PG_LEVEL_512G;
 	*nx |= pgd_flags(*pgd) & _PAGE_NX;
-	*rw &= pgd_flags(*pgd) & _PAGE_RW;
+	*rw &= !!(pgd_flags(*pgd) & _PAGE_RW);
 
 	p4d = p4d_offset(pgd, address);
 	if (p4d_none(*p4d))
@@ -765,7 +780,7 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
 
 	*level = PG_LEVEL_1G;
 	*nx |= p4d_flags(*p4d) & _PAGE_NX;
-	*rw &= p4d_flags(*p4d) & _PAGE_RW;
+	*rw &= !!(p4d_flags(*p4d) & _PAGE_RW);
 
 	pud = pud_offset(p4d, address);
 	if (pud_none(*pud))
@@ -776,7 +791,7 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
 
 	*level = PG_LEVEL_2M;
 	*nx |= pud_flags(*pud) & _PAGE_NX;
-	*rw &= pud_flags(*pud) & _PAGE_RW;
+	*rw &= !!(pud_flags(*pud) & _PAGE_RW);
 
 	pmd = pmd_offset(pud, address);
 	if (pmd_none(*pmd))
@@ -787,7 +802,7 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
 
 	*level = PG_LEVEL_4K;
 	*nx |= pmd_flags(*pmd) & _PAGE_NX;
-	*rw &= pmd_flags(*pmd) & _PAGE_RW;
+	*rw &= !!(pmd_flags(*pmd) & _PAGE_RW);
 
 	return pte_offset_kernel(pmd, address);
 }
@@ -2685,22 +2700,14 @@ static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flag
 	return __change_page_attr_set_clr(&cpa, 1);
 }
 
-int set_direct_map_invalid_noflush(struct page *page)
-{
-	return __set_pages_np(page, 1, 0);
-}
-
-int set_direct_map_default_noflush(struct page *page)
+int set_direct_map_invalid_noflush(struct page *page, unsigned int nr)
 {
-	return __set_pages_p(page, 1, 0);
+	return __set_pages_np(page, nr, 0);
 }
 
-int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid)
+int set_direct_map_default_noflush(struct page *page, unsigned int nr)
 {
-	if (valid)
-		return __set_pages_p(page, nr, 0);
-
-	return __set_pages_np(page, nr, 0);
+	return __set_pages_p(page, nr, 0);
 }
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index cb03f5a2b24338..4a105f283cfbb6 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -705,47 +705,37 @@ int pmd_clear_huge(pmd_t *pmd)
 }
 
 #ifdef CONFIG_X86_64
-/**
- * pud_free_pmd_page - Clear PUD entry and free PMD page
- * @pud: Pointer to a PUD
- * @addr: Virtual address associated with PUD
- *
- * Context: The PUD range has been unmapped and TLB purged.
- * Return: 1 if clearing the entry succeeded. 0 otherwise.
- *
- * NOTE: Callers must allow a single page allocation.
+/*
+ * Given a PUD poitner, detach and free the pointed-to
+ * PMD page and any PTE page children. The entire range
+ * under the PUD must not have any valid translations
+ * and the TLB must have already been flushed.
  */
 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
 {
-	pmd_t *pmd, *pmd_sv;
 	struct ptdesc *pt;
+	pmd_t *pmd;
 	int i;
 
 	pmd = pud_pgtable(*pud);
-	pmd_sv = (pmd_t *)__get_free_page(GFP_KERNEL);
-	if (!pmd_sv)
-		return 0;
-
-	for (i = 0; i < PTRS_PER_PMD; i++) {
-		pmd_sv[i] = pmd[i];
-		if (!pmd_none(pmd[i]))
-			pmd_clear(&pmd[i]);
-	}
 
+	/* Detach the PMD page: */
 	pud_clear(pud);
 
-	/* INVLPG to clear all paging-structure caches */
+	/*
+	 * PMD and all its descendents are unreachable
+	 * via normal page walks. Make them unreachable
+	 * in cached mid-level walks too:
+	 */
 	flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1);
 
 	for (i = 0; i < PTRS_PER_PMD; i++) {
-		if (!pmd_none(pmd_sv[i])) {
-			pt = page_ptdesc(pmd_page(pmd_sv[i]));
+		if (!pmd_none(pmd[i])) {
+			pt = page_ptdesc(pmd_page(pmd[i]));
 			pagetable_dtor_free(pt);
 		}
 	}
 
-	free_page((unsigned long)pmd_sv);
-
 	pmd_free(&init_mm, pmd);
 
 	return 1;
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 598f553cc8713c..31055ee6f1de8e 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -688,5 +688,5 @@ void pti_finalize(void)
 	pti_clone_entry_text(true);
 	pti_clone_kernel_text();
 
-	debug_checkwx_user();
+	pgtable_checkwx_user();
 }
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 2853e87797a784..e38466ad44b20b 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -9,6 +9,7 @@
 #include <linux/filter.h>
 #include <linux/if_vlan.h>
 #include <linux/bitfield.h>
+#include <linux/init.h>
 #include <linux/bpf.h>
 #include <linux/bpf_verifier.h>
 #include <linux/memory.h>
@@ -22,8 +23,34 @@
 #include <asm/unwind.h>
 #include <asm/cfi.h>
 
+#if IS_ENABLED(CONFIG_BPF_JIT_KASAN)
+void __asan_load1(void *p);
+void __asan_store1(void *p);
+void __asan_load2(void *p);
+void __asan_store2(void *p);
+void __asan_load4(void *p);
+void __asan_store4(void *p);
+void __asan_load8(void *p);
+void __asan_store8(void *p);
+#endif
+
 static bool all_callee_regs_used[4] = {true, true, true, true};
 
+static void *trampoline_size_image;
+
+static int __init init_trampoline_size_image(void)
+{
+	/*
+	 * The generated trampoline contains calls and jumps with 32bit relative
+	 * offsets, so the scratch image must be in the execmem range.
+	 * On x86, module data and executable memory share the same address range,
+	 * so using EXECMEM_MODULE_DATA to get writable memory.
+	 */
+	trampoline_size_image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE);
+	return trampoline_size_image ? 0 : -ENOMEM;
+}
+late_initcall(init_trampoline_size_image);
+
 static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
 {
 	if (len == 1)
@@ -722,6 +749,46 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)
 	*pprog = prog;
 }
 
+static void __emit_indirect_call(u8 **pprog, int reg, bool ereg)
+{
+	u8 *prog = *pprog;
+
+	if (ereg)
+		EMIT1(0x41);
+
+	EMIT2(0xFF, 0xD0 + reg);
+
+	*pprog = prog;
+}
+
+/* call *bpf_reg */
+static int emit_indirect_call(u8 **pprog, int bpf_reg, u8 *ip)
+{
+	u8 *prog = *pprog;
+	int reg = reg2hex[bpf_reg];
+	bool ereg = is_ereg(bpf_reg);
+	int err = 0;
+
+	if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) {
+		OPTIMIZER_HIDE_VAR(reg);
+		err = emit_call(&prog, its_static_thunk(reg + 8*ereg), ip);
+	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
+		EMIT_LFENCE();
+		__emit_indirect_call(&prog, reg, ereg);
+	} else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {
+		OPTIMIZER_HIDE_VAR(reg);
+		if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
+			err = emit_call(&prog, &__x86_indirect_call_thunk_array[reg + 8*ereg], ip);
+		else
+			err = emit_call(&prog, &__x86_indirect_thunk_array[reg + 8*ereg], ip);
+	} else {
+		__emit_indirect_call(&prog, reg, ereg);
+	}
+
+	*pprog = prog;
+	return err;
+}
+
 static void emit_return(u8 **pprog, u8 *ip)
 {
 	u8 *prog = *pprog;
@@ -1111,6 +1178,93 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64)
 	*pprog = prog;
 }
 
+static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog,
+			    u32 addr_reg, s32 off, u32 bpf_size, u8 *ip,
+			    bool is_write)
+{
+#ifdef CONFIG_BPF_JIT_KASAN
+	u8 *prog = *pprog;
+	void *kasan_func;
+
+	if (!env)
+		return 0;
+
+	if (addr_reg == BPF_REG_FP || addr_reg == BPF_REG_PARAMS)
+		return 0;
+
+	/* Derive KASAN check function from access type and size */
+	switch (bpf_size) {
+	case BPF_B:
+		kasan_func = is_write ? __asan_store1 : __asan_load1;
+		break;
+	case BPF_H:
+		kasan_func = is_write ? __asan_store2 : __asan_load2;
+		break;
+	case BPF_W:
+		kasan_func = is_write ? __asan_store4 : __asan_load4;
+		break;
+	case BPF_DW:
+		kasan_func = is_write ? __asan_store8 : __asan_load8;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Save rax */
+	EMIT1(0x50);
+	/* Save rcx */
+	EMIT1(0x51);
+	/* Save rdx */
+	EMIT1(0x52);
+	/* Save rsi */
+	EMIT1(0x56);
+	/* Save rdi */
+	EMIT1(0x57);
+	/* Save r8 */
+	EMIT2(0x41, 0x50);
+	/* Save r9 */
+	EMIT2(0x41, 0x51);
+	/* Save r10 */
+	EMIT2(0x41, 0x52);
+	/* Save r11 */
+	EMIT2(0x41, 0x53);
+	/* mov rdi, addr_reg */
+	EMIT_mov(BPF_REG_1, addr_reg);
+
+	/* add rdi, off (if offset is non-zero) */
+	if (off) {
+		if (is_imm8(off)) {
+			/* add rdi, imm8 */
+			EMIT4(0x48, 0x83, 0xC7, (u8)off);
+		} else {
+			/* add rdi, imm32 */
+			EMIT3_off32(0x48, 0x81, 0xC7, off);
+		}
+	}
+
+	/* Adjust ip to account for the instrumentation generated so far */
+	ip += (prog - *pprog);
+	/* We emit a call, so update call depth counting */
+	ip += x86_call_depth_emit_accounting(&prog, kasan_func, ip);
+	/* call kasan_func */
+	if (emit_call(&prog, kasan_func, ip))
+		return -ERANGE;
+
+	EMIT2(0x41, 0x5B);
+	EMIT2(0x41, 0x5A);
+	EMIT2(0x41, 0x59);
+	EMIT2(0x41, 0x58);
+	EMIT1(0x5F);
+	EMIT1(0x5E);
+	EMIT1(0x5A);
+	EMIT1(0x59);
+	EMIT1(0x58);
+
+	*pprog = prog;
+#endif /* CONFIG_BPF_JIT_KASAN */
+	return 0;
+}
+
 /* LDX: dst_reg = *(u8*)(src_reg + off) */
 static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
 {
@@ -1316,6 +1470,46 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int
 	*pprog = prog;
 }
 
+/* ST: *(u8*)(dst_reg + off) = imm */
+static void emit_st(u8 **pprog, struct bpf_insn *insn, u32 dst_reg,
+		    s32 insn_off)
+{
+	s32 imm32 = insn->imm;
+	u8 *prog = *pprog;
+
+	switch (BPF_SIZE(insn->code)) {
+	case BPF_B:
+		if (is_ereg(dst_reg))
+			EMIT2(0x41, 0xC6);
+		else
+			EMIT1(0xC6);
+		break;
+	case BPF_H:
+		if (is_ereg(dst_reg))
+			EMIT3(0x66, 0x41, 0xC7);
+		else
+			EMIT2(0x66, 0xC7);
+		break;
+	case BPF_W:
+		if (is_ereg(dst_reg))
+			EMIT2(0x41, 0xC7);
+		else
+			EMIT1(0xC7);
+		break;
+	case BPF_DW:
+		EMIT2(add_1mod(0x48, dst_reg), 0xC7);
+		break;
+	}
+
+	if (is_imm8(insn_off))
+		EMIT2(add_1reg(0x40, dst_reg), insn_off);
+	else
+		EMIT1_off32(add_1reg(0x80, dst_reg), insn_off);
+
+	EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
+	*pprog = prog;
+}
+
 static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm)
 {
 	emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);
@@ -1424,17 +1618,35 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size,
 	return 0;
 }
 
-static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg,
-			     u32 src_reg, s16 off, u8 bpf_size)
+static int emit_atomic_ld_st(struct bpf_verifier_env *env, u8 **pprog,
+			     struct bpf_insn *insn, u8 *ip, u32 dst_reg,
+			     u32 src_reg, bool accesses_stack_only)
 {
+	u32 atomic_op = insn->imm;
+	int err;
+
 	switch (atomic_op) {
 	case BPF_LOAD_ACQ:
+		if (!accesses_stack_only) {
+			err = emit_kasan_check(env, pprog, src_reg, insn->off,
+					       BPF_SIZE(insn->code), ip, false);
+			if (err)
+				return err;
+		}
 		/* dst_reg = smp_load_acquire(src_reg + off16) */
-		emit_ldx(pprog, bpf_size, dst_reg, src_reg, off);
+		emit_ldx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg,
+			 insn->off);
 		break;
 	case BPF_STORE_REL:
+		if (!accesses_stack_only) {
+			err = emit_kasan_check(env, pprog, dst_reg, insn->off,
+					       BPF_SIZE(insn->code), ip, true);
+			if (err)
+				return err;
+		}
 		/* smp_store_release(dst_reg + off16, src_reg) */
-		emit_stx(pprog, bpf_size, dst_reg, src_reg, off);
+		emit_stx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg,
+			 insn->off);
 		break;
 	default:
 		pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
@@ -1683,6 +1895,65 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,
 	return 0;
 }
 
+static const struct bpf_jit_arg_abi x86_arg_abi = {
+	.nr_arg_regs		= 6,
+	.backfill_after_stack	= true,
+	.even_stack_align	= true,
+};
+
+static const u8 x86_arg_reg[] = {
+	BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9,
+};
+
+/*
+ * Move the arguments the x86-64 ABI places somewhere other than the argument
+ * slot the BPF calling convention gave them. @stack_base addresses the
+ * outgoing stack argument area from RBP. Return the number of emitted bytes.
+ */
+static int emit_kfunc_arg_moves(const struct btf_func_model *fm, s32 stack_base, u8 **pprog)
+{
+	struct bpf_jit_arg_move moves[BPF_JIT_MAX_ARG_MOVES];
+	const u8 nreg = x86_arg_abi.nr_arg_regs;
+	u8 *prog = *pprog, *start = prog;
+	u32 i, n;
+
+	n = bpf_jit_plan_arg_moves(&x86_arg_abi, fm, moves);
+
+	for (i = 0; i < n; i++) {
+		u8 dst = moves[i].dst, src = moves[i].src, reg;
+		bool dst_mem = dst != BPF_JIT_ARG_TMP && dst >= nreg;
+		bool src_mem = src != BPF_JIT_ARG_TMP && src >= nreg;
+
+		/*
+		 * Take the value into a register: the one it belongs in, the
+		 * scratch when it is carried past its own destination, and
+		 * BPF_REG_AX only to pass one stack slot to another.
+		 */
+		if (src == BPF_JIT_ARG_TMP) {
+			reg = AUX_REG;
+		} else if (src_mem) {
+			reg = dst == BPF_JIT_ARG_TMP ? AUX_REG :
+			      dst_mem ? BPF_REG_AX : x86_arg_reg[dst];
+			emit_ldx(&prog, BPF_DW, reg, BPF_REG_FP,
+				 stack_base + (src - nreg) * 8);
+		} else {
+			reg = x86_arg_reg[src];
+		}
+
+		/* And leave it where the argument belongs. */
+		if (dst == BPF_JIT_ARG_TMP)
+			emit_mov_reg(&prog, true, AUX_REG, reg);
+		else if (dst_mem)
+			emit_stx(&prog, BPF_DW, BPF_REG_FP, reg,
+				 stack_base + (dst - nreg) * 8);
+		else if (reg != x86_arg_reg[dst])
+			emit_mov_reg(&prog, true, x86_arg_reg[dst], reg);
+	}
+
+	*pprog = prog;
+	return prog - start;
+}
+
 /*
  * Rebase the __arena args of a kfunc call to arena kernel addresses,
  * rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable
@@ -1690,20 +1961,21 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,
  * arena NULL is offset 0. Return the number of emitted bytes.
  */
 static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog,
-				 const struct bpf_insn *insn, u8 **pprog)
+				 const struct btf_func_model *fm, u8 **pprog)
 {
-	const struct btf_func_model *fm;
 	u8 *prog = *pprog;
 	u8 *start = prog;
-	int i;
-
-	fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
-	if (!fm)
-		return -EINVAL;
+	int i, slot;
 
-	for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) {
+	for (i = 0, slot = 0; i < fm->nr_args; i++) {
+		u32 arg_regs = (fm->arg_size[i] + 7) / 8;
 		u8 flags = fm->arg_flags[i];
-		u32 reg = BPF_REG_1 + i;
+		u32 reg;
+
+		if (slot + arg_regs > MAX_BPF_FUNC_REG_ARGS)
+			break;
+		reg = BPF_REG_1 + slot;
+		slot += arg_regs;
 
 		if (!(flags & BTF_FMODEL_ARENA_ARG))
 			continue;
@@ -1860,10 +2132,12 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 		const s32 imm32 = insn->imm;
 		u32 dst_reg = insn->dst_reg;
 		u32 src_reg = insn->src_reg;
+		bool accesses_stack_only;
 		u8 b2 = 0, b3 = 0;
 		u8 *start_of_ldx;
 		s64 jmp_offset;
 		s32 insn_off;
+		int insn_idx;
 		u8 jmp_cond;
 		u8 *func;
 		int nops;
@@ -1880,6 +2154,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 			EMIT_ENDBR();
 
 		ip = image + addrs[i - 1] + (prog - temp);
+		insn_idx = i - 1 + bpf_prog->aux->subprog_start;
+		accesses_stack_only =
+			env ? !env->insn_aux_data[insn_idx].non_stack_access :
+			      false;
 
 		switch (insn->code) {
 			/* ALU */
@@ -2256,49 +2534,37 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 			EMIT_LFENCE();
 			break;
 
-			/* ST: *(u8*)(dst_reg + off) = imm */
 		case BPF_ST | BPF_MEM | BPF_B:
-			if (is_ereg(dst_reg))
-				EMIT2(0x41, 0xC6);
-			else
-				EMIT1(0xC6);
-			goto st;
 		case BPF_ST | BPF_MEM | BPF_H:
-			if (is_ereg(dst_reg))
-				EMIT3(0x66, 0x41, 0xC7);
-			else
-				EMIT2(0x66, 0xC7);
-			goto st;
 		case BPF_ST | BPF_MEM | BPF_W:
-			if (is_ereg(dst_reg))
-				EMIT2(0x41, 0xC7);
-			else
-				EMIT1(0xC7);
-			goto st;
 		case BPF_ST | BPF_MEM | BPF_DW:
 			if (dst_reg == BPF_REG_PARAMS && insn->off == -8) {
 				/* Arg 6: store immediate in r9 register */
-				emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32);
+				emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31,
+					       imm32);
 				break;
 			}
-			EMIT2(add_1mod(0x48, dst_reg), 0xC7);
-
-st:			insn_off = insn->off;
+			insn_off = insn->off;
 			if (dst_reg == BPF_REG_PARAMS) {
 				/*
 				 * Args 7+: reverse BPF negative offsets to
 				 * x86 positive rsp offsets.
 				 * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ...
 				 */
-				insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16;
+				insn_off = outgoing_arg_base - outgoing_rsp -
+					   insn_off - 16;
 				dst_reg = BPF_REG_FP;
 			}
-			if (is_imm8(insn_off))
-				EMIT2(add_1reg(0x40, dst_reg), insn_off);
-			else
-				EMIT1_off32(add_1reg(0x80, dst_reg), insn_off);
+			if (!accesses_stack_only) {
+				err = emit_kasan_check(env, &prog, dst_reg,
+						       insn_off,
+						       BPF_SIZE(insn->code), ip,
+						       true);
+				if (err)
+					return err;
+			}
 
-			EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
+			emit_st(&prog, insn, dst_reg, insn_off);
 			break;
 
 			/* STX: *(u8*)(dst_reg + off) = src_reg */
@@ -2316,6 +2582,14 @@ st:			insn_off = insn->off;
 				insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16;
 				dst_reg = BPF_REG_FP;
 			}
+			if (!accesses_stack_only) {
+				err = emit_kasan_check(env, &prog, dst_reg,
+						       insn_off,
+						       BPF_SIZE(insn->code), ip,
+						       true);
+				if (err)
+					return err;
+			}
 			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);
 			break;
 
@@ -2497,6 +2771,13 @@ st:			insn_off = insn->off;
 				/* populate jmp_offset for JAE above to jump to start_of_ldx */
 				start_of_ldx = prog;
 				end_of_jmp[-1] = start_of_ldx - end_of_jmp;
+			} else if (!accesses_stack_only) {
+				err = emit_kasan_check(env, &prog, src_reg,
+						       insn_off,
+						       BPF_SIZE(insn->code), ip,
+						       false);
+				if (err)
+					return err;
 			}
 			if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
 			    BPF_MODE(insn->code) == BPF_MEMSX)
@@ -2558,28 +2839,42 @@ st:			insn_off = insn->off;
 			}
 			fallthrough;
 		case BPF_STX | BPF_ATOMIC | BPF_W:
-		case BPF_STX | BPF_ATOMIC | BPF_DW:
-			if (insn->imm == (BPF_AND | BPF_FETCH) ||
-			    insn->imm == (BPF_OR | BPF_FETCH) ||
-			    insn->imm == (BPF_XOR | BPF_FETCH)) {
-				bool is64 = BPF_SIZE(insn->code) == BPF_DW;
-				u32 real_src_reg = src_reg;
-				u32 real_dst_reg = dst_reg;
-				u8 *branch_target;
-
+		case BPF_STX | BPF_ATOMIC | BPF_DW: {
+			bool is64 = BPF_SIZE(insn->code) == BPF_DW;
+			u32 real_src_reg = src_reg;
+			u32 real_dst_reg = dst_reg;
+			u8 *old_prog;
+			bool is_atomic_fetch =
+				(insn->imm == (BPF_AND | BPF_FETCH) ||
+				 insn->imm == (BPF_OR | BPF_FETCH) ||
+				 insn->imm == (BPF_XOR | BPF_FETCH));
+			if (is_atomic_fetch) {
 				/*
 				 * Can't be implemented with a single x86 insn.
 				 * Need to do a CMPXCHG loop.
 				 */
 
 				/* Will need RAX as a CMPXCHG operand so save R0 */
+				old_prog = prog;
 				emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0);
 				if (src_reg == BPF_REG_0)
 					real_src_reg = BPF_REG_AX;
 				if (dst_reg == BPF_REG_0)
 					real_dst_reg = BPF_REG_AX;
-
-				branch_target = prog;
+				ip += (prog - old_prog);
+			}
+			if (!bpf_atomic_is_load_store(insn)) {
+				if (!accesses_stack_only) {
+					err = emit_kasan_check(
+						env, &prog, real_dst_reg,
+						insn->off, BPF_SIZE(insn->code),
+						ip, true);
+					if (err)
+						return err;
+				}
+			}
+			if (is_atomic_fetch) {
+				u8 *branch_target = prog;
 				/* Load old value */
 				emit_ldx(&prog, BPF_SIZE(insn->code),
 					 BPF_REG_0, real_dst_reg, insn->off);
@@ -2611,15 +2906,16 @@ st:			insn_off = insn->off;
 			}
 
 			if (bpf_atomic_is_load_store(insn))
-				err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg,
-							insn->off, BPF_SIZE(insn->code));
+				err = emit_atomic_ld_st(env, &prog, insn, ip,
+							dst_reg, src_reg,
+							accesses_stack_only);
 			else
 				err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg,
 						      insn->off, BPF_SIZE(insn->code));
 			if (err)
 				return err;
 			break;
-
+		}
 		case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
 		case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
 			if (!bpf_atomic_is_load_store(insn)) {
@@ -2645,6 +2941,8 @@ st:			insn_off = insn->off;
 
 			/* call */
 		case BPF_JMP | BPF_CALL: {
+			const struct btf_func_model *fm = NULL;
+
 			func = (u8 *) __bpf_call_base + imm32;
 			if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
 				LOAD_TAIL_CALL_CNT_PTR(stack_depth);
@@ -2653,10 +2951,27 @@ st:			insn_off = insn->off;
 			if (!imm32)
 				return -EINVAL;
 			if (src_reg == BPF_PSEUDO_KFUNC_CALL) {
-				err = emit_kfunc_arena_args(bpf_prog, insn, &prog);
+				fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
+				if (!fm)
+					return -EINVAL;
+				err = emit_kfunc_arena_args(bpf_prog, fm, &prog);
 				if (err < 0)
 					return err;
 				ip += err;
+				ip += emit_kfunc_arg_moves(fm, outgoing_arg_base -
+							   outgoing_rsp, &prog);
+			}
+			if (func == (u8 *)arch_bpf_timed_may_goto) {
+				u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP;
+
+				/*
+				 * AX has the offset of count and timestamp
+				 * in the stack. Turn it into a pointer.
+				 * add r10, rbp or add r10, r9
+				 */
+				maybe_emit_mod(&prog, BPF_REG_AX, fp, true);
+				EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp));
+				ip += 3;
 			}
 			if (priv_frame_ptr) {
 				push_r9(&prog);
@@ -2667,9 +2982,35 @@ st:			insn_off = insn->off;
 				return -EINVAL;
 			if (priv_frame_ptr)
 				pop_r9(&prog);
+			/*
+			 * A kfunc returning more than 8 bytes hands the second
+			 * half back in RDX (the native ABI's second return reg),
+			 * but BPF expects it in R0:R2. BPF R0 is RAX (no move
+			 * needed), while BPF R2 is RSI, so copy RDX into RSI.
+			 */
+			if (fm && fm->ret_size > 8)
+				emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3);
 			break;
 		}
 
+			/* callx: call of a bpf subprog whose address is in dst_reg */
+		case BPF_JMP | BPF_CALL | BPF_X:
+			/*
+			 * The verifier makes sure that callees of callx are
+			 * not tail call reachable, hence unlike a direct call
+			 * of a subprog there is no need to pass
+			 * tail_call_cnt_ptr in rax.
+			 */
+			if (priv_frame_ptr) {
+				push_r9(&prog);
+				ip += 2;
+			}
+			if (emit_indirect_call(&prog, insn->dst_reg, ip))
+				return -EINVAL;
+			if (priv_frame_ptr)
+				pop_r9(&prog);
+			break;
+
 		case BPF_JMP | BPF_TAIL_CALL:
 			if (imm32)
 				emit_bpf_tail_call_direct(bpf_prog,
@@ -3217,16 +3558,21 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog,
 }
 
 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
+			   struct bpf_tramp_image *im,
 			   struct bpf_tramp_node *node, int stack_size,
 			   int run_ctx_off, bool save_ret,
 			   void *image, void *rw_image)
 {
 	u8 *prog = *pprog;
-	u8 *jmp_insn;
+	u8 *jmp_insn, *skip;
 	int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
 	struct bpf_prog *p = node->link->prog;
 	u64 cookie = node->cookie;
 
+	/* nop, patched to skip this prog when it is detached */
+	skip = image + (prog - (u8 *)rw_image);
+	emit_nops(&prog, X86_PATCH_SIZE);
+
 	/* mov rdi, cookie */
 	emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie);
 
@@ -3301,6 +3647,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 	if (emit_rsb_call(&prog, bpf_trampoline_exit(p), image + (prog - (u8 *)rw_image)))
 		return -EINVAL;
 
+	bpf_tramp_image_add_skip(im, p, skip, image + (prog - (u8 *)rw_image));
+
 	*pprog = prog;
 	return 0;
 }
@@ -3332,6 +3680,7 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
 }
 
 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
+		      struct bpf_tramp_image *im,
 		      struct bpf_tramp_nodes *tl, int stack_size,
 		      int run_ctx_off, int func_meta_off, bool save_ret,
 		      void *image, void *rw_image, u64 func_meta,
@@ -3346,7 +3695,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
 				func_meta | (cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT));
 			cur_cookie--;
 		}
-		if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size,
+		if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size,
 				    run_ctx_off, save_ret, image, rw_image))
 			return -EINVAL;
 	}
@@ -3355,6 +3704,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
 }
 
 static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
+			      struct bpf_tramp_image *im,
 			      struct bpf_tramp_nodes *tl, int stack_size,
 			      int run_ctx_off, u8 **branches,
 			      void *image, void *rw_image)
@@ -3368,7 +3718,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
 	emit_mov_imm32(&prog, false, BPF_REG_0, 0);
 	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
 	for (i = 0; i < tl->nr_nodes; i++) {
-		if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size, run_ctx_off, true,
+		if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size, run_ctx_off, true,
 				    image, rw_image))
 			return -EINVAL;
 
@@ -3640,7 +3990,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 	}
 
 	if (fentry->nr_nodes) {
-		if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, func_meta_off,
+		if (invoke_bpf(m, &prog, im, fentry, regs_off, run_ctx_off, func_meta_off,
 			       flags & BPF_TRAMP_F_RET_FENTRY_RET, image, rw_image,
 			       func_meta, cookie_off))
 			return -EINVAL;
@@ -3652,7 +4002,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		if (!branches)
 			return -ENOMEM;
 
-		if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off,
+		if (invoke_bpf_mod_ret(m, &prog, im, fmod_ret, regs_off,
 				       run_ctx_off, branches, image, rw_image)) {
 			ret = -EINVAL;
 			goto cleanup;
@@ -3682,8 +4032,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		}
 		/* remember return value in a stack for bpf prog to access */
 		emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
-		im->ip_after_call = image + (prog - (u8 *)rw_image);
-		emit_nops(&prog, X86_PATCH_SIZE);
 	}
 
 	if (fmod_ret->nr_nodes) {
@@ -3708,7 +4056,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 		emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta);
 
 	if (fexit->nr_nodes) {
-		if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, func_meta_off,
+		if (invoke_bpf(m, &prog, im, fexit, regs_off, run_ctx_off, func_meta_off,
 			       false, image, rw_image, func_meta, cookie_off)) {
 			ret = -EINVAL;
 			goto cleanup;
@@ -3723,7 +4071,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
 	 * restored to R0.
 	 */
 	if (flags & BPF_TRAMP_F_CALL_ORIG) {
-		im->ip_epilogue = image + (prog - (u8 *)rw_image);
 		/* arg1: mov rdi, im */
 		emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im);
 		if (emit_rsb_call(&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) {
@@ -3811,25 +4158,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 			     struct bpf_tramp_nodes *tnodes, void *func_addr)
 {
-	struct bpf_tramp_image im;
-	void *image;
-	int ret;
+	struct bpf_tramp_image im = {};
 
-	/* Allocate a temporary buffer for __arch_prepare_bpf_trampoline().
-	 *
-	 * We cannot use kvmalloc here, because we need image to be in
-	 * module memory range.
-	 * Since it must be writable use execmem_alloc(EXECMEM_MODULE_DATA)
-	 * that returns writable memory in the module address space.
-	 */
-	image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE);
-	if (!image)
+	if (!trampoline_size_image)
 		return -ENOMEM;
 
-	ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image,
-					    m, flags, tnodes, func_addr);
-	execmem_free(image);
-	return ret;
+	return __arch_prepare_bpf_trampoline(&im, trampoline_size_image,
+					     trampoline_size_image + PAGE_SIZE,
+					     trampoline_size_image, m, flags,
+					     tnodes, func_addr);
 }
 
 static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)
@@ -4153,11 +4490,31 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	return prog;
 }
 
+bool bpf_jit_supports_callx(void)
+{
+	/*
+	 * FineIBT poisons ENDBR at the entry of a JITed function and expects
+	 * indirect callers to go through the CFI preamble instead.
+	 * callx doesn't do that yet.
+	 */
+	return cfi_mode != CFI_FINEIBT;
+}
+
 bool bpf_jit_supports_kfunc_call(void)
 {
 	return true;
 }
 
+bool bpf_jit_supports_kfunc_ret_reg_pair(void)
+{
+	return true;
+}
+
+const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void)
+{
+	return &x86_arg_abi;
+}
+
 bool bpf_jit_supports_stack_args(void)
 {
 	return true;
@@ -4181,6 +4538,18 @@ bool bpf_jit_supports_subprog_tailcalls(void)
 	return true;
 }
 
+/*
+ * Frame sizes are 32-bit immediates in the prologue, epilogue and tail call
+ * sequences, a tail call pops the caller's frame and lands in the target's
+ * prologue before the target allocates its own, and private stacks are
+ * allocated from the program's own depth, so MAX_BPF_STACK_JIT frames need
+ * nothing special.
+ */
+bool bpf_jit_supports_large_stack(void)
+{
+	return true;
+}
+
 bool bpf_jit_supports_percpu_insn(void)
 {
 	return true;
diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S
index 54c690cae19087..6c6e03f725a301 100644
--- a/arch/x86/net/bpf_timed_may_goto.S
+++ b/arch/x86/net/bpf_timed_may_goto.S
@@ -11,12 +11,6 @@
 SYM_FUNC_START(arch_bpf_timed_may_goto)
 	ANNOTATE_NOENDBR
 
-	/*
-	 * r10 passes us stack depth, load the pointer to count and timestamp
-	 * into r10 by adding it to BPF frame pointer.
-	 */
-	leaq (%rbp, %r10, 1), %r10
-
 	/* Setup frame. */
 	pushq %rbp
 	movq %rsp, %rbp
@@ -30,8 +24,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
 	pushq %r8
 
 	/*
-	 * r10 has the pointer to count and timestamp, pass it as first
-	 * argument.
+	 * r10 has the pointer to count and timestamp. JIT added the frame
+	 * pointer to the offset. Pass it as first argument.
 	 */
 	movq %r10, %rdi
 
diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c
index b301c6c8df7532..795d81c7a4de4b 100644
--- a/arch/x86/pci/fixup.c
+++ b/arch/x86/pci/fixup.c
@@ -886,6 +886,105 @@ static void quirk_clear_strap_no_soft_reset_dev2_f0(struct pci_dev *dev)
 	}
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x15b8, quirk_clear_strap_no_soft_reset_dev2_f0);
+
+/*
+ * Enhanced atomic operations can cause corruption with 64-bit DMA
+ * on these devices.
+ */
+#define RX_ENH_ATOMIC_EN		BIT(8)
+
+static const u32 nbio_7_7_pcie_smn_addrs[] = {
+	0x111401d0,
+	0x111411d0,
+	0x111421d0,
+	0x111431d0,
+	0x111441d0,
+	0x112401d0,
+	0x112411d0,
+	0x112421d0,
+	0x112431d0,
+	0x112441d0,
+	0x112451d0,
+	0x113401d0,
+	0x114401d0,
+};
+
+static const u32 nbio_7_11_pcie_smn_addrs[] = {
+	0x112401d0,
+	0x112411d0,
+	0x112421d0,
+	0x112431d0,
+	0x112441d0,
+	0x112451d0,
+	0x113401d0,
+	0x113411d0,
+	0x113421d0,
+	0x113431d0,
+	0x113441d0,
+	0x113451d0,
+};
+
+static void quirk_amd_nbio_enhanced_atomic(struct pci_dev *host_bridge,
+					   const u32 *smn_addrs,
+					   size_t nr_smn_addrs)
+{
+	bool changed = false;
+	size_t i;
+	u32 data;
+	int ret;
+
+	for (i = 0; i < nr_smn_addrs; i++) {
+		ret = amd_smn_read(0, smn_addrs[i], &data);
+		if (ret)
+			continue;
+		if (!(data & RX_ENH_ATOMIC_EN))
+			continue;
+		data = data & ~RX_ENH_ATOMIC_EN;
+		ret = amd_smn_write(0, smn_addrs[i], data);
+		if (ret)
+			continue;
+		if (changed)
+			continue;
+		ret = amd_smn_read(0, smn_addrs[i], &data);
+		if (ret)
+			continue;
+		if (data & RX_ENH_ATOMIC_EN)
+			continue;
+		changed = true;
+	}
+
+	if (changed)
+		pci_info(host_bridge, "enhanced atomics disabled\n");
+}
+
+static void quirk_amd_nbio_7_7_disable_enhanced_atomic(struct pci_dev *dev)
+{
+	quirk_amd_nbio_enhanced_atomic(dev, nbio_7_7_pcie_smn_addrs,
+				       ARRAY_SIZE(nbio_7_7_pcie_smn_addrs));
+}
+
+static void quirk_amd_nbio_7_11_disable_enhanced_atomic(struct pci_dev *dev)
+{
+	quirk_amd_nbio_enhanced_atomic(dev, nbio_7_11_pcie_smn_addrs,
+				       ARRAY_SIZE(nbio_7_11_pcie_smn_addrs));
+}
+
+/* Phoenix, Hawk Point (NBIO 7.7) */
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x14E8,
+			quirk_amd_nbio_7_7_disable_enhanced_atomic);
+DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x14E8,
+			quirk_amd_nbio_7_7_disable_enhanced_atomic);
+
+/* Strix, Krackan, Strix Halo (NBIO 7.11) */
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x1507,
+			quirk_amd_nbio_7_11_disable_enhanced_atomic);
+DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x1507,
+			quirk_amd_nbio_7_11_disable_enhanced_atomic);
+DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x1122,
+			quirk_amd_nbio_7_11_disable_enhanced_atomic);
+DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x1122,
+			quirk_amd_nbio_7_11_disable_enhanced_atomic);
+
 #endif
 
 /*
diff --git a/arch/x86/platform/Makefile b/arch/x86/platform/Makefile
index 3ed03a2552d012..727b92d0ca25e3 100644
--- a/arch/x86/platform/Makefile
+++ b/arch/x86/platform/Makefile
@@ -10,5 +10,4 @@ obj-y	+= intel-mid/
 obj-y	+= intel-quark/
 obj-y	+= olpc/
 obj-y	+= scx200/
-obj-y	+= ts5500/
 obj-y	+= uv/
diff --git a/arch/x86/platform/olpc/olpc-xo15-sci.c b/arch/x86/platform/olpc/olpc-xo15-sci.c
index 75ed6ceb9df33d..a13dd6b8bc6ee5 100644
--- a/arch/x86/platform/olpc/olpc-xo15-sci.c
+++ b/arch/x86/platform/olpc/olpc-xo15-sci.c
@@ -210,8 +210,8 @@ static int xo15_sci_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(xo15_sci_pm, NULL, xo15_sci_resume);
 
 static const struct acpi_device_id xo15_sci_device_ids[] = {
-	{"XO15EC", 0},
-	{"", 0},
+	{ .id = "XO15EC" },
+	{ }
 };
 
 static struct platform_driver xo15_sci_drv = {
diff --git a/arch/x86/platform/pvh/enlighten.c b/arch/x86/platform/pvh/enlighten.c
index f2053cbe9b0ce3..cb442cbd9d8286 100644
--- a/arch/x86/platform/pvh/enlighten.c
+++ b/arch/x86/platform/pvh/enlighten.c
@@ -8,6 +8,7 @@
 #include <asm/hypervisor.h>
 #include <asm/e820/api.h>
 #include <asm/x86_init.h>
+#include <asm/string.h>
 
 #include <asm/xen/interface.h>
 
@@ -129,7 +130,7 @@ void __init xen_prepare_pvh(void)
 	 * This must not compile to "call memset" because memset() may be
 	 * instrumented.
 	 */
-	__builtin_memset(&pvh_bootparams, 0, sizeof(pvh_bootparams));
+	__inline_memset(&pvh_bootparams, 0, sizeof(pvh_bootparams));
 
 	hypervisor_specific_init(xen_guest);
 
diff --git a/arch/x86/platform/ts5500/Makefile b/arch/x86/platform/ts5500/Makefile
deleted file mode 100644
index 910fe9e3ffb4ad..00000000000000
--- a/arch/x86/platform/ts5500/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_TS5500)	+= ts5500.o
diff --git a/arch/x86/platform/ts5500/ts5500.c b/arch/x86/platform/ts5500/ts5500.c
deleted file mode 100644
index 0b67da056fd927..00000000000000
--- a/arch/x86/platform/ts5500/ts5500.c
+++ /dev/null
@@ -1,341 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Technologic Systems TS-5500 Single Board Computer support
- *
- * Copyright (C) 2013-2014 Savoir-faire Linux Inc.
- *	Vivien Didelot <vivien.didelot@savoirfairelinux.com>
- *
- * This driver registers the Technologic Systems TS-5500 Single Board Computer
- * (SBC) and its devices, and exposes information to userspace such as jumpers'
- * state or available options. For further information about sysfs entries, see
- * Documentation/ABI/testing/sysfs-platform-ts5500.
- *
- * This code may be extended to support similar x86-based platforms.
- * Actually, the TS-5500 and TS-5400 are supported.
- */
-
-#include <linux/delay.h>
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <linux/leds.h>
-#include <linux/init.h>
-#include <linux/platform_data/max197.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-/* Product code register */
-#define TS5500_PRODUCT_CODE_ADDR	0x74
-#define TS5500_PRODUCT_CODE		0x60	/* TS-5500 product code */
-#define TS5400_PRODUCT_CODE		0x40	/* TS-5400 product code */
-
-/* SRAM/RS-485/ADC options, and RS-485 RTS/Automatic RS-485 flags register */
-#define TS5500_SRAM_RS485_ADC_ADDR	0x75
-#define TS5500_SRAM			BIT(0)	/* SRAM option */
-#define TS5500_RS485			BIT(1)	/* RS-485 option */
-#define TS5500_ADC			BIT(2)	/* A/D converter option */
-#define TS5500_RS485_RTS		BIT(6)	/* RTS for RS-485 */
-#define TS5500_RS485_AUTO		BIT(7)	/* Automatic RS-485 */
-
-/* External Reset/Industrial Temperature Range options register */
-#define TS5500_ERESET_ITR_ADDR		0x76
-#define TS5500_ERESET			BIT(0)	/* External Reset option */
-#define TS5500_ITR			BIT(1)	/* Indust. Temp. Range option */
-
-/* LED/Jumpers register */
-#define TS5500_LED_JP_ADDR		0x77
-#define TS5500_LED			BIT(0)	/* LED flag */
-#define TS5500_JP1			BIT(1)	/* Automatic CMOS */
-#define TS5500_JP2			BIT(2)	/* Enable Serial Console */
-#define TS5500_JP3			BIT(3)	/* Write Enable Drive A */
-#define TS5500_JP4			BIT(4)	/* Fast Console (115K baud) */
-#define TS5500_JP5			BIT(5)	/* User Jumper */
-#define TS5500_JP6			BIT(6)	/* Console on COM1 (req. JP2) */
-#define TS5500_JP7			BIT(7)	/* Undocumented (Unused) */
-
-/* A/D Converter registers */
-#define TS5500_ADC_CONV_BUSY_ADDR	0x195	/* Conversion state register */
-#define TS5500_ADC_CONV_BUSY		BIT(0)
-#define TS5500_ADC_CONV_INIT_LSB_ADDR	0x196	/* Start conv. / LSB register */
-#define TS5500_ADC_CONV_MSB_ADDR	0x197	/* MSB register */
-#define TS5500_ADC_CONV_DELAY		12	/* usec */
-
-/**
- * struct ts5500_sbc - TS-5500 board description
- * @name:	Board model name.
- * @id:		Board product ID.
- * @sram:	Flag for SRAM option.
- * @rs485:	Flag for RS-485 option.
- * @adc:	Flag for Analog/Digital converter option.
- * @ereset:	Flag for External Reset option.
- * @itr:	Flag for Industrial Temperature Range option.
- * @jumpers:	Bitfield for jumpers' state.
- */
-struct ts5500_sbc {
-	const char *name;
-	int	id;
-	bool	sram;
-	bool	rs485;
-	bool	adc;
-	bool	ereset;
-	bool	itr;
-	u8	jumpers;
-};
-
-/* Board signatures in BIOS shadow RAM */
-static const struct {
-	const char * const string;
-	const ssize_t offset;
-} ts5500_signatures[] __initconst = {
-	{ "TS-5x00 AMD Elan", 0xb14 },
-};
-
-static int __init ts5500_check_signature(void)
-{
-	void __iomem *bios;
-	int i, ret = -ENODEV;
-
-	bios = ioremap(0xf0000, 0x10000);
-	if (!bios)
-		return -ENOMEM;
-
-	for (i = 0; i < ARRAY_SIZE(ts5500_signatures); i++) {
-		if (check_signature(bios + ts5500_signatures[i].offset,
-				    ts5500_signatures[i].string,
-				    strlen(ts5500_signatures[i].string))) {
-			ret = 0;
-			break;
-		}
-	}
-
-	iounmap(bios);
-	return ret;
-}
-
-static int __init ts5500_detect_config(struct ts5500_sbc *sbc)
-{
-	u8 tmp;
-	int ret = 0;
-
-	if (!request_region(TS5500_PRODUCT_CODE_ADDR, 4, "ts5500"))
-		return -EBUSY;
-
-	sbc->id = inb(TS5500_PRODUCT_CODE_ADDR);
-	if (sbc->id == TS5500_PRODUCT_CODE) {
-		sbc->name = "TS-5500";
-	} else if (sbc->id == TS5400_PRODUCT_CODE) {
-		sbc->name = "TS-5400";
-	} else {
-		pr_err("ts5500: unknown product code 0x%x\n", sbc->id);
-		ret = -ENODEV;
-		goto cleanup;
-	}
-
-	tmp = inb(TS5500_SRAM_RS485_ADC_ADDR);
-	sbc->sram = tmp & TS5500_SRAM;
-	sbc->rs485 = tmp & TS5500_RS485;
-	sbc->adc = tmp & TS5500_ADC;
-
-	tmp = inb(TS5500_ERESET_ITR_ADDR);
-	sbc->ereset = tmp & TS5500_ERESET;
-	sbc->itr = tmp & TS5500_ITR;
-
-	tmp = inb(TS5500_LED_JP_ADDR);
-	sbc->jumpers = tmp & ~TS5500_LED;
-
-cleanup:
-	release_region(TS5500_PRODUCT_CODE_ADDR, 4);
-	return ret;
-}
-
-static ssize_t name_show(struct device *dev, struct device_attribute *attr,
-		char *buf)
-{
-	struct ts5500_sbc *sbc = dev_get_drvdata(dev);
-
-	return sprintf(buf, "%s\n", sbc->name);
-}
-static DEVICE_ATTR_RO(name);
-
-static ssize_t id_show(struct device *dev, struct device_attribute *attr,
-		char *buf)
-{
-	struct ts5500_sbc *sbc = dev_get_drvdata(dev);
-
-	return sprintf(buf, "0x%.2x\n", sbc->id);
-}
-static DEVICE_ATTR_RO(id);
-
-static ssize_t jumpers_show(struct device *dev, struct device_attribute *attr,
-		char *buf)
-{
-	struct ts5500_sbc *sbc = dev_get_drvdata(dev);
-
-	return sprintf(buf, "0x%.2x\n", sbc->jumpers >> 1);
-}
-static DEVICE_ATTR_RO(jumpers);
-
-#define TS5500_ATTR_BOOL(_field)					\
-	static ssize_t _field##_show(struct device *dev,		\
-			struct device_attribute *attr, char *buf)	\
-	{								\
-		struct ts5500_sbc *sbc = dev_get_drvdata(dev);		\
-									\
-		return sprintf(buf, "%d\n", sbc->_field);		\
-	}								\
-	static DEVICE_ATTR_RO(_field)
-
-TS5500_ATTR_BOOL(sram);
-TS5500_ATTR_BOOL(rs485);
-TS5500_ATTR_BOOL(adc);
-TS5500_ATTR_BOOL(ereset);
-TS5500_ATTR_BOOL(itr);
-
-static struct attribute *ts5500_attributes[] = {
-	&dev_attr_id.attr,
-	&dev_attr_name.attr,
-	&dev_attr_jumpers.attr,
-	&dev_attr_sram.attr,
-	&dev_attr_rs485.attr,
-	&dev_attr_adc.attr,
-	&dev_attr_ereset.attr,
-	&dev_attr_itr.attr,
-	NULL
-};
-
-static const struct attribute_group ts5500_attr_group = {
-	.attrs = ts5500_attributes,
-};
-
-static struct resource ts5500_dio1_resource[] = {
-	DEFINE_RES_IRQ_NAMED(7, "DIO1 interrupt"),
-};
-
-static struct platform_device ts5500_dio1_pdev = {
-	.name = "ts5500-dio1",
-	.id = -1,
-	.resource = ts5500_dio1_resource,
-	.num_resources = 1,
-};
-
-static struct resource ts5500_dio2_resource[] = {
-	DEFINE_RES_IRQ_NAMED(6, "DIO2 interrupt"),
-};
-
-static struct platform_device ts5500_dio2_pdev = {
-	.name = "ts5500-dio2",
-	.id = -1,
-	.resource = ts5500_dio2_resource,
-	.num_resources = 1,
-};
-
-static void ts5500_led_set(struct led_classdev *led_cdev,
-			   enum led_brightness brightness)
-{
-	outb(!!brightness, TS5500_LED_JP_ADDR);
-}
-
-static enum led_brightness ts5500_led_get(struct led_classdev *led_cdev)
-{
-	return (inb(TS5500_LED_JP_ADDR) & TS5500_LED) ? LED_FULL : LED_OFF;
-}
-
-static struct led_classdev ts5500_led_cdev = {
-	.name = "ts5500:green:",
-	.brightness_set = ts5500_led_set,
-	.brightness_get = ts5500_led_get,
-};
-
-static int ts5500_adc_convert(u8 ctrl)
-{
-	u8 lsb, msb;
-
-	/* Start conversion (ensure the 3 MSB are set to 0) */
-	outb(ctrl & 0x1f, TS5500_ADC_CONV_INIT_LSB_ADDR);
-
-	/*
-	 * The platform has CPLD logic driving the A/D converter.
-	 * The conversion must complete within 11 microseconds,
-	 * otherwise we have to re-initiate a conversion.
-	 */
-	udelay(TS5500_ADC_CONV_DELAY);
-	if (inb(TS5500_ADC_CONV_BUSY_ADDR) & TS5500_ADC_CONV_BUSY)
-		return -EBUSY;
-
-	/* Read the raw data */
-	lsb = inb(TS5500_ADC_CONV_INIT_LSB_ADDR);
-	msb = inb(TS5500_ADC_CONV_MSB_ADDR);
-
-	return (msb << 8) | lsb;
-}
-
-static struct max197_platform_data ts5500_adc_pdata = {
-	.convert = ts5500_adc_convert,
-};
-
-static struct platform_device ts5500_adc_pdev = {
-	.name = "max197",
-	.id = -1,
-	.dev = {
-		.platform_data = &ts5500_adc_pdata,
-	},
-};
-
-static int __init ts5500_init(void)
-{
-	struct platform_device *pdev;
-	struct ts5500_sbc *sbc;
-	int err;
-
-	/*
-	 * There is no DMI available or PCI bridge subvendor info,
-	 * only the BIOS provides a 16-bit identification call.
-	 * It is safer to find a signature in the BIOS shadow RAM.
-	 */
-	err = ts5500_check_signature();
-	if (err)
-		return err;
-
-	pdev = platform_device_register_simple("ts5500", -1, NULL, 0);
-	if (IS_ERR(pdev))
-		return PTR_ERR(pdev);
-
-	sbc = devm_kzalloc(&pdev->dev, sizeof(struct ts5500_sbc), GFP_KERNEL);
-	if (!sbc) {
-		err = -ENOMEM;
-		goto error;
-	}
-
-	err = ts5500_detect_config(sbc);
-	if (err)
-		goto error;
-
-	platform_set_drvdata(pdev, sbc);
-
-	err = sysfs_create_group(&pdev->dev.kobj, &ts5500_attr_group);
-	if (err)
-		goto error;
-
-	if (sbc->id == TS5500_PRODUCT_CODE) {
-		ts5500_dio1_pdev.dev.parent = &pdev->dev;
-		if (platform_device_register(&ts5500_dio1_pdev))
-			dev_warn(&pdev->dev, "DIO1 block registration failed\n");
-		ts5500_dio2_pdev.dev.parent = &pdev->dev;
-		if (platform_device_register(&ts5500_dio2_pdev))
-			dev_warn(&pdev->dev, "DIO2 block registration failed\n");
-	}
-
-	if (led_classdev_register(&pdev->dev, &ts5500_led_cdev))
-		dev_warn(&pdev->dev, "LED registration failed\n");
-
-	if (sbc->adc) {
-		ts5500_adc_pdev.dev.parent = &pdev->dev;
-		if (platform_device_register(&ts5500_adc_pdev))
-			dev_warn(&pdev->dev, "ADC registration failed\n");
-	}
-
-	return 0;
-error:
-	platform_device_unregister(pdev);
-	return err;
-}
-device_initcall(ts5500_init);
diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c
index 694d80a5c68e14..65110b2ee1a1d1 100644
--- a/arch/x86/realmode/init.c
+++ b/arch/x86/realmode/init.c
@@ -66,7 +66,7 @@ void __init reserve_real_mode(void)
 	 */
 	memblock_reserve(0, SZ_1M);
 
-	memblock_clear_kho_scratch(0, SZ_1M);
+	memblock_clear_kho_noprsrv(0, SZ_1M);
 }
 
 static void __init sme_sev_setup_real_mode(struct trampoline_header *th)
diff --git a/arch/x86/um/Makefile b/arch/x86/um/Makefile
index f9ea75bf43aca3..100b68f601f929 100644
--- a/arch/x86/um/Makefile
+++ b/arch/x86/um/Makefile
@@ -47,4 +47,7 @@ include/generated/user_constants.h: $(obj)/user-offsets.s FORCE
 UNPROFILE_OBJS := stub_segv.o
 CFLAGS_stub_segv.o := $(CFLAGS_NO_HARDENING)
 
+# see arch/um/kernel/skas/Makefile - this goes into the stub
+CFLAGS_stub_segv.o += $(call cc-option, -ftrivial-auto-var-init=uninitialized)
+
 include $(srctree)/arch/um/scripts/Makefile.rules
diff --git a/arch/x86/um/asm/elf.h b/arch/x86/um/asm/elf.h
index 22d0111b543b34..388fe669886cf6 100644
--- a/arch/x86/um/asm/elf.h
+++ b/arch/x86/um/asm/elf.h
@@ -9,6 +9,7 @@
 #include <skas.h>
 
 #define CORE_DUMP_USE_REGSET
+#define ELF_FDPIC_CORE_EFLAGS  0
 
 #ifdef CONFIG_X86_32
 
@@ -158,8 +159,11 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm,
 
 extern unsigned long um_vdso_addr;
 #define AT_SYSINFO_EHDR 33
-#define ARCH_DLINFO	NEW_AUX_ENT(AT_SYSINFO_EHDR, um_vdso_addr)
-
+#define ARCH_DLINFO						\
+do {								\
+	NEW_AUX_ENT(AT_SYSINFO_EHDR, um_vdso_addr);		\
+	NEW_AUX_ENT(AT_MINSIGSTKSZ, 0);			\
+} while (0)
 #endif
 
 typedef unsigned long elf_greg_t;
diff --git a/arch/x86/um/asm/ptrace.h b/arch/x86/um/asm/ptrace.h
index 2641d28d115ca6..439c4151f6b72e 100644
--- a/arch/x86/um/asm/ptrace.h
+++ b/arch/x86/um/asm/ptrace.h
@@ -13,9 +13,7 @@ enum {
 };
 
 #include <linux/compiler.h>
-#ifndef CONFIG_X86_32
-#define __FRAME_OFFSETS /* Needed to get the R* macros */
-#endif
+#define __FRAME_OFFSETS /* Needed to get the register macros */
 #include <asm/ptrace-generic.h>
 
 #define user_mode(r) UPT_IS_USER(&(r)->regs)
diff --git a/arch/x86/um/asm/required-features.h b/arch/x86/um/asm/required-features.h
deleted file mode 100644
index dfb967b2d2f375..00000000000000
--- a/arch/x86/um/asm/required-features.h
+++ /dev/null
@@ -1,9 +0,0 @@
-#ifndef __UM_REQUIRED_FEATURES_H
-#define __UM_REQUIRED_FEATURES_H
-
-/*
- * Nothing to see, just need something for the i386 and x86_64 asm
- * headers to include.
- */
-
-#endif
diff --git a/arch/x86/um/ptrace_32.c b/arch/x86/um/ptrace_32.c
index 3af3cb821524bf..9e9155b0e91828 100644
--- a/arch/x86/um/ptrace_32.c
+++ b/arch/x86/um/ptrace_32.c
@@ -7,6 +7,7 @@
 #include <linux/sched.h>
 #include <linux/uaccess.h>
 #include <linux/regset.h>
+#define __FRAME_OFFSETS
 #include <asm/ptrace-abi.h>
 #include <registers.h>
 #include <skas.h>
diff --git a/arch/x86/um/vdso/vma.c b/arch/x86/um/vdso/vma.c
index 51a2b9f2eca95d..8e1f5ee44f4e68 100644
--- a/arch/x86/um/vdso/vma.c
+++ b/arch/x86/um/vdso/vma.c
@@ -9,6 +9,7 @@
 #include <asm/page.h>
 #include <asm/elf.h>
 #include <linux/init.h>
+#include <os.h>
 
 unsigned long um_vdso_addr;
 static struct page *um_vdso;
@@ -20,18 +21,25 @@ static int __init init_vdso(void)
 {
 	BUG_ON(vdso_end - vdso_start > PAGE_SIZE);
 
-	um_vdso_addr = task_size - PAGE_SIZE;
-
 	um_vdso = alloc_page(GFP_KERNEL);
 	if (!um_vdso)
 		panic("Cannot allocate vdso\n");
 
 	copy_page(page_address(um_vdso), vdso_start);
 
+#ifdef CONFIG_MMU
+	um_vdso_addr = task_size - PAGE_SIZE;
+#else
+	/* this is fine with NOMMU as everything is accessible */
+	um_vdso_addr = (unsigned long)page_address(um_vdso);
+	os_protect_memory((void *)um_vdso_addr, vdso_end - vdso_start, 1, 0, 1);
+#endif
+
 	return 0;
 }
 subsys_initcall(init_vdso);
 
+#ifdef CONFIG_MMU
 int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 {
 	struct vm_area_struct *vma;
@@ -53,3 +61,4 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 
 	return IS_ERR(vma) ? PTR_ERR(vma) : 0;
 }
+#endif /* CONFIG_MMU */
diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
index cff285d8ad8e3e..bd70c4d0b774a3 100644
--- a/arch/x86/virt/svm/sev.c
+++ b/arch/x86/virt/svm/sev.c
@@ -19,6 +19,7 @@
 #include <linux/iommu.h>
 #include <linux/amd-iommu.h>
 #include <linux/nospec.h>
+#include <linux/workqueue.h>
 
 #include <asm/sev.h>
 #include <asm/processor.h>
@@ -124,6 +125,28 @@ static void *rmp_bookkeeping __ro_after_init;
 
 static u64 probed_rmp_base, probed_rmp_size;
 
+static u64 rmpopt_pa_start, rmpopt_pa_end;
+
+enum rmpopt_op_type {
+	RMPOPT_OP_VERIFY_AND_REPORT_STATUS,
+	RMPOPT_OP_REPORT_STATUS
+};
+
+static struct workqueue_struct *rmpopt_wq;
+static struct delayed_work rmpopt_delayed_work;
+
+/* Software RMPOPT facilities initialized */
+static bool rmpopt_soft_init;
+
+/*
+ * Delay, in milliseconds, before the RMP re-optimization pass runs after an
+ * SNP guest is torn down. This coalesces a burst of teardowns into a single
+ * scan and gives each guest's pages time to be converted back to the shared,
+ * hypervisor-owned state.  The 10 second value is a heuristic trading
+ * re-optimization latency against scanning too eagerly.
+ */
+#define RMPOPT_WORK_TIMEOUT	(10 * MSEC_PER_SEC)
+
 static LIST_HEAD(snp_leaked_pages_list);
 static DEFINE_SPINLOCK(snp_leaked_pages_list_lock);
 
@@ -513,7 +536,6 @@ static void clear_hsave_pa(void *arg)
 
 int snp_prepare(void)
 {
-	int ret;
 	u64 val;
 
 	/*
@@ -526,14 +548,18 @@ int snp_prepare(void)
 
 	clear_rmp();
 
-	cpus_read_lock();
+	/*
+	 * No CPU may come online without SnpEn while SNP is active; disable
+	 * hotplug here and re-enable it in snp_shutdown().
+	 */
+	cpu_hotplug_disable();
 
 	if (!cpumask_equal(cpu_online_mask, cpu_present_mask)) {
-		ret = -EOPNOTSUPP;
+		cpu_hotplug_enable();
 		pr_warn("SNP init failed: not all CPUs online. (%*pbl online <-> %*pbl present masks).\n",
 			cpumask_pr_args(cpu_online_mask),
 			cpumask_pr_args(cpu_present_mask));
-		goto unlock;
+		return -EOPNOTSUPP;
 	}
 
 	wbinvd_on_all_cpus();
@@ -548,12 +574,7 @@ int snp_prepare(void)
 	/* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */
 	on_each_cpu(clear_hsave_pa, NULL, 1);
 
-	ret = 0;
-
-unlock:
-	cpus_read_unlock();
-
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp");
 
@@ -565,11 +586,113 @@ void snp_shutdown(void)
 	if (syscfg & MSR_AMD64_SYSCFG_SNP_EN)
 		return;
 
+	if (rmpopt_soft_init)
+		cancel_delayed_work_sync(&rmpopt_delayed_work);
+
 	clear_rmp();
 	on_each_cpu(mfd_reconfigure, NULL, 1);
+
+	/*
+	 * The firmware has disabled SNP (SnpEn is clear), so re-enable CPU
+	 * hotplug.  A legacy SNP shutdown returns above with SnpEn still set and
+	 * leaves hotplug disabled.
+	 */
+	cpu_hotplug_enable();
 }
 EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp");
 
+/*
+ * RMPOPT optimizations skip RMP checks at 1GB granularity if this range of
+ * memory does not contain any SNP guest memory.
+ *
+ * @pa is a system physical address; RMPOPT operates on the containing 1GB.
+ */
+static void rmpopt(u64 pa)
+{
+	enum rmpopt_op_type op = RMPOPT_OP_VERIFY_AND_REPORT_STATUS;
+	u64 pa_start = ALIGN_DOWN(pa, SZ_1G);
+
+	/* Supported by binutils 2.48+ */
+	asm volatile(".byte 0xf2, 0x0f, 0x01, 0xfc"
+		     :: "a" (pa_start), "c" (op)
+		     : "memory", "cc");
+}
+
+static void rmpopt_scan_range(void *arg)
+{
+	u64 pa;
+
+	for (pa = rmpopt_pa_start; pa < rmpopt_pa_end; pa += SZ_1G)
+		rmpopt(pa);
+}
+
+static void do_rmpopt_work(struct work_struct *work)
+{
+	/*
+	 * Warm up the RMPOPT cache on this pinned per-CPU worker with interrupts
+	 * enabled, so the IRQ-disabled fan-out below only issues cache-hit RMPOPTs.
+	 */
+	rmpopt_scan_range(NULL);
+
+	on_each_cpu_mask(cpu_primary_thread_mask, rmpopt_scan_range, NULL, true);
+}
+
+static int __init rmpopt_init(void)
+{
+	if (!cpu_feature_enabled(X86_FEATURE_RMPOPT))
+		return -ENODEV;
+
+	rmpopt_wq = alloc_workqueue("rmpopt_wq", WQ_PERCPU, 1);
+	if (!rmpopt_wq) {
+		pr_err("Failed to allocate RMPOPT workqueue\n");
+		return -ENOMEM;
+	}
+
+	INIT_DELAYED_WORK(&rmpopt_delayed_work, do_rmpopt_work);
+
+	/* The optimization range is fixed at boot; compute it once. */
+	rmpopt_pa_start = ALIGN_DOWN(PFN_PHYS(min_low_pfn), SZ_1G);
+	rmpopt_pa_end = ALIGN(PFN_PHYS(max_pfn), SZ_1G);
+	if ((rmpopt_pa_end - rmpopt_pa_start) > SZ_2T)
+		rmpopt_pa_end = rmpopt_pa_start + SZ_2T;
+
+	pr_info("RMPOPT optimizations enabled\n");
+
+	rmpopt_soft_init = true;
+
+	return 0;
+}
+device_initcall(rmpopt_init);
+
+void snp_enable_rmpopt(void)
+{
+	u64 base;
+	int cpu;
+
+	if (!cpu_feature_enabled(X86_FEATURE_RMPOPT))
+		return;
+
+	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
+		return;
+
+	if (!rmpopt_soft_init)
+		return;
+
+	/*
+	 * Per-CPU RMPOPT tables cover at most 2 TB.  Program each core's
+	 * RMPOPT_BASE with the start of RAM to optimize up to 2 TB.
+	 */
+	rdmsrq(MSR_AMD64_RMPOPT_BASE, base);
+	if (!(base & MSR_AMD64_RMPOPT_ENABLE))
+		for_each_cpu(cpu, cpu_primary_thread_mask)
+			wrmsrq_on_cpu(cpu, MSR_AMD64_RMPOPT_BASE,
+				      rmpopt_pa_start | MSR_AMD64_RMPOPT_ENABLE);
+
+	mod_delayed_work(rmpopt_wq, &rmpopt_delayed_work,
+			 msecs_to_jiffies(RMPOPT_WORK_TIMEOUT));
+}
+EXPORT_SYMBOL_FOR_MODULES(snp_enable_rmpopt, "ccp,kvm-amd");
+
 /*
  * Do the necessary preparations which are verified by the firmware as
  * described in the SNP_INIT_EX firmware command description in the SNP
@@ -577,6 +700,8 @@ EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp");
  */
 int __init snp_rmptable_init(void)
 {
+	u64 val;
+
 	if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP)))
 		return -ENOSYS;
 
@@ -586,6 +711,15 @@ int __init snp_rmptable_init(void)
 	if (!setup_rmptable())
 		return -ENOSYS;
 
+	/*
+	 * On a kexec boot SNP may already be enabled (legacy firmware leaves
+	 * SnpEn set across shutdown), in which case snp_prepare() bails without
+	 * disabling CPU hotplug, so disable it here.
+	 */
+	rdmsrq(MSR_AMD64_SYSCFG, val);
+	if (val & MSR_AMD64_SYSCFG_SNP_EN)
+		cpu_hotplug_disable();
+
 	/*
 	 * Setting crash_kexec_post_notifiers to 'true' to ensure that SNP panic
 	 * notifier is invoked to do SNP IOMMU shutdown before kdump.
@@ -683,13 +817,21 @@ static bool probe_segmented_rmptable_info(void)
 
 bool snp_probe_rmptable_info(void)
 {
-	if (cpu_feature_enabled(X86_FEATURE_SEGMENTED_RMP))
+	if (cpu_feature_enabled(X86_FEATURE_SEGMENTED_RMP)) {
 		rdmsrq(MSR_AMD64_RMP_CFG, rmp_cfg);
 
-	if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED)
-		return probe_segmented_rmptable_info();
-	else
-		return probe_contiguous_rmptable_info();
+		if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) {
+			if (probe_segmented_rmptable_info())
+				return true;
+
+			setup_clear_cpu_cap(X86_FEATURE_RMPOPT);
+			return false;
+		}
+	} else {
+		setup_clear_cpu_cap(X86_FEATURE_RMPOPT);
+	}
+
+	return probe_contiguous_rmptable_info();
 }
 
 /*
diff --git a/arch/x86/virt/vmx/tdx/seamcall_internal.h b/arch/x86/virt/vmx/tdx/seamcall_internal.h
index be5f446467dfad..051ad2d45cab77 100644
--- a/arch/x86/virt/vmx/tdx/seamcall_internal.h
+++ b/arch/x86/virt/vmx/tdx/seamcall_internal.h
@@ -11,6 +11,7 @@
 #ifndef _X86_VIRT_SEAMCALL_INTERNAL_H
 #define _X86_VIRT_SEAMCALL_INTERNAL_H
 
+#include <linux/bitfield.h>
 #include <linux/printk.h>
 #include <linux/types.h>
 #include <asm/archrandom.h>
@@ -23,9 +24,27 @@ u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args);
 
 typedef u64 (*sc_func_t)(u64 fn, struct tdx_module_args *args);
 
+/*
+ * SEAMCALL leaf:
+ *
+ * Bit 15:0	Leaf number
+ * Bit 23:16	Leaf ABI version number
+ * Bit 24	Pending interrupts detection mode
+ * Bit 63	1 for P-SEAMLDR leaf, 0 for TDX module leaf
+ */
+#define SEAMCALL_LEAF_MASK		GENMASK_U64(15, 0)
+#define SEAMCALL_SEAMLDR_MASK		BIT_U64(63)
+
 static __always_inline u64 __seamcall_dirty_cache(sc_func_t func, u64 fn,
 						  struct tdx_module_args *args)
 {
+	/*
+	 * fn contains leaf number for TDX module calls and P-SEAMLDR calls.
+	 * Other fields in SEAMCALL leaf like leaf ABI version number are in
+	 * struct tdx_module_args.
+	 */
+	BUILD_BUG_ON(fn & ~(SEAMCALL_LEAF_MASK | SEAMCALL_SEAMLDR_MASK));
+
 	lockdep_assert_preemption_disabled();
 
 	/*
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index 1b9ff749dd8e02..680e5178a86f20 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -30,7 +30,9 @@
 #include <linux/suspend.h>
 #include <linux/syscore_ops.h>
 #include <linux/idr.h>
+#include <linux/vmalloc.h>
 #include <asm/page.h>
+#include <asm/cacheflush.h>
 #include <asm/special_insns.h>
 #include <asm/msr-index.h>
 #include <asm/msr.h>
@@ -46,6 +48,9 @@
 #include "seamcall_internal.h"
 #include "tdx.h"
 
+/* Number of DPAMT pages to be provided to TDX module per 2MB region of PA */
+#define TDX_DPAMT_ENTRY_PAGE_CNT 2
+
 struct tdx_module_state {
 	bool initialized;
 	bool sysinit_done;
@@ -63,6 +68,14 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized);
 
 static struct tdmr_info_list tdx_tdmr_list;
 
+/*
+ * On a machine with DPAMT, the kernel maintains a reference counter
+ * for every 2MB range. The counter indicates how many users there are for
+ * the DPAMT at the 2MB range. The kernel allocates DPAMT refcounts at
+ * initialization.
+ */
+static atomic_t *dpamt_refcounts;
+
 /* All TDX-usable memory regions.  Protected by mem_hotplug_lock. */
 static LIST_HEAD(tdx_memlist);
 
@@ -252,6 +265,42 @@ static struct syscore tdx_syscore = {
 	.ops = &tdx_syscore_ops,
 };
 
+/*
+ * Allocate DPAMT reference counters for all physical memory.
+ *
+ * It consumes 2MB for every 1TB of physical memory.
+ */
+static __init int init_dpamt_refcounts(void)
+{
+	size_t size = DIV_ROUND_UP(max_pfn, PTRS_PER_PTE) * sizeof(*dpamt_refcounts);
+
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return 0;
+
+	dpamt_refcounts = vzalloc(size);
+	if (!dpamt_refcounts)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static __init void free_dpamt_refcounts(void)
+{
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return;
+
+	vfree(dpamt_refcounts);
+	dpamt_refcounts = NULL;
+}
+
+static atomic_t *tdx_find_dpamt_refcount(unsigned long pfn)
+{
+	/* Find which PMD a PFN is in. */
+	unsigned long index = pfn >> (PMD_SHIFT - PAGE_SHIFT);
+
+	return &dpamt_refcounts[index];
+}
+
 /*
  * Add a memory region as a TDX memory block.  The caller must make sure
  * all memory regions are added in address ascending order and don't
@@ -510,35 +559,37 @@ static __init int fill_out_tdmrs(struct list_head *tmb_list,
 	return 0;
 }
 
+static __init unsigned long tdmr_get_pamt_bitmap_sz(struct tdmr_info *tdmr)
+{
+	unsigned long pamt_sz, nr_pamt_entries;
+	int bits_per_entry;
+
+	bits_per_entry = tdx_sysinfo.tdmr.pamt_page_bitmap_entry_bits;
+	nr_pamt_entries = tdmr->size >> PAGE_SHIFT;
+	pamt_sz = DIV_ROUND_UP(nr_pamt_entries * bits_per_entry, BITS_PER_BYTE);
+
+	return PAGE_ALIGN(pamt_sz);
+}
+
 /*
  * Calculate PAMT size given a TDMR and a page size.  The returned
  * PAMT size is always aligned up to 4K page boundary.
  */
-static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz,
-					     u16 pamt_entry_size)
+static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz)
 {
 	unsigned long pamt_sz, nr_pamt_entries;
+	const int tdx_pg_size_shift[TDX_PS_NR] = { PAGE_SHIFT, PMD_SHIFT, PUD_SHIFT };
+	const u16 pamt_entry_size[TDX_PS_NR] = {
+		tdx_sysinfo.tdmr.pamt_4k_entry_size,
+		tdx_sysinfo.tdmr.pamt_2m_entry_size,
+		tdx_sysinfo.tdmr.pamt_1g_entry_size,
+	};
 
-	switch (pgsz) {
-	case TDX_PS_4K:
-		nr_pamt_entries = tdmr->size >> PAGE_SHIFT;
-		break;
-	case TDX_PS_2M:
-		nr_pamt_entries = tdmr->size >> PMD_SHIFT;
-		break;
-	case TDX_PS_1G:
-		nr_pamt_entries = tdmr->size >> PUD_SHIFT;
-		break;
-	default:
-		WARN_ON_ONCE(1);
-		return 0;
-	}
+	nr_pamt_entries = tdmr->size >> tdx_pg_size_shift[pgsz];
+	pamt_sz = nr_pamt_entries * pamt_entry_size[pgsz];
 
-	pamt_sz = nr_pamt_entries * pamt_entry_size;
 	/* TDX requires PAMT size must be 4K aligned */
-	pamt_sz = ALIGN(pamt_sz, PAGE_SIZE);
-
-	return pamt_sz;
+	return PAGE_ALIGN(pamt_sz);
 }
 
 /*
@@ -576,15 +627,11 @@ static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_lis
  * within @tdmr, and set up PAMTs for @tdmr.
  */
 static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
-				   struct list_head *tmb_list,
-				   u16 pamt_entry_size[])
+				   struct list_head *tmb_list)
 {
-	unsigned long pamt_base[TDX_PS_NR];
-	unsigned long pamt_size[TDX_PS_NR];
-	unsigned long tdmr_pamt_base;
 	unsigned long tdmr_pamt_size;
 	struct page *pamt;
-	int pgsz, nid;
+	int nid;
 
 	nid = tdmr_get_nid(tdmr, tmb_list);
 
@@ -592,13 +639,18 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
 	 * Calculate the PAMT size for each TDX supported page size
 	 * and the total PAMT size.
 	 */
-	tdmr_pamt_size = 0;
-	for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) {
-		pamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz,
-					pamt_entry_size[pgsz]);
-		tdmr_pamt_size += pamt_size[pgsz];
+	tdmr->pamt_1g_size = tdmr_get_pamt_sz(tdmr, TDX_PS_1G);
+	tdmr->pamt_2m_size = tdmr_get_pamt_sz(tdmr, TDX_PS_2M);
+
+	if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) {
+		/* With DPAMT, PAMT_4K is replaced with a bitmap */
+		tdmr->pamt_4k_size = tdmr_get_pamt_bitmap_sz(tdmr);
+	} else {
+		tdmr->pamt_4k_size = tdmr_get_pamt_sz(tdmr, TDX_PS_4K);
 	}
 
+	tdmr_pamt_size = tdmr->pamt_4k_size + tdmr->pamt_2m_size + tdmr->pamt_1g_size;
+
 	/*
 	 * Allocate one chunk of physically contiguous memory for all
 	 * PAMTs.  This helps minimize the PAMT's use of reserved areas
@@ -606,25 +658,17 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
 	 */
 	pamt = alloc_contig_pages(tdmr_pamt_size >> PAGE_SHIFT, GFP_KERNEL,
 			nid, &node_online_map);
-	if (!pamt)
-		return -ENOMEM;
 
 	/*
-	 * Break the contiguous allocation back up into the
-	 * individual PAMTs for each page size.
+	 * tdmr->pamt_4k_base is still zero so the error
+	 * path of the caller will skip freeing the PAMT.
 	 */
-	tdmr_pamt_base = page_to_pfn(pamt) << PAGE_SHIFT;
-	for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) {
-		pamt_base[pgsz] = tdmr_pamt_base;
-		tdmr_pamt_base += pamt_size[pgsz];
-	}
+	if (!pamt)
+		return -ENOMEM;
 
-	tdmr->pamt_4k_base = pamt_base[TDX_PS_4K];
-	tdmr->pamt_4k_size = pamt_size[TDX_PS_4K];
-	tdmr->pamt_2m_base = pamt_base[TDX_PS_2M];
-	tdmr->pamt_2m_size = pamt_size[TDX_PS_2M];
-	tdmr->pamt_1g_base = pamt_base[TDX_PS_1G];
-	tdmr->pamt_1g_size = pamt_size[TDX_PS_1G];
+	tdmr->pamt_4k_base = page_to_phys(pamt);
+	tdmr->pamt_2m_base = tdmr->pamt_4k_base + tdmr->pamt_4k_size;
+	tdmr->pamt_1g_base = tdmr->pamt_2m_base + tdmr->pamt_2m_size;
 
 	return 0;
 }
@@ -655,10 +699,7 @@ static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr,
 	tdmr_get_pamt(tdmr, &pamt_base, &pamt_size);
 
 	/* Do nothing if PAMT hasn't been allocated for this TDMR */
-	if (!pamt_size)
-		return;
-
-	if (WARN_ON_ONCE(!pamt_base))
+	if (!pamt_base)
 		return;
 
 	pamt_func(pamt_base, pamt_size);
@@ -684,14 +725,12 @@ static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list)
 
 /* Allocate and set up PAMTs for all TDMRs */
 static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list,
-					struct list_head *tmb_list,
-					u16 pamt_entry_size[])
+					struct list_head *tmb_list)
 {
 	int i, ret = 0;
 
 	for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) {
-		ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list,
-				pamt_entry_size);
+		ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list);
 		if (ret)
 			goto err;
 	}
@@ -968,18 +1007,13 @@ static __init int construct_tdmrs(struct list_head *tmb_list,
 				  struct tdmr_info_list *tdmr_list,
 				  struct tdx_sys_info_tdmr *sysinfo_tdmr)
 {
-	u16 pamt_entry_size[TDX_PS_NR] = {
-		sysinfo_tdmr->pamt_4k_entry_size,
-		sysinfo_tdmr->pamt_2m_entry_size,
-		sysinfo_tdmr->pamt_1g_entry_size,
-	};
 	int ret;
 
 	ret = fill_out_tdmrs(tmb_list, tdmr_list);
 	if (ret)
 		return ret;
 
-	ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size);
+	ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list);
 	if (ret)
 		return ret;
 
@@ -998,6 +1032,8 @@ static __init int construct_tdmrs(struct list_head *tmb_list,
 	return ret;
 }
 
+#define TDX_SYS_CONFIG_DYNAMIC_PAMT	BIT(16)
+
 static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
 				    u64 global_keyid)
 {
@@ -1026,6 +1062,12 @@ static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
 	args.rcx = __pa(tdmr_pa_array);
 	args.rdx = tdmr_list->nr_consumed_tdmrs;
 	args.r8 = global_keyid;
+
+	if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) {
+		pr_info("Enable Dynamic PAMT\n");
+		args.r8 |= TDX_SYS_CONFIG_DYNAMIC_PAMT;
+	}
+
 	ret = seamcall_prerr(TDH_SYS_CONFIG, &args);
 
 	/* Free the array as it is not required anymore. */
@@ -1167,10 +1209,14 @@ static __init int init_tdx_module(void)
 	 */
 	get_online_mems();
 
-	ret = build_tdx_memlist(&tdx_memlist);
+	ret = init_dpamt_refcounts();
 	if (ret)
 		goto out_put_tdxmem;
 
+	ret = build_tdx_memlist(&tdx_memlist);
+	if (ret)
+		goto err_free_dpamt_refcounts;
+
 	/* Allocate enough space for constructing TDMRs */
 	ret = alloc_tdmr_list(&tdx_tdmr_list, &tdx_sysinfo.tdmr);
 	if (ret)
@@ -1220,6 +1266,8 @@ static __init int init_tdx_module(void)
 	free_tdmr_list(&tdx_tdmr_list);
 err_free_tdxmem:
 	free_tdx_memlist(&tdx_memlist);
+err_free_dpamt_refcounts:
+	free_dpamt_refcounts();
 	goto out_put_tdxmem;
 }
 
@@ -1912,10 +1960,11 @@ u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid)
 		.rcx = vp->tdvpr_pa,
 		.rdx = initial_rcx,
 		.r8 = x2apicid,
+		/* apicid requires version == 1. */
+		.version = 1,
 	};
 
-	/* apicid requires version == 1. */
-	return seamcall(TDH_VP_INIT | (1ULL << TDX_VERSION_SHIFT), &args);
+	return seamcall(TDH_VP_INIT, &args);
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_vp_init);
 
@@ -2005,6 +2054,269 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn)
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid);
 
+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo)
+{
+	return sysinfo->features.tdx_features0 & TDX_FEATURES0_DYNAMIC_PAMT;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_supports_dynamic_pamt);
+
+static struct page *tdx_alloc_page_pamt_cache(struct tdx_pamt_cache *cache)
+{
+	struct page *page;
+
+	page = list_first_entry_or_null(&cache->page_list, struct page, lru);
+	if (page) {
+		list_del(&page->lru);
+		cache->cnt--;
+	}
+
+	return page;
+}
+
+static struct page *alloc_dpamt_page(struct tdx_pamt_cache *cache)
+{
+	if (cache)
+		return tdx_alloc_page_pamt_cache(cache);
+
+	return alloc_page(GFP_KERNEL_ACCOUNT);
+}
+
+static int alloc_pamt_array(struct page **pamt_pages, struct tdx_pamt_cache *cache)
+{
+	int i, j;
+
+	for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) {
+		pamt_pages[i] = alloc_dpamt_page(cache);
+		if (!pamt_pages[i])
+			goto err;
+	}
+
+	return 0;
+
+err:
+	for (j = 0; j < i; j++)
+		__free_page(pamt_pages[j]);
+
+	return -ENOMEM;
+}
+
+static void free_pamt_array(struct page **pamt_pages)
+{
+	int i;
+
+	for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) {
+		/*
+		 * Reset pages unconditionally to cover cases
+		 * where they were passed to the TDX module.
+		 */
+		tdx_quirk_reset_paddr(page_to_phys(pamt_pages[i]), PAGE_SIZE);
+
+		__free_page(pamt_pages[i]);
+	}
+}
+
+/* Helper for building DPAMT seamcall() arguments. */
+static u64 pamt_2mb_arg(kvm_pfn_t pfn)
+{
+	/* Find the 2MB-wide DPAMT region for 'pfn': */
+	unsigned long hpa_2mb = ALIGN_DOWN(pfn << PAGE_SHIFT, PMD_SIZE);
+
+	/*
+	 * TDX ABI requires specifying the page level the installed DPAMT
+	 * backing will cover, even though today only 2MB is supported.
+	 */
+	return hpa_2mb | TDX_PS_2M;
+}
+
+/* Add DPAMT backing for the 2MB region surrounding the given pfn. */
+static u64 tdh_phymem_pamt_add(kvm_pfn_t pfn, struct page **pamt_pages)
+{
+	struct tdx_module_args args = {
+		.rcx = pamt_2mb_arg(pfn),
+		.rdx = page_to_phys(pamt_pages[0]),
+		.r8  = page_to_phys(pamt_pages[1]),
+	};
+
+	return seamcall(TDH_PHYMEM_PAMT_ADD, &args);
+}
+
+/* Remove DPAMT backing for the 2MB region surrounding the given pfn. */
+static u64 tdh_phymem_pamt_remove(kvm_pfn_t pfn, struct page **pamt_pages)
+{
+	struct tdx_module_args args = {
+		.rcx = pamt_2mb_arg(pfn),
+	};
+	u64 ret;
+
+	ret = seamcall_ret(TDH_PHYMEM_PAMT_REMOVE, &args);
+	if (ret)
+		return ret;
+
+	/* Copy PAMT pages out of the struct per the TDX ABI */
+	pamt_pages[0] = phys_to_page(args.rdx);
+	pamt_pages[1] = phys_to_page(args.r8);
+
+	return 0;
+}
+
+/* Serializes adding/removing DPAMT memory */
+static DEFINE_SPINLOCK(dpamt_lock);
+
+/* Bump DPAMT refcount for the given pfn and allocate DPAMT backing if needed. */
+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache)
+{
+	struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT];
+	atomic_t *dpamt_refcount;
+	u64 tdx_status;
+	int ret;
+
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return 0;
+
+	ret = alloc_pamt_array(pamt_pages, cache);
+	if (ret)
+		return ret;
+
+	dpamt_refcount = tdx_find_dpamt_refcount(pfn);
+
+	spin_lock(&dpamt_lock);
+
+	/*
+	 * If the DPAMT entry is already added (i.e. refcount >= 1),
+	 * then just increment the refcount.
+	 */
+	if (atomic_inc_not_zero(dpamt_refcount))
+		goto out_free;
+
+	/* Try to add the PAMT page and take the refcount 0->1. */
+	tdx_status = tdh_phymem_pamt_add(pfn, pamt_pages);
+	if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {
+		ret = -EIO;
+		goto out_free;
+	}
+
+	atomic_set(dpamt_refcount, 1);
+	spin_unlock(&dpamt_lock);
+	return 0;
+
+out_free:
+	spin_unlock(&dpamt_lock);
+	free_pamt_array(pamt_pages);
+
+	return ret;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_get);
+
+/* Drop DPAMT refcount for the given pfn and free DPAMT backing if needed. */
+void tdx_pamt_put(kvm_pfn_t pfn)
+{
+	struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT] = {};
+	atomic_t *dpamt_refcount;
+	u64 tdx_status;
+
+	if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+		return;
+
+	dpamt_refcount = tdx_find_dpamt_refcount(pfn);
+
+	spin_lock(&dpamt_lock);
+	/*
+	 * If there is more than 1 reference on the DPAMT entry, don't
+	 * remove it yet. Just decrement the refcount.
+	 */
+	if (atomic_read(dpamt_refcount) > 1) {
+		atomic_dec(dpamt_refcount);
+		goto out_unlock;
+	}
+
+	/* Try to remove the pamt page and take the refcount 1->0. */
+	tdx_status = tdh_phymem_pamt_remove(pfn, pamt_pages);
+
+	/*
+	 * Don't free pamt_pages as it could hold garbage when
+	 * tdh_phymem_pamt_remove() fails.  Don't panic/BUG_ON(), as
+	 * there is no risk of data corruption, but do yell loudly as
+	 * failure indicates a kernel bug, memory is being leaked, and
+	 * the dangling DPAMT entry may cause future operations to fail.
+	 */
+	if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS))
+		goto out_unlock;
+
+	atomic_set(dpamt_refcount, 0);
+	spin_unlock(&dpamt_lock);
+	free_pamt_array(pamt_pages);
+	return;
+out_unlock:
+	spin_unlock(&dpamt_lock);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_put);
+
+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache)
+{
+	struct page *page;
+
+	while ((page = tdx_alloc_page_pamt_cache(cache)))
+		__free_page(page);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_free_pamt_cache);
+
+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages)
+{
+	if (WARN_ON_ONCE(!tdx_supports_dynamic_pamt(&tdx_sysinfo)))
+		return 0;
+
+	npages *= TDX_DPAMT_ENTRY_PAGE_CNT;
+
+	while (cache->cnt < npages) {
+		struct page *page = alloc_page(GFP_KERNEL_ACCOUNT);
+
+		if (!page)
+			return -ENOMEM;
+
+		list_add(&page->lru, &cache->page_list);
+		cache->cnt++;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_topup_pamt_cache);
+
+/*
+ * Return a page that can be gifted to the TDX module for use as a "control"
+ * page, i.e. pages that are used for control structures for a given TDX
+ * guest, and thus obtain TDX protections, including DPAMT tracking.
+ */
+struct page *tdx_alloc_control_page(void)
+{
+	struct page *page;
+
+	page = alloc_page(GFP_KERNEL_ACCOUNT);
+	if (!page)
+		return NULL;
+
+	if (tdx_pamt_get(page_to_pfn(page), NULL)) {
+		__free_page(page);
+		return NULL;
+	}
+
+	return page;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_alloc_control_page);
+
+/*
+ * Free a page that was gifted to the TDX module for use as a control
+ * page. After this, the page is no longer protected by TDX.
+ */
+void tdx_free_control_page(struct page *page)
+{
+	if (!page)
+		return;
+
+	tdx_pamt_put(page_to_pfn(page));
+	__free_page(page);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_free_control_page);
+
 void tdx_sys_disable(void)
 {
 	struct tdx_module_args args = {};
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index bdfd0e1e337ac0..db209541d3cdfb 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -48,16 +48,10 @@
 #define TDH_SYS_CONFIG			45
 #define TDH_SYS_SHUTDOWN		52
 #define TDH_SYS_UPDATE			53
+#define TDH_PHYMEM_PAMT_ADD		58
+#define TDH_PHYMEM_PAMT_REMOVE		59
 #define TDH_SYS_DISABLE			69
 
-/*
- * SEAMCALL leaf:
- *
- * Bit 15:0	Leaf number
- * Bit 23:16	Version number
- */
-#define TDX_VERSION_SHIFT		16
-
 /* TDX page types */
 #define	PT_NDA		0x0
 #define	PT_RSVD		0x1
diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
index e49c300f23d435..98ebf17aab1ce9 100644
--- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Automatically generated functions to read TDX global metadata.
+ * Functions to read TDX global metadata.
  *
  * This file doesn't compile on its own as it lacks of inclusion
  * of SEAMCALL wrapper primitive which reads global metadata.
@@ -33,6 +33,18 @@ static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinf
 	return ret;
 }
 
+static __init int get_tdx_sys_info_tdmr_dpamt(struct tdx_sys_info_tdmr *sysinfo_tdmr)
+{
+	int ret;
+	u64 val;
+
+	ret = read_sys_metadata_field(0x9100000000000013, &val);
+	if (!ret)
+		sysinfo_tdmr->pamt_page_bitmap_entry_bits = val;
+
+	return ret;
+}
+
 static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr)
 {
 	int ret = 0;
@@ -129,5 +141,14 @@ static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo)
 	ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl);
 	ret = ret ?: get_tdx_sys_info_td_conf(&sysinfo->td_conf);
 
+	/*
+	 * The kernel supports using TDX without DPAMT, so
+	 * avoid reporting failure if it's not supported. Don't
+	 * try to support buggy TDX modules that advertise
+	 * DPAMT but don't expose the metadata.
+	 */
+	if (!ret && tdx_supports_dynamic_pamt(sysinfo))
+		ret = get_tdx_sys_info_tdmr_dpamt(&sysinfo->tdmr);
+
 	return ret;
 }
diff --git a/arch/x86/virt/vmx/tdx/tdxcall.S b/arch/x86/virt/vmx/tdx/tdxcall.S
index 016a2a1ec1d691..a194e83613e7bf 100644
--- a/arch/x86/virt/vmx/tdx/tdxcall.S
+++ b/arch/x86/virt/vmx/tdx/tdxcall.S
@@ -45,8 +45,14 @@
 .macro TDX_MODULE_CALL host:req ret=0 saved=0
 	FRAME_BEGIN
 
-	/* Move Leaf ID to RAX */
-	mov %rdi, %rax
+	/* Leaf ABI version -> RAX[23:16]. Zero rest of RAX. */
+	movzbl	TDX_MODULE_version(%rsi), %eax
+	shl	$16, %eax
+	/*
+	 * Combine leaf number arg and leaf ABI version into RAX, they don't
+	 * overlap.
+	 */
+	or	%rdi, %rax
 
 	/* Move other input regs from 'struct tdx_module_args' */
 	movq	TDX_MODULE_rcx(%rsi), %rcx
diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c
index 5f50a3ee08f509..3f4dd3f50f56bc 100644
--- a/arch/x86/xen/pmu.c
+++ b/arch/x86/xen/pmu.c
@@ -456,12 +456,14 @@ static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
 	}
 }
 
+static DEFINE_PER_CPU(struct x86_perf_regs, x86_xen_intr_regs);
 irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
 {
 	int err, ret = IRQ_NONE;
 	struct pt_regs regs = {0};
 	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
 	uint8_t xenpmu_flags = get_xenpmu_flags();
+	struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_xen_intr_regs);
 
 	if (!xenpmu_data) {
 		pr_warn_once("%s: pmudata not initialized\n", __func__);
@@ -472,7 +474,8 @@ irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
 		xenpmu_flags | XENPMU_IRQ_PROCESSING;
 	xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
 			 xenpmu_data->pmu.pmu_flags);
-	if (x86_pmu.handle_irq(&regs))
+	x86_regs->regs = regs;
+	if (x86_pmu.handle_irq(&x86_regs->regs))
 		ret = IRQ_HANDLED;
 
 	/* Write out cached context to HW */
diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h
index 7308b7f777d79d..e00f55e457e0e3 100644
--- a/arch/xtensa/include/asm/atomic.h
+++ b/arch/xtensa/include/asm/atomic.h
@@ -37,21 +37,8 @@
  * where no register reference will cause an overflow.
  */
 
-/**
- * atomic_read - read atomic variable
- * @v: pointer of type atomic_t
- *
- * Atomically reads the value of @v.
- */
 #define arch_atomic_read(v)		READ_ONCE((v)->counter)
 
-/**
- * atomic_set - set atomic variable
- * @v: pointer of type atomic_t
- * @i: required value
- *
- * Atomically sets the value of @v to @i.
- */
 #define arch_atomic_set(v,i)		WRITE_ONCE((v)->counter, (i))
 
 #if XCHAL_HAVE_EXCLUSIVE
diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h
index f00a879dc298a5..60fb67a9972907 100644
--- a/arch/xtensa/include/asm/pgtable.h
+++ b/arch/xtensa/include/asm/pgtable.h
@@ -204,10 +204,6 @@
  */
 #ifndef __ASSEMBLER__
 
-#define pte_ERROR(e) \
-	printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
-#define pgd_ERROR(e) \
-	printk("%s:%d: bad pgd entry %08lx.\n", __FILE__, __LINE__, pgd_val(e))
 
 #ifdef CONFIG_MMU
 extern pgd_t swapper_pg_dir[PAGE_SIZE/sizeof(pgd_t)];
diff --git a/arch/xtensa/include/asm/tlb.h b/arch/xtensa/include/asm/tlb.h
index 8c3ceb4270180b..6fb7b78154f62a 100644
--- a/arch/xtensa/include/asm/tlb.h
+++ b/arch/xtensa/include/asm/tlb.h
@@ -16,7 +16,7 @@
 
 #include <asm-generic/tlb.h>
 
-#define __pte_free_tlb(tlb, pte, address)	pte_free((tlb)->mm, pte)
+#define __pte_free_tlb(tlb, pte, address)	tlb_remove_ptdesc((tlb), page_ptdesc(pte))
 
 void check_tlb_sanity(void);
 
diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h
index 99d4ccee7f6e8a..aa64ab9109c501 100644
--- a/arch/xtensa/include/uapi/asm/mman.h
+++ b/arch/xtensa/include/uapi/asm/mman.h
@@ -96,7 +96,7 @@
 #define MADV_HUGEPAGE	14		/* Worth backing with hugepages */
 #define MADV_NOHUGEPAGE	15		/* Not worth backing with hugepages */
 
-#define MADV_DONTDUMP   16		/* Explicity exclude from the core dump,
+#define MADV_DONTDUMP   16		/* Explicitly exclude from the core dump,
 					   overrides the coredump filter bits */
 #define MADV_DODUMP	17		/* Clear the MADV_NODUMP flag */
 
diff --git a/arch/xtensa/include/uapi/asm/setup.h b/arch/xtensa/include/uapi/asm/setup.h
index 5356a5fd4d1737..dcf33a403527a3 100644
--- a/arch/xtensa/include/uapi/asm/setup.h
+++ b/arch/xtensa/include/uapi/asm/setup.h
@@ -12,6 +12,10 @@
 #ifndef _XTENSA_SETUP_H
 #define _XTENSA_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	256
+#endif
 
 #endif
diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S
index 272fff587907fa..49b4c9eca80e33 100644
--- a/arch/xtensa/kernel/entry.S
+++ b/arch/xtensa/kernel/entry.S
@@ -1671,7 +1671,7 @@ ENTRY(fast_second_level_miss)
 	 * This allows to map the three most common regions to three different
 	 * DTLBs:
 	 *  0,1 -> way 7	program (0040.0000) and virtual (c000.0000)
-	 *  2   -> way 8	shared libaries (2000.0000)
+	 *  2   -> way 8	shared libraries (2000.0000)
 	 *  3   -> way 0	stack (3000.0000)
 	 */
 
diff --git a/arch/xtensa/kernel/hibernate.c b/arch/xtensa/kernel/hibernate.c
index 06984327d6e277..fba3ab10c0eb1a 100644
--- a/arch/xtensa/kernel/hibernate.c
+++ b/arch/xtensa/kernel/hibernate.c
@@ -3,6 +3,7 @@
 #include <linux/mm.h>
 #include <linux/suspend.h>
 #include <asm/coprocessor.h>
+#include <asm/sections.h>
 
 int pfn_is_nosave(unsigned long pfn)
 {
diff --git a/arch/xtensa/kernel/time.c b/arch/xtensa/kernel/time.c
index 09fda6a3fe3bee..bccc0e9e56073e 100644
--- a/arch/xtensa/kernel/time.c
+++ b/arch/xtensa/kernel/time.c
@@ -152,6 +152,7 @@ static void __init calibrate_ccount(void)
 		of_node_put(cpu);
 		if (!IS_ERR(clk)) {
 			ccount_freq = clk_get_rate(clk);
+			clk_put(clk);
 			return;
 		} else {
 			pr_warn("%s: CPU input clock not found\n",
diff --git a/arch/xtensa/kernel/vectors.S b/arch/xtensa/kernel/vectors.S
index 1073fe4a584da7..063a6acc64e9ba 100644
--- a/arch/xtensa/kernel/vectors.S
+++ b/arch/xtensa/kernel/vectors.S
@@ -21,7 +21,7 @@
  * use a first-level dispatch table to dispatch the exception to a registered
  * fast handler or the default handler, if no fast handler was registered.
  * The default handler sets up a C-stack and dispatches the exception to a
- * registerd C handler in the second-level dispatch table.
+ * registered C handler in the second-level dispatch table.
  *
  * Fast handler entry condition:
  *
diff --git a/arch/xtensa/mm/init.c b/arch/xtensa/mm/init.c
index fe83a68335da29..7b5f82932a4557 100644
--- a/arch/xtensa/mm/init.c
+++ b/arch/xtensa/mm/init.c
@@ -43,7 +43,7 @@ void __init bootmem_init(void)
 	 * accounting doesn't work for pages below that address.
 	 *
 	 * If PHYS_OFFSET is zero reserve page at address 0:
-	 * successfull allocations should never return NULL.
+	 * successful allocations should never return NULL.
 	 */
 	memblock_reserve(0, PHYS_OFFSET ? PHYS_OFFSET : 1);
 
diff --git a/block/bdev.c b/block/bdev.c
index cd832308374060..fac74319e9fbac 100644
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -292,7 +292,7 @@ EXPORT_SYMBOL(sync_blockdev_range);
  * The reference counter (bd_fsfreeze_count) guarantees that only the last
  * unfreeze process can unfreeze the frozen filesystem actually when multiple
  * freeze requests arrive simultaneously. It counts up in bdev_freeze() and
- * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze
+ * count down in bdev_thaw(). When it becomes 0, bdev_thaw() will unfreeze
  * actually.
  *
  * Return: On success zero is returned, negative error code on failure.
diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c
index 692403dfa047bb..c8e91ada8ca61c 100644
--- a/block/bio-integrity-fs.c
+++ b/block/bio-integrity-fs.c
@@ -31,6 +31,7 @@ unsigned int fs_bio_integrity_alloc(struct bio *bio)
 		bio_integrity_setup_default(bio);
 	return action;
 }
+EXPORT_SYMBOL_GPL(fs_bio_integrity_alloc);
 
 void fs_bio_integrity_free(struct bio *bio)
 {
@@ -43,6 +44,7 @@ void fs_bio_integrity_free(struct bio *bio)
 	bio->bi_integrity = NULL;
 	bio->bi_opf &= ~REQ_INTEGRITY;
 }
+EXPORT_SYMBOL_GPL(fs_bio_integrity_free);
 
 void fs_bio_integrity_generate(struct bio *bio)
 {
@@ -52,14 +54,10 @@ void fs_bio_integrity_generate(struct bio *bio)
 }
 EXPORT_SYMBOL_GPL(fs_bio_integrity_generate);
 
-int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
+int fs_bio_integrity_verify(struct bio *bio, struct bvec_iter *data_iter)
 {
 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
 	struct bio_integrity_payload *bip = bio_integrity(bio);
-	struct bvec_iter data_iter = {
-		.bi_sector	= sector,
-		.bi_size	= size,
-	};
 
 	if (!bip || !(bip->bip_flags & BIP_CHECK_FLAGS))
 		return 0;
@@ -71,9 +69,10 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
 	 * bio.  Requires the submitter to remember the sector and the size.
 	 */
 	memset(&bip->bip_iter, 0, sizeof(bip->bip_iter));
-	bip->bip_iter.bi_sector = sector;
-	bip->bip_iter.bi_size = bio_integrity_bytes(bi, size >> SECTOR_SHIFT);
-	return blk_status_to_errno(bio_integrity_verify(bio, &data_iter));
+	bip->bip_iter.bi_sector = data_iter->bi_sector;
+	bip->bip_iter.bi_size =
+		bio_integrity_bytes(bi, data_iter->bi_size >> SECTOR_SHIFT);
+	return blk_status_to_errno(bio_integrity_verify(bio, data_iter));
 }
 
 static int __init fs_bio_integrity_init(void)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index b23e2434d80c2c..d3df726e0f0881 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -72,6 +72,7 @@ void bio_integrity_alloc_buf(struct bio *bio, gfp_t gfp, bool zero_buffer)
 	unsigned int len = bio_integrity_bytes(bi, bio_sectors(bio));
 	void *buf;
 
+	WARN_ON_ONCE(len > BLK_INTEGRITY_MAX_SIZE);
 	buf = kmalloc(len, gfp | __GFP_NOWARN | (zero_buffer ? __GFP_ZERO : 0));
 	if (unlikely(!buf)) {
 		struct page *page;
diff --git a/block/bio.c b/block/bio.c
index f95b63c0604afd..b48091c7663fa4 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -320,6 +320,26 @@ void bio_reuse(struct bio *bio, blk_opf_t opf)
 }
 EXPORT_SYMBOL_GPL(bio_reuse);
 
+/**
+ * bio_prepare_reissue - prepare a bio for reuissing the original I/O
+ * @bio:	bio to reuse
+ * @bdev:	block device to use the bio for
+ *
+ * Prepare @bio to be resubmitted to retry the original operation.
+ * The caller must reset bio->bi_iter to the original state.
+ */
+void bio_prepare_reissue(struct bio *bio, struct block_device *bdev)
+{
+	bio->bi_bdev = bdev;
+	bio_associate_blkg(bio);
+	bio->bi_flags &=
+		(BIO_PAGE_PINNED | BIO_CLONED | BIO_QUIET | BIO_REFFED);
+	bio->bi_status = BLK_STS_OK;
+	bio->bi_bvec_gap_bit = 0;
+	atomic_set(&bio->__bi_remaining, 1);
+}
+EXPORT_SYMBOL_GPL(bio_prepare_reissue);
+
 static struct bio *__bio_chain_endio(struct bio *bio)
 {
 	struct bio *parent = bio->bi_private;
@@ -1203,8 +1223,9 @@ bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter)
  * for the next iteration.
  */
 static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
-				   struct bio_vec *bv, unsigned len_align_mask)
+				   unsigned len_align_mask)
 {
+	struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
 	size_t nbytes = bio->bi_iter.bi_size & len_align_mask;
 
 	if (!nbytes)
@@ -1262,6 +1283,7 @@ static inline bool bio_iov_bvec_aligned(const struct bio *bio,
  * bio_iov_iter_get_pages - add user or kernel pages to a bio
  * @bio: bio to add pages to
  * @iter: iov iterator describing the region to be added
+ * @maxlen: maximum size to consume from @iter
  * @mem_align_mask: the mask the source address and length must be aligned to,
  *	0 for no requirement
  * @len_align_mask: the mask to align the total size to, 0 for any length
@@ -1282,7 +1304,8 @@ static inline bool bio_iov_bvec_aligned(const struct bio *bio,
  * is returned only if 0 pages could be pinned.
  */
 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
-			   unsigned mem_align_mask, unsigned len_align_mask)
+			   unsigned maxlen, unsigned mem_align_mask,
+			   unsigned len_align_mask)
 {
 	iov_iter_extraction_t flags = 0;
 
@@ -1294,6 +1317,8 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
 		    !bio_iov_bvec_aligned(bio, mem_align_mask))
 			return -EINVAL;
 
+		/* Truncate to the maximum size that the caller can handle */
+		bio->bi_iter.bi_size = min(bio->bi_iter.bi_size, maxlen);
 		iov_iter_advance(iter, bio->bi_iter.bi_size);
 		return 0;
 	}
@@ -1307,7 +1332,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
 		ssize_t ret;
 
 		ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec,
-				BIO_MAX_SIZE - bio->bi_iter.bi_size,
+				maxlen - bio->bi_iter.bi_size,
 				&bio->bi_vcnt, bio->bi_max_vecs,
 				mem_align_mask, flags);
 		if (ret <= 0) {
@@ -1330,8 +1355,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
 
 	if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page))
 		bio->bi_opf |= REQ_NOMERGE;
-	return bio_iov_iter_align_down(bio, iter,
-			&bio->bi_io_vec[bio->bi_vcnt - 1], len_align_mask);
+	return bio_iov_iter_align_down(bio, iter, len_align_mask);
 }
 
 static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
@@ -1350,7 +1374,7 @@ static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size,
 	return folio_alloc(gfp, get_order(*size));
 }
 
-static void bio_free_folios(struct bio *bio)
+void bio_free_folios(struct bio *bio)
 {
 	struct bio_vec *bv;
 	int i;
@@ -1363,11 +1387,8 @@ static void bio_free_folios(struct bio *bio)
 	}
 }
 
-static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
-		size_t maxlen, size_t minsize)
+int bio_alloc_bounce_folios(struct bio *bio, size_t total_len, size_t minsize)
 {
-	size_t total_len = min(maxlen, iov_iter_count(iter));
-
 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
 		return -EINVAL;
 	if (WARN_ON_ONCE(bio->bi_iter.bi_size))
@@ -1377,7 +1398,6 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
 
 	do {
 		size_t this_len = min(total_len, SZ_1M);
-		size_t copied;
 		struct folio *folio;
 
 		if (this_len > minsize * 2)
@@ -1389,164 +1409,69 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
 		folio = folio_alloc_greedy(GFP_KERNEL, &this_len, minsize);
 		if (!folio)
 			break;
-		bio_add_folio_nofail(bio, folio, this_len, 0);
 
-		if (iter->nofault)
-			copied = copy_folio_from_iter_atomic(folio, 0, this_len,
-							     iter);
-		else
-			copied = copy_folio_from_iter(folio, 0, this_len, iter);
-		if (copied < this_len) {
-			/*
-			 * Need to revert the iov iter for all bytes we have
-			 * copied.
-			 *
-			 * However the bio size differs from the real copied
-			 * bytes as @this_len is queued but only advanced
-			 * less than that.
-			 * Need to compensate that for the revert.
-			 */
-			iov_iter_revert(iter, bio->bi_iter.bi_size - this_len +
-					copied);
-			bio_free_folios(bio);
-			return -EFAULT;
-		}
+		/*
+		 * Align down the size to the minimum alignment.  In practice
+		 * this should not happen as minsize is expected to be a power
+		 * of two, as is the allocation size, but it offers us a cheap
+		 * extra safety belt.
+		 */
+		this_len &= ~(minsize - 1);
+		bio_add_folio_nofail(bio, folio, this_len, 0);
 		total_len -= this_len;
 	} while (total_len && bio->bi_vcnt < bio->bi_max_vecs);
 
 	if (!bio->bi_iter.bi_size)
 		return -ENOMEM;
-	return bio_iov_iter_align_down(bio, iter,
-			&bio->bi_io_vec[bio->bi_vcnt - 1], minsize - 1);
-}
-
-static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter,
-		size_t maxlen, size_t minsize)
-{
-	size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M);
-	struct folio *folio;
-	ssize_t ret;
-
-	folio = folio_alloc_greedy(GFP_KERNEL, &len, minsize);
-	if (!folio)
-		return -ENOMEM;
-
-	do {
-		ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len,
-				&bio->bi_vcnt, bio->bi_max_vecs - 1, 0, 0);
-		if (ret <= 0) {
-			if (!bio->bi_vcnt)
-				goto out_folio_put;
-			break;
-		}
-		len -= ret;
-		bio->bi_iter.bi_size += ret;
-	} while (len && bio->bi_vcnt < bio->bi_max_vecs - 1);
-
-	/*
-	 * Set the folio directly here.  The above loop has already calculated
-	 * the correct bi_size, and we use bi_vcnt for the user buffers.  That
-	 * is safe as bi_vcnt is only used by the submitter and not the actual
-	 * I/O path.
-	 */
-	bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0);
-	if (iov_iter_extract_will_pin(iter))
-		bio_set_flag(bio, BIO_PAGE_PINNED);
-
-	/* The first vec stores the bounce buffer, so do not subtract 1 here. */
-	ret = bio_iov_iter_align_down(bio, iter,
-			&bio->bi_io_vec[bio->bi_vcnt], minsize - 1);
-	if (ret)
-		goto out_folio_put;
-
-	/* Update the bounc buffer bv_len to the aligned down size. */
-	bio->bi_io_vec[0].bv_len = bio->bi_iter.bi_size;
 	return 0;
-
-out_folio_put:
-	folio_put(folio);
-	return ret;
 }
 
 /**
- * bio_iov_iter_bounce - bounce buffer data from an iter into a bio
+ * bio_iov_iter_bounce_write - bounce buffer data from an iter into a bio
  * @bio:	bio to send
- * @iter:	iter to read from / write into
+ * @iter:	iter to read from
  * @maxlen:	maximum size to bounce
  * @minsize:	minimum folio allocation size
  *
- * Helper for direct I/O implementations that need to bounce buffer because
- * we need to checksum the data or perform other operations that require
- * consistency.  Allocates folios to back the bounce buffer, and for writes
- * copies the data into it.  Needs to be paired with bio_iov_iter_unbounce()
- * called on completion.
+ * Helper for direct I/O write implementations that need to bounce buffer
+ * because they need need to checksum the data or perform other operations that
+ * require consistency.  Allocates folios to back the bounce buffer, and copies
+ * the data into it.  Needs to be paired with bio_free_folios() called on
+ * completion.
  */
-int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen,
-			size_t minsize)
-{
-	if (op_is_write(bio_op(bio)))
-		return bio_iov_iter_bounce_write(bio, iter, maxlen, minsize);
-	return bio_iov_iter_bounce_read(bio, iter, maxlen, minsize);
-}
-
-static void bvec_unpin(struct bio_vec *bv, bool mark_dirty)
-{
-	struct folio *folio = bvec_folio(bv);
-	size_t nr_pages = (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE -
-			bv->bv_offset / PAGE_SIZE + 1;
-
-	if (mark_dirty)
-		folio_mark_dirty_lock(folio);
-	unpin_user_folio(folio, nr_pages);
-}
-
-static void bio_iov_iter_unbounce_read(struct bio *bio, bool is_error,
-		bool mark_dirty)
+int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
+		size_t maxlen, size_t minsize)
 {
-	unsigned int len = bio->bi_io_vec[0].bv_len;
-
-	if (likely(!is_error)) {
-		void *buf = bvec_virt(&bio->bi_io_vec[0]);
-		struct iov_iter to;
+	size_t total_len = min(maxlen, iov_iter_count(iter));
+	size_t total_copied = 0;
+	struct bio_vec *bv;
+	int i, error;
 
-		iov_iter_bvec(&to, ITER_DEST, bio->bi_io_vec + 1, bio->bi_vcnt,
-				len);
-		/* copying to pinned pages should always work */
-		WARN_ON_ONCE(copy_to_iter(buf, len, &to) != len);
-	} else {
-		/* No need to mark folios dirty if never copied to them */
-		mark_dirty = false;
-	}
+	error = bio_alloc_bounce_folios(bio, total_len, minsize);
+	if (error)
+		return error;
 
-	if (bio_flagged(bio, BIO_PAGE_PINNED)) {
-		int i;
+	bio_for_each_bvec_all(bv, bio, i) {
+		struct folio *folio = page_folio(bv->bv_page);
+		size_t copied;
 
-		for (i = 0; i < bio->bi_vcnt; i++)
-			bvec_unpin(&bio->bi_io_vec[1 + i], mark_dirty);
+		if (iter->nofault)
+			copied = copy_folio_from_iter_atomic(folio, 0,
+					bv->bv_len, iter);
+		else
+			copied = copy_folio_from_iter(folio, 0, bv->bv_len,
+					iter);
+		total_copied += copied;
+		if (copied < bv->bv_len) {
+			iov_iter_revert(iter, total_copied);
+			bio_free_folios(bio);
+			return -EFAULT;
+		}
 	}
 
-	folio_put(bvec_folio(&bio->bi_io_vec[0]));
-}
-
-/**
- * bio_iov_iter_unbounce - finish a bounce buffer operation
- * @bio:	completed bio
- * @is_error:	%true if an I/O error occurred and data should not be copied
- * @mark_dirty:	If %true, folios will be marked dirty.
- *
- * Helper for direct I/O implementations that need to bounce buffer because
- * we need to checksum the data or perform other operations that require
- * consistency.  Called to complete a bio set up by bio_iov_iter_bounce().
- * Copies data back for reads, and marks the original folios dirty if
- * requested and then frees the bounce buffer.
- */
-void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty)
-{
-	if (op_is_write(bio_op(bio)))
-		bio_free_folios(bio);
-	else
-		bio_iov_iter_unbounce_read(bio, is_error, mark_dirty);
+	return 0;
 }
+EXPORT_SYMBOL_GPL(bio_iov_iter_bounce_write);
 
 static void bio_wait_end_io(struct bio *bio)
 {
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 2b5c29434e4264..82b66147d778f9 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -1189,10 +1189,17 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)
 	struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
 	struct blkcg_gq *blkg;
 
-	if (!seq_css(sf)->parent)
+	if (!seq_css(sf)->parent) {
 		blkcg_fill_root_iostats();
-	else
+	} else {
+		/*
+		 * Descendant blkgs imply ancestor blkgs, so an empty
+		 * ->blkg_list has no stats to report.
+		 */
+		if (hlist_empty(&blkcg->blkg_list))
+			return 0;
 		css_rstat_flush(&blkcg->css);
+	}
 
 	guard(spinlock_irq)(&blkcg->lock);
 	hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node)
@@ -2091,6 +2098,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
 	struct request_queue *q = bio->bi_bdev->bd_queue;
 	struct blkcg *blkcg = css_to_blkcg(css);
 	struct blkcg_gq *blkg;
+	unsigned long flags;
 
 	rcu_read_lock();
 	blkg = blkg_lookup(blkcg, q);
@@ -2105,11 +2113,11 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
 	 * Fast path failed, we're probably issuing IO in this cgroup the first
 	 * time, hold lock to create new blkg.
 	 */
-	spin_lock_irq(&q->queue_lock);
+	spin_lock_irqsave(&q->queue_lock, flags);
 	blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk);
 	if (blkg)
 		blkg = blkg_lookup_tryget(blkg);
-	spin_unlock_irq(&q->queue_lock);
+	spin_unlock_irqrestore(&q->queue_lock, flags);
 
 	return blkg;
 }
diff --git a/block/blk-core.c b/block/blk-core.c
index 196bccf27f58dc..13dc70e8f55d9f 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -901,8 +901,11 @@ void submit_bio_noacct(struct bio *bio)
 	case REQ_OP_ZONE_CLOSE:
 	case REQ_OP_ZONE_RESET:
 	case REQ_OP_ZONE_FINISH:
-		/* Zone management operations require sequential zones. */
-		if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector))
+		/*
+		 * Zone management operations require sequential zones that are
+		 * not offline nor read-only.
+		 */
+		if (!bdev_zone_mgmt_allowed(bdev, bio->bi_iter.bi_sector))
 			goto end_io;
 		break;
 	case REQ_OP_ZONE_RESET_ALL:
diff --git a/block/blk-map.c b/block/blk-map.c
index 9cb9605d1f62da..81cba3af4e9c1d 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -274,7 +274,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,
 	 * No alignment requirements on our part to support arbitrary
 	 * passthrough commands.
 	 */
-	ret = bio_iov_iter_get_pages(bio, iter, 0, 0);
+	ret = bio_iov_iter_get_pages(bio, iter, BIO_MAX_SIZE, 0, 0);
 	if (ret)
 		goto out_put;
 	ret = blk_rq_append_bio(rq, bio);
diff --git a/block/blk-mq.c b/block/blk-mq.c
index a26a11c73ee3e0..025a799f35b924 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -447,16 +447,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 	WRITE_ONCE(rq->deadline, 0);
 	req_ref_set(rq, 1);
 
-	if (rq->rq_flags & RQF_USE_SCHED) {
-		struct elevator_queue *e = data->q->elevator;
-
-		INIT_HLIST_NODE(&rq->hash);
-		RB_CLEAR_NODE(&rq->rb_node);
-
-		if (e->type->ops.prepare_request)
-			e->type->ops.prepare_request(rq);
-	}
-
 	return rq;
 }
 
@@ -498,6 +488,12 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
 	return rq_list_pop(data->cached_rqs);
 }
 
+static bool blk_op_bypass_sched(blk_opf_t opf)
+{
+	return (opf & REQ_OP_MASK) == REQ_OP_FLUSH ||
+	       blk_op_is_passthrough(opf);
+}
+
 static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
 {
 	struct elevator_mq_ops *ops;
@@ -518,12 +514,10 @@ static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
 	 * Flush/passthrough requests are special and go directly to the
 	 * dispatch list, they are not subject to the async_depth limit.
 	 */
-	if ((data->cmd_flags & REQ_OP_MASK) == REQ_OP_FLUSH ||
-	    blk_op_is_passthrough(data->cmd_flags))
+	if (blk_op_bypass_sched(data->cmd_flags))
 		return;
 
 	WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
-	data->rq_flags |= RQF_USE_SCHED;
 
 	/*
 	 * By default, sync requests have no limit, and async requests are
@@ -534,6 +528,29 @@ static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
 		ops->limit_depth(data->cmd_flags, data);
 }
 
+/*
+ * Finish initializing a request once it has been claimed for an operation.
+ * Cached requests are allocated before that operation is known.
+ */
+static void blk_mq_rq_late_init(struct request *rq, u64 alloc_time_ns)
+{
+	struct elevator_queue *e;
+
+	blk_mq_rq_time_init(rq, alloc_time_ns);
+
+	if (!(rq->rq_flags & RQF_SCHED_TAGS) || (rq->rq_flags & RQF_RESV) ||
+	    blk_op_bypass_sched(rq->cmd_flags))
+		return;
+
+	rq->rq_flags |= RQF_USE_SCHED;
+	INIT_HLIST_NODE(&rq->hash);
+	RB_CLEAR_NODE(&rq->rb_node);
+
+	e = rq->q->elevator;
+	if (e->type->ops.prepare_request)
+		e->type->ops.prepare_request(rq);
+}
+
 static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
 {
 	struct request_queue *q = data->q;
@@ -562,7 +579,7 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
 	if (data->nr_tags > 1) {
 		rq = __blk_mq_alloc_requests_batch(data);
 		if (rq) {
-			blk_mq_rq_time_init(rq, alloc_time_ns);
+			blk_mq_rq_late_init(rq, alloc_time_ns);
 			return rq;
 		}
 		data->nr_tags = 1;
@@ -590,7 +607,7 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
 	if (!(data->rq_flags & RQF_SCHED_TAGS))
 		blk_mq_inc_active_requests(data->hctx);
 	rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
-	blk_mq_rq_time_init(rq, alloc_time_ns);
+	blk_mq_rq_late_init(rq, alloc_time_ns);
 	return rq;
 }
 
@@ -637,8 +654,6 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
 		if (plug->nr_ios == 1)
 			return NULL;
 		rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
-		if (!rq)
-			return NULL;
 	} else {
 		rq = rq_list_peek(&plug->cached_rqs);
 		if (!rq || rq->q != q)
@@ -646,15 +661,13 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
 
 		if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
 			return NULL;
-		if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
-			return NULL;
 
 		rq_list_pop(&plug->cached_rqs);
-		blk_mq_rq_time_init(rq, blk_time_get_ns());
+		rq->cmd_flags = opf;
+		INIT_LIST_HEAD(&rq->queuelist);
+		blk_mq_rq_late_init(rq, blk_time_get_ns());
 	}
 
-	rq->cmd_flags = opf;
-	INIT_LIST_HEAD(&rq->queuelist);
 	return rq;
 }
 
@@ -766,7 +779,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
 	if (!(data.rq_flags & RQF_SCHED_TAGS))
 		blk_mq_inc_active_requests(data.hctx);
 	rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
-	blk_mq_rq_time_init(rq, alloc_time_ns);
+	blk_mq_rq_late_init(rq, alloc_time_ns);
 	rq->__data_len = 0;
 	rq->phys_gap_bit = 0;
 	rq->__sector = (sector_t) -1;
@@ -3060,8 +3073,6 @@ static struct request *blk_mq_get_cached_request(struct blk_plug *plug,
 	if (type != rq->mq_hctx->type &&
 	    (type != HCTX_TYPE_READ || rq->mq_hctx->type != HCTX_TYPE_DEFAULT))
 		return NULL;
-	if (op_is_flush(rq->cmd_flags) != op_is_flush(opf))
-		return NULL;
 	rq_list_pop(&plug->cached_rqs);
 	return rq;
 }
@@ -3163,9 +3174,9 @@ void blk_mq_submit_bio(struct bio *bio)
 new_request:
 	if (rq) {
 		rq_qos_throttle(rq->q, bio);
-		blk_mq_rq_time_init(rq, blk_time_get_ns());
 		rq->cmd_flags = bio->bi_opf;
 		INIT_LIST_HEAD(&rq->queuelist);
+		blk_mq_rq_late_init(rq, blk_time_get_ns());
 	} else {
 		rq = blk_mq_get_new_requests(q, plug, bio);
 		if (unlikely(!rq)) {
diff --git a/block/blk-settings.c b/block/blk-settings.c
index 8274631290dbf4..1f5ee2453269f0 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -230,14 +230,20 @@ static unsigned int blk_queue_max_guaranteed_bio(struct queue_limits *lim)
 
 static void blk_atomic_writes_update_limits(struct queue_limits *lim)
 {
+	unsigned int integrity_max = lim->integrity.metadata_size ?
+					max_integrity_io_size(lim) : 0;
 	unsigned int unit_limit = min(lim->max_hw_sectors << SECTOR_SHIFT,
 					blk_queue_max_guaranteed_bio(lim));
 
+	unit_limit = min_not_zero(unit_limit, integrity_max);
 	unit_limit = rounddown_pow_of_two(unit_limit);
 
 	lim->atomic_write_max_sectors =
 		min(lim->atomic_write_hw_max >> SECTOR_SHIFT,
 			lim->max_hw_sectors);
+	lim->atomic_write_max_sectors =
+		min_not_zero(lim->atomic_write_max_sectors,
+				integrity_max >> SECTOR_SHIFT);
 	lim->atomic_write_unit_min =
 		min(lim->atomic_write_hw_unit_min, unit_limit);
 	lim->atomic_write_unit_max =
@@ -505,11 +511,13 @@ int blk_validate_limits(struct queue_limits *lim)
 	if (!(lim->features & BLK_FEAT_WRITE_CACHE))
 		lim->features &= ~BLK_FEAT_FUA;
 
-	blk_validate_atomic_write_limits(lim);
-
 	err = blk_validate_integrity_limits(lim);
 	if (err)
 		return err;
+
+	/* atomics limits depend on integrity limits */
+	blk_validate_atomic_write_limits(lim);
+
 	return blk_validate_zoned_limits(lim);
 }
 EXPORT_SYMBOL_GPL(blk_validate_limits);
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
index a5afb842bf35e0..19268afb8752e6 100644
--- a/block/blk-zoned.c
+++ b/block/blk-zoned.c
@@ -39,6 +39,103 @@ static const char *const zone_cond_name[] = {
 };
 #undef ZONE_COND_NAME
 
+/*
+ * Internal and compact representation of enum blk_zone_cond values for zone
+ * conditions. All these values fit into 4-bits, allowing using the high order
+ * bits as the zone type.
+ */
+enum blk_zstate {
+	BLK_ZSTATE_NOT_WP	= 0x00,
+	BLK_ZSTATE_EMPTY	= 0x01,
+	BLK_ZSTATE_IMP_OPEN	= 0x02,
+	BLK_ZSTATE_EXP_OPEN	= 0x03,
+	BLK_ZSTATE_CLOSED	= 0x04,
+	BLK_ZSTATE_READONLY	= 0x05,
+	BLK_ZSTATE_FULL		= 0x06,
+	BLK_ZSTATE_OFFLINE	= 0x07,
+	BLK_ZSTATE_ACTIVE	= 0x08,
+
+	BLK_ZSTATE_COND_MASK	= 0x0F,
+
+	/* Conventional zone. */
+	BLK_ZFLAG_CONV		= 0x80,
+	BLK_ZSTATE_FLAGS_MASK	= ~BLK_ZSTATE_COND_MASK,
+};
+
+/*
+ * Lookup table and helper to convert enum blk_zstate conditions into enum
+ * blk_zone_condition values.
+ */
+static const u8 blk_zstate2zcond[] = {
+	[BLK_ZSTATE_NOT_WP]	= BLK_ZONE_COND_NOT_WP,
+	[BLK_ZSTATE_EMPTY]	= BLK_ZONE_COND_EMPTY,
+	[BLK_ZSTATE_IMP_OPEN]	= BLK_ZONE_COND_IMP_OPEN,
+	[BLK_ZSTATE_EXP_OPEN]	= BLK_ZONE_COND_EXP_OPEN,
+	[BLK_ZSTATE_CLOSED]	= BLK_ZONE_COND_CLOSED,
+	[BLK_ZSTATE_READONLY]	= BLK_ZONE_COND_READONLY,
+	[BLK_ZSTATE_FULL]	= BLK_ZONE_COND_FULL,
+	[BLK_ZSTATE_OFFLINE]	= BLK_ZONE_COND_OFFLINE,
+	[BLK_ZSTATE_ACTIVE]	= BLK_ZONE_COND_ACTIVE,
+};
+
+static inline enum blk_zone_cond blk_zstate_to_zone_cond(enum blk_zstate zs)
+{
+	u8 idx = zs & BLK_ZSTATE_COND_MASK;
+
+	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_zstate2zcond)))
+		return 0;
+
+	return blk_zstate2zcond[idx];
+}
+
+/*
+ * Lookup table and helper to convert an enum blk_zone_condition into an enum
+ * blk_zstate condition value. To keep the lookup table small, the
+ * BLK_ZONE_COND_ACTIVE condition is not added and handled separately.
+ */
+static const u8 blk_zcond2zstate[] = {
+	[BLK_ZONE_COND_NOT_WP]		= BLK_ZSTATE_NOT_WP,
+	[BLK_ZONE_COND_EMPTY]		= BLK_ZSTATE_EMPTY,
+	[BLK_ZONE_COND_IMP_OPEN]	= BLK_ZSTATE_ACTIVE,
+	[BLK_ZONE_COND_EXP_OPEN]	= BLK_ZSTATE_ACTIVE,
+	[BLK_ZONE_COND_CLOSED]		= BLK_ZSTATE_ACTIVE,
+	[BLK_ZONE_COND_READONLY]	= BLK_ZSTATE_READONLY,
+	[BLK_ZONE_COND_FULL]		= BLK_ZSTATE_FULL,
+	[BLK_ZONE_COND_OFFLINE]		= BLK_ZSTATE_OFFLINE,
+};
+
+static inline enum blk_zstate blk_zone_cond_to_zstate(enum blk_zone_cond cond)
+{
+	if (cond == BLK_ZONE_COND_ACTIVE)
+		return BLK_ZSTATE_ACTIVE;
+
+	if (WARN_ON_ONCE(cond >= ARRAY_SIZE(blk_zcond2zstate)))
+		return 0;
+
+	return blk_zcond2zstate[cond];
+}
+
+/*
+ * Combine an enum blk_zone_condition and zone flags into a zones_state array
+ * entry.
+ */
+static inline void blk_zstate_set(u8 *zones_state, unsigned int nr_zones,
+			unsigned int zno, enum blk_zone_cond cond, u8 flags)
+{
+	if (zones_state && zno < nr_zones)
+		zones_state[zno] = flags | blk_zone_cond_to_zstate(cond);
+}
+
+static inline u8 blk_zstate_flags(enum blk_zstate zs)
+{
+	return zs & BLK_ZSTATE_FLAGS_MASK;
+}
+
+static inline bool blk_zstate_is_conv(enum blk_zstate zs)
+{
+	return blk_zstate_flags(zs) & BLK_ZFLAG_CONV;
+}
+
 /*
  * Per-zone write plug.
  * @node: hlist_node structure for managing the plug using a hash table.
@@ -135,55 +232,64 @@ const char *blk_zone_cond_str(enum blk_zone_cond zone_cond)
 }
 EXPORT_SYMBOL_GPL(blk_zone_cond_str);
 
-static void blk_zone_set_cond(u8 *zones_cond, unsigned int zno,
-			      enum blk_zone_cond cond)
+static void disk_zone_set_cond(struct gendisk *disk, sector_t sector,
+			       enum blk_zone_cond cond)
 {
-	if (!zones_cond)
-		return;
+	unsigned int zno = disk_zone_no(disk, sector);
+	u8 *zones_state;
 
-	switch (cond) {
-	case BLK_ZONE_COND_IMP_OPEN:
-	case BLK_ZONE_COND_EXP_OPEN:
-	case BLK_ZONE_COND_CLOSED:
-		zones_cond[zno] = BLK_ZONE_COND_ACTIVE;
-		return;
-	case BLK_ZONE_COND_NOT_WP:
-	case BLK_ZONE_COND_EMPTY:
-	case BLK_ZONE_COND_FULL:
-	case BLK_ZONE_COND_OFFLINE:
-	case BLK_ZONE_COND_READONLY:
-	default:
-		zones_cond[zno] = cond;
-		return;
-	}
+	rcu_read_lock();
+	zones_state = rcu_dereference(disk->zones_state);
+	if (likely(zones_state && zno < disk->nr_zones))
+		blk_zstate_set(zones_state, disk->nr_zones, zno, cond,
+			       blk_zstate_flags(zones_state[zno]));
+	rcu_read_unlock();
 }
 
-static void disk_zone_set_cond(struct gendisk *disk, sector_t sector,
-			       enum blk_zone_cond cond)
+static inline u8 disk_zone_get_state(struct gendisk *disk, sector_t sector)
 {
-	u8 *zones_cond;
+	unsigned int zno = disk_zone_no(disk, sector);
+	u8 *zones_state, zs;
 
 	rcu_read_lock();
-	zones_cond = rcu_dereference(disk->zones_cond);
-	if (zones_cond) {
-		unsigned int zno = disk_zone_no(disk, sector);
-
-		/*
-		 * The condition of a conventional, readonly and offline zones
-		 * never changes, so do nothing if the target zone is in one of
-		 * these conditions.
-		 */
-		switch (zones_cond[zno]) {
-		case BLK_ZONE_COND_NOT_WP:
-		case BLK_ZONE_COND_READONLY:
-		case BLK_ZONE_COND_OFFLINE:
-			break;
-		default:
-			blk_zone_set_cond(zones_cond, zno, cond);
-			break;
-		}
-	}
+	zones_state = rcu_dereference(disk->zones_state);
+	if (likely(zones_state && zno < disk->nr_zones))
+		zs = zones_state[zno];
+	else
+		zs = BLK_ZFLAG_CONV;
 	rcu_read_unlock();
+
+	return zs;
+}
+
+static enum blk_zone_cond disk_zone_get_cond(struct gendisk *disk,
+					     sector_t sector)
+{
+	u8 zs = disk_zone_get_state(disk, sector);
+
+	return blk_zstate_to_zone_cond(zs);
+}
+
+static inline bool
+disk_zone_cond_is_offline_or_readonly(enum blk_zone_cond cond)
+{
+	return cond == BLK_ZONE_COND_READONLY ||
+		cond == BLK_ZONE_COND_OFFLINE;
+}
+
+static inline bool disk_zone_is_offline_or_readonly(struct gendisk *disk,
+						    sector_t sector)
+{
+	enum blk_zone_cond cond = disk_zone_get_cond(disk, sector);
+
+	return disk_zone_cond_is_offline_or_readonly(cond);
+}
+
+static bool disk_zone_is_seq(struct gendisk *disk, sector_t sector)
+{
+	u8 zs = disk_zone_get_state(disk, sector);
+
+	return !blk_zstate_is_conv(zs);
 }
 
 /**
@@ -195,23 +301,37 @@ static void disk_zone_set_cond(struct gendisk *disk, sector_t sector,
  */
 bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector)
 {
-	struct gendisk *disk = bdev->bd_disk;
-	unsigned int zno = disk_zone_no(disk, sector);
-	bool is_seq = false;
-	u8 *zones_cond;
+	if (!bdev_is_zoned(bdev))
+		return false;
+
+	return disk_zone_is_seq(bdev->bd_disk, sector);
+}
+EXPORT_SYMBOL_GPL(bdev_zone_is_seq);
+
+/**
+ * bdev_zone_mgmt_allowed - check if management operations are allowed on a zone
+ * @bdev:       block device to check
+ * @sector:     sector number
+ *
+ * Check if the zone containing @sector on @bdev can be a target for a zone
+ * management operation, that is, if the zone is a sequential write required
+ * zone that is not offline nor read-only.
+ */
+bool bdev_zone_mgmt_allowed(struct block_device *bdev, sector_t sector)
+{
+	enum blk_zone_cond cond;
+	u8 zs;
 
 	if (!bdev_is_zoned(bdev))
 		return false;
 
-	rcu_read_lock();
-	zones_cond = rcu_dereference(disk->zones_cond);
-	if (zones_cond && zno < disk->nr_zones)
-		is_seq = zones_cond[zno] != BLK_ZONE_COND_NOT_WP;
-	rcu_read_unlock();
+	zs = disk_zone_get_state(bdev->bd_disk, sector);
+	if (blk_zstate_is_conv(zs))
+		return false;
 
-	return is_seq;
+	cond = blk_zstate_to_zone_cond(zs);
+	return !disk_zone_cond_is_offline_or_readonly(cond);
 }
-EXPORT_SYMBOL_GPL(bdev_zone_is_seq);
 
 /*
  * Zone report arguments for block device drivers report_zones operation.
@@ -500,12 +620,17 @@ static bool disk_zone_wplug_is_full(struct gendisk *disk,
 	return zwplug->wp_offset >= disk->last_zone_capacity;
 }
 
+static bool disk_zone_wplug_is_offline_or_readonly(struct blk_zone_wplug *zwplug)
+{
+	return disk_zone_cond_is_offline_or_readonly(zwplug->cond);
+}
+
 static bool disk_insert_zone_wplug(struct gendisk *disk,
 				   struct blk_zone_wplug *zwplug)
 {
 	struct blk_zone_wplug *zwplg;
 	unsigned long flags;
-	u8 *zones_cond;
+	u8 *zones_state;
 	unsigned int idx =
 		hash_32(zwplug->zone_no, disk->zone_wplugs_hash_bits);
 
@@ -524,15 +649,16 @@ static bool disk_insert_zone_wplug(struct gendisk *disk,
 	}
 
 	/*
-	 * Set the zone condition: if we do not yet have a zones_cond array
+	 * Set the zone condition: if we do not yet have a zones_state array
 	 * attached to the disk, then this is a zone write plug insert from the
 	 * first call to blk_revalidate_disk_zones(), in which case the zone is
 	 * necessarilly in the active condition.
 	 */
-	zones_cond = rcu_dereference_check(disk->zones_cond,
+	zones_state = rcu_dereference_check(disk->zones_state,
 				lockdep_is_held(&disk->zone_wplugs_hash_lock));
-	if (zones_cond)
-		zwplug->cond = zones_cond[zwplug->zone_no];
+	if (zones_state)
+		zwplug->cond =
+			blk_zstate_to_zone_cond(zones_state[zwplug->zone_no]);
 	else
 		zwplug->cond = BLK_ZONE_COND_ACTIVE;
 
@@ -574,6 +700,26 @@ static inline struct blk_zone_wplug *disk_get_zone_wplug(struct gendisk *disk,
 	return disk_get_hashed_zone_wplug(disk, sector);
 }
 
+static void disk_for_all_zone_wplugs(struct gendisk *disk,
+				     void (*actor)(struct blk_zone_wplug *,
+						   void *),
+				     void *data)
+{
+	struct blk_zone_wplug *zwplug;
+	unsigned int i;
+
+	if (!disk->zone_wplugs_hash)
+		return;
+
+	rcu_read_lock();
+	for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++) {
+		hlist_for_each_entry_rcu(zwplug, &disk->zone_wplugs_hash[i],
+					 node)
+			actor(zwplug, data);
+	}
+	rcu_read_unlock();
+}
+
 static void disk_free_zone_wplug_rcu(struct rcu_head *rcu_head)
 {
 	struct blk_zone_wplug *zwplug =
@@ -592,9 +738,9 @@ static void disk_free_zone_wplug(struct blk_zone_wplug *zwplug)
 	WARN_ON_ONCE(!bio_list_empty(&zwplug->bio_list));
 
 	spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags);
-	blk_zone_set_cond(rcu_dereference_check(disk->zones_cond,
+	blk_zstate_set(rcu_dereference_check(disk->zones_state,
 				lockdep_is_held(&disk->zone_wplugs_hash_lock)),
-			  zwplug->zone_no, zwplug->cond);
+		       disk->nr_zones, zwplug->zone_no, zwplug->cond, 0);
 	hlist_del_init_rcu(&zwplug->node);
 	atomic_dec(&disk->nr_zone_wplugs);
 	spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags);
@@ -608,6 +754,53 @@ static inline void disk_put_zone_wplug(struct blk_zone_wplug *zwplug)
 		disk_free_zone_wplug(zwplug);
 }
 
+static inline void blk_zone_wplug_bio_io_error(struct blk_zone_wplug *zwplug,
+					       struct bio *bio)
+{
+	struct request_queue *q = zwplug->disk->queue;
+
+	bio_clear_flag(bio, BIO_ZONE_WRITE_PLUGGING);
+	bio_io_error(bio);
+	disk_put_zone_wplug(zwplug);
+	/* Drop the reference taken by disk_zone_wplug_add_bio(). */
+	blk_queue_exit(q);
+}
+
+/*
+ * Abort (fail) all plugged BIOs of a zone write plug.
+ */
+static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug)
+{
+	struct gendisk *disk = zwplug->disk;
+	struct bio *bio;
+
+	lockdep_assert_held(&zwplug->lock);
+
+	if (bio_list_empty(&zwplug->bio_list))
+		return;
+
+	pr_warn_ratelimited("%s: zone %u: Aborting plugged BIOs\n",
+			    zwplug->disk->disk_name, zwplug->zone_no);
+	while ((bio = bio_list_pop(&zwplug->bio_list)))
+		blk_zone_wplug_bio_io_error(zwplug, bio);
+
+	zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED;
+
+	/*
+	 * If we are using the per disk zone write plugs worker thread, remove
+	 * the zone write plug from the work list and drop the reference we
+	 * took when the zone write plug was added to that list.
+	 */
+	if (blk_queue_zoned_qd1_writes(disk->queue)) {
+		spin_lock(&disk->zone_wplugs_list_lock);
+		if (!list_empty(&zwplug->entry)) {
+			list_del_init(&zwplug->entry);
+			disk_put_zone_wplug(zwplug);
+		}
+		spin_unlock(&disk->zone_wplugs_list_lock);
+	}
+}
+
 /*
  * Flag the zone write plug as dead and drop the initial reference we got when
  * the zone write plug was added to the hash table. The zone write plug will be
@@ -625,6 +818,12 @@ static void disk_mark_zone_wplug_dead(struct blk_zone_wplug *zwplug)
 
 static inline bool disk_check_zone_wplug_dead(struct blk_zone_wplug *zwplug)
 {
+	if (disk_zone_wplug_is_offline_or_readonly(zwplug)) {
+		disk_zone_wplug_abort(zwplug);
+		disk_mark_zone_wplug_dead(zwplug);
+		return true;
+	}
+
 	if (!(zwplug->flags & BLK_ZONE_WPLUG_DEAD))
 		return false;
 
@@ -708,53 +907,6 @@ static struct blk_zone_wplug *disk_get_or_alloc_zone_wplug(struct gendisk *disk,
 	return zwplug;
 }
 
-static inline void blk_zone_wplug_bio_io_error(struct blk_zone_wplug *zwplug,
-					       struct bio *bio)
-{
-	struct request_queue *q = zwplug->disk->queue;
-
-	bio_clear_flag(bio, BIO_ZONE_WRITE_PLUGGING);
-	bio_io_error(bio);
-	disk_put_zone_wplug(zwplug);
-	/* Drop the reference taken by disk_zone_wplug_add_bio(). */
-	blk_queue_exit(q);
-}
-
-/*
- * Abort (fail) all plugged BIOs of a zone write plug.
- */
-static void disk_zone_wplug_abort(struct blk_zone_wplug *zwplug)
-{
-	struct gendisk *disk = zwplug->disk;
-	struct bio *bio;
-
-	lockdep_assert_held(&zwplug->lock);
-
-	if (bio_list_empty(&zwplug->bio_list))
-		return;
-
-	pr_warn_ratelimited("%s: zone %u: Aborting plugged BIOs\n",
-			    zwplug->disk->disk_name, zwplug->zone_no);
-	while ((bio = bio_list_pop(&zwplug->bio_list)))
-		blk_zone_wplug_bio_io_error(zwplug, bio);
-
-	zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED;
-
-	/*
-	 * If we are using the per disk zone write plugs worker thread, remove
-	 * the zone write plug from the work list and drop the reference we
-	 * took when the zone write plug was added to that list.
-	 */
-	if (blk_queue_zoned_qd1_writes(disk->queue)) {
-		spin_lock(&disk->zone_wplugs_list_lock);
-		if (!list_empty(&zwplug->entry)) {
-			list_del_init(&zwplug->entry);
-			disk_put_zone_wplug(zwplug);
-		}
-		spin_unlock(&disk->zone_wplugs_list_lock);
-	}
-}
-
 /*
  * Update a zone write plug condition based on the write pointer offset.
  */
@@ -785,8 +937,10 @@ static void disk_zone_wplug_set_wp_offset(struct gendisk *disk,
 
 	/* Update the zone write pointer and abort all plugged BIOs. */
 	zwplug->flags &= ~BLK_ZONE_WPLUG_NEED_WP_UPDATE;
-	zwplug->wp_offset = wp_offset;
-	disk_zone_wplug_update_cond(disk, zwplug);
+	if (!disk_zone_wplug_is_offline_or_readonly(zwplug)) {
+		zwplug->wp_offset = wp_offset;
+		disk_zone_wplug_update_cond(disk, zwplug);
+	}
 
 	disk_zone_wplug_abort(zwplug);
 	if (!zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug))
@@ -816,8 +970,8 @@ static unsigned int blk_zone_wp_offset(struct blk_zone *zone)
 	}
 }
 
-static unsigned int disk_zone_wplug_sync_wp_offset(struct gendisk *disk,
-						   struct blk_zone *zone)
+static unsigned int disk_zone_wplug_sync_state(struct gendisk *disk,
+					       struct blk_zone *zone)
 {
 	struct blk_zone_wplug *zwplug;
 	unsigned int wp_offset = blk_zone_wp_offset(zone);
@@ -827,6 +981,12 @@ static unsigned int disk_zone_wplug_sync_wp_offset(struct gendisk *disk,
 		unsigned long flags;
 
 		spin_lock_irqsave(&zwplug->lock, flags);
+		if (disk_zone_cond_is_offline_or_readonly(zone->cond)) {
+			zwplug->flags &= ~BLK_ZONE_WPLUG_NEED_WP_UPDATE;
+			zwplug->cond = zone->cond;
+			zwplug->wp_offset = UINT_MAX;
+			disk_mark_zone_wplug_dead(zwplug);
+		}
 		if (zwplug->flags & BLK_ZONE_WPLUG_NEED_WP_UPDATE)
 			disk_zone_wplug_set_wp_offset(disk, zwplug, wp_offset);
 		spin_unlock_irqrestore(&zwplug->lock, flags);
@@ -871,7 +1031,7 @@ int disk_report_zone(struct gendisk *disk, struct blk_zone *zone,
 	}
 
 	if (disk->zone_wplugs_hash)
-		disk_zone_wplug_sync_wp_offset(disk, zone);
+		disk_zone_wplug_sync_state(disk, zone);
 
 	if (args && args->cb)
 		return args->cb(zone, idx, args->data);
@@ -938,29 +1098,36 @@ int blkdev_get_zone_info(struct block_device *bdev, sector_t sector,
 {
 	struct gendisk *disk = bdev->bd_disk;
 	sector_t zone_sectors = bdev_zone_sectors(bdev);
+	unsigned int zno = disk_zone_no(disk, sector);
 	struct blk_zone_wplug *zwplug;
 	unsigned long flags;
-	u8 *zones_cond;
+	u8 *zones_state, zs;
 
 	if (!bdev_is_zoned(bdev))
 		return -EOPNOTSUPP;
 
-	if (sector >= get_capacity(disk))
+	if (sector >= get_capacity(disk) || zno >= disk->nr_zones)
 		return -EINVAL;
 
 	memset(zone, 0, sizeof(*zone));
 	sector = bdev_zone_start(bdev, sector);
 
 	if (!blkdev_has_cached_report_zones(bdev))
-		return blkdev_report_zone_fallback(bdev, sector, zone);
+		goto fallback;
 
 	rcu_read_lock();
-	zones_cond = rcu_dereference(disk->zones_cond);
-	if (!disk->zone_wplugs_hash || !zones_cond) {
+	zones_state = rcu_dereference(disk->zones_state);
+	if (!disk->zone_wplugs_hash || !zones_state) {
 		rcu_read_unlock();
-		return blkdev_report_zone_fallback(bdev, sector, zone);
+		goto fallback;
 	}
-	zone->cond = zones_cond[disk_zone_no(disk, sector)];
+
+	zs = zones_state[zno];
+	zone->cond = blk_zstate_to_zone_cond(zs);
+	if (blk_zstate_is_conv(zs))
+		zone->type = BLK_ZONE_TYPE_CONVENTIONAL;
+	else
+		zone->type = BLK_ZONE_TYPE_SEQWRITE_REQ;
 	rcu_read_unlock();
 
 	zone->start = sector;
@@ -970,8 +1137,7 @@ int blkdev_get_zone_info(struct block_device *bdev, sector_t sector,
 	 * If this is a conventional zone, we do not have a zone write plug and
 	 * can report the zone immediately.
 	 */
-	if (zone->cond == BLK_ZONE_COND_NOT_WP) {
-		zone->type = BLK_ZONE_TYPE_CONVENTIONAL;
+	if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) {
 		zone->capacity = zone_sectors;
 		zone->wp = ULLONG_MAX;
 		return 0;
@@ -982,14 +1148,12 @@ int blkdev_get_zone_info(struct block_device *bdev, sector_t sector,
 	 * offline, only set the zone write pointer to an invalid value and
 	 * report the zone.
 	 */
-	zone->type = BLK_ZONE_TYPE_SEQWRITE_REQ;
 	if (disk_zone_is_last(disk, zone))
 		zone->capacity = disk->last_zone_capacity;
 	else
 		zone->capacity = disk->zone_capacity;
 
-	if (zone->cond == BLK_ZONE_COND_READONLY ||
-	    zone->cond == BLK_ZONE_COND_OFFLINE) {
+	if (disk_zone_cond_is_offline_or_readonly(zone->cond)) {
 		zone->wp = ULLONG_MAX;
 		return 0;
 	}
@@ -1022,6 +1186,9 @@ int blkdev_get_zone_info(struct block_device *bdev, sector_t sector,
 	disk_put_zone_wplug(zwplug);
 
 	return 0;
+
+fallback:
+	return blkdev_report_zone_fallback(bdev, sector, zone);
 }
 EXPORT_SYMBOL_GPL(blkdev_get_zone_info);
 
@@ -1108,34 +1275,32 @@ static void blk_zone_reset_bio_endio(struct bio *bio)
 	}
 }
 
+static void disk_zone_wplug_reset_wp(struct blk_zone_wplug *zwplug, void *data)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&zwplug->lock, flags);
+	disk_zone_wplug_set_wp_offset(zwplug->disk, zwplug, 0);
+	spin_unlock_irqrestore(&zwplug->lock, flags);
+}
+
 static void blk_zone_reset_all_bio_endio(struct bio *bio)
 {
 	struct gendisk *disk = bio->bi_bdev->bd_disk;
-	sector_t capacity = get_capacity(disk);
-	struct blk_zone_wplug *zwplug;
-	unsigned long flags;
 	sector_t sector;
-	unsigned int i;
 
-	if (atomic_read(&disk->nr_zone_wplugs)) {
-		/* Update the condition of all zone write plugs. */
-		rcu_read_lock();
-		for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++) {
-			hlist_for_each_entry_rcu(zwplug,
-						 &disk->zone_wplugs_hash[i],
-						 node) {
-				spin_lock_irqsave(&zwplug->lock, flags);
-				disk_zone_wplug_set_wp_offset(disk, zwplug, 0);
-				spin_unlock_irqrestore(&zwplug->lock, flags);
-			}
-		}
-		rcu_read_unlock();
-	}
+	/* Update the condition of all zone write plugs. */
+	if (atomic_read(&disk->nr_zone_wplugs))
+		disk_for_all_zone_wplugs(disk, disk_zone_wplug_reset_wp, NULL);
 
 	/* Update the cached zone conditions. */
-	for (sector = 0; sector < capacity;
-	     sector += bdev_zone_sectors(bio->bi_bdev))
+	for (sector = 0; sector < get_capacity(disk);
+	     sector += bdev_zone_sectors(bio->bi_bdev)) {
+		if (!disk_zone_is_seq(disk, sector) ||
+		    disk_zone_is_offline_or_readonly(disk, sector))
+			continue;
 		disk_zone_set_cond(disk, sector, BLK_ZONE_COND_EMPTY);
+	}
 	clear_bit(GD_ZONE_APPEND_USED, &disk->state);
 }
 
@@ -1381,11 +1546,12 @@ static bool blk_zone_wplug_prepare_bio(struct blk_zone_wplug *zwplug,
 		return false;
 
 	/*
-	 * Check that the user is not attempting to write to a full zone.
-	 * We know such BIO will fail, and that would potentially overflow our
-	 * write pointer offset beyond the end of the zone.
+	 * Check that the user is not attempting to write to a full, read-only
+	 * or offline zone. We know such BIOs will fail, so there is no point
+	 * in issuing them.
 	 */
-	if (disk_zone_wplug_is_full(disk, zwplug))
+	if (disk_zone_wplug_is_full(disk, zwplug) ||
+	    disk_zone_wplug_is_offline_or_readonly(zwplug))
 		return false;
 
 	if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
@@ -1442,7 +1608,7 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs)
 	}
 
 	/* Conventional zones do not need write plugging. */
-	if (!bdev_zone_is_seq(bio->bi_bdev, sector)) {
+	if (!disk_zone_is_seq(disk, sector)) {
 		/* Zone append to conventional zones is not allowed. */
 		if (bio_op(bio) == REQ_OP_ZONE_APPEND) {
 			bio_io_error(bio);
@@ -1854,12 +2020,29 @@ static int disk_zone_wplugs_worker(void *data)
 
 void disk_init_zone_resources(struct gendisk *disk)
 {
+	mutex_init(&disk->zone_revalidate_mutex);
+	atomic_set(&disk->nr_zone_wplugs, 0);
 	spin_lock_init(&disk->zone_wplugs_hash_lock);
 	spin_lock_init(&disk->zone_wplugs_list_lock);
 	INIT_LIST_HEAD(&disk->zone_wplugs_list);
 	init_completion(&disk->zone_wplugs_worker_bio_done);
 }
 
+static unsigned int disk_get_nr_zones(struct gendisk *disk, sector_t capacity)
+{
+	struct queue_limits *lim = &disk->queue->limits;
+	unsigned long long nr_zones;
+
+	if (!capacity || !lim->chunk_sectors)
+		return 0;
+
+	nr_zones = DIV_ROUND_UP_ULL(capacity, lim->chunk_sectors);
+	if (nr_zones > UINT_MAX)
+		return 0;
+
+	return nr_zones;
+}
+
 /*
  * For the size of a disk zone write plug hash table, use the size of the
  * zone write plug mempool, which is the maximum of the disk open zones and
@@ -1869,13 +2052,24 @@ void disk_init_zone_resources(struct gendisk *disk)
 #define BLK_ZONE_WPLUG_MAX_HASH_BITS		9
 #define BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE	128
 
-static int disk_alloc_zone_resources(struct gendisk *disk,
-				     unsigned int pool_size)
+static int disk_alloc_zone_resources(struct gendisk *disk, sector_t capacity)
 {
-	unsigned int i;
+	struct queue_limits *lim = &disk->queue->limits;
+	unsigned int nr_zones, pool_size, i;
 	int ret = -ENOMEM;
 
-	atomic_set(&disk->nr_zone_wplugs, 0);
+	nr_zones = disk_get_nr_zones(disk, capacity);
+	if (!nr_zones)
+		return -ENODEV;
+
+	/*
+	 * If the device has no limit on the maximum number of open and active
+	 * zones, use BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE.
+	 */
+	pool_size = max(lim->max_open_zones, lim->max_active_zones);
+	if (!pool_size)
+		pool_size = min(BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE, nr_zones);
+
 	disk->zone_wplugs_hash_bits =
 		min(ilog2(pool_size) + 1, BLK_ZONE_WPLUG_MAX_HASH_BITS);
 
@@ -1893,21 +2087,6 @@ static int disk_alloc_zone_resources(struct gendisk *disk,
 	if (!disk->zone_wplugs_pool)
 		goto free_hash;
 
-	/*
-	 * We may already have a zone write plug workqueue as this function may
-	 * be called after disk_free_zone_resources(), which does not destroy
-	 * the workqueue (the zone write plugs workqueue is destroyed at
-	 * disk_release() time).
-	 */
-	if (!disk->zone_wplugs_wq) {
-		disk->zone_wplugs_wq =
-			alloc_workqueue("%s_zwplugs",
-					WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU,
-					pool_size, disk->disk_name);
-		if (!disk->zone_wplugs_wq)
-			goto destroy_pool;
-	}
-
 	disk->zone_wplugs_worker =
 		kthread_create(disk_zone_wplugs_worker, disk,
 			       "%s_zwplugs_worker", disk->disk_name);
@@ -1918,8 +2097,18 @@ static int disk_alloc_zone_resources(struct gendisk *disk,
 	}
 	wake_up_process(disk->zone_wplugs_worker);
 
+	disk->zone_wplugs_wq =
+		alloc_workqueue("%s_zwplugs",
+				WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU,
+				pool_size, disk->disk_name);
+	if (!disk->zone_wplugs_wq)
+		goto stop_worker;
+
 	return 0;
 
+stop_worker:
+	kthread_stop(disk->zone_wplugs_worker);
+	disk->zone_wplugs_worker = NULL;
 destroy_pool:
 	mempool_destroy(disk->zone_wplugs_pool);
 	disk->zone_wplugs_pool = NULL;
@@ -1963,19 +2152,19 @@ static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk)
 	disk->zone_wplugs_pool = NULL;
 }
 
-static void disk_set_zones_cond_array(struct gendisk *disk, u8 *zones_cond)
+static void disk_set_zones_state_array(struct gendisk *disk, u8 *zones_state)
 {
 	unsigned long flags;
 
 	spin_lock_irqsave(&disk->zone_wplugs_hash_lock, flags);
-	zones_cond = rcu_replace_pointer(disk->zones_cond, zones_cond,
+	zones_state = rcu_replace_pointer(disk->zones_state, zones_state,
 				lockdep_is_held(&disk->zone_wplugs_hash_lock));
 	spin_unlock_irqrestore(&disk->zone_wplugs_hash_lock, flags);
 
-	kfree_rcu_mightsleep(zones_cond);
+	kfree_rcu_mightsleep(zones_state);
 }
 
-static void disk_free_zone_resources(struct gendisk *disk)
+void disk_release_zone_resources(struct gendisk *disk)
 {
 	if (disk->zone_wplugs_worker) {
 		kthread_stop(disk->zone_wplugs_worker);
@@ -1983,30 +2172,24 @@ static void disk_free_zone_resources(struct gendisk *disk)
 	}
 	WARN_ON_ONCE(!list_empty(&disk->zone_wplugs_list));
 
-	if (disk->zone_wplugs_wq)
-		drain_workqueue(disk->zone_wplugs_wq);
+	if (disk->zone_wplugs_wq) {
+		destroy_workqueue(disk->zone_wplugs_wq);
+		disk->zone_wplugs_wq = NULL;
+	}
 
 	disk_destroy_zone_wplugs_hash_table(disk);
 
-	disk_set_zones_cond_array(disk, NULL);
+	disk_set_zones_state_array(disk, NULL);
 	disk->zone_capacity = 0;
 	disk->last_zone_capacity = 0;
 	disk->nr_zones = 0;
-}
-
-void disk_release_zone_resources(struct gendisk *disk)
-{
-	if (disk->zone_wplugs_wq) {
-		destroy_workqueue(disk->zone_wplugs_wq);
-		disk->zone_wplugs_wq = NULL;
-	}
-
-	disk_free_zone_resources(disk);
+	mutex_destroy(&disk->zone_revalidate_mutex);
 }
 
 struct blk_revalidate_zone_args {
 	struct gendisk	*disk;
-	u8		*zones_cond;
+	sector_t	capacity;
+	u8		*zones_state;
 	unsigned int	nr_zones;
 	unsigned int	nr_conv_zones;
 	unsigned int	zone_capacity;
@@ -2014,73 +2197,76 @@ struct blk_revalidate_zone_args {
 	sector_t	sector;
 };
 
-static int disk_revalidate_zone_resources(struct gendisk *disk,
-				struct blk_revalidate_zone_args *args)
+static int disk_init_revalidate_args(struct gendisk *disk,
+				     struct blk_revalidate_zone_args *args)
 {
-	struct queue_limits *lim = &disk->queue->limits;
-	unsigned int pool_size;
-	int ret = 0;
-
 	args->disk = disk;
-	args->nr_zones =
-		DIV_ROUND_UP_ULL(get_capacity(disk), lim->chunk_sectors);
+	args->nr_zones = disk_get_nr_zones(disk, args->capacity);
+	if (!args->nr_zones)
+		return -ENODEV;
 
 	/* Cached zone conditions: 1 byte per zone */
-	args->zones_cond = kzalloc(args->nr_zones, GFP_NOIO);
-	if (!args->zones_cond)
+	args->zones_state = kzalloc(args->nr_zones, GFP_NOIO);
+	if (!args->zones_state)
 		return -ENOMEM;
 
-	if (!disk_need_zone_resources(disk))
-		return 0;
-
-	/*
-	 * If the device has no limit on the maximum number of open and active
-	 * zones, use BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE.
-	 */
-	pool_size = max(lim->max_open_zones, lim->max_active_zones);
-	if (!pool_size)
-		pool_size =
-			min(BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE, args->nr_zones);
-
-	if (!disk->zone_wplugs_hash) {
-		ret = disk_alloc_zone_resources(disk, pool_size);
-		if (ret)
-			kfree(args->zones_cond);
-	}
-
-	return ret;
+	return 0;
 }
 
 /*
- * Update the disk zone resources information and device queue limits.
- * The disk queue is frozen when this is executed.
+ * Revalidate and update the disk zone resources information and device queue
+ * limits.
  */
-static int disk_update_zone_resources(struct gendisk *disk,
-				      struct blk_revalidate_zone_args *args)
+static int disk_revalidate_zone_resources(struct gendisk *disk,
+					  struct blk_revalidate_zone_args *args)
 {
 	struct request_queue *q = disk->queue;
 	unsigned int nr_seq_zones;
 	unsigned int pool_size, memflags;
 	struct queue_limits lim;
+	sector_t capacity;
 	int ret = 0;
 
 	lim = queue_limits_start_update(q);
 
 	memflags = blk_mq_freeze_queue(q);
 
-	disk->nr_zones = args->nr_zones;
-	if (args->nr_conv_zones >= disk->nr_zones) {
-		queue_limits_cancel_update(q);
+	/*
+	 * Using the re-evaluated disk capacity, make sure that the entire disk
+	 * has been checked.
+	 */
+	capacity = get_capacity(disk);
+	if (args->capacity != capacity) {
+		pr_warn("%s: Capacity has changed (%llu -> %llu)\n",
+			disk->disk_name, args->capacity, capacity);
+		/* Force a retry if we have a valid (non-zero) capacity. */
+		if (capacity)
+			ret = -EAGAIN;
+		else
+			ret = -ENODEV;
+		goto unfreeze;
+	}
+
+	/* Make sure that all zones have been checked. */
+	if (args->sector != capacity) {
+		pr_warn("%s: last zone and capacity mismatch (%llu != %llu)\n",
+			disk->disk_name, args->sector, capacity);
+		ret = -ENODEV;
+		goto unfreeze;
+	}
+
+	if (args->nr_conv_zones >= args->nr_zones) {
 		pr_warn("%s: Invalid number of conventional zones %u / %u\n",
-			disk->disk_name, args->nr_conv_zones, disk->nr_zones);
+			disk->disk_name, args->nr_conv_zones, args->nr_zones);
 		ret = -ENODEV;
 		goto unfreeze;
 	}
 
+	disk->nr_zones = args->nr_zones;
 	disk->zone_capacity = args->zone_capacity;
 	disk->last_zone_capacity = args->last_zone_capacity;
-	disk_set_zones_cond_array(disk, args->zones_cond);
-	args->zones_cond = NULL;
+	disk_set_zones_state_array(disk, args->zones_state);
+	args->zones_state = NULL;
 
 	/*
 	 * Some devices can advertise zone resource limits that are larger than
@@ -2095,7 +2281,7 @@ static int disk_update_zone_resources(struct gendisk *disk,
 		lim.max_active_zones = 0;
 
 	if (!disk->zone_wplugs_pool)
-		goto commit;
+		goto unfreeze;
 
 	/*
 	 * If the device has no limit on the maximum number of open and active
@@ -2117,51 +2303,65 @@ static int disk_update_zone_resources(struct gendisk *disk,
 			lim.max_open_zones = 0;
 	}
 
-commit:
-	ret = queue_limits_commit_update(q, &lim);
-
 unfreeze:
+	if (ret)
+		queue_limits_cancel_update(q);
+	else
+		ret = queue_limits_commit_update(q, &lim);
+
 	blk_mq_unfreeze_queue(q, memflags);
 
 	return ret;
 }
 
-static int blk_revalidate_zone_cond(struct blk_zone *zone, unsigned int idx,
+static void disk_drop_zone_wplug(struct blk_zone_wplug *zwplug, void *data)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&zwplug->lock, flags);
+	disk_zone_wplug_abort(zwplug);
+	disk_mark_zone_wplug_dead(zwplug);
+	spin_unlock_irqrestore(&zwplug->lock, flags);
+}
+
+static int disk_revalidate_capacity(struct gendisk *disk,
 				    struct blk_revalidate_zone_args *args)
 {
-	enum blk_zone_cond cond = zone->cond;
+	struct queue_limits *lim = &disk->queue->limits;
+        sector_t zone_sectors = lim->chunk_sectors;
+	unsigned int nr_zones;
+	int ret = -ENODEV;
 
-	/* Check that the zone condition is consistent with the zone type. */
-	switch (cond) {
-	case BLK_ZONE_COND_NOT_WP:
-		if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL)
-			goto invalid_condition;
-		break;
-	case BLK_ZONE_COND_IMP_OPEN:
-	case BLK_ZONE_COND_EXP_OPEN:
-	case BLK_ZONE_COND_CLOSED:
-	case BLK_ZONE_COND_EMPTY:
-	case BLK_ZONE_COND_FULL:
-	case BLK_ZONE_COND_OFFLINE:
-	case BLK_ZONE_COND_READONLY:
-		if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
-			goto invalid_condition;
-		break;
-	default:
-		pr_warn("%s: Invalid zone condition 0x%X\n",
-			args->disk->disk_name, cond);
-		return -ENODEV;
+	/* Checks that the device driver indicated a valid zone size. */
+	if (!zone_sectors || !is_power_of_2(zone_sectors)) {
+		pr_warn("%s: Invalid non power of two zone size (%llu)\n",
+			disk->disk_name, zone_sectors);
+		goto drop_all_zwplugs;
 	}
 
-	blk_zone_set_cond(args->zones_cond, idx, cond);
+	args->capacity = get_capacity(disk);
+	nr_zones = disk_get_nr_zones(disk, args->capacity);
+	if (!args->capacity || !nr_zones)
+		goto drop_all_zwplugs;
+
+	/*
+	 * Check if the capacity has changed. If it did, assume that the device
+	 * was reformatted and that all sequential zones are now empty. So drop
+	 * all zone write plug.
+	 */
+	if (disk->nr_zones && disk->nr_zones != nr_zones) {
+		pr_warn("%s: Number of zones changed (%u -> %u)\n",
+			disk->disk_name, disk->nr_zones, nr_zones);
+		ret = 0;
+		goto drop_all_zwplugs;
+	}
 
 	return 0;
 
-invalid_condition:
-	pr_warn("%s: Invalid zone condition 0x%x for type 0x%x\n",
-		args->disk->disk_name, cond, zone->type);
+drop_all_zwplugs:
+	disk_for_all_zone_wplugs(disk, disk_drop_zone_wplug, NULL);
 
-	return -ENODEV;
+	return ret;
 }
 
 static int blk_revalidate_conv_zone(struct blk_zone *zone, unsigned int idx,
@@ -2169,12 +2369,27 @@ static int blk_revalidate_conv_zone(struct blk_zone *zone, unsigned int idx,
 {
 	struct gendisk *disk = args->disk;
 
+	/* Check the zone condition. */
+	switch (zone->cond) {
+	case BLK_ZONE_COND_NOT_WP:
+	case BLK_ZONE_COND_OFFLINE:
+	case BLK_ZONE_COND_READONLY:
+		break;
+	default:
+		pr_warn("%s: Invalid conv. zone condition 0x%X at sector %llu\n",
+			disk->disk_name, zone->cond, zone->start);
+		return -ENODEV;
+	}
+
 	if (zone->capacity != zone->len) {
 		pr_warn("%s: Invalid conventional zone capacity\n",
 			disk->disk_name);
 		return -ENODEV;
 	}
 
+	blk_zstate_set(args->zones_state, args->nr_zones, idx,
+		       zone->cond, BLK_ZFLAG_CONV);
+
 	if (disk_zone_is_last(disk, zone))
 		args->last_zone_capacity = zone->capacity;
 
@@ -2190,6 +2405,24 @@ static int blk_revalidate_seq_zone(struct blk_zone *zone, unsigned int idx,
 	struct blk_zone_wplug *zwplug;
 	unsigned int wp_offset;
 
+	/* Check the zone condition. */
+	switch (zone->cond) {
+	case BLK_ZONE_COND_IMP_OPEN:
+	case BLK_ZONE_COND_EXP_OPEN:
+	case BLK_ZONE_COND_CLOSED:
+	case BLK_ZONE_COND_EMPTY:
+	case BLK_ZONE_COND_FULL:
+	case BLK_ZONE_COND_OFFLINE:
+	case BLK_ZONE_COND_READONLY:
+		break;
+	default:
+		pr_warn("%s: Invalid seq. zone condition 0x%X at sector %llu\n",
+			disk->disk_name, zone->cond, zone->start);
+		return -ENODEV;
+	}
+
+	blk_zstate_set(args->zones_state, args->nr_zones, idx, zone->cond, 0);
+
 	/*
 	 * Remember the capacity of the first sequential zone and check
 	 * if it is constant for all zones, ignoring the last zone as it can be
@@ -2214,7 +2447,7 @@ static int blk_revalidate_seq_zone(struct blk_zone *zone, unsigned int idx,
 	if (!disk->zone_wplugs_hash)
 		return 0;
 
-	wp_offset = disk_zone_wplug_sync_wp_offset(disk, zone);
+	wp_offset = disk_zone_wplug_sync_state(disk, zone);
 	if (!wp_offset || wp_offset >= zone->capacity)
 		return 0;
 
@@ -2244,7 +2477,7 @@ static int blk_revalidate_zone_cb(struct blk_zone *zone, unsigned int idx,
 		return -ENODEV;
 	}
 
-	if (zone->start >= get_capacity(disk) || !zone->len) {
+	if (zone->start >= args->capacity || !zone->len) {
 		pr_warn("%s: Invalid zone start %llu, length %llu\n",
 			disk->disk_name, zone->start, zone->len);
 		return -ENODEV;
@@ -2272,11 +2505,6 @@ static int blk_revalidate_zone_cb(struct blk_zone *zone, unsigned int idx,
 		return -ENODEV;
 	}
 
-	/* Check zone condition */
-	ret = blk_revalidate_zone_cond(zone, idx, args);
-	if (ret)
-		return ret;
-
 	/* Check zone type */
 	switch (zone->type) {
 	case BLK_ZONE_TYPE_CONVENTIONAL:
@@ -2313,42 +2541,48 @@ static int blk_revalidate_zone_cb(struct blk_zone *zone, unsigned int idx,
  */
 int blk_revalidate_disk_zones(struct gendisk *disk)
 {
-	struct request_queue *q = disk->queue;
-	sector_t zone_sectors = q->limits.chunk_sectors;
-	sector_t capacity = get_capacity(disk);
 	struct blk_revalidate_zone_args args = { };
-	unsigned int memflags, noio_flag;
 	struct blk_report_zones_args rep_args = {
 		.cb = blk_revalidate_zone_cb,
 		.data = &args,
 	};
-	int ret = -ENOMEM;
+	unsigned int noio_flag;
+	int retries = 2;
+	int ret;
 
-	if (WARN_ON_ONCE(!blk_queue_is_zoned(q)))
+	if (WARN_ON_ONCE(!blk_queue_is_zoned(disk->queue)))
 		return -EIO;
 
-	if (!capacity)
-		return -ENODEV;
-
 	/*
-	 * Checks that the device driver indicated a valid zone size and that
-	 * the max zone append limit is set.
+	 * Serialize calls to this function so that we can safely look at and
+	 * eventually change the disk zone information.
 	 */
-	if (!zone_sectors || !is_power_of_2(zone_sectors)) {
-		pr_warn("%s: Invalid non power of two zone size (%llu)\n",
-			disk->disk_name, zone_sectors);
-		return -ENODEV;
-	}
+	mutex_lock(&disk->zone_revalidate_mutex);
+
+again:
+	ret = disk_revalidate_capacity(disk, &args);
+	if (ret)
+		goto unlock;
 
 	/*
-	 * Ensure that all memory allocations in this context are done as if
-	 * GFP_NOIO was specified.
+	 * Allocate zone resources if they are needed and we have not done
+	 * so yet, and initialize the revalidation arguments passed to report
+	 * zones. Ensure that all memory allocations in this context are done as
+	 * if GFP_NOIO was specified.
 	 */
 	noio_flag = memalloc_noio_save();
-	ret = disk_revalidate_zone_resources(disk, &args);
+	if (disk_need_zone_resources(disk) && !disk->zone_wplugs_hash) {
+		ret = disk_alloc_zone_resources(disk, args.capacity);
+		if (ret) {
+			memalloc_noio_restore(noio_flag);
+			goto unlock;
+		}
+	}
+
+	ret = disk_init_revalidate_args(disk, &args);
 	if (ret) {
 		memalloc_noio_restore(noio_flag);
-		return ret;
+		goto unlock;
 	}
 
 	ret = disk->fops->report_zones(disk, 0, UINT_MAX, &rep_args);
@@ -2358,33 +2592,32 @@ int blk_revalidate_disk_zones(struct gendisk *disk)
 	}
 	memalloc_noio_restore(noio_flag);
 
-	if (ret <= 0)
-		goto free_resources;
-
-	/*
-	 * If zones where reported, make sure that the entire disk capacity
-	 * has been checked.
-	 */
-	if (args.sector != capacity) {
-		pr_warn("%s: Missing zones from sector %llu\n",
-			disk->disk_name, args.sector);
-		ret = -ENODEV;
-		goto free_resources;
-	}
+	if (ret < 0)
+		goto free_args;
 
-	ret = disk_update_zone_resources(disk, &args);
+	ret = disk_revalidate_zone_resources(disk, &args);
 	if (ret)
-		goto free_resources;
+		goto free_args;
+
+	mutex_unlock(&disk->zone_revalidate_mutex);
 
 	return 0;
 
-free_resources:
-	pr_warn("%s: failed to revalidate zones\n", disk->disk_name);
+free_args:
+	kfree(args.zones_state);
 
-	kfree(args.zones_cond);
-	memflags = blk_mq_freeze_queue(q);
-	disk_free_zone_resources(disk);
-	blk_mq_unfreeze_queue(q, memflags);
+	if (ret == -EAGAIN) {
+		if (retries) {
+			memset(&args, 0, sizeof(args));
+			retries--;
+			goto again;
+		}
+		ret = -ENODEV;
+	}
+
+	pr_warn("%s: failed to revalidate zones\n", disk->disk_name);
+unlock:
+	mutex_unlock(&disk->zone_revalidate_mutex);
 
 	return ret;
 }
@@ -2435,8 +2668,9 @@ EXPORT_SYMBOL_GPL(blk_zone_issue_zeroout);
 
 #ifdef CONFIG_BLK_DEBUG_FS
 static void queue_zone_wplug_show(struct blk_zone_wplug *zwplug,
-				  struct seq_file *m)
+				  void *data)
 {
+	struct seq_file *m = data;
 	unsigned int zwp_wp_offset, zwp_flags;
 	unsigned int zwp_zone_no, zwp_ref;
 	unsigned int zwp_bio_list_size;
@@ -2461,19 +2695,8 @@ static void queue_zone_wplug_show(struct blk_zone_wplug *zwplug,
 int queue_zone_wplugs_show(void *data, struct seq_file *m)
 {
 	struct request_queue *q = data;
-	struct gendisk *disk = q->disk;
-	struct blk_zone_wplug *zwplug;
-	unsigned int i;
 
-	if (!disk->zone_wplugs_hash)
-		return 0;
-
-	rcu_read_lock();
-	for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++)
-		hlist_for_each_entry_rcu(zwplug, &disk->zone_wplugs_hash[i],
-					 node)
-			queue_zone_wplug_show(zwplug, m);
-	rcu_read_unlock();
+	disk_for_all_zone_wplugs(q->disk, queue_zone_wplug_show, m);
 
 	return 0;
 }
diff --git a/block/blk.h b/block/blk.h
index 50abfd9328861e..2cc03aa54c5327 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -577,6 +577,7 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, unsigned int cmd,
 		unsigned long arg);
 int blkdev_zone_mgmt_ioctl(struct block_device *bdev, blk_mode_t mode,
 		unsigned int cmd, unsigned long arg);
+bool bdev_zone_mgmt_allowed(struct block_device *bdev, sector_t sector);
 #else /* CONFIG_BLK_DEV_ZONED */
 static inline void disk_init_zone_resources(struct gendisk *disk)
 {
@@ -619,6 +620,11 @@ static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
 {
 	return -ENOTTY;
 }
+static inline bool bdev_zone_mgmt_allowed(struct block_device *bdev,
+					  sector_t sector)
+{
+	return false;
+}
 #endif /* CONFIG_BLK_DEV_ZONED */
 
 struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
diff --git a/block/fops.c b/block/fops.c
index 2ce7c6c4714e42..d5f2b8068875f4 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -6,6 +6,7 @@
  */
 #include <linux/init.h>
 #include <linux/mm.h>
+#include <linux/writeback.h>
 #include <linux/blkdev.h>
 #include <linux/blk-integrity.h>
 #include <linux/buffer_head.h>
@@ -26,12 +27,26 @@ static inline struct inode *bdev_file_inode(struct file *file)
 	return file->f_mapping->host;
 }
 
-static blk_opf_t dio_bio_write_op(struct kiocb *iocb)
+static bool blkdev_dio_fua(struct kiocb *iocb, struct block_device *bdev)
+{
+	if (!iocb_is_dsync(iocb))
+		return false;
+	/*
+	 * Async writes cannot fall back to generic_write_sync(), so they must
+	 * use FUA (emulated if needed); sync writes only need it when the
+	 * device supports FUA natively.
+	 */
+	if (!is_sync_kiocb(iocb))
+		return true;
+	return bdev_fua(bdev);
+}
+
+static blk_opf_t dio_bio_write_op(struct kiocb *iocb, struct block_device *bdev)
 {
 	blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
 
 	/* avoid the need for a I/O completion work item */
-	if (iocb_is_dsync(iocb))
+	if (blkdev_dio_fua(iocb, bdev))
 		opf |= REQ_FUA;
 	return opf;
 }
@@ -46,7 +61,8 @@ static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb,
 static inline int blkdev_iov_iter_get_pages(struct bio *bio,
 		struct iov_iter *iter, struct block_device *bdev)
 {
-	return bio_iov_iter_get_pages(bio, iter, bdev_dma_alignment(bdev),
+	return bio_iov_iter_get_pages(bio, iter, BIO_MAX_SIZE,
+			bdev_dma_alignment(bdev),
 			bdev_logical_block_size(bdev) - 1);
 }
 
@@ -75,7 +91,8 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,
 		if (user_backed_iter(iter))
 			should_dirty = true;
 	} else {
-		bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));
+		bio_init(&bio, bdev, vecs, nr_pages,
+			 dio_bio_write_op(iocb, bdev));
 	}
 	bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT;
 	bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;
@@ -179,7 +196,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
 	struct blkdev_dio *dio;
 	struct bio *bio;
 	bool is_read = (iov_iter_rw(iter) == READ), is_sync;
-	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
+	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev);
 	loff_t pos = iocb->ki_pos;
 	int ret = 0;
 
@@ -325,7 +342,7 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
 					unsigned int nr_pages)
 {
 	bool is_read = iov_iter_rw(iter) == READ;
-	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
+	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb, bdev);
 	struct blkdev_dio *dio;
 	struct bio *bio;
 	loff_t pos = iocb->ki_pos;
@@ -402,6 +419,11 @@ static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
 	if (blkdev_dio_invalid(bdev, iocb, iter))
 		return -EINVAL;
 
+	/* HIPRI needs private as bio; HAS_METADATA keeps it as uio_meta */
+	if ((iocb->ki_flags & IOCB_HIPRI) &&
+	    (iocb->ki_flags & IOCB_HAS_METADATA))
+		return -EOPNOTSUPP;
+
 	if (iov_iter_rw(iter) == WRITE) {
 		u16 max_write_streams = bdev_max_write_streams(bdev);
 
@@ -728,6 +750,7 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	bool atomic = iocb->ki_flags & IOCB_ATOMIC;
 	loff_t size = bdev_nr_bytes(bdev);
 	size_t shorted = 0;
+	bool need_sync = false;
 	ssize_t ret;
 
 	if (bdev_read_only(bdev))
@@ -765,9 +788,18 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 
 	if (iocb->ki_flags & IOCB_DIRECT) {
 		ret = blkdev_direct_write(iocb, from);
-		if (ret >= 0 && iov_iter_count(from))
+		if (ret >= 0 && iov_iter_count(from)) {
 			ret = direct_write_fallback(iocb, from, ret,
 					blkdev_buffered_write(iocb, from));
+			need_sync = true;
+		} else if (ret > 0 && !blkdev_dio_fua(iocb, bdev)) {
+			/*
+			 * The device does not support FUA, so REQ_FUA was not
+			 * set and the O_DSYNC direct write still needs a flush
+			 * to be made durable.
+			 */
+			need_sync = true;
+		}
 	} else {
 		/*
 		 * Take i_rwsem and invalidate_lock to avoid racing with
@@ -777,9 +809,10 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		inode_lock_shared(bd_inode);
 		ret = blkdev_buffered_write(iocb, from);
 		inode_unlock_shared(bd_inode);
+		need_sync = true;
 	}
 
-	if (ret > 0)
+	if (ret > 0 && need_sync)
 		ret = generic_write_sync(iocb, ret);
 	iov_iter_reexpand(from, iov_iter_count(from) + shorted);
 	return ret;
diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index 5f643c0ce2a863..e5db1ee097c35d 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -685,7 +685,7 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx,
 	blk_mq_free_requests(&free);
 }
 
-/* Callback from inside blk_mq_rq_ctx_init(). */
+/* Callback from inside blk_mq_rq_late_init(). */
 static void dd_prepare_request(struct request *rq)
 {
 	rq->elv.priv[0] = NULL;
diff --git a/block/partitions/efi.h b/block/partitions/efi.h
index 84b9f36b9e4797..1f56f93b28048f 100644
--- a/block/partitions/efi.h
+++ b/block/partitions/efi.h
@@ -75,18 +75,12 @@ typedef struct _gpt_header {
 	 */
 } __packed gpt_header;
 
-typedef struct _gpt_entry_attributes {
-	u64 required_to_function:1;
-	u64 reserved:47;
-        u64 type_guid_specific:16;
-} __packed gpt_entry_attributes;
-
 typedef struct _gpt_entry {
 	efi_guid_t partition_type_guid;
 	efi_guid_t unique_partition_guid;
 	__le64 starting_lba;
 	__le64 ending_lba;
-	gpt_entry_attributes attributes;
+	__le64 attributes;
 	__le16 partition_name[72/sizeof(__le16)];
 } __packed gpt_entry;
 
diff --git a/crypto/Makefile b/crypto/Makefile
index 8386d55a9755e7..33bb5ad595e8f5 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -22,9 +22,6 @@ crypto_skcipher-y += lskcipher.o
 crypto_skcipher-y += skcipher.o
 
 obj-$(CONFIG_CRYPTO_SKCIPHER2) += crypto_skcipher.o
-ifeq ($(CONFIG_BPF_SYSCALL),y)
-obj-$(CONFIG_CRYPTO_SKCIPHER2) += bpf_crypto_skcipher.o
-endif
 
 obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o
 obj-$(CONFIG_CRYPTO_ECHAINIV) += echainiv.o
diff --git a/crypto/aes.c b/crypto/aes.c
index 94791f481e9842..51eee78396ead5 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -637,7 +637,18 @@ static struct skcipher_alg skcipher_algs[] = {
 		.decrypt = crypto_aes_cbc_decrypt,
 	},
 #endif
-#if IS_ENABLED(CONFIG_CRYPTO_CTS)
+	/*
+	 * Don't register library-based "cts(cbc(aes))" on architectures where
+	 * it might block a "better" implementation from being instantiated via
+	 * the "cts" template.  These exclusions are temporary and will go away
+	 * as the arch-optimized AES code is migrated into the library.
+	 */
+#if IS_ENABLED(CONFIG_CRYPTO_CTS) && \
+	!(IS_ENABLED(CONFIG_ARM) || \
+	  IS_ENABLED(CONFIG_ARM64) || \
+	  IS_ENABLED(CONFIG_PPC) || \
+	  IS_ENABLED(CONFIG_S390) || \
+	  IS_ENABLED(CONFIG_SPARC))
 	{
 		.base.cra_name = "cts(cbc(aes))",
 		.base.cra_driver_name = "cts-cbc-aes-lib",
@@ -687,7 +698,13 @@ static struct skcipher_alg skcipher_algs[] = {
 		.decrypt = crypto_aes_xctr_crypt,
 	},
 #endif
-#if IS_ENABLED(CONFIG_CRYPTO_XTS)
+	/*
+	 * Don't register library-based "xts(aes)" on architectures where it
+	 * might block a "better" implementation from being instantiated via the
+	 * "xts" template.  This exclusion is temporary and will go away when
+	 * the library AES-XTS is optimized for SPARC.
+	 */
+#if IS_ENABLED(CONFIG_CRYPTO_XTS) && !IS_ENABLED(CONFIG_SPARC)
 	{
 		.base.cra_name = "xts(aes)",
 		.base.cra_driver_name = "xts-aes-lib",
@@ -980,7 +997,20 @@ static __maybe_unused int crypto_aes_ccm_decrypt(struct aead_request *req)
 }
 
 static struct aead_alg aead_algs[] = {
-#if IS_ENABLED(CONFIG_CRYPTO_GCM)
+	/*
+	 * Don't register library-based "gcm(aes)" and "rfc4106(gcm(aes))" on
+	 * architectures where they might block a "better" implementation from
+	 * being instantiated via the "gcm" and "rfc4106" templates.  These
+	 * exclusions are temporary and will go away as the arch-optimized AES
+	 * code is migrated into the library.
+	 */
+#if IS_ENABLED(CONFIG_CRYPTO_GCM) && \
+	!(IS_ENABLED(CONFIG_ARM) || \
+	  IS_ENABLED(CONFIG_ARM64) || \
+	  IS_ENABLED(CONFIG_PPC) || \
+	  IS_ENABLED(CONFIG_RISCV) || \
+	  IS_ENABLED(CONFIG_S390) || \
+	  IS_ENABLED(CONFIG_SPARC))
 	{
 		.base.cra_name = "gcm(aes)",
 		.base.cra_driver_name = "gcm-aes-lib",
@@ -1012,7 +1042,20 @@ static struct aead_alg aead_algs[] = {
 		.chunksize = AES_BLOCK_SIZE,
 	},
 #endif /* CONFIG_CRYPTO_GCM */
-#if IS_ENABLED(CONFIG_CRYPTO_CCM)
+	/*
+	 * Don't register library-based "ccm(aes)" on architectures where it
+	 * might block a "better" implementation from being instantiated via the
+	 * "ccm" template.  These exclusions are temporary and will go away as
+	 * the arch-optimized AES code is migrated into the library.
+	 */
+#if IS_ENABLED(CONFIG_CRYPTO_CCM) && \
+	!(IS_ENABLED(CONFIG_ARM) || \
+	  IS_ENABLED(CONFIG_ARM64) || \
+	  IS_ENABLED(CONFIG_PPC) || \
+	  IS_ENABLED(CONFIG_RISCV) || \
+	  IS_ENABLED(CONFIG_S390) || \
+	  IS_ENABLED(CONFIG_SPARC) || \
+	  IS_ENABLED(CONFIG_X86))
 	{
 		.base.cra_name = "ccm(aes)",
 		.base.cra_driver_name = "ccm-aes-lib",
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 37de377719aec7..a6839e2a20f381 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -424,7 +424,6 @@ static void crypto_free_alg(struct crypto_alg *alg)
 	unsigned int algsize = alg->cra_type->algsize;
 	u8 *p = (u8 *)alg - algsize;
 
-	crypto_destroy_alg(alg);
 	kfree(p);
 }
 
@@ -494,9 +493,31 @@ void crypto_unregister_alg(struct crypto_alg *alg)
 	LIST_HEAD(list);
 
 	down_write(&crypto_alg_sem);
+	if (alg->cra_flags & CRYPTO_ALG_DUP_FIRST) {
+		struct crypto_alg *q, *r = NULL;
+
+		list_for_each_entry(q, &crypto_alg_list, cra_list) {
+			if (crypto_is_larval(q))
+				continue;
+			if (strcmp(q->cra_driver_name, alg->cra_driver_name))
+				continue;
+			r = q;
+			break;
+		}
+
+		if (!r) {
+			up_write(&crypto_alg_sem);
+			ret = -ENOENT;
+			goto not_found;
+		}
+
+		alg = r;
+	}
+
 	ret = crypto_remove_alg(alg, &list);
 	up_write(&crypto_alg_sem);
 
+not_found:
 	if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
 		return;
 
diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig
index 6a2f664046efe6..c73979b2e98e98 100644
--- a/crypto/asymmetric_keys/Kconfig
+++ b/crypto/asymmetric_keys/Kconfig
@@ -114,4 +114,16 @@ config FIPS_SIGNATURE_SELFTEST_ECDSA
 	depends on CRYPTO_SHA256=y || CRYPTO_SHA256=FIPS_SIGNATURE_SELFTEST
 	depends on CRYPTO_ECDSA=y || CRYPTO_ECDSA=FIPS_SIGNATURE_SELFTEST
 
+config VERIFY_PEFILE_KUNIT_TEST
+	bool "KUnit tests for signed PE file parsing" if !KUNIT_ALL_TESTS
+	depends on KUNIT=y && SIGNED_PE_FILE_VERIFICATION
+	default KUNIT_ALL_TESTS
+	help
+	  Enable KUnit tests for the PE binary parser used by signed PE file
+	  signature verification.  The tests build small malformed PE images in
+	  memory and check that pefile_parse_binary() rejects them instead of
+	  reading out of bounds.
+
+	  If unsure, say N.
+
 endif # ASYMMETRIC_KEY_TYPE
diff --git a/crypto/asymmetric_keys/Makefile b/crypto/asymmetric_keys/Makefile
index bc65d3b98dcbfa..35172685c533cf 100644
--- a/crypto/asymmetric_keys/Makefile
+++ b/crypto/asymmetric_keys/Makefile
@@ -79,3 +79,5 @@ verify_signed_pefile-y := \
 
 $(obj)/mscode_parser.o: $(obj)/mscode.asn1.h $(obj)/mscode.asn1.h
 $(obj)/mscode.asn1.o: $(obj)/mscode.asn1.c $(obj)/mscode.asn1.h
+
+obj-$(CONFIG_VERIFY_PEFILE_KUNIT_TEST) += verify_pefile_test.o
diff --git a/crypto/asymmetric_keys/pkcs7_parser.c b/crypto/asymmetric_keys/pkcs7_parser.c
index 6e3ffdac83ace7..7f00e45a33cf69 100644
--- a/crypto/asymmetric_keys/pkcs7_parser.c
+++ b/crypto/asymmetric_keys/pkcs7_parser.c
@@ -111,6 +111,12 @@ static int pkcs7_check_authattrs(struct pkcs7_message *msg)
 		if (!!sinfo->authattrs != want)
 			goto inconsistent;
 
+#ifdef CONFIG_PKCS7_WAIVE_AUTHATTRS_REJECTION_FOR_MLDSA
+		if (sinfo->authattrs &&
+		    strncmp(sinfo->sig->pkey_algo, "mldsa", 5) != 0)
+			msg->authattrs_rej_waivable = false;
+#endif
+
 		if (!sinfo->authattrs &&
 		    sinfo->sig->algo_takes_data)
 			sinfo->sig->hash_algo = "none";
diff --git a/crypto/asymmetric_keys/restrict.c b/crypto/asymmetric_keys/restrict.c
index 86292965f4930c..0e4783ff7d22c6 100644
--- a/crypto/asymmetric_keys/restrict.c
+++ b/crypto/asymmetric_keys/restrict.c
@@ -243,10 +243,14 @@ static int key_or_keyring_common(struct key *dest_keyring,
 			if (IS_ERR(key))
 				key = NULL;
 		} else if (trusted->type == &key_type_asymmetric) {
+			const struct asymmetric_key_ids *kids;
 			const struct asymmetric_key_id **signer_ids;
 
-			signer_ids = (const struct asymmetric_key_id **)
-				asymmetric_key_ids(trusted)->id;
+			kids = asymmetric_key_ids(trusted);
+			if (!kids)
+				return -ENOKEY;
+
+			signer_ids = (const struct asymmetric_key_id **)kids->id;
 
 			/*
 			 * The auth_ids come from the candidate key (the
diff --git a/crypto/asymmetric_keys/verify_pefile.c b/crypto/asymmetric_keys/verify_pefile.c
index cec99db14129af..1efd5c0d0e9ce4 100644
--- a/crypto/asymmetric_keys/verify_pefile.c
+++ b/crypto/asymmetric_keys/verify_pefile.c
@@ -87,6 +87,11 @@ static int pefile_parse_binary(const void *pebuf, unsigned int pelen,
 	if (ctx->n_data_dirents > (ctx->header_size - cursor) / sizeof(*dde))
 		return -ELIBBAD;
 
+	/* the certificate table entry must be present in the data directory */
+	if (ctx->n_data_dirents <=
+	    offsetof(struct data_directory, certs) / sizeof(*dde))
+		return -ELIBBAD;
+
 	ddir = pebuf + cursor;
 	cursor += sizeof(*dde) * ctx->n_data_dirents;
 
diff --git a/crypto/asymmetric_keys/verify_pefile_test.c b/crypto/asymmetric_keys/verify_pefile_test.c
new file mode 100644
index 00000000000000..8b42eb41edd554
--- /dev/null
+++ b/crypto/asymmetric_keys/verify_pefile_test.c
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit tests for the signed PE binary parser in verify_pefile.c.
+ *
+ * The cases build small malformed PE images in memory and call
+ * verify_pefile_signature(), which parses the image before any signature
+ * check, so the parser is exercised without a keyring or a real signature.
+ */
+#include <kunit/test.h>
+#include <linux/slab.h>
+#include <linux/pe.h>
+#include <linux/verification.h>
+
+/*
+ * Build a minimal PE32 image whose optional header declares @data_dirs data
+ * directory entries.  header_size is sized to hold them and the image is a few
+ * bytes larger.  With @data_dirs < 5 the certificate table entry (fixed index
+ * 4) lies outside the declared directory.
+ */
+static void *build_pe32(u16 data_dirs, unsigned int *out_len)
+{
+	size_t cursor = sizeof(struct mz_hdr) + sizeof(struct pe_hdr) +
+			sizeof(struct pe32_opt_hdr);
+	size_t header_size = cursor + ((size_t)data_dirs + 1) *
+				      sizeof(struct data_dirent);
+	size_t pelen = header_size + 8;
+	struct mz_hdr *mz;
+	struct pe_hdr *pe;
+	struct pe32_opt_hdr *opt;
+	void *buf = kzalloc(pelen, GFP_KERNEL);
+
+	if (!buf)
+		return NULL;
+
+	mz = buf;
+	mz->magic = IMAGE_DOS_SIGNATURE;
+	mz->peaddr = sizeof(struct mz_hdr);
+
+	pe = buf + sizeof(struct mz_hdr);
+	pe->magic = IMAGE_NT_SIGNATURE;
+
+	opt = buf + sizeof(struct mz_hdr) + sizeof(struct pe_hdr);
+	opt->magic = IMAGE_NT_OPTIONAL_HDR32_MAGIC;
+	opt->header_size = header_size;
+	opt->data_dirs = data_dirs;
+
+	*out_len = pelen;
+	return buf;
+}
+
+/*
+ * data_dirs = 0: the certificate table entry (index 4) is absent, so the
+ * parser must reject the image instead of reading ddir->certs past the end.
+ */
+static void pefile_missing_certs_dirent(struct kunit *test)
+{
+	unsigned int len;
+	void *buf = build_pe32(0, &len);
+	int ret;
+
+	KUNIT_ASSERT_NOT_NULL(test, buf);
+	ret = verify_pefile_signature(buf, len, NULL,
+				      VERIFYING_KEXEC_PE_SIGNATURE);
+	KUNIT_EXPECT_EQ(test, ret, -ELIBBAD);
+	kfree(buf);
+}
+
+/*
+ * data_dirs = 5: the certificate table entry is present (and zero, i.e.
+ * unsigned), so the parser gets past the directory bound and rejects the image
+ * as unsigned (-ENODATA) rather than -ELIBBAD.
+ */
+static void pefile_present_certs_dirent(struct kunit *test)
+{
+	unsigned int len;
+	void *buf = build_pe32(5, &len);
+	int ret;
+
+	KUNIT_ASSERT_NOT_NULL(test, buf);
+	ret = verify_pefile_signature(buf, len, NULL,
+				      VERIFYING_KEXEC_PE_SIGNATURE);
+	KUNIT_EXPECT_EQ(test, ret, -ENODATA);
+	kfree(buf);
+}
+
+static struct kunit_case verify_pefile_cases[] = {
+	KUNIT_CASE(pefile_missing_certs_dirent),
+	KUNIT_CASE(pefile_present_certs_dirent),
+	{}
+};
+
+static struct kunit_suite verify_pefile_suite = {
+	.name = "verify_pefile",
+	.test_cases = verify_pefile_cases,
+};
+
+kunit_test_suite(verify_pefile_suite);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("KUnit tests for the signed PE binary parser");
diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c
index 25cf8ac7f25792..5c9165a83f914a 100644
--- a/crypto/asymmetric_keys/x509_public_key.c
+++ b/crypto/asymmetric_keys/x509_public_key.c
@@ -53,9 +53,11 @@ int x509_get_sig_params(struct x509_certificate *cert)
 
 	if (sig->algo_takes_data) {
 		/* The signature algorithm does whatever passes for hashing. */
-		sig->m = (u8 *)cert->tbs;
+		sig->m = kmemdup(cert->tbs, cert->tbs_size, GFP_KERNEL);
+		if (!sig->m)
+			return -ENOMEM;
 		sig->m_size = cert->tbs_size;
-		sig->m_free = false;
+		sig->m_free = true;
 		goto out;
 	}
 
diff --git a/crypto/bpf_crypto_skcipher.c b/crypto/bpf_crypto_skcipher.c
deleted file mode 100644
index a88798d3e8c872..00000000000000
--- a/crypto/bpf_crypto_skcipher.c
+++ /dev/null
@@ -1,83 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2024 Meta, Inc */
-#include <linux/types.h>
-#include <linux/module.h>
-#include <linux/bpf_crypto.h>
-#include <crypto/skcipher.h>
-
-static void *bpf_crypto_lskcipher_alloc_tfm(const char *algo)
-{
-	return crypto_alloc_lskcipher(algo, 0, 0);
-}
-
-static void bpf_crypto_lskcipher_free_tfm(void *tfm)
-{
-	crypto_free_lskcipher(tfm);
-}
-
-static int bpf_crypto_lskcipher_has_algo(const char *algo)
-{
-	return crypto_has_skcipher(algo, CRYPTO_ALG_TYPE_LSKCIPHER, CRYPTO_ALG_TYPE_MASK);
-}
-
-static int bpf_crypto_lskcipher_setkey(void *tfm, const u8 *key, unsigned int keylen)
-{
-	return crypto_lskcipher_setkey(tfm, key, keylen);
-}
-
-static u32 bpf_crypto_lskcipher_get_flags(void *tfm)
-{
-	return crypto_lskcipher_get_flags(tfm);
-}
-
-static unsigned int bpf_crypto_lskcipher_ivsize(void *tfm)
-{
-	return crypto_lskcipher_ivsize(tfm);
-}
-
-static unsigned int bpf_crypto_lskcipher_statesize(void *tfm)
-{
-	return crypto_lskcipher_statesize(tfm);
-}
-
-static int bpf_crypto_lskcipher_encrypt(void *tfm, const u8 *src, u8 *dst,
-					unsigned int len, u8 *siv)
-{
-	return crypto_lskcipher_encrypt(tfm, src, dst, len, siv);
-}
-
-static int bpf_crypto_lskcipher_decrypt(void *tfm, const u8 *src, u8 *dst,
-					unsigned int len, u8 *siv)
-{
-	return crypto_lskcipher_decrypt(tfm, src, dst, len, siv);
-}
-
-static const struct bpf_crypto_type bpf_crypto_lskcipher_type = {
-	.alloc_tfm	= bpf_crypto_lskcipher_alloc_tfm,
-	.free_tfm	= bpf_crypto_lskcipher_free_tfm,
-	.has_algo	= bpf_crypto_lskcipher_has_algo,
-	.setkey		= bpf_crypto_lskcipher_setkey,
-	.encrypt	= bpf_crypto_lskcipher_encrypt,
-	.decrypt	= bpf_crypto_lskcipher_decrypt,
-	.ivsize		= bpf_crypto_lskcipher_ivsize,
-	.statesize	= bpf_crypto_lskcipher_statesize,
-	.get_flags	= bpf_crypto_lskcipher_get_flags,
-	.owner		= THIS_MODULE,
-	.name		= "skcipher",
-};
-
-static int __init bpf_crypto_skcipher_init(void)
-{
-	return bpf_crypto_register_type(&bpf_crypto_lskcipher_type);
-}
-
-static void __exit bpf_crypto_skcipher_exit(void)
-{
-	int err = bpf_crypto_unregister_type(&bpf_crypto_lskcipher_type);
-	WARN_ON_ONCE(err);
-}
-
-module_init(bpf_crypto_skcipher_init);
-module_exit(bpf_crypto_skcipher_exit);
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Symmetric key cipher support for BPF");
diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c
index a8b07594005db8..6c30436a113547 100644
--- a/crypto/lskcipher.c
+++ b/crypto/lskcipher.c
@@ -75,6 +75,7 @@ static int crypto_lskcipher_crypt_unaligned(
 	unsigned ivsize = crypto_lskcipher_ivsize(tfm);
 	unsigned bs = crypto_lskcipher_blocksize(tfm);
 	unsigned cs = crypto_lskcipher_chunksize(tfm);
+	u32 flags = 0;
 	int err;
 	u8 *tiv;
 	u8 *p;
@@ -98,13 +99,16 @@ static int crypto_lskcipher_crypt_unaligned(
 
 		if (chunk > cs)
 			chunk &= ~(cs - 1);
+		if (chunk == len)
+			flags |= CRYPTO_LSKCIPHER_FLAG_FINAL;
 
 		memcpy(p, src, chunk);
-		err = crypt(tfm, p, p, chunk, tiv, CRYPTO_LSKCIPHER_FLAG_FINAL);
+		err = crypt(tfm, p, p, chunk, tiv, flags);
 		if (err)
 			goto out;
 
 		memcpy(dst, p, chunk);
+		flags |= CRYPTO_LSKCIPHER_FLAG_CONT;
 		src += chunk;
 		dst += chunk;
 		len -= chunk;
diff --git a/crypto/rsassa-pkcs1.c b/crypto/rsassa-pkcs1.c
index 94fa5e9600e79d..39137476ce818b 100644
--- a/crypto/rsassa-pkcs1.c
+++ b/crypto/rsassa-pkcs1.c
@@ -169,7 +169,7 @@ static int rsassa_pkcs1_sign(struct crypto_sig *tfm,
 	u8 *in_buf;
 	int err;
 
-	if (!ctx->key_size)
+	if (ctx->key_size < 11)
 		return -EINVAL;
 
 	if (dlen < ctx->key_size)
@@ -231,7 +231,7 @@ static int rsassa_pkcs1_verify(struct crypto_sig *tfm,
 	int err;
 
 	/* RFC 8017 sec 8.2.2 step 1 - length checking */
-	if (!ctx->key_size ||
+	if (ctx->key_size < 11 ||
 	    slen != ctx->key_size ||
 	    rsassa_pkcs1_invalid_hash_len(dlen, hash_prefix))
 		return -EINVAL;
diff --git a/crypto/zstd.c b/crypto/zstd.c
index 556f5d2bdd5fbd..76bdaca3273305 100644
--- a/crypto/zstd.c
+++ b/crypto/zstd.c
@@ -96,6 +96,7 @@ static int zstd_compress_one(struct acomp_req *req, struct zstd_ctx *ctx,
 
 static int zstd_compress(struct acomp_req *req)
 {
+	bool stream_initialized = false;
 	struct crypto_acomp_stream *s;
 	unsigned int pos, scur, dcur;
 	unsigned int total_out = 0;
@@ -115,12 +116,6 @@ static int zstd_compress(struct acomp_req *req)
 	if (ret)
 		goto out;
 
-	ctx->cctx = zstd_init_cstream(&ctx->params, 0, ctx->wksp, ctx->wksp_size);
-	if (!ctx->cctx) {
-		ret = -EINVAL;
-		goto out;
-	}
-
 	do {
 		dcur = acomp_walk_next_dst(&walk);
 		if (!dcur) {
@@ -142,6 +137,19 @@ static int zstd_compress(struct acomp_req *req)
 				goto out;
 			}
 
+			if (!stream_initialized) {
+				ctx->cctx = zstd_init_cstream(&ctx->params, 0,
+							      ctx->wksp, ctx->wksp_size);
+				if (!ctx->cctx) {
+					/* Release in the reverse of the map order. */
+					acomp_walk_done_src(&walk, 0);
+					acomp_walk_done_dst(&walk, 0);
+					ret = -EINVAL;
+					goto out;
+				}
+				stream_initialized = true;
+			}
+
 			if (scur) {
 				inbuf.pos = 0;
 				inbuf.src = walk.src.virt.addr;
@@ -207,6 +215,7 @@ static int zstd_decompress_one(struct acomp_req *req, struct zstd_ctx *ctx,
 
 static int zstd_decompress(struct acomp_req *req)
 {
+	bool stream_initialized = false;
 	struct crypto_acomp_stream *s;
 	unsigned int total_out = 0;
 	unsigned int scur, dcur;
@@ -224,12 +233,6 @@ static int zstd_decompress(struct acomp_req *req)
 	if (ret)
 		goto out;
 
-	ctx->dctx = zstd_init_dstream(ZSTD_MAX_SIZE, ctx->wksp, ctx->wksp_size);
-	if (!ctx->dctx) {
-		ret = -EINVAL;
-		goto out;
-	}
-
 	do {
 		scur = acomp_walk_next_src(&walk);
 		if (scur) {
@@ -255,6 +258,19 @@ static int zstd_decompress(struct acomp_req *req)
 				goto out;
 			}
 
+			if (!stream_initialized) {
+				ctx->dctx = zstd_init_dstream(ZSTD_MAX_SIZE, ctx->wksp,
+							      ctx->wksp_size);
+				if (!ctx->dctx) {
+					/* Release in the reverse of the map order. */
+					acomp_walk_done_dst(&walk, 0);
+					acomp_walk_done_src(&walk, 0);
+					ret = -EINVAL;
+					goto out;
+				}
+				stream_initialized = true;
+			}
+
 			outbuf.pos = 0;
 			outbuf.dst = (u8 *)walk.dst.virt.addr;
 			outbuf.size = dcur;
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index baf9a8b90a4d68..d927c8c9d557a8 100644
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -7,6 +7,7 @@
 #include <drm/drm_device.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 #include <drm/drm_syncobj.h>
 #include <linux/hmm.h>
@@ -99,9 +100,9 @@ static void aie2_job_put(struct amdxdna_sched_job *job)
 static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx,
 			    struct drm_sched_job *bad_job)
 {
-	drm_sched_stop(&hwctx->priv->sched, bad_job);
+	drm_sched_stop(hwctx->priv->sched, bad_job);
 	aie2_destroy_context(xdna->dev_handle, hwctx);
-	drm_sched_start(&hwctx->priv->sched, 0);
+	drm_sched_start(hwctx->priv->sched, 0);
 }
 
 static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx)
@@ -659,10 +660,19 @@ static void aie2_ctx_syncobj_destroy(struct amdxdna_hwctx *hwctx)
 	drm_syncobj_put(hwctx->priv->syncobj);
 }
 
-int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
+void aie2_hwctx_sched_fini(struct amdxdna_dev_hdl *ndev)
 {
-	struct amdxdna_client *client = hwctx->client;
-	struct amdxdna_dev *xdna = client->xdna;
+	int i;
+
+	for (i = 0; i < ndev->priv->hwctx_limit; i++)
+		drm_sched_fini(&ndev->hwctx_sched[i]);
+
+	ida_destroy(&ndev->hwctx_sched_ida);
+}
+
+int aie2_hwctx_sched_init(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	const struct drm_sched_init_args args = {
 		.ops = &sched_ops,
 		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
@@ -673,7 +683,55 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 		.name = "amdxdna_js",
 		.dev = xdna->ddev.dev,
 	};
-	struct drm_gpu_scheduler *sched;
+	int i, ret;
+
+	ndev->hwctx_sched = drmm_kcalloc(&xdna->ddev, ndev->priv->hwctx_limit,
+					 sizeof(*ndev->hwctx_sched), GFP_KERNEL);
+	if (!ndev->hwctx_sched)
+		return -ENOMEM;
+
+	for (i = 0; i < ndev->priv->hwctx_limit; i++) {
+		ret = drm_sched_init(&ndev->hwctx_sched[i], &args);
+		if (ret) {
+			XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret);
+			goto fini_sched;
+		}
+	}
+
+	ida_init(&ndev->hwctx_sched_ida);
+
+	return 0;
+
+fini_sched:
+	while (--i >= 0)
+		drm_sched_fini(&ndev->hwctx_sched[i]);
+
+	return ret;
+}
+
+static struct drm_gpu_scheduler *aie2_hwctx_sched_alloc(struct amdxdna_dev_hdl *ndev)
+{
+	int ret;
+
+	ret = ida_alloc_range(&ndev->hwctx_sched_ida, 0,
+			      ndev->priv->hwctx_limit - 1, GFP_KERNEL);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	return &ndev->hwctx_sched[ret];
+}
+
+static void aie2_hwctx_sched_free(struct amdxdna_dev_hdl *ndev,
+				  struct drm_gpu_scheduler *sched)
+{
+	ida_free(&ndev->hwctx_sched_ida, sched - ndev->hwctx_sched);
+}
+
+int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
+{
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
 	struct amdxdna_hwctx_priv *priv;
 	struct amdxdna_gem_obj *heap;
 	int i, ret;
@@ -683,13 +741,26 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 		return -ENOMEM;
 	hwctx->priv = priv;
 
+	priv->sched = aie2_hwctx_sched_alloc(ndev);
+	if (IS_ERR(priv->sched)) {
+		ret = PTR_ERR(priv->sched);
+		goto free_priv;
+	}
+
+	ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL,
+				    &priv->sched, 1, NULL);
+	if (ret) {
+		XDNA_ERR(xdna, "Failed to initial sched entity. ret %d", ret);
+		goto free_sched;
+	}
+
 	mutex_lock(&client->mm_lock);
 	heap = xa_load(&client->dev_heap_xa, 0);
 	if (!heap) {
 		XDNA_ERR(xdna, "The client dev heap object not exist");
 		mutex_unlock(&client->mm_lock);
 		ret = -ENOENT;
-		goto free_priv;
+		goto free_entity;
 	}
 	drm_gem_object_get(to_gobj(heap));
 	mutex_unlock(&client->mm_lock);
@@ -722,30 +793,16 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 		priv->cmd_buf[i] = abo;
 	}
 
-	sched = &priv->sched;
 	mutex_init(&priv->io_lock);
 
 	fs_reclaim_acquire(GFP_KERNEL);
 	might_lock(&priv->io_lock);
 	fs_reclaim_release(GFP_KERNEL);
 
-	ret = drm_sched_init(sched, &args);
-	if (ret) {
-		XDNA_ERR(xdna, "Failed to init DRM scheduler. ret %d", ret);
-		goto free_cmd_bufs;
-	}
-
-	ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL,
-				    &sched, 1, NULL);
-	if (ret) {
-		XDNA_ERR(xdna, "Failed to initial sched entiry. ret %d", ret);
-		goto free_sched;
-	}
-
 	ret = aie2_hwctx_col_list(hwctx);
 	if (ret) {
 		XDNA_ERR(xdna, "Create col list failed, ret %d", ret);
-		goto free_entity;
+		goto free_cmd_bufs;
 	}
 
 	ret = amdxdna_pm_resume_get_locked(xdna);
@@ -791,19 +848,20 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 	amdxdna_pm_suspend_put(xdna);
 free_col_list:
 	kfree(hwctx->col_list);
-free_entity:
-	drm_sched_entity_destroy(&priv->entity);
-free_sched:
-	drm_sched_fini(&priv->sched);
 free_cmd_bufs:
 	for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) {
 		if (!priv->cmd_buf[i])
 			continue;
+		amdxdna_gem_heap_free(client, priv->cmd_buf[i]);
 		drm_gem_object_put(to_gobj(priv->cmd_buf[i]));
 	}
 	amdxdna_gem_unpin(heap);
 put_heap:
 	drm_gem_object_put(to_gobj(heap));
+free_entity:
+	drm_sched_entity_destroy(&priv->entity);
+free_sched:
+	aie2_hwctx_sched_free(ndev, priv->sched);
 free_priv:
 	kfree(priv);
 	return ret;
@@ -811,18 +869,20 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 
 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)
 {
+	struct amdxdna_dev_hdl *ndev;
 	struct amdxdna_dev *xdna;
 	int idx;
 
 	xdna = hwctx->client->xdna;
+	ndev = xdna->dev_handle;
 
 	XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq);
 	aie2_hwctx_wait_for_idle(hwctx);
 
 	/* Request fw to destroy hwctx and cancel the rest pending requests */
-	drm_sched_stop(&hwctx->priv->sched, NULL);
+	drm_sched_stop(hwctx->priv->sched, NULL);
 	aie2_release_resource(hwctx);
-	drm_sched_start(&hwctx->priv->sched, 0);
+	drm_sched_start(hwctx->priv->sched, 0);
 
 	mutex_unlock(&xdna->dev_lock);
 	drm_sched_entity_destroy(&hwctx->priv->entity);
@@ -833,11 +893,17 @@ void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx)
 		   atomic64_read(&hwctx->job_free_cnt));
 	mutex_lock(&xdna->dev_lock);
 
-	drm_sched_fini(&hwctx->priv->sched);
+	aie2_hwctx_sched_free(ndev, hwctx->priv->sched);
 	aie2_ctx_syncobj_destroy(hwctx);
 
-	for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++)
+	for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) {
+		/*
+		 * The open/close will never be called for driver allocated
+		 * dev bo. Call amdxdna_gem_heap_free explicitly.
+		 */
+		amdxdna_gem_heap_free(hwctx->client, hwctx->priv->cmd_buf[idx]);
 		drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx]));
+	}
 	amdxdna_gem_unpin(hwctx->priv->heap);
 	drm_gem_object_put(to_gobj(hwctx->priv->heap));
 
diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c
index b4c49259a1a23b..f658760c3d480e 100644
--- a/drivers/accel/amdxdna/aie2_message.c
+++ b/drivers/accel/amdxdna/aie2_message.c
@@ -277,6 +277,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 
 free_channel:
 	xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
+	hwctx->priv->mbox_chann = NULL;
 del_ctx_req:
 	aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
 	return ret;
diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h
index fd65a4236d491d..8ed2b5701dcc25 100644
--- a/drivers/accel/amdxdna/aie2_msg_priv.h
+++ b/drivers/accel/amdxdna/aie2_msg_priv.h
@@ -530,6 +530,7 @@ enum aie2_dev_revision {
 	AIE2_DEV_REVISION_GPT1,
 	AIE2_DEV_REVISION_GPT2,
 	AIE2_DEV_REVISION_GPT3,
+	AIE2_DEV_REVISION_GRGH,
 	AIE2_DEV_REVISION_UNKN,
 };
 
diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c
index daec1f6b490797..7a4314ca843bc5 100644
--- a/drivers/accel/amdxdna/aie2_pci.c
+++ b/drivers/accel/amdxdna/aie2_pci.c
@@ -503,10 +503,16 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	ndev->priv = xdna->dev_info->dev_priv;
 	ndev->aie.xdna = xdna;
 
+	ret = aie2_hwctx_sched_init(ndev);
+	if (ret)
+		return ret;
+
 	for (i = 0; i < ARRAY_SIZE(npu_fw); i++) {
 		fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]);
-		if (!fw_full_path)
-			return -ENOMEM;
+		if (!fw_full_path) {
+			ret = -ENOMEM;
+			goto free_hwctx_sched;
+		}
 
 		ret = firmware_request_nowarn(&fw, fw_full_path, &pdev->dev);
 		kfree(fw_full_path);
@@ -519,7 +525,7 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	if (ret) {
 		XDNA_ERR(xdna, "failed to request_firmware %s, ret %d",
 			 ndev->priv->fw_path, ret);
-		return ret;
+		goto free_hwctx_sched;
 	}
 
 	ret = pcim_enable_device(pdev);
@@ -623,6 +629,8 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	aie2_hw_stop(xdna);
 release_fw:
 	release_firmware(fw);
+free_hwctx_sched:
+	aie2_hwctx_sched_fini(ndev);
 
 	return ret;
 }
@@ -631,6 +639,7 @@ static void aie2_fini(struct amdxdna_dev *xdna)
 {
 	amdxdna_pm_fini(xdna);
 	aie2_hw_stop(xdna);
+	aie2_hwctx_sched_fini(xdna->dev_handle);
 }
 
 static int aie2_get_aie_status(struct amdxdna_client *client,
diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h
index ea1dac10640042..2c7019bd26b5ee 100644
--- a/drivers/accel/amdxdna/aie2_pci.h
+++ b/drivers/accel/amdxdna/aie2_pci.h
@@ -107,7 +107,7 @@ struct amdxdna_hwctx_priv {
 	struct amdxdna_gem_obj		*heap;
 	void				*mbox_chann;
 
-	struct drm_gpu_scheduler	sched;
+	struct drm_gpu_scheduler	*sched;
 	struct drm_sched_entity		entity;
 
 	struct mutex			io_lock; /* protect seq and cmd order */
@@ -152,6 +152,8 @@ struct amdxdna_dev_hdl {
 	u32				total_col;
 	struct amdxdna_drm_query_aie_version version;
 	struct aie2_exec_msg_ops	*exec_msg_ops;
+	struct drm_gpu_scheduler	*hwctx_sched;
+	struct ida			hwctx_sched_ida;
 
 	/* power management and clock*/
 	enum amdxdna_power_mode_type	pw_mode;
@@ -294,6 +296,8 @@ int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us);
 /* aie2_hwctx.c */
 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx);
 void aie2_hwctx_fini(struct amdxdna_hwctx *hwctx);
+int aie2_hwctx_sched_init(struct amdxdna_dev_hdl *ndev);
+void aie2_hwctx_sched_fini(struct amdxdna_dev_hdl *ndev);
 int aie2_hwctx_config(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size);
 int aie2_hwctx_sync_debug_bo(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl);
 void aie2_hwctx_suspend(struct amdxdna_client *client);
diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index 4b0d58d0329b75..f4832337ec31d1 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -14,6 +14,7 @@
 #include <linux/dma-direct.h>
 #include <linux/iosys-map.h>
 #include <linux/pagemap.h>
+#include <linux/swap.h>
 #include <linux/vmalloc.h>
 
 #include "amdxdna_cbuf.h"
@@ -37,6 +38,7 @@ amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo)
 	struct amdxdna_gem_obj *heap;
 	u64 heap_addr, exp_heap_uva;
 	u32 heap_id;
+	int ret;
 
 	if (xa_empty(&client->dev_heap_xa)) {
 		XDNA_DBG(xdna, "Empty heap xa");
@@ -58,24 +60,32 @@ amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo)
 	heap = xa_load(&client->dev_heap_xa, heap_id);
 	exp_heap_uva = amdxdna_gem_uva(heap);
 	heap_addr = amdxdna_gem_dev_addr(heap);
-	dev_bo->heap_start_id = heap_id;
 	dev_bo->mem.uva = dev_bo->mm_node.start - heap_addr + exp_heap_uva;
 
 	for (; heap_id < client->dev_heap_nid; heap_id++) {
 		heap = xa_load(&client->dev_heap_xa, heap_id);
 		if (!heap) {
 			XDNA_ERR(xdna, "Failed to load heap %d", heap_id);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto cleanup_heap_xa;
 		}
 		heap_addr = amdxdna_gem_uva(heap);
 		if (heap_addr == AMDXDNA_INVALID_ADDR) {
 			XDNA_ERR(xdna, "Heap %d is not mapped", heap_id);
-			return -EAGAIN;
+			ret = -EAGAIN;
+			goto cleanup_heap_xa;
 		}
 
 		if (heap_addr != exp_heap_uva) {
 			XDNA_ERR(xdna, "Heap %d uva is not contiguous", heap_id);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto cleanup_heap_xa;
+		}
+
+		ret = xa_insert(&dev_bo->heap_xa, heap_id, heap, GFP_KERNEL);
+		if (ret) {
+			ret = -ENOMEM;
+			goto cleanup_heap_xa;
 		}
 
 		if (heap->dev_addr + heap->mem.size >=
@@ -87,12 +97,15 @@ amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo)
 
 	if (heap_id == client->dev_heap_nid) {
 		XDNA_DBG(xdna, "Can not find heap end");
-		return -EAGAIN;
+		ret = -EAGAIN;
+		goto cleanup_heap_xa;
 	}
 
-	dev_bo->heap_end_id = heap_id;
-
 	return 0;
+
+cleanup_heap_xa:
+	xa_destroy(&dev_bo->heap_xa);
+	return ret;
 }
 
 static int
@@ -132,8 +145,7 @@ amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
 	}
 
 	client->heap_usage += mem->size;
-	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
-			  abo->heap_start_id, abo->heap_end_id)
+	xa_for_each(&abo->heap_xa, heap_id, heap)
 		drm_gem_object_get(to_gobj(heap));
 
 unlock_out:
@@ -142,48 +154,16 @@ amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
 	return ret;
 }
 
-static void
-amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo)
+void amdxdna_gem_heap_free(struct amdxdna_client *client, struct amdxdna_gem_obj *abo)
 {
-	struct amdxdna_client *client = abo->client;
-	struct amdxdna_gem_obj *heap;
-	unsigned long heap_id;
-
 	mutex_lock(&client->mm_lock);
 
 	drm_mm_remove_node(&abo->mm_node);
 	client->heap_usage -= abo->mem.size;
 
-	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
-			  abo->heap_start_id, abo->heap_end_id)
-		drm_gem_object_put(to_gobj(heap));
-
 	mutex_unlock(&client->mm_lock);
 }
 
-static struct amdxdna_gem_obj *
-amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
-{
-	struct amdxdna_gem_obj *abo;
-
-	abo = kzalloc_obj(*abo);
-	if (!abo)
-		return ERR_PTR(-ENOMEM);
-
-	abo->pinned = false;
-	abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
-	mutex_init(&abo->lock);
-
-	abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
-	abo->mem.uva = AMDXDNA_INVALID_ADDR;
-	abo->mem.size = size;
-	abo->open_ref = 0;
-	abo->internal = false;
-	INIT_LIST_HEAD(&abo->mem.umap_list);
-
-	return abo;
-}
-
 static void
 amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo)
 {
@@ -253,13 +233,16 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
 	struct amdxdna_gem_obj *abo = mapp->abo;
 	struct amdxdna_dev *xdna;
 
+	if (!mmu_notifier_range_blockable(range))
+		return false;
+
+	if (mapp->unmapped)
+		return true;
+
 	xdna = to_xdna_dev(to_gobj(abo)->dev);
 	XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d",
 		 mapp->range.start, mapp->range.end, abo->type);
 
-	if (!mmu_notifier_range_blockable(range))
-		return false;
-
 	down_write(&xdna->notifier_lock);
 	abo->mem.map_invalid = true;
 	mapp->invalid = true;
@@ -271,10 +254,8 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
 
 	if (range->event == MMU_NOTIFY_UNMAP) {
 		down_write(&xdna->notifier_lock);
-		if (!mapp->unmapped) {
-			queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
-			mapp->unmapped = true;
-		}
+		mapp->unmapped = true;
+		queue_work(xdna->notifier_wq, &abo->hmm_unreg_work);
 		up_write(&xdna->notifier_lock);
 	}
 
@@ -301,33 +282,42 @@ static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo,
 
 	down_write(&xdna->notifier_lock);
 	list_for_each_entry(mapp, &abo->mem.umap_list, node) {
-		if (!vma || compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end)) {
-			if (!mapp->unmapped) {
-				queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work);
-				mapp->unmapped = true;
-			}
-			if (vma)
-				break;
-		}
+		if (!compare_range(mapp, vma->vm_mm, vma->vm_start, vma->vm_end))
+			continue;
+
+		mapp->unmapped = true;
+		queue_work(xdna->notifier_wq, &abo->hmm_unreg_work);
 	}
 	up_write(&xdna->notifier_lock);
 }
 
-static void amdxdna_umap_release(struct kref *ref)
+static void amdxdna_hmm_unreg_umaps(struct amdxdna_gem_obj *abo, bool force)
 {
-	struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt);
-	struct amdxdna_gem_obj *abo = mapp->abo;
-	struct amdxdna_dev *xdna;
-
-	mmu_interval_notifier_remove(&mapp->notifier);
+	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
+	struct amdxdna_umap *mapp, *tmp;
+	LIST_HEAD(dead);
 
-	xdna = to_xdna_dev(to_gobj(mapp->abo)->dev);
 	down_write(&xdna->notifier_lock);
-	list_del(&mapp->node);
+	list_for_each_entry_safe(mapp, tmp, &abo->mem.umap_list, node) {
+		if (!force && !mapp->unmapped)
+			continue;
+
+		mapp->unmapped = true;
+		list_move(&mapp->node, &dead);
+	}
 	if (list_empty(&abo->mem.umap_list))
 		abo->mem.uva = AMDXDNA_INVALID_ADDR;
 	up_write(&xdna->notifier_lock);
 
+	list_for_each_entry_safe(mapp, tmp, &dead, node)
+		amdxdna_umap_put(mapp);
+}
+
+static void amdxdna_umap_release(struct kref *ref)
+{
+	struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt);
+
+	mmu_interval_notifier_remove(&mapp->notifier);
 	kvfree(mapp->range.hmm_pfns);
 	kfree(mapp);
 }
@@ -339,10 +329,10 @@ void amdxdna_umap_put(struct amdxdna_umap *mapp)
 
 static void amdxdna_hmm_unreg_work(struct work_struct *work)
 {
-	struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap,
-						 hmm_unreg_work);
+	struct amdxdna_gem_obj *abo = container_of(work, struct amdxdna_gem_obj,
+						   hmm_unreg_work);
 
-	amdxdna_umap_put(mapp);
+	amdxdna_hmm_unreg_umaps(abo, false);
 }
 
 static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
@@ -387,6 +377,21 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
 		goto free_map;
 	}
 
+	mapp->range.notifier = &mapp->notifier;
+	mapp->range.start = vma->vm_start;
+	mapp->range.end = vma->vm_end;
+	/*
+	 * Access permissions are fixed at mmap() time. Changing them later
+	 * with mprotect() is not supported: the range keeps requesting the
+	 * original permissions, so the application may see a fault failure
+	 * or an IOMMU fault.
+	 */
+	mapp->range.default_flags = HMM_PFN_REQ_FAULT;
+	if (vma->vm_flags & VM_WRITE)
+		mapp->range.default_flags |= HMM_PFN_REQ_WRITE;
+	mapp->abo = abo;
+	kref_init(&mapp->refcnt);
+
 	ret = mmu_interval_notifier_insert_locked(&mapp->notifier,
 						  current->mm,
 						  addr,
@@ -397,15 +402,6 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
 		goto free_pfns;
 	}
 
-	mapp->range.notifier = &mapp->notifier;
-	mapp->range.start = vma->vm_start;
-	mapp->range.end = vma->vm_end;
-	mapp->range.default_flags = HMM_PFN_REQ_FAULT;
-	mapp->abo = abo;
-	kref_init(&mapp->refcnt);
-
-	INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work);
-
 	down_write(&xdna->notifier_lock);
 	if (list_empty(&abo->mem.umap_list))
 		abo->mem.uva = addr;
@@ -421,17 +417,47 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
 	return ret;
 }
 
+static struct amdxdna_gem_obj *
+amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
+{
+	struct amdxdna_gem_obj *abo;
+
+	abo = kzalloc_obj(*abo);
+	if (!abo)
+		return ERR_PTR(-ENOMEM);
+
+	abo->pinned = false;
+	abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
+	mutex_init(&abo->lock);
+
+	abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
+	abo->mem.uva = AMDXDNA_INVALID_ADDR;
+	abo->mem.size = size;
+	abo->open_ref = 0;
+	abo->internal = false;
+	INIT_LIST_HEAD(&abo->mem.umap_list);
+	xa_init_flags(&abo->heap_xa, XA_FLAGS_ALLOC);
+	INIT_WORK(&abo->hmm_unreg_work, amdxdna_hmm_unreg_work);
+
+	return abo;
+}
+
 static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj)
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+	struct amdxdna_gem_obj *heap;
+	unsigned long heap_id;
 
 	XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, amdxdna_gem_dev_addr(abo));
 	if (abo->pinned)
 		amdxdna_gem_unpin(abo);
 
 	amdxdna_gem_vunmap(abo);
-	amdxdna_gem_heap_free(abo);
+	xa_for_each(&abo->heap_xa, heap_id, heap)
+		drm_gem_object_put(to_gobj(heap));
+	xa_destroy(&abo->heap_xa);
+
 	drm_gem_object_release(gobj);
 	amdxdna_gem_destroy_obj(abo);
 }
@@ -458,16 +484,16 @@ static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo,
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
 	unsigned long num_pages = vma_pages(vma);
-	unsigned long offset = 0;
 	int ret;
 
-	if (!is_import_bo(abo)) {
-		ret = drm_gem_shmem_mmap(&abo->base, vma);
-		if (ret) {
-			XDNA_ERR(xdna, "Failed shmem mmap %d", ret);
-			return ret;
-		}
-	} else {
+	/*
+	 * Until today there is not any use case to mmap with non-zero
+	 * offset. Put an explicit check here.
+	 */
+	if (vma->vm_pgoff - drm_vma_node_start(&to_gobj(abo)->vma_node))
+		return -EINVAL;
+
+	if (is_import_bo(abo)) {
 		vma->vm_private_data = NULL;
 		vma->vm_ops = NULL;
 		ret = dma_buf_mmap(abo->dma_buf, vma, 0);
@@ -476,23 +502,28 @@ static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo,
 			return ret;
 		}
 
+		amdxdna_mark_mapp_invalid(abo, vma);
+
 		/* Drop the reference drm_gem_mmap_obj() acquired.*/
 		drm_gem_object_put(to_gobj(abo));
+		return 0;
 	}
 
-	do {
-		vm_fault_t fault_ret;
-
-		fault_ret = handle_mm_fault(vma, vma->vm_start + offset,
-					    FAULT_FLAG_WRITE, NULL);
-		if (fault_ret & VM_FAULT_ERROR) {
-			XDNA_ERR(xdna, "Fault in page failed");
-			amdxdna_mark_mapp_invalid(abo, vma);
-			break;
-		}
+	ret = drm_gem_shmem_mmap(&abo->base, vma);
+	if (ret) {
+		XDNA_ERR(xdna, "Failed shmem mmap %d", ret);
+		return ret;
+	}
 
-		offset += PAGE_SIZE;
-	} while (--num_pages);
+	vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
+	ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, &num_pages);
+	if (ret) {
+		XDNA_ERR(xdna, "Failed to insert pages %d", ret);
+		dma_resv_lock(to_gobj(abo)->resv, NULL);
+		drm_gem_shmem_put_pages_locked(&abo->base);
+		dma_resv_unlock(to_gobj(abo)->resv);
+		return ret;
+	}
 
 	return 0;
 }
@@ -504,6 +535,10 @@ static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
 	int ret;
 
+	XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx",
+		 drm_vma_node_offset_addr(&gobj->vma_node), abo->type,
+		 vma->vm_start, gobj->size);
+
 	ret = amdxdna_hmm_register(abo, vma);
 	if (ret)
 		return ret;
@@ -514,9 +549,6 @@ static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
 		goto hmm_unreg;
 	}
 
-	XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx",
-		 drm_vma_node_offset_addr(&gobj->vma_node), abo->type,
-		 vma->vm_start, gobj->size);
 	return 0;
 
 hmm_unreg:
@@ -524,14 +556,99 @@ static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
 	return ret;
 }
 
+/*
+ * VM operations for amdxdna shmem VMAs that use VM_MIXEDMAP.
+ *
+ * drm_gem_shmem_vm_ops cannot be used on VM_MIXEDMAP VMAs because its fault
+ * handler calls vmf_insert_pfn() → vmf_insert_pfn_prot() which contains:
+ *   BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn))
+ * All amdxdna shmem pages are ordinary struct pages so pfn_valid() is always
+ * true, making the combination fatal.
+ *
+ * These ops use vmf_insert_page() (struct-page based) instead, which is the
+ * correct API for VM_MIXEDMAP VMAs backed by real struct pages.  The open and
+ * close handlers replicate drm_gem_shmem_vm_open/close using only exported
+ * symbols.
+ */
+static vm_fault_t amdxdna_gem_mixedmap_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct drm_gem_object *gobj = vma->vm_private_data;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(gobj);
+	loff_t num_pages = gobj->size >> PAGE_SHIFT;
+	vm_fault_t ret = VM_FAULT_SIGBUS;
+	pgoff_t page_offset;
+	struct page *page;
+
+	/*
+	 * Partial free of vma is unexpected. Otherwise, the wrong page
+	 * will be faulted in and the user application may crash itself.
+	 */
+	page_offset = linear_page_delta(vma, vmf->address);
+
+	dma_resv_lock(gobj->resv, NULL);
+
+	if (!shmem->pages || shmem->madv < 0 || page_offset >= num_pages)
+		goto out;
+
+	page = shmem->pages[page_offset];
+	if (WARN_ON_ONCE(!page))
+		goto out;
+
+	/*
+	 * Use vmf_insert_page() (struct-page path) not vmf_insert_pfn()
+	 * (PFN path) because this VMA carries VM_MIXEDMAP.
+	 */
+	ret = vmf_insert_page(vma, vmf->address, page);
+	if (ret == VM_FAULT_NOPAGE)
+		folio_mark_accessed(page_folio(page));
+
+out:
+	dma_resv_unlock(gobj->resv);
+	return ret;
+}
+
+static void amdxdna_gem_mixedmap_vm_open(struct vm_area_struct *vma)
+{
+	struct drm_gem_object *gobj = vma->vm_private_data;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(gobj);
+
+	/*
+	 * Bump pages_use_count so the page array stays alive for the new
+	 * mapping copy created by fork().  Mirrors drm_gem_shmem_vm_open().
+	 */
+	dma_resv_lock(gobj->resv, NULL);
+	drm_WARN_ON_ONCE(gobj->dev, !refcount_inc_not_zero(&shmem->pages_use_count));
+	dma_resv_unlock(gobj->resv);
+
+	drm_gem_vm_open(vma);
+}
+
+static void amdxdna_gem_mixedmap_vm_close(struct vm_area_struct *vma)
+{
+	struct drm_gem_object *gobj = vma->vm_private_data;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(gobj);
+
+	dma_resv_lock(gobj->resv, NULL);
+	drm_gem_shmem_put_pages_locked(shmem);
+	dma_resv_unlock(gobj->resv);
+
+	drm_gem_vm_close(vma);
+}
+
+static const struct vm_operations_struct amdxdna_gem_mixedmap_vm_ops = {
+	.fault  = amdxdna_gem_mixedmap_fault,
+	.open   = amdxdna_gem_mixedmap_vm_open,
+	.close  = amdxdna_gem_mixedmap_vm_close,
+};
+
 static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma)
 {
 	struct drm_gem_object *gobj = dma_buf->priv;
 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
-	unsigned long num_pages = vma_pages(vma);
 	int ret;
 
-	vma->vm_ops = &drm_gem_shmem_vm_ops;
+	vma->vm_ops = &amdxdna_gem_mixedmap_vm_ops;
 	vma->vm_private_data = gobj;
 
 	drm_gem_object_get(gobj);
@@ -541,16 +658,9 @@ static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struc
 
 	/* The buffer is based on memory pages. Fix the flag. */
 	vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
-	ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages,
-			      &num_pages);
-	if (ret)
-		goto close_vma;
 
 	return 0;
 
-close_vma:
-	vma->vm_ops->close(vma);
-	return ret;
 put_obj:
 	drm_gem_object_put(gobj);
 	return ret;
@@ -609,10 +719,8 @@ amdxdna_gem_skip_bo_usage(struct amdxdna_gem_obj *abo)
 }
 
 static void
-amdxdna_gem_add_bo_usage(struct amdxdna_gem_obj *abo)
+amdxdna_gem_add_bo_usage(struct amdxdna_client *client, struct amdxdna_gem_obj *abo)
 {
-	struct amdxdna_client *client = abo->client;
-
 	if (amdxdna_gem_skip_bo_usage(abo))
 		return;
 
@@ -624,10 +732,8 @@ amdxdna_gem_add_bo_usage(struct amdxdna_gem_obj *abo)
 }
 
 static void
-amdxdna_gem_del_bo_usage(struct amdxdna_gem_obj *abo)
+amdxdna_gem_del_bo_usage(struct amdxdna_client *client, struct amdxdna_gem_obj *abo)
 {
-	struct amdxdna_client *client = abo->client;
-
 	if (amdxdna_gem_skip_bo_usage(abo))
 		return;
 
@@ -643,8 +749,9 @@ static void amdxdna_gem_obj_free(struct drm_gem_object *gobj)
 	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
 
-	amdxdna_hmm_unregister(abo, NULL);
-	flush_workqueue(xdna->notifier_wq);
+	/* No notifier survives this, so no new work can be queued. */
+	amdxdna_hmm_unreg_umaps(abo, true);
+	cancel_work_sync(&abo->hmm_unreg_work);
 
 	if (abo->pinned)
 		amdxdna_gem_unpin(abo);
@@ -663,12 +770,20 @@ static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *fi
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+	struct amdxdna_client *client;
 	int ret;
 
-	guard(mutex)(&abo->lock);
+	mutex_lock(&abo->lock);
+	if (abo->open_ref > 0 && filp->driver_priv != abo->client) {
+		mutex_unlock(&abo->lock);
+		return -EPERM;
+	}
+
 	abo->open_ref++;
-	if (abo->open_ref > 1)
+	if (abo->open_ref > 1) {
+		mutex_unlock(&abo->lock);
 		return 0;
+	}
 
 	/* Attached to the client when first opened by it. */
 	abo->client = filp->driver_priv;
@@ -679,26 +794,34 @@ static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *fi
 		if (ret) {
 			abo->open_ref--;
 			abo->client = NULL;
+			mutex_unlock(&abo->lock);
 			return ret;
 		}
 	}
+	client = abo->client;
+	mutex_unlock(&abo->lock);
 
-	amdxdna_gem_add_bo_usage(abo);
+	amdxdna_gem_add_bo_usage(client, abo);
 	return 0;
 }
 
 static void amdxdna_gem_obj_close(struct drm_gem_object *gobj, struct drm_file *filp)
 {
 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+	struct amdxdna_client *client = NULL;
 
-	guard(mutex)(&abo->lock);
+	mutex_lock(&abo->lock);
 	abo->open_ref--;
 
 	if (abo->open_ref == 0) {
-		amdxdna_gem_del_bo_usage(abo);
 		/* Detach from the client when last closed by it. */
+		client = abo->client;
 		abo->client = NULL;
 	}
+	mutex_unlock(&abo->lock);
+
+	if (client)
+		amdxdna_gem_del_bo_usage(client, abo);
 }
 
 static int amdxdna_gem_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map)
@@ -734,19 +857,72 @@ static void amdxdna_gem_obj_vunmap(struct drm_gem_object *obj, struct iosys_map
 		drm_gem_shmem_object_vunmap(obj, map);
 }
 
+static int amdxdna_gem_dev_obj_open(struct drm_gem_object *gobj, struct drm_file *filp)
+{
+	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+
+	guard(mutex)(&abo->lock);
+	if (filp->driver_priv != abo->client)
+		return -EPERM;
+	abo->open_ref++;
+
+	return 0;
+}
+
+static void amdxdna_gem_dev_obj_close(struct drm_gem_object *gobj, struct drm_file *filp)
+{
+	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
+	struct amdxdna_client *client = NULL;
+
+	mutex_lock(&abo->lock);
+	abo->open_ref--;
+
+	/*
+	 * Freeing the heap allocation here, when the handle is closed, is
+	 * intentional.  DEV BOs are carved out of a per-client drm_mm heap;
+	 * any subsequent allocation that lands on the same device address will
+	 * also belong to the same client.  If the user closes the handle while
+	 * a job is still in flight the only consequence is self-inflicted
+	 * corruption within their own context -- it cannot affect other
+	 * processes.  The GEM reference held by the in-flight job keeps the
+	 * amdxdna_gem_obj struct alive until the job completes; it does not
+	 * prevent the device address from being reclaimed by the allocator.
+	 *
+	 * Cross-process handle creation for DEV BOs is rejected in
+	 * amdxdna_gem_dev_obj_open(), which prevents the following UAF:
+	 * if a second process shared the handle via GEM flink and the
+	 * original creator exited (freeing client), the importer would later
+	 * reach open_ref == 0 here and call amdxdna_gem_heap_free() with a
+	 * dangling abo->client pointer.  Because cross-process opens are
+	 * rejected, the process arriving here is always the owning client,
+	 * which is still alive.  abo->client is nulled out afterwards so that
+	 * any code path running on a lingering GEM reference (e.g. an
+	 * in-flight job) cannot silently dereference a stale pointer.
+	 */
+	if (abo->open_ref == 0) {
+		client = abo->client;
+		abo->client = NULL;
+	}
+	mutex_unlock(&abo->lock);
+
+	if (client)
+		amdxdna_gem_heap_free(client, abo);
+}
+
 static int amdxdna_gem_dev_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map)
 {
 	struct amdxdna_gem_obj *abo = to_xdna_obj(obj);
 	struct amdxdna_gem_obj *heap;
+	unsigned long index = 0;
 	void *base;
 	u64 offset;
 
-	/* vmap dev bo which is across more than 1 heap is not allowed */
-	if (abo->heap_start_id != abo->heap_end_id)
+	heap = xa_find(&abo->heap_xa, &index, ULONG_MAX, XA_PRESENT);
+	if (!heap)
 		return -ENOMEM;
 
-	heap = xa_load(&abo->client->dev_heap_xa, abo->heap_start_id);
-	if (!heap)
+	/* vmap dev bo which is across more than 1 heap is not allowed */
+	if (xa_find_after(&abo->heap_xa, &index, ULONG_MAX, XA_PRESENT))
 		return -ENOMEM;
 
 	base = amdxdna_gem_vmap(heap);
@@ -765,6 +941,8 @@ static struct dma_buf *amdxdna_gem_dev_obj_export(struct drm_gem_object *gobj, i
 
 static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = {
 	.free = amdxdna_gem_dev_obj_free,
+	.open = amdxdna_gem_dev_obj_open,
+	.close = amdxdna_gem_dev_obj_close,
 	.vmap = amdxdna_gem_dev_obj_vmap,
 	.export = amdxdna_gem_dev_obj_export,
 };
@@ -780,7 +958,7 @@ static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
 	.vmap = amdxdna_gem_obj_vmap,
 	.vunmap = amdxdna_gem_obj_vunmap,
 	.mmap = amdxdna_gem_obj_mmap,
-	.vm_ops = &drm_gem_shmem_vm_ops,
+	.vm_ops = &amdxdna_gem_mixedmap_vm_ops,
 	.export = amdxdna_gem_prime_export,
 };
 
@@ -1103,6 +1281,8 @@ int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_f
 		 args->handle, args->type, amdxdna_gem_uva(abo),
 		 amdxdna_gem_dev_addr(abo), abo->mem.size);
 put_obj:
+	if (ret && abo->type == AMDXDNA_BO_DEV)
+		amdxdna_gem_heap_free(client, abo);
 	/* Dereference object reference. Handle holds it now. */
 	drm_gem_object_put(to_gobj(abo));
 	return ret;
@@ -1136,7 +1316,6 @@ static void amdxdna_bo_unpin(struct amdxdna_gem_obj *abo)
 
 int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
 {
-	struct amdxdna_client *client = abo->client;
 	struct amdxdna_gem_obj *heap;
 	unsigned long heap_id, last = ULONG_MAX;
 	int ret = 0;
@@ -1144,17 +1323,15 @@ int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
 	if (abo->type != AMDXDNA_BO_DEV)
 		return amdxdna_bo_pin(abo);
 
-	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
-			  abo->heap_start_id, abo->heap_end_id) {
+	xa_for_each(&abo->heap_xa, heap_id, heap) {
 		ret = amdxdna_bo_pin(heap);
 		if (ret)
 			break;
 		last = heap_id;
 	}
 
-	if (ret && last <= abo->heap_end_id) {
-		xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
-				  abo->heap_start_id, last)
+	if (ret && last != ULONG_MAX) {
+		xa_for_each_range(&abo->heap_xa, heap_id, heap, 0, last)
 			amdxdna_bo_unpin(heap);
 	}
 
@@ -1179,8 +1356,7 @@ void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo)
 		struct amdxdna_gem_obj *heap;
 		unsigned long heap_id;
 
-		xa_for_each_range(&abo->client->dev_heap_xa, heap_id, heap,
-				  abo->heap_start_id, abo->heap_end_id)
+		xa_for_each(&abo->heap_xa, heap_id, heap)
 			amdxdna_bo_unpin(heap);
 	} else {
 		amdxdna_bo_unpin(abo);
@@ -1300,8 +1476,7 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
 		u64 flush_start = bo_start + args->offset;
 		u64 flush_end = flush_start + args->size;
 
-		xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
-				  abo->heap_start_id, abo->heap_end_id) {
+		xa_for_each(&abo->heap_xa, heap_id, heap) {
 			u64 heap_start = amdxdna_gem_dev_addr(heap);
 			u64 heap_end = heap_start + heap->mem.size;
 			u64 start = max(flush_start, heap_start);
@@ -1328,7 +1503,7 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
 		amdxdna_gem_unpin(abo);
 
 		if (ret) {
-			drm_WARN(&xdna->ddev, 1, "Can not get flush memory");
+			XDNA_DBG(xdna, "Flush BO %d failed, ret %d", args->handle, ret);
 			goto put_obj;
 		}
 	}
@@ -1336,7 +1511,8 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
 	XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n",
 		 args->handle, args->offset, args->size);
 
-	if (args->direction == SYNC_DIRECT_FROM_DEVICE)
+	if (abo->assigned_hwctx != AMDXDNA_INVALID_CTX_HANDLE &&
+	    args->direction == SYNC_DIRECT_FROM_DEVICE)
 		ret = amdxdna_hwctx_sync_debug_bo(client, args->handle);
 
 put_obj:
diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h
index 1e90e32bf3cd79..9b4aa21a37c99b 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.h
+++ b/drivers/accel/amdxdna/amdxdna_gem.h
@@ -14,7 +14,6 @@
 struct amdxdna_umap {
 	struct mmu_interval_notifier	notifier;
 	struct hmm_range		range;
-	struct work_struct		hmm_unreg_work;
 	struct amdxdna_gem_obj		*abo;
 	struct list_head		node;
 	struct kref			refcnt;
@@ -43,11 +42,11 @@ struct amdxdna_gem_obj {
 	struct mutex			lock; /* Protects: pinned, mem.kva, open_ref */
 	struct amdxdna_mem		mem;
 	int				open_ref;
+	struct work_struct		hmm_unreg_work;
 
 	/* Below members are initialized when needed */
 	struct drm_mm_node		mm_node; /* For AMDXDNA_BO_DEV */
-	u32				heap_start_id;
-	u32				heap_end_id;
+	struct xarray			heap_xa;
 	u64				dev_addr; /* For heap bo */
 	u32				assigned_hwctx;
 	struct dma_buf			*dma_buf;
@@ -104,6 +103,7 @@ static inline u64 amdxdna_obj_dma_addr(struct amdxdna_gem_obj *abo)
 }
 
 void amdxdna_umap_put(struct amdxdna_umap *mapp);
+void amdxdna_gem_heap_free(struct amdxdna_client *client, struct amdxdna_gem_obj *abo);
 
 struct drm_gem_object *
 amdxdna_gem_create_shmem_object_cb(struct drm_device *dev, size_t size);
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c
index cc8865f4e79c9f..05c3786de13519 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox.c
+++ b/drivers/accel/amdxdna/amdxdna_mailbox.c
@@ -481,7 +481,7 @@ struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb)
 		return NULL;
 
 	INIT_WORK(&mb_chann->rx_work, mailbox_rx_worker);
-	mb_chann->work_q = create_singlethread_workqueue(MAILBOX_NAME);
+	mb_chann->work_q = alloc_ordered_workqueue(MAILBOX_NAME, WQ_MEM_RECLAIM);
 	if (!mb_chann->work_q) {
 		MB_ERR(mb_chann, "Create workqueue failed");
 		goto free_chann;
diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c
index 15a1613846252b..c9648c8032ec55 100644
--- a/drivers/accel/amdxdna/npu4_regs.c
+++ b/drivers/accel/amdxdna/npu4_regs.c
@@ -153,6 +153,7 @@ const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[] = {
 	{ AIE2_DEV_REVISION_GPT1, "NPU Gorgon Point 1" },
 	{ AIE2_DEV_REVISION_GPT2, "NPU Gorgon Point 2" },
 	{ AIE2_DEV_REVISION_GPT3, "NPU Gorgon Point 3" },
+	{ AIE2_DEV_REVISION_GRGH, "NPU Gorgon Halo" },
 	{ 0 }
 };
 
diff --git a/drivers/accel/ethosu/ethosu_job.c b/drivers/accel/ethosu/ethosu_job.c
index 8dce74db0cb489..d656b3ec670e82 100644
--- a/drivers/accel/ethosu/ethosu_job.c
+++ b/drivers/accel/ethosu/ethosu_job.c
@@ -345,10 +345,8 @@ int ethosu_job_init(struct ethosu_device *edev)
 					ethosu_job_irq_handler_thread,
 					0, KBUILD_MODNAME,
 					edev);
-	if (ret) {
-		dev_err(dev, "failed to request irq\n");
+	if (ret)
 		return ret;
-	}
 
 	edev->fence_context = dma_fence_context_alloc(1);
 
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 95120957f42a58..647684219e9d3b 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -205,6 +205,7 @@ bool ivpu_is_capable(struct ivpu_device *vdev, u32 capability)
 	case DRM_IVPU_CAP_BO_CREATE_FROM_USERPTR:
 		return true;
 	case DRM_IVPU_CAP_MANAGE_CMDQ:
+	case DRM_IVPU_CAP_CMDQ_SET_PRIORITY:
 		return vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW;
 	default:
 		return false;
@@ -273,6 +274,9 @@ static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_f
 	case DRM_IVPU_PARAM_PREEMPT_BUFFER_SIZE:
 		args->value = ivpu_fw_preempt_buf_size(vdev);
 		break;
+	case DRM_IVPU_PARAM_CMDQ_PRIORITY:
+		ret = ivpu_cmdq_get_priority(file_priv, args->index, &args->value);
+		break;
 	default:
 		ret = -EINVAL;
 		break;
@@ -284,14 +288,23 @@ static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_f
 
 static int ivpu_set_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
+	struct ivpu_file_priv *file_priv = file->driver_priv;
 	struct drm_ivpu_param *args = data;
 	int ret = 0;
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx))
+		return -ENODEV;
 
 	switch (args->param) {
+	case DRM_IVPU_PARAM_CMDQ_PRIORITY:
+		ret = ivpu_cmdq_set_priority(file_priv, args->index, args->value);
+		break;
 	default:
 		ret = -EINVAL;
 	}
 
+	drm_dev_exit(idx);
 	return ret;
 }
 
@@ -515,6 +528,7 @@ void ivpu_prepare_for_reset(struct ivpu_device *vdev)
 {
 	ivpu_hw_irq_disable(vdev);
 	disable_irq(vdev->irq);
+	atomic_set(&vdev->job_timeout_detected, 0);
 	flush_work(&vdev->irq_dct_work);
 	flush_work(&vdev->context_abort_work);
 	flush_work(&vdev->job_destroy_work);
@@ -710,7 +724,7 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
 	vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID;
 	atomic64_set(&vdev->unique_id_counter, 0);
 	atomic_set(&vdev->job_timeout_counter, 0);
-	atomic_set(&vdev->faults_detected, 0);
+	atomic_set(&vdev->job_timeout_detected, 0);
 	xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 	xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);
 	xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1);
diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h
index 86d7c9966cacbf..8762b8969d961a 100644
--- a/drivers/accel/ivpu/ivpu_drv.h
+++ b/drivers/accel/ivpu/ivpu_drv.h
@@ -91,7 +91,7 @@
 	drm_warn(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__)
 
 #define ivpu_warn_ratelimited(vdev, fmt, ...) \
-	drm_err_ratelimited(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__)
+	drm_warn_ratelimited(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__)
 
 #define ivpu_info(vdev, fmt, ...) drm_info(&(vdev)->drm, fmt, ##__VA_ARGS__)
 
@@ -171,7 +171,7 @@ struct ivpu_device {
 	struct xarray submitted_jobs_xa;
 	struct ivpu_ipc_consumer job_done_consumer;
 	atomic_t job_timeout_counter;
-	atomic_t faults_detected;
+	atomic_t job_timeout_detected;
 
 	atomic64_t unique_id_counter;
 
@@ -186,6 +186,9 @@ struct ivpu_device {
 		int autosuspend;
 		int d0i3_entry_msg;
 		int state_dump_msg;
+		int mmu_reg;
+		int mmu_queue;
+		int pwr_island_status;
 	} timeout;
 };
 
diff --git a/drivers/accel/ivpu/ivpu_fw.c b/drivers/accel/ivpu/ivpu_fw.c
index b78b7c35cd9b5d..f9ddaa2420c4c6 100644
--- a/drivers/accel/ivpu/ivpu_fw.c
+++ b/drivers/accel/ivpu/ivpu_fw.c
@@ -364,6 +364,25 @@ static void ivpu_fw_release(struct ivpu_device *vdev)
 }
 
 
+static void ivpu_fw_mem_fini(struct ivpu_device *vdev)
+{
+	struct ivpu_fw_info *fw = vdev->fw;
+
+	ivpu_bo_free(fw->mem_shave_nn);
+	ivpu_bo_free(fw->mem_log_verb);
+	ivpu_bo_free(fw->mem_log_crit);
+	ivpu_bo_free(fw->mem);
+	ivpu_bo_free(fw->mem_fw_ver);
+	ivpu_bo_free(fw->mem_bp);
+
+	fw->mem_shave_nn = NULL;
+	fw->mem_log_verb = NULL;
+	fw->mem_log_crit = NULL;
+	fw->mem = NULL;
+	fw->mem_fw_ver = NULL;
+	fw->mem_bp = NULL;
+}
+
 static int ivpu_fw_mem_init(struct ivpu_device *vdev)
 {
 	struct ivpu_fw_info *fw = vdev->fw;
@@ -382,7 +401,7 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev)
 	if (!fw->mem_fw_ver) {
 		ivpu_err(vdev, "Failed to create firmware version memory buffer\n");
 		ret = -ENOMEM;
-		goto err_free_bp;
+		goto err_free;
 	}
 
 	fw->mem = ivpu_bo_create_runtime(vdev, fw->runtime_addr, fw->runtime_size,
@@ -390,14 +409,14 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev)
 	if (!fw->mem) {
 		ivpu_err(vdev, "Failed to create firmware runtime memory buffer\n");
 		ret = -ENOMEM;
-		goto err_free_fw_ver;
+		goto err_free;
 	}
 
 	ret = ivpu_mmu_context_set_pages_ro(vdev, &vdev->gctx, fw->read_only_addr,
 					    fw->read_only_size);
 	if (ret) {
 		ivpu_err(vdev, "Failed to set firmware image read-only\n");
-		goto err_free_fw_mem;
+		goto err_free;
 	}
 
 	fw->mem_log_crit = ivpu_bo_create_global(vdev, IVPU_FW_CRITICAL_BUFFER_SIZE,
@@ -405,7 +424,7 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev)
 	if (!fw->mem_log_crit) {
 		ivpu_err(vdev, "Failed to create critical log buffer\n");
 		ret = -ENOMEM;
-		goto err_free_fw_mem;
+		goto err_free;
 	}
 
 	if (ivpu_fw_log_level <= IVPU_FW_LOG_INFO)
@@ -418,7 +437,7 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev)
 	if (!fw->mem_log_verb) {
 		ivpu_err(vdev, "Failed to create verbose log buffer\n");
 		ret = -ENOMEM;
-		goto err_free_log_crit;
+		goto err_free;
 	}
 
 	if (fw->shave_nn_size) {
@@ -427,47 +446,17 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev)
 		if (!fw->mem_shave_nn) {
 			ivpu_err(vdev, "Failed to create shavenn buffer\n");
 			ret = -ENOMEM;
-			goto err_free_log_verb;
+			goto err_free;
 		}
 	}
 
 	return 0;
 
-err_free_log_verb:
-	ivpu_bo_free(fw->mem_log_verb);
-err_free_log_crit:
-	ivpu_bo_free(fw->mem_log_crit);
-err_free_fw_mem:
-	ivpu_bo_free(fw->mem);
-err_free_fw_ver:
-	ivpu_bo_free(fw->mem_fw_ver);
-err_free_bp:
-	ivpu_bo_free(fw->mem_bp);
+err_free:
+	ivpu_fw_mem_fini(vdev);
 	return ret;
 }
 
-static void ivpu_fw_mem_fini(struct ivpu_device *vdev)
-{
-	struct ivpu_fw_info *fw = vdev->fw;
-
-	if (fw->mem_shave_nn) {
-		ivpu_bo_free(fw->mem_shave_nn);
-		fw->mem_shave_nn = NULL;
-	}
-
-	ivpu_bo_free(fw->mem_log_verb);
-	ivpu_bo_free(fw->mem_log_crit);
-	ivpu_bo_free(fw->mem);
-	ivpu_bo_free(fw->mem_fw_ver);
-	ivpu_bo_free(fw->mem_bp);
-
-	fw->mem_log_verb = NULL;
-	fw->mem_log_crit = NULL;
-	fw->mem = NULL;
-	fw->mem_fw_ver = NULL;
-	fw->mem_bp = NULL;
-}
-
 int ivpu_fw_init(struct ivpu_device *vdev)
 {
 	int ret;
diff --git a/drivers/accel/ivpu/ivpu_gem.c b/drivers/accel/ivpu/ivpu_gem.c
index 4f2005a8d4966e..62dbc2ef6e9d74 100644
--- a/drivers/accel/ivpu/ivpu_gem.c
+++ b/drivers/accel/ivpu/ivpu_gem.c
@@ -471,6 +471,9 @@ struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 fl
 
 void ivpu_bo_free(struct ivpu_bo *bo)
 {
+	if (!bo)
+		return;
+
 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
 
 	if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
diff --git a/drivers/accel/ivpu/ivpu_hw.c b/drivers/accel/ivpu/ivpu_hw.c
index 647dc045c231ab..47ed9beeb3fbef 100644
--- a/drivers/accel/ivpu/ivpu_hw.c
+++ b/drivers/accel/ivpu/ivpu_hw.c
@@ -99,13 +99,21 @@ static void wa_init(struct ivpu_device *vdev)
 static void timeouts_init(struct ivpu_device *vdev)
 {
 	if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) {
+		/* in milliseconds, negative value disables the timeout */
 		vdev->timeout.boot = -1;
 		vdev->timeout.jsm = -1;
 		vdev->timeout.tdr = -1;
 		vdev->timeout.inference = -1;
 		vdev->timeout.autosuspend = -1;
 		vdev->timeout.d0i3_entry_msg = -1;
+		/* fixed delay, not a timeout - has to remain finite */
+		vdev->timeout.state_dump_msg = 100;
+		/* in microseconds, zero disables the timeout */
+		vdev->timeout.mmu_reg = 0;
+		vdev->timeout.mmu_queue = 0;
+		vdev->timeout.pwr_island_status = 0;
 	} else if (ivpu_is_fpga(vdev)) {
+		/* in milliseconds */
 		vdev->timeout.boot = 50;
 		vdev->timeout.jsm = 15000;
 		vdev->timeout.tdr = 30000;
@@ -113,7 +121,12 @@ static void timeouts_init(struct ivpu_device *vdev)
 		vdev->timeout.autosuspend = -1;
 		vdev->timeout.d0i3_entry_msg = 500;
 		vdev->timeout.state_dump_msg = 10000;
+		/* in microseconds */
+		vdev->timeout.mmu_reg = 100 * USEC_PER_MSEC;
+		vdev->timeout.mmu_queue = 1000 * USEC_PER_MSEC;
+		vdev->timeout.pwr_island_status = 5 * USEC_PER_MSEC;
 	} else if (ivpu_is_simics(vdev)) {
+		/* in milliseconds */
 		vdev->timeout.boot = 50;
 		vdev->timeout.jsm = 500;
 		vdev->timeout.tdr = 10000;
@@ -121,7 +134,12 @@ static void timeouts_init(struct ivpu_device *vdev)
 		vdev->timeout.autosuspend = 100;
 		vdev->timeout.d0i3_entry_msg = 100;
 		vdev->timeout.state_dump_msg = 10;
+		/* in microseconds */
+		vdev->timeout.mmu_reg = 10 * USEC_PER_MSEC;
+		vdev->timeout.mmu_queue = 100 * USEC_PER_MSEC;
+		vdev->timeout.pwr_island_status = 5 * USEC_PER_MSEC;
 	} else {
+		/* in milliseconds */
 		vdev->timeout.boot = 1000;
 		vdev->timeout.jsm = 500;
 		vdev->timeout.tdr = 2000;
@@ -132,6 +150,10 @@ static void timeouts_init(struct ivpu_device *vdev)
 			vdev->timeout.autosuspend = 100;
 		vdev->timeout.d0i3_entry_msg = 5;
 		vdev->timeout.state_dump_msg = 100;
+		/* in microseconds */
+		vdev->timeout.mmu_reg = 10 * USEC_PER_MSEC;
+		vdev->timeout.mmu_queue = 100 * USEC_PER_MSEC;
+		vdev->timeout.pwr_island_status = 5 * USEC_PER_MSEC;
 	}
 }
 
diff --git a/drivers/accel/ivpu/ivpu_hw_ip.c b/drivers/accel/ivpu/ivpu_hw_ip.c
index 81f0b1f8f5a663..46661fbf9d0528 100644
--- a/drivers/accel/ivpu/ivpu_hw_ip.c
+++ b/drivers/accel/ivpu/ivpu_hw_ip.c
@@ -15,8 +15,6 @@
 #include "ivpu_mmu.h"
 #include "ivpu_pm.h"
 
-#define PWR_ISLAND_STATUS_TIMEOUT_US        (5 * USEC_PER_MSEC)
-
 #define TIM_SAFE_ENABLE		            0xf1d0dead
 #define TIM_WATCHDOG_RESET_VALUE            0xffffffff
 
@@ -350,10 +348,10 @@ static int wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val)
 
 	if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 		return REGV_POLL_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0, MSS_CPU, exp_val,
-				     PWR_ISLAND_STATUS_TIMEOUT_US);
+				     vdev->timeout.pwr_island_status);
 	else
 		return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU, exp_val,
-				     PWR_ISLAND_STATUS_TIMEOUT_US);
+				     vdev->timeout.pwr_island_status);
 }
 
 static void pwr_island_isolation_drive_37xx(struct ivpu_device *vdev, bool enable)
diff --git a/drivers/accel/ivpu/ivpu_ipc.c b/drivers/accel/ivpu/ivpu_ipc.c
index 8e960293b77a84..79d34f9b1f0747 100644
--- a/drivers/accel/ivpu/ivpu_ipc.c
+++ b/drivers/accel/ivpu/ivpu_ipc.c
@@ -509,7 +509,7 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
 	if (!ipc->mem_rx) {
 		ivpu_err(vdev, "Failed to allocate mem_rx\n");
 		ret = -ENOMEM;
-		goto err_free_tx;
+		goto err_free_bo;
 	}
 
 	ipc->mm_tx = devm_gen_pool_create(vdev->drm.dev, __ffs(IVPU_IPC_ALIGNMENT),
@@ -517,13 +517,13 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
 	if (IS_ERR(ipc->mm_tx)) {
 		ret = PTR_ERR(ipc->mm_tx);
 		ivpu_err(vdev, "Failed to create gen pool, %pe\n", ipc->mm_tx);
-		goto err_free_rx;
+		goto err_free_bo;
 	}
 
 	ret = gen_pool_add(ipc->mm_tx, ipc->mem_tx->vpu_addr, ivpu_bo_size(ipc->mem_tx), -1);
 	if (ret) {
 		ivpu_err(vdev, "gen_pool_add failed, ret %d\n", ret);
-		goto err_free_rx;
+		goto err_free_bo;
 	}
 
 	spin_lock_init(&ipc->cons_lock);
@@ -532,14 +532,13 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
 	ret = drmm_mutex_init(&vdev->drm, &ipc->lock);
 	if (ret) {
 		ivpu_err(vdev, "Failed to initialize ipc->lock, ret %d\n", ret);
-		goto err_free_rx;
+		goto err_free_bo;
 	}
 	ivpu_ipc_reset(vdev);
 	return 0;
 
-err_free_rx:
+err_free_bo:
 	ivpu_bo_free(ipc->mem_rx);
-err_free_tx:
 	ivpu_bo_free(ipc->mem_tx);
 err_destroy_cache:
 	kmem_cache_destroy(ipc->rx_msg_cache);
diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c
index ebb2c865b09a08..b9b6efea43e800 100644
--- a/drivers/accel/ivpu/ivpu_job.c
+++ b/drivers/accel/ivpu/ivpu_job.c
@@ -64,10 +64,8 @@ static int ivpu_preemption_buffers_create(struct ivpu_device *vdev,
 static void ivpu_preemption_buffers_free(struct ivpu_device *vdev,
 					 struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 {
-	if (cmdq->primary_preempt_buf)
-		ivpu_bo_free(cmdq->primary_preempt_buf);
-	if (cmdq->secondary_preempt_buf)
-		ivpu_bo_free(cmdq->secondary_preempt_buf);
+	ivpu_bo_free(cmdq->primary_preempt_buf);
+	ivpu_bo_free(cmdq->secondary_preempt_buf);
 }
 
 static int ivpu_preemption_job_init(struct ivpu_device *vdev, struct ivpu_file_priv *file_priv,
@@ -323,6 +321,11 @@ static inline u8 ivpu_job_to_jsm_priority(u8 priority)
 	return priority - 1;
 }
 
+static inline u8 ivpu_jsm_to_job_priority(u8 priority)
+{
+	return priority + 1;
+}
+
 static void ivpu_cmdq_destroy(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 {
 	lockdep_assert_held(&file_priv->lock);
@@ -621,7 +624,6 @@ bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_
 		 * status and ensure both are handled in the same way
 		 */
 		job->file_priv->has_mmu_faults = true;
-		atomic_set(&vdev->faults_detected, 1);
 		queue_work(system_percpu_wq, &vdev->context_abort_work);
 		return true;
 	}
@@ -1127,6 +1129,86 @@ int ivpu_cmdq_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file
 	return ret;
 }
 
+static int ivpu_cmdq_priority_args_check(struct ivpu_device *vdev, u32 cmdq_id, u64 priority)
+{
+	if (!ivpu_is_capable(vdev, DRM_IVPU_CAP_CMDQ_SET_PRIORITY)) {
+		ivpu_dbg(vdev, IOCTL, "Command queue priority not supported\n");
+		return -ENODEV;
+	}
+
+	if (cmdq_id < IVPU_CMDQ_MIN_ID || cmdq_id > IVPU_CMDQ_MAX_ID) {
+		ivpu_dbg(vdev, IOCTL, "Invalid command queue ID %u\n", cmdq_id);
+		return -EINVAL;
+	}
+
+	if (priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
+		return -EINVAL;
+
+	return 0;
+}
+
+int ivpu_cmdq_get_priority(struct ivpu_file_priv *file_priv, u32 cmdq_id, u64 *priority)
+{
+	struct ivpu_device *vdev = file_priv->vdev;
+	struct ivpu_cmdq *cmdq;
+	int ret;
+
+	ret = ivpu_cmdq_priority_args_check(vdev, cmdq_id, DRM_IVPU_JOB_PRIORITY_NORMAL);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&file_priv->lock);
+
+	cmdq = ivpu_cmdq_acquire(file_priv, cmdq_id);
+	if (!cmdq || cmdq->is_legacy)
+		return -EINVAL;
+
+	*priority = ivpu_jsm_to_job_priority(cmdq->priority);
+	return 0;
+}
+
+int ivpu_cmdq_set_priority(struct ivpu_file_priv *file_priv, u32 cmdq_id, u64 priority)
+{
+	struct ivpu_device *vdev = file_priv->vdev;
+	struct ivpu_cmdq *cmdq;
+	u8 jsm_priority;
+	int ret = 0;
+
+	ret = ivpu_cmdq_priority_args_check(vdev, cmdq_id, priority);
+	if (ret)
+		return ret;
+
+	ret = ivpu_rpm_get(vdev);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&file_priv->lock);
+
+	cmdq = ivpu_cmdq_acquire(file_priv, cmdq_id);
+	if (!cmdq || cmdq->is_legacy) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	jsm_priority = ivpu_job_to_jsm_priority(priority);
+	if (cmdq->priority == jsm_priority)
+		goto out;
+
+	if (cmdq->db_id) {
+		ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq_id,
+								jsm_priority);
+		if (ret)
+			goto out;
+	}
+
+	cmdq->priority = jsm_priority;
+
+out:
+	mutex_unlock(&file_priv->lock);
+	ivpu_rpm_put(vdev);
+	return ret;
+}
+
 static void
 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
 		       struct vpu_jsm_msg *jsm_msg)
@@ -1175,10 +1257,10 @@ static int reset_engine_and_mark_faulty_contexts(struct ivpu_device *vdev)
 		return ret;
 
 	/*
-	 * If faults are detected, ignore guilty contexts from engine reset as NPU may not be stuck
-	 * and could return currently running good context and faulty contexts are already marked
+	 * If job timeout is detected, read guilty context from engine reset, for other reasons
+	 * faulty context is already known
 	 */
-	if (atomic_cmpxchg(&vdev->faults_detected, 1, 0) == 1)
+	if (atomic_cmpxchg(&vdev->job_timeout_detected, 1, 0) == 0)
 		return 0;
 
 	num_impacted_contexts = resp.payload.engine_reset_done.num_impacted_contexts;
@@ -1216,9 +1298,12 @@ void ivpu_context_abort_work_fn(struct work_struct *work)
 	unsigned long ctx_id;
 	unsigned long id;
 
-	if (drm_WARN_ON(&vdev->drm, pm_runtime_get_if_active(vdev->drm.dev) <= 0))
+	if (!down_read_trylock(&vdev->pm->reset_lock))
 		return;
 
+	if (drm_WARN_ON(&vdev->drm, pm_runtime_get_if_active(vdev->drm.dev) <= 0))
+		goto unlock;
+
 	if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW)
 		if (reset_engine_and_mark_faulty_contexts(vdev))
 			goto runtime_put;
@@ -1259,4 +1344,6 @@ void ivpu_context_abort_work_fn(struct work_struct *work)
 
 runtime_put:
 	pm_runtime_put_autosuspend(vdev->drm.dev);
+unlock:
+	up_read(&vdev->pm->reset_lock);
 }
diff --git a/drivers/accel/ivpu/ivpu_job.h b/drivers/accel/ivpu/ivpu_job.h
index d8dbce82447a1e..6e32fea1eb62c8 100644
--- a/drivers/accel/ivpu/ivpu_job.h
+++ b/drivers/accel/ivpu/ivpu_job.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #ifndef __IVPU_JOB_H__
@@ -81,6 +81,9 @@ int ivpu_cmdq_create_ioctl(struct drm_device *dev, void *data, struct drm_file *
 int ivpu_cmdq_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
 int ivpu_cmdq_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
 
+int ivpu_cmdq_get_priority(struct ivpu_file_priv *file_priv, u32 cmdq_id, u64 *priority);
+int ivpu_cmdq_set_priority(struct ivpu_file_priv *file_priv, u32 cmdq_id, u64 priority);
+
 void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv);
 
 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv);
diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c
index 41efd8985fa67f..c846499d96ce0e 100644
--- a/drivers/accel/ivpu/ivpu_mmu.c
+++ b/drivers/accel/ivpu/ivpu_mmu.c
@@ -220,9 +220,6 @@
 #define IVPU_MMU_STE_1_S1DSS		GENMASK_ULL(1, 0)
 #define IVPU_MMU_STE_1_S1DSS_TERMINATE	0x0
 
-#define IVPU_MMU_REG_TIMEOUT_US		(10 * USEC_PER_MSEC)
-#define IVPU_MMU_QUEUE_TIMEOUT_US	(100 * USEC_PER_MSEC)
-
 #define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \
 				  (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \
 				  (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \
@@ -438,14 +435,14 @@ static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val)
 {
 	REGV_WR32(IVPU_MMU_REG_CR0, val);
 
-	return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
+	return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, vdev->timeout.mmu_reg);
 }
 
 static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val)
 {
 	REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val);
 
-	return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
+	return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, vdev->timeout.mmu_reg);
 }
 
 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)
@@ -465,8 +462,7 @@ static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)
 	struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
 	int ret;
 
-	ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod,
-			    IVPU_MMU_QUEUE_TIMEOUT_US);
+	ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod, vdev->timeout.mmu_queue);
 	if (ret)
 		return ret;
 
@@ -880,7 +876,7 @@ static int ivpu_mmu_evtq_set(struct ivpu_device *vdev, bool enable)
 		val = REG_CLR_FLD(IVPU_MMU_REG_CR0, EVTQEN, val);
 	REGV_WR32(IVPU_MMU_REG_CR0, val);
 
-	return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);
+	return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, vdev->timeout.mmu_reg);
 }
 
 static int ivpu_mmu_evtq_enable(struct ivpu_device *vdev)
@@ -964,7 +960,6 @@ void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
 		file_priv = xa_load(&vdev->context_xa, ssid);
 		if (file_priv) {
 			if (!READ_ONCE(file_priv->has_mmu_faults)) {
-				atomic_set(&vdev->faults_detected, 1);
 				ivpu_mmu_dump_event(vdev, event);
 				WRITE_ONCE(file_priv->has_mmu_faults, true);
 			}
diff --git a/drivers/accel/ivpu/ivpu_ms.c b/drivers/accel/ivpu/ivpu_ms.c
index cd176e77b9a01e..489ab51d33410e 100644
--- a/drivers/accel/ivpu/ivpu_ms.c
+++ b/drivers/accel/ivpu/ivpu_ms.c
@@ -69,7 +69,7 @@ int ivpu_ms_start_ioctl(struct drm_device *dev, void *data, struct drm_file *fil
 
 	ret = ivpu_jsm_metric_streamer_info(vdev, ms->mask, 0, 0, &sample_size, NULL);
 	if (ret)
-		goto err_free_ms;
+		goto err_free;
 
 	buf_size = PAGE_ALIGN((u64)args->read_period_samples * sample_size *
 			      MS_READ_PERIOD_MULTIPLIER * MS_NUM_BUFFERS);
@@ -77,14 +77,14 @@ int ivpu_ms_start_ioctl(struct drm_device *dev, void *data, struct drm_file *fil
 		ivpu_dbg(vdev, IOCTL, "Requested MS buffer size %llu exceeds range size %llu\n",
 			 buf_size, ivpu_hw_range_size(&vdev->hw->ranges.global));
 		ret = -EINVAL;
-		goto err_free_ms;
+		goto err_free;
 	}
 
 	ms->bo = ivpu_bo_create_global(vdev, buf_size, DRM_IVPU_BO_CACHED | DRM_IVPU_BO_MAPPABLE);
 	if (!ms->bo) {
 		ivpu_dbg(vdev, IOCTL, "Failed to allocate MS buffer (size %llu)\n", buf_size);
 		ret = -ENOMEM;
-		goto err_free_ms;
+		goto err_free;
 	}
 
 	ms->buff_size = ivpu_bo_size(ms->bo) / MS_NUM_BUFFERS;
@@ -96,16 +96,15 @@ int ivpu_ms_start_ioctl(struct drm_device *dev, void *data, struct drm_file *fil
 	ret = ivpu_jsm_metric_streamer_start(vdev, ms->mask, args->sampling_period_ns,
 					     ms->active_buff_vpu_addr, ms->buff_size);
 	if (ret)
-		goto err_free_bo;
+		goto err_free;
 
 	args->sample_size = sample_size;
 	args->max_data_size = ivpu_bo_size(ms->bo);
 	list_add_tail(&ms->ms_instance_node, &file_priv->ms_instance_list);
 	goto unlock;
 
-err_free_bo:
+err_free:
 	ivpu_bo_free(ms->bo);
-err_free_ms:
 	kfree(ms);
 unlock:
 	mutex_unlock(&file_priv->ms_lock);
@@ -322,10 +321,8 @@ void ivpu_ms_cleanup(struct ivpu_file_priv *file_priv)
 
 	mutex_lock(&file_priv->ms_lock);
 
-	if (file_priv->ms_info_bo) {
-		ivpu_bo_free(file_priv->ms_info_bo);
-		file_priv->ms_info_bo = NULL;
-	}
+	ivpu_bo_free(file_priv->ms_info_bo);
+	file_priv->ms_info_bo = NULL;
 
 	list_for_each_entry_safe(ms, tmp, &file_priv->ms_instance_list, ms_instance_node)
 		free_instance(file_priv, ms);
diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c
index c1ce8329790e00..de0becbfdffb2a 100644
--- a/drivers/accel/ivpu/ivpu_pm.c
+++ b/drivers/accel/ivpu/ivpu_pm.c
@@ -229,6 +229,7 @@ static void ivpu_job_timeout_work(struct work_struct *work)
 
 	ivpu_jsm_state_dump(vdev);
 	ivpu_dev_coredump(vdev);
+	atomic_set(&vdev->job_timeout_detected, 1);
 	queue_work(system_percpu_wq, &vdev->context_abort_work);
 }
 
diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c
index ef0351b6dc9b5d..4d55531bf1c9e9 100644
--- a/drivers/accel/qaic/qaic_data.c
+++ b/drivers/accel/qaic/qaic_data.c
@@ -15,6 +15,7 @@
 #include <linux/math64.h>
 #include <linux/mm.h>
 #include <linux/moduleparam.h>
+#include <linux/overflow.h>
 #include <linux/scatterlist.h>
 #include <linux/spinlock.h>
 #include <linux/srcu.h>
@@ -455,7 +456,12 @@ static int create_sgt(struct qaic_device *qdev, struct sg_table **sgt_out, u64 s
 	int order;
 
 	if (size) {
-		nr_pages = DIV_ROUND_UP(size, PAGE_SIZE);
+		u64 nr_pages_u64 = DIV_ROUND_UP_ULL(size, PAGE_SIZE);
+
+		if (overflows_type(nr_pages_u64, int))
+			return -E2BIG;
+
+		nr_pages = nr_pages_u64;
 		/*
 		 * calculate how much extra we are going to allocate, to remove
 		 * later
diff --git a/drivers/accel/qaic/qaic_debugfs.c b/drivers/accel/qaic/qaic_debugfs.c
index 95c78e12dd61b6..909a4d774d906f 100644
--- a/drivers/accel/qaic/qaic_debugfs.c
+++ b/drivers/accel/qaic/qaic_debugfs.c
@@ -5,6 +5,7 @@
 
 #include <linux/debugfs.h>
 #include <linux/device.h>
+#include <linux/device-id/mhi.h>
 #include <linux/fs.h>
 #include <linux/list.h>
 #include <linux/mhi.h>
diff --git a/drivers/accel/qaic/qaic_drv.c b/drivers/accel/qaic/qaic_drv.c
index 0acf6cdd44f814..c1bb6db78d1299 100644
--- a/drivers/accel/qaic/qaic_drv.c
+++ b/drivers/accel/qaic/qaic_drv.c
@@ -4,6 +4,7 @@
 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/delay.h>
+#include <linux/device-id/mhi.h>
 #include <linux/dma-mapping.h>
 #include <linux/idr.h>
 #include <linux/interrupt.h>
diff --git a/drivers/accel/qaic/qaic_ras.c b/drivers/accel/qaic/qaic_ras.c
index e0205c2cda3dfc..236e520f708f71 100644
--- a/drivers/accel/qaic/qaic_ras.c
+++ b/drivers/accel/qaic/qaic_ras.c
@@ -5,6 +5,7 @@
 
 #include <asm/byteorder.h>
 #include <linux/device.h>
+#include <linux/device-id/mhi.h>
 #include <linux/kernel.h>
 #include <linux/mhi.h>
 
diff --git a/drivers/accel/qaic/qaic_ssr.c b/drivers/accel/qaic/qaic_ssr.c
index a989286549599a..418f14936f9a00 100644
--- a/drivers/accel/qaic/qaic_ssr.c
+++ b/drivers/accel/qaic/qaic_ssr.c
@@ -8,6 +8,7 @@
 #include <drm/drm_managed.h>
 #include <linux/devcoredump.h>
 #include <linux/device.h>
+#include <linux/device-id/mhi.h>
 #include <linux/kernel.h>
 #include <linux/mhi.h>
 #include <linux/workqueue.h>
diff --git a/drivers/accel/qaic/qaic_timesync.c b/drivers/accel/qaic/qaic_timesync.c
index 45e5f0728ebe84..b5162677ccc463 100644
--- a/drivers/accel/qaic/qaic_timesync.c
+++ b/drivers/accel/qaic/qaic_timesync.c
@@ -2,6 +2,7 @@
 
 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
+#include <linux/device-id/mhi.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/math64.h>
diff --git a/drivers/accel/qaic/sahara.c b/drivers/accel/qaic/sahara.c
index c7c0b3eb4b65fb..8476be9b9148e1 100644
--- a/drivers/accel/qaic/sahara.c
+++ b/drivers/accel/qaic/sahara.c
@@ -3,6 +3,7 @@
 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/devcoredump.h>
+#include <linux/device-id/mhi.h>
 #include <linux/firmware.h>
 #include <linux/limits.h>
 #include <linux/mhi.h>
diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c
index e7366fcb76eeb8..9fc667d6776c83 100644
--- a/drivers/acpi/acpi_apd.c
+++ b/drivers/acpi/acpi_apd.c
@@ -256,6 +256,7 @@ static const struct acpi_device_id acpi_apd_device_ids[] = {
 	{ "AMDI0015", APD_ADDR(wt_i3c_desc) },
 	{ "AMDI0019", APD_ADDR(wt_i2c_desc) },
 	{ "AMDI0020", APD_ADDR(cz_uart_desc) },
+	{ "AMDI0021", APD_ADDR(cz_uart_desc) },
 	{ "AMDI0022", APD_ADDR(cz_uart_desc) },
 	{ "HYGO0010", APD_ADDR(wt_i2c_desc) },
 #endif
diff --git a/drivers/acpi/acpi_configfs.c b/drivers/acpi/acpi_configfs.c
index 12ffec79580311..6071699c716559 100644
--- a/drivers/acpi/acpi_configfs.c
+++ b/drivers/acpi/acpi_configfs.c
@@ -91,7 +91,7 @@ static ssize_t acpi_table_aml_read(struct config_item *cfg,
 
 CONFIGFS_BIN_ATTR(acpi_table_, aml, NULL, MAX_ACPI_TABLE_SIZE);
 
-static struct configfs_bin_attribute *acpi_table_bin_attrs[] = {
+static const struct configfs_bin_attribute *const acpi_table_bin_attrs[] = {
 	&acpi_table_attr_aml,
 	NULL,
 };
diff --git a/drivers/acpi/acpi_extlog.c b/drivers/acpi/acpi_extlog.c
index 7ad3b36013cc61..9e61354a807b33 100644
--- a/drivers/acpi/acpi_extlog.c
+++ b/drivers/acpi/acpi_extlog.c
@@ -134,22 +134,42 @@ static int print_extlog_rcd(const char *pfx,
 }
 
 static void extlog_print_pcie(struct cper_sec_pcie *pcie_err,
-			      int severity)
+			      int severity, u32 len)
 {
-#ifdef ACPI_APEI_PCIEAER
-	struct aer_capability_regs *aer;
+#ifdef CONFIG_ACPI_APEI_PCIEAER
+	struct aer_capability_regs aer_regs = {};
 	struct pci_dev *pdev;
 	unsigned int devfn;
 	unsigned int bus;
 	int aer_severity;
 	int domain;
 
+	if (len < sizeof(*pcie_err)) {
+		pr_warn_ratelimited(FW_WARN
+				    "PCIe error section too small (%u)\n", len);
+		return;
+	}
+
 	if (!(pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID &&
 	      pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO))
 		return;
 
 	aer_severity = cper_severity_to_aer(severity);
-	aer = (struct aer_capability_regs *)pcie_err->aer_info;
+
+	/*
+	 * struct pcie_tlp_log is larger than the hardware layout, so aer_info
+	 * only maps onto the struct up to the four Header Log DWORDs. Copy that
+	 * much, then place the TLP Prefix Log from where the hardware keeps it.
+	 * Everything else stays zero: nothing reads root_command, root_status or
+	 * the error source IDs, and header_len and flit are software-only.
+	 */
+	memcpy(&aer_regs, pcie_err->aer_info,
+	       offsetof(struct aer_capability_regs, header_log) +
+	       PCIE_STD_NUM_TLP_HEADERLOG * sizeof(u32));
+	memcpy(aer_regs.header_log.prefix,
+	       pcie_err->aer_info + PCI_ERR_PREFIX_LOG,
+	       sizeof(aer_regs.header_log.prefix));
+
 	domain = pcie_err->device_id.segment;
 	bus = pcie_err->device_id.bus;
 	devfn = PCI_DEVFN(pcie_err->device_id.device,
@@ -158,28 +178,11 @@ static void extlog_print_pcie(struct cper_sec_pcie *pcie_err,
 	if (!pdev)
 		return;
 
-	pci_print_aer(pdev, aer_severity, aer);
+	pci_print_aer(pdev, aer_severity, &aer_regs);
 	pci_dev_put(pdev);
 #endif
 }
 
-static void
-extlog_cxl_cper_handle_prot_err(struct cxl_cper_sec_prot_err *prot_err,
-				int severity)
-{
-#ifdef ACPI_APEI_PCIEAER
-	struct cxl_cper_prot_err_work_data wd;
-
-	if (cxl_cper_sec_prot_err_valid(prot_err))
-		return;
-
-	if (cxl_cper_setup_prot_err_work_data(&wd, prot_err, severity))
-		return;
-
-	cxl_cper_handle_prot_err(&wd);
-#endif
-}
-
 static int extlog_print(struct notifier_block *nb, unsigned long val,
 			void *data)
 {
@@ -208,6 +211,15 @@ static int extlog_print(struct notifier_block *nb, unsigned long val,
 
 	tmp = (struct acpi_hest_generic_status *)elog_buf;
 
+	/*
+	 * Bound the length before cper_estatus_check() walks the sections: it
+	 * iterates over data_length, which is not yet known to fit elog_buf.
+	 * cper_estatus_check_header() then rejects a length that wrapped, which
+	 * the bound cannot see.
+	 */
+	if (cper_estatus_len(tmp) > ELOG_ENTRY_LEN || cper_estatus_check(tmp))
+		return NOTIFY_DONE;
+
 	if (!ras_userspace_consumers()) {
 		print_extlog_rcd(NULL, tmp, cpu);
 		goto out;
@@ -235,12 +247,14 @@ static int extlog_print(struct notifier_block *nb, unsigned long val,
 			struct cxl_cper_sec_prot_err *prot_err =
 				acpi_hest_get_payload(gdata);
 
-			extlog_cxl_cper_handle_prot_err(prot_err,
-							gdata->error_severity);
+			cxl_cper_post_prot_err(prot_err,
+					       gdata->error_severity,
+					       gdata->error_data_length);
 		} else if (guid_equal(sec_type, &CPER_SEC_PCIE)) {
 			struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata);
 
-			extlog_print_pcie(pcie_err, gdata->error_severity);
+			extlog_print_pcie(pcie_err, gdata->error_severity,
+					  gdata->error_data_length);
 		} else {
 			void *err = acpi_hest_get_payload(gdata);
 
diff --git a/drivers/acpi/acpi_mrrm.c b/drivers/acpi/acpi_mrrm.c
index e99cbda8345211..83934e2aa0d4ab 100644
--- a/drivers/acpi/acpi_mrrm.c
+++ b/drivers/acpi/acpi_mrrm.c
@@ -36,17 +36,11 @@ static u32 mrrm_mem_entry_num;
 
 static int get_node_num(struct mrrm_mem_range_entry *e)
 {
-	unsigned int nid;
+	struct zone *zone;
 
-	for_each_online_node(nid) {
-		for (int z = 0; z < MAX_NR_ZONES; z++) {
-			struct zone *zone = NODE_DATA(nid)->node_zones + z;
-
-			if (!populated_zone(zone))
-				continue;
-			if (zone_intersects(zone, PHYS_PFN(e->base), PHYS_PFN(e->length)))
-				return zone_to_nid(zone);
-		}
+	for_each_populated_zone(zone) {
+		if (zone_intersects(zone, PHYS_PFN(e->base), PHYS_PFN(e->length)))
+			return zone_to_nid(zone);
 	}
 
 	return -ENOENT;
diff --git a/drivers/acpi/acpi_pcc.c b/drivers/acpi/acpi_pcc.c
index 438c6718951128..345f233d77cdcf 100644
--- a/drivers/acpi/acpi_pcc.c
+++ b/drivers/acpi/acpi_pcc.c
@@ -28,6 +28,7 @@
  * to PCC commands
  */
 #define PCC_CMD_WAIT_RETRIES_NUM	500ULL
+#define PCC_SIGNATURE_SIZE		sizeof(u32)
 
 struct pcc_data {
 	struct pcc_mbox_chan *pcc_chan;
@@ -49,10 +50,24 @@ static acpi_status
 acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function,
 			     void *handler_context,  void **region_context)
 {
-	struct pcc_data *data;
 	struct acpi_pcc_info *ctx = handler_context;
 	struct pcc_mbox_chan *pcc_chan;
+	struct pcc_data *data;
 	acpi_status ret;
+	u64 usecs_lat;
+
+	if (function == ACPI_REGION_DEACTIVATE) {
+		data = *region_context;
+		if (data) {
+			pcc_mbox_free_channel(data->pcc_chan);
+			kfree(data);
+			*region_context = NULL;
+		}
+		return AE_OK;
+	}
+
+	if (function != ACPI_REGION_ACTIVATE)
+		return AE_BAD_PARAMETER;
 
 	data = kzalloc_obj(*data);
 	if (!data)
@@ -74,6 +89,14 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function,
 	}
 
 	pcc_chan = data->pcc_chan;
+	if (pcc_chan->shmem_size < PCC_SIGNATURE_SIZE ||
+	    ctx->length > pcc_chan->shmem_size - PCC_SIGNATURE_SIZE) {
+		pr_err("PCC channel-%d shared memory is too small.\n",
+		       ctx->subspace_id);
+		ret = AE_AML_REGION_LIMIT;
+		goto err_free_channel;
+	}
+
 	if (!pcc_chan->mchan->mbox->txdone_irq) {
 		pr_err("This channel-%d does not support interrupt.\n",
 		       ctx->subspace_id);
@@ -81,6 +104,16 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function,
 		goto err_free_channel;
 	}
 
+	/*
+	 * pcc_chan->latency is just a Nominal value. In reality the remote
+	 * processor could be much slower to reply. So add an arbitrary
+	 * amount of wait on top of Nominal.
+	 */
+	usecs_lat = PCC_CMD_WAIT_RETRIES_NUM * pcc_chan->latency;
+	data->cl.tx_tout = DIV_ROUND_UP_ULL(usecs_lat, 1000);
+	if (!data->cl.tx_tout)
+		data->cl.tx_tout = 1;
+
 	*region_context = data;
 	return AE_OK;
 
@@ -97,27 +130,23 @@ acpi_pcc_address_space_handler(u32 function, acpi_physical_address addr,
 			       u32 bits, acpi_integer *value,
 			       void *handler_context, void *region_context)
 {
-	int ret;
 	struct pcc_data *data = region_context;
-	u64 usecs_lat;
+	void __iomem *pcc_opregion;
+	int ret;
+
+	pcc_opregion = data->pcc_chan->shmem + PCC_SIGNATURE_SIZE;
 
 	reinit_completion(&data->done);
 
-	/* Write to Shared Memory */
-	memcpy_toio(data->pcc_chan->shmem, (void *)value, data->ctx.length);
+	/* Write to the PCC OperationRegion after the shared memory signature. */
+	memcpy_toio(pcc_opregion, (void *)value, data->ctx.length);
 
 	ret = mbox_send_message(data->pcc_chan->mchan, NULL);
 	if (ret < 0)
 		return AE_ERROR;
 
-	/*
-	 * pcc_chan->latency is just a Nominal value. In reality the remote
-	 * processor could be much slower to reply. So add an arbitrary
-	 * amount of wait on top of Nominal.
-	 */
-	usecs_lat = PCC_CMD_WAIT_RETRIES_NUM * data->pcc_chan->latency;
 	ret = wait_for_completion_timeout(&data->done,
-						usecs_to_jiffies(usecs_lat));
+					  msecs_to_jiffies(data->cl.tx_tout));
 	if (ret == 0) {
 		pr_err("PCC command executed timeout!\n");
 		return AE_TIME;
@@ -125,7 +154,7 @@ acpi_pcc_address_space_handler(u32 function, acpi_physical_address addr,
 
 	mbox_chan_txdone(data->pcc_chan->mchan, ret);
 
-	memcpy_fromio(value, data->pcc_chan->shmem, data->ctx.length);
+	memcpy_fromio(value, pcc_opregion, data->ctx.length);
 
 	return AE_OK;
 }
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index 4d6fd9f6e9ada7..2a2822516665a5 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -1700,9 +1700,9 @@ static int acpi_video_resume(struct notifier_block *nb,
 
 static void acpi_video_dev_register_backlight(struct acpi_video_device *device)
 {
+	struct device *phys_dev, *parent = NULL;
 	struct backlight_properties props;
 	struct pci_dev *pdev;
-	struct device *parent = NULL;
 	int result;
 	static int count;
 	char *name;
@@ -1729,11 +1729,10 @@ static void acpi_video_dev_register_backlight(struct acpi_video_device *device)
 						      device,
 						      &acpi_backlight_ops,
 						      &props);
-	put_device(parent);
 	kfree(name);
 	if (IS_ERR(device->backlight)) {
 		device->backlight = NULL;
-		return;
+		goto put_parent;
 	}
 
 	/*
@@ -1743,8 +1742,12 @@ static void acpi_video_dev_register_backlight(struct acpi_video_device *device)
 	device->backlight->props.brightness =
 			acpi_video_get_brightness(device->backlight);
 
-	device->cooling_dev = thermal_cooling_device_register("LCD", device,
-							      &video_cooling_ops);
+	phys_dev = acpi_bus_get_primary_device(device->dev);
+	if (!phys_dev)
+		phys_dev = get_device(parent);
+
+	device->cooling_dev = thermal_cooling_device_create(phys_dev, "LCD", device,
+							    &video_cooling_ops);
 	if (IS_ERR(device->cooling_dev)) {
 		/*
 		 * Set cooling_dev to NULL so we don't crash trying to free it.
@@ -1753,21 +1756,17 @@ static void acpi_video_dev_register_backlight(struct acpi_video_device *device)
 		 * -- dtor
 		 */
 		device->cooling_dev = NULL;
-		return;
+		goto put_phys_dev;
 	}
 
-	dev_info(&device->dev->dev, "registered as cooling_device%d\n",
-		 device->cooling_dev->id);
-	result = sysfs_create_link(&device->dev->dev.kobj,
-			&device->cooling_dev->device.kobj,
-			"thermal_cooling");
-	if (result)
-		pr_info("sysfs link creation failed\n");
+	dev_info(&device->cooling_dev->device, "Using ACPI device %s\n",
+		acpi_dev_name(device->dev));
 
-	result = sysfs_create_link(&device->cooling_dev->device.kobj,
-			&device->dev->dev.kobj, "device");
-	if (result)
-		pr_info("Reverse sysfs link creation failed\n");
+put_phys_dev:
+	put_device(phys_dev);
+
+put_parent:
+	put_device(parent);
 }
 
 static void acpi_video_run_bcl_for_osi(struct acpi_video_bus *video)
@@ -1825,6 +1824,10 @@ static int acpi_video_bus_register_backlight(struct acpi_video_bus *video)
 
 static void acpi_video_dev_unregister_backlight(struct acpi_video_device *device)
 {
+	if (device->cooling_dev) {
+		thermal_cooling_device_unregister(device->cooling_dev);
+		device->cooling_dev = NULL;
+	}
 	if (device->backlight) {
 		backlight_device_unregister(device->backlight);
 		device->backlight = NULL;
@@ -1834,12 +1837,6 @@ static void acpi_video_dev_unregister_backlight(struct acpi_video_device *device
 		kfree(device->brightness);
 		device->brightness = NULL;
 	}
-	if (device->cooling_dev) {
-		sysfs_remove_link(&device->dev->dev.kobj, "thermal_cooling");
-		sysfs_remove_link(&device->cooling_dev->device.kobj, "device");
-		thermal_cooling_device_unregister(device->cooling_dev);
-		device->cooling_dev = NULL;
-	}
 }
 
 static int acpi_video_bus_unregister_backlight(struct acpi_video_bus *video)
diff --git a/drivers/acpi/acpica/dsfield.c b/drivers/acpi/acpica/dsfield.c
index d7d56cc600d318..f7e503bf57c87c 100644
--- a/drivers/acpi/acpica/dsfield.c
+++ b/drivers/acpi/acpica/dsfield.c
@@ -522,7 +522,8 @@ acpi_ds_create_field(union acpi_parse_object *op,
 	}
 
 	if (info.region_node->object->region.space_id ==
-	    ACPI_ADR_SPACE_PLATFORM_COMM) {
+	    ACPI_ADR_SPACE_PLATFORM_COMM &&
+	    !region_node->object->field.internal_pcc_buffer) {
 		region_node->object->field.internal_pcc_buffer =
 		    ACPI_ALLOCATE_ZEROED(info.region_node->object->region.
 					 length);
diff --git a/drivers/acpi/acpica/exfield.c b/drivers/acpi/acpica/exfield.c
index 9a55524ed8f423..50317fe0cbd5ec 100644
--- a/drivers/acpi/acpica/exfield.c
+++ b/drivers/acpi/acpica/exfield.c
@@ -45,12 +45,13 @@ static const u8 acpi_protocol_lengths[] = {
 
 /*
  * The following macros determine a given offset is a COMD field.
- * According to the specification, generic subspaces (types 0-2) contains a
- * 2-byte COMD field at offset 4 and master subspaces (type 3) contains a 4-byte
- * COMD field starting at offset 12.
+ * According to the specification, the PCC OperationRegion begins after
+ * the PCC signature. The raw shared memory COMD offsets of 4 for generic
+ * subspaces (types 0-2) and 12 for master subspaces (type 3) therefore
+ * appear at OperationRegion offsets 0 and 8.
  */
-#define GENERIC_SUBSPACE_COMMAND(a)     (4 == a || a == 5)
-#define MASTER_SUBSPACE_COMMAND(a)      (12 <= a && a <= 15)
+#define GENERIC_SUBSPACE_COMMAND(a)     (0 == a || a == 1)
+#define MASTER_SUBSPACE_COMMAND(a)      (8 <= a && a <= 11)
 
 /*******************************************************************************
  *
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index fe10ab0e02f689..08c985e729d6c7 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -642,11 +642,14 @@ static void ghes_handle_aer(struct acpi_hest_generic_data *gdata)
 #ifdef CONFIG_ACPI_APEI_PCIEAER
 	struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata);
 
+	if (gdata->error_data_length < sizeof(*pcie_err))
+		return;
+
 	if (pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID &&
 	    pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) {
+		struct aer_capability_regs *aer_info;
 		unsigned int devfn;
 		int aer_severity;
-		u8 *aer_info;
 
 		devfn = PCI_DEVFN(pcie_err->device_id.device,
 				  pcie_err->device_id.function);
@@ -664,13 +667,25 @@ static void ghes_handle_aer(struct acpi_hest_generic_data *gdata)
 						  sizeof(struct aer_capability_regs));
 		if (!aer_info)
 			return;
-		memcpy(aer_info, pcie_err->aer_info, sizeof(struct aer_capability_regs));
+
+		/*
+		 * Map aer_info onto the struct as extlog_print_pcie() does:
+		 * copy up to the four Header Log DWORDs, then place the TLP
+		 * Prefix Log from where the hardware keeps it. The rest stays
+		 * zero, so firmware cannot drive the pcie_print_tlp_log() loop
+		 * over dw[] out of bounds.
+		 */
+		memset(aer_info, 0, sizeof(struct aer_capability_regs));
+		memcpy(aer_info, pcie_err->aer_info,
+		       offsetof(struct aer_capability_regs, header_log) +
+		       PCIE_STD_NUM_TLP_HEADERLOG * sizeof(u32));
+		memcpy(aer_info->header_log.prefix,
+		       pcie_err->aer_info + PCI_ERR_PREFIX_LOG,
+		       sizeof(aer_info->header_log.prefix));
 
 		aer_recover_queue(pcie_err->device_id.segment,
 				  pcie_err->device_id.bus,
-				  devfn, aer_severity,
-				  (struct aer_capability_regs *)
-				  aer_info);
+				  devfn, aer_severity, aer_info);
 	}
 #endif
 }
@@ -752,13 +767,13 @@ static DEFINE_KFIFO(cxl_cper_prot_err_fifo, struct cxl_cper_prot_err_work_data,
 static DEFINE_RAW_SPINLOCK(cxl_cper_prot_err_work_lock);
 struct work_struct *cxl_cper_prot_err_work;
 
-static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err,
-				   int severity)
+void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err,
+			    int severity, u32 len)
 {
 #ifdef CONFIG_ACPI_APEI_PCIEAER
 	struct cxl_cper_prot_err_work_data wd;
 
-	if (cxl_cper_sec_prot_err_valid(prot_err))
+	if (cxl_cper_sec_prot_err_valid(prot_err, len))
 		return;
 
 	guard(raw_spinlock_irqsave)(&cxl_cper_prot_err_work_lock);
@@ -777,6 +792,7 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err,
 	schedule_work(cxl_cper_prot_err_work);
 #endif
 }
+EXPORT_SYMBOL_FOR_MODULES(cxl_cper_post_prot_err, "acpi_extlog");
 
 void cxl_cper_register_prot_err_work(struct work_struct *work)
 {
@@ -823,10 +839,15 @@ static DEFINE_RAW_SPINLOCK(cxl_cper_work_lock);
 struct work_struct *cxl_cper_work;
 
 static void cxl_cper_post_event(enum cxl_event_type event_type,
-				struct cxl_cper_event_rec *rec)
+				struct cxl_cper_event_rec *rec, u32 len)
 {
 	struct cxl_cper_work_data wd;
 
+	if (len < sizeof(*rec)) {
+		pr_err(FW_WARN "CXL CPER section too small (%u)\n", len);
+		return;
+	}
+
 	if (rec->hdr.length <= sizeof(rec->hdr) ||
 	    rec->hdr.length > sizeof(*rec)) {
 		pr_err(FW_WARN "CXL CPER Invalid section length (%u)\n",
@@ -923,6 +944,28 @@ static void ghes_log_hwerr(int sev, guid_t *sec_type)
 	hwerr_log_error_type(HWERR_RECOV_OTHERS);
 }
 
+/*
+ * The fields from "extended" on are absent from the 73-byte UEFI 2.1/2.2
+ * layout that older firmware still emits. Return the length needed for the
+ * fields the validation bits claim, so over-claiming is rejected without
+ * rejecting an honest short record.
+ */
+static u32 ghes_mem_err_min_len(u64 validation_bits)
+{
+	u32 len = sizeof(struct cper_sec_mem_err_old);
+
+	if (validation_bits & (CPER_MEM_VALID_ROW_EXT | CPER_MEM_VALID_CHIP_ID))
+		len = offsetof(struct cper_sec_mem_err, rank);
+	if (validation_bits & CPER_MEM_VALID_RANK_NUMBER)
+		len = offsetof(struct cper_sec_mem_err, mem_array_handle);
+	if (validation_bits & CPER_MEM_VALID_CARD_HANDLE)
+		len = offsetof(struct cper_sec_mem_err, mem_dev_handle);
+	if (validation_bits & CPER_MEM_VALID_MODULE_HANDLE)
+		len = sizeof(struct cper_sec_mem_err);
+
+	return len;
+}
+
 static void ghes_do_proc(struct ghes *ghes,
 			 const struct acpi_hest_generic_status *estatus)
 {
@@ -948,6 +991,25 @@ static void ghes_do_proc(struct ghes *ghes,
 		if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) {
 			struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata);
 
+			/*
+			 * Check once for all three consumers below. The 73-byte
+			 * UEFI 2.1/2.2 layout is the floor, matching
+			 * cper_estatus_print_section() and making
+			 * validation_bits safe to read.
+			 */
+			if (gdata->error_data_length <
+			    sizeof(struct cper_sec_mem_err_old))
+				continue;
+
+			/* Then require what the claimed fields actually need. */
+			if (gdata->error_data_length <
+			    ghes_mem_err_min_len(mem_err->validation_bits)) {
+				pr_warn_ratelimited(FW_WARN GHES_PFX
+						    "memory error section too small (%u) for the fields it claims\n",
+						    gdata->error_data_length);
+				continue;
+			}
+
 			atomic_notifier_call_chain(&ghes_report_chain, sev, mem_err);
 
 			arch_apei_report_mem_error(sev, mem_err);
@@ -959,19 +1021,23 @@ static void ghes_do_proc(struct ghes *ghes,
 		} else if (guid_equal(sec_type, &CPER_SEC_CXL_PROT_ERR)) {
 			struct cxl_cper_sec_prot_err *prot_err = acpi_hest_get_payload(gdata);
 
-			cxl_cper_post_prot_err(prot_err, gdata->error_severity);
+			cxl_cper_post_prot_err(prot_err, gdata->error_severity,
+					       gdata->error_data_length);
 		} else if (guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID)) {
 			struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata);
 
-			cxl_cper_post_event(CXL_CPER_EVENT_GEN_MEDIA, rec);
+			cxl_cper_post_event(CXL_CPER_EVENT_GEN_MEDIA, rec,
+					    gdata->error_data_length);
 		} else if (guid_equal(sec_type, &CPER_SEC_CXL_DRAM_GUID)) {
 			struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata);
 
-			cxl_cper_post_event(CXL_CPER_EVENT_DRAM, rec);
+			cxl_cper_post_event(CXL_CPER_EVENT_DRAM, rec,
+					    gdata->error_data_length);
 		} else if (guid_equal(sec_type, &CPER_SEC_CXL_MEM_MODULE_GUID)) {
 			struct cxl_cper_event_rec *rec = acpi_hest_get_payload(gdata);
 
-			cxl_cper_post_event(CXL_CPER_EVENT_MEM_MODULE, rec);
+			cxl_cper_post_event(CXL_CPER_EVENT_MEM_MODULE, rec,
+					    gdata->error_data_length);
 		} else {
 			void *err = acpi_hest_get_payload(gdata);
 
diff --git a/drivers/acpi/apei/ghes_helpers.c b/drivers/acpi/apei/ghes_helpers.c
index bc7111b740afb5..df41b993f4130a 100644
--- a/drivers/acpi/apei/ghes_helpers.c
+++ b/drivers/acpi/apei/ghes_helpers.c
@@ -5,8 +5,15 @@
 #include <linux/aer.h>
 #include <cxl/event.h>
 
-int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err)
+int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err, u32 len)
 {
+	if (len < sizeof(*prot_err)) {
+		pr_err_ratelimited(FW_WARN
+				   "CXL CPER prot err section too small (%u)\n",
+				   len);
+		return -EINVAL;
+	}
+
 	if (!(prot_err->valid_bits & PROT_ERR_VALID_AGENT_ADDRESS)) {
 		pr_err_ratelimited("CXL CPER invalid agent type\n");
 		return -EINVAL;
@@ -23,6 +30,15 @@ int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err)
 		return -EINVAL;
 	}
 
+	/* The RAS Capability block sits after a firmware-sized DVSEC. */
+	if (sizeof(*prot_err) + prot_err->dvsec_len +
+	    sizeof(struct cxl_ras_capability_regs) > len) {
+		pr_err_ratelimited(FW_WARN
+				   "CXL CPER prot err DVSEC (%u) overruns section (%u)\n",
+				   prot_err->dvsec_len, len);
+		return -EINVAL;
+	}
+
 	if ((prot_err->agent_type == RCD || prot_err->agent_type == DEVICE ||
 	     prot_err->agent_type == LD || prot_err->agent_type == FMLD) &&
 	    !(prot_err->valid_bits & PROT_ERR_VALID_SERIAL_NUMBER))
diff --git a/drivers/acpi/arm64/amba.c b/drivers/acpi/arm64/amba.c
index 1350083bce5f1f..7ee431d9464ae1 100644
--- a/drivers/acpi/arm64/amba.c
+++ b/drivers/acpi/arm64/amba.c
@@ -87,11 +87,12 @@ static int amba_handler_attach(struct acpi_device *adev,
 	 * the amba device we are about to create.
 	 */
 	if (parent)
-		dev->dev.parent = acpi_get_first_physical_node(parent);
+		dev->dev.parent = acpi_bus_get_primary_device(parent);
 
 	device_set_node(&dev->dev, acpi_fwnode_handle(adev));
 
 	ret = amba_device_add(dev, &iomem_resource);
+	put_device(dev->dev.parent);
 	if (ret) {
 		dev_err(&adev->dev, "%s(): amba_device_add() failed (%d)\n",
 		       __func__, ret);
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index 670853ec3a4d75..8599949f878675 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -341,7 +341,9 @@ static int acpi_battery_get_property(struct power_supply *psy,
 	return ret;
 }
 
-static const enum power_supply_property charge_battery_props[] = {
+/* For devices supporting the _BIX ACPI control method */
+
+static const enum power_supply_property charge_battery_extended_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
@@ -359,7 +361,7 @@ static const enum power_supply_property charge_battery_props[] = {
 	POWER_SUPPLY_PROP_SERIAL_NUMBER,
 };
 
-static const enum power_supply_property charge_battery_full_cap_broken_props[] = {
+static const enum power_supply_property charge_battery_full_cap_broken_extended_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
@@ -373,7 +375,7 @@ static const enum power_supply_property charge_battery_full_cap_broken_props[] =
 	POWER_SUPPLY_PROP_SERIAL_NUMBER,
 };
 
-static const enum power_supply_property energy_battery_props[] = {
+static const enum power_supply_property energy_battery_extended_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
@@ -391,7 +393,7 @@ static const enum power_supply_property energy_battery_props[] = {
 	POWER_SUPPLY_PROP_SERIAL_NUMBER,
 };
 
-static const enum power_supply_property energy_battery_full_cap_broken_props[] = {
+static const enum power_supply_property energy_battery_full_cap_broken_extended_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
 	POWER_SUPPLY_PROP_TECHNOLOGY,
@@ -405,6 +407,68 @@ static const enum power_supply_property energy_battery_full_cap_broken_props[] =
 	POWER_SUPPLY_PROP_SERIAL_NUMBER,
 };
 
+/* For devices supporting only the _BIF ACPI control method */
+
+static const enum power_supply_property charge_battery_props[] = {
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
+	POWER_SUPPLY_PROP_CHARGE_FULL,
+	POWER_SUPPLY_PROP_CHARGE_NOW,
+	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
+	POWER_SUPPLY_PROP_MODEL_NAME,
+	POWER_SUPPLY_PROP_MANUFACTURER,
+	POWER_SUPPLY_PROP_SERIAL_NUMBER,
+};
+
+static const enum power_supply_property charge_battery_full_cap_broken_props[] = {
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_CURRENT_NOW,
+	POWER_SUPPLY_PROP_CHARGE_NOW,
+	POWER_SUPPLY_PROP_MODEL_NAME,
+	POWER_SUPPLY_PROP_MANUFACTURER,
+	POWER_SUPPLY_PROP_SERIAL_NUMBER,
+};
+
+static const enum power_supply_property energy_battery_props[] = {
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_POWER_NOW,
+	POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN,
+	POWER_SUPPLY_PROP_ENERGY_FULL,
+	POWER_SUPPLY_PROP_ENERGY_NOW,
+	POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_CAPACITY_LEVEL,
+	POWER_SUPPLY_PROP_MODEL_NAME,
+	POWER_SUPPLY_PROP_MANUFACTURER,
+	POWER_SUPPLY_PROP_SERIAL_NUMBER,
+};
+
+static const enum power_supply_property energy_battery_full_cap_broken_props[] = {
+	POWER_SUPPLY_PROP_STATUS,
+	POWER_SUPPLY_PROP_PRESENT,
+	POWER_SUPPLY_PROP_TECHNOLOGY,
+	POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW,
+	POWER_SUPPLY_PROP_POWER_NOW,
+	POWER_SUPPLY_PROP_ENERGY_NOW,
+	POWER_SUPPLY_PROP_MODEL_NAME,
+	POWER_SUPPLY_PROP_MANUFACTURER,
+	POWER_SUPPLY_PROP_SERIAL_NUMBER,
+};
+
 /* Battery Management */
 struct acpi_offsets {
 	size_t offset;		/* offset inside struct acpi_sbs_battery */
@@ -904,6 +968,7 @@ static void __exit battery_hook_exit(void)
 
 static int sysfs_add_battery(struct acpi_battery *battery)
 {
+	bool extended_info_available = test_bit(ACPI_BATTERY_XINFO_PRESENT, &battery->flags);
 	struct power_supply_config psy_cfg = {
 		.drv_data = battery,
 		.attr_grp = acpi_battery_groups,
@@ -922,25 +987,55 @@ static int sysfs_add_battery(struct acpi_battery *battery)
 
 	if (power_unit == ACPI_BATTERY_POWER_UNIT_MA) {
 		if (full_cap_broken) {
-			battery->bat_desc.properties =
-			    charge_battery_full_cap_broken_props;
-			battery->bat_desc.num_properties =
-			    ARRAY_SIZE(charge_battery_full_cap_broken_props);
+			if (extended_info_available) {
+				battery->bat_desc.properties =
+					charge_battery_full_cap_broken_extended_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(charge_battery_full_cap_broken_extended_props);
+			} else {
+				battery->bat_desc.properties =
+					charge_battery_full_cap_broken_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(charge_battery_full_cap_broken_props);
+			}
 		} else {
-			battery->bat_desc.properties = charge_battery_props;
-			battery->bat_desc.num_properties =
-			    ARRAY_SIZE(charge_battery_props);
+			if (extended_info_available) {
+				battery->bat_desc.properties =
+					charge_battery_extended_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(charge_battery_extended_props);
+			} else {
+				battery->bat_desc.properties =
+					charge_battery_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(charge_battery_props);
+			}
 		}
 	} else {
 		if (full_cap_broken) {
-			battery->bat_desc.properties =
-			    energy_battery_full_cap_broken_props;
-			battery->bat_desc.num_properties =
-			    ARRAY_SIZE(energy_battery_full_cap_broken_props);
+			if (extended_info_available) {
+				battery->bat_desc.properties =
+					energy_battery_full_cap_broken_extended_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(energy_battery_full_cap_broken_extended_props);
+			} else {
+				battery->bat_desc.properties =
+					energy_battery_full_cap_broken_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(energy_battery_full_cap_broken_props);
+			}
 		} else {
-			battery->bat_desc.properties = energy_battery_props;
-			battery->bat_desc.num_properties =
-			    ARRAY_SIZE(energy_battery_props);
+			if (extended_info_available) {
+				battery->bat_desc.properties =
+					energy_battery_extended_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(energy_battery_extended_props);
+			} else {
+				battery->bat_desc.properties =
+					energy_battery_props;
+				battery->bat_desc.num_properties =
+					ARRAY_SIZE(energy_battery_props);
+			}
 		}
 	}
 
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index 808c6746be14dd..48f0b4809f9495 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -1120,11 +1120,6 @@ EXPORT_SYMBOL_GPL(acpi_driver_match_device);
                               ACPI Bus operations
    -------------------------------------------------------------------------- */
 
-static int acpi_bus_match(struct device *dev, const struct device_driver *drv)
-{
-	return 0;
-}
-
 static int acpi_device_uevent(const struct device *dev, struct kobj_uevent_env *env)
 {
 	return __acpi_device_uevent_modalias(to_acpi_device(dev), env);
@@ -1132,7 +1127,6 @@ static int acpi_device_uevent(const struct device *dev, struct kobj_uevent_env *
 
 const struct bus_type acpi_bus_type = {
 	.name		= "acpi",
-	.match		= acpi_bus_match,
 	.uevent		= acpi_device_uevent,
 };
 
@@ -1451,7 +1445,7 @@ static int __init acpi_bus_init(void)
 	 */
 	acpi_root_dir = proc_mkdir(ACPI_BUS_FILE_ROOT, NULL);
 
-	result = bus_register(&acpi_bus_type);
+	result = companion_bus_register(&acpi_bus_type);
 	if (!result)
 		return 0;
 
diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c
index cdbb1023a8ee4e..34c7173b5a6c97 100644
--- a/drivers/acpi/button.c
+++ b/drivers/acpi/button.c
@@ -144,6 +144,18 @@ static const struct dmi_system_id dmi_lid_quirks[] = {
 		},
 		.driver_data = (void *)(long)ACPI_BUTTON_LID_INIT_OPEN,
 	},
+	{
+		/*
+		 * Razer Blade Stealth 13 early 2020, when the lid is opened
+		 * while suspended the open notification is lost and _LID keeps
+		 * returning closed after resume, causing spurious re-suspends.
+		 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Razer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Blade Stealth 13 (Early 2020) - RZ09-0310"),
+		},
+		.driver_data = (void *)(long)ACPI_BUTTON_LID_INIT_OPEN,
+	},
 	{
 		/*
 		 * Samsung galaxybook2 ,initial _LID device notification returns
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index fef54fcd00b78f..80e2e6b32ce357 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -34,8 +34,12 @@
 #define pr_fmt(fmt)	"ACPI CPPC: " fmt
 
 #include <linux/delay.h>
+#include <linux/interval_tree_generic.h>
 #include <linux/iopoll.h>
 #include <linux/ktime.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/rbtree.h>
 #include <linux/rwsem.h>
 #include <linux/wait.h>
 #include <linux/topology.h>
@@ -70,6 +74,8 @@ struct cppc_pcc_data {
 	 *	Take write_lock for all purposes which gives exclusive access
 	 */
 	struct rw_semaphore pcc_lock;
+	/* Serialize byte-oriented accesses to aliased PCC payload fields. */
+	raw_spinlock_t payload_lock;
 
 	/* Wait queue for CPUs whose requests were batched */
 	wait_queue_head_t pcc_write_wait_q;
@@ -81,6 +87,7 @@ struct cppc_pcc_data {
 
 /* Array to represent the PCC channel per subspace ID */
 static struct cppc_pcc_data *pcc_data[MAX_PCC_SUBSPACES];
+static DEFINE_MUTEX(pcc_data_lock);
 /* The cpu_pcc_subspace_idx contains per CPU subspace ID */
 static DEFINE_PER_CPU(int, cpu_pcc_subspace_idx);
 
@@ -93,9 +100,77 @@ static DEFINE_PER_CPU(int, cpu_pcc_subspace_idx);
  */
 static DEFINE_PER_CPU(struct cpc_desc *, cpc_desc_ptr);
 
+/* Protect immutable capability queries against descriptor removal. */
+static DEFINE_MUTEX(cpc_desc_lock);
+
+static void cpc_set_desc(unsigned int cpu, struct cpc_desc *desc)
+{
+	guard(mutex)(&cpc_desc_lock);
+	per_cpu(cpc_desc_ptr, cpu) = desc;
+}
+
+struct cpc_sysmem_node {
+	struct rb_node rb;
+	u64 subtree_last;
+	u64 start;
+	u64 last;
+	struct cpc_desc *desc;
+	unsigned int reg_idx;
+	struct list_head aliases;
+	struct list_head alias_node;
+	struct cpc_sysmem_node *alias_of;
+	bool registered;
+};
+
+struct cpc_non_mmio_node {
+	struct rb_node rb;
+	u64 subtree_last;
+	u64 start;
+	u64 last;
+	struct cpc_desc *desc;
+	unsigned int reg_idx;
+	u8 space_id;
+	u8 pcc_ss_id;
+	bool registered;
+};
+
+#define CPC_SYSMEM_START(node) ((node)->start)
+#define CPC_SYSMEM_LAST(node) ((node)->last)
+
+INTERVAL_TREE_DEFINE(struct cpc_sysmem_node, rb, u64, subtree_last,
+		     CPC_SYSMEM_START, CPC_SYSMEM_LAST, static inline,
+		     cpc_sysmem_itree)
+
+static struct rb_root_cached cpc_sysmem_tree = RB_ROOT_CACHED;
+static DEFINE_MUTEX(cpc_sysmem_lock);
+
+#define CPC_NON_MMIO_START(node) ((node)->start)
+#define CPC_NON_MMIO_LAST(node) ((node)->last)
+
+INTERVAL_TREE_DEFINE(struct cpc_non_mmio_node, rb, u64, subtree_last,
+		     CPC_NON_MMIO_START, CPC_NON_MMIO_LAST, static inline,
+		     cpc_non_mmio_itree)
+
+static struct rb_root_cached cpc_pcc_trees[MAX_PCC_SUBSPACES];
+static struct rb_root_cached cpc_sysio_tree = RB_ROOT_CACHED;
+static DEFINE_MUTEX(cpc_non_mmio_lock);
+
+static struct cpc_sysmem_node *cpc_sysmem_first(u64 start, u64 last)
+{
+	return cpc_sysmem_itree_iter_first(&cpc_sysmem_tree, start, last);
+}
+
+static struct cpc_sysmem_node *cpc_sysmem_next(struct cpc_sysmem_node *node,
+					       u64 start, u64 last)
+{
+	return cpc_sysmem_itree_iter_next(node, start, last);
+}
+
+#define CPC_PCC_HEADER_SIZE	0x8
+
 /* pcc mapped address + header size + offset within PCC subspace */
 #define GET_PCC_VADDR(offs, pcc_ss_id) (pcc_data[pcc_ss_id]->pcc_channel->shmem + \
-						0x8 + (offs))
+						CPC_PCC_HEADER_SIZE + (offs))
 
 /* Check if a CPC register is in PCC */
 #define CPC_IN_PCC(cpc) ((cpc)->type == ACPI_TYPE_BUFFER &&		\
@@ -129,6 +204,28 @@ static DEFINE_PER_CPU(struct cpc_desc *, cpc_desc_ptr);
 				!!(cpc)->cpc_entry.int_value :		\
 				!IS_NULL_REG(&(cpc)->cpc_entry.reg))
 
+static bool cpc_is_writable(const struct cpc_register_resource *cpc)
+{
+	return cpc->type == ACPI_TYPE_BUFFER &&
+	       !IS_NULL_REG(&cpc->cpc_entry.reg) &&
+	       !cpc->cpc_entry.write_unsupported;
+}
+
+static bool cpc_is_readable(const struct cpc_register_resource *cpc)
+{
+	return cpc->type != ACPI_TYPE_BUFFER ||
+	       !cpc->cpc_entry.read_unsupported;
+}
+
+static bool cpc_entry_present(const struct cpc_register_resource *cpc)
+{
+	if (cpc->type == ACPI_TYPE_INTEGER)
+		return true;
+
+	return cpc->type == ACPI_TYPE_BUFFER &&
+	       !IS_NULL_REG(&cpc->cpc_entry.reg);
+}
+
 /*
  * Each bit indicates the optionality of the register in per-cpu
  * cpc_regs[] with the corresponding index. 0 means mandatory and 1
@@ -142,6 +239,36 @@ static DEFINE_PER_CPU(struct cpc_desc *, cpc_desc_ptr);
  */
 #define IS_OPTIONAL_CPC_REG(reg_idx) (REG_OPTIONAL & (1U << (reg_idx)))
 
+static bool cpc_integer_entry_valid(unsigned int reg_idx, u64 value,
+				    bool *legacy_null)
+{
+	*legacy_null = false;
+
+	switch (reg_idx) {
+	case HIGHEST_PERF:
+	case NOMINAL_PERF:
+	case LOW_NON_LINEAR_PERF:
+	case LOWEST_PERF:
+	case REFERENCE_PERF:
+	case LOWEST_FREQ:
+	case NOMINAL_FREQ:
+		return value <= U32_MAX;
+	case CTR_WRAP_TIME:
+		/* AML Integers and the kernel interface are both 64-bit. */
+		return true;
+	case AUTO_SEL_ENABLE:
+		return value <= 1;
+	case DESIRED_PERF:
+		/* Validated against Autonomous Selection after parsing. */
+		*legacy_null = value == 0;
+		return *legacy_null;
+	default:
+		/* Tolerate legacy Integer 0 placeholders for absent options. */
+		*legacy_null = value == 0 && IS_OPTIONAL_CPC_REG(reg_idx);
+		return *legacy_null;
+	}
+}
+
 /*
  * Arbitrary Retries in case the remote processor is slow to respond
  * to PCC commands. Keeping it high enough to cover emulators where
@@ -149,7 +276,8 @@ static DEFINE_PER_CPU(struct cpc_desc *, cpc_desc_ptr);
  */
 #define NUM_RETRIES 500ULL
 
-#define OVER_16BTS_MASK ~0xFFFFULL
+#define CPC_GENERIC_REGISTER_DESCRIPTOR 0x82
+#define CPC_GENERIC_REGISTER_LENGTH (sizeof(struct cpc_reg) - 3)
 
 #define define_one_cppc_ro(_name)		\
 static struct kobj_attribute _name =		\
@@ -200,15 +328,82 @@ show_cppc_data(cppc_get_perf_ctrs, cppc_perf_fb_ctrs, wraparound_time);
 	((((val) & GENMASK(((reg)->bit_width) - 1, 0)) << (reg)->bit_offset) |		\
 	((prev_val) & ~(GENMASK(((reg)->bit_width) - 1, 0) << (reg)->bit_offset)))	\
 
-static u64 cpc_sysmem_access_size(const struct cpc_register_resource *reg)
+static unsigned int cpc_reg_access_width(const struct cpc_reg *reg)
 {
-	const struct cpc_reg *gas = &reg->cpc_entry.reg;
-	unsigned int width;
-
-	if (gas->access_width > 4)
+	if (reg->access_width > 4)
 		return 0;
 
-	width = GET_BIT_WIDTH(gas);
+	if (reg->access_width)
+		return 8U << (reg->access_width - 1);
+
+	return reg->bit_width;
+}
+
+enum cpc_platform_quirk {
+	CPC_QUIRK_PERF_LIMITED_OWNS_UNIT = BIT(0),
+};
+
+static const struct acpi_platform_list cpc_platform_quirk_list[] = {
+	{
+		.oem_id = "NVIDIA",
+		.oem_table_id = "T41",
+		.table = ACPI_SIG_DSDT,
+		.pred = all_versions,
+		.reason = "Performance Limited owns its access unit",
+		.data = CPC_QUIRK_PERF_LIMITED_OWNS_UNIT,
+	},
+	{ }
+};
+
+static DEFINE_MUTEX(cpc_platform_quirk_lock);
+static bool cpc_platform_quirks_initialized;
+static u32 cpc_platform_quirks;
+
+static int cpc_get_platform_quirks(u32 *quirks)
+{
+	int idx, ret = 0;
+
+	mutex_lock(&cpc_platform_quirk_lock);
+	if (!cpc_platform_quirks_initialized) {
+		idx = acpi_match_platform_list(cpc_platform_quirk_list);
+		if (idx < 0 && idx != -ENODEV) {
+			ret = idx;
+			goto out;
+		}
+		if (idx >= 0)
+			cpc_platform_quirks = cpc_platform_quirk_list[idx].data;
+		cpc_platform_quirks_initialized = true;
+	}
+	*quirks = cpc_platform_quirks;
+out:
+	mutex_unlock(&cpc_platform_quirk_lock);
+
+	return ret;
+}
+
+static void cpc_apply_platform_quirks(struct cpc_reg *reg,
+				      unsigned int reg_idx, u32 quirks)
+{
+	unsigned int access_width;
+
+	if (!(quirks & CPC_QUIRK_PERF_LIMITED_OWNS_UNIT) ||
+	    reg_idx != PERF_LIMITED ||
+	    reg->space_id != ACPI_ADR_SPACE_SYSTEM_MEMORY ||
+	    reg->bit_width != 2 || reg->bit_offset)
+		return;
+
+	access_width = cpc_reg_access_width(reg);
+	if (access_width != 32)
+		return;
+
+	reg->bit_width = access_width;
+	pr_info_once("firmware quirk: Performance Limited owns its access unit, using Bit Width %u\n",
+		     access_width);
+}
+
+static u64 cpc_sysmem_access_size(const struct cpc_register_resource *reg)
+{
+	unsigned int width = cpc_reg_access_width(&reg->cpc_entry.reg);
 
 	if (width != 8 && width != 16 && width != 32 && width != 64)
 		return 0;
@@ -216,13 +411,35 @@ static u64 cpc_sysmem_access_size(const struct cpc_register_resource *reg)
 	return width / 8;
 }
 
+static u64 cpc_sysmem_field_size(const struct cpc_reg *gas)
+{
+	return DIV_ROUND_UP((u64)gas->bit_offset + gas->bit_width, 8);
+}
+
+static u64 cpc_sysmem_claim_size(const struct cpc_register_resource *reg)
+{
+	const struct cpc_reg *gas = &reg->cpc_entry.reg;
+	u64 access_size = cpc_sysmem_access_size(reg);
+
+	if (!gas->bit_width)
+		return access_size;
+
+	return max(access_size, cpc_sysmem_field_size(gas));
+}
+
+static bool cpc_reg_access_aligned(const struct cpc_reg *reg, u64 access_size)
+{
+	/* x86 MMIO and port-I/O accessors support unaligned addresses. */
+	return IS_ENABLED(CONFIG_X86) || IS_ALIGNED(reg->address, access_size);
+}
+
 static bool cpc_sysmem_access_units_overlap(const struct cpc_register_resource *a,
 					    const struct cpc_register_resource *b)
 {
 	const struct cpc_reg *a_gas = &a->cpc_entry.reg;
 	const struct cpc_reg *b_gas = &b->cpc_entry.reg;
-	u64 a_size = cpc_sysmem_access_size(a);
-	u64 b_size = cpc_sysmem_access_size(b);
+	u64 a_size = cpc_sysmem_claim_size(a);
+	u64 b_size = cpc_sysmem_claim_size(b);
 
 	/* Keep the conservative locking path for malformed access widths. */
 	if (!a_size || !b_size)
@@ -234,36 +451,883 @@ static bool cpc_sysmem_access_units_overlap(const struct cpc_register_resource *
 	return a_gas->address - b_gas->address < b_size;
 }
 
-static void cpc_mark_rmw_lock_users(struct cpc_desc *cpc_desc)
+static bool cpc_reg_is_writable(unsigned int reg_idx)
+{
+	/* Only controls written by this driver can be competing writers. */
+	switch (reg_idx) {
+	case DESIRED_PERF:
+	case MIN_PERF:
+	case MAX_PERF:
+	case PERF_LIMITED:
+	case ENABLE:
+	case AUTO_SEL_ENABLE:
+	case AUTO_ACT_WINDOW:
+	case ENERGY_PERF:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool cpc_reg_is_write_only(const struct cpc_desc *cpc_desc,
+				  unsigned int reg_idx)
+{
+	return cpc_desc->version >= CPPC_V4_REV &&
+	       (reg_idx == DESIRED_PERF || reg_idx == OSPM_NOMINAL_PERF);
+}
+
+static void cpc_disable_reg(struct cpc_desc *cpc_desc, unsigned int reg_idx)
+{
+	struct cpc_register_resource *reg = &cpc_desc->cpc_regs[reg_idx];
+
+	reg->type = ACPI_TYPE_INTEGER;
+	reg->cpc_entry.int_value = 0;
+}
+
+static bool cpc_optional_writer_can_be_disabled(unsigned int reg_idx)
+{
+	if (!IS_OPTIONAL_CPC_REG(reg_idx) || !cpc_reg_is_writable(reg_idx) ||
+	    reg_idx == MIN_PERF || reg_idx == MAX_PERF || reg_idx == ENABLE ||
+	    reg_idx == AUTO_SEL_ENABLE)
+		return false;
+
+	return true;
+}
+
+static bool cpc_sysmem_reg_needs_rmw(const struct cpc_register_resource *reg)
+{
+	const struct cpc_reg *gas = &reg->cpc_entry.reg;
+	u64 access_size = cpc_sysmem_access_size(reg);
+
+	return gas->bit_offset || gas->bit_width != access_size * 8;
+}
+
+static int cpc_validate_sysmem_reg(struct cpc_desc *cpc_desc,
+				   const struct cpc_reg *gas,
+				   unsigned int reg_idx)
+{
+	unsigned int access_width = cpc_reg_access_width(gas);
+	u64 access_size;
+
+	if (access_width != 8 && access_width != 16 &&
+	    access_width != 32 && access_width != 64)
+		goto invalid;
+
+	if (!gas->bit_width || gas->bit_width > access_width ||
+	    gas->bit_offset >= access_width ||
+	    gas->bit_width > access_width - gas->bit_offset)
+		goto invalid;
+
+	access_size = access_width / 8;
+	if (!gas->address || gas->address > U64_MAX - (access_size - 1))
+		goto invalid;
+	if (!cpc_reg_access_aligned(gas, access_size))
+		goto invalid;
+
+	if (reg_idx == PERF_LIMITED) {
+		if (access_width == 64 && !IS_ENABLED(CONFIG_64BIT)) {
+			pr_warn_once("CPU%d: Performance Limited register cannot be accessed atomically; keeping its range reserved\n",
+				     cpc_desc->cpu_id);
+			cpc_desc->cpc_regs[reg_idx].cpc_entry.read_unsupported = true;
+			cpc_desc->cpc_regs[reg_idx].cpc_entry.write_unsupported = true;
+			return 0;
+		}
+
+		if (gas->bit_offset || gas->bit_width != access_width) {
+			pr_warn_once("CPU%d: Performance Limited register cannot be cleared safely; keeping it readable\n",
+				     cpc_desc->cpu_id);
+			cpc_desc->cpc_regs[reg_idx].cpc_entry.write_unsupported = true;
+		}
+	}
+
+	return 0;
+
+invalid:
+	access_size = 0;
+	if (access_width == 8 || access_width == 16 ||
+	    access_width == 32 || access_width == 64)
+		access_size = access_width / 8;
+	if (gas->bit_width)
+		access_size = max(access_size, cpc_sysmem_field_size(gas));
+	if ((cpc_reg_is_write_only(cpc_desc, reg_idx) ||
+	     reg_idx == PERF_LIMITED) && gas->address && access_size &&
+	    gas->address <= U64_MAX - (access_size - 1)) {
+		struct cpc_register_resource *reg = &cpc_desc->cpc_regs[reg_idx];
+
+		if (reg_idx == PERF_LIMITED)
+			pr_warn_once("CPU%d: _CPC v%d register %u is inaccessible; keeping its range reserved\n",
+				     cpc_desc->cpu_id, cpc_desc->version, reg_idx);
+		else
+			pr_warn("CPU%d: _CPC v%d register %u is inaccessible; keeping its range reserved\n",
+				cpc_desc->cpu_id, cpc_desc->version, reg_idx);
+		reg->cpc_entry.read_unsupported = true;
+		reg->cpc_entry.write_unsupported = true;
+		return 0;
+	}
+
+	pr_debug("CPU:%d invalid SystemMemory GAS for _CPC register %u\n",
+		 cpc_desc->cpu_id, reg_idx);
+	return -EINVAL;
+}
+
+static bool cpc_immutable_autonomous(const struct cpc_desc *cpc_desc)
 {
-	int i, j;
+	const struct cpc_register_resource *reg;
+
+	reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
+	return osc_sb_cppc2_support_acked && reg->type == ACPI_TYPE_INTEGER &&
+	       reg->cpc_entry.int_value == 1;
+}
+
+static bool cpc_retain_pcc_status(struct cpc_desc *cpc_desc,
+				  unsigned int reg_idx);
+
+static int cpc_resolve_unsupported(struct cpc_desc *cpc_desc,
+				   u32 unsupported)
+{
+	unsigned int i;
+	u32 bounds = BIT(MIN_PERF) | BIT(MAX_PERF);
+	bool min_unusable, max_unusable;
+
+	if (unsupported & bounds) {
+		min_unusable = (unsupported & BIT(MIN_PERF)) ||
+			       !cpc_is_writable(&cpc_desc->cpc_regs[MIN_PERF]);
+		max_unusable = (unsupported & BIT(MAX_PERF)) ||
+			       !cpc_is_writable(&cpc_desc->cpc_regs[MAX_PERF]);
+		if (min_unusable && max_unusable) {
+			pr_warn("CPU%d: ignoring inaccessible Minimum and Maximum Performance registers\n",
+				cpc_desc->cpu_id);
+			cpc_disable_reg(cpc_desc, MIN_PERF);
+			cpc_disable_reg(cpc_desc, MAX_PERF);
+			unsupported &= ~bounds;
+		}
+	}
+
+	for (i = 0; i < cpc_desc->num_entries - 2; i++) {
+		if (!(unsupported & BIT(i)))
+			continue;
+
+		/* CPPC control does not depend on Performance Limited status. */
+		if (i == PERF_LIMITED) {
+			if (CPC_IN_PCC(&cpc_desc->cpc_regs[i]) &&
+			    cpc_retain_pcc_status(cpc_desc, i))
+				continue;
+
+			pr_warn_once("CPU%d: ignoring inaccessible Performance Limited register\n",
+				     cpc_desc->cpu_id);
+			cpc_disable_reg(cpc_desc, i);
+			continue;
+		}
+
+		if (i == DESIRED_PERF && cpc_immutable_autonomous(cpc_desc)) {
+			pr_warn("CPU%d: ignoring inaccessible Desired Performance register in autonomous mode\n",
+				cpc_desc->cpu_id);
+			cpc_disable_reg(cpc_desc, i);
+			continue;
+		}
+
+		/*
+		 * A present Enable or Autonomous Selection control must remain
+		 * usable.  Disabling the latter could leave autonomous selection
+		 * enabled while OSPM believes that it has disabled it.
+		 */
+		if (i == ENABLE ||
+		    (i == AUTO_SEL_ENABLE && cpc_entry_present(&cpc_desc->cpc_regs[i])) ||
+		    i == MIN_PERF || i == MAX_PERF ||
+		    !IS_OPTIONAL_CPC_REG(i)) {
+			pr_err("CPU%d: cannot access _CPC register %u\n",
+			       cpc_desc->cpu_id, i);
+			return -EINVAL;
+		}
+
+		pr_warn("CPU%d: ignoring inaccessible optional _CPC register %u\n",
+			cpc_desc->cpu_id, i);
+		cpc_disable_reg(cpc_desc, i);
+	}
+
+	return 0;
+}
+
+static int cpc_validate_required_controls(struct cpc_desc *cpc_desc)
+{
+	unsigned int i;
+
+	/*
+	 * Performance Limited is required by the specification, but tolerate a
+	 * NULL descriptor used by firmware which cannot report limiting events.
+	 * CPPC control does not depend on this status.
+	 */
+	for (i = 0; i < cpc_desc->num_entries - 2; i++) {
+		if (i != DESIRED_PERF && i != PERF_LIMITED &&
+		    !IS_OPTIONAL_CPC_REG(i) &&
+		    !cpc_entry_present(&cpc_desc->cpc_regs[i])) {
+			pr_debug("CPU:%d lacks mandatory _CPC register %u\n",
+				 cpc_desc->cpu_id, i);
+			return -EINVAL;
+		}
+	}
+
+	/* Desired may be absent only for immutable autonomous operation. */
+	if (!cpc_is_writable(&cpc_desc->cpc_regs[DESIRED_PERF]) &&
+	    !cpc_immutable_autonomous(cpc_desc)) {
+		pr_debug("CPU:%d lacks a writable Desired Performance register\n",
+			 cpc_desc->cpu_id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int cpc_validate_bound_controls(struct cpc_desc *cpc_desc)
+{
+	bool have_min, have_max;
+
+	have_min = cpc_is_writable(&cpc_desc->cpc_regs[MIN_PERF]);
+	have_max = cpc_is_writable(&cpc_desc->cpc_regs[MAX_PERF]);
+	if (have_min != have_max) {
+		pr_err("CPU%d: _CPC must provide both Minimum and Maximum Performance or neither\n",
+		       cpc_desc->cpu_id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static bool
+cpc_retain_pcc_status(struct cpc_desc *cpc_desc, unsigned int reg_idx)
+{
+	struct cpc_register_resource *reg = &cpc_desc->cpc_regs[reg_idx];
+	const struct cpc_reg *gas = &reg->cpc_entry.reg;
+	u64 size;
+
+	if (reg_idx != PERF_LIMITED || !gas->bit_width)
+		return false;
+
+	size = DIV_ROUND_UP((u64)gas->bit_offset + gas->bit_width, 8);
+	if (!size || gas->address > U64_MAX - (size - 1))
+		return false;
+
+	reg->cpc_entry.read_unsupported = true;
+	reg->cpc_entry.write_unsupported = true;
+	pr_warn_once("CPU%d: Performance Limited register cannot be accessed; keeping its PCC range reserved\n",
+		     cpc_desc->cpu_id);
+	return true;
+}
+
+static bool
+cpc_retain_sysio_status(struct cpc_desc *cpc_desc, unsigned int reg_idx)
+{
+	struct cpc_register_resource *reg = &cpc_desc->cpc_regs[reg_idx];
+	const struct cpc_reg *gas = &reg->cpc_entry.reg;
+
+	if (reg_idx != PERF_LIMITED || !gas->bit_width)
+		return false;
+
+	/* Retain any in-range portion for overlap validation only. */
+	if (gas->address > U16_MAX)
+		return false;
+
+	pr_warn_once("CPU%d: Performance Limited register cannot be accessed; keeping its SystemIO range reserved\n",
+		     cpc_desc->cpu_id);
+	reg->cpc_entry.read_unsupported = true;
+	reg->cpc_entry.write_unsupported = true;
+	return true;
+}
+
+static u64 cpc_non_mmio_access_size(const struct cpc_register_resource *reg)
+{
+	const struct cpc_reg *gas = &reg->cpc_entry.reg;
+
+	if (gas->space_id == ACPI_ADR_SPACE_PLATFORM_COMM)
+		return DIV_ROUND_UP((u64)gas->bit_offset + gas->bit_width, 8);
+
+	return max((u64)cpc_reg_access_width(gas) / 8,
+		   DIV_ROUND_UP((u64)gas->bit_offset + gas->bit_width, 8));
+}
+
+static void cpc_validate_pcc_bounds(struct cpc_desc *cpc_desc,
+				    int pcc_ss_id, struct cppc_pcc_data *data,
+				    u32 *unsupported)
+{
+	u64 shmem_size = data->pcc_channel->shmem_size;
+	unsigned int i;
 
 	for (i = 0; i < cpc_desc->num_entries - 2; i++) {
-		struct cpc_register_resource *a = &cpc_desc->cpc_regs[i];
+		struct cpc_register_resource *reg = &cpc_desc->cpc_regs[i];
 		struct cpc_reg *gas;
 		u64 access_size;
 
-		if (!CPC_SUPPORTED(a) || !CPC_IN_SYSTEM_MEMORY(a))
+		if ((*unsupported & BIT(i)) || !CPC_SUPPORTED(reg) ||
+		    !CPC_IN_PCC(reg))
 			continue;
 
-		gas = &a->cpc_entry.reg;
-		access_size = cpc_sysmem_access_size(a);
-		if (gas->bit_offset || !access_size ||
-		    gas->bit_width != access_size * 8)
-			a->cpc_entry.use_rmw_lock = true;
+		gas = &reg->cpc_entry.reg;
+		if (gas->access_width != pcc_ss_id)
+			continue;
+		access_size = cpc_non_mmio_access_size(reg);
+		if (shmem_size >= CPC_PCC_HEADER_SIZE &&
+		    gas->address <= shmem_size - CPC_PCC_HEADER_SIZE &&
+		    access_size <= shmem_size - CPC_PCC_HEADER_SIZE - gas->address)
+			continue;
 
-		for (j = i + 1; j < cpc_desc->num_entries - 2; j++) {
-			struct cpc_register_resource *b = &cpc_desc->cpc_regs[j];
+		pr_debug("CPU%d: _CPC register %u exceeds the PCC shared region\n",
+			 cpc_desc->cpu_id, i);
+		*unsupported |= BIT(i);
+	}
+}
 
-			if (!CPC_SUPPORTED(b) || !CPC_IN_SYSTEM_MEMORY(b))
-				continue;
-			if (!cpc_sysmem_access_units_overlap(a, b))
-				continue;
+static bool cpc_pcc_access_needed(const struct cpc_desc *cpc_desc)
+{
+	unsigned int i;
+
+	for (i = 0; i < cpc_desc->num_entries - 2; i++) {
+		const struct cpc_register_resource *reg = &cpc_desc->cpc_regs[i];
+
+		if (CPC_SUPPORTED(reg) && CPC_IN_PCC(reg) &&
+		    (cpc_is_readable(reg) || cpc_is_writable(reg)))
+			return true;
+	}
+
+	return false;
+}
+
+static bool cpc_non_mmio_overlap_conflicts(u8 space_id, bool a_writable,
+					   bool b_writable, bool a_write_only,
+					   bool b_write_only)
+{
+	/* Only a write-only control can use a separate read-side port alias. */
+	if (space_id == ACPI_ADR_SPACE_SYSTEM_IO && a_writable != b_writable)
+		return a_writable ? !a_write_only : !b_write_only;
+
+	return a_writable || b_writable;
+}
+
+static bool
+cpc_sysio_perf_limited_conflicts(unsigned int a_idx, bool a_writable,
+				 unsigned int b_idx, bool b_writable)
+{
+	return (a_idx == PERF_LIMITED && b_writable) ||
+	       (b_idx == PERF_LIMITED && a_writable);
+}
+
+static struct rb_root_cached *cpc_non_mmio_tree(u8 space_id, u8 pcc_ss_id)
+{
+	if (space_id == ACPI_ADR_SPACE_PLATFORM_COMM)
+		return &cpc_pcc_trees[pcc_ss_id];
+	if (space_id == ACPI_ADR_SPACE_SYSTEM_IO)
+		return &cpc_sysio_tree;
+	return NULL;
+}
+
+static bool cpc_same_non_mmio_register(const struct cpc_non_mmio_node *a,
+				       const struct cpc_non_mmio_node *b)
+{
+	const struct cpc_reg *a_gas =
+		&a->desc->cpc_regs[a->reg_idx].cpc_entry.reg;
+	const struct cpc_reg *b_gas =
+		&b->desc->cpc_regs[b->reg_idx].cpc_entry.reg;
+
+	return a->space_id == b->space_id && a->pcc_ss_id == b->pcc_ss_id &&
+	       a->reg_idx == b->reg_idx && a->start == b->start &&
+	       a->last == b->last && a_gas->bit_offset == b_gas->bit_offset &&
+	       a_gas->bit_width == b_gas->bit_width &&
+	       (a->space_id == ACPI_ADR_SPACE_PLATFORM_COMM ||
+		cpc_reg_access_width(a_gas) == cpc_reg_access_width(b_gas));
+}
 
-			a->cpc_entry.use_rmw_lock = true;
-			b->cpc_entry.use_rmw_lock = true;
+static int cpc_validate_non_mmio_pair(const struct cpc_non_mmio_node *a,
+				      const struct cpc_non_mmio_node *b)
+{
+	const struct cpc_register_resource *a_reg;
+	const struct cpc_register_resource *b_reg;
+	bool a_writable, b_writable;
+	const char *name;
+
+	a_reg = &a->desc->cpc_regs[a->reg_idx];
+	b_reg = &b->desc->cpc_regs[b->reg_idx];
+	a_writable = cpc_reg_is_writable(a->reg_idx) && cpc_is_writable(a_reg);
+	b_writable = cpc_reg_is_writable(b->reg_idx) && cpc_is_writable(b_reg);
+
+	if (!cpc_non_mmio_overlap_conflicts(a->space_id, a_writable,
+					    b_writable,
+					    cpc_reg_is_write_only(a->desc, a->reg_idx),
+					    cpc_reg_is_write_only(b->desc, b->reg_idx)) &&
+	    !(a->space_id == ACPI_ADR_SPACE_SYSTEM_IO &&
+	      cpc_sysio_perf_limited_conflicts(a->reg_idx, a_writable,
+					       b->reg_idx, b_writable)))
+		return 0;
+
+	if (cpc_same_non_mmio_register(a, b))
+		return 0;
+
+	name = a->space_id == ACPI_ADR_SPACE_PLATFORM_COMM ?
+	       "PCC" : "SystemIO";
+	pr_err("CPU%d: %s _CPC register %u conflicts with CPU%d register %u\n",
+	       a->desc->cpu_id, name, a->reg_idx, b->desc->cpu_id,
+	       b->reg_idx);
+	return -EINVAL;
+}
+
+static void cpc_unregister_non_mmio_desc_locked(struct cpc_desc *cpc_desc)
+{
+	unsigned int i;
+
+	if (!cpc_desc->non_mmio_nodes)
+		return;
+
+	for (i = 0; i < cpc_desc->num_entries - 2; i++) {
+		struct cpc_non_mmio_node *node = &cpc_desc->non_mmio_nodes[i];
+		struct rb_root_cached *tree;
+
+		if (!node->registered)
+			continue;
+
+		tree = cpc_non_mmio_tree(node->space_id, node->pcc_ss_id);
+		cpc_non_mmio_itree_remove(node, tree);
+	}
+
+	kfree(cpc_desc->non_mmio_nodes);
+	cpc_desc->non_mmio_nodes = NULL;
+}
+
+static int cpc_register_non_mmio_desc(struct cpc_desc *cpc_desc)
+{
+	unsigned int nr_regs = cpc_desc->num_entries - 2;
+	unsigned int i;
+	int ret = 0;
+	bool found = false;
+
+	for (i = 0; i < nr_regs; i++) {
+		struct cpc_register_resource *reg = &cpc_desc->cpc_regs[i];
+		u8 space_id;
+
+		if (!CPC_SUPPORTED(reg) || reg->type != ACPI_TYPE_BUFFER)
+			continue;
+		space_id = reg->cpc_entry.reg.space_id;
+		if (space_id == ACPI_ADR_SPACE_PLATFORM_COMM ||
+		    space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
+			found = true;
+			break;
+		}
+	}
+	if (!found)
+		return 0;
+
+	cpc_desc->non_mmio_nodes = kcalloc(nr_regs,
+					   sizeof(*cpc_desc->non_mmio_nodes),
+					   GFP_KERNEL);
+	if (!cpc_desc->non_mmio_nodes)
+		return -ENOMEM;
+
+	mutex_lock(&cpc_non_mmio_lock);
+
+	for (i = 0; i < nr_regs; i++) {
+		struct cpc_register_resource *reg = &cpc_desc->cpc_regs[i];
+		struct cpc_non_mmio_node *match, *node;
+		struct rb_root_cached *tree;
+		u8 space_id;
+		u64 size;
+
+		if (!CPC_SUPPORTED(reg) || reg->type != ACPI_TYPE_BUFFER)
+			continue;
+
+		space_id = reg->cpc_entry.reg.space_id;
+		if (space_id != ACPI_ADR_SPACE_PLATFORM_COMM &&
+		    space_id != ACPI_ADR_SPACE_SYSTEM_IO)
+			continue;
+
+		node = &cpc_desc->non_mmio_nodes[i];
+		size = cpc_non_mmio_access_size(reg);
+		node->start = reg->cpc_entry.reg.address;
+		node->last = node->start + size - 1;
+		node->desc = cpc_desc;
+		node->reg_idx = i;
+		node->space_id = space_id;
+		node->pcc_ss_id = space_id == ACPI_ADR_SPACE_PLATFORM_COMM ?
+				      reg->cpc_entry.reg.access_width : 0;
+		tree = cpc_non_mmio_tree(space_id, node->pcc_ss_id);
+
+		match = cpc_non_mmio_itree_iter_first(tree, node->start,
+						      node->last);
+		while (match) {
+			ret = cpc_validate_non_mmio_pair(node, match);
+			if (ret)
+				goto out_unregister;
+
+			match = cpc_non_mmio_itree_iter_next(match, node->start,
+							     node->last);
+		}
+		cpc_non_mmio_itree_insert(node, tree);
+		node->registered = true;
+	}
+
+	mutex_unlock(&cpc_non_mmio_lock);
+	return 0;
+
+out_unregister:
+	cpc_unregister_non_mmio_desc_locked(cpc_desc);
+	mutex_unlock(&cpc_non_mmio_lock);
+	return ret;
+}
+
+static void cpc_unregister_non_mmio_desc(struct cpc_desc *cpc_desc)
+{
+	if (!cpc_desc->non_mmio_nodes)
+		return;
+
+	mutex_lock(&cpc_non_mmio_lock);
+	cpc_unregister_non_mmio_desc_locked(cpc_desc);
+	mutex_unlock(&cpc_non_mmio_lock);
+}
+
+static void cpc_mark_rmw_lock_users(struct cpc_desc *cpc_desc)
+{
+	int i;
+
+	for (i = 0; i < cpc_desc->num_entries - 2; i++) {
+		struct cpc_register_resource *reg = &cpc_desc->cpc_regs[i];
+
+		if (CPC_SUPPORTED(reg) && CPC_IN_SYSTEM_MEMORY(reg) &&
+		    cpc_is_writable(reg))
+			reg->cpc_entry.use_rmw_lock =
+				cpc_sysmem_reg_needs_rmw(reg);
+	}
+}
+
+struct cpc_bit_position {
+	u64 byte;
+	u8 bit;
+};
+
+static bool cpc_bit_position_before(const struct cpc_bit_position *a,
+				    const struct cpc_bit_position *b)
+{
+	return a->byte < b->byte || (a->byte == b->byte && a->bit < b->bit);
+}
+
+static bool cpc_sysmem_fields_overlap(const struct cpc_register_resource *a,
+				      const struct cpc_register_resource *b)
+{
+	const struct cpc_reg *a_gas = &a->cpc_entry.reg;
+	const struct cpc_reg *b_gas = &b->cpc_entry.reg;
+	unsigned int a_last_bit = a_gas->bit_offset + a_gas->bit_width - 1;
+	unsigned int b_last_bit = b_gas->bit_offset + b_gas->bit_width - 1;
+	struct cpc_bit_position a_start = {
+		.byte = a_gas->address + a_gas->bit_offset / 8,
+		.bit = a_gas->bit_offset % 8,
+	};
+	struct cpc_bit_position a_end = {
+		.byte = a_gas->address + a_last_bit / 8,
+		.bit = a_last_bit % 8,
+	};
+	struct cpc_bit_position b_start = {
+		.byte = b_gas->address + b_gas->bit_offset / 8,
+		.bit = b_gas->bit_offset % 8,
+	};
+	struct cpc_bit_position b_end = {
+		.byte = b_gas->address + b_last_bit / 8,
+		.bit = b_last_bit % 8,
+	};
+
+	return !cpc_bit_position_before(&a_end, &b_start) &&
+	       !cpc_bit_position_before(&b_end, &a_start);
+}
+
+static bool cpc_sysmem_access_overlaps_field(const struct cpc_register_resource *access,
+					     const struct cpc_register_resource *field)
+{
+	const struct cpc_reg *access_gas = &access->cpc_entry.reg;
+	const struct cpc_reg *field_gas = &field->cpc_entry.reg;
+	u64 access_last;
+	u64 field_start;
+	u64 field_last;
+
+	if (!field_gas->bit_width)
+		return cpc_sysmem_access_units_overlap(access, field);
+
+	access_last = access_gas->address +
+		      cpc_sysmem_access_size(access) - 1;
+	field_start = field_gas->address + field_gas->bit_offset / 8;
+	field_last = field_gas->address +
+		     (field_gas->bit_offset + field_gas->bit_width - 1) / 8;
+
+	return access_gas->address <= field_last && field_start <= access_last;
+}
+
+static bool cpc_same_sysmem_register(unsigned int a_idx,
+				     const struct cpc_register_resource *a,
+				     unsigned int b_idx,
+				     const struct cpc_register_resource *b)
+{
+	const struct cpc_reg *a_gas = &a->cpc_entry.reg;
+	const struct cpc_reg *b_gas = &b->cpc_entry.reg;
+
+	return a_idx == b_idx &&
+	       a_gas->address == b_gas->address &&
+	       a_gas->bit_width == b_gas->bit_width &&
+	       a_gas->bit_offset == b_gas->bit_offset &&
+	       cpc_reg_access_width(a_gas) == cpc_reg_access_width(b_gas);
+}
+
+static int cpc_validate_sysmem_pair(const struct cpc_desc *a_desc,
+				    unsigned int a_idx,
+				    const struct cpc_desc *b_desc,
+				    unsigned int b_idx)
+{
+	const struct cpc_register_resource *a = &a_desc->cpc_regs[a_idx];
+	const struct cpc_register_resource *b = &b_desc->cpc_regs[b_idx];
+	bool a_write_only, b_write_only;
+	bool a_writable, b_writable;
+	bool fields_overlap;
+
+	/* The overlap helper includes each descriptor's conservative claim. */
+	if (!CPC_SUPPORTED(a) || !CPC_IN_SYSTEM_MEMORY(a) ||
+	    !CPC_SUPPORTED(b) || !CPC_IN_SYSTEM_MEMORY(b) ||
+	    !cpc_sysmem_access_units_overlap(a, b))
+		return 0;
+
+	a_write_only = cpc_reg_is_write_only(a_desc, a_idx);
+	b_write_only = cpc_reg_is_write_only(b_desc, b_idx);
+	fields_overlap = !a->cpc_entry.reg.bit_width ||
+			 !b->cpc_entry.reg.bit_width ||
+			 cpc_sysmem_fields_overlap(a, b);
+	/* A readable field must not expose another field's undefined bits. */
+	if (a_write_only != b_write_only &&
+	    cpc_is_readable(a_write_only ? b : a) &&
+	    fields_overlap)
+		goto conflict;
+
+	a_writable = cpc_reg_is_writable(a_idx) && cpc_is_writable(a);
+	b_writable = cpc_reg_is_writable(b_idx) && cpc_is_writable(b);
+	if (!a_writable && !b_writable)
+		return 0;
+
+	if (cpc_same_sysmem_register(a_idx, a, b_idx, b)) {
+		u64 access_size = cpc_sysmem_access_size(a);
+
+		/*
+		 * Exact partial aliases update the same field and retain
+		 * last-writer-wins semantics when the complete access is one native
+		 * transaction.  A 64-bit MMIO write may be split on 32-bit kernels,
+		 * and an unaligned x86 access is not guaranteed to be one device
+		 * transaction.
+		 */
+		if (!a_writable ||
+		    (IS_ALIGNED(a->cpc_entry.reg.address, access_size) &&
+		     (access_size < sizeof(u64) ||
+		      IS_ENABLED(CONFIG_64BIT))))
+			return 0;
+		goto conflict;
+	}
+
+	/*
+	 * The platform may set Performance Limited asynchronously.  A write to
+	 * another field in the same access unit could write back stale status
+	 * bits, which an OSPM lock cannot prevent.
+	 */
+	if ((a_idx == PERF_LIMITED && b_writable) ||
+	    (b_idx == PERF_LIMITED && a_writable))
+		goto conflict;
+
+	/* A full-width writer must not overwrite another logical field. */
+	if (fields_overlap &&
+	    ((a_writable && b_writable) ||
+	     (a_writable && !cpc_sysmem_reg_needs_rmw(a)) ||
+	     (b_writable && !cpc_sysmem_reg_needs_rmw(b))))
+		goto conflict;
+
+	/* Different descriptors do not share their partial-write locks. */
+	if (a_desc != b_desc && a_writable && b_writable)
+		goto conflict;
+
+	/*
+	 * RMW of either writer preserves the other field.  If that field is
+	 * write-only, its readback is undefined and cannot safely be replayed.
+	 */
+	if ((a_write_only && b_writable &&
+	     cpc_sysmem_reg_needs_rmw(b) &&
+	     cpc_sysmem_access_overlaps_field(b, a)) ||
+	    (b_write_only && a_writable &&
+	     cpc_sysmem_reg_needs_rmw(a) &&
+	     cpc_sysmem_access_overlaps_field(a, b)))
+		goto conflict;
+
+	return 0;
+
+conflict:
+	pr_err("CPU%d: SystemMemory _CPC register %u conflicts with CPU%d register %u\n",
+	       a_desc->cpu_id, a_idx, b_desc->cpu_id, b_idx);
+	return -EINVAL;
+}
+
+static bool cpc_disable_new_sysmem_writer(struct cpc_desc *cpc_desc,
+					  unsigned int reg_idx,
+					  const struct cpc_sysmem_node *node)
+{
+	struct cpc_register_resource *reg = &cpc_desc->cpc_regs[reg_idx];
+	struct cpc_sysmem_node *match;
+	unsigned int status_cpu = 0;
+	bool found = false;
+
+	if (!cpc_optional_writer_can_be_disabled(reg_idx) ||
+	    !cpc_is_writable(reg))
+		return false;
+
+	match = cpc_sysmem_first(node->start, node->last);
+	while (match) {
+		const struct cpc_register_resource *status;
+
+		if (match->reg_idx == PERF_LIMITED) {
+			status = &match->desc->cpc_regs[PERF_LIMITED];
+			if (!status->cpc_entry.reg.bit_width ||
+			    cpc_sysmem_fields_overlap(reg, status))
+				return false;
+			status_cpu = match->desc->cpu_id;
+			found = true;
+		}
+
+		match = cpc_sysmem_next(match, node->start, node->last);
+	}
+	if (!found)
+		return false;
+
+	pr_warn_once("CPU%d: ignoring optional _CPC register %u sharing CPU%d Performance Limited access unit\n",
+		     cpc_desc->cpu_id, reg_idx, status_cpu);
+	cpc_disable_reg(cpc_desc, reg_idx);
+	return true;
+}
+
+static void cpc_unregister_sysmem_desc_locked(struct cpc_desc *cpc_desc)
+{
+	unsigned int i;
+
+	if (!cpc_desc->sysmem_nodes)
+		return;
+
+	for (i = 0; i < cpc_desc->num_entries - 2; i++) {
+		struct cpc_sysmem_node *node = &cpc_desc->sysmem_nodes[i];
+		struct cpc_sysmem_node *alias, *child;
+
+		if (node->alias_of) {
+			list_del(&node->alias_node);
+			continue;
+		}
+		if (!node->registered)
+			continue;
+
+		cpc_sysmem_itree_remove(node, &cpc_sysmem_tree);
+		node->registered = false;
+		if (list_empty(&node->aliases))
+			continue;
+
+		/* Keep one representative for aliases owned by live descriptors. */
+		alias = list_first_entry(&node->aliases,
+					 struct cpc_sysmem_node, alias_node);
+		list_del_init(&alias->alias_node);
+		alias->alias_of = NULL;
+		alias->registered = true;
+		list_splice_init(&node->aliases, &alias->aliases);
+		list_for_each_entry(child, &alias->aliases, alias_node)
+			child->alias_of = alias;
+		cpc_sysmem_itree_insert(alias, &cpc_sysmem_tree);
+	}
+
+	kfree(cpc_desc->sysmem_nodes);
+	cpc_desc->sysmem_nodes = NULL;
+}
+
+static int cpc_register_sysmem_desc(struct cpc_desc *cpc_desc)
+{
+	unsigned int nr_regs = cpc_desc->num_entries - 2;
+	unsigned int i;
+	int ret = 0;
+	bool found = false;
+
+	for (i = 0; i < nr_regs; i++) {
+		struct cpc_register_resource *reg = &cpc_desc->cpc_regs[i];
+
+		if (CPC_SUPPORTED(reg) && CPC_IN_SYSTEM_MEMORY(reg)) {
+			found = true;
+			break;
 		}
 	}
+	if (!found)
+		return 0;
+
+	cpc_desc->sysmem_nodes = kcalloc(nr_regs,
+					 sizeof(*cpc_desc->sysmem_nodes),
+					 GFP_KERNEL);
+	if (!cpc_desc->sysmem_nodes)
+		return -ENOMEM;
+
+	mutex_lock(&cpc_sysmem_lock);
+
+	for (i = 0; i < nr_regs; i++) {
+		struct cpc_register_resource *reg = &cpc_desc->cpc_regs[i];
+		struct cpc_sysmem_node *alias = NULL, *match, *node;
+		u64 size;
+
+		if (!CPC_SUPPORTED(reg) || !CPC_IN_SYSTEM_MEMORY(reg))
+			continue;
+
+		node = &cpc_desc->sysmem_nodes[i];
+		size = cpc_sysmem_claim_size(reg);
+		node->start = reg->cpc_entry.reg.address;
+		node->last = node->start + size - 1;
+		node->desc = cpc_desc;
+		node->reg_idx = i;
+		INIT_LIST_HEAD(&node->aliases);
+		INIT_LIST_HEAD(&node->alias_node);
+
+		/* Performance Limited precedes every optional writer we may disable. */
+		if (cpc_disable_new_sysmem_writer(cpc_desc, i, node))
+			continue;
+
+		match = cpc_sysmem_first(node->start, node->last);
+		while (match) {
+			struct cpc_register_resource *match_reg;
+
+			match_reg = &match->desc->cpc_regs[match->reg_idx];
+			ret = cpc_validate_sysmem_pair(cpc_desc, i, match->desc,
+						       match->reg_idx);
+			if (ret)
+				goto out_unregister;
+			if (cpc_desc == match->desc) {
+				reg->cpc_entry.use_rmw_lock = true;
+				match_reg->cpc_entry.use_rmw_lock = true;
+			}
+			if (cpc_same_sysmem_register(i, reg, match->reg_idx, match_reg))
+				alias = match;
+
+			match = cpc_sysmem_next(match, node->start, node->last);
+		}
+		if (alias) {
+			node->alias_of = alias;
+			list_add_tail(&node->alias_node, &alias->aliases);
+			continue;
+		}
+
+		cpc_sysmem_itree_insert(node, &cpc_sysmem_tree);
+		node->registered = true;
+	}
+
+	mutex_unlock(&cpc_sysmem_lock);
+	return 0;
+
+out_unregister:
+	cpc_unregister_sysmem_desc_locked(cpc_desc);
+	mutex_unlock(&cpc_sysmem_lock);
+	return ret;
+}
+
+static void cpc_unregister_sysmem_desc(struct cpc_desc *cpc_desc)
+{
+	if (!cpc_desc->sysmem_nodes)
+		return;
+
+	mutex_lock(&cpc_sysmem_lock);
+	cpc_unregister_sysmem_desc_locked(cpc_desc);
+	mutex_unlock(&cpc_sysmem_lock);
 }
 
 static ssize_t show_feedback_ctrs(struct kobject *kobj,
@@ -297,7 +1361,30 @@ static struct attribute *cppc_attrs[] = {
 };
 ATTRIBUTE_GROUPS(cppc);
 
+static void cppc_free_desc(struct cpc_desc *cpc_ptr)
+{
+	unsigned int i;
+
+	cpc_unregister_non_mmio_desc(cpc_ptr);
+	cpc_unregister_sysmem_desc(cpc_ptr);
+
+	for (i = 2; i < cpc_ptr->num_entries; i++) {
+		void __iomem *addr = cpc_ptr->cpc_regs[i - 2].sys_mem_vaddr;
+
+		if (addr)
+			iounmap(addr);
+	}
+
+	kfree(cpc_ptr);
+}
+
+static void cppc_kobj_release(struct kobject *kobj)
+{
+	cppc_free_desc(to_cpc_desc(kobj));
+}
+
 static const struct kobj_type cppc_ktype = {
+	.release = cppc_kobj_release,
 	.sysfs_ops = &kobj_sysfs_ops,
 	.default_groups = cppc_groups,
 };
@@ -321,6 +1408,8 @@ static int check_pcc_chan(int pcc_ss_id, bool chk_err_bit)
 					pcc_ss_data->deadline_us);
 
 	if (likely(!ret)) {
+		/* Order completion status before reading the returned payload. */
+		rmb();
 		pcc_ss_data->platform_owns_pcc = false;
 		if (chk_err_bit && (status & PCC_ERROR_MASK))
 			ret = -EIO;
@@ -333,13 +1422,47 @@ static int check_pcc_chan(int pcc_ss_id, bool chk_err_bit)
 	return ret;
 }
 
+static void cppc_complete_pcc_write(int pcc_ss_id,
+				    struct cppc_pcc_data *pcc_ss_data, int ret)
+{
+	int i;
+
+	if (unlikely(ret)) {
+		for_each_possible_cpu(i) {
+			struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i);
+
+			if (!desc ||
+			    per_cpu(cpu_pcc_subspace_idx, i) != pcc_ss_id)
+				continue;
+
+			if (desc->write_cmd_id == pcc_ss_data->pcc_write_cnt)
+				desc->write_cmd_status = ret;
+		}
+	}
+
+	pcc_ss_data->pcc_write_cnt++;
+	wake_up_all(&pcc_ss_data->pcc_write_wait_q);
+}
+
+/* The caller must hold pcc_lock for write. */
+static void cppc_abort_pending_pcc_write(int pcc_ss_id,
+					 struct cppc_pcc_data *pcc_ss_data,
+					 int ret)
+{
+	if (!pcc_ss_data->pending_pcc_write_cmd)
+		return;
+
+	pcc_ss_data->pending_pcc_write_cmd = false;
+	cppc_complete_pcc_write(pcc_ss_id, pcc_ss_data, ret);
+}
+
 /*
  * This function transfers the ownership of the PCC to the platform
  * So it must be called while holding write_lock(pcc_lock)
  */
 static int send_pcc_cmd(int pcc_ss_id, u16 cmd)
 {
-	int ret = -EIO, i;
+	int ret = -EIO;
 	struct cppc_pcc_data *pcc_ss_data = pcc_data[pcc_ss_id];
 	struct acpi_pcct_shared_memory __iomem *generic_comm_base =
 					pcc_ss_data->pcc_channel->shmem;
@@ -431,21 +1554,8 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd)
 		mbox_client_txdone(pcc_ss_data->pcc_channel->mchan, ret);
 
 end:
-	if (cmd == CMD_WRITE) {
-		if (unlikely(ret)) {
-			for_each_possible_cpu(i) {
-				struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i);
-
-				if (!desc)
-					continue;
-
-				if (desc->write_cmd_id == pcc_ss_data->pcc_write_cnt)
-					desc->write_cmd_status = ret;
-			}
-		}
-		pcc_ss_data->pcc_write_cnt++;
-		wake_up_all(&pcc_ss_data->pcc_write_wait_q);
-	}
+	if (cmd == CMD_WRITE)
+		cppc_complete_pcc_write(pcc_ss_id, pcc_ss_data, ret);
 
 	return ret;
 }
@@ -555,7 +1665,7 @@ bool cppc_allow_fast_switch(const struct cpumask *cpus)
 		min_reg = &cpc_ptr->cpc_regs[MIN_PERF];
 		max_reg = &cpc_ptr->cpc_regs[MAX_PERF];
 
-		if (!CPC_SUPPORTED(desired_reg) ||
+		if (!cpc_is_writable(desired_reg) ||
 		    (!CPC_IN_SYSTEM_MEMORY(desired_reg) &&
 		     !CPC_IN_SYSTEM_IO(desired_reg)) ||
 		    (CPC_SUPPORTED(min_reg) &&
@@ -643,35 +1753,49 @@ EXPORT_SYMBOL_GPL(acpi_get_psd_map);
 
 static int register_pcc_channel(int pcc_ss_idx)
 {
+	struct cppc_pcc_data *data;
 	struct pcc_mbox_chan *pcc_chan;
 	u64 usecs_lat;
+	int ret = 0;
 
-	if (pcc_ss_idx >= 0) {
-		pcc_chan = pcc_mbox_request_channel(&cppc_mbox_cl, pcc_ss_idx);
-
-		if (IS_ERR(pcc_chan)) {
-			pr_err("Failed to find PCC channel for subspace %d\n",
-			       pcc_ss_idx);
-			return -ENODEV;
-		}
+	if (pcc_ss_idx < 0 || pcc_ss_idx >= MAX_PCC_SUBSPACES)
+		return -EINVAL;
 
-		pcc_data[pcc_ss_idx]->pcc_channel = pcc_chan;
-		/*
-		 * cppc_ss->latency is just a Nominal value. In reality
-		 * the remote processor could be much slower to reply.
-		 * So add an arbitrary amount of wait on top of Nominal.
-		 */
-		usecs_lat = NUM_RETRIES * pcc_chan->latency;
-		pcc_data[pcc_ss_idx]->deadline_us = usecs_lat;
-		pcc_data[pcc_ss_idx]->pcc_mrtt = pcc_chan->min_turnaround_time;
-		pcc_data[pcc_ss_idx]->pcc_mpar = pcc_chan->max_access_rate;
-		pcc_data[pcc_ss_idx]->pcc_nominal = pcc_chan->latency;
+	mutex_lock(&pcc_data_lock);
+	data = pcc_data[pcc_ss_idx];
+	if (!data) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+	if (data->pcc_channel_acquired)
+		goto out_unlock;
 
-		/* Set flag so that we don't come here for each CPU. */
-		pcc_data[pcc_ss_idx]->pcc_channel_acquired = true;
+	pcc_chan = pcc_mbox_request_channel(&cppc_mbox_cl, pcc_ss_idx);
+	if (IS_ERR(pcc_chan)) {
+		ret = -ENODEV;
+		goto out_unlock;
 	}
 
-	return 0;
+	data->pcc_channel = pcc_chan;
+	/*
+	 * cppc_ss->latency is just a Nominal value. In reality
+	 * the remote processor could be much slower to reply.
+	 * So add an arbitrary amount of wait on top of Nominal.
+	 */
+	usecs_lat = NUM_RETRIES * pcc_chan->latency;
+	data->deadline_us = usecs_lat;
+	data->pcc_mrtt = pcc_chan->min_turnaround_time;
+	data->pcc_mpar = pcc_chan->max_access_rate;
+	data->pcc_nominal = pcc_chan->latency;
+	init_rwsem(&data->pcc_lock);
+	init_waitqueue_head(&data->pcc_write_wait_q);
+
+	/* Reuse this channel when another CPU references the same subspace. */
+	data->pcc_channel_acquired = true;
+
+out_unlock:
+	mutex_unlock(&pcc_data_lock);
+	return ret;
 }
 
 /**
@@ -713,19 +1837,50 @@ bool __weak cpc_supported_by_cpu(void)
  */
 static int pcc_data_alloc(int pcc_ss_id)
 {
+	struct cppc_pcc_data *data;
+	int ret = 0;
+
 	if (pcc_ss_id < 0 || pcc_ss_id >= MAX_PCC_SUBSPACES)
 		return -EINVAL;
 
-	if (pcc_data[pcc_ss_id]) {
-		pcc_data[pcc_ss_id]->refcount++;
-	} else {
-		pcc_data[pcc_ss_id] = kzalloc_obj(struct cppc_pcc_data);
-		if (!pcc_data[pcc_ss_id])
-			return -ENOMEM;
-		pcc_data[pcc_ss_id]->refcount++;
+	mutex_lock(&pcc_data_lock);
+	data = pcc_data[pcc_ss_id];
+	if (!data) {
+		data = kzalloc_obj(struct cppc_pcc_data);
+		if (!data) {
+			ret = -ENOMEM;
+			goto out_unlock;
+		}
+		raw_spin_lock_init(&data->payload_lock);
+		pcc_data[pcc_ss_id] = data;
 	}
+	data->refcount++;
 
-	return 0;
+out_unlock:
+	mutex_unlock(&pcc_data_lock);
+	return ret;
+}
+
+static void pcc_data_put(int pcc_ss_id)
+{
+	struct cppc_pcc_data *data;
+
+	if (pcc_ss_id < 0 || pcc_ss_id >= MAX_PCC_SUBSPACES)
+		return;
+
+	mutex_lock(&pcc_data_lock);
+	data = pcc_data[pcc_ss_id];
+	if (!data || --data->refcount)
+		goto out_unlock;
+
+	pcc_data[pcc_ss_id] = NULL;
+	if (data->pcc_channel_acquired)
+		pcc_mbox_free_channel(data->pcc_channel);
+
+	kfree(data);
+
+out_unlock:
+	mutex_unlock(&pcc_data_lock);
 }
 
 /*
@@ -772,9 +1927,24 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	struct device *cpu_dev;
 	acpi_handle handle = pr->handle;
 	unsigned int num_ent, i, cpc_rev;
+	u32 unsupported_regs = 0;
+	u32 platform_quirks;
 	int pcc_subspace_id = -1;
+	bool pcc_data_ref = false;
+	bool cpc_present = false;
 	acpi_status status;
-	int ret = -ENODATA;
+	int ret = -EINVAL;
+	int err;
+
+	if (per_cpu(cpc_desc_ptr, pr->id))
+		return 0;
+	ret = cpc_get_platform_quirks(&platform_quirks);
+	if (ret) {
+		pr_err("CPU%d: failed to match CPPC platform quirks: %d\n",
+		       pr->id, ret);
+		return ret;
+	}
+	per_cpu(cpu_pcc_subspace_idx, pr->id) = -1;
 
 	if (!osc_sb_cppc2_support_acked) {
 		pr_debug("CPPC v2 _OSC not acked\n");
@@ -791,24 +1961,35 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 		ret = -ENODEV;
 		goto out_buf_free;
 	}
+	cpc_present = true;
+	ret = -EINVAL;
 
 	out_obj = (union acpi_object *) output.pointer;
+	if (out_obj->package.count < 2) {
+		pr_debug("Unexpected _CPC package count (%u) for CPU:%d\n",
+			 out_obj->package.count, pr->id);
+		goto out_buf_free;
+	}
 
 	cpc_ptr = kzalloc_obj(struct cpc_desc);
 	if (!cpc_ptr) {
 		ret = -ENOMEM;
 		goto out_buf_free;
 	}
+	cpc_ptr->cpu_id = pr->id;
 
 	/* First entry is NumEntries. */
 	cpc_obj = &out_obj->package.elements[0];
 	if (cpc_obj->type == ACPI_TYPE_INTEGER)	{
-		num_ent = cpc_obj->integer.value;
-		if (num_ent <= 1) {
-			pr_debug("Unexpected _CPC NumEntries value (%d) for CPU:%d\n",
-				 num_ent, pr->id);
+		if (cpc_obj->integer.value < 2 ||
+		    cpc_obj->integer.value > out_obj->package.count) {
+			pr_debug("Invalid _CPC NumEntries (%llu) for package count (%u) on CPU:%d\n",
+				 cpc_obj->integer.value, out_obj->package.count,
+				 pr->id);
 			goto out_free;
 		}
+
+		num_ent = cpc_obj->integer.value;
 	} else {
 		pr_debug("Unexpected _CPC NumEntries entry type (%d) for CPU:%d\n",
 			 cpc_obj->type, pr->id);
@@ -818,6 +1999,12 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	/* Second entry should be revision. */
 	cpc_obj = &out_obj->package.elements[1];
 	if (cpc_obj->type == ACPI_TYPE_INTEGER)	{
+		if (cpc_obj->integer.value > U8_MAX) {
+			pr_debug("Invalid _CPC Revision (%llu) for CPU:%d\n",
+				 cpc_obj->integer.value, pr->id);
+			ret = -EINVAL;
+			goto out_free;
+		}
 		cpc_rev = cpc_obj->integer.value;
 	} else {
 		pr_debug("Unexpected _CPC Revision entry type (%d) for CPU:%d\n",
@@ -857,11 +2044,45 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 		cpc_obj = &out_obj->package.elements[i];
 
 		if (cpc_obj->type == ACPI_TYPE_INTEGER)	{
-			cpc_ptr->cpc_regs[i-2].type = ACPI_TYPE_INTEGER;
-			cpc_ptr->cpc_regs[i-2].cpc_entry.int_value = cpc_obj->integer.value;
+			bool legacy_null;
+
+			if (!cpc_integer_entry_valid(i - 2,
+						     cpc_obj->integer.value,
+						     &legacy_null)) {
+				pr_debug("Invalid Integer _CPC register %u for CPU:%d\n",
+					 i - 2, pr->id);
+				ret = -EINVAL;
+				goto out_free;
+			}
+			if (legacy_null)
+				pr_warn_once(FW_BUG "_CPC register %u uses Integer 0 for an absent Buffer\n",
+					     i - 2);
+			cpc_ptr->cpc_regs[i - 2].type = ACPI_TYPE_INTEGER;
+			cpc_ptr->cpc_regs[i - 2].cpc_entry.int_value = cpc_obj->integer.value;
 		} else if (cpc_obj->type == ACPI_TYPE_BUFFER) {
+			if (cpc_obj->buffer.length < sizeof(*gas_t)) {
+				pr_debug("Invalid register descriptor for CPU:%d\n",
+					 pr->id);
+				ret = -EINVAL;
+				goto out_free;
+			}
+
 			gas_t = (struct cpc_reg *)
 				cpc_obj->buffer.pointer;
+			if (gas_t->descriptor != CPC_GENERIC_REGISTER_DESCRIPTOR ||
+			    gas_t->length != CPC_GENERIC_REGISTER_LENGTH) {
+				pr_debug("Invalid register resource for CPU:%d\n",
+					 pr->id);
+				ret = -EINVAL;
+				goto out_free;
+			}
+
+			cpc_ptr->cpc_regs[i - 2].type = ACPI_TYPE_BUFFER;
+			memcpy(&cpc_ptr->cpc_regs[i - 2].cpc_entry.reg, gas_t,
+			       sizeof(*gas_t));
+			gas_t = &cpc_ptr->cpc_regs[i - 2].cpc_entry.reg;
+			cpc_apply_platform_quirks(gas_t, i - 2,
+						  platform_quirks);
 
 			/*
 			 * The PCC Subspace index is encoded inside
@@ -870,65 +2091,142 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 			 * so extract it only once.
 			 */
 			if (gas_t->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
+				/* These registers have no specified 32-bit upper bound. */
+				bool wide_write = i - 2 == PERF_LIMITED ||
+						  i - 2 == ENABLE ||
+						  i - 2 == AUTO_SEL_ENABLE;
+				bool write_width_supported = gas_t->bit_width == 8 ||
+						     gas_t->bit_width == 16 ||
+						     gas_t->bit_width == 32 ||
+						     gas_t->bit_width == 64;
+				bool unsupported;
+
+				unsupported = !gas_t->bit_width ||
+					      gas_t->bit_width > 64 ||
+					      gas_t->bit_offset ||
+					      gas_t->bit_width % 8 ||
+					      (cpc_reg_is_writable(i - 2) &&
+					       (!write_width_supported ||
+						(!wide_write && gas_t->bit_width > 32)));
+				if (unsupported) {
+					if (!cpc_retain_pcc_status(cpc_ptr, i - 2))
+						unsupported_regs |= BIT(i - 2);
+					continue;
+				}
+
 				if (pcc_subspace_id < 0) {
 					pcc_subspace_id = gas_t->access_width;
-					if (pcc_data_alloc(pcc_subspace_id))
-						goto out_free;
 				} else if (pcc_subspace_id != gas_t->access_width) {
 					pr_debug("Mismatched PCC ids in _CPC for CPU:%d\n",
 						 pr->id);
+					ret = -EINVAL;
 					goto out_free;
 				}
+
+				if (!pcc_data_ref) {
+					err = pcc_data_alloc(pcc_subspace_id);
+					if (err) {
+						ret = err;
+						goto out_free;
+					}
+					pcc_data_ref = true;
+				}
 			} else if (gas_t->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
-				if (gas_t->address) {
+				if (!IS_NULL_REG(gas_t)) {
 					void __iomem *addr;
 					size_t access_width;
 
+					err = cpc_validate_sysmem_reg(cpc_ptr, gas_t,
+								      i - 2);
+					if (err) {
+						unsupported_regs |= BIT(i - 2);
+						continue;
+					}
+					if (!cpc_is_readable(&cpc_ptr->cpc_regs[i - 2]) &&
+					    !cpc_is_writable(&cpc_ptr->cpc_regs[i - 2]))
+						continue;
+
 					if (!osc_cpc_flexible_adr_space_confirmed) {
 						pr_debug("Flexible address space capability not supported\n");
+						ret = -EOPNOTSUPP;
 						if (!cpc_supported_by_cpu())
 							goto out_free;
+						ret = -EINVAL;
 					}
 
-					access_width = GET_BIT_WIDTH(gas_t) / 8;
+					access_width = cpc_reg_access_width(gas_t);
+					access_width /= 8;
 					addr = ioremap(gas_t->address, access_width);
-					if (!addr)
+					if (!addr) {
+						ret = -ENOMEM;
 						goto out_free;
-					cpc_ptr->cpc_regs[i-2].sys_mem_vaddr = addr;
+					}
+					cpc_ptr->cpc_regs[i - 2].sys_mem_vaddr = addr;
 				}
 			} else if (gas_t->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
-				if (gas_t->access_width < 1 || gas_t->access_width > 3) {
-					/*
-					 * 1 = 8-bit, 2 = 16-bit, and 3 = 32-bit.
-					 * SystemIO doesn't implement 64-bit
-					 * registers.
-					 */
-					pr_debug("Invalid access width %d for SystemIO register in _CPC\n",
-						 gas_t->access_width);
-					goto out_free;
+				u64 access_size;
+				const char *reason = "uses unsupported SystemIO geometry";
+				unsigned int access_width;
+				bool partial = false;
+				bool unsupported;
+
+				access_width = cpc_reg_access_width(gas_t);
+				unsupported = !IS_ENABLED(CONFIG_HAS_IOPORT);
+				if (unsupported)
+					reason = "requires unavailable SystemIO support";
+				else
+					unsupported = access_width != 8 &&
+					      access_width != 16 &&
+					      access_width != 32;
+				if (!unsupported) {
+					access_size = access_width / 8;
+					unsupported = !gas_t->bit_width ||
+						gas_t->bit_width > access_width ||
+						gas_t->bit_offset >= access_width ||
+						gas_t->bit_width > access_width -
+									   gas_t->bit_offset;
+					partial = gas_t->bit_offset ||
+						  gas_t->bit_width != access_width;
 				}
-				if (gas_t->address & OVER_16BTS_MASK) {
-					/* SystemIO registers use 16-bit integer addresses */
-					pr_debug("Invalid IO port %llu for SystemIO register in _CPC\n",
-						 gas_t->address);
-					goto out_free;
+				if (!unsupported) {
+					unsupported = (cpc_reg_is_writable(i - 2) &&
+						i - 2 != PERF_LIMITED &&
+						cpc_is_writable(&cpc_ptr->cpc_regs[i - 2]) &&
+						partial) ||
+						!cpc_reg_access_aligned(gas_t,
+									access_size) ||
+						gas_t->address >
+						U16_MAX - (access_size - 1);
+				}
+				if (unsupported) {
+					if (cpc_retain_sysio_status(cpc_ptr, i - 2))
+						continue;
+					pr_debug("CPU%d: _CPC register %u %s\n",
+						 pr->id, i - 2, reason);
+					unsupported_regs |= BIT(i - 2);
+					continue;
+				}
+				if (i - 2 == PERF_LIMITED && partial) {
+					pr_warn_once("CPU%d: Performance Limited register cannot be cleared safely; keeping it readable\n",
+						     cpc_ptr->cpu_id);
+					cpc_ptr->cpc_regs[i - 2].cpc_entry.write_unsupported = true;
 				}
 				if (!osc_cpc_flexible_adr_space_confirmed) {
 					pr_debug("Flexible address space capability not supported\n");
+					ret = -EOPNOTSUPP;
 					if (!cpc_supported_by_cpu())
 						goto out_free;
+					ret = -EINVAL;
 				}
 			} else {
 				if (gas_t->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE || !cpc_ffh_supported()) {
 					/* Support only PCC, SystemMemory, SystemIO, and FFH type regs. */
 					pr_debug("Unsupported register type (%d) in _CPC\n",
 						 gas_t->space_id);
+					ret = -EOPNOTSUPP;
 					goto out_free;
 				}
 			}
-
-			cpc_ptr->cpc_regs[i-2].type = ACPI_TYPE_BUFFER;
-			memcpy(&cpc_ptr->cpc_regs[i-2].cpc_entry.reg, gas_t, sizeof(*gas_t));
 		} else if (cpc_obj->type == ACPI_TYPE_PACKAGE && (i - 2) == RESOURCE_PRIORITY) {
 			/*
 			 * ACPI 6.6, s8.4.6.1.2.7 defines Resource Priority as a
@@ -945,17 +2243,18 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 			goto out_free;
 		}
 	}
-	per_cpu(cpu_pcc_subspace_idx, pr->id) = pcc_subspace_id;
 
-	/*
-	 * In CPPC v1, DESIRED_PERF is mandatory. In CPPC v2, it is optional
-	 * only when AUTO_SEL_ENABLE is supported.
-	 */
-	if (!CPC_SUPPORTED(&cpc_ptr->cpc_regs[DESIRED_PERF]) &&
-	    (!osc_sb_cppc2_support_acked ||
-	     !CPC_SUPPORTED(&cpc_ptr->cpc_regs[AUTO_SEL_ENABLE])))
-		pr_warn("Desired perf. register is mandatory if CPPC v2 is not supported "
-			"or autonomous selection is disabled\n");
+	per_cpu(cpu_pcc_subspace_idx, pr->id) = pcc_data_ref ?
+						 pcc_subspace_id : -1;
+
+	ret = cpc_resolve_unsupported(cpc_ptr, unsupported_regs);
+	if (ret)
+		goto out_free;
+	unsupported_regs = 0;
+
+	ret = cpc_validate_required_controls(cpc_ptr);
+	if (ret)
+		goto out_free;
 
 	/*
 	 * Initialize the remaining cpc_regs as unsupported.
@@ -968,8 +2267,6 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	}
 
 
-	/* Store CPU Logical ID */
-	cpc_ptr->cpu_id = pr->id;
 	cpc_mark_rmw_lock_users(cpc_ptr);
 	raw_spin_lock_init(&cpc_ptr->rmw_lock);
 
@@ -978,16 +2275,43 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	if (ret)
 		goto out_free;
 
+	ret = cpc_register_sysmem_desc(cpc_ptr);
+	if (ret)
+		goto out_free;
+
 	/* Register PCC channel once for all PCC subspace ID. */
-	if (pcc_subspace_id >= 0 && !pcc_data[pcc_subspace_id]->pcc_channel_acquired) {
+	if (pcc_data_ref) {
 		ret = register_pcc_channel(pcc_subspace_id);
+		if (ret) {
+			pr_err("Failed to find PCC channel for subspace %d\n",
+			       pcc_subspace_id);
+			goto out_free;
+		}
+
+		cpc_validate_pcc_bounds(cpc_ptr, pcc_subspace_id,
+					pcc_data[pcc_subspace_id],
+					&unsupported_regs);
+
+		ret = cpc_resolve_unsupported(cpc_ptr, unsupported_regs);
 		if (ret)
 			goto out_free;
 
-		init_rwsem(&pcc_data[pcc_subspace_id]->pcc_lock);
-		init_waitqueue_head(&pcc_data[pcc_subspace_id]->pcc_write_wait_q);
+		/* A range-only status entry needs the channel only for bounds. */
+		if (!cpc_pcc_access_needed(cpc_ptr)) {
+			pcc_data_put(pcc_subspace_id);
+			pcc_data_ref = false;
+			per_cpu(cpu_pcc_subspace_idx, pr->id) = -1;
+		}
 	}
 
+	ret = cpc_validate_bound_controls(cpc_ptr);
+	if (ret)
+		goto out_free;
+
+	ret = cpc_register_non_mmio_desc(cpc_ptr);
+	if (ret)
+		goto out_free;
+
 	/* Everything looks okay */
 	pr_debug("Parsed CPC struct for CPU: %d\n", pr->id);
 
@@ -999,30 +2323,32 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr)
 	}
 
 	/* Plug PSD data into this CPU's CPC descriptor. */
-	per_cpu(cpc_desc_ptr, pr->id) = cpc_ptr;
+	cpc_set_desc(pr->id, cpc_ptr);
 
 	ret = kobject_init_and_add(&cpc_ptr->kobj, &cppc_ktype, &cpu_dev->kobj,
 			"acpi_cppc");
 	if (ret) {
-		per_cpu(cpc_desc_ptr, pr->id) = NULL;
+		cpc_set_desc(pr->id, NULL);
+		cpc_unregister_non_mmio_desc(cpc_ptr);
+		cpc_unregister_sysmem_desc(cpc_ptr);
 		kobject_put(&cpc_ptr->kobj);
-		goto out_free;
+		goto out_pcc_put;
 	}
 
 	kfree(output.pointer);
 	return 0;
 
 out_free:
-	/* Free all the mapped sys mem areas for this CPU */
-	for (i = 2; i < cpc_ptr->num_entries; i++) {
-		void __iomem *addr = cpc_ptr->cpc_regs[i-2].sys_mem_vaddr;
+	cppc_free_desc(cpc_ptr);
 
-		if (addr)
-			iounmap(addr);
-	}
-	kfree(cpc_ptr);
+out_pcc_put:
+	if (pcc_data_ref)
+		pcc_data_put(pcc_subspace_id);
+	per_cpu(cpu_pcc_subspace_idx, pr->id) = -1;
 
 out_buf_free:
+	if (cpc_present)
+		pr_err("CPU%d: failed to initialize _CPC: %d\n", pr->id, ret);
 	kfree(output.pointer);
 	return ret;
 }
@@ -1037,34 +2363,24 @@ EXPORT_SYMBOL_GPL(acpi_cppc_processor_probe);
 void acpi_cppc_processor_exit(struct acpi_processor *pr)
 {
 	struct cpc_desc *cpc_ptr;
-	unsigned int i;
-	void __iomem *addr;
-	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, pr->id);
-
-	if (pcc_ss_id >= 0 && pcc_data[pcc_ss_id]) {
-		if (pcc_data[pcc_ss_id]->pcc_channel_acquired) {
-			pcc_data[pcc_ss_id]->refcount--;
-			if (!pcc_data[pcc_ss_id]->refcount) {
-				pcc_mbox_free_channel(pcc_data[pcc_ss_id]->pcc_channel);
-				kfree(pcc_data[pcc_ss_id]);
-				pcc_data[pcc_ss_id] = NULL;
-			}
-		}
-	}
+	int pcc_ss_id;
 
 	cpc_ptr = per_cpu(cpc_desc_ptr, pr->id);
-	if (!cpc_ptr)
+	if (!cpc_ptr) {
+		per_cpu(cpu_pcc_subspace_idx, pr->id) = -1;
 		return;
-
-	/* Free all the mapped sys mem areas for this CPU */
-	for (i = 2; i < cpc_ptr->num_entries; i++) {
-		addr = cpc_ptr->cpc_regs[i-2].sys_mem_vaddr;
-		if (addr)
-			iounmap(addr);
 	}
 
+	pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, pr->id);
+	cpc_set_desc(pr->id, NULL);
+	kobject_del(&cpc_ptr->kobj);
+	cpc_unregister_non_mmio_desc(cpc_ptr);
+	cpc_unregister_sysmem_desc(cpc_ptr);
+
+	pcc_data_put(pcc_ss_id);
+	per_cpu(cpu_pcc_subspace_idx, pr->id) = -1;
+
 	kobject_put(&cpc_ptr->kobj);
-	kfree(cpc_ptr);
 }
 EXPORT_SYMBOL_GPL(acpi_cppc_processor_exit);
 
@@ -1123,23 +2439,34 @@ int __weak cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
 static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val)
 {
 	void __iomem *vaddr = NULL;
+	unsigned long flags;
+	u8 buf[sizeof(*val)];
+	unsigned int i;
 	int size;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
 	struct cpc_reg *reg = &reg_res->cpc_entry.reg;
 
+	if (!cpc_is_readable(reg_res))
+		return -EOPNOTSUPP;
+
 	if (reg_res->type == ACPI_TYPE_INTEGER) {
 		*val = reg_res->cpc_entry.int_value;
 		return 0;
 	}
 
 	*val = 0;
+	if (reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE)
+		return cpc_read_ffh(cpu, reg, val);
+
 	size = GET_BIT_WIDTH(reg);
 
-	if (IS_ENABLED(CONFIG_HAS_IOPORT) &&
-	    reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
+	if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
 		u32 val_u32;
 		acpi_status status;
 
+		if (!IS_ENABLED(CONFIG_HAS_IOPORT))
+			return -EOPNOTSUPP;
+
 		status = acpi_os_read_port((acpi_io_address)reg->address,
 					   &val_u32, size);
 		if (ACPI_FAILURE(status)) {
@@ -1148,20 +2475,33 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val)
 			return -EFAULT;
 		}
 
-		*val = val_u32;
+		*val = MASK_VAL_READ(reg, val_u32);
 		return 0;
-	} else if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM && pcc_ss_id >= 0) {
+	} else if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
+		if (pcc_ss_id < 0 || !pcc_data[pcc_ss_id])
+			return -ENODEV;
+
 		/*
 		 * For registers in PCC space, the register size is determined
 		 * by the bit width field; the access size is used to indicate
 		 * the PCC subspace id.
 		 */
 		vaddr = GET_PCC_VADDR(reg->address, pcc_ss_id);
-	}
-	else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
+		size = reg->bit_width / 8;
+		if (!size || size > sizeof(buf) || reg->bit_width % 8)
+			return -EFAULT;
+
+		raw_spin_lock_irqsave(&pcc_data[pcc_ss_id]->payload_lock, flags);
+		memcpy_fromio(buf, vaddr, size);
+		raw_spin_unlock_irqrestore(&pcc_data[pcc_ss_id]->payload_lock,
+					   flags);
+
+		*val = 0;
+		for (i = 0; i < size; i++)
+			*val |= (u64)buf[i] << (i * 8);
+		return 0;
+	} else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
 		vaddr = reg_res->sys_mem_vaddr;
-	else if (reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE)
-		return cpc_read_ffh(cpu, reg, val);
 	else
 		return acpi_os_read_memory((acpi_physical_address)reg->address,
 				val, size);
@@ -1180,18 +2520,12 @@ static int cpc_read(int cpu, struct cpc_register_resource *reg_res, u64 *val)
 		*val = readq_relaxed(vaddr);
 		break;
 	default:
-		if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
-			pr_debug("Error: Cannot read %u bit width from system memory: 0x%llx\n",
-				size, reg->address);
-		} else if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
-			pr_debug("Error: Cannot read %u bit width from PCC for ss: %d\n",
-				size, pcc_ss_id);
-		}
+		pr_debug("Error: Cannot read %u bit width from system memory: 0x%llx\n",
+			 size, reg->address);
 		return -EFAULT;
 	}
 
-	if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
-		*val = MASK_VAL_READ(reg, *val);
+	*val = MASK_VAL_READ(reg, *val);
 
 	return 0;
 }
@@ -1203,17 +2537,28 @@ static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val)
 	u64 prev_val;
 	void __iomem *vaddr = NULL;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
-	struct cpc_reg *reg = &reg_res->cpc_entry.reg;
+	struct cpc_reg *reg;
 	struct cpc_desc *cpc_desc;
 	unsigned long flags;
+	u8 buf[sizeof(val)];
+	unsigned int i;
 	bool locked = false;
 
+	if (!cpc_is_writable(reg_res))
+		return -EOPNOTSUPP;
+
+	reg = &reg_res->cpc_entry.reg;
+	if (reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE)
+		return cpc_write_ffh(cpu, reg, val);
+
 	size = GET_BIT_WIDTH(reg);
 
-	if (IS_ENABLED(CONFIG_HAS_IOPORT) &&
-	    reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
+	if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
 		acpi_status status;
 
+		if (!IS_ENABLED(CONFIG_HAS_IOPORT))
+			return -EOPNOTSUPP;
+
 		status = acpi_os_write_port((acpi_io_address)reg->address,
 					    (u32)val, size);
 		if (ACPI_FAILURE(status)) {
@@ -1223,60 +2568,72 @@ static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val)
 		}
 
 		return 0;
-	} else if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM && pcc_ss_id >= 0) {
+	} else if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
+		if (pcc_ss_id < 0 || !pcc_data[pcc_ss_id])
+			return -ENODEV;
+
 		/*
 		 * For registers in PCC space, the register size is determined
 		 * by the bit width field; the access size is used to indicate
 		 * the PCC subspace id.
 		 */
 		vaddr = GET_PCC_VADDR(reg->address, pcc_ss_id);
-	}
-	else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
+		size = reg->bit_width / 8;
+		if (!size || size > sizeof(buf) || reg->bit_width % 8)
+			return -EFAULT;
+
+		for (i = 0; i < size; i++)
+			buf[i] = val >> (i * 8);
+
+		raw_spin_lock_irqsave(&pcc_data[pcc_ss_id]->payload_lock, flags);
+		memcpy_toio(vaddr, buf, size);
+		/* Publish every payload byte before another CPU can ring the doorbell. */
+		wmb();
+		raw_spin_unlock_irqrestore(&pcc_data[pcc_ss_id]->payload_lock,
+					   flags);
+		return 0;
+	} else if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY)
 		vaddr = reg_res->sys_mem_vaddr;
-	else if (reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE)
-		return cpc_write_ffh(cpu, reg, val);
 	else
 		return acpi_os_write_memory((acpi_physical_address)reg->address,
 				val, size);
 
-	if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
-		/*
-		 * The _CPC layout is immutable after probe. The precomputed flag
-		 * retains serialization for partial fields or overlapping access
-		 * units; standalone full-width registers avoid the lock.
-		 */
-		locked = reg_res->cpc_entry.use_rmw_lock;
-		if (locked) {
-			cpc_desc = per_cpu(cpc_desc_ptr, cpu);
-			if (!cpc_desc) {
-				pr_debug("No CPC descriptor for CPU:%d\n", cpu);
-				return -ENODEV;
-			}
-			raw_spin_lock_irqsave(&cpc_desc->rmw_lock, flags);
+	/* Partial fields and local overlaps use the descriptor lock. */
+	locked = reg_res->cpc_entry.use_rmw_lock;
+	if (locked) {
+		cpc_desc = per_cpu(cpc_desc_ptr, cpu);
+		if (!cpc_desc) {
+			pr_debug("No CPC descriptor for CPU:%d\n", cpu);
+			return -ENODEV;
 		}
+		raw_spin_lock_irqsave(&cpc_desc->rmw_lock, flags);
+	}
 
-		if (reg->bit_offset || reg->bit_width != size) {
-			switch (size) {
-			case 8:
-				prev_val = readb_relaxed(vaddr);
-				break;
-			case 16:
-				prev_val = readw_relaxed(vaddr);
-				break;
-			case 32:
-				prev_val = readl_relaxed(vaddr);
-				break;
-			case 64:
-				prev_val = readq_relaxed(vaddr);
-				break;
-			default:
-				if (locked)
-					raw_spin_unlock_irqrestore(&cpc_desc->rmw_lock,
-								   flags);
-				return -EFAULT;
-			}
-			val = MASK_VAL_WRITE(reg, prev_val, val);
+	if (reg->bit_offset || reg->bit_width != size) {
+		/*
+		 * MASK_VAL_WRITE() discards the field's old bits, so undefined
+		 * readback from a write-only field is not propagated.
+		 */
+		switch (size) {
+		case 8:
+			prev_val = readb_relaxed(vaddr);
+			break;
+		case 16:
+			prev_val = readw_relaxed(vaddr);
+			break;
+		case 32:
+			prev_val = readl_relaxed(vaddr);
+			break;
+		case 64:
+			prev_val = readq_relaxed(vaddr);
+			break;
+		default:
+			if (locked)
+				raw_spin_unlock_irqrestore(&cpc_desc->rmw_lock,
+							   flags);
+			return -EFAULT;
 		}
+		val = MASK_VAL_WRITE(reg, prev_val, val);
 	}
 
 	switch (size) {
@@ -1293,19 +2650,17 @@ static int cpc_write(int cpu, struct cpc_register_resource *reg_res, u64 val)
 		writeq_relaxed(val, vaddr);
 		break;
 	default:
-		if (reg->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
-			pr_debug("Error: Cannot write %u bit width to system memory: 0x%llx\n",
-				size, reg->address);
-		} else if (reg->space_id == ACPI_ADR_SPACE_PLATFORM_COMM) {
-			pr_debug("Error: Cannot write %u bit width to PCC for ss: %d\n",
-				size, pcc_ss_id);
-		}
+		pr_debug("Error: Cannot write %u bit width to system memory: 0x%llx\n",
+			 size, reg->address);
 		ret_val = -EFAULT;
 		break;
 	}
 
-	if (locked)
+	if (locked) {
+		if (!ret_val)
+			mmiowb_set_pending();
 		raw_spin_unlock_irqrestore(&cpc_desc->rmw_lock, flags);
+	}
 
 	return ret_val;
 }
@@ -1347,15 +2702,25 @@ static int cppc_get_reg_val(int cpu, enum cppc_regs reg_idx, u64 *val)
 		pr_debug("No CPC descriptor for CPU:%d\n", cpu);
 		return -ENODEV;
 	}
+	if (cpc_reg_is_write_only(cpc_desc, reg_idx))
+		return -EOPNOTSUPP;
 
 	reg = &cpc_desc->cpc_regs[reg_idx];
 
-	if ((reg->type == ACPI_TYPE_INTEGER && IS_OPTIONAL_CPC_REG(reg_idx) &&
+	/*
+	 * Desired and Performance Limited may be disabled despite not being
+	 * generally optional.
+	 */
+	if ((reg->type == ACPI_TYPE_INTEGER &&
+	     (IS_OPTIONAL_CPC_REG(reg_idx) || reg_idx == DESIRED_PERF ||
+	      reg_idx == PERF_LIMITED) &&
 	     !reg->cpc_entry.int_value) || (reg->type != ACPI_TYPE_INTEGER &&
 	     IS_NULL_REG(&reg->cpc_entry.reg))) {
 		pr_debug("CPC register is not supported\n");
 		return -EOPNOTSUPP;
 	}
+	if (!cpc_is_readable(reg))
+		return -EOPNOTSUPP;
 
 	if (CPC_IN_PCC(reg))
 		return cppc_get_reg_val_in_pcc(cpu, reg, val);
@@ -1366,23 +2731,33 @@ static int cppc_get_reg_val(int cpu, enum cppc_regs reg_idx, u64 *val)
 static int cppc_set_reg_val_in_pcc(int cpu, struct cpc_register_resource *reg, u64 val)
 {
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
-	struct cppc_pcc_data *pcc_ss_data = NULL;
+	struct cppc_pcc_data *pcc_ss_data;
 	int ret;
 
 	if (pcc_ss_id < 0) {
 		pr_debug("Invalid pcc_ss_id\n");
 		return -ENODEV;
 	}
+	pcc_ss_data = pcc_data[pcc_ss_id];
+	if (!pcc_ss_data)
+		return -ENODEV;
 
-	ret = cpc_write(cpu, reg, val);
+	down_write(&pcc_ss_data->pcc_lock);
+
+	ret = check_pcc_chan(pcc_ss_id, false);
 	if (ret)
-		return ret;
+		goto out;
 
-	pcc_ss_data = pcc_data[pcc_ss_id];
+	ret = cpc_write(cpu, reg, val);
+	if (ret)
+		goto out;
 
-	down_write(&pcc_ss_data->pcc_lock);
 	/* after writing CPC, transfer the ownership of PCC to platform */
 	ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE);
+
+out:
+	if (ret)
+		cppc_abort_pending_pcc_write(pcc_ss_id, pcc_ss_data, ret);
 	up_write(&pcc_ss_data->pcc_lock);
 
 	return ret;
@@ -1400,8 +2775,13 @@ static int cppc_set_reg_val(int cpu, enum cppc_regs reg_idx, u64 val)
 
 	reg = &cpc_desc->cpc_regs[reg_idx];
 
+	/* Integer 1 describes autonomous selection that is always enabled. */
+	if (reg_idx == AUTO_SEL_ENABLE && reg->type == ACPI_TYPE_INTEGER &&
+	    reg->cpc_entry.int_value == 1)
+		return val == 1 ? 0 : -EOPNOTSUPP;
+
 	/* if a register is writeable, it must be a buffer and not null */
-	if ((reg->type != ACPI_TYPE_BUFFER) || IS_NULL_REG(&reg->cpc_entry.reg)) {
+	if (!cpc_is_writable(reg)) {
 		pr_debug("CPC register is not supported\n");
 		return -EOPNOTSUPP;
 	}
@@ -1412,11 +2792,6 @@ static int cppc_set_reg_val(int cpu, enum cppc_regs reg_idx, u64 val)
 	return cpc_write(cpu, reg, val);
 }
 
-static bool cppc_desired_perf_readable(const struct cpc_desc *cpc_desc)
-{
-	return cpc_desc->version < CPPC_V4_REV;
-}
-
 /**
  * cppc_get_desired_perf - Get the desired performance register value.
  * @cpunum: CPU from which to get desired performance.
@@ -1427,15 +2802,6 @@ static bool cppc_desired_perf_readable(const struct cpc_desc *cpc_desc)
  */
 int cppc_get_desired_perf(int cpunum, u64 *desired_perf)
 {
-	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum);
-
-	if (!cpc_desc)
-		return -ENODEV;
-
-	/* _CPC revision 4 no longer specifies Desired Performance as readable. */
-	if (!cppc_desired_perf_readable(cpc_desc))
-		return -EOPNOTSUPP;
-
 	return cppc_get_reg_val(cpunum, DESIRED_PERF, desired_perf);
 }
 EXPORT_SYMBOL_GPL(cppc_get_desired_perf);
@@ -1491,7 +2857,7 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 	struct cpc_register_resource *highest_reg, *lowest_reg,
 		*lowest_non_linear_reg, *nominal_reg, *reference_reg,
 		*guaranteed_reg, *low_freq_reg = NULL, *nom_freq_reg = NULL;
-	u64 high, low, guaranteed, nom, ref, min_nonlinear,
+	u64 high, low, guaranteed = 0, nom, ref, min_nonlinear,
 	    low_f = 0, nom_f = 0;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpunum);
 	struct cppc_pcc_data *pcc_ss_data = NULL;
@@ -1574,7 +2940,12 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 		goto out_err;
 	perf_caps->lowest_nonlinear_perf = min_nonlinear;
 
-	if (!high || !low || !nom || !ref || !min_nonlinear) {
+	if (!high || !low || !nom || !ref || !min_nonlinear ||
+	    high > U32_MAX || low > U32_MAX || guaranteed > U32_MAX ||
+	    nom > U32_MAX || ref > U32_MAX || min_nonlinear > U32_MAX ||
+	    high < nom || nom < min_nonlinear || min_nonlinear < low ||
+	    (CPC_SUPPORTED(guaranteed_reg) &&
+	     (guaranteed < low || guaranteed > nom))) {
 		ret = -EFAULT;
 		goto out_err;
 	}
@@ -1591,6 +2962,14 @@ int cppc_get_perf_caps(int cpunum, struct cppc_perf_caps *perf_caps)
 		if (ret)
 			goto out_err;
 	}
+	/* Require ordered anchors and a nonzero slope when frequencies differ. */
+	if (low_f > U32_MAX || nom_f > U32_MAX ||
+	    (low_f && nom_f &&
+	     (nom_f < low_f || nom < low ||
+	      (nom_f != low_f && nom == low)))) {
+		ret = -EFAULT;
+		goto out_err;
+	}
 
 	perf_caps->lowest_freq = low_f;
 	perf_caps->nominal_freq = nom_f;
@@ -1613,6 +2992,9 @@ bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu)
 {
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
 
+	if (!cpc_desc)
+		return false;
+
 	return CPC_IN_PCC(&cpc_desc->cpc_regs[DELIVERED_CTR]) ||
 		CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]) ||
 		CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]);
@@ -1754,8 +3136,10 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable)
 	struct cpc_register_resource *auto_sel_reg;
 	struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu);
 	struct cppc_pcc_data *pcc_ss_data = NULL;
-	bool autosel_ffh_sysmem;
-	bool epp_ffh_sysmem;
+	bool auto_sel_pcc;
+	bool auto_sel_non_pcc;
+	bool epp_pcc;
+	bool epp_non_pcc;
 	int ret;
 
 	if (!cpc_desc) {
@@ -1765,53 +3149,69 @@ int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable)
 
 	auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
 	epp_set_reg = &cpc_desc->cpc_regs[ENERGY_PERF];
+	if (!enable && auto_sel_reg->type == ACPI_TYPE_INTEGER &&
+	    auto_sel_reg->cpc_entry.int_value == 1)
+		return -EOPNOTSUPP;
 
-	epp_ffh_sysmem = CPC_SUPPORTED(epp_set_reg) &&
-		(CPC_IN_FFH(epp_set_reg) || CPC_IN_SYSTEM_MEMORY(epp_set_reg));
-	autosel_ffh_sysmem = CPC_SUPPORTED(auto_sel_reg) &&
-		(CPC_IN_FFH(auto_sel_reg) || CPC_IN_SYSTEM_MEMORY(auto_sel_reg));
+	auto_sel_pcc = cpc_is_writable(auto_sel_reg) &&
+		CPC_IN_PCC(auto_sel_reg);
+	epp_pcc = cpc_is_writable(epp_set_reg) && CPC_IN_PCC(epp_set_reg);
+
+	auto_sel_non_pcc = cpc_is_writable(auto_sel_reg) && !auto_sel_pcc;
+	epp_non_pcc = cpc_is_writable(epp_set_reg) && !epp_pcc;
+
+	/* Complete fallible non-PCC writes before staging PCC data. */
+	if (auto_sel_non_pcc) {
+		ret = cpc_write(cpu, auto_sel_reg, enable);
+		if (ret)
+			return ret;
+	}
+	if (epp_non_pcc) {
+		ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf);
+		if (ret)
+			return ret;
+	}
 
-	if (CPC_IN_PCC(epp_set_reg) || CPC_IN_PCC(auto_sel_reg)) {
+	if (epp_pcc || auto_sel_pcc) {
 		if (pcc_ss_id < 0) {
 			pr_debug("Invalid pcc_ss_id for CPU:%d\n", cpu);
 			return -ENODEV;
 		}
 
-		if (CPC_SUPPORTED(auto_sel_reg)) {
+		pcc_ss_data = pcc_data[pcc_ss_id];
+		if (!pcc_ss_data)
+			return -ENODEV;
+
+		down_write(&pcc_ss_data->pcc_lock);
+
+		ret = check_pcc_chan(pcc_ss_id, false);
+		if (ret)
+			goto out_unlock;
+
+		if (auto_sel_pcc) {
 			ret = cpc_write(cpu, auto_sel_reg, enable);
 			if (ret)
-				return ret;
+				goto out_unlock;
 		}
 
-		if (CPC_SUPPORTED(epp_set_reg)) {
+		if (epp_pcc) {
 			ret = cpc_write(cpu, epp_set_reg, perf_ctrls->energy_perf);
 			if (ret)
-				return ret;
+				goto out_unlock;
 		}
 
-		pcc_ss_data = pcc_data[pcc_ss_id];
-
-		down_write(&pcc_ss_data->pcc_lock);
 		/* after writing CPC, transfer the ownership of PCC to platform */
 		ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE);
-		up_write(&pcc_ss_data->pcc_lock);
-	} else if (osc_cpc_flexible_adr_space_confirmed &&
-		   (epp_ffh_sysmem || autosel_ffh_sysmem)) {
-		if (autosel_ffh_sysmem) {
-			ret = cpc_write(cpu, auto_sel_reg, enable);
-			if (ret)
-				return ret;
-		}
 
-		if (epp_ffh_sysmem) {
-			ret = cpc_write(cpu, epp_set_reg,
-					perf_ctrls->energy_perf);
-			if (ret)
-				return ret;
-		}
+out_unlock:
+		if (ret)
+			cppc_abort_pending_pcc_write(pcc_ss_id, pcc_ss_data, ret);
+		up_write(&pcc_ss_data->pcc_lock);
+	} else if (epp_non_pcc || auto_sel_non_pcc) {
+		ret = 0;
 	} else {
-		ret = -ENOTSUPP;
-		pr_debug("_CPC in PCC/FFH/SystemMemory are not supported\n");
+		ret = -EOPNOTSUPP;
+		pr_debug("No writable EPP controls for CPU:%d\n", cpu);
 	}
 
 	return ret;
@@ -1924,6 +3324,28 @@ int cppc_get_auto_sel(int cpu, bool *enable)
 }
 EXPORT_SYMBOL_GPL(cppc_get_auto_sel);
 
+/**
+ * cppc_auto_sel_is_immutable - Check for always-enabled autonomous selection.
+ * @cpu: CPU whose _CPC descriptor to check.
+ *
+ * Context: Process context.
+ * Return: true for Integer 1, false for a register or an absent descriptor.
+ */
+bool cppc_auto_sel_is_immutable(int cpu)
+{
+	struct cpc_desc *cpc_desc;
+	struct cpc_register_resource *reg;
+
+	guard(mutex)(&cpc_desc_lock);
+	cpc_desc = per_cpu(cpc_desc_ptr, cpu);
+	if (!cpc_desc)
+		return false;
+
+	reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
+	return reg->type == ACPI_TYPE_INTEGER && reg->cpc_entry.int_value == 1;
+}
+EXPORT_SYMBOL_GPL(cppc_auto_sel_is_immutable);
+
 /**
  * cppc_set_auto_sel - Write autonomous selection register.
  * @cpu    : CPU to which to write register.
@@ -1982,6 +3404,7 @@ int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	max_perf_reg = &cpc_desc->cpc_regs[MAX_PERF];
 	energy_perf_reg = &cpc_desc->cpc_regs[ENERGY_PERF];
 	auto_sel_reg = &cpc_desc->cpc_regs[AUTO_SEL_ENABLE];
+	perf_ctrls->min_perf_valid = false;
 
 	/* Are any of the regs PCC ?*/
 	if (CPC_IN_PCC(min_perf_reg) || CPC_IN_PCC(max_perf_reg) ||
@@ -2006,6 +3429,10 @@ int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 		ret = cpc_read(cpu, max_perf_reg, &max);
 		if (ret)
 			goto out_err;
+		if (max > U32_MAX) {
+			ret = -EFAULT;
+			goto out_err;
+		}
 	}
 	perf_ctrls->max_perf = max;
 
@@ -2013,6 +3440,11 @@ int cppc_get_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 		ret = cpc_read(cpu, min_perf_reg, &min);
 		if (ret)
 			goto out_err;
+		if (min > U32_MAX) {
+			ret = -EFAULT;
+			goto out_err;
+		}
+		perf_ctrls->min_perf_valid = true;
 	}
 	perf_ctrls->min_perf = min;
 
@@ -2052,7 +3484,9 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	struct cpc_register_resource *desired_reg, *min_perf_reg, *max_perf_reg;
 	int pcc_ss_id = per_cpu(cpu_pcc_subspace_idx, cpu);
 	struct cppc_pcc_data *pcc_ss_data = NULL;
-	bool regs_in_pcc;
+	bool desired_update, min_update, max_update;
+	bool desired_pcc, min_pcc, max_pcc, pcc_update;
+	bool pcc_layout, direct_layout, mixed_layout;
 	int ret = 0;
 
 	if (!cpc_desc) {
@@ -2063,54 +3497,162 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	desired_reg = &cpc_desc->cpc_regs[DESIRED_PERF];
 	min_perf_reg = &cpc_desc->cpc_regs[MIN_PERF];
 	max_perf_reg = &cpc_desc->cpc_regs[MAX_PERF];
-	regs_in_pcc = CPC_IN_PCC(desired_reg) || CPC_IN_PCC(min_perf_reg) ||
-		      CPC_IN_PCC(max_perf_reg);
-
-	/*
-	 * This is Phase-I where we want to write to CPC registers
-	 * -> We want all CPUs to be able to execute this phase in parallel
-	 *
-	 * Since read_lock can be acquired by multiple CPUs simultaneously we
-	 * achieve that goal here
-	 */
-	if (regs_in_pcc) {
+	desired_update = cpc_is_writable(desired_reg);
+	min_update = cpc_is_writable(min_perf_reg) &&
+		     (perf_ctrls->min_perf || perf_ctrls->min_perf_valid);
+	max_update = cpc_is_writable(max_perf_reg) &&
+		     perf_ctrls->max_perf;
+	desired_pcc = desired_update && CPC_IN_PCC(desired_reg);
+	min_pcc = min_update && CPC_IN_PCC(min_perf_reg);
+	max_pcc = max_update && CPC_IN_PCC(max_perf_reg);
+	pcc_update = desired_pcc || min_pcc || max_pcc;
+	pcc_layout = (cpc_is_writable(desired_reg) && CPC_IN_PCC(desired_reg)) ||
+		     (cpc_is_writable(min_perf_reg) && CPC_IN_PCC(min_perf_reg)) ||
+		     (cpc_is_writable(max_perf_reg) && CPC_IN_PCC(max_perf_reg));
+	direct_layout = (cpc_is_writable(desired_reg) &&
+			 !CPC_IN_PCC(desired_reg)) ||
+			(cpc_is_writable(min_perf_reg) &&
+			 !CPC_IN_PCC(min_perf_reg)) ||
+			(cpc_is_writable(max_perf_reg) &&
+			 !CPC_IN_PCC(max_perf_reg));
+	mixed_layout = pcc_layout && direct_layout;
+
+	if (mixed_layout || pcc_update) {
 		if (pcc_ss_id < 0) {
 			pr_debug("Invalid pcc_ss_id\n");
 			return -ENODEV;
 		}
 		pcc_ss_data = pcc_data[pcc_ss_id];
-		down_read(&pcc_ss_data->pcc_lock); /* BEGIN Phase-I */
+		if (!pcc_ss_data)
+			return -ENODEV;
+	}
+
+	/*
+	 * A mixed layout cannot batch fallible direct writes safely: another
+	 * CPU's staged PCC values may no longer match if a direct write fails.
+	 * Serialize the complete mixed transaction and drain an older batch
+	 * before changing a direct control.
+	 */
+	if (mixed_layout) {
+		down_write(&pcc_ss_data->pcc_lock);
+		if (pcc_ss_data->pending_pcc_write_cmd) {
+			ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+
 		if (pcc_ss_data->platform_owns_pcc) {
 			ret = check_pcc_chan(pcc_ss_id, false);
-			if (ret) {
-				up_read(&pcc_ss_data->pcc_lock);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+
+		if (desired_update && !desired_pcc) {
+			ret = cpc_write(cpu, desired_reg,
+					perf_ctrls->desired_perf);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+		if (min_update && !min_pcc) {
+			ret = cpc_write(cpu, min_perf_reg,
+					perf_ctrls->min_perf);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+		if (max_update && !max_pcc) {
+			ret = cpc_write(cpu, max_perf_reg,
+					perf_ctrls->max_perf);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+
+		if (desired_pcc) {
+			ret = cpc_write(cpu, desired_reg,
+					perf_ctrls->desired_perf);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+		if (min_pcc) {
+			ret = cpc_write(cpu, min_perf_reg,
+					perf_ctrls->min_perf);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+		if (max_pcc) {
+			ret = cpc_write(cpu, max_perf_reg,
+					perf_ctrls->max_perf);
+			if (ret)
+				goto out_mixed_unlock;
+		}
+
+		if (pcc_update) {
+			WRITE_ONCE(pcc_ss_data->pending_pcc_write_cmd, true);
+			cpc_desc->write_cmd_id = pcc_ss_data->pcc_write_cnt;
+			cpc_desc->write_cmd_status = 0;
+			ret = send_pcc_cmd(pcc_ss_id, CMD_WRITE);
+		}
+
+out_mixed_unlock:
+		up_write(&pcc_ss_data->pcc_lock);
+		return ret;
+	}
+
+	/* A request without PCC updates has no payload to coordinate. */
+	if (!pcc_update) {
+		if (desired_update) {
+			ret = cpc_write(cpu, desired_reg,
+					perf_ctrls->desired_perf);
+			if (ret)
 				return ret;
-			}
 		}
-		/*
-		 * Update the pending_write to make sure a PCC CMD_READ will not
-		 * arrive and steal the channel during the switch to write lock
-		 */
-		pcc_ss_data->pending_pcc_write_cmd = true;
-		cpc_desc->write_cmd_id = pcc_ss_data->pcc_write_cnt;
-		cpc_desc->write_cmd_status = 0;
+		if (min_update) {
+			ret = cpc_write(cpu, min_perf_reg,
+					perf_ctrls->min_perf);
+			if (ret)
+				return ret;
+		}
+		if (max_update)
+			ret = cpc_write(cpu, max_perf_reg,
+					perf_ctrls->max_perf);
+		return ret;
 	}
 
-	if (CPC_SUPPORTED(desired_reg))
-		cpc_write(cpu, desired_reg, perf_ctrls->desired_perf);
+	down_read(&pcc_ss_data->pcc_lock); /* BEGIN Phase-I */
+	if (pcc_ss_data->platform_owns_pcc) {
+		ret = check_pcc_chan(pcc_ss_id, false);
+		if (ret)
+			goto out_pcc_read_unlock;
+	}
 
 	/*
-	 * Only write if min_perf and max_perf not zero. Some drivers pass zero
-	 * value to min and max perf, but they don't mean to set the zero value,
-	 * they just don't want to write to those registers.
+	 * This is Phase-I where we want to write to CPC registers
+	 * -> We want all CPUs to be able to execute this phase in parallel
+	 *
+	 * Since read_lock can be acquired by multiple CPUs simultaneously we
+	 * achieve that goal here.
 	 */
-	if (perf_ctrls->min_perf && CPC_SUPPORTED(min_perf_reg))
-		cpc_write(cpu, min_perf_reg, perf_ctrls->min_perf);
-	if (perf_ctrls->max_perf && CPC_SUPPORTED(max_perf_reg))
-		cpc_write(cpu, max_perf_reg, perf_ctrls->max_perf);
+	if (desired_pcc) {
+		ret = cpc_write(cpu, desired_reg, perf_ctrls->desired_perf);
+		if (ret)
+			goto out_pcc_read_unlock;
+	}
 
-	if (regs_in_pcc)
-		up_read(&pcc_ss_data->pcc_lock);	/* END Phase-I */
+	if (min_pcc) {
+		ret = cpc_write(cpu, min_perf_reg, perf_ctrls->min_perf);
+		if (ret)
+			goto out_pcc_read_unlock;
+	}
+	if (max_pcc) {
+		ret = cpc_write(cpu, max_perf_reg, perf_ctrls->max_perf);
+		if (ret)
+			goto out_pcc_read_unlock;
+	}
+
+	/* Block a PCC read until the staged payload has been submitted. */
+	WRITE_ONCE(pcc_ss_data->pending_pcc_write_cmd, true);
+	cpc_desc->write_cmd_id = pcc_ss_data->pcc_write_cnt;
+	cpc_desc->write_cmd_status = 0;
+	up_read(&pcc_ss_data->pcc_lock);	/* END Phase-I */
 	/*
 	 * This is Phase-II where we transfer the ownership of PCC to Platform
 	 *
@@ -2157,20 +3699,22 @@ int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls)
 	 * case during a CMD_READ and if there are pending writes it delivers
 	 * the write command before servicing the read command
 	 */
-	if (regs_in_pcc) {
-		if (down_write_trylock(&pcc_ss_data->pcc_lock)) {/* BEGIN Phase-II */
-			/* Update only if there are pending write commands */
-			if (pcc_ss_data->pending_pcc_write_cmd)
-				send_pcc_cmd(pcc_ss_id, CMD_WRITE);
-			up_write(&pcc_ss_data->pcc_lock);	/* END Phase-II */
-		} else
-			/* Wait until pcc_write_cnt is updated by send_pcc_cmd */
-			wait_event(pcc_ss_data->pcc_write_wait_q,
-				   cpc_desc->write_cmd_id != pcc_ss_data->pcc_write_cnt);
-
-		/* send_pcc_cmd updates the status in case of failure */
-		ret = cpc_desc->write_cmd_status;
+	if (down_write_trylock(&pcc_ss_data->pcc_lock)) {/* BEGIN Phase-II */
+		/* Update only if there are pending write commands */
+		if (pcc_ss_data->pending_pcc_write_cmd)
+			send_pcc_cmd(pcc_ss_id, CMD_WRITE);
+		up_write(&pcc_ss_data->pcc_lock);	/* END Phase-II */
+	} else {
+		/* Wait until pcc_write_cnt is updated by send_pcc_cmd */
+		wait_event(pcc_ss_data->pcc_write_wait_q,
+			   cpc_desc->write_cmd_id != pcc_ss_data->pcc_write_cnt);
 	}
+
+	/* send_pcc_cmd updates the status in case of failure */
+	return cpc_desc->write_cmd_status;
+
+out_pcc_read_unlock:
+	up_read(&pcc_ss_data->pcc_lock);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(cppc_set_perf);
@@ -2194,7 +3738,7 @@ EXPORT_SYMBOL_GPL(cppc_get_perf_limited);
 /**
  * cppc_set_perf_limited() - Clear bits in the Performance Limited register.
  * @cpu: CPU on which to write register.
- * @bits_to_clear: Bitmask of bits to clear in the perf_limited register.
+ * @bits_to_clear: Zero for no-op or CPPC_PERF_LIMITED_MASK to clear both bits.
  *
  * The Performance Limited register contains two sticky bits set by platform:
  *   - Bit 0 (Desired_Excursion): Set when delivered performance is constrained
@@ -2203,31 +3747,29 @@ EXPORT_SYMBOL_GPL(cppc_get_perf_limited);
  *     below minimum performance.
  *
  * These bits are sticky and remain set until OSPM explicitly clears them.
- * This function only allows clearing bits (the platform sets them).
+ * Selective clears are unsupported because they require an interlocked RMW.
  *
  * Return: 0 for success, -EINVAL for invalid bits, -EIO on register
  *         access failure, -EOPNOTSUPP if not supported.
  */
 int cppc_set_perf_limited(int cpu, u64 bits_to_clear)
 {
-	u64 current_val, new_val;
-	int ret;
-
 	/* Only bits 0 and 1 are valid */
-	if (bits_to_clear & ~CPPC_PERF_LIMITED_MASK)
+	if (bits_to_clear & ~(u64)CPPC_PERF_LIMITED_MASK)
 		return -EINVAL;
 
 	if (!bits_to_clear)
 		return 0;
 
-	ret = cppc_get_perf_limited(cpu, &current_val);
-	if (ret)
-		return ret;
-
-	/* Clear the specified bits */
-	new_val = current_val & ~bits_to_clear;
+	/*
+	 * Writing zero clears both bits without depending on how a platform
+	 * treats written ones. ACPI does not define the effect of writing one,
+	 * so a selective clear cannot be implemented without an interlocked RMW.
+	 */
+	if (bits_to_clear != CPPC_PERF_LIMITED_MASK)
+		return -EOPNOTSUPP;
 
-	return cppc_set_reg_val(cpu, PERF_LIMITED, new_val);
+	return cppc_set_reg_val(cpu, PERF_LIMITED, 0);
 }
 EXPORT_SYMBOL_GPL(cppc_set_perf_limited);
 
@@ -2267,6 +3809,9 @@ int cppc_get_transition_latency(int cpu_num)
 		return -ENODATA;
 
 	desired_reg = &cpc_desc->cpc_regs[DESIRED_PERF];
+	if (!cpc_is_writable(desired_reg))
+		return -ENODATA;
+
 	if (CPC_IN_SYSTEM_MEMORY(desired_reg) || CPC_IN_SYSTEM_IO(desired_reg))
 		return 0;
 
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index aa55ecfc2923cf..76104f715efa11 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -23,6 +23,8 @@
 #include "fan.h"
 #include "internal.h"
 
+#define ACPI_D_STATE_INVALID	ACPI_D_STATE_COUNT
+
 /**
  * acpi_power_state_string - String representation of ACPI device power state.
  * @state: ACPI device power state to return the string representation of.
@@ -75,15 +77,14 @@ static int acpi_dev_pm_explicit_get(struct acpi_device *device, int *state)
 int acpi_device_get_power(struct acpi_device *device, int *state)
 {
 	int result = ACPI_STATE_UNKNOWN;
-	struct acpi_device *parent;
 	int error;
 
 	if (!device || !state)
 		return -EINVAL;
 
-	parent = acpi_dev_parent(device);
-
 	if (!device->flags.power_manageable) {
+		struct acpi_device *parent = acpi_dev_parent(device);
+
 		/* TBD: Non-recursive algorithm for walking up hierarchy. */
 		*state = parent ? parent->power.state : ACPI_STATE_D0;
 		goto out;
@@ -119,16 +120,6 @@ int acpi_device_get_power(struct acpi_device *device, int *state)
 			result = psc > ACPI_STATE_D2 ? ACPI_STATE_D3_HOT : psc;
 	}
 
-	/*
-	 * If we were unsure about the device parent's power state up to this
-	 * point, the fact that the device is in D0 implies that the parent has
-	 * to be in D0 too, except if ignore_parent is set.
-	 */
-	if (!device->power.flags.ignore_parent && parent &&
-	    parent->power.state == ACPI_STATE_UNKNOWN &&
-	    result == ACPI_STATE_D0)
-		parent->power.state = ACPI_STATE_D0;
-
 	*state = result;
 
  out:
@@ -304,24 +295,28 @@ int acpi_bus_set_power(acpi_handle handle, int state)
 }
 EXPORT_SYMBOL(acpi_bus_set_power);
 
-int acpi_bus_init_power(struct acpi_device *device)
+static int acpi_device_init_power(struct acpi_device *device)
 {
 	int state;
 	int result;
 
-	if (!device)
-		return -EINVAL;
-
-	device->power.state = ACPI_STATE_UNKNOWN;
-	if (!acpi_device_is_present(device)) {
-		device->flags.initialized = false;
-		return -ENXIO;
-	}
-
 	result = acpi_device_get_power(device, &state);
 	if (result)
 		return result;
 
+	/*
+	 * If the current power state of the device is D0 and it has a parent
+	 * whose power state is not ignored, and the parent's power state
+	 * initialization has failed, the parent's power state can be updated to
+	 * D0 for consistency.
+	 */
+	if (!device->power.flags.ignore_parent && state == ACPI_STATE_D0) {
+		struct acpi_device *parent = acpi_dev_parent(device);
+
+		if (parent && parent->power.state == ACPI_D_STATE_INVALID)
+			parent->power.state = ACPI_STATE_D0;
+	}
+
 	if (state < ACPI_STATE_D3_COLD && device->power.flags.power_resources) {
 		/* Reference count the power resources. */
 		result = acpi_power_on_resources(device, state);
@@ -351,9 +346,36 @@ int acpi_bus_init_power(struct acpi_device *device)
 		state = ACPI_STATE_D0;
 	}
 	device->power.state = state;
+
+	acpi_handle_debug(device->handle, "Initial power state: %s\n",
+			  acpi_power_state_string(state));
+
 	return 0;
 }
 
+int acpi_bus_init_power(struct acpi_device *device)
+{
+	int result;
+
+	if (device->power.state != ACPI_STATE_UNKNOWN)
+		return 0;
+
+	/*
+	 * The ACPI device power state can be only initialized once.  If this
+	 * fails, ACPI power management will not be used for the device going
+	 * forward.
+	 */
+	result = acpi_device_init_power(device);
+	if (result) {
+		device->flags.power_manageable = 0;
+		device->power.state = ACPI_D_STATE_INVALID;
+		acpi_handle_info(device->handle,
+				 "Initial power state undetermined, ACPI PM disabled\n");
+	}
+
+	return result;
+}
+
 /**
  * acpi_device_fix_up_power - Force device with missing _PSC into D0.
  * @device: Device object whose power state is to be fixed up.
@@ -475,8 +497,13 @@ static int acpi_power_up_if_adr_present(struct acpi_device *adev, void *not_used
 	if (!(adev->flags.power_manageable && adev->pnp.type.bus_address))
 		return 0;
 
-	acpi_handle_debug(adev->handle, "Power state: %s\n",
-			  acpi_power_state_string(adev->power.state));
+	/*
+	 * This is done during the PCI root initialization which occurs before
+	 * acpi_bus_attach() is called for the device, so the ACPI power state
+	 * of the device needs to be initialized here.
+	 */
+	if (acpi_bus_init_power(adev))
+		return 0;
 
 	if (adev->power.state == ACPI_STATE_D3_COLD)
 		return acpi_device_set_power(adev, ACPI_STATE_D0);
diff --git a/drivers/acpi/fan.h b/drivers/acpi/fan.h
index e20d6ad9df8050..3faa247af5148d 100644
--- a/drivers/acpi/fan.h
+++ b/drivers/acpi/fan.h
@@ -52,7 +52,7 @@ struct acpi_fan_fst {
 };
 
 struct acpi_fan {
-	acpi_handle handle;
+	struct acpi_device *adev;
 	bool acpi4;
 	bool has_fst;
 	struct acpi_fan_fif fif;
diff --git a/drivers/acpi/fan_core.c b/drivers/acpi/fan_core.c
index 624d0736b581cd..3674f9613d40df 100644
--- a/drivers/acpi/fan_core.c
+++ b/drivers/acpi/fan_core.c
@@ -54,8 +54,7 @@ MODULE_DEVICE_TABLE(acpi, fan_device_ids);
 static int fan_get_max_state(struct thermal_cooling_device *cdev, unsigned long
 			     *state)
 {
-	struct acpi_device *device = cdev->devdata;
-	struct acpi_fan *fan = acpi_driver_data(device);
+	struct acpi_fan *fan = cdev->devdata;
 
 	if (fan->acpi4) {
 		if (fan->fif.fine_grain_ctrl)
@@ -72,42 +71,34 @@ static int fan_get_max_state(struct thermal_cooling_device *cdev, unsigned long
 int acpi_fan_get_fst(acpi_handle handle, struct acpi_fan_fst *fst)
 {
 	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *obj;
 	acpi_status status;
-	int ret = 0;
 
 	status = acpi_evaluate_object(handle, "_FST", NULL, &buffer);
 	if (ACPI_FAILURE(status))
-		return -EIO;
+		return -ENXIO;
 
-	obj = buffer.pointer;
+	union acpi_object *obj __free(acpi_object_free) = buffer.pointer;
 	if (!obj)
 		return -ENODATA;
 
-	if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != 3) {
-		ret = -EPROTO;
-		goto err;
-	}
+	if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != 3)
+		return -EPROTO;
 
 	if (obj->package.elements[0].type != ACPI_TYPE_INTEGER ||
 	    obj->package.elements[1].type != ACPI_TYPE_INTEGER ||
-	    obj->package.elements[2].type != ACPI_TYPE_INTEGER) {
-		ret = -EPROTO;
-		goto err;
-	}
+	    obj->package.elements[2].type != ACPI_TYPE_INTEGER)
+		return -EPROTO;
 
 	fst->revision = obj->package.elements[0].integer.value;
 	fst->control = obj->package.elements[1].integer.value;
 	fst->speed = obj->package.elements[2].integer.value;
 
-err:
-	kfree(obj);
-	return ret;
+	return 0;
 }
 
-static int fan_get_state_acpi4(struct acpi_device *device, unsigned long *state)
+static int fan_get_state_acpi4(struct acpi_fan *fan, unsigned long *state)
 {
-	struct acpi_fan *fan = acpi_driver_data(device);
+	struct acpi_device *device = fan->adev;
 	struct acpi_fan_fst fst;
 	int status, i;
 
@@ -159,13 +150,12 @@ static int fan_get_state(struct acpi_device *device, unsigned long *state)
 static int fan_get_cur_state(struct thermal_cooling_device *cdev, unsigned long
 			     *state)
 {
-	struct acpi_device *device = cdev->devdata;
-	struct acpi_fan *fan = acpi_driver_data(device);
+	struct acpi_fan *fan = cdev->devdata;
 
 	if (fan->acpi4)
-		return fan_get_state_acpi4(device, state);
+		return fan_get_state_acpi4(fan, state);
 	else
-		return fan_get_state(device, state);
+		return fan_get_state(fan->adev, state);
 }
 
 static int fan_set_state(struct acpi_device *device, unsigned long state)
@@ -177,9 +167,9 @@ static int fan_set_state(struct acpi_device *device, unsigned long state)
 				     state ? ACPI_STATE_D0 : ACPI_STATE_D3_COLD);
 }
 
-static int fan_set_state_acpi4(struct acpi_device *device, unsigned long state)
+static int fan_set_state_acpi4(struct acpi_fan *fan, unsigned long state)
 {
-	struct acpi_fan *fan = acpi_driver_data(device);
+	struct acpi_device *device = fan->adev;
 	acpi_status status;
 	u64 value = state;
 	int max_state;
@@ -213,13 +203,12 @@ static int fan_set_state_acpi4(struct acpi_device *device, unsigned long state)
 static int
 fan_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state)
 {
-	struct acpi_device *device = cdev->devdata;
-	struct acpi_fan *fan = acpi_driver_data(device);
+	struct acpi_fan *fan = cdev->devdata;
 
 	if (fan->acpi4)
-		return fan_set_state_acpi4(device, state);
+		return fan_set_state_acpi4(fan, state);
 	else
-		return fan_set_state(device, state);
+		return fan_set_state(fan->adev, state);
 }
 
 static const struct thermal_cooling_device_ops fan_cooling_ops = {
@@ -240,25 +229,22 @@ static int acpi_fan_get_fif(struct acpi_device *device)
 	struct acpi_buffer format = { sizeof("NNNN"), "NNNN" };
 	u64 fields[4];
 	struct acpi_buffer fif = { sizeof(fields), fields };
-	union acpi_object *obj;
 	acpi_status status;
 
 	status = acpi_evaluate_object(device->handle, "_FIF", NULL, &buffer);
 	if (ACPI_FAILURE(status))
-		return status;
+		return -ENXIO;
 
-	obj = buffer.pointer;
+	union acpi_object *obj __free(acpi_object_free) = buffer.pointer;
 	if (!obj || obj->type != ACPI_TYPE_PACKAGE) {
 		dev_err(&device->dev, "Invalid _FIF data\n");
-		status = -EINVAL;
-		goto err;
+		return -ENODATA;
 	}
 
 	status = acpi_extract_package(obj, &format, &fif);
 	if (ACPI_FAILURE(status)) {
 		dev_err(&device->dev, "Invalid _FIF element\n");
-		status = -EINVAL;
-		goto err;
+		return -ENODATA;
 	}
 
 	fan->fif.revision = fields[0];
@@ -272,9 +258,8 @@ static int acpi_fan_get_fif(struct acpi_device *device)
 	/* If step size > 9, change to 9 (by spec valid values 1-9) */
 	else if (fan->fif.step_size > 9)
 		fan->fif.step_size = 9;
-err:
-	kfree(obj);
-	return status;
+
+	return 0;
 }
 
 static int acpi_fan_speed_cmp(const void *a, const void *b)
@@ -284,34 +269,28 @@ static int acpi_fan_speed_cmp(const void *a, const void *b)
 	return fps1->speed - fps2->speed;
 }
 
-static int acpi_fan_get_fps(struct acpi_device *device)
+static int acpi_fan_get_fps(struct device *dev, struct acpi_device *device)
 {
 	struct acpi_fan *fan = acpi_driver_data(device);
 	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *obj;
 	acpi_status status;
 	int i;
 
 	status = acpi_evaluate_object(device->handle, "_FPS", NULL, &buffer);
 	if (ACPI_FAILURE(status))
-		return status;
+		return -ENXIO;
 
-	obj = buffer.pointer;
+	union acpi_object *obj __free(acpi_object_free) = buffer.pointer;
 	if (!obj || obj->type != ACPI_TYPE_PACKAGE || obj->package.count < 2) {
 		dev_err(&device->dev, "Invalid _FPS data\n");
-		status = -EINVAL;
-		goto err;
+		return -ENODATA;
 	}
 
 	fan->fps_count = obj->package.count - 1; /* minus revision field */
-	fan->fps = devm_kcalloc(&device->dev,
-				fan->fps_count, sizeof(struct acpi_fan_fps),
-				GFP_KERNEL);
-	if (!fan->fps) {
-		dev_err(&device->dev, "Not enough memory\n");
-		status = -ENOMEM;
-		goto err;
-	}
+	fan->fps = devm_kcalloc(dev, fan->fps_count, sizeof(*fan->fps), GFP_KERNEL);
+	if (!fan->fps)
+		return -ENOMEM;
+
 	for (i = 0; i < fan->fps_count; i++) {
 		struct acpi_buffer format = { sizeof("NNNNN"), "NNNNN" };
 		struct acpi_buffer fps = { offsetof(struct acpi_fan_fps, name),
@@ -320,7 +299,7 @@ static int acpi_fan_get_fps(struct acpi_device *device)
 					      &format, &fps);
 		if (ACPI_FAILURE(status)) {
 			dev_err(&device->dev, "Invalid _FPS element\n");
-			goto err;
+			return -ENODATA;
 		}
 	}
 
@@ -328,9 +307,7 @@ static int acpi_fan_get_fps(struct acpi_device *device)
 	sort(fan->fps, fan->fps_count, sizeof(*fan->fps),
 	     acpi_fan_speed_cmp, NULL);
 
-err:
-	kfree(obj);
-	return status;
+	return 0;
 }
 
 static int acpi_fan_dsm_init(struct device *dev)
@@ -343,30 +320,28 @@ static int acpi_fan_dsm_init(struct device *dev)
 		},
 	};
 	struct acpi_fan *fan = dev_get_drvdata(dev);
-	union acpi_object *obj;
-	int ret = 0;
+	acpi_handle fan_handle = fan->adev->handle;
 
-	if (!acpi_check_dsm(fan->handle, &acpi_fan_microsoft_guid, 0,
+	if (!acpi_check_dsm(fan_handle, &acpi_fan_microsoft_guid, 0,
 			    BIT(ACPI_FAN_DSM_GET_TRIP_POINT_GRANULARITY) |
 			    BIT(ACPI_FAN_DSM_SET_TRIP_POINTS)))
 		return 0;
 
 	dev_info(dev, "Using Microsoft fan extensions\n");
 
-	obj = acpi_evaluate_dsm_typed(fan->handle, &acpi_fan_microsoft_guid, 0,
-				      ACPI_FAN_DSM_GET_TRIP_POINT_GRANULARITY, &dummy,
-				      ACPI_TYPE_INTEGER);
+	union acpi_object *obj __free(acpi_object_free) =
+		acpi_evaluate_dsm_typed(fan_handle, &acpi_fan_microsoft_guid, 0,
+					ACPI_FAN_DSM_GET_TRIP_POINT_GRANULARITY,
+					&dummy, ACPI_TYPE_INTEGER);
 	if (!obj)
-		return -EIO;
+		return -ENXIO;
 
 	if (obj->integer.value > U32_MAX)
-		ret = -EOVERFLOW;
-	else
-		fan->fan_trip_granularity = obj->integer.value;
+		return -EOVERFLOW;
 
-	kfree(obj);
+	fan->fan_trip_granularity = obj->integer.value;
 
-	return ret;
+	return 0;
 }
 
 static int acpi_fan_dsm_set_trip_points(struct device *dev, u64 upper, u64 lower)
@@ -395,9 +370,9 @@ static int acpi_fan_dsm_set_trip_points(struct device *dev, u64 upper, u64 lower
 	};
 	union acpi_object *obj;
 
-	obj = acpi_evaluate_dsm(fan->handle, &acpi_fan_microsoft_guid, 0,
-				ACPI_FAN_DSM_SET_TRIP_POINTS, &in);
-	kfree(obj);
+	obj = acpi_evaluate_dsm(fan->adev->handle, &acpi_fan_microsoft_guid,
+				0, ACPI_FAN_DSM_SET_TRIP_POINTS, &in);
+	ACPI_FREE(obj);
 
 	return 0;
 }
@@ -506,7 +481,7 @@ static int acpi_fan_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 
-	fan->handle = device->handle;
+	fan->adev = device;
 	device->driver_data = fan;
 	platform_set_drvdata(pdev, fan);
 
@@ -522,7 +497,7 @@ static int acpi_fan_probe(struct platform_device *pdev)
 		if (result)
 			return result;
 
-		result = acpi_fan_get_fps(device);
+		result = acpi_fan_get_fps(&pdev->dev, device);
 		if (result)
 			return result;
 	}
@@ -567,8 +542,7 @@ static int acpi_fan_probe(struct platform_device *pdev)
 	else
 		name = acpi_device_bid(device);
 
-	cdev = thermal_cooling_device_register(name, device,
-						&fan_cooling_ops);
+	cdev = thermal_cooling_device_create(&pdev->dev, name, fan, &fan_cooling_ops);
 	if (IS_ERR(cdev)) {
 		result = PTR_ERR(cdev);
 		goto err_end;
@@ -577,28 +551,9 @@ static int acpi_fan_probe(struct platform_device *pdev)
 	dev_dbg(&pdev->dev, "registered as cooling_device%d\n", cdev->id);
 
 	fan->cdev = cdev;
-	result = sysfs_create_link(&pdev->dev.kobj,
-				   &cdev->device.kobj,
-				   "thermal_cooling");
-	if (result) {
-		dev_err(&pdev->dev, "Failed to create sysfs link 'thermal_cooling'\n");
-		goto err_unregister;
-	}
-
-	result = sysfs_create_link(&cdev->device.kobj,
-				   &pdev->dev.kobj,
-				   "device");
-	if (result) {
-		dev_err(&pdev->dev, "Failed to create sysfs link 'device'\n");
-		goto err_remove_link;
-	}
 
 	return 0;
 
-err_remove_link:
-	sysfs_remove_link(&pdev->dev.kobj, "thermal_cooling");
-err_unregister:
-	thermal_cooling_device_unregister(cdev);
 err_end:
 	if (fan->has_fst)
 		acpi_fan_delete_attributes(device);
@@ -615,8 +570,6 @@ static void acpi_fan_remove(struct platform_device *pdev)
 
 		acpi_fan_delete_attributes(device);
 	}
-	sysfs_remove_link(&pdev->dev.kobj, "thermal_cooling");
-	sysfs_remove_link(&fan->cdev->device.kobj, "device");
 	thermal_cooling_device_unregister(fan->cdev);
 }
 
diff --git a/drivers/acpi/fan_hwmon.c b/drivers/acpi/fan_hwmon.c
index d3374f8f524b72..c5d8419f41c559 100644
--- a/drivers/acpi/fan_hwmon.c
+++ b/drivers/acpi/fan_hwmon.c
@@ -94,7 +94,7 @@ static int acpi_fan_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
 	struct acpi_fan_fst fst;
 	int ret;
 
-	ret = acpi_fan_get_fst(fan->handle, &fst);
+	ret = acpi_fan_get_fst(fan->adev->handle, &fst);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/acpi/glue.c b/drivers/acpi/glue.c
index b1776809279ddb..0959f58cb77c65 100644
--- a/drivers/acpi/glue.c
+++ b/drivers/acpi/glue.c
@@ -59,19 +59,37 @@ int unregister_acpi_bus_type(struct acpi_bus_type *type)
 }
 EXPORT_SYMBOL_GPL(unregister_acpi_bus_type);
 
-static struct acpi_bus_type *acpi_get_bus_type(struct device *dev)
+static struct acpi_device *acpi_companion_lookup(struct device *dev)
 {
-	struct acpi_bus_type *tmp, *ret = NULL;
+	struct acpi_bus_type *type;
 
-	down_read(&bus_type_sem);
-	list_for_each_entry(tmp, &bus_type_list, list) {
-		if (tmp->match(dev)) {
-			ret = tmp;
-			break;
+	if (!dev->type)
+		return NULL;
+
+	guard(rwsem_read)(&bus_type_sem);
+
+	list_for_each_entry(type, &bus_type_list, list) {
+		struct acpi_device *adev;
+
+		if (!type->match(dev))
+			continue;
+
+		adev = type->find_companion(dev);
+		if (!adev) {
+			dev_dbg(dev, "ACPI companion not found\n");
+			return NULL;
 		}
+		if (acpi_bind_one(dev, adev)) {
+			dev_dbg(dev, "Binding to ACPI companion failed\n");
+			return NULL;
+		}
+		if (type->setup)
+			type->setup(dev);
+
+		return adev;
 	}
-	up_read(&bus_type_sem);
-	return ret;
+
+	return NULL;
 }
 
 #define FIND_CHILD_MIN_SCORE	1
@@ -228,31 +246,25 @@ static void acpi_physnode_link_name(char *buf, unsigned int node_id)
 int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
 {
 	struct acpi_device_physical_node *physical_node, *pn;
+	struct acpi_device *comp_dev = ACPI_COMPANION(dev);
 	char physical_node_name[PHYSICAL_NODE_NAME_SIZE];
 	struct list_head *physnode_list;
 	unsigned int node_id;
 	int retval = -EINVAL;
 
-	if (has_acpi_companion(dev)) {
-		if (acpi_dev) {
-			dev_warn(dev, "ACPI companion already set\n");
+	if (!acpi_dev) {
+		if (!comp_dev)
 			return -EINVAL;
-		} else {
-			acpi_dev = ACPI_COMPANION(dev);
-		}
-	}
-	if (!acpi_dev)
-		return -EINVAL;
 
-	acpi_dev_get(acpi_dev);
-	get_device(dev);
-	physical_node = kzalloc_obj(*physical_node);
-	if (!physical_node) {
-		retval = -ENOMEM;
-		goto err;
+		/* If the companion has been set upfront, pick it up. */
+		acpi_dev = comp_dev;
+	} else if (comp_dev && acpi_dev != comp_dev) {
+		dev_warn(dev, "ACPI companion already set to %s which is not %s\n",
+			 acpi_dev_name(comp_dev), acpi_dev_name(acpi_dev));
+		return -EEXIST;
 	}
 
-	mutex_lock(&acpi_dev->physical_node_lock);
+	guard(mutex)(&acpi_dev->physical_node_lock);
 
 	/*
 	 * Keep the list sorted by node_id so that the IDs of removed nodes can
@@ -263,15 +275,12 @@ int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
 	list_for_each_entry(pn, &acpi_dev->physical_node_list, node) {
 		/* Sanity check. */
 		if (pn->dev == dev) {
-			mutex_unlock(&acpi_dev->physical_node_lock);
-
-			dev_warn(dev, "Already associated with ACPI node\n");
-			kfree(physical_node);
-			if (ACPI_COMPANION(dev) != acpi_dev)
-				goto err;
-
-			put_device(dev);
-			acpi_dev_put(acpi_dev);
+			if (!comp_dev) {
+				/* Really unexpected. */
+				ACPI_COMPANION_SET(dev, acpi_dev);
+				dev_warn(&acpi_dev->dev,
+					 "Physical device list corruption fixed up\n");
+			}
 			return 0;
 		}
 		if (pn->node_id == node_id) {
@@ -280,12 +289,19 @@ int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
 		}
 	}
 
+	physical_node = kzalloc_obj(*physical_node);
+	if (!physical_node)
+		return -ENOMEM;
+
+	acpi_dev_get(acpi_dev);
+	get_device(dev);
+
 	physical_node->node_id = node_id;
 	physical_node->dev = dev;
 	list_add(&physical_node->node, physnode_list);
 	acpi_dev->physical_node_count++;
 
-	if (!has_acpi_companion(dev))
+	if (!comp_dev)
 		ACPI_COMPANION_SET(dev, acpi_dev);
 
 	acpi_physnode_link_name(physical_node_name, node_id);
@@ -301,28 +317,20 @@ int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev)
 		dev_err(dev, "Failed to create link firmware_node (%d)\n",
 			retval);
 
-	mutex_unlock(&acpi_dev->physical_node_lock);
-
 	if (acpi_dev->wakeup.flags.valid)
 		device_set_wakeup_capable(dev, true);
 
 	return 0;
-
- err:
-	ACPI_COMPANION_SET(dev, NULL);
-	put_device(dev);
-	acpi_dev_put(acpi_dev);
-	return retval;
 }
 EXPORT_SYMBOL_GPL(acpi_bind_one);
 
-int acpi_unbind_one(struct device *dev)
+void acpi_unbind_one(struct device *dev)
 {
 	struct acpi_device *acpi_dev = ACPI_COMPANION(dev);
 	struct acpi_device_physical_node *entry;
 
 	if (!acpi_dev)
-		return 0;
+		return;
 
 	mutex_lock(&acpi_dev->physical_node_lock);
 
@@ -337,15 +345,17 @@ int acpi_unbind_one(struct device *dev)
 			sysfs_remove_link(&acpi_dev->dev.kobj, physnode_name);
 			sysfs_remove_link(&dev->kobj, "firmware_node");
 			ACPI_COMPANION_SET(dev, NULL);
+
+			mutex_unlock(&acpi_dev->physical_node_lock);
+
 			/* Drop references taken by acpi_bind_one(). */
 			put_device(dev);
 			acpi_dev_put(acpi_dev);
 			kfree(entry);
-			break;
+			return;
 		}
 
 	mutex_unlock(&acpi_dev->physical_node_lock);
-	return 0;
 }
 EXPORT_SYMBOL_GPL(acpi_unbind_one);
 
@@ -354,48 +364,29 @@ void acpi_device_notify(struct device *dev)
 	struct acpi_device *adev;
 	int ret;
 
+	/* ACPI devices have no ACPI companions. */
+	if (dev->bus == &acpi_bus_type)
+		return;
+
 	ret = acpi_bind_one(dev, NULL);
 	if (ret) {
-		struct acpi_bus_type *type = acpi_get_bus_type(dev);
-
-		if (!type)
-			goto err;
-
-		adev = type->find_companion(dev);
-		if (!adev) {
-			dev_dbg(dev, "ACPI companion not found\n");
-			goto err;
-		}
-		ret = acpi_bind_one(dev, adev);
-		if (ret)
-			goto err;
-
-		if (type->setup) {
-			type->setup(dev);
-			goto done;
-		}
+		adev = acpi_companion_lookup(dev);
+		if (!adev)
+			return;
 	} else {
 		adev = ACPI_COMPANION(dev);
 
 		if (dev_is_pci(dev)) {
 			pci_acpi_setup(dev, adev);
-			goto done;
 		} else if (dev_is_platform(dev)) {
 			acpi_configure_pmsi_domain(dev);
+
+			if (adev->handler && adev->handler->bind)
+				adev->handler->bind(dev);
 		}
 	}
 
-	if (adev->handler && adev->handler->bind)
-		adev->handler->bind(dev);
-
-done:
-	acpi_handle_debug(ACPI_HANDLE(dev), "Bound to device %s\n",
-			  dev_name(dev));
-
-	return;
-
-err:
-	dev_dbg(dev, "No ACPI support\n");
+	dev_dbg(dev, "Bound to ACPI device %s\n", acpi_dev_name(adev));
 }
 
 void acpi_device_notify_remove(struct device *dev)
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index 40f875b265a942..011da4ed6a80ee 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -157,6 +157,7 @@ void acpi_turn_off_unused_power_resources(void);
                               Device Power Management
    -------------------------------------------------------------------------- */
 int acpi_device_get_power(struct acpi_device *device, int *state);
+int acpi_bus_init_power(struct acpi_device *device);
 int acpi_wakeup_device_init(void);
 
 /* --------------------------------------------------------------------------
diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c
index 9792dc3947561e..3006789ae31feb 100644
--- a/drivers/acpi/numa/hmat.c
+++ b/drivers/acpi/numa/hmat.c
@@ -995,7 +995,7 @@ static int hmat_calculate_adistance(struct notifier_block *self,
 	return NOTIFY_STOP;
 }
 
-static struct notifier_block hmat_adist_nb __meminitdata = {
+static struct notifier_block hmat_adist_nb = {
 	.notifier_call = hmat_calculate_adistance,
 	.priority = 100,
 };
diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c
index 5c407dc6401e8b..f5616830f5a975 100644
--- a/drivers/acpi/numa/srat.c
+++ b/drivers/acpi/numa/srat.c
@@ -111,7 +111,7 @@ int __init fix_pxm_node_maps(int max_nid)
 			for (j = 0; j <= max_nid; j++) {
 				if ((emu_nid_to_phys[j] == i) &&
 				    WARN(node_to_pxm_map_copy[j] != PXM_INVAL,
-					 "Node %d is already binded to PXM %d\n",
+					 "Node %d is already bound to PXM %d\n",
 					 j, node_to_pxm_map_copy[j]))
 					return -1;
 				if (emu_nid_to_phys[j] == i) {
diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c
index ed2162a97bd2aa..5b7aefb48eeb01 100644
--- a/drivers/acpi/osl.c
+++ b/drivers/acpi/osl.c
@@ -159,7 +159,7 @@ void __printf(1, 0) acpi_os_vprintf(const char *fmt, va_list args)
 {
 	static char buffer[512];
 
-	vsprintf(buffer, fmt, args);
+	vsnprintf(buffer, sizeof(buffer), fmt, args);
 
 #ifdef ENABLE_DEBUGGER
 	if (acpi_in_debugger) {
diff --git a/drivers/acpi/pfr_update.c b/drivers/acpi/pfr_update.c
index 9afd2c52fdbd88..98ace679601b85 100644
--- a/drivers/acpi/pfr_update.c
+++ b/drivers/acpi/pfr_update.c
@@ -422,7 +422,7 @@ static int start_update(int action, struct pfru_device *pfru_dev)
 
 static long pfru_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
-	struct pfru_update_cap_info cap_hdr;
+	struct pfru_update_cap_info cap_hdr = {};
 	struct pfru_device *pfru_dev = to_pfru_dev(file);
 	void __user *p = (void __user *)arg;
 	u32 rev;
diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c
index 23a4e207a01eb3..c6955d1ad29db7 100644
--- a/drivers/acpi/power.c
+++ b/drivers/acpi/power.c
@@ -1120,6 +1120,19 @@ static const struct dmi_system_id dmi_leave_unused_power_resources_on[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "ZERO"),
 		}
 
+	},
+	{
+		/*
+		 * Acer Swift 3 SF314-56G with NVIDIA MX250 suffers from the
+		 * same ACPI power resource regression as the Thunderobot ZERO.
+		 * The power resource controlling the dGPU is incorrectly turned off
+		 * during initialization, causing the GPU to fall off the bus (D3cold).
+		 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Swift SF314-56G"),
+		}
+
 	},
 	{}
 };
diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c
index cdc2ae1632b21a..bdd1529d79f035 100644
--- a/drivers/acpi/processor_driver.c
+++ b/drivers/acpi/processor_driver.c
@@ -164,7 +164,7 @@ static int __acpi_processor_start(struct acpi_device *device)
 
 	acpi_pss_perf_init(pr);
 
-	result = acpi_processor_thermal_init(pr, device);
+	result = acpi_processor_thermal_init(pr);
 	if (result)
 		goto err_power_exit;
 
@@ -179,7 +179,7 @@ static int __acpi_processor_start(struct acpi_device *device)
 	return 0;
 
 err_thermal_exit:
-	acpi_processor_thermal_exit(pr, device);
+	acpi_processor_thermal_exit(pr);
 err_power_exit:
 	acpi_processor_power_exit(pr);
 	return result;
@@ -203,7 +203,7 @@ static int acpi_processor_stop(struct device *dev)
 
 	acpi_cppc_processor_exit(pr);
 
-	acpi_processor_thermal_exit(pr, device);
+	acpi_processor_thermal_exit(pr);
 
 	return 0;
 }
diff --git a/drivers/acpi/processor_thermal.c b/drivers/acpi/processor_thermal.c
index c7b1dc5687eccc..11036f0d2b9422 100644
--- a/drivers/acpi/processor_thermal.c
+++ b/drivers/acpi/processor_thermal.c
@@ -235,15 +235,7 @@ static int
 processor_get_max_state(struct thermal_cooling_device *cdev,
 			unsigned long *state)
 {
-	struct acpi_device *device = cdev->devdata;
-	struct acpi_processor *pr;
-
-	if (!device)
-		return -EINVAL;
-
-	pr = acpi_driver_data(device);
-	if (!pr)
-		return -EINVAL;
+	struct acpi_processor *pr = cdev->devdata;
 
 	*state = acpi_processor_max_state(pr);
 	return 0;
@@ -253,15 +245,7 @@ static int
 processor_get_cur_state(struct thermal_cooling_device *cdev,
 			unsigned long *cur_state)
 {
-	struct acpi_device *device = cdev->devdata;
-	struct acpi_processor *pr;
-
-	if (!device)
-		return -EINVAL;
-
-	pr = acpi_driver_data(device);
-	if (!pr)
-		return -EINVAL;
+	struct acpi_processor *pr = cdev->devdata;
 
 	*cur_state = cpufreq_get_cur_state(pr->id);
 	if (pr->flags.throttling)
@@ -273,18 +257,10 @@ static int
 processor_set_cur_state(struct thermal_cooling_device *cdev,
 			unsigned long state)
 {
-	struct acpi_device *device = cdev->devdata;
-	struct acpi_processor *pr;
+	struct acpi_processor *pr = cdev->devdata;
 	int result = 0;
 	int max_pstate;
 
-	if (!device)
-		return -EINVAL;
-
-	pr = acpi_driver_data(device);
-	if (!pr)
-		return -EINVAL;
-
 	max_pstate = cpufreq_get_max_state(pr->id);
 
 	if (state > acpi_processor_max_state(pr))
@@ -308,55 +284,21 @@ const struct thermal_cooling_device_ops processor_cooling_ops = {
 	.set_cur_state = processor_set_cur_state,
 };
 
-int acpi_processor_thermal_init(struct acpi_processor *pr,
-				struct acpi_device *device)
+int acpi_processor_thermal_init(struct acpi_processor *pr)
 {
-	int result = 0;
-
-	pr->cdev = thermal_cooling_device_register("Processor", device,
-						   &processor_cooling_ops);
-	if (IS_ERR(pr->cdev)) {
-		result = PTR_ERR(pr->cdev);
-		return result;
-	}
-
-	dev_dbg(&device->dev, "registered as cooling_device%d\n",
-		pr->cdev->id);
+	pr->cdev = thermal_cooling_device_create(pr->dev, "Processor", pr,
+						 &processor_cooling_ops);
+	if (IS_ERR(pr->cdev))
+		return PTR_ERR(pr->cdev);
 
-	result = sysfs_create_link(&device->dev.kobj,
-				   &pr->cdev->device.kobj,
-				   "thermal_cooling");
-	if (result) {
-		dev_err(&device->dev,
-			"Failed to create sysfs link 'thermal_cooling'\n");
-		goto err_thermal_unregister;
-	}
-
-	result = sysfs_create_link(&pr->cdev->device.kobj,
-				   &device->dev.kobj,
-				   "device");
-	if (result) {
-		dev_err(&pr->cdev->device,
-			"Failed to create sysfs link 'device'\n");
-		goto err_remove_sysfs_thermal;
-	}
+	dev_dbg(pr->dev, "registered as cooling_device%d\n", pr->cdev->id);
 
 	return 0;
-
-err_remove_sysfs_thermal:
-	sysfs_remove_link(&device->dev.kobj, "thermal_cooling");
-err_thermal_unregister:
-	thermal_cooling_device_unregister(pr->cdev);
-
-	return result;
 }
 
-void acpi_processor_thermal_exit(struct acpi_processor *pr,
-				 struct acpi_device *device)
+void acpi_processor_thermal_exit(struct acpi_processor *pr)
 {
-	if (pr->cdev) {
-		sysfs_remove_link(&device->dev.kobj, "thermal_cooling");
-		sysfs_remove_link(&pr->cdev->device.kobj, "device");
+	if (!IS_ERR_OR_NULL(pr->cdev)) {
 		thermal_cooling_device_unregister(pr->cdev);
 		pr->cdev = NULL;
 	}
diff --git a/drivers/acpi/riscv/cppc.c b/drivers/acpi/riscv/cppc.c
index 42c1a90524707f..4ea4ddedd91f16 100644
--- a/drivers/acpi/riscv/cppc.c
+++ b/drivers/acpi/riscv/cppc.c
@@ -97,6 +97,7 @@ bool cpc_ffh_supported(void)
 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)
 {
 	struct sbi_cppc_data data;
+	int ret;
 
 	if (WARN_ON_ONCE(irqs_disabled()))
 		return -EPERM;
@@ -107,19 +108,27 @@ int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)
 
 		data.reg = FFH_CPPC_SBI_REG(reg->address);
 
-		smp_call_function_single(cpu, sbi_cppc_read, &data, 1);
+		ret = smp_call_function_single(cpu, sbi_cppc_read, &data, 1);
+		if (ret)
+			return ret;
+		if (data.ret.error)
+			return sbi_err_map_linux_errno(data.ret.error);
 
 		*val = data.ret.value;
 
-		return (data.ret.error) ? sbi_err_map_linux_errno(data.ret.error) : 0;
+		return 0;
 	} else if (FFH_CPPC_TYPE(reg->address) == FFH_CPPC_CSR) {
 		data.reg = FFH_CPPC_CSR_NUM(reg->address);
 
-		smp_call_function_single(cpu, cppc_ffh_csr_read, &data, 1);
+		ret = smp_call_function_single(cpu, cppc_ffh_csr_read, &data, 1);
+		if (ret)
+			return ret;
+		if (data.ret.error)
+			return data.ret.error;
 
 		*val = data.ret.value;
 
-		return data.ret.error;
+		return 0;
 	}
 
 	return -EINVAL;
@@ -128,6 +137,7 @@ int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val)
 int cpc_write_ffh(int cpu, struct cpc_reg *reg, u64 val)
 {
 	struct sbi_cppc_data data;
+	int ret;
 
 	if (WARN_ON_ONCE(irqs_disabled()))
 		return -EPERM;
@@ -139,14 +149,18 @@ int cpc_write_ffh(int cpu, struct cpc_reg *reg, u64 val)
 		data.reg = FFH_CPPC_SBI_REG(reg->address);
 		data.val = val;
 
-		smp_call_function_single(cpu, sbi_cppc_write, &data, 1);
+		ret = smp_call_function_single(cpu, sbi_cppc_write, &data, 1);
+		if (ret)
+			return ret;
 
 		return (data.ret.error) ? sbi_err_map_linux_errno(data.ret.error) : 0;
 	} else if (FFH_CPPC_TYPE(reg->address) == FFH_CPPC_CSR) {
 		data.reg = FFH_CPPC_CSR_NUM(reg->address);
 		data.val = val;
 
-		smp_call_function_single(cpu, cppc_ffh_csr_write, &data, 1);
+		ret = smp_call_function_single(cpu, cppc_ffh_csr_write, &data, 1);
+		if (ret)
+			return ret;
 
 		return data.ret.error;
 	}
diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c
index 86b7c797585263..f10bbf13c242f3 100644
--- a/drivers/acpi/sbs.c
+++ b/drivers/acpi/sbs.c
@@ -318,7 +318,7 @@ static struct acpi_battery_reader state_readers[] = {
 	{0x0a, SMBUS_READ_WORD, offsetof(struct acpi_battery, rate_now)},
 	{0x0b, SMBUS_READ_WORD, offsetof(struct acpi_battery, rate_avg)},
 	{0x0f, SMBUS_READ_WORD, offsetof(struct acpi_battery, capacity_now)},
-	{0x0e, SMBUS_READ_WORD, offsetof(struct acpi_battery, state_of_charge)},
+	{0x0d, SMBUS_READ_WORD, offsetof(struct acpi_battery, state_of_charge)},
 	{0x16, SMBUS_READ_WORD, offsetof(struct acpi_battery, state)},
 };
 
@@ -639,10 +639,8 @@ static int acpi_sbs_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	sbs = kzalloc_obj(struct acpi_sbs);
-	if (!sbs) {
-		result = -ENOMEM;
-		goto end;
-	}
+	if (!sbs)
+		return -ENOMEM;
 
 	mutex_init(&sbs->lock);
 
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 163a3cccf197b0..0b17a7e5195693 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -20,6 +20,7 @@
 #include <linux/kthread.h>
 #include <linux/dmi.h>
 #include <linux/dma-map-ops.h>
+#include <linux/pci.h>
 #include <linux/platform_data/x86/apple.h>
 #include <linux/pgtable.h>
 #include <linux/crc32.h>
@@ -637,10 +638,9 @@ static struct acpi_device *handle_to_device(acpi_handle handle,
 
 	status = acpi_get_data_full(handle, acpi_scan_drop_device,
 				    (void **)&adev, callback);
-	if (ACPI_FAILURE(status) || !adev) {
-		acpi_handle_debug(handle, "No context!\n");
+	if (ACPI_FAILURE(status) || !adev)
 		return NULL;
-	}
+
 	return adev;
 }
 
@@ -1143,8 +1143,7 @@ static void acpi_bus_get_power_flags(struct acpi_device *device)
 			device->power.states[ACPI_STATE_D3_COLD].flags.valid = 1;
 	}
 
-	if (acpi_bus_init_power(device))
-		device->flags.power_manageable = 0;
+	device->power.state = ACPI_STATE_UNKNOWN;
 }
 
 static void acpi_bus_get_flags(struct acpi_device *device)
@@ -1764,6 +1763,7 @@ static bool acpi_device_enumeration_by_parent(struct acpi_device *device)
 		/* Non-conforming _HID for Cirrus Logic already released */
 		{"CLSA0100", },
 		{"CLSA0101", },
+		{"CLSA0102", },
 	/*
 	 * Some ACPI devs contain SerialBus resources even though they are not
 	 * attached to a serial bus at all.
@@ -2338,50 +2338,57 @@ static int acpi_scan_attach_handler(struct acpi_device *device)
 static int acpi_bus_attach(struct acpi_device *device, void *first_pass)
 {
 	bool skip = !first_pass && device->flags.visited;
+	struct pci_dev *pci;
 	acpi_handle ejd;
 	int ret;
 
 	if (skip)
 		goto ok;
 
+	device->flags.initialized = true;
+
 	if (ACPI_SUCCESS(acpi_bus_get_ejd(device->handle, &ejd)))
 		register_dock_dependent_device(device, ejd);
 
 	acpi_bus_get_status(device);
-	/* Skip devices that are not ready for enumeration (e.g. not present) */
-	if (!acpi_dev_ready_for_enumeration(device)) {
-		device->flags.initialized = false;
+	/*
+	 * If the given ACPI device object has been already associated with a
+	 * PCI device found on the bus, its status is effectively "present and
+	 * enabled", and dependencies are not tracked for PCI devices, so it is
+	 * not necessary or even useful to check the device's readiness in that
+	 * case.
+	 */
+	pci = acpi_dev_get_pci_dev(device);
+	if (pci) {
+		acpi_handle_debug(device->handle, "PCI companion %s found\n",
+				  pci_name(pci));
+
+		if (!acpi_device_is_present(device))
+			pci_info(pci, FW_BUG "ACPI status differs from reality\n");
+
+		pci_dev_put(pci);
+	} else if (!acpi_dev_ready_for_enumeration(device)) {
+		/* The device is not ready (e.g. not present), so skip it. */
 		acpi_device_clear_enumerated(device);
-		device->flags.power_manageable = 0;
 		return 0;
 	}
+
 	if (device->handler)
 		goto ok;
 
 	acpi_ec_register_opregions(device);
 
-	if (!device->flags.initialized) {
-		device->flags.power_manageable =
-			device->power.states[ACPI_STATE_D0].flags.valid;
-		if (acpi_bus_init_power(device))
-			device->flags.power_manageable = 0;
+	acpi_bus_init_power(device);
 
-		device->flags.initialized = true;
-	} else if (device->flags.visited) {
+	if (device->flags.visited)
 		goto ok;
-	}
 
 	ret = acpi_scan_attach_handler(device);
 	if (ret < 0)
 		return 0;
 
-	if (ret > 0 && !device->flags.enumeration_by_parent) {
-		acpi_device_set_enumerated(device);
-		goto ok;
-	}
-
-	if (device->pnp.type.platform_id || device->pnp.type.backlight ||
-	    device->flags.enumeration_by_parent)
+	if (device->flags.enumeration_by_parent ||
+	    (!ret && (device->pnp.type.platform_id || device->pnp.type.backlight)))
 		acpi_default_enumeration(device);
 	else
 		acpi_device_set_enumerated(device);
diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c
index dd4a99f09efea6..a993735471261c 100644
--- a/drivers/acpi/sysfs.c
+++ b/drivers/acpi/sysfs.c
@@ -11,6 +11,7 @@
 #include <linux/kernel.h>
 #include <linux/kstrtox.h>
 #include <linux/moduleparam.h>
+#include <linux/string.h>
 
 #include "internal.h"
 
@@ -181,10 +182,10 @@ static int param_set_trace_method_name(const char *val,
 
 	/* This is a hack.  We can't kmalloc in early boot. */
 	if (is_abs_path)
-		strcpy(trace_method_name, val);
+		strscpy(trace_method_name, val, sizeof(trace_method_name));
 	else {
 		trace_method_name[0] = '\\';
-		strcpy(trace_method_name+1, val);
+		strscpy(trace_method_name + 1, val, sizeof(trace_method_name) - 1);
 	}
 
 	/* Restore the original tracer state */
diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c
index eb93c060426ffd..aaa1de79bccfd7 100644
--- a/drivers/acpi/tables.c
+++ b/drivers/acpi/tables.c
@@ -560,6 +560,9 @@ acpi_table_initrd_override(struct acpi_table_header *existing_table,
 	while (table_offset + ACPI_HEADER_SIZE <= all_tables_size) {
 		table = acpi_os_map_memory(acpi_tables_addr + table_offset,
 					   ACPI_HEADER_SIZE);
+		if (!table)
+			return AE_NO_MEMORY;
+
 		if (table_offset + table->length > all_tables_size) {
 			acpi_os_unmap_memory(table, ACPI_HEADER_SIZE);
 			WARN_ON(1);
diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c
index dd7666c176a057..13e9a3a79d92a3 100644
--- a/drivers/acpi/thermal.c
+++ b/drivers/acpi/thermal.c
@@ -564,17 +564,20 @@ static bool acpi_thermal_should_bind_cdev(struct thermal_zone_device *thermal,
 					  struct cooling_spec *c)
 {
 	struct acpi_thermal_trip *acpi_trip = trip->priv;
-	struct acpi_device *cdev_adev = cdev->devdata;
+	struct device *parent = cdev->device.parent;
+	acpi_handle parent_handle;
 	int i;
 
-	/* Skip critical and hot trips. */
-	if (!acpi_trip)
+	/* Skip critical and hot trips and parentless cooling devices. */
+	if (!acpi_trip || !parent)
 		return false;
 
-	for (i = 0; i < acpi_trip->devices.count; i++) {
-		acpi_handle handle = acpi_trip->devices.handles[i];
+	parent_handle = ACPI_HANDLE(parent);
+	if (!parent_handle)
+		return false;
 
-		if (acpi_fetch_acpi_dev(handle) == cdev_adev)
+	for (i = 0; i < acpi_trip->devices.count; i++) {
+		if (acpi_trip->devices.handles[i] == parent_handle)
 			return true;
 	}
 
@@ -738,7 +741,9 @@ static void acpi_thermal_aml_dependency_fix(struct acpi_thermal *tz)
  */
 static void acpi_thermal_guess_offset(struct acpi_thermal *tz, long crit_temp)
 {
-	if (crit_temp != THERMAL_TEMP_INVALID && crit_temp % 5 == 1)
+	if (crit_temp != THERMAL_TEMP_INVALID && crit_temp % 5 == 0)
+		tz->kelvin_offset = 273000;
+	else if (crit_temp != THERMAL_TEMP_INVALID && crit_temp % 5 == 1)
 		tz->kelvin_offset = 273100;
 	else
 		tz->kelvin_offset = 273200;
diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c
index d499b72574ab26..7c584d490d6a45 100644
--- a/drivers/acpi/utils.c
+++ b/drivers/acpi/utils.c
@@ -1069,19 +1069,26 @@ EXPORT_SYMBOL(acpi_dev_is_video_device);
  * @plat: pointer to acpi_platform_list table terminated by a NULL entry
  *
  * Return the matched index if the system is found in the platform list.
- * Otherwise, return a negative error code.
+ * Return -ENODEV for no match, or another negative error code if a table
+ * header could not be read and no entry matched.
  */
 int acpi_match_platform_list(const struct acpi_platform_list *plat)
 {
 	struct acpi_table_header hdr;
+	acpi_status status;
+	int ret = -ENODEV;
 	int idx = 0;
 
 	if (acpi_disabled)
 		return -ENODEV;
 
 	for (; plat->oem_id[0]; plat++, idx++) {
-		if (ACPI_FAILURE(acpi_get_table_header(plat->table, 0, &hdr)))
+		status = acpi_get_table_header(plat->table, 0, &hdr);
+		if (ACPI_FAILURE(status)) {
+			if (status != AE_NOT_FOUND)
+				ret = status == AE_NO_MEMORY ? -ENOMEM : -EIO;
 			continue;
+		}
 
 		if (strncmp(plat->oem_id, hdr.oem_id, ACPI_OEM_ID_SIZE))
 			continue;
@@ -1096,6 +1103,6 @@ int acpi_match_platform_list(const struct acpi_platform_list *plat)
 			return idx;
 	}
 
-	return -ENODEV;
+	return ret;
 }
 EXPORT_SYMBOL(acpi_match_platform_list);
diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
index e3b69f876d2474..3037fd18d42064 100644
--- a/drivers/acpi/video_detect.c
+++ b/drivers/acpi/video_detect.c
@@ -929,6 +929,14 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 		DMI_MATCH(DMI_PRODUCT_NAME, "Nitro AN515-46"),
 		},
 	},
+	{
+	 .callback = video_detect_force_native,
+	 /* Acer Nitro AN515-58 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "Nitro AN515-58"),
+		},
+	},
 
 	/*
 	 * x86 android tablets which directly control the backlight through
diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c
index b6b1dd76a06ba0..c278ea14c0b36c 100644
--- a/drivers/acpi/x86/s2idle.c
+++ b/drivers/acpi/x86/s2idle.c
@@ -24,6 +24,34 @@
 
 #ifdef CONFIG_SUSPEND
 
+static void acpi_setup_ixx_gpes(void)
+{
+	acpi_handle gpe_root;
+	unsigned int i;
+	char gpe_nr_str[5];
+
+	if (ACPI_FAILURE(acpi_get_handle(NULL, "\\_GPE", &gpe_root)))
+		return;
+
+	for (i = 0; i <= 0xff; i++) {
+		scnprintf(gpe_nr_str, sizeof(gpe_nr_str), "_I%02X", i);
+		if (!acpi_has_method(gpe_root, gpe_nr_str))
+			continue;
+
+		/*
+		 * Enable the GPE if it has a handler method because marking it
+		 * as wake-capable causes acpi_update_all_gpes() to skip it.
+		 */
+		if (ACPI_FAILURE(acpi_enable_gpe_cond(NULL, i, ACPI_GPE_DISPATCH_METHOD)))
+			continue;
+
+		acpi_mark_gpe_for_wake(NULL, i);
+		acpi_set_gpe_wake_mask(NULL, i, ACPI_GPE_ENABLE);
+
+		pm_pr_dbg("ACPI: GPE0x%02x armed for wake via %s\n", i, gpe_nr_str);
+	}
+}
+
 static bool sleep_no_lps0 __read_mostly;
 module_param(sleep_no_lps0, bool, 0644);
 MODULE_PARM_DESC(sleep_no_lps0, "Do not use the special LPS0 device interface");
@@ -649,6 +677,7 @@ void __init acpi_s2idle_setup(void)
 {
 	acpi_scan_add_handler(&lps0_handler);
 	s2idle_set_ops(&acpi_s2idle_ops_lps0);
+	acpi_setup_ixx_gpes();
 }
 
 int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg)
diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
index 606a9d07f77471..c920d52c022169 100644
--- a/drivers/android/Kconfig
+++ b/drivers/android/Kconfig
@@ -1,10 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0
 menu "Android"
 
-config ANDROID_BINDER_IPC
+config ANDROID_BINDER_IPC_RUST
 	bool "Android Binder IPC Driver"
 	depends on MMU
 	depends on NET
+	depends on RUST
 	default n
 	help
 	  Binder is used in Android for both communication between processes,
@@ -14,34 +15,9 @@ config ANDROID_BINDER_IPC
 	  Android process, using Binder to identify, invoke and pass arguments
 	  between said processes.
 
-config ANDROID_BINDER_IPC_RUST
-	bool "Rust version of Android Binder IPC Driver"
-	depends on RUST && MMU && NET && !ANDROID_BINDER_IPC
-	help
-	  This enables the Rust implementation of the Binder driver.
-
-	  Binder is used in Android for both communication between processes,
-	  and remote method invocation.
-
-	  This means one Android process can call a method/routine in another
-	  Android process, using Binder to identify, invoke and pass arguments
-	  between said processes.
-
-config ANDROID_BINDERFS
-	bool "Android Binderfs filesystem"
-	depends on ANDROID_BINDER_IPC
-	default n
-	help
-	  Binderfs is a pseudo-filesystem for the Android Binder IPC driver
-	  which can be mounted per-ipc namespace allowing to run multiple
-	  instances of Android.
-	  Each binderfs mount initially only contains a binder-control device.
-	  It can be used to dynamically allocate new binder IPC devices via
-	  ioctls.
-
 config ANDROID_BINDER_DEVICES
 	string "Android Binder devices"
-	depends on ANDROID_BINDER_IPC || ANDROID_BINDER_IPC_RUST
+	depends on ANDROID_BINDER_IPC_RUST
 	default "binder,hwbinder,vndbinder"
 	help
 	  Default value for the binder.devices parameter.
@@ -51,15 +27,4 @@ config ANDROID_BINDER_DEVICES
 	  created. Each binder device has its own context manager, and is
 	  therefore logically separated from the other devices.
 
-config ANDROID_BINDER_ALLOC_KUNIT_TEST
-	tristate "KUnit Tests for Android Binder Alloc" if !KUNIT_ALL_TESTS
-	depends on ANDROID_BINDER_IPC && KUNIT
-	default KUNIT_ALL_TESTS
-	help
-	  This feature builds the binder alloc KUnit tests.
-
-	  Each test case runs using a pared-down binder_alloc struct and
-	  test-specific freelist, which allows this KUnit module to be loaded
-	  for testing without interfering with a running system.
-
 endmenu
diff --git a/drivers/android/Makefile b/drivers/android/Makefile
index e0c650d3898ede..f83d39d16f6e94 100644
--- a/drivers/android/Makefile
+++ b/drivers/android/Makefile
@@ -1,7 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
 ccflags-y += -I$(src)			# needed for trace events
 
-obj-$(CONFIG_ANDROID_BINDERFS)		+= binderfs.o
-obj-$(CONFIG_ANDROID_BINDER_IPC)	+= binder.o binder_alloc.o binder_netlink.o
-obj-$(CONFIG_ANDROID_BINDER_ALLOC_KUNIT_TEST)	+= tests/
 obj-$(CONFIG_ANDROID_BINDER_IPC_RUST)	+= binder/
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
deleted file mode 100644
index 8f2ef1bd539f18..00000000000000
--- a/drivers/android/binder.c
+++ /dev/null
@@ -1,7186 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* binder.c
- *
- * Android IPC Subsystem
- *
- * Copyright (C) 2007-2008 Google, Inc.
- */
-
-/*
- * Locking overview
- *
- * There are 3 main spinlocks which must be acquired in the
- * order shown:
- *
- * 1) proc->outer_lock : protects binder_ref
- *    binder_proc_lock() and binder_proc_unlock() are
- *    used to acq/rel.
- * 2) node->lock : protects most fields of binder_node.
- *    binder_node_lock() and binder_node_unlock() are
- *    used to acq/rel
- * 3) proc->inner_lock : protects the thread and node lists
- *    (proc->threads, proc->waiting_threads, proc->nodes)
- *    and all todo lists associated with the binder_proc
- *    (proc->todo, thread->todo, proc->delivered_death and
- *    node->async_todo), as well as thread->transaction_stack
- *    binder_inner_proc_lock() and binder_inner_proc_unlock()
- *    are used to acq/rel
- *
- * Any lock under procA must never be nested under any lock at the same
- * level or below on procB.
- *
- * Functions that require a lock held on entry indicate which lock
- * in the suffix of the function name:
- *
- * foo_olocked() : requires node->outer_lock
- * foo_nlocked() : requires node->lock
- * foo_ilocked() : requires proc->inner_lock
- * foo_oilocked(): requires proc->outer_lock and proc->inner_lock
- * foo_nilocked(): requires node->lock and proc->inner_lock
- * ...
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/fdtable.h>
-#include <linux/file.h>
-#include <linux/freezer.h>
-#include <linux/fs.h>
-#include <linux/list.h>
-#include <linux/miscdevice.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/nsproxy.h>
-#include <linux/poll.h>
-#include <linux/debugfs.h>
-#include <linux/rbtree.h>
-#include <linux/sched/signal.h>
-#include <linux/sched/mm.h>
-#include <linux/seq_file.h>
-#include <linux/string.h>
-#include <linux/uaccess.h>
-#include <linux/pid_namespace.h>
-#include <linux/security.h>
-#include <linux/spinlock.h>
-#include <linux/ratelimit.h>
-#include <linux/syscalls.h>
-#include <linux/task_work.h>
-#include <linux/sizes.h>
-#include <linux/ktime.h>
-
-#include <kunit/visibility.h>
-
-#include <uapi/linux/android/binder.h>
-
-#include <linux/cacheflush.h>
-
-#include "binder_netlink.h"
-#include "binder_internal.h"
-#include "binder_trace.h"
-
-static HLIST_HEAD(binder_deferred_list);
-static DEFINE_MUTEX(binder_deferred_lock);
-
-static HLIST_HEAD(binder_devices);
-static DEFINE_SPINLOCK(binder_devices_lock);
-
-static HLIST_HEAD(binder_procs);
-static DEFINE_MUTEX(binder_procs_lock);
-
-static HLIST_HEAD(binder_dead_nodes);
-static DEFINE_SPINLOCK(binder_dead_nodes_lock);
-
-static struct dentry *binder_debugfs_dir_entry_root;
-static struct dentry *binder_debugfs_dir_entry_proc;
-static atomic_t binder_last_id;
-
-static int proc_show(struct seq_file *m, void *unused);
-DEFINE_SHOW_ATTRIBUTE(proc);
-
-#define FORBIDDEN_MMAP_FLAGS                (VM_WRITE)
-
-enum {
-	BINDER_DEBUG_USER_ERROR             = 1U << 0,
-	BINDER_DEBUG_FAILED_TRANSACTION     = 1U << 1,
-	BINDER_DEBUG_DEAD_TRANSACTION       = 1U << 2,
-	BINDER_DEBUG_OPEN_CLOSE             = 1U << 3,
-	BINDER_DEBUG_DEAD_BINDER            = 1U << 4,
-	BINDER_DEBUG_DEATH_NOTIFICATION     = 1U << 5,
-	BINDER_DEBUG_READ_WRITE             = 1U << 6,
-	BINDER_DEBUG_USER_REFS              = 1U << 7,
-	BINDER_DEBUG_THREADS                = 1U << 8,
-	BINDER_DEBUG_TRANSACTION            = 1U << 9,
-	BINDER_DEBUG_TRANSACTION_COMPLETE   = 1U << 10,
-	BINDER_DEBUG_FREE_BUFFER            = 1U << 11,
-	BINDER_DEBUG_INTERNAL_REFS          = 1U << 12,
-	BINDER_DEBUG_PRIORITY_CAP           = 1U << 13,
-	BINDER_DEBUG_SPINLOCKS              = 1U << 14,
-};
-static uint32_t binder_debug_mask = BINDER_DEBUG_USER_ERROR |
-	BINDER_DEBUG_FAILED_TRANSACTION | BINDER_DEBUG_DEAD_TRANSACTION;
-module_param_named(debug_mask, binder_debug_mask, uint, 0644);
-
-char *binder_devices_param = CONFIG_ANDROID_BINDER_DEVICES;
-module_param_named(devices, binder_devices_param, charp, 0444);
-
-static DECLARE_WAIT_QUEUE_HEAD(binder_user_error_wait);
-static int binder_stop_on_user_error;
-
-static int binder_set_stop_on_user_error(const char *val,
-					 const struct kernel_param *kp)
-{
-	int ret;
-
-	ret = param_set_int(val, kp);
-	if (binder_stop_on_user_error < 2)
-		wake_up(&binder_user_error_wait);
-	return ret;
-}
-module_param_call(stop_on_user_error, binder_set_stop_on_user_error,
-	param_get_int, &binder_stop_on_user_error, 0644);
-
-static __printf(2, 3) void binder_debug(int mask, const char *format, ...)
-{
-	struct va_format vaf;
-	va_list args;
-
-	if (binder_debug_mask & mask) {
-		va_start(args, format);
-		vaf.va = &args;
-		vaf.fmt = format;
-		pr_info_ratelimited("%pV", &vaf);
-		va_end(args);
-	}
-}
-
-#define binder_txn_error(x...) \
-	binder_debug(BINDER_DEBUG_FAILED_TRANSACTION, x)
-
-static __printf(1, 2) void binder_user_error(const char *format, ...)
-{
-	struct va_format vaf;
-	va_list args;
-
-	if (binder_debug_mask & BINDER_DEBUG_USER_ERROR) {
-		va_start(args, format);
-		vaf.va = &args;
-		vaf.fmt = format;
-		pr_info_ratelimited("%pV", &vaf);
-		va_end(args);
-	}
-
-	if (binder_stop_on_user_error)
-		binder_stop_on_user_error = 2;
-}
-
-#define binder_set_extended_error(ee, _id, _command, _param) \
-	do { \
-		(ee)->id = _id; \
-		(ee)->command = _command; \
-		(ee)->param = _param; \
-	} while (0)
-
-#define to_flat_binder_object(hdr) \
-	container_of(hdr, struct flat_binder_object, hdr)
-
-#define to_binder_fd_object(hdr) container_of(hdr, struct binder_fd_object, hdr)
-
-#define to_binder_buffer_object(hdr) \
-	container_of(hdr, struct binder_buffer_object, hdr)
-
-#define to_binder_fd_array_object(hdr) \
-	container_of(hdr, struct binder_fd_array_object, hdr)
-
-static struct binder_stats binder_stats;
-
-static inline void binder_stats_deleted(enum binder_stat_types type)
-{
-	atomic_inc(&binder_stats.obj_deleted[type]);
-}
-
-static inline void binder_stats_created(enum binder_stat_types type)
-{
-	atomic_inc(&binder_stats.obj_created[type]);
-}
-
-struct binder_transaction_log_entry {
-	int debug_id;
-	int debug_id_done;
-	int call_type;
-	int from_proc;
-	int from_thread;
-	int target_handle;
-	int to_proc;
-	int to_thread;
-	int to_node;
-	int data_size;
-	int offsets_size;
-	int return_error_line;
-	uint32_t return_error;
-	uint32_t return_error_param;
-	char context_name[BINDERFS_MAX_NAME + 1];
-};
-
-struct binder_transaction_log {
-	atomic_t cur;
-	bool full;
-	struct binder_transaction_log_entry entry[32];
-};
-
-static struct binder_transaction_log binder_transaction_log;
-static struct binder_transaction_log binder_transaction_log_failed;
-
-static struct binder_transaction_log_entry *binder_transaction_log_add(
-	struct binder_transaction_log *log)
-{
-	struct binder_transaction_log_entry *e;
-	unsigned int cur = atomic_inc_return(&log->cur);
-
-	if (cur >= ARRAY_SIZE(log->entry))
-		log->full = true;
-	e = &log->entry[cur % ARRAY_SIZE(log->entry)];
-	WRITE_ONCE(e->debug_id_done, 0);
-	/*
-	 * write-barrier to synchronize access to e->debug_id_done.
-	 * We make sure the initialized 0 value is seen before
-	 * memset() other fields are zeroed by memset.
-	 */
-	smp_wmb();
-	memset(e, 0, sizeof(*e));
-	return e;
-}
-
-enum binder_deferred_state {
-	BINDER_DEFERRED_FLUSH        = 0x01,
-	BINDER_DEFERRED_RELEASE      = 0x02,
-};
-
-enum {
-	BINDER_LOOPER_STATE_REGISTERED  = 0x01,
-	BINDER_LOOPER_STATE_ENTERED     = 0x02,
-	BINDER_LOOPER_STATE_EXITED      = 0x04,
-	BINDER_LOOPER_STATE_INVALID     = 0x08,
-	BINDER_LOOPER_STATE_WAITING     = 0x10,
-	BINDER_LOOPER_STATE_POLL        = 0x20,
-};
-
-/**
- * binder_proc_lock() - Acquire outer lock for given binder_proc
- * @proc:         struct binder_proc to acquire
- *
- * Acquires proc->outer_lock. Used to protect binder_ref
- * structures associated with the given proc.
- */
-#define binder_proc_lock(proc) _binder_proc_lock(proc, __LINE__)
-static void
-_binder_proc_lock(struct binder_proc *proc, int line)
-	__acquires(&proc->outer_lock)
-{
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	spin_lock(&proc->outer_lock);
-}
-
-/**
- * binder_proc_unlock() - Release outer lock for given binder_proc
- * @proc:                struct binder_proc to acquire
- *
- * Release lock acquired via binder_proc_lock()
- */
-#define binder_proc_unlock(proc) _binder_proc_unlock(proc, __LINE__)
-static void
-_binder_proc_unlock(struct binder_proc *proc, int line)
-	__releases(&proc->outer_lock)
-{
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	spin_unlock(&proc->outer_lock);
-}
-
-/**
- * binder_inner_proc_lock() - Acquire inner lock for given binder_proc
- * @proc:         struct binder_proc to acquire
- *
- * Acquires proc->inner_lock. Used to protect todo lists
- */
-#define binder_inner_proc_lock(proc) _binder_inner_proc_lock(proc, __LINE__)
-static void
-_binder_inner_proc_lock(struct binder_proc *proc, int line)
-	__acquires(&proc->inner_lock)
-{
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	spin_lock(&proc->inner_lock);
-}
-
-/**
- * binder_inner_proc_unlock() - Release inner lock for given binder_proc
- * @proc:         struct binder_proc to acquire
- *
- * Release lock acquired via binder_inner_proc_lock()
- */
-#define binder_inner_proc_unlock(proc) _binder_inner_proc_unlock(proc, __LINE__)
-static void
-_binder_inner_proc_unlock(struct binder_proc *proc, int line)
-	__releases(&proc->inner_lock)
-{
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	spin_unlock(&proc->inner_lock);
-}
-
-/**
- * binder_node_lock() - Acquire spinlock for given binder_node
- * @node:         struct binder_node to acquire
- *
- * Acquires node->lock. Used to protect binder_node fields
- */
-#define binder_node_lock(node) _binder_node_lock(node, __LINE__)
-static void
-_binder_node_lock(struct binder_node *node, int line)
-	__acquires(&node->lock)
-{
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	spin_lock(&node->lock);
-}
-
-/**
- * binder_node_unlock() - Release spinlock for given binder_proc
- * @node:         struct binder_node to acquire
- *
- * Release lock acquired via binder_node_lock()
- */
-#define binder_node_unlock(node) _binder_node_unlock(node, __LINE__)
-static void
-_binder_node_unlock(struct binder_node *node, int line)
-	__releases(&node->lock)
-{
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	spin_unlock(&node->lock);
-}
-
-/**
- * binder_node_inner_lock() - Acquire node and inner locks
- * @node:         struct binder_node to acquire
- *
- * Acquires node->lock. If node->proc also acquires
- * proc->inner_lock. Used to protect binder_node fields
- */
-#define binder_node_inner_lock(node) _binder_node_inner_lock(node, __LINE__)
-static void
-_binder_node_inner_lock(struct binder_node *node, int line)
-	__acquires(&node->lock) __acquires(&node->proc->inner_lock)
-{
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	spin_lock(&node->lock);
-	if (node->proc)
-		binder_inner_proc_lock(node->proc);
-	else
-		/* annotation for sparse */
-		__acquire(&node->proc->inner_lock);
-}
-
-/**
- * binder_node_inner_unlock() - Release node and inner locks
- * @node:         struct binder_node to acquire
- *
- * Release lock acquired via binder_node_lock()
- */
-#define binder_node_inner_unlock(node) _binder_node_inner_unlock(node, __LINE__)
-static void
-_binder_node_inner_unlock(struct binder_node *node, int line)
-	__releases(&node->lock) __releases(&node->proc->inner_lock)
-{
-	struct binder_proc *proc = node->proc;
-
-	binder_debug(BINDER_DEBUG_SPINLOCKS,
-		     "%s: line=%d\n", __func__, line);
-	if (proc)
-		binder_inner_proc_unlock(proc);
-	else
-		/* annotation for sparse */
-		__release(&node->proc->inner_lock);
-	spin_unlock(&node->lock);
-}
-
-static bool binder_worklist_empty_ilocked(struct list_head *list)
-{
-	return list_empty(list);
-}
-
-/**
- * binder_worklist_empty() - Check if no items on the work list
- * @proc:       binder_proc associated with list
- * @list:	list to check
- *
- * Return: true if there are no items on list, else false
- */
-static bool binder_worklist_empty(struct binder_proc *proc,
-				  struct list_head *list)
-{
-	bool ret;
-
-	binder_inner_proc_lock(proc);
-	ret = binder_worklist_empty_ilocked(list);
-	binder_inner_proc_unlock(proc);
-	return ret;
-}
-
-/**
- * binder_enqueue_work_ilocked() - Add an item to the work list
- * @work:         struct binder_work to add to list
- * @target_list:  list to add work to
- *
- * Adds the work to the specified list. Asserts that work
- * is not already on a list.
- *
- * Requires the proc->inner_lock to be held.
- */
-static void
-binder_enqueue_work_ilocked(struct binder_work *work,
-			   struct list_head *target_list)
-{
-	BUG_ON(target_list == NULL);
-	BUG_ON(work->entry.next && !list_empty(&work->entry));
-	list_add_tail(&work->entry, target_list);
-}
-
-/**
- * binder_enqueue_deferred_thread_work_ilocked() - Add deferred thread work
- * @thread:       thread to queue work to
- * @work:         struct binder_work to add to list
- *
- * Adds the work to the todo list of the thread. Doesn't set the process_todo
- * flag, which means that (if it wasn't already set) the thread will go to
- * sleep without handling this work when it calls read.
- *
- * Requires the proc->inner_lock to be held.
- */
-static void
-binder_enqueue_deferred_thread_work_ilocked(struct binder_thread *thread,
-					    struct binder_work *work)
-{
-	WARN_ON(!list_empty(&thread->waiting_thread_node));
-	binder_enqueue_work_ilocked(work, &thread->todo);
-}
-
-/**
- * binder_enqueue_thread_work_ilocked() - Add an item to the thread work list
- * @thread:       thread to queue work to
- * @work:         struct binder_work to add to list
- *
- * Adds the work to the todo list of the thread, and enables processing
- * of the todo queue.
- *
- * Requires the proc->inner_lock to be held.
- */
-static void
-binder_enqueue_thread_work_ilocked(struct binder_thread *thread,
-				   struct binder_work *work)
-{
-	WARN_ON(!list_empty(&thread->waiting_thread_node));
-	binder_enqueue_work_ilocked(work, &thread->todo);
-
-	/* (e)poll-based threads require an explicit wakeup signal when
-	 * queuing their own work; they rely on these events to consume
-	 * messages without I/O block. Without it, threads risk waiting
-	 * indefinitely without handling the work.
-	 */
-	if (thread->looper & BINDER_LOOPER_STATE_POLL &&
-	    thread->pid == current->pid && !thread->process_todo)
-		wake_up_interruptible_sync(&thread->wait);
-
-	thread->process_todo = true;
-}
-
-/**
- * binder_enqueue_thread_work() - Add an item to the thread work list
- * @thread:       thread to queue work to
- * @work:         struct binder_work to add to list
- *
- * Adds the work to the todo list of the thread, and enables processing
- * of the todo queue.
- */
-static void
-binder_enqueue_thread_work(struct binder_thread *thread,
-			   struct binder_work *work)
-{
-	binder_inner_proc_lock(thread->proc);
-	binder_enqueue_thread_work_ilocked(thread, work);
-	binder_inner_proc_unlock(thread->proc);
-}
-
-static void
-binder_dequeue_work_ilocked(struct binder_work *work)
-{
-	list_del_init(&work->entry);
-}
-
-/**
- * binder_dequeue_work() - Removes an item from the work list
- * @proc:         binder_proc associated with list
- * @work:         struct binder_work to remove from list
- *
- * Removes the specified work item from whatever list it is on.
- * Can safely be called if work is not on any list.
- */
-static void
-binder_dequeue_work(struct binder_proc *proc, struct binder_work *work)
-{
-	binder_inner_proc_lock(proc);
-	binder_dequeue_work_ilocked(work);
-	binder_inner_proc_unlock(proc);
-}
-
-static struct binder_work *binder_dequeue_work_head_ilocked(
-					struct list_head *list)
-{
-	struct binder_work *w;
-
-	w = list_first_entry_or_null(list, struct binder_work, entry);
-	if (w)
-		list_del_init(&w->entry);
-	return w;
-}
-
-static void
-binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer);
-static void binder_free_thread(struct binder_thread *thread);
-static void binder_free_proc(struct binder_proc *proc);
-static void binder_inc_node_tmpref_ilocked(struct binder_node *node);
-
-static bool binder_has_work_ilocked(struct binder_thread *thread,
-				    bool do_proc_work)
-{
-	return thread->process_todo ||
-		thread->looper_need_return ||
-		(do_proc_work &&
-		 !binder_worklist_empty_ilocked(&thread->proc->todo));
-}
-
-static bool binder_has_work(struct binder_thread *thread, bool do_proc_work)
-{
-	bool has_work;
-
-	binder_inner_proc_lock(thread->proc);
-	has_work = binder_has_work_ilocked(thread, do_proc_work);
-	binder_inner_proc_unlock(thread->proc);
-
-	return has_work;
-}
-
-static bool binder_available_for_proc_work_ilocked(struct binder_thread *thread)
-{
-	return !thread->transaction_stack &&
-		binder_worklist_empty_ilocked(&thread->todo);
-}
-
-static void binder_wakeup_poll_threads_ilocked(struct binder_proc *proc,
-					       bool sync)
-{
-	struct rb_node *n;
-	struct binder_thread *thread;
-
-	for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) {
-		thread = rb_entry(n, struct binder_thread, rb_node);
-		if (thread->looper & BINDER_LOOPER_STATE_POLL &&
-		    binder_available_for_proc_work_ilocked(thread)) {
-			if (sync)
-				wake_up_interruptible_sync(&thread->wait);
-			else
-				wake_up_interruptible(&thread->wait);
-		}
-	}
-}
-
-/**
- * binder_select_thread_ilocked() - selects a thread for doing proc work.
- * @proc:	process to select a thread from
- *
- * Note that calling this function moves the thread off the waiting_threads
- * list, so it can only be woken up by the caller of this function, or a
- * signal. Therefore, callers *should* always wake up the thread this function
- * returns.
- *
- * Return:	If there's a thread currently waiting for process work,
- *		returns that thread. Otherwise returns NULL.
- */
-static struct binder_thread *
-binder_select_thread_ilocked(struct binder_proc *proc)
-{
-	struct binder_thread *thread;
-
-	assert_spin_locked(&proc->inner_lock);
-	thread = list_first_entry_or_null(&proc->waiting_threads,
-					  struct binder_thread,
-					  waiting_thread_node);
-
-	if (thread)
-		list_del_init(&thread->waiting_thread_node);
-
-	return thread;
-}
-
-/**
- * binder_wakeup_thread_ilocked() - wakes up a thread for doing proc work.
- * @proc:	process to wake up a thread in
- * @thread:	specific thread to wake-up (may be NULL)
- * @sync:	whether to do a synchronous wake-up
- *
- * This function wakes up a thread in the @proc process.
- * The caller may provide a specific thread to wake-up in
- * the @thread parameter. If @thread is NULL, this function
- * will wake up threads that have called poll().
- *
- * Note that for this function to work as expected, callers
- * should first call binder_select_thread() to find a thread
- * to handle the work (if they don't have a thread already),
- * and pass the result into the @thread parameter.
- */
-static void binder_wakeup_thread_ilocked(struct binder_proc *proc,
-					 struct binder_thread *thread,
-					 bool sync)
-{
-	assert_spin_locked(&proc->inner_lock);
-
-	if (thread) {
-		if (sync)
-			wake_up_interruptible_sync(&thread->wait);
-		else
-			wake_up_interruptible(&thread->wait);
-		return;
-	}
-
-	/* Didn't find a thread waiting for proc work; this can happen
-	 * in two scenarios:
-	 * 1. All threads are busy handling transactions
-	 *    In that case, one of those threads should call back into
-	 *    the kernel driver soon and pick up this work.
-	 * 2. Threads are using the (e)poll interface, in which case
-	 *    they may be blocked on the waitqueue without having been
-	 *    added to waiting_threads. For this case, we just iterate
-	 *    over all threads not handling transaction work, and
-	 *    wake them all up. We wake all because we don't know whether
-	 *    a thread that called into (e)poll is handling non-binder
-	 *    work currently.
-	 */
-	binder_wakeup_poll_threads_ilocked(proc, sync);
-}
-
-static void binder_wakeup_proc_ilocked(struct binder_proc *proc)
-{
-	struct binder_thread *thread = binder_select_thread_ilocked(proc);
-
-	binder_wakeup_thread_ilocked(proc, thread, /* sync = */false);
-}
-
-static void binder_set_nice(long nice)
-{
-	long min_nice;
-
-	if (can_nice(current, nice)) {
-		set_user_nice(current, nice);
-		return;
-	}
-	min_nice = rlimit_to_nice(rlimit(RLIMIT_NICE));
-	binder_debug(BINDER_DEBUG_PRIORITY_CAP,
-		     "%d: nice value %ld not allowed use %ld instead\n",
-		      current->pid, nice, min_nice);
-	set_user_nice(current, min_nice);
-	if (min_nice <= MAX_NICE)
-		return;
-	binder_user_error("%d RLIMIT_NICE not set\n", current->pid);
-}
-
-static struct binder_node *binder_get_node_ilocked(struct binder_proc *proc,
-						   binder_uintptr_t ptr)
-{
-	struct rb_node *n = proc->nodes.rb_node;
-	struct binder_node *node;
-
-	assert_spin_locked(&proc->inner_lock);
-
-	while (n) {
-		node = rb_entry(n, struct binder_node, rb_node);
-
-		if (ptr < node->ptr)
-			n = n->rb_left;
-		else if (ptr > node->ptr)
-			n = n->rb_right;
-		else {
-			/*
-			 * take an implicit weak reference
-			 * to ensure node stays alive until
-			 * call to binder_put_node()
-			 */
-			binder_inc_node_tmpref_ilocked(node);
-			return node;
-		}
-	}
-	return NULL;
-}
-
-static struct binder_node *binder_get_node(struct binder_proc *proc,
-					   binder_uintptr_t ptr)
-{
-	struct binder_node *node;
-
-	binder_inner_proc_lock(proc);
-	node = binder_get_node_ilocked(proc, ptr);
-	binder_inner_proc_unlock(proc);
-	return node;
-}
-
-static struct binder_node *binder_init_node_ilocked(
-						struct binder_proc *proc,
-						struct binder_node *new_node,
-						struct flat_binder_object *fp)
-{
-	struct rb_node **p = &proc->nodes.rb_node;
-	struct rb_node *parent = NULL;
-	struct binder_node *node;
-	binder_uintptr_t ptr = fp ? fp->binder : 0;
-	binder_uintptr_t cookie = fp ? fp->cookie : 0;
-	__u32 flags = fp ? fp->flags : 0;
-
-	assert_spin_locked(&proc->inner_lock);
-
-	while (*p) {
-
-		parent = *p;
-		node = rb_entry(parent, struct binder_node, rb_node);
-
-		if (ptr < node->ptr)
-			p = &(*p)->rb_left;
-		else if (ptr > node->ptr)
-			p = &(*p)->rb_right;
-		else {
-			/*
-			 * A matching node is already in
-			 * the rb tree. Abandon the init
-			 * and return it.
-			 */
-			binder_inc_node_tmpref_ilocked(node);
-			return node;
-		}
-	}
-	node = new_node;
-	binder_stats_created(BINDER_STAT_NODE);
-	node->tmp_refs++;
-	rb_link_node(&node->rb_node, parent, p);
-	rb_insert_color(&node->rb_node, &proc->nodes);
-	node->debug_id = atomic_inc_return(&binder_last_id);
-	node->proc = proc;
-	node->ptr = ptr;
-	node->cookie = cookie;
-	node->work.type = BINDER_WORK_NODE;
-	node->min_priority = flags & FLAT_BINDER_FLAG_PRIORITY_MASK;
-	node->accept_fds = !!(flags & FLAT_BINDER_FLAG_ACCEPTS_FDS);
-	node->txn_security_ctx = !!(flags & FLAT_BINDER_FLAG_TXN_SECURITY_CTX);
-	spin_lock_init(&node->lock);
-	INIT_LIST_HEAD(&node->work.entry);
-	INIT_LIST_HEAD(&node->async_todo);
-	binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-		     "%d:%d node %d u%016llx c%016llx created\n",
-		     proc->pid, current->pid, node->debug_id,
-		     (u64)node->ptr, (u64)node->cookie);
-
-	return node;
-}
-
-static struct binder_node *binder_new_node(struct binder_proc *proc,
-					   struct flat_binder_object *fp)
-{
-	struct binder_node *node;
-	struct binder_node *new_node = kzalloc_obj(*node);
-
-	if (!new_node)
-		return NULL;
-	binder_inner_proc_lock(proc);
-	node = binder_init_node_ilocked(proc, new_node, fp);
-	binder_inner_proc_unlock(proc);
-	if (node != new_node)
-		/*
-		 * The node was already added by another thread
-		 */
-		kfree(new_node);
-
-	return node;
-}
-
-static void binder_free_node(struct binder_node *node)
-{
-	kfree(node);
-	binder_stats_deleted(BINDER_STAT_NODE);
-}
-
-static int binder_inc_node_nilocked(struct binder_node *node, int strong,
-				    int internal,
-				    struct list_head *target_list)
-{
-	struct binder_proc *proc = node->proc;
-
-	assert_spin_locked(&node->lock);
-	if (proc)
-		assert_spin_locked(&proc->inner_lock);
-	if (strong) {
-		if (internal) {
-			if (target_list == NULL &&
-			    node->internal_strong_refs == 0 &&
-			    !(node->proc &&
-			      node == node->proc->context->binder_context_mgr_node &&
-			      node->has_strong_ref)) {
-				pr_err("invalid inc strong node for %d\n",
-					node->debug_id);
-				return -EINVAL;
-			}
-			node->internal_strong_refs++;
-		} else
-			node->local_strong_refs++;
-		if (!node->has_strong_ref && target_list) {
-			struct binder_thread *thread = container_of(target_list,
-						    struct binder_thread, todo);
-			binder_dequeue_work_ilocked(&node->work);
-			BUG_ON(&thread->todo != target_list);
-			binder_enqueue_deferred_thread_work_ilocked(thread,
-								   &node->work);
-		}
-	} else {
-		if (!internal)
-			node->local_weak_refs++;
-		if (!node->has_weak_ref && target_list && list_empty(&node->work.entry))
-			binder_enqueue_work_ilocked(&node->work, target_list);
-	}
-	return 0;
-}
-
-static int binder_inc_node(struct binder_node *node, int strong, int internal,
-			   struct list_head *target_list)
-{
-	int ret;
-
-	binder_node_inner_lock(node);
-	ret = binder_inc_node_nilocked(node, strong, internal, target_list);
-	binder_node_inner_unlock(node);
-
-	return ret;
-}
-
-static bool binder_dec_node_nilocked(struct binder_node *node,
-				     int strong, int internal)
-{
-	struct binder_proc *proc = node->proc;
-
-	assert_spin_locked(&node->lock);
-	if (proc)
-		assert_spin_locked(&proc->inner_lock);
-	if (strong) {
-		if (internal)
-			node->internal_strong_refs--;
-		else
-			node->local_strong_refs--;
-		if (node->local_strong_refs || node->internal_strong_refs)
-			return false;
-	} else {
-		if (!internal)
-			node->local_weak_refs--;
-		if (node->local_weak_refs || node->tmp_refs ||
-				!hlist_empty(&node->refs))
-			return false;
-	}
-
-	if (proc && (node->has_strong_ref || node->has_weak_ref)) {
-		if (list_empty(&node->work.entry)) {
-			binder_enqueue_work_ilocked(&node->work, &proc->todo);
-			binder_wakeup_proc_ilocked(proc);
-		}
-	} else {
-		if (hlist_empty(&node->refs) && !node->local_strong_refs &&
-		    !node->local_weak_refs && !node->tmp_refs) {
-			if (proc) {
-				binder_dequeue_work_ilocked(&node->work);
-				rb_erase(&node->rb_node, &proc->nodes);
-				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-					     "refless node %d deleted\n",
-					     node->debug_id);
-			} else {
-				BUG_ON(!list_empty(&node->work.entry));
-				spin_lock(&binder_dead_nodes_lock);
-				/*
-				 * tmp_refs could have changed so
-				 * check it again
-				 */
-				if (node->tmp_refs) {
-					spin_unlock(&binder_dead_nodes_lock);
-					return false;
-				}
-				hlist_del(&node->dead_node);
-				spin_unlock(&binder_dead_nodes_lock);
-				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-					     "dead node %d deleted\n",
-					     node->debug_id);
-			}
-			return true;
-		}
-	}
-	return false;
-}
-
-static void binder_dec_node(struct binder_node *node, int strong, int internal)
-{
-	bool free_node;
-
-	binder_node_inner_lock(node);
-	free_node = binder_dec_node_nilocked(node, strong, internal);
-	binder_node_inner_unlock(node);
-	if (free_node)
-		binder_free_node(node);
-}
-
-static void binder_inc_node_tmpref_ilocked(struct binder_node *node)
-{
-	/*
-	 * No call to binder_inc_node() is needed since we
-	 * don't need to inform userspace of any changes to
-	 * tmp_refs
-	 */
-	node->tmp_refs++;
-}
-
-/**
- * binder_inc_node_tmpref() - take a temporary reference on node
- * @node:	node to reference
- *
- * Take reference on node to prevent the node from being freed
- * while referenced only by a local variable. The inner lock is
- * needed to serialize with the node work on the queue (which
- * isn't needed after the node is dead). If the node is dead
- * (node->proc is NULL), use binder_dead_nodes_lock to protect
- * node->tmp_refs against dead-node-only cases where the node
- * lock cannot be acquired (eg traversing the dead node list to
- * print nodes)
- */
-static void binder_inc_node_tmpref(struct binder_node *node)
-{
-	binder_node_lock(node);
-	if (node->proc)
-		binder_inner_proc_lock(node->proc);
-	else
-		spin_lock(&binder_dead_nodes_lock);
-	binder_inc_node_tmpref_ilocked(node);
-	if (node->proc)
-		binder_inner_proc_unlock(node->proc);
-	else
-		spin_unlock(&binder_dead_nodes_lock);
-	binder_node_unlock(node);
-}
-
-/**
- * binder_dec_node_tmpref() - remove a temporary reference on node
- * @node:	node to reference
- *
- * Release temporary reference on node taken via binder_inc_node_tmpref()
- */
-static void binder_dec_node_tmpref(struct binder_node *node)
-{
-	bool free_node;
-
-	binder_node_inner_lock(node);
-	if (!node->proc)
-		spin_lock(&binder_dead_nodes_lock);
-	else
-		__acquire(&binder_dead_nodes_lock);
-	node->tmp_refs--;
-	BUG_ON(node->tmp_refs < 0);
-	if (!node->proc)
-		spin_unlock(&binder_dead_nodes_lock);
-	else
-		__release(&binder_dead_nodes_lock);
-	/*
-	 * Call binder_dec_node() to check if all refcounts are 0
-	 * and cleanup is needed. Calling with strong=0 and internal=1
-	 * causes no actual reference to be released in binder_dec_node().
-	 * If that changes, a change is needed here too.
-	 */
-	free_node = binder_dec_node_nilocked(node, 0, 1);
-	binder_node_inner_unlock(node);
-	if (free_node)
-		binder_free_node(node);
-}
-
-static void binder_put_node(struct binder_node *node)
-{
-	binder_dec_node_tmpref(node);
-}
-
-static struct binder_ref *binder_get_ref_olocked(struct binder_proc *proc,
-						 u32 desc, bool need_strong_ref)
-{
-	struct rb_node *n = proc->refs_by_desc.rb_node;
-	struct binder_ref *ref;
-
-	while (n) {
-		ref = rb_entry(n, struct binder_ref, rb_node_desc);
-
-		if (desc < ref->data.desc) {
-			n = n->rb_left;
-		} else if (desc > ref->data.desc) {
-			n = n->rb_right;
-		} else if (need_strong_ref && !ref->data.strong) {
-			binder_user_error("tried to use weak ref as strong ref\n");
-			return NULL;
-		} else {
-			return ref;
-		}
-	}
-	return NULL;
-}
-
-/* Find the smallest unused descriptor the "slow way" */
-static u32 slow_desc_lookup_olocked(struct binder_proc *proc, u32 offset)
-{
-	struct binder_ref *ref;
-	struct rb_node *n;
-	u32 desc;
-
-	desc = offset;
-	for (n = rb_first(&proc->refs_by_desc); n; n = rb_next(n)) {
-		ref = rb_entry(n, struct binder_ref, rb_node_desc);
-		if (ref->data.desc > desc)
-			break;
-		desc = ref->data.desc + 1;
-	}
-
-	return desc;
-}
-
-/*
- * Find an available reference descriptor ID. The proc->outer_lock might
- * be released in the process, in which case -EAGAIN is returned and the
- * @desc should be considered invalid.
- */
-static int get_ref_desc_olocked(struct binder_proc *proc,
-				struct binder_node *node,
-				u32 *desc)
-{
-	struct dbitmap *dmap = &proc->dmap;
-	unsigned int nbits, offset;
-	unsigned long *new, bit;
-
-	/* 0 is reserved for the context manager */
-	offset = (node == proc->context->binder_context_mgr_node) ? 0 : 1;
-
-	if (!dbitmap_enabled(dmap)) {
-		*desc = slow_desc_lookup_olocked(proc, offset);
-		return 0;
-	}
-
-	if (dbitmap_acquire_next_zero_bit(dmap, offset, &bit) == 0) {
-		*desc = bit;
-		return 0;
-	}
-
-	/*
-	 * The dbitmap is full and needs to grow. The proc->outer_lock
-	 * is briefly released to allocate the new bitmap safely.
-	 */
-	nbits = dbitmap_grow_nbits(dmap);
-	binder_proc_unlock(proc);
-	new = bitmap_zalloc(nbits, GFP_KERNEL);
-	binder_proc_lock(proc);
-	dbitmap_grow(dmap, new, nbits);
-
-	return -EAGAIN;
-}
-
-/**
- * binder_get_ref_for_node_olocked() - get the ref associated with given node
- * @proc:	binder_proc that owns the ref
- * @node:	binder_node of target
- * @new_ref:	newly allocated binder_ref to be initialized or %NULL
- *
- * Look up the ref for the given node and return it if it exists
- *
- * If it doesn't exist and the caller provides a newly allocated
- * ref, initialize the fields of the newly allocated ref and insert
- * into the given proc rb_trees and node refs list.
- *
- * Return:	the ref for node. It is possible that another thread
- *		allocated/initialized the ref first in which case the
- *		returned ref would be different than the passed-in
- *		new_ref. new_ref must be kfree'd by the caller in
- *		this case.
- */
-static struct binder_ref *binder_get_ref_for_node_olocked(
-					struct binder_proc *proc,
-					struct binder_node *node,
-					struct binder_ref *new_ref)
-{
-	struct binder_ref *ref;
-	struct rb_node *parent;
-	struct rb_node **p;
-	u32 desc;
-
-retry:
-	p = &proc->refs_by_node.rb_node;
-	parent = NULL;
-	while (*p) {
-		parent = *p;
-		ref = rb_entry(parent, struct binder_ref, rb_node_node);
-
-		if (node < ref->node)
-			p = &(*p)->rb_left;
-		else if (node > ref->node)
-			p = &(*p)->rb_right;
-		else
-			return ref;
-	}
-	if (!new_ref)
-		return NULL;
-
-	/* might release the proc->outer_lock */
-	if (get_ref_desc_olocked(proc, node, &desc) == -EAGAIN)
-		goto retry;
-
-	binder_stats_created(BINDER_STAT_REF);
-	new_ref->data.debug_id = atomic_inc_return(&binder_last_id);
-	new_ref->proc = proc;
-	new_ref->node = node;
-	rb_link_node(&new_ref->rb_node_node, parent, p);
-	rb_insert_color(&new_ref->rb_node_node, &proc->refs_by_node);
-
-	new_ref->data.desc = desc;
-	p = &proc->refs_by_desc.rb_node;
-	while (*p) {
-		parent = *p;
-		ref = rb_entry(parent, struct binder_ref, rb_node_desc);
-
-		if (new_ref->data.desc < ref->data.desc)
-			p = &(*p)->rb_left;
-		else if (new_ref->data.desc > ref->data.desc)
-			p = &(*p)->rb_right;
-		else
-			BUG();
-	}
-	rb_link_node(&new_ref->rb_node_desc, parent, p);
-	rb_insert_color(&new_ref->rb_node_desc, &proc->refs_by_desc);
-
-	binder_node_lock(node);
-	hlist_add_head(&new_ref->node_entry, &node->refs);
-
-	binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-		     "%d new ref %d desc %d for node %d\n",
-		      proc->pid, new_ref->data.debug_id, new_ref->data.desc,
-		      node->debug_id);
-	binder_node_unlock(node);
-	return new_ref;
-}
-
-static void binder_cleanup_ref_olocked(struct binder_ref *ref)
-{
-	struct dbitmap *dmap = &ref->proc->dmap;
-	bool delete_node = false;
-
-	binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-		     "%d delete ref %d desc %d for node %d\n",
-		      ref->proc->pid, ref->data.debug_id, ref->data.desc,
-		      ref->node->debug_id);
-
-	if (dbitmap_enabled(dmap))
-		dbitmap_clear_bit(dmap, ref->data.desc);
-	rb_erase(&ref->rb_node_desc, &ref->proc->refs_by_desc);
-	rb_erase(&ref->rb_node_node, &ref->proc->refs_by_node);
-
-	binder_node_inner_lock(ref->node);
-	if (ref->data.strong)
-		binder_dec_node_nilocked(ref->node, 1, 1);
-
-	hlist_del(&ref->node_entry);
-	delete_node = binder_dec_node_nilocked(ref->node, 0, 1);
-	binder_node_inner_unlock(ref->node);
-	/*
-	 * Clear ref->node unless we want the caller to free the node
-	 */
-	if (!delete_node) {
-		/*
-		 * The caller uses ref->node to determine
-		 * whether the node needs to be freed. Clear
-		 * it since the node is still alive.
-		 */
-		ref->node = NULL;
-	}
-
-	if (ref->death) {
-		binder_debug(BINDER_DEBUG_DEAD_BINDER,
-			     "%d delete ref %d desc %d has death notification\n",
-			      ref->proc->pid, ref->data.debug_id,
-			      ref->data.desc);
-		binder_dequeue_work(ref->proc, &ref->death->work);
-		binder_stats_deleted(BINDER_STAT_DEATH);
-	}
-
-	if (ref->freeze) {
-		binder_dequeue_work(ref->proc, &ref->freeze->work);
-		binder_stats_deleted(BINDER_STAT_FREEZE);
-	}
-
-	binder_stats_deleted(BINDER_STAT_REF);
-}
-
-/**
- * binder_inc_ref_olocked() - increment the ref for given handle
- * @ref:         ref to be incremented
- * @strong:      if true, strong increment, else weak
- * @target_list: list to queue node work on
- *
- * Increment the ref. @ref->proc->outer_lock must be held on entry
- *
- * Return: 0, if successful, else errno
- */
-static int binder_inc_ref_olocked(struct binder_ref *ref, int strong,
-				  struct list_head *target_list)
-{
-	int ret;
-
-	if (strong) {
-		if (ref->data.strong == 0) {
-			ret = binder_inc_node(ref->node, 1, 1, target_list);
-			if (ret)
-				return ret;
-		}
-		ref->data.strong++;
-	} else {
-		if (ref->data.weak == 0) {
-			ret = binder_inc_node(ref->node, 0, 1, target_list);
-			if (ret)
-				return ret;
-		}
-		ref->data.weak++;
-	}
-	return 0;
-}
-
-/**
- * binder_dec_ref_olocked() - dec the ref for given handle
- * @ref:	ref to be decremented
- * @strong:	if true, strong decrement, else weak
- *
- * Decrement the ref.
- *
- * Return: %true if ref is cleaned up and ready to be freed.
- */
-static bool binder_dec_ref_olocked(struct binder_ref *ref, int strong)
-{
-	if (strong) {
-		if (ref->data.strong == 0) {
-			binder_user_error("%d invalid dec strong, ref %d desc %d s %d w %d\n",
-					  ref->proc->pid, ref->data.debug_id,
-					  ref->data.desc, ref->data.strong,
-					  ref->data.weak);
-			return false;
-		}
-		ref->data.strong--;
-		if (ref->data.strong == 0)
-			binder_dec_node(ref->node, strong, 1);
-	} else {
-		if (ref->data.weak == 0) {
-			binder_user_error("%d invalid dec weak, ref %d desc %d s %d w %d\n",
-					  ref->proc->pid, ref->data.debug_id,
-					  ref->data.desc, ref->data.strong,
-					  ref->data.weak);
-			return false;
-		}
-		ref->data.weak--;
-	}
-	if (ref->data.strong == 0 && ref->data.weak == 0) {
-		binder_cleanup_ref_olocked(ref);
-		return true;
-	}
-	return false;
-}
-
-/**
- * binder_get_node_from_ref() - get the node from the given proc/desc
- * @proc:	proc containing the ref
- * @desc:	the handle associated with the ref
- * @need_strong_ref: if true, only return node if ref is strong
- * @rdata:	the id/refcount data for the ref
- *
- * Given a proc and ref handle, return the associated binder_node
- *
- * Return: a binder_node or NULL if not found or not strong when strong required
- */
-static struct binder_node *binder_get_node_from_ref(
-		struct binder_proc *proc,
-		u32 desc, bool need_strong_ref,
-		struct binder_ref_data *rdata)
-{
-	struct binder_node *node;
-	struct binder_ref *ref;
-
-	binder_proc_lock(proc);
-	ref = binder_get_ref_olocked(proc, desc, need_strong_ref);
-	if (!ref)
-		goto err_no_ref;
-	node = ref->node;
-	/*
-	 * Take an implicit reference on the node to ensure
-	 * it stays alive until the call to binder_put_node()
-	 */
-	binder_inc_node_tmpref(node);
-	if (rdata)
-		*rdata = ref->data;
-	binder_proc_unlock(proc);
-
-	return node;
-
-err_no_ref:
-	binder_proc_unlock(proc);
-	return NULL;
-}
-
-/**
- * binder_free_ref() - free the binder_ref
- * @ref:	ref to free
- *
- * Free the binder_ref. Free the binder_node indicated by ref->node
- * (if non-NULL) and the binder_ref_death indicated by ref->death.
- */
-static void binder_free_ref(struct binder_ref *ref)
-{
-	if (ref->node)
-		binder_free_node(ref->node);
-	kfree(ref->death);
-	kfree(ref->freeze);
-	kfree(ref);
-}
-
-/* shrink descriptor bitmap if needed */
-static void try_shrink_dmap(struct binder_proc *proc)
-{
-	unsigned long *new;
-	int nbits;
-
-	binder_proc_lock(proc);
-	nbits = dbitmap_shrink_nbits(&proc->dmap);
-	binder_proc_unlock(proc);
-
-	if (!nbits)
-		return;
-
-	new = bitmap_zalloc(nbits, GFP_KERNEL);
-	binder_proc_lock(proc);
-	dbitmap_shrink(&proc->dmap, new, nbits);
-	binder_proc_unlock(proc);
-}
-
-/**
- * binder_update_ref_for_handle() - inc/dec the ref for given handle
- * @proc:	proc containing the ref
- * @desc:	the handle associated with the ref
- * @increment:	true=inc reference, false=dec reference
- * @strong:	true=strong reference, false=weak reference
- * @rdata:	the id/refcount data for the ref
- *
- * Given a proc and ref handle, increment or decrement the ref
- * according to "increment" arg.
- *
- * Return: 0 if successful, else errno
- */
-static int binder_update_ref_for_handle(struct binder_proc *proc,
-		uint32_t desc, bool increment, bool strong,
-		struct binder_ref_data *rdata)
-{
-	int ret = 0;
-	struct binder_ref *ref;
-	bool delete_ref = false;
-
-	binder_proc_lock(proc);
-	ref = binder_get_ref_olocked(proc, desc, strong);
-	if (!ref) {
-		ret = -EINVAL;
-		goto err_no_ref;
-	}
-	if (increment)
-		ret = binder_inc_ref_olocked(ref, strong, NULL);
-	else
-		delete_ref = binder_dec_ref_olocked(ref, strong);
-
-	if (rdata)
-		*rdata = ref->data;
-	binder_proc_unlock(proc);
-
-	if (delete_ref) {
-		binder_free_ref(ref);
-		try_shrink_dmap(proc);
-	}
-	return ret;
-
-err_no_ref:
-	binder_proc_unlock(proc);
-	return ret;
-}
-
-/**
- * binder_dec_ref_for_handle() - dec the ref for given handle
- * @proc:	proc containing the ref
- * @desc:	the handle associated with the ref
- * @strong:	true=strong reference, false=weak reference
- * @rdata:	the id/refcount data for the ref
- *
- * Just calls binder_update_ref_for_handle() to decrement the ref.
- *
- * Return: 0 if successful, else errno
- */
-static int binder_dec_ref_for_handle(struct binder_proc *proc,
-		uint32_t desc, bool strong, struct binder_ref_data *rdata)
-{
-	return binder_update_ref_for_handle(proc, desc, false, strong, rdata);
-}
-
-
-/**
- * binder_inc_ref_for_node() - increment the ref for given proc/node
- * @proc:	 proc containing the ref
- * @node:	 target node
- * @strong:	 true=strong reference, false=weak reference
- * @target_list: worklist to use if node is incremented
- * @rdata:	 the id/refcount data for the ref
- *
- * Given a proc and node, increment the ref. Create the ref if it
- * doesn't already exist
- *
- * Return: 0 if successful, else errno
- */
-static int binder_inc_ref_for_node(struct binder_proc *proc,
-			struct binder_node *node,
-			bool strong,
-			struct list_head *target_list,
-			struct binder_ref_data *rdata)
-{
-	struct binder_ref *ref;
-	struct binder_ref *new_ref = NULL;
-	int ret = 0;
-
-	binder_proc_lock(proc);
-	ref = binder_get_ref_for_node_olocked(proc, node, NULL);
-	if (!ref) {
-		binder_proc_unlock(proc);
-		new_ref = kzalloc_obj(*ref);
-		if (!new_ref)
-			return -ENOMEM;
-		binder_proc_lock(proc);
-		ref = binder_get_ref_for_node_olocked(proc, node, new_ref);
-	}
-	ret = binder_inc_ref_olocked(ref, strong, target_list);
-	*rdata = ref->data;
-	if (ret && ref == new_ref) {
-		/*
-		 * Cleanup the failed reference here as the target
-		 * could now be dead and have already released its
-		 * references by now. Calling on the new reference
-		 * with strong=0 and a tmp_refs will not decrement
-		 * the node. The new_ref gets kfree'd below.
-		 */
-		binder_cleanup_ref_olocked(new_ref);
-		ref = NULL;
-	}
-
-	binder_proc_unlock(proc);
-	if (new_ref && ref != new_ref)
-		/*
-		 * Another thread created the ref first so
-		 * free the one we allocated
-		 */
-		kfree(new_ref);
-	return ret;
-}
-
-static void binder_pop_transaction_ilocked(struct binder_thread *target_thread,
-					   struct binder_transaction *t)
-{
-	BUG_ON(!target_thread);
-	assert_spin_locked(&target_thread->proc->inner_lock);
-	BUG_ON(target_thread->transaction_stack != t);
-	BUG_ON(target_thread->transaction_stack->from != target_thread);
-	target_thread->transaction_stack =
-		target_thread->transaction_stack->from_parent;
-	t->from = NULL;
-}
-
-/**
- * binder_thread_dec_tmpref() - decrement thread->tmp_ref
- * @thread:	thread to decrement
- *
- * A thread needs to be kept alive while being used to create or
- * handle a transaction. binder_get_txn_from() is used to safely
- * extract t->from from a binder_transaction and keep the thread
- * indicated by t->from from being freed. When done with that
- * binder_thread, this function is called to decrement the
- * tmp_ref and free if appropriate (thread has been released
- * and no transaction being processed by the driver)
- */
-static void binder_thread_dec_tmpref(struct binder_thread *thread)
-{
-	/*
-	 * atomic is used to protect the counter value while
-	 * it cannot reach zero or thread->is_dead is false
-	 */
-	binder_inner_proc_lock(thread->proc);
-	atomic_dec(&thread->tmp_ref);
-	if (thread->is_dead && !atomic_read(&thread->tmp_ref)) {
-		binder_inner_proc_unlock(thread->proc);
-		binder_free_thread(thread);
-		return;
-	}
-	binder_inner_proc_unlock(thread->proc);
-}
-
-/**
- * binder_proc_dec_tmpref() - decrement proc->tmp_ref
- * @proc:	proc to decrement
- *
- * A binder_proc needs to be kept alive while being used to create or
- * handle a transaction. proc->tmp_ref is incremented when
- * creating a new transaction or the binder_proc is currently in-use
- * by threads that are being released. When done with the binder_proc,
- * this function is called to decrement the counter and free the
- * proc if appropriate (proc has been released, all threads have
- * been released and not currently in-use to process a transaction).
- */
-static void binder_proc_dec_tmpref(struct binder_proc *proc)
-{
-	binder_inner_proc_lock(proc);
-	proc->tmp_ref--;
-	if (proc->is_dead && RB_EMPTY_ROOT(&proc->threads) &&
-			!proc->tmp_ref) {
-		binder_inner_proc_unlock(proc);
-		binder_free_proc(proc);
-		return;
-	}
-	binder_inner_proc_unlock(proc);
-}
-
-/**
- * binder_get_txn_from() - safely extract the "from" thread in transaction
- * @t:	binder transaction for t->from
- *
- * Atomically return the "from" thread and increment the tmp_ref
- * count for the thread to ensure it stays alive until
- * binder_thread_dec_tmpref() is called.
- *
- * Return: the value of t->from
- */
-static struct binder_thread *binder_get_txn_from(
-		struct binder_transaction *t)
-{
-	struct binder_thread *from;
-
-	guard(spinlock)(&t->lock);
-	from = t->from;
-	if (from)
-		atomic_inc(&from->tmp_ref);
-	return from;
-}
-
-/**
- * binder_get_txn_from_and_acq_inner() - get t->from and acquire inner lock
- * @t:	binder transaction for t->from
- *
- * Same as binder_get_txn_from() except it also acquires the proc->inner_lock
- * to guarantee that the thread cannot be released while operating on it.
- * The caller must call binder_inner_proc_unlock() to release the inner lock
- * as well as call binder_dec_thread_txn() to release the reference.
- *
- * Return: the value of t->from
- */
-static struct binder_thread *binder_get_txn_from_and_acq_inner(
-		struct binder_transaction *t)
-	__acquires(&t->from->proc->inner_lock)
-{
-	struct binder_thread *from;
-
-	from = binder_get_txn_from(t);
-	if (!from) {
-		__acquire(&from->proc->inner_lock);
-		return NULL;
-	}
-	binder_inner_proc_lock(from->proc);
-	if (t->from) {
-		BUG_ON(from != t->from);
-		return from;
-	}
-	binder_inner_proc_unlock(from->proc);
-	__acquire(&from->proc->inner_lock);
-	binder_thread_dec_tmpref(from);
-	return NULL;
-}
-
-/**
- * binder_free_txn_fixups() - free unprocessed fd fixups
- * @t:	binder transaction for t->from
- *
- * If the transaction is being torn down prior to being
- * processed by the target process, free all of the
- * fd fixups and fput the file structs. It is safe to
- * call this function after the fixups have been
- * processed -- in that case, the list will be empty.
- */
-static void binder_free_txn_fixups(struct binder_transaction *t)
-{
-	struct binder_txn_fd_fixup *fixup, *tmp;
-
-	list_for_each_entry_safe(fixup, tmp, &t->fd_fixups, fixup_entry) {
-		fput(fixup->file);
-		if (fixup->target_fd >= 0)
-			put_unused_fd(fixup->target_fd);
-		list_del(&fixup->fixup_entry);
-		kfree(fixup);
-	}
-}
-
-static void binder_txn_latency_free(struct binder_transaction *t)
-{
-	int from_proc, from_thread, to_proc, to_thread;
-
-	spin_lock(&t->lock);
-	from_proc = t->from ? t->from->proc->pid : 0;
-	from_thread = t->from ? t->from->pid : 0;
-	to_proc = t->to_proc ? t->to_proc->pid : 0;
-	to_thread = t->to_thread ? t->to_thread->pid : 0;
-	spin_unlock(&t->lock);
-
-	trace_binder_txn_latency_free(t, from_proc, from_thread, to_proc, to_thread);
-}
-
-static void binder_free_transaction(struct binder_transaction *t)
-{
-	struct binder_thread *target_thread;
-	struct binder_proc *target_proc;
-
-	spin_lock(&t->lock);
-	target_proc = t->to_proc;
-	target_thread = t->to_thread;
-	/*
-	 * Pin target_thread to keep target_proc alive. Undelivered
-	 * transactions with !target_thread are safe, as target_proc
-	 * can only be the current context there.
-	 */
-	if (target_thread)
-		atomic_inc(&target_thread->tmp_ref);
-	spin_unlock(&t->lock);
-
-	if (target_proc) {
-		binder_inner_proc_lock(target_proc);
-		target_proc->outstanding_txns--;
-		if (target_proc->outstanding_txns < 0)
-			pr_warn("%s: Unexpected outstanding_txns %d\n",
-				__func__, target_proc->outstanding_txns);
-		if (!target_proc->outstanding_txns && target_proc->is_frozen)
-			wake_up_interruptible_all(&target_proc->freeze_wait);
-		if (t->buffer)
-			t->buffer->transaction = NULL;
-		binder_inner_proc_unlock(target_proc);
-	}
-
-	if (target_thread)
-		binder_thread_dec_tmpref(target_thread);
-
-	if (trace_binder_txn_latency_free_enabled())
-		binder_txn_latency_free(t);
-	/*
-	 * If the transaction has no target_proc, then
-	 * t->buffer->transaction has already been cleared.
-	 */
-	binder_free_txn_fixups(t);
-	kfree(t);
-	binder_stats_deleted(BINDER_STAT_TRANSACTION);
-}
-
-static void binder_send_failed_reply(struct binder_transaction *t,
-				     uint32_t error_code)
-{
-	struct binder_thread *target_thread;
-	struct binder_transaction *next;
-
-	BUG_ON(t->flags & TF_ONE_WAY);
-	while (1) {
-		target_thread = binder_get_txn_from_and_acq_inner(t);
-		if (target_thread) {
-			binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
-				     "send failed reply for transaction %d to %d:%d\n",
-				      t->debug_id,
-				      target_thread->proc->pid,
-				      target_thread->pid);
-
-			binder_pop_transaction_ilocked(target_thread, t);
-			if (target_thread->reply_error.cmd == BR_OK) {
-				target_thread->reply_error.cmd = error_code;
-				binder_enqueue_thread_work_ilocked(
-					target_thread,
-					&target_thread->reply_error.work);
-				wake_up_interruptible(&target_thread->wait);
-			} else {
-				/*
-				 * Cannot get here for normal operation, but
-				 * we can if multiple synchronous transactions
-				 * are sent without blocking for responses.
-				 * Just ignore the 2nd error in this case.
-				 */
-				pr_warn("Unexpected reply error: %u\n",
-					target_thread->reply_error.cmd);
-			}
-			binder_inner_proc_unlock(target_thread->proc);
-			binder_thread_dec_tmpref(target_thread);
-			binder_free_transaction(t);
-			return;
-		}
-		__release(&target_thread->proc->inner_lock);
-		next = t->from_parent;
-
-		binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
-			     "send failed reply for transaction %d, target dead\n",
-			     t->debug_id);
-
-		binder_free_transaction(t);
-		if (next == NULL) {
-			binder_debug(BINDER_DEBUG_DEAD_BINDER,
-				     "reply failed, no target thread at root\n");
-			return;
-		}
-		t = next;
-		binder_debug(BINDER_DEBUG_DEAD_BINDER,
-			     "reply failed, no target thread -- retry %d\n",
-			      t->debug_id);
-	}
-}
-
-/**
- * binder_cleanup_transaction() - cleans up undelivered transaction
- * @t:		transaction that needs to be cleaned up
- * @reason:	reason the transaction wasn't delivered
- * @error_code:	error to return to caller (if synchronous call)
- */
-static void binder_cleanup_transaction(struct binder_transaction *t,
-				       const char *reason,
-				       uint32_t error_code)
-{
-	if (t->buffer->target_node && !(t->flags & TF_ONE_WAY)) {
-		binder_send_failed_reply(t, error_code);
-	} else {
-		binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
-			"undelivered transaction %d, %s\n",
-			t->debug_id, reason);
-		binder_free_transaction(t);
-	}
-}
-
-/**
- * binder_get_object() - gets object and checks for valid metadata
- * @proc:	binder_proc owning the buffer
- * @u:		sender's user pointer to base of buffer
- * @buffer:	binder_buffer that we're parsing.
- * @offset:	offset in the @buffer at which to validate an object.
- * @object:	struct binder_object to read into
- *
- * Copy the binder object at the given offset into @object. If @u is
- * provided then the copy is from the sender's buffer. If not, then
- * it is copied from the target's @buffer.
- *
- * Return:	If there's a valid metadata object at @offset, the
- *		size of that object. Otherwise, it returns zero. The object
- *		is read into the struct binder_object pointed to by @object.
- */
-static size_t binder_get_object(struct binder_proc *proc,
-				const void __user *u,
-				struct binder_buffer *buffer,
-				unsigned long offset,
-				struct binder_object *object)
-{
-	size_t read_size;
-	struct binder_object_header *hdr;
-	size_t object_size = 0;
-
-	read_size = min_t(size_t, sizeof(*object), buffer->data_size - offset);
-	if (offset > buffer->data_size || read_size < sizeof(*hdr) ||
-	    !IS_ALIGNED(offset, sizeof(u32)))
-		return 0;
-
-	if (u) {
-		if (copy_from_user(object, u + offset, read_size))
-			return 0;
-	} else {
-		if (binder_alloc_copy_from_buffer(&proc->alloc, object, buffer,
-						  offset, read_size))
-			return 0;
-	}
-
-	/* Ok, now see if we read a complete object. */
-	hdr = &object->hdr;
-	switch (hdr->type) {
-	case BINDER_TYPE_BINDER:
-	case BINDER_TYPE_WEAK_BINDER:
-	case BINDER_TYPE_HANDLE:
-	case BINDER_TYPE_WEAK_HANDLE:
-		object_size = sizeof(struct flat_binder_object);
-		break;
-	case BINDER_TYPE_FD:
-		object_size = sizeof(struct binder_fd_object);
-		break;
-	case BINDER_TYPE_PTR:
-		object_size = sizeof(struct binder_buffer_object);
-		break;
-	case BINDER_TYPE_FDA:
-		object_size = sizeof(struct binder_fd_array_object);
-		break;
-	default:
-		return 0;
-	}
-	if (offset <= buffer->data_size - object_size &&
-	    buffer->data_size >= object_size)
-		return object_size;
-	else
-		return 0;
-}
-
-/**
- * binder_validate_ptr() - validates binder_buffer_object in a binder_buffer.
- * @proc:	binder_proc owning the buffer
- * @b:		binder_buffer containing the object
- * @object:	struct binder_object to read into
- * @index:	index in offset array at which the binder_buffer_object is
- *		located
- * @start_offset: points to the start of the offset array
- * @object_offsetp: offset of @object read from @b
- * @num_valid:	the number of valid offsets in the offset array
- *
- * Return:	If @index is within the valid range of the offset array
- *		described by @start and @num_valid, and if there's a valid
- *		binder_buffer_object at the offset found in index @index
- *		of the offset array, that object is returned. Otherwise,
- *		%NULL is returned.
- *		Note that the offset found in index @index itself is not
- *		verified; this function assumes that @num_valid elements
- *		from @start were previously verified to have valid offsets.
- *		If @object_offsetp is non-NULL, then the offset within
- *		@b is written to it.
- */
-static struct binder_buffer_object *binder_validate_ptr(
-						struct binder_proc *proc,
-						struct binder_buffer *b,
-						struct binder_object *object,
-						binder_size_t index,
-						binder_size_t start_offset,
-						binder_size_t *object_offsetp,
-						binder_size_t num_valid)
-{
-	size_t object_size;
-	binder_size_t object_offset;
-	unsigned long buffer_offset;
-
-	if (index >= num_valid)
-		return NULL;
-
-	buffer_offset = start_offset + sizeof(binder_size_t) * index;
-	if (binder_alloc_copy_from_buffer(&proc->alloc, &object_offset,
-					  b, buffer_offset,
-					  sizeof(object_offset)))
-		return NULL;
-	object_size = binder_get_object(proc, NULL, b, object_offset, object);
-	if (!object_size || object->hdr.type != BINDER_TYPE_PTR)
-		return NULL;
-	if (object_offsetp)
-		*object_offsetp = object_offset;
-
-	return &object->bbo;
-}
-
-/**
- * binder_validate_fixup() - validates pointer/fd fixups happen in order.
- * @proc:		binder_proc owning the buffer
- * @b:			transaction buffer
- * @objects_start_offset: offset to start of objects buffer
- * @buffer_obj_offset:	offset to binder_buffer_object in which to fix up
- * @fixup_offset:	start offset in @buffer to fix up
- * @last_obj_offset:	offset to last binder_buffer_object that we fixed
- * @last_min_offset:	minimum fixup offset in object at @last_obj_offset
- *
- * Return:		%true if a fixup in buffer @buffer at offset @offset is
- *			allowed.
- *
- * For safety reasons, we only allow fixups inside a buffer to happen
- * at increasing offsets; additionally, we only allow fixup on the last
- * buffer object that was verified, or one of its parents.
- *
- * Example of what is allowed:
- *
- * A
- *   B (parent = A, offset = 0)
- *   C (parent = A, offset = 16)
- *     D (parent = C, offset = 0)
- *   E (parent = A, offset = 32) // min_offset is 16 (C.parent_offset)
- *
- * Examples of what is not allowed:
- *
- * Decreasing offsets within the same parent:
- * A
- *   C (parent = A, offset = 16)
- *   B (parent = A, offset = 0) // decreasing offset within A
- *
- * Referring to a parent that wasn't the last object or any of its parents:
- * A
- *   B (parent = A, offset = 0)
- *   C (parent = A, offset = 0)
- *   C (parent = A, offset = 16)
- *     D (parent = B, offset = 0) // B is not A or any of A's parents
- */
-static bool binder_validate_fixup(struct binder_proc *proc,
-				  struct binder_buffer *b,
-				  binder_size_t objects_start_offset,
-				  binder_size_t buffer_obj_offset,
-				  binder_size_t fixup_offset,
-				  binder_size_t last_obj_offset,
-				  binder_size_t last_min_offset)
-{
-	if (!last_obj_offset) {
-		/* Nothing to fix up in */
-		return false;
-	}
-
-	while (last_obj_offset != buffer_obj_offset) {
-		unsigned long buffer_offset;
-		struct binder_object last_object;
-		struct binder_buffer_object *last_bbo;
-		size_t object_size = binder_get_object(proc, NULL, b,
-						       last_obj_offset,
-						       &last_object);
-		if (object_size != sizeof(*last_bbo))
-			return false;
-
-		last_bbo = &last_object.bbo;
-		/*
-		 * Safe to retrieve the parent of last_obj, since it
-		 * was already previously verified by the driver.
-		 */
-		if ((last_bbo->flags & BINDER_BUFFER_FLAG_HAS_PARENT) == 0)
-			return false;
-		last_min_offset = last_bbo->parent_offset + sizeof(uintptr_t);
-		buffer_offset = objects_start_offset +
-			sizeof(binder_size_t) * last_bbo->parent;
-		if (binder_alloc_copy_from_buffer(&proc->alloc,
-						  &last_obj_offset,
-						  b, buffer_offset,
-						  sizeof(last_obj_offset)))
-			return false;
-	}
-	return (fixup_offset >= last_min_offset);
-}
-
-/**
- * struct binder_task_work_cb - for deferred close
- *
- * @twork:                callback_head for task work
- * @file:                 file to close
- *
- * Structure to pass task work to be handled after
- * returning from binder_ioctl() via task_work_add().
- */
-struct binder_task_work_cb {
-	struct callback_head twork;
-	struct file *file;
-};
-
-/**
- * binder_do_fd_close() - close list of file descriptors
- * @twork:	callback head for task work
- *
- * It is not safe to call ksys_close() during the binder_ioctl()
- * function if there is a chance that binder's own file descriptor
- * might be closed. This is to meet the requirements for using
- * fdget() (see comments for __fget_light()). Therefore use
- * task_work_add() to schedule the close operation once we have
- * returned from binder_ioctl(). This function is a callback
- * for that mechanism and does the actual ksys_close() on the
- * given file descriptor.
- */
-static void binder_do_fd_close(struct callback_head *twork)
-{
-	struct binder_task_work_cb *twcb = container_of(twork,
-			struct binder_task_work_cb, twork);
-
-	fput(twcb->file);
-	kfree(twcb);
-}
-
-/**
- * binder_deferred_fd_close() - schedule a close for the given file-descriptor
- * @fd:		file-descriptor to close
- *
- * See comments in binder_do_fd_close(). This function is used to schedule
- * a file-descriptor to be closed after returning from binder_ioctl().
- */
-static void binder_deferred_fd_close(int fd)
-{
-	struct binder_task_work_cb *twcb;
-
-	twcb = kzalloc_obj(*twcb);
-	if (!twcb)
-		return;
-	init_task_work(&twcb->twork, binder_do_fd_close);
-	twcb->file = file_close_fd(fd);
-	if (twcb->file) {
-		// pin it until binder_do_fd_close(); see comments there
-		get_file(twcb->file);
-		filp_close(twcb->file, current->files);
-		task_work_add(current, &twcb->twork, TWA_RESUME);
-	} else {
-		kfree(twcb);
-	}
-}
-
-static void binder_transaction_buffer_release(struct binder_proc *proc,
-					      struct binder_thread *thread,
-					      struct binder_buffer *buffer,
-					      binder_size_t off_end_offset,
-					      bool is_failure)
-{
-	int debug_id = buffer->debug_id;
-	binder_size_t off_start_offset, buffer_offset;
-
-	binder_debug(BINDER_DEBUG_TRANSACTION,
-		     "%d buffer release %d, size %zd-%zd, failed at %llx\n",
-		     proc->pid, buffer->debug_id,
-		     buffer->data_size, buffer->offsets_size,
-		     (unsigned long long)off_end_offset);
-
-	if (buffer->target_node)
-		binder_dec_node(buffer->target_node, 1, 0);
-
-	off_start_offset = ALIGN(buffer->data_size, sizeof(void *));
-
-	for (buffer_offset = off_start_offset; buffer_offset < off_end_offset;
-	     buffer_offset += sizeof(binder_size_t)) {
-		struct binder_object_header *hdr;
-		size_t object_size = 0;
-		struct binder_object object;
-		binder_size_t object_offset;
-
-		if (!binder_alloc_copy_from_buffer(&proc->alloc, &object_offset,
-						   buffer, buffer_offset,
-						   sizeof(object_offset)))
-			object_size = binder_get_object(proc, NULL, buffer,
-							object_offset, &object);
-		if (object_size == 0) {
-			pr_err("transaction release %d bad object at offset %lld, size %zd\n",
-			       debug_id, (u64)object_offset, buffer->data_size);
-			continue;
-		}
-		hdr = &object.hdr;
-		switch (hdr->type) {
-		case BINDER_TYPE_BINDER:
-		case BINDER_TYPE_WEAK_BINDER: {
-			struct flat_binder_object *fp;
-			struct binder_node *node;
-
-			fp = to_flat_binder_object(hdr);
-			node = binder_get_node(proc, fp->binder);
-			if (node == NULL) {
-				pr_err("transaction release %d bad node %016llx\n",
-				       debug_id, (u64)fp->binder);
-				break;
-			}
-			binder_debug(BINDER_DEBUG_TRANSACTION,
-				     "        node %d u%016llx\n",
-				     node->debug_id, (u64)node->ptr);
-			binder_dec_node(node, hdr->type == BINDER_TYPE_BINDER,
-					0);
-			binder_put_node(node);
-		} break;
-		case BINDER_TYPE_HANDLE:
-		case BINDER_TYPE_WEAK_HANDLE: {
-			struct flat_binder_object *fp;
-			struct binder_ref_data rdata;
-			int ret;
-
-			fp = to_flat_binder_object(hdr);
-			ret = binder_dec_ref_for_handle(proc, fp->handle,
-				hdr->type == BINDER_TYPE_HANDLE, &rdata);
-
-			if (ret) {
-				pr_err("transaction release %d bad handle %d, ret = %d\n",
-				 debug_id, fp->handle, ret);
-				break;
-			}
-			binder_debug(BINDER_DEBUG_TRANSACTION,
-				     "        ref %d desc %d\n",
-				     rdata.debug_id, rdata.desc);
-		} break;
-
-		case BINDER_TYPE_FD: {
-			/*
-			 * No need to close the file here since user-space
-			 * closes it for successfully delivered
-			 * transactions. For transactions that weren't
-			 * delivered, the new fd was never allocated so
-			 * there is no need to close and the fput on the
-			 * file is done when the transaction is torn
-			 * down.
-			 */
-		} break;
-		case BINDER_TYPE_PTR:
-			/*
-			 * Nothing to do here, this will get cleaned up when the
-			 * transaction buffer gets freed
-			 */
-			break;
-		case BINDER_TYPE_FDA: {
-			struct binder_fd_array_object *fda;
-			struct binder_buffer_object *parent;
-			struct binder_object ptr_object;
-			binder_size_t fda_offset;
-			size_t fd_index;
-			binder_size_t fd_buf_size;
-			binder_size_t num_valid;
-
-			if (is_failure) {
-				/*
-				 * The fd fixups have not been applied so no
-				 * fds need to be closed.
-				 */
-				continue;
-			}
-
-			num_valid = (buffer_offset - off_start_offset) /
-						sizeof(binder_size_t);
-			fda = to_binder_fd_array_object(hdr);
-			parent = binder_validate_ptr(proc, buffer, &ptr_object,
-						     fda->parent,
-						     off_start_offset,
-						     NULL,
-						     num_valid);
-			if (!parent) {
-				pr_err("transaction release %d bad parent offset\n",
-				       debug_id);
-				continue;
-			}
-			fd_buf_size = sizeof(u32) * fda->num_fds;
-			if (fda->num_fds >= SIZE_MAX / sizeof(u32)) {
-				pr_err("transaction release %d invalid number of fds (%lld)\n",
-				       debug_id, (u64)fda->num_fds);
-				continue;
-			}
-			if (fd_buf_size > parent->length ||
-			    fda->parent_offset > parent->length - fd_buf_size) {
-				/* No space for all file descriptors here. */
-				pr_err("transaction release %d not enough space for %lld fds in buffer\n",
-				       debug_id, (u64)fda->num_fds);
-				continue;
-			}
-			/*
-			 * the source data for binder_buffer_object is visible
-			 * to user-space and the @buffer element is the user
-			 * pointer to the buffer_object containing the fd_array.
-			 * Convert the address to an offset relative to
-			 * the base of the transaction buffer.
-			 */
-			fda_offset = parent->buffer - buffer->user_data +
-				fda->parent_offset;
-			for (fd_index = 0; fd_index < fda->num_fds;
-			     fd_index++) {
-				u32 fd;
-				int err;
-				binder_size_t offset = fda_offset +
-					fd_index * sizeof(fd);
-
-				err = binder_alloc_copy_from_buffer(
-						&proc->alloc, &fd, buffer,
-						offset, sizeof(fd));
-				WARN_ON(err);
-				if (!err) {
-					binder_deferred_fd_close(fd);
-					/*
-					 * Need to make sure the thread goes
-					 * back to userspace to complete the
-					 * deferred close
-					 */
-					if (thread)
-						thread->looper_need_return = true;
-				}
-			}
-		} break;
-		default:
-			pr_err("transaction release %d bad object type %x\n",
-				debug_id, hdr->type);
-			break;
-		}
-	}
-}
-
-/* Clean up all the objects in the buffer */
-static inline void binder_release_entire_buffer(struct binder_proc *proc,
-						struct binder_thread *thread,
-						struct binder_buffer *buffer,
-						bool is_failure)
-{
-	binder_size_t off_end_offset;
-
-	off_end_offset = ALIGN(buffer->data_size, sizeof(void *));
-	off_end_offset += buffer->offsets_size;
-
-	binder_transaction_buffer_release(proc, thread, buffer,
-					  off_end_offset, is_failure);
-}
-
-static int binder_translate_binder(struct flat_binder_object *fp,
-				   struct binder_transaction *t,
-				   struct binder_thread *thread)
-{
-	struct binder_node *node;
-	struct binder_proc *proc = thread->proc;
-	struct binder_proc *target_proc = t->to_proc;
-	struct binder_ref_data rdata;
-	int ret = 0;
-
-	node = binder_get_node(proc, fp->binder);
-	if (!node) {
-		node = binder_new_node(proc, fp);
-		if (!node)
-			return -ENOMEM;
-	}
-	if (fp->cookie != node->cookie) {
-		binder_user_error("%d:%d sending u%016llx node %d, cookie mismatch %016llx != %016llx\n",
-				  proc->pid, thread->pid, (u64)fp->binder,
-				  node->debug_id, (u64)fp->cookie,
-				  (u64)node->cookie);
-		ret = -EINVAL;
-		goto done;
-	}
-	if (security_binder_transfer_binder(proc->cred, target_proc->cred)) {
-		ret = -EPERM;
-		goto done;
-	}
-
-	ret = binder_inc_ref_for_node(target_proc, node,
-			fp->hdr.type == BINDER_TYPE_BINDER,
-			&thread->todo, &rdata);
-	if (ret)
-		goto done;
-
-	if (fp->hdr.type == BINDER_TYPE_BINDER)
-		fp->hdr.type = BINDER_TYPE_HANDLE;
-	else
-		fp->hdr.type = BINDER_TYPE_WEAK_HANDLE;
-	fp->binder = 0;
-	fp->handle = rdata.desc;
-	fp->cookie = 0;
-
-	trace_binder_transaction_node_to_ref(t, node, &rdata);
-	binder_debug(BINDER_DEBUG_TRANSACTION,
-		     "        node %d u%016llx -> ref %d desc %d\n",
-		     node->debug_id, (u64)node->ptr,
-		     rdata.debug_id, rdata.desc);
-done:
-	binder_put_node(node);
-	return ret;
-}
-
-static int binder_translate_handle(struct flat_binder_object *fp,
-				   struct binder_transaction *t,
-				   struct binder_thread *thread)
-{
-	struct binder_proc *proc = thread->proc;
-	struct binder_proc *target_proc = t->to_proc;
-	struct binder_node *node;
-	struct binder_ref_data src_rdata;
-	int ret = 0;
-
-	node = binder_get_node_from_ref(proc, fp->handle,
-			fp->hdr.type == BINDER_TYPE_HANDLE, &src_rdata);
-	if (!node) {
-		binder_user_error("%d:%d got transaction with invalid handle, %d\n",
-				  proc->pid, thread->pid, fp->handle);
-		return -EINVAL;
-	}
-	if (security_binder_transfer_binder(proc->cred, target_proc->cred)) {
-		ret = -EPERM;
-		goto done;
-	}
-
-	binder_node_lock(node);
-	if (node->proc == target_proc) {
-		if (fp->hdr.type == BINDER_TYPE_HANDLE)
-			fp->hdr.type = BINDER_TYPE_BINDER;
-		else
-			fp->hdr.type = BINDER_TYPE_WEAK_BINDER;
-		fp->binder = node->ptr;
-		fp->cookie = node->cookie;
-		if (node->proc)
-			binder_inner_proc_lock(node->proc);
-		else
-			__acquire(&node->proc->inner_lock);
-		binder_inc_node_nilocked(node,
-					 fp->hdr.type == BINDER_TYPE_BINDER,
-					 0, NULL);
-		if (node->proc)
-			binder_inner_proc_unlock(node->proc);
-		else
-			__release(&node->proc->inner_lock);
-		trace_binder_transaction_ref_to_node(t, node, &src_rdata);
-		binder_debug(BINDER_DEBUG_TRANSACTION,
-			     "        ref %d desc %d -> node %d u%016llx\n",
-			     src_rdata.debug_id, src_rdata.desc, node->debug_id,
-			     (u64)node->ptr);
-		binder_node_unlock(node);
-	} else {
-		struct binder_ref_data dest_rdata;
-
-		binder_node_unlock(node);
-		ret = binder_inc_ref_for_node(target_proc, node,
-				fp->hdr.type == BINDER_TYPE_HANDLE,
-				NULL, &dest_rdata);
-		if (ret)
-			goto done;
-
-		fp->binder = 0;
-		fp->handle = dest_rdata.desc;
-		fp->cookie = 0;
-		trace_binder_transaction_ref_to_ref(t, node, &src_rdata,
-						    &dest_rdata);
-		binder_debug(BINDER_DEBUG_TRANSACTION,
-			     "        ref %d desc %d -> ref %d desc %d (node %d)\n",
-			     src_rdata.debug_id, src_rdata.desc,
-			     dest_rdata.debug_id, dest_rdata.desc,
-			     node->debug_id);
-	}
-done:
-	binder_put_node(node);
-	return ret;
-}
-
-static int binder_translate_fd(u32 fd, binder_size_t fd_offset,
-			       struct binder_transaction *t,
-			       struct binder_thread *thread,
-			       struct binder_transaction *in_reply_to)
-{
-	struct binder_proc *proc = thread->proc;
-	struct binder_proc *target_proc = t->to_proc;
-	struct binder_txn_fd_fixup *fixup;
-	struct file *file;
-	int ret = 0;
-	bool target_allows_fd;
-
-	if (in_reply_to)
-		target_allows_fd = !!(in_reply_to->flags & TF_ACCEPT_FDS);
-	else
-		target_allows_fd = t->buffer->target_node->accept_fds;
-	if (!target_allows_fd) {
-		binder_user_error("%d:%d got %s with fd, %d, but target does not allow fds\n",
-				  proc->pid, thread->pid,
-				  in_reply_to ? "reply" : "transaction",
-				  fd);
-		ret = -EPERM;
-		goto err_fd_not_accepted;
-	}
-
-	file = fget(fd);
-	if (!file) {
-		binder_user_error("%d:%d got transaction with invalid fd, %d\n",
-				  proc->pid, thread->pid, fd);
-		ret = -EBADF;
-		goto err_fget;
-	}
-	ret = security_binder_transfer_file(proc->cred, target_proc->cred, file);
-	if (ret < 0) {
-		ret = -EPERM;
-		goto err_security;
-	}
-
-	/*
-	 * Add fixup record for this transaction. The allocation
-	 * of the fd in the target needs to be done from a
-	 * target thread.
-	 */
-	fixup = kzalloc_obj(*fixup);
-	if (!fixup) {
-		ret = -ENOMEM;
-		goto err_alloc;
-	}
-	fixup->file = file;
-	fixup->offset = fd_offset;
-	fixup->target_fd = -1;
-	trace_binder_transaction_fd_send(t, fd, fixup->offset);
-	list_add_tail(&fixup->fixup_entry, &t->fd_fixups);
-
-	return ret;
-
-err_alloc:
-err_security:
-	fput(file);
-err_fget:
-err_fd_not_accepted:
-	return ret;
-}
-
-/**
- * struct binder_ptr_fixup - data to be fixed-up in target buffer
- * @offset:      offset in target buffer to fixup
- * @skip_size:   bytes to skip in copy (fixup will be written later)
- * @fixup_data:  data to write at fixup offset
- * @node:        list node
- *
- * This is used for the pointer fixup list (pf) which is created and consumed
- * during binder_transaction() and is only accessed locally. No
- * locking is necessary.
- *
- * The list is ordered by @offset.
- */
-struct binder_ptr_fixup {
-	binder_size_t offset;
-	size_t skip_size;
-	binder_uintptr_t fixup_data;
-	struct list_head node;
-};
-
-/**
- * struct binder_sg_copy - scatter-gather data to be copied
- * @offset:        offset in target buffer
- * @sender_uaddr:  user address in source buffer
- * @length:        bytes to copy
- * @node:          list node
- *
- * This is used for the sg copy list (sgc) which is created and consumed
- * during binder_transaction() and is only accessed locally. No
- * locking is necessary.
- *
- * The list is ordered by @offset.
- */
-struct binder_sg_copy {
-	binder_size_t offset;
-	const void __user *sender_uaddr;
-	size_t length;
-	struct list_head node;
-};
-
-/**
- * binder_do_deferred_txn_copies() - copy and fixup scatter-gather data
- * @alloc:	binder_alloc associated with @buffer
- * @buffer:	binder buffer in target process
- * @sgc_head:	list_head of scatter-gather copy list
- * @pf_head:	list_head of pointer fixup list
- *
- * Processes all elements of @sgc_head, applying fixups from @pf_head
- * and copying the scatter-gather data from the source process' user
- * buffer to the target's buffer. It is expected that the list creation
- * and processing all occurs during binder_transaction() so these lists
- * are only accessed in local context.
- *
- * Return: 0=success, else -errno
- */
-static int binder_do_deferred_txn_copies(struct binder_alloc *alloc,
-					 struct binder_buffer *buffer,
-					 struct list_head *sgc_head,
-					 struct list_head *pf_head)
-{
-	int ret = 0;
-	struct binder_sg_copy *sgc, *tmpsgc;
-	struct binder_ptr_fixup *tmppf;
-	struct binder_ptr_fixup *pf =
-		list_first_entry_or_null(pf_head, struct binder_ptr_fixup,
-					 node);
-
-	list_for_each_entry_safe(sgc, tmpsgc, sgc_head, node) {
-		size_t bytes_copied = 0;
-
-		while (bytes_copied < sgc->length) {
-			size_t copy_size;
-			size_t bytes_left = sgc->length - bytes_copied;
-			size_t offset = sgc->offset + bytes_copied;
-
-			/*
-			 * We copy up to the fixup (pointed to by pf)
-			 */
-			copy_size = pf ? min(bytes_left, (size_t)pf->offset - offset)
-				       : bytes_left;
-			if (!ret && copy_size)
-				ret = binder_alloc_copy_user_to_buffer(
-						alloc, buffer,
-						offset,
-						sgc->sender_uaddr + bytes_copied,
-						copy_size);
-			bytes_copied += copy_size;
-			if (copy_size != bytes_left) {
-				BUG_ON(!pf);
-				/* we stopped at a fixup offset */
-				if (pf->skip_size) {
-					/*
-					 * we are just skipping. This is for
-					 * BINDER_TYPE_FDA where the translated
-					 * fds will be fixed up when we get
-					 * to target context.
-					 */
-					bytes_copied += pf->skip_size;
-				} else {
-					/* apply the fixup indicated by pf */
-					if (!ret)
-						ret = binder_alloc_copy_to_buffer(
-							alloc, buffer,
-							pf->offset,
-							&pf->fixup_data,
-							sizeof(pf->fixup_data));
-					bytes_copied += sizeof(pf->fixup_data);
-				}
-				list_del(&pf->node);
-				kfree(pf);
-				pf = list_first_entry_or_null(pf_head,
-						struct binder_ptr_fixup, node);
-			}
-		}
-		list_del(&sgc->node);
-		kfree(sgc);
-	}
-	list_for_each_entry_safe(pf, tmppf, pf_head, node) {
-		BUG_ON(pf->skip_size == 0);
-		list_del(&pf->node);
-		kfree(pf);
-	}
-	BUG_ON(!list_empty(sgc_head));
-
-	return ret > 0 ? -EINVAL : ret;
-}
-
-/**
- * binder_cleanup_deferred_txn_lists() - free specified lists
- * @sgc_head:	list_head of scatter-gather copy list
- * @pf_head:	list_head of pointer fixup list
- *
- * Called to clean up @sgc_head and @pf_head if there is an
- * error.
- */
-static void binder_cleanup_deferred_txn_lists(struct list_head *sgc_head,
-					      struct list_head *pf_head)
-{
-	struct binder_sg_copy *sgc, *tmpsgc;
-	struct binder_ptr_fixup *pf, *tmppf;
-
-	list_for_each_entry_safe(sgc, tmpsgc, sgc_head, node) {
-		list_del(&sgc->node);
-		kfree(sgc);
-	}
-	list_for_each_entry_safe(pf, tmppf, pf_head, node) {
-		list_del(&pf->node);
-		kfree(pf);
-	}
-}
-
-/**
- * binder_defer_copy() - queue a scatter-gather buffer for copy
- * @sgc_head:		list_head of scatter-gather copy list
- * @offset:		binder buffer offset in target process
- * @sender_uaddr:	user address in source process
- * @length:		bytes to copy
- *
- * Specify a scatter-gather block to be copied. The actual copy must
- * be deferred until all the needed fixups are identified and queued.
- * Then the copy and fixups are done together so un-translated values
- * from the source are never visible in the target buffer.
- *
- * We are guaranteed that repeated calls to this function will have
- * monotonically increasing @offset values so the list will naturally
- * be ordered.
- *
- * Return: 0=success, else -errno
- */
-static int binder_defer_copy(struct list_head *sgc_head, binder_size_t offset,
-			     const void __user *sender_uaddr, size_t length)
-{
-	struct binder_sg_copy *bc = kzalloc_obj(*bc);
-
-	if (!bc)
-		return -ENOMEM;
-
-	bc->offset = offset;
-	bc->sender_uaddr = sender_uaddr;
-	bc->length = length;
-	INIT_LIST_HEAD(&bc->node);
-
-	/*
-	 * We are guaranteed that the deferred copies are in-order
-	 * so just add to the tail.
-	 */
-	list_add_tail(&bc->node, sgc_head);
-
-	return 0;
-}
-
-/**
- * binder_add_fixup() - queue a fixup to be applied to sg copy
- * @pf_head:	list_head of binder ptr fixup list
- * @offset:	binder buffer offset in target process
- * @fixup:	bytes to be copied for fixup
- * @skip_size:	bytes to skip when copying (fixup will be applied later)
- *
- * Add the specified fixup to a list ordered by @offset. When copying
- * the scatter-gather buffers, the fixup will be copied instead of
- * data from the source buffer. For BINDER_TYPE_FDA fixups, the fixup
- * will be applied later (in target process context), so we just skip
- * the bytes specified by @skip_size. If @skip_size is 0, we copy the
- * value in @fixup.
- *
- * This function is called *mostly* in @offset order, but there are
- * exceptions. Since out-of-order inserts are relatively uncommon,
- * we insert the new element by searching backward from the tail of
- * the list.
- *
- * Return: 0=success, else -errno
- */
-static int binder_add_fixup(struct list_head *pf_head, binder_size_t offset,
-			    binder_uintptr_t fixup, size_t skip_size)
-{
-	struct binder_ptr_fixup *pf = kzalloc_obj(*pf);
-	struct binder_ptr_fixup *tmppf;
-
-	if (!pf)
-		return -ENOMEM;
-
-	pf->offset = offset;
-	pf->fixup_data = fixup;
-	pf->skip_size = skip_size;
-	INIT_LIST_HEAD(&pf->node);
-
-	/* Fixups are *mostly* added in-order, but there are some
-	 * exceptions. Look backwards through list for insertion point.
-	 */
-	list_for_each_entry_reverse(tmppf, pf_head, node) {
-		if (tmppf->offset < pf->offset) {
-			list_add(&pf->node, &tmppf->node);
-			return 0;
-		}
-	}
-	/*
-	 * if we get here, then the new offset is the lowest so
-	 * insert at the head
-	 */
-	list_add(&pf->node, pf_head);
-	return 0;
-}
-
-static int binder_translate_fd_array(struct list_head *pf_head,
-				     struct binder_fd_array_object *fda,
-				     const void __user *sender_ubuffer,
-				     struct binder_buffer_object *parent,
-				     struct binder_buffer_object *sender_uparent,
-				     struct binder_transaction *t,
-				     struct binder_thread *thread,
-				     struct binder_transaction *in_reply_to)
-{
-	binder_size_t fdi, fd_buf_size;
-	binder_size_t fda_offset;
-	const void __user *sender_ufda_base;
-	struct binder_proc *proc = thread->proc;
-	int ret;
-
-	if (fda->num_fds == 0)
-		return 0;
-
-	fd_buf_size = sizeof(u32) * fda->num_fds;
-	if (fda->num_fds >= SIZE_MAX / sizeof(u32)) {
-		binder_user_error("%d:%d got transaction with invalid number of fds (%lld)\n",
-				  proc->pid, thread->pid, (u64)fda->num_fds);
-		return -EINVAL;
-	}
-	if (fd_buf_size > parent->length ||
-	    fda->parent_offset > parent->length - fd_buf_size) {
-		/* No space for all file descriptors here. */
-		binder_user_error("%d:%d not enough space to store %lld fds in buffer\n",
-				  proc->pid, thread->pid, (u64)fda->num_fds);
-		return -EINVAL;
-	}
-	/*
-	 * the source data for binder_buffer_object is visible
-	 * to user-space and the @buffer element is the user
-	 * pointer to the buffer_object containing the fd_array.
-	 * Convert the address to an offset relative to
-	 * the base of the transaction buffer.
-	 */
-	fda_offset = parent->buffer - t->buffer->user_data +
-		fda->parent_offset;
-	sender_ufda_base = (void __user *)(uintptr_t)sender_uparent->buffer +
-				fda->parent_offset;
-
-	if (!IS_ALIGNED((unsigned long)fda_offset, sizeof(u32)) ||
-	    !IS_ALIGNED((unsigned long)sender_ufda_base, sizeof(u32))) {
-		binder_user_error("%d:%d parent offset not aligned correctly.\n",
-				  proc->pid, thread->pid);
-		return -EINVAL;
-	}
-	ret = binder_add_fixup(pf_head, fda_offset, 0, fda->num_fds * sizeof(u32));
-	if (ret)
-		return ret;
-
-	for (fdi = 0; fdi < fda->num_fds; fdi++) {
-		u32 fd;
-		binder_size_t offset = fda_offset + fdi * sizeof(fd);
-		binder_size_t sender_uoffset = fdi * sizeof(fd);
-
-		ret = copy_from_user(&fd, sender_ufda_base + sender_uoffset, sizeof(fd));
-		if (!ret)
-			ret = binder_translate_fd(fd, offset, t, thread,
-						  in_reply_to);
-		if (ret)
-			return ret > 0 ? -EINVAL : ret;
-	}
-	return 0;
-}
-
-static int binder_fixup_parent(struct list_head *pf_head,
-			       struct binder_transaction *t,
-			       struct binder_thread *thread,
-			       struct binder_buffer_object *bp,
-			       binder_size_t off_start_offset,
-			       binder_size_t num_valid,
-			       binder_size_t last_fixup_obj_off,
-			       binder_size_t last_fixup_min_off)
-{
-	struct binder_buffer_object *parent;
-	struct binder_buffer *b = t->buffer;
-	struct binder_proc *proc = thread->proc;
-	struct binder_proc *target_proc = t->to_proc;
-	struct binder_object object;
-	binder_size_t buffer_offset;
-	binder_size_t parent_offset;
-
-	if (!(bp->flags & BINDER_BUFFER_FLAG_HAS_PARENT))
-		return 0;
-
-	parent = binder_validate_ptr(target_proc, b, &object, bp->parent,
-				     off_start_offset, &parent_offset,
-				     num_valid);
-	if (!parent) {
-		binder_user_error("%d:%d got transaction with invalid parent offset or type\n",
-				  proc->pid, thread->pid);
-		return -EINVAL;
-	}
-
-	if (!binder_validate_fixup(target_proc, b, off_start_offset,
-				   parent_offset, bp->parent_offset,
-				   last_fixup_obj_off,
-				   last_fixup_min_off)) {
-		binder_user_error("%d:%d got transaction with out-of-order buffer fixup\n",
-				  proc->pid, thread->pid);
-		return -EINVAL;
-	}
-
-	if (parent->length < sizeof(binder_uintptr_t) ||
-	    bp->parent_offset > parent->length - sizeof(binder_uintptr_t)) {
-		/* No space for a pointer here! */
-		binder_user_error("%d:%d got transaction with invalid parent offset\n",
-				  proc->pid, thread->pid);
-		return -EINVAL;
-	}
-
-	buffer_offset = bp->parent_offset + parent->buffer - b->user_data;
-
-	return binder_add_fixup(pf_head, buffer_offset, bp->buffer, 0);
-}
-
-/**
- * binder_can_update_transaction() - Can a txn be superseded by an updated one?
- * @t1: the pending async txn in the frozen process
- * @t2: the new async txn to supersede the outdated pending one
- *
- * Return:  true if t2 can supersede t1
- *          false if t2 can not supersede t1
- */
-static bool binder_can_update_transaction(struct binder_transaction *t1,
-					  struct binder_transaction *t2)
-{
-	if ((t1->flags & t2->flags & (TF_ONE_WAY | TF_UPDATE_TXN)) !=
-	    (TF_ONE_WAY | TF_UPDATE_TXN) || !t1->to_proc || !t2->to_proc)
-		return false;
-	if (t1->to_proc->tsk == t2->to_proc->tsk && t1->code == t2->code &&
-	    t1->flags == t2->flags && t1->buffer->pid == t2->buffer->pid &&
-	    t1->buffer->target_node->ptr == t2->buffer->target_node->ptr &&
-	    t1->buffer->target_node->cookie == t2->buffer->target_node->cookie)
-		return true;
-	return false;
-}
-
-/**
- * binder_find_outdated_transaction_ilocked() - Find the outdated transaction
- * @t:		 new async transaction
- * @target_list: list to find outdated transaction
- *
- * Return: the outdated transaction if found
- *         NULL if no outdated transacton can be found
- *
- * Requires the proc->inner_lock to be held.
- */
-static struct binder_transaction *
-binder_find_outdated_transaction_ilocked(struct binder_transaction *t,
-					 struct list_head *target_list)
-{
-	struct binder_work *w;
-
-	list_for_each_entry(w, target_list, entry) {
-		struct binder_transaction *t_queued;
-
-		if (w->type != BINDER_WORK_TRANSACTION)
-			continue;
-		t_queued = container_of(w, struct binder_transaction, work);
-		if (binder_can_update_transaction(t_queued, t))
-			return t_queued;
-	}
-	return NULL;
-}
-
-/**
- * binder_proc_transaction() - sends a transaction to a process and wakes it up
- * @t:		transaction to send
- * @proc:	process to send the transaction to
- * @thread:	thread in @proc to send the transaction to (may be NULL)
- *
- * This function queues a transaction to the specified process. It will try
- * to find a thread in the target process to handle the transaction and
- * wake it up. If no thread is found, the work is queued to the proc
- * waitqueue.
- *
- * If the @thread parameter is not NULL, the transaction is always queued
- * to the waitlist of that specific thread.
- *
- * Return:	0 if the transaction was successfully queued
- *		BR_DEAD_REPLY if the target process or thread is dead
- *		BR_FROZEN_REPLY if the target process or thread is frozen and
- *			the sync transaction was rejected
- *		BR_TRANSACTION_PENDING_FROZEN if the target process is frozen
- *		and the async transaction was successfully queued
- */
-static int binder_proc_transaction(struct binder_transaction *t,
-				    struct binder_proc *proc,
-				    struct binder_thread *thread)
-{
-	struct binder_node *node = t->buffer->target_node;
-	bool oneway = !!(t->flags & TF_ONE_WAY);
-	bool pending_async = false;
-	struct binder_transaction *t_outdated = NULL;
-	bool frozen = false;
-
-	BUG_ON(!node);
-	binder_node_lock(node);
-	if (oneway) {
-		BUG_ON(thread);
-		if (node->has_async_transaction)
-			pending_async = true;
-		else
-			node->has_async_transaction = true;
-	}
-
-	binder_inner_proc_lock(proc);
-	if (proc->is_frozen) {
-		frozen = true;
-		proc->sync_recv |= !oneway;
-		proc->async_recv |= oneway;
-	}
-
-	if ((frozen && !oneway) || proc->is_dead ||
-			(thread && thread->is_dead)) {
-		binder_inner_proc_unlock(proc);
-		binder_node_unlock(node);
-		return frozen ? BR_FROZEN_REPLY : BR_DEAD_REPLY;
-	}
-
-	if (!thread && !pending_async)
-		thread = binder_select_thread_ilocked(proc);
-
-	if (thread) {
-		binder_enqueue_thread_work_ilocked(thread, &t->work);
-	} else if (!pending_async) {
-		binder_enqueue_work_ilocked(&t->work, &proc->todo);
-	} else {
-		if ((t->flags & TF_UPDATE_TXN) && frozen) {
-			t_outdated = binder_find_outdated_transaction_ilocked(t,
-									      &node->async_todo);
-			if (t_outdated) {
-				binder_debug(BINDER_DEBUG_TRANSACTION,
-					     "txn %d supersedes %d\n",
-					     t->debug_id, t_outdated->debug_id);
-				list_del_init(&t_outdated->work.entry);
-				proc->outstanding_txns--;
-			}
-		}
-		binder_enqueue_work_ilocked(&t->work, &node->async_todo);
-	}
-
-	if (!pending_async)
-		binder_wakeup_thread_ilocked(proc, thread, !oneway /* sync */);
-
-	proc->outstanding_txns++;
-	binder_inner_proc_unlock(proc);
-	binder_node_unlock(node);
-
-	/*
-	 * To reduce potential contention, free the outdated transaction and
-	 * buffer after releasing the locks.
-	 */
-	if (t_outdated) {
-		struct binder_buffer *buffer = t_outdated->buffer;
-
-		t_outdated->buffer = NULL;
-		buffer->transaction = NULL;
-		trace_binder_transaction_update_buffer_release(buffer);
-		binder_release_entire_buffer(proc, NULL, buffer, false);
-		binder_alloc_free_buf(&proc->alloc, buffer);
-		kfree(t_outdated);
-		binder_stats_deleted(BINDER_STAT_TRANSACTION);
-	}
-
-	if (oneway && frozen)
-		return BR_TRANSACTION_PENDING_FROZEN;
-
-	return 0;
-}
-
-/**
- * binder_get_node_refs_for_txn() - Get required refs on node for txn
- * @node:         struct binder_node for which to get refs
- * @procp:        returns @node->proc if valid
- * @error:        if no @procp then returns BR_DEAD_REPLY
- *
- * User-space normally keeps the node alive when creating a transaction
- * since it has a reference to the target. The local strong ref keeps it
- * alive if the sending process dies before the target process processes
- * the transaction. If the source process is malicious or has a reference
- * counting bug, relying on the local strong ref can fail.
- *
- * Since user-space can cause the local strong ref to go away, we also take
- * a tmpref on the node to ensure it survives while we are constructing
- * the transaction. We also need a tmpref on the proc while we are
- * constructing the transaction, so we take that here as well.
- *
- * Return: The target_node with refs taken or NULL if no @node->proc is NULL.
- * Also sets @procp if valid. If the @node->proc is NULL indicating that the
- * target proc has died, @error is set to BR_DEAD_REPLY.
- */
-static struct binder_node *binder_get_node_refs_for_txn(
-		struct binder_node *node,
-		struct binder_proc **procp,
-		uint32_t *error)
-{
-	struct binder_node *target_node = NULL;
-
-	binder_node_inner_lock(node);
-	if (node->proc) {
-		target_node = node;
-		binder_inc_node_nilocked(node, 1, 0, NULL);
-		binder_inc_node_tmpref_ilocked(node);
-		node->proc->tmp_ref++;
-		*procp = node->proc;
-	} else
-		*error = BR_DEAD_REPLY;
-	binder_node_inner_unlock(node);
-
-	return target_node;
-}
-
-static void binder_set_txn_from_error(struct binder_transaction *t, int id,
-				      uint32_t command, int32_t param)
-{
-	struct binder_thread *from = binder_get_txn_from_and_acq_inner(t);
-
-	if (!from) {
-		/* annotation for sparse */
-		__release(&from->proc->inner_lock);
-		return;
-	}
-
-	/* don't override existing errors */
-	if (from->ee.command == BR_OK)
-		binder_set_extended_error(&from->ee, id, command, param);
-	binder_inner_proc_unlock(from->proc);
-	binder_thread_dec_tmpref(from);
-}
-
-/**
- * binder_netlink_report() - report a transaction failure via netlink
- * @proc:	the binder proc sending the transaction
- * @t:		the binder transaction that failed
- * @data_size:	the user provided data size for the transaction
- * @error:	enum binder_driver_return_protocol returned to sender
- *
- * Note that t->buffer is not safe to access here, as it may have been
- * released (or not yet allocated). Callers should guarantee all the
- * transaction items used here are safe to access.
- */
-static void binder_netlink_report(struct binder_proc *proc,
-				  struct binder_transaction *t,
-				  u32 data_size,
-				  u32 error)
-{
-	const char *context = proc->context->name;
-	struct sk_buff *skb;
-	void *hdr;
-
-	if (!genl_has_listeners(&binder_nl_family, &init_net,
-				BINDER_NLGRP_REPORT))
-		return;
-
-	trace_binder_netlink_report(context, t, data_size, error);
-
-	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
-	if (!skb)
-		return;
-
-	hdr = genlmsg_put(skb, 0, 0, &binder_nl_family, 0, BINDER_CMD_REPORT);
-	if (!hdr)
-		goto free_skb;
-
-	if (nla_put_u32(skb, BINDER_A_REPORT_ERROR, error) ||
-	    nla_put_string(skb, BINDER_A_REPORT_CONTEXT, context) ||
-	    nla_put_u32(skb, BINDER_A_REPORT_FROM_PID, t->from_pid) ||
-	    nla_put_u32(skb, BINDER_A_REPORT_FROM_TID, t->from_tid))
-		goto cancel_skb;
-
-	if (t->to_proc &&
-	    nla_put_u32(skb, BINDER_A_REPORT_TO_PID, t->to_proc->pid))
-		goto cancel_skb;
-
-	if (t->to_thread &&
-	    nla_put_u32(skb, BINDER_A_REPORT_TO_TID, t->to_thread->pid))
-		goto cancel_skb;
-
-	if (t->is_reply && nla_put_flag(skb, BINDER_A_REPORT_IS_REPLY))
-		goto cancel_skb;
-
-	if (nla_put_u32(skb, BINDER_A_REPORT_FLAGS, t->flags) ||
-	    nla_put_u32(skb, BINDER_A_REPORT_CODE, t->code) ||
-	    nla_put_u32(skb, BINDER_A_REPORT_DATA_SIZE, data_size))
-		goto cancel_skb;
-
-	genlmsg_end(skb, hdr);
-	genlmsg_multicast(&binder_nl_family, skb, 0, BINDER_NLGRP_REPORT,
-			  GFP_KERNEL);
-	return;
-
-cancel_skb:
-	genlmsg_cancel(skb, hdr);
-free_skb:
-	nlmsg_free(skb);
-}
-
-static void binder_transaction(struct binder_proc *proc,
-			       struct binder_thread *thread,
-			       struct binder_transaction_data *tr, int reply,
-			       binder_size_t extra_buffers_size)
-{
-	int ret;
-	struct binder_transaction *t;
-	struct binder_work *w;
-	struct binder_work *tcomplete;
-	binder_size_t buffer_offset = 0;
-	binder_size_t off_start_offset, off_end_offset;
-	binder_size_t off_min;
-	binder_size_t sg_buf_offset, sg_buf_end_offset;
-	binder_size_t user_offset = 0;
-	struct binder_proc *target_proc = NULL;
-	struct binder_thread *target_thread = NULL;
-	struct binder_node *target_node = NULL;
-	struct binder_transaction *in_reply_to = NULL;
-	struct binder_transaction_log_entry *e;
-	uint32_t return_error = 0;
-	uint32_t return_error_param = 0;
-	uint32_t return_error_line = 0;
-	binder_size_t last_fixup_obj_off = 0;
-	binder_size_t last_fixup_min_off = 0;
-	struct binder_context *context = proc->context;
-	int t_debug_id = atomic_inc_return(&binder_last_id);
-	ktime_t t_start_time = ktime_get();
-	struct lsm_context lsmctx = { };
-	size_t lsmctx_aligned_size = 0;
-	LIST_HEAD(sgc_head);
-	LIST_HEAD(pf_head);
-	const void __user *user_buffer = (const void __user *)
-				(uintptr_t)tr->data.ptr.buffer;
-
-	e = binder_transaction_log_add(&binder_transaction_log);
-	e->debug_id = t_debug_id;
-	e->call_type = reply ? 2 : !!(tr->flags & TF_ONE_WAY);
-	e->from_proc = proc->pid;
-	e->from_thread = thread->pid;
-	e->target_handle = tr->target.handle;
-	e->data_size = tr->data_size;
-	e->offsets_size = tr->offsets_size;
-	strscpy(e->context_name, proc->context->name, BINDERFS_MAX_NAME);
-
-	binder_inner_proc_lock(proc);
-	binder_set_extended_error(&thread->ee, t_debug_id, BR_OK, 0);
-	binder_inner_proc_unlock(proc);
-
-	t = kzalloc_obj(*t);
-	if (!t) {
-		binder_txn_error("%d:%d cannot allocate transaction\n",
-				 thread->pid, proc->pid);
-		return_error = BR_FAILED_REPLY;
-		return_error_param = -ENOMEM;
-		return_error_line = __LINE__;
-		goto err_alloc_t_failed;
-	}
-	INIT_LIST_HEAD(&t->fd_fixups);
-	binder_stats_created(BINDER_STAT_TRANSACTION);
-	spin_lock_init(&t->lock);
-	t->debug_id = t_debug_id;
-	t->start_time = t_start_time;
-	t->from_pid = proc->pid;
-	t->from_tid = thread->pid;
-	t->sender_euid = current_euid();
-	t->code = tr->code;
-	t->flags = tr->flags;
-	t->priority = task_nice(current);
-	t->work.type = BINDER_WORK_TRANSACTION;
-	t->is_async = !reply && (tr->flags & TF_ONE_WAY);
-	t->is_reply = reply;
-	if (!reply && !(tr->flags & TF_ONE_WAY))
-		t->from = thread;
-
-	if (reply) {
-		binder_inner_proc_lock(proc);
-		in_reply_to = thread->transaction_stack;
-		if (in_reply_to == NULL) {
-			binder_inner_proc_unlock(proc);
-			binder_user_error("%d:%d got reply transaction with no transaction stack\n",
-					  proc->pid, thread->pid);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EPROTO;
-			return_error_line = __LINE__;
-			goto err_empty_call_stack;
-		}
-		if (in_reply_to->to_thread != thread) {
-			spin_lock(&in_reply_to->lock);
-			binder_user_error("%d:%d got reply transaction with bad transaction stack, transaction %d has target %d:%d\n",
-				proc->pid, thread->pid, in_reply_to->debug_id,
-				in_reply_to->to_proc ?
-				in_reply_to->to_proc->pid : 0,
-				in_reply_to->to_thread ?
-				in_reply_to->to_thread->pid : 0);
-			spin_unlock(&in_reply_to->lock);
-			binder_inner_proc_unlock(proc);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EPROTO;
-			return_error_line = __LINE__;
-			in_reply_to = NULL;
-			goto err_bad_call_stack;
-		}
-		thread->transaction_stack = in_reply_to->to_parent;
-		binder_inner_proc_unlock(proc);
-		binder_set_nice(in_reply_to->saved_priority);
-		target_thread = binder_get_txn_from_and_acq_inner(in_reply_to);
-		if (target_thread == NULL) {
-			/* annotation for sparse */
-			__release(&target_thread->proc->inner_lock);
-			binder_txn_error("%d:%d reply target not found\n",
-				thread->pid, proc->pid);
-			return_error = BR_DEAD_REPLY;
-			return_error_line = __LINE__;
-			goto err_dead_binder;
-		}
-		if (target_thread->transaction_stack != in_reply_to) {
-			binder_user_error("%d:%d got reply transaction with bad target transaction stack %d, expected %d\n",
-				proc->pid, thread->pid,
-				target_thread->transaction_stack ?
-				target_thread->transaction_stack->debug_id : 0,
-				in_reply_to->debug_id);
-			binder_inner_proc_unlock(target_thread->proc);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EPROTO;
-			return_error_line = __LINE__;
-			in_reply_to = NULL;
-			target_thread = NULL;
-			goto err_dead_binder;
-		}
-		target_proc = target_thread->proc;
-		target_proc->tmp_ref++;
-		binder_inner_proc_unlock(target_thread->proc);
-	} else {
-		if (tr->target.handle) {
-			struct binder_ref *ref;
-
-			/*
-			 * There must already be a strong ref
-			 * on this node. If so, do a strong
-			 * increment on the node to ensure it
-			 * stays alive until the transaction is
-			 * done.
-			 */
-			binder_proc_lock(proc);
-			ref = binder_get_ref_olocked(proc, tr->target.handle,
-						     true);
-			if (ref) {
-				target_node = binder_get_node_refs_for_txn(
-						ref->node, &target_proc,
-						&return_error);
-			} else {
-				binder_user_error("%d:%d got transaction to invalid handle, %u\n",
-						  proc->pid, thread->pid, tr->target.handle);
-				return_error = BR_FAILED_REPLY;
-			}
-			binder_proc_unlock(proc);
-		} else {
-			mutex_lock(&context->context_mgr_node_lock);
-			target_node = context->binder_context_mgr_node;
-			if (target_node)
-				target_node = binder_get_node_refs_for_txn(
-						target_node, &target_proc,
-						&return_error);
-			else
-				return_error = BR_DEAD_REPLY;
-			mutex_unlock(&context->context_mgr_node_lock);
-			if (target_node && target_proc->pid == proc->pid) {
-				binder_user_error("%d:%d got transaction to context manager from process owning it\n",
-						  proc->pid, thread->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = -EINVAL;
-				return_error_line = __LINE__;
-				goto err_invalid_target_handle;
-			}
-		}
-		if (!target_node) {
-			binder_txn_error("%d:%d cannot find target node\n",
-					 proc->pid, thread->pid);
-			/* return_error is set above */
-			return_error_param = -EINVAL;
-			return_error_line = __LINE__;
-			goto err_dead_binder;
-		}
-		e->to_node = target_node->debug_id;
-		if (WARN_ON(proc == target_proc)) {
-			binder_txn_error("%d:%d self transactions not allowed\n",
-				thread->pid, proc->pid);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EINVAL;
-			return_error_line = __LINE__;
-			goto err_invalid_target_handle;
-		}
-		if (security_binder_transaction(proc->cred,
-						target_proc->cred) < 0) {
-			binder_txn_error("%d:%d transaction credentials failed\n",
-				thread->pid, proc->pid);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EPERM;
-			return_error_line = __LINE__;
-			goto err_invalid_target_handle;
-		}
-		binder_inner_proc_lock(proc);
-
-		w = list_first_entry_or_null(&thread->todo,
-					     struct binder_work, entry);
-		if (!(tr->flags & TF_ONE_WAY) && w &&
-		    w->type == BINDER_WORK_TRANSACTION) {
-			/*
-			 * Do not allow new outgoing transaction from a
-			 * thread that has a transaction at the head of
-			 * its todo list. Only need to check the head
-			 * because binder_select_thread_ilocked picks a
-			 * thread from proc->waiting_threads to enqueue
-			 * the transaction, and nothing is queued to the
-			 * todo list while the thread is on waiting_threads.
-			 */
-			binder_user_error("%d:%d new transaction not allowed when there is a transaction on thread todo\n",
-					  proc->pid, thread->pid);
-			binder_inner_proc_unlock(proc);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EPROTO;
-			return_error_line = __LINE__;
-			goto err_bad_todo_list;
-		}
-
-		if (!(tr->flags & TF_ONE_WAY) && thread->transaction_stack) {
-			struct binder_transaction *tmp;
-
-			tmp = thread->transaction_stack;
-			if (tmp->to_thread != thread) {
-				spin_lock(&tmp->lock);
-				binder_user_error("%d:%d got new transaction with bad transaction stack, transaction %d has target %d:%d\n",
-					proc->pid, thread->pid, tmp->debug_id,
-					tmp->to_proc ? tmp->to_proc->pid : 0,
-					tmp->to_thread ?
-					tmp->to_thread->pid : 0);
-				spin_unlock(&tmp->lock);
-				binder_inner_proc_unlock(proc);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = -EPROTO;
-				return_error_line = __LINE__;
-				goto err_bad_call_stack;
-			}
-			while (tmp) {
-				struct binder_thread *from;
-
-				spin_lock(&tmp->lock);
-				from = tmp->from;
-				if (from && from->proc == target_proc) {
-					atomic_inc(&from->tmp_ref);
-					target_thread = from;
-					spin_unlock(&tmp->lock);
-					break;
-				}
-				spin_unlock(&tmp->lock);
-				tmp = tmp->from_parent;
-			}
-		}
-		binder_inner_proc_unlock(proc);
-	}
-
-	t->to_proc = target_proc;
-	t->to_thread = target_thread;
-	if (target_thread)
-		e->to_thread = target_thread->pid;
-	e->to_proc = target_proc->pid;
-
-	tcomplete = kzalloc_obj(*tcomplete);
-	if (tcomplete == NULL) {
-		binder_txn_error("%d:%d cannot allocate work for transaction\n",
-			thread->pid, proc->pid);
-		return_error = BR_FAILED_REPLY;
-		return_error_param = -ENOMEM;
-		return_error_line = __LINE__;
-		goto err_alloc_tcomplete_failed;
-	}
-	binder_stats_created(BINDER_STAT_TRANSACTION_COMPLETE);
-
-	if (reply)
-		binder_debug(BINDER_DEBUG_TRANSACTION,
-			     "%d:%d BC_REPLY %d -> %d:%d, data size %lld-%lld-%lld\n",
-			     proc->pid, thread->pid, t->debug_id,
-			     target_proc->pid, target_thread->pid,
-			     (u64)tr->data_size, (u64)tr->offsets_size,
-			     (u64)extra_buffers_size);
-	else
-		binder_debug(BINDER_DEBUG_TRANSACTION,
-			     "%d:%d BC_TRANSACTION %d -> %d - node %d, data size %lld-%lld-%lld\n",
-			     proc->pid, thread->pid, t->debug_id,
-			     target_proc->pid, target_node->debug_id,
-			     (u64)tr->data_size, (u64)tr->offsets_size,
-			     (u64)extra_buffers_size);
-
-	if (target_node && target_node->txn_security_ctx) {
-		u32 secid;
-
-		security_cred_getsecid(proc->cred, &secid);
-		ret = security_secid_to_secctx(secid, &lsmctx);
-		if (ret < 0) {
-			binder_txn_error("%d:%d failed to get security context\n",
-				thread->pid, proc->pid);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = ret;
-			return_error_line = __LINE__;
-			goto err_get_secctx_failed;
-		}
-		lsmctx_aligned_size = ALIGN(lsmctx.len, sizeof(u64));
-		extra_buffers_size += lsmctx_aligned_size;
-		if (extra_buffers_size < lsmctx_aligned_size) {
-			binder_txn_error("%d:%d integer overflow of extra_buffers_size\n",
-				thread->pid, proc->pid);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EINVAL;
-			return_error_line = __LINE__;
-			goto err_bad_extra_size;
-		}
-	}
-
-	trace_binder_transaction(reply, t, target_node);
-
-	t->buffer = binder_alloc_new_buf(&target_proc->alloc, tr->data_size,
-		tr->offsets_size, extra_buffers_size,
-		!reply && (t->flags & TF_ONE_WAY));
-	if (IS_ERR(t->buffer)) {
-		char *s;
-
-		ret = PTR_ERR(t->buffer);
-		s = (ret == -ESRCH) ? ": vma cleared, target dead or dying"
-			: (ret == -ENOSPC) ? ": no space left"
-			: (ret == -ENOMEM) ? ": memory allocation failed"
-			: "";
-		binder_txn_error("cannot allocate buffer%s", s);
-
-		return_error_param = PTR_ERR(t->buffer);
-		return_error = return_error_param == -ESRCH ?
-			BR_DEAD_REPLY : BR_FAILED_REPLY;
-		return_error_line = __LINE__;
-		t->buffer = NULL;
-		goto err_binder_alloc_buf_failed;
-	}
-	if (lsmctx.context) {
-		int err;
-		size_t buf_offset = ALIGN(tr->data_size, sizeof(void *)) +
-				    ALIGN(tr->offsets_size, sizeof(void *)) +
-				    ALIGN(extra_buffers_size, sizeof(void *)) -
-				    lsmctx_aligned_size;
-
-		t->security_ctx = t->buffer->user_data + buf_offset;
-		err = binder_alloc_copy_to_buffer(&target_proc->alloc,
-						  t->buffer, buf_offset,
-						  lsmctx.context, lsmctx.len);
-		if (err) {
-			t->security_ctx = 0;
-			WARN_ON(1);
-		}
-		security_release_secctx(&lsmctx);
-		lsmctx.context = NULL;
-	}
-	t->buffer->debug_id = t->debug_id;
-	t->buffer->transaction = t;
-	t->buffer->target_node = target_node;
-	t->buffer->clear_on_free = !!(t->flags & TF_CLEAR_BUF);
-	trace_binder_transaction_alloc_buf(t->buffer);
-
-	if (binder_alloc_copy_user_to_buffer(
-				&target_proc->alloc,
-				t->buffer,
-				ALIGN(tr->data_size, sizeof(void *)),
-				(const void __user *)
-					(uintptr_t)tr->data.ptr.offsets,
-				tr->offsets_size)) {
-		binder_user_error("%d:%d got transaction with invalid offsets ptr\n",
-				proc->pid, thread->pid);
-		return_error = BR_FAILED_REPLY;
-		return_error_param = -EFAULT;
-		return_error_line = __LINE__;
-		goto err_copy_data_failed;
-	}
-	if (!IS_ALIGNED(tr->offsets_size, sizeof(binder_size_t))) {
-		binder_user_error("%d:%d got transaction with invalid offsets size, %lld\n",
-				proc->pid, thread->pid, (u64)tr->offsets_size);
-		return_error = BR_FAILED_REPLY;
-		return_error_param = -EINVAL;
-		return_error_line = __LINE__;
-		goto err_bad_offset;
-	}
-	if (!IS_ALIGNED(extra_buffers_size, sizeof(u64))) {
-		binder_user_error("%d:%d got transaction with unaligned buffers size, %lld\n",
-				  proc->pid, thread->pid,
-				  (u64)extra_buffers_size);
-		return_error = BR_FAILED_REPLY;
-		return_error_param = -EINVAL;
-		return_error_line = __LINE__;
-		goto err_bad_offset;
-	}
-	off_start_offset = ALIGN(tr->data_size, sizeof(void *));
-	buffer_offset = off_start_offset;
-	off_end_offset = off_start_offset + tr->offsets_size;
-	sg_buf_offset = ALIGN(off_end_offset, sizeof(void *));
-	sg_buf_end_offset = sg_buf_offset + extra_buffers_size -
-		lsmctx_aligned_size;
-	off_min = 0;
-	for (buffer_offset = off_start_offset; buffer_offset < off_end_offset;
-	     buffer_offset += sizeof(binder_size_t)) {
-		struct binder_object_header *hdr;
-		size_t object_size;
-		struct binder_object object;
-		binder_size_t object_offset;
-		binder_size_t copy_size;
-
-		if (binder_alloc_copy_from_buffer(&target_proc->alloc,
-						  &object_offset,
-						  t->buffer,
-						  buffer_offset,
-						  sizeof(object_offset))) {
-			binder_txn_error("%d:%d copy offset from buffer failed\n",
-				thread->pid, proc->pid);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EINVAL;
-			return_error_line = __LINE__;
-			goto err_bad_offset;
-		}
-
-		/*
-		 * Copy the source user buffer up to the next object
-		 * that will be processed.
-		 */
-		copy_size = object_offset - user_offset;
-		if (copy_size && (user_offset > object_offset ||
-				object_offset > tr->data_size ||
-				binder_alloc_copy_user_to_buffer(
-					&target_proc->alloc,
-					t->buffer, user_offset,
-					user_buffer + user_offset,
-					copy_size))) {
-			binder_user_error("%d:%d got transaction with invalid data ptr\n",
-					proc->pid, thread->pid);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EFAULT;
-			return_error_line = __LINE__;
-			goto err_copy_data_failed;
-		}
-		object_size = binder_get_object(target_proc, user_buffer,
-				t->buffer, object_offset, &object);
-		if (object_size == 0 || object_offset < off_min) {
-			binder_user_error("%d:%d got transaction with invalid offset (%lld, min %lld max %lld) or object.\n",
-					  proc->pid, thread->pid,
-					  (u64)object_offset,
-					  (u64)off_min,
-					  (u64)t->buffer->data_size);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EINVAL;
-			return_error_line = __LINE__;
-			goto err_bad_offset;
-		}
-		/*
-		 * Set offset to the next buffer fragment to be
-		 * copied
-		 */
-		user_offset = object_offset + object_size;
-
-		hdr = &object.hdr;
-		off_min = object_offset + object_size;
-		switch (hdr->type) {
-		case BINDER_TYPE_BINDER:
-		case BINDER_TYPE_WEAK_BINDER: {
-			struct flat_binder_object *fp;
-
-			fp = to_flat_binder_object(hdr);
-			ret = binder_translate_binder(fp, t, thread);
-
-			if (ret < 0 ||
-			    binder_alloc_copy_to_buffer(&target_proc->alloc,
-							t->buffer,
-							object_offset,
-							fp, sizeof(*fp))) {
-				binder_txn_error("%d:%d translate binder failed\n",
-					thread->pid, proc->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = ret;
-				return_error_line = __LINE__;
-				goto err_translate_failed;
-			}
-		} break;
-		case BINDER_TYPE_HANDLE:
-		case BINDER_TYPE_WEAK_HANDLE: {
-			struct flat_binder_object *fp;
-
-			fp = to_flat_binder_object(hdr);
-			ret = binder_translate_handle(fp, t, thread);
-			if (ret < 0 ||
-			    binder_alloc_copy_to_buffer(&target_proc->alloc,
-							t->buffer,
-							object_offset,
-							fp, sizeof(*fp))) {
-				binder_txn_error("%d:%d translate handle failed\n",
-					thread->pid, proc->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = ret;
-				return_error_line = __LINE__;
-				goto err_translate_failed;
-			}
-		} break;
-
-		case BINDER_TYPE_FD: {
-			struct binder_fd_object *fp = to_binder_fd_object(hdr);
-			binder_size_t fd_offset = object_offset +
-				(uintptr_t)&fp->fd - (uintptr_t)fp;
-			int ret = binder_translate_fd(fp->fd, fd_offset, t,
-						      thread, in_reply_to);
-
-			fp->pad_binder = 0;
-			if (ret < 0 ||
-			    binder_alloc_copy_to_buffer(&target_proc->alloc,
-							t->buffer,
-							object_offset,
-							fp, sizeof(*fp))) {
-				binder_txn_error("%d:%d translate fd failed\n",
-					thread->pid, proc->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = ret;
-				return_error_line = __LINE__;
-				goto err_translate_failed;
-			}
-		} break;
-		case BINDER_TYPE_FDA: {
-			struct binder_object ptr_object;
-			binder_size_t parent_offset;
-			struct binder_object user_object;
-			size_t user_parent_size;
-			struct binder_fd_array_object *fda =
-				to_binder_fd_array_object(hdr);
-			size_t num_valid = (buffer_offset - off_start_offset) /
-						sizeof(binder_size_t);
-			struct binder_buffer_object *parent =
-				binder_validate_ptr(target_proc, t->buffer,
-						    &ptr_object, fda->parent,
-						    off_start_offset,
-						    &parent_offset,
-						    num_valid);
-			if (!parent) {
-				binder_user_error("%d:%d got transaction with invalid parent offset or type\n",
-						  proc->pid, thread->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = -EINVAL;
-				return_error_line = __LINE__;
-				goto err_bad_parent;
-			}
-			if (!binder_validate_fixup(target_proc, t->buffer,
-						   off_start_offset,
-						   parent_offset,
-						   fda->parent_offset,
-						   last_fixup_obj_off,
-						   last_fixup_min_off)) {
-				binder_user_error("%d:%d got transaction with out-of-order buffer fixup\n",
-						  proc->pid, thread->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = -EINVAL;
-				return_error_line = __LINE__;
-				goto err_bad_parent;
-			}
-			/*
-			 * We need to read the user version of the parent
-			 * object to get the original user offset
-			 */
-			user_parent_size =
-				binder_get_object(proc, user_buffer, t->buffer,
-						  parent_offset, &user_object);
-			if (user_parent_size != sizeof(user_object.bbo)) {
-				binder_user_error("%d:%d invalid ptr object size: %zd vs %zd\n",
-						  proc->pid, thread->pid,
-						  user_parent_size,
-						  sizeof(user_object.bbo));
-				return_error = BR_FAILED_REPLY;
-				return_error_param = -EINVAL;
-				return_error_line = __LINE__;
-				goto err_bad_parent;
-			}
-			ret = binder_translate_fd_array(&pf_head, fda,
-							user_buffer, parent,
-							&user_object.bbo, t,
-							thread, in_reply_to);
-			if (!ret)
-				ret = binder_alloc_copy_to_buffer(&target_proc->alloc,
-								  t->buffer,
-								  object_offset,
-								  fda, sizeof(*fda));
-			if (ret) {
-				binder_txn_error("%d:%d translate fd array failed\n",
-					thread->pid, proc->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = ret > 0 ? -EINVAL : ret;
-				return_error_line = __LINE__;
-				goto err_translate_failed;
-			}
-			last_fixup_obj_off = parent_offset;
-			last_fixup_min_off =
-				fda->parent_offset + sizeof(u32) * fda->num_fds;
-		} break;
-		case BINDER_TYPE_PTR: {
-			struct binder_buffer_object *bp =
-				to_binder_buffer_object(hdr);
-			size_t buf_left = sg_buf_end_offset - sg_buf_offset;
-			size_t num_valid;
-
-			if (bp->length > buf_left) {
-				binder_user_error("%d:%d got transaction with too large buffer\n",
-						  proc->pid, thread->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = -EINVAL;
-				return_error_line = __LINE__;
-				goto err_bad_offset;
-			}
-			ret = binder_defer_copy(&sgc_head, sg_buf_offset,
-				(const void __user *)(uintptr_t)bp->buffer,
-				bp->length);
-			if (ret) {
-				binder_txn_error("%d:%d deferred copy failed\n",
-					thread->pid, proc->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = ret;
-				return_error_line = __LINE__;
-				goto err_translate_failed;
-			}
-			/* Fixup buffer pointer to target proc address space */
-			bp->buffer = t->buffer->user_data + sg_buf_offset;
-			sg_buf_offset += ALIGN(bp->length, sizeof(u64));
-
-			num_valid = (buffer_offset - off_start_offset) /
-					sizeof(binder_size_t);
-			ret = binder_fixup_parent(&pf_head, t,
-						  thread, bp,
-						  off_start_offset,
-						  num_valid,
-						  last_fixup_obj_off,
-						  last_fixup_min_off);
-			if (ret < 0 ||
-			    binder_alloc_copy_to_buffer(&target_proc->alloc,
-							t->buffer,
-							object_offset,
-							bp, sizeof(*bp))) {
-				binder_txn_error("%d:%d failed to fixup parent\n",
-					thread->pid, proc->pid);
-				return_error = BR_FAILED_REPLY;
-				return_error_param = ret;
-				return_error_line = __LINE__;
-				goto err_translate_failed;
-			}
-			last_fixup_obj_off = object_offset;
-			last_fixup_min_off = 0;
-		} break;
-		default:
-			binder_user_error("%d:%d got transaction with invalid object type, %x\n",
-				proc->pid, thread->pid, hdr->type);
-			return_error = BR_FAILED_REPLY;
-			return_error_param = -EINVAL;
-			return_error_line = __LINE__;
-			goto err_bad_object_type;
-		}
-	}
-	/* Done processing objects, copy the rest of the buffer */
-	if (binder_alloc_copy_user_to_buffer(
-				&target_proc->alloc,
-				t->buffer, user_offset,
-				user_buffer + user_offset,
-				tr->data_size - user_offset)) {
-		binder_user_error("%d:%d got transaction with invalid data ptr\n",
-				proc->pid, thread->pid);
-		return_error = BR_FAILED_REPLY;
-		return_error_param = -EFAULT;
-		return_error_line = __LINE__;
-		goto err_copy_data_failed;
-	}
-
-	ret = binder_do_deferred_txn_copies(&target_proc->alloc, t->buffer,
-					    &sgc_head, &pf_head);
-	if (ret) {
-		binder_user_error("%d:%d got transaction with invalid offsets ptr\n",
-				  proc->pid, thread->pid);
-		return_error = BR_FAILED_REPLY;
-		return_error_param = ret;
-		return_error_line = __LINE__;
-		goto err_copy_data_failed;
-	}
-	if (t->buffer->oneway_spam_suspect) {
-		tcomplete->type = BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT;
-		binder_netlink_report(proc, t, tr->data_size,
-				      BR_ONEWAY_SPAM_SUSPECT);
-	} else {
-		tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE;
-	}
-
-	if (reply) {
-		binder_enqueue_thread_work(thread, tcomplete);
-		binder_inner_proc_lock(target_proc);
-		if (target_thread->is_dead) {
-			return_error = BR_DEAD_REPLY;
-			binder_inner_proc_unlock(target_proc);
-			goto err_dead_proc_or_thread;
-		}
-		BUG_ON(t->buffer->async_transaction != 0);
-		binder_pop_transaction_ilocked(target_thread, in_reply_to);
-		binder_enqueue_thread_work_ilocked(target_thread, &t->work);
-		target_proc->outstanding_txns++;
-		binder_inner_proc_unlock(target_proc);
-		wake_up_interruptible_sync(&target_thread->wait);
-		binder_free_transaction(in_reply_to);
-	} else if (!(t->flags & TF_ONE_WAY)) {
-		BUG_ON(t->buffer->async_transaction != 0);
-		binder_inner_proc_lock(proc);
-		/*
-		 * Defer the TRANSACTION_COMPLETE, so we don't return to
-		 * userspace immediately; this allows the target process to
-		 * immediately start processing this transaction, reducing
-		 * latency. We will then return the TRANSACTION_COMPLETE when
-		 * the target replies (or there is an error).
-		 */
-		binder_enqueue_deferred_thread_work_ilocked(thread, tcomplete);
-		t->from_parent = thread->transaction_stack;
-		thread->transaction_stack = t;
-		binder_inner_proc_unlock(proc);
-		return_error = binder_proc_transaction(t,
-				target_proc, target_thread);
-		if (return_error) {
-			binder_inner_proc_lock(proc);
-			binder_pop_transaction_ilocked(thread, t);
-			binder_inner_proc_unlock(proc);
-			goto err_dead_proc_or_thread;
-		}
-	} else {
-		/*
-		 * Make a transaction copy. It is not safe to access 't' after
-		 * binder_proc_transaction() reported a pending frozen. The
-		 * target could thaw and consume the transaction at any point.
-		 * Instead, use a safe 't_copy' for binder_netlink_report().
-		 */
-		struct binder_transaction t_copy = *t;
-
-		BUG_ON(target_node == NULL);
-		BUG_ON(t->buffer->async_transaction != 1);
-		return_error = binder_proc_transaction(t, target_proc, NULL);
-		/*
-		 * Let the caller know when async transaction reaches a frozen
-		 * process and is put in a pending queue, waiting for the target
-		 * process to be unfrozen.
-		 */
-		if (return_error == BR_TRANSACTION_PENDING_FROZEN) {
-			tcomplete->type = BINDER_WORK_TRANSACTION_PENDING;
-			binder_netlink_report(proc, &t_copy, tr->data_size,
-					      return_error);
-		}
-		binder_enqueue_thread_work(thread, tcomplete);
-		if (return_error &&
-		    return_error != BR_TRANSACTION_PENDING_FROZEN)
-			goto err_dead_proc_or_thread;
-	}
-	if (target_thread)
-		binder_thread_dec_tmpref(target_thread);
-	binder_proc_dec_tmpref(target_proc);
-	if (target_node)
-		binder_dec_node_tmpref(target_node);
-	/*
-	 * write barrier to synchronize with initialization
-	 * of log entry
-	 */
-	smp_wmb();
-	WRITE_ONCE(e->debug_id_done, t_debug_id);
-	return;
-
-err_dead_proc_or_thread:
-	binder_txn_error("%d:%d %s process or thread\n",
-			 proc->pid, thread->pid,
-			 return_error == BR_FROZEN_REPLY ? "frozen" : "dead");
-	return_error_line = __LINE__;
-	binder_dequeue_work(proc, tcomplete);
-err_translate_failed:
-err_bad_object_type:
-err_bad_offset:
-err_bad_parent:
-err_copy_data_failed:
-	binder_cleanup_deferred_txn_lists(&sgc_head, &pf_head);
-	binder_free_txn_fixups(t);
-	trace_binder_transaction_failed_buffer_release(t->buffer);
-	binder_transaction_buffer_release(target_proc, NULL, t->buffer,
-					  buffer_offset, true);
-	if (target_node)
-		binder_dec_node_tmpref(target_node);
-	target_node = NULL;
-	t->buffer->transaction = NULL;
-	binder_alloc_free_buf(&target_proc->alloc, t->buffer);
-err_binder_alloc_buf_failed:
-err_bad_extra_size:
-	if (lsmctx.context)
-		security_release_secctx(&lsmctx);
-err_get_secctx_failed:
-	kfree(tcomplete);
-	binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE);
-err_alloc_tcomplete_failed:
-	if (trace_binder_txn_latency_free_enabled())
-		binder_txn_latency_free(t);
-err_bad_todo_list:
-err_bad_call_stack:
-err_empty_call_stack:
-err_dead_binder:
-err_invalid_target_handle:
-	if (target_node) {
-		binder_dec_node(target_node, 1, 0);
-		binder_dec_node_tmpref(target_node);
-	}
-
-	binder_netlink_report(proc, t, tr->data_size, return_error);
-	kfree(t);
-	binder_stats_deleted(BINDER_STAT_TRANSACTION);
-err_alloc_t_failed:
-
-	binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
-		     "%d:%d transaction %s to %d:%d failed %d/%d/%d, code %u size %lld-%lld line %d\n",
-		     proc->pid, thread->pid, reply ? "reply" :
-		     (tr->flags & TF_ONE_WAY ? "async" : "call"),
-		     target_proc ? target_proc->pid : 0,
-		     target_thread ? target_thread->pid : 0,
-		     t_debug_id, return_error, return_error_param,
-		     tr->code, (u64)tr->data_size, (u64)tr->offsets_size,
-		     return_error_line);
-
-	if (target_thread)
-		binder_thread_dec_tmpref(target_thread);
-	if (target_proc)
-		binder_proc_dec_tmpref(target_proc);
-
-	{
-		struct binder_transaction_log_entry *fe;
-
-		e->return_error = return_error;
-		e->return_error_param = return_error_param;
-		e->return_error_line = return_error_line;
-		fe = binder_transaction_log_add(&binder_transaction_log_failed);
-		*fe = *e;
-		/*
-		 * write barrier to synchronize with initialization
-		 * of log entry
-		 */
-		smp_wmb();
-		WRITE_ONCE(e->debug_id_done, t_debug_id);
-		WRITE_ONCE(fe->debug_id_done, t_debug_id);
-	}
-
-	BUG_ON(thread->return_error.cmd != BR_OK);
-	if (in_reply_to) {
-		binder_set_txn_from_error(in_reply_to, t_debug_id,
-				return_error, return_error_param);
-		thread->return_error.cmd = BR_TRANSACTION_COMPLETE;
-		binder_enqueue_thread_work(thread, &thread->return_error.work);
-		binder_send_failed_reply(in_reply_to, return_error);
-	} else {
-		binder_inner_proc_lock(proc);
-		binder_set_extended_error(&thread->ee, t_debug_id,
-				return_error, return_error_param);
-		binder_inner_proc_unlock(proc);
-		thread->return_error.cmd = return_error;
-		binder_enqueue_thread_work(thread, &thread->return_error.work);
-	}
-}
-
-static int
-binder_request_freeze_notification(struct binder_proc *proc,
-				   struct binder_thread *thread,
-				   struct binder_handle_cookie *handle_cookie)
-{
-	struct binder_ref_freeze *freeze;
-	struct binder_ref *ref;
-
-	freeze = kzalloc_obj(*freeze);
-	if (!freeze)
-		return -ENOMEM;
-	binder_proc_lock(proc);
-	ref = binder_get_ref_olocked(proc, handle_cookie->handle, false);
-	if (!ref) {
-		binder_user_error("%d:%d BC_REQUEST_FREEZE_NOTIFICATION invalid ref %d\n",
-				  proc->pid, thread->pid, handle_cookie->handle);
-		binder_proc_unlock(proc);
-		kfree(freeze);
-		return -EINVAL;
-	}
-
-	binder_node_lock(ref->node);
-	if (ref->freeze) {
-		binder_user_error("%d:%d BC_REQUEST_FREEZE_NOTIFICATION already set\n",
-				  proc->pid, thread->pid);
-		binder_node_unlock(ref->node);
-		binder_proc_unlock(proc);
-		kfree(freeze);
-		return -EINVAL;
-	}
-
-	binder_stats_created(BINDER_STAT_FREEZE);
-	INIT_LIST_HEAD(&freeze->work.entry);
-	freeze->cookie = handle_cookie->cookie;
-	freeze->work.type = BINDER_WORK_FROZEN_BINDER;
-	ref->freeze = freeze;
-
-	if (ref->node->proc) {
-		binder_inner_proc_lock(ref->node->proc);
-		freeze->is_frozen = ref->node->proc->is_frozen;
-		binder_inner_proc_unlock(ref->node->proc);
-
-		binder_inner_proc_lock(proc);
-		binder_enqueue_work_ilocked(&freeze->work, &proc->todo);
-		binder_wakeup_proc_ilocked(proc);
-		binder_inner_proc_unlock(proc);
-	}
-
-	binder_node_unlock(ref->node);
-	binder_proc_unlock(proc);
-	return 0;
-}
-
-static int
-binder_clear_freeze_notification(struct binder_proc *proc,
-				 struct binder_thread *thread,
-				 struct binder_handle_cookie *handle_cookie)
-{
-	struct binder_ref_freeze *freeze;
-	struct binder_ref *ref;
-
-	binder_proc_lock(proc);
-	ref = binder_get_ref_olocked(proc, handle_cookie->handle, false);
-	if (!ref) {
-		binder_user_error("%d:%d BC_CLEAR_FREEZE_NOTIFICATION invalid ref %d\n",
-				  proc->pid, thread->pid, handle_cookie->handle);
-		binder_proc_unlock(proc);
-		return -EINVAL;
-	}
-
-	binder_node_lock(ref->node);
-
-	if (!ref->freeze) {
-		binder_user_error("%d:%d BC_CLEAR_FREEZE_NOTIFICATION freeze notification not active\n",
-				  proc->pid, thread->pid);
-		binder_node_unlock(ref->node);
-		binder_proc_unlock(proc);
-		return -EINVAL;
-	}
-	freeze = ref->freeze;
-	binder_inner_proc_lock(proc);
-	if (freeze->cookie != handle_cookie->cookie) {
-		binder_user_error("%d:%d BC_CLEAR_FREEZE_NOTIFICATION freeze notification cookie mismatch %016llx != %016llx\n",
-				  proc->pid, thread->pid, (u64)freeze->cookie,
-				  (u64)handle_cookie->cookie);
-		binder_inner_proc_unlock(proc);
-		binder_node_unlock(ref->node);
-		binder_proc_unlock(proc);
-		return -EINVAL;
-	}
-	ref->freeze = NULL;
-	/*
-	 * Take the existing freeze object and overwrite its work type. There are three cases here:
-	 * 1. No pending notification. In this case just add the work to the queue.
-	 * 2. A notification was sent and is pending an ack from userspace. Once an ack arrives, we
-	 *    should resend with the new work type.
-	 * 3. A notification is pending to be sent. Since the work is already in the queue, nothing
-	 *    needs to be done here.
-	 */
-	freeze->work.type = BINDER_WORK_CLEAR_FREEZE_NOTIFICATION;
-	if (list_empty(&freeze->work.entry)) {
-		binder_enqueue_work_ilocked(&freeze->work, &proc->todo);
-		binder_wakeup_proc_ilocked(proc);
-	} else if (freeze->sent) {
-		freeze->resend = true;
-	}
-	binder_inner_proc_unlock(proc);
-	binder_node_unlock(ref->node);
-	binder_proc_unlock(proc);
-	return 0;
-}
-
-static int
-binder_freeze_notification_done(struct binder_proc *proc,
-				struct binder_thread *thread,
-				binder_uintptr_t cookie)
-{
-	struct binder_ref_freeze *freeze = NULL;
-	struct binder_work *w;
-
-	binder_inner_proc_lock(proc);
-	list_for_each_entry(w, &proc->delivered_freeze, entry) {
-		struct binder_ref_freeze *tmp_freeze =
-			container_of(w, struct binder_ref_freeze, work);
-
-		if (tmp_freeze->cookie == cookie) {
-			freeze = tmp_freeze;
-			break;
-		}
-	}
-	if (!freeze) {
-		binder_user_error("%d:%d BC_FREEZE_NOTIFICATION_DONE %016llx not found\n",
-				  proc->pid, thread->pid, (u64)cookie);
-		binder_inner_proc_unlock(proc);
-		return -EINVAL;
-	}
-	binder_dequeue_work_ilocked(&freeze->work);
-	freeze->sent = false;
-	if (freeze->resend) {
-		freeze->resend = false;
-		binder_enqueue_work_ilocked(&freeze->work, &proc->todo);
-		binder_wakeup_proc_ilocked(proc);
-	}
-	binder_inner_proc_unlock(proc);
-	return 0;
-}
-
-/**
- * binder_free_buf() - free the specified buffer
- * @proc:       binder proc that owns buffer
- * @thread:     binder thread performing the buffer release
- * @buffer:     buffer to be freed
- * @is_failure: failed to send transaction
- *
- * If the buffer is for an async transaction, enqueue the next async
- * transaction from the node.
- *
- * Cleanup the buffer and free it.
- */
-static void
-binder_free_buf(struct binder_proc *proc,
-		struct binder_thread *thread,
-		struct binder_buffer *buffer, bool is_failure)
-{
-	binder_inner_proc_lock(proc);
-	if (buffer->transaction) {
-		buffer->transaction->buffer = NULL;
-		buffer->transaction = NULL;
-	}
-	binder_inner_proc_unlock(proc);
-	if (buffer->async_transaction && buffer->target_node) {
-		struct binder_node *buf_node;
-		struct binder_work *w;
-
-		buf_node = buffer->target_node;
-		binder_node_inner_lock(buf_node);
-		BUG_ON(!buf_node->has_async_transaction);
-		BUG_ON(buf_node->proc != proc);
-		w = binder_dequeue_work_head_ilocked(
-				&buf_node->async_todo);
-		if (!w) {
-			buf_node->has_async_transaction = false;
-		} else {
-			binder_enqueue_work_ilocked(
-					w, &proc->todo);
-			binder_wakeup_proc_ilocked(proc);
-		}
-		binder_node_inner_unlock(buf_node);
-	}
-	trace_binder_transaction_buffer_release(buffer);
-	binder_release_entire_buffer(proc, thread, buffer, is_failure);
-	binder_alloc_free_buf(&proc->alloc, buffer);
-}
-
-static int binder_thread_write(struct binder_proc *proc,
-			struct binder_thread *thread,
-			binder_uintptr_t binder_buffer, size_t size,
-			binder_size_t *consumed)
-{
-	uint32_t cmd;
-	struct binder_context *context = proc->context;
-	void __user *buffer = (void __user *)(uintptr_t)binder_buffer;
-	void __user *ptr = buffer + *consumed;
-	void __user *end = buffer + size;
-
-	while (ptr < end && thread->return_error.cmd == BR_OK) {
-		int ret;
-
-		if (get_user(cmd, (uint32_t __user *)ptr))
-			return -EFAULT;
-		ptr += sizeof(uint32_t);
-		trace_binder_command(cmd);
-		if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.bc)) {
-			atomic_inc(&binder_stats.bc[_IOC_NR(cmd)]);
-			atomic_inc(&proc->stats.bc[_IOC_NR(cmd)]);
-			atomic_inc(&thread->stats.bc[_IOC_NR(cmd)]);
-		}
-		switch (cmd) {
-		case BC_INCREFS:
-		case BC_ACQUIRE:
-		case BC_RELEASE:
-		case BC_DECREFS: {
-			uint32_t target;
-			const char *debug_string;
-			bool strong = cmd == BC_ACQUIRE || cmd == BC_RELEASE;
-			bool increment = cmd == BC_INCREFS || cmd == BC_ACQUIRE;
-			struct binder_ref_data rdata;
-
-			if (get_user(target, (uint32_t __user *)ptr))
-				return -EFAULT;
-
-			ptr += sizeof(uint32_t);
-			ret = -1;
-			if (increment && !target) {
-				struct binder_node *ctx_mgr_node;
-
-				mutex_lock(&context->context_mgr_node_lock);
-				ctx_mgr_node = context->binder_context_mgr_node;
-				if (ctx_mgr_node) {
-					if (ctx_mgr_node->proc == proc) {
-						binder_user_error("%d:%d context manager tried to acquire desc 0\n",
-								  proc->pid, thread->pid);
-						mutex_unlock(&context->context_mgr_node_lock);
-						return -EINVAL;
-					}
-					ret = binder_inc_ref_for_node(
-							proc, ctx_mgr_node,
-							strong, NULL, &rdata);
-				}
-				mutex_unlock(&context->context_mgr_node_lock);
-			}
-			if (ret)
-				ret = binder_update_ref_for_handle(
-						proc, target, increment, strong,
-						&rdata);
-			if (!ret && rdata.desc != target) {
-				binder_user_error("%d:%d tried to acquire reference to desc %d, got %d instead\n",
-					proc->pid, thread->pid,
-					target, rdata.desc);
-			}
-			switch (cmd) {
-			case BC_INCREFS:
-				debug_string = "IncRefs";
-				break;
-			case BC_ACQUIRE:
-				debug_string = "Acquire";
-				break;
-			case BC_RELEASE:
-				debug_string = "Release";
-				break;
-			case BC_DECREFS:
-			default:
-				debug_string = "DecRefs";
-				break;
-			}
-			if (ret) {
-				binder_user_error("%d:%d %s %d refcount change on invalid ref %d ret %d\n",
-					proc->pid, thread->pid, debug_string,
-					strong, target, ret);
-				break;
-			}
-			binder_debug(BINDER_DEBUG_USER_REFS,
-				     "%d:%d %s ref %d desc %d s %d w %d\n",
-				     proc->pid, thread->pid, debug_string,
-				     rdata.debug_id, rdata.desc, rdata.strong,
-				     rdata.weak);
-			break;
-		}
-		case BC_INCREFS_DONE:
-		case BC_ACQUIRE_DONE: {
-			binder_uintptr_t node_ptr;
-			binder_uintptr_t cookie;
-			struct binder_node *node;
-			bool free_node;
-
-			if (get_user(node_ptr, (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(binder_uintptr_t);
-			if (get_user(cookie, (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(binder_uintptr_t);
-			node = binder_get_node(proc, node_ptr);
-			if (node == NULL) {
-				binder_user_error("%d:%d %s u%016llx no match\n",
-					proc->pid, thread->pid,
-					cmd == BC_INCREFS_DONE ?
-					"BC_INCREFS_DONE" :
-					"BC_ACQUIRE_DONE",
-					(u64)node_ptr);
-				break;
-			}
-			if (cookie != node->cookie) {
-				binder_user_error("%d:%d %s u%016llx node %d cookie mismatch %016llx != %016llx\n",
-					proc->pid, thread->pid,
-					cmd == BC_INCREFS_DONE ?
-					"BC_INCREFS_DONE" : "BC_ACQUIRE_DONE",
-					(u64)node_ptr, node->debug_id,
-					(u64)cookie, (u64)node->cookie);
-				binder_put_node(node);
-				break;
-			}
-			binder_node_inner_lock(node);
-			if (cmd == BC_ACQUIRE_DONE) {
-				if (node->pending_strong_ref == 0) {
-					binder_user_error("%d:%d BC_ACQUIRE_DONE node %d has no pending acquire request\n",
-						proc->pid, thread->pid,
-						node->debug_id);
-					binder_node_inner_unlock(node);
-					binder_put_node(node);
-					break;
-				}
-				node->pending_strong_ref = 0;
-			} else {
-				if (node->pending_weak_ref == 0) {
-					binder_user_error("%d:%d BC_INCREFS_DONE node %d has no pending increfs request\n",
-						proc->pid, thread->pid,
-						node->debug_id);
-					binder_node_inner_unlock(node);
-					binder_put_node(node);
-					break;
-				}
-				node->pending_weak_ref = 0;
-			}
-			free_node = binder_dec_node_nilocked(node,
-					cmd == BC_ACQUIRE_DONE, 0);
-			WARN_ON(free_node);
-			binder_debug(BINDER_DEBUG_USER_REFS,
-				     "%d:%d %s node %d ls %d lw %d tr %d\n",
-				     proc->pid, thread->pid,
-				     cmd == BC_INCREFS_DONE ? "BC_INCREFS_DONE" : "BC_ACQUIRE_DONE",
-				     node->debug_id, node->local_strong_refs,
-				     node->local_weak_refs, node->tmp_refs);
-			binder_node_inner_unlock(node);
-			binder_put_node(node);
-			break;
-		}
-		case BC_ATTEMPT_ACQUIRE:
-			pr_err("BC_ATTEMPT_ACQUIRE not supported\n");
-			return -EINVAL;
-		case BC_ACQUIRE_RESULT:
-			pr_err("BC_ACQUIRE_RESULT not supported\n");
-			return -EINVAL;
-
-		case BC_FREE_BUFFER: {
-			binder_uintptr_t data_ptr;
-			struct binder_buffer *buffer;
-
-			if (get_user(data_ptr, (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(binder_uintptr_t);
-
-			buffer = binder_alloc_prepare_to_free(&proc->alloc,
-							      data_ptr);
-			if (IS_ERR_OR_NULL(buffer)) {
-				if (PTR_ERR(buffer) == -EPERM) {
-					binder_user_error(
-						"%d:%d BC_FREE_BUFFER matched unreturned or currently freeing buffer at offset %lx\n",
-						proc->pid, thread->pid,
-						(unsigned long)data_ptr - proc->alloc.vm_start);
-				} else {
-					binder_user_error(
-						"%d:%d BC_FREE_BUFFER no match for buffer at offset %lx\n",
-						proc->pid, thread->pid,
-						(unsigned long)data_ptr - proc->alloc.vm_start);
-				}
-				break;
-			}
-			binder_debug(BINDER_DEBUG_FREE_BUFFER,
-				     "%d:%d BC_FREE_BUFFER at offset %lx found buffer %d for %s transaction\n",
-				     proc->pid, thread->pid,
-				     (unsigned long)data_ptr - proc->alloc.vm_start,
-				     buffer->debug_id,
-				     buffer->transaction ? "active" : "finished");
-			binder_free_buf(proc, thread, buffer, false);
-			break;
-		}
-
-		case BC_TRANSACTION_SG:
-		case BC_REPLY_SG: {
-			struct binder_transaction_data_sg tr;
-
-			if (copy_from_user(&tr, ptr, sizeof(tr)))
-				return -EFAULT;
-			ptr += sizeof(tr);
-			binder_transaction(proc, thread, &tr.transaction_data,
-					   cmd == BC_REPLY_SG, tr.buffers_size);
-			break;
-		}
-		case BC_TRANSACTION:
-		case BC_REPLY: {
-			struct binder_transaction_data tr;
-
-			if (copy_from_user(&tr, ptr, sizeof(tr)))
-				return -EFAULT;
-			ptr += sizeof(tr);
-			binder_transaction(proc, thread, &tr,
-					   cmd == BC_REPLY, 0);
-			break;
-		}
-
-		case BC_REGISTER_LOOPER:
-			binder_debug(BINDER_DEBUG_THREADS,
-				     "%d:%d BC_REGISTER_LOOPER\n",
-				     proc->pid, thread->pid);
-			binder_inner_proc_lock(proc);
-			if (thread->looper & BINDER_LOOPER_STATE_ENTERED) {
-				thread->looper |= BINDER_LOOPER_STATE_INVALID;
-				binder_user_error("%d:%d ERROR: BC_REGISTER_LOOPER called after BC_ENTER_LOOPER\n",
-					proc->pid, thread->pid);
-			} else if (proc->requested_threads == 0) {
-				thread->looper |= BINDER_LOOPER_STATE_INVALID;
-				binder_user_error("%d:%d ERROR: BC_REGISTER_LOOPER called without request\n",
-					proc->pid, thread->pid);
-			} else {
-				proc->requested_threads--;
-				proc->requested_threads_started++;
-			}
-			thread->looper |= BINDER_LOOPER_STATE_REGISTERED;
-			binder_inner_proc_unlock(proc);
-			break;
-		case BC_ENTER_LOOPER:
-			binder_debug(BINDER_DEBUG_THREADS,
-				     "%d:%d BC_ENTER_LOOPER\n",
-				     proc->pid, thread->pid);
-			if (thread->looper & BINDER_LOOPER_STATE_REGISTERED) {
-				thread->looper |= BINDER_LOOPER_STATE_INVALID;
-				binder_user_error("%d:%d ERROR: BC_ENTER_LOOPER called after BC_REGISTER_LOOPER\n",
-					proc->pid, thread->pid);
-			}
-			thread->looper |= BINDER_LOOPER_STATE_ENTERED;
-			break;
-		case BC_EXIT_LOOPER:
-			binder_debug(BINDER_DEBUG_THREADS,
-				     "%d:%d BC_EXIT_LOOPER\n",
-				     proc->pid, thread->pid);
-			thread->looper |= BINDER_LOOPER_STATE_EXITED;
-			break;
-
-		case BC_REQUEST_DEATH_NOTIFICATION:
-		case BC_CLEAR_DEATH_NOTIFICATION: {
-			uint32_t target;
-			binder_uintptr_t cookie;
-			struct binder_ref *ref;
-			struct binder_ref_death *death = NULL;
-
-			if (get_user(target, (uint32_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(uint32_t);
-			if (get_user(cookie, (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(binder_uintptr_t);
-			if (cmd == BC_REQUEST_DEATH_NOTIFICATION) {
-				/*
-				 * Allocate memory for death notification
-				 * before taking lock
-				 */
-				death = kzalloc_obj(*death);
-				if (death == NULL) {
-					WARN_ON(thread->return_error.cmd !=
-						BR_OK);
-					thread->return_error.cmd = BR_ERROR;
-					binder_enqueue_thread_work(
-						thread,
-						&thread->return_error.work);
-					binder_debug(
-						BINDER_DEBUG_FAILED_TRANSACTION,
-						"%d:%d BC_REQUEST_DEATH_NOTIFICATION failed\n",
-						proc->pid, thread->pid);
-					break;
-				}
-			}
-			binder_proc_lock(proc);
-			ref = binder_get_ref_olocked(proc, target, false);
-			if (ref == NULL) {
-				binder_user_error("%d:%d %s invalid ref %d\n",
-					proc->pid, thread->pid,
-					cmd == BC_REQUEST_DEATH_NOTIFICATION ?
-					"BC_REQUEST_DEATH_NOTIFICATION" :
-					"BC_CLEAR_DEATH_NOTIFICATION",
-					target);
-				binder_proc_unlock(proc);
-				kfree(death);
-				break;
-			}
-
-			binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION,
-				     "%d:%d %s %016llx ref %d desc %d s %d w %d for node %d\n",
-				     proc->pid, thread->pid,
-				     cmd == BC_REQUEST_DEATH_NOTIFICATION ?
-				     "BC_REQUEST_DEATH_NOTIFICATION" :
-				     "BC_CLEAR_DEATH_NOTIFICATION",
-				     (u64)cookie, ref->data.debug_id,
-				     ref->data.desc, ref->data.strong,
-				     ref->data.weak, ref->node->debug_id);
-
-			binder_node_lock(ref->node);
-			if (cmd == BC_REQUEST_DEATH_NOTIFICATION) {
-				if (ref->death) {
-					binder_user_error("%d:%d BC_REQUEST_DEATH_NOTIFICATION death notification already set\n",
-						proc->pid, thread->pid);
-					binder_node_unlock(ref->node);
-					binder_proc_unlock(proc);
-					kfree(death);
-					break;
-				}
-				binder_stats_created(BINDER_STAT_DEATH);
-				INIT_LIST_HEAD(&death->work.entry);
-				death->cookie = cookie;
-				ref->death = death;
-				if (ref->node->proc == NULL) {
-					ref->death->work.type = BINDER_WORK_DEAD_BINDER;
-
-					binder_inner_proc_lock(proc);
-					binder_enqueue_work_ilocked(
-						&ref->death->work, &proc->todo);
-					binder_wakeup_proc_ilocked(proc);
-					binder_inner_proc_unlock(proc);
-				}
-			} else {
-				if (ref->death == NULL) {
-					binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification not active\n",
-						proc->pid, thread->pid);
-					binder_node_unlock(ref->node);
-					binder_proc_unlock(proc);
-					break;
-				}
-				death = ref->death;
-				if (death->cookie != cookie) {
-					binder_user_error("%d:%d BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch %016llx != %016llx\n",
-						proc->pid, thread->pid,
-						(u64)death->cookie,
-						(u64)cookie);
-					binder_node_unlock(ref->node);
-					binder_proc_unlock(proc);
-					break;
-				}
-				ref->death = NULL;
-				binder_inner_proc_lock(proc);
-				if (list_empty(&death->work.entry)) {
-					death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION;
-					if (thread->looper &
-					    (BINDER_LOOPER_STATE_REGISTERED |
-					     BINDER_LOOPER_STATE_ENTERED))
-						binder_enqueue_thread_work_ilocked(
-								thread,
-								&death->work);
-					else {
-						binder_enqueue_work_ilocked(
-								&death->work,
-								&proc->todo);
-						binder_wakeup_proc_ilocked(
-								proc);
-					}
-				} else {
-					BUG_ON(death->work.type != BINDER_WORK_DEAD_BINDER);
-					death->work.type = BINDER_WORK_DEAD_BINDER_AND_CLEAR;
-				}
-				binder_inner_proc_unlock(proc);
-			}
-			binder_node_unlock(ref->node);
-			binder_proc_unlock(proc);
-		} break;
-		case BC_DEAD_BINDER_DONE: {
-			struct binder_work *w;
-			binder_uintptr_t cookie;
-			struct binder_ref_death *death = NULL;
-
-			if (get_user(cookie, (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-
-			ptr += sizeof(cookie);
-			binder_inner_proc_lock(proc);
-			list_for_each_entry(w, &proc->delivered_death,
-					    entry) {
-				struct binder_ref_death *tmp_death =
-					container_of(w,
-						     struct binder_ref_death,
-						     work);
-
-				if (tmp_death->cookie == cookie) {
-					death = tmp_death;
-					break;
-				}
-			}
-			binder_debug(BINDER_DEBUG_DEAD_BINDER,
-				     "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n",
-				     proc->pid, thread->pid, (u64)cookie,
-				     death);
-			if (death == NULL) {
-				binder_user_error("%d:%d BC_DEAD_BINDER_DONE %016llx not found\n",
-					proc->pid, thread->pid, (u64)cookie);
-				binder_inner_proc_unlock(proc);
-				break;
-			}
-			binder_dequeue_work_ilocked(&death->work);
-			if (death->work.type == BINDER_WORK_DEAD_BINDER_AND_CLEAR) {
-				death->work.type = BINDER_WORK_CLEAR_DEATH_NOTIFICATION;
-				if (thread->looper &
-					(BINDER_LOOPER_STATE_REGISTERED |
-					 BINDER_LOOPER_STATE_ENTERED))
-					binder_enqueue_thread_work_ilocked(
-						thread, &death->work);
-				else {
-					binder_enqueue_work_ilocked(
-							&death->work,
-							&proc->todo);
-					binder_wakeup_proc_ilocked(proc);
-				}
-			}
-			binder_inner_proc_unlock(proc);
-		} break;
-
-		case BC_REQUEST_FREEZE_NOTIFICATION: {
-			struct binder_handle_cookie handle_cookie;
-			int error;
-
-			if (copy_from_user(&handle_cookie, ptr, sizeof(handle_cookie)))
-				return -EFAULT;
-			ptr += sizeof(handle_cookie);
-			error = binder_request_freeze_notification(proc, thread,
-								   &handle_cookie);
-			if (error)
-				return error;
-		} break;
-
-		case BC_CLEAR_FREEZE_NOTIFICATION: {
-			struct binder_handle_cookie handle_cookie;
-			int error;
-
-			if (copy_from_user(&handle_cookie, ptr, sizeof(handle_cookie)))
-				return -EFAULT;
-			ptr += sizeof(handle_cookie);
-			error = binder_clear_freeze_notification(proc, thread, &handle_cookie);
-			if (error)
-				return error;
-		} break;
-
-		case BC_FREEZE_NOTIFICATION_DONE: {
-			binder_uintptr_t cookie;
-			int error;
-
-			if (get_user(cookie, (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-
-			ptr += sizeof(cookie);
-			error = binder_freeze_notification_done(proc, thread, cookie);
-			if (error)
-				return error;
-		} break;
-
-		default:
-			pr_err("%d:%d unknown command %u\n",
-			       proc->pid, thread->pid, cmd);
-			return -EINVAL;
-		}
-		*consumed = ptr - buffer;
-	}
-	return 0;
-}
-
-static void binder_stat_br(struct binder_proc *proc,
-			   struct binder_thread *thread, uint32_t cmd)
-{
-	trace_binder_return(cmd);
-	if (_IOC_NR(cmd) < ARRAY_SIZE(binder_stats.br)) {
-		atomic_inc(&binder_stats.br[_IOC_NR(cmd)]);
-		atomic_inc(&proc->stats.br[_IOC_NR(cmd)]);
-		atomic_inc(&thread->stats.br[_IOC_NR(cmd)]);
-	}
-}
-
-static int binder_put_node_cmd(struct binder_proc *proc,
-			       struct binder_thread *thread,
-			       void __user **ptrp,
-			       binder_uintptr_t node_ptr,
-			       binder_uintptr_t node_cookie,
-			       int node_debug_id,
-			       uint32_t cmd, const char *cmd_name)
-{
-	void __user *ptr = *ptrp;
-
-	if (put_user(cmd, (uint32_t __user *)ptr))
-		return -EFAULT;
-	ptr += sizeof(uint32_t);
-
-	if (put_user(node_ptr, (binder_uintptr_t __user *)ptr))
-		return -EFAULT;
-	ptr += sizeof(binder_uintptr_t);
-
-	if (put_user(node_cookie, (binder_uintptr_t __user *)ptr))
-		return -EFAULT;
-	ptr += sizeof(binder_uintptr_t);
-
-	binder_stat_br(proc, thread, cmd);
-	binder_debug(BINDER_DEBUG_USER_REFS, "%d:%d %s %d u%016llx c%016llx\n",
-		     proc->pid, thread->pid, cmd_name, node_debug_id,
-		     (u64)node_ptr, (u64)node_cookie);
-
-	*ptrp = ptr;
-	return 0;
-}
-
-static int binder_wait_for_work(struct binder_thread *thread,
-				bool do_proc_work)
-{
-	DEFINE_WAIT(wait);
-	struct binder_proc *proc = thread->proc;
-	int ret = 0;
-
-	binder_inner_proc_lock(proc);
-	for (;;) {
-		prepare_to_wait(&thread->wait, &wait, TASK_INTERRUPTIBLE|TASK_FREEZABLE);
-		if (binder_has_work_ilocked(thread, do_proc_work))
-			break;
-		if (do_proc_work)
-			list_add(&thread->waiting_thread_node,
-				 &proc->waiting_threads);
-		binder_inner_proc_unlock(proc);
-		schedule();
-		binder_inner_proc_lock(proc);
-		list_del_init(&thread->waiting_thread_node);
-		if (signal_pending(current)) {
-			ret = -EINTR;
-			break;
-		}
-	}
-	finish_wait(&thread->wait, &wait);
-	binder_inner_proc_unlock(proc);
-
-	return ret;
-}
-
-/**
- * binder_apply_fd_fixups() - finish fd translation
- * @proc:         binder_proc associated @t->buffer
- * @t:	binder transaction with list of fd fixups
- *
- * Now that we are in the context of the transaction target
- * process, we can allocate and install fds. Process the
- * list of fds to translate and fixup the buffer with the
- * new fds first and only then install the files.
- *
- * If we fail to allocate an fd, skip the install and release
- * any fds that have already been allocated.
- *
- * Return: 0 on success, a negative errno code on failure.
- */
-static int binder_apply_fd_fixups(struct binder_proc *proc,
-				  struct binder_transaction *t)
-{
-	struct binder_txn_fd_fixup *fixup, *tmp;
-	int ret = 0;
-
-	list_for_each_entry(fixup, &t->fd_fixups, fixup_entry) {
-		int fd = get_unused_fd_flags(O_CLOEXEC);
-
-		if (fd < 0) {
-			binder_debug(BINDER_DEBUG_TRANSACTION,
-				     "failed fd fixup txn %d fd %d\n",
-				     t->debug_id, fd);
-			ret = -ENOMEM;
-			goto err;
-		}
-		binder_debug(BINDER_DEBUG_TRANSACTION,
-			     "fd fixup txn %d fd %d\n",
-			     t->debug_id, fd);
-		trace_binder_transaction_fd_recv(t, fd, fixup->offset);
-		fixup->target_fd = fd;
-		if (binder_alloc_copy_to_buffer(&proc->alloc, t->buffer,
-						fixup->offset, &fd,
-						sizeof(u32))) {
-			ret = -EINVAL;
-			goto err;
-		}
-	}
-	list_for_each_entry_safe(fixup, tmp, &t->fd_fixups, fixup_entry) {
-		fd_install(fixup->target_fd, fixup->file);
-		list_del(&fixup->fixup_entry);
-		kfree(fixup);
-	}
-
-	return ret;
-
-err:
-	binder_free_txn_fixups(t);
-	return ret;
-}
-
-static int binder_thread_read(struct binder_proc *proc,
-			      struct binder_thread *thread,
-			      binder_uintptr_t binder_buffer, size_t size,
-			      binder_size_t *consumed, int non_block)
-{
-	void __user *buffer = (void __user *)(uintptr_t)binder_buffer;
-	void __user *ptr = buffer + *consumed;
-	void __user *end = buffer + size;
-
-	int ret = 0;
-	int wait_for_proc_work;
-
-	if (*consumed == 0) {
-		if (put_user(BR_NOOP, (uint32_t __user *)ptr))
-			return -EFAULT;
-		ptr += sizeof(uint32_t);
-	}
-
-retry:
-	binder_inner_proc_lock(proc);
-	wait_for_proc_work = binder_available_for_proc_work_ilocked(thread);
-	binder_inner_proc_unlock(proc);
-
-	thread->looper |= BINDER_LOOPER_STATE_WAITING;
-
-	trace_binder_wait_for_work(wait_for_proc_work,
-				   !!thread->transaction_stack,
-				   !binder_worklist_empty(proc, &thread->todo));
-	if (wait_for_proc_work) {
-		if (!(thread->looper & (BINDER_LOOPER_STATE_REGISTERED |
-					BINDER_LOOPER_STATE_ENTERED))) {
-			binder_user_error("%d:%d ERROR: Thread waiting for process work before calling BC_REGISTER_LOOPER or BC_ENTER_LOOPER (state %x)\n",
-				proc->pid, thread->pid, thread->looper);
-			wait_event_interruptible(binder_user_error_wait,
-						 binder_stop_on_user_error < 2);
-		}
-		binder_set_nice(proc->default_priority);
-	}
-
-	if (non_block) {
-		if (!binder_has_work(thread, wait_for_proc_work))
-			ret = -EAGAIN;
-	} else {
-		ret = binder_wait_for_work(thread, wait_for_proc_work);
-	}
-
-	thread->looper &= ~BINDER_LOOPER_STATE_WAITING;
-
-	if (ret)
-		return ret;
-
-	while (1) {
-		uint32_t cmd;
-		struct binder_transaction_data_secctx tr;
-		struct binder_transaction_data *trd = &tr.transaction_data;
-		struct binder_work *w = NULL;
-		struct list_head *list = NULL;
-		struct binder_transaction *t = NULL;
-		struct binder_thread *t_from;
-		size_t trsize = sizeof(*trd);
-
-		binder_inner_proc_lock(proc);
-		if (!binder_worklist_empty_ilocked(&thread->todo))
-			list = &thread->todo;
-		else if (!binder_worklist_empty_ilocked(&proc->todo) &&
-			   wait_for_proc_work)
-			list = &proc->todo;
-		else {
-			binder_inner_proc_unlock(proc);
-
-			/* no data added */
-			if (ptr - buffer == 4 && !thread->looper_need_return)
-				goto retry;
-			break;
-		}
-
-		if (end - ptr < sizeof(tr) + 4) {
-			binder_inner_proc_unlock(proc);
-			break;
-		}
-		w = binder_dequeue_work_head_ilocked(list);
-		if (binder_worklist_empty_ilocked(&thread->todo))
-			thread->process_todo = false;
-
-		switch (w->type) {
-		case BINDER_WORK_TRANSACTION: {
-			binder_inner_proc_unlock(proc);
-			t = container_of(w, struct binder_transaction, work);
-		} break;
-		case BINDER_WORK_RETURN_ERROR: {
-			struct binder_error *e = container_of(
-					w, struct binder_error, work);
-
-			WARN_ON(e->cmd == BR_OK);
-			binder_inner_proc_unlock(proc);
-			if (put_user(e->cmd, (uint32_t __user *)ptr))
-				return -EFAULT;
-			cmd = e->cmd;
-			e->cmd = BR_OK;
-			ptr += sizeof(uint32_t);
-
-			binder_stat_br(proc, thread, cmd);
-		} break;
-		case BINDER_WORK_TRANSACTION_COMPLETE:
-		case BINDER_WORK_TRANSACTION_PENDING:
-		case BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT: {
-			if (proc->oneway_spam_detection_enabled &&
-				   w->type == BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT)
-				cmd = BR_ONEWAY_SPAM_SUSPECT;
-			else if (w->type == BINDER_WORK_TRANSACTION_PENDING)
-				cmd = BR_TRANSACTION_PENDING_FROZEN;
-			else
-				cmd = BR_TRANSACTION_COMPLETE;
-			binder_inner_proc_unlock(proc);
-			kfree(w);
-			binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE);
-			if (put_user(cmd, (uint32_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(uint32_t);
-
-			binder_stat_br(proc, thread, cmd);
-			binder_debug(BINDER_DEBUG_TRANSACTION_COMPLETE,
-				     "%d:%d BR_TRANSACTION_COMPLETE\n",
-				     proc->pid, thread->pid);
-		} break;
-		case BINDER_WORK_NODE: {
-			struct binder_node *node = container_of(w, struct binder_node, work);
-			int strong, weak;
-			binder_uintptr_t node_ptr = node->ptr;
-			binder_uintptr_t node_cookie = node->cookie;
-			int node_debug_id = node->debug_id;
-			int has_weak_ref;
-			int has_strong_ref;
-			void __user *orig_ptr = ptr;
-
-			BUG_ON(proc != node->proc);
-			strong = node->internal_strong_refs ||
-					node->local_strong_refs;
-			weak = !hlist_empty(&node->refs) ||
-					node->local_weak_refs ||
-					node->tmp_refs || strong;
-			has_strong_ref = node->has_strong_ref;
-			has_weak_ref = node->has_weak_ref;
-
-			if (weak && !has_weak_ref) {
-				node->has_weak_ref = 1;
-				node->pending_weak_ref = 1;
-				node->local_weak_refs++;
-			}
-			if (strong && !has_strong_ref) {
-				node->has_strong_ref = 1;
-				node->pending_strong_ref = 1;
-				node->local_strong_refs++;
-			}
-			if (!strong && has_strong_ref)
-				node->has_strong_ref = 0;
-			if (!weak && has_weak_ref)
-				node->has_weak_ref = 0;
-			if (!weak && !strong) {
-				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-					     "%d:%d node %d u%016llx c%016llx deleted\n",
-					     proc->pid, thread->pid,
-					     node_debug_id,
-					     (u64)node_ptr,
-					     (u64)node_cookie);
-				rb_erase(&node->rb_node, &proc->nodes);
-				binder_inner_proc_unlock(proc);
-				binder_node_lock(node);
-				/*
-				 * Acquire the node lock before freeing the
-				 * node to serialize with other threads that
-				 * may have been holding the node lock while
-				 * decrementing this node (avoids race where
-				 * this thread frees while the other thread
-				 * is unlocking the node after the final
-				 * decrement)
-				 */
-				binder_node_unlock(node);
-				binder_free_node(node);
-			} else
-				binder_inner_proc_unlock(proc);
-
-			if (weak && !has_weak_ref)
-				ret = binder_put_node_cmd(
-						proc, thread, &ptr, node_ptr,
-						node_cookie, node_debug_id,
-						BR_INCREFS, "BR_INCREFS");
-			if (!ret && strong && !has_strong_ref)
-				ret = binder_put_node_cmd(
-						proc, thread, &ptr, node_ptr,
-						node_cookie, node_debug_id,
-						BR_ACQUIRE, "BR_ACQUIRE");
-			if (!ret && !strong && has_strong_ref)
-				ret = binder_put_node_cmd(
-						proc, thread, &ptr, node_ptr,
-						node_cookie, node_debug_id,
-						BR_RELEASE, "BR_RELEASE");
-			if (!ret && !weak && has_weak_ref)
-				ret = binder_put_node_cmd(
-						proc, thread, &ptr, node_ptr,
-						node_cookie, node_debug_id,
-						BR_DECREFS, "BR_DECREFS");
-			if (orig_ptr == ptr)
-				binder_debug(BINDER_DEBUG_INTERNAL_REFS,
-					     "%d:%d node %d u%016llx c%016llx state unchanged\n",
-					     proc->pid, thread->pid,
-					     node_debug_id,
-					     (u64)node_ptr,
-					     (u64)node_cookie);
-			if (ret)
-				return ret;
-		} break;
-		case BINDER_WORK_DEAD_BINDER:
-		case BINDER_WORK_DEAD_BINDER_AND_CLEAR:
-		case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: {
-			struct binder_ref_death *death;
-			uint32_t cmd;
-			binder_uintptr_t cookie;
-
-			death = container_of(w, struct binder_ref_death, work);
-			if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION)
-				cmd = BR_CLEAR_DEATH_NOTIFICATION_DONE;
-			else
-				cmd = BR_DEAD_BINDER;
-			cookie = death->cookie;
-
-			binder_debug(BINDER_DEBUG_DEATH_NOTIFICATION,
-				     "%d:%d %s %016llx\n",
-				      proc->pid, thread->pid,
-				      cmd == BR_DEAD_BINDER ?
-				      "BR_DEAD_BINDER" :
-				      "BR_CLEAR_DEATH_NOTIFICATION_DONE",
-				      (u64)cookie);
-			if (w->type == BINDER_WORK_CLEAR_DEATH_NOTIFICATION) {
-				binder_inner_proc_unlock(proc);
-				kfree(death);
-				binder_stats_deleted(BINDER_STAT_DEATH);
-			} else {
-				binder_enqueue_work_ilocked(
-						w, &proc->delivered_death);
-				binder_inner_proc_unlock(proc);
-			}
-			if (put_user(cmd, (uint32_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(uint32_t);
-			if (put_user(cookie,
-				     (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(binder_uintptr_t);
-			binder_stat_br(proc, thread, cmd);
-			if (cmd == BR_DEAD_BINDER)
-				goto done; /* DEAD_BINDER notifications can cause transactions */
-		} break;
-
-		case BINDER_WORK_FROZEN_BINDER: {
-			struct binder_ref_freeze *freeze;
-			struct binder_frozen_state_info info;
-
-			memset(&info, 0, sizeof(info));
-			freeze = container_of(w, struct binder_ref_freeze, work);
-			info.is_frozen = freeze->is_frozen;
-			info.cookie = freeze->cookie;
-			freeze->sent = true;
-			binder_enqueue_work_ilocked(w, &proc->delivered_freeze);
-			binder_inner_proc_unlock(proc);
-
-			if (put_user(BR_FROZEN_BINDER, (uint32_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(uint32_t);
-			if (copy_to_user(ptr, &info, sizeof(info)))
-				return -EFAULT;
-			ptr += sizeof(info);
-			binder_stat_br(proc, thread, BR_FROZEN_BINDER);
-			goto done; /* BR_FROZEN_BINDER notifications can cause transactions */
-		} break;
-
-		case BINDER_WORK_CLEAR_FREEZE_NOTIFICATION: {
-			struct binder_ref_freeze *freeze =
-			    container_of(w, struct binder_ref_freeze, work);
-			binder_uintptr_t cookie = freeze->cookie;
-
-			binder_inner_proc_unlock(proc);
-			kfree(freeze);
-			binder_stats_deleted(BINDER_STAT_FREEZE);
-			if (put_user(BR_CLEAR_FREEZE_NOTIFICATION_DONE, (uint32_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(uint32_t);
-			if (put_user(cookie, (binder_uintptr_t __user *)ptr))
-				return -EFAULT;
-			ptr += sizeof(binder_uintptr_t);
-			binder_stat_br(proc, thread, BR_CLEAR_FREEZE_NOTIFICATION_DONE);
-		} break;
-
-		default:
-			binder_inner_proc_unlock(proc);
-			pr_err("%d:%d: bad work type %d\n",
-			       proc->pid, thread->pid, w->type);
-			break;
-		}
-
-		if (!t)
-			continue;
-
-		BUG_ON(t->buffer == NULL);
-		if (t->buffer->target_node) {
-			struct binder_node *target_node = t->buffer->target_node;
-
-			trd->target.ptr = target_node->ptr;
-			trd->cookie =  target_node->cookie;
-			t->saved_priority = task_nice(current);
-			if (t->priority < target_node->min_priority &&
-			    !(t->flags & TF_ONE_WAY))
-				binder_set_nice(t->priority);
-			else if (!(t->flags & TF_ONE_WAY) ||
-				 t->saved_priority > target_node->min_priority)
-				binder_set_nice(target_node->min_priority);
-			cmd = BR_TRANSACTION;
-		} else {
-			trd->target.ptr = 0;
-			trd->cookie = 0;
-			cmd = BR_REPLY;
-		}
-		trd->code = t->code;
-		trd->flags = t->flags;
-		trd->sender_euid = from_kuid(current_user_ns(), t->sender_euid);
-
-		t_from = binder_get_txn_from(t);
-		if (t_from) {
-			struct task_struct *sender = t_from->proc->tsk;
-
-			trd->sender_pid =
-				task_tgid_nr_ns(sender,
-						task_active_pid_ns(current));
-		} else {
-			trd->sender_pid = 0;
-		}
-
-		ret = binder_apply_fd_fixups(proc, t);
-		if (ret) {
-			struct binder_buffer *buffer = t->buffer;
-			bool oneway = !!(t->flags & TF_ONE_WAY);
-			int tid = t->debug_id;
-
-			if (t_from)
-				binder_thread_dec_tmpref(t_from);
-			buffer->transaction = NULL;
-			binder_cleanup_transaction(t, "fd fixups failed",
-						   BR_FAILED_REPLY);
-			binder_free_buf(proc, thread, buffer, true);
-			binder_debug(BINDER_DEBUG_FAILED_TRANSACTION,
-				     "%d:%d %stransaction %d fd fixups failed %d/%d, line %d\n",
-				     proc->pid, thread->pid,
-				     oneway ? "async " :
-					(cmd == BR_REPLY ? "reply " : ""),
-				     tid, BR_FAILED_REPLY, ret, __LINE__);
-			if (cmd == BR_REPLY) {
-				cmd = BR_FAILED_REPLY;
-				if (put_user(cmd, (uint32_t __user *)ptr))
-					return -EFAULT;
-				ptr += sizeof(uint32_t);
-				binder_stat_br(proc, thread, cmd);
-				break;
-			}
-			continue;
-		}
-		trd->data_size = t->buffer->data_size;
-		trd->offsets_size = t->buffer->offsets_size;
-		trd->data.ptr.buffer = t->buffer->user_data;
-		trd->data.ptr.offsets = trd->data.ptr.buffer +
-					ALIGN(t->buffer->data_size,
-					    sizeof(void *));
-
-		tr.secctx = t->security_ctx;
-		if (t->security_ctx) {
-			cmd = BR_TRANSACTION_SEC_CTX;
-			trsize = sizeof(tr);
-		}
-		if (put_user(cmd, (uint32_t __user *)ptr)) {
-			if (t_from)
-				binder_thread_dec_tmpref(t_from);
-
-			binder_cleanup_transaction(t, "put_user failed",
-						   BR_FAILED_REPLY);
-
-			return -EFAULT;
-		}
-		ptr += sizeof(uint32_t);
-		if (copy_to_user(ptr, &tr, trsize)) {
-			if (t_from)
-				binder_thread_dec_tmpref(t_from);
-
-			binder_cleanup_transaction(t, "copy_to_user failed",
-						   BR_FAILED_REPLY);
-
-			return -EFAULT;
-		}
-		ptr += trsize;
-
-		trace_binder_transaction_received(t);
-		binder_stat_br(proc, thread, cmd);
-		binder_debug(BINDER_DEBUG_TRANSACTION,
-			     "%d:%d %s %d %d:%d, cmd %u size %zd-%zd\n",
-			     proc->pid, thread->pid,
-			     (cmd == BR_TRANSACTION) ? "BR_TRANSACTION" :
-				(cmd == BR_TRANSACTION_SEC_CTX) ?
-				     "BR_TRANSACTION_SEC_CTX" : "BR_REPLY",
-			     t->debug_id, t_from ? t_from->proc->pid : 0,
-			     t_from ? t_from->pid : 0, cmd,
-			     t->buffer->data_size, t->buffer->offsets_size);
-
-		if (t_from)
-			binder_thread_dec_tmpref(t_from);
-		t->buffer->allow_user_free = 1;
-		if (cmd != BR_REPLY && !(t->flags & TF_ONE_WAY)) {
-			binder_inner_proc_lock(thread->proc);
-			t->to_parent = thread->transaction_stack;
-			t->to_thread = thread;
-			thread->transaction_stack = t;
-			binder_inner_proc_unlock(thread->proc);
-		} else {
-			binder_free_transaction(t);
-		}
-		break;
-	}
-
-done:
-
-	*consumed = ptr - buffer;
-	binder_inner_proc_lock(proc);
-	if (proc->requested_threads == 0 &&
-	    list_empty(&thread->proc->waiting_threads) &&
-	    proc->requested_threads_started < proc->max_threads &&
-	    (thread->looper & (BINDER_LOOPER_STATE_REGISTERED |
-	     BINDER_LOOPER_STATE_ENTERED)) /* the user-space code fails to */
-	     /*spawn a new thread if we leave this out */) {
-		proc->requested_threads++;
-		binder_inner_proc_unlock(proc);
-		binder_debug(BINDER_DEBUG_THREADS,
-			     "%d:%d BR_SPAWN_LOOPER\n",
-			     proc->pid, thread->pid);
-		if (put_user(BR_SPAWN_LOOPER, (uint32_t __user *)buffer))
-			return -EFAULT;
-		binder_stat_br(proc, thread, BR_SPAWN_LOOPER);
-	} else
-		binder_inner_proc_unlock(proc);
-	return 0;
-}
-
-static void binder_release_work(struct binder_proc *proc,
-				struct list_head *list)
-{
-	struct binder_work *w;
-	enum binder_work_type wtype;
-
-	while (1) {
-		binder_inner_proc_lock(proc);
-		w = binder_dequeue_work_head_ilocked(list);
-		wtype = w ? w->type : 0;
-		binder_inner_proc_unlock(proc);
-		if (!w)
-			return;
-
-		switch (wtype) {
-		case BINDER_WORK_TRANSACTION: {
-			struct binder_transaction *t;
-
-			t = container_of(w, struct binder_transaction, work);
-
-			binder_cleanup_transaction(t, "process died.",
-						   BR_DEAD_REPLY);
-		} break;
-		case BINDER_WORK_RETURN_ERROR: {
-			struct binder_error *e = container_of(
-					w, struct binder_error, work);
-
-			binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
-				"undelivered TRANSACTION_ERROR: %u\n",
-				e->cmd);
-		} break;
-		case BINDER_WORK_TRANSACTION_PENDING:
-		case BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT:
-		case BINDER_WORK_TRANSACTION_COMPLETE: {
-			binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
-				"undelivered TRANSACTION_COMPLETE\n");
-			kfree(w);
-			binder_stats_deleted(BINDER_STAT_TRANSACTION_COMPLETE);
-		} break;
-		case BINDER_WORK_DEAD_BINDER_AND_CLEAR:
-		case BINDER_WORK_CLEAR_DEATH_NOTIFICATION: {
-			struct binder_ref_death *death;
-
-			death = container_of(w, struct binder_ref_death, work);
-			binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
-				"undelivered death notification, %016llx\n",
-				(u64)death->cookie);
-			kfree(death);
-			binder_stats_deleted(BINDER_STAT_DEATH);
-		} break;
-		case BINDER_WORK_NODE:
-			break;
-		case BINDER_WORK_CLEAR_FREEZE_NOTIFICATION: {
-			struct binder_ref_freeze *freeze;
-
-			freeze = container_of(w, struct binder_ref_freeze, work);
-			binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
-				     "undelivered freeze notification, %016llx\n",
-				     (u64)freeze->cookie);
-			kfree(freeze);
-			binder_stats_deleted(BINDER_STAT_FREEZE);
-		} break;
-		default:
-			pr_err("unexpected work type, %d, not freed\n",
-			       wtype);
-			break;
-		}
-	}
-
-}
-
-static struct binder_thread *binder_get_thread_ilocked(
-		struct binder_proc *proc, struct binder_thread *new_thread)
-{
-	struct binder_thread *thread = NULL;
-	struct rb_node *parent = NULL;
-	struct rb_node **p = &proc->threads.rb_node;
-
-	while (*p) {
-		parent = *p;
-		thread = rb_entry(parent, struct binder_thread, rb_node);
-
-		if (current->pid < thread->pid)
-			p = &(*p)->rb_left;
-		else if (current->pid > thread->pid)
-			p = &(*p)->rb_right;
-		else
-			return thread;
-	}
-	if (!new_thread)
-		return NULL;
-	thread = new_thread;
-	binder_stats_created(BINDER_STAT_THREAD);
-	thread->proc = proc;
-	thread->pid = current->pid;
-	atomic_set(&thread->tmp_ref, 0);
-	init_waitqueue_head(&thread->wait);
-	INIT_LIST_HEAD(&thread->todo);
-	rb_link_node(&thread->rb_node, parent, p);
-	rb_insert_color(&thread->rb_node, &proc->threads);
-	thread->looper_need_return = true;
-	thread->return_error.work.type = BINDER_WORK_RETURN_ERROR;
-	thread->return_error.cmd = BR_OK;
-	thread->reply_error.work.type = BINDER_WORK_RETURN_ERROR;
-	thread->reply_error.cmd = BR_OK;
-	thread->ee.command = BR_OK;
-	INIT_LIST_HEAD(&new_thread->waiting_thread_node);
-	return thread;
-}
-
-static struct binder_thread *binder_get_thread(struct binder_proc *proc)
-{
-	struct binder_thread *thread;
-	struct binder_thread *new_thread;
-
-	binder_inner_proc_lock(proc);
-	thread = binder_get_thread_ilocked(proc, NULL);
-	binder_inner_proc_unlock(proc);
-	if (!thread) {
-		new_thread = kzalloc_obj(*thread);
-		if (new_thread == NULL)
-			return NULL;
-		binder_inner_proc_lock(proc);
-		thread = binder_get_thread_ilocked(proc, new_thread);
-		binder_inner_proc_unlock(proc);
-		if (thread != new_thread)
-			kfree(new_thread);
-	}
-	return thread;
-}
-
-static void binder_free_proc(struct binder_proc *proc)
-{
-	struct binder_device *device;
-
-	BUG_ON(!list_empty(&proc->todo));
-	BUG_ON(!list_empty(&proc->delivered_death));
-	if (proc->outstanding_txns)
-		pr_warn("%s: Unexpected outstanding_txns %d\n",
-			__func__, proc->outstanding_txns);
-	device = container_of(proc->context, struct binder_device, context);
-	if (refcount_dec_and_test(&device->ref)) {
-		binder_remove_device(device);
-		kfree(proc->context->name);
-		kfree(device);
-	}
-	binder_alloc_deferred_release(&proc->alloc);
-	put_task_struct(proc->tsk);
-	put_cred(proc->cred);
-	binder_stats_deleted(BINDER_STAT_PROC);
-	dbitmap_free(&proc->dmap);
-	kfree(proc);
-}
-
-static void binder_free_thread(struct binder_thread *thread)
-{
-	BUG_ON(!list_empty(&thread->todo));
-	binder_stats_deleted(BINDER_STAT_THREAD);
-	binder_proc_dec_tmpref(thread->proc);
-	kfree(thread);
-}
-
-static int binder_thread_release(struct binder_proc *proc,
-				 struct binder_thread *thread)
-{
-	struct binder_transaction *t;
-	struct binder_transaction *send_reply = NULL;
-	int active_transactions = 0;
-	struct binder_transaction *last_t = NULL;
-
-	binder_inner_proc_lock(thread->proc);
-	/*
-	 * take a ref on the proc so it survives
-	 * after we remove this thread from proc->threads.
-	 * The corresponding dec is when we actually
-	 * free the thread in binder_free_thread()
-	 */
-	proc->tmp_ref++;
-	/*
-	 * take a ref on this thread to ensure it
-	 * survives while we are releasing it
-	 */
-	atomic_inc(&thread->tmp_ref);
-	rb_erase(&thread->rb_node, &proc->threads);
-	t = thread->transaction_stack;
-	if (t) {
-		spin_lock(&t->lock);
-		if (t->to_thread == thread)
-			send_reply = t;
-	} else {
-		__acquire(&t->lock);
-	}
-	thread->is_dead = true;
-
-	while (t) {
-		last_t = t;
-		active_transactions++;
-		binder_debug(BINDER_DEBUG_DEAD_TRANSACTION,
-			     "release %d:%d transaction %d %s, still active\n",
-			      proc->pid, thread->pid,
-			     t->debug_id,
-			     (t->to_thread == thread) ? "in" : "out");
-
-		if (t->to_thread == thread) {
-			thread->proc->outstanding_txns--;
-			t->to_proc = NULL;
-			t->to_thread = NULL;
-			if (t->buffer) {
-				t->buffer->transaction = NULL;
-				t->buffer = NULL;
-			}
-			t = t->to_parent;
-		} else if (t->from == thread) {
-			t->from = NULL;
-			t = t->from_parent;
-		} else
-			BUG();
-		spin_unlock(&last_t->lock);
-		if (t)
-			spin_lock(&t->lock);
-		else
-			__acquire(&t->lock);
-	}
-	/* annotation for sparse, lock not acquired in last iteration above */
-	__release(&t->lock);
-
-	/*
-	 * If this thread used poll, make sure we remove the waitqueue from any
-	 * poll data structures holding it.
-	 */
-	if (thread->looper & BINDER_LOOPER_STATE_POLL)
-		wake_up_pollfree(&thread->wait);
-
-	binder_inner_proc_unlock(thread->proc);
-
-	/*
-	 * This is needed to avoid races between wake_up_pollfree() above and
-	 * someone else removing the last entry from the queue for other reasons
-	 * (e.g. ep_remove_wait_queue() being called due to an epoll file
-	 * descriptor being closed).  Such other users hold an RCU read lock, so
-	 * we can be sure they're done after we call synchronize_rcu().
-	 */
-	if (thread->looper & BINDER_LOOPER_STATE_POLL)
-		synchronize_rcu();
-
-	if (send_reply)
-		binder_send_failed_reply(send_reply, BR_DEAD_REPLY);
-	binder_release_work(proc, &thread->todo);
-	binder_thread_dec_tmpref(thread);
-	return active_transactions;
-}
-
-static __poll_t binder_poll(struct file *filp,
-				struct poll_table_struct *wait)
-{
-	struct binder_proc *proc = filp->private_data;
-	struct binder_thread *thread = NULL;
-	bool wait_for_proc_work;
-
-	thread = binder_get_thread(proc);
-	if (!thread)
-		return EPOLLERR;
-
-	binder_inner_proc_lock(thread->proc);
-	thread->looper |= BINDER_LOOPER_STATE_POLL;
-	wait_for_proc_work = binder_available_for_proc_work_ilocked(thread);
-
-	binder_inner_proc_unlock(thread->proc);
-
-	poll_wait(filp, &thread->wait, wait);
-
-	if (binder_has_work(thread, wait_for_proc_work))
-		return EPOLLIN;
-
-	return 0;
-}
-
-static int binder_ioctl_write_read(struct file *filp, unsigned long arg,
-				struct binder_thread *thread)
-{
-	int ret = 0;
-	struct binder_proc *proc = filp->private_data;
-	void __user *ubuf = (void __user *)arg;
-	struct binder_write_read bwr;
-
-	if (copy_from_user(&bwr, ubuf, sizeof(bwr)))
-		return -EFAULT;
-
-	binder_debug(BINDER_DEBUG_READ_WRITE,
-		     "%d:%d write %lld at %016llx, read %lld at %016llx\n",
-		     proc->pid, thread->pid,
-		     (u64)bwr.write_size, (u64)bwr.write_buffer,
-		     (u64)bwr.read_size, (u64)bwr.read_buffer);
-
-	if (bwr.write_size > 0) {
-		ret = binder_thread_write(proc, thread,
-					  bwr.write_buffer,
-					  bwr.write_size,
-					  &bwr.write_consumed);
-		trace_binder_write_done(ret);
-		if (ret < 0) {
-			bwr.read_consumed = 0;
-			goto out;
-		}
-	}
-	if (bwr.read_size > 0) {
-		ret = binder_thread_read(proc, thread, bwr.read_buffer,
-					 bwr.read_size,
-					 &bwr.read_consumed,
-					 filp->f_flags & O_NONBLOCK);
-		trace_binder_read_done(ret);
-		binder_inner_proc_lock(proc);
-		if (!binder_worklist_empty_ilocked(&proc->todo))
-			binder_wakeup_proc_ilocked(proc);
-		binder_inner_proc_unlock(proc);
-		if (ret < 0)
-			goto out;
-	}
-	binder_debug(BINDER_DEBUG_READ_WRITE,
-		     "%d:%d wrote %lld of %lld, read return %lld of %lld\n",
-		     proc->pid, thread->pid,
-		     (u64)bwr.write_consumed, (u64)bwr.write_size,
-		     (u64)bwr.read_consumed, (u64)bwr.read_size);
-out:
-	if (copy_to_user(ubuf, &bwr, sizeof(bwr)))
-		ret = -EFAULT;
-	return ret;
-}
-
-static int binder_ioctl_set_ctx_mgr(struct file *filp,
-				    struct flat_binder_object *fbo)
-{
-	int ret = 0;
-	struct binder_proc *proc = filp->private_data;
-	struct binder_context *context = proc->context;
-	struct binder_node *new_node;
-	kuid_t curr_euid = current_euid();
-
-	guard(mutex)(&context->context_mgr_node_lock);
-	if (context->binder_context_mgr_node) {
-		pr_err("BINDER_SET_CONTEXT_MGR already set\n");
-		return -EBUSY;
-	}
-	ret = security_binder_set_context_mgr(proc->cred);
-	if (ret < 0)
-		return ret;
-	if (uid_valid(context->binder_context_mgr_uid)) {
-		if (!uid_eq(context->binder_context_mgr_uid, curr_euid)) {
-			pr_err("BINDER_SET_CONTEXT_MGR bad uid %d != %d\n",
-			       from_kuid(&init_user_ns, curr_euid),
-			       from_kuid(&init_user_ns,
-					 context->binder_context_mgr_uid));
-			return -EPERM;
-		}
-	} else {
-		context->binder_context_mgr_uid = curr_euid;
-	}
-	new_node = binder_new_node(proc, fbo);
-	if (!new_node)
-		return -ENOMEM;
-	binder_node_lock(new_node);
-	new_node->local_weak_refs++;
-	new_node->local_strong_refs++;
-	new_node->has_strong_ref = 1;
-	new_node->has_weak_ref = 1;
-	context->binder_context_mgr_node = new_node;
-	binder_node_unlock(new_node);
-	binder_put_node(new_node);
-	return ret;
-}
-
-static int binder_ioctl_get_node_info_for_ref(struct binder_proc *proc,
-		struct binder_node_info_for_ref *info)
-{
-	struct binder_node *node;
-	struct binder_context *context = proc->context;
-	__u32 handle = info->handle;
-
-	if (info->strong_count || info->weak_count || info->reserved1 ||
-	    info->reserved2 || info->reserved3) {
-		binder_user_error("%d BINDER_GET_NODE_INFO_FOR_REF: only handle may be non-zero.",
-				  proc->pid);
-		return -EINVAL;
-	}
-
-	/* This ioctl may only be used by the context manager */
-	mutex_lock(&context->context_mgr_node_lock);
-	if (!context->binder_context_mgr_node ||
-		context->binder_context_mgr_node->proc != proc) {
-		mutex_unlock(&context->context_mgr_node_lock);
-		return -EPERM;
-	}
-	mutex_unlock(&context->context_mgr_node_lock);
-
-	node = binder_get_node_from_ref(proc, handle, true, NULL);
-	if (!node)
-		return -EINVAL;
-
-	info->strong_count = node->local_strong_refs +
-		node->internal_strong_refs;
-	info->weak_count = node->local_weak_refs;
-
-	binder_put_node(node);
-
-	return 0;
-}
-
-static int binder_ioctl_get_node_debug_info(struct binder_proc *proc,
-				struct binder_node_debug_info *info)
-{
-	struct rb_node *n;
-	binder_uintptr_t ptr = info->ptr;
-
-	memset(info, 0, sizeof(*info));
-
-	binder_inner_proc_lock(proc);
-	for (n = rb_first(&proc->nodes); n != NULL; n = rb_next(n)) {
-		struct binder_node *node = rb_entry(n, struct binder_node,
-						    rb_node);
-		if (node->ptr > ptr) {
-			info->ptr = node->ptr;
-			info->cookie = node->cookie;
-			info->has_strong_ref = node->has_strong_ref;
-			info->has_weak_ref = node->has_weak_ref;
-			break;
-		}
-	}
-	binder_inner_proc_unlock(proc);
-
-	return 0;
-}
-
-static bool binder_txns_pending_ilocked(struct binder_proc *proc)
-{
-	struct rb_node *n;
-	struct binder_thread *thread;
-
-	if (proc->outstanding_txns > 0)
-		return true;
-
-	for (n = rb_first(&proc->threads); n; n = rb_next(n)) {
-		thread = rb_entry(n, struct binder_thread, rb_node);
-		if (thread->transaction_stack)
-			return true;
-	}
-	return false;
-}
-
-static void binder_add_freeze_work(struct binder_proc *proc, bool is_frozen)
-{
-	struct binder_node *prev = NULL;
-	struct rb_node *n;
-	struct binder_ref *ref;
-
-	binder_inner_proc_lock(proc);
-	for (n = rb_first(&proc->nodes); n; n = rb_next(n)) {
-		struct binder_node *node;
-
-		node = rb_entry(n, struct binder_node, rb_node);
-		binder_inc_node_tmpref_ilocked(node);
-		binder_inner_proc_unlock(proc);
-		if (prev)
-			binder_put_node(prev);
-		binder_node_lock(node);
-		hlist_for_each_entry(ref, &node->refs, node_entry) {
-			/*
-			 * Need the node lock to synchronize
-			 * with new notification requests and the
-			 * inner lock to synchronize with queued
-			 * freeze notifications.
-			 */
-			binder_inner_proc_lock(ref->proc);
-			if (!ref->freeze) {
-				binder_inner_proc_unlock(ref->proc);
-				continue;
-			}
-			ref->freeze->work.type = BINDER_WORK_FROZEN_BINDER;
-			if (list_empty(&ref->freeze->work.entry)) {
-				ref->freeze->is_frozen = is_frozen;
-				binder_enqueue_work_ilocked(&ref->freeze->work, &ref->proc->todo);
-				binder_wakeup_proc_ilocked(ref->proc);
-			} else {
-				if (ref->freeze->sent && ref->freeze->is_frozen != is_frozen)
-					ref->freeze->resend = true;
-				ref->freeze->is_frozen = is_frozen;
-			}
-			binder_inner_proc_unlock(ref->proc);
-		}
-		prev = node;
-		binder_node_unlock(node);
-		binder_inner_proc_lock(proc);
-		if (proc->is_dead)
-			break;
-	}
-	binder_inner_proc_unlock(proc);
-	if (prev)
-		binder_put_node(prev);
-}
-
-static int binder_ioctl_freeze(struct binder_freeze_info *info,
-			       struct binder_proc *target_proc)
-{
-	int ret = 0;
-
-	if (!info->enable) {
-		binder_inner_proc_lock(target_proc);
-		target_proc->sync_recv = false;
-		target_proc->async_recv = false;
-		target_proc->is_frozen = false;
-		binder_inner_proc_unlock(target_proc);
-		binder_add_freeze_work(target_proc, false);
-		return 0;
-	}
-
-	/*
-	 * Freezing the target. Prevent new transactions by
-	 * setting frozen state. If timeout specified, wait
-	 * for transactions to drain.
-	 */
-	binder_inner_proc_lock(target_proc);
-	target_proc->sync_recv = false;
-	target_proc->async_recv = false;
-	target_proc->is_frozen = true;
-	binder_inner_proc_unlock(target_proc);
-
-	if (info->timeout_ms > 0)
-		ret = wait_event_interruptible_timeout(
-			target_proc->freeze_wait,
-			(!target_proc->outstanding_txns),
-			msecs_to_jiffies(info->timeout_ms));
-
-	/* Check pending transactions that wait for reply */
-	if (ret >= 0) {
-		binder_inner_proc_lock(target_proc);
-		if (binder_txns_pending_ilocked(target_proc))
-			ret = -EAGAIN;
-		binder_inner_proc_unlock(target_proc);
-	}
-
-	if (ret < 0) {
-		binder_inner_proc_lock(target_proc);
-		target_proc->is_frozen = false;
-		binder_inner_proc_unlock(target_proc);
-	} else {
-		binder_add_freeze_work(target_proc, true);
-	}
-
-	return ret;
-}
-
-static int binder_ioctl_get_freezer_info(
-				struct binder_frozen_status_info *info)
-{
-	struct binder_proc *target_proc;
-	bool found = false;
-	__u32 txns_pending;
-
-	info->sync_recv = 0;
-	info->async_recv = 0;
-
-	mutex_lock(&binder_procs_lock);
-	hlist_for_each_entry(target_proc, &binder_procs, proc_node) {
-		if (target_proc->pid == info->pid) {
-			found = true;
-			binder_inner_proc_lock(target_proc);
-			txns_pending = binder_txns_pending_ilocked(target_proc);
-			info->sync_recv |= target_proc->sync_recv |
-					(txns_pending << 1);
-			info->async_recv |= target_proc->async_recv;
-			binder_inner_proc_unlock(target_proc);
-		}
-	}
-	mutex_unlock(&binder_procs_lock);
-
-	if (!found)
-		return -EINVAL;
-
-	return 0;
-}
-
-static int binder_ioctl_get_extended_error(struct binder_thread *thread,
-					   void __user *ubuf)
-{
-	struct binder_extended_error ee;
-
-	binder_inner_proc_lock(thread->proc);
-	ee = thread->ee;
-	binder_set_extended_error(&thread->ee, 0, BR_OK, 0);
-	binder_inner_proc_unlock(thread->proc);
-
-	if (copy_to_user(ubuf, &ee, sizeof(ee)))
-		return -EFAULT;
-
-	return 0;
-}
-
-static long binder_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	int ret;
-	struct binder_proc *proc = filp->private_data;
-	struct binder_thread *thread;
-	void __user *ubuf = (void __user *)arg;
-
-	trace_binder_ioctl(cmd, arg);
-
-	ret = wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2);
-	if (ret)
-		goto err_unlocked;
-
-	thread = binder_get_thread(proc);
-	if (thread == NULL) {
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	switch (cmd) {
-	case BINDER_WRITE_READ:
-		ret = binder_ioctl_write_read(filp, arg, thread);
-		if (ret)
-			goto err;
-		break;
-	case BINDER_SET_MAX_THREADS: {
-		u32 max_threads;
-
-		if (copy_from_user(&max_threads, ubuf,
-				   sizeof(max_threads))) {
-			ret = -EINVAL;
-			goto err;
-		}
-		binder_inner_proc_lock(proc);
-		proc->max_threads = max_threads;
-		binder_inner_proc_unlock(proc);
-		break;
-	}
-	case BINDER_SET_CONTEXT_MGR_EXT: {
-		struct flat_binder_object fbo;
-
-		if (copy_from_user(&fbo, ubuf, sizeof(fbo))) {
-			ret = -EINVAL;
-			goto err;
-		}
-		ret = binder_ioctl_set_ctx_mgr(filp, &fbo);
-		if (ret)
-			goto err;
-		break;
-	}
-	case BINDER_SET_CONTEXT_MGR:
-		ret = binder_ioctl_set_ctx_mgr(filp, NULL);
-		if (ret)
-			goto err;
-		break;
-	case BINDER_THREAD_EXIT:
-		binder_debug(BINDER_DEBUG_THREADS, "%d:%d exit\n",
-			     proc->pid, thread->pid);
-		binder_thread_release(proc, thread);
-		thread = NULL;
-		break;
-	case BINDER_VERSION: {
-		struct binder_version __user *ver = ubuf;
-
-		if (put_user(BINDER_CURRENT_PROTOCOL_VERSION,
-			     &ver->protocol_version)) {
-			ret = -EINVAL;
-			goto err;
-		}
-		break;
-	}
-	case BINDER_GET_NODE_INFO_FOR_REF: {
-		struct binder_node_info_for_ref info;
-
-		if (copy_from_user(&info, ubuf, sizeof(info))) {
-			ret = -EFAULT;
-			goto err;
-		}
-
-		ret = binder_ioctl_get_node_info_for_ref(proc, &info);
-		if (ret < 0)
-			goto err;
-
-		if (copy_to_user(ubuf, &info, sizeof(info))) {
-			ret = -EFAULT;
-			goto err;
-		}
-
-		break;
-	}
-	case BINDER_GET_NODE_DEBUG_INFO: {
-		struct binder_node_debug_info info;
-
-		if (copy_from_user(&info, ubuf, sizeof(info))) {
-			ret = -EFAULT;
-			goto err;
-		}
-
-		ret = binder_ioctl_get_node_debug_info(proc, &info);
-		if (ret < 0)
-			goto err;
-
-		if (copy_to_user(ubuf, &info, sizeof(info))) {
-			ret = -EFAULT;
-			goto err;
-		}
-		break;
-	}
-	case BINDER_FREEZE: {
-		struct binder_freeze_info info;
-		struct binder_proc **target_procs = NULL, *target_proc;
-		int target_procs_count = 0, i = 0;
-
-		ret = 0;
-
-		if (copy_from_user(&info, ubuf, sizeof(info))) {
-			ret = -EFAULT;
-			goto err;
-		}
-
-		mutex_lock(&binder_procs_lock);
-		hlist_for_each_entry(target_proc, &binder_procs, proc_node) {
-			if (target_proc->pid == info.pid)
-				target_procs_count++;
-		}
-
-		if (target_procs_count == 0) {
-			mutex_unlock(&binder_procs_lock);
-			ret = -EINVAL;
-			goto err;
-		}
-
-		target_procs = kzalloc_objs(struct binder_proc *,
-					    target_procs_count);
-
-		if (!target_procs) {
-			mutex_unlock(&binder_procs_lock);
-			ret = -ENOMEM;
-			goto err;
-		}
-
-		hlist_for_each_entry(target_proc, &binder_procs, proc_node) {
-			if (target_proc->pid != info.pid)
-				continue;
-
-			binder_inner_proc_lock(target_proc);
-			target_proc->tmp_ref++;
-			binder_inner_proc_unlock(target_proc);
-
-			target_procs[i++] = target_proc;
-		}
-		mutex_unlock(&binder_procs_lock);
-
-		for (i = 0; i < target_procs_count; i++) {
-			if (ret >= 0)
-				ret = binder_ioctl_freeze(&info,
-							  target_procs[i]);
-
-			binder_proc_dec_tmpref(target_procs[i]);
-		}
-
-		kfree(target_procs);
-
-		if (ret < 0)
-			goto err;
-		break;
-	}
-	case BINDER_GET_FROZEN_INFO: {
-		struct binder_frozen_status_info info;
-
-		if (copy_from_user(&info, ubuf, sizeof(info))) {
-			ret = -EFAULT;
-			goto err;
-		}
-
-		ret = binder_ioctl_get_freezer_info(&info);
-		if (ret < 0)
-			goto err;
-
-		if (copy_to_user(ubuf, &info, sizeof(info))) {
-			ret = -EFAULT;
-			goto err;
-		}
-		break;
-	}
-	case BINDER_ENABLE_ONEWAY_SPAM_DETECTION: {
-		uint32_t enable;
-
-		if (copy_from_user(&enable, ubuf, sizeof(enable))) {
-			ret = -EFAULT;
-			goto err;
-		}
-		binder_inner_proc_lock(proc);
-		proc->oneway_spam_detection_enabled = (bool)enable;
-		binder_inner_proc_unlock(proc);
-		break;
-	}
-	case BINDER_GET_EXTENDED_ERROR:
-		ret = binder_ioctl_get_extended_error(thread, ubuf);
-		if (ret < 0)
-			goto err;
-		break;
-	default:
-		ret = -EINVAL;
-		goto err;
-	}
-	ret = 0;
-err:
-	if (thread)
-		thread->looper_need_return = false;
-	wait_event_interruptible(binder_user_error_wait, binder_stop_on_user_error < 2);
-	if (ret && ret != -EINTR)
-		pr_info("%d:%d ioctl %x %lx returned %d\n", proc->pid, current->pid, cmd, arg, ret);
-err_unlocked:
-	trace_binder_ioctl_done(ret);
-	return ret;
-}
-
-static void binder_vma_open(struct vm_area_struct *vma)
-{
-	struct binder_proc *proc = vma->vm_private_data;
-
-	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "%d open vm area %lx-%lx (%ld K) vma %lx pagep %lx\n",
-		     proc->pid, vma->vm_start, vma->vm_end,
-		     (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags,
-		     (unsigned long)pgprot_val(vma->vm_page_prot));
-}
-
-static void binder_vma_close(struct vm_area_struct *vma)
-{
-	struct binder_proc *proc = vma->vm_private_data;
-
-	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "%d close vm area %lx-%lx (%ld K) vma %lx pagep %lx\n",
-		     proc->pid, vma->vm_start, vma->vm_end,
-		     (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags,
-		     (unsigned long)pgprot_val(vma->vm_page_prot));
-	binder_alloc_vma_close(&proc->alloc);
-}
-
-VISIBLE_IF_KUNIT vm_fault_t binder_vm_fault(struct vm_fault *vmf)
-{
-	return VM_FAULT_SIGBUS;
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_vm_fault);
-
-static const struct vm_operations_struct binder_vm_ops = {
-	.open = binder_vma_open,
-	.close = binder_vma_close,
-	.fault = binder_vm_fault,
-};
-
-static int binder_mmap(struct file *filp, struct vm_area_struct *vma)
-{
-	struct binder_proc *proc = filp->private_data;
-
-	if (!same_thread_group(proc->tsk, current))
-		return -EINVAL;
-
-	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "%s: %d %lx-%lx (%ld K) vma %lx pagep %lx\n",
-		     __func__, proc->pid, vma->vm_start, vma->vm_end,
-		     (vma->vm_end - vma->vm_start) / SZ_1K, vma->vm_flags,
-		     (unsigned long)pgprot_val(vma->vm_page_prot));
-
-	if (vma->vm_flags & FORBIDDEN_MMAP_FLAGS) {
-		pr_err("%s: %d %lx-%lx %s failed %d\n", __func__,
-		       proc->pid, vma->vm_start, vma->vm_end, "bad vm_flags", -EPERM);
-		return -EPERM;
-	}
-	vm_flags_mod(vma, VM_DONTCOPY | VM_MIXEDMAP, VM_MAYWRITE);
-
-	vma->vm_ops = &binder_vm_ops;
-	vma->vm_private_data = proc;
-
-	return binder_alloc_mmap_handler(&proc->alloc, vma);
-}
-
-static int binder_open(struct inode *nodp, struct file *filp)
-{
-	struct binder_proc *proc, *itr;
-	struct binder_device *binder_dev;
-	struct binderfs_info *info;
-	struct dentry *binder_binderfs_dir_entry_proc = NULL;
-	bool existing_pid = false;
-
-	binder_debug(BINDER_DEBUG_OPEN_CLOSE, "%s: %d:%d\n", __func__,
-		     current->tgid, current->pid);
-
-	proc = kzalloc_obj(*proc);
-	if (proc == NULL)
-		return -ENOMEM;
-
-	dbitmap_init(&proc->dmap);
-	spin_lock_init(&proc->inner_lock);
-	spin_lock_init(&proc->outer_lock);
-	proc->tsk = get_task_struct(current->group_leader);
-	proc->pid = current->tgid;
-	proc->cred = get_cred(filp->f_cred);
-	INIT_LIST_HEAD(&proc->todo);
-	init_waitqueue_head(&proc->freeze_wait);
-	proc->default_priority = task_nice(current);
-	/* binderfs stashes devices in i_private */
-	if (is_binderfs_device(nodp)) {
-		binder_dev = nodp->i_private;
-		info = nodp->i_sb->s_fs_info;
-		binder_binderfs_dir_entry_proc = info->proc_log_dir;
-	} else {
-		binder_dev = container_of(filp->private_data,
-					  struct binder_device, miscdev);
-	}
-	refcount_inc(&binder_dev->ref);
-	proc->context = &binder_dev->context;
-	binder_alloc_init(&proc->alloc);
-
-	binder_stats_created(BINDER_STAT_PROC);
-	INIT_LIST_HEAD(&proc->delivered_death);
-	INIT_LIST_HEAD(&proc->delivered_freeze);
-	INIT_LIST_HEAD(&proc->waiting_threads);
-	filp->private_data = proc;
-
-	mutex_lock(&binder_procs_lock);
-	hlist_for_each_entry(itr, &binder_procs, proc_node) {
-		if (itr->pid == proc->pid) {
-			existing_pid = true;
-			break;
-		}
-	}
-	hlist_add_head(&proc->proc_node, &binder_procs);
-	mutex_unlock(&binder_procs_lock);
-
-	if (binder_debugfs_dir_entry_proc && !existing_pid) {
-		char strbuf[11];
-
-		snprintf(strbuf, sizeof(strbuf), "%u", proc->pid);
-		/*
-		 * proc debug entries are shared between contexts.
-		 * Only create for the first PID to avoid debugfs log spamming
-		 * The printing code will anyway print all contexts for a given
-		 * PID so this is not a problem.
-		 */
-		proc->debugfs_entry = debugfs_create_file(strbuf, 0444,
-			binder_debugfs_dir_entry_proc,
-			(void *)(unsigned long)proc->pid,
-			&proc_fops);
-	}
-
-	if (binder_binderfs_dir_entry_proc && !existing_pid) {
-		char strbuf[11];
-		struct dentry *binderfs_entry;
-
-		snprintf(strbuf, sizeof(strbuf), "%u", proc->pid);
-		/*
-		 * Similar to debugfs, the process specific log file is shared
-		 * between contexts. Only create for the first PID.
-		 * This is ok since same as debugfs, the log file will contain
-		 * information on all contexts of a given PID.
-		 */
-		binderfs_entry = binderfs_create_file(binder_binderfs_dir_entry_proc,
-			strbuf, &proc_fops, (void *)(unsigned long)proc->pid);
-		if (!IS_ERR(binderfs_entry)) {
-			proc->binderfs_entry = binderfs_entry;
-		} else {
-			int error;
-
-			error = PTR_ERR(binderfs_entry);
-			pr_warn("Unable to create file %s in binderfs (error %d)\n",
-				strbuf, error);
-		}
-	}
-
-	return 0;
-}
-
-static int binder_flush(struct file *filp, fl_owner_t id)
-{
-	struct binder_proc *proc = filp->private_data;
-
-	binder_defer_work(proc, BINDER_DEFERRED_FLUSH);
-
-	return 0;
-}
-
-static void binder_deferred_flush(struct binder_proc *proc)
-{
-	struct rb_node *n;
-	int wake_count = 0;
-
-	binder_inner_proc_lock(proc);
-	for (n = rb_first(&proc->threads); n != NULL; n = rb_next(n)) {
-		struct binder_thread *thread = rb_entry(n, struct binder_thread, rb_node);
-
-		thread->looper_need_return = true;
-		if (thread->looper & BINDER_LOOPER_STATE_WAITING) {
-			wake_up_interruptible(&thread->wait);
-			wake_count++;
-		}
-	}
-	binder_inner_proc_unlock(proc);
-
-	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "binder_flush: %d woke %d threads\n", proc->pid,
-		     wake_count);
-}
-
-static int binder_release(struct inode *nodp, struct file *filp)
-{
-	struct binder_proc *proc = filp->private_data;
-
-	debugfs_remove(proc->debugfs_entry);
-
-	if (proc->binderfs_entry) {
-		simple_recursive_removal(proc->binderfs_entry, NULL);
-		proc->binderfs_entry = NULL;
-	}
-
-	binder_defer_work(proc, BINDER_DEFERRED_RELEASE);
-
-	return 0;
-}
-
-static int binder_node_release(struct binder_node *node, int refs)
-{
-	struct binder_ref *ref;
-	int death = 0;
-	struct binder_proc *proc = node->proc;
-
-	binder_release_work(proc, &node->async_todo);
-
-	binder_node_lock(node);
-	binder_inner_proc_lock(proc);
-	binder_dequeue_work_ilocked(&node->work);
-	/*
-	 * The caller must have taken a temporary ref on the node,
-	 */
-	BUG_ON(!node->tmp_refs);
-	if (hlist_empty(&node->refs) && node->tmp_refs == 1) {
-		binder_inner_proc_unlock(proc);
-		binder_node_unlock(node);
-		binder_free_node(node);
-
-		return refs;
-	}
-
-	node->proc = NULL;
-	node->local_strong_refs = 0;
-	node->local_weak_refs = 0;
-	binder_inner_proc_unlock(proc);
-
-	spin_lock(&binder_dead_nodes_lock);
-	hlist_add_head(&node->dead_node, &binder_dead_nodes);
-	spin_unlock(&binder_dead_nodes_lock);
-
-	hlist_for_each_entry(ref, &node->refs, node_entry) {
-		refs++;
-		/*
-		 * Need the node lock to synchronize
-		 * with new notification requests and the
-		 * inner lock to synchronize with queued
-		 * death notifications.
-		 */
-		binder_inner_proc_lock(ref->proc);
-		if (!ref->death) {
-			binder_inner_proc_unlock(ref->proc);
-			continue;
-		}
-
-		death++;
-
-		BUG_ON(!list_empty(&ref->death->work.entry));
-		ref->death->work.type = BINDER_WORK_DEAD_BINDER;
-		binder_enqueue_work_ilocked(&ref->death->work,
-					    &ref->proc->todo);
-		binder_wakeup_proc_ilocked(ref->proc);
-		binder_inner_proc_unlock(ref->proc);
-	}
-
-	binder_debug(BINDER_DEBUG_DEAD_BINDER,
-		     "node %d now dead, refs %d, death %d\n",
-		     node->debug_id, refs, death);
-	binder_node_unlock(node);
-	binder_put_node(node);
-
-	return refs;
-}
-
-static void binder_deferred_release(struct binder_proc *proc)
-{
-	struct binder_context *context = proc->context;
-	struct rb_node *n;
-	int threads, nodes, incoming_refs, outgoing_refs, active_transactions;
-
-	mutex_lock(&binder_procs_lock);
-	hlist_del(&proc->proc_node);
-	mutex_unlock(&binder_procs_lock);
-
-	mutex_lock(&context->context_mgr_node_lock);
-	if (context->binder_context_mgr_node &&
-	    context->binder_context_mgr_node->proc == proc) {
-		binder_debug(BINDER_DEBUG_DEAD_BINDER,
-			     "%s: %d context_mgr_node gone\n",
-			     __func__, proc->pid);
-		context->binder_context_mgr_node = NULL;
-	}
-	mutex_unlock(&context->context_mgr_node_lock);
-	binder_inner_proc_lock(proc);
-	/*
-	 * Make sure proc stays alive after we
-	 * remove all the threads
-	 */
-	proc->tmp_ref++;
-
-	proc->is_dead = true;
-	proc->is_frozen = false;
-	proc->sync_recv = false;
-	proc->async_recv = false;
-	threads = 0;
-	active_transactions = 0;
-	while ((n = rb_first(&proc->threads))) {
-		struct binder_thread *thread;
-
-		thread = rb_entry(n, struct binder_thread, rb_node);
-		binder_inner_proc_unlock(proc);
-		threads++;
-		active_transactions += binder_thread_release(proc, thread);
-		binder_inner_proc_lock(proc);
-	}
-
-	nodes = 0;
-	incoming_refs = 0;
-	while ((n = rb_first(&proc->nodes))) {
-		struct binder_node *node;
-
-		node = rb_entry(n, struct binder_node, rb_node);
-		nodes++;
-		/*
-		 * take a temporary ref on the node before
-		 * calling binder_node_release() which will either
-		 * kfree() the node or call binder_put_node()
-		 */
-		binder_inc_node_tmpref_ilocked(node);
-		rb_erase(&node->rb_node, &proc->nodes);
-		binder_inner_proc_unlock(proc);
-		incoming_refs = binder_node_release(node, incoming_refs);
-		binder_inner_proc_lock(proc);
-	}
-	binder_inner_proc_unlock(proc);
-
-	outgoing_refs = 0;
-	binder_proc_lock(proc);
-	while ((n = rb_first(&proc->refs_by_desc))) {
-		struct binder_ref *ref;
-
-		ref = rb_entry(n, struct binder_ref, rb_node_desc);
-		outgoing_refs++;
-		binder_cleanup_ref_olocked(ref);
-		binder_proc_unlock(proc);
-		binder_free_ref(ref);
-		binder_proc_lock(proc);
-	}
-	binder_proc_unlock(proc);
-
-	binder_release_work(proc, &proc->todo);
-	binder_release_work(proc, &proc->delivered_death);
-	binder_release_work(proc, &proc->delivered_freeze);
-
-	binder_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "%s: %d threads %d, nodes %d (ref %d), refs %d, active transactions %d\n",
-		     __func__, proc->pid, threads, nodes, incoming_refs,
-		     outgoing_refs, active_transactions);
-
-	binder_proc_dec_tmpref(proc);
-}
-
-static void binder_deferred_func(struct work_struct *work)
-{
-	struct binder_proc *proc;
-
-	int defer;
-
-	do {
-		mutex_lock(&binder_deferred_lock);
-		if (!hlist_empty(&binder_deferred_list)) {
-			proc = hlist_entry(binder_deferred_list.first,
-					struct binder_proc, deferred_work_node);
-			hlist_del_init(&proc->deferred_work_node);
-			defer = proc->deferred_work;
-			proc->deferred_work = 0;
-		} else {
-			proc = NULL;
-			defer = 0;
-		}
-		mutex_unlock(&binder_deferred_lock);
-
-		if (defer & BINDER_DEFERRED_FLUSH)
-			binder_deferred_flush(proc);
-
-		if (defer & BINDER_DEFERRED_RELEASE)
-			binder_deferred_release(proc); /* frees proc */
-	} while (proc);
-}
-static DECLARE_WORK(binder_deferred_work, binder_deferred_func);
-
-static void
-binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer)
-{
-	guard(mutex)(&binder_deferred_lock);
-	proc->deferred_work |= defer;
-	if (hlist_unhashed(&proc->deferred_work_node)) {
-		hlist_add_head(&proc->deferred_work_node,
-				&binder_deferred_list);
-		schedule_work(&binder_deferred_work);
-	}
-}
-
-static void print_binder_transaction_ilocked(struct seq_file *m,
-					     struct binder_proc *proc,
-					     const char *prefix,
-					     struct binder_transaction *t)
-{
-	struct binder_proc *to_proc;
-	struct binder_buffer *buffer = t->buffer;
-	ktime_t current_time = ktime_get();
-
-	spin_lock(&t->lock);
-	to_proc = t->to_proc;
-	seq_printf(m,
-		   "%s %d: %pK from %d:%d to %d:%d code %x flags %x pri %ld a%d r%d elapsed %lldms",
-		   prefix, t->debug_id, t,
-		   t->from_pid,
-		   t->from_tid,
-		   to_proc ? to_proc->pid : 0,
-		   t->to_thread ? t->to_thread->pid : 0,
-		   t->code, t->flags, t->priority, t->is_async, t->is_reply,
-		   ktime_ms_delta(current_time, t->start_time));
-	spin_unlock(&t->lock);
-
-	if (proc != to_proc) {
-		/*
-		 * Can only safely deref buffer if we are holding the
-		 * correct proc inner lock for this node
-		 */
-		seq_puts(m, "\n");
-		return;
-	}
-
-	if (buffer == NULL) {
-		seq_puts(m, " buffer free\n");
-		return;
-	}
-	if (buffer->target_node)
-		seq_printf(m, " node %d", buffer->target_node->debug_id);
-	seq_printf(m, " size %zd:%zd offset %lx\n",
-		   buffer->data_size, buffer->offsets_size,
-		   buffer->user_data - proc->alloc.vm_start);
-}
-
-static void print_binder_work_ilocked(struct seq_file *m,
-				      struct binder_proc *proc,
-				      const char *prefix,
-				      const char *transaction_prefix,
-				      struct binder_work *w, bool hash_ptrs)
-{
-	struct binder_node *node;
-	struct binder_transaction *t;
-
-	switch (w->type) {
-	case BINDER_WORK_TRANSACTION:
-		t = container_of(w, struct binder_transaction, work);
-		print_binder_transaction_ilocked(
-				m, proc, transaction_prefix, t);
-		break;
-	case BINDER_WORK_RETURN_ERROR: {
-		struct binder_error *e = container_of(
-				w, struct binder_error, work);
-
-		seq_printf(m, "%stransaction error: %u\n",
-			   prefix, e->cmd);
-	} break;
-	case BINDER_WORK_TRANSACTION_COMPLETE:
-		seq_printf(m, "%stransaction complete\n", prefix);
-		break;
-	case BINDER_WORK_NODE:
-		node = container_of(w, struct binder_node, work);
-		if (hash_ptrs)
-			seq_printf(m, "%snode work %d: u%p c%p\n",
-				   prefix, node->debug_id,
-				   (void *)(long)node->ptr,
-				   (void *)(long)node->cookie);
-		else
-			seq_printf(m, "%snode work %d: u%016llx c%016llx\n",
-				   prefix, node->debug_id,
-				   (u64)node->ptr, (u64)node->cookie);
-		break;
-	case BINDER_WORK_DEAD_BINDER:
-		seq_printf(m, "%shas dead binder\n", prefix);
-		break;
-	case BINDER_WORK_DEAD_BINDER_AND_CLEAR:
-		seq_printf(m, "%shas cleared dead binder\n", prefix);
-		break;
-	case BINDER_WORK_CLEAR_DEATH_NOTIFICATION:
-		seq_printf(m, "%shas cleared death notification\n", prefix);
-		break;
-	case BINDER_WORK_FROZEN_BINDER:
-		seq_printf(m, "%shas frozen binder\n", prefix);
-		break;
-	case BINDER_WORK_CLEAR_FREEZE_NOTIFICATION:
-		seq_printf(m, "%shas cleared freeze notification\n", prefix);
-		break;
-	default:
-		seq_printf(m, "%sunknown work: type %d\n", prefix, w->type);
-		break;
-	}
-}
-
-static void print_binder_thread_ilocked(struct seq_file *m,
-					struct binder_thread *thread,
-					bool print_always, bool hash_ptrs)
-{
-	struct binder_transaction *t;
-	struct binder_work *w;
-	size_t start_pos = m->count;
-	size_t header_pos;
-
-	seq_printf(m, "  thread %d: l %02x need_return %d tr %d\n",
-			thread->pid, thread->looper,
-			thread->looper_need_return,
-			atomic_read(&thread->tmp_ref));
-	header_pos = m->count;
-	t = thread->transaction_stack;
-	while (t) {
-		if (t->from == thread) {
-			print_binder_transaction_ilocked(m, thread->proc,
-					"    outgoing transaction", t);
-			t = t->from_parent;
-		} else if (t->to_thread == thread) {
-			print_binder_transaction_ilocked(m, thread->proc,
-						 "    incoming transaction", t);
-			t = t->to_parent;
-		} else {
-			print_binder_transaction_ilocked(m, thread->proc,
-					"    bad transaction", t);
-			t = NULL;
-		}
-	}
-	list_for_each_entry(w, &thread->todo, entry) {
-		print_binder_work_ilocked(m, thread->proc, "    ",
-					  "    pending transaction",
-					  w, hash_ptrs);
-	}
-	if (!print_always && m->count == header_pos)
-		m->count = start_pos;
-}
-
-static void print_binder_node_nilocked(struct seq_file *m,
-				       struct binder_node *node,
-				       bool hash_ptrs)
-{
-	struct binder_ref *ref;
-	struct binder_work *w;
-	int count;
-
-	count = hlist_count_nodes(&node->refs);
-
-	if (hash_ptrs)
-		seq_printf(m, "  node %d: u%p c%p", node->debug_id,
-			   (void *)(long)node->ptr, (void *)(long)node->cookie);
-	else
-		seq_printf(m, "  node %d: u%016llx c%016llx", node->debug_id,
-			   (u64)node->ptr, (u64)node->cookie);
-	seq_printf(m, " hs %d hw %d ls %d lw %d is %d iw %d tr %d",
-		   node->has_strong_ref, node->has_weak_ref,
-		   node->local_strong_refs, node->local_weak_refs,
-		   node->internal_strong_refs, count, node->tmp_refs);
-	if (count) {
-		seq_puts(m, " proc");
-		hlist_for_each_entry(ref, &node->refs, node_entry)
-			seq_printf(m, " %d", ref->proc->pid);
-	}
-	seq_puts(m, "\n");
-	if (node->proc) {
-		list_for_each_entry(w, &node->async_todo, entry)
-			print_binder_work_ilocked(m, node->proc, "    ",
-					  "    pending async transaction",
-					  w, hash_ptrs);
-	}
-}
-
-static void print_binder_ref_olocked(struct seq_file *m,
-				     struct binder_ref *ref)
-{
-	binder_node_lock(ref->node);
-	seq_printf(m, "  ref %d: desc %d %snode %d s %d w %d d %pK\n",
-		   ref->data.debug_id, ref->data.desc,
-		   ref->node->proc ? "" : "dead ",
-		   ref->node->debug_id, ref->data.strong,
-		   ref->data.weak, ref->death);
-	binder_node_unlock(ref->node);
-}
-
-/**
- * print_next_binder_node_ilocked() - Print binder_node from a locked list
- * @m:          struct seq_file for output via seq_printf()
- * @proc:       struct binder_proc we hold the inner_proc_lock to (if any)
- * @node:       struct binder_node to print fields of
- * @prev_node:	struct binder_node we hold a temporary reference to (if any)
- * @hash_ptrs:  whether to hash @node's binder_uintptr_t fields
- *
- * Helper function to handle synchronization around printing a struct
- * binder_node while iterating through @proc->nodes or the dead nodes list.
- * Caller must hold either @proc->inner_lock (for live nodes) or
- * binder_dead_nodes_lock. This lock will be released during the body of this
- * function, but it will be reacquired before returning to the caller.
- *
- * Return:	pointer to the struct binder_node we hold a tmpref on
- */
-static struct binder_node *
-print_next_binder_node_ilocked(struct seq_file *m, struct binder_proc *proc,
-			       struct binder_node *node,
-			       struct binder_node *prev_node, bool hash_ptrs)
-{
-	/*
-	 * Take a temporary reference on the node so that isn't freed while
-	 * we print it.
-	 */
-	binder_inc_node_tmpref_ilocked(node);
-	/*
-	 * Live nodes need to drop the inner proc lock and dead nodes need to
-	 * drop the binder_dead_nodes_lock before trying to take the node lock.
-	 */
-	if (proc)
-		binder_inner_proc_unlock(proc);
-	else
-		spin_unlock(&binder_dead_nodes_lock);
-	if (prev_node)
-		binder_put_node(prev_node);
-	binder_node_inner_lock(node);
-	print_binder_node_nilocked(m, node, hash_ptrs);
-	binder_node_inner_unlock(node);
-	if (proc)
-		binder_inner_proc_lock(proc);
-	else
-		spin_lock(&binder_dead_nodes_lock);
-	return node;
-}
-
-static void print_binder_proc(struct seq_file *m, struct binder_proc *proc,
-			      bool print_all, bool hash_ptrs)
-{
-	struct binder_work *w;
-	struct rb_node *n;
-	size_t start_pos = m->count;
-	size_t header_pos;
-	struct binder_node *last_node = NULL;
-
-	seq_printf(m, "proc %d\n", proc->pid);
-	seq_printf(m, "context %s\n", proc->context->name);
-	header_pos = m->count;
-
-	binder_inner_proc_lock(proc);
-	for (n = rb_first(&proc->threads); n; n = rb_next(n))
-		print_binder_thread_ilocked(m, rb_entry(n, struct binder_thread,
-						rb_node), print_all, hash_ptrs);
-
-	for (n = rb_first(&proc->nodes); n; n = rb_next(n)) {
-		struct binder_node *node = rb_entry(n, struct binder_node,
-						    rb_node);
-		if (!print_all && !node->has_async_transaction)
-			continue;
-
-		last_node = print_next_binder_node_ilocked(m, proc, node,
-							   last_node,
-							   hash_ptrs);
-	}
-	binder_inner_proc_unlock(proc);
-	if (last_node)
-		binder_put_node(last_node);
-
-	if (print_all) {
-		binder_proc_lock(proc);
-		for (n = rb_first(&proc->refs_by_desc); n; n = rb_next(n))
-			print_binder_ref_olocked(m, rb_entry(n,
-							     struct binder_ref,
-							     rb_node_desc));
-		binder_proc_unlock(proc);
-	}
-	binder_alloc_print_allocated(m, &proc->alloc);
-	binder_inner_proc_lock(proc);
-	list_for_each_entry(w, &proc->todo, entry)
-		print_binder_work_ilocked(m, proc, "  ",
-					  "  pending transaction", w,
-					  hash_ptrs);
-	list_for_each_entry(w, &proc->delivered_death, entry) {
-		seq_puts(m, "  has delivered dead binder\n");
-		break;
-	}
-	list_for_each_entry(w, &proc->delivered_freeze, entry) {
-		seq_puts(m, "  has delivered freeze binder\n");
-		break;
-	}
-	binder_inner_proc_unlock(proc);
-	if (!print_all && m->count == header_pos)
-		m->count = start_pos;
-}
-
-static const char * const binder_return_strings[] = {
-	"BR_ERROR",
-	"BR_OK",
-	"BR_TRANSACTION",
-	"BR_REPLY",
-	"BR_ACQUIRE_RESULT",
-	"BR_DEAD_REPLY",
-	"BR_TRANSACTION_COMPLETE",
-	"BR_INCREFS",
-	"BR_ACQUIRE",
-	"BR_RELEASE",
-	"BR_DECREFS",
-	"BR_ATTEMPT_ACQUIRE",
-	"BR_NOOP",
-	"BR_SPAWN_LOOPER",
-	"BR_FINISHED",
-	"BR_DEAD_BINDER",
-	"BR_CLEAR_DEATH_NOTIFICATION_DONE",
-	"BR_FAILED_REPLY",
-	"BR_FROZEN_REPLY",
-	"BR_ONEWAY_SPAM_SUSPECT",
-	"BR_TRANSACTION_PENDING_FROZEN",
-	"BR_FROZEN_BINDER",
-	"BR_CLEAR_FREEZE_NOTIFICATION_DONE",
-};
-
-static const char * const binder_command_strings[] = {
-	"BC_TRANSACTION",
-	"BC_REPLY",
-	"BC_ACQUIRE_RESULT",
-	"BC_FREE_BUFFER",
-	"BC_INCREFS",
-	"BC_ACQUIRE",
-	"BC_RELEASE",
-	"BC_DECREFS",
-	"BC_INCREFS_DONE",
-	"BC_ACQUIRE_DONE",
-	"BC_ATTEMPT_ACQUIRE",
-	"BC_REGISTER_LOOPER",
-	"BC_ENTER_LOOPER",
-	"BC_EXIT_LOOPER",
-	"BC_REQUEST_DEATH_NOTIFICATION",
-	"BC_CLEAR_DEATH_NOTIFICATION",
-	"BC_DEAD_BINDER_DONE",
-	"BC_TRANSACTION_SG",
-	"BC_REPLY_SG",
-	"BC_REQUEST_FREEZE_NOTIFICATION",
-	"BC_CLEAR_FREEZE_NOTIFICATION",
-	"BC_FREEZE_NOTIFICATION_DONE",
-};
-
-static const char * const binder_objstat_strings[] = {
-	"proc",
-	"thread",
-	"node",
-	"ref",
-	"death",
-	"transaction",
-	"transaction_complete",
-	"freeze",
-};
-
-static void print_binder_stats(struct seq_file *m, const char *prefix,
-			       struct binder_stats *stats)
-{
-	int i;
-
-	BUILD_BUG_ON(ARRAY_SIZE(stats->bc) !=
-		     ARRAY_SIZE(binder_command_strings));
-	for (i = 0; i < ARRAY_SIZE(stats->bc); i++) {
-		int temp = atomic_read(&stats->bc[i]);
-
-		if (temp)
-			seq_printf(m, "%s%s: %d\n", prefix,
-				   binder_command_strings[i], temp);
-	}
-
-	BUILD_BUG_ON(ARRAY_SIZE(stats->br) !=
-		     ARRAY_SIZE(binder_return_strings));
-	for (i = 0; i < ARRAY_SIZE(stats->br); i++) {
-		int temp = atomic_read(&stats->br[i]);
-
-		if (temp)
-			seq_printf(m, "%s%s: %d\n", prefix,
-				   binder_return_strings[i], temp);
-	}
-
-	BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) !=
-		     ARRAY_SIZE(binder_objstat_strings));
-	BUILD_BUG_ON(ARRAY_SIZE(stats->obj_created) !=
-		     ARRAY_SIZE(stats->obj_deleted));
-	for (i = 0; i < ARRAY_SIZE(stats->obj_created); i++) {
-		int created = atomic_read(&stats->obj_created[i]);
-		int deleted = atomic_read(&stats->obj_deleted[i]);
-
-		if (created || deleted)
-			seq_printf(m, "%s%s: active %d total %d\n",
-				prefix,
-				binder_objstat_strings[i],
-				created - deleted,
-				created);
-	}
-}
-
-static void print_binder_proc_stats(struct seq_file *m,
-				    struct binder_proc *proc)
-{
-	struct binder_work *w;
-	struct binder_thread *thread;
-	struct rb_node *n;
-	int count, strong, weak, ready_threads;
-	size_t free_async_space =
-		binder_alloc_get_free_async_space(&proc->alloc);
-
-	seq_printf(m, "proc %d\n", proc->pid);
-	seq_printf(m, "context %s\n", proc->context->name);
-	count = 0;
-	ready_threads = 0;
-	binder_inner_proc_lock(proc);
-	for (n = rb_first(&proc->threads); n; n = rb_next(n))
-		count++;
-
-	list_for_each_entry(thread, &proc->waiting_threads, waiting_thread_node)
-		ready_threads++;
-
-	seq_printf(m, "  threads: %d\n", count);
-	seq_printf(m, "  requested threads: %d+%d/%d\n"
-			"  ready threads %d\n"
-			"  free async space %zd\n", proc->requested_threads,
-			proc->requested_threads_started, proc->max_threads,
-			ready_threads,
-			free_async_space);
-	count = 0;
-	for (n = rb_first(&proc->nodes); n; n = rb_next(n))
-		count++;
-	binder_inner_proc_unlock(proc);
-	seq_printf(m, "  nodes: %d\n", count);
-	count = 0;
-	strong = 0;
-	weak = 0;
-	binder_proc_lock(proc);
-	for (n = rb_first(&proc->refs_by_desc); n; n = rb_next(n)) {
-		struct binder_ref *ref = rb_entry(n, struct binder_ref,
-						  rb_node_desc);
-		count++;
-		strong += ref->data.strong;
-		weak += ref->data.weak;
-	}
-	binder_proc_unlock(proc);
-	seq_printf(m, "  refs: %d s %d w %d\n", count, strong, weak);
-
-	count = binder_alloc_get_allocated_count(&proc->alloc);
-	seq_printf(m, "  buffers: %d\n", count);
-
-	binder_alloc_print_pages(m, &proc->alloc);
-
-	count = 0;
-	binder_inner_proc_lock(proc);
-	list_for_each_entry(w, &proc->todo, entry) {
-		if (w->type == BINDER_WORK_TRANSACTION)
-			count++;
-	}
-	binder_inner_proc_unlock(proc);
-	seq_printf(m, "  pending transactions: %d\n", count);
-
-	print_binder_stats(m, "  ", &proc->stats);
-}
-
-static void print_binder_state(struct seq_file *m, bool hash_ptrs)
-{
-	struct binder_proc *proc;
-	struct binder_node *node;
-	struct binder_node *last_node = NULL;
-
-	seq_puts(m, "binder state:\n");
-
-	spin_lock(&binder_dead_nodes_lock);
-	if (!hlist_empty(&binder_dead_nodes))
-		seq_puts(m, "dead nodes:\n");
-	hlist_for_each_entry(node, &binder_dead_nodes, dead_node)
-		last_node = print_next_binder_node_ilocked(m, NULL, node,
-							   last_node,
-							   hash_ptrs);
-	spin_unlock(&binder_dead_nodes_lock);
-	if (last_node)
-		binder_put_node(last_node);
-
-	mutex_lock(&binder_procs_lock);
-	hlist_for_each_entry(proc, &binder_procs, proc_node)
-		print_binder_proc(m, proc, true, hash_ptrs);
-	mutex_unlock(&binder_procs_lock);
-}
-
-static void print_binder_transactions(struct seq_file *m, bool hash_ptrs)
-{
-	struct binder_proc *proc;
-
-	seq_puts(m, "binder transactions:\n");
-	mutex_lock(&binder_procs_lock);
-	hlist_for_each_entry(proc, &binder_procs, proc_node)
-		print_binder_proc(m, proc, false, hash_ptrs);
-	mutex_unlock(&binder_procs_lock);
-}
-
-static int state_show(struct seq_file *m, void *unused)
-{
-	print_binder_state(m, false);
-	return 0;
-}
-
-static int state_hashed_show(struct seq_file *m, void *unused)
-{
-	print_binder_state(m, true);
-	return 0;
-}
-
-static int stats_show(struct seq_file *m, void *unused)
-{
-	struct binder_proc *proc;
-
-	seq_puts(m, "binder stats:\n");
-
-	print_binder_stats(m, "", &binder_stats);
-
-	mutex_lock(&binder_procs_lock);
-	hlist_for_each_entry(proc, &binder_procs, proc_node)
-		print_binder_proc_stats(m, proc);
-	mutex_unlock(&binder_procs_lock);
-
-	return 0;
-}
-
-static int transactions_show(struct seq_file *m, void *unused)
-{
-	print_binder_transactions(m, false);
-	return 0;
-}
-
-static int transactions_hashed_show(struct seq_file *m, void *unused)
-{
-	print_binder_transactions(m, true);
-	return 0;
-}
-
-static int proc_show(struct seq_file *m, void *unused)
-{
-	struct binder_proc *itr;
-	int pid = (unsigned long)m->private;
-
-	guard(mutex)(&binder_procs_lock);
-	hlist_for_each_entry(itr, &binder_procs, proc_node) {
-		if (itr->pid == pid) {
-			seq_puts(m, "binder proc state:\n");
-			print_binder_proc(m, itr, true, false);
-		}
-	}
-
-	return 0;
-}
-
-static void print_binder_transaction_log_entry(struct seq_file *m,
-					struct binder_transaction_log_entry *e)
-{
-	int debug_id = READ_ONCE(e->debug_id_done);
-	/*
-	 * read barrier to guarantee debug_id_done read before
-	 * we print the log values
-	 */
-	smp_rmb();
-	seq_printf(m,
-		   "%d: %s from %d:%d to %d:%d context %s node %d handle %d size %d:%d ret %d/%d l=%d",
-		   e->debug_id, (e->call_type == 2) ? "reply" :
-		   ((e->call_type == 1) ? "async" : "call "), e->from_proc,
-		   e->from_thread, e->to_proc, e->to_thread, e->context_name,
-		   e->to_node, e->target_handle, e->data_size, e->offsets_size,
-		   e->return_error, e->return_error_param,
-		   e->return_error_line);
-	/*
-	 * read-barrier to guarantee read of debug_id_done after
-	 * done printing the fields of the entry
-	 */
-	smp_rmb();
-	seq_printf(m, debug_id && debug_id == READ_ONCE(e->debug_id_done) ?
-			"\n" : " (incomplete)\n");
-}
-
-static int transaction_log_show(struct seq_file *m, void *unused)
-{
-	struct binder_transaction_log *log = m->private;
-	unsigned int log_cur = atomic_read(&log->cur);
-	unsigned int count;
-	unsigned int cur;
-	int i;
-
-	count = log_cur + 1;
-	cur = count < ARRAY_SIZE(log->entry) && !log->full ?
-		0 : count % ARRAY_SIZE(log->entry);
-	if (count > ARRAY_SIZE(log->entry) || log->full)
-		count = ARRAY_SIZE(log->entry);
-	for (i = 0; i < count; i++) {
-		unsigned int index = cur++ % ARRAY_SIZE(log->entry);
-
-		print_binder_transaction_log_entry(m, &log->entry[index]);
-	}
-	return 0;
-}
-
-const struct file_operations binder_fops = {
-	.owner = THIS_MODULE,
-	.poll = binder_poll,
-	.unlocked_ioctl = binder_ioctl,
-	.compat_ioctl = compat_ptr_ioctl,
-	.mmap = binder_mmap,
-	.open = binder_open,
-	.flush = binder_flush,
-	.release = binder_release,
-};
-
-DEFINE_SHOW_ATTRIBUTE(state);
-DEFINE_SHOW_ATTRIBUTE(state_hashed);
-DEFINE_SHOW_ATTRIBUTE(stats);
-DEFINE_SHOW_ATTRIBUTE(transactions);
-DEFINE_SHOW_ATTRIBUTE(transactions_hashed);
-DEFINE_SHOW_ATTRIBUTE(transaction_log);
-
-const struct binder_debugfs_entry binder_debugfs_entries[] = {
-	{
-		.name = "state",
-		.mode = 0444,
-		.fops = &state_fops,
-		.data = NULL,
-	},
-	{
-		.name = "state_hashed",
-		.mode = 0444,
-		.fops = &state_hashed_fops,
-		.data = NULL,
-	},
-	{
-		.name = "stats",
-		.mode = 0444,
-		.fops = &stats_fops,
-		.data = NULL,
-	},
-	{
-		.name = "transactions",
-		.mode = 0444,
-		.fops = &transactions_fops,
-		.data = NULL,
-	},
-	{
-		.name = "transactions_hashed",
-		.mode = 0444,
-		.fops = &transactions_hashed_fops,
-		.data = NULL,
-	},
-	{
-		.name = "transaction_log",
-		.mode = 0444,
-		.fops = &transaction_log_fops,
-		.data = &binder_transaction_log,
-	},
-	{
-		.name = "failed_transaction_log",
-		.mode = 0444,
-		.fops = &transaction_log_fops,
-		.data = &binder_transaction_log_failed,
-	},
-	{} /* terminator */
-};
-
-void binder_add_device(struct binder_device *device)
-{
-	guard(spinlock)(&binder_devices_lock);
-	hlist_add_head(&device->hlist, &binder_devices);
-}
-
-void binder_remove_device(struct binder_device *device)
-{
-	guard(spinlock)(&binder_devices_lock);
-	hlist_del_init(&device->hlist);
-}
-
-static int __init init_binder_device(const char *name)
-{
-	int ret;
-	struct binder_device *binder_device;
-
-	binder_device = kzalloc_obj(*binder_device);
-	if (!binder_device)
-		return -ENOMEM;
-
-	binder_device->miscdev.fops = &binder_fops;
-	binder_device->miscdev.minor = MISC_DYNAMIC_MINOR;
-	binder_device->miscdev.name = name;
-
-	refcount_set(&binder_device->ref, 1);
-	binder_device->context.binder_context_mgr_uid = INVALID_UID;
-	binder_device->context.name = name;
-	mutex_init(&binder_device->context.context_mgr_node_lock);
-
-	ret = misc_register(&binder_device->miscdev);
-	if (ret < 0) {
-		kfree(binder_device);
-		return ret;
-	}
-
-	binder_add_device(binder_device);
-
-	return ret;
-}
-
-static int __init binder_init(void)
-{
-	int ret;
-	char *device_name, *device_tmp;
-	struct binder_device *device;
-	struct hlist_node *tmp;
-	char *device_names = NULL;
-	const struct binder_debugfs_entry *db_entry;
-
-	ret = binder_alloc_shrinker_init();
-	if (ret)
-		return ret;
-
-	atomic_set(&binder_transaction_log.cur, ~0U);
-	atomic_set(&binder_transaction_log_failed.cur, ~0U);
-
-	binder_debugfs_dir_entry_root = debugfs_create_dir("binder", NULL);
-
-	binder_for_each_debugfs_entry(db_entry)
-		debugfs_create_file(db_entry->name,
-					db_entry->mode,
-					binder_debugfs_dir_entry_root,
-					db_entry->data,
-					db_entry->fops);
-
-	binder_debugfs_dir_entry_proc = debugfs_create_dir("proc",
-						binder_debugfs_dir_entry_root);
-
-	if (!IS_ENABLED(CONFIG_ANDROID_BINDERFS) &&
-	    strcmp(binder_devices_param, "") != 0) {
-		/*
-		* Copy the module_parameter string, because we don't want to
-		* tokenize it in-place.
-		 */
-		device_names = kstrdup(binder_devices_param, GFP_KERNEL);
-		if (!device_names) {
-			ret = -ENOMEM;
-			goto err_alloc_device_names_failed;
-		}
-
-		device_tmp = device_names;
-		while ((device_name = strsep(&device_tmp, ","))) {
-			ret = init_binder_device(device_name);
-			if (ret)
-				goto err_init_binder_device_failed;
-		}
-	}
-
-	ret = genl_register_family(&binder_nl_family);
-	if (ret)
-		goto err_init_binder_device_failed;
-
-	ret = init_binderfs();
-	if (ret)
-		goto err_init_binderfs_failed;
-
-	return ret;
-
-err_init_binderfs_failed:
-	genl_unregister_family(&binder_nl_family);
-
-err_init_binder_device_failed:
-	hlist_for_each_entry_safe(device, tmp, &binder_devices, hlist) {
-		misc_deregister(&device->miscdev);
-		binder_remove_device(device);
-		kfree(device);
-	}
-
-	kfree(device_names);
-
-err_alloc_device_names_failed:
-	debugfs_remove_recursive(binder_debugfs_dir_entry_root);
-	binder_alloc_shrinker_exit();
-
-	return ret;
-}
-
-device_initcall(binder_init);
-
-#define CREATE_TRACE_POINTS
-#include "binder_trace.h"
diff --git a/drivers/android/binder/Makefile b/drivers/android/binder/Makefile
index 7e0cd9782a8b24..fe474628a94e2d 100644
--- a/drivers/android/binder/Makefile
+++ b/drivers/android/binder/Makefile
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 ccflags-y += -I$(src)	# needed for trace events
 
-obj-$(CONFIG_ANDROID_BINDER_IPC_RUST) += rust_binder.o
-rust_binder-y := \
+obj-$(CONFIG_ANDROID_BINDER_IPC_RUST) += binder.o
+binder-y := \
 	rust_binder_main.o	\
 	rust_binderfs.o		\
 	rust_binder_events.o
diff --git a/drivers/android/binder/allocation.rs b/drivers/android/binder/allocation.rs
index 165cb797eb1e9d..2f0763e6868d9e 100644
--- a/drivers/android/binder/allocation.rs
+++ b/drivers/android/binder/allocation.rs
@@ -208,7 +208,7 @@ impl Allocation {
         let num_close_on_free = files.iter().filter(|entry| entry.close_on_free).count();
         let mut close_on_free = KVec::with_capacity(num_close_on_free, GFP_KERNEL)?;
 
-        let mut reservations = KVec::with_capacity(files.len(), GFP_KERNEL)?;
+        let mut reservations = KVVec::with_capacity(files.len(), GFP_KERNEL)?;
         for file_info in files {
             let res = FileDescriptorReservation::get_unused_fd_flags(bindings::O_CLOEXEC)?;
             let fd = res.reserved_fd();
@@ -503,7 +503,7 @@ impl BinderObject {
     /// The closure should write the bytes for the object into the provided slice.
     pub(crate) fn read_from_inner<R>(reader: R) -> Result<BinderObject>
     where
-        R: FnOnce(&mut [u8; size_of::<BinderObject>()]) -> Result<()>,
+        R: FnOnce(&mut [u8; size_of::<BinderObject>()]) -> Result,
     {
         let mut obj = MaybeUninit::<BinderObject>::zeroed();
 
@@ -567,7 +567,7 @@ impl BinderObject {
 
 #[derive(Default)]
 struct FileList {
-    files_to_translate: KVec<FileEntry>,
+    files_to_translate: KVVec<FileEntry>,
     close_on_free: KVec<u32>,
 }
 
@@ -581,7 +581,7 @@ struct FileEntry {
 }
 
 pub(crate) struct TranslatedFds {
-    reservations: KVec<Reservation>,
+    reservations: KVVec<Reservation>,
     /// If commit is called, then these fds should be closed. (If commit is not called, then they
     /// shouldn't be closed.)
     close_on_free: FdsCloseOnFree,
@@ -595,7 +595,7 @@ struct Reservation {
 impl TranslatedFds {
     pub(crate) fn new() -> Self {
         Self {
-            reservations: KVec::new(),
+            reservations: KVVec::new(),
             close_on_free: FdsCloseOnFree(KVec::new()),
         }
     }
diff --git a/drivers/android/binder/defs.rs b/drivers/android/binder/defs.rs
index 8ac9bdd7a49968..cc4becd6e168f9 100644
--- a/drivers/android/binder/defs.rs
+++ b/drivers/android/binder/defs.rs
@@ -77,7 +77,8 @@ pub_no_prefix!(
     TF_ONE_WAY,
     TF_ACCEPT_FDS,
     TF_CLEAR_BUF,
-    TF_UPDATE_TXN
+    TF_UPDATE_TXN,
+    TF_DEFER_COMPLETE,
 );
 
 pub(crate) use uapi::{
diff --git a/drivers/android/binder/freeze.rs b/drivers/android/binder/freeze.rs
index 66912b4cb527b8..ea2450f3f16fe4 100644
--- a/drivers/android/binder/freeze.rs
+++ b/drivers/android/binder/freeze.rs
@@ -155,7 +155,7 @@ impl DeliverToRead for FreezeMessage {
     }
 
     #[inline(never)]
-    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
+    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result {
         seq_print!(m, "{}has frozen binder\n", prefix);
         Ok(())
     }
@@ -177,10 +177,7 @@ impl FreezeListener {
 }
 
 impl Process {
-    pub(crate) fn request_freeze_notif(
-        self: &Arc<Self>,
-        reader: &mut UserSliceReader,
-    ) -> Result<()> {
+    pub(crate) fn request_freeze_notif(self: &Arc<Self>, reader: &mut UserSliceReader) -> Result {
         let hc = reader.read::<BinderHandleCookie>()?;
         let handle = hc.handle;
         let cookie = FreezeCookie(hc.cookie);
@@ -272,7 +269,7 @@ impl Process {
         Ok(())
     }
 
-    pub(crate) fn freeze_notif_done(self: &Arc<Self>, reader: &mut UserSliceReader) -> Result<()> {
+    pub(crate) fn freeze_notif_done(self: &Arc<Self>, reader: &mut UserSliceReader) -> Result {
         let cookie = FreezeCookie(reader.read()?);
         let alloc = FreezeMessage::new(GFP_KERNEL)?;
         let mut node_refs_guard = self.node_refs.lock();
@@ -313,7 +310,7 @@ impl Process {
         Ok(())
     }
 
-    pub(crate) fn clear_freeze_notif(self: &Arc<Self>, reader: &mut UserSliceReader) -> Result<()> {
+    pub(crate) fn clear_freeze_notif(self: &Arc<Self>, reader: &mut UserSliceReader) -> Result {
         let hc = reader.read::<BinderHandleCookie>()?;
         let handle = hc.handle;
         let cookie = FreezeCookie(hc.cookie);
diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs
index 0a82af14cda3ca..b16ea0ff9ed942 100644
--- a/drivers/android/binder/node.rs
+++ b/drivers/android/binder/node.rs
@@ -51,9 +51,9 @@ pub(crate) struct CouldNotDeliverCriticalIncrement;
 ///   about to drop the weak reference, then the strong increment could be processed after the
 ///   other thread has already exited, which would be too late.
 ///
-/// Note that trying to create a `ListArc` to the node can succeed even if `has_normal_push` is
+/// Note that trying to create a `ListArc` to the node can succeed even if `has_pushed_node` is
 /// set. This is because another thread might just have popped the node from a todo list, but not
-/// yet called `do_work`. However, if `has_normal_push` is false, then creating a `ListArc` should
+/// yet called `do_work`. However, if `has_pushed_node` is false, then creating a `ListArc` should
 /// always succeed.
 ///
 /// Like the other fields in `NodeInner`, the delivery state is protected by the process lock.
@@ -255,11 +255,7 @@ impl Node {
     }
 
     #[inline(never)]
-    pub(crate) fn full_debug_print(
-        &self,
-        m: &SeqFile,
-        owner_inner: &mut ProcessInner,
-    ) -> Result<()> {
+    pub(crate) fn full_debug_print(&self, m: &SeqFile, owner_inner: &mut ProcessInner) -> Result {
         let inner = self.inner.access_mut(owner_inner);
         seq_print!(
             m,
@@ -738,14 +734,28 @@ impl DeliverToRead for Node {
         self.do_work_locked(writer, owner_inner)
     }
 
-    fn cancel(self: DArc<Self>) {}
+    fn cancel(self: DArc<Self>) {
+        let _drop_outside_lock;
+        let mut owner_inner = self.owner.inner.lock();
+
+        // We only do something on BINDER_THREAD_EXIT, not process exit.
+        if owner_inner.is_dead {
+            return;
+        }
+
+        // If BINDER_THREAD_EXIT is invoked on a thread with a pending node refcount update, we
+        // should move ourselves to ensure the refcount update is still delivered.
+        if let Some(node) = ListArc::try_from_arc_borrow(self.as_arc_borrow()) {
+            _drop_outside_lock = owner_inner.push_work(&self.owner, node);
+        }
+    }
 
     fn should_sync_wakeup(&self) -> bool {
         false
     }
 
     #[inline(never)]
-    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
+    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result {
         seq_print!(
             m,
             "{}node work {}: u{:016x} c{:016x}\n",
@@ -1140,7 +1150,7 @@ impl DeliverToRead for NodeDeath {
     }
 
     #[inline(never)]
-    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
+    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result {
         let inner = self.inner.lock();
 
         let dead_binder = inner.dead && !inner.notification_done;
diff --git a/drivers/android/binder/node/wrapper.rs b/drivers/android/binder/node/wrapper.rs
index 6e4ca01c941ab0..5a5e5e9883a4e6 100644
--- a/drivers/android/binder/node/wrapper.rs
+++ b/drivers/android/binder/node/wrapper.rs
@@ -57,14 +57,46 @@ impl DeliverToRead for NodeWrapper {
         node.do_work_locked(writer, owner_inner)
     }
 
-    fn cancel(self: DArc<Self>) {}
+    fn cancel(self: DArc<Self>) {
+        let _drop_outside_lock;
+        let node = &self.node;
+        let mut owner_inner = node.owner.inner.lock();
+
+        // We only do something on BINDER_THREAD_EXIT, not process exit.
+        if owner_inner.is_dead {
+            return;
+        }
+
+        // We transfer the responsibility of the node refcount update to the scheduled Node because
+        // NodeWrapper has no way to re-create the ListArc.
+        let inner = node.inner.access_mut(&mut owner_inner);
+
+        let ds = &mut inner.delivery_state;
+        assert!(ds.has_pushed_wrapper);
+        assert!(ds.has_strong_zero2one);
+        ds.has_pushed_wrapper = false;
+
+        // We are changing the state to one where the Node is the strong zero2one update instead of
+        // the wrapper.
+        ds.has_weak_zero2one = false;
+
+        if !ds.has_pushed_node {
+            if let Some(node2) = ListArc::try_from_arc_borrow(node.as_arc_borrow()) {
+                ds.has_pushed_node = true;
+                _drop_outside_lock = owner_inner.push_work(&node.owner, node2);
+            } else {
+                // This can't actually happen.
+                ds.has_strong_zero2one = false;
+            }
+        }
+    }
 
     fn should_sync_wakeup(&self) -> bool {
         false
     }
 
     #[inline(never)]
-    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
+    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result {
         seq_print!(
             m,
             "{}node work {}: u{:016x} c{:016x}\n",
diff --git a/drivers/android/binder/page_range.rs b/drivers/android/binder/page_range.rs
index 52ffbf3504e7f7..74c99432f86768 100644
--- a/drivers/android/binder/page_range.rs
+++ b/drivers/android/binder/page_range.rs
@@ -66,7 +66,7 @@ impl Shrinker {
     }
 
     /// Register this shrinker with the kernel.
-    pub(crate) fn register(&'static self, name: &CStr) -> Result<()> {
+    pub(crate) fn register(&'static self, name: &CStr) -> Result {
         // SAFETY: These fields are not yet used, so it's okay to zero them.
         unsafe {
             self.inner.get().write(ptr::null_mut());
@@ -352,7 +352,7 @@ impl ShrinkablePageRange {
     /// Make sure that the given pages are allocated and mapped.
     ///
     /// Must not be called from an atomic context.
-    pub(crate) fn use_range(&self, start: usize, end: usize) -> Result<()> {
+    pub(crate) fn use_range(&self, start: usize, end: usize) -> Result {
         if start >= end {
             return Ok(());
         }
@@ -398,7 +398,7 @@ impl ShrinkablePageRange {
     ///
     /// Assumes that `i` is in bounds.
     #[cold]
-    unsafe fn use_page_slow(&self, i: usize) -> Result<()> {
+    unsafe fn use_page_slow(&self, i: usize) -> Result {
         let new_page = Page::alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO)?;
 
         let mm_mutex = self.mm_lock.lock();
@@ -439,22 +439,9 @@ impl ShrinkablePageRange {
         // workqueue.
         let mm = MmWithUser::into_mmput_async(self.mm.mmget_not_zero().ok_or(ESRCH)?);
         {
-            let vma_read;
-            let mmap_read;
-            let vma = if let Some(ret) = mm.lock_vma_under_rcu(vma_addr) {
-                vma_read = ret;
-                check_vma(&vma_read, self)
-            } else {
-                mmap_read = mm.mmap_read_lock();
-                mmap_read
-                    .vma_lookup(vma_addr)
-                    .and_then(|vma| check_vma(vma, self))
-            };
-
-            match vma {
-                Some(vma) => vma.vm_insert_page(user_page_addr, &new_page)?,
-                None => return Err(ESRCH),
-            }
+            let vma_read_guard = mm.vma_start_read_unlocked(vma_addr).ok_or(ESRCH)?;
+            let vma = check_vma(&vma_read_guard, self).ok_or(ESRCH)?;
+            vma.vm_insert_page(user_page_addr, &new_page)?;
         }
 
         let inner = self.lock.lock();
diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs
index 5372bfbd93b35d..58c84be8ab7753 100644
--- a/drivers/android/binder/process.rs
+++ b/drivers/android/binder/process.rs
@@ -548,7 +548,7 @@ impl Process {
     }
 
     #[inline(never)]
-    pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result<()> {
+    pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result {
         seq_print!(m, "proc {}\n", self.pid_in_current_ns());
         seq_print!(m, "context {}\n", &*ctx.name);
 
@@ -596,7 +596,7 @@ impl Process {
     }
 
     #[inline(never)]
-    pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result<()> {
+    pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result {
         seq_print!(m, "proc {}\n", self.pid_in_current_ns());
         seq_print!(m, "context {}\n", &*ctx.name);
 
@@ -686,8 +686,16 @@ impl Process {
     pub(crate) fn get_work_or_register<'a>(
         &'a self,
         thread: &'a Arc<Thread>,
+        thread_has_deferred_work: bool,
     ) -> GetWorkOrRegister<'a> {
         let mut inner = self.inner.lock();
+
+        if thread_has_deferred_work && !inner.work.is_empty() {
+            if let Some(work) = thread.pop_work_even_if_deferred() {
+                return GetWorkOrRegister::Work(work);
+            }
+        }
+
         // Try to get work from the process queue.
         if let Some(work) = inner.work.pop_front() {
             return GetWorkOrRegister::Work(work);
@@ -1208,11 +1216,10 @@ impl Process {
 
         {
             let inner = self.inner.lock();
-            for (node_ptr, node) in &inner.nodes {
-                if *node_ptr > ptr {
-                    node.populate_debug_info(&mut out, &inner);
-                    break;
-                }
+            // cursor_lower_bound retrieves the "key" passed or the next existing larger key
+            if let Some(cursor) = inner.nodes.cursor_lower_bound(&(ptr + 1)) {
+                let (_, node) = cursor.current();
+                node.populate_debug_info(&mut out, &inner);
             }
         }
 
diff --git a/drivers/android/binder/range_alloc/array.rs b/drivers/android/binder/range_alloc/array.rs
index 081d19b09d4bb4..71bf49f9db0df6 100644
--- a/drivers/android/binder/range_alloc/array.rs
+++ b/drivers/android/binder/range_alloc/array.rs
@@ -61,7 +61,7 @@ impl<T> ArrayRangeAllocator<T> {
         self.ranges.len() == self.ranges.capacity()
     }
 
-    pub(crate) fn debug_print(&self, m: &SeqFile) -> Result<()> {
+    pub(crate) fn debug_print(&self, m: &SeqFile) -> Result {
         for range in &self.ranges {
             seq_print!(
                 m,
diff --git a/drivers/android/binder/range_alloc/mod.rs b/drivers/android/binder/range_alloc/mod.rs
index 1f4734468ff11f..bb6e0ec9650e10 100644
--- a/drivers/android/binder/range_alloc/mod.rs
+++ b/drivers/android/binder/range_alloc/mod.rs
@@ -141,7 +141,7 @@ impl<T> RangeAllocator<T> {
         }
     }
 
-    pub(crate) fn debug_print(&self, m: &SeqFile) -> Result<()> {
+    pub(crate) fn debug_print(&self, m: &SeqFile) -> Result {
         match &self.inner {
             Impl::Empty(_size) => Ok(()),
             Impl::Array(array) => array.debug_print(m),
diff --git a/drivers/android/binder/range_alloc/tree.rs b/drivers/android/binder/range_alloc/tree.rs
index 48796fcdb36249..058ff7498fa50e 100644
--- a/drivers/android/binder/range_alloc/tree.rs
+++ b/drivers/android/binder/range_alloc/tree.rs
@@ -111,7 +111,7 @@ impl<T> TreeRangeAllocator<T> {
             .count()
     }
 
-    pub(crate) fn debug_print(&self, m: &SeqFile) -> Result<()> {
+    pub(crate) fn debug_print(&self, m: &SeqFile) -> Result {
         for desc in self.tree.values() {
             let state = match &desc.state {
                 Some(state) => &state.0,
diff --git a/drivers/android/binder/rust_binder_main.rs b/drivers/android/binder/rust_binder_main.rs
index 955c4c348f738d..92bf33f9ff1c38 100644
--- a/drivers/android/binder/rust_binder_main.rs
+++ b/drivers/android/binder/rust_binder_main.rs
@@ -81,7 +81,7 @@ mod binderfs {
 
 module! {
     type: BinderModule,
-    name: "rust_binder",
+    name: "binder",
     authors: ["Wedson Almeida Filho", "Alice Ryhl"],
     description: "Android Binder",
     license: "GPL",
@@ -158,7 +158,7 @@ trait DeliverToRead: ListArcSafe + Send + Sync {
     /// Generally only set to true for non-oneway transactions.
     fn should_sync_wakeup(&self) -> bool;
 
-    fn debug_print(&self, m: &SeqFile, prefix: &str, transaction_prefix: &str) -> Result<()>;
+    fn debug_print(&self, m: &SeqFile, prefix: &str, transaction_prefix: &str) -> Result;
 }
 
 // Wrapper around a `DeliverToRead` with linked list links.
@@ -274,7 +274,7 @@ impl DeliverToRead for DeliverCode {
         false
     }
 
-    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
+    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result {
         seq_print!(m, "{}", prefix);
         if self.skip.load(Relaxed) {
             seq_print!(m, "(skipped) ");
@@ -545,7 +545,7 @@ unsafe extern "C" fn rust_binder_transactions_show(
     0
 }
 
-fn rust_binder_transactions_show_impl(m: &SeqFile) -> Result<()> {
+fn rust_binder_transactions_show_impl(m: &SeqFile) -> Result {
     seq_print!(m, "binder transactions:\n");
     let contexts = context::get_all_contexts()?;
     for ctx in contexts {
@@ -558,7 +558,7 @@ fn rust_binder_transactions_show_impl(m: &SeqFile) -> Result<()> {
     Ok(())
 }
 
-fn rust_binder_stats_show_impl(m: &SeqFile) -> Result<()> {
+fn rust_binder_stats_show_impl(m: &SeqFile) -> Result {
     seq_print!(m, "binder stats:\n");
     stats::GLOBAL_STATS.debug_print("", m);
     let contexts = context::get_all_contexts()?;
@@ -572,7 +572,7 @@ fn rust_binder_stats_show_impl(m: &SeqFile) -> Result<()> {
     Ok(())
 }
 
-fn rust_binder_state_show_impl(m: &SeqFile) -> Result<()> {
+fn rust_binder_state_show_impl(m: &SeqFile) -> Result {
     seq_print!(m, "binder state:\n");
     let contexts = context::get_all_contexts()?;
     for ctx in contexts {
@@ -585,7 +585,7 @@ fn rust_binder_state_show_impl(m: &SeqFile) -> Result<()> {
     Ok(())
 }
 
-fn rust_binder_proc_show_impl(m: &SeqFile, pid: Pid) -> Result<()> {
+fn rust_binder_proc_show_impl(m: &SeqFile, pid: Pid) -> Result {
     seq_print!(m, "binder proc state:\n");
     let contexts = context::get_all_contexts()?;
     for ctx in contexts {
diff --git a/drivers/android/binder/rust_binderfs.c b/drivers/android/binder/rust_binderfs.c
index 300cc65562d1e9..c2c7c3b404689f 100644
--- a/drivers/android/binder/rust_binderfs.c
+++ b/drivers/android/binder/rust_binderfs.c
@@ -49,7 +49,7 @@ DEFINE_SHOW_ATTRIBUTE(rust_binder_transactions);
 DEFINE_SHOW_ATTRIBUTE(rust_binder_proc);
 
 char *rust_binder_devices_param = CONFIG_ANDROID_BINDER_DEVICES;
-module_param_named(rust_devices, rust_binder_devices_param, charp, 0444);
+module_param_named(devices, rust_binder_devices_param, charp, 0444);
 
 extern u32 rust_binder_debug_mask;
 module_param_named(debug_mask, rust_binder_debug_mask, uint, 0644);
@@ -72,6 +72,7 @@ struct binder_features {
 	bool oneway_spam_detection;
 	bool extended_error;
 	bool freeze_notification;
+	bool transaction_report;
 };
 
 static const struct constant_table binderfs_param_stats[] = {
@@ -89,6 +90,7 @@ static struct binder_features binder_features = {
 	.oneway_spam_detection = true,
 	.extended_error = true,
 	.freeze_notification = true,
+	.transaction_report = true,
 };
 
 static inline struct binderfs_info *BINDERFS_SB(const struct super_block *sb)
@@ -338,7 +340,7 @@ static inline bool is_binderfs_control_device(const struct dentry *dentry)
 	return info->control_dentry == dentry;
 }
 
-static int binderfs_rename(struct mnt_idmap *idmap,
+static int binderfs_rename(const struct mnt_idmap *idmap,
 			   struct inode *old_dir, struct dentry *old_dentry,
 			   struct inode *new_dir, struct dentry *new_dentry,
 			   unsigned int flags)
@@ -566,6 +568,12 @@ static int init_binder_features(struct super_block *sb)
 	if (IS_ERR(dentry))
 		return PTR_ERR(dentry);
 
+	dentry = rust_binderfs_create_file(dir, "transaction_report",
+				      &binder_features_fops,
+				      &binder_features.transaction_report);
+	if (IS_ERR(dentry))
+		return PTR_ERR(dentry);
+
 	return 0;
 }
 
diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs
index 18a14aa8a835ae..edafce806d2776 100644
--- a/drivers/android/binder/thread.rs
+++ b/drivers/android/binder/thread.rs
@@ -481,7 +481,7 @@ impl Thread {
     }
 
     #[inline(never)]
-    pub(crate) fn debug_print(self: &Arc<Self>, m: &SeqFile, print_all: bool) -> Result<()> {
+    pub(crate) fn debug_print(self: &Arc<Self>, m: &SeqFile, print_all: bool) -> Result {
         let inner = self.inner.lock();
 
         if print_all || inner.current_transaction.is_some() || !inner.work_list.is_empty() {
@@ -584,9 +584,21 @@ impl Thread {
     // mangled symbol names.
     #[export_name = "rust_binder_wait"]
     fn get_work(self: &Arc<Self>, wait: bool) -> Result<Option<DLArc<dyn DeliverToRead>>> {
+        let thread_has_deferred_work;
+
         // Try to get work from the thread's work queue, using only a local lock.
         {
             let mut inner = self.inner.lock();
+
+            // The process_work_list boolean is used to make us go to sleep even if there is work
+            // in the thread todo-list, but it doesn't apply to the process todo-list. Furthermore,
+            // work in the thread todo-list must still be delivered before the process list.
+            //
+            // Thus, in some scenarios we must return the thread work now even if we were requested
+            // to wait. Adjust `process_work_list` to `true` accordingly.
+            inner.process_work_list |= inner.looper_need_return;
+            inner.process_work_list |= !wait;
+
             if let Some(work) = inner.pop_work() {
                 return Ok(Some(work));
             }
@@ -594,18 +606,26 @@ impl Thread {
                 drop(inner);
                 return Ok(self.process.get_work());
             }
+
+            // Note that if the thread list is empty, then the call to `pop_work()` has changed
+            // `process_work_list` back to `false` even if we set it to `true` above.
+            thread_has_deferred_work = !inner.work_list.is_empty();
         }
 
         // If the caller doesn't want to wait, try to grab work from the process queue.
         //
         // We know nothing will have been queued directly to the thread queue because it is not in
-        // a transaction and it is not in the process' ready list.
+        // a transaction and it is not in the process' ready list. We also know the thread list has
+        // no deferred work due to the `inner.process_work_list |= !wait` call above.
         if !wait {
             return self.process.get_work().ok_or(EAGAIN).map(Some);
         }
 
         // Get work from the process queue. If none is available, atomically register as ready.
-        let reg = match self.process.get_work_or_register(self) {
+        let reg = match self
+            .process
+            .get_work_or_register(self, thread_has_deferred_work)
+        {
             GetWorkOrRegister::Work(work) => return Ok(Some(work)),
             GetWorkOrRegister::Register(reg) => reg,
         };
@@ -621,14 +641,18 @@ impl Thread {
             inner.looper_flags &= !(LooperFlag::Waiting | LooperFlag::WaitingProc);
 
             if signal_pending || inner.looper_need_return {
-                // We need to return now. We need to pull the thread off the list of ready threads
-                // (by dropping `reg`), then check the state again after it's off the list to
-                // ensure that something was not queued in the meantime. If something has been
-                // queued, we just return it (instead of the error).
+                // We need to return now.
+                //
+                // We need to pull the thread off the list of ready threads (by dropping `reg`),
+                // then check the state again after it's off the list to ensure that something was
+                // not queued in the meantime. If something has been queued (or if there is
+                // deferred work), we just return it (instead of the error).
                 drop(inner);
                 drop(reg);
 
-                let res = match self.inner.lock().pop_work() {
+                inner = self.inner.lock();
+                inner.process_work_list = true;
+                let res = match inner.pop_work() {
                     Some(work) => Ok(Some(work)),
                     None if signal_pending => Err(EINTR),
                     None => Ok(None),
@@ -686,6 +710,12 @@ impl Thread {
         self.inner.lock().push_return_work(reply);
     }
 
+    pub(crate) fn pop_work_even_if_deferred(&self) -> Option<DLArc<dyn DeliverToRead>> {
+        let mut thread_inner = self.inner.lock();
+        thread_inner.process_work_list = true;
+        thread_inner.pop_work()
+    }
+
     fn translate_object(
         &self,
         obj_index: usize,
@@ -1250,7 +1280,7 @@ impl Thread {
         cmd: u32,
         reader: &mut UserSliceReader,
         info: &mut TransactionInfo,
-    ) -> Result<()> {
+    ) -> Result {
         let td = match cmd {
             BC_TRANSACTION | BC_REPLY => {
                 reader.read::<BinderTransactionData>()?.with_buffers_size(0)
@@ -1281,7 +1311,7 @@ impl Thread {
     }
 
     #[inline(never)]
-    fn transaction(self: &Arc<Self>, cmd: u32, reader: &mut UserSliceReader) -> Result<()> {
+    fn transaction(self: &Arc<Self>, cmd: u32, reader: &mut UserSliceReader) -> Result {
         let mut info = TransactionInfo::zeroed();
         self.read_transaction_info(cmd, reader, &mut info)?;
 
@@ -1410,8 +1440,16 @@ impl Thread {
             let process = orig.from.process.clone();
             let allow_fds = orig.flags.contains(TransactionFlag::AcceptFds);
             let reply = Transaction::new_reply(self, process, info, allow_fds)?;
-            // Not notifying: Reply to current thread.
-            let _ = self.inner.lock().push_work(completion);
+            {
+                let mut inner = self.inner.lock();
+                if info.flags.contains(TransactionFlag::DeferComplete) {
+                    // The flag is set. Perform a deferred push so that `read` can wait for the
+                    // next incoming transaction without a userspace roundtrip.
+                    inner.push_work_deferred(completion);
+                } else {
+                    let _ = inner.push_work(completion);
+                }
+            }
             orig.from.deliver_reply(Ok(reply), &orig, None);
             Ok(())
         })()
@@ -1678,7 +1716,7 @@ impl Thread {
         self.unwind_transaction_stack();
 
         // Cancel all pending work items.
-        while let Ok(Some(work)) = self.get_work_local(false) {
+        while let Some(work) = self.pop_work_even_if_deferred() {
             work.into_arc().cancel();
         }
     }
@@ -1738,7 +1776,7 @@ impl DeliverToRead for ThreadError {
         false
     }
 
-    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
+    fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result {
         seq_print!(
             m,
             "{}transaction error: {}\n",
diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder/transaction.rs
index 245f1556b5db37..888a9e0eba8f23 100644
--- a/drivers/android/binder/transaction.rs
+++ b/drivers/android/binder/transaction.rs
@@ -39,6 +39,7 @@ kernel::impl_flags!(
         AcceptFds = TF_ACCEPT_FDS,
         ClearBuf = TF_CLEAR_BUF,
         UpdateTxn = TF_UPDATE_TXN,
+        DeferComplete = TF_DEFER_COMPLETE,
     }
 );
 
@@ -576,7 +577,7 @@ impl DeliverToRead for Transaction {
         !self.flags.is_oneway()
     }
 
-    fn debug_print(&self, m: &SeqFile, _prefix: &str, tprefix: &str) -> Result<()> {
+    fn debug_print(&self, m: &SeqFile, _prefix: &str, tprefix: &str) -> Result {
         self.debug_print_inner(m, tprefix);
         Ok(())
     }
diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
deleted file mode 100644
index e4488ad86a6557..00000000000000
--- a/drivers/android/binder_alloc.c
+++ /dev/null
@@ -1,1409 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* binder_alloc.c
- *
- * Android IPC Subsystem
- *
- * Copyright (C) 2007-2017 Google, Inc.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/list.h>
-#include <linux/sched/mm.h>
-#include <linux/module.h>
-#include <linux/rtmutex.h>
-#include <linux/rbtree.h>
-#include <linux/seq_file.h>
-#include <linux/vmalloc.h>
-#include <linux/slab.h>
-#include <linux/sched.h>
-#include <linux/list_lru.h>
-#include <linux/ratelimit.h>
-#include <asm/cacheflush.h>
-#include <linux/uaccess.h>
-#include <linux/highmem.h>
-#include <linux/sizes.h>
-#include <kunit/visibility.h>
-#include "binder_alloc.h"
-#include "binder_trace.h"
-
-static struct list_lru binder_freelist;
-
-static DEFINE_MUTEX(binder_alloc_mmap_lock);
-
-enum {
-	BINDER_DEBUG_USER_ERROR             = 1U << 0,
-	BINDER_DEBUG_OPEN_CLOSE             = 1U << 1,
-	BINDER_DEBUG_BUFFER_ALLOC           = 1U << 2,
-	BINDER_DEBUG_BUFFER_ALLOC_ASYNC     = 1U << 3,
-};
-static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR;
-
-module_param_named(debug_mask, binder_alloc_debug_mask,
-		   uint, 0644);
-
-#define binder_alloc_debug(mask, x...) \
-	do { \
-		if (binder_alloc_debug_mask & mask) \
-			pr_info_ratelimited(x); \
-	} while (0)
-
-static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer)
-{
-	return list_entry(buffer->entry.next, struct binder_buffer, entry);
-}
-
-static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer)
-{
-	return list_entry(buffer->entry.prev, struct binder_buffer, entry);
-}
-
-VISIBLE_IF_KUNIT size_t binder_alloc_buffer_size(struct binder_alloc *alloc,
-						 struct binder_buffer *buffer)
-{
-	if (list_is_last(&buffer->entry, &alloc->buffers))
-		return alloc->vm_start + alloc->buffer_size - buffer->user_data;
-	return binder_buffer_next(buffer)->user_data - buffer->user_data;
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_alloc_buffer_size);
-
-static void binder_insert_free_buffer(struct binder_alloc *alloc,
-				      struct binder_buffer *new_buffer)
-{
-	struct rb_node **p = &alloc->free_buffers.rb_node;
-	struct rb_node *parent = NULL;
-	struct binder_buffer *buffer;
-	size_t buffer_size;
-	size_t new_buffer_size;
-
-	BUG_ON(!new_buffer->free);
-
-	new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer);
-
-	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: add free buffer, size %zd, at %p\n",
-		      alloc->pid, new_buffer_size, new_buffer);
-
-	while (*p) {
-		parent = *p;
-		buffer = rb_entry(parent, struct binder_buffer, rb_node);
-		BUG_ON(!buffer->free);
-
-		buffer_size = binder_alloc_buffer_size(alloc, buffer);
-
-		if (new_buffer_size < buffer_size)
-			p = &parent->rb_left;
-		else
-			p = &parent->rb_right;
-	}
-	rb_link_node(&new_buffer->rb_node, parent, p);
-	rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers);
-}
-
-static void binder_insert_allocated_buffer_locked(
-		struct binder_alloc *alloc, struct binder_buffer *new_buffer)
-{
-	struct rb_node **p = &alloc->allocated_buffers.rb_node;
-	struct rb_node *parent = NULL;
-	struct binder_buffer *buffer;
-
-	BUG_ON(new_buffer->free);
-
-	while (*p) {
-		parent = *p;
-		buffer = rb_entry(parent, struct binder_buffer, rb_node);
-		BUG_ON(buffer->free);
-
-		if (new_buffer->user_data < buffer->user_data)
-			p = &parent->rb_left;
-		else if (new_buffer->user_data > buffer->user_data)
-			p = &parent->rb_right;
-		else
-			BUG();
-	}
-	rb_link_node(&new_buffer->rb_node, parent, p);
-	rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers);
-}
-
-static struct binder_buffer *binder_alloc_prepare_to_free_locked(
-		struct binder_alloc *alloc,
-		unsigned long user_ptr)
-{
-	struct rb_node *n = alloc->allocated_buffers.rb_node;
-	struct binder_buffer *buffer;
-
-	while (n) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		BUG_ON(buffer->free);
-
-		if (user_ptr < buffer->user_data) {
-			n = n->rb_left;
-		} else if (user_ptr > buffer->user_data) {
-			n = n->rb_right;
-		} else {
-			/*
-			 * Guard against user threads attempting to
-			 * free the buffer when in use by kernel or
-			 * after it's already been freed.
-			 */
-			if (!buffer->allow_user_free)
-				return ERR_PTR(-EPERM);
-			buffer->allow_user_free = 0;
-			return buffer;
-		}
-	}
-	return NULL;
-}
-
-/**
- * binder_alloc_prepare_to_free() - get buffer given user ptr
- * @alloc:	binder_alloc for this proc
- * @user_ptr:	User pointer to buffer data
- *
- * Validate userspace pointer to buffer data and return buffer corresponding to
- * that user pointer. Search the rb tree for buffer that matches user data
- * pointer.
- *
- * Return:	Pointer to buffer or NULL
- */
-struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc,
-						   unsigned long user_ptr)
-{
-	guard(mutex)(&alloc->mutex);
-	return binder_alloc_prepare_to_free_locked(alloc, user_ptr);
-}
-
-static inline void
-binder_set_installed_page(struct binder_alloc *alloc,
-			  unsigned long index,
-			  struct page *page)
-{
-	/* Pairs with acquire in binder_get_installed_page() */
-	smp_store_release(&alloc->pages[index], page);
-}
-
-static inline struct page *
-binder_get_installed_page(struct binder_alloc *alloc, unsigned long index)
-{
-	/* Pairs with release in binder_set_installed_page() */
-	return smp_load_acquire(&alloc->pages[index]);
-}
-
-static void binder_lru_freelist_add(struct binder_alloc *alloc,
-				    unsigned long start, unsigned long end)
-{
-	unsigned long page_addr;
-	struct page *page;
-
-	trace_binder_update_page_range(alloc, false, start, end);
-
-	for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
-		size_t index;
-		int ret;
-
-		index = (page_addr - alloc->vm_start) / PAGE_SIZE;
-		page = binder_get_installed_page(alloc, index);
-		if (!page)
-			continue;
-
-		trace_binder_free_lru_start(alloc, index);
-
-		ret = list_lru_add(alloc->freelist,
-				   page_to_lru(page),
-				   page_to_nid(page),
-				   NULL);
-		WARN_ON(!ret);
-
-		trace_binder_free_lru_end(alloc, index);
-	}
-}
-
-static inline
-void binder_alloc_set_mapped(struct binder_alloc *alloc, bool state)
-{
-	/* pairs with smp_load_acquire in binder_alloc_is_mapped() */
-	smp_store_release(&alloc->mapped, state);
-}
-
-static inline bool binder_alloc_is_mapped(struct binder_alloc *alloc)
-{
-	/* pairs with smp_store_release in binder_alloc_set_mapped() */
-	return smp_load_acquire(&alloc->mapped);
-}
-
-static struct page *binder_page_lookup(struct binder_alloc *alloc,
-				       unsigned long addr)
-{
-	struct mm_struct *mm = alloc->mm;
-	struct page *page;
-	long npages = 0;
-
-	/*
-	 * Find an existing page in the remote mm. If missing,
-	 * don't attempt to fault-in just propagate an error.
-	 */
-	mmap_read_lock(mm);
-	if (binder_alloc_is_mapped(alloc))
-		npages = get_user_pages_remote(mm, addr, 1, FOLL_NOFAULT,
-					       &page, NULL);
-	mmap_read_unlock(mm);
-
-	return npages > 0 ? page : NULL;
-}
-
-static int binder_page_insert(struct binder_alloc *alloc,
-			      unsigned long addr,
-			      struct page *page)
-{
-	struct mm_struct *mm = alloc->mm;
-	struct vm_area_struct *vma;
-	int ret = -ESRCH;
-
-	/* attempt per-vma lock first */
-	vma = lock_vma_under_rcu(mm, addr);
-	if (vma) {
-		if (binder_alloc_is_mapped(alloc))
-			ret = vm_insert_page(vma, addr, page);
-		vma_end_read(vma);
-		return ret;
-	}
-
-	/* fall back to mmap_lock */
-	mmap_read_lock(mm);
-	vma = vma_lookup(mm, addr);
-	if (vma && binder_alloc_is_mapped(alloc))
-		ret = vm_insert_page(vma, addr, page);
-	mmap_read_unlock(mm);
-
-	return ret;
-}
-
-static struct page *binder_page_alloc(struct binder_alloc *alloc,
-				      unsigned long index)
-{
-	struct binder_shrinker_mdata *mdata;
-	struct page *page;
-
-	page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
-	if (!page)
-		return NULL;
-
-	/* allocate and install shrinker metadata under page->private */
-	mdata = kzalloc_obj(*mdata);
-	if (!mdata) {
-		__free_page(page);
-		return NULL;
-	}
-
-	mdata->alloc = alloc;
-	mdata->page_index = index;
-	INIT_LIST_HEAD(&mdata->lru);
-	set_page_private(page, (unsigned long)mdata);
-
-	return page;
-}
-
-static void binder_free_page(struct page *page)
-{
-	kfree((struct binder_shrinker_mdata *)page_private(page));
-	__free_page(page);
-}
-
-static int binder_install_single_page(struct binder_alloc *alloc,
-				      unsigned long index,
-				      unsigned long addr)
-{
-	struct page *page;
-	int ret;
-
-	if (!mmget_not_zero(alloc->mm))
-		return -ESRCH;
-
-	page = binder_page_alloc(alloc, index);
-	if (!page) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	ret = binder_page_insert(alloc, addr, page);
-	switch (ret) {
-	case -EBUSY:
-		/*
-		 * EBUSY is ok. Someone installed the pte first but the
-		 * alloc->pages[index] has not been updated yet. Discard
-		 * our page and look up the one already installed.
-		 */
-		ret = 0;
-		binder_free_page(page);
-		page = binder_page_lookup(alloc, addr);
-		if (!page) {
-			pr_err("%d: failed to find page at offset %lx\n",
-			       alloc->pid, addr - alloc->vm_start);
-			ret = -ESRCH;
-			break;
-		}
-		fallthrough;
-	case 0:
-		/* Mark page installation complete and safe to use */
-		binder_set_installed_page(alloc, index, page);
-		break;
-	default:
-		binder_free_page(page);
-		pr_err("%d: %s failed to insert page at offset %lx with %d\n",
-		       alloc->pid, __func__, addr - alloc->vm_start, ret);
-		break;
-	}
-out:
-	mmput_async(alloc->mm);
-	return ret;
-}
-
-static int binder_install_buffer_pages(struct binder_alloc *alloc,
-				       struct binder_buffer *buffer,
-				       size_t size)
-{
-	unsigned long start, final;
-	unsigned long page_addr;
-
-	start = buffer->user_data & PAGE_MASK;
-	final = PAGE_ALIGN(buffer->user_data + size);
-
-	for (page_addr = start; page_addr < final; page_addr += PAGE_SIZE) {
-		unsigned long index;
-		int ret;
-
-		index = (page_addr - alloc->vm_start) / PAGE_SIZE;
-		if (binder_get_installed_page(alloc, index))
-			continue;
-
-		trace_binder_alloc_page_start(alloc, index);
-
-		ret = binder_install_single_page(alloc, index, page_addr);
-		if (ret)
-			return ret;
-
-		trace_binder_alloc_page_end(alloc, index);
-	}
-
-	return 0;
-}
-
-/* The range of pages should exclude those shared with other buffers */
-static void binder_lru_freelist_del(struct binder_alloc *alloc,
-				    unsigned long start, unsigned long end)
-{
-	unsigned long page_addr;
-	struct page *page;
-
-	trace_binder_update_page_range(alloc, true, start, end);
-
-	for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
-		unsigned long index;
-		bool on_lru;
-
-		index = (page_addr - alloc->vm_start) / PAGE_SIZE;
-		page = binder_get_installed_page(alloc, index);
-
-		if (page) {
-			trace_binder_alloc_lru_start(alloc, index);
-
-			on_lru = list_lru_del(alloc->freelist,
-					      page_to_lru(page),
-					      page_to_nid(page),
-					      NULL);
-			WARN_ON(!on_lru);
-
-			trace_binder_alloc_lru_end(alloc, index);
-			continue;
-		}
-
-		if (index + 1 > alloc->pages_high)
-			alloc->pages_high = index + 1;
-	}
-}
-
-static void debug_no_space_locked(struct binder_alloc *alloc)
-{
-	size_t largest_alloc_size = 0;
-	struct binder_buffer *buffer;
-	size_t allocated_buffers = 0;
-	size_t largest_free_size = 0;
-	size_t total_alloc_size = 0;
-	size_t total_free_size = 0;
-	size_t free_buffers = 0;
-	size_t buffer_size;
-	struct rb_node *n;
-
-	for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		buffer_size = binder_alloc_buffer_size(alloc, buffer);
-		allocated_buffers++;
-		total_alloc_size += buffer_size;
-		if (buffer_size > largest_alloc_size)
-			largest_alloc_size = buffer_size;
-	}
-
-	for (n = rb_first(&alloc->free_buffers); n; n = rb_next(n)) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		buffer_size = binder_alloc_buffer_size(alloc, buffer);
-		free_buffers++;
-		total_free_size += buffer_size;
-		if (buffer_size > largest_free_size)
-			largest_free_size = buffer_size;
-	}
-
-	binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
-			   "allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n",
-			   total_alloc_size, allocated_buffers,
-			   largest_alloc_size, total_free_size,
-			   free_buffers, largest_free_size);
-}
-
-static bool debug_low_async_space_locked(struct binder_alloc *alloc)
-{
-	/*
-	 * Find the amount and size of buffers allocated by the current caller;
-	 * The idea is that once we cross the threshold, whoever is responsible
-	 * for the low async space is likely to try to send another async txn,
-	 * and at some point we'll catch them in the act. This is more efficient
-	 * than keeping a map per pid.
-	 */
-	struct binder_buffer *buffer;
-	size_t total_alloc_size = 0;
-	int pid = current->tgid;
-	size_t num_buffers = 0;
-	struct rb_node *n;
-
-	/*
-	 * Only start detecting spammers once we have less than 20% of async
-	 * space left (which is less than 10% of total buffer size).
-	 */
-	if (alloc->free_async_space >= alloc->buffer_size / 10) {
-		alloc->oneway_spam_detected = false;
-		return false;
-	}
-
-	for (n = rb_first(&alloc->allocated_buffers); n != NULL;
-		 n = rb_next(n)) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		if (buffer->pid != pid)
-			continue;
-		if (!buffer->async_transaction)
-			continue;
-		total_alloc_size += binder_alloc_buffer_size(alloc, buffer);
-		num_buffers++;
-	}
-
-	/*
-	 * Warn if this pid has more than 50 transactions, or more than 50% of
-	 * async space (which is 25% of total buffer size). Oneway spam is only
-	 * detected when the threshold is exceeded.
-	 */
-	if (num_buffers > 50 || total_alloc_size > alloc->buffer_size / 4) {
-		binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
-			     "%d: pid %d spamming oneway? %zd buffers allocated for a total size of %zd\n",
-			      alloc->pid, pid, num_buffers, total_alloc_size);
-		if (!alloc->oneway_spam_detected) {
-			alloc->oneway_spam_detected = true;
-			return true;
-		}
-	}
-	return false;
-}
-
-/* Callers preallocate @new_buffer, it is freed by this function if unused */
-static struct binder_buffer *binder_alloc_new_buf_locked(
-				struct binder_alloc *alloc,
-				struct binder_buffer *new_buffer,
-				size_t size,
-				int is_async)
-{
-	struct rb_node *n = alloc->free_buffers.rb_node;
-	struct rb_node *best_fit = NULL;
-	struct binder_buffer *buffer;
-	unsigned long next_used_page;
-	unsigned long curr_last_page;
-	size_t buffer_size;
-
-	if (is_async && alloc->free_async_space < size) {
-		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
-			     "%d: binder_alloc_buf size %zd failed, no async space left\n",
-			      alloc->pid, size);
-		buffer = ERR_PTR(-ENOSPC);
-		goto out;
-	}
-
-	while (n) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		BUG_ON(!buffer->free);
-		buffer_size = binder_alloc_buffer_size(alloc, buffer);
-
-		if (size < buffer_size) {
-			best_fit = n;
-			n = n->rb_left;
-		} else if (size > buffer_size) {
-			n = n->rb_right;
-		} else {
-			best_fit = n;
-			break;
-		}
-	}
-
-	if (unlikely(!best_fit)) {
-		binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
-				   "%d: binder_alloc_buf size %zd failed, no address space\n",
-				   alloc->pid, size);
-		debug_no_space_locked(alloc);
-		buffer = ERR_PTR(-ENOSPC);
-		goto out;
-	}
-
-	if (buffer_size != size) {
-		/* Found an oversized buffer and needs to be split */
-		buffer = rb_entry(best_fit, struct binder_buffer, rb_node);
-		buffer_size = binder_alloc_buffer_size(alloc, buffer);
-
-		WARN_ON(n || buffer_size == size);
-		new_buffer->user_data = buffer->user_data + size;
-		list_add(&new_buffer->entry, &buffer->entry);
-		new_buffer->free = 1;
-		binder_insert_free_buffer(alloc, new_buffer);
-		new_buffer = NULL;
-	}
-
-	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: binder_alloc_buf size %zd got buffer %p size %zd\n",
-		      alloc->pid, size, buffer, buffer_size);
-
-	/*
-	 * Now we remove the pages from the freelist. A clever calculation
-	 * with buffer_size determines if the last page is shared with an
-	 * adjacent in-use buffer. In such case, the page has been already
-	 * removed from the freelist so we trim our range short.
-	 */
-	next_used_page = (buffer->user_data + buffer_size) & PAGE_MASK;
-	curr_last_page = PAGE_ALIGN(buffer->user_data + size);
-	binder_lru_freelist_del(alloc, PAGE_ALIGN(buffer->user_data),
-				min(next_used_page, curr_last_page));
-
-	rb_erase(&buffer->rb_node, &alloc->free_buffers);
-	buffer->free = 0;
-	buffer->allow_user_free = 0;
-	binder_insert_allocated_buffer_locked(alloc, buffer);
-	buffer->async_transaction = is_async;
-	buffer->oneway_spam_suspect = false;
-	if (is_async) {
-		alloc->free_async_space -= size;
-		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
-			     "%d: binder_alloc_buf size %zd async free %zd\n",
-			      alloc->pid, size, alloc->free_async_space);
-		if (debug_low_async_space_locked(alloc))
-			buffer->oneway_spam_suspect = true;
-	}
-
-out:
-	/* Discard possibly unused new_buffer */
-	kfree(new_buffer);
-	return buffer;
-}
-
-/* Calculate the sanitized total size, returns 0 for invalid request */
-static inline size_t sanitized_size(size_t data_size,
-				    size_t offsets_size,
-				    size_t extra_buffers_size)
-{
-	size_t total, tmp;
-
-	/* Align to pointer size and check for overflows */
-	tmp = ALIGN(data_size, sizeof(void *)) +
-		ALIGN(offsets_size, sizeof(void *));
-	if (tmp < data_size || tmp < offsets_size)
-		return 0;
-	total = tmp + ALIGN(extra_buffers_size, sizeof(void *));
-	if (total < tmp || total < extra_buffers_size)
-		return 0;
-
-	/* Pad 0-sized buffers so they get a unique address */
-	total = max(total, sizeof(void *));
-
-	return total;
-}
-
-/**
- * binder_alloc_new_buf() - Allocate a new binder buffer
- * @alloc:              binder_alloc for this proc
- * @data_size:          size of user data buffer
- * @offsets_size:       user specified buffer offset
- * @extra_buffers_size: size of extra space for meta-data (eg, security context)
- * @is_async:           buffer for async transaction
- *
- * Allocate a new buffer given the requested sizes. Returns
- * the kernel version of the buffer pointer. The size allocated
- * is the sum of the three given sizes (each rounded up to
- * pointer-sized boundary)
- *
- * Return:	The allocated buffer or %ERR_PTR(-errno) if error
- */
-struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,
-					   size_t data_size,
-					   size_t offsets_size,
-					   size_t extra_buffers_size,
-					   int is_async)
-{
-	struct binder_buffer *buffer, *next;
-	size_t size;
-	int ret;
-
-	/* Check binder_alloc is fully initialized */
-	if (!binder_alloc_is_mapped(alloc)) {
-		binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
-				   "%d: binder_alloc_buf, no vma\n",
-				   alloc->pid);
-		return ERR_PTR(-ESRCH);
-	}
-
-	size = sanitized_size(data_size, offsets_size, extra_buffers_size);
-	if (unlikely(!size)) {
-		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
-				   "%d: got transaction with invalid size %zd-%zd-%zd\n",
-				   alloc->pid, data_size, offsets_size,
-				   extra_buffers_size);
-		return ERR_PTR(-EINVAL);
-	}
-
-	/* Preallocate the next buffer */
-	next = kzalloc_obj(*next);
-	if (!next)
-		return ERR_PTR(-ENOMEM);
-
-	mutex_lock(&alloc->mutex);
-	buffer = binder_alloc_new_buf_locked(alloc, next, size, is_async);
-	if (IS_ERR(buffer)) {
-		mutex_unlock(&alloc->mutex);
-		goto out;
-	}
-
-	buffer->data_size = data_size;
-	buffer->offsets_size = offsets_size;
-	buffer->extra_buffers_size = extra_buffers_size;
-	buffer->pid = current->tgid;
-	mutex_unlock(&alloc->mutex);
-
-	ret = binder_install_buffer_pages(alloc, buffer, size);
-	if (ret) {
-		binder_alloc_free_buf(alloc, buffer);
-		buffer = ERR_PTR(ret);
-	}
-out:
-	return buffer;
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_alloc_new_buf);
-
-static unsigned long buffer_start_page(struct binder_buffer *buffer)
-{
-	return buffer->user_data & PAGE_MASK;
-}
-
-static unsigned long prev_buffer_end_page(struct binder_buffer *buffer)
-{
-	return (buffer->user_data - 1) & PAGE_MASK;
-}
-
-static void binder_delete_free_buffer(struct binder_alloc *alloc,
-				      struct binder_buffer *buffer)
-{
-	struct binder_buffer *prev, *next;
-
-	if (PAGE_ALIGNED(buffer->user_data))
-		goto skip_freelist;
-
-	BUG_ON(alloc->buffers.next == &buffer->entry);
-	prev = binder_buffer_prev(buffer);
-	BUG_ON(!prev->free);
-	if (prev_buffer_end_page(prev) == buffer_start_page(buffer))
-		goto skip_freelist;
-
-	if (!list_is_last(&buffer->entry, &alloc->buffers)) {
-		next = binder_buffer_next(buffer);
-		if (buffer_start_page(next) == buffer_start_page(buffer))
-			goto skip_freelist;
-	}
-
-	binder_lru_freelist_add(alloc, buffer_start_page(buffer),
-				buffer_start_page(buffer) + PAGE_SIZE);
-skip_freelist:
-	list_del(&buffer->entry);
-	kfree(buffer);
-}
-
-static void binder_free_buf_locked(struct binder_alloc *alloc,
-				   struct binder_buffer *buffer)
-{
-	size_t size, buffer_size;
-
-	buffer_size = binder_alloc_buffer_size(alloc, buffer);
-
-	size = ALIGN(buffer->data_size, sizeof(void *)) +
-		ALIGN(buffer->offsets_size, sizeof(void *)) +
-		ALIGN(buffer->extra_buffers_size, sizeof(void *));
-
-	binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
-		     "%d: binder_free_buf %p size %zd buffer_size %zd\n",
-		      alloc->pid, buffer, size, buffer_size);
-
-	BUG_ON(buffer->free);
-	BUG_ON(size > buffer_size);
-	BUG_ON(buffer->transaction != NULL);
-	BUG_ON(buffer->user_data < alloc->vm_start);
-	BUG_ON(buffer->user_data > alloc->vm_start + alloc->buffer_size);
-
-	if (buffer->async_transaction) {
-		alloc->free_async_space += buffer_size;
-		binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
-			     "%d: binder_free_buf size %zd async free %zd\n",
-			      alloc->pid, size, alloc->free_async_space);
-	}
-
-	binder_lru_freelist_add(alloc, PAGE_ALIGN(buffer->user_data),
-				(buffer->user_data + buffer_size) & PAGE_MASK);
-
-	rb_erase(&buffer->rb_node, &alloc->allocated_buffers);
-	buffer->free = 1;
-	if (!list_is_last(&buffer->entry, &alloc->buffers)) {
-		struct binder_buffer *next = binder_buffer_next(buffer);
-
-		if (next->free) {
-			rb_erase(&next->rb_node, &alloc->free_buffers);
-			binder_delete_free_buffer(alloc, next);
-		}
-	}
-	if (alloc->buffers.next != &buffer->entry) {
-		struct binder_buffer *prev = binder_buffer_prev(buffer);
-
-		if (prev->free) {
-			binder_delete_free_buffer(alloc, buffer);
-			rb_erase(&prev->rb_node, &alloc->free_buffers);
-			buffer = prev;
-		}
-	}
-	binder_insert_free_buffer(alloc, buffer);
-}
-
-/**
- * binder_alloc_get_page() - get kernel pointer for given buffer offset
- * @alloc: binder_alloc for this proc
- * @buffer: binder buffer to be accessed
- * @buffer_offset: offset into @buffer data
- * @pgoffp: address to copy final page offset to
- *
- * Lookup the struct page corresponding to the address
- * at @buffer_offset into @buffer->user_data. If @pgoffp is not
- * NULL, the byte-offset into the page is written there.
- *
- * The caller is responsible to ensure that the offset points
- * to a valid address within the @buffer and that @buffer is
- * not freeable by the user. Since it can't be freed, we are
- * guaranteed that the corresponding elements of @alloc->pages[]
- * cannot change.
- *
- * Return: struct page
- */
-static struct page *binder_alloc_get_page(struct binder_alloc *alloc,
-					  struct binder_buffer *buffer,
-					  binder_size_t buffer_offset,
-					  pgoff_t *pgoffp)
-{
-	binder_size_t buffer_space_offset = buffer_offset +
-		(buffer->user_data - alloc->vm_start);
-	pgoff_t pgoff = buffer_space_offset & ~PAGE_MASK;
-	size_t index = buffer_space_offset >> PAGE_SHIFT;
-
-	*pgoffp = pgoff;
-
-	return alloc->pages[index];
-}
-
-/**
- * binder_alloc_clear_buf() - zero out buffer
- * @alloc: binder_alloc for this proc
- * @buffer: binder buffer to be cleared
- *
- * memset the given buffer to 0
- */
-static void binder_alloc_clear_buf(struct binder_alloc *alloc,
-				   struct binder_buffer *buffer)
-{
-	size_t bytes = binder_alloc_buffer_size(alloc, buffer);
-	binder_size_t buffer_offset = 0;
-
-	while (bytes) {
-		unsigned long size;
-		struct page *page;
-		pgoff_t pgoff;
-
-		page = binder_alloc_get_page(alloc, buffer,
-					     buffer_offset, &pgoff);
-		size = min_t(size_t, bytes, PAGE_SIZE - pgoff);
-		memset_page(page, pgoff, 0, size);
-		bytes -= size;
-		buffer_offset += size;
-	}
-}
-
-/**
- * binder_alloc_free_buf() - free a binder buffer
- * @alloc:	binder_alloc for this proc
- * @buffer:	kernel pointer to buffer
- *
- * Free the buffer allocated via binder_alloc_new_buf()
- */
-void binder_alloc_free_buf(struct binder_alloc *alloc,
-			    struct binder_buffer *buffer)
-{
-	/*
-	 * We could eliminate the call to binder_alloc_clear_buf()
-	 * from binder_alloc_deferred_release() by moving this to
-	 * binder_free_buf_locked(). However, that could
-	 * increase contention for the alloc mutex if clear_on_free
-	 * is used frequently for large buffers. The mutex is not
-	 * needed for correctness here.
-	 */
-	if (buffer->clear_on_free) {
-		binder_alloc_clear_buf(alloc, buffer);
-		buffer->clear_on_free = false;
-	}
-	mutex_lock(&alloc->mutex);
-	binder_free_buf_locked(alloc, buffer);
-	mutex_unlock(&alloc->mutex);
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_alloc_free_buf);
-
-/**
- * binder_alloc_mmap_handler() - map virtual address space for proc
- * @alloc:	alloc structure for this proc
- * @vma:	vma passed to mmap()
- *
- * Called by binder_mmap() to initialize the space specified in
- * vma for allocating binder buffers
- *
- * Return:
- *      0 = success
- *      -EBUSY = address space already mapped
- *      -ENOMEM = failed to map memory to given address space
- */
-int binder_alloc_mmap_handler(struct binder_alloc *alloc,
-			      struct vm_area_struct *vma)
-{
-	struct binder_buffer *buffer;
-	const char *failure_string;
-	int ret;
-
-	if (unlikely(vma->vm_mm != alloc->mm)) {
-		ret = -EINVAL;
-		failure_string = "invalid vma->vm_mm";
-		goto err_invalid_mm;
-	}
-
-	mutex_lock(&binder_alloc_mmap_lock);
-	if (alloc->buffer_size) {
-		ret = -EBUSY;
-		failure_string = "already mapped";
-		goto err_already_mapped;
-	}
-	alloc->buffer_size = min_t(unsigned long, vma->vm_end - vma->vm_start,
-				   SZ_4M);
-	mutex_unlock(&binder_alloc_mmap_lock);
-
-	alloc->vm_start = vma->vm_start;
-
-	alloc->pages = kvzalloc_objs(alloc->pages[0],
-				     alloc->buffer_size / PAGE_SIZE);
-	if (!alloc->pages) {
-		ret = -ENOMEM;
-		failure_string = "alloc page array";
-		goto err_alloc_pages_failed;
-	}
-
-	buffer = kzalloc_obj(*buffer);
-	if (!buffer) {
-		ret = -ENOMEM;
-		failure_string = "alloc buffer struct";
-		goto err_alloc_buf_struct_failed;
-	}
-
-	buffer->user_data = alloc->vm_start;
-	list_add(&buffer->entry, &alloc->buffers);
-	buffer->free = 1;
-	binder_insert_free_buffer(alloc, buffer);
-	alloc->free_async_space = alloc->buffer_size / 2;
-
-	/* Signal binder_alloc is fully initialized */
-	binder_alloc_set_mapped(alloc, true);
-
-	return 0;
-
-err_alloc_buf_struct_failed:
-	kvfree(alloc->pages);
-	alloc->pages = NULL;
-err_alloc_pages_failed:
-	alloc->vm_start = 0;
-	mutex_lock(&binder_alloc_mmap_lock);
-	alloc->buffer_size = 0;
-err_already_mapped:
-	mutex_unlock(&binder_alloc_mmap_lock);
-err_invalid_mm:
-	binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
-			   "%s: %d %lx-%lx %s failed %d\n", __func__,
-			   alloc->pid, vma->vm_start, vma->vm_end,
-			   failure_string, ret);
-	return ret;
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_alloc_mmap_handler);
-
-void binder_alloc_deferred_release(struct binder_alloc *alloc)
-{
-	struct rb_node *n;
-	int buffers, page_count;
-	struct binder_buffer *buffer;
-
-	buffers = 0;
-	mutex_lock(&alloc->mutex);
-	BUG_ON(alloc->mapped);
-
-	while ((n = rb_first(&alloc->allocated_buffers))) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-
-		/* Transaction should already have been freed */
-		BUG_ON(buffer->transaction);
-
-		if (buffer->clear_on_free) {
-			binder_alloc_clear_buf(alloc, buffer);
-			buffer->clear_on_free = false;
-		}
-		binder_free_buf_locked(alloc, buffer);
-		buffers++;
-	}
-
-	while (!list_empty(&alloc->buffers)) {
-		buffer = list_first_entry(&alloc->buffers,
-					  struct binder_buffer, entry);
-		WARN_ON(!buffer->free);
-
-		list_del(&buffer->entry);
-		WARN_ON_ONCE(!list_empty(&alloc->buffers));
-		kfree(buffer);
-	}
-
-	page_count = 0;
-	if (alloc->pages) {
-		int i;
-
-		for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
-			struct page *page;
-			bool on_lru;
-
-			page = binder_get_installed_page(alloc, i);
-			if (!page)
-				continue;
-
-			on_lru = list_lru_del(alloc->freelist,
-					      page_to_lru(page),
-					      page_to_nid(page),
-					      NULL);
-			binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
-				     "%s: %d: page %d %s\n",
-				     __func__, alloc->pid, i,
-				     on_lru ? "on lru" : "active");
-			binder_free_page(page);
-			page_count++;
-		}
-	}
-	mutex_unlock(&alloc->mutex);
-	kvfree(alloc->pages);
-	if (alloc->mm)
-		mmdrop(alloc->mm);
-
-	binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE,
-		     "%s: %d buffers %d, pages %d\n",
-		     __func__, alloc->pid, buffers, page_count);
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_alloc_deferred_release);
-
-/**
- * binder_alloc_print_allocated() - print buffer info
- * @m:     seq_file for output via seq_printf()
- * @alloc: binder_alloc for this proc
- *
- * Prints information about every buffer associated with
- * the binder_alloc state to the given seq_file
- */
-void binder_alloc_print_allocated(struct seq_file *m,
-				  struct binder_alloc *alloc)
-{
-	struct binder_buffer *buffer;
-	struct rb_node *n;
-
-	guard(mutex)(&alloc->mutex);
-	for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) {
-		buffer = rb_entry(n, struct binder_buffer, rb_node);
-		seq_printf(m, "  buffer %d: %lx size %zd:%zd:%zd %s\n",
-			   buffer->debug_id,
-			   buffer->user_data - alloc->vm_start,
-			   buffer->data_size, buffer->offsets_size,
-			   buffer->extra_buffers_size,
-			   buffer->transaction ? "active" : "delivered");
-	}
-}
-
-/**
- * binder_alloc_print_pages() - print page usage
- * @m:     seq_file for output via seq_printf()
- * @alloc: binder_alloc for this proc
- */
-void binder_alloc_print_pages(struct seq_file *m,
-			      struct binder_alloc *alloc)
-{
-	struct page *page;
-	int i;
-	int active = 0;
-	int lru = 0;
-	int free = 0;
-
-	mutex_lock(&alloc->mutex);
-	/*
-	 * Make sure the binder_alloc is fully initialized, otherwise we might
-	 * read inconsistent state.
-	 */
-	if (binder_alloc_is_mapped(alloc)) {
-		for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
-			page = binder_get_installed_page(alloc, i);
-			if (!page)
-				free++;
-			else if (list_empty(page_to_lru(page)))
-				active++;
-			else
-				lru++;
-		}
-	}
-	mutex_unlock(&alloc->mutex);
-	seq_printf(m, "  pages: %d:%d:%d\n", active, lru, free);
-	seq_printf(m, "  pages high watermark: %zu\n", alloc->pages_high);
-}
-
-/**
- * binder_alloc_get_allocated_count() - return count of buffers
- * @alloc: binder_alloc for this proc
- *
- * Return: count of allocated buffers
- */
-int binder_alloc_get_allocated_count(struct binder_alloc *alloc)
-{
-	struct rb_node *n;
-	int count = 0;
-
-	guard(mutex)(&alloc->mutex);
-	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))
-		count++;
-	return count;
-}
-
-
-/**
- * binder_alloc_vma_close() - invalidate address space
- * @alloc: binder_alloc for this proc
- *
- * Called from binder_vma_close() when releasing address space.
- * Clears alloc->mapped to prevent new incoming transactions from
- * allocating more buffers.
- */
-void binder_alloc_vma_close(struct binder_alloc *alloc)
-{
-	binder_alloc_set_mapped(alloc, false);
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_alloc_vma_close);
-
-/**
- * binder_alloc_free_page() - shrinker callback to free pages
- * @item:   item to free
- * @lru:    list_lru instance of the item
- * @cb_arg: callback argument
- *
- * Called from list_lru_walk() in binder_shrink_scan() to free
- * up pages when the system is under memory pressure.
- */
-enum lru_status binder_alloc_free_page(struct list_head *item,
-				       struct list_lru_one *lru,
-				       void *cb_arg)
-	__must_hold(&lru->lock)
-{
-	struct binder_shrinker_mdata *mdata = container_of(item, typeof(*mdata), lru);
-	struct binder_alloc *alloc = mdata->alloc;
-	struct mm_struct *mm = alloc->mm;
-	struct vm_area_struct *vma;
-	struct page *page_to_free;
-	unsigned long page_addr;
-	int mm_locked = 0;
-	size_t index;
-
-	if (!mmget_not_zero(mm))
-		goto err_mmget;
-
-	index = mdata->page_index;
-	page_addr = alloc->vm_start + index * PAGE_SIZE;
-
-	/* attempt per-vma lock first */
-	vma = lock_vma_under_rcu(mm, page_addr);
-	if (!vma) {
-		/* fall back to mmap_lock */
-		if (!mmap_read_trylock(mm))
-			goto err_mmap_read_lock_failed;
-		mm_locked = 1;
-		vma = vma_lookup(mm, page_addr);
-	}
-
-	if (!mutex_trylock(&alloc->mutex))
-		goto err_get_alloc_mutex_failed;
-
-	/*
-	 * Since a binder_alloc can only be mapped once, we ensure
-	 * the vma corresponds to this mapping by checking whether
-	 * the binder_alloc is still mapped.
-	 */
-	if (vma && !binder_alloc_is_mapped(alloc))
-		goto err_invalid_vma;
-
-	trace_binder_unmap_kernel_start(alloc, index);
-
-	page_to_free = alloc->pages[index];
-	binder_set_installed_page(alloc, index, NULL);
-
-	trace_binder_unmap_kernel_end(alloc, index);
-
-	list_lru_isolate(lru, item);
-	spin_unlock(&lru->lock);
-
-	if (vma) {
-		trace_binder_unmap_user_start(alloc, index);
-
-		zap_vma_range(vma, page_addr, PAGE_SIZE);
-
-		trace_binder_unmap_user_end(alloc, index);
-	}
-
-	mutex_unlock(&alloc->mutex);
-	if (mm_locked)
-		mmap_read_unlock(mm);
-	else
-		vma_end_read(vma);
-	mmput_async(mm);
-	binder_free_page(page_to_free);
-
-	return LRU_REMOVED_RETRY;
-
-err_invalid_vma:
-	mutex_unlock(&alloc->mutex);
-err_get_alloc_mutex_failed:
-	if (mm_locked)
-		mmap_read_unlock(mm);
-	else
-		vma_end_read(vma);
-err_mmap_read_lock_failed:
-	mmput_async(mm);
-err_mmget:
-	return LRU_SKIP;
-}
-EXPORT_SYMBOL_IF_KUNIT(binder_alloc_free_page);
-
-static unsigned long
-binder_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
-{
-	return list_lru_count(&binder_freelist);
-}
-
-static unsigned long
-binder_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
-{
-	return list_lru_walk(&binder_freelist, binder_alloc_free_page,
-			    NULL, sc->nr_to_scan);
-}
-
-static struct shrinker *binder_shrinker;
-
-VISIBLE_IF_KUNIT void __binder_alloc_init(struct binder_alloc *alloc,
-					  struct list_lru *freelist)
-{
-	alloc->pid = current->tgid;
-	alloc->mm = current->mm;
-	mmgrab(alloc->mm);
-	mutex_init(&alloc->mutex);
-	INIT_LIST_HEAD(&alloc->buffers);
-	alloc->freelist = freelist;
-}
-EXPORT_SYMBOL_IF_KUNIT(__binder_alloc_init);
-
-/**
- * binder_alloc_init() - called by binder_open() for per-proc initialization
- * @alloc: binder_alloc for this proc
- *
- * Called from binder_open() to initialize binder_alloc fields for
- * new binder proc
- */
-void binder_alloc_init(struct binder_alloc *alloc)
-{
-	__binder_alloc_init(alloc, &binder_freelist);
-}
-
-int binder_alloc_shrinker_init(void)
-{
-	int ret;
-
-	ret = list_lru_init(&binder_freelist);
-	if (ret)
-		return ret;
-
-	binder_shrinker = shrinker_alloc(0, "android-binder");
-	if (!binder_shrinker) {
-		list_lru_destroy(&binder_freelist);
-		return -ENOMEM;
-	}
-
-	binder_shrinker->count_objects = binder_shrink_count;
-	binder_shrinker->scan_objects = binder_shrink_scan;
-
-	shrinker_register(binder_shrinker);
-
-	return 0;
-}
-
-void binder_alloc_shrinker_exit(void)
-{
-	shrinker_free(binder_shrinker);
-	list_lru_destroy(&binder_freelist);
-}
-
-/**
- * check_buffer() - verify that buffer/offset is safe to access
- * @alloc: binder_alloc for this proc
- * @buffer: binder buffer to be accessed
- * @offset: offset into @buffer data
- * @bytes: bytes to access from offset
- *
- * Check that the @offset/@bytes are within the size of the given
- * @buffer and that the buffer is currently active and not freeable.
- * Offsets must also be multiples of sizeof(u32). The kernel is
- * allowed to touch the buffer in two cases:
- *
- * 1) when the buffer is being created:
- *     (buffer->free == 0 && buffer->allow_user_free == 0)
- * 2) when the buffer is being torn down:
- *     (buffer->free == 0 && buffer->transaction == NULL).
- *
- * Return: true if the buffer is safe to access
- */
-static inline bool check_buffer(struct binder_alloc *alloc,
-				struct binder_buffer *buffer,
-				binder_size_t offset, size_t bytes)
-{
-	size_t buffer_size = binder_alloc_buffer_size(alloc, buffer);
-
-	return buffer_size >= bytes &&
-		offset <= buffer_size - bytes &&
-		IS_ALIGNED(offset, sizeof(u32)) &&
-		!buffer->free &&
-		(!buffer->allow_user_free || !buffer->transaction);
-}
-
-/**
- * binder_alloc_copy_user_to_buffer() - copy src user to tgt user
- * @alloc: binder_alloc for this proc
- * @buffer: binder buffer to be accessed
- * @buffer_offset: offset into @buffer data
- * @from: userspace pointer to source buffer
- * @bytes: bytes to copy
- *
- * Copy bytes from source userspace to target buffer.
- *
- * Return: bytes remaining to be copied
- */
-unsigned long
-binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,
-				 struct binder_buffer *buffer,
-				 binder_size_t buffer_offset,
-				 const void __user *from,
-				 size_t bytes)
-{
-	if (!check_buffer(alloc, buffer, buffer_offset, bytes))
-		return bytes;
-
-	while (bytes) {
-		unsigned long size;
-		unsigned long ret;
-		struct page *page;
-		pgoff_t pgoff;
-		void *kptr;
-
-		page = binder_alloc_get_page(alloc, buffer,
-					     buffer_offset, &pgoff);
-		size = min_t(size_t, bytes, PAGE_SIZE - pgoff);
-		kptr = kmap_local_page(page) + pgoff;
-		ret = copy_from_user(kptr, from, size);
-		kunmap_local(kptr);
-		if (ret)
-			return bytes - size + ret;
-		bytes -= size;
-		from += size;
-		buffer_offset += size;
-	}
-	return 0;
-}
-
-static int binder_alloc_do_buffer_copy(struct binder_alloc *alloc,
-				       bool to_buffer,
-				       struct binder_buffer *buffer,
-				       binder_size_t buffer_offset,
-				       void *ptr,
-				       size_t bytes)
-{
-	/* All copies must be 32-bit aligned and 32-bit size */
-	if (!check_buffer(alloc, buffer, buffer_offset, bytes))
-		return -EINVAL;
-
-	while (bytes) {
-		unsigned long size;
-		struct page *page;
-		pgoff_t pgoff;
-
-		page = binder_alloc_get_page(alloc, buffer,
-					     buffer_offset, &pgoff);
-		size = min_t(size_t, bytes, PAGE_SIZE - pgoff);
-		if (to_buffer)
-			memcpy_to_page(page, pgoff, ptr, size);
-		else
-			memcpy_from_page(ptr, page, pgoff, size);
-		bytes -= size;
-		pgoff = 0;
-		ptr = ptr + size;
-		buffer_offset += size;
-	}
-	return 0;
-}
-
-int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
-				struct binder_buffer *buffer,
-				binder_size_t buffer_offset,
-				void *src,
-				size_t bytes)
-{
-	return binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset,
-					   src, bytes);
-}
-
-int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
-				  void *dest,
-				  struct binder_buffer *buffer,
-				  binder_size_t buffer_offset,
-				  size_t bytes)
-{
-	return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,
-					   dest, bytes);
-}
diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h
deleted file mode 100644
index d6f1f6f2d00e7c..00000000000000
--- a/drivers/android/binder_alloc.h
+++ /dev/null
@@ -1,189 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2017 Google, Inc.
- */
-
-#ifndef _LINUX_BINDER_ALLOC_H
-#define _LINUX_BINDER_ALLOC_H
-
-#include <linux/rbtree.h>
-#include <linux/list.h>
-#include <linux/mm.h>
-#include <linux/rtmutex.h>
-#include <linux/vmalloc.h>
-#include <linux/slab.h>
-#include <linux/list_lru.h>
-#include <uapi/linux/android/binder.h>
-
-struct binder_transaction;
-
-/**
- * struct binder_buffer - buffer used for binder transactions
- * @entry:              entry alloc->buffers
- * @rb_node:            node for allocated_buffers/free_buffers rb trees
- * @free:               %true if buffer is free
- * @clear_on_free:      %true if buffer must be zeroed after use
- * @allow_user_free:    %true if user is allowed to free buffer
- * @async_transaction:  %true if buffer is in use for an async txn
- * @oneway_spam_suspect: %true if total async allocate size just exceed
- * spamming detect threshold
- * @debug_id:           unique ID for debugging
- * @transaction:        pointer to associated struct binder_transaction
- * @target_node:        struct binder_node associated with this buffer
- * @data_size:          size of @transaction data
- * @offsets_size:       size of array of offsets
- * @extra_buffers_size: size of space for other objects (like sg lists)
- * @user_data:          user pointer to base of buffer space
- * @pid:                pid to attribute the buffer to (caller)
- *
- * Bookkeeping structure for binder transaction buffers
- */
-struct binder_buffer {
-	struct list_head entry; /* free and allocated entries by address */
-	struct rb_node rb_node; /* free entry by size or allocated entry */
-				/* by address */
-	unsigned free:1;
-	unsigned clear_on_free:1;
-	unsigned allow_user_free:1;
-	unsigned async_transaction:1;
-	unsigned oneway_spam_suspect:1;
-	unsigned debug_id:27;
-	struct binder_transaction *transaction;
-	struct binder_node *target_node;
-	size_t data_size;
-	size_t offsets_size;
-	size_t extra_buffers_size;
-	unsigned long user_data;
-	int pid;
-};
-
-/**
- * struct binder_shrinker_mdata - binder metadata used to reclaim pages
- * @lru:         LRU entry in binder_freelist
- * @alloc:       binder_alloc owning the page to reclaim
- * @page_index:  offset in @alloc->pages[] into the page to reclaim
- */
-struct binder_shrinker_mdata {
-	struct list_head lru;
-	struct binder_alloc *alloc;
-	unsigned long page_index;
-};
-
-static inline struct list_head *page_to_lru(struct page *p)
-{
-	struct binder_shrinker_mdata *mdata;
-
-	mdata = (struct binder_shrinker_mdata *)page_private(p);
-
-	return &mdata->lru;
-}
-
-/**
- * struct binder_alloc - per-binder proc state for binder allocator
- * @mutex:              protects binder_alloc fields
- * @mm:                 copy of task->mm (invariant after open)
- * @vm_start:           base of per-proc address space mapped via mmap
- * @buffers:            list of all buffers for this proc
- * @free_buffers:       rb tree of buffers available for allocation
- *                      sorted by size
- * @allocated_buffers:  rb tree of allocated buffers sorted by address
- * @free_async_space:   VA space available for async buffers. This is
- *                      initialized at mmap time to 1/2 the full VA space
- * @pages:              array of struct page *
- * @freelist:           lru list to use for free pages (invariant after init)
- * @buffer_size:        size of address space specified via mmap
- * @pid:                pid for associated binder_proc (invariant after init)
- * @pages_high:         high watermark of offset in @pages
- * @mapped:             whether the vm area is mapped, each binder instance is
- *                      allowed a single mapping throughout its lifetime
- * @oneway_spam_detected: %true if oneway spam detection fired, clear that
- * flag once the async buffer has returned to a healthy state
- *
- * Bookkeeping structure for per-proc address space management for binder
- * buffers. It is normally initialized during binder_init() and binder_mmap()
- * calls. The address space is used for both user-visible buffers and for
- * struct binder_buffer objects used to track the user buffers
- */
-struct binder_alloc {
-	struct mutex mutex;
-	struct mm_struct *mm;
-	unsigned long vm_start;
-	struct list_head buffers;
-	struct rb_root free_buffers;
-	struct rb_root allocated_buffers;
-	size_t free_async_space;
-	struct page **pages;
-	struct list_lru *freelist;
-	size_t buffer_size;
-	int pid;
-	size_t pages_high;
-	bool mapped;
-	bool oneway_spam_detected;
-};
-
-enum lru_status binder_alloc_free_page(struct list_head *item,
-				       struct list_lru_one *lru,
-				       void *cb_arg);
-struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,
-					   size_t data_size,
-					   size_t offsets_size,
-					   size_t extra_buffers_size,
-					   int is_async);
-void binder_alloc_init(struct binder_alloc *alloc);
-int binder_alloc_shrinker_init(void);
-void binder_alloc_shrinker_exit(void);
-void binder_alloc_vma_close(struct binder_alloc *alloc);
-struct binder_buffer *
-binder_alloc_prepare_to_free(struct binder_alloc *alloc,
-			     unsigned long user_ptr);
-void binder_alloc_free_buf(struct binder_alloc *alloc,
-			   struct binder_buffer *buffer);
-int binder_alloc_mmap_handler(struct binder_alloc *alloc,
-			      struct vm_area_struct *vma);
-void binder_alloc_deferred_release(struct binder_alloc *alloc);
-int binder_alloc_get_allocated_count(struct binder_alloc *alloc);
-void binder_alloc_print_allocated(struct seq_file *m,
-				  struct binder_alloc *alloc);
-void binder_alloc_print_pages(struct seq_file *m,
-			      struct binder_alloc *alloc);
-
-/**
- * binder_alloc_get_free_async_space() - get free space available for async
- * @alloc:	binder_alloc for this proc
- *
- * Return:	the bytes remaining in the address-space for async transactions
- */
-static inline size_t
-binder_alloc_get_free_async_space(struct binder_alloc *alloc)
-{
-	guard(mutex)(&alloc->mutex);
-	return alloc->free_async_space;
-}
-
-unsigned long
-binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,
-				 struct binder_buffer *buffer,
-				 binder_size_t buffer_offset,
-				 const void __user *from,
-				 size_t bytes);
-
-int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
-				struct binder_buffer *buffer,
-				binder_size_t buffer_offset,
-				void *src,
-				size_t bytes);
-
-int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
-				  void *dest,
-				  struct binder_buffer *buffer,
-				  binder_size_t buffer_offset,
-				  size_t bytes);
-
-#if IS_ENABLED(CONFIG_KUNIT)
-void __binder_alloc_init(struct binder_alloc *alloc, struct list_lru *freelist);
-size_t binder_alloc_buffer_size(struct binder_alloc *alloc,
-				struct binder_buffer *buffer);
-#endif
-
-#endif /* _LINUX_BINDER_ALLOC_H */
-
diff --git a/drivers/android/binder_internal.h b/drivers/android/binder_internal.h
deleted file mode 100644
index 342574bfd28a88..00000000000000
--- a/drivers/android/binder_internal.h
+++ /dev/null
@@ -1,597 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-#ifndef _LINUX_BINDER_INTERNAL_H
-#define _LINUX_BINDER_INTERNAL_H
-
-#include <linux/fs.h>
-#include <linux/list.h>
-#include <linux/miscdevice.h>
-#include <linux/mutex.h>
-#include <linux/refcount.h>
-#include <linux/stddef.h>
-#include <linux/types.h>
-#include <linux/uidgid.h>
-#include <uapi/linux/android/binderfs.h>
-#include "binder_alloc.h"
-#include "dbitmap.h"
-
-struct binder_context {
-	struct binder_node *binder_context_mgr_node;
-	struct mutex context_mgr_node_lock;
-	kuid_t binder_context_mgr_uid;
-	const char *name;
-};
-
-/**
- * struct binder_device - information about a binder device node
- * @hlist:          list of binder devices
- * @miscdev:        information about a binder character device node
- * @context:        binder context information
- * @binderfs_inode: This is the inode of the root dentry of the super block
- *                  belonging to a binderfs mount.
- */
-struct binder_device {
-	struct hlist_node hlist;
-	struct miscdevice miscdev;
-	struct binder_context context;
-	struct inode *binderfs_inode;
-	refcount_t ref;
-};
-
-/**
- * binderfs_mount_opts - mount options for binderfs
- * @max: maximum number of allocatable binderfs binder devices
- * @stats_mode: enable binder stats in binderfs.
- */
-struct binderfs_mount_opts {
-	int max;
-	int stats_mode;
-};
-
-/**
- * binderfs_info - information about a binderfs mount
- * @ipc_ns:         The ipc namespace the binderfs mount belongs to.
- * @control_dentry: This records the dentry of this binderfs mount
- *                  binder-control device.
- * @root_uid:       uid that needs to be used when a new binder device is
- *                  created.
- * @root_gid:       gid that needs to be used when a new binder device is
- *                  created.
- * @mount_opts:     The mount options in use.
- * @device_count:   The current number of allocated binder devices.
- * @proc_log_dir:   Pointer to the directory dentry containing process-specific
- *                  logs.
- */
-struct binderfs_info {
-	struct ipc_namespace *ipc_ns;
-	struct dentry *control_dentry;
-	kuid_t root_uid;
-	kgid_t root_gid;
-	struct binderfs_mount_opts mount_opts;
-	int device_count;
-	struct dentry *proc_log_dir;
-};
-
-extern const struct file_operations binder_fops;
-
-extern char *binder_devices_param;
-
-#ifdef CONFIG_ANDROID_BINDERFS
-extern bool is_binderfs_device(const struct inode *inode);
-extern struct dentry *binderfs_create_file(struct dentry *dir, const char *name,
-					   const struct file_operations *fops,
-					   void *data);
-#else
-static inline bool is_binderfs_device(const struct inode *inode)
-{
-	return false;
-}
-static inline struct dentry *binderfs_create_file(struct dentry *dir,
-					   const char *name,
-					   const struct file_operations *fops,
-					   void *data)
-{
-	return NULL;
-}
-#endif
-
-#ifdef CONFIG_ANDROID_BINDERFS
-extern int __init init_binderfs(void);
-#else
-static inline int __init init_binderfs(void)
-{
-	return 0;
-}
-#endif
-
-struct binder_debugfs_entry {
-	const char *name;
-	umode_t mode;
-	const struct file_operations *fops;
-	void *data;
-};
-
-extern const struct binder_debugfs_entry binder_debugfs_entries[];
-
-#define binder_for_each_debugfs_entry(entry)	\
-	for ((entry) = binder_debugfs_entries;	\
-	     (entry)->name;			\
-	     (entry)++)
-
-enum binder_stat_types {
-	BINDER_STAT_PROC,
-	BINDER_STAT_THREAD,
-	BINDER_STAT_NODE,
-	BINDER_STAT_REF,
-	BINDER_STAT_DEATH,
-	BINDER_STAT_TRANSACTION,
-	BINDER_STAT_TRANSACTION_COMPLETE,
-	BINDER_STAT_FREEZE,
-	BINDER_STAT_COUNT
-};
-
-struct binder_stats {
-	atomic_t br[_IOC_NR(BR_CLEAR_FREEZE_NOTIFICATION_DONE) + 1];
-	atomic_t bc[_IOC_NR(BC_FREEZE_NOTIFICATION_DONE) + 1];
-	atomic_t obj_created[BINDER_STAT_COUNT];
-	atomic_t obj_deleted[BINDER_STAT_COUNT];
-};
-
-/**
- * struct binder_work - work enqueued on a worklist
- * @entry:             node enqueued on list
- * @type:              type of work to be performed
- *
- * There are separate work lists for proc, thread, and node (async).
- */
-struct binder_work {
-	struct list_head entry;
-
-	enum binder_work_type {
-		BINDER_WORK_TRANSACTION = 1,
-		BINDER_WORK_TRANSACTION_COMPLETE,
-		BINDER_WORK_TRANSACTION_PENDING,
-		BINDER_WORK_TRANSACTION_ONEWAY_SPAM_SUSPECT,
-		BINDER_WORK_RETURN_ERROR,
-		BINDER_WORK_NODE,
-		BINDER_WORK_DEAD_BINDER,
-		BINDER_WORK_DEAD_BINDER_AND_CLEAR,
-		BINDER_WORK_CLEAR_DEATH_NOTIFICATION,
-		BINDER_WORK_FROZEN_BINDER,
-		BINDER_WORK_CLEAR_FREEZE_NOTIFICATION,
-	} type;
-};
-
-struct binder_error {
-	struct binder_work work;
-	uint32_t cmd;
-};
-
-/**
- * struct binder_node - binder node bookkeeping
- * @debug_id:             unique ID for debugging
- *                        (invariant after initialized)
- * @lock:                 lock for node fields
- * @work:                 worklist element for node work
- *                        (protected by @proc->inner_lock)
- * @rb_node:              element for proc->nodes tree
- *                        (protected by @proc->inner_lock)
- * @dead_node:            element for binder_dead_nodes list
- *                        (protected by binder_dead_nodes_lock)
- * @proc:                 binder_proc that owns this node
- *                        (invariant after initialized)
- * @refs:                 list of references on this node
- *                        (protected by @lock)
- * @internal_strong_refs: used to take strong references when
- *                        initiating a transaction
- *                        (protected by @proc->inner_lock if @proc
- *                        and by @lock)
- * @local_weak_refs:      weak user refs from local process
- *                        (protected by @proc->inner_lock if @proc
- *                        and by @lock)
- * @local_strong_refs:    strong user refs from local process
- *                        (protected by @proc->inner_lock if @proc
- *                        and by @lock)
- * @tmp_refs:             temporary kernel refs
- *                        (protected by @proc->inner_lock while @proc
- *                        is valid, and by binder_dead_nodes_lock
- *                        if @proc is NULL. During inc/dec and node release
- *                        it is also protected by @lock to provide safety
- *                        as the node dies and @proc becomes NULL)
- * @ptr:                  userspace pointer for node
- *                        (invariant, no lock needed)
- * @cookie:               userspace cookie for node
- *                        (invariant, no lock needed)
- * @has_strong_ref:       userspace notified of strong ref
- *                        (protected by @proc->inner_lock if @proc
- *                        and by @lock)
- * @pending_strong_ref:   userspace has acked notification of strong ref
- *                        (protected by @proc->inner_lock if @proc
- *                        and by @lock)
- * @has_weak_ref:         userspace notified of weak ref
- *                        (protected by @proc->inner_lock if @proc
- *                        and by @lock)
- * @pending_weak_ref:     userspace has acked notification of weak ref
- *                        (protected by @proc->inner_lock if @proc
- *                        and by @lock)
- * @has_async_transaction: async transaction to node in progress
- *                        (protected by @lock)
- * @accept_fds:           file descriptor operations supported for node
- *                        (invariant after initialized)
- * @min_priority:         minimum scheduling priority
- *                        (invariant after initialized)
- * @txn_security_ctx:     require sender's security context
- *                        (invariant after initialized)
- * @async_todo:           list of async work items
- *                        (protected by @proc->inner_lock)
- *
- * Bookkeeping structure for binder nodes.
- */
-struct binder_node {
-	int debug_id;
-	spinlock_t lock;
-	struct binder_work work;
-	union {
-		struct rb_node rb_node;
-		struct hlist_node dead_node;
-	};
-	struct binder_proc *proc;
-	struct hlist_head refs;
-	int internal_strong_refs;
-	int local_weak_refs;
-	int local_strong_refs;
-	int tmp_refs;
-	binder_uintptr_t ptr;
-	binder_uintptr_t cookie;
-	struct {
-		/*
-		 * bitfield elements protected by
-		 * proc inner_lock
-		 */
-		u8 has_strong_ref:1;
-		u8 pending_strong_ref:1;
-		u8 has_weak_ref:1;
-		u8 pending_weak_ref:1;
-	};
-	struct {
-		/*
-		 * invariant after initialization
-		 */
-		u8 accept_fds:1;
-		u8 txn_security_ctx:1;
-		u8 min_priority;
-	};
-	bool has_async_transaction;
-	struct list_head async_todo;
-};
-
-struct binder_ref_death {
-	/**
-	 * @work: worklist element for death notifications
-	 *        (protected by inner_lock of the proc that
-	 *        this ref belongs to)
-	 */
-	struct binder_work work;
-	binder_uintptr_t cookie;
-};
-
-struct binder_ref_freeze {
-	struct binder_work work;
-	binder_uintptr_t cookie;
-	bool is_frozen:1;
-	bool sent:1;
-	bool resend:1;
-};
-
-/**
- * struct binder_ref_data - binder_ref counts and id
- * @debug_id:        unique ID for the ref
- * @desc:            unique userspace handle for ref
- * @strong:          strong ref count (debugging only if not locked)
- * @weak:            weak ref count (debugging only if not locked)
- *
- * Structure to hold ref count and ref id information. Since
- * the actual ref can only be accessed with a lock, this structure
- * is used to return information about the ref to callers of
- * ref inc/dec functions.
- */
-struct binder_ref_data {
-	int debug_id;
-	uint32_t desc;
-	int strong;
-	int weak;
-};
-
-/**
- * struct binder_ref - struct to track references on nodes
- * @data:        binder_ref_data containing id, handle, and current refcounts
- * @rb_node_desc: node for lookup by @data.desc in proc's rb_tree
- * @rb_node_node: node for lookup by @node in proc's rb_tree
- * @node_entry:  list entry for node->refs list in target node
- *               (protected by @node->lock)
- * @proc:        binder_proc containing ref
- * @node:        binder_node of target node. When cleaning up a
- *               ref for deletion in binder_cleanup_ref, a non-NULL
- *               @node indicates the node must be freed
- * @death:       pointer to death notification (ref_death) if requested
- *               (protected by @node->lock)
- * @freeze:      pointer to freeze notification (ref_freeze) if requested
- *               (protected by @node->lock)
- *
- * Structure to track references from procA to target node (on procB). This
- * structure is unsafe to access without holding @proc->outer_lock.
- */
-struct binder_ref {
-	/* Lookups needed: */
-	/*   node + proc => ref (transaction) */
-	/*   desc + proc => ref (transaction, inc/dec ref) */
-	/*   node => refs + procs (proc exit) */
-	struct binder_ref_data data;
-	struct rb_node rb_node_desc;
-	struct rb_node rb_node_node;
-	struct hlist_node node_entry;
-	struct binder_proc *proc;
-	struct binder_node *node;
-	struct binder_ref_death *death;
-	struct binder_ref_freeze *freeze;
-};
-
-/**
- * struct binder_proc - binder process bookkeeping
- * @proc_node:            element for binder_procs list
- * @threads:              rbtree of binder_threads in this proc
- *                        (protected by @inner_lock)
- * @nodes:                rbtree of binder nodes associated with
- *                        this proc ordered by node->ptr
- *                        (protected by @inner_lock)
- * @refs_by_desc:         rbtree of refs ordered by ref->desc
- *                        (protected by @outer_lock)
- * @refs_by_node:         rbtree of refs ordered by ref->node
- *                        (protected by @outer_lock)
- * @waiting_threads:      threads currently waiting for proc work
- *                        (protected by @inner_lock)
- * @pid                   PID of group_leader of process
- *                        (invariant after initialized)
- * @tsk                   task_struct for group_leader of process
- *                        (invariant after initialized)
- * @cred                  struct cred associated with the `struct file`
- *                        in binder_open()
- *                        (invariant after initialized)
- * @deferred_work_node:   element for binder_deferred_list
- *                        (protected by binder_deferred_lock)
- * @deferred_work:        bitmap of deferred work to perform
- *                        (protected by binder_deferred_lock)
- * @outstanding_txns:     number of transactions to be transmitted before
- *                        processes in freeze_wait are woken up
- *                        (protected by @inner_lock)
- * @is_dead:              process is dead and awaiting free
- *                        when outstanding transactions are cleaned up
- *                        (protected by @inner_lock)
- * @is_frozen:            process is frozen and unable to service
- *                        binder transactions
- *                        (protected by @inner_lock)
- * @sync_recv:            process received sync transactions since last frozen
- *                        bit 0: received sync transaction after being frozen
- *                        bit 1: new pending sync transaction during freezing
- *                        (protected by @inner_lock)
- * @async_recv:           process received async transactions since last frozen
- *                        (protected by @inner_lock)
- * @freeze_wait:          waitqueue of processes waiting for all outstanding
- *                        transactions to be processed
- *                        (protected by @inner_lock)
- * @dmap                  dbitmap to manage available reference descriptors
- *                        (protected by @outer_lock)
- * @todo:                 list of work for this process
- *                        (protected by @inner_lock)
- * @stats:                per-process binder statistics
- *                        (atomics, no lock needed)
- * @delivered_death:      list of delivered death notification
- *                        (protected by @inner_lock)
- * @delivered_freeze:     list of delivered freeze notification
- *                        (protected by @inner_lock)
- * @max_threads:          cap on number of binder threads
- *                        (protected by @inner_lock)
- * @requested_threads:    number of binder threads requested but not
- *                        yet started. In current implementation, can
- *                        only be 0 or 1.
- *                        (protected by @inner_lock)
- * @requested_threads_started: number binder threads started
- *                        (protected by @inner_lock)
- * @tmp_ref:              temporary reference to indicate proc is in use
- *                        (protected by @inner_lock)
- * @default_priority:     default scheduler priority
- *                        (invariant after initialized)
- * @debugfs_entry:        debugfs node
- * @alloc:                binder allocator bookkeeping
- * @context:              binder_context for this proc
- *                        (invariant after initialized)
- * @inner_lock:           can nest under outer_lock and/or node lock
- * @outer_lock:           no nesting under innor or node lock
- *                        Lock order: 1) outer, 2) node, 3) inner
- * @binderfs_entry:       process-specific binderfs log file
- * @oneway_spam_detection_enabled: process enabled oneway spam detection
- *                        or not
- *
- * Bookkeeping structure for binder processes
- */
-struct binder_proc {
-	struct hlist_node proc_node;
-	struct rb_root threads;
-	struct rb_root nodes;
-	struct rb_root refs_by_desc;
-	struct rb_root refs_by_node;
-	struct list_head waiting_threads;
-	int pid;
-	struct task_struct *tsk;
-	const struct cred *cred;
-	struct hlist_node deferred_work_node;
-	int deferred_work;
-	int outstanding_txns;
-	bool is_dead;
-	bool is_frozen;
-	bool sync_recv;
-	bool async_recv;
-	wait_queue_head_t freeze_wait;
-	struct dbitmap dmap;
-	struct list_head todo;
-	struct binder_stats stats;
-	struct list_head delivered_death;
-	struct list_head delivered_freeze;
-	u32 max_threads;
-	int requested_threads;
-	int requested_threads_started;
-	int tmp_ref;
-	long default_priority;
-	struct dentry *debugfs_entry;
-	struct binder_alloc alloc;
-	struct binder_context *context;
-	spinlock_t inner_lock;
-	spinlock_t outer_lock;
-	struct dentry *binderfs_entry;
-	bool oneway_spam_detection_enabled;
-};
-
-/**
- * struct binder_thread - binder thread bookkeeping
- * @proc:                 binder process for this thread
- *                        (invariant after initialization)
- * @rb_node:              element for proc->threads rbtree
- *                        (protected by @proc->inner_lock)
- * @waiting_thread_node:  element for @proc->waiting_threads list
- *                        (protected by @proc->inner_lock)
- * @pid:                  PID for this thread
- *                        (invariant after initialization)
- * @looper:               bitmap of looping state
- *                        (only accessed by this thread)
- * @looper_needs_return:  looping thread needs to exit driver
- *                        (no lock needed)
- * @transaction_stack:    stack of in-progress transactions for this thread
- *                        (protected by @proc->inner_lock)
- * @todo:                 list of work to do for this thread
- *                        (protected by @proc->inner_lock)
- * @process_todo:         whether work in @todo should be processed
- *                        (protected by @proc->inner_lock)
- * @return_error:         transaction errors reported by this thread
- *                        (only accessed by this thread)
- * @reply_error:          transaction errors reported by target thread
- *                        (protected by @proc->inner_lock)
- * @ee:                   extended error information from this thread
- *                        (protected by @proc->inner_lock)
- * @wait:                 wait queue for thread work
- * @stats:                per-thread statistics
- *                        (atomics, no lock needed)
- * @tmp_ref:              temporary reference to indicate thread is in use
- *                        (atomic since @proc->inner_lock cannot
- *                        always be acquired)
- * @is_dead:              thread is dead and awaiting free
- *                        when outstanding transactions are cleaned up
- *                        (protected by @proc->inner_lock)
- *
- * Bookkeeping structure for binder threads.
- */
-struct binder_thread {
-	struct binder_proc *proc;
-	struct rb_node rb_node;
-	struct list_head waiting_thread_node;
-	int pid;
-	int looper;              /* only modified by this thread */
-	bool looper_need_return; /* can be written by other thread */
-	struct binder_transaction *transaction_stack;
-	struct list_head todo;
-	bool process_todo;
-	struct binder_error return_error;
-	struct binder_error reply_error;
-	struct binder_extended_error ee;
-	wait_queue_head_t wait;
-	struct binder_stats stats;
-	atomic_t tmp_ref;
-	bool is_dead;
-};
-
-/**
- * struct binder_txn_fd_fixup - transaction fd fixup list element
- * @fixup_entry:          list entry
- * @file:                 struct file to be associated with new fd
- * @offset:               offset in buffer data to this fixup
- * @target_fd:            fd to use by the target to install @file
- *
- * List element for fd fixups in a transaction. Since file
- * descriptors need to be allocated in the context of the
- * target process, we pass each fd to be processed in this
- * struct.
- */
-struct binder_txn_fd_fixup {
-	struct list_head fixup_entry;
-	struct file *file;
-	size_t offset;
-	int target_fd;
-};
-
-struct binder_transaction {
-	int debug_id;
-	struct binder_work work;
-	struct binder_thread *from;
-	pid_t from_pid;
-	pid_t from_tid;
-	struct binder_transaction *from_parent;
-	struct binder_proc *to_proc;
-	struct binder_thread *to_thread;
-	struct binder_transaction *to_parent;
-	unsigned is_async:1;
-	unsigned is_reply:1;
-
-	struct binder_buffer *buffer;
-	unsigned int    code;
-	unsigned int    flags;
-	long    priority;
-	long    saved_priority;
-	kuid_t  sender_euid;
-	ktime_t start_time;
-	struct list_head fd_fixups;
-	binder_uintptr_t security_ctx;
-	/**
-	 * @lock:  protects @from, @to_proc, and @to_thread
-	 *
-	 * @from, @to_proc, and @to_thread can be set to NULL
-	 * during thread teardown
-	 */
-	spinlock_t lock;
-};
-
-/**
- * struct binder_object - union of flat binder object types
- * @hdr:   generic object header
- * @fbo:   binder object (nodes and refs)
- * @fdo:   file descriptor object
- * @bbo:   binder buffer pointer
- * @fdao:  file descriptor array
- *
- * Used for type-independent object copies
- */
-struct binder_object {
-	union {
-		struct binder_object_header hdr;
-		struct flat_binder_object fbo;
-		struct binder_fd_object fdo;
-		struct binder_buffer_object bbo;
-		struct binder_fd_array_object fdao;
-	};
-};
-
-/**
- * Add a binder device to binder_devices
- * @device: the new binder device to add to the global list
- */
-void binder_add_device(struct binder_device *device);
-
-/**
- * Remove a binder device to binder_devices
- * @device: the binder device to remove from the global list
- */
-void binder_remove_device(struct binder_device *device);
-
-#if IS_ENABLED(CONFIG_KUNIT)
-vm_fault_t binder_vm_fault(struct vm_fault *vmf);
-#endif
-
-#endif /* _LINUX_BINDER_INTERNAL_H */
diff --git a/drivers/android/binder_netlink.c b/drivers/android/binder_netlink.c
deleted file mode 100644
index 81e8432b5904fa..00000000000000
--- a/drivers/android/binder_netlink.c
+++ /dev/null
@@ -1,32 +0,0 @@
-// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
-/* Do not edit directly, auto-generated from: */
-/*	Documentation/netlink/specs/binder.yaml */
-/* YNL-GEN kernel source */
-/* To regenerate run: tools/net/ynl/ynl-regen.sh */
-
-#include <net/netlink.h>
-#include <net/genetlink.h>
-
-#include "binder_netlink.h"
-
-#include <uapi/linux/android/binder_netlink.h>
-
-/* Ops table for binder */
-static const struct genl_split_ops binder_nl_ops[] = {
-};
-
-static const struct genl_multicast_group binder_nl_mcgrps[] = {
-	[BINDER_NLGRP_REPORT] = { "report", },
-};
-
-struct genl_family binder_nl_family __ro_after_init = {
-	.name		= BINDER_FAMILY_NAME,
-	.version	= BINDER_FAMILY_VERSION,
-	.netnsok	= true,
-	.parallel_ops	= true,
-	.module		= THIS_MODULE,
-	.split_ops	= binder_nl_ops,
-	.n_split_ops	= ARRAY_SIZE(binder_nl_ops),
-	.mcgrps		= binder_nl_mcgrps,
-	.n_mcgrps	= ARRAY_SIZE(binder_nl_mcgrps),
-};
diff --git a/drivers/android/binder_netlink.h b/drivers/android/binder_netlink.h
deleted file mode 100644
index 57399942a5e3e4..00000000000000
--- a/drivers/android/binder_netlink.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
-/* Do not edit directly, auto-generated from: */
-/*	Documentation/netlink/specs/binder.yaml */
-/* YNL-GEN kernel header */
-/* To regenerate run: tools/net/ynl/ynl-regen.sh */
-
-#ifndef _LINUX_BINDER_GEN_H
-#define _LINUX_BINDER_GEN_H
-
-#include <net/netlink.h>
-#include <net/genetlink.h>
-
-#include <uapi/linux/android/binder_netlink.h>
-
-enum {
-	BINDER_NLGRP_REPORT,
-};
-
-extern struct genl_family binder_nl_family;
-
-#endif /* _LINUX_BINDER_GEN_H */
diff --git a/drivers/android/binder_trace.h b/drivers/android/binder_trace.h
deleted file mode 100644
index fa5eb61cf5805c..00000000000000
--- a/drivers/android/binder_trace.h
+++ /dev/null
@@ -1,448 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2012 Google, Inc.
- */
-
-#undef TRACE_SYSTEM
-#define TRACE_SYSTEM binder
-
-#if !defined(_BINDER_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
-#define _BINDER_TRACE_H
-
-#include <linux/tracepoint.h>
-
-struct binder_buffer;
-struct binder_node;
-struct binder_proc;
-struct binder_alloc;
-struct binder_ref_data;
-struct binder_thread;
-struct binder_transaction;
-
-TRACE_EVENT(binder_ioctl,
-	TP_PROTO(unsigned int cmd, unsigned long arg),
-	TP_ARGS(cmd, arg),
-
-	TP_STRUCT__entry(
-		__field(unsigned int, cmd)
-		__field(unsigned long, arg)
-	),
-	TP_fast_assign(
-		__entry->cmd = cmd;
-		__entry->arg = arg;
-	),
-	TP_printk("cmd=0x%x arg=0x%lx", __entry->cmd, __entry->arg)
-);
-
-DECLARE_EVENT_CLASS(binder_function_return_class,
-	TP_PROTO(int ret),
-	TP_ARGS(ret),
-	TP_STRUCT__entry(
-		__field(int, ret)
-	),
-	TP_fast_assign(
-		__entry->ret = ret;
-	),
-	TP_printk("ret=%d", __entry->ret)
-);
-
-#define DEFINE_BINDER_FUNCTION_RETURN_EVENT(name)	\
-DEFINE_EVENT(binder_function_return_class, name,	\
-	TP_PROTO(int ret), \
-	TP_ARGS(ret))
-
-DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_ioctl_done);
-DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_write_done);
-DEFINE_BINDER_FUNCTION_RETURN_EVENT(binder_read_done);
-
-TRACE_EVENT(binder_wait_for_work,
-	TP_PROTO(bool proc_work, bool transaction_stack, bool thread_todo),
-	TP_ARGS(proc_work, transaction_stack, thread_todo),
-
-	TP_STRUCT__entry(
-		__field(bool, proc_work)
-		__field(bool, transaction_stack)
-		__field(bool, thread_todo)
-	),
-	TP_fast_assign(
-		__entry->proc_work = proc_work;
-		__entry->transaction_stack = transaction_stack;
-		__entry->thread_todo = thread_todo;
-	),
-	TP_printk("proc_work=%d transaction_stack=%d thread_todo=%d",
-		  __entry->proc_work, __entry->transaction_stack,
-		  __entry->thread_todo)
-);
-
-TRACE_EVENT(binder_txn_latency_free,
-	TP_PROTO(struct binder_transaction *t,
-		 int from_proc, int from_thread,
-		 int to_proc, int to_thread),
-	TP_ARGS(t, from_proc, from_thread, to_proc, to_thread),
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(int, from_proc)
-		__field(int, from_thread)
-		__field(int, to_proc)
-		__field(int, to_thread)
-		__field(unsigned int, code)
-		__field(unsigned int, flags)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-		__entry->from_proc = from_proc;
-		__entry->from_thread = from_thread;
-		__entry->to_proc = to_proc;
-		__entry->to_thread = to_thread;
-		__entry->code = t->code;
-		__entry->flags = t->flags;
-	),
-	TP_printk("transaction=%d from %d:%d to %d:%d flags=0x%x code=0x%x",
-		  __entry->debug_id, __entry->from_proc, __entry->from_thread,
-		  __entry->to_proc, __entry->to_thread, __entry->code,
-		  __entry->flags)
-);
-
-TRACE_EVENT(binder_transaction,
-	TP_PROTO(bool reply, struct binder_transaction *t,
-		 struct binder_node *target_node),
-	TP_ARGS(reply, t, target_node),
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(int, target_node)
-		__field(int, to_proc)
-		__field(int, to_thread)
-		__field(int, reply)
-		__field(unsigned int, code)
-		__field(unsigned int, flags)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-		__entry->target_node = target_node ? target_node->debug_id : 0;
-		__entry->to_proc = t->to_proc->pid;
-		__entry->to_thread = t->to_thread ? t->to_thread->pid : 0;
-		__entry->reply = reply;
-		__entry->code = t->code;
-		__entry->flags = t->flags;
-	),
-	TP_printk("transaction=%d dest_node=%d dest_proc=%d dest_thread=%d reply=%d flags=0x%x code=0x%x",
-		  __entry->debug_id, __entry->target_node,
-		  __entry->to_proc, __entry->to_thread,
-		  __entry->reply, __entry->flags, __entry->code)
-);
-
-TRACE_EVENT(binder_transaction_received,
-	TP_PROTO(struct binder_transaction *t),
-	TP_ARGS(t),
-
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-	),
-	TP_printk("transaction=%d", __entry->debug_id)
-);
-
-TRACE_EVENT(binder_transaction_node_to_ref,
-	TP_PROTO(struct binder_transaction *t, struct binder_node *node,
-		 struct binder_ref_data *rdata),
-	TP_ARGS(t, node, rdata),
-
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(int, node_debug_id)
-		__field(binder_uintptr_t, node_ptr)
-		__field(int, ref_debug_id)
-		__field(uint32_t, ref_desc)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-		__entry->node_debug_id = node->debug_id;
-		__entry->node_ptr = node->ptr;
-		__entry->ref_debug_id = rdata->debug_id;
-		__entry->ref_desc = rdata->desc;
-	),
-	TP_printk("transaction=%d node=%d src_ptr=0x%016llx ==> dest_ref=%d dest_desc=%d",
-		  __entry->debug_id, __entry->node_debug_id,
-		  (u64)__entry->node_ptr,
-		  __entry->ref_debug_id, __entry->ref_desc)
-);
-
-TRACE_EVENT(binder_transaction_ref_to_node,
-	TP_PROTO(struct binder_transaction *t, struct binder_node *node,
-		 struct binder_ref_data *rdata),
-	TP_ARGS(t, node, rdata),
-
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(int, ref_debug_id)
-		__field(uint32_t, ref_desc)
-		__field(int, node_debug_id)
-		__field(binder_uintptr_t, node_ptr)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-		__entry->ref_debug_id = rdata->debug_id;
-		__entry->ref_desc = rdata->desc;
-		__entry->node_debug_id = node->debug_id;
-		__entry->node_ptr = node->ptr;
-	),
-	TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ptr=0x%016llx",
-		  __entry->debug_id, __entry->node_debug_id,
-		  __entry->ref_debug_id, __entry->ref_desc,
-		  (u64)__entry->node_ptr)
-);
-
-TRACE_EVENT(binder_transaction_ref_to_ref,
-	TP_PROTO(struct binder_transaction *t, struct binder_node *node,
-		 struct binder_ref_data *src_ref,
-		 struct binder_ref_data *dest_ref),
-	TP_ARGS(t, node, src_ref, dest_ref),
-
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(int, node_debug_id)
-		__field(int, src_ref_debug_id)
-		__field(uint32_t, src_ref_desc)
-		__field(int, dest_ref_debug_id)
-		__field(uint32_t, dest_ref_desc)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-		__entry->node_debug_id = node->debug_id;
-		__entry->src_ref_debug_id = src_ref->debug_id;
-		__entry->src_ref_desc = src_ref->desc;
-		__entry->dest_ref_debug_id = dest_ref->debug_id;
-		__entry->dest_ref_desc = dest_ref->desc;
-	),
-	TP_printk("transaction=%d node=%d src_ref=%d src_desc=%d ==> dest_ref=%d dest_desc=%d",
-		  __entry->debug_id, __entry->node_debug_id,
-		  __entry->src_ref_debug_id, __entry->src_ref_desc,
-		  __entry->dest_ref_debug_id, __entry->dest_ref_desc)
-);
-
-TRACE_EVENT(binder_transaction_fd_send,
-	TP_PROTO(struct binder_transaction *t, int fd, size_t offset),
-	TP_ARGS(t, fd, offset),
-
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(int, fd)
-		__field(size_t, offset)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-		__entry->fd = fd;
-		__entry->offset = offset;
-	),
-	TP_printk("transaction=%d src_fd=%d offset=%zu",
-		  __entry->debug_id, __entry->fd, __entry->offset)
-);
-
-TRACE_EVENT(binder_transaction_fd_recv,
-	TP_PROTO(struct binder_transaction *t, int fd, size_t offset),
-	TP_ARGS(t, fd, offset),
-
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(int, fd)
-		__field(size_t, offset)
-	),
-	TP_fast_assign(
-		__entry->debug_id = t->debug_id;
-		__entry->fd = fd;
-		__entry->offset = offset;
-	),
-	TP_printk("transaction=%d dest_fd=%d offset=%zu",
-		  __entry->debug_id, __entry->fd, __entry->offset)
-);
-
-DECLARE_EVENT_CLASS(binder_buffer_class,
-	TP_PROTO(struct binder_buffer *buf),
-	TP_ARGS(buf),
-	TP_STRUCT__entry(
-		__field(int, debug_id)
-		__field(size_t, data_size)
-		__field(size_t, offsets_size)
-		__field(size_t, extra_buffers_size)
-	),
-	TP_fast_assign(
-		__entry->debug_id = buf->debug_id;
-		__entry->data_size = buf->data_size;
-		__entry->offsets_size = buf->offsets_size;
-		__entry->extra_buffers_size = buf->extra_buffers_size;
-	),
-	TP_printk("transaction=%d data_size=%zd offsets_size=%zd extra_buffers_size=%zd",
-		  __entry->debug_id, __entry->data_size, __entry->offsets_size,
-		  __entry->extra_buffers_size)
-);
-
-DEFINE_EVENT(binder_buffer_class, binder_transaction_alloc_buf,
-	TP_PROTO(struct binder_buffer *buffer),
-	TP_ARGS(buffer));
-
-DEFINE_EVENT(binder_buffer_class, binder_transaction_buffer_release,
-	TP_PROTO(struct binder_buffer *buffer),
-	TP_ARGS(buffer));
-
-DEFINE_EVENT(binder_buffer_class, binder_transaction_failed_buffer_release,
-	TP_PROTO(struct binder_buffer *buffer),
-	TP_ARGS(buffer));
-
-DEFINE_EVENT(binder_buffer_class, binder_transaction_update_buffer_release,
-	     TP_PROTO(struct binder_buffer *buffer),
-	     TP_ARGS(buffer));
-
-TRACE_EVENT(binder_update_page_range,
-	TP_PROTO(struct binder_alloc *alloc, bool allocate,
-		 unsigned long start, unsigned long end),
-	TP_ARGS(alloc, allocate, start, end),
-	TP_STRUCT__entry(
-		__field(int, proc)
-		__field(bool, allocate)
-		__field(size_t, offset)
-		__field(size_t, size)
-	),
-	TP_fast_assign(
-		__entry->proc = alloc->pid;
-		__entry->allocate = allocate;
-		__entry->offset = start - alloc->vm_start;
-		__entry->size = end - start;
-	),
-	TP_printk("proc=%d allocate=%d offset=%zu size=%zu",
-		  __entry->proc, __entry->allocate,
-		  __entry->offset, __entry->size)
-);
-
-DECLARE_EVENT_CLASS(binder_lru_page_class,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index),
-	TP_STRUCT__entry(
-		__field(int, proc)
-		__field(size_t, page_index)
-	),
-	TP_fast_assign(
-		__entry->proc = alloc->pid;
-		__entry->page_index = page_index;
-	),
-	TP_printk("proc=%d page_index=%zu",
-		  __entry->proc, __entry->page_index)
-);
-
-DEFINE_EVENT(binder_lru_page_class, binder_alloc_lru_start,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_alloc_lru_end,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_free_lru_start,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_free_lru_end,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_alloc_page_start,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_alloc_page_end,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_unmap_user_start,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_unmap_user_end,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_unmap_kernel_start,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-DEFINE_EVENT(binder_lru_page_class, binder_unmap_kernel_end,
-	TP_PROTO(const struct binder_alloc *alloc, size_t page_index),
-	TP_ARGS(alloc, page_index));
-
-TRACE_EVENT(binder_command,
-	TP_PROTO(uint32_t cmd),
-	TP_ARGS(cmd),
-	TP_STRUCT__entry(
-		__field(uint32_t, cmd)
-	),
-	TP_fast_assign(
-		__entry->cmd = cmd;
-	),
-	TP_printk("cmd=0x%x %s",
-		  __entry->cmd,
-		  _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_command_strings) ?
-			  binder_command_strings[_IOC_NR(__entry->cmd)] :
-			  "unknown")
-);
-
-TRACE_EVENT(binder_return,
-	TP_PROTO(uint32_t cmd),
-	TP_ARGS(cmd),
-	TP_STRUCT__entry(
-		__field(uint32_t, cmd)
-	),
-	TP_fast_assign(
-		__entry->cmd = cmd;
-	),
-	TP_printk("cmd=0x%x %s",
-		  __entry->cmd,
-		  _IOC_NR(__entry->cmd) < ARRAY_SIZE(binder_return_strings) ?
-			  binder_return_strings[_IOC_NR(__entry->cmd)] :
-			  "unknown")
-);
-
-TRACE_EVENT(binder_netlink_report,
-	TP_PROTO(const char *context,
-		 struct binder_transaction *t,
-		 u32 data_size,
-		 u32 error),
-	TP_ARGS(context, t, data_size, error),
-	TP_STRUCT__entry(
-		__field(const char *, context)
-		__field(u32, error)
-		__field(int, from_pid)
-		__field(int, from_tid)
-		__field(int, to_pid)
-		__field(int, to_tid)
-		__field(bool, is_reply)
-		__field(unsigned int, flags)
-		__field(unsigned int, code)
-		__field(size_t, data_size)
-	),
-	TP_fast_assign(
-		__entry->context = context;
-		__entry->error = error;
-		__entry->from_pid = t->from_pid;
-		__entry->from_tid = t->from_tid;
-		__entry->to_pid = t->to_proc ? t->to_proc->pid : 0;
-		__entry->to_tid = t->to_thread ? t->to_thread->pid : 0;
-		__entry->is_reply = t->is_reply;
-		__entry->flags = t->flags;
-		__entry->code = t->code;
-		__entry->data_size = data_size;
-	),
-	TP_printk("from %d:%d to %d:%d context=%s error=%d is_reply=%d flags=0x%x code=0x%x size=%zu",
-		  __entry->from_pid, __entry->from_tid,
-		  __entry->to_pid, __entry->to_tid,
-		  __entry->context, __entry->error, __entry->is_reply,
-		  __entry->flags, __entry->code, __entry->data_size)
-);
-
-#endif /* _BINDER_TRACE_H */
-
-#undef TRACE_INCLUDE_PATH
-#undef TRACE_INCLUDE_FILE
-#define TRACE_INCLUDE_PATH .
-#define TRACE_INCLUDE_FILE binder_trace
-#include <trace/define_trace.h>
diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c
deleted file mode 100644
index 361d69f756f50e..00000000000000
--- a/drivers/android/binderfs.c
+++ /dev/null
@@ -1,786 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-#include <linux/compiler_types.h>
-#include <linux/errno.h>
-#include <linux/fs.h>
-#include <linux/fsnotify.h>
-#include <linux/gfp.h>
-#include <linux/idr.h>
-#include <linux/init.h>
-#include <linux/ipc_namespace.h>
-#include <linux/kdev_t.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/namei.h>
-#include <linux/magic.h>
-#include <linux/major.h>
-#include <linux/miscdevice.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/mount.h>
-#include <linux/fs_parser.h>
-#include <linux/sched.h>
-#include <linux/seq_file.h>
-#include <linux/slab.h>
-#include <linux/spinlock_types.h>
-#include <linux/stddef.h>
-#include <linux/string.h>
-#include <linux/types.h>
-#include <linux/uaccess.h>
-#include <linux/user_namespace.h>
-#include <linux/xarray.h>
-#include <uapi/linux/android/binder.h>
-#include <uapi/linux/android/binderfs.h>
-
-#include "binder_internal.h"
-
-#define FIRST_INODE 1
-#define SECOND_INODE 2
-#define INODE_OFFSET 3
-#define BINDERFS_MAX_MINOR (1U << MINORBITS)
-/* Ensure that the initial ipc namespace always has devices available. */
-#define BINDERFS_MAX_MINOR_CAPPED (BINDERFS_MAX_MINOR - 4)
-
-static dev_t binderfs_dev;
-static DEFINE_MUTEX(binderfs_minors_mutex);
-static DEFINE_IDA(binderfs_minors);
-
-enum binderfs_param {
-	Opt_max,
-	Opt_stats_mode,
-};
-
-enum binderfs_stats_mode {
-	binderfs_stats_mode_unset,
-	binderfs_stats_mode_global,
-};
-
-struct binder_features {
-	bool oneway_spam_detection;
-	bool extended_error;
-	bool freeze_notification;
-	bool transaction_report;
-};
-
-static const struct constant_table binderfs_param_stats[] = {
-	{ "global", binderfs_stats_mode_global },
-	{}
-};
-
-static const struct fs_parameter_spec binderfs_fs_parameters[] = {
-	fsparam_u32("max",	Opt_max),
-	fsparam_enum("stats",	Opt_stats_mode, binderfs_param_stats),
-	{}
-};
-
-static struct binder_features binder_features = {
-	.oneway_spam_detection = true,
-	.extended_error = true,
-	.freeze_notification = true,
-	.transaction_report = true,
-};
-
-static inline struct binderfs_info *BINDERFS_SB(const struct super_block *sb)
-{
-	return sb->s_fs_info;
-}
-
-bool is_binderfs_device(const struct inode *inode)
-{
-	if (inode->i_sb->s_magic == BINDERFS_SUPER_MAGIC)
-		return true;
-
-	return false;
-}
-
-/**
- * binderfs_binder_device_create - allocate inode from super block of a
- *                                 binderfs mount
- * @ref_inode: inode from which the super block will be taken
- * @userp:     buffer to copy information about new device for userspace to
- * @req:       struct binderfs_device as copied from userspace
- *
- * This function allocates a new binder_device and reserves a new minor
- * number for it.
- * Minor numbers are limited and tracked globally in binderfs_minors. The
- * function will stash a struct binder_device for the specific binder
- * device in i_private of the inode.
- * It will go on to allocate a new inode from the super block of the
- * filesystem mount, stash a struct binder_device in its i_private field
- * and attach a dentry to that inode.
- *
- * Return: 0 on success, negative errno on failure
- */
-static int binderfs_binder_device_create(struct inode *ref_inode,
-					 struct binderfs_device __user *userp,
-					 struct binderfs_device *req)
-{
-	int minor, ret;
-	struct dentry *dentry, *root;
-	struct binder_device *device;
-	char *name = NULL;
-	struct inode *inode = NULL;
-	struct super_block *sb = ref_inode->i_sb;
-	struct binderfs_info *info = sb->s_fs_info;
-#if defined(CONFIG_IPC_NS)
-	bool use_reserve = (info->ipc_ns == &init_ipc_ns);
-#else
-	bool use_reserve = true;
-#endif
-
-	/* Reserve new minor number for the new device. */
-	mutex_lock(&binderfs_minors_mutex);
-	if (++info->device_count <= info->mount_opts.max)
-		minor = ida_alloc_max(&binderfs_minors,
-				      use_reserve ? BINDERFS_MAX_MINOR - 1 :
-						    BINDERFS_MAX_MINOR_CAPPED - 1,
-				      GFP_KERNEL);
-	else
-		minor = -ENOSPC;
-	if (minor < 0) {
-		--info->device_count;
-		mutex_unlock(&binderfs_minors_mutex);
-		return minor;
-	}
-	mutex_unlock(&binderfs_minors_mutex);
-
-	ret = -ENOMEM;
-	device = kzalloc_obj(*device);
-	if (!device)
-		goto err;
-
-	inode = new_inode(sb);
-	if (!inode)
-		goto err;
-
-	inode->i_ino = minor + INODE_OFFSET;
-	simple_inode_init_ts(inode);
-	init_special_inode(inode, S_IFCHR | 0600,
-			   MKDEV(MAJOR(binderfs_dev), minor));
-	inode->i_fop = &binder_fops;
-	inode->i_uid = info->root_uid;
-	inode->i_gid = info->root_gid;
-
-	req->name[BINDERFS_MAX_NAME] = '\0'; /* NUL-terminate */
-	name = kstrdup(req->name, GFP_KERNEL);
-	if (!name)
-		goto err;
-
-	refcount_set(&device->ref, 1);
-	device->binderfs_inode = inode;
-	device->context.binder_context_mgr_uid = INVALID_UID;
-	device->context.name = name;
-	device->miscdev.name = name;
-	device->miscdev.minor = minor;
-	mutex_init(&device->context.context_mgr_node_lock);
-
-	req->major = MAJOR(binderfs_dev);
-	req->minor = minor;
-
-	if (userp && copy_to_user(userp, req, sizeof(*req))) {
-		ret = -EFAULT;
-		goto err;
-	}
-
-	root = sb->s_root;
-	dentry = simple_start_creating(root, name);
-	if (IS_ERR(dentry)) {
-		ret = PTR_ERR(dentry);
-		goto err;
-	}
-	inode->i_private = device;
-	d_make_persistent(dentry, inode);
-	fsnotify_create(root->d_inode, dentry);
-	simple_done_creating(dentry);
-
-	binder_add_device(device);
-
-	return 0;
-
-err:
-	kfree(name);
-	kfree(device);
-	mutex_lock(&binderfs_minors_mutex);
-	--info->device_count;
-	ida_free(&binderfs_minors, minor);
-	mutex_unlock(&binderfs_minors_mutex);
-	iput(inode);
-
-	return ret;
-}
-
-/**
- * binder_ctl_ioctl - handle binder device node allocation requests
- * @file: The file pointer for the binder-control device node.
- * @cmd: The ioctl command.
- * @arg: The ioctl argument.
- *
- * The request handler for the binder-control device. All requests operate on
- * the binderfs mount the binder-control device resides in:
- * - BINDER_CTL_ADD
- *   Allocate a new binder device.
- *
- * Return: %0 on success, negative errno on failure.
- */
-static long binder_ctl_ioctl(struct file *file, unsigned int cmd,
-			     unsigned long arg)
-{
-	int ret = -EINVAL;
-	struct inode *inode = file_inode(file);
-	struct binderfs_device __user *device = (struct binderfs_device __user *)arg;
-	struct binderfs_device device_req;
-
-	switch (cmd) {
-	case BINDER_CTL_ADD:
-		ret = copy_from_user(&device_req, device, sizeof(device_req));
-		if (ret) {
-			ret = -EFAULT;
-			break;
-		}
-
-		ret = binderfs_binder_device_create(inode, device, &device_req);
-		break;
-	default:
-		break;
-	}
-
-	return ret;
-}
-
-static void binderfs_evict_inode(struct inode *inode)
-{
-	struct binder_device *device = inode->i_private;
-	struct binderfs_info *info = BINDERFS_SB(inode->i_sb);
-
-	clear_inode(inode);
-
-	if (!S_ISCHR(inode->i_mode) || !device)
-		return;
-
-	mutex_lock(&binderfs_minors_mutex);
-	--info->device_count;
-	ida_free(&binderfs_minors, device->miscdev.minor);
-	mutex_unlock(&binderfs_minors_mutex);
-
-	if (refcount_dec_and_test(&device->ref)) {
-		binder_remove_device(device);
-		kfree(device->context.name);
-		kfree(device);
-	}
-}
-
-static int binderfs_fs_context_parse_param(struct fs_context *fc,
-					   struct fs_parameter *param)
-{
-	int opt;
-	struct binderfs_mount_opts *ctx = fc->fs_private;
-	struct fs_parse_result result;
-
-	opt = fs_parse(fc, binderfs_fs_parameters, param, &result);
-	if (opt < 0)
-		return opt;
-
-	switch (opt) {
-	case Opt_max:
-		if (result.uint_32 > BINDERFS_MAX_MINOR)
-			return invalfc(fc, "Bad value for '%s'", param->key);
-
-		ctx->max = result.uint_32;
-		break;
-	case Opt_stats_mode:
-		if (!capable(CAP_SYS_ADMIN))
-			return -EPERM;
-
-		ctx->stats_mode = result.uint_32;
-		break;
-	default:
-		return invalfc(fc, "Unsupported parameter '%s'", param->key);
-	}
-
-	return 0;
-}
-
-static int binderfs_fs_context_reconfigure(struct fs_context *fc)
-{
-	struct binderfs_mount_opts *ctx = fc->fs_private;
-	struct binderfs_info *info = BINDERFS_SB(fc->root->d_sb);
-
-	if (info->mount_opts.stats_mode != ctx->stats_mode)
-		return invalfc(fc, "Binderfs stats mode cannot be changed during a remount");
-
-	info->mount_opts.stats_mode = ctx->stats_mode;
-	info->mount_opts.max = ctx->max;
-	return 0;
-}
-
-static int binderfs_show_options(struct seq_file *seq, struct dentry *root)
-{
-	struct binderfs_info *info = BINDERFS_SB(root->d_sb);
-
-	if (info->mount_opts.max <= BINDERFS_MAX_MINOR)
-		seq_printf(seq, ",max=%d", info->mount_opts.max);
-
-	switch (info->mount_opts.stats_mode) {
-	case binderfs_stats_mode_unset:
-		break;
-	case binderfs_stats_mode_global:
-		seq_printf(seq, ",stats=global");
-		break;
-	}
-
-	return 0;
-}
-
-static const struct super_operations binderfs_super_ops = {
-	.evict_inode    = binderfs_evict_inode,
-	.show_options	= binderfs_show_options,
-	.statfs         = simple_statfs,
-};
-
-static inline bool is_binderfs_control_device(const struct dentry *dentry)
-{
-	struct binderfs_info *info = dentry->d_sb->s_fs_info;
-
-	return info->control_dentry == dentry;
-}
-
-static int binderfs_rename(struct mnt_idmap *idmap,
-			   struct inode *old_dir, struct dentry *old_dentry,
-			   struct inode *new_dir, struct dentry *new_dentry,
-			   unsigned int flags)
-{
-	if (is_binderfs_control_device(old_dentry) ||
-	    is_binderfs_control_device(new_dentry))
-		return -EPERM;
-
-	return simple_rename(idmap, old_dir, old_dentry, new_dir,
-			     new_dentry, flags);
-}
-
-static int binderfs_unlink(struct inode *dir, struct dentry *dentry)
-{
-	if (is_binderfs_control_device(dentry))
-		return -EPERM;
-
-	return simple_unlink(dir, dentry);
-}
-
-static const struct file_operations binder_ctl_fops = {
-	.owner		= THIS_MODULE,
-	.open		= nonseekable_open,
-	.unlocked_ioctl	= binder_ctl_ioctl,
-	.compat_ioctl	= binder_ctl_ioctl,
-	.llseek		= noop_llseek,
-};
-
-/**
- * binderfs_binder_ctl_create - create a new binder-control device
- * @sb: super block of the binderfs mount
- *
- * This function creates a new binder-control device node in the binderfs mount
- * referred to by @sb.
- *
- * Return: 0 on success, negative errno on failure
- */
-static int binderfs_binder_ctl_create(struct super_block *sb)
-{
-	int minor, ret;
-	struct dentry *dentry;
-	struct binder_device *device;
-	struct inode *inode = NULL;
-	struct dentry *root = sb->s_root;
-	struct binderfs_info *info = sb->s_fs_info;
-#if defined(CONFIG_IPC_NS)
-	bool use_reserve = (info->ipc_ns == &init_ipc_ns);
-#else
-	bool use_reserve = true;
-#endif
-
-	device = kzalloc_obj(*device);
-	if (!device)
-		return -ENOMEM;
-
-	ret = -ENOMEM;
-	inode = new_inode(sb);
-	if (!inode)
-		goto out;
-
-	/* Reserve a new minor number for the new device. */
-	mutex_lock(&binderfs_minors_mutex);
-	minor = ida_alloc_max(&binderfs_minors,
-			      use_reserve ? BINDERFS_MAX_MINOR - 1 :
-					    BINDERFS_MAX_MINOR_CAPPED - 1,
-			      GFP_KERNEL);
-	mutex_unlock(&binderfs_minors_mutex);
-	if (minor < 0) {
-		ret = minor;
-		goto out;
-	}
-
-	inode->i_ino = SECOND_INODE;
-	simple_inode_init_ts(inode);
-	init_special_inode(inode, S_IFCHR | 0600,
-			   MKDEV(MAJOR(binderfs_dev), minor));
-	inode->i_fop = &binder_ctl_fops;
-	inode->i_uid = info->root_uid;
-	inode->i_gid = info->root_gid;
-
-	refcount_set(&device->ref, 1);
-	device->binderfs_inode = inode;
-	device->miscdev.minor = minor;
-
-	dentry = d_alloc_name(root, "binder-control");
-	if (!dentry)
-		goto out;
-
-	inode->i_private = device;
-	info->control_dentry = dentry;
-	d_make_persistent(dentry, inode);
-	dput(dentry);
-
-	return 0;
-
-out:
-	kfree(device);
-	iput(inode);
-
-	return ret;
-}
-
-static const struct inode_operations binderfs_dir_inode_operations = {
-	.lookup = simple_lookup,
-	.rename = binderfs_rename,
-	.unlink = binderfs_unlink,
-};
-
-static struct inode *binderfs_make_inode(struct super_block *sb, int mode)
-{
-	struct inode *ret;
-
-	ret = new_inode(sb);
-	if (ret) {
-		ret->i_ino = iunique(sb, BINDERFS_MAX_MINOR + INODE_OFFSET);
-		ret->i_mode = mode;
-		simple_inode_init_ts(ret);
-	}
-	return ret;
-}
-
-struct dentry *binderfs_create_file(struct dentry *parent, const char *name,
-				    const struct file_operations *fops,
-				    void *data)
-{
-	struct dentry *dentry;
-	struct inode *new_inode, *parent_inode;
-	struct super_block *sb;
-
-	parent_inode = d_inode(parent);
-
-	dentry = simple_start_creating(parent, name);
-	if (IS_ERR(dentry))
-		return dentry;
-
-	sb = parent_inode->i_sb;
-	new_inode = binderfs_make_inode(sb, S_IFREG | 0444);
-	if (!new_inode) {
-		simple_done_creating(dentry);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	new_inode->i_fop = fops;
-	new_inode->i_private = data;
-	d_make_persistent(dentry, new_inode);
-	fsnotify_create(parent_inode, dentry);
-	simple_done_creating(dentry);
-	return dentry; // borrowed
-}
-
-static struct dentry *binderfs_create_dir(struct dentry *parent,
-					  const char *name)
-{
-	struct dentry *dentry;
-	struct inode *new_inode, *parent_inode;
-	struct super_block *sb;
-
-	parent_inode = d_inode(parent);
-
-	dentry = simple_start_creating(parent, name);
-	if (IS_ERR(dentry))
-		return dentry;
-
-	sb = parent_inode->i_sb;
-	new_inode = binderfs_make_inode(sb, S_IFDIR | 0755);
-	if (!new_inode) {
-		simple_done_creating(dentry);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	new_inode->i_fop = &simple_dir_operations;
-	new_inode->i_op = &simple_dir_inode_operations;
-
-	set_nlink(new_inode, 2);
-	d_make_persistent(dentry, new_inode);
-	inc_nlink(parent_inode);
-	fsnotify_mkdir(parent_inode, dentry);
-	simple_done_creating(dentry);
-	return dentry;
-}
-
-static int binder_features_show(struct seq_file *m, void *unused)
-{
-	bool *feature = m->private;
-
-	seq_printf(m, "%d\n", *feature);
-
-	return 0;
-}
-DEFINE_SHOW_ATTRIBUTE(binder_features);
-
-static int init_binder_features(struct super_block *sb)
-{
-	struct dentry *dentry, *dir;
-
-	dir = binderfs_create_dir(sb->s_root, "features");
-	if (IS_ERR(dir))
-		return PTR_ERR(dir);
-
-	dentry = binderfs_create_file(dir, "oneway_spam_detection",
-				      &binder_features_fops,
-				      &binder_features.oneway_spam_detection);
-	if (IS_ERR(dentry))
-		return PTR_ERR(dentry);
-
-	dentry = binderfs_create_file(dir, "extended_error",
-				      &binder_features_fops,
-				      &binder_features.extended_error);
-	if (IS_ERR(dentry))
-		return PTR_ERR(dentry);
-
-	dentry = binderfs_create_file(dir, "freeze_notification",
-				      &binder_features_fops,
-				      &binder_features.freeze_notification);
-	if (IS_ERR(dentry))
-		return PTR_ERR(dentry);
-
-	dentry = binderfs_create_file(dir, "transaction_report",
-				      &binder_features_fops,
-				      &binder_features.transaction_report);
-	if (IS_ERR(dentry))
-		return PTR_ERR(dentry);
-
-	return 0;
-}
-
-static int init_binder_logs(struct super_block *sb)
-{
-	struct dentry *binder_logs_root_dir, *dentry, *proc_log_dir;
-	const struct binder_debugfs_entry *db_entry;
-	struct binderfs_info *info;
-	int ret = 0;
-
-	binder_logs_root_dir = binderfs_create_dir(sb->s_root,
-						   "binder_logs");
-	if (IS_ERR(binder_logs_root_dir)) {
-		ret = PTR_ERR(binder_logs_root_dir);
-		goto out;
-	}
-
-	binder_for_each_debugfs_entry(db_entry) {
-		dentry = binderfs_create_file(binder_logs_root_dir,
-					      db_entry->name,
-					      db_entry->fops,
-					      db_entry->data);
-		if (IS_ERR(dentry)) {
-			ret = PTR_ERR(dentry);
-			goto out;
-		}
-	}
-
-	proc_log_dir = binderfs_create_dir(binder_logs_root_dir, "proc");
-	if (IS_ERR(proc_log_dir)) {
-		ret = PTR_ERR(proc_log_dir);
-		goto out;
-	}
-	info = sb->s_fs_info;
-	info->proc_log_dir = proc_log_dir;
-
-out:
-	return ret;
-}
-
-static int binderfs_fill_super(struct super_block *sb, struct fs_context *fc)
-{
-	int ret;
-	struct binderfs_info *info;
-	struct binderfs_mount_opts *ctx = fc->fs_private;
-	struct inode *inode = NULL;
-	struct binderfs_device device_info = {};
-	const char *name;
-	size_t len;
-
-	sb->s_blocksize = PAGE_SIZE;
-	sb->s_blocksize_bits = PAGE_SHIFT;
-
-	/*
-	 * The binderfs filesystem can be mounted by userns root in a
-	 * non-initial userns. By default such mounts have the SB_I_NODEV flag
-	 * set in s_iflags to prevent security issues where userns root can
-	 * just create random device nodes via mknod() since it owns the
-	 * filesystem mount. But binderfs does not allow to create any files
-	 * including devices nodes. The only way to create binder devices nodes
-	 * is through the binder-control device which userns root is explicitly
-	 * allowed to do. So removing the SB_I_NODEV flag from s_iflags is both
-	 * necessary and safe.
-	 */
-	sb->s_iflags &= ~SB_I_NODEV;
-	sb->s_iflags |= SB_I_NOEXEC;
-	sb->s_magic = BINDERFS_SUPER_MAGIC;
-	sb->s_op = &binderfs_super_ops;
-	sb->s_time_gran = 1;
-
-	sb->s_fs_info = kzalloc_obj(struct binderfs_info);
-	if (!sb->s_fs_info)
-		return -ENOMEM;
-	info = sb->s_fs_info;
-
-	info->ipc_ns = get_ipc_ns(current->nsproxy->ipc_ns);
-
-	info->root_gid = make_kgid(sb->s_user_ns, 0);
-	if (!gid_valid(info->root_gid))
-		info->root_gid = GLOBAL_ROOT_GID;
-	info->root_uid = make_kuid(sb->s_user_ns, 0);
-	if (!uid_valid(info->root_uid))
-		info->root_uid = GLOBAL_ROOT_UID;
-	info->mount_opts.max = ctx->max;
-	info->mount_opts.stats_mode = ctx->stats_mode;
-
-	inode = new_inode(sb);
-	if (!inode)
-		return -ENOMEM;
-
-	inode->i_ino = FIRST_INODE;
-	inode->i_fop = &simple_dir_operations;
-	inode->i_mode = S_IFDIR | 0755;
-	simple_inode_init_ts(inode);
-	inode->i_op = &binderfs_dir_inode_operations;
-	set_nlink(inode, 2);
-
-	sb->s_root = d_make_root(inode);
-	if (!sb->s_root)
-		return -ENOMEM;
-
-	ret = binderfs_binder_ctl_create(sb);
-	if (ret)
-		return ret;
-
-	name = binder_devices_param;
-	for (len = strcspn(name, ","); len > 0; len = strcspn(name, ",")) {
-		strscpy(device_info.name, name, len + 1);
-		ret = binderfs_binder_device_create(inode, NULL, &device_info);
-		if (ret)
-			return ret;
-		name += len;
-		if (*name == ',')
-			name++;
-	}
-
-	ret = init_binder_features(sb);
-	if (ret)
-		return ret;
-
-	if (info->mount_opts.stats_mode == binderfs_stats_mode_global)
-		return init_binder_logs(sb);
-
-	return 0;
-}
-
-static int binderfs_fs_context_get_tree(struct fs_context *fc)
-{
-	return get_tree_nodev(fc, binderfs_fill_super);
-}
-
-static void binderfs_fs_context_free(struct fs_context *fc)
-{
-	struct binderfs_mount_opts *ctx = fc->fs_private;
-
-	kfree(ctx);
-}
-
-static const struct fs_context_operations binderfs_fs_context_ops = {
-	.free		= binderfs_fs_context_free,
-	.get_tree	= binderfs_fs_context_get_tree,
-	.parse_param	= binderfs_fs_context_parse_param,
-	.reconfigure	= binderfs_fs_context_reconfigure,
-};
-
-static int binderfs_init_fs_context(struct fs_context *fc)
-{
-	struct binderfs_mount_opts *ctx;
-
-	ctx = kzalloc_obj(struct binderfs_mount_opts);
-	if (!ctx)
-		return -ENOMEM;
-
-	ctx->max = BINDERFS_MAX_MINOR;
-	ctx->stats_mode = binderfs_stats_mode_unset;
-
-	fc->fs_private = ctx;
-	fc->ops = &binderfs_fs_context_ops;
-
-	return 0;
-}
-
-static void binderfs_kill_super(struct super_block *sb)
-{
-	struct binderfs_info *info = sb->s_fs_info;
-
-	/*
-	 * During inode eviction struct binderfs_info is needed.
-	 * So first wipe the super_block then free struct binderfs_info.
-	 */
-	kill_anon_super(sb);
-
-	if (info && info->ipc_ns)
-		put_ipc_ns(info->ipc_ns);
-
-	kfree(info);
-}
-
-static struct file_system_type binder_fs_type = {
-	.name			= "binder",
-	.init_fs_context	= binderfs_init_fs_context,
-	.parameters		= binderfs_fs_parameters,
-	.kill_sb		= binderfs_kill_super,
-	.fs_flags		= FS_USERNS_MOUNT,
-};
-
-int __init init_binderfs(void)
-{
-	int ret;
-	const char *name;
-	size_t len;
-
-	/* Verify that the default binderfs device names are valid. */
-	name = binder_devices_param;
-	for (len = strcspn(name, ","); len > 0; len = strcspn(name, ",")) {
-		if (len > BINDERFS_MAX_NAME)
-			return -E2BIG;
-		name += len;
-		if (*name == ',')
-			name++;
-	}
-
-	/* Allocate new major number for binderfs. */
-	ret = alloc_chrdev_region(&binderfs_dev, 0, BINDERFS_MAX_MINOR,
-				  "binder");
-	if (ret)
-		return ret;
-
-	ret = register_filesystem(&binder_fs_type);
-	if (ret) {
-		unregister_chrdev_region(binderfs_dev, BINDERFS_MAX_MINOR);
-		return ret;
-	}
-
-	return ret;
-}
diff --git a/drivers/android/dbitmap.h b/drivers/android/dbitmap.h
deleted file mode 100644
index c7299ce8b37413..00000000000000
--- a/drivers/android/dbitmap.h
+++ /dev/null
@@ -1,169 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright 2024 Google LLC
- *
- * dbitmap - dynamically sized bitmap library.
- *
- * Used by the binder driver to optimize the allocation of the smallest
- * available descriptor ID. Each bit in the bitmap represents the state
- * of an ID.
- *
- * A dbitmap can grow or shrink as needed. This part has been designed
- * considering that users might need to briefly release their locks in
- * order to allocate memory for the new bitmap. These operations then,
- * are verified to determine if the grow or shrink is sill valid.
- *
- * This library does not provide protection against concurrent access
- * by itself. Binder uses the proc->outer_lock for this purpose.
- */
-
-#ifndef _LINUX_DBITMAP_H
-#define _LINUX_DBITMAP_H
-#include <linux/bitmap.h>
-
-#define NBITS_MIN	BITS_PER_TYPE(unsigned long)
-
-struct dbitmap {
-	unsigned int nbits;
-	unsigned long *map;
-};
-
-static inline int dbitmap_enabled(struct dbitmap *dmap)
-{
-	return !!dmap->nbits;
-}
-
-static inline void dbitmap_free(struct dbitmap *dmap)
-{
-	dmap->nbits = 0;
-	kfree(dmap->map);
-	dmap->map = NULL;
-}
-
-/* Returns the nbits that a dbitmap can shrink to, 0 if not possible. */
-static inline unsigned int dbitmap_shrink_nbits(struct dbitmap *dmap)
-{
-	unsigned int bit;
-
-	if (dmap->nbits <= NBITS_MIN)
-		return 0;
-
-	/*
-	 * Determine if the bitmap can shrink based on the position of
-	 * its last set bit. If the bit is within the first quarter of
-	 * the bitmap then shrinking is possible. In this case, the
-	 * bitmap should shrink to half its current size.
-	 */
-	bit = find_last_bit(dmap->map, dmap->nbits);
-	if (bit < (dmap->nbits >> 2))
-		return dmap->nbits >> 1;
-
-	/* find_last_bit() returns dmap->nbits when no bits are set. */
-	if (bit == dmap->nbits)
-		return NBITS_MIN;
-
-	return 0;
-}
-
-/* Replace the internal bitmap with a new one of different size */
-static inline void
-dbitmap_replace(struct dbitmap *dmap, unsigned long *new, unsigned int nbits)
-{
-	bitmap_copy(new, dmap->map, min(dmap->nbits, nbits));
-	kfree(dmap->map);
-	dmap->map = new;
-	dmap->nbits = nbits;
-}
-
-static inline void
-dbitmap_shrink(struct dbitmap *dmap, unsigned long *new, unsigned int nbits)
-{
-	if (!new)
-		return;
-
-	/*
-	 * Verify that shrinking to @nbits is still possible. The @new
-	 * bitmap might have been allocated without locks, so this call
-	 * could now be outdated. In this case, free @new and move on.
-	 */
-	if (!dbitmap_enabled(dmap) || dbitmap_shrink_nbits(dmap) != nbits) {
-		kfree(new);
-		return;
-	}
-
-	dbitmap_replace(dmap, new, nbits);
-}
-
-/* Returns the nbits that a dbitmap can grow to. */
-static inline unsigned int dbitmap_grow_nbits(struct dbitmap *dmap)
-{
-	return dmap->nbits << 1;
-}
-
-static inline void
-dbitmap_grow(struct dbitmap *dmap, unsigned long *new, unsigned int nbits)
-{
-	/*
-	 * Verify that growing to @nbits is still possible. The @new
-	 * bitmap might have been allocated without locks, so this call
-	 * could now be outdated. In this case, free @new and move on.
-	 */
-	if (!dbitmap_enabled(dmap) || nbits <= dmap->nbits) {
-		kfree(new);
-		return;
-	}
-
-	/*
-	 * Check for ENOMEM after confirming the grow operation is still
-	 * required. This ensures we only disable the dbitmap when it's
-	 * necessary. Once the dbitmap is disabled, binder will fallback
-	 * to slow_desc_lookup_olocked().
-	 */
-	if (!new) {
-		dbitmap_free(dmap);
-		return;
-	}
-
-	dbitmap_replace(dmap, new, nbits);
-}
-
-/*
- * Finds and sets the next zero bit in the bitmap. Upon success @bit
- * is populated with the index and 0 is returned. Otherwise, -ENOSPC
- * is returned to indicate that a dbitmap_grow() is needed.
- */
-static inline int
-dbitmap_acquire_next_zero_bit(struct dbitmap *dmap, unsigned long offset,
-			      unsigned long *bit)
-{
-	unsigned long n;
-
-	n = find_next_zero_bit(dmap->map, dmap->nbits, offset);
-	if (n == dmap->nbits)
-		return -ENOSPC;
-
-	*bit = n;
-	set_bit(n, dmap->map);
-
-	return 0;
-}
-
-static inline void
-dbitmap_clear_bit(struct dbitmap *dmap, unsigned long bit)
-{
-	clear_bit(bit, dmap->map);
-}
-
-static inline int dbitmap_init(struct dbitmap *dmap)
-{
-	dmap->map = bitmap_zalloc(NBITS_MIN, GFP_KERNEL);
-	if (!dmap->map) {
-		dmap->nbits = 0;
-		return -ENOMEM;
-	}
-
-	dmap->nbits = NBITS_MIN;
-
-	return 0;
-}
-#endif
diff --git a/drivers/android/tests/.kunitconfig b/drivers/android/tests/.kunitconfig
deleted file mode 100644
index 39b76bab9d9a15..00000000000000
--- a/drivers/android/tests/.kunitconfig
+++ /dev/null
@@ -1,7 +0,0 @@
-#
-# Copyright 2025 Google LLC.
-#
-
-CONFIG_KUNIT=y
-CONFIG_ANDROID_BINDER_IPC=y
-CONFIG_ANDROID_BINDER_ALLOC_KUNIT_TEST=y
diff --git a/drivers/android/tests/Makefile b/drivers/android/tests/Makefile
deleted file mode 100644
index 27268418eb033e..00000000000000
--- a/drivers/android/tests/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Copyright 2025 Google LLC.
-#
-
-obj-$(CONFIG_ANDROID_BINDER_ALLOC_KUNIT_TEST)	+= binder_alloc_kunit.o
diff --git a/drivers/android/tests/binder_alloc_kunit.c b/drivers/android/tests/binder_alloc_kunit.c
deleted file mode 100644
index 7f9cc003bbe3ba..00000000000000
--- a/drivers/android/tests/binder_alloc_kunit.c
+++ /dev/null
@@ -1,572 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Test cases for binder allocator code.
- *
- * Copyright 2025 Google LLC.
- * Author: Tiffany Yang <ynaffit@google.com>
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <kunit/test.h>
-#include <linux/anon_inodes.h>
-#include <linux/err.h>
-#include <linux/file.h>
-#include <linux/fs.h>
-#include <linux/mm.h>
-#include <linux/mman.h>
-#include <linux/seq_buf.h>
-#include <linux/sizes.h>
-
-#include "../binder_alloc.h"
-#include "../binder_internal.h"
-
-MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
-
-#define BINDER_MMAP_SIZE SZ_128K
-
-#define BUFFER_NUM 5
-#define BUFFER_MIN_SIZE (PAGE_SIZE / 8)
-
-#define FREESEQ_BUFLEN ((3 * BUFFER_NUM) + 1)
-
-#define ALIGN_TYPE_STRLEN (12)
-
-#define ALIGNMENTS_BUFLEN (((ALIGN_TYPE_STRLEN + 6) * BUFFER_NUM) + 1)
-
-#define PRINT_ALL_CASES (0)
-
-/* 5^5 alignment combinations * 2 places to share pages * 5! free sequences */
-#define TOTAL_EXHAUSTIVE_CASES (3125 * 2 * 120)
-
-/**
- * enum buf_end_align_type - Page alignment of a buffer
- * end with regard to the end of the previous buffer.
- *
- * In the pictures below, buf2 refers to the buffer we
- * are aligning. buf1 refers to previous buffer by addr.
- * Symbol [ means the start of a buffer, ] means the end
- * of a buffer, and | means page boundaries.
- */
-enum buf_end_align_type {
-	/**
-	 * @SAME_PAGE_UNALIGNED: The end of this buffer is on
-	 * the same page as the end of the previous buffer and
-	 * is not page aligned. Examples:
-	 * buf1 ][ buf2 ][ ...
-	 * buf1 ]|[ buf2 ][ ...
-	 */
-	SAME_PAGE_UNALIGNED = 0,
-	/**
-	 * @SAME_PAGE_ALIGNED: When the end of the previous buffer
-	 * is not page aligned, the end of this buffer is on the
-	 * same page as the end of the previous buffer and is page
-	 * aligned. When the previous buffer is page aligned, the
-	 * end of this buffer is aligned to the next page boundary.
-	 * Examples:
-	 * buf1 ][ buf2 ]| ...
-	 * buf1 ]|[ buf2 ]| ...
-	 */
-	SAME_PAGE_ALIGNED,
-	/**
-	 * @NEXT_PAGE_UNALIGNED: The end of this buffer is on
-	 * the page next to the end of the previous buffer and
-	 * is not page aligned. Examples:
-	 * buf1 ][ buf2 | buf2 ][ ...
-	 * buf1 ]|[ buf2 | buf2 ][ ...
-	 */
-	NEXT_PAGE_UNALIGNED,
-	/**
-	 * @NEXT_PAGE_ALIGNED: The end of this buffer is on
-	 * the page next to the end of the previous buffer and
-	 * is page aligned. Examples:
-	 * buf1 ][ buf2 | buf2 ]| ...
-	 * buf1 ]|[ buf2 | buf2 ]| ...
-	 */
-	NEXT_PAGE_ALIGNED,
-	/**
-	 * @NEXT_NEXT_UNALIGNED: The end of this buffer is on
-	 * the page that follows the page after the end of the
-	 * previous buffer and is not page aligned. Examples:
-	 * buf1 ][ buf2 | buf2 | buf2 ][ ...
-	 * buf1 ]|[ buf2 | buf2 | buf2 ][ ...
-	 */
-	NEXT_NEXT_UNALIGNED,
-	/**
-	 * @LOOP_END: The number of enum values in &buf_end_align_type.
-	 * It is used for controlling loop termination.
-	 */
-	LOOP_END,
-};
-
-static const char *const buf_end_align_type_strs[LOOP_END] = {
-	[SAME_PAGE_UNALIGNED] = "SP_UNALIGNED",
-	[SAME_PAGE_ALIGNED]   = " SP_ALIGNED ",
-	[NEXT_PAGE_UNALIGNED] = "NP_UNALIGNED",
-	[NEXT_PAGE_ALIGNED]   = " NP_ALIGNED ",
-	[NEXT_NEXT_UNALIGNED] = "NN_UNALIGNED",
-};
-
-struct binder_alloc_test_case_info {
-	char alignments[ALIGNMENTS_BUFLEN];
-	struct seq_buf alignments_sb;
-	size_t *buffer_sizes;
-	int *free_sequence;
-	bool front_pages;
-};
-
-static void stringify_free_seq(struct kunit *test, int *seq, struct seq_buf *sb)
-{
-	int i;
-
-	for (i = 0; i < BUFFER_NUM; i++)
-		seq_buf_printf(sb, "[%d]", seq[i]);
-
-	KUNIT_EXPECT_FALSE(test, seq_buf_has_overflowed(sb));
-}
-
-static void stringify_alignments(struct kunit *test, int *alignments,
-				 struct seq_buf *sb)
-{
-	int i;
-
-	for (i = 0; i < BUFFER_NUM; i++)
-		seq_buf_printf(sb, "[ %d:%s ]", i,
-			       buf_end_align_type_strs[alignments[i]]);
-
-	KUNIT_EXPECT_FALSE(test, seq_buf_has_overflowed(sb));
-}
-
-static bool check_buffer_pages_allocated(struct kunit *test,
-					 struct binder_alloc *alloc,
-					 struct binder_buffer *buffer,
-					 size_t size)
-{
-	unsigned long page_addr;
-	unsigned long end;
-	int page_index;
-
-	end = PAGE_ALIGN(buffer->user_data + size);
-	page_addr = buffer->user_data;
-	for (; page_addr < end; page_addr += PAGE_SIZE) {
-		page_index = (page_addr - alloc->vm_start) / PAGE_SIZE;
-		if (!alloc->pages[page_index] ||
-		    !list_empty(page_to_lru(alloc->pages[page_index]))) {
-			kunit_err(test, "expect alloc but is %s at page index %d\n",
-				  alloc->pages[page_index] ?
-				  "lru" : "free", page_index);
-			return false;
-		}
-	}
-	return true;
-}
-
-static unsigned long binder_alloc_test_alloc_buf(struct kunit *test,
-						 struct binder_alloc *alloc,
-						 struct binder_buffer *buffers[],
-						 size_t *sizes, int *seq)
-{
-	unsigned long failures = 0;
-	int i;
-
-	for (i = 0; i < BUFFER_NUM; i++) {
-		buffers[i] = binder_alloc_new_buf(alloc, sizes[i], 0, 0, 0);
-		if (IS_ERR(buffers[i]) ||
-		    !check_buffer_pages_allocated(test, alloc, buffers[i], sizes[i]))
-			failures++;
-	}
-
-	return failures;
-}
-
-static unsigned long binder_alloc_test_free_buf(struct kunit *test,
-						struct binder_alloc *alloc,
-						struct binder_buffer *buffers[],
-						size_t *sizes, int *seq, size_t end)
-{
-	unsigned long failures = 0;
-	int i;
-
-	for (i = 0; i < BUFFER_NUM; i++)
-		binder_alloc_free_buf(alloc, buffers[seq[i]]);
-
-	for (i = 0; i <= (end - 1) / PAGE_SIZE; i++) {
-		if (list_empty(page_to_lru(alloc->pages[i]))) {
-			kunit_err(test, "expect lru but is %s at page index %d\n",
-				  alloc->pages[i] ? "alloc" : "free", i);
-			failures++;
-		}
-	}
-
-	return failures;
-}
-
-static unsigned long binder_alloc_test_free_page(struct kunit *test,
-						 struct binder_alloc *alloc)
-{
-	unsigned long failures = 0;
-	unsigned long count;
-	int i;
-
-	while ((count = list_lru_count(alloc->freelist))) {
-		list_lru_walk(alloc->freelist, binder_alloc_free_page,
-			      NULL, count);
-	}
-
-	for (i = 0; i < (alloc->buffer_size / PAGE_SIZE); i++) {
-		if (alloc->pages[i]) {
-			kunit_err(test, "expect free but is %s at page index %d\n",
-				  list_empty(page_to_lru(alloc->pages[i])) ?
-				  "alloc" : "lru", i);
-			failures++;
-		}
-	}
-
-	return failures;
-}
-
-/* Executes one full test run for the given test case. */
-static bool binder_alloc_test_alloc_free(struct kunit *test,
-					 struct binder_alloc *alloc,
-					 struct binder_alloc_test_case_info *tc,
-					 size_t end)
-{
-	unsigned long pages = PAGE_ALIGN(end) / PAGE_SIZE;
-	struct binder_buffer *buffers[BUFFER_NUM];
-	unsigned long failures;
-	bool failed = false;
-
-	failures = binder_alloc_test_alloc_buf(test, alloc, buffers,
-					       tc->buffer_sizes,
-					       tc->free_sequence);
-	failed = failed || failures;
-	KUNIT_EXPECT_EQ_MSG(test, failures, 0,
-			    "Initial allocation failed: %lu/%u buffers with errors",
-			    failures, BUFFER_NUM);
-
-	failures = binder_alloc_test_free_buf(test, alloc, buffers,
-					      tc->buffer_sizes,
-					      tc->free_sequence, end);
-	failed = failed || failures;
-	KUNIT_EXPECT_EQ_MSG(test, failures, 0,
-			    "Initial buffers not freed correctly: %lu/%lu pages not on lru list",
-			    failures, pages);
-
-	/* Allocate from lru. */
-	failures = binder_alloc_test_alloc_buf(test, alloc, buffers,
-					       tc->buffer_sizes,
-					       tc->free_sequence);
-	failed = failed || failures;
-	KUNIT_EXPECT_EQ_MSG(test, failures, 0,
-			    "Reallocation failed: %lu/%u buffers with errors",
-			    failures, BUFFER_NUM);
-
-	failures = list_lru_count(alloc->freelist);
-	failed = failed || failures;
-	KUNIT_EXPECT_EQ_MSG(test, failures, 0,
-			    "lru list should be empty after reallocation but still has %lu pages",
-			    failures);
-
-	failures = binder_alloc_test_free_buf(test, alloc, buffers,
-					      tc->buffer_sizes,
-					      tc->free_sequence, end);
-	failed = failed || failures;
-	KUNIT_EXPECT_EQ_MSG(test, failures, 0,
-			    "Reallocated buffers not freed correctly: %lu/%lu pages not on lru list",
-			    failures, pages);
-
-	failures = binder_alloc_test_free_page(test, alloc);
-	failed = failed || failures;
-	KUNIT_EXPECT_EQ_MSG(test, failures, 0,
-			    "Failed to clean up allocated pages: %lu/%lu pages still installed",
-			    failures, (alloc->buffer_size / PAGE_SIZE));
-
-	return failed;
-}
-
-static bool is_dup(int *seq, int index, int val)
-{
-	int i;
-
-	for (i = 0; i < index; i++) {
-		if (seq[i] == val)
-			return true;
-	}
-	return false;
-}
-
-/* Generate BUFFER_NUM factorial free orders. */
-static void permute_frees(struct kunit *test, struct binder_alloc *alloc,
-			  struct binder_alloc_test_case_info *tc,
-			  unsigned long *runs, unsigned long *failures,
-			  int index, size_t end)
-{
-	bool case_failed;
-	int i;
-
-	if (index == BUFFER_NUM) {
-		DECLARE_SEQ_BUF(freeseq_sb, FREESEQ_BUFLEN);
-
-		case_failed = binder_alloc_test_alloc_free(test, alloc, tc, end);
-		*runs += 1;
-		*failures += case_failed;
-
-		if (case_failed || PRINT_ALL_CASES) {
-			stringify_free_seq(test, tc->free_sequence,
-					   &freeseq_sb);
-			kunit_err(test, "case %lu: [%s] | %s - %s - %s", *runs,
-				  case_failed ? "FAILED" : "PASSED",
-				  tc->front_pages ? "front" : "back ",
-				  seq_buf_str(&tc->alignments_sb),
-				  seq_buf_str(&freeseq_sb));
-		}
-
-		return;
-	}
-	for (i = 0; i < BUFFER_NUM; i++) {
-		if (is_dup(tc->free_sequence, index, i))
-			continue;
-		tc->free_sequence[index] = i;
-		permute_frees(test, alloc, tc, runs, failures, index + 1, end);
-	}
-}
-
-static void gen_buf_sizes(struct kunit *test,
-			  struct binder_alloc *alloc,
-			  struct binder_alloc_test_case_info *tc,
-			  size_t *end_offset, unsigned long *runs,
-			  unsigned long *failures)
-{
-	size_t last_offset, offset = 0;
-	size_t front_sizes[BUFFER_NUM];
-	size_t back_sizes[BUFFER_NUM];
-	int seq[BUFFER_NUM] = {0};
-	int i;
-
-	tc->free_sequence = seq;
-	for (i = 0; i < BUFFER_NUM; i++) {
-		last_offset = offset;
-		offset = end_offset[i];
-		front_sizes[i] = offset - last_offset;
-		back_sizes[BUFFER_NUM - i - 1] = front_sizes[i];
-	}
-	back_sizes[0] += alloc->buffer_size - end_offset[BUFFER_NUM - 1];
-
-	/*
-	 * Buffers share the first or last few pages.
-	 * Only BUFFER_NUM - 1 buffer sizes are adjustable since
-	 * we need one giant buffer before getting to the last page.
-	 */
-	tc->front_pages = true;
-	tc->buffer_sizes = front_sizes;
-	permute_frees(test, alloc, tc, runs, failures, 0,
-		      end_offset[BUFFER_NUM - 1]);
-
-	tc->front_pages = false;
-	tc->buffer_sizes = back_sizes;
-	permute_frees(test, alloc, tc, runs, failures, 0, alloc->buffer_size);
-}
-
-static void gen_buf_offsets(struct kunit *test, struct binder_alloc *alloc,
-			    size_t *end_offset, int *alignments,
-			    unsigned long *runs, unsigned long *failures,
-			    int index)
-{
-	size_t end, prev;
-	int align;
-
-	if (index == BUFFER_NUM) {
-		struct binder_alloc_test_case_info tc = {0};
-
-		seq_buf_init(&tc.alignments_sb, tc.alignments,
-			     ALIGNMENTS_BUFLEN);
-		stringify_alignments(test, alignments, &tc.alignments_sb);
-
-		gen_buf_sizes(test, alloc, &tc, end_offset, runs, failures);
-		return;
-	}
-	prev = index == 0 ? 0 : end_offset[index - 1];
-	end = prev;
-
-	BUILD_BUG_ON(BUFFER_MIN_SIZE * BUFFER_NUM >= PAGE_SIZE);
-
-	for (align = SAME_PAGE_UNALIGNED; align < LOOP_END; align++) {
-		if (align % 2)
-			end = ALIGN(end, PAGE_SIZE);
-		else
-			end += BUFFER_MIN_SIZE;
-		end_offset[index] = end;
-		alignments[index] = align;
-		gen_buf_offsets(test, alloc, end_offset, alignments, runs,
-				failures, index + 1);
-	}
-}
-
-struct binder_alloc_test {
-	struct binder_alloc alloc;
-	struct list_lru binder_test_freelist;
-	struct file *filp;
-	unsigned long mmap_uaddr;
-};
-
-static void binder_alloc_test_init_freelist(struct kunit *test)
-{
-	struct binder_alloc_test *priv = test->priv;
-
-	KUNIT_EXPECT_PTR_EQ(test, priv->alloc.freelist,
-			    &priv->binder_test_freelist);
-}
-
-static void binder_alloc_test_mmap(struct kunit *test)
-{
-	struct binder_alloc_test *priv = test->priv;
-	struct binder_alloc *alloc = &priv->alloc;
-	struct binder_buffer *buf;
-	struct rb_node *n;
-
-	KUNIT_EXPECT_EQ(test, alloc->mapped, true);
-	KUNIT_EXPECT_EQ(test, alloc->buffer_size, BINDER_MMAP_SIZE);
-
-	n = rb_first(&alloc->allocated_buffers);
-	KUNIT_EXPECT_PTR_EQ(test, n, NULL);
-
-	n = rb_first(&alloc->free_buffers);
-	buf = rb_entry(n, struct binder_buffer, rb_node);
-	KUNIT_EXPECT_EQ(test, binder_alloc_buffer_size(alloc, buf),
-			BINDER_MMAP_SIZE);
-	KUNIT_EXPECT_TRUE(test, list_is_last(&buf->entry, &alloc->buffers));
-}
-
-/**
- * binder_alloc_exhaustive_test() - Exhaustively test alloc and free of buffer pages.
- * @test: The test context object.
- *
- * Allocate BUFFER_NUM buffers to cover all page alignment cases,
- * then free them in all orders possible. Check that pages are
- * correctly allocated, put onto lru when buffers are freed, and
- * are freed when binder_alloc_free_page() is called.
- */
-static void binder_alloc_exhaustive_test(struct kunit *test)
-{
-	struct binder_alloc_test *priv = test->priv;
-	size_t end_offset[BUFFER_NUM];
-	int alignments[BUFFER_NUM];
-	unsigned long failures = 0;
-	unsigned long runs = 0;
-
-	gen_buf_offsets(test, &priv->alloc, end_offset, alignments, &runs,
-			&failures, 0);
-
-	KUNIT_EXPECT_EQ(test, runs, TOTAL_EXHAUSTIVE_CASES);
-	KUNIT_EXPECT_EQ(test, failures, 0);
-}
-
-/* ===== End test cases ===== */
-
-static void binder_alloc_test_vma_close(struct vm_area_struct *vma)
-{
-	struct binder_alloc *alloc = vma->vm_private_data;
-
-	binder_alloc_vma_close(alloc);
-}
-
-static const struct vm_operations_struct binder_alloc_test_vm_ops = {
-	.close = binder_alloc_test_vma_close,
-	.fault = binder_vm_fault,
-};
-
-static int binder_alloc_test_mmap_handler(struct file *filp,
-					  struct vm_area_struct *vma)
-{
-	struct binder_alloc *alloc = filp->private_data;
-
-	vm_flags_mod(vma, VM_DONTCOPY | VM_MIXEDMAP, VM_MAYWRITE);
-
-	vma->vm_ops = &binder_alloc_test_vm_ops;
-	vma->vm_private_data = alloc;
-
-	return binder_alloc_mmap_handler(alloc, vma);
-}
-
-static const struct file_operations binder_alloc_test_fops = {
-	.mmap = binder_alloc_test_mmap_handler,
-};
-
-static int binder_alloc_test_init(struct kunit *test)
-{
-	struct binder_alloc_test *priv;
-	int ret;
-
-	priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-	test->priv = priv;
-
-	ret = list_lru_init(&priv->binder_test_freelist);
-	if (ret) {
-		kunit_err(test, "Failed to initialize test freelist\n");
-		return ret;
-	}
-
-	/* __binder_alloc_init requires mm to be attached */
-	ret = kunit_attach_mm();
-	if (ret) {
-		kunit_err(test, "Failed to attach mm\n");
-		return ret;
-	}
-	__binder_alloc_init(&priv->alloc, &priv->binder_test_freelist);
-
-	priv->filp = anon_inode_getfile("binder_alloc_kunit",
-					&binder_alloc_test_fops, &priv->alloc,
-					O_RDWR | O_CLOEXEC);
-	if (IS_ERR_OR_NULL(priv->filp)) {
-		kunit_err(test, "Failed to open binder alloc test driver file\n");
-		return priv->filp ? PTR_ERR(priv->filp) : -ENOMEM;
-	}
-
-	priv->mmap_uaddr = kunit_vm_mmap(test, priv->filp, 0, BINDER_MMAP_SIZE,
-					 PROT_READ, MAP_PRIVATE | MAP_NORESERVE,
-					 0);
-	if (!priv->mmap_uaddr) {
-		kunit_err(test, "Could not map the test's transaction memory\n");
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-static void binder_alloc_test_exit(struct kunit *test)
-{
-	struct binder_alloc_test *priv = test->priv;
-
-	/* Close the backing file to make sure binder_alloc_vma_close runs */
-	if (!IS_ERR_OR_NULL(priv->filp))
-		fput(priv->filp);
-
-	if (priv->alloc.mm)
-		binder_alloc_deferred_release(&priv->alloc);
-
-	/* Make sure freelist is empty */
-	KUNIT_EXPECT_EQ(test, list_lru_count(&priv->binder_test_freelist), 0);
-	list_lru_destroy(&priv->binder_test_freelist);
-}
-
-static struct kunit_case binder_alloc_test_cases[] = {
-	KUNIT_CASE(binder_alloc_test_init_freelist),
-	KUNIT_CASE(binder_alloc_test_mmap),
-	KUNIT_CASE_SLOW(binder_alloc_exhaustive_test),
-	{}
-};
-
-static struct kunit_suite binder_alloc_test_suite = {
-	.name = "binder_alloc",
-	.test_cases = binder_alloc_test_cases,
-	.init = binder_alloc_test_init,
-	.exit = binder_alloc_test_exit,
-};
-
-kunit_test_suite(binder_alloc_test_suite);
-
-MODULE_AUTHOR("Tiffany Yang <ynaffit@google.com>");
-MODULE_DESCRIPTION("Binder Alloc KUnit tests");
-MODULE_LICENSE("GPL");
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index 28ca856ecc7540..9704ed688775bb 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -328,13 +328,6 @@ config SATA_AHCI_SEATTLE
 
 	 If unsure, say N
 
-config SATA_INIC162X
-	tristate "Initio 162x SATA support (Very Experimental)"
-	depends on PCI
-	select SATA_HOST
-	help
-	  This option enables support for Initio 162x Serial ATA.
-
 config SATA_ACARD_AHCI
 	tristate "ACard AHCI variant (ATP 8620)"
 	depends on PCI
@@ -1040,6 +1033,16 @@ config PATA_GAYLE
 
 	  If unsure, say N.
 
+config PATA_CSWARP
+	tristate "Amiga CS Warp PATA support"
+	depends on M68K && AMIGA && ZORRO
+	help
+	  This option enables support for the on-board IDE interface on
+	  CS-Lab Warp Expansion Card (NOTE that idemode=native has to be
+	  set in Warp Configuration)
+
+	  If unsure, say N.
+
 config PATA_BUDDHA
 	tristate "Buddha/Catweasel/X-Surf PATA support"
 	depends on ZORRO
diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile
index b96025abd45e36..20b400aea0c45c 100644
--- a/drivers/ata/Makefile
+++ b/drivers/ata/Makefile
@@ -11,7 +11,6 @@ obj-$(CONFIG_SATA_AHCI_SEATTLE)	+= ahci_seattle.o libahci.o libahci_platform.o
 obj-$(CONFIG_SATA_AHCI_PLATFORM) += ahci_platform.o libahci.o libahci_platform.o
 obj-$(CONFIG_SATA_FSL)		+= sata_fsl.o
 obj-$(CONFIG_SATA_GEMINI)	+= sata_gemini.o
-obj-$(CONFIG_SATA_INIC162X)	+= sata_inic162x.o
 obj-$(CONFIG_SATA_SIL24)	+= sata_sil24.o
 obj-$(CONFIG_SATA_DWC)		+= sata_dwc_460ex.o
 obj-$(CONFIG_SATA_HIGHBANK)	+= sata_highbank.o libahci.o
@@ -100,6 +99,7 @@ obj-$(CONFIG_PATA_WINBOND)	+= pata_sl82c105.o
 obj-$(CONFIG_PATA_CMD640_PCI)	+= pata_cmd640.o
 obj-$(CONFIG_PATA_FALCON)	+= pata_falcon.o
 obj-$(CONFIG_PATA_GAYLE)	+= pata_gayle.o
+obj-$(CONFIG_PATA_CSWARP)	+= pata_cswarp.o
 obj-$(CONFIG_PATA_BUDDHA)	+= pata_buddha.o
 obj-$(CONFIG_PATA_ISAPNP)	+= pata_isapnp.o
 obj-$(CONFIG_PATA_IXP4XX_CF)	+= pata_ixp4xx_cf.o
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index 9b8c0935001cb3..bce0128c81a1ad 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -26,6 +26,7 @@
 #include <linux/device.h>
 #include <linux/dmi.h>
 #include <linux/gfp.h>
+#include <linux/pm_runtime.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_cmnd.h>
 #include <linux/libata.h>
@@ -68,7 +69,6 @@ enum board_ids {
 	/* board IDs for specific chipsets in alphabetical order */
 	board_ahci_al,
 	board_ahci_avn,
-	board_ahci_jmb585,
 	board_ahci_mcp65,
 	board_ahci_mcp77,
 	board_ahci_mcp89,
@@ -213,15 +213,6 @@ static const struct ata_port_info ahci_port_info[] = {
 		.udma_mask	= ATA_UDMA6,
 		.port_ops	= &ahci_avn_ops,
 	},
-	/* JMicron JMB582/585: 64-bit DMA is broken, force 32-bit */
-	[board_ahci_jmb585] = {
-		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR |
-				 AHCI_HFLAG_32BIT_ONLY),
-		.flags		= AHCI_FLAG_COMMON,
-		.pio_mask	= ATA_PIO4,
-		.udma_mask	= ATA_UDMA6,
-		.port_ops	= &ahci_ops,
-	},
 	[board_ahci_mcp65] = {
 		AHCI_HFLAGS	(AHCI_HFLAG_NO_FPDMA_AA | AHCI_HFLAG_NO_PMP |
 				 AHCI_HFLAG_YES_NCQ),
@@ -954,14 +945,6 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 		/* Elkhart Lake AHCI */
 		PCI_VDEVICE(INTEL, 0x4b63),
 		.driver_data = board_ahci_pcs_quirk,
-	}, {
-		/* JMicron JMB582/585: force 32-bit DMA (broken 64-bit implementation) */
-		PCI_VDEVICE(JMICRON, 0x0582),
-		.driver_data = board_ahci_jmb585,
-
-	}, {
-		PCI_VDEVICE(JMICRON, 0x0585),
-		.driver_data = board_ahci_jmb585,
 	}, {
 		/* JMicron 360/1/3/5/6, match class to avoid IDE function */
 		PCI_DEVICE(PCI_VENDOR_ID_JMICRON, PCI_ANY_ID),
diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c
index 29be74fedcf019..48460e515722a9 100644
--- a/drivers/ata/ahci_brcm.c
+++ b/drivers/ata/ahci_brcm.c
@@ -16,6 +16,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/reset.h>
 #include <linux/string.h>
 
diff --git a/drivers/ata/ahci_ceva.c b/drivers/ata/ahci_ceva.c
index 2961e53288f429..008869d9499b8f 100644
--- a/drivers/ata/ahci_ceva.c
+++ b/drivers/ata/ahci_ceva.c
@@ -12,6 +12,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/reset.h>
 #include "ahci.h"
 
diff --git a/drivers/ata/ahci_da850.c b/drivers/ata/ahci_da850.c
index 4e53ff96d712c2..59b9ca26fd449f 100644
--- a/drivers/ata/ahci_da850.c
+++ b/drivers/ata/ahci_da850.c
@@ -218,8 +218,8 @@ static int ahci_da850_probe(struct platform_device *pdev)
 	return rc;
 }
 
-static SIMPLE_DEV_PM_OPS(ahci_da850_pm_ops, ahci_platform_suspend,
-			 ahci_platform_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(ahci_da850_pm_ops, ahci_platform_suspend,
+				ahci_platform_resume);
 
 static const struct of_device_id ahci_da850_of_match[] = {
 	{ .compatible = "ti,da850-ahci", },
@@ -233,7 +233,7 @@ static struct platform_driver ahci_da850_driver = {
 	.driver = {
 		.name = DRV_NAME,
 		.of_match_table = ahci_da850_of_match,
-		.pm = &ahci_da850_pm_ops,
+		.pm = pm_sleep_ptr(&ahci_da850_pm_ops),
 	},
 };
 module_platform_driver(ahci_da850_driver);
diff --git a/drivers/ata/ahci_qoriq.c b/drivers/ata/ahci_qoriq.c
index 96492159fa7c1e..fc6b8cd75b3200 100644
--- a/drivers/ata/ahci_qoriq.c
+++ b/drivers/ata/ahci_qoriq.c
@@ -14,6 +14,7 @@
 #include <linux/device.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/libata.h>
 #include "ahci.h"
 
diff --git a/drivers/ata/ahci_st.c b/drivers/ata/ahci_st.c
index 4336c8a6e20871..eea75961038cfb 100644
--- a/drivers/ata/ahci_st.c
+++ b/drivers/ata/ahci_st.c
@@ -83,18 +83,27 @@ static int st_ahci_deassert_resets(struct ahci_host_priv *hpriv,
 	return 0;
 }
 
-static void st_ahci_host_stop(struct ata_host *host)
+static int st_ahci_assert_pwrdwn(struct ahci_host_priv *hpriv,
+				 struct device *dev)
 {
-	struct ahci_host_priv *hpriv = host->private_data;
 	struct st_ahci_drv_data *drv_data = hpriv->plat_data;
-	struct device *dev = host->dev;
 	int err;
 
-	if (drv_data->pwr) {
-		err = reset_control_assert(drv_data->pwr);
-		if (err)
-			dev_err(dev, "unable to pwrdwn\n");
-	}
+	if (!drv_data->pwr)
+		return 0;
+
+	err = reset_control_assert(drv_data->pwr);
+	if (err)
+		dev_err(dev, "unable to pwrdwn\n");
+
+	return err;
+}
+
+static void st_ahci_host_stop(struct ata_host *host)
+{
+	struct ahci_host_priv *hpriv = host->private_data;
+
+	st_ahci_assert_pwrdwn(hpriv, host->dev);
 
 	ahci_platform_disable_resources(hpriv);
 }
@@ -104,23 +113,20 @@ static int st_ahci_probe_resets(struct ahci_host_priv *hpriv,
 {
 	struct st_ahci_drv_data *drv_data = hpriv->plat_data;
 
-	drv_data->pwr = devm_reset_control_get(dev, "pwr-dwn");
-	if (IS_ERR(drv_data->pwr)) {
-		dev_info(dev, "power reset control not defined\n");
-		drv_data->pwr = NULL;
-	}
+	drv_data->pwr = devm_reset_control_get_optional(dev, "pwr-dwn");
+	if (IS_ERR(drv_data->pwr))
+		return dev_err_probe(dev, PTR_ERR(drv_data->pwr),
+				     "failed to get pwr-dwn reset\n");
 
-	drv_data->sw_rst = devm_reset_control_get(dev, "sw-rst");
-	if (IS_ERR(drv_data->sw_rst)) {
-		dev_info(dev, "soft reset control not defined\n");
-		drv_data->sw_rst = NULL;
-	}
+	drv_data->sw_rst = devm_reset_control_get_optional(dev, "sw-rst");
+	if (IS_ERR(drv_data->sw_rst))
+		return dev_err_probe(dev, PTR_ERR(drv_data->sw_rst),
+				     "failed to get sw-rst reset\n");
 
-	drv_data->pwr_rst = devm_reset_control_get(dev, "pwr-rst");
-	if (IS_ERR(drv_data->pwr_rst)) {
-		dev_dbg(dev, "power soft reset control not defined\n");
-		drv_data->pwr_rst = NULL;
-	}
+	drv_data->pwr_rst = devm_reset_control_get_optional(dev, "pwr-rst");
+	if (IS_ERR(drv_data->pwr_rst))
+		return dev_err_probe(dev, PTR_ERR(drv_data->pwr_rst),
+				     "failed to get pwr-rst reset\n");
 
 	return st_ahci_deassert_resets(hpriv, dev);
 }
@@ -162,38 +168,38 @@ static int st_ahci_probe(struct platform_device *pdev)
 
 	err = ahci_platform_enable_resources(hpriv);
 	if (err)
-		return err;
+		goto assert_pwrdwn;
 
 	st_ahci_configure_oob(hpriv->mmio);
 
 	err = ahci_platform_init_host(pdev, hpriv, &st_ahci_port_info,
 				      &ahci_platform_sht);
-	if (err) {
-		ahci_platform_disable_resources(hpriv);
-		return err;
-	}
+	if (err)
+		goto disable_resources;
 
 	return 0;
+
+disable_resources:
+	ahci_platform_disable_resources(hpriv);
+assert_pwrdwn:
+	st_ahci_assert_pwrdwn(hpriv, &pdev->dev);
+
+	return err;
 }
 
 static int st_ahci_suspend(struct device *dev)
 {
 	struct ata_host *host = dev_get_drvdata(dev);
 	struct ahci_host_priv *hpriv = host->private_data;
-	struct st_ahci_drv_data *drv_data = hpriv->plat_data;
 	int err;
 
 	err = ahci_platform_suspend_host(dev);
 	if (err)
 		return err;
 
-	if (drv_data->pwr) {
-		err = reset_control_assert(drv_data->pwr);
-		if (err) {
-			dev_err(dev, "unable to pwrdwn");
-			return err;
-		}
-	}
+	err = st_ahci_assert_pwrdwn(hpriv, dev);
+	if (err)
+		return err;
 
 	ahci_platform_disable_resources(hpriv);
 
diff --git a/drivers/ata/ata_generic.c b/drivers/ata/ata_generic.c
index 18ea740ca58293..2858aed4a648ce 100644
--- a/drivers/ata/ata_generic.c
+++ b/drivers/ata/ata_generic.c
@@ -150,6 +150,9 @@ static int is_intel_ider(struct pci_dev *dev)
 	return 1;
 }
 
+/* Forward declaration for the pci_match_id() call in ata_generic_init_one() */
+static const struct pci_device_id ata_generic[];
+
 /**
  *	ata_generic_init_one		-	attach generic IDE
  *	@dev: PCI device found
@@ -172,6 +175,19 @@ static int ata_generic_init_one(struct pci_dev *dev, const struct pci_device_id
 	};
 	const struct ata_port_info *ppi[] = { &info, NULL };
 
+	/*
+	 * A device matched through driver_override or through an ID added
+	 * with new_id does not come from our ID table, so pci_match_device()
+	 * hands us a synthetic ID with no driver_data and none of the checks
+	 * below apply. Probing maps BAR0 and BAR1 as the ATA command and
+	 * control blocks and writes to the device control register, so only
+	 * continue for devices which report the IDE class or which we list
+	 * ourselves.
+	 */
+	if (!pci_match_id(ata_generic, dev) &&
+	    (dev->class >> 8) != PCI_CLASS_STORAGE_IDE)
+		return -ENODEV;
+
 	/* Don't use the generic entry unless instructed to do so */
 	if ((id->driver_data & ATA_GEN_CLASS_MATCH) && all_generic_ide == 0)
 		return -ENODEV;
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index 08ab56e955662a..e67b95e90f1063 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -30,6 +30,7 @@
 #include <scsi/scsi_cmnd.h>
 #include <linux/libata.h>
 #include <linux/pci.h>
+#include <linux/pm_runtime.h>
 #include "ahci.h"
 #include "libata.h"
 
@@ -2736,11 +2737,35 @@ static int ahci_host_activate_multi_irqs(struct ata_host *host,
 				0, pp->irq_desc, host->ports[i]);
 
 		if (rc)
-			return rc;
+			goto free_irqs;
 		ata_port_desc_misc(host->ports[i], irq);
 	}
 
-	return ata_host_register(host, sht);
+	rc = ata_host_register(host, sht);
+	if (rc)
+		goto free_irqs;
+
+	return 0;
+
+free_irqs:
+	/*
+	 * Free the IRQs which have been requested, so that the handlers can no
+	 * longer access the MMIO of the host once we return, e.g. after the
+	 * caller has disabled the clocks of the host.
+	 */
+	while (--i >= 0) {
+		struct ahci_port_priv *pp = host->ports[i]->private_data;
+
+		if (!pp)
+			continue;
+
+		devm_free_irq(host->dev, hpriv->get_irq_vector(host, i),
+			      host->ports[i]);
+	}
+
+	ata_host_undo_start(host);
+
+	return rc;
 }
 
 /**
diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c
index f6524135ea1171..ae342080556ae8 100644
--- a/drivers/ata/libahci_platform.c
+++ b/drivers/ata/libahci_platform.c
@@ -689,6 +689,10 @@ EXPORT_SYMBOL_GPL(ahci_platform_get_resources);
  * ahci-platform host, note any necessary resources (ie clks, phys, etc.)
  * must be initialized / enabled before calling this.
  *
+ * On failure, ->host_stop() is not called, so the caller has to release the
+ * resources it enabled (clocks, regulators, resets, PHYs) in its probe()
+ * error path.
+ *
  * RETURNS:
  * 0 on success otherwise a negative error code
  */
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index f482c0a6d7e986..c9a0d0b7428521 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -2972,9 +2972,10 @@ static void ata_dev_config_lpm(struct ata_device *dev)
 		    (dev->id[ATA_ID_SATA_CAPABILITY] & 0xe) == 0x2)
 			dev->quirks |= ATA_QUIRK_NOLPM;
 
-		/* ATI specific quirk */
-		if ((dev->quirks & ATA_QUIRK_NO_LPM_ON_ATI) &&
-		    ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI))
+		/* ATI and AMD specific quirk */
+		if ((dev->quirks & ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD) &&
+		    (ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI) ||
+		     ata_dev_check_adapter(dev, PCI_VENDOR_ID_AMD)))
 			dev->quirks |= ATA_QUIRK_NOLPM;
 	}
 
@@ -4136,7 +4137,7 @@ static const char * const ata_quirk_names[] = {
 	[__ATA_QUIRK_MAX_SEC]		= "maxsec",
 	[__ATA_QUIRK_MAX_TRIM_128M]	= "maxtrim128m",
 	[__ATA_QUIRK_NO_NCQ_ON_ATI]	= "noncqonati",
-	[__ATA_QUIRK_NO_LPM_ON_ATI]	= "nolpmonati",
+	[__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD] = "nolpmonatiandamd",
 	[__ATA_QUIRK_NO_ID_DEV_LOG]	= "noiddevlog",
 	[__ATA_QUIRK_NO_LOG_DIR]	= "nologdir",
 	[__ATA_QUIRK_NO_FUA]		= "nofua",
@@ -4420,15 +4421,15 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = {
 	{ "Samsung SSD 860*",		NULL,	ATA_QUIRK_NO_NCQ_TRIM |
 						ATA_QUIRK_ZERO_AFTER_TRIM |
 						ATA_QUIRK_NO_NCQ_ON_ATI |
-						ATA_QUIRK_NO_LPM_ON_ATI },
+						ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD },
 	{ "Samsung SSD 870*",		NULL,	ATA_QUIRK_NO_NCQ_TRIM |
 						ATA_QUIRK_ZERO_AFTER_TRIM |
 						ATA_QUIRK_NO_NCQ_ON_ATI |
-						ATA_QUIRK_NO_LPM_ON_ATI },
+						ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD },
 	{ "SAMSUNG*MZ7LH*",		NULL,	ATA_QUIRK_NO_NCQ_TRIM |
 						ATA_QUIRK_ZERO_AFTER_TRIM |
 						ATA_QUIRK_NO_NCQ_ON_ATI |
-						ATA_QUIRK_NO_LPM_ON_ATI },
+						ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD },
 	{ "FCCT*M500*",			NULL,	ATA_QUIRK_NO_NCQ_TRIM |
 						ATA_QUIRK_ZERO_AFTER_TRIM },
 
@@ -6127,6 +6128,46 @@ int ata_host_start(struct ata_host *host)
 }
 EXPORT_SYMBOL_GPL(ata_host_start);
 
+/**
+ *	ata_host_undo_start - undo ata_host_start()
+ *	@host: ATA host to undo_start
+ *
+ *	Stop the ports of @host and drop the devres action registered by
+ *	ata_host_start(), without calling ->host_stop().  Nothing is done if
+ *	@host has not been started.
+ *
+ *	This gives the release of the host resources back to the caller, which
+ *	is what a driver whose probe() error path releases those resources
+ *	itself needs when starting or activating the host fails.
+ *
+ *	LOCKING:
+ *	Inherited from calling layer (may sleep).
+ */
+void ata_host_undo_start(struct ata_host *host)
+{
+	int i;
+
+	if (!(host->flags & ATA_HOST_STARTED))
+		return;
+
+	for (i = 0; i < host->n_ports; i++) {
+		struct ata_port *ap = host->ports[i];
+
+		if (ap->ops->port_stop)
+			ap->ops->port_stop(ap);
+	}
+
+	/*
+	 * Drop the action added by ata_host_start() without calling it.
+	 * It does not exist if neither ->port_stop() nor ->host_stop() is
+	 * implemented, in which case there is nothing to drop.
+	 */
+	devres_destroy(host->dev, ata_host_stop, NULL, NULL);
+
+	host->flags &= ~ATA_HOST_STARTED;
+}
+EXPORT_SYMBOL_GPL(ata_host_undo_start);
+
 /**
  *	ata_host_init - Initialize a host struct for sas (ipr, libsas)
  *	@host:	host to initialize
@@ -6197,9 +6238,15 @@ static void async_port_probe(void *data, async_cookie_t cookie)
  *	@sht: template for SCSI host
  *
  *	Register initialized ATA host.  @host is allocated using
- *	ata_host_alloc() and fully initialized by LLD.  This function
- *	starts ports, registers @host with ATA and SCSI layers and
- *	probe registered devices.
+ *	ata_host_alloc(), fully initialized by the LLD and started using
+ *	ata_host_start().  This function registers @host with the ATA and
+ *	SCSI layers and probes the registered devices.
+ *
+ *	On failure, the host remains started, i.e. the devres action
+ *	registered by ata_host_start() is kept, so ->host_stop() is called by
+ *	the driver core when probe() fails.  A caller which releases the host
+ *	resources in its own probe() error path must therefore call
+ *	ata_host_undo_start().
  *
  *	LOCKING:
  *	Inherited from calling layer (may sleep).
@@ -6294,6 +6341,10 @@ EXPORT_SYMBOL_GPL(ata_host_register);
  *	have set polling mode on the port. In this case, @irq_handler
  *	should be NULL.
  *
+ *	On failure, the ports are stopped again and the devres action
+ *	registered by ata_host_start() is dropped without calling
+ *	->host_stop(), so releasing the host resources is left to the caller.
+ *
  *	LOCKING:
  *	Inherited from calling layer (may sleep).
  *
@@ -6314,27 +6365,40 @@ int ata_host_activate(struct ata_host *host, int irq,
 	/* Special case for polling mode */
 	if (!irq) {
 		WARN_ON(irq_handler);
-		return ata_host_register(host, sht);
+		rc = ata_host_register(host, sht);
+		if (rc)
+			goto undo_start;
+
+		return 0;
 	}
 
 	irq_desc = devm_kasprintf(host->dev, GFP_KERNEL, "%s[%s]",
 				  dev_driver_string(host->dev),
 				  dev_name(host->dev));
-	if (!irq_desc)
-		return -ENOMEM;
+	if (!irq_desc) {
+		rc = -ENOMEM;
+		goto undo_start;
+	}
 
 	rc = devm_request_irq(host->dev, irq, irq_handler, irq_flags,
 			      irq_desc, host);
 	if (rc)
-		return rc;
+		goto undo_start;
 
 	for (i = 0; i < host->n_ports; i++)
 		ata_port_desc_misc(host->ports[i], irq);
 
 	rc = ata_host_register(host, sht);
-	/* if failed, just free the IRQ and leave ports alone */
-	if (rc)
+	if (rc) {
+		/* Free the IRQ, so that the handler can no longer run */
 		devm_free_irq(host->dev, irq, host);
+		goto undo_start;
+	}
+
+	return 0;
+
+undo_start:
+	ata_host_undo_start(host);
 
 	return rc;
 }
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index c93c24b22d9ca3..b7e8545e569e63 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -573,9 +573,9 @@ void ata_scsi_error(struct Scsi_Host *host)
 	int nr_timedout;
 	LIST_HEAD(eh_work_q);
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	list_splice_init(&host->eh_cmd_q, &eh_work_q);
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 
 	/*
 	 * First check what errors we got with ata_scsi_cmd_error_handler().
@@ -1516,11 +1516,13 @@ static bool ata_eh_request_sense(struct ata_queued_cmd *qc)
 	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0);
 	/* Ignore err_mask; ATA_ERR might be set */
 	if (tf.status & ATA_SENSE) {
-		if (ata_scsi_sense_is_valid(tf.lbah, tf.lbam, tf.lbal)) {
+		u16 sense_code = scsi_sense_code(tf.lbam, tf.lbal);
+
+		if (ata_scsi_sense_is_valid(tf.lbah, sense_code)) {
 			/* Set sense without also setting scsicmd->result */
-			scsi_build_sense_buffer(dev->flags & ATA_DFLAG_D_SENSE,
-						cmd->sense_buffer, tf.lbah,
-						tf.lbam, tf.lbal);
+			scsi_set_sense_buffer(dev->flags & ATA_DFLAG_D_SENSE,
+					      cmd->sense_buffer, tf.lbah,
+					      sense_code);
 			qc->flags |= ATA_QCFLAG_SENSE_VALID;
 			return true;
 		}
@@ -2094,7 +2096,8 @@ static void ata_eh_get_success_sense(struct ata_link *link)
 			continue;
 
 		/* This success command had sense data, but we failed to get. */
-		ata_scsi_set_sense(dev, qc->scsicmd, ABORTED_COMMAND, 0, 0);
+		ata_scsi_set_sense(dev, qc->scsicmd, ABORTED_COMMAND,
+				   NO_ADDITIONAL_SENSE_INFORMATION);
 		qc->flags |= ATA_QCFLAG_SENSE_VALID;
 	}
 	ata_eh_done(link, dev, ATA_EH_GET_SUCCESS_SENSE);
diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c
index ad40f516c6d435..32fd0e804654b2 100644
--- a/drivers/ata/libata-sata.c
+++ b/drivers/ata/libata-sata.c
@@ -1588,7 +1588,7 @@ int ata_eh_get_ncq_success_sense(struct ata_link *link)
 		asc = sense[1];
 		ascq = sense[2];
 
-		if (!ata_scsi_sense_is_valid(sk, asc, ascq)) {
+		if (!ata_scsi_sense_is_valid(sk, scsi_sense_code(asc, ascq))) {
 			ret = -EIO;
 			continue;
 		}
@@ -1605,9 +1605,9 @@ int ata_eh_get_ncq_success_sense(struct ata_link *link)
 			qc->result_tf.auxiliary = get_unaligned_le32(&sense[16]);
 
 		/* Set sense without also setting scsicmd->result */
-		scsi_build_sense_buffer(dev->flags & ATA_DFLAG_D_SENSE,
-					qc->scsicmd->sense_buffer, sk,
-					asc, ascq);
+		scsi_set_sense_buffer(dev->flags & ATA_DFLAG_D_SENSE,
+				      qc->scsicmd->sense_buffer, sk,
+				      scsi_sense_code(asc, ascq));
 		qc->flags |= ATA_QCFLAG_SENSE_VALID;
 
 		/*
@@ -1684,14 +1684,14 @@ void ata_eh_analyze_ncq_error(struct ata_link *link)
 	 * stored the sense data in qc->result_tf.auxiliary.
 	 */
 	if (qc->result_tf.auxiliary) {
-		char sense_key, asc, ascq;
+		u16 sense_code;
+		u8 sense_key;
 
 		sense_key = (qc->result_tf.auxiliary >> 16) & 0xff;
-		asc = (qc->result_tf.auxiliary >> 8) & 0xff;
-		ascq = qc->result_tf.auxiliary & 0xff;
-		if (ata_scsi_sense_is_valid(sense_key, asc, ascq)) {
-			ata_scsi_set_sense(dev, qc->scsicmd, sense_key, asc,
-					   ascq);
+		sense_code = qc->result_tf.auxiliary & 0xffff;
+		if (ata_scsi_sense_is_valid(sense_key, sense_code)) {
+			ata_scsi_set_sense(dev, qc->scsicmd, sense_key,
+					   sense_code);
 			qc->flags |= ATA_QCFLAG_SENSE_VALID;
 		}
 	}
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 7e22bbc382384e..3045dd72aa558b 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -190,28 +190,25 @@ DEVICE_ATTR(unload_heads, S_IRUGO | S_IWUSR,
 	    ata_scsi_park_show, ata_scsi_park_store);
 EXPORT_SYMBOL_GPL(dev_attr_unload_heads);
 
-bool ata_scsi_sense_is_valid(u8 sk, u8 asc, u8 ascq)
+bool ata_scsi_sense_is_valid(u8 sense_key, u16 sense_code)
 {
-	/*
-	 * If sk == NO_SENSE, and asc + ascq == NO ADDITIONAL SENSE INFORMATION,
-	 * then there is no sense data to add.
-	 */
-	if (sk == 0 && asc == 0 && ascq == 0)
+	if (sense_key == NO_SENSE &&
+	    sense_code == NO_ADDITIONAL_SENSE_INFORMATION)
 		return false;
 
 	/* If sk > COMPLETED, sense data is bogus. */
-	if (sk > COMPLETED)
+	if (sense_key > COMPLETED)
 		return false;
 
 	return true;
 }
 
 void ata_scsi_set_sense(struct ata_device *dev, struct scsi_cmnd *cmd,
-			u8 sk, u8 asc, u8 ascq)
+			u8 sk, u16 code)
 {
 	bool d_sense = (dev->flags & ATA_DFLAG_D_SENSE);
 
-	scsi_build_sense(cmd, d_sense, sk, asc, ascq);
+	scsi_set_sense(cmd, d_sense, sk, code);
 }
 
 static void ata_scsi_set_sense_information(struct ata_queued_cmd *qc)
@@ -261,12 +258,18 @@ static void ata_scsi_set_passthru_sense_fields(struct ata_queued_cmd *qc)
 
 		/* descriptor format */
 		len = sb[7];
-		desc = (char *)scsi_sense_desc_find(sb, len + 8, 9);
+		desc = (char *)scsi_sense_desc_find(sb, SCSI_SENSE_BUFFERSIZE, 9);
 		if (!desc) {
-			if (SCSI_SENSE_BUFFERSIZE < len + 14)
+			/*
+			 * The descriptor is written at sb[8 + len] and is 14
+			 * bytes long, so it needs len + 22 bytes of buffer.
+			 */
+			if (len + 22 > SCSI_SENSE_BUFFERSIZE)
 				return;
 			sb[7] = len + 14;
 			desc = sb + 8 + len;
+		} else if (desc - sb > SCSI_SENSE_BUFFERSIZE - 14) {
+			return;
 		}
 		desc[0] = 9;
 		desc[1] = 12;
@@ -316,8 +319,7 @@ static void ata_scsi_set_passthru_sense_fields(struct ata_queued_cmd *qc)
 static void ata_scsi_set_invalid_field(struct ata_device *dev,
 				       struct scsi_cmnd *cmd, u16 field, u8 bit)
 {
-	ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x24, 0x0);
-	/* "Invalid field in CDB" */
+	ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 	scsi_set_sense_field_pointer(cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE,
 				     field, bit, 1);
 }
@@ -325,8 +327,8 @@ static void ata_scsi_set_invalid_field(struct ata_device *dev,
 static void ata_scsi_set_invalid_parameter(struct ata_device *dev,
 					   struct scsi_cmnd *cmd, u16 field)
 {
-	/* "Invalid field in parameter list" */
-	ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x26, 0x0);
+	ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST,
+			   INVALID_FIELD_IN_PARAMETER_LIST);
 	scsi_set_sense_field_pointer(cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE,
 				     field, 0xff, 0);
 }
@@ -525,7 +527,8 @@ int ata_cmd_ioctl(struct scsi_device *scsidev, void __user *arg)
 		 * check condition even if no error. Filter that. */
 		if (scsi_status_is_check_condition(cmd_result)) {
 			if (sshdr.sense_key == RECOVERED_ERROR &&
-			    sshdr.asc == 0 && sshdr.ascq == 0x1d)
+			    sshdr.sense_code ==
+			    ATA_PASS_THROUGH_INFORMATION_AVAILABLE)
 				cmd_result &= ~SAM_STAT_CHECK_CONDITION;
 		}
 
@@ -613,7 +616,8 @@ int ata_task_ioctl(struct scsi_device *scsidev, void __user *arg)
 		 * check condition even if no error. Filter that. */
 		if (cmd_result & SAM_STAT_CHECK_CONDITION) {
 			if (sshdr.sense_key == RECOVERED_ERROR &&
-			    sshdr.asc == 0 && sshdr.ascq == 0x1d)
+			    sshdr.sense_code ==
+			    ATA_PASS_THROUGH_INFORMATION_AVAILABLE)
 				cmd_result &= ~SAM_STAT_CHECK_CONDITION;
 		}
 
@@ -792,79 +796,143 @@ static void ata_qc_set_pc_nbytes(struct ata_queued_cmd *qc)
 	qc->nbytes = scsi_bufflen(scmd) + qc->extrabytes;
 }
 
+struct ata_err_sense {
+	u8	err_mask;
+	u8	sense_key;
+	u16	sense_code;
+};
+
 /**
  *	ata_to_sense_error - convert ATA error to SCSI error
  *	@drv_stat: value contained in ATA status register
  *	@drv_err: value contained in ATA error register
  *	@sk: the sense key we'll fill out
- *	@asc: the additional sense code we'll fill out
- *	@ascq: the additional sense code qualifier we'll fill out
+ *	@scode: the additional sense code and its qualifier we'll fill out
  *
  *	Converts an ATA error into a SCSI error.  Fill out pointers to
- *	SK, ASC, and ASCQ bytes for later use in fixed or descriptor
+ *	the sense key and sense code for later use in fixed or descriptor
  *	format sense blocks.
  *
  *	LOCKING:
  *	spin_lock_irqsave(host lock)
  */
-static void ata_to_sense_error(u8 drv_stat, u8 drv_err, u8 *sk, u8 *asc,
-			       u8 *ascq)
+static void ata_to_sense_error(u8 drv_stat, u8 drv_err, u8 *sk, u16 *scode)
 {
 	int i;
 
 	/* Based on the 3ware driver translation table */
-	static const unsigned char sense_table[][4] = {
-		/* BBD|ECC|ID|MAR */
-		{0xd1,		ABORTED_COMMAND, 0x00, 0x00},
-			// Device busy                  Aborted command
-		/* BBD|ECC|ID */
-		{0xd0,		ABORTED_COMMAND, 0x00, 0x00},
-			// Device busy                  Aborted command
-		/* ECC|MC|MARK */
-		{0x61,		HARDWARE_ERROR, 0x00, 0x00},
-			// Device fault                 Hardware error
-		/* ICRC|ABRT */		/* NB: ICRC & !ABRT is BBD */
-		{0x84,		ABORTED_COMMAND, 0x47, 0x00},
-			// Data CRC error               SCSI parity error
-		/* MC|ID|ABRT|TRK0|MARK */
-		{0x37,		NOT_READY, 0x04, 0x00},
-			// Unit offline                 Not ready
-		/* MCR|MARK */
-		{0x09,		NOT_READY, 0x04, 0x00},
-			// Unrecovered disk error       Not ready
-		/*  Bad address mark */
-		{0x01,		MEDIUM_ERROR, 0x13, 0x00},
-			// Address mark not found for data field
-		/* TRK0 - Track 0 not found */
-		{0x02,		HARDWARE_ERROR, 0x00, 0x00},
-			// Hardware error
-		/* Abort: 0x04 is not translated here, see below */
-		/* Media change request */
-		{0x08,		NOT_READY, 0x04, 0x00},
-			// FIXME: faking offline
-		/* SRV/IDNF - ID not found */
-		{0x10,		ILLEGAL_REQUEST, 0x21, 0x00},
-			// Logical address out of range
-		/* MC - Media Changed */
-		{0x20,		UNIT_ATTENTION, 0x28, 0x00},
-			// Not ready to ready change, medium may have changed
-		/* ECC - Uncorrectable ECC error */
-		{0x40,		MEDIUM_ERROR, 0x11, 0x04},
-			// Unrecovered read error
-		/* BBD - block marked bad */
-		{0x80,		MEDIUM_ERROR, 0x11, 0x04},
-			// Block marked bad	Medium error, unrecovered read error
-		{0xFF, 0xFF, 0xFF, 0xFF}, // END mark
+	static const struct ata_err_sense sense_table[] = {
+		{
+			/* BBD|ECC|ID|MAR - Device busy */
+			0xd1,
+			ABORTED_COMMAND,
+			NO_ADDITIONAL_SENSE_INFORMATION
+		},
+		{
+			/* BBD|ECC|ID - Device busy */
+			0xd0,
+			ABORTED_COMMAND,
+			NO_ADDITIONAL_SENSE_INFORMATION
+		},
+		{
+			/* ECC|MC|MARK - Device fault */
+			0x61,
+			HARDWARE_ERROR,
+			NO_ADDITIONAL_SENSE_INFORMATION
+		},
+		{
+			/* ICRC|ABRT - Data CRC error */
+			/* NB: ICRC & !ABRT is BBD */
+			0x84,
+			ABORTED_COMMAND,
+			SCSI_PARITY_ERROR
+		},
+		{
+			/* MC|ID|ABRT|TRK0|MARK - Unit offline */
+			0x37,
+			NOT_READY,
+			LU_NOT_READY
+		},
+		{
+			/* MCR|MARK - Unrecovered disk error */
+			0x09,
+			NOT_READY,
+			LU_NOT_READY
+		},
+		{
+			/* Bad address mark */
+			0x01,
+			MEDIUM_ERROR,
+			ADDRESS_MARK_NOT_FOUND_FOR_DATA_FIELD
+		},
+		{
+			/* TRK0 - Track 0 not found */
+			0x02,
+			HARDWARE_ERROR,
+			NO_ADDITIONAL_SENSE_INFORMATION
+		},
+		{
+			/* Abort: 0x04 is not translated here, see below */
+			/* Media change request */
+			0x08,
+			NOT_READY,
+			LU_NOT_READY
+		},
+		{
+			/* SRV/IDNF - ID not found */
+			0x10,
+			ILLEGAL_REQUEST,
+			LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE
+		},
+		{
+			/* MC - Media Changed */
+			0x20,
+			UNIT_ATTENTION,
+			NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED
+		},
+		{
+			/* ECC - Uncorrectable ECC error */
+			0x40,
+			MEDIUM_ERROR,
+			UNRECOVERED_READ_ERROR_AUTO_REALLOCATE_FAILED
+		},
+		{
+			/* BBD - block marked bad */
+			0x80,
+			MEDIUM_ERROR,
+			UNRECOVERED_READ_ERROR_AUTO_REALLOCATE_FAILED
+		},
+		{
+			/* END mark */
+			0xFF, 0xFF, 0xFFFF
+		},
 	};
-	static const unsigned char stat_table[][4] = {
-		/* Busy: must be first because BUSY means no other bits valid */
-		{ ATA_BUSY,	ABORTED_COMMAND, 0x00, 0x00 },
-		/* Device fault: INTERNAL TARGET FAILURE */
-		{ ATA_DF,	HARDWARE_ERROR,  0x44, 0x00 },
-		/* Corrected data error */
-		{ ATA_CORR,	RECOVERED_ERROR, 0x00, 0x00 },
-
-		{ 0xFF, 0xFF, 0xFF, 0xFF }, /* END mark */
+	static const struct ata_err_sense stat_table[] = {
+		{
+			/*
+			 * Busy: must be first because BUSY means no other bits
+			 * valid.
+			 */
+			ATA_BUSY,
+			ABORTED_COMMAND,
+			NO_ADDITIONAL_SENSE_INFORMATION
+		},
+		{
+			/* Device fault */
+			ATA_DF,
+			HARDWARE_ERROR,
+			INTERNAL_TARGET_FAILURE
+		},
+		{
+			/* Corrected data error */
+			ATA_CORR,
+			RECOVERED_ERROR,
+			NO_ADDITIONAL_SENSE_INFORMATION
+		},
+		{
+			/* END mark */
+			0xFF, 0xFF, 0xFFFF
+		},
 	};
 
 	/*
@@ -876,13 +944,12 @@ static void ata_to_sense_error(u8 drv_stat, u8 drv_err, u8 *sk, u8 *asc,
 
 	if (drv_err) {
 		/* Look for drv_err */
-		for (i = 0; sense_table[i][0] != 0xFF; i++) {
+		for (i = 0; sense_table[i].err_mask != 0xFF; i++) {
 			/* Look for best matches first */
-			if ((sense_table[i][0] & drv_err) ==
-			    sense_table[i][0]) {
-				*sk = sense_table[i][1];
-				*asc = sense_table[i][2];
-				*ascq = sense_table[i][3];
+			if ((sense_table[i].err_mask & drv_err) ==
+			    sense_table[i].err_mask) {
+				*sk = sense_table[i].sense_key;
+				*scode = sense_table[i].sense_code;
 				return;
 			}
 		}
@@ -893,11 +960,10 @@ static void ata_to_sense_error(u8 drv_stat, u8 drv_err, u8 *sk, u8 *asc,
 	 * has only the ABRT bit set, we decode drv_stat.  ABRT by itself
 	 * is not descriptive enough.
 	 */
-	for (i = 0; stat_table[i][0] != 0xFF; i++) {
-		if (stat_table[i][0] & drv_stat) {
-			*sk = stat_table[i][1];
-			*asc = stat_table[i][2];
-			*ascq = stat_table[i][3];
+	for (i = 0; stat_table[i].err_mask != 0xFF; i++) {
+		if (stat_table[i].err_mask & drv_stat) {
+			*sk = stat_table[i].sense_key;
+			*scode = stat_table[i].sense_code;
 			return;
 		}
 	}
@@ -907,8 +973,7 @@ static void ata_to_sense_error(u8 drv_stat, u8 drv_err, u8 *sk, u8 *asc,
 	 * that won't cause people to do things like return a disk wrongly.
 	 */
 	*sk = ABORTED_COMMAND;
-	*asc = 0x00;
-	*ascq = 0x00;
+	*scode = NO_ADDITIONAL_SENSE_INFORMATION;
 }
 
 /*
@@ -931,40 +996,41 @@ static void ata_gen_passthru_sense(struct ata_queued_cmd *qc)
 	struct ata_device *dev = qc->dev;
 	struct scsi_cmnd *cmd = qc->scsicmd;
 	struct ata_taskfile *tf = &qc->result_tf;
-	u8 sense_key, asc, ascq;
+	u16 sense_code;
+	u8 sense_key;
 
 	if (!(qc->flags & ATA_QCFLAG_RTF_FILLED)) {
 		ata_dev_dbg(dev,
 			    "missing result TF: can't generate ATA PT sense data\n");
 		if (qc->err_mask)
-			ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND, 0, 0);
+			ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND,
+					   NO_ADDITIONAL_SENSE_INFORMATION);
 		return;
 	}
 
 	/*
-	 * Use ata_to_sense_error() to map status register bits
-	 * onto sense key, asc & ascq.
+	 * Use ata_to_sense_error() to map status register bits onto sense key
+	 * and sense code.
 	 */
 	if (qc->err_mask ||
 	    tf->status & (ATA_BUSY | ATA_DF | ATA_ERR | ATA_DRQ)) {
 		ata_to_sense_error(tf->status, tf->error,
-				   &sense_key, &asc, &ascq);
-		ata_scsi_set_sense(qc->dev, cmd, sense_key, asc, ascq);
-	} else {
-		/*
-		 * ATA PASS-THROUGH INFORMATION AVAILABLE
-		 *
-		 * Note: we are supposed to call ata_scsi_set_sense(), which
-		 * respects the D_SENSE bit, instead of unconditionally
-		 * generating the sense data in descriptor format. However,
-		 * because hdparm, hddtemp, and udisks incorrectly assume sense
-		 * data in descriptor format, without even looking at the
-		 * RESPONSE CODE field in the returned sense data (to see which
-		 * format the returned sense data is in), we are stuck with
-		 * being bug compatible with older kernels.
-		 */
-		scsi_build_sense(cmd, 1, RECOVERED_ERROR, 0, 0x1D);
+				   &sense_key, &sense_code);
+		ata_scsi_set_sense(qc->dev, cmd, sense_key, sense_code);
+		return;
 	}
+
+	/*
+	 * Note: we are supposed to call ata_scsi_set_sense(), which respects
+	 * the D_SENSE bit, instead of unconditionally generating the sense data
+	 * in descriptor format. However, because hdparm, hddtemp, and udisks
+	 * incorrectly assume sense data in descriptor format, without even
+	 * looking at the RESPONSE CODE field in the returned sense data (to see
+	 * which format the returned sense data is in), we are stuck with being
+	 * bug compatible with older kernels.
+	 */
+	scsi_set_sense(cmd, 1, RECOVERED_ERROR,
+		       ATA_PASS_THROUGH_INFORMATION_AVAILABLE);
 }
 
 /**
@@ -981,19 +1047,18 @@ static void ata_gen_ata_sense(struct ata_queued_cmd *qc)
 	struct ata_device *dev = qc->dev;
 	struct scsi_cmnd *cmd = qc->scsicmd;
 	struct ata_taskfile *tf = &qc->result_tf;
-	u8 sense_key, asc, ascq;
 
 	if (ata_dev_disabled(dev)) {
 		/* Device disabled after error recovery */
-		/* LOGICAL UNIT NOT READY, HARD RESET REQUIRED */
-		ata_scsi_set_sense(dev, cmd, NOT_READY, 0x04, 0x21);
+		ata_scsi_set_sense(dev, cmd, NOT_READY,
+				   LU_NOT_READY_HARD_RESET_REQUIRED);
 		return;
 	}
 
 	if (ata_id_is_locked(dev->id)) {
 		/* Security locked */
-		/* LOGICAL UNIT ACCESS NOT AUTHORIZED */
-		ata_scsi_set_sense(dev, cmd, DATA_PROTECT, 0x74, 0x71);
+		ata_scsi_set_sense(dev, cmd, DATA_PROTECT,
+				   LU_ACCESS_NOT_AUTHORIZED);
 		return;
 	}
 
@@ -1008,9 +1073,12 @@ static void ata_gen_ata_sense(struct ata_queued_cmd *qc)
 	 */
 	if (qc->err_mask ||
 	    tf->status & (ATA_BUSY | ATA_DF | ATA_ERR | ATA_DRQ)) {
+		u16 sense_code;
+		u8 sense_key;
+
 		ata_to_sense_error(tf->status, tf->error,
-				   &sense_key, &asc, &ascq);
-		ata_scsi_set_sense(dev, cmd, sense_key, asc, ascq);
+				   &sense_key, &sense_code);
+		ata_scsi_set_sense(dev, cmd, sense_key, sense_code);
 		return;
 	}
 
@@ -1019,7 +1087,8 @@ static void ata_gen_ata_sense(struct ata_queued_cmd *qc)
 		"Could not decode error 0x%x, status 0x%x (err_mask=0x%x)\n",
 		tf->error, tf->status, qc->err_mask);
 aborted:
-	ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND, 0, 0);
+	ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND,
+			   NO_ADDITIONAL_SENSE_INFORMATION);
 }
 
 void ata_scsi_sdev_config(struct scsi_device *sdev)
@@ -1273,7 +1342,8 @@ static unsigned int ata_scsi_start_stop_xlat(struct ata_queued_cmd *qc)
 
 	/* Ignore IMMED bit (cdb[1] & 0x1), violates sat-r05 */
 	if (!ata_dev_power_init_tf(qc->dev, &qc->tf, cdb[4] & 0x1)) {
-		ata_scsi_set_sense(qc->dev, scmd, ABORTED_COMMAND, 0, 0);
+		ata_scsi_set_sense(qc->dev, scmd, ABORTED_COMMAND,
+				   NO_ADDITIONAL_SENSE_INFORMATION);
 		return 1;
 	}
 
@@ -1501,8 +1571,8 @@ static unsigned int ata_scsi_verify_xlat(struct ata_queued_cmd *qc)
 	return 1;
 
 out_of_range:
-	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST, 0x21, 0x0);
-	/* "Logical Block Address out of range" */
+	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST,
+			   LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 	return 1;
 
 nothing_to_do:
@@ -1638,8 +1708,8 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
 	return 1;
 
 out_of_range:
-	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST, 0x21, 0x0);
-	/* "Logical Block Address out of range" */
+	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST,
+			   LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 	return 1;
 
 nothing_to_do:
@@ -2015,7 +2085,8 @@ static void ata_scsi_rbuf_fill(struct ata_device *dev, struct scsi_cmnd *cmd,
 	len = actor(dev, cmd, ata_scsi_rbuf);
 	if (len) {
 		if (WARN_ON(len > ATA_SCSI_RBUF_SIZE)) {
-			ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND, 0, 0);
+			ata_scsi_set_sense(dev, cmd, ABORTED_COMMAND,
+					   NO_ADDITIONAL_SENSE_INFORMATION);
 			spin_unlock_irqrestore(&ata_scsi_rbuf_lock, flags);
 			return;
 		}
@@ -2893,8 +2964,8 @@ static unsigned int ata_scsiop_mode_sense(struct ata_device *dev,
 	return 0;
 
 saving_not_supp:
-	ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x39, 0x0);
-	 /* "Saving parameters not supported" */
+	ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST,
+			   SAVING_PARAMETERS_NOT_SUPPORTED);
 	return 0;
 }
 
@@ -3506,6 +3577,11 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc)
 	if (is_multi_taskfile(tf)) {
 		unsigned int multi_count = 1 << (cdb[1] >> 5);
 
+		if (!dev->multi_count) {
+			ata_scsi_set_sense(dev, scmd, ABORTED_COMMAND, NO_ADDITIONAL_SENSE_INFORMATION);
+			return 1;
+		}
+
 		/* compare the passed through multi_count
 		 * with the cached multi_count of libata
 		 */
@@ -3731,12 +3807,12 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 	ata_scsi_set_invalid_field(dev, scmd, fp, bp);
 	return 1;
 invalid_param_len:
-	/* "Parameter list length error" */
-	ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x1a, 0x0);
+	ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST,
+			   PARAMETER_LIST_LENGTH_ERROR);
 	return 1;
 invalid_opcode:
-	/* "Invalid command operation code" */
-	ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+	ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST,
+			   INVALID_COMMAND_OP_CODE);
 	return 1;
 }
 
@@ -4198,8 +4274,8 @@ static unsigned int ata_scsi_zbc_in_xlat(struct ata_queued_cmd *qc)
 	return 1;
 
 invalid_param_len:
-	/* "Parameter list length error" */
-	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST, 0x1a, 0x0);
+	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST,
+			   PARAMETER_LIST_LENGTH_ERROR);
 	return 1;
 }
 
@@ -4276,8 +4352,8 @@ static unsigned int ata_scsi_zbc_out_xlat(struct ata_queued_cmd *qc)
 	ata_scsi_set_invalid_field(qc->dev, scmd, fp, 0xff);
 	return 1;
 invalid_param_len:
-	/* "Parameter list length error" */
-	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST, 0x1a, 0x0);
+	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST,
+			   PARAMETER_LIST_LENGTH_ERROR);
 	return 1;
 }
 
@@ -4628,8 +4704,8 @@ static unsigned int ata_scsi_mode_select_xlat(struct ata_queued_cmd *qc)
 	return 1;
 
  invalid_param_len:
-	/* "Parameter list length error" */
-	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST, 0x1a, 0x0);
+	ata_scsi_set_sense(qc->dev, scmd, ILLEGAL_REQUEST,
+			   PARAMETER_LIST_LENGTH_ERROR);
 	return 1;
 
  skip:
@@ -4788,7 +4864,8 @@ ata_scsi_get_phys_element_status_xlat(struct ata_queued_cmd *qc)
 
 	/* ATA_CMD_GET_PHYS_ELEMENT_STATUS is a DMA command. */
 	if (!(dev->flags & ATA_DFLAG_DEPOP) || !ata_dma_enabled(dev)) {
-		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST,
+				   INVALID_COMMAND_OP_CODE);
 		return 1;
 	}
 
@@ -4833,7 +4910,8 @@ ata_scsi_remove_element_and_truncate_xlat(struct ata_queued_cmd *qc)
 	u32 id;
 
 	if (!(dev->flags & ATA_DFLAG_DEPOP)) {
-		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST,
+				   INVALID_COMMAND_OP_CODE);
 		return 1;
 	}
 
@@ -4875,7 +4953,8 @@ ata_scsi_remove_element_and_modify_zones_xlat(struct ata_queued_cmd *qc)
 	u32 id;
 
 	if (!(dev->flags & ATA_DFLAG_DEPOP_MODIFY)) {
-		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST,
+				   INVALID_COMMAND_OP_CODE);
 		return 1;
 	}
 
@@ -4903,7 +4982,8 @@ ata_scsi_restore_elements_and_rebuild_xlat(struct ata_queued_cmd *qc)
 	struct ata_taskfile *tf = &qc->tf;
 
 	if (!(dev->flags & ATA_DFLAG_DEPOP_RESTORE)) {
-		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST, 0x20, 0x0);
+		ata_scsi_set_sense(dev, scmd, ILLEGAL_REQUEST,
+				   INVALID_COMMAND_OP_CODE);
 		return 1;
 	}
 
@@ -5063,7 +5143,8 @@ static void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd)
 		break;
 
 	case REQUEST_SENSE:
-		ata_scsi_set_sense(dev, cmd, 0, 0, 0);
+		ata_scsi_set_sense(dev, cmd, NO_SENSE,
+				   NO_ADDITIONAL_SENSE_INFORMATION);
 		break;
 
 	/* if we reach this, then writeback caching is disabled,
@@ -5092,8 +5173,8 @@ static void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd)
 
 	/* all other commands */
 	default:
-		ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST, 0x20, 0x0);
-		/* "Invalid command operation code" */
+		ata_scsi_set_sense(dev, cmd, ILLEGAL_REQUEST,
+				   INVALID_COMMAND_OP_CODE);
 		break;
 	}
 
@@ -5381,7 +5462,7 @@ bool ata_scsi_offline_dev(struct ata_device *dev)
  *	ata_scsi_remove_dev - remove attached SCSI device
  *	@dev: ATA device to remove attached SCSI device for
  *
- *	This function is called from ata_eh_scsi_hotplug() and
+ *	This function is called from ata_scsi_hotplug() and
  *	responsible for removing the SCSI device attached to @dev.
  *
  *	LOCKING:
diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c
index 976e4e160494d3..a263f886abc711 100644
--- a/drivers/ata/libata-sff.c
+++ b/drivers/ata/libata-sff.c
@@ -580,9 +580,9 @@ static void ata_pio_xfer(struct ata_queued_cmd *qc, struct page *page,
 	bool do_write = (qc->tf.flags & ATA_TFLAG_WRITE);
 	unsigned char *buf;
 
-	buf = kmap_atomic(page);
+	buf = kmap_local_page(page);
 	qc->ap->ops->sff_data_xfer(qc, buf + offset, xfer_size, do_write);
-	kunmap_atomic(buf);
+	kunmap_local(buf);
 
 	if (!do_write && !PageSlab(page))
 		flush_dcache_page(page);
@@ -763,9 +763,9 @@ static int __atapi_pio_bytes(struct ata_queued_cmd *qc, unsigned int bytes)
 	trace_atapi_pio_transfer_data(qc, offset, count);
 
 	/* do the actual data transfer */
-	buf = kmap_atomic(page);
+	buf = kmap_local_page(page);
 	consumed = ap->ops->sff_data_xfer(qc, buf + offset, count, rw);
-	kunmap_atomic(buf);
+	kunmap_local(buf);
 
 	bytes -= min(bytes, consumed);
 	qc->curbytes += count;
@@ -2267,6 +2267,11 @@ EXPORT_SYMBOL_GPL(ata_pci_sff_prepare_host);
  *	hosts.  This separate helper is necessary because SFF hosts
  *	use two separate interrupts in legacy mode.
  *
+ *	Note that, unlike ata_host_activate(), the devres action registered
+ *	by ata_host_start() is kept on failure, i.e. ->host_stop() is called
+ *	by the driver core when probe() fails.  All callers of this function
+ *	rely on that, none of them releases the host resources itself.
+ *
  *	LOCKING:
  *	Inherited from calling layer (may sleep).
  *
diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
index 48274691cd0e31..d222a2c92efcef 100644
--- a/drivers/ata/libata.h
+++ b/drivers/ata/libata.h
@@ -164,9 +164,10 @@ extern int ata_scsi_add_hosts(struct ata_host *host,
 			      const struct scsi_host_template *sht);
 extern void ata_scsi_scan_host(struct ata_port *ap, int sync);
 extern bool ata_scsi_offline_dev(struct ata_device *dev);
-extern bool ata_scsi_sense_is_valid(u8 sk, u8 asc, u8 ascq);
+extern bool ata_scsi_sense_is_valid(u8 sense_key, u16 sense_code);
 extern void ata_scsi_set_sense(struct ata_device *dev,
-			       struct scsi_cmnd *cmd, u8 sk, u8 asc, u8 ascq);
+			       struct scsi_cmnd *cmd, u8 sense_key,
+			       u16 sense_code);
 extern void ata_scsi_media_change_notify(struct ata_device *dev);
 extern void ata_scsi_hotplug(struct work_struct *work);
 extern void ata_scsi_dev_rescan(struct work_struct *work);
diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c
index e05a4847e03713..a652a1b57338c1 100644
--- a/drivers/ata/pata_arasan_cf.c
+++ b/drivers/ata/pata_arasan_cf.c
@@ -400,7 +400,7 @@ dma_xfer(struct arasan_cf_dev *acdev, dma_addr_t src, dma_addr_t dest, u32 len)
 	unsigned long flags = DMA_PREP_INTERRUPT;
 	int ret = 0;
 
-	tx = chan->device->device_prep_dma_memcpy(chan, dest, src, len, flags);
+	tx = dmaengine_prep_dma_memcpy(chan, dest, src, len, flags);
 	if (!tx) {
 		dev_err(acdev->host->dev, "device_prep_dma_memcpy failed\n");
 		return -EAGAIN;
@@ -408,7 +408,7 @@ dma_xfer(struct arasan_cf_dev *acdev, dma_addr_t src, dma_addr_t dest, u32 len)
 
 	tx->callback = dma_callback;
 	tx->callback_param = acdev;
-	cookie = tx->tx_submit(tx);
+	cookie = dmaengine_submit(tx);
 
 	ret = dma_submit_error(cookie);
 	if (ret) {
@@ -416,7 +416,7 @@ dma_xfer(struct arasan_cf_dev *acdev, dma_addr_t src, dma_addr_t dest, u32 len)
 		return ret;
 	}
 
-	chan->device->device_issue_pending(chan);
+	dma_async_issue_pending(chan);
 
 	/* Wait for DMA to complete */
 	if (!wait_for_completion_timeout(&acdev->dma_completion, TIMEOUT)) {
diff --git a/drivers/ata/pata_cswarp.c b/drivers/ata/pata_cswarp.c
new file mode 100644
index 00000000000000..58fe6d0012dc5a
--- /dev/null
+++ b/drivers/ata/pata_cswarp.c
@@ -0,0 +1,183 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Amiga CS Warp PATA controller driver
+ *
+ * Copyright (c) 2024 CS-Lab s.c.
+ *		http://www.cs-lab.eu
+ *
+ * Based on pata_gayle.c, pata_buddha.c and warpATA.device:
+ *
+ *     Created 2 Jun 2024 by Andrzej Rogozynski
+ */
+
+#include <linux/ata.h>
+#include <linux/blkdev.h>
+#include <linux/delay.h>
+#include <linux/device-id/zorro.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/libata.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/zorro.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_host.h>
+
+#include <asm/amigahw.h>
+#include <asm/amigaints.h>
+#include <asm/amigayle.h>
+#include <asm/setup.h>
+
+#define DRV_NAME "pata_cswarp"
+
+#define WARP_OFFSET_ATA         0x6000
+
+static const struct scsi_host_template pata_cswarp_sht = {
+	ATA_PIO_SHT(DRV_NAME),
+};
+
+static unsigned int pata_cswarp_data_xfer(struct ata_queued_cmd *qc,
+					  unsigned char *buf,
+					  unsigned int buflen, int rw)
+{
+	struct ata_device *dev = qc->dev;
+	struct ata_port *ap = dev->link->ap;
+	void __iomem *data_addr = ap->ioaddr.data_addr;
+	unsigned int words = buflen >> 1;
+	u16 *buf16 = (u16 *)buf;
+
+	/* Transfer multiple of 2 bytes */
+	if (rw == READ)
+		raw_insw(data_addr, buf16, words);
+	else
+		raw_outsw(data_addr, buf16, words);
+
+	/* Transfer trailing byte, if any. */
+	if (unlikely(buflen & 0x01)) {
+		if (rw == READ)
+			buf[buflen - 1] = raw_inw(data_addr) >> 8;
+		else
+			raw_outw(buf[buflen - 1] << 8, data_addr);
+		words++;
+	}
+
+	return words << 1;
+}
+
+/*
+ * Provide our own set_mode() as we don't want to change anything that has
+ * already been configured..
+ */
+static int pata_cswarp_set_mode(struct ata_link *link,
+				struct ata_device **unused)
+{
+	struct ata_device *dev;
+
+	ata_for_each_dev(dev, link, ENABLED) {
+		/* We don't really care */
+		dev->pio_mode = dev->xfer_mode = XFER_PIO_0;
+		dev->xfer_shift = ATA_SHIFT_PIO;
+		dev->flags |= ATA_DFLAG_PIO;
+		ata_dev_info(dev, "configured for PIO\n");
+	}
+	return 0;
+}
+
+static struct ata_port_operations pata_cswarp_ops = {
+	.inherits	= &ata_sff_port_ops,
+	.sff_data_xfer	= pata_cswarp_data_xfer,
+	.cable_detect	= ata_cable_unknown,
+	.set_mode	= pata_cswarp_set_mode,
+};
+
+static int pata_cswarp_probe(struct zorro_dev *z,
+			     const struct zorro_device_id *ent)
+{
+	static const char board_name[] = "csWarp";
+	struct ata_host *host;
+	struct ata_port *ap;
+	struct resource *res;
+	void __iomem *base;
+	unsigned long board = z->resource.start;
+
+	dev_info(&z->dev, "%s IDE controller (board: 0x%lx)\n", board_name,
+		 board);
+
+	res = devm_request_mem_region(&z->dev, board + WARP_OFFSET_ATA, 0x1800,
+				      DRV_NAME);
+	if (!res)
+		return -ENXIO;
+
+	host = ata_host_alloc(&z->dev, 1);
+	if (!host)
+		return -ENXIO;
+
+	ap = host->ports[0];
+	base = devm_ioremap(&z->dev, board + WARP_OFFSET_ATA, 0x1800);
+	if (!base)
+		return -ENOMEM;
+
+	ap->ops = &pata_cswarp_ops;
+
+	ap->pio_mask = ATA_PIO4;
+	ap->flags |= ATA_FLAG_SLAVE_POSS | ATA_FLAG_NO_IORDY |
+		ATA_FLAG_PIO_POLLING;
+
+	ap->ioaddr.data_addr		= base;
+	ap->ioaddr.error_addr		= base + 1 * 4;
+	ap->ioaddr.feature_addr		= base + 1 * 4;
+	ap->ioaddr.nsect_addr		= base + 2 * 4;
+	ap->ioaddr.lbal_addr		= base + 3 * 4;
+	ap->ioaddr.lbam_addr		= base + 4 * 4;
+	ap->ioaddr.lbah_addr		= base + 5 * 4;
+	ap->ioaddr.device_addr		= base + 6 * 4;
+	ap->ioaddr.status_addr		= base + 7 * 4;
+	ap->ioaddr.command_addr		= base + 7 * 4;
+
+	ap->ioaddr.altstatus_addr	= base + (0x1000 | (6UL << 2));
+	ap->ioaddr.ctl_addr		= base + (0x1000 | (6UL << 2));
+
+	ata_port_desc(ap, "mmio %pR", res);
+
+	return ata_host_activate(host, 0, NULL,
+			  IRQF_SHARED, &pata_cswarp_sht);
+}
+
+static void pata_cswarp_remove(struct zorro_dev *z)
+{
+	struct ata_host *host = dev_get_drvdata(&z->dev);
+
+	ata_host_detach(host);
+}
+
+static const struct zorro_device_id pata_cswarp_zorro_tbl[] = {
+	{ .id = ZORRO_PROD_CSLAB_WARP_1260 },
+	{ }
+};
+MODULE_DEVICE_TABLE(zorro, pata_cswarp_zorro_tbl);
+
+static struct zorro_driver pata_cswarp_driver = {
+	.name           = "pata_cswarp",
+	.id_table       = pata_cswarp_zorro_tbl,
+	.probe          = pata_cswarp_probe,
+	.remove         = pata_cswarp_remove,
+};
+
+static int __init pata_cswarp_init(void)
+{
+	return zorro_register_driver(&pata_cswarp_driver);
+}
+
+static void __exit pata_cswarp_unregister(void)
+{
+	zorro_unregister_driver(&pata_cswarp_driver);
+}
+
+module_init(pata_cswarp_init);
+module_exit(pata_cswarp_unregister);
+
+MODULE_AUTHOR("Andrzej Rogozynski");
+MODULE_DESCRIPTION("low-level driver for CSWarp PATA");
+MODULE_LICENSE("GPL");
diff --git a/drivers/ata/pata_parport/pata_parport.c b/drivers/ata/pata_parport/pata_parport.c
index 40baeac594a9f9..f0fe921a10b4ea 100644
--- a/drivers/ata/pata_parport/pata_parport.c
+++ b/drivers/ata/pata_parport/pata_parport.c
@@ -618,8 +618,9 @@ void pata_parport_unregister_driver(struct pi_protocol *pr)
 			break;
 	}
 	idr_remove(&protocols, id);
-	mutex_unlock(&pi_mutex);
 	driver_unregister(&pr->driver);
+	mutex_unlock(&pi_mutex);
+
 }
 EXPORT_SYMBOL_GPL(pata_parport_unregister_driver);
 
@@ -646,18 +647,18 @@ static ssize_t new_device_store(const struct bus_type *bus, const char *buf, siz
 		port_wanted = -1;
 	}
 
+	mutex_lock(&pi_mutex);
 	drv = driver_find(protocol, &pata_parport_bus_type);
 	if (!drv) {
 		if (strcmp(protocol, "auto")) {
 			pr_err("protocol %s not found\n", protocol);
+			mutex_unlock(&pi_mutex);
 			return -EINVAL;
 		}
 		pr_wanted = NULL;
 	} else {
 		pr_wanted = container_of(drv, struct pi_protocol, driver);
 	}
-
-	mutex_lock(&pi_mutex);
 	/* walk all parports */
 	idr_for_each_entry(&parport_list, parport, port_num) {
 		if (port_num == port_wanted || port_wanted == -1) {
@@ -722,7 +723,7 @@ static void pata_parport_attach(struct parport *port)
 	int pr_num, id;
 
 	mutex_lock(&pi_mutex);
-	id = idr_alloc(&parport_list, port, port->number, port->number,
+	id = idr_alloc(&parport_list, port, port->number, port->number + 1,
 		       GFP_KERNEL);
 	if (id < 0) {
 		mutex_unlock(&pi_mutex);
diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c
index 8e3fc713891afc..aa816a1a4c5416 100644
--- a/drivers/ata/sata_dwc_460ex.c
+++ b/drivers/ata/sata_dwc_460ex.c
@@ -1185,11 +1185,14 @@ static int sata_dwc_probe(struct platform_device *ofdev)
 	 * error_handler() to execute a dummy Softreset EH session
 	 */
 	err = ata_host_activate(host, irq, sata_dwc_isr, 0, &sata_dwc_sht);
-	if (err)
+	if (err) {
 		dev_err(dev, "failed to activate host");
+		goto error_out;
+	}
 
 	/* Enable SATA Interrupts */
 	sata_dwc_enable_interrupts(hsdev);
+
 	return 0;
 
 error_out:
diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c
index 70b210afd2911e..987d8acf1486c8 100644
--- a/drivers/ata/sata_fsl.c
+++ b/drivers/ata/sata_fsl.c
@@ -1490,8 +1490,10 @@ static int sata_fsl_probe(struct platform_device *ofdev)
 	 * device discovery process, invoking our port_start() handler &
 	 * error_handler() to execute a dummy Softreset EH session
 	 */
-	ata_host_activate(host, irq, sata_fsl_interrupt, SATA_FSL_IRQ_FLAG,
-			  &sata_fsl_sht);
+	retval = ata_host_activate(host, irq, sata_fsl_interrupt,
+				   SATA_FSL_IRQ_FLAG, &sata_fsl_sht);
+	if (retval)
+		goto error_exit_with_cleanup;
 
 	host_priv->intr_coalescing.show = fsl_sata_intr_coalescing_show;
 	host_priv->intr_coalescing.store = fsl_sata_intr_coalescing_store;
@@ -1500,7 +1502,7 @@ static int sata_fsl_probe(struct platform_device *ofdev)
 	host_priv->intr_coalescing.attr.mode = S_IRUGO | S_IWUSR;
 	retval = device_create_file(host->dev, &host_priv->intr_coalescing);
 	if (retval)
-		goto error_exit_with_cleanup;
+		goto error_exit_detach;
 
 	host_priv->rx_watermark.show = fsl_sata_rx_watermark_show;
 	host_priv->rx_watermark.store = fsl_sata_rx_watermark_store;
@@ -1510,16 +1512,24 @@ static int sata_fsl_probe(struct platform_device *ofdev)
 	retval = device_create_file(host->dev, &host_priv->rx_watermark);
 	if (retval) {
 		device_remove_file(&ofdev->dev, &host_priv->intr_coalescing);
-		goto error_exit_with_cleanup;
+		goto error_exit_detach;
 	}
 
 	return 0;
 
-error_exit_with_cleanup:
+error_exit_detach:
+	/*
+	 * Once the host has been activated, hcr_base and host_priv are
+	 * released by sata_fsl_host_stop(), which is called by the driver core
+	 * through the ata_host_stop() devres action registered by
+	 * ata_host_start(). Releasing them here as well would result in a
+	 * double iounmap() and a use-after-free.
+	 */
+	ata_host_detach(host);
 
-	if (host)
-		ata_host_detach(host);
+	return retval;
 
+error_exit_with_cleanup:
 	if (hcr_base)
 		iounmap(hcr_base);
 	kfree(host_priv);
diff --git a/drivers/ata/sata_inic162x.c b/drivers/ata/sata_inic162x.c
deleted file mode 100644
index 065408975f5b56..00000000000000
--- a/drivers/ata/sata_inic162x.c
+++ /dev/null
@@ -1,903 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * sata_inic162x.c - Driver for Initio 162x SATA controllers
- *
- * Copyright 2006  SUSE Linux Products GmbH
- * Copyright 2006  Tejun Heo <teheo@novell.com>
- *
- * **** WARNING ****
- *
- * This driver never worked properly and unfortunately data corruption is
- * relatively common.  There isn't anyone working on the driver and there's
- * no support from the vendor.  Do not use this driver in any production
- * environment.
- *
- * http://thread.gmane.org/gmane.linux.debian.devel.bugs.rc/378525/focus=54491
- * https://bugzilla.kernel.org/show_bug.cgi?id=60565
- *
- * *****************
- *
- * This controller is eccentric and easily locks up if something isn't
- * right.  Documentation is available at initio's website but it only
- * documents registers (not programming model).
- *
- * This driver has interesting history.  The first version was written
- * from the documentation and a 2.4 IDE driver posted on a Taiwan
- * company, which didn't use any IDMA features and couldn't handle
- * LBA48.  The resulting driver couldn't handle LBA48 devices either
- * making it pretty useless.
- *
- * After a while, initio picked the driver up, renamed it to
- * sata_initio162x, updated it to use IDMA for ATA DMA commands and
- * posted it on their website.  It only used ATA_PROT_DMA for IDMA and
- * attaching both devices and issuing IDMA and !IDMA commands
- * simultaneously broke it due to PIRQ masking interaction but it did
- * show how to use the IDMA (ADMA + some initio specific twists)
- * engine.
- *
- * Then, I picked up their changes again and here's the usable driver
- * which uses IDMA for everything.  Everything works now including
- * LBA48, CD/DVD burning, suspend/resume and hotplug.  There are some
- * issues tho.  Result Tf is not resported properly, NCQ isn't
- * supported yet and CD/DVD writing works with DMA assisted PIO
- * protocol (which, for native SATA devices, shouldn't cause any
- * noticeable difference).
- *
- * Anyways, so, here's finally a working driver for inic162x.  Enjoy!
- *
- * initio: If you guys wanna improve the driver regarding result TF
- * access and other stuff, please feel free to contact me.  I'll be
- * happy to assist.
- */
-
-#include <linux/gfp.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <scsi/scsi_host.h>
-#include <linux/libata.h>
-#include <linux/blkdev.h>
-#include <scsi/scsi_device.h>
-
-#define DRV_NAME	"sata_inic162x"
-#define DRV_VERSION	"0.4"
-
-enum {
-	MMIO_BAR_PCI		= 5,
-	MMIO_BAR_CARDBUS	= 1,
-
-	NR_PORTS		= 2,
-
-	IDMA_CPB_TBL_SIZE	= 4 * 32,
-
-	INIC_DMA_BOUNDARY	= 0xffffff,
-
-	HOST_ACTRL		= 0x08,
-	HOST_CTL		= 0x7c,
-	HOST_STAT		= 0x7e,
-	HOST_IRQ_STAT		= 0xbc,
-	HOST_IRQ_MASK		= 0xbe,
-
-	PORT_SIZE		= 0x40,
-
-	/* registers for ATA TF operation */
-	PORT_TF_DATA		= 0x00,
-	PORT_TF_FEATURE		= 0x01,
-	PORT_TF_NSECT		= 0x02,
-	PORT_TF_LBAL		= 0x03,
-	PORT_TF_LBAM		= 0x04,
-	PORT_TF_LBAH		= 0x05,
-	PORT_TF_DEVICE		= 0x06,
-	PORT_TF_COMMAND		= 0x07,
-	PORT_TF_ALT_STAT	= 0x08,
-	PORT_IRQ_STAT		= 0x09,
-	PORT_IRQ_MASK		= 0x0a,
-	PORT_PRD_CTL		= 0x0b,
-	PORT_PRD_ADDR		= 0x0c,
-	PORT_PRD_XFERLEN	= 0x10,
-	PORT_CPB_CPBLAR		= 0x18,
-	PORT_CPB_PTQFIFO	= 0x1c,
-
-	/* IDMA register */
-	PORT_IDMA_CTL		= 0x14,
-	PORT_IDMA_STAT		= 0x16,
-
-	PORT_RPQ_FIFO		= 0x1e,
-	PORT_RPQ_CNT		= 0x1f,
-
-	PORT_SCR		= 0x20,
-
-	/* HOST_CTL bits */
-	HCTL_LEDEN		= (1 << 3),  /* enable LED operation */
-	HCTL_IRQOFF		= (1 << 8),  /* global IRQ off */
-	HCTL_FTHD0		= (1 << 10), /* fifo threshold 0 */
-	HCTL_FTHD1		= (1 << 11), /* fifo threshold 1*/
-	HCTL_PWRDWN		= (1 << 12), /* power down PHYs */
-	HCTL_SOFTRST		= (1 << 13), /* global reset (no phy reset) */
-	HCTL_RPGSEL		= (1 << 15), /* register page select */
-
-	HCTL_KNOWN_BITS		= HCTL_IRQOFF | HCTL_PWRDWN | HCTL_SOFTRST |
-				  HCTL_RPGSEL,
-
-	/* HOST_IRQ_(STAT|MASK) bits */
-	HIRQ_PORT0		= (1 << 0),
-	HIRQ_PORT1		= (1 << 1),
-	HIRQ_SOFT		= (1 << 14),
-	HIRQ_GLOBAL		= (1 << 15), /* STAT only */
-
-	/* PORT_IRQ_(STAT|MASK) bits */
-	PIRQ_OFFLINE		= (1 << 0),  /* device unplugged */
-	PIRQ_ONLINE		= (1 << 1),  /* device plugged */
-	PIRQ_COMPLETE		= (1 << 2),  /* completion interrupt */
-	PIRQ_FATAL		= (1 << 3),  /* fatal error */
-	PIRQ_ATA		= (1 << 4),  /* ATA interrupt */
-	PIRQ_REPLY		= (1 << 5),  /* reply FIFO not empty */
-	PIRQ_PENDING		= (1 << 7),  /* port IRQ pending (STAT only) */
-
-	PIRQ_ERR		= PIRQ_OFFLINE | PIRQ_ONLINE | PIRQ_FATAL,
-	PIRQ_MASK_DEFAULT	= PIRQ_REPLY | PIRQ_ATA,
-	PIRQ_MASK_FREEZE	= 0xff,
-
-	/* PORT_PRD_CTL bits */
-	PRD_CTL_START		= (1 << 0),
-	PRD_CTL_WR		= (1 << 3),
-	PRD_CTL_DMAEN		= (1 << 7),  /* DMA enable */
-
-	/* PORT_IDMA_CTL bits */
-	IDMA_CTL_RST_ATA	= (1 << 2),  /* hardreset ATA bus */
-	IDMA_CTL_RST_IDMA	= (1 << 5),  /* reset IDMA machinery */
-	IDMA_CTL_GO		= (1 << 7),  /* IDMA mode go */
-	IDMA_CTL_ATA_NIEN	= (1 << 8),  /* ATA IRQ disable */
-
-	/* PORT_IDMA_STAT bits */
-	IDMA_STAT_PERR		= (1 << 0),  /* PCI ERROR MODE */
-	IDMA_STAT_CPBERR	= (1 << 1),  /* ADMA CPB error */
-	IDMA_STAT_LGCY		= (1 << 3),  /* ADMA legacy */
-	IDMA_STAT_UIRQ		= (1 << 4),  /* ADMA unsolicited irq */
-	IDMA_STAT_STPD		= (1 << 5),  /* ADMA stopped */
-	IDMA_STAT_PSD		= (1 << 6),  /* ADMA pause */
-	IDMA_STAT_DONE		= (1 << 7),  /* ADMA done */
-
-	IDMA_STAT_ERR		= IDMA_STAT_PERR | IDMA_STAT_CPBERR,
-
-	/* CPB Control Flags*/
-	CPB_CTL_VALID		= (1 << 0),  /* CPB valid */
-	CPB_CTL_QUEUED		= (1 << 1),  /* queued command */
-	CPB_CTL_DATA		= (1 << 2),  /* data, rsvd in datasheet */
-	CPB_CTL_IEN		= (1 << 3),  /* PCI interrupt enable */
-	CPB_CTL_DEVDIR		= (1 << 4),  /* device direction control */
-
-	/* CPB Response Flags */
-	CPB_RESP_DONE		= (1 << 0),  /* ATA command complete */
-	CPB_RESP_REL		= (1 << 1),  /* ATA release */
-	CPB_RESP_IGNORED	= (1 << 2),  /* CPB ignored */
-	CPB_RESP_ATA_ERR	= (1 << 3),  /* ATA command error */
-	CPB_RESP_SPURIOUS	= (1 << 4),  /* ATA spurious interrupt error */
-	CPB_RESP_UNDERFLOW	= (1 << 5),  /* APRD deficiency length error */
-	CPB_RESP_OVERFLOW	= (1 << 6),  /* APRD exccess length error */
-	CPB_RESP_CPB_ERR	= (1 << 7),  /* CPB error flag */
-
-	/* PRD Control Flags */
-	PRD_DRAIN		= (1 << 1),  /* ignore data excess */
-	PRD_CDB			= (1 << 2),  /* atapi packet command pointer */
-	PRD_DIRECT_INTR		= (1 << 3),  /* direct interrupt */
-	PRD_DMA			= (1 << 4),  /* data transfer method */
-	PRD_WRITE		= (1 << 5),  /* data dir, rsvd in datasheet */
-	PRD_IOM			= (1 << 6),  /* io/memory transfer */
-	PRD_END			= (1 << 7),  /* APRD chain end */
-};
-
-/* Comman Parameter Block */
-struct inic_cpb {
-	u8		resp_flags;	/* Response Flags */
-	u8		error;		/* ATA Error */
-	u8		status;		/* ATA Status */
-	u8		ctl_flags;	/* Control Flags */
-	__le32		len;		/* Total Transfer Length */
-	__le32		prd;		/* First PRD pointer */
-	u8		rsvd[4];
-	/* 16 bytes */
-	u8		feature;	/* ATA Feature */
-	u8		hob_feature;	/* ATA Ex. Feature */
-	u8		device;		/* ATA Device/Head */
-	u8		mirctl;		/* Mirror Control */
-	u8		nsect;		/* ATA Sector Count */
-	u8		hob_nsect;	/* ATA Ex. Sector Count */
-	u8		lbal;		/* ATA Sector Number */
-	u8		hob_lbal;	/* ATA Ex. Sector Number */
-	u8		lbam;		/* ATA Cylinder Low */
-	u8		hob_lbam;	/* ATA Ex. Cylinder Low */
-	u8		lbah;		/* ATA Cylinder High */
-	u8		hob_lbah;	/* ATA Ex. Cylinder High */
-	u8		command;	/* ATA Command */
-	u8		ctl;		/* ATA Control */
-	u8		slave_error;	/* Slave ATA Error */
-	u8		slave_status;	/* Slave ATA Status */
-	/* 32 bytes */
-} __packed;
-
-/* Physical Region Descriptor */
-struct inic_prd {
-	__le32		mad;		/* Physical Memory Address */
-	__le16		len;		/* Transfer Length */
-	u8		rsvd;
-	u8		flags;		/* Control Flags */
-} __packed;
-
-struct inic_pkt {
-	struct inic_cpb	cpb;
-	struct inic_prd	prd[LIBATA_MAX_PRD + 1];	/* + 1 for cdb */
-	u8		cdb[ATAPI_CDB_LEN];
-} __packed;
-
-struct inic_host_priv {
-	void __iomem	*mmio_base;
-	u16		cached_hctl;
-};
-
-struct inic_port_priv {
-	struct inic_pkt	*pkt;
-	dma_addr_t	pkt_dma;
-	u32		*cpb_tbl;
-	dma_addr_t	cpb_tbl_dma;
-};
-
-static const struct scsi_host_template inic_sht = {
-	ATA_BASE_SHT(DRV_NAME),
-	.sg_tablesize		= LIBATA_MAX_PRD, /* maybe it can be larger? */
-
-	/*
-	 * This controller is braindamaged.  dma_boundary is 0xffff like others
-	 * but it will lock up the whole machine HARD if 65536 byte PRD entry
-	 * is fed.  Reduce maximum segment size.
-	 */
-	.dma_boundary		= INIC_DMA_BOUNDARY,
-	.max_segment_size	= 65536 - 512,
-};
-
-static const int scr_map[] = {
-	[SCR_STATUS]	= 0,
-	[SCR_ERROR]	= 1,
-	[SCR_CONTROL]	= 2,
-};
-
-static void __iomem *inic_port_base(struct ata_port *ap)
-{
-	struct inic_host_priv *hpriv = ap->host->private_data;
-
-	return hpriv->mmio_base + ap->port_no * PORT_SIZE;
-}
-
-static void inic_reset_port(void __iomem *port_base)
-{
-	void __iomem *idma_ctl = port_base + PORT_IDMA_CTL;
-
-	/* stop IDMA engine */
-	readw(idma_ctl); /* flush */
-	msleep(1);
-
-	/* mask IRQ and assert reset */
-	writew(IDMA_CTL_RST_IDMA, idma_ctl);
-	readw(idma_ctl); /* flush */
-	msleep(1);
-
-	/* release reset */
-	writew(0, idma_ctl);
-
-	/* clear irq */
-	writeb(0xff, port_base + PORT_IRQ_STAT);
-}
-
-static int inic_scr_read(struct ata_link *link, unsigned sc_reg, u32 *val)
-{
-	void __iomem *scr_addr = inic_port_base(link->ap) + PORT_SCR;
-
-	if (unlikely(sc_reg >= ARRAY_SIZE(scr_map)))
-		return -EINVAL;
-
-	*val = readl(scr_addr + scr_map[sc_reg] * 4);
-
-	/* this controller has stuck DIAG.N, ignore it */
-	if (sc_reg == SCR_ERROR)
-		*val &= ~SERR_PHYRDY_CHG;
-	return 0;
-}
-
-static int inic_scr_write(struct ata_link *link, unsigned sc_reg, u32 val)
-{
-	void __iomem *scr_addr = inic_port_base(link->ap) + PORT_SCR;
-
-	if (unlikely(sc_reg >= ARRAY_SIZE(scr_map)))
-		return -EINVAL;
-
-	writel(val, scr_addr + scr_map[sc_reg] * 4);
-	return 0;
-}
-
-static void inic_stop_idma(struct ata_port *ap)
-{
-	void __iomem *port_base = inic_port_base(ap);
-
-	readb(port_base + PORT_RPQ_FIFO);
-	readb(port_base + PORT_RPQ_CNT);
-	writew(0, port_base + PORT_IDMA_CTL);
-}
-
-static void inic_host_err_intr(struct ata_port *ap, u8 irq_stat, u16 idma_stat)
-{
-	struct ata_eh_info *ehi = &ap->link.eh_info;
-	struct inic_port_priv *pp = ap->private_data;
-	struct inic_cpb *cpb = &pp->pkt->cpb;
-	bool freeze = false;
-
-	ata_ehi_clear_desc(ehi);
-	ata_ehi_push_desc(ehi, "irq_stat=0x%x idma_stat=0x%x",
-			  irq_stat, idma_stat);
-
-	inic_stop_idma(ap);
-
-	if (irq_stat & (PIRQ_OFFLINE | PIRQ_ONLINE)) {
-		ata_ehi_push_desc(ehi, "hotplug");
-		ata_ehi_hotplugged(ehi);
-		freeze = true;
-	}
-
-	if (idma_stat & IDMA_STAT_PERR) {
-		ata_ehi_push_desc(ehi, "PCI error");
-		freeze = true;
-	}
-
-	if (idma_stat & IDMA_STAT_CPBERR) {
-		ata_ehi_push_desc(ehi, "CPB error");
-
-		if (cpb->resp_flags & CPB_RESP_IGNORED) {
-			__ata_ehi_push_desc(ehi, " ignored");
-			ehi->err_mask |= AC_ERR_INVALID;
-			freeze = true;
-		}
-
-		if (cpb->resp_flags & CPB_RESP_ATA_ERR)
-			ehi->err_mask |= AC_ERR_DEV;
-
-		if (cpb->resp_flags & CPB_RESP_SPURIOUS) {
-			__ata_ehi_push_desc(ehi, " spurious-intr");
-			ehi->err_mask |= AC_ERR_HSM;
-			freeze = true;
-		}
-
-		if (cpb->resp_flags &
-		    (CPB_RESP_UNDERFLOW | CPB_RESP_OVERFLOW)) {
-			__ata_ehi_push_desc(ehi, " data-over/underflow");
-			ehi->err_mask |= AC_ERR_HSM;
-			freeze = true;
-		}
-	}
-
-	if (freeze)
-		ata_port_freeze(ap);
-	else
-		ata_port_abort(ap);
-}
-
-static void inic_host_intr(struct ata_port *ap)
-{
-	void __iomem *port_base = inic_port_base(ap);
-	struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
-	u8 irq_stat;
-	u16 idma_stat;
-
-	/* read and clear IRQ status */
-	irq_stat = readb(port_base + PORT_IRQ_STAT);
-	writeb(irq_stat, port_base + PORT_IRQ_STAT);
-	idma_stat = readw(port_base + PORT_IDMA_STAT);
-
-	if (unlikely((irq_stat & PIRQ_ERR) || (idma_stat & IDMA_STAT_ERR)))
-		inic_host_err_intr(ap, irq_stat, idma_stat);
-
-	if (unlikely(!qc))
-		goto spurious;
-
-	if (likely(idma_stat & IDMA_STAT_DONE)) {
-		inic_stop_idma(ap);
-
-		/* Depending on circumstances, device error
-		 * isn't reported by IDMA, check it explicitly.
-		 */
-		if (unlikely(readb(port_base + PORT_TF_COMMAND) &
-			     (ATA_DF | ATA_ERR)))
-			qc->err_mask |= AC_ERR_DEV;
-
-		ata_qc_complete(qc);
-		return;
-	}
-
- spurious:
-	ata_port_warn(ap, "unhandled interrupt: cmd=0x%x irq_stat=0x%x idma_stat=0x%x\n",
-		      qc ? qc->tf.command : 0xff, irq_stat, idma_stat);
-}
-
-static irqreturn_t inic_interrupt(int irq, void *dev_instance)
-{
-	struct ata_host *host = dev_instance;
-	struct inic_host_priv *hpriv = host->private_data;
-	u16 host_irq_stat;
-	int i, handled = 0;
-
-	host_irq_stat = readw(hpriv->mmio_base + HOST_IRQ_STAT);
-
-	if (unlikely(!(host_irq_stat & HIRQ_GLOBAL)))
-		goto out;
-
-	spin_lock(&host->lock);
-
-	for (i = 0; i < NR_PORTS; i++)
-		if (host_irq_stat & (HIRQ_PORT0 << i)) {
-			inic_host_intr(host->ports[i]);
-			handled++;
-		}
-
-	spin_unlock(&host->lock);
-
- out:
-	return IRQ_RETVAL(handled);
-}
-
-static int inic_check_atapi_dma(struct ata_queued_cmd *qc)
-{
-	/* For some reason ATAPI_PROT_DMA doesn't work for some
-	 * commands including writes and other misc ops.  Use PIO
-	 * protocol instead, which BTW is driven by the DMA engine
-	 * anyway, so it shouldn't make much difference for native
-	 * SATA devices.
-	 */
-	if (atapi_cmd_type(qc->cdb[0]) == READ)
-		return 0;
-	return 1;
-}
-
-static void inic_fill_sg(struct inic_prd *prd, struct ata_queued_cmd *qc)
-{
-	struct scatterlist *sg;
-	unsigned int si;
-	u8 flags = 0;
-
-	if (qc->tf.flags & ATA_TFLAG_WRITE)
-		flags |= PRD_WRITE;
-
-	if (ata_is_dma(qc->tf.protocol))
-		flags |= PRD_DMA;
-
-	for_each_sg(qc->sg, sg, qc->n_elem, si) {
-		prd->mad = cpu_to_le32(sg_dma_address(sg));
-		prd->len = cpu_to_le16(sg_dma_len(sg));
-		prd->flags = flags;
-		prd++;
-	}
-
-	WARN_ON(!si);
-	prd[-1].flags |= PRD_END;
-}
-
-static enum ata_completion_errors inic_qc_prep(struct ata_queued_cmd *qc)
-{
-	struct inic_port_priv *pp = qc->ap->private_data;
-	struct inic_pkt *pkt = pp->pkt;
-	struct inic_cpb *cpb = &pkt->cpb;
-	struct inic_prd *prd = pkt->prd;
-	bool is_atapi = ata_is_atapi(qc->tf.protocol);
-	bool is_data = ata_is_data(qc->tf.protocol);
-	unsigned int cdb_len = 0;
-
-	if (is_atapi)
-		cdb_len = qc->dev->cdb_len;
-
-	/* prepare packet, based on initio driver */
-	memset(pkt, 0, sizeof(struct inic_pkt));
-
-	cpb->ctl_flags = CPB_CTL_VALID | CPB_CTL_IEN;
-	if (is_atapi || is_data)
-		cpb->ctl_flags |= CPB_CTL_DATA;
-
-	cpb->len = cpu_to_le32(qc->nbytes + cdb_len);
-	cpb->prd = cpu_to_le32(pp->pkt_dma + offsetof(struct inic_pkt, prd));
-
-	cpb->device = qc->tf.device;
-	cpb->feature = qc->tf.feature;
-	cpb->nsect = qc->tf.nsect;
-	cpb->lbal = qc->tf.lbal;
-	cpb->lbam = qc->tf.lbam;
-	cpb->lbah = qc->tf.lbah;
-
-	if (qc->tf.flags & ATA_TFLAG_LBA48) {
-		cpb->hob_feature = qc->tf.hob_feature;
-		cpb->hob_nsect = qc->tf.hob_nsect;
-		cpb->hob_lbal = qc->tf.hob_lbal;
-		cpb->hob_lbam = qc->tf.hob_lbam;
-		cpb->hob_lbah = qc->tf.hob_lbah;
-	}
-
-	cpb->command = qc->tf.command;
-	/* don't load ctl - dunno why.  it's like that in the initio driver */
-
-	/* setup PRD for CDB */
-	if (is_atapi) {
-		memcpy(pkt->cdb, qc->cdb, ATAPI_CDB_LEN);
-		prd->mad = cpu_to_le32(pp->pkt_dma +
-				       offsetof(struct inic_pkt, cdb));
-		prd->len = cpu_to_le16(cdb_len);
-		prd->flags = PRD_CDB | PRD_WRITE;
-		if (!is_data)
-			prd->flags |= PRD_END;
-		prd++;
-	}
-
-	/* setup sg table */
-	if (is_data)
-		inic_fill_sg(prd, qc);
-
-	pp->cpb_tbl[0] = pp->pkt_dma;
-
-	return AC_ERR_OK;
-}
-
-static unsigned int inic_qc_issue(struct ata_queued_cmd *qc)
-{
-	struct ata_port *ap = qc->ap;
-	void __iomem *port_base = inic_port_base(ap);
-
-	/* fire up the ADMA engine */
-	writew(HCTL_FTHD0 | HCTL_LEDEN, port_base + HOST_CTL);
-	writew(IDMA_CTL_GO, port_base + PORT_IDMA_CTL);
-	writeb(0, port_base + PORT_CPB_PTQFIFO);
-
-	return 0;
-}
-
-static void inic_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
-{
-	void __iomem *port_base = inic_port_base(ap);
-
-	tf->error	= readb(port_base + PORT_TF_FEATURE);
-	tf->nsect	= readb(port_base + PORT_TF_NSECT);
-	tf->lbal	= readb(port_base + PORT_TF_LBAL);
-	tf->lbam	= readb(port_base + PORT_TF_LBAM);
-	tf->lbah	= readb(port_base + PORT_TF_LBAH);
-	tf->device	= readb(port_base + PORT_TF_DEVICE);
-	tf->status	= readb(port_base + PORT_TF_COMMAND);
-}
-
-static void inic_qc_fill_rtf(struct ata_queued_cmd *qc)
-{
-	struct ata_taskfile *rtf = &qc->result_tf;
-	struct ata_taskfile tf;
-
-	/* FIXME: Except for status and error, result TF access
-	 * doesn't work.  I tried reading from BAR0/2, CPB and BAR5.
-	 * None works regardless of which command interface is used.
-	 * For now return true iff status indicates device error.
-	 * This means that we're reporting bogus sector for RW
-	 * failures.  Eeekk....
-	 */
-	inic_tf_read(qc->ap, &tf);
-
-	if (tf.status & ATA_ERR) {
-		rtf->status = tf.status;
-		rtf->error = tf.error;
-	}
-}
-
-static void inic_freeze(struct ata_port *ap)
-{
-	void __iomem *port_base = inic_port_base(ap);
-
-	writeb(PIRQ_MASK_FREEZE, port_base + PORT_IRQ_MASK);
-	writeb(0xff, port_base + PORT_IRQ_STAT);
-}
-
-static void inic_thaw(struct ata_port *ap)
-{
-	void __iomem *port_base = inic_port_base(ap);
-
-	writeb(0xff, port_base + PORT_IRQ_STAT);
-	writeb(PIRQ_MASK_DEFAULT, port_base + PORT_IRQ_MASK);
-}
-
-static int inic_check_ready(struct ata_link *link)
-{
-	void __iomem *port_base = inic_port_base(link->ap);
-
-	return ata_check_ready(readb(port_base + PORT_TF_COMMAND));
-}
-
-/*
- * SRST and SControl hardreset don't give valid signature on this
- * controller.  Only controller specific hardreset mechanism works.
- */
-static int inic_hardreset(struct ata_link *link, unsigned int *class,
-			  unsigned long deadline)
-{
-	struct ata_port *ap = link->ap;
-	void __iomem *port_base = inic_port_base(ap);
-	void __iomem *idma_ctl = port_base + PORT_IDMA_CTL;
-	const unsigned int *timing = sata_ehc_deb_timing(&link->eh_context);
-	int rc;
-
-	/* hammer it into sane state */
-	inic_reset_port(port_base);
-
-	writew(IDMA_CTL_RST_ATA, idma_ctl);
-	readw(idma_ctl);	/* flush */
-	ata_msleep(ap, 1);
-	writew(0, idma_ctl);
-
-	rc = sata_link_resume(link, timing, deadline);
-	if (rc) {
-		ata_link_warn(link,
-			      "failed to resume link after reset (errno=%d)\n",
-			      rc);
-		return rc;
-	}
-
-	*class = ATA_DEV_NONE;
-	if (ata_link_online(link)) {
-		struct ata_taskfile tf;
-
-		/* wait for link to become ready */
-		rc = ata_wait_after_reset(link, deadline, inic_check_ready);
-		/* link occupied, -ENODEV too is an error */
-		if (rc) {
-			ata_link_warn(link,
-				      "device not ready after hardreset (errno=%d)\n",
-				      rc);
-			return rc;
-		}
-
-		inic_tf_read(ap, &tf);
-		*class = ata_port_classify(ap, &tf);
-	}
-
-	return 0;
-}
-
-static void inic_error_handler(struct ata_port *ap)
-	__must_hold(&ap->host->eh_mutex)
-{
-	void __iomem *port_base = inic_port_base(ap);
-
-	inic_reset_port(port_base);
-	ata_std_error_handler(ap);
-}
-
-static void inic_post_internal_cmd(struct ata_queued_cmd *qc)
-{
-	/* make DMA engine forget about the failed command */
-	if (qc->flags & ATA_QCFLAG_EH)
-		inic_reset_port(inic_port_base(qc->ap));
-}
-
-static void init_port(struct ata_port *ap)
-{
-	void __iomem *port_base = inic_port_base(ap);
-	struct inic_port_priv *pp = ap->private_data;
-
-	/* clear packet and CPB table */
-	memset(pp->pkt, 0, sizeof(struct inic_pkt));
-	memset(pp->cpb_tbl, 0, IDMA_CPB_TBL_SIZE);
-
-	/* setup CPB lookup table addresses */
-	writel(pp->cpb_tbl_dma, port_base + PORT_CPB_CPBLAR);
-}
-
-static int inic_port_resume(struct ata_port *ap)
-{
-	init_port(ap);
-	return 0;
-}
-
-static int inic_port_start(struct ata_port *ap)
-{
-	struct device *dev = ap->host->dev;
-	struct inic_port_priv *pp;
-
-	/* alloc and initialize private data */
-	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
-	if (!pp)
-		return -ENOMEM;
-	ap->private_data = pp;
-
-	/* Alloc resources */
-	pp->pkt = dmam_alloc_coherent(dev, sizeof(struct inic_pkt),
-				      &pp->pkt_dma, GFP_KERNEL);
-	if (!pp->pkt)
-		return -ENOMEM;
-
-	pp->cpb_tbl = dmam_alloc_coherent(dev, IDMA_CPB_TBL_SIZE,
-					  &pp->cpb_tbl_dma, GFP_KERNEL);
-	if (!pp->cpb_tbl)
-		return -ENOMEM;
-
-	init_port(ap);
-
-	return 0;
-}
-
-static struct ata_port_operations inic_port_ops = {
-	.inherits		= &sata_port_ops,
-
-	.check_atapi_dma	= inic_check_atapi_dma,
-	.qc_prep		= inic_qc_prep,
-	.qc_issue		= inic_qc_issue,
-	.qc_fill_rtf		= inic_qc_fill_rtf,
-
-	.freeze			= inic_freeze,
-	.thaw			= inic_thaw,
-	.reset.hardreset	= inic_hardreset,
-	.error_handler		= inic_error_handler,
-	.post_internal_cmd	= inic_post_internal_cmd,
-
-	.scr_read		= inic_scr_read,
-	.scr_write		= inic_scr_write,
-
-	.port_resume		= inic_port_resume,
-	.port_start		= inic_port_start,
-};
-
-static const struct ata_port_info inic_port_info = {
-	.flags			= ATA_FLAG_SATA | ATA_FLAG_PIO_DMA,
-	.pio_mask		= ATA_PIO4,
-	.mwdma_mask		= ATA_MWDMA2,
-	.udma_mask		= ATA_UDMA6,
-	.port_ops		= &inic_port_ops
-};
-
-static int init_controller(void __iomem *mmio_base, u16 hctl)
-{
-	int i;
-	u16 val;
-
-	hctl &= ~HCTL_KNOWN_BITS;
-
-	/* Soft reset whole controller.  Spec says reset duration is 3
-	 * PCI clocks, be generous and give it 10ms.
-	 */
-	writew(hctl | HCTL_SOFTRST, mmio_base + HOST_CTL);
-	readw(mmio_base + HOST_CTL); /* flush */
-
-	for (i = 0; i < 10; i++) {
-		msleep(1);
-		val = readw(mmio_base + HOST_CTL);
-		if (!(val & HCTL_SOFTRST))
-			break;
-	}
-
-	if (val & HCTL_SOFTRST)
-		return -EIO;
-
-	/* mask all interrupts and reset ports */
-	for (i = 0; i < NR_PORTS; i++) {
-		void __iomem *port_base = mmio_base + i * PORT_SIZE;
-
-		writeb(0xff, port_base + PORT_IRQ_MASK);
-		inic_reset_port(port_base);
-	}
-
-	/* port IRQ is masked now, unmask global IRQ */
-	writew(hctl & ~HCTL_IRQOFF, mmio_base + HOST_CTL);
-	val = readw(mmio_base + HOST_IRQ_MASK);
-	val &= ~(HIRQ_PORT0 | HIRQ_PORT1);
-	writew(val, mmio_base + HOST_IRQ_MASK);
-
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int inic_pci_device_resume(struct pci_dev *pdev)
-{
-	struct ata_host *host = pci_get_drvdata(pdev);
-	struct inic_host_priv *hpriv = host->private_data;
-	int rc;
-
-	rc = ata_pci_device_do_resume(pdev);
-	if (rc)
-		return rc;
-
-	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
-		rc = init_controller(hpriv->mmio_base, hpriv->cached_hctl);
-		if (rc)
-			return rc;
-	}
-
-	ata_host_resume(host);
-
-	return 0;
-}
-#endif
-
-static int inic_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	const struct ata_port_info *ppi[] = { &inic_port_info, NULL };
-	struct ata_host *host;
-	struct inic_host_priv *hpriv;
-	void __iomem * const *iomap;
-	int mmio_bar;
-	int i, rc;
-
-	ata_print_version_once(&pdev->dev, DRV_VERSION);
-
-	dev_alert(&pdev->dev, "inic162x support is broken with common data corruption issues and will be disabled by default, contact linux-ide@vger.kernel.org if in production use\n");
-
-	/* alloc host */
-	host = ata_host_alloc_pinfo(&pdev->dev, ppi, NR_PORTS);
-	hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
-	if (!host || !hpriv)
-		return -ENOMEM;
-
-	host->private_data = hpriv;
-
-	/* Acquire resources and fill host.  Note that PCI and cardbus
-	 * use different BARs.
-	 */
-	rc = pcim_enable_device(pdev);
-	if (rc)
-		return rc;
-
-	if (pci_resource_flags(pdev, MMIO_BAR_PCI) & IORESOURCE_MEM)
-		mmio_bar = MMIO_BAR_PCI;
-	else
-		mmio_bar = MMIO_BAR_CARDBUS;
-
-	rc = pcim_iomap_regions(pdev, 1 << mmio_bar, DRV_NAME);
-	if (rc)
-		return rc;
-	host->iomap = iomap = pcim_iomap_table(pdev);
-	hpriv->mmio_base = iomap[mmio_bar];
-	hpriv->cached_hctl = readw(hpriv->mmio_base + HOST_CTL);
-
-	for (i = 0; i < NR_PORTS; i++) {
-		struct ata_port *ap = host->ports[i];
-
-		ata_port_pbar_desc(ap, mmio_bar, -1, "mmio");
-		ata_port_pbar_desc(ap, mmio_bar, i * PORT_SIZE, "port");
-	}
-
-	/* Set dma_mask.  This devices doesn't support 64bit addressing. */
-	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-	if (rc) {
-		dev_err(&pdev->dev, "32-bit DMA enable failed\n");
-		return rc;
-	}
-
-	rc = init_controller(hpriv->mmio_base, hpriv->cached_hctl);
-	if (rc) {
-		dev_err(&pdev->dev, "failed to initialize controller\n");
-		return rc;
-	}
-
-	pci_set_master(pdev);
-	return ata_host_activate(host, pdev->irq, inic_interrupt, IRQF_SHARED,
-				 &inic_sht);
-}
-
-static const struct pci_device_id inic_pci_tbl[] = {
-	{ PCI_VDEVICE(INIT, 0x1622), },
-	{ },
-};
-
-static struct pci_driver inic_pci_driver = {
-	.name 		= DRV_NAME,
-	.id_table	= inic_pci_tbl,
-#ifdef CONFIG_PM_SLEEP
-	.suspend	= ata_pci_device_suspend,
-	.resume		= inic_pci_device_resume,
-#endif
-	.probe 		= inic_init_one,
-	.remove		= ata_pci_remove_one,
-};
-
-module_pci_driver(inic_pci_driver);
-
-MODULE_AUTHOR("Tejun Heo");
-MODULE_DESCRIPTION("low-level driver for Initio 162x SATA");
-MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, inic_pci_tbl);
-MODULE_VERSION(DRV_VERSION);
diff --git a/drivers/ata/sata_qstor.c b/drivers/ata/sata_qstor.c
index 4e7f5b2ff3f698..f38310b84f51cf 100644
--- a/drivers/ata/sata_qstor.c
+++ b/drivers/ata/sata_qstor.c
@@ -584,8 +584,12 @@ static int qs_ata_init_one(struct pci_dev *pdev,
 	qs_host_init(host, board_idx);
 
 	pci_set_master(pdev);
-	return ata_host_activate(host, pdev->irq, qs_intr, IRQF_SHARED,
-				 &qs_ata_sht);
+	rc = ata_host_activate(host, pdev->irq, qs_intr, IRQF_SHARED,
+			       &qs_ata_sht);
+	if (rc)
+		qs_host_stop(host);
+
+	return rc;
 }
 
 module_pci_driver(qs_ata_pci_driver);
diff --git a/drivers/auxdisplay/arm-charlcd.c b/drivers/auxdisplay/arm-charlcd.c
index 70efda4f767ead..ea684f3446c37d 100644
--- a/drivers/auxdisplay/arm-charlcd.c
+++ b/drivers/auxdisplay/arm-charlcd.c
@@ -306,10 +306,7 @@ static int charlcd_resume(struct device *dev)
 	return 0;
 }
 
-static const struct dev_pm_ops charlcd_pm_ops = {
-	.suspend = charlcd_suspend,
-	.resume = charlcd_resume,
-};
+static DEFINE_SIMPLE_DEV_PM_OPS(charlcd_pm_ops, charlcd_suspend, charlcd_resume);
 
 static const struct of_device_id charlcd_match[] = {
 	{ .compatible = "arm,versatile-lcd", },
@@ -319,7 +316,7 @@ static const struct of_device_id charlcd_match[] = {
 static struct platform_driver charlcd_driver = {
 	.driver = {
 		.name = DRIVERNAME,
-		.pm = &charlcd_pm_ops,
+		.pm = pm_sleep_ptr(&charlcd_pm_ops),
 		.suppress_bind_attrs = true,
 		.of_match_table = charlcd_match,
 	},
diff --git a/drivers/auxdisplay/panel.c b/drivers/auxdisplay/panel.c
index a7dfc53157bc2f..2a567a075e2b29 100644
--- a/drivers/auxdisplay/panel.c
+++ b/drivers/auxdisplay/panel.c
@@ -714,10 +714,7 @@ static void lcd_backlight(struct charlcd *charlcd, enum charlcd_onoff on)
 
 	/* The backlight is activated by setting the AUTOFEED line to +5V  */
 	spin_lock_irq(&pprt_lock);
-	if (on)
-		set_bit(LCD_BIT_BL, bits);
-	else
-		clear_bit(LCD_BIT_BL, bits);
+	assign_bit(LCD_BIT_BL, bits, on);
 	panel_set_bits();
 	spin_unlock_irq(&pprt_lock);
 }
diff --git a/drivers/base/base.h b/drivers/base/base.h
index a5b7abc10ff02c..21c7d244f8aa7c 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -27,6 +27,7 @@
  * @drivers_autoprobe: gate whether new devices are automatically attached to
  *		       registered drivers, or new drivers automatically attach
  *		       to existing devices.
+ * @no_drivers: gate whether drivers can be registered.
  * @bus: pointer back to the struct bus_type that this structure is associated
  *	 with.
  * @dev_root: Default device to use as the parent.
@@ -51,6 +52,7 @@ struct subsys_private {
 	struct klist klist_drivers;
 	struct blocking_notifier_head bus_notifier;
 	unsigned int drivers_autoprobe:1;
+	unsigned int no_drivers:1;
 	const struct bus_type *bus;
 	struct device *dev_root;
 
diff --git a/drivers/base/bus.c b/drivers/base/bus.c
index d17bd91490ee4e..18f12c6f910a04 100644
--- a/drivers/base/bus.c
+++ b/drivers/base/bus.c
@@ -19,6 +19,7 @@
 #include <linux/string.h>
 #include <linux/mutex.h>
 #include <linux/sysfs.h>
+#include <linux/panic.h>
 #include "base.h"
 #include "power/power.h"
 
@@ -241,6 +242,7 @@ static ssize_t unbind_store(struct device_driver *drv, const char *buf,
 
 	dev = bus_find_device_by_name(bus, NULL, buf);
 	if (dev && dev->driver == drv) {
+		add_taint_module(drv->owner, TAINT_FORCED_BIND, LOCKDEP_STILL_OK);
 		device_driver_detach(dev);
 		err = count;
 	}
@@ -264,6 +266,7 @@ static ssize_t bind_store(struct device_driver *drv, const char *buf,
 
 	dev = bus_find_device_by_name(bus, NULL, buf);
 	if (dev && driver_match_device(drv, dev)) {
+		add_taint_module(drv->owner, TAINT_FORCED_BIND, LOCKDEP_STILL_OK);
 		err = device_driver_attach(drv, dev);
 		if (!err) {
 			/* success */
@@ -738,6 +741,11 @@ int bus_add_driver(struct device_driver *drv)
 	if (!sp)
 		return -EINVAL;
 
+	if (sp->no_drivers) {
+		error = -ENXIO;
+		goto out_put_bus;
+	}
+
 	/*
 	 * Reference in sp is now incremented and will be dropped when
 	 * the driver is removed from the bus
@@ -930,15 +938,7 @@ static ssize_t bus_uevent_store(const struct bus_type *bus,
 static struct bus_attribute bus_attr_uevent = __ATTR(uevent, 0200, NULL,
 						     bus_uevent_store);
 
-/**
- * bus_register - register a driver-core subsystem
- * @bus: bus to register
- *
- * Once we have that, we register the bus with the kobject
- * infrastructure, then register the children subsystems it has:
- * the devices and drivers that belong to the subsystem.
- */
-int bus_register(const struct bus_type *bus)
+static int bus_register_internal(const struct bus_type *bus, bool use_drivers)
 {
 	int retval;
 	struct subsys_private *priv;
@@ -960,7 +960,8 @@ int bus_register(const struct bus_type *bus)
 
 	bus_kobj->kset = bus_kset;
 	bus_kobj->ktype = &bus_ktype;
-	priv->drivers_autoprobe = 1;
+	priv->drivers_autoprobe = use_drivers;
+	priv->no_drivers = !use_drivers;
 
 	retval = kset_register(&priv->subsys);
 	if (retval)
@@ -976,6 +977,10 @@ int bus_register(const struct bus_type *bus)
 		goto bus_devices_fail;
 	}
 
+	/*
+	 * Create the drivers kset even if use_drivers is unset to prevent some
+	 * older versions of systemd from failing.
+	 */
 	priv->drivers_kset = kset_create_and_add("drivers", NULL, bus_kobj);
 	if (!priv->drivers_kset) {
 		retval = -ENOMEM;
@@ -989,9 +994,11 @@ int bus_register(const struct bus_type *bus)
 	klist_init(&priv->klist_devices, klist_devices_get, klist_devices_put);
 	klist_init(&priv->klist_drivers, NULL, NULL);
 
-	retval = add_probe_files(bus);
-	if (retval)
-		goto bus_probe_files_fail;
+	if (use_drivers) {
+		retval = add_probe_files(bus);
+		if (retval)
+			goto bus_probe_files_fail;
+	}
 
 	retval = sysfs_create_groups(bus_kobj, bus->bus_groups);
 	if (retval)
@@ -1016,8 +1023,40 @@ int bus_register(const struct bus_type *bus)
 	kfree(priv);
 	return retval;
 }
+
+/**
+ * bus_register - register a driver-core subsystem
+ * @bus: bus to register
+ *
+ * Once we have that, we register the bus with the kobject
+ * infrastructure, then register the children subsystems it has:
+ * the devices and drivers that belong to the subsystem.
+ */
+int bus_register(const struct bus_type *bus)
+{
+	return bus_register_internal(bus, true);
+}
 EXPORT_SYMBOL_GPL(bus_register);
 
+/**
+ * companion_bus_register - register a companion bus type
+ * @bus: companion bus to register
+ *
+ * A companion bus is a bus without drivers.  Devices that belong to it can be
+ * bound to other devices as their "companions" and represent interfaces that
+ * can be used by the drivers of those other devices.  They may also be used for
+ * the enumeration of those other devices.
+ *
+ * The ACPI bus is a specific example of a companion bus.
+ *
+ * Registering a companion bus is like registering a regular bus except that it
+ * skips the creation of sysfs interfaces related to drivers for @bus.
+ */
+int companion_bus_register(const struct bus_type *bus)
+{
+	return bus_register_internal(bus, false);
+}
+
 /**
  * bus_unregister - remove a bus from the system
  * @bus: bus.
@@ -1415,6 +1454,11 @@ struct device_driver *driver_find(const char *name, const struct bus_type *bus)
 	if (!sp)
 		return NULL;
 
+	if (sp->no_drivers) {
+		subsys_put(sp);
+		return NULL;
+	}
+
 	k = kset_find_obj(sp->drivers_kset, name);
 	subsys_put(sp);
 	if (!k)
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index 9f9c72727a0590..eb8a907f6d50bd 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -38,6 +38,23 @@ struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu)
 	return ci_cacheinfo(cpu);
 }
 
+/**
+ * get_cpu_cacheinfo_id - Return the cache ID for a CPU and cache level
+ * @cpu: CPU number
+ * @level: Cache level
+ *
+ * The caller must hold the cpuhp lock.
+ *
+ * Return: Cache ID on success, or -1 if no matching cache exists.
+ */
+int get_cpu_cacheinfo_id(int cpu, int level)
+{
+	struct cacheinfo *ci = get_cpu_cacheinfo_level(cpu, level);
+
+	return ci ? ci->id : -1;
+}
+EXPORT_SYMBOL_GPL(get_cpu_cacheinfo_id);
+
 static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf,
 					   struct cacheinfo *sib_leaf)
 {
@@ -1040,9 +1057,10 @@ static int cacheinfo_cpu_online(unsigned int cpu)
 	rc = cache_add_dev(cpu);
 	if (rc)
 		goto err;
-	if (cpu_map_shared_cache(true, cpu, &cpu_map))
+	if (cpu_map_shared_cache(true, cpu, &cpu_map)) {
 		update_per_cpu_data_slice_size(true, cpu, cpu_map);
-	sched_update_llc_bytes(cpu);
+		sched_update_llc_bytes(cpu_map);
+	}
 	return 0;
 err:
 	free_cache_attributes(cpu);
@@ -1059,10 +1077,10 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu)
 		cpu_cache_sysfs_exit(cpu);
 
 	free_cache_attributes(cpu);
-	if (nr_shared > 1)
+	if (nr_shared > 1) {
 		update_per_cpu_data_slice_size(false, cpu, cpu_map);
-
-	sched_update_llc_bytes(cpu);
+		sched_update_llc_bytes(cpu_map);
+	}
 
 	return 0;
 }
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 4c0c373998a19f..5daa88fa724abb 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -2517,13 +2517,13 @@ static const struct sysfs_ops dev_sysfs_ops = {
 	.store	= dev_attr_store,
 };
 
-#define to_ext_attr(x) container_of(x, struct dev_ext_attribute, attr)
+#define to_ext_attr(x) container_of_const(x, struct dev_ext_attribute, attr)
 
 ssize_t device_store_ulong(struct device *dev,
-			   struct device_attribute *attr,
+			   const struct device_attribute *attr,
 			   const char *buf, size_t size)
 {
-	struct dev_ext_attribute *ea = to_ext_attr(attr);
+	const struct dev_ext_attribute *ea = to_ext_attr(attr);
 	int ret;
 	unsigned long new;
 
@@ -2537,19 +2537,19 @@ ssize_t device_store_ulong(struct device *dev,
 EXPORT_SYMBOL_GPL(device_store_ulong);
 
 ssize_t device_show_ulong(struct device *dev,
-			  struct device_attribute *attr,
+			  const struct device_attribute *attr,
 			  char *buf)
 {
-	struct dev_ext_attribute *ea = to_ext_attr(attr);
+	const struct dev_ext_attribute *ea = to_ext_attr(attr);
 	return sysfs_emit(buf, "%lx\n", *(unsigned long *)(ea->var));
 }
 EXPORT_SYMBOL_GPL(device_show_ulong);
 
 ssize_t device_store_int(struct device *dev,
-			 struct device_attribute *attr,
+			 const struct device_attribute *attr,
 			 const char *buf, size_t size)
 {
-	struct dev_ext_attribute *ea = to_ext_attr(attr);
+	const struct dev_ext_attribute *ea = to_ext_attr(attr);
 	int ret;
 	long new;
 
@@ -2566,19 +2566,19 @@ ssize_t device_store_int(struct device *dev,
 EXPORT_SYMBOL_GPL(device_store_int);
 
 ssize_t device_show_int(struct device *dev,
-			struct device_attribute *attr,
+			const struct device_attribute *attr,
 			char *buf)
 {
-	struct dev_ext_attribute *ea = to_ext_attr(attr);
+	const struct dev_ext_attribute *ea = to_ext_attr(attr);
 
 	return sysfs_emit(buf, "%d\n", *(int *)(ea->var));
 }
 EXPORT_SYMBOL_GPL(device_show_int);
 
-ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
+ssize_t device_store_bool(struct device *dev, const struct device_attribute *attr,
 			  const char *buf, size_t size)
 {
-	struct dev_ext_attribute *ea = to_ext_attr(attr);
+	const struct dev_ext_attribute *ea = to_ext_attr(attr);
 
 	if (kstrtobool(buf, ea->var) < 0)
 		return -EINVAL;
@@ -2587,19 +2587,19 @@ ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
 }
 EXPORT_SYMBOL_GPL(device_store_bool);
 
-ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
+ssize_t device_show_bool(struct device *dev, const struct device_attribute *attr,
 			 char *buf)
 {
-	struct dev_ext_attribute *ea = to_ext_attr(attr);
+	const struct dev_ext_attribute *ea = to_ext_attr(attr);
 
 	return sysfs_emit(buf, "%d\n", *(bool *)(ea->var));
 }
 EXPORT_SYMBOL_GPL(device_show_bool);
 
 ssize_t device_show_string(struct device *dev,
-			   struct device_attribute *attr, char *buf)
+			   const struct device_attribute *attr, char *buf)
 {
-	struct dev_ext_attribute *ea = to_ext_attr(attr);
+	const struct dev_ext_attribute *ea = to_ext_attr(attr);
 
 	return sysfs_emit(buf, "%s\n", (char *)ea->var);
 }
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 69e52fed424159..747915ff974fb1 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -391,6 +391,15 @@ static int cpu_uevent(const struct device *dev, struct kobj_uevent_env *env)
 }
 #endif
 
+#ifdef CONFIG_PREFERRED_CPU
+static ssize_t preferred_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpu_preferred_mask));
+}
+static DEVICE_ATTR_RO(preferred);
+#endif
+
 const struct bus_type cpu_subsys = {
 	.name = "cpu",
 	.dev_name = "cpu",
@@ -531,6 +540,9 @@ static struct attribute *cpu_root_attrs[] = {
 #endif
 #ifdef CONFIG_GENERIC_CPU_AUTOPROBE
 	&dev_attr_modalias.attr,
+#endif
+#ifdef CONFIG_PREFERRED_CPU
+	&dev_attr_preferred.attr,
 #endif
 	NULL
 };
diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c
index aef0fcc6aba1e0..11c70888f38be5 100644
--- a/drivers/base/devtmpfs.c
+++ b/drivers/base/devtmpfs.c
@@ -72,39 +72,90 @@ static struct file_system_type internal_fs_type = {
 	.kill_sb = kill_anon_super,
 };
 
-/* Simply take a ref on the existing mount */
+struct devtmpfs_context {
+	struct fs_context *fc;
+};
+
+static void devtmpfs_free(struct fs_context *fc)
+{
+	struct devtmpfs_context *ctx = fc->fs_private;
+
+	if (ctx) {
+		put_fs_context(ctx->fc);
+		kfree(ctx);
+	}
+}
+
+static int devtmpfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
+{
+	struct devtmpfs_context *ctx = fc->fs_private;
+
+	return ctx->fc->ops->parse_param(ctx->fc, param);
+}
+
+static int devtmpfs_parse_monolithic(struct fs_context *fc, void *data)
+{
+	struct devtmpfs_context *ctx = fc->fs_private;
+
+	if (ctx->fc->ops->parse_monolithic)
+		return ctx->fc->ops->parse_monolithic(ctx->fc, data);
+	return generic_parse_monolithic(ctx->fc, data);
+}
+
 static int devtmpfs_get_tree(struct fs_context *fc)
 {
+	struct devtmpfs_context *ctx = fc->fs_private;
 	struct super_block *sb = mnt->mnt_sb;
+	int err;
 
 	atomic_inc(&sb->s_active);
 	down_write(&sb->s_umount);
+
+	if (ctx->fc->ops->reconfigure) {
+		err = ctx->fc->ops->reconfigure(ctx->fc);
+		if (err) {
+			deactivate_locked_super(sb);
+			return err;
+		}
+	}
+
 	fc->root = dget(sb->s_root);
 	return 0;
 }
 
-/* Ops are filled in during init depending on underlying shmem or ramfs type */
-static struct fs_context_operations devtmpfs_context_ops = {};
+static const struct fs_context_operations devtmpfs_context_ops = {
+	.free		  = devtmpfs_free,
+	.parse_param	  = devtmpfs_parse_param,
+	.parse_monolithic = devtmpfs_parse_monolithic,
+	.get_tree	  = devtmpfs_get_tree,
+};
 
-/* Call the underlying initialization and set to our ops */
 static int devtmpfs_init_fs_context(struct fs_context *fc)
 {
-	int ret;
-#ifdef CONFIG_TMPFS
-	ret = shmem_init_fs_context(fc);
-#else
-	ret = ramfs_init_fs_context(fc);
-#endif
-	if (ret < 0)
-		return ret;
+	struct devtmpfs_context *ctx;
+	int err;
+
+	ctx = kzalloc_obj(struct devtmpfs_context);
+	if (!ctx)
+		return -ENOMEM;
+
+	/* Each mount will reconfigure the shared superblock w/ new options */
+	ctx->fc = fs_context_for_reconfigure(mnt->mnt_root,
+					     mnt->mnt_sb->s_flags, MS_RMT_MASK);
+	if (IS_ERR(ctx->fc)) {
+		err = PTR_ERR(ctx->fc);
+		kfree(ctx);
+		return err;
+	}
 
+	fc->fs_private = ctx;
 	fc->ops = &devtmpfs_context_ops;
 
 	return 0;
 }
 
 static struct file_system_type dev_fs_type = {
-	.name = "devtmpfs",
+	.name		 = "devtmpfs",
 	.init_fs_context = devtmpfs_init_fs_context,
 };
 
@@ -442,31 +493,6 @@ static int __ref devtmpfsd(void *p)
 	return 0;
 }
 
-/*
- * Get the underlying (shmem/ramfs) context ops to build ours
- */
-static int devtmpfs_configure_context(void)
-{
-	struct fs_context *fc;
-
-	fc = fs_context_for_reconfigure(mnt->mnt_root, mnt->mnt_sb->s_flags,
-					MS_RMT_MASK);
-	if (IS_ERR(fc))
-		return PTR_ERR(fc);
-
-	/* Set up devtmpfs_context_ops based on underlying type */
-	devtmpfs_context_ops.free	      = fc->ops->free;
-	devtmpfs_context_ops.dup	      = fc->ops->dup;
-	devtmpfs_context_ops.parse_param      = fc->ops->parse_param;
-	devtmpfs_context_ops.parse_monolithic = fc->ops->parse_monolithic;
-	devtmpfs_context_ops.get_tree	      = &devtmpfs_get_tree;
-	devtmpfs_context_ops.reconfigure      = fc->ops->reconfigure;
-
-	put_fs_context(fc);
-
-	return 0;
-}
-
 /*
  * Create devtmpfs instance, driver-core devices will add their device
  * nodes here.
@@ -482,12 +508,6 @@ int __init devtmpfs_init(void)
 		return PTR_ERR(mnt);
 	}
 
-	err = devtmpfs_configure_context();
-	if (err) {
-		pr_err("unable to configure devtmpfs type %d\n", err);
-		return err;
-	}
-
 	err = register_filesystem(&dev_fs_type);
 	if (err) {
 		pr_err("unable to register devtmpfs type %d\n", err);
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index 5eead3346f1e32..1ac12c424200e1 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -255,6 +255,7 @@ static int memory_block_online(struct memory_block *mem)
 		nr_vmemmap_pages = mem->altmap->free;
 
 	mem_hotplug_begin();
+	clear_zone_contiguous(zone);
 	if (nr_vmemmap_pages) {
 		ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone);
 		if (ret)
@@ -279,6 +280,7 @@ static int memory_block_online(struct memory_block *mem)
 
 	mem->zone = zone;
 out:
+	set_zone_contiguous(zone);
 	mem_hotplug_done();
 	return ret;
 }
@@ -304,6 +306,7 @@ static int memory_block_offline(struct memory_block *mem)
 		nr_vmemmap_pages = mem->altmap->free;
 
 	mem_hotplug_begin();
+	clear_zone_contiguous(mem->zone);
 	if (nr_vmemmap_pages)
 		adjust_present_page_count(pfn_to_page(start_pfn), mem->group,
 					  -nr_vmemmap_pages);
@@ -321,8 +324,10 @@ static int memory_block_offline(struct memory_block *mem)
 	if (nr_vmemmap_pages)
 		mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages);
 
-	mem->zone = NULL;
 out:
+	set_zone_contiguous(mem->zone);
+	if (!ret)
+		mem->zone = NULL;
 	mem_hotplug_done();
 	return ret;
 }
@@ -972,7 +977,7 @@ void __init memory_dev_init(void)
 	 * block so that it can be covered.
 	 */
 	block_id = ULONG_MAX;
-	for_each_present_section_nr(0, nr) {
+	for_each_early_section_nr(0, nr) {
 		if (block_id != ULONG_MAX && memory_block_id(nr) == block_id)
 			continue;
 
diff --git a/drivers/base/node.c b/drivers/base/node.c
index 3da91929ad4e37..dcdc8626627a87 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -167,20 +167,19 @@ static struct node_access_nodes *node_init_node_access(struct node *node,
 	dev->parent = &node->dev;
 	dev->release = node_access_release;
 	dev->groups = node_access_node_groups;
-	if (dev_set_name(dev, "access%u", access))
-		goto free;
+	if (dev_set_name(dev, "access%u", access)) {
+		kfree(access_node);
+		return NULL;
+	}
 
-	if (device_register(dev))
-		goto free_name;
+	if (device_register(dev)) {
+		put_device(dev);
+		return NULL;
+	}
 
 	pm_runtime_no_callbacks(dev);
 	list_add_tail(&access_node->list_node, &node->access_list);
 	return access_node;
-free_name:
-	kfree_const(dev->kobj.name);
-free:
-	kfree(access_node);
-	return NULL;
 }
 
 #ifdef CONFIG_HMEM_REPORTING
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 66f9ec73d47e90..3593d45f4a7ae8 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -632,16 +632,17 @@ struct platform_device *platform_device_alloc(const char *name, int id)
 	struct platform_object *pa;
 
 	pa = kzalloc(sizeof(*pa) + strlen(name) + 1, GFP_KERNEL);
-	if (pa) {
-		strcpy(pa->name, name);
-		pa->pdev.name = pa->name;
-		pa->pdev.id = id;
-		device_initialize(&pa->pdev.dev);
-		pa->pdev.dev.release = platform_device_release;
-		setup_pdev_dma_masks(&pa->pdev);
-	}
+	if (!pa)
+		return NULL;
+
+	strcpy(pa->name, name);
+	pa->pdev.name = pa->name;
+	pa->pdev.id = id;
+	device_initialize(&pa->pdev.dev);
+	pa->pdev.dev.release = platform_device_release;
+	setup_pdev_dma_masks(&pa->pdev);
 
-	return pa ? &pa->pdev : NULL;
+	return &pa->pdev;
 }
 EXPORT_SYMBOL_GPL(platform_device_alloc);
 
diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c
index 59bb37e8244c57..3b8ba27802a800 100644
--- a/drivers/base/power/clock_ops.c
+++ b/drivers/base/power/clock_ops.c
@@ -250,7 +250,7 @@ EXPORT_SYMBOL_GPL(pm_clk_add);
  *
  * Add the clock to the list of clocks used for the power management of @dev.
  * The power-management code will take control of the clock reference, so
- * callers should not call clk_put() on @clk after this function sucessfully
+ * callers should not call clk_put() on @clk after this function successfully
  * returned.
  */
 int pm_clk_add_clk(struct device *dev, struct clk *clk)
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index e130da42814106..4e0d6357a37cb7 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -802,6 +802,7 @@ static void device_resume_noirq(struct device *dev, pm_message_t state, bool asy
 	const char *info = NULL;
 	bool skip_resume;
 	int error = 0;
+	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
 	TRACE_DEVICE(dev);
 	TRACE_RESUME(0);
@@ -827,6 +828,7 @@ static void device_resume_noirq(struct device *dev, pm_message_t state, bool asy
 	if (!dpm_wait_for_superior(dev, async))
 		goto Out;
 
+	dpm_watchdog_set(&wd, dev);
 	skip_resume = dev_pm_skip_resume(dev);
 	/*
 	 * If the driver callback is skipped below or by the middle layer
@@ -871,6 +873,7 @@ static void device_resume_noirq(struct device *dev, pm_message_t state, bool asy
 	error = dpm_run_callback(callback, dev, state, info);
 
 Skip:
+	dpm_watchdog_clear(&wd);
 	dev->power.is_noirq_suspended = false;
 
 Out:
@@ -971,6 +974,7 @@ static void device_resume_early(struct device *dev, pm_message_t state, bool asy
 	pm_callback_t callback = NULL;
 	const char *info = NULL;
 	int error = 0;
+	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
 	TRACE_DEVICE(dev);
 	TRACE_RESUME(0);
@@ -987,6 +991,7 @@ static void device_resume_early(struct device *dev, pm_message_t state, bool asy
 	if (!dpm_wait_for_superior(dev, async))
 		goto Out;
 
+	dpm_watchdog_set(&wd, dev);
 	if (dev->pm_domain) {
 		info = "early power domain ";
 		callback = pm_late_early_op(&dev->pm_domain->ops, state);
@@ -1004,7 +1009,7 @@ static void device_resume_early(struct device *dev, pm_message_t state, bool asy
 		goto Run;
 
 	if (dev_pm_skip_resume(dev))
-		goto Skip;
+		goto End;
 
 	if (dev->driver && dev->driver->pm) {
 		info = "early driver ";
@@ -1014,6 +1019,9 @@ static void device_resume_early(struct device *dev, pm_message_t state, bool asy
 Run:
 	error = dpm_run_callback(callback, dev, state, info);
 
+End:
+	dpm_watchdog_clear(&wd);
+
 Skip:
 	dev->power.is_late_suspended = false;
 	pm_runtime_enable(dev);
@@ -1281,10 +1289,12 @@ static void device_complete(struct device *dev, pm_message_t state)
 {
 	void (*callback)(struct device *) = NULL;
 	const char *info = NULL;
+	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
 	if (dev->power.syscore)
 		goto out;
 
+	dpm_watchdog_set(&wd, dev);
 	device_lock(dev);
 
 	if (dev->pm_domain) {
@@ -1312,6 +1322,7 @@ static void device_complete(struct device *dev, pm_message_t state)
 	}
 
 	device_unlock(dev);
+	dpm_watchdog_clear(&wd);
 
 out:
 	/* If enabling runtime PM for the device is blocked, unblock it. */
@@ -1508,6 +1519,7 @@ static void device_suspend_noirq(struct device *dev, pm_message_t state, bool as
 	pm_callback_t callback = NULL;
 	const char *info = NULL;
 	int error = 0;
+	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
 	TRACE_DEVICE(dev);
 	TRACE_SUSPEND(0);
@@ -1520,6 +1532,7 @@ static void device_suspend_noirq(struct device *dev, pm_message_t state, bool as
 	if (dev->power.syscore || dev->power.direct_complete)
 		goto Complete;
 
+	dpm_watchdog_set(&wd, dev);
 	if (dev->pm_domain) {
 		info = "noirq power domain ";
 		callback = pm_noirq_op(&dev->pm_domain->ops, state);
@@ -1550,7 +1563,7 @@ static void device_suspend_noirq(struct device *dev, pm_message_t state, bool as
 		WRITE_ONCE(async_error, error);
 		dpm_save_failed_dev(dev_name(dev));
 		pm_dev_err(dev, state, async ? " async noirq" : " noirq", error);
-		goto Complete;
+		goto End;
 	}
 
 Skip:
@@ -1569,6 +1582,9 @@ static void device_suspend_noirq(struct device *dev, pm_message_t state, bool as
 	if (dev->power.must_resume)
 		dpm_superior_set_must_resume(dev);
 
+End:
+	dpm_watchdog_clear(&wd);
+
 Complete:
 	complete_all(&dev->power.completion);
 	TRACE_SUSPEND(error);
@@ -1703,6 +1719,7 @@ static void device_suspend_late(struct device *dev, pm_message_t state, bool asy
 	pm_callback_t callback = NULL;
 	const char *info = NULL;
 	int error = 0;
+	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
 	TRACE_DEVICE(dev);
 	TRACE_SUSPEND(0);
@@ -1720,6 +1737,8 @@ static void device_suspend_late(struct device *dev, pm_message_t state, bool asy
 	if (dev->power.direct_complete)
 		goto Complete;
 
+	dpm_watchdog_set(&wd, dev);
+
 	/*
 	 * After this point, any runtime PM operations targeting the device
 	 * will fail until the corresponding pm_runtime_enable() call in
@@ -1761,13 +1780,16 @@ static void device_suspend_late(struct device *dev, pm_message_t state, bool asy
 		dpm_save_failed_dev(dev_name(dev));
 		pm_dev_err(dev, state, async ? " async late" : " late", error);
 		pm_runtime_enable(dev);
-		goto Complete;
+		goto End;
 	}
 	dpm_propagate_wakeup_to_parent(dev);
 
 Skip:
 	dev->power.is_late_suspended = true;
 
+End:
+	dpm_watchdog_clear(&wd);
+
 Complete:
 	TRACE_SUSPEND(error);
 	complete_all(&dev->power.completion);
@@ -2201,6 +2223,7 @@ static int device_prepare(struct device *dev, pm_message_t state)
 	int (*callback)(struct device *) = NULL;
 	bool smart_suspend;
 	int ret = 0;
+	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
 	/*
 	 * If a device's parent goes into runtime suspend at the wrong time,
@@ -2220,6 +2243,7 @@ static int device_prepare(struct device *dev, pm_message_t state)
 	if (dev->power.syscore)
 		return 0;
 
+	dpm_watchdog_set(&wd, dev);
 	device_lock(dev);
 
 	dev->power.wakeup_path = false;
@@ -2245,6 +2269,7 @@ static int device_prepare(struct device *dev, pm_message_t state)
 
 unlock:
 	device_unlock(dev);
+	dpm_watchdog_clear(&wd);
 
 	if (ret < 0) {
 		suspend_report_result(dev, callback, ret);
diff --git a/drivers/base/power/runtime-test.c b/drivers/base/power/runtime-test.c
index 1535ad2b02649f..24865ce844fc6a 100644
--- a/drivers/base/power/runtime-test.c
+++ b/drivers/base/power/runtime-test.c
@@ -5,6 +5,7 @@
 
 #include <linux/cleanup.h>
 #include <linux/pm_runtime.h>
+#include <linux/workqueue.h>
 #include <kunit/device.h>
 #include <kunit/test.h>
 
@@ -229,6 +230,61 @@ static void pm_runtime_probe_active_test(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev));
 }
 
+static void pm_runtime_supplier_suspend_test(struct kunit *test)
+{
+	struct device *dev = kunit_device_register(test, DEVICE_NAME);
+	struct device *supplier = kunit_device_register(test, DEVICE_NAME "_supplier");
+	struct device *norpm_supplier = kunit_device_register(test, DEVICE_NAME "_norpm_supplier");
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, supplier);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, norpm_supplier);
+
+	KUNIT_ASSERT_NOT_NULL(test, device_link_add(dev, supplier, DL_FLAG_PM_RUNTIME));
+	KUNIT_ASSERT_NOT_NULL(test, device_link_add(dev, norpm_supplier, 0));
+
+	pm_runtime_enable(dev);
+	pm_runtime_enable(supplier);
+	pm_runtime_enable(norpm_supplier);
+
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(supplier));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(norpm_supplier));
+
+	/*
+	 * Resume the device and RPM-linked supplier; non-RPM-linked supplier
+	 * stays suspended.
+	 */
+	KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(dev));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_active(dev));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_active(supplier));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(norpm_supplier));
+
+	/* Also resume the non-RPM-linked supplier. */
+	KUNIT_EXPECT_EQ(test, 0, pm_runtime_get_sync(norpm_supplier));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_active(norpm_supplier));
+
+	pm_runtime_put_noidle(norpm_supplier);
+	KUNIT_EXPECT_TRUE(test, pm_runtime_active(norpm_supplier));
+
+	/*
+	 * Suspend device. This should only suspend the device and its
+	 * RPM-linked supplier, not the non-RPM-linked supplier.
+	 */
+	KUNIT_EXPECT_EQ(test, 0, pm_runtime_put_sync(dev));
+	/* Note: supplier suspend is async, so we have to flush the queue. */
+	flush_workqueue(pm_wq);
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(supplier));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_active(norpm_supplier));
+
+	/* Now suspend non-RPM-linked supplier. */
+	KUNIT_EXPECT_EQ(test, 0, pm_runtime_suspend(norpm_supplier));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(dev));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(supplier));
+	KUNIT_EXPECT_TRUE(test, pm_runtime_suspended(norpm_supplier));
+}
+
 static struct kunit_case pm_runtime_test_cases[] = {
 	KUNIT_CASE(pm_runtime_depth_test),
 	KUNIT_CASE(pm_runtime_already_suspended_test),
@@ -236,6 +292,7 @@ static struct kunit_case pm_runtime_test_cases[] = {
 	KUNIT_CASE(pm_runtime_disabled_test),
 	KUNIT_CASE(pm_runtime_error_test),
 	KUNIT_CASE(pm_runtime_probe_active_test),
+	KUNIT_CASE(pm_runtime_supplier_suspend_test),
 	{}
 };
 
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index fab38bc981139c..655c0b8af09545 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -162,7 +162,7 @@ static void pm_runtime_cancel_pending(struct device *dev)
 	dev->power.request = RPM_REQ_NONE;
 }
 
-/*
+/**
  * pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
  * @dev: Device to handle.
  *
@@ -200,7 +200,7 @@ static int dev_memalloc_noio(struct device *dev, void *data)
 	return dev->power.memalloc_noio;
 }
 
-/*
+/**
  * pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
  * @dev: Device to handle.
  * @enable: True for setting the flag and False for clearing the flag.
@@ -361,7 +361,8 @@ static void rpm_suspend_suppliers(struct device *dev)
 
 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
 				device_links_read_lock_held())
-		pm_request_idle(link->supplier);
+		if (device_link_test(link, DL_FLAG_PM_RUNTIME))
+			pm_request_idle(link->supplier);
 
 	device_links_read_unlock(idx);
 }
@@ -1043,6 +1044,11 @@ static enum hrtimer_restart  pm_suspend_timer_fn(struct hrtimer *timer)
  * pm_schedule_suspend - Set up a timer to submit a suspend request in future.
  * @dev: Device to suspend.
  * @delay: Time to wait before submitting a suspend request, in milliseconds.
+ *
+ * Return:
+ * * %1: Success; @dev is already %RPM_SUSPENDED.
+ * * %0: Success.
+ * * Error code on failure.
  */
 int pm_schedule_suspend(struct device *dev, unsigned int delay)
 {
@@ -1096,16 +1102,15 @@ static int rpm_drop_usage_count(struct device *dev)
 }
 
 /**
- * __pm_runtime_idle - Entry point for runtime idle operations.
+ * __pm_runtime_idle - Core entry point for runtime idle operations.
  * @dev: Device to send idle notification for.
  * @rpmflags: Flag bits.
  *
- * If the RPM_GET_PUT flag is set, decrement the device's usage count and
- * return immediately if it is larger than zero (if it becomes negative, log a
- * warning, increment it, and return an error).  Then carry out an idle
- * notification, either synchronous or asynchronous.
+ * Carry out an idle check for @dev, either synchronous or asynchronous.
+ * If %RPM_GET_PUT is set in @rpmflags, decrement the device's usage count
+ * first, proceeding with idle notification only if the counter drops to zero.
  *
- * This routine may be called in atomic context if the RPM_ASYNC flag is set,
+ * This routine may be called in atomic context if the %RPM_ASYNC flag is set,
  * or if pm_runtime_irq_safe() has been called.
  */
 int __pm_runtime_idle(struct device *dev, int rpmflags)
@@ -1134,16 +1139,15 @@ int __pm_runtime_idle(struct device *dev, int rpmflags)
 EXPORT_SYMBOL_GPL(__pm_runtime_idle);
 
 /**
- * __pm_runtime_suspend - Entry point for runtime put/suspend operations.
+ * __pm_runtime_suspend - Core entry point for runtime put/suspend operations.
  * @dev: Device to suspend.
  * @rpmflags: Flag bits.
  *
- * If the RPM_GET_PUT flag is set, decrement the device's usage count and
- * return immediately if it is larger than zero (if it becomes negative, log a
- * warning, increment it, and return an error).  Then carry out a suspend,
- * either synchronous or asynchronous.
+ * Carry out a suspend operation for @dev, either synchronous or asynchronous.
+ * If %RPM_GET_PUT is set in @rpmflags, decrement the device's usage count
+ * first, proceeding with suspend only if the counter drops to zero.
  *
- * This routine may be called in atomic context if the RPM_ASYNC flag is set,
+ * This routine may be called in atomic context if the %RPM_ASYNC flag is set,
  * or if pm_runtime_irq_safe() has been called.
  */
 int __pm_runtime_suspend(struct device *dev, int rpmflags)
@@ -1172,14 +1176,15 @@ int __pm_runtime_suspend(struct device *dev, int rpmflags)
 EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
 
 /**
- * __pm_runtime_resume - Entry point for runtime resume operations.
+ * __pm_runtime_resume - Core entry point for runtime resume operations.
  * @dev: Device to resume.
  * @rpmflags: Flag bits.
  *
- * If the RPM_GET_PUT flag is set, increment the device's usage count.  Then
- * carry out a resume, either synchronous or asynchronous.
+ * Carry out a runtime resume operation for @dev, either synchronous or
+ * asynchronous. If %RPM_GET_PUT is set in @rpmflags, increment the device's
+ * usage count first, then bring the device to %RPM_ACTIVE state.
  *
- * This routine may be called in atomic context if the RPM_ASYNC flag is set,
+ * This routine may be called in atomic context if the %RPM_ASYNC flag is set,
  * or if pm_runtime_irq_safe() has been called.
  */
 int __pm_runtime_resume(struct device *dev, int rpmflags)
@@ -1254,9 +1259,12 @@ static int pm_runtime_get_conditional(struct device *dev, bool ign_usage_count)
  * @dev: Target device.
  *
  * Increment the runtime PM usage counter of @dev if its runtime PM status is
- * %RPM_ACTIVE, in which case it returns 1. If the device is in a different
- * state, 0 is returned. -EINVAL is returned if runtime PM is disabled for the
- * device, in which case also the usage_count will remain unmodified.
+ * already %RPM_ACTIVE
+ *
+ * Return:
+ * * %-EINVAL: Runtime PM is disabled for @dev. The usage counter is not incremented.
+ * * %1: Success; usage counter is incremented.
+ * * %0: @dev was not active.
  */
 int pm_runtime_get_if_active(struct device *dev)
 {
@@ -1268,17 +1276,15 @@ EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
  * pm_runtime_get_if_in_use - Conditionally bump up runtime PM usage counter.
  * @dev: Target device.
  *
- * Increment the runtime PM usage counter of @dev if its runtime PM status is
- * %RPM_ACTIVE and its runtime PM usage counter is greater than 0 or it is not
- * ignoring children and its active child count is nonzero.  1 is returned in
- * this case.
- *
- * If @dev is in a different state or it is not in use (that is, its usage
- * counter is 0, or it is ignoring children, or its active child count is 0),
- * 0 is returned.
+ * Increment the runtime PM usage counter of @dev if it is "in use." A device
+ * is considered in use if its runtime PM status is %RPM_ACTIVE and its runtime
+ * PM usage counter is greater than 0, or if it is not ignoring children and
+ * its active child count is nonzero.
  *
- * -EINVAL is returned if runtime PM is disabled for the device, in which case
- * also the usage counter of @dev is not updated.
+ * Return:
+ * * %-EINVAL: Runtime PM is disabled for @dev. The usage counter is not incremented.
+ * * %1: Success; usage counter is incremented.
+ * * %0: @dev was not in use; usage counter is not incremented.
  */
 int pm_runtime_get_if_in_use(struct device *dev)
 {
@@ -1287,7 +1293,7 @@ int pm_runtime_get_if_in_use(struct device *dev)
 EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use);
 
 /**
- * __pm_runtime_set_status - Set runtime PM status of a device.
+ * __pm_runtime_set_status - Set runtime PM status of a device and clear errors.
  * @dev: Device to handle.
  * @status: New runtime PM status of the device.
  *
@@ -1306,7 +1312,7 @@ EXPORT_SYMBOL_GPL(pm_runtime_get_if_in_use);
  * If @dev has any suppliers (as reflected by device links to them), and @status
  * is RPM_ACTIVE, they will be activated upfront and if the activation of one
  * of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead
- * of the @status value) and the suppliers will be deacticated on exit.  The
+ * of the @status value) and the suppliers will be deactivated on exit.  The
  * error returned by the failing supplier activation will be returned in that
  * case.
  */
@@ -1462,11 +1468,13 @@ static void __pm_runtime_barrier(struct device *dev)
  * pm_runtime_barrier - Flush pending requests and wait for completions.
  * @dev: Device to handle.
  *
- * Prevent the device from being suspended by incrementing its usage counter and
- * if there's a pending resume request for the device, wake the device up.
- * Next, make sure that all pending requests for the device have been flushed
- * from pm_wq and wait for all runtime PM operations involving the device in
- * progress to complete.
+ * If the device has a pending resume request, resume it synchronously. For all
+ * other request types, cancel any queued request, and wait for running
+ * operations to complete.
+ *
+ * Note that this is intentionally asymmetric, as it guarantees any queued
+ * asynchronous resume request will complete, but it may cancel asynchronous
+ * suspend requests.
  */
 void pm_runtime_barrier(struct device *dev)
 {
@@ -1550,8 +1558,16 @@ void __pm_runtime_disable(struct device *dev, bool check_resume)
 EXPORT_SYMBOL_GPL(__pm_runtime_disable);
 
 /**
- * pm_runtime_enable - Enable runtime PM of a device.
+ * pm_runtime_enable - Enable runtime PM for a device.
  * @dev: Device to handle.
+ *
+ * Enable runtime PM transitions for @dev by decrementing its disable counter.
+ * Once the counter reaches zero, the PM core is permitted to execute runtime
+ * PM callbacks for @dev as power conditions change.
+ *
+ * Callers should ensure that the device's runtime PM status accurately reflects
+ * its physical hardware state (via pm_runtime_set_active() or
+ * pm_runtime_set_suspended()) before enabling runtime PM.
  */
 void pm_runtime_enable(struct device *dev)
 {
@@ -1874,6 +1890,9 @@ void pm_runtime_reinit(struct device *dev)
 	if (dev->power.runtime_status == RPM_ACTIVE)
 		pm_runtime_set_suspended(dev);
 
+	if (dev->power.use_autosuspend)
+		pm_runtime_dont_use_autosuspend(dev);
+
 	if (dev->power.irq_safe) {
 		spin_lock_irq(&dev->power.lock);
 		dev->power.irq_safe = 0;
diff --git a/drivers/base/property.c b/drivers/base/property.c
index cceaa3240ca489..51109794d7275d 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -705,7 +705,7 @@ bool fwnode_name_eq(const struct fwnode_handle *fwnode, const char *name)
 EXPORT_SYMBOL_GPL(fwnode_name_eq);
 
 /**
- * fwnode_get_parent - Return parent firwmare node
+ * fwnode_get_parent - Return parent firmware node
  * @fwnode: Firmware whose parent is retrieved
  *
  * The caller is responsible for calling fwnode_handle_put() on the returned
diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h
index a6e4689000af80..ea375cc8ae87d4 100644
--- a/drivers/base/regmap/internal.h
+++ b/drivers/base/regmap/internal.h
@@ -10,6 +10,7 @@
 #ifndef _REGMAP_INTERNAL_H
 #define _REGMAP_INTERNAL_H
 
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/regmap.h>
 #include <linux/fs.h>
@@ -185,6 +186,16 @@ struct regmap {
 	struct hwspinlock *hwlock;
 };
 
+/*
+ * Scoped guard for the regmap lock. The lock/unlock callbacks are selected
+ * at init time (mutex, spinlock, raw spinlock, hwspinlock or none) and never
+ * fail, so an unconditional guard is sufficient. Use with guard(regmap)(map)
+ * or scoped_guard(regmap, map) { ... }.
+ */
+DEFINE_GUARD(regmap, struct regmap *,
+	     _T->lock(_T->lock_arg),
+	     _T->unlock(_T->lock_arg))
+
 struct regcache_ops {
 	const char *name;
 	enum regcache_type type;
@@ -322,6 +333,7 @@ static inline unsigned int regcache_get_index_by_order(const struct regmap *map,
 }
 
 struct regmap_ram_data {
+	unsigned int base_reg;
 	unsigned int *vals;  /* Allocatd by caller */
 	bool *read;
 	bool *written;
diff --git a/drivers/base/regmap/regcache-rbtree.c b/drivers/base/regmap/regcache-rbtree.c
index 520d5f8ba3cd99..e2feed6dbecffd 100644
--- a/drivers/base/regmap/regcache-rbtree.c
+++ b/drivers/base/regmap/regcache-rbtree.c
@@ -141,7 +141,7 @@ static int rbtree_show(struct seq_file *s, void *ignored)
 	int registers = 0;
 	int this_registers, average;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	mem_size = sizeof(*rbtree_ctx);
 
@@ -168,8 +168,6 @@ static int rbtree_show(struct seq_file *s, void *ignored)
 	seq_printf(s, "%d nodes, %d registers, average %d registers, used %zu bytes\n",
 		   nodes, registers, average, mem_size);
 
-	map->unlock(map->lock_arg);
-
 	return 0;
 }
 
diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c
index 0d58d900a2ca8e..4c80db5d6f6f32 100644
--- a/drivers/base/regmap/regcache.c
+++ b/drivers/base/regmap/regcache.c
@@ -281,9 +281,8 @@ void regcache_exit(struct regmap *map)
 	if (map->cache_ops->exit) {
 		dev_dbg(map->dev, "Destroying %s cache\n",
 			map->cache_ops->name);
-		map->lock(map->lock_arg);
-		map->cache_ops->exit(map);
-		map->unlock(map->lock_arg);
+		scoped_guard(regmap, map)
+			map->cache_ops->exit(map);
 	}
 
 	kfree(map->reg_defaults);
@@ -584,20 +583,14 @@ EXPORT_SYMBOL_GPL(regcache_sync_region);
 int regcache_drop_region(struct regmap *map, unsigned int min,
 			 unsigned int max)
 {
-	int ret = 0;
-
 	if (!map->cache_ops || !map->cache_ops->drop)
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	trace_regcache_drop_region(map, min, max);
 
-	ret = map->cache_ops->drop(map, min, max);
-
-	map->unlock(map->lock_arg);
-
-	return ret;
+	return map->cache_ops->drop(map, min, max);
 }
 EXPORT_SYMBOL_GPL(regcache_drop_region);
 
@@ -615,12 +608,11 @@ EXPORT_SYMBOL_GPL(regcache_drop_region);
  */
 void regcache_cache_only(struct regmap *map, bool enable)
 {
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 	WARN_ON(map->cache_type != REGCACHE_NONE &&
 		map->cache_bypass && enable);
 	map->cache_only = enable;
 	trace_regmap_cache_only(map, enable);
-	map->unlock(map->lock_arg);
 }
 EXPORT_SYMBOL_GPL(regcache_cache_only);
 
@@ -639,10 +631,9 @@ EXPORT_SYMBOL_GPL(regcache_cache_only);
  */
 void regcache_mark_dirty(struct regmap *map)
 {
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 	map->cache_dirty = true;
 	map->no_sync_defaults = true;
-	map->unlock(map->lock_arg);
 }
 EXPORT_SYMBOL_GPL(regcache_mark_dirty);
 
@@ -659,11 +650,10 @@ EXPORT_SYMBOL_GPL(regcache_mark_dirty);
  */
 void regcache_cache_bypass(struct regmap *map, bool enable)
 {
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 	WARN_ON(map->cache_only && enable);
 	map->cache_bypass = enable;
 	trace_regmap_cache_bypass(map, enable);
-	map->unlock(map->lock_arg);
 }
 EXPORT_SYMBOL_GPL(regcache_cache_bypass);
 
@@ -680,12 +670,10 @@ bool regcache_reg_cached(struct regmap *map, unsigned int reg)
 	unsigned int val;
 	int ret;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	ret = regcache_read(map, reg, &val);
 
-	map->unlock(map->lock_arg);
-
 	return ret == 0;
 }
 EXPORT_SYMBOL_GPL(regcache_reg_cached);
diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c
index 18f1c60749fed5..ec207548a6bbe7 100644
--- a/drivers/base/regmap/regmap-debugfs.c
+++ b/drivers/base/regmap/regmap-debugfs.c
@@ -471,18 +471,16 @@ static ssize_t regmap_cache_only_write_file(struct file *file,
 	if (err)
 		return count;
 
-	map->lock(map->lock_arg);
-
-	if (new_val && !map->cache_only) {
-		dev_warn(map->dev, "debugfs cache_only=Y forced\n");
-		add_taint(TAINT_USER, LOCKDEP_STILL_OK);
-	} else if (!new_val && map->cache_only) {
-		dev_warn(map->dev, "debugfs cache_only=N forced: syncing cache\n");
-		require_sync = true;
+	scoped_guard(regmap, map) {
+		if (new_val && !map->cache_only) {
+			dev_warn(map->dev, "debugfs cache_only=Y forced\n");
+			add_taint(TAINT_USER, LOCKDEP_STILL_OK);
+		} else if (!new_val && map->cache_only) {
+			dev_warn(map->dev, "debugfs cache_only=N forced: syncing cache\n");
+			require_sync = true;
+		}
+		map->cache_only = new_val;
 	}
-	map->cache_only = new_val;
-
-	map->unlock(map->lock_arg);
 
 	if (require_sync) {
 		err = regcache_sync(map);
@@ -513,7 +511,7 @@ static ssize_t regmap_cache_bypass_write_file(struct file *file,
 	if (err)
 		return count;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	if (new_val && !map->cache_bypass) {
 		dev_warn(map->dev, "debugfs cache_bypass=Y forced\n");
@@ -523,8 +521,6 @@ static ssize_t regmap_cache_bypass_write_file(struct file *file,
 	}
 	map->cache_bypass = new_val;
 
-	map->unlock(map->lock_arg);
-
 	return count;
 }
 
diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c
index 99b55b1053eed4..715eb9e7aa2ad2 100644
--- a/drivers/base/regmap/regmap-irq.c
+++ b/drivers/base/regmap/regmap-irq.c
@@ -963,7 +963,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode,
 	return 0;
 
 err_domain:
-	/* Should really dispose of the domain but... */
+	irq_domain_remove(d->domain);
 err_mutex:
 	mutex_destroy(&d->lock);
 	lockdep_unregister_key(&d->lock_key);
diff --git a/drivers/base/regmap/regmap-kunit.c b/drivers/base/regmap/regmap-kunit.c
index 2999d9c185d52f..c78772b660d6c5 100644
--- a/drivers/base/regmap/regmap-kunit.c
+++ b/drivers/base/regmap/regmap-kunit.c
@@ -11,6 +11,8 @@
 
 #define BLOCK_TEST_SIZE 12
 
+#define HIGH_BIT_REG ((UINT_MAX / 2) - 3)
+
 KUNIT_DEFINE_ACTION_WRAPPER(regmap_exit_action, regmap_exit, struct regmap *);
 
 struct regmap_test_priv {
@@ -23,6 +25,7 @@ struct regmap_test_param {
 	enum regcache_type cache;
 	enum regmap_endian val_endian;
 
+	unsigned int base_reg;
 	unsigned int from_reg;
 	bool fast_io;
 };
@@ -204,7 +207,8 @@ static struct regmap *gen_regmap(struct kunit *test,
 			config->max_register += (BLOCK_TEST_SIZE * config->reg_stride);
 	}
 
-	size = array_size(config->max_register + 1, sizeof(*buf));
+	size = array_size(config->max_register - param->base_reg + 1,
+			  sizeof(*buf));
 	buf = kmalloc(size, GFP_KERNEL);
 	if (!buf)
 		return ERR_PTR(-ENOMEM);
@@ -215,6 +219,7 @@ static struct regmap *gen_regmap(struct kunit *test,
 	if (!(*data))
 		goto out_free;
 	(*data)->vals = buf;
+	(*data)->base_reg = param->base_reg;
 
 	if (config->num_reg_defaults) {
 		defaults = kunit_kcalloc(test,
@@ -227,8 +232,8 @@ static struct regmap *gen_regmap(struct kunit *test,
 		config->reg_defaults = defaults;
 
 		for (i = 0; i < config->num_reg_defaults; i++) {
-			defaults[i].reg = param->from_reg + (i * config->reg_stride);
-			defaults[i].def = buf[param->from_reg + (i * config->reg_stride)];
+			defaults[i].reg = param->from_reg - param->base_reg + (i * config->reg_stride);
+			defaults[i].def = buf[param->from_reg - param->base_reg + (i * config->reg_stride)];
 		}
 	}
 
@@ -1648,6 +1653,92 @@ static void cache_write_zero(struct kunit *test)
 	KUNIT_ASSERT_FALSE(test, regcache_reg_cached(map, 1));
 }
 
+static const struct regmap_test_param high_bit_cache_types_list[] = {
+	{ .cache = REGCACHE_RBTREE, .from_reg = HIGH_BIT_REG,
+				    .base_reg = HIGH_BIT_REG },
+	{ .cache = REGCACHE_MAPLE,  .from_reg = HIGH_BIT_REG,
+				    .base_reg = HIGH_BIT_REG },
+};
+
+KUNIT_ARRAY_PARAM(high_bit_cache_types, high_bit_cache_types_list, param_to_desc);
+
+/*
+ * Dynamically allocated cache types should support registers with the
+ * high bit set, the backend will return an error for any operation on
+ * a register below the expected window.
+ */
+static void cache_high_bit_reg(struct kunit *test)
+{
+	const struct regmap_test_param *param = test->param_value;
+	struct regmap_ram_data *data;
+	struct regmap_config config;
+	struct regmap *map;
+	unsigned int val[BLOCK_TEST_SIZE], rval;
+	int i;
+
+	config = test_regmap_config;
+
+	map = gen_regmap(test, &config, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	get_random_bytes(&val, sizeof(val));
+
+	/* No defaults so no registers cached */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_FALSE(test, regcache_reg_cached(map, param->from_reg + i));
+
+	/* Writes should reach the device */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, 0, regmap_write(map, param->from_reg + i,
+						      val[i]));
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		KUNIT_EXPECT_TRUE(test, data->written[i]);
+		KUNIT_EXPECT_EQ(test, val[i], data->vals[i]);
+	}
+
+	/* Reads should be satisfied from the cache */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		KUNIT_EXPECT_TRUE(test, regcache_reg_cached(map, param->from_reg + i));
+		KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + i,
+						     &rval));
+		KUNIT_EXPECT_EQ(test, val[i], rval);
+		KUNIT_EXPECT_FALSE(test, data->read[i]);
+	}
+
+	/* Trash the data on the device then resync */
+	regcache_mark_dirty(map);
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		data->vals[i] = 0;
+		data->written[i] = false;
+	}
+	KUNIT_EXPECT_EQ(test, 0, regcache_sync(map));
+
+	/* Did we just write the correct data out? */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		KUNIT_EXPECT_TRUE(test, data->written[i]);
+		KUNIT_EXPECT_EQ(test, val[i], data->vals[i]);
+	}
+
+	/* Drop some registers */
+	KUNIT_EXPECT_EQ(test, 0, regcache_drop_region(map, param->from_reg + 3,
+						      param->from_reg + 5));
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		KUNIT_EXPECT_EQ(test, i < 3 || i > 5,
+				regcache_reg_cached(map, param->from_reg + i));
+
+	/* Reread and check only the dropped registers hit the device */
+	for (i = 0; i < BLOCK_TEST_SIZE; i++)
+		data->read[i] = false;
+	for (i = 0; i < BLOCK_TEST_SIZE; i++) {
+		KUNIT_EXPECT_EQ(test, 0, regmap_read(map, param->from_reg + i,
+						     &rval));
+		KUNIT_EXPECT_EQ(test, val[i], rval);
+		KUNIT_EXPECT_EQ(test, i >= 3 && i <= 5, data->read[i]);
+	}
+}
+
 /* Check that caching the window register works with sync */
 static void cache_range_window_reg(struct kunit *test)
 {
@@ -2168,6 +2259,7 @@ static struct kunit_case regmap_test_cases[] = {
 	KUNIT_CASE_PARAM(cache_drop_all_and_sync_has_defaults, sparse_cache_types_gen_params),
 	KUNIT_CASE_PARAM(cache_present, sparse_cache_types_gen_params),
 	KUNIT_CASE_PARAM(cache_write_zero, sparse_cache_types_gen_params),
+	KUNIT_CASE_PARAM(cache_high_bit_reg, high_bit_cache_types_gen_params),
 	KUNIT_CASE_PARAM(cache_range_window_reg, real_cache_types_only_gen_params),
 
 	KUNIT_CASE_PARAM(raw_read_defaults_single, raw_test_types_gen_params),
diff --git a/drivers/base/regmap/regmap-ram.c b/drivers/base/regmap/regmap-ram.c
index c7356b0d8c8327..a666bfb4d667c1 100644
--- a/drivers/base/regmap/regmap-ram.c
+++ b/drivers/base/regmap/regmap-ram.c
@@ -20,8 +20,11 @@ static int regmap_ram_write(void *context, unsigned int reg, unsigned int val)
 {
 	struct regmap_ram_data *data = context;
 
-	data->vals[reg] = val;
-	data->written[reg] = true;
+	if (reg < data->base_reg)
+		return -EINVAL;
+
+	data->vals[reg - data->base_reg] = val;
+	data->written[reg - data->base_reg] = true;
 
 	return 0;
 }
@@ -30,8 +33,11 @@ static int regmap_ram_read(void *context, unsigned int reg, unsigned int *val)
 {
 	struct regmap_ram_data *data = context;
 
-	*val = data->vals[reg];
-	data->read[reg] = true;
+	if (reg < data->base_reg)
+		return -EINVAL;
+
+	*val = data->vals[reg - data->base_reg];
+	data->read[reg - data->base_reg] = true;
 
 	return 0;
 }
@@ -60,17 +66,25 @@ struct regmap *__regmap_init_ram(struct device *dev,
 				 const char *lock_name)
 {
 	struct regmap *map;
+	unsigned int num_regs;
 
 	if (!config->max_register) {
 		pr_crit("No max_register specified for RAM regmap\n");
 		return ERR_PTR(-EINVAL);
 	}
 
-	data->read = kzalloc_objs(bool, config->max_register + 1);
+	if (data->base_reg > config->max_register) {
+		pr_crit("base_reg above max_register for RAM regmap\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	num_regs = config->max_register - data->base_reg + 1;
+
+	data->read = kzalloc_objs(bool, num_regs);
 	if (!data->read)
 		return ERR_PTR(-ENOMEM);
 
-	data->written = kzalloc_objs(bool, config->max_register + 1);
+	data->written = kzalloc_objs(bool, num_regs);
 	if (!data->written) {
 		kfree(data->read);
 		return ERR_PTR(-ENOMEM);
diff --git a/drivers/base/regmap/regmap-raw-ram.c b/drivers/base/regmap/regmap-raw-ram.c
index 60d6e95cdd1b79..c63c464549f978 100644
--- a/drivers/base/regmap/regmap-raw-ram.c
+++ b/drivers/base/regmap/regmap-raw-ram.c
@@ -16,14 +16,17 @@
 
 #include "internal.h"
 
-static unsigned int decode_reg(enum regmap_endian endian, const void *reg)
+static unsigned int decode_reg(struct regmap_ram_data *data, const void *reg)
 {
 	const u16 *r = reg;
+	unsigned int decode;
 
-	if (endian == REGMAP_ENDIAN_BIG)
-		return be16_to_cpu(*r);
+	if (data->reg_endian == REGMAP_ENDIAN_BIG)
+		decode = be16_to_cpu(*r);
 	else
-		return le16_to_cpu(*r);
+		decode = le16_to_cpu(*r);
+
+	return decode - data->base_reg;
 }
 
 static int regmap_raw_ram_gather_write(void *context,
@@ -40,7 +43,7 @@ static int regmap_raw_ram_gather_write(void *context,
 	if (val_len % 2)
 		return -EINVAL;
 
-	r = decode_reg(data->reg_endian, reg);
+	r = decode_reg(data, reg);
 	if (data->noinc_reg && data->noinc_reg(data, r)) {
 		memcpy(&our_buf[r], val + val_len - 2, 2);
 		data->written[r] = true;
@@ -74,7 +77,7 @@ static int regmap_raw_ram_read(void *context,
 	if (val_len % 2)
 		return -EINVAL;
 
-	r = decode_reg(data->reg_endian, reg);
+	r = decode_reg(data, reg);
 	if (data->noinc_reg && data->noinc_reg(data, r)) {
 		for (i = 0; i < val_len; i += 2)
 			memcpy(val + i, &our_buf[r], 2);
@@ -114,6 +117,7 @@ struct regmap *__regmap_init_raw_ram(struct device *dev,
 				     const char *lock_name)
 {
 	struct regmap *map;
+	unsigned int regs;
 
 	if (config->reg_bits != 16)
 		return ERR_PTR(-EINVAL);
@@ -123,11 +127,12 @@ struct regmap *__regmap_init_raw_ram(struct device *dev,
 		return ERR_PTR(-EINVAL);
 	}
 
-	data->read = kzalloc_objs(bool, config->max_register + 1);
+	regs = config->max_register - data->base_reg + 1;
+	data->read = kzalloc_objs(bool, regs);
 	if (!data->read)
 		return ERR_PTR(-ENOMEM);
 
-	data->written = kzalloc_objs(bool, config->max_register + 1);
+	data->written = kzalloc_objs(bool, regs);
 	if (!data->written)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index e6e022b0263753..0b40e1e7817c55 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -115,9 +115,8 @@ bool regmap_cached(struct regmap *map, unsigned int reg)
 	if (map->max_register_is_set && reg > map->max_register)
 		return false;
 
-	map->lock(map->lock_arg);
-	ret = regcache_read(map, reg, &val);
-	map->unlock(map->lock_arg);
+	scoped_guard(regmap, map)
+		ret = regcache_read(map, reg, &val);
 	if (ret)
 		return false;
 
@@ -1977,18 +1976,12 @@ int _regmap_write(struct regmap *map, unsigned int reg,
  */
 int regmap_write(struct regmap *map, unsigned int reg, unsigned int val)
 {
-	int ret;
-
 	if (!IS_ALIGNED(reg, map->reg_stride))
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
-
-	ret = _regmap_write(map, reg, val);
-
-	map->unlock(map->lock_arg);
+	guard(regmap)(map);
 
-	return ret;
+	return _regmap_write(map, reg, val);
 }
 EXPORT_SYMBOL_GPL(regmap_write);
 
@@ -2009,7 +2002,7 @@ int regmap_write_async(struct regmap *map, unsigned int reg, unsigned int val)
 	if (!IS_ALIGNED(reg, map->reg_stride))
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	map->async = true;
 
@@ -2017,8 +2010,6 @@ int regmap_write_async(struct regmap *map, unsigned int reg, unsigned int val)
 
 	map->async = false;
 
-	map->unlock(map->lock_arg);
-
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regmap_write_async);
@@ -2080,20 +2071,14 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg,
 int regmap_raw_write(struct regmap *map, unsigned int reg,
 		     const void *val, size_t val_len)
 {
-	int ret;
-
 	if (!regmap_can_raw_write(map))
 		return -EINVAL;
 	if (val_len % map->format.val_bytes)
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
-	ret = _regmap_raw_write(map, reg, val, val_len, false);
-
-	map->unlock(map->lock_arg);
-
-	return ret;
+	return _regmap_raw_write(map, reg, val, val_len, false);
 }
 EXPORT_SYMBOL_GPL(regmap_raw_write);
 
@@ -2211,21 +2196,17 @@ int regmap_noinc_write(struct regmap *map, unsigned int reg,
 	if (val_len == 0)
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
-	if (!regmap_volatile(map, reg) || !regmap_writeable_noinc(map, reg)) {
-		ret = -EINVAL;
-		goto out_unlock;
-	}
+	if (!regmap_volatile(map, reg) || !regmap_writeable_noinc(map, reg))
+		return -EINVAL;
 
 	/*
 	 * Use the accelerated operation if we can. The val drops the const
 	 * typing in order to facilitate code reuse in regmap_noinc_readwrite().
 	 */
-	if (map->bus->reg_noinc_write) {
-		ret = regmap_noinc_readwrite(map, reg, (void *)val, val_len, true);
-		goto out_unlock;
-	}
+	if (map->bus->reg_noinc_write)
+		return regmap_noinc_readwrite(map, reg, (void *)val, val_len, true);
 
 	while (val_len) {
 		if (map->max_raw_write && map->max_raw_write < val_len)
@@ -2234,14 +2215,12 @@ int regmap_noinc_write(struct regmap *map, unsigned int reg,
 			write_len = val_len;
 		ret = _regmap_raw_write(map, reg, val, write_len, true);
 		if (ret)
-			goto out_unlock;
+			return ret;
 		val = ((u8 *)val) + write_len;
 		val_len -= write_len;
 	}
 
-out_unlock:
-	map->unlock(map->lock_arg);
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(regmap_noinc_write);
 
@@ -2357,7 +2336,8 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val,
 	 * single write operations.
 	 */
 	if (!map->write || !map->format.parse_inplace) {
-		map->lock(map->lock_arg);
+		guard(regmap)(map);
+
 		for (i = 0; i < val_count; i++) {
 			unsigned int ival;
 
@@ -2372,18 +2352,15 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val,
 				ival = *(u32 *)(val + (i * val_bytes));
 				break;
 			default:
-				ret = -EINVAL;
-				goto out;
+				return -EINVAL;
 			}
 
 			ret = _regmap_write(map,
 					    reg + regmap_get_offset(map, i),
 					    ival);
 			if (ret != 0)
-				goto out;
+				return ret;
 		}
-out:
-		map->unlock(map->lock_arg);
 	} else {
 		void *wval;
 
@@ -2653,15 +2630,9 @@ static int _regmap_multi_reg_write(struct regmap *map,
 int regmap_multi_reg_write(struct regmap *map, const struct reg_sequence *regs,
 			   int num_regs)
 {
-	int ret;
-
-	map->lock(map->lock_arg);
-
-	ret = _regmap_multi_reg_write(map, regs, num_regs);
-
-	map->unlock(map->lock_arg);
+	guard(regmap)(map);
 
-	return ret;
+	return _regmap_multi_reg_write(map, regs, num_regs);
 }
 EXPORT_SYMBOL_GPL(regmap_multi_reg_write);
 
@@ -2690,7 +2661,7 @@ int regmap_multi_reg_write_bypassed(struct regmap *map,
 	int ret;
 	bool bypass;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	bypass = map->cache_bypass;
 	map->cache_bypass = true;
@@ -2699,8 +2670,6 @@ int regmap_multi_reg_write_bypassed(struct regmap *map,
 
 	map->cache_bypass = bypass;
 
-	map->unlock(map->lock_arg);
-
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regmap_multi_reg_write_bypassed);
@@ -2722,7 +2691,7 @@ EXPORT_SYMBOL_GPL(regmap_multi_reg_write_bypassed);
  * If supported by the underlying bus the write will be scheduled
  * asynchronously, helping maximise I/O speed on higher speed buses
  * like SPI.  regmap_async_complete() can be called to ensure that all
- * asynchrnous writes have been completed.
+ * asynchronous writes have been completed.
  *
  * A value of zero will be returned on success, a negative errno will
  * be returned in error cases.
@@ -2737,7 +2706,7 @@ int regmap_raw_write_async(struct regmap *map, unsigned int reg,
 	if (!IS_ALIGNED(reg, map->reg_stride))
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	map->async = true;
 
@@ -2745,8 +2714,6 @@ int regmap_raw_write_async(struct regmap *map, unsigned int reg,
 
 	map->async = false;
 
-	map->unlock(map->lock_arg);
-
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regmap_raw_write_async);
@@ -2863,18 +2830,12 @@ static int _regmap_read(struct regmap *map, unsigned int reg,
  */
 int regmap_read(struct regmap *map, unsigned int reg, unsigned int *val)
 {
-	int ret;
-
 	if (!IS_ALIGNED(reg, map->reg_stride))
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
-	ret = _regmap_read(map, reg, val);
-
-	map->unlock(map->lock_arg);
-
-	return ret;
+	return _regmap_read(map, reg, val);
 }
 EXPORT_SYMBOL_GPL(regmap_read);
 
@@ -2897,7 +2858,7 @@ int regmap_read_bypassed(struct regmap *map, unsigned int reg, unsigned int *val
 	if (!IS_ALIGNED(reg, map->reg_stride))
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	bypass = map->cache_bypass;
 	cache_only = map->cache_only;
@@ -2909,8 +2870,6 @@ int regmap_read_bypassed(struct regmap *map, unsigned int reg, unsigned int *val
 	map->cache_bypass = bypass;
 	map->cache_only = cache_only;
 
-	map->unlock(map->lock_arg);
-
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regmap_read_bypassed);
@@ -2941,22 +2900,18 @@ int regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
 	if (val_count == 0)
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	if (regmap_volatile_range(map, reg, val_count) || map->cache_bypass ||
 	    map->cache_type == REGCACHE_NONE) {
 		size_t chunk_count, chunk_bytes;
 		size_t chunk_regs = val_count;
 
-		if (!map->cache_bypass && map->cache_only) {
-			ret = -EBUSY;
-			goto out;
-		}
+		if (!map->cache_bypass && map->cache_only)
+			return -EBUSY;
 
-		if (!map->read) {
-			ret = -ENOTSUPP;
-			goto out;
-		}
+		if (!map->read)
+			return -ENOTSUPP;
 
 		if (map->use_single_read)
 			chunk_regs = 1;
@@ -2970,7 +2925,7 @@ int regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
 		for (i = 0; i < chunk_count; i++) {
 			ret = _regmap_raw_read(map, reg, val, chunk_bytes, false);
 			if (ret != 0)
-				goto out;
+				return ret;
 
 			reg += regmap_get_offset(map, chunk_regs);
 			val += chunk_bytes;
@@ -2981,7 +2936,7 @@ int regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
 		if (val_len) {
 			ret = _regmap_raw_read(map, reg, val, val_len, false);
 			if (ret != 0)
-				goto out;
+				return ret;
 		}
 	} else {
 		/* Otherwise go word by word for the cache; should be low
@@ -2991,16 +2946,13 @@ int regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
 			ret = _regmap_read(map, reg + regmap_get_offset(map, i),
 					   &v);
 			if (ret != 0)
-				goto out;
+				return ret;
 
 			map->format.format_val(val + (i * val_bytes), v, 0);
 		}
 	}
 
- out:
-	map->unlock(map->lock_arg);
-
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(regmap_raw_read);
 
@@ -3041,12 +2993,10 @@ int regmap_noinc_read(struct regmap *map, unsigned int reg,
 	if (val_len == 0)
 		return -EINVAL;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
-	if (!regmap_volatile(map, reg) || !regmap_readable_noinc(map, reg)) {
-		ret = -EINVAL;
-		goto out_unlock;
-	}
+	if (!regmap_volatile(map, reg) || !regmap_readable_noinc(map, reg))
+		return -EINVAL;
 
 	/*
 	 * We have not defined the FIFO semantics for cache, as the
@@ -3054,16 +3004,12 @@ int regmap_noinc_read(struct regmap *map, unsigned int reg,
 	 * written value? Just avoid this by always reading the FIFO
 	 * even when using cache. Cache only will not work.
 	 */
-	if (!map->cache_bypass && map->cache_only) {
-		ret = -EBUSY;
-		goto out_unlock;
-	}
+	if (!map->cache_bypass && map->cache_only)
+		return -EBUSY;
 
 	/* Use the accelerated operation if we can */
-	if (map->bus->reg_noinc_read) {
-		ret = regmap_noinc_readwrite(map, reg, val, val_len, false);
-		goto out_unlock;
-	}
+	if (map->bus->reg_noinc_read)
+		return regmap_noinc_readwrite(map, reg, val, val_len, false);
 
 	while (val_len) {
 		if (map->max_raw_read && map->max_raw_read < val_len)
@@ -3072,14 +3018,12 @@ int regmap_noinc_read(struct regmap *map, unsigned int reg,
 			read_len = val_len;
 		ret = _regmap_raw_read(map, reg, val, read_len, true);
 		if (ret)
-			goto out_unlock;
+			return ret;
 		val = ((u8 *)val) + read_len;
 		val_len -= read_len;
 	}
 
-out_unlock:
-	map->unlock(map->lock_arg);
-	return ret;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(regmap_noinc_read);
 
@@ -3149,22 +3093,20 @@ static int _regmap_bulk_read(struct regmap *map, unsigned int reg,
 	u8 *u8 = val;
 	int ret, i;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	for (i = 0; i < val_count; i++) {
 		unsigned int ival;
 
 		if (regs) {
-			if (!IS_ALIGNED(regs[i], map->reg_stride)) {
-				ret = -EINVAL;
-				goto out;
-			}
+			if (!IS_ALIGNED(regs[i], map->reg_stride))
+				return -EINVAL;
 			ret = _regmap_read(map, regs[i], &ival);
 		} else {
 			ret = _regmap_read(map, reg + regmap_get_offset(map, i), &ival);
 		}
 		if (ret != 0)
-			goto out;
+			return ret;
 
 		switch (map->format.val_bytes) {
 		case 4:
@@ -3177,13 +3119,11 @@ static int _regmap_bulk_read(struct regmap *map, unsigned int reg,
 			u8[i] = ival;
 			break;
 		default:
-			ret = -EINVAL;
-			goto out;
+			return -EINVAL;
 		}
 	}
-out:
-	map->unlock(map->lock_arg);
-	return ret;
+
+	return 0;
 }
 
 /**
@@ -3310,7 +3250,7 @@ int regmap_update_bits_base(struct regmap *map, unsigned int reg,
 {
 	int ret;
 
-	map->lock(map->lock_arg);
+	guard(regmap)(map);
 
 	map->async = async;
 
@@ -3318,8 +3258,6 @@ int regmap_update_bits_base(struct regmap *map, unsigned int reg,
 
 	map->async = false;
 
-	map->unlock(map->lock_arg);
-
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regmap_update_bits_base);
@@ -3452,19 +3390,17 @@ int regmap_register_patch(struct regmap *map, const struct reg_sequence *regs,
 		return -ENOMEM;
 	}
 
-	map->lock(map->lock_arg);
+	scoped_guard(regmap, map) {
+		bypass = map->cache_bypass;
 
-	bypass = map->cache_bypass;
+		map->cache_bypass = true;
+		map->async = true;
 
-	map->cache_bypass = true;
-	map->async = true;
-
-	ret = _regmap_multi_reg_write(map, regs, num_regs);
+		ret = _regmap_multi_reg_write(map, regs, num_regs);
 
-	map->async = false;
-	map->cache_bypass = bypass;
-
-	map->unlock(map->lock_arg);
+		map->async = false;
+		map->cache_bypass = bypass;
+	}
 
 	regmap_async_complete(map);
 
diff --git a/drivers/base/transport_class.c b/drivers/base/transport_class.c
index 351c3d3ce6a080..2bee796601ed35 100644
--- a/drivers/base/transport_class.c
+++ b/drivers/base/transport_class.c
@@ -7,7 +7,7 @@
  *
  * The basic idea here is to allow any "device controller" (which
  * would most often be a Host Bus Adapter to use the services of one
- * or more tranport classes for performing transport specific
+ * or more transport classes for performing transport specific
  * services.  Transport specific services are things that the generic
  * command layer doesn't want to know about (speed settings, line
  * condidtioning, etc), but which the user might be interested in.
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index b77f901fc3ef49..6f5d059394b314 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -14,6 +14,7 @@
 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
 
 #include <linux/module.h>
+#include <linux/umh.h>
 #include <linux/drbd.h>
 #include <linux/in.h>
 #include <linux/fs.h>
diff --git a/drivers/block/drbd/drbd_nl_gen.c b/drivers/block/drbd/drbd_nl_gen.c
index 9753dc789bdea9..a3b24aa582977e 100644
--- a/drivers/block/drbd/drbd_nl_gen.c
+++ b/drivers/block/drbd/drbd_nl_gen.c
@@ -638,10 +638,6 @@ const struct genl_split_ops drbd_nl_ops[32] = {
 	},
 };
 
-static const struct genl_multicast_group drbd_nl_mcgrps[] = {
-	[DRBD_NLGRP_EVENTS] = { "events", },
-};
-
 static int __drbd_cfg_context_from_attrs(struct drbd_cfg_context *s,
 		struct nlattr ***ret_nested_attribute_table,
 		struct genl_info *info)
diff --git a/drivers/block/rnull/configfs.rs b/drivers/block/rnull/configfs.rs
index 32c10c3f4d0fd7..34ef96fa125f0a 100644
--- a/drivers/block/rnull/configfs.rs
+++ b/drivers/block/rnull/configfs.rs
@@ -2,15 +2,27 @@
 
 use super::NullBlkDevice;
 use kernel::{
-    block::mq::gen_disk::{GenDisk, GenDiskBuilder},
-    configfs::{self, AttributeOperations},
+    block::mq::gen_disk::{
+        GenDisk,
+        GenDiskBuilder, //
+    },
+    configfs::{
+        self,
+        AttributeOperations, //
+    },
     configfs_attrs,
-    fmt::{self, Write as _},
+    fmt::{
+        self,
+        Write as _, //
+    },
     new_mutex,
     page::PAGE_SIZE,
     prelude::*,
-    str::{kstrtobool_bytes, CString},
-    sync::Mutex,
+    str::{
+        kstrtobool_bytes,
+        CString, //
+    },
+    sync::Mutex, //
 };
 
 pub(crate) fn subsystem() -> impl PinInit<kernel::configfs::Subsystem<Config>, Error> {
@@ -35,7 +47,7 @@ impl AttributeOperations<0> for Config {
 
     fn show(_this: &Config, page: &mut [u8; PAGE_SIZE]) -> Result<usize> {
         let mut writer = kernel::str::Formatter::new(page);
-        writer.write_str("blocksize,size,rotational,irqmode\n")?;
+        writer.write_str("blocksize,size,rotational,irqmode,power\n")?;
         Ok(writer.bytes_written())
     }
 }
@@ -174,7 +186,8 @@ impl configfs::AttributeOperations<1> for DeviceConfig {
     }
 
     fn store(this: &DeviceConfig, page: &[u8]) -> Result {
-        if this.data.lock().powered {
+        let mut guard = this.data.lock();
+        if guard.powered {
             return Err(EBUSY);
         }
 
@@ -182,7 +195,7 @@ impl configfs::AttributeOperations<1> for DeviceConfig {
         let value = text.parse::<u32>().map_err(|_| EINVAL)?;
 
         GenDiskBuilder::validate_block_size(value)?;
-        this.data.lock().block_size = value;
+        guard.block_size = value;
         Ok(())
     }
 }
@@ -204,11 +217,12 @@ impl configfs::AttributeOperations<2> for DeviceConfig {
     }
 
     fn store(this: &DeviceConfig, page: &[u8]) -> Result {
-        if this.data.lock().powered {
+        let mut guard = this.data.lock();
+        if guard.powered {
             return Err(EBUSY);
         }
 
-        this.data.lock().rotational = kstrtobool_bytes(page)?;
+        guard.rotational = kstrtobool_bytes(page)?;
 
         Ok(())
     }
@@ -225,14 +239,15 @@ impl configfs::AttributeOperations<3> for DeviceConfig {
     }
 
     fn store(this: &DeviceConfig, page: &[u8]) -> Result {
-        if this.data.lock().powered {
+        let mut guard = this.data.lock();
+        if guard.powered {
             return Err(EBUSY);
         }
 
         let text = core::str::from_utf8(page)?.trim();
         let value = text.parse::<u64>().map_err(|_| EINVAL)?;
 
-        this.data.lock().capacity_mib = value;
+        guard.capacity_mib = value;
         Ok(())
     }
 }
@@ -248,14 +263,15 @@ impl configfs::AttributeOperations<4> for DeviceConfig {
     }
 
     fn store(this: &DeviceConfig, page: &[u8]) -> Result {
-        if this.data.lock().powered {
+        let mut guard = this.data.lock();
+        if guard.powered {
             return Err(EBUSY);
         }
 
         let text = core::str::from_utf8(page)?.trim();
         let value = text.parse::<u8>().map_err(|_| EINVAL)?;
 
-        this.data.lock().irq_mode = IRQMode::try_from(value)?;
+        guard.irq_mode = IRQMode::try_from(value)?;
         Ok(())
     }
 }
diff --git a/drivers/block/rnull/rnull.rs b/drivers/block/rnull/rnull.rs
index 0ca8715febe833..13048cea8bb0dc 100644
--- a/drivers/block/rnull/rnull.rs
+++ b/drivers/block/rnull/rnull.rs
@@ -10,12 +10,19 @@ use kernel::{
         self,
         mq::{
             self,
-            gen_disk::{self, GenDisk},
-            Operations, TagSet,
-        },
+            gen_disk::{
+                self,
+                GenDisk, //
+            },
+            Operations,
+            TagSet, //
+        }, //
     },
     prelude::*,
-    sync::{aref::ARef, Arc},
+    sync::{
+        aref::ARef,
+        Arc, //
+    }, //
 };
 
 module! {
diff --git a/drivers/block/swim.c b/drivers/block/swim.c
index 0ccc12a72388d5..baeb05f3e0039a 100644
--- a/drivers/block/swim.c
+++ b/drivers/block/swim.c
@@ -2,12 +2,20 @@
 /*
  * Driver for SWIM (Sander Woz Integrated Machine) floppy controller
  *
- * Copyright (C) 2004,2008 Laurent Vivier <Laurent@lvivier.info>
+ * Copyright (C) 2004, 2008 Laurent Vivier <Laurent@lvivier.info>
  *
  * based on Alastair Bridgewater SWIM analysis, 2001
  * based on SWIM3 driver (c) Paul Mackerras, 1996
  * based on netBSD IWM driver (c) 1997, 1998 Hauke Fath.
  *
+ * See also:
+ * Inside Macintosh, vol. III, ch. 2
+ * https://archive.org/details/SWIMDesignDocs
+ * NEC uPD72070 FDC datasheet
+ * mkLinux source file swimiiicommonhal.c
+ * http://www.mac.linux-m68k.org/devel/iwm.php.html
+ * MAME source file sonydriv.cpp
+ *
  * 2004-08-21 (lv) - Initial implementation
  * 2008-10-30 (lv) - Port to 2.6
  */
@@ -19,6 +27,7 @@
 #include <linux/major.h>
 #include <linux/mutex.h>
 #include <linux/hdreg.h>
+#include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/delay.h>
 #include <linux/platform_device.h>
@@ -60,7 +69,7 @@ struct swim {
 	REG(read_handshake)
 } __attribute__((packed));
 
-#define swim_write(base, reg, v) 	out_8(&(base)->write_##reg, (v))
+#define swim_write(base, reg, v)	out_8(&(base)->write_##reg, (v))
 #define swim_read(base, reg)		in_8(&(base)->read_##reg)
 
 /* IWM registers */
@@ -84,62 +93,54 @@ struct iwm {
 	REG(q7H)
 } __attribute__((packed));
 
-#define iwm_write(base, reg, v) 	out_8(&(base)->reg, (v))
+#define iwm_write(base, reg, v)		out_8(&(base)->reg, (v))
 #define iwm_read(base, reg)		in_8(&(base)->reg)
 
-/* bits in phase register */
-
-#define SEEK_POSITIVE	0x070
-#define SEEK_NEGATIVE	0x074
-#define STEP		0x071
-#define MOTOR_ON	0x072
-#define MOTOR_OFF	0x076
-#define INDEX		0x073
-#define EJECT		0x077
-#define SETMFM		0x171
-#define SETGCR		0x175
-
-#define RELAX		0x033
-#define LSTRB		0x008
+/* Bits in phase register */
 
-#define CA_MASK		0x077
+#define RELAX		0x03
+#define LSTRB		0x08
+#define CA_MASK		0x07
+#define PHASE_PIN_DIR	0xF0
 
 /* Select values for swim_select and swim_readbit */
 
-#define READ_DATA_0	0x074
-#define ONEMEG_DRIVE	0x075
-#define SINGLE_SIDED	0x076
-#define DRIVE_PRESENT	0x077
-#define DISK_IN		0x170
-#define WRITE_PROT	0x171
-#define TRACK_ZERO	0x172
-#define TACHO		0x173
-#define READ_DATA_1	0x174
-#define GCR_MODE	0x175
-#define SEEK_COMPLETE	0x176
-#define TWOMEG_MEDIA	0x177
+#define SEL_MASK	0x100
+
+#define SEEK_POSITIVE	0x000
+#define SEEK_NEGATIVE	0x004
+#define STEP		0x001
+#define MOTOR_ON	0x002
+#define MOTOR_OFF	0x006
+#define INDEX		0x003
+#define EJECT		0x007
+#define SETMFM		0x101
+#define SETGCR		0x105
+
+#define STEPPING	0x001
+#define READ_DATA_0	0x004
+#define ONEMEG_DRIVE	0x005
+#define SINGLE_SIDED	0x006
+#define DRIVE_PRESENT	0x007
+#define DISK_IN		0x100
+#define WRITE_PROT	0x101
+#define TRACK_ZERO	0x102
+#define TACHO		0x103
+#define READ_DATA_1	0x104
+#define GCR_MODE	0x105
+#define READY		0x106
+#define TWOMEG_MEDIA	0x107
 
 /* Bits in handshake register */
 
-#define MARK_BYTE	0x01
-#define CRC_ZERO	0x02
-#define RDDATA		0x04
 #define SENSE		0x08
-#define MOTEN		0x10
-#define ERROR		0x20
 #define DAT2BYTE	0x40
 #define DAT1BYTE	0x80
 
 /* bits in setup register */
 
-#define S_INV_WDATA	0x01
-#define S_3_5_SELECT	0x02
-#define S_GCR		0x04
 #define S_FCLK_DIV2	0x08
-#define S_ERROR_CORR	0x10
 #define S_IBM_DRIVE	0x20
-#define S_GCR_WRITE	0x40
-#define S_TIMEOUT	0x80
 
 /* bits in mode register */
 
@@ -148,14 +149,13 @@ struct iwm {
 #define ENBL2		0x04
 #define ACTION		0x08
 #define WRITE_MODE	0x10
-#define HEDSEL		0x20
+#define ISM_SELECT	0x40
 #define MOTON		0x80
 
-/*----------------------------------------------------------------------------*/
-
 enum drive_location {
-	INTERNAL_DRIVE = 0x02,
-	EXTERNAL_DRIVE = 0x04,
+	NO_DRIVE = 0,
+	INTERNAL_DRIVE = BIT(1),
+	EXTERNAL_DRIVE = BIT(2),
 };
 
 enum media_type {
@@ -168,23 +168,23 @@ struct floppy_state {
 	/* physical properties */
 
 	enum drive_location location;	/* internal or external drive */
-	int		 head_number;	/* single- or double-sided drive */
+	int head_number;		/* single- or double-sided drive */
 
 	/* media */
 
-	int		 disk_in;
-	int		 ejected;
-	enum media_type	 type;
-	int		 write_protected;
+	int disk_in;
+	int ejected;
+	enum media_type type;
+	int write_protected;
 
-	int		 total_secs;
-	int		 secpercyl;
-	int		 secpertrack;
+	unsigned int total_secs;
+	unsigned int secpercyl;
+	unsigned int secpertrack;
 
 	/* in-use information */
 
-	int		track;
-	int		ref_count;
+	int track;
+	int ref_count;
 	bool registered;
 
 	struct gendisk *disk;
@@ -209,7 +209,6 @@ enum head {
 
 struct swim_priv {
 	struct swim __iomem *base;
-	spinlock_t lock;
 	int floppy_count;
 	struct floppy_state unit[FD_MAX_UNIT];
 };
@@ -220,20 +219,24 @@ extern int swim_read_sector_data(struct swim __iomem *base,
 				 unsigned char *data);
 
 static DEFINE_MUTEX(swim_mutex);
-static inline void set_swim_mode(struct swim __iomem *base, int enable)
+
+static void set_swim_mode(struct swim __iomem *base, int enable)
 {
-	struct iwm __iomem *iwm_base;
+	struct iwm __iomem *iwm_base = (struct iwm __iomem *)base;
 	unsigned long flags;
 
 	if (!enable) {
-		swim_write(base, mode0, 0xf8);
+		swim_write(base, mode0, ACTION);
+		swim_write(base, mode0, ENBL1 | ENBL2 | MOTON);
+		swim_write(base, mode0, ISM_SELECT);
+		iwm_read(iwm_base, q7L);
 		return;
 	}
 
-	iwm_base = (struct iwm __iomem *)base;
 	local_irq_save(flags);
 
 	iwm_read(iwm_base, q7L);
+	iwm_read(iwm_base, q6L);
 	iwm_read(iwm_base, mtrOff);
 	iwm_read(iwm_base, q6H);
 
@@ -245,7 +248,7 @@ static inline void set_swim_mode(struct swim __iomem *base, int enable)
 	local_irq_restore(flags);
 }
 
-static inline int get_swim_mode(struct swim __iomem *base)
+static int get_swim_mode(struct swim __iomem *base)
 {
 	unsigned long flags;
 
@@ -269,11 +272,11 @@ static inline int get_swim_mode(struct swim __iomem *base)
 
 static inline void swim_select(struct swim __iomem *base, int sel)
 {
-	swim_write(base, phase, RELAX);
+	swim_write(base, phase, RELAX | PHASE_PIN_DIR);
 
-	via1_set_head(sel & 0x100);
+	via1_set_head(sel & SEL_MASK);
 
-	swim_write(base, phase, sel & CA_MASK);
+	swim_write(base, phase, (sel & CA_MASK) | PHASE_PIN_DIR);
 }
 
 static inline void swim_action(struct swim __iomem *base, int action)
@@ -284,15 +287,15 @@ static inline void swim_action(struct swim __iomem *base, int action)
 
 	swim_select(base, action);
 	udelay(1);
-	swim_write(base, phase, (LSTRB<<4) | LSTRB);
-	udelay(1);
-	swim_write(base, phase, (LSTRB<<4) | ((~LSTRB) & 0x0F));
+	swim_write(base, phase, LSTRB | action | PHASE_PIN_DIR);
+	udelay(2);
+	swim_write(base, phase, action | PHASE_PIN_DIR);
 	udelay(1);
 
 	local_irq_restore(flags);
 }
 
-static inline int swim_readbit(struct swim __iomem *base, int bit)
+static inline bool swim_readbit(struct swim __iomem *base, int bit)
 {
 	int stat;
 
@@ -305,125 +308,116 @@ static inline int swim_readbit(struct swim __iomem *base, int bit)
 	return (stat & SENSE) == 0;
 }
 
-static inline void swim_drive(struct swim __iomem *base,
+#define swim_readbit_timeout(base, bit, val, timeout_us) \
+	poll_timeout_us(, swim_readbit(base, bit) == val, 1000, timeout_us, false)
+
+#define swim_READY_timeout(base) \
+({ \
+	int ret = swim_readbit_timeout(base, READY, true, 1000 * 1000); \
+	if (ret) \
+		printk(KERN_DEBUG "%s: drive not ready\n", __func__); \
+	ret; \
+})
+
+static void swim_drive(struct swim __iomem *base,
 			      enum drive_location location)
 {
 	if (location == INTERNAL_DRIVE) {
 		swim_write(base, mode0, EXTERNAL_DRIVE); /* clear drive 1 bit */
 		swim_write(base, mode1, INTERNAL_DRIVE); /* set drive 0 bit */
+		swim_write(base, mode1, MOTON);
 	} else if (location == EXTERNAL_DRIVE) {
 		swim_write(base, mode0, INTERNAL_DRIVE); /* clear drive 0 bit */
 		swim_write(base, mode1, EXTERNAL_DRIVE); /* set drive 1 bit */
+		swim_write(base, mode1, MOTON);
+	} else {
+		swim_write(base, mode0, INTERNAL_DRIVE);
+		swim_write(base, mode0, EXTERNAL_DRIVE);
+		swim_write(base, mode0, MOTON);
 	}
+	udelay(1);
 }
 
-static inline void swim_motor(struct swim __iomem *base,
-			      enum motor_action action)
+static void swim_motor(struct swim __iomem *base,
+		       enum motor_action action)
 {
 	if (action == ON) {
-		int i;
-
 		swim_action(base, MOTOR_ON);
-
-		for (i = 0; i < 2*HZ; i++) {
-			swim_select(base, RELAX);
-			if (swim_readbit(base, MOTOR_ON))
-				break;
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(1);
-		}
+		swim_readbit_timeout(base, MOTOR_ON, true, 2000 * 1000);
 	} else if (action == OFF) {
 		swim_action(base, MOTOR_OFF);
-		swim_select(base, RELAX);
+		swim_write(base, phase, RELAX | PHASE_PIN_DIR);
 	}
 }
 
-static inline void swim_eject(struct swim __iomem *base)
+static void swim_eject(struct swim __iomem *base)
 {
-	int i;
-
 	swim_action(base, EJECT);
-
-	for (i = 0; i < 2*HZ; i++) {
-		swim_select(base, RELAX);
-		if (!swim_readbit(base, DISK_IN))
-			break;
-		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(1);
-	}
-	swim_select(base, RELAX);
+	swim_readbit_timeout(base, DISK_IN, false, 2000 * 1000);
+	msleep(1);
 }
 
 static inline void swim_head(struct swim __iomem *base, enum head head)
 {
-	/* wait drive is ready */
-
 	if (head == UPPER_HEAD)
 		swim_select(base, READ_DATA_1);
 	else if (head == LOWER_HEAD)
 		swim_select(base, READ_DATA_0);
 }
 
-static inline int swim_step(struct swim __iomem *base)
+static int swim_step(struct swim __iomem *base)
 {
-	int wait;
-
 	swim_action(base, STEP);
-
-	for (wait = 0; wait < HZ; wait++) {
-
-		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(1);
-
-		swim_select(base, RELAX);
-		if (!swim_readbit(base, STEP))
-			return 0;
-	}
-	return -1;
+	udelay(150);
+	return swim_readbit_timeout(base, STEPPING, false, 20 * 1000);
 }
 
-static inline int swim_track00(struct swim __iomem *base)
+static int swim_track00(struct swim __iomem *base)
 {
 	int try;
 
 	swim_action(base, SEEK_NEGATIVE);
 
 	for (try = 0; try < 100; try++) {
+		msleep(3);
 
-		swim_select(base, RELAX);
 		if (swim_readbit(base, TRACK_ZERO))
-			break;
+			return 0;
 
 		if (swim_step(base))
-			return -1;
+			break;
 	}
 
-	if (swim_readbit(base, TRACK_ZERO))
-		return 0;
-
+	pr_err("swim: track zero recalibration failed\n");
 	return -1;
 }
 
-static inline int swim_seek(struct swim __iomem *base, int step)
+static int swim_seek(struct swim __iomem *base, int step)
 {
-	if (step == 0)
-		return 0;
-
 	if (step < 0) {
 		swim_action(base, SEEK_NEGATIVE);
 		step = -step;
-	} else
+	} else if (step > 0)
 		swim_action(base, SEEK_POSITIVE);
+	udelay(1);
+
+	swim_READY_timeout(base);
+
+	if (step == 0)
+		return 0;
 
-	for ( ; step > 0; step--) {
+	for (; step > 0; step--) {
 		if (swim_step(base))
 			return -1;
 	}
 
+	msleep(30);
+	swim_READY_timeout(base);
+
 	return 0;
 }
 
-static inline int swim_track(struct floppy_state *fs,  int track)
+static int swim_track(struct floppy_state *fs, int track)
 {
 	struct swim __iomem *base = fs->swd->base;
 	int ret;
@@ -445,8 +439,10 @@ static int floppy_eject(struct floppy_state *fs)
 	struct swim __iomem *base = fs->swd->base;
 
 	swim_drive(base, fs->location);
+	swim_track(fs, 40);
 	swim_motor(base, OFF);
 	swim_eject(base);
+	swim_drive(base, NO_DRIVE);
 
 	fs->disk_in = 0;
 	fs->ejected = 1;
@@ -454,70 +450,79 @@ static int floppy_eject(struct floppy_state *fs)
 	return 0;
 }
 
-static inline int swim_read_sector(struct floppy_state *fs,
-				   int side, int track,
-				   int sector, unsigned char *buffer)
+static unsigned int swim_read_sector_range(struct floppy_state *fs,
+					unsigned int side, unsigned int track,
+					unsigned int start, unsigned int count,
+					unsigned char *buffer)
 {
 	struct swim __iomem *base = fs->swd->base;
 	unsigned long flags;
 	struct sector_header header;
-	int ret = -1;
-	short i;
+	unsigned int i, bits = 0;
 
-	swim_track(fs, track);
-
-	swim_write(base, mode1, MOTON);
-	swim_head(base, side);
-	swim_write(base, mode0, side);
+	if (count > 0) {
+		count = min(count, fs->secpertrack);
+		bits = GENMASK(count - 1, 0);
+	}
 
 	local_irq_save(flags);
-	for (i = 0; i < 36; i++) {
-		ret = swim_read_sector_header(base, &header);
-		if (!ret && (header.sector == sector)) {
-			/* found */
-
-			ret = swim_read_sector_data(base, buffer);
+	for (i = 0; i < fs->secpertrack + 1; i++) {
+		if (bits == 0) /* All sectors were read ok */
 			break;
+
+		if (swim_read_sector_header(base, &header) == 0 &&
+		    swim_read(base, error) == 0) {
+			unsigned int offset = header.sector - start;
+			unsigned char *buf = NULL;
+			int len;
+
+			if (header.track == track && header.side == side &&
+			    header.size == 2 && header.sector >= start &&
+			    header.sector < start + count &&
+			    (bits & BIT(offset)))
+				buf = buffer + 512 * offset;
+			len = swim_read_sector_data(base, buf);
+			if (swim_read(base, error) == 0 && buf && len == 512)
+				bits &= ~BIT(offset);
 		}
 	}
 	local_irq_restore(flags);
 
-	swim_write(base, mode0, MOTON);
-
-	if ((header.side != side)  || (header.track != track) ||
-	     (header.sector != sector))
-		return 0;
-
-	return ret;
+	return bits ? ffs(bits) - 1 : count; /* No. of contiguous ok sectors */
 }
 
 static blk_status_t floppy_read_sectors(struct floppy_state *fs,
-			       int req_sector, int sectors_nb,
-			       unsigned char *buffer)
+					unsigned int req_sector,
+					unsigned int sectors_nb,
+					unsigned char *buffer)
 {
 	struct swim __iomem *base = fs->swd->base;
-	int ret;
-	int side, track, sector;
-	int i, try;
-
+	unsigned int try = 0;
 
 	swim_drive(base, fs->location);
-	for (i = req_sector; i < req_sector + sectors_nb; i++) {
-		int x;
-		track = i / fs->secpercyl;
-		x = i % fs->secpercyl;
-		side = x / fs->secpertrack;
-		sector = x % fs->secpertrack + 1;
-
-		try = 5;
-		do {
-			ret = swim_read_sector(fs, side, track, sector,
-						buffer);
-			if (try-- == 0)
-				return BLK_STS_IOERR;
-		} while (ret != 512);
-
-		buffer += ret;
+	swim_READY_timeout(base);
+
+	while (sectors_nb) {
+		unsigned int cyl, x, head, sector, n, n_ok;
+
+		cyl = req_sector / fs->secpercyl;
+		x = req_sector % fs->secpercyl;
+		head = (x >= fs->secpertrack) ? 1 : 0;
+		sector = x % fs->secpertrack;
+		n = min(sectors_nb, fs->secpertrack - sector);
+
+		swim_track(fs, cyl);
+		swim_head(base, head);
+
+		n_ok = swim_read_sector_range(fs, head, cyl, sector + 1, n, buffer);
+		if (n_ok == n)
+			try = 0;
+		else if (++try >= 5)
+			return BLK_STS_IOERR;
+
+		buffer += 512 * n_ok;
+		sectors_nb -= n_ok;
+		req_sector += n_ok;
 	}
 
 	return 0;
@@ -527,12 +532,10 @@ static blk_status_t swim_queue_rq(struct blk_mq_hw_ctx *hctx,
 				  const struct blk_mq_queue_data *bd)
 {
 	struct floppy_state *fs = hctx->queue->queuedata;
-	struct swim_priv *swd = fs->swd;
 	struct request *req = bd->rq;
 	blk_status_t err;
 
-	if (!spin_trylock_irq(&swd->lock))
-		return BLK_STS_DEV_RESOURCE;
+	mutex_lock(&swim_mutex);
 
 	blk_mq_start_request(req);
 
@@ -550,16 +553,16 @@ static blk_status_t swim_queue_rq(struct blk_mq_hw_ctx *hctx,
 
 	err = BLK_STS_OK;
 out:
-	spin_unlock_irq(&swd->lock);
+	mutex_unlock(&swim_mutex);
 	return err;
 
 }
 
 static struct floppy_struct floppy_type[4] = {
-	{    0,  0, 0,  0, 0, 0x00, 0x00, 0x00, 0x00, NULL }, /* no testing   */
+	{    0,  0, 0,  0, 0, 0x00, 0x00, 0x00, 0x00, NULL }, /* no testing */
 	{  720,  9, 1, 80, 0, 0x2A, 0x02, 0xDF, 0x50, NULL }, /* 360KB SS 3.5"*/
-	{ 1440,  9, 2, 80, 0, 0x2A, 0x02, 0xDF, 0x50, NULL }, /* 720KB 3.5"   */
-	{ 2880, 18, 2, 80, 0, 0x1B, 0x00, 0xCF, 0x6C, NULL }, /* 1.44MB 3.5"  */
+	{ 1440,  9, 2, 80, 0, 0x2A, 0x02, 0xDF, 0x50, NULL }, /* 720KB 3.5" */
+	{ 2880, 18, 2, 80, 0, 0x1B, 0x00, 0xCF, 0x6C, NULL }, /* 1.44MB 3.5" */
 };
 
 static int get_floppy_geometry(struct floppy_state *fs, int type,
@@ -586,23 +589,16 @@ static void setup_medium(struct floppy_state *fs)
 
 	if (swim_readbit(base, DISK_IN)) {
 		struct floppy_struct *g;
+
 		fs->disk_in = 1;
 		fs->write_protected = swim_readbit(base, WRITE_PROT);
-
-		if (swim_track00(base))
-			printk(KERN_ERR
-				"SWIM: cannot move floppy head to track 0\n");
-
-		swim_track00(base);
-
 		fs->type = swim_readbit(base, TWOMEG_MEDIA) ?
-			HD_MEDIA : DD_MEDIA;
+			   HD_MEDIA : DD_MEDIA;
 		fs->head_number = swim_readbit(base, SINGLE_SIDED) ? 1 : 2;
 		get_floppy_geometry(fs, 0, &g);
 		fs->total_secs = g->size;
 		fs->secpercyl = g->head * g->sect;
 		fs->secpertrack = g->sect;
-		fs->track = 0;
 	} else {
 		fs->disk_in = 0;
 	}
@@ -620,11 +616,9 @@ static int floppy_open(struct gendisk *disk, blk_mode_t mode)
 		fs->ref_count = -1;
 	else
 		fs->ref_count++;
-	swim_write(base, setup, S_IBM_DRIVE  | S_FCLK_DIV2);
-	udelay(10);
+
 	swim_drive(base, fs->location);
-	swim_motor(base, ON);
-	swim_action(base, SETMFM);
+
 	if (fs->ejected)
 		setup_medium(fs);
 	if (!fs->disk_in) {
@@ -632,6 +626,11 @@ static int floppy_open(struct gendisk *disk, blk_mode_t mode)
 		goto out;
 	}
 
+	swim_motor(base, ON);
+	swim_action(base, SETMFM);
+	msleep(30);
+	swim_READY_timeout(base);
+
 	set_capacity(fs->disk, fs->total_secs);
 
 	if (mode & BLK_OPEN_NDELAY)
@@ -640,7 +639,8 @@ static int floppy_open(struct gendisk *disk, blk_mode_t mode)
 	if (mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) {
 		if (disk_check_media_change(disk) && fs->disk_in)
 			fs->ejected = 0;
-		if ((mode & BLK_OPEN_WRITE) && fs->write_protected) {
+		if (mode & BLK_OPEN_WRITE) {
+			/* Write support isn't implemented. */
 			err = -EROFS;
 			goto out;
 		}
@@ -652,8 +652,10 @@ static int floppy_open(struct gendisk *disk, blk_mode_t mode)
 	else if (fs->ref_count > 0)
 		--fs->ref_count;
 
-	if (fs->ref_count == 0)
+	if (fs->ref_count == 0) {
 		swim_motor(base, OFF);
+		swim_drive(base, NO_DRIVE);
+	}
 	return err;
 }
 
@@ -679,8 +681,11 @@ static void floppy_release(struct gendisk *disk)
 	else if (fs->ref_count > 0)
 		--fs->ref_count;
 
-	if (fs->ref_count == 0)
+	if (fs->ref_count == 0) {
+		swim_drive(base, fs->location);
 		swim_motor(base, OFF);
+		swim_drive(base, NO_DRIVE);
+	}
 	mutex_unlock(&swim_mutex);
 }
 
@@ -691,14 +696,15 @@ static int floppy_ioctl(struct block_device *bdev, blk_mode_t mode,
 	int err;
 
 	if ((cmd & 0x80) && !capable(CAP_SYS_ADMIN))
-			return -EPERM;
+		return -EPERM;
 
 	switch (cmd) {
 	case FDEJECT:
-		if (fs->ref_count != 1)
-			return -EBUSY;
 		mutex_lock(&swim_mutex);
-		err = floppy_eject(fs);
+		if (fs->ref_count == -1 || fs->ref_count == 1)
+			err = floppy_eject(fs);
+		else
+			err = -EBUSY;
 		mutex_unlock(&swim_mutex);
 		return err;
 
@@ -745,26 +751,34 @@ static const struct block_device_operations floppy_fops = {
 	.check_events	 = floppy_check_events,
 };
 
-static int swim_add_floppy(struct swim_priv *swd, enum drive_location location)
+static void swim_add_floppy(struct swim_priv *swd, enum drive_location location)
 {
 	struct floppy_state *fs = &swd->unit[swd->floppy_count];
 	struct swim __iomem *base = swd->base;
 
-	fs->location = location;
-
 	swim_drive(base, location);
+	if (!swim_readbit(base, DRIVE_PRESENT) ||
+	    swim_readbit(base, ONEMEG_DRIVE))
+		goto out;
+	if (swim_readbit(base, DISK_IN))
+		swim_motor(base, ON);
+	if (swim_track00(base))
+		goto out;
 
-	swim_motor(base, OFF);
+	fs->swd = swd;
+	fs->location = location;
 
 	fs->type = HD_MEDIA;
 	fs->head_number = 2;
 
 	fs->ref_count = 0;
 	fs->ejected = 1;
+	fs->track = 0;
 
 	swd->floppy_count++;
 
-	return 0;
+out:
+	swim_motor(base, OFF);
 }
 
 static const struct blk_mq_ops swim_mq_ops = {
@@ -785,25 +799,38 @@ static void swim_cleanup_floppy_disk(struct floppy_state *fs)
 	blk_mq_free_tag_set(&fs->tag_set);
 }
 
-static int swim_floppy_init(struct swim_priv *swd)
+static void swim_set_parameters(struct swim __iomem *base)
 {
+	unsigned int i;
+	static const u8 mem[] = { 0x18, 0x41, 0x2e, 0x2e, 0x18, 0x18, 0x1b, 0x1b,
+				  0x2f, 0x2f, 0x19, 0x19, 0x97, 0x1b, 0x57, 0x3b, };
+
+	swim_write(base, mode0, 0); /* reset parameter memory index */
+	for (i = 0; i < 16; ++i)
+		swim_write(base, parameter, mem[i]);
+}
+
+static int swim_floppy_init(struct platform_device *pdev)
+{
+	struct swim_priv *swd = platform_get_drvdata(pdev);
+	bool *data = pdev->dev.platform_data;
+	bool fast_fclk = data && *data;
 	struct queue_limits lim = {
-		.features		= BLK_FEAT_ROTATIONAL,
+		.features = BLK_FEAT_ROTATIONAL,
 	};
 	int err;
 	int drive;
 	struct swim __iomem *base = swd->base;
 
+	swim_write(base, setup, S_IBM_DRIVE | (fast_fclk ? S_FCLK_DIV2 : 0));
+
+	swim_set_parameters(base);
+
 	/* scan floppy drives */
 
-	swim_drive(base, INTERNAL_DRIVE);
-	if (swim_readbit(base, DRIVE_PRESENT) &&
-	    !swim_readbit(base, ONEMEG_DRIVE))
-		swim_add_floppy(swd, INTERNAL_DRIVE);
-	swim_drive(base, EXTERNAL_DRIVE);
-	if (swim_readbit(base, DRIVE_PRESENT) &&
-	    !swim_readbit(base, ONEMEG_DRIVE))
-		swim_add_floppy(swd, EXTERNAL_DRIVE);
+	swim_add_floppy(swd, INTERNAL_DRIVE);
+	swim_add_floppy(swd, EXTERNAL_DRIVE);
+	swim_drive(base, NO_DRIVE);
 
 	/* register floppy drives */
 
@@ -814,11 +841,9 @@ static int swim_floppy_init(struct swim_priv *swd)
 		return -EBUSY;
 	}
 
-	spin_lock_init(&swd->lock);
-
 	for (drive = 0; drive < swd->floppy_count; drive++) {
 		err = blk_mq_alloc_sq_tag_set(&swd->unit[drive].tag_set,
-				&swim_mq_ops, 2, 0);
+				&swim_mq_ops, 2, BLK_MQ_F_BLOCKING);
 		if (err)
 			goto exit_put_disks;
 
@@ -830,8 +855,6 @@ static int swim_floppy_init(struct swim_priv *swd)
 			err = PTR_ERR(swd->unit[drive].disk);
 			goto exit_put_disks;
 		}
-
-		swd->unit[drive].swd = swd;
 	}
 
 	for (drive = 0; drive < swd->floppy_count; drive++) {
@@ -905,7 +928,7 @@ static int swim_probe(struct platform_device *dev)
 
 	swd->base = swim_base;
 
-	ret = swim_floppy_init(swd);
+	ret = swim_floppy_init(dev);
 	if (ret)
 		goto out_kfree;
 
@@ -945,7 +968,7 @@ static void swim_remove(struct platform_device *dev)
 static struct platform_driver swim_driver = {
 	.probe  = swim_probe,
 	.remove = swim_remove,
-	.driver   = {
+	.driver = {
 		.name	= CARDNAME,
 	},
 };
diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c
index d4f0d43e0a7235..a03de4c4d35f0f 100644
--- a/drivers/block/swim3.c
+++ b/drivers/block/swim3.c
@@ -887,7 +887,6 @@ static int floppy_locked_ioctl(struct block_device *bdev, blk_mode_t mode,
 			unsigned int cmd, unsigned long param)
 {
 	struct floppy_state *fs = bdev->bd_disk->private_data;
-	int err;
 		
 	if ((cmd & 0x80) && !capable(CAP_SYS_ADMIN))
 		return -EPERM;
@@ -898,10 +897,9 @@ static int floppy_locked_ioctl(struct block_device *bdev, blk_mode_t mode,
 
 	switch (cmd) {
 	case FDEJECT:
-		if (fs->ref_count != 1)
-			return -EBUSY;
-		err = fd_eject(fs);
-		return err;
+		if (fs->ref_count == -1 || fs->ref_count == 1)
+			return fd_eject(fs);
+		return -EBUSY;
 	case FDGETPRM:
 	        if (copy_to_user((void __user *) param, &floppy_type,
 				 sizeof(struct floppy_struct)))
diff --git a/drivers/block/swim_asm.S b/drivers/block/swim_asm.S
index 3d7a2d87595a57..489512a0f561b2 100644
--- a/drivers/block/swim_asm.S
+++ b/drivers/block/swim_asm.S
@@ -5,7 +5,7 @@
  * needs assembly language because is very timing dependent
  * this controller exists only on macintosh 680x0 based
  *
- * Copyright (C) 2004,2008 Laurent Vivier <Laurent@lvivier.info>
+ * Copyright (C) 2004, 2008 Laurent Vivier <Laurent@lvivier.info>
  *
  * based on Alastair Bridgewater SWIM analysis, 2001
  * based on netBSD IWM driver (c) 1997, 1998 Hauke Fath.
@@ -14,135 +14,160 @@
  * 2008-11-05 (lv) - add get_swim_mode
  */
 
-	.equ	write_data,	0x0000
-	.equ	write_mark,	0x0200
-	.equ	write_CRC,	0x0400
-	.equ	write_parameter,0x0600
-	.equ	write_phase,	0x0800
-	.equ	write_setup,	0x0a00
-	.equ	write_mode0,	0x0c00
-	.equ	write_mode1,	0x0e00
-	.equ	read_data,	0x1000
-	.equ	read_mark,	0x1200
-	.equ	read_error,	0x1400
-	.equ	read_parameter,	0x1600
-	.equ	read_phase,	0x1800
-	.equ	read_setup,	0x1a00
-	.equ	read_status,	0x1c00
-	.equ	read_handshake,	0x1e00
-
-	.equ	o_side, 0
-	.equ	o_track, 1
-	.equ	o_sector, 2
-	.equ	o_size, 3
-	.equ	o_crc0, 4
-	.equ	o_crc1, 5
-
-	.equ	seek_time, 30000
-	.equ	max_retry, 40
-	.equ	sector_size, 512
+	.equ	.Lwrite_data,		0x0000
+	.equ	.Lwrite_mark,		0x0200
+	.equ	.Lwrite_CRC,		0x0400
+	.equ	.Lwrite_parameter,	0x0600
+	.equ	.Lwrite_phase,		0x0800
+	.equ	.Lwrite_setup,		0x0a00
+	.equ	.Lwrite_mode0,		0x0c00
+	.equ	.Lwrite_mode1,		0x0e00
+	.equ	.Lread_data,		0x1000
+	.equ	.Lread_mark,		0x1200
+	.equ	.Lread_error,		0x1400
+	.equ	.Lread_parameter,	0x1600
+	.equ	.Lread_phase,		0x1800
+	.equ	.Lread_setup,		0x1a00
+	.equ	.Lread_status,		0x1c00
+	.equ	.Lread_handshake,	0x1e00
+
+	.equ	.Lo_side,		0
+	.equ	.Lo_track,		1
+	.equ	.Lo_sector,		2
+	.equ	.Lo_size,		3
+	.equ	.Lo_crc0,		4
+	.equ	.Lo_crc1,		5
+
+	.equ	.Lseek_time,		30000
+	.equ	.Lmax_retry,		40
+	.equ	.Lsector_size,		512
+	.equ	.Lmark_sequence_len,	4
+
+	.equ	.Lmr_clear_fifo,	0x01
+	.equ	.Lmr_action,		0x08
+	.equ	.Lmr_write_action,	0x18
+
+	.equ	.Lhr_crc_error,		0x02
+	.equ	.Lhr_fifo_2bytes,	0x40
+	.equ	.Lhr_fifo_1byte,	0x80
+
+.Lmfm_mark_check:
+	/*
+	 * This subroutine reads and validates a mark byte sequence.
+	 * On entry, %a1 and %d4 shall hold the location and length (resp.)
+	 * of the mark byte array.
+	 * %a2 and %a3 shall hold the locations of the handshake and mark
+	 * registers.
+	 * Returns zero in %d1 for success.
+	 */
+
+	moveq	#-1, %d1
+	subq	#1, %d4
+	movew	#.Lseek_time, %d2
+
+5:	tstb	%a2@
+	dbmi	%d2, 5b
+	bpl	6f
+
+	moveb	%a3@, %d3
+	cmpb	%a1@+, %d3
+	dbne	%d4, 5b
+	bne	6f
+
+	moveq	#0, %d1
+6:	rts
 
 	.global swim_read_sector_header
 swim_read_sector_header:
 	link	%a6, #0
-	moveml	%d1-%d5/%a0-%a4,%sp@-
+	moveml	%d1-%d5/%a0-%a5,%sp@-
 	movel	%a6@(0x0c), %a4
-	bsr	mfm_read_addrmark
-	moveml	%sp@+, %d1-%d5/%a0-%a4
+	moveq	#-1, %d0
+	bsr	.Lmfm_read_header
+	moveml	%sp@+, %d1-%d5/%a0-%a5
 	unlk	%a6
 	rts
 
-sector_address_mark:
+.Lsector_address_mark:
 	.byte	0xa1, 0xa1, 0xa1, 0xfe
-sector_data_mark:
+.Lsector_data_mark:
 	.byte	0xa1, 0xa1, 0xa1, 0xfb
 
-mfm_read_addrmark:
+.Lmfm_read_header:
 	movel	%a6@(0x08), %a3
-	lea	%a3@(read_handshake), %a2
-	lea	%a3@(read_mark), %a3
-	moveq	#-1, %d0
-	movew	#seek_time, %d2
-
-wait_header_init:
-	tstb	%a3@(read_error - read_mark)
-	moveb	#0x18, %a3@(write_mode0 - read_mark)
-	moveb	#0x01, %a3@(write_mode1 - read_mark)
-	moveb	#0x01, %a3@(write_mode0 - read_mark)
-	tstb	%a3@(read_error - read_mark)
-	moveb	#0x08, %a3@(write_mode1 - read_mark)
-
-	lea	sector_address_mark, %a0
-	moveq	#3, %d1
-
-wait_addr_mark_byte:
-
+	lea	%a3@(.Lread_handshake), %a2
+	lea	%a3@(.Lread_data), %a5
+	lea	%a3@(.Lread_mark), %a3
+
+	moveb	#.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
+	moveb	#.Lmr_clear_fifo, %a3@(.Lwrite_mode1 - .Lread_mark)
+	moveb	#.Lmr_clear_fifo, %a3@(.Lwrite_mode0 - .Lread_mark)
+	tstb	%a3@(.Lread_error - .Lread_mark)
+	moveb	#.Lmr_action, %a3@(.Lwrite_mode1 - .Lread_mark)
+
+	lea	.Lsector_address_mark, %a1
+	moveq	#.Lmark_sequence_len, %d4
+	bsr	.Lmfm_mark_check
+	tstl	%d1
+	bne	.Lsignal_nonyb
+
+	/* read header */
+
+	moveq	#.Lmax_retry, %d2
+.Lamark0:
 	tstb	%a2@
-	dbmi	%d2, wait_addr_mark_byte
-	bpl	header_exit
-
-	moveb	%a3@, %d3
-	cmpb	%a0@+, %d3
-	dbne	%d1, wait_addr_mark_byte
-	bne	wait_header_init
-
-	moveq	#max_retry, %d2
-
-amark0:	tstb	%a2@
-	dbmi	%d2, amark0
-	bpl	signal_nonyb
-
-	moveb	%a3@, %a4@(o_track)
-
-	moveq	#max_retry, %d2
-
-amark1:	tstb	%a2@
-	dbmi	%d2, amark1
-	bpl	signal_nonyb
+	dbmi	%d2, .Lamark0
+	bpl	.Lsignal_nonyb
 
-	moveb	%a3@, %a4@(o_side)
+	moveb	%a5@, %a4@(.Lo_track)
 
-	moveq	#max_retry, %d2
-
-amark2:	tstb	%a2@
-	dbmi	%d2, amark2
-	bpl	signal_nonyb
-
-	moveb	%a3@, %a4@(o_sector)
+	moveq	#.Lmax_retry, %d2
+.Lamark1:
+	tstb	%a2@
+	dbmi	%d2, .Lamark1
+	bpl	.Lsignal_nonyb
 
-	moveq	#max_retry, %d2
+	moveb	%a5@, %a4@(.Lo_side)
 
-amark3:	tstb	%a2@
-	dbmi	%d2, amark3
-	bpl	signal_nonyb
+	moveq	#.Lmax_retry, %d2
+.Lamark2:
+	tstb	%a2@
+	dbmi	%d2, .Lamark2
+	bpl	.Lsignal_nonyb
 
-	moveb	%a3@, %a4@(o_size)
+	moveb	%a5@, %a4@(.Lo_sector)
 
-	moveq	#max_retry, %d2
+	moveq	#.Lmax_retry, %d2
+.Lamark3:
+	tstb	%a2@
+	dbmi	%d2, .Lamark3
+	bpl	.Lsignal_nonyb
 
-crc0:	tstb	%a2@
-	dbmi	%d2, crc0
-	bpl	signal_nonyb
+	moveb	%a5@, %a4@(.Lo_size)
 
-	moveb	%a3@, %a4@(o_crc0)
+	moveq	#.Lmax_retry, %d2
+.Lcrc0:
+	tstb	%a2@
+	dbmi	%d2, .Lcrc0
+	bpl	.Lsignal_nonyb
 
-	moveq	#max_retry, %d2
+	moveb	%a5@, %a4@(.Lo_crc0)
 
-crc1:	tstb	%a2@
-	dbmi	%d2, crc1
-	bpl	signal_nonyb
+	moveq	#.Lmax_retry, %d2
+.Lcrc1:
+	tstb	%a2@
+	dbmi	%d2, .Lcrc1
+	bpl	.Lsignal_nonyb
 
-	moveb	%a3@, %a4@(o_crc1)
+	moveb	%a5@, %a4@(.Lo_crc1)
 
-	tstb	%a3@(read_error - read_mark)
+	moveb	%a2@, %d5
+	andb	#.Lhr_crc_error, %d5
+	bne	.Lsignal_nonyb
 
-header_exit:
 	moveq	#0, %d0
-	moveb	#0x18, %a3@(write_mode0 - read_mark)
-	rts
-signal_nonyb:
-	moveq	#-1, %d0
-	moveb	#0x18, %a3@(write_mode0 - read_mark)
+.Lsignal_nonyb:
+	moveb	#.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
 	rts
 
 	.global swim_read_sector_data
@@ -150,94 +175,85 @@ swim_read_sector_data:
 	link	%a6, #0
 	moveml	%d1-%d5/%a0-%a5,%sp@-
 	movel	%a6@(0x0c), %a4
-	bsr	mfm_read_data
+	moveq	#-1, %d0
+	bsr	.Lmfm_read_data
 	moveml	%sp@+, %d1-%d5/%a0-%a5
 	unlk	%a6
 	rts
 
-mfm_read_data:
+.Lmfm_read_data:
 	movel	%a6@(0x08), %a3
-	lea	%a3@(read_handshake), %a2
-	lea	%a3@(read_data), %a5
-	lea	%a3@(read_mark), %a3
-	movew	#seek_time, %d2
-
-wait_data_init:
-	tstb	%a3@(read_error - read_mark)
-	moveb	#0x18, %a3@(write_mode0 - read_mark)
-	moveb	#0x01, %a3@(write_mode1 - read_mark)
-	moveb	#0x01, %a3@(write_mode0 - read_mark)
-	tstb	%a3@(read_error - read_mark)
-	moveb	#0x08, %a3@(write_mode1 - read_mark)
-
-	lea	sector_data_mark, %a0
-	moveq	#3, %d1
-
-	/* wait data address mark */
-
-wait_data_mark_byte:
-
-	tstb	%a2@
-	dbmi	%d2, wait_data_mark_byte
-	bpl	data_exit
-
-	moveb	%a3@, %d3
-	cmpb	%a0@+, %d3
-	dbne	%d1, wait_data_mark_byte
-	bne	wait_data_init
+	lea	%a3@(.Lread_handshake), %a2
+	lea	%a3@(.Lread_data), %a5
+	lea	%a3@(.Lread_mark), %a3
+
+	moveb	#.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
+	moveb	#.Lmr_clear_fifo, %a3@(.Lwrite_mode1 - .Lread_mark)
+	moveb	#.Lmr_clear_fifo, %a3@(.Lwrite_mode0 - .Lread_mark)
+	tstb	%a3@(.Lread_error - .Lread_mark)
+	moveb	#.Lmr_action, %a3@(.Lwrite_mode1 - .Lread_mark)
+
+	lea	.Lsector_data_mark, %a1
+	moveq	#.Lmark_sequence_len, %d4
+	bsr	.Lmfm_mark_check
+	tstl	%d1
+	bne	.Ldata_exit
 
 	/* read data */
 
-	tstb	%a3@(read_error - read_mark)
-
-	movel	#sector_size-1, %d4		/* sector size */
-read_new_data:
-	movew	#max_retry, %d2
-read_data_loop:
+	movel	#.Lsector_size - 1, %d4		/* sector size */
+	movew	#.Lmax_retry, %d2
+.Lread_data_loop:
 	moveb	%a2@, %d5
-	andb	#0xc0, %d5
-	dbne	%d2, read_data_loop
-	beq	data_exit
-	moveb	%a5@, %a4@+
-	andb	#0x40, %d5
-	dbne	%d4, read_new_data
-	beq	exit_loop
-	moveb	%a5@, %a4@+
-	dbra	%d4, read_new_data
-exit_loop:
+	andb	#(.Lhr_fifo_1byte + .Lhr_fifo_2bytes), %d5
+	dbne	%d2, .Lread_data_loop
+	beq	.Ldata_exit
+
+	moveq	#.Lmax_retry, %d2
+	moveb	%a5@, %d3
+	bsr	.Lmaybe_store
+	dbra	%d4, 1f
+	bra	.Ldata_crc0
+1:	andb	#.Lhr_fifo_2bytes, %d5
+	beq	.Lread_data_loop
+	moveb	%a5@, %d3
+	bsr	.Lmaybe_store
+	dbra	%d4, .Lread_data_loop
 
 	/* read CRC */
 
-	movew	#max_retry, %d2
-data_crc0:
-
+	movew	#.Lmax_retry, %d2
+.Ldata_crc0:
 	tstb	%a2@
-	dbmi	%d2, data_crc0
-	bpl	data_exit
+	dbmi	%d2, .Ldata_crc0
+	bpl	.Ldata_exit
 
-	moveb	%a3@, %d5
-
-	moveq	#max_retry, %d2
-
-data_crc1:
+	moveb	%a5@, %d5
 
+	moveq	#.Lmax_retry, %d2
+.Ldata_crc1:
 	tstb	%a2@
-	dbmi	%d2, data_crc1
-	bpl	data_exit
-
-	moveb	%a3@, %d5
+	dbmi	%d2, .Ldata_crc1
+	bpl	.Ldata_exit
 
-	tstb	%a3@(read_error - read_mark)
+	moveb	%a5@, %d5
 
-	moveb	#0x18, %a3@(write_mode0 - read_mark)
+	moveb	%a2@, %d5
+	andb	#.Lhr_crc_error, %d5
+	bne	.Ldata_exit
 
 	/* return number of bytes read */
 
-	movel	#sector_size, %d0
+	movel	#.Lsector_size, %d0
 	addw	#1, %d4
 	subl	%d4, %d0
+
+.Ldata_exit:
+	moveb	#.Lmr_write_action, %a3@(.Lwrite_mode0 - .Lread_mark)
 	rts
-data_exit:
-	moveb	#0x18, %a3@(write_mode0 - read_mark)
-	moveq	#-1, %d0
-	rts
+
+.Lmaybe_store:
+	tstl	%a4
+	beq	9f
+	moveb	%d3, %a4@+
+9:	rts
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 32bf3ba07a9dcd..c8bfbf8435ac7e 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -743,6 +743,7 @@ static int virtblk_read_zoned_limits(struct virtio_blk *vblk,
 		dev_warn(&vdev->dev, "zero write granularity reported\n");
 		return -ENODEV;
 	}
+	lim->zone_write_granularity = wg;
 	lim->physical_block_size = wg;
 	lim->io_min = wg;
 
diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c
index 974c4691887e6f..028e2f0f587e3f 100644
--- a/drivers/block/zram/zcomp.c
+++ b/drivers/block/zram/zcomp.c
@@ -7,6 +7,8 @@
 #include <linux/wait.h>
 #include <linux/sched.h>
 #include <linux/cpuhotplug.h>
+#include <linux/highmem.h>
+#include <linux/scatterlist.h>
 #include <linux/vmalloc.h>
 #include <linux/sysfs.h>
 
@@ -158,17 +160,32 @@ int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm,
 }
 
 int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm,
-		     const void *src, unsigned int src_len, void *dst)
+		     struct scatterlist *sg, unsigned int src_len, void *dst)
 {
 	struct zcomp_req req = {
-		.src = src,
 		.dst = dst,
 		.src_len = src_len,
 		.dst_len = PAGE_SIZE,
 	};
+	void *src = NULL;
+	int ret;
 
 	might_sleep();
-	return comp->ops->decompress(comp->params, &zstrm->ctx, &req);
+
+	if (sg_is_last(sg)) {
+		/* the object is contained within one page, read it in-place */
+		src = kmap_local_page(sg_page(sg));
+		req.src = src + sg->offset;
+	} else {
+		/* the object spans two pages, linearize it into local copy */
+		sg_copy_to_buffer(sg, 2, zstrm->local_copy, src_len);
+		req.src = zstrm->local_copy;
+	}
+
+	ret = comp->ops->decompress(comp->params, &zstrm->ctx, &req);
+	if (src)
+		kunmap_local(src);
+	return ret;
 }
 
 int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h
index 81a0f3f6ff4871..f38fd31f9e4e02 100644
--- a/drivers/block/zram/zcomp.h
+++ b/drivers/block/zram/zcomp.h
@@ -5,6 +5,8 @@
 
 #include <linux/mutex.h>
 
+struct scatterlist;
+
 #define ZCOMP_PARAM_NOT_SET	INT_MIN
 
 struct deflate_params {
@@ -91,6 +93,6 @@ void zcomp_stream_put(struct zcomp_strm *zstrm);
 int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm,
 		   const void *src, unsigned int *dst_len);
 int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm,
-		     const void *src, unsigned int src_len, void *dst);
+		     struct scatterlist *sg, unsigned int src_len, void *dst);
 
 #endif /* _ZCOMP_H_ */
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index a9b3bb1d3bef35..024402438bd1d4 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -32,6 +32,7 @@
 #include <linux/debugfs.h>
 #include <linux/cpuhotplug.h>
 #include <linux/part_stat.h>
+#include <linux/scatterlist.h>
 #include <linux/kernel_read_file.h>
 #include <linux/rcupdate.h>
 
@@ -415,7 +416,7 @@ static ssize_t mem_used_max_store(struct device *dev,
  * Mark all pages which are older than or equal to cutoff as IDLE.
  * Callers should hold the zram init lock in read mode
  */
-static void mark_idle(struct zram *zram, ktime_t cutoff)
+static void mark_idle(struct zram *zram, time64_t cutoff)
 {
 	int is_idle = 1;
 	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
@@ -439,7 +440,7 @@ static void mark_idle(struct zram *zram, ktime_t cutoff)
 
 #ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
 		is_idle = !cutoff ||
-			ktime_after(cutoff, zram->table[index].attr.ac_time);
+			cutoff > zram->table[index].attr.ac_time;
 #endif
 		if (is_idle)
 			set_slot_flag(zram, index, ZRAM_IDLE);
@@ -453,21 +454,26 @@ static ssize_t idle_store(struct device *dev, struct device_attribute *attr,
 			  const char *buf, size_t len)
 {
 	struct zram *zram = dev_to_zram(dev);
-	ktime_t cutoff = 0;
+	time64_t cutoff = 0;
 
 	if (!sysfs_streq(buf, "all")) {
 		/*
 		 * If it did not parse as 'all' try to treat it as an integer
 		 * when we have memory tracking enabled.
 		 */
+		time64_t uptime;
 		u32 age_sec;
 
-		if (IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) &&
-		    !kstrtouint(buf, 0, &age_sec))
-			cutoff = ktime_sub((u32)ktime_get_boottime_seconds(),
-					   age_sec);
-		else
+		if (!IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) ||
+		    kstrtouint(buf, 0, &age_sec))
 			return -EINVAL;
+
+		/* No slot can be older than the system uptime */
+		uptime = ktime_get_boottime_seconds();
+		if (age_sec >= uptime)
+			return len;
+
+		cutoff = uptime - age_sec;
 	}
 
 	guard(rwsem_read)(&zram->dev_lock);
@@ -1342,9 +1348,9 @@ static int decompress_bdev_page(struct zram *zram, struct page *page,
 				unsigned long index)
 {
 	struct zcomp_strm *zstrm;
+	struct scatterlist sg[1];
 	unsigned int size;
 	int ret, prio;
-	void *src;
 
 	slot_lock(zram, index);
 	/* Since slot was unlocked we need to make sure it's still ZRAM_WB */
@@ -1363,13 +1369,18 @@ static int decompress_bdev_page(struct zram *zram, struct page *page,
 	size = get_slot_size(zram, index);
 	prio = get_slot_comp_priority(zram, index);
 
+	sg_init_table(sg, 1);
+	sg_set_page(sg, page, size, 0);
+
 	zstrm = zcomp_stream_get(zram->comps[prio]);
-	src = kmap_local_page(page);
-	ret = zcomp_decompress(zram->comps[prio], zstrm, src, size,
+	ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size,
 			       zstrm->local_copy);
-	if (!ret)
-		copy_page(src, zstrm->local_copy);
-	kunmap_local(src);
+	if (!ret) {
+		void *dst = kmap_local_page(page);
+
+		copy_page(dst, zstrm->local_copy);
+		kunmap_local(dst);
+	}
 	zcomp_stream_put(zstrm);
 	slot_unlock(zram, index);
 
@@ -1717,11 +1728,6 @@ static int comp_params_store(struct zram *zram, u32 prio, s32 level,
 			       dict_path, sz);
 			return sz;
 		}
-		if (sz == 0) {
-			pr_err("failed to load dictionary %s (empty file)\n",
-			       dict_path);
-			return -EINVAL;
-		}
 	}
 
 	zram->params[prio].dict_sz = sz;
@@ -2086,15 +2092,19 @@ static int read_same_filled_page(struct zram *zram, struct page *page,
 static int read_incompressible_page(struct zram *zram, struct page *page,
 				    unsigned long index)
 {
+	struct scatterlist sg[2];
 	unsigned long handle;
 	void *src, *dst;
 
 	handle = get_slot_handle(zram, index);
-	src = zs_obj_read_begin(zram->mem_pool, handle, PAGE_SIZE, NULL);
+	zs_obj_read_sg_begin(zram->mem_pool, handle, sg, PAGE_SIZE);
+	/* an incompressible object never spans two pages */
+	src = kmap_local_page(sg_page(sg));
 	dst = kmap_local_page(page);
 	copy_page(dst, src);
 	kunmap_local(dst);
-	zs_obj_read_end(zram->mem_pool, handle, PAGE_SIZE, src);
+	kunmap_local(src);
+	zs_obj_read_sg_end(zram->mem_pool, handle);
 
 	return 0;
 }
@@ -2103,9 +2113,10 @@ static int read_compressed_page(struct zram *zram, struct page *page,
 				unsigned long index)
 {
 	struct zcomp_strm *zstrm;
+	struct scatterlist sg[2];
 	unsigned long handle;
 	unsigned int size;
-	void *src, *dst;
+	void *dst;
 	int ret, prio;
 
 	handle = get_slot_handle(zram, index);
@@ -2113,12 +2124,11 @@ static int read_compressed_page(struct zram *zram, struct page *page,
 	prio = get_slot_comp_priority(zram, index);
 
 	zstrm = zcomp_stream_get(zram->comps[prio]);
-	src = zs_obj_read_begin(zram->mem_pool, handle, size,
-				zstrm->local_copy);
+	zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size);
 	dst = kmap_local_page(page);
-	ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, dst);
+	ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size, dst);
 	kunmap_local(dst);
-	zs_obj_read_end(zram->mem_pool, handle, size, src);
+	zs_obj_read_sg_end(zram->mem_pool, handle);
 	zcomp_stream_put(zstrm);
 
 	return ret;
@@ -2128,25 +2138,23 @@ static int read_compressed_page(struct zram *zram, struct page *page,
 static int read_from_zspool_raw(struct zram *zram, struct page *page,
 				unsigned long index)
 {
-	struct zcomp_strm *zstrm;
+	struct scatterlist sg[2];
 	unsigned long handle;
 	unsigned int size;
-	void *src;
+	void *dst;
 
 	handle = get_slot_handle(zram, index);
 	size = get_slot_size(zram, index);
 
 	/*
-	 * We need to get stream just for ->local_copy buffer, in
-	 * case if object spans two physical pages. No decompression
-	 * takes place here, as we read raw compressed data.
+	 * No decompression takes place here, we copy out raw compressed
+	 * data directly into the destination page.
 	 */
-	zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]);
-	src = zs_obj_read_begin(zram->mem_pool, handle, size,
-				zstrm->local_copy);
-	memcpy_to_page(page, 0, src, size);
-	zs_obj_read_end(zram->mem_pool, handle, size, src);
-	zcomp_stream_put(zstrm);
+	zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size);
+	dst = kmap_local_page(page);
+	sg_copy_to_buffer(sg, sg_nents(sg), dst, size);
+	kunmap_local(dst);
+	zs_obj_read_sg_end(zram->mem_pool, handle);
 
 	memzero_page(page, size, PAGE_SIZE - size);
 
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index 4e8c24d757e9a4..db0189ca3cbe2e 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -90,6 +90,20 @@ config BT_HCIBTUSB_RTL
 
 	  Say Y here to compile support for Realtek protocol.
 
+config BT_HCIBTUSB_QCOM
+	bool "Qualcomm protocol support"
+	depends on BT_HCIBTUSB
+	select FW_LOADER
+	select GPIOLIB
+	select WANT_DEV_COREDUMP
+	default y
+	help
+	  Enables various required support, such as configuration, firmware
+	  download, and error recovery, for Qualcomm Bluetooth controllers.
+
+	  Say Y here to compile support for Qualcomm protocol.
+	  If unsure, say Y.
+
 config BT_HCIBTSDIO
 	tristate "HCI SDIO driver"
 	depends on MMC
@@ -151,6 +165,7 @@ config BT_HCIUART_BCSP
 	bool "BCSP protocol support"
 	depends on BT_HCIUART
 	select BITREVERSE
+	select CRC_CCITT
 	help
 	  BCSP (BlueCore Serial Protocol) is serial protocol for communication
 	  between Bluetooth device and host. This protocol is required for non
diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile
index e6b1c1180d1dc9..d5eb2568af2e93 100644
--- a/drivers/bluetooth/Makefile
+++ b/drivers/bluetooth/Makefile
@@ -51,4 +51,7 @@ hci_uart-$(CONFIG_BT_HCIUART_MRVL)	+= hci_mrvl.o
 hci_uart-$(CONFIG_BT_HCIUART_AML)	+= hci_aml.o
 hci_uart-objs				:= $(hci_uart-y)
 
+btusb-y				:= btusb_main.o
+btusb-$(CONFIG_BT_HCIBTUSB_QCOM)	+= btusb_qcom.o
+
 CONTEXT_ANALYSIS := y
diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
index 463d59890befe5..63c4c788d9c48d 100644
--- a/drivers/bluetooth/btbcm.c
+++ b/drivers/bluetooth/btbcm.c
@@ -31,6 +31,7 @@
 #define BDADDR_BCM4334B0 (&(bdaddr_t) {{0x00, 0x00, 0x00, 0xb0, 0x34, 0x43}})
 #define BDADDR_BCM4345C5 (&(bdaddr_t) {{0xac, 0x1f, 0x00, 0xc5, 0x45, 0x43}})
 #define BDADDR_BCM43341B (&(bdaddr_t) {{0xac, 0x1f, 0x00, 0x1b, 0x34, 0x43}})
+#define BDADDR_BCM4384B0 (&(bdaddr_t) {{0x93, 0x76, 0x00, 0xb0, 0x84, 0x43}})
 
 #define BCM_FW_NAME_LEN			64
 #define BCM_FW_NAME_COUNT_MAX		4
@@ -130,7 +131,8 @@ int btbcm_check_bdaddr(struct hci_dev *hdev)
 	    !bacmp(&bda->bdaddr, BDADDR_BCM4345C5) ||
 	    !bacmp(&bda->bdaddr, BDADDR_BCM43430A0) ||
 	    !bacmp(&bda->bdaddr, BDADDR_BCM43430A1) ||
-	    !bacmp(&bda->bdaddr, BDADDR_BCM43341B)) {
+	    !bacmp(&bda->bdaddr, BDADDR_BCM43341B) ||
+	    !bacmp(&bda->bdaddr, BDADDR_BCM4384B0)) {
 		/* Try falling back to BDADDR EFI variable */
 		if (btbcm_set_bdaddr_from_efi(hdev) != 0) {
 			bt_dev_info(hdev, "BCM: Using default device address (%pMR)",
@@ -514,6 +516,8 @@ static const struct bcm_subver_table bcm_uart_subver_table[] = {
 	{ 0x4106, "BCM4335A0"	},	/* 002.001.006 */
 	{ 0x410c, "BCM43430B0"	},	/* 002.001.012 */
 	{ 0x2119, "BCM4373A0"	},	/* 001.001.025 */
+	{ 0x2128, "BCM4384A0"	},	/* 001.001.040 */
+	{ 0x4119, "BCM4384B0"	},	/* 002.001.025 */
 	{ }
 };
 
diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c
index 909a265fd9067e..f298c91f4f4e15 100644
--- a/drivers/bluetooth/btintel.c
+++ b/drivers/bluetooth/btintel.c
@@ -66,6 +66,7 @@ static struct {
 	const char *driver_name;
 	u8         hw_variant;
 	u32        fw_build_num;
+	u32        fw_sha;
 } coredump_info;
 
 const guid_t btintel_guid_dsm =
@@ -289,7 +290,8 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code)
 		goto unlock;
 	}
 
-	bt_dev_err(hdev, "Exception info %s", (char *)(skb->data + 1));
+	bt_dev_err(hdev, "Exception info %.*s", (int)(skb->len - 1),
+		   (char *)(skb->data + 1));
 
 	kfree_skb(skb);
 
@@ -408,8 +410,16 @@ int btintel_load_ddc_config(struct hci_dev *hdev, const char *ddc_name)
 	/* DDC file contains one or more DDC structure which has
 	 * Length (1 byte), DDC ID (2 bytes), and DDC value (Length - 2).
 	 */
-	while (fw->size > fw_ptr - fw->data) {
-		u8 cmd_plen = fw_ptr[0] + sizeof(u8);
+	while (fw->size > (size_t)(fw_ptr - fw->data)) {
+		size_t remaining = fw->size - (fw_ptr - fw->data);
+		unsigned int cmd_plen = fw_ptr[0] + 1U;
+
+		if (cmd_plen < 3 || cmd_plen > U8_MAX || cmd_plen > remaining) {
+			bt_dev_err(hdev, "Malformed DDC record (plen=%u, remaining=%zu)",
+				   cmd_plen, remaining);
+			release_firmware(fw);
+			return -EINVAL;
+		}
 
 		skb = __hci_cmd_sync(hdev, 0xfc8b, cmd_plen, fw_ptr,
 				     HCI_INIT_TIMEOUT);
@@ -507,6 +517,7 @@ int btintel_version_info_tlv(struct hci_dev *hdev,
 	case 0x20:	/* Scorpious Peak2 */
 	case 0x21:	/* Scorpious Peak2 F */
 	case 0x22:	/* BlazarIW (BzrIW) */
+	case 0x23:	/* Draco */
 		break;
 	default:
 		bt_dev_err(hdev, "Unsupported Intel hardware variant (0x%x)",
@@ -560,6 +571,7 @@ int btintel_version_info_tlv(struct hci_dev *hdev,
 
 	coredump_info.hw_variant = INTEL_HW_VARIANT(version->cnvi_bt);
 	coredump_info.fw_build_num = version->build_num;
+	coredump_info.fw_sha = version->git_sha1;
 
 	bt_dev_info(hdev, "%s timestamp %u.%u buildtype %u build %u", variant,
 		    2000 + (version->timestamp >> 8), version->timestamp & 0xff,
@@ -961,6 +973,41 @@ int btintel_send_intel_reset(struct hci_dev *hdev, u32 boot_param)
 }
 EXPORT_SYMBOL_GPL(btintel_send_intel_reset);
 
+static void btintel_get_rom_debug_info(struct hci_dev *hdev)
+{
+	struct btintel_rp_get_rom_debug_info *rom_debug_info;
+	struct sk_buff *skb;
+
+	skb = __hci_cmd_sync(hdev, BTINTEL_GET_ROM_DEBUG_INFO, 0, NULL,
+			     HCI_CMD_TIMEOUT);
+	if (IS_ERR(skb)) {
+		bt_dev_err(hdev, "Failed to send Intel get rom debug info command (%ld)",
+			   PTR_ERR(skb));
+		return;
+	}
+
+	if (skb->len != sizeof(*rom_debug_info)) {
+		bt_dev_err(hdev,
+			   "Intel get rom debug info size mismatch (%u != %zu)",
+			   skb->len, sizeof(*rom_debug_info));
+		kfree_skb(skb);
+		return;
+	}
+
+	rom_debug_info = (struct btintel_rp_get_rom_debug_info *)skb->data;
+
+	/* A non-zero status here would have already been turned into an
+	 * error by the HCI core and reported via the IS_ERR(skb) check above.
+	 */
+	bt_dev_info(hdev, "Intel get rom debug info: dr0:0x%08x dr1:0x%08x dr2:0x%08x dr3:0x%08x",
+		    le32_to_cpu(rom_debug_info->debug_reg0),
+		    le32_to_cpu(rom_debug_info->debug_reg1),
+		    le32_to_cpu(rom_debug_info->debug_reg2),
+		    le32_to_cpu(rom_debug_info->debug_reg3));
+
+	kfree_skb(skb);
+}
+
 int btintel_read_boot_params(struct hci_dev *hdev,
 			     struct intel_boot_params *params)
 {
@@ -2210,6 +2257,8 @@ static int btintel_download_fw(struct hci_dev *hdev,
 			goto done;
 		}
 
+		btintel_get_rom_debug_info(hdev);
+
 		/* When FW download fails, send Intel Reset to retry
 		 * FW download.
 		 */
@@ -2229,8 +2278,10 @@ static int btintel_download_fw(struct hci_dev *hdev,
 	 * of this device.
 	 */
 	err = btintel_download_wait(hdev, calltime, 5000);
-	if (err == -ETIMEDOUT)
+	if (err == -ETIMEDOUT) {
+		btintel_get_rom_debug_info(hdev);
 		btintel_reset_to_bootloader(hdev);
+	}
 
 done:
 	release_firmware(fw);
@@ -2373,6 +2424,7 @@ static int btintel_prepare_fw_download_tlv(struct hci_dev *hdev,
 					   struct intel_version_tlv *ver,
 					   u32 *boot_param)
 {
+	struct btintel_data *intel_data = hci_get_priv(hdev);
 	const struct firmware *fw;
 	char fwname[128];
 	int err;
@@ -2462,6 +2514,8 @@ static int btintel_prepare_fw_download_tlv(struct hci_dev *hdev,
 			goto done;
 		}
 
+		btintel_get_rom_debug_info(hdev);
+
 		/* When FW download fails, send Intel Reset to retry
 		 * FW download.
 		 */
@@ -2481,10 +2535,13 @@ static int btintel_prepare_fw_download_tlv(struct hci_dev *hdev,
 	 * of this device.
 	 */
 	err = btintel_download_wait(hdev, calltime, 5000);
-	if (err == -ETIMEDOUT)
+	if (err == -ETIMEDOUT) {
+		btintel_get_rom_debug_info(hdev);
 		btintel_reset_to_bootloader(hdev);
+	}
 
 done:
+	intel_data->cnvi_bt = ver->cnvi_bt;
 	release_firmware(fw);
 	return err;
 }
@@ -3522,6 +3579,9 @@ int btintel_bootloader_setup_tlv(struct hci_dev *hdev,
 
 	btintel_version_info_tlv(hdev, &new_ver);
 
+	/* Update ver with the operational firmware version */
+	*ver = new_ver;
+
 finish:
 	/* Set the event mask for Intel specific vendor events. This enables
 	 * a few extra events that are useful during general operation. It
@@ -3558,6 +3618,7 @@ void btintel_set_msft_opcode(struct hci_dev *hdev, u8 hw_variant)
 	case 0x20:
 	case 0x21:
 	case 0x22:
+	case 0x23:
 		hci_set_msft_opcode(hdev, 0xFC1E);
 		break;
 	default:
@@ -3901,6 +3962,7 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 	case 0x20:
 	case 0x21:
 	case 0x22:
+	case 0x23:
 		/* Display version information of TLV type */
 		btintel_version_info_tlv(hdev, &ver_tlv);
 
diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h
index 966ec1b02be257..ff7c1d623007e9 100644
--- a/drivers/bluetooth/btintel.h
+++ b/drivers/bluetooth/btintel.h
@@ -61,7 +61,9 @@ struct intel_tlv {
 #define BTINTEL_CNVI_BLAZARU		0x930	/* BlazarU - Meteor Lake */
 #define BTINTEL_CNVI_SCP		0xA00	/* Scorpius Peak - Panther Lake */
 #define BTINTEL_CNVI_SCP2		0xA10	/* Scorpius Peak2 - Nova Lake */
+#define BTINTEL_CNVI_SCP2_C0		0xA11	/* Scorpius Peak2 C0 - Titan Lake */
 #define BTINTEL_CNVI_SCP2F		0xA20	/* Scorpius Peak2F - Nova Lake */
+#define BTINTEL_CNVI_DRACO		0xA30	/* Draco */
 
 /* CNVR */
 #define BTINTEL_CNVR_FMP2		0x910
@@ -178,6 +180,15 @@ struct hci_ppag_enable_cmd {
 	__le32	ppag_enable_flags;
 } __packed;
 
+#define BTINTEL_GET_ROM_DEBUG_INFO	0xfcde
+struct btintel_rp_get_rom_debug_info {
+	__u8	status;
+	__le32	debug_reg0;
+	__le32	debug_reg1;
+	__le32	debug_reg2;
+	__le32	debug_reg3;
+} __packed;
+
 #define INTEL_TLV_TYPE_ID		0x01
 
 #define INTEL_TLV_SYSTEM_EXCEPTION	0x00
@@ -225,7 +236,9 @@ struct btintel_sar_rev2 {
 #define INTEL_HW_PLATFORM(cnvx_bt)	((u8)(((cnvx_bt) & 0x0000ff00) >> 8))
 #define INTEL_HW_VARIANT(cnvx_bt)	((u8)(((cnvx_bt) & 0x003f0000) >> 16))
 #define INTEL_CNVX_TOP_TYPE(cnvx_top)	((cnvx_top) & 0x00000fff)
+#define INTEL_CNVX_TOP_DASH(cnvx_top)	(((cnvx_top) & 0x00f00000) >> 20)
 #define INTEL_CNVX_TOP_STEP(cnvx_top)	(((cnvx_top) & 0x0f000000) >> 24)
+#define INTEL_CNVX_TOP_FLAVOR(cnvx_top)	(((cnvx_top) & 0xf0000000) >> 28)
 #define INTEL_CNVX_TOP_PACK_SWAB(t, s)	__swab16(((__u16)(((t) << 4) | (s))))
 
 enum {
@@ -247,6 +260,7 @@ enum {
 struct btintel_data {
 	DECLARE_BITMAP(flags, __INTEL_NUM_FLAGS);
 	int (*acpi_reset_method)(struct hci_dev *hdev);
+	u32 cnvi_bt;
 };
 
 #define btintel_set_flag(hdev, nr)					\
diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c
index 6e6e2b19815ca9..bcbfe2d5151ac6 100644
--- a/drivers/bluetooth/btintel_pcie.c
+++ b/drivers/bluetooth/btintel_pcie.c
@@ -16,9 +16,11 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/acpi.h>
+#include <linux/suspend.h>
 
 #include <linux/unaligned.h>
 #include <linux/devcoredump.h>
+#include <linux/scatterlist.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -54,6 +56,9 @@ static const struct pci_device_id btintel_pcie_table[] = {
 	{ BTINTEL_PCI_DEVICE(0xD346, PCI_ANY_ID) },
 	 /* Scorpious2, Nova Lake-PCD-S */
 	{ BTINTEL_PCI_DEVICE(0x6E74, PCI_ANY_ID) },
+	 /* Draco */
+	{ BTINTEL_PCI_DEVICE(0x2731, PCI_ANY_ID) },
+	{ BTINTEL_PCI_DEVICE(0x2732, PCI_ANY_ID) },
 	{ 0 }
 };
 MODULE_DEVICE_TABLE(pci, btintel_pcie_table);
@@ -73,17 +78,6 @@ struct btintel_pcie_dev_recovery {
 #define BTINTEL_PCIE_HCI_EVT_PKT	0x00000004
 #define BTINTEL_PCIE_HCI_ISO_PKT	0x00000005
 
-#define BTINTEL_PCIE_MAGIC_NUM    0xA5A5A5A5
-
-#define BTINTEL_PCIE_BLZR_HWEXP_SIZE		1024
-#define BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR	0xB00A7C00
-
-#define BTINTEL_PCIE_SCP_HWEXP_SIZE		4096
-#define BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR		0xB030F800
-
-#define BTINTEL_PCIE_SCP2_HWEXP_SIZE		4096
-#define BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR	0xB031D000
-
 #define BTINTEL_PCIE_MAGIC_NUM	0xA5A5A5A5
 
 #define BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER	0x17A2
@@ -145,6 +139,20 @@ struct btintel_pcie_dbgc_ctxt {
 	struct btintel_pcie_dbgc_ctxt_buf bufs[BTINTEL_PCIE_DBGC_BUFFER_COUNT];
 };
 
+struct btintel_pcie_mdbgc_ctxt {
+	u32     magic_num;
+	u32     ver;
+	u32     buf1_index;
+	u32     buf1_count;
+	struct btintel_pcie_dbgc_ctxt_buf buf1[BTINTEL_PCIE_DBGC_BUFFER_COUNT];
+	u32     buf2_index;
+	u32     buf2_count;
+	struct btintel_pcie_dbgc_ctxt_buf buf2[BTINTEL_PCIE_DBGC_BUFFER_COUNT];
+	u32     buf3_index;
+	u32     buf3_count;
+	struct btintel_pcie_dbgc_ctxt_buf buf3[BTINTEL_PCIE_DBGC_BUFFER_COUNT];
+};
+
 struct btintel_pcie_trigger_evt {
 	u8 type;
 	u8 len;
@@ -186,6 +194,138 @@ static inline char *btintel_pcie_alivectxt_state2str(u32 alive_intr_ctxt)
 	}
 }
 
+/* Returns true when firmware traces are routed to the WiFi DBGC. In that
+ * mode the host must not allocate DBGC buffers and must not publish their
+ * addresses in the context info.
+ */
+static inline bool btintel_pcie_dbg_to_wifi(struct btintel_pcie_data *data)
+{
+	return data->dbg_path_cache != BTINTEL_PCIE_DRAM;
+}
+
+/* Helper function to allocate and setup a debug buffer group
+ * @data: driver data structure
+ * @buf: pointer to data_buf array pointer
+ * @p_addr: pointer to physical DMA address
+ * @v_addr: pointer to virtual address
+ * @frag: pointer to fragment buffer array
+ * @buf_index: buffer index (for error messages)
+ * @buf_count: number of buffers to allocate
+ */
+static int btintel_pcie_alloc_dbgc_buf(struct btintel_pcie_data *data,
+				       struct data_buf **buf,
+				       dma_addr_t *p_addr,
+				       void **v_addr,
+				       struct btintel_pcie_dbgc_ctxt_buf *frag,
+				       u32 buf_index,
+				       u32 buf_count)
+{
+	struct data_buf *b;
+	int i;
+
+	*buf = devm_kcalloc(&data->pdev->dev, buf_count,
+			    sizeof(**buf), GFP_KERNEL);
+	if (!*buf) {
+		BT_ERR("Failed to allocate dbgc buf: %u", buf_index + 1);
+		return -ENOMEM;
+	}
+
+	*v_addr = dmam_alloc_coherent(&data->pdev->dev,
+				      buf_count *
+					BTINTEL_PCIE_DBGC_BUFFER_SIZE,
+					p_addr,
+					GFP_KERNEL | __GFP_NOWARN);
+	if (!*v_addr) {
+		BT_ERR("Failed to allocate dbgc buf: %u DMA", buf_index + 1);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < buf_count; i++) {
+		b = &(*buf)[i];
+		b->data_p_addr = *p_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
+		b->data = *v_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
+		frag[i].buf_addr_lsb = lower_32_bits(b->data_p_addr);
+		frag[i].buf_addr_msb = upper_32_bits(b->data_p_addr);
+		frag[i].buf_size = BTINTEL_PCIE_DBGC_BUFFER_SIZE;
+	}
+
+	return 0;
+}
+
+/* This function initializes the memory for MDBGC buffers and formats the
+ * DBGC fragment which consists header info and DBGC buffer's LSB, MSB and
+ * size as the payload
+ */
+static int btintel_pcie_setup_mdbgc(struct btintel_pcie_data *data)
+{
+	struct btintel_pcie_mdbgc_ctxt db_frag;
+	u32 frag_size = sizeof(db_frag);
+	void *frag_v_addr;
+	int err;
+
+	data->mdbgc.count = BTINTEL_PCIE_DBGC_BUFFER_COUNT;
+
+	/* Allocate fragment context structure */
+	frag_v_addr = dmam_alloc_coherent(&data->pdev->dev, frag_size,
+					  &data->mdbgc.frag_p_addr,
+					  GFP_KERNEL | __GFP_NOWARN);
+	if (!frag_v_addr) {
+		BT_ERR("Failed to allocate mdbgc context");
+		return -ENOMEM;
+	}
+
+	data->mdbgc.frag_v_addr = frag_v_addr;
+	data->mdbgc.frag_size = frag_size;
+
+	/* Initialize fragment header */
+	memset(&db_frag, 0, sizeof(db_frag));
+	db_frag.magic_num = BTINTEL_PCIE_MAGIC_NUM;
+	db_frag.ver = BTINTEL_PCIE_MDBGC_FRAG_VERSION;
+
+	/* Allocate DBGC buffer 1 */
+	db_frag.buf1_index = BTINTEL_PCIE_MDBGC_ALLOCATIONID_1;
+	db_frag.buf1_count = data->mdbgc.count;
+	err = btintel_pcie_alloc_dbgc_buf(data,
+					  &data->mdbgc.buf1,
+					  &data->mdbgc.buf1_p_addr,
+					  &data->mdbgc.buf1_v_addr,
+					  db_frag.buf1,
+					  0,
+					  data->mdbgc.count);
+	if (err)
+		return err;
+
+	/* Allocate DBGC buffer 2 */
+	db_frag.buf2_index = BTINTEL_PCIE_MDBGC_ALLOCATIONID_2;
+	db_frag.buf2_count = data->mdbgc.count;
+	err = btintel_pcie_alloc_dbgc_buf(data,
+					  &data->mdbgc.buf2,
+					  &data->mdbgc.buf2_p_addr,
+					  &data->mdbgc.buf2_v_addr,
+					  db_frag.buf2,
+					  1,
+					  data->mdbgc.count);
+	if (err)
+		return err;
+
+	/* Allocate DBGC buffer 3 */
+	db_frag.buf3_index = BTINTEL_PCIE_MDBGC_ALLOCATIONID_3;
+	db_frag.buf3_count = data->mdbgc.count;
+	err = btintel_pcie_alloc_dbgc_buf(data,
+					  &data->mdbgc.buf3,
+					  &data->mdbgc.buf3_p_addr,
+					  &data->mdbgc.buf3_v_addr,
+					  db_frag.buf3,
+					  2,
+					  data->mdbgc.count);
+	if (err)
+		return err;
+
+	/* Copy fragment to DMA coherent memory */
+	memcpy(data->mdbgc.frag_v_addr, &db_frag, sizeof(db_frag));
+	return 0;
+}
+
 /* This function initializes the memory for DBGC buffers and formats the
  * DBGC fragment which consists header info and DBGC buffer's LSB, MSB and
  * size as the payload
@@ -193,46 +333,42 @@ static inline char *btintel_pcie_alivectxt_state2str(u32 alive_intr_ctxt)
 static int btintel_pcie_setup_dbgc(struct btintel_pcie_data *data)
 {
 	struct btintel_pcie_dbgc_ctxt db_frag;
-	struct data_buf *buf;
-	int i;
+	u32 frag_size = sizeof(db_frag);
+	int err;
 
 	data->dbgc.count = BTINTEL_PCIE_DBGC_BUFFER_COUNT;
-	data->dbgc.bufs = devm_kcalloc(&data->pdev->dev, data->dbgc.count,
-				       sizeof(*buf), GFP_KERNEL);
-	if (!data->dbgc.bufs)
-		return -ENOMEM;
-
-	data->dbgc.buf_v_addr = dmam_alloc_coherent(&data->pdev->dev,
-						    data->dbgc.count *
-						    BTINTEL_PCIE_DBGC_BUFFER_SIZE,
-						    &data->dbgc.buf_p_addr,
-						    GFP_KERNEL | __GFP_NOWARN);
-	if (!data->dbgc.buf_v_addr)
-		return -ENOMEM;
 
+	/* Allocate fragment context structure */
 	data->dbgc.frag_v_addr = dmam_alloc_coherent(&data->pdev->dev,
-						     sizeof(struct btintel_pcie_dbgc_ctxt),
+						     frag_size,
 						     &data->dbgc.frag_p_addr,
 						     GFP_KERNEL | __GFP_NOWARN);
-	if (!data->dbgc.frag_v_addr)
+	if (!data->dbgc.frag_v_addr) {
+		BT_ERR("Failed to allocate dbgc context");
 		return -ENOMEM;
+	}
 
 	data->dbgc.frag_size = sizeof(struct btintel_pcie_dbgc_ctxt);
 
+	/* Initialize fragment header */
+	memset(&db_frag, 0, sizeof(db_frag));
 	db_frag.magic_num = BTINTEL_PCIE_MAGIC_NUM;
 	db_frag.ver = BTINTEL_PCIE_DBGC_FRAG_VERSION;
 	db_frag.total_size = BTINTEL_PCIE_DBGC_FRAG_PAYLOAD_SIZE;
 	db_frag.num_buf = BTINTEL_PCIE_DBGC_FRAG_BUFFER_COUNT;
 
-	for (i = 0; i < data->dbgc.count; i++) {
-		buf = &data->dbgc.bufs[i];
-		buf->data_p_addr = data->dbgc.buf_p_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
-		buf->data = data->dbgc.buf_v_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
-		db_frag.bufs[i].buf_addr_lsb = lower_32_bits(buf->data_p_addr);
-		db_frag.bufs[i].buf_addr_msb = upper_32_bits(buf->data_p_addr);
-		db_frag.bufs[i].buf_size = BTINTEL_PCIE_DBGC_BUFFER_SIZE;
-	}
+	/* Allocate DBGC buffers */
+	err = btintel_pcie_alloc_dbgc_buf(data,
+					  &data->dbgc.bufs,
+					  &data->dbgc.buf_p_addr,
+					  &data->dbgc.buf_v_addr,
+					  db_frag.bufs,
+					  0,
+					  data->dbgc.count);
+	if (err)
+		return err;
 
+	/* Copy fragment to DMA coherent memory */
 	memcpy(data->dbgc.frag_v_addr, &db_frag, sizeof(db_frag));
 	return 0;
 }
@@ -401,7 +537,7 @@ static int btintel_pcie_send_sync(struct btintel_pcie_data *data,
 
 	tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM];
 
-	if (tfd_index > txq->count)
+	if (tfd_index >= txq->count)
 		return -ERANGE;
 
 	if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) {
@@ -645,169 +781,616 @@ static void btintel_pcie_release_mac_access(struct btintel_pcie_data *data)
 	}
 }
 
-static void *btintel_pcie_copy_tlv(void *dest, enum btintel_pcie_tlv_type type,
-				   void *data, size_t size)
+static struct scatterlist *btintel_pcie_alloc_sgtable(ssize_t size)
 {
-	struct intel_tlv *tlv;
+	int nents;
+	struct page *page;
+	struct scatterlist *sg, *result;
 
-	tlv = dest;
-	tlv->type = type;
-	tlv->len = size;
-	memcpy(tlv->val, data, tlv->len);
-	return dest + sizeof(*tlv) + size;
+	if (size <= 0)
+		return NULL;
+
+	nents = DIV_ROUND_UP(size, PAGE_SIZE);
+	result = kcalloc(nents, sizeof(*result), GFP_KERNEL);
+	if (!result)
+		return NULL;
+
+	sg_init_table(result, nents);
+	sg = result;
+
+	while (size > 0) {
+		ssize_t bytes = min_t(ssize_t, size, PAGE_SIZE);
+
+		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+		if (!page)
+			goto err_free;
+
+		sg_set_page(sg, page, bytes, 0);
+		sg = sg_next(sg);
+		size -= bytes;
+	}
+	return result;
+
+err_free:
+	for (sg = result; sg; sg = sg_next(sg)) {
+		page = sg_page(sg);
+		if (page)
+			__free_page(page);
+	}
+	kfree(result);
+	return NULL;
 }
 
-static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data)
+static struct btintel_pcie_dump_entry *
+btintel_pcie_dump_entry_alloc(u32 data_size)
 {
-	u32 offset, prev_size, wr_ptr_status, dump_size, data_len;
-	u32 status_reg, wrap_reg;
-	struct btintel_pcie_dbgc *dbgc = &data->dbgc;
+	struct btintel_pcie_dump_entry *entry;
+
+	entry = vzalloc(sizeof(*entry) + data_size);
+	if (!entry)
+		return NULL;
+
+	entry->size = data_size;
+	INIT_LIST_HEAD(&entry->list);
+	return entry;
+}
+
+static void btintel_pcie_dump_list_free(struct list_head *list)
+{
+	struct btintel_pcie_dump_entry *entry, *tmp;
+
+	list_for_each_entry_safe(entry, tmp, list, list) {
+		list_del(&entry->list);
+		vfree(entry);
+	}
+}
+
+static u32 btintel_pcie_dump_list_total_size(struct list_head *list)
+{
+	struct btintel_pcie_dump_entry *entry;
+	u32 total = 0;
+
+	list_for_each_entry(entry, list, list)
+		total += entry->size;
+
+	return total;
+}
+
+static int btintel_pcie_dump_dram(struct list_head *list,
+				  u8 count, struct data_buf *bufs,
+				  u32 write_ptr, u32 wrap_ctr, u32 cur_frag,
+				  u32 region_id, const char *name)
+{
+	struct btintel_pcie_dump_entry *entry;
+	struct btintel_pcie_ini_dump_data *dump_data;
+	struct btintel_pcie_ini_monitor_dump *mon_dump;
+	struct btintel_pcie_dump_range *range;
+	u32 mon_hdr_size, ranges_size, payload_size, total_size;
+	int i;
+
+	mon_hdr_size = sizeof(*mon_dump);
+	ranges_size = count * (sizeof(*range) + BTINTEL_PCIE_DBGC_BUFFER_SIZE);
+	payload_size = mon_hdr_size + ranges_size;
+	total_size = sizeof(*dump_data) + payload_size;
+
+	entry = btintel_pcie_dump_entry_alloc(total_size);
+	if (!entry)
+		return -ENOMEM;
+
+	dump_data = (void *)entry->data;
+	dump_data->type = BTINTEL_PCIE_INI_REGION_DRAM_BUFFER;
+	dump_data->sub_type = 0;
+	dump_data->sub_type_ver = 0;
+	dump_data->reserved = 0;
+	dump_data->len = cpu_to_le32(payload_size);
+
+	mon_dump = (void *)dump_data->data;
+	mon_dump->header.version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER);
+	mon_dump->header.region_id = cpu_to_le32(region_id);
+	mon_dump->header.num_of_ranges = cpu_to_le32(count);
+	mon_dump->header.name_len = cpu_to_le32(BTINTEL_PCIE_INI_MAX_NAME);
+	memset(mon_dump->header.name, 0, BTINTEL_PCIE_INI_MAX_NAME);
+	strscpy(mon_dump->header.name, name, BTINTEL_PCIE_INI_MAX_NAME);
+
+	mon_dump->write_ptr = cpu_to_le32(write_ptr);
+	mon_dump->cycle_cnt = cpu_to_le32(wrap_ctr);
+	mon_dump->cur_frag = cpu_to_le32(cur_frag);
+
+	range = (void *)mon_dump->data;
+	for (i = 0; i < count; i++) {
+		range->range_data_size =
+			cpu_to_le32(BTINTEL_PCIE_DBGC_BUFFER_SIZE);
+		range->dram_base_addr = cpu_to_le64(bufs[i].data_p_addr);
+		memcpy(range->data, bufs[i].data,
+		       BTINTEL_PCIE_DBGC_BUFFER_SIZE);
+		range = (void *)range->data + BTINTEL_PCIE_DBGC_BUFFER_SIZE;
+	}
+
+	list_add_tail(&entry->list, list);
+	return 0;
+}
+
+static int
+btintel_pcie_dump_dram_monitor(struct btintel_pcie_data *data,
+			       struct list_head *list, u8 count,
+			       struct data_buf *bufs, u32 status_reg,
+			       u32 wrap_reg, u8 allocation_id, u32 region_id,
+			       const char *name)
+{
+	u32 allocation_offset, write_ptr, wrap_ctr, wr_ptr_status;
+	u8 cur_frag;
+
+	allocation_offset = allocation_id *
+			    BTINTEL_PCIE_DBGC_ALLOCATION_OFFSET;
+	wr_ptr_status = btintel_pcie_rd_dev_mem(data, status_reg +
+						allocation_offset);
+	wrap_ctr = btintel_pcie_rd_dev_mem(data, wrap_reg +
+					  allocation_offset);
+
+	write_ptr = wr_ptr_status & BTINTEL_PCIE_DBG_OFFSET_BIT_MASK;
+	write_ptr >>= 2;
+	cur_frag = BTINTEL_PCIE_DBGC_DBG_BUF_IDX(wr_ptr_status);
+	if (cur_frag >= count) {
+		bt_dev_warn(data->hdev,
+			    "Invalid DRAM monitor fragment %u for allocation %u",
+			    cur_frag, allocation_id);
+		return -EINVAL;
+	}
+
+	bt_dev_dbg(data->hdev,
+		   "allocation=%u wr_ptr_status=0x%08x write_ptr=0x%06x cur_frag=%u wrap_ctr=0x%08x",
+		   allocation_id, wr_ptr_status, write_ptr, cur_frag, wrap_ctr);
+
+	return btintel_pcie_dump_dram(list, count, bufs, write_ptr, wrap_ctr,
+				      cur_frag, region_id, name);
+}
+
+static int btintel_pcie_dump_target_region(struct btintel_pcie_data *data,
+					   struct list_head *list,
+					   u32 region_id, const char *name,
+					   u32 addr_start, u32 size)
+{
+	struct btintel_pcie_dump_entry *entry;
+	struct btintel_pcie_ini_dump_data *dump_data;
+	struct btintel_pcie_ini_dump_header *hdr;
+	struct btintel_pcie_dump_range *range;
+	u32 payload_size, total_size, target_mem_offset, tempdata;
+	u8 *dest;
+	int i;
+
+	if (!size) {
+		bt_dev_warn(data->hdev, "Skipping empty dump region: %s", name);
+		return 1;
+	}
+
+	if (!addr_start) {
+		bt_dev_warn(data->hdev,
+			    "Skipping dump region with zero address: %s",
+			    name);
+		return 1;
+	}
+
+	if (addr_start > U32_MAX - size) {
+		bt_dev_warn(data->hdev,
+			    "Skipping dump region %s: addr_start 0x%08x + size %u would overflow",
+			    name, addr_start, size);
+		return 1;
+	}
+
+	/* Align to 4 bytes - target access reads 32-bit words */
+	size = round_down(size, sizeof(u32));
+	if (!size) {
+		bt_dev_warn(data->hdev,
+			    "Skipping dump region %s: size is smaller than 4-byte access",
+			    name);
+		return 1;
+	}
+
+	bt_dev_dbg(data->hdev,
+		   "Target access: region=%s start=0x%08x size=%u",
+		   name, addr_start, size);
+
+	payload_size = sizeof(*hdr) + sizeof(*range) + size;
+	total_size = sizeof(*dump_data) + payload_size;
+	entry = btintel_pcie_dump_entry_alloc(total_size);
+	if (!entry)
+		return -ENOMEM;
+
+	dump_data = (void *)entry->data;
+	dump_data->type = BTINTEL_PCIE_INI_REGION_DEVICE_MEMORY;
+	dump_data->sub_type = 0;
+	dump_data->sub_type_ver = 0;
+	dump_data->reserved = 0;
+	dump_data->len = cpu_to_le32(payload_size);
+
+	hdr = (void *)dump_data->data;
+	hdr->version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER);
+	hdr->region_id = cpu_to_le32(region_id);
+	hdr->num_of_ranges = cpu_to_le32(1);
+	hdr->name_len = cpu_to_le32(BTINTEL_PCIE_INI_MAX_NAME);
+	memset(hdr->name, 0, BTINTEL_PCIE_INI_MAX_NAME);
+	strscpy(hdr->name, name, BTINTEL_PCIE_INI_MAX_NAME);
+
+	range = (void *)(hdr + 1);
+	range->range_data_size = cpu_to_le32(size);
+	range->internal_base_addr = cpu_to_le32(addr_start);
+
+	dest = (u8 *)range->data;
+	target_mem_offset = size / sizeof(u32);
+	for (i = 0; i < target_mem_offset; i++) {
+		u32 offset = BTINTEL_PCIE_TARGET_ACCESS_FRAG_OFFSET * i;
+
+		tempdata = btintel_pcie_rd_dev_mem(data,
+						   addr_start + offset);
+		memcpy(dest, &tempdata, sizeof(tempdata));
+		dest += sizeof(tempdata);
+	}
+
+	list_add_tail(&entry->list, list);
+	return 0;
+}
+
+static int btintel_pcie_dump_smem_monitor_region(struct btintel_pcie_data *data,
+						 struct list_head *list,
+						 u32 region_id,
+						 const char *name,
+						 u32 addr_start, u32 size)
+{
+	struct btintel_pcie_dump_entry *entry;
+	struct btintel_pcie_ini_dump_data *dump_data;
+	struct btintel_pcie_ini_monitor_dump *mon;
+	struct btintel_pcie_dump_range *range;
+	u32 payload_size, total_size, target_mem_offset, tempdata;
+	u8 *dest;
+	int i;
+
+	if (!size || !addr_start) {
+		bt_dev_err(data->hdev, "Skipping smem dump: size = %u addr = %8.8x",
+			   size, addr_start);
+		return -EINVAL;
+	}
+
+	payload_size = sizeof(*mon) + sizeof(*range) + size;
+	total_size = sizeof(*dump_data) + payload_size;
+	entry = btintel_pcie_dump_entry_alloc(total_size);
+	if (!entry)
+		return -ENOMEM;
+
+	dump_data = (void *)entry->data;
+	dump_data->type = BTINTEL_PCIE_INI_REGION_INTERNAL_BUFFER;
+	dump_data->sub_type = 0;
+	dump_data->sub_type_ver = 0;
+	dump_data->reserved = 0;
+	dump_data->len = cpu_to_le32(payload_size);
+
+	mon = (void *)dump_data->data;
+	mon->header.version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER);
+	mon->header.region_id = cpu_to_le32(region_id);
+	mon->header.num_of_ranges = cpu_to_le32(1);
+	mon->header.name_len = cpu_to_le32(BTINTEL_PCIE_INI_MAX_NAME);
+	memset(mon->header.name, 0, BTINTEL_PCIE_INI_MAX_NAME);
+	strscpy(mon->header.name, name, BTINTEL_PCIE_INI_MAX_NAME);
+
+	mon->write_ptr = cpu_to_le32(0);
+	mon->cycle_cnt = cpu_to_le32(0);
+	mon->cur_frag = cpu_to_le32(0);
+
+	range = (void *)mon->data;
+	range->range_data_size = cpu_to_le32(size);
+	range->internal_base_addr = cpu_to_le32(addr_start);
+
+	dest = (u8 *)range->data;
+	target_mem_offset = size / sizeof(u32);
+	for (i = 0; i < target_mem_offset; i++) {
+		u32 offset = BTINTEL_PCIE_TARGET_ACCESS_FRAG_OFFSET * i;
+
+		tempdata = btintel_pcie_rd_dev_mem(data,
+						   addr_start + offset);
+		memcpy(dest, &tempdata, sizeof(tempdata));
+		dest += sizeof(tempdata);
+	}
+
+	list_add_tail(&entry->list, list);
+	return 0;
+}
+
+static int btintel_pcie_dump_info(struct btintel_pcie_data *data,
+				  struct list_head *list,
+				  u64 regions_mask)
+{
+	struct btintel_pcie_dump_entry *entry;
+	struct btintel_pcie_error_dump_data *tlv;
+	struct btintel_pcie_ini_dump_info *dump;
+	u32 size = sizeof(*tlv) + sizeof(*dump);
+	char build_tag[64];
+
+	entry = btintel_pcie_dump_entry_alloc(size);
+	if (!entry)
+		return -ENOMEM;
+
+	tlv = (void *)entry->data;
+	tlv->type = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_INFO_TYPE);
+	tlv->len = cpu_to_le32(sizeof(*dump));
+
+	dump = (void *)tlv->data;
+	memset(dump, 0, sizeof(*dump));
+
+	dump->version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER);
+	dump->trigger_reason = cpu_to_le32(data->dmp_hdr.trigger_reason);
+
+	if (data->dmp_hdr.trigger_reason ==
+	    BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT)
+		dump->time_point =
+			cpu_to_le32(BTINTEL_PCIE_TIME_POINT_FW_ASSERT);
+	else if (data->dmp_hdr.trigger_reason ==
+		 BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER)
+		dump->time_point =
+			cpu_to_le32(BTINTEL_PCIE_TIME_POINT_USER_TRIGGER);
+
+	dump->hw_type = cpu_to_le32(INTEL_CNVX_TOP_TYPE(data->dmp_hdr.cnvi_top));
+	dump->hw_step = cpu_to_le32(INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvi_top));
+	bt_dev_dbg(data->hdev, "hw_type=0x%x hw_step=0x%x (cnvi_top=0x%x)",
+		   le32_to_cpu(dump->hw_type), le32_to_cpu(dump->hw_step),
+		   data->dmp_hdr.cnvi_top);
+
+	dump->ver_type = cpu_to_le32(data->dmp_hdr.cnvi_bt);
+	dump->ver_subtype = cpu_to_le32(data->dmp_hdr.fw_sha);
+	dump->rf_id_type = cpu_to_le32(INTEL_CNVX_TOP_TYPE(data->dmp_hdr.cnvr_top));
+	dump->rf_id_dash = cpu_to_le32(INTEL_CNVX_TOP_DASH(data->dmp_hdr.cnvr_top));
+	dump->rf_id_step = cpu_to_le32(INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvr_top));
+	dump->rf_id_flavor = cpu_to_le32(INTEL_CNVX_TOP_FLAVOR(data->dmp_hdr.cnvr_top));
+	bt_dev_dbg(data->hdev,
+		   "rf_id_type=0x%x rf_id_dash=0x%x rf_id_step=0x%x rf_id_flavor=0x%x (cnvr_top=0x%x)",
+		   le32_to_cpu(dump->rf_id_type),
+		   le32_to_cpu(dump->rf_id_dash),
+		   le32_to_cpu(dump->rf_id_step),
+		   le32_to_cpu(dump->rf_id_flavor),
+		   data->dmp_hdr.cnvr_top);
+	dump->lmac_major = cpu_to_le32(0);
+	dump->lmac_minor = cpu_to_le32(0);
+	dump->umac_major = cpu_to_le32(0);
+	dump->umac_minor = cpu_to_le32(0);
+	dump->fw_mon_mode = cpu_to_le32(BTINTEL_PCIE_FW_MON_MODE_DRAM);
+
+	dump->regions_mask = cpu_to_le64(regions_mask);
+
+	bt_dev_dbg(data->hdev, "ExpectedRegionIDs regions_mask=0x%016llx",
+		   le64_to_cpu(dump->regions_mask));
+
+	snprintf(build_tag, sizeof(build_tag), "%08X", data->dmp_hdr.fw_sha);
+
+	dump->build_tag_len = cpu_to_le32(strlen(build_tag));
+	memcpy(dump->build_tag, build_tag, min(strlen(build_tag),
+					       sizeof(dump->build_tag)));
+
+	dump->num_of_cfg_names = cpu_to_le32(0);
+
+	bt_dev_dbg(data->hdev,
+		   "HwType=0x%08x HwStep=0x%08x RfIdType=0x%08x RfIdDash=0x%08x RfIdStep=0x%08x RfIdFlavor=0x%08x",
+		   le32_to_cpu(dump->hw_type), le32_to_cpu(dump->hw_step),
+		   le32_to_cpu(dump->rf_id_type),
+		   le32_to_cpu(dump->rf_id_dash),
+		   le32_to_cpu(dump->rf_id_step),
+		   le32_to_cpu(dump->rf_id_flavor));
+	bt_dev_dbg(data->hdev, "VerType=0x%08x VerSubType=0x%08x",
+		   le32_to_cpu(dump->ver_type), le32_to_cpu(dump->ver_subtype));
+	bt_dev_dbg(data->hdev,
+		   "LmacMajor=0x%08x LmacMinor=0x%08x UmacMajor=0x%08x UmacMinor=0x%08x",
+		   le32_to_cpu(dump->lmac_major), le32_to_cpu(dump->lmac_minor),
+		   le32_to_cpu(dump->umac_major),
+		   le32_to_cpu(dump->umac_minor));
+	bt_dev_dbg(data->hdev, "TriggerReason=0x%04x MonMode=%u BuildTag=%.64s",
+		   le32_to_cpu(dump->trigger_reason),
+		   le32_to_cpu(dump->fw_mon_mode),
+		   dump->build_tag);
+
+	list_add(&entry->list, list);
+
+	return 0;
+}
+
+static bool btintel_pcie_is_mdbgc_supported(struct btintel_pcie_data *data)
+{
+	return data->pdev->device == BTINTEL_PCIE_DEVICE_ID_NVL_S_SCP2 ||
+		data->pdev->device == BTINTEL_PCIE_DEVICE_ID_NVL_Hx_SCP2 ||
+		data->pdev->device == BTINTEL_PCIE_DEVICE_ID_PTL_FMP2;
+}
+
+static void btintel_pcie_dump_mem_range(struct btintel_pcie_data *data,
+					struct list_head *list, u32 region_id,
+					const char *name, u32 addr_start,
+					u32 region_size, u64 *regions_mask)
+{
+	if (!region_size)
+		return;
+
+	if (!btintel_pcie_dump_target_region(data, list, region_id, name,
+					     addr_start, region_size))
+		*regions_mask |= BIT_ULL(region_id);
+}
+
+static int btintel_pcie_read_debug_regions(struct btintel_pcie_data *data)
+{
+	struct btintel_pcie_dbgc *dbgc = NULL;
+	struct btintel_pcie_mdbgc *mdbgc = NULL;
+	struct btintel_pcie_dump_mem_info *dump_info = &data->dump_info;
 	struct hci_dev *hdev = data->hdev;
-	u8 *pdata, *p, buf_idx, hw_variant;
-	struct intel_tlv *tlv;
-	struct timespec64 now;
-	struct tm tm_now;
-	char fw_build[128];
-	char ts[128];
-	char vendor[64];
-	char driver[64];
+	struct btintel_pcie_dump_entry *entry;
+	struct btintel_pcie_dump_file_hdr *file_hdr;
+	struct scatterlist *sg_dump_data;
+	u32 status_reg, wrap_reg;
+	u64 regions_mask = 0;
+	u8 hw_variant;
+	u32 file_len;
+	u8 count;
+	int ret;
+	LIST_HEAD(dump_list);
 
 	if (!IS_ENABLED(CONFIG_DEV_COREDUMP))
 		return -EOPNOTSUPP;
 
+	if (btintel_pcie_is_mdbgc_supported(data)) {
+		mdbgc = &data->mdbgc;
+		count = mdbgc->count;
+	} else {
+		dbgc = &data->dbgc;
+		count = dbgc->count;
+	}
 
 	hw_variant = INTEL_HW_VARIANT(data->cnvi);
-	switch (hw_variant) {
-	case BTINTEL_HWID_BZRI:
-	case BTINTEL_HWID_BZRIW:
+
+	if (hw_variant == BTINTEL_HWID_BZRI ||
+	    hw_variant == BTINTEL_HWID_BZRIW) {
 		status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS;
 		wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND;
-		break;
-	case BTINTEL_HWID_SCP:
-	case BTINTEL_HWID_SCP2:
-	case BTINTEL_HWID_SCP2F:
+	} else if (hw_variant >= BTINTEL_HWID_SCP) {
 		status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS_SCP;
 		wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND_SCP;
-		break;
-	default:
-		bt_dev_err(hdev, "Unsupported Intel hardware variant (0x%2.2x)",
+	} else {
+		bt_dev_err(hdev,
+			   "Unsupported Intel hardware variant (0x%2.2x)",
 			   hw_variant);
 		return -EINVAL;
 	}
 
-	wr_ptr_status = btintel_pcie_rd_dev_mem(data, status_reg);
-	data->dmp_hdr.wrap_ctr = btintel_pcie_rd_dev_mem(data, wrap_reg);
-
-	offset = wr_ptr_status & BTINTEL_PCIE_DBG_OFFSET_BIT_MASK;
+	if (btintel_pcie_is_mdbgc_supported(data)) {
+		ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count,
+						     mdbgc->buf1, status_reg,
+						     wrap_reg,
+						     BTINTEL_PCIE_MDBGC_ALLOCATIONID_1,
+						     BTINTEL_PCIE_INI_ID_DRAM_MONITOR1,
+						     "monitor");
+		if (!ret)
+			regions_mask |=
+				BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR1);
+		else
+			bt_dev_warn(hdev, "Failed to dump DRAM buf1: %d", ret);
+
+		ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count,
+						     mdbgc->buf2, status_reg,
+						     wrap_reg,
+						     BTINTEL_PCIE_MDBGC_ALLOCATIONID_2,
+						     BTINTEL_PCIE_INI_ID_DRAM_MONITOR2,
+						     "monitor2");
+		if (!ret)
+			regions_mask |=
+				BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR2);
+		else
+			bt_dev_warn(hdev, "Failed to dump DRAM buf2: %d", ret);
+
+		ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count,
+						     mdbgc->buf3, status_reg,
+						     wrap_reg,
+						     BTINTEL_PCIE_MDBGC_ALLOCATIONID_3,
+						     BTINTEL_PCIE_INI_ID_DRAM_MONITOR3,
+						     "monitor3");
+		if (!ret)
+			regions_mask |=
+				BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR3);
+		else
+			bt_dev_warn(hdev, "Failed to dump DRAM buf3: %d", ret);
+	} else {
+		ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count,
+						     dbgc->bufs, status_reg,
+						     wrap_reg,
+						     BTINTEL_PCIE_MDBGC_ALLOCATIONID_1,
+						     BTINTEL_PCIE_INI_ID_DRAM_MONITOR1,
+						     "monitor");
+		if (!ret)
+			regions_mask |=
+				BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR1);
+		else
+			bt_dev_warn(hdev,
+				    "Failed to dump DRAM region: %d", ret);
+	}
 
-	buf_idx = BTINTEL_PCIE_DBGC_DBG_BUF_IDX(wr_ptr_status);
-	if (buf_idx > dbgc->count) {
-		bt_dev_warn(hdev, "Buffer index is invalid");
-		return -EINVAL;
+	if (dump_info->smem_size &&
+	    !btintel_pcie_dump_smem_monitor_region(data, &dump_list,
+						   BTINTEL_PCIE_INI_ID_SMEM,
+						   "monitor_smem",
+						   dump_info->smem_addr_start,
+						   dump_info->smem_size))
+		regions_mask |= BIT_ULL(BTINTEL_PCIE_INI_ID_SMEM);
+
+	if (dump_info->exception_dump_addr && dump_info->exception_dump_len) {
+		ret = btintel_pcie_dump_target_region(data, &dump_list,
+						      BTINTEL_PCIE_INI_ID_EXCEPTION_EVT,
+						      "EXCEPTION_EVT_BUFFER",
+						      dump_info->exception_dump_addr,
+						      dump_info->exception_dump_len);
+		if (!ret)
+			regions_mask |= BIT_ULL(BTINTEL_PCIE_INI_ID_EXCEPTION_EVT);
 	}
 
-	prev_size = buf_idx * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
-	if (prev_size + offset >= prev_size)
-		data->dmp_hdr.write_ptr = prev_size + offset;
-	else
-		return -EINVAL;
+	btintel_pcie_dump_mem_range(data, &dump_list,
+				    BTINTEL_PCIE_INI_ID_DCCM, "DCCM",
+				    dump_info->dccm_addr_start,
+				    dump_info->dccm_size,
+				    &regions_mask);
+
+	btintel_pcie_dump_mem_range(data, &dump_list,
+				    BTINTEL_PCIE_INI_ID_SDS, "SDS",
+				    dump_info->sds_start_addr_start,
+				    dump_info->sds_start_size,
+				    &regions_mask);
+
+	btintel_pcie_dump_mem_range(data, &dump_list,
+				    BTINTEL_PCIE_INI_ID_SDS_IOSF, "SDS_IOSF",
+				    dump_info->sds_iosf_data_addr_start,
+				    dump_info->sds_iosf_data_size,
+				    &regions_mask);
+
+	btintel_pcie_dump_mem_range(data, &dump_list,
+				    BTINTEL_PCIE_INI_ID_ECL, "ECL_REGION",
+				    dump_info->ecl_addr_start,
+				    dump_info->ecl_size,
+				    &regions_mask);
+
+	ret = btintel_pcie_dump_info(data, &dump_list, regions_mask);
+	if (ret) {
+		btintel_pcie_dump_list_free(&dump_list);
+		return ret;
+	}
 
-	strscpy(vendor, "Vendor: Intel\n");
-	snprintf(driver, sizeof(driver), "Driver: %s\n",
-		 data->dmp_hdr.driver_name);
-
-	ktime_get_real_ts64(&now);
-	time64_to_tm(now.tv_sec, 0, &tm_now);
-	snprintf(ts, sizeof(ts), "Dump Time: %02d-%02d-%04ld %02d:%02d:%02d",
-				 tm_now.tm_mday, tm_now.tm_mon + 1, tm_now.tm_year + 1900,
-				 tm_now.tm_hour, tm_now.tm_min, tm_now.tm_sec);
-
-	snprintf(fw_build, sizeof(fw_build),
-			    "Firmware Timestamp: Year %u WW %02u buildtype %u build %u",
-			    2000 + (data->dmp_hdr.fw_timestamp >> 8),
-			    data->dmp_hdr.fw_timestamp & 0xff, data->dmp_hdr.fw_build_type,
-			    data->dmp_hdr.fw_build_num);
-
-	data_len = sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_bt) +
-		sizeof(*tlv) + sizeof(data->dmp_hdr.write_ptr) +
-		sizeof(*tlv) + sizeof(data->dmp_hdr.wrap_ctr) +
-		sizeof(*tlv) + sizeof(data->dmp_hdr.trigger_reason) +
-		sizeof(*tlv) + sizeof(data->dmp_hdr.fw_git_sha1) +
-		sizeof(*tlv) + sizeof(data->dmp_hdr.cnvr_top) +
-		sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_top) +
-		sizeof(*tlv) + strlen(ts) +
-		sizeof(*tlv) + strlen(fw_build) +
-		sizeof(*tlv) + strlen(vendor) +
-		sizeof(*tlv) + strlen(driver);
-
-	if (data->dmp_hdr.event_type && data->dmp_hdr.event_id) {
-		data_len += sizeof(*tlv) + sizeof(data->dmp_hdr.event_type);
-		data_len += sizeof(*tlv) + sizeof(data->dmp_hdr.event_id);
+	file_len = sizeof(*file_hdr) +
+		   btintel_pcie_dump_list_total_size(&dump_list);
+	entry = btintel_pcie_dump_entry_alloc(sizeof(*file_hdr));
+	if (!entry) {
+		btintel_pcie_dump_list_free(&dump_list);
+		return -ENOMEM;
 	}
 
-	/*
-	 * sizeof(u32) - signature
-	 * sizeof(data_len) - to store tlv data size
-	 * data_len - TLV data
-	 */
-	dump_size = sizeof(u32) + sizeof(data_len) + data_len;
+	file_hdr = (void *)entry->data;
+	file_hdr->barker = cpu_to_le32(BTINTEL_PCIE_INI_ERROR_DUMP_BARKER);
+	file_hdr->file_len = cpu_to_le32(file_len);
+	list_add(&entry->list, &dump_list);
 
+	sg_dump_data = btintel_pcie_alloc_sgtable(file_len);
+	if (sg_dump_data) {
+		int sg_entries = sg_nents(sg_dump_data);
+		u32 offs = 0;
 
-	/* Add debug buffers data length to dump size */
-	dump_size += BTINTEL_PCIE_DBGC_BUFFER_SIZE * dbgc->count;
+		list_for_each_entry(entry, &dump_list, list) {
+			sg_pcopy_from_buffer(sg_dump_data, sg_entries,
+					     entry->data, entry->size, offs);
+			offs += entry->size;
+		}
 
-	pdata = vmalloc(dump_size);
-	if (!pdata)
-		return -ENOMEM;
-	p = pdata;
-
-	*(u32 *)p = BTINTEL_PCIE_MAGIC_NUM;
-	p += sizeof(u32);
-
-	*(u32 *)p = data_len;
-	p += sizeof(u32);
-
-
-	p = btintel_pcie_copy_tlv(p, BTINTEL_VENDOR, vendor, strlen(vendor));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_DRIVER, driver, strlen(driver));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_DUMP_TIME, ts, strlen(ts));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_FW_BUILD, fw_build,
-				  strlen(fw_build));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_BT, &data->dmp_hdr.cnvi_bt,
-				  sizeof(data->dmp_hdr.cnvi_bt));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_WRITE_PTR, &data->dmp_hdr.write_ptr,
-				  sizeof(data->dmp_hdr.write_ptr));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_WRAP_CTR, &data->dmp_hdr.wrap_ctr,
-				  sizeof(data->dmp_hdr.wrap_ctr));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_TRIGGER_REASON, &data->dmp_hdr.trigger_reason,
-				  sizeof(data->dmp_hdr.trigger_reason));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_FW_SHA, &data->dmp_hdr.fw_git_sha1,
-				  sizeof(data->dmp_hdr.fw_git_sha1));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_CNVR_TOP, &data->dmp_hdr.cnvr_top,
-				  sizeof(data->dmp_hdr.cnvr_top));
-	p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_TOP, &data->dmp_hdr.cnvi_top,
-				  sizeof(data->dmp_hdr.cnvi_top));
-
-	if (data->dmp_hdr.event_type && data->dmp_hdr.event_id) {
-		p = btintel_pcie_copy_tlv(p, BTINTEL_EVENT_TYPE,
-					  &data->dmp_hdr.event_type,
-					  sizeof(data->dmp_hdr.event_type));
-		p = btintel_pcie_copy_tlv(p, BTINTEL_EVENT_ID,
-					  &data->dmp_hdr.event_id,
-					  sizeof(data->dmp_hdr.event_id));
-		data->dmp_hdr.event_type = 0;
-		data->dmp_hdr.event_id = 0;
-	}
-
-	memcpy(p, dbgc->bufs[0].data, dbgc->count * BTINTEL_PCIE_DBGC_BUFFER_SIZE);
-	dev_coredumpv(&hdev->dev, pdata, dump_size, GFP_KERNEL);
-	return 0;
+		bt_dev_dbg(hdev, "triggering dev_coredumpsg()");
+		dev_coredumpsg(&hdev->dev, sg_dump_data, file_len, GFP_KERNEL);
+	} else {
+		bt_dev_err(hdev, "Failed to allocate scatter-gather table for coredump");
+		ret = -ENOMEM;
+	}
+
+	btintel_pcie_dump_list_free(&dump_list);
+	return ret;
 }
 
 static void btintel_pcie_dump_traces(struct hci_dev *hdev)
 {
 	struct btintel_pcie_data *data = hci_get_drvdata(hdev);
-	int ret = 0;
+	int ret;
 
 	ret = btintel_pcie_get_mac_access(data);
 	if (ret) {
@@ -815,7 +1398,7 @@ static void btintel_pcie_dump_traces(struct hci_dev *hdev)
 		return;
 	}
 
-	ret = btintel_pcie_read_dram_buffers(data);
+	ret = btintel_pcie_read_debug_regions(data);
 
 	btintel_pcie_release_mac_access(data);
 
@@ -963,10 +1546,391 @@ static inline bool btintel_pcie_in_error(struct btintel_pcie_data *data)
 	return	data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER;
 }
 
+static u32 btintel_pcie_region_size(struct hci_dev *hdev, const char *name,
+				    u32 start, u32 end)
+{
+	u64 size;
+
+	if (!start || !end)
+		return 0;
+
+	if (end < start) {
+		bt_dev_warn(hdev, "%s region invalid: start=0x%08x end=0x%08x",
+			    name, start, end);
+		return 0;
+	}
+
+	size = (u64)end + 0x04 - start;
+	if (size % sizeof(u32))
+		bt_dev_warn(hdev, "%s region size %llu is not 4-byte aligned",
+			    name, size);
+
+	if (size > BTINTEL_PCIE_REGION_MAX_SIZE) {
+		bt_dev_warn(hdev, "%s region size %llu exceeds cap %u, ignoring",
+			    name, size, BTINTEL_PCIE_REGION_MAX_SIZE);
+		return 0;
+	}
+
+	return (u32)size;
+}
+
+static const char *btintel_pcie_tlv_str(u8 tlv_type)
+{
+	switch (tlv_type) {
+	case BTINTEL_PCIE_TLV_TYPE_EXCEPTION_DUMP_ADDRESS:
+		return "EXCEPTION_DUMP_ADDRESS";
+	case BTINTEL_PCIE_TLV_TYPE_DCCM_MEM_ADDRESS:
+		return "DCCM_MEM_ADDRESS";
+	case BTINTEL_PCIE_TLV_TYPE_SDS_MEM_ADDRESS:
+		return "SDS_MEM_ADDRESS";
+	case BTINTEL_PCIE_TLV_TYPE_ECL_MEM_ADDRESS:
+		return "ECL_MEM_ADDRESS";
+	case BTINTEL_PCIE_TLV_TYPE_SMEM_ADDRESS:
+		return "SMEM_ADDRESS";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static int btintel_parse_mbox_tlv(struct btintel_pcie_data *data)
+{
+	/* Custom TLV structure for mailbox parsing
+	 * len is __le16 as per agreement with FW
+	 */
+	struct mbox_tlv {
+		u8 type;
+		__le16 len;
+		u8 val[];
+	} __packed;
+
+	u8 *buffer, *ptr;
+	u32 buffer_len, remaining;
+	int err;
+	u32 tbl_addr, tbl_size;
+	struct mbox_tlv *tlv;
+	struct btintel_data *cnvi_data = hci_get_priv(data->hdev);
+	u8 hw_variant = INTEL_HW_VARIANT(cnvi_data->cnvi_bt);
+	long t;
+
+	/* Wait for GP0 alive interrupt to post RX buffers */
+	t = wait_event_timeout(data->mbox_parse_wait_q,
+			       test_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags),
+			       msecs_to_jiffies(BTINTEL_PCIE_MBOX_INTR_TIMEOUT_MS));
+	if (!t) {
+		bt_dev_warn(data->hdev,
+			    "Timeout (%u ms) waiting for alive interrupt before mbox TLV parse; skipping",
+			    BTINTEL_PCIE_MBOX_INTR_TIMEOUT_MS);
+		return 0;
+	}
+	clear_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags);
+
+	bt_dev_info(data->hdev,
+		    "mbox TLV parse started at %lld ns (%lld us after mbox interrupt)",
+		    ktime_to_ns(ktime_get()),
+		    ktime_to_us(ktime_sub(ktime_get(), data->mbox_intr_ts)));
+
+	memset(&data->dump_info, 0, sizeof(data->dump_info));
+
+	/* Snapshot to avoid TOCTOU with the GP1 IRQ handler */
+	tbl_size = READ_ONCE(data->debug_table_size);
+	tbl_addr = READ_ONCE(data->debug_table_addr);
+
+	if (!tbl_size || !tbl_addr)
+		return -EINVAL;
+
+	/* Ensure size is 4-byte aligned; btintel_pcie_read_device_mem()
+	 * reads device memory in 4-byte units.
+	 */
+	tbl_size = ALIGN_DOWN(tbl_size, 4);
+
+	if (!tbl_size)
+		return -EINVAL;
+
+	if (tbl_size > SZ_1M) {
+		bt_dev_err(data->hdev, "Debug table size too large: %u",
+			   tbl_size);
+		return -EINVAL;
+	}
+
+	buffer_len = tbl_size;
+
+	buffer = vmalloc(buffer_len);
+	if (!buffer)
+		return -ENOMEM;
+
+	btintel_pcie_mac_init(data);
+
+	err = btintel_pcie_read_device_mem(data, buffer, tbl_addr,
+					   buffer_len);
+	if (err)
+		goto exit_on_error;
+
+	print_hex_dump(KERN_INFO, "Bluetooth: mbox_tlv: ", DUMP_PREFIX_OFFSET, 16, 1,
+		       buffer, buffer_len, false);
+
+	ptr = buffer;
+	remaining = buffer_len;
+
+	/* Parse TLV structures: 1 byte type + 2 bytes length +
+	 * variable value
+	 */
+	while (remaining >= sizeof(struct mbox_tlv)) {
+		u16 tlv_len;
+		u32 tlv_total;
+
+		tlv = (struct mbox_tlv *)ptr;
+		tlv_len = le16_to_cpu(tlv->len);
+		tlv_total = sizeof(tlv->type) + sizeof(tlv->len) + tlv_len;
+
+		if (tlv_total > remaining) {
+			bt_dev_err(data->hdev, "TLV parse error: not enough data for TLV value (type=%u, len=%u)",
+				   tlv->type, tlv_len);
+			break;
+		}
+
+		switch (tlv->type) {
+		case BTINTEL_PCIE_TLV_TYPE_EXCEPTION_DUMP_ADDRESS:
+			if (tlv_len < 8) {
+				bt_dev_err(data->hdev,
+					   "TLV %s too short: len=%u (need 8)",
+					   btintel_pcie_tlv_str(tlv->type),
+					   tlv_len);
+				break;
+			}
+			data->dump_info.exception_dump_addr =
+				get_unaligned_le32(&tlv->val[0]);
+			data->dump_info.exception_dump_len =
+				get_unaligned_le32(&tlv->val[4]);
+			if (data->dump_info.exception_dump_len >
+			    BTINTEL_PCIE_REGION_MAX_SIZE) {
+				bt_dev_warn(data->hdev,
+					    "EXCEPTION_EVT_BUFFER size %u exceeds cap %u, ignoring",
+					    data->dump_info.exception_dump_len,
+					    BTINTEL_PCIE_REGION_MAX_SIZE);
+				data->dump_info.exception_dump_addr = 0;
+				data->dump_info.exception_dump_len = 0;
+			}
+			break;
+		case BTINTEL_PCIE_TLV_TYPE_DCCM_MEM_ADDRESS:
+			if (tlv_len < 8) {
+				bt_dev_err(data->hdev,
+					   "TLV %s too short: len=%u (need 8)",
+					   btintel_pcie_tlv_str(tlv->type),
+					   tlv_len);
+				break;
+			}
+			data->dump_info.dccm_addr_start =
+				get_unaligned_le32(&tlv->val[0]);
+			data->dump_info.dccm_size =
+				btintel_pcie_region_size(data->hdev, "DCCM",
+							 data->dump_info.dccm_addr_start,
+							 get_unaligned_le32(&tlv->val[4]));
+			if (!data->dump_info.dccm_size)
+				data->dump_info.dccm_addr_start = 0;
+			break;
+		case BTINTEL_PCIE_TLV_TYPE_SDS_MEM_ADDRESS: {
+			struct btintel_pcie_dump_mem_info *di = &data->dump_info;
+
+			/* hw_variant comes from cnvi_bt which is set during
+			 * setup. If mailbox fires before setup completes,
+			 * hw_variant is 0. Skip SDS parsing in that case.
+			 */
+			if (!hw_variant) {
+				bt_dev_dbg(data->hdev, "SDS TLV: skipped, hw_variant not yet known");
+				break;
+			}
+			if (tlv_len == 16 && hw_variant > BTINTEL_HWID_BZRI) {
+				di->sds_start_addr_start =
+					get_unaligned_le32(&tlv->val[0]);
+				di->sds_start_size =
+					btintel_pcie_region_size(data->hdev, "SDS",
+								 di->sds_start_addr_start,
+								 get_unaligned_le32(&tlv->val[4]));
+				di->sds_iosf_data_addr_start =
+					get_unaligned_le32(&tlv->val[8]);
+				di->sds_iosf_data_size =
+					btintel_pcie_region_size(data->hdev, "SDS_IOSF",
+								 di->sds_iosf_data_addr_start,
+								 get_unaligned_le32(&tlv->val[12]));
+			} else if (tlv_len == 24 &&
+				   (hw_variant == BTINTEL_HWID_BZRI ||
+				    hw_variant == BTINTEL_HWID_BZRIW)) {
+				di->sds_fixed_rom_addr_start =
+					get_unaligned_le32(&tlv->val[0]);
+				di->sds_fixed_rom_size =
+					btintel_pcie_region_size(data->hdev, "SDS_FIXED_ROM",
+								 di->sds_fixed_rom_addr_start,
+								 get_unaligned_le32(&tlv->val[4]));
+				di->sds_start_addr_start =
+					get_unaligned_le32(&tlv->val[8]);
+				di->sds_start_size =
+					btintel_pcie_region_size(data->hdev, "SDS",
+								 di->sds_start_addr_start,
+								 get_unaligned_le32(&tlv->val[12]));
+				di->sds_iosf_data_addr_start =
+					get_unaligned_le32(&tlv->val[16]);
+				di->sds_iosf_data_size =
+					btintel_pcie_region_size(data->hdev, "SDS_IOSF",
+								 di->sds_iosf_data_addr_start,
+								 get_unaligned_le32(&tlv->val[20]));
+			} else {
+				bt_dev_err(data->hdev,
+					   "SDS TLV: hw=0x%2.2x len=%u",
+					   hw_variant, tlv_len);
+			}
+			if (!di->sds_start_size)
+				di->sds_start_addr_start = 0;
+			if (!di->sds_iosf_data_size)
+				di->sds_iosf_data_addr_start = 0;
+			if (!di->sds_fixed_rom_size)
+				di->sds_fixed_rom_addr_start = 0;
+			break;
+		}
+		case BTINTEL_PCIE_TLV_TYPE_ECL_MEM_ADDRESS:
+			if (tlv_len < 8) {
+				bt_dev_err(data->hdev,
+					   "TLV %s too short: len=%u (need 8)",
+					   btintel_pcie_tlv_str(tlv->type),
+					   tlv_len);
+				break;
+			}
+			data->dump_info.ecl_addr_start =
+				get_unaligned_le32(&tlv->val[0]);
+			data->dump_info.ecl_size =
+				btintel_pcie_region_size(data->hdev, "ECL",
+							 data->dump_info.ecl_addr_start,
+							 get_unaligned_le32(&tlv->val[4]));
+			if (!data->dump_info.ecl_size)
+				data->dump_info.ecl_addr_start = 0;
+			break;
+		case BTINTEL_PCIE_TLV_TYPE_SMEM_ADDRESS:
+			if (tlv_len < 8) {
+				bt_dev_err(data->hdev,
+					   "TLV %s too short: len=%u (need 8)",
+					   btintel_pcie_tlv_str(tlv->type),
+					   tlv_len);
+				break;
+			}
+			data->dump_info.smem_addr_start =
+				get_unaligned_le32(&tlv->val[0]);
+			data->dump_info.smem_size =
+				btintel_pcie_region_size(data->hdev, "SMEM",
+							 data->dump_info.smem_addr_start,
+							 get_unaligned_le32(&tlv->val[4]));
+			if (!data->dump_info.smem_size)
+				data->dump_info.smem_addr_start = 0;
+			break;
+		default:
+			bt_dev_dbg(data->hdev, "Unknown TLV type: %u length: %u",
+				   tlv->type, tlv_len);
+			break;
+		}
+
+		/* Move to next TLV */
+		ptr += tlv_total;
+		remaining -= tlv_total;
+	}
+
+	bt_dev_info(data->hdev,
+		   "exception_dump: addr:0x%08x len:0x%08x",
+		   data->dump_info.exception_dump_addr,
+		   data->dump_info.exception_dump_len);
+	bt_dev_info(data->hdev,
+		   "dccm: start:0x%8.8x size:%u",
+		   data->dump_info.dccm_addr_start,
+		   data->dump_info.dccm_size);
+	bt_dev_info(data->hdev,
+		   "sds_fixed_rom: start:0x%8.8x size:%u",
+		   data->dump_info.sds_fixed_rom_addr_start,
+		   data->dump_info.sds_fixed_rom_size);
+	bt_dev_info(data->hdev,
+		   "sds: start:0x%8.8x size:%u",
+		   data->dump_info.sds_start_addr_start,
+		   data->dump_info.sds_start_size);
+	bt_dev_info(data->hdev,
+		   "sds_iosf: start:0x%8.8x size:%u",
+		   data->dump_info.sds_iosf_data_addr_start,
+		   data->dump_info.sds_iosf_data_size);
+	bt_dev_info(data->hdev,
+		   "ecl: start:0x%8.8x size:%u",
+		   data->dump_info.ecl_addr_start,
+		   data->dump_info.ecl_size);
+	bt_dev_info(data->hdev,
+		   "smem: start:0x%8.8x size:%u",
+		   data->dump_info.smem_addr_start,
+		   data->dump_info.smem_size);
+
+	vfree(buffer);
+	return 0;
+
+exit_on_error:
+	vfree(buffer);
+	return err;
+}
+
 static void btintel_pcie_msix_gp1_handler(struct btintel_pcie_data *data)
 {
-	bt_dev_err(data->hdev, "Received gp1 mailbox interrupt");
-	btintel_pcie_dump_debug_registers(data->hdev);
+	bool target_access = false;
+	u32 addr = 0, size = 0;
+
+	/* Read the Mail box status and registers */
+	data->mbox.mbox_status = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_STATUS_REG);
+	if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX1) {
+		data->mbox.mbox1 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_1_REG);
+		if (data->mbox.mbox1 ==
+		    BTINTEL_PCIE_BUILD_SPECIFIC_RESOURCES_MAPPING) {
+			bt_dev_info(data->hdev,
+				    "mailbox for target access");
+			target_access = true;
+		}
+	}
+
+	if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX2) {
+		data->mbox.mbox2 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_2_REG);
+		if (target_access)
+			addr = data->mbox.mbox2;
+	}
+
+	if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX3) {
+		data->mbox.mbox3 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_3_REG);
+		if (target_access)
+			size = data->mbox.mbox3;
+	}
+
+	if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4)
+		data->mbox.mbox4 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_4_REG);
+
+	bt_dev_dbg(data->hdev,
+		   "GP1: sts:0x%08x mb1:0x%08x mb2:0x%08x mb3:0x%08x mb4:0x%08x",
+		   data->mbox.mbox_status, data->mbox.mbox1,
+		   data->mbox.mbox2, data->mbox.mbox3,
+		   data->mbox.mbox4);
+
+	if (target_access &&
+	    !test_and_set_bit(BTINTEL_PCIE_MAIL_BOX_INTR,
+				   &data->flags)) {
+		/* Arm the mbox<->alive handshake */
+		clear_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags);
+		set_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING, &data->flags);
+		data->mbox_intr_ts = ktime_get();
+
+		bt_dev_info(data->hdev,
+			    "mbox interrupt received at %lld ns; queuing mbox_work",
+			    ktime_to_ns(data->mbox_intr_ts));
+
+		WRITE_ONCE(data->debug_table_addr, addr);
+		WRITE_ONCE(data->debug_table_size, size);
+		if (!queue_work(data->dump_workqueue,
+				&data->mbox_work)) {
+			clear_bit(BTINTEL_PCIE_MAIL_BOX_INTR, &data->flags);
+			clear_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING, &data->flags);
+		}
+	}
+
+	/* Mailbox is read, ack to FW */
+	btintel_pcie_set_reg_bits(data,
+				  BTINTEL_PCIE_CSR_IPC_DOORBELL_VEC_REG,
+				  BTINTEL_PCIE_CSR_DOORBELL_MBOX_READ_CONFIRM);
 }
 
 /* This function handles the MSI-X interrupt for gp0 cause (bit 0 in
@@ -1011,6 +1975,10 @@ static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data)
 	submit_rx = false;
 	signal_waitq = false;
 
+	/* Serialize the alive_intr_ctxt claim against set_dxstate()'s
+	 * register-based fallback, which can run concurrently.
+	 */
+	spin_lock(&data->irq_lock);
 	switch (data->alive_intr_ctxt) {
 	case BTINTEL_PCIE_ROM:
 		data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL;
@@ -1066,10 +2034,17 @@ static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data)
 			   data->alive_intr_ctxt);
 		break;
 	}
+	spin_unlock(&data->irq_lock);
 
 	if (submit_rx) {
 		btintel_pcie_reset_ia(data);
 		btintel_pcie_start_rx(data);
+
+		/* Complete the mbox<->alive handshake */
+		if (test_and_clear_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING, &data->flags)) {
+			set_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags);
+			wake_up(&data->mbox_parse_wait_q);
+		}
 	}
 
 	if (signal_waitq) {
@@ -1099,6 +2074,11 @@ static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data)
 
 	txq = &data->txq;
 
+	if (cr_hia >= txq->count) {
+		bt_dev_err(data->hdev, "TXQ: invalid cr_hia %u", cr_hia);
+		return;
+	}
+
 	while (cr_tia != cr_hia) {
 		data->tx_wait_done = true;
 		wake_up(&data->tx_wait_q);
@@ -1192,7 +2172,7 @@ static int btintel_pcie_recv_frame(struct btintel_pcie_data *data,
 {
 	int ret;
 	u8 pkt_type;
-	u16 plen;
+	u32 plen;
 	u32 pcie_pkt_type;
 	void *pdata;
 	struct hci_dev *hdev = data->hdev;
@@ -1293,7 +2273,8 @@ static int btintel_pcie_recv_frame(struct btintel_pcie_data *data,
 
 static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data)
 {
-	int len, err, offset, pending;
+	int err, offset, pending;
+	u32 len;
 	struct sk_buff *skb;
 	u8 *buf, prefix[64];
 	u32 addr, val;
@@ -1313,23 +2294,32 @@ static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data)
 		/* only from step B0 onwards */
 		if (INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvi_top) != 0x01)
 			return;
-		len = BTINTEL_PCIE_BLZR_HWEXP_SIZE; /* exception data length */
-		addr = BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR;
 		break;
 	case BTINTEL_CNVI_SCP:
-		len = BTINTEL_PCIE_SCP_HWEXP_SIZE;
-		addr = BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR;
-		break;
 	case BTINTEL_CNVI_SCP2:
+	case BTINTEL_CNVI_SCP2_C0:
 	case BTINTEL_CNVI_SCP2F:
-		len = BTINTEL_PCIE_SCP2_HWEXP_SIZE;
-		addr = BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR;
+	case BTINTEL_CNVI_DRACO:
 		break;
 	default:
 		bt_dev_err(data->hdev, "Unsupported cnvi 0x%8.8x", data->dmp_hdr.cnvi_top);
 		return;
 	}
 
+	len = data->dump_info.exception_dump_len;
+	addr = data->dump_info.exception_dump_addr;
+
+	if (!addr || len < sizeof(__le32) || len > SZ_4K) {
+		bt_dev_err(data->hdev, "Invalid exception address: 0x%8.8x or length: %u",
+			   addr, len);
+		return;
+	}
+
+	/* Ensure size is 4-byte aligned; btintel_pcie_read_device_mem()
+	 * reads device memory in 4-byte units.
+	 */
+	len = ALIGN_DOWN(len, 4);
+
 	buf = kzalloc(len, GFP_KERNEL);
 	if (!buf)
 		goto exit_on_error;
@@ -1431,11 +2421,8 @@ static int btintel_pcie_dump_fwtrigger_event(struct btintel_pcie_data *data)
 		goto exit_on_error;
 
 	evt = (void *)buf;
-	data->dmp_hdr.event_type = evt->event_type;
-	data->dmp_hdr.event_id = le16_to_cpu(evt->event_id);
-
 	bt_dev_dbg(data->hdev, "event type: 0x%2.2x event id: 0x%4.4x len: %u",
-		   data->dmp_hdr.event_type, data->dmp_hdr.event_id, len);
+		   evt->event_type, le16_to_cpu(evt->event_id), len);
 
 	skb = bt_skb_alloc(len, GFP_KERNEL);
 	if (!skb) {
@@ -1461,8 +2448,8 @@ static int btintel_pcie_dump_fwtrigger_event(struct btintel_pcie_data *data)
  *
  * Always queue this AFTER any companion event-reader work (hwexp /
  * fwtrigger) so that, on the ordered @dump_workqueue, the event reader
- * runs first and populates dmp_hdr.event_type / event_id before
- * dump_traces consumes them.
+ * runs first and the trigger metadata is populated before dump_traces
+ * consumes it.
  */
 static bool btintel_pcie_queue_coredump(struct btintel_pcie_data *data,
 					u16 trigger_reason)
@@ -1536,6 +2523,15 @@ static void btintel_pcie_coredump_worker(struct work_struct *work)
 	if (!data->hdev)
 		goto out;
 
+	/* When firmware routes debug traces to the WiFi DBGC, no host
+	 * DBGC buffers were allocated, so there is nothing to dump here.
+	 */
+	if (btintel_pcie_dbg_to_wifi(data)) {
+		bt_dev_info(data->hdev,
+			    "Skipping coredump: debug traces routed to WiFi DBGC");
+		goto out;
+	}
+
 	btintel_pcie_dump_traces(data->hdev);
 out:
 	/* Release guard last so a new trigger can run only after this
@@ -1582,6 +2578,20 @@ static void btintel_pcie_fwtrigger_worker(struct work_struct *work)
 	clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags);
 }
 
+static void btintel_pcie_mbox_worker(struct work_struct *work)
+{
+	struct btintel_pcie_data *data = container_of(work,
+					struct btintel_pcie_data, mbox_work);
+
+	if (!data->hdev)
+		goto out;
+
+	btintel_parse_mbox_tlv(data);
+out:
+	/* Release guard last; matches set in gp1 handler. */
+	clear_bit(BTINTEL_PCIE_MAIL_BOX_INTR, &data->flags);
+}
+
 static void btintel_pcie_rx_work(struct work_struct *work)
 {
 	struct btintel_pcie_data *data = container_of(work,
@@ -1650,6 +2660,11 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data)
 
 	rxq = &data->rxq;
 
+	if (cr_hia >= rxq->count) {
+		bt_dev_err(hdev, "RXQ: invalid cr_hia %u", cr_hia);
+		return;
+	}
+
 	/* The firmware sends multiple CD in a single MSI-X and it needs to
 	 * process all received CDs in this interrupt.
 	 */
@@ -1657,6 +2672,12 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data)
 		urbd1 = &rxq->urbd1s[cr_tia];
 		ipc_print_urbd1(data->hdev, urbd1, cr_tia);
 
+		if (urbd1->frbd_tag >= rxq->count) {
+			bt_dev_err(hdev, "RXQ: invalid frbd_tag %u",
+				   urbd1->frbd_tag);
+			return;
+		}
+
 		buf = &rxq->bufs[urbd1->frbd_tag];
 		if (!buf) {
 			bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d",
@@ -1861,6 +2882,13 @@ static int btintel_pcie_config_pcie(struct pci_dev *pdev,
 	if (IS_ERR(data->base_addr))
 		return PTR_ERR(data->base_addr);
 
+	/* Do shared hardware reset to ensure a clean start before
+	 * configuring interrupts.
+	 */
+	err = btintel_pcie_reset_bt(data);
+	if (err)
+		return err;
+
 	err = btintel_pcie_setup_irq(data);
 	if (err)
 		return err;
@@ -1897,9 +2925,22 @@ static void btintel_pcie_init_ci(struct btintel_pcie_data *data,
 	ci->num_urbdq1 = data->rxq.count;
 	ci->urbdq_db_vec = BTINTEL_PCIE_RXQ_NUM;
 
-	ci->dbg_output_mode = 0x01;
-	ci->dbgc_addr = data->dbgc.frag_p_addr;
-	ci->dbgc_size = data->dbgc.frag_size;
+	ci->dbg_output_mode = btintel_pcie_dbg_to_wifi(data) ?
+			      BTINTEL_PCIE_WIFI_DBGC : BTINTEL_PCIE_DRAM;
+	if (btintel_pcie_dbg_to_wifi(data)) {
+		/* Firmware forwards debug traces to the WiFi DBGC, so no
+		 * host DBGC buffer is needed; leave dbgc_addr/size as 0.
+		 */
+		ci->dbgc_addr = 0;
+		ci->dbgc_size = 0;
+	} else if (btintel_pcie_is_mdbgc_supported(data)) {
+		ci->dbgc_addr = data->mdbgc.frag_p_addr;
+		ci->dbgc_size = data->mdbgc.frag_size;
+	} else {
+		ci->dbgc_addr = data->dbgc.frag_p_addr;
+		ci->dbgc_size = data->dbgc.frag_size;
+	}
+
 	ci->dbg_preset = 0x00;
 }
 
@@ -2127,7 +3168,16 @@ static int btintel_pcie_alloc(struct btintel_pcie_data *data)
 	v_addr += ci_size;
 
 	/* Setup data buffers for dbgc */
-	err = btintel_pcie_setup_dbgc(data);
+	if (btintel_pcie_dbg_to_wifi(data)) {
+		/* Firmware routes traces to the WiFi DBGC; skip host DBGC
+		 * buffer allocation entirely.
+		 */
+		err = 0;
+	} else if (btintel_pcie_is_mdbgc_supported(data)) {
+		err = btintel_pcie_setup_mdbgc(data);
+	} else {
+		err = btintel_pcie_setup_dbgc(data);
+	}
 	if (err)
 		goto exit_error_txq;
 
@@ -2203,6 +3253,16 @@ static int btintel_pcie_send_frame(struct hci_dev *hdev,
 	if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags))
 		return -ENODEV;
 
+	/* Account for the 4-byte PCIe type header prepended before the
+	 * DMA copy.  Written as a subtraction to avoid wrap-around on
+	 * attacker-controlled skb->len.
+	 */
+	if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) {
+		bt_dev_err(hdev, "Packet too large: %u > %u", skb->len,
+			   BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN);
+		return -EMSGSIZE;
+	}
+
 	/* Due to the fw limitation, the type header of the packet should be
 	 * 4 bytes unlike 1 byte for UART. In UART, the firmware can read
 	 * the first byte to get the packet type and redirect the rest of data
@@ -2391,6 +3451,7 @@ static int btintel_pcie_setup_internal(struct hci_dev *hdev)
 		goto exit_error;
 	}
 
+	data->dmp_hdr.cnvi_bt = ver_tlv.cnvi_bt;
 	switch (INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)) {
 	case 0x37:
 		break;
@@ -2414,6 +3475,7 @@ static int btintel_pcie_setup_internal(struct hci_dev *hdev)
 	case 0x20:	/* ScP2 */
 	case 0x21:	/* ScP2 F */
 	case 0x22:	/* BzrIW */
+	case 0x23:	/* DrC */
 		/* Display version information of TLV type */
 		btintel_version_info_tlv(hdev, &ver_tlv);
 
@@ -2443,10 +3505,9 @@ static int btintel_pcie_setup_internal(struct hci_dev *hdev)
 	data->dmp_hdr.fw_timestamp = ver_tlv.timestamp;
 	data->dmp_hdr.fw_build_type = ver_tlv.build_type;
 	data->dmp_hdr.fw_build_num = ver_tlv.build_num;
-	data->dmp_hdr.cnvi_bt = ver_tlv.cnvi_bt;
 
 	if (ver_tlv.img_type == 0x02 || ver_tlv.img_type == 0x03)
-		data->dmp_hdr.fw_git_sha1 = ver_tlv.git_sha1;
+		data->dmp_hdr.fw_sha = ver_tlv.git_sha1;
 
 	err = btintel_pcie_get_debug_info_addr(hdev);
 	if (err)
@@ -2726,6 +3787,7 @@ static void btintel_pcie_reset_work(struct work_struct *wk)
 	disable_work_sync(&data->coredump_work);
 	disable_work_sync(&data->hwexp_work);
 	disable_work_sync(&data->fwtrigger_work);
+	disable_work_sync(&data->mbox_work);
 
 	bt_dev_dbg(data->hdev, "Release bluetooth interface");
 
@@ -2750,6 +3812,7 @@ static void btintel_pcie_reset_work(struct work_struct *wk)
 		enable_work(&data->coredump_work);
 		enable_work(&data->hwexp_work);
 		enable_work(&data->fwtrigger_work);
+		enable_work(&data->mbox_work);
 	}
 
 out:
@@ -3007,6 +4070,8 @@ static int btintel_pcie_probe(struct pci_dev *pdev,
 	init_waitqueue_head(&data->tx_wait_q);
 	data->tx_wait_done = false;
 
+	init_waitqueue_head(&data->mbox_parse_wait_q);
+
 	data->workqueue = alloc_ordered_workqueue(KBUILD_MODNAME, WQ_HIGHPRI);
 	if (!data->workqueue)
 		return -ENOMEM;
@@ -3023,9 +4088,11 @@ static int btintel_pcie_probe(struct pci_dev *pdev,
 	INIT_WORK(&data->coredump_work, btintel_pcie_coredump_worker);
 	INIT_WORK(&data->hwexp_work, btintel_pcie_hwexp_worker);
 	INIT_WORK(&data->fwtrigger_work, btintel_pcie_fwtrigger_worker);
+	INIT_WORK(&data->mbox_work, btintel_pcie_mbox_worker);
 
 	data->boot_stage_cache = 0x00;
 	data->img_resp_cache = 0x00;
+	data->dbg_path_cache = BTINTEL_PCIE_WIFI_DBGC;
 	/* FLR can be invoked by echoing to debugfs path, so explicitly
 	 * initialized
 	 */
@@ -3092,6 +4159,7 @@ static void btintel_pcie_remove(struct pci_dev *pdev)
 	disable_work_sync(&data->coredump_work);
 	disable_work_sync(&data->hwexp_work);
 	disable_work_sync(&data->fwtrigger_work);
+	disable_work_sync(&data->mbox_work);
 
 	/* Cancel pending reset work. Skip only when remove() is called from
 	 * within the reset work itself (PLDR device_reprobe path) to avoid
@@ -3145,7 +4213,8 @@ static void btintel_pcie_coredump(struct device *dev)
 
 static int btintel_pcie_set_dxstate(struct btintel_pcie_data *data, u32 dxstate)
 {
-	int retry = 0, status;
+	int retry = 0, err;
+	long status;
 	u32 dx_intr_timeout_ms = 200;
 
 	do {
@@ -3156,28 +4225,75 @@ static int btintel_pcie_set_dxstate(struct btintel_pcie_data *data, u32 dxstate)
 		status = wait_event_timeout(data->gp0_wait_q, data->gp0_received,
 			msecs_to_jiffies(dx_intr_timeout_ms));
 
-		if (status)
-			return 0;
-
-		bt_dev_warn(data->hdev,
-			   "Timeout (%u ms) on alive interrupt for D%d entry, retry count %d",
-			   dx_intr_timeout_ms, dxstate, retry);
+		if (!status) {
+			bt_dev_warn(data->hdev,
+				    "Timeout (%u ms) on alive interrupt for D%d entry, retry count %d",
+				    dx_intr_timeout_ms, dxstate, retry);
 
-		/* clear gp0 cause */
-		btintel_pcie_clr_reg_bits(data,
-					  BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES,
-					  BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0);
+			/* MSIX_HW_INT_CAUSES is W1C. Write only GP0 so other
+			 * pending causes are not acknowledged here.
+			 */
+			btintel_pcie_wr_reg32(data,
+					      BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES,
+					      BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0);
+		}
 
-		/* A hardware bug may cause the alive interrupt to be missed.
-		 * Check if the controller reached the expected state and retry
-		 * the operation only if it hasn't.
+		/* gp0_received is set at the top of the handler, before the switch on
+		 * alive_intr_ctxt. Error and lockdown are filtered out above it, but a
+		 * matched case can still complete without doing anything - D3 breaks
+		 * unchanged while the controller has not reached D0 - and a hardware
+		 * bug may drop the interrupt outright. Either way the flag says a gp0
+		 * was handled, not that the transition completed, and only the register
+		 * knows. Refresh the cache here and retry only if the state check still
+		 * fails.
 		 */
+		data->boot_stage_cache = btintel_pcie_rd_reg32(data,
+							       BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
 		if (dxstate == BTINTEL_PCIE_STATE_D0) {
-			if (btintel_pcie_in_d0(data))
+			if (btintel_pcie_in_d0(data)) {
+				bool armed;
+
+				/* Claim the transition under irq_lock so a
+				 * concurrent gp0 handler run cannot also arm RX.
+				 */
+				spin_lock(&data->irq_lock);
+				armed = data->alive_intr_ctxt == BTINTEL_PCIE_D0;
+				if (!armed)
+					data->alive_intr_ctxt = BTINTEL_PCIE_D0;
+				spin_unlock(&data->irq_lock);
+
+				if (armed)
+					return 0;
+
+				/* Do what the handler's D3 -> D0 branch
+				 * would have done, unless it already has.
+				 */
+				btintel_pcie_reset_ia(data);
+				err = btintel_pcie_start_rx(data);
+				if (err)
+					return err;
+
+				/* Complete the mbox<->alive handshake */
+				if (test_and_clear_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING,
+						       &data->flags)) {
+					set_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags);
+					wake_up(&data->mbox_parse_wait_q);
+				}
+
 				return 0;
+			}
 		} else {
-			if (btintel_pcie_in_d3(data))
+			if (btintel_pcie_in_d3(data)) {
+				/* Do what the handler's D0 -> D3 branch
+				 * would have done, unless it already has.
+				 */
+				spin_lock(&data->irq_lock);
+				if (data->alive_intr_ctxt != BTINTEL_PCIE_D3)
+					data->alive_intr_ctxt = BTINTEL_PCIE_D3;
+				spin_unlock(&data->irq_lock);
+
 				return 0;
+			}
 		}
 
 	} while (++retry < BTINTEL_PCIE_DX_TRANSITION_MAX_RETRIES);
@@ -3195,18 +4311,23 @@ static int btintel_pcie_suspend_late(struct device *dev, pm_message_t mesg)
 
 	data = pci_get_drvdata(pdev);
 
-	dxstate = (mesg.event == PM_EVENT_SUSPEND ?
-		   BTINTEL_PCIE_STATE_D3_HOT : BTINTEL_PCIE_STATE_D3_COLD);
-
-	data->pm_sx_event = mesg.event;
+	/* S0ix (s2idle) uses D3_HOT; S3, freeze and hibernate use D3_COLD. */
+	if (mesg.event == PM_EVENT_SUSPEND &&
+	    pm_suspend_target_state == PM_SUSPEND_TO_IDLE)
+		dxstate = BTINTEL_PCIE_STATE_D3_HOT;
+	else
+		dxstate = BTINTEL_PCIE_STATE_D3_COLD;
 
 	start = ktime_get();
 
 	/* Refer: 6.4.11.7 -> Platform power management */
 	err = btintel_pcie_set_dxstate(data, dxstate);
 
-	if (err)
+	if (err) {
+		bt_dev_err(data->hdev, "Failed to set dxstate:%u (%d)",
+			   dxstate, err);
 		return err;
+	}
 
 	bt_dev_dbg(data->hdev,
 		   "device entered into d3 state from d0 in %lld us",
@@ -3229,7 +4350,7 @@ static int btintel_pcie_freeze(struct device *dev)
 	return btintel_pcie_suspend_late(dev, PMSG_FREEZE);
 }
 
-static int btintel_pcie_resume(struct device *dev)
+static int btintel_pcie_resume_event(struct device *dev, pm_message_t mesg)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct btintel_pcie_data *data;
@@ -3237,19 +4358,15 @@ static int btintel_pcie_resume(struct device *dev)
 	int err;
 
 	data = pci_get_drvdata(pdev);
-	data->gp0_received = false;
 
 	start = ktime_get();
 
-	/* When the system enters S4 (hibernate) mode, bluetooth device loses
-	 * power, which results in the erasure of its loaded firmware.
-	 * Consequently, function level reset (flr) is required on system
-	 * resume to bring the controller back into an operational state by
-	 * initiating a new firmware download.
+	/* S3 and S4 may cut power, erasing the firmware. Force FLR to recover
+	 * instead of a normal D0 transition.
 	 */
-
-	if (data->pm_sx_event == PM_EVENT_FREEZE ||
-	    data->pm_sx_event == PM_EVENT_HIBERNATE) {
+	if (mesg.event == PM_EVENT_RESTORE ||
+	    (mesg.event == PM_EVENT_RESUME &&
+	     pm_suspend_target_state == PM_SUSPEND_MEM)) {
 		set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
 		btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR);
 		return 0;
@@ -3258,7 +4375,9 @@ static int btintel_pcie_resume(struct device *dev)
 	/* Refer: 6.4.11.7 -> Platform power management */
 	err = btintel_pcie_set_dxstate(data, BTINTEL_PCIE_STATE_D0);
 
-	if (err == 0) {
+	if (err) {
+		bt_dev_err(data->hdev, "Failed to set D0 state (%d)", err);
+	} else {
 		bt_dev_dbg(data->hdev,
 			   "device entered into d0 state from d3 in %lld us",
 			   ktime_to_us(ktime_get() - start));
@@ -3283,13 +4402,28 @@ static int btintel_pcie_resume(struct device *dev)
 	return err;
 }
 
+static int btintel_pcie_resume(struct device *dev)
+{
+	return btintel_pcie_resume_event(dev, PMSG_RESUME);
+}
+
+static int btintel_pcie_restore(struct device *dev)
+{
+	return btintel_pcie_resume_event(dev, PMSG_RESTORE);
+}
+
+static int btintel_pcie_thaw(struct device *dev)
+{
+	return btintel_pcie_resume_event(dev, PMSG_THAW);
+}
+
 static const struct dev_pm_ops btintel_pcie_pm_ops = {
 	.suspend = btintel_pcie_suspend,
 	.resume = btintel_pcie_resume,
 	.freeze = btintel_pcie_freeze,
-	.thaw = btintel_pcie_resume,
+	.thaw = btintel_pcie_thaw,
 	.poweroff = btintel_pcie_hibernate,
-	.restore = btintel_pcie_resume,
+	.restore = btintel_pcie_restore,
 };
 
 static struct pci_driver btintel_pcie_driver = {
diff --git a/drivers/bluetooth/btintel_pcie.h b/drivers/bluetooth/btintel_pcie.h
index 749369b24031ce..3030b4e8b75039 100644
--- a/drivers/bluetooth/btintel_pcie.h
+++ b/drivers/bluetooth/btintel_pcie.h
@@ -18,6 +18,7 @@
 #define BTINTEL_PCIE_CSR_CI_ADDR_LSB_REG	(BTINTEL_PCIE_CSR_BASE + 0x118)
 #define BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG	(BTINTEL_PCIE_CSR_BASE + 0x11C)
 #define BTINTEL_PCIE_CSR_IMG_RESPONSE_REG	(BTINTEL_PCIE_CSR_BASE + 0x12C)
+#define BTINTEL_PCIE_CSR_IPC_DOORBELL_VEC_REG	(BTINTEL_PCIE_CSR_BASE + 0x130)
 #define BTINTEL_PCIE_CSR_MBOX_1_REG		(BTINTEL_PCIE_CSR_BASE + 0x170)
 #define BTINTEL_PCIE_CSR_MBOX_2_REG		(BTINTEL_PCIE_CSR_BASE + 0x174)
 #define BTINTEL_PCIE_CSR_MBOX_3_REG		(BTINTEL_PCIE_CSR_BASE + 0x178)
@@ -50,8 +51,11 @@
 #define BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_HALTED	(BIT(14))
 #define BTINTEL_PCIE_CSR_BOOT_STAGE_MAC_ACCESS_ON	(BIT(16))
 #define BTINTEL_PCIE_CSR_BOOT_STAGE_ALIVE		(BIT(23))
+/* Reflects live D-state. Updated by hardware on every D-state transition. */
 #define BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY	(BIT(24))
 
+#define BTINTEL_PCIE_CSR_DOORBELL_MBOX_READ_CONFIRM	(BIT(4))
+
 /* Registers for MSI-X */
 #define BTINTEL_PCIE_CSR_MSIX_BASE		(0x2000)
 #define BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES	(BTINTEL_PCIE_CSR_MSIX_BASE + 0x0800)
@@ -71,11 +75,16 @@
 #define BTINTEL_PCIE_DBGC_BASE_ADDR_SCP			(0xf0d5d500)
 #define BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS_SCP	(BTINTEL_PCIE_DBGC_BASE_ADDR_SCP + 0x1C)
 #define BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND_SCP		(BTINTEL_PCIE_DBGC_BASE_ADDR_SCP + 0x2C)
+#define BTINTEL_PCIE_DBGC_ALLOCATION_OFFSET		0x100
 
+/* Upper bound for device-supplied debug-region sizes. */
+#define BTINTEL_PCIE_REGION_MAX_SIZE		(1 * 1024 * 1024)
 #define BTINTEL_PCIE_DBG_IDX_BIT_MASK		0x0F
 #define BTINTEL_PCIE_DBGC_DBG_BUF_IDX(data)	(((data) >> 24) & BTINTEL_PCIE_DBG_IDX_BIT_MASK)
 #define BTINTEL_PCIE_DBG_OFFSET_BIT_MASK	0xFFFFFF
 
+#define BTINTEL_PCIE_TARGET_ACCESS_FRAG_OFFSET	4
+
 /* The DRAM buffer count, each buffer size, and
  * fragment buffer size
  */
@@ -91,6 +100,31 @@
 /* Num of alloc Dbg buff (4) + (LSB(4), MSB(4), Size(4)) for each buffer */
 #define BTINTEL_PCIE_DBGC_FRAG_PAYLOAD_SIZE	196
 
+/* dbg_output_mode values for the context info.
+ *   BTINTEL_PCIE_DRAM: firmware writes traces to host DRAM DBGC buffers.
+ *   BTINTEL_PCIE_WIFI_DBGC: firmware forwards traces to the WiFi DBGC; the
+ *   host does NOT need to allocate DBGC fragment/data buffers and must
+ *   publish dbgc_addr/size as 0 in the context info.
+ *
+ * Encoding of BTINTEL_PCIE_WIFI_DBGC (0x06):
+ *   Bit[0] DBGC O/P  : 0 = SRAM (don't care, DBGI selected)
+ *   Bit[1] DBGC I/P  : 1 = DBGI
+ *   Bits[2:3] DBGI O/P : 01 = WiFi DBGC
+ */
+#define BTINTEL_PCIE_DRAM	0x01
+#define BTINTEL_PCIE_FW_MON_MODE_DRAM	0x02
+#define BTINTEL_PCIE_WIFI_DBGC	0x06
+
+#define BTINTEL_PCIE_MDBGC_FRAG_VERSION		2
+
+#define BTINTEL_PCIE_MDBGC_ALLOCATIONID_1		0
+#define BTINTEL_PCIE_MDBGC_ALLOCATIONID_2		1
+#define BTINTEL_PCIE_MDBGC_ALLOCATIONID_3		2
+
+#define BTINTEL_PCIE_DEVICE_ID_NVL_S_SCP2	0x6E74
+#define BTINTEL_PCIE_DEVICE_ID_NVL_Hx_SCP2	0xD346
+#define BTINTEL_PCIE_DEVICE_ID_PTL_FMP2		0xE476
+
 /* Causes for the FH register interrupts */
 enum msix_fh_int_causes {
 	BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0	= BIT(0),	/* cause 0 */
@@ -121,7 +155,10 @@ enum {
 	BTINTEL_PCIE_COREDUMP_INPROGRESS,
 	BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS,
 	BTINTEL_PCIE_RECOVERY_IN_PROGRESS,
-	BTINTEL_PCIE_SETUP_DONE
+	BTINTEL_PCIE_SETUP_DONE,
+	BTINTEL_PCIE_MAIL_BOX_INTR,
+	BTINTEL_PCIE_MBOX_PARSE_PENDING,
+	BTINTEL_PCIE_MBOX_PARSE_READY
 };
 
 enum btintel_pcie_tlv_type {
@@ -153,6 +190,14 @@ enum btintel_pcie_reset_type {
 	BTINTEL_PCIE_IOSF_PRR_PLDR = 1,
 };
 
+enum btintel_pcie_mbox_msg {
+	BTINTEL_PCIE_NO_USE = 0,
+	BTINTEL_PCIE_TOP_SILENT_RESET,
+	BTINTEL_PCIE_SB_AUDIO_DEVICE_REPORT,
+	BTINTEL_PCIE_BUILD_SPECIFIC_RESOURCES_MAPPING,
+	BTINTEL_PCIE_LAST_MESSAGE = 4095
+};
+
 #define BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE	BIT(7)
 
 /* Minimum and Maximum number of MSI-X Vector
@@ -166,6 +211,7 @@ enum btintel_pcie_reset_type {
 
 /* Default interrupt timeout in msec */
 #define BTINTEL_DEFAULT_INTR_TIMEOUT_MS	3000
+#define BTINTEL_PCIE_MBOX_INTR_TIMEOUT_MS	500
 
 #define BTINTEL_PCIE_DX_TRANSITION_MAX_RETRIES	3
 
@@ -198,6 +244,12 @@ enum {
 /* RBD buffer size mapping */
 #define BTINTEL_PCIE_RBD_SIZE_4K	0x04
 
+#define BTINTEL_PCIE_TLV_TYPE_EXCEPTION_DUMP_ADDRESS  0x04
+#define BTINTEL_PCIE_TLV_TYPE_DCCM_MEM_ADDRESS        0x05
+#define BTINTEL_PCIE_TLV_TYPE_SDS_MEM_ADDRESS         0x06
+#define BTINTEL_PCIE_TLV_TYPE_ECL_MEM_ADDRESS         0x07
+#define BTINTEL_PCIE_TLV_TYPE_SMEM_ADDRESS	      0x08
+
 /*
  * Struct for Context Information (v2)
  *
@@ -424,6 +476,50 @@ struct btintel_pcie_dbgc {
 	struct data_buf *bufs;
 };
 
+struct btintel_pcie_mdbgc {
+	u32		count;
+
+	void		*frag_v_addr;
+	dma_addr_t	frag_p_addr;
+	u32		frag_size;
+
+	dma_addr_t	buf1_p_addr;
+	void		*buf1_v_addr;
+	dma_addr_t	buf2_p_addr;
+	void		*buf2_v_addr;
+	dma_addr_t	buf3_p_addr;
+	void		*buf3_v_addr;
+	struct data_buf *buf1;
+	struct data_buf *buf2;
+	struct data_buf *buf3;
+};
+
+struct btintel_pcie_dump_mem_info {
+	u32	exception_dump_addr;
+	u32	exception_dump_len;
+	u32	dccm_addr_start;
+	u32	dccm_size;
+	u32	sds_fixed_rom_addr_start;
+	u32	sds_fixed_rom_size;
+	u32	sds_start_addr_start;
+	u32	sds_start_size;
+	u32	sds_iosf_data_addr_start;
+	u32	sds_iosf_data_size;
+	u32	ecl_addr_start;
+	u32	ecl_size;
+	u32	smem_addr_start;
+	u32	smem_size;
+};
+
+struct btintel_pcie_mbox {
+	u32 mbox_flags;
+	u32 mbox_status;
+	u32 mbox1;
+	u32 mbox2;
+	u32 mbox3;
+	u32 mbox4;
+};
+
 struct btintel_pcie_dump_header {
 	const char	*driver_name;
 	u32		cnvi_top;
@@ -431,16 +527,153 @@ struct btintel_pcie_dump_header {
 	u16		fw_timestamp;
 	u8		fw_build_type;
 	u32		fw_build_num;
-	u32		fw_git_sha1;
+	u32		fw_sha;
 	u32		cnvi_bt;
 	u32		write_ptr;
 	u32		wrap_ctr;
 	u16		trigger_reason;
 	int		state;
-	u8		event_type;
-	u16		event_id;
 };
 
+/* Per-fragment range descriptor for dump regions.
+ * Binary-compatible with iwl_fw_ini_error_dump_range.
+ */
+struct btintel_pcie_dump_range {
+	__le32	range_data_size;
+	union {
+		__le32	internal_base_addr;
+		__le64	dram_base_addr;
+		__le32	page_num;
+	};
+	__le32	data[];
+} __packed;
+
+/*
+ * INI region types for ini_dump_data.type field.
+ * The unified decoder dispatches parsing logic based on these values.
+ */
+#define BTINTEL_PCIE_INI_REGION_INTERNAL_BUFFER	2
+#define BTINTEL_PCIE_INI_REGION_DRAM_BUFFER	3
+#define BTINTEL_PCIE_INI_REGION_DEVICE_MEMORY	9
+
+/* INI region IDs - used in dump header region_id field and regions_mask */
+#define BTINTEL_PCIE_INI_ID_EXCEPTION_EVT	7
+#define BTINTEL_PCIE_INI_ID_SMEM		15
+#define BTINTEL_PCIE_INI_ID_DCCM		39
+#define BTINTEL_PCIE_INI_ID_SDS			40
+#define BTINTEL_PCIE_INI_ID_SDS_IOSF		41
+#define BTINTEL_PCIE_INI_ID_ECL			42
+#define BTINTEL_PCIE_INI_ID_DRAM_MONITOR3	61
+#define BTINTEL_PCIE_INI_ID_DRAM_MONITOR2	62
+#define BTINTEL_PCIE_INI_ID_DRAM_MONITOR1	63
+
+/*
+ * INI (Intel INI debug infrastructure) style dump data TLV - wraps each dump
+ * region. INI is the iwlwifi firmware debug format used by the unified decoder.
+ * Compatible with iwl_fw_ini_error_dump_data.
+ */
+struct btintel_pcie_ini_dump_data {
+	u8	type;
+	u8	sub_type;
+	u8	sub_type_ver;
+	u8	reserved;
+	__le32	len;
+	u8	data[];
+} __packed;
+
+/*
+ * INI-style region dump header.
+ * Compatible with iwl_fw_ini_error_dump_header.
+ */
+#define BTINTEL_PCIE_INI_MAX_NAME	32
+#define BTINTEL_PCIE_INI_DUMP_VER	1
+
+struct btintel_pcie_ini_dump_header {
+	__le32	version;
+	__le32	region_id;
+	__le32	num_of_ranges;
+	__le32	name_len;
+	u8	name[BTINTEL_PCIE_INI_MAX_NAME];
+};
+
+/*
+ * INI-style monitor dump - region header + monitor state.
+ * Compatible with iwl_fw_ini_monitor_dump.
+ */
+struct btintel_pcie_ini_monitor_dump {
+	struct btintel_pcie_ini_dump_header header;
+	__le32	write_ptr;
+	__le32	cycle_cnt;
+	__le32	cur_frag;
+	u8	data[];
+} __packed;
+
+/* Linked list entry for modular dump collection */
+struct btintel_pcie_dump_entry {
+	struct list_head	list;
+	u32			size;
+	u8			data[];
+};
+
+/* File-level header for coredump output.
+ * Compatible with iwl_fw_ini_dump_file_hdr.
+ * Uses IWL_FW_INI_ERROR_DUMP_BARKER (0x14789633) for decoder compatibility.
+ */
+#define BTINTEL_PCIE_INI_ERROR_DUMP_BARKER	0x14789633
+
+struct btintel_pcie_dump_file_hdr {
+	__le32	barker;
+	__le32	file_len;
+} __packed;
+
+/*
+ * Legacy-style dump data wrapper for dump info TLV.
+ * Compatible with iwl_fw_error_dump_data.
+ * Used only for the dump info entry (type=BTINTEL_PCIE_INI_DUMP_INFO_TYPE).
+ */
+struct btintel_pcie_error_dump_data {
+	__le32	type;
+	__le32	len;
+	u8	data[];
+} __packed;
+
+/* Use bit 31 as dump info type, matching IWL_INI_DUMP_INFO_TYPE */
+#define BTINTEL_PCIE_INI_DUMP_INFO_TYPE	BIT(31)
+
+/* Time point values matching iwl_fw_ini_time_point for fwdump parser */
+#define BTINTEL_PCIE_TIME_POINT_FW_ASSERT	4
+#define BTINTEL_PCIE_TIME_POINT_USER_TRIGGER	9
+
+/*
+ * Dump info struct - single TLV containing all metadata.
+ * Compatible with iwl_fw_ini_dump_info.
+ * Packs all device/firmware info that the decoder needs.
+ */
+struct btintel_pcie_ini_dump_info {
+	__le32	version;
+	__le32	time_point;
+	__le32	trigger_reason;
+	__le32	external_cfg_state;
+	__le32	ver_type;
+	__le32	ver_subtype;
+	__le32	hw_step;
+	__le32	hw_type;
+	__le32	rf_id_flavor;
+	__le32	rf_id_dash;
+	__le32	rf_id_step;
+	__le32	rf_id_type;
+	__le32	lmac_major;
+	__le32	lmac_minor;
+	__le32	umac_major;
+	__le32	umac_minor;
+	__le32	fw_mon_mode;
+	__le64	regions_mask;
+	__le32	build_tag_len;
+	u8	build_tag[64];
+	__le32	num_of_cfg_names;
+	/* no cfg_names for BT - keep zero-length */
+} __packed;
+
 /* struct btintel_pcie_data
  * @pdev: pci device
  * @hdev: hdev device
@@ -456,6 +689,7 @@ struct btintel_pcie_dump_header {
  * @hw_init_mask: initial unmaksed hw causes
  * @boot_stage_cache: cached value of boot stage register
  * @img_resp_cache: cached value of image response register
+ * @dbg_path_cache: cached debug output routing mode (BT DRAM or WiFi DBGC)
  * @cnvi: CNVi register value
  * @cnvr: CNVr register value
  * @gp0_received: condition for gp0 interrupt
@@ -479,7 +713,6 @@ struct btintel_pcie_dump_header {
  * @txq: TX Queue struct
  * @rxq: RX Queue struct
  * @alive_intr_ctxt: Alive interrupt context
- * @pm_sx_event: PM event on which system got suspended
  */
 struct btintel_pcie_data {
 	struct pci_dev	*pdev;
@@ -503,6 +736,7 @@ struct btintel_pcie_data {
 
 	u32	boot_stage_cache;
 	u32	img_resp_cache;
+	u32	dbg_path_cache;
 
 	u32	cnvi;
 	u32	cnvr;
@@ -522,6 +756,7 @@ struct btintel_pcie_data {
 	struct work_struct	coredump_work;
 	struct work_struct	hwexp_work;
 	struct work_struct	fwtrigger_work;
+	struct work_struct	mbox_work;
 
 	struct dma_pool	*dma_pool;
 	dma_addr_t	dma_p_addr;
@@ -535,10 +770,19 @@ struct btintel_pcie_data {
 	u32	alive_intr_ctxt;
 	enum btintel_pcie_reset_type	reset_type;
 	struct btintel_pcie_dbgc	dbgc;
+	struct btintel_pcie_mdbgc	mdbgc;
 	struct btintel_pcie_dump_header dmp_hdr;
-	u8	pm_sx_event;
 	u32	debug_evt_addr;
 	u32	debug_evt_size;
+	dma_addr_t	debug_table_addr;
+	u32	debug_table_size;
+	struct btintel_pcie_dump_mem_info	dump_info;
+	struct btintel_pcie_mbox	mbox;
+
+	/* Wait queue for mbox_worker to wait for GP0 alive interrupt */
+	wait_queue_head_t	mbox_parse_wait_q;
+	/* Timestamp captured in GP1 handler when mbox interrupt is received */
+	ktime_t	mbox_intr_ts;
 };
 
 static inline u32 btintel_pcie_rd_reg32(struct btintel_pcie_data *data,
@@ -585,4 +829,3 @@ static inline u32 btintel_pcie_rd_dev_mem(struct btintel_pcie_data *data,
 	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_PRPH_DEV_ADDR_REG, addr);
 	return btintel_pcie_rd_reg32(data, BTINTEL_PCIE_PRPH_DEV_RD_REG);
 }
-
diff --git a/drivers/bluetooth/btmrvl_main.c b/drivers/bluetooth/btmrvl_main.c
index e25930351f643c..d449fbdde46406 100644
--- a/drivers/bluetooth/btmrvl_main.c
+++ b/drivers/bluetooth/btmrvl_main.c
@@ -87,8 +87,7 @@ int btmrvl_process_event(struct btmrvl_private *priv, struct sk_buff *skb)
 	event = (struct btmrvl_event *) skb->data;
 	if (event->ec != 0xff) {
 		BT_DBG("Not Marvell Event=%x", event->ec);
-		ret = -EINVAL;
-		goto exit;
+		return -EINVAL;
 	}
 
 	switch (event->data[0]) {
@@ -165,7 +164,6 @@ int btmrvl_process_event(struct btmrvl_private *priv, struct sk_buff *skb)
 		break;
 	}
 
-exit:
 	if (!ret)
 		kfree_skb(skb);
 
diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c
index 7ea8bcd8a7ecaa..115a11fcb25483 100644
--- a/drivers/bluetooth/btmtk.c
+++ b/drivers/bluetooth/btmtk.c
@@ -6,6 +6,7 @@
 #include <linux/firmware.h>
 #include <linux/usb.h>
 #include <linux/iopoll.h>
+#include <linux/overflow.h>
 #include <linux/unaligned.h>
 
 #include <net/bluetooth/bluetooth.h>
@@ -59,6 +60,67 @@ struct btmtk_section_map {
 	};
 } __packed;
 
+/* Reject a firmware image whose header, global descriptor or section map
+ * array does not fit in the file, and hand back the validated section
+ * count, so that the section map walks stay inside the firmware buffer.
+ */
+static int btmtk_fw_validate_layout(struct hci_dev *hdev,
+				    const struct firmware *fw,
+				    u32 *section_num)
+{
+	const struct btmtk_global_desc *globaldesc;
+	size_t need;
+
+	if (fw->size < MTK_FW_ROM_PATCH_HEADER_SIZE + MTK_FW_ROM_PATCH_GD_SIZE) {
+		bt_dev_err(hdev, "Firmware too small: %zu bytes, need at least %u",
+			   fw->size,
+			   MTK_FW_ROM_PATCH_HEADER_SIZE + MTK_FW_ROM_PATCH_GD_SIZE);
+		return -EINVAL;
+	}
+
+	globaldesc = (const struct btmtk_global_desc *)(fw->data +
+			MTK_FW_ROM_PATCH_HEADER_SIZE);
+	*section_num = le32_to_cpu(globaldesc->section_num);
+
+	if (!*section_num) {
+		bt_dev_err(hdev, "Firmware declares no sections");
+		return -EINVAL;
+	}
+
+	if (check_mul_overflow((size_t)MTK_FW_ROM_PATCH_SEC_MAP_SIZE,
+			       (size_t)*section_num, &need) ||
+	    check_add_overflow(need,
+			       (size_t)(MTK_FW_ROM_PATCH_HEADER_SIZE +
+					MTK_FW_ROM_PATCH_GD_SIZE),
+			       &need)) {
+		bt_dev_err(hdev, "Firmware section count too large: %u",
+			   *section_num);
+		return -EINVAL;
+	}
+
+	if (fw->size < need) {
+		bt_dev_err(hdev, "Firmware truncated: %zu bytes, need %zu for %u sections",
+			   fw->size, need, *section_num);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Reject a section whose payload does not lie within the firmware file. */
+static int btmtk_fw_validate_section(struct hci_dev *hdev,
+				     const struct firmware *fw, int index,
+				     u32 offset, u32 size)
+{
+	if (offset > fw->size || size > fw->size - offset) {
+		bt_dev_err(hdev, "Firmware section %d out of bounds: offset=%u, size=%u, fw_size=%zu",
+			   index, offset, size, fw->size);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static void btmtk_coredump(struct hci_dev *hdev)
 {
 	int err;
@@ -137,7 +199,6 @@ int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname,
 {
 	struct btmtk_hci_wmt_params wmt_params;
 	struct btmtk_patch_header *hdr;
-	struct btmtk_global_desc *globaldesc = NULL;
 	struct btmtk_section_map *sectionmap;
 	const struct firmware *fw;
 	const u8 *fw_ptr;
@@ -153,14 +214,18 @@ int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname,
 		return err;
 	}
 
+	bt_dev_info(hdev, "Loading BT firmware: %s", fwname);
+
+	err = btmtk_fw_validate_layout(hdev, fw, &section_num);
+	if (err < 0)
+		goto err_release_fw;
+
 	fw_ptr = fw->data;
 	fw_bin_ptr = fw_ptr;
 	hdr = (struct btmtk_patch_header *)fw_ptr;
-	globaldesc = (struct btmtk_global_desc *)(fw_ptr + MTK_FW_ROM_PATCH_HEADER_SIZE);
-	section_num = le32_to_cpu(globaldesc->section_num);
 
-	bt_dev_info(hdev, "HW/SW Version: 0x%04x%04x, Build Time: %s",
-		    le16_to_cpu(hdr->hwver), le16_to_cpu(hdr->swver), hdr->datetime);
+	bt_dev_info(hdev, "BT HW ver: 0x%04x, SW ver: 0x%04x, Build Time: %.16s",
+		    dev_id & 0xffff, le16_to_cpu(hdr->swver), hdr->datetime);
 
 	for (i = 0; i < section_num; i++) {
 		first_block = 1;
@@ -180,6 +245,11 @@ int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname,
 			continue;
 
 		if (dl_size > 0) {
+			err = btmtk_fw_validate_section(hdev, fw, i,
+							section_offset, dl_size);
+			if (err < 0)
+				goto err_release_fw;
+
 			retry = 20;
 			while (retry > 0) {
 				cmd[0] = 0; /* 0 means legacy dl mode. */
@@ -230,12 +300,12 @@ int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname,
 			while (dl_size > 0) {
 				dlen = min_t(int, 250, dl_size);
 				if (first_block == 1) {
-					flag = 1;
+					flag = BTMTK_WMT_PKT_START;
 					first_block = 0;
 				} else if (dl_size - dlen <= 0) {
-					flag = 3;
+					flag = BTMTK_WMT_PKT_END;
 				} else {
-					flag = 2;
+					flag = BTMTK_WMT_PKT_CONTINUE;
 				}
 
 				wmt_params.flag = flag;
@@ -314,7 +384,7 @@ int btmtk_setup_firmware(struct hci_dev *hdev, const char *fwname,
 
 	fw_size -= 30;
 	fw_ptr += 30;
-	flag = 1;
+	flag = BTMTK_WMT_PKT_START;
 
 	wmt_params.op = BTMTK_WMT_PATCH_DWNLD;
 	wmt_params.status = NULL;
@@ -324,9 +394,9 @@ int btmtk_setup_firmware(struct hci_dev *hdev, const char *fwname,
 
 		/* Tell device the position in sequence */
 		if (fw_size - dlen <= 0)
-			flag = 3;
+			flag = BTMTK_WMT_PKT_END;
 		else if (fw_size < fw->size - 30)
-			flag = 2;
+			flag = BTMTK_WMT_PKT_CONTINUE;
 
 		wmt_params.flag = flag;
 		wmt_params.dlen = dlen;
@@ -990,6 +1060,13 @@ int btmtk_usb_recv_acl(struct hci_dev *hdev, struct sk_buff *skb)
 	struct btmtk_data *data = hci_get_priv(hdev);
 	u16 handle = le16_to_cpu(hci_acl_hdr(skb)->handle);
 
+	/* The handles below are vendor-reserved values MTK firmware uses to
+	 * tag out-of-band debug/dump data on the ACL channel rather than a
+	 * real connection. Each is always sent as a single, complete
+	 * ACL_START packet, so unlike genuine connection data they never
+	 * arrive fragmented (e.g. 0x2efd is never followed by an ACL_CONT
+	 * continuation, 0x1efd).
+	 */
 	switch (handle) {
 	case 0xfc6f:		/* Firmware dump from device */
 		/* When the firmware hangs, the device can no longer
@@ -1011,6 +1088,7 @@ int btmtk_usb_recv_acl(struct hci_dev *hdev, struct sk_buff *skb)
 		fallthrough;
 	case 0x05ff:		/* Firmware debug logging 1 */
 	case 0x05fe:		/* Firmware debug logging 2 */
+	case 0x2efd:		/* Firmware debug event */
 		return hci_recv_diag(hdev, skb);
 	}
 
@@ -1413,6 +1491,10 @@ int btmtk_usb_setup(struct hci_dev *hdev)
 		err = btmtk_usb_hci_wmt_sync(hdev, &wmt_params);
 		if (err < 0) {
 			bt_dev_err(hdev, "Failed to send wmt func ctrl (%d)", err);
+
+			if (dev_id == 0x7925 && err == -ETIMEDOUT)
+				btmtk_reset_sync(hdev);
+
 			return err;
 		}
 
diff --git a/drivers/bluetooth/btmtk.h b/drivers/bluetooth/btmtk.h
index bc26148ec5442a..e8ad281a93a021 100644
--- a/drivers/bluetooth/btmtk.h
+++ b/drivers/bluetooth/btmtk.h
@@ -67,6 +67,15 @@ enum {
 	BTMTK_WMT_ON_PROGRESS,
 };
 
+/* Position of a BTMTK_WMT_PATCH_DWNLD packet within a download sequence,
+ * carried in struct btmtk_wmt_hdr's flag field.
+ */
+enum btmtk_wmt_pkt_flag {
+	BTMTK_WMT_PKT_START = 1,
+	BTMTK_WMT_PKT_CONTINUE = 2,
+	BTMTK_WMT_PKT_END = 3,
+};
+
 struct btmtk_wmt_hdr {
 	u8	dir;
 	u8	op;
diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c
index a15ae6598c6654..b9563df7298373 100644
--- a/drivers/bluetooth/btmtksdio.c
+++ b/drivers/bluetooth/btmtksdio.c
@@ -378,18 +378,14 @@ static int btmtksdio_fw_pmctrl(struct btmtksdio_dev *bdev)
 
 	/* Return ownership to the device */
 	sdio_writel(bdev->func, C_FW_OWN_REQ_SET, MTK_REG_CHLPCR, &err);
+	if (err == 0)
+		err = readx_poll_timeout(btmtksdio_drv_own_query, bdev, status,
+					 !(status & C_COM_DRV_OWN), 2000, 1000000);
 	if (err < 0)
-		goto out;
-
-	err = readx_poll_timeout(btmtksdio_drv_own_query, bdev, status,
-				 !(status & C_COM_DRV_OWN), 2000, 1000000);
-
+		bt_dev_err(bdev->hdev, "Cannot return ownership to device");
 out:
 	sdio_release_host(bdev->func);
 
-	if (err < 0)
-		bt_dev_err(bdev->hdev, "Cannot return ownership to device");
-
 	return err;
 }
 
@@ -469,6 +465,13 @@ static int btmtksdio_recv_acl(struct hci_dev *hdev, struct sk_buff *skb)
 	struct btmtksdio_dev *bdev = hci_get_drvdata(hdev);
 	u16 handle = le16_to_cpu(hci_acl_hdr(skb)->handle);
 
+	/* The handles below are vendor-reserved values MTK firmware uses to
+	 * tag out-of-band debug/dump data on the ACL channel rather than a
+	 * real connection. Each is always sent as a single, complete
+	 * ACL_START packet, so unlike genuine connection data they never
+	 * arrive fragmented (e.g. 0x2efd is never followed by an ACL_CONT
+	 * continuation, 0x1efd).
+	 */
 	switch (handle) {
 	case 0xfc6f:
 		/* Firmware dump from device: when the firmware hangs, the
@@ -478,6 +481,7 @@ static int btmtksdio_recv_acl(struct hci_dev *hdev, struct sk_buff *skb)
 		fallthrough;
 	case 0x05ff:
 	case 0x05fe:
+	case 0x2efd:		/* Firmware debug event */
 		/* Firmware debug logging */
 		return hci_recv_diag(hdev, skb);
 	}
@@ -779,8 +783,16 @@ static int btmtksdio_close(struct hci_dev *hdev)
 
 	sdio_release_irq(bdev->func);
 
+	/* No new work can be scheduled after sdio_release_irq(), so cancel the
+	 * work outside the sdio host lock. btmtksdio_txrx_work() also claims
+	 * the host, so canceling it while holding the lock would deadlock.
+	 */
+	sdio_release_host(bdev->func);
+
 	cancel_work_sync(&bdev->txrx_work);
 
+	sdio_claim_host(bdev->func);
+
 	btmtksdio_fw_pmctrl(bdev);
 
 	clear_bit(BTMTKSDIO_FUNC_ENABLED, &bdev->tx_state);
@@ -909,14 +921,14 @@ static int mt76xx_setup(struct hci_dev *hdev, const char *fwname)
 	return 0;
 }
 
-static int mt79xx_setup(struct hci_dev *hdev, const char *fwname)
+static int mt79xx_setup(struct hci_dev *hdev, const char *fwname, u32 dev_id)
 {
 	struct btmtksdio_dev *bdev = hci_get_drvdata(hdev);
 	struct btmtk_hci_wmt_params wmt_params;
 	u8 param = 0x1;
 	int err;
 
-	err = btmtk_setup_firmware_79xx(hdev, fwname, mtk_hci_wmt_sync, 0);
+	err = btmtk_setup_firmware_79xx(hdev, fwname, mtk_hci_wmt_sync, dev_id);
 	if (err < 0) {
 		bt_dev_err(hdev, "Failed to setup 79xx firmware (%d)", err);
 		return err;
@@ -1129,8 +1141,8 @@ static int btmtksdio_setup(struct hci_dev *hdev)
 	ktime_t calltime, delta, rettime;
 	unsigned long long duration;
 	char fwname[64];
-	int err, dev_id;
-	u32 fw_version = 0, val;
+	int err;
+	u32 dev_id, fw_version = 0, val;
 
 	calltime = ktime_get();
 	set_bit(BTMTKSDIO_HW_TX_READY, &bdev->tx_state);
@@ -1172,10 +1184,7 @@ static int btmtksdio_setup(struct hci_dev *hdev)
 		btmtk_fw_get_filename(fwname, sizeof(fwname), dev_id,
 				      fw_version, 0);
 
-		snprintf(fwname, sizeof(fwname),
-			 "mediatek/BT_RAM_CODE_MT%04x_1_%x_hdr.bin",
-			 dev_id & 0xffff, (fw_version & 0xff) + 1);
-		err = mt79xx_setup(hdev, fwname);
+		err = mt79xx_setup(hdev, fwname, dev_id);
 		if (err < 0)
 			return err;
 
@@ -1324,8 +1333,22 @@ static void btmtksdio_reset(struct hci_dev *hdev)
 
 	sdio_writel(bdev->func, C_INT_EN_CLR, MTK_REG_CHLPCR, NULL);
 	skb_queue_purge(&bdev->txq);
+
+	/* Unregister the IRQ before releasing the host lock so that a
+	 * concurrently running btmtksdio_txrx_work() cannot re-enable the
+	 * device interrupt (C_INT_EN_SET) and be rescheduled while the device
+	 * is being reset. btmtksdio_txrx_work() also claims the host, so the
+	 * work must be cancelled outside the sdio host lock to avoid a
+	 * deadlock. The IRQ is re-claimed by btmtksdio_open() when the HCI
+	 * device is re-opened after the reset.
+	 */
+	sdio_release_irq(bdev->func);
+	sdio_release_host(bdev->func);
+
 	cancel_work_sync(&bdev->txrx_work);
 
+	sdio_claim_host(bdev->func);
+
 	gpiod_set_value_cansleep(bdev->reset, 1);
 	msleep(100);
 	gpiod_set_value_cansleep(bdev->reset, 0);
diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c
index 25e7b41b349f12..5c730e3361141d 100644
--- a/drivers/bluetooth/btnxpuart.c
+++ b/drivers/bluetooth/btnxpuart.c
@@ -1388,9 +1388,11 @@ static int nxp_process_fw_dump(struct hci_dev *hdev, struct sk_buff *skb)
 				      msecs_to_jiffies(20000));
 	}
 
-	err = hci_devcd_append(hdev, skb_clone(skb, GFP_ATOMIC));
-	if (err < 0)
-		goto free_skb;
+	if (IS_ENABLED(CONFIG_DEV_COREDUMP)) {
+		err = hci_devcd_append(hdev, skb_clone(skb, GFP_ATOMIC));
+		if (err < 0)
+			goto free_skb;
+	}
 
 	if (buf_len == 0) {
 		bt_dev_warn(hdev, "==== FW dump complete ===");
@@ -1405,10 +1407,8 @@ static int nxp_process_fw_dump(struct hci_dev *hdev, struct sk_buff *skb)
 
 static int nxp_recv_acl_pkt(struct hci_dev *hdev, struct sk_buff *skb)
 {
-	__u16 handle = __le16_to_cpu(hci_acl_hdr(skb)->handle);
-
 	/* FW dump chunks are ACL packets with conn handle 0xfff */
-	if ((handle & 0x0FFF) == 0xFFF)
+	if (hci_acl_handle(skb) == 0xFFF)
 		return nxp_process_fw_dump(hdev, skb);
 	else
 		return hci_recv_frame(hdev, skb);
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
deleted file mode 100644
index dc7191bf423490..00000000000000
--- a/drivers/bluetooth/btusb.c
+++ /dev/null
@@ -1,4864 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *
- *  Generic Bluetooth USB driver
- *
- *  Copyright (C) 2005-2008  Marcel Holtmann <marcel@holtmann.org>
- */
-
-#include <linux/cpufeature.h>
-#include <linux/dmi.h>
-#include <linux/module.h>
-#include <linux/usb.h>
-#include <linux/usb/quirks.h>
-#include <linux/firmware.h>
-#include <linux/iopoll.h>
-#include <linux/of_device.h>
-#include <linux/of_irq.h>
-#include <linux/suspend.h>
-#include <linux/gpio/consumer.h>
-#include <linux/debugfs.h>
-#include <linux/unaligned.h>
-
-#include <net/bluetooth/bluetooth.h>
-#include <net/bluetooth/hci_core.h>
-#include <net/bluetooth/hci_drv.h>
-
-#include "btintel.h"
-#include "btbcm.h"
-#include "btrtl.h"
-#include "btmtk.h"
-
-#define VERSION "0.8"
-
-static bool disable_scofix;
-static bool force_scofix;
-static bool enable_autosuspend = IS_ENABLED(CONFIG_BT_HCIBTUSB_AUTOSUSPEND);
-static bool enable_poll_sync = IS_ENABLED(CONFIG_BT_HCIBTUSB_POLL_SYNC);
-static bool reset = true;
-
-static struct usb_driver btusb_driver;
-
-#define BTUSB_IGNORE			BIT(0)
-#define BTUSB_DIGIANSWER		BIT(1)
-#define BTUSB_CSR			BIT(2)
-#define BTUSB_SNIFFER			BIT(3)
-#define BTUSB_BCM92035			BIT(4)
-#define BTUSB_BROKEN_ISOC		BIT(5)
-#define BTUSB_WRONG_SCO_MTU		BIT(6)
-#define BTUSB_ATH3012			BIT(7)
-#define BTUSB_INTEL_COMBINED		BIT(8)
-#define BTUSB_INTEL_BOOT		BIT(9)
-#define BTUSB_BCM_PATCHRAM		BIT(10)
-#define BTUSB_MARVELL			BIT(11)
-#define BTUSB_SWAVE			BIT(12)
-#define BTUSB_AMP			BIT(13)
-#define BTUSB_QCA_ROME			BIT(14)
-#define BTUSB_BCM_APPLE			BIT(15)
-#define BTUSB_REALTEK			BIT(16)
-#define BTUSB_BCM2045			BIT(17)
-#define BTUSB_IFNUM_2			BIT(18)
-#define BTUSB_CW6622			BIT(19)
-#define BTUSB_MEDIATEK			BIT(20)
-#define BTUSB_WIDEBAND_SPEECH		BIT(21)
-#define BTUSB_INVALID_LE_STATES		BIT(22)
-#define BTUSB_QCA_WCN6855		BIT(23)
-#define BTUSB_INTEL_BROKEN_SHUTDOWN_LED	BIT(24)
-#define BTUSB_INTEL_BROKEN_INITIAL_NCMD BIT(25)
-#define BTUSB_INTEL_NO_WBS_SUPPORT	BIT(26)
-#define BTUSB_ACTIONS_SEMI		BIT(27)
-#define BTUSB_BARROT			BIT(28)
-#define BTUSB_BROKEN_EXT_SCAN		BIT(29)
-
-static const struct usb_device_id btusb_table[] = {
-	/*
-	 * NXP IW610 (0471:0215): the composite device reports Bluetooth
-	 * class at the whole-device level, so the generic entry below
-	 * would also match this WiFi vendor interface. Ignore it here
-	 * first so mwifiex-nxp can bind it instead.
-	 */
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x0471, 0x0215, 0xff, 0xff, 0xff),
-	  .driver_info = BTUSB_IGNORE },
-
-	/* Generic Bluetooth USB device */
-	{ USB_DEVICE_INFO(0xe0, 0x01, 0x01) },
-
-	/* Generic Bluetooth AMP device */
-	{ USB_DEVICE_INFO(0xe0, 0x01, 0x04), .driver_info = BTUSB_AMP },
-
-	/* Generic Bluetooth USB interface */
-	{ USB_INTERFACE_INFO(0xe0, 0x01, 0x01) },
-
-	/* Apple-specific (Broadcom) devices */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x05ac, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_APPLE | BTUSB_IFNUM_2 },
-
-	/* MediaTek MT76x0E */
-	{ USB_DEVICE(0x0e8d, 0x763f) },
-
-	/* Broadcom SoftSailing reporting vendor specific */
-	{ USB_DEVICE(0x0a5c, 0x21e1) },
-
-	/* Apple MacBookPro 7,1 */
-	{ USB_DEVICE(0x05ac, 0x8213) },
-
-	/* Apple iMac11,1 */
-	{ USB_DEVICE(0x05ac, 0x8215) },
-
-	/* Apple MacBookPro6,2 */
-	{ USB_DEVICE(0x05ac, 0x8218) },
-
-	/* Apple MacBookAir3,1, MacBookAir3,2 */
-	{ USB_DEVICE(0x05ac, 0x821b) },
-
-	/* Apple MacBookAir4,1 */
-	{ USB_DEVICE(0x05ac, 0x821f) },
-
-	/* Apple MacBookPro8,2 */
-	{ USB_DEVICE(0x05ac, 0x821a) },
-
-	/* Apple MacMini5,1 */
-	{ USB_DEVICE(0x05ac, 0x8281) },
-
-	/* AVM BlueFRITZ! USB v2.0 */
-	{ USB_DEVICE(0x057c, 0x3800), .driver_info = BTUSB_SWAVE },
-
-	/* Bluetooth Ultraport Module from IBM */
-	{ USB_DEVICE(0x04bf, 0x030a) },
-
-	/* ALPS Modules with non-standard id */
-	{ USB_DEVICE(0x044e, 0x3001) },
-	{ USB_DEVICE(0x044e, 0x3002) },
-
-	/* Ericsson with non-standard id */
-	{ USB_DEVICE(0x0bdb, 0x1002) },
-
-	/* Canyon CN-BTU1 with HID interfaces */
-	{ USB_DEVICE(0x0c10, 0x0000) },
-
-	/* Broadcom BCM20702B0 (Dynex/Insignia) */
-	{ USB_DEVICE(0x19ff, 0x0239), .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Broadcom BCM43142A0 (Foxconn/Lenovo) */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x105b, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Broadcom BCM920703 (HTC Vive) */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x0bb4, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Foxconn - Hon Hai */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x0489, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Lite-On Technology - Broadcom based */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x04ca, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Broadcom devices with vendor specific id */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x0a5c, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* ASUSTek Computer - Broadcom based */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x0b05, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Belkin F8065bf - Broadcom based */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x050d, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* IMC Networks - Broadcom based */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x13d3, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Dell Computer - Broadcom based  */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x413c, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Toshiba Corp - Broadcom based */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x0930, 0xff, 0x01, 0x01),
-	  .driver_info = BTUSB_BCM_PATCHRAM },
-
-	/* Intel Bluetooth USB Bootloader (RAM module) */
-	{ USB_DEVICE(0x8087, 0x0a5a),
-	  .driver_info = BTUSB_INTEL_BOOT | BTUSB_BROKEN_ISOC },
-
-	{ }	/* Terminating entry */
-};
-
-MODULE_DEVICE_TABLE(usb, btusb_table);
-
-static const struct usb_device_id quirks_table[] = {
-	/* CSR BlueCore devices */
-	{ USB_DEVICE(0x0a12, 0x0001), .driver_info = BTUSB_CSR },
-
-	/* Broadcom BCM2033 without firmware */
-	{ USB_DEVICE(0x0a5c, 0x2033), .driver_info = BTUSB_IGNORE },
-
-	/* Broadcom BCM2045 devices */
-	{ USB_DEVICE(0x0a5c, 0x2045), .driver_info = BTUSB_BCM2045 },
-
-	/* Atheros 3011 with sflash firmware */
-	{ USB_DEVICE(0x0489, 0xe027), .driver_info = BTUSB_IGNORE },
-	{ USB_DEVICE(0x0489, 0xe03d), .driver_info = BTUSB_IGNORE },
-	{ USB_DEVICE(0x04f2, 0xaff1), .driver_info = BTUSB_IGNORE },
-	{ USB_DEVICE(0x0930, 0x0215), .driver_info = BTUSB_IGNORE },
-	{ USB_DEVICE(0x0cf3, 0x3002), .driver_info = BTUSB_IGNORE },
-	{ USB_DEVICE(0x0cf3, 0xe019), .driver_info = BTUSB_IGNORE },
-	{ USB_DEVICE(0x13d3, 0x3304), .driver_info = BTUSB_IGNORE },
-
-	/* Atheros AR9285 Malbec with sflash firmware */
-	{ USB_DEVICE(0x03f0, 0x311d), .driver_info = BTUSB_IGNORE },
-
-	/* Atheros 3012 with sflash firmware */
-	{ USB_DEVICE(0x0489, 0xe04d), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe04e), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe056), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe057), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe05f), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe076), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe078), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe095), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04c5, 0x1330), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3004), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3005), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3006), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3007), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3008), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x300b), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x300d), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x300f), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3010), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3014), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x04ca, 0x3018), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0930, 0x0219), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0930, 0x021c), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0930, 0x0220), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0930, 0x0227), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0b05, 0x17d0), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x0036), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x3004), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x3008), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x311d), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x311e), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x311f), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x3121), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x817a), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x817b), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0xe003), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0xe004), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0xe005), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0xe006), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3362), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3375), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3393), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3395), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3402), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3408), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3423), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3432), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3472), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3474), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3487), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x13d3, 0x3490), .driver_info = BTUSB_ATH3012 },
-
-	/* Atheros AR5BBU12 with sflash firmware */
-	{ USB_DEVICE(0x0489, 0xe02c), .driver_info = BTUSB_IGNORE },
-
-	/* Atheros AR5BBU12 with sflash firmware */
-	{ USB_DEVICE(0x0489, 0xe036), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0489, 0xe03c), .driver_info = BTUSB_ATH3012 },
-
-	/* QCA ROME chipset */
-	{ USB_DEVICE(0x0cf3, 0x535b), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe009), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe010), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe301), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe500), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe09f), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0a2), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3015), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x301a), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3021), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3491), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3496), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3501), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3503), .driver_info = BTUSB_QCA_ROME |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* QCA WCN6855 chipset */
-	{ USB_DEVICE(0x0489, 0xe0c7), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0c9), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0ca), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0cb), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0cc), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0ce), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0d0), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0d6), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0de), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0df), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0e1), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0e3), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0ea), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0ec), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3022), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3023), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3024), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3a22), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3a24), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3a26), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3a27), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cf3, 0xe600), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9108), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9109), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9208), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9209), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9308), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9309), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9408), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9409), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9508), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9509), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9608), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9609), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x10ab, 0x9f09), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x28de, 0x1401), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* QCA WCN785x chipset */
-	{ USB_DEVICE(0x0cf3, 0xe700), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0fc), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0f3), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe100), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe103), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe10a), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe10d), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe11b), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe11c), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe11f), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe141), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe14a), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe14b), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe14d), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3623), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3624), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2c7c, 0x0130), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2c7c, 0x0131), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2c7c, 0x0132), .driver_info = BTUSB_QCA_WCN6855 |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Broadcom BCM2035 */
-	{ USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 },
-	{ USB_DEVICE(0x0a5c, 0x200a), .driver_info = BTUSB_WRONG_SCO_MTU },
-	{ USB_DEVICE(0x0a5c, 0x2035), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* Broadcom BCM2045 */
-	{ USB_DEVICE(0x0a5c, 0x2039), .driver_info = BTUSB_WRONG_SCO_MTU },
-	{ USB_DEVICE(0x0a5c, 0x2101), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* IBM/Lenovo ThinkPad with Broadcom chip */
-	{ USB_DEVICE(0x0a5c, 0x201e), .driver_info = BTUSB_WRONG_SCO_MTU },
-	{ USB_DEVICE(0x0a5c, 0x2110), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* HP laptop with Broadcom chip */
-	{ USB_DEVICE(0x03f0, 0x171d), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* Dell laptop with Broadcom chip */
-	{ USB_DEVICE(0x413c, 0x8126), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* Dell Wireless 370 and 410 devices */
-	{ USB_DEVICE(0x413c, 0x8152), .driver_info = BTUSB_WRONG_SCO_MTU },
-	{ USB_DEVICE(0x413c, 0x8156), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* Belkin F8T012 and F8T013 devices */
-	{ USB_DEVICE(0x050d, 0x0012), .driver_info = BTUSB_WRONG_SCO_MTU },
-	{ USB_DEVICE(0x050d, 0x0013), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* Asus WL-BTD202 device */
-	{ USB_DEVICE(0x0b05, 0x1715), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* Kensington Bluetooth USB adapter */
-	{ USB_DEVICE(0x047d, 0x105e), .driver_info = BTUSB_WRONG_SCO_MTU },
-
-	/* RTX Telecom based adapters with buggy SCO support */
-	{ USB_DEVICE(0x0400, 0x0807), .driver_info = BTUSB_BROKEN_ISOC },
-	{ USB_DEVICE(0x0400, 0x080a), .driver_info = BTUSB_BROKEN_ISOC },
-
-	/* CONWISE Technology based adapters with buggy SCO support */
-	{ USB_DEVICE(0x0e5e, 0x6622),
-	  .driver_info = BTUSB_BROKEN_ISOC | BTUSB_CW6622},
-
-	/* Roper Class 1 Bluetooth Dongle (Silicon Wave based) */
-	{ USB_DEVICE(0x1310, 0x0001), .driver_info = BTUSB_SWAVE },
-
-	/* Digianswer devices */
-	{ USB_DEVICE(0x08fd, 0x0001), .driver_info = BTUSB_DIGIANSWER },
-	{ USB_DEVICE(0x08fd, 0x0002), .driver_info = BTUSB_IGNORE },
-
-	/* CSR BlueCore Bluetooth Sniffer */
-	{ USB_DEVICE(0x0a12, 0x0002),
-	  .driver_info = BTUSB_SNIFFER | BTUSB_BROKEN_ISOC },
-
-	/* Frontline ComProbe Bluetooth Sniffer */
-	{ USB_DEVICE(0x16d3, 0x0002),
-	  .driver_info = BTUSB_SNIFFER | BTUSB_BROKEN_ISOC },
-
-	/* Marvell Bluetooth devices */
-	{ USB_DEVICE(0x1286, 0x2044), .driver_info = BTUSB_MARVELL },
-	{ USB_DEVICE(0x1286, 0x2046), .driver_info = BTUSB_MARVELL },
-	{ USB_DEVICE(0x1286, 0x204e), .driver_info = BTUSB_MARVELL },
-
-	/*
-	 * NXP IW610 BT interfaces (Marvell-lineage silicon, same quirk as
-	 * the 0x1286 entries above). Scoped to the BT interface class,
-	 * not just VID/PID -- see the btusb_table entry above.
-	 */
-	{ USB_DEVICE_AND_INTERFACE_INFO(0x0471, 0x0215, 0xe0, 0x01, 0x01),
-	  .driver_info = BTUSB_MARVELL },
-
-	/* Intel Bluetooth devices */
-	{ USB_DEVICE(0x8087, 0x0025), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0026), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0029), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0032), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0033), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0035), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0036), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0037), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0038), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0039), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0040), .driver_info = BTUSB_INTEL_COMBINED }, /* Lizard Peak 2 */
-	{ USB_DEVICE(0x8087, 0x07da), .driver_info = BTUSB_CSR },
-	{ USB_DEVICE(0x8087, 0x07dc), .driver_info = BTUSB_INTEL_COMBINED |
-						     BTUSB_INTEL_NO_WBS_SUPPORT |
-						     BTUSB_INTEL_BROKEN_INITIAL_NCMD |
-						     BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
-	{ USB_DEVICE(0x8087, 0x0a2a), .driver_info = BTUSB_INTEL_COMBINED |
-						     BTUSB_INTEL_NO_WBS_SUPPORT |
-						     BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
-	{ USB_DEVICE(0x8087, 0x0a2b), .driver_info = BTUSB_INTEL_COMBINED },
-	{ USB_DEVICE(0x8087, 0x0aa7), .driver_info = BTUSB_INTEL_COMBINED |
-						     BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
-	{ USB_DEVICE(0x8087, 0x0aaa), .driver_info = BTUSB_INTEL_COMBINED },
-
-	/* Other Intel Bluetooth devices */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x8087, 0xe0, 0x01, 0x01),
-	  .driver_info = BTUSB_IGNORE },
-
-	/* Realtek 8821CE Bluetooth devices */
-	{ USB_DEVICE(0x13d3, 0x3529), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3533), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8822CE Bluetooth devices */
-	{ USB_DEVICE(0x0bda, 0xb00c), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0xc822), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8822CU Bluetooth devices */
-	{ USB_DEVICE(0x13d3, 0x3549), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8851BE Bluetooth devices */
-	{ USB_DEVICE(0x0bda, 0xb850), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3600), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3601), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x0489, 0xe112), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8851BU Bluetooth devices */
-	{ USB_DEVICE(0x3625, 0x010b), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2001, 0x332a), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x7392, 0xe611), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2c4e, 0x0128), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8852AE Bluetooth devices */
-	{ USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0xc852), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0x385a), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0x4852), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04c5, 0x165c), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x4006), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cb8, 0xc549), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8852CE Bluetooth devices */
-	{ USB_DEVICE(0x04ca, 0x4007), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04c5, 0x1675), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cb8, 0xc558), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3587), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3586), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3592), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3612), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe122), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8852BE Bluetooth devices */
-	{ USB_DEVICE(0x0cb8, 0xc559), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0x4853), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0x887b), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0xb85b), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3570), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3571), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3572), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3591), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe123), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe125), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8852BT/8852BE-VT Bluetooth devices */
-	{ USB_DEVICE(0x0bda, 0x8520), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe12f), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3618), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3619), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8922AE Bluetooth devices */
-	{ USB_DEVICE(0x0bda, 0x8922), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0xd922), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0xd923), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3617), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3616), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe130), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Realtek 8761BU Bluetooth devices */
-	{ USB_DEVICE(0x0bda, 0xa728), .driver_info = BTUSB_REALTEK |
-						     BTUSB_BROKEN_EXT_SCAN },
-
-	/* Realtek Bluetooth devices */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01),
-	  .driver_info = BTUSB_REALTEK },
-
-	/* MediaTek Bluetooth devices */
-	{ USB_VENDOR_AND_INTERFACE_INFO(0x0e8d, 0xe0, 0x01, 0x01),
-	  .driver_info = BTUSB_MEDIATEK |
-			 BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional MediaTek MT7615E Bluetooth devices */
-	{ USB_DEVICE(0x13d3, 0x3560), .driver_info = BTUSB_MEDIATEK},
-
-	/* Additional MediaTek MT7663 Bluetooth devices */
-	{ USB_DEVICE(0x043e, 0x310c), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3801), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional MediaTek MT7668 Bluetooth devices */
-	{ USB_DEVICE(0x043e, 0x3109), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional MediaTek MT7920 Bluetooth devices */
-	{ USB_DEVICE(0x0489, 0xe134), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe135), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3620), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3621), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3622), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe158), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional MediaTek MT7921 Bluetooth devices */
-	{ USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0cd), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0e0), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0f2), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3802), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0e8d, 0x0608), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3563), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3564), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3567), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3576), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3578), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3583), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3606), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	/* MediaTek MT7902 Bluetooth devices */
-	{ USB_DEVICE(0x0489, 0xe156), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0e8d, 0x1ede), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3579), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3580), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3594), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3596), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	/* MediaTek MT7922 Bluetooth devices */
-	{ USB_DEVICE(0x13d3, 0x3585), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3610), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* MediaTek MT7922A Bluetooth devices */
-	{ USB_DEVICE(0x0489, 0xe0d8), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0d9), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0e2), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0e4), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0f1), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0f2), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0f5), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0f6), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe102), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe11d), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe152), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe153), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe170), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe174), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3804), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x3807), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04ca, 0x38e4), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0e8d, 0x223c), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3568), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3584), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3605), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3607), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3614), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3615), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3633), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x35f5, 0x7922), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional MediaTek MT7925 Bluetooth devices */
-	{ USB_DEVICE(0x0489, 0xe111), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe113), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe118), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe11e), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe124), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe139), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe13a), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe0fa), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe10f), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe110), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe116), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3588), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe14e), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe14f), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe150), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0489, 0xe151), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0e8d, 0x8c38), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3602), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3603), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3604), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3608), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3609), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3613), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3625), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3627), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3628), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3630), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2c7c, 0x7009), .driver_info = BTUSB_MEDIATEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional Realtek 8723AE Bluetooth devices */
-	{ USB_DEVICE(0x0930, 0x021d), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3394), .driver_info = BTUSB_REALTEK },
-
-	/* Additional Realtek 8723BE Bluetooth devices */
-	{ USB_DEVICE(0x0489, 0xe085), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x0489, 0xe08b), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x04f2, 0xb49f), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3410), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3416), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3459), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3494), .driver_info = BTUSB_REALTEK },
-
-	/* Additional Realtek 8723BU Bluetooth devices */
-	{ USB_DEVICE(0x7392, 0xa611), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x2c0a, 0x8761), .driver_info = BTUSB_REALTEK },
-
-	/* Additional Realtek 8723DE Bluetooth devices */
-	{ USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK },
-
-	/* Additional Realtek 8761BUV Bluetooth devices */
-	{ USB_DEVICE(0x2c4e, 0x0115), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2357, 0x0607), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2550, 0x8761), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0x8771), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x6655, 0x8771), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x7392, 0xc611), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2b89, 0x8761), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2b89, 0x6275), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x37ad, 0x0600), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional Realtek 8761CU Bluetooth devices */
-	{ USB_DEVICE(0x0b05, 0x1bef), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0b05, 0x1d70), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional Realtek 8821AE Bluetooth devices */
-	{ USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3414), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3458), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3461), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x13d3, 0x3462), .driver_info = BTUSB_REALTEK },
-
-	/* Additional Realtek 8822BE Bluetooth devices */
-	{ USB_DEVICE(0x13d3, 0x3526), .driver_info = BTUSB_REALTEK },
-	{ USB_DEVICE(0x0b05, 0x185c), .driver_info = BTUSB_REALTEK },
-
-	/* Additional Realtek 8822CE Bluetooth devices */
-	{ USB_DEVICE(0x04ca, 0x4005), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x04c5, 0x161f), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0b05, 0x18ef), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3548), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3549), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3553), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x13d3, 0x3555), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x2ff8, 0x3051), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x1358, 0xc123), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0bda, 0xc123), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x1357, 0xc123), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-	{ USB_DEVICE(0x0cb5, 0xc547), .driver_info = BTUSB_REALTEK |
-						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Barrot Technology Bluetooth devices */
-	{ USB_DEVICE(0x33fa, 0x0010), .driver_info = BTUSB_BARROT },
-	{ USB_DEVICE(0x33fa, 0x0012), .driver_info = BTUSB_BARROT },
-
-	/* Actions Semiconductor ATS2851 based devices */
-	{ USB_DEVICE(0x10d7, 0xb012), .driver_info = BTUSB_ACTIONS_SEMI },
-
-	/* Silicon Wave based devices */
-	{ USB_DEVICE(0x0c10, 0x0000), .driver_info = BTUSB_SWAVE },
-
-	{ }	/* Terminating entry */
-};
-
-/* The Bluetooth USB module build into some devices needs to be reset on resume,
- * this is a problem with the platform (likely shutting off all power) not with
- * the module itself. So we use a DMI list to match known broken platforms.
- */
-static const struct dmi_system_id btusb_needs_reset_resume_table[] = {
-	{
-		/* Dell OptiPlex 3060 (QCA ROME device 0cf3:e007) */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 3060"),
-		},
-	},
-	{
-		/* Dell XPS 9360 (QCA ROME device 0cf3:e300) */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "XPS 13 9360"),
-		},
-	},
-	{
-		/* Dell Inspiron 5565 (QCA ROME device 0cf3:e009) */
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 5565"),
-		},
-	},
-	{}
-};
-
-struct qca_dump_info {
-	/* fields for dump collection */
-	u16 id_vendor;
-	u16 id_product;
-	u32 fw_version;
-	u32 controller_id;
-	u32 ram_dump_size;
-	u16 ram_dump_seqno;
-};
-
-struct btqca_data {
-	struct qca_dump_info qca_dump;
-};
-
-#define BTUSB_MAX_ISOC_FRAMES	10
-
-#define BTUSB_INTR_RUNNING	0
-#define BTUSB_BULK_RUNNING	1
-#define BTUSB_ISOC_RUNNING	2
-#define BTUSB_SUSPENDING	3
-#define BTUSB_DID_ISO_RESUME	4
-#define BTUSB_BOOTLOADER	5
-#define BTUSB_DOWNLOADING	6
-#define BTUSB_FIRMWARE_LOADED	7
-#define BTUSB_FIRMWARE_FAILED	8
-#define BTUSB_BOOTING		9
-#define BTUSB_DIAG_RUNNING	10
-#define BTUSB_OOB_WAKE_ENABLED	11
-#define BTUSB_HW_RESET_ACTIVE	12
-#define BTUSB_TX_WAIT_VND_EVT	13
-#define BTUSB_WAKEUP_AUTOSUSPEND	14
-#define BTUSB_USE_ALT3_FOR_WBS	15
-#define BTUSB_ALT6_CONTINUOUS_TX	16
-#define BTUSB_HW_SSR_ACTIVE	17
-#define BTUSB_WAKEUP_BROKEN	18
-#define BTUSB_RESET		19
-
-struct btusb_data {
-	struct hci_dev       *hdev;
-	struct usb_device    *udev;
-	struct usb_interface *intf;
-	struct usb_interface *isoc;
-	struct usb_interface *diag;
-	unsigned isoc_ifnum;
-
-	unsigned long flags;
-
-	bool poll_sync;
-	int intr_interval;
-	struct work_struct  work;
-	struct work_struct  waker;
-	struct delayed_work rx_work;
-
-	struct sk_buff_head acl_q;
-
-	struct usb_anchor deferred;
-	struct usb_anchor tx_anchor;
-	int tx_in_flight;
-	spinlock_t txlock;
-
-	struct usb_anchor intr_anchor;
-	struct usb_anchor bulk_anchor;
-	struct usb_anchor isoc_anchor;
-	struct usb_anchor diag_anchor;
-	struct usb_anchor ctrl_anchor;
-	spinlock_t rxlock;
-
-	struct sk_buff *evt_skb;
-	struct sk_buff *acl_skb;
-	struct sk_buff *sco_skb;
-
-	struct usb_endpoint_descriptor *intr_ep;
-	struct usb_endpoint_descriptor *bulk_tx_ep;
-	struct usb_endpoint_descriptor *bulk_rx_ep;
-	struct usb_endpoint_descriptor *isoc_tx_ep;
-	struct usb_endpoint_descriptor *isoc_rx_ep;
-	struct usb_endpoint_descriptor *diag_tx_ep;
-	struct usb_endpoint_descriptor *diag_rx_ep;
-
-	struct gpio_desc *reset_gpio;
-
-	__u8 cmdreq_type;
-	__u8 cmdreq;
-
-	unsigned int sco_num;
-	unsigned int air_mode;
-	bool usb_alt6_packet_flow;
-	int isoc_altsetting;
-	int suspend_count;
-	const struct usb_device_id *match_id;
-
-	int (*recv_event)(struct hci_dev *hdev, struct sk_buff *skb);
-	int (*recv_acl)(struct hci_dev *hdev, struct sk_buff *skb);
-	int (*recv_bulk)(struct btusb_data *data, void *buffer, int count);
-
-	int (*setup_on_usb)(struct hci_dev *hdev);
-
-	int (*suspend)(struct hci_dev *hdev);
-	int (*resume)(struct hci_dev *hdev);
-	int (*disconnect)(struct hci_dev *hdev);
-
-	int oob_wake_irq;   /* irq for out-of-band wake-on-bt */
-};
-
-static void btusb_reset(struct hci_dev *hdev)
-{
-	struct btusb_data *data;
-	int err;
-
-	data = hci_get_drvdata(hdev);
-	err = usb_autopm_get_interface(data->intf);
-	if (err) {
-		bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err);
-		return;
-	}
-
-	if (test_and_set_bit(BTUSB_RESET, &data->flags))
-		usb_autopm_put_interface_no_suspend(data->intf);
-
-	bt_dev_err(hdev, "Resetting usb device.");
-	usb_queue_reset_device(data->intf);
-}
-
-static void btusb_intel_reset(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct gpio_desc *reset_gpio = data->reset_gpio;
-	struct btintel_data *intel_data = hci_get_priv(hdev);
-
-	if (intel_data->acpi_reset_method) {
-		if (test_and_set_bit(INTEL_ACPI_RESET_ACTIVE, intel_data->flags)) {
-			bt_dev_err(hdev, "acpi: last reset failed ? Not resetting again");
-			return;
-		}
-
-		bt_dev_err(hdev, "Initiating acpi reset method");
-		/* If ACPI reset method fails, lets try with legacy GPIO
-		 * toggling
-		 */
-		if (!intel_data->acpi_reset_method(hdev)) {
-			return;
-		}
-	}
-
-	if (!reset_gpio) {
-		btusb_reset(hdev);
-		return;
-	}
-
-	/*
-	 * Toggle the hard reset line if the platform provides one. The reset
-	 * is going to yank the device off the USB and then replug. So doing
-	 * once is enough. The cleanup is handled correctly on the way out
-	 * (standard USB disconnect), and the new device is detected cleanly
-	 * and bound to the driver again like it should be.
-	 */
-	if (test_and_set_bit(BTUSB_HW_RESET_ACTIVE, &data->flags)) {
-		bt_dev_err(hdev, "last reset failed? Not resetting again");
-		return;
-	}
-
-	bt_dev_err(hdev, "Initiating HW reset via gpio");
-	gpiod_set_value_cansleep(reset_gpio, 1);
-	msleep(100);
-	gpiod_set_value_cansleep(reset_gpio, 0);
-}
-
-#define RTK_DEVCOREDUMP_CODE_MEMDUMP		0x01
-#define RTK_DEVCOREDUMP_CODE_HW_ERR		0x02
-#define RTK_DEVCOREDUMP_CODE_CMD_TIMEOUT	0x03
-
-#define RTK_SUB_EVENT_CODE_COREDUMP		0x34
-
-struct rtk_dev_coredump_hdr {
-	u8 type;
-	u8 code;
-	u8 reserved[2];
-} __packed;
-
-static inline void btusb_rtl_alloc_devcoredump(struct hci_dev *hdev,
-		struct rtk_dev_coredump_hdr *hdr, u8 *buf, u32 len)
-{
-	struct sk_buff *skb;
-
-	skb = alloc_skb(len + sizeof(*hdr), GFP_ATOMIC);
-	if (!skb)
-		return;
-
-	skb_put_data(skb, hdr, sizeof(*hdr));
-	if (len)
-		skb_put_data(skb, buf, len);
-
-	if (!hci_devcd_init(hdev, skb->len)) {
-		hci_devcd_append(hdev, skb);
-		hci_devcd_complete(hdev);
-	} else {
-		bt_dev_err(hdev, "RTL: Failed to generate devcoredump");
-		kfree_skb(skb);
-	}
-}
-
-static void btusb_rtl_reset(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct gpio_desc *reset_gpio = data->reset_gpio;
-	struct rtk_dev_coredump_hdr hdr = {
-		.type = RTK_DEVCOREDUMP_CODE_CMD_TIMEOUT,
-	};
-
-	btusb_rtl_alloc_devcoredump(hdev, &hdr, NULL, 0);
-
-	if (!reset_gpio) {
-		btusb_reset(hdev);
-		return;
-	}
-
-	/* Toggle the hard reset line. The Realtek device is going to
-	 * yank itself off the USB and then replug. The cleanup is handled
-	 * correctly on the way out (standard USB disconnect), and the new
-	 * device is detected cleanly and bound to the driver again like
-	 * it should be.
-	 */
-	if (test_and_set_bit(BTUSB_HW_RESET_ACTIVE, &data->flags)) {
-		bt_dev_err(hdev, "last reset failed? Not resetting again");
-		return;
-	}
-
-	bt_dev_err(hdev, "Reset Realtek device via gpio");
-	gpiod_set_value_cansleep(reset_gpio, 1);
-	msleep(200);
-	gpiod_set_value_cansleep(reset_gpio, 0);
-}
-
-static void btusb_rtl_hw_error(struct hci_dev *hdev, u8 code)
-{
-	struct rtk_dev_coredump_hdr hdr = {
-		.type = RTK_DEVCOREDUMP_CODE_HW_ERR,
-		.code = code,
-	};
-
-	bt_dev_err(hdev, "RTL: hw err, trigger devcoredump (%d)", code);
-
-	btusb_rtl_alloc_devcoredump(hdev, &hdr, NULL, 0);
-}
-
-static void btusb_qca_reset(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct gpio_desc *reset_gpio = data->reset_gpio;
-
-	if (test_bit(BTUSB_HW_SSR_ACTIVE, &data->flags)) {
-		bt_dev_info(hdev, "Ramdump in progress, defer reset");
-		return;
-	}
-
-	if (reset_gpio) {
-		bt_dev_err(hdev, "Reset qca device via bt_en gpio");
-
-		/* Toggle the hard reset line. The qca bt device is going to
-		 * yank itself off the USB and then replug. The cleanup is handled
-		 * correctly on the way out (standard USB disconnect), and the new
-		 * device is detected cleanly and bound to the driver again like
-		 * it should be.
-		 */
-		if (test_and_set_bit(BTUSB_HW_RESET_ACTIVE, &data->flags)) {
-			bt_dev_err(hdev, "last reset failed? Not resetting again");
-			return;
-		}
-
-		gpiod_set_value_cansleep(reset_gpio, 0);
-		msleep(200);
-		gpiod_set_value_cansleep(reset_gpio, 1);
-
-		return;
-	}
-
-	btusb_reset(hdev);
-}
-
-static u8 btusb_classify_qca_pkt_type(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	/* Some Qualcomm controllers, e.g., QCNFA765 with WCN6855 chip, send debug
-	 * packets as ACL frames with connection handle 0x2EDC. These are not real
-	 * ACL packets and should be reclassified as HCI_DIAG_PKT to prevent
-	 * "ACL packet for unknown connection handle 3804" errors.
-	 */
-	if (skb->len >= 2) {
-		u16 handle = get_unaligned_le16(skb->data);
-
-		if (handle == 0x2EDC)
-			return HCI_DIAG_PKT;
-	}
-
-	/* Use default packet type for other packets */
-	return hci_skb_pkt_type(skb);
-}
-
-static inline void btusb_free_frags(struct btusb_data *data)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&data->rxlock, flags);
-
-	dev_kfree_skb_irq(data->evt_skb);
-	data->evt_skb = NULL;
-
-	dev_kfree_skb_irq(data->acl_skb);
-	data->acl_skb = NULL;
-
-	dev_kfree_skb_irq(data->sco_skb);
-	data->sco_skb = NULL;
-
-	spin_unlock_irqrestore(&data->rxlock, flags);
-}
-
-static int btusb_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-
-	if (data->intr_interval) {
-		/* Trigger dequeue immediately if an event is received */
-		schedule_delayed_work(&data->rx_work, 0);
-	}
-
-	return data->recv_event(hdev, skb);
-}
-
-static int btusb_recv_intr(struct btusb_data *data, void *buffer, int count)
-{
-	struct sk_buff *skb;
-	unsigned long flags;
-	int err = 0;
-
-	spin_lock_irqsave(&data->rxlock, flags);
-	skb = data->evt_skb;
-
-	while (count) {
-		int len;
-
-		if (!skb) {
-			skb = bt_skb_alloc(HCI_MAX_EVENT_SIZE, GFP_ATOMIC);
-			if (!skb) {
-				err = -ENOMEM;
-				break;
-			}
-
-			hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
-			hci_skb_expect(skb) = HCI_EVENT_HDR_SIZE;
-		}
-
-		len = min_t(uint, hci_skb_expect(skb), count);
-		skb_put_data(skb, buffer, len);
-
-		count -= len;
-		buffer += len;
-		hci_skb_expect(skb) -= len;
-
-		if (skb->len == HCI_EVENT_HDR_SIZE) {
-			/* Complete event header */
-			hci_skb_expect(skb) = hci_event_hdr(skb)->plen;
-
-			if (skb_tailroom(skb) < hci_skb_expect(skb)) {
-				kfree_skb(skb);
-				skb = NULL;
-
-				err = -EILSEQ;
-				break;
-			}
-		}
-
-		if (!hci_skb_expect(skb)) {
-			/* Each chunk should correspond to at least 1 or more
-			 * events so if there are still bytes left that doesn't
-			 * constitute a new event this is likely a bug in the
-			 * controller.
-			 */
-			if (count && count < HCI_EVENT_HDR_SIZE) {
-				bt_dev_warn(data->hdev,
-					"Unexpected continuation: %d bytes",
-					count);
-				count = 0;
-			}
-
-			/* Complete frame */
-			btusb_recv_event(data->hdev, skb);
-			skb = NULL;
-		}
-	}
-
-	data->evt_skb = skb;
-	spin_unlock_irqrestore(&data->rxlock, flags);
-
-	return err;
-}
-
-static int btusb_recv_acl(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-
-	/* Only queue ACL packet if intr_interval is set as it means
-	 * force_poll_sync has been enabled.
-	 */
-	if (!data->intr_interval)
-		return data->recv_acl(hdev, skb);
-
-	skb_queue_tail(&data->acl_q, skb);
-	schedule_delayed_work(&data->rx_work, data->intr_interval);
-
-	return 0;
-}
-
-static int btusb_recv_bulk(struct btusb_data *data, void *buffer, int count)
-{
-	struct sk_buff *skb;
-	unsigned long flags;
-	int err = 0;
-
-	spin_lock_irqsave(&data->rxlock, flags);
-	skb = data->acl_skb;
-
-	while (count) {
-		int len;
-
-		if (!skb) {
-			skb = bt_skb_alloc(HCI_MAX_FRAME_SIZE, GFP_ATOMIC);
-			if (!skb) {
-				err = -ENOMEM;
-				break;
-			}
-
-			hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT;
-			hci_skb_expect(skb) = HCI_ACL_HDR_SIZE;
-		}
-
-		len = min_t(uint, hci_skb_expect(skb), count);
-		skb_put_data(skb, buffer, len);
-
-		count -= len;
-		buffer += len;
-		hci_skb_expect(skb) -= len;
-
-		if (skb->len == HCI_ACL_HDR_SIZE) {
-			/* Complete ACL header */
-			hci_skb_expect(skb) = hci_acl_dlen(skb);
-
-			if (skb_tailroom(skb) < hci_skb_expect(skb)) {
-				kfree_skb(skb);
-				skb = NULL;
-
-				err = -EILSEQ;
-				break;
-			}
-		}
-
-		if (!hci_skb_expect(skb)) {
-			/* Complete frame */
-			btusb_recv_acl(data->hdev, skb);
-			skb = NULL;
-		}
-	}
-
-	data->acl_skb = skb;
-	spin_unlock_irqrestore(&data->rxlock, flags);
-
-	return err;
-}
-
-static bool btusb_validate_sco_handle(struct hci_dev *hdev,
-				      struct hci_sco_hdr *hdr)
-{
-	__u16 handle;
-
-	if (hci_dev_test_flag(hdev, HCI_USER_CHANNEL))
-		// Can't validate, userspace controls everything.
-		return true;
-
-	/*
-	 * USB isochronous transfers are not designed to be reliable and may
-	 * lose fragments.  When this happens, the next first fragment
-	 * encountered might actually be a continuation fragment.
-	 * Validate the handle to detect it and drop it, or else the upper
-	 * layer will get garbage for a while.
-	 */
-
-	handle = hci_handle(__le16_to_cpu(hdr->handle));
-
-	switch (hci_conn_lookup_type(hdev, handle)) {
-	case SCO_LINK:
-	case ESCO_LINK:
-		return true;
-	default:
-		return false;
-	}
-}
-
-static int btusb_recv_isoc(struct btusb_data *data, void *buffer, int count)
-{
-	struct sk_buff *skb;
-	unsigned long flags;
-	int err = 0;
-
-	spin_lock_irqsave(&data->rxlock, flags);
-	skb = data->sco_skb;
-
-	while (count) {
-		int len;
-
-		if (!skb) {
-			skb = bt_skb_alloc(HCI_MAX_SCO_SIZE, GFP_ATOMIC);
-			if (!skb) {
-				err = -ENOMEM;
-				break;
-			}
-
-			hci_skb_pkt_type(skb) = HCI_SCODATA_PKT;
-			hci_skb_expect(skb) = HCI_SCO_HDR_SIZE;
-		}
-
-		len = min_t(uint, hci_skb_expect(skb), count);
-		skb_put_data(skb, buffer, len);
-
-		count -= len;
-		buffer += len;
-		hci_skb_expect(skb) -= len;
-
-		if (skb->len == HCI_SCO_HDR_SIZE) {
-			/* Complete SCO header */
-			struct hci_sco_hdr *hdr = hci_sco_hdr(skb);
-
-			hci_skb_expect(skb) = hdr->dlen;
-
-			if (skb_tailroom(skb) < hci_skb_expect(skb) ||
-			    !btusb_validate_sco_handle(data->hdev, hdr)) {
-				kfree_skb(skb);
-				skb = NULL;
-
-				err = -EILSEQ;
-				break;
-			}
-		}
-
-		if (!hci_skb_expect(skb)) {
-			/* Complete frame */
-			hci_recv_frame(data->hdev, skb);
-			skb = NULL;
-		}
-	}
-
-	data->sco_skb = skb;
-	spin_unlock_irqrestore(&data->rxlock, flags);
-
-	return err;
-}
-
-static void btusb_intr_complete(struct urb *urb)
-{
-	struct hci_dev *hdev = urb->context;
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int err;
-
-	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
-	       urb->actual_length);
-
-	if (!test_bit(HCI_RUNNING, &hdev->flags))
-		return;
-
-	if (urb->status == 0) {
-		hdev->stat.byte_rx += urb->actual_length;
-
-		if (btusb_recv_intr(data, urb->transfer_buffer,
-				    urb->actual_length) < 0) {
-			bt_dev_err(hdev, "corrupted event packet");
-			hdev->stat.err_rx++;
-		}
-	} else if (urb->status == -ENOENT) {
-		/* Avoid suspend failed when usb_kill_urb */
-		return;
-	}
-
-	if (!test_bit(BTUSB_INTR_RUNNING, &data->flags))
-		return;
-
-	usb_mark_last_busy(data->udev);
-	usb_anchor_urb(urb, &data->intr_anchor);
-
-	err = usb_submit_urb(urb, GFP_ATOMIC);
-	if (err < 0) {
-		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected
-		 */
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
-				   urb, -err);
-		if (err != -EPERM)
-			hci_cmd_sync_cancel(hdev, -err);
-		usb_unanchor_urb(urb);
-	}
-}
-
-static int btusb_submit_intr_urb(struct hci_dev *hdev, gfp_t mem_flags)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct urb *urb;
-	unsigned char *buf;
-	unsigned int pipe;
-	int err, size;
-
-	BT_DBG("%s", hdev->name);
-
-	if (!data->intr_ep)
-		return -ENODEV;
-
-	urb = usb_alloc_urb(0, mem_flags);
-	if (!urb)
-		return -ENOMEM;
-
-	if (le16_to_cpu(data->udev->descriptor.idVendor)  == 0x0a12 &&
-	    le16_to_cpu(data->udev->descriptor.idProduct) == 0x0001)
-		/* Fake CSR devices don't seem to support sort-transter */
-		size = le16_to_cpu(data->intr_ep->wMaxPacketSize);
-	else
-		/* Use maximum HCI Event size so the USB stack handles
-		 * ZPL/short-transfer automatically.
-		 */
-		size = HCI_MAX_EVENT_SIZE;
-
-	buf = kmalloc(size, mem_flags);
-	if (!buf) {
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-
-	pipe = usb_rcvintpipe(data->udev, data->intr_ep->bEndpointAddress);
-
-	usb_fill_int_urb(urb, data->udev, pipe, buf, size,
-			 btusb_intr_complete, hdev, data->intr_ep->bInterval);
-
-	urb->transfer_flags |= URB_FREE_BUFFER;
-
-	usb_anchor_urb(urb, &data->intr_anchor);
-
-	err = usb_submit_urb(urb, mem_flags);
-	if (err < 0) {
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p submission failed (%d)",
-				   urb, -err);
-		if (err != -EPERM)
-			hci_cmd_sync_cancel(hdev, -err);
-		usb_unanchor_urb(urb);
-	}
-
-	/* Only initialize intr_interval if URB poll sync is enabled */
-	if (!data->poll_sync)
-		goto done;
-
-	/* The units are frames (milliseconds) for full and low speed devices,
-	 * and microframes (1/8 millisecond) for highspeed and SuperSpeed
-	 * devices.
-	 *
-	 * This is done once on open/resume so it shouldn't change even if
-	 * force_poll_sync changes.
-	 */
-	switch (urb->dev->speed) {
-	case USB_SPEED_SUPER_PLUS:
-	case USB_SPEED_SUPER:	/* units are 125us */
-		data->intr_interval = usecs_to_jiffies(urb->interval * 125);
-		break;
-	default:
-		data->intr_interval = msecs_to_jiffies(urb->interval);
-		break;
-	}
-
-done:
-	usb_free_urb(urb);
-
-	return err;
-}
-
-static void btusb_bulk_complete(struct urb *urb)
-{
-	struct hci_dev *hdev = urb->context;
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int err;
-
-	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
-	       urb->actual_length);
-
-	if (!test_bit(HCI_RUNNING, &hdev->flags))
-		return;
-
-	if (urb->status == 0) {
-		hdev->stat.byte_rx += urb->actual_length;
-
-		if (data->recv_bulk(data, urb->transfer_buffer,
-				    urb->actual_length) < 0) {
-			bt_dev_err(hdev, "corrupted ACL packet");
-			hdev->stat.err_rx++;
-		}
-	} else if (urb->status == -ENOENT) {
-		/* Avoid suspend failed when usb_kill_urb */
-		return;
-	}
-
-	if (!test_bit(BTUSB_BULK_RUNNING, &data->flags))
-		return;
-
-	usb_anchor_urb(urb, &data->bulk_anchor);
-	usb_mark_last_busy(data->udev);
-
-	err = usb_submit_urb(urb, GFP_ATOMIC);
-	if (err < 0) {
-		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected
-		 */
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
-				   urb, -err);
-		usb_unanchor_urb(urb);
-	}
-}
-
-static int btusb_submit_bulk_urb(struct hci_dev *hdev, gfp_t mem_flags)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct urb *urb;
-	unsigned char *buf;
-	unsigned int pipe;
-	int err, size = HCI_MAX_FRAME_SIZE;
-
-	BT_DBG("%s", hdev->name);
-
-	if (!data->bulk_rx_ep)
-		return -ENODEV;
-
-	urb = usb_alloc_urb(0, mem_flags);
-	if (!urb)
-		return -ENOMEM;
-
-	buf = kmalloc(size, mem_flags);
-	if (!buf) {
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-
-	pipe = usb_rcvbulkpipe(data->udev, data->bulk_rx_ep->bEndpointAddress);
-
-	usb_fill_bulk_urb(urb, data->udev, pipe, buf, size,
-			  btusb_bulk_complete, hdev);
-
-	urb->transfer_flags |= URB_FREE_BUFFER;
-
-	usb_mark_last_busy(data->udev);
-	usb_anchor_urb(urb, &data->bulk_anchor);
-
-	err = usb_submit_urb(urb, mem_flags);
-	if (err < 0) {
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p submission failed (%d)",
-				   urb, -err);
-		usb_unanchor_urb(urb);
-	}
-
-	usb_free_urb(urb);
-
-	return err;
-}
-
-static void btusb_isoc_complete(struct urb *urb)
-{
-	struct hci_dev *hdev = urb->context;
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int i, err;
-
-	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
-	       urb->actual_length);
-
-	if (!test_bit(HCI_RUNNING, &hdev->flags))
-		return;
-
-	if (urb->status == 0) {
-		for (i = 0; i < urb->number_of_packets; i++) {
-			unsigned int offset = urb->iso_frame_desc[i].offset;
-			unsigned int length = urb->iso_frame_desc[i].actual_length;
-
-			if (urb->iso_frame_desc[i].status)
-				continue;
-
-			hdev->stat.byte_rx += length;
-
-			if (btusb_recv_isoc(data, urb->transfer_buffer + offset,
-					    length) < 0) {
-				bt_dev_err(hdev, "corrupted SCO packet");
-				hdev->stat.err_rx++;
-			}
-		}
-	} else if (urb->status == -ENOENT) {
-		/* Avoid suspend failed when usb_kill_urb */
-		return;
-	}
-
-	if (!test_bit(BTUSB_ISOC_RUNNING, &data->flags))
-		return;
-
-	usb_anchor_urb(urb, &data->isoc_anchor);
-
-	err = usb_submit_urb(urb, GFP_ATOMIC);
-	if (err < 0) {
-		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected
-		 */
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
-				   urb, -err);
-		usb_unanchor_urb(urb);
-	}
-}
-
-static inline void __fill_isoc_descriptor_msbc(struct urb *urb, int len,
-					       int mtu, struct btusb_data *data)
-{
-	int i = 0, offset = 0;
-	unsigned int interval;
-
-	BT_DBG("len %d mtu %d", len, mtu);
-
-	/* For mSBC ALT 6 settings some chips need to transmit the data
-	 * continuously without the zero length of USB packets.
-	 */
-	if (test_bit(BTUSB_ALT6_CONTINUOUS_TX, &data->flags))
-		goto ignore_usb_alt6_packet_flow;
-
-	/* For mSBC ALT 6 setting the host will send the packet at continuous
-	 * flow. As per core spec 5, vol 4, part B, table 2.1. For ALT setting
-	 * 6 the HCI PACKET INTERVAL should be 7.5ms for every usb packets.
-	 * To maintain the rate we send 63bytes of usb packets alternatively for
-	 * 7ms and 8ms to maintain the rate as 7.5ms.
-	 */
-	if (data->usb_alt6_packet_flow) {
-		interval = 7;
-		data->usb_alt6_packet_flow = false;
-	} else {
-		interval = 6;
-		data->usb_alt6_packet_flow = true;
-	}
-
-	for (i = 0; i < interval; i++) {
-		urb->iso_frame_desc[i].offset = offset;
-		urb->iso_frame_desc[i].length = offset;
-	}
-
-ignore_usb_alt6_packet_flow:
-	if (len && i < BTUSB_MAX_ISOC_FRAMES) {
-		urb->iso_frame_desc[i].offset = offset;
-		urb->iso_frame_desc[i].length = len;
-		i++;
-	}
-
-	urb->number_of_packets = i;
-}
-
-static inline void __fill_isoc_descriptor(struct urb *urb, int len, int mtu)
-{
-	int i, offset = 0;
-
-	BT_DBG("len %d mtu %d", len, mtu);
-
-	for (i = 0; i < BTUSB_MAX_ISOC_FRAMES && len >= mtu;
-					i++, offset += mtu, len -= mtu) {
-		urb->iso_frame_desc[i].offset = offset;
-		urb->iso_frame_desc[i].length = mtu;
-	}
-
-	if (len && i < BTUSB_MAX_ISOC_FRAMES) {
-		urb->iso_frame_desc[i].offset = offset;
-		urb->iso_frame_desc[i].length = len;
-		i++;
-	}
-
-	urb->number_of_packets = i;
-}
-
-static int btusb_submit_isoc_urb(struct hci_dev *hdev, gfp_t mem_flags)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct urb *urb;
-	unsigned char *buf;
-	unsigned int pipe;
-	int err, size;
-
-	BT_DBG("%s", hdev->name);
-
-	if (!data->isoc_rx_ep)
-		return -ENODEV;
-
-	urb = usb_alloc_urb(BTUSB_MAX_ISOC_FRAMES, mem_flags);
-	if (!urb)
-		return -ENOMEM;
-
-	size = le16_to_cpu(data->isoc_rx_ep->wMaxPacketSize) *
-						BTUSB_MAX_ISOC_FRAMES;
-
-	buf = kmalloc(size, mem_flags);
-	if (!buf) {
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-
-	pipe = usb_rcvisocpipe(data->udev, data->isoc_rx_ep->bEndpointAddress);
-
-	usb_fill_int_urb(urb, data->udev, pipe, buf, size, btusb_isoc_complete,
-			 hdev, data->isoc_rx_ep->bInterval);
-
-	urb->transfer_flags = URB_FREE_BUFFER | URB_ISO_ASAP;
-
-	__fill_isoc_descriptor(urb, size,
-			       le16_to_cpu(data->isoc_rx_ep->wMaxPacketSize));
-
-	usb_anchor_urb(urb, &data->isoc_anchor);
-
-	err = usb_submit_urb(urb, mem_flags);
-	if (err < 0) {
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p submission failed (%d)",
-				   urb, -err);
-		usb_unanchor_urb(urb);
-	}
-
-	usb_free_urb(urb);
-
-	return err;
-}
-
-static void btusb_diag_complete(struct urb *urb)
-{
-	struct hci_dev *hdev = urb->context;
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int err;
-
-	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
-	       urb->actual_length);
-
-	if (urb->status == 0) {
-		struct sk_buff *skb;
-
-		skb = bt_skb_alloc(urb->actual_length, GFP_ATOMIC);
-		if (skb) {
-			skb_put_data(skb, urb->transfer_buffer,
-				     urb->actual_length);
-			hci_recv_diag(hdev, skb);
-		}
-	} else if (urb->status == -ENOENT) {
-		/* Avoid suspend failed when usb_kill_urb */
-		return;
-	}
-
-	if (!test_bit(BTUSB_DIAG_RUNNING, &data->flags))
-		return;
-
-	usb_anchor_urb(urb, &data->diag_anchor);
-	usb_mark_last_busy(data->udev);
-
-	err = usb_submit_urb(urb, GFP_ATOMIC);
-	if (err < 0) {
-		/* -EPERM: urb is being killed;
-		 * -ENODEV: device got disconnected
-		 */
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
-				   urb, -err);
-		usb_unanchor_urb(urb);
-	}
-}
-
-static int btusb_submit_diag_urb(struct hci_dev *hdev, gfp_t mem_flags)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct urb *urb;
-	unsigned char *buf;
-	unsigned int pipe;
-	int err, size = HCI_MAX_FRAME_SIZE;
-
-	BT_DBG("%s", hdev->name);
-
-	if (!data->diag_rx_ep)
-		return -ENODEV;
-
-	urb = usb_alloc_urb(0, mem_flags);
-	if (!urb)
-		return -ENOMEM;
-
-	buf = kmalloc(size, mem_flags);
-	if (!buf) {
-		usb_free_urb(urb);
-		return -ENOMEM;
-	}
-
-	pipe = usb_rcvbulkpipe(data->udev, data->diag_rx_ep->bEndpointAddress);
-
-	usb_fill_bulk_urb(urb, data->udev, pipe, buf, size,
-			  btusb_diag_complete, hdev);
-
-	urb->transfer_flags |= URB_FREE_BUFFER;
-
-	usb_mark_last_busy(data->udev);
-	usb_anchor_urb(urb, &data->diag_anchor);
-
-	err = usb_submit_urb(urb, mem_flags);
-	if (err < 0) {
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p submission failed (%d)",
-				   urb, -err);
-		usb_unanchor_urb(urb);
-	}
-
-	usb_free_urb(urb);
-
-	return err;
-}
-
-static void btusb_tx_complete(struct urb *urb)
-{
-	struct sk_buff *skb = urb->context;
-	struct hci_dev *hdev = (struct hci_dev *)skb->dev;
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	unsigned long flags;
-
-	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
-	       urb->actual_length);
-
-	if (!test_bit(HCI_RUNNING, &hdev->flags))
-		goto done;
-
-	if (!urb->status) {
-		hdev->stat.byte_tx += urb->transfer_buffer_length;
-	} else {
-		if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT)
-			hci_cmd_sync_cancel(hdev, -urb->status);
-		hdev->stat.err_tx++;
-	}
-
-done:
-	spin_lock_irqsave(&data->txlock, flags);
-	data->tx_in_flight--;
-	spin_unlock_irqrestore(&data->txlock, flags);
-
-	kfree(urb->setup_packet);
-
-	kfree_skb(skb);
-}
-
-static void btusb_isoc_tx_complete(struct urb *urb)
-{
-	struct sk_buff *skb = urb->context;
-	struct hci_dev *hdev = (struct hci_dev *)skb->dev;
-
-	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
-	       urb->actual_length);
-
-	if (!test_bit(HCI_RUNNING, &hdev->flags))
-		goto done;
-
-	if (!urb->status)
-		hdev->stat.byte_tx += urb->transfer_buffer_length;
-	else
-		hdev->stat.err_tx++;
-
-done:
-	kfree(urb->setup_packet);
-
-	kfree_skb(skb);
-}
-
-static int btusb_open(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int err;
-
-	BT_DBG("%s", hdev->name);
-
-	err = usb_autopm_get_interface(data->intf);
-	if (err < 0)
-		return err;
-
-	/* Patching USB firmware files prior to starting any URBs of HCI path
-	 * It is more safe to use USB bulk channel for downloading USB patch
-	 */
-	if (data->setup_on_usb) {
-		err = data->setup_on_usb(hdev);
-		if (err < 0)
-			goto setup_fail;
-	}
-
-	data->intf->needs_remote_wakeup = 1;
-
-	if (test_and_set_bit(BTUSB_INTR_RUNNING, &data->flags))
-		goto done;
-
-	err = btusb_submit_intr_urb(hdev, GFP_KERNEL);
-	if (err < 0)
-		goto failed;
-
-	err = btusb_submit_bulk_urb(hdev, GFP_KERNEL);
-	if (err < 0) {
-		usb_kill_anchored_urbs(&data->intr_anchor);
-		goto failed;
-	}
-
-	set_bit(BTUSB_BULK_RUNNING, &data->flags);
-	btusb_submit_bulk_urb(hdev, GFP_KERNEL);
-
-	if (data->diag) {
-		if (!btusb_submit_diag_urb(hdev, GFP_KERNEL))
-			set_bit(BTUSB_DIAG_RUNNING, &data->flags);
-	}
-
-done:
-	usb_autopm_put_interface(data->intf);
-	return 0;
-
-failed:
-	clear_bit(BTUSB_INTR_RUNNING, &data->flags);
-setup_fail:
-	usb_autopm_put_interface(data->intf);
-	return err;
-}
-
-static void btusb_stop_traffic(struct btusb_data *data)
-{
-	usb_kill_anchored_urbs(&data->intr_anchor);
-	usb_kill_anchored_urbs(&data->bulk_anchor);
-	usb_kill_anchored_urbs(&data->isoc_anchor);
-	usb_kill_anchored_urbs(&data->diag_anchor);
-	usb_kill_anchored_urbs(&data->ctrl_anchor);
-}
-
-static void btusb_prepare_reset(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-
-	btusb_stop_traffic(data);
-	usb_kill_anchored_urbs(&data->tx_anchor);
-}
-
-static int btusb_close(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int err;
-
-	BT_DBG("%s", hdev->name);
-
-	cancel_work_sync(&data->work);
-	cancel_work_sync(&data->waker);
-
-	clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
-	clear_bit(BTUSB_BULK_RUNNING, &data->flags);
-	clear_bit(BTUSB_INTR_RUNNING, &data->flags);
-	clear_bit(BTUSB_DIAG_RUNNING, &data->flags);
-
-	btusb_stop_traffic(data);
-
-	/* rx_work must only be canceled once the URBs that can rearm it are
-	 * gone, and it must be canceled synchronously since btusb_disconnect()
-	 * frees the btusb_data it dereferences right after hci_unregister_dev().
-	 */
-	cancel_delayed_work_sync(&data->rx_work);
-
-	skb_queue_purge(&data->acl_q);
-
-	btusb_free_frags(data);
-
-	err = usb_autopm_get_interface(data->intf);
-	if (err < 0)
-		goto failed;
-
-	data->intf->needs_remote_wakeup = 0;
-
-	/* Enable remote wake up for auto-suspend */
-	if (test_bit(BTUSB_WAKEUP_AUTOSUSPEND, &data->flags))
-		data->intf->needs_remote_wakeup = 1;
-
-	usb_autopm_put_interface(data->intf);
-
-failed:
-	usb_scuttle_anchored_urbs(&data->deferred);
-	return 0;
-}
-
-static int btusb_flush(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-
-	BT_DBG("%s", hdev->name);
-
-	cancel_delayed_work_sync(&data->rx_work);
-
-	skb_queue_purge(&data->acl_q);
-
-	usb_kill_anchored_urbs(&data->tx_anchor);
-	btusb_free_frags(data);
-
-	return 0;
-}
-
-static struct urb *alloc_ctrl_urb(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct usb_ctrlrequest *dr;
-	struct urb *urb;
-	unsigned int pipe;
-
-	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb)
-		return ERR_PTR(-ENOMEM);
-
-	dr = kmalloc_obj(*dr);
-	if (!dr) {
-		usb_free_urb(urb);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	dr->bRequestType = data->cmdreq_type;
-	dr->bRequest     = data->cmdreq;
-	dr->wIndex       = 0;
-	dr->wValue       = 0;
-	dr->wLength      = __cpu_to_le16(skb->len);
-
-	pipe = usb_sndctrlpipe(data->udev, 0x00);
-
-	usb_fill_control_urb(urb, data->udev, pipe, (void *)dr,
-			     skb->data, skb->len, btusb_tx_complete, skb);
-
-	skb->dev = (void *)hdev;
-
-	return urb;
-}
-
-static struct urb *alloc_bulk_urb(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct urb *urb;
-	unsigned int pipe;
-
-	if (!data->bulk_tx_ep)
-		return ERR_PTR(-ENODEV);
-
-	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb)
-		return ERR_PTR(-ENOMEM);
-
-	pipe = usb_sndbulkpipe(data->udev, data->bulk_tx_ep->bEndpointAddress);
-
-	usb_fill_bulk_urb(urb, data->udev, pipe,
-			  skb->data, skb->len, btusb_tx_complete, skb);
-
-	skb->dev = (void *)hdev;
-
-	return urb;
-}
-
-static struct urb *alloc_isoc_urb(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct urb *urb;
-	unsigned int pipe;
-
-	if (!data->isoc_tx_ep)
-		return ERR_PTR(-ENODEV);
-
-	urb = usb_alloc_urb(BTUSB_MAX_ISOC_FRAMES, GFP_KERNEL);
-	if (!urb)
-		return ERR_PTR(-ENOMEM);
-
-	pipe = usb_sndisocpipe(data->udev, data->isoc_tx_ep->bEndpointAddress);
-
-	usb_fill_int_urb(urb, data->udev, pipe,
-			 skb->data, skb->len, btusb_isoc_tx_complete,
-			 skb, data->isoc_tx_ep->bInterval);
-
-	urb->transfer_flags  = URB_ISO_ASAP;
-
-	if (data->isoc_altsetting == 6)
-		__fill_isoc_descriptor_msbc(urb, skb->len,
-					    le16_to_cpu(data->isoc_tx_ep->wMaxPacketSize),
-					    data);
-	else
-		__fill_isoc_descriptor(urb, skb->len,
-				       le16_to_cpu(data->isoc_tx_ep->wMaxPacketSize));
-	skb->dev = (void *)hdev;
-
-	return urb;
-}
-
-static int submit_tx_urb(struct hci_dev *hdev, struct urb *urb)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int err;
-
-	usb_anchor_urb(urb, &data->tx_anchor);
-
-	err = usb_submit_urb(urb, GFP_KERNEL);
-	if (err < 0) {
-		if (err != -EPERM && err != -ENODEV)
-			bt_dev_err(hdev, "urb %p submission failed (%d)",
-				   urb, -err);
-		kfree(urb->setup_packet);
-		usb_unanchor_urb(urb);
-	} else {
-		usb_mark_last_busy(data->udev);
-	}
-
-	usb_free_urb(urb);
-	return err;
-}
-
-static int submit_or_queue_tx_urb(struct hci_dev *hdev, struct urb *urb)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	unsigned long flags;
-	bool suspending;
-
-	spin_lock_irqsave(&data->txlock, flags);
-	suspending = test_bit(BTUSB_SUSPENDING, &data->flags);
-	if (!suspending)
-		data->tx_in_flight++;
-	spin_unlock_irqrestore(&data->txlock, flags);
-
-	if (!suspending)
-		return submit_tx_urb(hdev, urb);
-
-	usb_anchor_urb(urb, &data->deferred);
-	schedule_work(&data->waker);
-
-	usb_free_urb(urb);
-	return 0;
-}
-
-static int btusb_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct urb *urb;
-
-	BT_DBG("%s", hdev->name);
-
-	switch (hci_skb_pkt_type(skb)) {
-	case HCI_COMMAND_PKT:
-		urb = alloc_ctrl_urb(hdev, skb);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		hdev->stat.cmd_tx++;
-		return submit_or_queue_tx_urb(hdev, urb);
-
-	case HCI_ACLDATA_PKT:
-		urb = alloc_bulk_urb(hdev, skb);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		hdev->stat.acl_tx++;
-		return submit_or_queue_tx_urb(hdev, urb);
-
-	case HCI_SCODATA_PKT:
-		if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
-		    hci_conn_num(hdev, SCO_LINK) < 1)
-			return -ENODEV;
-
-		urb = alloc_isoc_urb(hdev, skb);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		hdev->stat.sco_tx++;
-		return submit_tx_urb(hdev, urb);
-
-	case HCI_ISODATA_PKT:
-		urb = alloc_bulk_urb(hdev, skb);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		return submit_or_queue_tx_urb(hdev, urb);
-	}
-
-	return -EILSEQ;
-}
-
-static void btusb_notify(struct hci_dev *hdev, unsigned int evt)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-
-	BT_DBG("%s evt %d", hdev->name, evt);
-
-	if (hci_conn_num(hdev, SCO_LINK) != data->sco_num) {
-		data->sco_num = hci_conn_num(hdev, SCO_LINK);
-		data->air_mode = evt;
-		schedule_work(&data->work);
-	}
-}
-
-static inline int __set_isoc_interface(struct hci_dev *hdev, int altsetting)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct usb_interface *intf = data->isoc;
-	struct usb_endpoint_descriptor *ep_desc;
-	int i, err;
-
-	if (!data->isoc)
-		return -ENODEV;
-
-	err = usb_set_interface(data->udev, data->isoc_ifnum, altsetting);
-	if (err < 0) {
-		bt_dev_err(hdev, "setting interface failed (%d)", -err);
-		return err;
-	}
-
-	data->isoc_altsetting = altsetting;
-
-	data->isoc_tx_ep = NULL;
-	data->isoc_rx_ep = NULL;
-
-	for (i = 0; i < intf->cur_altsetting->desc.bNumEndpoints; i++) {
-		ep_desc = &intf->cur_altsetting->endpoint[i].desc;
-
-		if (!data->isoc_tx_ep && usb_endpoint_is_isoc_out(ep_desc)) {
-			data->isoc_tx_ep = ep_desc;
-			continue;
-		}
-
-		if (!data->isoc_rx_ep && usb_endpoint_is_isoc_in(ep_desc)) {
-			data->isoc_rx_ep = ep_desc;
-			continue;
-		}
-	}
-
-	if (!data->isoc_tx_ep || !data->isoc_rx_ep) {
-		bt_dev_err(hdev, "invalid SCO descriptors");
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-static int btusb_switch_alt_setting(struct hci_dev *hdev, int new_alts)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int err;
-
-	if (data->isoc_altsetting != new_alts) {
-		unsigned long flags;
-
-		clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
-		usb_kill_anchored_urbs(&data->isoc_anchor);
-
-		/* When isochronous alternate setting needs to be
-		 * changed, because SCO connection has been added
-		 * or removed, a packet fragment may be left in the
-		 * reassembling state. This could lead to wrongly
-		 * assembled fragments.
-		 *
-		 * Clear outstanding fragment when selecting a new
-		 * alternate setting.
-		 */
-		spin_lock_irqsave(&data->rxlock, flags);
-		dev_kfree_skb_irq(data->sco_skb);
-		data->sco_skb = NULL;
-		spin_unlock_irqrestore(&data->rxlock, flags);
-
-		err = __set_isoc_interface(hdev, new_alts);
-		if (err < 0)
-			return err;
-	}
-
-	if (!test_and_set_bit(BTUSB_ISOC_RUNNING, &data->flags)) {
-		if (btusb_submit_isoc_urb(hdev, GFP_KERNEL) < 0)
-			clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
-		else
-			btusb_submit_isoc_urb(hdev, GFP_KERNEL);
-	}
-
-	return 0;
-}
-
-static struct usb_host_interface *btusb_find_altsetting(struct btusb_data *data,
-							int alt)
-{
-	struct usb_interface *intf = data->isoc;
-	int i;
-
-	BT_DBG("Looking for Alt no :%d", alt);
-
-	if (!intf)
-		return NULL;
-
-	for (i = 0; i < intf->num_altsetting; i++) {
-		if (intf->altsetting[i].desc.bAlternateSetting == alt)
-			return &intf->altsetting[i];
-	}
-
-	return NULL;
-}
-
-static void btusb_work(struct work_struct *work)
-{
-	struct btusb_data *data = container_of(work, struct btusb_data, work);
-	struct hci_dev *hdev = data->hdev;
-	int new_alts = 0;
-	int err;
-
-	if (data->sco_num > 0) {
-		if (!test_bit(BTUSB_DID_ISO_RESUME, &data->flags)) {
-			err = usb_autopm_get_interface(data->isoc ? data->isoc : data->intf);
-			if (err < 0) {
-				clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
-				usb_kill_anchored_urbs(&data->isoc_anchor);
-				return;
-			}
-
-			set_bit(BTUSB_DID_ISO_RESUME, &data->flags);
-		}
-
-		if (data->air_mode == HCI_NOTIFY_ENABLE_SCO_CVSD) {
-			if (hdev->voice_setting & 0x0020) {
-				static const int alts[3] = { 2, 4, 5 };
-				unsigned int sco_idx;
-
-				sco_idx = min_t(unsigned int, data->sco_num - 1,
-						ARRAY_SIZE(alts) - 1);
-				new_alts = alts[sco_idx];
-			} else {
-				new_alts = data->sco_num;
-			}
-		} else if (data->air_mode == HCI_NOTIFY_ENABLE_SCO_TRANSP) {
-			/* Bluetooth USB spec recommends alt 6 (63 bytes), but
-			 * many adapters do not support it.  Alt 1 appears to
-			 * work for all adapters that do not have alt 6, and
-			 * which work with WBS at all.  Some devices prefer
-			 * alt 3 (HCI payload >= 60 Bytes let air packet
-			 * data satisfy 60 bytes), requiring
-			 * MTU >= 3 (packets) * 25 (size) - 3 (headers) = 72
-			 * see also Core spec 5, vol 4, B 2.1.1 & Table 2.1.
-			 */
-			if (btusb_find_altsetting(data, 6))
-				new_alts = 6;
-			else if (btusb_find_altsetting(data, 3) &&
-				 hdev->sco_mtu >= 72 &&
-				 test_bit(BTUSB_USE_ALT3_FOR_WBS, &data->flags))
-				new_alts = 3;
-			else
-				new_alts = 1;
-		}
-
-		if (btusb_switch_alt_setting(hdev, new_alts) < 0)
-			bt_dev_err(hdev, "set USB alt:(%d) failed!", new_alts);
-	} else {
-		usb_kill_anchored_urbs(&data->isoc_anchor);
-
-		if (test_and_clear_bit(BTUSB_ISOC_RUNNING, &data->flags))
-			__set_isoc_interface(hdev, 0);
-
-		if (test_and_clear_bit(BTUSB_DID_ISO_RESUME, &data->flags))
-			usb_autopm_put_interface(data->isoc ? data->isoc : data->intf);
-	}
-}
-
-static void btusb_waker(struct work_struct *work)
-{
-	struct btusb_data *data = container_of(work, struct btusb_data, waker);
-	int err;
-
-	err = usb_autopm_get_interface(data->intf);
-	if (err < 0)
-		return;
-
-	usb_autopm_put_interface(data->intf);
-}
-
-static void btusb_rx_work(struct work_struct *work)
-{
-	struct btusb_data *data = container_of(work, struct btusb_data,
-					       rx_work.work);
-	struct sk_buff *skb;
-
-	/* Dequeue ACL data received during the interval */
-	while ((skb = skb_dequeue(&data->acl_q)))
-		data->recv_acl(data->hdev, skb);
-}
-
-static int btusb_setup_bcm92035(struct hci_dev *hdev)
-{
-	struct sk_buff *skb;
-	u8 val = 0x00;
-
-	BT_DBG("%s", hdev->name);
-
-	skb = __hci_cmd_sync(hdev, 0xfc3b, 1, &val, HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb))
-		bt_dev_err(hdev, "BCM92035 command failed (%ld)", PTR_ERR(skb));
-	else
-		kfree_skb(skb);
-
-	return 0;
-}
-
-static int btusb_setup_csr(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	u16 bcdDevice = le16_to_cpu(data->udev->descriptor.bcdDevice);
-	struct hci_rp_read_local_version *rp;
-	struct sk_buff *skb;
-	bool is_fake = false;
-	int ret;
-
-	BT_DBG("%s", hdev->name);
-
-	skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_VERSION, 0, NULL,
-			     HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb)) {
-		int err = PTR_ERR(skb);
-
-		bt_dev_err(hdev, "CSR: Local version failed (%d)", err);
-		return err;
-	}
-
-	rp = skb_pull_data(skb, sizeof(*rp));
-	if (!rp) {
-		bt_dev_err(hdev, "CSR: Local version length mismatch");
-		kfree_skb(skb);
-		return -EIO;
-	}
-
-	bt_dev_info(hdev, "CSR: Setting up dongle with HCI ver=%u rev=%04x",
-		    rp->hci_ver, le16_to_cpu(rp->hci_rev));
-
-	bt_dev_info(hdev, "LMP ver=%u subver=%04x; manufacturer=%u",
-		    rp->lmp_ver, le16_to_cpu(rp->lmp_subver),
-		    le16_to_cpu(rp->manufacturer));
-
-	/* Detect a wide host of Chinese controllers that aren't CSR.
-	 *
-	 * Known fake bcdDevices: 0x0100, 0x0134, 0x1915, 0x2520, 0x7558, 0x8891
-	 *
-	 * The main thing they have in common is that these are really popular low-cost
-	 * options that support newer Bluetooth versions but rely on heavy VID/PID
-	 * squatting of this poor old Bluetooth 1.1 device. Even sold as such.
-	 *
-	 * We detect actual CSR devices by checking that the HCI manufacturer code
-	 * is Cambridge Silicon Radio (10) and ensuring that LMP sub-version and
-	 * HCI rev values always match. As they both store the firmware number.
-	 */
-	if (le16_to_cpu(rp->manufacturer) != 10 ||
-	    le16_to_cpu(rp->hci_rev) != le16_to_cpu(rp->lmp_subver))
-		is_fake = true;
-
-	/* Known legit CSR firmware build numbers and their supported BT versions:
-	 * - 1.1 (0x1) -> 0x0073, 0x020d, 0x033c, 0x034e
-	 * - 1.2 (0x2) ->                 0x04d9, 0x0529
-	 * - 2.0 (0x3) ->         0x07a6, 0x07ad, 0x0c5c
-	 * - 2.1 (0x4) ->         0x149c, 0x1735, 0x1899 (0x1899 is a BlueCore4-External)
-	 * - 4.0 (0x6) ->         0x1d86, 0x2031, 0x22bb
-	 *
-	 * e.g. Real CSR dongles with LMP subversion 0x73 are old enough that
-	 *      support BT 1.1 only; so it's a dead giveaway when some
-	 *      third-party BT 4.0 dongle reuses it.
-	 */
-	else if (le16_to_cpu(rp->lmp_subver) <= 0x034e &&
-		 rp->hci_ver > BLUETOOTH_VER_1_1)
-		is_fake = true;
-
-	else if (le16_to_cpu(rp->lmp_subver) <= 0x0529 &&
-		 rp->hci_ver > BLUETOOTH_VER_1_2)
-		is_fake = true;
-
-	else if (le16_to_cpu(rp->lmp_subver) <= 0x0c5c &&
-		 rp->hci_ver > BLUETOOTH_VER_2_0)
-		is_fake = true;
-
-	else if (le16_to_cpu(rp->lmp_subver) <= 0x1899 &&
-		 rp->hci_ver > BLUETOOTH_VER_2_1)
-		is_fake = true;
-
-	else if (le16_to_cpu(rp->lmp_subver) <= 0x22bb &&
-		 rp->hci_ver > BLUETOOTH_VER_4_0)
-		is_fake = true;
-
-	/* Other clones which beat all the above checks */
-	else if (bcdDevice == 0x0134 &&
-		 le16_to_cpu(rp->lmp_subver) == 0x0c5c &&
-		 rp->hci_ver == BLUETOOTH_VER_2_0)
-		is_fake = true;
-
-	if (is_fake) {
-		bt_dev_warn(hdev, "CSR: Unbranded CSR clone detected; adding workarounds and force-suspending once...");
-
-		/* Generally these clones have big discrepancies between
-		 * advertised features and what's actually supported.
-		 * Probably will need to be expanded in the future;
-		 * without these the controller will lock up.
-		 */
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_STORED_LINK_KEY);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ERR_DATA_REPORTING);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_FILTER_CLEAR_ALL);
-		hci_set_quirk(hdev, HCI_QUIRK_NO_SUSPEND_NOTIFIER);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_VOICE_SETTING);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_PAGE_SCAN_TYPE);
-
-		/* Clear the reset quirk since this is not an actual
-		 * early Bluetooth 1.1 device from CSR.
-		 */
-		hci_clear_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
-		hci_clear_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
-
-		/*
-		 * Special workaround for these BT 4.0 chip clones, and potentially more:
-		 *
-		 * - 0x0134: a Barrot 8041a02                 (HCI rev: 0x0810 sub: 0x1012)
-		 * - 0x7558: IC markings FR3191AHAL 749H15143 (HCI rev/sub-version: 0x0709)
-		 *
-		 * These controllers are really messed-up.
-		 *
-		 * 1. Their bulk RX endpoint will never report any data unless
-		 *    the device was suspended at least once (yes, really).
-		 * 2. They will not wakeup when autosuspended and receiving data
-		 *    on their bulk RX endpoint from e.g. a keyboard or mouse
-		 *    (IOW remote-wakeup support is broken for the bulk endpoint).
-		 *
-		 * To fix 1. enable runtime-suspend, force-suspend the
-		 * HCI and then wake-it up by disabling runtime-suspend.
-		 *
-		 * To fix 2. clear the HCI's can_wake flag, this way the HCI
-		 * will still be autosuspended when it is not open.
-		 *
-		 * --
-		 *
-		 * Because these are widespread problems we prefer generic solutions; so
-		 * apply this initialization quirk to every controller that gets here,
-		 * it should be harmless. The alternative is to not work at all.
-		 */
-		pm_runtime_allow(&data->udev->dev);
-
-		ret = pm_runtime_suspend(&data->udev->dev);
-		if (ret >= 0)
-			msleep(200);
-		else
-			bt_dev_warn(hdev, "CSR: Couldn't suspend the device for our Barrot 8041a02 receive-issue workaround");
-
-		pm_runtime_forbid(&data->udev->dev);
-
-		device_set_wakeup_capable(&data->udev->dev, false);
-
-		/* Re-enable autosuspend if this was requested */
-		if (enable_autosuspend)
-			usb_enable_autosuspend(data->udev);
-	}
-
-	kfree_skb(skb);
-
-	return 0;
-}
-
-static int inject_cmd_complete(struct hci_dev *hdev, __u16 opcode)
-{
-	struct sk_buff *skb;
-	struct hci_event_hdr *hdr;
-	struct hci_ev_cmd_complete *evt;
-
-	skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL);
-	if (!skb)
-		return -ENOMEM;
-
-	hdr = skb_put(skb, sizeof(*hdr));
-	hdr->evt = HCI_EV_CMD_COMPLETE;
-	hdr->plen = sizeof(*evt) + 1;
-
-	evt = skb_put(skb, sizeof(*evt));
-	evt->ncmd = 0x01;
-	evt->opcode = cpu_to_le16(opcode);
-
-	skb_put_u8(skb, 0x00);
-
-	hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
-
-	return hci_recv_frame(hdev, skb);
-}
-
-static int btusb_recv_bulk_intel(struct btusb_data *data, void *buffer,
-				 int count)
-{
-	struct hci_dev *hdev = data->hdev;
-
-	/* When the device is in bootloader mode, then it can send
-	 * events via the bulk endpoint. These events are treated the
-	 * same way as the ones received from the interrupt endpoint.
-	 */
-	if (btintel_test_flag(hdev, INTEL_BOOTLOADER))
-		return btusb_recv_intr(data, buffer, count);
-
-	return btusb_recv_bulk(data, buffer, count);
-}
-
-static int btusb_send_frame_intel(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct urb *urb;
-
-	BT_DBG("%s", hdev->name);
-
-	switch (hci_skb_pkt_type(skb)) {
-	case HCI_COMMAND_PKT:
-		if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
-			struct hci_command_hdr *cmd = (void *)skb->data;
-			__u16 opcode = le16_to_cpu(cmd->opcode);
-
-			/* When in bootloader mode and the command 0xfc09
-			 * is received, it needs to be send down the
-			 * bulk endpoint. So allocate a bulk URB instead.
-			 */
-			if (opcode == 0xfc09)
-				urb = alloc_bulk_urb(hdev, skb);
-			else
-				urb = alloc_ctrl_urb(hdev, skb);
-
-			/* When the BTINTEL_HCI_OP_RESET command is issued to
-			 * boot into the operational firmware, it will actually
-			 * not send a command complete event. To keep the flow
-			 * control working inject that event here.
-			 */
-			if (opcode == BTINTEL_HCI_OP_RESET)
-				inject_cmd_complete(hdev, opcode);
-		} else {
-			urb = alloc_ctrl_urb(hdev, skb);
-		}
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		hdev->stat.cmd_tx++;
-		return submit_or_queue_tx_urb(hdev, urb);
-
-	case HCI_ACLDATA_PKT:
-		urb = alloc_bulk_urb(hdev, skb);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		hdev->stat.acl_tx++;
-		return submit_or_queue_tx_urb(hdev, urb);
-
-	case HCI_SCODATA_PKT:
-		if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
-		    hci_conn_num(hdev, SCO_LINK) < 1)
-			return -ENODEV;
-
-		urb = alloc_isoc_urb(hdev, skb);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		hdev->stat.sco_tx++;
-		return submit_tx_urb(hdev, urb);
-
-	case HCI_ISODATA_PKT:
-		urb = alloc_bulk_urb(hdev, skb);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		return submit_or_queue_tx_urb(hdev, urb);
-	}
-
-	return -EILSEQ;
-}
-
-static int btusb_setup_realtek(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	int ret;
-
-	ret = btrtl_setup_realtek(hdev);
-
-	if (btrealtek_test_flag(data->hdev, REALTEK_ALT6_CONTINUOUS_TX_CHIP))
-		set_bit(BTUSB_ALT6_CONTINUOUS_TX, &data->flags);
-
-	return ret;
-}
-
-static int btusb_recv_event_realtek(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	if (skb->len >= HCI_EVENT_HDR_SIZE + 1 &&
-	    skb->data[0] == HCI_EV_VENDOR &&
-	    skb->data[2] == RTK_SUB_EVENT_CODE_COREDUMP) {
-		struct rtk_dev_coredump_hdr hdr = {
-			.code = RTK_DEVCOREDUMP_CODE_MEMDUMP,
-		};
-
-		bt_dev_dbg(hdev, "RTL: received coredump vendor evt, len %u",
-			skb->len);
-
-		btusb_rtl_alloc_devcoredump(hdev, &hdr, skb->data, skb->len);
-		kfree_skb(skb);
-
-		return 0;
-	}
-
-	return hci_recv_frame(hdev, skb);
-}
-
-static void btusb_mtk_claim_iso_intf(struct btusb_data *data)
-{
-	struct btmtk_data *btmtk_data;
-	int err;
-
-	if (!data->hdev)
-		return;
-
-	btmtk_data = hci_get_priv(data->hdev);
-	if (!btmtk_data)
-		return;
-
-	if (!btmtk_data->isopkt_intf) {
-		bt_dev_err(data->hdev, "Can't claim NULL iso interface");
-		return;
-	}
-
-	/*
-	 * The function usb_driver_claim_interface() is documented to need
-	 * locks held if it's not called from a probe routine. The code here
-	 * is called from the hci_power_on workqueue, so grab the lock.
-	 */
-	device_lock(&btmtk_data->isopkt_intf->dev);
-	err = usb_driver_claim_interface(&btusb_driver,
-					 btmtk_data->isopkt_intf, data);
-	device_unlock(&btmtk_data->isopkt_intf->dev);
-	if (err < 0) {
-		btmtk_data->isopkt_intf = NULL;
-		bt_dev_err(data->hdev, "Failed to claim iso interface: %d", err);
-		return;
-	}
-
-	set_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags);
-	init_usb_anchor(&btmtk_data->isopkt_anchor);
-}
-
-static void btusb_mtk_release_iso_intf(struct hci_dev *hdev)
-{
-	struct btmtk_data *btmtk_data;
-
-	if (!hdev)
-		return;
-
-	btmtk_data = hci_get_priv(hdev);
-	if (!btmtk_data)
-		return;
-
-	if (test_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags)) {
-		usb_kill_anchored_urbs(&btmtk_data->isopkt_anchor);
-		clear_bit(BTMTK_ISOPKT_RUNNING, &btmtk_data->flags);
-
-		if (btmtk_data->isopkt_skb) {
-			dev_kfree_skb_irq(btmtk_data->isopkt_skb);
-			btmtk_data->isopkt_skb = NULL;
-		}
-
-		if (btmtk_data->isopkt_intf) {
-			usb_set_intfdata(btmtk_data->isopkt_intf, NULL);
-			usb_driver_release_interface(&btusb_driver,
-						     btmtk_data->isopkt_intf);
-			btmtk_data->isopkt_intf = NULL;
-		}
-	}
-
-	clear_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags);
-}
-
-static int btusb_mtk_disconnect(struct hci_dev *hdev)
-{
-	/* This function describes the specific additional steps taken by MediaTek
-	 * when Bluetooth usb driver's resume function is called.
-	 */
-	btusb_mtk_release_iso_intf(hdev);
-
-	return 0;
-}
-
-static int btusb_mtk_reset(struct hci_dev *hdev, void *rst_data)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct btmtk_data *btmtk_data = hci_get_priv(hdev);
-	int err;
-
-	/* It's MediaTek specific bluetooth reset mechanism via USB */
-	if (test_and_set_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags)) {
-		bt_dev_err(hdev, "last reset failed? Not resetting again");
-		return -EBUSY;
-	}
-
-	err = usb_autopm_get_interface(data->intf);
-	if (err < 0) {
-		bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err);
-		clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags);
-		return err;
-	}
-
-	/* Release MediaTek ISO data interface */
-	btusb_mtk_release_iso_intf(hdev);
-
-	btusb_prepare_reset(hdev);
-
-	/* Toggle the hard reset line. The MediaTek device is going to
-	 * yank itself off the USB and then replug. The cleanup is handled
-	 * correctly on the way out (standard USB disconnect), and the new
-	 * device is detected cleanly and bound to the driver again like
-	 * it should be.
-	 */
-	if (data->reset_gpio) {
-		gpiod_set_value_cansleep(data->reset_gpio, 1);
-		msleep(200);
-		gpiod_set_value_cansleep(data->reset_gpio, 0);
-		return 0;
-	}
-
-	err = btmtk_usb_subsys_reset(hdev, btmtk_data->dev_id);
-
-	if (test_and_set_bit(BTUSB_RESET, &data->flags)) {
-		bt_dev_err(hdev, "last usb reset failed? Resetting again");
-		usb_autopm_put_interface_no_suspend(data->intf);
-	}
-
-	usb_queue_reset_device(data->intf);
-	clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags);
-
-	return err;
-}
-
-static int btusb_send_frame_mtk(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct urb *urb;
-
-	BT_DBG("%s", hdev->name);
-
-	if (hci_skb_pkt_type(skb) == HCI_ISODATA_PKT) {
-		urb = alloc_mtk_intr_urb(hdev, skb, btusb_tx_complete);
-		if (IS_ERR(urb))
-			return PTR_ERR(urb);
-
-		return submit_or_queue_tx_urb(hdev, urb);
-	} else {
-		return btusb_send_frame(hdev, skb);
-	}
-}
-
-static inline bool platform_is_ryzen(void)
-{
-#ifdef CONFIG_X86
-	return boot_cpu_has(X86_FEATURE_ZEN);
-#else
-	return false;
-#endif
-}
-
-static inline bool is_direct_child_of_root_hub(struct usb_device *udev)
-{
-	return udev->parent == udev->bus->root_hub;
-}
-
-static int btusb_mtk_setup(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct btmtk_data *btmtk_data = hci_get_priv(hdev);
-	int err;
-
-	/* MediaTek WMT vendor cmd requiring below USB resources to
-	 * complete the handshake.
-	 */
-	btmtk_data->drv_name = btusb_driver.name;
-	btmtk_data->intf = data->intf;
-	btmtk_data->udev = data->udev;
-	btmtk_data->ctrl_anchor = &data->ctrl_anchor;
-	btmtk_data->reset_sync = btusb_mtk_reset;
-
-	/* Claim ISO data interface and endpoint */
-	if (!test_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags)) {
-		btmtk_data->isopkt_intf = usb_ifnum_to_if(data->udev, MTK_ISO_IFNUM);
-		btusb_mtk_claim_iso_intf(data);
-	}
-
-	err = btmtk_usb_setup(hdev);
-	if (err)
-		return err;
-
-	switch (btmtk_data->dev_id) {
-	case 0x7922:
-	case 0x7925:
-		/*
-		 * All reports seen to be relevant to Ryzen-based laptops. These
-		 * NICs are usually used as OEM components thanks to some sort
-		 * of reference designs.
-		 *
-		 * Their popularity on other platforms is unclear. While there
-		 * is still a chance that the quirk may exist on other
-		 * platforms, be cautious and only apply the quirk to direct
-		 * children of Ryzen platforms's root hubs for the time being.
-		 *
-		 * In most cases the root hub is on the SoC or PCH, which needs
-		 * the quirk. Unfortunately, this can't distinguish root hubs on
-		 * PCIe add-in cards. Such roughness should be acceptable, as
-		 * PCIe USB controller add-in cards are less commonly used
-		 * nowadays. On the other hand, applying the quirk doesn't hurt
-		 * any functionalities either, as the device can still be used
-		 * as a wakeup source if desired.
-		 *
-		 * Theoretically, we could retrieve the root hub's PCI vendor ID
-		 * with some hierarchy magic, but that's too intrusive...
-		 */
-		if (platform_is_ryzen() && is_direct_child_of_root_hub(data->udev))
-			set_bit(BTUSB_WAKEUP_BROKEN, &data->flags);
-		break;
-	}
-
-	return 0;
-}
-
-static int btusb_mtk_shutdown(struct hci_dev *hdev)
-{
-	int ret;
-
-	ret = btmtk_usb_shutdown(hdev);
-
-	/* Release MediaTek iso interface after shutdown */
-	btusb_mtk_release_iso_intf(hdev);
-
-	return ret;
-}
-
-#ifdef CONFIG_PM
-/* Configure an out-of-band gpio as wake-up pin, if specified in device tree */
-static int marvell_config_oob_wake(struct hci_dev *hdev)
-{
-	struct sk_buff *skb;
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct device *dev = &data->udev->dev;
-	u16 pin, gap, opcode;
-	int ret;
-	u8 cmd[5];
-
-	/* Move on if no wakeup pin specified */
-	if (of_property_read_u16(dev->of_node, "marvell,wakeup-pin", &pin) ||
-	    of_property_read_u16(dev->of_node, "marvell,wakeup-gap-ms", &gap))
-		return 0;
-
-	/* Vendor specific command to configure a GPIO as wake-up pin */
-	opcode = hci_opcode_pack(0x3F, 0x59);
-	cmd[0] = opcode & 0xFF;
-	cmd[1] = opcode >> 8;
-	cmd[2] = 2; /* length of parameters that follow */
-	cmd[3] = pin;
-	cmd[4] = gap; /* time in ms, for which wakeup pin should be asserted */
-
-	skb = bt_skb_alloc(sizeof(cmd), GFP_KERNEL);
-	if (!skb) {
-		bt_dev_err(hdev, "%s: No memory", __func__);
-		return -ENOMEM;
-	}
-
-	skb_put_data(skb, cmd, sizeof(cmd));
-	hci_skb_pkt_type(skb) = HCI_COMMAND_PKT;
-
-	ret = btusb_send_frame(hdev, skb);
-	if (ret) {
-		bt_dev_err(hdev, "%s: configuration failed", __func__);
-		kfree_skb(skb);
-		return ret;
-	}
-
-	return 0;
-}
-#else
-static inline int marvell_config_oob_wake(struct hci_dev *hdev)
-{
-	return 0;
-}
-#endif
-
-static int btusb_set_bdaddr_marvell(struct hci_dev *hdev,
-				    const bdaddr_t *bdaddr)
-{
-	struct sk_buff *skb;
-	u8 buf[8];
-	long ret;
-
-	buf[0] = 0xfe;
-	buf[1] = sizeof(bdaddr_t);
-	memcpy(buf + 2, bdaddr, sizeof(bdaddr_t));
-
-	skb = __hci_cmd_sync(hdev, 0xfc22, sizeof(buf), buf, HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb)) {
-		ret = PTR_ERR(skb);
-		bt_dev_err(hdev, "changing Marvell device address failed (%ld)",
-			   ret);
-		return ret;
-	}
-	kfree_skb(skb);
-
-	return 0;
-}
-
-static int btusb_set_bdaddr_ath3012(struct hci_dev *hdev,
-				    const bdaddr_t *bdaddr)
-{
-	struct sk_buff *skb;
-	u8 buf[10];
-	long ret;
-
-	buf[0] = 0x01;
-	buf[1] = 0x01;
-	buf[2] = 0x00;
-	buf[3] = sizeof(bdaddr_t);
-	memcpy(buf + 4, bdaddr, sizeof(bdaddr_t));
-
-	skb = __hci_cmd_sync(hdev, 0xfc0b, sizeof(buf), buf, HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb)) {
-		ret = PTR_ERR(skb);
-		bt_dev_err(hdev, "Change address command failed (%ld)", ret);
-		return ret;
-	}
-	kfree_skb(skb);
-
-	return 0;
-}
-
-static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev,
-				const bdaddr_t *bdaddr)
-{
-	bdaddr_t bdaddr_swapped;
-	struct sk_buff *skb;
-	long ret;
-
-	baswap(&bdaddr_swapped, bdaddr);
-
-	skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(bdaddr_swapped),
-				&bdaddr_swapped, HCI_EV_CMD_COMPLETE,
-				HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb)) {
-		ret = PTR_ERR(skb);
-		bt_dev_err(hdev, "Change address command failed (%ld)", ret);
-		return ret;
-	}
-	kfree_skb(skb);
-
-	return 0;
-}
-
-#define QCA_MEMDUMP_ACL_HANDLE 0x2EDD
-#define QCA_MEMDUMP_SIZE_MAX  0x100000
-#define QCA_MEMDUMP_VSE_CLASS 0x01
-#define QCA_MEMDUMP_MSG_TYPE 0x08
-#define QCA_MEMDUMP_PKT_SIZE 248
-#define QCA_LAST_SEQUENCE_NUM 0xffff
-
-struct qca_dump_hdr {
-	u8 vse_class;
-	u8 msg_type;
-	__le16 seqno;
-	u8 reserved;
-	union {
-		u8 data[0];
-		struct {
-			__le32 ram_dump_size;
-			u8 data0[0];
-		} __packed;
-	};
-} __packed;
-
-
-static void btusb_dump_hdr_qca(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	char buf[128];
-	struct btqca_data *btqca_data = hci_get_priv(hdev);
-	struct qca_dump_info *qca_dump_ptr = &btqca_data->qca_dump;
-
-	snprintf(buf, sizeof(buf), "Controller Name: 0x%x\n",
-			qca_dump_ptr->controller_id);
-	skb_put_data(skb, buf, strlen(buf));
-
-	snprintf(buf, sizeof(buf), "Firmware Version: 0x%x\n",
-			qca_dump_ptr->fw_version);
-	skb_put_data(skb, buf, strlen(buf));
-
-	snprintf(buf, sizeof(buf), "Driver: %s\nVendor: qca\n",
-			btusb_driver.name);
-	skb_put_data(skb, buf, strlen(buf));
-
-	snprintf(buf, sizeof(buf), "VID: 0x%x\nPID:0x%x\n",
-			qca_dump_ptr->id_vendor, qca_dump_ptr->id_product);
-	skb_put_data(skb, buf, strlen(buf));
-
-	snprintf(buf, sizeof(buf), "Lmp Subversion: 0x%x\n",
-			hdev->lmp_subver);
-	skb_put_data(skb, buf, strlen(buf));
-}
-
-static void btusb_coredump_qca(struct hci_dev *hdev)
-{
-	int err;
-	static const u8 param[] = { 0x26 };
-
-	err = __hci_cmd_send(hdev, 0xfc0c, 1, param);
-	if (err < 0)
-		bt_dev_err(hdev, "%s: triggle crash failed (%d)", __func__, err);
-}
-
-/* Return: 0 on success, negative errno on failure. */
-static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	int ret = 0;
-	unsigned int skip = 0;
-	u8 pkt_type;
-	u16 seqno;
-	u32 dump_size;
-
-	struct qca_dump_hdr *dump_hdr;
-	struct btusb_data *btdata = hci_get_drvdata(hdev);
-	struct btqca_data *btqca_data = hci_get_priv(hdev);
-	struct qca_dump_info *qca_dump_ptr = &btqca_data->qca_dump;
-	struct usb_device *udev = btdata->udev;
-
-	pkt_type = hci_skb_pkt_type(skb);
-	skip = sizeof(struct hci_event_hdr);
-	if (pkt_type == HCI_ACLDATA_PKT)
-		skip += sizeof(struct hci_acl_hdr);
-
-	skb_pull(skb, skip);
-	dump_hdr = (struct qca_dump_hdr *)skb->data;
-
-	seqno = le16_to_cpu(dump_hdr->seqno);
-	if (seqno == 0) {
-		set_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags);
-		dump_size = le32_to_cpu(dump_hdr->ram_dump_size);
-		if (!dump_size || (dump_size > QCA_MEMDUMP_SIZE_MAX)) {
-			ret = -EILSEQ;
-			bt_dev_err(hdev, "Invalid memdump size(%u)",
-				   dump_size);
-			goto out;
-		}
-
-		ret = hci_devcd_init(hdev, dump_size);
-		if (ret < 0) {
-			bt_dev_err(hdev, "memdump init error(%d)", ret);
-			goto out;
-		}
-
-		qca_dump_ptr->ram_dump_size = dump_size;
-		qca_dump_ptr->ram_dump_seqno = 0;
-
-		skb_pull(skb, offsetof(struct qca_dump_hdr, data0));
-
-		usb_disable_autosuspend(udev);
-		bt_dev_info(hdev, "%s memdump size(%u)\n",
-			    (pkt_type == HCI_ACLDATA_PKT) ? "ACL" : "event",
-			    dump_size);
-	} else {
-		skb_pull(skb, offsetof(struct qca_dump_hdr, data));
-	}
-
-	if (!qca_dump_ptr->ram_dump_size) {
-		ret = -EINVAL;
-		bt_dev_err(hdev, "memdump is not active");
-		goto out;
-	}
-
-	if ((seqno > qca_dump_ptr->ram_dump_seqno + 1) && seqno != QCA_LAST_SEQUENCE_NUM) {
-		dump_size = QCA_MEMDUMP_PKT_SIZE * (seqno - qca_dump_ptr->ram_dump_seqno - 1);
-		hci_devcd_append_pattern(hdev, 0x0, dump_size);
-		bt_dev_err(hdev,
-			   "expected memdump seqno(%u) is not received(%u)\n",
-			   qca_dump_ptr->ram_dump_seqno, seqno);
-		qca_dump_ptr->ram_dump_seqno = seqno;
-		kfree_skb(skb);
-		return ret;
-	}
-
-	hci_devcd_append(hdev, skb);
-	qca_dump_ptr->ram_dump_seqno++;
-	if (seqno == QCA_LAST_SEQUENCE_NUM) {
-		bt_dev_info(hdev,
-				"memdump done: pkts(%u), total(%u)\n",
-				qca_dump_ptr->ram_dump_seqno, qca_dump_ptr->ram_dump_size);
-
-		hci_devcd_complete(hdev);
-		goto out;
-	}
-	return ret;
-
-out:
-	if (qca_dump_ptr->ram_dump_size)
-		usb_enable_autosuspend(udev);
-	qca_dump_ptr->ram_dump_size = 0;
-	qca_dump_ptr->ram_dump_seqno = 0;
-	clear_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags);
-
-	if (ret < 0)
-		kfree_skb(skb);
-	return ret;
-}
-
-/* Return: true if the ACL packet is a dump packet, false otherwise. */
-static bool acl_pkt_is_dump_qca(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct hci_event_hdr *event_hdr;
-	struct hci_acl_hdr *acl_hdr;
-	struct qca_dump_hdr *dump_hdr;
-	struct sk_buff *clone = skb_clone(skb, GFP_ATOMIC);
-	bool is_dump = false;
-
-	if (!clone)
-		return false;
-
-	acl_hdr = skb_pull_data(clone, sizeof(*acl_hdr));
-	if (!acl_hdr || (le16_to_cpu(acl_hdr->handle) != QCA_MEMDUMP_ACL_HANDLE))
-		goto out;
-
-	event_hdr = skb_pull_data(clone, sizeof(*event_hdr));
-	if (!event_hdr || event_hdr->evt != HCI_EV_VENDOR)
-		goto out;
-
-	dump_hdr = skb_pull_data(clone, sizeof(*dump_hdr));
-	if (!dump_hdr || (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS) ||
-	   (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE))
-		goto out;
-
-	is_dump = true;
-out:
-	consume_skb(clone);
-	return is_dump;
-}
-
-/* Return: true if the event packet is a dump packet, false otherwise. */
-static bool evt_pkt_is_dump_qca(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct hci_event_hdr *event_hdr;
-	struct qca_dump_hdr *dump_hdr;
-	struct sk_buff *clone = skb_clone(skb, GFP_ATOMIC);
-	bool is_dump = false;
-
-	if (!clone)
-		return false;
-
-	event_hdr = skb_pull_data(clone, sizeof(*event_hdr));
-	if (!event_hdr || event_hdr->evt != HCI_EV_VENDOR)
-		goto out;
-
-	dump_hdr = skb_pull_data(clone, sizeof(*dump_hdr));
-	if (!dump_hdr || (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS) ||
-	   (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE))
-		goto out;
-
-	is_dump = true;
-out:
-	consume_skb(clone);
-	return is_dump;
-}
-
-static int btusb_recv_acl_qca(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	if (acl_pkt_is_dump_qca(hdev, skb))
-		return handle_dump_pkt_qca(hdev, skb);
-	return hci_recv_frame(hdev, skb);
-}
-
-static int btusb_recv_evt_qca(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	if (evt_pkt_is_dump_qca(hdev, skb))
-		return handle_dump_pkt_qca(hdev, skb);
-	return hci_recv_frame(hdev, skb);
-}
-
-
-#define QCA_DFU_PACKET_LEN	4096
-
-#define QCA_GET_TARGET_VERSION	0x09
-#define QCA_CHECK_STATUS	0x05
-#define QCA_DFU_DOWNLOAD	0x01
-
-#define QCA_SYSCFG_UPDATED	0x40
-#define QCA_PATCH_UPDATED	0x80
-#define QCA_DFU_TIMEOUT		3000
-#define QCA_FLAG_MULTI_NVM      0x80
-#define QCA_BT_RESET_WAIT_MS    100
-
-#define WCN6855_2_0_RAM_VERSION_GF 0x400c1200
-#define WCN6855_2_1_RAM_VERSION_GF 0x400c1211
-
-struct qca_version {
-	__le32	rom_version;
-	__le32	patch_version;
-	__le32	ram_version;
-	__u8	chip_id;
-	__u8	platform_id;
-	__le16	flag;
-	__u8	reserved[4];
-} __packed;
-
-struct qca_rampatch_version {
-	__le16	rom_version_high;
-	__le16  rom_version_low;
-	__le16	patch_version;
-} __packed;
-
-struct qca_device_info {
-	u32	rom_version;
-	u8	rampatch_hdr;	/* length of header in rampatch */
-	u8	nvm_hdr;	/* length of header in NVM */
-	u8	ver_offset;	/* offset of version structure in rampatch */
-};
-
-struct qca_custom_firmware {
-	u32 rom_version;
-	u16 board_id;
-	const char *subdirectory;
-};
-
-static const struct qca_device_info qca_devices_table[] = {
-	{ 0x00000100, 20, 4,  8 }, /* Rome 1.0 */
-	{ 0x00000101, 20, 4,  8 }, /* Rome 1.1 */
-	{ 0x00000200, 28, 4, 16 }, /* Rome 2.0 */
-	{ 0x00000201, 28, 4, 16 }, /* Rome 2.1 */
-	{ 0x00000300, 28, 4, 16 }, /* Rome 3.0 */
-	{ 0x00000302, 28, 4, 16 }, /* Rome 3.2 */
-	{ 0x00130100, 40, 4, 16 }, /* WCN6855 1.0 */
-	{ 0x00130200, 40, 4, 16 }, /* WCN6855 2.0 */
-	{ 0x00130201, 40, 4, 16 }, /* WCN6855 2.1 */
-	{ 0x00190200, 40, 4, 16 }, /* WCN785x 2.0 */
-};
-
-static const struct qca_custom_firmware qca_custom_btfws[] = {
-	{ 0x00130201, 0x030A, "QCA2066" },
-	{ 0x00130201, 0x030B, "QCA2066" },
-	{ },
-};
-
-static u16 qca_extract_board_id(const struct qca_version *ver)
-{
-	u16 flag = le16_to_cpu(ver->flag);
-	u16 board_id = 0;
-
-	if (((flag >> 8) & 0xff) == QCA_FLAG_MULTI_NVM) {
-		/* The board_id should be split into two bytes
-		 * The 1st byte is chip ID, and the 2nd byte is platform ID
-		 * For example, board ID 0x010A, 0x01 is platform ID. 0x0A is chip ID
-		 * we have several platforms, and platform IDs are continuously added
-		 * Platform ID:
-		 * 0x00 is for Mobile
-		 * 0x01 is for X86
-		 * 0x02 is for Automotive
-		 * 0x03 is for Consumer electronic
-		 */
-		board_id = (ver->chip_id << 8) + ver->platform_id;
-	}
-
-	/* Take 0xffff as invalid board ID */
-	if (board_id == 0xffff)
-		board_id = 0;
-
-	return board_id;
-}
-
-static const char *qca_get_fw_subdirectory(const struct qca_version *ver)
-{
-	const struct qca_custom_firmware *ptr;
-	u32 rom_ver;
-	u16 board_id;
-
-	rom_ver = le32_to_cpu(ver->rom_version);
-	board_id = qca_extract_board_id(ver);
-	if (!board_id)
-		return NULL;
-
-	for (ptr = qca_custom_btfws; ptr->rom_version; ptr++) {
-		if (ptr->rom_version == rom_ver &&
-		    ptr->board_id == board_id)
-			return ptr->subdirectory;
-	}
-
-	return NULL;
-}
-
-static int btusb_qca_send_vendor_req(struct usb_device *udev, u8 request,
-				     void *data, u16 size)
-{
-	int err;
-
-	/* Found some of USB hosts have IOT issues with ours so that we should
-	 * not wait until HCI layer is ready.
-	 */
-	err = usb_control_msg_recv(udev, 0, request, USB_TYPE_VENDOR | USB_DIR_IN,
-				   0, 0, data, size, USB_CTRL_GET_TIMEOUT,
-				   GFP_KERNEL);
-	if (err)
-		dev_err(&udev->dev, "Failed to access otp area (%d)", err);
-
-	return err;
-}
-
-static int btusb_setup_qca_download_fw(struct hci_dev *hdev,
-				       const struct firmware *firmware,
-				       size_t hdr_size)
-{
-	struct btusb_data *btdata = hci_get_drvdata(hdev);
-	struct usb_device *udev = btdata->udev;
-	size_t count, size, sent = 0;
-	int pipe, len, err;
-	u8 *buf;
-
-	buf = kmalloc(QCA_DFU_PACKET_LEN, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	count = firmware->size;
-
-	size = min_t(size_t, count, hdr_size);
-	memcpy(buf, firmware->data, size);
-
-	/* USB patches should go down to controller through USB path
-	 * because binary format fits to go down through USB channel.
-	 * USB control path is for patching headers and USB bulk is for
-	 * patch body.
-	 */
-	pipe = usb_sndctrlpipe(udev, 0);
-	err = usb_control_msg(udev, pipe, QCA_DFU_DOWNLOAD, USB_TYPE_VENDOR,
-			      0, 0, buf, size, USB_CTRL_SET_TIMEOUT);
-	if (err < 0) {
-		bt_dev_err(hdev, "Failed to send headers (%d)", err);
-		goto done;
-	}
-
-	sent += size;
-	count -= size;
-
-	/* ep2 need time to switch from function acl to function dfu,
-	 * so we add 20ms delay here.
-	 */
-	msleep(20);
-
-	while (count) {
-		size = min_t(size_t, count, QCA_DFU_PACKET_LEN);
-
-		memcpy(buf, firmware->data + sent, size);
-
-		pipe = usb_sndbulkpipe(udev, 0x02);
-		err = usb_bulk_msg(udev, pipe, buf, size, &len,
-				   QCA_DFU_TIMEOUT);
-		if (err < 0) {
-			bt_dev_err(hdev, "Failed to send body at %zd of %zd (%d)",
-				   sent, firmware->size, err);
-			break;
-		}
-
-		if (size != len) {
-			bt_dev_err(hdev, "Failed to get bulk buffer");
-			err = -EILSEQ;
-			break;
-		}
-
-		sent  += size;
-		count -= size;
-	}
-
-done:
-	kfree(buf);
-	return err;
-}
-
-static int btusb_setup_qca_load_rampatch(struct hci_dev *hdev,
-					 struct qca_version *ver,
-					 const struct qca_device_info *info)
-{
-	struct qca_rampatch_version *rver;
-	const struct firmware *fw;
-	const char *fw_subdir;
-	u32 ver_rom, ver_patch, rver_rom;
-	u16 rver_rom_low, rver_rom_high, rver_patch;
-	char fwname[80];
-	int err;
-
-	ver_rom = le32_to_cpu(ver->rom_version);
-	ver_patch = le32_to_cpu(ver->patch_version);
-
-	fw_subdir = qca_get_fw_subdirectory(ver);
-	if (fw_subdir)
-		snprintf(fwname, sizeof(fwname), "qca/%s/rampatch_usb_%08x.bin",
-			 fw_subdir, ver_rom);
-	else
-		snprintf(fwname, sizeof(fwname), "qca/rampatch_usb_%08x.bin",
-			 ver_rom);
-
-	err = request_firmware(&fw, fwname, &hdev->dev);
-	if (err) {
-		bt_dev_err(hdev, "failed to request rampatch file: %s (%d)",
-			   fwname, err);
-		return err;
-	}
-
-	bt_dev_info(hdev, "using rampatch file: %s", fwname);
-
-	rver = (struct qca_rampatch_version *)(fw->data + info->ver_offset);
-	rver_rom_low = le16_to_cpu(rver->rom_version_low);
-	rver_patch = le16_to_cpu(rver->patch_version);
-
-	if (ver_rom & ~0xffffU) {
-		rver_rom_high = le16_to_cpu(rver->rom_version_high);
-		rver_rom = rver_rom_high << 16 | rver_rom_low;
-	} else {
-		rver_rom = rver_rom_low;
-	}
-
-	bt_dev_info(hdev, "QCA: patch rome 0x%x build 0x%x, "
-		    "firmware rome 0x%x build 0x%x",
-		    rver_rom, rver_patch, ver_rom, ver_patch);
-
-	/* Allow rampatch when the patch version equals the firmware version.
-	 * A firmware download may be aborted by a transient USB error (e.g.
-	 * disconnect) after the controller updates version info but before
-	 * completion.
-	 * Allowing equal versions enables re-flashing during recovery.
-	 */
-	if (rver_rom != ver_rom || rver_patch < ver_patch) {
-		bt_dev_err(hdev, "rampatch file version did not match with firmware");
-		err = -EINVAL;
-		goto done;
-	}
-
-	err = btusb_setup_qca_download_fw(hdev, fw, info->rampatch_hdr);
-
-done:
-	release_firmware(fw);
-
-	return err;
-}
-
-static void btusb_generate_qca_nvm_name(char *fwname, size_t max_size,
-					const struct qca_version *ver)
-{
-	u32 rom_version = le32_to_cpu(ver->rom_version);
-	const char *variant, *fw_subdir;
-	int len;
-	u16 board_id;
-
-	fw_subdir = qca_get_fw_subdirectory(ver);
-	board_id = qca_extract_board_id(ver);
-
-	switch (le32_to_cpu(ver->ram_version)) {
-	case WCN6855_2_0_RAM_VERSION_GF:
-	case WCN6855_2_1_RAM_VERSION_GF:
-		variant = "_gf";
-		break;
-	default:
-		variant = NULL;
-		break;
-	}
-
-	if (fw_subdir)
-		len = snprintf(fwname, max_size, "qca/%s/nvm_usb_%08x",
-			       fw_subdir, rom_version);
-	else
-		len = snprintf(fwname, max_size, "qca/nvm_usb_%08x",
-			       rom_version);
-	if (variant)
-		len += snprintf(fwname + len, max_size - len, "%s", variant);
-	if (board_id)
-		len += snprintf(fwname + len, max_size - len, "_%04x", board_id);
-	len += snprintf(fwname + len, max_size - len, ".bin");
-}
-
-static int btusb_setup_qca_load_nvm(struct hci_dev *hdev,
-				    struct qca_version *ver,
-				    const struct qca_device_info *info)
-{
-	const struct firmware *fw;
-	char fwname[80];
-	int err;
-
-	btusb_generate_qca_nvm_name(fwname, sizeof(fwname), ver);
-
-	err = request_firmware(&fw, fwname, &hdev->dev);
-	if (err) {
-		bt_dev_err(hdev, "failed to request NVM file: %s (%d)",
-			   fwname, err);
-		return err;
-	}
-
-	bt_dev_info(hdev, "using NVM file: %s", fwname);
-
-	err = btusb_setup_qca_download_fw(hdev, fw, info->nvm_hdr);
-
-	release_firmware(fw);
-
-	return err;
-}
-
-/* identify the ROM version and check whether patches are needed */
-static bool btusb_qca_need_patch(struct usb_device *udev)
-{
-	struct qca_version ver;
-
-	if (btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
-				      sizeof(ver)))
-		return false;
-	/* only low ROM versions need patches */
-	return !(le32_to_cpu(ver.rom_version) & ~0xffffU);
-}
-
-static int btusb_setup_qca(struct hci_dev *hdev)
-{
-	struct btusb_data *btdata = hci_get_drvdata(hdev);
-	struct usb_device *udev = btdata->udev;
-	const struct qca_device_info *info = NULL;
-	struct qca_version ver;
-	u32 ver_rom;
-	u8 status;
-	int i, err;
-
-	err = btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
-					sizeof(ver));
-	if (err)
-		return err;
-
-	ver_rom = le32_to_cpu(ver.rom_version);
-
-	for (i = 0; i < ARRAY_SIZE(qca_devices_table); i++) {
-		if (ver_rom == qca_devices_table[i].rom_version)
-			info = &qca_devices_table[i];
-	}
-	if (!info) {
-		/* If the rom_version is not matched in the qca_devices_table
-		 * and the high ROM version is not zero, we assume this chip no
-		 * need to load the rampatch and nvm.
-		 */
-		if (ver_rom & ~0xffffU)
-			return 0;
-
-		bt_dev_err(hdev, "don't support firmware rome 0x%x", ver_rom);
-		return -ENODEV;
-	}
-
-	err = btusb_qca_send_vendor_req(udev, QCA_CHECK_STATUS, &status,
-					sizeof(status));
-	if (err)
-		return err;
-
-	if (!(status & QCA_PATCH_UPDATED)) {
-		err = btusb_setup_qca_load_rampatch(hdev, &ver, info);
-		if (err < 0)
-			return err;
-	}
-
-	err = btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
-					sizeof(ver));
-	if (err)
-		return err;
-
-	if (btdata->match_id->driver_info & BTUSB_QCA_WCN6855) {
-		struct btqca_data *btqca_data = hci_get_priv(hdev);
-
-		btqca_data->qca_dump.fw_version = le32_to_cpu(ver.patch_version);
-		btqca_data->qca_dump.controller_id = le32_to_cpu(ver.rom_version);
-	}
-
-	if (!(status & QCA_SYSCFG_UPDATED)) {
-		err = btusb_setup_qca_load_nvm(hdev, &ver, info);
-		if (err < 0)
-			return err;
-
-		/* WCN6855 2.1 and later will reset to apply firmware downloaded here, so
-		 * wait ~100ms for reset Done then go ahead, otherwise, it maybe
-		 * cause potential enable failure.
-		 */
-		if (info->rom_version >= 0x00130201)
-			msleep(QCA_BT_RESET_WAIT_MS);
-	}
-
-	/* Mark HCI_OP_ENHANCED_SETUP_SYNC_CONN as broken as it doesn't seem to
-	 * work with the likes of HSP/HFP mSBC.
-	 */
-	hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN);
-
-	return 0;
-}
-
-static inline int __set_diag_interface(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct usb_interface *intf = data->diag;
-	int ret;
-
-	if (!data->diag)
-		return -ENODEV;
-
-	ret = usb_find_common_endpoints(intf->cur_altsetting, &data->diag_rx_ep,
-					&data->diag_tx_ep, NULL, NULL);
-	if (ret) {
-		bt_dev_err(hdev, "invalid diagnostic descriptors");
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-static struct urb *alloc_diag_urb(struct hci_dev *hdev, bool enable)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct sk_buff *skb;
-	struct urb *urb;
-	unsigned int pipe;
-
-	if (!data->diag_tx_ep)
-		return ERR_PTR(-ENODEV);
-
-	urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!urb)
-		return ERR_PTR(-ENOMEM);
-
-	skb = bt_skb_alloc(2, GFP_KERNEL);
-	if (!skb) {
-		usb_free_urb(urb);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	skb_put_u8(skb, 0xf0);
-	skb_put_u8(skb, enable);
-
-	pipe = usb_sndbulkpipe(data->udev, data->diag_tx_ep->bEndpointAddress);
-
-	usb_fill_bulk_urb(urb, data->udev, pipe,
-			  skb->data, skb->len, btusb_tx_complete, skb);
-
-	skb->dev = (void *)hdev;
-
-	return urb;
-}
-
-static int btusb_bcm_set_diag(struct hci_dev *hdev, bool enable)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct urb *urb;
-
-	if (!data->diag)
-		return -ENODEV;
-
-	if (!test_bit(HCI_RUNNING, &hdev->flags))
-		return -ENETDOWN;
-
-	urb = alloc_diag_urb(hdev, enable);
-	if (IS_ERR(urb))
-		return PTR_ERR(urb);
-
-	return submit_or_queue_tx_urb(hdev, urb);
-}
-
-#ifdef CONFIG_PM
-static irqreturn_t btusb_oob_wake_handler(int irq, void *priv)
-{
-	struct btusb_data *data = priv;
-
-	pm_wakeup_event(&data->udev->dev, 0);
-	pm_system_wakeup();
-
-	/* Disable only if not already disabled (keep it balanced) */
-	if (test_and_clear_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags)) {
-		disable_irq_nosync(irq);
-		disable_irq_wake(irq);
-	}
-	return IRQ_HANDLED;
-}
-
-static const struct of_device_id btusb_match_table[] = {
-	{ .compatible = "usb1286,204e" },
-	{ .compatible = "usbcf3,e300" }, /* QCA6174A */
-	{ .compatible = "usb4ca,301a" }, /* QCA6174A (Lite-On) */
-	{ }
-};
-MODULE_DEVICE_TABLE(of, btusb_match_table);
-
-/* Use an oob wakeup pin? */
-static int btusb_config_oob_wake(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct device *dev = &data->udev->dev;
-	int irq, ret;
-
-	clear_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags);
-
-	if (!of_match_device(btusb_match_table, dev))
-		return 0;
-
-	/* Move on if no IRQ specified */
-	irq = of_irq_get_byname(dev->of_node, "wakeup");
-	if (irq <= 0) {
-		bt_dev_dbg(hdev, "%s: no OOB Wakeup IRQ in DT", __func__);
-		return 0;
-	}
-
-	irq_set_status_flags(irq, IRQ_NOAUTOEN);
-	ret = request_irq(irq, btusb_oob_wake_handler, 0, "OOB Wake-on-BT", data);
-	if (ret) {
-		bt_dev_err(hdev, "%s: IRQ request failed", __func__);
-		return ret;
-	}
-
-	ret = device_init_wakeup(dev, true);
-	if (ret) {
-		bt_dev_err(hdev, "%s: failed to init_wakeup", __func__);
-		goto err_free_irq;
-	}
-
-	data->oob_wake_irq = irq;
-	bt_dev_info(hdev, "OOB Wake-on-BT configured at IRQ %u", irq);
-
-	return 0;
-
-err_free_irq:
-	free_irq(irq, data);
-
-	return ret;
-}
-#else
-static inline int btusb_config_oob_wake(struct hci_dev *hdev)
-{
-	return 0;
-}
-#endif
-
-static void btusb_check_needs_reset_resume(struct usb_interface *intf)
-{
-	if (dmi_check_system(btusb_needs_reset_resume_table))
-		interface_to_usbdev(intf)->quirks |= USB_QUIRK_RESET_RESUME;
-}
-
-static bool btusb_wakeup(struct hci_dev *hdev)
-{
-	struct btusb_data *data = hci_get_drvdata(hdev);
-
-	return device_may_wakeup(&data->udev->dev);
-}
-
-static int btusb_shutdown_qca(struct hci_dev *hdev)
-{
-	int err;
-
-	err = __hci_reset_sync(hdev);
-	if (err)
-		bt_dev_err(hdev, "HCI reset during shutdown failed");
-
-	return err;
-}
-
-static ssize_t force_poll_sync_read(struct file *file, char __user *user_buf,
-				    size_t count, loff_t *ppos)
-{
-	struct btusb_data *data = file->private_data;
-	char buf[3];
-
-	buf[0] = data->poll_sync ? 'Y' : 'N';
-	buf[1] = '\n';
-	buf[2] = '\0';
-	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
-}
-
-static ssize_t force_poll_sync_write(struct file *file,
-				     const char __user *user_buf,
-				     size_t count, loff_t *ppos)
-{
-	struct btusb_data *data = file->private_data;
-	bool enable;
-	int err;
-
-	err = kstrtobool_from_user(user_buf, count, &enable);
-	if (err)
-		return err;
-
-	/* Only allow changes while the adapter is down */
-	if (test_bit(HCI_UP, &data->hdev->flags))
-		return -EPERM;
-
-	if (data->poll_sync == enable)
-		return -EALREADY;
-
-	data->poll_sync = enable;
-
-	return count;
-}
-
-static const struct file_operations force_poll_sync_fops = {
-	.owner		= THIS_MODULE,
-	.open		= simple_open,
-	.read		= force_poll_sync_read,
-	.write		= force_poll_sync_write,
-	.llseek		= default_llseek,
-};
-
-#define BTUSB_HCI_DRV_OP_SUPPORTED_ALTSETTINGS \
-		hci_opcode_pack(HCI_DRV_OGF_DRIVER_SPECIFIC, 0x0000)
-#define BTUSB_HCI_DRV_SUPPORTED_ALTSETTINGS_SIZE	0
-struct btusb_hci_drv_rp_supported_altsettings {
-	__u8	num;
-	__u8	altsettings[];
-} __packed;
-
-#define BTUSB_HCI_DRV_OP_SWITCH_ALTSETTING \
-		hci_opcode_pack(HCI_DRV_OGF_DRIVER_SPECIFIC, 0x0001)
-#define BTUSB_HCI_DRV_SWITCH_ALTSETTING_SIZE		1
-struct btusb_hci_drv_cmd_switch_altsetting {
-	__u8	altsetting;
-} __packed;
-
-static const struct {
-	u16 opcode;
-	const char *desc;
-} btusb_hci_drv_supported_commands[] = {
-	/* Common commands */
-	{ HCI_DRV_OP_READ_INFO, "Read Info" },
-
-	/* Driver specific commands */
-	{ BTUSB_HCI_DRV_OP_SUPPORTED_ALTSETTINGS, "Supported Altsettings" },
-	{ BTUSB_HCI_DRV_OP_SWITCH_ALTSETTING,     "Switch Altsetting" },
-};
-static int btusb_hci_drv_read_info(struct hci_dev *hdev, void *data,
-				   u16 data_len)
-{
-	struct hci_drv_rp_read_info *rp;
-	size_t rp_size;
-	int err, i;
-	u16 opcode, num_supported_commands =
-		ARRAY_SIZE(btusb_hci_drv_supported_commands);
-
-	rp_size = sizeof(*rp) + num_supported_commands * 2;
-
-	rp = kmalloc(rp_size, GFP_KERNEL);
-	if (!rp)
-		return -ENOMEM;
-
-	strscpy_pad(rp->driver_name, btusb_driver.name);
-
-	rp->num_supported_commands = cpu_to_le16(num_supported_commands);
-	for (i = 0; i < num_supported_commands; i++) {
-		opcode = btusb_hci_drv_supported_commands[i].opcode;
-		bt_dev_info(hdev,
-			    "Supported HCI Drv command (0x%02x|0x%04x): %s",
-			    hci_opcode_ogf(opcode),
-			    hci_opcode_ocf(opcode),
-			    btusb_hci_drv_supported_commands[i].desc);
-		rp->supported_commands[i] = cpu_to_le16(opcode);
-	}
-
-	err = hci_drv_cmd_complete(hdev, HCI_DRV_OP_READ_INFO,
-				   HCI_DRV_STATUS_SUCCESS, rp, rp_size);
-
-	kfree(rp);
-	return err;
-}
-
-static int btusb_hci_drv_supported_altsettings(struct hci_dev *hdev, void *data,
-					       u16 data_len)
-{
-	struct btusb_data *drvdata = hci_get_drvdata(hdev);
-	struct btusb_hci_drv_rp_supported_altsettings *rp;
-	size_t rp_size;
-	int err;
-	u8 i;
-
-	/* There are at most 7 alt (0 - 6) */
-	rp = kmalloc(sizeof(*rp) + 7, GFP_KERNEL);
-	if (!rp)
-		return -ENOMEM;
-
-	rp->num = 0;
-	if (!drvdata->isoc)
-		goto done;
-
-	for (i = 0; i <= 6; i++) {
-		if (btusb_find_altsetting(drvdata, i))
-			rp->altsettings[rp->num++] = i;
-	}
-
-done:
-	rp_size = sizeof(*rp) + rp->num;
-
-	err = hci_drv_cmd_complete(hdev, BTUSB_HCI_DRV_OP_SUPPORTED_ALTSETTINGS,
-				   HCI_DRV_STATUS_SUCCESS, rp, rp_size);
-	kfree(rp);
-	return err;
-}
-
-static int btusb_hci_drv_switch_altsetting(struct hci_dev *hdev, void *data,
-					   u16 data_len)
-{
-	struct btusb_hci_drv_cmd_switch_altsetting *cmd = data;
-	u8 status;
-
-	if (cmd->altsetting > 6) {
-		status = HCI_DRV_STATUS_INVALID_PARAMETERS;
-	} else {
-		if (btusb_switch_alt_setting(hdev, cmd->altsetting))
-			status = HCI_DRV_STATUS_UNSPECIFIED_ERROR;
-		else
-			status = HCI_DRV_STATUS_SUCCESS;
-	}
-
-	return hci_drv_cmd_status(hdev, BTUSB_HCI_DRV_OP_SWITCH_ALTSETTING,
-				  status);
-}
-
-static const struct hci_drv_handler btusb_hci_drv_common_handlers[] = {
-	{ btusb_hci_drv_read_info,	HCI_DRV_READ_INFO_SIZE },
-};
-
-static const struct hci_drv_handler btusb_hci_drv_specific_handlers[] = {
-	{ btusb_hci_drv_supported_altsettings,
-				BTUSB_HCI_DRV_SUPPORTED_ALTSETTINGS_SIZE },
-	{ btusb_hci_drv_switch_altsetting,
-				BTUSB_HCI_DRV_SWITCH_ALTSETTING_SIZE },
-};
-
-static struct hci_drv btusb_hci_drv = {
-	.common_handler_count	= ARRAY_SIZE(btusb_hci_drv_common_handlers),
-	.common_handlers	= btusb_hci_drv_common_handlers,
-	.specific_handler_count	= ARRAY_SIZE(btusb_hci_drv_specific_handlers),
-	.specific_handlers	= btusb_hci_drv_specific_handlers,
-};
-
-static int btusb_probe(struct usb_interface *intf,
-		       const struct usb_device_id *id)
-{
-	struct gpio_desc *reset_gpio;
-	struct btusb_data *data;
-	struct hci_dev *hdev;
-	unsigned ifnum_base;
-	int err, priv_size = 0;
-
-	BT_DBG("intf %p id %p", intf, id);
-
-	if ((id->driver_info & BTUSB_IFNUM_2) &&
-	    (intf->cur_altsetting->desc.bInterfaceNumber != 0) &&
-	    (intf->cur_altsetting->desc.bInterfaceNumber != 2))
-		return -ENODEV;
-
-	ifnum_base = intf->cur_altsetting->desc.bInterfaceNumber;
-
-	if (!id->driver_info) {
-		const struct usb_device_id *match;
-
-		match = usb_match_id(intf, quirks_table);
-		if (match)
-			id = match;
-	}
-
-	if (id->driver_info & BTUSB_IGNORE)
-		return -ENODEV;
-
-	if (id->driver_info & BTUSB_ATH3012) {
-		struct usb_device *udev = interface_to_usbdev(intf);
-
-		/* Old firmware would otherwise let ath3k driver load
-		 * patch and sysconfig files
-		 */
-		if (le16_to_cpu(udev->descriptor.bcdDevice) <= 0x0001 &&
-		    !btusb_qca_need_patch(udev))
-			return -ENODEV;
-	}
-
-	data = kzalloc_obj(*data);
-	if (!data)
-		return -ENOMEM;
-
-	data->match_id = id;
-	err = usb_find_common_endpoints(intf->cur_altsetting, &data->bulk_rx_ep,
-					&data->bulk_tx_ep, &data->intr_ep, NULL);
-	if (err)
-		goto err_free_data;
-
-	if (id->driver_info & BTUSB_AMP) {
-		data->cmdreq_type = USB_TYPE_CLASS | 0x01;
-		data->cmdreq = 0x2b;
-	} else {
-		data->cmdreq_type = USB_TYPE_CLASS;
-		data->cmdreq = 0x00;
-	}
-
-	data->udev = interface_to_usbdev(intf);
-	data->intf = intf;
-
-	INIT_WORK(&data->work, btusb_work);
-	INIT_WORK(&data->waker, btusb_waker);
-	INIT_DELAYED_WORK(&data->rx_work, btusb_rx_work);
-
-	skb_queue_head_init(&data->acl_q);
-
-	init_usb_anchor(&data->deferred);
-	init_usb_anchor(&data->tx_anchor);
-	spin_lock_init(&data->txlock);
-
-	init_usb_anchor(&data->intr_anchor);
-	init_usb_anchor(&data->bulk_anchor);
-	init_usb_anchor(&data->isoc_anchor);
-	init_usb_anchor(&data->diag_anchor);
-	init_usb_anchor(&data->ctrl_anchor);
-	spin_lock_init(&data->rxlock);
-
-	data->recv_event = hci_recv_frame;
-	data->recv_bulk = btusb_recv_bulk;
-
-	if (id->driver_info & BTUSB_INTEL_COMBINED) {
-		/* Allocate extra space for Intel device */
-		priv_size += sizeof(struct btintel_data);
-
-		/* Override the rx handlers */
-		data->recv_event = btintel_recv_event;
-		data->recv_bulk = btusb_recv_bulk_intel;
-	} else if (id->driver_info & BTUSB_REALTEK) {
-		/* Allocate extra space for Realtek device */
-		priv_size += sizeof(struct btrealtek_data);
-
-		data->recv_event = btusb_recv_event_realtek;
-	} else if (id->driver_info & BTUSB_MEDIATEK) {
-		/* Allocate extra space for Mediatek device */
-		priv_size += sizeof(struct btmtk_data);
-	} else if (id->driver_info & BTUSB_QCA_WCN6855) {
-		/* Allocate extra space for QCA WCN6855 device */
-		priv_size += sizeof(struct btqca_data);
-	}
-
-	data->recv_acl = hci_recv_frame;
-
-	hdev = hci_alloc_dev_priv(priv_size);
-	if (!hdev) {
-		err = -ENOMEM;
-		goto err_free_data;
-	}
-
-	hdev->bus = HCI_USB;
-	hci_set_drvdata(hdev, data);
-
-	data->hdev = hdev;
-
-	SET_HCIDEV_DEV(hdev, &intf->dev);
-
-	reset_gpio = gpiod_get_optional(&data->udev->dev, "reset",
-					GPIOD_OUT_LOW);
-	if (IS_ERR(reset_gpio)) {
-		err = PTR_ERR(reset_gpio);
-		goto err_free_hdev;
-	} else if (reset_gpio) {
-		data->reset_gpio = reset_gpio;
-	}
-
-	hdev->open    = btusb_open;
-	hdev->close   = btusb_close;
-	hdev->flush   = btusb_flush;
-	hdev->send    = btusb_send_frame;
-	hdev->notify  = btusb_notify;
-	hdev->wakeup  = btusb_wakeup;
-	hdev->hci_drv = &btusb_hci_drv;
-
-	err = btusb_config_oob_wake(hdev);
-	if (err)
-		goto err_put_reset;
-
-	/* Marvell devices may need a specific chip configuration */
-	if (id->driver_info & BTUSB_MARVELL && data->oob_wake_irq) {
-		err = marvell_config_oob_wake(hdev);
-		if (err)
-			goto err_disable_wakeup;
-	}
-
-	if (id->driver_info & BTUSB_CW6622)
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_STORED_LINK_KEY);
-
-	if (id->driver_info & BTUSB_BCM2045)
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_STORED_LINK_KEY);
-
-	if (id->driver_info & BTUSB_BCM92035)
-		hdev->setup = btusb_setup_bcm92035;
-
-	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_BCM) &&
-	    (id->driver_info & BTUSB_BCM_PATCHRAM)) {
-		hdev->manufacturer = 15;
-		hdev->setup = btbcm_setup_patchram;
-		hdev->set_diag = btusb_bcm_set_diag;
-		hdev->set_bdaddr = btbcm_set_bdaddr;
-
-		/* Broadcom LM_DIAG Interface numbers are hardcoded */
-		data->diag = usb_ifnum_to_if(data->udev, ifnum_base + 2);
-	}
-
-	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_BCM) &&
-	    (id->driver_info & BTUSB_BCM_APPLE)) {
-		hdev->manufacturer = 15;
-		hdev->setup = btbcm_setup_apple;
-		hdev->set_diag = btusb_bcm_set_diag;
-
-		/* Broadcom LM_DIAG Interface numbers are hardcoded */
-		data->diag = usb_ifnum_to_if(data->udev, ifnum_base + 2);
-	}
-
-	/* Combined Intel Device setup to support multiple setup routine */
-	if (id->driver_info & BTUSB_INTEL_COMBINED) {
-		err = btintel_configure_setup(hdev, btusb_driver.name);
-		if (err)
-			goto err_kill_tx_urbs;
-
-		/* Transport specific configuration */
-		hdev->send = btusb_send_frame_intel;
-		hdev->reset = btusb_intel_reset;
-
-		if (id->driver_info & BTUSB_INTEL_NO_WBS_SUPPORT)
-			btintel_set_flag(hdev, INTEL_ROM_LEGACY_NO_WBS_SUPPORT);
-
-		if (id->driver_info & BTUSB_INTEL_BROKEN_INITIAL_NCMD)
-			btintel_set_flag(hdev, INTEL_BROKEN_INITIAL_NCMD);
-
-		if (id->driver_info & BTUSB_INTEL_BROKEN_SHUTDOWN_LED)
-			btintel_set_flag(hdev, INTEL_BROKEN_SHUTDOWN_LED);
-	}
-
-	if (id->driver_info & BTUSB_MARVELL)
-		hdev->set_bdaddr = btusb_set_bdaddr_marvell;
-
-	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_MTK) &&
-	    (id->driver_info & BTUSB_MEDIATEK)) {
-		hdev->setup = btusb_mtk_setup;
-		hdev->shutdown = btusb_mtk_shutdown;
-		hdev->manufacturer = 70;
-		hdev->reset = btmtk_reset_sync;
-		hdev->set_bdaddr = btmtk_set_bdaddr;
-		hdev->send = btusb_send_frame_mtk;
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN);
-		hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_SETUP);
-		data->recv_acl = btmtk_usb_recv_acl;
-		data->recv_event = btmtk_recv_event;
-		data->suspend = btmtk_usb_suspend;
-		data->resume = btmtk_usb_resume;
-		data->disconnect = btusb_mtk_disconnect;
-	}
-
-	if (id->driver_info & BTUSB_SWAVE) {
-		hci_set_quirk(hdev, HCI_QUIRK_FIXUP_INQUIRY_MODE);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_LOCAL_COMMANDS);
-	}
-
-	if (id->driver_info & BTUSB_INTEL_BOOT) {
-		hdev->manufacturer = 2;
-		hci_set_quirk(hdev, HCI_QUIRK_RAW_DEVICE);
-	}
-
-	if (id->driver_info & BTUSB_ATH3012) {
-		data->setup_on_usb = btusb_setup_qca;
-		hdev->set_bdaddr = btusb_set_bdaddr_ath3012;
-		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
-		hci_set_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER);
-	}
-
-	if (id->driver_info & BTUSB_QCA_ROME) {
-		data->setup_on_usb = btusb_setup_qca;
-		hdev->shutdown = btusb_shutdown_qca;
-		hdev->set_bdaddr = btusb_set_bdaddr_ath3012;
-		hdev->reset = btusb_qca_reset;
-		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
-		btusb_check_needs_reset_resume(intf);
-	}
-
-	if (id->driver_info & BTUSB_QCA_WCN6855) {
-		struct btqca_data *btqca_data = hci_get_priv(hdev);
-
-		btqca_data->qca_dump.id_vendor = id->idVendor;
-		btqca_data->qca_dump.id_product = id->idProduct;
-		data->recv_event = btusb_recv_evt_qca;
-		data->recv_acl = btusb_recv_acl_qca;
-		hci_devcd_register(hdev, btusb_coredump_qca, btusb_dump_hdr_qca, NULL);
-		data->setup_on_usb = btusb_setup_qca;
-		hdev->classify_pkt_type = btusb_classify_qca_pkt_type;
-		hdev->shutdown = btusb_shutdown_qca;
-		hdev->set_bdaddr = btusb_set_bdaddr_wcn6855;
-		hdev->reset = btusb_qca_reset;
-		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
-		hci_set_msft_opcode(hdev, 0xFD70);
-	}
-
-	if (id->driver_info & BTUSB_AMP) {
-		/* AMP controllers do not support SCO packets */
-		data->isoc = NULL;
-	} else {
-		/* Interface orders are hardcoded in the specification */
-		data->isoc = usb_ifnum_to_if(data->udev, ifnum_base + 1);
-		data->isoc_ifnum = ifnum_base + 1;
-	}
-
-	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_RTL) &&
-	    (id->driver_info & BTUSB_REALTEK)) {
-		btrtl_set_driver_name(hdev, btusb_driver.name);
-		hdev->setup = btusb_setup_realtek;
-		hdev->shutdown = btrtl_shutdown_realtek;
-		hdev->reset = btusb_rtl_reset;
-		hdev->hw_error = btusb_rtl_hw_error;
-
-		/* Realtek devices need to set remote wakeup on auto-suspend */
-		set_bit(BTUSB_WAKEUP_AUTOSUSPEND, &data->flags);
-		set_bit(BTUSB_USE_ALT3_FOR_WBS, &data->flags);
-	}
-
-	if (id->driver_info & BTUSB_ACTIONS_SEMI) {
-		/* Support is advertised, but not implemented */
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ERR_DATA_REPORTING);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_TRANSMIT_POWER);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_SET_RPA_TIMEOUT);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_SCAN);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_ENC_KEY_SIZE);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_CREATE_CONN);
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_WRITE_AUTH_PAYLOAD_TIMEOUT);
-	}
-
-	if (!reset)
-		hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
-
-	if (force_scofix || id->driver_info & BTUSB_WRONG_SCO_MTU) {
-		if (!disable_scofix)
-			hci_set_quirk(hdev, HCI_QUIRK_FIXUP_BUFFER_SIZE);
-	}
-
-	if (id->driver_info & BTUSB_BROKEN_ISOC)
-		data->isoc = NULL;
-
-	if (id->driver_info & BTUSB_WIDEBAND_SPEECH)
-		hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED);
-
-	if (id->driver_info & BTUSB_INVALID_LE_STATES)
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_LE_STATES);
-
-	if (id->driver_info & BTUSB_BROKEN_EXT_SCAN)
-		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_SCAN);
-
-	if (id->driver_info & BTUSB_DIGIANSWER) {
-		data->cmdreq_type = USB_TYPE_VENDOR;
-		hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
-	}
-
-	if (id->driver_info & BTUSB_CSR) {
-		struct usb_device *udev = data->udev;
-		u16 bcdDevice = le16_to_cpu(udev->descriptor.bcdDevice);
-
-		/* Old firmware would otherwise execute USB reset */
-		if (bcdDevice < 0x117)
-			hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
-
-		/* This must be set first in case we disable it for fakes */
-		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
-
-		/* Fake CSR devices with broken commands */
-		if (le16_to_cpu(udev->descriptor.idVendor)  == 0x0a12 &&
-		    le16_to_cpu(udev->descriptor.idProduct) == 0x0001)
-			hdev->setup = btusb_setup_csr;
-	}
-
-	if (id->driver_info & BTUSB_SNIFFER) {
-		struct usb_device *udev = data->udev;
-
-		/* New sniffer firmware has crippled HCI interface */
-		if (le16_to_cpu(udev->descriptor.bcdDevice) > 0x997)
-			hci_set_quirk(hdev, HCI_QUIRK_RAW_DEVICE);
-	}
-
-	if (id->driver_info & BTUSB_INTEL_BOOT) {
-		/* A bug in the bootloader causes that interrupt interface is
-		 * only enabled after receiving SetInterface(0, AltSetting=0).
-		 */
-		err = usb_set_interface(data->udev, 0, 0);
-		if (err < 0) {
-			BT_ERR("failed to set interface 0, alt 0 %d", err);
-			goto err_kill_tx_urbs;
-		}
-	}
-
-	if (data->isoc) {
-		err = usb_driver_claim_interface(&btusb_driver,
-						 data->isoc, data);
-		if (err < 0)
-			goto err_kill_tx_urbs;
-	}
-
-	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_BCM) && data->diag) {
-		if (!usb_driver_claim_interface(&btusb_driver,
-						data->diag, data))
-			__set_diag_interface(hdev);
-		else
-			data->diag = NULL;
-	}
-
-	if (enable_autosuspend)
-		usb_enable_autosuspend(data->udev);
-
-	data->poll_sync = enable_poll_sync;
-
-	err = hci_register_dev(hdev);
-	if (err < 0)
-		goto err_release_siblings;
-
-	usb_set_intfdata(intf, data);
-
-	debugfs_create_file("force_poll_sync", 0644, hdev->debugfs, data,
-			    &force_poll_sync_fops);
-
-	return 0;
-
-err_release_siblings:
-	if (data->diag) {
-		usb_set_intfdata(data->diag, NULL);
-		usb_driver_release_interface(&btusb_driver, data->diag);
-	}
-	if (data->isoc) {
-		usb_set_intfdata(data->isoc, NULL);
-		usb_driver_release_interface(&btusb_driver, data->isoc);
-	}
-err_kill_tx_urbs:
-	usb_kill_anchored_urbs(&data->tx_anchor);
-err_disable_wakeup:
-	if (data->oob_wake_irq) {
-		device_init_wakeup(&data->udev->dev, false);
-		free_irq(data->oob_wake_irq, data);
-	}
-err_put_reset:
-	if (data->reset_gpio)
-		gpiod_put(data->reset_gpio);
-err_free_hdev:
-	hci_free_dev(hdev);
-err_free_data:
-	kfree(data);
-
-	return err;
-}
-
-static void btusb_disconnect(struct usb_interface *intf)
-{
-	struct btusb_data *data = usb_get_intfdata(intf);
-	struct hci_dev *hdev;
-
-	BT_DBG("intf %p", intf);
-
-	if (!data)
-		return;
-
-	hdev = data->hdev;
-	usb_set_intfdata(data->intf, NULL);
-
-	if (data->isoc)
-		usb_set_intfdata(data->isoc, NULL);
-
-	if (data->diag)
-		usb_set_intfdata(data->diag, NULL);
-
-	if (data->disconnect)
-		data->disconnect(hdev);
-
-	hci_unregister_dev(hdev);
-
-	if (data->oob_wake_irq) {
-		device_init_wakeup(&data->udev->dev, false);
-		free_irq(data->oob_wake_irq, data);
-	}
-
-	if (data->reset_gpio)
-		gpiod_put(data->reset_gpio);
-
-	if (test_and_clear_bit(BTUSB_RESET, &data->flags))
-		usb_autopm_put_interface_no_suspend(data->intf);
-
-	if (intf == data->intf) {
-		if (data->isoc)
-			usb_driver_release_interface(&btusb_driver, data->isoc);
-		if (data->diag)
-			usb_driver_release_interface(&btusb_driver, data->diag);
-	} else if (intf == data->isoc) {
-		if (data->diag)
-			usb_driver_release_interface(&btusb_driver, data->diag);
-		usb_driver_release_interface(&btusb_driver, data->intf);
-	} else if (intf == data->diag) {
-		if (data->isoc)
-			usb_driver_release_interface(&btusb_driver, data->isoc);
-		usb_driver_release_interface(&btusb_driver, data->intf);
-	}
-
-	hci_free_dev(hdev);
-	kfree(data);
-}
-
-static int btusb_suspend(struct usb_interface *intf, pm_message_t message)
-{
-	struct btusb_data *data = usb_get_intfdata(intf);
-
-	BT_DBG("intf %p", intf);
-
-	/*
-	 * It is reported that remote wakeup events could sometimes cause some
-	 * adapters completely unresponsive. Resetting the xHCI root hub doesn't
-	 * help at all, and recovering from such a state needs a power cycle.
-	 * Since disabling remote wakeup simply causes the USB core to gate
-	 * runtime autosuspend as well due to needs_remote_wakeup == 1, let's do
-	 * this ourselves to make our life easier. The interface can be safely
-	 * autosuspended as long as remote wakeup is disabled, i.e., after
-	 * closing the HCI device.
-	 *
-	 * Don't auto-suspend if there are connections or discovery in progress.
-	 *
-	 * External suspend calls shall never fail. Specifically, a device with
-	 * broken remote wakeup may still take the advantage of remote wakeup in
-	 * order to wake up the system from sleep if userspace has enabled it as
-	 * a wakeup source.
-	 */
-	if (PMSG_IS_AUTO(message) &&
-	    ((test_bit(BTUSB_WAKEUP_BROKEN, &data->flags) && data->intf->needs_remote_wakeup) ||
-	     hci_conn_count(data->hdev) || hci_discovery_active(data->hdev)))
-		return -EBUSY;
-
-	if (data->suspend_count++)
-		return 0;
-
-	spin_lock_irq(&data->txlock);
-	if (!(PMSG_IS_AUTO(message) && data->tx_in_flight)) {
-		set_bit(BTUSB_SUSPENDING, &data->flags);
-		spin_unlock_irq(&data->txlock);
-	} else {
-		spin_unlock_irq(&data->txlock);
-		data->suspend_count--;
-		return -EBUSY;
-	}
-
-	cancel_work_sync(&data->work);
-
-	if (data->suspend)
-		data->suspend(data->hdev);
-
-	btusb_stop_traffic(data);
-	usb_kill_anchored_urbs(&data->tx_anchor);
-
-	if (data->oob_wake_irq && device_may_wakeup(&data->udev->dev)) {
-		set_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags);
-		enable_irq_wake(data->oob_wake_irq);
-		enable_irq(data->oob_wake_irq);
-	}
-
-	/* For global suspend, Realtek devices lose the loaded fw
-	 * in them. But for autosuspend, firmware should remain.
-	 * Actually, it depends on whether the usb host sends
-	 * set feature (enable wakeup) or not.
-	 */
-	if (test_bit(BTUSB_WAKEUP_AUTOSUSPEND, &data->flags)) {
-		if (PMSG_IS_AUTO(message) &&
-		    device_can_wakeup(&data->udev->dev))
-			data->udev->do_remote_wakeup = 1;
-		else if (!PMSG_IS_AUTO(message) &&
-			 !device_may_wakeup(&data->udev->dev)) {
-			data->udev->do_remote_wakeup = 0;
-			data->udev->reset_resume = 1;
-		}
-	}
-
-	return 0;
-}
-
-static void play_deferred(struct btusb_data *data)
-{
-	struct urb *urb;
-	int err;
-
-	while ((urb = usb_get_from_anchor(&data->deferred))) {
-		usb_anchor_urb(urb, &data->tx_anchor);
-
-		err = usb_submit_urb(urb, GFP_ATOMIC);
-		if (err < 0) {
-			if (err != -EPERM && err != -ENODEV)
-				BT_ERR("%s urb %p submission failed (%d)",
-				       data->hdev->name, urb, -err);
-			kfree(urb->setup_packet);
-			usb_unanchor_urb(urb);
-			usb_free_urb(urb);
-			break;
-		}
-
-		data->tx_in_flight++;
-		usb_free_urb(urb);
-	}
-
-	/* Cleanup the rest deferred urbs. */
-	while ((urb = usb_get_from_anchor(&data->deferred))) {
-		kfree(urb->setup_packet);
-		usb_free_urb(urb);
-	}
-}
-
-static int btusb_resume(struct usb_interface *intf)
-{
-	struct btusb_data *data = usb_get_intfdata(intf);
-	struct hci_dev *hdev = data->hdev;
-	int err = 0;
-
-	BT_DBG("intf %p", intf);
-
-	if (--data->suspend_count)
-		return 0;
-
-	/* Disable only if not already disabled (keep it balanced) */
-	if (test_and_clear_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags)) {
-		disable_irq(data->oob_wake_irq);
-		disable_irq_wake(data->oob_wake_irq);
-	}
-
-	if (!test_bit(HCI_RUNNING, &hdev->flags))
-		goto done;
-
-	if (test_bit(BTUSB_INTR_RUNNING, &data->flags)) {
-		err = btusb_submit_intr_urb(hdev, GFP_NOIO);
-		if (err < 0) {
-			clear_bit(BTUSB_INTR_RUNNING, &data->flags);
-			goto failed;
-		}
-	}
-
-	if (test_bit(BTUSB_BULK_RUNNING, &data->flags)) {
-		err = btusb_submit_bulk_urb(hdev, GFP_NOIO);
-		if (err < 0) {
-			clear_bit(BTUSB_BULK_RUNNING, &data->flags);
-			goto failed;
-		}
-
-		btusb_submit_bulk_urb(hdev, GFP_NOIO);
-	}
-
-	if (test_bit(BTUSB_ISOC_RUNNING, &data->flags)) {
-		if (btusb_submit_isoc_urb(hdev, GFP_NOIO) < 0)
-			clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
-		else
-			btusb_submit_isoc_urb(hdev, GFP_NOIO);
-	}
-
-	if (data->resume)
-		data->resume(hdev);
-
-	spin_lock_irq(&data->txlock);
-	play_deferred(data);
-	clear_bit(BTUSB_SUSPENDING, &data->flags);
-	spin_unlock_irq(&data->txlock);
-	schedule_work(&data->work);
-
-	return 0;
-
-failed:
-	usb_scuttle_anchored_urbs(&data->deferred);
-done:
-	spin_lock_irq(&data->txlock);
-	clear_bit(BTUSB_SUSPENDING, &data->flags);
-	spin_unlock_irq(&data->txlock);
-
-	return err;
-}
-
-#ifdef CONFIG_DEV_COREDUMP
-static void btusb_coredump(struct device *dev)
-{
-	struct btusb_data *data = dev_get_drvdata(dev);
-	struct hci_dev *hdev = data->hdev;
-
-	if (hdev->dump.coredump)
-		hdev->dump.coredump(hdev);
-}
-#endif
-
-static struct usb_driver btusb_driver = {
-	.name		= "btusb",
-	.probe		= btusb_probe,
-	.disconnect	= btusb_disconnect,
-	.suspend	= pm_ptr(btusb_suspend),
-	.resume		= pm_ptr(btusb_resume),
-	.id_table	= btusb_table,
-	.supports_autosuspend = 1,
-	.disable_hub_initiated_lpm = 1,
-
-#ifdef CONFIG_DEV_COREDUMP
-	.driver = {
-		.coredump = btusb_coredump,
-	},
-#endif
-};
-
-module_usb_driver(btusb_driver);
-
-module_param(disable_scofix, bool, 0644);
-MODULE_PARM_DESC(disable_scofix, "Disable fixup of wrong SCO buffer size");
-
-module_param(force_scofix, bool, 0644);
-MODULE_PARM_DESC(force_scofix, "Force fixup of wrong SCO buffers size");
-
-module_param(enable_autosuspend, bool, 0644);
-MODULE_PARM_DESC(enable_autosuspend, "Enable USB autosuspend by default");
-
-module_param(reset, bool, 0644);
-MODULE_PARM_DESC(reset, "Send HCI reset command on initialization");
-
-MODULE_AUTHOR("Marcel Holtmann <marcel@holtmann.org>");
-MODULE_DESCRIPTION("Generic Bluetooth USB driver ver " VERSION);
-MODULE_VERSION(VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/bluetooth/btusb_main.c b/drivers/bluetooth/btusb_main.c
new file mode 100644
index 00000000000000..3518f92ebbd3db
--- /dev/null
+++ b/drivers/bluetooth/btusb_main.c
@@ -0,0 +1,5121 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *
+ *  Generic Bluetooth USB driver
+ *
+ *  Copyright (C) 2005-2008  Marcel Holtmann <marcel@holtmann.org>
+ */
+
+#include <linux/cpufeature.h>
+#include <linux/dmi.h>
+#include <linux/module.h>
+#include <linux/usb.h>
+#include <linux/usb/quirks.h>
+#include <linux/firmware.h>
+#include <linux/iopoll.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/suspend.h>
+#include <linux/gpio/consumer.h>
+#include <linux/debugfs.h>
+#include <linux/unaligned.h>
+
+#include <net/bluetooth/bluetooth.h>
+#include <net/bluetooth/hci_core.h>
+#include <net/bluetooth/hci_drv.h>
+
+#include "btintel.h"
+#include "btbcm.h"
+#include "btrtl.h"
+#include "btmtk.h"
+#include "hci_uart.h"
+#include "btusb_qcom.h"
+
+#define VERSION "1.0"
+
+static bool disable_scofix;
+static bool force_scofix;
+static bool enable_autosuspend = IS_ENABLED(CONFIG_BT_HCIBTUSB_AUTOSUSPEND);
+static bool enable_poll_sync = IS_ENABLED(CONFIG_BT_HCIBTUSB_POLL_SYNC);
+static bool reset = true;
+
+static struct usb_driver btusb_driver;
+
+#define BTUSB_IGNORE			BIT(0)
+#define BTUSB_DIGIANSWER		BIT(1)
+#define BTUSB_CSR			BIT(2)
+#define BTUSB_SNIFFER			BIT(3)
+#define BTUSB_BCM92035			BIT(4)
+#define BTUSB_BROKEN_ISOC		BIT(5)
+#define BTUSB_WRONG_SCO_MTU		BIT(6)
+#define BTUSB_ATH3012			BIT(7)
+#define BTUSB_INTEL_COMBINED		BIT(8)
+#define BTUSB_INTEL_BOOT		BIT(9)
+#define BTUSB_BCM_PATCHRAM		BIT(10)
+#define BTUSB_MARVELL			BIT(11)
+#define BTUSB_SWAVE			BIT(12)
+#define BTUSB_AMP			BIT(13)
+#define BTUSB_QCA_ROME			BIT(14)
+#define BTUSB_BCM_APPLE			BIT(15)
+#define BTUSB_REALTEK			BIT(16)
+#define BTUSB_BCM2045			BIT(17)
+#define BTUSB_IFNUM_2			BIT(18)
+#define BTUSB_CW6622			BIT(19)
+#define BTUSB_MEDIATEK			BIT(20)
+#define BTUSB_WIDEBAND_SPEECH		BIT(21)
+#define BTUSB_INVALID_LE_STATES		BIT(22)
+#define BTUSB_QCA_WCN6855		BIT(23)
+#define BTUSB_INTEL_BROKEN_SHUTDOWN_LED	BIT(24)
+#define BTUSB_INTEL_BROKEN_INITIAL_NCMD BIT(25)
+#define BTUSB_INTEL_NO_WBS_SUPPORT	BIT(26)
+#define BTUSB_ACTIONS_SEMI		BIT(27)
+#define BTUSB_BARROT			BIT(28)
+#define BTUSB_QUALCOMM			BIT(30)
+
+static const struct usb_device_id btusb_table[] = {
+	/*
+	 * NXP IW610 (0471:0215): the composite device reports Bluetooth
+	 * class at the whole-device level, so the generic entry below
+	 * would also match this WiFi vendor interface. Ignore it here
+	 * first so mwifiex-nxp can bind it instead.
+	 */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0471, 0x0215, 0xff, 0xff, 0xff),
+	  .driver_info = BTUSB_IGNORE },
+
+	/* Generic Bluetooth USB device */
+	{ USB_DEVICE_INFO(0xe0, 0x01, 0x01) },
+
+	/* Generic Bluetooth AMP device */
+	{ USB_DEVICE_INFO(0xe0, 0x01, 0x04), .driver_info = BTUSB_AMP },
+
+	/* Generic Bluetooth USB interface */
+	{ USB_INTERFACE_INFO(0xe0, 0x01, 0x01) },
+
+	/* Apple-specific (Broadcom) devices */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x05ac, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_APPLE | BTUSB_IFNUM_2 },
+
+	/* MediaTek MT76x0E */
+	{ USB_DEVICE(0x0e8d, 0x763f) },
+
+	/* Broadcom SoftSailing reporting vendor specific */
+	{ USB_DEVICE(0x0a5c, 0x21e1) },
+
+	/* Apple MacBookPro 7,1 */
+	{ USB_DEVICE(0x05ac, 0x8213) },
+
+	/* Apple iMac11,1 */
+	{ USB_DEVICE(0x05ac, 0x8215) },
+
+	/* Apple MacBookPro6,2 */
+	{ USB_DEVICE(0x05ac, 0x8218) },
+
+	/* Apple MacBookAir3,1, MacBookAir3,2 */
+	{ USB_DEVICE(0x05ac, 0x821b) },
+
+	/* Apple MacBookAir4,1 */
+	{ USB_DEVICE(0x05ac, 0x821f) },
+
+	/* Apple MacBookPro8,2 */
+	{ USB_DEVICE(0x05ac, 0x821a) },
+
+	/* Apple MacMini5,1 */
+	{ USB_DEVICE(0x05ac, 0x8281) },
+
+	/* AVM BlueFRITZ! USB v2.0 */
+	{ USB_DEVICE(0x057c, 0x3800), .driver_info = BTUSB_SWAVE },
+
+	/* Bluetooth Ultraport Module from IBM */
+	{ USB_DEVICE(0x04bf, 0x030a) },
+
+	/* ALPS Modules with non-standard id */
+	{ USB_DEVICE(0x044e, 0x3001) },
+	{ USB_DEVICE(0x044e, 0x3002) },
+
+	/* Ericsson with non-standard id */
+	{ USB_DEVICE(0x0bdb, 0x1002) },
+
+	/* Canyon CN-BTU1 with HID interfaces */
+	{ USB_DEVICE(0x0c10, 0x0000) },
+
+	/* Broadcom BCM20702B0 (Dynex/Insignia) */
+	{ USB_DEVICE(0x19ff, 0x0239), .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Broadcom BCM43142A0 (Foxconn/Lenovo) */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x105b, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Broadcom BCM920703 (HTC Vive) */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x0bb4, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Foxconn - Hon Hai */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x0489, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Lite-On Technology - Broadcom based */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x04ca, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Broadcom devices with vendor specific id */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x0a5c, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* ASUSTek Computer - Broadcom based */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x0b05, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Belkin F8065bf - Broadcom based */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x050d, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* IMC Networks - Broadcom based */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x13d3, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Dell Computer - Broadcom based  */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x413c, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Toshiba Corp - Broadcom based */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x0930, 0xff, 0x01, 0x01),
+	  .driver_info = BTUSB_BCM_PATCHRAM },
+
+	/* Intel Bluetooth USB Bootloader (RAM module) */
+	{ USB_DEVICE(0x8087, 0x0a5a),
+	  .driver_info = BTUSB_INTEL_BOOT | BTUSB_BROKEN_ISOC },
+
+	{ }	/* Terminating entry */
+};
+
+MODULE_DEVICE_TABLE(usb, btusb_table);
+
+static const struct usb_device_id quirks_table[] = {
+	/* CSR BlueCore devices */
+	{ USB_DEVICE(0x0a12, 0x0001), .driver_info = BTUSB_CSR },
+
+	/* Broadcom BCM2033 without firmware */
+	{ USB_DEVICE(0x0a5c, 0x2033), .driver_info = BTUSB_IGNORE },
+
+	/* Broadcom BCM2045 devices */
+	{ USB_DEVICE(0x0a5c, 0x2045), .driver_info = BTUSB_BCM2045 },
+
+	/* Atheros 3011 with sflash firmware */
+	{ USB_DEVICE(0x0489, 0xe027), .driver_info = BTUSB_IGNORE },
+	{ USB_DEVICE(0x0489, 0xe03d), .driver_info = BTUSB_IGNORE },
+	{ USB_DEVICE(0x04f2, 0xaff1), .driver_info = BTUSB_IGNORE },
+	{ USB_DEVICE(0x0930, 0x0215), .driver_info = BTUSB_IGNORE },
+	{ USB_DEVICE(0x0cf3, 0x3002), .driver_info = BTUSB_IGNORE },
+	{ USB_DEVICE(0x0cf3, 0xe019), .driver_info = BTUSB_IGNORE },
+	{ USB_DEVICE(0x13d3, 0x3304), .driver_info = BTUSB_IGNORE },
+
+	/* Atheros AR9285 Malbec with sflash firmware */
+	{ USB_DEVICE(0x03f0, 0x311d), .driver_info = BTUSB_IGNORE },
+
+	/* Atheros 3012 with sflash firmware */
+	{ USB_DEVICE(0x0489, 0xe04d), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe04e), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe056), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe057), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe05f), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe076), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe078), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe095), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04c5, 0x1330), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3004), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3005), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3006), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3007), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3008), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x300b), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x300d), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x300f), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3010), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3014), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x04ca, 0x3018), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0930, 0x0219), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0930, 0x021c), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0930, 0x0220), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0930, 0x0227), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0b05, 0x17d0), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x0036), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x3004), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x3008), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x311d), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x311e), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x311f), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x3121), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x817a), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0x817b), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0xe003), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0xe004), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0xe005), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0cf3, 0xe006), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3362), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3375), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3393), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3395), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3402), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3408), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3423), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3432), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3472), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3474), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3487), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x13d3, 0x3490), .driver_info = BTUSB_ATH3012 },
+
+	/* Atheros AR5BBU12 with sflash firmware */
+	{ USB_DEVICE(0x0489, 0xe02c), .driver_info = BTUSB_IGNORE },
+
+	/* Atheros AR5BBU12 with sflash firmware */
+	{ USB_DEVICE(0x0489, 0xe036), .driver_info = BTUSB_ATH3012 },
+	{ USB_DEVICE(0x0489, 0xe03c), .driver_info = BTUSB_ATH3012 },
+
+	/* QCA ROME chipset */
+	{ USB_DEVICE(0x0b05, 0x1868), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0x535b), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe009), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe010), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe301), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe500), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe092), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe09f), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0a2), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3015), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x301a), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3021), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3491), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3496), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3501), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3503), .driver_info = BTUSB_QCA_ROME |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* QCA WCN6855 chipset */
+	{ USB_DEVICE(0x0489, 0xe0c7), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0c9), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0ca), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0cb), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0cc), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0ce), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0d0), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0d6), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0de), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0df), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0e1), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0e3), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0ea), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0ec), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3022), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3023), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3024), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3a22), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3a24), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3a26), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3a27), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cf3, 0xe600), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9108), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9109), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9208), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9209), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9308), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9309), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9408), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9409), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9508), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9509), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9608), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9609), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x10ab, 0x9f09), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x28de, 0x1401), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* QCA WCN785x chipset */
+	{ USB_DEVICE(0x0cf3, 0xe700), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0fc), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0f3), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe100), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe103), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe10a), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe10d), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe11b), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe11c), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe11f), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe141), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe14a), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe14b), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe14d), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3623), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3624), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2c7c, 0x0130), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2c7c, 0x0131), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2c7c, 0x0132), .driver_info = BTUSB_QCA_WCN6855 |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Qualcomm multi-subsystem chipset QCC2072 */
+	{ USB_DEVICE(0x0cf3, 0xea00), .driver_info = BTUSB_QUALCOMM |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Broadcom BCM2035 */
+	{ USB_DEVICE(0x0a5c, 0x2009), .driver_info = BTUSB_BCM92035 },
+	{ USB_DEVICE(0x0a5c, 0x200a), .driver_info = BTUSB_WRONG_SCO_MTU },
+	{ USB_DEVICE(0x0a5c, 0x2035), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* Broadcom BCM2045 */
+	{ USB_DEVICE(0x0a5c, 0x2039), .driver_info = BTUSB_WRONG_SCO_MTU },
+	{ USB_DEVICE(0x0a5c, 0x2101), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* IBM/Lenovo ThinkPad with Broadcom chip */
+	{ USB_DEVICE(0x0a5c, 0x201e), .driver_info = BTUSB_WRONG_SCO_MTU },
+	{ USB_DEVICE(0x0a5c, 0x2110), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* HP laptop with Broadcom chip */
+	{ USB_DEVICE(0x03f0, 0x171d), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* Dell laptop with Broadcom chip */
+	{ USB_DEVICE(0x413c, 0x8126), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* Dell Wireless 370 and 410 devices */
+	{ USB_DEVICE(0x413c, 0x8152), .driver_info = BTUSB_WRONG_SCO_MTU },
+	{ USB_DEVICE(0x413c, 0x8156), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* Belkin F8T012 and F8T013 devices */
+	{ USB_DEVICE(0x050d, 0x0012), .driver_info = BTUSB_WRONG_SCO_MTU },
+	{ USB_DEVICE(0x050d, 0x0013), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* Asus WL-BTD202 device */
+	{ USB_DEVICE(0x0b05, 0x1715), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* Kensington Bluetooth USB adapter */
+	{ USB_DEVICE(0x047d, 0x105e), .driver_info = BTUSB_WRONG_SCO_MTU },
+
+	/* RTX Telecom based adapters with buggy SCO support */
+	{ USB_DEVICE(0x0400, 0x0807), .driver_info = BTUSB_BROKEN_ISOC },
+	{ USB_DEVICE(0x0400, 0x080a), .driver_info = BTUSB_BROKEN_ISOC },
+
+	/* CONWISE Technology based adapters with buggy SCO support */
+	{ USB_DEVICE(0x0e5e, 0x6622),
+	  .driver_info = BTUSB_BROKEN_ISOC | BTUSB_CW6622},
+
+	/* Roper Class 1 Bluetooth Dongle (Silicon Wave based) */
+	{ USB_DEVICE(0x1310, 0x0001), .driver_info = BTUSB_SWAVE },
+
+	/* Digianswer devices */
+	{ USB_DEVICE(0x08fd, 0x0001), .driver_info = BTUSB_DIGIANSWER },
+	{ USB_DEVICE(0x08fd, 0x0002), .driver_info = BTUSB_IGNORE },
+
+	/* CSR BlueCore Bluetooth Sniffer */
+	{ USB_DEVICE(0x0a12, 0x0002),
+	  .driver_info = BTUSB_SNIFFER | BTUSB_BROKEN_ISOC },
+
+	/* Frontline ComProbe Bluetooth Sniffer */
+	{ USB_DEVICE(0x16d3, 0x0002),
+	  .driver_info = BTUSB_SNIFFER | BTUSB_BROKEN_ISOC },
+
+	/* Marvell Bluetooth devices */
+	{ USB_DEVICE(0x1286, 0x2044), .driver_info = BTUSB_MARVELL },
+	{ USB_DEVICE(0x1286, 0x2046), .driver_info = BTUSB_MARVELL },
+	{ USB_DEVICE(0x1286, 0x204e), .driver_info = BTUSB_MARVELL },
+
+	/*
+	 * NXP IW610 BT interfaces (Marvell-lineage silicon, same quirk as
+	 * the 0x1286 entries above). Scoped to the BT interface class,
+	 * not just VID/PID -- see the btusb_table entry above.
+	 */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x0471, 0x0215, 0xe0, 0x01, 0x01),
+	  .driver_info = BTUSB_MARVELL },
+
+	/* Intel Bluetooth devices */
+	{ USB_DEVICE(0x8087, 0x0025), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0026), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0029), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0032), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0033), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0035), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0036), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0037), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0038), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0039), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0043), .driver_info = BTUSB_INTEL_COMBINED }, /* Otter Peak 2 */
+	{ USB_DEVICE(0x8087, 0x0040), .driver_info = BTUSB_INTEL_COMBINED }, /* Lizard Peak 2 */
+	{ USB_DEVICE(0x8087, 0x07da), .driver_info = BTUSB_CSR },
+	{ USB_DEVICE(0x8087, 0x07dc), .driver_info = BTUSB_INTEL_COMBINED |
+						     BTUSB_INTEL_NO_WBS_SUPPORT |
+						     BTUSB_INTEL_BROKEN_INITIAL_NCMD |
+						     BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
+	{ USB_DEVICE(0x8087, 0x0a2a), .driver_info = BTUSB_INTEL_COMBINED |
+						     BTUSB_INTEL_NO_WBS_SUPPORT |
+						     BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
+	{ USB_DEVICE(0x8087, 0x0a2b), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0aa7), .driver_info = BTUSB_INTEL_COMBINED |
+						     BTUSB_INTEL_BROKEN_SHUTDOWN_LED },
+	{ USB_DEVICE(0x8087, 0x0aaa), .driver_info = BTUSB_INTEL_COMBINED },
+
+	/* Other Intel Bluetooth devices */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x8087, 0xe0, 0x01, 0x01),
+	  .driver_info = BTUSB_IGNORE },
+
+	/* Realtek 8821CE Bluetooth devices */
+	{ USB_DEVICE(0x13d3, 0x3529), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3533), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3556), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3558), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8822CE Bluetooth devices */
+	{ USB_DEVICE(0x0bda, 0xb00c), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xc822), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8822CU Bluetooth devices */
+	{ USB_DEVICE(0x13d3, 0x3549), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8851BE Bluetooth devices */
+	{ USB_DEVICE(0x0bda, 0xb850), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3600), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3601), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x0489, 0xe112), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8851BU Bluetooth devices */
+	{ USB_DEVICE(0x3625, 0x010b), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2001, 0x332a), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x7392, 0xe611), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2c4e, 0x0128), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8852AE Bluetooth devices */
+	{ USB_DEVICE(0x0bda, 0x2852), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xc852), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0x385a), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0x4852), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04c5, 0x165c), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x4006), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cb8, 0xc549), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8852CE Bluetooth devices */
+	{ USB_DEVICE(0x04ca, 0x4007), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04c5, 0x1675), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cb8, 0xc558), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3587), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3586), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3592), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3612), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe122), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8852BE Bluetooth devices */
+	{ USB_DEVICE(0x0cb8, 0xc559), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0x4853), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0x887b), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xb85b), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3570), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3571), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3572), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3591), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe123), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe125), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8852BT/8852BE-VT Bluetooth devices */
+	{ USB_DEVICE(0x0bda, 0x8520), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe12f), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3618), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3619), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek 8922AE Bluetooth devices */
+	{ USB_DEVICE(0x0bda, 0x8922), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xd922), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xd923), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3617), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3616), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe130), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Realtek Bluetooth devices */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01),
+	  .driver_info = BTUSB_REALTEK },
+
+	/* MediaTek Bluetooth devices */
+	{ USB_VENDOR_AND_INTERFACE_INFO(0x0e8d, 0xe0, 0x01, 0x01),
+	  .driver_info = BTUSB_MEDIATEK |
+			 BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional MediaTek MT7615E Bluetooth devices */
+	{ USB_DEVICE(0x13d3, 0x3560), .driver_info = BTUSB_MEDIATEK},
+
+	/* Additional MediaTek MT7663 Bluetooth devices */
+	{ USB_DEVICE(0x043e, 0x310c), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3801), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional MediaTek MT7668 Bluetooth devices */
+	{ USB_DEVICE(0x043e, 0x3109), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional MediaTek MT7920 Bluetooth devices */
+	{ USB_DEVICE(0x0489, 0xe134), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe135), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3620), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3621), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3622), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe158), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional MediaTek MT7921 Bluetooth devices */
+	{ USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0cd), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0e0), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0f2), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3802), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0e8d, 0x0608), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3563), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3564), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3567), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3576), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3578), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3583), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3606), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	/* MediaTek MT7902 Bluetooth devices */
+	{ USB_DEVICE(0x0489, 0xe156), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0e8d, 0x1ede), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3579), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3580), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3594), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3596), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	/* MediaTek MT7922 Bluetooth devices */
+	{ USB_DEVICE(0x13d3, 0x3585), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3610), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* MediaTek MT7922A Bluetooth devices */
+	{ USB_DEVICE(0x0489, 0xe0d8), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0d9), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0e2), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0e4), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0f1), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0f2), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0f5), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0f6), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe102), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe11d), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe152), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe153), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe170), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe174), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3804), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x3807), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04ca, 0x38e4), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0e8d, 0x223c), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3568), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3584), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3605), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3607), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3614), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3615), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3633), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x35f5, 0x7922), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional MediaTek MT7925 Bluetooth devices */
+	{ USB_DEVICE(0x0489, 0xe111), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe113), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe118), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe11e), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe124), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe139), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe13a), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe0fa), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe10f), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe110), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe116), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3588), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe14e), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe14f), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe150), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0489, 0xe151), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0e8d, 0x8c38), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3602), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3603), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3604), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3608), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3609), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3613), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3625), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3627), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3628), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3630), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3631), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2c7c, 0x7009), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional Realtek 8723AE Bluetooth devices */
+	{ USB_DEVICE(0x0930, 0x021d), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3394), .driver_info = BTUSB_REALTEK },
+
+	/* Additional Realtek 8723BE Bluetooth devices */
+	{ USB_DEVICE(0x0489, 0xe085), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x0489, 0xe08b), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x04f2, 0xb49f), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3410), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3416), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3459), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3494), .driver_info = BTUSB_REALTEK },
+
+	/* Additional Realtek 8723BU Bluetooth devices */
+	{ USB_DEVICE(0x7392, 0xa611), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x2c0a, 0x8761), .driver_info = BTUSB_REALTEK },
+
+	/* Additional Realtek 8723DE Bluetooth devices */
+	{ USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK },
+
+	/* Additional Realtek 8761BUV Bluetooth devices */
+	{ USB_DEVICE(0x2c4e, 0x0115), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2357, 0x0607), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2550, 0x8761), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0x8771), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x6655, 0x8771), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x7392, 0xc611), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2b89, 0x8761), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2b89, 0x6275), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x37ad, 0x0600), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x14b0, 0x047a), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional Realtek 8761CU Bluetooth devices */
+	{ USB_DEVICE(0x0b05, 0x1bef), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0b05, 0x1d70), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Additional Realtek 8821AE Bluetooth devices */
+	{ USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3414), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3458), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3461), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x13d3, 0x3462), .driver_info = BTUSB_REALTEK },
+
+	/* Additional Realtek 8822BE Bluetooth devices */
+	{ USB_DEVICE(0x13d3, 0x3526), .driver_info = BTUSB_REALTEK },
+	{ USB_DEVICE(0x0b05, 0x185c), .driver_info = BTUSB_REALTEK },
+
+	/* Additional Realtek 8822CE Bluetooth devices */
+	{ USB_DEVICE(0x04ca, 0x4005), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x04c5, 0x161f), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0b05, 0x18ef), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3548), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3549), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3553), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3555), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x2ff8, 0x3051), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x1358, 0xc123), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xc123), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x1357, 0xc123), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cb5, 0xc547), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+
+	/* Barrot Technology Bluetooth devices */
+	{ USB_DEVICE(0x33fa, 0x0010), .driver_info = BTUSB_BARROT },
+	{ USB_DEVICE(0x33fa, 0x0012), .driver_info = BTUSB_BARROT },
+
+	/* Actions Semiconductor ATS2851 based devices */
+	{ USB_DEVICE(0x10d7, 0xb012), .driver_info = BTUSB_ACTIONS_SEMI },
+
+	/* Silicon Wave based devices */
+	{ USB_DEVICE(0x0c10, 0x0000), .driver_info = BTUSB_SWAVE },
+
+	{ }	/* Terminating entry */
+};
+
+/* The Bluetooth USB module build into some devices needs to be reset on resume,
+ * this is a problem with the platform (likely shutting off all power) not with
+ * the module itself. So we use a DMI list to match known broken platforms.
+ */
+static const struct dmi_system_id btusb_needs_reset_resume_table[] = {
+	{
+		/* Dell OptiPlex 3060 (QCA ROME device 0cf3:e007) */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 3060"),
+		},
+	},
+	{
+		/* Dell XPS 9360 (QCA ROME device 0cf3:e300) */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "XPS 13 9360"),
+		},
+	},
+	{
+		/* Dell Inspiron 5565 (QCA ROME device 0cf3:e009) */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 5565"),
+		},
+	},
+	{}
+};
+
+struct qca_dump_info {
+	/* fields for dump collection */
+	u16 id_vendor;
+	u16 id_product;
+	u32 fw_version;
+	u32 controller_id;
+	u32 ram_dump_size;
+	u16 ram_dump_seqno;
+};
+
+struct btqca_data {
+	struct qca_dump_info qca_dump;
+};
+
+#define BTUSB_MAX_ISOC_FRAMES	10
+
+#define BTUSB_INTR_RUNNING	0
+#define BTUSB_BULK_RUNNING	1
+#define BTUSB_ISOC_RUNNING	2
+#define BTUSB_SUSPENDING	3
+#define BTUSB_DID_ISO_RESUME	4
+#define BTUSB_BOOTLOADER	5
+#define BTUSB_DOWNLOADING	6
+#define BTUSB_FIRMWARE_LOADED	7
+#define BTUSB_FIRMWARE_FAILED	8
+#define BTUSB_BOOTING		9
+#define BTUSB_DIAG_RUNNING	10
+#define BTUSB_OOB_WAKE_ENABLED	11
+#define BTUSB_HW_RESET_ACTIVE	12
+#define BTUSB_TX_WAIT_VND_EVT	13
+#define BTUSB_WAKEUP_AUTOSUSPEND	14
+#define BTUSB_USE_ALT3_FOR_WBS	15
+#define BTUSB_ALT6_CONTINUOUS_TX	16
+#define BTUSB_HW_SSR_ACTIVE	17
+#define BTUSB_WAKEUP_BROKEN	18
+#define BTUSB_RESET		19
+
+#define BTUSB_PROTO_LEGACY	0x00
+#define BTUSB_PROTO_H4		0x01
+
+struct btusb_data {
+	struct hci_dev       *hdev;
+	struct usb_device    *udev;
+	struct usb_interface *intf;
+	struct usb_interface *isoc;
+	struct usb_interface *diag;
+	unsigned isoc_ifnum;
+
+	unsigned long flags;
+
+	bool poll_sync;
+	int intr_interval;
+	struct work_struct  work;
+	struct work_struct  waker;
+	struct delayed_work rx_work;
+
+	struct sk_buff_head acl_q;
+
+	struct usb_anchor deferred;
+	struct usb_anchor tx_anchor;
+	int tx_in_flight;
+	spinlock_t txlock;
+
+	struct usb_anchor intr_anchor;
+	struct usb_anchor bulk_anchor;
+	struct usb_anchor isoc_anchor;
+	struct usb_anchor diag_anchor;
+	struct usb_anchor ctrl_anchor;
+	spinlock_t rxlock;
+
+	struct sk_buff *evt_skb;
+	struct sk_buff *acl_skb;
+	struct sk_buff *sco_skb;
+	struct sk_buff *rx_skb;
+
+	struct usb_endpoint_descriptor *intr_ep;
+	struct usb_endpoint_descriptor *bulk_tx_ep;
+	struct usb_endpoint_descriptor *bulk_rx_ep;
+	struct usb_endpoint_descriptor *isoc_tx_ep;
+	struct usb_endpoint_descriptor *isoc_rx_ep;
+	struct usb_endpoint_descriptor *diag_tx_ep;
+	struct usb_endpoint_descriptor *diag_rx_ep;
+
+	struct gpio_desc *reset_gpio;
+
+	__u8 cmdreq_type;
+	__u8 cmdreq;
+	__u8 proto;
+
+	unsigned int sco_num;
+	unsigned int air_mode;
+	bool usb_alt6_packet_flow;
+	int isoc_altsetting;
+	int suspend_count;
+	const struct usb_device_id *match_id;
+
+	int (*recv_event)(struct hci_dev *hdev, struct sk_buff *skb);
+	int (*recv_acl)(struct hci_dev *hdev, struct sk_buff *skb);
+	int (*recv_intr)(struct btusb_data *data, void *buffer, int count);
+	int (*recv_bulk)(struct btusb_data *data, void *buffer, int count);
+
+	int (*setup_on_usb)(struct hci_dev *hdev);
+
+	int (*suspend)(struct hci_dev *hdev);
+	int (*resume)(struct hci_dev *hdev);
+	int (*disconnect)(struct hci_dev *hdev);
+
+	int oob_wake_irq;   /* irq for out-of-band wake-on-bt */
+};
+
+static void btusb_reset(struct hci_dev *hdev)
+{
+	struct btusb_data *data;
+	int err;
+
+	data = hci_get_drvdata(hdev);
+	err = usb_autopm_get_interface(data->intf);
+	if (err) {
+		bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err);
+		return;
+	}
+
+	if (test_and_set_bit(BTUSB_RESET, &data->flags))
+		usb_autopm_put_interface_no_suspend(data->intf);
+
+	bt_dev_err(hdev, "Resetting usb device.");
+	usb_queue_reset_device(data->intf);
+}
+
+static void btusb_intel_reset(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct gpio_desc *reset_gpio = data->reset_gpio;
+	struct btintel_data *intel_data = hci_get_priv(hdev);
+
+	if (intel_data->acpi_reset_method) {
+		if (test_and_set_bit(INTEL_ACPI_RESET_ACTIVE, intel_data->flags)) {
+			bt_dev_err(hdev, "acpi: last reset failed ? Not resetting again");
+			return;
+		}
+
+		bt_dev_err(hdev, "Initiating acpi reset method");
+		/* If ACPI reset method fails, lets try with legacy GPIO
+		 * toggling
+		 */
+		if (!intel_data->acpi_reset_method(hdev)) {
+			return;
+		}
+	}
+
+	if (!reset_gpio) {
+		btusb_reset(hdev);
+		return;
+	}
+
+	/*
+	 * Toggle the hard reset line if the platform provides one. The reset
+	 * is going to yank the device off the USB and then replug. So doing
+	 * once is enough. The cleanup is handled correctly on the way out
+	 * (standard USB disconnect), and the new device is detected cleanly
+	 * and bound to the driver again like it should be.
+	 */
+	if (test_and_set_bit(BTUSB_HW_RESET_ACTIVE, &data->flags)) {
+		bt_dev_err(hdev, "last reset failed? Not resetting again");
+		return;
+	}
+
+	bt_dev_err(hdev, "Initiating HW reset via gpio");
+	gpiod_set_value_cansleep(reset_gpio, 1);
+	msleep(100);
+	gpiod_set_value_cansleep(reset_gpio, 0);
+}
+
+#define RTK_DEVCOREDUMP_CODE_MEMDUMP		0x01
+#define RTK_DEVCOREDUMP_CODE_HW_ERR		0x02
+#define RTK_DEVCOREDUMP_CODE_CMD_TIMEOUT	0x03
+
+#define RTK_SUB_EVENT_CODE_COREDUMP		0x34
+
+struct rtk_dev_coredump_hdr {
+	u8 type;
+	u8 code;
+	u8 reserved[2];
+} __packed;
+
+static inline void btusb_rtl_alloc_devcoredump(struct hci_dev *hdev,
+		struct rtk_dev_coredump_hdr *hdr, u8 *buf, u32 len)
+{
+	struct sk_buff *skb;
+
+	skb = alloc_skb(len + sizeof(*hdr), GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	skb_put_data(skb, hdr, sizeof(*hdr));
+	if (len)
+		skb_put_data(skb, buf, len);
+
+	if (!hci_devcd_init(hdev, skb->len)) {
+		hci_devcd_append(hdev, skb);
+		hci_devcd_complete(hdev);
+	} else {
+		bt_dev_err(hdev, "RTL: Failed to generate devcoredump");
+		kfree_skb(skb);
+	}
+}
+
+static void btusb_rtl_reset(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct gpio_desc *reset_gpio = data->reset_gpio;
+	struct rtk_dev_coredump_hdr hdr = {
+		.type = RTK_DEVCOREDUMP_CODE_CMD_TIMEOUT,
+	};
+
+	btusb_rtl_alloc_devcoredump(hdev, &hdr, NULL, 0);
+
+	if (!reset_gpio) {
+		btusb_reset(hdev);
+		return;
+	}
+
+	/* Toggle the hard reset line. The Realtek device is going to
+	 * yank itself off the USB and then replug. The cleanup is handled
+	 * correctly on the way out (standard USB disconnect), and the new
+	 * device is detected cleanly and bound to the driver again like
+	 * it should be.
+	 */
+	if (test_and_set_bit(BTUSB_HW_RESET_ACTIVE, &data->flags)) {
+		bt_dev_err(hdev, "last reset failed? Not resetting again");
+		return;
+	}
+
+	bt_dev_err(hdev, "Reset Realtek device via gpio");
+	gpiod_set_value_cansleep(reset_gpio, 1);
+	msleep(200);
+	gpiod_set_value_cansleep(reset_gpio, 0);
+}
+
+static void btusb_rtl_hw_error(struct hci_dev *hdev, u8 code)
+{
+	struct rtk_dev_coredump_hdr hdr = {
+		.type = RTK_DEVCOREDUMP_CODE_HW_ERR,
+		.code = code,
+	};
+
+	bt_dev_err(hdev, "RTL: hw err, trigger devcoredump (%d)", code);
+
+	btusb_rtl_alloc_devcoredump(hdev, &hdr, NULL, 0);
+}
+
+static void btusb_qca_reset(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct gpio_desc *reset_gpio = data->reset_gpio;
+
+	if (test_bit(BTUSB_HW_SSR_ACTIVE, &data->flags)) {
+		bt_dev_info(hdev, "Ramdump in progress, defer reset");
+		return;
+	}
+
+	if (reset_gpio) {
+		bt_dev_err(hdev, "Reset qca device via bt_en gpio");
+
+		/* Toggle the hard reset line. The qca bt device is going to
+		 * yank itself off the USB and then replug. The cleanup is handled
+		 * correctly on the way out (standard USB disconnect), and the new
+		 * device is detected cleanly and bound to the driver again like
+		 * it should be.
+		 */
+		if (test_and_set_bit(BTUSB_HW_RESET_ACTIVE, &data->flags)) {
+			bt_dev_err(hdev, "last reset failed? Not resetting again");
+			return;
+		}
+
+		gpiod_set_value_cansleep(reset_gpio, 0);
+		msleep(200);
+		gpiod_set_value_cansleep(reset_gpio, 1);
+
+		return;
+	}
+
+	btusb_reset(hdev);
+}
+
+static u8 btusb_classify_qca_pkt_type(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	/* Some Qualcomm controllers, e.g., QCNFA765 with WCN6855 chip, send debug
+	 * packets as ACL frames with connection handle 0x2EDC. These are not real
+	 * ACL packets and should be reclassified as HCI_DIAG_PKT to prevent
+	 * "ACL packet for unknown connection handle 3804" errors.
+	 */
+	if (skb->len >= 2) {
+		u16 handle = get_unaligned_le16(skb->data);
+
+		if (handle == 0x2EDC)
+			return HCI_DIAG_PKT;
+	}
+
+	/* Use default packet type for other packets */
+	return hci_skb_pkt_type(skb);
+}
+
+static inline void btusb_free_frags(struct btusb_data *data)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&data->rxlock, flags);
+
+	dev_kfree_skb_irq(data->evt_skb);
+	data->evt_skb = NULL;
+
+	dev_kfree_skb_irq(data->acl_skb);
+	data->acl_skb = NULL;
+
+	dev_kfree_skb_irq(data->sco_skb);
+	data->sco_skb = NULL;
+
+	/* rx_skb may hold an ERR_PTR from a previous h4_recv_skb() call */
+	if (!IS_ERR(data->rx_skb))
+		dev_kfree_skb_irq(data->rx_skb);
+	data->rx_skb = NULL;
+
+	spin_unlock_irqrestore(&data->rxlock, flags);
+}
+
+static int btusb_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	err = data->recv_event(hdev, skb);
+
+	if (data->intr_interval) {
+		/* Trigger dequeue immediately if an event is received */
+		schedule_delayed_work(&data->rx_work, 0);
+	}
+
+	return err;
+}
+
+static int btusb_recv_intr(struct btusb_data *data, void *buffer, int count)
+{
+	struct sk_buff *skb;
+	unsigned long flags;
+	int err = 0;
+
+	spin_lock_irqsave(&data->rxlock, flags);
+	skb = data->evt_skb;
+
+	while (count) {
+		int len;
+
+		if (!skb) {
+			skb = bt_skb_alloc(HCI_MAX_EVENT_SIZE, GFP_ATOMIC);
+			if (!skb) {
+				err = -ENOMEM;
+				break;
+			}
+
+			hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
+			hci_skb_expect(skb) = HCI_EVENT_HDR_SIZE;
+		}
+
+		len = min_t(uint, hci_skb_expect(skb), count);
+		skb_put_data(skb, buffer, len);
+
+		count -= len;
+		buffer += len;
+		hci_skb_expect(skb) -= len;
+
+		if (skb->len == HCI_EVENT_HDR_SIZE) {
+			/* Complete event header */
+			hci_skb_expect(skb) = hci_event_hdr(skb)->plen;
+
+			if (skb_tailroom(skb) < hci_skb_expect(skb)) {
+				kfree_skb(skb);
+				skb = NULL;
+
+				err = -EILSEQ;
+				break;
+			}
+		}
+
+		if (!hci_skb_expect(skb)) {
+			/* Each chunk should correspond to at least 1 or more
+			 * events so if there are still bytes left that doesn't
+			 * constitute a new event this is likely a bug in the
+			 * controller.
+			 */
+			if (count && count < HCI_EVENT_HDR_SIZE) {
+				bt_dev_warn(data->hdev,
+					"Unexpected continuation: %d bytes",
+					count);
+				count = 0;
+			}
+
+			/* Complete frame */
+			btusb_recv_event(data->hdev, skb);
+			skb = NULL;
+		}
+	}
+
+	data->evt_skb = skb;
+	spin_unlock_irqrestore(&data->rxlock, flags);
+
+	return err;
+}
+
+static int btusb_recv_acl(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+
+	/* Only queue ACL packet if intr_interval is set as it means
+	 * force_poll_sync has been enabled.
+	 */
+	if (!data->intr_interval)
+		return data->recv_acl(hdev, skb);
+
+	skb_queue_tail(&data->acl_q, skb);
+	schedule_delayed_work(&data->rx_work, data->intr_interval);
+
+	return 0;
+}
+
+static int btusb_recv_frame(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	u8 pkt_type = hci_skb_pkt_type(skb);
+	int ret;
+
+	switch (pkt_type) {
+	case HCI_EVENT_PKT:
+		ret = btusb_recv_event(hdev, skb);
+		break;
+	case HCI_ACLDATA_PKT:
+		ret = btusb_recv_acl(hdev, skb);
+		break;
+	case HCI_SCODATA_PKT:
+	case HCI_ISODATA_PKT:
+		ret = hci_recv_frame(hdev, skb);
+		break;
+	default:
+		dev_kfree_skb_irq(skb);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+/* Dispatch through the btusb_recv_* wrappers so that vendor specific
+ * handling (data->recv_event, data->recv_acl) is preserved in H:4 mode.
+ */
+static const struct h4_recv_pkt btusb_recv_pkts[] = {
+	{ H4_RECV_ACL,          .recv = btusb_recv_acl },
+	{ H4_RECV_SCO,          .recv = hci_recv_frame },
+	{ H4_RECV_EVENT,        .recv = btusb_recv_event },
+	{ H4_RECV_ISO,          .recv = hci_recv_frame },
+};
+
+static int btusb_recv_h4(struct btusb_data *data, void *buffer, int count)
+{
+	unsigned long flags;
+	int err = 0;
+
+	spin_lock_irqsave(&data->rxlock, flags);
+	data->rx_skb = h4_recv_skb(data->hdev, NULL, NULL, data->rx_skb, buffer,
+				   count, btusb_recv_pkts,
+				   ARRAY_SIZE(btusb_recv_pkts));
+	if (IS_ERR(data->rx_skb))
+		err = PTR_ERR(data->rx_skb);
+	spin_unlock_irqrestore(&data->rxlock, flags);
+
+	return err;
+}
+
+static int btusb_recv_bulk(struct btusb_data *data, void *buffer, int count)
+{
+	struct sk_buff *skb;
+	unsigned long flags;
+	int err = 0;
+
+	if (data->proto == BTUSB_PROTO_H4)
+		return btusb_recv_h4(data, buffer, count);
+
+	spin_lock_irqsave(&data->rxlock, flags);
+	skb = data->acl_skb;
+
+	while (count) {
+		int len;
+
+		if (!skb) {
+			skb = bt_skb_alloc(HCI_MAX_FRAME_SIZE, GFP_ATOMIC);
+			if (!skb) {
+				err = -ENOMEM;
+				break;
+			}
+
+			hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT;
+			hci_skb_expect(skb) = HCI_ACL_HDR_SIZE;
+		}
+
+		len = min_t(uint, hci_skb_expect(skb), count);
+		skb_put_data(skb, buffer, len);
+
+		count -= len;
+		buffer += len;
+		hci_skb_expect(skb) -= len;
+
+		if (skb->len == HCI_ACL_HDR_SIZE) {
+			/* Complete ACL header */
+			hci_skb_expect(skb) = hci_acl_dlen(skb);
+
+			if (skb_tailroom(skb) < hci_skb_expect(skb)) {
+				kfree_skb(skb);
+				skb = NULL;
+
+				err = -EILSEQ;
+				break;
+			}
+		}
+
+		if (!hci_skb_expect(skb)) {
+			/* Complete frame */
+			btusb_recv_acl(data->hdev, skb);
+			skb = NULL;
+		}
+	}
+
+	data->acl_skb = skb;
+	spin_unlock_irqrestore(&data->rxlock, flags);
+
+	return err;
+}
+
+static bool btusb_validate_sco_handle(struct hci_dev *hdev,
+				      struct hci_sco_hdr *hdr)
+{
+	__u16 handle;
+
+	if (hci_dev_test_flag(hdev, HCI_USER_CHANNEL))
+		// Can't validate, userspace controls everything.
+		return true;
+
+	/*
+	 * USB isochronous transfers are not designed to be reliable and may
+	 * lose fragments.  When this happens, the next first fragment
+	 * encountered might actually be a continuation fragment.
+	 * Validate the handle to detect it and drop it, or else the upper
+	 * layer will get garbage for a while.
+	 */
+
+	handle = hci_handle(__le16_to_cpu(hdr->handle));
+
+	switch (hci_conn_lookup_type(hdev, handle)) {
+	case SCO_LINK:
+	case ESCO_LINK:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static int btusb_recv_isoc(struct btusb_data *data, void *buffer, int count)
+{
+	struct sk_buff *skb;
+	unsigned long flags;
+	int err = 0;
+
+	spin_lock_irqsave(&data->rxlock, flags);
+	skb = data->sco_skb;
+
+	while (count) {
+		int len;
+
+		if (!skb) {
+			skb = bt_skb_alloc(HCI_MAX_SCO_SIZE, GFP_ATOMIC);
+			if (!skb) {
+				err = -ENOMEM;
+				break;
+			}
+
+			hci_skb_pkt_type(skb) = HCI_SCODATA_PKT;
+			hci_skb_expect(skb) = HCI_SCO_HDR_SIZE;
+		}
+
+		len = min_t(uint, hci_skb_expect(skb), count);
+		skb_put_data(skb, buffer, len);
+
+		count -= len;
+		buffer += len;
+		hci_skb_expect(skb) -= len;
+
+		if (skb->len == HCI_SCO_HDR_SIZE) {
+			/* Complete SCO header */
+			struct hci_sco_hdr *hdr = hci_sco_hdr(skb);
+
+			hci_skb_expect(skb) = hdr->dlen;
+
+			if (skb_tailroom(skb) < hci_skb_expect(skb) ||
+			    !btusb_validate_sco_handle(data->hdev, hdr)) {
+				kfree_skb(skb);
+				skb = NULL;
+
+				err = -EILSEQ;
+				break;
+			}
+		}
+
+		if (!hci_skb_expect(skb)) {
+			/* Complete frame */
+			hci_recv_frame(data->hdev, skb);
+			skb = NULL;
+		}
+	}
+
+	data->sco_skb = skb;
+	spin_unlock_irqrestore(&data->rxlock, flags);
+
+	return err;
+}
+
+static void btusb_intr_complete(struct urb *urb)
+{
+	struct hci_dev *hdev = urb->context;
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
+	       urb->actual_length);
+
+	if (!test_bit(HCI_RUNNING, &hdev->flags))
+		return;
+
+	if (urb->status == 0) {
+		hdev->stat.byte_rx += urb->actual_length;
+
+		if (data->recv_intr(data, urb->transfer_buffer,
+				    urb->actual_length) < 0) {
+			bt_dev_err(hdev, "corrupted event packet");
+			hdev->stat.err_rx++;
+		}
+	} else if (urb->status == -ENOENT) {
+		/* Avoid suspend failed when usb_kill_urb */
+		return;
+	}
+
+	if (!test_bit(BTUSB_INTR_RUNNING, &data->flags))
+		return;
+
+	usb_mark_last_busy(data->udev);
+	usb_anchor_urb(urb, &data->intr_anchor);
+
+	err = usb_submit_urb(urb, GFP_ATOMIC);
+	if (err < 0) {
+		/* -EPERM: urb is being killed;
+		 * -ENODEV: device got disconnected
+		 */
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
+				   urb, -err);
+		if (err != -EPERM)
+			hci_cmd_sync_cancel(hdev, -err);
+		usb_unanchor_urb(urb);
+	}
+}
+
+static int btusb_submit_intr_urb(struct hci_dev *hdev, gfp_t mem_flags)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct urb *urb;
+	unsigned char *buf;
+	unsigned int pipe;
+	int err, size;
+
+	BT_DBG("%s", hdev->name);
+
+	if (!data->intr_ep)
+		return -ENODEV;
+
+	urb = usb_alloc_urb(0, mem_flags);
+	if (!urb)
+		return -ENOMEM;
+
+	if (le16_to_cpu(data->udev->descriptor.idVendor)  == 0x0a12 &&
+	    le16_to_cpu(data->udev->descriptor.idProduct) == 0x0001)
+		/* Fake CSR devices don't seem to support sort-transter */
+		size = le16_to_cpu(data->intr_ep->wMaxPacketSize);
+	else
+		/* Use maximum HCI Event size so the USB stack handles
+		 * ZPL/short-transfer automatically.
+		 */
+		size = HCI_MAX_EVENT_SIZE;
+
+	buf = kmalloc(size, mem_flags);
+	if (!buf) {
+		usb_free_urb(urb);
+		return -ENOMEM;
+	}
+
+	pipe = usb_rcvintpipe(data->udev, data->intr_ep->bEndpointAddress);
+
+	usb_fill_int_urb(urb, data->udev, pipe, buf, size,
+			 btusb_intr_complete, hdev, data->intr_ep->bInterval);
+
+	urb->transfer_flags |= URB_FREE_BUFFER;
+
+	usb_anchor_urb(urb, &data->intr_anchor);
+
+	err = usb_submit_urb(urb, mem_flags);
+	if (err < 0) {
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p submission failed (%d)",
+				   urb, -err);
+		if (err != -EPERM)
+			hci_cmd_sync_cancel(hdev, -err);
+		usb_unanchor_urb(urb);
+	}
+
+	/* Only initialize intr_interval if URB poll sync is enabled */
+	if (!data->poll_sync)
+		goto done;
+
+	/* The units are frames (milliseconds) for full and low speed devices,
+	 * and microframes (1/8 millisecond) for highspeed and SuperSpeed
+	 * devices.
+	 *
+	 * This is done once on open/resume so it shouldn't change even if
+	 * force_poll_sync changes.
+	 */
+	switch (urb->dev->speed) {
+	case USB_SPEED_SUPER_PLUS:
+	case USB_SPEED_SUPER:	/* units are 125us */
+		data->intr_interval = usecs_to_jiffies(urb->interval * 125);
+		break;
+	default:
+		data->intr_interval = msecs_to_jiffies(urb->interval);
+		break;
+	}
+
+done:
+	usb_free_urb(urb);
+
+	return err;
+}
+
+static void btusb_bulk_complete(struct urb *urb)
+{
+	struct hci_dev *hdev = urb->context;
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
+	       urb->actual_length);
+
+	if (!test_bit(HCI_RUNNING, &hdev->flags))
+		return;
+
+	if (urb->status == 0) {
+		hdev->stat.byte_rx += urb->actual_length;
+
+		if (data->recv_bulk(data, urb->transfer_buffer,
+				    urb->actual_length) < 0) {
+			bt_dev_err(hdev, "corrupted ACL packet");
+			hdev->stat.err_rx++;
+		}
+	} else if (urb->status == -ENOENT) {
+		/* Avoid suspend failed when usb_kill_urb */
+		return;
+	}
+
+	if (!test_bit(BTUSB_BULK_RUNNING, &data->flags))
+		return;
+
+	usb_anchor_urb(urb, &data->bulk_anchor);
+	usb_mark_last_busy(data->udev);
+
+	err = usb_submit_urb(urb, GFP_ATOMIC);
+	if (err < 0) {
+		/* -EPERM: urb is being killed;
+		 * -ENODEV: device got disconnected
+		 */
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
+				   urb, -err);
+		usb_unanchor_urb(urb);
+	}
+}
+
+static int btusb_submit_bulk_urb(struct hci_dev *hdev, gfp_t mem_flags)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct urb *urb;
+	unsigned char *buf;
+	unsigned int pipe;
+	int err, size = HCI_MAX_FRAME_SIZE;
+
+	BT_DBG("%s", hdev->name);
+
+	if (!data->bulk_rx_ep)
+		return -ENODEV;
+
+	urb = usb_alloc_urb(0, mem_flags);
+	if (!urb)
+		return -ENOMEM;
+
+	buf = kmalloc(size, mem_flags);
+	if (!buf) {
+		usb_free_urb(urb);
+		return -ENOMEM;
+	}
+
+	pipe = usb_rcvbulkpipe(data->udev, data->bulk_rx_ep->bEndpointAddress);
+
+	usb_fill_bulk_urb(urb, data->udev, pipe, buf, size,
+			  btusb_bulk_complete, hdev);
+
+	urb->transfer_flags |= URB_FREE_BUFFER;
+
+	usb_mark_last_busy(data->udev);
+	usb_anchor_urb(urb, &data->bulk_anchor);
+
+	err = usb_submit_urb(urb, mem_flags);
+	if (err < 0) {
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p submission failed (%d)",
+				   urb, -err);
+		usb_unanchor_urb(urb);
+	}
+
+	usb_free_urb(urb);
+
+	return err;
+}
+
+static void btusb_isoc_complete(struct urb *urb)
+{
+	struct hci_dev *hdev = urb->context;
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int i, err;
+
+	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
+	       urb->actual_length);
+
+	if (!test_bit(HCI_RUNNING, &hdev->flags))
+		return;
+
+	if (urb->status == 0) {
+		for (i = 0; i < urb->number_of_packets; i++) {
+			unsigned int offset = urb->iso_frame_desc[i].offset;
+			unsigned int length = urb->iso_frame_desc[i].actual_length;
+
+			if (urb->iso_frame_desc[i].status)
+				continue;
+
+			hdev->stat.byte_rx += length;
+
+			if (btusb_recv_isoc(data, urb->transfer_buffer + offset,
+					    length) < 0) {
+				bt_dev_err(hdev, "corrupted SCO packet");
+				hdev->stat.err_rx++;
+			}
+		}
+	} else if (urb->status == -ENOENT) {
+		/* Avoid suspend failed when usb_kill_urb */
+		return;
+	}
+
+	if (!test_bit(BTUSB_ISOC_RUNNING, &data->flags))
+		return;
+
+	usb_anchor_urb(urb, &data->isoc_anchor);
+
+	err = usb_submit_urb(urb, GFP_ATOMIC);
+	if (err < 0) {
+		/* -EPERM: urb is being killed;
+		 * -ENODEV: device got disconnected
+		 */
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
+				   urb, -err);
+		usb_unanchor_urb(urb);
+	}
+}
+
+static inline void __fill_isoc_descriptor_msbc(struct urb *urb, int len,
+					       int mtu, struct btusb_data *data)
+{
+	int i = 0, offset = 0;
+	unsigned int interval;
+
+	BT_DBG("len %d mtu %d", len, mtu);
+
+	/* For mSBC ALT 6 settings some chips need to transmit the data
+	 * continuously without the zero length of USB packets.
+	 */
+	if (test_bit(BTUSB_ALT6_CONTINUOUS_TX, &data->flags))
+		goto ignore_usb_alt6_packet_flow;
+
+	/* For mSBC ALT 6 setting the host will send the packet at continuous
+	 * flow. As per core spec 5, vol 4, part B, table 2.1. For ALT setting
+	 * 6 the HCI PACKET INTERVAL should be 7.5ms for every usb packets.
+	 * To maintain the rate we send 63bytes of usb packets alternatively for
+	 * 7ms and 8ms to maintain the rate as 7.5ms.
+	 */
+	if (data->usb_alt6_packet_flow) {
+		interval = 7;
+		data->usb_alt6_packet_flow = false;
+	} else {
+		interval = 6;
+		data->usb_alt6_packet_flow = true;
+	}
+
+	for (i = 0; i < interval; i++) {
+		urb->iso_frame_desc[i].offset = offset;
+		urb->iso_frame_desc[i].length = offset;
+	}
+
+ignore_usb_alt6_packet_flow:
+	if (len && i < BTUSB_MAX_ISOC_FRAMES) {
+		urb->iso_frame_desc[i].offset = offset;
+		urb->iso_frame_desc[i].length = len;
+		i++;
+	}
+
+	urb->number_of_packets = i;
+}
+
+static inline void __fill_isoc_descriptor(struct urb *urb, int len, int mtu)
+{
+	int i, offset = 0;
+
+	BT_DBG("len %d mtu %d", len, mtu);
+
+	for (i = 0; i < BTUSB_MAX_ISOC_FRAMES && len >= mtu;
+					i++, offset += mtu, len -= mtu) {
+		urb->iso_frame_desc[i].offset = offset;
+		urb->iso_frame_desc[i].length = mtu;
+	}
+
+	if (len && i < BTUSB_MAX_ISOC_FRAMES) {
+		urb->iso_frame_desc[i].offset = offset;
+		urb->iso_frame_desc[i].length = len;
+		i++;
+	}
+
+	urb->number_of_packets = i;
+}
+
+static int btusb_submit_isoc_urb(struct hci_dev *hdev, gfp_t mem_flags)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct urb *urb;
+	unsigned char *buf;
+	unsigned int pipe;
+	int err, size;
+
+	BT_DBG("%s", hdev->name);
+
+	if (!data->isoc_rx_ep)
+		return -ENODEV;
+
+	urb = usb_alloc_urb(BTUSB_MAX_ISOC_FRAMES, mem_flags);
+	if (!urb)
+		return -ENOMEM;
+
+	size = le16_to_cpu(data->isoc_rx_ep->wMaxPacketSize) *
+						BTUSB_MAX_ISOC_FRAMES;
+
+	buf = kmalloc(size, mem_flags);
+	if (!buf) {
+		usb_free_urb(urb);
+		return -ENOMEM;
+	}
+
+	pipe = usb_rcvisocpipe(data->udev, data->isoc_rx_ep->bEndpointAddress);
+
+	usb_fill_int_urb(urb, data->udev, pipe, buf, size, btusb_isoc_complete,
+			 hdev, data->isoc_rx_ep->bInterval);
+
+	urb->transfer_flags = URB_FREE_BUFFER | URB_ISO_ASAP;
+
+	__fill_isoc_descriptor(urb, size,
+			       le16_to_cpu(data->isoc_rx_ep->wMaxPacketSize));
+
+	usb_anchor_urb(urb, &data->isoc_anchor);
+
+	err = usb_submit_urb(urb, mem_flags);
+	if (err < 0) {
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p submission failed (%d)",
+				   urb, -err);
+		usb_unanchor_urb(urb);
+	}
+
+	usb_free_urb(urb);
+
+	return err;
+}
+
+static void btusb_diag_complete(struct urb *urb)
+{
+	struct hci_dev *hdev = urb->context;
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
+	       urb->actual_length);
+
+	if (urb->status == 0) {
+		struct sk_buff *skb;
+
+		skb = bt_skb_alloc(urb->actual_length, GFP_ATOMIC);
+		if (skb) {
+			skb_put_data(skb, urb->transfer_buffer,
+				     urb->actual_length);
+			hci_recv_diag(hdev, skb);
+		}
+	} else if (urb->status == -ENOENT) {
+		/* Avoid suspend failed when usb_kill_urb */
+		return;
+	}
+
+	if (!test_bit(BTUSB_DIAG_RUNNING, &data->flags))
+		return;
+
+	usb_anchor_urb(urb, &data->diag_anchor);
+	usb_mark_last_busy(data->udev);
+
+	err = usb_submit_urb(urb, GFP_ATOMIC);
+	if (err < 0) {
+		/* -EPERM: urb is being killed;
+		 * -ENODEV: device got disconnected
+		 */
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p failed to resubmit (%d)",
+				   urb, -err);
+		usb_unanchor_urb(urb);
+	}
+}
+
+static int btusb_submit_diag_urb(struct hci_dev *hdev, gfp_t mem_flags)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct urb *urb;
+	unsigned char *buf;
+	unsigned int pipe;
+	int err, size = HCI_MAX_FRAME_SIZE;
+
+	BT_DBG("%s", hdev->name);
+
+	if (!data->diag_rx_ep)
+		return -ENODEV;
+
+	urb = usb_alloc_urb(0, mem_flags);
+	if (!urb)
+		return -ENOMEM;
+
+	buf = kmalloc(size, mem_flags);
+	if (!buf) {
+		usb_free_urb(urb);
+		return -ENOMEM;
+	}
+
+	pipe = usb_rcvbulkpipe(data->udev, data->diag_rx_ep->bEndpointAddress);
+
+	usb_fill_bulk_urb(urb, data->udev, pipe, buf, size,
+			  btusb_diag_complete, hdev);
+
+	urb->transfer_flags |= URB_FREE_BUFFER;
+
+	usb_mark_last_busy(data->udev);
+	usb_anchor_urb(urb, &data->diag_anchor);
+
+	err = usb_submit_urb(urb, mem_flags);
+	if (err < 0) {
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p submission failed (%d)",
+				   urb, -err);
+		usb_unanchor_urb(urb);
+	}
+
+	usb_free_urb(urb);
+
+	return err;
+}
+
+static void btusb_tx_complete(struct urb *urb)
+{
+	struct sk_buff *skb = urb->context;
+	struct hci_dev *hdev = (struct hci_dev *)skb->dev;
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	unsigned long flags;
+
+	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
+	       urb->actual_length);
+
+	if (!test_bit(HCI_RUNNING, &hdev->flags))
+		goto done;
+
+	if (!urb->status) {
+		hdev->stat.byte_tx += urb->transfer_buffer_length;
+	} else {
+		if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT)
+			hci_cmd_sync_cancel(hdev, -urb->status);
+		hdev->stat.err_tx++;
+	}
+
+done:
+	spin_lock_irqsave(&data->txlock, flags);
+	data->tx_in_flight--;
+	spin_unlock_irqrestore(&data->txlock, flags);
+
+	kfree(urb->setup_packet);
+
+	kfree_skb(skb);
+}
+
+static void btusb_isoc_tx_complete(struct urb *urb)
+{
+	struct sk_buff *skb = urb->context;
+	struct hci_dev *hdev = (struct hci_dev *)skb->dev;
+
+	BT_DBG("%s urb %p status %d count %d", hdev->name, urb, urb->status,
+	       urb->actual_length);
+
+	if (!test_bit(HCI_RUNNING, &hdev->flags))
+		goto done;
+
+	if (!urb->status)
+		hdev->stat.byte_tx += urb->transfer_buffer_length;
+	else
+		hdev->stat.err_tx++;
+
+done:
+	kfree(urb->setup_packet);
+
+	kfree_skb(skb);
+}
+
+static int btusb_open(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	BT_DBG("%s", hdev->name);
+
+	err = usb_autopm_get_interface(data->intf);
+	if (err < 0)
+		return err;
+
+	/* Patching USB firmware files prior to starting any URBs of HCI path
+	 * It is more safe to use USB bulk channel for downloading USB patch
+	 */
+	if (data->setup_on_usb) {
+		err = data->setup_on_usb(hdev);
+		if (err < 0)
+			goto setup_fail;
+	}
+
+	data->intf->needs_remote_wakeup = 1;
+
+	if (data->proto == BTUSB_PROTO_LEGACY) {
+		if (test_and_set_bit(BTUSB_INTR_RUNNING, &data->flags))
+			goto done;
+
+		err = btusb_submit_intr_urb(hdev, GFP_KERNEL);
+		if (err < 0)
+			goto failed;
+	}
+
+	err = btusb_submit_bulk_urb(hdev, GFP_KERNEL);
+	if (err < 0) {
+		usb_kill_anchored_urbs(&data->intr_anchor);
+		goto failed;
+	}
+
+	set_bit(BTUSB_BULK_RUNNING, &data->flags);
+	btusb_submit_bulk_urb(hdev, GFP_KERNEL);
+
+	if (data->diag) {
+		if (!btusb_submit_diag_urb(hdev, GFP_KERNEL))
+			set_bit(BTUSB_DIAG_RUNNING, &data->flags);
+	}
+
+done:
+	usb_autopm_put_interface(data->intf);
+	return 0;
+
+failed:
+	clear_bit(BTUSB_INTR_RUNNING, &data->flags);
+setup_fail:
+	usb_autopm_put_interface(data->intf);
+	return err;
+}
+
+static void btusb_stop_traffic(struct btusb_data *data)
+{
+	usb_kill_anchored_urbs(&data->intr_anchor);
+	usb_kill_anchored_urbs(&data->bulk_anchor);
+	usb_kill_anchored_urbs(&data->isoc_anchor);
+	usb_kill_anchored_urbs(&data->diag_anchor);
+	usb_kill_anchored_urbs(&data->ctrl_anchor);
+}
+
+static void btusb_prepare_reset(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+
+	btusb_stop_traffic(data);
+	usb_kill_anchored_urbs(&data->tx_anchor);
+}
+
+static int btusb_close(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	BT_DBG("%s", hdev->name);
+
+	cancel_work_sync(&data->work);
+	cancel_work_sync(&data->waker);
+
+	clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
+	clear_bit(BTUSB_BULK_RUNNING, &data->flags);
+	clear_bit(BTUSB_INTR_RUNNING, &data->flags);
+	clear_bit(BTUSB_DIAG_RUNNING, &data->flags);
+
+	btusb_stop_traffic(data);
+
+	/* rx_work must only be canceled once the URBs that can rearm it are
+	 * gone, and it must be canceled synchronously since btusb_disconnect()
+	 * frees the btusb_data it dereferences right after hci_unregister_dev().
+	 */
+	cancel_delayed_work_sync(&data->rx_work);
+
+	skb_queue_purge(&data->acl_q);
+
+	btusb_free_frags(data);
+
+	err = usb_autopm_get_interface(data->intf);
+	if (err < 0)
+		goto failed;
+
+	data->intf->needs_remote_wakeup = 0;
+
+	/* Enable remote wake up for auto-suspend */
+	if (test_bit(BTUSB_WAKEUP_AUTOSUSPEND, &data->flags))
+		data->intf->needs_remote_wakeup = 1;
+
+	usb_autopm_put_interface(data->intf);
+
+failed:
+	usb_scuttle_anchored_urbs(&data->deferred);
+	return 0;
+}
+
+static int btusb_flush(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+
+	BT_DBG("%s", hdev->name);
+
+	cancel_delayed_work_sync(&data->rx_work);
+
+	skb_queue_purge(&data->acl_q);
+
+	usb_kill_anchored_urbs(&data->tx_anchor);
+	btusb_free_frags(data);
+
+	return 0;
+}
+
+static struct urb *alloc_bulk_urb(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct urb *urb;
+	unsigned int pipe;
+
+	if (!data->bulk_tx_ep)
+		return ERR_PTR(-ENODEV);
+
+	if (data->proto == BTUSB_PROTO_H4) {
+		/* The frame type is prepended in place, so the buffer must not
+		 * be shared with anyone else.
+		 */
+		if (skb_cow_head(skb, 1))
+			return ERR_PTR(-ENOMEM);
+	}
+
+	urb = usb_alloc_urb(0, GFP_KERNEL);
+	if (!urb)
+		return ERR_PTR(-ENOMEM);
+
+	pipe = usb_sndbulkpipe(data->udev, data->bulk_tx_ep->bEndpointAddress);
+
+	if (data->proto == BTUSB_PROTO_H4) {
+		/* Prepend skb with frame type */
+		memcpy(skb_push(skb, 1), &hci_skb_pkt_type(skb), 1);
+	}
+
+	usb_fill_bulk_urb(urb, data->udev, pipe,
+			  skb->data, skb->len, btusb_tx_complete, skb);
+
+	skb->dev = (void *)hdev;
+
+	return urb;
+}
+
+static struct urb *alloc_ctrl_urb(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct usb_ctrlrequest *dr;
+	struct urb *urb;
+	unsigned int pipe;
+
+	if (data->proto == BTUSB_PROTO_H4)
+		return alloc_bulk_urb(hdev, skb);
+
+	urb = usb_alloc_urb(0, GFP_KERNEL);
+	if (!urb)
+		return ERR_PTR(-ENOMEM);
+
+	dr = kmalloc_obj(*dr);
+	if (!dr) {
+		usb_free_urb(urb);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	dr->bRequestType = data->cmdreq_type;
+	dr->bRequest     = data->cmdreq;
+	dr->wIndex       = 0;
+	dr->wValue       = 0;
+	dr->wLength      = __cpu_to_le16(skb->len);
+
+	pipe = usb_sndctrlpipe(data->udev, 0x00);
+
+	usb_fill_control_urb(urb, data->udev, pipe, (void *)dr,
+			     skb->data, skb->len, btusb_tx_complete, skb);
+
+	skb->dev = (void *)hdev;
+
+	return urb;
+}
+
+static struct urb *alloc_isoc_urb(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct urb *urb;
+	unsigned int pipe;
+
+	if (data->proto == BTUSB_PROTO_H4)
+		return alloc_bulk_urb(hdev, skb);
+
+	if (!data->isoc_tx_ep)
+		return ERR_PTR(-ENODEV);
+
+	urb = usb_alloc_urb(BTUSB_MAX_ISOC_FRAMES, GFP_KERNEL);
+	if (!urb)
+		return ERR_PTR(-ENOMEM);
+
+	pipe = usb_sndisocpipe(data->udev, data->isoc_tx_ep->bEndpointAddress);
+
+	usb_fill_int_urb(urb, data->udev, pipe,
+			 skb->data, skb->len, btusb_isoc_tx_complete,
+			 skb, data->isoc_tx_ep->bInterval);
+
+	urb->transfer_flags  = URB_ISO_ASAP;
+
+	if (data->isoc_altsetting == 6)
+		__fill_isoc_descriptor_msbc(urb, skb->len,
+					    le16_to_cpu(data->isoc_tx_ep->wMaxPacketSize),
+					    data);
+	else
+		__fill_isoc_descriptor(urb, skb->len,
+				       le16_to_cpu(data->isoc_tx_ep->wMaxPacketSize));
+	skb->dev = (void *)hdev;
+
+	return urb;
+}
+
+static int submit_tx_urb(struct hci_dev *hdev, struct urb *urb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	usb_anchor_urb(urb, &data->tx_anchor);
+
+	err = usb_submit_urb(urb, GFP_KERNEL);
+	if (err < 0) {
+		if (err != -EPERM && err != -ENODEV)
+			bt_dev_err(hdev, "urb %p submission failed (%d)",
+				   urb, -err);
+		kfree(urb->setup_packet);
+		usb_unanchor_urb(urb);
+	} else {
+		usb_mark_last_busy(data->udev);
+	}
+
+	usb_free_urb(urb);
+	return err;
+}
+
+static int submit_or_queue_tx_urb(struct hci_dev *hdev, struct urb *urb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	unsigned long flags;
+	bool suspending;
+
+	spin_lock_irqsave(&data->txlock, flags);
+	suspending = test_bit(BTUSB_SUSPENDING, &data->flags);
+	if (!suspending)
+		data->tx_in_flight++;
+	spin_unlock_irqrestore(&data->txlock, flags);
+
+	if (!suspending)
+		return submit_tx_urb(hdev, urb);
+
+	usb_anchor_urb(urb, &data->deferred);
+	schedule_work(&data->waker);
+
+	usb_free_urb(urb);
+	return 0;
+}
+
+static int submit_sco_urb(struct hci_dev *hdev, struct urb *urb)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+
+	/* In H:4 mode SCO frames are carried over the bulk endpoint and
+	 * complete via btusb_tx_complete(), which decrements tx_in_flight, so
+	 * they have to be accounted for like any other bulk transfer.
+	 * Isochronous transfers use btusb_isoc_tx_complete() instead, which
+	 * does not, so they must not be counted.
+	 */
+	if (data->proto == BTUSB_PROTO_H4)
+		return submit_or_queue_tx_urb(hdev, urb);
+
+	return submit_tx_urb(hdev, urb);
+}
+
+static int btusb_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct urb *urb;
+
+	BT_DBG("%s", hdev->name);
+
+	switch (hci_skb_pkt_type(skb)) {
+	case HCI_COMMAND_PKT:
+		urb = alloc_ctrl_urb(hdev, skb);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		hdev->stat.cmd_tx++;
+		return submit_or_queue_tx_urb(hdev, urb);
+
+	case HCI_ACLDATA_PKT:
+		urb = alloc_bulk_urb(hdev, skb);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		hdev->stat.acl_tx++;
+		return submit_or_queue_tx_urb(hdev, urb);
+
+	case HCI_SCODATA_PKT:
+		if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
+		    hci_conn_num(hdev, SCO_LINK) < 1)
+			return -ENODEV;
+
+		urb = alloc_isoc_urb(hdev, skb);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		hdev->stat.sco_tx++;
+		return submit_sco_urb(hdev, urb);
+
+	case HCI_ISODATA_PKT:
+		urb = alloc_bulk_urb(hdev, skb);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		return submit_or_queue_tx_urb(hdev, urb);
+	}
+
+	return -EILSEQ;
+}
+
+static void btusb_notify(struct hci_dev *hdev, unsigned int evt)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+
+	BT_DBG("%s evt %d", hdev->name, evt);
+
+	if (hci_conn_num(hdev, SCO_LINK) != data->sco_num) {
+		data->sco_num = hci_conn_num(hdev, SCO_LINK);
+		data->air_mode = evt;
+		schedule_work(&data->work);
+	}
+}
+
+static inline int __set_isoc_interface(struct hci_dev *hdev, int altsetting)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct usb_interface *intf = data->isoc;
+	struct usb_endpoint_descriptor *ep_desc;
+	int i, err;
+
+	if (!data->isoc)
+		return -ENODEV;
+
+	err = usb_set_interface(data->udev, data->isoc_ifnum, altsetting);
+	if (err < 0) {
+		bt_dev_err(hdev, "setting interface failed (%d)", -err);
+		return err;
+	}
+
+	data->isoc_altsetting = altsetting;
+
+	data->isoc_tx_ep = NULL;
+	data->isoc_rx_ep = NULL;
+
+	for (i = 0; i < intf->cur_altsetting->desc.bNumEndpoints; i++) {
+		ep_desc = &intf->cur_altsetting->endpoint[i].desc;
+
+		if (!data->isoc_tx_ep && usb_endpoint_is_isoc_out(ep_desc)) {
+			data->isoc_tx_ep = ep_desc;
+			continue;
+		}
+
+		if (!data->isoc_rx_ep && usb_endpoint_is_isoc_in(ep_desc)) {
+			data->isoc_rx_ep = ep_desc;
+			continue;
+		}
+	}
+
+	if (!data->isoc_tx_ep || !data->isoc_rx_ep) {
+		bt_dev_err(hdev, "invalid SCO descriptors");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int btusb_switch_alt_setting(struct hci_dev *hdev, int new_alts)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int err;
+
+	if (data->isoc_altsetting != new_alts) {
+		unsigned long flags;
+
+		clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
+		usb_kill_anchored_urbs(&data->isoc_anchor);
+
+		/* When isochronous alternate setting needs to be
+		 * changed, because SCO connection has been added
+		 * or removed, a packet fragment may be left in the
+		 * reassembling state. This could lead to wrongly
+		 * assembled fragments.
+		 *
+		 * Clear outstanding fragment when selecting a new
+		 * alternate setting.
+		 */
+		spin_lock_irqsave(&data->rxlock, flags);
+		dev_kfree_skb_irq(data->sco_skb);
+		data->sco_skb = NULL;
+		spin_unlock_irqrestore(&data->rxlock, flags);
+
+		err = __set_isoc_interface(hdev, new_alts);
+		if (err < 0)
+			return err;
+	}
+
+	if (!test_and_set_bit(BTUSB_ISOC_RUNNING, &data->flags)) {
+		if (btusb_submit_isoc_urb(hdev, GFP_KERNEL) < 0)
+			clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
+		else
+			btusb_submit_isoc_urb(hdev, GFP_KERNEL);
+	}
+
+	return 0;
+}
+
+static struct usb_host_interface *
+btusb_find_altsetting(struct usb_interface *intf, int alt)
+{
+	int i;
+
+	BT_DBG("Looking for Alt no :%d", alt);
+
+	if (!intf)
+		return NULL;
+
+	for (i = 0; i < intf->num_altsetting; i++) {
+		if (intf->altsetting[i].desc.bAlternateSetting == alt)
+			return &intf->altsetting[i];
+	}
+
+	return NULL;
+}
+
+static void btusb_work(struct work_struct *work)
+{
+	struct btusb_data *data = container_of(work, struct btusb_data, work);
+	struct hci_dev *hdev = data->hdev;
+	int new_alts = 0;
+	int err;
+
+	/* In H:4 mode SCO/ISO data is carried over the bulk endpoints, so
+	 * there is no isochronous interface to resume or to switch alternate
+	 * settings on.
+	 */
+	if (data->proto == BTUSB_PROTO_H4)
+		return;
+
+	if (data->sco_num > 0) {
+		if (!test_bit(BTUSB_DID_ISO_RESUME, &data->flags)) {
+			err = usb_autopm_get_interface(data->isoc ? data->isoc : data->intf);
+			if (err < 0) {
+				clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
+				usb_kill_anchored_urbs(&data->isoc_anchor);
+				return;
+			}
+
+			set_bit(BTUSB_DID_ISO_RESUME, &data->flags);
+		}
+
+		if (data->air_mode == HCI_NOTIFY_ENABLE_SCO_CVSD) {
+			if (hdev->voice_setting & 0x0020) {
+				static const int alts[3] = { 2, 4, 5 };
+				unsigned int sco_idx;
+
+				sco_idx = min_t(unsigned int, data->sco_num - 1,
+						ARRAY_SIZE(alts) - 1);
+				new_alts = alts[sco_idx];
+			} else {
+				new_alts = data->sco_num;
+			}
+		} else if (data->air_mode == HCI_NOTIFY_ENABLE_SCO_TRANSP) {
+			/* Bluetooth USB spec recommends alt 6 (63 bytes), but
+			 * many adapters do not support it.  Alt 1 appears to
+			 * work for all adapters that do not have alt 6, and
+			 * which work with WBS at all.  Some devices prefer
+			 * alt 3 (HCI payload >= 60 Bytes let air packet
+			 * data satisfy 60 bytes), requiring
+			 * MTU >= 3 (packets) * 25 (size) - 3 (headers) = 72
+			 * see also Core spec 5, vol 4, B 2.1.1 & Table 2.1.
+			 */
+			if (btusb_find_altsetting(data->isoc, 6))
+				new_alts = 6;
+			else if (btusb_find_altsetting(data->isoc, 3) &&
+				 hdev->sco_mtu >= 72 &&
+				 test_bit(BTUSB_USE_ALT3_FOR_WBS, &data->flags))
+				new_alts = 3;
+			else
+				new_alts = 1;
+		}
+
+		if (btusb_switch_alt_setting(hdev, new_alts) < 0)
+			bt_dev_err(hdev, "set USB alt:(%d) failed!", new_alts);
+	} else {
+		usb_kill_anchored_urbs(&data->isoc_anchor);
+
+		if (test_and_clear_bit(BTUSB_ISOC_RUNNING, &data->flags))
+			__set_isoc_interface(hdev, 0);
+
+		if (test_and_clear_bit(BTUSB_DID_ISO_RESUME, &data->flags))
+			usb_autopm_put_interface(data->isoc ? data->isoc : data->intf);
+	}
+}
+
+static void btusb_waker(struct work_struct *work)
+{
+	struct btusb_data *data = container_of(work, struct btusb_data, waker);
+	int err;
+
+	err = usb_autopm_get_interface(data->intf);
+	if (err < 0)
+		return;
+
+	usb_autopm_put_interface(data->intf);
+}
+
+static void btusb_rx_work(struct work_struct *work)
+{
+	struct btusb_data *data = container_of(work, struct btusb_data,
+					       rx_work.work);
+	struct sk_buff *skb;
+
+	/* Dequeue ACL data received during the interval */
+	while ((skb = skb_dequeue(&data->acl_q)))
+		data->recv_acl(data->hdev, skb);
+}
+
+static int btusb_setup_bcm92035(struct hci_dev *hdev)
+{
+	struct sk_buff *skb;
+	u8 val = 0x00;
+
+	BT_DBG("%s", hdev->name);
+
+	skb = __hci_cmd_sync(hdev, 0xfc3b, 1, &val, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb))
+		bt_dev_err(hdev, "BCM92035 command failed (%ld)", PTR_ERR(skb));
+	else
+		kfree_skb(skb);
+
+	return 0;
+}
+
+static int btusb_setup_csr(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	u16 bcdDevice = le16_to_cpu(data->udev->descriptor.bcdDevice);
+	struct hci_rp_read_local_version *rp;
+	struct sk_buff *skb;
+	bool is_fake = false;
+	int ret;
+
+	BT_DBG("%s", hdev->name);
+
+	skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_VERSION, 0, NULL,
+			     HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		int err = PTR_ERR(skb);
+
+		bt_dev_err(hdev, "CSR: Local version failed (%d)", err);
+		return err;
+	}
+
+	rp = skb_pull_data(skb, sizeof(*rp));
+	if (!rp) {
+		bt_dev_err(hdev, "CSR: Local version length mismatch");
+		kfree_skb(skb);
+		return -EIO;
+	}
+
+	bt_dev_info(hdev, "CSR: Setting up dongle with HCI ver=%u rev=%04x",
+		    rp->hci_ver, le16_to_cpu(rp->hci_rev));
+
+	bt_dev_info(hdev, "LMP ver=%u subver=%04x; manufacturer=%u",
+		    rp->lmp_ver, le16_to_cpu(rp->lmp_subver),
+		    le16_to_cpu(rp->manufacturer));
+
+	/* Detect a wide host of Chinese controllers that aren't CSR.
+	 *
+	 * Known fake bcdDevices: 0x0100, 0x0134, 0x1915, 0x2520, 0x7558, 0x8891
+	 *
+	 * The main thing they have in common is that these are really popular low-cost
+	 * options that support newer Bluetooth versions but rely on heavy VID/PID
+	 * squatting of this poor old Bluetooth 1.1 device. Even sold as such.
+	 *
+	 * We detect actual CSR devices by checking that the HCI manufacturer code
+	 * is Cambridge Silicon Radio (10) and ensuring that LMP sub-version and
+	 * HCI rev values always match. As they both store the firmware number.
+	 */
+	if (le16_to_cpu(rp->manufacturer) != 10 ||
+	    le16_to_cpu(rp->hci_rev) != le16_to_cpu(rp->lmp_subver))
+		is_fake = true;
+
+	/* Known legit CSR firmware build numbers and their supported BT versions:
+	 * - 1.1 (0x1) -> 0x0073, 0x020d, 0x033c, 0x034e
+	 * - 1.2 (0x2) ->                 0x04d9, 0x0529
+	 * - 2.0 (0x3) ->         0x07a6, 0x07ad, 0x0c5c
+	 * - 2.1 (0x4) ->         0x149c, 0x1735, 0x1899 (0x1899 is a BlueCore4-External)
+	 * - 4.0 (0x6) ->         0x1d86, 0x2031, 0x22bb
+	 *
+	 * e.g. Real CSR dongles with LMP subversion 0x73 are old enough that
+	 *      support BT 1.1 only; so it's a dead giveaway when some
+	 *      third-party BT 4.0 dongle reuses it.
+	 */
+	else if (le16_to_cpu(rp->lmp_subver) <= 0x034e &&
+		 rp->hci_ver > BLUETOOTH_VER_1_1)
+		is_fake = true;
+
+	else if (le16_to_cpu(rp->lmp_subver) <= 0x0529 &&
+		 rp->hci_ver > BLUETOOTH_VER_1_2)
+		is_fake = true;
+
+	else if (le16_to_cpu(rp->lmp_subver) <= 0x0c5c &&
+		 rp->hci_ver > BLUETOOTH_VER_2_0)
+		is_fake = true;
+
+	else if (le16_to_cpu(rp->lmp_subver) <= 0x1899 &&
+		 rp->hci_ver > BLUETOOTH_VER_2_1)
+		is_fake = true;
+
+	else if (le16_to_cpu(rp->lmp_subver) <= 0x22bb &&
+		 rp->hci_ver > BLUETOOTH_VER_4_0)
+		is_fake = true;
+
+	/* Other clones which beat all the above checks */
+	else if (bcdDevice == 0x0134 &&
+		 le16_to_cpu(rp->lmp_subver) == 0x0c5c &&
+		 rp->hci_ver == BLUETOOTH_VER_2_0)
+		is_fake = true;
+
+	if (is_fake) {
+		bt_dev_warn(hdev, "CSR: Unbranded CSR clone detected; adding workarounds and force-suspending once...");
+
+		/* Generally these clones have big discrepancies between
+		 * advertised features and what's actually supported.
+		 * Probably will need to be expanded in the future;
+		 * without these the controller will lock up.
+		 */
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_STORED_LINK_KEY);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ERR_DATA_REPORTING);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_FILTER_CLEAR_ALL);
+		hci_set_quirk(hdev, HCI_QUIRK_NO_SUSPEND_NOTIFIER);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_VOICE_SETTING);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_PAGE_SCAN_TYPE);
+
+		/* Clear the reset quirk since this is not an actual
+		 * early Bluetooth 1.1 device from CSR.
+		 */
+		hci_clear_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
+		hci_clear_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
+
+		/*
+		 * Special workaround for these BT 4.0 chip clones, and potentially more:
+		 *
+		 * - 0x0134: a Barrot 8041a02                 (HCI rev: 0x0810 sub: 0x1012)
+		 * - 0x7558: IC markings FR3191AHAL 749H15143 (HCI rev/sub-version: 0x0709)
+		 *
+		 * These controllers are really messed-up.
+		 *
+		 * 1. Their bulk RX endpoint will never report any data unless
+		 *    the device was suspended at least once (yes, really).
+		 * 2. They will not wakeup when autosuspended and receiving data
+		 *    on their bulk RX endpoint from e.g. a keyboard or mouse
+		 *    (IOW remote-wakeup support is broken for the bulk endpoint).
+		 *
+		 * To fix 1. enable runtime-suspend, force-suspend the
+		 * HCI and then wake-it up by disabling runtime-suspend.
+		 *
+		 * To fix 2. clear the HCI's can_wake flag, this way the HCI
+		 * will still be autosuspended when it is not open.
+		 *
+		 * --
+		 *
+		 * Because these are widespread problems we prefer generic solutions; so
+		 * apply this initialization quirk to every controller that gets here,
+		 * it should be harmless. The alternative is to not work at all.
+		 */
+		pm_runtime_allow(&data->udev->dev);
+
+		ret = pm_runtime_suspend(&data->udev->dev);
+		if (ret >= 0)
+			msleep(200);
+		else
+			bt_dev_warn(hdev, "CSR: Couldn't suspend the device for our Barrot 8041a02 receive-issue workaround");
+
+		pm_runtime_forbid(&data->udev->dev);
+
+		device_set_wakeup_capable(&data->udev->dev, false);
+
+		/* Re-enable autosuspend if this was requested */
+		if (enable_autosuspend)
+			usb_enable_autosuspend(data->udev);
+	}
+
+	kfree_skb(skb);
+
+	return 0;
+}
+
+static int inject_cmd_complete(struct hci_dev *hdev, __u16 opcode)
+{
+	struct sk_buff *skb;
+	struct hci_event_hdr *hdr;
+	struct hci_ev_cmd_complete *evt;
+
+	skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL);
+	if (!skb)
+		return -ENOMEM;
+
+	hdr = skb_put(skb, sizeof(*hdr));
+	hdr->evt = HCI_EV_CMD_COMPLETE;
+	hdr->plen = sizeof(*evt) + 1;
+
+	evt = skb_put(skb, sizeof(*evt));
+	evt->ncmd = 0x01;
+	evt->opcode = cpu_to_le16(opcode);
+
+	skb_put_u8(skb, 0x00);
+
+	hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
+
+	return hci_recv_frame(hdev, skb);
+}
+
+static int btusb_recv_bulk_intel(struct btusb_data *data, void *buffer,
+				 int count)
+{
+	struct hci_dev *hdev = data->hdev;
+
+	/* When the device is in bootloader mode, then it can send
+	 * events via the bulk endpoint. These events are treated the
+	 * same way as the ones received from the interrupt endpoint.
+	 *
+	 * In H:4 mode there is no interrupt endpoint and every frame on the
+	 * bulk endpoint carries an H:4 header, including the ones sent by the
+	 * bootloader, so the regular decoding applies.
+	 */
+	if (data->proto == BTUSB_PROTO_LEGACY &&
+	    btintel_test_flag(hdev, INTEL_BOOTLOADER))
+		return data->recv_intr(data, buffer, count);
+
+	return btusb_recv_bulk(data, buffer, count);
+}
+
+static int btusb_send_frame_intel(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct urb *urb;
+
+	BT_DBG("%s", hdev->name);
+
+	switch (hci_skb_pkt_type(skb)) {
+	case HCI_COMMAND_PKT:
+		if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
+			struct hci_command_hdr *cmd = (void *)skb->data;
+			__u16 opcode = le16_to_cpu(cmd->opcode);
+
+			/* When in bootloader mode and the command 0xfc09
+			 * is received, it needs to be send down the
+			 * bulk endpoint. So allocate a bulk URB instead.
+			 */
+			if (opcode == 0xfc09)
+				urb = alloc_bulk_urb(hdev, skb);
+			else
+				urb = alloc_ctrl_urb(hdev, skb);
+
+			/* When the BTINTEL_HCI_OP_RESET command is issued to
+			 * boot into the operational firmware, it will actually
+			 * not send a command complete event. To keep the flow
+			 * control working inject that event here.
+			 */
+			if (opcode == BTINTEL_HCI_OP_RESET)
+				inject_cmd_complete(hdev, opcode);
+		} else {
+			urb = alloc_ctrl_urb(hdev, skb);
+		}
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		hdev->stat.cmd_tx++;
+		return submit_or_queue_tx_urb(hdev, urb);
+
+	case HCI_ACLDATA_PKT:
+		urb = alloc_bulk_urb(hdev, skb);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		hdev->stat.acl_tx++;
+		return submit_or_queue_tx_urb(hdev, urb);
+
+	case HCI_SCODATA_PKT:
+		if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
+		    hci_conn_num(hdev, SCO_LINK) < 1)
+			return -ENODEV;
+
+		urb = alloc_isoc_urb(hdev, skb);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		hdev->stat.sco_tx++;
+		return submit_sco_urb(hdev, urb);
+
+	case HCI_ISODATA_PKT:
+		urb = alloc_bulk_urb(hdev, skb);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		return submit_or_queue_tx_urb(hdev, urb);
+	}
+
+	return -EILSEQ;
+}
+
+static int btusb_setup_realtek(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	int ret;
+
+	ret = btrtl_setup_realtek(hdev);
+
+	if (btrealtek_test_flag(data->hdev, REALTEK_ALT6_CONTINUOUS_TX_CHIP))
+		set_bit(BTUSB_ALT6_CONTINUOUS_TX, &data->flags);
+
+	return ret;
+}
+
+static int btusb_recv_event_realtek(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	if (skb->len >= HCI_EVENT_HDR_SIZE + 1 &&
+	    skb->data[0] == HCI_EV_VENDOR &&
+	    skb->data[2] == RTK_SUB_EVENT_CODE_COREDUMP) {
+		struct rtk_dev_coredump_hdr hdr = {
+			.code = RTK_DEVCOREDUMP_CODE_MEMDUMP,
+		};
+
+		bt_dev_dbg(hdev, "RTL: received coredump vendor evt, len %u",
+			skb->len);
+
+		btusb_rtl_alloc_devcoredump(hdev, &hdr, skb->data, skb->len);
+		kfree_skb(skb);
+
+		return 0;
+	}
+
+	return hci_recv_frame(hdev, skb);
+}
+
+static void btusb_mtk_claim_iso_intf(struct btusb_data *data)
+{
+	struct btmtk_data *btmtk_data;
+	int err;
+
+	if (!data->hdev)
+		return;
+
+	btmtk_data = hci_get_priv(data->hdev);
+	if (!btmtk_data)
+		return;
+
+	if (!btmtk_data->isopkt_intf) {
+		bt_dev_err(data->hdev, "Can't claim NULL iso interface");
+		return;
+	}
+
+	/*
+	 * The function usb_driver_claim_interface() is documented to need
+	 * locks held if it's not called from a probe routine. The code here
+	 * is called from the hci_power_on workqueue, so grab the lock.
+	 */
+	device_lock(&btmtk_data->isopkt_intf->dev);
+	err = usb_driver_claim_interface(&btusb_driver,
+					 btmtk_data->isopkt_intf, data);
+	device_unlock(&btmtk_data->isopkt_intf->dev);
+	if (err < 0) {
+		btmtk_data->isopkt_intf = NULL;
+		bt_dev_err(data->hdev, "Failed to claim iso interface: %d", err);
+		return;
+	}
+
+	set_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags);
+	init_usb_anchor(&btmtk_data->isopkt_anchor);
+}
+
+static void btusb_mtk_release_iso_intf(struct hci_dev *hdev)
+{
+	struct btmtk_data *btmtk_data;
+
+	if (!hdev)
+		return;
+
+	btmtk_data = hci_get_priv(hdev);
+	if (!btmtk_data)
+		return;
+
+	if (test_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags)) {
+		usb_kill_anchored_urbs(&btmtk_data->isopkt_anchor);
+		clear_bit(BTMTK_ISOPKT_RUNNING, &btmtk_data->flags);
+
+		if (btmtk_data->isopkt_skb) {
+			dev_kfree_skb_irq(btmtk_data->isopkt_skb);
+			btmtk_data->isopkt_skb = NULL;
+		}
+
+		if (btmtk_data->isopkt_intf) {
+			usb_set_intfdata(btmtk_data->isopkt_intf, NULL);
+			usb_driver_release_interface(&btusb_driver,
+						     btmtk_data->isopkt_intf);
+			btmtk_data->isopkt_intf = NULL;
+		}
+	}
+
+	clear_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags);
+}
+
+static int btusb_mtk_disconnect(struct hci_dev *hdev)
+{
+	/* This function describes the specific additional steps taken by MediaTek
+	 * when Bluetooth usb driver's resume function is called.
+	 */
+	btusb_mtk_release_iso_intf(hdev);
+
+	return 0;
+}
+
+static int btusb_mtk_reset(struct hci_dev *hdev, void *rst_data)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct btmtk_data *btmtk_data = hci_get_priv(hdev);
+	int err;
+
+	/* It's MediaTek specific bluetooth reset mechanism via USB */
+	if (test_and_set_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags)) {
+		bt_dev_err(hdev, "last reset failed? Not resetting again");
+		return -EBUSY;
+	}
+
+	err = usb_autopm_get_interface(data->intf);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err);
+		clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags);
+		return err;
+	}
+
+	/* Release MediaTek ISO data interface */
+	btusb_mtk_release_iso_intf(hdev);
+
+	btusb_prepare_reset(hdev);
+
+	/* Toggle the hard reset line. The MediaTek device is going to
+	 * yank itself off the USB and then replug. The cleanup is handled
+	 * correctly on the way out (standard USB disconnect), and the new
+	 * device is detected cleanly and bound to the driver again like
+	 * it should be.
+	 */
+	if (data->reset_gpio) {
+		gpiod_set_value_cansleep(data->reset_gpio, 1);
+		msleep(200);
+		gpiod_set_value_cansleep(data->reset_gpio, 0);
+		return 0;
+	}
+
+	err = btmtk_usb_subsys_reset(hdev, btmtk_data->dev_id);
+
+	if (test_and_set_bit(BTUSB_RESET, &data->flags)) {
+		bt_dev_err(hdev, "last usb reset failed? Resetting again");
+		usb_autopm_put_interface_no_suspend(data->intf);
+	}
+
+	usb_queue_reset_device(data->intf);
+	clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags);
+
+	return err;
+}
+
+static int btusb_send_frame_mtk(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct urb *urb;
+
+	BT_DBG("%s", hdev->name);
+
+	if (hci_skb_pkt_type(skb) == HCI_ISODATA_PKT) {
+		urb = alloc_mtk_intr_urb(hdev, skb, btusb_tx_complete);
+		if (IS_ERR(urb))
+			return PTR_ERR(urb);
+
+		return submit_or_queue_tx_urb(hdev, urb);
+	} else {
+		return btusb_send_frame(hdev, skb);
+	}
+}
+
+static inline bool platform_is_ryzen(void)
+{
+#ifdef CONFIG_X86
+	return boot_cpu_has(X86_FEATURE_ZEN);
+#else
+	return false;
+#endif
+}
+
+static inline bool is_direct_child_of_root_hub(struct usb_device *udev)
+{
+	return udev->parent == udev->bus->root_hub;
+}
+
+static int btusb_mtk_setup(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct btmtk_data *btmtk_data = hci_get_priv(hdev);
+	int err;
+
+	/* MediaTek WMT vendor cmd requiring below USB resources to
+	 * complete the handshake.
+	 */
+	btmtk_data->drv_name = btusb_driver.name;
+	btmtk_data->intf = data->intf;
+	btmtk_data->udev = data->udev;
+	btmtk_data->ctrl_anchor = &data->ctrl_anchor;
+	btmtk_data->reset_sync = btusb_mtk_reset;
+
+	/* Claim ISO data interface and endpoint */
+	if (!test_bit(BTMTK_ISOPKT_OVER_INTR, &btmtk_data->flags)) {
+		btmtk_data->isopkt_intf = usb_ifnum_to_if(data->udev, MTK_ISO_IFNUM);
+		btusb_mtk_claim_iso_intf(data);
+	}
+
+	err = btmtk_usb_setup(hdev);
+	if (err)
+		return err;
+
+	switch (btmtk_data->dev_id) {
+	case 0x7922:
+	case 0x7925:
+		/*
+		 * All reports seen to be relevant to Ryzen-based laptops. These
+		 * NICs are usually used as OEM components thanks to some sort
+		 * of reference designs.
+		 *
+		 * Their popularity on other platforms is unclear. While there
+		 * is still a chance that the quirk may exist on other
+		 * platforms, be cautious and only apply the quirk to direct
+		 * children of Ryzen platforms's root hubs for the time being.
+		 *
+		 * In most cases the root hub is on the SoC or PCH, which needs
+		 * the quirk. Unfortunately, this can't distinguish root hubs on
+		 * PCIe add-in cards. Such roughness should be acceptable, as
+		 * PCIe USB controller add-in cards are less commonly used
+		 * nowadays. On the other hand, applying the quirk doesn't hurt
+		 * any functionalities either, as the device can still be used
+		 * as a wakeup source if desired.
+		 *
+		 * Theoretically, we could retrieve the root hub's PCI vendor ID
+		 * with some hierarchy magic, but that's too intrusive...
+		 */
+		if (platform_is_ryzen() && is_direct_child_of_root_hub(data->udev))
+			set_bit(BTUSB_WAKEUP_BROKEN, &data->flags);
+		break;
+	}
+
+	return 0;
+}
+
+static int btusb_mtk_shutdown(struct hci_dev *hdev)
+{
+	int ret;
+
+	ret = btmtk_usb_shutdown(hdev);
+
+	/* Release MediaTek iso interface after shutdown */
+	btusb_mtk_release_iso_intf(hdev);
+
+	return ret;
+}
+
+#ifdef CONFIG_PM
+/* Configure an out-of-band gpio as wake-up pin, if specified in device tree */
+static int marvell_config_oob_wake(struct hci_dev *hdev)
+{
+	struct sk_buff *skb;
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct device *dev = &data->udev->dev;
+	u16 pin, gap, opcode;
+	int ret;
+	u8 cmd[5];
+
+	/* Move on if no wakeup pin specified */
+	if (of_property_read_u16(dev->of_node, "marvell,wakeup-pin", &pin) ||
+	    of_property_read_u16(dev->of_node, "marvell,wakeup-gap-ms", &gap))
+		return 0;
+
+	/* Vendor specific command to configure a GPIO as wake-up pin */
+	opcode = hci_opcode_pack(0x3F, 0x59);
+	cmd[0] = opcode & 0xFF;
+	cmd[1] = opcode >> 8;
+	cmd[2] = 2; /* length of parameters that follow */
+	cmd[3] = pin;
+	cmd[4] = gap; /* time in ms, for which wakeup pin should be asserted */
+
+	skb = bt_skb_alloc(sizeof(cmd), GFP_KERNEL);
+	if (!skb) {
+		bt_dev_err(hdev, "%s: No memory", __func__);
+		return -ENOMEM;
+	}
+
+	skb_put_data(skb, cmd, sizeof(cmd));
+	hci_skb_pkt_type(skb) = HCI_COMMAND_PKT;
+
+	ret = btusb_send_frame(hdev, skb);
+	if (ret) {
+		bt_dev_err(hdev, "%s: configuration failed", __func__);
+		kfree_skb(skb);
+		return ret;
+	}
+
+	return 0;
+}
+#else
+static inline int marvell_config_oob_wake(struct hci_dev *hdev)
+{
+	return 0;
+}
+#endif
+
+static int btusb_set_bdaddr_marvell(struct hci_dev *hdev,
+				    const bdaddr_t *bdaddr)
+{
+	struct sk_buff *skb;
+	u8 buf[8];
+	long ret;
+
+	buf[0] = 0xfe;
+	buf[1] = sizeof(bdaddr_t);
+	memcpy(buf + 2, bdaddr, sizeof(bdaddr_t));
+
+	skb = __hci_cmd_sync(hdev, 0xfc22, sizeof(buf), buf, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		ret = PTR_ERR(skb);
+		bt_dev_err(hdev, "changing Marvell device address failed (%ld)",
+			   ret);
+		return ret;
+	}
+	kfree_skb(skb);
+
+	return 0;
+}
+
+static int btusb_set_bdaddr_ath3012(struct hci_dev *hdev,
+				    const bdaddr_t *bdaddr)
+{
+	struct sk_buff *skb;
+	u8 buf[10];
+	long ret;
+
+	buf[0] = 0x01;
+	buf[1] = 0x01;
+	buf[2] = 0x00;
+	buf[3] = sizeof(bdaddr_t);
+	memcpy(buf + 4, bdaddr, sizeof(bdaddr_t));
+
+	skb = __hci_cmd_sync(hdev, 0xfc0b, sizeof(buf), buf, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		ret = PTR_ERR(skb);
+		bt_dev_err(hdev, "Change address command failed (%ld)", ret);
+		return ret;
+	}
+	kfree_skb(skb);
+
+	return 0;
+}
+
+static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev,
+				const bdaddr_t *bdaddr)
+{
+	bdaddr_t bdaddr_swapped;
+	struct sk_buff *skb;
+	long ret;
+
+	baswap(&bdaddr_swapped, bdaddr);
+
+	skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(bdaddr_swapped),
+				&bdaddr_swapped, HCI_EV_CMD_COMPLETE,
+				HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		ret = PTR_ERR(skb);
+		bt_dev_err(hdev, "Change address command failed (%ld)", ret);
+		return ret;
+	}
+	kfree_skb(skb);
+
+	return 0;
+}
+
+#define QCA_MEMDUMP_ACL_HANDLE 0x2EDD
+#define QCA_MEMDUMP_SIZE_MAX  0x100000
+#define QCA_MEMDUMP_VSE_CLASS 0x01
+#define QCA_MEMDUMP_MSG_TYPE 0x08
+#define QCA_MEMDUMP_PKT_SIZE 248
+#define QCA_LAST_SEQUENCE_NUM 0xffff
+
+struct qca_dump_hdr {
+	u8 vse_class;
+	u8 msg_type;
+	__le16 seqno;
+	u8 reserved;
+	union {
+		DECLARE_FLEX_ARRAY(u8, data);
+		struct {
+			__le32 ram_dump_size;
+			DECLARE_FLEX_ARRAY(u8, data0);
+		} __packed;
+	};
+} __packed;
+
+
+static void btusb_dump_hdr_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	char buf[128];
+	struct btqca_data *btqca_data = hci_get_priv(hdev);
+	struct qca_dump_info *qca_dump_ptr = &btqca_data->qca_dump;
+
+	snprintf(buf, sizeof(buf), "Controller Name: 0x%x\n",
+			qca_dump_ptr->controller_id);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Firmware Version: 0x%x\n",
+			qca_dump_ptr->fw_version);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Driver: %s\nVendor: qca\n",
+			btusb_driver.name);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "VID: 0x%x\nPID:0x%x\n",
+			qca_dump_ptr->id_vendor, qca_dump_ptr->id_product);
+	skb_put_data(skb, buf, strlen(buf));
+
+	snprintf(buf, sizeof(buf), "Lmp Subversion: 0x%x\n",
+			hdev->lmp_subver);
+	skb_put_data(skb, buf, strlen(buf));
+}
+
+static void btusb_coredump_qca(struct hci_dev *hdev)
+{
+	int err;
+	static const u8 param[] = { 0x26 };
+
+	err = __hci_cmd_send(hdev, 0xfc0c, 1, param);
+	if (err < 0)
+		bt_dev_err(hdev, "%s: triggle crash failed (%d)", __func__, err);
+}
+
+/* Return: 0 on success, negative errno on failure. */
+static int handle_dump_pkt_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	int ret = 0;
+	unsigned int skip = 0;
+	u8 pkt_type;
+	u16 seqno;
+	u32 dump_size;
+
+	struct qca_dump_hdr *dump_hdr;
+	struct btusb_data *btdata = hci_get_drvdata(hdev);
+	struct btqca_data *btqca_data = hci_get_priv(hdev);
+	struct qca_dump_info *qca_dump_ptr = &btqca_data->qca_dump;
+	struct usb_device *udev = btdata->udev;
+
+	pkt_type = hci_skb_pkt_type(skb);
+	skip = sizeof(struct hci_event_hdr);
+	if (pkt_type == HCI_ACLDATA_PKT)
+		skip += sizeof(struct hci_acl_hdr);
+
+	skb_pull(skb, skip);
+	dump_hdr = (struct qca_dump_hdr *)skb->data;
+
+	seqno = le16_to_cpu(dump_hdr->seqno);
+	if (seqno == 0) {
+		set_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags);
+		dump_size = le32_to_cpu(dump_hdr->ram_dump_size);
+		if (!dump_size || (dump_size > QCA_MEMDUMP_SIZE_MAX)) {
+			ret = -EILSEQ;
+			bt_dev_err(hdev, "Invalid memdump size(%u)",
+				   dump_size);
+			goto out;
+		}
+
+		ret = hci_devcd_init(hdev, dump_size);
+		if (ret < 0) {
+			bt_dev_err(hdev, "memdump init error(%d)", ret);
+			goto out;
+		}
+
+		qca_dump_ptr->ram_dump_size = dump_size;
+		qca_dump_ptr->ram_dump_seqno = 0;
+
+		skb_pull(skb, offsetof(struct qca_dump_hdr, data0));
+
+		usb_disable_autosuspend(udev);
+		bt_dev_info(hdev, "%s memdump size(%u)\n",
+			    (pkt_type == HCI_ACLDATA_PKT) ? "ACL" : "event",
+			    dump_size);
+	} else {
+		skb_pull(skb, offsetof(struct qca_dump_hdr, data));
+	}
+
+	if (!qca_dump_ptr->ram_dump_size) {
+		ret = -EINVAL;
+		bt_dev_err(hdev, "memdump is not active");
+		goto out;
+	}
+
+	if ((seqno > qca_dump_ptr->ram_dump_seqno + 1) && seqno != QCA_LAST_SEQUENCE_NUM) {
+		dump_size = QCA_MEMDUMP_PKT_SIZE * (seqno - qca_dump_ptr->ram_dump_seqno - 1);
+		hci_devcd_append_pattern(hdev, 0x0, dump_size);
+		bt_dev_err(hdev,
+			   "expected memdump seqno(%u) is not received(%u)\n",
+			   qca_dump_ptr->ram_dump_seqno, seqno);
+		qca_dump_ptr->ram_dump_seqno = seqno;
+		kfree_skb(skb);
+		return ret;
+	}
+
+	hci_devcd_append(hdev, skb);
+	qca_dump_ptr->ram_dump_seqno++;
+	if (seqno == QCA_LAST_SEQUENCE_NUM) {
+		bt_dev_info(hdev,
+				"memdump done: pkts(%u), total(%u)\n",
+				qca_dump_ptr->ram_dump_seqno, qca_dump_ptr->ram_dump_size);
+
+		hci_devcd_complete(hdev);
+		goto out;
+	}
+	return ret;
+
+out:
+	if (qca_dump_ptr->ram_dump_size)
+		usb_enable_autosuspend(udev);
+	qca_dump_ptr->ram_dump_size = 0;
+	qca_dump_ptr->ram_dump_seqno = 0;
+	clear_bit(BTUSB_HW_SSR_ACTIVE, &btdata->flags);
+
+	if (ret < 0)
+		kfree_skb(skb);
+	return ret;
+}
+
+/* Return: true if the ACL packet is a dump packet, false otherwise. */
+static bool acl_pkt_is_dump_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct hci_event_hdr *event_hdr;
+	struct hci_acl_hdr *acl_hdr;
+	struct qca_dump_hdr *dump_hdr;
+	struct sk_buff *clone = skb_clone(skb, GFP_ATOMIC);
+	bool is_dump = false;
+
+	if (!clone)
+		return false;
+
+	acl_hdr = skb_pull_data(clone, sizeof(*acl_hdr));
+	if (!acl_hdr || (le16_to_cpu(acl_hdr->handle) != QCA_MEMDUMP_ACL_HANDLE))
+		goto out;
+
+	event_hdr = skb_pull_data(clone, sizeof(*event_hdr));
+	if (!event_hdr || event_hdr->evt != HCI_EV_VENDOR)
+		goto out;
+
+	dump_hdr = skb_pull_data(clone, sizeof(*dump_hdr));
+	if (!dump_hdr || (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS) ||
+	   (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE))
+		goto out;
+
+	is_dump = true;
+out:
+	consume_skb(clone);
+	return is_dump;
+}
+
+/* Return: true if the event packet is a dump packet, false otherwise. */
+static bool evt_pkt_is_dump_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct hci_event_hdr *event_hdr;
+	struct qca_dump_hdr *dump_hdr;
+	struct sk_buff *clone = skb_clone(skb, GFP_ATOMIC);
+	bool is_dump = false;
+
+	if (!clone)
+		return false;
+
+	event_hdr = skb_pull_data(clone, sizeof(*event_hdr));
+	if (!event_hdr || event_hdr->evt != HCI_EV_VENDOR)
+		goto out;
+
+	dump_hdr = skb_pull_data(clone, sizeof(*dump_hdr));
+	if (!dump_hdr || (dump_hdr->vse_class != QCA_MEMDUMP_VSE_CLASS) ||
+	   (dump_hdr->msg_type != QCA_MEMDUMP_MSG_TYPE))
+		goto out;
+
+	is_dump = true;
+out:
+	consume_skb(clone);
+	return is_dump;
+}
+
+static int btusb_recv_acl_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	if (acl_pkt_is_dump_qca(hdev, skb))
+		return handle_dump_pkt_qca(hdev, skb);
+	return hci_recv_frame(hdev, skb);
+}
+
+static int btusb_recv_evt_qca(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	if (evt_pkt_is_dump_qca(hdev, skb))
+		return handle_dump_pkt_qca(hdev, skb);
+	return hci_recv_frame(hdev, skb);
+}
+
+
+#define QCA_DFU_PACKET_LEN	4096
+
+#define QCA_GET_TARGET_VERSION	0x09
+#define QCA_CHECK_STATUS	0x05
+#define QCA_DFU_DOWNLOAD	0x01
+
+#define QCA_SYSCFG_UPDATED	0x40
+#define QCA_PATCH_UPDATED	0x80
+#define QCA_DFU_TIMEOUT		3000
+#define QCA_FLAG_MULTI_NVM      0x80
+#define QCA_BT_RESET_WAIT_MS    100
+
+#define WCN6855_2_0_RAM_VERSION_GF 0x400c1200
+#define WCN6855_2_1_RAM_VERSION_GF 0x400c1211
+
+struct qca_version {
+	__le32	rom_version;
+	__le32	patch_version;
+	__le32	ram_version;
+	__u8	chip_id;
+	__u8	platform_id;
+	__le16	flag;
+	__u8	reserved[4];
+} __packed;
+
+struct qca_rampatch_version {
+	__le16	rom_version_high;
+	__le16  rom_version_low;
+	__le16	patch_version;
+} __packed;
+
+struct qca_device_info {
+	u32	rom_version;
+	u8	rampatch_hdr;	/* length of header in rampatch */
+	u8	nvm_hdr;	/* length of header in NVM */
+	u8	ver_offset;	/* offset of version structure in rampatch */
+};
+
+struct qca_custom_firmware {
+	u32 rom_version;
+	u16 board_id;
+	const char *subdirectory;
+};
+
+static const struct qca_device_info qca_devices_table[] = {
+	{ 0x00000100, 20, 4,  8 }, /* Rome 1.0 */
+	{ 0x00000101, 20, 4,  8 }, /* Rome 1.1 */
+	{ 0x00000200, 28, 4, 16 }, /* Rome 2.0 */
+	{ 0x00000201, 28, 4, 16 }, /* Rome 2.1 */
+	{ 0x00000300, 28, 4, 16 }, /* Rome 3.0 */
+	{ 0x00000302, 28, 4, 16 }, /* Rome 3.2 */
+	{ 0x00130100, 40, 4, 16 }, /* WCN6855 1.0 */
+	{ 0x00130200, 40, 4, 16 }, /* WCN6855 2.0 */
+	{ 0x00130201, 40, 4, 16 }, /* WCN6855 2.1 */
+	{ 0x00190200, 40, 4, 16 }, /* WCN785x 2.0 */
+};
+
+static const struct qca_custom_firmware qca_custom_btfws[] = {
+	{ 0x00130201, 0x030A, "QCA2066" },
+	{ 0x00130201, 0x030B, "QCA2066" },
+	{ },
+};
+
+static u16 qca_extract_board_id(const struct qca_version *ver)
+{
+	u16 flag = le16_to_cpu(ver->flag);
+	u16 board_id = 0;
+
+	if (((flag >> 8) & 0xff) == QCA_FLAG_MULTI_NVM) {
+		/* The board_id should be split into two bytes
+		 * The 1st byte is chip ID, and the 2nd byte is platform ID
+		 * For example, board ID 0x010A, 0x01 is platform ID. 0x0A is chip ID
+		 * we have several platforms, and platform IDs are continuously added
+		 * Platform ID:
+		 * 0x00 is for Mobile
+		 * 0x01 is for X86
+		 * 0x02 is for Automotive
+		 * 0x03 is for Consumer electronic
+		 */
+		board_id = (ver->chip_id << 8) + ver->platform_id;
+	}
+
+	/* Take 0xffff as invalid board ID */
+	if (board_id == 0xffff)
+		board_id = 0;
+
+	return board_id;
+}
+
+static const char *qca_get_fw_subdirectory(const struct qca_version *ver)
+{
+	const struct qca_custom_firmware *ptr;
+	u32 rom_ver;
+	u16 board_id;
+
+	rom_ver = le32_to_cpu(ver->rom_version);
+	board_id = qca_extract_board_id(ver);
+	if (!board_id)
+		return NULL;
+
+	for (ptr = qca_custom_btfws; ptr->rom_version; ptr++) {
+		if (ptr->rom_version == rom_ver &&
+		    ptr->board_id == board_id)
+			return ptr->subdirectory;
+	}
+
+	return NULL;
+}
+
+static int btusb_qca_send_vendor_req(struct usb_device *udev, u8 request,
+				     void *data, u16 size)
+{
+	int err;
+
+	/* Found some of USB hosts have IOT issues with ours so that we should
+	 * not wait until HCI layer is ready.
+	 */
+	err = usb_control_msg_recv(udev, 0, request, USB_TYPE_VENDOR | USB_DIR_IN,
+				   0, 0, data, size, USB_CTRL_GET_TIMEOUT,
+				   GFP_KERNEL);
+	if (err)
+		dev_err(&udev->dev, "Failed to access otp area (%d)", err);
+
+	return err;
+}
+
+static int btusb_setup_qca_download_fw(struct hci_dev *hdev,
+				       const struct firmware *firmware,
+				       size_t hdr_size)
+{
+	struct btusb_data *btdata = hci_get_drvdata(hdev);
+	struct usb_device *udev = btdata->udev;
+	size_t count, size, sent = 0;
+	int pipe, len, err;
+	u8 *buf;
+
+	buf = kmalloc(QCA_DFU_PACKET_LEN, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	count = firmware->size;
+
+	size = min_t(size_t, count, hdr_size);
+	memcpy(buf, firmware->data, size);
+
+	/* USB patches should go down to controller through USB path
+	 * because binary format fits to go down through USB channel.
+	 * USB control path is for patching headers and USB bulk is for
+	 * patch body.
+	 */
+	pipe = usb_sndctrlpipe(udev, 0);
+	err = usb_control_msg(udev, pipe, QCA_DFU_DOWNLOAD, USB_TYPE_VENDOR,
+			      0, 0, buf, size, USB_CTRL_SET_TIMEOUT);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send headers (%d)", err);
+		goto done;
+	}
+
+	sent += size;
+	count -= size;
+
+	/* ep2 need time to switch from function acl to function dfu,
+	 * so we add 20ms delay here.
+	 */
+	msleep(20);
+
+	while (count) {
+		size = min_t(size_t, count, QCA_DFU_PACKET_LEN);
+
+		memcpy(buf, firmware->data + sent, size);
+
+		pipe = usb_sndbulkpipe(udev, 0x02);
+		err = usb_bulk_msg(udev, pipe, buf, size, &len,
+				   QCA_DFU_TIMEOUT);
+		if (err < 0) {
+			bt_dev_err(hdev, "Failed to send body at %zd of %zd (%d)",
+				   sent, firmware->size, err);
+			break;
+		}
+
+		if (size != len) {
+			bt_dev_err(hdev, "Failed to get bulk buffer");
+			err = -EILSEQ;
+			break;
+		}
+
+		sent  += size;
+		count -= size;
+	}
+
+done:
+	kfree(buf);
+	return err;
+}
+
+static int btusb_setup_qca_load_rampatch(struct hci_dev *hdev,
+					 struct qca_version *ver,
+					 const struct qca_device_info *info)
+{
+	struct qca_rampatch_version *rver;
+	const struct firmware *fw;
+	const char *fw_subdir;
+	u32 ver_rom, ver_patch, rver_rom;
+	u16 rver_rom_low, rver_rom_high, rver_patch;
+	char fwname[80];
+	int err;
+
+	ver_rom = le32_to_cpu(ver->rom_version);
+	ver_patch = le32_to_cpu(ver->patch_version);
+
+	fw_subdir = qca_get_fw_subdirectory(ver);
+	if (fw_subdir)
+		snprintf(fwname, sizeof(fwname), "qca/%s/rampatch_usb_%08x.bin",
+			 fw_subdir, ver_rom);
+	else
+		snprintf(fwname, sizeof(fwname), "qca/rampatch_usb_%08x.bin",
+			 ver_rom);
+
+	err = request_firmware(&fw, fwname, &hdev->dev);
+	if (err) {
+		bt_dev_err(hdev, "failed to request rampatch file: %s (%d)",
+			   fwname, err);
+		return err;
+	}
+
+	bt_dev_info(hdev, "using rampatch file: %s", fwname);
+
+	rver = (struct qca_rampatch_version *)(fw->data + info->ver_offset);
+	rver_rom_low = le16_to_cpu(rver->rom_version_low);
+	rver_patch = le16_to_cpu(rver->patch_version);
+
+	if (ver_rom & ~0xffffU) {
+		rver_rom_high = le16_to_cpu(rver->rom_version_high);
+		rver_rom = rver_rom_high << 16 | rver_rom_low;
+	} else {
+		rver_rom = rver_rom_low;
+	}
+
+	bt_dev_info(hdev, "QCA: patch rome 0x%x build 0x%x, "
+		    "firmware rome 0x%x build 0x%x",
+		    rver_rom, rver_patch, ver_rom, ver_patch);
+
+	/* Allow rampatch when the patch version equals the firmware version.
+	 * A firmware download may be aborted by a transient USB error (e.g.
+	 * disconnect) after the controller updates version info but before
+	 * completion.
+	 * Allowing equal versions enables re-flashing during recovery.
+	 */
+	if (rver_rom != ver_rom || rver_patch < ver_patch) {
+		bt_dev_err(hdev, "rampatch file version did not match with firmware");
+		err = -EINVAL;
+		goto done;
+	}
+
+	err = btusb_setup_qca_download_fw(hdev, fw, info->rampatch_hdr);
+
+done:
+	release_firmware(fw);
+
+	return err;
+}
+
+static void btusb_generate_qca_nvm_name(char *fwname, size_t max_size,
+					const struct qca_version *ver)
+{
+	u32 rom_version = le32_to_cpu(ver->rom_version);
+	const char *variant, *fw_subdir;
+	int len;
+	u16 board_id;
+
+	fw_subdir = qca_get_fw_subdirectory(ver);
+	board_id = qca_extract_board_id(ver);
+
+	switch (le32_to_cpu(ver->ram_version)) {
+	case WCN6855_2_0_RAM_VERSION_GF:
+	case WCN6855_2_1_RAM_VERSION_GF:
+		variant = "_gf";
+		break;
+	default:
+		variant = NULL;
+		break;
+	}
+
+	if (fw_subdir)
+		len = snprintf(fwname, max_size, "qca/%s/nvm_usb_%08x",
+			       fw_subdir, rom_version);
+	else
+		len = snprintf(fwname, max_size, "qca/nvm_usb_%08x",
+			       rom_version);
+	if (variant)
+		len += snprintf(fwname + len, max_size - len, "%s", variant);
+	if (board_id)
+		len += snprintf(fwname + len, max_size - len, "_%04x", board_id);
+	len += snprintf(fwname + len, max_size - len, ".bin");
+}
+
+static int btusb_setup_qca_load_nvm(struct hci_dev *hdev,
+				    struct qca_version *ver,
+				    const struct qca_device_info *info)
+{
+	const struct firmware *fw;
+	char fwname[80];
+	int err;
+
+	btusb_generate_qca_nvm_name(fwname, sizeof(fwname), ver);
+
+	err = request_firmware(&fw, fwname, &hdev->dev);
+	if (err) {
+		bt_dev_err(hdev, "failed to request NVM file: %s (%d)",
+			   fwname, err);
+		return err;
+	}
+
+	bt_dev_info(hdev, "using NVM file: %s", fwname);
+
+	err = btusb_setup_qca_download_fw(hdev, fw, info->nvm_hdr);
+
+	release_firmware(fw);
+
+	return err;
+}
+
+/* identify the ROM version and check whether patches are needed */
+static bool btusb_qca_need_patch(struct usb_device *udev)
+{
+	struct qca_version ver;
+
+	if (btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
+				      sizeof(ver)))
+		return false;
+	/* only low ROM versions need patches */
+	return !(le32_to_cpu(ver.rom_version) & ~0xffffU);
+}
+
+static int btusb_setup_qca(struct hci_dev *hdev)
+{
+	struct btusb_data *btdata = hci_get_drvdata(hdev);
+	struct usb_device *udev = btdata->udev;
+	const struct qca_device_info *info = NULL;
+	struct qca_version ver;
+	u32 ver_rom;
+	u8 status;
+	int i, err;
+
+	err = btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
+					sizeof(ver));
+	if (err)
+		return err;
+
+	ver_rom = le32_to_cpu(ver.rom_version);
+
+	for (i = 0; i < ARRAY_SIZE(qca_devices_table); i++) {
+		if (ver_rom == qca_devices_table[i].rom_version)
+			info = &qca_devices_table[i];
+	}
+	if (!info) {
+		/* If the rom_version is not matched in the qca_devices_table
+		 * and the high ROM version is not zero, we assume this chip no
+		 * need to load the rampatch and nvm.
+		 */
+		if (ver_rom & ~0xffffU)
+			return 0;
+
+		bt_dev_err(hdev, "don't support firmware rome 0x%x", ver_rom);
+		return -ENODEV;
+	}
+
+	err = btusb_qca_send_vendor_req(udev, QCA_CHECK_STATUS, &status,
+					sizeof(status));
+	if (err)
+		return err;
+
+	if (!(status & QCA_PATCH_UPDATED)) {
+		err = btusb_setup_qca_load_rampatch(hdev, &ver, info);
+		if (err < 0)
+			return err;
+	}
+
+	err = btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver,
+					sizeof(ver));
+	if (err)
+		return err;
+
+	if (btdata->match_id->driver_info & BTUSB_QCA_WCN6855) {
+		struct btqca_data *btqca_data = hci_get_priv(hdev);
+
+		btqca_data->qca_dump.fw_version = le32_to_cpu(ver.patch_version);
+		btqca_data->qca_dump.controller_id = le32_to_cpu(ver.rom_version);
+	}
+
+	if (!(status & QCA_SYSCFG_UPDATED)) {
+		err = btusb_setup_qca_load_nvm(hdev, &ver, info);
+		if (err < 0)
+			return err;
+
+		/* WCN6855 2.1 and later will reset to apply firmware downloaded here, so
+		 * wait ~100ms for reset Done then go ahead, otherwise, it maybe
+		 * cause potential enable failure.
+		 */
+		if (info->rom_version >= 0x00130201)
+			msleep(QCA_BT_RESET_WAIT_MS);
+	}
+
+	/* Mark HCI_OP_ENHANCED_SETUP_SYNC_CONN as broken as it doesn't seem to
+	 * work with the likes of HSP/HFP mSBC.
+	 */
+	hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN);
+
+	return 0;
+}
+
+static inline int __set_diag_interface(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct usb_interface *intf = data->diag;
+	int ret;
+
+	if (!data->diag)
+		return -ENODEV;
+
+	ret = usb_find_common_endpoints(intf->cur_altsetting, &data->diag_rx_ep,
+					&data->diag_tx_ep, NULL, NULL);
+	if (ret) {
+		bt_dev_err(hdev, "invalid diagnostic descriptors");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static struct urb *alloc_diag_urb(struct hci_dev *hdev, bool enable)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct sk_buff *skb;
+	struct urb *urb;
+	unsigned int pipe;
+
+	if (!data->diag_tx_ep)
+		return ERR_PTR(-ENODEV);
+
+	urb = usb_alloc_urb(0, GFP_KERNEL);
+	if (!urb)
+		return ERR_PTR(-ENOMEM);
+
+	skb = bt_skb_alloc(2, GFP_KERNEL);
+	if (!skb) {
+		usb_free_urb(urb);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	skb_put_u8(skb, 0xf0);
+	skb_put_u8(skb, enable);
+
+	pipe = usb_sndbulkpipe(data->udev, data->diag_tx_ep->bEndpointAddress);
+
+	usb_fill_bulk_urb(urb, data->udev, pipe,
+			  skb->data, skb->len, btusb_tx_complete, skb);
+
+	skb->dev = (void *)hdev;
+
+	return urb;
+}
+
+static int btusb_bcm_set_diag(struct hci_dev *hdev, bool enable)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct urb *urb;
+
+	if (!data->diag)
+		return -ENODEV;
+
+	if (!test_bit(HCI_RUNNING, &hdev->flags))
+		return -ENETDOWN;
+
+	urb = alloc_diag_urb(hdev, enable);
+	if (IS_ERR(urb))
+		return PTR_ERR(urb);
+
+	return submit_or_queue_tx_urb(hdev, urb);
+}
+
+#ifdef CONFIG_PM
+static irqreturn_t btusb_oob_wake_handler(int irq, void *priv)
+{
+	struct btusb_data *data = priv;
+
+	pm_wakeup_event(&data->udev->dev, 0);
+	pm_system_wakeup();
+
+	/* Disable only if not already disabled (keep it balanced) */
+	if (test_and_clear_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags)) {
+		disable_irq_nosync(irq);
+		disable_irq_wake(irq);
+	}
+	return IRQ_HANDLED;
+}
+
+static const struct of_device_id btusb_match_table[] = {
+	{ .compatible = "usb1286,204e" },
+	{ .compatible = "usbcf3,e300" }, /* QCA6174A */
+	{ .compatible = "usb4ca,301a" }, /* QCA6174A (Lite-On) */
+	{ }
+};
+MODULE_DEVICE_TABLE(of, btusb_match_table);
+
+/* Use an oob wakeup pin? */
+static int btusb_config_oob_wake(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+	struct device *dev = &data->udev->dev;
+	int irq, ret;
+
+	clear_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags);
+
+	if (!of_match_device(btusb_match_table, dev))
+		return 0;
+
+	/* Move on if no IRQ specified */
+	irq = of_irq_get_byname(dev->of_node, "wakeup");
+	if (irq <= 0) {
+		bt_dev_dbg(hdev, "%s: no OOB Wakeup IRQ in DT", __func__);
+		return 0;
+	}
+
+	irq_set_status_flags(irq, IRQ_NOAUTOEN);
+	ret = request_irq(irq, btusb_oob_wake_handler, 0, "OOB Wake-on-BT", data);
+	if (ret) {
+		bt_dev_err(hdev, "%s: IRQ request failed", __func__);
+		return ret;
+	}
+
+	ret = device_init_wakeup(dev, true);
+	if (ret) {
+		bt_dev_err(hdev, "%s: failed to init_wakeup", __func__);
+		goto err_free_irq;
+	}
+
+	data->oob_wake_irq = irq;
+	bt_dev_info(hdev, "OOB Wake-on-BT configured at IRQ %u", irq);
+
+	return 0;
+
+err_free_irq:
+	free_irq(irq, data);
+
+	return ret;
+}
+#else
+static inline int btusb_config_oob_wake(struct hci_dev *hdev)
+{
+	return 0;
+}
+#endif
+
+static void btusb_check_needs_reset_resume(struct usb_interface *intf)
+{
+	if (dmi_check_system(btusb_needs_reset_resume_table))
+		interface_to_usbdev(intf)->quirks |= USB_QUIRK_RESET_RESUME;
+}
+
+static bool btusb_wakeup(struct hci_dev *hdev)
+{
+	struct btusb_data *data = hci_get_drvdata(hdev);
+
+	return device_may_wakeup(&data->udev->dev);
+}
+
+static int btusb_shutdown_qca(struct hci_dev *hdev)
+{
+	int err;
+
+	err = __hci_reset_sync(hdev);
+	if (err)
+		bt_dev_err(hdev, "HCI reset during shutdown failed");
+
+	return err;
+}
+
+static ssize_t force_poll_sync_read(struct file *file, char __user *user_buf,
+				    size_t count, loff_t *ppos)
+{
+	struct btusb_data *data = file->private_data;
+	char buf[3];
+
+	buf[0] = data->poll_sync ? 'Y' : 'N';
+	buf[1] = '\n';
+	buf[2] = '\0';
+	return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
+}
+
+static ssize_t force_poll_sync_write(struct file *file,
+				     const char __user *user_buf,
+				     size_t count, loff_t *ppos)
+{
+	struct btusb_data *data = file->private_data;
+	bool enable;
+	int err;
+
+	err = kstrtobool_from_user(user_buf, count, &enable);
+	if (err)
+		return err;
+
+	/* Only allow changes while the adapter is down and it is using legacy
+	 * protocol.
+	 */
+	if (test_bit(HCI_UP, &data->hdev->flags) ||
+	    data->proto != BTUSB_PROTO_LEGACY)
+		return -EPERM;
+
+	if (data->poll_sync == enable)
+		return -EALREADY;
+
+	data->poll_sync = enable;
+
+	return count;
+}
+
+static const struct file_operations force_poll_sync_fops = {
+	.owner		= THIS_MODULE,
+	.open		= simple_open,
+	.read		= force_poll_sync_read,
+	.write		= force_poll_sync_write,
+	.llseek		= default_llseek,
+};
+
+#define BTUSB_HCI_DRV_OP_SUPPORTED_ALTSETTINGS \
+		hci_opcode_pack(HCI_DRV_OGF_DRIVER_SPECIFIC, 0x0000)
+#define BTUSB_HCI_DRV_SUPPORTED_ALTSETTINGS_SIZE	0
+struct btusb_hci_drv_rp_supported_altsettings {
+	__u8	num;
+	__u8	altsettings[];
+} __packed;
+
+#define BTUSB_HCI_DRV_OP_SWITCH_ALTSETTING \
+		hci_opcode_pack(HCI_DRV_OGF_DRIVER_SPECIFIC, 0x0001)
+#define BTUSB_HCI_DRV_SWITCH_ALTSETTING_SIZE		1
+struct btusb_hci_drv_cmd_switch_altsetting {
+	__u8	altsetting;
+} __packed;
+
+static const struct {
+	u16 opcode;
+	const char *desc;
+} btusb_hci_drv_supported_commands[] = {
+	/* Common commands */
+	{ HCI_DRV_OP_READ_INFO, "Read Info" },
+
+	/* Driver specific commands */
+	{ BTUSB_HCI_DRV_OP_SUPPORTED_ALTSETTINGS, "Supported Altsettings" },
+	{ BTUSB_HCI_DRV_OP_SWITCH_ALTSETTING,     "Switch Altsetting" },
+};
+static int btusb_hci_drv_read_info(struct hci_dev *hdev, void *data,
+				   u16 data_len)
+{
+	struct hci_drv_rp_read_info *rp;
+	size_t rp_size;
+	int err, i;
+	u16 opcode, num_supported_commands =
+		ARRAY_SIZE(btusb_hci_drv_supported_commands);
+
+	rp_size = sizeof(*rp) + num_supported_commands * 2;
+
+	rp = kmalloc(rp_size, GFP_KERNEL);
+	if (!rp)
+		return -ENOMEM;
+
+	strscpy_pad(rp->driver_name, btusb_driver.name);
+
+	rp->num_supported_commands = cpu_to_le16(num_supported_commands);
+	for (i = 0; i < num_supported_commands; i++) {
+		opcode = btusb_hci_drv_supported_commands[i].opcode;
+		bt_dev_info(hdev,
+			    "Supported HCI Drv command (0x%02x|0x%04x): %s",
+			    hci_opcode_ogf(opcode),
+			    hci_opcode_ocf(opcode),
+			    btusb_hci_drv_supported_commands[i].desc);
+		rp->supported_commands[i] = cpu_to_le16(opcode);
+	}
+
+	err = hci_drv_cmd_complete(hdev, HCI_DRV_OP_READ_INFO,
+				   HCI_DRV_STATUS_SUCCESS, rp, rp_size);
+
+	kfree(rp);
+	return err;
+}
+
+static int btusb_hci_drv_supported_altsettings(struct hci_dev *hdev, void *data,
+					       u16 data_len)
+{
+	struct btusb_data *drvdata = hci_get_drvdata(hdev);
+	struct btusb_hci_drv_rp_supported_altsettings *rp;
+	size_t rp_size;
+	int err;
+	u8 i;
+
+	/* There are at most 7 alt (0 - 6) */
+	rp = kmalloc(sizeof(*rp) + 7, GFP_KERNEL);
+	if (!rp)
+		return -ENOMEM;
+
+	rp->num = 0;
+	if (!drvdata->isoc)
+		goto done;
+
+	for (i = 0; i <= 6; i++) {
+		if (btusb_find_altsetting(drvdata->isoc, i))
+			rp->altsettings[rp->num++] = i;
+	}
+
+done:
+	rp_size = sizeof(*rp) + rp->num;
+
+	err = hci_drv_cmd_complete(hdev, BTUSB_HCI_DRV_OP_SUPPORTED_ALTSETTINGS,
+				   HCI_DRV_STATUS_SUCCESS, rp, rp_size);
+	kfree(rp);
+	return err;
+}
+
+static int btusb_hci_drv_switch_altsetting(struct hci_dev *hdev, void *data,
+					   u16 data_len)
+{
+	struct btusb_hci_drv_cmd_switch_altsetting *cmd = data;
+	u8 status;
+
+	if (cmd->altsetting > 6) {
+		status = HCI_DRV_STATUS_INVALID_PARAMETERS;
+	} else {
+		if (btusb_switch_alt_setting(hdev, cmd->altsetting))
+			status = HCI_DRV_STATUS_UNSPECIFIED_ERROR;
+		else
+			status = HCI_DRV_STATUS_SUCCESS;
+	}
+
+	return hci_drv_cmd_status(hdev, BTUSB_HCI_DRV_OP_SWITCH_ALTSETTING,
+				  status);
+}
+
+static const struct hci_drv_handler btusb_hci_drv_common_handlers[] = {
+	{ btusb_hci_drv_read_info,	HCI_DRV_READ_INFO_SIZE },
+};
+
+static const struct hci_drv_handler btusb_hci_drv_specific_handlers[] = {
+	{ btusb_hci_drv_supported_altsettings,
+				BTUSB_HCI_DRV_SUPPORTED_ALTSETTINGS_SIZE },
+	{ btusb_hci_drv_switch_altsetting,
+				BTUSB_HCI_DRV_SWITCH_ALTSETTING_SIZE },
+};
+
+static struct hci_drv btusb_hci_drv = {
+	.common_handler_count	= ARRAY_SIZE(btusb_hci_drv_common_handlers),
+	.common_handlers	= btusb_hci_drv_common_handlers,
+	.specific_handler_count	= ARRAY_SIZE(btusb_hci_drv_specific_handlers),
+	.specific_handlers	= btusb_hci_drv_specific_handlers,
+};
+
+/*
+ * ============================================================================
+ * Qualcomm support
+ * ============================================================================
+ */
+
+static int btusb_recv_intr_qcom(struct btusb_data *data, void *buffer, int count)
+{
+	unsigned long flags;
+	int err = 0;
+
+	spin_lock_irqsave(&data->rxlock, flags);
+	data->evt_skb = btusb_qcom_recv_intr(data->hdev, data->evt_skb, buffer, count,
+					     &err);
+	spin_unlock_irqrestore(&data->rxlock, flags);
+
+	return err;
+}
+
+static int btusb_recv_bulk_qcom(struct btusb_data *data, void *buffer, int count)
+{
+	unsigned long flags;
+	int err = 0;
+
+	spin_lock_irqsave(&data->rxlock, flags);
+	data->acl_skb = btusb_qcom_recv_bulk(data->hdev, data->acl_skb, buffer, count,
+					     &err);
+	spin_unlock_irqrestore(&data->rxlock, flags);
+
+	return err;
+}
+
+static int btusb_send_vendor_frame_qcom(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	u8 pkt_type = hci_skb_pkt_type(skb);
+	struct urb *urb;
+
+	switch (pkt_type) {
+	case QPERI_COMMAND_PKT:
+		urb = alloc_ctrl_urb(hdev, skb);
+		break;
+
+	case QPERI_ACLDATA_PKT:
+		urb = alloc_bulk_urb(hdev, skb);
+		break;
+
+	default:
+		return -EILSEQ;
+	}
+
+	if (IS_ERR(urb))
+		return PTR_ERR(urb);
+
+	return submit_or_queue_tx_urb(hdev, urb);
+}
+
+/*
+ * ============================================================================
+ * Qualcomm support end
+ * ============================================================================
+ */
+
+static int btusb_probe(struct usb_interface *intf,
+		       const struct usb_device_id *id)
+{
+	struct gpio_desc *reset_gpio;
+	struct usb_host_interface *alt;
+	struct usb_endpoint_descriptor *bulk_rx_ep, *bulk_tx_ep, *intr_ep;
+	struct btusb_data *data;
+	struct hci_dev *hdev;
+	unsigned ifnum_base;
+	int err, priv_size = 0;
+
+	BT_DBG("intf %p id %p", intf, id);
+
+	if ((id->driver_info & BTUSB_IFNUM_2) &&
+	    (intf->cur_altsetting->desc.bInterfaceNumber != 0) &&
+	    (intf->cur_altsetting->desc.bInterfaceNumber != 2))
+		return -ENODEV;
+
+	ifnum_base = intf->cur_altsetting->desc.bInterfaceNumber;
+
+	if (!id->driver_info) {
+		const struct usb_device_id *match;
+
+		match = usb_match_id(intf, quirks_table);
+		if (match)
+			id = match;
+	}
+
+	if (id->driver_info & BTUSB_IGNORE)
+		return -ENODEV;
+
+	if (id->driver_info & BTUSB_ATH3012) {
+		struct usb_device *udev = interface_to_usbdev(intf);
+
+		/* Old firmware would otherwise let ath3k driver load
+		 * patch and sysconfig files
+		 */
+		if (le16_to_cpu(udev->descriptor.bcdDevice) <= 0x0001 &&
+		    !btusb_qca_need_patch(udev))
+			return -ENODEV;
+	}
+
+	data = kzalloc_obj(*data);
+	if (!data)
+		return -ENOMEM;
+
+	data->match_id = id;
+
+	/* Alternate setting 1 with a single pair of bulk endpoints and no
+	 * interrupt endpoint means the controller supports Bulk Serialization
+	 * Mode, in which every packet is prefixed with an H:4 header and
+	 * carried over the bulk endpoints.
+	 */
+	alt = btusb_find_altsetting(intf, 1);
+	if (alt && usb_find_int_in_endpoint(alt, &intr_ep) &&
+	    !usb_find_common_endpoints(alt, &bulk_rx_ep, &bulk_tx_ep, NULL,
+				       NULL)) {
+		err = usb_set_interface(interface_to_usbdev(intf), ifnum_base, 1);
+		if (!err)
+			data->proto = BTUSB_PROTO_H4;
+		else
+			dev_warn(&intf->dev,
+				 "failed to select alt setting 1 (%d), using legacy mode",
+				 err);
+	}
+
+	/* Check if all endpoints could be enumerated, legacy mode requires
+	 * interrupt and bulk endpoints while H4 mode only requires bulk
+	 * endpoints.
+	 */
+	err = usb_find_common_endpoints(intf->cur_altsetting, &data->bulk_rx_ep,
+					&data->bulk_tx_ep,
+					data->proto == BTUSB_PROTO_LEGACY ?
+						&data->intr_ep : NULL,
+					NULL);
+	if (err) {
+		dev_err(&intf->dev, "failed to enumerate endpoints\n");
+		goto err_free_data;
+	}
+
+	if (id->driver_info & BTUSB_AMP) {
+		data->cmdreq_type = USB_TYPE_CLASS | 0x01;
+		data->cmdreq = 0x2b;
+	} else {
+		data->cmdreq_type = USB_TYPE_CLASS;
+		data->cmdreq = 0x00;
+	}
+
+	data->udev = interface_to_usbdev(intf);
+	data->intf = intf;
+
+	INIT_WORK(&data->work, btusb_work);
+	INIT_WORK(&data->waker, btusb_waker);
+	INIT_DELAYED_WORK(&data->rx_work, btusb_rx_work);
+
+	skb_queue_head_init(&data->acl_q);
+
+	init_usb_anchor(&data->deferred);
+	init_usb_anchor(&data->tx_anchor);
+	spin_lock_init(&data->txlock);
+
+	init_usb_anchor(&data->intr_anchor);
+	init_usb_anchor(&data->bulk_anchor);
+	init_usb_anchor(&data->isoc_anchor);
+	init_usb_anchor(&data->diag_anchor);
+	init_usb_anchor(&data->ctrl_anchor);
+	spin_lock_init(&data->rxlock);
+
+	data->recv_event = hci_recv_frame;
+	data->recv_intr = btusb_recv_intr;
+	data->recv_bulk = btusb_recv_bulk;
+
+	if (id->driver_info & BTUSB_INTEL_COMBINED) {
+		/* Allocate extra space for Intel device */
+		priv_size += sizeof(struct btintel_data);
+
+		/* Override the rx handlers */
+		data->recv_event = btintel_recv_event;
+		data->recv_bulk = btusb_recv_bulk_intel;
+	} else if (id->driver_info & BTUSB_REALTEK) {
+		/* Allocate extra space for Realtek device */
+		priv_size += sizeof(struct btrealtek_data);
+
+		data->recv_event = btusb_recv_event_realtek;
+	} else if (id->driver_info & BTUSB_MEDIATEK) {
+		/* Allocate extra space for Mediatek device */
+		priv_size += sizeof(struct btmtk_data);
+	} else if (id->driver_info & BTUSB_QCA_WCN6855) {
+		/* Allocate extra space for QCA WCN6855 device */
+		priv_size += sizeof(struct btqca_data);
+	} else if (id->driver_info & BTUSB_QUALCOMM) {
+		/* Allocate extra space for Qualcomm device */
+		priv_size += btusb_qcom_hdev_priv_size();
+	}
+
+	data->recv_acl = hci_recv_frame;
+
+	hdev = hci_alloc_dev_priv(priv_size);
+	if (!hdev) {
+		err = -ENOMEM;
+		goto err_free_data;
+	}
+
+	hdev->bus = HCI_USB;
+	hci_set_drvdata(hdev, data);
+
+	data->hdev = hdev;
+
+	SET_HCIDEV_DEV(hdev, &intf->dev);
+
+	reset_gpio = gpiod_get_optional(&data->udev->dev, "reset",
+					GPIOD_OUT_LOW);
+	if (IS_ERR(reset_gpio)) {
+		err = PTR_ERR(reset_gpio);
+		goto err_free_hdev;
+	} else if (reset_gpio) {
+		data->reset_gpio = reset_gpio;
+	}
+
+	hdev->open    = btusb_open;
+	hdev->close   = btusb_close;
+	hdev->flush   = btusb_flush;
+	hdev->send    = btusb_send_frame;
+	hdev->notify  = btusb_notify;
+	hdev->wakeup  = btusb_wakeup;
+	hdev->hci_drv = &btusb_hci_drv;
+
+	err = btusb_config_oob_wake(hdev);
+	if (err)
+		goto err_put_reset;
+
+	/* Marvell devices may need a specific chip configuration */
+	if (id->driver_info & BTUSB_MARVELL && data->oob_wake_irq) {
+		err = marvell_config_oob_wake(hdev);
+		if (err)
+			goto err_disable_wakeup;
+	}
+
+	if (id->driver_info & BTUSB_CW6622)
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_STORED_LINK_KEY);
+
+	if (id->driver_info & BTUSB_BCM2045)
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_STORED_LINK_KEY);
+
+	if (id->driver_info & BTUSB_BCM92035)
+		hdev->setup = btusb_setup_bcm92035;
+
+	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_BCM) &&
+	    (id->driver_info & BTUSB_BCM_PATCHRAM)) {
+		hdev->manufacturer = 15;
+		hdev->setup = btbcm_setup_patchram;
+		hdev->set_diag = btusb_bcm_set_diag;
+		hdev->set_bdaddr = btbcm_set_bdaddr;
+
+		/* Broadcom LM_DIAG Interface numbers are hardcoded */
+		data->diag = usb_ifnum_to_if(data->udev, ifnum_base + 2);
+	}
+
+	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_BCM) &&
+	    (id->driver_info & BTUSB_BCM_APPLE)) {
+		hdev->manufacturer = 15;
+		hdev->setup = btbcm_setup_apple;
+		hdev->set_diag = btusb_bcm_set_diag;
+
+		/* Broadcom LM_DIAG Interface numbers are hardcoded */
+		data->diag = usb_ifnum_to_if(data->udev, ifnum_base + 2);
+	}
+
+	/* Combined Intel Device setup to support multiple setup routine */
+	if (id->driver_info & BTUSB_INTEL_COMBINED) {
+		err = btintel_configure_setup(hdev, btusb_driver.name);
+		if (err)
+			goto err_kill_tx_urbs;
+
+		/* Transport specific configuration */
+		hdev->send = btusb_send_frame_intel;
+		hdev->reset = btusb_intel_reset;
+
+		if (id->driver_info & BTUSB_INTEL_NO_WBS_SUPPORT)
+			btintel_set_flag(hdev, INTEL_ROM_LEGACY_NO_WBS_SUPPORT);
+
+		if (id->driver_info & BTUSB_INTEL_BROKEN_INITIAL_NCMD)
+			btintel_set_flag(hdev, INTEL_BROKEN_INITIAL_NCMD);
+
+		if (id->driver_info & BTUSB_INTEL_BROKEN_SHUTDOWN_LED)
+			btintel_set_flag(hdev, INTEL_BROKEN_SHUTDOWN_LED);
+	}
+
+	if (id->driver_info & BTUSB_MARVELL)
+		hdev->set_bdaddr = btusb_set_bdaddr_marvell;
+
+	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_MTK) &&
+	    (id->driver_info & BTUSB_MEDIATEK)) {
+		hdev->setup = btusb_mtk_setup;
+		hdev->shutdown = btusb_mtk_shutdown;
+		hdev->manufacturer = 70;
+		hdev->reset = btmtk_reset_sync;
+		hdev->set_bdaddr = btmtk_set_bdaddr;
+		hdev->send = btusb_send_frame_mtk;
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN);
+		hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_SETUP);
+		data->recv_acl = btmtk_usb_recv_acl;
+		data->recv_event = btmtk_recv_event;
+		data->suspend = btmtk_usb_suspend;
+		data->resume = btmtk_usb_resume;
+		data->disconnect = btusb_mtk_disconnect;
+	}
+
+	if (id->driver_info & BTUSB_SWAVE) {
+		hci_set_quirk(hdev, HCI_QUIRK_FIXUP_INQUIRY_MODE);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_LOCAL_COMMANDS);
+	}
+
+	if (id->driver_info & BTUSB_INTEL_BOOT) {
+		hdev->manufacturer = 2;
+		hci_set_quirk(hdev, HCI_QUIRK_RAW_DEVICE);
+	}
+
+	if (id->driver_info & BTUSB_ATH3012) {
+		data->setup_on_usb = btusb_setup_qca;
+		hdev->set_bdaddr = btusb_set_bdaddr_ath3012;
+		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
+		hci_set_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER);
+	}
+
+	if (id->driver_info & BTUSB_QCA_ROME) {
+		data->setup_on_usb = btusb_setup_qca;
+		hdev->shutdown = btusb_shutdown_qca;
+		hdev->set_bdaddr = btusb_set_bdaddr_ath3012;
+		hdev->reset = btusb_qca_reset;
+		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
+		btusb_check_needs_reset_resume(intf);
+	}
+
+	if (id->driver_info & BTUSB_QCA_WCN6855) {
+		struct btqca_data *btqca_data = hci_get_priv(hdev);
+
+		btqca_data->qca_dump.id_vendor = id->idVendor;
+		btqca_data->qca_dump.id_product = id->idProduct;
+		data->recv_event = btusb_recv_evt_qca;
+		data->recv_acl = btusb_recv_acl_qca;
+		hci_devcd_register(hdev, btusb_coredump_qca, btusb_dump_hdr_qca, NULL);
+		data->setup_on_usb = btusb_setup_qca;
+		hdev->classify_pkt_type = btusb_classify_qca_pkt_type;
+		hdev->shutdown = btusb_shutdown_qca;
+		hdev->set_bdaddr = btusb_set_bdaddr_wcn6855;
+		hdev->reset = btusb_qca_reset;
+		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
+		hci_set_msft_opcode(hdev, 0xFD70);
+	}
+
+	if (id->driver_info & BTUSB_QUALCOMM) {
+		struct btusb_qcom *xport_data;
+
+		if (!IS_ENABLED(CONFIG_BT_HCIBTUSB_QCOM)) {
+			err = -ENODEV;
+			bt_dev_err(hdev, "CONFIG_BT_HCIBTUSB_QCOM not enabled");
+			goto err_kill_tx_urbs;
+		}
+
+		xport_data = btusb_qcom_xport_data(hdev);
+		xport_data->reset_gpio = data->reset_gpio;
+		xport_data->prepare_reset = btusb_prepare_reset;
+		xport_data->recv_bt_frame = btusb_recv_frame;
+		xport_data->send_bt_frame = btusb_send_frame;
+		xport_data->send_vendor_frame = btusb_send_vendor_frame_qcom;
+
+		data->recv_intr = btusb_recv_intr_qcom;
+		data->recv_bulk = btusb_recv_bulk_qcom;
+		data->disconnect = btusb_qcom_disconnect;
+
+		hdev->manufacturer = 0x1d;
+		hdev->send  = btusb_qcom_send_frame;
+		hdev->setup = btusb_qcom_setup;
+	}
+
+	if (id->driver_info & BTUSB_AMP) {
+		/* AMP controllers do not support SCO packets */
+		data->isoc = NULL;
+	} else if (data->proto == BTUSB_PROTO_H4) {
+		/* In H:4 mode every packet, including SCO/ISO, is carried over
+		 * the bulk endpoints, so the isochronous interface must not be
+		 * claimed nor have its alternate settings switched.
+		 */
+		data->isoc = NULL;
+	} else {
+		/* Interface orders are hardcoded in the specification */
+		data->isoc = usb_ifnum_to_if(data->udev, ifnum_base + 1);
+		data->isoc_ifnum = ifnum_base + 1;
+	}
+
+	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_RTL) &&
+	    (id->driver_info & BTUSB_REALTEK)) {
+		btrtl_set_driver_name(hdev, btusb_driver.name);
+		hdev->setup = btusb_setup_realtek;
+		hdev->shutdown = btrtl_shutdown_realtek;
+		hdev->reset = btusb_rtl_reset;
+		hdev->hw_error = btusb_rtl_hw_error;
+
+		/* Realtek devices need to set remote wakeup on auto-suspend */
+		set_bit(BTUSB_WAKEUP_AUTOSUSPEND, &data->flags);
+		set_bit(BTUSB_USE_ALT3_FOR_WBS, &data->flags);
+	}
+
+	if (id->driver_info & BTUSB_ACTIONS_SEMI) {
+		/* Support is advertised, but not implemented */
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ERR_DATA_REPORTING);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_TRANSMIT_POWER);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_SET_RPA_TIMEOUT);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_SCAN);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_READ_ENC_KEY_SIZE);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_CREATE_CONN);
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_WRITE_AUTH_PAYLOAD_TIMEOUT);
+	}
+
+	if (!reset)
+		hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
+
+	if (force_scofix || id->driver_info & BTUSB_WRONG_SCO_MTU) {
+		if (!disable_scofix)
+			hci_set_quirk(hdev, HCI_QUIRK_FIXUP_BUFFER_SIZE);
+	}
+
+	if (id->driver_info & BTUSB_BROKEN_ISOC)
+		data->isoc = NULL;
+
+	if (id->driver_info & BTUSB_WIDEBAND_SPEECH)
+		hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED);
+
+	if (id->driver_info & BTUSB_INVALID_LE_STATES)
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_LE_STATES);
+
+	if (id->driver_info & BTUSB_DIGIANSWER) {
+		data->cmdreq_type = USB_TYPE_VENDOR;
+		hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
+	}
+
+	if (id->driver_info & BTUSB_CSR) {
+		struct usb_device *udev = data->udev;
+		u16 bcdDevice = le16_to_cpu(udev->descriptor.bcdDevice);
+
+		/* Old firmware would otherwise execute USB reset */
+		if (bcdDevice < 0x117)
+			hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE);
+
+		/* This must be set first in case we disable it for fakes */
+		hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
+
+		/* Fake CSR devices with broken commands */
+		if (le16_to_cpu(udev->descriptor.idVendor)  == 0x0a12 &&
+		    le16_to_cpu(udev->descriptor.idProduct) == 0x0001)
+			hdev->setup = btusb_setup_csr;
+	}
+
+	if (id->driver_info & BTUSB_SNIFFER) {
+		struct usb_device *udev = data->udev;
+
+		/* New sniffer firmware has crippled HCI interface */
+		if (le16_to_cpu(udev->descriptor.bcdDevice) > 0x997)
+			hci_set_quirk(hdev, HCI_QUIRK_RAW_DEVICE);
+	}
+
+	if (id->driver_info & BTUSB_INTEL_BOOT) {
+		/* A bug in the bootloader causes that interrupt interface is
+		 * only enabled after receiving SetInterface(0, AltSetting=0).
+		 */
+		err = usb_set_interface(data->udev, 0, 0);
+		if (err < 0) {
+			BT_ERR("failed to set interface 0, alt 0 %d", err);
+			goto err_kill_tx_urbs;
+		}
+	}
+
+	if (data->isoc) {
+		err = usb_driver_claim_interface(&btusb_driver,
+						 data->isoc, data);
+		if (err < 0)
+			goto err_kill_tx_urbs;
+	}
+
+	if (IS_ENABLED(CONFIG_BT_HCIBTUSB_BCM) && data->diag) {
+		if (!usb_driver_claim_interface(&btusb_driver,
+						data->diag, data))
+			__set_diag_interface(hdev);
+		else
+			data->diag = NULL;
+	}
+
+	if (enable_autosuspend)
+		usb_enable_autosuspend(data->udev);
+
+	if (data->proto == BTUSB_PROTO_LEGACY)
+		data->poll_sync = enable_poll_sync;
+
+	err = hci_register_dev(hdev);
+	if (err < 0)
+		goto err_release_siblings;
+
+	usb_set_intfdata(intf, data);
+
+	debugfs_create_file("force_poll_sync", 0644, hdev->debugfs, data,
+			    &force_poll_sync_fops);
+
+	return 0;
+
+err_release_siblings:
+	if (data->diag) {
+		usb_set_intfdata(data->diag, NULL);
+		usb_driver_release_interface(&btusb_driver, data->diag);
+	}
+	if (data->isoc) {
+		usb_set_intfdata(data->isoc, NULL);
+		usb_driver_release_interface(&btusb_driver, data->isoc);
+	}
+err_kill_tx_urbs:
+	usb_kill_anchored_urbs(&data->tx_anchor);
+err_disable_wakeup:
+	if (data->oob_wake_irq) {
+		device_init_wakeup(&data->udev->dev, false);
+		free_irq(data->oob_wake_irq, data);
+	}
+err_put_reset:
+	if (data->reset_gpio)
+		gpiod_put(data->reset_gpio);
+err_free_hdev:
+	hci_free_dev(hdev);
+err_free_data:
+	kfree(data);
+
+	return err;
+}
+
+static void btusb_disconnect(struct usb_interface *intf)
+{
+	struct btusb_data *data = usb_get_intfdata(intf);
+	struct hci_dev *hdev;
+
+	BT_DBG("intf %p", intf);
+
+	if (!data)
+		return;
+
+	hdev = data->hdev;
+	usb_set_intfdata(data->intf, NULL);
+
+	if (data->isoc)
+		usb_set_intfdata(data->isoc, NULL);
+
+	if (data->diag)
+		usb_set_intfdata(data->diag, NULL);
+
+	if (data->disconnect)
+		data->disconnect(hdev);
+
+	hci_unregister_dev(hdev);
+
+	if (data->oob_wake_irq) {
+		device_init_wakeup(&data->udev->dev, false);
+		free_irq(data->oob_wake_irq, data);
+	}
+
+	if (data->reset_gpio)
+		gpiod_put(data->reset_gpio);
+
+	if (test_and_clear_bit(BTUSB_RESET, &data->flags))
+		usb_autopm_put_interface_no_suspend(data->intf);
+
+	if (intf == data->intf) {
+		if (data->isoc)
+			usb_driver_release_interface(&btusb_driver, data->isoc);
+		if (data->diag)
+			usb_driver_release_interface(&btusb_driver, data->diag);
+	} else if (intf == data->isoc) {
+		if (data->diag)
+			usb_driver_release_interface(&btusb_driver, data->diag);
+		usb_driver_release_interface(&btusb_driver, data->intf);
+	} else if (intf == data->diag) {
+		if (data->isoc)
+			usb_driver_release_interface(&btusb_driver, data->isoc);
+		usb_driver_release_interface(&btusb_driver, data->intf);
+	}
+
+	hci_free_dev(hdev);
+	kfree(data);
+}
+
+static int btusb_suspend(struct usb_interface *intf, pm_message_t message)
+{
+	struct btusb_data *data = usb_get_intfdata(intf);
+
+	BT_DBG("intf %p", intf);
+
+	/*
+	 * It is reported that remote wakeup events could sometimes cause some
+	 * adapters completely unresponsive. Resetting the xHCI root hub doesn't
+	 * help at all, and recovering from such a state needs a power cycle.
+	 * Since disabling remote wakeup simply causes the USB core to gate
+	 * runtime autosuspend as well due to needs_remote_wakeup == 1, let's do
+	 * this ourselves to make our life easier. The interface can be safely
+	 * autosuspended as long as remote wakeup is disabled, i.e., after
+	 * closing the HCI device.
+	 *
+	 * Don't auto-suspend if there are connections or discovery in progress.
+	 *
+	 * External suspend calls shall never fail. Specifically, a device with
+	 * broken remote wakeup may still take the advantage of remote wakeup in
+	 * order to wake up the system from sleep if userspace has enabled it as
+	 * a wakeup source.
+	 */
+	if (PMSG_IS_AUTO(message) &&
+	    ((test_bit(BTUSB_WAKEUP_BROKEN, &data->flags) && data->intf->needs_remote_wakeup) ||
+	     hci_conn_count(data->hdev) || hci_discovery_active(data->hdev)))
+		return -EBUSY;
+
+	if (data->suspend_count++)
+		return 0;
+
+	spin_lock_irq(&data->txlock);
+	if (!(PMSG_IS_AUTO(message) && data->tx_in_flight)) {
+		set_bit(BTUSB_SUSPENDING, &data->flags);
+		spin_unlock_irq(&data->txlock);
+	} else {
+		spin_unlock_irq(&data->txlock);
+		data->suspend_count--;
+		return -EBUSY;
+	}
+
+	cancel_work_sync(&data->work);
+
+	if (data->suspend)
+		data->suspend(data->hdev);
+
+	btusb_stop_traffic(data);
+	usb_kill_anchored_urbs(&data->tx_anchor);
+
+	if (data->oob_wake_irq && device_may_wakeup(&data->udev->dev)) {
+		set_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags);
+		enable_irq_wake(data->oob_wake_irq);
+		enable_irq(data->oob_wake_irq);
+	}
+
+	/* For global suspend, Realtek devices lose the loaded fw
+	 * in them. But for autosuspend, firmware should remain.
+	 * Actually, it depends on whether the usb host sends
+	 * set feature (enable wakeup) or not.
+	 */
+	if (test_bit(BTUSB_WAKEUP_AUTOSUSPEND, &data->flags)) {
+		if (PMSG_IS_AUTO(message) &&
+		    device_can_wakeup(&data->udev->dev))
+			data->udev->do_remote_wakeup = 1;
+		else if (!PMSG_IS_AUTO(message) &&
+			 !device_may_wakeup(&data->udev->dev)) {
+			data->udev->do_remote_wakeup = 0;
+			data->udev->reset_resume = 1;
+		}
+	}
+
+	return 0;
+}
+
+static void play_deferred(struct btusb_data *data)
+{
+	struct urb *urb;
+	int err;
+
+	while ((urb = usb_get_from_anchor(&data->deferred))) {
+		usb_anchor_urb(urb, &data->tx_anchor);
+
+		err = usb_submit_urb(urb, GFP_ATOMIC);
+		if (err < 0) {
+			if (err != -EPERM && err != -ENODEV)
+				BT_ERR("%s urb %p submission failed (%d)",
+				       data->hdev->name, urb, -err);
+			kfree(urb->setup_packet);
+			usb_unanchor_urb(urb);
+			usb_free_urb(urb);
+			break;
+		}
+
+		data->tx_in_flight++;
+		usb_free_urb(urb);
+	}
+
+	/* Cleanup the rest deferred urbs. */
+	while ((urb = usb_get_from_anchor(&data->deferred))) {
+		kfree(urb->setup_packet);
+		usb_free_urb(urb);
+	}
+}
+
+static int btusb_resume(struct usb_interface *intf)
+{
+	struct btusb_data *data = usb_get_intfdata(intf);
+	struct hci_dev *hdev = data->hdev;
+	int err = 0;
+
+	BT_DBG("intf %p", intf);
+
+	if (--data->suspend_count)
+		return 0;
+
+	/* Disable only if not already disabled (keep it balanced) */
+	if (test_and_clear_bit(BTUSB_OOB_WAKE_ENABLED, &data->flags)) {
+		disable_irq(data->oob_wake_irq);
+		disable_irq_wake(data->oob_wake_irq);
+	}
+
+	if (!test_bit(HCI_RUNNING, &hdev->flags))
+		goto done;
+
+	if (test_bit(BTUSB_INTR_RUNNING, &data->flags)) {
+		err = btusb_submit_intr_urb(hdev, GFP_NOIO);
+		if (err < 0) {
+			clear_bit(BTUSB_INTR_RUNNING, &data->flags);
+			goto failed;
+		}
+	}
+
+	if (test_bit(BTUSB_BULK_RUNNING, &data->flags)) {
+		err = btusb_submit_bulk_urb(hdev, GFP_NOIO);
+		if (err < 0) {
+			clear_bit(BTUSB_BULK_RUNNING, &data->flags);
+			goto failed;
+		}
+
+		btusb_submit_bulk_urb(hdev, GFP_NOIO);
+	}
+
+	if (test_bit(BTUSB_ISOC_RUNNING, &data->flags)) {
+		if (btusb_submit_isoc_urb(hdev, GFP_NOIO) < 0)
+			clear_bit(BTUSB_ISOC_RUNNING, &data->flags);
+		else
+			btusb_submit_isoc_urb(hdev, GFP_NOIO);
+	}
+
+	if (data->resume)
+		data->resume(hdev);
+
+	spin_lock_irq(&data->txlock);
+	play_deferred(data);
+	clear_bit(BTUSB_SUSPENDING, &data->flags);
+	spin_unlock_irq(&data->txlock);
+	schedule_work(&data->work);
+
+	return 0;
+
+failed:
+	usb_scuttle_anchored_urbs(&data->deferred);
+done:
+	spin_lock_irq(&data->txlock);
+	clear_bit(BTUSB_SUSPENDING, &data->flags);
+	spin_unlock_irq(&data->txlock);
+
+	return err;
+}
+
+#ifdef CONFIG_DEV_COREDUMP
+static void btusb_coredump(struct device *dev)
+{
+	struct btusb_data *data = dev_get_drvdata(dev);
+	struct hci_dev *hdev = data->hdev;
+
+	if (hdev->dump.coredump)
+		hdev->dump.coredump(hdev);
+}
+#endif
+
+static struct usb_driver btusb_driver = {
+	.name		= "btusb",
+	.probe		= btusb_probe,
+	.disconnect	= btusb_disconnect,
+	.suspend	= pm_ptr(btusb_suspend),
+	.resume		= pm_ptr(btusb_resume),
+	.id_table	= btusb_table,
+	.supports_autosuspend = 1,
+	.disable_hub_initiated_lpm = 1,
+
+#ifdef CONFIG_DEV_COREDUMP
+	.driver = {
+		.coredump = btusb_coredump,
+	},
+#endif
+};
+
+module_usb_driver(btusb_driver);
+
+module_param(disable_scofix, bool, 0644);
+MODULE_PARM_DESC(disable_scofix, "Disable fixup of wrong SCO buffer size");
+
+module_param(force_scofix, bool, 0644);
+MODULE_PARM_DESC(force_scofix, "Force fixup of wrong SCO buffers size");
+
+module_param(enable_autosuspend, bool, 0644);
+MODULE_PARM_DESC(enable_autosuspend, "Enable USB autosuspend by default");
+
+module_param(reset, bool, 0644);
+MODULE_PARM_DESC(reset, "Send HCI reset command on initialization");
+
+MODULE_AUTHOR("Marcel Holtmann <marcel@holtmann.org>");
+MODULE_DESCRIPTION("Generic Bluetooth USB driver ver " VERSION);
+MODULE_VERSION(VERSION);
+MODULE_LICENSE("GPL");
diff --git a/drivers/bluetooth/btusb_qcom.c b/drivers/bluetooth/btusb_qcom.c
new file mode 100644
index 00000000000000..cc4cfcde2afa04
--- /dev/null
+++ b/drivers/bluetooth/btusb_qcom.c
@@ -0,0 +1,4557 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Qualcomm Bluetooth USB transport-specific support
+ *
+ * Abbreviations:
+ *   BTC   - BT controller
+ *   PERI  - Peripheral subsystem of a multi-subsys BTC
+ *   TME-L - Trust Management Engine Lite subsystem of a multi-subsys BTC
+ *   DFU   - Device Firmware Update
+ *   EDL   - Embedded Downloader
+ *   TLV   - Type-Length-Value, a firmware file format
+ *   VSC   - Vendor-Specific Command
+ *   VSE   - Vendor-Specific Event
+ *   CCE   - Command Complete Event
+ *   CSE   - Command Status Event
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/args.h>
+#include <linux/bitfield.h>
+#include <linux/firmware.h>
+#include <linux/gpio/consumer.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/sizes.h>
+#include <linux/unaligned.h>
+#include <linux/usb.h>
+
+#include <net/bluetooth/bluetooth.h>
+#include <net/bluetooth/coredump.h>
+#include <net/bluetooth/hci_core.h>
+
+#include "btusb_qcom.h"
+
+/*
+ * ============================================================================
+ * Supported BTCs and their configuration
+ * ============================================================================
+ */
+
+/*
+ * enum qbtc_category - driver-perspective BTC category
+ * @QBTC_CAT_LEGACY: legacy
+ * @QBTC_CAT_UNIFIED: unified HCI, VSC gets CCE as response
+ * @QBTC_CAT_MSUBSYS: multi-subsys, always has PERI
+ * @QBTC_CAT_MAX: number of categories
+ */
+enum qbtc_category {
+	QBTC_CAT_LEGACY,
+	QBTC_CAT_UNIFIED,
+	QBTC_CAT_MSUBSYS,
+	QBTC_CAT_MAX,
+};
+
+static const char *qbtc_category_name(int category)
+{
+	const char *category_name = "unknown";
+
+	switch (category) {
+	case QBTC_CAT_LEGACY:
+		category_name = "legacy";
+		break;
+	case QBTC_CAT_UNIFIED:
+		category_name = "unified";
+		break;
+	case QBTC_CAT_MSUBSYS:
+		category_name = "multi-subsys";
+		break;
+	default:
+		break;
+	}
+
+	return category_name;
+}
+
+/* flags for BTC info and default configuration */
+
+/* BTC has PERI / TME-L subsystem */
+#define QBT_FLAG_HAS_PERI		BIT(0)
+#define QBT_FLAG_HAS_TMEL		BIT(1)
+/* BTC supports AOSP / MSFT vendor extension */
+#define QBT_FLAG_AOSP_EXT		BIT(4)
+#define QBT_FLAG_MSFT_EXT		BIT(5)
+/* BTC supports software reset */
+#define QBT_FLAG_SW_RESET		BIT(6)
+/* BTC supports memdump */
+#define QBT_FLAG_MEMDUMP		BIT(7)
+/* take foundry as a factor to select NVM */
+#define QBT_FLAG_FOUNDRY_NVM		BIT(8)
+/* select NVM based on board ID to download */
+#define QBT_FLAG_BID_NVM		BIT(9)
+/* fall back to the default NVM if no board-ID-specific NVM exists */
+#define QBT_FLAG_NVM_FALLBACK		BIT(10)
+/* reset PERI HCI by QHCI instead of QDFU */
+#define QBT_FLAG_RESET_PERI_HCI		BIT(11)
+/* trigger memdump on command timeout */
+#define QBT_FLAG_CMD_TIMEOUT_MEMDUMP	BIT(12)
+/* reserve bits [31:24] for board-level flags */
+#define QBT_FLAG_BTC_CFG_MASK		GENMASK(23, 0)
+
+/* rare board ID to custom firmware directroy map */
+struct qbtc_bid_fwdir {
+	u16 board_id;
+	const char *fw_dir;
+};
+
+/*
+ * struct qbtc_info - BTC information and default configuration
+ * @category: driver-perspective BTC category (legacy/unified/multi-subsys)
+ * @flags: QBT_FLAG_* — BTC attributes and default configuration
+ * @fw_dir: firmware folder, "qca" if NULL
+ * @custom_fw_table: {}-terminated array or NULL
+ */
+struct qbtc_info {
+	enum qbtc_category category;
+	unsigned long flags;
+	const char *fw_dir;
+	const struct qbtc_bid_fwdir *custom_fw_table;
+};
+
+/*
+ * struct qbtc_id - BTC ID entry in qbtc_id_table[] below
+ * @rom_version: ID to match against rom_version read from BTC
+ * @name: human-readable BTC name
+ * @btc_info: BTC information and default configuration, != NULL
+ */
+struct qbtc_id {
+	u32		rom_version;
+	const char	*name;
+	const struct qbtc_info *btc_info;
+};
+
+/* multi-subsys BTC here */
+#define QBTC_INFO_FLAGS_MSUBSYS	(QBT_FLAG_HAS_PERI | QBT_FLAG_AOSP_EXT | \
+				 QBT_FLAG_MSFT_EXT | QBT_FLAG_SW_RESET | \
+				 QBT_FLAG_MEMDUMP | QBT_FLAG_BID_NVM | \
+				 QBT_FLAG_NVM_FALLBACK | QBT_FLAG_CMD_TIMEOUT_MEMDUMP)
+
+static const struct qbtc_info qbtc_msubsys_qcc2072 = {
+	.category = QBTC_CAT_MSUBSYS,
+	.flags    = QBTC_INFO_FLAGS_MSUBSYS,
+	.fw_dir   = "qca/QCC2072",
+};
+
+static const struct qbtc_id qbtc_id_table[] = {
+	{ 0x00220100, "QCC2072 1.x", &qbtc_msubsys_qcc2072 },
+	{ }
+};
+
+/*
+ * ============================================================================
+ * Qualcomm DFU for rampatch/NVM download
+ * ============================================================================
+ */
+
+/* QDFU USB vendor command codes (bRequest) */
+#define QDFU_BT_CMD_VSC_REQ_DOWNLOAD_LOCAL	0x01
+#define QDFU_BT_CMD_CHECK_TARGET_STATE	0x05
+#define QDFU_BT_CMD_GET_TARGET_VERSION	0x09
+#define QDFU_BT_CMD_VSC_REQ_DOWNLOAD_REMOTE	0x10
+#define QDFU_BT_CMD_RESET_PERI_HCI			0x11
+#define QDFU_BT_CMD_ACTIVATE_REMOTE_BTSS	0x12
+#define QDFU_BT_CMD_BT_ENABLE_RESET		0x13
+
+/* QDFU target status bits (u8 bitmask from QDFU_BT_CMD_CHECK_TARGET_STATE) */
+#define QDFU_BT_STATE_LOADING_LOCAL		BIT(3)
+#define QDFU_BT_STATE_PATCHED_REMOTE	BIT(4)
+#define QDFU_BT_STATE_NVMED_REMOTE		BIT(5)
+#define QDFU_BT_STATE_NVMED_LOCAL		BIT(6)
+#define QDFU_BT_STATE_PATCHED_LOCAL		BIT(7)
+
+/* QDFU_BT_CMD_GET_TARGET_VERSION response */
+struct qdfu_bt_version {
+	__le32	rom_version;
+	__le32	patch_version;
+	__le32	soc_ver;
+	__be16	board_id;
+	__le16	flag;
+	u8	reserved[4];
+} __packed;
+
+struct qdfu_bt_id {
+	u32 rom_version;
+	u32 patch_version;
+	u32 soc_id;
+	u16 board_id;
+};
+
+static inline int qdfu_recv_vendor_req(struct hci_dev *hdev, u8 request,
+				       u16 value, void *buf, u16 size)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+
+	return usb_control_msg_recv(xport_data->udev, 0, request,
+				    USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT,
+				    value, 0, buf, size,
+				    USB_CTRL_GET_TIMEOUT, GFP_KERNEL);
+}
+
+static inline int qdfu_send_vendor_req(struct hci_dev *hdev, u8 request,
+				       u16 value, const void *buf, u16 size)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+
+	return usb_control_msg_send(xport_data->udev, 0, request,
+				    USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT,
+				    value, 0, buf, size,
+				    USB_CTRL_SET_TIMEOUT, GFP_KERNEL);
+}
+
+static int qdfu_vsc_req_download(struct hci_dev *hdev, u8 request,
+				 const void *buf, u16 size)
+{
+	int ret;
+
+	if (request != QDFU_BT_CMD_VSC_REQ_DOWNLOAD_LOCAL &&
+	    request != QDFU_BT_CMD_VSC_REQ_DOWNLOAD_REMOTE) {
+		bt_dev_err(hdev,
+			   "QDFU download invalid request code 0x%02x", request);
+		return -EINVAL;
+	}
+
+	ret = qdfu_send_vendor_req(hdev, request, 0, buf, size);
+	if (ret)
+		bt_dev_err(hdev,
+			   "QDFU download request 0x%02x failed: %pe",
+			   request, ERR_PTR(ret));
+
+	return ret;
+}
+
+static int qdfu_get_target_state(struct hci_dev *hdev, u8 *state_ptr)
+{
+	u8 state;
+	int ret;
+
+	ret = qdfu_recv_vendor_req(hdev, QDFU_BT_CMD_CHECK_TARGET_STATE, 0,
+				   &state, sizeof(state));
+	if (!ret)
+		*state_ptr = state;
+	else
+		bt_dev_err(hdev,
+			   "QDFU get target state failed: %pe",
+			   ERR_PTR(ret));
+
+	return ret;
+}
+
+static int qdfu_get_target_version(struct hci_dev *hdev, struct qdfu_bt_id *id_info)
+{
+	struct qdfu_bt_version dfu_ver;
+	u16 board_id = 0;
+	int ret;
+
+	ret = qdfu_recv_vendor_req(hdev, QDFU_BT_CMD_GET_TARGET_VERSION, 0,
+				   &dfu_ver, sizeof(dfu_ver));
+	if (ret) {
+		bt_dev_err(hdev,
+			   "QDFU get target version failed: %pe",
+			   ERR_PTR(ret));
+		return ret;
+	}
+
+	id_info->rom_version = le32_to_cpu(dfu_ver.rom_version);
+	id_info->patch_version = le32_to_cpu(dfu_ver.patch_version);
+	id_info->soc_id = le32_to_cpu(dfu_ver.soc_ver);
+
+	if ((le16_to_cpu(dfu_ver.flag) >> 8) == 0x80)
+		board_id = be16_to_cpu(dfu_ver.board_id);
+
+	/* Take 0xffff as invalid board ID */
+	if (board_id == 0xffff)
+		board_id = 0;
+
+	id_info->board_id = board_id;
+
+	return 0;
+}
+
+static int qdfu_activate_remote_btss(struct hci_dev *hdev, bool on,
+				     unsigned int wait_us)
+{
+	u8 status = 0;
+	int ret;
+
+	ret = qdfu_recv_vendor_req(hdev, QDFU_BT_CMD_ACTIVATE_REMOTE_BTSS, on,
+				   &status, sizeof(status));
+	if (ret) {
+		bt_dev_err(hdev, "QDFU turn %s remote BTSS failed: %pe",
+			   str_on_off(on), ERR_PTR(ret));
+		return ret;
+	}
+
+	switch (status) {
+	case 0:
+		bt_dev_dbg(hdev, "QDFU Remote BTSS turned %s", str_on_off(on));
+		fsleep(wait_us);
+		break;
+	case 0x17:
+		bt_dev_dbg(hdev, "QDFU Remote BTSS already %s", str_on_off(on));
+		break;
+	default:
+		bt_dev_err(hdev,
+			   "QDFU turn remote BTSS %s failed, unexpected status 0x%02x",
+			   str_on_off(on), status);
+		ret = -ENODEV;
+	}
+
+	return ret;
+}
+
+static int qdfu_reset_peri_hci(struct hci_dev *hdev)
+{
+	int ret;
+
+	ret = qdfu_send_vendor_req(hdev, QDFU_BT_CMD_RESET_PERI_HCI, 0, NULL, 0);
+	if (ret) {
+		bt_dev_err(hdev, "PERI HCI reset via QDFU failed: %pe",
+			   ERR_PTR(ret));
+		return ret;
+	}
+	bt_dev_info(hdev, "PERI HCI reset via QDFU succeeded");
+	fsleep(20 * 1000);
+
+	return 0;
+}
+
+static int qdfu_sw_reset(struct hci_dev *hdev)
+{
+	int ret;
+
+	ret = qdfu_send_vendor_req(hdev, QDFU_BT_CMD_BT_ENABLE_RESET, 0, NULL, 0);
+	/* SW reset succeeds even if the request returns an error code */
+	if (ret)
+		bt_dev_dbg(hdev, "QDFU SW reset failed: %pe", ERR_PTR(ret));
+
+	bt_dev_info(hdev, "QDFU SW reset succeeded");
+
+	return 0;
+}
+
+/*
+ * qdfu_poll_state - wait for DFU target status flags to reach a wanted state
+ * @hdev: the HCI device to poll
+ * @state_ptr: optional storage for the last status read; may be NULL
+ * @set: true to wait until @flags are all set, false until all cleared
+ * @flags: the status flag bits to wait on
+ *
+ * Repeatedly reads the DFU target status until @flags reach the requested
+ * state, a status read fails, or the overall timeout expires.
+ *
+ * Return: 0 on success, -ETIMEDOUT on timeout, or a negative errno on read
+ *	failure.
+ */
+static int qdfu_poll_state(struct hci_dev *hdev, u8 *state_ptr, bool set, u8 flags)
+{
+	int err, ret;
+	u8 dfu_state;
+
+	if (!state_ptr)
+		state_ptr = &dfu_state;
+
+	ret = read_poll_timeout(qdfu_get_target_state, err,
+				err || (set ? (*state_ptr & flags) == flags
+					    : !(*state_ptr & flags)),
+				5 * 1000,
+				3000 * 1000, true,
+				hdev, state_ptr);
+	if (ret) {
+		bt_dev_err(hdev,
+			   "Timed out waiting for QDFU state flags 0x%02x to be %s: %pe",
+			   flags, set ? "set" : "cleared", ERR_PTR(ret));
+		return ret;
+	}
+
+	return err;
+}
+
+static inline int qdfu_reset_msubsys_bt(struct hci_dev *hdev)
+{
+	int ret;
+
+	ret = qdfu_activate_remote_btss(hdev, false, 100 * 1000);
+	if (!ret)
+		ret = qdfu_activate_remote_btss(hdev, true, 100 * 1000);
+	if (!ret)
+		bt_dev_info(hdev, "QDFU reset msubsys BT succeeded");
+
+	return ret;
+}
+
+/*
+ * ============================================================================
+ * Common definitions, per-device struct btqcom_data, and PERI frame format
+ * ============================================================================
+ */
+
+/* BTC version, read out via an HCI command */
+struct qhci_btc_ver {
+	u32 product_id;
+	u32 soc_ver;
+	u16 rom_ver;
+	u16 patch_ver;
+	u8  sec_ver;
+};
+
+/* Subsystem field appears in a PERI command/response/event */
+enum qhci_subsys {
+	QHCI_SUBSYS_PERI,
+	QHCI_SUBSYS_BT,
+	QHCI_SUBSYS_UWB,
+	QHCI_SUBSYS_TMEL,
+	QHCI_SUBSYS_MAX,
+};
+
+/*
+ * The pseudo QHCI subsys: for BT-only BTC, or when the memdump comes
+ * from the BT channel (HCI_EVENT_PKT or HCI_ACLDATA_PKT) on a msubsys
+ * BTC, there is no real subsys — it stands in for these cases.
+ */
+#define QHCI_SUBSYS_INVALID	QHCI_SUBSYS_MAX
+
+static const char *qhci_subsys_name(int subsys)
+{
+	const char *subsys_name = "unknown";
+
+	switch (subsys) {
+	case QHCI_SUBSYS_PERI:
+		subsys_name = "PERI";
+		break;
+	case QHCI_SUBSYS_BT:
+		subsys_name = "PERI_BT";
+		break;
+	case QHCI_SUBSYS_UWB:
+		subsys_name = "UWB";
+		break;
+	case QHCI_SUBSYS_TMEL:
+		subsys_name = "TME-L";
+		break;
+	case QHCI_SUBSYS_INVALID:
+		subsys_name = "BT";
+		break;
+	default:
+		break;
+	}
+
+	return subsys_name;
+}
+
+/* BTC subsystems we care about that support the BT function unit */
+enum qbtc_subsys {
+	QBTC_SUBSYS_PERI,
+	QBTC_SUBSYS_TMEL,
+	QBTC_SUBSYS_MAX,
+};
+
+/*
+ * The pseudo BTC subsys stands in for the BT function unit, not a
+ * real subsys, when building a firmware file path.
+ */
+#define QBTC_SUBSYS_INVALID QBTC_SUBSYS_MAX
+
+static const char *qbtc_subsys_name(int subsys)
+{
+	const char *subsys_name = "unknown";
+
+	switch (subsys) {
+	case QBTC_SUBSYS_PERI:
+		subsys_name = "PERI";
+		break;
+	case QBTC_SUBSYS_TMEL:
+		subsys_name = "TME-L";
+		break;
+	case QBTC_SUBSYS_INVALID:
+		subsys_name = "BT";
+		break;
+	default:
+		break;
+	}
+
+	return subsys_name;
+}
+
+/*
+ * struct qbtc_subsys_data - per-subsys data
+ * @ver: subsys version
+ * @board_id: subsys board ID
+ * @build_info: the subsys firmware's build info string
+ */
+struct qbtc_subsys_data {
+	struct qhci_btc_ver ver;
+	u16	board_id;
+	char	build_info[160];
+};
+
+/* simple command payload */
+struct qhci_cp_simple {
+	u8	sub_opcode;
+} __packed;
+
+/* simple CCE(response) payload */
+struct qhci_rp_simple {
+	u8	status;
+} __packed;
+
+/* generic CCE(response) payload */
+struct qhci_rp_generic {
+	u8		status;
+	u8		sub_opcode;
+} __packed;
+
+/* common VSE payload */
+struct qhci_vse_comm {
+	u8   ev_class;
+	u8   ev_type;
+} __packed;
+#define QHCI_VSE_COMM_SIZE	(sizeof(struct qhci_vse_comm))
+
+#define QHCI_MEMDUMP_SEQ_LAST		0xFFFF
+#define QBT_EV_CLASS_DATALOG 0x01
+#define QBT_EV_TYPE_MEMDUMP  0x08
+
+/*
+ * struct qhci_vse_memdump - one memdump segment
+ * @seqno: segment sequence number; 0 = first, QHCI_MEMDUMP_SEQ_LAST = last
+ * @subsys: subsystem this segment belongs to (see enum qhci_subsys)
+ * @segdata: payload of a middle or last segment
+ * @dump_size: total dump size, valid only in the first segment (@seqno == 0)
+ * @first_segdata: payload of the first segment, following @dump_size
+ */
+struct qhci_vse_memdump {
+	__le16 seqno;
+	u8     subsys;
+	union {
+		u8 segdata[0];
+		struct {
+			__le32 dump_size;
+			u8 first_segdata[];
+		} __packed;
+	};
+} __packed;
+
+#define QHCI_MEMDUMP_SEGDATA_GAP \
+	(offsetof(struct qhci_vse_memdump, first_segdata) - \
+	 offsetof(struct qhci_vse_memdump, segdata))
+
+enum qhci_req_state {
+	QHCI_REQ_DONE,
+	QHCI_REQ_PEND,
+	QHCI_REQ_CANCELED,
+	/* unused for now */
+	QHCI_REQ_STOP,
+};
+
+#define QHCI_OPCODE_INVALID		HCI_OP_NOP
+#define QHCI_SUB_OPCODE_INVALID		0xff
+#define QHCI_REQ_EVENT_MAX		2
+
+/*
+ * struct qhci_req_spec - request/response specification
+ * @opcode: command opcode, ignored if invalid
+ * @sub_opcode: command sub-opcode, ignored if invalid
+ * @event: a sequence of events, ending with an invalid marker
+ */
+struct qhci_req_spec {
+	u16	opcode;
+	u8	sub_opcode;
+	u8	event[QHCI_REQ_EVENT_MAX + 1];
+};
+
+/*
+ * struct qhci_req - per-request data, often stack allocated by the requester
+ * @pkt_type: the packet type to sync
+ * @spec: the request spec to sync
+ * @req_rsp: response skb on success, NULL on no data, or ERR_PTR() on failure
+ * @req_result: >= 0 an HCI status code, or a negative errno on failure
+ * @event_idx: cursor into @spec->event
+ */
+struct qhci_req {
+	u8				pkt_type;
+	const struct qhci_req_spec	*spec;
+	struct sk_buff			*req_rsp;
+	int				req_result;
+	u8				event_idx;
+};
+
+/* board has a BT_EN pin to reset BTC */
+#define QBT_FLAG_HW_RESET		BIT(24)
+
+/*
+ * enum qbt_work_bit - bits in btqcom_data.work_flags
+ * @QBT_WORK_RESET_HDEV: request to reset hdev
+ */
+enum qbt_work_bit {
+	QBT_WORK_RESET_HDEV,
+};
+
+/*
+ * enum qbt_misc_bit - bits in btqcom_data.misc_flags
+ * @QBT_MISC_MEMDUMP_PERI: PERI memdump
+ * @QBT_MISC_MEMDUMP_BT: BT memdump
+ * @QBT_MISC_MEMDUMP_UWB: UWB memdump, never happens here
+ * @QBT_MISC_MEMDUMP_TMEL: TME-L memdump
+ * @QBT_MISC_MEMDUMP_INCOMING: memdump is arriving from the BTC
+ * @QBT_MISC_RESET_ACTIVE: a reset is in progress
+ * @QBT_MISC_HWERR_PERI: PERI hardware error event happens
+ * @QBT_MISC_HWERR_BT: BT hardware error event happens
+ * @QBT_MISC_CMD_TIMEOUT: command timeout happens
+ *
+ * The memdump bits are indexed by enum qhci_subsys, so a subsystem's bit can
+ * be derived from its QHCI subsys value.
+ */
+enum qbt_misc_bit {
+	QBT_MISC_MEMDUMP_PERI = QHCI_SUBSYS_PERI,
+	QBT_MISC_MEMDUMP_BT = QHCI_SUBSYS_BT,
+	QBT_MISC_MEMDUMP_UWB = QHCI_SUBSYS_UWB,
+	QBT_MISC_MEMDUMP_TMEL = QHCI_SUBSYS_TMEL,
+	QBT_MISC_MEMDUMP_INCOMING,
+	QBT_MISC_RESET_ACTIVE,
+	QBT_MISC_HWERR_PERI,
+	QBT_MISC_HWERR_BT,
+	QBT_MISC_CMD_TIMEOUT,
+};
+
+/* each subsys's memdump pending flag in btqcom_data.md_pending_flags */
+#define QMD_FLAG_PENDING_PERI	BIT(QBT_MISC_MEMDUMP_PERI)
+#define QMD_FLAG_PENDING_BT	BIT(QBT_MISC_MEMDUMP_BT)
+#define QMD_FLAG_PENDING_TMEL	BIT(QBT_MISC_MEMDUMP_TMEL)
+#define QMD_FLAG_PENDING_MASK	(QMD_FLAG_PENDING_PERI | \
+				 QMD_FLAG_PENDING_BT | \
+				 QMD_FLAG_PENDING_TMEL)
+
+static inline int qmd_subsys_to_bit(int qhci_subsys)
+{
+	return qhci_subsys == QHCI_SUBSYS_INVALID ?
+	       QBT_MISC_MEMDUMP_BT : qhci_subsys;
+}
+
+static inline unsigned long qmd_subsys_to_flag(int qhci_subsys)
+{
+	return BIT(qmd_subsys_to_bit(qhci_subsys));
+}
+
+/*
+ * struct btqcom_memdump - collect memdump from a QHCI subsys
+ * @size: total memdump size to collect
+ * @subsys: QHCI subsys this memdump belongs to
+ * @seqno: next expected sequence number
+ * @from: which channel this memdump comes from, marked by hci_skb_pkt_type()
+ * @submit_size: bytes submitted to HCI devcoredump for the subsys
+ */
+struct btqcom_memdump {
+	u32 size;
+	int subsys;
+	u16 seqno;
+	u8 from;
+
+	u32 submit_size;
+};
+
+/*
+ * struct btqcom_data - transport-independent per-device common data, allocated as hci priv
+ * @drv_name: driver name
+ * @btc_name: human-readable BTC name
+ * @category: driver-perspective BTC category (legacy/unified/multi-subsys)
+ * @flags: QBT_FLAG_* flags for BTC/board and default config
+ * @hdev: the owning HCI device
+ * @xport_data: opaque transport-specific data (e.g. struct btusb_qcom for USB)
+ * @inited: whether the hdev-lifetime fields are initialized
+ * @board_id: BT board ID
+ * @ver: BT version
+ * @build_info: BT firmware build info string
+ * @fw_dir: firmware directory configured by user via sysfs
+ * @fw_logging: firmware logging configured by user via sysfs
+ * @misc_flags: flags made from QBT_MISC_* bits defined above
+ * @work_flags: flags made from QBT_WORK_* bits defined above
+ * @dwork: delayed work to handle @work_flags
+ * @md_ready: memdump functionality is ready
+ * @md_state: devcoredump state as notified by the HCI devcoredump core
+ * @md_submit_err: first error submitting to HCI devcoredump, 0 if none
+ * @md_submit_size: total bytes submitted to HCI devcoredump, headers included
+ * @md_pending_flags: QMD_FLAG_PENDING_* flags tracking per-subsystem memdump
+ * @md: per-subsystem memdump collection track
+ * @req_mutex: serializes sync requesters
+ * @req_wait_q: wait queue for the requester
+ * @req_spinlock: protects @req_state, @req, and @*req
+ * @req_state: request state, see enum qhci_req_state
+ * @req: the in-flight request, valid while @req_state is QHCI_REQ_PEND
+ * @subsys: per-subsystem info for MSUBSYS chips
+ */
+struct btqcom_data {
+	const char *drv_name;
+	const char *btc_name;
+	enum qbtc_category category;
+	unsigned long flags;
+
+	struct hci_dev *hdev;
+	void *xport_data;
+
+	bool inited;
+
+	u16	board_id;
+	struct qhci_btc_ver ver;
+	char	build_info[160];
+
+	const char *fw_dir;
+	u8 fw_logging;
+
+	unsigned long misc_flags;
+	unsigned long		work_flags;
+	struct delayed_work	dwork;
+
+	bool md_ready;
+	enum devcoredump_state md_state;
+	int md_submit_err;
+	u32 md_submit_size;
+	unsigned long md_pending_flags;
+	struct btqcom_memdump md;
+
+	struct mutex		req_mutex;
+	wait_queue_head_t	req_wait_q;
+	spinlock_t		req_spinlock;
+	int			req_state;
+	struct qhci_req		*req;
+
+	struct qbtc_subsys_data subsys[QBTC_SUBSYS_MAX];
+};
+
+/* BT host ID in PERI command/event/ACL */
+#define QHCI_HOST_ID_BT 0
+
+struct qperi_command_hdr {
+	u8	host_id;
+	__le16	opcode;
+	u8	plen;
+} __packed;
+#define QPERI_COMMAND_HDR_SIZE (sizeof(struct qperi_command_hdr))
+
+struct qperi_acl_hdr {
+	u8	host_id;
+	__le16	handle;
+	__le16	dlen;
+} __packed;
+#define QPERI_ACL_HDR_SIZE	(sizeof(struct qperi_acl_hdr))
+#define QPERI_MAX_FRAME_SIZE	(HCI_MAX_FRAME_SIZE + 1)
+
+struct qperi_event_hdr {
+	u8	host_id;
+	u8	evt;
+	u8	plen;
+} __packed;
+#define QPERI_EVENT_HDR_SIZE	(sizeof(struct qperi_event_hdr))
+#define QPERI_MAX_EVENT_SIZE	(HCI_MAX_EVENT_SIZE + 1)
+
+static inline struct qperi_event_hdr *qperi_event_header(const struct sk_buff *skb)
+{
+	return (struct qperi_event_hdr *)skb->data;
+}
+
+static inline struct qperi_acl_hdr *qperi_acl_header(const struct sk_buff *skb)
+{
+	return (struct qperi_acl_hdr *)skb->data;
+}
+
+static inline __u16 qperi_acl_handle(const struct sk_buff *skb)
+{
+	struct qperi_acl_hdr *hdr = qperi_acl_header(skb);
+
+	return hci_handle(__le16_to_cpu(hdr->handle));
+}
+
+static inline __u16 qperi_acl_dlen(const struct sk_buff *skb)
+{
+	return __le16_to_cpu(qperi_acl_header(skb)->dlen);
+}
+
+static const char *qhci_pkt_type_name(u8 pkt_type)
+{
+	const char *pkt_type_name = "Unknown";
+
+	switch (pkt_type) {
+	case HCI_EVENT_PKT:
+		pkt_type_name = "BT EVT";
+		break;
+	case QPERI_EVENT_PKT:
+		pkt_type_name = "PERI EVT";
+		break;
+	case HCI_ACLDATA_PKT:
+		pkt_type_name = "BT ACL";
+		break;
+	case QPERI_ACLDATA_PKT:
+		pkt_type_name = "PERI ACL";
+		break;
+	default:
+		break;
+	}
+
+	return pkt_type_name;
+}
+
+static void btusb_qcom_reset_sync(struct hci_dev *hdev);
+
+/*
+ * ============================================================================
+ * Multi-subsys memdump design
+ * ============================================================================
+ */
+
+/*
+ * Design compatible with BT-only BTC as well:
+ *
+ * Abbreviations: SS - Subsystem, MD - Memdump, HDR - Header
+ *
+ * +--------------+-------------+---------+     +-------------+---------+
+ * | file HDR     | SS_A MD HDR | SS_A    | ... | SS_N MD HDR | SS_N    |
+ * | 512B         | 512B        | MD data |     | 512B        | MD data |
+ * | by dmp_hdr() | appended as |         |     | appended as |         |
+ * |              | MD data     |         |     | MD data     |         |
+ * +--------------+-------------+---------+     +-------------+---------+
+ *
+ * 1) All SS MDs are collected into a SINGLE file.
+ * 2) Each SS_X MD HDR records the size of its own MD data.
+ * 3) On the 1st segment of the 1st SS: call hci_devcd_init() with
+ *    dump_size — for a BT-only BTC, a 512B MD HDR plus the size carried
+ *    in the segment itself; for a multi-subsys BTC, the sum of (a 512B
+ *    MD HDR plus the max MD data size) over every SS. The file HDR is
+ *    not counted, since the HCI devcoredump adds it on top of dump_size.
+ * 4) On the 1st segment of every SS: call hci_devcd_append() to append
+ *    its 512B MD HDR as ordinary MD data.
+ * 5) On the last segment of a SS: call hci_devcd_complete() only if
+ *    no other SS's MD is still pending.
+ * 6) On the hardware error event (either PERI or BT), which always
+ *    comes after all MDs have been reported by the BTC: call
+ *    hci_devcd_complete() there if it has not been called yet, since a
+ *    SS's MD is optional.
+ */
+
+static const u32 btqcom_memdump_maxsize[] = {
+	[QHCI_SUBSYS_PERI]	= SZ_256K,
+	[QHCI_SUBSYS_BT]	= SZ_1M,
+	[QHCI_SUBSYS_UWB]	= 0,
+	[QHCI_SUBSYS_TMEL]	= SZ_256K,
+	[QHCI_SUBSYS_INVALID]	= SZ_1M,
+};
+
+/*
+ * btqcom_memdump_hdr_room - helper to fill a memdump header up to 512 bytes
+ * @skb: the devcoredump header skb to fill
+ *
+ * Return: bytes safe to write into @skb.
+ */
+static int btqcom_memdump_hdr_room(struct sk_buff *skb)
+{
+	int end_marker_len = sizeof(HCI_DEVCD_HDR_END_MARKER) - 1;
+	/*
+	 * don't use skb_tailroom(): observed to differ from requested
+	 * alloc_size - skb->len
+	 */
+	int avail_room = HCI_DEVCD_HDR_SIZE_MAX - skb->len;
+	u8 *ptr;
+
+	/* 1 char + '\n' at least */
+	if (avail_room >= end_marker_len + 2)
+		return avail_room - end_marker_len;
+
+	if (avail_room == end_marker_len)
+		return 0;
+
+	/* Avoid a blank line ('\n\n') in the header */
+	skb_trim(skb, HCI_DEVCD_HDR_SIZE_MAX - end_marker_len - 1);
+	ptr = skb_tail_pointer(skb) - 1;
+	if (*ptr == '\n')
+		*ptr = ' ';
+	skb_put_u8(skb, '\n');
+
+	return 0;
+}
+
+/*
+ * btqcom_memdump_hdr - build the 512B memdump header
+ * @hdev: the HCI device
+ * @skb: the skb to build the header into
+ *
+ * Its 'Memdump Size' field is the size of the following payload.
+ */
+static void btqcom_memdump_hdr(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	int avail_room = btqcom_memdump_hdr_room(skb);
+	struct btqcom_memdump *md = &qbt_data->md;
+	struct qbtc_subsys_data *subsys_data;
+	char buf[HCI_DEVCD_HDR_SIZE_MAX];
+	struct qhci_btc_ver *btc_ver;
+	const char *build_info;
+	u16 board_id;
+	int len = 0;
+
+	if (!avail_room)
+		return;
+
+	len += scnprintf(buf + len, sizeof(buf) - len, "Header Type: memdump\n");
+	len += scnprintf(buf + len, sizeof(buf) - len, "Header Size: %d\n",
+			 HCI_DEVCD_HDR_SIZE_MAX);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Memdump Size: %u\n",
+			 md->size);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Channel: %s\n",
+			 qhci_pkt_type_name(md->from));
+	len += scnprintf(buf + len, sizeof(buf) - len, "Owner: %s\n",
+			 qhci_subsys_name(md->subsys));
+
+	switch (md->subsys) {
+	case QHCI_SUBSYS_PERI:
+		subsys_data = &qbt_data->subsys[QBTC_SUBSYS_PERI];
+		btc_ver = &subsys_data->ver;
+		board_id = subsys_data->board_id;
+		build_info = subsys_data->build_info;
+		break;
+	case QHCI_SUBSYS_TMEL:
+		subsys_data = &qbt_data->subsys[QBTC_SUBSYS_TMEL];
+		btc_ver = &subsys_data->ver;
+		board_id = subsys_data->board_id;
+		build_info = subsys_data->build_info;
+		break;
+	default:
+		btc_ver = &qbt_data->ver;
+		board_id = qbt_data->board_id;
+		build_info = qbt_data->build_info;
+		break;
+	}
+
+	len += scnprintf(buf + len, sizeof(buf) - len, "SoC Version: 0x%08x\n",
+			 btc_ver->soc_ver);
+	len += scnprintf(buf + len, sizeof(buf) - len, "ROM Version: 0x%04x\n",
+			 btc_ver->rom_ver);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Patch Version: 0x%04x\n",
+			 btc_ver->patch_ver);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Board ID: 0x%04x\n",
+			 board_id);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Firmware Version: %s\n",
+			 build_info);
+
+	if (len > avail_room) {
+		bt_dev_warn(hdev, "memdump header truncated (%d -> %d bytes)",
+			    len, avail_room);
+		len = avail_room;
+		if (buf[len - 2] == '\n')
+			buf[len - 2] = ' ';
+		buf[len - 1] = '\n';
+	}
+
+	skb_put_data(skb, buf, len);
+}
+
+/*
+ * btusb_qcom_memdump_hdr - build the 512B file header
+ * @hdev: the HCI device
+ * @skb: the skb to build the header into
+ *
+ * Implements the dmp_hdr_t for hci_devcd_register().
+ */
+static void btusb_qcom_memdump_hdr(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	char buf[HCI_DEVCD_HDR_SIZE_MAX];
+	const char *vendor = "Qualcomm";
+	int avail_room;
+	int len = 0;
+
+	avail_room = btqcom_memdump_hdr_room(skb);
+	if (!avail_room)
+		return;
+
+	len += scnprintf(buf + len, sizeof(buf) - len, "Header Type: file\n");
+	len += scnprintf(buf + len, sizeof(buf) - len, "Header Size: %d\n",
+			 HCI_DEVCD_HDR_SIZE_MAX);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Driver: %s\n",
+			 qbt_data->drv_name);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Vendor: %s\n", vendor);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Controller Name: %s\n",
+			 qbt_data->btc_name);
+	len += scnprintf(buf + len, sizeof(buf) - len, "Controller Category: %s\n",
+			 qbtc_category_name(qbt_data->category));
+
+	if (qbt_data->category == QBTC_CAT_MSUBSYS) {
+		if (qbt_data->flags & QBT_FLAG_HAS_TMEL)
+			len += scnprintf(buf + len, sizeof(buf) - len,
+					 "Controller Subsys: PERI, TME-L\n");
+		else
+			len += scnprintf(buf + len, sizeof(buf) - len,
+					 "Controller Subsys: PERI\n");
+	}
+	len += scnprintf(buf + len, sizeof(buf) - len, "VID: 0x%04x\n",
+			 xport_data->idVendor);
+	len += scnprintf(buf + len, sizeof(buf) - len, "PID: 0x%04x\n",
+			 xport_data->idProduct);
+
+	if (len > avail_room)
+		bt_dev_warn(hdev, "dump header truncated (%d -> %d bytes)",
+			    len, avail_room);
+
+	memset(buf + len, 'P', sizeof(buf) - len);
+
+	/*
+	 * Deliberately overfill to skb's full 512 bytes, then leverage
+	 * btqcom_memdump_hdr_room() to trim it back to the expected boundary.
+	 */
+	skb_put_data(skb, buf, HCI_DEVCD_HDR_SIZE_MAX - skb->len);
+	btqcom_memdump_hdr_room(skb);
+}
+
+/* Used as the notify_change_t callback for hci_devcd_register(). */
+static void btusb_qcom_notify_memdump(struct hci_dev *hdev, int state)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	enum devcoredump_state old_state;
+
+	old_state = qbt_data->md_state;
+
+	bt_dev_dbg(hdev, "memdump notify state: %s -> %s",
+		   hci_devcd_state_name(old_state),
+		   hci_devcd_state_name(state));
+	bt_dev_dbg(hdev, "misc_flags: 0x%lx", READ_ONCE(qbt_data->misc_flags));
+	bt_dev_dbg(hdev, "md_pending_flags: 0x%lx, md_submit_err: %d",
+		   READ_ONCE(qbt_data->md_pending_flags),
+		   READ_ONCE(qbt_data->md_submit_err));
+
+	switch (state) {
+	case HCI_DEVCOREDUMP_IDLE:
+		break;
+	case HCI_DEVCOREDUMP_ACTIVE:
+		usb_autopm_get_interface_no_resume(xport_data->intf);
+		bt_dev_dbg(hdev, "memdump notify: get autopm refcount");
+		break;
+	case HCI_DEVCOREDUMP_TIMEOUT:
+		bt_dev_err(hdev, "memdump notify: %s", hci_devcd_state_name(state));
+		qbt_data->md_submit_err = -ETIMEDOUT;
+		qbt_data->md_pending_flags = 0;
+		test_and_clear_bit(QBT_MISC_MEMDUMP_INCOMING,
+				   &qbt_data->misc_flags);
+		fallthrough;
+	case HCI_DEVCOREDUMP_DONE:
+	case HCI_DEVCOREDUMP_ABORT:
+		usb_autopm_put_interface_no_suspend(xport_data->intf);
+		bt_dev_dbg(hdev, "memdump notify: put autopm refcount");
+		break;
+	}
+
+	qbt_data->md_state = state;
+
+	if (state == HCI_DEVCOREDUMP_TIMEOUT)
+		btusb_qcom_reset_sync(hdev);
+}
+
+/* handle received memdump frame here */
+
+static inline void btqcom_reset_memdump(struct btqcom_memdump *md)
+{
+	memset(md, 0x00, sizeof(*md));
+	md->subsys  = QHCI_SUBSYS_INVALID;
+}
+
+/*
+ * btqcom_submit_memdump - submit one memdump segment to HCI devcoredump
+ * @hdev: the HCI device the memdump comes from
+ * @skb: the memdump segment payload
+ *
+ * See the "Multi-subsys memdump design" block above for the overall scheme
+ * this implements.
+ *
+ * Return: 0 on success,
+ *	1 if the memdump ended normally,
+ *	a negative errno on failure.
+ */
+static int btqcom_submit_memdump(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct btqcom_memdump *md = &qbt_data->md;
+	unsigned int dump_size, seg_len;
+	bool is_first_subsys = false;
+	int ret = 0;
+
+	dump_size = 0;
+	seg_len = skb->len;
+
+	if (md->seqno == 0) {
+		if (!test_and_set_bit(QBT_MISC_MEMDUMP_INCOMING,
+				      &qbt_data->misc_flags)) {
+			/* wake the waiter in btusb_do_cmd_timeout_work() */
+			wake_up_var(&qbt_data->misc_flags);
+			is_first_subsys = true;
+			qbt_data->md_submit_err = 0;
+			qbt_data->md_submit_size = 0;
+			if (qbt_data->category == QBTC_CAT_MSUBSYS) {
+				qbt_data->md_pending_flags = QMD_FLAG_PENDING_BT |
+							     QMD_FLAG_PENDING_PERI;
+				dump_size = HCI_DEVCD_HDR_SIZE_MAX +
+					    btqcom_memdump_maxsize[QHCI_SUBSYS_BT] +
+					    HCI_DEVCD_HDR_SIZE_MAX +
+					    btqcom_memdump_maxsize[QHCI_SUBSYS_PERI];
+
+				if (qbt_data->flags & QBT_FLAG_HAS_TMEL) {
+					qbt_data->md_pending_flags |= QMD_FLAG_PENDING_TMEL;
+					dump_size += HCI_DEVCD_HDR_SIZE_MAX +
+						     btqcom_memdump_maxsize[QHCI_SUBSYS_TMEL];
+				}
+			} else {
+				qbt_data->md_pending_flags = QMD_FLAG_PENDING_BT;
+				dump_size = HCI_DEVCD_HDR_SIZE_MAX + md->size;
+			}
+		}
+
+		bt_dev_info(hdev, "%s memdump incoming: %u bytes",
+			    qhci_subsys_name(md->subsys), md->size);
+		bt_dev_dbg(hdev, "md_pending_flags: 0x%lx",
+			   qbt_data->md_pending_flags);
+
+		if (is_first_subsys) {
+			ret = hci_devcd_init(hdev, dump_size);
+			if (ret) {
+				kfree_skb(skb);
+				bt_dev_err(hdev, "init memdump failed: %pe",
+					   ERR_PTR(ret));
+				return ret;
+			}
+			qbt_data->md_submit_size += HCI_DEVCD_HDR_SIZE_MAX;
+			bt_dev_info(hdev, "file header: %u bytes",
+				    HCI_DEVCD_HDR_SIZE_MAX);
+		}
+
+		if (!qbt_data->md_submit_err) {
+			struct sk_buff *hdr_skb;
+
+			hdr_skb = alloc_skb(HCI_DEVCD_HDR_SIZE_MAX, GFP_KERNEL);
+			if (hdr_skb) {
+				btqcom_memdump_hdr(hdev, hdr_skb);
+				if (hdr_skb->len < HCI_DEVCD_HDR_SIZE_MAX)
+					skb_put_zero(hdr_skb,
+						     HCI_DEVCD_HDR_SIZE_MAX - hdr_skb->len);
+				ret = hci_devcd_append(hdev, hdr_skb);
+			} else {
+				ret = -ENOMEM;
+			}
+			if (ret) {
+				hci_devcd_abort(hdev);
+				kfree_skb(skb);
+				bt_dev_err(hdev, "append memdump header failed: %pe",
+					   ERR_PTR(ret));
+				return ret;
+			}
+			qbt_data->md_submit_size += HCI_DEVCD_HDR_SIZE_MAX;
+			bt_dev_info(hdev, "%s memdump header appended: %u bytes",
+				    qhci_subsys_name(md->subsys), HCI_DEVCD_HDR_SIZE_MAX);
+		}
+	}
+
+	ret = qbt_data->md_submit_err;
+	if (ret) {
+		kfree_skb(skb);
+	} else {
+		ret = hci_devcd_append(hdev, skb);
+		if (unlikely(ret)) {
+			hci_devcd_abort(hdev);
+			bt_dev_err(hdev, "append memdump failed: %pe", ERR_PTR(ret));
+		} else {
+			md->submit_size += seg_len;
+			qbt_data->md_submit_size += seg_len;
+		}
+	}
+
+	if (md->seqno != QHCI_MEMDUMP_SEQ_LAST)
+		return ret;
+
+	if (md->submit_size == md->size)
+		bt_dev_info(hdev, "%s memdump fully collected",
+			    qhci_subsys_name(md->subsys));
+	else
+		bt_dev_err(hdev, "%s memdump partially collected %u/%u bytes",
+			   qhci_subsys_name(md->subsys), md->submit_size, md->size);
+
+	qbt_data->md_pending_flags &= ~qmd_subsys_to_flag(md->subsys);
+	if (qbt_data->md_pending_flags) {
+		bt_dev_dbg(hdev, "md_pending_flags: 0x%lx",
+			   qbt_data->md_pending_flags);
+		return ret;
+	}
+
+	if (test_and_clear_bit(QBT_MISC_MEMDUMP_INCOMING, &qbt_data->misc_flags))
+		bt_dev_dbg(hdev, "clear MEMDUMP_INCOMING on memdump completion");
+
+	if (ret) {
+		bt_dev_err(hdev, "memdump complete failed: %pe", ERR_PTR(ret));
+		return ret;
+	}
+
+	ret = hci_devcd_complete(hdev);
+	if (ret) {
+		hci_devcd_abort(hdev);
+		bt_dev_err(hdev, "memdump complete failed: %pe", ERR_PTR(ret));
+	} else {
+		ret = 1;
+		bt_dev_info(hdev, "memdump completed: %u bytes",
+			    qbt_data->md_submit_size);
+	}
+	return ret;
+}
+
+/*
+ * btqcom_handle_memdump - handle one memdump segment
+ * @hdev: the HCI device the @skb comes from
+ * @skb: the memdump segment to handle
+ *
+ * Return: 0 on success,
+ *	1 if the memdump ended normally,
+ *	a negative errno on failure.
+ */
+static int btqcom_handle_memdump(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct btqcom_memdump *md = &qbt_data->md;
+	const struct qhci_vse_memdump *md_vse;
+	u8 pkt_type = hci_skb_pkt_type(skb);
+	int subsys = QHCI_SUBSYS_INVALID;
+	u16 seq_no = 0;
+	u32 dump_size;
+	int ret = 0;
+
+	md_vse = skb_pull_data(skb, offsetof(struct qhci_vse_memdump, segdata));
+	if (!md_vse) {
+		ret = -EMSGSIZE;
+		bt_dev_err(hdev, "bad memdump segment: too short (%u bytes)", skb->len);
+		goto out_abort_md;
+	}
+
+	if (pkt_type == QPERI_EVENT_PKT || pkt_type == QPERI_ACLDATA_PKT) {
+		subsys = md_vse->subsys;
+		if (subsys >= QHCI_SUBSYS_MAX || !btqcom_memdump_maxsize[subsys] ||
+		    (subsys == QHCI_SUBSYS_TMEL &&
+		     !(qbt_data->flags & QBT_FLAG_HAS_TMEL))) {
+			kfree_skb(skb);
+			bt_dev_warn_ratelimited(hdev,
+						"drop memdump of unsupported subsys %d",
+						subsys);
+			return 0;
+		}
+	}
+
+	seq_no = le16_to_cpu(md_vse->seqno);
+	if (seq_no == 0) {
+		set_bit(qmd_subsys_to_bit(subsys), &qbt_data->misc_flags);
+
+		if (!skb_pull(skb, QHCI_MEMDUMP_SEGDATA_GAP)) {
+			ret = -EMSGSIZE;
+			bt_dev_err(hdev, "bad first %s memdump segment: too short for dump_size",
+				   qhci_subsys_name(subsys));
+			goto out_abort_md;
+		}
+
+		dump_size = le32_to_cpu(md_vse->dump_size);
+		if (!dump_size || dump_size > btqcom_memdump_maxsize[subsys]) {
+			ret = -EILSEQ;
+			bt_dev_err(hdev, "wrong %s memdump dump_size: %u",
+				   qhci_subsys_name(subsys), dump_size);
+			goto out_abort_md;
+		}
+
+		btqcom_reset_memdump(md);
+		md->from   = pkt_type;
+		md->subsys = subsys;
+		md->size   = dump_size;
+	} else {
+		if (md->subsys != subsys) {
+			ret = -EILSEQ;
+			bt_dev_err_ratelimited(hdev,
+					       "wrong memdump subsys: expected(%d), coming(%d)",
+					       md->subsys, subsys);
+			goto out_abort_md;
+		}
+
+		if (seq_no == QHCI_MEMDUMP_SEQ_LAST) {
+			md->seqno = QHCI_MEMDUMP_SEQ_LAST;
+		} else if (seq_no != md->seqno) {
+			ret = -EILSEQ;
+			bt_dev_err_ratelimited(hdev,
+					       "wrong memdump seqno: expected(%u), coming(%u)",
+					       md->seqno, seq_no);
+			goto out_abort_md;
+		}
+	}
+
+	/* @skb is consumed here */
+	ret = btqcom_submit_memdump(hdev, skb);
+	if (seq_no != QHCI_MEMDUMP_SEQ_LAST)
+		md->seqno = seq_no + 1;
+
+	goto out_reset_md;
+
+out_abort_md:
+	kfree_skb(skb);
+	if (qbt_data->md_pending_flags && !qbt_data->md_submit_err) {
+		hci_devcd_abort(hdev);
+		bt_dev_err(hdev, "abort memdump");
+	}
+
+out_reset_md:
+	if (ret < 0 && !qbt_data->md_submit_err)
+		qbt_data->md_submit_err = ret;
+
+	if (seq_no == QHCI_MEMDUMP_SEQ_LAST)
+		btqcom_reset_memdump(md);
+
+	return ret;
+}
+
+/*
+ * ============================================================================
+ * BT-HCI vendor-specific command/response
+ * ============================================================================
+ */
+
+/* reserved BT ACL handle for enhanced logging */
+#define QBT_HANDLE_ENHANCED_LOGGING 0xEDC
+/* reserved BT ACL handle for memdump */
+#define QBT_HANDLE_MEMDUMP 0xEDD
+
+/*
+ * Unless otherwise noted, for the VSCs below:
+ * - command payload: struct qhci_cp_simple
+ * - response payload: struct qhci_rp_generic
+ *
+ * Grouped by { opcode, sub_opcode, cp, rp }.
+ */
+
+/* BT EDL opcode and its sub-opcodes */
+#define QBT_OP_EDL		0xFC00
+
+#define BEDL_PATCH_GETVER	0x19
+struct qbt_rp_patch_getver {
+	u8	status;
+	u8	sub_opcode;
+	u8	dlen;
+	__le32	product_id;
+	__le16	patch_ver;
+	__le16	rom_ver;
+	__le32	soc_ver;
+} __packed;
+
+#define	BEDL_GET_BUILD_INFO 0x20
+struct qbt_rp_get_build_info {
+	u8	status;
+	u8	sub_opcode;
+	u8	dlen;
+	u8	data[];
+} __packed;
+
+#define BEDL_GET_BOARD_ID	0x23
+struct qbt_rp_get_board_id {
+	u8	status;
+	u8	sub_opcode;
+	u8	dlen;
+	__be16	board_id;
+} __packed;
+
+/* BT DEBUG opcode and its sub-opcodes */
+#define QBT_OP_DEBUG		0xFC0C
+
+#define BDBG_ERROR_FATAL_CMD	0x26
+/* no response */
+
+/* BT write BD_ADDR opcode, no sub-opcode */
+#define QBT_OP_WRITE_BD_ADDR	0xFC14
+struct qbt_cp_write_bd_addr {
+	bdaddr_t bdaddr;
+} __packed;
+
+/* BT firmware logging opcode and its sub-opcodes */
+#define QBT_OP_HOST_LOG		0xFC17
+
+#define BHL_ENH_ENABLE_LOG	0x14
+struct qbt_cp_config_fw_logging {
+	u8	sub_opcode;
+	u8	flags;
+} __packed;
+
+/*
+ * QBT_CHECK_RP_GENERIC - sanity check rp @_skb against type @_rp_type
+ * @_skb: the rp to check
+ * @_rp_type: the type to interpret @_skb as
+ * @_sub_opcode: sub_opcode to check @_skb against
+ *
+ * It is safe to evaluate @_skb and @_sub_opcode more than once for its
+ * usages.
+ *
+ * To simplify logic, @_skb's length is checked first to keep later field
+ * accesses safe.
+ *
+ * Returns a negative errno on failure, 0 on success, or error status code
+ * otherwise.
+ */
+#define QBT_CHECK_RP_GENERIC(_skb, _rp_type, _sub_opcode)		\
+({									\
+	_rp_type *_rp_ptr;						\
+	int _err = 0;							\
+	do {								\
+		if ((_skb)->len < sizeof(_rp_type)) {			\
+			_err = -EBADMSG;				\
+			break;						\
+		}							\
+		_rp_ptr = (void *)(_skb)->data;				\
+		if (_rp_ptr->sub_opcode != (_sub_opcode)) {		\
+			_err = -EILSEQ;					\
+			break;						\
+		}							\
+		if (_rp_ptr->status) {					\
+			_err = _rp_ptr->status;				\
+			break;						\
+		}							\
+	} while (0);							\
+	_err;								\
+})
+
+/* check the @dlen field on top of QBT_CHECK_RP_GENERIC() */
+#define QBT_CHECK_RP_DLEN(_skb, _rp_type, _sub_opcode)			\
+({									\
+	_rp_type *_rp_ptr;						\
+	int _err = 0;							\
+	do {								\
+		_err = QBT_CHECK_RP_GENERIC(_skb, _rp_type, _sub_opcode);\
+		if (_err)						\
+			break;						\
+		_rp_ptr = (void *)(_skb)->data;				\
+		if ((_skb)->len != offsetofend(_rp_type, dlen) +	\
+				   _rp_ptr->dlen) {			\
+			_err = -EMSGSIZE;				\
+			break;						\
+		}							\
+	} while (0);							\
+	_err;								\
+})
+
+/*
+ * qbt_edl_patch_getver - read BTC version info
+ * @hdev: the HCI device to query
+ * @ver: output version info, filled on success
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qbt_edl_patch_getver(struct hci_dev *hdev, struct qhci_btc_ver *ver)
+{
+	const struct qhci_cp_simple cp = { .sub_opcode = BEDL_PATCH_GETVER };
+	struct qbt_rp_patch_getver *rp;
+	struct sk_buff *skb;
+	int err;
+
+	skb = __hci_cmd_sync_ev(hdev, QBT_OP_EDL, sizeof(cp),
+				&cp, 0, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		err = PTR_ERR(skb);
+		goto out;
+	}
+
+	err = QBT_CHECK_RP_DLEN(skb, struct qbt_rp_patch_getver, cp.sub_opcode);
+	if (err)
+		goto out_free_skb;
+
+	rp = (void *)skb->data;
+	ver->product_id = le32_to_cpu(rp->product_id);
+	ver->soc_ver    = le32_to_cpu(rp->soc_ver);
+	ver->rom_ver    = le16_to_cpu(rp->rom_ver);
+	ver->patch_ver  = le16_to_cpu(rp->patch_ver);
+
+	bt_dev_dbg(hdev, "Product ID   :0x%08x", ver->product_id);
+	bt_dev_dbg(hdev, "SOC Version  :0x%08x", ver->soc_ver);
+	bt_dev_dbg(hdev, "ROM Version  :0x%04x", ver->rom_ver);
+	bt_dev_dbg(hdev, "Patch Version:0x%04x", ver->patch_ver);
+
+	if (ver->soc_ver == 0 || ver->rom_ver == 0)
+		err = -EILSEQ;
+
+out_free_skb:
+	kfree_skb(skb);
+
+	if (err > 0) {
+		bt_dev_dbg(hdev, "get BT version status error: 0x%02x",
+			   err);
+		err = -bt_to_errno(err);
+	}
+out:
+	if (err)
+		bt_dev_err(hdev, "get BT version failed: %pe", ERR_PTR(err));
+
+	return err;
+}
+
+/*
+ * qbt_edl_get_build_info - get BTC build info string
+ * @hdev: the HCI device to query
+ * @build_info: output build info string, allocated on success; caller frees
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qbt_edl_get_build_info(struct hci_dev *hdev, char **build_info)
+{
+	const struct qhci_cp_simple cp = { .sub_opcode = BEDL_GET_BUILD_INFO };
+	struct qbt_rp_get_build_info *rp;
+	struct sk_buff *skb;
+	int err;
+
+	skb = __hci_cmd_sync_ev(hdev, QBT_OP_EDL, sizeof(cp),
+				&cp, 0, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		err = PTR_ERR(skb);
+		goto out;
+	}
+
+	err = QBT_CHECK_RP_DLEN(skb, struct qbt_rp_get_build_info, cp.sub_opcode);
+	if (err)
+		goto out_free_skb;
+
+	rp = (void *)skb->data;
+	*build_info = kmemdup_nul(rp->data, rp->dlen, GFP_KERNEL);
+	if (!*build_info) {
+		err = -ENOMEM;
+		goto out_free_skb;
+	}
+
+	bt_dev_dbg(hdev, "BT build info: %s", *build_info);
+
+out_free_skb:
+	kfree_skb(skb);
+
+	if (err > 0) {
+		bt_dev_err(hdev, "get BT build info status error: 0x%02x",
+			   err);
+		err = -bt_to_errno(err);
+	}
+out:
+	if (err)
+		bt_dev_err(hdev, "get BT build info failed: %pe", ERR_PTR(err));
+
+	return err;
+}
+
+/*
+ * qbt_edl_get_board_id - get BTC board ID
+ * @hdev: the HCI device to query
+ * @board_id: output board ID, filled on success
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qbt_edl_get_board_id(struct hci_dev *hdev, u16 *board_id)
+{
+	const struct qhci_cp_simple cp = { .sub_opcode = BEDL_GET_BOARD_ID };
+	struct qbt_rp_get_board_id *rp;
+	struct sk_buff *skb;
+	int err;
+
+	skb = __hci_cmd_sync_ev(hdev, QBT_OP_EDL, sizeof(cp),
+				&cp, 0, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		err = PTR_ERR(skb);
+		goto out;
+	}
+
+	err = QBT_CHECK_RP_DLEN(skb, struct qbt_rp_get_board_id, cp.sub_opcode);
+	if (err)
+		goto out_free_skb;
+
+	rp = (void *)skb->data;
+	*board_id = be16_to_cpu(rp->board_id);
+	/* Take 0xffff as invalid board ID */
+	if (*board_id == 0xffff)
+		*board_id = 0;
+
+	bt_dev_dbg(hdev, "BT Board ID: 0x%04x", *board_id);
+
+out_free_skb:
+	kfree_skb(skb);
+
+	if (err > 0) {
+		bt_dev_dbg(hdev, "get BT board ID status error: 0x%02x",
+			   err);
+		err = -bt_to_errno(err);
+	}
+out:
+	if (err)
+		bt_dev_err(hdev, "get BT board ID failed: %pe", ERR_PTR(err));
+
+	return err;
+}
+
+/*
+ * qbt_error_fatal_cmd - trigger a controller-side fatal error for debugging
+ * @hdev: the HCI device to command
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qbt_error_fatal_cmd(struct hci_dev *hdev)
+{
+	const struct qhci_cp_simple cp = { .sub_opcode = BDBG_ERROR_FATAL_CMD };
+	int err;
+
+	err = __hci_cmd_send(hdev, QBT_OP_DEBUG, sizeof(cp), &cp);
+	if (err < 0)
+		bt_dev_err(hdev, "send error fatal cmd failed: %d", err);
+	else
+		bt_dev_info(hdev, "send error fatal cmd succeeded");
+
+	return err;
+}
+
+/*
+ * qbt_write_bda - set the controller's BD address
+ * @hdev: the HCI device to configure
+ * @bdaddr: the address to set
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qbt_write_bda(struct hci_dev *hdev, const bdaddr_t *bdaddr)
+{
+	struct qbt_cp_write_bd_addr cp;
+	int err;
+
+	/* The controller expects the address in reversed byte order. */
+	baswap(&cp.bdaddr, bdaddr);
+
+	err = __hci_cmd_sync_status(hdev, QBT_OP_WRITE_BD_ADDR,
+				    sizeof(cp), &cp, HCI_INIT_TIMEOUT);
+	if (err < 0) {
+		bt_dev_err(hdev, "set BT address failed: %pe",
+			   ERR_PTR(err));
+		return err;
+	}
+	if (err > 0) {
+		bt_dev_err(hdev, "set BT address status error: 0x%02x",
+			   err);
+		return -bt_to_errno(err);
+	}
+
+	bt_dev_info(hdev, "BT address set to %pMR", bdaddr);
+
+	return 0;
+}
+
+/*
+ * qbt_config_fw_logging - configure enhanced firmware logging
+ * @hdev: the HCI device to configure
+ * @flags: logging configuration flags, normally 1 (enable) or 0 (disable)
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qbt_config_fw_logging(struct hci_dev *hdev, u8 flags)
+{
+	const struct qbt_cp_config_fw_logging cp = {
+		.sub_opcode = BHL_ENH_ENABLE_LOG,
+		.flags      = flags,
+	};
+	struct sk_buff *skb;
+	int err;
+
+	skb = __hci_cmd_sync_ev(hdev, QBT_OP_HOST_LOG, sizeof(cp),
+				&cp, 0, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		err = PTR_ERR(skb);
+		goto out;
+	}
+
+	err = QBT_CHECK_RP_GENERIC(skb, struct qhci_rp_generic, cp.sub_opcode);
+	if (err)
+		goto out_free_skb;
+
+	bt_dev_dbg(hdev, "firmware logging configured (flags=0x%02x)", flags);
+
+out_free_skb:
+	kfree_skb(skb);
+
+	if (err > 0) {
+		bt_dev_dbg(hdev, "firmware logging config status error: 0x%02x", err);
+		err = -bt_to_errno(err);
+	}
+out:
+	if (err)
+		bt_dev_err(hdev, "firmware logging config failed: %pe", ERR_PTR(err));
+
+	return err;
+}
+
+/*
+ * ============================================================================
+ * qperi_handle_evt(): per-event handler and TX sync event sequence
+ * ============================================================================
+ */
+
+#define QPERI_EV_CLASS_PERI	0xF0
+#define QPERI_EV_TYPE_INVALID	0xff
+
+/* Pass QPERI_EV_TYPE_INVALID as event if no events to sync */
+#define DEFINE_QPERI_REQ_SPEC(specname, _opcode, _sub_opcode, _events...) \
+	const struct qhci_req_spec specname = {			\
+		.opcode     = (_opcode),				\
+		.sub_opcode = (_sub_opcode),				\
+		.event      = { _events, QPERI_EV_TYPE_INVALID },	\
+	}
+
+/*
+ * struct qperi_evt - per-event helper for qperi_handle_evt()
+ *
+ * @handler: event handler, executed before the TX sync event check.
+ *	Return: true if @skb was consumed, false otherwise.
+ *
+ * @verify_wakeup: extra verify if @skb belongs to the event sequence being synced.
+ *
+ *	Pull @skb empty if all its own checks pass and nothing is left for the
+ *	requester to verify, and the requester will get NULL as response, e.g. a
+ *	CSE, or a CCE whose payload has only status, or only status and
+ *	sub_opcode.
+ *
+ *	Return: 0 if @skb doesn't belong, 1 if it does, or a negative errno if
+ *		@skb is malformed.
+ */
+struct qperi_evt {
+	bool (*handler)(struct hci_dev *hdev, struct sk_buff *skb);
+	int (*verify_wakeup)(struct hci_dev *hdev, struct qhci_req *req,
+			     struct sk_buff *skb);
+};
+
+#define PERI_EV_CMD_STATUS	0x00
+struct peri_ev_cmd_status {
+	u8	status;
+	u8	ncmd;
+	__le16	opcode;
+} __packed;
+
+#define PERI_EV_CMD_COMPLETE	0x01
+struct peri_ev_cmd_complete {
+	u8	ncmd;
+	__le16	opcode;
+} __packed;
+
+#define PERI_EV_SUBSYS_ACTIVATE_COMPLETE	0x02
+struct peri_ev_subsys_activate_complete {
+	u8	subsys;
+	u8	action;
+} __packed;
+
+#define PERI_EV_SUBSYS_PATCH_NOTIFICATION	0x03
+struct peri_ev_subsys_patch_notification {
+	u8	status;
+	u8	subsys;
+} __packed;
+
+#define PERI_EV_CRASH_DUMP_MEMDUMP 0x04
+/* struct qhci_vse_memdump */
+
+#define PERI_EV_HARDWARE_ERROR	0x06
+struct peri_ev_hardware_error {
+	u8	code;
+} __packed;
+
+/* As qperi_evt::verify_wakeup() for PERI_EV_CMD_STATUS */
+static int peri_verify_cmd_status(struct hci_dev *hdev __maybe_unused,
+				  struct qhci_req *req, struct sk_buff *skb)
+{
+	const struct peri_ev_cmd_status *ev;
+
+	if (skb->len != sizeof(*ev))
+		return -EMSGSIZE;
+
+	ev = (const void *)skb->data;
+
+	if (le16_to_cpu(ev->opcode) != req->spec->opcode)
+		return 0;
+
+	req->req_result = ev->status;
+	if (!req->req_result)
+		skb_pull(skb, sizeof(*ev));
+
+	return 1;
+}
+
+/* CCE rp payload: [0] status, [1] sub_opcode if present, then rp-specific data if any */
+static int peri_verify_rp_payload(struct hci_dev *hdev __maybe_unused,
+				  struct qhci_req *req, struct sk_buff *skb)
+{
+	if (!skb->len)
+		return -EMSGSIZE;
+
+	if (req->spec->sub_opcode != QHCI_SUB_OPCODE_INVALID) {
+		if (skb->len < 2)
+			return -EMSGSIZE;
+
+		if (skb->data[1] != req->spec->sub_opcode)
+			return 0;
+	}
+
+	req->req_result = skb->data[0];
+	if (!req->req_result && skb->len <= 2)
+		skb_pull(skb, skb->len);
+
+	return 1;
+}
+
+/* As qperi_evt::verify_wakeup() for PERI_EV_CMD_COMPLETE */
+static int peri_verify_cmd_complete(struct hci_dev *hdev, struct qhci_req *req,
+				    struct sk_buff *skb)
+{
+	const struct peri_ev_cmd_complete *ev;
+	u16 opcode;
+
+	ev = skb_pull_data(skb, sizeof(*ev));
+	if (!ev)
+		return -EMSGSIZE;
+
+	opcode = le16_to_cpu(ev->opcode);
+	if (opcode != req->spec->opcode)
+		return 0;
+
+	hci_skb_opcode(skb) = opcode;
+
+	return peri_verify_rp_payload(hdev, req, skb);
+}
+
+/* As qperi_evt::verify_wakeup() for PERI_EV_SUBSYS_ACTIVATE_COMPLETE */
+static int peri_verify_subsys_activate_complete(struct hci_dev *hdev __maybe_unused,
+						struct qhci_req *req __maybe_unused,
+						struct sk_buff *skb)
+{
+	if (skb->len != sizeof(struct peri_ev_subsys_activate_complete))
+		return -EMSGSIZE;
+
+	return 1;
+}
+
+/* As qperi_evt::verify_wakeup() for PERI_EV_SUBSYS_PATCH_NOTIFICATION */
+static int peri_verify_subsys_patch_notification(struct hci_dev *hdev __maybe_unused,
+						 struct qhci_req *req,
+						 struct sk_buff *skb)
+{
+	const struct peri_ev_subsys_patch_notification *ev;
+
+	if (skb->len != sizeof(*ev))
+		return -EMSGSIZE;
+
+	ev = (const void *)skb->data;
+	req->req_result = ev->status;
+
+	return 1;
+}
+
+/* As qperi_evt::handler() for PERI_EV_CRASH_DUMP_MEMDUMP */
+static bool peri_crash_dump_memdump(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	btqcom_handle_memdump(hdev, skb);
+
+	return true;
+}
+
+/* As qperi_evt::handler() for PERI_EV_HARDWARE_ERROR */
+static bool peri_hardware_error(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	const struct peri_ev_hardware_error *ev;
+	u8 code = HCI_ERROR_UNSPECIFIED;
+
+	ev = skb_pull_data(skb, sizeof(*ev));
+	if (ev)
+		code = ev->code;
+	else
+		bt_dev_err(hdev, "malformed PERI hardware error event");
+
+	set_bit(QBT_MISC_HWERR_PERI, &qbt_data->misc_flags);
+	__hci_reset_dev(hdev, code);
+
+	return false;
+}
+
+static const struct qperi_evt qperi_evt_table[] = {
+	/* [0x00 = PERI_EV_CMD_STATUS] */
+	[PERI_EV_CMD_STATUS] = {
+		.verify_wakeup = peri_verify_cmd_status,
+	},
+	/* [0x01 = PERI_EV_CMD_COMPLETE] */
+	[PERI_EV_CMD_COMPLETE] = {
+		.verify_wakeup = peri_verify_cmd_complete,
+	},
+	/* [0x02 = PERI_EV_SUBSYS_ACTIVATE_COMPLETE] */
+	[PERI_EV_SUBSYS_ACTIVATE_COMPLETE] = {
+		.verify_wakeup = peri_verify_subsys_activate_complete,
+	},
+	/* [0x03 = PERI_EV_SUBSYS_PATCH_NOTIFICATION] */
+	[PERI_EV_SUBSYS_PATCH_NOTIFICATION] = {
+		.verify_wakeup = peri_verify_subsys_patch_notification,
+	},
+	/* [0x04 = PERI_EV_CRASH_DUMP_MEMDUMP] */
+	[PERI_EV_CRASH_DUMP_MEMDUMP] = {
+		.handler = peri_crash_dump_memdump,
+	},
+	/* [0x06 = PERI_EV_HARDWARE_ERROR] */
+	[PERI_EV_HARDWARE_ERROR] = {
+		.handler = peri_hardware_error,
+	},
+};
+
+/*
+ * qperi_try_wakeup - check if event @skb with type @ev_type can wake up the requester
+ * @hdev: the HCI device the event comes from
+ * @ev_type: the event type
+ * @skb: the event payload, or NULL on skb clone failure
+ *
+ * Note: the caller only cares about the return value rather than the wakeup result.
+ *
+ * Return: true if @skb was consumed, false otherwise.
+ */
+static bool qperi_try_wakeup(struct hci_dev *hdev, u8 ev_type,
+			     struct sk_buff *skb)
+{
+	const struct qperi_evt *evt = &qperi_evt_table[ev_type];
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct sk_buff *req_rsp = skb;
+	struct qhci_req *req;
+	int res;
+
+	/* fast path */
+	if (READ_ONCE(qbt_data->req_state) != QHCI_REQ_PEND)
+		return false;
+
+	guard(spinlock)(&qbt_data->req_spinlock);
+
+	if (qbt_data->req_state != QHCI_REQ_PEND)
+		return false;
+
+	req = qbt_data->req;
+	if (WARN_ON_ONCE(!req) || WARN_ON_ONCE(!req->spec))
+		return false;
+
+	if (req->spec->event[req->event_idx] != ev_type)
+		return false;
+
+	/* skb clone failure */
+	if (!skb) {
+		req->req_result = -ENOMEM;
+		goto out_wakeup;
+	}
+
+	/* malformed frame */
+	if (!skb->len) {
+		req->req_result = -EMSGSIZE;
+		goto out_wakeup;
+	}
+
+	req->req_result = 0;
+	if (evt->verify_wakeup) {
+		res = evt->verify_wakeup(hdev, req, skb);
+		if (!res)
+			return false;
+
+		if (res < 0) {
+			req->req_result = res;
+			goto out_wakeup;
+		}
+
+		/* HCI status error */
+		if (req->req_result)
+			goto out_wakeup;
+
+		/* nothing left for the requester to verify */
+		if (!req_rsp->len)
+			req_rsp = NULL;
+	}
+
+	if (req->spec->event[req->event_idx + 1] != QPERI_EV_TYPE_INVALID) {
+		req->event_idx++;
+		return false;
+	}
+
+out_wakeup:
+	if (req->req_result < 0)
+		req_rsp = NULL;
+
+	req->req_rsp = req_rsp;
+	if (req->req_result)
+		bt_dev_err(hdev, "PERI cmd opcode(0x%04x) sub_opcode(0x%02x) failed on ev_type(0x%02x): %d",
+			   req->spec->opcode, req->spec->sub_opcode, ev_type, req->req_result);
+	qbt_data->req_state = QHCI_REQ_DONE;
+	wake_up_interruptible(&qbt_data->req_wait_q);
+	/* test if the requester now owns @skb */
+	return req_rsp == skb;
+}
+
+/*
+ * __qperi_tx_sync_evt - send a frame and sync its event sequence
+ * @hdev: the HCI device
+ * @iter: the frame to send, NULL to sync only
+ * @spec: the request spec to sync, NULL to send only
+ * @timeout: timeout in jiffies for the sync
+ * @evt_idx: index of the waking event
+ * @status: HCI status code if it has one, 0 otherwise
+ *
+ * Requires the caller holds qbt_data->req_mutex.
+ *
+ * Return: ERR_PTR() on failure,
+ *	NULL on success for send-only or nothing left for the caller to verify,
+ *	the waking event otherwise.
+ */
+static struct sk_buff *__qperi_tx_sync_evt(struct hci_dev *hdev,
+					   struct iov_iter *iter,
+					   const struct qhci_req_spec *spec,
+					   unsigned int timeout,
+					   u8 *evt_idx, u8 *status)
+{
+	struct qhci_req req = { .pkt_type = QPERI_EVENT_PKT, .spec = spec };
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct sk_buff *ret = NULL;
+	int res = 0, wait_res = 0;
+	int req_state;
+
+	if (!iter && !spec)
+		return ERR_PTR(-EINVAL);
+
+	/* send only, no event to sync */
+	if (!spec) {
+		res = hci_send_vendor_frame(hdev, iter);
+		if (res < 0) {
+			bt_dev_err(hdev, "send-only PERI frame failed: %pe", ERR_PTR(res));
+			return ERR_PTR(res);
+		}
+		return NULL;
+	}
+
+	scoped_guard(spinlock, &qbt_data->req_spinlock) {
+		qbt_data->req = &req;
+		qbt_data->req_state = QHCI_REQ_PEND;
+	}
+
+	if (iter)
+		res = hci_send_vendor_frame(hdev, iter);
+
+	if (!res)
+		wait_res = wait_event_interruptible_timeout(qbt_data->req_wait_q,
+							    READ_ONCE(qbt_data->req_state) !=
+							    QHCI_REQ_PEND,
+							    timeout);
+
+	scoped_guard(spinlock, &qbt_data->req_spinlock) {
+		req_state = qbt_data->req_state;
+		qbt_data->req = NULL;
+		qbt_data->req_state = QHCI_REQ_DONE;
+	}
+
+	if (res >= 0) {
+		switch (req_state) {
+		case QHCI_REQ_DONE:
+			res = req.req_result;
+			break;
+		case QHCI_REQ_CANCELED:
+			if (req.req_result < 0)
+				bt_dev_info(hdev, "PERI cmd opcode(0x%04x) sub_opcode(0x%02x) canceled: %pe",
+					    spec->opcode, spec->sub_opcode,
+					    ERR_PTR(req.req_result));
+			else
+				bt_dev_info(hdev, "PERI cmd opcode(0x%04x) sub_opcode(0x%02x) canceled",
+					    spec->opcode, spec->sub_opcode);
+			res = req.req_result < 0 ? req.req_result : -ECANCELED;
+			break;
+		case QHCI_REQ_PEND:
+		default:
+			res = wait_res < 0 ? -EINTR : -ETIMEDOUT;
+			break;
+		}
+	}
+
+	ret = req.req_rsp;
+	if (res < 0) {
+		/* log unless qperi_try_wakeup() already did */
+		if (res != req.req_result)
+			bt_dev_err(hdev, "PERI cmd opcode(0x%04x) sub_opcode(0x%02x) failed: %pe",
+				   spec->opcode, spec->sub_opcode, ERR_PTR(res));
+
+		if (WARN_ON_ONCE(req.req_rsp))
+			kfree_skb(req.req_rsp);
+
+		ret = ERR_PTR(res);
+	}
+
+	if (evt_idx)
+		*evt_idx = req.event_idx;
+	if (status)
+		*status = (u8)req.req_result;
+	return ret;
+}
+
+/* synchronously cancel pending request with error code @err */
+static void qperi_tx_sync_cancel_sync(struct hci_dev *hdev, int err)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct qhci_req *req;
+
+	guard(spinlock)(&qbt_data->req_spinlock);
+
+	if (qbt_data->req_state != QHCI_REQ_PEND)
+		return;
+
+	req = qbt_data->req;
+	if (WARN_ON_ONCE(!req))
+		return;
+
+	req->req_result = err;
+	qbt_data->req = NULL;
+	qbt_data->req_state = QHCI_REQ_CANCELED;
+
+	wake_up_interruptible(&qbt_data->req_wait_q);
+}
+
+/*
+ * __qperi_cmd_sync_evt - send a command and sync its event sequence
+ * @hdev: the HCI device
+ * @opcode: command opcode, QHCI_OPCODE_INVALID to sync only
+ * @plen: length of @cp
+ * @cp: the command payload
+ * @spec: the request spec to sync, NULL to send only
+ * @evt_idx: index of the waking event
+ * @status: HCI status code if it has one, 0 otherwise
+ *
+ * Requires the caller holds qbt_data->req_mutex.
+ *
+ * Return: ERR_PTR() on failure,
+ *	NULL on success for send-only or nothing left for the caller to verify,
+ *	the waking event otherwise.
+ */
+static struct sk_buff *__qperi_cmd_sync_evt(struct hci_dev *hdev,
+					    u16 opcode, u32 plen, const void *cp,
+					    const struct qhci_req_spec *spec,
+					    u8 *evt_idx, u8 *status)
+{
+	u8 pkt_type = QPERI_COMMAND_PKT;
+	struct qperi_command_hdr cmd_hdr = {
+		.host_id = QHCI_HOST_ID_BT,
+		.opcode  = cpu_to_le16(opcode),
+		.plen    = (u8)plen,
+	};
+	struct kvec cmd_kv[] = {
+		{ .iov_base = &pkt_type,       .iov_len = 1 },
+		{ .iov_base = &cmd_hdr,        .iov_len = sizeof(cmd_hdr) },
+		{ .iov_base = (void *)cp,      .iov_len = plen },
+	};
+	struct iov_iter iter;
+
+	/* no command to send, sync only */
+	if (opcode == QHCI_OPCODE_INVALID)
+		return __qperi_tx_sync_evt(hdev, NULL, spec, HCI_INIT_TIMEOUT, evt_idx, status);
+
+	iov_iter_kvec(&iter, ITER_SOURCE, cmd_kv, plen ? 3 : 2,
+		      1 + sizeof(cmd_hdr) + plen);
+
+	return __qperi_tx_sync_evt(hdev, &iter, spec, HCI_INIT_TIMEOUT, evt_idx, status);
+}
+
+static inline struct sk_buff *qperi_cmd_sync_evt(struct hci_dev *hdev,
+						 u16 opcode, u32 plen, const void *cp,
+						 const struct qhci_req_spec *spec,
+						 u8 *evt_idx, u8 *status)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+
+	guard(mutex)(&qbt_data->req_mutex);
+
+	return __qperi_cmd_sync_evt(hdev, opcode, plen, cp, spec, evt_idx, status);
+}
+
+/*
+ * send a command and sync an event with type @ev_type;
+ * @cp's first byte is sub_opcode for all commands so far
+ *
+ * Requires the caller holds qbt_data->req_mutex.
+ */
+static struct sk_buff *__qperi_cmd_sync_one_evt(struct hci_dev *hdev,
+						u16 opcode, u32 plen, const void *cp,
+						u8 ev_type, u8 *status)
+{
+	u8 sub_opcode = plen ? *(const u8 *)cp : QHCI_SUB_OPCODE_INVALID;
+	DEFINE_QPERI_REQ_SPEC(spec, opcode, sub_opcode, ev_type);
+
+	return __qperi_cmd_sync_evt(hdev, opcode, plen, cp, &spec, NULL, status);
+}
+
+static inline struct sk_buff *qperi_cmd_sync_one_evt(struct hci_dev *hdev,
+						     u16 opcode, u32 plen, const void *cp,
+						     u8 ev_type, u8 *status)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+
+	guard(mutex)(&qbt_data->req_mutex);
+
+	return __qperi_cmd_sync_one_evt(hdev, opcode, plen, cp, ev_type, status);
+}
+
+/*
+ * only wait for an unsolicited event with type @ev_type
+ *
+ * Requires the caller holds qbt_data->req_mutex.
+ */
+static struct sk_buff __maybe_unused *__qperi_wait_one_evt(struct hci_dev *hdev,
+							   u8 ev_type, u8 *status)
+{
+	DEFINE_QPERI_REQ_SPEC(spec, QHCI_OPCODE_INVALID, QHCI_SUB_OPCODE_INVALID, ev_type);
+
+	return __qperi_tx_sync_evt(hdev, NULL, &spec, HCI_INIT_TIMEOUT, NULL, status);
+}
+
+/*
+ * handle a received PERI event @skb, called from hdev->recv_vendor_pkt();
+ * generic processing first, then the TX sync event, without interfering
+ */
+static void qperi_handle_evt(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	const struct qhci_vse_comm *vse_comm;
+	const struct qperi_event_hdr *hdr;
+	const struct qperi_evt *evt_entry;
+	/* for qperi_try_wakeup() call below */
+	struct sk_buff *orig_skb = NULL;
+	u8 ev_type;
+	int res;
+
+	ev_type = QPERI_EV_TYPE_INVALID;
+	res = -EBADMSG;
+	hdr = skb_pull_data(skb, QPERI_EVENT_HDR_SIZE);
+	if (!hdr || hdr->evt != HCI_EV_VENDOR)
+		goto out_free_skb;
+
+	res = -EILSEQ;
+	vse_comm = skb_pull_data(skb, QHCI_VSE_COMM_SIZE);
+	if (!vse_comm || vse_comm->ev_class != QPERI_EV_CLASS_PERI)
+		goto out_free_skb;
+
+	ev_type = vse_comm->ev_type;
+	res = -ENOENT;
+	if (ev_type >= ARRAY_SIZE(qperi_evt_table))
+		goto out_free_skb;
+
+	res = 0;
+	hci_skb_event(skb) = ev_type;
+	evt_entry = &qperi_evt_table[ev_type];
+
+	if (evt_entry->handler) {
+		const struct qhci_req *req;
+		bool may_wakeup;
+
+		spin_lock(&qbt_data->req_spinlock);
+		req = qbt_data->req;
+		may_wakeup = qbt_data->req_state == QHCI_REQ_PEND &&
+			req && req->spec &&
+			req->spec->event[req->event_idx] == ev_type;
+		spin_unlock(&qbt_data->req_spinlock);
+
+		if (may_wakeup) {
+			orig_skb = skb_clone(skb, GFP_KERNEL);
+			if (!orig_skb)
+				res = -ENOMEM;
+		}
+
+		if (evt_entry->handler(hdev, skb))
+			skb = NULL;
+	} else {
+		orig_skb = skb;
+		skb = NULL;
+	}
+
+	if (qperi_try_wakeup(hdev, ev_type, orig_skb))
+		orig_skb = NULL;
+
+out_free_skb:
+	kfree_skb(orig_skb);
+	kfree_skb(skb);
+	if (res < 0)
+		bt_dev_err(hdev, "fails to handle PERI event with type 0x%02x: %pe",
+			   ev_type, ERR_PTR(res));
+}
+
+/*
+ * ============================================================================
+ * PERI-HCI vendor-specific command/response
+ * ============================================================================
+ */
+
+/* reserved PERI ACL handle for enhanced logging */
+#define QPERI_HANDLE_ENHANCED_LOGGING 0xEC0
+/* reserved PERI ACL handle for memdump */
+#define QPERI_HANDLE_MEMDUMP 0xEC1
+
+/*
+ * Unless otherwise noted, for the VSCs below:
+ * - command payload: struct qhci_cp_simple
+ * - response payload: struct qhci_rp_generic
+ *
+ * Grouped by { opcode, sub_opcode, cp, rp }.
+ */
+
+/* PERI EDL opcode and its sub-opcodes */
+#define QPERI_OP_EDL		0xFFF0
+
+#define PEDL_GET_BUILD_INFO	0x09
+struct qperi_cp_generic {
+	u8	sub_opcode;
+	u8	subsys;
+} __packed;
+struct qperi_rp_get_build_info {
+	u8	status;
+	u8	sub_opcode;
+	u8	subsys;
+	u8	dlen;
+	u8	data[];
+} __packed;
+
+/* PERI Generic opcode and its sub-opcodes */
+#define QPERI_OP_GENERIC	0xFFF1
+
+#define PGEN_PERI_RESET		0x03
+/* struct qhci_rp_generic */
+/* struct peri_ev_subsys_patch_notification */
+
+#define PGEN_INITIATE_BT_CRASH	0x05
+/* struct peri_ev_cmd_status */
+
+#define __QPERI_CP_INIT_SUBSYS_0(...)
+#define __QPERI_CP_INIT_SUBSYS_1(_subsys, ...)	.subsys = (_subsys),
+
+#define QPERI_CP_INITIALIZER(_sub_opcode, _subsys...) {			\
+	.sub_opcode = (_sub_opcode),					\
+	CONCATENATE(__QPERI_CP_INIT_SUBSYS_, COUNT_ARGS(_subsys))(_subsys)	\
+}
+
+/*
+ * QPERI_CHECK_RP_LEN - check rp @_skb's length and its @dlen
+ * @_skb: the rp to check
+ * @_rp_type: the type to interpret @_skb as
+ *
+ * It is safe to evaluate @_skb more than once for its usages.
+ *
+ * Returns a negative errno on failure, or 0 on success.
+ */
+#define QPERI_CHECK_RP_LEN(_skb, _rp_type)				\
+({									\
+	_rp_type *_rp_ptr;						\
+	int _err = 0;							\
+	do {								\
+		if (!(_skb) || (_skb)->len < sizeof(_rp_type)) {	\
+			_err = -EBADMSG;				\
+			break;						\
+		}							\
+		_rp_ptr = (void *)(_skb)->data;				\
+		if ((_skb)->len != offsetofend(_rp_type, dlen) +	\
+				   _rp_ptr->dlen) {			\
+			_err = -EMSGSIZE;				\
+			break;						\
+		}							\
+	} while (0);							\
+	_err;								\
+})
+
+/*
+ * qperi_get_subsys_build_info - get subsystem build info string
+ * @hdev: the HCI device to query
+ * @subsys: the subsystem to query
+ * @build_info: output build info string, allocated on success; caller frees
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qperi_get_subsys_build_info(struct hci_dev *hdev, u8 subsys,
+				       char **build_info)
+{
+	struct qperi_cp_generic cp = QPERI_CP_INITIALIZER(PEDL_GET_BUILD_INFO, subsys);
+	struct qperi_rp_get_build_info *rp;
+	struct sk_buff *skb;
+	u8 status = 0;
+	int ret;
+
+	skb = qperi_cmd_sync_one_evt(hdev, QPERI_OP_EDL, sizeof(cp), &cp,
+				     PERI_EV_CMD_COMPLETE, &status);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	if (status) {
+		ret = -bt_to_errno(status);
+		goto out_free_skb;
+	}
+
+	ret = QPERI_CHECK_RP_LEN(skb, struct qperi_rp_get_build_info);
+	if (ret)
+		goto out_free_skb;
+	rp = (void *)skb->data;
+
+	if (rp->subsys != subsys) {
+		ret = -EILSEQ;
+		goto out_free_skb;
+	}
+
+	*build_info = kmemdup_nul(rp->data, rp->dlen, GFP_KERNEL);
+	ret = *build_info ? 0 : -ENOMEM;
+
+out_free_skb:
+	kfree_skb(skb);
+
+	if (ret)
+		bt_dev_err(hdev, "get %s build info failed: %pe",
+			   qhci_subsys_name(subsys), ERR_PTR(ret));
+	else
+		bt_dev_dbg(hdev, "%s build info: %s",
+			   qhci_subsys_name(subsys), *build_info);
+
+	return ret;
+}
+
+/*
+ * qperi_hci_reset - reset PERI HCI
+ * @hdev: the HCI device to reset
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qperi_hci_reset(struct hci_dev *hdev)
+{
+	DEFINE_QPERI_REQ_SPEC(spec, QPERI_OP_GENERIC, PGEN_PERI_RESET,
+			      PERI_EV_CMD_COMPLETE, PERI_EV_SUBSYS_PATCH_NOTIFICATION);
+	struct qhci_cp_simple cp = QPERI_CP_INITIALIZER(PGEN_PERI_RESET);
+	struct sk_buff *skb;
+	u8 evt_idx = 0;
+	u8 status = 0;
+	int ret = 0;
+
+	skb = qperi_cmd_sync_evt(hdev, QPERI_OP_GENERIC, sizeof(cp), &cp,
+				 &spec, &evt_idx, &status);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	if (status) {
+		ret = -bt_to_errno(status);
+	} else if (evt_idx == 1) {
+		const struct peri_ev_subsys_patch_notification *ev;
+
+		ev = skb_pull_data(skb, sizeof(*ev));
+		if (ev->subsys != QHCI_SUBSYS_PERI) {
+			ret = -EILSEQ;
+			bt_dev_err(hdev, "PERI HCI reset via QHCI failed: wrong subsys %d",
+				   ev->subsys);
+		}
+	}
+
+	kfree_skb(skb);
+
+	if (!ret)
+		bt_dev_info(hdev, "PERI HCI reset via QHCI succeeded");
+
+	return ret;
+}
+
+/*
+ * qperi_initiate_bt_crash - initiate a BT subsystem crash
+ * @hdev: the HCI device
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int qperi_initiate_bt_crash(struct hci_dev *hdev)
+{
+	struct qhci_cp_simple cp = QPERI_CP_INITIALIZER(PGEN_INITIATE_BT_CRASH);
+	struct sk_buff *skb;
+	u8 status = 0;
+	int ret = 0;
+
+	skb = qperi_cmd_sync_one_evt(hdev, QPERI_OP_GENERIC, sizeof(cp), &cp,
+				     PERI_EV_CMD_STATUS, &status);
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	if (skb) {
+		ret = -bt_to_errno(status);
+		kfree_skb(skb);
+	}
+
+	if (!ret)
+		bt_dev_info(hdev, "PERI initiate BT crash succeeded");
+
+	return ret;
+}
+
+/*
+ * ============================================================================
+ * hdev callbacks
+ * ============================================================================
+ */
+
+/* test if a HW or SW reset is available */
+static inline bool btqcom_has_btc_reset(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+
+	return qbt_data->flags & (QBT_FLAG_HW_RESET | QBT_FLAG_SW_RESET);
+}
+
+/*
+ * btqcom_set_bdaddr - set BD_ADDR
+ * @hdev: the HCI device
+ * @bdaddr: the BD_ADDR to set
+ *
+ * Implements hdev->set_bdaddr().
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btqcom_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
+{
+	int ret;
+
+	ret = __hci_reset_sync(hdev);
+	if (ret) {
+		bt_dev_err(hdev, "HCI reset before setting BD_ADDR failed: %pe",
+			   ERR_PTR(ret));
+		return ret;
+	}
+
+	ret = qbt_write_bda(hdev, bdaddr);
+
+	return ret;
+}
+
+/*
+ * btqcom_handle_ev_vendor - handle @skb if interested
+ * @hdev: the HCI device @skb comes from
+ * @skb: the VSE payload
+ *
+ * Implements hdev->handle_ev_vendor().
+ *
+ * Return: true if @skb was handled, false otherwise.
+ */
+static bool btqcom_handle_ev_vendor(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	const struct qhci_vse_comm *vse_comm;
+	u16 vse_id;
+
+	if (skb->len < QHCI_VSE_COMM_SIZE)
+		return false;
+
+	vse_comm = (const void *)skb->data;
+	vse_id = vse_comm->ev_class << 8 | vse_comm->ev_type;
+
+	switch (vse_id) {
+	case (QBT_EV_CLASS_DATALOG << 8) | QBT_EV_TYPE_MEMDUMP:
+		skb_pull_data(skb, QHCI_VSE_COMM_SIZE);
+		btqcom_handle_memdump(hdev, skb_get(skb));
+		return true;
+	default:
+		return false;
+	}
+}
+
+/* handle a received BT vendor ACL frame */
+static void handle_acl_vendor(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	const struct qhci_vse_comm *vse_comm;
+	const struct hci_event_hdr *evt_hdr;
+	u16 vse_id;
+
+	if (hci_acl_handle(skb) != QBT_HANDLE_MEMDUMP)
+		goto out_free_skb;
+
+	skb_pull(skb, HCI_ACL_HDR_SIZE);
+
+	evt_hdr = skb_pull_data(skb, HCI_EVENT_HDR_SIZE);
+	if (!evt_hdr || evt_hdr->evt != HCI_EV_VENDOR)
+		goto out_free_skb;
+
+	vse_comm = skb_pull_data(skb, QHCI_VSE_COMM_SIZE);
+	if (!vse_comm)
+		goto out_free_skb;
+
+	vse_id = vse_comm->ev_class << 8 | vse_comm->ev_type;
+
+	switch (vse_id) {
+	case (QBT_EV_CLASS_DATALOG << 8) | QBT_EV_TYPE_MEMDUMP:
+		btqcom_handle_memdump(hdev, skb);
+		return;
+	default:
+		break;
+	}
+
+out_free_skb:
+	kfree_skb(skb);
+}
+
+/* handle a received PERI ACL frame */
+static void qperi_handle_acl(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	const struct hci_event_hdr *bt_evt_hdr;
+	const struct qhci_vse_comm *vse_comm;
+	u8 ev_type;
+
+	if (qperi_acl_handle(skb) != QPERI_HANDLE_MEMDUMP)
+		goto out_free_skb;
+
+	skb_pull(skb, QPERI_ACL_HDR_SIZE);
+
+	bt_evt_hdr = skb_pull_data(skb, HCI_EVENT_HDR_SIZE);
+	if (!bt_evt_hdr || bt_evt_hdr->evt != HCI_EV_VENDOR)
+		goto out_free_skb;
+
+	vse_comm = skb_pull_data(skb, sizeof(*vse_comm));
+	if (!vse_comm || vse_comm->ev_class != QPERI_EV_CLASS_PERI)
+		goto out_free_skb;
+
+	ev_type = vse_comm->ev_type;
+	switch (ev_type) {
+	case PERI_EV_CRASH_DUMP_MEMDUMP:
+		btqcom_handle_memdump(hdev, skb);
+		return;
+	default:
+		break;
+	}
+
+out_free_skb:
+	kfree_skb(skb);
+}
+
+/* Implements hdev->recv_vendor_pkt(). */
+static void btqcom_recv_vendor_pkt(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	hci_skb_pkt_type(skb) = *(const u8 *)skb_pull_data(skb, 1);
+
+	switch (hci_skb_pkt_type(skb)) {
+	case QPERI_EVENT_PKT:
+		qperi_handle_evt(hdev, skb);
+		break;
+	case QPERI_ACLDATA_PKT:
+		qperi_handle_acl(hdev, skb);
+		break;
+	case HCI_ACLDATA_PKT:
+		handle_acl_vendor(hdev, skb);
+		break;
+	default:
+		bt_dev_err(hdev, "unexpected vendor HCI frame with pkt_type 0x%02x",
+			   hci_skb_pkt_type(skb));
+		kfree_skb(skb);
+		break;
+	}
+}
+
+/* Implements hdev->dump.coredump() for hci_devcd_register(). */
+static void btusb_qcom_trigger_memdump(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+
+	if (qbt_data->category == QBTC_CAT_MSUBSYS)
+		qperi_initiate_bt_crash(hdev);
+	else
+		qbt_error_fatal_cmd(hdev);
+}
+
+/* return 0 on success, or a negative errno on failure */
+static int btusb_qcom_deactivate_msubsys_bt(struct hci_dev *hdev)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	int ret;
+
+	ret = usb_autopm_get_interface(xport_data->intf);
+	if (ret) {
+		bt_dev_err(hdev, "get autopm for deactivate BT failed: %pe", ERR_PTR(ret));
+		return ret;
+	}
+
+	ret = qdfu_activate_remote_btss(hdev, false, 100 * 1000);
+
+	usb_autopm_put_interface(xport_data->intf);
+
+	if (!ret)
+		bt_dev_info(hdev, "deactivate BT succeeded");
+
+	return ret;
+}
+
+/* return 0 on success, or a negative errno on failure */
+static int btusb_qcom_sw_reset(struct hci_dev *hdev)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	int ret = -EOPNOTSUPP;
+
+	if (!(qbt_data->flags & QBT_FLAG_SW_RESET))
+		return ret;
+
+	ret = usb_autopm_get_interface(xport_data->intf);
+	if (ret) {
+		bt_dev_err(hdev, "get autopm for SW reset failed: %pe", ERR_PTR(ret));
+		return ret;
+	}
+
+	ret = qdfu_sw_reset(hdev);
+
+	usb_autopm_put_interface(xport_data->intf);
+
+	return ret;
+}
+
+#define QBT_RESET_TYPE_SYNC	"sync"
+#define QBT_RESET_TYPE_ASYNC	"async"
+#define QBT_RESET_TYPE_DIRECT	"direct"
+
+/*
+ * btusb_do_reset_work - reset the BTC
+ * @hdev: the HCI device
+ * @type: QBT_RESET_TYPE_* above, for logging
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_do_reset_work(struct hci_dev *hdev, void *type)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	struct gpio_desc *reset_gpio = xport_data->reset_gpio;
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	int ret;
+
+	bt_dev_dbg(hdev, "%s reset: misc_flags(0x%lx)", (char *)type,
+		   READ_ONCE(qbt_data->misc_flags));
+	bt_dev_dbg(hdev, "md_state: %s, md_submit_err: %d",
+		   hci_devcd_state_name(READ_ONCE(qbt_data->md_state)),
+		   READ_ONCE(qbt_data->md_submit_err));
+
+	if (test_and_set_bit(QBT_MISC_RESET_ACTIVE, &qbt_data->misc_flags)) {
+		bt_dev_info(hdev, "reset already in progress");
+		return 0;
+	}
+
+	if (xport_data->prepare_reset)
+		xport_data->prepare_reset(hdev);
+
+	if (reset_gpio) {
+		bt_dev_info(hdev, "hardware reset");
+		gpiod_set_value_cansleep(reset_gpio, 0);
+		fsleep(200 * 1000);
+		gpiod_set_value_cansleep(reset_gpio, 1);
+		return 0;
+	}
+
+	ret = btusb_qcom_sw_reset(hdev);
+	if (!ret)
+		return 0;
+
+	ret = usb_autopm_get_interface(xport_data->intf);
+	if (ret) {
+		bt_dev_err(hdev, "reset: autopm get failed: %pe", ERR_PTR(ret));
+		clear_bit(QBT_MISC_RESET_ACTIVE, &qbt_data->misc_flags);
+		return ret;
+	}
+	usb_autopm_put_interface_no_suspend(xport_data->intf);
+
+	bt_dev_info(hdev, "usb reset");
+	/* Clear it here since usb reset isn't guaranteed to succeed. */
+	clear_bit(QBT_MISC_RESET_ACTIVE, &qbt_data->misc_flags);
+	usb_queue_reset_device(xport_data->intf);
+
+	return 0;
+}
+
+static void btusb_qcom_reset_sync(struct hci_dev *hdev)
+{
+	int res;
+
+	res = hci_cmd_sync_queue_once(hdev, btusb_do_reset_work, QBT_RESET_TYPE_SYNC, NULL);
+	if (res)
+		bt_dev_dbg(hdev, "queue reset work failed: %pe", ERR_PTR(res));
+	if (!res || res == -EEXIST || res == -ENODEV)
+		return;
+
+	btusb_do_reset_work(hdev, QBT_RESET_TYPE_DIRECT);
+}
+
+static void btqcom_reset_async(struct hci_dev *hdev, unsigned int delay_ms)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+
+	set_bit(QBT_WORK_RESET_HDEV, &qbt_data->work_flags);
+	schedule_delayed_work(&qbt_data->dwork, msecs_to_jiffies(delay_ms));
+}
+
+/*
+ * btusb_do_cmd_timeout_work - HCI cmd sync work function to handle command timeout
+ * @hdev: the HCI device
+ * @data: unused
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_do_cmd_timeout_work(struct hci_dev *hdev,
+				     void *data __maybe_unused)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	unsigned int wait_ms = 0;
+	int ret = 0;
+
+	if (!(qbt_data->flags & QBT_FLAG_MEMDUMP) ||
+	    !(qbt_data->flags & QBT_FLAG_CMD_TIMEOUT_MEMDUMP))
+		goto out_reset;
+
+	switch (qbt_data->category) {
+	case QBTC_CAT_LEGACY:
+		return -EOPNOTSUPP;
+	case QBTC_CAT_UNIFIED:
+		wait_ms = 800;
+		ret = qbt_error_fatal_cmd(hdev);
+		break;
+	case QBTC_CAT_MSUBSYS:
+		wait_ms = 1200;
+		ret = qperi_initiate_bt_crash(hdev);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (ret)
+		goto out_reset;
+
+	/*
+	 * Command timeout is solved by self-recovery after memdump.
+	 * Testing CMD_TIMEOUT in this wait is only to abandon it
+	 * immediately if required, not a sign the timeout is resolved.
+	 */
+	ret = wait_var_event_timeout(&qbt_data->misc_flags,
+				     test_bit(QBT_MISC_MEMDUMP_INCOMING,
+					      &qbt_data->misc_flags) ||
+				     !test_bit(QBT_MISC_CMD_TIMEOUT,
+					       &qbt_data->misc_flags),
+				     msecs_to_jiffies(wait_ms));
+	if (!test_bit(QBT_MISC_CMD_TIMEOUT, &qbt_data->misc_flags)) {
+		bt_dev_dbg(hdev, "Flag CMD_TIMEOUT cleared");
+		return 0;
+	}
+	if (ret) {
+		bt_dev_dbg(hdev, "trigger memdump on command timeout succeeded");
+		return 0;
+	}
+	bt_dev_err(hdev, "trigger memdump on command timeout failed");
+
+out_reset:
+	btusb_do_reset_work(hdev, QBT_RESET_TYPE_SYNC);
+
+	return 0;
+}
+
+/* Implements hdev->reset(). */
+static void btusb_qcom_reset(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	bool has_cmd_timeout;
+	int res;
+
+	has_cmd_timeout = current_work() == &hdev->cmd_timer.work;
+	bt_dev_info(hdev, "reset triggered by %s",
+		    has_cmd_timeout ? "command timeout" : "user");
+
+	if (test_bit(QBT_MISC_MEMDUMP_PERI, &qbt_data->misc_flags) ||
+	    test_bit(QBT_MISC_MEMDUMP_BT, &qbt_data->misc_flags) ||
+	    test_bit(QBT_MISC_MEMDUMP_TMEL, &qbt_data->misc_flags)) {
+		bt_dev_info(hdev, "reset will happen after memdump");
+		return;
+	}
+
+	if (!has_cmd_timeout) {
+		btusb_qcom_reset_sync(hdev);
+		return;
+	}
+
+	if (test_and_set_bit(QBT_MISC_CMD_TIMEOUT, &qbt_data->misc_flags)) {
+		bt_dev_info(hdev, "handling command timeout is in progress");
+		return;
+	}
+
+	res = hci_cmd_sync_queue_once(hdev, btusb_do_cmd_timeout_work, NULL, NULL);
+	if (res)
+		bt_dev_dbg(hdev, "queue command timeout work failed: %pe", ERR_PTR(res));
+	if (!res || res == -EEXIST || res == -ENODEV)
+		return;
+
+	btqcom_reset_async(hdev, 0);
+}
+
+/* Implements hdev->hw_error(). */
+static void btqcom_hw_error(struct hci_dev *hdev, u8 code)
+{
+	const char *subsys_name = qhci_subsys_name(QHCI_SUBSYS_PERI);
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	int res;
+
+	if (!test_bit(QBT_MISC_HWERR_PERI, &qbt_data->misc_flags)) {
+		set_bit(QBT_MISC_HWERR_BT, &qbt_data->misc_flags);
+		subsys_name = qhci_subsys_name(QHCI_SUBSYS_INVALID);
+	}
+
+	bt_dev_info(hdev, "%s hardware error (0x%02x)", subsys_name, code);
+	bt_dev_dbg(hdev, "md_pending_flags: 0x%lx", qbt_data->md_pending_flags);
+	bt_dev_dbg(hdev, "md_submit_err: %d", qbt_data->md_submit_err);
+
+	if (qbt_data->md_pending_flags && !qbt_data->md_submit_err) {
+		res = hci_devcd_complete(hdev);
+		if (res)
+			bt_dev_err(hdev, "memdump complete on %s hardware error failed: %pe",
+				   subsys_name, ERR_PTR(res));
+		else
+			bt_dev_info(hdev, "memdump completed on %s hardware error: %u bytes",
+				    subsys_name, qbt_data->md_submit_size);
+	}
+
+	qbt_data->md_pending_flags = 0;
+
+	if (test_and_clear_bit(QBT_MISC_MEMDUMP_INCOMING, &qbt_data->misc_flags))
+		bt_dev_dbg(hdev, "clear MEMDUMP_INCOMING on %s hardware error", subsys_name);
+}
+
+static int btusb_qcom_shutdown_unified(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	bool had_hwerr, had_memdump;
+	unsigned int delay_ms = 10;
+	int ret;
+
+	had_memdump = test_bit(QBT_MISC_MEMDUMP_BT, &qbt_data->misc_flags);
+	had_hwerr = test_bit(QBT_MISC_HWERR_BT, &qbt_data->misc_flags);
+
+	bt_dev_dbg(hdev, "shutdown: had_memdump %d, had_hwerr %d",
+		   had_memdump, had_hwerr);
+
+	if (!had_hwerr && !had_memdump) {
+		ret = __hci_reset_sync(hdev);
+		if (ret)
+			bt_dev_err(hdev, "shutdown: HCI reset failed: %pe",
+				   ERR_PTR(ret));
+		else
+			bt_dev_info(hdev, "shutdown: HCI reset succeeded");
+		goto out;
+	}
+
+	if (!had_memdump) {
+		ret = __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL,
+					    HCI_CMD_TIMEOUT);
+		if (!ret) {
+			clear_bit(QBT_MISC_HWERR_BT, &qbt_data->misc_flags);
+			bt_dev_info(hdev, "shutdown: recovered from hardware error via HCI reset");
+			goto out;
+		}
+
+		bt_dev_warn(hdev, "shutdown: HCI reset failed to recover from hardware error: %pe",
+			    ERR_PTR(ret));
+	}
+
+	ret = -EIO;
+	if (had_memdump && !btqcom_has_btc_reset(hdev)) {
+		bt_dev_info(hdev, "shutdown: BTC will self-recover via re-enumeration");
+		return ret;
+	}
+
+	bt_dev_info(hdev, "shutdown: reset BTC after %u msec", delay_ms);
+	btqcom_reset_async(hdev, delay_ms);
+
+	return ret;
+out:
+	/*
+	 * The flag is usually already unset here in a normal shutdown;
+	 * clear it just in case. No wake_up_var() is needed since the
+	 * waiter is never waiting at this time.
+	 */
+	if (test_and_clear_bit(QBT_MISC_CMD_TIMEOUT, &qbt_data->misc_flags))
+		bt_dev_dbg(hdev, "clear CMD_TIMEOUT on shutdown");
+
+	return ret;
+}
+
+static int btusb_qcom_shutdown_msubsys(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	bool had_peri_hwerr, had_peri_memdump;
+	bool had_bt_hwerr, had_bt_memdump;
+	bool had_tmel_memdump;
+	unsigned int delay_ms;
+	int ret;
+
+	delay_ms = 10;
+	had_peri_memdump = test_bit(QBT_MISC_MEMDUMP_PERI, &qbt_data->misc_flags);
+	had_peri_hwerr = test_bit(QBT_MISC_HWERR_PERI, &qbt_data->misc_flags);
+	had_bt_memdump = test_bit(QBT_MISC_MEMDUMP_BT, &qbt_data->misc_flags);
+	had_bt_hwerr = test_bit(QBT_MISC_HWERR_BT, &qbt_data->misc_flags);
+	had_tmel_memdump = test_bit(QBT_MISC_MEMDUMP_TMEL, &qbt_data->misc_flags);
+
+	bt_dev_dbg(hdev, "shutdown: had_peri_memdump %d, had_peri_hwerr %d",
+		   had_peri_memdump, had_peri_hwerr);
+	bt_dev_dbg(hdev, "shutdown: had_bt_memdump %d, had_bt_hwerr %d",
+		   had_bt_memdump, had_bt_hwerr);
+	bt_dev_dbg(hdev, "shutdown: had_tmel_memdump %d", had_tmel_memdump);
+
+	ret = -EIO;
+	if (had_peri_memdump || had_peri_hwerr || had_tmel_memdump)
+		goto out_reset;
+
+	ret = btusb_qcom_deactivate_msubsys_bt(hdev);
+	if (ret && (had_bt_memdump || had_bt_hwerr))
+		goto out_reset;
+
+	clear_bit(QBT_MISC_HWERR_BT, &qbt_data->misc_flags);
+	clear_bit(QBT_MISC_MEMDUMP_BT, &qbt_data->misc_flags);
+	/* Same as in btusb_qcom_shutdown_unified(). */
+	if (test_and_clear_bit(QBT_MISC_CMD_TIMEOUT, &qbt_data->misc_flags))
+		bt_dev_dbg(hdev, "clear CMD_TIMEOUT on shutdown");
+
+	return ret;
+
+out_reset:
+	bt_dev_info(hdev, "shutdown: reset BTC after %u msec", delay_ms);
+	btqcom_reset_async(hdev, delay_ms);
+	return ret;
+}
+
+/* Implements hdev->shutdown(). */
+static int btusb_qcom_shutdown(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	int ret;
+
+	switch (qbt_data->category) {
+	case QBTC_CAT_LEGACY:
+		ret = -EOPNOTSUPP;
+		break;
+	case QBTC_CAT_UNIFIED:
+		ret = btusb_qcom_shutdown_unified(hdev);
+		break;
+	case QBTC_CAT_MSUBSYS:
+		ret = btusb_qcom_shutdown_msubsys(hdev);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+/*
+ * ============================================================================
+ * QDFU firmware downloading
+ * ============================================================================
+ */
+
+#define QBT_FW_PATH_MAX  96
+
+enum qbt_fw_type {
+	QBT_FW_TYPE_PATCH,
+	QBT_FW_TYPE_NVM,
+	QBT_FW_TYPE_MAX,
+};
+
+#define QBT_PATCH_TYPE_TLV		0x01
+#define QBT_NVM_TYPE_TLV		0x02
+struct qbt_tlv_file_hdr {
+	u8	type;
+	u8	length[3];
+} __packed;
+
+struct qbt_patch_tlv_hdr {
+	struct  qbt_tlv_file_hdr tlv_hdr;
+	__le32	total_len;
+	__le32	patch_data_len;
+	u8	sign_ver;
+	u8	sign_algo;
+	u8	download_cfg;
+	u8	image_type;
+	__le16	product_id;
+	__le16	rom_ver;
+	__le16	patch_ver;
+	u8	reserved1[2];
+	__le32	anti_rollback_ver;
+	__le32	serial_low;
+	__le16	serial_high;
+	u8	debug_option;
+	u8	reserved2;
+	__le32	entry_addr;
+} __packed;
+
+struct qbt_nvm_tlv_hdr {
+	struct  qbt_tlv_file_hdr tlv_hdr;
+} __packed;
+
+/*
+ * struct qdfu_fw_cfg - QDFU firmware download configuration
+ * @desc: short description for logging (e.g. "PERI patch", "PERI NVM")
+ * @format: QBT_PATCH_TYPE_TLV or QBT_NVM_TYPE_TLV
+ * @hdr_size: TLV header size for @format
+ * @request: QDFU_BT_CMD_VSC_REQ_DOWNLOAD_(LOCAL|REMOTE)
+ * @loaded_flag: QDFU_BT_STATE_* flag that marks fw downloaded or not
+ * @settle_us: default time to wait after download for the BTC to settle
+ * @get_path: build firmware file path, return number of paths or error
+ * @check: check firmware, return 0 on success or a negative errno
+ * @download: download firmware, return 0 on success or a negative errno
+ * @get_settle_us: get the time to wait for BTC to settle
+ * @post_download: run after download, return 0 on success or a negative errno
+ */
+struct qdfu_fw_cfg {
+	const char	*desc;
+	u8		format;
+	u8		hdr_size;
+	u8		request;
+	u8		loaded_flag;
+	u32		settle_us;
+
+	int (*get_path)(struct hci_dev *hdev, const struct qdfu_fw_cfg *fw_cfg,
+			const struct qbtc_info *info, const struct qdfu_bt_id *ctrl_id,
+			char path[], size_t size);
+
+	int (*check)(struct hci_dev *hdev, const struct qdfu_fw_cfg *fw_cfg,
+		     const struct qbtc_info *info, const struct qdfu_bt_id *ctrl_id,
+		     const struct firmware *fw);
+
+	int (*download)(struct hci_dev *hdev, const struct qdfu_fw_cfg *fw_cfg,
+			const struct qbtc_info *info, const struct qdfu_bt_id *ctrl_id,
+			const void *data, size_t size);
+
+	u32 (*get_settle_us)(struct hci_dev *hdev, const struct qdfu_fw_cfg *fw_cfg,
+			     const struct qbtc_info *info, const struct qdfu_bt_id *ctrl_id);
+
+	int (*post_download)(struct hci_dev *hdev, const struct qdfu_fw_cfg *fw_cfg,
+			     const struct qbtc_info *info, const struct qdfu_bt_id *ctrl_id);
+};
+
+static int get_qbtc_subsys(const struct qdfu_fw_cfg *fw_cfg);
+
+static void qhci_to_qdfu_id(struct qdfu_bt_id *dfu_id,
+			    const struct qhci_btc_ver *hci_ver, u16 board_id)
+{
+	dfu_id->rom_version   = (u32)hci_ver->product_id << 16 | hci_ver->rom_ver;
+	dfu_id->patch_version = hci_ver->patch_ver;
+	dfu_id->soc_id        = hci_ver->soc_ver;
+	dfu_id->board_id      = board_id;
+}
+
+static void qdfu_to_qhci_id(struct qhci_btc_ver *hci_ver, u16 *board_id,
+			    const struct qdfu_bt_id *dfu_id)
+{
+	hci_ver->product_id = upper_16_bits(dfu_id->rom_version);
+	hci_ver->rom_ver    = lower_16_bits(dfu_id->rom_version);
+	hci_ver->patch_ver  = (u16)dfu_id->patch_version;
+	hci_ver->soc_ver    = dfu_id->soc_id;
+	*board_id           = dfu_id->board_id;
+}
+
+static int get_qdfu_id_via_hci(struct hci_dev *hdev,
+			       struct qdfu_bt_id *dfu_id)
+{
+	struct qhci_btc_ver hci_ver;
+	u16 board_id;
+	int ret;
+
+	ret = qbt_edl_patch_getver(hdev, &hci_ver);
+	if (ret)
+		return ret;
+
+	ret = qbt_edl_get_board_id(hdev, &board_id);
+	if (ret)
+		return ret;
+
+	qhci_to_qdfu_id(dfu_id, &hci_ver, board_id);
+
+	return 0;
+}
+
+static const char *get_fw_directory(struct hci_dev *hdev, const struct qbtc_info *info,
+				    const struct qdfu_bt_id *ctrl_id)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	const struct qbtc_bid_fwdir *custom_fwdir;
+
+	if (qbt_data->fw_dir)
+		return qbt_data->fw_dir;
+
+	if (!info->custom_fw_table)
+		goto info_fw_dir;
+
+	for (custom_fwdir = info->custom_fw_table; custom_fwdir->board_id; custom_fwdir++) {
+		if (custom_fwdir->board_id == ctrl_id->board_id)
+			return custom_fwdir->fw_dir;
+	}
+
+info_fw_dir:
+	if (info->fw_dir)
+		return info->fw_dir;
+
+	return "qca";
+}
+
+/*
+ * btusb_get_patch_path - build the patch firmware file path
+ * @hdev: the HCI device to build the path for
+ * @fw_cfg: the firmware download config
+ * @info: BTC info for this hdev
+ * @ctrl_id: BTC QDFU ID
+ * @path: output buffer for the built path
+ * @size: size of @path
+ *
+ * Used as the qdfu_fw_cfg ->get_path() callback.
+ *
+ * Return: 1 path written into @path, or a negative errno on failure.
+ */
+static int btusb_get_patch_path(struct hci_dev *hdev,
+				const struct qdfu_fw_cfg *fw_cfg,
+				const struct qbtc_info *info,
+				const struct qdfu_bt_id *ctrl_id,
+				char path[], size_t size)
+{
+	const char *fw_dir = get_fw_directory(hdev, info, ctrl_id);
+	int qbtc_subsys;
+	int len;
+
+	qbtc_subsys = get_qbtc_subsys(fw_cfg);
+
+	if (qbtc_subsys == QBTC_SUBSYS_INVALID) {
+		/* BT function unit. */
+		len = snprintf(path, size, "%s/rampatch_usb_%08x.bin",
+			       fw_dir, ctrl_id->rom_version);
+		goto out;
+	}
+
+	switch (qbtc_subsys) {
+	case QBTC_SUBSYS_PERI:
+		len = snprintf(path, size, "%s/peripatch_usb_%08x.bin",
+			       fw_dir, ctrl_id->rom_version);
+		break;
+	case QBTC_SUBSYS_TMEL:
+		return -EOPNOTSUPP;
+	default:
+		return -EINVAL;
+	}
+
+out:
+	if (len >= size)
+		return -ENAMETOOLONG;
+
+	return 1;
+}
+
+/*
+ * btusb_get_nvm_path - build the NVM firmware file path(s)
+ * @hdev: the HCI device to build the path for
+ * @fw_cfg: the firmware download config
+ * @info: BTC info for this hdev
+ * @ctrl_id: BTC QDFU ID
+ * @path: output buffer for the built path(s)
+ * @size: size of @path
+ *
+ * @path may hold two NUL-terminated paths back-to-back: a board-ID path
+ * followed by a fallback path, when NVM fallback is enabled.
+ *
+ * Used as the qdfu_fw_cfg ->get_path() callback.
+ *
+ * Return: number of paths (1 or 2) written into @path, or a negative
+ *	errno on failure.
+ */
+static int btusb_get_nvm_path(struct hci_dev *hdev,
+			      const struct qdfu_fw_cfg *fw_cfg,
+			      const struct qbtc_info *info,
+			      const struct qdfu_bt_id *ctrl_id,
+			      char path[], size_t size)
+{
+	const char *fw_dir = get_fw_directory(hdev, info, ctrl_id);
+	const char *prefix = NULL, *foundry_str = NULL;
+	bool has_bid = false, need_fb = false;
+	char fw_fb[QBT_FW_PATH_MAX] = { 0 };
+	int qbtc_subsys;
+	int len = 0;
+
+	qbtc_subsys = get_qbtc_subsys(fw_cfg);
+	if (qbtc_subsys == QBTC_SUBSYS_INVALID) {
+		/* BT function unit. */
+		prefix = "nvm_usb";
+	} else {
+		switch (qbtc_subsys) {
+		case QBTC_SUBSYS_PERI:
+			prefix = "perinvm_usb";
+			break;
+		case QBTC_SUBSYS_TMEL:
+			return -EOPNOTSUPP;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	if (info->flags & QBT_FLAG_FOUNDRY_NVM) {
+		u8 foundry = FIELD_GET(GENMASK(15, 12), ctrl_id->soc_id);
+
+		switch (foundry) {
+		/* GlobalFoundries */
+		case 0x01:
+			foundry_str = "_gf";
+			break;
+		default:
+			break;
+		}
+	}
+
+	if ((info->flags & QBT_FLAG_BID_NVM) && ctrl_id->board_id) {
+		has_bid = true;
+		if (info->flags & QBT_FLAG_NVM_FALLBACK)
+			need_fb = true;
+	}
+
+	len = snprintf(path, size, "%s/%s_%08x", fw_dir, prefix, ctrl_id->rom_version);
+	if (len >= size)
+		return -ENAMETOOLONG;
+
+	if (foundry_str) {
+		len += snprintf(path + len, size - len, "%s", foundry_str);
+		if (len >= size)
+			return -ENAMETOOLONG;
+	}
+
+	if (!has_bid) {
+		len += snprintf(path + len, size - len, ".bin");
+		if (len >= size)
+			return -ENAMETOOLONG;
+		return 1;
+	}
+
+	if (need_fb) {
+		int fb_len = snprintf(fw_fb, sizeof(fw_fb), "%s.bin", path);
+
+		if (fb_len >= sizeof(fw_fb))
+			return -ENAMETOOLONG;
+
+		len += snprintf(path + len, size - len, "_%04x.bin",
+				ctrl_id->board_id);
+		if (len >= size)
+			return -ENAMETOOLONG;
+
+		/* path holds two NUL-terminated strings back-to-back:
+		 * [board-id path '\0'][fallback path '\0']
+		 * Verify the buffer fits both before writing.
+		 */
+		if (len + fb_len + 2 > size)
+			return -ENAMETOOLONG;
+
+		len += 1;
+		snprintf(path + len, size - len, "%s", fw_fb);
+		return 2;
+	}
+
+	len += snprintf(path + len, size - len, "_%04x.bin", ctrl_id->board_id);
+	if (len >= size)
+		return -ENAMETOOLONG;
+
+	return 1;
+}
+
+/* check the TLV file header; return 0 on success, or a negative errno */
+static int qbt_check_tlv_file(struct hci_dev *hdev, const struct firmware *fw, u8 format)
+{
+	const struct qbt_tlv_file_hdr *tlv_hdr = (const struct qbt_tlv_file_hdr *)fw->data;
+	size_t file_len;
+
+	if (fw->size < sizeof(*tlv_hdr))
+		return -ENODATA;
+
+	if (tlv_hdr->type != format)
+		return -EINVAL;
+
+	file_len = get_unaligned_le24(tlv_hdr->length) + sizeof(*tlv_hdr);
+	if (file_len != fw->size)
+		return -EBADF;
+
+	return 0;
+}
+
+/*
+ * btusb_check_patch_tlv - validate a patch TLV file against the BTC
+ *	before download
+ * @hdev: the HCI device to validate the file for
+ * @fw_cfg: the firmware download config (unused)
+ * @info: BTC info for this hdev (unused)
+ * @ctrl_id: BTC QDFU ID
+ * @fw: the firmware to validate
+ *
+ * Used as the qdfu_fw_cfg ->check() callback.
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_check_patch_tlv(struct hci_dev *hdev,
+				 const struct qdfu_fw_cfg *fw_cfg,
+				 const struct qbtc_info *info __maybe_unused,
+				 const struct qdfu_bt_id *ctrl_id,
+				 const struct firmware *fw)
+{
+	const struct qbt_patch_tlv_hdr *patch_hdr = (const void *)fw->data;
+	u32 fw_rom_version, fw_patch_version;
+	int ret;
+
+	ret = qbt_check_tlv_file(hdev, fw, QBT_PATCH_TYPE_TLV);
+	if (ret) {
+		bt_dev_err(hdev, "Check %s TLV file header failed: %pe",
+			   fw_cfg->desc, ERR_PTR(ret));
+		return ret;
+	}
+
+	if (fw->size < sizeof(*patch_hdr)) {
+		bt_dev_err(hdev, "%s header truncated (%zu bytes, header needs %zu)",
+			   fw_cfg->desc, fw->size, sizeof(*patch_hdr));
+		return -ENODATA;
+	}
+
+	if (likely(upper_16_bits(ctrl_id->rom_version))) {
+		fw_rom_version = (le16_to_cpu(patch_hdr->product_id) << 16) |
+				 le16_to_cpu(patch_hdr->rom_ver);
+	} else {
+		bt_dev_warn(hdev, "check %s: invalid rom_version 0x%08x",
+			    fw_cfg->desc, ctrl_id->rom_version);
+		fw_rom_version = le16_to_cpu(patch_hdr->rom_ver);
+	}
+
+	fw_patch_version = le16_to_cpu(patch_hdr->patch_ver);
+
+	if (fw_rom_version != ctrl_id->rom_version) {
+		bt_dev_err(hdev,
+			   "%s ROM version 0x%08x does not match controller 0x%08x",
+			   fw_cfg->desc, fw_rom_version, ctrl_id->rom_version);
+		return -EINVAL;
+	}
+
+	if (fw_patch_version < ctrl_id->patch_version) {
+		bt_dev_err(hdev,
+			   "%s version 0x%x older than controller 0x%x, anti-rollback",
+			   fw_cfg->desc, fw_patch_version, ctrl_id->patch_version);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+/*
+ * validate a NVM TLV file against the BTC
+ * Used as the qdfu_fw_cfg ->check() callback.
+ */
+static int btusb_check_nvm_tlv(struct hci_dev *hdev,
+			       const struct qdfu_fw_cfg *fw_cfg,
+			       const struct qbtc_info *info __maybe_unused,
+			       const struct qdfu_bt_id *ctrl_id __maybe_unused,
+			       const struct firmware *fw)
+{
+	int ret;
+
+	ret = qbt_check_tlv_file(hdev, fw, QBT_NVM_TYPE_TLV);
+	if (ret) {
+		bt_dev_err(hdev, "Check %s TLV file header failed: %pe",
+			   fw_cfg->desc, ERR_PTR(ret));
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * btusb_download_via_qdfu - download a firmware image via QDFU to BTC
+ * @hdev: the HCI device to send the firmware to
+ * @fw_cfg: the firmware download config
+ * @info: BTC info for this hdev (unused)
+ * @ctrl_id: BTC QDFU ID (unused)
+ * @fw_data: the firmware image to send, header included
+ * @fw_size: size of @fw_data
+ *
+ * Used as the qdfu_fw_cfg ->download() callback.
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_download_via_qdfu(struct hci_dev *hdev,
+				   const struct qdfu_fw_cfg *fw_cfg,
+				   const struct qbtc_info *info __maybe_unused,
+				   const struct qdfu_bt_id *ctrl_id __maybe_unused,
+				   const void *fw_data, size_t fw_size)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	unsigned int pipe = usb_sndbulkpipe(xport_data->udev, 0x02);
+	const unsigned int xfer_timeout_ms = 3000;
+	const size_t xfer_size = SZ_4K;
+	const u8 *base = fw_data;
+	const u8 *ptr = fw_data;
+	size_t seg_size, size;
+	int snd_len, ret;
+	u8 *buf;
+
+	ret = qdfu_vsc_req_download(hdev, fw_cfg->request, ptr, fw_cfg->hdr_size);
+	if (ret)
+		return ret;
+
+	ptr += fw_cfg->hdr_size;
+	size = fw_size - fw_cfg->hdr_size;
+
+	/* ep2 needs time to switch from ACL to DFU function mode. */
+	fsleep(20 * 1000);
+
+	buf = kmalloc(xfer_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	while (size) {
+		seg_size = min_t(size_t, size, xfer_size);
+		memcpy(buf, ptr, seg_size);
+
+		snd_len = 0;
+		ret = usb_bulk_msg(xport_data->udev, pipe, buf, seg_size, &snd_len,
+				   xfer_timeout_ms);
+		if (ret < 0) {
+			bt_dev_err(hdev,
+				   "QDFU bulk send failed at offset %zu: %pe",
+				   ptr - base, ERR_PTR(ret));
+			break;
+		}
+
+		if (seg_size != snd_len) {
+			bt_dev_err(hdev,
+				   "QDFU bulk short write (%d of %zu bytes) at offset %zu",
+				   snd_len, seg_size, ptr - base);
+			ret = -EIO;
+			break;
+		}
+
+		ptr += seg_size;
+		size -= seg_size;
+	}
+
+	kfree(buf);
+	return ret;
+}
+
+/*
+ * Used as the qdfu_fw_cfg ->get_settle_us() callback.
+ * Kept here for future per-BTC settle time use.
+ */
+static u32 btusb_get_settle_us(struct hci_dev *hdev __maybe_unused,
+			       const struct qdfu_fw_cfg *fw_cfg,
+			       const struct qbtc_info *info __maybe_unused,
+			       const struct qdfu_bt_id *ctrl_id __maybe_unused)
+{
+	return fw_cfg->settle_us;
+}
+
+/* Used as the qdfu_fw_cfg ->post_download() callback. */
+static int btusb_post_download_unified_bt(struct hci_dev *hdev,
+					  const struct qdfu_fw_cfg *fw_cfg __maybe_unused,
+					  const struct qbtc_info *info __maybe_unused,
+					  const struct qdfu_bt_id *ctrl_id __maybe_unused)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	char *build_info = NULL;
+	int ret;
+
+	ret = qbt_edl_get_build_info(hdev, &build_info);
+	if (!ret) {
+		strscpy(qbt_data->build_info, build_info, sizeof(qbt_data->build_info));
+		kfree(build_info);
+	}
+
+	ret = __hci_reset_sync(hdev);
+	if (ret)
+		bt_dev_err(hdev, "HCI reset after BT NVM download failed: %pe", ERR_PTR(ret));
+	else
+		bt_dev_info(hdev, "HCI reset after BT NVM download succeeded");
+
+	return ret;
+}
+
+/* Used as the qdfu_fw_cfg ->post_download() callback. */
+static int btusb_post_download_msubsys_peri(struct hci_dev *hdev,
+					    const struct qdfu_fw_cfg *fw_cfg __maybe_unused,
+					    const struct qbtc_info *info __maybe_unused,
+					    const struct qdfu_bt_id *ctrl_id __maybe_unused)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct qbtc_subsys_data *subsys_data;
+	char *build_info = NULL;
+	int ret;
+
+	subsys_data = &qbt_data->subsys[QBTC_SUBSYS_PERI];
+
+	ret = qperi_get_subsys_build_info(hdev, QHCI_SUBSYS_PERI, &build_info);
+	if (!ret) {
+		strscpy(subsys_data->build_info, build_info, sizeof(subsys_data->build_info));
+		kfree(build_info);
+	}
+
+	/* Resetting PERI HCI is only allowed after downloading PERI NVM. */
+	if (qbt_data->flags & QBT_FLAG_RESET_PERI_HCI)
+		ret = qperi_hci_reset(hdev);
+	else
+		ret = qdfu_reset_peri_hci(hdev);
+
+	return ret;
+}
+
+static const struct qdfu_fw_cfg qdfu_fw_cfgs_bt[QBTC_CAT_MAX][QBT_FW_TYPE_MAX] = {
+	[QBTC_CAT_UNIFIED][QBT_FW_TYPE_PATCH] = {
+		.desc          = "patch",
+		.format        = QBT_PATCH_TYPE_TLV,
+		.hdr_size      = sizeof(struct qbt_patch_tlv_hdr),
+		.request       = QDFU_BT_CMD_VSC_REQ_DOWNLOAD_LOCAL,
+		.loaded_flag   = QDFU_BT_STATE_PATCHED_LOCAL,
+		.settle_us     = 10 * 1000,
+		.get_path      = btusb_get_patch_path,
+		.check         = btusb_check_patch_tlv,
+		.download      = btusb_download_via_qdfu,
+		.get_settle_us = btusb_get_settle_us,
+		.post_download = NULL,
+	},
+	[QBTC_CAT_UNIFIED][QBT_FW_TYPE_NVM] = {
+		.desc          = "NVM",
+		.format        = QBT_NVM_TYPE_TLV,
+		.hdr_size      = sizeof(struct qbt_nvm_tlv_hdr),
+		.request       = QDFU_BT_CMD_VSC_REQ_DOWNLOAD_LOCAL,
+		.loaded_flag   = QDFU_BT_STATE_NVMED_LOCAL,
+		.settle_us     = 40 * 1000,
+		.get_path      = btusb_get_nvm_path,
+		.check         = btusb_check_nvm_tlv,
+		.download      = btusb_download_via_qdfu,
+		.get_settle_us = btusb_get_settle_us,
+		.post_download = btusb_post_download_unified_bt,
+	},
+	[QBTC_CAT_MSUBSYS][QBT_FW_TYPE_PATCH] = {
+		.desc          = "BT patch",
+		.format        = QBT_PATCH_TYPE_TLV,
+		.hdr_size      = sizeof(struct qbt_patch_tlv_hdr),
+		.request       = QDFU_BT_CMD_VSC_REQ_DOWNLOAD_REMOTE,
+		.loaded_flag   = QDFU_BT_STATE_PATCHED_REMOTE,
+		.settle_us     = 30 * 1000,
+		.get_path      = btusb_get_patch_path,
+		.check         = btusb_check_patch_tlv,
+		.download      = btusb_download_via_qdfu,
+		.get_settle_us = btusb_get_settle_us,
+		.post_download = NULL,
+	},
+	[QBTC_CAT_MSUBSYS][QBT_FW_TYPE_NVM] = {
+		.desc          = "BT NVM",
+		.format        = QBT_NVM_TYPE_TLV,
+		.hdr_size      = sizeof(struct qbt_nvm_tlv_hdr),
+		.request       = QDFU_BT_CMD_VSC_REQ_DOWNLOAD_REMOTE,
+		.loaded_flag   = QDFU_BT_STATE_NVMED_REMOTE,
+		.settle_us     = 60 * 1000,
+		.get_path      = btusb_get_nvm_path,
+		.check         = btusb_check_nvm_tlv,
+		.download      = btusb_download_via_qdfu,
+		.get_settle_us = btusb_get_settle_us,
+		.post_download = btusb_post_download_unified_bt,
+	},
+};
+
+static const struct qdfu_fw_cfg qdfu_fw_cfgs_subsys[QBTC_SUBSYS_MAX][QBT_FW_TYPE_MAX] = {
+	[QBTC_SUBSYS_PERI][QBT_FW_TYPE_PATCH] = {
+		.desc          = "PERI patch",
+		.format        = QBT_PATCH_TYPE_TLV,
+		.hdr_size      = sizeof(struct qbt_patch_tlv_hdr),
+		.request       = QDFU_BT_CMD_VSC_REQ_DOWNLOAD_LOCAL,
+		.loaded_flag   = QDFU_BT_STATE_PATCHED_LOCAL,
+		.settle_us     = 10 * 1000,
+		.get_path      = btusb_get_patch_path,
+		.check         = btusb_check_patch_tlv,
+		.download      = btusb_download_via_qdfu,
+		.get_settle_us = btusb_get_settle_us,
+		.post_download = NULL,
+	},
+	[QBTC_SUBSYS_PERI][QBT_FW_TYPE_NVM] = {
+		.desc          = "PERI NVM",
+		.format        = QBT_NVM_TYPE_TLV,
+		.hdr_size      = sizeof(struct qbt_nvm_tlv_hdr),
+		.request       = QDFU_BT_CMD_VSC_REQ_DOWNLOAD_LOCAL,
+		.loaded_flag   = QDFU_BT_STATE_NVMED_LOCAL,
+		.settle_us     = 20 * 1000,
+		.get_path      = btusb_get_nvm_path,
+		.check         = btusb_check_nvm_tlv,
+		.download      = btusb_download_via_qdfu,
+		.get_settle_us = btusb_get_settle_us,
+		.post_download = btusb_post_download_msubsys_peri,
+	},
+};
+
+static int get_qbtc_subsys(const struct qdfu_fw_cfg *fw_cfg)
+{
+	int qbtc_subsys;
+	int fw_type;
+
+	for (qbtc_subsys = 0; qbtc_subsys < QBTC_SUBSYS_MAX; qbtc_subsys++) {
+		for (fw_type = 0; fw_type < QBT_FW_TYPE_MAX; fw_type++) {
+			if (&qdfu_fw_cfgs_subsys[qbtc_subsys][fw_type] == fw_cfg)
+				return qbtc_subsys;
+		}
+	}
+
+	return QBTC_SUBSYS_INVALID;
+}
+
+/*
+ * btusb_download_fw - download a firmware image
+ * @hdev: the HCI device to download the firmware to
+ * @fw_cfg: the firmware download config
+ * @info: BTC info for this hdev
+ * @ctrl_id: BTC ID
+ *
+ * Return: 1 if downloaded, 0 if already downloaded, or a negative errno
+ *	on failure.
+ */
+static int btusb_download_fw(struct hci_dev *hdev,
+			     const struct qdfu_fw_cfg *fw_cfg,
+			     const struct qbtc_info *info,
+			     const struct qdfu_bt_id *ctrl_id)
+{
+	const struct firmware *fw = NULL;
+	char paths[QBT_FW_PATH_MAX * 2];
+	const char *loaded_name;
+	u8 state, state_new;
+	u32 settle_us;
+	int n_paths;
+	int ret;
+
+	bt_dev_dbg(hdev, "download %s", fw_cfg->desc);
+	ret = qdfu_get_target_state(hdev, &state);
+	if (ret)
+		goto out;
+
+	if (state & fw_cfg->loaded_flag) {
+		bt_dev_info(hdev, "%s already downloaded", fw_cfg->desc);
+		goto out;
+	}
+
+	n_paths = fw_cfg->get_path(hdev, fw_cfg, info, ctrl_id, paths, sizeof(paths));
+	if (n_paths < 0) {
+		bt_dev_err(hdev, "Failed to get %s path: %pe",
+			   fw_cfg->desc, ERR_PTR(n_paths));
+		ret = n_paths;
+		goto out;
+	}
+	bt_dev_dbg(hdev, "%s path: %s", fw_cfg->desc, paths);
+	if (n_paths == 2)
+		bt_dev_dbg(hdev, "%s fallback path: %s", fw_cfg->desc,
+			   paths + strlen(paths) + 1);
+
+	loaded_name = paths;
+	ret = request_firmware(&fw, paths, &hdev->dev);
+	if (ret == -ENOENT && n_paths == 2) {
+		bt_dev_err(hdev, "Failed to request %s %s: %pe",
+			   fw_cfg->desc, loaded_name, ERR_PTR(ret));
+		loaded_name = paths + strlen(paths) + 1;
+		ret = request_firmware(&fw, loaded_name, &hdev->dev);
+	}
+	if (ret) {
+		bt_dev_err(hdev, "Failed to request %s %s: %pe",
+			   fw_cfg->desc, loaded_name, ERR_PTR(ret));
+		goto out;
+	}
+	bt_dev_dbg(hdev, "%s request succeeded: %s", fw_cfg->desc, loaded_name);
+
+	ret = fw_cfg->check(hdev, fw_cfg, info, ctrl_id, fw);
+	if (ret)
+		goto out_free_fw;
+
+	ret = fw_cfg->download(hdev, fw_cfg, info, ctrl_id, fw->data, fw->size);
+	if (ret)
+		goto out_free_fw;
+	bt_dev_info(hdev, "%s download succeeded: %s", fw_cfg->desc, loaded_name);
+
+	ret = qdfu_poll_state(hdev, &state_new, true, fw_cfg->loaded_flag);
+	if (ret)
+		goto out_free_fw;
+	if (state_new != state)
+		bt_dev_dbg(hdev, "%s state: 0x%02x -> 0x%02x",
+			   fw_cfg->desc, state, state_new);
+
+	if (fw_cfg->get_settle_us)
+		settle_us = fw_cfg->get_settle_us(hdev, fw_cfg, info, ctrl_id);
+	else
+		settle_us = fw_cfg->settle_us;
+	fsleep(settle_us);
+
+	if (fw_cfg->post_download) {
+		ret = fw_cfg->post_download(hdev, fw_cfg, info, ctrl_id);
+		if (ret)
+			bt_dev_err(hdev, "%s post-download failed: %pe",
+				   fw_cfg->desc, ERR_PTR(ret));
+		else
+			bt_dev_dbg(hdev, "%s post-download succeeded", fw_cfg->desc);
+	}
+
+	if (!ret) {
+		bt_dev_dbg(hdev, "download %s succeeded", fw_cfg->desc);
+		ret = 1;
+	}
+
+out_free_fw:
+	release_firmware(fw);
+out:
+	if (ret < 0)
+		bt_dev_err(hdev, "download %s failed: %pe", fw_cfg->desc, ERR_PTR(ret));
+	return ret;
+}
+
+/*
+ * ============================================================================
+ * btusb_qcom.h APIs
+ * ============================================================================
+ */
+
+/* size of the hci priv area */
+int btusb_qcom_hdev_priv_size(void)
+{
+	return sizeof(struct btqcom_data) + sizeof(struct btusb_qcom);
+}
+
+/* get @hdev's transport-specific data */
+struct btusb_qcom *btusb_qcom_xport_data(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+
+	return (struct btusb_qcom *)(qbt_data + 1);
+}
+
+/* work function for qbt_data->dwork */
+static void btqcom_work(struct work_struct *work)
+{
+	struct btqcom_data *qbt_data;
+	struct hci_dev *hdev;
+
+	qbt_data = container_of(to_delayed_work(work), struct btqcom_data, dwork);
+	hdev = qbt_data->hdev;
+
+	if (test_and_clear_bit(QBT_WORK_RESET_HDEV, &qbt_data->work_flags)) {
+		hci_dev_hold(hdev);
+		hci_req_sync_lock(hdev);
+
+		btusb_do_reset_work(hdev, QBT_RESET_TYPE_ASYNC);
+
+		hci_req_sync_unlock(hdev);
+		hci_dev_put(hdev);
+	}
+}
+
+static void devm_btusb_qcom_deinit(void *data)
+{
+	struct btqcom_data *qbt_data = data;
+	struct btusb_qcom *xport_data;
+
+	if (!qbt_data->inited)
+		return;
+
+	xport_data = qbt_data->xport_data;
+	mutex_destroy(&qbt_data->req_mutex);
+	mutex_destroy(&xport_data->tx_mutex);
+
+	qbt_data->inited = false;
+	bt_dev_dbg(qbt_data->hdev, "btusb qcom deinited");
+}
+
+static void __maybe_unused btusb_qcom_deinit(struct btqcom_data *qbt_data)
+{
+	devm_release_action(&qbt_data->hdev->dev, devm_btusb_qcom_deinit,
+			    qbt_data);
+}
+
+/*
+ * btusb_qcom_init - initialize hdev and per-device btqcom_data
+ * @qbt_data: the btqcom_data to initialize
+ *
+ * Doesn't touch btqcom_data fields that live for the hdev's lifetime,
+ * e.g. @fw_dir and @fw_logging.
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_qcom_init(struct btqcom_data *qbt_data)
+{
+	struct btusb_qcom *xport_data = qbt_data->xport_data;
+	struct hci_dev *hdev = qbt_data->hdev;
+	int res;
+
+	WRITE_ONCE(qbt_data->misc_flags, 0);
+	WRITE_ONCE(qbt_data->work_flags, 0);
+
+	qbt_data->md_state = HCI_DEVCOREDUMP_IDLE;
+	qbt_data->md_submit_err = 0;
+	qbt_data->md_submit_size = 0;
+	qbt_data->md_pending_flags = 0;
+	btqcom_reset_memdump(&qbt_data->md);
+
+	WRITE_ONCE(qbt_data->req, NULL);
+	WRITE_ONCE(qbt_data->req_state, QHCI_REQ_DONE);
+
+	if (qbt_data->inited)
+		return 0;
+
+	hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_SETUP);
+	hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
+	hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED);
+
+	if (qbt_data->flags & QBT_FLAG_AOSP_EXT)
+		hci_set_aosp_capable(hdev);
+	if (qbt_data->flags & QBT_FLAG_MSFT_EXT)
+		hci_set_msft_opcode(hdev, 0xFD70);
+
+	res = devm_add_action(&hdev->dev, devm_btusb_qcom_deinit, qbt_data);
+	if (res) {
+		bt_dev_err(hdev, "Add btusb qcom deinit action failed: %pe",
+			   ERR_PTR(res));
+		return res;
+	}
+
+	INIT_DELAYED_WORK(&qbt_data->dwork, btqcom_work);
+	mutex_init(&xport_data->tx_mutex);
+	mutex_init(&qbt_data->req_mutex);
+	init_waitqueue_head(&qbt_data->req_wait_q);
+	spin_lock_init(&qbt_data->req_spinlock);
+
+	hdev->set_bdaddr        = btqcom_set_bdaddr;
+	hdev->reset             = btusb_qcom_reset;
+	hdev->shutdown          = btusb_qcom_shutdown;
+	hdev->hw_error          = btqcom_hw_error;
+	hdev->handle_ev_vendor  = btqcom_handle_ev_vendor;
+	hdev->recv_vendor_pkt   = btqcom_recv_vendor_pkt;
+
+	qbt_data->md_ready = false;
+	if (qbt_data->flags & QBT_FLAG_MEMDUMP) {
+		res = hci_devcd_register(hdev, btusb_qcom_trigger_memdump,
+					 btusb_qcom_memdump_hdr,
+					 btusb_qcom_notify_memdump);
+		if (!res)
+			qbt_data->md_ready = true;
+		else if (res == -EOPNOTSUPP)
+			bt_dev_warn(hdev, "CONFIG_DEV_COREDUMP not enabled");
+		else
+			bt_dev_err(hdev, "register devcoredump failed: %pe",
+				   ERR_PTR(res));
+	}
+
+	/* pairs with smp_load_acquire() in btusb_qcom_disconnect() */
+	smp_store_release(&qbt_data->inited, true);
+
+	bt_dev_dbg(hdev, "btusb qcom inited");
+	return 0;
+}
+
+/*
+ * btusb_qcom_setup_unified - setup for a unified BTC
+ * @hdev: the HCI device to set up
+ * @info: BTC info for this hdev
+ * @ctrl_id: BTC QDFU ID
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_qcom_setup_unified(struct hci_dev *hdev,
+				    const struct qbtc_info *info,
+				    const struct qdfu_bt_id *ctrl_id)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	enum qbtc_category btc_cat = info->category;
+	const struct qdfu_fw_cfg *cfgs_bt;
+	struct qdfu_bt_id bt_id;
+	int ret;
+
+	cfgs_bt = qdfu_fw_cfgs_bt[btc_cat];
+
+	ret = btusb_download_fw(hdev, &cfgs_bt[QBT_FW_TYPE_PATCH], info, ctrl_id);
+	if (ret < 0)
+		return ret;
+
+	ret = qdfu_get_target_version(hdev, &bt_id);
+	if (ret)
+		return ret;
+	qdfu_to_qhci_id(&qbt_data->ver, &qbt_data->board_id, &bt_id);
+
+	ret = btusb_download_fw(hdev, &cfgs_bt[QBT_FW_TYPE_NVM], info, &bt_id);
+	if (ret)
+		return ret < 0 ? ret : 0;
+
+	ret = __hci_reset_sync(hdev);
+	if (ret)
+		bt_dev_err(hdev, "HCI reset failed: %pe", ERR_PTR(ret));
+	else
+		bt_dev_dbg(hdev, "HCI reset succeeded");
+
+	return ret;
+}
+
+/*
+ * btusb_qcom_setup_msubsys - setup for a multi-subsystem BTC
+ * @hdev: the HCI device to set up
+ * @info: BTC info for this hdev
+ * @ctrl_id: BTC QDFU ID
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_qcom_setup_msubsys(struct hci_dev *hdev,
+				    const struct qbtc_info *info,
+				    const struct qdfu_bt_id *ctrl_id)
+{
+	const struct qdfu_fw_cfg *cfgs_peri = qdfu_fw_cfgs_subsys[QBTC_SUBSYS_PERI];
+	const struct qdfu_fw_cfg *cfgs_bt = qdfu_fw_cfgs_bt[QBTC_CAT_MSUBSYS];
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct qbtc_subsys_data *subsys_peri;
+	struct qdfu_bt_id peri_id, bt_id;
+	int ret;
+
+	subsys_peri = &qbt_data->subsys[QBTC_SUBSYS_PERI];
+
+	ret = btusb_download_fw(hdev, &cfgs_peri[QBT_FW_TYPE_PATCH], info, ctrl_id);
+	if (ret < 0)
+		return ret;
+
+	ret = qdfu_get_target_version(hdev, &peri_id);
+	if (ret)
+		return ret;
+	qdfu_to_qhci_id(&subsys_peri->ver, &subsys_peri->board_id, &peri_id);
+
+	ret = btusb_download_fw(hdev, &cfgs_peri[QBT_FW_TYPE_NVM], info, &peri_id);
+	if (ret < 0)
+		return ret;
+
+	ret = qdfu_reset_msubsys_bt(hdev);
+	if (ret)
+		return ret;
+
+	ret = get_qdfu_id_via_hci(hdev, &bt_id);
+	if (ret)
+		return ret;
+
+	if (bt_id.rom_version != peri_id.rom_version ||
+	    bt_id.soc_id != peri_id.soc_id ||
+	    bt_id.board_id != peri_id.board_id) {
+		bt_dev_info(hdev, "BT ROM Version: 0x%08x",
+			    bt_id.rom_version);
+		bt_dev_info(hdev, "BT Patch Version: 0x%08x",
+			    bt_id.patch_version);
+		bt_dev_info(hdev, "BT SoC Version: 0x%08x",
+			    bt_id.soc_id);
+		bt_dev_info(hdev, "BT Board ID: 0x%04x",
+			    bt_id.board_id);
+	}
+
+	ret = btusb_download_fw(hdev, &cfgs_bt[QBT_FW_TYPE_PATCH], info, &bt_id);
+	if (ret < 0)
+		return ret;
+
+	ret = get_qdfu_id_via_hci(hdev, &bt_id);
+	if (ret)
+		return ret;
+	qdfu_to_qhci_id(&qbt_data->ver, &qbt_data->board_id, &bt_id);
+
+	ret = btusb_download_fw(hdev, &cfgs_bt[QBT_FW_TYPE_NVM], info, &bt_id);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int btqcom_post_setup(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	struct qbtc_subsys_data *subsys_data;
+	char *build_info = NULL;
+	char buf[512];
+	int len = 0;
+	int res;
+
+	bt_dev_dbg(hdev, "flags: 0x%lx", qbt_data->flags);
+	bt_dev_info(hdev, "memdump ready? %s", str_yes_no(qbt_data->md_ready));
+
+	qbt_config_fw_logging(hdev, qbt_data->fw_logging);
+
+	hci_set_hw_info(hdev, "%s", qbt_data->btc_name);
+	if (qbt_data->category == QBTC_CAT_MSUBSYS) {
+		subsys_data = &qbt_data->subsys[QBTC_SUBSYS_PERI];
+		if (subsys_data->build_info[0] == '\0') {
+			res = qperi_get_subsys_build_info(hdev, QHCI_SUBSYS_PERI,
+							  &build_info);
+			if (!res) {
+				strscpy(subsys_data->build_info, build_info,
+					sizeof(subsys_data->build_info));
+				kfree(build_info);
+			}
+		}
+
+		bt_dev_info(hdev, "%s build info: %s",
+			    qbtc_subsys_name(QBTC_SUBSYS_PERI), subsys_data->build_info);
+		len = scnprintf(buf, sizeof(buf), "%s: %s\n",
+				qbtc_subsys_name(QBTC_SUBSYS_PERI), subsys_data->build_info);
+	}
+
+	if (qbt_data->build_info[0] == '\0') {
+		build_info = NULL;
+		res = qbt_edl_get_build_info(hdev, &build_info);
+		if (!res) {
+			strscpy(qbt_data->build_info, build_info,
+				sizeof(qbt_data->build_info));
+			kfree(build_info);
+		}
+	}
+
+	bt_dev_info(hdev, "BT build info: %s", qbt_data->build_info);
+	len += snprintf(buf + len, sizeof(buf) - len, "BT: %s", qbt_data->build_info);
+	hci_set_fw_info(hdev, "%s", buf);
+
+	return 0;
+}
+
+/*
+ * btusb_qcom_setup - setup a BTC
+ * @hdev: the HCI device
+ *
+ * Identifies the BTC, then dispatches setup based on its category.
+ * Implements hdev->setup().
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int btusb_qcom_setup(struct hci_dev *hdev)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	bool is_first_setup = hci_dev_test_flag(hdev, HCI_SETUP);
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	const struct qbtc_id *id_entry = qbtc_id_table;
+	struct qdfu_bt_id ctrl_id;
+	int ret;
+
+	qbt_data->hdev = hdev;
+	xport_data->intf = to_usb_interface(hdev->dev.parent);
+	xport_data->udev = interface_to_usbdev(xport_data->intf);
+	xport_data->idVendor  = le16_to_cpu(xport_data->udev->descriptor.idVendor);
+	xport_data->idProduct = le16_to_cpu(xport_data->udev->descriptor.idProduct);
+	qbt_data->xport_data = xport_data;
+
+	ret = usb_autopm_get_interface(xport_data->intf);
+	if (ret) {
+		bt_dev_err(hdev, "get autopm for QCOM BTUSB setup failed: %pe",
+			   ERR_PTR(ret));
+		return ret;
+	}
+
+	ret = qdfu_get_target_version(hdev, &ctrl_id);
+	if (ret)
+		goto out;
+
+	while (id_entry->rom_version) {
+		if (id_entry->rom_version == ctrl_id.rom_version)
+			break;
+		id_entry++;
+	}
+
+	if (!id_entry->rom_version) {
+		ret = -ENODEV;
+		bt_dev_warn(hdev, "Detected unsupported BT controller:");
+	} else {
+		bt_dev_info(hdev, "%s QCOM %s BT controller: %s",
+			    is_first_setup ? "Detected" : "Setup",
+			    qbtc_category_name(id_entry->btc_info->category),
+			    id_entry->name);
+	}
+
+	bt_dev_info(hdev, "ROM Version  : 0x%08x", ctrl_id.rom_version);
+	bt_dev_info(hdev, "Patch Version: 0x%08x", ctrl_id.patch_version);
+	bt_dev_info(hdev, "SoC Version  : 0x%08x", ctrl_id.soc_id);
+	bt_dev_info(hdev, "Board ID     : 0x%04x", ctrl_id.board_id);
+
+	if (ret)
+		goto out;
+
+	if (!qbt_data->drv_name)
+		qbt_data->drv_name = dev_driver_string(&xport_data->intf->dev);
+	qbt_data->btc_name = id_entry->name;
+	qbt_data->category = id_entry->btc_info->category;
+	qbt_data->flags    = id_entry->btc_info->flags & QBT_FLAG_BTC_CFG_MASK;
+	if (xport_data->reset_gpio)
+		qbt_data->flags |= QBT_FLAG_HW_RESET;
+	ret = btusb_qcom_init(qbt_data);
+	if (ret)
+		goto out;
+
+	switch (qbt_data->category) {
+	case QBTC_CAT_LEGACY:
+		ret = -EOPNOTSUPP;
+		break;
+	case QBTC_CAT_UNIFIED:
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN);
+		ret = btusb_qcom_setup_unified(hdev, id_entry->btc_info, &ctrl_id);
+		break;
+	case QBTC_CAT_MSUBSYS:
+		ret = btusb_qcom_setup_msubsys(hdev, id_entry->btc_info, &ctrl_id);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (!ret)
+		ret = btqcom_post_setup(hdev);
+
+out:
+	usb_autopm_put_interface(xport_data->intf);
+	if (!ret)
+		bt_dev_info(hdev, "QCOM BTUSB setup succeeded (^_^)");
+	else
+		bt_dev_err(hdev, "QCOM BTUSB setup failed: %pe", ERR_PTR(ret));
+	return ret;
+}
+
+/*
+ * btusb_qcom_disconnect - clean up on USB disconnect
+ * @hdev: the HCI device being disconnected
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int btusb_qcom_disconnect(struct hci_dev *hdev)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+
+	/* pairs with smp_store_release() in btusb_qcom_init() */
+	if (!smp_load_acquire(&qbt_data->inited))
+		return 0;
+
+	bt_dev_dbg(hdev, "disconnection: misc_flags(0x%lx)",
+		   READ_ONCE(qbt_data->misc_flags));
+
+	if (test_bit(QBT_MISC_MEMDUMP_INCOMING, &qbt_data->misc_flags) &&
+	    !qbt_data->md_submit_err)
+		bt_dev_warn(hdev, "memdump collection interrupted by disconnection");
+
+	qperi_tx_sync_cancel_sync(hdev, -ENODEV);
+	if (test_and_clear_bit(QBT_MISC_CMD_TIMEOUT, &qbt_data->misc_flags)) {
+		wake_up_var(&qbt_data->misc_flags);
+		bt_dev_dbg(hdev, "wake command-timeout waiter on disconnection");
+	}
+	disable_delayed_work_sync(&qbt_data->dwork);
+
+	return 0;
+}
+
+/*
+ * btusb_qcom_send_frame - send a frame for BT or PERI
+ * @hdev: the HCI device
+ * @skb: the frame to send
+ *
+ * Implements hdev->send().
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int btusb_qcom_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
+	__context_unsafe(/* conditional locking */)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	bool is_vendor;
+	bool need_lock;
+	u8 pkt_type;
+	int ret;
+
+	if (qbt_data->category != QBTC_CAT_MSUBSYS)
+		return xport_data->send_bt_frame(hdev, skb);
+
+	pkt_type = hci_skb_pkt_type(skb);
+	is_vendor = pkt_type == HCI_VENDOR_PKT;
+	/* an empty vendor packet has no path to reach here */
+	if (is_vendor)
+		hci_skb_pkt_type(skb) = *(const u8 *)skb_pull_data(skb, 1);
+
+	pkt_type = hci_skb_pkt_type(skb);
+	need_lock = pkt_type == HCI_COMMAND_PKT ||
+		    pkt_type == QPERI_COMMAND_PKT;
+
+	if (need_lock)
+		mutex_lock(&xport_data->tx_mutex);
+
+	if (is_vendor)
+		ret = xport_data->send_vendor_frame(hdev, skb);
+	else
+		ret = xport_data->send_bt_frame(hdev, skb);
+
+	if (need_lock)
+		mutex_unlock(&xport_data->tx_mutex);
+
+	return ret;
+}
+
+/*
+ * RX flow:
+ *
+ * USB intr/bulk endpoint (byte stream)
+ *  -> btusb_qcom_recv_{intr,bulk}() (reassemble to frame)
+ *      -> btqcom_recv_frame()
+ *          -> btusb_upward_frame() (dispatch frame upwards)
+ *              -> hci_recv_frame() (PERI frame as HCI_VENDOR_PKT)
+ *              -> xport_data->recv_bt_frame() (BT, back to btusb_main.c)
+ */
+
+/*
+ * btusb_upward_frame - dispatch a received frame upwards
+ * @hdev: the HCI device the @skb comes from
+ * @skb: the frame to dispatch upwards
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+static int btusb_upward_frame(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct btusb_qcom *xport_data = btusb_qcom_xport_data(hdev);
+	u8 pkt_type = hci_skb_pkt_type(skb);
+	u16 handle;
+	int ret;
+
+	switch (pkt_type) {
+	case QPERI_EVENT_PKT:
+	case QPERI_ACLDATA_PKT:
+		*(u8 *)skb_push(skb, 1) = pkt_type;
+		hci_skb_pkt_type(skb) = HCI_VENDOR_PKT;
+		break;
+	case HCI_EVENT_PKT:
+		break;
+	case HCI_ACLDATA_PKT:
+		handle = hci_acl_handle(skb);
+		/* reroute vendor ACLs as HCI_VENDOR_PKT */
+		if (handle == QBT_HANDLE_ENHANCED_LOGGING ||
+		    handle == QBT_HANDLE_MEMDUMP) {
+			*(u8 *)skb_push(skb, 1) = pkt_type;
+			hci_skb_pkt_type(skb) = HCI_VENDOR_PKT;
+		}
+		break;
+	default:
+		dev_kfree_skb_irq(skb);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (hci_skb_pkt_type(skb) == HCI_VENDOR_PKT)
+		ret = hci_recv_frame(hdev, skb);
+	else
+		ret = xport_data->recv_bt_frame(hdev, skb);
+
+out:
+	if (ret)
+		bt_dev_err(hdev, "upward frame with type (0x%02x) failed: %pe",
+			   pkt_type, ERR_PTR(ret));
+
+	return ret;
+}
+
+/* Every frame lands here first — the ideal spot for pre-processing. */
+static int btqcom_recv_frame(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	u8 pkt_type = hci_skb_pkt_type(skb);
+
+	switch (pkt_type) {
+	case HCI_EVENT_PKT:
+	case QPERI_EVENT_PKT:
+	case HCI_ACLDATA_PKT:
+	case QPERI_ACLDATA_PKT:
+		break;
+	default:
+		dev_kfree_skb_irq(skb);
+		bt_dev_err_ratelimited(hdev, "unexpected pkt type 0x%02x", pkt_type);
+		return -EINVAL;
+	}
+
+	return btusb_upward_frame(hdev, skb);
+}
+
+/*
+ * RX reassembly state, stashed in hci_skb_pkt_seqnum(skb) between calls:
+ * QRX_STATE_INDICATOR - determining the packet type
+ * QRX_STATE_HEADER    - receiving the packet header
+ * QRX_STATE_PAYLOAD  - receiving the payload
+ */
+enum {
+	QRX_STATE_INDICATOR,
+	QRX_STATE_HEADER,
+	QRX_STATE_PAYLOAD,
+};
+
+/*
+ * btusb_qcom_recv_intr - reassemble byte stream from the intr endpoint to frame
+ * @hdev: the HCI device
+ * @skb: the in-progress frame, or NULL to start a new one
+ * @buffer: bytes received
+ * @count: bytes count
+ * @err: output error code
+ *
+ * See the RX flow diagram above for further dispatch.
+ *
+ * Return: the in-progress frame to resume on the next call, or NULL
+ *	otherwise.
+ */
+struct sk_buff *btusb_qcom_recv_intr(struct hci_dev *hdev, struct sk_buff *skb,
+				     void *buffer, int count, int *err)
+{
+	struct btqcom_data *qbt_data = hci_get_priv(hdev);
+	enum qbtc_category btc_cat = qbt_data->category;
+	int rx_state;
+	u8 pkt_type;
+	int len;
+	int res;
+
+	*err = 0;
+	while (count) {
+		if (!skb) {
+			u8 host_id = *(u8 *)buffer;
+
+			skb = bt_skb_alloc(QPERI_MAX_EVENT_SIZE, GFP_ATOMIC);
+			if (!skb) {
+				*err = -ENOMEM;
+				break;
+			}
+
+			/* see struct qperi_event_hdr for PERI event format */
+			if (host_id == QHCI_HOST_ID_BT) {
+				hci_skb_pkt_type(skb) = QPERI_EVENT_PKT;
+				hci_skb_pkt_seqnum(skb) = QRX_STATE_HEADER;
+				hci_skb_expect(skb) = QPERI_EVENT_HDR_SIZE;
+			} else {
+				hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
+				hci_skb_pkt_seqnum(skb) = QRX_STATE_HEADER;
+				hci_skb_expect(skb) = HCI_EVENT_HDR_SIZE;
+			}
+		}
+
+		len = min_t(uint, hci_skb_expect(skb), count);
+		skb_put_data(skb, buffer, len);
+
+		count -= len;
+		buffer += len;
+		hci_skb_expect(skb) -= len;
+
+		if (hci_skb_expect(skb))
+			continue;
+
+		rx_state = hci_skb_pkt_seqnum(skb);
+		if (rx_state == QRX_STATE_PAYLOAD)
+			goto frame_done;
+
+		hci_skb_pkt_seqnum(skb) = QRX_STATE_PAYLOAD;
+		pkt_type = hci_skb_pkt_type(skb);
+		if (pkt_type == QPERI_EVENT_PKT)
+			hci_skb_expect(skb) = qperi_event_header(skb)->plen;
+		else if (pkt_type == HCI_EVENT_PKT)
+			hci_skb_expect(skb) = hci_event_hdr(skb)->plen;
+
+		if (hci_skb_expect(skb))
+			continue;
+
+frame_done:
+		if (count && count < HCI_EVENT_HDR_SIZE) {
+			bt_dev_warn(hdev,
+				    "Unexpected continuation: %d bytes",
+				    count);
+			if (btc_cat != QBTC_CAT_MSUBSYS)
+				count = 0;
+		}
+
+		hci_skb_pkt_seqnum(skb) = 0;
+		res = btqcom_recv_frame(hdev, skb);
+		if (res)
+			bt_dev_err_ratelimited(hdev, "recv intr frame failed: %pe",
+					       ERR_PTR(res));
+		skb = NULL;
+	}
+
+	return skb;
+}
+
+/*
+ * btusb_qcom_recv_bulk - reassemble byte stream from the bulk endpoint to frame
+ * @hdev: the HCI device
+ * @skb: the in-progress frame, or NULL to start a new one
+ * @buffer: bytes received
+ * @count: bytes count
+ * @err: output error code
+ *
+ * See the RX flow diagram above for further dispatch.
+ *
+ * Return: the in-progress frame to resume on the next call, or NULL
+ *	otherwise.
+ */
+struct sk_buff *btusb_qcom_recv_bulk(struct hci_dev *hdev, struct sk_buff *skb,
+				     void *buffer, int count, int *err)
+{
+	struct qperi_acl_hdr *peri_hdr;
+	u16 peri_handle;
+	int rx_state;
+	u8 pkt_type;
+	int res;
+	int len;
+
+	*err = 0;
+	while (count) {
+		if (!skb) {
+			skb = bt_skb_alloc(QPERI_MAX_FRAME_SIZE, GFP_ATOMIC);
+			if (!skb) {
+				*err = -ENOMEM;
+				break;
+			}
+			hci_skb_pkt_seqnum(skb) = QRX_STATE_INDICATOR;
+			hci_skb_expect(skb) = HCI_ACL_HDR_SIZE;
+		}
+
+		len = min_t(uint, hci_skb_expect(skb), count);
+		skb_put_data(skb, buffer, len);
+
+		count -= len;
+		buffer += len;
+		hci_skb_expect(skb) -= len;
+
+		if (hci_skb_expect(skb))
+			continue;
+
+		rx_state = hci_skb_pkt_seqnum(skb);
+		switch (rx_state) {
+		case QRX_STATE_INDICATOR:
+			hci_skb_pkt_seqnum(skb) = QRX_STATE_HEADER;
+			peri_hdr = qperi_acl_header(skb);
+			peri_handle = qperi_acl_handle(skb);
+			if (peri_hdr->host_id == QHCI_HOST_ID_BT &&
+			    peri_handle >= 0xEC0 && peri_handle <= 0xECF) {
+				hci_skb_pkt_type(skb) = QPERI_ACLDATA_PKT;
+				hci_skb_expect(skb) = QPERI_ACL_HDR_SIZE - HCI_ACL_HDR_SIZE;
+				continue;
+			}
+			hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT;
+			fallthrough;
+
+		case QRX_STATE_HEADER:
+			pkt_type = hci_skb_pkt_type(skb);
+			if (pkt_type == QPERI_ACLDATA_PKT)
+				hci_skb_expect(skb) = qperi_acl_dlen(skb);
+			else if (pkt_type == HCI_ACLDATA_PKT)
+				hci_skb_expect(skb) = hci_acl_dlen(skb);
+
+			if (hci_skb_expect(skb) > QPERI_MAX_FRAME_SIZE - skb->len) {
+				dev_kfree_skb_irq(skb);
+				skb = NULL;
+				*err = -EILSEQ;
+				return NULL;
+			}
+			hci_skb_pkt_seqnum(skb) = QRX_STATE_PAYLOAD;
+			if (hci_skb_expect(skb))
+				continue;
+			fallthrough;
+
+		case QRX_STATE_PAYLOAD:
+			break;
+		}
+
+		hci_skb_pkt_seqnum(skb) = 0;
+		res = btqcom_recv_frame(hdev, skb);
+		if (res)
+			bt_dev_err_ratelimited(hdev, "recv bulk frame failed: %pe",
+					       ERR_PTR(res));
+		skb = NULL;
+	}
+
+	return skb;
+}
+
+MODULE_AUTHOR("Zijun Hu <zijun.hu@oss.qualcomm.com>");
diff --git a/drivers/bluetooth/btusb_qcom.h b/drivers/bluetooth/btusb_qcom.h
new file mode 100644
index 00000000000000..8fbc105f4eaf53
--- /dev/null
+++ b/drivers/bluetooth/btusb_qcom.h
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Qualcomm Bluetooth USB transport-specific support
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __BTUSB_QCOM_H
+#define __BTUSB_QCOM_H
+
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/skbuff.h>
+#include <linux/usb.h>
+
+#include <net/bluetooth/hci_core.h>
+
+#define QPERI_COMMAND_PKT	0x31
+#define QPERI_ACLDATA_PKT	0x32
+#define QPERI_EVENT_PKT		0x34
+
+struct btusb_qcom {
+	__u16                 idVendor;
+	__u16                 idProduct;
+	struct usb_device    *udev;
+	struct usb_interface *intf;
+	struct gpio_desc     *reset_gpio;
+
+	/* serializes sending BT and vendor frames to the transport */
+	struct mutex          tx_mutex;
+
+	void (*prepare_reset)(struct hci_dev *hdev);
+	int (*recv_bt_frame)(struct hci_dev *hdev, struct sk_buff *skb);
+	int (*send_bt_frame)(struct hci_dev *hdev, struct sk_buff *skb);
+	int (*send_vendor_frame)(struct hci_dev *hdev, struct sk_buff *skb);
+};
+
+#if IS_ENABLED(CONFIG_BT_HCIBTUSB_QCOM)
+
+int btusb_qcom_hdev_priv_size(void);
+struct btusb_qcom *btusb_qcom_xport_data(struct hci_dev *hdev);
+
+int btusb_qcom_setup(struct hci_dev *hdev);
+int btusb_qcom_disconnect(struct hci_dev *hdev);
+int btusb_qcom_send_frame(struct hci_dev *hdev, struct sk_buff *skb);
+
+struct sk_buff *btusb_qcom_recv_intr(struct hci_dev *hdev, struct sk_buff *skb,
+				     void *buffer, int count, int *err);
+
+struct sk_buff *btusb_qcom_recv_bulk(struct hci_dev *hdev, struct sk_buff *skb,
+				     void *buffer, int count, int *err);
+
+#else
+
+static inline int btusb_qcom_hdev_priv_size(void)
+{
+	return 0;
+}
+
+static inline struct btusb_qcom *btusb_qcom_xport_data(struct hci_dev *hdev)
+{
+	return NULL;
+}
+
+static inline int btusb_qcom_setup(struct hci_dev *hdev)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int btusb_qcom_disconnect(struct hci_dev *hdev)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int btusb_qcom_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline struct sk_buff *btusb_qcom_recv_intr(struct hci_dev *hdev,
+						   struct sk_buff *skb,
+						   void *buffer, int count,
+						   int *err)
+{
+	*err = -EOPNOTSUPP;
+	return NULL;
+}
+
+static inline struct sk_buff *btusb_qcom_recv_bulk(struct hci_dev *hdev,
+						   struct sk_buff *skb,
+						   void *buffer, int count,
+						   int *err)
+{
+	*err = -EOPNOTSUPP;
+	return NULL;
+}
+
+#endif
+#endif
diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
index 9a103db7e3558b..e70b9f3fd96857 100644
--- a/drivers/bluetooth/hci_bcm.c
+++ b/drivers/bluetooth/hci_bcm.c
@@ -1593,6 +1593,7 @@ static const struct of_device_id bcm_bluetooth_of_match[] = {
 	{ .compatible = "brcm,bcm4349-bt", .data = &bcm43438_device_data },
 	{ .compatible = "brcm,bcm43540-bt", .data = &bcm4354_device_data },
 	{ .compatible = "brcm,bcm4335a0" },
+	{ .compatible = "brcm,bcm4384-bt" },
 	{ .compatible = "cypress,cyw4373a0-bt", .data = &cyw4373a0_device_data },
 	{ .compatible = "infineon,cyw55572-bt", .data = &cyw55572_device_data },
 	{ },
diff --git a/drivers/bluetooth/hci_bcsp.c b/drivers/bluetooth/hci_bcsp.c
index 0323db21c42877..ef71a349e7772a 100644
--- a/drivers/bluetooth/hci_bcsp.c
+++ b/drivers/bluetooth/hci_bcsp.c
@@ -25,6 +25,7 @@
 #include <linux/ioctl.h>
 #include <linux/skbuff.h>
 #include <linux/bitrev.h>
+#include <linux/crc-ccitt.h>
 #include <linux/unaligned.h>
 
 #include <net/bluetooth/bluetooth.h>
@@ -75,34 +76,13 @@ struct bcsp_struct {
 
 /* ---- BCSP CRC calculation ---- */
 
-/* Table for calculating CRC for polynomial 0x1021, LSB processed first,
- * initial value 0xffff, bits shifted in reverse order.
- */
-
-static const u16 crc_table[] = {
-	0x0000, 0x1081, 0x2102, 0x3183,
-	0x4204, 0x5285, 0x6306, 0x7387,
-	0x8408, 0x9489, 0xa50a, 0xb58b,
-	0xc60c, 0xd68d, 0xe70e, 0xf78f
-};
-
 /* Initialise the crc calculator */
 #define BCSP_CRC_INIT(x) x = 0xffff
 
-/* Update crc with next data byte
- *
- * Implementation note
- *     The data byte is treated as two nibbles.  The crc is generated
- *     in reverse, i.e., bits are fed into the register from the top.
- */
+/* Update crc with next data byte */
 static void bcsp_crc_update(u16 *crc, u8 d)
 {
-	u16 reg = *crc;
-
-	reg = (reg >> 4) ^ crc_table[(reg ^ d) & 0x000f];
-	reg = (reg >> 4) ^ crc_table[(reg ^ (d >> 4)) & 0x000f];
-
-	*crc = reg;
+	*crc = crc_ccitt_byte(*crc, d);
 }
 
 /* ---- BCSP core ---- */
diff --git a/drivers/bluetooth/hci_h4.c b/drivers/bluetooth/hci_h4.c
index 7673727074985c..cbdf51458ec3c4 100644
--- a/drivers/bluetooth/hci_h4.c
+++ b/drivers/bluetooth/hci_h4.c
@@ -29,6 +29,7 @@
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
+#include <net/bluetooth/hci_h4.h>
 
 #include "hci_uart.h"
 
@@ -112,8 +113,9 @@ static int h4_recv(struct hci_uart *hu, const void *data, int count)
 	if (!h4)
 		return -ENODEV;
 
-	h4->rx_skb = h4_recv_buf(hu, h4->rx_skb, data, count,
-				 h4_recv_pkts, ARRAY_SIZE(h4_recv_pkts));
+	h4->rx_skb = h4_recv_skb(hu->hdev, &hu->alignment, &hu->padding,
+				 h4->rx_skb, data, count, h4_recv_pkts,
+				 ARRAY_SIZE(h4_recv_pkts));
 	if (IS_ERR(h4->rx_skb)) {
 		int err = PTR_ERR(h4->rx_skb);
 		bt_dev_err(hu->hdev, "Frame reassembly failed (%d)", err);
@@ -155,120 +157,7 @@ struct sk_buff *h4_recv_buf(struct hci_uart *hu, struct sk_buff *skb,
 			    const unsigned char *buffer, int count,
 			    const struct h4_recv_pkt *pkts, int pkts_count)
 {
-	u8 alignment = hu->alignment ? hu->alignment : 1;
-	struct hci_dev *hdev = hu->hdev;
-
-	/* Check for error from previous call */
-	if (IS_ERR(skb))
-		skb = NULL;
-
-	while (count) {
-		int i, len;
-
-		/* remove padding bytes from buffer */
-		for (; hu->padding && count > 0; hu->padding--) {
-			count--;
-			buffer++;
-		}
-		if (!count)
-			break;
-
-		if (!skb) {
-			for (i = 0; i < pkts_count; i++) {
-				if (buffer[0] != (&pkts[i])->type)
-					continue;
-
-				skb = bt_skb_alloc((&pkts[i])->maxlen,
-						   GFP_ATOMIC);
-				if (!skb)
-					return ERR_PTR(-ENOMEM);
-
-				hci_skb_pkt_type(skb) = (&pkts[i])->type;
-				hci_skb_expect(skb) = (&pkts[i])->hlen;
-				break;
-			}
-
-			/* Check for invalid packet type */
-			if (!skb)
-				return ERR_PTR(-EILSEQ);
-
-			count -= 1;
-			buffer += 1;
-		}
-
-		len = min_t(uint, hci_skb_expect(skb) - skb->len, count);
-		skb_put_data(skb, buffer, len);
-
-		count -= len;
-		buffer += len;
-
-		/* Check for partial packet */
-		if (skb->len < hci_skb_expect(skb))
-			continue;
-
-		for (i = 0; i < pkts_count; i++) {
-			if (hci_skb_pkt_type(skb) == (&pkts[i])->type)
-				break;
-		}
-
-		if (i >= pkts_count) {
-			kfree_skb(skb);
-			return ERR_PTR(-EILSEQ);
-		}
-
-		if (skb->len == (&pkts[i])->hlen) {
-			u16 dlen;
-
-			switch ((&pkts[i])->lsize) {
-			case 0:
-				/* No variable data length */
-				dlen = 0;
-				break;
-			case 1:
-				/* Single octet variable length */
-				dlen = skb->data[(&pkts[i])->loff];
-				hci_skb_expect(skb) += dlen;
-
-				if (skb_tailroom(skb) < dlen) {
-					kfree_skb(skb);
-					return ERR_PTR(-EMSGSIZE);
-				}
-				break;
-			case 2:
-				/* Double octet variable length */
-				dlen = get_unaligned_le16(skb->data +
-							  (&pkts[i])->loff);
-				hci_skb_expect(skb) += dlen;
-
-				if (skb_tailroom(skb) < dlen) {
-					kfree_skb(skb);
-					return ERR_PTR(-EMSGSIZE);
-				}
-				break;
-			default:
-				/* Unsupported variable length */
-				kfree_skb(skb);
-				return ERR_PTR(-EILSEQ);
-			}
-
-			if (!dlen) {
-				hu->padding = (skb->len + 1) % alignment;
-				hu->padding = (alignment - hu->padding) % alignment;
-
-				/* No more data, complete frame */
-				(&pkts[i])->recv(hdev, skb);
-				skb = NULL;
-			}
-		} else {
-			hu->padding = (skb->len + 1) % alignment;
-			hu->padding = (alignment - hu->padding) % alignment;
-
-			/* Complete frame */
-			(&pkts[i])->recv(hdev, skb);
-			skb = NULL;
-		}
-	}
-
-	return skb;
+	return h4_recv_skb(hu->hdev, &hu->alignment, &hu->padding, skb, buffer,
+			   count, pkts, pkts_count);
 }
 EXPORT_SYMBOL_GPL(h4_recv_buf);
diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c
index b1999e14aadef1..37f4d9327e724d 100644
--- a/drivers/bluetooth/hci_h5.c
+++ b/drivers/bluetooth/hci_h5.c
@@ -543,12 +543,8 @@ static void h5_unslip_one_byte(struct h5 *h5, unsigned char c)
 	const u8 delim = SLIP_DELIMITER, esc = SLIP_ESC;
 	const u8 *byte = &c;
 
-	if (!test_bit(H5_RX_ESC, &h5->flags) && c == SLIP_ESC) {
-		set_bit(H5_RX_ESC, &h5->flags);
-		return;
-	}
-
-	if (test_and_clear_bit(H5_RX_ESC, &h5->flags)) {
+	if (test_bit(H5_RX_ESC, &h5->flags)) {
+		clear_bit(H5_RX_ESC, &h5->flags);
 		switch (c) {
 		case SLIP_ESC_DELIM:
 			byte = &delim;
@@ -561,6 +557,9 @@ static void h5_unslip_one_byte(struct h5 *h5, unsigned char c)
 			h5_reset_rx(h5);
 			return;
 		}
+	} else if (c == SLIP_ESC) {
+		set_bit(H5_RX_ESC, &h5->flags);
+		return;
 	}
 
 	skb_put_data(h5->rx_skb, byte, 1);
diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c
index ab744001dafc43..35adf869cc1f28 100644
--- a/drivers/bluetooth/hci_ll.c
+++ b/drivers/bluetooth/hci_ll.c
@@ -629,7 +629,7 @@ static int ll_setup(struct hci_uart *hu)
 		gpiod_set_value_cansleep(lldev->enable_gpio, 0);
 		msleep(5);
 		gpiod_set_value_cansleep(lldev->enable_gpio, 1);
-		mdelay(100);
+		msleep(100);
 		err = serdev_device_wait_for_cts(serdev, true, 200);
 		if (err) {
 			bt_dev_err(hu->hdev, "Failed to get CTS");
diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c
index d2eaf2f12aa278..48ba017e219bc3 100644
--- a/drivers/bluetooth/hci_serdev.c
+++ b/drivers/bluetooth/hci_serdev.c
@@ -395,20 +395,54 @@ EXPORT_SYMBOL_GPL(hci_uart_register_device_priv);
 void hci_uart_unregister_device(struct hci_uart *hu)
 {
 	struct hci_dev *hdev = hu->hdev;
+	bool proto_ready;
 
+	/* Wait for init_ready to finish to prevent registration races */
 	cancel_work_sync(&hu->init_ready);
-	if (test_bit(HCI_UART_REGISTERED, &hu->flags))
-		hci_unregister_dev(hdev);
-	hci_free_dev(hdev);
 
+	proto_ready = test_bit(HCI_UART_PROTO_READY, &hu->flags);
+	if (proto_ready) {
+		/* Clear HCI_UART_PROTO_READY under the write lock so a
+		 * concurrent hci_uart_tx_wakeup() cannot re-schedule
+		 * write_work via the write_wakeup callback once the device
+		 * is torn down.
+		 */
+		percpu_down_write(&hu->proto_lock);
+		clear_bit(HCI_UART_PROTO_READY, &hu->flags);
+		percpu_up_write(&hu->proto_lock);
+	}
+
+	/* Unconditionally cancel write_work AFTER clearing PROTO_READY.
+	 * This ensures that concurrent protocol timers cannot requeue
+	 * write_work, permanently preventing double-free races and UAFs,
+	 * and guarantees no write_work is in flight before the serdev
+	 * device is closed.
+	 */
 	cancel_work_sync(&hu->write_work);
 
+	/* Free any partially transmitted frame left over by write_work now
+	 * that the transmit path is fully quiesced. hci_uart_close() would
+	 * skip hci_uart_flush() because HCI_UART_PROTO_READY is cleared.
+	 */
+	if (hu->tx_skb) {
+		kfree_skb(hu->tx_skb);
+		hu->tx_skb = NULL;
+	}
+
+	if (test_bit(HCI_UART_REGISTERED, &hu->flags))
+		hci_unregister_dev(hdev);
+
+	/* Close the protocol before freeing hdev (intrinsically purges queues).
+	 * Some protocol close handlers (e.g. qca_close) may still access the
+	 * serdev device, so keep the serdev port open until this completes.
+	 */
 	hu->proto->close(hu);
 
-	if (test_bit(HCI_UART_PROTO_READY, &hu->flags)) {
-		clear_bit(HCI_UART_PROTO_READY, &hu->flags);
+	if (proto_ready)
 		serdev_device_close(hu->serdev);
-	}
+
+	hci_free_dev(hdev);
+
 	percpu_free_rwsem(&hu->proto_lock);
 }
 EXPORT_SYMBOL_GPL(hci_uart_unregister_device);
diff --git a/drivers/bluetooth/hci_uart.h b/drivers/bluetooth/hci_uart.h
index 48ac7ca9334e57..7fbe8dffab988d 100644
--- a/drivers/bluetooth/hci_uart.h
+++ b/drivers/bluetooth/hci_uart.h
@@ -8,6 +8,8 @@
  *  Copyright (C) 2004-2005  Marcel Holtmann <marcel@holtmann.org>
  */
 
+#include <net/bluetooth/hci_h4.h>
+
 #ifndef N_HCI
 #define N_HCI	15
 #endif
@@ -121,43 +123,6 @@ void hci_uart_set_flow_control(struct hci_uart *hu, bool enable);
 void hci_uart_set_speeds(struct hci_uart *hu, unsigned int init_speed,
 			 unsigned int oper_speed);
 
-struct h4_recv_pkt {
-	u8  type;	/* Packet type */
-	u8  hlen;	/* Header length */
-	u8  loff;	/* Data length offset in header */
-	u8  lsize;	/* Data length field size */
-	u16 maxlen;	/* Max overall packet length */
-	int (*recv)(struct hci_dev *hdev, struct sk_buff *skb);
-};
-
-#define H4_RECV_ACL \
-	.type = HCI_ACLDATA_PKT, \
-	.hlen = HCI_ACL_HDR_SIZE, \
-	.loff = 2, \
-	.lsize = 2, \
-	.maxlen = HCI_MAX_FRAME_SIZE \
-
-#define H4_RECV_SCO \
-	.type = HCI_SCODATA_PKT, \
-	.hlen = HCI_SCO_HDR_SIZE, \
-	.loff = 2, \
-	.lsize = 1, \
-	.maxlen = HCI_MAX_SCO_SIZE
-
-#define H4_RECV_EVENT \
-	.type = HCI_EVENT_PKT, \
-	.hlen = HCI_EVENT_HDR_SIZE, \
-	.loff = 1, \
-	.lsize = 1, \
-	.maxlen = HCI_MAX_EVENT_SIZE
-
-#define H4_RECV_ISO \
-	.type = HCI_ISODATA_PKT, \
-	.hlen = HCI_ISO_HDR_SIZE, \
-	.loff = 2, \
-	.lsize = 2, \
-	.maxlen = HCI_MAX_FRAME_SIZE \
-
 #ifdef CONFIG_BT_HCIUART_H4
 int h4_init(void);
 int h4_deinit(void);
diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c
index c20d54088c8c4e..8c55b538deefdb 100644
--- a/drivers/bluetooth/virtio_bt.c
+++ b/drivers/bluetooth/virtio_bt.c
@@ -315,12 +315,12 @@ static int virtbt_probe(struct virtio_device *vdev)
 
 	err = virtio_find_vqs(vdev, VIRTBT_NUM_VQS, vbt->vqs, vqs_info, NULL);
 	if (err)
-		return err;
+		goto err_free_vbt;
 
 	hdev = hci_alloc_dev();
 	if (!hdev) {
 		err = -ENOMEM;
-		goto failed;
+		goto err_del_vqs;
 	}
 
 	vbt->hdev = hdev;
@@ -390,20 +390,25 @@ static int virtbt_probe(struct virtio_device *vdev)
 	if (hci_register_dev(hdev) < 0) {
 		hci_free_dev(hdev);
 		err = -EBUSY;
-		goto failed;
+		goto err_del_vqs;
 	}
 
 	virtio_device_ready(vdev);
 	err = virtbt_open_vdev(vbt);
-	if (err)
-		goto open_failed;
+	if (err) {
+		hci_unregister_dev(hdev);
+		virtio_reset_device(vdev);
+		virtbt_close_vdev(vbt);
+		hci_free_dev(hdev);
+		goto err_del_vqs;
+	}
 
 	return 0;
 
-open_failed:
-	hci_free_dev(hdev);
-failed:
+err_del_vqs:
 	vdev->config->del_vqs(vdev);
+err_free_vbt:
+	kfree(vbt);
 	return err;
 }
 
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index 66a4fa73c5b864..5878eafd8d4462 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -797,6 +797,10 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc,
 	}
 	dev_set_name(&mc_dev->dev, "%s.%d", obj_desc->type, obj_desc->id);
 
+	mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK;
+	mc_dev->dev.dma_mask = &mc_dev->dma_mask;
+	mc_dev->dev.coherent_dma_mask = mc_dev->dma_mask;
+
 	if (strcmp(obj_desc->type, "dprc") == 0) {
 		struct fsl_mc_io *mc_io2;
 
@@ -838,9 +842,6 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc,
 		 * parent's ICID.
 		 */
 		mc_dev->icid = parent_mc_dev->icid;
-		mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK;
-		mc_dev->dev.dma_mask = &mc_dev->dma_mask;
-		mc_dev->dev.coherent_dma_mask = mc_dev->dma_mask;
 	}
 
 	/*
@@ -1264,35 +1265,43 @@ static int __init fsl_mc_bus_driver_init(void)
 	error = bus_register(&fsl_mc_bus_type);
 	if (error < 0) {
 		pr_err("bus type registration failed: %d\n", error);
-		goto error_cleanup_cache;
+		return error;
 	}
 
-	error = platform_driver_register(&fsl_mc_bus_driver);
-	if (error < 0) {
-		pr_err("platform_driver_register() failed: %d\n", error);
+	error = bus_register_notifier(&platform_bus_type, &fsl_mc_nb);
+	if (error < 0)
 		goto error_cleanup_bus;
-	}
+
+	return 0;
+
+error_cleanup_bus:
+	bus_unregister(&fsl_mc_bus_type);
+	return error;
+}
+postcore_initcall(fsl_mc_bus_driver_init);
+
+static int __init fsl_mc_bus_drivers_init(void)
+{
+	int error;
 
 	error = dprc_driver_init();
 	if (error < 0)
-		goto error_cleanup_driver;
+		return error;
 
 	error = fsl_mc_allocator_driver_init();
 	if (error < 0)
 		goto error_cleanup_dprc_driver;
 
-	return bus_register_notifier(&platform_bus_type, &fsl_mc_nb);
+	error = platform_driver_register(&fsl_mc_bus_driver);
+	if (error < 0) {
+		pr_err("platform_driver_register() failed: %d\n", error);
+		goto error_cleanup_dprc_driver;
+	}
+
+	return 0;
 
 error_cleanup_dprc_driver:
 	dprc_driver_exit();
-
-error_cleanup_driver:
-	platform_driver_unregister(&fsl_mc_bus_driver);
-
-error_cleanup_bus:
-	bus_unregister(&fsl_mc_bus_type);
-
-error_cleanup_cache:
 	return error;
 }
-postcore_initcall(fsl_mc_bus_driver_init);
+subsys_initcall_sync(fsl_mc_bus_drivers_init);
diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c
index be38b43803deae..7d760bfa63c117 100644
--- a/drivers/bus/fsl-mc/fsl-mc-msi.c
+++ b/drivers/bus/fsl-mc/fsl-mc-msi.c
@@ -57,7 +57,7 @@ static void fsl_mc_write_msi_msg(struct msi_desc *msi_desc, struct msi_msg *msg)
 		}
 	} else {
 		/*
-		 * IRQ is for for a child device of mc_bus_dev
+		 * IRQ is for a child device of mc_bus_dev
 		 */
 		error = dprc_set_obj_irq(mc_bus_dev->mc_io,
 					 MC_CMD_FLAG_INTR_DIS | MC_CMD_FLAG_PRI,
diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c
index db7ee6547eefa7..da498570113a3a 100644
--- a/drivers/bus/mhi/ep/main.c
+++ b/drivers/bus/mhi/ep/main.c
@@ -1534,8 +1534,8 @@ int mhi_ep_register_controller(struct mhi_ep_cntrl *mhi_cntrl,
 
 	mhi_cntrl->index = ret;
 
-	irq_set_status_flags(mhi_cntrl->irq, IRQ_NOAUTOEN);
-	ret = request_irq(mhi_cntrl->irq, mhi_ep_irq, IRQF_TRIGGER_HIGH,
+	ret = request_irq(mhi_cntrl->irq, mhi_ep_irq,
+			  IRQF_TRIGGER_HIGH | IRQF_NO_AUTOEN,
 			  "doorbell_irq", mhi_cntrl);
 	if (ret) {
 		dev_err(mhi_cntrl->cntrl_dev, "Failed to request Doorbell IRQ\n");
diff --git a/drivers/bus/mhi/host/debugfs.c b/drivers/bus/mhi/host/debugfs.c
index 39e45748a24c0c..93f458a2557dc3 100644
--- a/drivers/bus/mhi/host/debugfs.c
+++ b/drivers/bus/mhi/host/debugfs.c
@@ -72,7 +72,7 @@ static int mhi_debugfs_events_show(struct seq_file *m, void *d)
 		seq_printf(m, " rp: 0x%llx wp: 0x%llx", le64_to_cpu(er_ctxt->rp),
 			   le64_to_cpu(er_ctxt->wp));
 
-		seq_printf(m, " local rp: 0x%pK db: 0x%pad\n", ring->rp,
+		seq_printf(m, " local rp: 0x%p db: 0x%pad\n", ring->rp,
 			   &mhi_event->db_cfg.db_val);
 	}
 
@@ -120,7 +120,7 @@ static int mhi_debugfs_channels_show(struct seq_file *m, void *d)
 			   le64_to_cpu(chan_ctxt->rbase), le64_to_cpu(chan_ctxt->rlen),
 			   le64_to_cpu(chan_ctxt->rp), le64_to_cpu(chan_ctxt->wp));
 
-		seq_printf(m, " local rp: 0x%pK local wp: 0x%pK db: 0x%pad\n",
+		seq_printf(m, " local rp: 0x%p local wp: 0x%p db: 0x%pad\n",
 			   ring->rp, ring->wp,
 			   &mhi_chan->db_cfg.db_val);
 	}
diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c
index b636e2c23b4d6a..5471a6a0f12859 100644
--- a/drivers/bus/mhi/host/pci_generic.c
+++ b/drivers/bus/mhi/host/pci_generic.c
@@ -322,6 +322,8 @@ static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = {
 	MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0),
 	MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0),
 	MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0),
+	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
+	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
 	MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0),
 	MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
 	MHI_CHANNEL_CONFIG_UL(46, "IP_SW0", 64, 2),
@@ -732,6 +734,8 @@ static const struct mhi_channel_config mhi_sierra_em919x_channels[] = {
 	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 128, 0),
 	MHI_CHANNEL_CONFIG_UL(14, "QMI", 32, 0),
 	MHI_CHANNEL_CONFIG_DL(15, "QMI", 32, 0),
+	MHI_CHANNEL_CONFIG_UL(18, "IP_CTRL", 32, 0),
+	MHI_CHANNEL_CONFIG_DL(19, "IP_CTRL", 32, 0),
 	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
 	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
 	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 512, 1),
@@ -1097,6 +1101,7 @@ struct mhi_pci_device {
 	struct work_struct recovery_work;
 	struct timer_list health_check_timer;
 	unsigned long status;
+	bool no_m3;
 	bool reset_on_remove;
 };
 
@@ -1409,6 +1414,8 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	mhi_cntrl->mru = info->mru_default;
 	mhi_cntrl->name = info->name;
 
+	mhi_pdev->no_m3 = info->no_m3;
+
 	if (!pdev->is_virtfn)
 		mhi_pdev->reset_on_remove = info->reset_on_remove;
 
@@ -1497,7 +1504,7 @@ static void mhi_pci_remove(struct pci_dev *pdev)
 	}
 
 	/* balancing probe put_noidle */
-	if (pci_pme_capable(pdev, PCI_D3hot))
+	if (pci_pme_capable(pdev, PCI_D3hot) && !mhi_pdev->no_m3)
 		pm_runtime_get_noresume(&pdev->dev);
 
 	if (mhi_pdev->reset_on_remove)
diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c
index 4f1fd389260f53..c717e35ea2afcc 100644
--- a/drivers/cdrom/cdrom.c
+++ b/drivers/cdrom/cdrom.c
@@ -345,9 +345,8 @@ int cdrom_dummy_generic_packet(struct cdrom_device_info *cdi,
 			       struct packet_command *cgc)
 {
 	if (cgc->sshdr) {
-		cgc->sshdr->sense_key = 0x05;
-		cgc->sshdr->asc = 0x20;
-		cgc->sshdr->ascq = 0x00;
+		cgc->sshdr->sense_key = ILLEGAL_REQUEST;
+		cgc->sshdr->sense_code = INVALID_COMMAND_OP_CODE;
 	}
 
 	cgc->stat = -EIO;
@@ -2979,9 +2978,8 @@ static noinline int mmc_ioctl_cdrom_read_data(struct cdrom_device_info *cdi,
 	cgc->sshdr = &sshdr;
 	cgc->data_direction = CGC_DATA_READ;
 	ret = cdrom_read_block(cdi, cgc, lba, 1, format, blocksize);
-	if (ret && sshdr.sense_key == 0x05 &&
-	    sshdr.asc == 0x20 &&
-	    sshdr.ascq == 0x00) {
+	if (ret && sshdr.sense_key == ILLEGAL_REQUEST &&
+	    sshdr.sense_code == INVALID_COMMAND_OP_CODE) {
 		/*
 		 * SCSI-II devices are not required to support
 		 * READ_CD, so let's try switching block size
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index a46d7bf7c4c828..bb3bf66937b37b 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -3,7 +3,7 @@
 # Makefile for the kernel character device drivers.
 #
 
-obj-y				+= mem.o random.o
+obj-y				+= random.o
 obj-$(CONFIG_TTY_PRINTK)	+= ttyprintk.o
 obj-y				+= misc.o
 obj-$(CONFIG_TEST_MISC_MINOR)	+= misc_minor_kunit.o
diff --git a/drivers/char/agp/backend.c b/drivers/char/agp/backend.c
index 8983addca6959b..5ce4ea4eb895b6 100644
--- a/drivers/char/agp/backend.c
+++ b/drivers/char/agp/backend.c
@@ -31,7 +31,6 @@
 #include <linux/pci.h>
 #include <linux/init.h>
 #include <linux/slab.h>
-#include <linux/pagemap.h>
 #include <linux/miscdevice.h>
 #include <linux/pm.h>
 #include <linux/agp_backend.h>
diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index 41752ed2b0759e..417f007967a484 100644
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -29,7 +29,6 @@
  */
 #include <linux/module.h>
 #include <linux/pci.h>
-#include <linux/pagemap.h>
 #include <linux/miscdevice.h>
 #include <linux/pm.h>
 #include <linux/agp_backend.h>
diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index 3111e320b2c5af..4c9a79a0bcc5a6 100644
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -7,7 +7,6 @@
 #include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
-#include <linux/pagemap.h>
 #include <linux/agp_backend.h>
 #include <asm/smp.h>
 #include "agp.h"
diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index bcc26785175d39..f589a6e02c6506 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -18,7 +18,6 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/kernel.h>
-#include <linux/pagemap.h>
 #include <linux/agp_backend.h>
 #include <linux/iommu.h>
 #include <linux/delay.h>
diff --git a/drivers/char/agp/uninorth-agp.c b/drivers/char/agp/uninorth-agp.c
index 08175fe2f530dc..229294c15b40b0 100644
--- a/drivers/char/agp/uninorth-agp.c
+++ b/drivers/char/agp/uninorth-agp.c
@@ -7,10 +7,11 @@
 #include <linux/pci.h>
 #include <linux/slab.h>
 #include <linux/init.h>
-#include <linux/pagemap.h>
 #include <linux/agp_backend.h>
 #include <linux/delay.h>
 #include <linux/vmalloc.h>
+
+#include <asm/cacheflush.h>
 #include <asm/uninorth.h>
 #include <asm/prom.h>
 #include <asm/pmac_feature.h>
diff --git a/drivers/char/hw_random/arm_smccc_trng.c b/drivers/char/hw_random/arm_smccc_trng.c
index dcb8e7f37f25c6..01c1a30cbab8d7 100644
--- a/drivers/char/hw_random/arm_smccc_trng.c
+++ b/drivers/char/hw_random/arm_smccc_trng.c
@@ -16,8 +16,10 @@
 #include <linux/device.h>
 #include <linux/hw_random.h>
 #include <linux/module.h>
-#include <linux/platform_device.h>
 #include <linux/arm-smccc.h>
+#include <linux/arm-smccc-bus.h>
+
+#include <asm/archrandom.h>
 
 #ifdef CONFIG_ARM64
 #define ARM_SMCCC_TRNG_RND	ARM_SMCCC_TRNG_RND64
@@ -94,29 +96,37 @@ static int smccc_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
 	return copied;
 }
 
-static int smccc_trng_probe(struct platform_device *pdev)
+static int smccc_trng_probe(struct arm_smccc_device *sdev)
 {
 	struct hwrng *trng;
 
-	trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL);
+	/* validate the minimum version requirement */
+	if (!smccc_probe_trng())
+		return -ENODEV;
+
+	trng = devm_kzalloc(&sdev->dev, sizeof(*trng), GFP_KERNEL);
 	if (!trng)
 		return -ENOMEM;
 
 	trng->name = "smccc_trng";
 	trng->read = smccc_trng_read;
 
-	return devm_hwrng_register(&pdev->dev, trng);
+	return devm_hwrng_register(&sdev->dev, trng);
 }
 
-static struct platform_driver smccc_trng_driver = {
-	.driver = {
-		.name		= "smccc_trng",
-	},
-	.probe		= smccc_trng_probe,
+static const struct arm_smccc_device_id smccc_trng_id_table[] = {
+	{ .func_id = ARM_SMCCC_TRNG_VERSION },
+	{}
+};
+MODULE_DEVICE_TABLE(arm_smccc, smccc_trng_id_table);
+
+static struct arm_smccc_driver smccc_trng_driver = {
+	.name	  = KBUILD_MODNAME,
+	.probe	  = smccc_trng_probe,
+	.id_table = smccc_trng_id_table,
 };
-module_platform_driver(smccc_trng_driver);
+module_arm_smccc_driver(smccc_trng_driver);
 
-MODULE_ALIAS("platform:smccc_trng");
 MODULE_AUTHOR("Andre Przywara");
 MODULE_DESCRIPTION("Arm SMCCC TRNG firmware interface support");
 MODULE_LICENSE("GPL");
diff --git a/drivers/char/hw_random/cctrng.c b/drivers/char/hw_random/cctrng.c
index a6925211c3b508..51290ea345c327 100644
--- a/drivers/char/hw_random/cctrng.c
+++ b/drivers/char/hw_random/cctrng.c
@@ -86,12 +86,7 @@ static inline u32 cc_ioread(struct cctrng_drvdata *drvdata, u32 reg)
 
 static int cc_trng_pm_get(struct device *dev)
 {
-	int rc = 0;
-
-	rc = pm_runtime_get_sync(dev);
-
-	/* pm_runtime_get_sync() can return 1 as a valid return code */
-	return (rc == 1 ? 0 : rc);
+	return pm_runtime_resume_and_get(dev);
 }
 
 static void cc_trng_pm_put_suspend(struct device *dev)
diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c
index 7693328f105646..0545a18701467e 100644
--- a/drivers/char/hw_random/imx-rngc.c
+++ b/drivers/char/hw_random/imx-rngc.c
@@ -266,7 +266,9 @@ static int __init imx_rngc_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	clk_prepare_enable(rngc->clk);
+	ret = clk_prepare_enable(rngc->clk);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "Cannot enable rng_clk\n");
 
 	ver_id = readl(rngc->base + RNGC_VER_ID);
 	rng_type = FIELD_GET(RNG_TYPE, ver_id);
@@ -338,9 +340,7 @@ static int imx_rngc_resume(struct device *dev)
 {
 	struct imx_rngc *rngc = dev_get_drvdata(dev);
 
-	clk_prepare_enable(rngc->clk);
-
-	return 0;
+	return clk_prepare_enable(rngc->clk);
 }
 
 static const struct dev_pm_ops imx_rngc_pm_ops = {
diff --git a/drivers/char/hw_random/jh7110-trng.c b/drivers/char/hw_random/jh7110-trng.c
index aee12caab57805..61eb4db7ac0144 100644
--- a/drivers/char/hw_random/jh7110-trng.c
+++ b/drivers/char/hw_random/jh7110-trng.c
@@ -320,9 +320,17 @@ static int starfive_trng_probe(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, PTR_ERR(trng->rst),
 				     "Error getting hardware reset line\n");
 
-	clk_prepare_enable(trng->hclk);
-	clk_prepare_enable(trng->ahb);
-	reset_control_deassert(trng->rst);
+	ret = clk_prepare_enable(trng->hclk);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(trng->ahb);
+	if (ret)
+		goto disable_hclk;
+
+	ret = reset_control_deassert(trng->rst);
+	if (ret)
+		goto disable_ahb;
 
 	trng->rng.name = dev_driver_string(&pdev->dev);
 	trng->rng.init = starfive_trng_init;
@@ -342,13 +350,18 @@ static int starfive_trng_probe(struct platform_device *pdev)
 		pm_runtime_disable(&pdev->dev);
 
 		reset_control_assert(trng->rst);
-		clk_disable_unprepare(trng->ahb);
-		clk_disable_unprepare(trng->hclk);
-
-		return dev_err_probe(&pdev->dev, ret, "Failed to register hwrng\n");
+		ret = dev_err_probe(&pdev->dev, ret, "Failed to register hwrng\n");
+		goto disable_ahb;
 	}
 
 	return 0;
+
+disable_ahb:
+	clk_disable_unprepare(trng->ahb);
+disable_hclk:
+	clk_disable_unprepare(trng->hclk);
+
+	return ret;
 }
 
 static int __maybe_unused starfive_trng_suspend(struct device *dev)
@@ -364,11 +377,17 @@ static int __maybe_unused starfive_trng_suspend(struct device *dev)
 static int __maybe_unused starfive_trng_resume(struct device *dev)
 {
 	struct starfive_trng *trng = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(trng->hclk);
+	if (ret)
+		return ret;
 
-	clk_prepare_enable(trng->hclk);
-	clk_prepare_enable(trng->ahb);
+	ret = clk_prepare_enable(trng->ahb);
+	if (ret)
+		clk_disable_unprepare(trng->hclk);
 
-	return 0;
+	return ret;
 }
 
 static const struct dev_pm_ops starfive_trng_pm_ops = {
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index 9f0545c6777419..22cde35ebe5dc7 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -530,7 +530,7 @@ static void omap_rng_remove(struct platform_device *pdev)
 	clk_disable_unprepare(priv->clk_reg);
 }
 
-static int __maybe_unused omap_rng_suspend(struct device *dev)
+static int omap_rng_suspend(struct device *dev)
 {
 	struct omap_rng_dev *priv = dev_get_drvdata(dev);
 
@@ -540,7 +540,7 @@ static int __maybe_unused omap_rng_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused omap_rng_resume(struct device *dev)
+static int omap_rng_resume(struct device *dev)
 {
 	struct omap_rng_dev *priv = dev_get_drvdata(dev);
 	int ret;
@@ -556,12 +556,12 @@ static int __maybe_unused omap_rng_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(omap_rng_pm, omap_rng_suspend, omap_rng_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(omap_rng_pm, omap_rng_suspend, omap_rng_resume);
 
 static struct platform_driver omap_rng_driver = {
 	.driver = {
 		.name		= "omap_rng",
-		.pm		= &omap_rng_pm,
+		.pm		= pm_sleep_ptr(&omap_rng_pm),
 		.of_match_table = of_match_ptr(omap_rng_of_match),
 	},
 	.probe		= omap_rng_probe,
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
deleted file mode 100644
index 63253d1de5d70b..00000000000000
--- a/drivers/char/mem.c
+++ /dev/null
@@ -1,778 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *  linux/drivers/char/mem.c
- *
- *  Copyright (C) 1991, 1992  Linus Torvalds
- *
- *  Added devfs support.
- *    Jan-11-1998, C. Scott Ananian <cananian@alumni.princeton.edu>
- *  Shared /dev/zero mmapping support, Feb 2000, Kanoj Sarcar <kanoj@sgi.com>
- */
-
-#include <linux/mm.h>
-#include <linux/miscdevice.h>
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-#include <linux/mman.h>
-#include <linux/random.h>
-#include <linux/init.h>
-#include <linux/tty.h>
-#include <linux/capability.h>
-#include <linux/ptrace.h>
-#include <linux/device.h>
-#include <linux/highmem.h>
-#include <linux/backing-dev.h>
-#include <linux/shmem_fs.h>
-#include <linux/splice.h>
-#include <linux/pfn.h>
-#include <linux/export.h>
-#include <linux/io.h>
-#include <linux/uio.h>
-#include <linux/uaccess.h>
-#include <linux/security.h>
-
-#define DEVMEM_MINOR	1
-#define DEVPORT_MINOR	4
-
-static inline unsigned long size_inside_page(unsigned long start,
-					     unsigned long size)
-{
-	unsigned long sz;
-
-	sz = PAGE_SIZE - (start & (PAGE_SIZE - 1));
-
-	return min(sz, size);
-}
-
-#ifndef ARCH_HAS_VALID_PHYS_ADDR_RANGE
-static inline int valid_phys_addr_range(phys_addr_t addr, size_t count)
-{
-	return addr + count <= __pa(high_memory);
-}
-
-static inline int valid_mmap_phys_addr_range(unsigned long pfn, size_t size)
-{
-	return 1;
-}
-#endif
-
-#ifdef CONFIG_STRICT_DEVMEM
-static inline int page_is_allowed(unsigned long pfn)
-{
-	return devmem_is_allowed(pfn);
-}
-#else
-static inline int page_is_allowed(unsigned long pfn)
-{
-	return 1;
-}
-#endif
-
-static inline bool should_stop_iteration(void)
-{
-	if (need_resched())
-		cond_resched();
-	return signal_pending(current);
-}
-
-/*
- * This funcion reads the *physical* memory. The f_pos points directly to the
- * memory location.
- */
-static ssize_t read_mem(struct file *file, char __user *buf,
-			size_t count, loff_t *ppos)
-{
-	phys_addr_t p = *ppos;
-	ssize_t read, sz;
-	void *ptr;
-	char *bounce;
-	int err;
-
-	if (p != *ppos)
-		return 0;
-
-	if (!valid_phys_addr_range(p, count))
-		return -EFAULT;
-	read = 0;
-#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPED
-	/* we don't have page 0 mapped on sparc and m68k.. */
-	if (p < PAGE_SIZE) {
-		sz = size_inside_page(p, count);
-		if (sz > 0) {
-			if (clear_user(buf, sz))
-				return -EFAULT;
-			buf += sz;
-			p += sz;
-			count -= sz;
-			read += sz;
-		}
-	}
-#endif
-
-	bounce = kmalloc(PAGE_SIZE, GFP_KERNEL);
-	if (!bounce)
-		return -ENOMEM;
-
-	while (count > 0) {
-		unsigned long remaining;
-		int allowed, probe;
-
-		sz = size_inside_page(p, count);
-
-		err = -EPERM;
-		allowed = page_is_allowed(p >> PAGE_SHIFT);
-		if (!allowed)
-			goto failed;
-
-		err = -EFAULT;
-		if (allowed == 2) {
-			/* Show zeros for restricted memory. */
-			remaining = clear_user(buf, sz);
-		} else {
-			/*
-			 * On ia64 if a page has been mapped somewhere as
-			 * uncached, then it must also be accessed uncached
-			 * by the kernel or data corruption may occur.
-			 */
-			ptr = xlate_dev_mem_ptr(p);
-			if (!ptr)
-				goto failed;
-
-			probe = copy_from_kernel_nofault(bounce, ptr, sz);
-			unxlate_dev_mem_ptr(p, ptr);
-			if (probe)
-				goto failed;
-
-			remaining = copy_to_user(buf, bounce, sz);
-		}
-
-		if (remaining)
-			goto failed;
-
-		buf += sz;
-		p += sz;
-		count -= sz;
-		read += sz;
-		if (should_stop_iteration())
-			break;
-	}
-	kfree(bounce);
-
-	*ppos += read;
-	return read;
-
-failed:
-	kfree(bounce);
-	return err;
-}
-
-static ssize_t write_mem(struct file *file, const char __user *buf,
-			 size_t count, loff_t *ppos)
-{
-	phys_addr_t p = *ppos;
-	ssize_t written, sz;
-	unsigned long copied;
-	void *ptr;
-
-	if (p != *ppos)
-		return -EFBIG;
-
-	if (!valid_phys_addr_range(p, count))
-		return -EFAULT;
-
-	written = 0;
-
-#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPED
-	/* we don't have page 0 mapped on sparc and m68k.. */
-	if (p < PAGE_SIZE) {
-		sz = size_inside_page(p, count);
-		/* Hmm. Do something? */
-		buf += sz;
-		p += sz;
-		count -= sz;
-		written += sz;
-	}
-#endif
-
-	while (count > 0) {
-		int allowed;
-
-		sz = size_inside_page(p, count);
-
-		allowed = page_is_allowed(p >> PAGE_SHIFT);
-		if (!allowed)
-			return -EPERM;
-
-		/* Skip actual writing when a page is marked as restricted. */
-		if (allowed == 1) {
-			/*
-			 * On ia64 if a page has been mapped somewhere as
-			 * uncached, then it must also be accessed uncached
-			 * by the kernel or data corruption may occur.
-			 */
-			ptr = xlate_dev_mem_ptr(p);
-			if (!ptr) {
-				if (written)
-					break;
-				return -EFAULT;
-			}
-
-			copied = copy_from_user(ptr, buf, sz);
-			unxlate_dev_mem_ptr(p, ptr);
-			if (copied) {
-				written += sz - copied;
-				if (written)
-					break;
-				return -EFAULT;
-			}
-		}
-
-		buf += sz;
-		p += sz;
-		count -= sz;
-		written += sz;
-		if (should_stop_iteration())
-			break;
-	}
-
-	*ppos += written;
-	return written;
-}
-
-int __weak phys_mem_access_prot_allowed(struct file *file,
-	unsigned long pfn, unsigned long size, pgprot_t *vma_prot)
-{
-	return 1;
-}
-
-#ifndef __HAVE_PHYS_MEM_ACCESS_PROT
-
-/*
- * Architectures vary in how they handle caching for addresses
- * outside of main memory.
- *
- */
-#ifdef pgprot_noncached
-static int uncached_access(struct file *file, phys_addr_t addr)
-{
-	/*
-	 * Accessing memory above the top the kernel knows about or through a
-	 * file pointer
-	 * that was marked O_DSYNC will be done non-cached.
-	 */
-	if (file->f_flags & O_DSYNC)
-		return 1;
-	return addr >= __pa(high_memory);
-}
-#endif
-
-static pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
-				     unsigned long size, pgprot_t vma_prot)
-{
-#ifdef pgprot_noncached
-	phys_addr_t offset = pfn << PAGE_SHIFT;
-
-	if (uncached_access(file, offset))
-		return pgprot_noncached(vma_prot);
-#endif
-	return vma_prot;
-}
-#endif
-
-#ifndef CONFIG_MMU
-static unsigned long get_unmapped_area_mem(struct file *file,
-					   unsigned long addr,
-					   unsigned long len,
-					   unsigned long pgoff,
-					   unsigned long flags)
-{
-	if (!valid_mmap_phys_addr_range(pgoff, len))
-		return (unsigned long) -EINVAL;
-	return pgoff << PAGE_SHIFT;
-}
-
-/* permit direct mmap, for read, write or exec */
-static unsigned memory_mmap_capabilities(struct file *file)
-{
-	return NOMMU_MAP_DIRECT |
-		NOMMU_MAP_READ | NOMMU_MAP_WRITE | NOMMU_MAP_EXEC;
-}
-
-static unsigned zero_mmap_capabilities(struct file *file)
-{
-	return NOMMU_MAP_COPY;
-}
-
-/* can't do an in-place private mapping if there's no MMU */
-static inline int private_mapping_ok(struct vm_area_desc *desc)
-{
-	return is_nommu_shared_vma_flags(&desc->vma_flags);
-}
-#else
-
-static inline int private_mapping_ok(struct vm_area_desc *desc)
-{
-	return 1;
-}
-#endif
-
-static const struct vm_operations_struct mmap_mem_ops = {
-#ifdef CONFIG_HAVE_IOREMAP_PROT
-	.access = generic_access_phys
-#endif
-};
-
-static int mmap_mem_prepare(struct vm_area_desc *desc)
-{
-	struct file *file = desc->file;
-	const size_t size = vma_desc_size(desc);
-	const phys_addr_t offset = (phys_addr_t)desc->pgoff << PAGE_SHIFT;
-
-	/* Does it even fit in phys_addr_t? */
-	if (offset >> PAGE_SHIFT != desc->pgoff)
-		return -EINVAL;
-
-	/* It's illegal to wrap around the end of the physical address space. */
-	if (offset + (phys_addr_t)size - 1 < offset)
-		return -EINVAL;
-
-	if (!valid_mmap_phys_addr_range(desc->pgoff, size))
-		return -EINVAL;
-
-	if (!private_mapping_ok(desc))
-		return -ENOSYS;
-
-	if (!range_is_allowed(desc->pgoff, size))
-		return -EPERM;
-
-	if (!phys_mem_access_prot_allowed(file, desc->pgoff, size,
-					  &desc->page_prot))
-		return -EINVAL;
-
-	desc->page_prot = phys_mem_access_prot(file, desc->pgoff,
-					       size,
-					       desc->page_prot);
-
-	desc->vm_ops = &mmap_mem_ops;
-
-	/* Remap-pfn-range will mark the range with the I/O flag. */
-	mmap_action_remap_full(desc, desc->pgoff);
-	desc->action.error_override = -EAGAIN;
-
-	return 0;
-}
-
-#ifdef CONFIG_DEVPORT
-static ssize_t read_port(struct file *file, char __user *buf,
-			 size_t count, loff_t *ppos)
-{
-	unsigned long i = *ppos;
-	char __user *tmp = buf;
-
-	if (!access_ok(buf, count))
-		return -EFAULT;
-	while (count-- > 0 && i < 65536) {
-		if (__put_user(inb(i), tmp) < 0)
-			return -EFAULT;
-		i++;
-		tmp++;
-	}
-	*ppos = i;
-	return tmp-buf;
-}
-
-static ssize_t write_port(struct file *file, const char __user *buf,
-			  size_t count, loff_t *ppos)
-{
-	unsigned long i = *ppos;
-	const char __user *tmp = buf;
-
-	if (!access_ok(buf, count))
-		return -EFAULT;
-	while (count-- > 0 && i < 65536) {
-		char c;
-
-		if (__get_user(c, tmp)) {
-			if (tmp > buf)
-				break;
-			return -EFAULT;
-		}
-		outb(c, i);
-		i++;
-		tmp++;
-	}
-	*ppos = i;
-	return tmp-buf;
-}
-#endif
-
-static ssize_t read_null(struct file *file, char __user *buf,
-			 size_t count, loff_t *ppos)
-{
-	return 0;
-}
-
-static ssize_t write_null(struct file *file, const char __user *buf,
-			  size_t count, loff_t *ppos)
-{
-	return count;
-}
-
-static ssize_t read_iter_null(struct kiocb *iocb, struct iov_iter *to)
-{
-	return 0;
-}
-
-static ssize_t write_iter_null(struct kiocb *iocb, struct iov_iter *from)
-{
-	size_t count = iov_iter_count(from);
-	iov_iter_advance(from, count);
-	return count;
-}
-
-static int pipe_to_null(struct pipe_inode_info *info, struct pipe_buffer *buf,
-			struct splice_desc *sd)
-{
-	return sd->len;
-}
-
-static ssize_t splice_write_null(struct pipe_inode_info *pipe, struct file *out,
-				 loff_t *ppos, size_t len, unsigned int flags)
-{
-	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_null);
-}
-
-static int uring_cmd_null(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
-{
-	return 0;
-}
-
-static ssize_t read_iter_zero(struct kiocb *iocb, struct iov_iter *iter)
-{
-	size_t written = 0;
-
-	while (iov_iter_count(iter)) {
-		size_t chunk = iov_iter_count(iter), n;
-
-		if (chunk > PAGE_SIZE)
-			chunk = PAGE_SIZE;	/* Just for latency reasons */
-		n = iov_iter_zero(chunk, iter);
-		if (!n && iov_iter_count(iter))
-			return written ? written : -EFAULT;
-		written += n;
-		if (signal_pending(current))
-			return written ? written : -ERESTARTSYS;
-		if (!need_resched())
-			continue;
-		if (iocb->ki_flags & IOCB_NOWAIT)
-			return written ? written : -EAGAIN;
-		cond_resched();
-	}
-	return written;
-}
-
-static ssize_t read_zero(struct file *file, char __user *buf,
-			 size_t count, loff_t *ppos)
-{
-	size_t cleared = 0;
-
-	while (count) {
-		size_t chunk = min_t(size_t, count, PAGE_SIZE);
-		size_t left;
-
-		left = clear_user(buf + cleared, chunk);
-		if (unlikely(left)) {
-			cleared += (chunk - left);
-			if (!cleared)
-				return -EFAULT;
-			break;
-		}
-		cleared += chunk;
-		count -= chunk;
-
-		if (signal_pending(current))
-			break;
-		cond_resched();
-	}
-
-	return cleared;
-}
-
-static int mmap_zero_prepare(struct vm_area_desc *desc)
-{
-#ifndef CONFIG_MMU
-	return -ENOSYS;
-#endif
-	if (vma_desc_test(desc, VMA_SHARED_BIT))
-		return shmem_zero_setup_desc(desc);
-
-	/*
-	 * This is a highly unique situation where we mark a MAP_PRIVATE mapping
-	 * of /dev/zero anonymous, despite it not being.
-	 */
-	vma_desc_set_anonymous(desc);
-	return 0;
-}
-
-#ifndef CONFIG_MMU
-static unsigned long get_unmapped_area_zero(struct file *file,
-				unsigned long addr, unsigned long len,
-				unsigned long pgoff, unsigned long flags)
-{
-	return -ENOSYS;
-}
-#else
-static unsigned long get_unmapped_area_zero(struct file *file,
-				unsigned long addr, unsigned long len,
-				unsigned long pgoff, unsigned long flags)
-{
-	if (flags & MAP_SHARED) {
-		/*
-		 * mmap_zero_prepare() will call shmem_zero_setup() to create a
-		 * file, so use shmem's get_unmapped_area in case it can be
-		 * huge; and pass NULL for file as in mmap.c's
-		 * get_unmapped_area(), so as not to confuse shmem with our
-		 * handle on "/dev/zero".
-		 */
-		return shmem_get_unmapped_area(NULL, addr, len, pgoff, flags);
-	}
-
-	/*
-	 * Otherwise flags & MAP_PRIVATE: with no shmem object beneath it,
-	 * attempt to map aligned to huge page size if possible, otherwise we
-	 * fall back to system page size mappings.
-	 */
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-	return thp_get_unmapped_area(file, addr, len, pgoff, flags);
-#else
-	return mm_get_unmapped_area(file, addr, len, pgoff, flags);
-#endif
-}
-#endif /* CONFIG_MMU */
-
-static ssize_t write_full(struct file *file, const char __user *buf,
-			  size_t count, loff_t *ppos)
-{
-	return -ENOSPC;
-}
-
-/*
- * Special lseek() function for /dev/null and /dev/zero.  Most notably, you
- * can fopen() both devices with "a" now.  This was previously impossible.
- * -- SRB.
- */
-static loff_t null_lseek(struct file *file, loff_t offset, int orig)
-{
-	return file->f_pos = 0;
-}
-
-/*
- * The memory devices use the full 32/64 bits of the offset, and so we cannot
- * check against negative addresses: they are ok. The return value is weird,
- * though, in that case (0).
- *
- * also note that seeking relative to the "end of file" isn't supported:
- * it has no meaning, so it returns -EINVAL.
- */
-static loff_t memory_lseek(struct file *file, loff_t offset, int orig)
-{
-	loff_t ret;
-
-	inode_lock(file_inode(file));
-	switch (orig) {
-	case SEEK_CUR:
-		offset += file->f_pos;
-		fallthrough;
-	case SEEK_SET:
-		/* to avoid userland mistaking f_pos=-9 as -EBADF=-9 */
-		if ((unsigned long long)offset >= -MAX_ERRNO) {
-			ret = -EOVERFLOW;
-			break;
-		}
-		file->f_pos = offset;
-		ret = file->f_pos;
-		force_successful_syscall_return();
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	inode_unlock(file_inode(file));
-	return ret;
-}
-
-static int open_port(struct inode *inode, struct file *filp)
-{
-	int rc;
-
-	if (!capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	rc = security_locked_down(LOCKDOWN_DEV_MEM);
-	if (rc)
-		return rc;
-
-	if (iminor(inode) != DEVMEM_MINOR)
-		return 0;
-
-	/*
-	 * Use a unified address space to have a single point to manage
-	 * revocations when drivers want to take over a /dev/mem mapped
-	 * range.
-	 */
-	filp->f_mapping = iomem_get_mapping();
-
-	return 0;
-}
-
-#define zero_lseek	null_lseek
-#define full_lseek      null_lseek
-#define write_zero	write_null
-#define write_iter_zero	write_iter_null
-#define splice_write_zero	splice_write_null
-#define open_mem	open_port
-
-static const struct file_operations __maybe_unused mem_fops = {
-	.llseek		= memory_lseek,
-	.read		= read_mem,
-	.write		= write_mem,
-	.mmap_prepare	= mmap_mem_prepare,
-	.open		= open_mem,
-#ifndef CONFIG_MMU
-	.get_unmapped_area = get_unmapped_area_mem,
-	.mmap_capabilities = memory_mmap_capabilities,
-#endif
-	.fop_flags	= FOP_UNSIGNED_OFFSET,
-};
-
-static const struct file_operations null_fops = {
-	.llseek		= null_lseek,
-	.read		= read_null,
-	.write		= write_null,
-	.read_iter	= read_iter_null,
-	.write_iter	= write_iter_null,
-	.splice_write	= splice_write_null,
-	.uring_cmd	= uring_cmd_null,
-};
-
-#ifdef CONFIG_DEVPORT
-static const struct file_operations port_fops = {
-	.llseek		= memory_lseek,
-	.read		= read_port,
-	.write		= write_port,
-	.open		= open_port,
-};
-#endif
-
-static const struct file_operations zero_fops = {
-	.llseek		= zero_lseek,
-	.write		= write_zero,
-	.read_iter	= read_iter_zero,
-	.read		= read_zero,
-	.write_iter	= write_iter_zero,
-	.splice_read	= copy_splice_read,
-	.splice_write	= splice_write_zero,
-	.mmap_prepare	= mmap_zero_prepare,
-	.get_unmapped_area = get_unmapped_area_zero,
-#ifndef CONFIG_MMU
-	.mmap_capabilities = zero_mmap_capabilities,
-#endif
-};
-
-static const struct file_operations full_fops = {
-	.llseek		= full_lseek,
-	.read_iter	= read_iter_zero,
-	.write		= write_full,
-	.splice_read	= copy_splice_read,
-};
-
-static const struct memdev {
-	const char *name;
-	const struct file_operations *fops;
-	fmode_t fmode;
-	umode_t mode;
-} devlist[] = {
-#ifdef CONFIG_DEVMEM
-	[DEVMEM_MINOR] = { "mem", &mem_fops, 0, 0 },
-#endif
-	[3] = { "null", &null_fops, FMODE_NOWAIT, 0666 },
-#ifdef CONFIG_DEVPORT
-	[4] = { "port", &port_fops, 0, 0 },
-#endif
-	[5] = { "zero", &zero_fops, FMODE_NOWAIT, 0666 },
-	[7] = { "full", &full_fops, 0, 0666 },
-	[8] = { "random", &random_fops, FMODE_NOWAIT, 0666 },
-	[9] = { "urandom", &urandom_fops, FMODE_NOWAIT, 0666 },
-#ifdef CONFIG_PRINTK
-	[11] = { "kmsg", &kmsg_fops, 0, 0644 },
-#endif
-};
-
-static int memory_open(struct inode *inode, struct file *filp)
-{
-	int minor;
-	const struct memdev *dev;
-
-	minor = iminor(inode);
-	if (minor >= ARRAY_SIZE(devlist))
-		return -ENXIO;
-
-	dev = &devlist[minor];
-	if (!dev->fops)
-		return -ENXIO;
-
-	filp->f_op = dev->fops;
-	filp->f_mode |= dev->fmode;
-
-	if (dev->fops->open)
-		return dev->fops->open(inode, filp);
-
-	return 0;
-}
-
-static const struct file_operations memory_fops = {
-	.open = memory_open,
-	.llseek = noop_llseek,
-};
-
-static char *mem_devnode(const struct device *dev, umode_t *mode)
-{
-	if (mode && devlist[MINOR(dev->devt)].mode)
-		*mode = devlist[MINOR(dev->devt)].mode;
-	return NULL;
-}
-
-static const struct class mem_class = {
-	.name		= "mem",
-	.devnode	= mem_devnode,
-};
-
-static int __init chr_dev_init(void)
-{
-	int retval;
-	int minor;
-
-	if (register_chrdev(MEM_MAJOR, "mem", &memory_fops))
-		printk("unable to get major %d for memory devs\n", MEM_MAJOR);
-
-	retval = class_register(&mem_class);
-	if (retval)
-		return retval;
-
-	for (minor = 1; minor < ARRAY_SIZE(devlist); minor++) {
-		if (!devlist[minor].name)
-			continue;
-
-		/*
-		 * Create /dev/port?
-		 */
-		if ((minor == DEVPORT_MINOR) && !arch_has_dev_port())
-			continue;
-
-		device_create(&mem_class, NULL, MKDEV(MEM_MAJOR, minor),
-			      NULL, devlist[minor].name);
-	}
-
-	return tty_init();
-}
-
-fs_initcall(chr_dev_init);
diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c
index f745a098908b37..1ccdbde98b69a8 100644
--- a/drivers/char/tpm/tpm-interface.c
+++ b/drivers/char/tpm/tpm-interface.c
@@ -268,7 +268,7 @@ ssize_t tpm_transmit_cmd(struct tpm_chip *chip, struct tpm_buf *buf,
 	int err;
 	ssize_t len;
 
-	len = tpm_transmit(chip, buf->data, PAGE_SIZE);
+	len = tpm_transmit(chip, buf->data, buf->capacity);
 	if (len <  0)
 		return len;
 
diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c
index 48cec39995fe22..ae22295df7981a 100644
--- a/drivers/char/tpm/tpm2-cmd.c
+++ b/drivers/char/tpm/tpm2-cmd.c
@@ -281,8 +281,10 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max)
 		}
 		tpm_buf_append_u16(buf, num_bytes);
 		err = tpm_buf_fill_hmac_session(chip, buf);
-		if (err)
+		if (err) {
+			tpm2_end_auth_session(chip);
 			return err;
+		}
 
 		err = tpm_transmit_cmd(chip, buf,
 				       offsetof(struct tpm2_get_random_out,
@@ -305,7 +307,7 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max)
 		out = (struct tpm2_get_random_out *)&buf->data[offset];
 		recd = min_t(u32, be16_to_cpu(out->size), num_bytes);
 		if (tpm_buf_length(buf) <
-		    TPM_HEADER_SIZE +
+		    offset +
 		    offsetof(struct tpm2_get_random_out, buffer) +
 		    recd) {
 			tpm2_end_auth_session(chip);
diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c
index cf8f1fd6790ba6..ca1e2bf424e140 100644
--- a/drivers/char/tpm/tpm2-sessions.c
+++ b/drivers/char/tpm/tpm2-sessions.c
@@ -975,6 +975,7 @@ static int tpm2_load_null(struct tpm_chip *chip, u32 *null_key)
 	/* Deduce from the name change TPM interference: */
 	dev_err(&chip->dev, "null key integrity check failed\n");
 	tpm2_flush_context(chip, tmp_null_key);
+	rc = -ENODEV;
 
 err:
 	if (rc) {
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index f9592fd9ec2bb6..983bf74b86fb4a 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -221,6 +221,7 @@ config COMMON_CLK_EN7523
 	bool "Clock driver for Airoha/EcoNet SoC system clocks"
 	depends on OF
 	depends on ARCH_AIROHA || ECONET || COMPILE_TEST
+	select GENERIC_PHY
 	default ARCH_AIROHA
 	help
 	  This driver provides the fixed clocks and gates present on Airoha
@@ -450,12 +451,14 @@ config COMMON_CLK_STM32H7
 
 config COMMON_CLK_MMP2
 	def_bool COMMON_CLK && (MACH_MMP2_DT || MACH_MMP3_DT)
+	select COMMON_CLK_MMP
 	help
 	  Support for Marvell MMP2 and MMP3 SoC clocks
 
 config COMMON_CLK_MMP2_AUDIO
         tristate "Clock driver for MMP2 Audio subsystem"
         depends on COMMON_CLK_MMP2 || COMPILE_TEST
+        select COMMON_CLK_MMP
         help
           This driver supports clocks for Audio subsystem on MMP2 SoC.
 
@@ -507,9 +510,11 @@ config COMMON_CLK_RPMI
 
 source "drivers/clk/actions/Kconfig"
 source "drivers/clk/analogbits/Kconfig"
+source "drivers/clk/anlogic/Kconfig"
 source "drivers/clk/aspeed/Kconfig"
 source "drivers/clk/bcm/Kconfig"
 source "drivers/clk/cix/Kconfig"
+source "drivers/clk/davinci/Kconfig"
 source "drivers/clk/eswin/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
 source "drivers/clk/imgtec/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index b18af485d7f031..d94de2988167f0 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -29,6 +29,7 @@ clk-test-y			:= clk_test.o \
 				   kunit_clk_assigned_sscs_zero.dtbo.o \
 				   kunit_clk_assigned_sscs_zero_consumer.dtbo.o \
 				   kunit_clk_hw_get_dev_of_node.dtbo.o \
+				   kunit_clk_parse_clkspec.dtbo.o \
 				   kunit_clk_parent_data_test.dtbo.o
 obj-$(CONFIG_COMMON_CLK)	+= clk-divider.o
 obj-$(CONFIG_CLK_DIVIDER_KUNIT_TEST) += clk-divider_test.o
@@ -125,6 +126,7 @@ obj-$(CONFIG_COMMON_CLK_XGENE)		+= clk-xgene.o
 # please keep this section sorted lexicographically by directory path name
 obj-y					+= actions/
 obj-y					+= analogbits/
+obj-y					+= anlogic/
 obj-y					+= aspeed/
 obj-$(CONFIG_COMMON_CLK_AT91)		+= at91/
 obj-$(CONFIG_ARCH_ARTPEC)		+= axis/
@@ -132,7 +134,7 @@ obj-$(CONFIG_ARC_PLAT_AXS10X)		+= axs10x/
 obj-y					+= bcm/
 obj-$(CONFIG_ARCH_BERLIN)		+= berlin/
 obj-y					+= cix/
-obj-$(CONFIG_ARCH_DAVINCI)		+= davinci/
+obj-$(CONFIG_COMMON_CLK_DAVINCI)	+= davinci/
 obj-$(CONFIG_COMMON_CLK_ESWIN)		+= eswin/
 obj-$(CONFIG_COMMON_CLK_HISI)		+= hisilicon/
 obj-y					+= imgtec/
@@ -143,12 +145,12 @@ obj-y					+= mediatek/
 obj-$(CONFIG_ARCH_MESON)		+= meson/
 obj-y					+= microchip/
 ifeq ($(CONFIG_COMMON_CLK), y)
-obj-$(CONFIG_ARCH_MMP)			+= mmp/
+obj-$(CONFIG_COMMON_CLK_MMP)		+= mmp/
 endif
 obj-y					+= mstar/
 obj-y					+= mvebu/
 obj-$(CONFIG_ARCH_MXS)			+= mxs/
-obj-$(CONFIG_ARCH_MA35)			+= nuvoton/
+obj-y					+= nuvoton/
 obj-$(CONFIG_COMMON_CLK_NXP)		+= nxp/
 obj-$(CONFIG_COMMON_CLK_PISTACHIO)	+= pistachio/
 obj-$(CONFIG_COMMON_CLK_PXA)		+= pxa/
diff --git a/drivers/clk/actions/owl-composite.h b/drivers/clk/actions/owl-composite.h
index 6d7c6f0c47c8b7..ca055599b2fceb 100644
--- a/drivers/clk/actions/owl-composite.h
+++ b/drivers/clk/actions/owl-composite.h
@@ -119,6 +119,5 @@ extern const struct clk_ops owl_comp_div_ops;
 extern const struct clk_ops owl_comp_fact_ops;
 extern const struct clk_ops owl_comp_fix_fact_ops;
 extern const struct clk_ops owl_comp_pass_ops;
-extern const struct clk_ops clk_fixed_factor_ops;
 
 #endif /* _OWL_COMPOSITE_H_ */
diff --git a/drivers/clk/actions/owl-fixed-factor.h b/drivers/clk/actions/owl-fixed-factor.h
index 3dfd7fd7d292a4..bbac5afadd670d 100644
--- a/drivers/clk/actions/owl-fixed-factor.h
+++ b/drivers/clk/actions/owl-fixed-factor.h
@@ -23,6 +23,4 @@
 					      _flags),			\
 	}
 
-extern const struct clk_ops clk_fixed_factor_ops;
-
 #endif /* _OWL_FIXED_FACTOR_H_ */
diff --git a/drivers/clk/actions/owl-pll.c b/drivers/clk/actions/owl-pll.c
index 869690b79cc16f..5e30ce35f2f5ef 100644
--- a/drivers/clk/actions/owl-pll.c
+++ b/drivers/clk/actions/owl-pll.c
@@ -186,7 +186,7 @@ static int owl_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 
 	regmap_write(common->regmap, pll_hw->reg, reg);
 
-	udelay(pll_hw->delay);
+	udelay((unsigned long)pll_hw->delay);
 
 	return 0;
 }
diff --git a/drivers/clk/anlogic/Kconfig b/drivers/clk/anlogic/Kconfig
new file mode 100644
index 00000000000000..b3b17540176d46
--- /dev/null
+++ b/drivers/clk/anlogic/Kconfig
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config ANLOGIC_DR1_CRU
+	tristate "Clock support for Anlogic DR1 series SoCs"
+	depends on ARCH_ANLOGIC || COMPILE_TEST
+	select AUXILIARY_BUS
+	default ARCH_ANLOGIC
+	help
+	  Say Y to enable clock controller unit support for Anlogic DR1 series
+	  SoCs.
+
+if ANLOGIC_DR1_CRU
+
+config ANLOGIC_DR1V90_CRU
+	tristate "Anlogic DR1V90 clock support"
+	depends on ARCH_ANLOGIC || COMPILE_TEST
+	default ARCH_ANLOGIC
+	help
+	  Support for the Clock and Reset Unit in Anlogic DR1V90 SoCs.
+
+endif
diff --git a/drivers/clk/anlogic/Makefile b/drivers/clk/anlogic/Makefile
new file mode 100644
index 00000000000000..fab7f29bb09758
--- /dev/null
+++ b/drivers/clk/anlogic/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+obj-$(CONFIG_ANLOGIC_DR1_CRU)		+= anlogic-dr1-cru.o
+anlogic-dr1-cru-y			+= cru_dr1.o
+
+obj-$(CONFIG_ANLOGIC_DR1V90_CRU)	+= anlogic-dr1v90-cru.o
+anlogic-dr1v90-cru-y			+= cru-dr1v90.o
diff --git a/drivers/clk/anlogic/cru-dr1v90.c b/drivers/clk/anlogic/cru-dr1v90.c
new file mode 100644
index 00000000000000..b513dfba3dbd97
--- /dev/null
+++ b/drivers/clk/anlogic/cru-dr1v90.c
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024-2025 Anlogic, Inc.
+ * Copyright (C) 2026 Junhui Liu <junhui.liu@pigmoral.tech>
+ */
+
+#include <linux/array_size.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "cru_dr1.h"
+
+#include <dt-bindings/clock/anlogic,dr1v90-cru.h>
+
+static const struct clk_div_table cru_div_table_24[] = {
+	{ 0xFFFFFF, 1 },  { 0x555555, 2 },  { 0x249249, 3 },  { 0x111111, 4 },
+	{ 0x084210, 5 },  { 0x041041, 6 },  { 0x020408, 7 },  { 0x010101, 8 },
+	{ 0x008040, 9 },  { 0x004010, 10 }, { 0x002004, 11 }, { 0x001001, 12 },
+	{ 0x000800, 13 }, { 0x000400, 14 }, { 0x000200, 15 }, { 0x000100, 16 },
+	{ 0x000080, 17 }, { 0x000040, 18 }, { 0x000020, 19 }, { 0x000010, 20 },
+	{ 0x000008, 21 }, { 0x000004, 22 }, { 0x000002, 23 }, { 0x000001, 24 },
+	{ /* sentinel */ }
+};
+
+static const struct clk_div_table cru_div_table_32[] = {
+	{ 0xFFFFFFFF, 1 },  { 0x55555555, 2 },	{ 0x24924924, 3 },
+	{ 0x11111111, 4 },  { 0x08421084, 5 },	{ 0x04104104, 6 },
+	{ 0x02040810, 7 },  { 0x01010101, 8 },	{ 0x00804020, 9 },
+	{ 0x00401004, 10 }, { 0x00200400, 11 }, { 0x00100100, 12 },
+	{ 0x00080040, 13 }, { 0x00040010, 14 }, { 0x00020004, 15 },
+	{ 0x00010001, 16 }, { 0x00008000, 17 }, { 0x00004000, 18 },
+	{ 0x00002000, 19 }, { 0x00001000, 20 }, { 0x00000800, 21 },
+	{ 0x00000400, 22 }, { 0x00000200, 23 }, { 0x00000100, 24 },
+	{ 0x00000080, 25 }, { 0x00000040, 26 }, { 0x00000020, 27 },
+	{ 0x00000010, 28 }, { 0x00000008, 29 }, { 0x00000004, 30 },
+	{ 0x00000002, 31 }, { 0x00000001, 32 }, { /* sentinel */ }
+};
+
+CLK_FIXED_FACTOR_FW_NAME(osc_div2, "osc_div2", "osc", 2, 1, 0);
+
+CRU_PLL_NM_DEFINE(cpu_pll, CRU_PARENT_NAME(osc), 0x120);
+CRU_PLL_C_DEFINE(cpu_pll_4x, CRU_PARENT_HW(cpu_pll), 0x14c);
+
+CRU_DIV_DEFINE(cpu_4x_div1, CRU_PARENT_HW(cpu_pll_4x), 0x010, 0, 24,
+	       cru_div_table_24, CLK_DIVIDER_READ_ONLY);
+CRU_DIV_DEFINE(cpu_4x_div2, CRU_PARENT_HW(cpu_pll_4x), 0x014, 0, 24,
+	       cru_div_table_24, CLK_DIVIDER_READ_ONLY);
+CRU_DIV_DEFINE(cpu_4x_div4, CRU_PARENT_HW(cpu_pll_4x), 0x018, 0, 24,
+	       cru_div_table_24, CLK_DIVIDER_READ_ONLY);
+
+CRU_PLL_NM_DEFINE(io_pll, CRU_PARENT_NAME(osc), 0x220);
+CRU_PLL_C_DEFINE(io_1000m, CRU_PARENT_HW(io_pll), 0x248);
+CRU_PLL_C_DEFINE(io_400m, CRU_PARENT_HW(io_pll), 0x24c);
+CRU_PLL_C_DEFINE(io_25m, CRU_PARENT_HW(io_pll), 0x250);
+CRU_PLL_C_DEFINE(io_80m, CRU_PARENT_HW(io_pll), 0x254);
+
+CRU_DIV_DEFINE(io_400m_div2, CRU_PARENT_HW(io_400m), 0x020, 0, 32,
+	       cru_div_table_32, CLK_DIVIDER_READ_ONLY);
+CRU_DIV_DEFINE(io_400m_div4, CRU_PARENT_HW(io_400m), 0x024, 0, 32,
+	       cru_div_table_32, CLK_DIVIDER_READ_ONLY);
+CRU_DIV_DEFINE(io_400m_div8, CRU_PARENT_HW(io_400m), 0x028, 0, 32,
+	       cru_div_table_32, CLK_DIVIDER_READ_ONLY);
+CRU_DIV_DEFINE(io_400m_div16, CRU_PARENT_HW(io_400m), 0x02c, 0, 32,
+	       cru_div_table_32, CLK_DIVIDER_READ_ONLY);
+
+CRU_DIV_GATE_DEFINE(qspi, CRU_PARENT_HW(io_1000m), 0x030, 0, 6, NULL, 0, 2);
+CRU_DIV_GATE_DEFINE(spi, CRU_PARENT_HW(io_1000m), 0x030, 8, 6, NULL, 0, 4);
+CRU_DIV_GATE_DEFINE(smc, CRU_PARENT_HW(io_1000m), 0x030, 16, 6, NULL, 0, 4);
+CRU_DIV_DEFINE(sdio, CRU_PARENT_HW(io_400m), 0x030, 24, 6, NULL, 0);
+
+CRU_DIV_GATE_DEFINE(gpio_db, CRU_PARENT_HW(io_25m), 0x034, 0, 6, NULL, 0, 1);
+CRU_DIV_GATE_DEFINE(efuse, CRU_PARENT_HW(io_25m), 0x034, 8, 6, NULL, 0, 1);
+CRU_DIV_GATE_DEFINE(tvs, CRU_PARENT_HW(io_25m), 0x034, 16, 6, NULL, 0, 1);
+CRU_DIV_GATE_DEFINE(trng, CRU_PARENT_HW(io_25m), 0x034, 24, 7, NULL, 0, 1);
+
+CRU_DIV_GATE_DEFINE(osc_div, CRU_PARENT_NAME(osc), 0x038, 0, 6, NULL, 0, 1);
+CRU_DIV_GATE_DEFINE(pwm, CRU_PARENT_NAME(osc), 0x038, 8, 12, NULL, 0, 1);
+
+CRU_DIV_GATE_DEFINE(fclk0, CRU_PARENT_HW(io_400m), 0x03c, 0, 6, NULL, 0, 1);
+CRU_DIV_GATE_DEFINE(fclk1, CRU_PARENT_HW(io_400m), 0x03c, 8, 6, NULL, 0, 1);
+CRU_DIV_GATE_DEFINE(fclk2, CRU_PARENT_HW(io_400m), 0x03c, 16, 6, NULL, 0, 1);
+CRU_DIV_GATE_DEFINE(fclk3, CRU_PARENT_HW(io_400m), 0x03c, 24, 6, NULL, 0, 1);
+
+static const struct clk_parent_data wdt_parents[] = {
+	CRU_PARENT_HW(osc_div2),
+	CRU_PARENT_NAME(wdt_ext)
+};
+CRU_MUX_DEFINE(wdt_sel, wdt_parents, 0x040, 1, 1);
+
+static const struct clk_parent_data efuse_parents[] = {
+	CRU_PARENT_NAME(osc),
+	CRU_PARENT_DIV_HW(efuse)
+};
+CRU_MUX_DEFINE(efuse_sel, efuse_parents, 0x040, 2, 1);
+
+static const struct clk_parent_data can_parents[] = {
+	CRU_PARENT_HW(io_80m),
+	CRU_PARENT_NAME(can_ext)
+};
+CRU_MUX_DEFINE(can_sel, can_parents, 0x040, 3, 1);
+
+static const struct clk_parent_data cpu_parents[] = {
+	CRU_PARENT_HW(cpu_4x_div1),
+	CRU_PARENT_HW(cpu_4x_div2)
+};
+CRU_MUX_DEFINE(cpu_sel, cpu_parents, 0x040, 5, 1);
+
+CRU_GATE_DEFINE(can0, CRU_PARENT_HW(can_sel), 0x08c, 20, CLK_GATE_SET_TO_DISABLE);
+CRU_GATE_DEFINE(can1, CRU_PARENT_HW(can_sel), 0x08c, 21, CLK_GATE_SET_TO_DISABLE);
+
+static const struct cru_clk dr1v90_cru_clks[] = {
+	[CLK_OSC_DIV2]		= { &osc_div2.hw,	NULL },
+	[CLK_CPU_PLL]		= { &cpu_pll.hw,	&cpu_pll.reg },
+	[CLK_CPU_PLL_4X]	= { &cpu_pll_4x.hw,	&cpu_pll_4x.reg },
+	[CLK_CPU_4X]		= { &cpu_4x_div1.hw,	&cpu_4x_div1.reg },
+	[CLK_CPU_2X]		= { &cpu_4x_div2.hw,	&cpu_4x_div2.reg },
+	[CLK_CPU_1X]		= { &cpu_4x_div4.hw,	&cpu_4x_div4.reg },
+	[CLK_IO_PLL]		= { &io_pll.hw,		&io_pll.reg },
+	[CLK_IO_1000M]		= { &io_1000m.hw,	&io_1000m.reg },
+	[CLK_IO_400M]		= { &io_400m.hw,	&io_400m.reg },
+	[CLK_IO_25M]		= { &io_25m.hw,		&io_25m.reg },
+	[CLK_IO_80M]		= { &io_80m.hw,		&io_80m.reg },
+	[CLK_IO_400M_DIV2]	= { &io_400m_div2.hw,	&io_400m_div2.reg },
+	[CLK_IO_400M_DIV4]	= { &io_400m_div4.hw,	&io_400m_div4.reg },
+	[CLK_IO_400M_DIV8]	= { &io_400m_div8.hw,	&io_400m_div8.reg },
+	[CLK_IO_400M_DIV16]	= { &io_400m_div16.hw,	&io_400m_div16.reg },
+	[CLK_QSPI]		= { &qspi.divider.hw,	&qspi.divider.reg },
+	[CLK_SPI]		= { &spi.divider.hw,	&spi.divider.reg },
+	[CLK_SMC]		= { &smc.divider.hw,	&smc.divider.reg },
+	[CLK_SDIO]		= { &sdio.hw,		&sdio.reg },
+	[CLK_GPIO_DB]		= { &gpio_db.divider.hw, &gpio_db.divider.reg },
+	[CLK_EFUSE]		= { &efuse.divider.hw,	&efuse.divider.reg },
+	[CLK_TVS]		= { &tvs.divider.hw,	&tvs.divider.reg },
+	[CLK_TRNG]		= { &trng.divider.hw,	&trng.divider.reg },
+	[CLK_OSC_DIV]		= { &osc_div.divider.hw, &osc_div.divider.reg },
+	[CLK_PWM]		= { &pwm.divider.hw,	&pwm.divider.reg },
+	[CLK_FCLK0]		= { &fclk0.divider.hw,	&fclk0.divider.reg },
+	[CLK_FCLK1]		= { &fclk1.divider.hw,	&fclk1.divider.reg },
+	[CLK_FCLK2]		= { &fclk2.divider.hw,	&fclk2.divider.reg },
+	[CLK_FCLK3]		= { &fclk3.divider.hw,	&fclk3.divider.reg },
+	[CLK_WDT_SEL]		= { &wdt_sel.hw,	&wdt_sel.reg },
+	[CLK_EFUSE_SEL]		= { &efuse_sel.hw,	&efuse_sel.reg },
+	[CLK_CAN_SEL]		= { &can_sel.hw,	&can_sel.reg },
+	[CLK_CPU_SEL]		= { &cpu_sel.hw,	&cpu_sel.reg },
+	[CLK_CAN0]		= { &can0.hw,		&can0.reg },
+	[CLK_CAN1]		= { &can1.hw,		&can1.reg }
+};
+
+static int dr1v90_cru_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	void __iomem *base;
+	int ret;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	ret = dr1_cru_clk_register(dev, base, dr1v90_cru_clks,
+				   ARRAY_SIZE(dr1v90_cru_clks));
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register clocks\n");
+
+	ret = dr1_cru_reset_register(dev, base);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register resets\n");
+
+	return 0;
+}
+
+static const struct of_device_id dr1v90_cru_ids[] = {
+	{ .compatible = "anlogic,dr1v90-cru" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, dr1v90_cru_ids);
+
+static struct platform_driver dr1v90_cru_driver = {
+	.driver = {
+		.name = "dr1v90-cru",
+		.of_match_table = dr1v90_cru_ids,
+	},
+	.probe = dr1v90_cru_probe,
+};
+module_platform_driver(dr1v90_cru_driver);
+
+MODULE_AUTHOR("Fushan Zeng <fushan.zeng@anlogic.com>");
+MODULE_AUTHOR("Junhui Liu <junhui.liu@pigmoral.tech>");
+MODULE_DESCRIPTION("Anlogic DR1V90 CRU driver");
+MODULE_IMPORT_NS("CLK_ANLOGIC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/anlogic/cru_dr1.c b/drivers/clk/anlogic/cru_dr1.c
new file mode 100644
index 00000000000000..10f8751a88383d
--- /dev/null
+++ b/drivers/clk/anlogic/cru_dr1.c
@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024-2025 Anlogic, Inc.
+ * Copyright (C) 2026 Junhui Liu <junhui.liu@pigmoral.tech>
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "cru_dr1.h"
+
+static unsigned long cru_pll_nm_recalc_rate(struct clk_hw *hw,
+					    unsigned long parent_rate)
+{
+	struct cru_pll *pll = hw_to_cru_pll(hw);
+	u32 mult, div;
+
+	div = FIELD_GET(GENMASK(6, 0), readl(pll->reg)) + 1;
+	mult = FIELD_GET(GENMASK(6, 0), readl(pll->reg + 4)) + 1;
+
+	return parent_rate * mult / div;
+}
+
+const struct clk_ops dr1_cru_pll_nm_ops = {
+	.recalc_rate = cru_pll_nm_recalc_rate,
+};
+EXPORT_SYMBOL_NS_GPL(dr1_cru_pll_nm_ops, "CLK_ANLOGIC");
+
+static unsigned long cru_pll_c_recalc_rate(struct clk_hw *hw,
+					   unsigned long parent_rate)
+{
+	struct cru_pll *pll = hw_to_cru_pll(hw);
+	u32 div;
+
+	div = FIELD_GET(GENMASK(30, 24), readl(pll->reg)) + 1;
+
+	return parent_rate / div;
+}
+
+const struct clk_ops dr1_cru_pll_c_ops = {
+	.recalc_rate = cru_pll_c_recalc_rate,
+};
+EXPORT_SYMBOL_NS_GPL(dr1_cru_pll_c_ops, "CLK_ANLOGIC");
+
+static void cru_div_gate_endisable(struct clk_hw *hw, int enable)
+{
+	struct cru_div_gate *div_gate = hw_to_cru_div_gate(hw);
+	struct clk_divider *divider = &div_gate->divider;
+	u32 reg;
+
+	reg = readl(divider->reg);
+	reg &= ~(clk_div_mask(divider->width) << divider->shift);
+
+	if (enable)
+		reg |= div_gate->val << divider->shift;
+
+	writel(reg, divider->reg);
+}
+
+static int cru_div_gate_enable(struct clk_hw *hw)
+{
+	cru_div_gate_endisable(hw, 1);
+
+	return 0;
+}
+
+static void cru_div_gate_disable(struct clk_hw *hw)
+{
+	cru_div_gate_endisable(hw, 0);
+}
+
+static int cru_div_gate_is_enabled(struct clk_hw *hw)
+{
+	struct cru_div_gate *div_gate = hw_to_cru_div_gate(hw);
+	struct clk_divider *divider = &div_gate->divider;
+	u32 val;
+
+	val = readl(divider->reg) >> divider->shift;
+	val &= clk_div_mask(divider->width);
+
+	return !!val;
+}
+
+static unsigned long cru_div_gate_recalc_rate(struct clk_hw *hw,
+					      unsigned long parent_rate)
+{
+	struct cru_div_gate *div_gate = hw_to_cru_div_gate(hw);
+	struct clk_divider *divider = &div_gate->divider;
+	unsigned int val;
+
+	val = readl(divider->reg) >> divider->shift;
+	val &= clk_div_mask(divider->width);
+
+	if (val < div_gate->min)
+		return 0;
+
+	return divider_recalc_rate(hw, parent_rate, val, divider->table,
+				   divider->flags, divider->width);
+}
+
+static int cru_div_gate_determine_rate(struct clk_hw *hw,
+				       struct clk_rate_request *req)
+{
+	struct cru_div_gate *div_gate = hw_to_cru_div_gate(hw);
+	struct clk_divider *divider = &div_gate->divider;
+	unsigned long maxdiv, mindiv;
+	int div = 0;
+
+	maxdiv = clk_div_mask(divider->width) + 1;
+	mindiv = div_gate->min + 1;
+
+	div = DIV_ROUND_UP_ULL(req->best_parent_rate, req->rate);
+	div = div > maxdiv ? maxdiv : div;
+	div = div < mindiv ? mindiv : div;
+
+	req->rate = DIV_ROUND_UP_ULL(req->best_parent_rate, div);
+
+	return 0;
+}
+
+static int cru_div_gate_set_rate(struct clk_hw *hw, unsigned long rate,
+				 unsigned long parent_rate)
+{
+	struct cru_div_gate *div_gate = hw_to_cru_div_gate(hw);
+	struct clk_divider *divider = &div_gate->divider;
+	int value;
+	u32 reg;
+
+	if (!__clk_is_enabled(hw->clk))
+		return 0;
+
+	value = divider_get_val(rate, parent_rate, divider->table,
+				divider->width, divider->flags);
+	if (value < 0)
+		return value;
+
+	if (value < div_gate->min)
+		value = div_gate->min;
+
+	reg = readl(divider->reg);
+	reg &= ~(clk_div_mask(divider->width) << divider->shift);
+	reg |= (u32)value << divider->shift;
+	writel(reg, divider->reg);
+
+	div_gate->val = value;
+
+	return 0;
+}
+
+static int cru_div_gate_init(struct clk_hw *hw)
+{
+	struct cru_div_gate *div_gate = hw_to_cru_div_gate(hw);
+	struct clk_divider *divider = &div_gate->divider;
+	u32 val;
+
+	val = readl(divider->reg) >> divider->shift;
+	val &= clk_div_mask(divider->width);
+	div_gate->val = val;
+
+	return 0;
+}
+
+const struct clk_ops dr1_cru_div_gate_ops = {
+	.enable = cru_div_gate_enable,
+	.disable = cru_div_gate_disable,
+	.is_enabled = cru_div_gate_is_enabled,
+	.recalc_rate = cru_div_gate_recalc_rate,
+	.determine_rate = cru_div_gate_determine_rate,
+	.set_rate = cru_div_gate_set_rate,
+	.init = cru_div_gate_init,
+};
+EXPORT_SYMBOL_NS_GPL(dr1_cru_div_gate_ops, "CLK_ANLOGIC");
+
+int dr1_cru_clk_register(struct device *dev, void __iomem *base,
+			 const struct cru_clk *clks, int nr_clks)
+{
+	struct clk_hw_onecell_data *priv;
+	int i, ret;
+
+	priv = devm_kzalloc(dev, struct_size(priv, hws, nr_clks), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	for (i = 0; i < nr_clks; i++) {
+		const struct cru_clk *clk = &clks[i];
+
+		if (clk->reg)
+			*(clk->reg) += (uintptr_t)base;
+
+		ret = devm_clk_hw_register(dev, clk->hw);
+		if (ret)
+			return ret;
+
+		priv->hws[i] = clk->hw;
+	}
+
+	priv->num = nr_clks;
+
+	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, priv);
+	if (ret)
+		dev_err(dev, "failed to add clock hardware provider\n");
+
+	return ret;
+}
+EXPORT_SYMBOL_NS_GPL(dr1_cru_clk_register, "CLK_ANLOGIC");
+
+int dr1_cru_reset_register(struct device *dev, void __iomem *base)
+{
+	struct auxiliary_device *adev;
+
+	adev = devm_auxiliary_device_create(dev, "reset", base);
+	if (!adev)
+		return -ENODEV;
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(dr1_cru_reset_register, "CLK_ANLOGIC");
+
+MODULE_DESCRIPTION("Anlogic DR1 CRU driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/anlogic/cru_dr1.h b/drivers/clk/anlogic/cru_dr1.h
new file mode 100644
index 00000000000000..5386aed9e9f3f0
--- /dev/null
+++ b/drivers/clk/anlogic/cru_dr1.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024-2025 Anlogic, Inc.
+ * Copyright (C) 2026 Junhui Liu <junhui.liu@pigmoral.tech>
+ */
+
+#ifndef _CRU_DR1_H_
+#define _CRU_DR1_H_
+
+#include <linux/clk-provider.h>
+
+struct cru_pll {
+	struct clk_hw hw;
+	void __iomem *reg;
+};
+
+struct cru_div_gate {
+	struct clk_divider divider;
+	u32 val; /* Cached divider value for restoring on enable */
+	u8 min; /* Minimum divider value to avoid timing issues */
+};
+
+struct cru_clk {
+	struct clk_hw *hw;
+	void **reg;
+};
+
+#define CRU_PARENT_NAME(_name)		{ .fw_name = #_name }
+#define CRU_PARENT_HW(_parent)		{ .hw = &_parent.hw }
+#define CRU_PARENT_DIV_HW(_parent)	{ .hw = &_parent.divider.hw }
+
+#define CRU_INITHW(_name, _parent, _ops)				\
+	.hw.init = &(struct clk_init_data) {				\
+		.name		= #_name,				\
+		.parent_data	= (const struct clk_parent_data[])	\
+					{ _parent },			\
+		.num_parents	= 1,					\
+		.ops		= &_ops,				\
+	}
+
+#define CRU_INITHW_PARENTS(_name, _parents, _ops)			\
+	.hw.init = CLK_HW_INIT_PARENTS_DATA(#_name, _parents, &_ops, 0)
+
+#define CRU_PLL_NM_DEFINE(_name, _parent, _reg)				\
+static struct cru_pll _name = {						\
+	.reg = (void __iomem *)(_reg),					\
+	CRU_INITHW(_name, _parent, dr1_cru_pll_nm_ops),			\
+}
+
+#define CRU_PLL_C_DEFINE(_name, _parent, _reg)				\
+static struct cru_pll _name = {						\
+	.reg = (void __iomem *)(_reg),					\
+	CRU_INITHW(_name, _parent, dr1_cru_pll_c_ops),			\
+}
+
+#define CRU_DIV_DEFINE(_name, _parent, _reg, _shift, _width, _table,	\
+		       _flags)						\
+static struct clk_divider _name = {					\
+	.shift = _shift,						\
+	.width = _width,						\
+	.flags = _flags,						\
+	.table = _table,						\
+	.reg = (void __iomem *)(_reg),					\
+	CRU_INITHW(_name, _parent, clk_divider_ops),			\
+}
+
+#define CRU_DIV_GATE_DEFINE(_name, _parent, _reg, _shift, _width,	\
+			    _table, _flags, _min)			\
+static struct cru_div_gate _name = {					\
+	.min = _min,							\
+	.divider = {							\
+		.shift = _shift,					\
+		.width = _width,					\
+		.flags = _flags,					\
+		.table = _table,					\
+		.reg = (void __iomem *)(_reg),				\
+		CRU_INITHW(_name, _parent, dr1_cru_div_gate_ops),	\
+	}								\
+}
+
+#define CRU_MUX_DEFINE(_name, _parents, _reg, _shift, _width)		\
+static struct clk_mux _name = {						\
+	.shift = _shift,						\
+	.mask = GENMASK(_width - 1, 0),					\
+	.reg = (void __iomem *)(_reg),					\
+	CRU_INITHW_PARENTS(_name, _parents, clk_mux_ops)		\
+}
+
+#define CRU_GATE_DEFINE(_name, _parent, _reg, _bit_idx, _flags)		\
+static struct clk_gate _name = {					\
+	.bit_idx = _bit_idx,						\
+	.flags = _flags,						\
+	.reg = (void __iomem *)(_reg),					\
+	CRU_INITHW(_name, _parent, clk_gate_ops)			\
+}
+
+static inline struct cru_pll *hw_to_cru_pll(struct clk_hw *hw)
+{
+	return container_of(hw, struct cru_pll, hw);
+}
+
+static inline struct cru_div_gate *hw_to_cru_div_gate(struct clk_hw *hw)
+{
+	struct clk_divider *divider = to_clk_divider(hw);
+
+	return container_of(divider, struct cru_div_gate, divider);
+}
+
+extern const struct clk_ops dr1_cru_pll_nm_ops;
+extern const struct clk_ops dr1_cru_pll_c_ops;
+extern const struct clk_ops dr1_cru_div_gate_ops;
+
+int dr1_cru_clk_register(struct device *dev, void __iomem *base,
+			 const struct cru_clk *clks, int nr_clks);
+int dr1_cru_reset_register(struct device *dev, void __iomem *base);
+
+#endif /* _CRU_DR1_H_ */
diff --git a/drivers/clk/aspeed/Kconfig b/drivers/clk/aspeed/Kconfig
index ef50481c31ff64..2f505eb3ca95f7 100644
--- a/drivers/clk/aspeed/Kconfig
+++ b/drivers/clk/aspeed/Kconfig
@@ -15,6 +15,7 @@ config COMMON_CLK_ASPEED
 config COMMON_CLK_AST2700
         bool "Clock driver for AST2700 SoC"
         depends on ARCH_ASPEED || COMPILE_TEST
+        select AUXILIARY_BUS
         help
           This driver provides support for clock on AST2700 SoC.
           The driver is responsible for managing the various clocks required
diff --git a/drivers/clk/aspeed/clk-aspeed.c b/drivers/clk/aspeed/clk-aspeed.c
index 38a5766a435ca2..d32a3e9f300a2d 100644
--- a/drivers/clk/aspeed/clk-aspeed.c
+++ b/drivers/clk/aspeed/clk-aspeed.c
@@ -349,8 +349,8 @@ static struct clk_hw *aspeed_clk_hw_register_gate(struct device *dev,
 		struct regmap *map, u8 clock_idx, u8 reset_idx,
 		u8 clk_gate_flags, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct aspeed_clk_gate *gate;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int ret;
 
diff --git a/drivers/clk/aspeed/clk-ast2600.c b/drivers/clk/aspeed/clk-ast2600.c
index 70061c961b69ec..cc3d26602b16eb 100644
--- a/drivers/clk/aspeed/clk-ast2600.c
+++ b/drivers/clk/aspeed/clk-ast2600.c
@@ -425,8 +425,8 @@ static struct clk_hw *aspeed_g6_clk_hw_register_gate(struct device *dev,
 		struct regmap *map, u8 clock_idx, u8 reset_idx,
 		u8 clk_gate_flags, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct aspeed_clk_gate *gate;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int ret;
 
diff --git a/drivers/clk/aspeed/clk-ast2700.c b/drivers/clk/aspeed/clk-ast2700.c
index 536600fcd0244e..495b7fa3395b21 100644
--- a/drivers/clk/aspeed/clk-ast2700.c
+++ b/drivers/clk/aspeed/clk-ast2700.c
@@ -824,7 +824,7 @@ static struct clk_hw *ast2700_clk_hw_register_gate(struct device *dev, const cha
 						   void __iomem *reg, u8 clock_idx,
 						   unsigned long flags, spinlock_t *lock)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_gate *gate;
 	struct clk_hw *hw;
 	int ret = -EINVAL;
diff --git a/drivers/clk/at91/clk-audio-pll.c b/drivers/clk/at91/clk-audio-pll.c
index 732037bfbda836..26875ad0cce79a 100644
--- a/drivers/clk/at91/clk-audio-pll.c
+++ b/drivers/clk/at91/clk-audio-pll.c
@@ -486,8 +486,8 @@ struct clk_hw * __init
 at91_clk_register_audio_pll_pad(struct regmap *regmap, const char *name,
 				const char *parent_name)
 {
+	struct clk_init_data init = {};
 	struct clk_audio_pad *apad_ck;
-	struct clk_init_data init;
 	int ret;
 
 	apad_ck = kzalloc_obj(*apad_ck);
@@ -517,8 +517,8 @@ struct clk_hw * __init
 at91_clk_register_audio_pll_pmc(struct regmap *regmap, const char *name,
 				const char *parent_name)
 {
+	struct clk_init_data init = {};
 	struct clk_audio_pmc *apmc_ck;
-	struct clk_init_data init;
 	int ret;
 
 	apmc_ck = kzalloc_obj(*apmc_ck);
diff --git a/drivers/clk/at91/clk-h32mx.c b/drivers/clk/at91/clk-h32mx.c
index 12bd112d3c509f..8d5488223e4937 100644
--- a/drivers/clk/at91/clk-h32mx.c
+++ b/drivers/clk/at91/clk-h32mx.c
@@ -97,7 +97,7 @@ at91_clk_register_h32mx(struct regmap *regmap, const char *name,
 			const char *parent_name)
 {
 	struct clk_sama5d4_h32mx *h32mxclk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int ret;
 
 	h32mxclk = kzalloc_obj(*h32mxclk);
diff --git a/drivers/clk/at91/clk-main.c b/drivers/clk/at91/clk-main.c
index ab515239adcf49..77f905e57e1ab4 100644
--- a/drivers/clk/at91/clk-main.c
+++ b/drivers/clk/at91/clk-main.c
@@ -170,7 +170,7 @@ at91_clk_register_main_osc(struct regmap *regmap,
 	init.name = name;
 	init.ops = &main_osc_ops;
 	if (parent_data)
-		init.parent_data = (const struct clk_parent_data *)parent_data;
+		init.parent_data = parent_data;
 	else
 		init.parent_names = &parent_name;
 	init.num_parents = 1;
@@ -298,8 +298,8 @@ at91_clk_register_main_rc_osc(struct regmap *regmap,
 			      const char *name,
 			      u32 frequency, u32 accuracy)
 {
+	struct clk_init_data init = {};
 	struct clk_main_rc_osc *osc;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int ret;
 
diff --git a/drivers/clk/at91/clk-pll.c b/drivers/clk/at91/clk-pll.c
index b89f60cf02416e..1b9a37a9e0701b 100644
--- a/drivers/clk/at91/clk-pll.c
+++ b/drivers/clk/at91/clk-pll.c
@@ -318,7 +318,7 @@ at91_clk_register_pll(struct regmap *regmap, const char *name,
 {
 	struct clk_pll *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int offset = PLL_REG(id);
 	unsigned int pllr;
 	int ret;
diff --git a/drivers/clk/at91/clk-plldiv.c b/drivers/clk/at91/clk-plldiv.c
index 5f93aead03fa58..135ca0abeb1889 100644
--- a/drivers/clk/at91/clk-plldiv.c
+++ b/drivers/clk/at91/clk-plldiv.c
@@ -86,9 +86,9 @@ struct clk_hw * __init
 at91_clk_register_plldiv(struct regmap *regmap, const char *name,
 			 const char *parent_name)
 {
+	struct clk_init_data init = {};
 	struct clk_plldiv *plldiv;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	plldiv = kzalloc_obj(*plldiv);
diff --git a/drivers/clk/at91/clk-slow.c b/drivers/clk/at91/clk-slow.c
index f0cad50e490bf1..dc0009ca2cef55 100644
--- a/drivers/clk/at91/clk-slow.c
+++ b/drivers/clk/at91/clk-slow.c
@@ -42,8 +42,8 @@ at91_clk_register_sam9260_slow(struct regmap *regmap,
 			       int num_parents)
 {
 	struct clk_sam9260_slow *slowck;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	if (!name)
diff --git a/drivers/clk/at91/clk-smd.c b/drivers/clk/at91/clk-smd.c
index 77f2b8fe6cabd5..4eff52a9798ba4 100644
--- a/drivers/clk/at91/clk-smd.c
+++ b/drivers/clk/at91/clk-smd.c
@@ -113,9 +113,9 @@ struct clk_hw * __init
 at91sam9x5_clk_register_smd(struct regmap *regmap, const char *name,
 			    const char **parent_names, u8 num_parents)
 {
+	struct clk_init_data init = {};
 	struct at91sam9x5_clk_smd *smd;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	smd = kzalloc_obj(*smd);
diff --git a/drivers/clk/at91/clk-usb.c b/drivers/clk/at91/clk-usb.c
index b4dc86193e09fb..0e4c0367c1afa7 100644
--- a/drivers/clk/at91/clk-usb.c
+++ b/drivers/clk/at91/clk-usb.c
@@ -224,9 +224,9 @@ _at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
 			     const char **parent_names, u8 num_parents,
 			     u32 usbs_mask)
 {
+	struct clk_init_data init = {};
 	struct at91sam9x5_clk_usb *usb;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	usb = kzalloc_obj(*usb);
@@ -275,9 +275,9 @@ struct clk_hw * __init
 at91sam9n12_clk_register_usb(struct regmap *regmap, const char *name,
 			     const char *parent_name)
 {
+	struct clk_init_data init = {};
 	struct at91sam9x5_clk_usb *usb;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	usb = kzalloc_obj(*usb);
@@ -394,9 +394,9 @@ struct clk_hw * __init
 at91rm9200_clk_register_usb(struct regmap *regmap, const char *name,
 			    const char *parent_name, const u32 *divisors)
 {
+	struct clk_init_data init = {};
 	struct at91rm9200_clk_usb *usb;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	usb = kzalloc_obj(*usb);
diff --git a/drivers/clk/at91/sckc.c b/drivers/clk/at91/sckc.c
index 776118f14867de..fc5ca22d181a75 100644
--- a/drivers/clk/at91/sckc.c
+++ b/drivers/clk/at91/sckc.c
@@ -231,9 +231,9 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr,
 			      unsigned long startup,
 			      const struct clk_slow_bits *bits)
 {
+	struct clk_init_data init = {};
 	struct clk_slow_rc_osc *osc;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	if (!sckcr || !name)
diff --git a/drivers/clk/bcm/clk-iproc-armpll.c b/drivers/clk/bcm/clk-iproc-armpll.c
index 8485428fe49ce6..aca53e2bda010c 100644
--- a/drivers/clk/bcm/clk-iproc-armpll.c
+++ b/drivers/clk/bcm/clk-iproc-armpll.c
@@ -235,7 +235,7 @@ void __init iproc_armpll_setup(struct device_node *node)
 {
 	int ret;
 	struct iproc_arm_pll *pll;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *parent_name;
 
 	pll = kzalloc_obj(*pll);
diff --git a/drivers/clk/bcm/clk-iproc-asiu.c b/drivers/clk/bcm/clk-iproc-asiu.c
index 56a8132279814d..86c0100934f81b 100644
--- a/drivers/clk/bcm/clk-iproc-asiu.c
+++ b/drivers/clk/bcm/clk-iproc-asiu.c
@@ -206,7 +206,7 @@ void __init iproc_asiu_setup(struct device_node *node,
 		goto err_iomap_gate;
 
 	for (i = 0; i < num_clks; i++) {
-		struct clk_init_data init;
+		struct clk_init_data init = {};
 		const char *parent_name;
 		struct iproc_asiu_clk *asiu_clk;
 		const char *clk_name;
diff --git a/drivers/clk/bcm/clk-iproc-pll.c b/drivers/clk/bcm/clk-iproc-pll.c
index d708f770971cee..4354aac2d2707d 100644
--- a/drivers/clk/bcm/clk-iproc-pll.c
+++ b/drivers/clk/bcm/clk-iproc-pll.c
@@ -722,7 +722,7 @@ void iproc_pll_clk_setup(struct device_node *node,
 	int i, ret;
 	struct iproc_pll *pll;
 	struct iproc_clk *iclk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *parent_name;
 	struct iproc_clk *iclk_array;
 	struct clk_hw_onecell_data *clk_data;
diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c
index f1a99de6de4f14..9882f243d68133 100644
--- a/drivers/clk/bcm/clk-raspberrypi.c
+++ b/drivers/clk/bcm/clk-raspberrypi.c
@@ -553,6 +553,8 @@ static int raspberrypi_clk_probe(struct platform_device *pdev)
 
 	rpi->cpufreq = platform_device_register_data(dev, "raspberrypi-cpufreq",
 						     -1, NULL, 0);
+	if (IS_ERR(rpi->cpufreq))
+		return PTR_ERR(rpi->cpufreq);
 
 	return 0;
 }
diff --git a/drivers/clk/berlin/berlin2-avpll.c b/drivers/clk/berlin/berlin2-avpll.c
index c801d42397bf06..33a3cc16b0cc5f 100644
--- a/drivers/clk/berlin/berlin2-avpll.c
+++ b/drivers/clk/berlin/berlin2-avpll.c
@@ -181,8 +181,8 @@ int __init berlin2_avpll_vco_register(void __iomem *base,
 			       const char *name, const char *parent_name,
 			       u8 vco_flags, unsigned long flags)
 {
+	struct clk_init_data init = {};
 	struct berlin2_avpll_vco *vco;
-	struct clk_init_data init;
 
 	vco = kzalloc_obj(*vco);
 	if (!vco)
@@ -358,7 +358,7 @@ int __init berlin2_avpll_channel_register(void __iomem *base,
 			   u8 ch_flags, unsigned long flags)
 {
 	struct berlin2_avpll_channel *ch;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 
 	ch = kzalloc_obj(*ch);
 	if (!ch)
diff --git a/drivers/clk/berlin/berlin2-pll.c b/drivers/clk/berlin/berlin2-pll.c
index d0ec3d0bcf3e57..1c4203736ecf4f 100644
--- a/drivers/clk/berlin/berlin2-pll.c
+++ b/drivers/clk/berlin/berlin2-pll.c
@@ -78,7 +78,7 @@ berlin2_pll_register(const struct berlin2_pll_map *map,
 		     void __iomem *base, const char *name,
 		     const char *parent_name, unsigned long flags)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct berlin2_pll *pll;
 
 	pll = kzalloc_obj(*pll);
diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c
index 6fcee41447e455..e53256c56dfabc 100644
--- a/drivers/clk/clk-axi-clkgen.c
+++ b/drivers/clk/clk-axi-clkgen.c
@@ -568,7 +568,7 @@ static int axi_clkgen_probe(struct platform_device *pdev)
 	const struct axi_clkgen_limits *dflt_limits;
 	struct axi_clkgen *axi_clkgen;
 	unsigned int pcore_version;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *parent_names[2];
 	const char *clk_name;
 	struct clk *axi_clk;
diff --git a/drivers/clk/clk-bd718x7.c b/drivers/clk/clk-bd718x7.c
index 8ddb50c677577a..1698b763bcb78d 100644
--- a/drivers/clk/clk-bd718x7.c
+++ b/drivers/clk/clk-bd718x7.c
@@ -21,6 +21,9 @@
 #define BD718XX_REG_OUT32K	0x2E
 /* BD72720 */
 #define BD72720_REG_OUT32K	0x9a
+/* BD73800 */
+#define BD73800_REG_OUT32K	0x50
+
 /*
  * BD71837, BD71847, and BD71828 all use bit [0] to clk output control
  */
@@ -123,6 +126,10 @@ static int bd71837_clk_probe(struct platform_device *pdev)
 		c->reg = BD72720_REG_OUT32K;
 		c->mask = CLK_OUT_EN_MASK;
 		break;
+	case ROHM_CHIP_TYPE_BD73800:
+		c->reg = BD73800_REG_OUT32K;
+		c->mask = CLK_OUT_EN_MASK;
+		break;
 	default:
 		dev_err(&pdev->dev, "Unknown clk chip\n");
 		return -EINVAL;
@@ -152,6 +159,7 @@ static const struct platform_device_id bd718x7_clk_id[] = {
 	{ .name = "bd71828-clk", .driver_data = ROHM_CHIP_TYPE_BD71828 },
 	{ .name = "bd71815-clk", .driver_data = ROHM_CHIP_TYPE_BD71815 },
 	{ .name = "bd72720-clk", .driver_data = ROHM_CHIP_TYPE_BD72720 },
+	{ .name = "bd73800-clk", .driver_data = ROHM_CHIP_TYPE_BD73800 },
 	{ }
 };
 MODULE_DEVICE_TABLE(platform, bd718x7_clk_id);
diff --git a/drivers/clk/clk-cdce925.c b/drivers/clk/clk-cdce925.c
index 30d0493c771818..4d49cc1659c07b 100644
--- a/drivers/clk/clk-cdce925.c
+++ b/drivers/clk/clk-cdce925.c
@@ -620,7 +620,7 @@ static int cdce925_probe(struct i2c_client *client)
 	struct device_node *node = client->dev.of_node;
 	const char *parent_name;
 	const char *pll_clk_name[MAX_NUMBER_OF_PLLS] = {NULL,};
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	u32 value;
 	int i;
 	int err;
diff --git a/drivers/clk/clk-conf.c b/drivers/clk/clk-conf.c
index 550b8ae375a2ca..da66b9ae3de540 100644
--- a/drivers/clk/clk-conf.c
+++ b/drivers/clk/clk-conf.c
@@ -25,8 +25,8 @@ static int __set_clk_parents(struct device_node *node, bool clk_supplier)
 		       node);
 
 	for (index = 0; index < num_parents; index++) {
-		rc = of_parse_phandle_with_args(node, "assigned-clock-parents",
-					"#clock-cells",	index, &clkspec);
+		rc = of_parse_phandle_with_args_map(node, "assigned-clock-parents",
+						    "clock", index, &clkspec);
 		if (rc < 0) {
 			/* skip empty (null) phandles */
 			if (rc == -ENOENT)
@@ -47,8 +47,8 @@ static int __set_clk_parents(struct device_node *node, bool clk_supplier)
 			return PTR_ERR(pclk);
 		}
 
-		rc = of_parse_phandle_with_args(node, "assigned-clocks",
-					"#clock-cells", index, &clkspec);
+		rc = of_parse_phandle_with_args_map(node, "assigned-clocks",
+						    "clock", index, &clkspec);
 		if (rc < 0)
 			goto err;
 		if (clkspec.np == node && !clk_supplier) {
@@ -121,8 +121,8 @@ static int __set_clk_rates(struct device_node *node, bool clk_supplier)
 			rate = rates[index];
 
 		if (rate) {
-			rc = of_parse_phandle_with_args(node, "assigned-clocks",
-					"#clock-cells",	index, &clkspec);
+			rc = of_parse_phandle_with_args_map(node, "assigned-clocks",
+							    "clock", index, &clkspec);
 			if (rc < 0) {
 				/* skip empty (null) phandles */
 				if (rc == -ENOENT)
diff --git a/drivers/clk/clk-cs2000-cp.c b/drivers/clk/clk-cs2000-cp.c
index e415bdff4d970a..aa1823b3100eea 100644
--- a/drivers/clk/clk-cs2000-cp.c
+++ b/drivers/clk/clk-cs2000-cp.c
@@ -461,7 +461,7 @@ static int cs2000_clk_register(struct cs2000_priv *priv)
 {
 	struct device *dev = priv_to_dev(priv);
 	struct device_node *np = dev->of_node;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = np->name;
 	static const char *parent_names[CLK_MAX];
 	u32 aux_out = 0;
diff --git a/drivers/clk/clk-en7523.c b/drivers/clk/clk-en7523.c
index 5c45aacb6409ee..24fdec2e6198c4 100644
--- a/drivers/clk/clk-en7523.c
+++ b/drivers/clk/clk-en7523.c
@@ -6,14 +6,18 @@
 #include <linux/io.h>
 #include <linux/mfd/syscon.h>
 #include <linux/platform_device.h>
+#include <linux/phy.h>
+#include <linux/phy/phy.h>
 #include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/reset-controller.h>
+#include <linux/spinlock.h>
 #include <dt-bindings/clock/en7523-clk.h>
 #include <dt-bindings/reset/airoha,en7523-reset.h>
 #include <dt-bindings/reset/airoha,en7581-reset.h>
 #include <dt-bindings/clock/econet,en751221-scu.h>
 #include <dt-bindings/reset/econet,en751221-scu.h>
+#include <dt-bindings/soc/airoha,scu-ssr.h>
 
 #define RST_NR_PER_BANK			32
 
@@ -40,9 +44,22 @@
 #define   REG_HIR_MASK			GENMASK(31, 16)
 /* EN7581 */
 #define REG_NP_SCU_PCIC			0x88
+#define REG_NP_SCU_SSR3			0x94
+#define REG_SSUSB_HSGMII_SEL_MASK	BIT(29)
+#define REG_SSUSB_HSGMII_SEL_HSGMII	FIELD_PREP_CONST(REG_SSUSB_HSGMII_SEL_MASK, 0x0)
+#define REG_SSUSB_HSGMII_SEL_USB	FIELD_PREP_CONST(REG_SSUSB_HSGMII_SEL_MASK, 0x1)
 #define REG_NP_SCU_SSTR			0x9c
 #define REG_PCIE_XSI0_SEL_MASK		GENMASK(14, 13)
+#define REG_PCIE_XSI0_SEL_PCIE		FIELD_PREP_CONST(REG_PCIE_XSI0_SEL_MASK, 0x0)
+#define REG_PCIE_XSI0_SEL_XFI		FIELD_PREP_CONST(REG_PCIE_XSI0_SEL_MASK, 0x1)
+#define REG_PCIE_XSI0_SEL_HSGMII	FIELD_PREP_CONST(REG_PCIE_XSI0_SEL_MASK, 0x2)
 #define REG_PCIE_XSI1_SEL_MASK		GENMASK(12, 11)
+#define REG_PCIE_XSI1_SEL_PCIE		FIELD_PREP_CONST(REG_PCIE_XSI1_SEL_MASK, 0x0)
+#define REG_PCIE_XSI1_SEL_XFI		FIELD_PREP_CONST(REG_PCIE_XSI1_SEL_MASK, 0x1)
+#define REG_PCIE_XSI1_SEL_HSGMII	FIELD_PREP_CONST(REG_PCIE_XSI1_SEL_MASK, 0x2)
+#define REG_USB_PCIE_SEL_MASK		BIT(3)
+#define REG_USB_PCIE_SEL_PCIE		FIELD_PREP_CONST(REG_USB_PCIE_SEL_MASK, 0x0)
+#define REG_USB_PCIE_SEL_USB		FIELD_PREP_CONST(REG_USB_PCIE_SEL_MASK, 0x1)
 #define REG_CRYPTO_CLKSRC2		0x20c
 /* EN751221 */
 #define EN751221_REG_SPI_DIV		0x0cc
@@ -82,6 +99,8 @@ enum en_hir {
 	HIR_MAX		= 14,
 };
 
+#define EN_SERDES_PHY_NUM		4
+
 struct en_clk_desc {
 	int id;
 	const char *name;
@@ -114,6 +133,18 @@ struct en_rst_data {
 	struct reset_controller_dev rcdev;
 };
 
+struct en_serdes_phy_instance {
+	struct phy *phy;
+	unsigned int serdes_port;
+};
+
+struct en_clk_priv {
+	void __iomem *base;
+	/* protect SCU register */
+	spinlock_t lock;
+	struct en_serdes_phy_instance *serdes_phys[EN_SERDES_PHY_NUM];
+};
+
 struct en_clk_soc_data {
 	u32 num_clocks;
 	const struct clk_ops pcie_ops;
@@ -847,12 +878,179 @@ static int en7581_reset_register(struct device *dev, void __iomem *base,
 	return devm_reset_controller_register(dev, &rst_data->rcdev);
 }
 
+static int en7581_serdes_phy_set_mode(struct phy *phy, enum phy_mode mode,
+				      int submode)
+{
+	struct en_serdes_phy_instance *instance = phy_get_drvdata(phy);
+	struct en_clk_priv *priv = dev_get_drvdata(phy->dev.parent);
+	u32 reg, mask, sel, val;
+	unsigned long flags;
+
+	switch (instance->serdes_port) {
+	case AIROHA_SCU_SERDES_PCIE1:
+		reg = REG_NP_SCU_SSTR;
+		mask = REG_PCIE_XSI0_SEL_MASK;
+
+		if (mode != PHY_MODE_ETHERNET && mode != PHY_MODE_PCIE)
+			return -EINVAL;
+
+		if (mode == PHY_MODE_ETHERNET) {
+			switch (submode) {
+			case PHY_INTERFACE_MODE_USXGMII:
+			case PHY_INTERFACE_MODE_10GBASER:
+				sel = REG_PCIE_XSI0_SEL_XFI;
+				break;
+			case PHY_INTERFACE_MODE_SGMII:
+			case PHY_INTERFACE_MODE_1000BASEX:
+			case PHY_INTERFACE_MODE_2500BASEX:
+				sel = REG_PCIE_XSI0_SEL_HSGMII;
+				break;
+			default:
+				return -EINVAL;
+			}
+		} else {
+			sel = REG_PCIE_XSI0_SEL_PCIE;
+		}
+
+		break;
+	case AIROHA_SCU_SERDES_PCIE2:
+		reg = REG_NP_SCU_SSTR;
+		mask = REG_PCIE_XSI1_SEL_MASK;
+
+		if (mode != PHY_MODE_ETHERNET && mode != PHY_MODE_PCIE)
+			return -EINVAL;
+
+		if (mode == PHY_MODE_ETHERNET) {
+			switch (submode) {
+			case PHY_INTERFACE_MODE_USXGMII:
+			case PHY_INTERFACE_MODE_10GBASER:
+				sel = REG_PCIE_XSI1_SEL_XFI;
+				break;
+			case PHY_INTERFACE_MODE_SGMII:
+			case PHY_INTERFACE_MODE_1000BASEX:
+			case PHY_INTERFACE_MODE_2500BASEX:
+				sel = REG_PCIE_XSI1_SEL_HSGMII;
+				break;
+			default:
+				return -EINVAL;
+			}
+		} else {
+			sel = REG_PCIE_XSI1_SEL_PCIE;
+		}
+
+		break;
+	case AIROHA_SCU_SERDES_USB1:
+		reg = REG_NP_SCU_SSR3;
+		mask = REG_SSUSB_HSGMII_SEL_MASK;
+
+		if (mode != PHY_MODE_ETHERNET && mode != PHY_MODE_USB_DEVICE &&
+		    mode != PHY_MODE_USB_DEVICE_SS)
+			return -EINVAL;
+
+		if (mode == PHY_MODE_ETHERNET)
+			sel = REG_SSUSB_HSGMII_SEL_HSGMII;
+		else
+			sel = REG_SSUSB_HSGMII_SEL_USB;
+
+		break;
+	case AIROHA_SCU_SERDES_USB2:
+		reg = REG_NP_SCU_SSTR;
+		mask = REG_USB_PCIE_SEL_MASK;
+
+		if (mode != PHY_MODE_PCIE && mode != PHY_MODE_USB_DEVICE &&
+		    mode != PHY_MODE_USB_DEVICE_SS)
+			return -EINVAL;
+
+		if (mode == PHY_MODE_PCIE)
+			sel = REG_USB_PCIE_SEL_PCIE;
+		else
+			sel = REG_USB_PCIE_SEL_USB;
+
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&priv->lock, flags);
+	val = readl(priv->base + reg);
+	val &= ~mask;
+	val |= sel;
+	writel(val, priv->base + reg);
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	return 0;
+}
+
+static const struct phy_ops en7581_serdes_phy_ops = {
+	.set_mode	= en7581_serdes_phy_set_mode,
+	.owner		= THIS_MODULE,
+};
+
+static struct phy *en7581_serdes_phy_xlate(struct device *dev,
+					   const struct of_phandle_args *args)
+{
+	struct en_clk_priv *priv = dev_get_drvdata(dev);
+	struct en_serdes_phy_instance *instance;
+	unsigned int serdes_port;
+
+	if (args->args_count != 1) {
+		dev_err(dev, "invalid number of cells in 'phy' property\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	serdes_port = args->args[0];
+	if (serdes_port >= EN_SERDES_PHY_NUM) {
+		dev_err(dev, "invalid serdes port: %d\n", serdes_port);
+		return ERR_PTR(-EINVAL);
+	}
+
+	instance = priv->serdes_phys[serdes_port];
+	if (!instance) {
+		dev_err(dev, "failed to find appropriate serdes phy\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	return instance->phy;
+}
+
+static int en7581_serdes_phy_register(struct device *dev)
+{
+	struct en_clk_priv *priv = dev_get_drvdata(dev);
+	struct phy_provider *phy_provider;
+	int i;
+
+	for (i = 0; i < EN_SERDES_PHY_NUM; i++) {
+		struct en_serdes_phy_instance *instance;
+
+		instance = devm_kzalloc(dev, sizeof(*instance),
+					GFP_KERNEL);
+		if (!instance)
+			return -ENOMEM;
+
+		instance->phy = devm_phy_create(dev, NULL,
+						&en7581_serdes_phy_ops);
+		if (IS_ERR(instance->phy))
+			return dev_err_probe(dev, PTR_ERR(instance->phy), "failed to create phy\n");
+
+		instance->serdes_port = i;
+		priv->serdes_phys[i] = instance;
+
+		phy_set_drvdata(instance->phy, instance);
+	}
+
+	phy_provider = devm_of_phy_provider_register(dev, en7581_serdes_phy_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
 static int en7581_clk_hw_init(struct platform_device *pdev,
 			      struct clk_hw_onecell_data *clk_data)
 {
+	struct en_clk_priv *priv = platform_get_drvdata(pdev);
 	struct regmap *map;
 	void __iomem *base;
 	u32 val;
+	int ret;
 
 	map = syscon_regmap_lookup_by_compatible("airoha,en7581-chip-scu");
 	if (IS_ERR(map))
@@ -862,6 +1060,8 @@ static int en7581_clk_hw_init(struct platform_device *pdev,
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
+	priv->base = base;
+
 	en7581_register_clocks(&pdev->dev, clk_data, map, base);
 
 	val = readl(base + REG_NP_SCU_SSTR);
@@ -876,9 +1076,12 @@ static int en7581_clk_hw_init(struct platform_device *pdev,
 	val &= ~REG_PCIE_HB_RST;
 	writel(val, base + REG_RST_CTRL1);
 
-	return en7581_reset_register(&pdev->dev, base, en7581_rst_map,
-				     ARRAY_SIZE(en7581_rst_map),
-				     en7581_rst_ofs);
+	ret = en7581_reset_register(&pdev->dev, base, en7581_rst_map,
+				    ARRAY_SIZE(en7581_rst_map), en7581_rst_ofs);
+	if (ret)
+		return ret;
+
+	return en7581_serdes_phy_register(&pdev->dev);
 }
 
 static enum en_hir get_hw_id(void __iomem *np_base)
@@ -985,16 +1188,25 @@ static int en7523_clk_probe(struct platform_device *pdev)
 	struct device_node *node = pdev->dev.of_node;
 	const struct en_clk_soc_data *soc_data;
 	struct clk_hw_onecell_data *clk_data;
+	struct en_clk_priv *priv;
 	int r;
 
 	soc_data = device_get_match_data(&pdev->dev);
 
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	spin_lock_init(&priv->lock);
+
 	clk_data = devm_kzalloc(&pdev->dev,
 				struct_size(clk_data, hws, soc_data->num_clocks),
 				GFP_KERNEL);
 	if (!clk_data)
 		return -ENOMEM;
 
+	platform_set_drvdata(pdev, priv);
+
 	clk_data->num = soc_data->num_clocks;
 	r = soc_data->hw_init(pdev, clk_data);
 	if (r)
diff --git a/drivers/clk/clk-eyeq.c b/drivers/clk/clk-eyeq.c
index cf37feccc734fb..ae0faee3168902 100644
--- a/drivers/clk/clk-eyeq.c
+++ b/drivers/clk/clk-eyeq.c
@@ -513,21 +513,16 @@ static int eqc_probe(struct platform_device *pdev)
 	const struct eqc_match_data *data;
 	struct clk_hw_onecell_data *cells;
 	unsigned int i, clk_count;
-	struct resource *res;
 	void __iomem *base;
 	int ret;
 
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
 	data = device_get_match_data(dev);
 	if (!data)
-		return 0; /* No clocks nor auxdevs, we are done. */
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
-	base = ioremap(res->start, resource_size(res));
-	if (!base)
-		return -ENOMEM;
+		return 0; /* No clocks nor auxdevs, stop here but keep resource reserved */
 
 	/* Init optional auxiliary devices. */
 	eqc_auxdev_create_optional(dev, base, data->reset_auxdev_name);
@@ -559,7 +554,20 @@ static int eqc_probe(struct platform_device *pdev)
 			dev_warn(dev, "failed probing clock %s: %d\n", clk->name, ret);
 	}
 
-	return of_clk_add_hw_provider(np, of_clk_hw_onecell_get, cells);
+	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, cells);
+	if (ret) {
+		for (i = 0; i < data->clk_count; i++) {
+			const struct eqc_clock *clk = &data->clks[i];
+			struct clk_hw *hw = cells->hws[clk->index];
+
+			if (!IS_ERR_OR_NULL(hw) && clk->unregister)
+				clk->unregister(hw);
+		}
+
+		kfree(cells);
+	}
+
+	return ret;
 }
 
 #define DIV(_index, _parent_idx, _name, _parent_name,			\
@@ -1175,6 +1183,7 @@ static const struct of_device_id eqc_match_table[] = {
 	{ .compatible = "mobileye,eyeq5-olb", .data = &eqc_eyeq5_match_data },
 	{ .compatible = "mobileye,eyeq6l-olb", .data = &eqc_eyeq6l_match_data },
 	{ .compatible = "mobileye,eyeq6lplus-olb", .data = &eqc_eyeq6lplus_match_data },
+	{ .compatible = "mobileye,eyeq6h-central-olb" /* no data, early only */ },
 	{ .compatible = "mobileye,eyeq6h-west-olb", .data = &eqc_eyeq6h_west_match_data },
 	{ .compatible = "mobileye,eyeq6h-east-olb", .data = &eqc_eyeq6h_east_match_data },
 	{ .compatible = "mobileye,eyeq6h-south-olb", .data = &eqc_eyeq6h_south_match_data },
diff --git a/drivers/clk/clk-fractional-divider.c b/drivers/clk/clk-fractional-divider.c
index 1c5fdcb68a5048..c8e00d75cd14d8 100644
--- a/drivers/clk/clk-fractional-divider.c
+++ b/drivers/clk/clk-fractional-divider.c
@@ -271,7 +271,7 @@ struct clk_hw *clk_hw_register_fractional_divider(struct device *dev,
 		u8 clk_divider_flags, spinlock_t *lock)
 {
 	struct clk_fractional_divider *fd;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
 	int ret;
 
diff --git a/drivers/clk/clk-gemini.c b/drivers/clk/clk-gemini.c
index 9c0e6e7a7cacdf..4e874392e3b0b5 100644
--- a/drivers/clk/clk-gemini.c
+++ b/drivers/clk/clk-gemini.c
@@ -193,8 +193,8 @@ static struct clk_hw *gemini_pci_clk_setup(const char *name,
 					   const char *parent_name,
 					   struct regmap *map)
 {
+	struct clk_init_data init = {};
 	struct clk_gemini_pci *pciclk;
-	struct clk_init_data init;
 	int ret;
 
 	pciclk = kzalloc_obj(*pciclk);
diff --git a/drivers/clk/clk-highbank.c b/drivers/clk/clk-highbank.c
index e9b177811dfc2c..61c23199763508 100644
--- a/drivers/clk/clk-highbank.c
+++ b/drivers/clk/clk-highbank.c
@@ -269,7 +269,7 @@ static void __init hb_clk_init(struct device_node *node, const struct clk_ops *o
 	struct hb_clk *hb_clk;
 	const char *clk_name = node->name;
 	const char *parent_name;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct device_node *srnp;
 	int rc;
 
diff --git a/drivers/clk/clk-lmk04832.c b/drivers/clk/clk-lmk04832.c
index f9a0684d2628eb..10e5cfa96e6be1 100644
--- a/drivers/clk/clk-lmk04832.c
+++ b/drivers/clk/clk-lmk04832.c
@@ -587,8 +587,8 @@ static const struct clk_ops lmk04832_vco_ops = {
  */
 static int lmk04832_register_vco(struct lmk04832 *lmk)
 {
+	struct clk_init_data init = {};
 	const char *parent_names[1];
-	struct clk_init_data init;
 	int ret;
 
 	init.name = "lmk-vco";
@@ -947,8 +947,8 @@ static const struct clk_ops lmk04832_sclk_ops = {
 
 static int lmk04832_register_sclk(struct lmk04832 *lmk)
 {
+	struct clk_init_data init = {};
 	const char *parent_names[1];
-	struct clk_init_data init;
 	int ret;
 
 	init.name = "lmk-sclk";
@@ -1292,8 +1292,8 @@ static int lmk04832_register_clkout(struct lmk04832 *lmk, const int num)
 {
 	char name[] = "lmk-clkoutXX";
 	char dclk_name[] = "lmk-dclkXX_YY";
+	struct clk_init_data init = {};
 	const char *parent_names[2];
-	struct clk_init_data init;
 	int dclk_num = num / 2;
 	int ret;
 
diff --git a/drivers/clk/clk-milbeaut.c b/drivers/clk/clk-milbeaut.c
index 4265bc442dfd29..29c1e5f68c7a80 100644
--- a/drivers/clk/clk-milbeaut.c
+++ b/drivers/clk/clk-milbeaut.c
@@ -328,9 +328,9 @@ static struct clk_hw *m10v_clk_hw_register_mux(struct device *dev,
 			u8 shift, u32 mask, u8 clk_mux_flags, u32 *table,
 			spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clk_mux *mux;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	mux = kzalloc_obj(*mux);
@@ -410,8 +410,8 @@ static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 				unsigned long parent_rate)
 {
 	struct m10v_clk_divider *divider = to_m10v_div(hw);
-	int value;
 	unsigned long flags = 0;
+	int value, ret = 0;
 	u32 val;
 	u32 write_en = BIT(divider->width - 1);
 
@@ -433,8 +433,10 @@ static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 
 	if (divider->write_valid_reg) {
 		writel(M10V_DCHREQ, divider->write_valid_reg);
-		if (readl_poll_timeout(divider->write_valid_reg, val,
-			!val, M10V_UPOLL_RATE, M10V_UTIMEOUT))
+		ret = readl_poll_timeout_atomic(divider->write_valid_reg, val,
+						!val, M10V_UPOLL_RATE,
+						M10V_UTIMEOUT);
+		if (ret)
 			pr_err("%s:%s couldn't stabilize\n",
 				__func__, clk_hw_get_name(hw));
 	}
@@ -444,7 +446,7 @@ static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 	else
 		__release(divider->lock);
 
-	return 0;
+	return ret;
 }
 
 static const struct clk_ops m10v_clk_divider_ops = {
@@ -459,9 +461,9 @@ static struct clk_hw *m10v_clk_hw_register_divider(struct device *dev,
 		u8 clk_divider_flags, const struct clk_div_table *table,
 		spinlock_t *lock, void __iomem *write_valid_reg)
 {
+	struct clk_init_data init = {};
 	struct m10v_clk_divider *div;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	div = kzalloc_obj(*div);
diff --git a/drivers/clk/clk-nomadik.c b/drivers/clk/clk-nomadik.c
index da6b89759a14ec..890084b3912b4c 100644
--- a/drivers/clk/clk-nomadik.c
+++ b/drivers/clk/clk-nomadik.c
@@ -263,7 +263,7 @@ pll_clk_register(struct device *dev, const char *name,
 {
 	int ret;
 	struct clk_pll *pll;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 
 	if (id != 1 && id != 2) {
 		pr_err("%s: the Nomadik has only PLL 1 & 2\n", __func__);
@@ -355,7 +355,7 @@ src_clk_register(struct device *dev, const char *name,
 {
 	int ret;
 	struct clk_src *sclk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 
 	sclk = kzalloc_obj(*sclk);
 	if (!sclk)
diff --git a/drivers/clk/clk-npcm7xx.c b/drivers/clk/clk-npcm7xx.c
index 79eca6ee3120c6..562d2d406d7a00 100644
--- a/drivers/clk/clk-npcm7xx.c
+++ b/drivers/clk/clk-npcm7xx.c
@@ -69,8 +69,8 @@ static struct clk_hw *
 npcm7xx_clk_register_pll(void __iomem *pllcon, const char *name,
 			 const char *parent_name, unsigned long flags)
 {
+	struct clk_init_data init = {};
 	struct npcm7xx_clk_pll *pll;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int ret;
 
diff --git a/drivers/clk/clk-pwm.c b/drivers/clk/clk-pwm.c
index 4709f0338e373b..2001c7c9367c00 100644
--- a/drivers/clk/clk-pwm.c
+++ b/drivers/clk/clk-pwm.c
@@ -95,7 +95,7 @@ static const struct clk_ops clk_pwm_ops = {
 static int clk_pwm_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_pwm *clk_pwm;
 	struct pwm_device *pwm;
 	struct pwm_args pargs;
diff --git a/drivers/clk/clk-renesas-pcie.c b/drivers/clk/clk-renesas-pcie.c
index 564550c8038967..864bf89cecd065 100644
--- a/drivers/clk/clk-renesas-pcie.c
+++ b/drivers/clk/clk-renesas-pcie.c
@@ -63,10 +63,11 @@ struct rs9_driver_data {
 	struct i2c_client	*client;
 	struct regmap		*regmap;
 	const struct rs9_chip_info *chip_info;
-	struct clk_hw		*clk_dif[8];
 	u8			pll_amplitude;
 	u8			pll_ssc;
 	u8			clk_dif_sr;
+	/* must be last */
+	struct clk_hw_onecell_data onecell;
 };
 
 /*
@@ -270,32 +271,28 @@ static void rs9_update_config(struct rs9_driver_data *rs9)
 	}
 }
 
-static struct clk_hw *
-rs9_of_clk_get(struct of_phandle_args *clkspec, void *data)
-{
-	struct rs9_driver_data *rs9 = data;
-	unsigned int idx = clkspec->args[0];
-
-	return rs9->clk_dif[idx];
-}
-
 static int rs9_probe(struct i2c_client *client)
 {
+	const struct rs9_chip_info *chip_info;
 	unsigned char name[5] = "DIF0";
 	struct rs9_driver_data *rs9;
 	unsigned int vid, did;
 	struct clk_hw *hw;
 	int i, ret;
 
-	rs9 = devm_kzalloc(&client->dev, sizeof(*rs9), GFP_KERNEL);
+	chip_info = i2c_get_match_data(client);
+	if (!chip_info)
+		return -EINVAL;
+
+	rs9 = devm_kzalloc(&client->dev, struct_size(rs9, onecell.hws,
+			   chip_info->num_clks), GFP_KERNEL);
 	if (!rs9)
 		return -ENOMEM;
 
 	i2c_set_clientdata(client, rs9);
 	rs9->client = client;
-	rs9->chip_info = i2c_get_match_data(client);
-	if (!rs9->chip_info)
-		return -EINVAL;
+	rs9->chip_info = chip_info;
+	rs9->onecell.num = chip_info->num_clks;
 
 	/* Fetch common configuration from DT (if specified) */
 	ret = rs9_get_common_config(rs9);
@@ -303,7 +300,7 @@ static int rs9_probe(struct i2c_client *client)
 		return ret;
 
 	/* Fetch DIFx output configuration from DT (if specified) */
-	for (i = 0; i < rs9->chip_info->num_clks; i++) {
+	for (i = 0; i < rs9->onecell.num; i++) {
 		ret = rs9_get_output_config(rs9, i);
 		if (ret)
 			return ret;
@@ -329,24 +326,24 @@ static int rs9_probe(struct i2c_client *client)
 		return ret;
 
 	vid &= RS9_REG_VID_MASK;
-	if (vid != RS9_REG_VID_IDT || did != rs9->chip_info->did)
+	if (vid != RS9_REG_VID_IDT || did != chip_info->did)
 		return dev_err_probe(&client->dev, -ENODEV,
 				     "Incorrect VID/DID: %#02x, %#02x. Expected %#02x, %#02x\n",
-				     vid, did, RS9_REG_VID_IDT,
-				     rs9->chip_info->did);
+				     vid, did, RS9_REG_VID_IDT, chip_info->did);
 
 	/* Register clock */
-	for (i = 0; i < rs9->chip_info->num_clks; i++) {
+	for (i = 0; i < rs9->onecell.num; i++) {
 		snprintf(name, 5, "DIF%d", i);
 		hw = devm_clk_hw_register_fixed_factor_index(&client->dev, name,
 						    0, 0, 4, 1);
 		if (IS_ERR(hw))
 			return PTR_ERR(hw);
 
-		rs9->clk_dif[i] = hw;
+		rs9->onecell.hws[i] = hw;
 	}
 
-	ret = devm_of_clk_add_hw_provider(&client->dev, rs9_of_clk_get, rs9);
+	ret = devm_of_clk_add_hw_provider(&client->dev, of_clk_hw_onecell_get,
+					  &rs9->onecell);
 	if (!ret)
 		rs9_update_config(rs9);
 
@@ -370,8 +367,10 @@ static int rs9_resume(struct device *dev)
 
 	regcache_cache_only(rs9->regmap, false);
 	ret = regcache_sync(rs9->regmap);
-	if (ret)
+	if (ret) {
+		regcache_cache_only(rs9->regmap, true);
 		dev_err(dev, "Failed to restore register map: %d\n", ret);
+	}
 	return ret;
 }
 
diff --git a/drivers/clk/clk-s2mps11.c b/drivers/clk/clk-s2mps11.c
index fa5ac8f673f6ef..94969d97d6e995 100644
--- a/drivers/clk/clk-s2mps11.c
+++ b/drivers/clk/clk-s2mps11.c
@@ -192,8 +192,10 @@ static int s2mps11_clk_probe(struct platform_device *pdev)
 		clk_data->hws[i] = &s2mps11_clks[i].hw;
 	}
 
-	of_clk_add_hw_provider(s2mps11_clks->clk_np, of_clk_hw_onecell_get,
-			       clk_data);
+	ret = of_clk_add_hw_provider(s2mps11_clks->clk_np,
+				     of_clk_hw_onecell_get, clk_data);
+	if (ret)
+		goto err_reg;
 
 	platform_set_drvdata(pdev, s2mps11_clks);
 
@@ -201,8 +203,10 @@ static int s2mps11_clk_probe(struct platform_device *pdev)
 
 err_reg:
 	of_node_put(s2mps11_clks[0].clk_np);
-	while (--i >= 0)
-		clkdev_drop(s2mps11_clks[i].lookup);
+	while (--i >= 0) {
+		if (s2mps11_clks[i].lookup)
+			clkdev_drop(s2mps11_clks[i].lookup);
+	}
 
 	return ret;
 }
diff --git a/drivers/clk/clk-scpi.c b/drivers/clk/clk-scpi.c
index b2d412ca1d2222..e2eec7be5c1961 100644
--- a/drivers/clk/clk-scpi.c
+++ b/drivers/clk/clk-scpi.c
@@ -128,7 +128,7 @@ static int
 scpi_clk_ops_init(struct device *dev, const struct of_device_id *match,
 		  struct scpi_clk *sclk, const char *name)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	unsigned long min = 0, max = 0;
 	int ret;
 
diff --git a/drivers/clk/clk-si514.c b/drivers/clk/clk-si514.c
index c0b433ad937283..07091b91420d16 100644
--- a/drivers/clk/clk-si514.c
+++ b/drivers/clk/clk-si514.c
@@ -337,8 +337,8 @@ static const struct regmap_config si514_regmap_config = {
 
 static int si514_probe(struct i2c_client *client)
 {
+	struct clk_init_data init = {};
 	struct clk_si514 *data;
-	struct clk_init_data init;
 	int err;
 
 	data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
diff --git a/drivers/clk/clk-si521xx.c b/drivers/clk/clk-si521xx.c
index e3fb8df712effb..84dc3621df93c0 100644
--- a/drivers/clk/clk-si521xx.c
+++ b/drivers/clk/clk-si521xx.c
@@ -13,6 +13,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/bitops.h>
 #include <linux/bitrev.h>
 #include <linux/clk-provider.h>
 #include <linux/i2c.h>
@@ -255,18 +256,24 @@ static void si521xx_diff_idx_to_reg_bit(const u16 chip_info, const int idx,
 					struct si_clk *clk)
 {
 	unsigned long mask;
-	int oe, b, ctr = 0;
-
-	for (oe = 1; oe <= 2; oe++) {
-		mask = bitrev8(SI521XX_OE_MAP_GET_OE(oe, chip_info));
-		for_each_set_bit(b, &mask, 8) {
-			if (ctr++ != idx)
-				continue;
-			clk->reg = SI521XX_REG_OE(oe);
-			clk->bit = 7 - b;
-			return;
-		}
-	}
+	unsigned int bit;
+	int oe = 1;
+
+	mask = bitrev8(SI521XX_OE_MAP_GET_OE(1, chip_info));
+	bit = fns(mask, idx);
+	if (bit < 8)
+		goto out;
+
+	oe = 2;
+	bit = idx - hweight8(mask);
+	mask = bitrev8(SI521XX_OE_MAP_GET_OE(2, chip_info));
+	bit = fns(mask, bit);
+	if (bit >= 8)
+		return;
+
+out:
+	clk->reg = SI521XX_REG_OE(oe);
+	clk->bit = 7 - bit;
 }
 
 static struct clk_hw *
@@ -341,7 +348,7 @@ static int si521xx_probe(struct i2c_client *client)
 	return ret;
 }
 
-static int __maybe_unused si521xx_suspend(struct device *dev)
+static int si521xx_suspend(struct device *dev)
 {
 	struct si521xx *si = dev_get_drvdata(dev);
 
@@ -351,15 +358,17 @@ static int __maybe_unused si521xx_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused si521xx_resume(struct device *dev)
+static int si521xx_resume(struct device *dev)
 {
 	struct si521xx *si = dev_get_drvdata(dev);
 	int ret;
 
 	regcache_cache_only(si->regmap, false);
 	ret = regcache_sync(si->regmap);
-	if (ret)
+	if (ret) {
+		regcache_cache_only(si->regmap, true);
 		dev_err(dev, "Failed to restore register map: %d\n", ret);
+	}
 	return ret;
 }
 
@@ -374,17 +383,17 @@ MODULE_DEVICE_TABLE(i2c, si521xx_id);
 static const struct of_device_id clk_si521xx_of_match[] = {
 	{ .compatible = "skyworks,si52144", .data = (void *)SI521XX_OE_MAP(0x5, 0xc0) },
 	{ .compatible = "skyworks,si52146", .data = (void *)SI521XX_OE_MAP(0x15, 0xe0) },
-	{ .compatible = "skyworks,si52147", .data = (void *)SI521XX_OE_MAP(0x15, 0xf8) },
+	{ .compatible = "skyworks,si52147", .data = (void *)SI521XX_OE_MAP(0x17, 0xf8) },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, clk_si521xx_of_match);
 
-static SIMPLE_DEV_PM_OPS(si521xx_pm_ops, si521xx_suspend, si521xx_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(si521xx_pm_ops, si521xx_suspend, si521xx_resume);
 
 static struct i2c_driver si521xx_driver = {
 	.driver = {
 		.name = "clk-si521xx",
-		.pm	= &si521xx_pm_ops,
+		.pm	= pm_sleep_ptr(&si521xx_pm_ops),
 		.of_match_table = clk_si521xx_of_match,
 	},
 	.probe		= si521xx_probe,
diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c
index 5311c23532b44b..88522333525b68 100644
--- a/drivers/clk/clk-si5341.c
+++ b/drivers/clk/clk-si5341.c
@@ -1556,8 +1556,8 @@ static const struct attribute *si5341_attributes[] = {
 
 static int si5341_probe(struct i2c_client *client)
 {
+	struct clk_init_data init = {};
 	struct clk_si5341 *data;
-	struct clk_init_data init;
 	struct clk *input;
 	const char *root_clock_name;
 	const char *synth_clock_names[SI5341_NUM_SYNTH] = { NULL };
diff --git a/drivers/clk/clk-si5351.c b/drivers/clk/clk-si5351.c
index 20b0c3c48b8638..70daba62305c54 100644
--- a/drivers/clk/clk-si5351.c
+++ b/drivers/clk/clk-si5351.c
@@ -907,7 +907,7 @@ static int _si5351_clkout_set_disable_state(
 	return 0;
 }
 
-static void _si5351_clkout_reset_pll(struct si5351_driver_data *drvdata, int num)
+static int _si5351_clkout_reset_pll(struct si5351_driver_data *drvdata, int num)
 {
 	u8 val = si5351_reg_read(drvdata, SI5351_CLK0_CTRL + num);
 	u8 mask = val & SI5351_CLK_PLL_SELECT ? SI5351_PLL_RESET_B :
@@ -918,7 +918,7 @@ static void _si5351_clkout_reset_pll(struct si5351_driver_data *drvdata, int num
 	switch (val & SI5351_CLK_INPUT_MASK) {
 	case SI5351_CLK_INPUT_XTAL:
 	case SI5351_CLK_INPUT_CLKIN:
-		return;  /* pll not used, no need to reset */
+		return 0;  /* pll not used, no need to reset */
 	}
 
 	si5351_reg_write(drvdata, SI5351_PLL_RESET, mask);
@@ -931,6 +931,8 @@ static void _si5351_clkout_reset_pll(struct si5351_driver_data *drvdata, int num
 	dev_dbg(&drvdata->client->dev, "%s - %s: pll = %d\n",
 		__func__, clk_hw_get_name(&drvdata->clkout[num].hw),
 		(val & SI5351_CLK_PLL_SELECT) ? 1 : 0);
+
+	return err;
 }
 
 static int si5351_clkout_prepare(struct clk_hw *hw)
@@ -947,8 +949,18 @@ static int si5351_clkout_prepare(struct clk_hw *hw)
 	 * Do a pll soft reset on the parent pll -- needed to get a
 	 * deterministic phase relationship between the output clocks.
 	 */
-	if (pdata->clkout[hwdata->num].pll_reset)
-		_si5351_clkout_reset_pll(hwdata->drvdata, hwdata->num);
+	if (pdata->clkout[hwdata->num].pll_reset) {
+		int ret;
+
+		ret = _si5351_clkout_reset_pll(hwdata->drvdata, hwdata->num);
+		if (ret) {
+			si5351_set_bits(hwdata->drvdata,
+					SI5351_CLK0_CTRL + hwdata->num,
+					SI5351_CLK_POWERDOWN,
+					SI5351_CLK_POWERDOWN);
+			return ret;
+		}
+	}
 
 	si5351_set_bits(hwdata->drvdata, SI5351_OUTPUT_ENABLE_CTRL,
 			(1 << hwdata->num), 0);
diff --git a/drivers/clk/clk-si544.c b/drivers/clk/clk-si544.c
index 9c520128e0a2c4..3924ec3e216369 100644
--- a/drivers/clk/clk-si544.c
+++ b/drivers/clk/clk-si544.c
@@ -445,8 +445,8 @@ static const struct regmap_config si544_regmap_config = {
 
 static int si544_probe(struct i2c_client *client)
 {
+	struct clk_init_data init = {};
 	struct clk_si544 *data;
-	struct clk_init_data init;
 	int err;
 
 	data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
diff --git a/drivers/clk/clk-si570.c b/drivers/clk/clk-si570.c
index 3063e38c66da0e..ac9013e7379c80 100644
--- a/drivers/clk/clk-si570.c
+++ b/drivers/clk/clk-si570.c
@@ -412,7 +412,7 @@ static const struct regmap_config si570_regmap_config = {
 static int si570_probe(struct i2c_client *client)
 {
 	struct clk_si570 *data;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	u32 initial_fout, factory_fout, stability;
 	bool skip_recall;
 	int err;
diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c
index 565b4d99bebb06..6603d481551415 100644
--- a/drivers/clk/clk-stm32f4.c
+++ b/drivers/clk/clk-stm32f4.c
@@ -485,8 +485,8 @@ static struct clk *clk_register_apb_mul(struct device *dev, const char *name,
 					const char *parent_name,
 					unsigned long flags, u8 bit_idx)
 {
+	struct clk_init_data init = {};
 	struct clk_apb_mul *am;
-	struct clk_init_data init;
 	struct clk *clk;
 
 	am = kzalloc_obj(*am);
@@ -810,8 +810,8 @@ static struct clk_hw *clk_register_pll_div(const char *name,
 		struct clk_hw *pll_hw, spinlock_t *lock)
 {
 	struct stm32f4_pll_div *pll_div;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	/* allocate the divider */
diff --git a/drivers/clk/clk-stm32h7.c b/drivers/clk/clk-stm32h7.c
index 82e0789cf189c7..c6afa9b202351c 100644
--- a/drivers/clk/clk-stm32h7.c
+++ b/drivers/clk/clk-stm32h7.c
@@ -462,8 +462,8 @@ static struct clk_hw *clk_register_stm32_timer_ker(struct device *dev,
 		u8 dppre_shift,
 		spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct timer_ker *element;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int err;
 
diff --git a/drivers/clk/clk-versaclock5.c b/drivers/clk/clk-versaclock5.c
index b5114892021692..d91749be785c35 100644
--- a/drivers/clk/clk-versaclock5.c
+++ b/drivers/clk/clk-versaclock5.c
@@ -1223,7 +1223,7 @@ static void vc5_remove(struct i2c_client *client)
 		clk_unregister_fixed_rate(vc5->pin_xin);
 }
 
-static int __maybe_unused vc5_suspend(struct device *dev)
+static int vc5_suspend(struct device *dev)
 {
 	struct vc5_driver_data *vc5 = dev_get_drvdata(dev);
 
@@ -1233,15 +1233,17 @@ static int __maybe_unused vc5_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused vc5_resume(struct device *dev)
+static int vc5_resume(struct device *dev)
 {
 	struct vc5_driver_data *vc5 = dev_get_drvdata(dev);
 	int ret;
 
 	regcache_cache_only(vc5->regmap, false);
 	ret = regcache_sync(vc5->regmap);
-	if (ret)
+	if (ret) {
+		regcache_cache_only(vc5->regmap, true);
 		dev_err(dev, "Failed to restore register map: %d\n", ret);
+	}
 	return ret;
 }
 
@@ -1335,12 +1337,12 @@ static const struct of_device_id clk_vc5_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, clk_vc5_of_match);
 
-static SIMPLE_DEV_PM_OPS(vc5_pm_ops, vc5_suspend, vc5_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(vc5_pm_ops, vc5_suspend, vc5_resume);
 
 static struct i2c_driver vc5_driver = {
 	.driver = {
 		.name = "vc5",
-		.pm	= &vc5_pm_ops,
+		.pm	= pm_sleep_ptr(&vc5_pm_ops),
 		.of_match_table = clk_vc5_of_match,
 	},
 	.probe		= vc5_probe,
diff --git a/drivers/clk/clk-vt8500.c b/drivers/clk/clk-vt8500.c
index 3e62cb853d5e90..581ac00c31bc04 100644
--- a/drivers/clk/clk-vt8500.c
+++ b/drivers/clk/clk-vt8500.c
@@ -227,8 +227,8 @@ static __init void vtwm_device_clk_init(struct device_node *node)
 	struct clk_hw *hw;
 	struct clk_device *dev_clk;
 	const char *clk_name = node->name;
+	struct clk_init_data init = {};
 	const char *parent_name;
-	struct clk_init_data init;
 	int rc;
 	int clk_init_flags = 0;
 
@@ -687,8 +687,8 @@ static __init void vtwm_pll_clk_init(struct device_node *node, int pll_type)
 	struct clk_hw *hw;
 	struct clk_pll *pll_clk;
 	const char *clk_name = node->name;
+	struct clk_init_data init = {};
 	const char *parent_name;
-	struct clk_init_data init;
 	int rc;
 
 	if (!pmc_base)
diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c
index abb6c8fcdc91fa..2029962ae5aae5 100644
--- a/drivers/clk/clk-xgene.c
+++ b/drivers/clk/clk-xgene.c
@@ -126,9 +126,9 @@ static struct clk *xgene_register_clk_pll(struct device *dev,
 	unsigned long flags, void __iomem *reg, u32 pll_offset,
 	u32 type, spinlock_t *lock, int version)
 {
+	struct clk_init_data init = {};
 	struct xgene_clk_pll *apmclk;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	/* allocate the APM clock structure */
 	apmclk = kzalloc_obj(*apmclk);
@@ -350,8 +350,8 @@ xgene_register_clk_pmd(struct device *dev,
 		       unsigned long flags, void __iomem *reg, u8 shift,
 		       u8 width, u64 denom, u32 clk_flags, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct xgene_clk_pmd *fd;
-	struct clk_init_data init;
 	struct clk *clk;
 
 	fd = kzalloc_obj(*fd);
@@ -634,9 +634,9 @@ static struct clk *xgene_register_clk(struct device *dev,
 		const char *name, const char *parent_name,
 		struct xgene_dev_parameters *parameters, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct xgene_clk *apmclk;
 	struct clk *clk;
-	struct clk_init_data init;
 	int rc;
 
 	/* allocate the APM clock structure */
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index fef87167a60b5e..4d2cc86bfbeb64 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -24,6 +24,7 @@
 #include <linux/spinlock.h>
 #include <linux/string.h>
 #include <linux/stringhash.h>
+#include <kunit/visibility.h>
 
 #include "clk.h"
 
@@ -166,6 +167,7 @@ struct clk {
 	unsigned long max_rate;
 	unsigned int exclusive_count;
 	struct hlist_node clks_node;
+	bool pinning;
 };
 
 /***           runtime pm          ***/
@@ -2851,6 +2853,20 @@ int clk_hw_set_spread_spectrum(struct clk_hw *hw, const struct clk_spread_spectr
 	if (!hw)
 		return 0;
 
+	switch (ss_conf->method) {
+	case CLK_SPREAD_NO:
+		break;
+	case CLK_SPREAD_CENTER:
+	case CLK_SPREAD_UP:
+	case CLK_SPREAD_DOWN:
+		if (!ss_conf->modfreq_hz || !ss_conf->spread_bp ||
+		    ss_conf->spread_bp > 10000)
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	core = hw->core;
 
 	clk_prepare_lock();
@@ -4237,13 +4253,15 @@ static void free_clk(struct clk *clk)
  * @hw: clk_hw associated with the clk being consumed
  * @dev_id: string describing device name
  * @con_id: connection ID string on device
+ * @owner: reference to the module creating the clock
  *
  * This is the main function used to create a clk pointer for use by clk
  * consumers. It connects a consumer to the clk_core and clk_hw structures
  * used by the framework and clk provider respectively.
  */
 struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw,
-			      const char *dev_id, const char *con_id)
+			      const char *dev_id, const char *con_id,
+			      struct module *owner)
 {
 	struct clk *clk;
 	struct clk_core *core;
@@ -4258,7 +4276,13 @@ struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw,
 		return clk;
 	clk->dev = dev;
 
-	if (!try_module_get(core->owner)) {
+	/*
+	 * Pin the provider module only when the consumer lives in a different
+	 * module. A provider getting a clk from its own clk_hw would otherwise
+	 * pin itself and could never be unloaded.
+	 */
+	clk->pinning = owner != core->owner;
+	if (clk->pinning && !try_module_get(core->owner)) {
 		free_clk(clk);
 		return ERR_PTR(-ENOENT);
 	}
@@ -4269,24 +4293,19 @@ struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw,
 	return clk;
 }
 
-/**
- * clk_hw_get_clk - get clk consumer given an clk_hw
- * @hw: clk_hw associated with the clk being consumed
- * @con_id: connection ID string on device
- *
- * Returns: new clk consumer
- * This is the function to be used by providers which need
- * to get a consumer clk and act on the clock element
- * Calls to this function must be balanced with calls clk_put()
+/*
+ * Internal helper backing the clk_hw_get_clk() macro, which passes the caller's
+ * module via THIS_MODULE.
  */
-struct clk *clk_hw_get_clk(struct clk_hw *hw, const char *con_id)
+struct clk *__clk_hw_get_clk(struct clk_hw *hw, const char *con_id,
+			     struct module *owner)
 {
 	struct device *dev = hw->core->dev;
 	const char *name = dev ? dev_name(dev) : NULL;
 
-	return clk_hw_create_clk(dev, hw, name, con_id);
+	return clk_hw_create_clk(dev, hw, name, con_id, owner);
 }
-EXPORT_SYMBOL(clk_hw_get_clk);
+EXPORT_SYMBOL(__clk_hw_get_clk);
 
 static int clk_cpy_name(const char **dst_p, const char *src, bool must_exist)
 {
@@ -4450,8 +4469,9 @@ __clk_register(struct device *dev, struct device_node *np, struct clk_hw *hw)
 	INIT_HLIST_HEAD(&core->clks);
 
 	/*
-	 * Don't call clk_hw_create_clk() here because that would pin the
-	 * provider module to itself and prevent it from ever being removed.
+	 * Don't call clk_hw_create_clk() here because it would systematically
+	 * add a consumer to the hw clock and all clocks would appear to have
+	 * consumer in the clock summary.
 	 */
 	hw->clk = alloc_clk(core, NULL, NULL);
 	if (IS_ERR(hw->clk)) {
@@ -4808,7 +4828,7 @@ struct clk *devm_clk_hw_get_clk(struct device *dev, struct clk_hw *hw,
 	if (!clkp)
 		return ERR_PTR(-ENOMEM);
 
-	clk = clk_hw_get_clk(hw, con_id);
+	clk = __clk_hw_get_clk(hw, con_id, dev->driver->owner);
 	if (!IS_ERR(clk)) {
 		*clkp = clk;
 		devres_add(dev, clkp);
@@ -4855,7 +4875,8 @@ void __clk_put(struct clk *clk)
 
 	owner = clk->core->owner;
 	kref_put(&clk->core->ref, __clk_release);
-	module_put(owner);
+	if (clk->pinning)
+		module_put(owner);
 	free_clk(clk);
 }
 
@@ -5297,13 +5318,13 @@ static int of_parse_clkspec(const struct device_node *np, int index,
 		/*
 		 * For named clocks, first look up the name in the
 		 * "clock-names" property.  If it cannot be found, then index
-		 * will be an error code and of_parse_phandle_with_args() will
+		 * will be an error code and the OF phandle parser will
 		 * return -EINVAL.
 		 */
 		if (name)
 			index = of_property_match_string(np, "clock-names", name);
-		ret = of_parse_phandle_with_args(np, "clocks", "#clock-cells",
-						 index, out_args);
+		ret = of_parse_phandle_with_args_map(np, "clocks", "clock",
+						     index, out_args);
 		if (!ret)
 			break;
 		if (name && index >= 0)
@@ -5314,6 +5335,8 @@ static int of_parse_clkspec(const struct device_node *np, int index,
 		 * has a "clock-ranges" property, then we can try one of its
 		 * clocks.
 		 */
+		if (of_property_present(np, "clocks"))
+			break;
 		np = np->parent;
 		if (np && !of_property_present(np, "clock-ranges"))
 			break;
@@ -5370,13 +5393,13 @@ of_clk_get_hw_from_clkspec(struct of_phandle_args *clkspec)
  *
  * This function looks up a struct clk from the registered list of clock
  * providers, an input is a clock specifier data structure as returned
- * from the of_parse_phandle_with_args() function call.
+ * from the OF phandle parser.
  */
 struct clk *of_clk_get_from_provider(struct of_phandle_args *clkspec)
 {
 	struct clk_hw *hw = of_clk_get_hw_from_clkspec(clkspec);
 
-	return clk_hw_create_clk(NULL, hw, NULL, __func__);
+	return clk_hw_create_clk(NULL, hw, NULL, __func__, NULL);
 }
 EXPORT_SYMBOL_GPL(of_clk_get_from_provider);
 
@@ -5396,6 +5419,7 @@ struct clk_hw *of_clk_get_hw(struct device_node *np, int index,
 
 	return hw;
 }
+EXPORT_SYMBOL_IF_KUNIT(of_clk_get_hw);
 
 static struct clk *__of_clk_get(struct device_node *np,
 				int index, const char *dev_id,
@@ -5403,7 +5427,7 @@ static struct clk *__of_clk_get(struct device_node *np,
 {
 	struct clk_hw *hw = of_clk_get_hw(np, index, con_id);
 
-	return clk_hw_create_clk(NULL, hw, dev_id, con_id);
+	return clk_hw_create_clk(NULL, hw, dev_id, con_id, NULL);
 }
 
 struct clk *of_clk_get(struct device_node *np, int index)
@@ -5458,8 +5482,7 @@ const char *of_clk_get_parent_name(const struct device_node *np, int index)
 	int count;
 	struct clk *clk;
 
-	rc = of_parse_phandle_with_args(np, "clocks", "#clock-cells", index,
-					&clkspec);
+	rc = of_parse_clkspec(np, index, NULL, &clkspec);
 	if (rc)
 		return NULL;
 
diff --git a/drivers/clk/clk.h b/drivers/clk/clk.h
index 2d801900cad598..5009380863a9dc 100644
--- a/drivers/clk/clk.h
+++ b/drivers/clk/clk.h
@@ -23,13 +23,14 @@ struct clk_hw *clk_find_hw(const char *dev_id, const char *con_id);
 
 #ifdef CONFIG_COMMON_CLK
 struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw,
-			      const char *dev_id, const char *con_id);
+			      const char *dev_id, const char *con_id,
+			      struct module *owner);
 void __clk_put(struct clk *clk);
 #else
 /* All these casts to avoid ifdefs in clkdev... */
 static inline struct clk *
 clk_hw_create_clk(struct device *dev, struct clk_hw *hw, const char *dev_id,
-		  const char *con_id)
+		  const char *con_id, struct module *owner)
 {
 	return (struct clk *)hw;
 }
diff --git a/drivers/clk/clk_kunit_helpers.c b/drivers/clk/clk_kunit_helpers.c
index 68a28e70bb61cd..ab6b2cffcad606 100644
--- a/drivers/clk/clk_kunit_helpers.c
+++ b/drivers/clk/clk_kunit_helpers.c
@@ -233,5 +233,36 @@ int of_clk_add_hw_provider_kunit(struct kunit *test, struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(of_clk_add_hw_provider_kunit);
 
+KUNIT_DEFINE_ACTION_WRAPPER(of_node_put_wrapper, of_node_put, struct device_node *);
+
+/**
+ * clk_of_find_node_by_name_kunit() - Test managed of_find_node_by_name()
+ * @test: The test context
+ * @from: Parent device node to start searching from, or NULL to search from root
+ * @name: The name string to match against
+ *
+ * Just like of_find_node_by_name(), except the device_node is managed by
+ * the test case and is automatically put after the test case concludes.
+ *
+ * Return: the device_node on success, NULL if not found, or an error pointer on failure.
+ */
+struct device_node *clk_of_find_node_by_name_kunit(struct kunit *test, struct device_node *from,
+						   const char *name)
+{
+	struct device_node *np;
+	int ret;
+
+	np = of_find_node_by_name(from, name);
+	if (!np)
+		return NULL;
+
+	ret = kunit_add_action_or_reset(test, of_node_put_wrapper, np);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return np;
+}
+EXPORT_SYMBOL_GPL(clk_of_find_node_by_name_kunit);
+
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("KUnit helpers for clk providers and consumers");
diff --git a/drivers/clk/clk_test.c b/drivers/clk/clk_test.c
index 1c5c8b7c1f3c17..0505ed357b20f1 100644
--- a/drivers/clk/clk_test.c
+++ b/drivers/clk/clk_test.c
@@ -3526,7 +3526,7 @@ static const struct clk_assigned_rates_test_param clk_assigned_sscs_skips_test_p
 		 */
 		.desc = "provider missing assigned-clocks",
 		TEST_PARAM_OVERLAY(kunit_clk_assigned_sscs_without),
-		.sscs = {50000, 60000, 3},
+		.sscs = {50000, 600, 3},
 	},
 	{
 		/*
@@ -3535,7 +3535,7 @@ static const struct clk_assigned_rates_test_param clk_assigned_sscs_skips_test_p
 		 */
 		.desc = "consumer missing assigned-clocks",
 		TEST_PARAM_OVERLAY(kunit_clk_assigned_sscs_without_consumer),
-		.sscs = {50000, 60000, 3},
+		.sscs = {50000, 600, 3},
 		.consumer_test = true,
 	},
 	{
@@ -3545,7 +3545,7 @@ static const struct clk_assigned_rates_test_param clk_assigned_sscs_skips_test_p
 		 */
 		.desc = "provider assigned-clock-sscs of zero",
 		TEST_PARAM_OVERLAY(kunit_clk_assigned_sscs_zero),
-		.sscs = {50000, 60000, 3},
+		.sscs = {50000, 600, 3},
 	},
 	{
 		/*
@@ -3554,7 +3554,7 @@ static const struct clk_assigned_rates_test_param clk_assigned_sscs_skips_test_p
 		 */
 		.desc = "consumer assigned-clock-sscs of zero",
 		TEST_PARAM_OVERLAY(kunit_clk_assigned_sscs_zero_consumer),
-		.sscs = {50000, 60000, 3},
+		.sscs = {50000, 600, 3},
 		.consumer_test = true,
 	},
 	{
@@ -3564,7 +3564,7 @@ static const struct clk_assigned_rates_test_param clk_assigned_sscs_skips_test_p
 		 */
 		.desc = "provider assigned-clocks null phandle",
 		TEST_PARAM_OVERLAY(kunit_clk_assigned_sscs_null),
-		.sscs = {50000, 60000, 3},
+		.sscs = {50000, 600, 3},
 	},
 	{
 		/*
@@ -3573,7 +3573,7 @@ static const struct clk_assigned_rates_test_param clk_assigned_sscs_skips_test_p
 		 */
 		.desc = "consumer assigned-clocks null phandle",
 		TEST_PARAM_OVERLAY(kunit_clk_assigned_sscs_null_consumer),
-		.sscs = {50000, 60000, 3},
+		.sscs = {50000, 600, 3},
 		.consumer_test = true,
 	},
 };
@@ -3738,11 +3738,149 @@ static struct kunit_suite clk_hw_get_dev_of_node_test_suite = {
 	.test_cases = clk_hw_get_dev_of_node_test_cases,
 };
 
+static const struct clk_init_data clk_parse_clkspec_1_init_data = {
+	.name = "clk_parse_clkspec_1",
+	.ops = &empty_clk_ops,
+};
+
+static const struct clk_init_data clk_parse_clkspec_2_init_data = {
+	.name = "clk_parse_clkspec_2",
+	.ops = &empty_clk_ops,
+};
+
+struct clk_parse_clkspec_ctx {
+	struct device_node *cons_np;
+};
+
+static int clk_parse_clkspec_init(struct kunit *test)
+{
+	struct device_node *prov1_np, *prov2_np;
+	struct clk_parse_clkspec_ctx *ctx;
+	struct clk_hw *hw1, *hw2;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+	test->priv = ctx;
+
+	KUNIT_ASSERT_EQ(test, 0, of_overlay_apply_kunit(test, kunit_clk_parse_clkspec));
+
+	/* Register provider 1 */
+	hw1 = kunit_kzalloc(test, sizeof(*hw1), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hw1);
+	hw1->init = &clk_parse_clkspec_1_init_data;
+
+	prov1_np = of_find_compatible_node(NULL, NULL, "test,clock-provider1");
+	KUNIT_ASSERT_NOT_NULL(test, prov1_np);
+	of_node_put_kunit(test, prov1_np);
+
+	KUNIT_ASSERT_EQ(test, 0, of_clk_hw_register_kunit(test, prov1_np, hw1));
+	KUNIT_ASSERT_EQ(test, 0, of_clk_add_hw_provider_kunit(test, prov1_np, of_clk_hw_simple_get, hw1));
+
+	/* Register provider 2 */
+	hw2 = kunit_kzalloc(test, sizeof(*hw2), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hw2);
+	hw2->init = &clk_parse_clkspec_2_init_data;
+
+	prov2_np = of_find_compatible_node(NULL, NULL, "test,clock-provider2");
+	KUNIT_ASSERT_NOT_NULL(test, prov2_np);
+	of_node_put_kunit(test, prov2_np);
+
+	KUNIT_ASSERT_EQ(test, 0, of_clk_hw_register_kunit(test, prov2_np, hw2));
+	KUNIT_ASSERT_EQ(test, 0, of_clk_add_hw_provider_kunit(test, prov2_np, of_clk_hw_simple_get, hw2));
+
+	ctx->cons_np = of_find_compatible_node(NULL, NULL, "test,clock-consumer");
+	KUNIT_ASSERT_NOT_NULL(test, ctx->cons_np);
+	of_node_put_kunit(test, ctx->cons_np);
+
+	return 0;
+}
+
+/* Test DT phandle lookups using correct index or name succeed */
+static void clk_parse_clkspec_with_correct_index_and_name(struct kunit *test)
+{
+	struct clk_parse_clkspec_ctx *ctx = test->priv;
+	struct clk_hw *hw1, *hw2, *hw3, *hw4;
+
+	/* Get clocks by index */
+	hw1 = of_clk_get_hw(ctx->cons_np, 0, NULL);
+	KUNIT_EXPECT_NOT_ERR_OR_NULL(test, hw1);
+
+	hw2 = of_clk_get_hw(ctx->cons_np, 1, NULL);
+	KUNIT_EXPECT_NOT_ERR_OR_NULL(test, hw2);
+	KUNIT_EXPECT_PTR_NE(test, hw1, hw2);
+
+	/* Get clocks by name */
+	hw3 = of_clk_get_hw(ctx->cons_np, 0, "first_clock");
+	KUNIT_EXPECT_NOT_ERR_OR_NULL(test, hw3);
+	KUNIT_EXPECT_PTR_EQ(test, hw1, hw3);
+
+	hw4 = of_clk_get_hw(ctx->cons_np, 0, "second_clock");
+	KUNIT_EXPECT_NOT_ERR_OR_NULL(test, hw4);
+	KUNIT_EXPECT_PTR_EQ(test, hw2, hw4);
+}
+
+/* Test DT phandle lookups using wrong index or name fail */
+static void clk_parse_clkspec_with_incorrect_index_and_name(struct kunit *test)
+{
+	struct clk_parse_clkspec_ctx *ctx = test->priv;
+	struct clk_hw *hw;
+
+	/* Get clock by index */
+	hw = of_clk_get_hw(ctx->cons_np, 2, NULL);
+	KUNIT_EXPECT_TRUE(test, IS_ERR(hw));
+
+	/* Get clock by name */
+	hw = of_clk_get_hw(ctx->cons_np, 0, "third_clock");
+	KUNIT_EXPECT_TRUE(test, IS_ERR(hw));
+}
+
+/*
+ * Verify that of_clk_get_parent_name() returns the correct clock name when
+ * looking up by index through the consumer's clocks property.
+ */
+static void of_clk_get_parent_name_gets_parent_name(struct kunit *test)
+{
+	struct clk_parse_clkspec_ctx *ctx = test->priv;
+	const char *expected_name = "clk_parse_clkspec_1";
+
+	KUNIT_EXPECT_STREQ(test, expected_name,
+			   of_clk_get_parent_name(ctx->cons_np, 0));
+}
+
+static void of_clk_get_hw_maps_thru_nexus(struct kunit *test)
+{
+	struct clk_parse_clkspec_ctx *ctx = test->priv;
+	struct clk_hw *expected;
+	struct device_node *np;
+
+	np = clk_of_find_node_by_name_kunit(test, NULL, "kunit-clock-nexus-child");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, np);
+	expected = of_clk_get_hw(ctx->cons_np, 1, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, expected);
+
+	KUNIT_EXPECT_PTR_EQ(test, expected, of_clk_get_hw(np, 0, NULL));
+}
+
+static struct kunit_case clk_parse_clkspec_test_cases[] = {
+	KUNIT_CASE(clk_parse_clkspec_with_correct_index_and_name),
+	KUNIT_CASE(clk_parse_clkspec_with_incorrect_index_and_name),
+	KUNIT_CASE(of_clk_get_parent_name_gets_parent_name),
+	KUNIT_CASE(of_clk_get_hw_maps_thru_nexus),
+	{}
+};
+
+/* Test suite to verify clk_parse_clkspec() */
+static struct kunit_suite clk_parse_clkspec_test_suite = {
+	.name = "clk_parse_clkspec",
+	.init = clk_parse_clkspec_init,
+	.test_cases = clk_parse_clkspec_test_cases,
+};
 
 kunit_test_suites(
 	&clk_assigned_rates_suite,
 	&clk_assigned_sscs_suite,
 	&clk_hw_get_dev_of_node_test_suite,
+	&clk_parse_clkspec_test_suite,
 	&clk_leaf_mux_set_rate_parent_test_suite,
 	&clk_test_suite,
 	&clk_multiple_parents_mux_test_suite,
@@ -3760,4 +3898,5 @@ kunit_test_suites(
 	&clk_uncached_test_suite,
 );
 MODULE_DESCRIPTION("Kunit tests for clk framework");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c
index abaa0f9e00831b..919fc680ae4dd6 100644
--- a/drivers/clk/clkdev.c
+++ b/drivers/clk/clkdev.c
@@ -88,7 +88,7 @@ static struct clk *__clk_get_sys(struct device *dev, const char *dev_id,
 {
 	struct clk_hw *hw = clk_find_hw(dev_id, con_id);
 
-	return clk_hw_create_clk(dev, hw, dev_id, con_id);
+	return clk_hw_create_clk(dev, hw, dev_id, con_id, NULL);
 }
 
 struct clk *clk_get_sys(const char *dev_id, const char *con_id)
@@ -105,7 +105,7 @@ struct clk *clk_get(struct device *dev, const char *con_id)
 	if (dev && dev->of_node) {
 		hw = of_clk_get_hw(dev->of_node, 0, con_id);
 		if (!IS_ERR(hw) || PTR_ERR(hw) == -EPROBE_DEFER)
-			return clk_hw_create_clk(dev, hw, dev_id, con_id);
+			return clk_hw_create_clk(dev, hw, dev_id, con_id, NULL);
 	}
 
 	return __clk_get_sys(dev, dev_id, con_id);
diff --git a/drivers/clk/davinci/Kconfig b/drivers/clk/davinci/Kconfig
new file mode 100644
index 00000000000000..7b8c518ef6b455
--- /dev/null
+++ b/drivers/clk/davinci/Kconfig
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config COMMON_CLK_DAVINCI
+	bool "Clock drivers for TI DaVinci SoCs"
+	depends on ARCH_DAVINCI || COMPILE_TEST
+	depends on HAS_IOMEM
+	select REGMAP_MMIO
+	select RESET_CONTROLLER
+	default ARCH_DAVINCI
+	help
+	  Support for the clock controllers found on TI DaVinci SoCs.
diff --git a/drivers/clk/davinci/Makefile b/drivers/clk/davinci/Makefile
index f9d5c9a392e483..7cb030b5e0469d 100644
--- a/drivers/clk/davinci/Makefile
+++ b/drivers/clk/davinci/Makefile
@@ -1,11 +1,9 @@
 # SPDX-License-Identifier: GPL-2.0
 
-ifeq ($(CONFIG_COMMON_CLK), y)
-obj-$(CONFIG_ARCH_DAVINCI_DA8XX)	+= da8xx-cfgchip.o
+obj-y += da8xx-cfgchip.o
 
 obj-y += pll.o
-obj-$(CONFIG_ARCH_DAVINCI_DA850)	+= pll-da850.o
+obj-y += pll-da850.o
 
 obj-y += psc.o
-obj-$(CONFIG_ARCH_DAVINCI_DA850)	+= psc-da850.o
-endif
+obj-y += psc-da850.o
diff --git a/drivers/clk/davinci/da8xx-cfgchip.c b/drivers/clk/davinci/da8xx-cfgchip.c
index a5109fe8b16e9b..9d7fced898ae07 100644
--- a/drivers/clk/davinci/da8xx-cfgchip.c
+++ b/drivers/clk/davinci/da8xx-cfgchip.c
@@ -91,7 +91,7 @@ da8xx_cfgchip_gate_clk_register(struct device *dev,
 	struct clk *parent;
 	const char *parent_name;
 	struct da8xx_cfgchip_gate_clk *gate;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int ret;
 
 	parent = devm_clk_get(dev, NULL);
@@ -241,7 +241,7 @@ da8xx_cfgchip_mux_clk_register(struct device *dev,
 {
 	const char * const parent_names[] = { info->parent0, info->parent1 };
 	struct da8xx_cfgchip_mux_clk *mux;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int ret;
 
 	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
diff --git a/drivers/clk/davinci/pll.c b/drivers/clk/davinci/pll.c
index f73b0ac5d8a08b..0e58b02d7acaf1 100644
--- a/drivers/clk/davinci/pll.c
+++ b/drivers/clk/davinci/pll.c
@@ -371,7 +371,7 @@ struct clk *davinci_pll_clk_register(struct device *dev,
 	char pllout_name[MAX_NAME_SIZE];
 	char postdiv_name[MAX_NAME_SIZE];
 	char pllen_name[MAX_NAME_SIZE];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct davinci_pll_clk *pllout;
 	struct davinci_pllen_clk *pllen;
 	struct clk *oscin_clk = NULL;
diff --git a/drivers/clk/davinci/psc.c b/drivers/clk/davinci/psc.c
index ff603520d56f94..73dce7ce2be6cc 100644
--- a/drivers/clk/davinci/psc.c
+++ b/drivers/clk/davinci/psc.c
@@ -234,7 +234,7 @@ davinci_lpsc_clk_register(struct device *dev, const char *name,
 			  const char *parent_name, struct regmap *regmap,
 			  u32 md, u32 pd, u32 flags)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct davinci_lpsc_clk *lpsc;
 	int ret;
 	bool is_on;
diff --git a/drivers/clk/hisilicon/clk-hi3559a.c b/drivers/clk/hisilicon/clk-hi3559a.c
index f297fb25c512a3..fea6fb9e98f59a 100644
--- a/drivers/clk/hisilicon/clk-hi3559a.c
+++ b/drivers/clk/hisilicon/clk-hi3559a.c
@@ -456,8 +456,8 @@ static void hisi_clk_register_pll(struct hi3559av100_pll_clock *clks,
 {
 	void __iomem *base = data->base;
 	struct hi3559av100_clk_pll *p_clk = NULL;
+	struct clk_init_data init = {};
 	struct clk *clk = NULL;
-	struct clk_init_data init;
 	int i;
 
 	p_clk = devm_kcalloc(dev, nums, sizeof(*p_clk), GFP_KERNEL);
diff --git a/drivers/clk/hisilicon/clk-hi3620.c b/drivers/clk/hisilicon/clk-hi3620.c
index 8dcad74e679aa4..a2c4637eaa9880 100644
--- a/drivers/clk/hisilicon/clk-hi3620.c
+++ b/drivers/clk/hisilicon/clk-hi3620.c
@@ -410,9 +410,9 @@ static const struct clk_ops clk_mmc_ops = {
 static struct clk *hisi_register_clk_mmc(struct hisi_mmc_clock *mmc_clk,
 			void __iomem *base, struct device_node *np)
 {
+	struct clk_init_data init = {};
 	struct clk_mmc *mclk;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	mclk = kzalloc_obj(*mclk);
 	if (!mclk)
diff --git a/drivers/clk/hisilicon/clk-hi6220-stub.c b/drivers/clk/hisilicon/clk-hi6220-stub.c
index bf99cfafafa013..a70544d2bdb9a8 100644
--- a/drivers/clk/hisilicon/clk-hi6220-stub.c
+++ b/drivers/clk/hisilicon/clk-hi6220-stub.c
@@ -195,7 +195,7 @@ static const struct clk_ops hi6220_stub_clk_ops = {
 static int hi6220_stub_clk_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct hi6220_stub_clk *stub_clk;
 	struct clk *clk;
 	struct device_node *np = pdev->dev.of_node;
diff --git a/drivers/clk/hisilicon/clk-hisi-phase.c b/drivers/clk/hisilicon/clk-hisi-phase.c
index ba6afad66a2b0c..d8e555d9728de9 100644
--- a/drivers/clk/hisilicon/clk-hisi-phase.c
+++ b/drivers/clk/hisilicon/clk-hisi-phase.c
@@ -94,8 +94,8 @@ struct clk *clk_register_hisi_phase(struct device *dev,
 		const struct hisi_phase_clock *clks,
 		void __iomem *base, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clk_hisi_phase *phase;
-	struct clk_init_data init;
 
 	phase = devm_kzalloc(dev, sizeof(struct clk_hisi_phase), GFP_KERNEL);
 	if (!phase)
diff --git a/drivers/clk/hisilicon/clk-hix5hd2.c b/drivers/clk/hisilicon/clk-hix5hd2.c
index 57db6fab43aacd..4b7ebebfac4be6 100644
--- a/drivers/clk/hisilicon/clk-hix5hd2.c
+++ b/drivers/clk/hisilicon/clk-hix5hd2.c
@@ -258,8 +258,8 @@ hix5hd2_clk_register_complex(struct hix5hd2_complex_clock *clks, int nums,
 
 	for (i = 0; i < nums; i++) {
 		struct hix5hd2_clk_complex *p_clk;
+		struct clk_init_data init = {};
 		struct clk *clk;
-		struct clk_init_data init;
 
 		p_clk = kzalloc_obj(*p_clk);
 		if (!p_clk)
diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c
index 20a337383a1e1f..cc53d0a33e3c4e 100644
--- a/drivers/clk/hisilicon/clkdivider-hi6220.c
+++ b/drivers/clk/hisilicon/clkdivider-hi6220.c
@@ -103,7 +103,7 @@ struct clk *hi6220_register_clkdiv(struct device *dev, const char *name,
 {
 	struct hi6220_clk_divider *div;
 	struct clk *clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_div_table *table;
 	u32 max_div, min_div;
 	int i;
diff --git a/drivers/clk/hisilicon/clkgate-separated.c b/drivers/clk/hisilicon/clkgate-separated.c
index 199303157ebd84..e02d6bb5ada79a 100644
--- a/drivers/clk/hisilicon/clkgate-separated.c
+++ b/drivers/clk/hisilicon/clkgate-separated.c
@@ -87,8 +87,8 @@ struct clk *hisi_register_clkgate_sep(struct device *dev, const char *name,
 				      u8 clk_gate_flags, spinlock_t *lock)
 {
 	struct clkgate_separated *sclk;
+	struct clk_init_data init = {};
 	struct clk *clk;
-	struct clk_init_data init;
 
 	sclk = kzalloc_obj(*sclk);
 	if (!sclk)
diff --git a/drivers/clk/imx/clk-busy.c b/drivers/clk/imx/clk-busy.c
index b8b6e7c7e1fe35..75a2bf29e42fc6 100644
--- a/drivers/clk/imx/clk-busy.c
+++ b/drivers/clk/imx/clk-busy.c
@@ -78,8 +78,8 @@ struct clk_hw *imx_clk_hw_busy_divider(const char *name, const char *parent_name
 				 void __iomem *busy_reg, u8 busy_shift)
 {
 	struct clk_busy_divider *busy;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	busy = kzalloc_obj(*busy);
@@ -157,9 +157,9 @@ struct clk_hw *imx_clk_hw_busy_mux(const char *name, void __iomem *reg, u8 shift
 			     u8 width, void __iomem *busy_reg, u8 busy_shift,
 			     const char * const *parent_names, int num_parents)
 {
+	struct clk_init_data init = {};
 	struct clk_busy_mux *busy;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	busy = kzalloc_obj(*busy);
diff --git a/drivers/clk/imx/clk-cpu.c b/drivers/clk/imx/clk-cpu.c
index f53dbaacddcc9f..b4d9722eb17e63 100644
--- a/drivers/clk/imx/clk-cpu.c
+++ b/drivers/clk/imx/clk-cpu.c
@@ -76,9 +76,9 @@ struct clk_hw *imx_clk_hw_cpu(const char *name, const char *parent_name,
 		struct clk *div, struct clk *mux, struct clk *pll,
 		struct clk *step)
 {
+	struct clk_init_data init = {};
 	struct clk_cpu *cpu;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	cpu = kzalloc_obj(*cpu);
diff --git a/drivers/clk/imx/clk-divider-gate.c b/drivers/clk/imx/clk-divider-gate.c
index 957d132c4607dc..a1b7c109069c3b 100644
--- a/drivers/clk/imx/clk-divider-gate.c
+++ b/drivers/clk/imx/clk-divider-gate.c
@@ -179,8 +179,8 @@ struct clk_hw *imx_clk_hw_divider_gate(const char *name, const char *parent_name
 				    const struct clk_div_table *table,
 				    spinlock_t *lock)
 {
-	struct clk_init_data init;
 	struct clk_divider_gate *div_gate;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
 	u32 val;
 	int ret;
diff --git a/drivers/clk/imx/clk-fixup-div.c b/drivers/clk/imx/clk-fixup-div.c
index 3866319af54ef8..a6f68317cd2e52 100644
--- a/drivers/clk/imx/clk-fixup-div.c
+++ b/drivers/clk/imx/clk-fixup-div.c
@@ -90,8 +90,8 @@ struct clk_hw *imx_clk_hw_fixup_divider(const char *name, const char *parent,
 				  void (*fixup)(u32 *val))
 {
 	struct clk_fixup_div *fixup_div;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	if (!fixup)
diff --git a/drivers/clk/imx/clk-fixup-mux.c b/drivers/clk/imx/clk-fixup-mux.c
index 7188cdad79d3c8..f43648f7fb45a9 100644
--- a/drivers/clk/imx/clk-fixup-mux.c
+++ b/drivers/clk/imx/clk-fixup-mux.c
@@ -70,8 +70,8 @@ struct clk_hw *imx_clk_hw_fixup_mux(const char *name, void __iomem *reg,
 			      int num_parents, void (*fixup)(u32 *val))
 {
 	struct clk_fixup_mux *fixup_mux;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	if (!fixup)
diff --git a/drivers/clk/imx/clk-frac-pll.c b/drivers/clk/imx/clk-frac-pll.c
index cd4e26bac19c48..d6c59634c0c028 100644
--- a/drivers/clk/imx/clk-frac-pll.c
+++ b/drivers/clk/imx/clk-frac-pll.c
@@ -208,7 +208,7 @@ struct clk_hw *imx_clk_hw_frac_pll(const char *name,
 				   const char *parent_name,
 				   void __iomem *base)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_frac_pll *pll;
 	struct clk_hw *hw;
 	int ret;
diff --git a/drivers/clk/imx/clk-fracn-gppll.c b/drivers/clk/imx/clk-fracn-gppll.c
index 4048c16c0578b8..32bb3e9a8d6ec2 100644
--- a/drivers/clk/imx/clk-fracn-gppll.c
+++ b/drivers/clk/imx/clk-fracn-gppll.c
@@ -363,9 +363,9 @@ static struct clk_hw *_imx_clk_fracn_gppll(const char *name, const char *parent_
 					   const struct imx_fracn_gppll_clk *pll_clk,
 					   u32 pll_flags)
 {
+	struct clk_init_data init = {};
 	struct clk_fracn_gppll *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	pll = kzalloc_obj(*pll);
diff --git a/drivers/clk/imx/clk-gate-93.c b/drivers/clk/imx/clk-gate-93.c
index 7acbcfec9610b3..7208138d615dfb 100644
--- a/drivers/clk/imx/clk-gate-93.c
+++ b/drivers/clk/imx/clk-gate-93.c
@@ -158,9 +158,9 @@ struct clk_hw *imx93_clk_gate(struct device *dev, const char *name, const char *
 			      unsigned long flags, void __iomem *reg, u32 bit_idx, u32 val,
 			      u32 mask, u32 domain_id, unsigned int *share_count)
 {
+	struct clk_init_data init = {};
 	struct imx93_clk_gate *gate;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 	u32 authen;
 
diff --git a/drivers/clk/imx/clk-gate-exclusive.c b/drivers/clk/imx/clk-gate-exclusive.c
index 504cfeef28a08b..cf873e490362a2 100644
--- a/drivers/clk/imx/clk-gate-exclusive.c
+++ b/drivers/clk/imx/clk-gate-exclusive.c
@@ -59,9 +59,9 @@ struct clk_hw *imx_clk_hw_gate_exclusive(const char *name, const char *parent,
 	 void __iomem *reg, u8 shift, u32 exclusive_mask)
 {
 	struct clk_gate_exclusive *exgate;
+	struct clk_init_data init = {};
 	struct clk_gate *gate;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	if (exclusive_mask == 0)
diff --git a/drivers/clk/imx/clk-gate2.c b/drivers/clk/imx/clk-gate2.c
index f07cf82dd9c9a2..c8b6323571eec9 100644
--- a/drivers/clk/imx/clk-gate2.c
+++ b/drivers/clk/imx/clk-gate2.c
@@ -139,9 +139,9 @@ struct clk_hw *clk_hw_register_gate2(struct device *dev, const char *name,
 		u8 clk_gate2_flags, spinlock_t *lock,
 		unsigned int *share_count)
 {
+	struct clk_init_data init = {};
 	struct clk_gate2 *gate;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	gate = kzalloc_obj(struct clk_gate2);
diff --git a/drivers/clk/imx/clk-lpcg-scu.c b/drivers/clk/imx/clk-lpcg-scu.c
index 03bbfbe9bff3d3..da110a823fd5ba 100644
--- a/drivers/clk/imx/clk-lpcg-scu.c
+++ b/drivers/clk/imx/clk-lpcg-scu.c
@@ -114,8 +114,8 @@ struct clk_hw *__imx_clk_lpcg_scu(struct device *dev, const char *name,
 				  const char *parent_name, unsigned long flags,
 				  void __iomem *reg, u8 bit_idx, bool hw_gate)
 {
+	struct clk_init_data init = {};
 	struct clk_lpcg_scu *clk;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int ret;
 
diff --git a/drivers/clk/imx/clk-pfd.c b/drivers/clk/imx/clk-pfd.c
index e7a5d80c50083b..e56bb27e57cc92 100644
--- a/drivers/clk/imx/clk-pfd.c
+++ b/drivers/clk/imx/clk-pfd.c
@@ -127,9 +127,9 @@ static const struct clk_ops clk_pfd_ops = {
 struct clk_hw *imx_clk_hw_pfd(const char *name, const char *parent_name,
 			void __iomem *reg, u8 idx)
 {
+	struct clk_init_data init = {};
 	struct clk_pfd *pfd;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	pfd = kzalloc_obj(*pfd);
diff --git a/drivers/clk/imx/clk-pfdv2.c b/drivers/clk/imx/clk-pfdv2.c
index 0f92b92e2229fd..43ba79e47eac56 100644
--- a/drivers/clk/imx/clk-pfdv2.c
+++ b/drivers/clk/imx/clk-pfdv2.c
@@ -203,7 +203,7 @@ static const struct clk_ops clk_pfdv2_ops = {
 struct clk_hw *imx_clk_hw_pfdv2(enum imx_pfdv2_type type, const char *name,
 			     const char *parent_name, void __iomem *reg, u8 idx)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_pfdv2 *pfd;
 	struct clk_hw *hw;
 	int ret;
diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c
index b6f1cc9f570059..2f9e36f9e2409a 100644
--- a/drivers/clk/imx/clk-pll14xx.c
+++ b/drivers/clk/imx/clk-pll14xx.c
@@ -515,9 +515,9 @@ struct clk_hw *imx_dev_clk_hw_pll14xx(struct device *dev, const char *name,
 				const char *parent_name, void __iomem *base,
 				const struct imx_pll14xx_clk *pll_clk)
 {
+	struct clk_init_data init = {};
 	struct clk_pll14xx *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 	u32 val;
 
diff --git a/drivers/clk/imx/clk-pllv1.c b/drivers/clk/imx/clk-pllv1.c
index da5531e57da2a7..ee60c7660ef490 100644
--- a/drivers/clk/imx/clk-pllv1.c
+++ b/drivers/clk/imx/clk-pllv1.c
@@ -114,9 +114,9 @@ static const struct clk_ops clk_pllv1_ops = {
 struct clk_hw *imx_clk_hw_pllv1(enum imx_pllv1_type type, const char *name,
 		const char *parent, void __iomem *base)
 {
+	struct clk_init_data init = {};
 	struct clk_pllv1 *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	pll = kmalloc_obj(*pll);
diff --git a/drivers/clk/imx/clk-pllv2.c b/drivers/clk/imx/clk-pllv2.c
index 43396388030e30..aacc3d84e797c2 100644
--- a/drivers/clk/imx/clk-pllv2.c
+++ b/drivers/clk/imx/clk-pllv2.c
@@ -249,9 +249,9 @@ static const struct clk_ops clk_pllv2_ops = {
 struct clk_hw *imx_clk_hw_pllv2(const char *name, const char *parent,
 		void __iomem *base)
 {
+	struct clk_init_data init = {};
 	struct clk_pllv2 *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	pll = kzalloc_obj(*pll);
diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c
index a0ad6f4aae8fa5..714d9c723e167d 100644
--- a/drivers/clk/imx/clk-pllv3.c
+++ b/drivers/clk/imx/clk-pllv3.c
@@ -422,8 +422,8 @@ struct clk_hw *imx_clk_hw_pllv3(enum imx_pllv3_type type, const char *name,
 {
 	struct clk_pllv3 *pll;
 	const struct clk_ops *ops;
+	struct clk_init_data init = {};
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	pll = kzalloc_obj(*pll);
diff --git a/drivers/clk/imx/clk-pllv4.c b/drivers/clk/imx/clk-pllv4.c
index 8fa2f7111ce9b2..6dc9640d2f19b8 100644
--- a/drivers/clk/imx/clk-pllv4.c
+++ b/drivers/clk/imx/clk-pllv4.c
@@ -246,9 +246,9 @@ static const struct clk_ops clk_pllv4_ops = {
 struct clk_hw *imx_clk_hw_pllv4(enum imx_pllv4_type type, const char *name,
 		 const char *parent_name, void __iomem *base)
 {
+	struct clk_init_data init = {};
 	struct clk_pllv4 *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	pll = kzalloc_obj(*pll);
diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c
index 44220e9c5c2a23..0d034e206c6bc8 100644
--- a/drivers/clk/imx/clk-scu.c
+++ b/drivers/clk/imx/clk-scu.c
@@ -445,7 +445,7 @@ struct clk_hw *__imx_clk_scu(struct device *dev, const char *name,
 			     const char * const *parents, int num_parents,
 			     u32 rsrc_id, u8 clk_type)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_scu *clk;
 	struct clk_hw *hw;
 	int ret;
@@ -839,9 +839,9 @@ struct clk_hw *__imx_clk_gpr_scu(const char *name, const char * const *parent_na
 				 bool invert)
 {
 	struct imx_scu_clk_node *clk_node;
+	struct clk_init_data init = {};
 	struct clk_gpr_scu *clk;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	if (rsrc_id >= IMX_SC_R_LAST || gpr_id >= IMX_SC_C_LAST)
diff --git a/drivers/clk/imx/clk-sscg-pll.c b/drivers/clk/imx/clk-sscg-pll.c
index 7f104ecfa80fee..f6286ec11e0c07 100644
--- a/drivers/clk/imx/clk-sscg-pll.c
+++ b/drivers/clk/imx/clk-sscg-pll.c
@@ -504,8 +504,8 @@ struct clk_hw *imx_clk_hw_sscg_pll(const char *name,
 				void __iomem *base,
 				unsigned long flags)
 {
+	struct clk_init_data init = {};
 	struct clk_sscg_pll *pll;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int ret;
 
diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c
index 41e4c69131bd6e..ad223aa20e8db5 100644
--- a/drivers/clk/ingenic/cgu.c
+++ b/drivers/clk/ingenic/cgu.c
@@ -644,8 +644,8 @@ static const struct clk_ops ingenic_clk_ops = {
 static int ingenic_register_clock(struct ingenic_cgu *cgu, unsigned idx)
 {
 	const struct ingenic_cgu_clk_info *clk_info = &cgu->clock_info[idx];
-	struct clk_init_data clk_init;
 	struct ingenic_clk *ingenic_clk = NULL;
+	struct clk_init_data clk_init = {};
 	struct clk *clk, *parent;
 	const char *parent_names[4];
 	unsigned caps, i, num_possible;
diff --git a/drivers/clk/keystone/gate.c b/drivers/clk/keystone/gate.c
index 03e854f10fd98a..e83e3d4c7d67a1 100644
--- a/drivers/clk/keystone/gate.c
+++ b/drivers/clk/keystone/gate.c
@@ -164,7 +164,7 @@ static struct clk *clk_register_psc(struct device *dev,
 			struct clk_psc_data *psc_data,
 			spinlock_t *lock)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_psc *psc;
 	struct clk *clk;
 
diff --git a/drivers/clk/keystone/pll.c b/drivers/clk/keystone/pll.c
index ed197ee0cc88f8..020d5f4e9ed641 100644
--- a/drivers/clk/keystone/pll.c
+++ b/drivers/clk/keystone/pll.c
@@ -122,7 +122,7 @@ static struct clk *clk_register_pll(struct device *dev,
 			const char *parent_name,
 			struct clk_pll_data *pll_data)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_pll *pll;
 	struct clk *clk;
 
diff --git a/drivers/clk/keystone/syscon-clk.c b/drivers/clk/keystone/syscon-clk.c
index ecf180a7949c93..338174be41215f 100644
--- a/drivers/clk/keystone/syscon-clk.c
+++ b/drivers/clk/keystone/syscon-clk.c
@@ -67,7 +67,7 @@ static struct clk_hw
 			     const struct ti_syscon_gate_clk_data *data)
 {
 	struct ti_syscon_gate_clk_priv *priv;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	char *name = NULL;
 	int ret;
 
diff --git a/drivers/clk/kunit_clk_assigned_rates.h b/drivers/clk/kunit_clk_assigned_rates.h
index d7ae5ec2d25bed..c5d9f004ee8150 100644
--- a/drivers/clk/kunit_clk_assigned_rates.h
+++ b/drivers/clk/kunit_clk_assigned_rates.h
@@ -9,10 +9,10 @@
 #define ASSIGNED_RATES_1_RATE		9700000
 
 #define ASSIGNED_SSCS_0_MODFREQ		10000
-#define ASSIGNED_SSCS_0_SPREAD		30000
+#define ASSIGNED_SSCS_0_SPREAD		300
 #define ASSIGNED_SSCS_0_METHOD		CLK_SSC_CENTER_SPREAD
 #define ASSIGNED_SSCS_1_MODFREQ		20000
-#define ASSIGNED_SSCS_1_SPREAD		40000
+#define ASSIGNED_SSCS_1_SPREAD		400
 #define ASSIGNED_SSCS_1_METHOD		CLK_SSC_UP_SPREAD
 
 #endif
diff --git a/drivers/clk/kunit_clk_parse_clkspec.dtso b/drivers/clk/kunit_clk_parse_clkspec.dtso
new file mode 100644
index 00000000000000..a4115216d2aa2e
--- /dev/null
+++ b/drivers/clk/kunit_clk_parse_clkspec.dtso
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-2.0
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	kunit_clock_provider1: kunit-clock-provider1 {
+		compatible = "test,clock-provider1";
+		#clock-cells = <1>;
+	};
+
+	kunit_clock_provider2: kunit-clock-provider2 {
+		compatible = "test,clock-provider2";
+		#clock-cells = <1>;
+	};
+
+	kunit-clock-consumer {
+		compatible = "test,clock-consumer";
+		clocks = <&kunit_clock_provider1 0>, <&kunit_clock_provider2 0>;
+		clock-names = "first_clock", "second_clock";
+	};
+
+	kunit_clock_nexus: kunit-clock-nexus {
+		clocks = <&kunit_clock_provider2 0>;
+		clock-map = <&kunit_clock_provider2 0>;
+		#clock-cells = <0>;
+
+		kunit-clock-nexus-child {
+			clocks = <&kunit_clock_nexus>;
+		};
+	};
+};
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
index b059d44577f1e3..ba1ca9c90dbf1e 100644
--- a/drivers/clk/mediatek/Kconfig
+++ b/drivers/clk/mediatek/Kconfig
@@ -541,6 +541,7 @@ config COMMON_CLK_MT8173_MFGTOP
 	tristate "Clock and power driver for MediaTek MT8173 mfgtop"
 	depends on COMMON_CLK_MT8173
 	depends on PM
+	depends on OF
 	default COMMON_CLK_MT8173
 	select PM_GENERIC_DOMAINS
 	select PM_GENERIC_DOMAINS_OF
diff --git a/drivers/clk/mediatek/clk-cpumux.c b/drivers/clk/mediatek/clk-cpumux.c
index 412558b9512772..9799bc156dccd6 100644
--- a/drivers/clk/mediatek/clk-cpumux.c
+++ b/drivers/clk/mediatek/clk-cpumux.c
@@ -62,9 +62,9 @@ static struct clk_hw *
 mtk_clk_register_cpumux(struct device *dev, const struct mtk_composite *mux,
 			struct regmap *regmap)
 {
+	struct clk_init_data init = {};
 	struct mtk_clk_cpumux *cpumux;
 	int ret;
-	struct clk_init_data init;
 
 	cpumux = kzalloc_obj(*cpumux);
 	if (!cpumux)
diff --git a/drivers/clk/mediatek/reset.h b/drivers/clk/mediatek/reset.h
index 562ffd290a224a..b09cdcab0c4ba2 100644
--- a/drivers/clk/mediatek/reset.h
+++ b/drivers/clk/mediatek/reset.h
@@ -60,11 +60,11 @@ struct mtk_clk_rst_data {
 };
 
 /**
- * mtk_register_reset_controller - Register mediatek clock reset controller with device
- * @np: Pointer to device.
+ * mtk_register_reset_controller_with_dev - Register a MediaTek reset controller
+ * @dev: Device used to manage the reset controller resources.
  * @desc: Constant pointer to description of clock reset.
  *
- * Return: 0 on success and errorno otherwise.
+ * Return: 0 on success, negative errno otherwise.
  */
 int mtk_register_reset_controller_with_dev(struct device *dev,
 					   const struct mtk_clk_rst_desc *desc);
diff --git a/drivers/clk/mmp/Kconfig b/drivers/clk/mmp/Kconfig
index b0d2fea3cda5de..35a069034b2828 100644
--- a/drivers/clk/mmp/Kconfig
+++ b/drivers/clk/mmp/Kconfig
@@ -1,10 +1,15 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
+config COMMON_CLK_MMP
+	bool
+	default ARCH_MMP
+
 config COMMON_CLK_PXA1908
 	bool "Clock driver for Marvell PXA1908"
 	depends on ARCH_MMP || COMPILE_TEST
 	depends on OF
 	default y if ARCH_MMP && ARM64
+	select COMMON_CLK_MMP
 	select AUXILIARY_BUS
 	help
 	  This driver supports the Marvell PXA1908 SoC clocks.
diff --git a/drivers/clk/mmp/clk-apbc.c b/drivers/clk/mmp/clk-apbc.c
index 84532ce8fd0943..338ccd8fe99ca6 100644
--- a/drivers/clk/mmp/clk-apbc.c
+++ b/drivers/clk/mmp/clk-apbc.c
@@ -120,9 +120,9 @@ struct clk *mmp_clk_register_apbc(const char *name, const char *parent_name,
 		void __iomem *base, unsigned int delay,
 		unsigned int apbc_flags, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clk_apbc *apbc;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	apbc = kzalloc_obj(*apbc);
 	if (!apbc)
diff --git a/drivers/clk/mmp/clk-apmu.c b/drivers/clk/mmp/clk-apmu.c
index 30bbac1deca142..887854d8f5b1d6 100644
--- a/drivers/clk/mmp/clk-apmu.c
+++ b/drivers/clk/mmp/clk-apmu.c
@@ -65,9 +65,9 @@ static const struct clk_ops clk_apmu_ops = {
 struct clk *mmp_clk_register_apmu(const char *name, const char *parent_name,
 		void __iomem *base, u32 enable_mask, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clk_apmu *apmu;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	apmu = kzalloc_obj(*apmu);
 	if (!apmu)
diff --git a/drivers/clk/mmp/clk-frac.c b/drivers/clk/mmp/clk-frac.c
index 13bfc4c99f3ae5..5774354563aa67 100644
--- a/drivers/clk/mmp/clk-frac.c
+++ b/drivers/clk/mmp/clk-frac.c
@@ -171,8 +171,8 @@ struct clk *mmp_clk_register_factor(const char *name, const char *parent_name,
 		struct mmp_clk_factor_masks *masks,
 		struct u32_fract *ftbl, unsigned int ftbl_cnt, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct mmp_clk_factor *factor;
-	struct clk_init_data init;
 	struct clk *clk;
 
 	if (!masks) {
diff --git a/drivers/clk/mmp/clk-gate.c b/drivers/clk/mmp/clk-gate.c
index cc2f841ae9d963..e8dad4ed0125ef 100644
--- a/drivers/clk/mmp/clk-gate.c
+++ b/drivers/clk/mmp/clk-gate.c
@@ -94,9 +94,9 @@ struct clk *mmp_clk_register_gate(struct device *dev, const char *name,
 		void __iomem *reg, u32 mask, u32 val_enable, u32 val_disable,
 		unsigned int gate_flags, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct mmp_clk_gate *gate;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	/* allocate the gate */
 	gate = kzalloc_obj(*gate);
diff --git a/drivers/clk/mmp/clk-mix.c b/drivers/clk/mmp/clk-mix.c
index b1c9899c71afba..2f3d1cb6f836c6 100644
--- a/drivers/clk/mmp/clk-mix.c
+++ b/drivers/clk/mmp/clk-mix.c
@@ -444,9 +444,9 @@ struct clk *mmp_clk_register_mix(struct device *dev,
 					struct mmp_clk_mix_config *config,
 					spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct mmp_clk_mix *mix;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	mix = kzalloc_obj(*mix);
 	if (!mix)
diff --git a/drivers/clk/mmp/clk-pll.c b/drivers/clk/mmp/clk-pll.c
index 430f5e3c8540e8..e0ea29b5bb66cf 100644
--- a/drivers/clk/mmp/clk-pll.c
+++ b/drivers/clk/mmp/clk-pll.c
@@ -104,9 +104,9 @@ static struct clk *mmp_clk_register_pll(char *name,
 			unsigned long input_rate,
 			void __iomem *postdiv_reg, u8 postdiv_shift)
 {
+	struct clk_init_data init = {};
 	struct mmp_clk_pll *pll;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	pll = kzalloc_obj(*pll);
 	if (!pll)
diff --git a/drivers/clk/mstar/clk-msc313-mpll.c b/drivers/clk/mstar/clk-msc313-mpll.c
index 61beb4e8752509..bd45aa0aec0116 100644
--- a/drivers/clk/mstar/clk-msc313-mpll.c
+++ b/drivers/clk/mstar/clk-msc313-mpll.c
@@ -105,7 +105,7 @@ static int msc313_mpll_probe(struct platform_device *pdev)
 		return PTR_ERR(mpll->loop_div_second);
 
 	mpll->clk_data = devm_kzalloc(dev, struct_size(mpll->clk_data, hws,
-			ARRAY_SIZE(output_dividers)), GFP_KERNEL);
+				NUMOUTPUTS), GFP_KERNEL);
 	if (!mpll->clk_data)
 		return -ENOMEM;
 
diff --git a/drivers/clk/mvebu/ap-cpu-clk.c b/drivers/clk/mvebu/ap-cpu-clk.c
index 1e44ace7d95197..b1c6fceb91b29d 100644
--- a/drivers/clk/mvebu/ap-cpu-clk.c
+++ b/drivers/clk/mvebu/ap-cpu-clk.c
@@ -284,9 +284,11 @@ static int ap_cpu_clock_probe(struct platform_device *pdev)
 	if (!ap_cpu_data)
 		return -ENOMEM;
 
+	ap_cpu_data->num = nclusters;
+
 	for_each_of_cpu_node(dn) {
 		char *clk_name = "cpu-cluster-0";
-		struct clk_init_data init;
+		struct clk_init_data init = {};
 		const char *parent_name;
 		struct clk *parent;
 		u64 cpu;
@@ -333,8 +335,6 @@ static int ap_cpu_clock_probe(struct platform_device *pdev)
 		ap_cpu_data->hws[cluster_index] = &ap_cpu_clk[cluster_index].hw;
 	}
 
-	ap_cpu_data->num = cluster_index + 1;
-
 	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, ap_cpu_data);
 	if (ret)
 		dev_err(dev, "failed to register OF clock provider\n");
diff --git a/drivers/clk/mvebu/clk-corediv.c b/drivers/clk/mvebu/clk-corediv.c
index ae48293b0c413f..e26ada3c29aeb0 100644
--- a/drivers/clk/mvebu/clk-corediv.c
+++ b/drivers/clk/mvebu/clk-corediv.c
@@ -253,7 +253,7 @@ static void __init
 mvebu_corediv_clk_init(struct device_node *node,
 		       const struct clk_corediv_soc_desc *soc_desc)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_corediv *corediv;
 	struct clk **clks;
 	void __iomem *base;
diff --git a/drivers/clk/mvebu/clk-cpu.c b/drivers/clk/mvebu/clk-cpu.c
index 24196e288fe29c..a7646a25726b64 100644
--- a/drivers/clk/mvebu/clk-cpu.c
+++ b/drivers/clk/mvebu/clk-cpu.c
@@ -192,7 +192,7 @@ static void __init of_cpu_clk_setup(struct device_node *node)
 		goto clks_out;
 
 	for_each_possible_cpu(cpu) {
-		struct clk_init_data init;
+		struct clk_init_data init = {};
 		struct clk *clk;
 		char *clk_name = kzalloc(5, GFP_KERNEL);
 
diff --git a/drivers/clk/mxs/clk-div.c b/drivers/clk/mxs/clk-div.c
index fe7224162b3cc0..d4ee785c651190 100644
--- a/drivers/clk/mxs/clk-div.c
+++ b/drivers/clk/mxs/clk-div.c
@@ -70,9 +70,9 @@ static const struct clk_ops clk_div_ops = {
 struct clk *mxs_clk_div(const char *name, const char *parent_name,
 			void __iomem *reg, u8 shift, u8 width, u8 busy)
 {
+	struct clk_init_data init = {};
 	struct clk_div *div;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	div = kzalloc_obj(*div);
 	if (!div)
diff --git a/drivers/clk/mxs/clk-frac.c b/drivers/clk/mxs/clk-frac.c
index 52eb82fde322dc..e317710ab503a8 100644
--- a/drivers/clk/mxs/clk-frac.c
+++ b/drivers/clk/mxs/clk-frac.c
@@ -112,9 +112,9 @@ static const struct clk_ops clk_frac_ops = {
 struct clk *mxs_clk_frac(const char *name, const char *parent_name,
 			 void __iomem *reg, u8 shift, u8 width, u8 busy)
 {
+	struct clk_init_data init = {};
 	struct clk_frac *frac;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	frac = kzalloc_obj(*frac);
 	if (!frac)
diff --git a/drivers/clk/mxs/clk-pll.c b/drivers/clk/mxs/clk-pll.c
index 581868e0693c6e..5acf48cddbb285 100644
--- a/drivers/clk/mxs/clk-pll.c
+++ b/drivers/clk/mxs/clk-pll.c
@@ -82,9 +82,9 @@ static const struct clk_ops clk_pll_ops = {
 struct clk *mxs_clk_pll(const char *name, const char *parent_name,
 			void __iomem *base, u8 power, unsigned long rate)
 {
+	struct clk_init_data init = {};
 	struct clk_pll *pll;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	pll = kzalloc_obj(*pll);
 	if (!pll)
diff --git a/drivers/clk/mxs/clk-ref.c b/drivers/clk/mxs/clk-ref.c
index 9667d4d3c2a885..66ea1c229014ed 100644
--- a/drivers/clk/mxs/clk-ref.c
+++ b/drivers/clk/mxs/clk-ref.c
@@ -113,9 +113,9 @@ static const struct clk_ops clk_ref_ops = {
 struct clk *mxs_clk_ref(const char *name, const char *parent_name,
 			void __iomem *reg, u8 idx)
 {
+	struct clk_init_data init = {};
 	struct clk_ref *ref;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	ref = kzalloc_obj(*ref);
 	if (!ref)
diff --git a/drivers/clk/nxp/clk-lpc18xx-creg.c b/drivers/clk/nxp/clk-lpc18xx-creg.c
index 3d3982e9c661a5..de3c214a80fcf9 100644
--- a/drivers/clk/nxp/clk-lpc18xx-creg.c
+++ b/drivers/clk/nxp/clk-lpc18xx-creg.c
@@ -138,7 +138,7 @@ static struct clk *clk_register_creg_clk(struct device *dev,
 					 const char **parent_name,
 					 struct regmap *syscon)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 
 	init.ops = creg_clk->ops;
 	init.name = creg_clk->name;
diff --git a/drivers/clk/pistachio/clk-pll.c b/drivers/clk/pistachio/clk-pll.c
index 5b268a985a9c7d..ad997a777632a8 100644
--- a/drivers/clk/pistachio/clk-pll.c
+++ b/drivers/clk/pistachio/clk-pll.c
@@ -453,8 +453,8 @@ static struct clk *pll_register(const char *name, const char *parent_name,
 				struct pistachio_pll_rate_table *rates,
 				unsigned int nr_rates)
 {
+	struct clk_init_data init = {};
 	struct pistachio_clk_pll *pll;
-	struct clk_init_data init;
 	struct clk *clk;
 
 	pll = kzalloc_obj(*pll);
diff --git a/drivers/clk/pistachio/clk.c b/drivers/clk/pistachio/clk.c
index 80c722c77a1bd7..03e28779011f1c 100644
--- a/drivers/clk/pistachio/clk.c
+++ b/drivers/clk/pistachio/clk.c
@@ -17,28 +17,21 @@ pistachio_clk_alloc_provider(struct device_node *node, unsigned int num_clks)
 {
 	struct pistachio_clk_provider *p;
 
-	p = kzalloc_obj(*p);
+	p = kzalloc_flex(*p, clks, num_clks);
 	if (!p)
 		return p;
 
-	p->clk_data.clks = kzalloc_objs(struct clk *, num_clks);
-	if (!p->clk_data.clks)
-		goto free_provider;
+	p->clk_data.clks = p->clks;
 	p->clk_data.clk_num = num_clks;
 	p->node = node;
 	p->base = of_iomap(node, 0);
 	if (!p->base) {
 		pr_err("Failed to map clock provider registers\n");
-		goto free_clks;
+		kfree(p);
+		return NULL;
 	}
 
 	return p;
-
-free_clks:
-	kfree(p->clk_data.clks);
-free_provider:
-	kfree(p);
-	return NULL;
 }
 
 void pistachio_clk_register_provider(struct pistachio_clk_provider *p)
diff --git a/drivers/clk/pistachio/clk.h b/drivers/clk/pistachio/clk.h
index 2f4ffbd9828210..1b9b311b9656ef 100644
--- a/drivers/clk/pistachio/clk.h
+++ b/drivers/clk/pistachio/clk.h
@@ -142,6 +142,7 @@ struct pistachio_clk_provider {
 	struct device_node *node;
 	void __iomem *base;
 	struct clk_onecell_data clk_data;
+	struct clk *clks[];
 };
 
 extern struct pistachio_clk_provider *
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index d1fc9bf87126b2..27d0ab24d50a4c 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -19,6 +19,17 @@ menuconfig COMMON_CLK_QCOM
 
 if COMMON_CLK_QCOM
 
+config CLK_ELIZA_CAMCC
+	tristate "Eliza Camera Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_ELIZA_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the camera clock controller on Qualcomm Technologies, Inc
+	  Eliza devices.
+	  Say Y if you want to support camera devices and functionality such as
+	  capturing pictures.
+
 config CLK_ELIZA_DISPCC
 	tristate "Eliza Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -39,6 +50,16 @@ config CLK_ELIZA_GCC
 	  Say Y if you want to use peripheral devices such as UART, SPI,
 	  I2C, USB, UFS, SDCC, etc.
 
+config CLK_ELIZA_GPUCC
+	tristate "Eliza Graphics Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_ELIZA_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the graphics clock controller on Eliza devices.
+	  Say Y if you want to support graphics controller devices and
+	  functionality such as 3D graphics.
+
 config CLK_ELIZA_TCSRCC
 	tristate "Eliza TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -59,6 +80,16 @@ config CLK_GLYMUR_CAMCC
 	  Say Y if you want to support camera devices and functionality such as
 	  capturing pictures.
 
+config CLK_ELIZA_VIDEOCC
+	tristate "Eliza Video Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_ELIZA_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the video clock controller on Eliza devices.
+	  Say Y if you want to support video devices and functionality such as
+	  video encode and decode.
+
 config CLK_GLYMUR_DISPCC
 	tristate "Glymur Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -73,7 +104,6 @@ config CLK_GLYMUR_DISPCC
 config CLK_GLYMUR_EVACC
 	tristate "Glymur EVA Clock Controller"
 	depends on ARM64 || COMPILE_TEST
-	default m if ARCH_QCOM
 	select CLK_GLYMUR_GCC
 	default m if ARCH_QCOM
 	help
@@ -96,6 +126,7 @@ config CLK_GLYMUR_GPUCC
 	tristate "Glymur Graphics Clock Controller"
 	depends on ARM64 || COMPILE_TEST
 	select CLK_GLYMUR_GCC
+	default m if ARCH_QCOM
 	help
 	  Support for the graphics clock controller on Glymur devices.
 	  Say Y if you want to support graphics controller devices and
@@ -181,6 +212,27 @@ config CLK_KAANAPALI_VIDEOCC
 	  Say Y if you want to support video devices and functionality such as
 	  video encode/decode.
 
+config CLK_KUNO_GCC
+	tristate "Kuno Global Clock Controller"
+	depends on ARM || COMPILE_TEST
+	select QCOM_GDSC
+	default ARCH_QCOM
+	help
+	  Support for the global clock controller (GCC) on Kuno devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  I2C, USB, SD, PCIe and Ethernet on the Kuno SoC. This clock
+	  controller supplies the clocks and resets to those peripherals.
+
+config CLK_MAILI_VIDEOCC
+	tristate "Maili Video Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_HAWI_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the video clock controller on Maili devices.
+	  Say Y if you want to support video devices and functionality such as
+	  video encode/decode.
+
 config CLK_NORD_DISPCC
 	tristate "Nord Display Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -192,15 +244,16 @@ config CLK_NORD_DISPCC
 	  Say Y if you want to support display devices and functionality such as
 	  splash screen.
 
-config CLK_MAILI_VIDEOCC
-	tristate "Maili Video Clock Controller"
+config CLK_NORD_CAMCC
+	tristate "Nord Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
-	select CLK_HAWI_GCC
+	select CLK_NORD_GCC
 	default m if ARCH_QCOM
 	help
-	  Support for the video clock controller on Maili devices.
-	  Say Y if you want to support video devices and functionality such as
-	  video encode/decode.
+	  Support for the camera clock controller on Qualcomm Technologies, Inc
+	  Nord devices.
+	  Say Y if you want to support camera devices and functionality such as
+	  capturing pictures.
 
 config CLK_NORD_GCC
 	tristate "Nord Global Clock Controller"
@@ -213,6 +266,25 @@ config CLK_NORD_GCC
 	  SPI, I2C, USB, SD/UFS, PCIe etc. The clock controller is a combination
 	  of GCC, SE_GCC, NE_GCC and NW_GCC.
 
+config CLK_NORD_GPUCC
+	tristate "Nord Graphics Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_NORD_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the graphics clock controllers on Nord devices. There are two
+	  graphics clock controllers on Nord SoC.
+	  Say Y if you want to support graphics controller devices and
+	  functionality such as 3D graphics.
+
+config CLK_NORD_TCSRCC
+	tristate "Nord TCSR Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	default m if ARCH_QCOM
+	help
+	  Support for the TCSR clock controller on Nord devices.
+	  Say Y if you want to use peripheral devices such as PCIe, USB, UFS etc.
+
 config CLK_SHIKRA_AUDIOCORECC
 	tristate "Shikra Audio Core Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -223,6 +295,17 @@ config CLK_SHIKRA_AUDIOCORECC
 	  Say Y if you want to use AudioCoreCC clocks required to support audio
 	  devices and it's functionality.
 
+config CLK_NORD_VIDEOCC
+	tristate "Nord VIDEO Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_NORD_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the video clock controller on Qualcomm Technologies, Inc.
+	  Nord devices.
+	  Say Y if you want to support video devices and functionality such as
+	  video encode/decode.
+
 config CLK_SHIKRA_GCC
 	tristate "Shikra Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -233,17 +316,6 @@ config CLK_SHIKRA_GCC
 	  Say Y if you want to use multimedia devices or peripheral
 	  devices such as Camera, Video, UART, SPI, I2C, USB, SD/eMMC etc.
 
-config CLK_NORD_GPUCC
-	tristate "Nord Graphics Clock Controller"
-	depends on ARM64 || COMPILE_TEST
-	select CLK_NORD_GCC
-	default m if ARCH_QCOM
-	help
-	  Support for the graphics clock controllers on Nord devices. There are two
-	  graphics clock controllers on Nord SoC.
-	  Say Y if you want to support graphics controller devices and
-	  functionality such as 3D graphics.
-
 config CLK_X1E80100_CAMCC
 	tristate "X1E80100 Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -423,6 +495,17 @@ config QCOM_CLK_RPMH
 	 Say Y if you want to support the clocks exposed by RPMh on
 	 platforms such as SDM845.
 
+config CLK_HAWI_CAMCC
+	tristate "Hawi Camera Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_HAWI_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the camera clock controller on Qualcomm Technologies, Inc
+	  Hawi devices.
+	  Say Y if you want to support camera devices and functionality such as
+	  capturing pictures.
+
 config CLK_HAWI_GCC
 	tristate "Hawi Global Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -433,6 +516,18 @@ config CLK_HAWI_GCC
 	  Say Y if you want to use peripheral devices such as UART, SPI,
 	  I2C, USB, UFS, SD/eMMC, PCIe, etc.
 
+config CLK_HAWI_GPUCC
+	tristate "Hawi Graphics Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select CLK_HAWI_GCC
+	default m if ARCH_QCOM
+	help
+	  Support for the graphics clock controller on Hawi devices.
+	  Say Y if you want to support graphics controller devices and
+	  functionality such as 3D graphics. It also provides GX clock
+	  domain control required for GPU power management on Hawi-based
+	  devices.
+
 config CLK_HAWI_TCSRCC
 	tristate "Hawi TCSR Clock Controller"
 	depends on ARM64 || COMPILE_TEST
@@ -694,6 +789,16 @@ config MSM_GCC_8939
 	  Say Y if you want to use devices such as UART, SPI i2c, USB,
 	  SD/eMMC, display, graphics, camera etc.
 
+config MSM_GCC_8952
+	tristate "MSM8952 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select QCOM_GDSC
+	default ARCH_QCOM
+	help
+	  Support for the global clock controller on MSM8952 devices.
+	  Say Y if you want to use devices such as UART, SPI i2c, USB,
+	  SD/eMMC, display, graphics, camera etc.
+
 config MSM_GCC_8960
 	tristate "APQ8064/MSM8960 Global Clock Controller"
 	depends on ARM || COMPILE_TEST
@@ -895,14 +1000,6 @@ config QCS_GCC_404
 	  Say Y if you want to use multimedia devices or peripheral
 	  devices such as UART, SPI, I2C, USB, SD/eMMC, PCIe etc.
 
-config CLK_NORD_TCSRCC
-	tristate "Nord TCSR Clock Controller"
-	depends on ARM64 || COMPILE_TEST
-	default m if ARCH_QCOM
-	help
-	  Support for the TCSR clock controller on Nord devices.
-	  Say Y if you want to use peripheral devices such as PCIe, USB, UFS etc.
-
 config SA_CAMCC_8775P
 	tristate "SA8775P Camera Clock Controller"
 	depends on ARM64 || COMPILE_TEST
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index daf25d583be28b..6affb5073f269d 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -21,9 +21,12 @@ clk-qcom-$(CONFIG_QCOM_GDSC) += gdsc.o
 # Keep alphabetically sorted by config
 obj-$(CONFIG_APQ_GCC_8084) += gcc-apq8084.o
 obj-$(CONFIG_APQ_MMCC_8084) += mmcc-apq8084.o
+obj-$(CONFIG_CLK_ELIZA_CAMCC) += cambistmclkcc-eliza.o camcc-eliza.o
 obj-$(CONFIG_CLK_ELIZA_DISPCC) += dispcc-eliza.o
 obj-$(CONFIG_CLK_ELIZA_GCC) += gcc-eliza.o
+obj-$(CONFIG_CLK_ELIZA_GPUCC) += gpucc-eliza.o
 obj-$(CONFIG_CLK_ELIZA_TCSRCC) += tcsrcc-eliza.o
+obj-$(CONFIG_CLK_ELIZA_VIDEOCC) += videocc-eliza.o
 obj-$(CONFIG_CLK_GFM_LPASS_SM8250) += lpass-gfm-sm8250.o
 obj-$(CONFIG_CLK_GLYMUR_CAMCC) += camcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_DISPCC) += dispcc-glymur.o
@@ -32,7 +35,9 @@ obj-$(CONFIG_CLK_GLYMUR_GCC) += gcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_GPUCC) += gpucc-glymur.o gxclkctl-kaanapali.o
 obj-$(CONFIG_CLK_GLYMUR_TCSRCC) += tcsrcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_VIDEOCC) += videocc-glymur.o
+obj-$(CONFIG_CLK_HAWI_CAMCC) += cambistmclkcc-kaanapali.o camcc-hawi.o
 obj-$(CONFIG_CLK_HAWI_GCC) += gcc-hawi.o
+obj-$(CONFIG_CLK_HAWI_GPUCC) += gpucc-hawi.o gxclkctl-kaanapali.o
 obj-$(CONFIG_CLK_HAWI_TCSRCC) += tcsrcc-hawi.o
 obj-$(CONFIG_CLK_HAWI_VIDEOCC) += videocc-hawi.o
 obj-$(CONFIG_CLK_KAANAPALI_CAMCC) += cambistmclkcc-kaanapali.o camcc-kaanapali.o
@@ -41,11 +46,14 @@ obj-$(CONFIG_CLK_KAANAPALI_GCC) += gcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_GPUCC) += gpucc-kaanapali.o gxclkctl-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_TCSRCC) += tcsrcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_VIDEOCC) += videocc-kaanapali.o
+obj-$(CONFIG_CLK_KUNO_GCC) += gcc-kuno.o
 obj-$(CONFIG_CLK_MAILI_VIDEOCC) += videocc-maili.o
+obj-$(CONFIG_CLK_NORD_CAMCC) += camcc-nord.o
 obj-$(CONFIG_CLK_NORD_DISPCC) += dispcc0-nord.o dispcc1-nord.o
 obj-$(CONFIG_CLK_NORD_GCC) += gcc-nord.o negcc-nord.o nwgcc-nord.o segcc-nord.o
 obj-$(CONFIG_CLK_NORD_GPUCC) += gpucc-nord.o gpu2cc-nord.o
 obj-$(CONFIG_CLK_NORD_TCSRCC) += tcsrcc-nord.o
+obj-$(CONFIG_CLK_NORD_VIDEOCC) += videocc-glymur.o
 obj-$(CONFIG_CLK_SHIKRA_AUDIOCORECC) += audiocorecc-shikra.o
 obj-$(CONFIG_CLK_SHIKRA_GCC) += gcc-shikra.o
 obj-$(CONFIG_CLK_X1E80100_CAMCC) += camcc-x1e80100.o
@@ -82,6 +90,7 @@ obj-$(CONFIG_MSM_GCC_8909) += gcc-msm8909.o
 obj-$(CONFIG_MSM_GCC_8916) += gcc-msm8916.o
 obj-$(CONFIG_MSM_GCC_8917) += gcc-msm8917.o
 obj-$(CONFIG_MSM_GCC_8939) += gcc-msm8939.o
+obj-$(CONFIG_MSM_GCC_8952) += gcc-msm8952.o
 obj-$(CONFIG_MSM_GCC_8953) += gcc-msm8953.o
 obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o
 obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o
diff --git a/drivers/clk/qcom/cambistmclkcc-eliza.c b/drivers/clk/qcom/cambistmclkcc-eliza.c
new file mode 100644
index 00000000000000..5cb68b81095ebf
--- /dev/null
+++ b/drivers/clk/qcom/cambistmclkcc-eliza.c
@@ -0,0 +1,464 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,eliza-cambistmclkcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_SLEEP_CLK,
+	DT_AHB_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_CAM_BIST_MCLK_CC_PLL0_OUT_EVEN,
+	P_CAM_BIST_MCLK_CC_PLL0_OUT_MAIN,
+	P_SLEEP_CLK,
+};
+
+static const struct pll_vco rivian_ole_vco[] = {
+	{ 777000000, 1285000000, 0 },
+};
+
+/* 960.0 MHz Configuration */
+static const struct alpha_pll_config cam_bist_mclk_cc_pll0_config = {
+	.l = 0x32,
+	.cal_l = 0x32,
+	.alpha = 0x0,
+	.config_ctl_val = 0x10000030,
+	.config_ctl_hi_val = 0x80890263,
+	.config_ctl_hi1_val = 0x00000217,
+	.user_ctl_val = 0x00000001,
+	.user_ctl_hi_val = 0x00000000,
+};
+
+static struct clk_alpha_pll cam_bist_mclk_cc_pll0 = {
+	.offset = 0x0,
+	.config = &cam_bist_mclk_cc_pll0_config,
+	.vco_table = rivian_ole_vco,
+	.num_vco = ARRAY_SIZE(rivian_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_RIVIAN_EVO],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_rivian_evo_ops,
+		},
+	},
+};
+
+static const struct parent_map cam_bist_mclk_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_BIST_MCLK_CC_PLL0_OUT_EVEN, 3 },
+	{ P_CAM_BIST_MCLK_CC_PLL0_OUT_MAIN, 5 },
+};
+
+static const struct clk_parent_data cam_bist_mclk_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_bist_mclk_cc_pll0.clkr.hw },
+	{ .hw = &cam_bist_mclk_cc_pll0.clkr.hw },
+};
+
+static const struct parent_map cam_bist_mclk_cc_parent_map_1[] = {
+	{ P_SLEEP_CLK, 0 },
+};
+
+static const struct clk_parent_data cam_bist_mclk_cc_parent_data_1[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct freq_tbl ftbl_cam_bist_mclk_cc_mclk0_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(24000000, P_CAM_BIST_MCLK_CC_PLL0_OUT_EVEN, 10, 1, 4),
+	F(68571429, P_CAM_BIST_MCLK_CC_PLL0_OUT_MAIN, 14, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk0_clk_src = {
+	.cmd_rcgr = 0x4000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk0_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk1_clk_src = {
+	.cmd_rcgr = 0x401c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk1_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk2_clk_src = {
+	.cmd_rcgr = 0x4038,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk2_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk3_clk_src = {
+	.cmd_rcgr = 0x4054,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk3_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk4_clk_src = {
+	.cmd_rcgr = 0x4070,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk4_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk5_clk_src = {
+	.cmd_rcgr = 0x408c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk5_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk6_clk_src = {
+	.cmd_rcgr = 0x40a8,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk6_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_mclk7_clk_src = {
+	.cmd_rcgr = 0x40c4,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_mclk0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_mclk7_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_bist_mclk_cc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_bist_mclk_cc_sleep_clk_src = {
+	.cmd_rcgr = 0x40e0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_bist_mclk_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_bist_mclk_cc_sleep_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_bist_mclk_cc_sleep_clk_src",
+		.parent_data = cam_bist_mclk_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_bist_mclk_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk0_clk = {
+	.halt_reg = 0x4018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk1_clk = {
+	.halt_reg = 0x4034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk2_clk = {
+	.halt_reg = 0x4050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk3_clk = {
+	.halt_reg = 0x406c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x406c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk4_clk = {
+	.halt_reg = 0x4088,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4088,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk5_clk = {
+	.halt_reg = 0x40a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x40a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk6_clk = {
+	.halt_reg = 0x40c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x40c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_bist_mclk_cc_mclk7_clk = {
+	.halt_reg = 0x40dc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x40dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_bist_mclk_cc_mclk7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_bist_mclk_cc_mclk7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap *cam_bist_mclk_cc_eliza_clocks[] = {
+	[CAM_BIST_MCLK_CC_MCLK0_CLK] = &cam_bist_mclk_cc_mclk0_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK0_CLK_SRC] = &cam_bist_mclk_cc_mclk0_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_MCLK1_CLK] = &cam_bist_mclk_cc_mclk1_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK1_CLK_SRC] = &cam_bist_mclk_cc_mclk1_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_MCLK2_CLK] = &cam_bist_mclk_cc_mclk2_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK2_CLK_SRC] = &cam_bist_mclk_cc_mclk2_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_MCLK3_CLK] = &cam_bist_mclk_cc_mclk3_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK3_CLK_SRC] = &cam_bist_mclk_cc_mclk3_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_MCLK4_CLK] = &cam_bist_mclk_cc_mclk4_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK4_CLK_SRC] = &cam_bist_mclk_cc_mclk4_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_MCLK5_CLK] = &cam_bist_mclk_cc_mclk5_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK5_CLK_SRC] = &cam_bist_mclk_cc_mclk5_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_MCLK6_CLK] = &cam_bist_mclk_cc_mclk6_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK6_CLK_SRC] = &cam_bist_mclk_cc_mclk6_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_MCLK7_CLK] = &cam_bist_mclk_cc_mclk7_clk.clkr,
+	[CAM_BIST_MCLK_CC_MCLK7_CLK_SRC] = &cam_bist_mclk_cc_mclk7_clk_src.clkr,
+	[CAM_BIST_MCLK_CC_PLL0] = &cam_bist_mclk_cc_pll0.clkr,
+	[CAM_BIST_MCLK_CC_SLEEP_CLK_SRC] = &cam_bist_mclk_cc_sleep_clk_src.clkr,
+};
+
+static struct clk_alpha_pll *cam_bist_mclk_cc_eliza_plls[] = {
+	&cam_bist_mclk_cc_pll0,
+};
+
+static const u32 cam_bist_mclk_cc_eliza_critical_cbcrs[] = {
+	0x40f8, /* CAM_BIST_MCLK_CC_SLEEP_CLK */
+};
+
+static const struct regmap_config cam_bist_mclk_cc_eliza_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5010,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_driver_data cam_bist_mclk_cc_eliza_driver_data = {
+	.alpha_plls = cam_bist_mclk_cc_eliza_plls,
+	.num_alpha_plls = ARRAY_SIZE(cam_bist_mclk_cc_eliza_plls),
+	.clk_cbcrs = cam_bist_mclk_cc_eliza_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_bist_mclk_cc_eliza_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc cam_bist_mclk_cc_eliza_desc = {
+	.config = &cam_bist_mclk_cc_eliza_regmap_config,
+	.clks = cam_bist_mclk_cc_eliza_clocks,
+	.num_clks = ARRAY_SIZE(cam_bist_mclk_cc_eliza_clocks),
+	.use_rpm = true,
+	.driver_data = &cam_bist_mclk_cc_eliza_driver_data,
+};
+
+static const struct of_device_id cam_bist_mclk_cc_eliza_match_table[] = {
+	{ .compatible = "qcom,eliza-cambistmclkcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, cam_bist_mclk_cc_eliza_match_table);
+
+static int cam_bist_mclk_cc_eliza_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &cam_bist_mclk_cc_eliza_desc);
+}
+
+static struct platform_driver cam_bist_mclk_cc_eliza_driver = {
+	.probe = cam_bist_mclk_cc_eliza_probe,
+	.driver = {
+		.name = "cambistmclkcc-eliza",
+		.of_match_table = cam_bist_mclk_cc_eliza_match_table,
+	},
+};
+
+module_platform_driver(cam_bist_mclk_cc_eliza_driver);
+
+MODULE_DESCRIPTION("QTI CAMBISTMCLKCC Eliza Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/cambistmclkcc-kaanapali.c b/drivers/clk/qcom/cambistmclkcc-kaanapali.c
index c96e9196d908e5..b5bafc1e7d4285 100644
--- a/drivers/clk/qcom/cambistmclkcc-kaanapali.c
+++ b/drivers/clk/qcom/cambistmclkcc-kaanapali.c
@@ -394,11 +394,18 @@ static const struct regmap_config cam_bist_mclk_cc_kaanapali_regmap_config = {
 	.fast_io = true,
 };
 
+static void cam_bist_mclk_cc_kaanapali_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	regmap_write(regmap, 0x18, 0); /* CAM_BIST_MCLK_CC_PLL0_USER_CTL_U */
+	regmap_write(regmap, 0x28, 0); /* CAM_BIST_MCLK_CC_PLL0_CONFIG_CTL_U2 */
+}
+
 static const struct qcom_cc_driver_data cam_bist_mclk_cc_kaanapali_driver_data = {
 	.alpha_plls = cam_bist_mclk_cc_kaanapali_plls,
 	.num_alpha_plls = ARRAY_SIZE(cam_bist_mclk_cc_kaanapali_plls),
 	.clk_cbcrs = cam_bist_mclk_cc_kaanapali_critical_cbcrs,
 	.num_clk_cbcrs = ARRAY_SIZE(cam_bist_mclk_cc_kaanapali_critical_cbcrs),
+	.clk_regs_configure = cam_bist_mclk_cc_kaanapali_regs_configure,
 };
 
 static const struct qcom_cc_desc cam_bist_mclk_cc_kaanapali_desc = {
diff --git a/drivers/clk/qcom/cambistmclkcc-sm8750.c b/drivers/clk/qcom/cambistmclkcc-sm8750.c
index 69abb756c04f2a..45788ddb9dcc62 100644
--- a/drivers/clk/qcom/cambistmclkcc-sm8750.c
+++ b/drivers/clk/qcom/cambistmclkcc-sm8750.c
@@ -413,11 +413,18 @@ static const struct regmap_config cam_bist_mclk_cc_sm8750_regmap_config = {
 	.fast_io = true,
 };
 
+static void cam_bist_mclk_cc_sm8750_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	regmap_write(regmap, 0x18, 0); /* CAM_BIST_MCLK_CC_PLL0_USER_CTL_U */
+	regmap_write(regmap, 0x28, 0); /* CAM_BIST_MCLK_CC_PLL0_CONFIG_CTL_U2 */
+}
+
 static const struct qcom_cc_driver_data cam_bist_mclk_cc_sm8750_driver_data = {
 	.alpha_plls = cam_bist_mclk_cc_sm8750_plls,
 	.num_alpha_plls = ARRAY_SIZE(cam_bist_mclk_cc_sm8750_plls),
 	.clk_cbcrs = cam_bist_mclk_cc_sm8750_critical_cbcrs,
 	.num_clk_cbcrs = ARRAY_SIZE(cam_bist_mclk_cc_sm8750_critical_cbcrs),
+	.clk_regs_configure = cam_bist_mclk_cc_sm8750_regs_configure,
 };
 
 static const struct qcom_cc_desc cam_bist_mclk_cc_sm8750_desc = {
diff --git a/drivers/clk/qcom/camcc-eliza.c b/drivers/clk/qcom/camcc-eliza.c
new file mode 100644
index 00000000000000..7348f4dde65aea
--- /dev/null
+++ b/drivers/clk/qcom/camcc-eliza.c
@@ -0,0 +1,2803 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,eliza-camcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_SLEEP_CLK,
+	DT_AHB_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_CAM_CC_PLL0_OUT_EVEN,
+	P_CAM_CC_PLL0_OUT_MAIN,
+	P_CAM_CC_PLL0_OUT_ODD,
+	P_CAM_CC_PLL1_OUT_EVEN,
+	P_CAM_CC_PLL2_OUT_EVEN,
+	P_CAM_CC_PLL3_OUT_EVEN,
+	P_CAM_CC_PLL4_OUT_EVEN,
+	P_CAM_CC_PLL5_OUT_EVEN,
+	P_CAM_CC_PLL6_OUT_EVEN,
+	P_CAM_CC_PLL6_OUT_ODD,
+	P_SLEEP_CLK,
+};
+
+static const struct pll_vco lucid_ole_vco[] = {
+	{ 249600000, 2300000000, 0 },
+};
+
+/* 1200.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll0_config = {
+	.l = 0x3e,
+	.cal_l = 0x44,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00008400,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll cam_cc_pll0 = {
+	.offset = 0x0,
+	.config = &cam_cc_pll0_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_odd[] = {
+	{ 0x2, 3 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_odd = {
+	.offset = 0x0,
+	.post_div_shift = 14,
+	.post_div_table = post_div_table_cam_cc_pll0_out_odd,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_odd),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_odd",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+/* 900.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll1_config = {
+	.l = 0x2e,
+	.cal_l = 0x44,
+	.alpha = 0xe000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll cam_cc_pll1 = {
+	.offset = 0x1000,
+	.config = &cam_cc_pll1_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll1",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll1_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll1_out_even = {
+	.offset = 0x1000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll1_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll1_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll1_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll1.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+/* 872.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll2_config = {
+	.l = 0x2d,
+	.cal_l = 0x44,
+	.alpha = 0x6aaa,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll cam_cc_pll2 = {
+	.offset = 0x2000,
+	.config = &cam_cc_pll2_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll2",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll2_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll2_out_even = {
+	.offset = 0x2000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll2_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll2_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll2_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll2.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+/* 890.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll3_config = {
+	.l = 0x2e,
+	.cal_l = 0x44,
+	.alpha = 0x5aaa,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll cam_cc_pll3 = {
+	.offset = 0x3000,
+	.config = &cam_cc_pll3_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll3",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll3_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll3_out_even = {
+	.offset = 0x3000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll3_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll3_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll3_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll3.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+/* 890.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll4_config = {
+	.l = 0x2e,
+	.cal_l = 0x44,
+	.alpha = 0x5aaa,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll cam_cc_pll4 = {
+	.offset = 0x4000,
+	.config = &cam_cc_pll4_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll4_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll4_out_even = {
+	.offset = 0x4000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll4_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll4_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll4_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll4.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+/* 890.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll5_config = {
+	.l = 0x2e,
+	.cal_l = 0x44,
+	.alpha = 0x5aaa,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll cam_cc_pll5 = {
+	.offset = 0x5000,
+	.config = &cam_cc_pll5_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll5",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll5_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll5_out_even = {
+	.offset = 0x5000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll5_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll5_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll5_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll5.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+/* 960.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll6_config = {
+	.l = 0x32,
+	.cal_l = 0x44,
+	.alpha = 0x0,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00008400,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll cam_cc_pll6 = {
+	.offset = 0x6000,
+	.config = &cam_cc_pll6_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll6",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll6_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll6_out_even = {
+	.offset = 0x6000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll6_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll6_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll6_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll6_out_odd[] = {
+	{ 0x2, 3 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll6_out_odd = {
+	.offset = 0x6000,
+	.post_div_shift = 14,
+	.post_div_table = post_div_table_cam_cc_pll6_out_odd,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll6_out_odd),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll6_out_odd",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_ole_ops,
+	},
+};
+
+static const struct parent_map cam_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+	{ P_CAM_CC_PLL6_OUT_ODD, 4 },
+	{ P_CAM_CC_PLL6_OUT_EVEN, 5 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+	{ P_CAM_CC_PLL6_OUT_ODD, 4 },
+	{ P_CAM_CC_PLL6_OUT_EVEN, 5 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL1_OUT_EVEN, 4 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll1_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL2_OUT_EVEN, 5 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll2_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_4[] = {
+	{ P_SLEEP_CLK, 0 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_4[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map cam_cc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL3_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll3_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_6[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL4_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_6[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll4_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_7[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL5_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_7[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll5_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_8[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_8[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct freq_tbl ftbl_cam_cc_camnoc_rt_axi_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_camnoc_rt_axi_clk_src = {
+	.cmd_rcgr = 0x112e8,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_camnoc_rt_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_camnoc_rt_axi_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cci_0_clk_src[] = {
+	F(37500000, P_CAM_CC_PLL0_OUT_EVEN, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cci_0_clk_src = {
+	.cmd_rcgr = 0x1126c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_0_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_1_clk_src = {
+	.cmd_rcgr = 0x11288,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_1_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_2_clk_src = {
+	.cmd_rcgr = 0x112a4,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_2_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cphy_rx_clk_src[] = {
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cphy_rx_clk_src = {
+	.cmd_rcgr = 0x11068,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cphy_rx_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cre_clk_src[] = {
+	F(200000000, P_CAM_CC_PLL0_OUT_ODD, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cre_clk_src = {
+	.cmd_rcgr = 0x111ac,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cre_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cre_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_csi0phytimer_clk_src[] = {
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x10000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi0phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x10024,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi1phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi2phytimer_clk_src = {
+	.cmd_rcgr = 0x10044,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi2phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi3phytimer_clk_src = {
+	.cmd_rcgr = 0x10064,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi3phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi4phytimer_clk_src = {
+	.cmd_rcgr = 0x10084,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi4phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi5phytimer_clk_src = {
+	.cmd_rcgr = 0x100a4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi5phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csid_clk_src = {
+	.cmd_rcgr = 0x112c0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csid_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_fast_ahb_clk_src[] = {
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_fast_ahb_clk_src = {
+	.cmd_rcgr = 0x100dc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_fast_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_fast_ahb_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_icp_0_clk_src[] = {
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL0_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_icp_0_clk_src = {
+	.cmd_rcgr = 0x11214,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_icp_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_icp_0_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_icp_1_clk_src = {
+	.cmd_rcgr = 0x1123c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_icp_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_icp_1_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_clk_src = {
+	.cmd_rcgr = 0x11150,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_lite_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_csid_clk_src = {
+	.cmd_rcgr = 0x1117c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_lite_csid_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ipe_nps_clk_src[] = {
+	F(450000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(575000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(675000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(825000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ipe_nps_clk_src = {
+	.cmd_rcgr = 0x10190,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_2,
+	.freq_tbl = ftbl_cam_cc_ipe_nps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ipe_nps_clk_src",
+		.parent_data = cam_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_jpeg_clk_src = {
+	.cmd_rcgr = 0x111d0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cre_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_jpeg_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ofe_clk_src[] = {
+	F(436000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(570000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(675000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(757000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ofe_clk_src = {
+	.cmd_rcgr = 0x1011c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_3,
+	.freq_tbl = ftbl_cam_cc_ofe_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ofe_clk_src",
+		.parent_data = cam_cc_parent_data_3,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_qdss_debug_clk_src[] = {
+	F(60000000, P_CAM_CC_PLL6_OUT_EVEN, 8, 0, 0),
+	F(120000000, P_CAM_CC_PLL0_OUT_EVEN, 5, 0, 0),
+	F(240000000, P_CAM_CC_PLL0_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_qdss_debug_clk_src = {
+	.cmd_rcgr = 0x1132c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_qdss_debug_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_qdss_debug_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_sleep_clk_src = {
+	.cmd_rcgr = 0x11380,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_4,
+	.freq_tbl = ftbl_cam_cc_sleep_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_sleep_clk_src",
+		.parent_data = cam_cc_parent_data_4,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_slow_ahb_clk_src[] = {
+	F(80000000, P_CAM_CC_PLL0_OUT_EVEN, 7.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_slow_ahb_clk_src = {
+	.cmd_rcgr = 0x10100,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_slow_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_slow_ahb_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_tfe_0_clk_src[] = {
+	F(445000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(567000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(644000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(785000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_tfe_0_clk_src = {
+	.cmd_rcgr = 0x11018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_5,
+	.freq_tbl = ftbl_cam_cc_tfe_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_tfe_0_clk_src",
+		.parent_data = cam_cc_parent_data_5,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_tfe_1_clk_src[] = {
+	F(445000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(567000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(644000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(785000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_tfe_1_clk_src = {
+	.cmd_rcgr = 0x11098,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_6,
+	.freq_tbl = ftbl_cam_cc_tfe_1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_tfe_1_clk_src",
+		.parent_data = cam_cc_parent_data_6,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_6),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_tfe_2_clk_src[] = {
+	F(445000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(567000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(644000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(785000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_tfe_2_clk_src = {
+	.cmd_rcgr = 0x11100,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_7,
+	.freq_tbl = ftbl_cam_cc_tfe_2_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_tfe_2_clk_src",
+		.parent_data = cam_cc_parent_data_7,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_7),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_xo_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_xo_clk_src = {
+	.cmd_rcgr = 0x11364,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_8,
+	.freq_tbl = ftbl_cam_cc_xo_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_xo_clk_src",
+		.parent_data = cam_cc_parent_data_8,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_8),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_branch cam_cc_cam_top_ahb_clk = {
+	.halt_reg = 0x113ac,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x113ac,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cam_top_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cam_top_fast_ahb_clk = {
+	.halt_reg = 0x1139c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1139c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cam_top_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_dcd_xo_clk = {
+	.halt_reg = 0x11320,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11320,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_dcd_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_axi_clk = {
+	.halt_reg = 0x11310,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x11310,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x11310,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_rt_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_cre_clk = {
+	.halt_reg = 0x111c8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x111c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_cre_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cre_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_ipe_nps_clk = {
+	.halt_reg = 0x101b8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x101b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_ipe_nps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_ofe_anchor_clk = {
+	.halt_reg = 0x10158,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10158,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_ofe_anchor_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_ofe_hdr_clk = {
+	.halt_reg = 0x1016c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1016c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_ofe_hdr_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_ofe_main_clk = {
+	.halt_reg = 0x10144,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10144,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_ofe_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_axi_clk = {
+	.halt_reg = 0x11300,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11300,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_rt_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_ife_lite_clk = {
+	.halt_reg = 0x11178,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11178,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_0_bayer_clk = {
+	.halt_reg = 0x11054,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_0_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_0_main_clk = {
+	.halt_reg = 0x11040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_0_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_1_bayer_clk = {
+	.halt_reg = 0x110d4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_1_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_1_main_clk = {
+	.halt_reg = 0x110c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_1_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_2_bayer_clk = {
+	.halt_reg = 0x1113c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1113c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_2_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_2_main_clk = {
+	.halt_reg = 0x11128,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11128,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_2_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_xo_clk = {
+	.halt_reg = 0x11324,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11324,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_0_clk = {
+	.halt_reg = 0x11284,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11284,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_1_clk = {
+	.halt_reg = 0x112a0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x112a0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_2_clk = {
+	.halt_reg = 0x112bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x112bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_core_ahb_clk = {
+	.halt_reg = 0x11360,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x11360,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_core_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cre_ahb_clk = {
+	.halt_reg = 0x111cc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x111cc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cre_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cre_clk = {
+	.halt_reg = 0x111c4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x111c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cre_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cre_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi0phytimer_clk = {
+	.halt_reg = 0x10018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi0phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi0phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi1phytimer_clk = {
+	.halt_reg = 0x1003c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1003c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi1phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi1phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi2phytimer_clk = {
+	.halt_reg = 0x1005c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1005c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi2phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi2phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi3phytimer_clk = {
+	.halt_reg = 0x1007c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1007c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi3phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi3phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi4phytimer_clk = {
+	.halt_reg = 0x1009c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1009c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi4phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi4phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi5phytimer_clk = {
+	.halt_reg = 0x100bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi5phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi5phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_clk = {
+	.halt_reg = 0x112d8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x112d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_csiphy_rx_clk = {
+	.halt_reg = 0x10020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_csiphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy0_clk = {
+	.halt_reg = 0x1001c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1001c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy1_clk = {
+	.halt_reg = 0x10040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy2_clk = {
+	.halt_reg = 0x10060,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy3_clk = {
+	.halt_reg = 0x10080,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10080,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy4_clk = {
+	.halt_reg = 0x100a0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100a0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy5_clk = {
+	.halt_reg = 0x100c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_drv_ahb_clk = {
+	.halt_reg = 0x113c4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x113c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_drv_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_drv_xo_clk = {
+	.halt_reg = 0x113c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x113c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_drv_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_0_ahb_clk = {
+	.halt_reg = 0x11264,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11264,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_0_clk = {
+	.halt_reg = 0x1122c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1122c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_icp_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_1_ahb_clk = {
+	.halt_reg = 0x11268,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11268,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_1_clk = {
+	.halt_reg = 0x11254,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11254,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_icp_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_ahb_clk = {
+	.halt_reg = 0x111a8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x111a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_clk = {
+	.halt_reg = 0x11168,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11168,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_cphy_rx_clk = {
+	.halt_reg = 0x111a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x111a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_cphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_csid_clk = {
+	.halt_reg = 0x11194,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11194,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_ahb_clk = {
+	.halt_reg = 0x101d4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x101d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_clk = {
+	.halt_reg = 0x101a8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x101a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_fast_ahb_clk = {
+	.halt_reg = 0x101d8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x101d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_pps_clk = {
+	.halt_reg = 0x101bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x101bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_pps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_pps_fast_ahb_clk = {
+	.halt_reg = 0x101dc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x101dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_pps_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_jpeg_0_clk = {
+	.halt_reg = 0x111e8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x111e8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_jpeg_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_jpeg_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_jpeg_1_clk = {
+	.halt_reg = 0x111f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x111f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_jpeg_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_jpeg_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_ahb_clk = {
+	.halt_reg = 0x10118,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10118,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_anchor_clk = {
+	.halt_reg = 0x10148,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10148,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_anchor_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_anchor_fast_ahb_clk = {
+	.halt_reg = 0x100f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_anchor_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_hdr_clk = {
+	.halt_reg = 0x1015c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1015c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_hdr_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_hdr_fast_ahb_clk = {
+	.halt_reg = 0x100fc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100fc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_hdr_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_main_clk = {
+	.halt_reg = 0x10134,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10134,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_main_fast_ahb_clk = {
+	.halt_reg = 0x100f4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_clk = {
+	.halt_reg = 0x11344,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11344,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qdss_debug_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_xo_clk = {
+	.halt_reg = 0x11348,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11348,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_bayer_clk = {
+	.halt_reg = 0x11044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_bayer_fast_ahb_clk = {
+	.halt_reg = 0x11064,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11064,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_bayer_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_main_clk = {
+	.halt_reg = 0x11030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_main_fast_ahb_clk = {
+	.halt_reg = 0x11060,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_bayer_clk = {
+	.halt_reg = 0x110c4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_bayer_fast_ahb_clk = {
+	.halt_reg = 0x110e4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110e4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_bayer_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_main_clk = {
+	.halt_reg = 0x110b0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_main_fast_ahb_clk = {
+	.halt_reg = 0x110e0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110e0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_bayer_clk = {
+	.halt_reg = 0x1112c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1112c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_bayer_fast_ahb_clk = {
+	.halt_reg = 0x1114c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1114c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_bayer_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_main_clk = {
+	.halt_reg = 0x11118,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11118,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_main_fast_ahb_clk = {
+	.halt_reg = 0x11148,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11148,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc cam_cc_titan_top_gdsc = {
+	.gdscr = 0x1134c,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_titan_top_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ipe_0_gdsc = {
+	.gdscr = 0x1017c,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ipe_0_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ofe_gdsc = {
+	.gdscr = 0x100c8,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ofe_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_tfe_0_gdsc = {
+	.gdscr = 0x11004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_tfe_0_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_tfe_1_gdsc = {
+	.gdscr = 0x11084,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_tfe_1_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_tfe_2_gdsc = {
+	.gdscr = 0x110ec,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_tfe_2_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *cam_cc_eliza_clocks[] = {
+	[CAM_CC_CAM_TOP_AHB_CLK] = &cam_cc_cam_top_ahb_clk.clkr,
+	[CAM_CC_CAM_TOP_FAST_AHB_CLK] = &cam_cc_cam_top_fast_ahb_clk.clkr,
+	[CAM_CC_CAMNOC_DCD_XO_CLK] = &cam_cc_camnoc_dcd_xo_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_AXI_CLK] = &cam_cc_camnoc_nrt_axi_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_CRE_CLK] = &cam_cc_camnoc_nrt_cre_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_IPE_NPS_CLK] = &cam_cc_camnoc_nrt_ipe_nps_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_OFE_ANCHOR_CLK] = &cam_cc_camnoc_nrt_ofe_anchor_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_OFE_HDR_CLK] = &cam_cc_camnoc_nrt_ofe_hdr_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_OFE_MAIN_CLK] = &cam_cc_camnoc_nrt_ofe_main_clk.clkr,
+	[CAM_CC_CAMNOC_RT_AXI_CLK] = &cam_cc_camnoc_rt_axi_clk.clkr,
+	[CAM_CC_CAMNOC_RT_AXI_CLK_SRC] = &cam_cc_camnoc_rt_axi_clk_src.clkr,
+	[CAM_CC_CAMNOC_RT_IFE_LITE_CLK] = &cam_cc_camnoc_rt_ife_lite_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_0_BAYER_CLK] = &cam_cc_camnoc_rt_tfe_0_bayer_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_0_MAIN_CLK] = &cam_cc_camnoc_rt_tfe_0_main_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_1_BAYER_CLK] = &cam_cc_camnoc_rt_tfe_1_bayer_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_1_MAIN_CLK] = &cam_cc_camnoc_rt_tfe_1_main_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_2_BAYER_CLK] = &cam_cc_camnoc_rt_tfe_2_bayer_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_2_MAIN_CLK] = &cam_cc_camnoc_rt_tfe_2_main_clk.clkr,
+	[CAM_CC_CAMNOC_XO_CLK] = &cam_cc_camnoc_xo_clk.clkr,
+	[CAM_CC_CCI_0_CLK] = &cam_cc_cci_0_clk.clkr,
+	[CAM_CC_CCI_0_CLK_SRC] = &cam_cc_cci_0_clk_src.clkr,
+	[CAM_CC_CCI_1_CLK] = &cam_cc_cci_1_clk.clkr,
+	[CAM_CC_CCI_1_CLK_SRC] = &cam_cc_cci_1_clk_src.clkr,
+	[CAM_CC_CCI_2_CLK] = &cam_cc_cci_2_clk.clkr,
+	[CAM_CC_CCI_2_CLK_SRC] = &cam_cc_cci_2_clk_src.clkr,
+	[CAM_CC_CORE_AHB_CLK] = &cam_cc_core_ahb_clk.clkr,
+	[CAM_CC_CPHY_RX_CLK_SRC] = &cam_cc_cphy_rx_clk_src.clkr,
+	[CAM_CC_CRE_AHB_CLK] = &cam_cc_cre_ahb_clk.clkr,
+	[CAM_CC_CRE_CLK] = &cam_cc_cre_clk.clkr,
+	[CAM_CC_CRE_CLK_SRC] = &cam_cc_cre_clk_src.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK] = &cam_cc_csi0phytimer_clk.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK_SRC] = &cam_cc_csi0phytimer_clk_src.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK] = &cam_cc_csi1phytimer_clk.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK_SRC] = &cam_cc_csi1phytimer_clk_src.clkr,
+	[CAM_CC_CSI2PHYTIMER_CLK] = &cam_cc_csi2phytimer_clk.clkr,
+	[CAM_CC_CSI2PHYTIMER_CLK_SRC] = &cam_cc_csi2phytimer_clk_src.clkr,
+	[CAM_CC_CSI3PHYTIMER_CLK] = &cam_cc_csi3phytimer_clk.clkr,
+	[CAM_CC_CSI3PHYTIMER_CLK_SRC] = &cam_cc_csi3phytimer_clk_src.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK] = &cam_cc_csi4phytimer_clk.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK_SRC] = &cam_cc_csi4phytimer_clk_src.clkr,
+	[CAM_CC_CSI5PHYTIMER_CLK] = &cam_cc_csi5phytimer_clk.clkr,
+	[CAM_CC_CSI5PHYTIMER_CLK_SRC] = &cam_cc_csi5phytimer_clk_src.clkr,
+	[CAM_CC_CSID_CLK] = &cam_cc_csid_clk.clkr,
+	[CAM_CC_CSID_CLK_SRC] = &cam_cc_csid_clk_src.clkr,
+	[CAM_CC_CSID_CSIPHY_RX_CLK] = &cam_cc_csid_csiphy_rx_clk.clkr,
+	[CAM_CC_CSIPHY0_CLK] = &cam_cc_csiphy0_clk.clkr,
+	[CAM_CC_CSIPHY1_CLK] = &cam_cc_csiphy1_clk.clkr,
+	[CAM_CC_CSIPHY2_CLK] = &cam_cc_csiphy2_clk.clkr,
+	[CAM_CC_CSIPHY3_CLK] = &cam_cc_csiphy3_clk.clkr,
+	[CAM_CC_CSIPHY4_CLK] = &cam_cc_csiphy4_clk.clkr,
+	[CAM_CC_CSIPHY5_CLK] = &cam_cc_csiphy5_clk.clkr,
+	[CAM_CC_DRV_AHB_CLK] = &cam_cc_drv_ahb_clk.clkr,
+	[CAM_CC_DRV_XO_CLK] = &cam_cc_drv_xo_clk.clkr,
+	[CAM_CC_FAST_AHB_CLK_SRC] = &cam_cc_fast_ahb_clk_src.clkr,
+	[CAM_CC_ICP_0_AHB_CLK] = &cam_cc_icp_0_ahb_clk.clkr,
+	[CAM_CC_ICP_0_CLK] = &cam_cc_icp_0_clk.clkr,
+	[CAM_CC_ICP_0_CLK_SRC] = &cam_cc_icp_0_clk_src.clkr,
+	[CAM_CC_ICP_1_AHB_CLK] = &cam_cc_icp_1_ahb_clk.clkr,
+	[CAM_CC_ICP_1_CLK] = &cam_cc_icp_1_clk.clkr,
+	[CAM_CC_ICP_1_CLK_SRC] = &cam_cc_icp_1_clk_src.clkr,
+	[CAM_CC_IFE_LITE_AHB_CLK] = &cam_cc_ife_lite_ahb_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK] = &cam_cc_ife_lite_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK_SRC] = &cam_cc_ife_lite_clk_src.clkr,
+	[CAM_CC_IFE_LITE_CPHY_RX_CLK] = &cam_cc_ife_lite_cphy_rx_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK] = &cam_cc_ife_lite_csid_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK_SRC] = &cam_cc_ife_lite_csid_clk_src.clkr,
+	[CAM_CC_IPE_NPS_AHB_CLK] = &cam_cc_ipe_nps_ahb_clk.clkr,
+	[CAM_CC_IPE_NPS_CLK] = &cam_cc_ipe_nps_clk.clkr,
+	[CAM_CC_IPE_NPS_CLK_SRC] = &cam_cc_ipe_nps_clk_src.clkr,
+	[CAM_CC_IPE_NPS_FAST_AHB_CLK] = &cam_cc_ipe_nps_fast_ahb_clk.clkr,
+	[CAM_CC_IPE_PPS_CLK] = &cam_cc_ipe_pps_clk.clkr,
+	[CAM_CC_IPE_PPS_FAST_AHB_CLK] = &cam_cc_ipe_pps_fast_ahb_clk.clkr,
+	[CAM_CC_JPEG_0_CLK] = &cam_cc_jpeg_0_clk.clkr,
+	[CAM_CC_JPEG_1_CLK] = &cam_cc_jpeg_1_clk.clkr,
+	[CAM_CC_JPEG_CLK_SRC] = &cam_cc_jpeg_clk_src.clkr,
+	[CAM_CC_OFE_AHB_CLK] = &cam_cc_ofe_ahb_clk.clkr,
+	[CAM_CC_OFE_ANCHOR_CLK] = &cam_cc_ofe_anchor_clk.clkr,
+	[CAM_CC_OFE_ANCHOR_FAST_AHB_CLK] = &cam_cc_ofe_anchor_fast_ahb_clk.clkr,
+	[CAM_CC_OFE_CLK_SRC] = &cam_cc_ofe_clk_src.clkr,
+	[CAM_CC_OFE_HDR_CLK] = &cam_cc_ofe_hdr_clk.clkr,
+	[CAM_CC_OFE_HDR_FAST_AHB_CLK] = &cam_cc_ofe_hdr_fast_ahb_clk.clkr,
+	[CAM_CC_OFE_MAIN_CLK] = &cam_cc_ofe_main_clk.clkr,
+	[CAM_CC_OFE_MAIN_FAST_AHB_CLK] = &cam_cc_ofe_main_fast_ahb_clk.clkr,
+	[CAM_CC_PLL0] = &cam_cc_pll0.clkr,
+	[CAM_CC_PLL0_OUT_EVEN] = &cam_cc_pll0_out_even.clkr,
+	[CAM_CC_PLL0_OUT_ODD] = &cam_cc_pll0_out_odd.clkr,
+	[CAM_CC_PLL1] = &cam_cc_pll1.clkr,
+	[CAM_CC_PLL1_OUT_EVEN] = &cam_cc_pll1_out_even.clkr,
+	[CAM_CC_PLL2] = &cam_cc_pll2.clkr,
+	[CAM_CC_PLL2_OUT_EVEN] = &cam_cc_pll2_out_even.clkr,
+	[CAM_CC_PLL3] = &cam_cc_pll3.clkr,
+	[CAM_CC_PLL3_OUT_EVEN] = &cam_cc_pll3_out_even.clkr,
+	[CAM_CC_PLL4] = &cam_cc_pll4.clkr,
+	[CAM_CC_PLL4_OUT_EVEN] = &cam_cc_pll4_out_even.clkr,
+	[CAM_CC_PLL5] = &cam_cc_pll5.clkr,
+	[CAM_CC_PLL5_OUT_EVEN] = &cam_cc_pll5_out_even.clkr,
+	[CAM_CC_PLL6] = &cam_cc_pll6.clkr,
+	[CAM_CC_PLL6_OUT_EVEN] = &cam_cc_pll6_out_even.clkr,
+	[CAM_CC_PLL6_OUT_ODD] = &cam_cc_pll6_out_odd.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK] = &cam_cc_qdss_debug_clk.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK_SRC] = &cam_cc_qdss_debug_clk_src.clkr,
+	[CAM_CC_QDSS_DEBUG_XO_CLK] = &cam_cc_qdss_debug_xo_clk.clkr,
+	[CAM_CC_SLEEP_CLK_SRC] = &cam_cc_sleep_clk_src.clkr,
+	[CAM_CC_SLOW_AHB_CLK_SRC] = &cam_cc_slow_ahb_clk_src.clkr,
+	[CAM_CC_TFE_0_BAYER_CLK] = &cam_cc_tfe_0_bayer_clk.clkr,
+	[CAM_CC_TFE_0_BAYER_FAST_AHB_CLK] = &cam_cc_tfe_0_bayer_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_0_CLK_SRC] = &cam_cc_tfe_0_clk_src.clkr,
+	[CAM_CC_TFE_0_MAIN_CLK] = &cam_cc_tfe_0_main_clk.clkr,
+	[CAM_CC_TFE_0_MAIN_FAST_AHB_CLK] = &cam_cc_tfe_0_main_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_1_BAYER_CLK] = &cam_cc_tfe_1_bayer_clk.clkr,
+	[CAM_CC_TFE_1_BAYER_FAST_AHB_CLK] = &cam_cc_tfe_1_bayer_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_1_CLK_SRC] = &cam_cc_tfe_1_clk_src.clkr,
+	[CAM_CC_TFE_1_MAIN_CLK] = &cam_cc_tfe_1_main_clk.clkr,
+	[CAM_CC_TFE_1_MAIN_FAST_AHB_CLK] = &cam_cc_tfe_1_main_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_2_BAYER_CLK] = &cam_cc_tfe_2_bayer_clk.clkr,
+	[CAM_CC_TFE_2_BAYER_FAST_AHB_CLK] = &cam_cc_tfe_2_bayer_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_2_CLK_SRC] = &cam_cc_tfe_2_clk_src.clkr,
+	[CAM_CC_TFE_2_MAIN_CLK] = &cam_cc_tfe_2_main_clk.clkr,
+	[CAM_CC_TFE_2_MAIN_FAST_AHB_CLK] = &cam_cc_tfe_2_main_fast_ahb_clk.clkr,
+	[CAM_CC_XO_CLK_SRC] = &cam_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *cam_cc_eliza_gdscs[] = {
+	[CAM_CC_IPE_0_GDSC] = &cam_cc_ipe_0_gdsc,
+	[CAM_CC_OFE_GDSC] = &cam_cc_ofe_gdsc,
+	[CAM_CC_TFE_0_GDSC] = &cam_cc_tfe_0_gdsc,
+	[CAM_CC_TFE_1_GDSC] = &cam_cc_tfe_1_gdsc,
+	[CAM_CC_TFE_2_GDSC] = &cam_cc_tfe_2_gdsc,
+	[CAM_CC_TITAN_TOP_GDSC] = &cam_cc_titan_top_gdsc,
+};
+
+static const struct qcom_reset_map cam_cc_eliza_resets[] = {
+	[CAM_CC_DRV_BCR] = { 0x113bc },
+	[CAM_CC_ICP_BCR] = { 0x11210 },
+	[CAM_CC_IPE_0_BCR] = { 0x10178 },
+	[CAM_CC_OFE_BCR] = { 0x100c4 },
+	[CAM_CC_QDSS_DEBUG_BCR] = { 0x11328 },
+	[CAM_CC_TFE_0_BCR] = { 0x11000 },
+	[CAM_CC_TFE_1_BCR] = { 0x11080 },
+	[CAM_CC_TFE_2_BCR] = { 0x110e8 },
+};
+
+static struct clk_alpha_pll *cam_cc_eliza_plls[] = {
+	&cam_cc_pll0,
+	&cam_cc_pll1,
+	&cam_cc_pll2,
+	&cam_cc_pll3,
+	&cam_cc_pll4,
+	&cam_cc_pll5,
+	&cam_cc_pll6,
+};
+
+static const u32 cam_cc_eliza_critical_cbcrs[] = {
+	0x1137c, /* CAM_CC_GDSC_CLK */
+	0x11398, /* CAM_CC_SLEEP_CLK */
+};
+
+static const struct regmap_config cam_cc_eliza_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1601c,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_driver_data cam_cc_eliza_driver_data = {
+	.alpha_plls = cam_cc_eliza_plls,
+	.num_alpha_plls = ARRAY_SIZE(cam_cc_eliza_plls),
+	.clk_cbcrs = cam_cc_eliza_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_eliza_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc cam_cc_eliza_desc = {
+	.config = &cam_cc_eliza_regmap_config,
+	.clks = cam_cc_eliza_clocks,
+	.num_clks = ARRAY_SIZE(cam_cc_eliza_clocks),
+	.resets = cam_cc_eliza_resets,
+	.num_resets = ARRAY_SIZE(cam_cc_eliza_resets),
+	.gdscs = cam_cc_eliza_gdscs,
+	.num_gdscs = ARRAY_SIZE(cam_cc_eliza_gdscs),
+	.use_rpm = true,
+	.driver_data = &cam_cc_eliza_driver_data,
+};
+
+static const struct of_device_id cam_cc_eliza_match_table[] = {
+	{ .compatible = "qcom,eliza-camcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, cam_cc_eliza_match_table);
+
+static int cam_cc_eliza_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &cam_cc_eliza_desc);
+}
+
+static struct platform_driver cam_cc_eliza_driver = {
+	.probe = cam_cc_eliza_probe,
+	.driver = {
+		.name = "camcc-eliza",
+		.of_match_table = cam_cc_eliza_match_table,
+	},
+};
+
+module_platform_driver(cam_cc_eliza_driver);
+
+MODULE_DESCRIPTION("QTI CAMCC Eliza Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/camcc-glymur.c b/drivers/clk/qcom/camcc-glymur.c
index 81c1e102b110e5..6d022bbd2ce2ef 100644
--- a/drivers/clk/qcom/camcc-glymur.c
+++ b/drivers/clk/qcom/camcc-glymur.c
@@ -2235,11 +2235,18 @@ static const struct regmap_config cam_cc_glymur_regmap_config = {
 	.fast_io = true,
 };
 
+static void cam_cc_glymur_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	regmap_write(regmap, 0x2018, 0); /* CAM_CC_PLL2_PLL_USER_CTL_U */
+	regmap_write(regmap, 0x2028, 0); /* CAM_CC_PLL2_PLL_CONFIG_CTL_U2 */
+}
+
 static const struct qcom_cc_driver_data cam_cc_glymur_driver_data = {
 	.alpha_plls = cam_cc_glymur_plls,
 	.num_alpha_plls = ARRAY_SIZE(cam_cc_glymur_plls),
 	.clk_cbcrs = cam_cc_glymur_critical_cbcrs,
 	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_glymur_critical_cbcrs),
+	.clk_regs_configure = cam_cc_glymur_regs_configure,
 };
 
 static const struct qcom_cc_desc cam_cc_glymur_desc = {
diff --git a/drivers/clk/qcom/camcc-hawi.c b/drivers/clk/qcom/camcc-hawi.c
new file mode 100644
index 00000000000000..9842a9a4bf853b
--- /dev/null
+++ b/drivers/clk/qcom/camcc-hawi.c
@@ -0,0 +1,3152 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,hawi-camcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_AHB_CLK,
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK,
+};
+
+/* Need to match the order of interconnects in DT binding */
+enum {
+	DT_ICC_TOP_GDSC,
+};
+
+enum {
+	P_BI_TCXO,
+	P_CAM_CC_PLL0_OUT_EVEN,
+	P_CAM_CC_PLL0_OUT_MAIN,
+	P_CAM_CC_PLL0_OUT_ODD,
+	P_CAM_CC_PLL1_OUT_EVEN,
+	P_CAM_CC_PLL2_OUT_EVEN,
+	P_CAM_CC_PLL3_OUT_EVEN,
+	P_CAM_CC_PLL4_OUT_EVEN,
+	P_CAM_CC_PLL5_OUT_EVEN,
+	P_CAM_CC_PLL6_OUT_EVEN,
+	P_CAM_CC_PLL6_OUT_MAIN,
+	P_CAM_CC_PLL6_OUT_ODD,
+	P_CAM_CC_PLL7_OUT_EVEN,
+	P_CAM_CC_PLL7_OUT_MAIN,
+	P_CAM_CC_PLL8_OUT_EVEN,
+};
+
+static const struct pll_vco taycan_eha_t_vco[] = {
+	{ 249600000, 2500000000, 0 },
+};
+
+/* 1200.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll0_config = {
+	.l = 0x3e,
+	.alpha = 0x8000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00008400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 1200.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll0_config_maili = {
+	.l = 0x3e,
+	.cal_l = 0x42,
+	.alpha = 0x8000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00008400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll0 = {
+	.offset = 0x0,
+	.config = &cam_cc_pll0_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_odd[] = {
+	{ 0x2, 3 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_odd = {
+	.offset = 0x0,
+	.post_div_shift = 14,
+	.post_div_table = post_div_table_cam_cc_pll0_out_odd,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_odd),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_odd",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 493.2 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll1_config = {
+	.l = 0x19,
+	.alpha = 0xb000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 493.2 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll1_config_maili = {
+	.l = 0x19,
+	.cal_l = 0x42,
+	.alpha = 0xb000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll1 = {
+	.offset = 0x1000,
+	.config = &cam_cc_pll1_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll1",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll1_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll1_out_even = {
+	.offset = 0x1000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll1_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll1_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll1_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll1.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 451.2 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll2_config = {
+	.l = 0x17,
+	.alpha = 0x8000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 600.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll2_config_maili = {
+	.l = 0x1f,
+	.cal_l = 0x42,
+	.alpha = 0x4000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll2 = {
+	.offset = 0x2000,
+	.config = &cam_cc_pll2_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll2",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll2_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll2_out_even = {
+	.offset = 0x2000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll2_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll2_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll2_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll2.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 476.4 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll3_config = {
+	.l = 0x18,
+	.alpha = 0xd000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 476.4 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll3_config_maili = {
+	.l = 0x18,
+	.cal_l = 0x42,
+	.alpha = 0xd000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll3 = {
+	.offset = 0x3000,
+	.config = &cam_cc_pll3_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll3",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll3_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll3_out_even = {
+	.offset = 0x3000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll3_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll3_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll3_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll3.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 476.4 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll4_config = {
+	.l = 0x18,
+	.alpha = 0xd000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 476.4 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll4_config_maili = {
+	.l = 0x18,
+	.cal_l = 0x42,
+	.alpha = 0xd000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll4 = {
+	.offset = 0x4000,
+	.config = &cam_cc_pll4_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll4_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll4_out_even = {
+	.offset = 0x4000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll4_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll4_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll4_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll4.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 476.4 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll5_config = {
+	.l = 0x18,
+	.alpha = 0xd000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 476.4 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll5_config_maili = {
+	.l = 0x18,
+	.cal_l = 0x42,
+	.alpha = 0xd000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll5 = {
+	.offset = 0x5000,
+	.config = &cam_cc_pll5_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll5",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll5_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll5_out_even = {
+	.offset = 0x5000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll5_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll5_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll5_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll5.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 960.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll6_config = {
+	.l = 0x32,
+	.alpha = 0x0,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00008400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 960.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll6_config_maili = {
+	.l = 0x32,
+	.cal_l = 0x42,
+	.alpha = 0x0,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00008400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll6 = {
+	.offset = 0x6000,
+	.config = &cam_cc_pll6_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll6",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll6_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll6_out_even = {
+	.offset = 0x6000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll6_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll6_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll6_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll6_out_odd[] = {
+	{ 0x2, 3 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll6_out_odd = {
+	.offset = 0x6000,
+	.post_div_shift = 14,
+	.post_div_table = post_div_table_cam_cc_pll6_out_odd,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll6_out_odd),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll6_out_odd",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+/* 345.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll7_config = {
+	.l = 0x11,
+	.alpha = 0xf800,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 345.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll7_config_maili = {
+	.l = 0x11,
+	.cal_l = 0x42,
+	.alpha = 0xf800,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll7 = {
+	.offset = 0x7000,
+	.config = &cam_cc_pll7_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll7",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+/* 493.2 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll8_config = {
+	.l = 0x19,
+	.alpha = 0xb000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 493.2 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll8_config_maili = {
+	.l = 0x19,
+	.cal_l = 0x42,
+	.alpha = 0xb000,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll cam_cc_pll8 = {
+	.offset = 0x8000,
+	.config = &cam_cc_pll8_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll8",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll8_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll8_out_even = {
+	.offset = 0x8000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll8_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll8_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll8_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll8.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+static const struct parent_map cam_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+	{ P_CAM_CC_PLL6_OUT_ODD, 4 },
+	{ P_CAM_CC_PLL6_OUT_EVEN, 5 },
+	{ P_CAM_CC_PLL6_OUT_MAIN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll6.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+	{ P_CAM_CC_PLL6_OUT_ODD, 4 },
+	{ P_CAM_CC_PLL6_OUT_EVEN, 5 },
+	{ P_CAM_CC_PLL6_OUT_MAIN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll6_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll6.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+	{ P_CAM_CC_PLL7_OUT_EVEN, 5 },
+	{ P_CAM_CC_PLL7_OUT_MAIN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+	{ .hw = &cam_cc_pll7.clkr.hw },
+	{ .hw = &cam_cc_pll7.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL1_OUT_EVEN, 4 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll1_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL2_OUT_EVEN, 5 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll2_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL8_OUT_EVEN, 4 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll8_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_6[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL3_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_6[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll3_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_7[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL4_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_7[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll4_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_8[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL5_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_8[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll5_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_9[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_9[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct freq_tbl ftbl_cam_cc_camnoc_rt_axi_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(120000000, P_CAM_CC_PLL6_OUT_EVEN, 4, 0, 0),
+	F(160000000, P_CAM_CC_PLL6_OUT_EVEN, 3, 0, 0),
+	F(240000000, P_CAM_CC_PLL6_OUT_EVEN, 2, 0, 0),
+	F(320000000, P_CAM_CC_PLL6_OUT_EVEN, 1.5, 0, 0),
+	F(480000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static const struct freq_tbl ftbl_cam_cc_camnoc_rt_axi_clk_src_maili[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(200000000, P_CAM_CC_PLL0_OUT_MAIN, 6, 0, 0),
+	F(240000000, P_CAM_CC_PLL0_OUT_MAIN, 5, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	F(600000000, P_CAM_CC_PLL0_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_camnoc_rt_axi_clk_src = {
+	.cmd_rcgr = 0x212cc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_camnoc_rt_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_camnoc_rt_axi_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cci_0_clk_src[] = {
+	F(37500000, P_CAM_CC_PLL0_OUT_ODD, 1, 3, 32),
+	{ }
+};
+
+static const struct freq_tbl ftbl_cam_cc_cci_0_clk_src_maili[] = {
+	F(37500000, P_CAM_CC_PLL0_OUT_EVEN, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cci_0_clk_src = {
+	.cmd_rcgr = 0x21250,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_0_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_1_clk_src = {
+	.cmd_rcgr = 0x2126c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_1_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_2_clk_src = {
+	.cmd_rcgr = 0x21288,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_2_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cphy_rx_clk_src[] = {
+	F(200000000, P_CAM_CC_PLL0_OUT_MAIN, 6, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cphy_rx_clk_src = {
+	.cmd_rcgr = 0x21064,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cphy_rx_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cre_clk_src[] = {
+	F(80000000, P_CAM_CC_PLL6_OUT_EVEN, 6, 0, 0),
+	F(137142857, P_CAM_CC_PLL6_OUT_EVEN, 3.5, 0, 0),
+	F(200000000, P_CAM_CC_PLL0_OUT_ODD, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	F(600000000, P_CAM_CC_PLL0_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cre_clk_src = {
+	.cmd_rcgr = 0x211a0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cre_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cre_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_csi0phytimer_clk_src[] = {
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x20000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi0phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x20024,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi1phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi2phytimer_clk_src = {
+	.cmd_rcgr = 0x20044,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi2phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi3phytimer_clk_src = {
+	.cmd_rcgr = 0x20064,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi3phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi4phytimer_clk_src = {
+	.cmd_rcgr = 0x20084,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi4phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi5phytimer_clk_src = {
+	.cmd_rcgr = 0x200a4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi5phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csid_clk_src = {
+	.cmd_rcgr = 0x212a4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csid_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_fast_ahb_clk_src[] = {
+	F(128000000, P_CAM_CC_PLL6_OUT_ODD, 2.5, 0, 0),
+	F(213333333, P_CAM_CC_PLL6_OUT_ODD, 1.5, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_fast_ahb_clk_src = {
+	.cmd_rcgr = 0x200dc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_fast_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_fast_ahb_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_icp_0_clk_src[] = {
+	F(345000000, P_CAM_CC_PLL7_OUT_MAIN, 1, 0, 0),
+	F(500000000, P_CAM_CC_PLL7_OUT_MAIN, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL7_OUT_MAIN, 1, 0, 0),
+	F(770000000, P_CAM_CC_PLL7_OUT_MAIN, 1, 0, 0),
+	F(900000000, P_CAM_CC_PLL7_OUT_MAIN, 1, 0, 0),
+	F(1150000000, P_CAM_CC_PLL7_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_icp_0_clk_src = {
+	.cmd_rcgr = 0x211f8,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_2,
+	.freq_tbl = ftbl_cam_cc_icp_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_icp_0_clk_src",
+		.parent_data = cam_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_icp_1_clk_src = {
+	.cmd_rcgr = 0x21220,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_2,
+	.freq_tbl = ftbl_cam_cc_icp_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_icp_1_clk_src",
+		.parent_data = cam_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ife_lite_clk_src[] = {
+	F(200000000, P_CAM_CC_PLL0_OUT_MAIN, 6, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_clk_src = {
+	.cmd_rcgr = 0x21144,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_ife_lite_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_lite_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_csid_clk_src = {
+	.cmd_rcgr = 0x21170,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_lite_csid_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ipe_nps_clk_src[] = {
+	F(246600000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(411000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(475000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(575000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(675000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	F(825000000, P_CAM_CC_PLL1_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ipe_nps_clk_src = {
+	.cmd_rcgr = 0x2019c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_3,
+	.freq_tbl = ftbl_cam_cc_ipe_nps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ipe_nps_clk_src",
+		.parent_data = cam_cc_parent_data_3,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_jpeg_clk_src = {
+	.cmd_rcgr = 0x211c4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cre_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_jpeg_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ofe_clk_src[] = {
+	F(225600000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(300000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(376000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(445000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(500000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(609000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(688000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(841000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static const struct freq_tbl ftbl_cam_cc_ofe_clk_src_maili[] = {
+	F(300000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(376000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(445000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(500000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(609000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(688000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(841000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ofe_clk_src = {
+	.cmd_rcgr = 0x20120,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_4,
+	.freq_tbl = ftbl_cam_cc_ofe_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ofe_clk_src",
+		.parent_data = cam_cc_parent_data_4,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_qdss_debug_clk_src[] = {
+	F(40000000, P_CAM_CC_PLL6_OUT_ODD, 8, 0, 0),
+	F(60000000, P_CAM_CC_PLL0_OUT_EVEN, 10, 0, 0),
+	F(120000000, P_CAM_CC_PLL0_OUT_EVEN, 5, 0, 0),
+	F(240000000, P_CAM_CC_PLL0_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_qdss_debug_clk_src = {
+	.cmd_rcgr = 0x21310,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_qdss_debug_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_qdss_debug_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_slow_ahb_clk_src[] = {
+	F(56470588, P_CAM_CC_PLL6_OUT_EVEN, 8.5, 0, 0),
+	F(80000000, P_CAM_CC_PLL0_OUT_EVEN, 7.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_slow_ahb_clk_src = {
+	.cmd_rcgr = 0x20104,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_slow_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_slow_ahb_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_sqpf_clk_src[] = {
+	F(246600000, P_CAM_CC_PLL8_OUT_EVEN, 1, 0, 0),
+	F(411000000, P_CAM_CC_PLL8_OUT_EVEN, 1, 0, 0),
+	F(475000000, P_CAM_CC_PLL8_OUT_EVEN, 1, 0, 0),
+	F(575000000, P_CAM_CC_PLL8_OUT_EVEN, 1, 0, 0),
+	F(675000000, P_CAM_CC_PLL8_OUT_EVEN, 1, 0, 0),
+	F(825000000, P_CAM_CC_PLL8_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_sqpf_clk_src = {
+	.cmd_rcgr = 0x2021c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_5,
+	.freq_tbl = ftbl_cam_cc_sqpf_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_sqpf_clk_src",
+		.parent_data = cam_cc_parent_data_5,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_tfe_0_clk_src[] = {
+	F(238200000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(397000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(440000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(630000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(750000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(833000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_tfe_0_clk_src = {
+	.cmd_rcgr = 0x21018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_6,
+	.freq_tbl = ftbl_cam_cc_tfe_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_tfe_0_clk_src",
+		.parent_data = cam_cc_parent_data_6,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_6),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_tfe_1_clk_src[] = {
+	F(238200000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(397000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(440000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(630000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(750000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(833000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_tfe_1_clk_src = {
+	.cmd_rcgr = 0x21094,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_7,
+	.freq_tbl = ftbl_cam_cc_tfe_1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_tfe_1_clk_src",
+		.parent_data = cam_cc_parent_data_7,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_7),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_tfe_2_clk_src[] = {
+	F(238200000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(397000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(440000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(630000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(750000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(833000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_tfe_2_clk_src = {
+	.cmd_rcgr = 0x210f8,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_8,
+	.freq_tbl = ftbl_cam_cc_tfe_2_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_tfe_2_clk_src",
+		.parent_data = cam_cc_parent_data_8,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_8),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_xo_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_xo_clk_src = {
+	.cmd_rcgr = 0x21348,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_9,
+	.freq_tbl = ftbl_cam_cc_xo_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_xo_clk_src",
+		.parent_data = cam_cc_parent_data_9,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_9),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_branch cam_cc_cam_top_ahb_clk = {
+	.halt_reg = 0x21378,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21378,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cam_top_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cam_top_fast_ahb_clk = {
+	.halt_reg = 0x21368,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21368,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cam_top_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_axi_clk = {
+	.halt_reg = 0x212f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x212f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_rt_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_cre_clk = {
+	.halt_reg = 0x211bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x211bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_cre_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cre_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_ipe_nps_clk = {
+	.halt_reg = 0x201c4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_ipe_nps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_ipe_scalers_clk = {
+	.halt_reg = 0x201e8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201e8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_ipe_scalers_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_ofe_main_clk = {
+	.halt_reg = 0x20148,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20148,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_ofe_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_sqpf_clk = {
+	.halt_reg = 0x20244,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20244,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_sqpf_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_sqpf_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_axi_clk = {
+	.halt_reg = 0x212e4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x212e4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_rt_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_ife_lite_clk = {
+	.halt_reg = 0x2116c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2116c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_0_main_clk = {
+	.halt_reg = 0x21040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_0_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_1_main_clk = {
+	.halt_reg = 0x210bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x210bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_1_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_tfe_2_main_clk = {
+	.halt_reg = 0x21120,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21120,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_tfe_2_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_0_clk = {
+	.halt_reg = 0x21268,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21268,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_1_clk = {
+	.halt_reg = 0x21284,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21284,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_2_clk = {
+	.halt_reg = 0x212a0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x212a0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_core_ahb_clk = {
+	.halt_reg = 0x21344,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x21344,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_core_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cre_ahb_clk = {
+	.halt_reg = 0x211c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x211c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cre_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cre_clk = {
+	.halt_reg = 0x211b8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x211b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cre_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cre_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi0phytimer_clk = {
+	.halt_reg = 0x20018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi0phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi0phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi1phytimer_clk = {
+	.halt_reg = 0x2003c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2003c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi1phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi1phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi2phytimer_clk = {
+	.halt_reg = 0x2005c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2005c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi2phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi2phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi3phytimer_clk = {
+	.halt_reg = 0x2007c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2007c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi3phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi3phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi4phytimer_clk = {
+	.halt_reg = 0x2009c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2009c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi4phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi4phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi5phytimer_clk = {
+	.halt_reg = 0x200bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x200bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi5phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi5phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_clk = {
+	.halt_reg = 0x212bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x212bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_csiphy_rx_clk = {
+	.halt_reg = 0x20020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_csiphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy0_clk = {
+	.halt_reg = 0x2001c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2001c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy1_clk = {
+	.halt_reg = 0x20040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy2_clk = {
+	.halt_reg = 0x20060,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy3_clk = {
+	.halt_reg = 0x20080,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20080,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy4_clk = {
+	.halt_reg = 0x200a0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x200a0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy5_clk = {
+	.halt_reg = 0x200c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x200c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_0_ahb_clk = {
+	.halt_reg = 0x21248,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21248,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_0_clk = {
+	.halt_reg = 0x21210,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21210,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_icp_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_1_ahb_clk = {
+	.halt_reg = 0x2124c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2124c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_1_clk = {
+	.halt_reg = 0x21238,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21238,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_icp_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_ahb_clk = {
+	.halt_reg = 0x2119c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2119c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_clk = {
+	.halt_reg = 0x2115c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2115c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_cphy_rx_clk = {
+	.halt_reg = 0x21198,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21198,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_cphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_csid_clk = {
+	.halt_reg = 0x21188,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21188,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_ahb_clk = {
+	.halt_reg = 0x201f4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_clk = {
+	.halt_reg = 0x201b4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_nps_fast_ahb_clk = {
+	.halt_reg = 0x201f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_nps_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_pps_clk = {
+	.halt_reg = 0x201c8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_pps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_pps_fast_ahb_clk = {
+	.halt_reg = 0x201fc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201fc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_pps_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_scalers_clk = {
+	.halt_reg = 0x201d8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x201d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_scalers_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_nps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_scalers_fast_ahb_clk = {
+	.halt_reg = 0x20200,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20200,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_scalers_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_jpeg_clk = {
+	.halt_reg = 0x211dc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x211dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_jpeg_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_jpeg_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_ahb_clk = {
+	.halt_reg = 0x2011c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2011c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_anchor_clk = {
+	.halt_reg = 0x2014c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2014c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_anchor_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_anchor_fast_ahb_clk = {
+	.halt_reg = 0x200f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x200f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_anchor_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_hdr_clk = {
+	.halt_reg = 0x2015c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2015c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_hdr_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_hdr_fast_ahb_clk = {
+	.halt_reg = 0x200fc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x200fc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_hdr_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_main_clk = {
+	.halt_reg = 0x20138,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20138,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_main_fast_ahb_clk = {
+	.halt_reg = 0x200f4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x200f4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_mp_clk = {
+	.halt_reg = 0x2016c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2016c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_mp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ofe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ofe_mp_fast_ahb_clk = {
+	.halt_reg = 0x20100,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20100,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ofe_mp_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_clk = {
+	.halt_reg = 0x21328,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21328,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qdss_debug_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_xo_clk = {
+	.halt_reg = 0x2132c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2132c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sqpf_ahb_clk = {
+	.halt_reg = 0x20250,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20250,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sqpf_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sqpf_clk = {
+	.halt_reg = 0x20234,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20234,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sqpf_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_sqpf_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sqpf_fast_ahb_clk = {
+	.halt_reg = 0x20254,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x20254,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sqpf_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_bayer_clk = {
+	.halt_reg = 0x21044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_bayer_fast_ahb_clk = {
+	.halt_reg = 0x21060,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_bayer_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_main_clk = {
+	.halt_reg = 0x21030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_0_main_fast_ahb_clk = {
+	.halt_reg = 0x2105c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2105c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_0_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_bayer_clk = {
+	.halt_reg = 0x210c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x210c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_bayer_fast_ahb_clk = {
+	.halt_reg = 0x210dc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x210dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_bayer_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_main_clk = {
+	.halt_reg = 0x210ac,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x210ac,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_1_main_fast_ahb_clk = {
+	.halt_reg = 0x210d8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x210d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_1_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_bayer_clk = {
+	.halt_reg = 0x21124,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21124,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_bayer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_bayer_fast_ahb_clk = {
+	.halt_reg = 0x21140,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21140,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_bayer_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_main_clk = {
+	.halt_reg = 0x21110,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21110,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_tfe_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tfe_2_main_fast_ahb_clk = {
+	.halt_reg = 0x2113c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2113c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tfe_2_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_titan_mnoc_hf_clk = {
+	.halt_reg = 0x213a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x213a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_titan_mnoc_hf_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_titan_mnoc_sf_clk = {
+	.halt_reg = 0x213b4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x213b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_titan_mnoc_sf_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tracenoc_tpdm_1_cmb_clk = {
+	.halt_reg = 0x213dc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x213dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tracenoc_tpdm_1_cmb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc cam_cc_titan_top_cx_gdsc = {
+	.gdscr = 0x21388,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_titan_top_cx_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.needs_icc = true,
+	.icc_path_index = DT_ICC_TOP_GDSC,
+};
+
+static struct gdsc cam_cc_titan_top_gdsc = {
+	.gdscr = 0x21330,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_titan_top_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.needs_icc = true,
+	.icc_path_index = DT_ICC_TOP_GDSC,
+	.parent = &cam_cc_titan_top_cx_gdsc.pd,
+};
+
+static struct gdsc cam_cc_ipe_0_gdsc = {
+	.gdscr = 0x20188,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ipe_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+};
+
+static struct gdsc cam_cc_ofe_gdsc = {
+	.gdscr = 0x200c8,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ofe_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+};
+
+static struct gdsc cam_cc_sqpf_gdsc = {
+	.gdscr = 0x20208,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_sqpf_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+};
+
+static struct gdsc cam_cc_tfe_0_gdsc = {
+	.gdscr = 0x21004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_tfe_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+};
+
+static struct gdsc cam_cc_tfe_1_gdsc = {
+	.gdscr = 0x21080,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_tfe_1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+};
+
+static struct gdsc cam_cc_tfe_2_gdsc = {
+	.gdscr = 0x210e4,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_tfe_2_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.parent = &cam_cc_titan_top_gdsc.pd,
+};
+
+static struct clk_regmap *cam_cc_hawi_clocks[] = {
+	[CAM_CC_CAM_TOP_AHB_CLK] = &cam_cc_cam_top_ahb_clk.clkr,
+	[CAM_CC_CAM_TOP_FAST_AHB_CLK] = &cam_cc_cam_top_fast_ahb_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_AXI_CLK] = &cam_cc_camnoc_nrt_axi_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_CRE_CLK] = &cam_cc_camnoc_nrt_cre_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_IPE_NPS_CLK] = &cam_cc_camnoc_nrt_ipe_nps_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_IPE_SCALERS_CLK] = &cam_cc_camnoc_nrt_ipe_scalers_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_OFE_MAIN_CLK] = &cam_cc_camnoc_nrt_ofe_main_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_SQPF_CLK] = &cam_cc_camnoc_nrt_sqpf_clk.clkr,
+	[CAM_CC_CAMNOC_RT_AXI_CLK] = &cam_cc_camnoc_rt_axi_clk.clkr,
+	[CAM_CC_CAMNOC_RT_AXI_CLK_SRC] = &cam_cc_camnoc_rt_axi_clk_src.clkr,
+	[CAM_CC_CAMNOC_RT_IFE_LITE_CLK] = &cam_cc_camnoc_rt_ife_lite_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_0_MAIN_CLK] = &cam_cc_camnoc_rt_tfe_0_main_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_1_MAIN_CLK] = &cam_cc_camnoc_rt_tfe_1_main_clk.clkr,
+	[CAM_CC_CAMNOC_RT_TFE_2_MAIN_CLK] = &cam_cc_camnoc_rt_tfe_2_main_clk.clkr,
+	[CAM_CC_CCI_0_CLK] = &cam_cc_cci_0_clk.clkr,
+	[CAM_CC_CCI_0_CLK_SRC] = &cam_cc_cci_0_clk_src.clkr,
+	[CAM_CC_CCI_1_CLK] = &cam_cc_cci_1_clk.clkr,
+	[CAM_CC_CCI_1_CLK_SRC] = &cam_cc_cci_1_clk_src.clkr,
+	[CAM_CC_CCI_2_CLK] = &cam_cc_cci_2_clk.clkr,
+	[CAM_CC_CCI_2_CLK_SRC] = &cam_cc_cci_2_clk_src.clkr,
+	[CAM_CC_CORE_AHB_CLK] = &cam_cc_core_ahb_clk.clkr,
+	[CAM_CC_CPHY_RX_CLK_SRC] = &cam_cc_cphy_rx_clk_src.clkr,
+	[CAM_CC_CRE_AHB_CLK] = &cam_cc_cre_ahb_clk.clkr,
+	[CAM_CC_CRE_CLK] = &cam_cc_cre_clk.clkr,
+	[CAM_CC_CRE_CLK_SRC] = &cam_cc_cre_clk_src.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK] = &cam_cc_csi0phytimer_clk.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK_SRC] = &cam_cc_csi0phytimer_clk_src.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK] = &cam_cc_csi1phytimer_clk.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK_SRC] = &cam_cc_csi1phytimer_clk_src.clkr,
+	[CAM_CC_CSI2PHYTIMER_CLK] = &cam_cc_csi2phytimer_clk.clkr,
+	[CAM_CC_CSI2PHYTIMER_CLK_SRC] = &cam_cc_csi2phytimer_clk_src.clkr,
+	[CAM_CC_CSI3PHYTIMER_CLK] = &cam_cc_csi3phytimer_clk.clkr,
+	[CAM_CC_CSI3PHYTIMER_CLK_SRC] = &cam_cc_csi3phytimer_clk_src.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK] = &cam_cc_csi4phytimer_clk.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK_SRC] = &cam_cc_csi4phytimer_clk_src.clkr,
+	[CAM_CC_CSI5PHYTIMER_CLK] = &cam_cc_csi5phytimer_clk.clkr,
+	[CAM_CC_CSI5PHYTIMER_CLK_SRC] = &cam_cc_csi5phytimer_clk_src.clkr,
+	[CAM_CC_CSID_CLK] = &cam_cc_csid_clk.clkr,
+	[CAM_CC_CSID_CLK_SRC] = &cam_cc_csid_clk_src.clkr,
+	[CAM_CC_CSID_CSIPHY_RX_CLK] = &cam_cc_csid_csiphy_rx_clk.clkr,
+	[CAM_CC_CSIPHY0_CLK] = &cam_cc_csiphy0_clk.clkr,
+	[CAM_CC_CSIPHY1_CLK] = &cam_cc_csiphy1_clk.clkr,
+	[CAM_CC_CSIPHY2_CLK] = &cam_cc_csiphy2_clk.clkr,
+	[CAM_CC_CSIPHY3_CLK] = &cam_cc_csiphy3_clk.clkr,
+	[CAM_CC_CSIPHY4_CLK] = &cam_cc_csiphy4_clk.clkr,
+	[CAM_CC_CSIPHY5_CLK] = &cam_cc_csiphy5_clk.clkr,
+	[CAM_CC_FAST_AHB_CLK_SRC] = &cam_cc_fast_ahb_clk_src.clkr,
+	[CAM_CC_ICP_0_AHB_CLK] = &cam_cc_icp_0_ahb_clk.clkr,
+	[CAM_CC_ICP_0_CLK] = &cam_cc_icp_0_clk.clkr,
+	[CAM_CC_ICP_0_CLK_SRC] = &cam_cc_icp_0_clk_src.clkr,
+	[CAM_CC_ICP_1_AHB_CLK] = &cam_cc_icp_1_ahb_clk.clkr,
+	[CAM_CC_ICP_1_CLK] = &cam_cc_icp_1_clk.clkr,
+	[CAM_CC_ICP_1_CLK_SRC] = &cam_cc_icp_1_clk_src.clkr,
+	[CAM_CC_IFE_LITE_AHB_CLK] = &cam_cc_ife_lite_ahb_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK] = &cam_cc_ife_lite_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK_SRC] = &cam_cc_ife_lite_clk_src.clkr,
+	[CAM_CC_IFE_LITE_CPHY_RX_CLK] = &cam_cc_ife_lite_cphy_rx_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK] = &cam_cc_ife_lite_csid_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK_SRC] = &cam_cc_ife_lite_csid_clk_src.clkr,
+	[CAM_CC_IPE_NPS_AHB_CLK] = &cam_cc_ipe_nps_ahb_clk.clkr,
+	[CAM_CC_IPE_NPS_CLK] = &cam_cc_ipe_nps_clk.clkr,
+	[CAM_CC_IPE_NPS_CLK_SRC] = &cam_cc_ipe_nps_clk_src.clkr,
+	[CAM_CC_IPE_NPS_FAST_AHB_CLK] = &cam_cc_ipe_nps_fast_ahb_clk.clkr,
+	[CAM_CC_IPE_PPS_CLK] = &cam_cc_ipe_pps_clk.clkr,
+	[CAM_CC_IPE_PPS_FAST_AHB_CLK] = &cam_cc_ipe_pps_fast_ahb_clk.clkr,
+	[CAM_CC_IPE_SCALERS_CLK] = &cam_cc_ipe_scalers_clk.clkr,
+	[CAM_CC_IPE_SCALERS_FAST_AHB_CLK] = &cam_cc_ipe_scalers_fast_ahb_clk.clkr,
+	[CAM_CC_JPEG_CLK] = &cam_cc_jpeg_clk.clkr,
+	[CAM_CC_JPEG_CLK_SRC] = &cam_cc_jpeg_clk_src.clkr,
+	[CAM_CC_OFE_AHB_CLK] = &cam_cc_ofe_ahb_clk.clkr,
+	[CAM_CC_OFE_ANCHOR_CLK] = &cam_cc_ofe_anchor_clk.clkr,
+	[CAM_CC_OFE_ANCHOR_FAST_AHB_CLK] = &cam_cc_ofe_anchor_fast_ahb_clk.clkr,
+	[CAM_CC_OFE_CLK_SRC] = &cam_cc_ofe_clk_src.clkr,
+	[CAM_CC_OFE_HDR_CLK] = &cam_cc_ofe_hdr_clk.clkr,
+	[CAM_CC_OFE_HDR_FAST_AHB_CLK] = &cam_cc_ofe_hdr_fast_ahb_clk.clkr,
+	[CAM_CC_OFE_MAIN_CLK] = &cam_cc_ofe_main_clk.clkr,
+	[CAM_CC_OFE_MAIN_FAST_AHB_CLK] = &cam_cc_ofe_main_fast_ahb_clk.clkr,
+	[CAM_CC_OFE_MP_CLK] = &cam_cc_ofe_mp_clk.clkr,
+	[CAM_CC_OFE_MP_FAST_AHB_CLK] = &cam_cc_ofe_mp_fast_ahb_clk.clkr,
+	[CAM_CC_PLL0] = &cam_cc_pll0.clkr,
+	[CAM_CC_PLL0_OUT_EVEN] = &cam_cc_pll0_out_even.clkr,
+	[CAM_CC_PLL0_OUT_ODD] = &cam_cc_pll0_out_odd.clkr,
+	[CAM_CC_PLL1] = &cam_cc_pll1.clkr,
+	[CAM_CC_PLL1_OUT_EVEN] = &cam_cc_pll1_out_even.clkr,
+	[CAM_CC_PLL2] = &cam_cc_pll2.clkr,
+	[CAM_CC_PLL2_OUT_EVEN] = &cam_cc_pll2_out_even.clkr,
+	[CAM_CC_PLL3] = &cam_cc_pll3.clkr,
+	[CAM_CC_PLL3_OUT_EVEN] = &cam_cc_pll3_out_even.clkr,
+	[CAM_CC_PLL4] = &cam_cc_pll4.clkr,
+	[CAM_CC_PLL4_OUT_EVEN] = &cam_cc_pll4_out_even.clkr,
+	[CAM_CC_PLL5] = &cam_cc_pll5.clkr,
+	[CAM_CC_PLL5_OUT_EVEN] = &cam_cc_pll5_out_even.clkr,
+	[CAM_CC_PLL6] = &cam_cc_pll6.clkr,
+	[CAM_CC_PLL6_OUT_EVEN] = &cam_cc_pll6_out_even.clkr,
+	[CAM_CC_PLL6_OUT_ODD] = &cam_cc_pll6_out_odd.clkr,
+	[CAM_CC_PLL7] = &cam_cc_pll7.clkr,
+	[CAM_CC_PLL8] = &cam_cc_pll8.clkr,
+	[CAM_CC_PLL8_OUT_EVEN] = &cam_cc_pll8_out_even.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK] = &cam_cc_qdss_debug_clk.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK_SRC] = &cam_cc_qdss_debug_clk_src.clkr,
+	[CAM_CC_QDSS_DEBUG_XO_CLK] = &cam_cc_qdss_debug_xo_clk.clkr,
+	[CAM_CC_SLOW_AHB_CLK_SRC] = &cam_cc_slow_ahb_clk_src.clkr,
+	[CAM_CC_SQPF_AHB_CLK] = &cam_cc_sqpf_ahb_clk.clkr,
+	[CAM_CC_SQPF_CLK] = &cam_cc_sqpf_clk.clkr,
+	[CAM_CC_SQPF_CLK_SRC] = &cam_cc_sqpf_clk_src.clkr,
+	[CAM_CC_SQPF_FAST_AHB_CLK] = &cam_cc_sqpf_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_0_BAYER_CLK] = &cam_cc_tfe_0_bayer_clk.clkr,
+	[CAM_CC_TFE_0_BAYER_FAST_AHB_CLK] = &cam_cc_tfe_0_bayer_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_0_CLK_SRC] = &cam_cc_tfe_0_clk_src.clkr,
+	[CAM_CC_TFE_0_MAIN_CLK] = &cam_cc_tfe_0_main_clk.clkr,
+	[CAM_CC_TFE_0_MAIN_FAST_AHB_CLK] = &cam_cc_tfe_0_main_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_1_BAYER_CLK] = &cam_cc_tfe_1_bayer_clk.clkr,
+	[CAM_CC_TFE_1_BAYER_FAST_AHB_CLK] = &cam_cc_tfe_1_bayer_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_1_CLK_SRC] = &cam_cc_tfe_1_clk_src.clkr,
+	[CAM_CC_TFE_1_MAIN_CLK] = &cam_cc_tfe_1_main_clk.clkr,
+	[CAM_CC_TFE_1_MAIN_FAST_AHB_CLK] = &cam_cc_tfe_1_main_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_2_BAYER_CLK] = &cam_cc_tfe_2_bayer_clk.clkr,
+	[CAM_CC_TFE_2_BAYER_FAST_AHB_CLK] = &cam_cc_tfe_2_bayer_fast_ahb_clk.clkr,
+	[CAM_CC_TFE_2_CLK_SRC] = &cam_cc_tfe_2_clk_src.clkr,
+	[CAM_CC_TFE_2_MAIN_CLK] = &cam_cc_tfe_2_main_clk.clkr,
+	[CAM_CC_TFE_2_MAIN_FAST_AHB_CLK] = &cam_cc_tfe_2_main_fast_ahb_clk.clkr,
+	[CAM_CC_TITAN_MNOC_HF_CLK] = &cam_cc_titan_mnoc_hf_clk.clkr,
+	[CAM_CC_TITAN_MNOC_SF_CLK] = &cam_cc_titan_mnoc_sf_clk.clkr,
+	[CAM_CC_TRACENOC_TPDM_1_CMB_CLK] = &cam_cc_tracenoc_tpdm_1_cmb_clk.clkr,
+	[CAM_CC_XO_CLK_SRC] = &cam_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *cam_cc_hawi_gdscs[] = {
+	[CAM_CC_IPE_0_GDSC] = &cam_cc_ipe_0_gdsc,
+	[CAM_CC_OFE_GDSC] = &cam_cc_ofe_gdsc,
+	[CAM_CC_SQPF_GDSC] = &cam_cc_sqpf_gdsc,
+	[CAM_CC_TFE_0_GDSC] = &cam_cc_tfe_0_gdsc,
+	[CAM_CC_TFE_1_GDSC] = &cam_cc_tfe_1_gdsc,
+	[CAM_CC_TFE_2_GDSC] = &cam_cc_tfe_2_gdsc,
+	[CAM_CC_TITAN_TOP_CX_GDSC] = &cam_cc_titan_top_cx_gdsc,
+	[CAM_CC_TITAN_TOP_GDSC] = &cam_cc_titan_top_gdsc,
+};
+
+static const struct qcom_reset_map cam_cc_hawi_resets[] = {
+	[CAM_CC_DRV_BCR] = { 0x213d4 },
+	[CAM_CC_ICP_BCR] = { 0x211f4 },
+	[CAM_CC_IPE_0_BCR] = { 0x20184 },
+	[CAM_CC_OFE_BCR] = { 0x200c4 },
+	[CAM_CC_QDSS_DEBUG_BCR] = { 0x2130c },
+	[CAM_CC_SQPF_BCR] = { 0x20204 },
+	[CAM_CC_TFE_0_BCR] = { 0x21000 },
+	[CAM_CC_TFE_1_BCR] = { 0x2107c },
+	[CAM_CC_TFE_2_BCR] = { 0x210e0 },
+};
+
+static struct clk_alpha_pll *cam_cc_hawi_plls[] = {
+	&cam_cc_pll0,
+	&cam_cc_pll1,
+	&cam_cc_pll2,
+	&cam_cc_pll3,
+	&cam_cc_pll4,
+	&cam_cc_pll5,
+	&cam_cc_pll6,
+	&cam_cc_pll7,
+	&cam_cc_pll8,
+};
+
+static const u32 cam_cc_hawi_critical_cbcrs[] = {
+	0x21308, /* CAM_CC_CAMNOC_DCD_XO_CLK */
+	0x2139c, /* CAM_CC_CX_GDSC_CLK */
+	0x213e0, /* CAM_CC_DRV_AHB_CLK */
+	0x213d8, /* CAM_CC_DRV_XO_CLK */
+	0x21360, /* CAM_CC_GDSC_CLK */
+	0x213d0, /* CAM_CC_MMSS_AHB_CLK */
+	0x21364, /* CAM_CC_SLEEP_CLK */
+	0x213a0, /* CAM_CC_TITAN_CAMNOC_XO_CLK */
+};
+
+static const struct regmap_config cam_cc_hawi_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x26024,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_driver_data cam_cc_hawi_driver_data = {
+	.alpha_plls = cam_cc_hawi_plls,
+	.num_alpha_plls = ARRAY_SIZE(cam_cc_hawi_plls),
+	.clk_cbcrs = cam_cc_hawi_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_hawi_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc cam_cc_hawi_desc = {
+	.config = &cam_cc_hawi_regmap_config,
+	.clks = cam_cc_hawi_clocks,
+	.num_clks = ARRAY_SIZE(cam_cc_hawi_clocks),
+	.resets = cam_cc_hawi_resets,
+	.num_resets = ARRAY_SIZE(cam_cc_hawi_resets),
+	.gdscs = cam_cc_hawi_gdscs,
+	.num_gdscs = ARRAY_SIZE(cam_cc_hawi_gdscs),
+	.use_rpm = true,
+	.driver_data = &cam_cc_hawi_driver_data,
+};
+
+static const struct of_device_id cam_cc_hawi_match_table[] = {
+	{ .compatible = "qcom,hawi-camcc" },
+	{ .compatible = "qcom,maili-camcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, cam_cc_hawi_match_table);
+
+static int cam_cc_hawi_probe(struct platform_device *pdev)
+{
+	if (device_is_compatible(&pdev->dev, "qcom,maili-camcc")) {
+		cam_cc_pll0.config = &cam_cc_pll0_config_maili;
+		cam_cc_pll1.config = &cam_cc_pll1_config_maili;
+		cam_cc_pll2.config = &cam_cc_pll2_config_maili;
+		cam_cc_pll3.config = &cam_cc_pll3_config_maili;
+		cam_cc_pll4.config = &cam_cc_pll4_config_maili;
+		cam_cc_pll5.config = &cam_cc_pll5_config_maili;
+		cam_cc_pll6.config = &cam_cc_pll6_config_maili;
+		cam_cc_pll7.config = &cam_cc_pll7_config_maili;
+		cam_cc_pll8.config = &cam_cc_pll8_config_maili;
+
+		cam_cc_camnoc_rt_axi_clk_src.freq_tbl = ftbl_cam_cc_camnoc_rt_axi_clk_src_maili;
+		cam_cc_cci_0_clk_src.freq_tbl = ftbl_cam_cc_cci_0_clk_src_maili;
+		cam_cc_cci_1_clk_src.freq_tbl = ftbl_cam_cc_cci_0_clk_src_maili;
+		cam_cc_ofe_clk_src.freq_tbl = ftbl_cam_cc_ofe_clk_src_maili;
+
+		cam_cc_hawi_clocks[CAM_CC_CCI_2_CLK_SRC] = NULL;
+		cam_cc_hawi_clocks[CAM_CC_CCI_2_CLK] = NULL;
+		cam_cc_hawi_clocks[CAM_CC_CSI5PHYTIMER_CLK_SRC] = NULL;
+		cam_cc_hawi_clocks[CAM_CC_CSI5PHYTIMER_CLK] = NULL;
+		cam_cc_hawi_clocks[CAM_CC_CSIPHY5_CLK] = NULL;
+	}
+
+	return qcom_cc_probe(pdev, &cam_cc_hawi_desc);
+}
+
+static struct platform_driver cam_cc_hawi_driver = {
+	.probe = cam_cc_hawi_probe,
+	.driver = {
+		.name = "camcc-hawi",
+		.of_match_table = cam_cc_hawi_match_table,
+	},
+};
+
+module_platform_driver(cam_cc_hawi_driver);
+
+MODULE_DESCRIPTION("QTI CAMCC Hawi Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/camcc-nord.c b/drivers/clk/qcom/camcc-nord.c
new file mode 100644
index 00000000000000..9e3c40cb3ad5f0
--- /dev/null
+++ b/drivers/clk/qcom/camcc-nord.c
@@ -0,0 +1,2941 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,nord-camcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_IFACE,
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK
+};
+
+enum {
+	P_BI_TCXO,
+	P_CAM_CC_PLL0_OUT_EVEN,
+	P_CAM_CC_PLL0_OUT_MAIN,
+	P_CAM_CC_PLL0_OUT_ODD,
+	P_CAM_CC_PLL2_OUT_EVEN,
+	P_CAM_CC_PLL3_OUT_EVEN,
+	P_CAM_CC_PLL4_OUT_EVEN,
+	P_CAM_CC_PLL5_OUT_EVEN,
+	P_CAM_CC_PLL6_OUT_EVEN,
+	P_SLEEP_CLK,
+};
+
+static const struct pll_vco lucid_ole_vco[] = {
+	{ 249600000, 2300000000, 0 },
+};
+
+/* 1200.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll0_config = {
+	.l = 0x3e,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00008400,
+	.user_ctl_hi_val = 0x00400005,
+};
+
+static struct clk_alpha_pll cam_cc_pll0 = {
+	.offset = 0x0,
+	.config = &cam_cc_pll0_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO_AO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll0_out_odd[] = {
+	{ 0x2, 3 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll0_out_odd = {
+	.offset = 0x0,
+	.post_div_shift = 14,
+	.post_div_table = post_div_table_cam_cc_pll0_out_odd,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll0_out_odd),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll0_out_odd",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+/* 720.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll2_config = {
+	.l = 0x25,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00400005,
+};
+
+static struct clk_alpha_pll cam_cc_pll2 = {
+	.offset = 0x2000,
+	.config = &cam_cc_pll2_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll2",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll2_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll2_out_even = {
+	.offset = 0x2000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll2_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll2_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll2_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll2.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+/* 720.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll3_config = {
+	.l = 0x25,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00400005,
+};
+
+static struct clk_alpha_pll cam_cc_pll3 = {
+	.offset = 0x3000,
+	.config = &cam_cc_pll3_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll3",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll3_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll3_out_even = {
+	.offset = 0x3000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll3_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll3_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll3_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll3.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+/* 720.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll4_config = {
+	.l = 0x25,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00400005,
+};
+
+static struct clk_alpha_pll cam_cc_pll4 = {
+	.offset = 0x4000,
+	.config = &cam_cc_pll4_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll4_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll4_out_even = {
+	.offset = 0x4000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll4_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll4_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll4_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll4.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+/* 720.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll5_config = {
+	.l = 0x25,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00400005,
+};
+
+static struct clk_alpha_pll cam_cc_pll5 = {
+	.offset = 0x5000,
+	.config = &cam_cc_pll5_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll5",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll5_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll5_out_even = {
+	.offset = 0x5000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll5_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll5_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll5_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll5.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+/* 720.0 MHz Configuration */
+static const struct alpha_pll_config cam_cc_pll6_config = {
+	.l = 0x25,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000400,
+	.user_ctl_hi_val = 0x00400005,
+};
+
+static struct clk_alpha_pll cam_cc_pll6 = {
+	.offset = 0x6000,
+	.config = &cam_cc_pll6_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_pll6",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_cam_cc_pll6_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv cam_cc_pll6_out_even = {
+	.offset = 0x6000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_cam_cc_pll6_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_cam_cc_pll6_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_pll6_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_pll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+static const struct parent_map cam_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL0_OUT_ODD, 3 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_odd.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+	{ P_CAM_CC_PLL3_OUT_EVEN, 3 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+	{ .hw = &cam_cc_pll3_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL2_OUT_EVEN, 2 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll2_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL4_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll4_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL5_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll5_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_6[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL6_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_6[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll6_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_7[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_CAM_CC_PLL0_OUT_MAIN, 1 },
+	{ P_CAM_CC_PLL0_OUT_EVEN, 2 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_7[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &cam_cc_pll0.clkr.hw },
+	{ .hw = &cam_cc_pll0_out_even.clkr.hw },
+};
+
+static const struct parent_map cam_cc_parent_map_8[] = {
+	{ P_SLEEP_CLK, 0 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_8[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map cam_cc_parent_map_9[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data cam_cc_parent_data_9[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct freq_tbl ftbl_cam_cc_camnoc_rt_axi_clk_src[] = {
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_camnoc_rt_axi_clk_src = {
+	.cmd_rcgr = 0x13244,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_camnoc_rt_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_camnoc_rt_axi_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cci_0_clk_src[] = {
+	F(37500000, P_CAM_CC_PLL0_OUT_EVEN, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cci_0_clk_src = {
+	.cmd_rcgr = 0x13110,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_0_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_1_clk_src = {
+	.cmd_rcgr = 0x13130,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_1_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_2_clk_src = {
+	.cmd_rcgr = 0x13150,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_2_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_3_clk_src = {
+	.cmd_rcgr = 0x13170,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_3_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_cci_4_clk_src = {
+	.cmd_rcgr = 0x13190,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cci_4_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_cphy_rx_clk_src[] = {
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_cphy_rx_clk_src = {
+	.cmd_rcgr = 0x120ac,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_cphy_rx_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_cphy_rx_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_csi0phytimer_clk_src[] = {
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_ODD, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x10000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi0phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x10028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi1phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi2phytimer_clk_src = {
+	.cmd_rcgr = 0x1004c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi2phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi3phytimer_clk_src = {
+	.cmd_rcgr = 0x10070,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi3phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_csi4phytimer_clk_src = {
+	.cmd_rcgr = 0x10094,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csi0phytimer_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csi4phytimer_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_csid_clk_src[] = {
+	F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	F(480000000, P_CAM_CC_PLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_csid_clk_src = {
+	.cmd_rcgr = 0x13214,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csid_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_csid_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_fast_ahb_clk_src[] = {
+	F(200000000, P_CAM_CC_PLL0_OUT_EVEN, 3, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_MAIN, 4, 0, 0),
+	F(400000000, P_CAM_CC_PLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_fast_ahb_clk_src = {
+	.cmd_rcgr = 0x131dc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_fast_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_fast_ahb_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_icp_0_clk_src[] = {
+	F(360000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	F(600000000, P_CAM_CC_PLL3_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_icp_0_clk_src = {
+	.cmd_rcgr = 0x130a4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_2,
+	.freq_tbl = ftbl_cam_cc_icp_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_icp_0_clk_src",
+		.parent_data = cam_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_icp_1_clk_src = {
+	.cmd_rcgr = 0x130dc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_2,
+	.freq_tbl = ftbl_cam_cc_icp_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_icp_1_clk_src",
+		.parent_data = cam_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ife_0_main_clk_src[] = {
+	F(360000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	F(650000000, P_CAM_CC_PLL4_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ife_0_main_clk_src = {
+	.cmd_rcgr = 0x12018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_4,
+	.freq_tbl = ftbl_cam_cc_ife_0_main_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_0_main_clk_src",
+		.parent_data = cam_cc_parent_data_4,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ife_1_main_clk_src[] = {
+	F(360000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	F(650000000, P_CAM_CC_PLL5_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ife_1_main_clk_src = {
+	.cmd_rcgr = 0x120dc,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_5,
+	.freq_tbl = ftbl_cam_cc_ife_1_main_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_1_main_clk_src",
+		.parent_data = cam_cc_parent_data_5,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ife_2_main_clk_src[] = {
+	F(360000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0),
+	F(650000000, P_CAM_CC_PLL6_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ife_2_main_clk_src = {
+	.cmd_rcgr = 0x12188,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_6,
+	.freq_tbl = ftbl_cam_cc_ife_2_main_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_2_main_clk_src",
+		.parent_data = cam_cc_parent_data_6,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_6),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_clk_src = {
+	.cmd_rcgr = 0x13000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csid_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_lite_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_ife_lite_csid_clk_src = {
+	.cmd_rcgr = 0x13024,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_1,
+	.freq_tbl = ftbl_cam_cc_csid_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ife_lite_csid_clk_src",
+		.parent_data = cam_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_ipe_0_clk_src[] = {
+	F(360000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(480000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	F(650000000, P_CAM_CC_PLL2_OUT_EVEN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_ipe_0_clk_src = {
+	.cmd_rcgr = 0x11018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_3,
+	.freq_tbl = ftbl_cam_cc_ipe_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ipe_0_clk_src",
+		.parent_data = cam_cc_parent_data_3,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_ipe_1_clk_src = {
+	.cmd_rcgr = 0x11074,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_3,
+	.freq_tbl = ftbl_cam_cc_ipe_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_ipe_1_clk_src",
+		.parent_data = cam_cc_parent_data_3,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_qdss_debug_clk_src[] = {
+	F(200000000, P_CAM_CC_PLL0_OUT_EVEN, 3, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_qdss_debug_clk_src = {
+	.cmd_rcgr = 0x13330,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_qdss_debug_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_qdss_debug_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_qup_ahbm_clk_src[] = {
+	F(150000000, P_CAM_CC_PLL0_OUT_EVEN, 4, 0, 0),
+	F(240000000, P_CAM_CC_PLL0_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_qup_ahbm_clk_src = {
+	.cmd_rcgr = 0x132e8,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_qup_ahbm_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_qup_ahbm_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_qup_core_2x_clk_src[] = {
+	F(200000000, P_CAM_CC_PLL0_OUT_ODD, 2, 0, 0),
+	F(300000000, P_CAM_CC_PLL0_OUT_EVEN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_qup_core_2x_clk_src = {
+	.cmd_rcgr = 0x132a0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_qup_core_2x_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_qup_core_2x_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 cam_cc_qup_se_clk_src = {
+	.cmd_rcgr = 0x13308,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_7,
+	.freq_tbl = ftbl_cam_cc_cci_0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_qup_se_clk_src",
+		.parent_data = cam_cc_parent_data_7,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_7),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_sleep_clk_src = {
+	.cmd_rcgr = 0x13384,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_8,
+	.freq_tbl = ftbl_cam_cc_sleep_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_sleep_clk_src",
+		.parent_data = cam_cc_parent_data_8,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_8),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_slow_ahb_clk_src[] = {
+	F(80000000, P_CAM_CC_PLL0_OUT_EVEN, 7.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_slow_ahb_clk_src = {
+	.cmd_rcgr = 0x131f8,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_0,
+	.freq_tbl = ftbl_cam_cc_slow_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_slow_ahb_clk_src",
+		.parent_data = cam_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_cam_cc_xo_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cam_cc_xo_clk_src = {
+	.cmd_rcgr = 0x13368,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = cam_cc_parent_map_9,
+	.freq_tbl = ftbl_cam_cc_xo_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_xo_clk_src",
+		.parent_data = cam_cc_parent_data_9,
+		.num_parents = ARRAY_SIZE(cam_cc_parent_data_9),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div cam_cc_qup_core_2x_div_clk_src = {
+	.reg = 0x132cc,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "cam_cc_qup_core_2x_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&cam_cc_qup_core_2x_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_dcd_xo_clk = {
+	.halt_reg = 0x13290,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13290,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_dcd_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_nrt_axi_clk = {
+	.halt_reg = 0x13278,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13278,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_nrt_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_rt_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_camnoc_rt_axi_clk = {
+	.halt_reg = 0x13260,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13260,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_camnoc_rt_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_camnoc_rt_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_0_clk = {
+	.halt_reg = 0x1312c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1312c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_1_clk = {
+	.halt_reg = 0x1314c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1314c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_2_clk = {
+	.halt_reg = 0x1316c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1316c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_3_clk = {
+	.halt_reg = 0x1318c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1318c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_cci_4_clk = {
+	.halt_reg = 0x131ac,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x131ac,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_cci_4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cci_4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ccu_fast_ahb_clk = {
+	.halt_reg = 0x1329c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1329c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ccu_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi0phytimer_clk = {
+	.halt_reg = 0x1001c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1001c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi0phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi0phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi1phytimer_clk = {
+	.halt_reg = 0x10044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi1phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi1phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi2phytimer_clk = {
+	.halt_reg = 0x10068,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi2phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi2phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi3phytimer_clk = {
+	.halt_reg = 0x1008c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1008c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi3phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi3phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csi4phytimer_clk = {
+	.halt_reg = 0x100b0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csi4phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csi4phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_clk = {
+	.halt_reg = 0x13230,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13230,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csid_csiphy_rx_clk = {
+	.halt_reg = 0x10024,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csid_csiphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy0_clk = {
+	.halt_reg = 0x10020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy1_clk = {
+	.halt_reg = 0x10048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy2_clk = {
+	.halt_reg = 0x1006c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1006c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy3_clk = {
+	.halt_reg = 0x10090,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_csiphy4_clk = {
+	.halt_reg = 0x100b4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_csiphy4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_0_ahb_clk = {
+	.halt_reg = 0x130d4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x130d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_0_clk = {
+	.halt_reg = 0x130c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x130c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_icp_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_1_ahb_clk = {
+	.halt_reg = 0x1310c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1310c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_icp_1_clk = {
+	.halt_reg = 0x130f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x130f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_icp_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_icp_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_main_clk = {
+	.halt_reg = 0x12034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_main_fast_ahb_clk = {
+	.halt_reg = 0x12054,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_pcp_clk = {
+	.halt_reg = 0x12058,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12058,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_pcp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_pcp_fast_ahb_clk = {
+	.halt_reg = 0x12070,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12070,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_pcp_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_scalar_clk = {
+	.halt_reg = 0x12074,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12074,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_scalar_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_scalar_fast_ahb_clk = {
+	.halt_reg = 0x1208c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1208c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_scalar_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_tmc_clk = {
+	.halt_reg = 0x12090,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_tmc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_0_tmc_fast_ahb_clk = {
+	.halt_reg = 0x120a8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x120a8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_0_tmc_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_main_clk = {
+	.halt_reg = 0x120f8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x120f8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_main_fast_ahb_clk = {
+	.halt_reg = 0x12118,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12118,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_pcp_clk = {
+	.halt_reg = 0x1211c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1211c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_pcp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_pcp_fast_ahb_clk = {
+	.halt_reg = 0x12134,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12134,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_pcp_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_scalar_clk = {
+	.halt_reg = 0x12138,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12138,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_scalar_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_scalar_fast_ahb_clk = {
+	.halt_reg = 0x12150,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12150,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_scalar_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_tmc_clk = {
+	.halt_reg = 0x12154,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12154,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_tmc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_1_tmc_fast_ahb_clk = {
+	.halt_reg = 0x1216c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1216c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_1_tmc_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_main_clk = {
+	.halt_reg = 0x121a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x121a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_main_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_2_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_main_fast_ahb_clk = {
+	.halt_reg = 0x121c4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x121c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_main_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_pcp_clk = {
+	.halt_reg = 0x121c8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x121c8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_pcp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_2_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_pcp_fast_ahb_clk = {
+	.halt_reg = 0x121e0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x121e0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_pcp_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_scalar_clk = {
+	.halt_reg = 0x121e4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x121e4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_scalar_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_2_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_scalar_fast_ahb_clk = {
+	.halt_reg = 0x121fc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x121fc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_scalar_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_tmc_clk = {
+	.halt_reg = 0x12200,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12200,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_tmc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_2_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_2_tmc_fast_ahb_clk = {
+	.halt_reg = 0x12218,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12218,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_2_tmc_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_ahb_clk = {
+	.halt_reg = 0x13054,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_clk = {
+	.halt_reg = 0x1301c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1301c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_cphy_rx_clk = {
+	.halt_reg = 0x13050,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_cphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ife_lite_csid_clk = {
+	.halt_reg = 0x1303c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1303c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ife_lite_csid_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_csid_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_0_ahb_clk = {
+	.halt_reg = 0x11054,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_0_clk = {
+	.halt_reg = 0x11034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_0_fast_ahb_clk = {
+	.halt_reg = 0x11058,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11058,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_0_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_1_ahb_clk = {
+	.halt_reg = 0x110b0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_1_clk = {
+	.halt_reg = 0x11090,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_ipe_1_fast_ahb_clk = {
+	.halt_reg = 0x110b4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110b4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_ipe_1_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_clk = {
+	.halt_reg = 0x13348,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13348,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qdss_debug_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qdss_debug_xo_clk = {
+	.halt_reg = 0x1334c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1334c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qdss_debug_xo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qup_ahbm_clk = {
+	.halt_reg = 0x13300,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13300,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qup_ahbm_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qup_ahbm_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qup_core_2x_clk = {
+	.halt_reg = 0x132b8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x132b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qup_core_2x_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qup_core_2x_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qup_core_clk = {
+	.halt_reg = 0x132d0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x132d0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qup_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qup_core_2x_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_qup_se_clk = {
+	.halt_reg = 0x13320,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13320,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_qup_se_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qup_se_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sfe_lite_0_clk = {
+	.halt_reg = 0x1305c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1305c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sfe_lite_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sfe_lite_0_fast_ahb_clk = {
+	.halt_reg = 0x1306c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1306c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sfe_lite_0_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sfe_lite_1_clk = {
+	.halt_reg = 0x13074,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13074,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sfe_lite_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sfe_lite_1_fast_ahb_clk = {
+	.halt_reg = 0x13084,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13084,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sfe_lite_1_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sfe_lite_2_clk = {
+	.halt_reg = 0x1308c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1308c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sfe_lite_2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_2_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sfe_lite_2_fast_ahb_clk = {
+	.halt_reg = 0x1309c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1309c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sfe_lite_2_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_sm_obs_clk = {
+	.halt_reg = 0x1401c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1401c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_sm_obs_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_ahb_clk = {
+	.halt_reg = 0x131b0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x131b0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_slow_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_fast_ahb_clk = {
+	.halt_reg = 0x131c4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x131c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_fast_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_fast_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_ife_0_clk = {
+	.halt_reg = 0x12048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x12048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_ife_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_ife_1_clk = {
+	.halt_reg = 0x1210c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1210c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_ife_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_ife_2_clk = {
+	.halt_reg = 0x121b8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x121b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_ife_2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_2_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_ife_lite_clk = {
+	.halt_reg = 0x13020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_ife_lite_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_lite_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_ipe_0_clk = {
+	.halt_reg = 0x11048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_ipe_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_ipe_1_clk = {
+	.halt_reg = 0x110a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x110a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_ipe_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ipe_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_qup_ahbm_clk = {
+	.halt_reg = 0x13304,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13304,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_qup_ahbm_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_qup_ahbm_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_sfe_lite_0_clk = {
+	.halt_reg = 0x13060,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_sfe_lite_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_0_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_sfe_lite_1_clk = {
+	.halt_reg = 0x13078,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13078,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_sfe_lite_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_1_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_top_sfe_lite_2_clk = {
+	.halt_reg = 0x13090,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x13090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_top_sfe_lite_2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_ife_2_main_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch cam_cc_tpg_csiphy_rx_clk = {
+	.halt_reg = 0x100b8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x100b8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "cam_cc_tpg_csiphy_rx_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&cam_cc_cphy_rx_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc cam_cc_titan_top_gdsc = {
+	.gdscr = 0x13350,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_titan_top_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ife_0_gdsc = {
+	.gdscr = 0x12004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ife_0_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ife_1_gdsc = {
+	.gdscr = 0x120c8,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ife_1_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ife_2_gdsc = {
+	.gdscr = 0x12174,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ife_2_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ipe_0_gdsc = {
+	.gdscr = 0x11004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ipe_0_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc cam_cc_ipe_1_gdsc = {
+	.gdscr = 0x11060,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "cam_cc_ipe_1_gdsc",
+	},
+	.parent = &cam_cc_titan_top_gdsc.pd,
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = HW_CTRL_TRIGGER | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *cam_cc_nord_clocks[] = {
+	[CAM_CC_CAMNOC_DCD_XO_CLK] = &cam_cc_camnoc_dcd_xo_clk.clkr,
+	[CAM_CC_CAMNOC_NRT_AXI_CLK] = &cam_cc_camnoc_nrt_axi_clk.clkr,
+	[CAM_CC_CAMNOC_RT_AXI_CLK] = &cam_cc_camnoc_rt_axi_clk.clkr,
+	[CAM_CC_CAMNOC_RT_AXI_CLK_SRC] = &cam_cc_camnoc_rt_axi_clk_src.clkr,
+	[CAM_CC_CCI_0_CLK] = &cam_cc_cci_0_clk.clkr,
+	[CAM_CC_CCI_0_CLK_SRC] = &cam_cc_cci_0_clk_src.clkr,
+	[CAM_CC_CCI_1_CLK] = &cam_cc_cci_1_clk.clkr,
+	[CAM_CC_CCI_1_CLK_SRC] = &cam_cc_cci_1_clk_src.clkr,
+	[CAM_CC_CCI_2_CLK] = &cam_cc_cci_2_clk.clkr,
+	[CAM_CC_CCI_2_CLK_SRC] = &cam_cc_cci_2_clk_src.clkr,
+	[CAM_CC_CCI_3_CLK] = &cam_cc_cci_3_clk.clkr,
+	[CAM_CC_CCI_3_CLK_SRC] = &cam_cc_cci_3_clk_src.clkr,
+	[CAM_CC_CCI_4_CLK] = &cam_cc_cci_4_clk.clkr,
+	[CAM_CC_CCI_4_CLK_SRC] = &cam_cc_cci_4_clk_src.clkr,
+	[CAM_CC_CCU_FAST_AHB_CLK] = &cam_cc_ccu_fast_ahb_clk.clkr,
+	[CAM_CC_CPHY_RX_CLK_SRC] = &cam_cc_cphy_rx_clk_src.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK] = &cam_cc_csi0phytimer_clk.clkr,
+	[CAM_CC_CSI0PHYTIMER_CLK_SRC] = &cam_cc_csi0phytimer_clk_src.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK] = &cam_cc_csi1phytimer_clk.clkr,
+	[CAM_CC_CSI1PHYTIMER_CLK_SRC] = &cam_cc_csi1phytimer_clk_src.clkr,
+	[CAM_CC_CSI2PHYTIMER_CLK] = &cam_cc_csi2phytimer_clk.clkr,
+	[CAM_CC_CSI2PHYTIMER_CLK_SRC] = &cam_cc_csi2phytimer_clk_src.clkr,
+	[CAM_CC_CSI3PHYTIMER_CLK] = &cam_cc_csi3phytimer_clk.clkr,
+	[CAM_CC_CSI3PHYTIMER_CLK_SRC] = &cam_cc_csi3phytimer_clk_src.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK] = &cam_cc_csi4phytimer_clk.clkr,
+	[CAM_CC_CSI4PHYTIMER_CLK_SRC] = &cam_cc_csi4phytimer_clk_src.clkr,
+	[CAM_CC_CSID_CLK] = &cam_cc_csid_clk.clkr,
+	[CAM_CC_CSID_CLK_SRC] = &cam_cc_csid_clk_src.clkr,
+	[CAM_CC_CSID_CSIPHY_RX_CLK] = &cam_cc_csid_csiphy_rx_clk.clkr,
+	[CAM_CC_CSIPHY0_CLK] = &cam_cc_csiphy0_clk.clkr,
+	[CAM_CC_CSIPHY1_CLK] = &cam_cc_csiphy1_clk.clkr,
+	[CAM_CC_CSIPHY2_CLK] = &cam_cc_csiphy2_clk.clkr,
+	[CAM_CC_CSIPHY3_CLK] = &cam_cc_csiphy3_clk.clkr,
+	[CAM_CC_CSIPHY4_CLK] = &cam_cc_csiphy4_clk.clkr,
+	[CAM_CC_FAST_AHB_CLK_SRC] = &cam_cc_fast_ahb_clk_src.clkr,
+	[CAM_CC_ICP_0_AHB_CLK] = &cam_cc_icp_0_ahb_clk.clkr,
+	[CAM_CC_ICP_0_CLK] = &cam_cc_icp_0_clk.clkr,
+	[CAM_CC_ICP_0_CLK_SRC] = &cam_cc_icp_0_clk_src.clkr,
+	[CAM_CC_ICP_1_AHB_CLK] = &cam_cc_icp_1_ahb_clk.clkr,
+	[CAM_CC_ICP_1_CLK] = &cam_cc_icp_1_clk.clkr,
+	[CAM_CC_ICP_1_CLK_SRC] = &cam_cc_icp_1_clk_src.clkr,
+	[CAM_CC_IFE_0_MAIN_CLK] = &cam_cc_ife_0_main_clk.clkr,
+	[CAM_CC_IFE_0_MAIN_CLK_SRC] = &cam_cc_ife_0_main_clk_src.clkr,
+	[CAM_CC_IFE_0_MAIN_FAST_AHB_CLK] = &cam_cc_ife_0_main_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_0_PCP_CLK] = &cam_cc_ife_0_pcp_clk.clkr,
+	[CAM_CC_IFE_0_PCP_FAST_AHB_CLK] = &cam_cc_ife_0_pcp_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_0_SCALAR_CLK] = &cam_cc_ife_0_scalar_clk.clkr,
+	[CAM_CC_IFE_0_SCALAR_FAST_AHB_CLK] = &cam_cc_ife_0_scalar_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_0_TMC_CLK] = &cam_cc_ife_0_tmc_clk.clkr,
+	[CAM_CC_IFE_0_TMC_FAST_AHB_CLK] = &cam_cc_ife_0_tmc_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_1_MAIN_CLK] = &cam_cc_ife_1_main_clk.clkr,
+	[CAM_CC_IFE_1_MAIN_CLK_SRC] = &cam_cc_ife_1_main_clk_src.clkr,
+	[CAM_CC_IFE_1_MAIN_FAST_AHB_CLK] = &cam_cc_ife_1_main_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_1_PCP_CLK] = &cam_cc_ife_1_pcp_clk.clkr,
+	[CAM_CC_IFE_1_PCP_FAST_AHB_CLK] = &cam_cc_ife_1_pcp_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_1_SCALAR_CLK] = &cam_cc_ife_1_scalar_clk.clkr,
+	[CAM_CC_IFE_1_SCALAR_FAST_AHB_CLK] = &cam_cc_ife_1_scalar_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_1_TMC_CLK] = &cam_cc_ife_1_tmc_clk.clkr,
+	[CAM_CC_IFE_1_TMC_FAST_AHB_CLK] = &cam_cc_ife_1_tmc_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_2_MAIN_CLK] = &cam_cc_ife_2_main_clk.clkr,
+	[CAM_CC_IFE_2_MAIN_CLK_SRC] = &cam_cc_ife_2_main_clk_src.clkr,
+	[CAM_CC_IFE_2_MAIN_FAST_AHB_CLK] = &cam_cc_ife_2_main_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_2_PCP_CLK] = &cam_cc_ife_2_pcp_clk.clkr,
+	[CAM_CC_IFE_2_PCP_FAST_AHB_CLK] = &cam_cc_ife_2_pcp_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_2_SCALAR_CLK] = &cam_cc_ife_2_scalar_clk.clkr,
+	[CAM_CC_IFE_2_SCALAR_FAST_AHB_CLK] = &cam_cc_ife_2_scalar_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_2_TMC_CLK] = &cam_cc_ife_2_tmc_clk.clkr,
+	[CAM_CC_IFE_2_TMC_FAST_AHB_CLK] = &cam_cc_ife_2_tmc_fast_ahb_clk.clkr,
+	[CAM_CC_IFE_LITE_AHB_CLK] = &cam_cc_ife_lite_ahb_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK] = &cam_cc_ife_lite_clk.clkr,
+	[CAM_CC_IFE_LITE_CLK_SRC] = &cam_cc_ife_lite_clk_src.clkr,
+	[CAM_CC_IFE_LITE_CPHY_RX_CLK] = &cam_cc_ife_lite_cphy_rx_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK] = &cam_cc_ife_lite_csid_clk.clkr,
+	[CAM_CC_IFE_LITE_CSID_CLK_SRC] = &cam_cc_ife_lite_csid_clk_src.clkr,
+	[CAM_CC_IPE_0_AHB_CLK] = &cam_cc_ipe_0_ahb_clk.clkr,
+	[CAM_CC_IPE_0_CLK] = &cam_cc_ipe_0_clk.clkr,
+	[CAM_CC_IPE_0_CLK_SRC] = &cam_cc_ipe_0_clk_src.clkr,
+	[CAM_CC_IPE_0_FAST_AHB_CLK] = &cam_cc_ipe_0_fast_ahb_clk.clkr,
+	[CAM_CC_IPE_1_AHB_CLK] = &cam_cc_ipe_1_ahb_clk.clkr,
+	[CAM_CC_IPE_1_CLK] = &cam_cc_ipe_1_clk.clkr,
+	[CAM_CC_IPE_1_CLK_SRC] = &cam_cc_ipe_1_clk_src.clkr,
+	[CAM_CC_IPE_1_FAST_AHB_CLK] = &cam_cc_ipe_1_fast_ahb_clk.clkr,
+	[CAM_CC_PLL0] = &cam_cc_pll0.clkr,
+	[CAM_CC_PLL0_OUT_EVEN] = &cam_cc_pll0_out_even.clkr,
+	[CAM_CC_PLL0_OUT_ODD] = &cam_cc_pll0_out_odd.clkr,
+	[CAM_CC_PLL2] = &cam_cc_pll2.clkr,
+	[CAM_CC_PLL2_OUT_EVEN] = &cam_cc_pll2_out_even.clkr,
+	[CAM_CC_PLL3] = &cam_cc_pll3.clkr,
+	[CAM_CC_PLL3_OUT_EVEN] = &cam_cc_pll3_out_even.clkr,
+	[CAM_CC_PLL4] = &cam_cc_pll4.clkr,
+	[CAM_CC_PLL4_OUT_EVEN] = &cam_cc_pll4_out_even.clkr,
+	[CAM_CC_PLL5] = &cam_cc_pll5.clkr,
+	[CAM_CC_PLL5_OUT_EVEN] = &cam_cc_pll5_out_even.clkr,
+	[CAM_CC_PLL6] = &cam_cc_pll6.clkr,
+	[CAM_CC_PLL6_OUT_EVEN] = &cam_cc_pll6_out_even.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK] = &cam_cc_qdss_debug_clk.clkr,
+	[CAM_CC_QDSS_DEBUG_CLK_SRC] = &cam_cc_qdss_debug_clk_src.clkr,
+	[CAM_CC_QDSS_DEBUG_XO_CLK] = &cam_cc_qdss_debug_xo_clk.clkr,
+	[CAM_CC_QUP_AHBM_CLK] = &cam_cc_qup_ahbm_clk.clkr,
+	[CAM_CC_QUP_AHBM_CLK_SRC] = &cam_cc_qup_ahbm_clk_src.clkr,
+	[CAM_CC_QUP_CORE_2X_CLK] = &cam_cc_qup_core_2x_clk.clkr,
+	[CAM_CC_QUP_CORE_2X_CLK_SRC] = &cam_cc_qup_core_2x_clk_src.clkr,
+	[CAM_CC_QUP_CORE_2X_DIV_CLK_SRC] = &cam_cc_qup_core_2x_div_clk_src.clkr,
+	[CAM_CC_QUP_CORE_CLK] = &cam_cc_qup_core_clk.clkr,
+	[CAM_CC_QUP_SE_CLK] = &cam_cc_qup_se_clk.clkr,
+	[CAM_CC_QUP_SE_CLK_SRC] = &cam_cc_qup_se_clk_src.clkr,
+	[CAM_CC_SFE_LITE_0_CLK] = &cam_cc_sfe_lite_0_clk.clkr,
+	[CAM_CC_SFE_LITE_0_FAST_AHB_CLK] = &cam_cc_sfe_lite_0_fast_ahb_clk.clkr,
+	[CAM_CC_SFE_LITE_1_CLK] = &cam_cc_sfe_lite_1_clk.clkr,
+	[CAM_CC_SFE_LITE_1_FAST_AHB_CLK] = &cam_cc_sfe_lite_1_fast_ahb_clk.clkr,
+	[CAM_CC_SFE_LITE_2_CLK] = &cam_cc_sfe_lite_2_clk.clkr,
+	[CAM_CC_SFE_LITE_2_FAST_AHB_CLK] = &cam_cc_sfe_lite_2_fast_ahb_clk.clkr,
+	[CAM_CC_SLEEP_CLK_SRC] = &cam_cc_sleep_clk_src.clkr,
+	[CAM_CC_SLOW_AHB_CLK_SRC] = &cam_cc_slow_ahb_clk_src.clkr,
+	[CAM_CC_SM_OBS_CLK] = &cam_cc_sm_obs_clk.clkr,
+	[CAM_CC_TOP_AHB_CLK] = &cam_cc_top_ahb_clk.clkr,
+	[CAM_CC_TOP_FAST_AHB_CLK] = &cam_cc_top_fast_ahb_clk.clkr,
+	[CAM_CC_TOP_IFE_0_CLK] = &cam_cc_top_ife_0_clk.clkr,
+	[CAM_CC_TOP_IFE_1_CLK] = &cam_cc_top_ife_1_clk.clkr,
+	[CAM_CC_TOP_IFE_2_CLK] = &cam_cc_top_ife_2_clk.clkr,
+	[CAM_CC_TOP_IFE_LITE_CLK] = &cam_cc_top_ife_lite_clk.clkr,
+	[CAM_CC_TOP_IPE_0_CLK] = &cam_cc_top_ipe_0_clk.clkr,
+	[CAM_CC_TOP_IPE_1_CLK] = &cam_cc_top_ipe_1_clk.clkr,
+	[CAM_CC_TOP_QUP_AHBM_CLK] = &cam_cc_top_qup_ahbm_clk.clkr,
+	[CAM_CC_TOP_SFE_LITE_0_CLK] = &cam_cc_top_sfe_lite_0_clk.clkr,
+	[CAM_CC_TOP_SFE_LITE_1_CLK] = &cam_cc_top_sfe_lite_1_clk.clkr,
+	[CAM_CC_TOP_SFE_LITE_2_CLK] = &cam_cc_top_sfe_lite_2_clk.clkr,
+	[CAM_CC_TPG_CSIPHY_RX_CLK] = &cam_cc_tpg_csiphy_rx_clk.clkr,
+	[CAM_CC_XO_CLK_SRC] = &cam_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *cam_cc_nord_gdscs[] = {
+	[CAM_CC_TITAN_TOP_GDSC] = &cam_cc_titan_top_gdsc,
+	[CAM_CC_IFE_0_GDSC] = &cam_cc_ife_0_gdsc,
+	[CAM_CC_IFE_1_GDSC] = &cam_cc_ife_1_gdsc,
+	[CAM_CC_IFE_2_GDSC] = &cam_cc_ife_2_gdsc,
+	[CAM_CC_IPE_0_GDSC] = &cam_cc_ipe_0_gdsc,
+	[CAM_CC_IPE_1_GDSC] = &cam_cc_ipe_1_gdsc,
+};
+
+static const struct qcom_reset_map cam_cc_nord_resets[] = {
+	[CAM_CC_CCU_BCR] = { 0x13298 },
+	[CAM_CC_ICP_0_BCR] = { 0x130a0 },
+	[CAM_CC_ICP_1_BCR] = { 0x130d8 },
+	[CAM_CC_IFE_0_BCR] = { 0x12000 },
+	[CAM_CC_IFE_1_BCR] = { 0x120c4 },
+	[CAM_CC_IFE_2_BCR] = { 0x12170 },
+	[CAM_CC_IPE_0_BCR] = { 0x11000 },
+	[CAM_CC_IPE_1_BCR] = { 0x1105c },
+	[CAM_CC_QDSS_DEBUG_BCR] = { 0x1332c },
+	[CAM_CC_SFE_LITE_0_BCR] = { 0x13058 },
+	[CAM_CC_SFE_LITE_1_BCR] = { 0x13070 },
+	[CAM_CC_SFE_LITE_2_BCR] = { 0x13088 },
+};
+
+static struct clk_alpha_pll *cam_cc_nord_plls[] = {
+	&cam_cc_pll0,
+	&cam_cc_pll2,
+	&cam_cc_pll3,
+	&cam_cc_pll4,
+	&cam_cc_pll5,
+	&cam_cc_pll6,
+};
+
+static const u32 cam_cc_nord_critical_cbcrs[] = {
+	0x13294, /* CAM_CC_CAMNOC_XO_CLK */
+	0x13364, /* CAM_CC_CORE_AHB_CLK */
+	0x13380, /* CAM_CC_GDSC_CLK */
+	0x132e4, /* CAM_CC_QUP_SLEEP_CLK */
+	0x1339c, /* CAM_CC_SLEEP_CLK */
+};
+
+static const struct regmap_config cam_cc_nord_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x17000,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_driver_data cam_cc_nord_driver_data = {
+	.alpha_plls = cam_cc_nord_plls,
+	.num_alpha_plls = ARRAY_SIZE(cam_cc_nord_plls),
+	.clk_cbcrs = cam_cc_nord_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_nord_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc cam_cc_nord_desc = {
+	.config = &cam_cc_nord_regmap_config,
+	.clks = cam_cc_nord_clocks,
+	.num_clks = ARRAY_SIZE(cam_cc_nord_clocks),
+	.resets = cam_cc_nord_resets,
+	.num_resets = ARRAY_SIZE(cam_cc_nord_resets),
+	.gdscs = cam_cc_nord_gdscs,
+	.num_gdscs = ARRAY_SIZE(cam_cc_nord_gdscs),
+	.use_rpm = true,
+	.driver_data = &cam_cc_nord_driver_data,
+};
+
+static const struct of_device_id cam_cc_nord_match_table[] = {
+	{ .compatible = "qcom,nord-camcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, cam_cc_nord_match_table);
+
+static int cam_cc_nord_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &cam_cc_nord_desc);
+}
+
+static struct platform_driver cam_cc_nord_driver = {
+	.probe = cam_cc_nord_probe,
+	.driver = {
+		.name = "camcc-nord",
+		.of_match_table = cam_cc_nord_match_table,
+	},
+};
+
+module_platform_driver(cam_cc_nord_driver);
+
+MODULE_DESCRIPTION("QTI CAMCC Nord Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/camcc-sa8775p.c b/drivers/clk/qcom/camcc-sa8775p.c
index 914478139e970a..b0a8f0077dd871 100644
--- a/drivers/clk/qcom/camcc-sa8775p.c
+++ b/drivers/clk/qcom/camcc-sa8775p.c
@@ -1818,7 +1818,30 @@ static const struct regmap_config cam_cc_sa8775p_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 cam_cc_sa8775p_critical_cbcrs[] = {
+	0x13194, /* CAM_CC_CAMNOC_XO_CLK */
+	0x131ec, /* CAM_CC_GDSC_CLK */
+	0x13208, /* CAM_CC_SLEEP_CLK */
+};
+
+static const u32 cam_cc_qcs8300_critical_cbcrs[] = {
+	0x13178, /* CAM_CC_CAMNOC_XO_CLK */
+	0x131d0, /* CAM_CC_GDSC_CLK */
+	0x131ec, /* CAM_CC_SLEEP_CLK */
+};
+
+static const struct qcom_cc_driver_data cam_cc_sa8775p_driver_data = {
+	.clk_cbcrs = cam_cc_sa8775p_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_sa8775p_critical_cbcrs),
+};
+
+static const struct qcom_cc_driver_data cam_cc_qcs8300_driver_data = {
+	.clk_cbcrs = cam_cc_qcs8300_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_qcs8300_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc cam_cc_sa8775p_desc = {
+	.driver_data = &cam_cc_sa8775p_driver_data,
 	.config = &cam_cc_sa8775p_regmap_config,
 	.clks = cam_cc_sa8775p_clocks,
 	.num_clks = ARRAY_SIZE(cam_cc_sa8775p_clocks),
@@ -1837,6 +1860,7 @@ MODULE_DEVICE_TABLE(of, cam_cc_sa8775p_match_table);
 
 static int cam_cc_sa8775p_probe(struct platform_device *pdev)
 {
+	struct qcom_cc_desc desc = cam_cc_sa8775p_desc;
 	struct regmap *regmap;
 	int ret;
 
@@ -1927,18 +1951,10 @@ static int cam_cc_sa8775p_probe(struct platform_device *pdev)
 		cam_cc_sa8775p_clocks[CAM_CC_TITAN_TOP_ACCU_SHIFT_CLK] =
 				&cam_cc_titan_top_accu_shift_clk.clkr;
 
-		/* Keep some clocks always enabled */
-		qcom_branch_set_clk_en(regmap, 0x13178); /* CAM_CC_CAMNOC_XO_CLK */
-		qcom_branch_set_clk_en(regmap, 0x131d0); /* CAM_CC_GDSC_CLK */
-		qcom_branch_set_clk_en(regmap, 0x131ec); /* CAM_CC_SLEEP_CLK */
-	} else {
-		/* Keep some clocks always enabled */
-		qcom_branch_set_clk_en(regmap, 0x13194); /* CAM_CC_CAMNOC_XO_CLK */
-		qcom_branch_set_clk_en(regmap, 0x131ec); /* CAM_CC_GDSC_CLK */
-		qcom_branch_set_clk_en(regmap, 0x13208); /* CAM_CC_SLEEP_CLK */
+		desc.driver_data = &cam_cc_qcs8300_driver_data;
 	}
 
-	ret = qcom_cc_really_probe(&pdev->dev, &cam_cc_sa8775p_desc, regmap);
+	ret = qcom_cc_really_probe(&pdev->dev, &desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
 
diff --git a/drivers/clk/qcom/camcc-sc8280xp.c b/drivers/clk/qcom/camcc-sc8280xp.c
index 660d8655d3919a..b0f1d40d1df5ef 100644
--- a/drivers/clk/qcom/camcc-sc8280xp.c
+++ b/drivers/clk/qcom/camcc-sc8280xp.c
@@ -2967,7 +2967,17 @@ static const struct regmap_config camcc_sc8280xp_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 camcc_sc8280xp_critical_cbcrs[] = {
+	0xc1e4, /* CAMCC_GDSC_CLK */
+};
+
+static const struct qcom_cc_driver_data camcc_sc8280xp_driver_data = {
+	.clk_cbcrs = camcc_sc8280xp_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(camcc_sc8280xp_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc camcc_sc8280xp_desc = {
+	.driver_data = &camcc_sc8280xp_driver_data,
 	.config = &camcc_sc8280xp_regmap_config,
 	.clks = camcc_sc8280xp_clocks,
 	.num_clks = ARRAY_SIZE(camcc_sc8280xp_clocks),
@@ -3011,9 +3021,6 @@ static int camcc_sc8280xp_probe(struct platform_device *pdev)
 	clk_lucid_pll_configure(&camcc_pll6, regmap, &camcc_pll6_config);
 	clk_lucid_pll_configure(&camcc_pll7, regmap, &camcc_pll7_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xc1e4); /* CAMCC_GDSC_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &camcc_sc8280xp_desc, regmap);
 	if (ret)
 		goto err_disable;
diff --git a/drivers/clk/qcom/camcc-sm7150.c b/drivers/clk/qcom/camcc-sm7150.c
index 6f75689e9847a2..d06258327ba740 100644
--- a/drivers/clk/qcom/camcc-sm7150.c
+++ b/drivers/clk/qcom/camcc-sm7150.c
@@ -2008,7 +2008,17 @@ static const struct regmap_config camcc_sm7150_regmap_config = {
 	.fast_io	= true,
 };
 
+static const u32 camcc_sm7150_critical_cbcrs[] = {
+	0xc1a0, /* CAMCC_GDSC_CLK */
+};
+
+static const struct qcom_cc_driver_data camcc_sm7150_driver_data = {
+	.clk_cbcrs = camcc_sm7150_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(camcc_sm7150_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc camcc_sm7150_desc = {
+	.driver_data = &camcc_sm7150_driver_data,
 	.config = &camcc_sm7150_regmap_config,
 	.clk_hws = camcc_sm7150_hws,
 	.num_clk_hws = ARRAY_SIZE(camcc_sm7150_hws),
@@ -2038,9 +2048,6 @@ static int camcc_sm7150_probe(struct platform_device *pdev)
 	clk_fabia_pll_configure(&camcc_pll3, regmap, &camcc_pll3_config);
 	clk_fabia_pll_configure(&camcc_pll4, regmap, &camcc_pll3_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xc1a0); /* CAMCC_GDSC_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &camcc_sm7150_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/camcc-sm8150.c b/drivers/clk/qcom/camcc-sm8150.c
index fcbaff55fc2721..27a51abddce256 100644
--- a/drivers/clk/qcom/camcc-sm8150.c
+++ b/drivers/clk/qcom/camcc-sm8150.c
@@ -2093,7 +2093,17 @@ static const struct regmap_config cam_cc_sm8150_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 cam_cc_sm8150_critical_cbcrs[] = {
+	0xc1e4, /* CAM_CC_GDSC_CLK */
+};
+
+static const struct qcom_cc_driver_data cam_cc_sm8150_driver_data = {
+	.clk_cbcrs = cam_cc_sm8150_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(cam_cc_sm8150_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc cam_cc_sm8150_desc = {
+	.driver_data = &cam_cc_sm8150_driver_data,
 	.config = &cam_cc_sm8150_regmap_config,
 	.clks = cam_cc_sm8150_clocks,
 	.num_clks = ARRAY_SIZE(cam_cc_sm8150_clocks),
@@ -2134,9 +2144,6 @@ static int cam_cc_sm8150_probe(struct platform_device *pdev)
 	clk_trion_pll_configure(&cam_cc_pll3, regmap, &cam_cc_pll3_config);
 	clk_trion_pll_configure(&cam_cc_pll4, regmap, &cam_cc_pll4_config);
 
-	/* Keep the critical clock always-on */
-	qcom_branch_set_clk_en(regmap, 0xc1e4); /* cam_cc_gdsc_clk */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &cam_cc_sm8150_desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c
index f8313f9d0e30ff..60173b076cc5f1 100644
--- a/drivers/clk/qcom/clk-alpha-pll.c
+++ b/drivers/clk/qcom/clk-alpha-pll.c
@@ -2373,6 +2373,15 @@ void clk_lucid_ole_pll_configure(struct clk_alpha_pll *pll, struct regmap *regma
 {
 	u32 lval = config->l;
 
+	/*
+	 * If the bootloader left the PLL enabled it's likely that there are
+	 * RCGs that will lock up if we disable the PLL below.
+	 */
+	if (trion_pll_is_enabled(pll, regmap)) {
+		pr_debug("Lucid Ole PLL is already enabled, skipping configuration\n");
+		return;
+	}
+
 	lval |= TRION_PLL_CAL_VAL << LUCID_EVO_PLL_CAL_L_VAL_SHIFT;
 	lval |= TRION_PLL_CAL_VAL << LUCID_OLE_PLL_RINGOSC_CAL_L_VAL_SHIFT;
 	clk_alpha_pll_write_config(regmap, PLL_L_VAL(pll), lval);
diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c
index d7914e59129f61..c44c51d860e1fc 100644
--- a/drivers/clk/qcom/clk-rcg2.c
+++ b/drivers/clk/qcom/clk-rcg2.c
@@ -1332,7 +1332,10 @@ static int clk_rcg2_set_force_enable(struct clk_hw *hw)
 
 	/* wait for RCG to turn ON */
 	for (count = 500; count > 0; count--) {
-		if (clk_rcg2_is_enabled(hw))
+		ret = clk_rcg2_is_enabled(hw);
+		if (ret < 0)
+			return ret;
+		if (ret)
 			return 0;
 
 		udelay(1);
@@ -1361,8 +1364,10 @@ clk_rcg2_shared_force_enable_clear(struct clk_hw *hw, const struct freq_tbl *f)
 		return ret;
 
 	ret = clk_rcg2_configure(rcg, f);
-	if (ret)
+	if (ret) {
+		clk_rcg2_clear_force_enable(hw);
 		return ret;
+	}
 
 	return clk_rcg2_clear_force_enable(hw);
 }
@@ -1555,7 +1560,20 @@ const struct clk_ops clk_rcg2_shared_ops = {
 };
 EXPORT_SYMBOL_GPL(clk_rcg2_shared_ops);
 
+static int clk_rcg2_shared_no_init_park(struct clk_hw *hw)
+{
+	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
+
+	/*
+	 * Read the config register so that the parent is properly mapped at
+	 * registration time.
+	 */
+	return regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG,
+			   &rcg->parked_cfg);
+}
+
 const struct clk_ops clk_rcg2_shared_floor_ops = {
+	.init = clk_rcg2_shared_no_init_park,
 	.enable = clk_rcg2_shared_enable,
 	.disable = clk_rcg2_shared_disable,
 	.get_parent = clk_rcg2_shared_get_parent,
@@ -1567,19 +1585,6 @@ const struct clk_ops clk_rcg2_shared_floor_ops = {
 };
 EXPORT_SYMBOL_GPL(clk_rcg2_shared_floor_ops);
 
-static int clk_rcg2_shared_no_init_park(struct clk_hw *hw)
-{
-	struct clk_rcg2 *rcg = to_clk_rcg2(hw);
-
-	/*
-	 * Read the config register so that the parent is properly mapped at
-	 * registration time.
-	 */
-	regmap_read(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, &rcg->parked_cfg);
-
-	return 0;
-}
-
 /*
  * Like clk_rcg2_shared_ops but skip the init so that the clk frequency is left
  * unchanged at registration time.
diff --git a/drivers/clk/qcom/clk-regmap-divider.c b/drivers/clk/qcom/clk-regmap-divider.c
index 672e82caf20504..6e12aebf14b6a3 100644
--- a/drivers/clk/qcom/clk-regmap-divider.c
+++ b/drivers/clk/qcom/clk-regmap-divider.c
@@ -27,7 +27,8 @@ static int div_ro_determine_rate(struct clk_hw *hw,
 	val &= BIT(divider->width) - 1;
 
 	return divider_ro_determine_rate(hw, req, NULL, divider->width,
-					 CLK_DIVIDER_ROUND_CLOSEST, val);
+					 CLK_DIVIDER_ROUND_CLOSEST | divider->flags,
+					 val);
 }
 
 static int div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req)
@@ -35,7 +36,7 @@ static int div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req)
 	struct clk_regmap_div *divider = to_clk_regmap_div(hw);
 
 	return divider_determine_rate(hw, req, NULL, divider->width,
-				      CLK_DIVIDER_ROUND_CLOSEST);
+				      CLK_DIVIDER_ROUND_CLOSEST | divider->flags);
 }
 
 static int div_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -43,14 +44,16 @@ static int div_set_rate(struct clk_hw *hw, unsigned long rate,
 {
 	struct clk_regmap_div *divider = to_clk_regmap_div(hw);
 	struct clk_regmap *clkr = &divider->clkr;
-	u32 div;
+	int div;
 
 	div = divider_get_val(rate, parent_rate, NULL, divider->width,
-			      CLK_DIVIDER_ROUND_CLOSEST);
+			      CLK_DIVIDER_ROUND_CLOSEST | divider->flags);
+	if (div < 0)
+		return div;
 
 	return regmap_update_bits(clkr->regmap, divider->reg,
 				  (BIT(divider->width) - 1) << divider->shift,
-				  div << divider->shift);
+				  (u32)div << divider->shift);
 }
 
 static unsigned long div_recalc_rate(struct clk_hw *hw,
@@ -65,7 +68,8 @@ static unsigned long div_recalc_rate(struct clk_hw *hw,
 	div &= BIT(divider->width) - 1;
 
 	return divider_recalc_rate(hw, parent_rate, div, NULL,
-				   CLK_DIVIDER_ROUND_CLOSEST, divider->width);
+				   CLK_DIVIDER_ROUND_CLOSEST | divider->flags,
+				   divider->width);
 }
 
 const struct clk_ops clk_regmap_div_ops = {
diff --git a/drivers/clk/qcom/clk-regmap-divider.h b/drivers/clk/qcom/clk-regmap-divider.h
index e75a65c3839c37..4fde766b2633be 100644
--- a/drivers/clk/qcom/clk-regmap-divider.h
+++ b/drivers/clk/qcom/clk-regmap-divider.h
@@ -13,6 +13,7 @@ struct clk_regmap_div {
 	u32			reg;
 	u32			shift;
 	u32			width;
+	unsigned long		flags;
 	struct clk_regmap	clkr;
 };
 
diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c
index a224a94ae27337..68640e5904a648 100644
--- a/drivers/clk/qcom/clk-rpmh.c
+++ b/drivers/clk/qcom/clk-rpmh.c
@@ -393,6 +393,7 @@ DEFINE_CLK_RPMH_VRM(clk2, _a1_e0, "C2A_E0", 1);
 DEFINE_CLK_RPMH_VRM(clk3, _a1_e0, "C3A_E0", 1);
 DEFINE_CLK_RPMH_VRM(clk4, _a1_e0, "C4A_E0", 1);
 DEFINE_CLK_RPMH_VRM(clk5, _a1_e0, "C5A_E0", 1);
+DEFINE_CLK_RPMH_VRM(clk7, _a1_e0, "C7A_E0", 1);
 DEFINE_CLK_RPMH_VRM(clk8, _a1_e0, "C8A_E0", 1);
 
 DEFINE_CLK_RPMH_VRM(clk3, _a2_e0, "C3A_E0", 2);
@@ -902,6 +903,10 @@ static struct clk_hw *glymur_rpmh_clocks[] = {
 	[RPMH_RF_CLK4_A]	= &clk_rpmh_clk4_a1_e0_ao.hw,
 	[RPMH_RF_CLK5]		= &clk_rpmh_clk5_a1_e0.hw,
 	[RPMH_RF_CLK5_A]	= &clk_rpmh_clk5_a1_e0_ao.hw,
+	[RPMH_LN_BB_CLK2]	= &clk_rpmh_clk7_a1_e0.hw,
+	[RPMH_LN_BB_CLK2_A]	= &clk_rpmh_clk7_a1_e0_ao.hw,
+	[RPMH_LN_BB_CLK3]	= &clk_rpmh_clk8_a1_e0.hw,
+	[RPMH_LN_BB_CLK3_A]	= &clk_rpmh_clk8_a1_e0_ao.hw,
 };
 
 static const struct clk_rpmh_desc clk_rpmh_glymur = {
@@ -937,6 +942,20 @@ static const struct clk_rpmh_desc clk_rpmh_kaanapali = {
 	.num_clks = ARRAY_SIZE(kaanapali_rpmh_clocks),
 };
 
+static struct clk_hw *kuno_rpmh_clocks[] = {
+	[RPMH_CXO_CLK]		= &clk_rpmh_bi_tcxo_div2.hw,
+	[RPMH_CXO_CLK_A]	= &clk_rpmh_bi_tcxo_div2_ao.hw,
+	[RPMH_RF_CLK1]		= &clk_rpmh_rf_clk1_a.hw,
+	[RPMH_RF_CLK1_A]	= &clk_rpmh_rf_clk1_a_ao.hw,
+	[RPMH_QPIC_CLK]		= &clk_rpmh_qpic_clk.hw,
+	[RPMH_IPA_CLK]		= &clk_rpmh_ipa.hw,
+};
+
+static const struct clk_rpmh_desc clk_rpmh_kuno = {
+	.clks = kuno_rpmh_clocks,
+	.num_clks = ARRAY_SIZE(kuno_rpmh_clocks),
+};
+
 static struct clk_hw *eliza_rpmh_clocks[] = {
 	[RPMH_CXO_CLK]          = &clk_rpmh_bi_tcxo_div2.hw,
 	[RPMH_CXO_CLK_A]        = &clk_rpmh_bi_tcxo_div2_ao.hw,
@@ -1005,6 +1024,29 @@ static const struct clk_rpmh_desc clk_rpmh_hawi = {
 	.num_clks = ARRAY_SIZE(hawi_rpmh_clocks),
 };
 
+static struct clk_hw *sm7250_rpmh_clocks[] = {
+	[RPMH_CXO_CLK]		= &clk_rpmh_bi_tcxo_div2.hw,
+	[RPMH_CXO_CLK_A]	= &clk_rpmh_bi_tcxo_div2_ao.hw,
+	[RPMH_LN_BB_CLK2]	= &clk_rpmh_ln_bb_clk2_a2.hw,
+	[RPMH_LN_BB_CLK2_A]	= &clk_rpmh_ln_bb_clk2_a2_ao.hw,
+	[RPMH_LN_BB_CLK3]	= &clk_rpmh_ln_bb_clk3_a2.hw,
+	[RPMH_LN_BB_CLK3_A]	= &clk_rpmh_ln_bb_clk3_a2_ao.hw,
+	[RPMH_RF_CLK1]		= &clk_rpmh_rf_clk1_d.hw,
+	[RPMH_RF_CLK1_A]	= &clk_rpmh_rf_clk1_d_ao.hw,
+	[RPMH_RF_CLK2]		= &clk_rpmh_rf_clk2_d.hw,
+	[RPMH_RF_CLK2_A]	= &clk_rpmh_rf_clk2_d_ao.hw,
+	[RPMH_RF_CLK3]		= &clk_rpmh_rf_clk3_d.hw,
+	[RPMH_RF_CLK3_A]	= &clk_rpmh_rf_clk3_d_ao.hw,
+	[RPMH_RF_CLK4]		= &clk_rpmh_rf_clk4_d.hw,
+	[RPMH_RF_CLK4_A]	= &clk_rpmh_rf_clk4_d_ao.hw,
+	[RPMH_IPA_CLK]		= &clk_rpmh_ipa.hw,
+};
+
+static const struct clk_rpmh_desc clk_rpmh_sm7250 = {
+	.clks = sm7250_rpmh_clocks,
+	.num_clks = ARRAY_SIZE(sm7250_rpmh_clocks),
+};
+
 static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec,
 					 void *data)
 {
@@ -1097,6 +1139,7 @@ static const struct of_device_id clk_rpmh_match_table[] = {
 	{ .compatible = "qcom,glymur-rpmh-clk", .data = &clk_rpmh_glymur},
 	{ .compatible = "qcom,hawi-rpmh-clk", .data = &clk_rpmh_hawi},
 	{ .compatible = "qcom,kaanapali-rpmh-clk", .data = &clk_rpmh_kaanapali},
+	{ .compatible = "qcom,kuno-rpmh-clk", .data = &clk_rpmh_kuno},
 	{ .compatible = "qcom,milos-rpmh-clk", .data = &clk_rpmh_milos},
 	{ .compatible = "qcom,nord-rpmh-clk", .data = &clk_rpmh_nord},
 	{ .compatible = "qcom,qcs615-rpmh-clk", .data = &clk_rpmh_qcs615},
@@ -1114,6 +1157,7 @@ static const struct of_device_id clk_rpmh_match_table[] = {
 	{ .compatible = "qcom,sdx75-rpmh-clk",  .data = &clk_rpmh_sdx75},
 	{ .compatible = "qcom,sm4450-rpmh-clk", .data = &clk_rpmh_sm4450},
 	{ .compatible = "qcom,sm6350-rpmh-clk", .data = &clk_rpmh_sm6350},
+	{ .compatible = "qcom,sm7250-rpmh-clk", .data = &clk_rpmh_sm7250},
 	{ .compatible = "qcom,sm8150-rpmh-clk", .data = &clk_rpmh_sm8150},
 	{ .compatible = "qcom,sm8250-rpmh-clk", .data = &clk_rpmh_sm8250},
 	{ .compatible = "qcom,sm8350-rpmh-clk", .data = &clk_rpmh_sm8350},
diff --git a/drivers/clk/qcom/dispcc-sc7280.c b/drivers/clk/qcom/dispcc-sc7280.c
index d11265debaf9fa..24a65cbb1ecfff 100644
--- a/drivers/clk/qcom/dispcc-sc7280.c
+++ b/drivers/clk/qcom/dispcc-sc7280.c
@@ -860,7 +860,17 @@ static const struct regmap_config disp_cc_sc7280_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_sc7280_critical_cbcrs[] = {
+	0x5008, /* DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_sc7280_driver_data = {
+	.clk_cbcrs = disp_cc_sc7280_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sc7280_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_sc7280_desc = {
+	.driver_data = &disp_cc_sc7280_driver_data,
 	.config = &disp_cc_sc7280_regmap_config,
 	.clks = disp_cc_sc7280_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sc7280_clocks),
@@ -886,9 +896,6 @@ static int disp_cc_sc7280_probe(struct platform_device *pdev)
 
 	clk_lucid_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x5008); /* DISP_CC_XO_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &disp_cc_sc7280_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/dispcc-sc8280xp.c b/drivers/clk/qcom/dispcc-sc8280xp.c
index 96609bd5233f3c..96b3e5505279b5 100644
--- a/drivers/clk/qcom/dispcc-sc8280xp.c
+++ b/drivers/clk/qcom/dispcc-sc8280xp.c
@@ -3108,7 +3108,17 @@ static const struct regmap_config disp_cc_sc8280xp_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_sc8280xp_critical_cbcrs[] = {
+	0x605c, /* DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_sc8280xp_driver_data = {
+	.clk_cbcrs = disp_cc_sc8280xp_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sc8280xp_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp0_cc_sc8280xp_desc = {
+	.driver_data = &disp_cc_sc8280xp_driver_data,
 	.config = &disp_cc_sc8280xp_regmap_config,
 	.clks = disp0_cc_sc8280xp_clocks,
 	.num_clks = ARRAY_SIZE(disp0_cc_sc8280xp_clocks),
@@ -3119,6 +3129,7 @@ static const struct qcom_cc_desc disp0_cc_sc8280xp_desc = {
 };
 
 static const struct qcom_cc_desc disp1_cc_sc8280xp_desc = {
+	.driver_data = &disp_cc_sc8280xp_driver_data,
 	.config = &disp_cc_sc8280xp_regmap_config,
 	.clks = disp1_cc_sc8280xp_clocks,
 	.num_clks = ARRAY_SIZE(disp1_cc_sc8280xp_clocks),
@@ -3167,13 +3178,8 @@ static int disp_cc_sc8280xp_probe(struct platform_device *pdev)
 	clk_lucid_pll_configure(clkr_to_alpha_clk_pll(desc->clks[DISP_CC_PLL2]), regmap, &disp_cc_pll2_config);
 
 	ret = qcom_cc_really_probe(&pdev->dev, desc, regmap);
-	if (ret) {
+	if (ret)
 		dev_err(&pdev->dev, "Failed to register display clock controller\n");
-		goto out_pm_runtime_put;
-	}
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x605c); /* DISP_CC_XO_CLK */
 
 out_pm_runtime_put:
 	pm_runtime_put_sync(&pdev->dev);
diff --git a/drivers/clk/qcom/dispcc-sm4450.c b/drivers/clk/qcom/dispcc-sm4450.c
index 4a4811db55dd51..f39afa067cc66a 100644
--- a/drivers/clk/qcom/dispcc-sm4450.c
+++ b/drivers/clk/qcom/dispcc-sm4450.c
@@ -719,7 +719,18 @@ static const struct regmap_config disp_cc_sm4450_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_sm4450_critical_cbcrs[] = {
+	0xe070, /* DISP_CC_SLEEP_CLK */
+	0xe054, /* DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_sm4450_driver_data = {
+	.clk_cbcrs = disp_cc_sm4450_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sm4450_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_sm4450_desc = {
+	.driver_data = &disp_cc_sm4450_driver_data,
 	.config = &disp_cc_sm4450_regmap_config,
 	.clks = disp_cc_sm4450_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sm4450_clocks),
@@ -746,10 +757,6 @@ static int disp_cc_sm4450_probe(struct platform_device *pdev)
 	clk_lucid_evo_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config);
 	clk_lucid_evo_pll_configure(&disp_cc_pll1, regmap, &disp_cc_pll0_config);
 
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0xe070); /* DISP_CC_SLEEP_CLK */
-	qcom_branch_set_clk_en(regmap, 0xe054); /* DISP_CC_XO_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &disp_cc_sm4450_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/dispcc-sm6115.c b/drivers/clk/qcom/dispcc-sm6115.c
index 9a7b8ad646ed88..f6f5edf07f0319 100644
--- a/drivers/clk/qcom/dispcc-sm6115.c
+++ b/drivers/clk/qcom/dispcc-sm6115.c
@@ -561,7 +561,17 @@ static const struct regmap_config disp_cc_sm6115_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_sm6115_critical_cbcrs[] = {
+	0x604c, /* DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_sm6115_driver_data = {
+	.clk_cbcrs = disp_cc_sm6115_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sm6115_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_sm6115_desc = {
+	.driver_data = &disp_cc_sm6115_driver_data,
 	.config = &disp_cc_sm6115_regmap_config,
 	.clks = disp_cc_sm6115_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sm6115_clocks),
@@ -588,9 +598,6 @@ static int disp_cc_sm6115_probe(struct platform_device *pdev)
 
 	clk_alpha_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x604c); /* DISP_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_sm6115_desc, regmap);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register DISP CC clocks\n");
diff --git a/drivers/clk/qcom/dispcc-sm7150.c b/drivers/clk/qcom/dispcc-sm7150.c
index b9df6153e50f78..da7610856e5621 100644
--- a/drivers/clk/qcom/dispcc-sm7150.c
+++ b/drivers/clk/qcom/dispcc-sm7150.c
@@ -963,7 +963,17 @@ static const struct regmap_config dispcc_sm7150_regmap_config = {
 	.fast_io	= true,
 };
 
+static const u32 dispcc_sm7150_critical_cbcrs[] = {
+	0x605c, /* DISPCC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data dispcc_sm7150_driver_data = {
+	.clk_cbcrs = dispcc_sm7150_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(dispcc_sm7150_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc dispcc_sm7150_desc = {
+	.driver_data = &dispcc_sm7150_driver_data,
 	.config = &dispcc_sm7150_regmap_config,
 	.clks = dispcc_sm7150_clocks,
 	.num_clks = ARRAY_SIZE(dispcc_sm7150_clocks),
@@ -991,9 +1001,6 @@ static int dispcc_sm7150_probe(struct platform_device *pdev)
 	/* Enable clock gating for DSI and MDP clocks */
 	regmap_update_bits(regmap, 0x8000, 0x7f0, 0x7f0);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x605c); /* DISPCC_XO_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &dispcc_sm7150_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/dispcc-sm8250.c b/drivers/clk/qcom/dispcc-sm8250.c
index fdc07323f2986c..dc62eb18deeda2 100644
--- a/drivers/clk/qcom/dispcc-sm8250.c
+++ b/drivers/clk/qcom/dispcc-sm8250.c
@@ -1232,7 +1232,17 @@ static const struct regmap_config disp_cc_sm8250_regmap_config = {
 	.fast_io	= true,
 };
 
+static const u32 disp_cc_sm8250_critical_cbcrs[] = {
+	0x605c, /* DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_sm8250_driver_data = {
+	.clk_cbcrs = disp_cc_sm8250_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sm8250_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_sm8250_desc = {
+	.driver_data = &disp_cc_sm8250_driver_data,
 	.config = &disp_cc_sm8250_regmap_config,
 	.clks = disp_cc_sm8250_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sm8250_clocks),
@@ -1370,9 +1380,6 @@ static int disp_cc_sm8250_probe(struct platform_device *pdev)
 	/* Enable clock gating for MDP clocks */
 	regmap_update_bits(regmap, 0x8000, 0x10, 0x10);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x605c); /* DISP_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_sm8250_desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
diff --git a/drivers/clk/qcom/dispcc-sm8450.c b/drivers/clk/qcom/dispcc-sm8450.c
index 3af120e54cddb2..facbf040ab9a52 100644
--- a/drivers/clk/qcom/dispcc-sm8450.c
+++ b/drivers/clk/qcom/dispcc-sm8450.c
@@ -613,7 +613,7 @@ static struct clk_rcg2 disp_cc_mdss_mdp_clk_src = {
 		.parent_data = disp_cc_parent_data_5,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_5),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_shared_ops,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
 	},
 };
 
@@ -1778,6 +1778,29 @@ static const struct regmap_config disp_cc_sm8450_regmap_config = {
 	.fast_io = true,
 };
 
+static struct clk_alpha_pll *disp_cc_sm8450_plls[] = {
+	&disp_cc_pll0,
+	&disp_cc_pll1,
+};
+
+static const u32 disp_cc_sm8450_critical_cbcrs[] = {
+	0xe05c, /* DISP_CC_XO_CLK */
+};
+
+static void disp_cc_sm8450_clk_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Enable clock gating for MDP clocks */
+	regmap_set_bits(regmap, DISP_CC_MISC_CMD, BIT(4));
+}
+
+static const struct qcom_cc_driver_data disp_cc_sm8450_driver_data = {
+	.alpha_plls = disp_cc_sm8450_plls,
+	.num_alpha_plls = ARRAY_SIZE(disp_cc_sm8450_plls),
+	.clk_cbcrs = disp_cc_sm8450_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sm8450_critical_cbcrs),
+	.clk_regs_configure = disp_cc_sm8450_clk_regs_configure,
+};
+
 static const struct qcom_cc_desc disp_cc_sm8450_desc = {
 	.config = &disp_cc_sm8450_regmap_config,
 	.clks = disp_cc_sm8450_clocks,
@@ -1786,6 +1809,7 @@ static const struct qcom_cc_desc disp_cc_sm8450_desc = {
 	.num_resets = ARRAY_SIZE(disp_cc_sm8450_resets),
 	.gdscs = disp_cc_sm8450_gdscs,
 	.num_gdscs = ARRAY_SIZE(disp_cc_sm8450_gdscs),
+	.driver_data = &disp_cc_sm8450_driver_data,
 };
 
 static const struct of_device_id disp_cc_sm8450_match_table[] = {
@@ -1823,19 +1847,13 @@ static int disp_cc_sm8450_probe(struct platform_device *pdev)
 		disp_cc_pll1.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE];
 		disp_cc_pll1.clkr.hw.init = &sm8475_disp_cc_pll1_init;
 
-		clk_lucid_ole_pll_configure(&disp_cc_pll0, regmap, &sm8475_disp_cc_pll0_config);
-		clk_lucid_ole_pll_configure(&disp_cc_pll1, regmap, &sm8475_disp_cc_pll1_config);
+		disp_cc_pll0.config = &sm8475_disp_cc_pll0_config;
+		disp_cc_pll1.config = &sm8475_disp_cc_pll1_config;
 	} else {
-		clk_lucid_evo_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config);
-		clk_lucid_evo_pll_configure(&disp_cc_pll1, regmap, &disp_cc_pll1_config);
+		disp_cc_pll0.config = &disp_cc_pll0_config;
+		disp_cc_pll1.config = &disp_cc_pll1_config;
 	}
 
-	/* Enable clock gating for MDP clocks */
-	regmap_update_bits(regmap, DISP_CC_MISC_CMD, 0x10, 0x10);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xe05c); /* DISP_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_sm8450_desc, regmap);
 	if (ret)
 		goto err_put_rpm;
diff --git a/drivers/clk/qcom/dispcc-sm8550.c b/drivers/clk/qcom/dispcc-sm8550.c
index 418dcea20f005a..030fc7ef1fbff0 100644
--- a/drivers/clk/qcom/dispcc-sm8550.c
+++ b/drivers/clk/qcom/dispcc-sm8550.c
@@ -1744,7 +1744,17 @@ static const struct regmap_config disp_cc_sm8550_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_sm8550_critical_cbcrs[] = {
+	0xe054, /* DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_sm8550_driver_data = {
+	.clk_cbcrs = disp_cc_sm8550_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sm8550_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_sm8550_desc = {
+	.driver_data = &disp_cc_sm8550_driver_data,
 	.config = &disp_cc_sm8550_regmap_config,
 	.clks = disp_cc_sm8550_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sm8550_clocks),
@@ -1800,9 +1810,6 @@ static int disp_cc_sm8550_probe(struct platform_device *pdev)
 	/* Enable clock gating for MDP clocks */
 	regmap_update_bits(regmap, DISP_CC_MISC_CMD, 0x10, 0x10);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xe054); /* DISP_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_sm8550_desc, regmap);
 	if (ret)
 		goto err_put_rpm;
diff --git a/drivers/clk/qcom/dispcc-sm8750.c b/drivers/clk/qcom/dispcc-sm8750.c
index 18e86b80e581b9..3205ce5c1ad9e0 100644
--- a/drivers/clk/qcom/dispcc-sm8750.c
+++ b/drivers/clk/qcom/dispcc-sm8750.c
@@ -1884,7 +1884,20 @@ static const struct regmap_config disp_cc_sm8750_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_sm8750_critical_cbcrs[] = {
+	0xe07c, /* DISP_CC_SLEEP_CLK */
+	0xe05c, /* DISP_CC_XO_CLK */
+	0xc00c, /* DISP_CC_MDSS_RSCC_AHB_CLK */
+	0xc008, /* DISP_CC_MDSS_RSCC_VSYNC_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_sm8750_driver_data = {
+	.clk_cbcrs = disp_cc_sm8750_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_sm8750_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_sm8750_desc = {
+	.driver_data = &disp_cc_sm8750_driver_data,
 	.config = &disp_cc_sm8750_regmap_config,
 	.clks = disp_cc_sm8750_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_sm8750_clocks),
@@ -1926,12 +1939,6 @@ static int disp_cc_sm8750_probe(struct platform_device *pdev)
 	/* Enable clock gating for MDP clocks */
 	regmap_update_bits(regmap, DISP_CC_MISC_CMD, 0x10, 0x10);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xe07c); /* DISP_CC_SLEEP_CLK */
-	qcom_branch_set_clk_en(regmap, 0xe05c); /* DISP_CC_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0xc00c); /* DISP_CC_MDSS_RSCC_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xc008); /* DISP_CC_MDSS_RSCC_VSYNC_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_sm8750_desc, regmap);
 	if (ret)
 		goto err_put_rpm;
diff --git a/drivers/clk/qcom/dispcc-x1e80100.c b/drivers/clk/qcom/dispcc-x1e80100.c
index 795279609c0b75..aed06203a886ad 100644
--- a/drivers/clk/qcom/dispcc-x1e80100.c
+++ b/drivers/clk/qcom/dispcc-x1e80100.c
@@ -1637,7 +1637,18 @@ static const struct regmap_config disp_cc_x1e80100_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_x1e80100_critical_cbcrs[] = {
+	0xe074, /* DISP_CC_SLEEP_CLK */
+	0xe054, /* DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_x1e80100_driver_data = {
+	.clk_cbcrs = disp_cc_x1e80100_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_x1e80100_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_x1e80100_desc = {
+	.driver_data = &disp_cc_x1e80100_driver_data,
 	.config = &disp_cc_x1e80100_regmap_config,
 	.clks = disp_cc_x1e80100_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_x1e80100_clocks),
@@ -1678,10 +1689,6 @@ static int disp_cc_x1e80100_probe(struct platform_device *pdev)
 	/* Enable clock gating for MDP clocks */
 	regmap_update_bits(regmap, DISP_CC_MISC_CMD, 0x10, 0x10);
 
-	/* Keep clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0xe074); /* DISP_CC_SLEEP_CLK */
-	qcom_branch_set_clk_en(regmap, 0xe054); /* DISP_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_x1e80100_desc, regmap);
 	if (ret)
 		goto err_put_rpm;
@@ -1704,17 +1711,7 @@ static struct platform_driver disp_cc_x1e80100_driver = {
 	},
 };
 
-static int __init disp_cc_x1e80100_init(void)
-{
-	return platform_driver_register(&disp_cc_x1e80100_driver);
-}
-subsys_initcall(disp_cc_x1e80100_init);
-
-static void __exit disp_cc_x1e80100_exit(void)
-{
-	platform_driver_unregister(&disp_cc_x1e80100_driver);
-}
-module_exit(disp_cc_x1e80100_exit);
+subsys_platform_driver(disp_cc_x1e80100_driver);
 
 MODULE_DESCRIPTION("QTI Display Clock Controller X1E80100 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/dispcc0-sa8775p.c b/drivers/clk/qcom/dispcc0-sa8775p.c
index 0e976442834aeb..a1861cc8343e2d 100644
--- a/drivers/clk/qcom/dispcc0-sa8775p.c
+++ b/drivers/clk/qcom/dispcc0-sa8775p.c
@@ -1414,7 +1414,18 @@ static const struct regmap_config disp_cc_0_sa8775p_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_0_sa8775p_critical_cbcrs[] = {
+	0xc070, /* MDSS_0_DISP_CC_SLEEP_CLK */
+	0xc054, /* MDSS_0_DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_0_sa8775p_driver_data = {
+	.clk_cbcrs = disp_cc_0_sa8775p_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_0_sa8775p_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_0_sa8775p_desc = {
+	.driver_data = &disp_cc_0_sa8775p_driver_data,
 	.config = &disp_cc_0_sa8775p_regmap_config,
 	.clks = disp_cc_0_sa8775p_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_0_sa8775p_clocks),
@@ -1452,10 +1463,6 @@ static int disp_cc_0_sa8775p_probe(struct platform_device *pdev)
 	clk_lucid_evo_pll_configure(&mdss_0_disp_cc_pll0, regmap, &mdss_0_disp_cc_pll0_config);
 	clk_lucid_evo_pll_configure(&mdss_0_disp_cc_pll1, regmap, &mdss_0_disp_cc_pll1_config);
 
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0xc070); /* MDSS_0_DISP_CC_SLEEP_CLK */
-	qcom_branch_set_clk_en(regmap, 0xc054); /* MDSS_0_DISP_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_0_sa8775p_desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
diff --git a/drivers/clk/qcom/dispcc1-sa8775p.c b/drivers/clk/qcom/dispcc1-sa8775p.c
index 58008c1afc7671..330d953b5e9e7d 100644
--- a/drivers/clk/qcom/dispcc1-sa8775p.c
+++ b/drivers/clk/qcom/dispcc1-sa8775p.c
@@ -1414,7 +1414,18 @@ static const struct regmap_config disp_cc_1_sa8775p_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 disp_cc_1_sa8775p_critical_cbcrs[] = {
+	0xc070, /* MDSS_1_DISP_CC_SLEEP_CLK */
+	0xc054, /* MDSS_1_DISP_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data disp_cc_1_sa8775p_driver_data = {
+	.clk_cbcrs = disp_cc_1_sa8775p_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(disp_cc_1_sa8775p_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc disp_cc_1_sa8775p_desc = {
+	.driver_data = &disp_cc_1_sa8775p_driver_data,
 	.config = &disp_cc_1_sa8775p_regmap_config,
 	.clks = disp_cc_1_sa8775p_clocks,
 	.num_clks = ARRAY_SIZE(disp_cc_1_sa8775p_clocks),
@@ -1452,10 +1463,6 @@ static int disp_cc_1_sa8775p_probe(struct platform_device *pdev)
 	clk_lucid_evo_pll_configure(&mdss_1_disp_cc_pll0, regmap, &mdss_1_disp_cc_pll0_config);
 	clk_lucid_evo_pll_configure(&mdss_1_disp_cc_pll1, regmap, &mdss_1_disp_cc_pll1_config);
 
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0xc070); /* MDSS_1_DISP_CC_SLEEP_CLK */
-	qcom_branch_set_clk_en(regmap, 0xc054); /* MDSS_1_DISP_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &disp_cc_1_sa8775p_desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
diff --git a/drivers/clk/qcom/gcc-eliza.c b/drivers/clk/qcom/gcc-eliza.c
index 3e26c7a1e5b546..064771726e9bb5 100644
--- a/drivers/clk/qcom/gcc-eliza.c
+++ b/drivers/clk/qcom/gcc-eliza.c
@@ -3010,8 +3010,6 @@ static const u32 gcc_eliza_critical_cbcrs[] = {
 	0x26034, /* GCC_CAMERA_XO_CLK */
 	0x27004, /* GCC_DISP_AHB_CLK */
 	0x71004, /* GCC_GPU_CFG_AHB_CLK */
-	0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */
-	0x52010, /* GCC_PCIE_RSCC_XO_CLK */
 	0x32004, /* GCC_VIDEO_AHB_CLK */
 	0x32038, /* GCC_VIDEO_XO_CLK */
 };
@@ -3045,6 +3043,10 @@ static const struct regmap_config gcc_eliza_regmap_config = {
 
 static void clk_eliza_regs_configure(struct device *dev, struct regmap *regmap)
 {
+	/* Keep clocks always enabled */
+	regmap_update_bits(regmap, 0x52010, BIT(20), BIT(20)); /* GCC_PCIE_RSCC_CFG_AHB_CLK */
+	regmap_update_bits(regmap, 0x52010, BIT(21), BIT(21)); /* GCC_PCIE_RSCC_XO_CLK */
+
 	/* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks  */
 	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
 	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true);
@@ -3066,6 +3068,7 @@ static const struct qcom_cc_desc gcc_eliza_desc = {
 	.num_resets = ARRAY_SIZE(gcc_eliza_resets),
 	.gdscs = gcc_eliza_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_eliza_gdscs),
+	.use_rpm = true,
 	.driver_data = &gcc_eliza_driver_data,
 };
 
@@ -3088,17 +3091,7 @@ static struct platform_driver gcc_eliza_driver = {
 	},
 };
 
-static int __init gcc_eliza_init(void)
-{
-	return platform_driver_register(&gcc_eliza_driver);
-}
-subsys_initcall(gcc_eliza_init);
-
-static void __exit gcc_eliza_exit(void)
-{
-	platform_driver_unregister(&gcc_eliza_driver);
-}
-module_exit(gcc_eliza_exit);
+subsys_platform_driver(gcc_eliza_driver);
 
 MODULE_DESCRIPTION("QTI GCC Eliza Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c
index 2ee4820b6fdfb1..be18a555fe314a 100644
--- a/drivers/clk/qcom/gcc-glymur.c
+++ b/drivers/clk/qcom/gcc-glymur.c
@@ -481,36 +481,6 @@ static const struct clk_parent_data gcc_parent_data_17[] = {
 	{ .hw = &gcc_gpll0_out_even.clkr.hw },
 };
 
-static const struct parent_map gcc_parent_map_18[] = {
-	{ P_UFS_PHY_RX_SYMBOL_0_CLK, 0 },
-	{ P_BI_TCXO, 2 },
-};
-
-static const struct clk_parent_data gcc_parent_data_18[] = {
-	{ .index = DT_UFS_PHY_RX_SYMBOL_0_CLK },
-	{ .index = DT_BI_TCXO },
-};
-
-static const struct parent_map gcc_parent_map_19[] = {
-	{ P_UFS_PHY_RX_SYMBOL_1_CLK, 0 },
-	{ P_BI_TCXO, 2 },
-};
-
-static const struct clk_parent_data gcc_parent_data_19[] = {
-	{ .index = DT_UFS_PHY_RX_SYMBOL_1_CLK },
-	{ .index = DT_BI_TCXO },
-};
-
-static const struct parent_map gcc_parent_map_20[] = {
-	{ P_UFS_PHY_TX_SYMBOL_0_CLK, 0 },
-	{ P_BI_TCXO, 2 },
-};
-
-static const struct clk_parent_data gcc_parent_data_20[] = {
-	{ .index = DT_UFS_PHY_TX_SYMBOL_0_CLK },
-	{ .index = DT_BI_TCXO },
-};
-
 static const struct parent_map gcc_parent_map_21[] = {
 	{ P_GCC_USB3_PRIM_PHY_PIPE_CLK_SRC, 0 },
 	{ P_USB4_0_PHY_GCC_USB4RTR_MAX_PIPE_CLK, 1 },
@@ -907,47 +877,44 @@ static struct clk_regmap_phy_mux gcc_pcie_6_pipe_clk_src = {
 	},
 };
 
-static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_0_clk_src = {
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_0_clk_src = {
 	.reg = 0x7706c,
-	.shift = 0,
-	.width = 2,
-	.parent_map = gcc_parent_map_18,
 	.clkr = {
 		.hw.init = &(const struct clk_init_data) {
 			.name = "gcc_ufs_phy_rx_symbol_0_clk_src",
-			.parent_data = gcc_parent_data_18,
-			.num_parents = ARRAY_SIZE(gcc_parent_data_18),
-			.ops = &clk_regmap_mux_closest_ops,
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
 		},
 	},
 };
 
-static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_1_clk_src = {
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_1_clk_src = {
 	.reg = 0x770f0,
-	.shift = 0,
-	.width = 2,
-	.parent_map = gcc_parent_map_19,
 	.clkr = {
 		.hw.init = &(const struct clk_init_data) {
 			.name = "gcc_ufs_phy_rx_symbol_1_clk_src",
-			.parent_data = gcc_parent_data_19,
-			.num_parents = ARRAY_SIZE(gcc_parent_data_19),
-			.ops = &clk_regmap_mux_closest_ops,
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_1_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
 		},
 	},
 };
 
-static struct clk_regmap_mux gcc_ufs_phy_tx_symbol_0_clk_src = {
+static struct clk_regmap_phy_mux gcc_ufs_phy_tx_symbol_0_clk_src = {
 	.reg = 0x7705c,
-	.shift = 0,
-	.width = 2,
-	.parent_map = gcc_parent_map_20,
 	.clkr = {
 		.hw.init = &(const struct clk_init_data) {
 			.name = "gcc_ufs_phy_tx_symbol_0_clk_src",
-			.parent_data = gcc_parent_data_20,
-			.num_parents = ARRAY_SIZE(gcc_parent_data_20),
-			.ops = &clk_regmap_mux_closest_ops,
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_TX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
 		},
 	},
 };
@@ -8571,17 +8538,7 @@ static struct platform_driver gcc_glymur_driver = {
 	},
 };
 
-static int __init gcc_glymur_init(void)
-{
-	return platform_driver_register(&gcc_glymur_driver);
-}
-subsys_initcall(gcc_glymur_init);
-
-static void __exit gcc_glymur_exit(void)
-{
-	platform_driver_unregister(&gcc_glymur_driver);
-}
-module_exit(gcc_glymur_exit);
+subsys_platform_driver(gcc_glymur_driver);
 
 MODULE_DESCRIPTION("QTI GCC Glymur Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-hawi.c b/drivers/clk/qcom/gcc-hawi.c
index 6fb5a3db958640..39fd5f56ca614a 100644
--- a/drivers/clk/qcom/gcc-hawi.c
+++ b/drivers/clk/qcom/gcc-hawi.c
@@ -3692,8 +3692,6 @@ static const u32 gcc_hawi_critical_cbcrs[] = {
 	0x67084, /* GCC_PCIE_1_RSC_CORE_CLK */
 	0x43014, /* GCC_PCIE_LINK_XO_CLK */
 	0x6b088, /* GCC_PCIE_RSC_CORE_CLK */
-	0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */
-	0x52010, /* GCC_PCIE_RSCC_XO_CLK */
 	0x32004, /* GCC_VIDEO_AHB_CLK */
 	0x32028, /* GCC_VIDEO_XO_CLK */
 };
@@ -3763,6 +3761,10 @@ static const struct regmap_config gcc_hawi_regmap_config = {
 
 static void clk_hawi_regs_configure(struct device *dev, struct regmap *regmap)
 {
+	/* Keep clocks always enabled */
+	regmap_update_bits(regmap, 0x52010, BIT(20), BIT(20)); /* GCC_PCIE_RSCC_CFG_AHB_CLK */
+	regmap_update_bits(regmap, 0x52010, BIT(21), BIT(21)); /* GCC_PCIE_RSCC_XO_CLK */
+
 	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
 	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
 }
@@ -3841,17 +3843,7 @@ static struct platform_driver gcc_hawi_driver = {
 	},
 };
 
-static int __init gcc_hawi_init(void)
-{
-	return platform_driver_register(&gcc_hawi_driver);
-}
-subsys_initcall(gcc_hawi_init);
-
-static void __exit gcc_hawi_exit(void)
-{
-	platform_driver_unregister(&gcc_hawi_driver);
-}
-module_exit(gcc_hawi_exit);
+subsys_platform_driver(gcc_hawi_driver);
 
 MODULE_DESCRIPTION("QTI GCC HAWI Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-ipq5018.c b/drivers/clk/qcom/gcc-ipq5018.c
index 594dae3bac4c1d..add6a53d4df55f 100644
--- a/drivers/clk/qcom/gcc-ipq5018.c
+++ b/drivers/clk/qcom/gcc-ipq5018.c
@@ -68,6 +68,15 @@ static struct clk_alpha_pll gpll0_main = {
 			.parent_data = gcc_xo_data,
 			.num_parents = ARRAY_SIZE(gcc_xo_data),
 			.ops = &clk_alpha_pll_stromer_ops,
+			/*
+			 * During system boot, this PLL feeds the CPUs.
+			 * Mark it as critical to ensure that CCF does
+			 * not disable it, even if there are no active
+			 * consumers. This is needed to avoid a system
+			 * hang caused by turning off the clock driving
+			 * the CPUs.
+			 */
+			.flags = CLK_IS_CRITICAL,
 		},
 	},
 };
@@ -1260,6 +1269,7 @@ static struct clk_rcg2 usb0_master_clk_src = {
 };
 
 static const struct freq_tbl ftbl_usb0_mock_utmi_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
 	F(60000000, P_GPLL4, 10, 1, 2),
 	{ }
 };
diff --git a/drivers/clk/qcom/gcc-ipq5210.c b/drivers/clk/qcom/gcc-ipq5210.c
index 3a786a21bdff56..76adfa8bf2ad95 100644
--- a/drivers/clk/qcom/gcc-ipq5210.c
+++ b/drivers/clk/qcom/gcc-ipq5210.c
@@ -716,6 +716,7 @@ static struct clk_rcg2 gcc_usb0_master_clk_src = {
 };
 
 static const struct freq_tbl ftbl_gcc_usb0_mock_utmi_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
 	F(60000000, P_GPLL4_OUT_AUX, 10, 1, 2),
 	{ }
 };
diff --git a/drivers/clk/qcom/gcc-ipq5332.c b/drivers/clk/qcom/gcc-ipq5332.c
index ecd9ebeed7549c..4625b447c2351f 100644
--- a/drivers/clk/qcom/gcc-ipq5332.c
+++ b/drivers/clk/qcom/gcc-ipq5332.c
@@ -1038,6 +1038,7 @@ static struct clk_rcg2 gcc_usb0_master_clk_src = {
 };
 
 static const struct freq_tbl ftbl_gcc_usb0_mock_utmi_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
 	F(60000000, P_GPLL4_OUT_AUX, 10, 1, 2),
 	{ }
 };
diff --git a/drivers/clk/qcom/gcc-ipq5424.c b/drivers/clk/qcom/gcc-ipq5424.c
index 3ddd4b3aedea9c..32243e110b279e 100644
--- a/drivers/clk/qcom/gcc-ipq5424.c
+++ b/drivers/clk/qcom/gcc-ipq5424.c
@@ -2949,7 +2949,7 @@ static const struct qcom_reset_map gcc_ipq5424_resets[] = {
 	[GCC_QUPV3_SPI0_BCR] = { 0x04000, 0 },
 	[GCC_QUPV3_SPI1_BCR] = { 0x05000, 0 },
 	[GCC_IMEM_BCR] = { 0x0e000, 0 },
-	[GCC_TME_BCR] = { 0x100000, 0 },
+	[GCC_TME_BCR] = { 0x10000, 0 },
 	[GCC_DDRSS_BCR] = { 0x11000, 0 },
 	[GCC_PRNG_BCR] = { 0x13020, 0 },
 	[GCC_BOOT_ROM_BCR] = { 0x13028, 0 },
@@ -2960,7 +2960,7 @@ static const struct qcom_reset_map gcc_ipq5424_resets[] = {
 	[GCC_UNIPHY2_BCR] = { 0x17064, 0 },
 	[GCC_WCSS_BCR] = { 0x18004, 0 },
 	[GCC_SEC_CTRL_BCR] = { 0x1a000, 0 },
-	[GCC_TME_SEC_BUS_BCR] = { 0xa1030, 0 },
+	[GCC_TME_SEC_BUS_BCR] = { 0x1a030, 0 },
 	[GCC_ADSS_BCR] = { 0x1c000, 0 },
 	[GCC_LPASS_BCR] = { 0x27000, 0 },
 	[GCC_PCIE0_BCR] = { 0x28000, 0 },
diff --git a/drivers/clk/qcom/gcc-ipq9650.c b/drivers/clk/qcom/gcc-ipq9650.c
index e65a52fdfa91fd..8c09a757c11d49 100644
--- a/drivers/clk/qcom/gcc-ipq9650.c
+++ b/drivers/clk/qcom/gcc-ipq9650.c
@@ -4,11 +4,13 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/interconnect-provider.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,ipq9650-gcc.h>
+#include <dt-bindings/interconnect/qcom,ipq9650.h>
 #include <dt-bindings/reset/qcom,ipq9650-gcc.h>
 
 #include "clk-alpha-pll.h"
@@ -3474,6 +3476,22 @@ static const struct qcom_reset_map gcc_ipq9650_resets[] = {
 	[GCC_USB_BCR] = { 0x2c000 },
 };
 
+#define IPQ_APPS_ID			9650	/* some unique value */
+
+static const struct qcom_icc_hws_data icc_ipq9650_hws[] = {
+	{ MASTER_ANOC_PCIE0, SLAVE_ANOC_PCIE0, GCC_ANOC_PCIE0_1LANE_M_CLK },
+	{ MASTER_CNOC_PCIE0, SLAVE_CNOC_PCIE0, GCC_ANOC_PCIE0_1LANE_S_CLK },
+	{ MASTER_ANOC_PCIE1, SLAVE_ANOC_PCIE1, GCC_ANOC_PCIE1_2LANE_M_CLK },
+	{ MASTER_CNOC_PCIE1, SLAVE_CNOC_PCIE1, GCC_ANOC_PCIE1_2LANE_S_CLK },
+	{ MASTER_ANOC_PCIE2, SLAVE_ANOC_PCIE2, GCC_ANOC_PCIE2_2LANE_M_CLK },
+	{ MASTER_CNOC_PCIE2, SLAVE_CNOC_PCIE2, GCC_ANOC_PCIE2_2LANE_S_CLK },
+	{ MASTER_ANOC_PCIE3, SLAVE_ANOC_PCIE3, GCC_ANOC_PCIE3_2LANE_M_CLK },
+	{ MASTER_CNOC_PCIE3, SLAVE_CNOC_PCIE3, GCC_ANOC_PCIE3_2LANE_S_CLK },
+	{ MASTER_ANOC_PCIE4, SLAVE_ANOC_PCIE4, GCC_ANOC_PCIE4_1LANE_M_CLK },
+	{ MASTER_CNOC_PCIE4, SLAVE_CNOC_PCIE4, GCC_ANOC_PCIE4_1LANE_S_CLK },
+	{ MASTER_SNOC_USB, SLAVE_SNOC_USB, GCC_SNOC_USB_CLK },
+};
+
 static const struct of_device_id gcc_ipq9650_match_table[] = {
 	{ .compatible = "qcom,ipq9650-gcc" },
 	{ }
@@ -3503,6 +3521,9 @@ static const struct qcom_cc_desc gcc_ipq9650_desc = {
 	.num_resets = ARRAY_SIZE(gcc_ipq9650_resets),
 	.clk_hws = gcc_ipq9650_hws,
 	.num_clk_hws = ARRAY_SIZE(gcc_ipq9650_hws),
+	.icc_hws = icc_ipq9650_hws,
+	.num_icc_hws = ARRAY_SIZE(icc_ipq9650_hws),
+	.icc_first_node_id = IPQ_APPS_ID,
 };
 
 static int gcc_ipq9650_probe(struct platform_device *pdev)
@@ -3515,6 +3536,7 @@ static struct platform_driver gcc_ipq9650_driver = {
 	.driver = {
 		.name   = "qcom,gcc-ipq9650",
 		.of_match_table = gcc_ipq9650_match_table,
+		.sync_state = icc_sync_state,
 	},
 };
 
diff --git a/drivers/clk/qcom/gcc-kaanapali.c b/drivers/clk/qcom/gcc-kaanapali.c
index 842c1a70c69168..c9473fb79ccc91 100644
--- a/drivers/clk/qcom/gcc-kaanapali.c
+++ b/drivers/clk/qcom/gcc-kaanapali.c
@@ -3464,8 +3464,6 @@ static const u32 gcc_kaanapali_critical_cbcrs[] = {
 	0x9f004, /* GCC_EVA_AHB_CLK */
 	0x9f024, /* GCC_EVA_XO_CLK */
 	0x71004, /* GCC_GPU_CFG_AHB_CLK */
-	0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */
-	0x52010, /* GCC_PCIE_RSCC_XO_CLK */
 	0x32004, /* GCC_VIDEO_AHB_CLK */
 	0x32040, /* GCC_VIDEO_XO_CLK */
 };
@@ -3480,6 +3478,10 @@ static const struct regmap_config gcc_kaanapali_regmap_config = {
 
 static void clk_kaanapali_regs_configure(struct device *dev, struct regmap *regmap)
 {
+	/* Keep clocks always enabled */
+	regmap_update_bits(regmap, 0x52010, BIT(20), BIT(20)); /* GCC_PCIE_RSCC_CFG_AHB_CLK */
+	regmap_update_bits(regmap, 0x52010, BIT(21), BIT(21)); /* GCC_PCIE_RSCC_XO_CLK */
+
 	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
 	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
 }
@@ -3500,6 +3502,7 @@ static const struct qcom_cc_desc gcc_kaanapali_desc = {
 	.num_resets = ARRAY_SIZE(gcc_kaanapali_resets),
 	.gdscs = gcc_kaanapali_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_kaanapali_gdscs),
+	.use_rpm = true,
 	.driver_data = &gcc_kaanapali_driver_data,
 };
 
@@ -3522,17 +3525,7 @@ static struct platform_driver gcc_kaanapali_driver = {
 	},
 };
 
-static int __init gcc_kaanapali_init(void)
-{
-	return platform_driver_register(&gcc_kaanapali_driver);
-}
-subsys_initcall(gcc_kaanapali_init);
-
-static void __exit gcc_kaanapali_exit(void)
-{
-	platform_driver_unregister(&gcc_kaanapali_driver);
-}
-module_exit(gcc_kaanapali_exit);
+subsys_platform_driver(gcc_kaanapali_driver);
 
 MODULE_DESCRIPTION("QTI GCC Kaanapali Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-kuno.c b/drivers/clk/qcom/gcc-kuno.c
new file mode 100644
index 00000000000000..55a2fbc74268c5
--- /dev/null
+++ b/drivers/clk/qcom/gcc-kuno.c
@@ -0,0 +1,1473 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,kuno-gcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "clk-regmap-phy-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+/* Need to match the order of clocks in DT binding */
+enum {
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK,
+	DT_PCIE_PIPE_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_GPLL0_OUT_EVEN,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL2_OUT_MAIN,
+	P_GPLL3_OUT_MAIN,
+	P_GPLL4_OUT_EVEN,
+	P_GPLL4_OUT_MAIN,
+	P_SLEEP_CLK,
+};
+
+static struct clk_alpha_pll gpll0 = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
+	.clkr = {
+		.enable_reg = 0x7d000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_gpll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll2 = {
+	.offset = 0x2000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
+	.clkr = {
+		.enable_reg = 0x7d000,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll2",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_evo_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gpll3 = {
+	.offset = 0x3000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
+	.clkr = {
+		.enable_reg = 0x7d000,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll3",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_evo_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gpll4 = {
+	.offset = 0x4000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
+	.clkr = {
+		.enable_reg = 0x7d000,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll4_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll4_out_even = {
+	.offset = 0x4000,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_gpll4_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll4_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll4_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll4.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_lucid_evo_ops,
+	},
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_EVEN, 2 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4_out_even.clkr.hw },
+	{ .hw = &gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_EVEN, 2 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4_out_even.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+	{ .hw = &gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_EVEN, 2 },
+	{ P_GPLL4_OUT_MAIN, 3 },
+	{ P_GPLL2_OUT_MAIN, 4 },
+	{ P_GPLL3_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4_out_even.clkr.hw },
+	{ .hw = &gpll4.clkr.hw },
+	{ .hw = &gpll2.clkr.hw },
+	{ .hw = &gpll3.clkr.hw },
+	{ .hw = &gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_6[] = {
+	{ P_BI_TCXO, 2 },
+};
+
+static struct clk_regmap_mux gcc_pcie_aux_clk_src = {
+	.reg = 0x5308c,
+	.shift = 0,
+	.width = 2,
+	.parent_map = gcc_parent_map_6,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_aux_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_mux_closest_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie_pipe_clk_src = {
+	.reg = 0x53070,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_emac0_phy_aux_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_emac0_phy_aux_clk_src = {
+	.cmd_rcgr = 0x7102c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac0_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_emac0_ptp_clk_src[] = {
+	F(62500000, P_GPLL4_OUT_EVEN, 4, 0, 0),
+	F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(125000000, P_GPLL2_OUT_MAIN, 4, 0, 0),
+	F(230400000, P_GPLL3_OUT_MAIN, 3.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_emac0_ptp_clk_src = {
+	.cmd_rcgr = 0x71064,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_emac0_ptp_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac0_ptp_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_emac0_rgmii_clk_src[] = {
+	F(41666667, P_GPLL4_OUT_EVEN, 6, 0, 0),
+	F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(125000000, P_GPLL2_OUT_MAIN, 4, 0, 0),
+	F(250000000, P_GPLL2_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_emac0_rgmii_clk_src = {
+	.cmd_rcgr = 0x7104c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_emac0_rgmii_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_emac0_rgmii_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(41666667, P_GPLL4_OUT_EVEN, 6, 0, 0),
+	F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_gp1_clk_src = {
+	.cmd_rcgr = 0x47004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp1_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp2_clk_src = {
+	.cmd_rcgr = 0x48004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp2_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp3_clk_src = {
+	.cmd_rcgr = 0x49004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp3_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_aux_phy_clk_src = {
+	.cmd_rcgr = 0x53074,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_aux_phy_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_rchng_phy_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(83333333, P_GPLL4_OUT_EVEN, 3, 0, 0),
+	F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_rchng_phy_clk_src = {
+	.cmd_rcgr = 0x53038,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_pcie_rchng_phy_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_rchng_phy_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pdm2_clk_src = {
+	.cmd_rcgr = 0x34010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pdm2_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pdm2_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = {
+	F(7372800, P_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(62500000, P_GPLL4_OUT_EVEN, 4, 0, 0),
+	F(64000000, P_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.ops = &clk_rcg2_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = {
+	.cmd_rcgr = 0x6c004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.ops = &clk_rcg2_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = {
+	.cmd_rcgr = 0x6c13c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s2_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.ops = &clk_rcg2_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = {
+	.cmd_rcgr = 0x6c274,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.ops = &clk_rcg2_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = {
+	.cmd_rcgr = 0x6c3ac,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap0_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.ops = &clk_rcg2_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = {
+	.cmd_rcgr = 0x6c4e4,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src,
+	.clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_EVEN, 12, 1, 1),
+	F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(83333333, P_GPLL4_OUT_EVEN, 3, 0, 0),
+	F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
+	.cmd_rcgr = 0x6a01c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_sdcc4_apps_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc4_apps_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb20_master_clk_src[] = {
+	F(50000000, P_GPLL4_OUT_EVEN, 5, 0, 0),
+	F(60000000, P_GPLL0_OUT_EVEN, 5, 0, 0),
+	F(120000000, P_GPLL0_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb20_master_clk_src = {
+	.cmd_rcgr = 0x27048,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb20_master_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb20_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb20_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x2702c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_emac0_phy_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb20_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb20_mock_utmi_postdiv_clk_src = {
+	.reg = 0x27044,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb20_mock_utmi_postdiv_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb20_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x37004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x37004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_boot_rom_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_axi_clk = {
+	.halt_reg = 0x71018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x71018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_phy_aux_clk = {
+	.halt_reg = 0x71028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x71028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_ptp_clk = {
+	.halt_reg = 0x71044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x71044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_ptp_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_ptp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_rgmii_clk = {
+	.halt_reg = 0x71048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x71048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_rgmii_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_emac0_rgmii_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac0_slv_ahb_clk = {
+	.halt_reg = 0x71024,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x71024,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac0_slv_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_emac_0_clkref_en = {
+	.halt_reg = 0x94004,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x94004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_emac_0_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x47000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x47000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x48000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x48000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x49000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x49000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_clkref_en = {
+	.halt_reg = 0x94000,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x94000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_aux_clk = {
+	.halt_reg = 0x53054,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x53054,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_cfg_ahb_clk = {
+	.halt_reg = 0x53034,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x53034,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_mstr_axi_clk = {
+	.halt_reg = 0x53028,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x53028,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_pipe_clk = {
+	.halt_reg = 0x53064,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x53064,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(17),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_rchng_phy_clk = {
+	.halt_reg = 0x53050,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x53050,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_rchng_phy_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_rchng_phy_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_sleep_clk = {
+	.halt_reg = 0x53060,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x53060,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_sleep_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_phy_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_slv_axi_clk = {
+	.halt_reg = 0x5301c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_slv_q2a_axi_clk = {
+	.halt_reg = 0x53018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x53018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d010,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x3400c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3400c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pdm2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x34004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x34004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_xo4_clk = {
+	.halt_reg = 0x34008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x34008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_xo4_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = {
+	.halt_reg = 0x2d018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_core_clk = {
+	.halt_reg = 0x2d008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s0_clk = {
+	.halt_reg = 0x6c130,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s1_clk = {
+	.halt_reg = 0x6c268,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(17),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s2_clk = {
+	.halt_reg = 0x6c3a0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s3_clk = {
+	.halt_reg = 0x6c4d8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap0_s4_clk = {
+	.halt_reg = 0x6c610,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap0_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap0_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = {
+	.halt_reg = 0x2d000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2d000,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_0_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = {
+	.halt_reg = 0x2d004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2d004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7d008,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_0_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_ahb_clk = {
+	.halt_reg = 0x6a010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6a010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc4_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_apps_clk = {
+	.halt_reg = 0x6a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x6a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc4_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc4_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_snoc_cnoc_usb3_clk = {
+	.halt_reg = 0x27060,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x27060,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x27060,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_snoc_cnoc_usb3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_usb_sf_axi_clk = {
+	.halt_reg = 0x27064,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x27064,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x27064,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sys_noc_usb_sf_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_master_clk = {
+	.halt_reg = 0x27018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x27018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb20_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_mock_utmi_clk = {
+	.halt_reg = 0x27028,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x27028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb20_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb20_mock_utmi_postdiv_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb20_sleep_clk = {
+	.halt_reg = 0x27024,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x27024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb20_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb2_clkref_en = {
+	.halt_reg = 0x94008,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x94008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb2_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_clkref_en = {
+	.halt_reg = 0x9400c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x9400c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_clkref_en",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_phy_cfg_ahb2phy_clk = {
+	.halt_reg = 0x29004,
+	.halt_check = BRANCH_HALT,
+	.hwcg_reg = 0x29004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x29004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb_phy_cfg_ahb2phy_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct gdsc gcc_emac0_gdsc = {
+	.gdscr = 0x71004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_emac0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_pcie_gdsc = {
+	.gdscr = 0x53004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_pcie_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb20_gdsc = {
+	.gdscr = 0x27004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_usb20_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gcc_kuno_clocks[] = {
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_EMAC0_AXI_CLK] = &gcc_emac0_axi_clk.clkr,
+	[GCC_EMAC0_PHY_AUX_CLK] = &gcc_emac0_phy_aux_clk.clkr,
+	[GCC_EMAC0_PHY_AUX_CLK_SRC] = &gcc_emac0_phy_aux_clk_src.clkr,
+	[GCC_EMAC0_PTP_CLK] = &gcc_emac0_ptp_clk.clkr,
+	[GCC_EMAC0_PTP_CLK_SRC] = &gcc_emac0_ptp_clk_src.clkr,
+	[GCC_EMAC0_RGMII_CLK] = &gcc_emac0_rgmii_clk.clkr,
+	[GCC_EMAC0_RGMII_CLK_SRC] = &gcc_emac0_rgmii_clk_src.clkr,
+	[GCC_EMAC0_SLV_AHB_CLK] = &gcc_emac0_slv_ahb_clk.clkr,
+	[GCC_EMAC_0_CLKREF_EN] = &gcc_emac_0_clkref_en.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr,
+	[GCC_PCIE_0_CLKREF_EN] = &gcc_pcie_0_clkref_en.clkr,
+	[GCC_PCIE_AUX_CLK] = &gcc_pcie_aux_clk.clkr,
+	[GCC_PCIE_AUX_CLK_SRC] = &gcc_pcie_aux_clk_src.clkr,
+	[GCC_PCIE_AUX_PHY_CLK_SRC] = &gcc_pcie_aux_phy_clk_src.clkr,
+	[GCC_PCIE_CFG_AHB_CLK] = &gcc_pcie_cfg_ahb_clk.clkr,
+	[GCC_PCIE_MSTR_AXI_CLK] = &gcc_pcie_mstr_axi_clk.clkr,
+	[GCC_PCIE_PIPE_CLK] = &gcc_pcie_pipe_clk.clkr,
+	[GCC_PCIE_PIPE_CLK_SRC] = &gcc_pcie_pipe_clk_src.clkr,
+	[GCC_PCIE_RCHNG_PHY_CLK] = &gcc_pcie_rchng_phy_clk.clkr,
+	[GCC_PCIE_RCHNG_PHY_CLK_SRC] = &gcc_pcie_rchng_phy_clk_src.clkr,
+	[GCC_PCIE_SLEEP_CLK] = &gcc_pcie_sleep_clk.clkr,
+	[GCC_PCIE_SLV_AXI_CLK] = &gcc_pcie_slv_axi_clk.clkr,
+	[GCC_PCIE_SLV_Q2A_AXI_CLK] = &gcc_pcie_slv_q2a_axi_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr,
+	[GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr,
+	[GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr,
+	[GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr,
+	[GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr,
+	[GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr,
+	[GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr,
+	[GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr,
+	[GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr,
+	[GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr,
+	[GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr,
+	[GCC_SNOC_CNOC_USB3_CLK] = &gcc_snoc_cnoc_usb3_clk.clkr,
+	[GCC_SYS_NOC_USB_SF_AXI_CLK] = &gcc_sys_noc_usb_sf_axi_clk.clkr,
+	[GCC_USB20_MASTER_CLK] = &gcc_usb20_master_clk.clkr,
+	[GCC_USB20_MASTER_CLK_SRC] = &gcc_usb20_master_clk_src.clkr,
+	[GCC_USB20_MOCK_UTMI_CLK] = &gcc_usb20_mock_utmi_clk.clkr,
+	[GCC_USB20_MOCK_UTMI_CLK_SRC] = &gcc_usb20_mock_utmi_clk_src.clkr,
+	[GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb20_mock_utmi_postdiv_clk_src.clkr,
+	[GCC_USB20_SLEEP_CLK] = &gcc_usb20_sleep_clk.clkr,
+	[GCC_USB2_CLKREF_EN] = &gcc_usb2_clkref_en.clkr,
+	[GCC_USB3_PRIM_CLKREF_EN] = &gcc_usb3_prim_clkref_en.clkr,
+	[GCC_USB_PHY_CFG_AHB2PHY_CLK] = &gcc_usb_phy_cfg_ahb2phy_clk.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL0_OUT_EVEN] = &gpll0_out_even.clkr,
+	[GPLL2] = &gpll2.clkr,
+	[GPLL3] = &gpll3.clkr,
+	[GPLL4] = &gpll4.clkr,
+	[GPLL4_OUT_EVEN] = &gpll4_out_even.clkr,
+};
+
+static const struct qcom_reset_map gcc_kuno_resets[] = {
+	[GCC_EMAC0_BCR] = { 0x71000 },
+	[GCC_PCIE_BCR] = { 0x53000 },
+	[GCC_PCIE_LINK_DOWN_BCR] = { 0x87000 },
+	[GCC_PCIE_NOCSR_COM_PHY_BCR] = { 0x88008 },
+	[GCC_PCIE_PHY_BCR] = { 0x54000 },
+	[GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x88000 },
+	[GCC_PCIE_PHY_COM_BCR] = { 0x88004 },
+	[GCC_PCIE_PHY_NOCSR_COM_PHY_BCR] = { 0x8800c },
+	[GCC_PDM_BCR] = { 0x34000 },
+	[GCC_QUPV3_WRAPPER_0_BCR] = { 0x6c000 },
+	[GCC_QUSB2PHY_BCR] = { 0x2a000 },
+	[GCC_SDCC4_BCR] = { 0x6a000 },
+	[GCC_TCSR_PCIE_BCR] = { 0x84000 },
+	[GCC_USB20_BCR] = { 0x27000 },
+	[GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x29000 },
+};
+
+static struct gdsc *gcc_kuno_gdscs[] = {
+	[GCC_EMAC0_GDSC] = &gcc_emac0_gdsc,
+	[GCC_PCIE_GDSC] = &gcc_pcie_gdsc,
+	[GCC_USB20_GDSC] = &gcc_usb20_gdsc,
+};
+
+static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = {
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src),
+};
+
+static const struct regmap_config gcc_kuno_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1f41f0,
+	.fast_io = true,
+};
+
+static const u32 gcc_kuno_critical_cbcrs[] = {
+	0x3e004, /* GCC_AHB_PCIE_LINK_CLK */
+	0x3e008, /* GCC_XO_PCIE_LINK_CLK */
+};
+
+static const struct qcom_cc_driver_data gcc_kuno_driver_data = {
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_cbcrs = gcc_kuno_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_kuno_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc gcc_kuno_desc = {
+	.config = &gcc_kuno_regmap_config,
+	.clks = gcc_kuno_clocks,
+	.num_clks = ARRAY_SIZE(gcc_kuno_clocks),
+	.resets = gcc_kuno_resets,
+	.num_resets = ARRAY_SIZE(gcc_kuno_resets),
+	.gdscs = gcc_kuno_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_kuno_gdscs),
+	.use_rpm = true,
+	.driver_data = &gcc_kuno_driver_data,
+};
+
+static const struct of_device_id gcc_kuno_match_table[] = {
+	{ .compatible = "qcom,kuno-gcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_kuno_match_table);
+
+static int gcc_kuno_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_kuno_desc);
+}
+
+static struct platform_driver gcc_kuno_driver = {
+	.probe = gcc_kuno_probe,
+	.driver = {
+		.name = "gcc-kuno",
+		.of_match_table = gcc_kuno_match_table,
+	},
+};
+
+subsys_platform_driver(gcc_kuno_driver);
+
+MODULE_DESCRIPTION("QTI GCC Kuno Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-milos.c b/drivers/clk/qcom/gcc-milos.c
index 4219af2879d934..10a5f3ab0d42ae 100644
--- a/drivers/clk/qcom/gcc-milos.c
+++ b/drivers/clk/qcom/gcc-milos.c
@@ -3208,17 +3208,7 @@ static struct platform_driver gcc_milos_driver = {
 	},
 };
 
-static int __init gcc_milos_init(void)
-{
-	return platform_driver_register(&gcc_milos_driver);
-}
-subsys_initcall(gcc_milos_init);
-
-static void __exit gcc_milos_exit(void)
-{
-	platform_driver_unregister(&gcc_milos_driver);
-}
-module_exit(gcc_milos_exit);
+subsys_platform_driver(gcc_milos_driver);
 
 MODULE_DESCRIPTION("QTI GCC Milos Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-msm8939.c b/drivers/clk/qcom/gcc-msm8939.c
index ffd7f14fcbaf8b..3900a06189ff96 100644
--- a/drivers/clk/qcom/gcc-msm8939.c
+++ b/drivers/clk/qcom/gcc-msm8939.c
@@ -3665,6 +3665,7 @@ static struct clk_branch gcc_venus0_vcodec0_clk = {
 
 static struct clk_branch gcc_venus0_core0_vcodec0_clk = {
 	.halt_reg = 0x4c02c,
+	.halt_check = BRANCH_HALT,
 	.clkr = {
 		.enable_reg = 0x4c02c,
 		.enable_mask = BIT(0),
@@ -3682,6 +3683,7 @@ static struct clk_branch gcc_venus0_core0_vcodec0_clk = {
 
 static struct clk_branch gcc_venus0_core1_vcodec0_clk = {
 	.halt_reg = 0x4c034,
+	.halt_check = BRANCH_HALT,
 	.clkr = {
 		.enable_reg = 0x4c034,
 		.enable_mask = BIT(0),
@@ -3754,6 +3756,7 @@ static struct gdsc venus_core0_gdsc = {
 	.pd = {
 		.name = "venus_core0",
 	},
+	.flags = HW_CTRL_TRIGGER,
 	.pwrsts = PWRSTS_OFF_ON,
 };
 
@@ -3762,6 +3765,7 @@ static struct gdsc venus_core1_gdsc = {
 	.pd = {
 		.name = "venus_core1",
 	},
+	.flags = HW_CTRL_TRIGGER,
 	.pwrsts = PWRSTS_OFF_ON,
 };
 
diff --git a/drivers/clk/qcom/gcc-msm8952.c b/drivers/clk/qcom/gcc-msm8952.c
new file mode 100644
index 00000000000000..de2811307a8681
--- /dev/null
+++ b/drivers/clk/qcom/gcc-msm8952.c
@@ -0,0 +1,3548 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2026 Muzaffer Kadir <muzafferkadir@mainlining.org>
+ *
+ * Based on gcc-msm8939.c:
+ *   Copyright 2020 Linaro Limited
+ * adapted with data from clock-gcc-8952.c in Qualcomm's msm-3.10 release
+ *   Copyright (c) 2014-2016, The Linux Foundation. All rights reserved.
+ * with parts taken from gcc-msm8976.c:
+ *   Copyright (c) 2016-2021, AngeloGioacchino Del Regno
+ *                     <angelogioacchino.delregno@somainline.org>
+ *   Copyright (c) 2021, Konrad Dybcio <konrad.dybcio@somainline.org>
+ *                     Marijn Suijten <marijn.suijten@somainline.org>
+ * with some resets copied from gcc-msm8953:
+ *   Copyright (c) 2021, The Linux Foundation. All rights reserved
+ */
+
+#include <linux/kernel.h>
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/clock/qcom,gcc-msm8952.h>
+
+#include "common.h"
+#include "clk-regmap.h"
+#include "clk-pll.h"
+#include "clk-alpha-pll.h"
+#include "clk-rcg.h"
+#include "clk-branch.h"
+#include "reset.h"
+#include "gdsc.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_DSI0PLL,
+	DT_DSI0PLL_BYTE,
+};
+
+enum {
+	P_XO,
+	P_XO_A,
+	P_GPLL0,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL0_AUX,
+	P_GPLL3,
+	P_GPLL4,
+	P_GPLL6,
+	P_GPLL6_AUX,
+	P_GPLL6_OUT_MAIN,
+	P_DSI0_PHYPLL_BYTE,
+	P_DSI0_PHYPLL_DSI,
+};
+
+static const struct clk_parent_data gcc_xo_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static struct clk_pll gpll0 = {
+	.l_reg = 0x21004,
+	.m_reg = 0x21008,
+	.n_reg = 0x2100c,
+	.config_reg = 0x21014,
+	.mode_reg = 0x21000,
+	.status_reg = 0x2101c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll0",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll0_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(0),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll0_vote",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static const struct pll_vco gpll3_p_vco[] = {
+	{ 700000000, 1400000000, 0 },
+};
+
+static const struct alpha_pll_config gpll3_early_config = {
+	.l = 63,
+	.config_ctl_val = 0x4001055b,
+	.main_output_mask = BIT(0),
+	.test_ctl_val = 0x0,
+	.test_ctl_mask = GENMASK(31, 0),
+	.test_ctl_hi_val = 0x40000600,
+	.test_ctl_hi_mask = GENMASK(31, 0),
+	.alpha_en_mask = BIT(24),
+	.post_div_val = BIT(8),
+	.post_div_mask = GENMASK(11, 8),
+};
+
+static struct clk_alpha_pll gpll3_early = {
+	.offset = 0x22000,
+	.config = &gpll3_early_config,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.vco_table = gpll3_p_vco,
+	.num_vco = ARRAY_SIZE(gpll3_p_vco),
+	.flags = SUPPORTS_DYNAMIC_UPDATE,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll3_early",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_alpha_pll_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll3 = {
+	.offset = 0x22000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll3",
+		.parent_hws = (const struct clk_hw*[]){
+			&gpll3_early.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_pll gpll4 = {
+	.l_reg = 0x24004,
+	.m_reg = 0x24008,
+	.n_reg = 0x2400c,
+	.config_reg = 0x24018,
+	.mode_reg = 0x24000,
+	.status_reg = 0x24000,
+	.status_bit = 30,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll4_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(5),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll4_vote",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll4.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static struct clk_pll gpll6 = {
+	.l_reg = 0x37004,
+	.m_reg = 0x37008,
+	.n_reg = 0x3700c,
+	.config_reg = 0x37014,
+	.mode_reg = 0x37000,
+	.status_reg = 0x3701c,
+	.status_bit = 17,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll6",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_pll_ops,
+	},
+};
+
+static struct clk_regmap gpll6_vote = {
+	.enable_reg = 0x45000,
+	.enable_mask = BIT(7),
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll6_vote",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll6.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_pll_vote_ops,
+	},
+};
+
+static const struct parent_map gcc_xo_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_out_main_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_out_main_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_a_gpll0_map[] = {
+	{ P_XO_A, 0 },
+	{ P_GPLL0, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_a_gpll0_parent_data[] = {
+	{ .index = DT_BI_TCXO_AO },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll3_gpll6a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL3, 2 },
+	{ P_GPLL6_AUX, 3 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll3_gpll6a_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll3.clkr.hw },
+	{ .hw = &gpll6_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll4_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL4, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll4_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0_vote.hw },
+	{ .hw = &gpll4_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll6_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL6, 2 },
+	{ P_GPLL0, 1 },
+};
+
+static const struct parent_map gcc_xo_gpll6_out_main_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL6_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_gpll6_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll6_vote.hw },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_xo_gpll0a_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_AUX, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0a_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct clk_parent_data gcc_gpll0_parent_data[] = {
+	{ .hw = &gpll0_vote.hw },
+};
+
+static const struct parent_map gcc_gpll0_map[] = {
+	{ P_GPLL0, 1 },
+};
+
+static const struct parent_map gcc_xo_map[] = {
+	{ P_XO, 0, },
+};
+
+static const struct parent_map gcc_xo_dsibyte_map[] = {
+	{ P_XO, 0 },
+	{ P_DSI0_PHYPLL_BYTE, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_dsibyte_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_DSI0PLL_BYTE },
+};
+
+static const struct parent_map gcc_xo_dsiphy_map[] = {
+	{ P_XO, 0 },
+	{ P_DSI0_PHYPLL_DSI, 1 },
+};
+
+static const struct clk_parent_data gcc_xo_dsiphy_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_DSI0PLL },
+};
+
+static const struct parent_map gcc_xo_gpll0_dsiphy_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_DSI0_PHYPLL_DSI, 2 },
+};
+
+static const struct clk_parent_data gcc_xo_gpll0_dsiphy_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll0_vote.hw },
+	{ .index = DT_DSI0PLL },
+};
+
+static const struct clk_parent_data gcc_xo_gpll6_out_main_parent_data[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpll6_vote.hw },
+};
+
+static const struct freq_tbl ftbl_gcc_camss_top_ahb_clk_src[] = {
+	F(40000000, P_GPLL0, 10, 1, 2),
+	F(61540000, P_GPLL0, 13, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 camss_top_ahb_clk_src = {
+	.cmd_rcgr = 0x5a000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_top_ahb_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_top_ahb_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_apss_ahb_clk[] = {
+	F(19200000, P_XO_A, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 apss_ahb_clk_src = {
+	.cmd_rcgr = 0x46000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_a_gpll0_map,
+	.freq_tbl = ftbl_apss_ahb_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "apss_ahb_clk_src",
+		.parent_data = gcc_xo_a_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_a_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+		/*
+		 * This clock allows the CPUs to communicate with
+		 * the rest of the SoC. Without it, the brain will
+		 * operate without the rest of the body.
+		 */
+		.flags = CLK_IS_CRITICAL,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_csi0_1_2_clk[] = {
+	F(100000000, P_GPLL0, 8, 0,	0),
+	F(160000000, P_GPLL0, 5, 0,	0),
+	F(200000000, P_GPLL0, 4, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 csi0_clk_src = {
+	.cmd_rcgr = 0x4e020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1_2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi0_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi1_clk_src = {
+	.cmd_rcgr = 0x4f020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1_2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi1_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi2_clk_src = {
+	.cmd_rcgr = 0x3c020,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1_2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi2_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_oxili_gfx3d_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(228570000, P_GPLL0, 3.5, 0, 0),
+	F(240000000, P_GPLL6_AUX, 4.5, 0, 0),
+	F(266670000, P_GPLL0, 3, 0, 0),
+	F(400000000, P_GPLL0, 2, 0, 0),
+	F(465000000, P_GPLL3, 1, 0, 0),
+	F(500000000, P_GPLL3, 1, 0, 0),
+	F(550000000, P_GPLL3, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gfx3d_clk_src = {
+	.cmd_rcgr = 0x59000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll3_gpll6a_map,
+	.freq_tbl = ftbl_gcc_oxili_gfx3d_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gfx3d_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll3_gpll6a_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll3_gpll6a_parent_data),
+		.ops = &clk_rcg2_ops,
+		.flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_vfe0_clk[] = {
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(133330000, P_GPLL0, 6, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(177780000, P_GPLL0, 4.5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(266670000, P_GPLL0, 3, 0, 0),
+	F(308570000, P_GPLL6, 3.5, 0, 0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	F(360000000, P_GPLL6, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 vfe0_clk_src = {
+	.cmd_rcgr = 0x58000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_map,
+	.freq_tbl = ftbl_gcc_camss_vfe0_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vfe0_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 vfe1_clk_src = {
+	.cmd_rcgr = 0x58054,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_map,
+	.freq_tbl = ftbl_gcc_camss_vfe0_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vfe1_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_blsp1_qup1_6_i2c_apps_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0200c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_blsp1_qup1_6_spi_apps_clk[] = {
+	F(960000, P_XO, 10, 1, 2),
+	F(4800000, P_XO, 4, 0, 0),
+	F(9600000, P_XO, 2, 0, 0),
+	F(16000000, P_GPLL0, 10, 1, 5),
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0, 16, 1, 2),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x02024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x03000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x03014,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup2_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x04000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x04024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup3_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x05000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x05024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup4_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_blsp1_uart1_6_apps_clk[] = {
+	F(3686400, P_GPLL0, 1, 72, 15625),
+	F(7372800, P_GPLL0, 1, 144, 15625),
+	F(14745600, P_GPLL0, 1, 288, 15625),
+	F(16000000, P_GPLL0, 10, 1, 5),
+	F(19200000, P_XO, 1, 0, 0),
+	F(24000000, P_GPLL0, 1, 3, 100),
+	F(25000000, P_GPLL0, 16, 1, 2),
+	F(32000000, P_GPLL0, 1, 1, 25),
+	F(40000000, P_GPLL0, 1, 1, 20),
+	F(46400000, P_GPLL0, 1, 29, 500),
+	F(48000000, P_GPLL0, 1, 3, 50),
+	F(51200000, P_GPLL0, 1, 8, 125),
+	F(56000000, P_GPLL0, 1, 7, 100),
+	F(58982400, P_GPLL0, 1, 1152, 15625),
+	F(60000000, P_GPLL0, 1, 3, 40),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x02044,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart1_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x03034,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart2_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0c00c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup1_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup1_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0c024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup1_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0d000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup2_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup2_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0d014,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup2_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0f000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup3_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup3_spi_apps_clk_src = {
+	.cmd_rcgr = 0x0f024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup3_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x18000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup4_i2c_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_qup4_spi_apps_clk_src = {
+	.cmd_rcgr = 0x18024,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_qup4_spi_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart1_apps_clk_src = {
+	.cmd_rcgr = 0x0c044,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart1_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp2_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x0d034,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp2_uart2_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_cci_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(37500000, P_GPLL0_AUX, 1, 3, 64),
+	{ }
+};
+
+static struct clk_rcg2 cci_clk_src = {
+	.cmd_rcgr = 0x51000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_map,
+	.freq_tbl = ftbl_gcc_camss_cci_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "cci_clk_src",
+		.parent_data = gcc_xo_gpll0a_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0a_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_camss_gp0_1_clk_src[] = {
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 camss_gp0_clk_src = {
+	.cmd_rcgr = 0x54000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_gpll0_map,
+	.freq_tbl = ftbl_camss_gp0_1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_gp0_clk_src",
+		.parent_data = gcc_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 camss_gp1_clk_src = {
+	.cmd_rcgr = 0x55000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_gpll0_map,
+	.freq_tbl = ftbl_camss_gp0_1_clk_src,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "camss_gp1_clk_src",
+		.parent_data = gcc_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_jpeg0_clk[] = {
+	F(133330000, P_GPLL0, 6, 0,	0),
+	F(266670000, P_GPLL0, 3, 0,	0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 jpeg0_clk_src = {
+	.cmd_rcgr = 0x57000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_jpeg0_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "jpeg0_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_mclk0_1_2_clk[] = {
+	F(6150000, P_GPLL0, 1, 1, 130),
+	F(8000000, P_GPLL0, 1, 1, 100),
+	F(24000000, P_GPLL6, 1, 1, 45),
+	F(66670000, P_GPLL0, 12, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 mclk0_clk_src = {
+	.cmd_rcgr = 0x52000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_map,
+	.freq_tbl = ftbl_gcc_camss_mclk0_1_2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk0_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 mclk1_clk_src = {
+	.cmd_rcgr = 0x53000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_map,
+	.freq_tbl = ftbl_gcc_camss_mclk0_1_2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk1_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 mclk2_clk_src = {
+	.cmd_rcgr = 0x5c000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_map,
+	.freq_tbl = ftbl_gcc_camss_mclk0_1_2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mclk2_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_csi0_1phytimer_clk[] = {
+	F(100000000, P_GPLL0, 8, 0,	0),
+	F(160000000, P_GPLL0, 5, 0,	0),
+	F(200000000, P_GPLL0, 4, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 csi0phytimer_clk_src = {
+	.cmd_rcgr = 0x4e000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1phytimer_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi0phytimer_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 csi1phytimer_clk_src = {
+	.cmd_rcgr = 0x4f000,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_camss_csi0_1phytimer_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "csi1phytimer_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_camss_cpp_clk[] = {
+	F(133330000, P_GPLL0, 6, 0, 0),
+	F(180000000, P_GPLL6, 6, 0, 0),
+	F(266670000, P_GPLL0, 3, 0, 0),
+	F(308570000, P_GPLL6, 3.5, 0, 0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cpp_clk_src = {
+	.cmd_rcgr = 0x58018,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_map,
+	.freq_tbl = ftbl_gcc_camss_cpp_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "cpp_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_crypto_clk[] = {
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 crypto_clk_src = {
+	.cmd_rcgr = 0x16004,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_crypto_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "crypto_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+/*
+ * This is a frequency table for "General Purpose" clocks.
+ * These clocks can be muxed to the SoC pins and may be used by
+ * external devices. They're often used as PWM source.
+ *
+ * Please note that MND divider must be enabled for duty-cycle
+ * control to be possible. (M != N) Also since D register is configured
+ * with a value multiplied by 2, and duty cycle is calculated as
+ *                             (2 * D) % 2^W
+ *                DutyCycle = ----------------
+ *                              2 * (N % 2^W)
+ * (where W = .mnd_width)
+ * N must be half or less than maximum value for the register.
+ * Otherwise duty-cycle control would be limited.
+ * (e.g. for 8-bit NMD N should be less than 128)
+ */
+static const struct freq_tbl ftbl_gcc_gp1_3_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gp1_clk_src = {
+	.cmd_rcgr = 0x08004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_map,
+	.freq_tbl = ftbl_gcc_gp1_3_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp1_clk_src",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp2_clk_src = {
+	.cmd_rcgr = 0x09004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_map,
+	.freq_tbl = ftbl_gcc_gp1_3_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp2_clk_src",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp3_clk_src = {
+	.cmd_rcgr = 0x0a004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_map,
+	.freq_tbl = ftbl_gcc_gp1_3_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp3_clk_src",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 byte0_clk_src = {
+	.cmd_rcgr = 0x4d044,
+	.hid_width = 5,
+	.parent_map = gcc_xo_dsibyte_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "byte0_clk_src",
+		.parent_data = gcc_xo_dsibyte_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_dsibyte_parent_data),
+		.ops = &clk_byte2_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_mdss_esc_clk[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 esc0_clk_src = {
+	.cmd_rcgr = 0x4d05c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_map,
+	.freq_tbl = ftbl_gcc_mdss_esc_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "esc0_clk_src",
+		.parent_data = gcc_xo_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_mdss_mdp_clk[] = {
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(145450000, P_GPLL0, 5.5, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(177780000, P_GPLL0, 4.5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(266670000, P_GPLL0, 3, 0, 0),
+	F(320000000, P_GPLL0, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 mdp_clk_src = {
+	.cmd_rcgr = 0x4d014,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_dsiphy_map,
+	.freq_tbl = ftbl_gcc_mdss_mdp_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "mdp_clk_src",
+		.parent_data = gcc_xo_gpll0_dsiphy_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_dsiphy_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 pclk0_clk_src = {
+	.cmd_rcgr = 0x4d000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_dsiphy_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pclk0_clk_src",
+		.parent_data = gcc_xo_dsiphy_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_dsiphy_parent_data),
+		.ops = &clk_pixel_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_mdss_vsync_clk[] = {
+	F(19200000, P_XO, 1, 0,	0),
+	{ }
+};
+
+static struct clk_rcg2 vsync_clk_src = {
+	.cmd_rcgr = 0x4d02c,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_map,
+	.freq_tbl = ftbl_gcc_mdss_vsync_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vsync_clk_src",
+		.parent_data = gcc_xo_gpll0a_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0a_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk[] = {
+	F(64000000, P_GPLL0, 12.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 pdm2_clk_src = {
+	.cmd_rcgr = 0x44010,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_pdm2_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pdm2_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 10, 1, 4),
+	F(25000000, P_GPLL0, 16, 1, 2),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(177770000, P_GPLL0, 4.5, 0, 0),
+	F(192000000, P_GPLL4, 6, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(384000000, P_GPLL4, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x42004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll4_map,
+	.freq_tbl = ftbl_gcc_sdcc1_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc1_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll4_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll4_parent_data),
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(20000000, P_GPLL0, 10, 1, 4),
+	F(25000000, P_GPLL0, 16, 1, 2),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(177770000, P_GPLL0, 4.5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x43004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_sdcc2_apps_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc2_apps_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk[] = {
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc1_ice_core_clk_src = {
+	.cmd_rcgr = 0x5d000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_out_main_map,
+	.freq_tbl = ftbl_gcc_sdcc1_ice_core_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc1_ice_core_clk_src",
+		.parent_data = gcc_xo_gpll0_out_main_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_out_main_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ice_core_clk = {
+	.halt_reg = 0x5d014,
+	.clkr = {
+		.enable_reg = 0x5d014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&sdcc1_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb_hs_system_clk[] = {
+	F(57140000, P_GPLL0, 14, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(133330000, P_GPLL0, 6, 0, 0),
+	F(177780000, P_GPLL0, 4.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb_hs_system_clk_src = {
+	.cmd_rcgr = 0x41010,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.freq_tbl = ftbl_gcc_usb_hs_system_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb_hs_system_clk_src",
+		.parent_data = gcc_xo_gpll0_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb_fs_system_clk[] = {
+	F(64000000, P_GPLL0_AUX, 12.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb_fs_system_clk_src = {
+	.cmd_rcgr = 0x3f010,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0a_map,
+	.freq_tbl = ftbl_gcc_usb_fs_system_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb_fs_system_clk_src",
+		.parent_data = gcc_xo_gpll0a_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0a_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb_fs_ic_clk[] = {
+	F(60000000, P_GPLL6_OUT_MAIN, 9, 1, 2),
+	{ }
+};
+
+static struct clk_rcg2 usb_fs_ic_clk_src = {
+	.cmd_rcgr = 0x3f034,
+	.hid_width = 5,
+	.mnd_width = 8,
+	.parent_map = gcc_xo_gpll6_out_main_map,
+	.freq_tbl = ftbl_gcc_usb_fs_ic_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb_fs_ic_clk_src",
+		.parent_data = gcc_xo_gpll6_out_main_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll6_out_main_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_venus0_vcodec0_clk[] = {
+	F(133330000, P_GPLL0, 6, 0, 0),
+	F(180000000, P_GPLL6, 6, 0, 0),
+	F(228570000, P_GPLL0, 3.5, 0, 0),
+	F(266670000, P_GPLL0, 3, 0, 0),
+	F(308570000, P_GPLL6, 3.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 vcodec0_clk_src = {
+	.cmd_rcgr = 0x4C000,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_map,
+	.freq_tbl = ftbl_gcc_venus0_vcodec0_clk,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "vcodec0_clk_src",
+		.parent_data = gcc_xo_gpll0_gpll6_parent_data,
+		.num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6_parent_data),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_blsp1_ahb_clk = {
+	.halt_reg = 0x01008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
+	.halt_reg = 0x02008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x02008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup1_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
+	.halt_reg = 0x02004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x02004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_spi_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup1_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
+	.halt_reg = 0x03010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x03010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup2_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
+	.halt_reg = 0x0300c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_spi_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup2_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
+	.halt_reg = 0x04020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x04020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup3_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
+	.halt_reg = 0x0401c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0401c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_spi_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup3_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
+	.halt_reg = 0x05020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x05020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup4_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
+	.halt_reg = 0x0501c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0501c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_spi_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_qup4_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart1_apps_clk = {
+	.halt_reg = 0x0203c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0203c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart1_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_uart1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart2_apps_clk = {
+	.halt_reg = 0x0302c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0302c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart2_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&blsp1_uart2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_ahb_clk = {
+	.halt_reg = 0x0b008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(20),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_i2c_apps_clk = {
+	.halt_reg = 0x0c008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0c008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup1_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup1_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup1_spi_apps_clk = {
+	.halt_reg = 0x0c004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0c004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup1_spi_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup1_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart1_apps_clk = {
+	.halt_reg = 0x0c03c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0c03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart1_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_uart1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_i2c_apps_clk = {
+	.halt_reg = 0x0d010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0d010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup2_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup2_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup2_spi_apps_clk = {
+	.halt_reg = 0x0d00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0d00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup2_spi_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup2_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_uart2_apps_clk = {
+	.halt_reg = 0x0d02c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0d02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_uart2_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_uart2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_i2c_apps_clk = {
+	.halt_reg = 0x0f020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0f020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup3_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup3_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup3_spi_apps_clk = {
+	.halt_reg = 0x0f01c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x0f01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup3_spi_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup3_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_i2c_apps_clk = {
+	.halt_reg = 0x18020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x18020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup4_i2c_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup4_i2c_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp2_qup4_spi_apps_clk = {
+	.halt_reg = 0x1801c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1801c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp2_qup4_spi_apps_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&blsp2_qup4_spi_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x1300c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(7),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_boot_rom_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cci_ahb_clk = {
+	.halt_reg = 0x5101c,
+	.clkr = {
+		.enable_reg = 0x5101c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cci_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cci_clk = {
+	.halt_reg = 0x51018,
+	.clkr = {
+		.enable_reg = 0x51018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cci_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&cci_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0_ahb_clk = {
+	.halt_reg = 0x4e040,
+	.clkr = {
+		.enable_reg = 0x4e040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0_clk = {
+	.halt_reg = 0x4e03c,
+	.clkr = {
+		.enable_reg = 0x4e03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0phy_clk = {
+	.halt_reg = 0x4e048,
+	.clkr = {
+		.enable_reg = 0x4e048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0phy_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0pix_clk = {
+	.halt_reg = 0x4e058,
+	.clkr = {
+		.enable_reg = 0x4e058,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0pix_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0rdi_clk = {
+	.halt_reg = 0x4e050,
+	.clkr = {
+		.enable_reg = 0x4e050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0rdi_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1_ahb_clk = {
+	.halt_reg = 0x4f040,
+	.clkr = {
+		.enable_reg = 0x4f040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1_clk = {
+	.halt_reg = 0x4f03c,
+	.clkr = {
+		.enable_reg = 0x4f03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1phy_clk = {
+	.halt_reg = 0x4f048,
+	.clkr = {
+		.enable_reg = 0x4f048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1phy_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1pix_clk = {
+	.halt_reg = 0x4f058,
+	.clkr = {
+		.enable_reg = 0x4f058,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1pix_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1rdi_clk = {
+	.halt_reg = 0x4f050,
+	.clkr = {
+		.enable_reg = 0x4f050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1rdi_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi2_ahb_clk = {
+	.halt_reg = 0x3c040,
+	.clkr = {
+		.enable_reg = 0x3c040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi2_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi2_clk = {
+	.halt_reg = 0x3c03c,
+	.clkr = {
+		.enable_reg = 0x3c03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi2_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi2phy_clk = {
+	.halt_reg = 0x3c048,
+	.clkr = {
+		.enable_reg = 0x3c048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi2phy_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi2pix_clk = {
+	.halt_reg = 0x3c058,
+	.clkr = {
+		.enable_reg = 0x3c058,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi2pix_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi2rdi_clk = {
+	.halt_reg = 0x3c050,
+	.clkr = {
+		.enable_reg = 0x3c050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi2rdi_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk2_clk = {
+	.halt_reg = 0x5c018,
+	.clkr = {
+		.enable_reg = 0x5c018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_mclk2_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&mclk2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi_vfe0_clk = {
+	.halt_reg = 0x58050,
+	.clkr = {
+		.enable_reg = 0x58050,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi_vfe0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&vfe0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_gp0_clk = {
+	.halt_reg = 0x54018,
+	.clkr = {
+		.enable_reg = 0x54018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_gp0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_gp0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_gp1_clk = {
+	.halt_reg = 0x55018,
+	.clkr = {
+		.enable_reg = 0x55018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_gp1_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_ispif_ahb_clk = {
+	.halt_reg = 0x50004,
+	.clkr = {
+		.enable_reg = 0x50004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_ispif_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_jpeg0_clk = {
+	.halt_reg = 0x57020,
+	.clkr = {
+		.enable_reg = 0x57020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_jpeg0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&jpeg0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_jpeg_ahb_clk = {
+	.halt_reg = 0x57024,
+	.clkr = {
+		.enable_reg = 0x57024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_jpeg_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_jpeg_axi_clk = {
+	.halt_reg = 0x57028,
+	.clkr = {
+		.enable_reg = 0x57028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_jpeg_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk0_clk = {
+	.halt_reg = 0x52018,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_mclk0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&mclk0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_mclk1_clk = {
+	.halt_reg = 0x53018,
+	.clkr = {
+		.enable_reg = 0x53018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_mclk1_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&mclk1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_micro_ahb_clk = {
+	.halt_reg = 0x5600c,
+	.clkr = {
+		.enable_reg = 0x5600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_micro_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi0phytimer_clk = {
+	.halt_reg = 0x4e01c,
+	.clkr = {
+		.enable_reg = 0x4e01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi0phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi0phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi1phytimer_clk = {
+	.halt_reg = 0x4f01c,
+	.clkr = {
+		.enable_reg = 0x4f01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi1phytimer_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&csi1phytimer_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_ahb_clk = {
+	.halt_reg = 0x56004,
+	.clkr = {
+		.enable_reg = 0x56004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_top_ahb_clk = {
+	.halt_reg = 0x5a014,
+	.clkr = {
+		.enable_reg = 0x5a014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_top_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe0_ahb_clk = {
+	.halt_reg = 0x58044,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x58044,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "gcc_camss_vfe0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		}
+	}
+};
+
+static struct clk_branch gcc_camss_vfe0_axi_clk = {
+	.halt_reg = 0x58048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x58048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "gcc_camss_vfe0_axi_clk",
+			.ops = &clk_branch2_ops,
+		}
+	}
+};
+
+static struct clk_branch gcc_camss_cpp_ahb_clk = {
+	.halt_reg = 0x58040,
+	.clkr = {
+		.enable_reg = 0x58040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cpp_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cpp_clk = {
+	.halt_reg = 0x5803c,
+	.clkr = {
+		.enable_reg = 0x5803c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cpp_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&cpp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe0_clk = {
+	.halt_reg = 0x58038,
+	.clkr = {
+		.enable_reg = 0x58038,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_vfe0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&vfe0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_cpp_axi_clk = {
+	.halt_reg = 0x58064,
+	.clkr = {
+		.enable_reg = 0x58064,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_cpp_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_crypto_ahb_clk = {
+	.halt_reg = 0x16024,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_crypto_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_crypto_axi_clk = {
+	.halt_reg = 0x16020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_crypto_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_crypto_clk = {
+	.halt_reg = 0x1601c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_crypto_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&crypto_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_gmem_clk = {
+	.halt_reg = 0x59024,
+	.clkr = {
+		.enable_reg = 0x59024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_gmem_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&gfx3d_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x08000,
+	.clkr = {
+		.enable_reg = 0x08000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp1_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x09000,
+	.clkr = {
+		.enable_reg = 0x09000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp2_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&gp2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x0a000,
+	.clkr = {
+		.enable_reg = 0x0a000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp3_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&gp3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_ahb_clk = {
+	.halt_reg = 0x4d07c,
+	.clkr = {
+		.enable_reg = 0x4d07c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_axi_clk = {
+	.halt_reg = 0x4d080,
+	.clkr = {
+		.enable_reg = 0x4d080,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_byte0_clk = {
+	.halt_reg = 0x4d094,
+	.clkr = {
+		.enable_reg = 0x4d094,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_byte0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&byte0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_esc0_clk = {
+	.halt_reg = 0x4d098,
+	.clkr = {
+		.enable_reg = 0x4d098,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_esc0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&esc0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_mdp_clk = {
+	.halt_reg = 0x4D088,
+	.clkr = {
+		.enable_reg = 0x4D088,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_mdp_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&mdp_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_pclk0_clk = {
+	.halt_reg = 0x4d084,
+	.clkr = {
+		.enable_reg = 0x4d084,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_pclk0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&pclk0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdss_vsync_clk = {
+	.halt_reg = 0x4d090,
+	.clkr = {
+		.enable_reg = 0x4d090,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdss_vsync_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&vsync_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_cfg_ahb_clk = {
+	.halt_reg = 0x49000,
+	.clkr = {
+		.enable_reg = 0x49000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mss_q6_bimc_axi_clk = {
+	.halt_reg = 0x49004,
+	.clkr = {
+		.enable_reg = 0x49004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mss_q6_bimc_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_ahb_clk = {
+	.halt_reg = 0x59028,
+	.clkr = {
+		.enable_reg = 0x59028,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_gfx3d_clk = {
+	.halt_reg = 0x59020,
+	.clkr = {
+		.enable_reg = 0x59020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_gfx3d_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&gfx3d_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x4400c,
+	.clkr = {
+		.enable_reg = 0x4400c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm2_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&pdm2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x44004,
+	.clkr = {
+		.enable_reg = 0x44004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_prng_ahb_clk = {
+	.halt_reg = 0x13004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x45004,
+		.enable_mask = BIT(8),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_prng_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0x4201c,
+	.clkr = {
+		.enable_reg = 0x4201c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x42018,
+	.clkr = {
+		.enable_reg = 0x42018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&sdcc1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x4301c,
+	.clkr = {
+		.enable_reg = 0x4301c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x43018,
+	.clkr = {
+		.enable_reg = 0x43018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&sdcc2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_apss_tcu_clk = {
+	.halt_reg = 0x12018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_apss_tcu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gfx_tcu_clk = {
+	.halt_reg = 0x12020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gfx_tcu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gfx_tbu_clk = {
+	.halt_reg = 0x12010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(3),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gfx_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdp_tbu_clk = {
+	.halt_reg = 0x1201c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdp_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus_tbu_clk = {
+	.halt_reg = 0x12014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(5),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_csi_vfe1_clk = {
+	.halt_reg = 0x58074,
+	.clkr = {
+		.enable_reg = 0x58074,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_csi_vfe1_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&vfe1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe1_clk = {
+	.halt_reg = 0x5805c,
+	.clkr = {
+		.enable_reg = 0x5805c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_vfe1_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&vfe1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe1_ahb_clk = {
+	.halt_reg = 0x58060,
+	.clkr = {
+		.enable_reg = 0x58060,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_vfe1_ahb_clk",
+			.parent_hws = (const struct clk_hw *[]){
+				&camss_top_ahb_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camss_vfe1_axi_clk = {
+	.halt_reg = 0x58068,
+	.clkr = {
+		.enable_reg = 0x58068,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_camss_vfe1_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vfe_tbu_clk = {
+	.halt_reg = 0x1203c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vfe_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_jpeg_tbu_clk = {
+	.halt_reg = 0x12034,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(10),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_jpeg_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_smmu_cfg_clk = {
+	.halt_reg = 0x12038,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(12),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_smmu_cfg_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gtcu_ahb_clk = {
+	.halt_reg = 0x12044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(13),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gtcu_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cpp_tbu_clk = {
+	.halt_reg = 0x12040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(14),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cpp_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_vfe1_tbu_clk = {
+	.halt_reg = 0x12090,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(17),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_vfe1_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ipa_tbu_clk = {
+	.halt_reg = 0x120a0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x4500c,
+		.enable_mask = BIT(16),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ipa_tbu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_bimc_gfx_clk = {
+	.halt_reg = 0x59034,
+	.clkr = {
+		.enable_reg = 0x59034,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_bimc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_bimc_gpu_clk = {
+	.halt_reg = 0x59030,
+	.clkr = {
+		.enable_reg = 0x59030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_bimc_gpu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb2a_phy_sleep_clk = {
+	.halt_reg = 0x4102c,
+	.clkr = {
+		.enable_reg = 0x4102c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb2a_phy_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_fs_ahb_clk = {
+	.halt_reg = 0x3f008,
+	.clkr = {
+		.enable_reg = 0x3f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_fs_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_hs_phy_cfg_ahb_clk = {
+	.halt_reg = 0x41030,
+	.clkr = {
+		.enable_reg = 0x41030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_hs_phy_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_fs_ic_clk = {
+	.halt_reg = 0x3f030,
+	.clkr = {
+		.enable_reg = 0x3f030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_fs_ic_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&usb_fs_ic_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_fs_system_clk = {
+	.halt_reg = 0x3f004,
+	.clkr = {
+		.enable_reg = 0x3f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_fs_system_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&usb_fs_system_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_hs_ahb_clk = {
+	.halt_reg = 0x41008,
+	.clkr = {
+		.enable_reg = 0x41008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_hs_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb_hs_system_clk = {
+	.halt_reg = 0x41004,
+	.clkr = {
+		.enable_reg = 0x41004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb_hs_system_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&usb_hs_system_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_ahb_clk = {
+	.halt_reg = 0x4c020,
+	.clkr = {
+		.enable_reg = 0x4c020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_axi_clk = {
+	.halt_reg = 0x4c024,
+	.clkr = {
+		.enable_reg = 0x4c024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_vcodec0_clk = {
+	.halt_reg = 0x4c01c,
+	.clkr = {
+		.enable_reg = 0x4c01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_vcodec0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&vcodec0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_core0_vcodec0_clk = {
+	.halt_reg = 0x4c02c,
+	.clkr = {
+		.enable_reg = 0x4c02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_core0_vcodec0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&vcodec0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_venus0_core1_vcodec0_clk = {
+	.halt_reg = 0x4c034,
+	.clkr = {
+		.enable_reg = 0x4c034,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_venus0_core1_vcodec0_clk",
+			.parent_hws = (const struct clk_hw*[]){
+				&vcodec0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_oxili_timer_clk = {
+	.halt_reg = 0x59040,
+	.clkr = {
+		.enable_reg = 0x59040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_oxili_timer_clk",
+			.parent_data = gcc_xo_parent_data,
+			.num_parents = ARRAY_SIZE(gcc_xo_parent_data),
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc venus_gdsc = {
+	.gdscr = 0x4c018,
+	.cxcs = (unsigned int []){ 0x4c024, 0x4c01c },
+	.cxc_count = 2,
+	.pd = {
+		.name = "venus_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc mdss_gdsc = {
+	.gdscr = 0x4d078,
+	.cxcs = (unsigned int []){ 0x4d080, 0x4d088 },
+	.cxc_count = 2,
+	.pd = {
+		.name = "mdss_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc jpeg_gdsc = {
+	.gdscr = 0x5701c,
+	.cxcs = (unsigned int []){ 0x57020, 0x57028 },
+	.cxc_count = 2,
+	.pd = {
+		.name = "jpeg_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc vfe0_gdsc = {
+	.gdscr = 0x58034,
+	.cxcs = (unsigned int []){ 0x58038, 0x58048, 0x5600c, 0x58050 },
+	.cxc_count = 4,
+	.pd = {
+		.name = "vfe0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc vfe1_gdsc = {
+	.gdscr = 0x5806c,
+	.cxcs = (unsigned int []){ 0x5805c, 0x58068, 0x5600c, 0x58074 },
+	.cxc_count = 4,
+	.pd = {
+		.name = "vfe1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc cpp_gdsc = {
+	.gdscr = 0x58078,
+	.cxcs = (unsigned int []){ 0x5803c, 0x58064 },
+	.cxc_count = 2,
+	.pd = {
+		.name = "cpp_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc oxili_gx_gdsc = {
+	.gdscr = 0x5901c,
+	.cxcs = (unsigned int []){ 0x59000, 0x59024 },
+	.cxc_count = 2,
+	.pd = {
+		.name = "oxili_gx_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc venus_core0_gdsc = {
+	.gdscr = 0x4c028,
+	.cxcs = (unsigned int []){ 0x4c02c },
+	.cxc_count = 1,
+	.pd = {
+		.name = "venus_core0_gdsc",
+	},
+	.flags = HW_CTRL_TRIGGER,
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct gdsc venus_core1_gdsc = {
+	.gdscr = 0x4c030,
+	.cxcs = (unsigned int []){ 0x4c034 },
+	.cxc_count = 1,
+	.pd = {
+		.name = "venus_core1_gdsc",
+	},
+	.flags = HW_CTRL_TRIGGER,
+	.pwrsts = PWRSTS_OFF_ON,
+};
+
+static struct clk_regmap *gcc_msm8952_clocks[] = {
+	[GPLL0] = &gpll0.clkr,
+	[GPLL0_VOTE] = &gpll0_vote,
+	[CAMSS_TOP_AHB_CLK_SRC] = &camss_top_ahb_clk_src.clkr,
+	[APSS_AHB_CLK_SRC] = &apss_ahb_clk_src.clkr,
+	[CSI0_CLK_SRC] = &csi0_clk_src.clkr,
+	[CSI1_CLK_SRC] = &csi1_clk_src.clkr,
+	[CSI2_CLK_SRC] = &csi2_clk_src.clkr,
+	[GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr,
+	[VFE0_CLK_SRC] = &vfe0_clk_src.clkr,
+	[BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr,
+	[BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr,
+	[BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr,
+	[BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr,
+	[BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr,
+	[BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr,
+	[BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr,
+	[CCI_CLK_SRC] = &cci_clk_src.clkr,
+	[CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr,
+	[CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr,
+	[JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr,
+	[MCLK0_CLK_SRC] = &mclk0_clk_src.clkr,
+	[MCLK1_CLK_SRC] = &mclk1_clk_src.clkr,
+	[CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr,
+	[CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr,
+	[CPP_CLK_SRC] = &cpp_clk_src.clkr,
+	[CRYPTO_CLK_SRC] = &crypto_clk_src.clkr,
+	[GP1_CLK_SRC] = &gp1_clk_src.clkr,
+	[GP2_CLK_SRC] = &gp2_clk_src.clkr,
+	[GP3_CLK_SRC] = &gp3_clk_src.clkr,
+	[BYTE0_CLK_SRC] = &byte0_clk_src.clkr,
+	[ESC0_CLK_SRC] = &esc0_clk_src.clkr,
+	[MDP_CLK_SRC] = &mdp_clk_src.clkr,
+	[PCLK0_CLK_SRC] = &pclk0_clk_src.clkr,
+	[VSYNC_CLK_SRC] = &vsync_clk_src.clkr,
+	[PDM2_CLK_SRC] = &pdm2_clk_src.clkr,
+	[SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr,
+	[SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr,
+	[SDCC1_ICE_CORE_CLK_SRC] = &sdcc1_ice_core_clk_src.clkr,
+	[USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr,
+	[VCODEC0_CLK_SRC] = &vcodec0_clk_src.clkr,
+	[GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr,
+	[GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr,
+	[GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_CAMSS_CCI_AHB_CLK] = &gcc_camss_cci_ahb_clk.clkr,
+	[GCC_CAMSS_CCI_CLK] = &gcc_camss_cci_clk.clkr,
+	[GCC_CAMSS_CSI0_AHB_CLK] = &gcc_camss_csi0_ahb_clk.clkr,
+	[GCC_CAMSS_CSI0_CLK] = &gcc_camss_csi0_clk.clkr,
+	[GCC_CAMSS_CSI0PHY_CLK] = &gcc_camss_csi0phy_clk.clkr,
+	[GCC_CAMSS_CSI0PIX_CLK] = &gcc_camss_csi0pix_clk.clkr,
+	[GCC_CAMSS_CSI0RDI_CLK] = &gcc_camss_csi0rdi_clk.clkr,
+	[GCC_CAMSS_CSI1_AHB_CLK] = &gcc_camss_csi1_ahb_clk.clkr,
+	[GCC_CAMSS_CSI1_CLK] = &gcc_camss_csi1_clk.clkr,
+	[GCC_CAMSS_CSI1PHY_CLK] = &gcc_camss_csi1phy_clk.clkr,
+	[GCC_CAMSS_CSI1PIX_CLK] = &gcc_camss_csi1pix_clk.clkr,
+	[GCC_CAMSS_CSI1RDI_CLK] = &gcc_camss_csi1rdi_clk.clkr,
+	[GCC_CAMSS_CSI2_AHB_CLK] = &gcc_camss_csi2_ahb_clk.clkr,
+	[GCC_CAMSS_CSI2_CLK] = &gcc_camss_csi2_clk.clkr,
+	[GCC_CAMSS_CSI2PHY_CLK] = &gcc_camss_csi2phy_clk.clkr,
+	[GCC_CAMSS_CSI2PIX_CLK] = &gcc_camss_csi2pix_clk.clkr,
+	[GCC_CAMSS_CSI2RDI_CLK] = &gcc_camss_csi2rdi_clk.clkr,
+	[GCC_CAMSS_CSI_VFE0_CLK] = &gcc_camss_csi_vfe0_clk.clkr,
+	[GCC_CAMSS_GP0_CLK] = &gcc_camss_gp0_clk.clkr,
+	[GCC_CAMSS_GP1_CLK] = &gcc_camss_gp1_clk.clkr,
+	[GCC_CAMSS_ISPIF_AHB_CLK] = &gcc_camss_ispif_ahb_clk.clkr,
+	[GCC_CAMSS_JPEG0_CLK] = &gcc_camss_jpeg0_clk.clkr,
+	[GCC_CAMSS_JPEG_AHB_CLK] = &gcc_camss_jpeg_ahb_clk.clkr,
+	[GCC_CAMSS_JPEG_AXI_CLK] = &gcc_camss_jpeg_axi_clk.clkr,
+	[GCC_CAMSS_MCLK0_CLK] = &gcc_camss_mclk0_clk.clkr,
+	[GCC_CAMSS_MCLK1_CLK] = &gcc_camss_mclk1_clk.clkr,
+	[GCC_CAMSS_MICRO_AHB_CLK] = &gcc_camss_micro_ahb_clk.clkr,
+	[GCC_CAMSS_CSI0PHYTIMER_CLK] = &gcc_camss_csi0phytimer_clk.clkr,
+	[GCC_CAMSS_CSI1PHYTIMER_CLK] = &gcc_camss_csi1phytimer_clk.clkr,
+	[GCC_CAMSS_AHB_CLK] = &gcc_camss_ahb_clk.clkr,
+	[GCC_CAMSS_TOP_AHB_CLK] = &gcc_camss_top_ahb_clk.clkr,
+	[GCC_CAMSS_VFE0_AHB_CLK] = &gcc_camss_vfe0_ahb_clk.clkr,
+	[GCC_CAMSS_VFE0_AXI_CLK] = &gcc_camss_vfe0_axi_clk.clkr,
+	[GCC_CAMSS_CPP_AHB_CLK] = &gcc_camss_cpp_ahb_clk.clkr,
+	[GCC_CAMSS_CPP_CLK] = &gcc_camss_cpp_clk.clkr,
+	[GCC_CAMSS_VFE0_CLK] = &gcc_camss_vfe0_clk.clkr,
+	[GCC_CRYPTO_AHB_CLK] = &gcc_crypto_ahb_clk.clkr,
+	[GCC_CRYPTO_AXI_CLK] = &gcc_crypto_axi_clk.clkr,
+	[GCC_CRYPTO_CLK] = &gcc_crypto_clk.clkr,
+	[GCC_OXILI_GMEM_CLK] = &gcc_oxili_gmem_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_MDSS_AHB_CLK] = &gcc_mdss_ahb_clk.clkr,
+	[GCC_MDSS_AXI_CLK] = &gcc_mdss_axi_clk.clkr,
+	[GCC_MDSS_BYTE0_CLK] = &gcc_mdss_byte0_clk.clkr,
+	[GCC_MDSS_ESC0_CLK] = &gcc_mdss_esc0_clk.clkr,
+	[GCC_MDSS_MDP_CLK] = &gcc_mdss_mdp_clk.clkr,
+	[GCC_MDSS_PCLK0_CLK] = &gcc_mdss_pclk0_clk.clkr,
+	[GCC_MDSS_VSYNC_CLK] = &gcc_mdss_vsync_clk.clkr,
+	[GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr,
+	[GCC_OXILI_AHB_CLK] = &gcc_oxili_ahb_clk.clkr,
+	[GCC_OXILI_GFX3D_CLK] = &gcc_oxili_gfx3d_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_GTCU_AHB_CLK] = &gcc_gtcu_ahb_clk.clkr,
+	[GCC_JPEG_TBU_CLK] = &gcc_jpeg_tbu_clk.clkr,
+	[GCC_MDP_TBU_CLK] = &gcc_mdp_tbu_clk.clkr,
+	[GCC_SMMU_CFG_CLK] = &gcc_smmu_cfg_clk.clkr,
+	[GCC_VENUS_TBU_CLK] = &gcc_venus_tbu_clk.clkr,
+	[GCC_VFE_TBU_CLK] = &gcc_vfe_tbu_clk.clkr,
+	[GCC_USB2A_PHY_SLEEP_CLK] = &gcc_usb2a_phy_sleep_clk.clkr,
+	[GCC_USB_HS_AHB_CLK] = &gcc_usb_hs_ahb_clk.clkr,
+	[GCC_USB_HS_SYSTEM_CLK] = &gcc_usb_hs_system_clk.clkr,
+	[GCC_VENUS0_AHB_CLK] = &gcc_venus0_ahb_clk.clkr,
+	[GCC_VENUS0_AXI_CLK] = &gcc_venus0_axi_clk.clkr,
+	[GCC_VENUS0_VCODEC0_CLK] = &gcc_venus0_vcodec0_clk.clkr,
+	[GCC_APSS_TCU_CLK] = &gcc_apss_tcu_clk.clkr,
+	[GCC_GFX_TCU_CLK] = &gcc_gfx_tcu_clk.clkr,
+	[GCC_BIMC_GFX_CLK] = &gcc_bimc_gfx_clk.clkr,
+	[GCC_BIMC_GPU_CLK] = &gcc_bimc_gpu_clk.clkr,
+	[GCC_MSS_Q6_BIMC_AXI_CLK] = &gcc_mss_q6_bimc_axi_clk.clkr,
+	[GPLL3] = &gpll3.clkr,
+	[GPLL3_EARLY] = &gpll3_early.clkr,
+	[GPLL4] = &gpll4.clkr,
+	[GPLL4_VOTE] = &gpll4_vote,
+	[GPLL6] = &gpll6.clkr,
+	[GPLL6_VOTE] = &gpll6_vote,
+	[GCC_GFX_TBU_CLK] = &gcc_gfx_tbu_clk.clkr,
+	[GCC_CPP_TBU_CLK] = &gcc_cpp_tbu_clk.clkr,
+	[USB_FS_SYSTEM_CLK_SRC] = &usb_fs_system_clk_src.clkr,
+	[USB_FS_IC_CLK_SRC] = &usb_fs_ic_clk_src.clkr,
+	[GCC_USB_FS_AHB_CLK] = &gcc_usb_fs_ahb_clk.clkr,
+	[GCC_USB_HS_PHY_CFG_AHB_CLK] = &gcc_usb_hs_phy_cfg_ahb_clk.clkr,
+	[GCC_USB_FS_IC_CLK] = &gcc_usb_fs_ic_clk.clkr,
+	[GCC_USB_FS_SYSTEM_CLK] = &gcc_usb_fs_system_clk.clkr,
+	[GCC_VENUS0_CORE0_VCODEC0_CLK] = &gcc_venus0_core0_vcodec0_clk.clkr,
+	[GCC_VENUS0_CORE1_VCODEC0_CLK] = &gcc_venus0_core1_vcodec0_clk.clkr,
+	[GCC_OXILI_TIMER_CLK] = &gcc_oxili_timer_clk.clkr,
+	[BLSP2_QUP1_I2C_APPS_CLK_SRC] = &blsp2_qup1_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP1_SPI_APPS_CLK_SRC] = &blsp2_qup1_spi_apps_clk_src.clkr,
+	[BLSP2_QUP2_I2C_APPS_CLK_SRC] = &blsp2_qup2_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP2_SPI_APPS_CLK_SRC] = &blsp2_qup2_spi_apps_clk_src.clkr,
+	[BLSP2_QUP3_I2C_APPS_CLK_SRC] = &blsp2_qup3_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP3_SPI_APPS_CLK_SRC] = &blsp2_qup3_spi_apps_clk_src.clkr,
+	[BLSP2_QUP4_I2C_APPS_CLK_SRC] = &blsp2_qup4_i2c_apps_clk_src.clkr,
+	[BLSP2_QUP4_SPI_APPS_CLK_SRC] = &blsp2_qup4_spi_apps_clk_src.clkr,
+	[BLSP2_UART1_APPS_CLK_SRC]    = &blsp2_uart1_apps_clk_src.clkr,
+	[BLSP2_UART2_APPS_CLK_SRC]    = &blsp2_uart2_apps_clk_src.clkr,
+	[GCC_BLSP2_AHB_CLK]           = &gcc_blsp2_ahb_clk.clkr,
+	[GCC_BLSP2_QUP1_I2C_APPS_CLK] = &gcc_blsp2_qup1_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP1_SPI_APPS_CLK] = &gcc_blsp2_qup1_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_I2C_APPS_CLK] = &gcc_blsp2_qup2_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP2_SPI_APPS_CLK] = &gcc_blsp2_qup2_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_I2C_APPS_CLK] = &gcc_blsp2_qup3_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP3_SPI_APPS_CLK] = &gcc_blsp2_qup3_spi_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_I2C_APPS_CLK] = &gcc_blsp2_qup4_i2c_apps_clk.clkr,
+	[GCC_BLSP2_QUP4_SPI_APPS_CLK] = &gcc_blsp2_qup4_spi_apps_clk.clkr,
+	[GCC_BLSP2_UART1_APPS_CLK]    = &gcc_blsp2_uart1_apps_clk.clkr,
+	[GCC_BLSP2_UART2_APPS_CLK]    = &gcc_blsp2_uart2_apps_clk.clkr,
+	[VFE1_CLK_SRC]                 = &vfe1_clk_src.clkr,
+	[GCC_CAMSS_VFE1_CLK]           = &gcc_camss_vfe1_clk.clkr,
+	[GCC_CAMSS_VFE1_AHB_CLK]       = &gcc_camss_vfe1_ahb_clk.clkr,
+	[GCC_CAMSS_VFE1_AXI_CLK]       = &gcc_camss_vfe1_axi_clk.clkr,
+	[GCC_CAMSS_CSI_VFE1_CLK]       = &gcc_camss_csi_vfe1_clk.clkr,
+	[MCLK2_CLK_SRC]                = &mclk2_clk_src.clkr,
+	[GCC_CAMSS_MCLK2_CLK]          = &gcc_camss_mclk2_clk.clkr,
+	[GCC_VFE1_TBU_CLK]             = &gcc_vfe1_tbu_clk.clkr,
+	[GCC_IPA_TBU_CLK]              = &gcc_ipa_tbu_clk.clkr,
+	[GCC_CAMSS_CPP_AXI_CLK]        = &gcc_camss_cpp_axi_clk.clkr,
+};
+
+static struct gdsc *gcc_msm8952_gdscs[] = {
+	[CPP_GDSC] = &cpp_gdsc,
+	[VENUS_GDSC] = &venus_gdsc,
+	[MDSS_GDSC] = &mdss_gdsc,
+	[JPEG_GDSC] = &jpeg_gdsc,
+	[VFE0_GDSC] = &vfe0_gdsc,
+	[VFE1_GDSC] = &vfe1_gdsc,
+	[OXILI_GX_GDSC] = &oxili_gx_gdsc,
+	[VENUS_CORE0_GDSC] = &venus_core0_gdsc,
+	[VENUS_CORE1_GDSC] = &venus_core1_gdsc,
+};
+
+static const struct qcom_reset_map gcc_msm8952_resets[] = {
+	[GCC_CRYPTO_BCR]		= { 0x16000 },
+	[GCC_USB_FS_BCR]		= { 0x3f000 },
+	[GCC_USB_HS_BCR]		= { 0x41000 },
+	[GCC_QUSB2_PHY_BCR]		= { 0x4103c },
+	[GCC_USB2_HS_PHY_ONLY_BCR]	= { 0x41034 },
+	[GCC_SDCC1_BCR]			= { 0x42000 },
+	[GCC_SDCC2_BCR]			= { 0x43000 },
+	[GCC_MDSS_BCR]			= { 0x4d074 },
+	[GCC_CAMSS_MICRO_BCR]		= { 0x56008 },
+	[GCC_MSS_BCR]			= { 0x71000 },
+};
+
+static const struct regmap_config gcc_msm8952_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x80000,
+	.fast_io	= true,
+};
+
+static struct clk_alpha_pll *gcc_msm8952_alpha_plls[] = {
+	&gpll3_early,
+};
+
+static void clk_msm8952_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Set Sleep and Wakeup cycles to 0 for GMEM clock */
+	regmap_update_bits(regmap, gcc_oxili_gmem_clk.clkr.enable_reg, 0xff0, 0);
+
+	/* Disable GMEM HW Dynamic, Without this write, GMEM can't get enabled */
+	regmap_write(regmap, 0x7E004, 0x1);
+}
+
+static const struct qcom_cc_driver_data gcc_msm8952_driver_data = {
+	.alpha_plls = gcc_msm8952_alpha_plls,
+	.num_alpha_plls = ARRAY_SIZE(gcc_msm8952_alpha_plls),
+	.clk_regs_configure = clk_msm8952_regs_configure,
+};
+
+static const struct qcom_cc_desc gcc_msm8952_desc = {
+	.config = &gcc_msm8952_regmap_config,
+	.clks = gcc_msm8952_clocks,
+	.num_clks = ARRAY_SIZE(gcc_msm8952_clocks),
+	.resets = gcc_msm8952_resets,
+	.num_resets = ARRAY_SIZE(gcc_msm8952_resets),
+	.gdscs = gcc_msm8952_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_msm8952_gdscs),
+	.use_rpm = true,
+	.driver_data = &gcc_msm8952_driver_data
+};
+
+static const struct of_device_id gcc_msm8952_match_table[] = {
+	{ .compatible = "qcom,gcc-msm8952" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_msm8952_match_table);
+
+static int gcc_msm8952_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_msm8952_desc);
+}
+
+static struct platform_driver gcc_msm8952_driver = {
+	.probe		= gcc_msm8952_probe,
+	.driver		= {
+		.name	= "gcc-msm8952",
+		.of_match_table = gcc_msm8952_match_table,
+	},
+};
+
+static int __init gcc_msm8952_init(void)
+{
+	return platform_driver_register(&gcc_msm8952_driver);
+}
+core_initcall(gcc_msm8952_init);
+
+static void __exit gcc_msm8952_exit(void)
+{
+	platform_driver_unregister(&gcc_msm8952_driver);
+}
+module_exit(gcc_msm8952_exit);
+
+MODULE_DESCRIPTION("Qualcomm GCC MSM8952 Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-nord.c b/drivers/clk/qcom/gcc-nord.c
index 5c9d25f53a6d25..4f87e06b33ab4f 100644
--- a/drivers/clk/qcom/gcc-nord.c
+++ b/drivers/clk/qcom/gcc-nord.c
@@ -1862,17 +1862,7 @@ static struct platform_driver gcc_nord_driver = {
 	},
 };
 
-static int __init gcc_nord_init(void)
-{
-	return platform_driver_register(&gcc_nord_driver);
-}
-subsys_initcall(gcc_nord_init);
-
-static void __exit gcc_nord_exit(void)
-{
-	platform_driver_unregister(&gcc_nord_driver);
-}
-module_exit(gcc_nord_exit);
+subsys_platform_driver(gcc_nord_driver);
 
 MODULE_DESCRIPTION("QTI GCC NORD Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-qcm2290.c b/drivers/clk/qcom/gcc-qcm2290.c
index 2532c0b36bc52b..cf7d05558be9b7 100644
--- a/drivers/clk/qcom/gcc-qcm2290.c
+++ b/drivers/clk/qcom/gcc-qcm2290.c
@@ -2923,17 +2923,7 @@ static struct platform_driver gcc_qcm2290_driver = {
 	},
 };
 
-static int __init gcc_qcm2290_init(void)
-{
-	return platform_driver_register(&gcc_qcm2290_driver);
-}
-subsys_initcall(gcc_qcm2290_init);
-
-static void __exit gcc_qcm2290_exit(void)
-{
-	platform_driver_unregister(&gcc_qcm2290_driver);
-}
-module_exit(gcc_qcm2290_exit);
+subsys_platform_driver(gcc_qcm2290_driver);
 
 MODULE_DESCRIPTION("QTI GCC QCM2290 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-qcs615.c b/drivers/clk/qcom/gcc-qcs615.c
index 57f8c80c6f3249..e096af7d82bf55 100644
--- a/drivers/clk/qcom/gcc-qcs615.c
+++ b/drivers/clk/qcom/gcc-qcs615.c
@@ -2958,7 +2958,34 @@ static const struct regmap_config gcc_qcs615_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_qcs615_critical_cbcrs[] = {
+	0xb008, /* GCC_CAMERA_AHB_CLK */
+	0xb044, /* GCC_CAMERA_XO_CLK */
+	0xb00c, /* GCC_DISP_AHB_CLK */
+	0xb048, /* GCC_DISP_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0xb004, /* GCC_VIDEO_AHB_CLK */
+	0xb040, /* GCC_VIDEO_XO_CLK */
+	0x48004, /* GCC_CPUSS_GNOC_CLK */
+};
+
+static void gcc_qcs615_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Disable the GPLL0 active input to MM blocks and GPU via MISC registers. */
+	regmap_update_bits(regmap, 0x0b084, BIT(0), BIT(0));
+	regmap_update_bits(regmap, 0x9b000, BIT(0), BIT(0));
+}
+
+static const struct qcom_cc_driver_data gcc_qcs615_driver_data = {
+	.clk_cbcrs = gcc_qcs615_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_qcs615_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_qcs615_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_qcs615_desc = {
+	.driver_data = &gcc_qcs615_driver_data,
 	.config = &gcc_qcs615_regmap_config,
 	.clk_hws = gcc_qcs615_hws,
 	.num_clk_hws = ARRAY_SIZE(gcc_qcs615_hws),
@@ -2978,35 +3005,7 @@ MODULE_DEVICE_TABLE(of, gcc_qcs615_match_table);
 
 static int gcc_qcs615_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_qcs615_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-	/*
-	 * Disable the GPLL0 active input to MM blocks and GPU
-	 * via MISC registers.
-	 */
-	regmap_update_bits(regmap, 0x0b084, BIT(0), BIT(0));
-	regmap_update_bits(regmap, 0x9b000, BIT(0), BIT(0));
-
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0xb008); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb044); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb00c); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb048); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xb040); /* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x480040); /* GCC_CPUSS_GNOC_CLK */
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_qcs615_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_qcs615_desc);
 }
 
 static struct platform_driver gcc_qcs615_driver = {
@@ -3017,17 +3016,7 @@ static struct platform_driver gcc_qcs615_driver = {
 	},
 };
 
-static int __init gcc_qcs615_init(void)
-{
-	return platform_driver_register(&gcc_qcs615_driver);
-}
-subsys_initcall(gcc_qcs615_init);
-
-static void __exit gcc_qcs615_exit(void)
-{
-	platform_driver_unregister(&gcc_qcs615_driver);
-}
-module_exit(gcc_qcs615_exit);
+subsys_platform_driver(gcc_qcs615_driver);
 
 MODULE_DESCRIPTION("QTI GCC QCS615 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-qcs8300.c b/drivers/clk/qcom/gcc-qcs8300.c
index 31fd870b10f7ae..916617a001c854 100644
--- a/drivers/clk/qcom/gcc-qcs8300.c
+++ b/drivers/clk/qcom/gcc-qcs8300.c
@@ -42,6 +42,7 @@ enum {
 	P_GCC_GPLL0_OUT_MAIN,
 	P_GCC_GPLL1_OUT_MAIN,
 	P_GCC_GPLL4_OUT_MAIN,
+	P_GCC_GPLL5_OUT_MAIN,
 	P_GCC_GPLL7_OUT_MAIN,
 	P_GCC_GPLL9_OUT_MAIN,
 	P_PCIE_0_PIPE_CLK,
@@ -128,6 +129,23 @@ static struct clk_alpha_pll gcc_gpll4 = {
 	},
 };
 
+static struct clk_alpha_pll gcc_gpll5 = {
+	.offset = 0x5000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
+	.clkr = {
+		.enable_reg = 0x4b028,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll5",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_evo_ops,
+		},
+	},
+};
+
 static struct clk_alpha_pll gcc_gpll7 = {
 	.offset = 0x7000,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO],
@@ -364,6 +382,22 @@ static const struct clk_parent_data gcc_parent_data_18[] = {
 	{ .index = DT_BI_TCXO },
 };
 
+static const struct parent_map gcc_parent_map_19[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL7_OUT_MAIN, 2 },
+	{ P_GCC_GPLL5_OUT_MAIN, 3 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_19[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll7.clkr.hw },
+	{ .hw = &gcc_gpll5.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
 static struct clk_regmap_mux gcc_pcie_0_phy_aux_clk_src = {
 	.reg = 0xa9074,
 	.shift = 0,
@@ -1094,6 +1128,25 @@ static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
 	},
 };
 
+static const struct freq_tbl ftbl_gcc_tscss_cntr_clk_src[] = {
+	F(15625000, P_GCC_GPLL7_OUT_MAIN, 16, 1, 4),
+	{ }
+};
+
+static struct clk_rcg2 gcc_tscss_cntr_clk_src = {
+	.cmd_rcgr = 0x21008,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_19,
+	.freq_tbl = ftbl_gcc_tscss_cntr_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_tscss_cntr_clk_src",
+		.parent_data = gcc_parent_data_19,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_19),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
 static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = {
 	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
 	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
@@ -2899,6 +2952,50 @@ static struct clk_branch gcc_sgmi_clkref_en = {
 	},
 };
 
+static struct clk_branch gcc_tscss_ahb_clk = {
+	.halt_reg = 0x21024,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_tscss_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tscss_etu_clk = {
+	.halt_reg = 0x21020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x21020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_tscss_etu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_tscss_global_cntr_clk = {
+	.halt_reg = 0x21004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x21004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_tscss_global_cntr_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_tscss_cntr_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_ufs_phy_ahb_clk = {
 	.halt_reg = 0x83020,
 	.halt_check = BRANCH_HALT_VOTED,
@@ -3360,6 +3457,7 @@ static struct clk_regmap *gcc_qcs8300_clocks[] = {
 	[GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr,
 	[GCC_GPLL1] = &gcc_gpll1.clkr,
 	[GCC_GPLL4] = &gcc_gpll4.clkr,
+	[GCC_GPLL5] = &gcc_gpll5.clkr,
 	[GCC_GPLL7] = &gcc_gpll7.clkr,
 	[GCC_GPLL9] = &gcc_gpll9.clkr,
 	[GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr,
@@ -3464,6 +3562,10 @@ static struct clk_regmap *gcc_qcs8300_clocks[] = {
 	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
 	[GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr,
 	[GCC_SGMI_CLKREF_EN] = &gcc_sgmi_clkref_en.clkr,
+	[GCC_TSCSS_AHB_CLK] = &gcc_tscss_ahb_clk.clkr,
+	[GCC_TSCSS_CNTR_CLK_SRC] = &gcc_tscss_cntr_clk_src.clkr,
+	[GCC_TSCSS_ETU_CLK] = &gcc_tscss_etu_clk.clkr,
+	[GCC_TSCSS_GLOBAL_CNTR_CLK] = &gcc_tscss_global_cntr_clk.clkr,
 	[GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr,
 	[GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr,
 	[GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr,
@@ -3523,6 +3625,7 @@ static const struct qcom_reset_map gcc_qcs8300_resets[] = {
 	[GCC_PCIE_1_PHY_BCR] = { 0xae08c },
 	[GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR] = { 0xae094 },
 	[GCC_SDCC1_BCR] = { 0x20000 },
+	[GCC_TSCSS_BCR] = { 0x21000 },
 	[GCC_UFS_PHY_BCR] = { 0x83000 },
 	[GCC_USB20_PRIM_BCR] = { 0x1c000 },
 	[GCC_USB2_PHY_PRIM_BCR] = { 0x5c01c },
@@ -3570,7 +3673,32 @@ static const struct regmap_config gcc_qcs8300_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_qcs8300_critical_cbcrs[] = {
+	0x32004, /* GCC_CAMERA_AHB_CLK */
+	0x32020, /* GCC_CAMERA_XO_CLK */
+	0x33004, /* GCC_DISP_AHB_CLK */
+	0x33018, /* GCC_DISP_XO_CLK */
+	0x7d004, /* GCC_GPU_CFG_AHB_CLK */
+	0x34004, /* GCC_VIDEO_AHB_CLK */
+	0x34024, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_qcs8300_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+}
+
+static const struct qcom_cc_driver_data gcc_qcs8300_driver_data = {
+	.clk_cbcrs = gcc_qcs8300_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_qcs8300_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_qcs8300_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_qcs8300_desc = {
+	.driver_data = &gcc_qcs8300_driver_data,
 	.config = &gcc_qcs8300_regmap_config,
 	.clks = gcc_qcs8300_clocks,
 	.num_clks = ARRAY_SIZE(gcc_qcs8300_clocks),
@@ -3578,6 +3706,7 @@ static const struct qcom_cc_desc gcc_qcs8300_desc = {
 	.num_resets = ARRAY_SIZE(gcc_qcs8300_resets),
 	.gdscs = gcc_qcs8300_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_qcs8300_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_qcs8300_match_table[] = {
@@ -3588,31 +3717,7 @@ MODULE_DEVICE_TABLE(of, gcc_qcs8300_match_table);
 
 static int gcc_qcs8300_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_qcs8300_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0x32004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32020); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x33004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x33018); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x7d004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x34004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x34024); /* GCC_VIDEO_XO_CLK */
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_qcs8300_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_qcs8300_desc);
 }
 
 static struct platform_driver gcc_qcs8300_driver = {
@@ -3623,17 +3728,7 @@ static struct platform_driver gcc_qcs8300_driver = {
 	},
 };
 
-static int __init gcc_qcs8300_init(void)
-{
-	return platform_driver_register(&gcc_qcs8300_driver);
-}
-subsys_initcall(gcc_qcs8300_init);
-
-static void __exit gcc_qcs8300_exit(void)
-{
-	platform_driver_unregister(&gcc_qcs8300_driver);
-}
-module_exit(gcc_qcs8300_exit);
+subsys_platform_driver(gcc_qcs8300_driver);
 
 MODULE_DESCRIPTION("QTI GCC QCS8300 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-qdu1000.c b/drivers/clk/qcom/gcc-qdu1000.c
index 915bb9b4ff8130..a139fd8f722dd6 100644
--- a/drivers/clk/qcom/gcc-qdu1000.c
+++ b/drivers/clk/qcom/gcc-qdu1000.c
@@ -2689,17 +2689,7 @@ static struct platform_driver gcc_qdu1000_driver = {
 	},
 };
 
-static int __init gcc_qdu1000_init(void)
-{
-	return platform_driver_register(&gcc_qdu1000_driver);
-}
-subsys_initcall(gcc_qdu1000_init);
-
-static void __exit gcc_qdu1000_exit(void)
-{
-	platform_driver_unregister(&gcc_qdu1000_driver);
-}
-module_exit(gcc_qdu1000_exit);
+subsys_platform_driver(gcc_qdu1000_driver);
 
 MODULE_DESCRIPTION("QTI GCC QDU1000 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sa8775p.c b/drivers/clk/qcom/gcc-sa8775p.c
index dca316decd0e69..0e02af68661416 100644
--- a/drivers/clk/qcom/gcc-sa8775p.c
+++ b/drivers/clk/qcom/gcc-sa8775p.c
@@ -4644,7 +4644,34 @@ static const struct regmap_config gcc_sa8775p_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sa8775p_critical_cbcrs[] = {
+	0x32004, /* GCC_CAMERA_AHB_CLK */
+	0x32020, /* GCC_CAMERA_XO_CLK */
+	0xc7004, /* GCC_DISP1_AHB_CLK */
+	0xc7018, /* GCC_DISP1_XO_CLK */
+	0x33004, /* GCC_DISP_AHB_CLK */
+	0x33018, /* GCC_DISP_XO_CLK */
+	0x7d004, /* GCC_GPU_CFG_AHB_CLK */
+	0x34004, /* GCC_VIDEO_AHB_CLK */
+	0x34024, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_sa8775p_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+}
+
+static const struct qcom_cc_driver_data gcc_sa8775p_driver_data = {
+	.clk_cbcrs = gcc_sa8775p_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sa8775p_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sa8775p_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sa8775p_desc = {
+	.driver_data = &gcc_sa8775p_driver_data,
 	.config = &gcc_sa8775p_regmap_config,
 	.clks = gcc_sa8775p_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sa8775p_clocks),
@@ -4652,6 +4679,7 @@ static const struct qcom_cc_desc gcc_sa8775p_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sa8775p_resets),
 	.gdscs = gcc_sa8775p_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sa8775p_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sa8775p_match_table[] = {
@@ -4662,33 +4690,7 @@ MODULE_DEVICE_TABLE(of, gcc_sa8775p_match_table);
 
 static int gcc_sa8775p_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sa8775p_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x32004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32020); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0xc7004); /* GCC_DISP1_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xc7018); /* GCC_DISP1_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x33004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x33018); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x7d004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x34004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x34024); /* GCC_VIDEO_XO_CLK */
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sa8775p_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sa8775p_desc);
 }
 
 static struct platform_driver gcc_sa8775p_driver = {
diff --git a/drivers/clk/qcom/gcc-sar2130p.c b/drivers/clk/qcom/gcc-sar2130p.c
index 475e2cda3618b9..8b6118f9583551 100644
--- a/drivers/clk/qcom/gcc-sar2130p.c
+++ b/drivers/clk/qcom/gcc-sar2130p.c
@@ -2300,7 +2300,29 @@ static const struct regmap_config gcc_sar2130p_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sar2130p_critical_cbcrs[] = {
+	0x37004, /* GCC_DISP_AHB_CLK */
+	0x42004, /* GCC_VIDEO_AHB_CLK */
+	0x42028, /* GCC_VIDEO_XO_CLK */
+	0x9b004, /* GCC_GPU_CFG_AHB_CLK */
+};
+
+static void gcc_sar2130p_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
+	regmap_write(regmap, 0x62204, 0x0);
+}
+
+static const struct qcom_cc_driver_data gcc_sar2130p_driver_data = {
+	.clk_cbcrs = gcc_sar2130p_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sar2130p_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sar2130p_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sar2130p_desc = {
+	.driver_data = &gcc_sar2130p_driver_data,
 	.config = &gcc_sar2130p_regmap_config,
 	.clks = gcc_sar2130p_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sar2130p_clocks),
@@ -2308,6 +2330,7 @@ static const struct qcom_cc_desc gcc_sar2130p_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sar2130p_resets),
 	.gdscs = gcc_sar2130p_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sar2130p_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sar2130p_match_table[] = {
@@ -2318,28 +2341,7 @@ MODULE_DEVICE_TABLE(of, gcc_sar2130p_match_table);
 
 static int gcc_sar2130p_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sar2130p_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x37004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x42004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x42028); /* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x9b004); /* GCC_GPU_CFG_AHB_CLK */
-
-	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
-	regmap_write(regmap, 0x62204, 0x0);
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sar2130p_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sar2130p_desc);
 }
 
 static struct platform_driver gcc_sar2130p_driver = {
@@ -2350,17 +2352,7 @@ static struct platform_driver gcc_sar2130p_driver = {
 	},
 };
 
-static int __init gcc_sar2130p_init(void)
-{
-	return platform_driver_register(&gcc_sar2130p_driver);
-}
-subsys_initcall(gcc_sar2130p_init);
-
-static void __exit gcc_sar2130p_exit(void)
-{
-	platform_driver_unregister(&gcc_sar2130p_driver);
-}
-module_exit(gcc_sar2130p_exit);
+subsys_platform_driver(gcc_sar2130p_driver);
 
 MODULE_DESCRIPTION("QTI GCC SAR2130P Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sc7180.c b/drivers/clk/qcom/gcc-sc7180.c
index 4a49ad7a9e5be6..6d9ee31044a513 100644
--- a/drivers/clk/qcom/gcc-sc7180.c
+++ b/drivers/clk/qcom/gcc-sc7180.c
@@ -2408,7 +2408,35 @@ static const struct regmap_config gcc_sc7180_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sc7180_critical_cbcrs[] = {
+	0x48004, /* GCC_CPUSS_GNOC_CLK */
+	0x0b004, /* GCC_VIDEO_AHB_CLK */
+	0x0b008, /* GCC_CAMERA_AHB_CLK */
+	0x0b00c, /* GCC_DISP_AHB_CLK */
+	0x0b02c, /* GCC_CAMERA_XO_CLK */
+	0x0b028, /* GCC_VIDEO_XO_CLK */
+	0x0b030, /* GCC_DISP_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+};
+
+static void gcc_sc7180_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Disable the GPLL0 active input to MM blocks, NPU and GPU via MISC registers. */
+	regmap_update_bits(regmap, 0x09ffc, 0x3, 0x3);
+	regmap_update_bits(regmap, 0x4d110, 0x3, 0x3);
+	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
+}
+
+static const struct qcom_cc_driver_data gcc_sc7180_driver_data = {
+	.clk_cbcrs = gcc_sc7180_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sc7180_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sc7180_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sc7180_desc = {
+	.driver_data = &gcc_sc7180_driver_data,
 	.config = &gcc_sc7180_regmap_config,
 	.clk_hws = gcc_sc7180_hws,
 	.num_clk_hws = ARRAY_SIZE(gcc_sc7180_hws),
@@ -2428,37 +2456,7 @@ MODULE_DEVICE_TABLE(of, gcc_sc7180_match_table);
 
 static int gcc_sc7180_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sc7180_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	/*
-	 * Disable the GPLL0 active input to MM blocks, NPU
-	 * and GPU via MISC registers.
-	 */
-	regmap_update_bits(regmap, 0x09ffc, 0x3, 0x3);
-	regmap_update_bits(regmap, 0x4d110, 0x3, 0x3);
-	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x48004); /* GCC_CPUSS_GNOC_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b008); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b00c); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b02c); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b028); /* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b030); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-					ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sc7180_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sc7180_desc);
 }
 
 static struct platform_driver gcc_sc7180_driver = {
diff --git a/drivers/clk/qcom/gcc-sc7280.c b/drivers/clk/qcom/gcc-sc7280.c
index 4502926a2691a7..78399993cdcee2 100644
--- a/drivers/clk/qcom/gcc-sc7280.c
+++ b/drivers/clk/qcom/gcc-sc7280.c
@@ -3438,7 +3438,34 @@ static const struct regmap_config gcc_sc7280_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sc7280_critical_cbcrs[] = {
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26028, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x2701c, /* GCC_DISP_XO_CLK */
+	0x28004, /* GCC_VIDEO_AHB_CLK */
+	0x28014, /* GCC_VIDEO_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+};
+
+static void gcc_sc7280_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	regmap_update_bits(regmap, 0x7100C, BIT(13), BIT(13));
+
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+}
+
+static const struct qcom_cc_driver_data gcc_sc7280_driver_data = {
+	.clk_cbcrs = gcc_sc7280_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sc7280_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sc7280_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sc7280_desc = {
+	.driver_data = &gcc_sc7280_driver_data,
 	.config = &gcc_sc7280_regmap_config,
 	.clks = gcc_sc7280_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sc7280_clocks),
@@ -3446,6 +3473,7 @@ static const struct qcom_cc_desc gcc_sc7280_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sc7280_resets),
 	.gdscs = gcc_sc7280_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sc7280_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sc7280_match_table[] = {
@@ -3456,32 +3484,7 @@ MODULE_DEVICE_TABLE(of, gcc_sc7280_match_table);
 
 static int gcc_sc7280_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sc7280_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x26004);/* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x26028);/* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27004);/* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x2701c);/* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x28004);/* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x28014);/* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004);/* GCC_GPU_CFG_AHB_CLK */
-	regmap_update_bits(regmap, 0x7100C, BIT(13), BIT(13));
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-			ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sc7280_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sc7280_desc);
 }
 
 static struct platform_driver gcc_sc7280_driver = {
@@ -3492,17 +3495,7 @@ static struct platform_driver gcc_sc7280_driver = {
 	},
 };
 
-static int __init gcc_sc7280_init(void)
-{
-	return platform_driver_register(&gcc_sc7280_driver);
-}
-subsys_initcall(gcc_sc7280_init);
-
-static void __exit gcc_sc7280_exit(void)
-{
-	platform_driver_unregister(&gcc_sc7280_driver);
-}
-module_exit(gcc_sc7280_exit);
+subsys_platform_driver(gcc_sc7280_driver);
 
 MODULE_DESCRIPTION("QTI GCC SC7280 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sc8280xp.c b/drivers/clk/qcom/gcc-sc8280xp.c
index 2ab111585d7fcb..918c70648d5738 100644
--- a/drivers/clk/qcom/gcc-sc8280xp.c
+++ b/drivers/clk/qcom/gcc-sc8280xp.c
@@ -7509,7 +7509,27 @@ static const struct regmap_config gcc_sc8280xp_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sc8280xp_critical_cbcrs[] = {
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26020, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x27028, /* GCC_DISP_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x28004, /* GCC_VIDEO_AHB_CLK */
+	0x28028, /* GCC_VIDEO_XO_CLK */
+	0xbb004, /* GCC_DISP1_AHB_CLK */
+	0xbb028, /* GCC_DISP1_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data gcc_sc8280xp_driver_data = {
+	.clk_cbcrs = gcc_sc8280xp_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sc8280xp_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+};
+
 static const struct qcom_cc_desc gcc_sc8280xp_desc = {
+	.driver_data = &gcc_sc8280xp_driver_data,
 	.config = &gcc_sc8280xp_regmap_config,
 	.clks = gcc_sc8280xp_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sc8280xp_clocks),
@@ -7538,21 +7558,6 @@ static int gcc_sc8280xp_probe(struct platform_device *pdev)
 		goto err_put_rpm;
 	}
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x26004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x26020); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27028); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x28004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x28028); /* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0xbb004); /* GCC_DISP1_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xbb028); /* GCC_DISP1_XO_CLK */
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		goto err_put_rpm;
-
 	ret = qcom_cc_really_probe(&pdev->dev, &gcc_sc8280xp_desc, regmap);
 	if (ret)
 		goto err_put_rpm;
@@ -7581,17 +7586,7 @@ static struct platform_driver gcc_sc8280xp_driver = {
 	},
 };
 
-static int __init gcc_sc8280xp_init(void)
-{
-	return platform_driver_register(&gcc_sc8280xp_driver);
-}
-subsys_initcall(gcc_sc8280xp_init);
-
-static void __exit gcc_sc8280xp_exit(void)
-{
-	platform_driver_unregister(&gcc_sc8280xp_driver);
-}
-module_exit(gcc_sc8280xp_exit);
+subsys_platform_driver(gcc_sc8280xp_driver);
 
 MODULE_DESCRIPTION("Qualcomm SC8280XP GCC driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sdx55.c b/drivers/clk/qcom/gcc-sdx55.c
index 84c507656e8fe3..8d8547c12b192a 100644
--- a/drivers/clk/qcom/gcc-sdx55.c
+++ b/drivers/clk/qcom/gcc-sdx55.c
@@ -1587,7 +1587,25 @@ static const struct regmap_config gcc_sdx55_regmap_config = {
 	.fast_io	= true,
 };
 
+static const u32 gcc_sdx55_critical_cbcrs[] = {
+	0x6d008, /* GCC_SYS_NOC_CPUSS_AHB_CLK */
+};
+
+static void gcc_sdx55_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Keep some clocks always-on */
+	regmap_update_bits(regmap, 0x6d008, BIT(21), BIT(21)); /* GCC_CPUSS_AHB_CLK */
+	regmap_update_bits(regmap, 0x6d008, BIT(22), BIT(22)); /* GCC_CPUSS_GNOC_CLK */
+}
+
+static const struct qcom_cc_driver_data gcc_sdx55_driver_data = {
+	.clk_cbcrs = gcc_sdx55_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sdx55_critical_cbcrs),
+	.clk_regs_configure = gcc_sdx55_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sdx55_desc = {
+	.driver_data = &gcc_sdx55_driver_data,
 	.config = &gcc_sdx55_regmap_config,
 	.clks = gcc_sdx55_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sdx55_clocks),
@@ -1605,18 +1623,7 @@ MODULE_DEVICE_TABLE(of, gcc_sdx55_match_table);
 
 static int gcc_sdx55_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-
-	regmap = qcom_cc_map(pdev, &gcc_sdx55_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x6d008); /* GCC_SYS_NOC_CPUSS_AHB_CLK */
-	regmap_update_bits(regmap, 0x6d008, BIT(21), BIT(21)); /* GCC_CPUSS_AHB_CLK */
-	regmap_update_bits(regmap, 0x6d008, BIT(22), BIT(22)); /* GCC_CPUSS_GNOC_CLK */
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sdx55_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sdx55_desc);
 }
 
 static struct platform_driver gcc_sdx55_driver = {
@@ -1627,17 +1634,7 @@ static struct platform_driver gcc_sdx55_driver = {
 	},
 };
 
-static int __init gcc_sdx55_init(void)
-{
-	return platform_driver_register(&gcc_sdx55_driver);
-}
-subsys_initcall(gcc_sdx55_init);
-
-static void __exit gcc_sdx55_exit(void)
-{
-	platform_driver_unregister(&gcc_sdx55_driver);
-}
-module_exit(gcc_sdx55_exit);
+subsys_platform_driver(gcc_sdx55_driver);
 
 MODULE_DESCRIPTION("QTI GCC SDX55 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sdx65.c b/drivers/clk/qcom/gcc-sdx65.c
index fe297c606f97a9..b1ab265ba2c18b 100644
--- a/drivers/clk/qcom/gcc-sdx65.c
+++ b/drivers/clk/qcom/gcc-sdx65.c
@@ -1551,7 +1551,25 @@ static const struct regmap_config gcc_sdx65_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sdx65_critical_cbcrs[] = {
+	0x6d008, /* GCC_SYS_NOC_CPUSS_AHB_CLK */
+};
+
+static void gcc_sdx65_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Keep some clocks always-on */
+	regmap_update_bits(regmap, 0x6d008, BIT(21), BIT(21)); /* GCC_CPUSS_AHB_CLK */
+	regmap_update_bits(regmap, 0x6d008, BIT(22), BIT(22)); /* GCC_CPUSS_GNOC_CLK */
+}
+
+static const struct qcom_cc_driver_data gcc_sdx65_driver_data = {
+	.clk_cbcrs = gcc_sdx65_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sdx65_critical_cbcrs),
+	.clk_regs_configure = gcc_sdx65_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sdx65_desc = {
+	.driver_data = &gcc_sdx65_driver_data,
 	.config = &gcc_sdx65_regmap_config,
 	.clks = gcc_sdx65_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sdx65_clocks),
@@ -1569,18 +1587,7 @@ MODULE_DEVICE_TABLE(of, gcc_sdx65_match_table);
 
 static int gcc_sdx65_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-
-	regmap = qcom_cc_map(pdev, &gcc_sdx65_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x6d008); /* GCC_SYS_NOC_CPUSS_AHB_CLK */
-	regmap_update_bits(regmap, 0x6d008, BIT(21), BIT(21)); /* GCC_CPUSS_AHB_CLK */
-	regmap_update_bits(regmap, 0x6d008, BIT(22), BIT(22)); /* GCC_CPUSS_GNOC_CLK */
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sdx65_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sdx65_desc);
 }
 
 static struct platform_driver gcc_sdx65_driver = {
@@ -1591,17 +1598,7 @@ static struct platform_driver gcc_sdx65_driver = {
 	},
 };
 
-static int __init gcc_sdx65_init(void)
-{
-	return platform_driver_register(&gcc_sdx65_driver);
-}
-subsys_initcall(gcc_sdx65_init);
-
-static void __exit gcc_sdx65_exit(void)
-{
-	platform_driver_unregister(&gcc_sdx65_driver);
-}
-module_exit(gcc_sdx65_exit);
+subsys_platform_driver(gcc_sdx65_driver);
 
 MODULE_DESCRIPTION("QTI GCC SDX65 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sdx75.c b/drivers/clk/qcom/gcc-sdx75.c
index 6712e76f875cfe..080e14e832f039 100644
--- a/drivers/clk/qcom/gcc-sdx75.c
+++ b/drivers/clk/qcom/gcc-sdx75.c
@@ -2905,7 +2905,20 @@ static const struct regmap_config gcc_sdx75_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sdx75_critical_cbcrs[] = {
+	0x3e004, /* GCC_AHB_PCIE_LINK_CLK */
+	0x3e008, /* GCC_XO_PCIE_LINK_CLK */
+};
+
+static const struct qcom_cc_driver_data gcc_sdx75_driver_data = {
+	.clk_cbcrs = gcc_sdx75_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sdx75_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+};
+
 static const struct qcom_cc_desc gcc_sdx75_desc = {
+	.driver_data = &gcc_sdx75_driver_data,
 	.config = &gcc_sdx75_regmap_config,
 	.clks = gcc_sdx75_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sdx75_clocks),
@@ -2913,6 +2926,7 @@ static const struct qcom_cc_desc gcc_sdx75_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sdx75_resets),
 	.gdscs = gcc_sdx75_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sdx75_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sdx75_match_table[] = {
@@ -2923,23 +2937,7 @@ MODULE_DEVICE_TABLE(of, gcc_sdx75_match_table);
 
 static int gcc_sdx75_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sdx75_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x3e004); /* GCC_AHB_PCIE_LINK_CLK */
-	qcom_branch_set_clk_en(regmap, 0x3e008); /* GCC_XO_PCIE_LINK_CLK */
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sdx75_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sdx75_desc);
 }
 
 static struct platform_driver gcc_sdx75_driver = {
@@ -2950,17 +2948,7 @@ static struct platform_driver gcc_sdx75_driver = {
 	},
 };
 
-static int __init gcc_sdx75_init(void)
-{
-	return platform_driver_register(&gcc_sdx75_driver);
-}
-subsys_initcall(gcc_sdx75_init);
-
-static void __exit gcc_sdx75_exit(void)
-{
-	platform_driver_unregister(&gcc_sdx75_driver);
-}
-module_exit(gcc_sdx75_exit);
+subsys_platform_driver(gcc_sdx75_driver);
 
 MODULE_DESCRIPTION("QTI GCC SDX75 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-shikra.c b/drivers/clk/qcom/gcc-shikra.c
index 2a1c1fc253aa31..13c99a883e2987 100644
--- a/drivers/clk/qcom/gcc-shikra.c
+++ b/drivers/clk/qcom/gcc-shikra.c
@@ -4416,17 +4416,7 @@ static struct platform_driver gcc_shikra_driver = {
 	},
 };
 
-static int __init gcc_shikra_init(void)
-{
-	return platform_driver_register(&gcc_shikra_driver);
-}
-subsys_initcall(gcc_shikra_init);
-
-static void __exit gcc_shikra_exit(void)
-{
-	platform_driver_unregister(&gcc_shikra_driver);
-}
-module_exit(gcc_shikra_exit);
+subsys_platform_driver(gcc_shikra_driver);
 
 MODULE_DESCRIPTION("QTI GCC Shikra Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sm4450.c b/drivers/clk/qcom/gcc-sm4450.c
index 30fc7af09930fc..a3e157c52be6a3 100644
--- a/drivers/clk/qcom/gcc-sm4450.c
+++ b/drivers/clk/qcom/gcc-sm4450.c
@@ -140,6 +140,7 @@ static const struct alpha_pll_config gcc_gpll3_config = {
 };
 
 static struct clk_alpha_pll gcc_gpll3 = {
+	.config = &gcc_gpll3_config,
 	.offset = 0x3000,
 	.vco_table = lucid_evo_vco,
 	.num_vco = ARRAY_SIZE(lucid_evo_vco),
@@ -2815,7 +2816,41 @@ static const struct regmap_config gcc_sm4450_regmap_config = {
 	.fast_io = true,
 };
 
+static struct clk_alpha_pll *gcc_sm4450_plls[] = {
+	&gcc_gpll3,
+};
+
+static const u32 gcc_sm4450_critical_cbcrs[] = {
+	0x36004, /* GCC_CAMERA_AHB_CLK */
+	0x36018, /* GCC_CAMERA_SLEEP_CLK */
+	0x3601c, /* GCC_CAMERA_XO_CLK */
+	0x37004, /* GCC_DISP_AHB_CLK */
+	0x37014, /* GCC_DISP_XO_CLK */
+	0x81004, /* GCC_GPU_CFG_AHB_CLK */
+	0x42004, /* GCC_VIDEO_AHB_CLK */
+	0x42018, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_sm4450_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for the UFS PHY ICE core clock */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+
+	regmap_update_bits(regmap, 0x4201c, BIT(21), BIT(21));
+}
+
+static const struct qcom_cc_driver_data gcc_sm4450_driver_data = {
+	.alpha_plls = gcc_sm4450_plls,
+	.num_alpha_plls = ARRAY_SIZE(gcc_sm4450_plls),
+	.clk_cbcrs = gcc_sm4450_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm4450_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sm4450_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm4450_desc = {
+	.driver_data = &gcc_sm4450_driver_data,
 	.config = &gcc_sm4450_regmap_config,
 	.clks = gcc_sm4450_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sm4450_clocks),
@@ -2823,6 +2858,7 @@ static const struct qcom_cc_desc gcc_sm4450_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sm4450_resets),
 	.gdscs = gcc_sm4450_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sm4450_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sm4450_match_table[] = {
@@ -2833,34 +2869,7 @@ MODULE_DEVICE_TABLE(of, gcc_sm4450_match_table);
 
 static int gcc_sm4450_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm4450_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	clk_lucid_evo_pll_configure(&gcc_gpll3, regmap, &gcc_gpll3_config);
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x36004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x36018); /* GCC_CAMERA_SLEEP_CLK */
-	qcom_branch_set_clk_en(regmap, 0x3601c); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x37004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x37014); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x81004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x42004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x42018); /* GCC_VIDEO_XO_CLK */
-
-	regmap_update_bits(regmap, 0x4201c, BIT(21), BIT(21));
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm4450_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm4450_desc);
 }
 
 static struct platform_driver gcc_sm4450_driver = {
@@ -2871,17 +2880,7 @@ static struct platform_driver gcc_sm4450_driver = {
 	},
 };
 
-static int __init gcc_sm4450_init(void)
-{
-	return platform_driver_register(&gcc_sm4450_driver);
-}
-subsys_initcall(gcc_sm4450_init);
-
-static void __exit gcc_sm4450_exit(void)
-{
-	platform_driver_unregister(&gcc_sm4450_driver);
-}
-module_exit(gcc_sm4450_exit);
+subsys_platform_driver(gcc_sm4450_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM4450 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sm6115.c b/drivers/clk/qcom/gcc-sm6115.c
index a3c9dbe3a48af1..5736e59d76ce24 100644
--- a/drivers/clk/qcom/gcc-sm6115.c
+++ b/drivers/clk/qcom/gcc-sm6115.c
@@ -3525,17 +3525,7 @@ static struct platform_driver gcc_sm6115_driver = {
 	},
 };
 
-static int __init gcc_sm6115_init(void)
-{
-	return platform_driver_register(&gcc_sm6115_driver);
-}
-subsys_initcall(gcc_sm6115_init);
-
-static void __exit gcc_sm6115_exit(void)
-{
-	platform_driver_unregister(&gcc_sm6115_driver);
-}
-module_exit(gcc_sm6115_exit);
+subsys_platform_driver(gcc_sm6115_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM6115 and SM4250 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sm6125.c b/drivers/clk/qcom/gcc-sm6125.c
index 07bb1e5c4a303a..ee3cd0b00fc5e0 100644
--- a/drivers/clk/qcom/gcc-sm6125.c
+++ b/drivers/clk/qcom/gcc-sm6125.c
@@ -4172,17 +4172,7 @@ static struct platform_driver gcc_sm6125_driver = {
 	},
 };
 
-static int __init gcc_sm6125_init(void)
-{
-	return platform_driver_register(&gcc_sm6125_driver);
-}
-subsys_initcall(gcc_sm6125_init);
-
-static void __exit gcc_sm6125_exit(void)
-{
-	platform_driver_unregister(&gcc_sm6125_driver);
-}
-module_exit(gcc_sm6125_exit);
+subsys_platform_driver(gcc_sm6125_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM6125 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sm6375.c b/drivers/clk/qcom/gcc-sm6375.c
index f47dc2808095ee..9c2acf8dc4b7ae 100644
--- a/drivers/clk/qcom/gcc-sm6375.c
+++ b/drivers/clk/qcom/gcc-sm6375.c
@@ -3903,17 +3903,7 @@ static struct platform_driver gcc_sm6375_driver = {
 	},
 };
 
-static int __init gcc_sm6375_init(void)
-{
-	return platform_driver_register(&gcc_sm6375_driver);
-}
-subsys_initcall(gcc_sm6375_init);
-
-static void __exit gcc_sm6375_exit(void)
-{
-	platform_driver_unregister(&gcc_sm6375_driver);
-}
-module_exit(gcc_sm6375_exit);
+subsys_platform_driver(gcc_sm6375_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM6375 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sm7150.c b/drivers/clk/qcom/gcc-sm7150.c
index dcb5d82a1a31c3..169aef61e74908 100644
--- a/drivers/clk/qcom/gcc-sm7150.c
+++ b/drivers/clk/qcom/gcc-sm7150.c
@@ -2966,7 +2966,35 @@ static const struct regmap_config gcc_sm7150_regmap_config = {
 	.fast_io	= true,
 };
 
+static const u32 gcc_sm7150_critical_cbcrs[] = {
+	0x48004, /* GCC_CPUSS_GNOC_CLK */
+	0x0b004, /* GCC_VIDEO_AHB_CLK */
+	0x0b008, /* GCC_CAMERA_AHB_CLK */
+	0x0b00c, /* GCC_DISP_AHB_CLK */
+	0x0b02c, /* GCC_CAMERA_XO_CLK */
+	0x0b028, /* GCC_VIDEO_XO_CLK */
+	0x0b030, /* GCC_DISP_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+};
+
+static void gcc_sm7150_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Disable the GPLL0 active input to MM blocks, NPU and GPU via MISC registers. */
+	regmap_update_bits(regmap, 0x09ffc, 0x3, 0x3);
+	regmap_update_bits(regmap, 0x4d110, 0x3, 0x3);
+	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
+}
+
+static const struct qcom_cc_driver_data gcc_sm7150_driver_data = {
+	.clk_cbcrs = gcc_sm7150_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm7150_critical_cbcrs),
+	.dfs_rcgs = gcc_sm7150_dfs_desc,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_sm7150_dfs_desc),
+	.clk_regs_configure = gcc_sm7150_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm7150_desc = {
+	.driver_data = &gcc_sm7150_driver_data,
 	.config = &gcc_sm7150_regmap_config,
 	.clk_hws = gcc_sm7150_hws,
 	.num_clk_hws = ARRAY_SIZE(gcc_sm7150_hws),
@@ -2986,37 +3014,7 @@ MODULE_DEVICE_TABLE(of, gcc_sm7150_match_table);
 
 static int gcc_sm7150_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm7150_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	/*
-	 * Disable the GPLL0 active input to MM blocks, NPU
-	 * and GPU via MISC registers.
-	 */
-	regmap_update_bits(regmap, 0x09ffc, 0x3, 0x3);
-	regmap_update_bits(regmap, 0x4d110, 0x3, 0x3);
-	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x48004); /* GCC_CPUSS_GNOC_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b008); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b00c); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b02c); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b028); /* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b030); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_sm7150_dfs_desc,
-					ARRAY_SIZE(gcc_sm7150_dfs_desc));
-	if (ret)
-		return ret;
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm7150_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm7150_desc);
 }
 
 static struct platform_driver gcc_sm7150_driver = {
@@ -3027,17 +3025,7 @@ static struct platform_driver gcc_sm7150_driver = {
 	},
 };
 
-static int __init gcc_sm7150_init(void)
-{
-	return platform_driver_register(&gcc_sm7150_driver);
-}
-subsys_initcall(gcc_sm7150_init);
-
-static void __exit gcc_sm7150_exit(void)
-{
-	platform_driver_unregister(&gcc_sm7150_driver);
-}
-module_exit(gcc_sm7150_exit);
+subsys_platform_driver(gcc_sm7150_driver);
 
 MODULE_DESCRIPTION("Qualcomm SM7150 Global Clock Controller");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sm8150.c b/drivers/clk/qcom/gcc-sm8150.c
index a93d1f412a7b6a..b38a4484268666 100644
--- a/drivers/clk/qcom/gcc-sm8150.c
+++ b/drivers/clk/qcom/gcc-sm8150.c
@@ -3808,17 +3808,7 @@ static struct platform_driver gcc_sm8150_driver = {
 	},
 };
 
-static int __init gcc_sm8150_init(void)
-{
-	return platform_driver_register(&gcc_sm8150_driver);
-}
-subsys_initcall(gcc_sm8150_init);
-
-static void __exit gcc_sm8150_exit(void)
-{
-	platform_driver_unregister(&gcc_sm8150_driver);
-}
-module_exit(gcc_sm8150_exit);
+subsys_platform_driver(gcc_sm8150_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM8150 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sm8250.c b/drivers/clk/qcom/gcc-sm8250.c
index 1c59d70e0f96c5..7ed7876577d19b 100644
--- a/drivers/clk/qcom/gcc-sm8250.c
+++ b/drivers/clk/qcom/gcc-sm8250.c
@@ -3603,6 +3603,15 @@ static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = {
 	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s5_clk_src),
 };
 
+static const u32 gcc_sm8250_critical_cbcrs[] = {
+	0x0b004, /* GCC_VIDEO_AHB_CLK */
+	0x0b008, /* GCC_CAMERA_AHB_CLK */
+	0x0b00c, /* GCC_DISP_AHB_CLK */
+	0x4818c, /* GCC_CPUSS_DVM_BUS_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x52000, /* GCC_SYS_NOC_CPUSS_AHB_CLK */
+};
+
 static const struct regmap_config gcc_sm8250_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
@@ -3611,6 +3620,24 @@ static const struct regmap_config gcc_sm8250_regmap_config = {
 	.fast_io = true,
 };
 
+static void gcc_sm8250_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/*
+	 * Disable the GPLL0 active input to NPU and GPU
+	 * via MISC registers.
+	 */
+	regmap_set_bits(regmap, 0x4d110, 0x3);
+	regmap_set_bits(regmap, 0x71028, 0x3);
+}
+
+static const struct qcom_cc_driver_data gcc_sm8250_driver_data = {
+	.clk_cbcrs = gcc_sm8250_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm8250_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sm8250_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm8250_desc = {
 	.config = &gcc_sm8250_regmap_config,
 	.clks = gcc_sm8250_clocks,
@@ -3619,6 +3646,7 @@ static const struct qcom_cc_desc gcc_sm8250_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sm8250_resets),
 	.gdscs = gcc_sm8250_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sm8250_gdscs),
+	.driver_data = &gcc_sm8250_driver_data,
 };
 
 static const struct of_device_id gcc_sm8250_match_table[] = {
@@ -3629,34 +3657,7 @@ MODULE_DEVICE_TABLE(of, gcc_sm8250_match_table);
 
 static int gcc_sm8250_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm8250_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	/*
-	 * Disable the GPLL0 active input to NPU and GPU
-	 * via MISC registers.
-	 */
-	regmap_update_bits(regmap, 0x4d110, 0x3, 0x3);
-	regmap_update_bits(regmap, 0x71028, 0x3, 0x3);
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x0b004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b008); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x0b00c); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x4818c); /* GCC_CPUSS_DVM_BUS_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x52000); /* GCC_SYS_NOC_CPUSS_AHB_CLK */
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm8250_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm8250_desc);
 }
 
 static struct platform_driver gcc_sm8250_driver = {
@@ -3667,17 +3668,7 @@ static struct platform_driver gcc_sm8250_driver = {
 	},
 };
 
-static int __init gcc_sm8250_init(void)
-{
-	return platform_driver_register(&gcc_sm8250_driver);
-}
-subsys_initcall(gcc_sm8250_init);
-
-static void __exit gcc_sm8250_exit(void)
-{
-	platform_driver_unregister(&gcc_sm8250_driver);
-}
-module_exit(gcc_sm8250_exit);
+subsys_platform_driver(gcc_sm8250_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM8250 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sm8350.c b/drivers/clk/qcom/gcc-sm8350.c
index 2d94f3046b7117..3b7aef50cfe377 100644
--- a/drivers/clk/qcom/gcc-sm8350.c
+++ b/drivers/clk/qcom/gcc-sm8350.c
@@ -3779,7 +3779,32 @@ static const struct regmap_config gcc_sm8350_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sm8350_critical_cbcrs[] = {
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26018, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x2701c, /* GCC_DISP_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x28004, /* GCC_VIDEO_AHB_CLK */
+	0x28020, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_sm8350_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14));
+}
+
+static const struct qcom_cc_driver_data gcc_sm8350_driver_data = {
+	.clk_cbcrs = gcc_sm8350_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm8350_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sm8350_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm8350_desc = {
+	.driver_data = &gcc_sm8350_driver_data,
 	.config = &gcc_sm8350_regmap_config,
 	.clks = gcc_sm8350_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sm8350_clocks),
@@ -3787,6 +3812,7 @@ static const struct qcom_cc_desc gcc_sm8350_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sm8350_resets),
 	.gdscs = gcc_sm8350_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sm8350_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sm8350_match_table[] = {
@@ -3797,32 +3823,7 @@ MODULE_DEVICE_TABLE(of, gcc_sm8350_match_table);
 
 static int gcc_sm8350_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm8350_desc);
-	if (IS_ERR(regmap)) {
-		dev_err(&pdev->dev, "Failed to map gcc registers\n");
-		return PTR_ERR(regmap);
-	}
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x26004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x26018); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x2701c); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x28004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x28020); /* GCC_VIDEO_XO_CLK */
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
-	regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14));
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm8350_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm8350_desc);
 }
 
 static struct platform_driver gcc_sm8350_driver = {
@@ -3833,17 +3834,7 @@ static struct platform_driver gcc_sm8350_driver = {
 	},
 };
 
-static int __init gcc_sm8350_init(void)
-{
-	return platform_driver_register(&gcc_sm8350_driver);
-}
-subsys_initcall(gcc_sm8350_init);
-
-static void __exit gcc_sm8350_exit(void)
-{
-	platform_driver_unregister(&gcc_sm8350_driver);
-}
-module_exit(gcc_sm8350_exit);
+subsys_platform_driver(gcc_sm8350_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM8350 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sm8450.c b/drivers/clk/qcom/gcc-sm8450.c
index b18bb34889ab28..cdae5d9c8aece7 100644
--- a/drivers/clk/qcom/gcc-sm8450.c
+++ b/drivers/clk/qcom/gcc-sm8450.c
@@ -3392,7 +3392,32 @@ static const struct regmap_config gcc_sm8450_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sm8450_critical_cbcrs[] = {
+	0x36004, /* GCC_CAMERA_AHB_CLK */
+	0x36020, /* GCC_CAMERA_XO_CLK */
+	0x37004, /* GCC_DISP_AHB_CLK */
+	0x3701c, /* GCC_DISP_XO_CLK */
+	0x81004, /* GCC_GPU_CFG_AHB_CLK */
+	0x42004, /* GCC_VIDEO_AHB_CLK */
+	0x42028, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_sm8450_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14));
+}
+
+static const struct qcom_cc_driver_data gcc_sm8450_driver_data = {
+	.clk_cbcrs = gcc_sm8450_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm8450_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sm8450_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm8450_desc = {
+	.driver_data = &gcc_sm8450_driver_data,
 	.config = &gcc_sm8450_regmap_config,
 	.clks = gcc_sm8450_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sm8450_clocks),
@@ -3400,6 +3425,7 @@ static const struct qcom_cc_desc gcc_sm8450_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sm8450_resets),
 	.gdscs = gcc_sm8450_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sm8450_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sm8450_match_table[] = {
@@ -3411,18 +3437,6 @@ MODULE_DEVICE_TABLE(of, gcc_sm8450_match_table);
 
 static int gcc_sm8450_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm8450_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
 	if (of_device_is_compatible(pdev->dev.of_node, "qcom,sm8475-gcc")) {
 		/* Update GCC PLL0 */
 		gcc_gpll0.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE];
@@ -3456,19 +3470,7 @@ static int gcc_sm8450_probe(struct platform_device *pdev)
 		gcc_sm8450_desc.clks[SM8475_GCC_GPLL3] = &sm8475_gcc_gpll3.clkr;
 	}
 
-	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
-	regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14));
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x36004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x36020); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x37004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x3701c); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x81004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x42004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x42028); /* GCC_VIDEO_XO_CLK */
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm8450_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm8450_desc);
 }
 
 static struct platform_driver gcc_sm8450_driver = {
@@ -3479,17 +3481,7 @@ static struct platform_driver gcc_sm8450_driver = {
 	},
 };
 
-static int __init gcc_sm8450_init(void)
-{
-	return platform_driver_register(&gcc_sm8450_driver);
-}
-subsys_initcall(gcc_sm8450_init);
-
-static void __exit gcc_sm8450_exit(void)
-{
-	platform_driver_unregister(&gcc_sm8450_driver);
-}
-module_exit(gcc_sm8450_exit);
+subsys_platform_driver(gcc_sm8450_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM8450 / SM8475 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/gcc-sm8550.c b/drivers/clk/qcom/gcc-sm8550.c
index 36a5b7de5b55d2..80a44ff7d21184 100644
--- a/drivers/clk/qcom/gcc-sm8550.c
+++ b/drivers/clk/qcom/gcc-sm8550.c
@@ -3319,7 +3319,35 @@ static const struct regmap_config gcc_sm8550_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sm8550_critical_cbcrs[] = {
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26028, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x27018, /* GCC_DISP_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x32004, /* GCC_VIDEO_AHB_CLK */
+	0x32030, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_sm8550_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14));
+
+	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
+	regmap_write(regmap, 0x52024, 0x0);
+}
+
+static const struct qcom_cc_driver_data gcc_sm8550_driver_data = {
+	.clk_cbcrs = gcc_sm8550_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm8550_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sm8550_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm8550_desc = {
+	.driver_data = &gcc_sm8550_driver_data,
 	.config = &gcc_sm8550_regmap_config,
 	.clks = gcc_sm8550_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sm8550_clocks),
@@ -3327,6 +3355,7 @@ static const struct qcom_cc_desc gcc_sm8550_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sm8550_resets),
 	.gdscs = gcc_sm8550_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sm8550_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sm8550_match_table[] = {
@@ -3337,34 +3366,7 @@ MODULE_DEVICE_TABLE(of, gcc_sm8550_match_table);
 
 static int gcc_sm8550_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm8550_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
-	regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14));
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x26004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x26028); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27018); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32030); /* GCC_VIDEO_XO_CLK */
-
-	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
-	regmap_write(regmap, 0x52024, 0x0);
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm8550_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm8550_desc);
 }
 
 static struct platform_driver gcc_sm8550_driver = {
@@ -3375,17 +3377,7 @@ static struct platform_driver gcc_sm8550_driver = {
 	},
 };
 
-static int __init gcc_sm8550_init(void)
-{
-	return platform_driver_register(&gcc_sm8550_driver);
-}
-subsys_initcall(gcc_sm8550_init);
-
-static void __exit gcc_sm8550_exit(void)
-{
-	platform_driver_unregister(&gcc_sm8550_driver);
-}
-module_exit(gcc_sm8550_exit);
+subsys_platform_driver(gcc_sm8550_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM8550 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sm8650.c b/drivers/clk/qcom/gcc-sm8650.c
index f7e2d7ec60c921..e439c958ae167d 100644
--- a/drivers/clk/qcom/gcc-sm8650.c
+++ b/drivers/clk/qcom/gcc-sm8650.c
@@ -3777,7 +3777,36 @@ static const struct regmap_config gcc_sm8650_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sm8650_critical_cbcrs[] = {
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26028, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x27018, /* GCC_DISP_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x32004, /* GCC_VIDEO_AHB_CLK */
+	0x32030, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_sm8650_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true);
+
+	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
+	regmap_write(regmap, 0x52150, 0x0);
+}
+
+static const struct qcom_cc_driver_data gcc_sm8650_driver_data = {
+	.clk_cbcrs = gcc_sm8650_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm8650_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sm8650_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm8650_desc = {
+	.driver_data = &gcc_sm8650_driver_data,
 	.config = &gcc_sm8650_regmap_config,
 	.clks = gcc_sm8650_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sm8650_clocks),
@@ -3785,6 +3814,7 @@ static const struct qcom_cc_desc gcc_sm8650_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sm8650_resets),
 	.gdscs = gcc_sm8650_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sm8650_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sm8650_match_table[] = {
@@ -3795,35 +3825,7 @@ MODULE_DEVICE_TABLE(of, gcc_sm8650_match_table);
 
 static int gcc_sm8650_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm8650_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x26004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x26028); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27018); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32030); /* GCC_VIDEO_XO_CLK */
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks  */
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true);
-
-	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
-	regmap_write(regmap, 0x52150, 0x0);
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm8650_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm8650_desc);
 }
 
 static struct platform_driver gcc_sm8650_driver = {
@@ -3834,17 +3836,7 @@ static struct platform_driver gcc_sm8650_driver = {
 	},
 };
 
-static int __init gcc_sm8650_init(void)
-{
-	return platform_driver_register(&gcc_sm8650_driver);
-}
-subsys_initcall(gcc_sm8650_init);
-
-static void __exit gcc_sm8650_exit(void)
-{
-	platform_driver_unregister(&gcc_sm8650_driver);
-}
-module_exit(gcc_sm8650_exit);
+subsys_platform_driver(gcc_sm8650_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM8650 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-sm8750.c b/drivers/clk/qcom/gcc-sm8750.c
index 6cfe90122268a8..80b8ec119c553d 100644
--- a/drivers/clk/qcom/gcc-sm8750.c
+++ b/drivers/clk/qcom/gcc-sm8750.c
@@ -3188,7 +3188,39 @@ static const struct regmap_config gcc_sm8750_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_sm8750_critical_cbcrs[] = {
+	0xa0004, /* GCC_CAM_BIST_MCLK_AHB_CLK */
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26034, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x9f004, /* GCC_EVA_AHB_CLK */
+	0x9f01c, /* GCC_EVA_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x32004, /* GCC_VIDEO_AHB_CLK */
+	0x32038, /* GCC_VIDEO_XO_CLK */
+};
+
+static void gcc_sm8750_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Keep GCC_PCIE_RSCC_CFG_AHB_CLK and GCC_PCIE_RSCC_XO_CLK enabled. */
+	regmap_update_bits(regmap, 0x52010, BIT(20), BIT(20));
+	regmap_update_bits(regmap, 0x52010, BIT(21), BIT(21));
+
+	/* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true);
+}
+
+static const struct qcom_cc_driver_data gcc_sm8750_driver_data = {
+	.clk_cbcrs = gcc_sm8750_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_sm8750_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_sm8750_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_sm8750_desc = {
+	.driver_data = &gcc_sm8750_driver_data,
 	.config = &gcc_sm8750_regmap_config,
 	.clks = gcc_sm8750_clocks,
 	.num_clks = ARRAY_SIZE(gcc_sm8750_clocks),
@@ -3196,6 +3228,7 @@ static const struct qcom_cc_desc gcc_sm8750_desc = {
 	.num_resets = ARRAY_SIZE(gcc_sm8750_resets),
 	.gdscs = gcc_sm8750_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_sm8750_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_sm8750_match_table[] = {
@@ -3206,49 +3239,7 @@ MODULE_DEVICE_TABLE(of, gcc_sm8750_match_table);
 
 static int gcc_sm8750_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_sm8750_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/*
-	 * Keep clocks always enabled:
-	 *	gcc_cam_bist_mclk_ahb_clk
-	 *	gcc_camera_ahb_clk
-	 *	gcc_camera_xo_clk
-	 *	gcc_disp_ahb_clk
-	 *	gcc_eva_ahb_clk
-	 *	gcc_eva_xo_clk
-	 *	gcc_gpu_cfg_ahb_clk
-	 *	gcc_video_ahb_clk
-	 *	gcc_video_xo_clk
-	 *	gcc_pcie_rscc_cfg_ahb_clk
-	 *	gcc_pcie_rscc_xo_clk
-	 */
-	qcom_branch_set_clk_en(regmap, 0xa0004);
-	qcom_branch_set_clk_en(regmap, 0x26004);
-	qcom_branch_set_clk_en(regmap, 0x26034);
-	qcom_branch_set_clk_en(regmap, 0x27004);
-	qcom_branch_set_clk_en(regmap, 0x9f004);
-	qcom_branch_set_clk_en(regmap, 0x9f01c);
-	qcom_branch_set_clk_en(regmap, 0x71004);
-	qcom_branch_set_clk_en(regmap, 0x32004);
-	qcom_branch_set_clk_en(regmap, 0x32038);
-	regmap_update_bits(regmap, 0x52010, BIT(20), BIT(20));
-	regmap_update_bits(regmap, 0x52010, BIT(21), BIT(21));
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks  */
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true);
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_sm8750_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_sm8750_desc);
 }
 
 static struct platform_driver gcc_sm8750_driver = {
@@ -3259,17 +3250,7 @@ static struct platform_driver gcc_sm8750_driver = {
 	},
 };
 
-static int __init gcc_sm8750_init(void)
-{
-	return platform_driver_register(&gcc_sm8750_driver);
-}
-subsys_initcall(gcc_sm8750_init);
-
-static void __exit gcc_sm8750_exit(void)
-{
-	platform_driver_unregister(&gcc_sm8750_driver);
-}
-module_exit(gcc_sm8750_exit);
+subsys_platform_driver(gcc_sm8750_driver);
 
 MODULE_DESCRIPTION("QTI GCC SM8750 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-x1e80100.c b/drivers/clk/qcom/gcc-x1e80100.c
index 8c146d62c07798..3411ecb0225c8c 100644
--- a/drivers/clk/qcom/gcc-x1e80100.c
+++ b/drivers/clk/qcom/gcc-x1e80100.c
@@ -6049,7 +6049,7 @@ static struct clk_branch gcc_usb4_0_sb_if_clk = {
 
 static struct clk_branch gcc_usb4_0_sys_clk = {
 	.halt_reg = 0x9f054,
-	.halt_check = BRANCH_HALT,
+	.halt_check = BRANCH_HALT_SKIP,
 	.clkr = {
 		.enable_reg = 0x9f054,
 		.enable_mask = BIT(0),
@@ -6266,7 +6266,7 @@ static struct clk_branch gcc_usb4_1_sb_if_clk = {
 
 static struct clk_branch gcc_usb4_1_sys_clk = {
 	.halt_reg = 0x2b054,
-	.halt_check = BRANCH_HALT,
+	.halt_check = BRANCH_HALT_SKIP,
 	.clkr = {
 		.enable_reg = 0x2b054,
 		.enable_mask = BIT(0),
@@ -6483,7 +6483,7 @@ static struct clk_branch gcc_usb4_2_sb_if_clk = {
 
 static struct clk_branch gcc_usb4_2_sys_clk = {
 	.halt_reg = 0x11054,
-	.halt_check = BRANCH_HALT,
+	.halt_check = BRANCH_HALT_SKIP,
 	.clkr = {
 		.enable_reg = 0x11054,
 		.enable_mask = BIT(0),
@@ -7441,7 +7441,37 @@ static const struct regmap_config gcc_x1e80100_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gcc_x1e80100_critical_cbcrs[] = {
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26028, /* GCC_CAMERA_XO_CLK */
+	0x27004, /* GCC_DISP_AHB_CLK */
+	0x27018, /* GCC_DISP_XO_CLK */
+	0x32004, /* GCC_VIDEO_AHB_CLK */
+	0x32030, /* GCC_VIDEO_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x7d01c, /* GCC_HLOS1_VOTE_AGGRE_NOC_MMU_USB_QTB_CLK */
+};
+
+static void gcc_x1e80100_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
+	regmap_write(regmap, 0x52224, 0x0);
+
+	/* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true);
+}
+
+static const struct qcom_cc_driver_data gcc_x1e80100_driver_data = {
+	.clk_cbcrs = gcc_x1e80100_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_x1e80100_critical_cbcrs),
+	.dfs_rcgs = gcc_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_dfs_clocks),
+	.clk_regs_configure = gcc_x1e80100_regs_configure,
+};
+
 static const struct qcom_cc_desc gcc_x1e80100_desc = {
+	.driver_data = &gcc_x1e80100_driver_data,
 	.config = &gcc_x1e80100_regmap_config,
 	.clks = gcc_x1e80100_clocks,
 	.num_clks = ARRAY_SIZE(gcc_x1e80100_clocks),
@@ -7449,6 +7479,7 @@ static const struct qcom_cc_desc gcc_x1e80100_desc = {
 	.num_resets = ARRAY_SIZE(gcc_x1e80100_resets),
 	.gdscs = gcc_x1e80100_gdscs,
 	.num_gdscs = ARRAY_SIZE(gcc_x1e80100_gdscs),
+	.use_rpm = true,
 };
 
 static const struct of_device_id gcc_x1e80100_match_table[] = {
@@ -7459,36 +7490,7 @@ MODULE_DEVICE_TABLE(of, gcc_x1e80100_match_table);
 
 static int gcc_x1e80100_probe(struct platform_device *pdev)
 {
-	struct regmap *regmap;
-	int ret;
-
-	regmap = qcom_cc_map(pdev, &gcc_x1e80100_desc);
-	if (IS_ERR(regmap))
-		return PTR_ERR(regmap);
-
-	ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks,
-				       ARRAY_SIZE(gcc_dfs_clocks));
-	if (ret)
-		return ret;
-
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x26004); /* GCC_CAMERA_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x26028); /* GCC_CAMERA_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27004); /* GCC_DISP_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x27018); /* GCC_DISP_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32004); /* GCC_VIDEO_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x32030); /* GCC_VIDEO_XO_CLK */
-	qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x7d01c); /* GCC_HLOS1_VOTE_AGGRE_NOC_MMU_USB_QTB_CLK */
-
-	/* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */
-	regmap_write(regmap, 0x52224, 0x0);
-
-	/* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks  */
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
-	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true);
-
-	return qcom_cc_really_probe(&pdev->dev, &gcc_x1e80100_desc, regmap);
+	return qcom_cc_probe(pdev, &gcc_x1e80100_desc);
 }
 
 static struct platform_driver gcc_x1e80100_driver = {
@@ -7499,17 +7501,7 @@ static struct platform_driver gcc_x1e80100_driver = {
 	},
 };
 
-static int __init gcc_x1e80100_init(void)
-{
-	return platform_driver_register(&gcc_x1e80100_driver);
-}
-subsys_initcall(gcc_x1e80100_init);
-
-static void __exit gcc_x1e80100_exit(void)
-{
-	platform_driver_unregister(&gcc_x1e80100_driver);
-}
-module_exit(gcc_x1e80100_exit);
+subsys_platform_driver(gcc_x1e80100_driver);
 
 MODULE_DESCRIPTION("QTI GCC X1E80100 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gpucc-eliza.c b/drivers/clk/qcom/gpucc-eliza.c
new file mode 100644
index 00000000000000..57a28502f32576
--- /dev/null
+++ b/drivers/clk/qcom/gpucc-eliza.c
@@ -0,0 +1,606 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,eliza-gpucc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_GPLL0_OUT_MAIN,
+	DT_GPLL0_OUT_MAIN_DIV,
+};
+
+enum {
+	P_BI_TCXO,
+	P_BI_TCXO_AO,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL0_OUT_MAIN_DIV,
+	P_GPU_CC_PLL0_OUT_MAIN,
+	P_GPU_CC_PLL1_OUT_MAIN,
+};
+
+static const struct pll_vco lucid_ole_vco[] = {
+	{ 249600000, 2300000000, 0 },
+};
+
+/* 518.0 MHz Configuration */
+static const struct alpha_pll_config gpu_cc_pll0_config = {
+	.l = 0x1a,
+	.alpha = 0xfaaa,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll gpu_cc_pll0 = {
+	.offset = 0x0,
+	.config = &gpu_cc_pll0_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+/* 440.0 MHz Configuration */
+static const struct alpha_pll_config gpu_cc_pll1_config = {
+	.l = 0x16,
+	.alpha = 0xeaaa,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll gpu_cc_pll1 = {
+	.offset = 0x1000,
+	.config = &gpu_cc_pll1_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_pll1",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct parent_map gpu_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPLL0_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_MAIN_DIV, 6 },
+};
+
+static const struct clk_parent_data gpu_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_GPLL0_OUT_MAIN },
+	{ .index = DT_GPLL0_OUT_MAIN_DIV },
+};
+
+static const struct parent_map gpu_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPU_CC_PLL0_OUT_MAIN, 1 },
+	{ P_GPU_CC_PLL1_OUT_MAIN, 3 },
+	{ P_GPLL0_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_MAIN_DIV, 6 },
+};
+
+static const struct clk_parent_data gpu_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpu_cc_pll0.clkr.hw },
+	{ .hw = &gpu_cc_pll1.clkr.hw },
+	{ .index = DT_GPLL0_OUT_MAIN },
+	{ .index = DT_GPLL0_OUT_MAIN_DIV },
+};
+
+static const struct parent_map gpu_cc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPU_CC_PLL1_OUT_MAIN, 3 },
+	{ P_GPLL0_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_MAIN_DIV, 6 },
+};
+
+static const struct clk_parent_data gpu_cc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpu_cc_pll1.clkr.hw },
+	{ .index = DT_GPLL0_OUT_MAIN },
+	{ .index = DT_GPLL0_OUT_MAIN_DIV },
+};
+
+static const struct parent_map gpu_cc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data gpu_cc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct freq_tbl ftbl_gpu_cc_ff_clk_src[] = {
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_ff_clk_src = {
+	.cmd_rcgr = 0x94b8,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_0,
+	.freq_tbl = ftbl_gpu_cc_ff_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_ff_clk_src",
+		.parent_data = gpu_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gpu_cc_gmu_clk_src[] = {
+	F(220000000, P_GPU_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	F(550000000, P_GPU_CC_PLL1_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_gmu_clk_src = {
+	.cmd_rcgr = 0x935c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_1,
+	.freq_tbl = ftbl_gpu_cc_gmu_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_gmu_clk_src",
+		.parent_data = gpu_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 gpu_cc_hub_clk_src = {
+	.cmd_rcgr = 0x9430,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_2,
+	.freq_tbl = ftbl_gpu_cc_ff_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_hub_clk_src",
+		.parent_data = gpu_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gpu_cc_xo_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_xo_clk_src = {
+	.cmd_rcgr = 0x9010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_3,
+	.freq_tbl = ftbl_gpu_cc_xo_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_xo_clk_src",
+		.parent_data = gpu_cc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_3),
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div gpu_cc_xo_div_clk_src = {
+	.reg = 0x9050,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_xo_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpu_cc_xo_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gpu_cc_ahb_clk = {
+	.halt_reg = 0x914c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x914c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_crc_ahb_clk = {
+	.halt_reg = 0x9150,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9150,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_crc_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_accu_shift_clk = {
+	.halt_reg = 0x9480,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9480,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_accu_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_ff_clk = {
+	.halt_reg = 0x9184,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9184,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_ff_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_ff_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_gmu_clk = {
+	.halt_reg = 0x916c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x916c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_gmu_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_gmu_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cxo_clk = {
+	.halt_reg = 0x917c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x917c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cxo_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_freq_measure_clk = {
+	.halt_reg = 0x9008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_freq_measure_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_xo_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_hlos1_vote_gpu_smmu_clk = {
+	.halt_reg = 0x7000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_hlos1_vote_gpu_smmu_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_hub_aon_clk = {
+	.halt_reg = 0x942c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x942c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_hub_aon_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_hub_cx_int_clk = {
+	.halt_reg = 0x9180,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9180,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_hub_cx_int_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_memnoc_gfx_clk = {
+	.halt_reg = 0x9188,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9188,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_memnoc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_mnd1x_0_gfx3d_clk = {
+	.halt_reg = 0x92cc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x92cc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_mnd1x_0_gfx3d_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_mnd1x_1_gfx3d_clk = {
+	.halt_reg = 0x92d0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x92d0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_mnd1x_1_gfx3d_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_sleep_clk = {
+	.halt_reg = 0x9164,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9164,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc gpu_cc_cx_gdsc = {
+	.gdscr = 0x9110,
+	.gds_hw_ctrl = 0x9124,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x8,
+	.pd = {
+		.name = "gpu_cc_cx_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gpu_cc_gx_gdsc = {
+	.gdscr = 0x905c,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gpu_cc_gx_gdsc",
+		.power_on = gdsc_gx_do_nothing_enable,
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gpu_cc_eliza_clocks[] = {
+	[GPU_CC_AHB_CLK] = &gpu_cc_ahb_clk.clkr,
+	[GPU_CC_CRC_AHB_CLK] = &gpu_cc_crc_ahb_clk.clkr,
+	[GPU_CC_CX_ACCU_SHIFT_CLK] = &gpu_cc_cx_accu_shift_clk.clkr,
+	[GPU_CC_CX_FF_CLK] = &gpu_cc_cx_ff_clk.clkr,
+	[GPU_CC_CX_GMU_CLK] = &gpu_cc_cx_gmu_clk.clkr,
+	[GPU_CC_CXO_CLK] = &gpu_cc_cxo_clk.clkr,
+	[GPU_CC_FF_CLK_SRC] = &gpu_cc_ff_clk_src.clkr,
+	[GPU_CC_FREQ_MEASURE_CLK] = &gpu_cc_freq_measure_clk.clkr,
+	[GPU_CC_GMU_CLK_SRC] = &gpu_cc_gmu_clk_src.clkr,
+	[GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK] = &gpu_cc_hlos1_vote_gpu_smmu_clk.clkr,
+	[GPU_CC_HUB_AON_CLK] = &gpu_cc_hub_aon_clk.clkr,
+	[GPU_CC_HUB_CLK_SRC] = &gpu_cc_hub_clk_src.clkr,
+	[GPU_CC_HUB_CX_INT_CLK] = &gpu_cc_hub_cx_int_clk.clkr,
+	[GPU_CC_MEMNOC_GFX_CLK] = &gpu_cc_memnoc_gfx_clk.clkr,
+	[GPU_CC_MND1X_0_GFX3D_CLK] = &gpu_cc_mnd1x_0_gfx3d_clk.clkr,
+	[GPU_CC_MND1X_1_GFX3D_CLK] = &gpu_cc_mnd1x_1_gfx3d_clk.clkr,
+	[GPU_CC_PLL0] = &gpu_cc_pll0.clkr,
+	[GPU_CC_PLL1] = &gpu_cc_pll1.clkr,
+	[GPU_CC_SLEEP_CLK] = &gpu_cc_sleep_clk.clkr,
+	[GPU_CC_XO_CLK_SRC] = &gpu_cc_xo_clk_src.clkr,
+	[GPU_CC_XO_DIV_CLK_SRC] = &gpu_cc_xo_div_clk_src.clkr,
+};
+
+static struct gdsc *gpu_cc_eliza_gdscs[] = {
+	[GPU_CC_CX_GDSC] = &gpu_cc_cx_gdsc,
+	[GPU_CC_GX_GDSC] = &gpu_cc_gx_gdsc,
+};
+
+static const struct qcom_reset_map gpu_cc_eliza_resets[] = {
+	[GPU_CC_ACD_BCR] = { 0x939c },
+	[GPU_CC_CB_BCR] = { 0x93e4 },
+	[GPU_CC_CX_BCR] = { 0x910c },
+	[GPU_CC_FAST_HUB_BCR] = { 0x9428 },
+	[GPU_CC_FF_BCR] = { 0x94b4 },
+	[GPU_CC_GFX3D_AON_BCR] = { 0x91dc },
+	[GPU_CC_GMU_BCR] = { 0x9358 },
+	[GPU_CC_GX_BCR] = { 0x9058 },
+	[GPU_CC_RBCPR_BCR] = { 0x9224 },
+	[GPU_CC_XO_BCR] = { 0x9000 },
+};
+
+static struct clk_alpha_pll *gpu_cc_eliza_plls[] = {
+	&gpu_cc_pll0,
+	&gpu_cc_pll1,
+};
+
+static const u32 gpu_cc_eliza_critical_cbcrs[] = {
+	0x9004, /* GPU_CC_CXO_AON_CLK */
+	0x900c, /* GPU_CC_DEMET_CLK */
+};
+
+static const struct regmap_config gpu_cc_eliza_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9988,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_driver_data gpu_cc_eliza_driver_data = {
+	.alpha_plls = gpu_cc_eliza_plls,
+	.num_alpha_plls = ARRAY_SIZE(gpu_cc_eliza_plls),
+	.clk_cbcrs = gpu_cc_eliza_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_eliza_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc gpu_cc_eliza_desc = {
+	.config = &gpu_cc_eliza_regmap_config,
+	.clks = gpu_cc_eliza_clocks,
+	.num_clks = ARRAY_SIZE(gpu_cc_eliza_clocks),
+	.resets = gpu_cc_eliza_resets,
+	.num_resets = ARRAY_SIZE(gpu_cc_eliza_resets),
+	.gdscs = gpu_cc_eliza_gdscs,
+	.num_gdscs = ARRAY_SIZE(gpu_cc_eliza_gdscs),
+	.use_rpm = true,
+	.driver_data = &gpu_cc_eliza_driver_data,
+};
+
+static const struct of_device_id gpu_cc_eliza_match_table[] = {
+	{ .compatible = "qcom,eliza-gpucc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gpu_cc_eliza_match_table);
+
+static int gpu_cc_eliza_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gpu_cc_eliza_desc);
+}
+
+static struct platform_driver gpu_cc_eliza_driver = {
+	.probe = gpu_cc_eliza_probe,
+	.driver = {
+		.name = "gpucc-eliza",
+		.of_match_table = gpu_cc_eliza_match_table,
+	},
+};
+
+module_platform_driver(gpu_cc_eliza_driver);
+
+MODULE_DESCRIPTION("QTI GPUCC Eliza Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gpucc-glymur.c b/drivers/clk/qcom/gpucc-glymur.c
index 001b2454786a0c..cac51de20b2950 100644
--- a/drivers/clk/qcom/gpucc-glymur.c
+++ b/drivers/clk/qcom/gpucc-glymur.c
@@ -510,7 +510,7 @@ static struct gdsc gpu_cc_cx_gdsc = {
 		.name = "gpu_cc_cx_gdsc",
 	},
 	.pwrsts = PWRSTS_OFF_ON,
-	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.flags = RETAIN_FF_ENABLE | VOTABLE,
 };
 
 static struct clk_regmap *gpu_cc_glymur_clocks[] = {
diff --git a/drivers/clk/qcom/gpucc-hawi.c b/drivers/clk/qcom/gpucc-hawi.c
new file mode 100644
index 00000000000000..3deb7edf7810de
--- /dev/null
+++ b/drivers/clk/qcom/gpucc-hawi.c
@@ -0,0 +1,477 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,hawi-gpucc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_GPLL0_OUT_MAIN,
+	DT_GPLL0_OUT_MAIN_DIV,
+};
+
+enum {
+	P_BI_TCXO,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL0_OUT_MAIN_DIV,
+	P_GPU_CC_PLL0_OUT_EVEN,
+	P_GPU_CC_PLL0_OUT_MAIN,
+	P_GPU_CC_PLL0_OUT_ODD,
+};
+
+static const struct pll_vco taycan_eha_t_vco[] = {
+	{ 249600000, 2500000000, 0 },
+};
+
+/* 788.0 MHz Configuration */
+static const struct alpha_pll_config gpu_cc_pll0_config = {
+	.l = 0x29,
+	.alpha = 0xaaa,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a8060e0,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+/* 788.0 MHz Configuration */
+static const struct alpha_pll_config gpu_cc_pll0_config_maili = {
+	.l = 0x29,
+	.cal_l = 0x42,
+	.alpha = 0xaaa,
+	.config_ctl_val = 0xa5c400e7,
+	.config_ctl_hi_val = 0x0a806160,
+	.config_ctl_hi1_val = 0xf51dea20,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000002,
+};
+
+static struct clk_alpha_pll gpu_cc_pll0 = {
+	.offset = 0x0,
+	.config = &gpu_cc_pll0_config,
+	.vco_table = taycan_eha_t_vco,
+	.num_vco = ARRAY_SIZE(taycan_eha_t_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct parent_map gpu_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GPU_CC_PLL0_OUT_MAIN, 1 },
+	{ P_GPU_CC_PLL0_OUT_EVEN, 2 },
+	{ P_GPU_CC_PLL0_OUT_ODD, 3 },
+	{ P_GPLL0_OUT_MAIN, 5 },
+	{ P_GPLL0_OUT_MAIN_DIV, 6 },
+};
+
+static const struct clk_parent_data gpu_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gpu_cc_pll0.clkr.hw },
+	{ .hw = &gpu_cc_pll0.clkr.hw },
+	{ .hw = &gpu_cc_pll0.clkr.hw },
+	{ .index = DT_GPLL0_OUT_MAIN },
+	{ .index = DT_GPLL0_OUT_MAIN_DIV },
+};
+
+static const struct freq_tbl ftbl_gpu_cc_gmu_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(788000000, P_GPU_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(825000000, P_GPU_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(880000000, P_GPU_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(917000000, P_GPU_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_gmu_clk_src = {
+	.cmd_rcgr = 0x9318,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_1,
+	.freq_tbl = ftbl_gpu_cc_gmu_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_gmu_clk_src",
+		.parent_data = gpu_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gpu_cc_hub_clk_src[] = {
+	F(150000000, P_GPLL0_OUT_MAIN_DIV, 2, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0),
+	F(400000000, P_GPLL0_OUT_MAIN, 1.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gpu_cc_hub_clk_src = {
+	.cmd_rcgr = 0x93f0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gpu_cc_parent_map_1,
+	.freq_tbl = ftbl_gpu_cc_hub_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_hub_clk_src",
+		.parent_data = gpu_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gpu_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div gpu_cc_hub_div_clk_src = {
+	.reg = 0x9430,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpu_cc_hub_div_clk_src",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gpu_cc_hub_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gpu_cc_ahb_clk = {
+	.halt_reg = 0x90bc,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x90bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_ahb_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gpu_cc_hub_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_accu_shift_clk = {
+	.halt_reg = 0x9104,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9104,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_accu_shift_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_cx_gmu_clk = {
+	.halt_reg = 0x90d4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x90d4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x90d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_cx_gmu_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gpu_cc_gmu_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_demet_clk = {
+	.halt_reg = 0x9010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_demet_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_dpm_clk = {
+	.halt_reg = 0x9108,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9108,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_dpm_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_freq_measure_clk = {
+	.halt_reg = 0x900c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x900c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_freq_measure_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gpu_smmu_vote_clk = {
+	.halt_reg = 0x7000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gpu_smmu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_gx_accu_shift_clk = {
+	.halt_reg = 0x9070,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9070,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_gx_accu_shift_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_hub_aon_clk = {
+	.halt_reg = 0x93ec,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x93ec,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_hub_aon_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_hub_cx_int_clk = {
+	.halt_reg = 0x90e8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x90e8,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x90e8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_hub_cx_int_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gpu_cc_hub_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_memnoc_gfx_clk = {
+	.halt_reg = 0x90ec,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x90ec,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x90ec,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_memnoc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gpu_cc_mxg_ahb_clk = {
+	.halt_reg = 0x9744,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x9744,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpu_cc_mxg_ahb_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gpu_cc_hub_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc gpu_cc_cx_gdsc = {
+	.gdscr = 0x9080,
+	.gds_hw_ctrl = 0x9094,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gpu_cc_cx_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = VOTABLE | POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gpu_cc_mxg_gdsc = {
+	.gdscr = 0x961c,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gpu_cc_mxg_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gpu_cc_hawi_clocks[] = {
+	[GPU_CC_AHB_CLK] = &gpu_cc_ahb_clk.clkr,
+	[GPU_CC_CX_ACCU_SHIFT_CLK] = &gpu_cc_cx_accu_shift_clk.clkr,
+	[GPU_CC_CX_GMU_CLK] = &gpu_cc_cx_gmu_clk.clkr,
+	[GPU_CC_DEMET_CLK] = &gpu_cc_demet_clk.clkr,
+	[GPU_CC_DPM_CLK] = &gpu_cc_dpm_clk.clkr,
+	[GPU_CC_FREQ_MEASURE_CLK] = &gpu_cc_freq_measure_clk.clkr,
+	[GPU_CC_GMU_CLK_SRC] = &gpu_cc_gmu_clk_src.clkr,
+	[GPU_CC_GPU_SMMU_VOTE_CLK] = &gpu_cc_gpu_smmu_vote_clk.clkr,
+	[GPU_CC_GX_ACCU_SHIFT_CLK] = &gpu_cc_gx_accu_shift_clk.clkr,
+	[GPU_CC_HUB_AON_CLK] = &gpu_cc_hub_aon_clk.clkr,
+	[GPU_CC_HUB_CLK_SRC] = &gpu_cc_hub_clk_src.clkr,
+	[GPU_CC_HUB_CX_INT_CLK] = &gpu_cc_hub_cx_int_clk.clkr,
+	[GPU_CC_HUB_DIV_CLK_SRC] = &gpu_cc_hub_div_clk_src.clkr,
+	[GPU_CC_MEMNOC_GFX_CLK] = &gpu_cc_memnoc_gfx_clk.clkr,
+	[GPU_CC_MXG_AHB_CLK] = &gpu_cc_mxg_ahb_clk.clkr,
+	[GPU_CC_PLL0] = &gpu_cc_pll0.clkr,
+};
+
+static struct gdsc *gpu_cc_hawi_gdscs[] = {
+	[GPU_CC_CX_GDSC] = &gpu_cc_cx_gdsc,
+	[GPU_CC_MXG_GDSC] = &gpu_cc_mxg_gdsc,
+};
+
+static const struct qcom_reset_map gpu_cc_hawi_resets[] = {
+	[GPU_CC_CB_BCR] = { 0x93a0 },
+	[GPU_CC_CX_BCR] = { 0x907c },
+	[GPU_CC_FAST_HUB_BCR] = { 0x93e4 },
+	[GPU_CC_GMU_BCR] = { 0x9314 },
+	[GPU_CC_GX_BCR] = { 0x905c },
+	[GPU_CC_MXG_BCR] = { 0x9618 },
+	[GPU_CC_XO_BCR] = { 0x9000 },
+};
+
+static struct clk_alpha_pll *gpu_cc_hawi_plls[] = {
+	&gpu_cc_pll0,
+};
+
+static u32 gpu_cc_hawi_critical_cbcrs[] = {
+	0x93a4, /* GPU_CC_CB_CLK */
+	0x9704, /* GPU_CC_MXG_XO_CLK */
+	0x90e4, /* GPU_CC_CXO_CLK */
+	0x9008, /* GPU_CC_CXO_AON_CLK */
+	0x93e8, /* GPU_CC_RSCC_HUB_AON_CLK */
+	0x9004, /* GPU_CC_RSCC_XO_AON_CLK */
+	0x90cc, /* GPU_CC_SLEEP_CLK */
+};
+
+static const struct regmap_config gpu_cc_hawi_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9744,
+	.fast_io = true,
+};
+
+static struct qcom_cc_driver_data gpu_cc_hawi_driver_data = {
+	.alpha_plls = gpu_cc_hawi_plls,
+	.num_alpha_plls = ARRAY_SIZE(gpu_cc_hawi_plls),
+	.clk_cbcrs = gpu_cc_hawi_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_hawi_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc gpu_cc_hawi_desc = {
+	.config = &gpu_cc_hawi_regmap_config,
+	.clks = gpu_cc_hawi_clocks,
+	.num_clks = ARRAY_SIZE(gpu_cc_hawi_clocks),
+	.resets = gpu_cc_hawi_resets,
+	.num_resets = ARRAY_SIZE(gpu_cc_hawi_resets),
+	.gdscs = gpu_cc_hawi_gdscs,
+	.num_gdscs = ARRAY_SIZE(gpu_cc_hawi_gdscs),
+	.use_rpm = true,
+	.driver_data = &gpu_cc_hawi_driver_data,
+};
+
+static const struct of_device_id gpu_cc_hawi_match_table[] = {
+	{ .compatible = "qcom,hawi-gpucc" },
+	{ .compatible = "qcom,maili-gpucc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gpu_cc_hawi_match_table);
+
+static int gpu_cc_hawi_probe(struct platform_device *pdev)
+{
+	if (device_is_compatible(&pdev->dev, "qcom,maili-gpucc"))
+		gpu_cc_pll0.config = &gpu_cc_pll0_config_maili;
+
+	return qcom_cc_probe(pdev, &gpu_cc_hawi_desc);
+}
+
+static struct platform_driver gpu_cc_hawi_driver = {
+	.probe = gpu_cc_hawi_probe,
+	.driver = {
+		.name = "gpucc-hawi",
+		.of_match_table = gpu_cc_hawi_match_table,
+	},
+};
+
+module_platform_driver(gpu_cc_hawi_driver);
+
+MODULE_DESCRIPTION("QTI GPUCC HAWI Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gpucc-kaanapali.c b/drivers/clk/qcom/gpucc-kaanapali.c
index ae5563e516f677..f75f2caa2205c4 100644
--- a/drivers/clk/qcom/gpucc-kaanapali.c
+++ b/drivers/clk/qcom/gpucc-kaanapali.c
@@ -380,7 +380,7 @@ static struct gdsc gpu_cc_cx_gdsc = {
 		.name = "gpu_cc_cx_gdsc",
 	},
 	.pwrsts = PWRSTS_OFF_ON,
-	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+	.flags = RETAIN_FF_ENABLE | VOTABLE,
 };
 
 static struct clk_regmap *gpu_cc_kaanapali_clocks[] = {
diff --git a/drivers/clk/qcom/gpucc-sar2130p.c b/drivers/clk/qcom/gpucc-sar2130p.c
index dd72b2a48c42da..fa261664c1bf52 100644
--- a/drivers/clk/qcom/gpucc-sar2130p.c
+++ b/drivers/clk/qcom/gpucc-sar2130p.c
@@ -455,7 +455,17 @@ static const struct regmap_config gpu_cc_sar2130p_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gpu_cc_sar2130p_critical_cbcrs[] = {
+	0x900c, /* GPU_CC_DEMET_CLK */
+};
+
+static const struct qcom_cc_driver_data gpu_cc_sar2130p_driver_data = {
+	.clk_cbcrs = gpu_cc_sar2130p_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_sar2130p_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc gpu_cc_sar2130p_desc = {
+	.driver_data = &gpu_cc_sar2130p_driver_data,
 	.config = &gpu_cc_sar2130p_regmap_config,
 	.clks = gpu_cc_sar2130p_clocks,
 	.num_clks = ARRAY_SIZE(gpu_cc_sar2130p_clocks),
@@ -483,9 +493,6 @@ static int gpu_cc_sar2130p_probe(struct platform_device *pdev)
 	clk_lucid_ole_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config);
 	clk_lucid_ole_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x900c); /* GPU_CC_DEMET_CLK */
-
 	return qcom_cc_really_probe(dev, &gpu_cc_sar2130p_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/gpucc-sc7280.c b/drivers/clk/qcom/gpucc-sc7280.c
index bd699a62451784..9b267276eac7b3 100644
--- a/drivers/clk/qcom/gpucc-sc7280.c
+++ b/drivers/clk/qcom/gpucc-sc7280.c
@@ -440,7 +440,24 @@ static const struct regmap_config gpu_cc_sc7280_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gpu_cc_sc7280_critical_cbcrs[] = {
+	0x1170, /* GPU_CC_CB_CLK */
+	0x1098, /* GPUCC_CX_GMU_CLK */
+};
+
+static void gpu_cc_sc7280_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	regmap_update_bits(regmap, 0x1098, BIT(13), BIT(13));
+}
+
+static const struct qcom_cc_driver_data gpu_cc_sc7280_driver_data = {
+	.clk_cbcrs = gpu_cc_sc7280_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_sc7280_critical_cbcrs),
+	.clk_regs_configure = gpu_cc_sc7280_regs_configure,
+};
+
 static const struct qcom_cc_desc gpu_cc_sc7280_desc = {
+	.driver_data = &gpu_cc_sc7280_driver_data,
 	.config = &gpu_cc_sc7280_regmap_config,
 	.clks = gpu_cc_sc7280_clocks,
 	.num_clks = ARRAY_SIZE(gpu_cc_sc7280_clocks),
@@ -464,11 +481,6 @@ static int gpu_cc_sc7280_probe(struct platform_device *pdev)
 
 	clk_lucid_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x1170); /* GPU_CC_CB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x1098); /* GPUCC_CX_GMU_CLK */
-	regmap_update_bits(regmap, 0x1098, BIT(13), BIT(13));
-
 	return qcom_cc_really_probe(&pdev->dev, &gpu_cc_sc7280_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/gpucc-sc8280xp.c b/drivers/clk/qcom/gpucc-sc8280xp.c
index 5dd90b854afb6c..2ccc083a4f9876 100644
--- a/drivers/clk/qcom/gpucc-sc8280xp.c
+++ b/drivers/clk/qcom/gpucc-sc8280xp.c
@@ -415,7 +415,18 @@ static const struct regmap_config gpu_cc_sc8280xp_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gpu_cc_sc8280xp_critical_cbcrs[] = {
+	0x1170, /* GPU_CC_CB_CLK */
+	0x109c, /* GPU_CC_CXO_CLK */
+};
+
+static const struct qcom_cc_driver_data gpu_cc_sc8280xp_driver_data = {
+	.clk_cbcrs = gpu_cc_sc8280xp_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_sc8280xp_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc gpu_cc_sc8280xp_desc = {
+	.driver_data = &gpu_cc_sc8280xp_driver_data,
 	.config = &gpu_cc_sc8280xp_regmap_config,
 	.clks = gpu_cc_sc8280xp_clocks,
 	.num_clks = ARRAY_SIZE(gpu_cc_sc8280xp_clocks),
@@ -445,10 +456,6 @@ static int gpu_cc_sc8280xp_probe(struct platform_device *pdev)
 	clk_lucid_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config);
 	clk_lucid_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x1170); /* GPU_CC_CB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x109c); /* GPU_CC_CXO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &gpu_cc_sc8280xp_desc, regmap);
 	pm_runtime_put(&pdev->dev);
 
diff --git a/drivers/clk/qcom/gpucc-sm4450.c b/drivers/clk/qcom/gpucc-sm4450.c
index 808b1eaa59d123..bd0f3fc2b72608 100644
--- a/drivers/clk/qcom/gpucc-sm4450.c
+++ b/drivers/clk/qcom/gpucc-sm4450.c
@@ -754,7 +754,19 @@ static const struct regmap_config gpu_cc_sm4450_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gpu_cc_sm4450_critical_cbcrs[] = {
+	0x93a4, /* GPU_CC_CB_CLK */
+	0x9004, /* GPU_CC_CXO_AON_CLK */
+	0x900c, /* GPU_CC_DEMET_CLK */
+};
+
+static const struct qcom_cc_driver_data gpu_cc_sm4450_driver_data = {
+	.clk_cbcrs = gpu_cc_sm4450_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_sm4450_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc gpu_cc_sm4450_desc = {
+	.driver_data = &gpu_cc_sm4450_driver_data,
 	.config = &gpu_cc_sm4450_regmap_config,
 	.clks = gpu_cc_sm4450_clocks,
 	.num_clks = ARRAY_SIZE(gpu_cc_sm4450_clocks),
@@ -781,11 +793,6 @@ static int gpu_cc_sm4450_probe(struct platform_device *pdev)
 	clk_lucid_evo_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config);
 	clk_lucid_evo_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config);
 
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0x93a4); /* GPU_CC_CB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x9004); /* GPU_CC_CXO_AON_CLK */
-	qcom_branch_set_clk_en(regmap, 0x900c); /* GPU_CC_DEMET_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &gpu_cc_sm4450_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/gpucc-sm8550.c b/drivers/clk/qcom/gpucc-sm8550.c
index 53614e980b262b..f8dd1727c0804d 100644
--- a/drivers/clk/qcom/gpucc-sm8550.c
+++ b/drivers/clk/qcom/gpucc-sm8550.c
@@ -547,7 +547,18 @@ static const struct regmap_config gpu_cc_sm8550_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gpu_cc_sm8550_critical_cbcrs[] = {
+	0x9004, /* GPU_CC_CXO_AON_CLK */
+	0x900c, /* GPU_CC_DEMET_CLK */
+};
+
+static const struct qcom_cc_driver_data gpu_cc_sm8550_driver_data = {
+	.clk_cbcrs = gpu_cc_sm8550_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_sm8550_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc gpu_cc_sm8550_desc = {
+	.driver_data = &gpu_cc_sm8550_driver_data,
 	.config = &gpu_cc_sm8550_regmap_config,
 	.clks = gpu_cc_sm8550_clocks,
 	.num_clks = ARRAY_SIZE(gpu_cc_sm8550_clocks),
@@ -574,10 +585,6 @@ static int gpu_cc_sm8550_probe(struct platform_device *pdev)
 	clk_lucid_ole_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config);
 	clk_lucid_ole_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x9004); /* GPU_CC_CXO_AON_CLK */
-	qcom_branch_set_clk_en(regmap, 0x900c); /* GPU_CC_DEMET_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &gpu_cc_sm8550_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/gpucc-x1e80100.c b/drivers/clk/qcom/gpucc-x1e80100.c
index f9161dbaedd783..10a0b11927e389 100644
--- a/drivers/clk/qcom/gpucc-x1e80100.c
+++ b/drivers/clk/qcom/gpucc-x1e80100.c
@@ -609,7 +609,17 @@ static const struct regmap_config gpu_cc_x1e80100_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gpu_cc_x1e80100_critical_cbcrs[] = {
+	0x93a4, /* GPU_CC_CB_CLK */
+};
+
+static const struct qcom_cc_driver_data gpu_cc_x1e80100_driver_data = {
+	.clk_cbcrs = gpu_cc_x1e80100_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_x1e80100_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc gpu_cc_x1e80100_desc = {
+	.driver_data = &gpu_cc_x1e80100_driver_data,
 	.config = &gpu_cc_x1e80100_regmap_config,
 	.clks = gpu_cc_x1e80100_clocks,
 	.num_clks = ARRAY_SIZE(gpu_cc_x1e80100_clocks),
@@ -636,9 +646,6 @@ static int gpu_cc_x1e80100_probe(struct platform_device *pdev)
 	clk_zonda_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config);
 	clk_lucid_evo_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config);
 
-	/* Keep clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0x93a4); /* GPU_CC_CB_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &gpu_cc_x1e80100_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/gpucc-x1p42100.c b/drivers/clk/qcom/gpucc-x1p42100.c
index cfc34e0fd29049..3f3fec257f2d5b 100644
--- a/drivers/clk/qcom/gpucc-x1p42100.c
+++ b/drivers/clk/qcom/gpucc-x1p42100.c
@@ -522,7 +522,19 @@ static const struct regmap_config gpu_cc_x1p42100_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 gpu_cc_x1p42100_critical_cbcrs[] = {
+	0x93a4, /* GPU_CC_CB_CLK */
+	0x9004, /* GPU_CC_CXO_AON_CLK */
+	0x900c, /* GPU_CC_DEMET_CLK */
+};
+
+static const struct qcom_cc_driver_data gpu_cc_x1p42100_driver_data = {
+	.clk_cbcrs = gpu_cc_x1p42100_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gpu_cc_x1p42100_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc gpu_cc_x1p42100_desc = {
+	.driver_data = &gpu_cc_x1p42100_driver_data,
 	.config = &gpu_cc_x1p42100_regmap_config,
 	.clks = gpu_cc_x1p42100_clocks,
 	.num_clks = ARRAY_SIZE(gpu_cc_x1p42100_clocks),
@@ -560,11 +572,6 @@ static int gpu_cc_x1p42100_probe(struct platform_device *pdev)
 	clk_lucid_ole_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config);
 	clk_lucid_ole_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config);
 
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0x93a4); /* GPU_CC_CB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x9004); /* GPU_CC_CXO_AON_CLK */
-	qcom_branch_set_clk_en(regmap, 0x900c); /* GPU_CC_DEMET_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &gpu_cc_x1p42100_desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
diff --git a/drivers/clk/qcom/ipq-cmn-pll.c b/drivers/clk/qcom/ipq-cmn-pll.c
index dafe8c1738df83..74a2043632b7df 100644
--- a/drivers/clk/qcom/ipq-cmn-pll.c
+++ b/drivers/clk/qcom/ipq-cmn-pll.c
@@ -44,6 +44,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/bitops.h>
 #include <linux/clk-provider.h>
 #include <linux/delay.h>
 #include <linux/err.h>
@@ -55,17 +56,31 @@
 
 #include <dt-bindings/clock/qcom,ipq-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq5018-cmn-pll.h>
+#include <dt-bindings/clock/qcom,ipq5210-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq5332-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq5424-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq6018-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq8074-cmn-pll.h>
 
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+
 #define CMN_PLL_REFCLK_SRC_SELECTION		0x28
 #define CMN_PLL_REFCLK_SRC_DIV			GENMASK(9, 8)
 
 #define CMN_PLL_LOCKED				0x64
 #define CMN_PLL_CLKS_LOCKED			BIT(8)
 
+#define CMN_PLL_NSS_PPE_FREQ_CTRL		0x98
+#define CMN_PLL_NSS_CLK_SEL			GENMASK(13, 8)
+#define CMN_PLL_PPE_CLK_SEL			GENMASK(5, 0)
+
+#define CMN_PLL_PON_CONFIG			0x42c
+#define CMN_PLL_PON_MODE_SEL			BIT(9)
+#define CMN_PLL_PON_EN				BIT(8)
+#define CMN_PLL_PON_DIV_CTRL			GENMASK(7, 0)
+#define CMN_PLL_GEPHY_312P5M_125M_SEL		BIT(10)
+
 #define CMN_PLL_POWER_ON_AND_RESET		0x780
 #define CMN_ANA_EN_SW_RSTN			BIT(6)
 
@@ -80,36 +95,110 @@
 #define CMN_PLL_DIVIDER_CTRL			0x794
 #define CMN_PLL_DIVIDER_CTRL_FACTOR		GENMASK(9, 0)
 
+/* Clock gate enable bits. */
+#define CMN_PLL_OUTPUT_RELATED_1		0x79c
+#define CLK25M_EN_BIT				15
+#define CLK50M_EN_BIT3				14
+/* Documented as CLK250M_EN in the register description; gates ephy-50mhz. */
+#define CLK250M_EN_BIT				13
+#define CLK31P25M_EN_BIT			12
+#define CLK50M_EN_BIT1				11
+#define CLK50M_EN_BIT2				10
+
+/**
+ * enum cmn_pll_clk_type - CMN PLL output clock registration type
+ * @CMN_PLL_CLK_FIXED_RATE: plain fixed rate clock
+ * @CMN_PLL_CLK_FIXED_GATE: fixed rate clock with a hardware gate
+ * @CMN_PLL_CLK_NSS: NSS clock with configurable divider
+ * @CMN_PLL_CLK_PPE: PPE clock with configurable divider
+ * @CMN_PLL_CLK_PON: PON reference clock
+ * @CMN_PLL_CLK_EPHY_RAW: EPHY-RAW clock
+ */
+enum cmn_pll_clk_type {
+	CMN_PLL_CLK_FIXED_RATE,
+	CMN_PLL_CLK_FIXED_GATE,
+	CMN_PLL_CLK_NSS,
+	CMN_PLL_CLK_PPE,
+	CMN_PLL_CLK_PON,
+	CMN_PLL_CLK_EPHY_RAW,
+};
+
 /**
  * struct cmn_pll_fixed_output_clk - CMN PLL output clocks information
  * @id:	Clock specifier to be supplied
  * @name: Clock name to be registered
+ * @type: Clock registration type
  * @rate: Clock rate
+ * @enable_bit: Enable bit in CMN_PLL_OUTPUT_RELATED_1 for gate clock,
+ *              -1 for non-gated clocks.
  */
 struct cmn_pll_fixed_output_clk {
 	unsigned int id;
 	const char *name;
+	enum cmn_pll_clk_type type;
 	unsigned long rate;
+	int enable_bit;
 };
 
 /**
  * struct clk_cmn_pll - CMN PLL hardware specific data
  * @regmap: hardware regmap.
  * @hw: handle between common and hardware-specific interfaces
+ * @div2_hw: fixed /2 clock derived from the CMN PLL output; present on
+ *           every supported SoC.
  */
 struct clk_cmn_pll {
 	struct regmap *regmap;
 	struct clk_hw hw;
+	struct clk_hw *div2_hw;
 };
 
-#define CLK_PLL_OUTPUT(_id, _name, _rate) {		\
-	.id =		_id,				\
-	.name =		_name,				\
-	.rate =		_rate,				\
+/**
+ * struct clk_fixed_gate - fixed rate clock with a hardware gate
+ * @clkr: handle between common and hardware-specific interfaces; its
+ *        enable_reg/enable_mask are set to CMN_PLL_OUTPUT_RELATED_1 and
+ *        this clock's gate bit, and reused via clk_enable_regmap() and
+ *        friends. That register is shared across multiple gate clocks,
+ *        but regmap already serializes read-modify-write access to a
+ *        given register, so no additional locking is needed here.
+ * @rate: fixed clock rate.
+ */
+struct clk_fixed_gate {
+	struct clk_regmap clkr;
+	unsigned long rate;
+};
+
+#define CLK_PLL_OUTPUT_RAW(_id, _name, _type, _rate, _bit) {	\
+	.id =		_id,					\
+	.name =		_name,					\
+	.type =		_type,					\
+	.rate =		_rate,					\
+	.enable_bit =	_bit,					\
 }
 
+#define CLK_PLL_OUTPUT(_id, _name, _rate)	\
+	CLK_PLL_OUTPUT_RAW(_id, _name, CMN_PLL_CLK_FIXED_RATE, _rate, -1)
+
+#define CLK_PLL_GATE(_id, _name, _rate, _bit)	\
+	CLK_PLL_OUTPUT_RAW(_id, _name, CMN_PLL_CLK_FIXED_GATE, _rate, _bit)
+
+#define CLK_PLL_NSS(_id, _name)	\
+	CLK_PLL_OUTPUT_RAW(_id, _name, CMN_PLL_CLK_NSS, 0, -1)
+
+#define CLK_PLL_PPE(_id, _name)	\
+	CLK_PLL_OUTPUT_RAW(_id, _name, CMN_PLL_CLK_PPE, 0, -1)
+
+#define CLK_PLL_PON(_id, _name)	\
+	CLK_PLL_OUTPUT_RAW(_id, _name, CMN_PLL_CLK_PON, 0, -1)
+
+#define CLK_PLL_EPHY_RAW(_id, _name)	\
+	CLK_PLL_OUTPUT_RAW(_id, _name, CMN_PLL_CLK_EPHY_RAW, 0, -1)
+
 #define to_clk_cmn_pll(_hw) container_of(_hw, struct clk_cmn_pll, hw)
 
+#define to_clk_fixed_gate(_hw) \
+	container_of(to_clk_regmap(_hw), struct clk_fixed_gate, clkr)
+
 static const struct regmap_config ipq_cmn_pll_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
@@ -124,6 +213,22 @@ static const struct cmn_pll_fixed_output_clk ipq5018_output_clks[] = {
 	{ /* Sentinel */ }
 };
 
+static const struct cmn_pll_fixed_output_clk ipq5210_output_clks[] = {
+	CLK_PLL_OUTPUT(IPQ5210_XO_24MHZ_CLK, "xo-24mhz", 24000000UL),
+	CLK_PLL_OUTPUT(IPQ5210_SLEEP_32KHZ_CLK, "sleep-32khz", 32000UL),
+	CLK_PLL_GATE(IPQ5210_PCS_31P25MHZ_CLK, "pcs-31p25mhz", 31250000UL, CLK31P25M_EN_BIT),
+	CLK_PLL_GATE(IPQ5210_ETH0_50MHZ_CLK, "eth0-50mhz", 50000000UL, CLK50M_EN_BIT1),
+	CLK_PLL_GATE(IPQ5210_ETH1_50MHZ_CLK, "eth1-50mhz", 50000000UL, CLK50M_EN_BIT2),
+	CLK_PLL_GATE(IPQ5210_ETH2_50MHZ_CLK, "eth2-50mhz", 50000000UL, CLK50M_EN_BIT3),
+	CLK_PLL_GATE(IPQ5210_EPHY_50MHZ_CLK, "ephy-50mhz", 50000000UL, CLK250M_EN_BIT),
+	CLK_PLL_GATE(IPQ5210_ETH_25MHZ_CLK, "eth-25mhz", 25000000UL, CLK25M_EN_BIT),
+	CLK_PLL_NSS(IPQ5210_NSS_CLK, "nss"),
+	CLK_PLL_PPE(IPQ5210_PPE_CLK, "ppe"),
+	CLK_PLL_PON(IPQ5210_PON_CLK, "pon"),
+	CLK_PLL_EPHY_RAW(IPQ5210_EPHY_RAW_CLK, "ephy-raw"),
+	{ /* Sentinel */ }
+};
+
 static const struct cmn_pll_fixed_output_clk ipq6018_output_clks[] = {
 	CLK_PLL_OUTPUT(IPQ6018_BIAS_PLL_CC_CLK, "bias_pll_cc_clk", 300000000UL),
 	CLK_PLL_OUTPUT(IPQ6018_BIAS_PLL_NSS_NOC_CLK, "bias_pll_nss_noc_clk", 416500000UL),
@@ -357,11 +462,350 @@ static struct clk_hw *ipq_cmn_pll_clk_hw_register(struct platform_device *pdev)
 	return &cmn_pll->hw;
 }
 
+static struct clk_hw *ipq_cmn_pll_regmap_div_register(struct device *dev,
+						      const char *name,
+						      struct clk_hw *parent_hw,
+						      u32 field_mask)
+{
+	struct clk_parent_data pdata = { .hw = parent_hw };
+	struct clk_regmap_div *div_clk;
+	int ret;
+
+	div_clk = devm_kzalloc(dev, sizeof(*div_clk), GFP_KERNEL);
+	if (!div_clk)
+		return ERR_PTR(-ENOMEM);
+
+	div_clk->reg = CMN_PLL_NSS_PPE_FREQ_CTRL;
+	div_clk->shift = __ffs(field_mask);
+	div_clk->width = hweight32(field_mask);
+	/*
+	 * CMN_PLL_NSS_CLK_SEL / CMN_PLL_PPE_CLK_SEL reset to a valid, non-zero
+	 * divider in hardware. CLK_DIVIDER_ALLOW_ZERO is deliberately not set:
+	 * a divider field read back as 0 is genuinely invalid and should trip
+	 * the core clk-divider's zero-divisor WARN rather than be silently
+	 * tolerated.
+	 */
+	div_clk->flags = CLK_DIVIDER_ONE_BASED;
+	div_clk->clkr.hw.init = &(struct clk_init_data){
+		.name = name,
+		.parent_data = &pdata,
+		.num_parents = 1,
+		.ops = &clk_regmap_div_ops,
+	};
+
+	ret = devm_clk_register_regmap(dev, &div_clk->clkr);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &div_clk->clkr.hw;
+}
+
+/*
+ * PON (Passive Optical Network) reference clock operations.
+ * The PON refclk's parent is cmn_pll_div2 (CMN PLL rate / 2); it is
+ * then divided by a configurable 8-bit divider (1-255).
+ */
+static unsigned long clk_pon_clk_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
+{
+	struct clk_regmap *pon_clk = to_clk_regmap(hw);
+	u32 val, div;
+	int ret;
+
+	ret = regmap_read(pon_clk->regmap, CMN_PLL_PON_CONFIG, &val);
+	if (ret)
+		return 0;
+
+	/* Check if in UNIPHY mode (bit 9 = 0) - fixed 31.25 MHz */
+	if (!(val & CMN_PLL_PON_MODE_SEL))
+		return 31250000UL;
+
+	/* PON mode: calculate from divider */
+	div = FIELD_GET(CMN_PLL_PON_DIV_CTRL, val);
+	if (!div)
+		return 0;
+
+	return DIV_ROUND_CLOSEST_ULL((u64)parent_rate, div);
+}
+
+static int clk_pon_clk_determine_rate(struct clk_hw *hw,
+				      struct clk_rate_request *req)
+{
+	unsigned long div, pon_rate, uniphy_rate = 31250000UL;
+	bool uniphy_rate_valid, pon_rate_valid;
+
+	if (!req->rate || !req->best_parent_rate)
+		return -EINVAL;
+
+	div = DIV64_U64_ROUND_CLOSEST((u64)req->best_parent_rate, req->rate);
+
+	/* Clamp to valid range (1-255) */
+	div = clamp_t(unsigned long, div, 1, 255);
+
+	pon_rate = DIV_ROUND_CLOSEST_ULL((u64)req->best_parent_rate, div);
+
+	uniphy_rate_valid = uniphy_rate >= req->min_rate &&
+			    uniphy_rate <= req->max_rate;
+	pon_rate_valid = pon_rate >= req->min_rate &&
+			 pon_rate <= req->max_rate;
+
+	if (!uniphy_rate_valid && !pon_rate_valid)
+		return -EINVAL;
+
+	/* Pick whichever mode gets closer to the requested rate */
+	if (uniphy_rate_valid && pon_rate_valid) {
+		unsigned long diff_uniphy, diff_pon;
+
+		diff_uniphy = abs_diff(req->rate, uniphy_rate);
+		diff_pon = abs_diff(req->rate, pon_rate);
+		req->rate = diff_uniphy < diff_pon ? uniphy_rate : pon_rate;
+	} else {
+		req->rate = uniphy_rate_valid ? uniphy_rate : pon_rate;
+	}
+
+	return 0;
+}
+
+static int clk_pon_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct clk_regmap *pon_clk = to_clk_regmap(hw);
+	unsigned long div;
+
+	if (rate == 0)
+		return -EINVAL;
+
+	/*
+	 * An exact request for 31.25 MHz is always satisfiable by UNIPHY
+	 * mode, even though PON mode with a suitable divider can produce
+	 * the same frequency for some parent rates. Preferring UNIPHY here
+	 * is safe: this function is only ever called with a rate produced
+	 * by clk_pon_clk_determine_rate() through the standard
+	 * clk_set_rate() path, and either mode yields the identical output
+	 * rate for this value.
+	 */
+	if (rate == 31250000UL)
+		return regmap_clear_bits(pon_clk->regmap, CMN_PLL_PON_CONFIG,
+					 CMN_PLL_PON_MODE_SEL);
+
+	div = DIV64_U64_ROUND_CLOSEST((u64)parent_rate, rate);
+	if (div == 0 || div > 255)
+		return -EINVAL;
+
+	/* Switch to PON mode and program the divider in a single write */
+	return regmap_update_bits(pon_clk->regmap, CMN_PLL_PON_CONFIG,
+				  CMN_PLL_PON_MODE_SEL | CMN_PLL_PON_DIV_CTRL,
+				  CMN_PLL_PON_MODE_SEL |
+				  FIELD_PREP(CMN_PLL_PON_DIV_CTRL, div));
+}
+
+static const struct clk_ops clk_pon_clk_ops = {
+	.enable = clk_enable_regmap,
+	.disable = clk_disable_regmap,
+	.is_enabled = clk_is_enabled_regmap,
+	.recalc_rate = clk_pon_clk_recalc_rate,
+	.determine_rate = clk_pon_clk_determine_rate,
+	.set_rate = clk_pon_clk_set_rate,
+};
+
+static struct clk_hw *ipq_cmn_pll_pon_clk_register(struct device *dev,
+						   const char *name,
+						   struct clk_hw *parent_hw)
+{
+	struct clk_parent_data pdata = { .hw = parent_hw };
+	struct clk_init_data init = {};
+	struct clk_regmap *pon_clk;
+	int ret;
+
+	pon_clk = devm_kzalloc(dev, sizeof(*pon_clk), GFP_KERNEL);
+	if (!pon_clk)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.parent_data = &pdata;
+	init.num_parents = 1;
+	init.ops = &clk_pon_clk_ops;
+	/*
+	 * The PON reference clock may already be enabled by bootloader
+	 * or consumed by hardware without an in-kernel client driver.
+	 * Add CLK_IGNORE_UNUSED so the clock framework does not disable
+	 * it when no consumer has claimed it.
+	 */
+	init.flags = CLK_IGNORE_UNUSED;
+
+	pon_clk->hw.init = &init;
+	pon_clk->enable_reg = CMN_PLL_PON_CONFIG;
+	pon_clk->enable_mask = CMN_PLL_PON_EN;
+
+	ret = devm_clk_register_regmap(dev, pon_clk);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &pon_clk->hw;
+}
+
+/*
+ * EPHY-RAW clock operations for IPQ5210.
+ * The output clock rate is selected via bit 10 of CMN_PLL_PON_CONFIG:
+ *   0: 125 MHz  (for 1G link speed)
+ *   1: 312.5 MHz (for 2.5G link speed)
+ */
+static unsigned long clk_ephy_raw_recalc_rate(struct clk_hw *hw,
+					      unsigned long parent_rate)
+{
+	struct clk_regmap *ephy_raw_clk = to_clk_regmap(hw);
+	u32 val;
+	int ret;
+
+	ret = regmap_read(ephy_raw_clk->regmap, CMN_PLL_PON_CONFIG, &val);
+	if (WARN_ON_ONCE(ret))
+		return 0;
+
+	if (val & CMN_PLL_GEPHY_312P5M_125M_SEL)
+		return 312500000UL;
+
+	return 125000000UL;
+}
+
+static int clk_ephy_raw_determine_rate(struct clk_hw *hw,
+				       struct clk_rate_request *req)
+{
+	unsigned long rate_125m = 125000000UL, rate_312p5m = 312500000UL;
+	bool rate_125m_valid, rate_312p5m_valid;
+
+	rate_125m_valid = rate_125m >= req->min_rate &&
+			  rate_125m <= req->max_rate;
+	rate_312p5m_valid = rate_312p5m >= req->min_rate &&
+			    rate_312p5m <= req->max_rate;
+
+	if (!rate_125m_valid && !rate_312p5m_valid)
+		return -EINVAL;
+
+	/* Pick whichever of the two supported rates is closer to the request */
+	if (rate_125m_valid && rate_312p5m_valid) {
+		unsigned long diff_125m, diff_312p5m;
+
+		diff_125m = abs_diff(req->rate, rate_125m);
+		diff_312p5m = abs_diff(req->rate, rate_312p5m);
+		req->rate = diff_125m < diff_312p5m ? rate_125m : rate_312p5m;
+	} else {
+		req->rate = rate_125m_valid ? rate_125m : rate_312p5m;
+	}
+
+	return 0;
+}
+
+static int clk_ephy_raw_set_rate(struct clk_hw *hw, unsigned long rate,
+				 unsigned long parent_rate)
+{
+	struct clk_regmap *ephy_raw_clk = to_clk_regmap(hw);
+
+	if (rate == 125000000UL)
+		return regmap_clear_bits(ephy_raw_clk->regmap,
+					 CMN_PLL_PON_CONFIG,
+					 CMN_PLL_GEPHY_312P5M_125M_SEL);
+
+	if (rate == 312500000UL)
+		return regmap_set_bits(ephy_raw_clk->regmap,
+				       CMN_PLL_PON_CONFIG,
+				       CMN_PLL_GEPHY_312P5M_125M_SEL);
+
+	return -EINVAL;
+}
+
+static const struct clk_ops clk_ephy_raw_ops = {
+	.recalc_rate = clk_ephy_raw_recalc_rate,
+	.determine_rate = clk_ephy_raw_determine_rate,
+	.set_rate = clk_ephy_raw_set_rate,
+};
+
+static struct clk_hw *ipq_cmn_pll_ephy_raw_register(struct device *dev,
+						    const char *name,
+						    struct clk_hw *parent_hw)
+{
+	struct clk_parent_data pdata = { .hw = parent_hw };
+	struct clk_regmap *ephy_raw_clk;
+	struct clk_init_data init = {};
+	int ret;
+
+	ephy_raw_clk = devm_kzalloc(dev, sizeof(*ephy_raw_clk), GFP_KERNEL);
+	if (!ephy_raw_clk)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.parent_data = &pdata;
+	init.num_parents = 1;
+	init.ops = &clk_ephy_raw_ops;
+
+	ephy_raw_clk->hw.init = &init;
+
+	ret = devm_clk_register_regmap(dev, ephy_raw_clk);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &ephy_raw_clk->hw;
+}
+
+static unsigned long clk_fixed_gate_recalc_rate(struct clk_hw *hw,
+						unsigned long parent_rate)
+{
+	struct clk_fixed_gate *gate_clk = to_clk_fixed_gate(hw);
+
+	return gate_clk->rate;
+}
+
+static const struct clk_ops clk_fixed_gate_ops = {
+	.enable = clk_enable_regmap,
+	.disable = clk_disable_regmap,
+	.is_enabled = clk_is_enabled_regmap,
+	.recalc_rate = clk_fixed_gate_recalc_rate,
+};
+
+static struct clk_hw *ipq_cmn_pll_register_fixed_gate(struct device *dev,
+						      const char *name,
+						      struct clk_hw *parent_hw,
+						      int enable_bit,
+						      unsigned long rate)
+{
+	struct clk_parent_data pdata = { .hw = parent_hw };
+	struct clk_fixed_gate *gate_clk;
+	struct clk_init_data init = {};
+	int ret;
+
+	gate_clk = devm_kzalloc(dev, sizeof(*gate_clk), GFP_KERNEL);
+	if (!gate_clk)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.parent_data = &pdata;
+	init.num_parents = 1;
+	init.ops = &clk_fixed_gate_ops;
+	/*
+	 * These gated clocks may be relied on by external hardware or
+	 * bootloader-enabled paths without an in-kernel client driver.
+	 * Add CLK_IGNORE_UNUSED so the clock framework does not disable
+	 * them when no consumer has claimed them.
+	 */
+	init.flags = CLK_IGNORE_UNUSED;
+
+	gate_clk->clkr.hw.init = &init;
+	gate_clk->clkr.enable_reg = CMN_PLL_OUTPUT_RELATED_1;
+	gate_clk->clkr.enable_mask = BIT(enable_bit);
+	gate_clk->rate = rate;
+
+	ret = devm_clk_register_regmap(dev, &gate_clk->clkr);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &gate_clk->clkr.hw;
+}
+
 static int ipq_cmn_pll_register_clks(struct platform_device *pdev)
 {
 	const struct cmn_pll_fixed_output_clk *p, *fixed_clk;
 	struct clk_hw_onecell_data *hw_data;
 	struct device *dev = &pdev->dev;
+	struct clk_cmn_pll *cmn_pll;
 	struct clk_hw *cmn_pll_hw;
 	unsigned int num_clks;
 	struct clk_hw *hw;
@@ -380,6 +824,8 @@ static int ipq_cmn_pll_register_clks(struct platform_device *pdev)
 	if (!hw_data)
 		return -ENOMEM;
 
+	hw_data->num = num_clks + 1;
+
 	/*
 	 * Register the CMN PLL clock, which is the parent clock of
 	 * the fixed rate output clocks.
@@ -388,15 +834,61 @@ static int ipq_cmn_pll_register_clks(struct platform_device *pdev)
 	if (IS_ERR(cmn_pll_hw))
 		return PTR_ERR(cmn_pll_hw);
 
+	cmn_pll = to_clk_cmn_pll(cmn_pll_hw);
+	cmn_pll->div2_hw = devm_clk_hw_register_fixed_factor_parent_hw(dev,
+								       "cmn_pll_div2",
+								       cmn_pll_hw,
+								       0, 1, 2);
+	if (IS_ERR(cmn_pll->div2_hw))
+		return PTR_ERR(cmn_pll->div2_hw);
+
 	/* Register the fixed rate output clocks. */
 	for (i = 0; i < num_clks; i++) {
-		hw = clk_hw_register_fixed_rate_parent_hw(dev, fixed_clk[i].name,
-							  cmn_pll_hw, 0,
-							  fixed_clk[i].rate);
-		if (IS_ERR(hw)) {
-			ret = PTR_ERR(hw);
-			goto unregister_fixed_clk;
+		hw = ERR_PTR(-EINVAL);
+
+		switch (fixed_clk[i].type) {
+		case CMN_PLL_CLK_FIXED_RATE: {
+			struct clk_parent_data pdata = { .hw = cmn_pll_hw };
+
+			hw = devm_clk_hw_register_fixed_rate_parent_data(dev,
+									 fixed_clk[i].name,
+									 &pdata, 0,
+									 fixed_clk[i].rate);
+			break;
 		}
+		case CMN_PLL_CLK_FIXED_GATE:
+			hw = ipq_cmn_pll_register_fixed_gate(dev,
+							     fixed_clk[i].name,
+							     cmn_pll->div2_hw,
+							     fixed_clk[i].enable_bit,
+							     fixed_clk[i].rate);
+			break;
+		case CMN_PLL_CLK_NSS:
+			hw = ipq_cmn_pll_regmap_div_register(dev,
+							     fixed_clk[i].name,
+							     cmn_pll->div2_hw,
+							     CMN_PLL_NSS_CLK_SEL);
+			break;
+		case CMN_PLL_CLK_PPE:
+			hw = ipq_cmn_pll_regmap_div_register(dev,
+							     fixed_clk[i].name,
+							     cmn_pll->div2_hw,
+							     CMN_PLL_PPE_CLK_SEL);
+			break;
+		case CMN_PLL_CLK_PON:
+			hw = ipq_cmn_pll_pon_clk_register(dev,
+							  fixed_clk[i].name,
+							  cmn_pll->div2_hw);
+			break;
+		case CMN_PLL_CLK_EPHY_RAW:
+			hw = ipq_cmn_pll_ephy_raw_register(dev,
+							   fixed_clk[i].name,
+							   cmn_pll->div2_hw);
+			break;
+		}
+
+		if (IS_ERR(hw))
+			return PTR_ERR(hw);
 
 		hw_data->hws[fixed_clk[i].id] = hw;
 	}
@@ -406,21 +898,14 @@ static int ipq_cmn_pll_register_clks(struct platform_device *pdev)
 	 * is configured to 12 GHZ by DT property assigned-clock-rates-u64.
 	 */
 	hw_data->hws[CMN_PLL_CLK] = cmn_pll_hw;
-	hw_data->num = num_clks + 1;
 
 	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, hw_data);
 	if (ret)
-		goto unregister_fixed_clk;
+		return ret;
 
 	platform_set_drvdata(pdev, hw_data);
 
 	return 0;
-
-unregister_fixed_clk:
-	while (i > 0)
-		clk_hw_unregister(hw_data->hws[fixed_clk[--i].id]);
-
-	return ret;
 }
 
 static int ipq_cmn_pll_clk_probe(struct platform_device *pdev)
@@ -462,27 +947,13 @@ static int ipq_cmn_pll_clk_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static void ipq_cmn_pll_clk_remove(struct platform_device *pdev)
-{
-	struct clk_hw_onecell_data *hw_data = platform_get_drvdata(pdev);
-	int i;
-
-	/*
-	 * The clock with index CMN_PLL_CLK is unregistered by
-	 * device management.
-	 */
-	for (i = 0; i < hw_data->num; i++) {
-		if (i != CMN_PLL_CLK)
-			clk_hw_unregister(hw_data->hws[i]);
-	}
-}
-
 static const struct dev_pm_ops ipq_cmn_pll_pm_ops = {
 	SET_RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL)
 };
 
 static const struct of_device_id ipq_cmn_pll_clk_ids[] = {
 	{ .compatible = "qcom,ipq5018-cmn-pll", .data = &ipq5018_output_clks },
+	{ .compatible = "qcom,ipq5210-cmn-pll", .data = &ipq5210_output_clks },
 	{ .compatible = "qcom,ipq5332-cmn-pll", .data = &ipq5332_output_clks },
 	{ .compatible = "qcom,ipq5424-cmn-pll", .data = &ipq5424_output_clks },
 	{ .compatible = "qcom,ipq6018-cmn-pll", .data = &ipq6018_output_clks },
@@ -494,7 +965,6 @@ MODULE_DEVICE_TABLE(of, ipq_cmn_pll_clk_ids);
 
 static struct platform_driver ipq_cmn_pll_clk_driver = {
 	.probe = ipq_cmn_pll_clk_probe,
-	.remove = ipq_cmn_pll_clk_remove,
 	.driver = {
 		.name = "ipq_cmn_pll",
 		.of_match_table = ipq_cmn_pll_clk_ids,
diff --git a/drivers/clk/qcom/lpasscc-sc7280.c b/drivers/clk/qcom/lpasscc-sc7280.c
index e6b815aec46a15..3789a7b37bddbd 100644
--- a/drivers/clk/qcom/lpasscc-sc7280.c
+++ b/drivers/clk/qcom/lpasscc-sc7280.c
@@ -169,17 +169,7 @@ static struct platform_driver lpass_cc_sc7280_driver = {
 	},
 };
 
-static int __init lpass_cc_sc7280_init(void)
-{
-	return platform_driver_register(&lpass_cc_sc7280_driver);
-}
-subsys_initcall(lpass_cc_sc7280_init);
-
-static void __exit lpass_cc_sc7280_exit(void)
-{
-	platform_driver_unregister(&lpass_cc_sc7280_driver);
-}
-module_exit(lpass_cc_sc7280_exit);
+subsys_platform_driver(lpass_cc_sc7280_driver);
 
 MODULE_DESCRIPTION("QTI LPASS_CC SC7280 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/lpasscc-sdm845.c b/drivers/clk/qcom/lpasscc-sdm845.c
index 5c1ea75f9ba802..1d7e78dd6015e6 100644
--- a/drivers/clk/qcom/lpasscc-sdm845.c
+++ b/drivers/clk/qcom/lpasscc-sdm845.c
@@ -144,17 +144,7 @@ static struct platform_driver lpass_cc_sdm845_driver = {
 	},
 };
 
-static int __init lpass_cc_sdm845_init(void)
-{
-	return platform_driver_register(&lpass_cc_sdm845_driver);
-}
-subsys_initcall(lpass_cc_sdm845_init);
-
-static void __exit lpass_cc_sdm845_exit(void)
-{
-	platform_driver_unregister(&lpass_cc_sdm845_driver);
-}
-module_exit(lpass_cc_sdm845_exit);
+subsys_platform_driver(lpass_cc_sdm845_driver);
 
 MODULE_DESCRIPTION("QTI LPASS_CC SDM845 Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/lpasscc-sm6115.c b/drivers/clk/qcom/lpasscc-sm6115.c
index 226dc02fc42da6..fcc1512d4a991c 100644
--- a/drivers/clk/qcom/lpasscc-sm6115.c
+++ b/drivers/clk/qcom/lpasscc-sm6115.c
@@ -34,7 +34,7 @@ static const struct qcom_cc_desc lpass_audiocc_sm6115_reset_desc = {
 };
 
 static const struct qcom_reset_map lpasscc_sm6115_resets[] = {
-	[LPASS_SWR_TX_CONFIG_CGCR] = { .reg = 0x100, .bit = 1, .udelay = 500 },
+	[LPASS_SWR_TX_CONFIG_CGCR] = { .reg = 0xc100, .bit = 1, .udelay = 500 },
 };
 
 static struct regmap_config lpasscc_sm6115_regmap_config = {
@@ -42,7 +42,7 @@ static struct regmap_config lpasscc_sm6115_regmap_config = {
 	.reg_stride = 4,
 	.val_bits = 32,
 	.name = "lpass-tcsr",
-	.max_register = 0x1000,
+	.max_register = 0x12000,
 };
 
 static const struct qcom_cc_desc lpasscc_sm6115_reset_desc = {
diff --git a/drivers/clk/qcom/lpasscorecc-sc7180.c b/drivers/clk/qcom/lpasscorecc-sc7180.c
index 5174bd3dcdc571..af28da5f04b37e 100644
--- a/drivers/clk/qcom/lpasscorecc-sc7180.c
+++ b/drivers/clk/qcom/lpasscorecc-sc7180.c
@@ -343,7 +343,28 @@ static const struct qcom_cc_desc lpass_core_hm_sc7180_desc = {
 	.num_gdscs = ARRAY_SIZE(lpass_core_hm_sc7180_gdscs),
 };
 
+static const u32 lpass_core_cc_sc7180_critical_cbcrs[] = {
+	0x24000, /* LPASS_AUDIO_CORE_SYSNOC_SWAY_CORE_CLK */
+};
+
+static void lpass_core_cc_sc7180_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* PLL settings */
+	regmap_write(regmap, 0x1008, 0x20);
+	regmap_update_bits(regmap, 0x1014, BIT(0), BIT(0));
+
+	clk_fabia_pll_configure(&lpass_lpaaudio_dig_pll, regmap,
+			      &lpass_lpaaudio_dig_pll_config);
+}
+
+static const struct qcom_cc_driver_data lpass_core_cc_sc7180_driver_data = {
+	.clk_cbcrs = lpass_core_cc_sc7180_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(lpass_core_cc_sc7180_critical_cbcrs),
+	.clk_regs_configure = lpass_core_cc_sc7180_regs_configure,
+};
+
 static const struct qcom_cc_desc lpass_core_cc_sc7180_desc = {
+	.driver_data = &lpass_core_cc_sc7180_driver_data,
 	.config = &lpass_core_cc_sc7180_regmap_config,
 	.clks = lpass_core_cc_sc7180_clocks,
 	.num_clks = ARRAY_SIZE(lpass_core_cc_sc7180_clocks),
@@ -400,16 +421,6 @@ static int lpass_core_cc_sc7180_probe(struct platform_device *pdev)
 		goto exit;
 	}
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x24000); /* LPASS_AUDIO_CORE_SYSNOC_SWAY_CORE_CLK */
-
-	/* PLL settings */
-	regmap_write(regmap, 0x1008, 0x20);
-	regmap_update_bits(regmap, 0x1014, BIT(0), BIT(0));
-
-	clk_fabia_pll_configure(&lpass_lpaaudio_dig_pll, regmap,
-				&lpass_lpaaudio_dig_pll_config);
-
 	ret = qcom_cc_really_probe(&pdev->dev, &lpass_core_cc_sc7180_desc, regmap);
 
 exit:
diff --git a/drivers/clk/qcom/mmcc-msm8960.c b/drivers/clk/qcom/mmcc-msm8960.c
index a23440e13b71d6..a67df833592a77 100644
--- a/drivers/clk/qcom/mmcc-msm8960.c
+++ b/drivers/clk/qcom/mmcc-msm8960.c
@@ -3170,7 +3170,8 @@ static int mmcc_msm8960_probe(struct platform_device *pdev)
 	if (IS_ERR(regmap))
 		return PTR_ERR(regmap);
 
-	clk_pll_configure_sr(&pll15, regmap, &pll15_config, false);
+	if (desc == &mmcc_apq8064_desc)
+		clk_pll_configure_sr(&pll15, regmap, &pll15_config, false);
 
 	return qcom_cc_really_probe(&pdev->dev, desc, regmap);
 }
diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c
index 3426e3dde92478..ef7bde361fc38e 100644
--- a/drivers/clk/qcom/mmcc-msm8996.c
+++ b/drivers/clk/qcom/mmcc-msm8996.c
@@ -3133,9 +3133,9 @@ static struct clk_branch fd_core_uar_clk = {
 };
 
 static struct clk_branch fd_ahb_clk = {
-	.halt_reg = 0x3ba74,
+	.halt_reg = 0x3b74,
 	.clkr = {
-		.enable_reg = 0x3ba74,
+		.enable_reg = 0x3b74,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
 			.name = "fd_ahb_clk",
diff --git a/drivers/clk/qcom/mmcc-sdm660.c b/drivers/clk/qcom/mmcc-sdm660.c
index 200f986de9651d..9f8b8e29fa0f3b 100644
--- a/drivers/clk/qcom/mmcc-sdm660.c
+++ b/drivers/clk/qcom/mmcc-sdm660.c
@@ -2017,7 +2017,7 @@ static struct clk_branch csiphy_ahb2crif_clk = {
 static struct clk_branch mdss_ahb_clk = {
 	.halt_reg = 0x2308,
 	.halt_check = BRANCH_HALT,
-	.hwcg_reg = 0x8a004,
+	.hwcg_reg = 0x2308,
 	.hwcg_bit = 1,
 	.clkr = {
 		.enable_reg = 0x2308,
diff --git a/drivers/clk/qcom/negcc-nord.c b/drivers/clk/qcom/negcc-nord.c
index 0db284edc4e41e..68ce4d6278baa5 100644
--- a/drivers/clk/qcom/negcc-nord.c
+++ b/drivers/clk/qcom/negcc-nord.c
@@ -46,7 +46,7 @@ static struct clk_alpha_pll ne_gcc_gpll0 = {
 	.offset = 0x0,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
 	.clkr = {
-		.enable_reg = 0x0,
+		.enable_reg = 0x57010,
 		.enable_mask = BIT(0),
 		.hw.init = &(const struct clk_init_data) {
 			.name = "ne_gcc_gpll0",
@@ -85,7 +85,7 @@ static struct clk_alpha_pll ne_gcc_gpll2 = {
 	.offset = 0x2000,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
 	.clkr = {
-		.enable_reg = 0x0,
+		.enable_reg = 0x57010,
 		.enable_mask = BIT(2),
 		.hw.init = &(const struct clk_init_data) {
 			.name = "ne_gcc_gpll2",
@@ -1877,6 +1877,7 @@ static const struct qcom_reset_map ne_gcc_nord_resets[] = {
 	[NE_GCC_USB3PHY_PHY_PRIM_BCR] = { 0x2b004 },
 	[NE_GCC_USB3PHY_PHY_SEC_BCR] = { 0x2d004 },
 	[NE_GCC_QUSB2PHY_PRIM_BCR] = { 0x2e000 },
+	[NE_GCC_QUSB2PHY_SEC_BCR] = { 0x2f000 },
 };
 
 static const struct clk_rcg_dfs_data ne_gcc_nord_dfs_clocks[] = {
diff --git a/drivers/clk/qcom/nwgcc-nord.c b/drivers/clk/qcom/nwgcc-nord.c
index e061c0623e9a46..1eb8e53139120d 100644
--- a/drivers/clk/qcom/nwgcc-nord.c
+++ b/drivers/clk/qcom/nwgcc-nord.c
@@ -36,7 +36,7 @@ static struct clk_alpha_pll nw_gcc_gpll0 = {
 	.offset = 0x0,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
 	.clkr = {
-		.enable_reg = 0x0,
+		.enable_reg = 0x76008,
 		.enable_mask = BIT(0),
 		.hw.init = &(const struct clk_init_data) {
 			.name = "nw_gcc_gpll0",
@@ -623,6 +623,9 @@ static const struct qcom_reset_map nw_gcc_nord_resets[] = {
 	[NW_GCC_GPU_2_BCR] = { 0x24000 },
 	[NW_GCC_GPU_BCR] = { 0x23000 },
 	[NW_GCC_VIDEO_BCR] = { 0x1a000 },
+	[NW_GCC_VIDEO_AXI0_CLK_ARES] = { 0x1a008, 2 },
+	[NW_GCC_VIDEO_AXI0C_CLK_ARES] = { 0x1a01c, 2 },
+	[NW_GCC_VIDEO_AXI1_CLK_ARES] = { 0x1a030, 2 },
 };
 
 static const u32 nw_gcc_nord_critical_cbcrs[] = {
diff --git a/drivers/clk/qcom/segcc-nord.c b/drivers/clk/qcom/segcc-nord.c
index 51d7f83e21b6b2..ef97da3852e357 100644
--- a/drivers/clk/qcom/segcc-nord.c
+++ b/drivers/clk/qcom/segcc-nord.c
@@ -39,7 +39,7 @@ static struct clk_alpha_pll se_gcc_gpll0 = {
 	.offset = 0x0,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
 	.clkr = {
-		.enable_reg = 0x0,
+		.enable_reg = 0x57010,
 		.enable_mask = BIT(0),
 		.hw.init = &(const struct clk_init_data) {
 			.name = "se_gcc_gpll0",
@@ -78,7 +78,7 @@ static struct clk_alpha_pll se_gcc_gpll2 = {
 	.offset = 0x2000,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
 	.clkr = {
-		.enable_reg = 0x0,
+		.enable_reg = 0x57010,
 		.enable_mask = BIT(2),
 		.hw.init = &(const struct clk_init_data) {
 			.name = "se_gcc_gpll2",
@@ -95,7 +95,7 @@ static struct clk_alpha_pll se_gcc_gpll4 = {
 	.offset = 0x4000,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
 	.clkr = {
-		.enable_reg = 0x0,
+		.enable_reg = 0x57010,
 		.enable_mask = BIT(4),
 		.hw.init = &(const struct clk_init_data) {
 			.name = "se_gcc_gpll4",
@@ -112,7 +112,7 @@ static struct clk_alpha_pll se_gcc_gpll5 = {
 	.offset = 0x5000,
 	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
 	.clkr = {
-		.enable_reg = 0x0,
+		.enable_reg = 0x57010,
 		.enable_mask = BIT(5),
 		.hw.init = &(const struct clk_init_data) {
 			.name = "se_gcc_gpll5",
@@ -747,6 +747,21 @@ static struct clk_branch se_gcc_emac0_cc_sgmiiphy_tx_clk = {
 	},
 };
 
+static struct clk_branch se_gcc_emac0_cnoc_apb_clk = {
+	.halt_reg = 0x24018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x24018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x24018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "se_gcc_emac0_cnoc_apb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch se_gcc_emac0_phy_aux_clk = {
 	.halt_reg = 0x2402c,
 	.halt_check = BRANCH_HALT,
@@ -894,6 +909,21 @@ static struct clk_branch se_gcc_emac1_cc_sgmiiphy_tx_clk = {
 	},
 };
 
+static struct clk_branch se_gcc_emac1_cnoc_apb_clk = {
+	.halt_reg = 0x25018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x25018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x25018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "se_gcc_emac1_cnoc_apb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch se_gcc_emac1_phy_aux_clk = {
 	.halt_reg = 0x2502c,
 	.halt_check = BRANCH_HALT,
@@ -1458,6 +1488,7 @@ static struct clk_regmap *se_gcc_nord_clocks[] = {
 	[SE_GCC_EMAC0_AXI_CLK] = &se_gcc_emac0_axi_clk.clkr,
 	[SE_GCC_EMAC0_CC_SGMIIPHY_RX_CLK] = &se_gcc_emac0_cc_sgmiiphy_rx_clk.clkr,
 	[SE_GCC_EMAC0_CC_SGMIIPHY_TX_CLK] = &se_gcc_emac0_cc_sgmiiphy_tx_clk.clkr,
+	[SE_GCC_EMAC0_CNOC_APB_CLK] = &se_gcc_emac0_cnoc_apb_clk.clkr,
 	[SE_GCC_EMAC0_PHY_AUX_CLK] = &se_gcc_emac0_phy_aux_clk.clkr,
 	[SE_GCC_EMAC0_PHY_AUX_CLK_SRC] = &se_gcc_emac0_phy_aux_clk_src.clkr,
 	[SE_GCC_EMAC0_PTP_CLK] = &se_gcc_emac0_ptp_clk.clkr,
@@ -1471,6 +1502,7 @@ static struct clk_regmap *se_gcc_nord_clocks[] = {
 	[SE_GCC_EMAC1_AXI_CLK] = &se_gcc_emac1_axi_clk.clkr,
 	[SE_GCC_EMAC1_CC_SGMIIPHY_RX_CLK] = &se_gcc_emac1_cc_sgmiiphy_rx_clk.clkr,
 	[SE_GCC_EMAC1_CC_SGMIIPHY_TX_CLK] = &se_gcc_emac1_cc_sgmiiphy_tx_clk.clkr,
+	[SE_GCC_EMAC1_CNOC_APB_CLK] = &se_gcc_emac1_cnoc_apb_clk.clkr,
 	[SE_GCC_EMAC1_PHY_AUX_CLK] = &se_gcc_emac1_phy_aux_clk.clkr,
 	[SE_GCC_EMAC1_PHY_AUX_CLK_SRC] = &se_gcc_emac1_phy_aux_clk_src.clkr,
 	[SE_GCC_EMAC1_PTP_CLK] = &se_gcc_emac1_ptp_clk.clkr,
diff --git a/drivers/clk/qcom/tcsrcc-eliza.c b/drivers/clk/qcom/tcsrcc-eliza.c
index 127d1f0a1442a2..daab9a62049007 100644
--- a/drivers/clk/qcom/tcsrcc-eliza.c
+++ b/drivers/clk/qcom/tcsrcc-eliza.c
@@ -162,17 +162,7 @@ static struct platform_driver tcsr_cc_eliza_driver = {
 	},
 };
 
-static int __init tcsr_cc_eliza_init(void)
-{
-	return platform_driver_register(&tcsr_cc_eliza_driver);
-}
-subsys_initcall(tcsr_cc_eliza_init);
-
-static void __exit tcsr_cc_eliza_exit(void)
-{
-	platform_driver_unregister(&tcsr_cc_eliza_driver);
-}
-module_exit(tcsr_cc_eliza_exit);
+subsys_platform_driver(tcsr_cc_eliza_driver);
 
 MODULE_DESCRIPTION("QTI TCSR_CC Eliza Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-glymur.c b/drivers/clk/qcom/tcsrcc-glymur.c
index 1791e23739ed45..8ac60052fb1448 100644
--- a/drivers/clk/qcom/tcsrcc-glymur.c
+++ b/drivers/clk/qcom/tcsrcc-glymur.c
@@ -296,17 +296,7 @@ static struct platform_driver tcsr_cc_glymur_driver = {
 	},
 };
 
-static int __init tcsr_cc_glymur_init(void)
-{
-	return platform_driver_register(&tcsr_cc_glymur_driver);
-}
-subsys_initcall(tcsr_cc_glymur_init);
-
-static void __exit tcsr_cc_glymur_exit(void)
-{
-	platform_driver_unregister(&tcsr_cc_glymur_driver);
-}
-module_exit(tcsr_cc_glymur_exit);
+subsys_platform_driver(tcsr_cc_glymur_driver);
 
 MODULE_DESCRIPTION("QTI TCSRCC Glymur Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-kaanapali.c b/drivers/clk/qcom/tcsrcc-kaanapali.c
index db46d639edb800..78b77e7cd396a2 100644
--- a/drivers/clk/qcom/tcsrcc-kaanapali.c
+++ b/drivers/clk/qcom/tcsrcc-kaanapali.c
@@ -124,17 +124,7 @@ static struct platform_driver tcsr_cc_kaanapali_driver = {
 	},
 };
 
-static int __init tcsr_cc_kaanapali_init(void)
-{
-	return platform_driver_register(&tcsr_cc_kaanapali_driver);
-}
-subsys_initcall(tcsr_cc_kaanapali_init);
-
-static void __exit tcsr_cc_kaanapali_exit(void)
-{
-	platform_driver_unregister(&tcsr_cc_kaanapali_driver);
-}
-module_exit(tcsr_cc_kaanapali_exit);
+subsys_platform_driver(tcsr_cc_kaanapali_driver);
 
 MODULE_DESCRIPTION("QTI TCSR_CC Kaanapali Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-sm8550.c b/drivers/clk/qcom/tcsrcc-sm8550.c
index 41d73f92a000ab..6cac619dd959aa 100644
--- a/drivers/clk/qcom/tcsrcc-sm8550.c
+++ b/drivers/clk/qcom/tcsrcc-sm8550.c
@@ -191,17 +191,7 @@ static struct platform_driver tcsr_cc_sm8550_driver = {
 	},
 };
 
-static int __init tcsr_cc_sm8550_init(void)
-{
-	return platform_driver_register(&tcsr_cc_sm8550_driver);
-}
-subsys_initcall(tcsr_cc_sm8550_init);
-
-static void __exit tcsr_cc_sm8550_exit(void)
-{
-	platform_driver_unregister(&tcsr_cc_sm8550_driver);
-}
-module_exit(tcsr_cc_sm8550_exit);
+subsys_platform_driver(tcsr_cc_sm8550_driver);
 
 MODULE_DESCRIPTION("QTI TCSRCC SM8550 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-sm8650.c b/drivers/clk/qcom/tcsrcc-sm8650.c
index 651e28067954a1..194aea6ec43a05 100644
--- a/drivers/clk/qcom/tcsrcc-sm8650.c
+++ b/drivers/clk/qcom/tcsrcc-sm8650.c
@@ -173,17 +173,7 @@ static struct platform_driver tcsr_cc_sm8650_driver = {
 	},
 };
 
-static int __init tcsr_cc_sm8650_init(void)
-{
-	return platform_driver_register(&tcsr_cc_sm8650_driver);
-}
-subsys_initcall(tcsr_cc_sm8650_init);
-
-static void __exit tcsr_cc_sm8650_exit(void)
-{
-	platform_driver_unregister(&tcsr_cc_sm8650_driver);
-}
-module_exit(tcsr_cc_sm8650_exit);
+subsys_platform_driver(tcsr_cc_sm8650_driver);
 
 MODULE_DESCRIPTION("QTI TCSRCC SM8650 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-sm8750.c b/drivers/clk/qcom/tcsrcc-sm8750.c
index 9fe840a448bfd7..b11ec403bb324c 100644
--- a/drivers/clk/qcom/tcsrcc-sm8750.c
+++ b/drivers/clk/qcom/tcsrcc-sm8750.c
@@ -124,17 +124,7 @@ static struct platform_driver tcsr_cc_sm8750_driver = {
 	},
 };
 
-static int __init tcsr_cc_sm8750_init(void)
-{
-	return platform_driver_register(&tcsr_cc_sm8750_driver);
-}
-subsys_initcall(tcsr_cc_sm8750_init);
-
-static void __exit tcsr_cc_sm8750_exit(void)
-{
-	platform_driver_unregister(&tcsr_cc_sm8750_driver);
-}
-module_exit(tcsr_cc_sm8750_exit);
+subsys_platform_driver(tcsr_cc_sm8750_driver);
 
 MODULE_DESCRIPTION("QTI TCSR_CC SM8750 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/tcsrcc-x1e80100.c b/drivers/clk/qcom/tcsrcc-x1e80100.c
index 0b05c27b619b63..7dbeff419f9989 100644
--- a/drivers/clk/qcom/tcsrcc-x1e80100.c
+++ b/drivers/clk/qcom/tcsrcc-x1e80100.c
@@ -5,252 +5,141 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/clk/qcom.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 
 #include <dt-bindings/clock/qcom,x1e80100-tcsr.h>
 
-#include "clk-branch.h"
-#include "clk-regmap.h"
-#include "common.h"
-#include "reset.h"
-
-enum {
-	DT_BI_TCXO_PAD,
+static const char * const x1e80100_tcsr_tx1_rpt0_rx0_regulators[] = {
+	"vdda-refgen0-0p9",
+	"vdda-refgen0-1p2",
+	"vdda-qreftx1-0p9",
+	"vdda-qreftx1-1p2",
+	"vdda-qrefrpt0-0p9",
+	"vdda-qrefrx0-0p9",
 };
 
-static struct clk_branch tcsr_edp_clkref_en = {
-	.halt_reg = 0x15130,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15130,
-		.enable_mask = BIT(0),
-		.hw.init = &(const struct clk_init_data) {
-			.name = "tcsr_edp_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
-	},
+static const char * const x1e80100_tcsr_tx1_rpt1_rx1_regulators[] = {
+	"vdda-refgen0-0p9",
+	"vdda-refgen0-1p2",
+	"vdda-qreftx1-0p9",
+	"vdda-qreftx1-1p2",
+	"vdda-qrefrpt1-0p9",
+	"vdda-qrefrx1-0p9",
 };
 
-static struct clk_branch tcsr_pcie_2l_4_clkref_en = {
-	.halt_reg = 0x15100,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15100,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_pcie_2l_4_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
-	},
+static const char * const x1e80100_tcsr_tx1_rpt12_rx2_regulators[] = {
+	"vdda-refgen0-0p9",
+	"vdda-refgen0-1p2",
+	"vdda-qreftx1-0p9",
+	"vdda-qreftx1-1p2",
+	"vdda-qrefrpt1-0p9",
+	"vdda-qrefrpt2-0p9",
+	"vdda-qrefrx2-0p9",
 };
 
-static struct clk_branch tcsr_pcie_2l_5_clkref_en = {
-	.halt_reg = 0x15104,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15104,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_pcie_2l_5_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
-	},
+static const char * const x1e80100_tcsr_tx0_rpt3_rx3_regulators[] = {
+	"vdda-refgen2-0p9",
+	"vdda-refgen2-1p2",
+	"vdda-qreftx0-0p9",
+	"vdda-qreftx0-1p2",
+	"vdda-qrefrpt3-0p9",
+	"vdda-qrefrx3-0p9",
 };
 
-static struct clk_branch tcsr_pcie_8l_clkref_en = {
-	.halt_reg = 0x15108,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15108,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_pcie_8l_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
-	},
+static const char * const x1e80100_tcsr_tx0_rpt4_rx4_regulators[] = {
+	"vdda-refgen2-0p9",
+	"vdda-refgen2-1p2",
+	"vdda-qreftx0-0p9",
+	"vdda-qreftx0-1p2",
+	"vdda-qrefrpt4-0p9",
+	"vdda-qrefrx4-0p9",
 };
 
-static struct clk_branch tcsr_usb3_mp0_clkref_en = {
-	.halt_reg = 0x1510c,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x1510c,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_usb3_mp0_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
-	},
+static const struct regmap_config tcsr_cc_x1e80100_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x2f000,
+	.fast_io = true,
 };
 
-static struct clk_branch tcsr_usb3_mp1_clkref_en = {
-	.halt_reg = 0x15110,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15110,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_usb3_mp1_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
+static const struct qcom_clk_ref_desc * const tcsr_cc_x1e80100_clk_descs[] = {
+	[TCSR_EDP_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_edp_clkref_en",
+		.offset = 0x15130,
+		.regulator_names = x1e80100_tcsr_tx0_rpt3_rx3_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx0_rpt3_rx3_regulators),
 	},
-};
-
-static struct clk_branch tcsr_usb2_1_clkref_en = {
-	.halt_reg = 0x15114,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15114,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_usb2_1_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
+	[TCSR_PCIE_2L_4_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_pcie_2l_4_clkref_en",
+		.offset = 0x15100,
+		.regulator_names = x1e80100_tcsr_tx1_rpt1_rx1_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx1_rpt1_rx1_regulators),
 	},
-};
-
-static struct clk_branch tcsr_ufs_phy_clkref_en = {
-	.halt_reg = 0x15118,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15118,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_ufs_phy_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
+	[TCSR_PCIE_2L_5_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_pcie_2l_5_clkref_en",
+		.offset = 0x15104,
+		.regulator_names = x1e80100_tcsr_tx1_rpt12_rx2_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx1_rpt12_rx2_regulators),
 	},
-};
-
-static struct clk_branch tcsr_usb4_1_clkref_en = {
-	.halt_reg = 0x15120,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15120,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_usb4_1_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
+	[TCSR_PCIE_8L_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_pcie_8l_clkref_en",
+		.offset = 0x15108,
+		.regulator_names = x1e80100_tcsr_tx1_rpt0_rx0_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx1_rpt0_rx0_regulators),
 	},
-};
-
-static struct clk_branch tcsr_usb4_2_clkref_en = {
-	.halt_reg = 0x15124,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15124,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_usb4_2_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
+	[TCSR_USB3_MP0_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_usb3_mp0_clkref_en",
+		.offset = 0x1510c,
+		.regulator_names = x1e80100_tcsr_tx1_rpt0_rx0_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx1_rpt0_rx0_regulators),
 	},
-};
-
-static struct clk_branch tcsr_usb2_2_clkref_en = {
-	.halt_reg = 0x15128,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x15128,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_usb2_2_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
+	[TCSR_USB3_MP1_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_usb3_mp1_clkref_en",
+		.offset = 0x15110,
+		.regulator_names = x1e80100_tcsr_tx1_rpt0_rx0_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx1_rpt0_rx0_regulators),
 	},
-};
-
-static struct clk_branch tcsr_pcie_4l_clkref_en = {
-	.halt_reg = 0x1512c,
-	.halt_check = BRANCH_HALT_DELAY,
-	.clkr = {
-		.enable_reg = 0x1512c,
-		.enable_mask = BIT(0),
-		.hw.init = &(struct clk_init_data){
-			.name = "tcsr_pcie_4l_clkref_en",
-			.parent_data = &(const struct clk_parent_data){
-				.index = DT_BI_TCXO_PAD,
-			},
-			.num_parents = 1,
-			.ops = &clk_branch2_ops,
-		},
+	[TCSR_USB2_1_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_usb2_1_clkref_en",
+		.offset = 0x15114,
+		.regulator_names = x1e80100_tcsr_tx0_rpt3_rx3_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx0_rpt3_rx3_regulators),
+	},
+	[TCSR_UFS_PHY_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_ufs_phy_clkref_en",
+		.offset = 0x15118,
+		.regulator_names = x1e80100_tcsr_tx1_rpt12_rx2_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx1_rpt12_rx2_regulators),
+	},
+	[TCSR_USB4_1_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_usb4_1_clkref_en",
+		.offset = 0x15120,
+		.regulator_names = x1e80100_tcsr_tx0_rpt4_rx4_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx0_rpt4_rx4_regulators),
+	},
+	[TCSR_USB4_2_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_usb4_2_clkref_en",
+		.offset = 0x15124,
+		.regulator_names = x1e80100_tcsr_tx0_rpt3_rx3_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx0_rpt3_rx3_regulators),
+	},
+	[TCSR_USB2_2_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_usb2_2_clkref_en",
+		.offset = 0x15128,
+		.regulator_names = x1e80100_tcsr_tx0_rpt3_rx3_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx0_rpt3_rx3_regulators),
+	},
+	[TCSR_PCIE_4L_CLKREF_EN] = &(const struct qcom_clk_ref_desc) {
+		.name = "tcsr_pcie_4l_clkref_en",
+		.offset = 0x1512c,
+		.regulator_names = x1e80100_tcsr_tx0_rpt4_rx4_regulators,
+		.num_regulators = ARRAY_SIZE(x1e80100_tcsr_tx0_rpt4_rx4_regulators),
 	},
-};
-
-static struct clk_regmap *tcsr_cc_x1e80100_clocks[] = {
-	[TCSR_EDP_CLKREF_EN] = &tcsr_edp_clkref_en.clkr,
-	[TCSR_PCIE_2L_4_CLKREF_EN] = &tcsr_pcie_2l_4_clkref_en.clkr,
-	[TCSR_PCIE_2L_5_CLKREF_EN] = &tcsr_pcie_2l_5_clkref_en.clkr,
-	[TCSR_PCIE_8L_CLKREF_EN] = &tcsr_pcie_8l_clkref_en.clkr,
-	[TCSR_USB3_MP0_CLKREF_EN] = &tcsr_usb3_mp0_clkref_en.clkr,
-	[TCSR_USB3_MP1_CLKREF_EN] = &tcsr_usb3_mp1_clkref_en.clkr,
-	[TCSR_USB2_1_CLKREF_EN] = &tcsr_usb2_1_clkref_en.clkr,
-	[TCSR_UFS_PHY_CLKREF_EN] = &tcsr_ufs_phy_clkref_en.clkr,
-	[TCSR_USB4_1_CLKREF_EN] = &tcsr_usb4_1_clkref_en.clkr,
-	[TCSR_USB4_2_CLKREF_EN] = &tcsr_usb4_2_clkref_en.clkr,
-	[TCSR_USB2_2_CLKREF_EN] = &tcsr_usb2_2_clkref_en.clkr,
-	[TCSR_PCIE_4L_CLKREF_EN] = &tcsr_pcie_4l_clkref_en.clkr,
-};
-
-static const struct regmap_config tcsr_cc_x1e80100_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x2f000,
-	.fast_io = true,
-};
-
-static const struct qcom_cc_desc tcsr_cc_x1e80100_desc = {
-	.config = &tcsr_cc_x1e80100_regmap_config,
-	.clks = tcsr_cc_x1e80100_clocks,
-	.num_clks = ARRAY_SIZE(tcsr_cc_x1e80100_clocks),
 };
 
 static const struct of_device_id tcsr_cc_x1e80100_match_table[] = {
@@ -261,7 +150,9 @@ MODULE_DEVICE_TABLE(of, tcsr_cc_x1e80100_match_table);
 
 static int tcsr_cc_x1e80100_probe(struct platform_device *pdev)
 {
-	return qcom_cc_probe(pdev, &tcsr_cc_x1e80100_desc);
+	return qcom_clk_ref_probe(pdev, &tcsr_cc_x1e80100_regmap_config,
+				  tcsr_cc_x1e80100_clk_descs,
+				  ARRAY_SIZE(tcsr_cc_x1e80100_clk_descs));
 }
 
 static struct platform_driver tcsr_cc_x1e80100_driver = {
@@ -272,17 +163,7 @@ static struct platform_driver tcsr_cc_x1e80100_driver = {
 	},
 };
 
-static int __init tcsr_cc_x1e80100_init(void)
-{
-	return platform_driver_register(&tcsr_cc_x1e80100_driver);
-}
-subsys_initcall(tcsr_cc_x1e80100_init);
-
-static void __exit tcsr_cc_x1e80100_exit(void)
-{
-	platform_driver_unregister(&tcsr_cc_x1e80100_driver);
-}
-module_exit(tcsr_cc_x1e80100_exit);
+subsys_platform_driver(tcsr_cc_x1e80100_driver);
 
 MODULE_DESCRIPTION("QTI TCSR Clock Controller X1E80100 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/videocc-eliza.c b/drivers/clk/qcom/videocc-eliza.c
new file mode 100644
index 00000000000000..9112ded781c235
--- /dev/null
+++ b/drivers/clk/qcom/videocc-eliza.c
@@ -0,0 +1,404 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,eliza-videocc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK,
+	DT_AHB_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_SLEEP_CLK,
+	P_VIDEO_CC_PLL0_OUT_MAIN,
+};
+
+static const struct pll_vco lucid_ole_vco[] = {
+	{ 249600000, 2300000000, 0 },
+};
+
+/* 576.0 MHz Configuration */
+static const struct alpha_pll_config video_cc_pll0_config = {
+	.l = 0x1e,
+	.alpha = 0x0,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00000005,
+};
+
+static struct clk_alpha_pll video_cc_pll0 = {
+	.offset = 0x0,
+	.config = &video_cc_pll0_config,
+	.vco_table = lucid_ole_vco,
+	.num_vco = ARRAY_SIZE(lucid_ole_vco),
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE],
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_pll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_lucid_evo_ops,
+		},
+	},
+};
+
+static const struct parent_map video_cc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct parent_map video_cc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_VIDEO_CC_PLL0_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &video_cc_pll0.clkr.hw },
+};
+
+static const struct parent_map video_cc_parent_map_2[] = {
+	{ P_SLEEP_CLK, 0 },
+};
+
+static const struct clk_parent_data video_cc_parent_data_2[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct freq_tbl ftbl_video_cc_ahb_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_ahb_clk_src = {
+	.cmd_rcgr = 0x8018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_0,
+	.freq_tbl = ftbl_video_cc_ahb_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_ahb_clk_src",
+		.parent_data = video_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_mvs0_clk_src[] = {
+	F(576000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(633000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(720000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1014000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1098000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1113000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1332000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1600000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_mvs0_clk_src = {
+	.cmd_rcgr = 0x8000,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_1,
+	.freq_tbl = ftbl_video_cc_mvs0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0_clk_src",
+		.parent_data = video_cc_parent_data_1,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_1),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_video_cc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 video_cc_sleep_clk_src = {
+	.cmd_rcgr = 0x8110,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_2,
+	.freq_tbl = ftbl_video_cc_sleep_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_sleep_clk_src",
+		.parent_data = video_cc_parent_data_2,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_rcg2 video_cc_xo_clk_src = {
+	.cmd_rcgr = 0x80f4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = video_cc_parent_map_0,
+	.freq_tbl = ftbl_video_cc_ahb_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_xo_clk_src",
+		.parent_data = video_cc_parent_data_0,
+		.num_parents = ARRAY_SIZE(video_cc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static struct clk_regmap_div video_cc_mvs0_div_clk_src = {
+	.reg = 0x80ac,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&video_cc_mvs0_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div video_cc_mvs0c_div2_div_clk_src = {
+	.reg = 0x8058,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "video_cc_mvs0c_div2_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&video_cc_mvs0_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch video_cc_mvs0_clk = {
+	.halt_reg = 0x80a0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x80a0,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x80a0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0_shift_clk = {
+	.halt_reg = 0x8144,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x8144,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x8144,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_clk = {
+	.halt_reg = 0x804c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x804c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_mvs0c_div2_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch video_cc_mvs0c_shift_clk = {
+	.halt_reg = 0x8148,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x8148,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x8148,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "video_cc_mvs0c_shift_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&video_cc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc video_cc_mvs0c_gdsc = {
+	.gdscr = 0x8034,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "video_cc_mvs0c_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc video_cc_mvs0_gdsc = {
+	.gdscr = 0x808c,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x6,
+	.pd = {
+		.name = "video_cc_mvs0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.parent = &video_cc_mvs0c_gdsc.pd,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | HW_CTRL_TRIGGER,
+};
+
+static struct clk_regmap *video_cc_eliza_clocks[] = {
+	[VIDEO_CC_AHB_CLK_SRC] = &video_cc_ahb_clk_src.clkr,
+	[VIDEO_CC_MVS0_CLK] = &video_cc_mvs0_clk.clkr,
+	[VIDEO_CC_MVS0_CLK_SRC] = &video_cc_mvs0_clk_src.clkr,
+	[VIDEO_CC_MVS0_DIV_CLK_SRC] = &video_cc_mvs0_div_clk_src.clkr,
+	[VIDEO_CC_MVS0_SHIFT_CLK] = &video_cc_mvs0_shift_clk.clkr,
+	[VIDEO_CC_MVS0C_CLK] = &video_cc_mvs0c_clk.clkr,
+	[VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC] = &video_cc_mvs0c_div2_div_clk_src.clkr,
+	[VIDEO_CC_MVS0C_SHIFT_CLK] = &video_cc_mvs0c_shift_clk.clkr,
+	[VIDEO_CC_PLL0] = &video_cc_pll0.clkr,
+	[VIDEO_CC_SLEEP_CLK_SRC] = &video_cc_sleep_clk_src.clkr,
+	[VIDEO_CC_XO_CLK_SRC] = &video_cc_xo_clk_src.clkr,
+};
+
+static struct gdsc *video_cc_eliza_gdscs[] = {
+	[VIDEO_CC_MVS0_GDSC] = &video_cc_mvs0_gdsc,
+	[VIDEO_CC_MVS0C_GDSC] = &video_cc_mvs0c_gdsc,
+};
+
+static const struct qcom_reset_map video_cc_eliza_resets[] = {
+	[VIDEO_CC_INTERFACE_BCR] = { 0x80d8 },
+	[VIDEO_CC_MVS0_CLK_ARES] = { 0x80a0, 2 },
+	[VIDEO_CC_MVS0_BCR] = { 0x8088 },
+	[VIDEO_CC_MVS0C_CLK_ARES] = { 0x804c, 2 },
+	[VIDEO_CC_MVS0C_BCR] = { 0x8030 },
+	[VIDEO_CC_XO_CLK_ARES] = { 0x810c, 2 },
+};
+
+static struct clk_alpha_pll *video_cc_eliza_plls[] = {
+	&video_cc_pll0,
+};
+
+static const u32 video_cc_eliza_critical_cbcrs[] = {
+	0x80dc, /* VIDEO_CC_AHB_CLK */
+	0x8128, /* VIDEO_CC_SLEEP_CLK */
+	0x810c, /* VIDEO_CC_XO_CLK */
+};
+
+static const struct regmap_config video_cc_eliza_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9f50,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_driver_data video_cc_eliza_driver_data = {
+	.alpha_plls = video_cc_eliza_plls,
+	.num_alpha_plls = ARRAY_SIZE(video_cc_eliza_plls),
+	.clk_cbcrs = video_cc_eliza_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_eliza_critical_cbcrs),
+};
+
+static const struct qcom_cc_desc video_cc_eliza_desc = {
+	.config = &video_cc_eliza_regmap_config,
+	.clks = video_cc_eliza_clocks,
+	.num_clks = ARRAY_SIZE(video_cc_eliza_clocks),
+	.resets = video_cc_eliza_resets,
+	.num_resets = ARRAY_SIZE(video_cc_eliza_resets),
+	.gdscs = video_cc_eliza_gdscs,
+	.num_gdscs = ARRAY_SIZE(video_cc_eliza_gdscs),
+	.use_rpm = true,
+	.driver_data = &video_cc_eliza_driver_data,
+};
+
+static const struct of_device_id video_cc_eliza_match_table[] = {
+	{ .compatible = "qcom,eliza-videocc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, video_cc_eliza_match_table);
+
+static int video_cc_eliza_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &video_cc_eliza_desc);
+}
+
+static struct platform_driver video_cc_eliza_driver = {
+	.probe = video_cc_eliza_probe,
+	.driver = {
+		.name = "videocc-eliza",
+		.of_match_table = video_cc_eliza_match_table,
+	},
+};
+
+module_platform_driver(video_cc_eliza_driver);
+
+MODULE_DESCRIPTION("QTI VIDEOCC Eliza Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/videocc-glymur.c b/drivers/clk/qcom/videocc-glymur.c
index 18313a65e78d95..2776140de3555d 100644
--- a/drivers/clk/qcom/videocc-glymur.c
+++ b/drivers/clk/qcom/videocc-glymur.c
@@ -37,6 +37,10 @@ static const struct pll_vco taycan_eko_t_vco[] = {
 	{ 249600000, 2500000000, 0 },
 };
 
+static const struct pll_vco lucid_ole_vco[] = {
+	{ 249600000, 2300000000, 0 },
+};
+
 /* 720.0 MHz Configuration */
 static const struct alpha_pll_config video_cc_pll0_config = {
 	.l = 0x25,
@@ -48,6 +52,21 @@ static const struct alpha_pll_config video_cc_pll0_config = {
 	.user_ctl_hi_val = 0x00000002,
 };
 
+/* 720.0 MHz Configuration */
+static const struct alpha_pll_config video_cc_pll0_config_nord = {
+	.l = 0x25,
+	.alpha = 0x8000,
+	.config_ctl_val = 0x20485699,
+	.config_ctl_hi_val = 0x00182261,
+	.config_ctl_hi1_val = 0x82aa299c,
+	.test_ctl_val = 0x00000000,
+	.test_ctl_hi_val = 0x00000003,
+	.test_ctl_hi1_val = 0x00009000,
+	.test_ctl_hi2_val = 0x00000034,
+	.user_ctl_val = 0x00000000,
+	.user_ctl_hi_val = 0x00400005,
+};
+
 static struct clk_alpha_pll video_cc_pll0 = {
 	.offset = 0x0,
 	.config = &video_cc_pll0_config,
@@ -112,6 +131,15 @@ static struct clk_rcg2 video_cc_ahb_clk_src = {
 	},
 };
 
+static const struct freq_tbl ftbl_video_cc_mvs0_clk_src_nord[] = {
+	F(720000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1305000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1440000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1600000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	F(1680000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
+	{ }
+};
+
 static const struct freq_tbl ftbl_video_cc_mvs0_clk_src[] = {
 	F(720000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
 	F(1014000000, P_VIDEO_CC_PLL0_OUT_MAIN, 1, 0, 0),
@@ -508,12 +536,22 @@ static const struct qcom_cc_desc video_cc_glymur_desc = {
 
 static const struct of_device_id video_cc_glymur_match_table[] = {
 	{ .compatible = "qcom,glymur-videocc" },
+	{ .compatible = "qcom,nord-videocc" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, video_cc_glymur_match_table);
 
 static int video_cc_glymur_probe(struct platform_device *pdev)
 {
+	if (of_device_is_compatible(pdev->dev.of_node, "qcom,nord-videocc")) {
+		video_cc_pll0.vco_table = lucid_ole_vco;
+		video_cc_pll0.num_vco = ARRAY_SIZE(lucid_ole_vco);
+		video_cc_pll0.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_OLE];
+		video_cc_pll0.config = &video_cc_pll0_config_nord;
+
+		video_cc_mvs0_clk_src.freq_tbl = ftbl_video_cc_mvs0_clk_src_nord;
+	}
+
 	return qcom_cc_probe(pdev, &video_cc_glymur_desc);
 }
 
diff --git a/drivers/clk/qcom/videocc-sa8775p.c b/drivers/clk/qcom/videocc-sa8775p.c
index 65a2d5bafea68c..2ca78af8c20c62 100644
--- a/drivers/clk/qcom/videocc-sa8775p.c
+++ b/drivers/clk/qcom/videocc-sa8775p.c
@@ -511,7 +511,19 @@ static const struct regmap_config video_cc_sa8775p_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 video_cc_sa8775p_critical_cbcrs[] = {
+	0x80ec, /* VIDEO_CC_AHB_CLK */
+	0x8144, /* VIDEO_CC_SLEEP_CLK */
+	0x8128, /* VIDEO_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data video_cc_sa8775p_driver_data = {
+	.clk_cbcrs = video_cc_sa8775p_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_sa8775p_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc video_cc_sa8775p_desc = {
+	.driver_data = &video_cc_sa8775p_driver_data,
 	.config = &video_cc_sa8775p_regmap_config,
 	.clks = video_cc_sa8775p_clocks,
 	.num_clks = ARRAY_SIZE(video_cc_sa8775p_clocks),
@@ -557,11 +569,6 @@ static int video_cc_sa8775p_probe(struct platform_device *pdev)
 	if (of_device_is_compatible(pdev->dev.of_node, "qcom,qcs8300-videocc"))
 		regmap_write(regmap, video_cc_mvs0c_div2_div_clk_src.reg, 2);
 
-	/* Keep some clocks always enabled */
-	qcom_branch_set_clk_en(regmap, 0x80ec); /* VIDEO_CC_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0x8144); /* VIDEO_CC_SLEEP_CLK */
-	qcom_branch_set_clk_en(regmap, 0x8128); /* VIDEO_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &video_cc_sa8775p_desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
diff --git a/drivers/clk/qcom/videocc-sm6350.c b/drivers/clk/qcom/videocc-sm6350.c
index 34bdc5aa865ac3..d3926f87497cc1 100644
--- a/drivers/clk/qcom/videocc-sm6350.c
+++ b/drivers/clk/qcom/videocc-sm6350.c
@@ -311,7 +311,17 @@ static const struct regmap_config video_cc_sm6350_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 video_cc_sm6350_critical_cbcrs[] = {
+	0x7018, /* VIDEO_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data video_cc_sm6350_driver_data = {
+	.clk_cbcrs = video_cc_sm6350_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_sm6350_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc video_cc_sm6350_desc = {
+	.driver_data = &video_cc_sm6350_driver_data,
 	.config = &video_cc_sm6350_regmap_config,
 	.clks = video_cc_sm6350_clocks,
 	.num_clks = ARRAY_SIZE(video_cc_sm6350_clocks),
@@ -335,9 +345,6 @@ static int video_cc_sm6350_probe(struct platform_device *pdev)
 
 	clk_fabia_pll_configure(&video_pll0, regmap, &video_pll0_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x7018); /* VIDEO_CC_XO_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &video_cc_sm6350_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/videocc-sm7150.c b/drivers/clk/qcom/videocc-sm7150.c
index 8d2e00799f968a..32bad5e09056e3 100644
--- a/drivers/clk/qcom/videocc-sm7150.c
+++ b/drivers/clk/qcom/videocc-sm7150.c
@@ -313,7 +313,17 @@ static const struct regmap_config videocc_sm7150_regmap_config = {
 	.fast_io	= true,
 };
 
+static const u32 videocc_sm7150_critical_cbcrs[] = {
+	0x984, /* VIDEOCC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data videocc_sm7150_driver_data = {
+	.clk_cbcrs = videocc_sm7150_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(videocc_sm7150_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc videocc_sm7150_desc = {
+	.driver_data = &videocc_sm7150_driver_data,
 	.config = &videocc_sm7150_regmap_config,
 	.clks = videocc_sm7150_clocks,
 	.num_clks = ARRAY_SIZE(videocc_sm7150_clocks),
@@ -337,9 +347,6 @@ static int videocc_sm7150_probe(struct platform_device *pdev)
 
 	clk_fabia_pll_configure(&videocc_pll0, regmap, &videocc_pll0_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0x984); /* VIDEOCC_XO_CLK */
-
 	return qcom_cc_really_probe(&pdev->dev, &videocc_sm7150_desc, regmap);
 }
 
diff --git a/drivers/clk/qcom/videocc-sm8250.c b/drivers/clk/qcom/videocc-sm8250.c
index df479a69cddd00..d29d64138508bd 100644
--- a/drivers/clk/qcom/videocc-sm8250.c
+++ b/drivers/clk/qcom/videocc-sm8250.c
@@ -345,7 +345,18 @@ static const struct regmap_config video_cc_sm8250_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 video_cc_sm8250_critical_cbcrs[] = {
+	0xe58, /* VIDEO_CC_AHB_CLK */
+	0xeec, /* VIDEO_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data video_cc_sm8250_driver_data = {
+	.clk_cbcrs = video_cc_sm8250_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_sm8250_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc video_cc_sm8250_desc = {
+	.driver_data = &video_cc_sm8250_driver_data,
 	.config = &video_cc_sm8250_regmap_config,
 	.clks = video_cc_sm8250_clocks,
 	.num_clks = ARRAY_SIZE(video_cc_sm8250_clocks),
@@ -383,10 +394,6 @@ static int video_cc_sm8250_probe(struct platform_device *pdev)
 	clk_lucid_pll_configure(&video_pll0, regmap, &video_pll0_config);
 	clk_lucid_pll_configure(&video_pll1, regmap, &video_pll1_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xe58); /* VIDEO_CC_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, 0xeec); /* VIDEO_CC_XO_CLK */
-
 	ret = qcom_cc_really_probe(&pdev->dev, &video_cc_sm8250_desc, regmap);
 
 	pm_runtime_put(&pdev->dev);
diff --git a/drivers/clk/qcom/videocc-sm8350.c b/drivers/clk/qcom/videocc-sm8350.c
index 057a9474894abf..8528aeb3a34257 100644
--- a/drivers/clk/qcom/videocc-sm8350.c
+++ b/drivers/clk/qcom/videocc-sm8350.c
@@ -510,7 +510,28 @@ static const struct regmap_config video_cc_sm8350_regmap_config = {
 	.fast_io = true,
 };
 
+static const u32 video_cc_sm8350_critical_cbcrs[] = {
+	0xe58, /* VIDEO_CC_AHB_CLK */
+	0xeec, /* VIDEO_CC_XO_CLK */
+};
+
+static const u32 video_cc_sc8280xp_critical_cbcrs[] = {
+	0xe58, /* VIDEO_CC_AHB_CLK */
+	0xf34, /* VIDEO_CC_XO_CLK */
+};
+
+static const struct qcom_cc_driver_data video_cc_sm8350_driver_data = {
+	.clk_cbcrs = video_cc_sm8350_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_sm8350_critical_cbcrs),
+};
+
+static const struct qcom_cc_driver_data video_cc_sc8280xp_driver_data = {
+	.clk_cbcrs = video_cc_sc8280xp_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(video_cc_sc8280xp_critical_cbcrs),
+};
+
 static const struct qcom_cc_desc video_cc_sm8350_desc = {
+	.driver_data = &video_cc_sm8350_driver_data,
 	.config = &video_cc_sm8350_regmap_config,
 	.clks = video_cc_sm8350_clocks,
 	.num_clks = ARRAY_SIZE(video_cc_sm8350_clocks),
@@ -522,7 +543,7 @@ static const struct qcom_cc_desc video_cc_sm8350_desc = {
 
 static int video_cc_sm8350_probe(struct platform_device *pdev)
 {
-	u32 video_cc_xo_clk_cbcr = 0xeec;
+	struct qcom_cc_desc desc = video_cc_sm8350_desc;
 	struct regmap *regmap;
 	int ret;
 
@@ -539,7 +560,7 @@ static int video_cc_sm8350_probe(struct platform_device *pdev)
 		video_cc_sleep_clk.halt_reg = 0xf58;
 		video_cc_sleep_clk.clkr.enable_reg = 0xf58;
 		video_cc_xo_clk_src.cmd_rcgr = 0xf14;
-		video_cc_xo_clk_cbcr = 0xf34;
+		desc.driver_data = &video_cc_sc8280xp_driver_data;
 
 		video_pll0.vco_table = video_pll1.vco_table = lucid_5lpe_vco_8280xp;
 		/* No change, but assign it for completeness */
@@ -558,11 +579,7 @@ static int video_cc_sm8350_probe(struct platform_device *pdev)
 	clk_lucid_pll_configure(&video_pll0, regmap, &video_pll0_config);
 	clk_lucid_pll_configure(&video_pll1, regmap, &video_pll1_config);
 
-	/* Keep some clocks always-on */
-	qcom_branch_set_clk_en(regmap, 0xe58); /* VIDEO_CC_AHB_CLK */
-	qcom_branch_set_clk_en(regmap, video_cc_xo_clk_cbcr); /* VIDEO_CC_XO_CLK */
-
-	ret = qcom_cc_really_probe(&pdev->dev, &video_cc_sm8350_desc, regmap);
+	ret = qcom_cc_really_probe(&pdev->dev, &desc, regmap);
 	pm_runtime_put(&pdev->dev);
 
 	return ret;
diff --git a/drivers/clk/qcom/videocc-sm8750.c b/drivers/clk/qcom/videocc-sm8750.c
index b62271a7dac6a6..91d18a90b84f29 100644
--- a/drivers/clk/qcom/videocc-sm8750.c
+++ b/drivers/clk/qcom/videocc-sm8750.c
@@ -445,17 +445,7 @@ static struct platform_driver video_cc_sm8750_driver = {
 	},
 };
 
-static int __init video_cc_sm8750_init(void)
-{
-	return platform_driver_register(&video_cc_sm8750_driver);
-}
-subsys_initcall(video_cc_sm8750_init);
-
-static void __exit video_cc_sm8750_exit(void)
-{
-	platform_driver_unregister(&video_cc_sm8750_driver);
-}
-module_exit(video_cc_sm8750_exit);
+subsys_platform_driver(video_cc_sm8750_driver);
 
 MODULE_DESCRIPTION("QTI VIDEO_CC SM8750 Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig
index 17b95a2e96fb05..a02ca6b096c0d1 100644
--- a/drivers/clk/renesas/Kconfig
+++ b/drivers/clk/renesas/Kconfig
@@ -13,6 +13,7 @@ config CLK_RENESAS
 	select CLK_R8A7745 if ARCH_R8A7745
 	select CLK_R8A77470 if ARCH_R8A77470
 	select CLK_R8A774A1 if ARCH_R8A774A1
+	select CLK_R8A774A3 if ARCH_R8A774A3
 	select CLK_R8A774B1 if ARCH_R8A774B1
 	select CLK_R8A774C0 if ARCH_R8A774C0
 	select CLK_R8A774E1 if ARCH_R8A774E1
@@ -93,6 +94,10 @@ config CLK_R8A774A1
 	bool "RZ/G2M clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN3_CPG
 
+config CLK_R8A774A3
+	bool "RZ/G2M v3.0 clock support" if COMPILE_TEST
+	select CLK_RCAR_GEN3_CPG
+
 config CLK_R8A774B1
 	bool "RZ/G2N clock support" if COMPILE_TEST
 	select CLK_RCAR_GEN3_CPG
@@ -223,11 +228,13 @@ config CLK_R9A09G077
 	bool "RZ/T2H clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSSR
 	select CLK_RZV2H_CPG_LIB
+	select MFD_SYSCON
 
 config CLK_R9A09G087
 	bool "RZ/N2H clock support" if COMPILE_TEST
 	select CLK_RENESAS_CPG_MSSR
 	select CLK_RZV2H_CPG_LIB
+	select MFD_SYSCON
 
 config CLK_SH73A0
 	bool "SH-Mobile AG5 clock support" if COMPILE_TEST
diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile
index 658f8aac5127cd..5c1b798118db72 100644
--- a/drivers/clk/renesas/Makefile
+++ b/drivers/clk/renesas/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_CLK_R8A7743)		+= r8a7743-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A7745)		+= r8a7745-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A77470)		+= r8a77470-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A774A1)		+= r8a774a1-cpg-mssr.o
+obj-$(CONFIG_CLK_R8A774A3)		+= r8a774a3-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A774B1)		+= r8a774b1-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A774C0)		+= r8a774c0-cpg-mssr.o
 obj-$(CONFIG_CLK_R8A774E1)		+= r8a774e1-cpg-mssr.o
diff --git a/drivers/clk/renesas/r8a774a3-cpg-mssr.c b/drivers/clk/renesas/r8a774a3-cpg-mssr.c
new file mode 100644
index 00000000000000..64c5106a9a548b
--- /dev/null
+++ b/drivers/clk/renesas/r8a774a3-cpg-mssr.c
@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * r8a774a3 Clock Pulse Generator / Module Standby and Software Reset
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ *
+ * Based on r8a7796-cpg-mssr.c
+ *
+ * Copyright (C) 2016 Glider bvba
+ */
+
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/soc/renesas/rcar-rst.h>
+
+#include <dt-bindings/clock/renesas,r8a774a3-cpg-mssr.h>
+
+#include "renesas-cpg-mssr.h"
+#include "rcar-gen3-cpg.h"
+
+enum clk_ids {
+	/* Core Clock Outputs exported to DT */
+	LAST_DT_CORE_CLK = R8A774A3_CLK_OSC,
+
+	/* External Input Clocks */
+	CLK_EXTAL,
+	CLK_EXTALR,
+
+	/* Internal Core Clocks */
+	CLK_MAIN,
+	CLK_PLL0,
+	CLK_PLL1,
+	CLK_PLL2,
+	CLK_PLL3,
+	CLK_PLL4,
+	CLK_PLL1_DIV2,
+	CLK_PLL1_DIV4,
+	CLK_S0,
+	CLK_S1,
+	CLK_S2,
+	CLK_S3,
+
+	/* Module Clocks */
+	MOD_CLK_BASE
+};
+
+static const struct cpg_core_clk r8a774a3_core_clks[] __initconst = {
+	/* External Clock Inputs */
+	DEF_INPUT("extal",      CLK_EXTAL),
+	DEF_INPUT("extalr",     CLK_EXTALR),
+
+	/* Internal Core Clocks */
+	DEF_BASE(".main",       CLK_MAIN, CLK_TYPE_GEN3_MAIN, CLK_EXTAL),
+	DEF_BASE(".pll0",       CLK_PLL0, CLK_TYPE_GEN3_PLL0, CLK_MAIN),
+	DEF_BASE(".pll1",       CLK_PLL1, CLK_TYPE_GEN3_PLL1, CLK_MAIN),
+	DEF_BASE(".pll2",       CLK_PLL2, CLK_TYPE_GEN3_PLL2, CLK_MAIN),
+
+	DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2,     CLK_PLL1,       2, 1),
+	DEF_FIXED(".s0",        CLK_S0,            CLK_PLL1_DIV2,  2, 1),
+	DEF_FIXED(".s3",        CLK_S3,            CLK_PLL1_DIV2,  6, 1),
+
+	/* Core Clock Outputs */
+	DEF_GEN3_Z("z",         R8A774A3_CLK_Z,     CLK_TYPE_GEN3_Z, CLK_PLL0, 2, 8),
+	DEF_GEN3_Z("z2",        R8A774A3_CLK_Z2,    CLK_TYPE_GEN3_Z,  CLK_PLL2, 2, 0),
+	DEF_FIXED("s0d3",       R8A774A3_CLK_S0D3,  CLK_S0,         3, 1),
+	DEF_FIXED("s3d1",       R8A774A3_CLK_S3D1,  CLK_S3,         1, 1),
+	DEF_FIXED("s3d4",       R8A774A3_CLK_S3D4,  CLK_S3,         4, 1),
+};
+
+static const struct mssr_mod_clk r8a774a3_mod_clks[] __initconst = {
+	DEF_MOD("scif2",		 310,	R8A774A3_CLK_S3D4),
+	DEF_MOD("intc-ap",		 408,	R8A774A3_CLK_S0D3),
+};
+
+static const unsigned int r8a774a3_crit_mod_clks[] __initconst = {
+	MOD_CLK_ID(408),	/* INTC-AP (GIC) */
+};
+
+/*
+ * CPG Clock Data
+ */
+
+/*
+ *   MD		EXTAL		PLL0	PLL1	PLL2	PLL3	PLL4	OSC
+ * 14 13 19 17	(MHz)
+ *-------------------------------------------------------------------------
+ * 0  0  0  0	16.66 x 1	x180	x192	x144	x192	x144	/16
+ * 0  0  0  1	16.66 x 1	x180	x192	x144	x128	x144	/16
+ * 0  0  1  0	Prohibited setting
+ * 0  0  1  1	16.66 x 1	x180	x192	x144	x192	x144	/16
+ * 0  1  0  0	20    x 1	x150	x160	x120	x160	x120	/19
+ * 0  1  0  1	20    x 1	x150	x160	x120	x106	x120	/19
+ * 0  1  1  0	Prohibited setting
+ * 0  1  1  1	20    x 1	x150	x160	x120	x160	x120	/19
+ * 1  0  0  0	25    x 1	x120	x128	x96	x128	x96	/24
+ * 1  0  0  1	25    x 1	x120	x128	x96	x84	x96	/24
+ * 1  0  1  0	Prohibited setting
+ * 1  0  1  1	25    x 1	x120	x128	x96	x128	x96	/24
+ * 1  1  0  0	33.33 / 2	x180	x192	x144	x192	x144	/32
+ * 1  1  0  1	33.33 / 2	x180	x192	x144	x128	x144	/32
+ * 1  1  1  0	Prohibited setting
+ * 1  1  1  1	33.33 / 2	x180	x192	x144	x192	x144	/32
+ */
+#define CPG_PLL_CONFIG_INDEX(md)	((((md) & BIT(14)) >> 11) | \
+					 (((md) & BIT(13)) >> 11) | \
+					 (((md) & BIT(19)) >> 18) | \
+					 (((md) & BIT(17)) >> 17))
+
+static const struct rcar_gen3_cpg_pll_config cpg_pll_configs[16] __initconst = {
+	/* EXTAL div	PLL1 mult/div	PLL3 mult/div	OSC prediv */
+	{ 1,		192,	1,	192,	1,	16,	},
+	{ 1,		192,	1,	128,	1,	16,	},
+	{ 0, /* Prohibited setting */				},
+	{ 1,		192,	1,	192,	1,	16,	},
+	{ 1,		160,	1,	160,	1,	19,	},
+	{ 1,		160,	1,	106,	1,	19,	},
+	{ 0, /* Prohibited setting */				},
+	{ 1,		160,	1,	160,	1,	19,	},
+	{ 1,		128,	1,	128,	1,	24,	},
+	{ 1,		128,	1,	84,	1,	24,	},
+	{ 0, /* Prohibited setting */				},
+	{ 1,		128,	1,	128,	1,	24,	},
+	{ 2,		192,	1,	192,	1,	32,	},
+	{ 2,		192,	1,	128,	1,	32,	},
+	{ 0, /* Prohibited setting */				},
+	{ 2,		192,	1,	192,	1,	32,	},
+};
+
+static int __init r8a774a3_cpg_mssr_init(struct device *dev)
+{
+	const struct rcar_gen3_cpg_pll_config *cpg_pll_config;
+	u32 cpg_mode;
+	int error;
+
+	error = rcar_rst_read_mode_pins(&cpg_mode);
+	if (error)
+		return error;
+
+	cpg_pll_config = &cpg_pll_configs[CPG_PLL_CONFIG_INDEX(cpg_mode)];
+	if (!cpg_pll_config->extal_div) {
+		dev_err(dev, "Prohibited setting (cpg_mode=0x%x)\n", cpg_mode);
+		return -EINVAL;
+	}
+
+	return rcar_gen3_cpg_init(cpg_pll_config, CLK_EXTALR, cpg_mode);
+}
+
+const struct cpg_mssr_info r8a774a3_cpg_mssr_info __initconst = {
+	/* Core Clocks */
+	.core_clks = r8a774a3_core_clks,
+	.num_core_clks = ARRAY_SIZE(r8a774a3_core_clks),
+	.last_dt_core_clk = LAST_DT_CORE_CLK,
+	.num_total_core_clks = MOD_CLK_BASE,
+
+	/* Module Clocks */
+	.mod_clks = r8a774a3_mod_clks,
+	.num_mod_clks = ARRAY_SIZE(r8a774a3_mod_clks),
+	.num_hw_mod_clks = 12 * 32,
+
+	/* Critical Module Clocks */
+	.crit_mod_clks = r8a774a3_crit_mod_clks,
+	.num_crit_mod_clks = ARRAY_SIZE(r8a774a3_crit_mod_clks),
+
+	/* Callbacks */
+	.init = r8a774a3_cpg_mssr_init,
+	.cpg_clk_register = rcar_gen3_cpg_clk_register,
+};
diff --git a/drivers/clk/renesas/r8a779a0-cpg-mssr.c b/drivers/clk/renesas/r8a779a0-cpg-mssr.c
index d67dff05d9f4a9..a18a51dab64fa2 100644
--- a/drivers/clk/renesas/r8a779a0-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a779a0-cpg-mssr.c
@@ -99,9 +99,9 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = {
 	DEF_BASE(".rpcsrc",		CLK_RPCSRC,	CLK_TYPE_GEN4_RPCSRC, CLK_PLL5),
 
 	/* Core Clock Outputs */
-	DEF_GEN4_Z("z0",	R8A779A0_CLK_Z0,	CLK_TYPE_GEN4_Z,	CLK_PLL20,	2, 0),
-	DEF_GEN4_Z("z1",	R8A779A0_CLK_Z1,	CLK_TYPE_GEN4_Z,	CLK_PLL21,	2, 8),
-	DEF_GEN4_Z("zg",	R8A779A0_CLK_ZG,	CLK_TYPE_GEN4_Z,	CLK_PLL4,	2, 88),
+	DEF_GEN4_Z("z0",	R8A779A0_CLK_Z0,	CLK_PLL20,	2, CPG_FRQCRC0, 0),
+	DEF_GEN4_Z("z1",	R8A779A0_CLK_Z1,	CLK_PLL21,	2, CPG_FRQCRC0, 8),
+	DEF_GEN4_Z("zg",	R8A779A0_CLK_ZG,	CLK_PLL4,	2, CPG_FRQCRB,  24),
 	DEF_FIXED("zx",		R8A779A0_CLK_ZX,	CLK_PLL20_DIV2,	2, 1),
 	DEF_FIXED("s1d1",	R8A779A0_CLK_S1D1,	CLK_S1,		1, 1),
 	DEF_FIXED("s1d2",	R8A779A0_CLK_S1D2,	CLK_S1,		2, 1),
diff --git a/drivers/clk/renesas/r8a779f0-cpg-mssr.c b/drivers/clk/renesas/r8a779f0-cpg-mssr.c
index f33342314b2ef9..1264c3fc50aad7 100644
--- a/drivers/clk/renesas/r8a779f0-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a779f0-cpg-mssr.c
@@ -79,8 +79,8 @@ static const struct cpg_core_clk r8a779f0_core_clks[] __initconst = {
 	DEF_BASE(".rpcsrc",	CLK_RPCSRC,	CLK_TYPE_GEN4_RPCSRC, CLK_PLL5),
 
 	/* Core Clock Outputs */
-	DEF_GEN4_Z("z0",	R8A779F0_CLK_Z0,	CLK_TYPE_GEN4_Z,	CLK_PLL2,	2, 0),
-	DEF_GEN4_Z("z1",	R8A779F0_CLK_Z1,	CLK_TYPE_GEN4_Z,	CLK_PLL2,	2, 8),
+	DEF_GEN4_Z("z0",	R8A779F0_CLK_Z0,	CLK_PLL2,	2, CPG_FRQCRC0, 0),
+	DEF_GEN4_Z("z1",	R8A779F0_CLK_Z1,	CLK_PLL2,	2, CPG_FRQCRC0, 8),
 	DEF_FIXED("s0d2",	R8A779F0_CLK_S0D2,	CLK_S0,		2, 1),
 	DEF_FIXED("s0d3",	R8A779F0_CLK_S0D3,	CLK_S0,		3, 1),
 	DEF_FIXED("s0d4",	R8A779F0_CLK_S0D4,	CLK_S0,		4, 1),
diff --git a/drivers/clk/renesas/r8a779g0-cpg-mssr.c b/drivers/clk/renesas/r8a779g0-cpg-mssr.c
index 54ba76ff5ab0fc..9a649f2524d932 100644
--- a/drivers/clk/renesas/r8a779g0-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a779g0-cpg-mssr.c
@@ -96,7 +96,8 @@ static const struct cpg_core_clk r8a779g0_core_clks[] __initconst = {
 	DEF_FIXED(".vc",	CLK_VC,		CLK_PLL5_DIV2,	3, 1),
 
 	/* Core Clock Outputs */
-	DEF_GEN4_Z("z0",	R8A779G0_CLK_Z0,	CLK_TYPE_GEN4_Z,	CLK_PLL2,	2, 0),
+	DEF_GEN4_Z("z0",	R8A779G0_CLK_Z0,	CLK_PLL2,	2, CPG_FRQCRC0, 0),
+	DEF_GEN4_Z("zg",	R8A779G0_CLK_ZG,	CLK_PLL4,	2, CPG_FRQCRB, 24),
 	DEF_FIXED("s0d2",	R8A779G0_CLK_S0D2,	CLK_S0,		2, 1),
 	DEF_FIXED("s0d3",	R8A779G0_CLK_S0D3,	CLK_S0,		3, 1),
 	DEF_FIXED("s0d4",	R8A779G0_CLK_S0D4,	CLK_S0,		4, 1),
@@ -163,6 +164,7 @@ static const struct cpg_core_clk r8a779g0_core_clks[] __initconst = {
 };
 
 static const struct mssr_mod_clk r8a779g0_mod_clks[] __initconst = {
+	DEF_MOD("rgx",		  0,	R8A779G0_CLK_ZG),
 	DEF_MOD("isp0",		 16,	R8A779G0_CLK_S0D2_VIO),
 	DEF_MOD("isp1",		 17,	R8A779G0_CLK_S0D2_VIO),
 	DEF_MOD("avb0",		211,	R8A779G0_CLK_S0D4_HSC),
diff --git a/drivers/clk/renesas/r8a779h0-cpg-mssr.c b/drivers/clk/renesas/r8a779h0-cpg-mssr.c
index ffea06d77d5eab..cde59cd352957c 100644
--- a/drivers/clk/renesas/r8a779h0-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a779h0-cpg-mssr.c
@@ -99,10 +99,10 @@ static const struct cpg_core_clk r8a779h0_core_clks[] __initconst = {
 	DEF_RATE(".oco",	CLK_OCO,	32768),
 
 	/* Core Clock Outputs */
-	DEF_GEN4_Z("zc0",	R8A779H0_CLK_ZC0,	CLK_TYPE_GEN4_Z,	CLK_PLL2,	4, 0),
-	DEF_GEN4_Z("zc1",	R8A779H0_CLK_ZC1,	CLK_TYPE_GEN4_Z,	CLK_PLL2,	4, 8),
-	DEF_GEN4_Z("zc2",	R8A779H0_CLK_ZC2,	CLK_TYPE_GEN4_Z,	CLK_PLL2,	4, 32),
-	DEF_GEN4_Z("zc3",	R8A779H0_CLK_ZC3,	CLK_TYPE_GEN4_Z,	CLK_PLL2,	4, 40),
+	DEF_GEN4_Z("zc0",	R8A779H0_CLK_ZC0,	CLK_PLL2,	4, CPG_FRQCRC0, 0),
+	DEF_GEN4_Z("zc1",	R8A779H0_CLK_ZC1,	CLK_PLL2,	4, CPG_FRQCRC0, 8),
+	DEF_GEN4_Z("zc2",	R8A779H0_CLK_ZC2,	CLK_PLL2,	4, CPG_FRQCRC1, 0),
+	DEF_GEN4_Z("zc3",	R8A779H0_CLK_ZC3,	CLK_PLL2,	4, CPG_FRQCRC1, 8),
 	DEF_FIXED("s0d2",	R8A779H0_CLK_S0D2,	CLK_S0,		2, 1),
 	DEF_FIXED("s0d3",	R8A779H0_CLK_S0D3,	CLK_S0,		3, 1),
 	DEF_FIXED("s0d4",	R8A779H0_CLK_S0D4,	CLK_S0,		4, 1),
diff --git a/drivers/clk/renesas/r8a78000-cpg.c b/drivers/clk/renesas/r8a78000-cpg.c
index 7946f9d0b423de..b8024e76743ea5 100644
--- a/drivers/clk/renesas/r8a78000-cpg.c
+++ b/drivers/clk/renesas/r8a78000-cpg.c
@@ -23,12 +23,14 @@ struct clk_map {
 
 enum fixed_clk {
 	FIXED_CLK_66M,
+	FIXED_CLK_133M,
 	FIXED_CLK_266M,
 	NUM_FIXED_CLKS
 };
 
 static const unsigned long fixed_clk_rates[NUM_FIXED_CLKS] = {
 	[FIXED_CLK_66M] = 66666000,
+	[FIXED_CLK_133M] = 133330000,
 	[FIXED_CLK_266M] = 266660000,
 };
 
@@ -140,6 +142,7 @@ static int r8a78000_cpg_probe(struct platform_device *pdev)
 
 static const struct clk_map r8a78000_cpg_default[] = {
 	{ R8A78000_CPG_SGASYNCD4_PERW_BUS,	FIXED_CLK(266M) },
+	{ R8A78000_CPG_SGASYNCD8_PERW_BUS,	FIXED_CLK(133M) },
 	{ R8A78000_CPG_SGASYNCD16_PERW_BUS,	FIXED_CLK(66M) },
 	{ -1 }
 };
diff --git a/drivers/clk/renesas/r9a06g032-clocks.c b/drivers/clk/renesas/r9a06g032-clocks.c
index 076f587dfd39d7..3df3f2b2c6de72 100644
--- a/drivers/clk/renesas/r9a06g032-clocks.c
+++ b/drivers/clk/renesas/r9a06g032-clocks.c
@@ -1014,7 +1014,7 @@ r9a06g032_div_determine_rate(struct clk_hw *hw, struct clk_rate_request *req)
 	if (clk->index == R9A06G032_DIV_UART ||
 	    clk->index == R9A06G032_DIV_P2_PG) {
 		pr_devel("%s div uart hack!\n", __func__);
-		req->rate = clk_get_rate(hw->clk);
+		req->rate = clk_hw_get_rate(hw);
 		return 0;
 	}
 	req->rate = DIV_ROUND_UP(req->best_parent_rate, div);
@@ -1419,10 +1419,4 @@ static struct platform_driver r9a06g032_clock_driver = {
 	},
 };
 
-static int __init r9a06g032_clocks_init(void)
-{
-	return platform_driver_probe(&r9a06g032_clock_driver,
-			r9a06g032_clocks_probe);
-}
-
-subsys_initcall(r9a06g032_clocks_init);
+subsys_platform_driver_probe(r9a06g032_clock_driver, r9a06g032_clocks_probe);
diff --git a/drivers/clk/renesas/r9a07g043-cpg.c b/drivers/clk/renesas/r9a07g043-cpg.c
index 59d6ee2e888fa7..f7e32abc5d0b31 100644
--- a/drivers/clk/renesas/r9a07g043-cpg.c
+++ b/drivers/clk/renesas/r9a07g043-cpg.c
@@ -119,7 +119,7 @@ static const struct cpg_core_clk r9a07g043_core_clks[] __initconst = {
 	DEF_FIXED(".pll3_400", CLK_PLL3_400, CLK_PLL3, 1, 4),
 	DEF_FIXED(".pll3_533", CLK_PLL3_533, CLK_PLL3, 1, 3),
 	DEF_MUX_RO(".sel_pll3_3", CLK_SEL_PLL3_3, SEL_PLL3_3, sel_pll3_3),
-	DEF_DIV("divpl3c", CLK_DIV_PLL3_C, CLK_SEL_PLL3_3, DIVPL3C, dtable_1_32),
+	DEF_DIV(".divpl3c", CLK_DIV_PLL3_C, CLK_SEL_PLL3_3, DIVPL3C, dtable_1_32),
 #ifdef CONFIG_ARM64
 	DEF_FIXED(".pll5", CLK_PLL5, CLK_EXTAL, 125, 1),
 	DEF_FIXED(".pll5_500", CLK_PLL5_500, CLK_PLL5, 1, 6),
@@ -135,7 +135,7 @@ static const struct cpg_core_clk r9a07g043_core_clks[] __initconst = {
 	DEF_FIXED("P0_DIV2", R9A07G043_CLK_P0_DIV2, R9A07G043_CLK_P0, 1, 2),
 	DEF_FIXED("TSU", R9A07G043_CLK_TSU, CLK_PLL2_DIV2_10, 1, 1),
 	DEF_DIV("P1", R9A07G043_CLK_P1, CLK_PLL3_DIV2_4, DIVPL3B, dtable_1_32),
-	DEF_FIXED("P1_DIV2", CLK_P1_DIV2, R9A07G043_CLK_P1, 1, 2),
+	DEF_FIXED(".p1_div2", CLK_P1_DIV2, R9A07G043_CLK_P1, 1, 2),
 	DEF_DIV("P2", R9A07G043_CLK_P2, CLK_PLL3_DIV2_4_2, DIVPL3A, dtable_1_32),
 	DEF_FIXED("M0", R9A07G043_CLK_M0, CLK_PLL3_DIV2_4, 1, 1),
 	DEF_FIXED("ZT", R9A07G043_CLK_ZT, CLK_PLL3_DIV2_4_2, 1, 1),
@@ -151,12 +151,12 @@ static const struct cpg_core_clk r9a07g043_core_clks[] __initconst = {
 		   mtable_sdhi, 0, rzg2l_cpg_sd_clk_mux_notifier),
 	DEF_SD_MUX("SD1", R9A07G043_CLK_SD1, SEL_SDHI1, SEL_SDHI1_STS, sel_sdhi,
 		   mtable_sdhi, 0, rzg2l_cpg_sd_clk_mux_notifier),
-	DEF_FIXED("SD0_DIV4", CLK_SD0_DIV4, R9A07G043_CLK_SD0, 1, 4),
-	DEF_FIXED("SD1_DIV4", CLK_SD1_DIV4, R9A07G043_CLK_SD1, 1, 4),
+	DEF_FIXED(".sd0_div4", CLK_SD0_DIV4, R9A07G043_CLK_SD0, 1, 4),
+	DEF_FIXED(".sd1_div4", CLK_SD1_DIV4, R9A07G043_CLK_SD1, 1, 4),
 #ifdef CONFIG_ARM64
 	DEF_FIXED("M2", R9A07G043_CLK_M2, CLK_PLL3_533, 1, 2),
-	DEF_FIXED("M2_DIV2", CLK_M2_DIV2, R9A07G043_CLK_M2, 1, 2),
-	DEF_DSI_DIV("DSI_DIV", CLK_DSI_DIV, CLK_PLL5_FOUTPOSTDIV, CLK_SET_RATE_PARENT),
+	DEF_FIXED(".m2_div2", CLK_M2_DIV2, R9A07G043_CLK_M2, 1, 2),
+	DEF_DSI_DIV(".dsi_div", CLK_DSI_DIV, CLK_PLL5_FOUTPOSTDIV, CLK_SET_RATE_PARENT),
 	DEF_FIXED("M3", R9A07G043_CLK_M3, CLK_DSI_DIV, 1, 1),
 #endif
 };
diff --git a/drivers/clk/renesas/r9a07g044-cpg.c b/drivers/clk/renesas/r9a07g044-cpg.c
index 913cca6dd46fc5..3b6d0bd1ec134b 100644
--- a/drivers/clk/renesas/r9a07g044-cpg.c
+++ b/drivers/clk/renesas/r9a07g044-cpg.c
@@ -187,7 +187,7 @@ static const struct {
 		DEF_FIXED(".pll3_div2_4", CLK_PLL3_DIV2_4, CLK_PLL3_DIV2, 1, 4),
 		DEF_FIXED(".pll3_div2_4_2", CLK_PLL3_DIV2_4_2, CLK_PLL3_DIV2_4, 1, 2),
 		DEF_MUX_RO(".sel_pll3_3", CLK_SEL_PLL3_3, SEL_PLL3_3, sel_pll3_3),
-		DEF_DIV("divpl3c", CLK_DIV_PLL3_C, CLK_SEL_PLL3_3, DIVPL3C, dtable_1_32),
+		DEF_DIV(".divpl3c", CLK_DIV_PLL3_C, CLK_SEL_PLL3_3, DIVPL3C, dtable_1_32),
 
 		DEF_FIXED(".pll5_250", CLK_PLL5_250, CLK_PLL5_FOUT3, 1, 2),
 		DEF_FIXED(".pll6_250", CLK_PLL6_250, CLK_PLL6, 1, 2),
@@ -204,7 +204,7 @@ static const struct {
 		DEF_FIXED("P0_DIV2", R9A07G044_CLK_P0_DIV2, R9A07G044_CLK_P0, 1, 2),
 		DEF_FIXED("TSU", R9A07G044_CLK_TSU, CLK_PLL2_DIV2_10, 1, 1),
 		DEF_DIV("P1", R9A07G044_CLK_P1, CLK_PLL3_DIV2_4, DIVPL3B, dtable_1_32),
-		DEF_FIXED("P1_DIV2", CLK_P1_DIV2, R9A07G044_CLK_P1, 1, 2),
+		DEF_FIXED(".p1_div2", CLK_P1_DIV2, R9A07G044_CLK_P1, 1, 2),
 		DEF_DIV("P2", R9A07G044_CLK_P2, CLK_PLL3_DIV2_4_2, DIVPL3A, dtable_1_32),
 		DEF_FIXED("M0", R9A07G044_CLK_M0, CLK_PLL3_DIV2_4, 1, 1),
 		DEF_FIXED("ZT", R9A07G044_CLK_ZT, CLK_PLL3_DIV2_4_2, 1, 1),
@@ -215,13 +215,13 @@ static const struct {
 			   mtable_sdhi, 0, rzg2l_cpg_sd_clk_mux_notifier),
 		DEF_SD_MUX("SD1", R9A07G044_CLK_SD1, SEL_SDHI1, SEL_SDHI1_STS, sel_sdhi,
 			   mtable_sdhi, 0, rzg2l_cpg_sd_clk_mux_notifier),
-		DEF_FIXED("SD0_DIV4", CLK_SD0_DIV4, R9A07G044_CLK_SD0, 1, 4),
-		DEF_FIXED("SD1_DIV4", CLK_SD1_DIV4, R9A07G044_CLK_SD1, 1, 4),
+		DEF_FIXED(".sd0_div4", CLK_SD0_DIV4, R9A07G044_CLK_SD0, 1, 4),
+		DEF_FIXED(".sd1_div4", CLK_SD1_DIV4, R9A07G044_CLK_SD1, 1, 4),
 		DEF_DIV("G", R9A07G044_CLK_G, CLK_SEL_GPU2, DIVGPU, dtable_1_8),
 		DEF_FIXED("M1", R9A07G044_CLK_M1, CLK_PLL5_FOUTPOSTDIV, 1, 1),
 		DEF_FIXED("M2", R9A07G044_CLK_M2, CLK_PLL3_533, 1, 2),
-		DEF_FIXED("M2_DIV2", CLK_M2_DIV2, R9A07G044_CLK_M2, 1, 2),
-		DEF_DSI_DIV("DSI_DIV", CLK_DSI_DIV, CLK_SEL_PLL5_4, CLK_SET_RATE_PARENT),
+		DEF_FIXED(".m2_div2", CLK_M2_DIV2, R9A07G044_CLK_M2, 1, 2),
+		DEF_DSI_DIV(".dsi_div", CLK_DSI_DIV, CLK_SEL_PLL5_4, CLK_SET_RATE_PARENT),
 		DEF_FIXED("M3", R9A07G044_CLK_M3, CLK_DSI_DIV, 1, 1),
 		DEF_FIXED("M4", R9A07G044_CLK_M4, CLK_DIV_DSI_LPCLK, 1, 1),
 	},
diff --git a/drivers/clk/renesas/r9a08g045-cpg.c b/drivers/clk/renesas/r9a08g045-cpg.c
index ea2a6a71aebd71..4e1cd3177ecd86 100644
--- a/drivers/clk/renesas/r9a08g045-cpg.c
+++ b/drivers/clk/renesas/r9a08g045-cpg.c
@@ -167,14 +167,14 @@ static const struct cpg_core_clk r9a08g045_core_clks[] __initconst = {
 	DEF_FIXED("M0", R9A08G045_CLK_M0, CLK_PLL3_DIV2_4, 1, 1),
 	DEF_G3S_DIV("P1", R9A08G045_CLK_P1, CLK_PLL3_DIV2_4, DIVPL3A, G3S_DIVPL3A_STS,
 		    dtable_1_32, 0, 0, 0, NULL),
-	DEF_FIXED("P1_DIV2", CLK_P1_DIV2, R9A08G045_CLK_P1, 1, 2),
+	DEF_FIXED(".p1_div2", CLK_P1_DIV2, R9A08G045_CLK_P1, 1, 2),
 	DEF_G3S_DIV("P2", R9A08G045_CLK_P2, CLK_PLL3_DIV2_8, DIVPL3B, G3S_DIVPL3B_STS,
 		    dtable_1_32, 0, 0, 0, NULL),
 	DEF_G3S_DIV("P3", R9A08G045_CLK_P3, CLK_PLL3_DIV2_4, DIVPL3C, G3S_DIVPL3C_STS,
 		    dtable_1_32, 0, 0, 0, NULL),
-	DEF_FIXED("P3_DIV2", CLK_P3_DIV2, R9A08G045_CLK_P3, 1, 2),
+	DEF_FIXED(".p3_div2", CLK_P3_DIV2, R9A08G045_CLK_P3, 1, 2),
 	DEF_FIXED("P4", R9A08G045_CLK_P4, CLK_PLL2_DIV2, 1, 5),
-	DEF_FIXED("P4_DIV2", CLK_P4_DIV2, R9A08G045_CLK_P4, 1, 2),
+	DEF_FIXED(".p4_div2", CLK_P4_DIV2, R9A08G045_CLK_P4, 1, 2),
 	DEF_FIXED("P5", R9A08G045_CLK_P5, CLK_PLL2_DIV2, 1, 4),
 	DEF_FIXED("ZT", R9A08G045_CLK_ZT, CLK_PLL3_DIV2_8, 1, 1),
 	DEF_FIXED("S0", R9A08G045_CLK_S0, CLK_SEL_PLL4, 1, 2),
diff --git a/drivers/clk/renesas/r9a08g046-cpg.c b/drivers/clk/renesas/r9a08g046-cpg.c
index 4488bd1988e822..b58057db9c4d09 100644
--- a/drivers/clk/renesas/r9a08g046-cpg.c
+++ b/drivers/clk/renesas/r9a08g046-cpg.c
@@ -19,30 +19,32 @@
 #define G3L_CPG_PL3_DDIV		(0x208)
 #define G3L_CPG_SDHI_DDIV		(0x218)
 #define G3L_CPG_GE3D_DDIV		(0x224)
+#define G3L_CPG_DSI_DDIV		(0x228)
 #define G3L_CPG_CA55CORE_DDIV		(0x234)
 #define G3L_CPG_RSCI_DDIV		(0x238)
 #define G3L_CPG_RSPI_DDIV		(0x23c)
 #define G3L_CPG_SDHI_DSEL		(0x244)
 #define G3L_CLKDIVSTATUS		(0x280)
 #define G3L_CLKSELSTATUS		(0x284)
+#define G3L_CPG_DSI_SSEL		(0x408)
 #define G3L_CPG_GE3D_SSEL		(0x40c)
 #define G3L_CPG_ETH_SSEL		(0x410)
 #define G3L_CPG_RSCI_SSEL		(0x414)
 #define G3L_CPG_RSPI_SSEL		(0x418)
+#define G3L_CPG_DSI_SDIV		(0x430)
 #define G3L_CPG_ETH_SDIV		(0x434)
 
 /* RZ/G3L Specific division configuration.  */
 #define G3L_DIVPL2A		DDIV_PACK(G3L_CPG_PL2_DDIV, 0, 2)
 #define G3L_DIVPL2B		DDIV_PACK(G3L_CPG_PL2_DDIV, 4, 2)
 #define G3L_DIVPL3A		DDIV_PACK(G3L_CPG_PL3_DDIV, 0, 2)
-#define G3L_DIV_SDHI0		DDIV_PACK(G3L_CPG_SDHI_DDIV, 0, 2)
-#define G3L_DIV_SDHI1		DDIV_PACK(G3L_CPG_SDHI_DDIV, 4, 2)
-#define G3L_DIV_SDHI2		DDIV_PACK(G3L_CPG_SDHI_DDIV, 8, 2)
-#define G3L_DIV_GE3D		DDIV_PACK(G3L_CPG_GE3D_DDIV, 0, 3)
+#define G3L_DIVPL3B		DDIV_PACK(G3L_CPG_PL3_DDIV, 4, 2)
 #define G3L_DIV_CA55_CORE0	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 0, 3)
 #define G3L_DIV_CA55_CORE1	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 4, 3)
 #define G3L_DIV_CA55_CORE2	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 8, 3)
 #define G3L_DIV_CA55_CORE3	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 12, 3)
+#define G3L_DIV_DSI		DDIV_PACK(G3L_CPG_DSI_DDIV, 0, 2)
+#define G3L_DIV_GE3D		DDIV_PACK(G3L_CPG_GE3D_DDIV, 0, 3)
 #define G3L_DIV_RSCI0		DDIV_PACK(G3L_CPG_RSCI_DDIV, 0, 2)
 #define G3L_DIV_RSCI1		DDIV_PACK(G3L_CPG_RSCI_DDIV, 2, 2)
 #define G3L_DIV_RSCI2		DDIV_PACK(G3L_CPG_RSCI_DDIV, 4, 2)
@@ -50,6 +52,10 @@
 #define G3L_DIV_RSPI0		DDIV_PACK(G3L_CPG_RSPI_DDIV, 0, 2)
 #define G3L_DIV_RSPI1		DDIV_PACK(G3L_CPG_RSPI_DDIV, 2, 2)
 #define G3L_DIV_RSPI2		DDIV_PACK(G3L_CPG_RSPI_DDIV, 4, 2)
+#define G3L_DIV_SDHI0		DDIV_PACK(G3L_CPG_SDHI_DDIV, 0, 2)
+#define G3L_DIV_SDHI1		DDIV_PACK(G3L_CPG_SDHI_DDIV, 4, 2)
+#define G3L_DIV_SDHI2		DDIV_PACK(G3L_CPG_SDHI_DDIV, 8, 2)
+#define G3L_SDIV_DSI_C_SET	DDIV_PACK(G3L_CPG_DSI_SDIV, 8, 1)
 #define G3L_SDIV_ETH_A		DDIV_PACK(G3L_CPG_ETH_SDIV, 0, 2)
 #define G3L_SDIV_ETH_B		DDIV_PACK(G3L_CPG_ETH_SDIV, 4, 1)
 #define G3L_SDIV_ETH_C		DDIV_PACK(G3L_CPG_ETH_SDIV, 8, 2)
@@ -59,6 +65,7 @@
 #define G3L_DIVPL2A_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 4, 1)
 #define G3L_DIVPL2B_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 5, 1)
 #define G3L_DIVPL3A_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 8, 1)
+#define G3L_DIVPL3B_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 9, 1)
 #define G3L_DIV_CA55_CORE0_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 12, 1)
 #define G3L_DIV_CA55_CORE1_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 13, 1)
 #define G3L_DIV_CA55_CORE2_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 14, 1)
@@ -73,16 +80,15 @@
 #define G3L_DIV_SDHI0_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 24, 1)
 #define G3L_DIV_SDHI1_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 25, 1)
 #define G3L_DIV_SDHI2_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 26, 1)
+#define G3L_DIV_GE3D_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 27, 1)
+#define G3L_DIV_DSI_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 28, 1)
 
 #define G3L_SEL_SDHI0_STS	SEL_PLL_PACK(G3L_CLKSELSTATUS, 16, 1)
 #define G3L_SEL_SDHI1_STS	SEL_PLL_PACK(G3L_CLKSELSTATUS, 17, 1)
 #define G3L_SEL_SDHI2_STS	SEL_PLL_PACK(G3L_CLKSELSTATUS, 18, 1)
-#define G3L_DIV_GE3D_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 27, 1)
 
 /* RZ/G3L Specific clocks select. */
-#define G3L_SEL_SDHI0		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 0, 2)
-#define G3L_SEL_SDHI1		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 4, 2)
-#define G3L_SEL_SDHI2		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 8, 2)
+#define G3L_SEL_DSI		SEL_PLL_PACK(G3L_CPG_DSI_SSEL, 0, 1)
 #define G3L_SEL_ETH0_TX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 0, 1)
 #define G3L_SEL_ETH0_RX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 1, 1)
 #define G3L_SEL_ETH0_RM		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 2, 1)
@@ -101,6 +107,9 @@
 #define G3L_SEL_RSPI0		SEL_PLL_PACK(G3L_CPG_RSPI_SSEL, 0, 2)
 #define G3L_SEL_RSPI1		SEL_PLL_PACK(G3L_CPG_RSPI_SSEL, 2, 2)
 #define G3L_SEL_RSPI2		SEL_PLL_PACK(G3L_CPG_RSPI_SSEL, 4, 2)
+#define G3L_SEL_SDHI0		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 0, 2)
+#define G3L_SEL_SDHI1		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 4, 2)
+#define G3L_SEL_SDHI2		SEL_PLL_PACK(G3L_CPG_SDHI_DSEL, 8, 2)
 
 enum clk_ids {
 	/* Core Clock Outputs exported to DT */
@@ -126,8 +135,11 @@ enum clk_ids {
 	CLK_PLL3_DIV2,
 	CLK_PLL3_DIV2_2,
 	CLK_PLL3_DIV3,
+	CLK_PLL3_DIV6,
 	CLK_PLL6,
 	CLK_PLL6_DIV10,
+	CLK_PLL7,
+	CLK_PLL7_DSI_SDIV,
 	CLK_SEL_ETH0_TX,
 	CLK_SEL_ETH0_RX,
 	CLK_SEL_ETH0_RM,
@@ -149,6 +161,8 @@ enum clk_ids {
 	CLK_ETH0_RM,
 	CLK_ETH1_TR,
 	CLK_ETH1_RM,
+	CLK_M2_DIV7,
+	CLK_P4_DIV2,
 	CLK_SD0_DIV2,
 	CLK_SD1_DIV2,
 	CLK_SD2_DIV2,
@@ -165,6 +179,12 @@ static const struct clk_div_table dtable_1_4[] = {
 	{ 0, 0 },
 };
 
+static const struct clk_div_table dtable_1_2[] = {
+	{ 0, 1 },
+	{ 1, 2 },
+	{ 0, 0 },
+};
+
 static const struct clk_div_table dtable_1_8[] = {
 	{ 0, 1 },
 	{ 1, 2 },
@@ -220,7 +240,16 @@ static const struct clk_div_table dtable_8_256[] = {
 	{ 0, 0 },
 };
 
+static const struct clk_div_table dtable_16_128[] = {
+	{ 0, 16 },
+	{ 1, 32 },
+	{ 2, 64 },
+	{ 3, 128 },
+	{ 0, 0 },
+};
+
 /* Mux clock names tables. */
+static const char * const sel_dsi[] = { ".m2_div7", ".pll7_dsi_div"};
 static const char * const sel_eth0_tx[] = { ".div_eth0_tr", "eth0_txc_tx_clk" };
 static const char * const sel_eth0_rx[] = { ".div_eth0_tr", "eth0_rxc_rx_clk" };
 static const char * const sel_eth0_rm[] = { ".pll6_div10", "eth0_rxc_rx_clk" };
@@ -254,6 +283,7 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_G3L_PLL(".pll6", CLK_PLL6, CLK_EXTAL, CPG_PLL_CONF(0x50, 0),
 		    500000000UL),
 	DEF_FIXED(".pll1_div2", CLK_PLL1_DIV2, CLK_PLL1, 1, 2),
+	DEF_G3L_PLLDSI(".pll7", CLK_PLL7, CLK_EXTAL, CPG_PLL_CONF(0x80, 0)),
 	DEF_FIXED(".pll2_div2", CLK_PLL2_DIV2, CLK_PLL2, 1, 2),
 	DEF_FIXED(".pll2_div2_4", CLK_PLL2_DIV2_4, CLK_PLL2_DIV2, 1, 4),
 	DEF_FIXED(".pll2_div5", CLK_PLL2_DIV5, CLK_PLL2, 1, 5),
@@ -262,6 +292,7 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_FIXED(".pll3_div2", CLK_PLL3_DIV2, CLK_PLL3, 1, 2),
 	DEF_FIXED(".pll3_div2_2", CLK_PLL3_DIV2_2, CLK_PLL3_DIV2, 1, 2),
 	DEF_FIXED(".pll3_div3", CLK_PLL3_DIV3, CLK_PLL3, 1, 3),
+	DEF_FIXED(".pll3_div6", CLK_PLL3_DIV6, CLK_PLL3, 1, 6),
 	DEF_FIXED(".pll6_div10", CLK_PLL6_DIV10, CLK_PLL6, 1, 10),
 	DEF_SD_MUX(".sel_sdhi0", CLK_SEL_SDHI0, G3L_SEL_SDHI0, G3L_SEL_SDHI0_STS, sel_sdhi,
 		   mtable_sd, 0, NULL),
@@ -277,10 +308,12 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_MUX(".sel_rspi0", CLK_SEL_RSPI0, G3L_SEL_RSPI0, sel_rsci_rspi),
 	DEF_MUX(".sel_rspi1", CLK_SEL_RSPI1, G3L_SEL_RSPI1, sel_rsci_rspi),
 	DEF_MUX(".sel_rspi2", CLK_SEL_RSPI2, G3L_SEL_RSPI2, sel_rsci_rspi),
-	DEF_MUX(".sel_eth0_tx", CLK_SEL_ETH0_TX, G3L_SEL_ETH0_TX, sel_eth0_tx),
+	DEF_MUX_FLAGS(".sel_eth0_tx", CLK_SEL_ETH0_TX, G3L_SEL_ETH0_TX, sel_eth0_tx,
+		      CLK_SET_RATE_PARENT),
 	DEF_MUX(".sel_eth0_rx", CLK_SEL_ETH0_RX, G3L_SEL_ETH0_RX, sel_eth0_rx),
 	DEF_MUX(".sel_eth0_rm", CLK_SEL_ETH0_RM, G3L_SEL_ETH0_RM, sel_eth0_rm),
-	DEF_MUX(".sel_eth1_tx", CLK_SEL_ETH1_TX, G3L_SEL_ETH1_TX, sel_eth1_tx),
+	DEF_MUX_FLAGS(".sel_eth1_tx", CLK_SEL_ETH1_TX, G3L_SEL_ETH1_TX, sel_eth1_tx,
+		      CLK_SET_RATE_PARENT),
 	DEF_MUX(".sel_eth1_rx", CLK_SEL_ETH1_RX, G3L_SEL_ETH1_RX, sel_eth1_rx),
 	DEF_MUX(".sel_eth1_rm", CLK_SEL_ETH1_RM, G3L_SEL_ETH1_RM, sel_eth1_rm),
 	DEF_DIV(".div_eth0_tr", CLK_ETH0_TR, CLK_PLL6, G3L_SDIV_ETH_A, dtable_4_200),
@@ -301,8 +334,13 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 		    dtable_8_256, 0, 0, 0, NULL),
 	DEF_G3S_DIV("P1", R9A08G046_CLK_P1, CLK_PLL3_DIV2, G3L_DIVPL3A, G3L_DIVPL3A_STS,
 		    dtable_4_128, 0, 0, 0, NULL),
+	DEF_G3S_DIV("P2", R9A08G046_CLK_P2, CLK_PLL3_DIV2, G3L_DIVPL3B, G3L_DIVPL3B_STS,
+		    dtable_8_256, 0, 0, 0, NULL),
 	DEF_G3S_DIV("P3", R9A08G046_CLK_P3, CLK_PLL2_DIV2, G3L_DIVPL2A, G3L_DIVPL2A_STS,
 		    dtable_4_128, 0, 0, 0, NULL),
+	DEF_FIXED("P4", R9A08G046_CLK_P4, CLK_PLL2_DIV2, 1, 5),
+	DEF_FIXED(".p4_div2", CLK_P4_DIV2, R9A08G046_CLK_P4, 1, 2),
+	DEF_FIXED("P8", R9A08G046_CLK_P8, CLK_PLL3_DIV2, 1, 8),
 	DEF_G3S_DIV("P13", R9A08G046_CLK_P13, CLK_SEL_RSCI0, G3L_DIV_RSCI0, G3L_DIV_RSCI0_STS,
 		    dtable_2_16, 0, 100000000UL, 0, NULL),
 	DEF_G3S_DIV("P14", R9A08G046_CLK_P14, CLK_SEL_RSCI1, G3L_DIV_RSCI1, G3L_DIV_RSCI1_STS,
@@ -317,6 +355,7 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 		    dtable_1_8, 0, 200000000UL, 0, NULL),
 	DEF_G3S_DIV("P19", R9A08G046_CLK_P19, CLK_SEL_RSPI2, G3L_DIV_RSPI2, G3L_DIV_RSPI2_STS,
 		    dtable_1_8, 0, 200000000UL, 0, NULL),
+	DEF_FIXED("P20", R9A08G046_CLK_P20, CLK_PLL2_DIV2, 1, 8),
 	DEF_G3S_DIV("SD0", R9A08G046_CLK_SD0, CLK_SEL_SDHI0, G3L_DIV_SDHI0, G3L_DIV_SDHI0_STS,
 		    dtable_1_4, 800000000UL, 600000000UL, CLK_SET_RATE_PARENT,
 		    rzg3s_cpg_div_clk_notifier),
@@ -329,6 +368,15 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_FIXED(".sd0_div2", CLK_SD0_DIV2, R9A08G046_CLK_SD0, 1, 2),
 	DEF_FIXED(".sd1_div2", CLK_SD1_DIV2, R9A08G046_CLK_SD1, 1, 2),
 	DEF_FIXED(".sd2_div2", CLK_SD2_DIV2, R9A08G046_CLK_SD2, 1, 2),
+	DEF_G3S_DIV("M1", R9A08G046_CLK_M1, CLK_PLL2_DIV6, G3L_DIV_DSI, G3L_DIV_DSI_STS,
+		    dtable_16_128, 0, 0, 0, NULL),
+	DEF_DIV_FLAGS("M2", R9A08G046_CLK_M2, CLK_PLL7, G3L_SDIV_DSI_C_SET, dtable_1_2,
+		      CLK_SET_RATE_PARENT, CLK_DIVIDER_ROUND_CLOSEST),
+	DEF_G3L_LVDS_DIV(".m2_div7", CLK_M2_DIV7, R9A08G046_CLK_M2, 7),
+	DEF_G3L_DSI_DIV(".pll7_dsi_div", CLK_PLL7_DSI_SDIV, CLK_PLL7, G3L_CPG_DSI_SDIV),
+	DEF_G3L_SEL_DSI_MUX("M3", R9A08G046_CLK_M3, G3L_SEL_DSI, sel_dsi),
+	DEF_FIXED("M4", R9A08G046_CLK_M4, CLK_PLL7, 1, 1),
+	DEF_FIXED("M5", R9A08G046_CLK_M5, CLK_PLL3_DIV6, 1, 2),
 	DEF_FIXED("HP", R9A08G046_CLK_HP, CLK_PLL6_DIV10, 1, 1),
 	DEF_MUX_FLAGS("ETHTX01", R9A08G046_CLK_ETHTX01, G3L_SEL_ETH0_CLK_TX_I, sel_eth0_clk_tx_i,
 		      CLK_SET_RATE_PARENT),
@@ -347,6 +395,7 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_G3S_DIV("G", R9A08G046_CLK_G, CLK_SEL_GE3D, G3L_DIV_GE3D, G3L_DIV_GE3D_STS,
 		    dtable_1_32, 0, 0, 0, NULL),
 	DEF_FIXED("OSCCLK", R9A08G046_OSCCLK, CLK_EXTAL, 1, 1),
+	DEF_FIXED("mipi_dsi_pllclk", R9A08G046_MIPI_DSI_PLLCLK, R9A08G046_CLK_M4, 1, 1),
 };
 
 static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
@@ -400,6 +449,22 @@ static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
 					MSTOP(BUS_PERI_VIDEO, BIT(12))),
 	DEF_MOD("ge3d_ace_clk",		R9A08G046_GE3D_ACE_CLK, R9A08G046_CLK_P1, 0x558, 2,
 					MSTOP(BUS_PERI_VIDEO, BIT(12))),
+	DEF_MOD("mipi_dsi_sysclk",	R9A08G046_MIPI_DSI_SYSCLK, R9A08G046_CLK_M5, 0x568, 1,
+					MSTOP(BUS_PERI_VIDEO, BIT(5) | BIT(6))),
+	DEF_MOD("mipi_dsi_aclk",	R9A08G046_MIPI_DSI_ACLK, R9A08G046_CLK_P1, 0x568, 2,
+					MSTOP(BUS_PERI_VIDEO, BIT(5) | BIT(6))),
+	DEF_MOD("mipi_dsi_pclk",	R9A08G046_MIPI_DSI_PCLK, R9A08G046_CLK_P2, 0x568, 3,
+					MSTOP(BUS_PERI_VIDEO, BIT(5) | BIT(6))),
+	DEF_MOD("mipi_dsi_vclk",	R9A08G046_MIPI_DSI_VCLK, R9A08G046_CLK_M3, 0x568, 4,
+					MSTOP(BUS_PERI_VIDEO, BIT(5) | BIT(6))),
+	DEF_MOD("mipi_dsi_lpclk",	R9A08G046_MIPI_DSI_LPCLK, R9A08G046_CLK_M1, 0x568, 5,
+					MSTOP(BUS_PERI_VIDEO, BIT(5) | BIT(6))),
+	DEF_MOD("lcdc_clk_a",		R9A08G046_LCDC_CLK_A, R9A08G046_CLK_P1, 0x56c, 0,
+					MSTOP(BUS_PERI_VIDEO, BIT(7) | BIT(8) | BIT(9))),
+	DEF_MOD("lcdc_clk_d",		R9A08G046_LCDC_CLK_D, R9A08G046_CLK_M3, 0x56c, 1,
+					MSTOP(BUS_PERI_VIDEO, BIT(7) | BIT(8) | BIT(9))),
+	DEF_MOD("lcdc_clk_p",		R9A08G046_LCDC_CLK_P, R9A08G046_CLK_P2, 0x56c, 2,
+					MSTOP(BUS_PERI_VIDEO, BIT(7) | BIT(8) | BIT(9))),
 	DEF_MOD("ssi0_pclk2",		R9A08G046_SSI0_PCLK2, R9A08G046_CLK_P0,	0x570, 0,
 					MSTOP(BUS_MCPU1, BIT(10))),
 	DEF_MOD("ssi0_pclk_sfr",	R9A08G046_SSI0_PCLK_SFR, R9A08G046_CLK_P0, 0x570, 1,
@@ -498,8 +563,24 @@ static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
 					MSTOP(BUS_MCPU1, BIT(15))),
 	DEF_MOD("rspi2_tclk",		R9A08G046_RSPI2_TCLK, R9A08G046_CLK_P19, 0x590, 10,
 					MSTOP(BUS_MCPU2, BIT(0))),
+	DEF_MOD("canfd_pclk",		R9A08G046_CANFD_PCLK, CLK_P4_DIV2, 0x594, 0,
+					MSTOP(BUS_MCPU2, BIT(9))),
+	DEF_MOD("canfd_clk_ram",	R9A08G046_CANFD_CLK_RAM, R9A08G046_CLK_P4, 0x594, 1,
+					MSTOP(BUS_MCPU2, BIT(9))),
 	DEF_MOD("gpio_hclk",		R9A08G046_GPIO_HCLK, R9A08G046_OSCCLK, 0x598, 0,
 					MSTOP(BUS_PERI_CPU, BIT(6))),
+	DEF_MOD("adc1_adclk",		R9A08G046_ADC1_ADCLK, R9A08G046_CLK_P20, 0x5a8, 2,
+					MSTOP(BUS_PERI_VIDEO, BIT(14))),
+	DEF_MOD("adc1_pclk",		R9A08G046_ADC1_PCLK, R9A08G046_CLK_P20, 0x5a8, 3,
+					MSTOP(BUS_PERI_VIDEO, BIT(14))),
+	DEF_MOD("tsu_pclk",		R9A08G046_TSU_PCLK, R9A08G046_CLK_P20, 0x5ac, 0,
+					MSTOP(BUS_PERI_VIDEO, BIT(15))),
+	DEF_MOD("pci_aclk",		R9A08G046_PCI_ACLK, R9A08G046_CLK_P1, 0x608, 0,
+					MSTOP(BUS_PERI_COM, BIT(10))),
+	DEF_MOD("pci_clkl1pm",		R9A08G046_PCI_CLKL1PM, R9A08G046_CLK_P8, 0x608, 1,
+					MSTOP(BUS_PERI_COM, BIT(10))),
+	DEF_MOD("pci_clk_pmu",		R9A08G046_PCI_CLK_PMU, R9A08G046_CLK_P1, 0x608, 2,
+					MSTOP(BUS_PERI_COM, BIT(10))),
 	DEF_MOD("rsci0_pclk",		R9A08G046_RSCI0_PCLK, R9A08G046_CLK_P0, 0x618, 0,
 					MSTOP(BUS_MCPU2, BIT(7))),
 	DEF_MOD("rsci1_pclk",		R9A08G046_RSCI1_PCLK, R9A08G046_CLK_P0, 0x618, 1,
@@ -516,6 +597,10 @@ static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
 					MSTOP(BUS_MCPU3, BIT(11))),
 	DEF_MOD("rsci3_tclk",		R9A08G046_RSCI3_TCLK, R9A08G046_CLK_P16, 0x618, 11,
 					MSTOP(BUS_MCPU3, BIT(12))),
+	DEF_MOD("lvds_pllclk",		R9A08G046_LVDS_PLLCLK, R9A08G046_CLK_M2, 0x61c, 0,
+					MSTOP(BUS_PERI_VIDEO, BIT(11))),
+	DEF_MOD("lvds_clk_dot0",	R9A08G046_LVDS_CLK_DOT0, R9A08G046_CLK_M3, 0x61c, 1,
+					MSTOP(BUS_PERI_VIDEO, BIT(11))),
 };
 
 static const struct rzg2l_reset r9a08g046_resets[] = {
@@ -537,6 +622,10 @@ static const struct rzg2l_reset r9a08g046_resets[] = {
 	DEF_RST(R9A08G046_GE3D_RESETN, 0x858, 0),
 	DEF_RST(R9A08G046_GE3D_AXI_RESETN, 0x858, 1),
 	DEF_RST(R9A08G046_GE3D_ACE_RESETN, 0x858, 2),
+	DEF_RST(R9A08G046_MIPI_DSI_CMN_RSTB, 0x868, 0),
+	DEF_RST(R9A08G046_MIPI_DSI_ARESET_N, 0x868, 1),
+	DEF_RST(R9A08G046_MIPI_DSI_PRESET_N, 0x868, 2),
+	DEF_RST(R9A08G046_LCDC_RESET_N, 0x86c, 0),
 	DEF_RST(R9A08G046_SSI0_RST_M2_REG, 0x870, 0),
 	DEF_RST(R9A08G046_SSI1_RST_M2_REG, 0x870, 1),
 	DEF_RST(R9A08G046_SSI2_RST_M2_REG, 0x870, 2),
@@ -564,9 +653,15 @@ static const struct rzg2l_reset r9a08g046_resets[] = {
 	DEF_RST(R9A08G046_RSPI0_TRESETN, 0x890, 8),
 	DEF_RST(R9A08G046_RSPI1_TRESETN, 0x890, 9),
 	DEF_RST(R9A08G046_RSPI2_TRESETN, 0x890, 10),
+	DEF_RST(R9A08G046_CANFD_RSTP_N, 0x894, 0),
+	DEF_RST(R9A08G046_CANFD_RSTC_N, 0x894, 1),
 	DEF_RST(R9A08G046_GPIO_RSTN, 0x898, 0),
 	DEF_RST(R9A08G046_GPIO_PORT_RESETN, 0x898, 1),
 	DEF_RST(R9A08G046_GPIO_SPARE_RESETN, 0x898, 2),
+	DEF_RST(R9A08G046_ADC1_PRESETN, 0x8a8, 2),
+	DEF_RST(R9A08G046_ADC1_ADRST_N, 0x8a8, 3),
+	DEF_RST(R9A08G046_TSU_PRESETN, 0x8ac, 0),
+	DEF_RST(R9A08G046_PCI_ARESETN, 0x908, 0),
 	DEF_RST(R9A08G046_RSCI0_PRESETN, 0x918, 0),
 	DEF_RST(R9A08G046_RSCI1_PRESETN, 0x918, 1),
 	DEF_RST(R9A08G046_RSCI2_PRESETN, 0x918, 2),
@@ -575,6 +670,7 @@ static const struct rzg2l_reset r9a08g046_resets[] = {
 	DEF_RST(R9A08G046_RSCI1_TRESETN, 0x918, 9),
 	DEF_RST(R9A08G046_RSCI2_TRESETN, 0x918, 10),
 	DEF_RST(R9A08G046_RSCI3_TRESETN, 0x918, 11),
+	DEF_RST(R9A08G046_LVDS_RESET_N, 0x91c, 0),
 };
 
 static const unsigned int r9a08g046_crit_mod_clks[] __initconst = {
@@ -604,6 +700,7 @@ static const unsigned int r9a08g046_no_pm_mod_clks[] = {
 	MOD_CLK_BASE + R9A08G046_ETH1_CLK_RMII_I,
 	MOD_CLK_BASE + R9A08G046_ETH1_CLK_TX_I_RMII,
 	MOD_CLK_BASE + R9A08G046_ETH1_CLK_RX_I_RMII,
+	MOD_CLK_BASE + R9A08G046_PCI_CLKL1PM,
 };
 
 const struct rzg2l_cpg_info r9a08g046_cpg_info = {
diff --git a/drivers/clk/renesas/r9a09g077-cpg.c b/drivers/clk/renesas/r9a09g077-cpg.c
index 5640c2035e5a40..5a0e7df1eb06c3 100644
--- a/drivers/clk/renesas/r9a09g077-cpg.c
+++ b/drivers/clk/renesas/r9a09g077-cpg.c
@@ -7,6 +7,8 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
 #include <linux/clk-provider.h>
 #include <linux/clk/renesas.h>
 #include <linux/device.h>
@@ -15,7 +17,11 @@
 #include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/math.h>
+#include <linux/mfd/syscon.h>
 #include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/units.h>
 
@@ -87,6 +93,20 @@ MODULE_IMPORT_NS("RZV2H_CPG");
 #define CPG_PLL_MON(x)		((x) - 0x10)
 #define CPG_PLL_MON_LOCK	BIT(0)
 
+/* Size of each mapped CPG/MSSR register region (base0, base1) */
+#define RZT2H_CPG_REG_SIZE		0x20000
+/* This is the size of two SYSC regions combined */
+#define RZT2H_SYSC_SIZE			0x20000
+#define RZT2H_SYSC_OFFSET		0x10000
+#define RZT2H_SYSC_BLOCK_MASK		BIT(16)
+#define RZT2H_SYSC_OFFSET_MASK		GENMASK(15, 0)
+#define RZT2H_SYSC_BLOCK(x)		FIELD_GET(RZT2H_SYSC_BLOCK_MASK, x)
+#define RZT2H_SYSC_REG_OFFSET(x)	FIELD_GET(RZT2H_SYSC_OFFSET_MASK, x)
+
+struct r9a09g077_sysc_reg {
+	void __iomem *base[2];
+};
+
 enum rzt2h_clk_types {
 	CLK_TYPE_RZT2H_DIV = CLK_TYPE_CUSTOM,	/* Clock with divider */
 	CLK_TYPE_RZT2H_MUX,			/* Clock with clock source selector */
@@ -875,6 +895,126 @@ r9a09g077_cpg_clk_register(struct device *dev, const struct cpg_core_clk *core,
 	}
 }
 
+static int r9a09g077_regmap_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct r9a09g077_sysc_reg *sysc = context;
+	void __iomem *base = sysc->base[RZT2H_SYSC_BLOCK(reg)];
+
+	*val = readl(base + RZT2H_SYSC_REG_OFFSET(reg));
+
+	return 0;
+}
+
+static int r9a09g077_regmap_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct r9a09g077_sysc_reg *sysc = context;
+	void __iomem *base = sysc->base[RZT2H_SYSC_BLOCK(reg)];
+
+	writel(val, base + RZT2H_SYSC_REG_OFFSET(reg));
+
+	return 0;
+}
+
+static const struct regmap_bus r9a09g077_sys_regmap_bus = {
+	.reg_write = r9a09g077_regmap_write,
+	.reg_read = r9a09g077_regmap_read,
+};
+
+static bool r9a09g077_writeable_readable_sysc(struct device *dev, unsigned int reg)
+{
+	unsigned int offset = RZT2H_SYSC_REG_OFFSET(reg);
+
+	switch (RZT2H_SYSC_BLOCK(reg)) {
+	case 0:
+		switch (offset) {
+		/* ELOPA/B and GTIOCSEL */
+		case 0x0000 ... 0x0008:
+		/* Encoder config */
+		case 0x1000 ... 0x1164:
+		/* PCIe config */
+		case 0x2000 ... 0x2024:
+		case 0x2030 ... 0x2054:
+		case 0x2060:
+		/* xSPI config */
+		case 0x3000 ... 0x300C:
+		case 0x3100 ... 0x310C:
+		/* MD_MON */
+		case 0x4100:
+		/* PRCRN */
+		case 0x4200:
+			return true;
+
+		default:
+			return false;
+		}
+	case 1:
+		switch (offset) {
+		/* WDTDCRm */
+		case 0x5100 ... 0x5114:
+		/* PRCRS */
+		case 0x6000:
+			return true;
+
+		default:
+			return false;
+		}
+	default:
+		return false;
+	}
+}
+
+static int r9a09g077_post_init(struct device *dev, struct cpg_mssr_pub *pub)
+{
+	struct regmap_config *regmap_cfg __free(kfree) = kzalloc_obj(*regmap_cfg);
+	struct r9a09g077_sysc_reg *sysc_reg;
+	struct regmap *regmap;
+	struct resource res;
+	int ret;
+
+	/*
+	 * Return early if the SYSC sizes are not as expected, for backwards
+	 * compatibility with older device trees that only expose the 64 KiB
+	 * CPG register window.
+	 */
+	ret = of_address_to_resource(dev->of_node, 0, &res);
+	if (ret)
+		return ret;
+	if (resource_size(&res) != RZT2H_CPG_REG_SIZE)
+		return 0;
+
+	ret = of_address_to_resource(dev->of_node, 1, &res);
+	if (ret)
+		return ret;
+	if (resource_size(&res) != RZT2H_CPG_REG_SIZE)
+		return 0;
+
+	if (!regmap_cfg)
+		return -ENOMEM;
+
+	sysc_reg = devm_kzalloc(dev, sizeof(*sysc_reg), GFP_KERNEL);
+	if (!sysc_reg)
+		return -ENOMEM;
+
+	/* Only allow access in the SYSC regions */
+	sysc_reg->base[0] = pub->base0 + RZT2H_SYSC_OFFSET;
+	sysc_reg->base[1] = pub->base1 + RZT2H_SYSC_OFFSET;
+
+	regmap_cfg->name = "rzt2h_sysc";
+	regmap_cfg->reg_bits = 32;
+	regmap_cfg->reg_stride = 4;
+	regmap_cfg->val_bits = 32;
+	regmap_cfg->fast_io = true;
+	regmap_cfg->max_register = RZT2H_SYSC_SIZE - 1;
+	regmap_cfg->readable_reg = r9a09g077_writeable_readable_sysc;
+	regmap_cfg->writeable_reg = r9a09g077_writeable_readable_sysc;
+
+	regmap = devm_regmap_init(dev, &r9a09g077_sys_regmap_bus, sysc_reg, regmap_cfg);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	return of_syscon_register_regmap(dev->of_node, regmap);
+}
+
 const struct cpg_mssr_info r9a09g077_cpg_mssr_info = {
 	/* Core Clocks */
 	.core_clks = r9a09g077_core_clks,
@@ -889,4 +1029,6 @@ const struct cpg_mssr_info r9a09g077_cpg_mssr_info = {
 
 	.reg_layout = CLK_REG_LAYOUT_RZ_T2H,
 	.cpg_clk_register = r9a09g077_cpg_clk_register,
+
+	.post_init = r9a09g077_post_init,
 };
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c
index e5b503332217dc..bbb76e4fa33355 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.c
+++ b/drivers/clk/renesas/rcar-gen3-cpg.c
@@ -36,7 +36,11 @@
 
 #define CPG_PLLnCR_STC_MASK	GENMASK(30, 24)	/* PLL Circuit Mult. Ratio */
 
-#define CPG_RCKCR_CKSEL	BIT(15)	/* RCLK Clock Source Select */
+#define CPG_RPCCKCR		0x238	/* RPC-IF Clock Frequency Control Reg */
+#define CPG_RPCCKCR_DIV_RPCSRC	GENMASK(4, 3)	/* RPCSRC divider */
+
+#define CPG_RCKCR_CKSEL		BIT(15)		/* RCLK Clock Source Select */
+#define CPG_RCKCR_DIV		GENMASK(5, 0)	/* RCLK Division Ratio */
 
 /* PLL Clocks */
 struct cpg_pll_clk {
@@ -395,7 +399,7 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 		 * the multiplier value.
 		 */
 		value = readl(base + CPG_PLL4CR);
-		mult = (((value >> 24) & 0x7f) + 1) * 2;
+		mult = (FIELD_GET(CPG_PLLnCR_STC_MASK, value) + 1) * 2;
 		break;
 
 	case CLK_TYPE_GEN3_SDH:
@@ -420,7 +424,7 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 			 * RINT is default.
 			 * Only if EXTALR is populated, we switch to it.
 			 */
-			value = readl(csn->reg) & 0x3f;
+			value = readl(csn->reg) & CPG_RCKCR_DIV;
 
 			if (clk_get_rate(clks[cpg_clk_extalr])) {
 				parent = clks[cpg_clk_extalr];
@@ -496,9 +500,8 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 		 * MD[4:1] pins and CPG_RPCCKCR[4:3] register value for
 		 * which has been set prior to booting the kernel.
 		 */
-		value = (readl(base + CPG_RPCCKCR) & GENMASK(4, 3)) >> 3;
-
-		switch (value) {
+		value = readl(base + CPG_RPCCKCR);
+		switch (FIELD_GET(CPG_RPCCKCR_DIV_RPCSRC, value)) {
 		case 0:
 			div = 5;
 			break;
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.h b/drivers/clk/renesas/rcar-gen3-cpg.h
index d15a5d1df71c78..fa25911b993edf 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.h
+++ b/drivers/clk/renesas/rcar-gen3-cpg.h
@@ -76,8 +76,7 @@ struct rcar_gen3_cpg_pll_config {
 	u8 osc_prediv;
 };
 
-#define CPG_RPCCKCR	0x238
-#define CPG_RCKCR	0x240
+#define CPG_RCKCR		0x240	/* RCLK Frequency Control Register */
 
 struct clk *rcar_gen3_cpg_clk_register(struct device *dev,
 	const struct cpg_core_clk *core, const struct cpg_mssr_info *info,
diff --git a/drivers/clk/renesas/rcar-gen4-cpg.c b/drivers/clk/renesas/rcar-gen4-cpg.c
index 96922789e02ad9..dd973e4feb1b24 100644
--- a/drivers/clk/renesas/rcar-gen4-cpg.c
+++ b/drivers/clk/renesas/rcar-gen4-cpg.c
@@ -259,10 +259,7 @@ static struct clk * __init cpg_pll_clk_register(const char *name,
 /*
  * Z0, Z1 and ZG Clock
  */
-#define CPG_FRQCRB			0x00000804
 #define CPG_FRQCRB_KICK			BIT(31)
-#define CPG_FRQCRC0			0x00000808
-#define CPG_FRQCRC1			0x000008e0
 
 struct cpg_z_clk {
 	struct clk_hw hw;
@@ -384,17 +381,17 @@ static struct clk * __init cpg_z_clk_register(const char *name,
 	init.parent_names = &parent_name;
 	init.num_parents = 1;
 
-	if (offset < 32) {
-		zclk->reg = reg + CPG_FRQCRC0;
-	} else if (offset < 64) {
-		zclk->reg = reg + CPG_FRQCRC1;
-		offset -= 32;
-	} else if (offset < 96) {
-		zclk->reg = reg + CPG_FRQCRB;
-		offset -= 64;
-	} else {
-		return ERR_PTR(-EINVAL);
-	}
+	/*
+	 * offset 0xAAABB
+	 * AAA : reg
+	 * BB  : pos
+	 *
+	 * see
+	 *	DEF_GEN4_Z()
+	 */
+	zclk->reg = reg + (offset >> FRQCR_offset);
+	offset &= 0x1f; /* 0 - 31 */
+
 	zclk->kick_reg = reg + CPG_FRQCRB;
 	zclk->hw.init = &init;
 	zclk->mask = GENMASK(offset + 4, offset);
diff --git a/drivers/clk/renesas/rcar-gen4-cpg.h b/drivers/clk/renesas/rcar-gen4-cpg.h
index 6c8280b37c37bc..3556cf31ee8f03 100644
--- a/drivers/clk/renesas/rcar-gen4-cpg.h
+++ b/drivers/clk/renesas/rcar-gen4-cpg.h
@@ -58,8 +58,15 @@ enum rcar_gen4_clk_types {
 #define DEF_GEN4_PLL_V9_24(_name, _idx, _id, _parent)	\
 	DEF_BASE(_name, _id, CLK_TYPE_GEN4_PLL_V9_24, _parent, .offset = _idx)
 
-#define DEF_GEN4_Z(_name, _id, _type, _parent, _div, _offset)	\
-	DEF_BASE(_name, _id, _type, _parent, .div = _div, .offset = _offset)
+/*
+ * offset 0xAAABB
+ * AAA : reg
+ * BB  : pos
+ */
+#define FRQCR_offset	8
+#define DEF_GEN4_Z(_name, _id, _parent, _div, reg, pos)	\
+	DEF_BASE(_name, _id, CLK_TYPE_GEN4_Z, _parent, .div = _div,	\
+		 .offset = ((reg) << FRQCR_offset) | (pos))
 
 struct rcar_gen4_cpg_pll_config {
 	u8 extal_div;
@@ -70,6 +77,9 @@ struct rcar_gen4_cpg_pll_config {
 	u8 osc_prediv;
 };
 
+#define CPG_FRQCRB	0x804	/* Frequency Control Register B */
+#define CPG_FRQCRC0	0x808	/* Frequency Control Register C0 */
+#define CPG_FRQCRC1	0x8e0	/* Frequency Control Register C1 */
 #define CPG_SD0CKCR	0x870	/* SD-IF0 Clock Frequency Control Register */
 #define CPG_CANFDCKCR	0x878	/* CAN-FD Clock Frequency Control Register */
 #define CPG_MSOCKCR	0x87c	/* MSIOF Clock Frequency Control Register */
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index 80f4403ea2bafc..ffac4409a16184 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -944,6 +944,12 @@ static const struct of_device_id cpg_mssr_match[] = {
 		.data = &r8a774a1_cpg_mssr_info,
 	},
 #endif
+#ifdef CONFIG_CLK_R8A774A3
+	{
+		.compatible = "renesas,r8a774a3-cpg-mssr",
+		.data = &r8a774a3_cpg_mssr_info,
+	},
+#endif
 #ifdef CONFIG_CLK_R8A774B1
 	{
 		.compatible = "renesas,r8a774b1-cpg-mssr",
@@ -1414,6 +1420,9 @@ static int __init cpg_mssr_probe(struct platform_device *pdev)
 
 	error = cpg_mssr_reset_controller_register(priv);
 
+	if (!error && info->post_init)
+		error = info->post_init(priv->dev, &priv->pub);
+
 reserve_exit:
 	cpg_mssr_reserved_exit(priv);
 
@@ -1428,12 +1437,7 @@ static struct platform_driver cpg_mssr_driver = {
 	},
 };
 
-static int __init cpg_mssr_init(void)
-{
-	return platform_driver_probe(&cpg_mssr_driver, cpg_mssr_probe);
-}
-
-subsys_initcall(cpg_mssr_init);
+subsys_platform_driver_probe(cpg_mssr_driver, cpg_mssr_probe);
 
 void __init mssr_mod_nullify(struct mssr_mod_clk *mod_clks,
 			     unsigned int num_mod_clks,
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h
index ad11ab5f006947..dc3015b3032d4d 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.h
+++ b/drivers/clk/renesas/renesas-cpg-mssr.h
@@ -178,6 +178,7 @@ struct cpg_mssr_info {
 
 	/* Callbacks */
 	int (*init)(struct device *dev);
+	int (*post_init)(struct device *dev, struct cpg_mssr_pub *pub);
 	struct clk *(*cpg_clk_register)(struct device *dev,
 					const struct cpg_core_clk *core,
 					const struct cpg_mssr_info *info,
@@ -190,6 +191,7 @@ extern const struct cpg_mssr_info r8a7743_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a7745_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a77470_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a774a1_cpg_mssr_info;
+extern const struct cpg_mssr_info r8a774a3_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a774b1_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a774c0_cpg_mssr_info;
 extern const struct cpg_mssr_info r8a774e1_cpg_mssr_info;
diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c
index a899c37dd4cd2a..f6293c968bcf9a 100644
--- a/drivers/clk/renesas/rzg2l-cpg.c
+++ b/drivers/clk/renesas/rzg2l-cpg.c
@@ -67,6 +67,17 @@
 #define CPG_PLL_MON_LOCK		BIT(4)
 #define CPG_PLL_MON_RESETB		BIT(0)
 
+#define DIV_DSI_A_SET    GENMASK(2, 0)
+#define DIV_DSI_B_SET    GENMASK(7, 4)
+
+#define RZG3L_SDIV_DIV_DSI_A_WEN	BIT(16)
+#define RZG3L_SDIV_DIV_DSI_B_WEN	BIT(20)
+
+#define CPG_PLL_CLK1_DIV_NF	GENMASK(12, 1)
+#define CPG_PLL_CLK1_DIV_NI	GENMASK(21, 13)
+#define CPG_PLL_CLK1_DIV_M	GENMASK(25, 22)
+#define CPG_PLL_CLK1_DIV_P	GENMASK(28, 26)
+
 #define CLK_ON_R(reg)		(reg)
 #define CLK_MON_R(reg)		(0x180 + (reg))
 #define CLK_RST_R(reg)		(reg)
@@ -92,6 +103,26 @@
 #define PLL5_HSCLK_MIN		10000000
 #define PLL5_HSCLK_MAX		187500000
 
+#define RZG3L_OSC_CLK			(24 * MEGA)
+#define RZG3L_PLL7_FDCO_RANGE_0_MIN	(900 * MEGA)
+#define RZG3L_PLL7_FDCO_RANGE_0_MAX	(2000 * MEGA)
+#define RZG3L_PLL7_FDCO_RANGE_1_MIN	(2000 * MEGA)
+#define RZG3L_PLL7_FDCO_RANGE_1_MAX	(3000ULL * MEGA)
+#define RZG3L_PLL7_PR_MIN		(0)
+#define RZG3L_PLL7_PR_MAX		(4)
+#define RZG3L_PLL7_MR_MIN		(0)
+#define RZG3L_PLL7_MR_MAX		(11)
+#define RZG3L_PLL7_NIR_MIN		(55)
+#define RZG3L_PLL7_NIR_MAX		(374)
+#define RZG3L_PLL7_NFR_MIN		(0)
+#define RZG3L_PLL7_NFR_MAX		(4095)
+#define RZG3L_PLL7_NR_MIN		(56250)	 /* Multiplied value 56.25 * 1000 */
+#define RZG3L_PLL7_NR_MAX		(375000) /* Multiplied value 375 * 1000 */
+#define RZG3L_PLL7_MULT_MIN		(293)	 /* Multiplied value 0.293 * 1000 */
+#define RZG3L_PLL7_MULT_MAX		(375000) /* Multiplied value 375 * 1000 */
+#define RZG3L_PLL7_FSTD_DIV_MR_MIN	(8 * MEGA)
+#define RZG3L_PLL7_FSTD_DIV_MR_MAX	(16 * MEGA)
+
 /**
  * struct clk_hw_data - clock hardware data
  * @hw: clock hw
@@ -402,7 +433,7 @@ rzg3s_cpg_div_clk_register(const struct cpg_core_clk *core, struct rzg2l_cpg_pri
 		return ERR_PTR(-ENOMEM);
 
 	init.name = core->name;
-	init.flags = core->flag;
+	init.flags = core->core_flags;
 	init.ops = &rzg3s_div_clk_ops;
 	init.parent_names = &parent_name;
 	init.num_parents = 1;
@@ -456,20 +487,20 @@ rzg2l_cpg_div_clk_register(const struct cpg_core_clk *core,
 
 	if (core->dtable)
 		clk_hw = clk_hw_register_divider_table(dev, core->name,
-						       parent_name, 0,
+						       parent_name, core->core_flags,
 						       base + GET_REG_OFFSET(core->conf),
 						       GET_SHIFT(core->conf),
 						       GET_WIDTH(core->conf),
-						       core->flag,
+						       core->div_flags,
 						       core->dtable,
 						       &priv->rmw_lock);
 	else
 		clk_hw = clk_hw_register_divider(dev, core->name,
-						 parent_name, 0,
+						 parent_name, core->core_flags,
 						 base + GET_REG_OFFSET(core->conf),
 						 GET_SHIFT(core->conf),
 						 GET_WIDTH(core->conf),
-						 core->flag, &priv->rmw_lock);
+						 core->div_flags, &priv->rmw_lock);
 
 	if (IS_ERR(clk_hw))
 		return ERR_CAST(clk_hw);
@@ -485,7 +516,7 @@ rzg2l_cpg_mux_clk_register(const struct cpg_core_clk *core,
 
 	clk_hw = devm_clk_hw_register_mux(priv->dev, core->name,
 					  core->parent_names, core->num_parents,
-					  core->flag,
+					  core->core_flags,
 					  priv->base + GET_REG_OFFSET(core->conf),
 					  GET_SHIFT(core->conf),
 					  GET_WIDTH(core->conf),
@@ -549,7 +580,7 @@ rzg2l_cpg_sd_mux_clk_register(const struct cpg_core_clk *core,
 			      struct rzg2l_cpg_priv *priv)
 {
 	struct sd_mux_hw_data *sd_mux_hw_data;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_hw *clk_hw;
 	int ret;
 
@@ -564,7 +595,7 @@ rzg2l_cpg_sd_mux_clk_register(const struct cpg_core_clk *core,
 
 	init.name = core->name;
 	init.ops = &rzg2l_cpg_sd_clk_mux_ops;
-	init.flags = core->flag;
+	init.flags = core->core_flags;
 	init.num_parents = core->num_parents;
 	init.parent_names = core->parent_names;
 
@@ -800,9 +831,9 @@ rzg2l_cpg_dsi_div_clk_register(const struct cpg_core_clk *core,
 			       struct rzg2l_cpg_priv *priv)
 {
 	struct dsi_div_hw_data *clk_hw_data;
+	struct clk_init_data init = {};
 	const struct clk *parent;
 	const char *parent_name;
-	struct clk_init_data init;
 	struct clk_hw *clk_hw;
 	int ret;
 
@@ -833,6 +864,207 @@ rzg2l_cpg_dsi_div_clk_register(const struct cpg_core_clk *core,
 	return clk_hw->clk;
 }
 
+static int rzg3l_cpg_lvds_div_determine_rate(struct clk_hw *hw,
+					     struct clk_rate_request *req)
+{
+	return clk_fixed_factor_ops.determine_rate(hw, req);
+}
+
+static int rzg3l_cpg_lvds_div_set_rate(struct clk_hw *hw, unsigned long rate,
+				       unsigned long parent_rate)
+{
+	return clk_fixed_factor_ops.set_rate(hw, rate, parent_rate);
+}
+
+static unsigned long rzg3l_cpg_lvds_div_recalc_rate(struct clk_hw *hw,
+						    unsigned long parent_rate)
+{
+	return clk_fixed_factor_ops.recalc_rate(hw, parent_rate);
+}
+
+static unsigned long rzg3l_cpg_lvds_div_recalc_accuracy(struct clk_hw *hw,
+							unsigned long parent_accuracy)
+{
+	return clk_fixed_factor_ops.recalc_accuracy(hw, parent_accuracy);
+}
+
+static int rzg3l_cpg_lvds_div_get_duty_cycle(struct clk_hw *hw,
+					     struct clk_duty *duty)
+{
+	struct clk_fixed_factor *fix = to_clk_fixed_factor(hw);
+
+	/*
+	 * An odd divider cannot generate a 50% duty cycle: the output stays
+	 * high for (div + 1) / 2 input clock cycles out of div, e.g. a divider
+	 * of 7 gives a duty cycle of 4/7.
+	 */
+	duty->num = DIV_ROUND_UP(fix->div, 2);
+	duty->den = fix->div;
+
+	return 0;
+}
+
+static const struct clk_ops rzg3l_cpg_lvds_div_ops = {
+	.determine_rate	= rzg3l_cpg_lvds_div_determine_rate,
+	.set_rate	= rzg3l_cpg_lvds_div_set_rate,
+	.recalc_rate	= rzg3l_cpg_lvds_div_recalc_rate,
+	.recalc_accuracy = rzg3l_cpg_lvds_div_recalc_accuracy,
+	.get_duty_cycle	= rzg3l_cpg_lvds_div_get_duty_cycle,
+};
+
+static struct clk * __init
+rzg3l_cpg_lvds_div_clk_register(const struct cpg_core_clk *core,
+				struct rzg2l_cpg_priv *priv)
+{
+	struct clk_init_data init = {};
+	struct clk_fixed_factor *ff;
+	const struct clk *parent;
+	const char *parent_name;
+	int ret;
+
+	parent = priv->clks[core->parent];
+	if (IS_ERR(parent))
+		return ERR_CAST(parent);
+
+	ff = devm_kzalloc(priv->dev, sizeof(*ff), GFP_KERNEL);
+	if (!ff)
+		return ERR_PTR(-ENOMEM);
+
+	parent_name = __clk_get_name(parent);
+	init.name = core->name;
+	init.ops = &rzg3l_cpg_lvds_div_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	ff->hw.init = &init;
+	ff->mult = 1;
+	ff->div = core->conf;
+
+	ret = devm_clk_hw_register(priv->dev, &ff->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return ff->hw.clk;
+}
+
+struct g3l_dsi_div_hw_data {
+	struct clk_hw hw;
+	struct rzg2l_cpg_priv *priv;
+	unsigned long rate;
+	u32 off;
+	u8 div_a;
+	u8 div_b;
+};
+
+#define to_g3l_dsi_div_hw_data(_hw)	container_of(_hw, struct g3l_dsi_div_hw_data, hw)
+
+static unsigned long rzg3l_cpg_dsi_div_recalc_rate(struct clk_hw *hw,
+						   unsigned long parent_rate)
+{
+	struct g3l_dsi_div_hw_data *dsi_div = to_g3l_dsi_div_hw_data(hw);
+	struct rzg2l_cpg_priv *priv = dsi_div->priv;
+	int div_a, div_b, val;
+
+	val = readl(priv->base + dsi_div->off);
+	div_a = FIELD_GET(DIV_DSI_A_SET, val);
+	div_b = FIELD_GET(DIV_DSI_B_SET, val);
+
+	return DIV_ROUND_CLOSEST_ULL((u64)parent_rate, (div_b + 1) << div_a);
+}
+
+static int rzg3l_cpg_dsi_div_determine_rate(struct clk_hw *hw,
+					    struct clk_rate_request *req)
+{
+	struct g3l_dsi_div_hw_data *dsi_div = to_g3l_dsi_div_hw_data(hw);
+	struct rzg2l_cpg_priv *priv = dsi_div->priv;
+	unsigned int divider = dsi_div_ab_desired;
+	unsigned int div_b_max = dsi_div_target ? 15 : 0;
+	bool divider_found = false;
+
+	for (unsigned int i = 0; i <= 5 && !divider_found; i++) {
+		unsigned int div_a = 5 - i;
+
+		for (unsigned int div_b = 0; div_b <= div_b_max; div_b++) {
+			divider = (div_b + 1) << div_a;
+			if (divider == dsi_div_ab_desired) {
+				dsi_div->div_a = div_a;
+				dsi_div->div_b = div_b;
+				divider_found = true;
+				break;
+			}
+		}
+	}
+
+	if (!divider_found) {
+		dev_err(priv->dev, "failed dsi div for: %u\n", divider);
+		return -EINVAL;
+	}
+
+	req->best_parent_rate = req->rate * divider;
+
+	return 0;
+}
+
+static int rzg3l_cpg_dsi_div_set_rate(struct clk_hw *hw, unsigned long rate,
+				      unsigned long parent_rate)
+{
+	struct g3l_dsi_div_hw_data *dsi_div = to_g3l_dsi_div_hw_data(hw);
+	struct rzg2l_cpg_priv *priv = dsi_div->priv;
+
+	writel(RZG3L_SDIV_DIV_DSI_A_WEN | RZG3L_SDIV_DIV_DSI_B_WEN |
+	       FIELD_PREP(DIV_DSI_A_SET, dsi_div->div_a) |
+	       FIELD_PREP(DIV_DSI_B_SET, dsi_div->div_b),
+	       priv->base + dsi_div->off);
+
+	return 0;
+}
+
+static const struct clk_ops rzg3l_cpg_dsi_div_ops = {
+	.recalc_rate = rzg3l_cpg_dsi_div_recalc_rate,
+	.determine_rate = rzg3l_cpg_dsi_div_determine_rate,
+	.set_rate = rzg3l_cpg_dsi_div_set_rate,
+};
+
+static struct clk * __init
+rzg3l_cpg_dsi_div_clk_register(const struct cpg_core_clk *core,
+			       struct rzg2l_cpg_priv *priv)
+{
+	struct g3l_dsi_div_hw_data *clk_hw_data;
+	struct clk_init_data init = {};
+	const struct clk *parent;
+	const char *parent_name;
+	struct clk_hw *clk_hw;
+	int ret;
+
+	parent = priv->clks[core->parent];
+	if (IS_ERR(parent))
+		return ERR_CAST(parent);
+
+	clk_hw_data = devm_kzalloc(priv->dev, sizeof(*clk_hw_data), GFP_KERNEL);
+	if (!clk_hw_data)
+		return ERR_PTR(-ENOMEM);
+
+	clk_hw_data->priv = priv;
+	clk_hw_data->off = core->conf;
+
+	parent_name = __clk_get_name(parent);
+	init.name = core->name;
+	init.ops = &rzg3l_cpg_dsi_div_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	clk_hw = &clk_hw_data->hw;
+	clk_hw->init = &init;
+
+	ret = devm_clk_hw_register(priv->dev, clk_hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return clk_hw->clk;
+}
+
 struct pll5_mux_hw_data {
 	struct clk_hw hw;
 	u32 conf;
@@ -896,7 +1128,7 @@ rzg2l_cpg_pll5_4_mux_clk_register(const struct cpg_core_clk *core,
 				  struct rzg2l_cpg_priv *priv)
 {
 	struct pll5_mux_hw_data *clk_hw_data;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_hw *clk_hw;
 	int ret;
 
@@ -1023,8 +1255,8 @@ static struct clk * __init
 rzg2l_cpg_sipll5_register(const struct cpg_core_clk *core,
 			  struct rzg2l_cpg_priv *priv)
 {
+	struct clk_init_data init = {};
 	const struct clk *parent;
-	struct clk_init_data init;
 	const char *parent_name;
 	struct sipll5 *sipll5;
 	struct clk_hw *clk_hw;
@@ -1064,6 +1296,15 @@ rzg2l_cpg_sipll5_register(const struct cpg_core_clk *core,
 	return clk_hw->clk;
 }
 
+/* RZ/G3L PLL7 parameters */
+struct rzg3l_plldsi_parameters {
+	s64 error_millihz;
+	u64 freq_millihz;
+	u16 nir, nfr;
+	u8 rangesel;
+	u8 pr, mr;
+};
+
 struct pll_clk {
 	struct clk_hw hw;
 	unsigned long default_rate;
@@ -1071,6 +1312,7 @@ struct pll_clk {
 	unsigned int type;
 	void __iomem *base;
 	struct rzg2l_cpg_priv *priv;
+	struct rzg3l_plldsi_parameters pll7_dsi_params;
 };
 
 #define to_pll(_hw)	container_of(_hw, struct pll_clk, hw)
@@ -1139,9 +1381,9 @@ rzg2l_cpg_pll_clk_register(const struct cpg_core_clk *core,
 			   struct rzg2l_cpg_priv *priv,
 			   const struct clk_ops *ops)
 {
+	struct clk_init_data init = {};
 	struct device *dev = priv->dev;
 	const struct clk *parent;
-	struct clk_init_data init;
 	const char *parent_name;
 	struct pll_clk *pll_clk;
 	int ret;
@@ -1230,6 +1472,280 @@ static const struct clk_ops rzg3l_cpg_pll_ops = {
 	.recalc_rate = rzg3s_cpg_pll_clk_recalc_rate,
 };
 
+static u8 rzg3l_cpg_dsi_smux_get_parent(struct clk_hw *hw)
+{
+	return clk_mux_ops.get_parent(hw);
+}
+
+static int rzg3l_cpg_dsi_smux_set_parent(struct clk_hw *hw, u8 index)
+{
+	return clk_mux_ops.set_parent(hw, index);
+}
+
+static int rzg3l_cpg_dsi_smux_determine_rate(struct clk_hw *hw,
+					     struct clk_rate_request *req)
+{
+	return clk_mux_ops.determine_rate(hw, req);
+}
+
+static int rzg3l_cpg_dsi_smux_set_duty_cycle(struct clk_hw *hw,
+					     struct clk_duty *duty)
+{
+	struct clk_hw *parent_hw;
+	u8 parent_idx;
+
+	/*
+	 * Select parent based on requested duty cycle:
+	 * - If duty > 50% (num/den > 1/2), select LVDS path (parent 0)
+	 * - Otherwise, select DSI/RGB path (parent 1)
+	 */
+	if (duty->num * 2 > duty->den)
+		parent_idx = 0;
+	else
+		parent_idx = 1;
+
+	if (parent_idx >= clk_hw_get_num_parents(hw))
+		return -EINVAL;
+
+	parent_hw = clk_hw_get_parent_by_index(hw, parent_idx);
+	if (!parent_hw)
+		return -EINVAL;
+
+	return clk_hw_set_parent(hw, parent_hw);
+}
+
+static const struct clk_ops rzg3l_cpg_dsi_smux_ops = {
+	.determine_rate = rzg3l_cpg_dsi_smux_determine_rate,
+	.get_parent = rzg3l_cpg_dsi_smux_get_parent,
+	.set_parent = rzg3l_cpg_dsi_smux_set_parent,
+	.set_duty_cycle = rzg3l_cpg_dsi_smux_set_duty_cycle,
+};
+
+static struct clk * __init
+rzg3l_cpg_dsi_mux_clk_register(const struct cpg_core_clk *core,
+			       struct rzg2l_cpg_priv *priv)
+{
+	struct clk_init_data init = {};
+	struct clk_mux *mux_data;
+	int ret;
+
+	mux_data = devm_kzalloc(priv->dev, sizeof(*mux_data), GFP_KERNEL);
+	if (!mux_data)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = core->name;
+	init.ops = &rzg3l_cpg_dsi_smux_ops;
+	init.flags = core->core_flags | CLK_SET_RATE_PARENT;
+	init.parent_names = core->parent_names;
+	init.num_parents = core->num_parents;
+
+	mux_data->reg = priv->base + GET_REG_OFFSET(core->conf);
+	mux_data->shift = GET_SHIFT(core->conf);
+	mux_data->mask = clk_div_mask(GET_WIDTH(core->conf));
+	mux_data->flags = core->mux_flags;
+	mux_data->lock = &priv->rmw_lock;
+	mux_data->hw.init = &init;
+
+	ret = devm_clk_hw_register(priv->dev, &mux_data->hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return mux_data->hw.clk;
+}
+
+static inline bool
+rzg3l_dsi_compute_pll_parameters(struct rzg3l_plldsi_parameters *pars,
+				 struct rzg3l_plldsi_parameters *p,
+				 struct rzg3l_plldsi_parameters *best,
+				 u64 freq_millihz, u32 fpfd, u32 pr)
+{
+	for (p->nir = RZG3L_PLL7_NIR_MIN; p->nir <= RZG3L_PLL7_NIR_MAX; p->nir++) {
+		u64 output_nir, output_nfr_range;
+		s64 nfr, output_nfr;
+		u64 fdco, output;
+		u64 nr_div_mr_pr;
+
+		/*
+		 * The frequency generated by the PLL is calculated as follows:
+		 *
+		 * With:
+		 * Freq = Ffout = Ffdco / pr
+		 * input frequency(fstd) = 24MHz
+		 * fpfd = fstd / mr
+		 * nr = nir + nfr / 4096
+		 * Ffdco = nr * fpfd
+		 * Ffdco = (nir + (nfr / 4096)) * fpfd
+		 *
+		 * Freq can also be rewritten as:
+		 * Freq = Ffdco / pr
+		 *      = (nir * fpfd) / pr + ((nfr / 4096) * fpfd) / pr
+		 *      = output_nir + output_nfr
+		 *
+		 * Every parameter has been determined at this point, but nfr.
+		 * Considering that:
+		 * 0 <= nfr <= 4095
+		 * Then:
+		 * 0 <= (nfr / 4096) < 1
+		 * Therefore:
+		 * 0 <= output_nfr < fpfd / pr
+		 */
+
+		/* Compute output nir component (in mHz) */
+		output_nir = DIV_ROUND_CLOSEST_ULL(mul_u32_u32((p->nir + 1) * MILLI, fpfd), pr);
+		/* Compute range for output nfr (in mHz) */
+		output_nfr_range = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(fpfd, MILLI), pr);
+		/* No point in continuing if we can't achieve the desired frequency */
+		if (freq_millihz < output_nir  || freq_millihz >= (output_nir + output_nfr_range))
+			continue;
+
+		/*
+		 * Compute the nfr component
+		 *
+		 * Since:
+		 * Freq = output_nir + output_nfr
+		 * Then:
+		 * output_nfr = Freq - output_nir
+		 *            = ((nfr / 4096) * fpfd) / pr
+		 * Therefore:
+		 * nfr = (output_nfr * 4096 * pr) / fpfd
+		 */
+		output_nfr = freq_millihz - output_nir;
+		nfr = div_s64(output_nfr * 4096 * pr, fpfd);
+		nfr = DIV_S64_ROUND_CLOSEST(nfr, 1000);
+
+		/* Validate nfr value within allowed limits */
+		if (nfr < RZG3L_PLL7_NFR_MIN || nfr > RZG3L_PLL7_NFR_MAX)
+			continue;
+
+		p->nfr = nfr;
+
+		/* Compute (Ffdco * 4096) */
+		output = mul_u32_u32(p->nir + 1, 4096);
+		fdco = (output + p->nfr) * fpfd;
+		if (fdco < (RZG3L_PLL7_FDCO_RANGE_0_MIN * 4096ULL) ||
+		    fdco > (RZG3L_PLL7_FDCO_RANGE_1_MAX * 4096ULL))
+			continue;
+
+		if (fdco <= (RZG3L_PLL7_FDCO_RANGE_0_MAX * 4096ULL))
+			p->rangesel = 0;
+		else
+			p->rangesel = 1;
+
+		/* compute the nr and magnify by 1000 */
+		output = mul_u32_u32(p->nir + 1, 4096);
+		output += p->nfr;
+		output *= 1000;
+		nr_div_mr_pr = div_u64(output, 4096);
+		if (nr_div_mr_pr < RZG3L_PLL7_NR_MIN || nr_div_mr_pr > RZG3L_PLL7_NR_MAX)
+			continue;
+
+		/* compute the magnified multipier = nr(magnified)/(mr *pr)  */
+		nr_div_mr_pr = div_u64(nr_div_mr_pr, (p->mr + 1) * pr);
+		if (nr_div_mr_pr < RZG3L_PLL7_MULT_MIN || nr_div_mr_pr > RZG3L_PLL7_MULT_MAX)
+			continue;
+
+		output *= RZG3L_OSC_CLK;
+		output = div_u64(output, (p->mr + 1) * pr * 4096);
+
+		p->error_millihz = freq_millihz - output;
+		p->freq_millihz = output;
+
+		/* If an exact match is found, return immediately */
+		if (p->error_millihz == 0) {
+			*pars = *p;
+			return true;
+		}
+
+		/* Update best match if error is smaller */
+		if (abs(p->error_millihz) < abs(best->error_millihz))
+			*best = *p;
+	}
+
+	return false;
+}
+
+static bool
+rzg3l_dsi_get_pll_parameters_values(struct rzg3l_plldsi_parameters *pars,
+				    u64 freq_millihz)
+{
+	struct rzg3l_plldsi_parameters p, best;
+
+	/* Initialize best error to maximum possible value */
+	best.error_millihz = S64_MAX;
+	p.error_millihz = S64_MAX;
+	for (p.mr = RZG3L_PLL7_MR_MIN; p.mr <= RZG3L_PLL7_MR_MAX; p.mr++) {
+		u32 fpfd = RZG3L_OSC_CLK / (p.mr + 1);
+
+		if (fpfd > RZG3L_PLL7_FSTD_DIV_MR_MAX || fpfd < RZG3L_PLL7_FSTD_DIV_MR_MIN)
+			continue;
+
+		for (p.pr = RZG3L_PLL7_PR_MIN; p.pr <= RZG3L_PLL7_PR_MAX; p.pr++)
+			if (rzg3l_dsi_compute_pll_parameters(pars, &p, &best,
+							     freq_millihz, fpfd, 1 << p.pr))
+				return true;
+	}
+
+	/* If no valid parameters were found, return false */
+	if (best.error_millihz == S64_MAX)
+		return false;
+
+	*pars = best;
+	return true;
+}
+
+static int rzg3l_cpg_plldsi_determine_rate(struct clk_hw *hw, struct clk_rate_request *req)
+{
+	struct pll_clk *pll_clk = to_pll(hw);
+	struct rzg2l_cpg_priv *priv = pll_clk->priv;
+	struct rzg3l_plldsi_parameters *pll7_dsi_params = &pll_clk->pll7_dsi_params;
+	u64 rate_millihz;
+
+	rate_millihz = mul_u32_u32(req->rate, MILLI);
+	if (rate_millihz == pll7_dsi_params->error_millihz + pll7_dsi_params->freq_millihz)
+		goto exit_determine_rate;
+
+	if (!rzg3l_dsi_get_pll_parameters_values(pll7_dsi_params, rate_millihz)) {
+		dev_err(priv->dev, "failed %s for req->rate: %lu\n", __func__, req->rate);
+		return -EINVAL;
+	}
+
+exit_determine_rate:
+	req->rate = DIV_ROUND_CLOSEST_ULL(pll7_dsi_params->freq_millihz, MILLI);
+	return 0;
+}
+
+static int rzg3l_cpg_plldsi_set_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long parent_rate)
+{
+	struct pll_clk *pll_clk = to_pll(hw);
+	struct rzg3l_plldsi_parameters *dsi_dividers = &pll_clk->pll7_dsi_params;
+	u32 val;
+
+	/* Put PLL into standby mode */
+	rzg3l_cpg_pll_clk_endisable(hw, false);
+
+	/* Output clock setting 1 */
+	val = FIELD_PREP(CPG_PLL_CLK1_DIV_P, dsi_dividers->pr) |
+	      FIELD_PREP(CPG_PLL_CLK1_DIV_M, dsi_dividers->mr) |
+	      FIELD_PREP(CPG_PLL_CLK1_DIV_NI, dsi_dividers->nir) |
+	      FIELD_PREP(CPG_PLL_CLK1_DIV_NF, dsi_dividers->nfr) |
+	      dsi_dividers->rangesel;
+	writel(val, pll_clk->base + CPG_PLL_CLK1_OFFSET(pll_clk->conf));
+
+	/* Put PLL to normal mode */
+	rzg3l_cpg_pll_clk_endisable(hw, true);
+
+	return 0;
+};
+
+static const struct clk_ops rzg3l_cpg_plldsi_ops = {
+	.recalc_rate = rzg3s_cpg_pll_clk_recalc_rate,
+	.determine_rate = rzg3l_cpg_plldsi_determine_rate,
+	.set_rate = rzg3l_cpg_plldsi_set_rate,
+	.is_enabled = rzg3l_cpg_pll_clk_is_enabled,
+	.enable = rzg3l_cpg_pll_clk_enable,
+};
+
 static struct clk
 *rzg2l_cpg_clk_src_twocell_get(struct of_phandle_args *clkspec,
 			       void *data)
@@ -1340,6 +1856,18 @@ rzg2l_cpg_register_core_clk(const struct cpg_core_clk *core,
 	case CLK_TYPE_DSI_DIV:
 		clk = rzg2l_cpg_dsi_div_clk_register(core, priv);
 		break;
+	case CLK_TYPE_G3L_DSI_DIV:
+		clk = rzg3l_cpg_dsi_div_clk_register(core, priv);
+		break;
+	case CLK_TYPE_G3L_LVDS_DIV:
+		clk = rzg3l_cpg_lvds_div_clk_register(core, priv);
+		break;
+	case CLK_TYPE_G3L_PLLDSI:
+		clk = rzg2l_cpg_pll_clk_register(core, priv, &rzg3l_cpg_plldsi_ops);
+		break;
+	case CLK_TYPE_G3L_DSI_MUX:
+		clk = rzg3l_cpg_dsi_mux_clk_register(core, priv);
+		break;
 	default:
 		goto fail;
 	}
@@ -1736,10 +2264,10 @@ rzg2l_cpg_register_mod_clk(const struct rzg2l_mod_clk *mod,
 			   const struct rzg2l_cpg_info *info,
 			   struct rzg2l_cpg_priv *priv)
 {
+	struct clk_init_data init = {};
 	struct mod_clock *clock = NULL;
 	struct device *dev = priv->dev;
 	unsigned int id = mod->id;
-	struct clk_init_data init;
 	struct clk *parent, *clk;
 	const char *parent_name;
 	unsigned int i;
@@ -2230,11 +2758,6 @@ static struct platform_driver rzg2l_cpg_driver = {
 	},
 };
 
-static int __init rzg2l_cpg_init(void)
-{
-	return platform_driver_probe(&rzg2l_cpg_driver, rzg2l_cpg_probe);
-}
-
-subsys_initcall(rzg2l_cpg_init);
+subsys_platform_driver_probe(rzg2l_cpg_driver, rzg2l_cpg_probe);
 
 MODULE_DESCRIPTION("Renesas RZ/G2L CPG Driver");
diff --git a/drivers/clk/renesas/rzg2l-cpg.h b/drivers/clk/renesas/rzg2l-cpg.h
index bd6169f625386a..81296046e35318 100644
--- a/drivers/clk/renesas/rzg2l-cpg.h
+++ b/drivers/clk/renesas/rzg2l-cpg.h
@@ -110,8 +110,9 @@ struct cpg_core_clk {
 	};
 	const char * const *parent_names;
 	notifier_fn_t notifier;
-	u32 flag;
+	u32 core_flags;
 	u32 mux_flags;
+	u32 div_flags;
 	int num_parents;
 };
 
@@ -142,6 +143,17 @@ enum clk_types {
 	/* Clock for DSI divider */
 	CLK_TYPE_DSI_DIV,
 
+	/* Clock for G3L DSI PLL */
+	CLK_TYPE_G3L_PLLDSI,
+
+	/* Clock for G3L DSI divider */
+	CLK_TYPE_G3L_DSI_DIV,
+
+	/* Clock for G3L LVDS divider */
+	CLK_TYPE_G3L_LVDS_DIV,
+
+	/* Clock for G3L DSI clock source selector */
+	CLK_TYPE_G3L_DSI_MUX,
 };
 
 #define DEF_TYPE(_name, _id, _type...) \
@@ -160,27 +172,31 @@ enum clk_types {
 	DEF_TYPE(_name, _id, CLK_TYPE_IN)
 #define DEF_FIXED(_name, _id, _parent, _mult, _div) \
 	DEF_BASE(_name, _id, CLK_TYPE_FF, _parent, .div = _div, .mult = _mult)
-#define DEF_DIV(_name, _id, _parent, _conf, _dtable) \
+#define DEF_DIV_FLAGS(_name, _id, _parent, _conf, _dtable, _flags, _div_flags) \
 	DEF_TYPE(_name, _id, CLK_TYPE_DIV, .conf = _conf, \
 		 .parent = _parent, .dtable = _dtable, \
-		 .flag = CLK_DIVIDER_HIWORD_MASK)
+		 .core_flags = _flags, \
+		 .div_flags = CLK_DIVIDER_HIWORD_MASK | (_div_flags))
+#define DEF_DIV(_name, _id, _parent, _conf, _dtable) \
+	DEF_DIV_FLAGS(_name, _id, _parent, _conf, _dtable, 0, 0)
 #define DEF_DIV_RO(_name, _id, _parent, _conf, _dtable) \
 	DEF_TYPE(_name, _id, CLK_TYPE_DIV, .conf = _conf, \
 		 .parent = _parent, .dtable = _dtable, \
-		 .flag = CLK_DIVIDER_READ_ONLY)
+		 .core_flags = 0, \
+		 .div_flags = CLK_DIVIDER_READ_ONLY)
 #define DEF_G3S_DIV(_name, _id, _parent, _conf, _sconf, _dtable, _invalid_rate, \
 		    _max_rate, _clk_flags, _notif) \
 	DEF_TYPE(_name, _id, CLK_TYPE_G3S_DIV, .conf = _conf, .sconf = _sconf, \
 		 .parent = _parent, .dtable = _dtable, \
 		 .invalid_rate = _invalid_rate, \
-		 .max_rate = _max_rate, .flag = (_clk_flags), \
+		 .max_rate = _max_rate, .core_flags = (_clk_flags), \
 		 .notifier = _notif)
-#define DEF_MUX_FLAGS(_name, _id, _conf, _parent_names, _flag) \
+#define DEF_MUX_FLAGS(_name, _id, _conf, _parent_names, _flags) \
 	DEF_TYPE(_name, _id, CLK_TYPE_MUX, .conf = _conf, \
 		 .parent_names = _parent_names, \
 		 .num_parents = ARRAY_SIZE(_parent_names), \
 		 .mux_flags = CLK_MUX_HIWORD_MASK, \
-		 .flag = _flag)
+		 .core_flags = _flags)
 #define DEF_MUX(_name, _id, _conf, _parent_names) \
 	DEF_MUX_FLAGS(_name, _id, _conf, _parent_names, 0)
 #define DEF_MUX_RO(_name, _id, _conf, _parent_names) \
@@ -188,19 +204,34 @@ enum clk_types {
 		 .parent_names = _parent_names, \
 		 .num_parents = ARRAY_SIZE(_parent_names), \
 		 .mux_flags = CLK_MUX_READ_ONLY)
+#define DEF_G3L_SEL_DSI_MUX(_name, _id, _conf, _parent_names) \
+	DEF_TYPE(_name, _id, CLK_TYPE_G3L_DSI_MUX, .conf = _conf, \
+		 .parent_names = _parent_names, \
+		 .num_parents = ARRAY_SIZE(_parent_names), \
+		 .core_flags = CLK_SET_RATE_PARENT | CLK_DUTY_CYCLE_PARENT | \
+			       CLK_SET_RATE_NO_REPARENT, \
+		 .mux_flags = CLK_MUX_HIWORD_MASK)
 #define DEF_SD_MUX(_name, _id, _conf, _sconf, _parent_names, _mtable, _clk_flags, _notifier) \
 	DEF_TYPE(_name, _id, CLK_TYPE_SD_MUX, .conf = _conf, .sconf = _sconf, \
 		 .parent_names = _parent_names, \
 		 .num_parents = ARRAY_SIZE(_parent_names), \
-		 .mtable = _mtable, .flag = _clk_flags, .notifier = _notifier)
+		 .mtable = _mtable, .core_flags = _clk_flags, .notifier = _notifier)
 #define DEF_PLL5_FOUTPOSTDIV(_name, _id, _parent) \
 	DEF_TYPE(_name, _id, CLK_TYPE_SIPLL5, .parent = _parent)
 #define DEF_PLL5_4_MUX(_name, _id, _conf, _parent_names) \
 	DEF_TYPE(_name, _id, CLK_TYPE_PLL5_4_MUX, .conf = _conf, \
 		 .parent_names = _parent_names, \
 		 .num_parents = ARRAY_SIZE(_parent_names))
-#define DEF_DSI_DIV(_name, _id, _parent, _flag) \
-	DEF_TYPE(_name, _id, CLK_TYPE_DSI_DIV, .parent = _parent, .flag = _flag)
+#define DEF_DSI_DIV(_name, _id, _parent, _flags) \
+	DEF_TYPE(_name, _id, CLK_TYPE_DSI_DIV, .parent = _parent, .core_flags = _flags)
+#define DEF_G3L_DSI_DIV(_name, _id, _parent, _conf) \
+	DEF_TYPE(_name, _id, CLK_TYPE_G3L_DSI_DIV, .parent = _parent, .conf = _conf, \
+		 .core_flags = CLK_SET_RATE_PARENT)
+#define DEF_G3L_LVDS_DIV(_name, _id, _parent, _conf) \
+	DEF_TYPE(_name, _id, CLK_TYPE_G3L_LVDS_DIV, .parent = _parent, .conf = _conf, \
+		 .core_flags = CLK_SET_RATE_PARENT)
+#define DEF_G3L_PLLDSI(_name, _id, _parent, _conf) \
+	DEF_TYPE(_name, _id, CLK_TYPE_G3L_PLLDSI, .parent = _parent, .conf = _conf)
 
 /**
  * struct rzg2l_mod_clk - Module Clocks definitions
diff --git a/drivers/clk/renesas/rzv2h-cpg.c b/drivers/clk/renesas/rzv2h-cpg.c
index 917fc737f254b9..7ce94680c79a29 100644
--- a/drivers/clk/renesas/rzv2h-cpg.c
+++ b/drivers/clk/renesas/rzv2h-cpg.c
@@ -99,7 +99,7 @@ struct rzv2h_pll_dsi_info {
  * @dev: CPG device
  * @base: CPG register block base address
  * @rmw_lock: protects register accesses
- * @clks: Array containing all Core and Module Clocks
+ * @clks: Array containing all Core and Module clk_hw entries
  * @num_core_clks: Number of Core Clocks in clks[]
  * @num_mod_clks: Number of Module Clocks in clks[]
  * @resets: Array of resets
@@ -115,7 +115,7 @@ struct rzv2h_cpg_priv {
 	void __iomem *base;
 	spinlock_t rmw_lock;
 
-	struct clk **clks;
+	struct clk_hw **clks;
 	unsigned int num_core_clks;
 	unsigned int num_mod_clks;
 	struct rzv2h_reset *resets;
@@ -332,14 +332,14 @@ static const struct clk_ops rzv2h_cpg_plldsi_div_ops = {
 	.set_rate = rzv2h_cpg_plldsi_div_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 rzv2h_cpg_plldsi_div_clk_register(const struct cpg_core_clk *core,
 				  struct rzv2h_cpg_priv *priv)
 {
 	struct rzv2h_plldsi_div_clk *clk_hw_data;
-	struct clk **clks = priv->clks;
-	struct clk_init_data init;
-	const struct clk *parent;
+	struct clk_hw **clks = priv->clks;
+	struct clk_init_data init = {};
+	const struct clk_hw *parent;
 	const char *parent_name;
 	struct clk_hw *clk_hw;
 	int ret;
@@ -356,7 +356,7 @@ rzv2h_cpg_plldsi_div_clk_register(const struct cpg_core_clk *core,
 	clk_hw_data->ddiv = core->cfg.ddiv;
 	clk_hw_data->dtable = core->dtable;
 
-	parent_name = __clk_get_name(parent);
+	parent_name = clk_hw_get_name(parent);
 	init.name = core->name;
 	init.ops = &rzv2h_cpg_plldsi_div_ops;
 	init.flags = core->flag;
@@ -370,7 +370,7 @@ rzv2h_cpg_plldsi_div_clk_register(const struct cpg_core_clk *core,
 	if (ret)
 		return ERR_PTR(ret);
 
-	return clk_hw->clk;
+	return clk_hw;
 }
 
 static int rzv2h_cpg_plldsi_determine_rate(struct clk_hw *hw,
@@ -577,12 +577,12 @@ static const struct clk_ops rzv2h_cpg_plldsi_smux_ops = {
 	.set_duty_cycle = rzv2h_cpg_plldsi_smux_set_duty_cycle,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 rzv2h_cpg_plldsi_smux_clk_register(const struct cpg_core_clk *core,
 				   struct rzv2h_cpg_priv *priv)
 {
 	struct rzv2h_plldsi_mux_clk *clk_hw_data;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_hw *clk_hw;
 	struct smuxed smux;
 	int ret;
@@ -620,7 +620,7 @@ rzv2h_cpg_plldsi_smux_clk_register(const struct cpg_core_clk *core,
 	if (ret)
 		return ERR_PTR(ret);
 
-	return clk_hw->clk;
+	return clk_hw;
 }
 
 static int rzv2h_cpg_pll_clk_is_enabled(struct clk_hw *hw)
@@ -705,14 +705,14 @@ static const struct clk_ops rzv2h_cpg_pll_ops = {
 	.recalc_rate = rzv2h_cpg_pll_clk_recalc_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 rzv2h_cpg_pll_clk_register(const struct cpg_core_clk *core,
 			   struct rzv2h_cpg_priv *priv,
 			   const struct clk_ops *ops)
 {
 	struct device *dev = priv->dev;
-	struct clk_init_data init;
-	const struct clk *parent;
+	struct clk_init_data init = {};
+	const struct clk_hw *parent;
 	const char *parent_name;
 	struct pll_clk *pll_clk;
 	int ret;
@@ -729,7 +729,7 @@ rzv2h_cpg_pll_clk_register(const struct cpg_core_clk *core,
 		priv->pll_dsi_info[core->cfg.pll.instance].pll_dsi_limits =
 			core->cfg.pll.limits;
 
-	parent_name = __clk_get_name(parent);
+	parent_name = clk_hw_get_name(parent);
 	init.name = core->name;
 	init.ops = ops;
 	init.flags = 0;
@@ -744,7 +744,7 @@ rzv2h_cpg_pll_clk_register(const struct cpg_core_clk *core,
 	if (ret)
 		return ERR_PTR(ret);
 
-	return pll_clk->hw.clk;
+	return &pll_clk->hw;
 }
 
 static unsigned long rzv2h_ddiv_recalc_rate(struct clk_hw *hw,
@@ -820,7 +820,7 @@ static const struct clk_ops rzv2h_ddiv_clk_divider_ops = {
 	.set_rate = rzv2h_ddiv_set_rate,
 };
 
-static struct clk * __init
+static struct clk_hw * __init
 rzv2h_cpg_ddiv_clk_register(const struct cpg_core_clk *core,
 			    struct rzv2h_cpg_priv *priv)
 {
@@ -829,7 +829,7 @@ rzv2h_cpg_ddiv_clk_register(const struct cpg_core_clk *core,
 	struct device *dev = priv->dev;
 	u8 shift = cfg_ddiv.shift;
 	u8 width = cfg_ddiv.width;
-	const struct clk *parent;
+	const struct clk_hw *parent;
 	const char *parent_name;
 	struct clk_divider *div;
 	struct ddiv_clk *ddiv;
@@ -839,7 +839,7 @@ rzv2h_cpg_ddiv_clk_register(const struct cpg_core_clk *core,
 	if (IS_ERR(parent))
 		return ERR_CAST(parent);
 
-	parent_name = __clk_get_name(parent);
+	parent_name = clk_hw_get_name(parent);
 
 	if ((shift + width) > 16)
 		return ERR_PTR(-EINVAL);
@@ -872,15 +872,15 @@ rzv2h_cpg_ddiv_clk_register(const struct cpg_core_clk *core,
 	if (ret)
 		return ERR_PTR(ret);
 
-	return div->hw.clk;
+	return &div->hw;
 }
 
-static struct clk * __init
+static struct clk_hw * __init
 rzv2h_cpg_mux_clk_register(const struct cpg_core_clk *core,
 			   struct rzv2h_cpg_priv *priv)
 {
 	struct smuxed mux = core->cfg.smux;
-	const struct clk_hw *clk_hw;
+	struct clk_hw *clk_hw;
 
 	clk_hw = devm_clk_hw_register_mux(priv->dev, core->name,
 					  core->parent_names, core->num_parents,
@@ -890,7 +890,7 @@ rzv2h_cpg_mux_clk_register(const struct cpg_core_clk *core,
 	if (IS_ERR(clk_hw))
 		return ERR_CAST(clk_hw);
 
-	return clk_hw->clk;
+	return clk_hw;
 }
 
 static int
@@ -906,14 +906,14 @@ rzv2h_clk_ff_mod_status_is_enabled(struct clk_hw *hw)
 	return !!(val & bitmask);
 }
 
-static struct clk * __init
+static struct clk_hw * __init
 rzv2h_cpg_fixed_mod_status_clk_register(const struct cpg_core_clk *core,
 					struct rzv2h_cpg_priv *priv)
 {
 	struct rzv2h_ff_mod_status_clk *clk_hw_data;
 	struct clk_init_data init = { };
 	struct clk_fixed_factor *fix;
-	const struct clk *parent;
+	const struct clk_hw *parent;
 	const char *parent_name;
 	int ret;
 
@@ -922,10 +922,7 @@ rzv2h_cpg_fixed_mod_status_clk_register(const struct cpg_core_clk *core,
 	if (IS_ERR(parent))
 		return ERR_CAST(parent);
 
-	parent_name = __clk_get_name(parent);
-	parent = priv->clks[core->parent];
-	if (IS_ERR(parent))
-		return ERR_CAST(parent);
+	parent_name = clk_hw_get_name(parent);
 
 	clk_hw_data = devm_kzalloc(priv->dev, sizeof(*clk_hw_data), GFP_KERNEL);
 	if (!clk_hw_data)
@@ -949,18 +946,18 @@ rzv2h_cpg_fixed_mod_status_clk_register(const struct cpg_core_clk *core,
 	if (ret)
 		return ERR_PTR(ret);
 
-	return clk_hw_data->fix.hw.clk;
+	return &clk_hw_data->fix.hw;
 }
 
-static struct clk
+static struct clk_hw
 *rzv2h_cpg_clk_src_twocell_get(struct of_phandle_args *clkspec,
 			       void *data)
 {
 	unsigned int clkidx = clkspec->args[1];
 	struct rzv2h_cpg_priv *priv = data;
 	struct device *dev = priv->dev;
+	struct clk_hw *clk_hw;
 	const char *type;
-	struct clk *clk;
 
 	switch (clkspec->args[0]) {
 	case CPG_CORE:
@@ -969,7 +966,7 @@ static struct clk
 			dev_err(dev, "Invalid %s clock index %u\n", type, clkidx);
 			return ERR_PTR(-EINVAL);
 		}
-		clk = priv->clks[clkidx];
+		clk_hw = priv->clks[clkidx];
 		break;
 
 	case CPG_MOD:
@@ -978,7 +975,7 @@ static struct clk
 			dev_err(dev, "Invalid %s clock index %u\n", type, clkidx);
 			return ERR_PTR(-EINVAL);
 		}
-		clk = priv->clks[priv->num_core_clks + clkidx];
+		clk_hw = priv->clks[priv->num_core_clks + clkidx];
 		break;
 
 	default:
@@ -986,25 +983,25 @@ static struct clk
 		return ERR_PTR(-EINVAL);
 	}
 
-	if (IS_ERR(clk))
+	if (IS_ERR(clk_hw))
 		dev_err(dev, "Cannot get %s clock %u: %ld\n", type, clkidx,
-			PTR_ERR(clk));
+			PTR_ERR(clk_hw));
 	else
-		dev_dbg(dev, "clock (%u, %u) is %pC at %lu Hz\n",
-			clkspec->args[0], clkspec->args[1], clk,
-			clk_get_rate(clk));
-	return clk;
+		dev_dbg(dev, "clock (%u, %u) is %s at %lu Hz\n",
+			clkspec->args[0], clkspec->args[1],
+			clk_hw_get_name(clk_hw), clk_hw_get_rate(clk_hw));
+	return clk_hw;
 }
 
 static void __init
 rzv2h_cpg_register_core_clk(const struct cpg_core_clk *core,
 			    struct rzv2h_cpg_priv *priv)
 {
-	struct clk *clk = ERR_PTR(-EOPNOTSUPP), *parent;
+	struct clk_hw *clk_hw = ERR_PTR(-EOPNOTSUPP), *parent;
 	unsigned int id = core->id, div = core->div;
 	struct device *dev = priv->dev;
 	const char *parent_name;
-	struct clk_hw *clk_hw;
+	struct clk *clk;
 
 	WARN_DEBUG(id >= priv->num_core_clks);
 	WARN_DEBUG(PTR_ERR(priv->clks[id]) != -ENOENT);
@@ -1012,70 +1009,72 @@ rzv2h_cpg_register_core_clk(const struct cpg_core_clk *core,
 	switch (core->type) {
 	case CLK_TYPE_IN:
 		clk = of_clk_get_by_name(priv->dev->of_node, core->name);
+		if (IS_ERR(clk)) {
+			clk_hw = ERR_CAST(clk);
+			goto fail;
+		}
+		clk_hw = __clk_get_hw(clk);
 		break;
 	case CLK_TYPE_FF:
 		WARN_DEBUG(core->parent >= priv->num_core_clks);
 		parent = priv->clks[core->parent];
 		if (IS_ERR(parent)) {
-			clk = parent;
+			clk_hw = parent;
 			goto fail;
 		}
 
-		parent_name = __clk_get_name(parent);
+		parent_name = clk_hw_get_name(parent);
 		clk_hw = devm_clk_hw_register_fixed_factor(dev, core->name,
 							   parent_name, CLK_SET_RATE_PARENT,
 							   core->mult, div);
-		if (IS_ERR(clk_hw))
-			clk = ERR_CAST(clk_hw);
-		else
-			clk = clk_hw->clk;
 		break;
 	case CLK_TYPE_FF_MOD_STATUS:
 		if (!priv->ff_mod_status_ops) {
 			priv->ff_mod_status_ops =
 				devm_kzalloc(dev, sizeof(*priv->ff_mod_status_ops), GFP_KERNEL);
 			if (!priv->ff_mod_status_ops) {
-				clk = ERR_PTR(-ENOMEM);
+				clk_hw = ERR_PTR(-ENOMEM);
 				goto fail;
 			}
 			memcpy(priv->ff_mod_status_ops, &clk_fixed_factor_ops,
 			       sizeof(const struct clk_ops));
 			priv->ff_mod_status_ops->is_enabled = rzv2h_clk_ff_mod_status_is_enabled;
 		}
-		clk = rzv2h_cpg_fixed_mod_status_clk_register(core, priv);
+		clk_hw = rzv2h_cpg_fixed_mod_status_clk_register(core, priv);
 		break;
 	case CLK_TYPE_PLL:
-		clk = rzv2h_cpg_pll_clk_register(core, priv, &rzv2h_cpg_pll_ops);
+		clk_hw = rzv2h_cpg_pll_clk_register(core, priv, &rzv2h_cpg_pll_ops);
 		break;
 	case CLK_TYPE_DDIV:
-		clk = rzv2h_cpg_ddiv_clk_register(core, priv);
+		clk_hw = rzv2h_cpg_ddiv_clk_register(core, priv);
 		break;
 	case CLK_TYPE_SMUX:
-		clk = rzv2h_cpg_mux_clk_register(core, priv);
+		clk_hw = rzv2h_cpg_mux_clk_register(core, priv);
 		break;
 	case CLK_TYPE_PLLDSI:
-		clk = rzv2h_cpg_pll_clk_register(core, priv, &rzv2h_cpg_plldsi_ops);
+		clk_hw = rzv2h_cpg_pll_clk_register(core, priv, &rzv2h_cpg_plldsi_ops);
 		break;
 	case CLK_TYPE_PLLDSI_DIV:
-		clk = rzv2h_cpg_plldsi_div_clk_register(core, priv);
+		clk_hw = rzv2h_cpg_plldsi_div_clk_register(core, priv);
 		break;
 	case CLK_TYPE_PLLDSI_SMUX:
-		clk = rzv2h_cpg_plldsi_smux_clk_register(core, priv);
+		clk_hw = rzv2h_cpg_plldsi_smux_clk_register(core, priv);
 		break;
 	default:
 		goto fail;
 	}
 
-	if (IS_ERR(clk))
+	if (IS_ERR(clk_hw))
 		goto fail;
 
-	dev_dbg(dev, "Core clock %pC at %lu Hz\n", clk, clk_get_rate(clk));
-	priv->clks[id] = clk;
+	dev_dbg(dev, "Core clock %s at %lu Hz\n",
+		clk_hw_get_name(clk_hw), clk_hw_get_rate(clk_hw));
+	priv->clks[id] = clk_hw;
 	return;
 
 fail:
-	dev_err(dev, "Failed to register core clock %s: %ld\n",
-		core->name, PTR_ERR(clk));
+	dev_err(dev, "Failed to register core clock %s: %ld\n", core->name,
+		PTR_ERR(clk_hw));
 }
 
 static void rzv2h_mod_clock_mstop_enable(struct rzv2h_cpg_priv *priv,
@@ -1122,15 +1121,10 @@ static void rzv2h_mod_clock_mstop_disable(struct rzv2h_cpg_priv *priv,
 
 static int rzv2h_parent_clk_mux_to_index(struct clk_hw *hw)
 {
-	struct clk_hw *parent_hw;
-	struct clk *parent_clk;
+	struct clk_hw *parent_hw = clk_hw_get_parent(hw);
 	struct clk_mux *mux;
 	u32 val;
 
-	/* This will always succeed, so no need to check for IS_ERR() */
-	parent_clk = clk_get_parent(hw->clk);
-
-	parent_hw = __clk_get_hw(parent_clk);
 	mux = to_clk_mux(parent_hw);
 
 	val = readl(mux->reg) >> mux->shift;
@@ -1229,9 +1223,9 @@ rzv2h_cpg_register_mod_clk(const struct rzv2h_mod_clk *mod,
 {
 	struct mod_clock *clock = NULL;
 	struct device *dev = priv->dev;
-	struct clk_init_data init;
-	struct clk *parent, *clk;
+	struct clk_init_data init = {};
 	const char *parent_name;
+	struct clk_hw *parent;
 	unsigned int id;
 	int ret;
 
@@ -1242,13 +1236,13 @@ rzv2h_cpg_register_mod_clk(const struct rzv2h_mod_clk *mod,
 
 	parent = priv->clks[mod->parent];
 	if (IS_ERR(parent)) {
-		clk = parent;
+		ret = PTR_ERR(parent);
 		goto fail;
 	}
 
 	clock = devm_kzalloc(dev, sizeof(*clock), GFP_KERNEL);
 	if (!clock) {
-		clk = ERR_PTR(-ENOMEM);
+		ret = -ENOMEM;
 		goto fail;
 	}
 
@@ -1258,7 +1252,7 @@ rzv2h_cpg_register_mod_clk(const struct rzv2h_mod_clk *mod,
 	if (mod->critical)
 		init.flags |= CLK_IS_CRITICAL;
 
-	parent_name = __clk_get_name(parent);
+	parent_name = clk_hw_get_name(parent);
 	init.parent_names = &parent_name;
 	init.num_parents = 1;
 
@@ -1273,12 +1267,10 @@ rzv2h_cpg_register_mod_clk(const struct rzv2h_mod_clk *mod,
 	clock->mstop_data = mod->mstop_data;
 
 	ret = devm_clk_hw_register(dev, &clock->hw);
-	if (ret) {
-		clk = ERR_PTR(ret);
+	if (ret)
 		goto fail;
-	}
 
-	priv->clks[id] = clock->hw.clk;
+	priv->clks[id] = &clock->hw;
 
 	/*
 	 * Ensure the module clocks and MSTOP bits are synchronized when they are
@@ -1317,8 +1309,8 @@ rzv2h_cpg_register_mod_clk(const struct rzv2h_mod_clk *mod,
 	return;
 
 fail:
-	dev_err(dev, "Failed to register module clock %s: %ld\n",
-		mod->name, PTR_ERR(clk));
+	dev_err(dev, "Failed to register module clock %s: %d\n", mod->name,
+		ret);
 }
 
 static int __rzv2h_cpg_assert(struct reset_controller_dev *rcdev,
@@ -1451,7 +1443,7 @@ static bool rzv2h_cpg_is_pm_clk(struct rzv2h_cpg_pd *pd,
 		if (priv->clks[priv->num_core_clks + id] == ERR_PTR(-ENOENT))
 			return false;
 
-		clock = to_mod_clock(__clk_get_hw(priv->clks[priv->num_core_clks + id]));
+		clock = to_mod_clock(priv->clks[priv->num_core_clks + id]);
 
 		return !clock->no_pm;
 	}
@@ -1550,19 +1542,13 @@ static int __init rzv2h_cpg_add_pm_domains(struct rzv2h_cpg_priv *priv)
 	return of_genpd_add_provider_simple(np, &pd->genpd);
 }
 
-static void rzv2h_cpg_del_clk_provider(void *data)
-{
-	of_clk_del_provider(data);
-}
-
 static int __init rzv2h_cpg_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *np = dev->of_node;
 	const struct rzv2h_cpg_info *info;
 	struct rzv2h_cpg_priv *priv;
 	unsigned int nclks, i;
-	struct clk **clks;
+	struct clk_hw **clks;
 	int error;
 
 	info = of_device_get_match_data(dev);
@@ -1613,11 +1599,7 @@ static int __init rzv2h_cpg_probe(struct platform_device *pdev)
 	for (i = 0; i < info->num_mod_clks; i++)
 		rzv2h_cpg_register_mod_clk(&info->mod_clks[i], priv);
 
-	error = of_clk_add_provider(np, rzv2h_cpg_clk_src_twocell_get, priv);
-	if (error)
-		return error;
-
-	error = devm_add_action_or_reset(dev, rzv2h_cpg_del_clk_provider, np);
+	error = devm_of_clk_add_hw_provider(dev, rzv2h_cpg_clk_src_twocell_get, priv);
 	if (error)
 		return error;
 
@@ -1661,11 +1643,6 @@ static struct platform_driver rzv2h_cpg_driver = {
 	},
 };
 
-static int __init rzv2h_cpg_init(void)
-{
-	return platform_driver_probe(&rzv2h_cpg_driver, rzv2h_cpg_probe);
-}
-
-subsys_initcall(rzv2h_cpg_init);
+subsys_platform_driver_probe(rzv2h_cpg_driver, rzv2h_cpg_probe);
 
 MODULE_DESCRIPTION("Renesas RZ/V2H CPG Driver");
diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c
index 6205af9ba88a8d..eb6b8b878e4dbe 100644
--- a/drivers/clk/rockchip/clk-cpu.c
+++ b/drivers/clk/rockchip/clk-cpu.c
@@ -304,7 +304,7 @@ struct clk *rockchip_clk_register_cpuclk(const char *name,
 			int nrates, void __iomem *reg_base, spinlock_t *lock)
 {
 	struct rockchip_cpuclk *cpuclk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk *clk, *cclk;
 	int ret;
 
diff --git a/drivers/clk/rockchip/clk-ddr.c b/drivers/clk/rockchip/clk-ddr.c
index 81ae8a4ee30ed9..fbb702d0b9d996 100644
--- a/drivers/clk/rockchip/clk-ddr.c
+++ b/drivers/clk/rockchip/clk-ddr.c
@@ -97,7 +97,7 @@ struct clk *rockchip_clk_register_ddrclk(const char *name, int flags,
 					 spinlock_t *lock)
 {
 	struct rockchip_ddrclk *ddrclk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk *clk;
 
 	ddrclk = kzalloc_obj(*ddrclk);
diff --git a/drivers/clk/rockchip/clk-gate-grf.c b/drivers/clk/rockchip/clk-gate-grf.c
index 88e409e91a636a..458314a2569181 100644
--- a/drivers/clk/rockchip/clk-gate-grf.c
+++ b/drivers/clk/rockchip/clk-gate-grf.c
@@ -73,7 +73,7 @@ struct clk *rockchip_clk_register_gate_grf(const char *name,
 		u8 gate_flags)
 {
 	struct rockchip_gate_grf *gate;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk *clk;
 
 	if (IS_ERR(regmap)) {
diff --git a/drivers/clk/rockchip/clk-inverter.c b/drivers/clk/rockchip/clk-inverter.c
index bf9072ed9fd4da..e6c7dad60e9662 100644
--- a/drivers/clk/rockchip/clk-inverter.c
+++ b/drivers/clk/rockchip/clk-inverter.c
@@ -75,8 +75,8 @@ struct clk *rockchip_clk_register_inverter(const char *name,
 				void __iomem *reg, int shift, int flags,
 				spinlock_t *lock)
 {
-	struct clk_init_data init;
 	struct rockchip_inv_clock *inv_clock;
+	struct clk_init_data init = {};
 	struct clk *clk;
 
 	inv_clock = kmalloc_obj(*inv_clock);
diff --git a/drivers/clk/rockchip/clk-mmc-phase.c b/drivers/clk/rockchip/clk-mmc-phase.c
index 4efd22953f3e05..d92bea75faf737 100644
--- a/drivers/clk/rockchip/clk-mmc-phase.c
+++ b/drivers/clk/rockchip/clk-mmc-phase.c
@@ -205,8 +205,8 @@ struct clk *rockchip_clk_register_mmc(const char *name,
 				struct regmap *grf, int grf_reg,
 				int shift)
 {
-	struct clk_init_data init;
 	struct rockchip_mmc_clock *mmc_clock;
+	struct clk_init_data init = {};
 	struct clk *clk;
 	int ret;
 
diff --git a/drivers/clk/rockchip/clk-muxgrf.c b/drivers/clk/rockchip/clk-muxgrf.c
index ccbb1379501ab4..ecdc77654813a3 100644
--- a/drivers/clk/rockchip/clk-muxgrf.c
+++ b/drivers/clk/rockchip/clk-muxgrf.c
@@ -59,7 +59,7 @@ struct clk *rockchip_clk_register_muxgrf(const char *name,
 				int shift, int width, int mux_flags)
 {
 	struct rockchip_muxgrf_clock *muxgrf_clock;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk *clk;
 
 	if (IS_ERR(regmap)) {
diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c
index 706ca4b344d399..7204b09243d483 100644
--- a/drivers/clk/rockchip/clk-pll.c
+++ b/drivers/clk/rockchip/clk-pll.c
@@ -1069,7 +1069,7 @@ struct clk *rockchip_clk_register_pll(struct rockchip_clk_provider *ctx,
 		unsigned long flags, u8 clk_pll_flags)
 {
 	const char *pll_parents[3];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct rockchip_clk_pll *pll;
 	struct clk_mux *pll_mux;
 	struct clk *pll_clk, *mux_clk;
diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
index ee8c79b938d3c1..10df21178ba483 100644
--- a/drivers/clk/rockchip/clk.c
+++ b/drivers/clk/rockchip/clk.c
@@ -257,7 +257,7 @@ static struct clk *rockchip_clk_register_frac_branch(
 
 	if (child) {
 		struct clk_mux *frac_mux = &frac->mux;
-		struct clk_init_data init;
+		struct clk_init_data init = {};
 		struct clk *mux_clk;
 		int ret;
 
diff --git a/drivers/clk/samsung/Makefile b/drivers/clk/samsung/Makefile
index b3c4ef4e0dbff4..b3e994bd525d9d 100644
--- a/drivers/clk/samsung/Makefile
+++ b/drivers/clk/samsung/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_EXYNOS_AUDSS_CLK_CON) += clk-exynos-audss.o
 obj-$(CONFIG_EXYNOS_CLKOUT)	+= clk-exynos-clkout.o
 obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-exynos-arm64.o
 obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-exynos2200.o
+obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-exynos5515.o
 obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-exynos7.o
 obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-exynos7870.o
 obj-$(CONFIG_EXYNOS_ARM64_COMMON_CLK)	+= clk-exynos7885.o
diff --git a/drivers/clk/samsung/clk-acpm.c b/drivers/clk/samsung/clk-acpm.c
index 7680bc8c5275b9..e5a2d3ab34cbd3 100644
--- a/drivers/clk/samsung/clk-acpm.c
+++ b/drivers/clk/samsung/clk-acpm.c
@@ -21,6 +21,7 @@ struct acpm_clk {
 	struct clk_hw hw;
 	unsigned int mbox_chan_id;
 	struct acpm_handle *handle;
+	const struct acpm_clk_driver_data *drv_data;
 };
 
 struct acpm_clk_variant {
@@ -31,6 +32,8 @@ struct acpm_clk_driver_data {
 	const struct acpm_clk_variant *clks;
 	unsigned int nr_clks;
 	unsigned int mbox_chan_id;
+	/* For SoCs where ACPM firmware doesn't implement get_rate call */
+	unsigned long (*get_rate)(struct clk_hw *clk, unsigned long parent_rate);
 };
 
 #define to_acpm_clk(clk) container_of(clk, struct acpm_clk, hw)
@@ -40,6 +43,18 @@ struct acpm_clk_driver_data {
 		.name		= cname,		\
 	}
 
+static const struct acpm_clk_variant exynos850_acpm_clks[] = {
+	ACPM_CLK("mif"),
+	ACPM_CLK("int"),
+	ACPM_CLK("cpucl0"),
+	ACPM_CLK("cpucl1"),
+	ACPM_CLK("g3d"),
+	ACPM_CLK("aud"),
+	ACPM_CLK("cam"),
+	ACPM_CLK("disp"),
+	ACPM_CLK("cp"),
+};
+
 static const struct acpm_clk_variant gs101_acpm_clks[] = {
 	ACPM_CLK("mif"),
 	ACPM_CLK("int"),
@@ -57,6 +72,19 @@ static const struct acpm_clk_variant gs101_acpm_clks[] = {
 	ACPM_CLK("bo"),
 };
 
+static unsigned long bypass_acpm_exynos850_get_rate(struct clk_hw *hw,
+						    unsigned long parent_rate)
+{
+	return parent_rate;
+}
+
+static const struct acpm_clk_driver_data acpm_clk_exynos850 = {
+	.clks = exynos850_acpm_clks,
+	.nr_clks = ARRAY_SIZE(exynos850_acpm_clks),
+	.mbox_chan_id = 0,
+	.get_rate = bypass_acpm_exynos850_get_rate,
+};
+
 static const struct acpm_clk_driver_data acpm_clk_gs101 = {
 	.clks = gs101_acpm_clks,
 	.nr_clks = ARRAY_SIZE(gs101_acpm_clks),
@@ -68,6 +96,9 @@ static unsigned long acpm_clk_recalc_rate(struct clk_hw *hw,
 {
 	struct acpm_clk *clk = to_acpm_clk(hw);
 
+	if (clk->drv_data->get_rate)
+		return clk->drv_data->get_rate(hw, parent_rate);
+
 	return clk->handle->ops->dvfs.get_rate(clk->handle, clk->mbox_chan_id,
 					       clk->id);
 }
@@ -88,19 +119,30 @@ static const struct clk_ops acpm_clk_ops = {
 };
 
 static int acpm_clk_register(struct device *dev, struct acpm_clk *aclk,
-			     const char *name)
+			     const char *name, const struct acpm_clk_driver_data *drv_data)
 {
 	struct clk_init_data init = {};
+	struct clk_parent_data pdata = {};
 
 	init.name = name;
 	init.ops = &acpm_clk_ops;
 	aclk->hw.init = &init;
 
+	/* If get_rate is set, the SoC relies on parent topology */
+	if (drv_data->get_rate) {
+		pdata.fw_name = name;
+		init.parent_data = &pdata;
+		init.num_parents = 1;
+		init.flags = CLK_GET_RATE_NOCACHE;
+	}
+
 	return devm_clk_hw_register(dev, &aclk->hw);
 }
 
 static int acpm_clk_probe(struct platform_device *pdev)
 {
+	const struct acpm_clk_driver_data *drv_data;
+	const struct platform_device_id *id;
 	struct acpm_handle *acpm_handle;
 	struct clk_hw_onecell_data *clk_data;
 	struct clk_hw **hws;
@@ -114,8 +156,14 @@ static int acpm_clk_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(acpm_handle),
 				     "Failed to get acpm handle\n");
 
-	count = acpm_clk_gs101.nr_clks;
-	mbox_chan_id = acpm_clk_gs101.mbox_chan_id;
+	id = platform_get_device_id(pdev);
+	if (!id || !id->driver_data)
+		return -ENODEV;
+
+	drv_data  = (const struct acpm_clk_driver_data *)id->driver_data;
+
+	count = drv_data->nr_clks;
+	mbox_chan_id = drv_data->mbox_chan_id;
 
 	clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, count),
 				GFP_KERNEL);
@@ -136,14 +184,14 @@ static int acpm_clk_probe(struct platform_device *pdev)
 		 * The code assumes the clock IDs start from zero,
 		 * are sequential and do not have gaps.
 		 */
+		aclk->drv_data = drv_data;
 		aclk->id = i;
 		aclk->handle = acpm_handle;
 		aclk->mbox_chan_id = mbox_chan_id;
 
 		hws[i] = &aclk->hw;
 
-		err = acpm_clk_register(dev, aclk,
-					acpm_clk_gs101.clks[i].name);
+		err = acpm_clk_register(dev, aclk, drv_data->clks[i].name, drv_data);
 		if (err)
 			return dev_err_probe(dev, err,
 					     "Failed to register clock\n");
@@ -154,7 +202,8 @@ static int acpm_clk_probe(struct platform_device *pdev)
 }
 
 static const struct platform_device_id acpm_clk_id[] = {
-	{ .name = "gs101-acpm-clk" },
+	{ .name = "exynos850-acpm-clk", (kernel_ulong_t)&acpm_clk_exynos850 },
+	{ .name = "gs101-acpm-clk", (kernel_ulong_t)&acpm_clk_gs101 },
 	{ }
 };
 MODULE_DEVICE_TABLE(platform, acpm_clk_id);
diff --git a/drivers/clk/samsung/clk-cpu.c b/drivers/clk/samsung/clk-cpu.c
index 37cf47533dc610..a17ce26494f3eb 100644
--- a/drivers/clk/samsung/clk-cpu.c
+++ b/drivers/clk/samsung/clk-cpu.c
@@ -647,7 +647,7 @@ static int __init exynos_register_cpu_clock(struct samsung_clk_provider *ctx,
 	const struct clk_hw *parent, *alt_parent;
 	struct clk_hw **hws;
 	struct exynos_cpuclk *cpuclk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *parent_name;
 	unsigned int num_cfgs;
 	int ret = 0;
diff --git a/drivers/clk/samsung/clk-exynos-clkout.c b/drivers/clk/samsung/clk-exynos-clkout.c
index 5f64d93b2facf5..9c8b17cb5e7c12 100644
--- a/drivers/clk/samsung/clk-exynos-clkout.c
+++ b/drivers/clk/samsung/clk-exynos-clkout.c
@@ -214,7 +214,7 @@ static void exynos_clkout_remove(struct platform_device *pdev)
 	iounmap(clkout->reg);
 }
 
-static int __maybe_unused exynos_clkout_suspend(struct device *dev)
+static int exynos_clkout_suspend(struct device *dev)
 {
 	struct exynos_clkout *clkout = dev_get_drvdata(dev);
 
@@ -223,7 +223,7 @@ static int __maybe_unused exynos_clkout_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused exynos_clkout_resume(struct device *dev)
+static int exynos_clkout_resume(struct device *dev)
 {
 	struct exynos_clkout *clkout = dev_get_drvdata(dev);
 
@@ -232,13 +232,13 @@ static int __maybe_unused exynos_clkout_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(exynos_clkout_pm_ops, exynos_clkout_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(exynos_clkout_pm_ops, exynos_clkout_suspend,
 			 exynos_clkout_resume);
 
 static struct platform_driver exynos_clkout_driver = {
 	.driver = {
 		.name = DRV_NAME,
-		.pm = &exynos_clkout_pm_ops,
+		.pm = pm_sleep_ptr(&exynos_clkout_pm_ops),
 	},
 	.probe = exynos_clkout_probe,
 	.remove = exynos_clkout_remove,
diff --git a/drivers/clk/samsung/clk-exynos5515.c b/drivers/clk/samsung/clk-exynos5515.c
new file mode 100644
index 00000000000000..ddcef581903421
--- /dev/null
+++ b/drivers/clk/samsung/clk-exynos5515.c
@@ -0,0 +1,1484 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2026 Aiden Isik <aidenisik+git@member.fsf.org>
+ * Author: Aiden Isik <aidenisik+git@member.fsf.org>
+ *
+ * Common Clock Framework support for Exynos5515 SoC.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include <dt-bindings/clock/samsung,exynos5515-cmu.h>
+
+#include "clk.h"
+#include "clk-exynos-arm64.h"
+#include "clk-pll.h"
+
+/* NOTE: Must be equal to the last clock ID increased by one */
+#define CLKS_NR_TOP	(CLK_GOUT_CMU_PERI_UART + 1)
+#define CLKS_NR_PERI	(CLK_GOUT_PERI_USI00_I2C + 1)
+#define CLKS_NR_FSYS	(CLK_GOUT_FSYS_XIU_P_ACLK + 1)
+
+#define EXYNOS5515_GATE_DBG_OFFSET	0x4000
+#define EXYNOS5515_DRCG_EN_OFFSET	0x0104
+#define EXYNOS5515_MEMCLK_OFFSET	0x0108
+
+/* ---- CMU_TOP ------------------------------------------------------------- */
+
+/*
+ * CMU_TOP:	0x15410000
+ * SYSREG_TOP:	0x15430000
+ */
+
+/* Register Offset definitions for CMU_TOP */
+#define PLL_LOCKTIME_PLL_AUD			0x0000
+#define PLL_LOCKTIME_PLL_SHARED0		0x0004
+#define PLL_LOCKTIME_PLL_SHARED1		0x0008
+#define PLL_CON3_PLL_AUD			0x010c
+#define PLL_CON8_PLL_AUD			0x0120
+#define PLL_CON9_PLL_AUD			0x0124
+#define PLL_CON3_PLL_SHARED0			0x014c
+#define PLL_CON8_PLL_SHARED0			0x0160
+#define PLL_CON9_PLL_SHARED0			0x0164
+#define PLL_CON3_PLL_SHARED1			0x018c
+#define PLL_CON8_PLL_SHARED1			0x01a0
+#define PLL_CON9_PLL_SHARED1			0x01a4
+#define CMU_CMU_TOP_CONTROLLER_OPTION		0x0800
+#define CLK_CON_MUX_MUX_CLKCMU_APM_BUS		0x1000
+#define CLK_CON_MUX_MUX_CLKCMU_CORE_BUS		0x1004
+#define CLK_CON_MUX_MUX_CLKCMU_CORE_SSS		0x1008
+#define CLK_CON_MUX_MUX_CLKCMU_CPUCL0_BUSP	0x100c
+#define CLK_CON_MUX_MUX_CLKCMU_CPUCL0_CPU	0x1010
+#define CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG	0x1014
+#define CLK_CON_MUX_MUX_CLKCMU_DPU_AUDIF	0x1018
+#define CLK_CON_MUX_MUX_CLKCMU_DPU_AUD_BUS	0x101c
+#define CLK_CON_MUX_MUX_CLKCMU_DPU_AUD_CPU	0x1020
+#define CLK_CON_MUX_MUX_CLKCMU_DPU_BUS		0x1024
+#define CLK_CON_MUX_MUX_CLKCMU_FSYS_BUS		0x1028
+#define CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_CARD	0x102c
+#define CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_EMBD	0x1030
+#define CLK_CON_MUX_MUX_CLKCMU_FSYS_USB20DRD	0x1034
+#define CLK_CON_MUX_MUX_CLKCMU_G3D_BUS		0x1038
+#define CLK_CON_MUX_MUX_CLKCMU_MFC_BUS		0x103c
+#define CLK_CON_MUX_MUX_CLKCMU_MIF_BUSP		0x1040
+#define CLK_CON_MUX_MUX_CLKCMU_MIF_SWITCH	0x1044
+#define CLK_CON_MUX_MUX_CLKCMU_MODEM_SHARED1	0x1048
+#define CLK_CON_MUX_MUX_CLKCMU_PERI_BUS		0x104c
+#define CLK_CON_MUX_MUX_CLKCMU_PERI_IP		0x1050
+#define CLK_CON_MUX_MUX_CLKCMU_PERI_UART	0x1054
+#define CLK_CON_MUX_MUX_CMU_CMUREF		0x1058
+#define CLK_CON_DIV_AP2CP_SHARED0_PLL_CLK	0x1800
+#define CLK_CON_DIV_AP2CP_SHARED1_PLL_CLK	0x1804
+#define CLK_CON_DIV_AP2CP_SHARED2_PLL_CLK	0x1808
+#define CLK_CON_DIV_CLKCMU_APM_BUS		0x180c
+#define CLK_CON_DIV_CLKCMU_CMU_BOOST		0x1810
+#define CLK_CON_DIV_CLKCMU_CORE_BUS		0x1814
+#define CLK_CON_DIV_CLKCMU_CORE_SSS		0x1818
+#define CLK_CON_DIV_CLKCMU_CPUCL0_BUSP		0x181c
+#define CLK_CON_DIV_CLKCMU_CPUCL0_CPU		0x1820
+#define CLK_CON_DIV_CLKCMU_CPUCL0_DBG		0x1824
+#define CLK_CON_DIV_CLKCMU_DPU_AUDIF		0x1828
+#define CLK_CON_DIV_CLKCMU_DPU_AUD_BUS		0x182c
+#define CLK_CON_DIV_CLKCMU_DPU_AUD_CPU		0x1830
+#define CLK_CON_DIV_CLKCMU_DPU_BUS		0x1834
+#define CLK_CON_DIV_CLKCMU_FSYS_BUS		0x1838
+#define CLK_CON_DIV_CLKCMU_FSYS_MMC_CARD	0x183c
+#define CLK_CON_DIV_CLKCMU_FSYS_MMC_EMBD	0x1840
+#define CLK_CON_DIV_CLKCMU_FSYS_USB20DRD	0x1844
+#define CLK_CON_DIV_CLKCMU_G3D_BUS		0x1848
+#define CLK_CON_DIV_CLKCMU_MFC_BUS		0x184c
+#define CLK_CON_DIV_CLKCMU_MIF_BUSP		0x1850
+#define CLK_CON_DIV_CLKCMU_PERI_BUS		0x1854
+#define CLK_CON_DIV_CLKCMU_PERI_IP		0x1858
+#define CLK_CON_DIV_CLKCMU_PERI_UART		0x185c
+#define CLK_CON_DIV_DIV_CLK_CMU_CMUREF		0x1860
+#define CLK_CON_GAT_CLKCMU_MIF_SWITCH		0x2030
+#define CLK_CON_GAT_GATE_CLKCMU_APM_BUS		0x2034
+#define CLK_CON_GAT_GATE_CLKCMU_CORE_BUS	0x2038
+#define CLK_CON_GAT_GATE_CLKCMU_CORE_SSS	0x203c
+#define CLK_CON_GAT_GATE_CLKCMU_CPUCL0_BUSP	0x2040
+#define CLK_CON_GAT_GATE_CLKCMU_CPUCL0_CPU	0x2044
+#define CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG	0x2048
+#define CLK_CON_GAT_GATE_CLKCMU_DPU_AUDIF	0x204c
+#define CLK_CON_GAT_GATE_CLKCMU_DPU_AUD_BUS	0x2050
+#define CLK_CON_GAT_GATE_CLKCMU_DPU_AUD_CPU	0x2054
+#define CLK_CON_GAT_GATE_CLKCMU_DPU_BUS		0x2058
+#define CLK_CON_GAT_GATE_CLKCMU_FSYS_BUS	0x205c
+#define CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_CARD	0x2060
+#define CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_EMBD	0x2064
+#define CLK_CON_GAT_GATE_CLKCMU_FSYS_USB20DRD	0x2068
+#define CLK_CON_GAT_GATE_CLKCMU_G3D_BUS		0x206c
+#define CLK_CON_GAT_GATE_CLKCMU_MFC_BUS		0x2070
+#define CLK_CON_GAT_GATE_CLKCMU_MIF_BUSP	0x2074
+#define CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED0	0x2078
+#define CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED1	0x207c
+#define CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED2	0x2080
+#define CLK_CON_GAT_GATE_CLKCMU_PERI_BUS	0x2084
+#define CLK_CON_GAT_GATE_CLKCMU_PERI_IP		0x2088
+#define CLK_CON_GAT_GATE_CLKCMU_PERI_UART	0x208c
+
+static const unsigned long top_clk_regs[] __initconst = {
+	PLL_LOCKTIME_PLL_AUD,
+	PLL_LOCKTIME_PLL_SHARED0,
+	PLL_LOCKTIME_PLL_SHARED1,
+	PLL_CON3_PLL_AUD,
+	PLL_CON8_PLL_AUD,
+	PLL_CON9_PLL_AUD,
+	PLL_CON3_PLL_SHARED0,
+	PLL_CON8_PLL_SHARED0,
+	PLL_CON9_PLL_SHARED0,
+	PLL_CON3_PLL_SHARED1,
+	PLL_CON8_PLL_SHARED1,
+	PLL_CON9_PLL_SHARED1,
+	CMU_CMU_TOP_CONTROLLER_OPTION,
+	CLK_CON_MUX_MUX_CLKCMU_APM_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_CORE_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_CORE_SSS,
+	CLK_CON_MUX_MUX_CLKCMU_CPUCL0_BUSP,
+	CLK_CON_MUX_MUX_CLKCMU_CPUCL0_CPU,
+	CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG,
+	CLK_CON_MUX_MUX_CLKCMU_DPU_AUDIF,
+	CLK_CON_MUX_MUX_CLKCMU_DPU_AUD_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_DPU_AUD_CPU,
+	CLK_CON_MUX_MUX_CLKCMU_DPU_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_FSYS_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_CARD,
+	CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_EMBD,
+	CLK_CON_MUX_MUX_CLKCMU_FSYS_USB20DRD,
+	CLK_CON_MUX_MUX_CLKCMU_G3D_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_MFC_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_MIF_BUSP,
+	CLK_CON_MUX_MUX_CLKCMU_MIF_SWITCH,
+	CLK_CON_MUX_MUX_CLKCMU_MODEM_SHARED1,
+	CLK_CON_MUX_MUX_CLKCMU_PERI_BUS,
+	CLK_CON_MUX_MUX_CLKCMU_PERI_IP,
+	CLK_CON_MUX_MUX_CLKCMU_PERI_UART,
+	CLK_CON_MUX_MUX_CMU_CMUREF,
+	CLK_CON_DIV_AP2CP_SHARED0_PLL_CLK,
+	CLK_CON_DIV_AP2CP_SHARED1_PLL_CLK,
+	CLK_CON_DIV_AP2CP_SHARED2_PLL_CLK,
+	CLK_CON_DIV_CLKCMU_APM_BUS,
+	CLK_CON_DIV_CLKCMU_CMU_BOOST,
+	CLK_CON_DIV_CLKCMU_CORE_BUS,
+	CLK_CON_DIV_CLKCMU_CORE_SSS,
+	CLK_CON_DIV_CLKCMU_CPUCL0_BUSP,
+	CLK_CON_DIV_CLKCMU_CPUCL0_CPU,
+	CLK_CON_DIV_CLKCMU_CPUCL0_DBG,
+	CLK_CON_DIV_CLKCMU_DPU_AUDIF,
+	CLK_CON_DIV_CLKCMU_DPU_AUD_BUS,
+	CLK_CON_DIV_CLKCMU_DPU_AUD_CPU,
+	CLK_CON_DIV_CLKCMU_DPU_BUS,
+	CLK_CON_DIV_CLKCMU_FSYS_BUS,
+	CLK_CON_DIV_CLKCMU_FSYS_MMC_CARD,
+	CLK_CON_DIV_CLKCMU_FSYS_MMC_EMBD,
+	CLK_CON_DIV_CLKCMU_FSYS_USB20DRD,
+	CLK_CON_DIV_CLKCMU_G3D_BUS,
+	CLK_CON_DIV_CLKCMU_MFC_BUS,
+	CLK_CON_DIV_CLKCMU_MIF_BUSP,
+	CLK_CON_DIV_CLKCMU_PERI_BUS,
+	CLK_CON_DIV_CLKCMU_PERI_IP,
+	CLK_CON_DIV_CLKCMU_PERI_UART,
+	CLK_CON_DIV_DIV_CLK_CMU_CMUREF,
+	CLK_CON_GAT_CLKCMU_MIF_SWITCH,
+	CLK_CON_GAT_GATE_CLKCMU_APM_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_CORE_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_CORE_SSS,
+	CLK_CON_GAT_GATE_CLKCMU_CPUCL0_BUSP,
+	CLK_CON_GAT_GATE_CLKCMU_CPUCL0_CPU,
+	CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG,
+	CLK_CON_GAT_GATE_CLKCMU_DPU_AUDIF,
+	CLK_CON_GAT_GATE_CLKCMU_DPU_AUD_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_DPU_AUD_CPU,
+	CLK_CON_GAT_GATE_CLKCMU_DPU_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_FSYS_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_CARD,
+	CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_EMBD,
+	CLK_CON_GAT_GATE_CLKCMU_FSYS_USB20DRD,
+	CLK_CON_GAT_GATE_CLKCMU_G3D_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_MFC_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_MIF_BUSP,
+	CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED0,
+	CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED1,
+	CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED2,
+	CLK_CON_GAT_GATE_CLKCMU_PERI_BUS,
+	CLK_CON_GAT_GATE_CLKCMU_PERI_IP,
+	CLK_CON_GAT_GATE_CLKCMU_PERI_UART,
+};
+
+static const struct samsung_pll_clock top_pll_clks[] __initconst = {
+	PLL(pll_309, CLK_FOUT_AUD_PLL, "fout_aud_pll", "oscclk",
+	    PLL_LOCKTIME_PLL_AUD, PLL_CON3_PLL_AUD, NULL),
+	PLL(pll_309, CLK_FOUT_SHARED0_PLL, "fout_shared0_pll", "oscclk",
+	    PLL_LOCKTIME_PLL_SHARED0, PLL_CON3_PLL_SHARED0, NULL),
+	PLL(pll_309, CLK_FOUT_SHARED1_PLL, "fout_shared1_pll", "oscclk",
+	    PLL_LOCKTIME_PLL_SHARED1, PLL_CON3_PLL_SHARED1, NULL),
+};
+
+/* Parent clock list for CMU_TOP muxes */
+PNAME(mout_cmu_cmuref_p)	= { "oscclk",
+				    "dout_cmu_cmuref" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_APM */
+PNAME(mout_cmu_apm_bus_p)	= { "dout_shared0_div2",
+				    "dout_shared0_div3" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_CORE */
+PNAME(mout_cmu_core_bus_p)	= { "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_shared0_div3",
+				    "dout_aud_div3" };
+
+PNAME(mout_cmu_core_sss_p)	= { "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_shared0_div3",
+				    "dout_aud_div3"};
+
+/* Parent clock list for CMU_TOP muxes: for CMU_CPUCL0 */
+PNAME(mout_cmu_cpucl0_busp_p)	= { "dout_shared0_div3",
+				    "dout_shared0_div4" };
+
+PNAME(mout_cmu_cpucl0_cpu_p)	= { "fout_aud_pll",
+				    "dout_shared1_div2",
+				    "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_aud_div2",
+				    "dout_shared0_div3",
+				    "dout_shared1_div4",
+				    "dout_shared0_div4" };
+
+PNAME(mout_cmu_cpucl0_dbg_p)	= { "dout_shared0_div3",
+				    "dout_shared0_div4" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_DPU */
+PNAME(mout_cmu_dpu_audif_p)	= { "dout_aud_div2",
+				    "dout_aud_div3" };
+
+PNAME(mout_cmu_dpu_aud_bus_p)	= { "dout_aud_div2",
+				    "dout_shared0_div3",
+				    "dout_aud_div3",
+				    "dout_shared0_div4" };
+
+PNAME(mout_cmu_dpu_aud_cpu_p)	= { "fout_aud_pll",
+				    "dout_shared1_div2",
+				    "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_aud_div2",
+				    "dout_shared0_div3",
+				    "dout_aud_div3",
+				    "dout_aud_div4" };
+
+PNAME(mout_cmu_dpu_bus_p)	= { "dout_shared1_div3",
+				    "dout_shared0_div3",
+				    "dout_shared0_div4" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_FSYS */
+PNAME(mout_cmu_fsys_bus_p)	= { "dout_shared0_div2",
+				    "dout_shared1_div3" };
+
+PNAME(mout_cmu_fsys_mmc_card_p)	= { "oscclk",
+				    "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_shared0_div3",
+				    "dout_aud_div2" };
+
+PNAME(mout_cmu_fsys_mmc_embd_p)	= { "oscclk",
+				    "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_shared0_div3",
+				    "dout_aud_div2" };
+
+PNAME(mout_cmu_fsys_usb20drd_p)	= { "oscclk",
+				    "dout_shared0_div4" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_G3D */
+PNAME(mout_cmu_g3d_bus_p)	= { "dout_shared1_div2",
+				    "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_shared0_div3" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_MFC */
+PNAME(mout_cmu_mfc_bus_p)	= { "dout_shared1_div3",
+				    "dout_shared0_div3",
+				    "dout_shared1_div4",
+				    "dout_shared0_div4" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_MIF */
+PNAME(mout_cmu_mif_busp_p)	= { "dout_shared0_div3",
+				    "dout_shared0_div4" };
+
+PNAME(mout_cmu_mif_switch_p)	= { "fout_shared1_pll",
+				    "fout_shared0_pll",
+				    "dout_shared1_div2",
+				    "dout_shared0_div2",
+				    "dout_shared1_div3",
+				    "dout_shared0_div3",
+				    "dout_shared1_div4",
+				    "dout_shared0_div4" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_MODEM */
+PNAME(mout_cmu_modem_shared1_p)	= { "dout_shared1_div2",
+				    "dout_aud_div2",
+				    "dout_shared1_div3",
+				    "dout_shared0_div3" };
+
+/* Parent clock list for CMU_TOP muxes: for CMU_PERI */
+PNAME(mout_cmu_peri_bus_p)	= { "dout_shared0_div3",
+				    "dout_shared0_div4" };
+
+PNAME(mout_cmu_peri_ip_p)	= { "oscclk",
+				    "dout_shared0_div4",
+				    "dout_shared1_div4",
+				    "dout_aud_div3" };
+
+PNAME(mout_cmu_peri_uart_p)	= { "oscclk",
+				    "dout_shared0_div4",
+				    "dout_shared1_div4",
+				    "dout_aud_div3" };
+
+static const struct samsung_mux_clock top_mux_clks[] __initconst = {
+	/* APM (Active Power Management) */
+	MUX(CLK_MOUT_CMU_APM_BUS, "mout_cmu_apm_bus",
+	    mout_cmu_apm_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_APM_BUS,
+	    0, 1),
+
+	/* CMU (Clock Management Unit) */
+	MUX(CLK_MOUT_CMU_CMUREF, "mout_cmu_cmuref",
+	    mout_cmu_cmuref_p,
+	    CLK_CON_MUX_MUX_CMU_CMUREF,
+	    0, 1),
+
+	/* CORE */
+	MUX(CLK_MOUT_CMU_CORE_BUS, "mout_cmu_core_bus",
+	    mout_cmu_core_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_CORE_BUS,
+	    0, 2),
+	MUX(CLK_MOUT_CMU_CORE_SSS, "mout_cmu_core_sss",
+	    mout_cmu_core_sss_p,
+	    CLK_CON_MUX_MUX_CLKCMU_CORE_SSS,
+	    0, 2),
+
+	/* CPUCL0 (CPU Cluster 0) */
+	MUX(CLK_MOUT_CMU_CPUCL0_BUSP, "mout_cmu_cpucl0_busp",
+	    mout_cmu_cpucl0_busp_p,
+	    CLK_CON_MUX_MUX_CLKCMU_CPUCL0_BUSP,
+	    0, 1),
+	MUX(CLK_MOUT_CMU_CPUCL0_CPU, "mout_cmu_cpucl0_cpu",
+	    mout_cmu_cpucl0_cpu_p,
+	    CLK_CON_MUX_MUX_CLKCMU_CPUCL0_CPU,
+	    0, 3),
+	MUX(CLK_MOUT_CMU_CPUCL0_DBG, "mout_cmu_cpucl0_dbg",
+	    mout_cmu_cpucl0_dbg_p,
+	    CLK_CON_MUX_MUX_CLKCMU_CPUCL0_DBG,
+	    0, 1),
+
+	/* DPU (Display Processing Unit) */
+	MUX(CLK_MOUT_CMU_DPU_AUDIF, "mout_cmu_dpu_audif",
+	    mout_cmu_dpu_audif_p,
+	    CLK_CON_MUX_MUX_CLKCMU_DPU_AUDIF,
+	    0, 1),
+	MUX(CLK_MOUT_CMU_DPU_AUD_BUS, "mout_cmu_dpu_aud_bus",
+	    mout_cmu_dpu_aud_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_DPU_AUD_BUS,
+	    0, 2),
+	MUX(CLK_MOUT_CMU_DPU_AUD_CPU, "mout_cmu_dpu_aud_cpu",
+	    mout_cmu_dpu_aud_cpu_p,
+	    CLK_CON_MUX_MUX_CLKCMU_DPU_AUD_CPU,
+	    0, 3),
+	MUX(CLK_MOUT_CMU_DPU_BUS, "mout_cmu_dpu_bus",
+	    mout_cmu_dpu_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_DPU_BUS,
+	    0, 2),
+
+	/* FSYS */
+	MUX(CLK_MOUT_CMU_FSYS_BUS, "mout_cmu_fsys_bus",
+	    mout_cmu_fsys_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_FSYS_BUS,
+	    0, 1),
+	MUX(CLK_MOUT_CMU_FSYS_MMC_CARD, "mout_cmu_fsys_mmc_card",
+	    mout_cmu_fsys_mmc_card_p,
+	    CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_CARD,
+	    0, 3),
+	MUX(CLK_MOUT_CMU_FSYS_MMC_EMBD, "mout_cmu_fsys_mmc_embd",
+	    mout_cmu_fsys_mmc_embd_p,
+	    CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_EMBD,
+	    0, 3),
+	MUX(CLK_MOUT_CMU_FSYS_USB20DRD, "mout_cmu_fsys_usb20drd",
+	    mout_cmu_fsys_usb20drd_p,
+	    CLK_CON_MUX_MUX_CLKCMU_FSYS_USB20DRD,
+	    0, 1),
+
+	/* G3D */
+	MUX(CLK_MOUT_CMU_G3D_BUS, "mout_cmu_g3d_bus",
+	    mout_cmu_g3d_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_G3D_BUS,
+	    0, 2),
+
+	/* MFC */
+	MUX(CLK_MOUT_CMU_MFC_BUS, "mout_cmu_mfc_bus",
+	    mout_cmu_mfc_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_MFC_BUS,
+	    0, 2),
+
+	/* MIF */
+	MUX(CLK_MOUT_CMU_MIF_BUSP, "mout_cmu_mif_busp",
+	    mout_cmu_mif_busp_p,
+	    CLK_CON_MUX_MUX_CLKCMU_MIF_BUSP,
+	    0, 1),
+	MUX(CLK_MOUT_CMU_MIF_SWITCH, "mout_cmu_mif_switch",
+	    mout_cmu_mif_switch_p,
+	    CLK_CON_MUX_MUX_CLKCMU_MIF_SWITCH,
+	    0, 3),
+
+	/* MODEM */
+	MUX(CLK_MOUT_CMU_MODEM_SHARED1, "mout_cmu_modem_shared1",
+	    mout_cmu_modem_shared1_p,
+	    CLK_CON_MUX_MUX_CLKCMU_MODEM_SHARED1,
+	    0, 2),
+
+	/* PERI */
+	MUX(CLK_MOUT_CMU_PERI_BUS, "mout_cmu_peri_bus",
+	    mout_cmu_peri_bus_p,
+	    CLK_CON_MUX_MUX_CLKCMU_PERI_BUS,
+	    0, 1),
+	MUX(CLK_MOUT_CMU_PERI_IP, "mout_cmu_peri_ip",
+	    mout_cmu_peri_ip_p,
+	    CLK_CON_MUX_MUX_CLKCMU_PERI_IP,
+	    0, 2),
+	MUX(CLK_MOUT_CMU_PERI_UART, "mout_cmu_peri_uart",
+	    mout_cmu_peri_uart_p,
+	    CLK_CON_MUX_MUX_CLKCMU_PERI_UART,
+	    0, 2),
+};
+
+static const struct samsung_div_clock top_div_clks[] __initconst = {
+	/* APM */
+	DIV(CLK_DOUT_CMU_APM_BUS, "dout_cmu_apm_bus",
+	    "gout_cmu_apm_bus",
+	    CLK_CON_DIV_CLKCMU_APM_BUS,
+	    0, 3),
+
+	/* CMU */
+	DIV(CLK_DOUT_CMU_CMU_BOOST, "dout_cmu_cmu_boost",
+	    "fout_shared0_pll",
+	    CLK_CON_DIV_CLKCMU_CMU_BOOST,
+	    0, 3),
+	DIV(CLK_DOUT_CMU_CMUREF, "dout_cmu_cmuref",
+	    "fout_shared0_pll",
+	    CLK_CON_DIV_DIV_CLK_CMU_CMUREF,
+	    0, 2),
+
+	/* CORE */
+	DIV(CLK_DOUT_CMU_CORE_BUS, "dout_cmu_core_bus",
+	    "gout_cmu_core_bus",
+	    CLK_CON_DIV_CLKCMU_CORE_BUS,
+	    0, 4),
+	DIV(CLK_DOUT_CMU_CORE_SSS, "dout_cmu_core_sss",
+	    "gout_cmu_core_sss",
+	    CLK_CON_DIV_CLKCMU_CORE_SSS,
+	    0, 4),
+
+	/* CPUCL0 */
+	DIV(CLK_DOUT_CMU_CPUCL0_BUSP, "dout_cmu_cpucl0_busp",
+	    "gout_cmu_cpucl0_busp",
+	    CLK_CON_DIV_CLKCMU_CPUCL0_BUSP,
+	    0, 3),
+	DIV(CLK_DOUT_CMU_CPUCL0_CPU, "dout_cmu_cpucl0_cpu",
+	    "gout_cmu_cpucl0_cpu",
+	    CLK_CON_DIV_CLKCMU_CPUCL0_CPU,
+	    0, 3),
+	DIV(CLK_DOUT_CMU_CPUCL0_DBG, "dout_cmu_cpucl0_dbg",
+	    "gout_cmu_cpucl0_dbg",
+	    CLK_CON_DIV_CLKCMU_CPUCL0_DBG,
+	    0, 3),
+
+	/* DPU */
+	DIV(CLK_DOUT_CMU_DPU_AUDIF, "dout_cmu_dpu_audif",
+	    "gout_cmu_dpu_audif",
+	    CLK_CON_DIV_CLKCMU_DPU_AUDIF,
+	    0, 9),
+	DIV(CLK_DOUT_CMU_DPU_AUD_BUS, "dout_cmu_dpu_aud_bus",
+	    "gout_cmu_dpu_aud_bus",
+	    CLK_CON_DIV_CLKCMU_DPU_AUD_BUS,
+	    0, 3),
+	DIV(CLK_DOUT_CMU_DPU_AUD_CPU, "dout_cmu_dpu_aud_cpu",
+	    "gout_cmu_dpu_aud_cpu",
+	    CLK_CON_DIV_CLKCMU_DPU_AUD_CPU,
+	    0, 3),
+	DIV(CLK_DOUT_CMU_DPU_BUS, "dout_cmu_dpu_bus",
+	    "gout_cmu_dpu_bus",
+	    CLK_CON_DIV_CLKCMU_DPU_BUS,
+	    0, 3),
+
+	/* FSYS */
+	DIV(CLK_DOUT_CMU_FSYS_BUS, "dout_cmu_fsys_bus",
+	    "gout_cmu_fsys_bus",
+	    CLK_CON_DIV_CLKCMU_FSYS_BUS,
+	    0, 4),
+	DIV(CLK_DOUT_CMU_FSYS_MMC_CARD, "dout_cmu_fsys_mmc_card",
+	    "gout_cmu_fsys_mmc_card",
+	    CLK_CON_DIV_CLKCMU_FSYS_MMC_CARD,
+	    0, 9),
+	DIV(CLK_DOUT_CMU_FSYS_MMC_EMBD, "dout_cmu_fsys_mmc_embd",
+	    "gout_cmu_fsys_mmc_embd",
+	    CLK_CON_DIV_CLKCMU_FSYS_MMC_EMBD,
+	    0, 9),
+	DIV(CLK_DOUT_CMU_FSYS_USB20DRD, "dout_cmu_fsys_usb20drd",
+	    "gout_cmu_fsys_usb20drd",
+	    CLK_CON_DIV_CLKCMU_FSYS_USB20DRD,
+	    0, 4),
+
+	/* G3D */
+	DIV(CLK_DOUT_CMU_G3D_BUS, "dout_cmu_g3d_bus",
+	    "gout_cmu_g3d_bus",
+	    CLK_CON_DIV_CLKCMU_G3D_BUS,
+	    0, 3),
+
+	/* MFC */
+	DIV(CLK_DOUT_CMU_MFC_BUS, "dout_cmu_mfc_bus",
+	    "gout_cmu_mfc_bus",
+	    CLK_CON_DIV_CLKCMU_MFC_BUS,
+	    0, 4),
+
+	/* MIF */
+	DIV(CLK_DOUT_CMU_MIF_BUSP, "dout_cmu_mif_busp",
+	    "gout_cmu_mif_busp",
+	    CLK_CON_DIV_CLKCMU_MIF_BUSP,
+	    0, 3),
+
+	/* PERI */
+	DIV(CLK_DOUT_CMU_PERI_BUS, "dout_cmu_peri_bus",
+	    "gout_cmu_peri_bus",
+	    CLK_CON_DIV_CLKCMU_PERI_BUS,
+	    0, 3),
+	DIV(CLK_DOUT_CMU_PERI_IP, "dout_cmu_peri_ip",
+	    "gout_cmu_peri_ip",
+	    CLK_CON_DIV_CLKCMU_PERI_IP,
+	    0, 4),
+	DIV(CLK_DOUT_CMU_PERI_UART, "dout_cmu_peri_uart",
+	    "gout_cmu_peri_uart",
+	    CLK_CON_DIV_CLKCMU_PERI_UART,
+	    0, 4),
+
+	/* SHARED */
+	DIV(CLK_DOUT_AP2CP_SHARED0_PLL, "dout_ap2cp_shared0_pll",
+	    "gout_cmu_modem_shared0",
+	    CLK_CON_DIV_AP2CP_SHARED0_PLL_CLK,
+	    0, 3),
+	DIV(CLK_DOUT_AP2CP_SHARED1_PLL, "dout_ap2cp_shared1_pll",
+	    "gout_cmu_modem_shared1",
+	    CLK_CON_DIV_AP2CP_SHARED1_PLL_CLK,
+	    0, 3),
+	DIV(CLK_DOUT_AP2CP_SHARED2_PLL, "dout_ap2cp_shared2_pll",
+	    "gout_cmu_modem_shared2",
+	    CLK_CON_DIV_AP2CP_SHARED2_PLL_CLK,
+	    0, 3),
+};
+
+static const struct samsung_fixed_factor_clock top_fixed_factor_clks[] __initconst = {
+	FFACTOR(CLK_DOUT_SHARED0_DIV2, "dout_shared0_div2", "fout_shared0_pll",
+		1, 2, 0),
+	FFACTOR(CLK_DOUT_SHARED0_DIV3, "dout_shared0_div3", "fout_shared0_pll",
+		1, 3, 0),
+	FFACTOR(CLK_DOUT_SHARED0_DIV4, "dout_shared0_div4", "fout_shared0_pll",
+		1, 4, 0),
+	FFACTOR(CLK_DOUT_SHARED1_DIV2, "dout_shared1_div2", "fout_shared1_pll",
+		1, 2, 0),
+	FFACTOR(CLK_DOUT_SHARED1_DIV3, "dout_shared1_div3", "fout_shared1_pll",
+		1, 3, 0),
+	FFACTOR(CLK_DOUT_SHARED1_DIV4, "dout_shared1_div4", "fout_shared1_pll",
+		1, 4, 0),
+	FFACTOR(CLK_DOUT_AUD_DIV2, "dout_aud_div2", "fout_aud_pll",
+		1, 2, 0),
+	FFACTOR(CLK_DOUT_AUD_DIV3, "dout_aud_div3", "fout_aud_pll",
+		1, 3, 0),
+	FFACTOR(CLK_DOUT_AUD_DIV4, "dout_aud_div4", "fout_aud_pll",
+		1, 4, 0),
+};
+
+static const struct samsung_gate_clock top_gate_clks[] __initconst = {
+	/* APM */
+	GATE(CLK_GOUT_CMU_APM_BUS, "gout_cmu_apm_bus",
+	     "mout_cmu_apm_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_APM_BUS,
+	     21, 0, 0),
+
+	/* CORE */
+	GATE(CLK_GOUT_CMU_CORE_BUS, "gout_cmu_core_bus",
+	     "mout_cmu_core_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_CORE_BUS,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_CORE_SSS, "gout_cmu_core_sss",
+	     "mout_cmu_core_sss",
+	     CLK_CON_GAT_GATE_CLKCMU_CORE_SSS,
+	     21, 0, 0),
+
+	/* CPUCL0 */
+	GATE(CLK_GOUT_CMU_CPUCL0_BUSP, "gout_cmu_cpucl0_busp",
+	     "mout_cmu_cpucl0_busp",
+	     CLK_CON_GAT_GATE_CLKCMU_CPUCL0_BUSP,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_CPUCL0_CPU, "gout_cmu_cpucl0_cpu",
+	     "mout_cmu_cpucl0_cpu",
+	     CLK_CON_GAT_GATE_CLKCMU_CPUCL0_CPU,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_CPUCL0_DBG, "gout_cmu_cpucl0_dbg",
+	     "mout_cmu_cpucl0_dbg",
+	     CLK_CON_GAT_GATE_CLKCMU_CPUCL0_DBG,
+	     21, 0, 0),
+
+	/* DPU */
+	GATE(CLK_GOUT_CMU_DPU_AUDIF, "gout_cmu_dpu_audif",
+	     "mout_cmu_dpu_audif",
+	     CLK_CON_GAT_GATE_CLKCMU_DPU_AUDIF,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_DPU_AUD_BUS, "gout_cmu_dpu_aud_bus",
+	     "mout_cmu_dpu_aud_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_DPU_AUD_BUS,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_DPU_AUD_CPU, "gout_cmu_dpu_aud_cpu",
+	     "mout_cmu_dpu_aud_cpu",
+	     CLK_CON_GAT_GATE_CLKCMU_DPU_AUD_CPU,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_DPU_BUS, "gout_cmu_dpu_bus",
+	     "mout_cmu_dpu_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_DPU_BUS,
+	     21, 0, 0),
+
+	/* FSYS */
+	GATE(CLK_GOUT_CMU_FSYS_BUS, "gout_cmu_fsys_bus",
+	     "mout_cmu_fsys_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_FSYS_BUS,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_FSYS_MMC_CARD, "gout_cmu_fsys_mmc_card",
+	     "mout_cmu_fsys_mmc_card",
+	     CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_CARD,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_FSYS_MMC_EMBD, "gout_cmu_fsys_mmc_embd",
+	     "mout_cmu_fsys_mmc_embd",
+	     CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_EMBD,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_FSYS_USB20DRD, "gout_cmu_fsys_usb20drd",
+	     "mout_cmu_fsys_usb20drd",
+	     CLK_CON_GAT_GATE_CLKCMU_FSYS_USB20DRD,
+	     21, 0, 0),
+
+	/* G3D */
+	GATE(CLK_GOUT_CMU_G3D_BUS, "gout_cmu_g3d_bus",
+	     "mout_cmu_g3d_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_G3D_BUS,
+	     21, 0, 0),
+
+	/* MFC */
+	GATE(CLK_GOUT_CMU_MFC_BUS, "gout_cmu_mfc_bus",
+	     "mout_cmu_mfc_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_MFC_BUS,
+	     21, 0, 0),
+
+	/* MIF */
+	GATE(CLK_GOUT_CMU_MIF_SWITCH, "gout_cmu_mif_switch",
+	     "mout_cmu_mif_switch",
+	     CLK_CON_GAT_CLKCMU_MIF_SWITCH,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_MIF_BUSP, "gout_cmu_mif_busp",
+	     "mout_cmu_mif_busp",
+	     CLK_CON_GAT_GATE_CLKCMU_MIF_BUSP,
+	     21, 0, 0),
+
+	/* MODEM */
+	GATE(CLK_GOUT_CMU_MODEM_SHARED0, "gout_cmu_modem_shared0",
+	     "fout_shared0_pll",
+	     CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED0,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_MODEM_SHARED1, "gout_cmu_modem_shared1",
+	     "mout_cmu_modem_shared1",
+	     CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED1,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_MODEM_SHARED2, "gout_cmu_modem_shared2",
+	     "fout_shared0_pll",
+	     CLK_CON_GAT_GATE_CLKCMU_MODEM_SHARED2,
+	     21, 0, 0),
+
+	/* PERI */
+	GATE(CLK_GOUT_CMU_PERI_BUS, "gout_cmu_peri_bus",
+	     "mout_cmu_peri_bus",
+	     CLK_CON_GAT_GATE_CLKCMU_PERI_BUS,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_PERI_IP, "gout_cmu_peri_ip",
+	     "mout_cmu_peri_ip",
+	     CLK_CON_GAT_GATE_CLKCMU_PERI_IP,
+	     21, 0, 0),
+	GATE(CLK_GOUT_CMU_PERI_UART, "gout_cmu_peri_uart",
+	     "mout_cmu_peri_uart",
+	     CLK_CON_GAT_GATE_CLKCMU_PERI_UART,
+	     21, 0, 0),
+};
+
+static const unsigned long drcg_sysreg[] __initconst = {
+	EXYNOS5515_DRCG_EN_OFFSET,
+};
+
+static const struct samsung_cmu_info top_cmu_info __initconst = {
+	.pll_clks		= top_pll_clks,
+	.nr_pll_clks		= ARRAY_SIZE(top_pll_clks),
+	.mux_clks		= top_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(top_mux_clks),
+	.div_clks		= top_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(top_div_clks),
+	.fixed_factor_clks	= top_fixed_factor_clks,
+	.nr_fixed_factor_clks	= ARRAY_SIZE(top_fixed_factor_clks),
+	.gate_clks		= top_gate_clks,
+	.nr_gate_clks		= ARRAY_SIZE(top_gate_clks),
+	.nr_clk_ids		= CLKS_NR_TOP,
+	.clk_regs		= top_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(top_clk_regs),
+	.sysreg_clk_regs	= drcg_sysreg,
+	.nr_sysreg_clk_regs	= ARRAY_SIZE(drcg_sysreg),
+	.auto_clock_gate	= true,
+	.gate_dbg_offset	= EXYNOS5515_GATE_DBG_OFFSET,
+	.option_offset		= CMU_CMU_TOP_CONTROLLER_OPTION,
+	.drcg_offset		= EXYNOS5515_DRCG_EN_OFFSET,
+};
+
+static void __init exynos5515_cmu_top_init(struct device_node *np)
+{
+	exynos_arm64_register_cmu(NULL, np, &top_cmu_info);
+}
+
+/* Register CMU_TOP early, as it's a dependency for other early domains */
+CLK_OF_DECLARE(exynos5515_cmu_top, "samsung,exynos5515-cmu-top",
+	       exynos5515_cmu_top_init);
+
+/* ---- CMU_PERI ------------------------------------------------------------ */
+
+/*
+ * CMU_PERI:	0x10030000
+ * SYSREG_PERI:	0x10020000
+ */
+
+#define PLL_CON0_MUX_CLKCMU_PERI_BUS_USER					0x0600
+#define PLL_CON1_MUX_CLKCMU_PERI_BUS_USER					0x0604
+#define PLL_CON0_MUX_CLKCMU_PERI_IP_USER					0x0610
+#define PLL_CON1_MUX_CLKCMU_PERI_IP_USER					0x0614
+#define PLL_CON0_MUX_CLKCMU_PERI_UART_USER					0x0620
+#define PLL_CON1_MUX_CLKCMU_PERI_UART_USER					0x0624
+#define PERI_CMU_PERI_CONTROLLER_OPTION						0x0800
+#define CLK_CON_MUX_MUX_CLK_PERI_SPI						0x1000
+#define CLK_CON_MUX_MUX_CLK_PERI_USI00_USI					0x1004
+#define CLK_CON_DIV_CLKCMU_OTP							0x1800
+#define CLK_CON_DIV_DIV_CLK_PERI_I2C_0						0x1804
+#define CLK_CON_DIV_DIV_CLK_PERI_I2C_1						0x1808
+#define CLK_CON_DIV_DIV_CLK_PERI_I2C_2						0x180c
+#define CLK_CON_DIV_DIV_CLK_PERI_I2C_3						0x1810
+#define CLK_CON_DIV_DIV_CLK_PERI_SPI						0x1814
+#define CLK_CON_DIV_DIV_CLK_PERI_USI00_I2C					0x1818
+#define CLK_CON_DIV_DIV_CLK_PERI_USI00_USI					0x181c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_BUSIF_TMU_IPCLKPORT_PCLK			0x2000
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_D_TZPC_PERI_IPCLKPORT_PCLK			0x2004
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_GPIO_PERI_IPCLKPORT_PCLK			0x2008
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_0_IPCLKPORT_IPCLK			0x200c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_0_IPCLKPORT_PCLK			0x2010
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_1_IPCLKPORT_IPCLK			0x2014
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_1_IPCLKPORT_PCLK			0x2018
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_2_IPCLKPORT_IPCLK			0x201c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_2_IPCLKPORT_PCLK			0x2020
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_3_IPCLKPORT_IPCLK			0x2024
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_3_IPCLKPORT_PCLK			0x2028
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_MCT_IPCLKPORT_PCLK				0x202c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_OTP_CON_TOP_IPCLKPORT_I_OSCCLK		0x2030
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_OTP_CON_TOP_IPCLKPORT_PCLK			0x2034
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_PERI_CMU_PERI_IPCLKPORT_PCLK		0x2038
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_PWM_MOTOR_IPCLKPORT_I_PCLK_S0		0x203c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_BUS_IPCLKPORT_CLK	0x2040
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_0_IPCLKPORT_CLK	0x2044
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_1_IPCLKPORT_CLK	0x2048
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_2_IPCLKPORT_CLK	0x204c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_3_IPCLKPORT_CLK	0x2050
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_OSCCLK_IPCLKPORT_CLK	0x2054
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_SPI_IPCLKPORT_CLK	0x2058
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_UART_IPCLKPORT_CLK	0x205c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_USI00_I2C_IPCLKPORT_CLK	0x2060
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_USI00_USI_IPCLKPORT_CLK	0x2064
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_SLH_AXI_MI_P_PERI_IPCLKPORT_I_CLK		0x2068
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_SYSREG_PERI_IPCLKPORT_PCLK			0x206c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_I2C_IPCLKPORT_IPCLK			0x2070
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_I2C_IPCLKPORT_PCLK			0x2074
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_USI_IPCLKPORT_IPCLK			0x2078
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_USI_IPCLKPORT_PCLK			0x207c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI_SPI_IPCLKPORT_IPCLK			0x2080
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI_SPI_IPCLKPORT_PCLK			0x2084
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI_UART_IPCLKPORT_IPCLK			0x2088
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_USI_UART_IPCLKPORT_PCLK			0x208c
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_WDT0_IPCLKPORT_PCLK			0x2090
+#define CLK_CON_GAT_CLK_BLK_PERI_UID_WDT1_IPCLKPORT_PCLK			0x2094
+#define CLK_CON_GAT_GATE_CLK_PERI_I2C_0						0x2098
+#define CLK_CON_GAT_GATE_CLK_PERI_I2C_1						0x209c
+#define CLK_CON_GAT_GATE_CLK_PERI_I2C_2						0x20a0
+#define CLK_CON_GAT_GATE_CLK_PERI_I2C_3						0x20a4
+#define CLK_CON_GAT_GATE_CLK_PERI_USI00_I2C					0x20a8
+
+static const unsigned long peri_clk_regs[] __initconst = {
+	PLL_CON0_MUX_CLKCMU_PERI_BUS_USER,
+	PLL_CON1_MUX_CLKCMU_PERI_BUS_USER,
+	PLL_CON0_MUX_CLKCMU_PERI_IP_USER,
+	PLL_CON1_MUX_CLKCMU_PERI_IP_USER,
+	PLL_CON0_MUX_CLKCMU_PERI_UART_USER,
+	PLL_CON1_MUX_CLKCMU_PERI_UART_USER,
+	PERI_CMU_PERI_CONTROLLER_OPTION,
+	CLK_CON_MUX_MUX_CLK_PERI_SPI,
+	CLK_CON_MUX_MUX_CLK_PERI_USI00_USI,
+	CLK_CON_DIV_CLKCMU_OTP,
+	CLK_CON_DIV_DIV_CLK_PERI_I2C_0,
+	CLK_CON_DIV_DIV_CLK_PERI_I2C_1,
+	CLK_CON_DIV_DIV_CLK_PERI_I2C_2,
+	CLK_CON_DIV_DIV_CLK_PERI_I2C_3,
+	CLK_CON_DIV_DIV_CLK_PERI_SPI,
+	CLK_CON_DIV_DIV_CLK_PERI_USI00_I2C,
+	CLK_CON_DIV_DIV_CLK_PERI_USI00_USI,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_BUSIF_TMU_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_D_TZPC_PERI_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_GPIO_PERI_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_0_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_0_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_1_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_1_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_2_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_2_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_3_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_3_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_MCT_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_OTP_CON_TOP_IPCLKPORT_I_OSCCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_OTP_CON_TOP_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_PERI_CMU_PERI_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_PWM_MOTOR_IPCLKPORT_I_PCLK_S0,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_BUS_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_0_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_1_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_2_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_3_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_OSCCLK_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_SPI_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_UART_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_USI00_I2C_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_USI00_USI_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_SLH_AXI_MI_P_PERI_IPCLKPORT_I_CLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_SYSREG_PERI_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_I2C_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_I2C_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_USI_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_USI_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI_SPI_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI_SPI_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI_UART_IPCLKPORT_IPCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_USI_UART_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_WDT0_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_PERI_UID_WDT1_IPCLKPORT_PCLK,
+	CLK_CON_GAT_GATE_CLK_PERI_I2C_0,
+	CLK_CON_GAT_GATE_CLK_PERI_I2C_1,
+	CLK_CON_GAT_GATE_CLK_PERI_I2C_2,
+	CLK_CON_GAT_GATE_CLK_PERI_I2C_3,
+	CLK_CON_GAT_GATE_CLK_PERI_USI00_I2C,
+};
+
+/* Parent clock list for CMU_PERI muxes */
+PNAME(mout_pll_peri_bus_user_p)		= { "oscclk",
+					    "mout_cmu_peri_bus" };
+
+PNAME(mout_pll_peri_ip_user_p)		= { "oscclk",
+					    "mout_cmu_peri_ip" };
+
+/* Parent clock list for CMU_PERI muxes: for SPI */
+PNAME(mout_peri_spi_p)			= { "oscclk",
+					    "mout_pll_peri_ip_user" };
+
+/* Parent clock list for CMU_PERI muxes: for UART */
+PNAME(mout_pll_peri_uart_user_p)	= { "oscclk",
+					    "mout_cmu_peri_uart" };
+
+/* Parent clock list for CMU_PERI muxes: for USI */
+PNAME(mout_peri_usi00_usi_p)		= { "oscclk",
+					    "mout_pll_peri_ip_user" };
+
+static const struct samsung_mux_clock peri_mux_clks[] __initconst = {
+	/* BUS */
+	MUX(CLK_MOUT_PLL_PERI_BUS_USER, "mout_pll_peri_bus_user",
+	    mout_pll_peri_bus_user_p,
+	    PLL_CON0_MUX_CLKCMU_PERI_BUS_USER,
+	    4, 1),
+
+	/* IP */
+	MUX(CLK_MOUT_PLL_PERI_IP_USER, "mout_pll_peri_ip_user",
+	    mout_pll_peri_ip_user_p,
+	    PLL_CON0_MUX_CLKCMU_PERI_IP_USER,
+	    4, 1),
+
+	/* SPI */
+	MUX(CLK_MOUT_PERI_SPI, "mout_peri_spi",
+	    mout_peri_spi_p,
+	    CLK_CON_MUX_MUX_CLK_PERI_SPI,
+	    0, 1),
+
+	/* UART */
+	MUX(CLK_MOUT_PLL_PERI_UART_USER, "mout_pll_peri_uart_user",
+	    mout_pll_peri_uart_user_p,
+	    PLL_CON0_MUX_CLKCMU_PERI_UART_USER,
+	    4, 1),
+
+	/* USI (Universal Serial Interface) */
+	MUX(CLK_MOUT_PERI_USI00_USI, "mout_peri_usi00_usi",
+	    mout_peri_usi00_usi_p,
+	    CLK_CON_MUX_MUX_CLK_PERI_USI00_USI,
+	    0, 1),
+};
+
+static const struct samsung_div_clock peri_div_clks[] __initconst = {
+	/* I2C */
+	DIV(CLK_DOUT_PERI_I2C_0, "dout_peri_i2c_0",
+	    "gout_peri_i2c_0",
+	    CLK_CON_DIV_DIV_CLK_PERI_I2C_0,
+	    0, 4),
+	DIV(CLK_DOUT_PERI_I2C_1, "dout_peri_i2c_1",
+	    "gout_peri_i2c_1",
+	    CLK_CON_DIV_DIV_CLK_PERI_I2C_1,
+	    0, 4),
+	DIV(CLK_DOUT_PERI_I2C_2, "dout_peri_i2c_2",
+	    "gout_peri_i2c_2",
+	    CLK_CON_DIV_DIV_CLK_PERI_I2C_2,
+	    0, 4),
+	DIV(CLK_DOUT_PERI_I2C_3, "dout_peri_i2c_3",
+	    "gout_peri_i2c_3",
+	    CLK_CON_DIV_DIV_CLK_PERI_I2C_3,
+	    0, 4),
+
+	/* SPI */
+	DIV(CLK_DOUT_PERI_SPI, "dout_peri_spi",
+	    "mout_peri_spi",
+	    CLK_CON_DIV_DIV_CLK_PERI_SPI,
+	    0, 5),
+
+	/* USI */
+	DIV(CLK_DOUT_PERI_USI00_I2C, "dout_peri_usi00_i2c",
+	    "gout_peri_usi00_i2c",
+	    CLK_CON_DIV_DIV_CLK_PERI_USI00_I2C,
+	    0, 4),
+	DIV(CLK_DOUT_PERI_USI00_USI, "dout_peri_usi00_usi",
+	    "mout_peri_usi00_usi",
+	    CLK_CON_DIV_DIV_CLK_PERI_USI00_USI,
+	    0, 5),
+};
+
+static const struct samsung_fixed_factor_clock peri_fixed_factor_clks[] __initconst = {
+	/* OTP (One-Time Programmable eFuses) */
+	FFACTOR(CLK_DOUT_PERI_OTP, "dout_peri_otp", "oscclk",
+		1, 8, 0),
+};
+
+static const struct samsung_gate_clock peri_gate_clks[] __initconst = {
+	/* BUS */
+	GATE(CLK_GOUT_PERI_BUSIF_TMU_PCLK, "gout_peri_busif_tmu_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_BUSIF_TMU_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_BUS_CLK, "gout_peri_bus_clk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_BUS_IPCLKPORT_CLK,
+	     21, 0, 0),
+
+	/* CMU */
+	GATE(CLK_GOUT_PERI_CMU_PCLK, "gout_peri_cmu_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_PERI_CMU_PERI_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* GPIO */
+	GATE(CLK_GOUT_PERI_GPIO_PCLK, "gout_peri_gpio_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_GPIO_PERI_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* I2C */
+	GATE(CLK_GOUT_PERI_I2C_0_IPCLK, "gout_peri_i2c_0_ipclk",
+	     "dout_peri_i2c_0",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_0_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_0_PCLK, "gout_peri_i2c_0_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_0_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_1_IPCLK, "gout_peri_i2c_1_ipclk",
+	     "dout_peri_i2c_1",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_1_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_1_PCLK, "gout_peri_i2c_1_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_1_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_2_IPCLK, "gout_peri_i2c_2_ipclk",
+	     "dout_peri_i2c_2",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_2_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_2_PCLK, "gout_peri_i2c_2_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_2_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_3_IPCLK, "gout_peri_i2c_3_ipclk",
+	     "dout_peri_i2c_3",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_3_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_3_PCLK, "gout_peri_i2c_3_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_I2C_3_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_0_CLK, "gout_peri_i2c_0_clk",
+	     "dout_peri_i2c_0",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_0_IPCLKPORT_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_1_CLK, "gout_peri_i2c_1_clk",
+	     "dout_peri_i2c_1",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_1_IPCLKPORT_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_2_CLK, "gout_peri_i2c_2_clk",
+	     "dout_peri_i2c_2",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_2_IPCLKPORT_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_3_CLK, "gout_peri_i2c_3_clk",
+	     "dout_peri_i2c_3",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_I2C_3_IPCLKPORT_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_0, "gout_peri_i2c_0",
+	     "mout_pll_peri_ip_user",
+	     CLK_CON_GAT_GATE_CLK_PERI_I2C_0,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_1, "gout_peri_i2c_1",
+	     "mout_pll_peri_ip_user",
+	     CLK_CON_GAT_GATE_CLK_PERI_I2C_1,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_2, "gout_peri_i2c_2",
+	     "mout_pll_peri_ip_user",
+	     CLK_CON_GAT_GATE_CLK_PERI_I2C_2,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_I2C_3, "gout_peri_i2c_3",
+	     "mout_pll_peri_ip_user",
+	     CLK_CON_GAT_GATE_CLK_PERI_I2C_3,
+	     21, 0, 0),
+
+	/* MCT (Multi-Core Timer) */
+	GATE(CLK_GOUT_PERI_MCT_PCLK, "gout_peri_mct_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_MCT_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* OTP */
+	GATE(CLK_GOUT_PERI_OTP_OSCCLK, "gout_peri_otp_oscclk",
+	     "oscclk",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_OTP_CON_TOP_IPCLKPORT_I_OSCCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_OTP_PCLK, "gout_peri_otp_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_OTP_CON_TOP_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* OSCCLK */
+	GATE(CLK_GOUT_PERI_OSCCLK, "gout_peri_oscclk",
+	     "oscclk",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_OSCCLK_IPCLKPORT_CLK,
+	     21, 0, 0),
+
+	/* PWM */
+	GATE(CLK_GOUT_PERI_PWM_MOTOR_PCLK, "gout_peri_pwm_motor_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_PWM_MOTOR_IPCLKPORT_I_PCLK_S0,
+	     21, 0, 0),
+
+	/* SPI */
+	GATE(CLK_GOUT_PERI_SPI_CLK, "gout_peri_spi_clk",
+	     "dout_peri_spi",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_SPI_IPCLKPORT_CLK,
+	     21, 0, 0),
+
+	/* SYSREG */
+	GATE(CLK_GOUT_PERI_SYSREG_PCLK, "gout_peri_sysreg_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_SYSREG_PERI_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* TZ */
+	GATE(CLK_GOUT_PERI_D_TZPC_PCLK, "gout_peri_d_tzpc_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_D_TZPC_PERI_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* UART */
+	GATE(CLK_GOUT_PERI_UART_CLK, "gout_peri_uart_clk",
+	     "mout_pll_peri_uart_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_UART_IPCLKPORT_CLK,
+	     21, 0, 0),
+
+	/* USI */
+	GATE(CLK_GOUT_PERI_USI00_I2C_CLK, "gout_peri_usi00_i2c_clk",
+	     "dout_peri_usi00_i2c",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_USI00_I2C_IPCLKPORT_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI00_USI_CLK, "gout_peri_usi00_usi_clk",
+	     "dout_peri_usi00_usi",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_RSTNSYNC_CLK_PERI_USI00_USI_IPCLKPORT_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI00_I2C_IPCLK, "gout_peri_usi00_i2c_ipclk",
+	     "dout_peri_usi00_i2c",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_I2C_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI00_I2C_PCLK, "gout_peri_usi00_i2c_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_I2C_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI00_USI_IPCLK, "gout_peri_usi00_usi_ipclk",
+	     "dout_peri_usi00_usi",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_USI_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI00_USI_PCLK, "gout_peri_usi00_usi_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI00_USI_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI_SPI_IPCLK, "gout_peri_usi_spi_ipclk",
+	     "dout_peri_spi",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI_SPI_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI_SPI_PCLK, "gout_peri_usi_spi_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI_SPI_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI_UART_IPCLK, "gout_peri_usi_uart_ipclk",
+	     "mout_pll_peri_uart_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI_UART_IPCLKPORT_IPCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI_UART_PCLK, "gout_peri_usi_uart_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_USI_UART_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_USI00_I2C, "gout_peri_usi00_i2c",
+	     "mout_pll_peri_ip_user",
+	     CLK_CON_GAT_GATE_CLK_PERI_USI00_I2C,
+	     21, 0, 0),
+
+	/* WDT (Watchdog Timer) */
+	GATE(CLK_GOUT_PERI_WDT0_PCLK, "gout_peri_wdt0_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_WDT0_IPCLKPORT_PCLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_PERI_WDT1_PCLK, "gout_peri_wdt1_pclk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_WDT1_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* UNKNOWN */
+	GATE(CLK_GOUT_PERI_SLH_AXI_MI_P_CLK, "gout_peri_slh_axi_mi_p_clk",
+	     "mout_pll_peri_bus_user",
+	     CLK_CON_GAT_CLK_BLK_PERI_UID_SLH_AXI_MI_P_PERI_IPCLKPORT_I_CLK,
+	     21, 0, 0),
+};
+
+static const struct samsung_cmu_info peri_cmu_info __initconst = {
+	.mux_clks		= peri_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(peri_mux_clks),
+	.div_clks		= peri_div_clks,
+	.nr_div_clks		= ARRAY_SIZE(peri_div_clks),
+	.fixed_factor_clks	= peri_fixed_factor_clks,
+	.nr_fixed_factor_clks	= ARRAY_SIZE(peri_fixed_factor_clks),
+	.gate_clks		= peri_gate_clks,
+	.nr_gate_clks		= ARRAY_SIZE(peri_gate_clks),
+	.nr_clk_ids		= CLKS_NR_PERI,
+	.clk_regs		= peri_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(peri_clk_regs),
+	.sysreg_clk_regs	= drcg_sysreg,
+	.nr_sysreg_clk_regs	= ARRAY_SIZE(drcg_sysreg),
+	.clk_name		= "bus",
+	.auto_clock_gate	= true,
+	.gate_dbg_offset	= EXYNOS5515_GATE_DBG_OFFSET,
+	.option_offset		= PERI_CMU_PERI_CONTROLLER_OPTION,
+	.drcg_offset		= EXYNOS5515_DRCG_EN_OFFSET,
+};
+
+static void __init exynos5515_cmu_peri_init(struct device_node *np)
+{
+	exynos_arm64_register_cmu(NULL, np, &peri_cmu_info);
+}
+
+/* Register CMU_PERI early, as it's a dependency for the MCT. */
+CLK_OF_DECLARE(exynos5515_cmu_peri, "samsung,exynos5515-cmu-peri",
+	       exynos5515_cmu_peri_init);
+
+/* ---- CMU_FSYS ------------------------------------------------------------ */
+
+/*
+ * CMU_FSYS:	0x10400000
+ * SYSREG_FSYS:	0x10420000
+ */
+
+#define PLL_CON0_MUX_CLKCMU_FSYS_BUS_USER						0x0600
+#define PLL_CON1_MUX_CLKCMU_FSYS_BUS_USER						0x0604
+#define PLL_CON0_MUX_CLKCMU_FSYS_MMC_CARD_USER						0x0610
+#define PLL_CON1_MUX_CLKCMU_FSYS_MMC_CARD_USER						0x0614
+#define PLL_CON0_MUX_CLKCMU_FSYS_MMC_EMBD_USER						0x0620
+#define PLL_CON1_MUX_CLKCMU_FSYS_MMC_EMBD_USER						0x0624
+#define PLL_CON0_MUX_CLKCMU_FSYS_USB20DRD_USER						0x0630
+#define PLL_CON1_MUX_CLKCMU_FSYS_USB20DRD_USER						0x0634
+#define FSYS_CMU_FSYS_CONTROLLER_OPTION							0x0800
+#define CLK_CON_GAT_CLK_BLK_FSYS_UID_FSYS_CMU_FSYS_IPCLKPORT_PCLK			0x2000
+#define CLK_CON_GAT_CLK_BLK_FSYS_UID_MMC_CARD_IPCLKPORT_SDCLKIN				0x2004
+#define CLK_CON_GAT_CLK_BLK_FSYS_UID_MMC_EMBD_IPCLKPORT_SDCLKIN				0x2008
+#define CLK_CON_GAT_CLK_BLK_FSYS_UID_RSTNSYNC_CLK_FSYS_OSCCLK_IPCLKPORT_CLK		0x200c
+#define CLK_CON_GAT_CLK_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_I_USB20DRD_REF_CLK_50	0x2010
+#define CLK_CON_GAT_CLK_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_I_USB20_PHY_REFCLK_26	0x2014
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_D_TZPC_FSYS_IPCLKPORT_PCLK			0x2018
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_GPIO_FSYS_IPCLKPORT_PCLK				0x201c
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_MMC_CARD_IPCLKPORT_I_ACLK				0x2020
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_MMC_EMBD_IPCLKPORT_I_ACLK				0x2024
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_PPMU_FSYS_IPCLKPORT_ACLK				0x2028
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_PPMU_FSYS_IPCLKPORT_PCLK				0x202c
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_RSTNSYNC_CLK_FSYS_BUS_IPCLKPORT_CLK		0x2030
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_SLH_AXI_MI_P_FSYS_IPCLKPORT_I_CLK			0x2034
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_SLH_AXI_SI_D_FSYS_IPCLKPORT_I_CLK			0x2038
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_SYSREG_FSYS_IPCLKPORT_PCLK			0x203c
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_ACLK_PHYCTRL_20		0x2040
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_BUS_CLK_EARLY		0x2044
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_US_64TO128_FSYS_IPCLKPORT_ACLK			0x2048
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_XIU_D_FSYS_IPCLKPORT_ACLK				0x204c
+#define CLK_CON_GAT_GOUT_BLK_FSYS_UID_XIU_P_FSYS_IPCLKPORT_ACLK				0x2050
+
+static const unsigned long fsys_clk_regs[] __initconst = {
+	PLL_CON0_MUX_CLKCMU_FSYS_BUS_USER,
+	PLL_CON1_MUX_CLKCMU_FSYS_BUS_USER,
+	PLL_CON0_MUX_CLKCMU_FSYS_MMC_CARD_USER,
+	PLL_CON1_MUX_CLKCMU_FSYS_MMC_CARD_USER,
+	PLL_CON0_MUX_CLKCMU_FSYS_MMC_EMBD_USER,
+	PLL_CON1_MUX_CLKCMU_FSYS_MMC_EMBD_USER,
+	PLL_CON0_MUX_CLKCMU_FSYS_USB20DRD_USER,
+	PLL_CON1_MUX_CLKCMU_FSYS_USB20DRD_USER,
+	FSYS_CMU_FSYS_CONTROLLER_OPTION,
+	CLK_CON_GAT_CLK_BLK_FSYS_UID_FSYS_CMU_FSYS_IPCLKPORT_PCLK,
+	CLK_CON_GAT_CLK_BLK_FSYS_UID_MMC_CARD_IPCLKPORT_SDCLKIN,
+	CLK_CON_GAT_CLK_BLK_FSYS_UID_MMC_EMBD_IPCLKPORT_SDCLKIN,
+	CLK_CON_GAT_CLK_BLK_FSYS_UID_RSTNSYNC_CLK_FSYS_OSCCLK_IPCLKPORT_CLK,
+	CLK_CON_GAT_CLK_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_I_USB20DRD_REF_CLK_50,
+	CLK_CON_GAT_CLK_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_I_USB20_PHY_REFCLK_26,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_D_TZPC_FSYS_IPCLKPORT_PCLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_GPIO_FSYS_IPCLKPORT_PCLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_MMC_CARD_IPCLKPORT_I_ACLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_MMC_EMBD_IPCLKPORT_I_ACLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_PPMU_FSYS_IPCLKPORT_ACLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_PPMU_FSYS_IPCLKPORT_PCLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_RSTNSYNC_CLK_FSYS_BUS_IPCLKPORT_CLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_SLH_AXI_MI_P_FSYS_IPCLKPORT_I_CLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_SLH_AXI_SI_D_FSYS_IPCLKPORT_I_CLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_SYSREG_FSYS_IPCLKPORT_PCLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_ACLK_PHYCTRL_20,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_BUS_CLK_EARLY,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_US_64TO128_FSYS_IPCLKPORT_ACLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_XIU_D_FSYS_IPCLKPORT_ACLK,
+	CLK_CON_GAT_GOUT_BLK_FSYS_UID_XIU_P_FSYS_IPCLKPORT_ACLK,
+};
+
+/* Parent clock list for CMU_FSYS muxes */
+PNAME(mout_pll_fsys_bus_user_p)		= { "oscclk",
+					    "dout_cmu_fsys_bus" };
+
+/* Parent clock list for CMU_FSYS muxes: for MMC */
+PNAME(mout_pll_fsys_mmc_card_user_p)	= { "oscclk",
+					    "dout_cmu_fsys_mmc_card" };
+
+PNAME(mout_pll_fsys_mmc_embd_user_p)	= { "oscclk",
+					    "dout_cmu_fsys_mmc_embd" };
+
+/* Parent clock list for CMU_FSYS muxes: for USB */
+PNAME(mout_pll_fsys_usb20drd_user_p)	= { "oscclk",
+					    "dout_cmu_fsys_usb20drd" };
+
+static const struct samsung_mux_clock fsys_mux_clks[] __initconst = {
+	/* BUS */
+	MUX(CLK_MOUT_PLL_FSYS_BUS_USER, "mout_pll_fsys_bus_user",
+	    mout_pll_fsys_bus_user_p,
+	    PLL_CON0_MUX_CLKCMU_FSYS_BUS_USER,
+	    4, 1),
+
+	/* MMC */
+	MUX(CLK_MOUT_PLL_FSYS_MMC_CARD_USER, "mout_pll_fsys_mmc_card_user",
+	    mout_pll_fsys_mmc_card_user_p,
+	    PLL_CON0_MUX_CLKCMU_FSYS_MMC_CARD_USER,
+	    4, 1),
+	MUX(CLK_MOUT_PLL_FSYS_MMC_EMBD_USER, "mout_pll_fsys_mmc_embd_user",
+	    mout_pll_fsys_mmc_embd_user_p,
+	    PLL_CON0_MUX_CLKCMU_FSYS_MMC_EMBD_USER,
+	    4, 1),
+
+	/* USB */
+	MUX(CLK_MOUT_PLL_FSYS_USB20DRD_USER, "mout_pll_fsys_usb20drd_user",
+	    mout_pll_fsys_usb20drd_user_p,
+	    PLL_CON0_MUX_CLKCMU_FSYS_USB20DRD_USER,
+	    4, 1),
+};
+
+static const struct samsung_gate_clock fsys_gate_clks[] __initconst = {
+	/* BUS */
+	GATE(CLK_GOUT_FSYS_BUS_CLK, "gout_fsys_bus_clk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_RSTNSYNC_CLK_FSYS_BUS_IPCLKPORT_CLK,
+	     21, 0, 0),
+
+	/* CMU */
+	GATE(CLK_GOUT_FSYS_CMU_PCLK, "gout_fsys_cmu_pclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_CLK_BLK_FSYS_UID_FSYS_CMU_FSYS_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* GPIO */
+	GATE(CLK_GOUT_FSYS_GPIO_PCLK, "gout_fsys_gpio_pclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_GPIO_FSYS_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* MMC */
+	GATE(CLK_GOUT_FSYS_MMC_CARD_SDCLKIN, "gout_fsys_mmc_card_sdclkin",
+	     "mout_pll_fsys_mmc_card_user",
+	     CLK_CON_GAT_CLK_BLK_FSYS_UID_MMC_CARD_IPCLKPORT_SDCLKIN,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_MMC_EMBD_SDCLKIN, "gout_fsys_mmc_embd_sdclkin",
+	     "mout_pll_fsys_mmc_embd_user",
+	     CLK_CON_GAT_CLK_BLK_FSYS_UID_MMC_EMBD_IPCLKPORT_SDCLKIN,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_MMC_CARD_ACLK, "gout_fsys_mmc_card_aclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_MMC_CARD_IPCLKPORT_I_ACLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_MMC_EMBD_ACLK, "gout_fsys_mmc_embd_aclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_MMC_EMBD_IPCLKPORT_I_ACLK,
+	     21, 0, 0),
+
+	/* OSCCLK */
+	GATE(CLK_GOUT_FSYS_OSCCLK, "gout_fsys_oscclk",
+	     "oscclk",
+	     CLK_CON_GAT_CLK_BLK_FSYS_UID_RSTNSYNC_CLK_FSYS_OSCCLK_IPCLKPORT_CLK,
+	     21, 0, 0),
+
+	/* PMU */
+	GATE(CLK_GOUT_FSYS_PPMU_ACLK, "gout_fsys_ppmu_aclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_PPMU_FSYS_IPCLKPORT_ACLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_PPMU_PCLK, "gout_fsys_ppmu_pclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_PPMU_FSYS_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* SYSREG */
+	GATE(CLK_GOUT_FSYS_SYSREG_PCLK, "gout_fsys_sysreg_pclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_SYSREG_FSYS_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* TZ */
+	GATE(CLK_GOUT_FSYS_D_TZPC_PCLK, "gout_fsys_d_tzpc_pclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_D_TZPC_FSYS_IPCLKPORT_PCLK,
+	     21, 0, 0),
+
+	/* USB */
+	GATE(CLK_GOUT_FSYS_USB20DRD_REFCLK,
+	     "gout_fsys_usb20drd_refclk",
+	     "mout_pll_fsys_usb20drd_user",
+	     CLK_CON_GAT_CLK_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_I_USB20DRD_REF_CLK_50,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_USB20DRD_PHY_REFCLK,
+	     "gout_fsys_usb20drd_phy_refclk",
+	     "oscclk",
+	     CLK_CON_GAT_CLK_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_I_USB20_PHY_REFCLK_26,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_USB20DRD_ACLK_PHYCTRL,
+	     "gout_fsys_usb20drd_aclk_phyctrl",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_ACLK_PHYCTRL_20,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_USB20DRD_BUS_CLK_EARLY,
+	     "gout_fsys_usb20drd_bus_clk_early",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_USB20DRD_TOP_IPCLKPORT_BUS_CLK_EARLY,
+	     21, 0, 0),
+
+	/* UNKNOWN */
+	GATE(CLK_GOUT_FSYS_SLH_AXI_MI_P_CLK, "gout_fsys_slh_axi_mi_p_clk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_SLH_AXI_MI_P_FSYS_IPCLKPORT_I_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_SLH_AXI_SI_D_CLK, "gout_fsys_slh_axi_si_d_clk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_SLH_AXI_SI_D_FSYS_IPCLKPORT_I_CLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_US_64TO128_ACLK, "gout_fsys_us_64to128_aclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_US_64TO128_FSYS_IPCLKPORT_ACLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_XIU_D_ACLK, "gout_fsys_xiu_d_aclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_XIU_D_FSYS_IPCLKPORT_ACLK,
+	     21, 0, 0),
+	GATE(CLK_GOUT_FSYS_XIU_P_ACLK, "gout_fsys_xiu_p_aclk",
+	     "mout_pll_fsys_bus_user",
+	     CLK_CON_GAT_GOUT_BLK_FSYS_UID_XIU_P_FSYS_IPCLKPORT_ACLK,
+	     21, 0, 0),
+};
+
+static const struct samsung_cmu_info fsys_cmu_info __initconst = {
+	.mux_clks		= fsys_mux_clks,
+	.nr_mux_clks		= ARRAY_SIZE(fsys_mux_clks),
+	.gate_clks		= fsys_gate_clks,
+	.nr_gate_clks		= ARRAY_SIZE(fsys_gate_clks),
+	.nr_clk_ids		= CLKS_NR_FSYS,
+	.clk_regs		= fsys_clk_regs,
+	.nr_clk_regs		= ARRAY_SIZE(fsys_clk_regs),
+	.sysreg_clk_regs	= drcg_sysreg,
+	.nr_sysreg_clk_regs	= ARRAY_SIZE(drcg_sysreg),
+	.clk_name		= "bus",
+	.auto_clock_gate	= true,
+	.gate_dbg_offset	= EXYNOS5515_GATE_DBG_OFFSET,
+	.option_offset		= FSYS_CMU_FSYS_CONTROLLER_OPTION,
+	.drcg_offset		= EXYNOS5515_DRCG_EN_OFFSET,
+};
+
+/* ----- platform_driver ----- */
+
+static int __init exynos5515_cmu_probe(struct platform_device *pdev)
+{
+	const struct samsung_cmu_info *info;
+	struct device *dev = &pdev->dev;
+
+	info = of_device_get_match_data(dev);
+	exynos_arm64_register_cmu(dev, dev->of_node, info);
+
+	return 0;
+}
+
+static const struct of_device_id exynos5515_cmu_of_match[] = {
+	{
+		.compatible = "samsung,exynos5515-cmu-fsys",
+		.data = &fsys_cmu_info,
+	},
+	{ },
+};
+
+static struct platform_driver exynos5515_cmu_driver __refdata = {
+	.driver = {
+		.name = "exynos5515-cmu",
+		.of_match_table = exynos5515_cmu_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe = exynos5515_cmu_probe,
+};
+
+static int __init exynos5515_cmu_init(void)
+{
+	return platform_driver_register(&exynos5515_cmu_driver);
+}
+
+core_initcall(exynos5515_cmu_init);
diff --git a/drivers/clk/samsung/clk-pll.c b/drivers/clk/samsung/clk-pll.c
index e74552846ba341..3d837bd7435583 100644
--- a/drivers/clk/samsung/clk-pll.c
+++ b/drivers/clk/samsung/clk-pll.c
@@ -1591,8 +1591,8 @@ static const struct clk_ops samsung_a9fraco_clk_min_ops = {
 static void __init _samsung_clk_register_pll(struct samsung_clk_provider *ctx,
 				const struct samsung_pll_clock *pll_clk)
 {
+	struct clk_init_data init = {};
 	struct samsung_clk_pll *pll;
-	struct clk_init_data init;
 	unsigned int len = 0;
 	int ret;
 
@@ -1726,6 +1726,7 @@ static void __init _samsung_clk_register_pll(struct samsung_clk_provider *ctx,
 		else
 			init.ops = &samsung_pll2650xx_clk_ops;
 		break;
+	case pll_309:
 	case pll_531x:
 	case pll_4311:
 		init.ops = &samsung_pll531x_clk_ops;
diff --git a/drivers/clk/samsung/clk-pll.h b/drivers/clk/samsung/clk-pll.h
index d6eb3246611b86..2457cf47c109f0 100644
--- a/drivers/clk/samsung/clk-pll.h
+++ b/drivers/clk/samsung/clk-pll.h
@@ -53,6 +53,7 @@ enum samsung_pll_type {
 	pll_1031x,
 	pll_a9fracm,
 	pll_a9fraco,
+	pll_309,
 };
 
 #define PLL_RATE(_fin, _m, _p, _s, _k, _ks) \
diff --git a/drivers/clk/socfpga/clk-gate-a10.c b/drivers/clk/socfpga/clk-gate-a10.c
index dd8d1713aaff76..ba12179061b3fe 100644
--- a/drivers/clk/socfpga/clk-gate-a10.c
+++ b/drivers/clk/socfpga/clk-gate-a10.c
@@ -49,7 +49,7 @@ static void __init __socfpga_gate_init(struct device_node *node,
 	struct socfpga_gate_clk *socfpga_clk;
 	const char *clk_name = node->name;
 	const char *parent_name[SOCFPGA_MAX_PARENTS];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int rc;
 
 	socfpga_clk = kzalloc_obj(*socfpga_clk);
diff --git a/drivers/clk/socfpga/clk-gate-s10.c b/drivers/clk/socfpga/clk-gate-s10.c
index 24f56a9edc44e5..4b23909c581065 100644
--- a/drivers/clk/socfpga/clk-gate-s10.c
+++ b/drivers/clk/socfpga/clk-gate-s10.c
@@ -128,7 +128,7 @@ struct clk_hw *s10_register_gate(const struct stratix10_gate_clock *clks, void _
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_gate_clk *socfpga_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *parent_name = clks->parent_name;
 	int ret;
 
@@ -186,7 +186,7 @@ struct clk_hw *agilex_register_gate(const struct stratix10_gate_clock *clks, voi
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_gate_clk *socfpga_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *parent_name = clks->parent_name;
 	int ret;
 
@@ -244,7 +244,7 @@ struct clk_hw *agilex5_register_gate(const struct agilex5_gate_clock *clks, void
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_gate_clk *socfpga_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int ret;
 
 	socfpga_clk = kzalloc_obj(*socfpga_clk);
diff --git a/drivers/clk/socfpga/clk-gate.c b/drivers/clk/socfpga/clk-gate.c
index ced8e09884068a..d0f9fa44ad1825 100644
--- a/drivers/clk/socfpga/clk-gate.c
+++ b/drivers/clk/socfpga/clk-gate.c
@@ -143,7 +143,7 @@ void __init socfpga_gate_init(struct device_node *node)
 	struct socfpga_gate_clk *socfpga_clk;
 	const char *clk_name = node->name;
 	const char *parent_name[SOCFPGA_MAX_PARENTS];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_ops *ops;
 	int rc;
 
diff --git a/drivers/clk/socfpga/clk-periph-a10.c b/drivers/clk/socfpga/clk-periph-a10.c
index 4eb5787f0dea3f..8aefc58ba4ec39 100644
--- a/drivers/clk/socfpga/clk-periph-a10.c
+++ b/drivers/clk/socfpga/clk-periph-a10.c
@@ -65,7 +65,7 @@ static void __init __socfpga_periph_init(struct device_node *node,
 	struct socfpga_periph_clk *periph_clk;
 	const char *clk_name = node->name;
 	const char *parent_name[SOCFPGA_MAX_PARENTS];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int rc;
 	u32 fixed_div;
 	u32 div_reg[3];
diff --git a/drivers/clk/socfpga/clk-periph-s10.c b/drivers/clk/socfpga/clk-periph-s10.c
index 5195b9476da169..dc923d0d878697 100644
--- a/drivers/clk/socfpga/clk-periph-s10.c
+++ b/drivers/clk/socfpga/clk-periph-s10.c
@@ -103,7 +103,7 @@ struct clk_hw *s10_register_periph(const struct stratix10_perip_c_clock *clks,
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_periph_clk *periph_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	const char *parent_name = clks->parent_name;
 	int ret;
@@ -139,7 +139,7 @@ struct clk_hw *n5x_register_periph(const struct n5x_perip_c_clock *clks,
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_periph_clk *periph_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	const char *parent_name = clks->parent_name;
 	int ret;
@@ -174,7 +174,7 @@ struct clk_hw *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *c
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_periph_clk *periph_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	const char *parent_name = clks->parent_name;
 	int ret;
@@ -220,7 +220,7 @@ struct clk_hw *agilex5_register_cnt_periph(const struct agilex5_perip_cnt_clock
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_periph_clk *periph_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	int ret;
 
diff --git a/drivers/clk/socfpga/clk-periph.c b/drivers/clk/socfpga/clk-periph.c
index 3821db6777e754..8fa3d8e20042d7 100644
--- a/drivers/clk/socfpga/clk-periph.c
+++ b/drivers/clk/socfpga/clk-periph.c
@@ -55,7 +55,7 @@ static void __init __socfpga_periph_init(struct device_node *node,
 	struct socfpga_periph_clk *periph_clk;
 	const char *clk_name = node->name;
 	const char *parent_name[SOCFPGA_MAX_PARENTS];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int rc;
 	u32 fixed_div;
 	u32 div_reg[3];
diff --git a/drivers/clk/socfpga/clk-pll-a10.c b/drivers/clk/socfpga/clk-pll-a10.c
index c56f276e8d51e0..63b8b75045aead 100644
--- a/drivers/clk/socfpga/clk-pll-a10.c
+++ b/drivers/clk/socfpga/clk-pll-a10.c
@@ -71,7 +71,7 @@ static void __init __socfpga_pll_init(struct device_node *node,
 	struct socfpga_pll *pll_clk;
 	const char *clk_name = node->name;
 	const char *parent_name[SOCFGPA_MAX_PARENTS];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct device_node *clkmgr_np;
 	int rc;
 	int i = 0;
diff --git a/drivers/clk/socfpga/clk-pll-s10.c b/drivers/clk/socfpga/clk-pll-s10.c
index ee236e87da33c0..1dca54b48375b1 100644
--- a/drivers/clk/socfpga/clk-pll-s10.c
+++ b/drivers/clk/socfpga/clk-pll-s10.c
@@ -192,7 +192,7 @@ struct clk_hw *s10_register_pll(const struct stratix10_pll_clock *clks,
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_pll *pll_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	int ret;
 
@@ -232,7 +232,7 @@ struct clk_hw *agilex_register_pll(const struct stratix10_pll_clock *clks,
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_pll *pll_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	int ret;
 
@@ -271,7 +271,7 @@ struct clk_hw *n5x_register_pll(const struct stratix10_pll_clock *clks,
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_pll *pll_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	int ret;
 
@@ -310,7 +310,7 @@ struct clk_hw *agilex5_register_pll(const struct agilex5_pll_clock *clks,
 {
 	struct clk_hw *hw_clk;
 	struct socfpga_pll *pll_clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *name = clks->name;
 	int ret;
 
diff --git a/drivers/clk/socfpga/clk-pll.c b/drivers/clk/socfpga/clk-pll.c
index f85adb16d41404..aa13cc9d6db115 100644
--- a/drivers/clk/socfpga/clk-pll.c
+++ b/drivers/clk/socfpga/clk-pll.c
@@ -78,7 +78,7 @@ static void __init __socfpga_pll_init(struct device_node *node,
 	struct socfpga_pll *pll_clk;
 	const char *clk_name = node->name;
 	const char *parent_name[SOCFPGA_MAX_PARENTS];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct device_node *clkmgr_np;
 	int rc;
 
diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c
index 92b930d5ff3057..39df15c8a3e6d9 100644
--- a/drivers/clk/spacemit/ccu-k3.c
+++ b/drivers/clk/spacemit/ccu-k3.c
@@ -30,14 +30,87 @@ static const struct ccu_pll_rate_tbl pll2_rate_tbl[] = {
 };
 
 static const struct ccu_pll_rate_tbl pll3_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
+	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
+	CCU_PLLA_RATE(2050000000UL, 0x0b2a2d55, 0x00005500, 0xa0558686),
+	CCU_PLLA_RATE(2100000000UL, 0x0b2b3000, 0x00000000, 0xa0558787),
+	CCU_PLLA_RATE(2150000000UL, 0x0b2c32aa, 0x0000ab00, 0xa0558787),
 	CCU_PLLA_RATE(2200000000UL, 0x0b2d3555, 0x00005500, 0xa0558787),
+	CCU_PLLA_RATE(2250000000UL, 0x0b2e3800, 0x00000000, 0xa0558888),
+	CCU_PLLA_RATE(2300000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0558888),
+	CCU_PLLA_RATE(2350000000UL, 0x0b303d55, 0x00005500, 0xa0558888),
+	CCU_PLLA_RATE(2400000000UL, 0x0b320000, 0x00000000, 0xa0558989),
 };
 
 static const struct ccu_pll_rate_tbl pll4_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
+	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
+	CCU_PLLA_RATE(2050000000UL, 0x0b2a2d55, 0x00005500, 0xa0558686),
+	CCU_PLLA_RATE(2100000000UL, 0x0b2b3000, 0x00000000, 0xa0558787),
+	CCU_PLLA_RATE(2150000000UL, 0x0b2c32aa, 0x0000ab00, 0xa0558787),
 	CCU_PLLA_RATE(2200000000UL, 0x0b2d3555, 0x00005500, 0xa0558787),
+	CCU_PLLA_RATE(2250000000UL, 0x0b2e3800, 0x00000000, 0xa0558888),
+	CCU_PLLA_RATE(2300000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0558888),
+	CCU_PLLA_RATE(2350000000UL, 0x0b303d55, 0x00005500, 0xa0558888),
+	CCU_PLLA_RATE(2400000000UL, 0x0b320000, 0x00000000, 0xa0558989),
 };
 
 static const struct ccu_pll_rate_tbl pll5_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
 	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
 };
 
@@ -50,6 +123,25 @@ static const struct ccu_pll_rate_tbl pll7_rate_tbl[] = {
 };
 
 static const struct ccu_pll_rate_tbl pll8_rate_tbl[] = {
+	CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080),
+	CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181),
+	CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181),
+	CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181),
+	CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282),
+	CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282),
+	CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282),
+	CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383),
+	CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383),
+	CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383),
+	CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484),
+	CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484),
+	CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484),
+	CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585),
+	CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585),
+	CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585),
+	CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686),
+	CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686),
 	CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686),
 };
 
diff --git a/drivers/clk/spear/clk-aux-synth.c b/drivers/clk/spear/clk-aux-synth.c
index 61a8c3d4c1e40d..85cfb9b5ec3534 100644
--- a/drivers/clk/spear/clk-aux-synth.c
+++ b/drivers/clk/spear/clk-aux-synth.c
@@ -138,8 +138,8 @@ struct clk *clk_register_aux(const char *aux_name, const char *gate_name,
 	        const struct aux_clk_masks *masks, struct aux_rate_tbl *rtbl,
 		u8 rtbl_cnt, spinlock_t *lock, struct clk **gate_clk)
 {
+	struct clk_init_data init = {};
 	struct clk_aux *aux;
-	struct clk_init_data init;
 	struct clk *clk;
 
 	if (!aux_name || !parent_name || !reg || !rtbl || !rtbl_cnt) {
diff --git a/drivers/clk/spear/clk-frac-synth.c b/drivers/clk/spear/clk-frac-synth.c
index 3549f1cfdae116..1b5e184d9668fd 100644
--- a/drivers/clk/spear/clk-frac-synth.c
+++ b/drivers/clk/spear/clk-frac-synth.c
@@ -125,7 +125,7 @@ struct clk *clk_register_frac(const char *name, const char *parent_name,
 		unsigned long flags, void __iomem *reg,
 		struct frac_rate_tbl *rtbl, u8 rtbl_cnt, spinlock_t *lock)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_frac *frac;
 	struct clk *clk;
 
diff --git a/drivers/clk/spear/clk-gpt-synth.c b/drivers/clk/spear/clk-gpt-synth.c
index 1a431d7ec20c0f..411cacdfc11f51 100644
--- a/drivers/clk/spear/clk-gpt-synth.c
+++ b/drivers/clk/spear/clk-gpt-synth.c
@@ -114,7 +114,7 @@ struct clk *clk_register_gpt(const char *name, const char *parent_name, unsigned
 		long flags, void __iomem *reg, struct gpt_rate_tbl *rtbl, u8
 		rtbl_cnt, spinlock_t *lock)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_gpt *gpt;
 	struct clk *clk;
 
diff --git a/drivers/clk/spear/clk-vco-pll.c b/drivers/clk/spear/clk-vco-pll.c
index faba727e2f8436..21fcc522c738e6 100644
--- a/drivers/clk/spear/clk-vco-pll.c
+++ b/drivers/clk/spear/clk-vco-pll.c
@@ -284,7 +284,7 @@ struct clk *clk_register_vco_pll(const char *vco_name, const char *pll_name,
 	struct clk_vco *vco;
 	struct clk_pll *pll;
 	struct clk *vco_clk, *tpll_clk, *tvco_gate_clk;
-	struct clk_init_data vco_init, pll_init;
+	struct clk_init_data vco_init = {}, pll_init = {};
 	const char **vco_parent_name;
 
 	if (!vco_name || !pll_name || !parent_name || !mode_reg || !cfg_reg ||
diff --git a/drivers/clk/st/clk-flexgen.c b/drivers/clk/st/clk-flexgen.c
index 211535fd8ac9bf..46e5c7be43e827 100644
--- a/drivers/clk/st/clk-flexgen.c
+++ b/drivers/clk/st/clk-flexgen.c
@@ -209,7 +209,7 @@ static struct clk *clk_register_flexgen(const char *name,
 				unsigned long flexgen_flags, bool mode) {
 	struct flexgen *fgxbar;
 	struct clk *clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	u32  xbar_shift;
 	void __iomem *xbar_reg, *fdiv_reg;
 
diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c
index 3fcc1da20c04ed..705133b959d74e 100644
--- a/drivers/clk/st/clkgen-fsyn.c
+++ b/drivers/clk/st/clkgen-fsyn.c
@@ -444,9 +444,9 @@ static struct clk * __init st_clk_register_quadfs_pll(
 		struct clkgen_quadfs_data *quadfs, void __iomem *reg,
 		spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct st_clk_quadfs_pll *pll;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	/*
 	 * Sanity check required pointers.
@@ -885,8 +885,8 @@ static struct clk * __init st_clk_register_quadfs_fsynth(
 		unsigned long flags, spinlock_t *lock)
 {
 	struct st_clk_quadfs_fsynth *fs;
+	struct clk_init_data init = {};
 	struct clk *clk;
-	struct clk_init_data init;
 
 	/*
 	 * Sanity check required pointers, note that nsdiv3 is optional.
diff --git a/drivers/clk/st/clkgen-pll.c b/drivers/clk/st/clkgen-pll.c
index 17551c6a879ac4..aaa28aaff62725 100644
--- a/drivers/clk/st/clkgen-pll.c
+++ b/drivers/clk/st/clkgen-pll.c
@@ -652,9 +652,9 @@ static struct clk * __init clkgen_pll_register(const char *parent_name,
 				void __iomem *reg, unsigned long pll_flags,
 				const char *clk_name, spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clkgen_pll *pll;
 	struct clk *clk;
-	struct clk_init_data init;
 
 	pll = kzalloc_obj(*pll);
 	if (!pll)
diff --git a/drivers/clk/starfive/clk-starfive-jh7110-isp.c b/drivers/clk/starfive/clk-starfive-jh7110-isp.c
index f3fa069db193e3..c02c8b29a12304 100644
--- a/drivers/clk/starfive/clk-starfive-jh7110-isp.c
+++ b/drivers/clk/starfive/clk-starfive-jh7110-isp.c
@@ -130,7 +130,7 @@ static int jh7110_ispcrg_probe(struct platform_device *pdev)
 
 	/* enable power domain and clocks */
 	pm_runtime_enable(priv->dev);
-	ret = pm_runtime_get_sync(priv->dev);
+	ret = pm_runtime_resume_and_get(priv->dev);
 	if (ret < 0)
 		return dev_err_probe(priv->dev, ret, "failed to turn on power\n");
 
diff --git a/drivers/clk/starfive/clk-starfive-jh7110-sys.c b/drivers/clk/starfive/clk-starfive-jh7110-sys.c
index 03c17cd2032f5a..847c0a39e5ab55 100644
--- a/drivers/clk/starfive/clk-starfive-jh7110-sys.c
+++ b/drivers/clk/starfive/clk-starfive-jh7110-sys.c
@@ -390,6 +390,9 @@ static int jh7110_pll0_clk_notifier_cb(struct notifier_block *nb,
 	if (action == PRE_RATE_CHANGE) {
 		struct clk *osc = clk_get(priv->dev, "osc");
 
+		if (IS_ERR(osc))
+			return notifier_from_errno(PTR_ERR(osc));
+
 		priv->original_clk = clk_get_parent(cpu_root);
 		ret = clk_set_parent(cpu_root, osc);
 		clk_put(osc);
diff --git a/drivers/clk/starfive/clk-starfive-jh7110-vout.c b/drivers/clk/starfive/clk-starfive-jh7110-vout.c
index bad20d5d794a72..aeba42cd3c44ba 100644
--- a/drivers/clk/starfive/clk-starfive-jh7110-vout.c
+++ b/drivers/clk/starfive/clk-starfive-jh7110-vout.c
@@ -40,10 +40,12 @@ static const struct jh71x0_clk_data jh7110_voutclk_data[] = {
 	JH71X0_GATE(JH7110_VOUTCLK_DC8200_AXI, "dc8200_axi", 0, JH7110_VOUTCLK_VOUT_TOP_AXI),
 	JH71X0_GATE(JH7110_VOUTCLK_DC8200_CORE, "dc8200_core", 0, JH7110_VOUTCLK_VOUT_TOP_AXI),
 	JH71X0_GATE(JH7110_VOUTCLK_DC8200_AHB, "dc8200_ahb", 0, JH7110_VOUTCLK_VOUT_TOP_AHB),
-	JH71X0_GMUX(JH7110_VOUTCLK_DC8200_PIX0, "dc8200_pix0", 0, 2,
+	JH71X0_GMUX(JH7110_VOUTCLK_DC8200_PIX0, "dc8200_pix0",
+		    CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, 2,
 		    JH7110_VOUTCLK_DC8200_PIX,
 		    JH7110_VOUTCLK_HDMITX0_PIXELCLK),
-	JH71X0_GMUX(JH7110_VOUTCLK_DC8200_PIX1, "dc8200_pix1", 0, 2,
+	JH71X0_GMUX(JH7110_VOUTCLK_DC8200_PIX1, "dc8200_pix1",
+		    CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, 2,
 		    JH7110_VOUTCLK_DC8200_PIX,
 		    JH7110_VOUTCLK_HDMITX0_PIXELCLK),
 	/* LCD */
diff --git a/drivers/clk/stm32/clk-stm32mp1.c b/drivers/clk/stm32/clk-stm32mp1.c
index 2d9ccd96ec98c0..b265e3f3e94c28 100644
--- a/drivers/clk/stm32/clk-stm32mp1.c
+++ b/drivers/clk/stm32/clk-stm32mp1.c
@@ -902,8 +902,8 @@ static struct clk_hw *clk_register_pll(struct device *dev, const char *name,
 				       unsigned long flags,
 				       spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct stm32_pll_obj *element;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int err;
 
@@ -1041,8 +1041,8 @@ static struct clk_hw *clk_register_cktim(struct device *dev, const char *name,
 					 void __iomem *timpre,
 					 spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct timer_cker *tim_ker;
-	struct clk_init_data init;
 	struct clk_hw *hw;
 	int err;
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c b/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c
index db0e36d8838e74..349230e28c1c3a 100644
--- a/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c
+++ b/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c
@@ -43,27 +43,24 @@ static SUNXI_CCU_M_DATA_WITH_MUX(r_apb1_clk, "r-apb1",
 				 24, 3,	/* mux */
 				 0);
 
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(r_cpu_timer0, "r-timer0",
-				       r_ahb_apb_parents, 0x100,
-				       0, 0,	/* no M */
-				       1, 3,	/* P */
-				       4, 3,	/* mux */
-				       BIT(0),
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(r_cpu_timer0, "r-timer0",
+				      r_ahb_apb_parents, 0x100,
+				      1, 3,	/* P */
+				      4, 3,	/* mux */
+				      BIT(0),
+				      0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(r_cpu_timer1, "r-timer1",
+				      r_ahb_apb_parents, 0x104,
+				      1, 3,	/* P */
+				      4, 3,	/* mux */
+				      BIT(0),
+				      0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(r_cpu_timer2, "r-timer2",
+				      r_ahb_apb_parents, 0x108,
+				      1, 3,	/* P */
+				      4, 3,	/* mux */
+				      BIT(0),
 				      0);
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(r_cpu_timer1, "r-timer1",
-				       r_ahb_apb_parents, 0x104,
-				       0, 0,	/* no M */
-				       1, 3,	/* P */
-				       4, 3,	/* mux */
-				       BIT(0),
-				       0);
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(r_cpu_timer2, "r-timer2",
-				       r_ahb_apb_parents, 0x108,
-				       0, 0,	/* no M */
-				       1, 3,	/* P */
-				       4, 3,	/* mux */
-				       BIT(0),
-				       0);
 
 static SUNXI_CCU_GATE_HW(bus_r_timer_clk, "bus-r-timer", &r_ahb_clk.common.hw,
 			 0x11c, BIT(0), 0);
diff --git a/drivers/clk/sunxi-ng/ccu-sun55i-a523.c b/drivers/clk/sunxi-ng/ccu-sun55i-a523.c
index 20dad06b37ca31..0714090cc3b08a 100644
--- a/drivers/clk/sunxi-ng/ccu-sun55i-a523.c
+++ b/drivers/clk/sunxi-ng/ccu-sun55i-a523.c
@@ -380,14 +380,13 @@ static const struct clk_parent_data mbus_parents[] = {
 	{ .hw = &pll_periph1_150M_clk.hw },
 	{ .fw_name = "hosc" },
 };
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_FEAT(mbus_clk, "mbus", mbus_parents,
-					    0x540,
-					    0, 5,		/* M */
-					    0, 0,		/* no P */
-					    24, 3,	/* mux */
-					    BIT(31),	/* gate */
-					    CLK_IS_CRITICAL,
-					    CCU_FEATURE_UPDATE_BIT);
+static SUNXI_CCU_M_DATA_WITH_MUX_GATE_FEAT(mbus_clk, "mbus", mbus_parents,
+					   0x540,
+					   0, 5,	/* M */
+					   24, 3,	/* mux */
+					   BIT(31),	/* gate */
+					   CLK_IS_CRITICAL,
+					   CCU_FEATURE_UPDATE_BIT);
 
 static const struct clk_hw *mbus_hws[] = { &mbus_clk.common.hw };
 
@@ -513,58 +512,47 @@ static const struct clk_parent_data hstimer_parents[] = {
 	{ .fw_name = "losc" },
 	{ .hw = &pll_periph0_200M_clk.hw },
 };
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(hstimer0_clk, "hstimer0",
-				       hstimer_parents, 0x730,
-				       0, 0,	/* M */
-				       0, 3,	/* P */
-				       24, 3,	/* mux */
-				       BIT(31),	/* gate */
-				       0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(hstimer0_clk, "hstimer0",
+				      hstimer_parents, 0x730,
+				      0, 3,	/* P */
+				      24, 3,	/* mux */
+				      BIT(31),	/* gate */
+				      0);
 
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(hstimer1_clk, "hstimer1",
-				       hstimer_parents,
-				       0x734,
-				       0, 0,	/* M */
-				       0, 3,	/* P */
-				       24, 3,	/* mux */
-				       BIT(31),	/* gate */
-				       0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(hstimer1_clk, "hstimer1",
+				      hstimer_parents, 0x734,
+				      0, 3,	/* P */
+				      24, 3,	/* mux */
+				      BIT(31),	/* gate */
+				      0);
 
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(hstimer2_clk, "hstimer2",
-				       hstimer_parents,
-				       0x738,
-				       0, 0,	/* M */
-				       0, 3,	/* P */
-				       24, 3,	/* mux */
-				       BIT(31),	/* gate */
-				       0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(hstimer2_clk, "hstimer2",
+				      hstimer_parents, 0x738,
+				      0, 3,	/* P */
+				      24, 3,	/* mux */
+				      BIT(31),	/* gate */
+				      0);
 
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(hstimer3_clk, "hstimer3",
-				       hstimer_parents,
-				       0x73c,
-				       0, 0,	/* M */
-				       0, 3,	/* P */
-				       24, 3,	/* mux */
-				       BIT(31),	/* gate */
-				       0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(hstimer3_clk, "hstimer3",
+				      hstimer_parents, 0x73c,
+				      0, 3,	/* P */
+				      24, 3,	/* mux */
+				      BIT(31),	/* gate */
+				      0);
 
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(hstimer4_clk, "hstimer4",
-				       hstimer_parents,
-				       0x740,
-				       0, 0,	/* M */
-				       0, 3,	/* P */
-				       24, 3,	/* mux */
-				       BIT(31),	/* gate */
-				       0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(hstimer4_clk, "hstimer4",
+				      hstimer_parents, 0x740,
+				      0, 3,	/* P */
+				      24, 3,	/* mux */
+				      BIT(31),	/* gate */
+				      0);
 
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE(hstimer5_clk, "hstimer5",
-				       hstimer_parents,
-				       0x744,
-				       0, 0,	/* M */
-				       0, 3,	/* P */
-				       24, 3,	/* mux */
-				       BIT(31),	/* gate */
-				       0);
+static SUNXI_CCU_P_DATA_WITH_MUX_GATE(hstimer5_clk, "hstimer5",
+				      hstimer_parents, 0x744,
+				      0, 3,	/* P */
+				      24, 3,	/* mux */
+				      BIT(31),	/* gate */
+				      0);
 
 static SUNXI_CCU_GATE_HWS(bus_hstimer_clk, "bus-hstimer", ahb_hws, 0x74c,
 			  BIT(0), 0);
@@ -584,14 +572,13 @@ static const struct clk_parent_data iommu_parents[] = {
 	{ .fw_name = "hosc" },
 };
 
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_FEAT(iommu_clk, "iommu", iommu_parents,
-					    0x7b0,
-					    0, 5,	/* M */
-					    0, 0,	/* no P */
-					    24, 3,	/* mux */
-					    BIT(31),	/* gate */
-					    CLK_SET_RATE_PARENT,
-					    CCU_FEATURE_UPDATE_BIT);
+static SUNXI_CCU_M_DATA_WITH_MUX_GATE_FEAT(iommu_clk, "iommu", iommu_parents,
+					   0x7b0,
+					   0, 5,	/* M */
+					   24, 3,	/* mux */
+					   BIT(31),	/* gate */
+					   CLK_SET_RATE_PARENT,
+					   CCU_FEATURE_UPDATE_BIT);
 
 static SUNXI_CCU_GATE_HWS(bus_iommu_clk, "bus-iommu", apb0_hws, 0x7bc,
 			  BIT(0), 0);
@@ -603,14 +590,13 @@ static const struct clk_parent_data dram_parents[] = {
 	{ .hw = &pll_periph0_400M_clk.hw },
 	{ .hw = &pll_periph0_150M_clk.hw },
 };
-static SUNXI_CCU_MP_DATA_WITH_MUX_GATE_FEAT(dram_clk, "dram", dram_parents,
-					    0x800,
-					    0, 5,	/* M */
-					    0, 0,	/* no P */
-					    24, 3,	/* mux */
-					    BIT(31),	/* gate */
-					    CLK_IS_CRITICAL,
-					    CCU_FEATURE_UPDATE_BIT);
+static SUNXI_CCU_M_DATA_WITH_MUX_GATE_FEAT(dram_clk, "dram", dram_parents,
+					   0x800,
+					   0, 5,	/* M */
+					   24, 3,	/* mux */
+					   BIT(31),	/* gate */
+					   CLK_IS_CRITICAL,
+					   CCU_FEATURE_UPDATE_BIT);
 
 static SUNXI_CCU_GATE_HWS(mbus_dma_clk, "mbus-dma", mbus_hws,
 			  0x804, BIT(0), 0);
diff --git a/drivers/clk/sunxi-ng/ccu_common.c b/drivers/clk/sunxi-ng/ccu_common.c
index 43d8eca6abeed8..248e58c9562d8a 100644
--- a/drivers/clk/sunxi-ng/ccu_common.c
+++ b/drivers/clk/sunxi-ng/ccu_common.c
@@ -35,7 +35,7 @@ void ccu_helper_wait_for_lock(struct ccu_common *common, u32 lock)
 	else
 		addr = common->base + common->reg;
 
-	WARN_ON(readl_relaxed_poll_timeout(addr, reg, reg & lock, 100, 70000));
+	WARN_ON(readl_relaxed_poll_timeout_atomic(addr, reg, reg & lock, 100, 70000));
 }
 EXPORT_SYMBOL_NS_GPL(ccu_helper_wait_for_lock, "SUNXI_CCU");
 
diff --git a/drivers/clk/sunxi-ng/ccu_div.h b/drivers/clk/sunxi-ng/ccu_div.h
index a30a92780a0515..6d3541a3b51910 100644
--- a/drivers/clk/sunxi-ng/ccu_div.h
+++ b/drivers/clk/sunxi-ng/ccu_div.h
@@ -212,16 +212,17 @@ struct ccu_div {
 	SUNXI_CCU_M_WITH_GATE(_struct, _name, _parent, _reg,		\
 			      _mshift, _mwidth, 0, _flags)
 
-#define SUNXI_CCU_M_DATA_WITH_MUX_GATE(_struct, _name, _parents, _reg,	\
-				       _mshift, _mwidth,		\
-				       _muxshift, _muxwidth,		\
-				       _gate, _flags)			\
+#define SUNXI_CCU_M_DATA_WITH_MUX_GATE_FEAT(_struct, _name, _parents, _reg, \
+					    _mshift, _mwidth,		\
+					    _muxshift, _muxwidth,	\
+					    _gate, _flags, _features)	\
 	struct ccu_div _struct = {					\
 		.enable	= _gate,					\
 		.div	= _SUNXI_CCU_DIV(_mshift, _mwidth),		\
 		.mux	= _SUNXI_CCU_MUX(_muxshift, _muxwidth),		\
 		.common	= {						\
 			.reg		= _reg,				\
+			.features	= _features,			\
 			.hw.init	= CLK_HW_INIT_PARENTS_DATA(_name, \
 								   _parents, \
 								   &ccu_div_ops, \
@@ -229,6 +230,15 @@ struct ccu_div {
 		},							\
 	}
 
+#define SUNXI_CCU_M_DATA_WITH_MUX_GATE(_struct, _name, _parents, _reg,	\
+				       _mshift, _mwidth,		\
+				       _muxshift, _muxwidth,		\
+				       _gate, _flags)			\
+	SUNXI_CCU_M_DATA_WITH_MUX_GATE_FEAT(_struct, _name, _parents, _reg, \
+					    _mshift, _mwidth,		\
+					    _muxshift, _muxwidth,	\
+					    _gate, _flags, 0)
+
 #define SUNXI_CCU_M_DATA_WITH_MUX(_struct, _name, _parents, _reg,	\
 				  _mshift, _mwidth,			\
 				  _muxshift, _muxwidth,			\
diff --git a/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c b/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c
index b61eba112a7fec..b7f55864ea2a51 100644
--- a/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c
+++ b/drivers/clk/sunxi/clk-sun4i-tcon-ch1.c
@@ -225,7 +225,7 @@ static void __init tcon_ch1_setup(struct device_node *node)
 {
 	const char *parents[TCON_CH1_SCLK2_PARENTS];
 	const char *clk_name = node->name;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct tcon_ch1_clk *tclk;
 	struct resource res;
 	struct clk *clk;
diff --git a/drivers/clk/tegra/clk-audio-sync.c b/drivers/clk/tegra/clk-audio-sync.c
index 0bbfcf46bedd52..0fb4377c80bf2a 100644
--- a/drivers/clk/tegra/clk-audio-sync.c
+++ b/drivers/clk/tegra/clk-audio-sync.c
@@ -47,7 +47,7 @@ struct clk *tegra_clk_register_sync_source(const char *name,
 					   unsigned long max_rate)
 {
 	struct tegra_clk_sync_source *sync;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk *clk;
 
 	sync = kzalloc_obj(*sync);
diff --git a/drivers/clk/tegra/clk-divider.c b/drivers/clk/tegra/clk-divider.c
index 27f43c3da1563e..2dc77134334517 100644
--- a/drivers/clk/tegra/clk-divider.c
+++ b/drivers/clk/tegra/clk-divider.c
@@ -145,8 +145,8 @@ struct clk *tegra_clk_register_divider(const char *name,
 		u8 frac_width, spinlock_t *lock)
 {
 	struct tegra_clk_frac_div *divider;
+	struct clk_init_data init = {};
 	struct clk *clk;
-	struct clk_init_data init;
 
 	divider = kzalloc_obj(*divider);
 	if (!divider) {
diff --git a/drivers/clk/tegra/clk-periph-fixed.c b/drivers/clk/tegra/clk-periph-fixed.c
index bb33e32cad4a17..9ea9d41f479631 100644
--- a/drivers/clk/tegra/clk-periph-fixed.c
+++ b/drivers/clk/tegra/clk-periph-fixed.c
@@ -77,7 +77,7 @@ struct clk *tegra_clk_register_periph_fixed(const char *name,
 {
 	const struct tegra_clk_periph_regs *regs;
 	struct tegra_clk_periph_fixed *fixed;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk *clk;
 
 	regs = get_reg_bank(num);
diff --git a/drivers/clk/tegra/clk-periph-gate.c b/drivers/clk/tegra/clk-periph-gate.c
index fa39d64875154e..d821e1b9f77e6b 100644
--- a/drivers/clk/tegra/clk-periph-gate.c
+++ b/drivers/clk/tegra/clk-periph-gate.c
@@ -139,7 +139,7 @@ struct clk *tegra_clk_register_periph_gate(const char *name,
 {
 	struct tegra_clk_periph_gate *gate;
 	struct clk *clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const struct tegra_clk_periph_regs *pregs;
 
 	pregs = get_reg_bank(clk_num);
diff --git a/drivers/clk/tegra/clk-periph.c b/drivers/clk/tegra/clk-periph.c
index 6ebeaa7cb65648..7af832d9af9b01 100644
--- a/drivers/clk/tegra/clk-periph.c
+++ b/drivers/clk/tegra/clk-periph.c
@@ -166,7 +166,7 @@ static struct clk *_tegra_clk_register_periph(const char *name,
 			unsigned long flags)
 {
 	struct clk *clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const struct tegra_clk_periph_regs *bank;
 	bool div = !(periph->gate.flags & TEGRA_PERIPH_NO_DIV);
 
diff --git a/drivers/clk/tegra/clk-pll-out.c b/drivers/clk/tegra/clk-pll-out.c
index 2299545fc386f8..285b0af811fb19 100644
--- a/drivers/clk/tegra/clk-pll-out.c
+++ b/drivers/clk/tegra/clk-pll-out.c
@@ -90,8 +90,8 @@ struct clk *tegra_clk_register_pll_out(const char *name,
 		spinlock_t *lock)
 {
 	struct tegra_clk_pll_out *pll_out;
+	struct clk_init_data init = {};
 	struct clk *clk;
-	struct clk_init_data init;
 
 	pll_out = kzalloc_obj(*pll_out);
 	if (!pll_out)
diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c
index adfb74f111ef1a..fc3abe86414ddd 100644
--- a/drivers/clk/tegra/clk-pll.c
+++ b/drivers/clk/tegra/clk-pll.c
@@ -1903,7 +1903,7 @@ static struct clk *_tegra_clk_register_pll(struct tegra_clk_pll *pll,
 		const char *name, const char *parent_name, unsigned long flags,
 		const struct clk_ops *ops)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 
 	init.name = name;
 	init.ops = ops;
diff --git a/drivers/clk/tegra/clk-sdmmc-mux.c b/drivers/clk/tegra/clk-sdmmc-mux.c
index 14b5439a100a0c..74fe06fa546abf 100644
--- a/drivers/clk/tegra/clk-sdmmc-mux.c
+++ b/drivers/clk/tegra/clk-sdmmc-mux.c
@@ -236,7 +236,7 @@ struct clk *tegra_clk_register_sdmmc_mux_div(const char *name,
 	unsigned long flags, void *lock)
 {
 	struct clk *clk;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const struct tegra_clk_periph_regs *bank;
 	struct tegra_sdmmc_mux *sdmmc_mux;
 
diff --git a/drivers/clk/tegra/clk-super.c b/drivers/clk/tegra/clk-super.c
index 370445e3d5acc3..7aa3c3774445d3 100644
--- a/drivers/clk/tegra/clk-super.c
+++ b/drivers/clk/tegra/clk-super.c
@@ -204,8 +204,8 @@ struct clk *tegra_clk_register_super_mux(const char *name,
 		u8 width, u8 pllx_index, u8 div2_index, spinlock_t *lock)
 {
 	struct tegra_clk_super_mux *super;
+	struct clk_init_data init = {};
 	struct clk *clk;
-	struct clk_init_data init;
 
 	super = kzalloc_obj(*super);
 	if (!super)
@@ -240,8 +240,8 @@ struct clk *tegra_clk_register_super_clk(const char *name,
 		spinlock_t *lock)
 {
 	struct tegra_clk_super_mux *super;
+	struct clk_init_data init = {};
 	struct clk *clk;
-	struct clk_init_data init;
 
 	super = kzalloc_obj(*super);
 	if (!super)
diff --git a/drivers/clk/tegra/clk-tegra-super-cclk.c b/drivers/clk/tegra/clk-tegra-super-cclk.c
index e7aeb67606b113..1047a1104ec1f5 100644
--- a/drivers/clk/tegra/clk-tegra-super-cclk.c
+++ b/drivers/clk/tegra/clk-tegra-super-cclk.c
@@ -135,8 +135,8 @@ struct clk *tegra_clk_register_super_cclk(const char *name,
 		spinlock_t *lock)
 {
 	struct tegra_clk_super_mux *super;
+	struct clk_init_data init = {};
 	struct clk *clk;
-	struct clk_init_data init;
 	u32 val;
 
 	if (WARN_ON(cclk_super))
diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c
index 94ac24ea1e6adc..0fc74808fe39b2 100644
--- a/drivers/clk/tegra/clk-tegra124-emc.c
+++ b/drivers/clk/tegra/clk-tegra124-emc.c
@@ -485,8 +485,8 @@ static const struct clk_ops tegra_clk_emc_ops = {
 struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np,
 				      spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct tegra_clk_emc *tegra;
-	struct clk_init_data init;
 	struct device_node *node;
 	u32 node_ram_code;
 	struct clk *clk;
diff --git a/drivers/clk/tegra/clk-tegra20-emc.c b/drivers/clk/tegra/clk-tegra20-emc.c
index 44465726a8a670..5cd026e8f305a3 100644
--- a/drivers/clk/tegra/clk-tegra20-emc.c
+++ b/drivers/clk/tegra/clk-tegra20-emc.c
@@ -245,8 +245,8 @@ bool tegra20_clk_emc_driver_available(struct clk_hw *emc_hw)
 
 struct clk *tegra20_clk_register_emc(void __iomem *ioaddr, bool low_jitter)
 {
+	struct clk_init_data init = {};
 	struct tegra_clk_emc *emc;
-	struct clk_init_data init;
 	struct clk *clk;
 
 	emc = kzalloc_obj(*emc);
diff --git a/drivers/clk/tegra/clk-tegra210-emc.c b/drivers/clk/tegra/clk-tegra210-emc.c
index 9f140d94bef755..2e559d95d40e7d 100644
--- a/drivers/clk/tegra/clk-tegra210-emc.c
+++ b/drivers/clk/tegra/clk-tegra210-emc.c
@@ -274,8 +274,8 @@ static const struct clk_ops tegra210_clk_emc_ops = {
 struct clk *tegra210_clk_register_emc(struct device_node *np,
 				      void __iomem *regs)
 {
+	struct clk_init_data init = {};
 	struct tegra210_clk_emc *emc;
-	struct clk_init_data init;
 	struct clk *clk;
 
 	emc = kzalloc_obj(*emc);
diff --git a/drivers/clk/tegra/clk.h b/drivers/clk/tegra/clk.h
index 73efd2ff37c98b..afa93ba505abff 100644
--- a/drivers/clk/tegra/clk.h
+++ b/drivers/clk/tegra/clk.h
@@ -255,7 +255,7 @@ struct tegra_clk_pll;
  * @set_gain:			Callback to adjust N div for SDM enabled
  *				PLL's based on fractional divider value.
  * @calc_rate:			Callback used to change how out of table
- *				rates (dividers and multipler) are calculated.
+ *				rates (dividers and multiplier) are calculated.
  * @adjust_vco:			Callback to adjust the programming range of the
  *				divider range (if SDM is present)
  * @set_defaults:		Callback which will try to initialize PLL
diff --git a/drivers/clk/tenstorrent/atlantis-prcm.c b/drivers/clk/tenstorrent/atlantis-prcm.c
index 6d4386eeb7daea..6d28aa2ba8816e 100644
--- a/drivers/clk/tenstorrent/atlantis-prcm.c
+++ b/drivers/clk/tenstorrent/atlantis-prcm.c
@@ -205,14 +205,9 @@ static int atlantis_clk_gate_endisable(struct clk_hw *hw, int enable)
 {
 	struct atlantis_clk_gate *gate = hw_to_atlantis_clk_gate(hw);
 
-	if (enable)
-		return regmap_set_bits(gate->common.regmap,
-				       gate->config.reg_offset,
-				       gate->config.enable);
-	else
-		return regmap_clear_bits(gate->common.regmap,
-					 gate->config.reg_offset,
-					 gate->config.enable);
+	return regmap_assign_bits(gate->common.regmap,
+				  gate->config.reg_offset,
+				  gate->config.enable, enable);
 }
 
 static int atlantis_clk_gate_enable(struct clk_hw *hw)
@@ -397,22 +392,21 @@ static int atlantis_clk_gate_shared_enable(struct clk_hw *hw)
 
 	scoped_guard(spinlock_irqsave, gate->config.refcount_lock)
 	{
-		need_enable = (*gate->config.share_count)++ == 0;
+		need_enable = (*gate->config.share_count) == 0;
 		if (need_enable) {
 			regmap_set_bits(gate->common.regmap,
 					gate->config.reg_offset,
 					gate->config.enable);
-		}
-	}
 
-	if (need_enable) {
-		if (!regmap_test_bits(gate->common.regmap,
-				      gate->config.reg_offset,
-				      gate->config.enable)) {
-			pr_warn("%s: gate enable %d failed to enable\n",
-				clk_hw_get_name(hw), gate->config.enable);
-			return -EIO;
+			if (!regmap_test_bits(gate->common.regmap,
+					      gate->config.reg_offset,
+					      gate->config.enable)) {
+				pr_warn("%s: gate enable %d failed to enable\n",
+					clk_hw_get_name(hw), gate->config.enable);
+				return -EIO;
+			}
 		}
+		(*gate->config.share_count)++;
 	}
 
 	return 0;
@@ -789,13 +783,14 @@ static int atlantis_prcm_clocks_register(struct device *dev,
 {
 	struct clk_hw_onecell_data *clk_data;
 	int i, ret;
-	size_t num_clks = data->num;
 
 	clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, data->num),
 				GFP_KERNEL);
 	if (!clk_data)
 		return -ENOMEM;
 
+	clk_data->num = data->num;
+
 	for (i = 0; i < data->num; i++) {
 		struct clk_hw *hw = data->hws[i];
 		struct atlantis_clk_common *common =
@@ -809,8 +804,6 @@ static int atlantis_prcm_clocks_register(struct device *dev,
 		clk_data->hws[common->clkid] = hw;
 	}
 
-	clk_data->num = num_clks;
-
 	return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data);
 }
 
diff --git a/drivers/clk/ti/adpll.c b/drivers/clk/ti/adpll.c
index 8885d28face50b..0d729c9980a55b 100644
--- a/drivers/clk/ti/adpll.c
+++ b/drivers/clk/ti/adpll.c
@@ -925,6 +925,7 @@ static void ti_adpll_remove(struct platform_device *pdev)
 {
 	struct ti_adpll_data *d = dev_get_drvdata(&pdev->dev);
 
+	of_clk_del_provider(d->np);
 	ti_adpll_free_resources(d);
 }
 
diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c
index c897ad7e681ef0..aedbe78bf1e409 100644
--- a/drivers/clk/ti/clockdomain.c
+++ b/drivers/clk/ti/clockdomain.c
@@ -104,7 +104,7 @@ int omap2_init_clk_clkdm(struct clk_hw *hw)
 	if (!clk->clkdm_name)
 		return 0;
 
-	clk_name = __clk_get_name(hw->clk);
+	clk_name = clk_hw_get_name(hw);
 
 	clkdm = ti_clk_ll_ops->clkdm_lookup(clk->clkdm_name);
 	if (clkdm) {
diff --git a/drivers/clk/ti/composite.c b/drivers/clk/ti/composite.c
index 83c3592cd179e1..c379bbdae25aa0 100644
--- a/drivers/clk/ti/composite.c
+++ b/drivers/clk/ti/composite.c
@@ -52,7 +52,7 @@ static const struct clk_ops ti_composite_gate_ops = {
 
 struct component_clk {
 	int num_parents;
-	struct clk_parent_data *parent_data;
+	const char **parent_names;
 	struct device_node *node;
 	int type;
 	struct clk_hw *hw;
@@ -116,7 +116,7 @@ static void __init _register_composite(void *user,
 	struct clk_hw_omap_comp *cclk = to_clk_hw_comp(hw);
 	struct component_clk *comp;
 	int num_parents = 0;
-	struct clk_parent_data *parent_data = NULL;
+	const char **parent_names = NULL;
 	const char *name;
 	int i;
 	int ret;
@@ -155,7 +155,7 @@ static void __init _register_composite(void *user,
 			continue;
 		if (comp->num_parents) {
 			num_parents = comp->num_parents;
-			parent_data = comp->parent_data;
+			parent_names = comp->parent_names;
 			break;
 		}
 	}
@@ -166,8 +166,8 @@ static void __init _register_composite(void *user,
 	}
 
 	name = ti_dt_clk_name(node);
-	clk = clk_register_composite_pdata(NULL, name,
-				     parent_data, num_parents,
+	clk = clk_register_composite(NULL, name,
+				     parent_names, num_parents,
 				     _get_hw(cclk, CLK_COMPONENT_TYPE_MUX),
 				     &ti_clk_mux_ops,
 				     _get_hw(cclk, CLK_COMPONENT_TYPE_DIVIDER),
@@ -190,7 +190,7 @@ static void __init _register_composite(void *user,
 		if (!cclk->comp_clks[i])
 			continue;
 		list_del(&cclk->comp_clks[i]->link);
-		kfree(cclk->comp_clks[i]->parent_data);
+		kfree(cclk->comp_clks[i]->parent_names);
 		kfree(cclk->comp_clks[i]);
 	}
 
@@ -237,9 +237,8 @@ int __init ti_clk_add_component(struct device_node *node, struct clk_hw *hw,
 				int type)
 {
 	unsigned int num_parents;
-	struct clk_parent_data *parent_data;
+	const char **parent_names;
 	struct component_clk *clk;
-	unsigned int i;
 
 	num_parents = of_clk_get_parent_count(node);
 
@@ -248,21 +247,20 @@ int __init ti_clk_add_component(struct device_node *node, struct clk_hw *hw,
 		return -EINVAL;
 	}
 
-	parent_data = kzalloc_objs(*parent_data, num_parents);
-	if (!parent_data)
+	parent_names = kcalloc(num_parents, sizeof(char *), GFP_KERNEL);
+	if (!parent_names)
 		return -ENOMEM;
 
-	for (i = 0; i < num_parents; i++)
-		parent_data[i].index = i;
+	of_clk_parent_fill(node, parent_names, num_parents);
 
 	clk = kzalloc_obj(*clk);
 	if (!clk) {
-		kfree(parent_data);
+		kfree(parent_names);
 		return -ENOMEM;
 	}
 
 	clk->num_parents = num_parents;
-	clk->parent_data = parent_data;
+	clk->parent_names = parent_names;
 	clk->hw = hw;
 	clk->node = node;
 	clk->type = type;
diff --git a/drivers/clk/uniphier/clk-uniphier-cpugear.c b/drivers/clk/uniphier/clk-uniphier-cpugear.c
index a2f01a4da12769..165c6832b722fa 100644
--- a/drivers/clk/uniphier/clk-uniphier-cpugear.c
+++ b/drivers/clk/uniphier/clk-uniphier-cpugear.c
@@ -79,7 +79,7 @@ struct clk_hw *uniphier_clk_register_cpugear(struct device *dev,
 				const struct uniphier_clk_cpugear_data *data)
 {
 	struct uniphier_clk_cpugear *gear;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int ret;
 
 	gear = devm_kzalloc(dev, sizeof(*gear), GFP_KERNEL);
diff --git a/drivers/clk/uniphier/clk-uniphier-fixed-factor.c b/drivers/clk/uniphier/clk-uniphier-fixed-factor.c
index a6a37a6bca3ab0..c86683fce20549 100644
--- a/drivers/clk/uniphier/clk-uniphier-fixed-factor.c
+++ b/drivers/clk/uniphier/clk-uniphier-fixed-factor.c
@@ -13,8 +13,8 @@ struct clk_hw *uniphier_clk_register_fixed_factor(struct device *dev,
 						  const char *name,
 			const struct uniphier_clk_fixed_factor_data *data)
 {
+	struct clk_init_data init = {};
 	struct clk_fixed_factor *fix;
-	struct clk_init_data init;
 	int ret;
 
 	fix = devm_kzalloc(dev, sizeof(*fix), GFP_KERNEL);
diff --git a/drivers/clk/uniphier/clk-uniphier-fixed-rate.c b/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
index 3bc55ab75314bf..63f49d3bab80ba 100644
--- a/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
+++ b/drivers/clk/uniphier/clk-uniphier-fixed-rate.c
@@ -13,8 +13,8 @@ struct clk_hw *uniphier_clk_register_fixed_rate(struct device *dev,
 						const char *name,
 				const struct uniphier_clk_fixed_rate_data *data)
 {
+	struct clk_init_data init = {};
 	struct clk_fixed_rate *fixed;
-	struct clk_init_data init;
 	int ret;
 
 	/* allocate fixed-rate clock */
diff --git a/drivers/clk/uniphier/clk-uniphier-gate.c b/drivers/clk/uniphier/clk-uniphier-gate.c
index e58093e79dd7dc..9f49e6bb6b2042 100644
--- a/drivers/clk/uniphier/clk-uniphier-gate.c
+++ b/drivers/clk/uniphier/clk-uniphier-gate.c
@@ -61,8 +61,8 @@ struct clk_hw *uniphier_clk_register_gate(struct device *dev,
 					  const char *name,
 				const struct uniphier_clk_gate_data *data)
 {
+	struct clk_init_data init = {};
 	struct uniphier_clk_gate *gate;
-	struct clk_init_data init;
 	int ret;
 
 	gate = devm_kzalloc(dev, sizeof(*gate), GFP_KERNEL);
diff --git a/drivers/clk/uniphier/clk-uniphier-mux.c b/drivers/clk/uniphier/clk-uniphier-mux.c
index 1998e9d4cfc022..a63ad4df8e65eb 100644
--- a/drivers/clk/uniphier/clk-uniphier-mux.c
+++ b/drivers/clk/uniphier/clk-uniphier-mux.c
@@ -58,8 +58,8 @@ struct clk_hw *uniphier_clk_register_mux(struct device *dev,
 					 const char *name,
 				const struct uniphier_clk_mux_data *data)
 {
+	struct clk_init_data init = {};
 	struct uniphier_clk_mux *mux;
-	struct clk_init_data init;
 	int ret;
 
 	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
diff --git a/drivers/clk/ux500/clk-prcc.c b/drivers/clk/ux500/clk-prcc.c
index dce235df2576df..dde72d2fd0ac67 100644
--- a/drivers/clk/ux500/clk-prcc.c
+++ b/drivers/clk/ux500/clk-prcc.c
@@ -97,8 +97,8 @@ static struct clk *clk_reg_prcc(const char *name,
 				unsigned long flags,
 				const struct clk_ops *clk_prcc_ops)
 {
+	struct clk_init_data clk_prcc_init = {};
 	struct clk_prcc *clk;
-	struct clk_init_data clk_prcc_init;
 	struct clk *clk_reg;
 
 	if (!name) {
diff --git a/drivers/clk/ux500/clk-prcmu.c b/drivers/clk/ux500/clk-prcmu.c
index ac96c46bd1bbf7..d89f3b634f8bbd 100644
--- a/drivers/clk/ux500/clk-prcmu.c
+++ b/drivers/clk/ux500/clk-prcmu.c
@@ -200,8 +200,8 @@ static struct clk_hw *clk_reg_prcmu(const char *name,
 				    unsigned long flags,
 				    const struct clk_ops *clk_prcmu_ops)
 {
+	struct clk_init_data clk_prcmu_init = {};
 	struct clk_prcmu *clk;
-	struct clk_init_data clk_prcmu_init;
 	int ret;
 
 	if (!name) {
@@ -357,8 +357,8 @@ struct clk_hw *clk_reg_prcmu_clkout(const char *name,
 				    u8 source, u8 divider)
 
 {
+	struct clk_init_data clk_prcmu_clkout_init = {};
 	struct clk_prcmu_clkout *clk;
-	struct clk_init_data clk_prcmu_clkout_init;
 	u8 clkout_id;
 	int ret;
 
diff --git a/drivers/clk/ux500/clk-sysctrl.c b/drivers/clk/ux500/clk-sysctrl.c
index ba3258c88d28e1..2127a949e56ccf 100644
--- a/drivers/clk/ux500/clk-sysctrl.c
+++ b/drivers/clk/ux500/clk-sysctrl.c
@@ -127,8 +127,8 @@ static struct clk *clk_reg_sysctrl(struct device *dev,
 				unsigned long flags,
 				const struct clk_ops *clk_sysctrl_ops)
 {
+	struct clk_init_data clk_sysctrl_init = {};
 	struct clk_sysctrl *clk;
-	struct clk_init_data clk_sysctrl_init;
 	struct clk *clk_reg;
 	int i;
 
diff --git a/drivers/clk/versatile/clk-icst.c b/drivers/clk/versatile/clk-icst.c
index 6bd3db54dd229d..37334afee040b9 100644
--- a/drivers/clk/versatile/clk-icst.c
+++ b/drivers/clk/versatile/clk-icst.c
@@ -360,8 +360,8 @@ struct clk *icst_clk_setup(struct device *dev,
 {
 	struct clk *clk;
 	struct clk_icst *icst;
-	struct clk_init_data init;
 	struct icst_params *pclone;
+	struct clk_init_data init = {};
 
 	icst = kzalloc_obj(*icst);
 	if (!icst)
@@ -516,16 +516,19 @@ static void __init of_syscon_icst_setup(struct device_node *np)
 	map = syscon_node_to_regmap(parent);
 	if (IS_ERR(map)) {
 		pr_err("no regmap for syscon ICST clock parent\n");
+		of_node_put(parent);
 		return;
 	}
 
 	if (of_property_read_u32(np, "reg", &icst_desc.vco_offset) &&
 	    of_property_read_u32(np, "vco-offset", &icst_desc.vco_offset)) {
 		pr_err("no VCO register offset for ICST clock\n");
+		of_node_put(parent);
 		return;
 	}
 	if (of_property_read_u32(np, "lock-offset", &icst_desc.lock_offset)) {
 		pr_err("no lock register offset for ICST clock\n");
+		of_node_put(parent);
 		return;
 	}
 
@@ -552,6 +555,7 @@ static void __init of_syscon_icst_setup(struct device_node *np)
 		ctype = ICST_INTEGRATOR_CP_CM_MEM;
 	} else {
 		pr_err("unknown ICST clock %pOF\n", np);
+		of_node_put(parent);
 		return;
 	}
 
diff --git a/drivers/clk/versatile/clk-sp810.c b/drivers/clk/versatile/clk-sp810.c
index b0e69686f7a9a9..28e5c0cf1fda38 100644
--- a/drivers/clk/versatile/clk-sp810.c
+++ b/drivers/clk/versatile/clk-sp810.c
@@ -86,7 +86,7 @@ static void __init clk_sp810_of_setup(struct device_node *node)
 	const char *parent_names[2];
 	int num = ARRAY_SIZE(parent_names);
 	char name[12];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	static int instance;
 	int i;
 	bool deprecated;
diff --git a/drivers/clk/versatile/clk-vexpress-osc.c b/drivers/clk/versatile/clk-vexpress-osc.c
index 9adbf5c33bd1ca..72e872ec6b7c9b 100644
--- a/drivers/clk/versatile/clk-vexpress-osc.c
+++ b/drivers/clk/versatile/clk-vexpress-osc.c
@@ -64,7 +64,7 @@ static const struct clk_ops vexpress_osc_ops = {
 
 static int vexpress_osc_probe(struct platform_device *pdev)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct vexpress_osc *osc;
 	u32 range[2];
 	int ret;
diff --git a/drivers/clk/x86/clk-pmc-atom.c b/drivers/clk/x86/clk-pmc-atom.c
index 08c83e0abc41d9..43dda274e329d1 100644
--- a/drivers/clk/x86/clk-pmc-atom.c
+++ b/drivers/clk/x86/clk-pmc-atom.c
@@ -151,8 +151,8 @@ static struct clk_plt *plt_clk_register(struct platform_device *pdev, int id,
 					const char **parent_names,
 					int num_parents)
 {
+	struct clk_init_data init = {};
 	struct clk_plt *pclk;
-	struct clk_init_data init;
 	int ret;
 
 	pclk = devm_kzalloc(&pdev->dev, sizeof(*pclk), GFP_KERNEL);
diff --git a/drivers/clk/xilinx/clk-xlnx-clock-wizard.c b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c
index edf8c0eaca27ca..71abd46b3593f0 100644
--- a/drivers/clk/xilinx/clk-xlnx-clock-wizard.c
+++ b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c
@@ -881,10 +881,10 @@ static int clk_wzrd_dynamic_reconfig_f(struct clk_hw *hw, unsigned long rate,
 	struct clk_wzrd_divider *divider = to_clk_wzrd_divider(hw);
 	void __iomem *div_addr = divider->base + divider->offset;
 
-	rate_div = DIV_ROUND_DOWN_ULL(parent_rate * 1000, rate);
+	rate_div = DIV_ROUND_DOWN_ULL((u64)parent_rate * 1000, rate);
 	clockout0_div = rate_div / 1000;
 
-	pre = DIV_ROUND_CLOSEST((parent_rate * 1000), rate);
+	pre = DIV_ROUND_CLOSEST_ULL((u64)parent_rate * 1000, rate);
 	f = (u32)(pre - (clockout0_div * 1000));
 	f = f & WZRD_CLKOUT_FRAC_MASK;
 	f = f << WZRD_CLKOUT_DIVIDE_WIDTH;
@@ -936,9 +936,9 @@ static struct clk_hw *clk_wzrd_register_divf(struct device *dev,
 					  u32 div_type,
 					  spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clk_wzrd_divider *div;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL);
@@ -980,9 +980,9 @@ static struct clk_hw *clk_wzrd_ver_register_divider(struct device *dev,
 						 u32 div_type,
 						 spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clk_wzrd_divider *div;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL);
@@ -1026,9 +1026,9 @@ static struct clk_hw *clk_wzrd_register_divider(struct device *dev,
 					     u32 div_type,
 					     spinlock_t *lock)
 {
+	struct clk_init_data init = {};
 	struct clk_wzrd_divider *div;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	div = devm_kzalloc(dev, sizeof(*div), GFP_KERNEL);
@@ -1091,7 +1091,7 @@ static int clk_wzrd_clk_notifier(struct notifier_block *nb, unsigned long event,
 	}
 }
 
-static int __maybe_unused clk_wzrd_suspend(struct device *dev)
+static int clk_wzrd_suspend(struct device *dev)
 {
 	struct clk_wzrd *clk_wzrd = dev_get_drvdata(dev);
 
@@ -1101,7 +1101,7 @@ static int __maybe_unused clk_wzrd_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused clk_wzrd_resume(struct device *dev)
+static int clk_wzrd_resume(struct device *dev)
 {
 	int ret;
 	struct clk_wzrd *clk_wzrd = dev_get_drvdata(dev);
@@ -1117,7 +1117,7 @@ static int __maybe_unused clk_wzrd_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(clk_wzrd_dev_pm_ops, clk_wzrd_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(clk_wzrd_dev_pm_ops, clk_wzrd_suspend,
 			 clk_wzrd_resume);
 
 static const struct versal_clk_data versal_data = {
@@ -1378,7 +1378,7 @@ static struct platform_driver clk_wzrd_driver = {
 	.driver = {
 		.name = "clk-wizard",
 		.of_match_table = clk_wzrd_ids,
-		.pm = &clk_wzrd_dev_pm_ops,
+		.pm = pm_sleep_ptr(&clk_wzrd_dev_pm_ops),
 	},
 	.probe = clk_wzrd_probe,
 };
diff --git a/drivers/clk/xilinx/xlnx_vcu.c b/drivers/clk/xilinx/xlnx_vcu.c
index f14bda375e353e..ccd9d7c0b18098 100644
--- a/drivers/clk/xilinx/xlnx_vcu.c
+++ b/drivers/clk/xilinx/xlnx_vcu.c
@@ -406,9 +406,9 @@ static struct clk_hw *xvcu_register_pll(struct device *dev,
 					const char *name, const char *parent,
 					unsigned long flags)
 {
+	struct clk_init_data init = {};
 	struct vcu_pll *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	init.name = name;
diff --git a/drivers/clk/zynqmp/clk-gate-zynqmp.c b/drivers/clk/zynqmp/clk-gate-zynqmp.c
index db10eb69f3fe01..1a462af7b8ea3e 100644
--- a/drivers/clk/zynqmp/clk-gate-zynqmp.c
+++ b/drivers/clk/zynqmp/clk-gate-zynqmp.c
@@ -109,10 +109,10 @@ struct clk_hw *zynqmp_clk_register_gate(const char *name, u32 clk_id,
 					u8 num_parents,
 					const struct clock_topology *nodes)
 {
+	struct clk_init_data init = {};
 	struct zynqmp_clk_gate *gate;
 	struct clk_hw *hw;
 	int ret;
-	struct clk_init_data init;
 
 	/* allocate the gate */
 	gate = kzalloc_obj(*gate);
diff --git a/drivers/clk/zynqmp/clk-mux-zynqmp.c b/drivers/clk/zynqmp/clk-mux-zynqmp.c
index e37bd3516974c1..79509e387b51fe 100644
--- a/drivers/clk/zynqmp/clk-mux-zynqmp.c
+++ b/drivers/clk/zynqmp/clk-mux-zynqmp.c
@@ -133,9 +133,9 @@ struct clk_hw *zynqmp_clk_register_mux(const char *name, u32 clk_id,
 				       u8 num_parents,
 				       const struct clock_topology *nodes)
 {
+	struct clk_init_data init = {};
 	struct zynqmp_clk_mux *mux;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	mux = kzalloc_obj(*mux);
diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c
index 4075f6ecf076a8..3eabd1cf211ddb 100644
--- a/drivers/clk/zynqmp/divider.c
+++ b/drivers/clk/zynqmp/divider.c
@@ -278,9 +278,9 @@ struct clk_hw *zynqmp_clk_register_divider(const char *name,
 					   u8 num_parents,
 					   const struct clock_topology *nodes)
 {
+	struct clk_init_data init = {};
 	struct zynqmp_clk_divider *div;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	/* allocate the divider */
diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c
index 6463787cf57b60..6158386ca743d1 100644
--- a/drivers/clk/zynqmp/pll.c
+++ b/drivers/clk/zynqmp/pll.c
@@ -313,9 +313,9 @@ struct clk_hw *zynqmp_clk_register_pll(const char *name, u32 clk_id,
 				       u8 num_parents,
 				       const struct clock_topology *nodes)
 {
+	struct clk_init_data init = {};
 	struct zynqmp_pll *pll;
 	struct clk_hw *hw;
-	struct clk_init_data init;
 	int ret;
 
 	init.name = name;
diff --git a/drivers/clocksource/timer-clint.c b/drivers/clocksource/timer-clint.c
index 0bdd9d7ec54583..e56eee7e3781dd 100644
--- a/drivers/clocksource/timer-clint.c
+++ b/drivers/clocksource/timer-clint.c
@@ -243,7 +243,7 @@ static int __init clint_timer_init_dt(struct device_node *np)
 	}
 
 #ifdef CONFIG_SMP
-	rc = ipi_mux_create(BITS_PER_BYTE, clint_send_ipi);
+	rc = ipi_mux_create(IPI_MAX, clint_send_ipi);
 	if (rc <= 0) {
 		pr_err("unable to create muxed IPIs\n");
 		rc = (rc < 0) ? rc : -ENODEV;
@@ -251,7 +251,7 @@ static int __init clint_timer_init_dt(struct device_node *np)
 	}
 
 	irq_set_chained_handler(clint_ipi_irq, clint_ipi_interrupt);
-	riscv_ipi_set_virq_range(rc, BITS_PER_BYTE);
+	riscv_ipi_set_virq_range(rc, IPI_MAX);
 	clint_clear_ipi();
 #endif
 
diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c
index 6787acac9a4387..922a8f14562c95 100644
--- a/drivers/clocksource/timer-ti-dm.c
+++ b/drivers/clocksource/timer-ti-dm.c
@@ -1528,7 +1528,7 @@ static int omap_dm_timer_probe(struct platform_device *pdev)
  */
 static void omap_dm_timer_remove(struct platform_device *pdev)
 {
-	struct dmtimer *timer;
+	struct dmtimer *timer, *found = NULL;
 	unsigned long flags;
 	int ret = -EINVAL;
 
@@ -1536,14 +1536,17 @@ static void omap_dm_timer_remove(struct platform_device *pdev)
 	list_for_each_entry(timer, &omap_timer_list, node)
 		if (!strcmp(dev_name(&timer->pdev->dev),
 			    dev_name(&pdev->dev))) {
-			if (!(timer->capability & OMAP_TIMER_ALWON))
-				cpu_pm_unregister_notifier(&timer->nb);
 			list_del(&timer->node);
+			found = timer;
 			ret = 0;
 			break;
 		}
 	spin_unlock_irqrestore(&dm_timer_lock, flags);
 
+	/* Unregister outside the lock: cpu_pm_unregister_notifier() may sleep. */
+	if (found && !(found->capability & OMAP_TIMER_ALWON))
+		cpu_pm_unregister_notifier(&found->nb);
+
 	pm_runtime_disable(&pdev->dev);
 
 	if (ret)
diff --git a/drivers/cpufreq/airoha-cpufreq.c b/drivers/cpufreq/airoha-cpufreq.c
index 3e7770860d13c8..8283c528d0c145 100644
--- a/drivers/cpufreq/airoha-cpufreq.c
+++ b/drivers/cpufreq/airoha-cpufreq.c
@@ -64,7 +64,7 @@ static int airoha_cpufreq_probe(struct platform_device *pdev)
 	/* Attach PM for OPP */
 	ret = dev_pm_domain_attach_list(cpu_dev, &attach_data,
 					&priv->pd_list);
-	if (ret)
+	if (ret < 0)
 		goto clear_opp_config;
 
 	cpufreq_dt = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c
index e23773680e050d..c2c1a166b3a9e7 100644
--- a/drivers/cpufreq/amd-pstate-ut.c
+++ b/drivers/cpufreq/amd-pstate-ut.c
@@ -226,11 +226,14 @@ static int amd_pstate_ut_check_freq(u32 index)
 	for_each_online_cpu(cpu) {
 		struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL;
 		struct amd_cpudata *cpudata;
+		union perf_cached perf;
 
 		policy = cpufreq_cpu_get(cpu);
 		if (!policy)
 			continue;
+
 		cpudata = policy->driver_data;
+		perf = READ_ONCE(cpudata->perf);
 
 		if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) &&
 			(cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) &&
@@ -242,7 +245,25 @@ static int amd_pstate_ut_check_freq(u32 index)
 			return -EINVAL;
 		}
 
-		if (cpudata->lowest_nonlinear_freq != policy->min) {
+		if (perf.bios_min_perf) {
+			u32 bios_min_freq = perf_to_freq(perf, cpudata->nominal_freq,
+							 perf.bios_min_perf);
+
+			/*
+			 * User set bios_min_freq cannot be trusted to
+			 * be within the driver limits. Clamp it similar
+			 * to cpufreq_verify_within_cpu_limits().
+			 */
+			bios_min_freq = clamp_t(u32, bios_min_freq,
+						     policy->cpuinfo.min_freq,
+						     policy->cpuinfo.max_freq);
+
+			if (bios_min_freq != policy->min) {
+				pr_err("%s cpu%d bios_min_freq=%d policy_min=%d, they should be equal!\n",
+					__func__, cpu, bios_min_freq, policy->min);
+				return -EINVAL;
+			}
+		} else if (cpudata->lowest_nonlinear_freq != policy->min) {
 			pr_err("%s cpu%d cpudata_lowest_nonlinear_freq=%d policy_min=%d, they should be equal!\n",
 				__func__, cpu, cpudata->lowest_nonlinear_freq, policy->min);
 			return -EINVAL;
diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c
index 8bfd46d60843dd..fbccbe86c94d03 100644
--- a/drivers/cpufreq/amd-pstate.c
+++ b/drivers/cpufreq/amd-pstate.c
@@ -55,10 +55,10 @@
 #define AMD_PSTATE_TRANSITION_DELAY	1000
 #define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600
 
-#define AMD_CPPC_EPP_PERFORMANCE		0x00
-#define AMD_CPPC_EPP_BALANCE_PERFORMANCE	0x80
-#define AMD_CPPC_EPP_BALANCE_POWERSAVE		0xBF
-#define AMD_CPPC_EPP_POWERSAVE			0xFF
+#define AMD_CPPC_EPP_LEGACY_PERFORMANCE			0x00
+#define AMD_CPPC_EPP_LEGACY_BALANCE_PERFORMANCE		0x80
+#define AMD_CPPC_EPP_LEGACY_BALANCE_POWERSAVE		0xBF
+#define AMD_CPPC_EPP_LEGACY_POWERSAVE			0xFF
 
 static const char * const amd_pstate_mode_string[] = {
 	[AMD_PSTATE_UNDEFINED]   = "undefined",
@@ -129,14 +129,112 @@ static const char * const energy_perf_strings[] = {
 };
 static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX);
 
-static unsigned int epp_values[] = {
-	[EPP_INDEX_DEFAULT] = 0,
-	[EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
-	[EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
-	[EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
-	[EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
+/*
+ * The numeric EPP value programmed for each named preference. First dimension
+ * is CPU type (TOPO_CPU_TYPE_ANY for non-hybrid, TOPO_CPU_TYPE_PERFORMANCE/
+ * EFFICIENCY/LOW_POWER for hybrid). The initializer holds the legacy values
+ * used as the fallback on any platform not listed in amd_pstate_epp_soc_ids[];
+ * amd_pstate_init_epp_values() overwrites slots at boot when the running SoC
+ * has a per-SoC (and potentially per-CPU-type) override.
+ */
+static u8 epp_values[][EPP_INDEX_MAX] = {
+	/*
+	 * Initialize all CPU types to legacy defaults.
+	 * amd_pstate_init_epp_values() will fix these up
+	 * based on the platform during boot.
+	 */
+	[TOPO_CPU_TYPE_ANY ... TOPO_CPU_TYPE_LOW_POWER] = {
+		[EPP_INDEX_DEFAULT] = 0,
+		[EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_LEGACY_PERFORMANCE,
+		[EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_LEGACY_BALANCE_PERFORMANCE,
+		[EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_LEGACY_BALANCE_POWERSAVE,
+		[EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_LEGACY_POWERSAVE,
+	},
+};
+static_assert(ARRAY_SIZE(epp_values) == TOPO_CPU_TYPE_LOW_POWER + 1,
+	      "epp_values must have entries for all CPU types up to TOPO_CPU_TYPE_LOW_POWER");
+
+/*
+ * Get the EPP value row for a given CPU, accounting for hybrid CPU types.
+ * Non-hybrid systems use TOPO_CPU_TYPE_ANY; hybrid systems use the CPU's
+ * actual type (PERFORMANCE/EFFICIENCY/LOW_POWER).
+ */
+static inline u8 *amd_pstate_cpu_epp_values(enum x86_topology_cpu_type cpu_type)
+{
+	switch (cpu_type) {
+	case TOPO_CPU_TYPE_PERFORMANCE:
+	case TOPO_CPU_TYPE_EFFICIENCY:
+	case TOPO_CPU_TYPE_LOW_POWER:
+		return epp_values[cpu_type];
+	default:
+		return epp_values[TOPO_CPU_TYPE_ANY];
+	}
+}
+
+/**
+ * struct amd_pstate_epp_values - EPP values for the four named preferences
+ * @performance:	value for the "performance" preference
+ * @balance_performance: value for the "balance_performance" preference
+ * @balance_power:	value for the "balance_power" preference
+ * @power:		value for the "power" preference
+ */
+struct amd_pstate_epp_values {
+	u8 performance;
+	u8 balance_performance;
+	u8 balance_power;
+	u8 power;
+};
+
+/**
+ * struct amd_pstate_epp_soc - per-CPU-type EPP overrides for hybrid systems
+ * @performance_core:	values for TOPO_CPU_TYPE_PERFORMANCE cores
+ * @efficiency_core:	values for TOPO_CPU_TYPE_EFFICIENCY cores
+ * @low_power_core:	values for TOPO_CPU_TYPE_LOW_POWER cores
+ *
+ * Referenced from amd_pstate_epp_soc_ids[] to give a hybrid platform its own
+ * numeric EPP values for the four named preferences, with distinct values per
+ * CPU type. Non-hybrid systems are not listed in the table and always use the
+ * legacy defaults.
+ */
+struct amd_pstate_epp_soc {
+	struct amd_pstate_epp_values performance_core;
+	struct amd_pstate_epp_values efficiency_core;
+	struct amd_pstate_epp_values low_power_core;
+};
+
+/*
+ * Per-CPU-type EPP overrides for hybrid systems. Only hybrid SoCs should be
+ * listed here; non-hybrid systems always use the legacy defaults.
+ */
+static const struct amd_pstate_epp_soc epp_soc_zen6_client __initconst = {
+	.performance_core = {
+		.performance		= 25,
+		.balance_performance	= 51,
+		.balance_power		= 64,
+		.power			= 64,
+	},
+	.efficiency_core = {
+		.performance		= 25,
+		.balance_performance	= 51,
+		.balance_power		= 64,
+		.power			= 115,
+	},
+	.low_power_core = {
+		.performance		= 25,
+		.balance_performance	= 51,
+		.balance_power		= 64,
+		.power			= 115,
+	},
+};
+
+static const struct x86_cpu_id amd_pstate_epp_soc_ids[] __initconst = {
+	X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x80, &epp_soc_zen6_client),
+	X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x81, &epp_soc_zen6_client),
+	X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x84, &epp_soc_zen6_client),
+	X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0x85, &epp_soc_zen6_client),
+	X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x1A, 0xe0, &epp_soc_zen6_client),
+	{}
 };
-static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 2);
 
 typedef int (*cppc_mode_transition_fn)(int);
 
@@ -145,19 +243,6 @@ static struct quirk_entry quirk_amd_7k62 = {
 	.lowest_freq = 550,
 };
 
-static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val)
-{
-	u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq);
-
-	return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf);
-}
-
-static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val)
-{
-	return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val,
-				perf.nominal_perf);
-}
-
 static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi)
 {
 	/**
@@ -544,7 +629,11 @@ static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf,
 			     u8 des_perf, u8 max_perf, u8 epp, bool fast_switch)
 {
 	struct amd_cpudata *cpudata = policy->driver_data;
-	struct cppc_perf_ctrls perf_ctrls;
+	struct cppc_perf_ctrls perf_ctrls = {
+		.max_perf = max_perf,
+		.min_perf = min_perf,
+		.desired_perf = des_perf,
+	};
 	u64 value, prev;
 	int ret;
 
@@ -577,10 +666,6 @@ static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf,
 	if (value == prev)
 		return 0;
 
-	perf_ctrls.max_perf = max_perf;
-	perf_ctrls.min_perf = min_perf;
-	perf_ctrls.desired_perf = des_perf;
-
 	ret = cppc_set_perf(cpudata->cpu, &perf_ctrls);
 	if (ret)
 		return ret;
@@ -1068,6 +1153,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
 		return -ENOMEM;
 
 	cpudata->cpu = policy->cpu;
+	cpudata->cpu_type = cpu_data(policy->cpu).topo.cpu_type;
 
 	ret = amd_pstate_init_perf(cpudata);
 	if (ret)
@@ -1195,13 +1281,16 @@ static int amd_pstate_power_supply_notifier(struct notifier_block *nb,
 static int amd_pstate_get_epp_from_platform_profile(struct cpufreq_policy *policy,
 						    enum platform_profile_option profile)
 {
+	struct amd_cpudata *cpudata = policy->driver_data;
+	u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type);
+
 	switch (profile) {
 	case PLATFORM_PROFILE_PERFORMANCE:
-		return AMD_CPPC_EPP_PERFORMANCE;
+		return values[EPP_INDEX_PERFORMANCE];
 	case PLATFORM_PROFILE_BALANCED:
 		return amd_pstate_get_balanced_epp(policy);
 	case PLATFORM_PROFILE_LOW_POWER:
-		return AMD_CPPC_EPP_POWERSAVE;
+		return values[EPP_INDEX_POWERSAVE];
 	default:
 		break;
 	}
@@ -1411,6 +1500,7 @@ ssize_t store_energy_performance_preference(struct cpufreq_policy *policy,
 				    const char *buf, size_t count)
 {
 	struct amd_cpudata *cpudata = policy->driver_data;
+	u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type);
 	ssize_t ret;
 	bool raw_epp = false;
 	u8 epp;
@@ -1445,12 +1535,13 @@ ssize_t store_energy_performance_preference(struct cpufreq_policy *policy,
 		}
 
 		if (ret)
-			epp = epp_values[ret];
+			epp = values[ret];
 		else
 			epp = cpudata->epp_default_dc;
 	}
 
-	if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
+	if (epp > 0 && epp != values[EPP_INDEX_PERFORMANCE] &&
+	    cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
 		pr_debug("EPP cannot be set under performance policy\n");
 		return -EBUSY;
 	}
@@ -1475,34 +1566,31 @@ EXPORT_SYMBOL_FOR_PSTATE_UT(store_energy_performance_preference);
 ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf)
 {
 	struct amd_cpudata *cpudata = policy->driver_data;
-	u8 preference, epp;
+	u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type);
+	u8 epp;
+	int i;
 
 	epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached);
 
 	if (!cpudata->dynamic_epp && cpudata->raw_epp)
 		return sysfs_emit(buf, "%u\n", epp);
 
-	switch (epp) {
-	case AMD_CPPC_EPP_PERFORMANCE:
-		preference = EPP_INDEX_PERFORMANCE;
-		break;
-	case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
-		preference = EPP_INDEX_BALANCE_PERFORMANCE;
-		break;
-	case AMD_CPPC_EPP_BALANCE_POWERSAVE:
-		preference = EPP_INDEX_BALANCE_POWERSAVE;
-		break;
-	case AMD_CPPC_EPP_POWERSAVE:
-		preference = EPP_INDEX_POWERSAVE;
-		break;
-	default:
-		return -EINVAL;
-	}
+	/*
+	 * Map the cached EPP value back to a named preference. Skip the
+	 * "default" slot (index 0) so an EPP of 0 reports as "performance".
+	 * Stop at POWERSAVE; CUSTOM and DYNAMIC are not initialized in epp_values.
+	 */
+	for (i = EPP_INDEX_PERFORMANCE; i <= EPP_INDEX_POWERSAVE; i++) {
+		const char *name = energy_perf_strings[i];
 
-	if (cpudata->dynamic_epp)
-		return sysfs_emit(buf, "dynamic(profile:%s)\n", energy_perf_strings[preference]);
+		if (epp == values[i]) {
+			if (cpudata->dynamic_epp)
+				return sysfs_emit(buf, "dynamic(profile:%s)\n", name);
+			return sysfs_emit(buf, "%s\n", name);
+		}
+	}
 
-	return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
+	return sysfs_emit(buf, "%u\n", epp);
 }
 EXPORT_SYMBOL_FOR_PSTATE_UT(show_energy_performance_preference);
 
@@ -1794,7 +1882,7 @@ EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_get_status);
 
 int amd_pstate_update_status(const char *buf, size_t size)
 {
-	int mode_idx;
+	int cpu, mode_idx;
 
 	if (size > strlen("passive") || size < strlen("active"))
 		return -EINVAL;
@@ -1803,12 +1891,23 @@ int amd_pstate_update_status(const char *buf, size_t size)
 	if (mode_idx < 0)
 		return mode_idx;
 
-	if (mode_state_machine[cppc_state][mode_idx]) {
-		guard(mutex)(&amd_pstate_driver_lock);
-		return mode_state_machine[cppc_state][mode_idx](mode_idx);
+	guard(mutex)(&amd_pstate_driver_lock);
+
+	if (!mode_state_machine[cppc_state][mode_idx])
+		return 0;
+
+	if (mode_idx == AMD_PSTATE_PASSIVE &&
+	    !cpu_feature_enabled(X86_FEATURE_CPPC)) {
+		guard(cpus_read_lock)();
+
+		/* Check offline CPUs too, before changing or removing the driver. */
+		for_each_present_cpu(cpu) {
+			if (cppc_auto_sel_is_immutable(cpu))
+				return -EOPNOTSUPP;
+		}
 	}
 
-	return 0;
+	return mode_state_machine[cppc_state][mode_idx](mode_idx);
 }
 EXPORT_SYMBOL_FOR_PSTATE_UT(amd_pstate_update_status);
 
@@ -1894,6 +1993,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
 		return -ENOMEM;
 
 	cpudata->cpu = policy->cpu;
+	cpudata->cpu_type = cpu_data(policy->cpu).topo.cpu_type;
 
 	ret = amd_pstate_init_perf(cpudata);
 	if (ret)
@@ -1944,9 +2044,11 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
 		cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp;
 		cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE;
 	} else {
+		u8 *values = amd_pstate_cpu_epp_values(cpudata->cpu_type);
+
 		policy->policy = CPUFREQ_POLICY_POWERSAVE;
-		cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE;
-		cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE;
+		cpudata->epp_default_ac = values[EPP_INDEX_PERFORMANCE];
+		cpudata->epp_default_dc = values[EPP_INDEX_BALANCE_PERFORMANCE];
 		cpudata->current_profile = PLATFORM_PROFILE_BALANCED;
 	}
 
@@ -2245,6 +2347,38 @@ static bool amd_cppc_supported(void)
 	return true;
 }
 
+/*
+ * Resolve the numeric EPP values for hybrid systems. Only hybrid SoCs are listed
+ * in amd_pstate_epp_soc_ids[]; non-hybrid systems always use the legacy defaults.
+ */
+static inline void __init amd_pstate_set_epp_values(enum x86_topology_cpu_type type,
+						    const struct amd_pstate_epp_values *core)
+{
+	epp_values[type][EPP_INDEX_PERFORMANCE] = core->performance;
+	epp_values[type][EPP_INDEX_BALANCE_PERFORMANCE] = core->balance_performance;
+	epp_values[type][EPP_INDEX_BALANCE_POWERSAVE] = core->balance_power;
+	epp_values[type][EPP_INDEX_POWERSAVE] = core->power;
+}
+
+static void __init amd_pstate_init_epp_values(void)
+{
+	const struct x86_cpu_id *id = x86_match_cpu(amd_pstate_epp_soc_ids);
+	const struct amd_pstate_epp_soc *soc;
+
+	if (!id || !id->driver_data) {
+		if (cpu_feature_enabled(X86_FEATURE_ZEN6) &&
+		    cpu_feature_enabled(X86_FEATURE_AMD_HTR_CORES))
+			pr_warn_once("No EPP tunings found for platform\n");
+		return;
+	}
+
+	soc = (const struct amd_pstate_epp_soc *)id->driver_data;
+
+	amd_pstate_set_epp_values(TOPO_CPU_TYPE_PERFORMANCE, &soc->performance_core);
+	amd_pstate_set_epp_values(TOPO_CPU_TYPE_EFFICIENCY, &soc->efficiency_core);
+	amd_pstate_set_epp_values(TOPO_CPU_TYPE_LOW_POWER, &soc->low_power_core);
+}
+
 static int __init amd_pstate_init(void)
 {
 	struct device *dev_root;
@@ -2272,6 +2406,9 @@ static int __init amd_pstate_init(void)
 	/* check if this machine need CPPC quirks */
 	dmi_check_system(amd_pstate_quirks_table);
 
+	/* resolve per-SoC EPP values for the named preferences */
+	amd_pstate_init_epp_values();
+
 	/*
 	* determine the driver mode from the command line or kernel config.
 	* If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED.
diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h
index 9f5a81976eae06..c7189d177b81a5 100644
--- a/drivers/cpufreq/amd-pstate.h
+++ b/drivers/cpufreq/amd-pstate.h
@@ -146,6 +146,8 @@ struct amd_cpudata {
 	enum platform_profile_option current_profile;
 	struct device *ppdev;
 	char *profile_name;
+
+	enum x86_topology_cpu_type cpu_type;
 };
 
 /*
@@ -159,6 +161,20 @@ enum amd_pstate_mode {
 	AMD_PSTATE_GUIDED,
 	AMD_PSTATE_MAX,
 };
+
+static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val)
+{
+	u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq);
+
+	return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf);
+}
+
+static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val)
+{
+	return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val,
+				perf.nominal_perf);
+}
+
 const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode);
 int amd_pstate_get_status(void);
 int amd_pstate_update_status(const char *buf, size_t size);
diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c
index a8e35bc75fb2b3..389fff5e6f6529 100644
--- a/drivers/cpufreq/bmips-cpufreq.c
+++ b/drivers/cpufreq/bmips-cpufreq.c
@@ -132,8 +132,8 @@ static int bmips_cpufreq_init(struct cpufreq_policy *policy)
 
 	freq_table = bmips_cpufreq_get_freq_table(policy);
 	if (IS_ERR(freq_table)) {
-		pr_err("%s: couldn't determine frequency table (%ld).\n",
-			BMIPS_CPUFREQ_NAME, PTR_ERR(freq_table));
+		pr_err("%s: couldn't determine frequency table (%pe).\n",
+			BMIPS_CPUFREQ_NAME, freq_table);
 		return PTR_ERR(freq_table);
 	}
 
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 80893844353ca1..4ea444ff889ea1 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -18,6 +18,7 @@
 #include <linux/cpufreq.h>
 #include <linux/irq_work.h>
 #include <linux/kthread.h>
+#include <linux/mutex.h>
 #include <linux/time.h>
 #include <linux/vmalloc.h>
 #include <uapi/linux/sched/types.h>
@@ -41,6 +42,7 @@ MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");
 /* Frequency invariance support */
 struct cppc_freq_invariance {
 	int cpu;
+	bool pcc_work_initialized;
 	struct irq_work irq_work;
 	struct kthread_work work;
 	struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
@@ -49,10 +51,12 @@ struct cppc_freq_invariance {
 
 static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
 static struct kthread_worker *kworker_fie;
+static DEFINE_MUTEX(cppc_fie_lock);
 
 static int cppc_perf_from_fbctrs(u64 reference_perf,
 				 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
 				 struct cppc_perf_fb_ctrs *fb_ctrs_t1);
+static int cppc_fie_kworker_init(void);
 
 /**
  * __cppc_scale_freq_tick - CPPC arch_freq_scale updater for frequency invariance
@@ -149,7 +153,7 @@ static struct scale_freq_data cppc_sftd_pcc = {
 
 static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
 {
-	struct scale_freq_data *sftd = &cppc_sftd;
+	struct scale_freq_data *sftd;
 	struct cppc_freq_invariance *cppc_fi;
 	int cpu, ret;
 
@@ -161,9 +165,12 @@ static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
 		cppc_fi->cpu = cpu;
 		cppc_fi->cpu_data = policy->driver_data;
 		if (cppc_perf_ctrs_in_pcc_cpu(cpu)) {
+			if (cppc_fie_kworker_init())
+				return;
+
 			kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);
 			init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
-			sftd = &cppc_sftd_pcc;
+			cppc_fi->pcc_work_initialized = true;
 		}
 
 		ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);
@@ -179,17 +186,20 @@ static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
 		}
 	}
 
-	/* Register for freq-invariance */
-	topology_set_scale_freq_source(sftd, policy->cpus);
+	/* A shared policy may contain both PCC and non-PCC counters. */
+	for_each_cpu(cpu, policy->cpus) {
+		cppc_fi = &per_cpu(cppc_freq_inv, cpu);
+		if (cppc_fi->pcc_work_initialized)
+			sftd = &cppc_sftd_pcc;
+		else
+			sftd = &cppc_sftd;
+		topology_set_scale_freq_source(sftd, cpumask_of(cpu));
+	}
 }
 
 /*
- * We free all the resources on policy's removal and not on CPU removal as the
- * irq-work are per-cpu and the hotplug core takes care of flushing the pending
- * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work
- * fires on another CPU after the concerned CPU is removed, it won't harm.
- *
- * We just need to make sure to remove them all on policy->exit().
+ * Drain work initialized by this policy even if processor removal has
+ * already unpublished the CPU's CPC descriptor.
  */
 static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
 {
@@ -203,16 +213,18 @@ static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
 	topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);
 
 	for_each_cpu(cpu, policy->related_cpus) {
-		if (!cppc_perf_ctrs_in_pcc_cpu(cpu))
-			continue;
 		cppc_fi = &per_cpu(cppc_freq_inv, cpu);
+		if (!cppc_fi->pcc_work_initialized)
+			continue;
 		irq_work_sync(&cppc_fi->irq_work);
 		kthread_cancel_work_sync(&cppc_fi->work);
+		cppc_fi->pcc_work_initialized = false;
 	}
 }
 
-static void cppc_fie_kworker_init(void)
+static int cppc_fie_kworker_init(void)
 {
+	struct kthread_worker *worker;
 	struct sched_attr attr = {
 		.size		= sizeof(struct sched_attr),
 		.sched_policy	= SCHED_DEADLINE,
@@ -228,23 +240,28 @@ static void cppc_fie_kworker_init(void)
 	};
 	int ret;
 
-	kworker_fie = kthread_run_worker(0, "cppc_fie");
-	if (IS_ERR(kworker_fie)) {
-		pr_warn("%s: failed to create kworker_fie: %ld\n", __func__,
-			PTR_ERR(kworker_fie));
-		fie_disabled = FIE_DISABLED;
-		kworker_fie = NULL;
-		return;
+	guard(mutex)(&cppc_fie_lock);
+
+	if (kworker_fie)
+		return 0;
+
+	worker = kthread_run_worker(0, "cppc_fie");
+	if (IS_ERR(worker)) {
+		pr_warn("%s: failed to create kworker_fie: %pe\n", __func__,
+			worker);
+		return PTR_ERR(worker);
 	}
 
-	ret = sched_setattr_nocheck(kworker_fie->task, &attr);
+	ret = sched_setattr_nocheck(worker->task, &attr);
 	if (ret) {
 		pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__,
 			ret);
-		kthread_destroy_worker(kworker_fie);
-		fie_disabled = FIE_DISABLED;
-		kworker_fie = NULL;
+		kthread_destroy_worker(worker);
+		return ret;
 	}
+
+	kworker_fie = worker;
+	return 0;
 }
 
 static void __init cppc_freq_invariance_init(void)
@@ -259,11 +276,6 @@ static void __init cppc_freq_invariance_init(void)
 			fie_disabled = FIE_ENABLED;
 		}
 	}
-
-	if (fie_disabled || !perf_ctrs_in_pcc)
-		return;
-
-	cppc_fie_kworker_init();
 }
 
 static void cppc_freq_invariance_exit(void)
@@ -883,6 +895,7 @@ static ssize_t store_auto_select(struct cpufreq_policy *policy,
 				 const char *buf, size_t count)
 {
 	struct cppc_cpudata *cpu_data = policy->driver_data;
+	bool old_auto_sel = cpu_data->perf_ctrls.auto_sel;
 	bool val;
 	int ret;
 
@@ -911,8 +924,8 @@ static ssize_t store_auto_select(struct cpufreq_policy *policy,
 		if (ret) {
 			cpu_data->perf_ctrls.min_perf = old_min_perf;
 			cpu_data->perf_ctrls.max_perf = old_max_perf;
-			cppc_set_auto_sel(policy->cpu, false);
-			cpu_data->perf_ctrls.auto_sel = false;
+			cppc_set_auto_sel(policy->cpu, old_auto_sel);
+			cpu_data->perf_ctrls.auto_sel = old_auto_sel;
 			return ret;
 		}
 	}
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 96515880b4ac1c..54dde8419bdc4b 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -2590,8 +2590,8 @@ static void cpufreq_update_pressure(struct cpufreq_policy *policy)
 
 	cpu = cpumask_first(policy->related_cpus);
 	max_freq = arch_scale_freq_ref(cpu);
-	if (!max_freq)
-		max_freq = policy->cpuinfo.max_freq;
+	if (!max_freq && cpufreq_driver->scale_freq_ref)
+		max_freq = cpufreq_driver->scale_freq_ref(policy);
 
 	capped_freq = policy->max;
 
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index 0b32ae28ec857f..02bfd46543e99c 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -85,10 +85,12 @@ static unsigned int cs_dbs_update(struct cpufreq_policy *policy)
 	freq_step = get_freq_step(cs_tuners, policy);
 
 	/*
-	 * Decrease requested_freq one freq_step for each idle period that
-	 * we didn't update the frequency.
+	 * Apply deferred down steps only when the policy sample load is
+	 * below down_threshold. Otherwise, multiple deferred down steps may
+	 * cause a net frequency decrease outside the downscaling region.
 	 */
-	if (policy_dbs->idle_periods < UINT_MAX) {
+	if (policy_dbs->max_sample_load < cs_tuners->down_threshold &&
+	    policy_dbs->idle_periods < UINT_MAX) {
 		unsigned int freq_steps = policy_dbs->idle_periods * freq_step;
 
 		if (requested_freq > policy->min + freq_steps)
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 710d93ec89b533..f3bd3dbe7c419c 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -124,7 +124,8 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
 	struct policy_dbs_info *policy_dbs = policy->governor_data;
 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
 	unsigned int ignore_nice = dbs_data->ignore_nice_load;
-	unsigned int max_load = 0, idle_periods = UINT_MAX;
+	unsigned int max_load = 0, max_sample_load = 0;
+	unsigned int idle_periods = UINT_MAX;
 	unsigned int sampling_rate, io_busy, j;
 	u64 cur_nice;
 
@@ -147,7 +148,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
 		struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j);
 		u64 update_time, cur_idle_time;
 		unsigned int idle_time, time_elapsed;
-		unsigned int load;
+		unsigned int load, sample_load;
 
 		cur_idle_time = get_cpu_idle_time(j, &update_time, io_busy);
 
@@ -186,6 +187,20 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
 
 		j_cdbs->prev_cpu_nice = cur_nice;
 
+		/*
+		 * Compute the sample load separately from the prev_load value
+		 * that may be reused after a long idle interval. The conservative
+		 * governor uses it to decide whether to apply deferred down steps.
+		 * If no time has elapsed, retain the existing behavior and use
+		 * prev_load.
+		 */
+		if (unlikely(!time_elapsed))
+			sample_load = j_cdbs->prev_load;
+		else if (time_elapsed > idle_time)
+			sample_load = 100 * (time_elapsed - idle_time) / time_elapsed;
+		else
+			sample_load = 0;
+
 		if (unlikely(!time_elapsed)) {
 			/*
 			 * That can only happen when this function is called
@@ -220,11 +235,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
 			load = j_cdbs->prev_load;
 			j_cdbs->prev_load = 0;
 		} else {
-			if (time_elapsed > idle_time)
-				load = 100 * (time_elapsed - idle_time) / time_elapsed;
-			else
-				load = 0;
-
+			load = sample_load;
 			j_cdbs->prev_load = load;
 		}
 
@@ -237,9 +248,13 @@ unsigned int dbs_update(struct cpufreq_policy *policy)
 
 		if (load > max_load)
 			max_load = load;
+
+		if (sample_load > max_sample_load)
+			max_sample_load = sample_load;
 	}
 
 	policy_dbs->idle_periods = idle_periods;
+	policy_dbs->max_sample_load = max_sample_load;
 
 	return max_load;
 }
diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h
index 73b8ed7cfaae76..806d8fb4dff1d9 100644
--- a/drivers/cpufreq/cpufreq_governor.h
+++ b/drivers/cpufreq/cpufreq_governor.h
@@ -95,6 +95,8 @@ struct policy_dbs_info {
 	/* Multiplier for increasing sample delay temporarily. */
 	unsigned int rate_mult;
 	unsigned int idle_periods;	/* For conservative */
+	/* Maximum load from the current policy sample. */
+	unsigned int max_sample_load;
 	/* Status indicators */
 	bool is_shared;		/* This object is used by multiple CPUs */
 	bool work_in_progress;	/* Work is being queued up or in progress */
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index ceb340f7a110ba..17158cb25dd89b 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1135,6 +1135,14 @@ static bool hybrid_clear_max_perf_cpu(void)
 	return ret;
 }
 
+static unsigned int intel_pstate_scale_freq_ref(struct cpufreq_policy *policy)
+{
+	if (READ_ONCE(all_cpu_data[policy->cpu]->capacity_perf))
+		return policy->cpuinfo.max_freq;
+
+	return 0;
+}
+
 static void intel_pstate_update_freq_limits(struct cpudata *cpu)
 {
 	int scaling = cpu->pstate.scaling;
@@ -3088,6 +3096,7 @@ static struct cpufreq_driver intel_pstate = {
 	.offline	= intel_pstate_cpu_offline,
 	.online		= intel_pstate_cpu_online,
 	.update_limits	= intel_pstate_update_limits,
+	.scale_freq_ref = intel_pstate_scale_freq_ref,
 	.name		= "intel_pstate",
 };
 
@@ -3411,6 +3420,7 @@ static struct cpufreq_driver intel_cpufreq = {
 	.suspend	= intel_cpufreq_suspend,
 	.resume		= intel_pstate_resume,
 	.update_limits	= intel_pstate_update_limits,
+	.scale_freq_ref = intel_pstate_scale_freq_ref,
 	.name		= "intel_cpufreq",
 };
 
diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c
index 42edb41ad45950..a21b6f01218ac5 100644
--- a/drivers/cpufreq/qoriq-cpufreq.c
+++ b/drivers/cpufreq/qoriq-cpufreq.c
@@ -57,8 +57,7 @@ static u32 get_bus_freq(void)
 	/* get platform freq by its clock name */
 	pltclk = clk_get(NULL, "cg-pll0-div1");
 	if (IS_ERR(pltclk)) {
-		pr_err("%s: can't get bus frequency %ld\n",
-		       __func__, PTR_ERR(pltclk));
+		pr_err("%s: can't get bus frequency %pe\n", __func__, pltclk);
 		return PTR_ERR(pltclk);
 	}
 
diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs
index d7ead60bf8c244..2b1f18b7a1e777 100644
--- a/drivers/cpufreq/rcpufreq_dt.rs
+++ b/drivers/cpufreq/rcpufreq_dt.rs
@@ -225,4 +225,5 @@ module_platform_driver! {
     authors: ["Viresh Kumar <viresh.kumar@linaro.org>"],
     description: "Generic CPUFreq DT driver",
     license: "GPL v2",
+    alias: ["platform:cpufreq-dt"],
 }
diff --git a/drivers/cpufreq/s3c64xx-cpufreq.c b/drivers/cpufreq/s3c64xx-cpufreq.c
index 9cef7152807626..9a01592425ee6e 100644
--- a/drivers/cpufreq/s3c64xx-cpufreq.c
+++ b/drivers/cpufreq/s3c64xx-cpufreq.c
@@ -152,15 +152,14 @@ static int s3c64xx_cpufreq_driver_init(struct cpufreq_policy *policy)
 
 	policy->clk = clk_get(NULL, "armclk");
 	if (IS_ERR(policy->clk)) {
-		pr_err("Unable to obtain ARMCLK: %ld\n",
-		       PTR_ERR(policy->clk));
+		pr_err("Unable to obtain ARMCLK: %pe\n", policy->clk);
 		return PTR_ERR(policy->clk);
 	}
 
 #ifdef CONFIG_REGULATOR
 	vddarm = regulator_get(NULL, "vddarm");
 	if (IS_ERR(vddarm)) {
-		pr_err("Failed to obtain VDDARM: %ld\n", PTR_ERR(vddarm));
+		pr_err("Failed to obtain VDDARM: %pe\n", vddarm);
 		pr_err("Only frequency scaling available\n");
 		vddarm = NULL;
 	} else {
diff --git a/drivers/cpufreq/sparc-us2e-cpufreq.c b/drivers/cpufreq/sparc-us2e-cpufreq.c
index a68706406b8805..5cda391ad03bbf 100644
--- a/drivers/cpufreq/sparc-us2e-cpufreq.c
+++ b/drivers/cpufreq/sparc-us2e-cpufreq.c
@@ -282,12 +282,11 @@ static int us2e_freq_cpu_init(struct cpufreq_policy *policy)
 	table[1].frequency = clock_tick / 2;
 	table[2].driver_data = 2;
 	table[2].frequency = clock_tick / 4;
-	table[2].driver_data = 3;
-	table[2].frequency = clock_tick / 6;
-	table[2].driver_data = 4;
-	table[2].frequency = clock_tick / 8;
-	table[2].driver_data = 5;
-	table[3].frequency = CPUFREQ_TABLE_END;
+	table[3].driver_data = 3;
+	table[3].frequency = clock_tick / 6;
+	table[4].driver_data = 4;
+	table[4].frequency = clock_tick / 8;
+	table[5].frequency = CPUFREQ_TABLE_END;
 
 	policy->cpuinfo.transition_latency = 0;
 	policy->cur = clock_tick;
diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c
index b15b3142b5fec3..144ac0f1d85d79 100644
--- a/drivers/cpufreq/sti-cpufreq.c
+++ b/drivers/cpufreq/sti-cpufreq.c
@@ -263,7 +263,7 @@ static int __init sti_cpufreq_init(void)
 
 	ddata.cpu = get_cpu_device(0);
 	if (!ddata.cpu) {
-		dev_err(ddata.cpu, "Failed to get device for CPU0\n");
+		pr_err("Failed to get device for CPU0\n");
 		goto skip_voltage_scaling;
 	}
 
@@ -281,7 +281,7 @@ static int __init sti_cpufreq_init(void)
 		goto register_cpufreq_dt;
 
 skip_voltage_scaling:
-	dev_err(ddata.cpu, "Not doing voltage scaling\n");
+	pr_err("Not doing voltage scaling\n");
 
 register_cpufreq_dt:
 	platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c
index c6375e14d445d7..1eb96e9f3310db 100644
--- a/drivers/cpufreq/tegra194-cpufreq.c
+++ b/drivers/cpufreq/tegra194-cpufreq.c
@@ -367,7 +367,7 @@ static void tegra194_get_cpu_ndiv_sysreg(void *ndiv)
 
 static int tegra194_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv)
 {
-	return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true);
+	return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, ndiv, true);
 }
 
 static void tegra194_set_cpu_ndiv_sysreg(void *data)
diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c
index b9e4ad7d43a331..5722365b943ea2 100644
--- a/drivers/cpuidle/cpuidle-psci-domain.c
+++ b/drivers/cpuidle/cpuidle-psci-domain.c
@@ -68,9 +68,8 @@ static int psci_pd_init(struct device_node *np, bool use_osi)
 	 */
 	if (use_osi) {
 		pd->power_off = psci_pd_power_off;
-		pd->flags |= GENPD_FLAG_ACTIVE_WAKEUP;
-		if (IS_ENABLED(CONFIG_PREEMPT_RT))
-			pd->flags |= GENPD_FLAG_RPM_ALWAYS_ON;
+		pd->flags |= GENPD_FLAG_ACTIVE_WAKEUP | GENPD_FLAG_POWER_UNKNOWN;
+		pd->state_idx = pd->state_count ? pd->state_count - 1 : 0;
 	} else {
 		pd->flags |= GENPD_FLAG_ALWAYS_ON;
 	}
@@ -78,7 +77,7 @@ static int psci_pd_init(struct device_node *np, bool use_osi)
 	/* Use governor for CPU PM domains if it has some states to manage. */
 	pd_gov = pd->states ? &pm_domain_cpu_gov : NULL;
 
-	ret = pm_genpd_init(pd, pd_gov, false);
+	ret = pm_genpd_init(pd, pd_gov, use_osi);
 	if (ret)
 		goto free_pd_prov;
 
diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
index b250d0dde76072..0187d6a9096a4c 100644
--- a/drivers/cpuidle/cpuidle-psci.c
+++ b/drivers/cpuidle/cpuidle-psci.c
@@ -479,4 +479,4 @@ static int __init psci_idle_init(void)
 
 	return 0;
 }
-device_initcall(psci_idle_init);
+subsys_initcall(psci_idle_init);
diff --git a/drivers/cpuidle/cpuidle-tegra.c b/drivers/cpuidle/cpuidle-tegra.c
index aca907a62bb5de..d2fc216c38fc56 100644
--- a/drivers/cpuidle/cpuidle-tegra.c
+++ b/drivers/cpuidle/cpuidle-tegra.c
@@ -281,7 +281,7 @@ static int tegra114_enter_s2idle(struct cpuidle_device *dev,
  * LP2 | C7	(CPU core power gating)
  * LP2 | CC6	(CPU cluster power gating)
  *
- * Note that that the older CPUIDLE driver versions didn't explicitly
+ * Note that the older CPUIDLE driver versions didn't explicitly
  * differentiate the LP2 states because these states either used the same
  * code path or because CC6 wasn't supported.
  */
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index 544a5d59300778..eb529d73d4b752 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -284,10 +284,10 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		data->bucket = BUCKETS - 1;
 	}
 
-	if (latency_req == 0 ||
-	    ((data->next_timer_ns < drv->states[1].target_residency_ns ||
-	      latency_req < drv->states[1].exit_latency_ns) &&
-	     !dev->states_usage[0].disable)) {
+	if (!dev->states_usage[0].disable &&
+	    (latency_req == 0 ||
+	     data->next_timer_ns < drv->states[1].target_residency_ns ||
+	     latency_req < drv->states[1].exit_latency_ns)) {
 		/*
 		 * In this case state[0] will be used no matter what, so return
 		 * it right away and keep the tick running if state[0] is a
diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c
index ac43b9b013b3fc..906f4f89e3659d 100644
--- a/drivers/cpuidle/governors/teo.c
+++ b/drivers/cpuidle/governors/teo.c
@@ -426,6 +426,17 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
 		}
 	}
 
+	/*
+	 * If the latency constraint does not allow any of the enabled idle
+	 * states to be used, the candidate state index will be capped to 0
+	 * by the check below, but state 0 may be disabled.  To prevent the
+	 * selection of a disabled state in that case, ensure that
+	 * constraint_idx is at least equal to the index of the first
+	 * enabled idle state.
+	 */
+	if (constraint_idx < idx0)
+		constraint_idx = idx0;
+
 	/*
 	 * If there is a latency constraint, it may be necessary to select an
 	 * idle state shallower than the current candidate one.
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index fd010bfb702027..acf18b71e54d7c 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -1046,8 +1046,7 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data)
 static inline irqreturn_t crypto4xx_interrupt_handler(int irq, void *data,
 						      u32 clr_val)
 {
-	struct device *dev = data;
-	struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
+	struct crypto4xx_core_device *core_dev = data;
 
 	writel(clr_val, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR);
 	tasklet_schedule(&core_dev->tasklet);
@@ -1206,9 +1205,19 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	struct device *dev = &ofdev->dev;
 	struct crypto4xx_core_device *core_dev;
 	struct device_node *np;
+	void __iomem *ce_base;
+	int irq;
 	u32 pvr;
 	bool is_revb = true;
 
+	irq = platform_get_irq(ofdev, 0);
+	if (irq < 0)
+		return irq;
+
+	ce_base = devm_platform_ioremap_resource(ofdev, 0);
+	if (IS_ERR(ce_base))
+		return PTR_ERR(ce_base);
+
 	np = of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto");
 	if (np) {
 		mtdcri(SDR0, PPC460EX_SDR0_SRST,
@@ -1251,9 +1260,7 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	if (!core_dev->dev)
 		return -ENOMEM;
 
-	core_dev->dev->ce_base = devm_platform_ioremap_resource(ofdev, 0);
-	if (IS_ERR(core_dev->dev->ce_base))
-		return PTR_ERR(core_dev->dev->ce_base);
+	core_dev->dev->ce_base = ce_base;
 
 	/*
 	 * Older version of 460EX/GT have a hardware bug.
@@ -1291,15 +1298,11 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 		     (unsigned long) dev);
 
 	/* Register for Crypto isr, Crypto Engine IRQ */
-	core_dev->irq = platform_get_irq(ofdev, 0);
-	if (core_dev->irq < 0) {
-		rc = core_dev->irq;
-		goto err_tasklet;
-	}
+	core_dev->irq = irq;
 	rc = request_irq(core_dev->irq,
 			 is_revb ? crypto4xx_ce_interrupt_handler_revb :
 				   crypto4xx_ce_interrupt_handler,
-			 0, KBUILD_MODNAME, dev);
+			 0, KBUILD_MODNAME, core_dev);
 	if (rc)
 		goto err_tasklet;
 
@@ -1312,11 +1315,15 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	if (rc)
 		goto err_irq;
 
-	ppc4xx_trng_probe(core_dev);
+	rc = ppc4xx_trng_probe(core_dev);
+	if (rc)
+		goto err_crypto;
 	return 0;
 
+err_crypto:
+	crypto4xx_unregister_alg(core_dev->dev);
 err_irq:
-	free_irq(core_dev->irq, dev);
+	free_irq(core_dev->irq, core_dev);
 err_tasklet:
 	tasklet_kill(&core_dev->tasklet);
 err_build_sdr:
@@ -1337,7 +1344,7 @@ static void crypto4xx_remove(struct platform_device *ofdev)
 	 * Free IRQ before killing the tasklet to prevent the interrupt
 	 * handler from rescheduling the tasklet after it has been killed.
 	 */
-	free_irq(core_dev->irq, dev);
+	free_irq(core_dev->irq, core_dev);
 	tasklet_kill(&core_dev->tasklet);
 	/* Un-register with Linux CryptoAPI */
 	crypto4xx_unregister_alg(core_dev->dev);
diff --git a/drivers/crypto/amcc/crypto4xx_trng.c b/drivers/crypto/amcc/crypto4xx_trng.c
index cfd66b779ce10d..83e9353529fcaf 100644
--- a/drivers/crypto/amcc/crypto4xx_trng.c
+++ b/drivers/crypto/amcc/crypto4xx_trng.c
@@ -68,7 +68,7 @@ static const struct of_device_id ppc4xx_trng_match[] = {
 	{},
 };
 
-void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev)
+int ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev)
 {
 	struct crypto4xx_device *dev = core_dev->dev;
 	struct device_node *trng = NULL;
@@ -79,17 +79,21 @@ void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev)
 	trng = of_find_matching_node(NULL, ppc4xx_trng_match);
 	if (!trng || !of_device_is_available(trng)) {
 		of_node_put(trng);
-		return;
+		return 0;
 	}
 
 	dev->trng_base = of_iomap(trng, 0);
 	of_node_put(trng);
-	if (!dev->trng_base)
+	if (!dev->trng_base) {
+		err = -EINVAL;
 		goto err_out;
+	}
 
 	rng = kzalloc_obj(*rng);
-	if (!rng)
+	if (!rng) {
+		err = -ENOMEM;
 		goto err_out;
+	}
 
 	rng->name = KBUILD_MODNAME;
 	rng->data_present = ppc4xx_trng_data_present;
@@ -105,13 +109,14 @@ void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev)
 			err);
 		goto err_out;
 	}
-	return;
+	return 0;
 
 err_out:
 	iounmap(dev->trng_base);
 	kfree(rng);
 	dev->trng_base = NULL;
 	core_dev->trng = NULL;
+	return err;
 }
 
 void ppc4xx_trng_remove(struct crypto4xx_core_device *core_dev)
diff --git a/drivers/crypto/amcc/crypto4xx_trng.h b/drivers/crypto/amcc/crypto4xx_trng.h
index 7356716274cba0..1030c3cfafd04b 100644
--- a/drivers/crypto/amcc/crypto4xx_trng.h
+++ b/drivers/crypto/amcc/crypto4xx_trng.h
@@ -13,11 +13,11 @@
 #define __CRYPTO4XX_TRNG_H__
 
 #ifdef CONFIG_HW_RANDOM_PPC4XX
-void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev);
+int ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev);
 void ppc4xx_trng_remove(struct crypto4xx_core_device *core_dev);
 #else
-static inline void ppc4xx_trng_probe(
-	struct crypto4xx_core_device *dev __maybe_unused) { }
+static inline int ppc4xx_trng_probe(
+	struct crypto4xx_core_device *dev __maybe_unused) { return 0; }
 static inline void ppc4xx_trng_remove(
 	struct crypto4xx_core_device *dev __maybe_unused) { }
 #endif
diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c
index 29048da6f50a7f..d78815b0547d87 100644
--- a/drivers/crypto/amlogic/amlogic-gxl-cipher.c
+++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c
@@ -177,10 +177,13 @@ static int meson_cipher(struct skcipher_request *areq)
 	if (areq->src == areq->dst) {
 		nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src),
 				    DMA_BIDIRECTIONAL);
-		if (!nr_sgs) {
-			dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
+		if (!nr_sgs || nr_sgs > MAXDESC - 3) {
+			dev_err(mc->dev, "Invalid BIDIR SG count %d\n", nr_sgs);
 			err = -EINVAL;
-			goto theend;
+
+			if (nr_sgs)
+				dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL);
+			goto error_keyiv;
 		}
 		nr_sgd = nr_sgs;
 	} else {
@@ -189,14 +192,20 @@ static int meson_cipher(struct skcipher_request *areq)
 		if (!nr_sgs || nr_sgs > MAXDESC - 3) {
 			dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
 			err = -EINVAL;
-			goto theend;
+
+			if (nr_sgs)
+				dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
+			goto error_keyiv;
 		}
 		nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst),
 				    DMA_FROM_DEVICE);
 		if (!nr_sgd || nr_sgd > MAXDESC - 3) {
 			dev_err(mc->dev, "Invalid SG count %d\n", nr_sgd);
 			err = -EINVAL;
-			goto theend;
+
+			if (nr_sgd)
+				dma_unmap_sg(mc->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE);
+			goto error_src;
 		}
 	}
 
@@ -251,6 +260,12 @@ static int meson_cipher(struct skcipher_request *areq)
 						 ivsize, 0);
 		}
 	}
+	goto theend;
+
+error_src:
+	dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
+error_keyiv:
+	dma_unmap_single(mc->dev, phykeyiv, keyivlen, DMA_TO_DEVICE);
 theend:
 	kfree_sensitive(bkeyiv);
 	kfree_sensitive(backup_iv);
diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c
index 169c6eeb51e51b..d5d84524c11042 100644
--- a/drivers/crypto/amlogic/amlogic-gxl-core.c
+++ b/drivers/crypto/amlogic/amlogic-gxl-core.c
@@ -37,12 +37,13 @@ static irqreturn_t meson_irq_handler(int irq, void *data)
 				complete(&mc->chanlist[flow].complete);
 				return IRQ_HANDLED;
 			}
-			dev_err(mc->dev, "%s %d Got irq for flow %d but ctrl is empty\n", __func__, irq, flow);
+			dev_err_ratelimited(mc->dev, "%s %d Got irq for flow %d but ctrl is empty\n", __func__, irq, flow);
+			return IRQ_NONE;
 		}
 	}
 
-	dev_err(mc->dev, "%s %d from unknown irq\n", __func__, irq);
-	return IRQ_HANDLED;
+	dev_err_ratelimited(mc->dev, "%s %d from unknown irq\n", __func__, irq);
+	return IRQ_NONE;
 }
 
 static struct meson_alg_template mc_algs[] = {
@@ -243,34 +244,30 @@ static int meson_crypto_probe(struct platform_device *pdev)
 	if (IS_ERR(mc->base))
 		return PTR_ERR(mc->base);
 
-	mc->busclk = devm_clk_get(&pdev->dev, "blkmv");
+	mc->busclk = devm_clk_get_enabled(&pdev->dev, "blkmv");
 	if (IS_ERR(mc->busclk)) {
 		err = PTR_ERR(mc->busclk);
-		dev_err(&pdev->dev, "Cannot get core clock err=%d\n", err);
+		dev_err(&pdev->dev, "Cannot get/enable core clock err=%d\n", err);
 		return err;
 	}
 
+	err = meson_allocate_chanlist(mc);
+	if (err)
+		return err;
+
 	for (i = 0; i < MAXFLOW; i++) {
 		mc->irqs[i] = platform_get_irq(pdev, i);
-		if (mc->irqs[i] < 0)
-			return mc->irqs[i];
+		if (mc->irqs[i] < 0) {
+			err = mc->irqs[i];
+			goto error_chanlist;
+		}
 
 		err = devm_request_irq(&pdev->dev, mc->irqs[i], meson_irq_handler, 0,
 				       "gxl-crypto", mc);
 		if (err < 0)
-			return err;
-	}
-
-	err = clk_prepare_enable(mc->busclk);
-	if (err != 0) {
-		dev_err(&pdev->dev, "Cannot prepare_enable busclk\n");
-		return err;
+			goto error_chanlist;
 	}
 
-	err = meson_allocate_chanlist(mc);
-	if (err)
-		goto error_flow;
-
 	err = meson_register_algs(mc);
 	if (err)
 		goto error_alg;
@@ -289,9 +286,8 @@ static int meson_crypto_probe(struct platform_device *pdev)
 	return 0;
 error_alg:
 	meson_unregister_algs(mc);
+error_chanlist:
 	meson_free_chanlist(mc, MAXFLOW - 1);
-error_flow:
-	clk_disable_unprepare(mc->busclk);
 	return err;
 }
 
@@ -306,8 +302,6 @@ static void meson_crypto_remove(struct platform_device *pdev)
 	meson_unregister_algs(mc);
 
 	meson_free_chanlist(mc, MAXFLOW - 1);
-
-	clk_disable_unprepare(mc->busclk);
 }
 
 static const struct of_device_id meson_crypto_of_match_table[] = {
diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c
index fa201dae1f81b4..e7d3f611df05cb 100644
--- a/drivers/crypto/aspeed/aspeed-hace-crypto.c
+++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c
@@ -576,7 +576,6 @@ static int aspeed_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
 {
 	struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
 	struct aspeed_hace_dev *hace_dev = ctx->hace_dev;
-	struct crypto_aes_ctx gen_aes_key;
 
 	CIPHER_DBG(hace_dev, "keylen: %d bits\n", (keylen * 8));
 
@@ -585,9 +584,9 @@ static int aspeed_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
 		return -EINVAL;
 
 	if (ctx->hace_dev->version == AST2500_VERSION) {
+		struct crypto_aes_ctx gen_aes_key __cleanup(aes_zeroize_ctx);
 		aes_expandkey(&gen_aes_key, key, keylen);
 		memcpy(ctx->key, gen_aes_key.key_enc, AES_MAX_KEYLENGTH);
-
 	} else {
 		memcpy(ctx->key, key, keylen);
 	}
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 50d97cef75b89d..8e4de859cadc08 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -1267,7 +1267,7 @@ static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd)
 	struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
 
 	/* Set the data length. */
-	atmel_aes_write(dd, AES_AADLENR, dd->total);
+	atmel_aes_write(dd, AES_AADLENR, dd->datalen);
 	atmel_aes_write(dd, AES_CLENR, 0);
 
 	/* If needed, overwrite the GCM Intermediate Hash Word Registers */
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index 075f38a67935c3..b538e818a26f4b 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -177,16 +177,14 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
 	work_data->client = ctx->client;
 
 	ret = atmel_i2c_init_ecdh_cmd(&work_data->cmd, req->src);
-	if (ret)
-		goto free_work_data;
+	if (ret) {
+		kfree(work_data);
+		return ret;
+	}
 
 	atmel_i2c_enqueue(work_data, atmel_ecdh_done, req);
 
 	return -EINPROGRESS;
-
-free_work_data:
-	kfree(work_data);
-	return ret;
 }
 
 static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
@@ -348,6 +346,7 @@ static void atmel_ecc_remove(struct i2c_client *client)
 
 static const struct of_device_id atmel_ecc_dt_ids[] = {
 	{ .compatible = "atmel,atecc508a", },
+	{ .compatible = "atmel,atecc608a", },
 	{ .compatible = "atmel,atecc608b", },
 	{ }
 };
@@ -355,6 +354,7 @@ MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids);
 
 static const struct i2c_device_id atmel_ecc_id[] = {
 	{ .name = "atecc508a" },
+	{ .name = "atecc608a" },
 	{ .name = "atecc608b" },
 	{ }
 };
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 48f54b61550423..2d70a4107693a1 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -422,6 +422,9 @@ static int atmel_sha_init(struct ahash_request *req)
 	struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 	struct atmel_sha_dev *dd = atmel_sha_find_dev(tctx);
 
+	if (!dd)
+		return -ENODEV;
+
 	ctx->dd = dd;
 
 	ctx->flags = 0;
@@ -1203,7 +1206,7 @@ static int atmel_sha_finup(struct ahash_request *req)
 
 	/*
 	 * final() has to be always called to cleanup resources
-	 * even if udpate() failed, except EINPROGRESS
+	 * even if update() failed, except EINPROGRESS
 	 */
 	err2 = atmel_sha_final(req);
 
@@ -1338,7 +1341,7 @@ static int atmel_sha_done(struct atmel_sha_dev *dd)
 	return err;
 
 finish:
-	/* finish curent request */
+	/* finish current request */
 	atmel_sha_finish_req(dd->req, err);
 
 	return err;
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index ffea1db74651d3..f084fda4242304 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -308,45 +308,33 @@ static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
 
 static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)
 {
-	int err = -ENOMEM;
-
 	dd->buf_in = (void *)__get_free_page(GFP_KERNEL);
 	dd->buf_out = (void *)__get_free_page(GFP_KERNEL);
-	dd->buflen = PAGE_SIZE;
-	dd->buflen &= ~(DES_BLOCK_SIZE - 1);
+	dd->buflen = PAGE_SIZE & ~(DES_BLOCK_SIZE - 1);
 
 	if (!dd->buf_in || !dd->buf_out) {
-		dev_dbg(dd->dev, "unable to alloc pages.\n");
+		dev_err(dd->dev, "failed to allocate DMA buffers\n");
 		goto err_alloc;
 	}
 
-	/* MAP here */
-	dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in,
-					dd->buflen, DMA_TO_DEVICE);
-	err = dma_mapping_error(dd->dev, dd->dma_addr_in);
-	if (err) {
-		dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen);
-		goto err_map_in;
-	}
+	dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen, DMA_TO_DEVICE);
+	if (dma_mapping_error(dd->dev, dd->dma_addr_in))
+		goto err_map;
 
-	dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out,
-					dd->buflen, DMA_FROM_DEVICE);
-	err = dma_mapping_error(dd->dev, dd->dma_addr_out);
-	if (err) {
-		dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen);
-		goto err_map_out;
+	dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen, DMA_FROM_DEVICE);
+	if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
+		dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
+		goto err_map;
 	}
 
 	return 0;
 
-err_map_out:
-	dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
-		DMA_TO_DEVICE);
-err_map_in:
+err_map:
+	dev_err(dd->dev, "failed to map %zu bytes for DMA\n", dd->buflen);
 err_alloc:
 	free_page((unsigned long)dd->buf_out);
 	free_page((unsigned long)dd->buf_in);
-	return err;
+	return -ENOMEM;
 }
 
 static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd)
@@ -361,18 +349,9 @@ static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd)
 
 static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd,
 				dma_addr_t dma_addr_in,
-				dma_addr_t dma_addr_out, int length)
+				dma_addr_t dma_addr_out)
 {
-	int len32;
-
-	dd->dma_size = length;
-
-	if (!(dd->flags & TDES_FLAGS_FAST)) {
-		dma_sync_single_for_device(dd->dev, dma_addr_in, length,
-					   DMA_TO_DEVICE);
-	}
-
-	len32 = DIV_ROUND_UP(length, sizeof(u32));
+	u32 len32 = DIV_ROUND_UP(dd->dma_size, sizeof(u32));
 
 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
 	atmel_tdes_write(dd, TDES_TPR, dma_addr_in);
@@ -391,19 +370,12 @@ static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd,
 
 static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd,
 				dma_addr_t dma_addr_in,
-				dma_addr_t dma_addr_out, int length)
+				dma_addr_t dma_addr_out)
 {
 	struct scatterlist sg[2];
 	struct dma_async_tx_descriptor	*in_desc, *out_desc;
 	enum dma_slave_buswidth addr_width;
 
-	dd->dma_size = length;
-
-	if (!(dd->flags & TDES_FLAGS_FAST)) {
-		dma_sync_single_for_device(dd->dev, dma_addr_in, length,
-					   DMA_TO_DEVICE);
-	}
-
 	addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 
 	dd->dma_lch_in.dma_conf.dst_addr_width = addr_width;
@@ -416,11 +388,11 @@ static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd,
 
 	sg_init_table(&sg[0], 1);
 	sg_dma_address(&sg[0]) = dma_addr_in;
-	sg_dma_len(&sg[0]) = length;
+	sg_dma_len(&sg[0]) = dd->dma_size;
 
 	sg_init_table(&sg[1], 1);
 	sg_dma_address(&sg[1]) = dma_addr_out;
-	sg_dma_len(&sg[1]) = length;
+	sg_dma_len(&sg[1]) = dd->dma_size;
 
 	in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0],
 				1, DMA_MEM_TO_DEV,
@@ -494,11 +466,17 @@ static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)
 	}
 
 	dd->total -= count;
+	dd->dma_size = count;
+
+	if (!fast) {
+		dma_sync_single_for_device(dd->dev, addr_in, dd->dma_size, DMA_TO_DEVICE);
+		dma_sync_single_for_device(dd->dev, addr_out, dd->dma_size, DMA_FROM_DEVICE);
+	}
 
 	if (dd->caps.has_dma)
-		err = atmel_tdes_crypt_dma(dd, addr_in, addr_out, count);
+		err = atmel_tdes_crypt_dma(dd, addr_in, addr_out);
 	else
-		err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out, count);
+		err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out);
 
 	if (err && (dd->flags & TDES_FLAGS_FAST)) {
 		dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
@@ -933,7 +911,7 @@ static int atmel_tdes_probe(struct platform_device *pdev)
 	struct resource *tdes_res;
 	int err;
 
-	tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL);
+	tdes_dd = devm_kzalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL);
 	if (!tdes_dd)
 		return -ENOMEM;
 
diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c
index 9358c1c041d4ea..4234f9f156a3d6 100644
--- a/drivers/crypto/cavium/cpt/cptpf_main.c
+++ b/drivers/crypto/cavium/cpt/cptpf_main.c
@@ -90,19 +90,19 @@ static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
 
 static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
 {
-	/* Clear mbox(0) interupts for all vfs */
+	/* Clear mbox(0) interrupts for all vfs */
 	cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
 }
 
 static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
 {
-	/* Clear ecc(0) interupts for all vfs */
+	/* Clear ecc(0) interrupts for all vfs */
 	cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
 }
 
 static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
 {
-	/* Clear exec interupts for all vfs */
+	/* Clear exec interrupts for all vfs */
 	cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
 }
 
@@ -115,7 +115,7 @@ static void cpt_disable_all_interrupts(struct cpt_device *cpt)
 
 static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
 {
-	/* Set mbox(0) interupts for all vfs */
+	/* Set mbox(0) interrupts for all vfs */
 	cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
 }
 
@@ -140,10 +140,10 @@ static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
 	 */
 	if (mcode->is_ae) {
 		core = CPT_MAX_SE_CORES; /* start couting from 10 */
-		total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
+		total_cores = CPT_MAX_TOTAL_CORES; /* up to 15 */
 	} else {
 		core = 0; /* start couting from 0 */
-		total_cores = CPT_MAX_SE_CORES; /* upto 9 */
+		total_cores = CPT_MAX_SE_CORES; /* up to 9 */
 	}
 
 	/* Point to microcode for each core of the group */
diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
index db76df14c4a007..e07448098ac035 100644
--- a/drivers/crypto/cavium/cpt/cptvf_main.c
+++ b/drivers/crypto/cavium/cpt/cptvf_main.c
@@ -417,7 +417,7 @@ static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
 
 	vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
 					CPTX_VQX_MISC_ENA_W1S(0, 0));
-	/* Set mbox(0) interupts for the requested vf */
+	/* Set mbox(0) interrupts for the requested vf */
 	vqx_misc_ena.s.swerr = 1;
 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
 			vqx_misc_ena.u);
@@ -429,7 +429,7 @@ static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
 
 	vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
 					CPTX_VQX_MISC_ENA_W1S(0, 0));
-	/* Set mbox(0) interupts for the requested vf */
+	/* Set mbox(0) interrupts for the requested vf */
 	vqx_misc_ena.s.mbox = 1;
 	cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
 			vqx_misc_ena.u);
diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c
index 1267e1eba7e953..9f2ae10bbd60d6 100644
--- a/drivers/crypto/cavium/cpt/cptvf_mbox.c
+++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c
@@ -120,7 +120,7 @@ int cptvf_send_vq_size_msg(struct cpt_vf *cptvf)
 }
 
 /*
- * Communicate VF group required to PF and get the VQ binded to that group
+ * Communicate VF group required to PF and get the VQ bound to that group
  */
 int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf)
 {
@@ -139,7 +139,7 @@ int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf)
 }
 
 /*
- * Communicate VF group required to PF and get the VQ binded to that group
+ * Communicate VF group required to PF and get the VQ bound to that group
  */
 int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf)
 {
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index e474c84d8d3811..54b2db6bb5d01c 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -468,7 +468,7 @@ static int nitrox_probe(struct pci_dev *pdev,
 		err = -EIO;
 		goto ioremap_err;
 	}
-	/* allocate command queus based on cpus, max queues are 64 */
+	/* allocate command queues based on cpus, max queues are 64 */
 	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
 	ndev->qlen = qlen;
 
diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c
index 60a133e23e5f5c..d92de4ad31ac5c 100644
--- a/drivers/crypto/ccp/ccp-dev-v3.c
+++ b/drivers/crypto/ccp/ccp-dev-v3.c
@@ -506,6 +506,8 @@ static int ccp_init(struct ccp_device *ccp)
 			kthread_stop(ccp->cmd_q[i].kthread);
 
 	sp_free_ccp_irq(ccp->sp, ccp);
+	if (ccp->use_tasklet)
+		tasklet_kill(&ccp->irq_tasklet);
 
 e_pool:
 	for (i = 0; i < ccp->cmd_q_count; i++)
@@ -545,6 +547,8 @@ static void ccp_destroy(struct ccp_device *ccp)
 			kthread_stop(ccp->cmd_q[i].kthread);
 
 	sp_free_ccp_irq(ccp->sp, ccp);
+	if (ccp->use_tasklet)
+		tasklet_kill(&ccp->irq_tasklet);
 
 	for (i = 0; i < ccp->cmd_q_count; i++)
 		dma_pool_destroy(ccp->cmd_q[i].dma_pool);
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index dcd6aab51e4767..dacde9614e8a79 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -995,6 +995,8 @@ static int ccp5_init(struct ccp_device *ccp)
 
 e_irq:
 	sp_free_ccp_irq(ccp->sp, ccp);
+	if (ccp->use_tasklet)
+		tasklet_kill(&ccp->irq_tasklet);
 
 e_pool:
 	for (i = 0; i < ccp->cmd_q_count; i++)
@@ -1046,6 +1048,8 @@ static void ccp5_destroy(struct ccp_device *ccp)
 			kthread_stop(ccp->cmd_q[i].kthread);
 
 	sp_free_ccp_irq(ccp->sp, ccp);
+	if (ccp->use_tasklet)
+		tasklet_kill(&ccp->irq_tasklet);
 
 	/* Flush the cmd and backlog queue */
 	while (!list_empty(&ccp->cmd)) {
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index 246801912e1a53..aff3348b83510d 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -30,7 +30,7 @@
 
 #define MAX_CCPS 32
 
-/* Limit CCP use to a specifed number of queues per device */
+/* Limit CCP use to a specified number of queues per device */
 static unsigned int nqueues;
 module_param(nqueues, uint, 0444);
 MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)");
diff --git a/drivers/crypto/ccp/dbc.c b/drivers/crypto/ccp/dbc.c
index 410084a9039c98..4db49a14550c97 100644
--- a/drivers/crypto/ccp/dbc.c
+++ b/drivers/crypto/ccp/dbc.c
@@ -236,6 +236,8 @@ int dbc_dev_init(struct psp_device *psp)
 		goto cleanup_mbox;
 	}
 
+	mutex_init(&dbc_dev->ioctl_mutex);
+
 	dbc_dev->char_dev.minor = MISC_DYNAMIC_MINOR;
 	dbc_dev->char_dev.name = "dbc";
 	dbc_dev->char_dev.fops = &dbc_fops;
@@ -244,8 +246,6 @@ int dbc_dev_init(struct psp_device *psp)
 	if (ret)
 		goto cleanup_mbox;
 
-	mutex_init(&dbc_dev->ioctl_mutex);
-
 	return 0;
 
 cleanup_mbox:
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index f833cb7e4da360..5c996ab6389594 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -1663,6 +1663,8 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 
 	sev_es_tmr_size = SNP_TMR_SIZE;
 
+	snp_enable_rmpopt();
+
 	return 0;
 }
 
diff --git a/drivers/crypto/ccp/sfs.c b/drivers/crypto/ccp/sfs.c
index a4777839790b4f..530141f8bc743b 100644
--- a/drivers/crypto/ccp/sfs.c
+++ b/drivers/crypto/ccp/sfs.c
@@ -111,13 +111,13 @@ static long sfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	struct sfs_device *sfs_dev;
 	int ret = 0;
 
+	guard(mutex)(&sfs_ioctl_mutex);
+
 	if (!psp_master || !psp_master->sfs_data)
 		return -ENODEV;
 
 	sfs_dev = psp_master->sfs_data;
 
-	guard(mutex)(&sfs_ioctl_mutex);
-
 	switch (cmd) {
 	case SFSIOCFWVERS:
 		dev_dbg(sfs_dev->dev, "in SFSIOCFWVERS\n");
@@ -191,8 +191,11 @@ static void sfs_exit(struct kref *ref)
 
 void sfs_dev_destroy(struct psp_device *psp)
 {
-	struct sfs_device *sfs_dev = psp->sfs_data;
+	struct sfs_device *sfs_dev;
+
+	guard(mutex)(&sfs_ioctl_mutex);
 
+	sfs_dev = psp->sfs_data;
 	if (!sfs_dev)
 		return;
 
diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c
index 3f9843fa77827c..c7ac840cbe136a 100644
--- a/drivers/crypto/ccp/sp-platform.c
+++ b/drivers/crypto/ccp/sp-platform.c
@@ -40,8 +40,8 @@ static const struct sp_dev_vdata dev_vdata[] = {
 };
 
 static const struct acpi_device_id sp_acpi_match[] = {
-	{ "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] },
-	{ },
+	{ .id = "AMDI0C00", .driver_data = (kernel_ulong_t)&dev_vdata[0] },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, sp_acpi_match);
 
diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c
index 088c4603047f04..d57a98f89e536c 100644
--- a/drivers/crypto/ccree/cc_aead.c
+++ b/drivers/crypto/ccree/cc_aead.c
@@ -226,8 +226,8 @@ static void cc_aead_complete(struct device *dev, void *cc_req, int err)
 		goto done;
 
 	if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
-		if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
-			   ctx->authsize) != 0) {
+		if (crypto_memneq(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
+				  ctx->authsize)) {
 			dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
 				ctx->authsize, ctx->cipher_mode);
 			/* In case of payload authentication failure, MUST NOT
diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c
index dc7e0cd51c2596..a592f5ec1b6a67 100644
--- a/drivers/crypto/ccree/cc_buffer_mgr.c
+++ b/drivers/crypto/ccree/cc_buffer_mgr.c
@@ -717,7 +717,7 @@ static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata,
 			}
 		} else { /* Contig. ICV */
 			sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
-			/*Should hanlde if the sg is not contig.*/
+			/*Should handle if the sg is not contig.*/
 			areq_ctx->icv_dma_addr = sg_dma_address(sg) +
 				(*src_last_bytes - authsize);
 			areq_ctx->icv_virt_addr = sg_virt(sg) +
@@ -749,7 +749,7 @@ static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata,
 
 		} else { /* Contig. ICV */
 			sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
-			/*Should hanlde if the sg is not contig.*/
+			/*Should handle if the sg is not contig.*/
 			areq_ctx->icv_dma_addr = sg_dma_address(sg) +
 				(*src_last_bytes - authsize);
 			areq_ctx->icv_virt_addr = sg_virt(sg) +
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 6dec4228276827..e2f2b2a6200ede 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -1605,7 +1605,7 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
 		req_ctx->hctx_wr.dma_addr = 0;
 	}
 	chcr_add_hash_src_ent(req, ulptx, param);
-	/* Request upto max wr size */
+	/* Request up to max wr size */
 	temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
 				(param->sg_len + param->bfr_len) : 0);
 	atomic_inc(&adap->chcr_stats.digest_rqst);
diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig
index aeff08ccbaddd4..e38a71d962016f 100644
--- a/drivers/crypto/hisilicon/Kconfig
+++ b/drivers/crypto/hisilicon/Kconfig
@@ -1,19 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 
-config CRYPTO_DEV_HISI_SEC
-	tristate "Support for Hisilicon SEC crypto block cipher accelerator"
-	select CRYPTO_SKCIPHER
-	select CRYPTO_ALGAPI
-	select CRYPTO_LIB_DES
-	select SG_SPLIT
-	depends on ARM64 || COMPILE_TEST
-	depends on HAS_IOMEM
-	help
-	  Support for Hisilicon SEC Engine in Hip06 and Hip07
-
-	  To compile this as a module, choose M here: the module
-	  will be called hisi_sec.
-
 config CRYPTO_DEV_HISI_SEC2
 	tristate "Support for HiSilicon SEC2 crypto block cipher accelerator"
 	select CRYPTO_SKCIPHER
diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile
index e1068ee9f97318..a62965be863314 100644
--- a/drivers/crypto/hisilicon/Makefile
+++ b/drivers/crypto/hisilicon/Makefile
@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hpre/
-obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/
 obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/
 obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o
 hisi_qm-objs = qm.o sgl.o debugfs.o
diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h
index 021dbd9a1d48fb..f73d01f6da878d 100644
--- a/drivers/crypto/hisilicon/hpre/hpre.h
+++ b/drivers/crypto/hisilicon/hpre/hpre.h
@@ -70,7 +70,6 @@ struct hpre_debug {
 struct hpre {
 	struct hisi_qm qm;
 	struct hpre_debug debug;
-	unsigned long status;
 };
 
 enum hpre_alg_type {
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
index 09077abbf6adee..66a276786a7d01 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
@@ -21,19 +21,15 @@ struct hpre_ctx;
 #define HPRE_CRYPTO_ALG_PRI	1000
 #define HPRE_ALIGN_SZ		64
 #define HPRE_BITS_2_BYTES_SHIFT	3
-#define HPRE_RSA_512BITS_KSZ	64
-#define HPRE_RSA_1536BITS_KSZ	192
 #define HPRE_CRT_PRMS		5
 #define HPRE_CRT_Q		2
 #define HPRE_CRT_P		3
 #define HPRE_CRT_INV		4
 #define HPRE_DH_G_FLAG		0x02
 #define HPRE_TRY_SEND_TIMES	100
-#define HPRE_INVLD_REQ_ID		(-1)
 
 #define HPRE_SQE_ALG_BITS	5
 #define HPRE_SQE_DONE_SHIFT	30
-#define HPRE_DH_MAX_P_SZ	512
 
 #define HPRE_DFX_SEC_TO_US	1000000
 #define HPRE_DFX_US_TO_NS	1000
@@ -56,7 +52,6 @@ struct hpre_ctx;
 #define HPRE_DRV_RSA_MASK_CAP		BIT(0)
 #define HPRE_DRV_DH_MASK_CAP		BIT(1)
 #define HPRE_DRV_ECDH_MASK_CAP		BIT(2)
-#define HPRE_DRV_X25519_MASK_CAP	BIT(5)
 
 static DEFINE_MUTEX(hpre_algs_lock);
 static unsigned int hpre_available_devs;
@@ -135,9 +130,9 @@ struct hpre_asym_request {
 		struct kpp_request *dh;
 		struct kpp_request *ecdh;
 	} areq;
-	int err;
 	hpre_cb cb;
 	struct timespec64 req_time;
+	u32 flags;
 };
 
 static inline unsigned int hpre_align_sz(void)
@@ -186,6 +181,7 @@ static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
 				struct scatterlist *data, unsigned int len,
 				int is_src, dma_addr_t *tmp)
 {
+	gfp_t gfp = hpre_req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
 	struct hpre_ctx *ctx = hpre_req->ctx;
 	struct device *dev = ctx->dev;
 	void *ptr;
@@ -195,7 +191,7 @@ static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
 	if (unlikely(shift < 0))
 		return -EINVAL;
 
-	ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_ATOMIC);
+	ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, gfp);
 	if (unlikely(!ptr))
 		return -ENOMEM;
 
@@ -419,6 +415,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
 		h_req = PTR_ALIGN(tmp, hpre_align_sz());
 		h_req->cb = hpre_rsa_cb;
 		h_req->areq.rsa = akreq;
+		h_req->flags = akreq->base.flags;
 		msg = &h_req->req;
 		memset(msg, 0, sizeof(*msg));
 	} else {
@@ -433,6 +430,7 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
 		h_req = PTR_ALIGN(tmp, hpre_align_sz());
 		h_req->cb = hpre_dh_cb;
 		h_req->areq.dh = kreq;
+		h_req->flags = kreq->base.flags;
 		msg = &h_req->req;
 		memset(msg, 0, sizeof(*msg));
 		msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
@@ -720,7 +718,7 @@ static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
 
 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
 {
-	while (!**ptr && *len) {
+	while (*len && !**ptr) {
 		(*ptr)++;
 		(*len)--;
 	}
@@ -1407,8 +1405,6 @@ static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
 	if (unlikely(dma_mapping_error(dev, dma)))
 		return;
 
-	if (req->dst)
-		dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
 	if (dst)
 		dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
 }
@@ -1452,13 +1448,14 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
 
 	if (req->dst_len < ctx->key_sz << 1) {
 		req->dst_len = ctx->key_sz << 1;
-		return -EINVAL;
+		return -EOVERFLOW;
 	}
 
 	tmp = kpp_request_ctx(req);
 	h_req = PTR_ALIGN(tmp, hpre_align_sz());
 	h_req->cb = hpre_ecdh_cb;
 	h_req->areq.ecdh = req;
+	h_req->flags = req->base.flags;
 	msg = &h_req->req;
 	memset(msg, 0, sizeof(*msg));
 	msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
@@ -1477,6 +1474,7 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
 static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
 				   struct scatterlist *data, unsigned int len)
 {
+	gfp_t gfp = hpre_req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
 	struct hpre_sqe *msg = &hpre_req->req;
 	struct hpre_ctx *ctx = hpre_req->ctx;
 	struct device *dev = ctx->dev;
@@ -1490,7 +1488,7 @@ static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
 	if (unlikely(shift < 0))
 		return -EINVAL;
 
-	ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
+	ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, gfp);
 	if (unlikely(!ptr))
 		return -ENOMEM;
 
@@ -1512,13 +1510,13 @@ static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
 	struct device *dev = ctx->dev;
 	dma_addr_t dma;
 
-	if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
-		dev_err(dev, "data or data length is illegal!\n");
+	if (unlikely(!data || !sg_is_last(data))) {
+		dev_err(dev, "data is illegal!\n");
 		return -EINVAL;
 	}
 
 	hpre_req->dst = NULL;
-	dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
+	dma = dma_map_single(dev, sg_virt(data), ctx->key_sz << 1, DMA_FROM_DEVICE);
 	if (unlikely(dma_mapping_error(dev, dma))) {
 		dev_err(dev, "dma map data err!\n");
 		return -ENOMEM;
diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c
index b6903fce60717f..03aa11a7a3b4f8 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_main.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_main.c
@@ -16,7 +16,6 @@
 #define CAP_FILE_PERMISSION		0444
 #define HPRE_CTRL_CNT_CLR_CE_BIT	BIT(0)
 #define HPRE_CTRL_CNT_CLR_CE		0x301000
-#define HPRE_FSM_MAX_CNT		0x301008
 #define HPRE_VFG_AXQOS			0x30100c
 #define HPRE_VFG_AXCACHE		0x301010
 #define HPRE_RDCHN_INI_CFG		0x301014
@@ -41,7 +40,6 @@
 #define HPRE_HAC_INT_SET		0x301500
 #define HPRE_AXI_ERROR_MASK		GENMASK(21, 10)
 #define HPRE_RNG_TIMEOUT_NUM		0x301A34
-#define HPRE_CORE_INT_ENABLE		0
 #define HPRE_RDCHN_INI_ST		0x301a00
 #define HPRE_CLSTR_BASE			0x302000
 #define HPRE_CORE_EN_OFFSET		0x04
@@ -61,8 +59,6 @@
 #define HPRE_CORE_ENB		(HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET)
 #define HPRE_CORE_INI_CFG	(HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET)
 #define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET)
-#define HPRE_HAC_ECC1_CNT		0x301a04
-#define HPRE_HAC_ECC2_CNT		0x301a08
 #define HPRE_HAC_SOURCE_INT		0x301600
 #define HPRE_CLSTR_ADDR_INTRVL		0x1000
 #define HPRE_CLUSTER_INQURY		0x100
@@ -381,6 +377,14 @@ static const char *hpre_channel_name[HPRE_MAX_CHANNEL_NUM] = {
 
 static const struct hisi_qm_err_ini hpre_err_ini;
 
+static inline u8 hpre_get_clusters_num(struct hisi_qm *qm)
+{
+	u32 cap = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
+
+	return (cap >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
+		hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+}
+
 bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg)
 {
 	u32 cap_val;
@@ -557,15 +561,12 @@ static int hpre_set_cluster(struct hisi_qm *qm)
 	struct device *dev = &qm->pdev->dev;
 	u32 cluster_core_mask;
 	unsigned long offset;
-	u32 hpre_core_info;
 	u8 clusters_num;
 	u32 val = 0;
 	int ret, i;
 
 	cluster_core_mask = qm->cap_tables.dev_cap_table[HPRE_CORE_EN].cap_val;
-	hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
-	clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
-			hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+	clusters_num = hpre_get_clusters_num(qm);
 	for (i = 0; i < clusters_num; i++) {
 		offset = i * HPRE_CLSTR_ADDR_INTRVL;
 
@@ -659,7 +660,6 @@ static void hpre_enable_clock_gate(struct hisi_qm *qm)
 {
 	unsigned long offset;
 	u8 clusters_num, i;
-	u32 hpre_core_info;
 	u32 val;
 
 	if (qm->ver < QM_HW_V3)
@@ -673,9 +673,7 @@ static void hpre_enable_clock_gate(struct hisi_qm *qm)
 	val |= HPRE_PEH_CFG_AUTO_GATE_EN;
 	writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE);
 
-	hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
-	clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
-			hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+	clusters_num = hpre_get_clusters_num(qm);
 	for (i = 0; i < clusters_num; i++) {
 		offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL;
 		val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL);
@@ -692,7 +690,6 @@ static void hpre_disable_clock_gate(struct hisi_qm *qm)
 {
 	unsigned long offset;
 	u8 clusters_num, i;
-	u32 hpre_core_info;
 	u32 val;
 
 	if (qm->ver < QM_HW_V3)
@@ -706,9 +703,7 @@ static void hpre_disable_clock_gate(struct hisi_qm *qm)
 	val &= ~HPRE_PEH_CFG_AUTO_GATE_EN;
 	writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE);
 
-	hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
-	clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
-			hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+	clusters_num = hpre_get_clusters_num(qm);
 	for (i = 0; i < clusters_num; i++) {
 		offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL;
 		val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL);
@@ -782,14 +777,11 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
 static void hpre_cnt_regs_clear(struct hisi_qm *qm)
 {
 	unsigned long offset;
-	u32 hpre_core_info;
 	u8 clusters_num;
 	int i;
 
 	/* clear clusterX/cluster_ctrl */
-	hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
-	clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
-			hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+	clusters_num = hpre_get_clusters_num(qm);
 	for (i = 0; i < clusters_num; i++) {
 		offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL;
 		writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY);
@@ -1072,13 +1064,10 @@ static int hpre_cluster_debugfs_init(struct hisi_qm *qm)
 	char buf[HPRE_DBGFS_VAL_MAX_LEN];
 	struct debugfs_regset32 *regset;
 	struct dentry *tmp_d;
-	u32 hpre_core_info;
 	u8 clusters_num;
 	int i, ret;
 
-	hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
-	clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
-			hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+	clusters_num = hpre_get_clusters_num(qm);
 	for (i = 0; i < clusters_num; i++) {
 		ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i);
 		if (ret >= HPRE_DBGFS_VAL_MAX_LEN)
@@ -1308,13 +1297,10 @@ static int hpre_show_last_regs_init(struct hisi_qm *qm)
 	int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs);
 	struct qm_debug *debug = &qm->debug;
 	void __iomem *io_base;
-	u32 hpre_core_info;
 	u8 clusters_num;
 	int i, j, idx;
 
-	hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
-	clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
-			hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+	clusters_num = hpre_get_clusters_num(qm);
 	debug->last_words = kcalloc(cluster_dfx_regs_num * clusters_num +
 			com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL);
 	if (!debug->last_words)
@@ -1354,7 +1340,6 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm)
 	struct qm_debug *debug = &qm->debug;
 	struct pci_dev *pdev = qm->pdev;
 	void __iomem *io_base;
-	u32 hpre_core_info;
 	u8 clusters_num;
 	int i, j, idx;
 	u32 val;
@@ -1370,9 +1355,7 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm)
 			  hpre_com_dfx_regs[i].name, debug->last_words[i], val);
 	}
 
-	hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
-	clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
-			hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
+	clusters_num = hpre_get_clusters_num(qm);
 	for (i = 0; i < clusters_num; i++) {
 		io_base = qm->io_base + hpre_cluster_offsets[i];
 		for (j = 0; j <  cluster_dfx_regs_num; j++) {
diff --git a/drivers/crypto/hisilicon/sec/Makefile b/drivers/crypto/hisilicon/sec/Makefile
deleted file mode 100644
index a55b698e0c2733..00000000000000
--- a/drivers/crypto/hisilicon/sec/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += hisi_sec.o
-hisi_sec-y = sec_algs.o sec_drv.o
diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c
deleted file mode 100644
index 85eecbb40e7e79..00000000000000
--- a/drivers/crypto/hisilicon/sec/sec_algs.c
+++ /dev/null
@@ -1,1122 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright (c) 2016-2017 HiSilicon Limited. */
-#include <linux/crypto.h>
-#include <linux/dma-mapping.h>
-#include <linux/dmapool.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-
-#include <crypto/aes.h>
-#include <crypto/algapi.h>
-#include <crypto/internal/des.h>
-#include <crypto/skcipher.h>
-#include <crypto/xts.h>
-#include <crypto/internal/skcipher.h>
-
-#include "sec_drv.h"
-
-#define SEC_MAX_CIPHER_KEY		64
-#define SEC_REQ_LIMIT SZ_32M
-
-struct sec_c_alg_cfg {
-	unsigned c_alg		: 3;
-	unsigned c_mode		: 3;
-	unsigned key_len	: 2;
-	unsigned c_width	: 2;
-};
-
-static const struct sec_c_alg_cfg sec_c_alg_cfgs[] =  {
-	[SEC_C_DES_ECB_64] = {
-		.c_alg = SEC_C_ALG_DES,
-		.c_mode = SEC_C_MODE_ECB,
-		.key_len = SEC_KEY_LEN_DES,
-	},
-	[SEC_C_DES_CBC_64] = {
-		.c_alg = SEC_C_ALG_DES,
-		.c_mode = SEC_C_MODE_CBC,
-		.key_len = SEC_KEY_LEN_DES,
-	},
-	[SEC_C_3DES_ECB_192_3KEY] = {
-		.c_alg = SEC_C_ALG_3DES,
-		.c_mode = SEC_C_MODE_ECB,
-		.key_len = SEC_KEY_LEN_3DES_3_KEY,
-	},
-	[SEC_C_3DES_ECB_192_2KEY] = {
-		.c_alg = SEC_C_ALG_3DES,
-		.c_mode = SEC_C_MODE_ECB,
-		.key_len = SEC_KEY_LEN_3DES_2_KEY,
-	},
-	[SEC_C_3DES_CBC_192_3KEY] = {
-		.c_alg = SEC_C_ALG_3DES,
-		.c_mode = SEC_C_MODE_CBC,
-		.key_len = SEC_KEY_LEN_3DES_3_KEY,
-	},
-	[SEC_C_3DES_CBC_192_2KEY] = {
-		.c_alg = SEC_C_ALG_3DES,
-		.c_mode = SEC_C_MODE_CBC,
-		.key_len = SEC_KEY_LEN_3DES_2_KEY,
-	},
-	[SEC_C_AES_ECB_128] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_ECB,
-		.key_len = SEC_KEY_LEN_AES_128,
-	},
-	[SEC_C_AES_ECB_192] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_ECB,
-		.key_len = SEC_KEY_LEN_AES_192,
-	},
-	[SEC_C_AES_ECB_256] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_ECB,
-		.key_len = SEC_KEY_LEN_AES_256,
-	},
-	[SEC_C_AES_CBC_128] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_CBC,
-		.key_len = SEC_KEY_LEN_AES_128,
-	},
-	[SEC_C_AES_CBC_192] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_CBC,
-		.key_len = SEC_KEY_LEN_AES_192,
-	},
-	[SEC_C_AES_CBC_256] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_CBC,
-		.key_len = SEC_KEY_LEN_AES_256,
-	},
-	[SEC_C_AES_CTR_128] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_CTR,
-		.key_len = SEC_KEY_LEN_AES_128,
-	},
-	[SEC_C_AES_CTR_192] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_CTR,
-		.key_len = SEC_KEY_LEN_AES_192,
-	},
-	[SEC_C_AES_CTR_256] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_CTR,
-		.key_len = SEC_KEY_LEN_AES_256,
-	},
-	[SEC_C_AES_XTS_128] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_XTS,
-		.key_len = SEC_KEY_LEN_AES_128,
-	},
-	[SEC_C_AES_XTS_256] = {
-		.c_alg = SEC_C_ALG_AES,
-		.c_mode = SEC_C_MODE_XTS,
-		.key_len = SEC_KEY_LEN_AES_256,
-	},
-	[SEC_C_NULL] = {
-	},
-};
-
-/*
- * Mutex used to ensure safe operation of reference count of
- * alg providers
- */
-static DEFINE_MUTEX(algs_lock);
-static unsigned int active_devs;
-
-static void sec_alg_skcipher_init_template(struct sec_alg_tfm_ctx *ctx,
-					   struct sec_bd_info *req,
-					   enum sec_cipher_alg alg)
-{
-	const struct sec_c_alg_cfg *cfg = &sec_c_alg_cfgs[alg];
-
-	memset(req, 0, sizeof(*req));
-	req->w0 |= cfg->c_mode << SEC_BD_W0_C_MODE_S;
-	req->w1 |= cfg->c_alg << SEC_BD_W1_C_ALG_S;
-	req->w3 |= cfg->key_len << SEC_BD_W3_C_KEY_LEN_S;
-	req->w0 |= cfg->c_width << SEC_BD_W0_C_WIDTH_S;
-
-	req->cipher_key_addr_lo = lower_32_bits(ctx->pkey);
-	req->cipher_key_addr_hi = upper_32_bits(ctx->pkey);
-}
-
-static void sec_alg_skcipher_init_context(struct crypto_skcipher *atfm,
-					  const u8 *key,
-					  unsigned int keylen,
-					  enum sec_cipher_alg alg)
-{
-	struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
-	struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	ctx->cipher_alg = alg;
-	memcpy(ctx->key, key, keylen);
-	sec_alg_skcipher_init_template(ctx, &ctx->req_template,
-				       ctx->cipher_alg);
-}
-
-static void sec_free_hw_sgl(struct sec_hw_sgl *hw_sgl,
-			    dma_addr_t psec_sgl, struct sec_dev_info *info)
-{
-	struct sec_hw_sgl *sgl_current, *sgl_next;
-	dma_addr_t sgl_next_dma;
-
-	sgl_current = hw_sgl;
-	while (sgl_current) {
-		sgl_next = sgl_current->next;
-		sgl_next_dma = sgl_current->next_sgl;
-
-		dma_pool_free(info->hw_sgl_pool, sgl_current, psec_sgl);
-
-		sgl_current = sgl_next;
-		psec_sgl = sgl_next_dma;
-	}
-}
-
-static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl,
-				     dma_addr_t *psec_sgl,
-				     struct scatterlist *sgl,
-				     int count,
-				     struct sec_dev_info *info,
-				     gfp_t gfp)
-{
-	struct sec_hw_sgl *sgl_current = NULL;
-	struct sec_hw_sgl *sgl_next;
-	dma_addr_t sgl_next_dma;
-	struct scatterlist *sg;
-	int ret, sge_index, i;
-
-	if (!count)
-		return -EINVAL;
-
-	for_each_sg(sgl, sg, count, i) {
-		sge_index = i % SEC_MAX_SGE_NUM;
-		if (sge_index == 0) {
-			sgl_next = dma_pool_zalloc(info->hw_sgl_pool,
-						   gfp, &sgl_next_dma);
-			if (!sgl_next) {
-				ret = -ENOMEM;
-				goto err_free_hw_sgls;
-			}
-
-			if (!sgl_current) { /* First one */
-				*psec_sgl = sgl_next_dma;
-				*sec_sgl = sgl_next;
-			} else { /* Chained */
-				sgl_current->entry_sum_in_sgl = SEC_MAX_SGE_NUM;
-				sgl_current->next_sgl = sgl_next_dma;
-				sgl_current->next = sgl_next;
-			}
-			sgl_current = sgl_next;
-		}
-		sgl_current->sge_entries[sge_index].buf = sg_dma_address(sg);
-		sgl_current->sge_entries[sge_index].len = sg_dma_len(sg);
-		sgl_current->data_bytes_in_sgl += sg_dma_len(sg);
-	}
-	sgl_current->entry_sum_in_sgl = count % SEC_MAX_SGE_NUM;
-	sgl_current->next_sgl = 0;
-	(*sec_sgl)->entry_sum_in_chain = count;
-
-	return 0;
-
-err_free_hw_sgls:
-	sec_free_hw_sgl(*sec_sgl, *psec_sgl, info);
-	*psec_sgl = 0;
-
-	return ret;
-}
-
-static int sec_alg_skcipher_setkey(struct crypto_skcipher *tfm,
-				   const u8 *key, unsigned int keylen,
-				   enum sec_cipher_alg alg)
-{
-	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct device *dev = ctx->queue->dev_info->dev;
-
-	mutex_lock(&ctx->lock);
-	if (ctx->key) {
-		/* rekeying */
-		memset(ctx->key, 0, SEC_MAX_CIPHER_KEY);
-	} else {
-		/* new key */
-		ctx->key = dma_alloc_coherent(dev, SEC_MAX_CIPHER_KEY,
-					      &ctx->pkey, GFP_KERNEL);
-		if (!ctx->key) {
-			mutex_unlock(&ctx->lock);
-			return -ENOMEM;
-		}
-	}
-	mutex_unlock(&ctx->lock);
-	sec_alg_skcipher_init_context(tfm, key, keylen, alg);
-
-	return 0;
-}
-
-static int sec_alg_skcipher_setkey_aes_ecb(struct crypto_skcipher *tfm,
-					   const u8 *key, unsigned int keylen)
-{
-	enum sec_cipher_alg alg;
-
-	switch (keylen) {
-	case AES_KEYSIZE_128:
-		alg = SEC_C_AES_ECB_128;
-		break;
-	case AES_KEYSIZE_192:
-		alg = SEC_C_AES_ECB_192;
-		break;
-	case AES_KEYSIZE_256:
-		alg = SEC_C_AES_ECB_256;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
-}
-
-static int sec_alg_skcipher_setkey_aes_cbc(struct crypto_skcipher *tfm,
-					   const u8 *key, unsigned int keylen)
-{
-	enum sec_cipher_alg alg;
-
-	switch (keylen) {
-	case AES_KEYSIZE_128:
-		alg = SEC_C_AES_CBC_128;
-		break;
-	case AES_KEYSIZE_192:
-		alg = SEC_C_AES_CBC_192;
-		break;
-	case AES_KEYSIZE_256:
-		alg = SEC_C_AES_CBC_256;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
-}
-
-static int sec_alg_skcipher_setkey_aes_ctr(struct crypto_skcipher *tfm,
-					   const u8 *key, unsigned int keylen)
-{
-	enum sec_cipher_alg alg;
-
-	switch (keylen) {
-	case AES_KEYSIZE_128:
-		alg = SEC_C_AES_CTR_128;
-		break;
-	case AES_KEYSIZE_192:
-		alg = SEC_C_AES_CTR_192;
-		break;
-	case AES_KEYSIZE_256:
-		alg = SEC_C_AES_CTR_256;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
-}
-
-static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
-					   const u8 *key, unsigned int keylen)
-{
-	enum sec_cipher_alg alg;
-	int ret;
-
-	ret = xts_verify_key(tfm, key, keylen);
-	if (ret)
-		return ret;
-
-	switch (keylen) {
-	case AES_KEYSIZE_128 * 2:
-		alg = SEC_C_AES_XTS_128;
-		break;
-	case AES_KEYSIZE_256 * 2:
-		alg = SEC_C_AES_XTS_256;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
-}
-
-static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
-					   const u8 *key, unsigned int keylen)
-{
-	return verify_skcipher_des_key(tfm, key) ?:
-	       sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
-}
-
-static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
-					   const u8 *key, unsigned int keylen)
-{
-	return verify_skcipher_des_key(tfm, key) ?:
-	       sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
-}
-
-static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
-					    const u8 *key, unsigned int keylen)
-{
-	return verify_skcipher_des3_key(tfm, key) ?:
-	       sec_alg_skcipher_setkey(tfm, key, keylen,
-				       SEC_C_3DES_ECB_192_3KEY);
-}
-
-static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
-					    const u8 *key, unsigned int keylen)
-{
-	return verify_skcipher_des3_key(tfm, key) ?:
-	       sec_alg_skcipher_setkey(tfm, key, keylen,
-				       SEC_C_3DES_CBC_192_3KEY);
-}
-
-static void sec_alg_free_el(struct sec_request_el *el,
-			    struct sec_dev_info *info)
-{
-	sec_free_hw_sgl(el->out, el->dma_out, info);
-	sec_free_hw_sgl(el->in, el->dma_in, info);
-	kfree(el->sgl_in);
-	kfree(el->sgl_out);
-	kfree(el);
-}
-
-/* queuelock must be held */
-static int sec_send_request(struct sec_request *sec_req, struct sec_queue *queue)
-{
-	struct sec_request_el *el, *temp;
-	int ret = 0;
-
-	mutex_lock(&sec_req->lock);
-	list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
-		/*
-		 * Add to hardware queue only under following circumstances
-		 * 1) Software and hardware queue empty so no chain dependencies
-		 * 2) No dependencies as new IV - (check software queue empty
-		 *    to maintain order)
-		 * 3) No dependencies because the mode does no chaining.
-		 *
-		 * In other cases first insert onto the software queue which
-		 * is then emptied as requests complete
-		 */
-		if (!queue->havesoftqueue ||
-		    (kfifo_is_empty(&queue->softqueue) &&
-		     sec_queue_empty(queue))) {
-			ret = sec_queue_send(queue, &el->req, sec_req);
-			if (ret == -EAGAIN) {
-				/* Wait unti we can send then try again */
-				/* DEAD if here - should not happen */
-				ret = -EBUSY;
-				goto err_unlock;
-			}
-		} else {
-			kfifo_put(&queue->softqueue, el);
-		}
-	}
-err_unlock:
-	mutex_unlock(&sec_req->lock);
-
-	return ret;
-}
-
-static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp,
-				      struct crypto_async_request *req_base)
-{
-	struct skcipher_request *skreq = container_of(req_base,
-						      struct skcipher_request,
-						      base);
-	struct sec_request *sec_req = skcipher_request_ctx(skreq);
-	struct sec_request *backlog_req;
-	struct sec_request_el *sec_req_el, *nextrequest;
-	struct sec_alg_tfm_ctx *ctx = sec_req->tfm_ctx;
-	struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
-	struct device *dev = ctx->queue->dev_info->dev;
-	int icv_or_skey_en, ret;
-	bool done;
-
-	sec_req_el = list_first_entry(&sec_req->elements, struct sec_request_el,
-				      head);
-	icv_or_skey_en = (sec_resp->w0 & SEC_BD_W0_ICV_OR_SKEY_EN_M) >>
-		SEC_BD_W0_ICV_OR_SKEY_EN_S;
-	if (sec_resp->w1 & SEC_BD_W1_BD_INVALID || icv_or_skey_en == 3) {
-		dev_err(dev, "Got an invalid answer %lu %d\n",
-			sec_resp->w1 & SEC_BD_W1_BD_INVALID,
-			icv_or_skey_en);
-		sec_req->err = -EINVAL;
-		/*
-		 * We need to muddle on to avoid getting stuck with elements
-		 * on the queue. Error will be reported so requester so
-		 * it should be able to handle appropriately.
-		 */
-	}
-
-	spin_lock_bh(&ctx->queue->queuelock);
-	/* Put the IV in place for chained cases */
-	switch (ctx->cipher_alg) {
-	case SEC_C_AES_CBC_128:
-	case SEC_C_AES_CBC_192:
-	case SEC_C_AES_CBC_256:
-		if (sec_req_el->req.w0 & SEC_BD_W0_DE)
-			sg_pcopy_to_buffer(sec_req_el->sgl_out,
-					   sg_nents(sec_req_el->sgl_out),
-					   skreq->iv,
-					   crypto_skcipher_ivsize(atfm),
-					   sec_req_el->el_length -
-					   crypto_skcipher_ivsize(atfm));
-		else
-			sg_pcopy_to_buffer(sec_req_el->sgl_in,
-					   sg_nents(sec_req_el->sgl_in),
-					   skreq->iv,
-					   crypto_skcipher_ivsize(atfm),
-					   sec_req_el->el_length -
-					   crypto_skcipher_ivsize(atfm));
-		/* No need to sync to the device as coherent DMA */
-		break;
-	case SEC_C_AES_CTR_128:
-	case SEC_C_AES_CTR_192:
-	case SEC_C_AES_CTR_256:
-		crypto_inc(skreq->iv, 16);
-		break;
-	default:
-		/* Do not update */
-		break;
-	}
-
-	if (ctx->queue->havesoftqueue &&
-	    !kfifo_is_empty(&ctx->queue->softqueue) &&
-	    sec_queue_empty(ctx->queue)) {
-		ret = kfifo_get(&ctx->queue->softqueue, &nextrequest);
-		if (ret <= 0)
-			dev_err(dev,
-				"Error getting next element from kfifo %d\n",
-				ret);
-		else
-			/* We know there is space so this cannot fail */
-			sec_queue_send(ctx->queue, &nextrequest->req,
-				       nextrequest->sec_req);
-	} else if (!list_empty(&ctx->backlog)) {
-		/* Need to verify there is room first */
-		backlog_req = list_first_entry(&ctx->backlog,
-					       typeof(*backlog_req),
-					       backlog_head);
-		if (sec_queue_can_enqueue(ctx->queue,
-		    backlog_req->num_elements) ||
-		    (ctx->queue->havesoftqueue &&
-		     kfifo_avail(&ctx->queue->softqueue) >
-		     backlog_req->num_elements)) {
-			sec_send_request(backlog_req, ctx->queue);
-			crypto_request_complete(backlog_req->req_base,
-						-EINPROGRESS);
-			list_del(&backlog_req->backlog_head);
-		}
-	}
-	spin_unlock_bh(&ctx->queue->queuelock);
-
-	mutex_lock(&sec_req->lock);
-	list_del(&sec_req_el->head);
-	mutex_unlock(&sec_req->lock);
-	sec_alg_free_el(sec_req_el, ctx->queue->dev_info);
-
-	/*
-	 * Request is done.
-	 * The dance is needed as the lock is freed in the completion
-	 */
-	mutex_lock(&sec_req->lock);
-	done = list_empty(&sec_req->elements);
-	mutex_unlock(&sec_req->lock);
-	if (done) {
-		if (crypto_skcipher_ivsize(atfm)) {
-			dma_unmap_single(dev, sec_req->dma_iv,
-					 crypto_skcipher_ivsize(atfm),
-					 DMA_TO_DEVICE);
-		}
-		dma_unmap_sg(dev, skreq->src, sec_req->len_in,
-			     DMA_BIDIRECTIONAL);
-		if (skreq->src != skreq->dst)
-			dma_unmap_sg(dev, skreq->dst, sec_req->len_out,
-				     DMA_BIDIRECTIONAL);
-		skcipher_request_complete(skreq, sec_req->err);
-	}
-}
-
-void sec_alg_callback(struct sec_bd_info *resp, void *shadow)
-{
-	struct sec_request *sec_req = shadow;
-
-	sec_req->cb(resp, sec_req->req_base);
-}
-
-static int sec_alg_alloc_and_calc_split_sizes(int length, size_t **split_sizes,
-					      int *steps, gfp_t gfp)
-{
-	size_t *sizes;
-	int i;
-
-	/* Split into suitable sized blocks */
-	*steps = roundup(length, SEC_REQ_LIMIT) / SEC_REQ_LIMIT;
-	sizes = kzalloc_objs(*sizes, *steps, gfp);
-	if (!sizes)
-		return -ENOMEM;
-
-	for (i = 0; i < *steps - 1; i++)
-		sizes[i] = SEC_REQ_LIMIT;
-	sizes[*steps - 1] = length - SEC_REQ_LIMIT * (*steps - 1);
-	*split_sizes = sizes;
-
-	return 0;
-}
-
-static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes,
-				int steps, struct scatterlist ***splits,
-				int **splits_nents,
-				int sgl_len_in,
-				struct device *dev, gfp_t gfp)
-{
-	int ret, count;
-
-	count = dma_map_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
-	if (!count)
-		return -EINVAL;
-
-	*splits = kzalloc_objs(struct scatterlist *, steps, gfp);
-	if (!*splits) {
-		ret = -ENOMEM;
-		goto err_unmap_sg;
-	}
-	*splits_nents = kzalloc_objs(int, steps, gfp);
-	if (!*splits_nents) {
-		ret = -ENOMEM;
-		goto err_free_splits;
-	}
-
-	/* output the scatter list before and after this */
-	ret = sg_split(sgl, count, 0, steps, split_sizes,
-		       *splits, *splits_nents, gfp);
-	if (ret) {
-		ret = -ENOMEM;
-		goto err_free_splits_nents;
-	}
-
-	return 0;
-
-err_free_splits_nents:
-	kfree(*splits_nents);
-err_free_splits:
-	kfree(*splits);
-err_unmap_sg:
-	dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
-
-	return ret;
-}
-
-/*
- * Reverses the sec_map_and_split_sg call for messages not yet added to
- * the queues.
- */
-static void sec_unmap_sg_on_err(struct scatterlist *sgl, int steps,
-				struct scatterlist **splits, int *splits_nents,
-				int sgl_len_in, struct device *dev)
-{
-	int i;
-
-	for (i = 0; i < steps; i++)
-		kfree(splits[i]);
-	kfree(splits_nents);
-	kfree(splits);
-
-	dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
-}
-
-static struct sec_request_el
-*sec_alg_alloc_and_fill_el(struct sec_bd_info *template, int encrypt,
-			   int el_size, bool different_dest,
-			   struct scatterlist *sgl_in, int n_ents_in,
-			   struct scatterlist *sgl_out, int n_ents_out,
-			   struct sec_dev_info *info, gfp_t gfp)
-{
-	struct sec_request_el *el;
-	struct sec_bd_info *req;
-	int ret;
-
-	el = kzalloc_obj(*el, gfp);
-	if (!el)
-		return ERR_PTR(-ENOMEM);
-	el->el_length = el_size;
-	req = &el->req;
-	memcpy(req, template, sizeof(*req));
-
-	req->w0 &= ~SEC_BD_W0_CIPHER_M;
-	if (encrypt)
-		req->w0 |= SEC_CIPHER_ENCRYPT << SEC_BD_W0_CIPHER_S;
-	else
-		req->w0 |= SEC_CIPHER_DECRYPT << SEC_BD_W0_CIPHER_S;
-
-	req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_19_16_M;
-	req->w0 |= ((el_size >> 16) << SEC_BD_W0_C_GRAN_SIZE_19_16_S) &
-		SEC_BD_W0_C_GRAN_SIZE_19_16_M;
-
-	req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_21_20_M;
-	req->w0 |= ((el_size >> 20) << SEC_BD_W0_C_GRAN_SIZE_21_20_S) &
-		SEC_BD_W0_C_GRAN_SIZE_21_20_M;
-
-	/* Writing whole u32 so no need to take care of masking */
-	req->w2 = ((1 << SEC_BD_W2_GRAN_NUM_S) & SEC_BD_W2_GRAN_NUM_M) |
-		((el_size << SEC_BD_W2_C_GRAN_SIZE_15_0_S) &
-		 SEC_BD_W2_C_GRAN_SIZE_15_0_M);
-
-	req->w3 &= ~SEC_BD_W3_CIPHER_LEN_OFFSET_M;
-	req->w1 |= SEC_BD_W1_ADDR_TYPE;
-
-	el->sgl_in = sgl_in;
-
-	ret = sec_alloc_and_fill_hw_sgl(&el->in, &el->dma_in, el->sgl_in,
-					n_ents_in, info, gfp);
-	if (ret)
-		goto err_free_el;
-
-	req->data_addr_lo = lower_32_bits(el->dma_in);
-	req->data_addr_hi = upper_32_bits(el->dma_in);
-
-	if (different_dest) {
-		el->sgl_out = sgl_out;
-		ret = sec_alloc_and_fill_hw_sgl(&el->out, &el->dma_out,
-						el->sgl_out,
-						n_ents_out, info, gfp);
-		if (ret)
-			goto err_free_hw_sgl_in;
-
-		req->w0 |= SEC_BD_W0_DE;
-		req->cipher_destin_addr_lo = lower_32_bits(el->dma_out);
-		req->cipher_destin_addr_hi = upper_32_bits(el->dma_out);
-
-	} else {
-		req->w0 &= ~SEC_BD_W0_DE;
-		req->cipher_destin_addr_lo = lower_32_bits(el->dma_in);
-		req->cipher_destin_addr_hi = upper_32_bits(el->dma_in);
-	}
-
-	return el;
-
-err_free_hw_sgl_in:
-	sec_free_hw_sgl(el->in, el->dma_in, info);
-err_free_el:
-	kfree(el);
-
-	return ERR_PTR(ret);
-}
-
-static int sec_alg_skcipher_crypto(struct skcipher_request *skreq,
-				   bool encrypt)
-{
-	struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
-	struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
-	struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
-	struct sec_queue *queue = ctx->queue;
-	struct sec_request *sec_req = skcipher_request_ctx(skreq);
-	struct sec_dev_info *info = queue->dev_info;
-	int i, ret, steps;
-	size_t *split_sizes;
-	struct scatterlist **splits_in;
-	struct scatterlist **splits_out = NULL;
-	int *splits_in_nents;
-	int *splits_out_nents = NULL;
-	struct sec_request_el *el, *temp;
-	bool split = skreq->src != skreq->dst;
-	gfp_t gfp = skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
-
-	mutex_init(&sec_req->lock);
-	sec_req->req_base = &skreq->base;
-	sec_req->err = 0;
-	/* SGL mapping out here to allow us to break it up as necessary */
-	sec_req->len_in = sg_nents(skreq->src);
-
-	ret = sec_alg_alloc_and_calc_split_sizes(skreq->cryptlen, &split_sizes,
-						 &steps, gfp);
-	if (ret)
-		return ret;
-	sec_req->num_elements = steps;
-	ret = sec_map_and_split_sg(skreq->src, split_sizes, steps, &splits_in,
-				   &splits_in_nents, sec_req->len_in,
-				   info->dev, gfp);
-	if (ret)
-		goto err_free_split_sizes;
-
-	if (split) {
-		sec_req->len_out = sg_nents(skreq->dst);
-		ret = sec_map_and_split_sg(skreq->dst, split_sizes, steps,
-					   &splits_out, &splits_out_nents,
-					   sec_req->len_out, info->dev, gfp);
-		if (ret)
-			goto err_unmap_in_sg;
-	}
-	/* Shared info stored in seq_req - applies to all BDs */
-	sec_req->tfm_ctx = ctx;
-	sec_req->cb = sec_skcipher_alg_callback;
-	INIT_LIST_HEAD(&sec_req->elements);
-
-	/*
-	 * Future optimization.
-	 * In the chaining case we can't use a dma pool bounce buffer
-	 * but in the case where we know there is no chaining we can
-	 */
-	if (crypto_skcipher_ivsize(atfm)) {
-		sec_req->dma_iv = dma_map_single(info->dev, skreq->iv,
-						 crypto_skcipher_ivsize(atfm),
-						 DMA_TO_DEVICE);
-		if (dma_mapping_error(info->dev, sec_req->dma_iv)) {
-			ret = -ENOMEM;
-			goto err_unmap_out_sg;
-		}
-	}
-
-	/* Set them all up then queue - cleaner error handling. */
-	for (i = 0; i < steps; i++) {
-		el = sec_alg_alloc_and_fill_el(&ctx->req_template,
-					       encrypt ? 1 : 0,
-					       split_sizes[i],
-					       skreq->src != skreq->dst,
-					       splits_in[i], splits_in_nents[i],
-					       split ? splits_out[i] : NULL,
-					       split ? splits_out_nents[i] : 0,
-					       info, gfp);
-		if (IS_ERR(el)) {
-			ret = PTR_ERR(el);
-			goto err_free_elements;
-		}
-		el->req.cipher_iv_addr_lo = lower_32_bits(sec_req->dma_iv);
-		el->req.cipher_iv_addr_hi = upper_32_bits(sec_req->dma_iv);
-		el->sec_req = sec_req;
-		list_add_tail(&el->head, &sec_req->elements);
-	}
-
-	/*
-	 * Only attempt to queue if the whole lot can fit in the queue -
-	 * we can't successfully cleanup after a partial queing so this
-	 * must succeed or fail atomically.
-	 *
-	 * Big hammer test of both software and hardware queues - could be
-	 * more refined but this is unlikely to happen so no need.
-	 */
-
-	/* Grab a big lock for a long time to avoid concurrency issues */
-	spin_lock_bh(&queue->queuelock);
-
-	/*
-	 * Can go on to queue if we have space in either:
-	 * 1) The hardware queue and no software queue
-	 * 2) The software queue
-	 * AND there is nothing in the backlog.  If there is backlog we
-	 * have to only queue to the backlog queue and return busy.
-	 */
-	if ((!sec_queue_can_enqueue(queue, steps) &&
-	     (!queue->havesoftqueue ||
-	      kfifo_avail(&queue->softqueue) > steps)) ||
-	    !list_empty(&ctx->backlog)) {
-		ret = -EBUSY;
-		if ((skreq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
-			list_add_tail(&sec_req->backlog_head, &ctx->backlog);
-			spin_unlock_bh(&queue->queuelock);
-			goto out;
-		}
-
-		spin_unlock_bh(&queue->queuelock);
-		goto err_free_elements;
-	}
-	ret = sec_send_request(sec_req, queue);
-	spin_unlock_bh(&queue->queuelock);
-	if (ret)
-		goto err_free_elements;
-
-	ret = -EINPROGRESS;
-out:
-	/* Cleanup - all elements in pointer arrays have been copied */
-	kfree(splits_in_nents);
-	kfree(splits_in);
-	kfree(splits_out_nents);
-	kfree(splits_out);
-	kfree(split_sizes);
-	return ret;
-
-err_free_elements:
-	list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
-		list_del(&el->head);
-		sec_alg_free_el(el, info);
-	}
-	if (crypto_skcipher_ivsize(atfm))
-		dma_unmap_single(info->dev, sec_req->dma_iv,
-				 crypto_skcipher_ivsize(atfm),
-				 DMA_TO_DEVICE);
-err_unmap_out_sg:
-	if (split)
-		sec_unmap_sg_on_err(skreq->dst, steps, splits_out,
-				    splits_out_nents, sec_req->len_out,
-				    info->dev);
-err_unmap_in_sg:
-	sec_unmap_sg_on_err(skreq->src, steps, splits_in, splits_in_nents,
-			    sec_req->len_in, info->dev);
-err_free_split_sizes:
-	kfree(split_sizes);
-
-	return ret;
-}
-
-static int sec_alg_skcipher_encrypt(struct skcipher_request *req)
-{
-	return sec_alg_skcipher_crypto(req, true);
-}
-
-static int sec_alg_skcipher_decrypt(struct skcipher_request *req)
-{
-	return sec_alg_skcipher_crypto(req, false);
-}
-
-static int sec_alg_skcipher_init(struct crypto_skcipher *tfm)
-{
-	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-
-	mutex_init(&ctx->lock);
-	INIT_LIST_HEAD(&ctx->backlog);
-	crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_request));
-
-	ctx->queue = sec_queue_alloc_start_safe();
-	if (IS_ERR(ctx->queue))
-		return PTR_ERR(ctx->queue);
-
-	spin_lock_init(&ctx->queue->queuelock);
-	ctx->queue->havesoftqueue = false;
-
-	return 0;
-}
-
-static void sec_alg_skcipher_exit(struct crypto_skcipher *tfm)
-{
-	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct device *dev = ctx->queue->dev_info->dev;
-
-	if (ctx->key) {
-		memzero_explicit(ctx->key, SEC_MAX_CIPHER_KEY);
-		dma_free_coherent(dev, SEC_MAX_CIPHER_KEY, ctx->key,
-				  ctx->pkey);
-	}
-	sec_queue_stop_release(ctx->queue);
-}
-
-static int sec_alg_skcipher_init_with_queue(struct crypto_skcipher *tfm)
-{
-	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-	int ret;
-
-	ret = sec_alg_skcipher_init(tfm);
-	if (ret)
-		return ret;
-
-	INIT_KFIFO(ctx->queue->softqueue);
-	ret = kfifo_alloc(&ctx->queue->softqueue, 512, GFP_KERNEL);
-	if (ret) {
-		sec_alg_skcipher_exit(tfm);
-		return ret;
-	}
-	ctx->queue->havesoftqueue = true;
-
-	return 0;
-}
-
-static void sec_alg_skcipher_exit_with_queue(struct crypto_skcipher *tfm)
-{
-	struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
-
-	kfifo_free(&ctx->queue->softqueue);
-	sec_alg_skcipher_exit(tfm);
-}
-
-static struct skcipher_alg sec_algs[] = {
-	{
-		.base = {
-			.cra_name = "ecb(aes)",
-			.cra_driver_name = "hisi_sec_aes_ecb",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = AES_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init,
-		.exit = sec_alg_skcipher_exit,
-		.setkey = sec_alg_skcipher_setkey_aes_ecb,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = AES_MIN_KEY_SIZE,
-		.max_keysize = AES_MAX_KEY_SIZE,
-		.ivsize = 0,
-	}, {
-		.base = {
-			.cra_name = "cbc(aes)",
-			.cra_driver_name = "hisi_sec_aes_cbc",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = AES_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init_with_queue,
-		.exit = sec_alg_skcipher_exit_with_queue,
-		.setkey = sec_alg_skcipher_setkey_aes_cbc,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = AES_MIN_KEY_SIZE,
-		.max_keysize = AES_MAX_KEY_SIZE,
-		.ivsize = AES_BLOCK_SIZE,
-	}, {
-		.base = {
-			.cra_name = "ctr(aes)",
-			.cra_driver_name = "hisi_sec_aes_ctr",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = AES_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init_with_queue,
-		.exit = sec_alg_skcipher_exit_with_queue,
-		.setkey = sec_alg_skcipher_setkey_aes_ctr,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = AES_MIN_KEY_SIZE,
-		.max_keysize = AES_MAX_KEY_SIZE,
-		.ivsize = AES_BLOCK_SIZE,
-	}, {
-		.base = {
-			.cra_name = "xts(aes)",
-			.cra_driver_name = "hisi_sec_aes_xts",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = AES_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init,
-		.exit = sec_alg_skcipher_exit,
-		.setkey = sec_alg_skcipher_setkey_aes_xts,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = 2 * AES_MIN_KEY_SIZE,
-		.max_keysize = 2 * AES_MAX_KEY_SIZE,
-		.ivsize = AES_BLOCK_SIZE,
-	}, {
-	/* Unable to find any test vectors so untested */
-		.base = {
-			.cra_name = "ecb(des)",
-			.cra_driver_name = "hisi_sec_des_ecb",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = DES_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init,
-		.exit = sec_alg_skcipher_exit,
-		.setkey = sec_alg_skcipher_setkey_des_ecb,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = DES_KEY_SIZE,
-		.max_keysize = DES_KEY_SIZE,
-		.ivsize = 0,
-	}, {
-		.base = {
-			.cra_name = "cbc(des)",
-			.cra_driver_name = "hisi_sec_des_cbc",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = DES_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init_with_queue,
-		.exit = sec_alg_skcipher_exit_with_queue,
-		.setkey = sec_alg_skcipher_setkey_des_cbc,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = DES_KEY_SIZE,
-		.max_keysize = DES_KEY_SIZE,
-		.ivsize = DES_BLOCK_SIZE,
-	}, {
-		.base = {
-			.cra_name = "cbc(des3_ede)",
-			.cra_driver_name = "hisi_sec_3des_cbc",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init_with_queue,
-		.exit = sec_alg_skcipher_exit_with_queue,
-		.setkey = sec_alg_skcipher_setkey_3des_cbc,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = DES3_EDE_KEY_SIZE,
-		.max_keysize = DES3_EDE_KEY_SIZE,
-		.ivsize = DES3_EDE_BLOCK_SIZE,
-	}, {
-		.base = {
-			.cra_name = "ecb(des3_ede)",
-			.cra_driver_name = "hisi_sec_3des_ecb",
-			.cra_priority = 4001,
-			.cra_flags = CRYPTO_ALG_ASYNC |
-				     CRYPTO_ALG_ALLOCATES_MEMORY,
-			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
-			.cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
-			.cra_alignmask = 0,
-			.cra_module = THIS_MODULE,
-		},
-		.init = sec_alg_skcipher_init,
-		.exit = sec_alg_skcipher_exit,
-		.setkey = sec_alg_skcipher_setkey_3des_ecb,
-		.decrypt = sec_alg_skcipher_decrypt,
-		.encrypt = sec_alg_skcipher_encrypt,
-		.min_keysize = DES3_EDE_KEY_SIZE,
-		.max_keysize = DES3_EDE_KEY_SIZE,
-		.ivsize = 0,
-	}
-};
-
-int sec_algs_register(void)
-{
-	int ret = 0;
-
-	mutex_lock(&algs_lock);
-	if (++active_devs != 1)
-		goto unlock;
-
-	ret = crypto_register_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
-	if (ret)
-		--active_devs;
-unlock:
-	mutex_unlock(&algs_lock);
-
-	return ret;
-}
-
-void sec_algs_unregister(void)
-{
-	mutex_lock(&algs_lock);
-	if (--active_devs != 0)
-		goto unlock;
-	crypto_unregister_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
-
-unlock:
-	mutex_unlock(&algs_lock);
-}
diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c
deleted file mode 100644
index 2514a5e1f9b41f..00000000000000
--- a/drivers/crypto/hisilicon/sec/sec_drv.c
+++ /dev/null
@@ -1,1307 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Driver for the HiSilicon SEC units found on Hip06 Hip07
- *
- * Copyright (c) 2016-2017 HiSilicon Limited.
- */
-#include <linux/acpi.h>
-#include <linux/atomic.h>
-#include <linux/delay.h>
-#include <linux/dma-direction.h>
-#include <linux/dma-mapping.h>
-#include <linux/dmapool.h>
-#include <linux/io.h>
-#include <linux/iommu.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/irqreturn.h>
-#include <linux/mm.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-#include "sec_drv.h"
-
-#define SEC_QUEUE_AR_FROCE_ALLOC			0
-#define SEC_QUEUE_AR_FROCE_NOALLOC			1
-#define SEC_QUEUE_AR_FROCE_DIS				2
-
-#define SEC_QUEUE_AW_FROCE_ALLOC			0
-#define SEC_QUEUE_AW_FROCE_NOALLOC			1
-#define SEC_QUEUE_AW_FROCE_DIS				2
-
-/* SEC_ALGSUB registers */
-#define SEC_ALGSUB_CLK_EN_REG				0x03b8
-#define SEC_ALGSUB_CLK_DIS_REG				0x03bc
-#define SEC_ALGSUB_CLK_ST_REG				0x535c
-#define SEC_ALGSUB_RST_REQ_REG				0x0aa8
-#define SEC_ALGSUB_RST_DREQ_REG				0x0aac
-#define SEC_ALGSUB_RST_ST_REG				0x5a54
-#define   SEC_ALGSUB_RST_ST_IS_RST			BIT(0)
-
-#define SEC_ALGSUB_BUILD_RST_REQ_REG			0x0ab8
-#define SEC_ALGSUB_BUILD_RST_DREQ_REG			0x0abc
-#define SEC_ALGSUB_BUILD_RST_ST_REG			0x5a5c
-#define   SEC_ALGSUB_BUILD_RST_ST_IS_RST		BIT(0)
-
-#define SEC_SAA_BASE					0x00001000UL
-
-/* SEC_SAA registers */
-#define SEC_SAA_CTRL_REG(x)	((x) * SEC_SAA_ADDR_SIZE)
-#define   SEC_SAA_CTRL_GET_QM_EN			BIT(0)
-
-#define SEC_ST_INTMSK1_REG				0x0200
-#define SEC_ST_RINT1_REG				0x0400
-#define SEC_ST_INTSTS1_REG				0x0600
-#define SEC_BD_MNG_STAT_REG				0x0800
-#define SEC_PARSING_STAT_REG				0x0804
-#define SEC_LOAD_TIME_OUT_CNT_REG			0x0808
-#define SEC_CORE_WORK_TIME_OUT_CNT_REG			0x080c
-#define SEC_BACK_TIME_OUT_CNT_REG			0x0810
-#define SEC_BD1_PARSING_RD_TIME_OUT_CNT_REG		0x0814
-#define SEC_BD1_PARSING_WR_TIME_OUT_CNT_REG		0x0818
-#define SEC_BD2_PARSING_RD_TIME_OUT_CNT_REG		0x081c
-#define SEC_BD2_PARSING_WR_TIME_OUT_CNT_REG		0x0820
-#define SEC_SAA_ACC_REG					0x083c
-#define SEC_BD_NUM_CNT_IN_SEC_REG			0x0858
-#define SEC_LOAD_WORK_TIME_CNT_REG			0x0860
-#define SEC_CORE_WORK_WORK_TIME_CNT_REG			0x0864
-#define SEC_BACK_WORK_TIME_CNT_REG			0x0868
-#define SEC_SAA_IDLE_TIME_CNT_REG			0x086c
-#define SEC_SAA_CLK_CNT_REG				0x0870
-
-/* SEC_COMMON registers */
-#define SEC_CLK_EN_REG					0x0000
-#define SEC_CTRL_REG					0x0004
-
-#define SEC_COMMON_CNT_CLR_CE_REG			0x0008
-#define   SEC_COMMON_CNT_CLR_CE_CLEAR			BIT(0)
-#define   SEC_COMMON_CNT_CLR_CE_SNAP_EN			BIT(1)
-
-#define SEC_SECURE_CTRL_REG				0x000c
-#define SEC_AXI_CACHE_CFG_REG				0x0010
-#define SEC_AXI_QOS_CFG_REG				0x0014
-#define SEC_IPV4_MASK_TABLE_REG				0x0020
-#define SEC_IPV6_MASK_TABLE_X_REG(x)	(0x0024 + (x) * 4)
-#define SEC_FSM_MAX_CNT_REG				0x0064
-
-#define SEC_CTRL2_REG					0x0068
-#define   SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M		GENMASK(3, 0)
-#define   SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S		0
-#define   SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M		GENMASK(6, 4)
-#define   SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S		4
-#define   SEC_CTRL2_CLK_GATE_EN				BIT(7)
-#define   SEC_CTRL2_ENDIAN_BD				BIT(8)
-#define   SEC_CTRL2_ENDIAN_BD_TYPE			BIT(9)
-
-#define SEC_CNT_PRECISION_CFG_REG			0x006c
-#define SEC_DEBUG_BD_CFG_REG				0x0070
-#define   SEC_DEBUG_BD_CFG_WB_NORMAL			BIT(0)
-#define   SEC_DEBUG_BD_CFG_WB_EN			BIT(1)
-
-#define SEC_Q_SIGHT_SEL					0x0074
-#define SEC_Q_SIGHT_HIS_CLR				0x0078
-#define SEC_Q_VMID_CFG_REG(q)		(0x0100 + (q) * 4)
-#define SEC_Q_WEIGHT_CFG_REG(q)		(0x200 + (q) * 4)
-#define SEC_STAT_CLR_REG				0x0a00
-#define SEC_SAA_IDLE_CNT_CLR_REG			0x0a04
-#define SEC_QM_CPL_Q_IDBUF_DFX_CFG_REG			0x0b00
-#define SEC_QM_CPL_Q_IDBUF_DFX_RESULT_REG		0x0b04
-#define SEC_QM_BD_DFX_CFG_REG				0x0b08
-#define SEC_QM_BD_DFX_RESULT_REG			0x0b0c
-#define SEC_QM_BDID_DFX_RESULT_REG			0x0b10
-#define SEC_QM_BD_DFIFO_STATUS_REG			0x0b14
-#define SEC_QM_BD_DFX_CFG2_REG				0x0b1c
-#define SEC_QM_BD_DFX_RESULT2_REG			0x0b20
-#define SEC_QM_BD_IDFIFO_STATUS_REG			0x0b18
-#define SEC_QM_BD_DFIFO_STATUS2_REG			0x0b28
-#define SEC_QM_BD_IDFIFO_STATUS2_REG			0x0b2c
-
-#define SEC_HASH_IPV4_MASK				0xfff00000
-#define SEC_MAX_SAA_NUM					0xa
-#define SEC_SAA_ADDR_SIZE				0x1000
-
-#define SEC_Q_INIT_REG					0x0
-#define   SEC_Q_INIT_WO_STAT_CLEAR			0x2
-#define   SEC_Q_INIT_AND_STAT_CLEAR			0x3
-
-#define SEC_Q_CFG_REG					0x8
-#define   SEC_Q_CFG_REORDER				BIT(0)
-
-#define SEC_Q_PROC_NUM_CFG_REG				0x10
-#define SEC_QUEUE_ENB_REG				0x18
-
-#define SEC_Q_DEPTH_CFG_REG				0x50
-#define   SEC_Q_DEPTH_CFG_DEPTH_M			GENMASK(11, 0)
-#define   SEC_Q_DEPTH_CFG_DEPTH_S			0
-
-#define SEC_Q_BASE_HADDR_REG				0x54
-#define SEC_Q_BASE_LADDR_REG				0x58
-#define SEC_Q_WR_PTR_REG				0x5c
-#define SEC_Q_OUTORDER_BASE_HADDR_REG			0x60
-#define SEC_Q_OUTORDER_BASE_LADDR_REG			0x64
-#define SEC_Q_OUTORDER_RD_PTR_REG			0x68
-#define SEC_Q_OT_TH_REG					0x6c
-
-#define SEC_Q_ARUSER_CFG_REG				0x70
-#define   SEC_Q_ARUSER_CFG_FA				BIT(0)
-#define   SEC_Q_ARUSER_CFG_FNA				BIT(1)
-#define   SEC_Q_ARUSER_CFG_RINVLD			BIT(2)
-#define   SEC_Q_ARUSER_CFG_PKG				BIT(3)
-
-#define SEC_Q_AWUSER_CFG_REG				0x74
-#define   SEC_Q_AWUSER_CFG_FA				BIT(0)
-#define   SEC_Q_AWUSER_CFG_FNA				BIT(1)
-#define   SEC_Q_AWUSER_CFG_PKG				BIT(2)
-
-#define SEC_Q_ERR_BASE_HADDR_REG			0x7c
-#define SEC_Q_ERR_BASE_LADDR_REG			0x80
-#define SEC_Q_CFG_VF_NUM_REG				0x84
-#define SEC_Q_SOFT_PROC_PTR_REG				0x88
-#define SEC_Q_FAIL_INT_MSK_REG				0x300
-#define SEC_Q_FLOW_INT_MKS_REG				0x304
-#define SEC_Q_FAIL_RINT_REG				0x400
-#define SEC_Q_FLOW_RINT_REG				0x404
-#define SEC_Q_FAIL_INT_STATUS_REG			0x500
-#define SEC_Q_FLOW_INT_STATUS_REG			0x504
-#define SEC_Q_STATUS_REG				0x600
-#define SEC_Q_RD_PTR_REG				0x604
-#define SEC_Q_PRO_PTR_REG				0x608
-#define SEC_Q_OUTORDER_WR_PTR_REG			0x60c
-#define SEC_Q_OT_CNT_STATUS_REG				0x610
-#define SEC_Q_INORDER_BD_NUM_ST_REG			0x650
-#define SEC_Q_INORDER_GET_FLAG_ST_REG			0x654
-#define SEC_Q_INORDER_ADD_FLAG_ST_REG			0x658
-#define SEC_Q_INORDER_TASK_INT_NUM_LEFT_ST_REG		0x65c
-#define SEC_Q_RD_DONE_PTR_REG				0x660
-#define SEC_Q_CPL_Q_BD_NUM_ST_REG			0x700
-#define SEC_Q_CPL_Q_PTR_ST_REG				0x704
-#define SEC_Q_CPL_Q_H_ADDR_ST_REG			0x708
-#define SEC_Q_CPL_Q_L_ADDR_ST_REG			0x70c
-#define SEC_Q_CPL_TASK_INT_NUM_LEFT_ST_REG		0x710
-#define SEC_Q_WRR_ID_CHECK_REG				0x714
-#define SEC_Q_CPLQ_FULL_CHECK_REG			0x718
-#define SEC_Q_SUCCESS_BD_CNT_REG			0x800
-#define SEC_Q_FAIL_BD_CNT_REG				0x804
-#define SEC_Q_GET_BD_CNT_REG				0x808
-#define SEC_Q_IVLD_CNT_REG				0x80c
-#define SEC_Q_BD_PROC_GET_CNT_REG			0x810
-#define SEC_Q_BD_PROC_DONE_CNT_REG			0x814
-#define SEC_Q_LAT_CLR_REG				0x850
-#define SEC_Q_PKT_LAT_MAX_REG				0x854
-#define SEC_Q_PKT_LAT_AVG_REG				0x858
-#define SEC_Q_PKT_LAT_MIN_REG				0x85c
-#define SEC_Q_ID_CLR_CFG_REG				0x900
-#define SEC_Q_1ST_BD_ERR_ID_REG				0x904
-#define SEC_Q_1ST_AUTH_FAIL_ID_REG			0x908
-#define SEC_Q_1ST_RD_ERR_ID_REG				0x90c
-#define SEC_Q_1ST_ECC2_ERR_ID_REG			0x910
-#define SEC_Q_1ST_IVLD_ID_REG				0x914
-#define SEC_Q_1ST_BD_WR_ERR_ID_REG			0x918
-#define SEC_Q_1ST_ERR_BD_WR_ERR_ID_REG			0x91c
-#define SEC_Q_1ST_BD_MAC_WR_ERR_ID_REG			0x920
-
-struct sec_debug_bd_info {
-#define SEC_DEBUG_BD_INFO_SOFT_ERR_CHECK_M	GENMASK(22, 0)
-	u32 soft_err_check;
-#define SEC_DEBUG_BD_INFO_HARD_ERR_CHECK_M	GENMASK(9, 0)
-	u32 hard_err_check;
-	u32 icv_mac1st_word;
-#define SEC_DEBUG_BD_INFO_GET_ID_M		GENMASK(19, 0)
-	u32 sec_get_id;
-	/* W4---W15 */
-	u32 reserv_left[12];
-};
-
-struct sec_out_bd_info	{
-#define SEC_OUT_BD_INFO_Q_ID_M			GENMASK(11, 0)
-#define SEC_OUT_BD_INFO_ECC_2BIT_ERR		BIT(14)
-	u16 data;
-};
-
-#define SEC_MAX_DEVICES				8
-static struct sec_dev_info *sec_devices[SEC_MAX_DEVICES];
-static DEFINE_MUTEX(sec_id_lock);
-
-static int sec_queue_map_io(struct sec_queue *queue)
-{
-	struct device *dev = queue->dev_info->dev;
-	struct resource *res;
-
-	res = platform_get_resource(to_platform_device(dev),
-				    IORESOURCE_MEM,
-				    2 + queue->queue_id);
-	if (!res) {
-		dev_err(dev, "Failed to get queue %u memory resource\n",
-			queue->queue_id);
-		return -ENOMEM;
-	}
-	queue->regs = ioremap(res->start, resource_size(res));
-	if (!queue->regs)
-		return -ENOMEM;
-
-	return 0;
-}
-
-static void sec_queue_unmap_io(struct sec_queue *queue)
-{
-	 iounmap(queue->regs);
-}
-
-static int sec_queue_ar_pkgattr(struct sec_queue *queue, u32 ar_pkg)
-{
-	void __iomem *addr = queue->regs +  SEC_Q_ARUSER_CFG_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	if (ar_pkg)
-		regval |= SEC_Q_ARUSER_CFG_PKG;
-	else
-		regval &= ~SEC_Q_ARUSER_CFG_PKG;
-	writel_relaxed(regval, addr);
-
-	return 0;
-}
-
-static int sec_queue_aw_pkgattr(struct sec_queue *queue, u32 aw_pkg)
-{
-	void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	regval |= SEC_Q_AWUSER_CFG_PKG;
-	writel_relaxed(regval, addr);
-
-	return 0;
-}
-
-static int sec_clk_en(struct sec_dev_info *info)
-{
-	void __iomem *base = info->regs[SEC_COMMON];
-	u32 i = 0;
-
-	writel_relaxed(0x7, base + SEC_ALGSUB_CLK_EN_REG);
-	do {
-		usleep_range(1000, 10000);
-		if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0x7)
-			return 0;
-		i++;
-	} while (i < 10);
-	dev_err(info->dev, "sec clock enable fail!\n");
-
-	return -EIO;
-}
-
-static int sec_clk_dis(struct sec_dev_info *info)
-{
-	void __iomem *base = info->regs[SEC_COMMON];
-	u32 i = 0;
-
-	writel_relaxed(0x7, base + SEC_ALGSUB_CLK_DIS_REG);
-	do {
-		usleep_range(1000, 10000);
-		if ((readl_relaxed(base + SEC_ALGSUB_CLK_ST_REG) & 0x7) == 0)
-			return 0;
-		i++;
-	} while (i < 10);
-	dev_err(info->dev, "sec clock disable fail!\n");
-
-	return -EIO;
-}
-
-static int sec_reset_whole_module(struct sec_dev_info *info)
-{
-	void __iomem *base = info->regs[SEC_COMMON];
-	bool is_reset, b_is_reset;
-	u32 i = 0;
-
-	writel_relaxed(1, base + SEC_ALGSUB_RST_REQ_REG);
-	writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_REQ_REG);
-	while (1) {
-		usleep_range(1000, 10000);
-		is_reset = readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) &
-			SEC_ALGSUB_RST_ST_IS_RST;
-		b_is_reset = readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) &
-			SEC_ALGSUB_BUILD_RST_ST_IS_RST;
-		if (is_reset && b_is_reset)
-			break;
-		i++;
-		if (i > 10) {
-			dev_err(info->dev, "Reset req failed\n");
-			return -EIO;
-		}
-	}
-
-	i = 0;
-	writel_relaxed(1, base + SEC_ALGSUB_RST_DREQ_REG);
-	writel_relaxed(1, base + SEC_ALGSUB_BUILD_RST_DREQ_REG);
-	while (1) {
-		usleep_range(1000, 10000);
-		is_reset = readl_relaxed(base + SEC_ALGSUB_RST_ST_REG) &
-			SEC_ALGSUB_RST_ST_IS_RST;
-		b_is_reset = readl_relaxed(base + SEC_ALGSUB_BUILD_RST_ST_REG) &
-			SEC_ALGSUB_BUILD_RST_ST_IS_RST;
-		if (!is_reset && !b_is_reset)
-			break;
-
-		i++;
-		if (i > 10) {
-			dev_err(info->dev, "Reset dreq failed\n");
-			return -EIO;
-		}
-	}
-
-	return 0;
-}
-
-static void sec_bd_endian_little(struct sec_dev_info *info)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	regval &= ~(SEC_CTRL2_ENDIAN_BD | SEC_CTRL2_ENDIAN_BD_TYPE);
-	writel_relaxed(regval, addr);
-}
-
-/*
- * sec_cache_config - configure optimum cache placement
- */
-static void sec_cache_config(struct sec_dev_info *info)
-{
-	struct iommu_domain *domain;
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL_REG;
-
-	domain = iommu_get_domain_for_dev(info->dev);
-
-	/* Check that translation is occurring */
-	if (domain && (domain->type & __IOMMU_DOMAIN_PAGING))
-		writel_relaxed(0x44cf9e, addr);
-	else
-		writel_relaxed(0x4cfd9, addr);
-}
-
-static void sec_data_axiwr_otsd_cfg(struct sec_dev_info *info, u32 cfg)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	regval &= ~SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M;
-	regval |= (cfg << SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_S) &
-		SEC_CTRL2_DATA_AXI_WR_OTSD_CFG_M;
-	writel_relaxed(regval, addr);
-}
-
-static void sec_data_axird_otsd_cfg(struct sec_dev_info *info, u32 cfg)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	regval &= ~SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M;
-	regval |= (cfg << SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_S) &
-		SEC_CTRL2_DATA_AXI_RD_OTSD_CFG_M;
-	writel_relaxed(regval, addr);
-}
-
-static void sec_clk_gate_en(struct sec_dev_info *info, bool clkgate)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_CTRL2_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	if (clkgate)
-		regval |= SEC_CTRL2_CLK_GATE_EN;
-	else
-		regval &= ~SEC_CTRL2_CLK_GATE_EN;
-	writel_relaxed(regval, addr);
-}
-
-static void sec_comm_cnt_cfg(struct sec_dev_info *info, bool clr_ce)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_COMMON_CNT_CLR_CE_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	if (clr_ce)
-		regval |= SEC_COMMON_CNT_CLR_CE_CLEAR;
-	else
-		regval &= ~SEC_COMMON_CNT_CLR_CE_CLEAR;
-	writel_relaxed(regval, addr);
-}
-
-static void sec_commsnap_en(struct sec_dev_info *info, bool snap_en)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_COMMON_CNT_CLR_CE_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	if (snap_en)
-		regval |= SEC_COMMON_CNT_CLR_CE_SNAP_EN;
-	else
-		regval &= ~SEC_COMMON_CNT_CLR_CE_SNAP_EN;
-	writel_relaxed(regval, addr);
-}
-
-static void sec_ipv6_hashmask(struct sec_dev_info *info, u32 hash_mask[])
-{
-	void __iomem *base = info->regs[SEC_SAA];
-	int i;
-
-	for (i = 0; i < 10; i++)
-		writel_relaxed(hash_mask[0],
-			       base + SEC_IPV6_MASK_TABLE_X_REG(i));
-}
-
-static int sec_ipv4_hashmask(struct sec_dev_info *info, u32 hash_mask)
-{
-	if (hash_mask & SEC_HASH_IPV4_MASK) {
-		dev_err(info->dev, "Sec Ipv4 Hash Mask Input Error!\n");
-		return -EINVAL;
-	}
-
-	writel_relaxed(hash_mask,
-		       info->regs[SEC_SAA] + SEC_IPV4_MASK_TABLE_REG);
-
-	return 0;
-}
-
-static void sec_set_dbg_bd_cfg(struct sec_dev_info *info, u32 cfg)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_DEBUG_BD_CFG_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	/* Always disable write back of normal bd */
-	regval &= ~SEC_DEBUG_BD_CFG_WB_NORMAL;
-
-	if (cfg)
-		regval &= ~SEC_DEBUG_BD_CFG_WB_EN;
-	else
-		regval |= SEC_DEBUG_BD_CFG_WB_EN;
-
-	writel_relaxed(regval, addr);
-}
-
-static void sec_saa_getqm_en(struct sec_dev_info *info, u32 saa_indx, u32 en)
-{
-	void __iomem *addr = info->regs[SEC_SAA] + SEC_SAA_BASE +
-		SEC_SAA_CTRL_REG(saa_indx);
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	if (en)
-		regval |= SEC_SAA_CTRL_GET_QM_EN;
-	else
-		regval &= ~SEC_SAA_CTRL_GET_QM_EN;
-	writel_relaxed(regval, addr);
-}
-
-static void sec_saa_int_mask(struct sec_dev_info *info, u32 saa_indx,
-			     u32 saa_int_mask)
-{
-	writel_relaxed(saa_int_mask,
-		       info->regs[SEC_SAA] + SEC_SAA_BASE + SEC_ST_INTMSK1_REG +
-		       saa_indx * SEC_SAA_ADDR_SIZE);
-}
-
-static void sec_streamid(struct sec_dev_info *info, int i)
-{
-	#define SEC_SID 0x600
-	#define SEC_VMID 0
-
-	writel_relaxed((SEC_VMID | ((SEC_SID & 0xffff) << 8)),
-		       info->regs[SEC_SAA] + SEC_Q_VMID_CFG_REG(i));
-}
-
-static void sec_queue_ar_alloc(struct sec_queue *queue, u32 alloc)
-{
-	void __iomem *addr = queue->regs + SEC_Q_ARUSER_CFG_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	if (alloc == SEC_QUEUE_AR_FROCE_ALLOC) {
-		regval |= SEC_Q_ARUSER_CFG_FA;
-		regval &= ~SEC_Q_ARUSER_CFG_FNA;
-	} else {
-		regval &= ~SEC_Q_ARUSER_CFG_FA;
-		regval |= SEC_Q_ARUSER_CFG_FNA;
-	}
-
-	writel_relaxed(regval, addr);
-}
-
-static void sec_queue_aw_alloc(struct sec_queue *queue, u32 alloc)
-{
-	void __iomem *addr = queue->regs + SEC_Q_AWUSER_CFG_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	if (alloc == SEC_QUEUE_AW_FROCE_ALLOC) {
-		regval |= SEC_Q_AWUSER_CFG_FA;
-		regval &= ~SEC_Q_AWUSER_CFG_FNA;
-	} else {
-		regval &= ~SEC_Q_AWUSER_CFG_FA;
-		regval |= SEC_Q_AWUSER_CFG_FNA;
-	}
-
-	writel_relaxed(regval, addr);
-}
-
-static void sec_queue_reorder(struct sec_queue *queue, bool reorder)
-{
-	void __iomem *base = queue->regs;
-	u32 regval;
-
-	regval = readl_relaxed(base + SEC_Q_CFG_REG);
-	if (reorder)
-		regval |= SEC_Q_CFG_REORDER;
-	else
-		regval &= ~SEC_Q_CFG_REORDER;
-	writel_relaxed(regval, base + SEC_Q_CFG_REG);
-}
-
-static void sec_queue_depth(struct sec_queue *queue, u32 depth)
-{
-	void __iomem *addr = queue->regs + SEC_Q_DEPTH_CFG_REG;
-	u32 regval;
-
-	regval = readl_relaxed(addr);
-	regval &= ~SEC_Q_DEPTH_CFG_DEPTH_M;
-	regval |= (depth << SEC_Q_DEPTH_CFG_DEPTH_S) & SEC_Q_DEPTH_CFG_DEPTH_M;
-
-	writel_relaxed(regval, addr);
-}
-
-static void sec_queue_cmdbase_addr(struct sec_queue *queue, u64 addr)
-{
-	writel_relaxed(upper_32_bits(addr), queue->regs + SEC_Q_BASE_HADDR_REG);
-	writel_relaxed(lower_32_bits(addr), queue->regs + SEC_Q_BASE_LADDR_REG);
-}
-
-static void sec_queue_outorder_addr(struct sec_queue *queue, u64 addr)
-{
-	writel_relaxed(upper_32_bits(addr),
-		       queue->regs + SEC_Q_OUTORDER_BASE_HADDR_REG);
-	writel_relaxed(lower_32_bits(addr),
-		       queue->regs + SEC_Q_OUTORDER_BASE_LADDR_REG);
-}
-
-static void sec_queue_errbase_addr(struct sec_queue *queue, u64 addr)
-{
-	writel_relaxed(upper_32_bits(addr),
-		       queue->regs + SEC_Q_ERR_BASE_HADDR_REG);
-	writel_relaxed(lower_32_bits(addr),
-		       queue->regs + SEC_Q_ERR_BASE_LADDR_REG);
-}
-
-static void sec_queue_irq_disable(struct sec_queue *queue)
-{
-	writel_relaxed((u32)~0, queue->regs + SEC_Q_FLOW_INT_MKS_REG);
-}
-
-static void sec_queue_irq_enable(struct sec_queue *queue)
-{
-	writel_relaxed(0, queue->regs + SEC_Q_FLOW_INT_MKS_REG);
-}
-
-static void sec_queue_abn_irq_disable(struct sec_queue *queue)
-{
-	writel_relaxed((u32)~0, queue->regs + SEC_Q_FAIL_INT_MSK_REG);
-}
-
-static void sec_queue_stop(struct sec_queue *queue)
-{
-	disable_irq(queue->task_irq);
-	sec_queue_irq_disable(queue);
-	writel_relaxed(0x0, queue->regs + SEC_QUEUE_ENB_REG);
-}
-
-static void sec_queue_start(struct sec_queue *queue)
-{
-	sec_queue_irq_enable(queue);
-	enable_irq(queue->task_irq);
-	queue->expected = 0;
-	writel_relaxed(SEC_Q_INIT_AND_STAT_CLEAR, queue->regs + SEC_Q_INIT_REG);
-	writel_relaxed(0x1, queue->regs + SEC_QUEUE_ENB_REG);
-}
-
-static struct sec_queue *sec_alloc_queue(struct sec_dev_info *info)
-{
-	int i;
-
-	mutex_lock(&info->dev_lock);
-
-	/* Get the first idle queue in SEC device */
-	for (i = 0; i < SEC_Q_NUM; i++)
-		if (!info->queues[i].in_use) {
-			info->queues[i].in_use = true;
-			info->queues_in_use++;
-			mutex_unlock(&info->dev_lock);
-
-			return &info->queues[i];
-		}
-	mutex_unlock(&info->dev_lock);
-
-	return ERR_PTR(-ENODEV);
-}
-
-static int sec_queue_free(struct sec_queue *queue)
-{
-	struct sec_dev_info *info = queue->dev_info;
-
-	if (queue->queue_id >= SEC_Q_NUM) {
-		dev_err(info->dev, "No queue %u\n", queue->queue_id);
-		return -ENODEV;
-	}
-
-	if (!queue->in_use) {
-		dev_err(info->dev, "Queue %u is idle\n", queue->queue_id);
-		return -ENODEV;
-	}
-
-	mutex_lock(&info->dev_lock);
-	queue->in_use = false;
-	info->queues_in_use--;
-	mutex_unlock(&info->dev_lock);
-
-	return 0;
-}
-
-static irqreturn_t sec_isr_handle_th(int irq, void *q)
-{
-	sec_queue_irq_disable(q);
-	return IRQ_WAKE_THREAD;
-}
-
-static irqreturn_t sec_isr_handle(int irq, void *q)
-{
-	struct sec_queue *queue = q;
-	struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
-	struct sec_queue_ring_cq *cq_ring = &queue->ring_cq;
-	struct sec_out_bd_info *outorder_msg;
-	struct sec_bd_info *msg;
-	u32 ooo_read, ooo_write;
-	void __iomem *base = queue->regs;
-	int q_id;
-
-	ooo_read = readl(base + SEC_Q_OUTORDER_RD_PTR_REG);
-	ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG);
-	outorder_msg = cq_ring->vaddr + ooo_read;
-	q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
-	msg = msg_ring->vaddr + q_id;
-
-	while ((ooo_write != ooo_read) && msg->w0 & SEC_BD_W0_DONE) {
-		/*
-		 * Must be before callback otherwise blocks adding other chained
-		 * elements
-		 */
-		set_bit(q_id, queue->unprocessed);
-		if (q_id == queue->expected)
-			while (test_bit(queue->expected, queue->unprocessed)) {
-				clear_bit(queue->expected, queue->unprocessed);
-				msg = msg_ring->vaddr + queue->expected;
-				msg->w0 &= ~SEC_BD_W0_DONE;
-				msg_ring->callback(msg,
-						queue->shadow[queue->expected]);
-				queue->shadow[queue->expected] = NULL;
-				queue->expected = (queue->expected + 1) %
-					SEC_QUEUE_LEN;
-				atomic_dec(&msg_ring->used);
-			}
-
-		ooo_read = (ooo_read + 1) % SEC_QUEUE_LEN;
-		writel(ooo_read, base + SEC_Q_OUTORDER_RD_PTR_REG);
-		ooo_write = readl(base + SEC_Q_OUTORDER_WR_PTR_REG);
-		outorder_msg = cq_ring->vaddr + ooo_read;
-		q_id = outorder_msg->data & SEC_OUT_BD_INFO_Q_ID_M;
-		msg = msg_ring->vaddr + q_id;
-	}
-
-	sec_queue_irq_enable(queue);
-
-	return IRQ_HANDLED;
-}
-
-static int sec_queue_irq_init(struct sec_queue *queue)
-{
-	struct sec_dev_info *info = queue->dev_info;
-	int irq = queue->task_irq;
-	int ret;
-
-	ret = request_threaded_irq(irq, sec_isr_handle_th, sec_isr_handle,
-				   IRQF_TRIGGER_RISING, queue->name, queue);
-	if (ret) {
-		dev_err(info->dev, "request irq(%d) failed %d\n", irq, ret);
-		return ret;
-	}
-	disable_irq(irq);
-
-	return 0;
-}
-
-static int sec_queue_irq_uninit(struct sec_queue *queue)
-{
-	free_irq(queue->task_irq, queue);
-
-	return 0;
-}
-
-static struct sec_dev_info *sec_device_get(void)
-{
-	struct sec_dev_info *sec_dev = NULL;
-	struct sec_dev_info *this_sec_dev;
-	int least_busy_n = SEC_Q_NUM + 1;
-	int i;
-
-	/* Find which one is least busy and use that first */
-	for (i = 0; i < SEC_MAX_DEVICES; i++) {
-		this_sec_dev = sec_devices[i];
-		if (this_sec_dev &&
-		    this_sec_dev->queues_in_use < least_busy_n) {
-			least_busy_n = this_sec_dev->queues_in_use;
-			sec_dev = this_sec_dev;
-		}
-	}
-
-	return sec_dev;
-}
-
-static struct sec_queue *sec_queue_alloc_start(struct sec_dev_info *info)
-{
-	struct sec_queue *queue;
-
-	queue = sec_alloc_queue(info);
-	if (IS_ERR(queue)) {
-		dev_err(info->dev, "alloc sec queue failed! %ld\n",
-			PTR_ERR(queue));
-		return queue;
-	}
-
-	sec_queue_start(queue);
-
-	return queue;
-}
-
-/**
- * sec_queue_alloc_start_safe - get a hw queue from appropriate instance
- *
- * This function does extremely simplistic load balancing. It does not take into
- * account NUMA locality of the accelerator, or which cpu has requested the
- * queue.  Future work may focus on optimizing this in order to improve full
- * machine throughput.
- */
-struct sec_queue *sec_queue_alloc_start_safe(void)
-{
-	struct sec_dev_info *info;
-	struct sec_queue *queue = ERR_PTR(-ENODEV);
-
-	mutex_lock(&sec_id_lock);
-	info = sec_device_get();
-	if (!info)
-		goto unlock;
-
-	queue = sec_queue_alloc_start(info);
-
-unlock:
-	mutex_unlock(&sec_id_lock);
-
-	return queue;
-}
-
-/**
- * sec_queue_stop_release() - free up a hw queue for reuse
- * @queue: The queue we are done with.
- *
- * This will stop the current queue, terminanting any transactions
- * that are inflight an return it to the pool of available hw queuess
- */
-int sec_queue_stop_release(struct sec_queue *queue)
-{
-	struct device *dev = queue->dev_info->dev;
-	int ret;
-
-	sec_queue_stop(queue);
-
-	ret = sec_queue_free(queue);
-	if (ret)
-		dev_err(dev, "Releasing queue failed %d\n", ret);
-
-	return ret;
-}
-
-/**
- * sec_queue_empty() - Is this hardware queue currently empty.
- * @queue: The queue to test
- *
- * We need to know if we have an empty queue for some of the chaining modes
- * as if it is not empty we may need to hold the message in a software queue
- * until the hw queue is drained.
- */
-bool sec_queue_empty(struct sec_queue *queue)
-{
-	struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
-
-	return !atomic_read(&msg_ring->used);
-}
-
-/**
- * sec_queue_send() - queue up a single operation in the hw queue
- * @queue: The queue in which to put the message
- * @msg: The message
- * @ctx: Context to be put in the shadow array and passed back to cb on result.
- *
- * This function will return -EAGAIN if the queue is currently full.
- */
-int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx)
-{
-	struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
-	void __iomem *base = queue->regs;
-	u32 write, read;
-
-	mutex_lock(&msg_ring->lock);
-	read = readl(base + SEC_Q_RD_PTR_REG);
-	write = readl(base + SEC_Q_WR_PTR_REG);
-	if (write == read && atomic_read(&msg_ring->used) == SEC_QUEUE_LEN) {
-		mutex_unlock(&msg_ring->lock);
-		return -EAGAIN;
-	}
-	memcpy(msg_ring->vaddr + write, msg, sizeof(*msg));
-	queue->shadow[write] = ctx;
-	write = (write + 1) % SEC_QUEUE_LEN;
-
-	/* Ensure content updated before queue advance */
-	wmb();
-	writel(write, base + SEC_Q_WR_PTR_REG);
-
-	atomic_inc(&msg_ring->used);
-	mutex_unlock(&msg_ring->lock);
-
-	return 0;
-}
-
-bool sec_queue_can_enqueue(struct sec_queue *queue, int num)
-{
-	struct sec_queue_ring_cmd *msg_ring = &queue->ring_cmd;
-
-	return SEC_QUEUE_LEN - atomic_read(&msg_ring->used) >= num;
-}
-
-static void sec_queue_hw_init(struct sec_queue *queue)
-{
-	sec_queue_ar_alloc(queue, SEC_QUEUE_AR_FROCE_NOALLOC);
-	sec_queue_aw_alloc(queue, SEC_QUEUE_AW_FROCE_NOALLOC);
-	sec_queue_ar_pkgattr(queue, 1);
-	sec_queue_aw_pkgattr(queue, 1);
-
-	/* Enable out of order queue */
-	sec_queue_reorder(queue, true);
-
-	/* Interrupt after a single complete element */
-	writel_relaxed(1, queue->regs + SEC_Q_PROC_NUM_CFG_REG);
-
-	sec_queue_depth(queue, SEC_QUEUE_LEN - 1);
-
-	sec_queue_cmdbase_addr(queue, queue->ring_cmd.paddr);
-
-	sec_queue_outorder_addr(queue, queue->ring_cq.paddr);
-
-	sec_queue_errbase_addr(queue, queue->ring_db.paddr);
-
-	writel_relaxed(0x100, queue->regs + SEC_Q_OT_TH_REG);
-
-	sec_queue_abn_irq_disable(queue);
-	sec_queue_irq_disable(queue);
-	writel_relaxed(SEC_Q_INIT_AND_STAT_CLEAR, queue->regs + SEC_Q_INIT_REG);
-}
-
-static int sec_hw_init(struct sec_dev_info *info)
-{
-	struct iommu_domain *domain;
-	u32 sec_ipv4_mask = 0;
-	u32 sec_ipv6_mask[10] = {};
-	int ret;
-	u32 i;
-
-	domain = iommu_get_domain_for_dev(info->dev);
-
-	/*
-	 * Enable all available processing unit clocks.
-	 * Only the first cluster is usable with translations.
-	 */
-	if (domain && (domain->type & __IOMMU_DOMAIN_PAGING))
-		info->num_saas = 5;
-
-	else
-		info->num_saas = 10;
-
-	writel_relaxed(GENMASK(info->num_saas - 1, 0),
-		       info->regs[SEC_SAA] + SEC_CLK_EN_REG);
-
-	/* 32 bit little endian */
-	sec_bd_endian_little(info);
-
-	sec_cache_config(info);
-
-	/* Data axi port write and read outstanding config as per datasheet */
-	sec_data_axiwr_otsd_cfg(info, 0x7);
-	sec_data_axird_otsd_cfg(info, 0x7);
-
-	/* Enable clock gating */
-	sec_clk_gate_en(info, true);
-
-	/* Set CNT_CYC register not read clear */
-	sec_comm_cnt_cfg(info, false);
-
-	/* Enable CNT_CYC */
-	sec_commsnap_en(info, false);
-
-	writel_relaxed((u32)~0, info->regs[SEC_SAA] + SEC_FSM_MAX_CNT_REG);
-
-	ret = sec_ipv4_hashmask(info, sec_ipv4_mask);
-	if (ret) {
-		dev_err(info->dev, "Failed to set ipv4 hashmask %d\n", ret);
-		return -EIO;
-	}
-
-	sec_ipv6_hashmask(info, sec_ipv6_mask);
-
-	/*  do not use debug bd */
-	sec_set_dbg_bd_cfg(info, 0);
-
-	if (domain && (domain->type & __IOMMU_DOMAIN_PAGING)) {
-		for (i = 0; i < SEC_Q_NUM; i++) {
-			sec_streamid(info, i);
-			/* Same QoS for all queues */
-			writel_relaxed(0x3f,
-				       info->regs[SEC_SAA] +
-				       SEC_Q_WEIGHT_CFG_REG(i));
-		}
-	}
-
-	for (i = 0; i < info->num_saas; i++) {
-		sec_saa_getqm_en(info, i, 1);
-		sec_saa_int_mask(info, i, 0);
-	}
-
-	return 0;
-}
-
-static void sec_hw_exit(struct sec_dev_info *info)
-{
-	int i;
-
-	for (i = 0; i < SEC_MAX_SAA_NUM; i++) {
-		sec_saa_int_mask(info, i, (u32)~0);
-		sec_saa_getqm_en(info, i, 0);
-	}
-}
-
-static void sec_queue_base_init(struct sec_dev_info *info,
-				struct sec_queue *queue, int queue_id)
-{
-	queue->dev_info = info;
-	queue->queue_id = queue_id;
-	snprintf(queue->name, sizeof(queue->name),
-		 "%s_%d", dev_name(info->dev), queue->queue_id);
-}
-
-static int sec_map_io(struct sec_dev_info *info, struct platform_device *pdev)
-{
-	int i;
-
-	for (i = 0; i < SEC_NUM_ADDR_REGIONS; i++) {
-		info->regs[i] = devm_platform_ioremap_resource(pdev, i);
-		if (IS_ERR(info->regs[i]))
-			return PTR_ERR(info->regs[i]);
-	}
-
-	return 0;
-}
-
-static int sec_base_init(struct sec_dev_info *info,
-			 struct platform_device *pdev)
-{
-	int ret;
-
-	ret = sec_map_io(info, pdev);
-	if (ret)
-		return ret;
-
-	ret = sec_clk_en(info);
-	if (ret)
-		return ret;
-
-	ret = sec_reset_whole_module(info);
-	if (ret)
-		goto sec_clk_disable;
-
-	ret = sec_hw_init(info);
-	if (ret)
-		goto sec_clk_disable;
-
-	return 0;
-
-sec_clk_disable:
-	sec_clk_dis(info);
-
-	return ret;
-}
-
-static void sec_base_exit(struct sec_dev_info *info)
-{
-	sec_hw_exit(info);
-	sec_clk_dis(info);
-}
-
-#define SEC_Q_CMD_SIZE \
-	round_up(SEC_QUEUE_LEN * sizeof(struct sec_bd_info), PAGE_SIZE)
-#define SEC_Q_CQ_SIZE \
-	round_up(SEC_QUEUE_LEN * sizeof(struct sec_out_bd_info), PAGE_SIZE)
-#define SEC_Q_DB_SIZE \
-	round_up(SEC_QUEUE_LEN * sizeof(struct sec_debug_bd_info), PAGE_SIZE)
-
-static int sec_queue_res_cfg(struct sec_queue *queue)
-{
-	struct device *dev = queue->dev_info->dev;
-	struct sec_queue_ring_cmd *ring_cmd = &queue->ring_cmd;
-	struct sec_queue_ring_cq *ring_cq = &queue->ring_cq;
-	struct sec_queue_ring_db *ring_db = &queue->ring_db;
-	int ret;
-
-	ring_cmd->vaddr = dma_alloc_coherent(dev, SEC_Q_CMD_SIZE,
-					     &ring_cmd->paddr, GFP_KERNEL);
-	if (!ring_cmd->vaddr)
-		return -ENOMEM;
-
-	atomic_set(&ring_cmd->used, 0);
-	mutex_init(&ring_cmd->lock);
-	ring_cmd->callback = sec_alg_callback;
-
-	ring_cq->vaddr = dma_alloc_coherent(dev, SEC_Q_CQ_SIZE,
-					    &ring_cq->paddr, GFP_KERNEL);
-	if (!ring_cq->vaddr) {
-		ret = -ENOMEM;
-		goto err_free_ring_cmd;
-	}
-
-	ring_db->vaddr = dma_alloc_coherent(dev, SEC_Q_DB_SIZE,
-					    &ring_db->paddr, GFP_KERNEL);
-	if (!ring_db->vaddr) {
-		ret = -ENOMEM;
-		goto err_free_ring_cq;
-	}
-	queue->task_irq = platform_get_irq(to_platform_device(dev),
-					   queue->queue_id * 2 + 1);
-	if (queue->task_irq < 0) {
-		ret = queue->task_irq;
-		goto err_free_ring_db;
-	}
-
-	return 0;
-
-err_free_ring_db:
-	dma_free_coherent(dev, SEC_Q_DB_SIZE, queue->ring_db.vaddr,
-			  queue->ring_db.paddr);
-err_free_ring_cq:
-	dma_free_coherent(dev, SEC_Q_CQ_SIZE, queue->ring_cq.vaddr,
-			  queue->ring_cq.paddr);
-err_free_ring_cmd:
-	dma_free_coherent(dev, SEC_Q_CMD_SIZE, queue->ring_cmd.vaddr,
-			  queue->ring_cmd.paddr);
-
-	return ret;
-}
-
-static void sec_queue_free_ring_pages(struct sec_queue *queue)
-{
-	struct device *dev = queue->dev_info->dev;
-
-	dma_free_coherent(dev, SEC_Q_DB_SIZE, queue->ring_db.vaddr,
-			  queue->ring_db.paddr);
-	dma_free_coherent(dev, SEC_Q_CQ_SIZE, queue->ring_cq.vaddr,
-			  queue->ring_cq.paddr);
-	dma_free_coherent(dev, SEC_Q_CMD_SIZE, queue->ring_cmd.vaddr,
-			  queue->ring_cmd.paddr);
-}
-
-static int sec_queue_config(struct sec_dev_info *info, struct sec_queue *queue,
-			    int queue_id)
-{
-	int ret;
-
-	sec_queue_base_init(info, queue, queue_id);
-
-	ret = sec_queue_res_cfg(queue);
-	if (ret)
-		return ret;
-
-	ret = sec_queue_map_io(queue);
-	if (ret) {
-		dev_err(info->dev, "Queue map failed %d\n", ret);
-		sec_queue_free_ring_pages(queue);
-		return ret;
-	}
-
-	sec_queue_hw_init(queue);
-
-	return 0;
-}
-
-static void sec_queue_unconfig(struct sec_dev_info *info,
-			       struct sec_queue *queue)
-{
-	sec_queue_unmap_io(queue);
-	sec_queue_free_ring_pages(queue);
-}
-
-static int sec_id_alloc(struct sec_dev_info *info)
-{
-	int ret = 0;
-	int i;
-
-	mutex_lock(&sec_id_lock);
-
-	for (i = 0; i < SEC_MAX_DEVICES; i++)
-		if (!sec_devices[i])
-			break;
-	if (i == SEC_MAX_DEVICES) {
-		ret = -ENOMEM;
-		goto unlock;
-	}
-	info->sec_id = i;
-	sec_devices[info->sec_id] = info;
-
-unlock:
-	mutex_unlock(&sec_id_lock);
-
-	return ret;
-}
-
-static void sec_id_free(struct sec_dev_info *info)
-{
-	mutex_lock(&sec_id_lock);
-	sec_devices[info->sec_id] = NULL;
-	mutex_unlock(&sec_id_lock);
-}
-
-static int sec_probe(struct platform_device *pdev)
-{
-	struct sec_dev_info *info;
-	struct device *dev = &pdev->dev;
-	int i, j;
-	int ret;
-
-	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
-	if (ret) {
-		dev_err(dev, "Failed to set 64 bit dma mask %d", ret);
-		return -ENODEV;
-	}
-
-	info = devm_kzalloc(dev, (sizeof(*info)), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	info->dev = dev;
-	mutex_init(&info->dev_lock);
-
-	info->hw_sgl_pool = dmam_pool_create("sgl", dev,
-					     sizeof(struct sec_hw_sgl), 64, 0);
-	if (!info->hw_sgl_pool) {
-		dev_err(dev, "Failed to create sec sgl dma pool\n");
-		return -ENOMEM;
-	}
-
-	ret = sec_base_init(info, pdev);
-	if (ret) {
-		dev_err(dev, "Base initialization fail! %d\n", ret);
-		return ret;
-	}
-
-	for (i = 0; i < SEC_Q_NUM; i++) {
-		ret = sec_queue_config(info, &info->queues[i], i);
-		if (ret)
-			goto queues_unconfig;
-
-		ret = sec_queue_irq_init(&info->queues[i]);
-		if (ret) {
-			sec_queue_unconfig(info, &info->queues[i]);
-			goto queues_unconfig;
-		}
-	}
-
-	ret = sec_algs_register();
-	if (ret) {
-		dev_err(dev, "Failed to register algorithms with crypto %d\n",
-			ret);
-		goto queues_unconfig;
-	}
-
-	platform_set_drvdata(pdev, info);
-
-	ret = sec_id_alloc(info);
-	if (ret)
-		goto algs_unregister;
-
-	return 0;
-
-algs_unregister:
-	sec_algs_unregister();
-queues_unconfig:
-	for (j = i - 1; j >= 0; j--) {
-		sec_queue_irq_uninit(&info->queues[j]);
-		sec_queue_unconfig(info, &info->queues[j]);
-	}
-	sec_base_exit(info);
-
-	return ret;
-}
-
-static void sec_remove(struct platform_device *pdev)
-{
-	struct sec_dev_info *info = platform_get_drvdata(pdev);
-	int i;
-
-	/* Unexpose as soon as possible, reuse during remove is fine */
-	sec_id_free(info);
-
-	sec_algs_unregister();
-
-	for (i = 0; i < SEC_Q_NUM; i++) {
-		sec_queue_irq_uninit(&info->queues[i]);
-		sec_queue_unconfig(info, &info->queues[i]);
-	}
-
-	sec_base_exit(info);
-}
-
-static const __maybe_unused struct of_device_id sec_match[] = {
-	{ .compatible = "hisilicon,hip06-sec" },
-	{ .compatible = "hisilicon,hip07-sec" },
-	{}
-};
-MODULE_DEVICE_TABLE(of, sec_match);
-
-static const __maybe_unused struct acpi_device_id sec_acpi_match[] = {
-	{ "HISI02C1", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(acpi, sec_acpi_match);
-
-static struct platform_driver sec_driver = {
-	.probe = sec_probe,
-	.remove = sec_remove,
-	.driver = {
-		.name = "hisi_sec_platform_driver",
-		.of_match_table = sec_match,
-		.acpi_match_table = ACPI_PTR(sec_acpi_match),
-	},
-};
-module_platform_driver(sec_driver);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("HiSilicon Security Accelerators");
-MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com");
-MODULE_AUTHOR("Jonathan Cameron <jonathan.cameron@huawei.com>");
diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h
deleted file mode 100644
index e2a50bf2234b9d..00000000000000
--- a/drivers/crypto/hisilicon/sec/sec_drv.h
+++ /dev/null
@@ -1,428 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright (c) 2016-2017 HiSilicon Limited. */
-
-#ifndef _SEC_DRV_H_
-#define _SEC_DRV_H_
-
-#include <crypto/algapi.h>
-#include <linux/kfifo.h>
-
-#define SEC_MAX_SGE_NUM			64
-#define SEC_HW_RING_NUM			3
-
-#define SEC_CMD_RING			0
-#define SEC_OUTORDER_RING		1
-#define SEC_DBG_RING			2
-
-/* A reasonable length to balance memory use against flexibility */
-#define SEC_QUEUE_LEN			512
-
-#define SEC_MAX_SGE_NUM   64
-
-struct sec_bd_info {
-#define SEC_BD_W0_T_LEN_M			GENMASK(4, 0)
-#define SEC_BD_W0_T_LEN_S			0
-
-#define SEC_BD_W0_C_WIDTH_M			GENMASK(6, 5)
-#define SEC_BD_W0_C_WIDTH_S			5
-#define   SEC_C_WIDTH_AES_128BIT		0
-#define   SEC_C_WIDTH_AES_8BIT		1
-#define   SEC_C_WIDTH_AES_1BIT		2
-#define   SEC_C_WIDTH_DES_64BIT		0
-#define   SEC_C_WIDTH_DES_8BIT		1
-#define   SEC_C_WIDTH_DES_1BIT		2
-
-#define SEC_BD_W0_C_MODE_M			GENMASK(9, 7)
-#define SEC_BD_W0_C_MODE_S			7
-#define   SEC_C_MODE_ECB			0
-#define   SEC_C_MODE_CBC			1
-#define   SEC_C_MODE_CTR			4
-#define   SEC_C_MODE_CCM			5
-#define   SEC_C_MODE_GCM			6
-#define   SEC_C_MODE_XTS			7
-
-#define SEC_BD_W0_SEQ				BIT(10)
-#define SEC_BD_W0_DE				BIT(11)
-#define SEC_BD_W0_DAT_SKIP_M			GENMASK(13, 12)
-#define SEC_BD_W0_DAT_SKIP_S			12
-#define SEC_BD_W0_C_GRAN_SIZE_19_16_M		GENMASK(17, 14)
-#define SEC_BD_W0_C_GRAN_SIZE_19_16_S		14
-
-#define SEC_BD_W0_CIPHER_M			GENMASK(19, 18)
-#define SEC_BD_W0_CIPHER_S			18
-#define   SEC_CIPHER_NULL			0
-#define   SEC_CIPHER_ENCRYPT			1
-#define   SEC_CIPHER_DECRYPT			2
-
-#define SEC_BD_W0_AUTH_M			GENMASK(21, 20)
-#define SEC_BD_W0_AUTH_S			20
-#define   SEC_AUTH_NULL				0
-#define   SEC_AUTH_MAC				1
-#define   SEC_AUTH_VERIF			2
-
-#define SEC_BD_W0_AI_GEN			BIT(22)
-#define SEC_BD_W0_CI_GEN			BIT(23)
-#define SEC_BD_W0_NO_HPAD			BIT(24)
-#define SEC_BD_W0_HM_M				GENMASK(26, 25)
-#define SEC_BD_W0_HM_S				25
-#define SEC_BD_W0_ICV_OR_SKEY_EN_M		GENMASK(28, 27)
-#define SEC_BD_W0_ICV_OR_SKEY_EN_S		27
-
-/* Multi purpose field - gran size bits for send, flag for recv */
-#define SEC_BD_W0_FLAG_M			GENMASK(30, 29)
-#define SEC_BD_W0_C_GRAN_SIZE_21_20_M		GENMASK(30, 29)
-#define SEC_BD_W0_FLAG_S			29
-#define SEC_BD_W0_C_GRAN_SIZE_21_20_S		29
-
-#define SEC_BD_W0_DONE				BIT(31)
-	u32 w0;
-
-#define SEC_BD_W1_AUTH_GRAN_SIZE_M		GENMASK(21, 0)
-#define SEC_BD_W1_AUTH_GRAN_SIZE_S		0
-#define SEC_BD_W1_M_KEY_EN			BIT(22)
-#define SEC_BD_W1_BD_INVALID			BIT(23)
-#define SEC_BD_W1_ADDR_TYPE			BIT(24)
-
-#define SEC_BD_W1_A_ALG_M			GENMASK(28, 25)
-#define SEC_BD_W1_A_ALG_S			25
-#define   SEC_A_ALG_SHA1			0
-#define   SEC_A_ALG_SHA256			1
-#define   SEC_A_ALG_MD5				2
-#define   SEC_A_ALG_SHA224			3
-#define   SEC_A_ALG_HMAC_SHA1			8
-#define   SEC_A_ALG_HMAC_SHA224			10
-#define   SEC_A_ALG_HMAC_SHA256			11
-#define   SEC_A_ALG_HMAC_MD5			12
-#define   SEC_A_ALG_AES_XCBC			13
-#define   SEC_A_ALG_AES_CMAC			14
-
-#define SEC_BD_W1_C_ALG_M			GENMASK(31, 29)
-#define SEC_BD_W1_C_ALG_S			29
-#define   SEC_C_ALG_DES				0
-#define   SEC_C_ALG_3DES			1
-#define   SEC_C_ALG_AES				2
-
-	u32 w1;
-
-#define SEC_BD_W2_C_GRAN_SIZE_15_0_M		GENMASK(15, 0)
-#define SEC_BD_W2_C_GRAN_SIZE_15_0_S		0
-#define SEC_BD_W2_GRAN_NUM_M			GENMASK(31, 16)
-#define SEC_BD_W2_GRAN_NUM_S			16
-	u32 w2;
-
-#define SEC_BD_W3_AUTH_LEN_OFFSET_M		GENMASK(9, 0)
-#define SEC_BD_W3_AUTH_LEN_OFFSET_S		0
-#define SEC_BD_W3_CIPHER_LEN_OFFSET_M		GENMASK(19, 10)
-#define SEC_BD_W3_CIPHER_LEN_OFFSET_S		10
-#define SEC_BD_W3_MAC_LEN_M			GENMASK(24, 20)
-#define SEC_BD_W3_MAC_LEN_S			20
-#define SEC_BD_W3_A_KEY_LEN_M			GENMASK(29, 25)
-#define SEC_BD_W3_A_KEY_LEN_S			25
-#define SEC_BD_W3_C_KEY_LEN_M			GENMASK(31, 30)
-#define SEC_BD_W3_C_KEY_LEN_S			30
-#define   SEC_KEY_LEN_AES_128			0
-#define   SEC_KEY_LEN_AES_192			1
-#define   SEC_KEY_LEN_AES_256			2
-#define   SEC_KEY_LEN_DES			1
-#define   SEC_KEY_LEN_3DES_3_KEY		1
-#define   SEC_KEY_LEN_3DES_2_KEY		3
-	u32 w3;
-
-	/* W4,5 */
-	union {
-		u32 authkey_addr_lo;
-		u32 authiv_addr_lo;
-	};
-	union {
-		u32 authkey_addr_hi;
-		u32 authiv_addr_hi;
-	};
-
-	/* W6,7 */
-	u32 cipher_key_addr_lo;
-	u32 cipher_key_addr_hi;
-
-	/* W8,9 */
-	u32 cipher_iv_addr_lo;
-	u32 cipher_iv_addr_hi;
-
-	/* W10,11 */
-	u32 data_addr_lo;
-	u32 data_addr_hi;
-
-	/* W12,13 */
-	u32 mac_addr_lo;
-	u32 mac_addr_hi;
-
-	/* W14,15 */
-	u32 cipher_destin_addr_lo;
-	u32 cipher_destin_addr_hi;
-};
-
-enum sec_mem_region {
-	SEC_COMMON = 0,
-	SEC_SAA,
-	SEC_NUM_ADDR_REGIONS
-};
-
-#define SEC_NAME_SIZE				64
-#define SEC_Q_NUM				16
-
-
-/**
- * struct sec_queue_ring_cmd - store information about a SEC HW cmd ring
- * @used: Local counter used to cheaply establish if the ring is empty.
- * @lock: Protect against simultaneous adjusting of the read and write pointers.
- * @vaddr: Virtual address for the ram pages used for the ring.
- * @paddr: Physical address of the dma mapped region of ram used for the ring.
- * @callback: Callback function called on a ring element completing.
- */
-struct sec_queue_ring_cmd {
-	atomic_t used;
-	struct mutex lock;
-	struct sec_bd_info *vaddr;
-	dma_addr_t paddr;
-	void (*callback)(struct sec_bd_info *resp, void *ctx);
-};
-
-struct sec_debug_bd_info;
-struct sec_queue_ring_db {
-	struct sec_debug_bd_info *vaddr;
-	dma_addr_t paddr;
-};
-
-struct sec_out_bd_info;
-struct sec_queue_ring_cq {
-	struct sec_out_bd_info *vaddr;
-	dma_addr_t paddr;
-};
-
-struct sec_dev_info;
-
-enum sec_cipher_alg {
-	SEC_C_DES_ECB_64,
-	SEC_C_DES_CBC_64,
-
-	SEC_C_3DES_ECB_192_3KEY,
-	SEC_C_3DES_ECB_192_2KEY,
-
-	SEC_C_3DES_CBC_192_3KEY,
-	SEC_C_3DES_CBC_192_2KEY,
-
-	SEC_C_AES_ECB_128,
-	SEC_C_AES_ECB_192,
-	SEC_C_AES_ECB_256,
-
-	SEC_C_AES_CBC_128,
-	SEC_C_AES_CBC_192,
-	SEC_C_AES_CBC_256,
-
-	SEC_C_AES_CTR_128,
-	SEC_C_AES_CTR_192,
-	SEC_C_AES_CTR_256,
-
-	SEC_C_AES_XTS_128,
-	SEC_C_AES_XTS_256,
-
-	SEC_C_NULL,
-};
-
-/**
- * struct sec_alg_tfm_ctx - hardware specific tranformation context
- * @cipher_alg: Cipher algorithm enabled include encryption mode.
- * @key: Key storage if required.
- * @pkey: DMA address for the key storage.
- * @req_template: Request template to save time on setup.
- * @queue: The hardware queue associated with this tfm context.
- * @lock: Protect key and pkey to ensure they are consistent
- * @auth_buf: Current context buffer for auth operations.
- * @backlog: The backlog queue used for cases where our buffers aren't
- * large enough.
- */
-struct sec_alg_tfm_ctx {
-	enum sec_cipher_alg cipher_alg;
-	u8 *key;
-	dma_addr_t pkey;
-	struct sec_bd_info req_template;
-	struct sec_queue *queue;
-	struct mutex lock;
-	u8 *auth_buf;
-	struct list_head backlog;
-};
-
-/**
- * struct sec_request - data associate with a single crypto request
- * @elements: List of subparts of this request (hardware size restriction)
- * @num_elements: The number of subparts (used as an optimization)
- * @lock: Protect elements of this structure against concurrent change.
- * @tfm_ctx: hardware specific context.
- * @len_in: length of in sgl from upper layers
- * @len_out: length of out sgl from upper layers
- * @dma_iv: initialization vector - phsyical address
- * @err: store used to track errors across subelements of this request.
- * @req_base: pointer to base element of associate crypto context.
- * This is needed to allow shared handling skcipher, ahash etc.
- * @cb: completion callback.
- * @backlog_head: list head to allow backlog maintenance.
- *
- * The hardware is limited in the maximum size of data that it can
- * process from a single BD.  Typically this is fairly large (32MB)
- * but still requires the complexity of splitting the incoming
- * skreq up into a number of elements complete with appropriate
- * iv chaining.
- */
-struct sec_request {
-	struct list_head elements;
-	int num_elements;
-	struct mutex lock;
-	struct sec_alg_tfm_ctx *tfm_ctx;
-	int len_in;
-	int len_out;
-	dma_addr_t dma_iv;
-	int err;
-	struct crypto_async_request *req_base;
-	void (*cb)(struct sec_bd_info *resp, struct crypto_async_request *req);
-	struct list_head backlog_head;
-};
-
-/**
- * struct sec_request_el - A subpart of a request.
- * @head: allow us to attach this to the list in the sec_request
- * @req: hardware block descriptor corresponding to this request subpart
- * @in: hardware sgl for input - virtual address
- * @dma_in: hardware sgl for input - physical address
- * @sgl_in: scatterlist for this request subpart
- * @out: hardware sgl for output - virtual address
- * @dma_out: hardware sgl for output - physical address
- * @sgl_out: scatterlist for this request subpart
- * @sec_req: The request which this subpart forms a part of
- * @el_length: Number of bytes in this subpart. Needed to locate
- * last ivsize chunk for iv chaining.
- */
-struct sec_request_el {
-	struct list_head head;
-	struct sec_bd_info req;
-	struct sec_hw_sgl *in;
-	dma_addr_t dma_in;
-	struct scatterlist *sgl_in;
-	struct sec_hw_sgl *out;
-	dma_addr_t dma_out;
-	struct scatterlist *sgl_out;
-	struct sec_request *sec_req;
-	size_t el_length;
-};
-
-/**
- * struct sec_queue - All the information about a HW queue
- * @dev_info: The parent SEC device to which this queue belongs.
- * @task_irq: Completion interrupt for the queue.
- * @name: Human readable queue description also used as irq name.
- * @ring: The several HW rings associated with one queue.
- * @regs: The iomapped device registers
- * @queue_id: Index of the queue used for naming and resource selection.
- * @in_use: Flag to say if the queue is in use.
- * @expected: The next expected element to finish assuming we were in order.
- * @uprocessed: A bitmap to track which OoO elements are done but not handled.
- * @softqueue: A software queue used when chaining requirements prevent direct
- *   use of the hardware queues.
- * @havesoftqueue: A flag to say we have a queues - as we may need one for the
- *   current mode.
- * @queuelock: Protect the soft queue from concurrent changes to avoid some
- *   potential loss of data races.
- * @shadow: Pointers back to the shadow copy of the hardware ring element
- *   need because we can't store any context reference in the bd element.
- */
-struct sec_queue {
-	struct sec_dev_info *dev_info;
-	int task_irq;
-	char name[SEC_NAME_SIZE];
-	struct sec_queue_ring_cmd ring_cmd;
-	struct sec_queue_ring_cq ring_cq;
-	struct sec_queue_ring_db ring_db;
-	void __iomem *regs;
-	u32 queue_id;
-	bool in_use;
-	int expected;
-
-	DECLARE_BITMAP(unprocessed, SEC_QUEUE_LEN);
-	DECLARE_KFIFO_PTR(softqueue, typeof(struct sec_request_el *));
-	bool havesoftqueue;
-	spinlock_t queuelock;
-	void *shadow[SEC_QUEUE_LEN];
-};
-
-/**
- * struct sec_hw_sge: Track each of the 64 element SEC HW SGL entries
- * @buf: The IOV dma address for this entry.
- * @len: Length of this IOV.
- * @pad: Reserved space.
- */
-struct sec_hw_sge {
-	dma_addr_t buf;
-	unsigned int len;
-	unsigned int pad;
-};
-
-/**
- * struct sec_hw_sgl: One hardware SGL entry.
- * @next_sgl: The next entry if we need to chain dma address. Null if last.
- * @entry_sum_in_chain: The full count of SGEs - only matters for first SGL.
- * @entry_sum_in_sgl: The number of SGEs in this SGL element.
- * @flag: Unused in skciphers.
- * @serial_num: Unsued in skciphers.
- * @cpuid: Currently unused.
- * @data_bytes_in_sgl: Count of bytes from all SGEs in this SGL.
- * @next: Virtual address used to stash the next sgl - useful in completion.
- * @reserved: A reserved field not currently used.
- * @sge_entries: The (up to) 64 Scatter Gather Entries, representing IOVs.
- * @node: Currently unused.
- */
-struct sec_hw_sgl {
-	dma_addr_t next_sgl;
-	u16 entry_sum_in_chain;
-	u16 entry_sum_in_sgl;
-	u32 flag;
-	u64 serial_num;
-	u32 cpuid;
-	u32 data_bytes_in_sgl;
-	struct sec_hw_sgl *next;
-	u64 reserved;
-	struct sec_hw_sge  sge_entries[SEC_MAX_SGE_NUM];
-	u8 node[16];
-};
-
-struct dma_pool;
-
-/**
- * struct sec_dev_info: The full SEC unit comprising queues and processors.
- * @sec_id: Index used to track which SEC this is when more than one is present.
- * @num_saas: The number of backed processors enabled.
- * @regs: iomapped register regions shared by whole SEC unit.
- * @dev_lock: Protects concurrent queue allocation / freeing for the SEC.
- * @queues: The 16 queues that this SEC instance provides.
- * @dev: Device pointer.
- * @hw_sgl_pool: DMA pool used to mimise mapping for the scatter gather lists.
- */
-struct sec_dev_info {
-	int sec_id;
-	int num_saas;
-	void __iomem *regs[SEC_NUM_ADDR_REGIONS];
-	struct mutex dev_lock;
-	int queues_in_use;
-	struct sec_queue queues[SEC_Q_NUM];
-	struct device *dev;
-	struct dma_pool *hw_sgl_pool;
-};
-
-int sec_queue_send(struct sec_queue *queue, struct sec_bd_info *msg, void *ctx);
-bool sec_queue_can_enqueue(struct sec_queue *queue, int num);
-int sec_queue_stop_release(struct sec_queue *queue);
-struct sec_queue *sec_queue_alloc_start_safe(void);
-bool sec_queue_empty(struct sec_queue *queue);
-
-/* Algorithm specific elements from sec_algs.c */
-void sec_alg_callback(struct sec_bd_info *resp, void *ctx);
-int sec_algs_register(void);
-void sec_algs_unregister(void);
-
-#endif /* _SEC_DRV_H_ */
diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c
index 752565200c1651..03523cfbb28e2e 100644
--- a/drivers/crypto/hisilicon/sec2/sec_main.c
+++ b/drivers/crypto/hisilicon/sec2/sec_main.c
@@ -484,7 +484,7 @@ static void sec_set_endian(struct hisi_qm *qm)
 	if (!IS_ENABLED(CONFIG_64BIT))
 		reg |= BIT(1);
 
-	if (!IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
+	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 		reg |= BIT(0);
 
 	writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);
diff --git a/drivers/crypto/hisilicon/zip/dae_main.c b/drivers/crypto/hisilicon/zip/dae_main.c
index 68aebd02fc84dc..57ea836a265e05 100644
--- a/drivers/crypto/hisilicon/zip/dae_main.c
+++ b/drivers/crypto/hisilicon/zip/dae_main.c
@@ -39,6 +39,8 @@
 #define DAE_AM_RETURN_MASK		0x3
 #define DAE_AXI_CFG_OFFSET		0x331000
 #define DAE_AXI_SHUTDOWN_EN_MASK	(BIT(0) | BIT(5))
+#define DAE_AUTO_CG_EN_OFFSET		0x331070
+#define DAE_AUTO_CG_EN_MASK		(~(BIT_ULL(4) | BIT_ULL(18) | BIT_ULL(19)))
 
 struct hisi_dae_hw_error {
 	u32 int_msk;
@@ -61,6 +63,17 @@ static inline bool dae_is_support(struct hisi_qm *qm)
 	return false;
 }
 
+static void hisi_dae_set_clock_gating(struct hisi_qm *qm, bool enable)
+{
+	if (qm->ver < QM_HW_V5)
+		return;
+
+	if (enable)
+		writeq(DAE_AUTO_CG_EN_MASK, qm->io_base + DAE_AUTO_CG_EN_OFFSET);
+	else
+		writeq(0, qm->io_base + DAE_AUTO_CG_EN_OFFSET);
+}
+
 int hisi_dae_set_user_domain(struct hisi_qm *qm)
 {
 	u32 val;
@@ -69,16 +82,22 @@ int hisi_dae_set_user_domain(struct hisi_qm *qm)
 	if (!dae_is_support(qm))
 		return 0;
 
+	hisi_dae_set_clock_gating(qm, false);
+
 	val = readl(qm->io_base + DAE_MEM_START_OFFSET);
 	val |= DAE_MEM_START_MASK;
 	writel(val, qm->io_base + DAE_MEM_START_OFFSET);
 	ret = readl_relaxed_poll_timeout(qm->io_base + DAE_MEM_DONE_OFFSET, val,
 					 val & DAE_MEM_DONE_MASK,
 					 DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US);
-	if (ret)
+	if (ret) {
 		pci_err(qm->pdev, "failed to init dae memory!\n");
+		return ret;
+	}
 
-	return ret;
+	hisi_dae_set_clock_gating(qm, true);
+
+	return 0;
 }
 
 int hisi_dae_set_alg(struct hisi_qm *qm)
diff --git a/drivers/crypto/inside-secure/eip93/eip93-aead.c b/drivers/crypto/inside-secure/eip93/eip93-aead.c
index 2bbd0af7b0e0ed..0b7a899b14d76b 100644
--- a/drivers/crypto/inside-secure/eip93/eip93-aead.c
+++ b/drivers/crypto/inside-secure/eip93/eip93-aead.c
@@ -92,7 +92,6 @@ static int eip93_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 	struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
 	struct eip93_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_authenc_keys keys;
-	struct crypto_aes_ctx aes;
 	struct sa_record *sa_record = ctx->sa_record;
 	u32 nonce = 0;
 	int ret;
@@ -126,7 +125,7 @@ static int eip93_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 
 		break;
 	case EIP93_ALG_AES:
-		ret = aes_expandkey(&aes, keys.enckey, keys.enckeylen);
+		ret = aes_check_keylen(keys.enckeylen);
 		if (ret)
 			return ret;
 
diff --git a/drivers/crypto/inside-secure/eip93/eip93-regs.h b/drivers/crypto/inside-secure/eip93/eip93-regs.h
index 96d28c6651bdb7..1d726bcdd77f32 100644
--- a/drivers/crypto/inside-secure/eip93/eip93-regs.h
+++ b/drivers/crypto/inside-secure/eip93/eip93-regs.h
@@ -281,7 +281,7 @@
 #define   EIP93_SA_CMD_HASH_CRYPT_OFFSET	GENMASK(23, 16)
 #define   EIP93_SA_CMD_BYTE_OFFSET		BIT(13) /* 0: CRYPT_OFFSET in 32bit word 1: CRYPT_OFFSET in 8bit bytes */
 #define   EIP93_SA_CMD_HMAC			BIT(12)
-#define   EIP93_SA_CMD_SSL_MAC			BIT(12)
+#define   EIP93_SA_CMD_SSL_MAC			BIT(11)
 /* This mask can be either used for ARC4 or AES */
 #define   EIP93_SA_CMD_CHIPER_MODE		GENMASK(9, 8)
 /* AES or DES operations */
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
index a8349b684693e4..b031cb9652ec3f 100644
--- a/drivers/crypto/inside-secure/safexcel_cipher.c
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -375,7 +375,7 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
 	struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct safexcel_crypto_priv *priv = ctx->base.priv;
-	struct crypto_aes_ctx aes;
+	struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx);
 	int ret, i;
 
 	ret = aes_expandkey(&aes, key, len);
@@ -396,7 +396,6 @@ static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
 
 	ctx->key_len = len;
 
-	memzero_explicit(&aes, sizeof(aes));
 	return 0;
 }
 
@@ -407,7 +406,6 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct safexcel_crypto_priv *priv = ctx->base.priv;
 	struct crypto_authenc_keys keys;
-	struct crypto_aes_ctx aes;
 	int err = -EINVAL, i;
 	const char *alg;
 
@@ -438,7 +436,7 @@ static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key,
 			goto badkey;
 		break;
 	case SAFEXCEL_AES:
-		err = aes_expandkey(&aes, keys.enckey, keys.enckeylen);
+		err = aes_check_keylen(keys.enckeylen);
 		if (unlikely(err))
 			goto badkey;
 		break;
@@ -1362,7 +1360,7 @@ static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm,
 	struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct safexcel_crypto_priv *priv = ctx->base.priv;
-	struct crypto_aes_ctx aes;
+	struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx);
 	int ret, i;
 	unsigned int keylen;
 
@@ -1388,7 +1386,6 @@ static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm,
 
 	ctx->key_len = keylen;
 
-	memzero_explicit(&aes, sizeof(aes));
 	return 0;
 }
 
@@ -2542,7 +2539,7 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm,
 	struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct safexcel_crypto_priv *priv = ctx->base.priv;
-	struct crypto_aes_ctx aes;
+	struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx);
 	int ret, i;
 	unsigned int keylen;
 
@@ -2590,7 +2587,6 @@ static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm,
 
 	ctx->key_len = keylen << 1;
 
-	memzero_explicit(&aes, sizeof(aes));
 	return 0;
 }
 
@@ -2756,12 +2752,11 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
 	struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct safexcel_crypto_priv *priv = ctx->base.priv;
-	struct crypto_aes_ctx aes;
+	struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx);
 	int ret, i;
 
 	ret = aes_expandkey(&aes, key, len);
 	if (ret) {
-		memzero_explicit(&aes, sizeof(aes));
 		return ret;
 	}
 
@@ -2790,7 +2785,6 @@ static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key,
 	else
 		ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
 
-	memzero_explicit(&aes, sizeof(aes));
 	return 0;
 }
 
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
index 3402e570d045c1..8f107da66685c2 100644
--- a/drivers/crypto/inside-secure/safexcel_hash.c
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -1905,7 +1905,7 @@ static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
 				 unsigned int len)
 {
 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
-	struct crypto_aes_ctx aes;
+	struct crypto_aes_ctx aes __cleanup(aes_zeroize_ctx);
 	int ret, i;
 
 	ret = aes_expandkey(&aes, key, len);
@@ -1928,7 +1928,6 @@ static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
 	}
 	ctx->cbcmac  = true;
 
-	memzero_explicit(&aes, sizeof(aes));
 	return 0;
 }
 
@@ -1993,6 +1992,7 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
 	ret = aes_prepareenckey(ctx->aes,
 				(u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
 				AES_MIN_KEY_SIZE);
+	memzero_explicit(key_tmp, sizeof(key_tmp));
 	if (ret)
 		return ret;
 
@@ -2104,6 +2104,8 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
 	}
 	ctx->cbcmac = false;
 
+	memzero_explicit(consts, sizeof(consts));
+	memzero_explicit(_const, sizeof(_const));
 	return 0;
 }
 
diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c
index b88febf53a19d9..e2ef5284abef10 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_cfg.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c
@@ -1,6 +1,5 @@
 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
 /* Copyright(c) 2014 - 2020 Intel Corporation */
-#include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/list.h>
@@ -9,13 +8,11 @@
 #include "adf_cfg.h"
 #include "adf_common_drv.h"
 
-static DEFINE_MUTEX(qat_cfg_read_lock);
-
 static void *qat_dev_cfg_start(struct seq_file *sfile, loff_t *pos)
 {
 	struct adf_cfg_device_data *dev_cfg = sfile->private;
 
-	mutex_lock(&qat_cfg_read_lock);
+	down_read(&dev_cfg->lock);
 	return seq_list_start(&dev_cfg->sec_list, *pos);
 }
 
@@ -43,7 +40,9 @@ static void *qat_dev_cfg_next(struct seq_file *sfile, void *v, loff_t *pos)
 
 static void qat_dev_cfg_stop(struct seq_file *sfile, void *v)
 {
-	mutex_unlock(&qat_cfg_read_lock);
+	struct adf_cfg_device_data *dev_cfg = sfile->private;
+
+	up_read(&dev_cfg->lock);
 }
 
 static const struct seq_operations qat_dev_cfg_sops = {
@@ -146,24 +145,6 @@ static void adf_cfg_keyval_add(struct adf_cfg_key_val *new,
 	list_add_tail(&new->list, &sec->param_head);
 }
 
-static void adf_cfg_keyval_remove(const char *key, struct adf_cfg_section *sec)
-{
-	struct list_head *head = &sec->param_head;
-	struct list_head *list_ptr, *tmp;
-
-	list_for_each_prev_safe(list_ptr, tmp, head) {
-		struct adf_cfg_key_val *ptr =
-			list_entry(list_ptr, struct adf_cfg_key_val, list);
-
-		if (strncmp(ptr->key, key, sizeof(ptr->key)))
-			continue;
-
-		list_del(list_ptr);
-		kfree(ptr);
-		break;
-	}
-}
-
 static void adf_cfg_keyval_del_all(struct list_head *head)
 {
 	struct list_head *list_ptr, *tmp;
@@ -272,13 +253,9 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev,
 				enum adf_cfg_val_type type)
 {
 	struct adf_cfg_device_data *cfg = accel_dev->cfg;
-	struct adf_cfg_key_val *key_val;
-	struct adf_cfg_section *section = adf_cfg_sec_find(accel_dev,
-							   section_name);
-	char temp_val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
-
-	if (!section)
-		return -EFAULT;
+	struct adf_cfg_key_val *key_val, *existing;
+	struct adf_cfg_section *section;
+	int ret = 0;
 
 	key_val = kzalloc_obj(*key_val);
 	if (!key_val)
@@ -299,7 +276,17 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev,
 	}
 	key_val->type = type;
 
-	/* Add the key-value pair as below policy:
+	down_write(&cfg->lock);
+
+	section = adf_cfg_sec_find(accel_dev, section_name);
+	if (!section) {
+		kfree(key_val);
+		ret = -ENOENT;
+		goto unlock;
+	}
+
+	/*
+	 * Add the key-value pair as below policy:
 	 * 1. if the key doesn't exist, add it;
 	 * 2. if the key already exists with a different value then update it
 	 *    to the new value (the key is deleted and the newly created
@@ -307,21 +294,21 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev,
 	 * 3. if the key exists with the same value, then return without doing
 	 *    anything (the newly created key_val is freed).
 	 */
-	down_write(&cfg->lock);
-	if (!adf_cfg_key_val_get(accel_dev, section_name, key, temp_val)) {
-		if (strncmp(temp_val, key_val->val, sizeof(temp_val))) {
-			adf_cfg_keyval_remove(key, section);
-		} else {
+	existing = adf_cfg_key_value_find(section, key);
+	if (existing) {
+		if (!strncmp(existing->val, key_val->val, sizeof(existing->val))) {
 			kfree(key_val);
-			goto out;
+			goto unlock;
 		}
+		list_del(&existing->list);
+		kfree(existing);
 	}
 
 	adf_cfg_keyval_add(key_val, section);
 
-out:
+unlock:
 	up_write(&cfg->lock);
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL_GPL(adf_cfg_add_key_value_param);
 
@@ -339,21 +326,26 @@ EXPORT_SYMBOL_GPL(adf_cfg_add_key_value_param);
 int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name)
 {
 	struct adf_cfg_device_data *cfg = accel_dev->cfg;
-	struct adf_cfg_section *sec = adf_cfg_sec_find(accel_dev, name);
+	struct adf_cfg_section *sec;
+	int ret = 0;
 
-	if (sec)
-		return 0;
+	down_write(&cfg->lock);
+
+	if (adf_cfg_sec_find(accel_dev, name))
+		goto unlock;
 
 	sec = kzalloc_obj(*sec);
-	if (!sec)
-		return -ENOMEM;
+	if (!sec) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
 
 	strscpy(sec->name, name);
 	INIT_LIST_HEAD(&sec->param_head);
-	down_write(&cfg->lock);
 	list_add_tail(&sec->list, &cfg->sec_list);
+unlock:
 	up_write(&cfg->lock);
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL_GPL(adf_cfg_section_add);
 
diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c
index bf2bae78a40d58..d8b961b2f2f7f7 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c
@@ -837,6 +837,9 @@ static int adf_gen4_vfmig_save_state(struct qat_mig_dev *mdev)
 		return ret;
 	}
 
+	memset(mdev->state + mdev->setup_size, 0,
+	       mdev->state_size - mdev->setup_size);
+
 	adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state + mdev->setup_size,
 			    mdev->state_size - mdev->setup_size);
 	if (!adf_mstate_preamble_add(vfmig->mstate_mgr))
@@ -934,6 +937,8 @@ static int adf_gen4_vfmig_save_setup(struct qat_mig_dev *mdev)
 	if (mdev->setup_size)
 		return 0;
 
+	memset(mdev->state, 0, mdev->state_size);
+
 	adf_mstate_mgr_init(vfmig->mstate_mgr, mdev->state, mdev->state_size);
 	if (!adf_mstate_preamble_add(vfmig->mstate_mgr))
 		return -EINVAL;
diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c
index 967253082a984b..902c4b1a23e3b1 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c
@@ -323,14 +323,26 @@ static void adf_handle_ti_cd_par_sts(struct adf_accel_dev *accel_dev, void __iom
 	u32 ti_cd_par_sts;
 
 	ti_cd_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_STS);
-	ti_cd_par_sts &= ADF_GEN6_TI_CD_PAR_STS_MASK;
+	ti_cd_par_sts &= ADF_GEN6_TI_CD_PAR_STS_TI_INT_ERR_DEVHALT_MASK;
 	if (ti_cd_par_sts) {
 		dev_err(&GET_DEV(accel_dev), "TI CD parity error: %#x\n", ti_cd_par_sts);
-		ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR);
 		ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_STS, ti_cd_par_sts);
 	}
 }
 
+static void adf_handle_ti_misc_cd_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr)
+{
+	u32 ti_misc_cd_par_sts;
+
+	ti_misc_cd_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_STS);
+	ti_misc_cd_par_sts &= ADF_GEN6_TI_CD_PAR_STS_TI_MISC_MASK;
+	if (ti_misc_cd_par_sts) {
+		dev_err(&GET_DEV(accel_dev), "TI MISC CD parity error: %#x\n", ti_misc_cd_par_sts);
+		ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR);
+		ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_STS, ti_misc_cd_par_sts);
+	}
+}
+
 static void adf_handle_ti_trnsb_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr)
 {
 	u32 ti_trnsb_par_sts;
@@ -367,7 +379,6 @@ static void adf_handle_ti_err(struct adf_accel_dev *accel_dev, void __iomem *csr
 	adf_handle_ti_ci_par_sts(accel_dev, csr);
 	adf_handle_ti_pullfub_par_sts(accel_dev, csr);
 	adf_handle_ti_pushfub_par_sts(accel_dev, csr);
-	adf_handle_ti_cd_par_sts(accel_dev, csr);
 	adf_handle_ti_trnsb_par_sts(accel_dev, csr);
 	adf_handle_iosfp_cmd_parerr(accel_dev, csr);
 }
@@ -591,6 +602,8 @@ static void adf_handle_timiscsts(struct adf_accel_dev *accel_dev, void __iomem *
 	if (!(errsou & ADF_GEN6_ERRSOU3_TIMISCSTS_BIT))
 		return;
 
+	adf_handle_ti_misc_cd_par_sts(accel_dev, csr);
+
 	timiscsts = ADF_CSR_RD(csr, ADF_GEN6_TIMISCSTS);
 	if (timiscsts) {
 		dev_err(&GET_DEV(accel_dev), "Fatal error in transmit interface: %#x\n",
@@ -730,6 +743,7 @@ static void adf_handle_ti_int_err_devhalt(struct adf_accel_dev *accel_dev, void
 	if (!(errsou & ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT))
 		return;
 
+	adf_handle_ti_cd_par_sts(accel_dev, csr);
 	dev_err(&GET_DEV(accel_dev), "DEVHALT due to a TI internal memory error\n");
 	ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL);
 }
diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h
index 66ced271d17339..11f7b2986f567e 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h
@@ -170,23 +170,25 @@ struct adf_ras_ops;
 #define ADF_GEN6_TI_CD_PAR_ERR_MASK			0x500638
 
 /*
- * TI CD parity status mask
+ * TI CD parity status mask - TI_INT_ERR_DEVHALT bits
  * BIT(0) - BIT(15) - CtxMdRam[0:15]_sts parity status
  * BIT(16) - Leaf2ClusterRam_sts parity status
  * BIT(17) - BIT(18) - Ring2LeafRam[0:1]_sts parity status
- * BIT(19) - VirtualQ_sts parity status
- * BIT(20) - DtRdQ_sts parity status
- * BIT(21) - DtWrQ_sts parity status
- * BIT(22) - RiCmdQ_sts parity status
- * BIT(23) - BypassQ_sts parity status
- * BIT(24) - DtRdQ_sc_sts parity status
- * BIT(25) - DtWrQ_sc_sts parity status
  */
-#define ADF_GEN6_TI_CD_PAR_STS_MASK \
+#define ADF_GEN6_TI_CD_PAR_STS_TI_INT_ERR_DEVHALT_MASK \
 	(BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \
 	 BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | \
-	 BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \
-	 BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25))
+	 BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18))
+
+/*
+ * TI CD parity status mask - TIMISCSTS bit
+ * BIT(19) - VirtualQ_sts parity status
+ */
+#define ADF_GEN6_TI_CD_PAR_STS_TI_MISC_MASK		BIT(19)
+
+#define ADF_GEN6_TI_CD_PAR_STS_MASK \
+	(ADF_GEN6_TI_CD_PAR_STS_TI_INT_ERR_DEVHALT_MASK | \
+	 ADF_GEN6_TI_CD_PAR_STS_TI_MISC_MASK)
 
 /* TI TRNSB parity status */
 #define ADF_GEN6_TI_TRNSB_PAR_STS			0x500648
diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c
index 3e39c53814ded1..81cb5e6c9f4a12 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_init.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_init.c
@@ -260,13 +260,23 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev)
 	clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
 	set_bit(ADF_STATUS_STARTED, &accel_dev->status);
 
-	if (!list_empty(&accel_dev->crypto_list) &&
-	    (qat_algs_register() || qat_asym_algs_register())) {
-		dev_err(&GET_DEV(accel_dev),
-			"Failed to register crypto algs\n");
-		set_bit(ADF_STATUS_STARTING, &accel_dev->status);
-		clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
-		return -EFAULT;
+	if (!list_empty(&accel_dev->crypto_list)) {
+		ret = qat_algs_register();
+		if (ret) {
+			dev_err(&GET_DEV(accel_dev), "Failed to register crypto algs\n");
+			set_bit(ADF_STATUS_STARTING, &accel_dev->status);
+			clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
+			return ret;
+		}
+
+		ret = qat_asym_algs_register();
+		if (ret) {
+			dev_err(&GET_DEV(accel_dev), "Failed to register crypto asym algs\n");
+			qat_algs_unregister();
+			set_bit(ADF_STATUS_STARTING, &accel_dev->status);
+			clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
+			return ret;
+		}
 	}
 	set_bit(ADF_STATUS_CRYPTO_ALGS_REGISTERED, &accel_dev->status);
 
diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c
index 1f733ae80bdfac..a0db98f33f1fe1 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_kpt.c
@@ -31,6 +31,7 @@ static ssize_t enable_store(struct device *dev, struct device_attribute *attr,
 	struct adf_hw_device_data *hw_data;
 	struct adf_accel_dev *accel_dev;
 	bool enable;
+	int svc;
 	int ret;
 
 	accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev));
@@ -38,14 +39,25 @@ static ssize_t enable_store(struct device *dev, struct device_attribute *attr,
 		return -EINVAL;
 
 	if (adf_dev_started(accel_dev)) {
-		dev_info(dev, "Device qat_dev%d must be down before enabling KPT\n",
+		dev_info(dev, "Device qat_dev%d must be down before changing KPT state\n",
 			 accel_dev->accel_id);
 		return -EINVAL;
 	}
 
-	if (adf_get_service_enabled(accel_dev) != SVC_ASYM) {
-		dev_info(dev, "KPT can only be enabled when the asymmetric service is enabled\n");
-		return -EINVAL;
+	ret = kstrtobool(buf, &enable);
+	if (ret)
+		return ret;
+
+	if (enable) {
+		svc = adf_get_service_enabled(accel_dev);
+		if (svc < 0)
+			return svc;
+
+		if (svc != SVC_ASYM) {
+			dev_info(dev,
+				 "KPT can only be enabled when the asymmetric crypto service is enabled\n");
+			return -EINVAL;
+		}
 	}
 
 	hw_data = GET_HW_DATA(accel_dev);
@@ -59,10 +71,6 @@ static ssize_t enable_store(struct device *dev, struct device_attribute *attr,
 	if (!hw_data->accel_capabilities_mask)
 		return -EINVAL;
 
-	ret = kstrtobool(buf, &enable);
-	if (ret)
-		return ret;
-
 	user_data = GET_KPT_USER_DATA(accel_dev);
 	user_data->enable = enable;
 
diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c
index 91663805d9e606..2ec19104787ff0 100644
--- a/drivers/crypto/intel/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c
@@ -388,7 +388,7 @@ static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,
 static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
 				    u8 *key_reverse)
 {
-	struct crypto_aes_ctx aes_expanded;
+	struct crypto_aes_ctx aes_expanded __cleanup(aes_zeroize_ctx);
 	int nrounds;
 	u8 *key;
 
@@ -405,7 +405,6 @@ static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
 		memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,
 		       AES_BLOCK_SIZE);
 	}
-	memzero_explicit(&aes_expanded, sizeof(aes_expanded));
 }
 
 static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
@@ -1320,19 +1319,22 @@ int qat_algs_register(void)
 	ret = crypto_register_skciphers(qat_skciphers,
 					ARRAY_SIZE(qat_skciphers));
 	if (ret)
-		goto unlock;
+		goto err_dec;
 
 	ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
 	if (ret)
-		goto unreg_algs;
+		goto err_unreg_skciphers;
 
-unlock:
 	mutex_unlock(&algs_lock);
-	return ret;
+	return 0;
 
-unreg_algs:
+err_unreg_skciphers:
 	crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
-	goto unlock;
+err_dec:
+	active_devs--;
+unlock:
+	mutex_unlock(&algs_lock);
+	return ret;
 }
 
 void qat_algs_unregister(void)
diff --git a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c
index 75c15c8e41dba7..77d59e977472df 100644
--- a/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/intel/qat/qat_common/qat_asym_algs.c
@@ -535,8 +535,10 @@ static int qat_dh_init_tfm(struct crypto_kpp *tfm)
 		return -EINVAL;
 
 	ctx->ftfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
-	if (IS_ERR(ctx->ftfm))
+	if (IS_ERR(ctx->ftfm)) {
+		qat_crypto_put_instance(inst);
 		return PTR_ERR(ctx->ftfm);
+	}
 
 	crypto_kpp_set_flags(ctx->ftfm, crypto_kpp_get_flags(tfm));
 
@@ -987,7 +989,7 @@ static int qat_rsa_set_n(struct qat_rsa_ctx *ctx, const char *value,
 	const char *ptr = value;
 	int ret;
 
-	while (!*ptr && vlen) {
+	while (vlen && !*ptr) {
 		ptr++;
 		vlen--;
 	}
@@ -1018,7 +1020,7 @@ static int qat_rsa_set_e(struct qat_rsa_ctx *ctx, const char *value,
 	struct device *dev = &GET_DEV(inst->accel_dev);
 	const char *ptr = value;
 
-	while (!*ptr && vlen) {
+	while (vlen && !*ptr) {
 		ptr++;
 		vlen--;
 	}
@@ -1044,7 +1046,7 @@ static int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value,
 	const char *ptr = value;
 	int ret;
 
-	while (!*ptr && vlen) {
+	while (vlen && !*ptr) {
 		ptr++;
 		vlen--;
 	}
@@ -1067,7 +1069,7 @@ static int qat_rsa_set_d(struct qat_rsa_ctx *ctx, const char *value,
 
 static void qat_rsa_drop_leading_zeros(const char **ptr, unsigned int *len)
 {
-	while (!**ptr && *len) {
+	while (*len && !**ptr) {
 		(*ptr)++;
 		(*len)--;
 	}
@@ -1338,13 +1340,25 @@ int qat_asym_algs_register(void)
 	int ret = 0;
 
 	mutex_lock(&algs_lock);
-	if (++active_devs == 1) {
-		rsa.base.cra_flags = 0;
-		ret = crypto_register_akcipher(&rsa);
-		if (ret)
-			goto unlock;
-		ret = crypto_register_kpp(&dh);
-	}
+	if (++active_devs != 1)
+		goto unlock;
+
+	rsa.base.cra_flags = 0;
+	ret = crypto_register_akcipher(&rsa);
+	if (ret)
+		goto err_dec;
+
+	ret = crypto_register_kpp(&dh);
+	if (ret)
+		goto err_unreg_akcipher;
+
+	mutex_unlock(&algs_lock);
+	return 0;
+
+err_unreg_akcipher:
+	crypto_unregister_akcipher(&rsa);
+err_dec:
+	active_devs--;
 unlock:
 	mutex_unlock(&algs_lock);
 	return ret;
diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c
index 096be42e9d0319..70befbdd442b83 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c
@@ -16,6 +16,7 @@
 #include <crypto/sha1.h>
 #include <crypto/sha2.h>
 #include <crypto/xts.h>
+#include <crypto/utils.h>
 #include <crypto/scatterwalk.h>
 #include <linux/sort.h>
 #include <linux/module.h>
@@ -105,9 +106,9 @@ static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req)
 	req = container_of(cpt_req->areq, struct aead_request, base);
 	tfm = crypto_aead_reqtfm(req);
 	rctx = aead_request_ctx_dma(req);
-	if (memcmp(rctx->fctx.hmac.s.hmac_calc,
-		   rctx->fctx.hmac.s.hmac_recv,
-		   crypto_aead_authsize(tfm)) != 0)
+	if (crypto_memneq(rctx->fctx.hmac.s.hmac_calc,
+			  rctx->fctx.hmac.s.hmac_recv,
+			  crypto_aead_authsize(tfm)))
 		return -EBADMSG;
 
 	return 0;
diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c
index 90fdafb7c4685b..38d1de24652b41 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c
@@ -180,7 +180,7 @@ int otx_cptvf_send_vq_size_msg(struct otx_cptvf *cptvf)
 }
 
 /*
- * Communicate VF group required to PF and get the VQ binded to that group
+ * Communicate VF group required to PF and get the VQ bound to that group
  */
 int otx_cptvf_send_vf_to_grp_msg(struct otx_cptvf *cptvf, int group)
 {
@@ -199,7 +199,7 @@ int otx_cptvf_send_vf_to_grp_msg(struct otx_cptvf *cptvf, int group)
 }
 
 /*
- * Communicate VF group required to PF and get the VQ binded to that group
+ * Communicate VF group required to PF and get the VQ bound to that group
  */
 int otx_cptvf_send_vf_priority_msg(struct otx_cptvf *cptvf)
 {
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
index 8d9f394d6b50ea..3ccf17d031c68c 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
@@ -11,6 +11,7 @@
 #include <crypto/xts.h>
 #include <crypto/gcm.h>
 #include <crypto/scatterwalk.h>
+#include <crypto/utils.h>
 #include <linux/sort.h>
 #include <linux/module.h>
 #include "otx2_cptvf.h"
@@ -90,9 +91,9 @@ static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
 	req = container_of(cpt_req->areq, struct aead_request, base);
 	tfm = crypto_aead_reqtfm(req);
 	rctx = aead_request_ctx_dma(req);
-	if (memcmp(rctx->fctx.hmac.s.hmac_calc,
-		   rctx->fctx.hmac.s.hmac_recv,
-		   crypto_aead_authsize(tfm)) != 0)
+	if (crypto_memneq(rctx->fctx.hmac.s.hmac_calc,
+			  rctx->fctx.hmac.s.hmac_recv,
+			  crypto_aead_authsize(tfm)))
 		return -EBADMSG;
 
 	return 0;
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index c75a069997876b..d104d958cb61ae 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -473,6 +473,9 @@ static int mxs_dcp_aes_enqueue(struct skcipher_request *req, int enc, int ecb)
 	struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req);
 	int ret;
 
+	if (!req->cryptlen)
+		return 0;
+
 	if (unlikely(actx->key_len != AES_KEYSIZE_128 && !actx->key_referenced))
 		return mxs_dcp_block_fallback(req, enc);
 
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index 3b7104cb4487bd..4b5a8d13f505e7 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -1082,6 +1082,8 @@ static void omap_des_remove(struct platform_device *pdev)
 
 	omap_des_unregister_algs(dd->pdata);
 
+	crypto_engine_exit(dd->engine);
+
 	cancel_work_sync(&dd->done_task);
 	omap_des_dma_cleanup(dd);
 	pm_runtime_disable(dd->dev);
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 6d2dd0b88f1519..5e739415a0102a 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -1265,7 +1265,7 @@ static int omap_sham_finup(struct ahash_request *req)
 		return err1;
 	/*
 	 * final() has to be always called to cleanup resources
-	 * even if udpate() failed, except EINPROGRESS
+	 * even if update() failed, except EINPROGRESS
 	 */
 	err2 = omap_sham_final(req);
 
@@ -1732,7 +1732,7 @@ static void omap_sham_done_task(struct work_struct *t)
 
 finish:
 	dev_dbg(dd->dev, "update done: err: %d\n", err);
-	/* finish curent request */
+	/* finish current request */
 	omap_sham_finish_req(dd->req, err);
 }
 
@@ -2129,6 +2129,7 @@ static void omap_sham_remove(struct platform_device *pdev)
 	list_del(&dd->list);
 	spin_unlock_bh(&sham.lock);
 	omap_sham_unregister_algs(dd->pdata);
+	crypto_engine_exit(dd->engine);
 	cancel_work_sync(&dd->done_task);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index 1be549a07a2197..400a889e924d76 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -109,7 +109,7 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 {
 	struct aes_ctx *ctx = aes_ctx(tfm);
 	const __le32 *key = (const __le32 *)in_key;
-	struct crypto_aes_ctx gen_aes;
+	struct crypto_aes_ctx gen_aes __cleanup(aes_zeroize_ctx);
 	int cpu;
 
 	if (key_len % 8)
@@ -137,20 +137,18 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 	ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
 
 	/* Don't generate extended keys if the hardware can do it. */
-	if (aes_hw_extkey_available(key_len))
-		goto ok;
+	if (!aes_hw_extkey_available(key_len)) {
+		ctx->D = ctx->d_data;
+		ctx->cword.encrypt.keygen = 1;
+		ctx->cword.decrypt.keygen = 1;
 
-	ctx->D = ctx->d_data;
-	ctx->cword.encrypt.keygen = 1;
-	ctx->cword.decrypt.keygen = 1;
+		if (aes_expandkey(&gen_aes, in_key, key_len))
+			return -EINVAL;
 
-	if (aes_expandkey(&gen_aes, in_key, key_len))
-		return -EINVAL;
-
-	memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
-	memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
+		memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
+		memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
+	}
 
-ok:
 	for_each_online_cpu(cpu)
 		if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) ||
 		    &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu))
diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c
index a9642c67f47614..e0061bfab21686 100644
--- a/drivers/crypto/qce/aead.c
+++ b/drivers/crypto/qce/aead.c
@@ -379,7 +379,7 @@ static int qce_aead_create_ccm_nonce(struct qce_aead_reqctx *rctx, struct qce_ae
 
 	/*
 	 * The crypto framework encodes cryptlen as unsigned int. Thus, even though
-	 * spec allows for upto 8 bytes to encode msg_len only 4 bytes are needed.
+	 * spec allows for up to 8 bytes to encode msg_len only 4 bytes are needed.
 	 */
 	if (msglen_size > 4)
 		msglen_size = 4;
@@ -769,14 +769,13 @@ static int qce_aead_register(struct qce_device *qce)
 
 	for (i = 0; i < ARRAY_SIZE(aead_def); i++) {
 		ret = qce_aead_register_one(&aead_def[i], qce);
-		if (ret)
-			goto err;
+		if (ret) {
+			qce_aead_unregister(qce);
+			return ret;
+		}
 	}
 
 	return 0;
-err:
-	qce_aead_unregister(qce);
-	return ret;
 }
 
 const struct qce_algo_ops aead_ops = {
diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c
index eb8a5b0c4173f2..b67d749db05f2b 100644
--- a/drivers/crypto/qce/sha.c
+++ b/drivers/crypto/qce/sha.c
@@ -480,14 +480,13 @@ static int qce_ahash_register(struct qce_device *qce)
 
 	for (i = 0; i < ARRAY_SIZE(ahash_def); i++) {
 		ret = qce_ahash_register_one(&ahash_def[i], qce);
-		if (ret)
-			goto err;
+		if (ret) {
+			qce_ahash_unregister(qce);
+			return ret;
+		}
 	}
 
 	return 0;
-err:
-	qce_ahash_unregister(qce);
-	return ret;
 }
 
 const struct qce_algo_ops ahash_ops = {
diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c
index 4931b064f10e37..03a3cec2bb90e8 100644
--- a/drivers/crypto/qce/skcipher.c
+++ b/drivers/crypto/qce/skcipher.c
@@ -416,14 +416,13 @@ static int qce_skcipher_register(struct qce_device *qce)
 
 	for (i = 0; i < ARRAY_SIZE(skcipher_def); i++) {
 		ret = qce_skcipher_register_one(&skcipher_def[i], qce);
-		if (ret)
-			goto err;
+		if (ret) {
+			qce_skcipher_unregister(qce);
+			return ret;
+		}
 	}
 
 	return 0;
-err:
-	qce_skcipher_unregister(qce);
-	return ret;
 }
 
 const struct qce_algo_ops skcipher_ops = {
diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c
index 7cb1316b6f189f..b2b629fffc056b 100644
--- a/drivers/crypto/rockchip/rk3288_crypto.c
+++ b/drivers/crypto/rockchip/rk3288_crypto.c
@@ -163,7 +163,7 @@ static void rk_crypto_pm_exit(struct rk_crypto_info *rkdev)
 
 static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id)
 {
-	struct rk_crypto_info *dev  = platform_get_drvdata(dev_id);
+	struct rk_crypto_info *dev = dev_id;
 	u32 interrupt_status;
 
 	interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS);
@@ -363,7 +363,7 @@ static int rk_crypto_probe(struct platform_device *pdev)
 
 	err = devm_request_irq(&pdev->dev, crypto_info->irq,
 			       rk_crypto_irq_handle, IRQF_SHARED,
-			       "rk-crypto", pdev);
+			       "rk-crypto", crypto_info);
 
 	if (err)
 		goto err_crypto;
diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c
index bdda7b39af85df..a70102799223d5 100644
--- a/drivers/crypto/s5p-sss.c
+++ b/drivers/crypto/s5p-sss.c
@@ -638,8 +638,7 @@ static int s5p_hash_rx(struct s5p_aes_dev *dev)
 
 static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
 {
-	struct platform_device *pdev = dev_id;
-	struct s5p_aes_dev *dev = platform_get_drvdata(pdev);
+	struct s5p_aes_dev *dev = dev_id;
 	struct skcipher_request *req;
 	int err_dma_tx = 0;
 	int err_dma_rx = 0;
@@ -1446,7 +1445,7 @@ static void s5p_hash_tasklet_cb(unsigned long data)
 	return;
 
 finish:
-	/* finish curent request */
+	/* finish current request */
 	s5p_hash_finish_req(dd->hash_req, 0);
 
 	/* If we are not busy, process next req */
@@ -2221,7 +2220,7 @@ static int s5p_aes_probe(struct platform_device *pdev)
 	}
 	err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL,
 					s5p_aes_interrupt, IRQF_ONESHOT,
-					pdev->name, pdev);
+					pdev->name, pdata);
 	if (err < 0) {
 		dev_warn(dev, "feed control interrupt is not available.\n");
 		goto err_irq;
diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c
index 9846cbeb3449ca..0dce2b98886a8c 100644
--- a/drivers/crypto/sa2ul.c
+++ b/drivers/crypto/sa2ul.c
@@ -466,7 +466,7 @@ static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
 /* Derive the inverse key used in AES-CBC decryption operation */
 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
 {
-	struct crypto_aes_ctx ctx;
+	struct crypto_aes_ctx ctx __cleanup(aes_zeroize_ctx);
 	int key_pos;
 
 	if (aes_expandkey(&ctx, key, key_sz)) {
diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c
index 842f76b0f11484..856290347b95bd 100644
--- a/drivers/crypto/starfive/jh7110-cryp.c
+++ b/drivers/crypto/starfive/jh7110-cryp.c
@@ -120,9 +120,17 @@ static int starfive_cryp_probe(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, PTR_ERR(cryp->rst),
 				     "Error getting hardware reset line\n");
 
-	clk_prepare_enable(cryp->hclk);
-	clk_prepare_enable(cryp->ahb);
-	reset_control_deassert(cryp->rst);
+	ret = clk_prepare_enable(cryp->hclk);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(cryp->ahb);
+	if (ret)
+		goto disable_hclk;
+
+	ret = reset_control_deassert(cryp->rst);
+	if (ret)
+		goto disable_ahb;
 
 	spin_lock(&dev_list.lock);
 	list_add(&cryp->list, &dev_list.dev_list);
@@ -170,9 +178,11 @@ static int starfive_cryp_probe(struct platform_device *pdev)
 	list_del(&cryp->list);
 	spin_unlock(&dev_list.lock);
 
-	clk_disable_unprepare(cryp->hclk);
-	clk_disable_unprepare(cryp->ahb);
 	reset_control_assert(cryp->rst);
+disable_ahb:
+	clk_disable_unprepare(cryp->ahb);
+disable_hclk:
+	clk_disable_unprepare(cryp->hclk);
 
 	return ret;
 }
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index fa96b6bf17e327..41a87d7c30a038 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -751,8 +751,7 @@ static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)	       \
 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)	       \
 {									       \
-	struct device *dev = data;					       \
-	struct talitos_private *priv = dev_get_drvdata(dev);		       \
+	struct talitos_private *priv = data;				       \
 	u32 isr, isr_lo;						       \
 	unsigned long flags;						       \
 									       \
@@ -765,7 +764,7 @@ static irqreturn_t talitos1_interrupt_##name(int irq, void *data)	       \
 									       \
 	if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
 		spin_unlock_irqrestore(&priv->reg_lock, flags);		       \
-		talitos_error(dev, isr & ch_err_mask, isr_lo);		       \
+		talitos_error(priv->dev, isr & ch_err_mask, isr_lo);	       \
 	}								       \
 	else {								       \
 		if (likely(isr & ch_done_mask)) {			       \
@@ -786,8 +785,7 @@ DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)	       \
 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)	       \
 {									       \
-	struct device *dev = data;					       \
-	struct talitos_private *priv = dev_get_drvdata(dev);		       \
+	struct talitos_private *priv = data;				       \
 	u32 isr, isr_lo;						       \
 	unsigned long flags;						       \
 									       \
@@ -800,7 +798,7 @@ static irqreturn_t talitos2_interrupt_##name(int irq, void *data)	       \
 									       \
 	if (unlikely(isr & ch_err_mask || isr_lo)) {			       \
 		spin_unlock_irqrestore(&priv->reg_lock, flags);		       \
-		talitos_error(dev, isr & ch_err_mask, isr_lo);		       \
+		talitos_error(priv->dev, isr & ch_err_mask, isr_lo);	       \
 	}								       \
 	else {								       \
 		if (likely(isr & ch_done_mask)) {			       \
@@ -3242,7 +3240,7 @@ static void talitos_remove(struct platform_device *ofdev)
 
 	for (i = 0; i < 2; i++)
 		if (priv->irq[i] > 0)
-			free_irq(priv->irq[i], dev);
+			free_irq(priv->irq[i], priv);
 
 	tasklet_kill(&priv->done_task[0]);
 	if (priv->irq[1] > 0)
@@ -3361,7 +3359,7 @@ static int talitos_probe_irq(struct platform_device *ofdev)
 
 	if (is_sec1) {
 		err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
-				  dev_driver_string(dev), dev);
+				  dev_driver_string(dev), priv);
 		goto primary_out;
 	}
 
@@ -3372,18 +3370,18 @@ static int talitos_probe_irq(struct platform_device *ofdev)
 	/* get the primary irq line */
 	if (priv->irq[1] < 0) {
 		err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
-				  dev_driver_string(dev), dev);
+				  dev_driver_string(dev), priv);
 		goto primary_out;
 	}
 
 	err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
-			  dev_driver_string(dev), dev);
+			  dev_driver_string(dev), priv);
 	if (err)
 		goto primary_out;
 
 	/* get the secondary irq line */
 	err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
-			  dev_driver_string(dev), dev);
+			  dev_driver_string(dev), priv);
 	if (err) {
 		dev_err(dev, "failed to request secondary irq\n");
 		priv->irq[1] = 0;
@@ -3422,6 +3420,7 @@ static int talitos_probe(struct platform_device *ofdev)
 
 	dev_set_drvdata(dev, priv);
 
+	priv->dev = dev;
 	priv->ofdev = ofdev;
 
 	spin_lock_init(&priv->reg_lock);
diff --git a/drivers/crypto/xilinx/zynqmp-aes-gcm.c b/drivers/crypto/xilinx/zynqmp-aes-gcm.c
index d54c268dfe34a7..d9bd6dcd23a6da 100644
--- a/drivers/crypto/xilinx/zynqmp-aes-gcm.c
+++ b/drivers/crypto/xilinx/zynqmp-aes-gcm.c
@@ -230,10 +230,8 @@ static int zynqmp_aes_aead_cipher(struct aead_request *req)
 	}
 
 freemem:
-	memzero_explicit(kbuf, dma_size);
-	kfree(kbuf);
-	memzero_explicit(dmabuf, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE);
-	kfree(dmabuf);
+	kfree_sensitive(kbuf);
+	kfree_sensitive(dmabuf);
 
 	return ret;
 }
@@ -364,11 +362,9 @@ static int versal_aes_aead_cipher(struct aead_request *req)
 	if (unlikely(dma_addr_hw_req))
 		dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL);
 buf2_free:
-	memzero_explicit(dmabuf, dmabuf_size);
-	kfree(dmabuf);
+	kfree_sensitive(dmabuf);
 buf1_free:
-	memzero_explicit(kbuf, kbuf_size);
-	kfree(kbuf);
+	kfree_sensitive(kbuf);
 err:
 	return ret;
 }
@@ -766,7 +762,7 @@ static void xilinx_aes_aead_exit(struct crypto_aead *aead)
 	struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
 
 	dma_unmap_single(tfm_ctx->dev, tfm_ctx->key_dma_addr, AES_KEYSIZE_256, DMA_TO_DEVICE);
-	kfree(tfm_ctx->key);
+	kfree_sensitive(tfm_ctx->key);
 	if (tfm_ctx->fbk_cipher) {
 		crypto_free_aead(tfm_ctx->fbk_cipher);
 		tfm_ctx->fbk_cipher = NULL;
diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c
index 5813017b6b79d2..cd051e1f427535 100644
--- a/drivers/crypto/xilinx/zynqmp-sha.c
+++ b/drivers/crypto/xilinx/zynqmp-sha.c
@@ -59,7 +59,7 @@ static int zynqmp_sha_init_tfm(struct crypto_shash *hash)
 		return PTR_ERR(fallback_tfm);
 
 	if (crypto_shash_descsize(hash) <
-	    crypto_shash_statesize(tfm_ctx->fbk_tfm)) {
+	    crypto_shash_statesize(fallback_tfm)) {
 		crypto_free_shash(fallback_tfm);
 		return -EINVAL;
 	}
diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c
index 3b818adbd38b59..fb09a5ff48c160 100644
--- a/drivers/cxl/acpi.c
+++ b/drivers/cxl/acpi.c
@@ -885,13 +885,17 @@ static int pair_cxl_resource(struct device *dev, void *data)
 
 static int cxl_acpi_probe(struct platform_device *pdev)
 {
-	int rc;
+	struct cxl_cfmws_context ctx;
+	struct acpi_device *adev;
 	struct resource *cxl_res;
 	struct cxl_root *cxl_root;
 	struct cxl_port *root_port;
 	struct device *host = &pdev->dev;
-	struct acpi_device *adev = ACPI_COMPANION(host);
-	struct cxl_cfmws_context ctx;
+	int rc;
+
+	adev = ACPI_COMPANION(host);
+	if (!adev)
+		return -ENODEV;
 
 	device_lock_set_class(&pdev->dev, &cxl_root_key);
 	rc = devm_add_action_or_reset(&pdev->dev, cxl_acpi_lock_reset_class,
diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c
index 5c9f0726251351..df9261ba84e5a5 100644
--- a/drivers/cxl/core/cdat.c
+++ b/drivers/cxl/core/cdat.c
@@ -633,9 +633,9 @@ static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr,
 	struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent);
 	struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent);
 	struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
-	struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX];
-	struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX];
-	struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX];
+	struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX] = { };
+	struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX] = { };
+	struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX] = { };
 	struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 	struct cxl_dev_state *cxlds = cxlmd->cxlds;
 	struct pci_dev *pdev = to_pci_dev(cxlds->dev);
@@ -757,7 +757,7 @@ static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr,
 {
 	struct xarray *res_xa __free(free_perf_xa) =
 		kzalloc_obj(*res_xa);
-	struct access_coordinate coords[ACCESS_COORDINATE_MAX];
+	struct access_coordinate coords[ACCESS_COORDINATE_MAX] = { };
 	struct cxl_perf_ctx *ctx, *us_ctx;
 	unsigned long index, us_index;
 	int dev_count = 0;
diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 35eaf636adc9c0..bb380ec6daeb04 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -217,10 +217,10 @@ int cxl_port_get_possible_dports(struct cxl_port *port);
 #ifdef CONFIG_CXL_FEATURES
 struct cxl_feat_entry *
 cxl_feature_info(struct cxl_features_state *cxlfs, const uuid_t *uuid);
-size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
-		       enum cxl_get_feat_selection selection,
-		       void *feat_out, size_t feat_out_size, u16 offset,
-		       u16 *return_code);
+ssize_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
+			enum cxl_get_feat_selection selection,
+			void *feat_out, size_t feat_out_size, u16 offset,
+			u16 *return_code);
 int cxl_set_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
 		    u8 feat_version, const void *feat_data,
 		    size_t feat_data_size, u32 feat_flag, u16 offset,
diff --git a/drivers/cxl/core/edac.c b/drivers/cxl/core/edac.c
index b321971fef5828..34b81e8dfbbb6f 100644
--- a/drivers/cxl/core/edac.c
+++ b/drivers/cxl/core/edac.c
@@ -78,7 +78,7 @@ static int cxl_mem_scrub_get_attrbs(struct cxl_mailbox *cxl_mbox, u8 *cap,
 				    u16 *cycle, u8 *flags, u8 *min_cycle)
 {
 	size_t rd_data_size = sizeof(struct cxl_scrub_rd_attrbs);
-	size_t data_size;
+	ssize_t data_size;
 	struct cxl_scrub_rd_attrbs *rd_attrbs __free(kfree) =
 		kzalloc(rd_data_size, GFP_KERNEL);
 	if (!rd_attrbs)
@@ -87,7 +87,9 @@ static int cxl_mem_scrub_get_attrbs(struct cxl_mailbox *cxl_mbox, u8 *cap,
 	data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_PATROL_SCRUB_UUID,
 				    CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
 				    rd_data_size, 0, NULL);
-	if (!data_size)
+	if (data_size < 0)
+		return data_size;
+	if ((size_t)data_size != rd_data_size)
 		return -EIO;
 
 	*cap = rd_attrbs->scrub_cycle_cap;
@@ -551,7 +553,7 @@ static int cxl_mem_ecs_get_attrbs(struct device *dev,
 	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
 	struct cxl_ecs_fru_rd_attrbs *fru_rd_attrbs;
 	size_t rd_data_size;
-	size_t data_size;
+	ssize_t data_size;
 
 	rd_data_size = cxl_ecs_ctx->get_feat_size;
 
@@ -563,7 +565,9 @@ static int cxl_mem_ecs_get_attrbs(struct device *dev,
 	data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID,
 				    CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
 				    rd_data_size, 0, NULL);
-	if (!data_size)
+	if (data_size < 0)
+		return data_size;
+	if ((size_t)data_size != rd_data_size)
 		return -EIO;
 
 	fru_rd_attrbs = rd_attrbs->fru_attrbs;
@@ -583,7 +587,7 @@ static int cxl_mem_ecs_set_attrbs(struct device *dev,
 	struct cxl_ecs_fru_wr_attrbs *fru_wr_attrbs;
 	size_t rd_data_size, wr_data_size;
 	u16 num_media_frus, count;
-	size_t data_size;
+	ssize_t data_size;
 
 	num_media_frus = cxl_ecs_ctx->num_media_frus;
 	rd_data_size = cxl_ecs_ctx->get_feat_size;
@@ -596,7 +600,9 @@ static int cxl_mem_ecs_set_attrbs(struct device *dev,
 	data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID,
 				    CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
 				    rd_data_size, 0, NULL);
-	if (!data_size)
+	if (data_size < 0)
+		return data_size;
+	if ((size_t)data_size != rd_data_size)
 		return -EIO;
 
 	struct cxl_ecs_wr_attrbs *wr_attrbs __free(kvfree) =
@@ -1264,7 +1270,7 @@ cxl_mem_sparing_get_attrbs(struct cxl_mem_sparing_context *cxl_sparing_ctx)
 	struct cxl_memdev *cxlmd = cxl_sparing_ctx->cxlmd;
 	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
 	u16 restriction_flags;
-	size_t data_size;
+	ssize_t data_size;
 	u16 return_code;
 	struct cxl_memdev_sparing_rd_attrbs *rd_attrbs __free(kfree) =
 		kzalloc(rd_data_size, GFP_KERNEL);
@@ -1274,7 +1280,9 @@ cxl_mem_sparing_get_attrbs(struct cxl_mem_sparing_context *cxl_sparing_ctx)
 	data_size = cxl_get_feature(cxl_mbox, &cxl_sparing_ctx->repair_uuid,
 				    CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
 				    rd_data_size, 0, &return_code);
-	if (!data_size)
+	if (data_size < 0)
+		return data_size;
+	if ((size_t)data_size != rd_data_size)
 		return -EIO;
 
 	cxl_sparing_ctx->op_class = rd_attrbs->hdr.op_class;
@@ -1750,7 +1758,7 @@ static int cxl_mem_ppr_get_attrbs(struct cxl_ppr_context *cxl_ppr_ctx)
 	struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
 	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
 	u16 restriction_flags;
-	size_t data_size;
+	ssize_t data_size;
 	u16 return_code;
 
 	struct cxl_memdev_ppr_rd_attrbs *rd_attrbs __free(kfree) =
@@ -1761,7 +1769,9 @@ static int cxl_mem_ppr_get_attrbs(struct cxl_ppr_context *cxl_ppr_ctx)
 	data_size = cxl_get_feature(cxl_mbox, &cxl_ppr_ctx->repair_uuid,
 				    CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
 				    rd_data_size, 0, &return_code);
-	if (!data_size)
+	if (data_size < 0)
+		return data_size;
+	if ((size_t)data_size != rd_data_size)
 		return -EIO;
 
 	cxl_ppr_ctx->op_class = rd_attrbs->hdr.op_class;
diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c
index ba6d2a5acb74ad..8024c2dcae1d11 100644
--- a/drivers/cxl/core/features.c
+++ b/drivers/cxl/core/features.c
@@ -97,6 +97,7 @@ get_supported_features(struct cxl_features_state *cxlfs)
 		kvmalloc_flex(*entries, ent, count);
 	if (!entries)
 		return NULL;
+	entries->num_features = count;
 
 	struct cxl_mbox_get_sup_feats_out *mbox_out __free(kvfree) =
 		kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
@@ -174,7 +175,6 @@ get_supported_features(struct cxl_features_state *cxlfs)
 		start += num_entries;
 	} while (remain_feats);
 
-	entries->num_features = count;
 	entries->num_user_features = user_feats;
 
 	return no_free_ptr(entries);
@@ -220,10 +220,10 @@ int devm_cxl_setup_features(struct cxl_dev_state *cxlds)
 }
 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_features, "CXL");
 
-size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
-		       enum cxl_get_feat_selection selection,
-		       void *feat_out, size_t feat_out_size, u16 offset,
-		       u16 *return_code)
+ssize_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
+			enum cxl_get_feat_selection selection,
+			void *feat_out, size_t feat_out_size, u16 offset,
+			u16 *return_code)
 {
 	size_t data_to_rd_size;
 	struct cxl_mbox_get_feat_in pi;
@@ -232,10 +232,13 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
 	int rc;
 
 	if (return_code)
-		*return_code = CXL_MBOX_CMD_RC_INPUT;
+		*return_code = CXL_MBOX_CMD_RC_SUCCESS;
 
 	if (!feat_out || !feat_out_size)
-		return 0;
+		return -EINVAL;
+
+	if (feat_out_size > U16_MAX - offset)
+		return -EINVAL;
 
 	uuid_copy(&pi.uuid, feat_uuid);
 	pi.selection = selection;
@@ -256,17 +259,25 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
 			.min_out = data_to_rd_size,
 		};
 		rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
+		/*
+		 * Per CXL r4.0 8.2.10.6.2, when Offset + Count runs past the
+		 * end of the Feature the device returns only the bytes up to
+		 * the Feature size. cxl_internal_send_cmd() reports that as
+		 * -EIO with a short payload, so stop and return what arrived.
+		 */
+		if (rc == -EIO && mbox_cmd.size_out &&
+		    mbox_cmd.size_out < data_to_rd_size) {
+			data_rcvd_size += mbox_cmd.size_out;
+			break;
+		}
 		if (rc < 0 || !mbox_cmd.size_out) {
 			if (return_code)
 				*return_code = mbox_cmd.return_code;
-			return 0;
+			return rc < 0 ? rc : -EIO;
 		}
 		data_rcvd_size += mbox_cmd.size_out;
 	} while (data_rcvd_size < feat_out_size);
 
-	if (return_code)
-		*return_code = CXL_MBOX_CMD_RC_SUCCESS;
-
 	return data_rcvd_size;
 }
 
@@ -286,7 +297,10 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox,
 	size_t hdr_size;
 
 	if (return_code)
-		*return_code = CXL_MBOX_CMD_RC_INPUT;
+		*return_code = CXL_MBOX_CMD_RC_SUCCESS;
+
+	if (feat_data_size > U16_MAX - offset)
+		return -EINVAL;
 
 	struct cxl_mbox_set_feat_in *pi __free(kfree) =
 			kzalloc(cxl_mbox->payload_size, GFP_KERNEL);
@@ -334,11 +348,8 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox,
 		}
 
 		data_sent_size += data_in_size;
-		if (data_sent_size >= feat_data_size) {
-			if (return_code)
-				*return_code = CXL_MBOX_CMD_RC_SUCCESS;
+		if (data_sent_size >= feat_data_size)
 			return 0;
-		}
 
 		if ((feat_data_size - data_sent_size) <= (cxl_mbox->payload_size - hdr_size)) {
 			data_in_size = feat_data_size - data_sent_size;
@@ -462,6 +473,7 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs,
 	const struct cxl_mbox_get_feat_in *feat_in;
 	u16 offset, count, return_code;
 	size_t out_size = *out_len;
+	ssize_t data_size;
 
 	if (rpc_in->op_size != sizeof(*feat_in))
 		return ERR_PTR(-EINVAL);
@@ -482,16 +494,20 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs,
 	if (!rpc_out)
 		return ERR_PTR(-ENOMEM);
 
-	out_size = cxl_get_feature(cxl_mbox, &feat_in->uuid,
-				   feat_in->selection, rpc_out->payload,
-				   count, offset, &return_code);
+	data_size = cxl_get_feature(cxl_mbox, &feat_in->uuid,
+				    feat_in->selection, rpc_out->payload,
+				    count, offset, &return_code);
+	if (data_size <= 0 &&
+	    return_code == CXL_MBOX_CMD_RC_SUCCESS)
+		return ERR_PTR(data_size ?: -EIO);
 	*out_len = sizeof(struct fwctl_rpc_cxl_out);
-	if (!out_size) {
+	if (data_size <= 0) {
 		rpc_out->size = 0;
 		rpc_out->retval = return_code;
 		return no_free_ptr(rpc_out);
 	}
 
+	out_size = data_size;
 	rpc_out->size = out_size;
 	rpc_out->retval = CXL_MBOX_CMD_RC_SUCCESS;
 	*out_len += out_size;
@@ -536,6 +552,8 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs,
 	rc = cxl_set_feature(cxl_mbox, &feat_in->uuid,
 			     feat_in->version, feat_in->feat_data,
 			     data_size, flags, offset, &return_code);
+	if (rc && return_code == CXL_MBOX_CMD_RC_SUCCESS)
+		return ERR_PTR(rc);
 	*out_len = sizeof(*rpc_out);
 	if (rc) {
 		rpc_out->retval = return_code;
diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index 0c80b76a5f9b4b..c51b2614aa39c6 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
 #include <linux/seq_file.h>
+#include <linux/unaligned.h>
 #include <linux/device.h>
 #include <linux/delay.h>
 
@@ -76,6 +77,8 @@ static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
 
 	hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
 	cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
+
+	/* target_count is a direct count (1h..8h), not 0-based like decoder_count */
 	cxlhdm->target_count =
 		FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
 	if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
@@ -240,6 +243,18 @@ static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds,
 }
 #define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL)
 
+static void cxl_dpa_release_region(struct resource *parent,
+				   struct resource *res)
+{
+	/* zero sized decoders are not tracked in the resource tree */
+	if (resource_size(res) == 0) {
+		kfree(res);
+		return;
+	}
+
+	__release_region(parent, res->start, resource_size(res));
+}
+
 /*
  * Must be called in a context that synchronizes against this decoder's
  * port ->remove() callback (like an endpoint decoder sysfs attribute)
@@ -256,7 +271,7 @@ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
 
 	/* save @skip_start, before @res is released */
 	skip_start = res->start - cxled->skip;
-	__release_region(&cxlds->dpa_res, res->start, resource_size(res));
+	cxl_dpa_release_region(&cxlds->dpa_res, res);
 	if (cxled->skip)
 		release_skip(cxlds, skip_start, cxled->skip);
 	cxled->skip = 0;
@@ -336,6 +351,27 @@ static int request_skip(struct cxl_dev_state *cxlds,
 	return -EBUSY;
 }
 
+static struct resource *cxl_dpa_request_region(struct resource *parent,
+					       resource_size_t start,
+					       resource_size_t n,
+					       const char *name)
+{
+	struct resource *res;
+
+	if (!n) {
+		res = kmalloc_obj(*res);
+		if (!res)
+			return ERR_PTR(-ENOMEM);
+
+		*res = DEFINE_RES_NAMED(start, 0, name, IORESOURCE_MEM);
+
+		return res;
+	}
+
+	res = __request_region(parent, start, n, name, 0);
+	return res ?: ERR_PTR(-EBUSY);
+}
+
 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 			     resource_size_t base, resource_size_t len,
 			     resource_size_t skipped)
@@ -349,12 +385,6 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 
 	lockdep_assert_held_write(&cxl_rwsem.dpa);
 
-	if (!len) {
-		dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
-			 port->id, cxled->cxld.id);
-		return -EINVAL;
-	}
-
 	if (cxled->dpa_res) {
 		dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
 			port->id, cxled->cxld.id, cxled->dpa_res);
@@ -378,14 +408,14 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 		if (rc)
 			return rc;
 	}
-	res = __request_region(&cxlds->dpa_res, base, len,
-			       dev_name(&cxled->cxld.dev), 0);
-	if (!res) {
+	res = cxl_dpa_request_region(&cxlds->dpa_res, base, len,
+				     dev_name(&cxled->cxld.dev));
+	if (IS_ERR(res)) {
 		dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
 			port->id, cxled->cxld.id);
 		if (skipped)
 			release_skip(cxlds, base - skipped, skipped);
-		return -EBUSY;
+		return PTR_ERR(res);
 	}
 	cxled->dpa_res = res;
 	cxled->skip = skipped;
@@ -402,7 +432,8 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
 				break;
 			}
 
-	if (cxled->part < 0)
+	/* Empty decoders may not be contained by a partition boundary */
+	if (cxled->part < 0 && resource_size(res))
 		dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n",
 			 port->id, cxled->cxld.id, res);
 
@@ -953,7 +984,8 @@ static int cxl_setup_hdm_decoder_from_dvsec(
 	 * change the range registers at run time.
 	 */
 	cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
-	port->commit_end = cxld->id;
+	scoped_guard(rwsem_write, &cxl_rwsem.region)
+		port->commit_end = cxld->id;
 
 	rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
 	if (rc) {
@@ -973,15 +1005,12 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
 			    u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
 {
 	struct cxl_endpoint_decoder *cxled = NULL;
+	u8 target_id[CXL_HDM_DECODER0_TL_TARGETS];
 	u64 size, base, skip, dpa_size, lo, hi;
 	bool committed;
 	u32 remainder;
 	int i, rc;
 	u32 ctrl;
-	union {
-		u64 value;
-		unsigned char target_id[8];
-	} target_list;
 
 	if (should_emulate_decoders(info))
 		return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
@@ -1031,12 +1060,6 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
 			return -ENXIO;
 		}
 
-		if (size == 0) {
-			dev_warn(&port->dev,
-				 "decoder%d.%d: Committed with zero size\n",
-				 port->id, cxld->id);
-			return -ENXIO;
-		}
 		port->commit_end = cxld->id;
 	} else {
 		if (cxled) {
@@ -1084,11 +1107,22 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
 		cxld->interleave_ways, cxld->interleave_granularity);
 
 	if (!cxled) {
-		lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
-		hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
-		target_list.value = (hi << 32) + lo;
+		struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(&cxld->dev);
+
+		if (cxld->interleave_ways > cxlsd->nr_targets) {
+			dev_err(&port->dev,
+				"decoder%d.%d: interleave ways: %d exceeds targets: %d\n",
+				port->id, cxld->id, cxld->interleave_ways,
+				cxlsd->nr_targets);
+			return -ENXIO;
+		}
+
+		put_unaligned_le32(readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)),
+				   &target_id[0]);
+		put_unaligned_le32(readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)),
+				   &target_id[4]);
 		for (i = 0; i < cxld->interleave_ways; i++)
-			cxld->target_map[i] = target_list.target_id[i];
+			cxld->target_map[i] = target_id[i];
 
 		return 0;
 	}
@@ -1148,6 +1182,14 @@ static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
 		msleep(20);
 }
 
+static void cxl_reset_commit_end(void *data)
+{
+	struct cxl_port *port = data;
+
+	guard(rwsem_write)(&cxl_rwsem.region);
+	port->commit_end = -1;
+}
+
 /**
  * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
  * @cxlhdm: Structure to populate with HDM capabilities
@@ -1158,13 +1200,33 @@ static int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
 {
 	void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 	struct cxl_port *port = cxlhdm->port;
-	int i;
 	u64 dpa_base = 0;
+	int i, rc;
+
+	/*
+	 * Per CXL 4.0 8.2.4.20.1 Target Count is the number of target ports
+	 * per decoder, max 8. Endpoint decoders have no targets.
+	 */
+	if (!is_cxl_endpoint(port) &&
+	    (cxlhdm->target_count < 1 ||
+	     cxlhdm->target_count > CXL_HDM_DECODER0_TL_TARGETS)) {
+		dev_err(&port->dev, "Invalid decoder target count: %u\n",
+			cxlhdm->target_count);
+		return -ENXIO;
+	}
 
 	cxl_settle_decoders(cxlhdm);
 
+	/*
+	 * Reset commit_end after all decoders have been torn down so a
+	 * subsequent probe rebuilds it from scratch.
+	 */
+	rc = devm_add_action(&port->dev, cxl_reset_commit_end, port);
+	if (rc)
+		return rc;
+
 	for (i = 0; i < cxlhdm->decoder_count; i++) {
-		int rc, target_count = cxlhdm->target_count;
+		int target_count = cxlhdm->target_count;
 		struct cxl_decoder *cxld;
 
 		if (is_cxl_endpoint(port)) {
diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c
index 55828a836c01aa..1a255333280110 100644
--- a/drivers/cxl/core/mbox.c
+++ b/drivers/cxl/core/mbox.c
@@ -1386,6 +1386,9 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
 	int nr_records = 0;
 	int rc;
 
+	if (!len)
+		return 0;
+
 	ACQUIRE(mutex_intr, lock)(&mds->poison.mutex);
 	if ((rc = ACQUIRE_ERR(mutex_intr, &lock)))
 		return rc;
diff --git a/drivers/cxl/core/mce.c b/drivers/cxl/core/mce.c
index 65fed913b22179..3ac6802e750d25 100644
--- a/drivers/cxl/core/mce.c
+++ b/drivers/cxl/core/mce.c
@@ -18,7 +18,13 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val,
 	u64 spa, spa_alias;
 	unsigned long pfn;
 
-	if (!mce || !mce_usable_address(mce))
+	if (!mce)
+		return NOTIFY_DONE;
+
+	if (mce_is_correctable(mce))
+		return NOTIFY_DONE;
+
+	if (!mce_usable_address(mce))
 		return NOTIFY_DONE;
 
 	spa = mce->addr & MCI_ADDR_PHYSADDR;
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 9d807c1a002ca9..43b9b7afff2967 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -683,6 +683,15 @@ static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data)
 	if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
 		return 0;
 
+	/*
+	 * Decoders emulated from the DVSEC range registers have no commit
+	 * callback and no HDM decoder registers to consult.
+	 */
+	if (!cxld->commit) {
+		dev_warn(dev, "DVSEC emulated decode may have been cleared by reset\n");
+		return 0;
+	}
+
 	cxlhdm = dev_get_drvdata(&port->dev);
 	hdm = cxlhdm->regs.hdm_decoder;
 	ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 625e4aa427db07..6024bc9c13762f 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -148,20 +148,29 @@ static ssize_t target_type_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(target_type);
 
+/*
+ * Interleave ways selects how many targets a decoder uses, but the target
+ * array is only nr_targets long. Bound array access by both.
+ */
+static int cxlsd_nr_used_targets(struct cxl_switch_decoder *cxlsd)
+{
+	return min(cxlsd->cxld.interleave_ways, cxlsd->nr_targets);
+}
+
 static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
 {
-	struct cxl_decoder *cxld = &cxlsd->cxld;
+	int nr_used = cxlsd_nr_used_targets(cxlsd);
 	ssize_t offset = 0;
 	int i, rc = 0;
 
-	for (i = 0; i < cxld->interleave_ways; i++) {
+	for (i = 0; i < nr_used; i++) {
 		struct cxl_dport *dport = cxlsd->target[i];
 		struct cxl_dport *next = NULL;
 
 		if (!dport)
 			break;
 
-		if (i + 1 < cxld->interleave_ways)
+		if (i + 1 < nr_used)
 			next = cxlsd->target[i + 1];
 		rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
 				   next ? "," : "");
@@ -814,6 +823,11 @@ static int cxl_einj_inject(void *data, u64 type)
 DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject,
 			 "0x%llx\n");
 
+static void remove_debugfs(void *dentry)
+{
+	debugfs_remove_recursive(dentry);
+}
+
 static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
 {
 	struct cxl_port *parent = parent_port_of(dport->port);
@@ -832,6 +846,9 @@ static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
 
 	dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev));
 
+	if (devm_add_action_or_reset(dport_to_host(dport), remove_debugfs, dir))
+		return;
+
 	debugfs_create_file("einj_inject", 0200, dir, dport,
 			    &cxl_einj_inject_fops);
 }
@@ -1611,7 +1628,7 @@ static int update_decoder_targets(struct device *dev, void *data)
 	struct cxl_dport *dport = data;
 	struct cxl_switch_decoder *cxlsd;
 	struct cxl_decoder *cxld;
-	int i;
+	int i, nr_used;
 
 	if (!is_switch_decoder(dev))
 		return 0;
@@ -1619,8 +1636,9 @@ static int update_decoder_targets(struct device *dev, void *data)
 	cxlsd = to_cxl_switch_decoder(dev);
 	cxld = &cxlsd->cxld;
 	guard(rwsem_write)(&cxl_rwsem.region);
+	nr_used = cxlsd_nr_used_targets(cxlsd);
 
-	for (i = 0; i < cxld->interleave_ways; i++) {
+	for (i = 0; i < nr_used; i++) {
 		if (cxld->target_map[i] == dport->port_id) {
 			cxlsd->target[i] = dport;
 			dev_dbg(dev, "dport%d found in target list, index %d\n",
@@ -1918,7 +1936,7 @@ static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
 				    struct cxl_port *port)
 {
 	struct cxl_decoder *cxld = &cxlsd->cxld;
-	int i;
+	int i, nr_used;
 
 	device_lock_assert(&port->dev);
 
@@ -1926,7 +1944,8 @@ static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
 		return 0;
 
 	guard(rwsem_write)(&cxl_rwsem.region);
-	for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
+	nr_used = cxlsd_nr_used_targets(cxlsd);
+	for (i = 0; i < nr_used; i++) {
 		struct cxl_dport *dport = find_dport(port, cxld->target_map[i]);
 
 		if (!dport) {
@@ -1987,7 +2006,7 @@ static int cxl_switch_decoder_init(struct cxl_port *port,
 				   struct cxl_switch_decoder *cxlsd,
 				   int nr_targets)
 {
-	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
+	if (nr_targets < 1 || nr_targets > CXL_DECODER_MAX_INTERLEAVE)
 		return -EINVAL;
 
 	cxlsd->nr_targets = nr_targets;
@@ -2394,8 +2413,10 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
 	 * Skip calculation for RCD. Expectation is HMAT already covers RCD case
 	 * since RCH does not support hotplug.
 	 */
-	if (cxlmd->cxlds->rcd)
+	if (cxlmd->cxlds->rcd) {
+		memset(coord, 0, sizeof(*coord) * ACCESS_COORDINATE_MAX);
 		return 0;
+	}
 
 	/*
 	 * Exit the loop when the parent port of the current iter port is cxl
diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c
index e307361bb39e46..a2c7c9479f2b46 100644
--- a/drivers/cxl/core/ras.c
+++ b/drivers/cxl/core/ras.c
@@ -70,14 +70,14 @@ cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd,
 	trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl);
 }
 
-static int match_memdev_by_parent(struct device *dev, const void *uport)
+static int match_memdev_by_parent(struct device *dev, const void *parent)
 {
-	if (is_cxl_memdev(dev) && dev->parent == uport)
+	if (is_cxl_memdev(dev) && dev->parent == parent)
 		return 1;
 	return 0;
 }
 
-void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data)
+static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data)
 {
 	unsigned int devfn = PCI_DEVFN(data->prot_err.agent_addr.device,
 				       data->prot_err.agent_addr.function);
@@ -108,7 +108,7 @@ void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data)
 		return;
 
 	struct device *mem_dev __free(put_device) = bus_find_device(
-		&cxl_bus_type, NULL, pdev, match_memdev_by_parent);
+		&cxl_bus_type, NULL, &pdev->dev, match_memdev_by_parent);
 	if (!mem_dev)
 		return;
 
@@ -118,7 +118,6 @@ void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data)
 	else
 		cxl_cper_trace_uncorr_prot_err(cxlmd, data->ras_cap);
 }
-EXPORT_SYMBOL_GPL(cxl_cper_handle_prot_err);
 
 static void cxl_cper_prot_err_work_fn(struct work_struct *work)
 {
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 27e63e6dab7c82..a048bd5c06cb64 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
+#include <linux/acpi.h>
 #include <linux/memregion.h>
 #include <linux/genalloc.h>
 #include <linux/debugfs.h>
@@ -1367,7 +1368,8 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled,
 	/*
 	 * If this position wants to share a dport with the last endpoint mapped
 	 * then that endpoint, at index 'position - distance', must also be
-	 * mapped by this dport.
+	 * mapped by this dport. An endpoint that this port does not map at all
+	 * fails that requirement.
 	 */
 	if (pos < distance) {
 		dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
@@ -1378,7 +1380,7 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled,
 	cxled_peer = p->targets[pos - distance];
 	cxlmd_peer = cxled_to_memdev(cxled_peer);
 	ep_peer = cxl_ep_load(port, cxlmd_peer);
-	if (ep->dport != ep_peer->dport) {
+	if (!ep_peer || ep->dport != ep_peer->dport) {
 		dev_dbg(&cxlr->dev,
 			"%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
 			dev_name(port->uport_dev), dev_name(&port->dev),
@@ -2112,7 +2114,7 @@ static int cxl_region_attach(struct cxl_region *cxlr,
 		return -ENXIO;
 	}
 
-	if (!cxled->dpa_res) {
+	if (cxled_empty(cxled)) {
 		dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
 		return -ENXIO;
@@ -2215,10 +2217,11 @@ static int cxl_region_attach(struct cxl_region *cxlr,
 		int test_pos;
 
 		test_pos = cxl_calc_interleave_pos(target, &cxlr->hpa_range);
-		dev_dbg(&target->cxld.dev,
-			"Test cxl_calc_interleave_pos(): %s test_pos:%d target->pos:%d\n",
-			(test_pos == target->pos) ? "success" : "fail",
-			test_pos, target->pos);
+		if (test_pos != target->pos)
+			dev_warn(&cxlr->dev,
+				 "%s: position mismatch: calculated:%d assigned:%d\n",
+				 dev_name(&target->cxld.dev), test_pos,
+				 target->pos);
 	}
 
 	return 0;
@@ -2630,7 +2633,19 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
 
 	for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
 		if (cxlr->coord[i].read_bandwidth) {
-			node_update_perf_attrs(nid, &cxlr->coord[i], i);
+			/*
+			 * A CFMWS-only node (not backed by a real SRAT pxm)
+			 * has no HMAT memory_target, so hmat_callback()
+			 * never created the access attributes for it:
+			 * create them here instead of only updating them.
+			 * This is called at most once per node (see
+			 * nodemask_region_seen), so the attributes are not
+			 * created twice.
+			 */
+			if (!acpi_node_backed_by_real_pxm(nid))
+				node_set_perf_attrs(nid, &cxlr->coord[i], i);
+			else
+				node_update_perf_attrs(nid, &cxlr->coord[i], i);
 			cset++;
 		}
 	}
@@ -2727,7 +2742,7 @@ void kill_regions(struct cxl_root_decoder *cxlrd)
  * This is the second step of region initialization. Regions exist within an
  * address space which is mapped by a @cxlrd.
  *
- * Return: 0 if the region was added to the @cxlrd, else returns negative error
+ * Return: the new region if it was added to the @cxlrd, else an ERR_PTR() with the error
  * code. The region will be named "regionZ" where Z is the unique region number.
  */
 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
@@ -2902,6 +2917,16 @@ struct cxl_poison_context {
 	u64 offset;
 };
 
+/*
+ * A device may answer a Get Poison List request with "physical address
+ * specified is invalid" (-EFAULT). That answer is tolerated for a RAM
+ * partition and the poison walk continues.
+ */
+static inline bool poison_efault_forgiven(int rc, enum cxl_partition_mode mode)
+{
+	return rc == -EFAULT && mode == CXL_PARTMODE_RAM;
+}
+
 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
 				   struct cxl_poison_context *ctx)
 {
@@ -2930,7 +2955,7 @@ static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
 		if (!length)
 			break;
 		rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
-		if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
+		if (poison_efault_forgiven(rc, cxlds->part[i].mode))
 			continue;
 		if (rc)
 			break;
@@ -2947,37 +2972,40 @@ static int poison_by_decoder(struct device *dev, void *arg)
 	struct cxl_dev_state *cxlds;
 	struct cxl_memdev *cxlmd;
 	u64 offset, length;
-	int rc = 0;
+	int rc;
 
 	if (!is_endpoint_decoder(dev))
-		return rc;
+		return 0;
 
 	cxled = to_cxl_endpoint_decoder(dev);
 	if (!cxled->dpa_res)
-		return rc;
+		return 0;
+
+	/*
+	 * Handle the degenerate case of a device with only empty decoders. An
+	 * empty decoder can still map a non-zero skip range, so advance the
+	 * walk to commit_end either way.
+	 */
+	if (cxled->part >= 0) {
+		cxlmd = cxled_to_memdev(cxled);
+		cxlds = cxlmd->cxlds;
+		mode = cxlds->part[cxled->part].mode;
 
-	cxlmd = cxled_to_memdev(cxled);
-	cxlds = cxlmd->cxlds;
-	mode = cxlds->part[cxled->part].mode;
+		if (cxled->skip) {
+			offset = cxled->dpa_res->start - cxled->skip;
+			length = cxled->skip;
+			rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
+			if (rc && !poison_efault_forgiven(rc, mode))
+				return rc;
+		}
 
-	if (cxled->skip) {
-		offset = cxled->dpa_res->start - cxled->skip;
-		length = cxled->skip;
-		rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
-		if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
-			rc = 0;
-		if (rc)
+		offset = cxled->dpa_res->start;
+		length = cxled->dpa_res->end - offset + 1;
+		rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
+		if (rc && !poison_efault_forgiven(rc, mode))
 			return rc;
 	}
 
-	offset = cxled->dpa_res->start;
-	length = cxled->dpa_res->end - offset + 1;
-	rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
-	if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
-		rc = 0;
-	if (rc)
-		return rc;
-
 	/* Iterate until commit_end is reached */
 	if (cxled->cxld.id == ctx->port->commit_end) {
 		ctx->offset = cxled->dpa_res->end + 1;
@@ -2999,9 +3027,17 @@ int cxl_get_poison_by_endpoint(struct cxl_port *port)
 	};
 
 	rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
-	if (rc == 1)
+	if (rc == 1) {
+		/*
+		 * No decoder with a sized DPA reservation was walked
+		 * (every committed decoder is zero-size): scan all
+		 * partitions in full.
+		 */
+		if (ctx.part < 0)
+			ctx.part = 0;
 		rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
 					     &ctx);
+	}
 
 	return rc;
 }
diff --git a/drivers/cxl/core/region_pmem.c b/drivers/cxl/core/region_pmem.c
index 23d97e3d78b6ab..7ab1373a95e01b 100644
--- a/drivers/cxl/core/region_pmem.c
+++ b/drivers/cxl/core/region_pmem.c
@@ -168,12 +168,14 @@ int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
 		dev_name(dev));
 
 	scoped_guard(device, &cxl_nvb->dev) {
-		if (cxl_nvb->dev.driver)
+		if (cxl_nvb->dev.driver) {
 			rc = devm_add_action_or_reset(&cxl_nvb->dev,
 						      cxlr_pmem_unregister,
 						      cxlr_pmem);
-		else
+		} else {
 			rc = -ENXIO;
+			cxlr_pmem_unregister(cxlr_pmem);
+		}
 	}
 
 	if (rc)
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index 20c2d9fbcfe7df..41416fa2ce4a42 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -277,6 +277,20 @@ static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi,
 	u64 offset = ((u64)reg_hi << 32) |
 		     (reg_lo & PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW);
 
+	if (reg_type == CXL_REGLOC_RBI_EMPTY)
+		return false;
+
+	/*
+	 * A BAR the PCI core could not place is reset to zero; decoding it
+	 * would map the block at physical address 0.
+	 */
+	if (!pci_resource_len(pdev, bar) ||
+	    (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)) {
+		dev_warn(&pdev->dev, "BAR%d: not assigned (type: %d)\n", bar,
+			 reg_type);
+		return false;
+	}
+
 	if (offset > pci_resource_len(pdev, bar)) {
 		dev_warn(&pdev->dev,
 			 "BAR%d: %pr: too small (offset: %pa, type: %d)\n", bar,
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index cab8ce39f46533..f03ee0addc4551 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -67,6 +67,8 @@ extern const struct nvdimm_security_ops *cxl_security_ops;
 #define   CXL_HDM_DECODER0_CTRL_HOSTONLY BIT(12)
 #define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
 #define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
+/* Target list capacity, i.e. max interleave ways for a non-endpoint decoder */
+#define CXL_HDM_DECODER0_TL_TARGETS 8
 #define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)
 #define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i)
 
@@ -336,6 +338,16 @@ struct cxl_endpoint_decoder {
 	int pos;
 };
 
+/*
+ * The common case is decoders with no reservation, but also handle
+ * decoders with a zero-sized reservation that firmware may install for
+ * security lockdown purposes.
+ */
+static inline bool cxled_empty(struct cxl_endpoint_decoder *cxled)
+{
+	return !cxled->dpa_res || !resource_size(cxled->dpa_res);
+}
+
 /**
  * struct cxl_switch_decoder - Switch specific CXL HDM Decoder
  * @cxld: base cxl_decoder object
diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index 798e5c369cfc22..3959ec9630264c 100644
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -50,8 +50,13 @@ static int cxl_debugfs_poison_inject(void *data, u64 dpa)
 	struct cxl_memdev *cxlmd = data;
 	int rc;
 
-	ACQUIRE(device_intr, devlock)(&cxlmd->dev);
-	if ((rc = ACQUIRE_ERR(device_intr, &devlock)))
+	/*
+	 * Never wait for this lock: the debugfs proxy holds a file reference
+	 * across the callback and unbind removes the file under the same
+	 * device lock, so waiting here deadlocks against unbind.
+	 */
+	ACQUIRE(device_try, devlock)(&cxlmd->dev);
+	if ((rc = ACQUIRE_ERR(device_try, &devlock)))
 		return rc;
 
 	return cxl_inject_poison(cxlmd, dpa);
@@ -65,8 +70,9 @@ static int cxl_debugfs_poison_clear(void *data, u64 dpa)
 	struct cxl_memdev *cxlmd = data;
 	int rc;
 
-	ACQUIRE(device_intr, devlock)(&cxlmd->dev);
-	if ((rc = ACQUIRE_ERR(device_intr, &devlock)))
+	/* Never wait, per the inject path above. */
+	ACQUIRE(device_try, devlock)(&cxlmd->dev);
+	if ((rc = ACQUIRE_ERR(device_try, &devlock)))
 		return rc;
 
 	return cxl_clear_poison(cxlmd, dpa);
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index c7c91e8dc51dc9..8b560cae91f2e3 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -515,6 +515,7 @@ static irqreturn_t cxl_event_thread(int irq, void *id)
 	struct cxl_dev_id *dev_id = id;
 	struct cxl_dev_state *cxlds = dev_id->cxlds;
 	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
+	bool handled = false;
 	u32 status;
 
 	do {
@@ -527,11 +528,13 @@ static irqreturn_t cxl_event_thread(int irq, void *id)
 		status &= CXLDEV_EVENT_STATUS_ALL;
 		if (!status)
 			break;
+
+		handled = true;
 		cxl_mem_get_event_records(mds, status);
 		cond_resched();
 	} while (status);
 
-	return IRQ_HANDLED;
+	return handled ? IRQ_HANDLED : IRQ_NONE;
 }
 
 static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting)
diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c
index 99cf77b6b6990c..c12fd0b89883a6 100644
--- a/drivers/cxl/port.c
+++ b/drivers/cxl/port.c
@@ -46,6 +46,9 @@ static int discover_region(struct device *dev, void *unused)
 	if (cxled->state != CXL_DECODER_STATE_AUTO)
 		return 0;
 
+	if (cxled_empty(cxled))
+		return 0;
+
 	/*
 	 * Region enumeration is opportunistic, if this add-event fails,
 	 * continue to the next endpoint decoder.
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index 82dd9a43dc6218..f08fc6966eaee3 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -1261,7 +1261,6 @@ void devfreq_resume(void)
 int devfreq_add_governor(struct devfreq_governor *governor)
 {
 	struct devfreq_governor *g;
-	struct devfreq *devfreq;
 	int err = 0;
 
 	if (!governor) {
@@ -1280,38 +1279,6 @@ int devfreq_add_governor(struct devfreq_governor *governor)
 
 	list_add(&governor->node, &devfreq_governor_list);
 
-	list_for_each_entry(devfreq, &devfreq_list, node) {
-		int ret = 0;
-		struct device *dev = devfreq->dev.parent;
-
-		if (!strncmp(devfreq->governor->name, governor->name,
-			     DEVFREQ_NAME_LEN)) {
-			/* The following should never occur */
-			if (devfreq->governor) {
-				dev_warn(dev,
-					 "%s: Governor %s already present\n",
-					 __func__, devfreq->governor->name);
-				ret = devfreq->governor->event_handler(devfreq,
-							DEVFREQ_GOV_STOP, NULL);
-				if (ret) {
-					dev_warn(dev,
-						 "%s: Governor %s stop = %d\n",
-						 __func__,
-						 devfreq->governor->name, ret);
-				}
-				/* Fall through */
-			}
-			devfreq->governor = governor;
-			ret = devfreq->governor->event_handler(devfreq,
-						DEVFREQ_GOV_START, NULL);
-			if (ret) {
-				dev_warn(dev, "%s: Governor %s start=%d\n",
-					 __func__, devfreq->governor->name,
-					 ret);
-			}
-		}
-	}
-
 err_out:
 	mutex_unlock(&devfreq_list_lock);
 
@@ -1410,7 +1377,7 @@ static ssize_t governor_show(struct device *dev,
 	struct devfreq *df = to_devfreq(dev);
 
 	if (!df->governor)
-		return -EINVAL;
+		return -ENOENT;
 
 	return sprintf(buf, "%s\n", df->governor->name);
 }
@@ -1423,9 +1390,6 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 	char str_governor[DEVFREQ_NAME_LEN + 1];
 	const struct devfreq_governor *governor, *prev_governor;
 
-	if (!df->governor)
-		return -EINVAL;
-
 	ret = sscanf(buf, "%" __stringify(DEVFREQ_NAME_LEN) "s", str_governor);
 	if (ret != 1)
 		return -EINVAL;
@@ -1436,6 +1400,9 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 		ret = PTR_ERR(governor);
 		goto out;
 	}
+	if (!df->governor)
+		goto start_new_governor;
+
 	if (df->governor == governor) {
 		ret = 0;
 		goto out;
@@ -1456,6 +1423,7 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 		goto out;
 	}
 
+start_new_governor:
 	/*
 	 * Start the new governor and create the specific sysfs files
 	 * which depend on the new governor.
@@ -1469,6 +1437,9 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 
 		/* Restore previous governor */
 		df->governor = prev_governor;
+		if (!df->governor)
+			goto out;
+
 		ret = df->governor->event_handler(df, DEVFREQ_GOV_START, NULL);
 		if (ret) {
 			dev_err(dev,
@@ -1501,16 +1472,13 @@ static ssize_t available_governors_show(struct device *d,
 	struct devfreq *df = to_devfreq(d);
 	ssize_t count = 0;
 
-	if (!df->governor)
-		return -EINVAL;
-
 	mutex_lock(&devfreq_list_lock);
 
 	/*
 	 * The devfreq with immutable governor (e.g., passive) shows
 	 * only own governor.
 	 */
-	if (IS_SUPPORTED_FLAG(df->governor->flags, IMMUTABLE)) {
+	if (df->governor && IS_SUPPORTED_FLAG(df->governor->flags, IMMUTABLE)) {
 		count = scnprintf(&buf[count], DEVFREQ_NAME_LEN,
 				  "%s ", df->governor->name);
 	/*
diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c
index 255aee1bdd91c7..64014f95fb66dd 100644
--- a/drivers/devfreq/event/rockchip-dfi.c
+++ b/drivers/devfreq/event/rockchip-dfi.c
@@ -185,8 +185,10 @@ static int rockchip_dfi_enable(struct rockchip_dfi *dfi)
 	}
 
 	ret = rockchip_dfi_ddrtype_to_ctrl(dfi, &ctrl);
-	if (ret)
+	if (ret) {
+		clk_disable_unprepare(dfi->clk);
 		goto out;
+	}
 
 	for (i = 0; i < dfi->max_channels; i++) {
 
diff --git a/drivers/dibs/dibs_loopback.c b/drivers/dibs/dibs_loopback.c
index 649e4e375be332..44a2e74c2efcb6 100644
--- a/drivers/dibs/dibs_loopback.c
+++ b/drivers/dibs/dibs_loopback.c
@@ -238,6 +238,7 @@ static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok,
 {
 	struct dibs_lo_dmb_node *rmb_node = NULL, *tmp_node;
 	struct dibs_lo_dev *ldev;
+	unsigned long flags;
 	u16 s_mask;
 	u8 client_id;
 	u32 sba_idx;
@@ -267,12 +268,12 @@ static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok,
 	if (!sf)
 		return 0;
 
-	spin_lock(&dibs->lock);
+	spin_lock_irqsave(&dibs->lock, flags);
 	client_id = dibs->dmb_clientid_arr[sba_idx];
 	s_mask = ror16(0x1000, idx);
 	if (likely(client_id != NO_DIBS_CLIENT && dibs->subs[client_id]))
 		dibs->subs[client_id]->ops->handle_irq(dibs, sba_idx, s_mask);
-	spin_unlock(&dibs->lock);
+	spin_unlock_irqrestore(&dibs->lock, flags);
 
 	return 0;
 }
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 4c9add51f9ef55..36cdc342627ee8 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -260,7 +260,7 @@ static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence)
 
 	/* only support discovering the end of the buffer,
 	 * but also allow SEEK_SET to maintain the idiomatic
-	 * SEEK_END(0), SEEK_CUR(0) pattern.
+	 * SEEK_END(0), SEEK_SET(0) pattern.
 	 */
 	if (whence == SEEK_END)
 		base = dmabuf->size;
diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c
index 39a92d9f241368..2365a6d8ca15f6 100644
--- a/drivers/dma-buf/dma-resv.c
+++ b/drivers/dma-buf/dma-resv.c
@@ -633,7 +633,7 @@ int dma_resv_get_singleton(struct dma_resv *obj, enum dma_resv_usage usage,
 	int r;
 
 	r = dma_resv_get_fences(obj, usage, &count, &fences);
-        if (r)
+	if (r)
 		return r;
 
 	if (count == 0) {
diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c
index e23e5b441a24e9..412f4bce1d4aac 100644
--- a/drivers/dma/altera-msgdma.c
+++ b/drivers/dma/altera-msgdma.c
@@ -847,11 +847,6 @@ static int msgdma_probe(struct platform_device *pdev)
 	if (mdev->irq < 0)
 		return -ENXIO;
 
-	ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler,
-			       0, dev_name(&pdev->dev), mdev);
-	if (ret)
-		return ret;
-
 	tasklet_setup(&mdev->irq_tasklet, msgdma_tasklet);
 
 	dma_cookie_init(&mdev->dmachan);
@@ -863,6 +858,11 @@ static int msgdma_probe(struct platform_device *pdev)
 	INIT_LIST_HEAD(&mdev->done_list);
 	INIT_LIST_HEAD(&mdev->free_list);
 
+	ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler,
+			       0, dev_name(&pdev->dev), mdev);
+	if (ret)
+		return ret;
+
 	dma_dev = &mdev->dmadev;
 
 	/* Set DMA capabilities */
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 5e88fd44812de1..543c2eb8cfa81e 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -2500,8 +2500,8 @@ DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);
 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
 {
 	/* Expose a simple debugfs interface to view all clocks */
-	debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,
-			    NULL, pl08x, &pl08x_debugfs_fops);
+	debugfs_create_file("stats", 0444, pl08x->memcpy.dbg_dev_root,
+			    pl08x, &pl08x_debugfs_fops);
 }
 
 #else
diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c
index 14a5ee14a48156..9a1c79c12268cd 100644
--- a/drivers/dma/apple-admac.c
+++ b/drivers/dma/apple-admac.c
@@ -39,10 +39,14 @@
 
 #define FLAG_DESC_NOTIFY	BIT(16)
 
+#define T8122_UNK_28_VAL	0x200000
+
 #define REG_TX_START		0x0000
 #define REG_TX_STOP		0x0004
 #define REG_RX_START		0x0008
 #define REG_RX_STOP		0x000c
+#define REG_UNK_28		0x0028
+#define REG_UNK_2C		0x002c
 #define REG_IMPRINT		0x0090
 #define REG_TX_SRAM_SIZE	0x0094
 #define REG_RX_SRAM_SIZE	0x0098
@@ -85,6 +89,10 @@
 struct admac_data;
 struct admac_tx;
 
+struct admac_hw {
+	bool set_unk28;
+};
+
 struct admac_chan {
 	unsigned int no;
 	struct admac_data *host;
@@ -127,6 +135,7 @@ struct admac_data {
 	struct mutex cache_alloc_lock;
 	struct admac_sram txcache, rxcache;
 
+	const struct admac_hw *hw;
 	int irq;
 	int irq_index;
 	int nchannels;
@@ -747,6 +756,11 @@ static int admac_device_config(struct dma_chan *chan,
 	u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) &
 		~(BUS_WIDTH_WORD_SIZE | BUS_WIDTH_FRAME_SIZE);
 
+	if (ad->hw->set_unk28) {
+		writel_relaxed(T8122_UNK_28_VAL, ad->base + REG_UNK_28);
+		writel_relaxed(T8122_UNK_28_VAL, ad->base + REG_UNK_2C);
+	}
+
 	switch (is_tx ? config->dst_addr_width : config->src_addr_width) {
 	case DMA_SLAVE_BUSWIDTH_1_BYTE:
 		wordsize = 1;
@@ -800,11 +814,18 @@ static int admac_device_config(struct dma_chan *chan,
 	return 0;
 }
 
+static const struct admac_hw admac_base_hw = {};
+
+static const struct admac_hw admac_t8122_hw = {
+	.set_unk28 = true,
+};
+
 static int admac_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
 	struct admac_data *ad;
 	struct dma_device *dma;
+	const struct admac_hw *hw;
 	int nchannels;
 	int err, irq, i;
 
@@ -822,6 +843,8 @@ static int admac_probe(struct platform_device *pdev)
 	ad->dev = &pdev->dev;
 	ad->nchannels = nchannels;
 	mutex_init(&ad->cache_alloc_lock);
+	hw = of_device_get_match_data(&pdev->dev);
+	ad->hw = hw ? hw : &admac_base_hw;
 
 	/*
 	 * The controller has 4 IRQ outputs. Try them all until
@@ -936,6 +959,7 @@ static void admac_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id admac_of_match[] = {
+	{ .compatible = "apple,t8122-admac", .data = &admac_t8122_hw },
 	{ .compatible = "apple,t8103-admac", },
 	{ .compatible = "apple,admac", },
 	{ }
diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c
index 09403aca8bb06b..52f5242ea33270 100644
--- a/drivers/dma/arm-dma350.c
+++ b/drivers/dma/arm-dma350.c
@@ -512,7 +512,7 @@ static int d350_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct d350_chan *dch = to_d350_chan(chan);
 	int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED,
-			      dev_name(&dch->vc.chan.dev->device), dch);
+			      vchan_chan_name(&dch->vc), dch);
 	if (!ret)
 		writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN);
 
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index e5b30a57c477ad..3fedc907db6383 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -244,7 +244,7 @@ struct at_desc {
 	/* Memset temporary buffer */
 	bool				memset_buffer;
 	dma_addr_t			memset_paddr;
-	int				*memset_vaddr;
+	u32				*memset_vaddr;
 	struct atdma_sg			sg[] __counted_by(sglen);
 };
 
@@ -381,23 +381,18 @@ static inline struct at_dma *to_at_dma(struct dma_device *ddev)
 
 /*--  Helper functions  ------------------------------------------------*/
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 #if defined(VERBOSE_DEBUG)
 static void vdbg_dump_regs(struct at_dma_chan *atchan)
 {
 	struct at_dma	*atdma = to_at_dma(atchan->vc.chan.device);
 
-	dev_err(chan2dev(&atchan->vc.chan),
+	dev_err(vchan_chan_dev(&atchan->vc),
 		"  channel %d : imr = 0x%x, chsr = 0x%x\n",
 		atchan->vc.chan.chan_id,
 		dma_readl(atdma, EBCIMR),
 		dma_readl(atdma, CHSR));
 
-	dev_err(chan2dev(&atchan->vc.chan),
+	dev_err(vchan_chan_dev(&atchan->vc),
 		"  channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n",
 		channel_readl(atchan, SADDR),
 		channel_readl(atchan, DADDR),
@@ -412,7 +407,7 @@ static void vdbg_dump_regs(struct at_dma_chan *atchan) {}
 
 static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli)
 {
-	dev_crit(chan2dev(&atchan->vc.chan),
+	dev_crit(vchan_chan_dev(&atchan->vc),
 		 "desc: s%pad d%pad ctrl0x%x:0x%x l%pad\n",
 		 &lli->saddr, &lli->daddr,
 		 lli->ctrla, lli->ctrlb, &lli->dscr);
@@ -790,8 +785,8 @@ static void atc_handle_error(struct at_dma_chan *atchan, unsigned int i)
 	 * controller flagged an error instead of scribbling over
 	 * random memory locations.
 	 */
-	dev_crit(chan2dev(&atchan->vc.chan), "Bad descriptor submitted for DMA!\n");
-	dev_crit(chan2dev(&atchan->vc.chan), "cookie: %d\n",
+	dev_crit(vchan_chan_dev(&atchan->vc), "Bad descriptor submitted for DMA!\n");
+	dev_crit(vchan_chan_dev(&atchan->vc), "cookie: %d\n",
 		 desc->vd.tx.cookie);
 	for (i = 0; i < desc->sglen; i++)
 		atc_dump_lli(atchan, desc->sg[i].lli);
@@ -886,7 +881,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan,
 
 	first = xt->sgl;
 
-	dev_info(chan2dev(chan),
+	dev_info(dmaengine_chan_dev(chan),
 		 "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n",
 		__func__, &xt->src_start, &xt->dst_start, xt->numf,
 		xt->frame_size, flags);
@@ -903,7 +898,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan,
 		if ((chunk->size != xt->sgl->size) ||
 		    (dmaengine_get_dst_icg(xt, chunk) != dmaengine_get_dst_icg(xt, first)) ||
 		    (dmaengine_get_src_icg(xt, chunk) != dmaengine_get_src_icg(xt, first))) {
-			dev_err(chan2dev(chan),
+			dev_err(dmaengine_chan_dev(chan),
 				"%s: the controller can transfer only identical chunks\n",
 				__func__);
 			return NULL;
@@ -916,7 +911,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan,
 
 	xfer_count = len >> dwidth;
 	if (xfer_count > ATC_BTSIZE_MAX) {
-		dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__);
+		dev_err(dmaengine_chan_dev(chan), "%s: buffer is too big\n", __func__);
 		return NULL;
 	}
 
@@ -983,11 +978,11 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	u32			ctrla;
 	u32			ctrlb;
 
-	dev_dbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n",
+	dev_dbg(dmaengine_chan_dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n",
 		&dest, &src, len, flags);
 
 	if (unlikely(!len)) {
-		dev_err(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
+		dev_err(dmaengine_chan_dev(chan), "prep_dma_memcpy: length is zero!\n");
 		return NULL;
 	}
 
@@ -1062,7 +1057,7 @@ static int atdma_create_memset_lli(struct dma_chan *chan,
 
 	xfer_count = len >> 2;
 	if (xfer_count > ATC_BTSIZE_MAX) {
-		dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__);
+		dev_err(dmaengine_chan_dev(chan), "%s: buffer is too big\n", __func__);
 		return -EINVAL;
 	}
 
@@ -1097,28 +1092,28 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	struct at_dma		*atdma = to_at_dma(chan->device);
 	struct at_desc		*desc;
-	void __iomem		*vaddr;
+	u32			*vaddr;
 	dma_addr_t		paddr;
 	char			fill_pattern;
 	int			ret;
 
-	dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__,
+	dev_vdbg(dmaengine_chan_dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__,
 		&dest, value, len, flags);
 
 	if (unlikely(!len)) {
-		dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: length is zero!\n", __func__);
 		return NULL;
 	}
 
 	if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {
-		dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: buffer is not aligned\n",
 			__func__);
 		return NULL;
 	}
 
 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr);
 	if (!vaddr) {
-		dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",
+		dev_err(dmaengine_chan_dev(chan), "%s: couldn't allocate buffer\n",
 			__func__);
 		return NULL;
 	}
@@ -1126,10 +1121,10 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
 	/* Only the first byte of value is to be used according to dmaengine */
 	fill_pattern = (char)value;
 
-	*(u32*)vaddr = (fill_pattern << 24) |
-		       (fill_pattern << 16) |
-		       (fill_pattern << 8) |
-		       fill_pattern;
+	*vaddr = (fill_pattern << 24) |
+		 (fill_pattern << 16) |
+		 (fill_pattern << 8) |
+		  fill_pattern;
 
 	desc = kzalloc_flex(*desc, sg, 1, GFP_ATOMIC);
 	if (!desc)
@@ -1168,28 +1163,28 @@ atc_prep_dma_memset_sg(struct dma_chan *chan,
 	struct at_dma		*atdma = to_at_dma(chan->device);
 	struct at_desc		*desc;
 	struct scatterlist	*sg;
-	void __iomem		*vaddr;
+	u32			*vaddr;
 	dma_addr_t		paddr;
 	size_t			total_len = 0;
 	int			i;
 	int			ret;
 
-	dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%x f0x%lx\n", __func__,
+	dev_vdbg(dmaengine_chan_dev(chan), "%s: v0x%x l0x%x f0x%lx\n", __func__,
 		 value, sg_len, flags);
 
 	if (unlikely(!sgl || !sg_len)) {
-		dev_dbg(chan2dev(chan), "%s: scatterlist is empty!\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: scatterlist is empty!\n",
 			__func__);
 		return NULL;
 	}
 
 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_NOWAIT, &paddr);
 	if (!vaddr) {
-		dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",
+		dev_err(dmaengine_chan_dev(chan), "%s: couldn't allocate buffer\n",
 			__func__);
 		return NULL;
 	}
-	*(u32*)vaddr = value;
+	*vaddr = value;
 
 	desc = kzalloc_flex(*desc, sg, sg_len, GFP_ATOMIC);
 	if (!desc)
@@ -1200,11 +1195,11 @@ atc_prep_dma_memset_sg(struct dma_chan *chan,
 		dma_addr_t dest = sg_dma_address(sg);
 		size_t len = sg_dma_len(sg);
 
-		dev_vdbg(chan2dev(chan), "%s: d%pad, l0x%zx\n",
+		dev_vdbg(dmaengine_chan_dev(chan), "%s: d%pad, l0x%zx\n",
 			 __func__, &dest, len);
 
 		if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {
-			dev_err(chan2dev(chan), "%s: buffer is not aligned\n",
+			dev_err(dmaengine_chan_dev(chan), "%s: buffer is not aligned\n",
 				__func__);
 			goto err_free_desc;
 		}
@@ -1264,13 +1259,13 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	struct scatterlist	*sg;
 	size_t			total_len = 0;
 
-	dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n",
+	dev_vdbg(dmaengine_chan_dev(chan), "prep_slave_sg (%d): %s f0x%lx\n",
 			sg_len,
 			direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE",
 			flags);
 
 	if (unlikely(!atslave || !sg_len)) {
-		dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n");
+		dev_dbg(dmaengine_chan_dev(chan), "prep_slave_sg: sg length is zero!\n");
 		return NULL;
 	}
 
@@ -1310,7 +1305,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 			mem = sg_dma_address(sg);
 			len = sg_dma_len(sg);
 			if (unlikely(!len)) {
-				dev_dbg(chan2dev(chan),
+				dev_dbg(dmaengine_chan_dev(chan),
 					"prep_slave_sg: sg(%d) data length is zero\n", i);
 				goto err;
 			}
@@ -1359,7 +1354,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 			mem = sg_dma_address(sg);
 			len = sg_dma_len(sg);
 			if (unlikely(!len)) {
-				dev_dbg(chan2dev(chan),
+				dev_dbg(dmaengine_chan_dev(chan),
 					"prep_slave_sg: sg(%d) data length is zero\n", i);
 				goto err;
 			}
@@ -1392,7 +1387,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	return vchan_tx_prep(&atchan->vc, &desc->vd, flags);
 
 err_desc_get:
-	dev_err(chan2dev(chan), "not enough descriptors available\n");
+	dev_err(dmaengine_chan_dev(chan), "not enough descriptors available\n");
 err:
 	atdma_desc_free(&desc->vd);
 	return NULL;
@@ -1503,19 +1498,19 @@ atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 	unsigned int		periods = buf_len / period_len;
 	unsigned int		i;
 
-	dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%zu/%zu)\n",
+	dev_vdbg(dmaengine_chan_dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%zu/%zu)\n",
 			direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE",
 			&buf_addr,
 			periods, buf_len, period_len);
 
 	if (unlikely(!atslave || !buf_len || !period_len)) {
-		dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n");
+		dev_dbg(dmaengine_chan_dev(chan), "prep_dma_cyclic: length is zero!\n");
 		return NULL;
 	}
 
 	was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status);
 	if (was_cyclic) {
-		dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n");
+		dev_dbg(dmaengine_chan_dev(chan), "prep_dma_cyclic: channel in use!\n");
 		return NULL;
 	}
 
@@ -1561,7 +1556,7 @@ static int atc_config(struct dma_chan *chan,
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 
-	dev_vdbg(chan2dev(chan), "%s\n", __func__);
+	dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	/* Check if it is chan is configured for slave transfers */
 	if (!chan->private)
@@ -1582,7 +1577,7 @@ static int atc_pause(struct dma_chan *chan)
 	int			chan_id = atchan->vc.chan.chan_id;
 	unsigned long		flags;
 
-	dev_vdbg(chan2dev(chan), "%s\n", __func__);
+	dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	spin_lock_irqsave(&atchan->vc.lock, flags);
 
@@ -1601,7 +1596,7 @@ static int atc_resume(struct dma_chan *chan)
 	int			chan_id = atchan->vc.chan.chan_id;
 	unsigned long		flags;
 
-	dev_vdbg(chan2dev(chan), "%s\n", __func__);
+	dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	if (!atc_chan_is_paused(atchan))
 		return 0;
@@ -1625,7 +1620,7 @@ static int atc_terminate_all(struct dma_chan *chan)
 
 	LIST_HEAD(list);
 
-	dev_vdbg(chan2dev(chan), "%s\n", __func__);
+	dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	/*
 	 * This is only called when something went wrong elsewhere, so
@@ -1691,13 +1686,13 @@ atc_tx_status(struct dma_chan *chan,
 	spin_unlock_irqrestore(&atchan->vc.lock, flags);
 
 	if (unlikely(ret < 0)) {
-		dev_vdbg(chan2dev(chan), "get residual bytes error\n");
+		dev_vdbg(dmaengine_chan_dev(chan), "get residual bytes error\n");
 		return DMA_ERROR;
 	} else {
 		dma_set_residue(txstate, residue);
 	}
 
-	dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %u\n",
+	dev_vdbg(dmaengine_chan_dev(chan), "tx_status %d: cookie = %d residue = %u\n",
 		 dma_status, cookie, residue);
 
 	return dma_status;
@@ -1729,11 +1724,11 @@ static int atc_alloc_chan_resources(struct dma_chan *chan)
 	struct at_dma_slave	*atslave;
 	u32			cfg;
 
-	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
+	dev_vdbg(dmaengine_chan_dev(chan), "alloc_chan_resources\n");
 
 	/* ASSERT:  channel is idle */
 	if (atc_chan_is_enabled(atchan)) {
-		dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
+		dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle ?\n");
 		return -EIO;
 	}
 
@@ -1782,7 +1777,7 @@ static void atc_free_chan_resources(struct dma_chan *chan)
 		chan->private = NULL;
 	}
 
-	dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
+	dev_vdbg(dmaengine_chan_dev(chan), "free_chan_resources: done\n");
 }
 
 #ifdef CONFIG_OF
@@ -1940,6 +1935,20 @@ static void at_dma_off(struct at_dma *atdma)
 		cpu_relax();
 }
 
+static void at_dma_cleanup_channels(struct at_dma *atdma)
+{
+	struct dma_chan *chan, *_chan;
+	int i = 0;
+
+	list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels,
+			device_node) {
+		/* Disable interrupts */
+		atc_disable_chan_irq(atdma, i++);
+		tasklet_kill(&to_at_dma_chan(chan)->vc.task);
+		list_del(&chan->device_node);
+	}
+}
+
 static int __init at_dma_probe(struct platform_device *pdev)
 {
 	struct at_dma		*atdma;
@@ -1980,20 +1989,16 @@ static int __init at_dma_probe(struct platform_device *pdev)
 	atdma->dma_device.cap_mask = plat_dat->cap_mask;
 	atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1;
 
-	atdma->clk = devm_clk_get(&pdev->dev, "dma_clk");
+	atdma->clk = devm_clk_get_enabled(&pdev->dev, "dma_clk");
 	if (IS_ERR(atdma->clk))
 		return PTR_ERR(atdma->clk);
 
-	err = clk_prepare_enable(atdma->clk);
-	if (err)
-		return err;
-
 	/* force dma off, just in case */
 	at_dma_off(atdma);
 
 	err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);
 	if (err)
-		goto err_irq;
+		return err;
 
 	platform_set_drvdata(pdev, atdma);
 
@@ -2105,38 +2110,30 @@ static int __init at_dma_probe(struct platform_device *pdev)
 err_of_dma_controller_register:
 	dma_async_device_unregister(&atdma->dma_device);
 err_dma_async_device_register:
+	disable_irq(platform_get_irq(pdev, 0));
+	at_dma_cleanup_channels(atdma);
 	dma_pool_destroy(atdma->memset_pool);
 err_memset_pool_create:
 	dma_pool_destroy(atdma->lli_pool);
 err_desc_pool_create:
-	free_irq(platform_get_irq(pdev, 0), atdma);
-err_irq:
-	clk_disable_unprepare(atdma->clk);
+	free_irq(irq, atdma);
 	return err;
 }
 
 static void at_dma_remove(struct platform_device *pdev)
 {
 	struct at_dma		*atdma = platform_get_drvdata(pdev);
-	struct dma_chan		*chan, *_chan;
 
 	at_dma_off(atdma);
 	if (pdev->dev.of_node)
 		of_dma_controller_free(pdev->dev.of_node);
 	dma_async_device_unregister(&atdma->dma_device);
 
-	dma_pool_destroy(atdma->memset_pool);
-	dma_pool_destroy(atdma->lli_pool);
 	free_irq(platform_get_irq(pdev, 0), atdma);
 
-	list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels,
-			device_node) {
-		/* Disable interrupts */
-		atc_disable_chan_irq(atdma, chan->chan_id);
-		list_del(&chan->device_node);
-	}
-
-	clk_disable_unprepare(atdma->clk);
+	at_dma_cleanup_channels(atdma);
+	dma_pool_destroy(atdma->memset_pool);
+	dma_pool_destroy(atdma->lli_pool);
 }
 
 static void at_dma_shutdown(struct platform_device *pdev)
@@ -2169,7 +2166,7 @@ static void atc_suspend_cyclic(struct at_dma_chan *atchan)
 	/* Channel should be paused by user
 	 * do it anyway even if it is not done already */
 	if (!atc_chan_is_paused(atchan)) {
-		dev_warn(chan2dev(chan),
+		dev_warn(dmaengine_chan_dev(chan),
 		"cyclic channel not paused, should be done by channel user\n");
 		atc_pause(chan);
 	}
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index 901971e8bae696..d40a8b40dddc39 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -324,11 +324,6 @@ static inline struct at_xdmac_chan *to_at_xdmac_chan(struct dma_chan *dchan)
 	return container_of(dchan, struct at_xdmac_chan, chan);
 }
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static inline struct at_xdmac *to_at_xdmac(struct dma_device *ddev)
 {
 	return container_of(ddev, struct at_xdmac, dma);
@@ -459,7 +454,7 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
 	if (ret < 0)
 		return;
 
-	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);
+	dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);
 
 	/* Set transfer as active to not try to start it again. */
 	first->active_xfer = true;
@@ -496,7 +491,7 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
 	       | AT_XDMAC_CNDC_NDE;
 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);
 
-	dev_vdbg(chan2dev(&atchan->chan),
+	dev_vdbg(dmaengine_chan_dev(&atchan->chan),
 		 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
@@ -524,12 +519,12 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,
 				    reg | AT_XDMAC_CIE_LIE);
 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);
-	dev_vdbg(chan2dev(&atchan->chan),
+	dev_vdbg(dmaengine_chan_dev(&atchan->chan),
 		 "%s: enable channel (0x%08x)\n", __func__, atchan->mask);
 	wmb();
 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);
 
-	dev_vdbg(chan2dev(&atchan->chan),
+	dev_vdbg(dmaengine_chan_dev(&atchan->chan),
 		 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
@@ -552,7 +547,7 @@ static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx)
 	list_add_tail(&desc->xfer_node, &atchan->xfers_list);
 	spin_unlock_irqrestore(&atchan->lock, irqflags);
 
-	dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",
+	dev_vdbg(dmaengine_chan_dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",
 		 __func__, atchan, desc);
 
 	return cookie;
@@ -612,7 +607,7 @@ static void at_xdmac_queue_desc(struct dma_chan *chan,
 	prev->lld.mbr_nda = desc->tx_dma_desc.phys;
 	prev->lld.mbr_ubc |= AT_XDMAC_MBR_UBC_NDE;
 
-	dev_dbg(chan2dev(chan),	"%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
+	dev_dbg(dmaengine_chan_dev(chan),	"%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
 		__func__, prev, &prev->lld.mbr_nda);
 }
 
@@ -624,7 +619,7 @@ static inline void at_xdmac_increment_block_count(struct dma_chan *chan,
 
 	desc->lld.mbr_bc++;
 
-	dev_dbg(chan2dev(chan),
+	dev_dbg(dmaengine_chan_dev(chan),
 		"%s: incrementing the block count of the desc 0x%p\n",
 		__func__, desc);
 }
@@ -680,13 +675,13 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
 
 		csize = ffs(atchan->sconfig.src_maxburst) - 1;
 		if (csize < 0) {
-			dev_err(chan2dev(chan), "invalid src maxburst value\n");
+			dev_err(dmaengine_chan_dev(chan), "invalid src maxburst value\n");
 			return -EINVAL;
 		}
 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
 		dwidth = ffs(atchan->sconfig.src_addr_width) - 1;
 		if (dwidth < 0) {
-			dev_err(chan2dev(chan), "invalid src addr width value\n");
+			dev_err(dmaengine_chan_dev(chan), "invalid src addr width value\n");
 			return -EINVAL;
 		}
 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
@@ -705,19 +700,19 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
 
 		csize = ffs(atchan->sconfig.dst_maxburst) - 1;
 		if (csize < 0) {
-			dev_err(chan2dev(chan), "invalid src maxburst value\n");
+			dev_err(dmaengine_chan_dev(chan), "invalid src maxburst value\n");
 			return -EINVAL;
 		}
 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
 		dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;
 		if (dwidth < 0) {
-			dev_err(chan2dev(chan), "invalid dst addr width value\n");
+			dev_err(dmaengine_chan_dev(chan), "invalid dst addr width value\n");
 			return -EINVAL;
 		}
 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
 	}
 
-	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);
+	dev_dbg(dmaengine_chan_dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);
 
 	return 0;
 }
@@ -746,7 +741,7 @@ static int at_xdmac_set_slave_config(struct dma_chan *chan,
 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
 
 	if (at_xdmac_check_slave_config(sconfig)) {
-		dev_err(chan2dev(chan), "invalid slave configuration\n");
+		dev_err(dmaengine_chan_dev(chan), "invalid slave configuration\n");
 		return -EINVAL;
 	}
 
@@ -772,11 +767,11 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		return NULL;
 
 	if (!is_slave_direction(direction)) {
-		dev_err(chan2dev(chan), "invalid DMA direction\n");
+		dev_err(dmaengine_chan_dev(chan), "invalid DMA direction\n");
 		return NULL;
 	}
 
-	dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",
 		 __func__, sg_len,
 		 direction == DMA_MEM_TO_DEV ? "to device" : "from device",
 		 flags);
@@ -795,15 +790,15 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		len = sg_dma_len(sg);
 		mem = sg_dma_address(sg);
 		if (unlikely(!len)) {
-			dev_err(chan2dev(chan), "sg data length is zero\n");
+			dev_err(dmaengine_chan_dev(chan), "sg data length is zero\n");
 			goto spin_unlock;
 		}
-		dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",
 			 __func__, i, len, mem);
 
 		desc = at_xdmac_get_desc(atchan);
 		if (!desc) {
-			dev_err(chan2dev(chan), "can't get descriptor\n");
+			dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n");
 			if (first)
 				list_splice_tail_init(&first->descs_list,
 						      &atchan->free_descs_list);
@@ -828,7 +823,7 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 			| (len >> fixed_dwidth);				/* microblock length */
 		desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |
 				    AT_XDMAC_CC_DWIDTH(fixed_dwidth);
-		dev_dbg(chan2dev(chan),
+		dev_dbg(dmaengine_chan_dev(chan),
 			 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
 			 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
 
@@ -840,7 +835,7 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		if (!first)
 			first = desc;
 
-		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 			 __func__, desc, first);
 		list_add_tail(&desc->desc_node, &first->descs_list);
 		xfer_size += len;
@@ -869,17 +864,17 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 	int			i;
 	unsigned long		irqflags;
 
-	dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",
 		__func__, &buf_addr, buf_len, period_len,
 		direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags);
 
 	if (!is_slave_direction(direction)) {
-		dev_err(chan2dev(chan), "invalid DMA direction\n");
+		dev_err(dmaengine_chan_dev(chan), "invalid DMA direction\n");
 		return NULL;
 	}
 
 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {
-		dev_err(chan2dev(chan), "channel currently used\n");
+		dev_err(dmaengine_chan_dev(chan), "channel currently used\n");
 		return NULL;
 	}
 
@@ -892,7 +887,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 		spin_lock_irqsave(&atchan->lock, irqflags);
 		desc = at_xdmac_get_desc(atchan);
 		if (!desc) {
-			dev_err(chan2dev(chan), "can't get descriptor\n");
+			dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n");
 			if (first)
 				list_splice_tail_init(&first->descs_list,
 						      &atchan->free_descs_list);
@@ -900,7 +895,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 			return NULL;
 		}
 		spin_unlock_irqrestore(&atchan->lock, irqflags);
-		dev_dbg(chan2dev(chan),
+		dev_dbg(dmaengine_chan_dev(chan),
 			"%s: desc=0x%p, tx_dma_desc.phys=%pad\n",
 			__func__, desc, &desc->tx_dma_desc.phys);
 
@@ -917,7 +912,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 			| AT_XDMAC_MBR_UBC_NSEN
 			| period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg);
 
-		dev_dbg(chan2dev(chan),
+		dev_dbg(dmaengine_chan_dev(chan),
 			 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
 			 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
 
@@ -929,7 +924,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 		if (!first)
 			first = desc;
 
-		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 			 __func__, desc, first);
 		list_add_tail(&desc->desc_node, &first->descs_list);
 	}
@@ -956,16 +951,16 @@ static inline u32 at_xdmac_align_width(struct dma_chan *chan, dma_addr_t addr)
 	 */
 	if (!(addr & 7)) {
 		width = AT_XDMAC_CC_DWIDTH_DWORD;
-		dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: double word\n", __func__);
 	} else if (!(addr & 3)) {
 		width = AT_XDMAC_CC_DWIDTH_WORD;
-		dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: word\n", __func__);
 	} else if (!(addr & 1)) {
 		width = AT_XDMAC_CC_DWIDTH_HALFWORD;
-		dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: half word\n", __func__);
 	} else {
 		width = AT_XDMAC_CC_DWIDTH_BYTE;
-		dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: dwidth: byte\n", __func__);
 	}
 
 	return width;
@@ -1005,7 +1000,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
 
 	dwidth = at_xdmac_align_width(chan, src | dst | chunk->size);
 	if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
-		dev_dbg(chan2dev(chan),
+		dev_dbg(dmaengine_chan_dev(chan),
 			"%s: chunk too big (%zu, max size %lu)...\n",
 			__func__, chunk->size,
 			AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth);
@@ -1013,7 +1008,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
 	}
 
 	if (prev)
-		dev_dbg(chan2dev(chan),
+		dev_dbg(dmaengine_chan_dev(chan),
 			"Adding items at the end of desc 0x%p\n", prev);
 
 	if (xt->src_inc) {
@@ -1034,7 +1029,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
 	desc = at_xdmac_get_desc(atchan);
 	spin_unlock_irqrestore(&atchan->lock, flags);
 	if (!desc) {
-		dev_err(chan2dev(chan), "can't get descriptor\n");
+		dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n");
 		return NULL;
 	}
 
@@ -1053,7 +1048,7 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
 		| ublen;
 	desc->lld.mbr_cfg = chan_cc;
 
-	dev_dbg(chan2dev(chan),
+	dev_dbg(dmaengine_chan_dev(chan),
 		"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
 		__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da,
 		desc->lld.mbr_ubc, desc->lld.mbr_cfg);
@@ -1087,7 +1082,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan,
 	if ((xt->numf > 1) && (xt->frame_size > 1))
 		return NULL;
 
-	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n",
 		__func__, &xt->src_start, &xt->dst_start,	xt->numf,
 		xt->frame_size, flags);
 
@@ -1106,7 +1101,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan,
 		for (i = 0; i < xt->numf - 1; i++)
 			at_xdmac_increment_block_count(chan, first);
 
-		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 			__func__, first, first);
 		list_add_tail(&first->desc_node, &first->descs_list);
 	} else {
@@ -1122,7 +1117,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan,
 			src_skip = chunk->size + src_icg;
 			dst_skip = chunk->size + dst_icg;
 
-			dev_dbg(chan2dev(chan),
+			dev_dbg(dmaengine_chan_dev(chan),
 				"%s: chunk size=%zu, src icg=%zu, dst icg=%zu\n",
 				__func__, chunk->size, src_icg, dst_icg);
 
@@ -1140,7 +1135,7 @@ at_xdmac_prep_interleaved(struct dma_chan *chan,
 			if (!first)
 				first = desc;
 
-			dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
+			dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 				__func__, desc, first);
 			list_add_tail(&desc->desc_node, &first->descs_list);
 
@@ -1193,7 +1188,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 					| AT_XDMAC_CC_TYPE_MEM_TRAN;
 	unsigned long		irqflags;
 
-	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n",
 		__func__, &src, &dest, len, flags);
 
 	if (unlikely(!len))
@@ -1205,13 +1200,13 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	while (remaining_size) {
 		struct at_xdmac_desc	*desc = NULL;
 
-		dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size);
 
 		spin_lock_irqsave(&atchan->lock, irqflags);
 		desc = at_xdmac_get_desc(atchan);
 		spin_unlock_irqrestore(&atchan->lock, irqflags);
 		if (!desc) {
-			dev_err(chan2dev(chan), "can't get descriptor\n");
+			dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n");
 			if (first)
 				list_splice_tail_init(&first->descs_list,
 						      &atchan->free_descs_list);
@@ -1227,7 +1222,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		else
 			xfer_size = remaining_size;
 
-		dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size);
 
 		/* Check remaining length and change data width if needed. */
 		dwidth = at_xdmac_align_width(chan,
@@ -1246,7 +1241,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 			| ublen;
 		desc->lld.mbr_cfg = chan_cc;
 
-		dev_dbg(chan2dev(chan),
+		dev_dbg(dmaengine_chan_dev(chan),
 			 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
 			 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg);
 
@@ -1258,7 +1253,7 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		if (!first)
 			first = desc;
 
-		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 			 __func__, desc, first);
 		list_add_tail(&desc->desc_node, &first->descs_list);
 	}
@@ -1306,7 +1301,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
 	dwidth = at_xdmac_align_width(chan, dst_addr);
 
 	if (len >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
-		dev_err(chan2dev(chan),
+		dev_err(dmaengine_chan_dev(chan),
 			"%s: Transfer too large, aborting...\n",
 			__func__);
 		return NULL;
@@ -1316,7 +1311,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
 	desc = at_xdmac_get_desc(atchan);
 	spin_unlock_irqrestore(&atchan->lock, flags);
 	if (!desc) {
-		dev_err(chan2dev(chan), "can't get descriptor\n");
+		dev_err(dmaengine_chan_dev(chan), "can't get descriptor\n");
 		return NULL;
 	}
 
@@ -1338,7 +1333,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
 		| ublen;
 	desc->lld.mbr_cfg = chan_cc;
 
-	dev_dbg(chan2dev(chan),
+	dev_dbg(dmaengine_chan_dev(chan),
 		"%s: lld: mbr_da=%pad, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
 		__func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc,
 		desc->lld.mbr_cfg);
@@ -1353,7 +1348,7 @@ at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
 	struct at_xdmac_desc	*desc;
 
-	dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n",
 		__func__, &dest, len, value, flags);
 
 	if (unlikely(!len))
@@ -1386,12 +1381,12 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	if (!sgl)
 		return NULL;
 
-	dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n",
 		__func__, sg_len, value, flags);
 
 	/* Prepare descriptors. */
 	for_each_sg(sgl, sg, sg_len, i) {
-		dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n",
+		dev_dbg(dmaengine_chan_dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n",
 			__func__, &sg_dma_address(sg), sg_dma_len(sg),
 			value, flags);
 		desc = at_xdmac_memset_create_desc(chan, atchan,
@@ -1433,7 +1428,7 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		if (ppdesc && pdesc) {
 			if ((stride == pstride) &&
 			    (sg_dma_len(ppsg) == sg_dma_len(psg))) {
-				dev_dbg(chan2dev(chan),
+				dev_dbg(dmaengine_chan_dev(chan),
 					"%s: desc 0x%p can be merged with desc 0x%p\n",
 					__func__, pdesc, ppdesc);
 
@@ -1481,7 +1476,7 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,
 				 */
 				list_add_tail(&desc->desc_node,
 					      &first->descs_list);
-				dev_dbg(chan2dev(chan),
+				dev_dbg(dmaengine_chan_dev(chan),
 					"%s: add desc 0x%p to descs_list 0x%p\n",
 					__func__, desc, first);
 			}
@@ -1496,7 +1491,7 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		 */
 		if ((i == (sg_len - 1)) &&
 		    sg_dma_len(psg) == sg_dma_len(sg)) {
-			dev_dbg(chan2dev(chan),
+			dev_dbg(dmaengine_chan_dev(chan),
 				"%s: desc 0x%p can be merged with desc 0x%p\n",
 				__func__, desc, pdesc);
 
@@ -1667,7 +1662,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 
 	dma_set_residue(txstate, residue);
 
-	dev_dbg(chan2dev(chan),
+	dev_dbg(dmaengine_chan_dev(chan),
 		 "%s: desc=0x%p, tx_dma_desc.phys=%pad, tx_status=%d, cookie=%d, residue=%d\n",
 		 __func__, desc, &desc->tx_dma_desc.phys, ret, cookie, residue);
 
@@ -1690,7 +1685,7 @@ static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)
 
 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc,
 				xfer_node);
-	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
+	dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
 	if (!desc->active_xfer)
 		at_xdmac_start_xfer(atchan, desc);
 }
@@ -1701,7 +1696,7 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)
 	struct dma_async_tx_descriptor	*txd;
 
 	spin_lock_irq(&atchan->lock);
-	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",
+	dev_dbg(dmaengine_chan_dev(&atchan->chan), "%s: status=0x%08x\n",
 		__func__, atchan->irq_status);
 	if (list_empty(&atchan->xfers_list)) {
 		spin_unlock_irq(&atchan->lock);
@@ -1733,11 +1728,11 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
 	 * descriptors queued (if any).
 	 */
 	if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)
-		dev_err(chan2dev(&atchan->chan), "read bus error!!!");
+		dev_err(dmaengine_chan_dev(&atchan->chan), "read bus error!!!");
 	if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)
-		dev_err(chan2dev(&atchan->chan), "write bus error!!!");
+		dev_err(dmaengine_chan_dev(&atchan->chan), "write bus error!!!");
 	if (atchan->irq_status & AT_XDMAC_CIS_ROIS)
-		dev_err(chan2dev(&atchan->chan), "request overflow error!!!");
+		dev_err(dmaengine_chan_dev(&atchan->chan), "request overflow error!!!");
 
 	/* Channel must be disabled first as it's not done automatically */
 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
@@ -1749,7 +1744,7 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
 				    xfer_node);
 
 	/* Print bad descriptor's details if needed */
-	dev_dbg(chan2dev(&atchan->chan),
+	dev_dbg(dmaengine_chan_dev(&atchan->chan),
 		"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
 		__func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da,
 		bad_desc->lld.mbr_ubc);
@@ -1775,7 +1770,7 @@ static void at_xdmac_tasklet(struct tasklet_struct *t)
 
 	spin_lock_irq(&atchan->lock);
 
-	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",
+	dev_dbg(dmaengine_chan_dev(&atchan->chan), "%s: status=0x%08x\n",
 		__func__, atchan->irq_status);
 
 	if (!(atchan->irq_status & AT_XDMAC_CIS_LIS) &&
@@ -1789,9 +1784,9 @@ static void at_xdmac_tasklet(struct tasklet_struct *t)
 
 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc,
 				xfer_node);
-	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
+	dev_vdbg(dmaengine_chan_dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
 	if (!desc->active_xfer) {
-		dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting");
+		dev_err(dmaengine_chan_dev(&atchan->chan), "Xfer not active: exiting");
 		spin_unlock_irq(&atchan->lock);
 		return;
 	}
@@ -1852,7 +1847,7 @@ static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id)
 			dev_vdbg(atxdmac->dma.dev,
 				 "%s: chan%d: imr=0x%x, status=0x%x\n",
 				 __func__, i, chan_imr, chan_status);
-			dev_vdbg(chan2dev(&atchan->chan),
+			dev_vdbg(dmaengine_chan_dev(&atchan->chan),
 				 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
 				 __func__,
 				 at_xdmac_chan_read(atchan, AT_XDMAC_CC),
@@ -1879,7 +1874,7 @@ static void at_xdmac_issue_pending(struct dma_chan *chan)
 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
 	unsigned long flags;
 
-	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);
+	dev_dbg(dmaengine_chan_dev(&atchan->chan), "%s\n", __func__);
 
 	spin_lock_irqsave(&atchan->lock, flags);
 	at_xdmac_advance_work(atchan);
@@ -1895,7 +1890,7 @@ static int at_xdmac_device_config(struct dma_chan *chan,
 	int ret;
 	unsigned long		flags;
 
-	dev_dbg(chan2dev(chan), "%s\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	spin_lock_irqsave(&atchan->lock, flags);
 	ret = at_xdmac_set_slave_config(chan, config);
@@ -1931,7 +1926,7 @@ static int at_xdmac_device_pause(struct dma_chan *chan)
 	unsigned long		flags;
 	int			ret;
 
-	dev_dbg(chan2dev(chan), "%s\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))
 		return 0;
@@ -1971,7 +1966,7 @@ static int at_xdmac_device_resume(struct dma_chan *chan)
 	unsigned long		flags;
 	int			ret;
 
-	dev_dbg(chan2dev(chan), "%s\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	ret = pm_runtime_resume_and_get(atxdmac->dev);
 	if (ret < 0)
@@ -2004,7 +1999,7 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan)
 	unsigned long		flags;
 	int			ret;
 
-	dev_dbg(chan2dev(chan), "%s\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	ret = pm_runtime_resume_and_get(atxdmac->dev);
 	if (ret < 0)
@@ -2045,13 +2040,13 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan)
 	int			i;
 
 	if (at_xdmac_chan_is_enabled(atchan)) {
-		dev_err(chan2dev(chan),
+		dev_err(dmaengine_chan_dev(chan),
 			"can't allocate channel resources (channel enabled)\n");
 		return -EIO;
 	}
 
 	if (!list_empty(&atchan->free_descs_list)) {
-		dev_err(chan2dev(chan),
+		dev_err(dmaengine_chan_dev(chan),
 			"can't allocate channel resources (channel not free from a previous use)\n");
 		return -EIO;
 	}
@@ -2060,11 +2055,11 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan)
 		desc = at_xdmac_alloc_desc(chan, GFP_KERNEL);
 		if (!desc) {
 			if (i == 0) {
-				dev_warn(chan2dev(chan),
+				dev_warn(dmaengine_chan_dev(chan),
 					 "can't allocate any descriptors\n");
 				return -EIO;
 			}
-			dev_warn(chan2dev(chan),
+			dev_warn(dmaengine_chan_dev(chan),
 				"only %d descriptors have been allocated\n", i);
 			break;
 		}
@@ -2073,7 +2068,7 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan)
 
 	dma_cookie_init(chan);
 
-	dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: allocated %d descriptors\n", __func__, i);
 
 	return i;
 }
@@ -2085,7 +2080,7 @@ static void at_xdmac_free_chan_resources(struct dma_chan *chan)
 	struct at_xdmac_desc	*desc, *_desc;
 
 	list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {
-		dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: freeing descriptor %p\n", __func__, desc);
 		list_del(&desc->desc_node);
 		dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys);
 	}
@@ -2148,7 +2143,7 @@ static int __maybe_unused atmel_xdmac_suspend(struct device *dev)
 		atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);
 		if (at_xdmac_chan_is_cyclic(atchan)) {
 			if (!at_xdmac_chan_is_paused(atchan)) {
-				dev_warn(chan2dev(chan), "%s: channel %d not paused\n",
+				dev_warn(dmaengine_chan_dev(chan), "%s: channel %d not paused\n",
 					 __func__, chan->chan_id);
 				at_xdmac_device_pause_internal(atchan);
 				at_xdmac_runtime_suspend_descriptors(atchan);
diff --git a/drivers/dma/bestcomm/ata.c b/drivers/dma/bestcomm/ata.c
index 502a45d76adca6..40126f3f189ef1 100644
--- a/drivers/dma/bestcomm/ata.c
+++ b/drivers/dma/bestcomm/ata.c
@@ -55,7 +55,6 @@ bcom_ata_init(int queue_len, int maxbufsize)
 {
 	struct bcom_task *tsk;
 	struct bcom_ata_var *var;
-	struct bcom_ata_inc *inc;
 
 	/* Prefetch breaks ATA DMA.  Turn it off for ATA DMA */
 	bcom_disable_prefetch();
@@ -69,7 +68,6 @@ bcom_ata_init(int queue_len, int maxbufsize)
 	bcom_ata_reset_bd(tsk);
 
 	var = (struct bcom_ata_var *) bcom_task_var(tsk->tasknum);
-	inc = (struct bcom_ata_inc *) bcom_task_inc(tsk->tasknum);
 
 	if (bcom_load_image(tsk->tasknum, bcom_ata_task)) {
 		bcom_task_free(tsk);
diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c
index 432b43520ddc95..241a91936fe8a3 100644
--- a/drivers/dma/bestcomm/bestcomm.c
+++ b/drivers/dma/bestcomm/bestcomm.c
@@ -13,7 +13,6 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/of.h>
-#include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <asm/io.h>
@@ -365,13 +364,19 @@ bcom_engine_cleanup(void)
 static int mpc52xx_bcom_probe(struct platform_device *op)
 {
 	struct device_node *ofn_sram;
-	struct resource res_bcom;
+	struct resource *res_bcom;
+	void __iomem *regs;
 
 	int rv;
 
 	/* Inform user we're ok so far */
 	printk(KERN_INFO "DMA: MPC52xx BestComm driver\n");
 
+	/* Get, reserve & map io */
+	regs = devm_platform_get_and_ioremap_resource(op, 0, &res_bcom);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
 	/* Get the bestcomm node */
 	of_node_get(op->dev.of_node);
 
@@ -402,35 +407,13 @@ static int mpc52xx_bcom_probe(struct platform_device *op)
 	/* Save the node */
 	bcom_eng->ofnode = op->dev.of_node;
 
-	/* Get, reserve & map io */
-	if (of_address_to_resource(op->dev.of_node, 0, &res_bcom)) {
-		printk(KERN_ERR DRIVER_NAME ": "
-			"Can't get resource\n");
-		rv = -EINVAL;
-		goto error_sramclean;
-	}
-
-	if (!request_mem_region(res_bcom.start, resource_size(&res_bcom),
-				DRIVER_NAME)) {
-		printk(KERN_ERR DRIVER_NAME ": "
-			"Can't request registers region\n");
-		rv = -EBUSY;
-		goto error_sramclean;
-	}
-
-	bcom_eng->regs_base = res_bcom.start;
-	bcom_eng->regs = ioremap(res_bcom.start, sizeof(struct mpc52xx_sdma));
-	if (!bcom_eng->regs) {
-		printk(KERN_ERR DRIVER_NAME ": "
-			"Can't map registers\n");
-		rv = -ENOMEM;
-		goto error_release;
-	}
+	bcom_eng->regs = regs;
+	bcom_eng->regs_base = res_bcom->start;
 
 	/* Now, do the real init */
 	rv = bcom_engine_init();
 	if (rv)
-		goto error_unmap;
+		goto error_sramclean;
 
 	/* Done ! */
 	printk(KERN_INFO "DMA: MPC52xx BestComm engine @%08lx ok !\n",
@@ -439,12 +422,9 @@ static int mpc52xx_bcom_probe(struct platform_device *op)
 	return 0;
 
 	/* Error path */
-error_unmap:
-	iounmap(bcom_eng->regs);
-error_release:
-	release_mem_region(res_bcom.start, sizeof(struct mpc52xx_sdma));
 error_sramclean:
 	kfree(bcom_eng);
+	bcom_eng = NULL;
 	bcom_sram_cleanup();
 error_ofput:
 	of_node_put(op->dev.of_node);
@@ -463,10 +443,6 @@ static void mpc52xx_bcom_remove(struct platform_device *op)
 	/* Cleanup SRAM */
 	bcom_sram_cleanup();
 
-	/* Release regs */
-	iounmap(bcom_eng->regs);
-	release_mem_region(bcom_eng->regs_base, sizeof(struct mpc52xx_sdma));
-
 	/* Release the node */
 	of_node_put(bcom_eng->ofnode);
 
diff --git a/drivers/dma/bestcomm/gen_bd.c b/drivers/dma/bestcomm/gen_bd.c
index 8a24a5cbc26330..3b4fabb4d12b06 100644
--- a/drivers/dma/bestcomm/gen_bd.c
+++ b/drivers/dma/bestcomm/gen_bd.c
@@ -81,7 +81,7 @@ struct bcom_gen_bd_priv {
 /* Task support code                                                        */
 /* ======================================================================== */
 
-struct bcom_task *
+static struct bcom_task *
 bcom_gen_bd_rx_init(int queue_len, phys_addr_t fifo,
 			int initiator, int ipr, int maxbufsize)
 {
@@ -108,7 +108,6 @@ bcom_gen_bd_rx_init(int queue_len, phys_addr_t fifo,
 
 	return tsk;
 }
-EXPORT_SYMBOL_GPL(bcom_gen_bd_rx_init);
 
 int
 bcom_gen_bd_rx_reset(struct bcom_task *tsk)
@@ -166,7 +165,7 @@ bcom_gen_bd_rx_release(struct bcom_task *tsk)
 EXPORT_SYMBOL_GPL(bcom_gen_bd_rx_release);
 
 
-extern struct bcom_task *
+static struct bcom_task *
 bcom_gen_bd_tx_init(int queue_len, phys_addr_t fifo,
 			int initiator, int ipr)
 {
@@ -192,7 +191,6 @@ bcom_gen_bd_tx_init(int queue_len, phys_addr_t fifo,
 
 	return tsk;
 }
-EXPORT_SYMBOL_GPL(bcom_gen_bd_tx_init);
 
 int
 bcom_gen_bd_tx_reset(struct bcom_task *tsk)
@@ -254,17 +252,23 @@ EXPORT_SYMBOL_GPL(bcom_gen_bd_tx_release);
  */
 
 /**
- * bcom_psc_parameters - Bestcomm initialization value table for PSC devices
+ * struct bcom_psc_params - Bestcomm initialization value table for PSC devices
+ * @rx_initiator: RX initiator ID
+ * @rx_ipr: RX interrupt priority register value
+ * @tx_initiator: TX initiator ID
+ * @tx_ipr: TX interrupt priority register value
  *
  * This structure is only used internally.  It is a lookup table for PSC
  * specific parameters to bestcomm tasks.
  */
-static struct bcom_psc_params {
+struct bcom_psc_params {
 	int rx_initiator;
 	int rx_ipr;
 	int tx_initiator;
 	int tx_ipr;
-} bcom_psc_params[] = {
+};
+
+static const struct bcom_psc_params bcom_psc_params[] = {
 	[0] = {
 		.rx_initiator = BCOM_INITIATOR_PSC1_RX,
 		.rx_ipr = BCOM_IPR_PSC1_RX,
@@ -315,7 +319,7 @@ static struct bcom_psc_params {
 struct bcom_task * bcom_psc_gen_bd_rx_init(unsigned psc_num, int queue_len,
 					   phys_addr_t fifo, int maxbufsize)
 {
-	if (psc_num >= MPC52xx_PSC_MAXNUM)
+	if (psc_num >= ARRAY_SIZE(bcom_psc_params))
 		return NULL;
 
 	return bcom_gen_bd_rx_init(queue_len, fifo,
@@ -336,6 +340,9 @@ EXPORT_SYMBOL_GPL(bcom_psc_gen_bd_rx_init);
 struct bcom_task *
 bcom_psc_gen_bd_tx_init(unsigned psc_num, int queue_len, phys_addr_t fifo)
 {
+	if (psc_num >= ARRAY_SIZE(bcom_psc_params))
+		return NULL;
+
 	struct psc;
 	return bcom_gen_bd_tx_init(queue_len, fifo,
 				   bcom_psc_params[psc_num].tx_initiator,
diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c
index 6070dfdb7114ce..ac2094e65ecf33 100644
--- a/drivers/dma/dma-jz4780.c
+++ b/drivers/dma/dma-jz4780.c
@@ -687,12 +687,12 @@ static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma,
 	jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0);
 
 	if (dcs & JZ_DMA_DCS_AR) {
-		dev_warn(&jzchan->vchan.chan.dev->device,
+		dev_warn(vchan_chan_dev(&jzchan->vchan),
 			 "address error (DCS=0x%x)\n", dcs);
 	}
 
 	if (dcs & JZ_DMA_DCS_HLT) {
-		dev_warn(&jzchan->vchan.chan.dev->device,
+		dev_warn(vchan_chan_dev(&jzchan->vchan),
 			 "channel halt (DCS=0x%x)\n", dcs);
 	}
 
@@ -721,7 +721,7 @@ static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma,
 			}
 		}
 	} else {
-		dev_err(&jzchan->vchan.chan.dev->device,
+		dev_err(vchan_chan_dev(&jzchan->vchan),
 			"channel IRQ with no active transfer\n");
 	}
 
@@ -760,12 +760,12 @@ static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
 
-	jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device),
+	jzchan->desc_pool = dma_pool_create(dma_chan_name(chan),
 					    chan->device->dev,
 					    JZ_DMA_DESC_BLOCK_SIZE,
 					    PAGE_SIZE, 0);
 	if (!jzchan->desc_pool) {
-		dev_err(&chan->dev->device,
+		dev_err(dmaengine_chan_dev(chan),
 			"failed to allocate descriptor pool\n");
 		return -ENOMEM;
 	}
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 6ffd8bd82154af..cc1c0a7c9f33da 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -428,11 +428,18 @@ static void dma_device_release(struct kref *ref)
 
 	list_del_rcu(&device->global_node);
 	dma_channel_rebalance();
+	synchronize_rcu();
 
 	if (device->device_release)
 		device->device_release(device);
 }
 
+static int __must_check dma_device_get(struct dma_device *device)
+{
+	lockdep_assert_held(&dma_list_mutex);
+	return kref_get_unless_zero(&device->ref);
+}
+
 static void dma_device_put(struct dma_device *device)
 {
 	lockdep_assert_held(&dma_list_mutex);
@@ -460,8 +467,7 @@ static int dma_chan_get(struct dma_chan *chan)
 	if (!try_module_get(owner))
 		return -ENODEV;
 
-	ret = kref_get_unless_zero(&chan->device->ref);
-	if (!ret) {
+	if (!dma_device_get(chan->device)) {
 		ret = -ENODEV;
 		goto module_put_out;
 	}
@@ -495,10 +501,13 @@ static int dma_chan_get(struct dma_chan *chan)
  */
 static void dma_chan_put(struct dma_chan *chan)
 {
+	struct module *owner;
+
 	/* This channel is not in use, bail out */
 	if (!chan->client_count)
 		return;
 
+	owner = dma_chan_to_owner(chan);
 	chan->client_count--;
 
 	/* This channel is not in use anymore, free it */
@@ -515,8 +524,10 @@ static void dma_chan_put(struct dma_chan *chan)
 		chan->route_data = NULL;
 	}
 
-	dma_device_put(chan->device);
-	module_put(dma_chan_to_owner(chan));
+	/* This channel is not in use anymore, drop the device ref */
+	if (!chan->client_count)
+		dma_device_put(chan->device);
+	module_put(owner);
 }
 
 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
@@ -868,10 +879,10 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name)
 		return chan;
 	chan->slave = dev;
 
-	if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,
+	if (sysfs_create_link(&dmaengine_chan_dev(chan)->kobj, &dev->kobj,
 			      DMA_SLAVE_NAME))
 		dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);
-	if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))
+	if (sysfs_create_link(&dev->kobj, &dmaengine_chan_dev(chan)->kobj, chan->name))
 		dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);
 
 	return chan;
@@ -917,7 +928,7 @@ void dma_release_channel(struct dma_chan *chan)
 	dma_chan_put(chan);
 
 	if (chan->slave) {
-		sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);
+		sysfs_remove_link(&dmaengine_chan_dev(chan)->kobj, DMA_SLAVE_NAME);
 		sysfs_remove_link(&chan->slave->kobj, chan->name);
 		kfree(chan->name);
 		chan->name = NULL;
@@ -1083,8 +1094,8 @@ static int __dma_async_device_channel_register(struct dma_device *device,
 	chan->local = alloc_percpu(typeof(*chan->local));
 	if (!chan->local)
 		return -ENOMEM;
-	chan->dev = kzalloc_obj(*chan->dev);
-	if (!chan->dev) {
+	chan->chan_dev = kzalloc_obj(*chan->chan_dev);
+	if (!chan->chan_dev) {
 		rc = -ENOMEM;
 		goto err_free_local;
 	}
@@ -1101,17 +1112,17 @@ static int __dma_async_device_channel_register(struct dma_device *device,
 		goto err_free_dev;
 	}
 
-	chan->dev->device.class = &dma_devclass;
-	chan->dev->device.parent = device->dev;
-	chan->dev->chan = chan;
-	chan->dev->dev_id = device->dev_id;
+	dmaengine_chan_dev(chan)->class = &dma_devclass;
+	dmaengine_chan_dev(chan)->parent = device->dev;
+	chan->chan_dev->chan = chan;
+	chan->chan_dev->dev_id = device->dev_id;
 	spin_lock_init(&chan->lock);
 
 	if (!name)
-		dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id);
+		dev_set_name(dmaengine_chan_dev(chan), "dma%dchan%d", device->dev_id, chan->chan_id);
 	else
-		dev_set_name(&chan->dev->device, "%s", name);
-	rc = device_register(&chan->dev->device);
+		dev_set_name(dmaengine_chan_dev(chan), "%s", name);
+	rc = device_register(dmaengine_chan_dev(chan));
 	if (rc)
 		goto err_out_ida;
 	chan->client_count = 0;
@@ -1122,7 +1133,7 @@ static int __dma_async_device_channel_register(struct dma_device *device,
  err_out_ida:
 	ida_free(&device->chan_ida, chan->chan_id);
  err_free_dev:
-	kfree(chan->dev);
+	kfree(chan->chan_dev);
  err_free_local:
 	free_percpu(chan->local);
 	chan->local = NULL;
@@ -1155,10 +1166,10 @@ static void __dma_async_device_channel_unregister(struct dma_device *device,
 		  __func__, chan->client_count);
 	mutex_lock(&dma_list_mutex);
 	device->chancnt--;
-	chan->dev->chan = NULL;
+	chan->chan_dev->chan = NULL;
 	mutex_unlock(&dma_list_mutex);
 	ida_free(&device->chan_ida, chan->chan_id);
-	device_unregister(&chan->dev->device);
+	device_unregister(dmaengine_chan_dev(chan));
 	free_percpu(chan->local);
 }
 
@@ -1290,9 +1301,9 @@ int dma_async_device_register(struct dma_device *device)
 		if (chan->local == NULL)
 			continue;
 		mutex_lock(&dma_list_mutex);
-		chan->dev->chan = NULL;
+		chan->chan_dev->chan = NULL;
 		mutex_unlock(&dma_list_mutex);
-		device_unregister(&chan->dev->device);
+		device_unregister(dmaengine_chan_dev(chan));
 		free_percpu(chan->local);
 	}
 	return rc;
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
index eebed2474210d8..7520c78f55ccf6 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
@@ -7,6 +7,7 @@
  * Author: Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/bitops.h>
 #include <linux/delay.h>
 #include <linux/device.h>
@@ -101,23 +102,25 @@ static inline void axi_chan_config_write(struct axi_dma_chan *chan,
 {
 	u32 cfg_lo, cfg_hi;
 
-	cfg_lo = (config->dst_multblk_type << CH_CFG_L_DST_MULTBLK_TYPE_POS |
-		  config->src_multblk_type << CH_CFG_L_SRC_MULTBLK_TYPE_POS);
 	if (chan->chip->dw->hdata->reg_map_8_channels &&
 	    !chan->chip->dw->hdata->use_cfg2) {
-		cfg_hi = config->tt_fc << CH_CFG_H_TT_FC_POS |
-			 config->hs_sel_src << CH_CFG_H_HS_SEL_SRC_POS |
-			 config->hs_sel_dst << CH_CFG_H_HS_SEL_DST_POS |
-			 config->src_per << CH_CFG_H_SRC_PER_POS |
-			 config->dst_per << CH_CFG_H_DST_PER_POS |
-			 config->prior << CH_CFG_H_PRIORITY_POS;
+		cfg_lo = FIELD_PREP(CH_CFG_L_DST_MULTBLK_TYPE, config->dst_multblk_type) |
+			 FIELD_PREP(CH_CFG_L_SRC_MULTBLK_TYPE, config->src_multblk_type);
+		cfg_hi = FIELD_PREP(CH_CFG_H_TT_FC, config->tt_fc) |
+			 FIELD_PREP(CH_CFG_H_HS_SEL_SRC, config->hs_sel_src) |
+			 FIELD_PREP(CH_CFG_H_HS_SEL_DST, config->hs_sel_dst) |
+			 FIELD_PREP(CH_CFG_H_SRC_PER, config->src_per) |
+			 FIELD_PREP(CH_CFG_H_DST_PER, config->dst_per) |
+			 FIELD_PREP(CH_CFG_H_PRIORITY, config->prior);
 	} else {
-		cfg_lo |= config->src_per << CH_CFG2_L_SRC_PER_POS |
-			  config->dst_per << CH_CFG2_L_DST_PER_POS;
-		cfg_hi = config->tt_fc << CH_CFG2_H_TT_FC_POS |
-			 config->hs_sel_src << CH_CFG2_H_HS_SEL_SRC_POS |
-			 config->hs_sel_dst << CH_CFG2_H_HS_SEL_DST_POS |
-			 config->prior << CH_CFG2_H_PRIORITY_POS;
+		cfg_lo = FIELD_PREP(CH_CFG_L_DST_MULTBLK_TYPE, config->dst_multblk_type) |
+			 FIELD_PREP(CH_CFG_L_SRC_MULTBLK_TYPE, config->src_multblk_type) |
+			 FIELD_PREP(CH_CFG2_L_SRC_PER, config->src_per) |
+			 FIELD_PREP(CH_CFG2_L_DST_PER, config->dst_per);
+		cfg_hi = FIELD_PREP(CH_CFG2_H_TT_FC, config->tt_fc) |
+			 FIELD_PREP(CH_CFG2_H_HS_SEL_SRC, config->hs_sel_src) |
+			 FIELD_PREP(CH_CFG2_H_HS_SEL_DST, config->hs_sel_dst) |
+			 FIELD_PREP(CH_CFG2_H_PRIORITY, config->prior);
 	}
 	axi_chan_iowrite32(chan, CH_CFG_L, cfg_lo);
 	axi_chan_iowrite32(chan, CH_CFG_H, cfg_hi);
@@ -316,7 +319,7 @@ static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan,
 
 	lli = dma_pool_zalloc(chan->desc_pool, GFP_NOWAIT, &phys);
 	if (unlikely(!lli)) {
-		dev_err(chan2dev(chan), "%s: not enough descriptors available\n",
+		dev_err(vchan_chan_dev(&chan->vc), "%s: not enough descriptors available\n",
 			axi_chan_name(chan));
 		return NULL;
 	}
@@ -342,7 +345,7 @@ static void axi_desc_put(struct axi_dma_desc *desc)
 	kfree(desc->hw_desc);
 	kfree(desc);
 	atomic_sub(descs_put, &chan->descs_allocated);
-	dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n",
+	dev_vdbg(vchan_chan_dev(&chan->vc), "%s: %d descs put, %d still allocated\n",
 		 axi_chan_name(chan), descs_put,
 		 atomic_read(&chan->descs_allocated));
 }
@@ -431,7 +434,7 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan,
 	u8 lms = 0; /* Select AXI0 master for LLI fetching */
 
 	if (unlikely(axi_chan_is_hw_enable(chan))) {
-		dev_err(chan2dev(chan), "%s is non-idle!\n",
+		dev_err(vchan_chan_dev(&chan->vc), "%s is non-idle!\n",
 			axi_chan_name(chan));
 
 		return;
@@ -490,7 +493,7 @@ static void axi_chan_start_first_queued(struct axi_dma_chan *chan)
 		return;
 
 	desc = vd_to_axi_desc(vd);
-	dev_vdbg(chan2dev(chan), "%s: started %u\n", axi_chan_name(chan),
+	dev_vdbg(vchan_chan_dev(&chan->vc), "%s: started %u\n", axi_chan_name(chan),
 		 vd->tx.cookie);
 	axi_chan_block_xfer_start(chan, desc);
 }
@@ -524,23 +527,22 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan)
 
 	/* ASSERT: channel is idle */
 	if (axi_chan_is_hw_enable(chan)) {
-		dev_err(chan2dev(chan), "%s is non-idle!\n",
+		dev_err(vchan_chan_dev(&chan->vc), "%s is non-idle!\n",
 			axi_chan_name(chan));
 		pm_runtime_put(chan->chip->dev);
 		return -EBUSY;
 	}
 
 	/* LLI address must be aligned to a 64-byte boundary */
-	chan->desc_pool = dma_pool_create(dev_name(chan2dev(chan)),
-					  chan->chip->dev,
+	chan->desc_pool = dma_pool_create(dma_chan_name(dchan), chan->chip->dev,
 					  sizeof(struct axi_dma_lli),
 					  64, 0);
 	if (!chan->desc_pool) {
-		dev_err(chan2dev(chan), "No memory for descriptors\n");
+		dev_err(vchan_chan_dev(&chan->vc), "No memory for descriptors\n");
 		pm_runtime_put(chan->chip->dev);
 		return -ENOMEM;
 	}
-	dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan));
+	dev_vdbg(dmaengine_chan_dev(dchan), "%s: allocating\n", axi_chan_name(chan));
 
 	return 0;
 }
@@ -551,7 +553,7 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan)
 
 	/* ASSERT: channel is idle */
 	if (axi_chan_is_hw_enable(chan))
-		dev_err(dchan2dev(dchan), "%s is non-idle!\n",
+		dev_err(dmaengine_chan_dev(dchan), "%s is non-idle!\n",
 			axi_chan_name(chan));
 
 	axi_chan_disable(chan);
@@ -561,7 +563,7 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan)
 
 	dma_pool_destroy(chan->desc_pool);
 	chan->desc_pool = NULL;
-	dev_vdbg(dchan2dev(dchan),
+	dev_vdbg(dmaengine_chan_dev(dchan),
 		 "%s: free resources, descriptor still allocated: %u\n",
 		 axi_chan_name(chan), atomic_read(&chan->descs_allocated));
 
@@ -677,19 +679,19 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 	case DMA_MEM_TO_DEV:
 		reg_width = __ffs(chan->config.dst_addr_width);
 		device_addr = chan->config.dst_addr;
-		ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS |
-			mem_width << CH_CTL_L_SRC_WIDTH_POS |
-			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS |
-			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS;
+		ctllo = FIELD_PREP(CH_CTL_L_DST_WIDTH, reg_width) |
+			FIELD_PREP(CH_CTL_L_SRC_WIDTH, mem_width) |
+			FIELD_PREP(CH_CTL_L_DST_INC, DWAXIDMAC_CH_CTL_L_NOINC) |
+			FIELD_PREP(CH_CTL_L_SRC_INC, DWAXIDMAC_CH_CTL_L_INC);
 		block_ts = len >> mem_width;
 		break;
 	case DMA_DEV_TO_MEM:
 		reg_width = __ffs(chan->config.src_addr_width);
 		device_addr = chan->config.src_addr;
-		ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS |
-			mem_width << CH_CTL_L_DST_WIDTH_POS |
-			DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |
-			DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS;
+		ctllo = FIELD_PREP(CH_CTL_L_SRC_WIDTH, reg_width) |
+			FIELD_PREP(CH_CTL_L_DST_WIDTH, mem_width) |
+			FIELD_PREP(CH_CTL_L_DST_INC, DWAXIDMAC_CH_CTL_L_INC) |
+			FIELD_PREP(CH_CTL_L_SRC_INC, DWAXIDMAC_CH_CTL_L_NOINC);
 		block_ts = len >> reg_width;
 		break;
 	default:
@@ -706,10 +708,10 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 	ctlhi = CH_CTL_H_LLI_VALID;
 
 	if (chan->chip->dw->hdata->restrict_axi_burst_len) {
-		burst_len = chan->chip->dw->hdata->axi_rw_burst_len;
+		burst_len = chan->chip->dw->hdata->axi_rw_burst_len - 1;
 		ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN |
-			 burst_len << CH_CTL_H_ARLEN_POS |
-			 burst_len << CH_CTL_H_AWLEN_POS;
+			 FIELD_PREP(CH_CTL_H_ARLEN, burst_len) |
+			 FIELD_PREP(CH_CTL_H_AWLEN, burst_len);
 	}
 
 	hw_desc->lli->ctl_hi = cpu_to_le32(ctlhi);
@@ -724,8 +726,8 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 
 	hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1);
 
-	ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
-		 DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS;
+	ctllo |= FIELD_PREP(CH_CTL_L_DST_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4) |
+		 FIELD_PREP(CH_CTL_L_SRC_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4);
 	hw_desc->lli->ctl_lo = cpu_to_le32(ctllo);
 
 	set_desc_src_master(hw_desc);
@@ -930,7 +932,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 	u64 llp = 0;
 	u8 lms = 0; /* Select AXI0 master for LLI fetching */
 
-	dev_dbg(chan2dev(chan), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx",
+	dev_dbg(vchan_chan_dev(&chan->vc), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx",
 		axi_chan_name(chan), &src_adr, &dst_adr, len, flags);
 
 	max_block_ts = chan->chip->dw->hdata->block_size[chan->id];
@@ -975,21 +977,21 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 
 		reg = CH_CTL_H_LLI_VALID;
 		if (chan->chip->dw->hdata->restrict_axi_burst_len) {
-			u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len;
+			u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len - 1;
 
-			reg |= (CH_CTL_H_ARLEN_EN |
-				burst_len << CH_CTL_H_ARLEN_POS |
-				CH_CTL_H_AWLEN_EN |
-				burst_len << CH_CTL_H_AWLEN_POS);
+			reg |= CH_CTL_H_ARLEN_EN |
+			       FIELD_PREP(CH_CTL_H_ARLEN, burst_len) |
+			       CH_CTL_H_AWLEN_EN |
+			       FIELD_PREP(CH_CTL_H_AWLEN, burst_len);
 		}
 		hw_desc->lli->ctl_hi = cpu_to_le32(reg);
 
-		reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS |
-		       DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS |
-		       xfer_width << CH_CTL_L_DST_WIDTH_POS |
-		       xfer_width << CH_CTL_L_SRC_WIDTH_POS |
-		       DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS |
-		       DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS);
+		reg = FIELD_PREP(CH_CTL_L_DST_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4) |
+		      FIELD_PREP(CH_CTL_L_SRC_MSIZE, DWAXIDMAC_BURST_TRANS_LEN_4) |
+		      FIELD_PREP(CH_CTL_L_DST_WIDTH, xfer_width) |
+		      FIELD_PREP(CH_CTL_L_SRC_WIDTH, xfer_width) |
+		      FIELD_PREP(CH_CTL_L_DST_INC, DWAXIDMAC_CH_CTL_L_INC) |
+		      FIELD_PREP(CH_CTL_L_SRC_INC, DWAXIDMAC_CH_CTL_L_INC);
 		hw_desc->lli->ctl_lo = cpu_to_le32(reg);
 
 		set_desc_src_master(hw_desc);
@@ -1035,11 +1037,11 @@ static void axi_chan_dump_lli(struct axi_dma_chan *chan,
 			      struct axi_dma_hw_desc *desc)
 {
 	if (!desc->lli) {
-		dev_err(dchan2dev(&chan->vc.chan), "NULL LLI\n");
+		dev_err(vchan_chan_dev(&chan->vc), "NULL LLI\n");
 		return;
 	}
 
-	dev_err(dchan2dev(&chan->vc.chan),
+	dev_err(vchan_chan_dev(&chan->vc),
 		"SAR: 0x%llx DAR: 0x%llx LLP: 0x%llx BTS 0x%x CTL: 0x%x:%08x",
 		le64_to_cpu(desc->lli->sar),
 		le64_to_cpu(desc->lli->dar),
@@ -1052,10 +1054,7 @@ static void axi_chan_dump_lli(struct axi_dma_chan *chan,
 static void axi_chan_list_dump_lli(struct axi_dma_chan *chan,
 				   struct axi_dma_desc *desc_head)
 {
-	int count = atomic_read(&chan->descs_allocated);
-	int i;
-
-	for (i = 0; i < count; i++)
+	for (unsigned int i = 0; i < desc_head->nr_hw_descs; i++)
 		axi_chan_dump_lli(chan, &desc_head->hw_desc[i]);
 }
 
@@ -1071,7 +1070,7 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status)
 	/* The bad descriptor currently is in the head of vc list */
 	vd = vchan_next_desc(&chan->vc);
 	if (!vd) {
-		dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n",
+		dev_err(vchan_chan_dev(&chan->vc), "BUG: %s, IRQ with no descriptors\n",
 			axi_chan_name(chan));
 		goto out;
 	}
@@ -1079,7 +1078,7 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status)
 	list_del(&vd->node);
 
 	/* WARN about bad descriptor */
-	dev_err(chan2dev(chan),
+	dev_err(vchan_chan_dev(&chan->vc),
 		"Bad descriptor submitted for %s, cookie: %d, irq: 0x%08x\n",
 		axi_chan_name(chan), vd->tx.cookie, status);
 	axi_chan_list_dump_lli(chan, vd_to_axi_desc(vd));
@@ -1105,7 +1104,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan)
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
 	if (unlikely(axi_chan_is_hw_enable(chan))) {
-		dev_err(chan2dev(chan), "BUG: %s caught DWAXIDMAC_IRQ_DMA_TRF, but channel not idle!\n",
+		dev_err(vchan_chan_dev(&chan->vc), "BUG: %s caught DWAXIDMAC_IRQ_DMA_TRF, but channel not idle!\n",
 			axi_chan_name(chan));
 		axi_chan_disable(chan);
 	}
@@ -1113,7 +1112,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan)
 	/* The completed descriptor currently is in the head of vc list */
 	vd = vchan_next_desc(&chan->vc);
 	if (!vd) {
-		dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n",
+		dev_err(vchan_chan_dev(&chan->vc), "BUG: %s, IRQ with no descriptors\n",
 			axi_chan_name(chan));
 		goto out;
 	}
@@ -1193,7 +1192,7 @@ static int dma_chan_terminate_all(struct dma_chan *dchan)
 	ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val,
 					!(val & chan_active), 1000, 50000);
 	if (ret == -ETIMEDOUT)
-		dev_warn(dchan2dev(dchan),
+		dev_warn(dmaengine_chan_dev(dchan),
 			 "%s failed to stop\n", axi_chan_name(chan));
 
 	if (chan->direction != DMA_MEM_TO_MEM)
@@ -1210,7 +1209,7 @@ static int dma_chan_terminate_all(struct dma_chan *dchan)
 
 	vchan_dma_desc_free_list(&chan->vc, &head);
 
-	dev_vdbg(dchan2dev(dchan), "terminated: %s\n", axi_chan_name(chan));
+	dev_vdbg(dmaengine_chan_dev(dchan), "terminated: %s\n", axi_chan_name(chan));
 
 	return 0;
 }
@@ -1320,8 +1319,7 @@ static int axi_dma_suspend(struct axi_dma_chip *chip)
 	axi_dma_irq_disable(chip);
 	axi_dma_disable(chip);
 
-	clk_disable_unprepare(chip->core_clk);
-	clk_disable_unprepare(chip->cfgr_clk);
+	clk_bulk_disable_unprepare(ARRAY_SIZE(chip->clks), chip->clks);
 
 	return 0;
 }
@@ -1330,11 +1328,7 @@ static int axi_dma_resume(struct axi_dma_chip *chip)
 {
 	int ret;
 
-	ret = clk_prepare_enable(chip->cfgr_clk);
-	if (ret < 0)
-		return ret;
-
-	ret = clk_prepare_enable(chip->core_clk);
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(chip->clks), chip->clks);
 	if (ret < 0)
 		return ret;
 
@@ -1524,13 +1518,11 @@ static int dw_probe(struct platform_device *pdev)
 
 	chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2);
 
-	chip->core_clk = devm_clk_get(chip->dev, "core-clk");
-	if (IS_ERR(chip->core_clk))
-		return PTR_ERR(chip->core_clk);
-
-	chip->cfgr_clk = devm_clk_get(chip->dev, "cfgr-clk");
-	if (IS_ERR(chip->cfgr_clk))
-		return PTR_ERR(chip->cfgr_clk);
+	chip->clks[0].id = "core-clk";
+	chip->clks[1].id = "cfgr-clk";
+	ret = devm_clk_bulk_get(chip->dev, ARRAY_SIZE(chip->clks), chip->clks);
+	if (ret)
+		return dev_err_probe(chip->dev, ret, "failed to get clocks\n");
 
 	ret = parse_device_properties(chip);
 	if (ret)
@@ -1640,10 +1632,13 @@ static void dw_remove(struct platform_device *pdev)
 	struct dw_axi_dma *dw = chip->dw;
 	struct axi_dma_chan *chan, *_chan;
 	u32 i;
+	int ret;
 
 	/* Enable clk before accessing to registers */
-	clk_prepare_enable(chip->cfgr_clk);
-	clk_prepare_enable(chip->core_clk);
+	ret = clk_bulk_prepare_enable(ARRAY_SIZE(chip->clks), chip->clks);
+	if (ret)
+		dev_warn(chip->dev, "failed to enable clocks before remove: %d\n",
+			 ret);
 	axi_dma_irq_disable(chip);
 	for (i = 0; i < dw->hdata->nr_channels; i++) {
 		axi_chan_disable(&chip->dw->chan[i]);
diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
index 67cc199e24d1fa..d8dd07efe60fe8 100644
--- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
+++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h
@@ -69,8 +69,7 @@ struct axi_dma_chip {
 	int			irq[DMAC_MAX_CHANNELS];
 	void __iomem		*regs;
 	void __iomem		*apb_regs;
-	struct clk		*core_clk;
-	struct clk		*cfgr_clk;
+	struct clk_bulk_data	clks[2];
 	struct dw_axi_dma	*dw;
 };
 
@@ -119,16 +118,6 @@ struct axi_dma_chan_config {
 	u8 hs_sel_src;
 };
 
-static inline struct device *dchan2dev(struct dma_chan *dchan)
-{
-	return &dchan->dev->device;
-}
-
-static inline struct device *chan2dev(struct axi_dma_chan *chan)
-{
-	return &chan->vc.chan.dev->device;
-}
-
 static inline struct axi_dma_desc *vd_to_axi_desc(struct virt_dma_desc *vd)
 {
 	return container_of(vd, struct axi_dma_desc, vd);
@@ -234,9 +223,9 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan)
 
 /* CH_CTL_H */
 #define CH_CTL_H_ARLEN_EN		BIT(6)
-#define CH_CTL_H_ARLEN_POS		7
+#define CH_CTL_H_ARLEN			GENMASK(14, 7)
 #define CH_CTL_H_AWLEN_EN		BIT(15)
-#define CH_CTL_H_AWLEN_POS		16
+#define CH_CTL_H_AWLEN			GENMASK(23, 16)
 
 enum {
 	DWAXIDMAC_ARWLEN_1		= 0,
@@ -258,8 +247,8 @@ enum {
 /* CH_CTL_L */
 #define CH_CTL_L_LAST_WRITE_EN		BIT(30)
 
-#define CH_CTL_L_DST_MSIZE_POS		18
-#define CH_CTL_L_SRC_MSIZE_POS		14
+#define CH_CTL_L_DST_MSIZE		GENMASK(21, 18)
+#define CH_CTL_L_SRC_MSIZE		GENMASK(17, 14)
 
 enum {
 	DWAXIDMAC_BURST_TRANS_LEN_1	= 0,
@@ -274,11 +263,11 @@ enum {
 	DWAXIDMAC_BURST_TRANS_LEN_1024
 };
 
-#define CH_CTL_L_DST_WIDTH_POS		11
-#define CH_CTL_L_SRC_WIDTH_POS		8
+#define CH_CTL_L_DST_WIDTH		GENMASK(13, 11)
+#define CH_CTL_L_SRC_WIDTH		GENMASK(10, 8)
 
-#define CH_CTL_L_DST_INC_POS		6
-#define CH_CTL_L_SRC_INC_POS		4
+#define CH_CTL_L_DST_INC		BIT(6)
+#define CH_CTL_L_SRC_INC		BIT(4)
 enum {
 	DWAXIDMAC_CH_CTL_L_INC		= 0,
 	DWAXIDMAC_CH_CTL_L_NOINC
@@ -288,17 +277,17 @@ enum {
 #define CH_CTL_L_SRC_MAST		BIT(0)
 
 /* CH_CFG_H */
-#define CH_CFG_H_PRIORITY_POS		17
-#define CH_CFG_H_DST_PER_POS		12
-#define CH_CFG_H_SRC_PER_POS		7
-#define CH_CFG_H_HS_SEL_DST_POS		4
-#define CH_CFG_H_HS_SEL_SRC_POS		3
+#define CH_CFG_H_PRIORITY		GENMASK(19, 17)
+#define CH_CFG_H_DST_PER		GENMASK(15, 12)
+#define CH_CFG_H_SRC_PER		GENMASK(10, 7)
+#define CH_CFG_H_HS_SEL_DST		BIT(4)
+#define CH_CFG_H_HS_SEL_SRC		BIT(3)
 enum {
 	DWAXIDMAC_HS_SEL_HW		= 0,
 	DWAXIDMAC_HS_SEL_SW
 };
 
-#define CH_CFG_H_TT_FC_POS		0
+#define CH_CFG_H_TT_FC			GENMASK(2, 0)
 enum {
 	DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC	= 0,
 	DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC,
@@ -311,8 +300,8 @@ enum {
 };
 
 /* CH_CFG_L */
-#define CH_CFG_L_DST_MULTBLK_TYPE_POS	2
-#define CH_CFG_L_SRC_MULTBLK_TYPE_POS	0
+#define CH_CFG_L_DST_MULTBLK_TYPE	GENMASK(3, 2)
+#define CH_CFG_L_SRC_MULTBLK_TYPE	GENMASK(1, 0)
 enum {
 	DWAXIDMAC_MBLK_TYPE_CONTIGUOUS	= 0,
 	DWAXIDMAC_MBLK_TYPE_RELOAD,
@@ -321,13 +310,13 @@ enum {
 };
 
 /* CH_CFG2 */
-#define CH_CFG2_L_SRC_PER_POS		4
-#define CH_CFG2_L_DST_PER_POS		11
+#define CH_CFG2_L_SRC_PER		GENMASK(9, 4)
+#define CH_CFG2_L_DST_PER		GENMASK(16, 11)
 
-#define CH_CFG2_H_TT_FC_POS		0
-#define CH_CFG2_H_HS_SEL_SRC_POS	3
-#define CH_CFG2_H_HS_SEL_DST_POS	4
-#define CH_CFG2_H_PRIORITY_POS		20
+#define CH_CFG2_H_TT_FC			GENMASK(2, 0)
+#define CH_CFG2_H_HS_SEL_SRC		BIT(3)
+#define CH_CFG2_H_HS_SEL_DST		BIT(4)
+#define CH_CFG2_H_PRIORITY		GENMASK(19, 15)
 
 /**
  * DW AXI DMA channel interrupts
diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index 03b2c2188351c8..e2eba6d72b5326 100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -13,6 +13,7 @@
 #include <linux/dmaengine.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
+#include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/dma/edma.h>
 #include <linux/dma-mapping.h>
@@ -30,9 +31,9 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
 	return container_of(vd, struct dw_edma_desc, vd);
 }
 
-enum dw_edma_irq_event {
-	DW_EDMA_IRQ_DONE	= BIT(0),
-	DW_EDMA_IRQ_ABORT	= BIT(1),
+enum dw_edma_deferred_event {
+	DW_EDMA_DEFERRED_DONE	= BIT(0),
+	DW_EDMA_DEFERRED_ABORT	= BIT(1),
 };
 
 static inline
@@ -51,13 +52,19 @@ dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
 {
 	struct dw_edma_desc *desc;
 
+	/*
+	 * For now, a descriptor that does not fit would stall the channel
+	 * forever: reject it up front.
+	 */
+	if (!chan->non_ll && nburst > chan->ll_max - 1)
+		return NULL;
+
 	desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
 	if (unlikely(!desc))
 		return NULL;
 
 	desc->chan = chan;
 	desc->nburst = nburst;
-	desc->cb = true;
 
 	return desc;
 }
@@ -67,38 +74,122 @@ static void vchan_free_desc(struct virt_dma_desc *vdesc)
 	kfree(vd2dw_edma_desc(vdesc));
 }
 
-static void dw_edma_core_start(struct dw_edma_desc *desc, bool first)
+/* Must be called with vc.lock held. */
+static void
+dw_edma_set_request(struct dw_edma_chan *chan, enum dw_edma_request request)
 {
-	struct dw_edma_chan *chan = desc->chan;
-	size_t i = 0;
+	chan->request = request;
+}
 
-	if (chan->non_ll) {
-		chan->dw->core->non_ll_start(chan, &desc->burst[desc->start_burst]);
-		desc->done_burst = desc->start_burst;
-		desc->start_burst += 1;
+static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
+					enum dmaengine_tx_result result)
+{
+	u32 residue = 0;
+	struct dw_edma_desc *desc;
+	struct dmaengine_result *res;
+
+	if (!vd->tx.callback_result)
 		return;
+
+	desc = vd2dw_edma_desc(vd);
+	if (desc) {
+		residue = desc->alloc_sz;
+
+		if (result == DMA_TRANS_NOERROR)
+			residue -= desc->burst[desc->start_burst - 1].xfer_sz;
+		else if (desc->done_burst)
+			residue -= desc->burst[desc->done_burst - 1].xfer_sz;
 	}
 
-	for (i = 0; i + desc->start_burst < desc->nburst; i++) {
-		u32 idx = i + desc->start_burst;
+	res = &vd->tx_result;
+	res->result = result;
+	res->residue = residue;
+}
 
-		if (i == chan->ll_max)
-			break;
+static void dw_edma_core_reset_ll(struct dw_edma_chan *chan)
+{
+	u32 i;
+
+	chan->ll_head = 0;
+	chan->ll_done = 0;
+	/* Drop stale CB bits before reusing the circular LL ring. */
+	for (i = 0; i < chan->ll_max; i++)
+		dw_edma_core_ll_clear(chan, i);
+	chan->cb = true;
+
+	dw_edma_core_ll_link(chan, chan->ll_max, chan->cb,
+			     chan->ll_region.paddr);
+
+	dw_edma_core_ch_enable(chan);
+	chan->ll_valid = true;
+}
 
-		dw_edma_core_ll_data(chan, &desc->burst[idx],
-				     i, desc->cb,
-				     idx == desc->nburst - 1 || i == chan->ll_max - 1);
+static u32 dw_edma_core_get_ll_dist(struct dw_edma_chan *chan, u32 from, u32 to)
+{
+	return (to + chan->ll_max - from) % chan->ll_max;
+}
+
+static u32 dw_edma_core_get_used_num(struct dw_edma_chan *chan)
+{
+	return dw_edma_core_get_ll_dist(chan, chan->ll_done, chan->ll_head);
+}
+
+static u32 dw_edma_core_get_free_num(struct dw_edma_chan *chan)
+{
+	/* Keep one data entry free so equal indices mean an empty ring. */
+	return chan->ll_max - 1 - dw_edma_core_get_used_num(chan);
+}
+
+static bool dw_edma_ll_pending(struct dw_edma_chan *chan)
+{
+	return chan->ll_head != chan->ll_done;
+}
+
+static void dw_edma_core_ll_start(struct dw_edma_desc *desc)
+{
+	struct dw_edma_chan *chan = desc->chan;
+	size_t i;
+	u32 free;
+
+	free = dw_edma_core_get_free_num(chan);
+	for (i = desc->start_burst; i < desc->nburst && free; i++, free--) {
+		/*
+		 * Refresh the link element before filling the last data slot so
+		 * the next lap has the updated CB value.
+		 */
+		if (chan->ll_head == chan->ll_max - 1)
+			dw_edma_core_ll_link(chan, chan->ll_max, chan->cb,
+					     chan->ll_region.paddr);
+
+		dw_edma_core_ll_data(chan, &desc->burst[i],
+				     chan->ll_head, chan->cb,
+				     i == desc->nburst - 1 || free == 1);
+
+		chan->ll_head++;
+
+		if (chan->ll_head == chan->ll_max) {
+			chan->cb = !chan->cb;
+			chan->ll_head = 0;
+		}
 	}
 
 	desc->done_burst = desc->start_burst;
-	desc->start_burst += i;
+	desc->start_burst = i;
+}
 
-	dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr);
+static void dw_edma_core_start(struct dw_edma_desc *desc)
+{
+	struct dw_edma_chan *chan = desc->chan;
 
-	if (first)
-		dw_edma_core_ch_enable(chan);
+	if (chan->non_ll) {
+		chan->dw->core->non_ll_start(chan,
+					     &desc->burst[desc->start_burst]);
+		desc->done_burst = desc->start_burst;
+		desc->start_burst += 1;
+		return;
+	}
 
-	dw_edma_core_ch_doorbell(chan);
+	dw_edma_core_ll_start(desc);
 }
 
 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
@@ -114,9 +205,10 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 	if (!desc)
 		return 0;
 
-	dw_edma_core_start(desc, !desc->start_burst);
+	if (!chan->non_ll && !chan->ll_valid)
+		dw_edma_core_reset_ll(chan);
 
-	desc->cb = !desc->cb;
+	dw_edma_core_start(desc);
 
 	return 1;
 }
@@ -150,6 +242,46 @@ static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan)
 	dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted);
 }
 
+/* Must be called with vc.lock held after the channel has stopped. */
+static void dw_edma_finish_termination(struct dw_edma_chan *chan)
+{
+	dw_edma_terminate_all_descs(chan);
+
+	/* Preserve a clean ring; resync only if entries remain published. */
+	if (!chan->non_ll && dw_edma_ll_pending(chan))
+		dw_edma_core_reset_ll(chan);
+
+	dw_edma_set_request(chan, EDMA_REQ_NONE);
+	chan->status = EDMA_ST_IDLE;
+}
+
+static void dw_edma_core_ll_sync(struct dw_edma_chan *chan)
+{
+	/*
+	 * Remote controller registers and LL memory may be reached through
+	 * different paths. Complete posted LL writes before the doorbell.
+	 */
+	if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
+		readl(chan->ll_region.vaddr.io);
+}
+
+/* Must be called with vc.lock held for an LL channel. */
+static void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan)
+{
+	dw_edma_core_ll_sync(chan);
+	dw_edma_core_do_ch_doorbell(chan);
+}
+
+/* Must be called with vc.lock held. */
+static void dw_edma_core_ch_maybe_doorbell(struct dw_edma_chan *chan)
+{
+	if (chan->non_ll || chan->request != EDMA_REQ_NONE ||
+	    chan->status != EDMA_ST_BUSY || !dw_edma_ll_pending(chan))
+		return;
+
+	dw_edma_core_ch_doorbell(chan);
+}
+
 static void dw_edma_device_caps(struct dma_chan *dchan,
 				struct dma_slave_caps *caps)
 {
@@ -253,7 +385,7 @@ static int dw_edma_device_pause(struct dma_chan *dchan)
 	else if (chan->request != EDMA_REQ_NONE)
 		err = -EPERM;
 	else
-		chan->request = EDMA_REQ_PAUSE;
+		dw_edma_set_request(chan, EDMA_REQ_PAUSE);
 
 	return err;
 }
@@ -275,6 +407,7 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
 		chan->status = EDMA_ST_BUSY;
 		if (!dw_edma_start_transfer(chan))
 			chan->status = EDMA_ST_IDLE;
+		dw_edma_core_ch_maybe_doorbell(chan);
 	}
 
 	return err;
@@ -290,24 +423,22 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
 	if (!chan->configured) {
 		dw_edma_terminate_all_descs(chan);
 	} else if (chan->status == EDMA_ST_PAUSE) {
-		dw_edma_terminate_all_descs(chan);
-		chan->status = EDMA_ST_IDLE;
+		dw_edma_finish_termination(chan);
 	} else if (chan->status == EDMA_ST_IDLE) {
-		dw_edma_terminate_all_descs(chan);
+		dw_edma_finish_termination(chan);
 	} else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
 		/*
 		 * The channel is in a false BUSY state, probably didn't
 		 * receive or lost an interrupt
 		 */
-		dw_edma_terminate_all_descs(chan);
-		chan->status = EDMA_ST_IDLE;
+		dw_edma_finish_termination(chan);
 	} else if (chan->request > EDMA_REQ_PAUSE) {
 		err = -EPERM;
 	} else {
-		chan->request = EDMA_REQ_STOP;
+		dw_edma_set_request(chan, EDMA_REQ_STOP);
 	}
 	if (chan->status == EDMA_ST_IDLE)
-		chan->request = EDMA_REQ_NONE;
+		dw_edma_set_request(chan, EDMA_REQ_NONE);
 
 	return err;
 }
@@ -323,6 +454,7 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
 	    chan->status == EDMA_ST_IDLE) {
 		chan->status = EDMA_ST_BUSY;
 		dw_edma_start_transfer(chan);
+		dw_edma_core_ch_maybe_doorbell(chan);
 	}
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
@@ -588,77 +720,62 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
 	return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
 }
 
-static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
-					enum dmaengine_tx_result result)
+/* Must be called with vc.lock held. */
+static void dw_edma_done_interrupt_locked(struct dw_edma_chan *chan)
 {
-	u32 residue = 0;
 	struct dw_edma_desc *desc;
-	struct dmaengine_result *res;
+	struct virt_dma_desc *vd;
 
-	if (!vd->tx.callback_result)
+	lockdep_assert_held(&chan->vc.lock);
+
+	if (chan->status == EDMA_ST_PAUSE)
 		return;
 
-	desc = vd2dw_edma_desc(vd);
-	if (desc) {
-		residue = desc->alloc_sz;
+	switch (chan->request) {
+	case EDMA_REQ_NONE:
+	case EDMA_REQ_PAUSE:
+		vd = vchan_next_desc(&chan->vc);
+		if (!vd)
+			break;
 
-		if (result == DMA_TRANS_NOERROR)
-			residue -= desc->burst[desc->start_burst - 1].xfer_sz;
-		else if (desc->done_burst)
-			residue -= desc->burst[desc->done_burst - 1].xfer_sz;
-	}
+		desc = vd2dw_edma_desc(vd);
+		if (desc->start_burst >= desc->nburst) {
+			dw_hdma_set_callback_result(vd, DMA_TRANS_NOERROR);
+			list_del(&vd->node);
+			vchan_cookie_complete(vd);
+			if (!chan->non_ll)
+				chan->ll_done = chan->ll_head;
+		}
 
-	res = &vd->tx_result;
-	res->result = result;
-	res->residue = residue;
+		if (chan->request == EDMA_REQ_PAUSE) {
+			dw_edma_set_request(chan, EDMA_REQ_NONE);
+			chan->status = EDMA_ST_PAUSE;
+			break;
+		}
+
+		chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
+		break;
+
+	case EDMA_REQ_STOP:
+		vd = vchan_next_desc(&chan->vc);
+		if (!vd)
+			break;
+
+		dw_edma_finish_termination(chan);
+		break;
+
+	default:
+		break;
+	}
+	dw_edma_core_ch_maybe_doorbell(chan);
 }
 
 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 {
-	struct dw_edma_desc *desc;
-	struct virt_dma_desc *vd;
 	unsigned long flags;
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
-	if (chan->status == EDMA_ST_PAUSE) {
-		spin_unlock_irqrestore(&chan->vc.lock, flags);
-		return;
-	}
-
-	vd = vchan_next_desc(&chan->vc);
-	if (vd) {
-		switch (chan->request) {
-		case EDMA_REQ_NONE:
-		case EDMA_REQ_PAUSE:
-			desc = vd2dw_edma_desc(vd);
-			if (desc->start_burst >= desc->nburst) {
-				dw_hdma_set_callback_result(vd,
-							    DMA_TRANS_NOERROR);
-				list_del(&vd->node);
-				vchan_cookie_complete(vd);
-			}
-
-			if (chan->request == EDMA_REQ_PAUSE) {
-				chan->request = EDMA_REQ_NONE;
-				chan->status = EDMA_ST_PAUSE;
-				break;
-			}
-
-			/* Continue transferring if there are remaining chunks or issued requests.
-			 */
-			chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
-			break;
-
-		case EDMA_REQ_STOP:
-			dw_edma_terminate_all_descs(chan);
-			chan->request = EDMA_REQ_NONE;
-			chan->status = EDMA_ST_IDLE;
-			break;
-
-		default:
-			break;
-		}
-	}
+	dw_edma_done_interrupt_locked(chan);
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
 
@@ -676,7 +793,9 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 		list_del(&vd->node);
 		vchan_cookie_complete(vd);
 	}
-	chan->request = EDMA_REQ_NONE;
+	if (!chan->non_ll)
+		dw_edma_core_reset_ll(chan);
+	dw_edma_set_request(chan, EDMA_REQ_NONE);
 	chan->status = EDMA_ST_IDLE;
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
 }
@@ -690,28 +809,32 @@ static void dw_edma_irq_work(struct work_struct *work)
 	do {
 		events = atomic_xchg(&chan->irq_pending, 0);
 
-		if (events & DW_EDMA_IRQ_DONE)
+		if (events & DW_EDMA_DEFERRED_DONE)
 			dw_edma_done_interrupt(chan);
-		if (events & DW_EDMA_IRQ_ABORT)
+		if (events & DW_EDMA_DEFERRED_ABORT)
 			dw_edma_abort_interrupt(chan);
 	} while (atomic_read(&chan->irq_pending));
 }
 
 static void dw_edma_queue_irq_work(struct dw_edma_chan *chan,
-				   enum dw_edma_irq_event event)
+				   unsigned int events)
 {
-	atomic_or(event, &chan->irq_pending);
+	atomic_or(events, &chan->irq_pending);
 	queue_work(chan->dw->wq, &chan->irq_work);
 }
 
-static void dw_edma_done_interrupt_deferred(struct dw_edma_chan *chan)
+static void dw_edma_record_irq(struct dw_edma_chan *chan, unsigned int events)
 {
-	dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_DONE);
-}
+	unsigned int pending = 0;
 
-static void dw_edma_abort_interrupt_deferred(struct dw_edma_chan *chan)
-{
-	dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_ABORT);
+	if (events & (DW_EDMA_IRQ_DONE | DW_EDMA_IRQ_PROGRESS |
+		      DW_EDMA_IRQ_STOP))
+		pending |= DW_EDMA_DEFERRED_DONE;
+	if (events & DW_EDMA_IRQ_ABORT)
+		pending |= DW_EDMA_DEFERRED_ABORT;
+
+	if (pending)
+		dw_edma_queue_irq_work(chan, pending);
 }
 
 static void dw_edma_emul_irq_ack(struct irq_data *d)
@@ -812,8 +935,7 @@ static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data)
 	struct dw_edma_irq *dw_irq = data;
 
 	return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
-				       dw_edma_done_interrupt_deferred,
-				       dw_edma_abort_interrupt_deferred);
+				       dw_edma_record_irq);
 }
 
 static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data)
@@ -821,8 +943,7 @@ static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data)
 	struct dw_edma_irq *dw_irq = data;
 
 	return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
-				       dw_edma_done_interrupt_deferred,
-				       dw_edma_abort_interrupt_deferred);
+				       dw_edma_record_irq);
 }
 
 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
@@ -863,6 +984,9 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
 	if (chan->status != EDMA_ST_IDLE)
 		return -EBUSY;
 
+	/* The hardware context may have been invalidated while unowned. */
+	chan->ll_valid = false;
+
 	return 0;
 }
 
@@ -954,6 +1078,13 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 		else
 			chan->ll_region = chip->ll_region_rd[chan->id];
 
+		if (!chip->cfg_non_ll && chan->ll_region.sz < 3 * EDMA_LL_SZ) {
+			dev_err(dev,
+				"channel %s[%u]: LL region has fewer than 2 data entries\n",
+				str_write_read(chan->dir == EDMA_DIR_WRITE),
+				chan->id);
+			return -EINVAL;
+		}
 		chan->ll_max = chan->ll_region.sz / EDMA_LL_SZ - 1;
 
 		dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h
index f6a5ad31756723..76fd5a41a70746 100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -41,6 +41,13 @@ enum dw_edma_xfer_type {
 	EDMA_XFER_INTERLEAVED
 };
 
+enum dw_edma_irq_event {
+	DW_EDMA_IRQ_DONE	= BIT(0),
+	DW_EDMA_IRQ_PROGRESS	= BIT(1),
+	DW_EDMA_IRQ_STOP	= BIT(2),
+	DW_EDMA_IRQ_ABORT	= BIT(3),
+};
+
 struct dw_edma_chan;
 struct dw_edma_chunk;
 
@@ -60,7 +67,6 @@ struct dw_edma_desc {
 
 	size_t				done_burst;
 	size_t				start_burst;
-	u8				cb;
 	size_t				nburst;
 	struct dw_edma_burst            burst[] __counted_by(nburst);
 };
@@ -72,8 +78,32 @@ struct dw_edma_chan {
 	enum dw_edma_dir		dir;
 	u8				func_no;
 
-	u32				ll_max;
+	/*
+	 * New LL entries are appended at ll_head. Entries between ll_done
+	 * and ll_head, modulo the LL ring, are owned by DMA; the rest are
+	 * owned by software.
+	 *
+	 *   software-owned      DMA-owned       software-owned
+	 * +---------------+-------------------+---------------+
+	 * ^               ^                   ^
+	 * 0             ll_done             ll_head
+	 *
+	 * The link entry points back to the region start. ll_head == ll_done
+	 * means all entries are software-owned and previous DMA work is
+	 * done.
+	 *
+	 * Software always keeps at least one free entry, so the ring is
+	 * never completely DMA-owned. That keeps a hardware-reported physical
+	 * LL index unique within the current ll_done..ll_head producer window.
+	 */
+	u32				ll_head;
+	u32				ll_done;
+
+	u32				ll_max;		/* Data entries */
 	struct dw_edma_region		ll_region;	/* Linked list */
+	bool				ll_valid;	/* LL context programmed */
+
+	bool				cb;
 
 	struct msi_msg			msi;
 
@@ -124,7 +154,8 @@ struct dw_edma {
 	const struct dw_edma_core_ops	*core;
 };
 
-typedef void (*dw_edma_handler_t)(struct dw_edma_chan *);
+typedef void (*dw_edma_handler_t)(struct dw_edma_chan *chan,
+				  unsigned int events);
 
 struct dw_edma_core_ops {
 	void (*off)(struct dw_edma *dw);
@@ -133,11 +164,14 @@ struct dw_edma_core_ops {
 	u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir);
 	enum dma_status (*ch_status)(struct dw_edma_chan *chan);
 	irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
-				  dw_edma_handler_t done, dw_edma_handler_t abort);
+				  dw_edma_handler_t handler);
 	void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child);
 	void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
 			u32 idx, bool cb, bool irq);
 	void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr);
+	void (*ll_clear)(struct dw_edma_chan *chan, u32 idx);
+	int (*ll_cur_idx)(struct dw_edma_chan *chan);
+	void (*ll_irq_clear)(struct dw_edma_chan *chan);
 	void (*ch_doorbell)(struct dw_edma_chan *chan);
 	void (*ch_enable)(struct dw_edma_chan *chan);
 	void (*ch_config)(struct dw_edma_chan *chan);
@@ -181,6 +215,15 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan)
 	return vc2dw_edma_chan(to_virt_chan(dchan));
 }
 
+/*
+ * Return the current LL entry index. A negative value means that the channel
+ * context is not initialized or was lost after a link reset.
+ */
+static inline int dw_edma_core_ll_cur_idx(struct dw_edma_chan *chan)
+{
+	return chan->dw->core->ll_cur_idx(chan);
+}
+
 static inline u64 dw_edma_core_get_ll_paddr(struct dw_edma_chan *chan)
 {
 	if (chan->dir == EDMA_DIR_WRITE)
@@ -221,9 +264,9 @@ enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan)
 
 static inline irqreturn_t
 dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
-			dw_edma_handler_t done, dw_edma_handler_t abort)
+			dw_edma_handler_t handler)
 {
-	return dw_irq->dw->core->handle_int(dw_irq, dir, done, abort);
+	return dw_irq->dw->core->handle_int(dw_irq, dir, handler);
 }
 
 static inline
@@ -245,7 +288,17 @@ dw_edma_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
 	chan->dw->core->ll_link(chan, idx, cb, addr);
 }
 
-static inline void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan)
+static inline void dw_edma_core_ll_clear(struct dw_edma_chan *chan, u32 idx)
+{
+	chan->dw->core->ll_clear(chan, idx);
+}
+
+static inline void dw_edma_core_ll_irq_clear(struct dw_edma_chan *chan)
+{
+	chan->dw->core->ll_irq_clear(chan);
+}
+
+static inline void dw_edma_core_do_ch_doorbell(struct dw_edma_chan *chan)
 {
 	chan->dw->core->ch_doorbell(chan);
 }
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index 9f237ba916deaa..fab4faf68b7748 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -33,6 +33,8 @@
 #define DW_PCIE_XILINX_MDB_VSEC_ID		0x20
 #define DW_PCIE_XILINX_MDB_VSEC_DMA_BAR		GENMASK(10, 8)
 #define DW_PCIE_XILINX_MDB_VSEC_DMA_MAP		GENMASK(2, 0)
+/* AMD CPM6 (Xilinx) supported cap */
+#define DW_PCIE_XILINX_CPM6_VSEC_CH_SEP		GENMASK(18, 16)
 #define DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH	GENMASK(9, 0)
 #define DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH	GENMASK(25, 16)
 
@@ -76,6 +78,7 @@ struct dw_edma_pcie_data {
 	u16				rd_ch_cnt;
 	u64				devmem_phys_off;
 	bool				cfg_non_ll;
+	u32				ch_space_sz;
 };
 
 struct dw_edma_pcie_match_data {
@@ -130,6 +133,7 @@ static const struct dw_edma_pcie_data snps_edda_data = {
 	.irqs				= 1,
 	.wr_ch_cnt			= 2,
 	.rd_ch_cnt			= 2,
+	.ch_space_sz			= 256,
 };
 
 static const struct dw_edma_pcie_data xilinx_mdb_data = {
@@ -143,6 +147,7 @@ static const struct dw_edma_pcie_data xilinx_mdb_data = {
 	.irqs				= 1,
 	.wr_ch_cnt			= 8,
 	.rd_ch_cnt			= 8,
+	.ch_space_sz			= 256,
 };
 
 static const struct dw_edma_pcie_data xilinx_cpm6_dma_data = {
@@ -156,6 +161,7 @@ static const struct dw_edma_pcie_data xilinx_cpm6_dma_data = {
 	.irqs				= 1,
 	.wr_ch_cnt			= 8,
 	.rd_ch_cnt			= 8,
+	.ch_space_sz			= 512,
 };
 
 static void dw_edma_set_chan_region_offset(struct dw_edma_pcie_data *pdata,
@@ -208,6 +214,13 @@ static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr)
 	return pci_irq_vector(to_pci_dev(dev), nr);
 }
 
+static u32 dw_edma_get_ch_space_sz(u32 val)
+{
+	if (val > 0 && val <= 7)
+		return 256 << val;
+	return 256;
+}
+
 static u64 dw_edma_pcie_address(struct device *dev, phys_addr_t cpu_addr)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -298,6 +311,10 @@ static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev,
 	pdata->mf = map;
 	pdata->rg.bar = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_BAR, val);
 
+	if (pdev->device == PCI_DEVICE_ID_XILINX_B00F)
+		pdata->ch_space_sz = dw_edma_get_ch_space_sz
+					(FIELD_GET(DW_PCIE_XILINX_CPM6_VSEC_CH_SEP, val));
+
 	pci_read_config_dword(pdev, vsec + 0xc, &val);
 	pdata->wr_ch_cnt = min(pdata->wr_ch_cnt,
 			       FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH, val));
@@ -477,6 +494,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
 	chip->nr_irqs = nr_irqs;
 	chip->ops = match->plat_ops;
 	chip->cfg_non_ll = dma_data->cfg_non_ll;
+	chip->ch_space_sz = dma_data->ch_space_sz;
 
 	chip->ll_wr_cnt = dma_data->wr_ch_cnt;
 	chip->ll_rd_cnt = dma_data->rd_ch_cnt;
diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c
index abc1bd4c0be2eb..c053cb4c28444f 100644
--- a/drivers/dma/dw-edma/dw-edma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-edma-v0-core.c
@@ -336,7 +336,7 @@ static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
 
 static irqreturn_t
 dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
-			   dw_edma_handler_t done, dw_edma_handler_t abort)
+			   dw_edma_handler_t handler)
 {
 	struct dw_edma *dw = dw_irq->dw;
 	unsigned long total, pos, val;
@@ -375,7 +375,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 			continue;
 
 		dw_edma_v0_core_clear_done_int(chan);
-		done(chan);
+		handler(chan, DW_EDMA_IRQ_DONE);
 
 		ret = IRQ_HANDLED;
 	}
@@ -389,7 +389,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 			continue;
 
 		dw_edma_v0_core_clear_abort_int(chan);
-		abort(chan);
+		handler(chan, DW_EDMA_IRQ_ABORT);
 
 		ret = IRQ_HANDLED;
 	}
@@ -480,20 +480,6 @@ static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan)
 		  upper_32_bits(chan->ll_region.paddr));
 }
 
-static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan)
-{
-	/*
-	 * In case of remote eDMA engine setup, the DW PCIe RP/EP internal
-	 * configuration registers and application memory are normally accessed
-	 * over different buses. Ensure LL-data reaches the memory before the
-	 * doorbell register is toggled by issuing the dummy-read from the remote
-	 * LL memory in a hope that the MRd TLP will return only after the
-	 * last MWr TLP is completed
-	 */
-	if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
-		readl(chan->ll_region.vaddr.io);
-}
-
 static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
@@ -605,17 +591,49 @@ dw_edma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
 	dw_edma_v0_write_ll_link(chan, idx, control, addr);
 }
 
+static void dw_edma_v0_core_ll_clear(struct dw_edma_chan *chan, u32 idx)
+{
+	ptrdiff_t ofs = idx * sizeof(struct dw_edma_v0_lli);
+
+	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
+		struct dw_edma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs;
+
+		lli->control = 0;
+	} else {
+		struct dw_edma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs;
+
+		writel(0, &lli->control);
+	}
+}
+
 static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
 
-	dw_edma_v0_sync_ll_data(chan);
-
 	/* Doorbell */
 	SET_RW_32(dw, chan->dir, doorbell,
 		  FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
 }
 
+static int dw_edma_v0_core_ll_cur_idx(struct dw_edma_chan *chan)
+{
+	u32 base, val;
+
+	val = GET_CH_32(chan->dw, chan->dir, chan->id, llp.lsb);
+	base = lower_32_bits(dw_edma_core_get_ll_paddr(chan));
+
+	/*
+	 * LL regions stay within one 4 GiB address window. Reject an all-ones
+	 * MMIO value. If the low word is zero, use the high word to distinguish
+	 * a nonzero boundary address from an unprogrammed all-zero context.
+	 */
+	if (val == U32_MAX ||
+	    (!val && !GET_CH_32(chan->dw, chan->dir, chan->id, llp.msb)))
+		return -EINVAL;
+
+	return (val - base) / EDMA_LL_SZ;
+}
+
 /* eDMA debugfs callbacks */
 static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
 {
@@ -650,6 +668,9 @@ static const struct dw_edma_core_ops dw_edma_v0_core = {
 	.handle_int = dw_edma_v0_core_handle_int,
 	.ll_data = dw_edma_v0_core_ll_data,
 	.ll_link = dw_edma_v0_core_ll_link,
+	.ll_clear = dw_edma_v0_core_ll_clear,
+	.ll_cur_idx = dw_edma_v0_core_ll_cur_idx,
+	.ll_irq_clear = dw_edma_v0_core_clear_done_int,
 	.ch_doorbell = dw_edma_v0_core_ch_doorbell,
 	.ch_enable = dw_edma_v0_core_ch_enable,
 	.ch_config = dw_edma_v0_core_ch_config,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c
index 36ee72efcd3177..d42cb4ad49a267 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-core.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c
@@ -23,18 +23,23 @@ enum dw_hdma_control {
 	DW_HDMA_V0_LLE					= BIT(9),
 };
 
-static inline struct dw_hdma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
-{
-	return dw->chip->reg_base;
-}
-
 static inline struct dw_hdma_v0_ch_regs __iomem *
 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
 {
-	if (dir == EDMA_DIR_WRITE)
-		return &(__dw_regs(dw)->ch[ch].wr);
-	else
-		return &(__dw_regs(dw)->ch[ch].rd);
+	u32 ch_base;
+
+	/*
+	 * For Write, the channel register index starts at
+	 * wr_base(ch_idx) = (2 * ch_idx) * ch_space_sz
+	 *
+	 * For Read channel,
+	 * rd_base(ch_idx) = (2 * ch_idx + 1) * ch_space_sz
+	 */
+	ch_base = 2 * ch;
+	if (dir == EDMA_DIR_READ)
+		ch_base += 1;
+
+	return dw->chip->reg_base + (ch_base * dw->chip->ch_space_sz);
 }
 
 #define SET_CH_32(dw, dir, ch, name, value) \
@@ -174,7 +179,7 @@ static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
 
 static irqreturn_t
 dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
-			   dw_edma_handler_t done, dw_edma_handler_t abort)
+			   dw_edma_handler_t handler)
 {
 	struct dw_edma *dw = dw_irq->dw;
 	unsigned long total, pos, val;
@@ -200,14 +205,14 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 		val = dw_hdma_v0_core_status_int(chan);
 		if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
 			dw_hdma_v0_core_clear_done_int(chan);
-			done(chan);
+			handler(chan, DW_EDMA_IRQ_STOP);
 
 			ret = IRQ_HANDLED;
 		}
 
 		if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
 			dw_hdma_v0_core_clear_abort_int(chan);
-			abort(chan);
+			handler(chan, DW_EDMA_IRQ_ABORT);
 
 			ret = IRQ_HANDLED;
 		}
@@ -282,20 +287,6 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan)
 		  HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
 }
 
-static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan)
-{
-	/*
-	 * In case of remote HDMA engine setup, the DW PCIe RP/EP internal
-	 * configuration registers and application memory are normally accessed
-	 * over different buses. Ensure LL-data reaches the memory before the
-	 * doorbell register is toggled by issuing the dummy-read from the remote
-	 * LL memory in a hope that the MRd TLP will return only after the
-	 * last MWr TLP is completed
-	 */
-	if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
-		readl(chan->ll_region.vaddr.io);
-}
-
 static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan,
 					 struct dw_edma_burst *child)
 {
@@ -374,16 +365,48 @@ dw_hdma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
 	dw_hdma_v0_write_ll_link(chan, idx, control, addr);
 }
 
+static void dw_hdma_v0_core_ll_clear(struct dw_edma_chan *chan, u32 idx)
+{
+	ptrdiff_t ofs = idx * sizeof(struct dw_hdma_v0_lli);
+
+	if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
+		struct dw_hdma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs;
+
+		lli->control = 0;
+	} else {
+		struct dw_hdma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs;
+
+		writel(0, &lli->control);
+	}
+}
+
 static void dw_hdma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
 
-	dw_hdma_v0_sync_ll_data(chan);
-
 	/* Doorbell */
 	SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
 }
 
+static int dw_hdma_v0_core_ll_cur_idx(struct dw_edma_chan *chan)
+{
+	u32 base, val;
+
+	val = GET_CH_32(chan->dw, chan->dir, chan->id, llp.lsb);
+	base = lower_32_bits(dw_edma_core_get_ll_paddr(chan));
+
+	/*
+	 * LL regions stay within one 4 GiB address window. Reject an all-ones
+	 * MMIO value. If the low word is zero, use the high word to distinguish
+	 * a nonzero boundary address from an unprogrammed all-zero context.
+	 */
+	if (val == U32_MAX ||
+	    (!val && !GET_CH_32(chan->dw, chan->dir, chan->id, llp.msb)))
+		return -EINVAL;
+
+	return (val - base) / EDMA_LL_SZ;
+}
+
 /* HDMA debugfs callbacks */
 static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
 {
@@ -406,6 +429,9 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.non_ll_start = dw_hdma_v0_core_non_ll_start,
 	.ll_data = dw_hdma_v0_core_ll_data,
 	.ll_link = dw_hdma_v0_core_ll_link,
+	.ll_clear = dw_hdma_v0_core_ll_clear,
+	.ll_cur_idx = dw_hdma_v0_core_ll_cur_idx,
+	.ll_irq_clear = dw_hdma_v0_core_clear_done_int,
 	.ch_doorbell = dw_hdma_v0_core_ch_doorbell,
 	.ch_enable = dw_hdma_v0_core_ch_enable,
 	.ch_config = dw_hdma_v0_core_ch_config,
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c
index dcdc57fe976c13..3fa16e045a5843 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c
+++ b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c
@@ -13,22 +13,17 @@
 #include "dw-hdma-v0-regs.h"
 #include "dw-edma-core.h"
 
-#define REGS_ADDR(dw, name)						       \
-	({								       \
-		struct dw_hdma_v0_regs __iomem *__regs = (dw)->chip->reg_base; \
-									       \
-		(void __iomem *)&__regs->name;				       \
-	})
-
 #define REGS_CH_ADDR(dw, name, _dir, _ch)				       \
 	({								       \
-		struct dw_hdma_v0_ch_regs __iomem *__ch_regs;		       \
+		struct dw_hdma_v0_ch_regs __iomem *__ch_regs;                  \
+		off_t __off = (dw)->chip->ch_space_sz;			       \
 									       \
-		if (_dir == EDMA_DIR_READ)				       \
-			__ch_regs = REGS_ADDR(dw, ch[_ch].rd);		       \
+		if ((_dir) == EDMA_DIR_READ)				       \
+			__off *= (2 * (_ch) + 1);			       \
 		else							       \
-			__ch_regs = REGS_ADDR(dw, ch[_ch].wr);		       \
+			__off *= (2 * (_ch));				       \
 									       \
+		__ch_regs = ((dw)->chip->reg_base + __off);		       \
 		(void __iomem *)&__ch_regs->name;			       \
 	})
 
diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
index 2124c162a62f91..b88226cdd791c3 100644
--- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h
+++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h
@@ -85,16 +85,6 @@ struct dw_hdma_v0_ch_regs {
 		};
 	} msi_abort;
 	u32 msi_msgdata;			/* 0x00a8 */
-	u32 padding_2[21];			/* 0x00ac..0x00fc */
-} __packed;
-
-struct dw_hdma_v0_ch {
-	struct dw_hdma_v0_ch_regs wr;		/* 0x0000 */
-	struct dw_hdma_v0_ch_regs rd;		/* 0x0100 */
-} __packed;
-
-struct dw_hdma_v0_regs {
-	struct dw_hdma_v0_ch ch[HDMA_V0_MAX_NR_CH];	/* 0x0000..0x0fa8 */
 } __packed;
 
 struct dw_hdma_v0_lli {
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index dd75f97a33b3d3..575b7bb6af9a06 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -41,11 +41,6 @@
 
 /*----------------------------------------------------------------------*/
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
 {
 	return to_dw_desc(dwc->active_list.next);
@@ -69,7 +64,7 @@ static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
 
 	list_add_tail(&desc->desc_node, &dwc->queue);
 	spin_unlock_irqrestore(&dwc->lock, flags);
-	dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n",
+	dev_vdbg(dmaengine_chan_dev(tx->chan), "%s: queued %u\n",
 		 __func__, desc->txd.cookie);
 
 	return cookie;
@@ -127,7 +122,7 @@ static void dwc_initialize(struct dw_dma_chan *dwc)
 
 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
 {
-	dev_err(chan2dev(&dwc->chan),
+	dev_err(dmaengine_chan_dev(&dwc->chan),
 		"  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 		channel_readl(dwc, SAR),
 		channel_readl(dwc, DAR),
@@ -177,7 +172,7 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 
 	/* ASSERT:  channel is idle */
 	if (dma_readl(dw, CH_EN) & dwc->mask) {
-		dev_err(chan2dev(&dwc->chan),
+		dev_err(dmaengine_chan_dev(&dwc->chan),
 			"%s: BUG: Attempted to start non-idle channel\n",
 			__func__);
 		dwc_dump_chan_regs(dwc);
@@ -190,7 +185,7 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 		was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,
 						&dwc->flags);
 		if (was_soft_llp) {
-			dev_err(chan2dev(&dwc->chan),
+			dev_err(dmaengine_chan_dev(&dwc->chan),
 				"BUG: Attempted to start new LLP transfer inside ongoing one\n");
 			return;
 		}
@@ -223,7 +218,7 @@ static void dwc_dostart_first_queued(struct dw_dma_chan *dwc)
 
 	list_move(dwc->queue.next, &dwc->active_list);
 	desc = dwc_first_active(dwc);
-	dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);
+	dev_vdbg(dmaengine_chan_dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);
 	dwc_dostart(dwc, desc);
 }
 
@@ -238,7 +233,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
 	unsigned long			flags;
 	struct dmaengine_desc_callback	cb;
 
-	dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
+	dev_vdbg(dmaengine_chan_dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	dma_cookie_complete(txd);
@@ -265,7 +260,7 @@ static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
 
 	spin_lock_irqsave(&dwc->lock, flags);
 	if (dma_readl(dw, CH_EN) & dwc->mask) {
-		dev_err(chan2dev(&dwc->chan),
+		dev_err(dmaengine_chan_dev(&dwc->chan),
 			"BUG: XFER bit set, but channel not idle!\n");
 
 		/* Try to continue after resetting the channel... */
@@ -353,12 +348,12 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 	}
 
 	if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
-		dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__);
+		dev_vdbg(dmaengine_chan_dev(&dwc->chan), "%s: soft LLP mode\n", __func__);
 		spin_unlock_irqrestore(&dwc->lock, flags);
 		return;
 	}
 
-	dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp);
+	dev_vdbg(dmaengine_chan_dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp);
 
 	list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
 		/* Initial residue value */
@@ -398,7 +393,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 		spin_lock_irqsave(&dwc->lock, flags);
 	}
 
-	dev_err(chan2dev(&dwc->chan),
+	dev_err(dmaengine_chan_dev(&dwc->chan),
 		"BUG: All descriptors done, but channel not idle!\n");
 
 	/* Try to continue after resetting the channel... */
@@ -410,7 +405,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 
 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
-	dev_crit(chan2dev(&dwc->chan), "  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
+	dev_crit(dmaengine_chan_dev(&dwc->chan), "  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
 		 lli_read(desc, sar),
 		 lli_read(desc, dar),
 		 lli_read(desc, llp),
@@ -449,7 +444,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
 	 * controller flagged an error instead of scribbling over
 	 * random memory locations.
 	 */
-	dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
+	dev_WARN(dmaengine_chan_dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
 				       "  cookie: %d\n", bad_desc->txd.cookie);
 	dwc_dump_lli(dwc, bad_desc);
 	list_for_each_entry(child, &bad_desc->tx_list, desc_node)
@@ -552,12 +547,12 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	u32			ctllo, ctlhi;
 	u8			lms = DWC_LLP_LMS(m_master);
 
-	dev_vdbg(chan2dev(chan),
+	dev_vdbg(dmaengine_chan_dev(chan),
 			"%s: d%pad s%pad l0x%zx f0x%lx\n", __func__,
 			&dest, &src, len, flags);
 
 	if (unlikely(!len)) {
-		dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
+		dev_dbg(dmaengine_chan_dev(chan), "%s: length is zero!\n", __func__);
 		return NULL;
 	}
 
@@ -630,7 +625,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	struct scatterlist	*sg;
 	size_t			total_len = 0;
 
-	dev_vdbg(chan2dev(chan), "%s\n", __func__);
+	dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	if (unlikely(!is_slave_direction(direction) || !sg_len))
 		return NULL;
@@ -754,7 +749,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	return &first->txd;
 
 err_desc_get:
-	dev_err(chan2dev(chan),
+	dev_err(dmaengine_chan_dev(chan),
 		"not enough descriptors available. Direction %d\n", direction);
 	dwc_desc_put(dwc, first);
 	return NULL;
@@ -1063,11 +1058,11 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma		*dw = to_dw_dma(chan->device);
 
-	dev_vdbg(chan2dev(chan), "%s\n", __func__);
+	dev_vdbg(dmaengine_chan_dev(chan), "%s\n", __func__);
 
 	/* ASSERT:  channel is idle */
 	if (dma_readl(dw, CH_EN) & dwc->mask) {
-		dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
+		dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle?\n");
 		return -EIO;
 	}
 
@@ -1083,7 +1078,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
 	 * We need controller-specific data to set up slave transfers.
 	 */
 	if (chan->private && !dw_dma_filter(chan, chan->private)) {
-		dev_warn(chan2dev(chan), "Wrong controller-specific data\n");
+		dev_warn(dmaengine_chan_dev(chan), "Wrong controller-specific data\n");
 		return -EINVAL;
 	}
 
@@ -1101,7 +1096,7 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
 	struct dw_dma		*dw = to_dw_dma(chan->device);
 	unsigned long		flags;
 
-	dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
+	dev_dbg(dmaengine_chan_dev(chan), "%s: descs allocated=%u\n", __func__,
 			dwc->descs_allocated);
 
 	/* ASSERT:  channel is idle */
@@ -1126,7 +1121,7 @@ static void dwc_free_chan_resources(struct dma_chan *chan)
 	if (!dw->in_use)
 		do_dw_dma_off(dw);
 
-	dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
+	dev_vdbg(dmaengine_chan_dev(chan), "%s: done\n", __func__);
 }
 
 static void dwc_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c
index 311e55a97ba9f9..cbfe74fc8d71ed 100644
--- a/drivers/dma/ep93xx_dma.c
+++ b/drivers/dma/ep93xx_dma.c
@@ -250,11 +250,6 @@ struct ep93xx_edma_data {
 	size_t	num_channels;
 };
 
-static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)
-{
-	return &edmac->chan.dev->device;
-}
-
 static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan)
 {
 	return container_of(chan, struct ep93xx_dma_chan, chan);
@@ -415,7 +410,7 @@ static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)
 	m2p_set_control(edmac, 0);
 
 	while (m2p_channel_state(edmac) != M2P_STATE_IDLE)
-		dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n");
+		dev_warn(dmaengine_chan_dev(&edmac->chan), "M2P: Not yet IDLE\n");
 }
 
 static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)
@@ -425,7 +420,7 @@ static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)
 
 	desc = ep93xx_dma_get_active(edmac);
 	if (!desc) {
-		dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");
+		dev_warn(dmaengine_chan_dev(&edmac->chan), "M2P: empty descriptor list\n");
 		return;
 	}
 
@@ -479,7 +474,7 @@ static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)
 		 * Revisit this when there is a mechanism to report back the
 		 * errors.
 		 */
-		dev_err(chan2dev(edmac),
+		dev_err(dmaengine_chan_dev(&edmac->chan),
 			"DMA transfer failed! Details:\n"
 			"\tcookie	: %d\n"
 			"\tsrc_addr	: 0x%08x\n"
@@ -586,7 +581,7 @@ static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)
 
 	desc = ep93xx_dma_get_active(edmac);
 	if (!desc) {
-		dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");
+		dev_warn(dmaengine_chan_dev(&edmac->chan), "M2M: empty descriptor list\n");
 		return;
 	}
 
@@ -849,7 +844,7 @@ static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
 
 	desc = ep93xx_dma_get_active(edmac);
 	if (!desc) {
-		dev_warn(chan2dev(edmac),
+		dev_warn(dmaengine_chan_dev(&edmac->chan),
 			 "got interrupt while active list is empty\n");
 		spin_unlock(&edmac->lock);
 		return IRQ_NONE;
@@ -867,7 +862,7 @@ static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
 		break;
 
 	default:
-		dev_warn(chan2dev(edmac), "unknown interrupt!\n");
+		dev_warn(dmaengine_chan_dev(&edmac->chan), "unknown interrupt!\n");
 		ret = IRQ_NONE;
 		break;
 	}
@@ -967,7 +962,7 @@ static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
 
 		desc = kzalloc_obj(*desc);
 		if (!desc) {
-			dev_warn(chan2dev(edmac), "not enough descriptors\n");
+			dev_warn(dmaengine_chan_dev(&edmac->chan), "not enough descriptors\n");
 			break;
 		}
 
@@ -1044,7 +1039,7 @@ ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
 	for (offset = 0; offset < len; offset += bytes) {
 		desc = ep93xx_dma_desc_get(edmac);
 		if (!desc) {
-			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
+			dev_warn(dmaengine_chan_dev(&edmac->chan), "couldn't get descriptor\n");
 			goto fail;
 		}
 
@@ -1091,13 +1086,13 @@ ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	int i;
 
 	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
-		dev_warn(chan2dev(edmac),
+		dev_warn(dmaengine_chan_dev(&edmac->chan),
 			 "channel was configured with different direction\n");
 		return NULL;
 	}
 
 	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
-		dev_warn(chan2dev(edmac),
+		dev_warn(dmaengine_chan_dev(&edmac->chan),
 			 "channel is already used for cyclic transfers\n");
 		return NULL;
 	}
@@ -1109,14 +1104,14 @@ ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		size_t len = sg_dma_len(sg);
 
 		if (len > DMA_MAX_CHAN_BYTES) {
-			dev_warn(chan2dev(edmac), "too big transfer size %zu\n",
+			dev_warn(dmaengine_chan_dev(&edmac->chan), "too big transfer size %zu\n",
 				 len);
 			goto fail;
 		}
 
 		desc = ep93xx_dma_desc_get(edmac);
 		if (!desc) {
-			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
+			dev_warn(dmaengine_chan_dev(&edmac->chan), "couldn't get descriptor\n");
 			goto fail;
 		}
 
@@ -1172,19 +1167,19 @@ ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
 	size_t offset = 0;
 
 	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
-		dev_warn(chan2dev(edmac),
+		dev_warn(dmaengine_chan_dev(&edmac->chan),
 			 "channel was configured with different direction\n");
 		return NULL;
 	}
 
 	if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
-		dev_warn(chan2dev(edmac),
+		dev_warn(dmaengine_chan_dev(&edmac->chan),
 			 "channel is already used for cyclic transfers\n");
 		return NULL;
 	}
 
 	if (period_len > DMA_MAX_CHAN_BYTES) {
-		dev_warn(chan2dev(edmac), "too big period length %zu\n",
+		dev_warn(dmaengine_chan_dev(&edmac->chan), "too big period length %zu\n",
 			 period_len);
 		return NULL;
 	}
@@ -1196,7 +1191,7 @@ ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
 	for (offset = 0; offset < buf_len; offset += period_len) {
 		desc = ep93xx_dma_desc_get(edmac);
 		if (!desc) {
-			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
+			dev_warn(dmaengine_chan_dev(&edmac->chan), "couldn't get descriptor\n");
 			goto fail;
 		}
 
diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c
index d9fb717b5b53cc..6934ac8826976a 100644
--- a/drivers/dma/fsl-edma-main.c
+++ b/drivers/dma/fsl-edma-main.c
@@ -326,13 +326,16 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec,
 		if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1))
 			continue;
 
+		chan = dma_get_slave_channel(chan);
+		if (!chan)
+			return NULL;
+
 		fsl_chan->srcid = dma_spec->args[0];
 		fsl_chan->priority = dma_spec->args[1];
 		fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX;
 		fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE;
 		fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO;
 
-		chan = dma_get_slave_channel(chan);
 		chan->device->privatecnt++;
 		return chan;
 	}
diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c
index 2d86f61105e528..9cc289f7129edd 100644
--- a/drivers/dma/fsl_raid.c
+++ b/drivers/dma/fsl_raid.c
@@ -162,6 +162,7 @@ static void fsl_re_dequeue(struct tasklet_struct *t)
 	struct fsl_re_hw_desc *hwdesc;
 	unsigned long flags;
 	unsigned int count, oub_count;
+	LIST_HEAD(completed);
 	int found;
 
 	fsl_re_cleanup_descs(re_chan);
@@ -182,8 +183,7 @@ static void fsl_re_dequeue(struct tasklet_struct *t)
 		}
 
 		if (found) {
-			fsl_re_desc_done(desc);
-			list_move_tail(&desc->node, &re_chan->ack_q);
+			list_move_tail(&desc->node, &completed);
 		} else {
 			dev_err(re_chan->dev,
 				"found hwdesc not in sw queue, discard it\n");
@@ -196,6 +196,17 @@ static void fsl_re_dequeue(struct tasklet_struct *t)
 			 FSL_RE_RMVD_JOB(1));
 	}
 	spin_unlock_irqrestore(&re_chan->desc_lock, flags);
+
+	/* Invoke the client callbacks outside the channel lock. The callback
+	 * may submit new work which re-acquires desc_lock, so holding it here
+	 * would deadlock.
+	 */
+	list_for_each_entry_safe(desc, _desc, &completed, node) {
+		fsl_re_desc_done(desc);
+		spin_lock_irqsave(&re_chan->desc_lock, flags);
+		list_move_tail(&desc->node, &re_chan->ack_q);
+		spin_unlock_irqrestore(&re_chan->desc_lock, flags);
+	}
 }
 
 /* Per Job Ring interrupt handler */
@@ -295,8 +306,8 @@ static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan,
 		if (!desc)
 			return NULL;
 
-		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT,
-				    &paddr);
+		cf = dma_pool_zalloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT,
+				     &paddr);
 		if (!cf) {
 			kfree(desc);
 			return NULL;
@@ -350,7 +361,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq(
 	cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT;
 	cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT;
 	xor = desc->cdb_addr;
-	xor->cdb32 = cdb;
+	xor->cdb32 = cpu_to_be32(cdb);
 
 	if (scf) {
 		/* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */
@@ -470,7 +481,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq(
 	cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT;
 
 	pq = desc->cdb_addr;
-	pq->cdb32 = cdb;
+	pq->cdb32 = cpu_to_be32(cdb);
 
 	p = pq->gfm_q1;
 	/* Init gfm_q1[] */
@@ -553,7 +564,7 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy(
 	cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT;
 
 	move = desc->cdb_addr;
-	move->cdb32 = cdb;
+	move->cdb32 = cpu_to_be32(cdb);
 
 	/* Filling frame 0 of CFD with move CDB */
 	cf = desc->cf_addr;
@@ -584,8 +595,8 @@ static int fsl_re_alloc_chan_resources(struct dma_chan *chan)
 		if (!desc)
 			break;
 
-		cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL,
-				    &paddr);
+		cf = dma_pool_zalloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL,
+				     &paddr);
 		if (!cf) {
 			kfree(desc);
 			break;
@@ -603,18 +614,15 @@ static int fsl_re_alloc_chan_resources(struct dma_chan *chan)
 static void fsl_re_free_chan_resources(struct dma_chan *chan)
 {
 	struct fsl_re_chan *re_chan;
-	struct fsl_re_desc *desc;
+	struct fsl_re_desc *desc, *_desc;
 
 	re_chan = container_of(chan, struct fsl_re_chan, chan);
-	while (re_chan->alloc_count--) {
-		desc = list_first_entry(&re_chan->free_q,
-					struct fsl_re_desc,
-					node);
-
+	list_for_each_entry_safe(desc, _desc, &re_chan->free_q, node) {
 		list_del(&desc->node);
 		dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr,
 			      desc->cf_paddr);
 		kfree(desc);
+		re_chan->alloc_count--;
 	}
 
 	if (!list_empty(&re_chan->free_q))
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index 22d62d958abd90..99a878ed144453 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -681,7 +681,7 @@ static void fsldma_cleanup_descriptors(struct fsldma_chan *chan)
 
 /**
  * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
- * @chan : Freescale DMA channel
+ * @dchan : Freescale DMA channel
  *
  * This function will create a dma pool for descriptor allocation.
  *
@@ -738,7 +738,7 @@ static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,
 
 /**
  * fsl_dma_free_chan_resources - Free all resources of the channel.
- * @chan : Freescale DMA channel
+ * @dchan : Freescale DMA channel
  */
 static void fsl_dma_free_chan_resources(struct dma_chan *dchan)
 {
@@ -873,7 +873,7 @@ static int fsl_dma_device_config(struct dma_chan *dchan,
 
 /**
  * fsl_dma_memcpy_issue_pending - Issue the DMA start command
- * @chan : Freescale DMA channel
+ * @dchan : Freescale DMA channel
  */
 static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
 {
@@ -886,7 +886,9 @@ static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
 
 /**
  * fsl_tx_status - Determine the DMA status
- * @chan : Freescale DMA channel
+ * @dchan : Freescale DMA channel
+ * @cookie : DMA transaction identifier
+ * @txstate : DMA transaction state
  */
 static enum dma_status fsl_tx_status(struct dma_chan *dchan,
 					dma_cookie_t cookie,
@@ -1205,6 +1207,7 @@ static int fsl_dma_chan_probe(struct fsldma_device *fdev,
 
 static void fsl_dma_chan_remove(struct fsldma_chan *chan)
 {
+	tasklet_kill(&chan->tasklet);
 	irq_dispose_mapping(chan->irq);
 	list_del(&chan->common.device_node);
 	iounmap(chan->regs);
@@ -1216,8 +1219,13 @@ static int fsldma_of_probe(struct platform_device *op)
 	struct fsldma_device *fdev;
 	struct device_node *child;
 	unsigned int i;
+	int irq;
 	int err;
 
+	irq = platform_get_irq_optional(op, 0);
+	if (irq == -EPROBE_DEFER)
+		return irq;
+
 	fdev = kzalloc_obj(*fdev);
 	if (!fdev) {
 		err = -ENOMEM;
@@ -1238,7 +1246,9 @@ static int fsldma_of_probe(struct platform_device *op)
 	}
 
 	/* map the channel IRQ if it exists, but don't hookup the handler yet */
-	fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0);
+	fdev->irq = irq;
+	if (fdev->irq < 0)
+		fdev->irq = 0;
 
 	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
 	dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
@@ -1300,7 +1310,6 @@ static int fsldma_of_probe(struct platform_device *op)
 		if (fdev->chan[i])
 			fsl_dma_chan_remove(fdev->chan[i]);
 	}
-	irq_dispose_mapping(fdev->irq);
 	iounmap(fdev->regs);
 out_free:
 	kfree(fdev);
@@ -1322,7 +1331,6 @@ static void fsldma_of_remove(struct platform_device *op)
 		if (fdev->chan[i])
 			fsl_dma_chan_remove(fdev->chan[i]);
 	}
-	irq_dispose_mapping(fdev->irq);
 
 	iounmap(fdev->regs);
 	kfree(fdev);
diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c
index 5fcd1befc92da7..657629b100fa95 100644
--- a/drivers/dma/idma64.c
+++ b/drivers/dma/idma64.c
@@ -25,11 +25,6 @@
 
 /* ---------------------------------------------------------------------- */
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 /* ---------------------------------------------------------------------- */
 
 static void idma64_off(struct idma64 *idma64)
@@ -507,11 +502,10 @@ static int idma64_alloc_chan_resources(struct dma_chan *chan)
 	struct idma64_chan *idma64c = to_idma64_chan(chan);
 
 	/* Create a pool of consistent memory blocks for hardware descriptors */
-	idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)),
-					chan->device->dev,
+	idma64c->pool = dma_pool_create(dma_chan_name(chan), chan->device->dev,
 					sizeof(struct idma64_lli), 8, 0);
 	if (!idma64c->pool) {
-		dev_err(chan2dev(chan), "No memory for descriptors\n");
+		dev_err(dmaengine_chan_dev(chan), "No memory for descriptors\n");
 		return -ENOMEM;
 	}
 
diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c
index b3765ba1580308..a3192d49b6c860 100644
--- a/drivers/dma/img-mdc-dma.c
+++ b/drivers/dma/img-mdc-dma.c
@@ -738,7 +738,7 @@ static int mdc_alloc_chan_resources(struct dma_chan *chan)
 	struct mdc_chan *mchan = to_mdc_chan(chan);
 	struct device *dev = mdma2dev(mchan->mdma);
 
-	return pm_runtime_get_sync(dev);
+	return pm_runtime_resume_and_get(dev);
 }
 
 static void mdc_free_chan_resources(struct dma_chan *chan)
diff --git a/drivers/dma/loongson/loongson1-apb-dma.c b/drivers/dma/loongson/loongson1-apb-dma.c
index 89786cbd20ab50..46b4bfef45e26f 100644
--- a/drivers/dma/loongson/loongson1-apb-dma.c
+++ b/drivers/dma/loongson/loongson1-apb-dma.c
@@ -89,11 +89,6 @@ static irqreturn_t ls1x_dma_irq_handler(int irq, void *data);
 #define to_ls1x_dma_desc(d)		\
 	container_of(d, struct ls1x_dma_desc, vd)
 
-static inline struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static inline int ls1x_dma_query(struct ls1x_dma_chan *chan,
 				 dma_addr_t *lli_phys)
 {
@@ -107,7 +102,7 @@ static inline int ls1x_dma_query(struct ls1x_dma_chan *chan,
 	ret = readl_poll_timeout_atomic(chan->reg_base + LS1X_DMA_CTRL, val,
 					!(val & LS1X_DMA_ASK_VALID), 0, 3000);
 	if (ret)
-		dev_err(chan2dev(dchan), "failed to query DMA\n");
+		dev_err(dmaengine_chan_dev(dchan), "failed to query DMA\n");
 
 	return ret;
 }
@@ -116,7 +111,7 @@ static inline int ls1x_dma_start(struct ls1x_dma_chan *chan,
 				 dma_addr_t *lli_phys)
 {
 	struct dma_chan *dchan = &chan->vc.chan;
-	struct device *dev = chan2dev(dchan);
+	struct device *dev = dmaengine_chan_dev(dchan);
 	int val, ret;
 
 	val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK;
@@ -143,7 +138,7 @@ static inline void ls1x_dma_stop(struct ls1x_dma_chan *chan)
 static void ls1x_dma_free_chan_resources(struct dma_chan *dchan)
 {
 	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	struct device *dev = chan2dev(dchan);
+	struct device *dev = dmaengine_chan_dev(dchan);
 
 	dma_free_coherent(dev, sizeof(struct ls1x_dma_lli),
 			  chan->curr_lli, chan->curr_lli->phys);
@@ -156,7 +151,7 @@ static void ls1x_dma_free_chan_resources(struct dma_chan *dchan)
 static int ls1x_dma_alloc_chan_resources(struct dma_chan *dchan)
 {
 	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
-	struct device *dev = chan2dev(dchan);
+	struct device *dev = dmaengine_chan_dev(dchan);
 	dma_addr_t phys;
 	int ret;
 
@@ -219,7 +214,7 @@ static int ls1x_dma_prep_lli(struct dma_chan *dchan, struct ls1x_dma_desc *desc,
 {
 	struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan);
 	struct ls1x_dma_lli *lli, *prev = NULL, *first = NULL;
-	struct device *dev = chan2dev(dchan);
+	struct device *dev = dmaengine_chan_dev(dchan);
 	struct list_head *pos = NULL;
 	struct scatterlist *sg;
 	unsigned int dev_addr, cmd, i;
@@ -299,7 +294,7 @@ ls1x_dma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 {
 	struct ls1x_dma_desc *desc;
 
-	dev_dbg(chan2dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n",
+	dev_dbg(dmaengine_chan_dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n",
 		sg_len, flags, dmaengine_get_direction_text(dir));
 
 	desc = ls1x_dma_alloc_desc();
@@ -325,7 +320,7 @@ ls1x_dma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr,
 	unsigned int i;
 	int ret;
 
-	dev_dbg(chan2dev(dchan),
+	dev_dbg(dmaengine_chan_dev(dchan),
 		"buf_len=%zu period_len=%zu flags=0x%lx dir=%s\n",
 		buf_len, period_len, flags, dmaengine_get_direction_text(dir));
 
@@ -450,7 +445,7 @@ static enum dma_status ls1x_dma_tx_status(struct dma_chan *dchan,
 				if (lli->hw[LS1X_DMADESC_NEXT] == next_phys)
 					break;
 
-			dev_dbg(chan2dev(dchan), "current lli_phys=%pad",
+			dev_dbg(dmaengine_chan_dev(dchan), "current lli_phys=%pad",
 				&lli->phys);
 
 			/* count the residues */
@@ -489,7 +484,7 @@ static irqreturn_t ls1x_dma_irq_handler(int irq, void *data)
 {
 	struct ls1x_dma_chan *chan = data;
 	struct dma_chan *dchan = &chan->vc.chan;
-	struct device *dev = chan2dev(dchan);
+	struct device *dev = dmaengine_chan_dev(dchan);
 	struct virt_dma_desc *vd;
 
 	scoped_guard(spinlock, &chan->vc.lock) {
diff --git a/drivers/dma/loongson/loongson2-apb-cmc-dma.c b/drivers/dma/loongson/loongson2-apb-cmc-dma.c
index 1c9a542edc8596..bc336947e31ac3 100644
--- a/drivers/dma/loongson/loongson2-apb-cmc-dma.c
+++ b/drivers/dma/loongson/loongson2-apb-cmc-dma.c
@@ -90,7 +90,7 @@ struct loongson2_cmc_dma_chan {
 	struct dma_slave_config	dma_sconfig;
 	struct loongson2_cmc_dma_desc *desc;
 	u32 id;
-	u32 irq;
+	int irq;
 	u32 next_sg;
 	struct loongson2_cmc_dma_chan_reg chan_reg;
 };
@@ -134,11 +134,6 @@ static struct loongson2_cmc_dma_desc *to_lmdma_desc(struct virt_dma_desc *vdesc)
 	return container_of(vdesc, struct loongson2_cmc_dma_desc, vdesc);
 }
 
-static struct device *chan2dev(struct loongson2_cmc_dma_chan *lchan)
-{
-	return &lchan->vchan.chan.dev->device;
-}
-
 static u32 loongson2_cmc_dma_read(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id)
 {
 	return readl(lddev->base + (reg + lddev->chan_reg_offset * id));
@@ -302,7 +297,7 @@ static irqreturn_t loongson2_cmc_dma_chan_irq(int irq, void *devid)
 {
 	struct loongson2_cmc_dma_chan *lchan = devid;
 	struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan);
-	struct device *dev = chan2dev(lchan);
+	struct device *dev = vchan_chan_dev(&lchan->vchan);
 	u32 ists, status, ccr;
 
 	scoped_guard(spinlock, &lchan->vchan.lock) {
@@ -337,7 +332,7 @@ static void loongson2_cmc_dma_issue_pending(struct dma_chan *chan)
 	guard(spinlock_irqsave)(&lchan->vchan.lock);
 
 	if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) {
-		dev_dbg(chan2dev(lchan), "vchan %pK: issued\n", &lchan->vchan);
+		dev_dbg(vchan_chan_dev(&lchan->vchan), "vchan %pK: issued\n", &lchan->vchan);
 		loongson2_cmc_dma_start_transfer(lchan);
 	}
 }
@@ -347,7 +342,7 @@ static int loongson2_cmc_dma_set_xfer_param(struct loongson2_cmc_dma_chan *lchan
 					    enum dma_slave_buswidth *buswidth, u32 buf_len)
 {
 	struct dma_slave_config	sconfig = lchan->dma_sconfig;
-	struct device *dev = chan2dev(lchan);
+	struct device *dev = vchan_chan_dev(&lchan->vchan);
 	int dev_width;
 	u32 ccr;
 
@@ -409,7 +404,7 @@ loongson2_cmc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 
 		num_items = DIV_ROUND_UP(sg_dma_len(sg), buswidth);
 		if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
-			dev_err(chan2dev(lchan), "Number of items not supported\n");
+			dev_err(vchan_chan_dev(&lchan->vchan), "Number of items not supported\n");
 			kfree(desc);
 			return ERR_PTR(-EINVAL);
 		}
@@ -447,7 +442,7 @@ loongson2_cmc_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, si
 
 	num_items = DIV_ROUND_UP(period_len, buswidth);
 	if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) {
-		dev_err(chan2dev(lchan), "Number of items not supported\n");
+		dev_err(vchan_chan_dev(&lchan->vchan), "Number of items not supported\n");
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -678,7 +673,7 @@ static int loongson2_cmc_dma_probe(struct platform_device *pdev)
 			return lchan->irq;
 
 		ret = devm_request_irq(dev, lchan->irq, loongson2_cmc_dma_chan_irq, IRQF_SHARED,
-				       dev_name(chan2dev(lchan)), lchan);
+				       vchan_chan_name(&lchan->vchan), lchan);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/dma/loongson/loongson2-apb-dma.c b/drivers/dma/loongson/loongson2-apb-dma.c
index aceb069e71fca0..16a0c1c60d59ae 100644
--- a/drivers/dma/loongson/loongson2-apb-dma.c
+++ b/drivers/dma/loongson/loongson2-apb-dma.c
@@ -161,11 +161,6 @@ static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev)
 	return container_of(ddev, struct ls2x_dma_priv, ddev);
 }
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc)
 {
 	struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan);
@@ -282,11 +277,11 @@ static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan)
 	struct ls2x_dma_chan *lchan = to_ldma_chan(chan);
 
 	/* Create a pool of consistent memory blocks for hardware descriptors */
-	lchan->pool = dma_pool_create(dev_name(chan2dev(chan)),
+	lchan->pool = dma_pool_create(dma_chan_name(chan),
 				      chan->device->dev, PAGE_SIZE,
 				      __alignof__(struct ls2x_dma_hw_desc), 0);
 	if (!lchan->pool) {
-		dev_err(chan2dev(chan), "No memory for descriptors\n");
+		dev_err(dmaengine_chan_dev(chan), "No memory for descriptors\n");
 		return -ENOMEM;
 	}
 
diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c
index a43412ff5eddcd..27cb370106c777 100644
--- a/drivers/dma/mediatek/mtk-hsdma.c
+++ b/drivers/dma/mediatek/mtk-hsdma.c
@@ -848,17 +848,27 @@ static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)
 {
 	int err;
 
-	pm_runtime_enable(hsdma2dev(hsdma));
-	pm_runtime_get_sync(hsdma2dev(hsdma));
+	err = devm_pm_runtime_enable(hsdma2dev(hsdma));
+	if (err)
+		return err;
+
+	err = pm_runtime_resume_and_get(hsdma2dev(hsdma));
+	if (err < 0)
+		return err;
 
 	err = clk_prepare_enable(hsdma->clk);
 	if (err)
-		return err;
+		goto err_put_pm;
 
 	mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);
 	mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);
 
 	return 0;
+
+err_put_pm:
+	pm_runtime_put_sync(hsdma2dev(hsdma));
+
+	return err;
 }
 
 static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)
@@ -868,7 +878,6 @@ static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)
 	clk_disable_unprepare(hsdma->clk);
 
 	pm_runtime_put_sync(hsdma2dev(hsdma));
-	pm_runtime_disable(hsdma2dev(hsdma));
 
 	return 0;
 }
@@ -983,7 +992,9 @@ static int mtk_hsdma_probe(struct platform_device *pdev)
 		goto err_unregister;
 	}
 
-	mtk_hsdma_hw_init(hsdma);
+	err = mtk_hsdma_hw_init(hsdma);
+	if (err)
+		goto err_free;
 
 	err = devm_request_irq(&pdev->dev, hsdma->irq,
 			       mtk_hsdma_irq, 0,
@@ -991,7 +1002,7 @@ static int mtk_hsdma_probe(struct platform_device *pdev)
 	if (err) {
 		dev_err(&pdev->dev,
 			"request_irq failed with err %d\n", err);
-		goto err_free;
+		goto err_deinit;
 	}
 
 	platform_set_drvdata(pdev, hsdma);
@@ -1000,8 +1011,9 @@ static int mtk_hsdma_probe(struct platform_device *pdev)
 
 	return 0;
 
-err_free:
+err_deinit:
 	mtk_hsdma_hw_deinit(hsdma);
+err_free:
 	of_dma_controller_free(pdev->dev.of_node);
 err_unregister:
 	dma_async_device_unregister(dd);
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index 386e85cd4882a3..8abd24e2ffc74a 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -52,7 +52,6 @@
 #define DCSR_EORINTR	BIT(9)	/* The end of Receive */
 
 #define DRCMR_BASE		0x0100
-#define DRCMR_EXT_BASE_K3	0x1000
 #define DRCMR_EXT_BASE_DEFAULT	0x1100
 #define DRCMR_REQ_LIMIT		64
 #define DRCMR_MAPVLD	BIT(7)	/* Map Valid (read / write) */
@@ -565,8 +564,7 @@ static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan)
 	if (chan->desc_pool)
 		return 1;
 
-	chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device),
-					  chan->dev,
+	chan->desc_pool = dma_pool_create(dma_chan_name(dchan), chan->dev,
 					  sizeof(struct mmp_pdma_desc_sw),
 					  __alignof__(struct mmp_pdma_desc_sw),
 					  0);
@@ -713,7 +711,7 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 
 	for_each_sg(sgl, sg, sg_len, i) {
 		addr = sg_dma_address(sg);
-		avail = sg_dma_len(sgl);
+		avail = sg_dma_len(sg);
 
 		do {
 			len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
@@ -1219,7 +1217,7 @@ static const struct mmp_pdma_ops spacemit_k3_pdma_ops = {
 	.get_desc_dst_addr = get_desc_dst_addr_64,
 	.run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN),
 	.dma_width = 64,
-	.drcmr_ext_base = DRCMR_EXT_BASE_K3,
+	.drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
 };
 
 static const struct of_device_id mmp_pdma_dt_ids[] = {
diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c
index 442f5aa16031f2..28be93eed4cce4 100644
--- a/drivers/dma/moxart-dma.c
+++ b/drivers/dma/moxart-dma.c
@@ -154,11 +154,6 @@ static const unsigned int es_bytes[] = {
 	[MOXART_DMA_DATA_TYPE_S32] = 4,
 };
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static inline struct moxart_chan *to_moxart_dma_chan(struct dma_chan *c)
 {
 	return container_of(c, struct moxart_chan, vc.chan);
@@ -182,7 +177,7 @@ static int moxart_terminate_all(struct dma_chan *chan)
 	LIST_HEAD(head);
 	u32 ctrl;
 
-	dev_dbg(chan2dev(chan), "%s: ch=%p\n", __func__, ch);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: ch=%p\n", __func__, ch);
 
 	spin_lock_irqsave(&ch->vc.lock, flags);
 
@@ -272,7 +267,7 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg(
 	unsigned int i;
 
 	if (!is_slave_direction(dir)) {
-		dev_err(chan2dev(chan), "%s: invalid DMA direction\n",
+		dev_err(dmaengine_chan_dev(chan), "%s: invalid DMA direction\n",
 			__func__);
 		return NULL;
 	}
@@ -296,7 +291,7 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg(
 		es = MOXART_DMA_DATA_TYPE_S32;
 		break;
 	default:
-		dev_err(chan2dev(chan), "%s: unsupported data width (%u)\n",
+		dev_err(dmaengine_chan_dev(chan), "%s: unsupported data width (%u)\n",
 			__func__, dev_width);
 		return NULL;
 	}
@@ -341,7 +336,7 @@ static int moxart_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct moxart_chan *ch = to_moxart_dma_chan(chan);
 
-	dev_dbg(chan2dev(chan), "%s: allocating channel #%u\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: allocating channel #%u\n",
 		__func__, ch->ch_num);
 	ch->allocated = 1;
 
@@ -354,7 +349,7 @@ static void moxart_free_chan_resources(struct dma_chan *chan)
 
 	vchan_free_chan_resources(&ch->vc);
 
-	dev_dbg(chan2dev(chan), "%s: freeing channel #%u\n",
+	dev_dbg(dmaengine_chan_dev(chan), "%s: freeing channel #%u\n",
 		__func__, ch->ch_num);
 	ch->allocated = 0;
 }
@@ -379,7 +374,7 @@ static void moxart_set_transfer_params(struct moxart_chan *ch, unsigned int len)
 	 */
 	writel(d->dma_cycles, ch->base + REG_OFF_CYCLES);
 
-	dev_dbg(chan2dev(&ch->vc.chan), "%s: set %u DMA cycles (len=%u)\n",
+	dev_dbg(vchan_chan_dev(&ch->vc), "%s: set %u DMA cycles (len=%u)\n",
 		__func__, d->dma_cycles, len);
 }
 
@@ -460,7 +455,7 @@ static size_t moxart_dma_desc_size_in_flight(struct moxart_chan *ch)
 	completed_cycles = (ch->desc->dma_cycles - cycles);
 	size -= completed_cycles << es_bytes[ch->desc->es];
 
-	dev_dbg(chan2dev(&ch->vc.chan), "%s: size=%zu\n", __func__, size);
+	dev_dbg(vchan_chan_dev(&ch->vc), "%s: size=%zu\n", __func__, size);
 
 	return size;
 }
@@ -517,7 +512,7 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid)
 	unsigned int i;
 	u32 ctrl;
 
-	dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__);
+	dev_dbg(vchan_chan_dev(&ch->vc), "%s\n", __func__);
 
 	for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {
 		if (!ch->allocated)
@@ -525,7 +520,7 @@ static irqreturn_t moxart_dma_interrupt(int irq, void *devid)
 
 		ctrl = readl(ch->base + REG_OFF_CTRL);
 
-		dev_dbg(chan2dev(&ch->vc.chan), "%s: ch=%p ch->base=%p ctrl=%x\n",
+		dev_dbg(vchan_chan_dev(&ch->vc), "%s: ch=%p ch->base=%p ctrl=%x\n",
 			__func__, ch, ch->base, ctrl);
 
 		if (ctrl & APB_DMA_FIN_INT_STS) {
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 25ed61f1b08999..c4e0dce3ca64e3 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -583,8 +583,13 @@ mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 		while (src_cnt--) {
 			/* Check if a new window needs to get added for 'src' */
 			ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);
-			if (ret)
+			if (ret) {
+				spin_lock_bh(&mv_chan->lock);
+				list_move_tail(&sw_desc->node,
+					       &mv_chan->free_slots);
+				spin_unlock_bh(&mv_chan->lock);
 				return NULL;
+			}
 			mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
 		}
 	}
@@ -1162,6 +1167,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
 
 err_free_irq:
 	free_irq(mv_chan->irq, mv_chan);
+	tasklet_kill(&mv_chan->irq_tasklet);
 err_free_dma:
 	dma_free_wc(&pdev->dev, MV_XOR_POOL_SIZE,
 			  mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c
index 05d7321629cc8f..744cd14d78725b 100644
--- a/drivers/dma/nbpfaxi.c
+++ b/drivers/dma/nbpfaxi.c
@@ -1202,7 +1202,7 @@ static irqreturn_t nbpf_chan_irq(int irq, void *dev)
 
 	nbpf_status_ack(chan);
 
-	dev_dbg(&chan->dma_chan.dev->device, "%s()\n", __func__);
+	dev_dbg(dmaengine_chan_dev(&chan->dma_chan), "%s()\n", __func__);
 
 	spin_lock(&chan->lock);
 	desc = chan->running;
diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c
index 7c80572fc71d13..9dc88d804a2e47 100644
--- a/drivers/dma/owl-dma.c
+++ b/drivers/dma/owl-dma.c
@@ -292,11 +292,6 @@ static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
 	return container_of(dd, struct owl_dma, dma);
 }
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
 {
 	return container_of(chan, struct owl_dma_vchan, vc.chan);
@@ -573,7 +568,7 @@ static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
 
 	spin_unlock_irqrestore(&od->lock, flags);
 
-	dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d\n", pchan->id);
+	dev_dbg(vchan_chan_dev(&vchan->vc), "starting pchan %d\n", pchan->id);
 
 	/* Start DMA transfer for this pchan */
 	pchan_writel(pchan, OWL_DMAX_START, 0x1);
@@ -757,7 +752,7 @@ static int owl_dma_resume(struct dma_chan *chan)
 	if (!vchan->pchan && !vchan->txd)
 		return 0;
 
-	dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
+	dev_dbg(dmaengine_chan_dev(chan), "vchan %p: resume\n", &vchan->vc);
 
 	spin_lock_irqsave(&vchan->vc.lock, flags);
 
@@ -888,7 +883,7 @@ static struct dma_async_tx_descriptor
 	for (offset = 0; offset < len; offset += bytes) {
 		lli = owl_dma_alloc_lli(od);
 		if (!lli) {
-			dev_warn(chan2dev(chan), "failed to allocate lli\n");
+			dev_warn(dmaengine_chan_dev(chan), "failed to allocate lli\n");
 			goto err_txd_free;
 		}
 
@@ -898,7 +893,7 @@ static struct dma_async_tx_descriptor
 				      bytes, DMA_MEM_TO_MEM,
 				      &vchan->cfg, txd->cyclic);
 		if (ret) {
-			dev_warn(chan2dev(chan), "failed to config lli\n");
+			dev_warn(dmaengine_chan_dev(chan), "failed to config lli\n");
 			goto err_txd_free;
 		}
 
@@ -947,7 +942,7 @@ static struct dma_async_tx_descriptor
 
 		lli = owl_dma_alloc_lli(od);
 		if (!lli) {
-			dev_err(chan2dev(chan), "failed to allocate lli");
+			dev_err(dmaengine_chan_dev(chan), "failed to allocate lli");
 			goto err_txd_free;
 		}
 
@@ -962,7 +957,7 @@ static struct dma_async_tx_descriptor
 		ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,
 				      txd->cyclic);
 		if (ret) {
-			dev_warn(chan2dev(chan), "failed to config lli");
+			dev_warn(dmaengine_chan_dev(chan), "failed to config lli");
 			goto err_txd_free;
 		}
 
@@ -1003,7 +998,7 @@ static struct dma_async_tx_descriptor
 	for (i = 0; i < periods; i++) {
 		lli = owl_dma_alloc_lli(od);
 		if (!lli) {
-			dev_warn(chan2dev(chan), "failed to allocate lli");
+			dev_warn(dmaengine_chan_dev(chan), "failed to allocate lli");
 			goto err_txd_free;
 		}
 
@@ -1018,7 +1013,7 @@ static struct dma_async_tx_descriptor
 		ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,
 				      dir, sconfig, txd->cyclic);
 		if (ret) {
-			dev_warn(chan2dev(chan), "failed to config lli");
+			dev_warn(dmaengine_chan_dev(chan), "failed to config lli");
 			goto err_txd_free;
 		}
 
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index bf805f1024f649..88baf7ffec3751 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -150,11 +150,6 @@ static inline struct pch_dma *to_pd(struct dma_device *ddev)
 	return container_of(ddev, struct pch_dma, dma);
 }
 
-static inline struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static inline
 struct pch_dma_desc *pdc_first_active(struct pch_dma_chan *pd_chan)
 {
@@ -189,7 +184,7 @@ static void pdc_enable_irq(struct dma_chan *chan, int enable)
 
 	dma_writel(pd, CTL2, val);
 
-	dev_dbg(chan2dev(chan), "pdc_enable_irq: chan %d -> %x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "pdc_enable_irq: chan %d -> %x\n",
 		chan->chan_id, val);
 }
 
@@ -237,7 +232,7 @@ static void pdc_set_dir(struct dma_chan *chan)
 		dma_writel(pd, CTL3, val);
 	}
 
-	dev_dbg(chan2dev(chan), "pdc_set_dir: chan %d -> %x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "pdc_set_dir: chan %d -> %x\n",
 		chan->chan_id, val);
 }
 
@@ -271,7 +266,7 @@ static void pdc_set_mode(struct dma_chan *chan, u32 mode)
 		dma_writel(pd, CTL3, val);
 	}
 
-	dev_dbg(chan2dev(chan), "pdc_set_mode: chan %d -> %x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "pdc_set_mode: chan %d -> %x\n",
 		chan->chan_id, val);
 }
 
@@ -314,18 +309,18 @@ static bool pdc_is_idle(struct pch_dma_chan *pd_chan)
 static void pdc_dostart(struct pch_dma_chan *pd_chan, struct pch_dma_desc* desc)
 {
 	if (!pdc_is_idle(pd_chan)) {
-		dev_err(chan2dev(&pd_chan->chan),
+		dev_err(dmaengine_chan_dev(&pd_chan->chan),
 			"BUG: Attempt to start non-idle channel\n");
 		return;
 	}
 
-	dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> dev_addr: %x\n",
+	dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> dev_addr: %x\n",
 		pd_chan->chan.chan_id, desc->regs.dev_addr);
-	dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> mem_addr: %x\n",
+	dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> mem_addr: %x\n",
 		pd_chan->chan.chan_id, desc->regs.mem_addr);
-	dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> size: %x\n",
+	dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> size: %x\n",
 		pd_chan->chan.chan_id, desc->regs.size);
-	dev_dbg(chan2dev(&pd_chan->chan), "chan %d -> next: %x\n",
+	dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "chan %d -> next: %x\n",
 		pd_chan->chan.chan_id, desc->regs.next);
 
 	if (list_empty(&desc->tx_list)) {
@@ -382,8 +377,8 @@ static void pdc_handle_error(struct pch_dma_chan *pd_chan)
 	if (!list_empty(&pd_chan->active_list))
 		pdc_dostart(pd_chan, pdc_first_active(pd_chan));
 
-	dev_crit(chan2dev(&pd_chan->chan), "Bad descriptor submitted\n");
-	dev_crit(chan2dev(&pd_chan->chan), "descriptor cookie: %d\n",
+	dev_crit(dmaengine_chan_dev(&pd_chan->chan), "Bad descriptor submitted\n");
+	dev_crit(dmaengine_chan_dev(&pd_chan->chan), "descriptor cookie: %d\n",
 		 bad_desc->txd.cookie);
 
 	pdc_chain_complete(pd_chan, bad_desc);
@@ -450,10 +445,10 @@ static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan)
 			ret = desc;
 			break;
 		}
-		dev_dbg(chan2dev(&pd_chan->chan), "desc %p not ACKed\n", desc);
+		dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "desc %p not ACKed\n", desc);
 	}
 	spin_unlock(&pd_chan->lock);
-	dev_dbg(chan2dev(&pd_chan->chan), "scanned %d descriptors\n", i);
+	dev_dbg(dmaengine_chan_dev(&pd_chan->chan), "scanned %d descriptors\n", i);
 
 	if (!ret) {
 		ret = pdc_alloc_desc(&pd_chan->chan, GFP_ATOMIC);
@@ -462,7 +457,7 @@ static struct pch_dma_desc *pdc_desc_get(struct pch_dma_chan *pd_chan)
 			pd_chan->descs_allocated++;
 			spin_unlock(&pd_chan->lock);
 		} else {
-			dev_err(chan2dev(&pd_chan->chan),
+			dev_err(dmaengine_chan_dev(&pd_chan->chan),
 				"failed to alloc desc\n");
 		}
 	}
@@ -489,7 +484,7 @@ static int pd_alloc_chan_resources(struct dma_chan *chan)
 	int i;
 
 	if (!pdc_is_idle(pd_chan)) {
-		dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
+		dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle ?\n");
 		return -EIO;
 	}
 
@@ -500,7 +495,7 @@ static int pd_alloc_chan_resources(struct dma_chan *chan)
 		desc = pdc_alloc_desc(chan, GFP_KERNEL);
 
 		if (!desc) {
-			dev_warn(chan2dev(chan),
+			dev_warn(dmaengine_chan_dev(chan),
 				"Only allocated %d initial descriptors\n", i);
 			break;
 		}
@@ -573,7 +568,7 @@ static struct dma_async_tx_descriptor *pd_prep_slave_sg(struct dma_chan *chan,
 	int i;
 
 	if (unlikely(!sg_len)) {
-		dev_info(chan2dev(chan), "prep_slave_sg: length is zero!\n");
+		dev_info(dmaengine_chan_dev(chan), "prep_slave_sg: length is zero!\n");
 		return NULL;
 	}
 
@@ -639,7 +634,7 @@ static struct dma_async_tx_descriptor *pd_prep_slave_sg(struct dma_chan *chan,
 	return &first->txd;
 
 err_desc_get:
-	dev_err(chan2dev(chan), "failed to get desc or wrong parameters\n");
+	dev_err(dmaengine_chan_dev(chan), "failed to get desc or wrong parameters\n");
 	pdc_desc_put(pd_chan, first);
 	return NULL;
 }
@@ -671,7 +666,7 @@ static void pdc_tasklet(struct tasklet_struct *t)
 	unsigned long flags;
 
 	if (!pdc_is_idle(pd_chan)) {
-		dev_err(chan2dev(&pd_chan->chan),
+		dev_err(dmaengine_chan_dev(&pd_chan->chan),
 			"BUG: handle non-idle channel in tasklet\n");
 		return;
 	}
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 6214d9000db8eb..c2d0b473b8f485 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -2953,8 +2953,8 @@ DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
 
 static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
 {
-	pl330->dbgfs = debugfs_create_file(dev_name(pl330->ddma.dev),
-					   S_IFREG | 0444, NULL, pl330,
+	pl330->dbgfs = debugfs_create_file("stats", 0444,
+					   pl330->ddma.dbg_dev_root, pl330,
 					   &pl330_debugfs_fops);
 }
 
diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c
index fa2ee0b3e09f8c..4f0d1d531f0101 100644
--- a/drivers/dma/pxa_dma.c
+++ b/drivers/dma/pxa_dma.c
@@ -149,7 +149,7 @@ struct pxad_device {
 	({								\
 		u32 _v;							\
 		_v = readl_relaxed((phy)->base + _reg((phy)->idx));	\
-		dev_vdbg(&phy->vchan->vc.chan.dev->device,		\
+		dev_vdbg(vchan_chan_dev(&phy->vchan->vc),		\
 			 "%s(): readl(%s): 0x%08x\n", __func__, #_reg,	\
 			  _v);						\
 		_v;							\
@@ -157,14 +157,14 @@ struct pxad_device {
 #define phy_writel(phy, val, _reg)					\
 	do {								\
 		writel((val), (phy)->base + _reg((phy)->idx));		\
-		dev_vdbg(&phy->vchan->vc.chan.dev->device,		\
+		dev_vdbg(vchan_chan_dev(&phy->vchan->vc),		\
 			 "%s(): writel(0x%08x, %s)\n",			\
 			 __func__, (u32)(val), #_reg);			\
 	} while (0)
 #define phy_writel_relaxed(phy, val, _reg)				\
 	do {								\
 		writel_relaxed((val), (phy)->base + _reg((phy)->idx));	\
-		dev_vdbg(&phy->vchan->vc.chan.dev->device,		\
+		dev_vdbg(vchan_chan_dev(&phy->vchan->vc),		\
 			 "%s(): writel_relaxed(0x%08x, %s)\n",		\
 			 __func__, (u32)(val), #_reg);			\
 	} while (0)
@@ -396,7 +396,7 @@ static struct pxad_phy *lookup_phy(struct pxad_chan *pchan)
 
 out_unlock:
 	spin_unlock_irqrestore(&pdev->phy_lock, flags);
-	dev_dbg(&pchan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&pchan->vc),
 		"%s(): phy=%p(%d)\n", __func__, found,
 		found ? found->idx : -1);
 
@@ -409,7 +409,7 @@ static void pxad_free_phy(struct pxad_chan *chan)
 	unsigned long flags;
 	u32 reg;
 
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): freeing\n", __func__);
 	if (!chan->phy)
 		return;
@@ -454,7 +454,7 @@ static void phy_enable(struct pxad_phy *phy, bool misaligned)
 	if (!phy->vchan)
 		return;
 
-	dev_dbg(&phy->vchan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&phy->vchan->vc),
 		"%s(); phy=%p(%d) misaligned=%d\n", __func__,
 		phy, phy->idx, misaligned);
 
@@ -483,7 +483,7 @@ static void phy_disable(struct pxad_phy *phy)
 		return;
 
 	dcsr = phy_readl_relaxed(phy, DCSR);
-	dev_dbg(&phy->vchan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&phy->vchan->vc),
 		"%s(): phy=%p(%d)\n", __func__, phy, phy->idx);
 	phy_writel(phy, dcsr & ~PXA_DCSR_RUN & ~PXA_DCSR_STOPIRQEN, DCSR);
 }
@@ -491,12 +491,12 @@ static void phy_disable(struct pxad_phy *phy)
 static void pxad_launch_chan(struct pxad_chan *chan,
 				 struct pxad_desc_sw *desc)
 {
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): desc=%p\n", __func__, desc);
 	if (!chan->phy) {
 		chan->phy = lookup_phy(chan);
 		if (!chan->phy) {
-			dev_dbg(&chan->vc.chan.dev->device,
+			dev_dbg(vchan_chan_dev(&chan->vc),
 				"%s(): no free dma channel\n", __func__);
 			return;
 		}
@@ -592,7 +592,7 @@ static unsigned int clear_chan_irq(struct pxad_phy *phy)
 	dcsr = phy_readl_relaxed(phy, DCSR);
 	phy_writel(phy, dcsr, DCSR);
 	if ((dcsr & PXA_DCSR_BUSERR) && (phy->vchan))
-		dev_warn(&phy->vchan->vc.chan.dev->device,
+		dev_warn(vchan_chan_dev(&phy->vchan->vc),
 			 "%s(chan=%p): PXA_DCSR_BUSERR\n",
 			 __func__, &phy->vchan);
 
@@ -617,7 +617,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
 	spin_lock(&chan->vc.lock);
 	list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {
 		vd_completed = is_desc_completed(vd);
-		dev_dbg(&chan->vc.chan.dev->device,
+		dev_dbg(vchan_chan_dev(&chan->vc),
 			"%s(): checking txd %p[%x]: completed=%d dcsr=0x%x\n",
 			__func__, vd, vd->tx.cookie, vd_completed,
 			dcsr);
@@ -640,7 +640,7 @@ static irqreturn_t pxad_chan_handler(int irq, void *dev_id)
 	}
 
 	if (!chan->bus_error && dcsr & PXA_DCSR_STOPSTATE) {
-		dev_dbg(&chan->vc.chan.dev->device,
+		dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): channel stopped, submitted_empty=%d issued_empty=%d",
 			__func__,
 			list_empty(&chan->vc.desc_submitted),
@@ -694,7 +694,7 @@ static int pxad_alloc_chan_resources(struct dma_chan *dchan)
 					  __alignof__(struct pxad_desc_hw),
 					  0);
 	if (!chan->desc_pool) {
-		dev_err(&chan->vc.chan.dev->device,
+		dev_err(vchan_chan_dev(&chan->vc),
 			"%s(): unable to allocate descriptor pool\n",
 			__func__);
 		return -ENOMEM;
@@ -744,18 +744,19 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc)
 	sw_desc = kzalloc_flex(*sw_desc, hw_desc, nb_hw_desc, GFP_NOWAIT);
 	if (!sw_desc)
 		return NULL;
+	sw_desc->nb_desc = nb_hw_desc;
 	sw_desc->desc_pool = chan->desc_pool;
 
 	for (i = 0; i < nb_hw_desc; i++) {
 		desc = dma_pool_alloc(sw_desc->desc_pool, GFP_NOWAIT, &dma);
 		if (!desc) {
-			dev_err(&chan->vc.chan.dev->device,
+			dev_err(vchan_chan_dev(&chan->vc),
 				"%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n",
 				__func__, i, sw_desc->desc_pool);
+			sw_desc->nb_desc = i;
 			goto err;
 		}
 
-		sw_desc->nb_desc++;
 		sw_desc->hw_desc[i] = desc;
 
 		if (i == 0)
@@ -786,7 +787,7 @@ static dma_cookie_t pxad_tx_submit(struct dma_async_tx_descriptor *tx)
 
 	if (list_empty(&vc->desc_submitted) && pxad_try_hotchain(vc, vd)) {
 		list_move_tail(&vd->node, &vc->desc_issued);
-		dev_dbg(&chan->vc.chan.dev->device,
+		dev_dbg(vchan_chan_dev(&chan->vc),
 			"%s(): txd %p[%x]: submitted (hot linked)\n",
 			__func__, vd, cookie);
 		goto out;
@@ -809,7 +810,7 @@ static dma_cookie_t pxad_tx_submit(struct dma_async_tx_descriptor *tx)
 		else
 			vd_chained = NULL;
 	}
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): txd %p[%x]: submitted (%s linked)\n",
 		__func__, vd, cookie, vd_chained ? "cold" : "not");
 	list_move_tail(&vd->node, &vc->desc_submitted);
@@ -832,7 +833,7 @@ static void pxad_issue_pending(struct dma_chan *dchan)
 
 	vd_first = list_first_entry(&chan->vc.desc_submitted,
 				    struct virt_dma_desc, node);
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): txd %p[%x]", __func__, vd_first, vd_first->tx.cookie);
 
 	vchan_issue_pending(&chan->vc);
@@ -852,7 +853,7 @@ pxad_tx_prep(struct virt_dma_chan *vc, struct virt_dma_desc *vd,
 	INIT_LIST_HEAD(&vd->node);
 	tx = vchan_tx_prep(vc, vd, tx_flags);
 	tx->tx_submit = pxad_tx_submit;
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): vc=%p txd=%p[%x] flags=0x%lx\n", __func__,
 		vc, vd, vd->tx.cookie,
 		tx_flags);
@@ -891,7 +892,7 @@ static void pxad_get_config(struct pxad_chan *chan,
 		*dcmd |= PXA_DCMD_BURST32 | PXA_DCMD_INCTRGADDR |
 			PXA_DCMD_INCSRCADDR;
 
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): dev_addr=0x%x maxburst=%d width=%d  dir=%d\n",
 		__func__, dev_addr, maxburst, width, dir);
 
@@ -925,7 +926,7 @@ pxad_prep_memcpy(struct dma_chan *dchan,
 	if (!dchan || !len)
 		return NULL;
 
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): dma_dst=0x%lx dma_src=0x%lx len=%zu flags=%lx\n",
 		__func__, (unsigned long)dma_dst, (unsigned long)dma_src,
 		len, flags);
@@ -974,7 +975,7 @@ pxad_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 		return NULL;
 
 	pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr);
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): dir=%d flags=%lx\n", __func__, dir, flags);
 
 	nb_desc = sg_nents_for_dma(sgl, sg_len, PDMA_MAX_DESC_BYTES);
@@ -1021,7 +1022,7 @@ pxad_prep_dma_cyclic(struct dma_chan *dchan,
 	if (!dchan || !len || !period_len)
 		return NULL;
 	if ((dir != DMA_DEV_TO_MEM) && (dir != DMA_MEM_TO_DEV)) {
-		dev_err(&chan->vc.chan.dev->device,
+		dev_err(vchan_chan_dev(&chan->vc),
 			"Unsupported direction for cyclic DMA\n");
 		return NULL;
 	}
@@ -1032,7 +1033,7 @@ pxad_prep_dma_cyclic(struct dma_chan *dchan,
 
 	pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr);
 	dcmd |= PXA_DCMD_ENDIRQEN | (PXA_DCMD_LENGTH & period_len);
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): buf_addr=0x%lx len=%zu period=%zu dir=%d flags=%lx\n",
 		__func__, (unsigned long)buf_addr, len, period_len, dir, flags);
 
@@ -1080,14 +1081,14 @@ static int pxad_terminate_all(struct dma_chan *dchan)
 	struct pxad_phy *phy;
 	LIST_HEAD(head);
 
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): vchan %p: terminate all\n", __func__, &chan->vc);
 
 	spin_lock_irqsave(&chan->vc.lock, flags);
 	vchan_get_all_descriptors(&chan->vc, &head);
 
 	list_for_each_entry(vd, &head, node) {
-		dev_dbg(&chan->vc.chan.dev->device,
+		dev_dbg(vchan_chan_dev(&chan->vc),
 			"%s(): cancelling txd %p[%x] (completed=%d)", __func__,
 			vd, vd->tx.cookie, is_desc_completed(vd));
 	}
@@ -1177,7 +1178,7 @@ static unsigned int pxad_residue(struct pxad_chan *chan,
 
 out:
 	spin_unlock_irqrestore(&chan->vc.lock, flags);
-	dev_dbg(&chan->vc.chan.dev->device,
+	dev_dbg(vchan_chan_dev(&chan->vc),
 		"%s(): txd %p[%x] sw_desc=%p: %d\n",
 		__func__, vd, cookie, sw_desc, residue);
 	return residue;
diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c
index a5055a6273af62..c41bfac65ddf3f 100644
--- a/drivers/dma/qcom/gpi.c
+++ b/drivers/dma/qcom/gpi.c
@@ -1965,16 +1965,19 @@ static int gpi_ch_init(struct gchan *gchan)
 error_start_chan:
 	for (i = i - 1; i >= 0; i--) {
 		gpi_stop_chan(&gpii->gchan[i]);
-		gpi_send_cmd(gpii, gchan, GPI_CH_CMD_RESET);
+		gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_RESET);
 	}
 	i = 2;
 error_alloc_chan:
 	for (i = i - 1; i >= 0; i--)
-		gpi_reset_chan(gchan, GPI_CH_CMD_DE_ALLOC);
+		gpi_reset_chan(&gpii->gchan[i], GPI_CH_CMD_DE_ALLOC);
 error_alloc_ev_ring:
 	gpi_disable_interrupts(gpii);
 error_config_int:
 	gpi_free_ring(&gpii->ev_ring, gpii);
+	write_lock_irq(&gpii->pm_lock);
+	gpii->pm_state = DISABLE_STATE;
+	write_unlock_irq(&gpii->pm_lock);
 exit_gpi_init:
 	return ret;
 }
@@ -2065,6 +2068,8 @@ static int gpi_alloc_chan_resources(struct dma_chan *chan)
 		goto xfer_alloc_err;
 
 	ret = gpi_ch_init(gchan);
+	if (ret)
+		gpi_free_ring(&gchan->ch_ring, gpii);
 
 	mutex_unlock(&gpii->ctrl_lock);
 
diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c
index 087fea3af2e411..60b3067a009945 100644
--- a/drivers/dma/sprd-dma.c
+++ b/drivers/dma/sprd-dma.c
@@ -208,8 +208,7 @@ struct sprd_dma_chn {
 struct sprd_dma_dev {
 	struct dma_device	dma_dev;
 	void __iomem		*glb_base;
-	struct clk		*clk;
-	struct clk		*ashb_clk;
+	struct clk_bulk_data	clks[2];
 	int			irq;
 	u32			total_chns;
 	struct sprd_dma_chn	channels[] __counted_by(total_chns);
@@ -260,31 +259,12 @@ static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg,
 
 static int sprd_dma_enable(struct sprd_dma_dev *sdev)
 {
-	int ret;
-
-	ret = clk_prepare_enable(sdev->clk);
-	if (ret)
-		return ret;
-
-	/*
-	 * The ashb_clk is optional and only for AGCP DMA controller, so we
-	 * need add one condition to check if the ashb_clk need enable.
-	 */
-	if (!IS_ERR(sdev->ashb_clk))
-		ret = clk_prepare_enable(sdev->ashb_clk);
-
-	return ret;
+	return clk_bulk_prepare_enable(ARRAY_SIZE(sdev->clks), sdev->clks);
 }
 
 static void sprd_dma_disable(struct sprd_dma_dev *sdev)
 {
-	clk_disable_unprepare(sdev->clk);
-
-	/*
-	 * Need to check if we need disable the optional ashb_clk for AGCP DMA.
-	 */
-	if (!IS_ERR(sdev->ashb_clk))
-		clk_disable_unprepare(sdev->ashb_clk);
+	clk_bulk_disable_unprepare(ARRAY_SIZE(sdev->clks), sdev->clks);
 }
 
 static void sprd_dma_set_uid(struct sprd_dma_chn *schan)
@@ -1140,16 +1120,17 @@ static int sprd_dma_probe(struct platform_device *pdev)
 	if (!sdev)
 		return -ENOMEM;
 
-	sdev->clk = devm_clk_get(&pdev->dev, "enable");
-	if (IS_ERR(sdev->clk)) {
-		dev_err(&pdev->dev, "get enable clock failed\n");
-		return PTR_ERR(sdev->clk);
-	}
+	/* The ashb_eb clock is optional and only present for AGCP DMA */
+	sdev->clks[0].id = "enable";
+	sdev->clks[1].id = "ashb_eb";
+	ret = devm_clk_bulk_get_optional(&pdev->dev, ARRAY_SIZE(sdev->clks),
+					 sdev->clks);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "get clocks failed\n");
 
-	/* ashb clock is optional for AGCP DMA */
-	sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb");
-	if (IS_ERR(sdev->ashb_clk))
-		dev_warn(&pdev->dev, "no optional ashb eb clock\n");
+	if (!sdev->clks[0].clk)
+		return dev_err_probe(&pdev->dev, -ENOENT,
+				     "get enable clock failed\n");
 
 	/*
 	 * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP
@@ -1212,7 +1193,7 @@ static int sprd_dma_probe(struct platform_device *pdev)
 
 	ret = pm_runtime_get_sync(&pdev->dev);
 	if (ret < 0)
-		goto err_rpm;
+		goto err_register;
 
 	ret = dma_async_device_register(&sdev->dma_dev);
 	if (ret < 0) {
@@ -1234,7 +1215,6 @@ static int sprd_dma_probe(struct platform_device *pdev)
 err_register:
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-err_rpm:
 	sprd_dma_disable(sdev);
 	return ret;
 }
diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c
index d9547017f3bd61..35dbff5f9f1abb 100644
--- a/drivers/dma/st_fdma.c
+++ b/drivers/dma/st_fdma.c
@@ -268,7 +268,7 @@ static int st_fdma_alloc_chan_res(struct dma_chan *chan)
 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
 
 	/* Create the dma pool for descriptor allocation */
-	fchan->node_pool = dma_pool_create(dev_name(&chan->dev->device),
+	fchan->node_pool = dma_pool_create(dma_chan_name(chan),
 					    fchan->fdev->dev,
 					    sizeof(struct st_fdma_hw_node),
 					    __alignof__(struct st_fdma_hw_node),
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 0d9ffa3e266398..cb93e97ba5f6e0 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -15,6 +15,7 @@
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/log2.h>
+#include <linux/interrupt.h>
 #include <linux/pm.h>
 #include <linux/pm_runtime.h>
 #include <linux/err.h>
@@ -623,11 +624,6 @@ struct d40_base {
 	struct d40_chan			 phy_chans[];
 };
 
-static struct device *chan2dev(struct d40_chan *d40c)
-{
-	return &d40c->chan.dev->device;
-}
-
 static bool chan_is_physical(struct d40_chan *chan)
 {
 	return chan->log_num == D40_PHY_CHAN;
@@ -648,7 +644,7 @@ static void __iomem *chan_base(struct d40_chan *chan)
 	dev_err(dev, "[%s] " format, __func__, ## arg)
 
 #define chan_err(d40c, format, arg...)		\
-	d40_err(chan2dev(d40c), format, ## arg)
+	d40_err(dmaengine_chan_dev(&d40c->chan), format, ## arg)
 
 static int d40_set_runtime_config_write(struct dma_chan *chan,
 				  struct dma_slave_config *config,
@@ -1243,7 +1239,7 @@ static void __d40_config_set_event(struct d40_chan *d40c,
 		}
 
 		if (tries != 99)
-			dev_dbg(chan2dev(d40c),
+			dev_dbg(dmaengine_chan_dev(&d40c->chan),
 				"[%s] workaround enable S%cLNK (%d tries)\n",
 				__func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
 				100 - tries);
@@ -1924,7 +1920,7 @@ static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
 			i = d40c->dma_cfg.phy_channel;
 
 			if ((i != phy_num) && (i != phy_num + 1)) {
-				dev_err(chan2dev(d40c),
+				dev_err(dmaengine_chan_dev(&d40c->chan),
 					"invalid fixed phy channel %d\n", i);
 				return -EINVAL;
 			}
@@ -1933,7 +1929,7 @@ static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
 					       is_log, first_phy_user))
 				goto found_log;
 
-			dev_err(chan2dev(d40c),
+			dev_err(dmaengine_chan_dev(&d40c->chan),
 				"could not allocate fixed phy channel %d\n", i);
 			return -EINVAL;
 		}
@@ -2446,7 +2442,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
 		d40c->dst_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS);
 	}
 
-	dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",
+	dev_dbg(dmaengine_chan_dev(&d40c->chan), "allocated %s channel (phy %d%s)\n",
 		 chan_is_logical(d40c) ? "logical" : "physical",
 		 d40c->phy_chan->num,
 		 d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");
diff --git a/drivers/dma/stm32/stm32-dma.c b/drivers/dma/stm32/stm32-dma.c
index d3ad78562a1403..4688cebaf7f7a2 100644
--- a/drivers/dma/stm32/stm32-dma.c
+++ b/drivers/dma/stm32/stm32-dma.c
@@ -248,11 +248,6 @@ static struct stm32_dma_desc *to_stm32_dma_desc(struct virt_dma_desc *vdesc)
 	return container_of(vdesc, struct stm32_dma_desc, vdesc);
 }
 
-static struct device *chan2dev(struct stm32_dma_chan *chan)
-{
-	return &chan->vchan.chan.dev->device;
-}
-
 static u32 stm32_dma_read(struct stm32_dma_device *dmadev, u32 reg)
 {
 	return readl_relaxed(dmadev->base + reg);
@@ -274,7 +269,7 @@ static int stm32_dma_get_width(struct stm32_dma_chan *chan,
 	case DMA_SLAVE_BUSWIDTH_4_BYTES:
 		return STM32_DMA_WORD;
 	default:
-		dev_err(chan2dev(chan), "Dma bus width not supported\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Dma bus width not supported\n");
 		return -EINVAL;
 	}
 }
@@ -374,7 +369,7 @@ static int stm32_dma_get_burst(struct stm32_dma_chan *chan, u32 maxburst)
 	case 16:
 		return STM32_DMA_BURST_INCR16;
 	default:
-		dev_err(chan2dev(chan), "Dma burst size not supported\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Dma burst size not supported\n");
 		return -EINVAL;
 	}
 }
@@ -487,7 +482,7 @@ static void stm32_dma_stop(struct stm32_dma_chan *chan)
 	/* Clear interrupt status if it is there */
 	status = stm32_dma_irq_status(chan);
 	if (status) {
-		dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",
+		dev_dbg(vchan_chan_dev(&chan->vchan), "%s(): clearing interrupt: 0x%08x\n",
 			__func__, status);
 		stm32_dma_irq_clear(chan, status);
 	}
@@ -536,12 +531,12 @@ static void stm32_dma_dump_reg(struct stm32_dma_chan *chan)
 	u32 sm1ar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(chan->id));
 	u32 sfcr = stm32_dma_read(dmadev, STM32_DMA_SFCR(chan->id));
 
-	dev_dbg(chan2dev(chan), "SCR:   0x%08x\n", scr);
-	dev_dbg(chan2dev(chan), "NDTR:  0x%08x\n", ndtr);
-	dev_dbg(chan2dev(chan), "SPAR:  0x%08x\n", spar);
-	dev_dbg(chan2dev(chan), "SM0AR: 0x%08x\n", sm0ar);
-	dev_dbg(chan2dev(chan), "SM1AR: 0x%08x\n", sm1ar);
-	dev_dbg(chan2dev(chan), "SFCR:  0x%08x\n", sfcr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "SCR:   0x%08x\n", scr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "NDTR:  0x%08x\n", ndtr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "SPAR:  0x%08x\n", spar);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "SM0AR: 0x%08x\n", sm0ar);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "SM1AR: 0x%08x\n", sm1ar);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "SFCR:  0x%08x\n", sfcr);
 }
 
 static void stm32_dma_sg_inc(struct stm32_dma_chan *chan)
@@ -613,7 +608,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan)
 	reg->dma_scr |= STM32_DMA_SCR_EN;
 	stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr);
 
-	dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: started\n", &chan->vchan);
 }
 
 static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan)
@@ -630,12 +625,12 @@ static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan)
 	if (dma_scr & STM32_DMA_SCR_CT) {
 		dma_sm0ar = sg_req->chan_reg.dma_sm0ar;
 		stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar);
-		dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n",
+		dev_dbg(vchan_chan_dev(&chan->vchan), "CT=1 <=> SM0AR: 0x%08x\n",
 			stm32_dma_read(dmadev, STM32_DMA_SM0AR(id)));
 	} else {
 		dma_sm1ar = sg_req->chan_reg.dma_sm1ar;
 		stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar);
-		dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n",
+		dev_dbg(vchan_chan_dev(&chan->vchan), "CT=0 <=> SM1AR: 0x%08x\n",
 			stm32_dma_read(dmadev, STM32_DMA_SM1AR(id)));
 	}
 }
@@ -676,7 +671,7 @@ static void stm32_dma_handle_chan_paused(struct stm32_dma_chan *chan)
 
 	chan->status = DMA_PAUSED;
 
-	dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: paused\n", &chan->vchan);
 }
 
 static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan)
@@ -728,7 +723,7 @@ static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan)
 	dma_scr |= STM32_DMA_SCR_EN;
 	stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr);
 
-	dev_dbg(chan2dev(chan), "vchan %p: reconfigured after pause/resume\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: reconfigured after pause/resume\n", &chan->vchan);
 }
 
 static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr)
@@ -775,16 +770,16 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
 		if (sfcr & STM32_DMA_SFCR_FEIE) {
 			if (!(scr & STM32_DMA_SCR_EN) &&
 			    !(status & STM32_DMA_TCI))
-				dev_err(chan2dev(chan), "FIFO Error\n");
+				dev_err(vchan_chan_dev(&chan->vchan), "FIFO Error\n");
 			else
-				dev_dbg(chan2dev(chan), "FIFO over/underrun\n");
+				dev_dbg(vchan_chan_dev(&chan->vchan), "FIFO over/underrun\n");
 		}
 	}
 	if (status & STM32_DMA_DMEI) {
 		stm32_dma_irq_clear(chan, STM32_DMA_DMEI);
 		status &= ~STM32_DMA_DMEI;
 		if (sfcr & STM32_DMA_SCR_DMEIE)
-			dev_dbg(chan2dev(chan), "Direct mode overrun\n");
+			dev_dbg(vchan_chan_dev(&chan->vchan), "Direct mode overrun\n");
 	}
 
 	if (status & STM32_DMA_TCI) {
@@ -803,9 +798,9 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid)
 
 	if (status) {
 		stm32_dma_irq_clear(chan, status);
-		dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
+		dev_err(vchan_chan_dev(&chan->vchan), "DMA error: status=0x%08x\n", status);
 		if (!(scr & STM32_DMA_SCR_EN))
-			dev_err(chan2dev(chan), "chan disabled by HW\n");
+			dev_err(vchan_chan_dev(&chan->vchan), "chan disabled by HW\n");
 	}
 
 	spin_unlock(&chan->vchan.lock);
@@ -820,7 +815,7 @@ static void stm32_dma_issue_pending(struct dma_chan *c)
 
 	spin_lock_irqsave(&chan->vchan.lock, flags);
 	if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) {
-		dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: issued\n", &chan->vchan);
 		stm32_dma_start_transfer(chan);
 
 	}
@@ -922,7 +917,7 @@ static int stm32_dma_resume(struct dma_chan *c)
 
 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
 
-	dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: resumed\n", &chan->vchan);
 
 	return 0;
 }
@@ -1059,7 +1054,7 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan,
 		break;
 
 	default:
-		dev_err(chan2dev(chan), "Dma direction is not supported\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Dma direction is not supported\n");
 		return -EINVAL;
 	}
 
@@ -1092,12 +1087,12 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg(
 	int i, ret;
 
 	if (!chan->config_init) {
-		dev_err(chan2dev(chan), "dma channel is not configured\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "dma channel is not configured\n");
 		return NULL;
 	}
 
 	if (sg_len < 1) {
-		dev_err(chan2dev(chan), "Invalid segment length %d\n", sg_len);
+		dev_err(vchan_chan_dev(&chan->vchan), "Invalid segment length %d\n", sg_len);
 		return NULL;
 	}
 
@@ -1129,7 +1124,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg(
 
 		nb_data_items = desc->sg_req[i].len / buswidth;
 		if (nb_data_items > STM32_DMA_ALIGNED_MAX_DATA_ITEMS) {
-			dev_err(chan2dev(chan), "nb items not supported\n");
+			dev_err(vchan_chan_dev(&chan->vchan), "nb items not supported\n");
 			goto err;
 		}
 
@@ -1164,17 +1159,17 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic(
 	int i, ret;
 
 	if (!buf_len || !period_len) {
-		dev_err(chan2dev(chan), "Invalid buffer/period len\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Invalid buffer/period len\n");
 		return NULL;
 	}
 
 	if (!chan->config_init) {
-		dev_err(chan2dev(chan), "dma channel is not configured\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "dma channel is not configured\n");
 		return NULL;
 	}
 
 	if (buf_len % period_len) {
-		dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "buf_len not multiple of period_len\n");
 		return NULL;
 	}
 
@@ -1185,7 +1180,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic(
 	 * terminating the DMA.
 	 */
 	if (chan->busy) {
-		dev_err(chan2dev(chan), "Request not allowed when dma busy\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Request not allowed when dma busy\n");
 		return NULL;
 	}
 
@@ -1196,7 +1191,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic(
 
 	nb_data_items = period_len / buswidth;
 	if (nb_data_items > STM32_DMA_ALIGNED_MAX_DATA_ITEMS) {
-		dev_err(chan2dev(chan), "number of items not supported\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "number of items not supported\n");
 		return NULL;
 	}
 
@@ -1478,7 +1473,7 @@ static void stm32_dma_free_chan_resources(struct dma_chan *c)
 	struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
 	unsigned long flags;
 
-	dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "Freeing channel %d\n", chan->id);
 
 	if (chan->busy) {
 		spin_lock_irqsave(&chan->vchan.lock, flags);
@@ -1668,7 +1663,7 @@ static int stm32_dma_probe(struct platform_device *pdev)
 
 		ret = devm_request_irq(&pdev->dev, chan->irq,
 				       stm32_dma_chan_irq, 0,
-				       dev_name(chan2dev(chan)), chan);
+				       vchan_chan_name(&chan->vchan), chan);
 		if (ret) {
 			dev_err(&pdev->dev,
 				"request_irq failed with err %d channel %d\n",
diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c
index 4724e7fa0008e1..4fe226541efa49 100644
--- a/drivers/dma/stm32/stm32-dma3.c
+++ b/drivers/dma/stm32/stm32-dma3.c
@@ -328,11 +328,6 @@ static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_des
 	return container_of(vdesc, struct stm32_dma3_swdesc, vdesc);
 }
 
-static struct device *chan2dev(struct stm32_dma3_chan *chan)
-{
-	return &chan->vchan.chan.dev->device;
-}
-
 static struct device *ddata2dev(struct stm32_dma3_ddata *ddata)
 {
 	return ddata->dma_dev.dev;
@@ -341,7 +336,7 @@ static struct device *ddata2dev(struct stm32_dma3_ddata *ddata)
 static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan)
 {
 	struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
-	struct device *dev = chan2dev(chan);
+	struct device *dev = vchan_chan_dev(&chan->vchan);
 	u32 id = chan->id, offset;
 
 	offset = STM32_DMA3_SECCFGR;
@@ -381,21 +376,21 @@ static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan,
 	for (i = 0; i < swdesc->lli_size; i++) {
 		hwdesc = swdesc->lli[i].hwdesc;
 		if (i)
-			dev_dbg(chan2dev(chan), "V\n");
-		dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr);
-		dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1);
-		dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2);
-		dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1);
-		dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar);
-		dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar);
-		dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr);
+			dev_dbg(vchan_chan_dev(&chan->vchan), "V\n");
+		dev_dbg(vchan_chan_dev(&chan->vchan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr);
 	}
 
 	if (swdesc->cyclic) {
-		dev_dbg(chan2dev(chan), "|\n");
-		dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "|\n");
+		dev_dbg(vchan_chan_dev(&chan->vchan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr);
 	} else {
-		dev_dbg(chan2dev(chan), "X\n");
+		dev_dbg(vchan_chan_dev(&chan->vchan), "X\n");
 	}
 }
 
@@ -411,7 +406,7 @@ static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_ch
 	 * addressed, so abort the allocation.
 	 */
 	if ((count * 32) > CLLR_LA) {
-		dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE);
+		dev_err(vchan_chan_dev(&chan->vchan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE);
 		return NULL;
 	}
 
@@ -438,7 +433,7 @@ static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_ch
 	return swdesc;
 
 err_pool_free:
-	dev_err(chan2dev(chan), "Failed to alloc descriptors\n");
+	dev_err(vchan_chan_dev(&chan->vchan), "Failed to alloc descriptors\n");
 	while (--i >= 0)
 		dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);
 	kfree(swdesc);
@@ -468,7 +463,7 @@ static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc)
 static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan)
 {
 	struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
-	struct device *dev = chan2dev(chan);
+	struct device *dev = vchan_chan_dev(&chan->vchan);
 	u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id));
 	u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));
 	u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id));
@@ -579,7 +574,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 	u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw;
 	u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw;
 
-	dev_dbg(chan2dev(chan), "%s from %pad to %pad\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "%s from %pad to %pad\n",
 		dmaengine_get_direction_text(dir), &src_addr, &dst_addr);
 
 	sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst);
@@ -589,12 +584,12 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 
 	/* Following conditions would raise User Setting Error interrupt */
 	if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) {
-		dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw);
+		dev_err(vchan_chan_dev(&chan->vchan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw);
 		return -EINVAL;
 	}
 
 	if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) {
-		dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Only one master port, port 1 is not supported\n");
 		return -EINVAL;
 	}
 
@@ -602,7 +597,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 	dap_max_dw = ddata->ports_max_dw[dap];
 	if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) ||
 	    (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) {
-		dev_err(chan2dev(chan),
+		dev_err(vchan_chan_dev(&chan->vchan),
 			"8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n",
 			sdw, ddw, sap, dap);
 		return -EINVAL;
@@ -659,7 +654,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 			_ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 			/* Should never reach this case as ddw is clamped down */
 			if (len & (ddw - 1)) {
-				dev_err(chan2dev(chan),
+				dev_err(vchan_chan_dev(&chan->vchan),
 					"Packing mode is enabled and len is not multiple of ddw");
 				return -EINVAL;
 			}
@@ -695,7 +690,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 			_ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 			/* Should never reach this case as ddw is clamped down */
 			if (len & (ddw - 1)) {
-				dev_err(chan2dev(chan),
+				dev_err(vchan_chan_dev(&chan->vchan),
 					"Packing mode is enabled and len is not multiple of ddw\n");
 				return -EINVAL;
 			}
@@ -740,7 +735,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 			_ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
 			/* Should never reach this case as ddw is clamped down */
 			if (len & (ddw - 1)) {
-				dev_err(chan2dev(chan),
+				dev_err(vchan_chan_dev(&chan->vchan),
 					"Packing mode is enabled and len is not multiple of ddw");
 				return -EINVAL;
 			}
@@ -752,7 +747,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 		break;
 
 	default:
-		dev_err(chan2dev(chan), "Direction %s not supported\n",
+		dev_err(vchan_chan_dev(&chan->vchan), "Direction %s not supported\n",
 			dmaengine_get_direction_text(dir));
 		return -EINVAL;
 	}
@@ -761,7 +756,7 @@ static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transf
 	*ctr1 = _ctr1;
 	*ctr2 = _ctr2;
 
-	dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n",
 		__func__, sdw, sbl_max, ddw, dbl_max);
 
 	return 0;
@@ -807,7 +802,7 @@ static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan)
 
 	chan->dma_status = DMA_IN_PROGRESS;
 
-	dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: started\n", &chan->vchan);
 }
 
 static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp)
@@ -871,7 +866,7 @@ static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan,
 					struct dma_tx_state *txstate)
 {
 	struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
-	struct device *dev = chan2dev(chan);
+	struct device *dev = vchan_chan_dev(&chan->vchan);
 	struct stm32_dma3_hwdesc *hwdesc;
 	u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol;
 	bool pack_unpack;
@@ -921,7 +916,7 @@ static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan,
 	/* Get current hwdesc and cumulate residue of pending hwdesc BNDT */
 	ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue);
 	if (ret < 0) {
-		dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Can't get residue: current hwdesc not found\n");
 		return;
 	}
 	curr_lli = ret;
@@ -957,7 +952,7 @@ static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan,
 
 skip_fifol_update:
 	if (fifol) {
-		dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "%u byte(s) in the FIFO\n", fifol);
 		dma_set_in_flight_bytes(txstate, fifol);
 		/*
 		 * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from
@@ -987,7 +982,7 @@ static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan)
 		/* Suspend the channel */
 		ret = stm32_dma3_chan_suspend(chan, true);
 		if (ret)
-			dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__);
+			dev_warn(vchan_chan_dev(&chan->vchan), "%s: timeout, data might be lost\n", __func__);
 	}
 
 	/*
@@ -1034,7 +1029,7 @@ static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid)
 	}
 
 	if (csr & CSR_USEF && ccr & CCR_USEIE) {
-		dev_err(chan2dev(chan), "User setting error\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "User setting error\n");
 		chan->dma_status = DMA_ERROR;
 		/* CCR.EN automatically cleared by HW */
 		stm32_dma3_check_user_setting(chan);
@@ -1042,14 +1037,14 @@ static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid)
 	}
 
 	if (csr & CSR_ULEF && ccr & CCR_ULEIE) {
-		dev_err(chan2dev(chan), "Update link transfer error\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Update link transfer error\n");
 		chan->dma_status = DMA_ERROR;
 		/* CCR.EN automatically cleared by HW */
 		stm32_dma3_chan_reset(chan);
 	}
 
 	if (csr & CSR_DTEF && ccr & CCR_DTEIE) {
-		dev_err(chan2dev(chan), "Data transfer error\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Data transfer error\n");
 		chan->dma_status = DMA_ERROR;
 		/* CCR.EN automatically cleared by HW */
 		stm32_dma3_chan_reset(chan);
@@ -1087,13 +1082,13 @@ static int stm32_dma3_get_chan_sem(struct stm32_dma3_chan *chan)
 		goto bad_cid;
 
 	chan->semaphore_taken = true;
-	dev_dbg(chan2dev(chan), "under CID1 control (semcr=0x%08x)\n", csemcr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "under CID1 control (semcr=0x%08x)\n", csemcr);
 
 	return 0;
 
 bad_cid:
 	chan->semaphore_taken = false;
-	dev_err(chan2dev(chan), "not under CID1 control (in-use by CID%d)\n", ccid);
+	dev_err(vchan_chan_dev(&chan->vchan), "not under CID1 control (in-use by CID%d)\n", ccid);
 
 	return -EACCES;
 }
@@ -1105,7 +1100,7 @@ static void stm32_dma3_put_chan_sem(struct stm32_dma3_chan *chan)
 	if (chan->semaphore_taken) {
 		writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id));
 		chan->semaphore_taken = false;
-		dev_dbg(chan2dev(chan), "no more under CID1 control\n");
+		dev_dbg(vchan_chan_dev(&chan->vchan), "no more under CID1 control\n");
 	}
 }
 
@@ -1126,11 +1121,11 @@ static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)
 		goto err_put_sync;
 	}
 
-	chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev,
+	chan->lli_pool = dmam_pool_create(dma_chan_name(c), c->device->dev,
 					  sizeof(struct stm32_dma3_hwdesc),
 					  __alignof__(struct stm32_dma3_hwdesc), SZ_64K);
 	if (!chan->lli_pool) {
-		dev_err(chan2dev(chan), "Failed to create LLI pool\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Failed to create LLI pool\n");
 		ret = -ENOMEM;
 		goto err_put_sync;
 	}
@@ -1366,7 +1361,7 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan
 	}
 
 	if (count != sg_len && chan->tcem != CTR2_TCEM_CHANNEL)
-		dev_warn(chan2dev(chan), "Linked-list refactored, %d items instead of %d\n",
+		dev_warn(vchan_chan_dev(&chan->vchan), "Linked-list refactored, %d items instead of %d\n",
 			 count, sg_len);
 
 	/* Enable Error interrupts */
@@ -1401,12 +1396,12 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_cha
 		return NULL;
 
 	if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) {
-		dev_err(chan2dev(chan), "Invalid buffer/period length\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Invalid buffer/period length\n");
 		return NULL;
 	}
 
 	if (buf_len % period_len) {
-		dev_err(chan2dev(chan), "Buffer length not multiple of period length\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Buffer length not multiple of period length\n");
 		return NULL;
 	}
 
@@ -1428,7 +1423,7 @@ static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_cha
 		ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,
 					      src, dst, period_len);
 	} else {
-		dev_err(chan2dev(chan), "Invalid direction\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Invalid direction\n");
 		ret = -EINVAL;
 	}
 
@@ -1502,7 +1497,7 @@ static int stm32_dma3_pause(struct dma_chan *c)
 
 	chan->dma_status = DMA_PAUSED;
 
-	dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: paused\n", &chan->vchan);
 
 	return 0;
 }
@@ -1515,7 +1510,7 @@ static int stm32_dma3_resume(struct dma_chan *c)
 
 	chan->dma_status = DMA_IN_PROGRESS;
 
-	dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: resumed\n", &chan->vchan);
 
 	return 0;
 }
@@ -1540,7 +1535,7 @@ static int stm32_dma3_terminate_all(struct dma_chan *c)
 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
 	vchan_dma_desc_free_list(&chan->vchan, &head);
 
-	dev_dbg(chan2dev(chan), "vchan %p: terminated\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: terminated\n", &chan->vchan);
 
 	return 0;
 }
@@ -1593,7 +1588,7 @@ static void stm32_dma3_issue_pending(struct dma_chan *c)
 	spin_lock_irqsave(&chan->vchan.lock, flags);
 
 	if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) {
-		dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: issued\n", &chan->vchan);
 		stm32_dma3_chan_start(chan);
 	}
 
@@ -1892,10 +1887,10 @@ static int stm32_dma3_probe(struct platform_device *pdev)
 		chan->irq = ret;
 
 		ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0,
-				       dev_name(chan2dev(chan)), chan);
+				       vchan_chan_name(&chan->vchan), chan);
 		if (ret) {
 			dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n",
-				      dev_name(chan2dev(chan)));
+				      vchan_chan_name(&chan->vchan));
 			goto err_clk_disable;
 		}
 	}
diff --git a/drivers/dma/stm32/stm32-mdma.c b/drivers/dma/stm32/stm32-mdma.c
index e3bbdc9ee36edd..04639e4b554ea0 100644
--- a/drivers/dma/stm32/stm32-mdma.c
+++ b/drivers/dma/stm32/stm32-mdma.c
@@ -276,11 +276,6 @@ static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc)
 	return container_of(vdesc, struct stm32_mdma_desc, vdesc);
 }
 
-static struct device *chan2dev(struct stm32_mdma_chan *chan)
-{
-	return &chan->vchan.chan.dev->device;
-}
-
 static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev)
 {
 	return mdma_dev->ddev.dev;
@@ -334,7 +329,7 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
 	return desc;
 
 err:
-	dev_err(chan2dev(chan), "Failed to allocate descriptor\n");
+	dev_err(vchan_chan_dev(&chan->vchan), "Failed to allocate descriptor\n");
 	while (--i >= 0)
 		dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
 			      desc->node[i].hwdesc_phys);
@@ -364,7 +359,7 @@ static int stm32_mdma_get_width(struct stm32_mdma_chan *chan,
 	case DMA_SLAVE_BUSWIDTH_8_BYTES:
 		return ffs(width) - 1;
 	default:
-		dev_err(chan2dev(chan), "Dma bus width %i not supported\n",
+		dev_err(vchan_chan_dev(&chan->vchan), "Dma bus width %i not supported\n",
 			width);
 		return -EINVAL;
 	}
@@ -422,7 +417,7 @@ static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan)
 				dmadev->base + STM32_MDMA_CISR(id), cisr,
 				(cisr & STM32_MDMA_CISR_CTCIF), 10, 1000);
 		if (ret) {
-			dev_err(chan2dev(chan), "%s: timeout!\n", __func__);
+			dev_err(vchan_chan_dev(&chan->vchan), "%s: timeout!\n", __func__);
 			return -EBUSY;
 		}
 	}
@@ -444,7 +439,7 @@ static void stm32_mdma_stop(struct stm32_mdma_chan *chan)
 	/* Clear interrupt status if it is there */
 	status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
 	if (status) {
-		dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",
+		dev_dbg(vchan_chan_dev(&chan->vchan), "%s(): clearing interrupt: 0x%08x\n",
 			__func__, status);
 		stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);
 	}
@@ -513,7 +508,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
 	/* Check burst size constraints */
 	if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST ||
 	    dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) {
-		dev_err(chan2dev(chan),
+		dev_err(vchan_chan_dev(&chan->vchan),
 			"burst size * bus width higher than %d bytes\n",
 			STM32_MDMA_MAX_BURST);
 		return -EINVAL;
@@ -521,7 +516,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
 
 	if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) ||
 	    (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) {
-		dev_err(chan2dev(chan), "burst size must be a power of 2\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "burst size must be a power of 2\n");
 		return -EINVAL;
 	}
 
@@ -658,7 +653,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
 		break;
 
 	default:
-		dev_err(chan2dev(chan), "Dma direction is not supported\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Dma direction is not supported\n");
 		return -EINVAL;
 	}
 
@@ -672,16 +667,16 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
 static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan,
 				   struct stm32_mdma_desc_node *node)
 {
-	dev_dbg(chan2dev(chan), "hwdesc:  %pad\n", &node->hwdesc_phys);
-	dev_dbg(chan2dev(chan), "CTCR:    0x%08x\n", node->hwdesc->ctcr);
-	dev_dbg(chan2dev(chan), "CBNDTR:  0x%08x\n", node->hwdesc->cbndtr);
-	dev_dbg(chan2dev(chan), "CSAR:    0x%08x\n", node->hwdesc->csar);
-	dev_dbg(chan2dev(chan), "CDAR:    0x%08x\n", node->hwdesc->cdar);
-	dev_dbg(chan2dev(chan), "CBRUR:   0x%08x\n", node->hwdesc->cbrur);
-	dev_dbg(chan2dev(chan), "CLAR:    0x%08x\n", node->hwdesc->clar);
-	dev_dbg(chan2dev(chan), "CTBR:    0x%08x\n", node->hwdesc->ctbr);
-	dev_dbg(chan2dev(chan), "CMAR:    0x%08x\n", node->hwdesc->cmar);
-	dev_dbg(chan2dev(chan), "CMDR:    0x%08x\n\n", node->hwdesc->cmdr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "hwdesc:  %pad\n", &node->hwdesc_phys);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CTCR:    0x%08x\n", node->hwdesc->ctcr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CBNDTR:  0x%08x\n", node->hwdesc->cbndtr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CSAR:    0x%08x\n", node->hwdesc->csar);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CDAR:    0x%08x\n", node->hwdesc->cdar);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CBRUR:   0x%08x\n", node->hwdesc->cbrur);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CLAR:    0x%08x\n", node->hwdesc->clar);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CTBR:    0x%08x\n", node->hwdesc->ctbr);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CMAR:    0x%08x\n", node->hwdesc->cmar);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CMDR:    0x%08x\n\n", node->hwdesc->cmdr);
 }
 
 static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
@@ -739,7 +734,7 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan,
 
 	for_each_sg(sgl, sg, sg_len, i) {
 		if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) {
-			dev_err(chan2dev(chan), "Invalid block len\n");
+			dev_err(vchan_chan_dev(&chan->vchan), "Invalid block len\n");
 			return -EINVAL;
 		}
 
@@ -797,7 +792,7 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,
 	 * for allowing another request.
 	 */
 	if (chan->desc && chan->desc->cyclic) {
-		dev_err(chan2dev(chan),
+		dev_err(vchan_chan_dev(&chan->vchan),
 			"Request not allowed when dma in cyclic mode\n");
 		return NULL;
 	}
@@ -858,18 +853,18 @@ stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr,
 	 * for allowing another request.
 	 */
 	if (chan->desc && chan->desc->cyclic) {
-		dev_err(chan2dev(chan),
+		dev_err(vchan_chan_dev(&chan->vchan),
 			"Request not allowed when dma in cyclic mode\n");
 		return NULL;
 	}
 
 	if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) {
-		dev_err(chan2dev(chan), "Invalid buffer/period len\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "Invalid buffer/period len\n");
 		return NULL;
 	}
 
 	if (buf_len % period_len) {
-		dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "buf_len not multiple of period_len\n");
 		return NULL;
 	}
 
@@ -954,7 +949,7 @@ stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
 	 * to allow another request
 	 */
 	if (chan->desc && chan->desc->cyclic) {
-		dev_err(chan2dev(chan),
+		dev_err(vchan_chan_dev(&chan->vchan),
 			"Request not allowed when dma in cyclic mode\n");
 		return NULL;
 	}
@@ -1116,25 +1111,25 @@ static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan)
 {
 	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
 
-	dev_dbg(chan2dev(chan), "CCR:     0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CCR:     0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)));
-	dev_dbg(chan2dev(chan), "CTCR:    0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CTCR:    0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)));
-	dev_dbg(chan2dev(chan), "CBNDTR:  0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CBNDTR:  0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)));
-	dev_dbg(chan2dev(chan), "CSAR:    0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CSAR:    0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id)));
-	dev_dbg(chan2dev(chan), "CDAR:    0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CDAR:    0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id)));
-	dev_dbg(chan2dev(chan), "CBRUR:   0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CBRUR:   0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id)));
-	dev_dbg(chan2dev(chan), "CLAR:    0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CLAR:    0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)));
-	dev_dbg(chan2dev(chan), "CTBR:    0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CTBR:    0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)));
-	dev_dbg(chan2dev(chan), "CMAR:    0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CMAR:    0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id)));
-	dev_dbg(chan2dev(chan), "CMDR:    0x%08x\n",
+	dev_dbg(vchan_chan_dev(&chan->vchan), "CMDR:    0x%08x\n",
 		stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id)));
 }
 
@@ -1187,7 +1182,7 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)
 
 	chan->busy = true;
 
-	dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: started\n", &chan->vchan);
 }
 
 static void stm32_mdma_issue_pending(struct dma_chan *c)
@@ -1200,7 +1195,7 @@ static void stm32_mdma_issue_pending(struct dma_chan *c)
 	if (!vchan_issue_pending(&chan->vchan))
 		goto end;
 
-	dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: issued\n", &chan->vchan);
 
 	if (!chan->desc && !chan->busy)
 		stm32_mdma_start_transfer(chan);
@@ -1220,7 +1215,7 @@ static int stm32_mdma_pause(struct dma_chan *c)
 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
 
 	if (!ret)
-		dev_dbg(chan2dev(chan), "vchan %p: pause\n", &chan->vchan);
+		dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: pause\n", &chan->vchan);
 
 	return ret;
 }
@@ -1261,7 +1256,7 @@ static int stm32_mdma_resume(struct dma_chan *c)
 
 	spin_unlock_irqrestore(&chan->vchan.lock, flags);
 
-	dev_dbg(chan2dev(chan), "vchan %p: resume\n", &chan->vchan);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "vchan %p: resume\n", &chan->vchan);
 
 	return 0;
 }
@@ -1422,10 +1417,10 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
 	if (!(status & ien)) {
 		spin_unlock(&chan->vchan.lock);
 		if (chan->busy)
-			dev_warn(chan2dev(chan),
+			dev_warn(vchan_chan_dev(&chan->vchan),
 				 "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);
 		else
-			dev_dbg(chan2dev(chan),
+			dev_dbg(vchan_chan_dev(&chan->vchan),
 				"spurious it (status=0x%04x, ien=0x%04x)\n", status, ien);
 		return IRQ_NONE;
 	}
@@ -1433,7 +1428,7 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
 	reg = STM32_MDMA_CIFCR(id);
 
 	if (status & STM32_MDMA_CISR_TEIF) {
-		dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n",
+		dev_err(vchan_chan_dev(&chan->vchan), "Transfer Err: stat=0x%08x\n",
 			readl_relaxed(dmadev->base + STM32_MDMA_CESR(id)));
 		stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);
 		status &= ~STM32_MDMA_CISR_TEIF;
@@ -1468,9 +1463,9 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
 
 	if (status) {
 		stm32_mdma_set_bits(dmadev, reg, status);
-		dev_err(chan2dev(chan), "DMA error: status=0x%08x\n", status);
+		dev_err(vchan_chan_dev(&chan->vchan), "DMA error: status=0x%08x\n", status);
 		if (!(ccr & STM32_MDMA_CCR_EN))
-			dev_err(chan2dev(chan), "chan disabled by HW\n");
+			dev_err(vchan_chan_dev(&chan->vchan), "chan disabled by HW\n");
 	}
 
 	spin_unlock(&chan->vchan.lock);
@@ -1484,13 +1479,12 @@ static int stm32_mdma_alloc_chan_resources(struct dma_chan *c)
 	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
 	int ret;
 
-	chan->desc_pool = dmam_pool_create(dev_name(&c->dev->device),
-					   c->device->dev,
+	chan->desc_pool = dmam_pool_create(dma_chan_name(c), c->device->dev,
 					   sizeof(struct stm32_mdma_hwdesc),
 					  __alignof__(struct stm32_mdma_hwdesc),
 					   0);
 	if (!chan->desc_pool) {
-		dev_err(chan2dev(chan), "failed to allocate descriptor pool\n");
+		dev_err(vchan_chan_dev(&chan->vchan), "failed to allocate descriptor pool\n");
 		return -ENOMEM;
 	}
 
@@ -1511,7 +1505,7 @@ static void stm32_mdma_free_chan_resources(struct dma_chan *c)
 	struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
 	unsigned long flags;
 
-	dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);
+	dev_dbg(vchan_chan_dev(&chan->vchan), "Freeing channel %d\n", chan->id);
 
 	if (chan->busy) {
 		spin_lock_irqsave(&chan->vchan.lock, flags);
diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c
index d2321f7287d209..225e4a53502b4d 100644
--- a/drivers/dma/sun4i-dma.c
+++ b/drivers/dma/sun4i-dma.c
@@ -246,11 +246,6 @@ static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd
 	return container_of(vd, struct sun4i_dma_contract, vd);
 }
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static void set_dst_data_width_a10(u32 *p_cfg, s8 data_width)
 {
 	*p_cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(data_width);
@@ -428,7 +423,7 @@ static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
 	 * has already submitted some work, we can't do anything else
 	 */
 	if (vchan->processing) {
-		dev_dbg(chan2dev(&vchan->vc.chan),
+		dev_dbg(vchan_chan_dev(&vchan->vc),
 			"processing something to this endpoint already\n");
 		ret = -EBUSY;
 		goto release_pchan;
@@ -438,7 +433,7 @@ static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
 		/* Figure out which contract we're working with today */
 		vd = vchan_next_desc(&vchan->vc);
 		if (!vd) {
-			dev_dbg(chan2dev(&vchan->vc.chan),
+			dev_dbg(vchan_chan_dev(&vchan->vc),
 				"No pending contract found");
 			ret = 0;
 			goto release_pchan;
@@ -449,7 +444,7 @@ static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
 			/* The contract has been completed so mark it as such */
 			list_del(&contract->vd.node);
 			vchan_cookie_complete(&contract->vd);
-			dev_dbg(chan2dev(&vchan->vc.chan),
+			dev_dbg(vchan_chan_dev(&vchan->vc),
 				"Empty contract found and marked complete");
 		}
 	} while (list_empty(&contract->demands));
@@ -542,7 +537,7 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
 	promise->cfg = SUN4I_DMA_CFG_LOADING |
 		SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
 
-	dev_dbg(chan2dev(chan),
+	dev_dbg(dmaengine_chan_dev(chan),
 		"src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
 		sconfig->src_maxburst, sconfig->dst_maxburst,
 		sconfig->src_addr_width, sconfig->dst_addr_width);
@@ -769,7 +764,7 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
 	u8 ram_type, io_mode, linear_mode;
 
 	if (!is_slave_direction(dir)) {
-		dev_err(chan2dev(chan), "Invalid DMA direction\n");
+		dev_err(dmaengine_chan_dev(chan), "Invalid DMA direction\n");
 		return NULL;
 	}
 
@@ -894,7 +889,7 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		return NULL;
 
 	if (!is_slave_direction(dir)) {
-		dev_err(chan2dev(chan), "Invalid DMA direction\n");
+		dev_err(dmaengine_chan_dev(chan), "Invalid DMA direction\n");
 		return NULL;
 	}
 
diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c
index f47a326dd7ffa6..d4966b66b75374 100644
--- a/drivers/dma/sun6i-dma.c
+++ b/drivers/dma/sun6i-dma.c
@@ -213,11 +213,6 @@ struct sun6i_dma_dev {
 	u32			max_request;
 };
 
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
-
 static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d)
 {
 	return container_of(d, struct sun6i_dma_dev, slave);
@@ -354,8 +349,10 @@ static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan)
 	size_t bytes;
 	dma_addr_t pos;
 
-	pos = readl(pchan->base + DMA_CHAN_LLI_ADDR);
-	bytes = readl(pchan->base + DMA_CHAN_CUR_CNT);
+	do {
+		pos = readl(pchan->base + DMA_CHAN_LLI_ADDR);
+		bytes = readl(pchan->base + DMA_CHAN_CUR_CNT);
+	} while (pos != readl(pchan->base + DMA_CHAN_LLI_ADDR));
 
 	if (pos == LLI_LAST_ITEM)
 		return bytes;
@@ -397,7 +394,7 @@ static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan,
 				      struct sun6i_dma_lli *v_lli,
 				      dma_addr_t p_lli)
 {
-	dev_dbg(chan2dev(&vchan->vc.chan),
+	dev_dbg(vchan_chan_dev(&vchan->vc),
 		"\n\tdesc:\tp - %pad v - 0x%p\n"
 		"\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n"
 		"\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n",
@@ -406,16 +403,12 @@ static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan,
 		v_lli->len, v_lli->para, v_lli->p_lli_next);
 }
 
-static void sun6i_dma_free_desc(struct virt_dma_desc *vd)
+static void sun6i_dma_free_desc(struct sun6i_dma_dev *sdev,
+				struct sun6i_desc *txd)
 {
-	struct sun6i_desc *txd = to_sun6i_desc(&vd->tx);
-	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device);
 	struct sun6i_dma_lli *v_lli, *v_next;
 	dma_addr_t p_lli, p_next;
 
-	if (unlikely(!txd))
-		return;
-
 	p_lli = txd->p_lli;
 	v_lli = txd->v_lli;
 
@@ -432,6 +425,17 @@ static void sun6i_dma_free_desc(struct virt_dma_desc *vd)
 	kfree(txd);
 }
 
+static void sun6i_dma_free_desc_virt(struct virt_dma_desc *vd)
+{
+	struct sun6i_desc *txd = to_sun6i_desc(&vd->tx);
+	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device);
+
+	if (unlikely(!txd))
+		return;
+
+	sun6i_dma_free_desc(sdev, txd);
+}
+
 static int sun6i_dma_start_desc(struct sun6i_vchan *vchan)
 {
 	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device);
@@ -671,7 +675,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy(
 	dma_addr_t p_lli;
 	s8 burst, width;
 
-	dev_dbg(chan2dev(chan),
+	dev_dbg(dmaengine_chan_dev(chan),
 		"%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n",
 		__func__, vchan->vc.chan.chan_id, &dest, &src, len, flags);
 
@@ -732,7 +736,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
 
 	ret = set_config(sdev, sconfig, dir, &lli_cfg);
 	if (ret) {
-		dev_err(chan2dev(chan), "Invalid DMA configuration\n");
+		dev_err(dmaengine_chan_dev(chan), "Invalid DMA configuration\n");
 		return NULL;
 	}
 
@@ -756,7 +760,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
 			sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
 			sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
 
-			dev_dbg(chan2dev(chan),
+			dev_dbg(dmaengine_chan_dev(chan),
 				"%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
 				__func__, vchan->vc.chan.chan_id,
 				&sconfig->dst_addr, &sg_dma_address(sg),
@@ -770,7 +774,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
 			sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
 			sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
 
-			dev_dbg(chan2dev(chan),
+			dev_dbg(dmaengine_chan_dev(chan),
 				"%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
 				__func__, vchan->vc.chan.chan_id,
 				&sg_dma_address(sg), &sconfig->src_addr,
@@ -780,7 +784,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
 		prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
 	}
 
-	dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli);
+	dev_dbg(dmaengine_chan_dev(chan), "First: %pad\n", &txd->p_lli);
 	for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
 	     p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
 		sun6i_dma_dump_lli(vchan, v_lli, p_lli);
@@ -788,10 +792,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
 	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 
 err_lli_free:
-	for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
-	     p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
-		dma_pool_free(sdev->pool, v_lli, p_lli);
-	kfree(txd);
+	sun6i_dma_free_desc(sdev, txd);
 	return NULL;
 }
 
@@ -815,7 +816,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
 
 	ret = set_config(sdev, sconfig, dir, &lli_cfg);
 	if (ret) {
-		dev_err(chan2dev(chan), "Invalid DMA configuration\n");
+		dev_err(dmaengine_chan_dev(chan), "Invalid DMA configuration\n");
 		return NULL;
 	}
 
@@ -840,7 +841,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
 			v_lli->cfg = lli_cfg;
 			sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
 			sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
-			dev_dbg(chan2dev(chan),
+			dev_dbg(dmaengine_chan_dev(chan),
 				"%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n",
 				__func__, vchan->vc.chan.chan_id,
 				&sconfig->dst_addr, &buf_addr,
@@ -852,7 +853,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
 			v_lli->cfg = lli_cfg;
 			sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
 			sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
-			dev_dbg(chan2dev(chan),
+			dev_dbg(dmaengine_chan_dev(chan),
 				"%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n",
 				__func__, vchan->vc.chan.chan_id,
 				&buf_addr, &sconfig->src_addr,
@@ -869,10 +870,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
 	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 
 err_lli_free:
-	for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
-	     p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
-		dma_pool_free(sdev->pool, v_lli, p_lli);
-	kfree(txd);
+	sun6i_dma_free_desc(sdev, txd);
 	return NULL;
 }
 
@@ -892,7 +890,7 @@ static int sun6i_dma_pause(struct dma_chan *chan)
 	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 	struct sun6i_pchan *pchan = vchan->phy;
 
-	dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc);
+	dev_dbg(dmaengine_chan_dev(chan), "vchan %p: pause\n", &vchan->vc);
 
 	if (pchan) {
 		writel(DMA_CHAN_PAUSE_PAUSE,
@@ -913,7 +911,7 @@ static int sun6i_dma_resume(struct dma_chan *chan)
 	struct sun6i_pchan *pchan = vchan->phy;
 	unsigned long flags;
 
-	dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
+	dev_dbg(dmaengine_chan_dev(chan), "vchan %p: resume\n", &vchan->vc);
 
 	spin_lock_irqsave(&vchan->vc.lock, flags);
 
@@ -979,7 +977,6 @@ static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan,
 	struct sun6i_pchan *pchan = vchan->phy;
 	struct sun6i_dma_lli *lli;
 	struct virt_dma_desc *vd;
-	struct sun6i_desc *txd;
 	enum dma_status ret;
 	unsigned long flags;
 	size_t bytes = 0;
@@ -991,9 +988,9 @@ static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan,
 	spin_lock_irqsave(&vchan->vc.lock, flags);
 
 	vd = vchan_find_desc(&vchan->vc, cookie);
-	txd = to_sun6i_desc(&vd->tx);
 
 	if (vd) {
+		struct sun6i_desc *txd = to_sun6i_desc(&vd->tx);
 		for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next)
 			bytes += lli->len;
 	} else if (!pchan || !pchan->desc) {
@@ -1023,13 +1020,13 @@ static void sun6i_dma_issue_pending(struct dma_chan *chan)
 		if (!vchan->phy && list_empty(&vchan->node)) {
 			list_add_tail(&vchan->node, &sdev->pending);
 			tasklet_schedule(&sdev->task);
-			dev_dbg(chan2dev(chan), "vchan %p: issued\n",
+			dev_dbg(dmaengine_chan_dev(chan), "vchan %p: issued\n",
 				&vchan->vc);
 		}
 
 		spin_unlock(&sdev->lock);
 	} else {
-		dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n",
+		dev_dbg(dmaengine_chan_dev(chan), "vchan %p: nothing to issue\n",
 			&vchan->vc);
 	}
 
@@ -1428,7 +1425,7 @@ static int sun6i_dma_probe(struct platform_device *pdev)
 		struct sun6i_vchan *vchan = &sdc->vchans[i];
 
 		INIT_LIST_HEAD(&vchan->node);
-		vchan->vc.desc_free = sun6i_dma_free_desc;
+		vchan->vc.desc_free = sun6i_dma_free_desc_virt;
 		vchan_init(&vchan->vc, &sdc->slave);
 	}
 
diff --git a/drivers/dma/switchtec_dma.c b/drivers/dma/switchtec_dma.c
index c8e2169877e92c..df58ca760176b6 100644
--- a/drivers/dma/switchtec_dma.c
+++ b/drivers/dma/switchtec_dma.c
@@ -406,7 +406,7 @@ static int disable_channel(struct switchtec_dma_chan *swdma_chan)
 static void
 switchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan)
 {
-	struct device *chan_dev = &swdma_chan->dma_chan.dev->device;
+	struct device *chan_dev = dmaengine_chan_dev(&swdma_chan->dma_chan);
 	struct switchtec_dma_desc *desc;
 	struct switchtec_dma_hw_ce *ce;
 	struct dmaengine_result res;
@@ -851,7 +851,7 @@ static irqreturn_t switchtec_dma_chan_status_isr(int irq, void *dma)
 	list_for_each_entry(chan, &dma_dev->channels, device_node) {
 		swdma_chan = container_of(chan, struct switchtec_dma_chan,
 					  dma_chan);
-		chan_dev = &swdma_chan->dma_chan.dev->device;
+		chan_dev = dmaengine_chan_dev(&swdma_chan->dma_chan);
 		chan_hw = swdma_chan->mmio_chan_hw;
 
 		rcu_read_lock();
@@ -1009,19 +1009,19 @@ static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan)
 	perf_cfg = readl(&swdma_chan->mmio_chan_fw->perf_cfg);
 	rcu_read_unlock();
 
-	dev_dbg(&chan->dev->device, "Burst Size:  0x%x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "Burst Size:  0x%x\n",
 		FIELD_GET(PERF_BURST_SIZE_MASK, perf_cfg));
 
-	dev_dbg(&chan->dev->device, "Burst Scale: 0x%x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "Burst Scale: 0x%x\n",
 		FIELD_GET(PERF_BURST_SCALE_MASK, perf_cfg));
 
-	dev_dbg(&chan->dev->device, "Interval:    0x%x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "Interval:    0x%x\n",
 		FIELD_GET(PERF_INTERVAL_MASK, perf_cfg));
 
-	dev_dbg(&chan->dev->device, "Arb Weight:  0x%x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "Arb Weight:  0x%x\n",
 		FIELD_GET(PERF_ARB_WEIGHT_MASK, perf_cfg));
 
-	dev_dbg(&chan->dev->device, "MRRS:        0x%x\n",
+	dev_dbg(dmaengine_chan_dev(chan), "MRRS:        0x%x\n",
 		FIELD_GET(PERF_MRRS_MASK, perf_cfg));
 
 	return SWITCHTEC_DMA_SQ_SIZE;
diff --git a/drivers/dma/tegra186-gpc-dma.c b/drivers/dma/tegra186-gpc-dma.c
index 64cedef1050ae5..e76c58f213d84f 100644
--- a/drivers/dma/tegra186-gpc-dma.c
+++ b/drivers/dma/tegra186-gpc-dma.c
@@ -1537,7 +1537,7 @@ static int tegra_dma_probe(struct platform_device *pdev)
 	 * the channels available and registered for the DMA device are used.
 	 */
 	list_for_each_entry(chan, &tdma->dma_dev.channels, device_node) {
-		chdev = &chan->dev->device;
+		chdev = dmaengine_chan_dev(chan);
 		tdc = to_tegra_dma_chan(chan);
 
 		if (use_iommu_map) {
@@ -1554,7 +1554,7 @@ static int tegra_dma_probe(struct platform_device *pdev)
 				return dev_err_probe(chdev, -EINVAL,
 					   "Failed to get stream ID for channel %d\n", tdc->id);
 
-			chan->dev->chan_dma_dev = true;
+			chan->chan_dev->chan_dma_dev = true;
 		}
 
 		/* program stream-id for this channel */
diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c
index 640b8a218c9add..7ff8c0241f61f2 100644
--- a/drivers/dma/tegra20-apb-dma.c
+++ b/drivers/dma/tegra20-apb-dma.c
@@ -256,7 +256,7 @@ txd_to_tegra_dma_desc(struct dma_async_tx_descriptor *td)
 
 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
 {
-	return &tdc->dma_chan.dev->device;
+	return dmaengine_chan_dev(&tdc->dma_chan);
 }
 
 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx);
diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c
index 686dc140293eba..70eaf7ee57e681 100644
--- a/drivers/dma/ti/k3-udma-glue.c
+++ b/drivers/dma/ti/k3-udma-glue.c
@@ -1243,8 +1243,9 @@ void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
 		rx_chn->psil_paired = false;
 	}
 
-	for (i = 0; i < rx_chn->flow_num; i++)
-		k3_udma_glue_release_rx_flow(rx_chn, i);
+	if (rx_chn->flows)
+		for (i = 0; i < rx_chn->flow_num; i++)
+			k3_udma_glue_release_rx_flow(rx_chn, i);
 
 	if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
 		xudma_free_gp_rflow_range(rx_chn->common.udmax,
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index fb21e0df5ab7b2..78a67cb9d6e000 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -422,16 +422,16 @@ static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
 
 static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel)
 {
-	struct device *chan_dev = &chan->dev->device;
+	struct device *chan_dev = dmaengine_chan_dev(chan);
 
 	if (asel == 0) {
 		/* No special handling for the channel */
-		chan->dev->chan_dma_dev = false;
+		chan->chan_dev->chan_dma_dev = false;
 
 		dev_clear_dma_coherent(chan_dev);
 		chan_dev->dma_parms = NULL;
 	} else if (asel == 14 || asel == 15) {
-		chan->dev->chan_dma_dev = true;
+		chan->chan_dev->chan_dma_dev = true;
 
 		dev_set_dma_coherent(chan_dev);
 		dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48));
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index 4fc5119854ec12..0acd7a36a3d0d4 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -90,14 +90,9 @@ struct timb_dma {
 	struct tasklet_struct	tasklet;
 	struct timb_dma_chan	channels[];
 };
-
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
 static struct device *chan2dmadev(struct dma_chan *chan)
 {
-	return chan2dev(chan)->parent->parent;
+	return dmaengine_chan_dev(chan)->parent->parent;
 }
 
 static struct timb_dma *tdchantotd(struct timb_dma_chan *td_chan)
@@ -117,7 +112,7 @@ static void __td_enable_chan_irq(struct timb_dma_chan *td_chan)
 	/* enable interrupt for this channel */
 	ier = ioread32(td->membase + TIMBDMA_IER);
 	ier |= 1 << id;
-	dev_dbg(chan2dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id,
+	dev_dbg(dmaengine_chan_dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id,
 		ier);
 	iowrite32(ier, td->membase + TIMBDMA_IER);
 }
@@ -131,7 +126,7 @@ static bool __td_dma_done_ack(struct timb_dma_chan *td_chan)
 	u32 isr;
 	bool done = false;
 
-	dev_dbg(chan2dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td);
+	dev_dbg(dmaengine_chan_dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td);
 
 	isr = ioread32(td->membase + TIMBDMA_ISR) & (1 << id);
 	if (isr) {
@@ -146,18 +141,18 @@ static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,
 	struct scatterlist *sg, bool last)
 {
 	if (sg_dma_len(sg) > USHRT_MAX) {
-		dev_err(chan2dev(&td_chan->chan), "Too big sg element\n");
+		dev_err(dmaengine_chan_dev(&td_chan->chan), "Too big sg element\n");
 		return -EINVAL;
 	}
 
 	/* length must be word aligned */
 	if (sg_dma_len(sg) % sizeof(u32)) {
-		dev_err(chan2dev(&td_chan->chan), "Incorrect length: %d\n",
+		dev_err(dmaengine_chan_dev(&td_chan->chan), "Incorrect length: %d\n",
 			sg_dma_len(sg));
 		return -EINVAL;
 	}
 
-	dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n",
+	dev_dbg(dmaengine_chan_dev(&td_chan->chan), "desc: %p, addr: 0x%llx\n",
 		dma_desc, (unsigned long long)sg_dma_address(sg));
 
 	dma_desc[7] = (sg_dma_address(sg) >> 24) & 0xff;
@@ -180,7 +175,7 @@ static void __td_start_dma(struct timb_dma_chan *td_chan)
 	struct timb_dma_desc *td_desc;
 
 	if (td_chan->ongoing) {
-		dev_err(chan2dev(&td_chan->chan),
+		dev_err(dmaengine_chan_dev(&td_chan->chan),
 			"Transfer already ongoing\n");
 		return;
 	}
@@ -188,7 +183,7 @@ static void __td_start_dma(struct timb_dma_chan *td_chan)
 	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,
 		desc_node);
 
-	dev_dbg(chan2dev(&td_chan->chan),
+	dev_dbg(dmaengine_chan_dev(&td_chan->chan),
 		"td_chan: %p, chan: %d, membase: %p\n",
 		td_chan, td_chan->chan.chan_id, td_chan->membase);
 
@@ -230,7 +225,7 @@ static void __td_finish(struct timb_dma_chan *td_chan)
 		desc_node);
 	txd = &td_desc->txd;
 
-	dev_dbg(chan2dev(&td_chan->chan), "descriptor %u complete\n",
+	dev_dbg(dmaengine_chan_dev(&td_chan->chan), "descriptor %u complete\n",
 		txd->cookie);
 
 	/* make sure to stop the transfer */
@@ -284,7 +279,7 @@ static void __td_start_next(struct timb_dma_chan *td_chan)
 	td_desc = list_entry(td_chan->queue.next, struct timb_dma_desc,
 		desc_node);
 
-	dev_dbg(chan2dev(&td_chan->chan), "%s: started %u\n",
+	dev_dbg(dmaengine_chan_dev(&td_chan->chan), "%s: started %u\n",
 		__func__, td_desc->txd.cookie);
 
 	list_move(&td_desc->desc_node, &td_chan->active_list);
@@ -303,12 +298,12 @@ static dma_cookie_t td_tx_submit(struct dma_async_tx_descriptor *txd)
 	cookie = dma_cookie_assign(txd);
 
 	if (list_empty(&td_chan->active_list)) {
-		dev_dbg(chan2dev(txd->chan), "%s: started %u\n", __func__,
+		dev_dbg(dmaengine_chan_dev(txd->chan), "%s: started %u\n", __func__,
 			txd->cookie);
 		list_add_tail(&td_desc->desc_node, &td_chan->active_list);
 		__td_start_dma(td_chan);
 	} else {
-		dev_dbg(chan2dev(txd->chan), "tx_submit: queued %u\n",
+		dev_dbg(dmaengine_chan_dev(txd->chan), "tx_submit: queued %u\n",
 			txd->cookie);
 
 		list_add_tail(&td_desc->desc_node, &td_chan->queue);
@@ -344,7 +339,7 @@ static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)
 
 	err = dma_mapping_error(chan2dmadev(chan), td_desc->txd.phys);
 	if (err) {
-		dev_err(chan2dev(chan), "DMA mapping error: %d\n", err);
+		dev_err(dmaengine_chan_dev(chan), "DMA mapping error: %d\n", err);
 		goto err;
 	}
 
@@ -359,7 +354,7 @@ static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)
 
 static void td_free_desc(struct timb_dma_desc *td_desc)
 {
-	dev_dbg(chan2dev(td_desc->txd.chan), "Freeing desc: %p\n", td_desc);
+	dev_dbg(dmaengine_chan_dev(td_desc->txd.chan), "Freeing desc: %p\n", td_desc);
 	dma_unmap_single(chan2dmadev(td_desc->txd.chan), td_desc->txd.phys,
 		td_desc->desc_list_len, DMA_TO_DEVICE);
 
@@ -370,7 +365,7 @@ static void td_free_desc(struct timb_dma_desc *td_desc)
 static void td_desc_put(struct timb_dma_chan *td_chan,
 	struct timb_dma_desc *td_desc)
 {
-	dev_dbg(chan2dev(&td_chan->chan), "Putting desc: %p\n", td_desc);
+	dev_dbg(dmaengine_chan_dev(&td_chan->chan), "Putting desc: %p\n", td_desc);
 
 	spin_lock_bh(&td_chan->lock);
 	list_add(&td_desc->desc_node, &td_chan->free_list);
@@ -390,7 +385,7 @@ static struct timb_dma_desc *td_desc_get(struct timb_dma_chan *td_chan)
 			ret = td_desc;
 			break;
 		}
-		dev_dbg(chan2dev(&td_chan->chan), "desc %p not ACKed\n",
+		dev_dbg(dmaengine_chan_dev(&td_chan->chan), "desc %p not ACKed\n",
 			td_desc);
 	}
 	spin_unlock_bh(&td_chan->lock);
@@ -404,7 +399,7 @@ static int td_alloc_chan_resources(struct dma_chan *chan)
 		container_of(chan, struct timb_dma_chan, chan);
 	int i;
 
-	dev_dbg(chan2dev(chan), "%s: entry\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: entry\n", __func__);
 
 	BUG_ON(!list_empty(&td_chan->free_list));
 	for (i = 0; i < td_chan->descs; i++) {
@@ -413,7 +408,7 @@ static int td_alloc_chan_resources(struct dma_chan *chan)
 			if (i)
 				break;
 			else {
-				dev_err(chan2dev(chan),
+				dev_err(dmaengine_chan_dev(chan),
 					"Couldn't allocate any descriptors\n");
 				return -ENOMEM;
 			}
@@ -436,7 +431,7 @@ static void td_free_chan_resources(struct dma_chan *chan)
 	struct timb_dma_desc *td_desc, *_td_desc;
 	LIST_HEAD(list);
 
-	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__);
 
 	/* check that all descriptors are free */
 	BUG_ON(!list_empty(&td_chan->active_list));
@@ -447,7 +442,7 @@ static void td_free_chan_resources(struct dma_chan *chan)
 	spin_unlock_bh(&td_chan->lock);
 
 	list_for_each_entry_safe(td_desc, _td_desc, &list, desc_node) {
-		dev_dbg(chan2dev(chan), "%s: Freeing desc: %p\n", __func__,
+		dev_dbg(dmaengine_chan_dev(chan), "%s: Freeing desc: %p\n", __func__,
 			td_desc);
 		td_free_desc(td_desc);
 	}
@@ -458,11 +453,11 @@ static enum dma_status td_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 {
 	enum dma_status ret;
 
-	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__);
 
 	ret = dma_cookie_status(chan, cookie, txstate);
 
-	dev_dbg(chan2dev(chan), "%s: exit, ret: %d\n", 	__func__, ret);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: exit, ret: %d\n", 	__func__, ret);
 
 	return ret;
 }
@@ -472,7 +467,7 @@ static void td_issue_pending(struct dma_chan *chan)
 	struct timb_dma_chan *td_chan =
 		container_of(chan, struct timb_dma_chan, chan);
 
-	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__);
 	spin_lock_bh(&td_chan->lock);
 
 	if (!list_empty(&td_chan->active_list))
@@ -499,20 +494,20 @@ static struct dma_async_tx_descriptor *td_prep_slave_sg(struct dma_chan *chan,
 	unsigned int desc_usage = 0;
 
 	if (!sgl || !sg_len) {
-		dev_err(chan2dev(chan), "%s: No SG list\n", __func__);
+		dev_err(dmaengine_chan_dev(chan), "%s: No SG list\n", __func__);
 		return NULL;
 	}
 
 	/* even channels are for RX, odd for TX */
 	if (td_chan->direction != direction) {
-		dev_err(chan2dev(chan),
+		dev_err(dmaengine_chan_dev(chan),
 			"Requesting channel in wrong direction\n");
 		return NULL;
 	}
 
 	td_desc = td_desc_get(td_chan);
 	if (!td_desc) {
-		dev_err(chan2dev(chan), "Not enough descriptors available\n");
+		dev_err(dmaengine_chan_dev(chan), "Not enough descriptors available\n");
 		return NULL;
 	}
 
@@ -521,14 +516,14 @@ static struct dma_async_tx_descriptor *td_prep_slave_sg(struct dma_chan *chan,
 	for_each_sg(sgl, sg, sg_len, i) {
 		int err;
 		if (desc_usage > td_desc->desc_list_len) {
-			dev_err(chan2dev(chan), "No descriptor space\n");
+			dev_err(dmaengine_chan_dev(chan), "No descriptor space\n");
 			return NULL;
 		}
 
 		err = td_fill_desc(td_chan, td_desc->desc_list + desc_usage, sg,
 			i == (sg_len - 1));
 		if (err) {
-			dev_err(chan2dev(chan), "Failed to update desc: %d\n",
+			dev_err(dmaengine_chan_dev(chan), "Failed to update desc: %d\n",
 				err);
 			td_desc_put(td_chan, td_desc);
 			return NULL;
@@ -548,7 +543,7 @@ static int td_terminate_all(struct dma_chan *chan)
 		container_of(chan, struct timb_dma_chan, chan);
 	struct timb_dma_desc *td_desc, *_td_desc;
 
-	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
+	dev_dbg(dmaengine_chan_dev(chan), "%s: Entry\n", __func__);
 
 	/* first the easy part, put the queue into the free list */
 	spin_lock_bh(&td_chan->lock);
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
index 05622b68a9363f..ecfaa6a7651672 100644
--- a/drivers/dma/txx9dmac.c
+++ b/drivers/dma/txx9dmac.c
@@ -125,14 +125,9 @@ static struct txx9dmac_regs32 __iomem *__txx9dmac_regs32(
 #define dma_writel(ddev, name, val) \
 	(__is_dmac64(ddev) ? \
 	dma64_writel(ddev, name, val) : dma32_writel(ddev, name, val))
-
-static struct device *chan2dev(struct dma_chan *chan)
-{
-	return &chan->dev->device;
-}
 static struct device *chan2parent(struct dma_chan *chan)
 {
-	return chan->dev->device.parent;
+	return dmaengine_chan_dev(chan)->parent;
 }
 
 static struct txx9dmac_desc *
@@ -218,12 +213,12 @@ static struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc)
 			ret = desc;
 			break;
 		}
-		dev_dbg(chan2dev(&dc->chan), "desc %p not ACKed\n", desc);
+		dev_dbg(dmaengine_chan_dev(&dc->chan), "desc %p not ACKed\n", desc);
 		i++;
 	}
 	spin_unlock_bh(&dc->lock);
 
-	dev_vdbg(chan2dev(&dc->chan), "scanned %u descriptors on freelist\n",
+	dev_vdbg(dmaengine_chan_dev(&dc->chan), "scanned %u descriptors on freelist\n",
 		 i);
 	if (!ret) {
 		ret = txx9dmac_desc_alloc(dc, GFP_ATOMIC);
@@ -232,7 +227,7 @@ static struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc)
 			dc->descs_allocated++;
 			spin_unlock_bh(&dc->lock);
 		} else
-			dev_err(chan2dev(&dc->chan),
+			dev_err(dmaengine_chan_dev(&dc->chan),
 				"not enough descriptors available\n");
 	}
 	return ret;
@@ -267,11 +262,11 @@ static void txx9dmac_desc_put(struct txx9dmac_chan *dc,
 
 		spin_lock_bh(&dc->lock);
 		list_for_each_entry(child, &desc->tx_list, desc_node)
-			dev_vdbg(chan2dev(&dc->chan),
+			dev_vdbg(dmaengine_chan_dev(&dc->chan),
 				 "moving child desc %p to freelist\n",
 				 child);
 		list_splice_init(&desc->tx_list, &dc->free_list);
-		dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n",
+		dev_vdbg(dmaengine_chan_dev(&dc->chan), "moving desc %p to freelist\n",
 			 desc);
 		list_add(&desc->desc_node, &dc->free_list);
 		spin_unlock_bh(&dc->lock);
@@ -283,7 +278,7 @@ static void txx9dmac_desc_put(struct txx9dmac_chan *dc,
 static void txx9dmac_dump_regs(struct txx9dmac_chan *dc)
 {
 	if (is_dmac64(dc))
-		dev_err(chan2dev(&dc->chan),
+		dev_err(dmaengine_chan_dev(&dc->chan),
 			"  CHAR: %#llx SAR: %#llx DAR: %#llx CNTR: %#x"
 			" SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n",
 			(u64)channel64_read_CHAR(dc),
@@ -295,7 +290,7 @@ static void txx9dmac_dump_regs(struct txx9dmac_chan *dc)
 			channel64_readl(dc, CCR),
 			channel64_readl(dc, CSR));
 	else
-		dev_err(chan2dev(&dc->chan),
+		dev_err(dmaengine_chan_dev(&dc->chan),
 			"  CHAR: %#x SAR: %#x DAR: %#x CNTR: %#x"
 			" SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n",
 			channel32_readl(dc, CHAR),
@@ -333,11 +328,11 @@ static void txx9dmac_dostart(struct txx9dmac_chan *dc,
 	struct txx9dmac_slave *ds = dc->chan.private;
 	u32 sai, dai;
 
-	dev_vdbg(chan2dev(&dc->chan), "dostart %u %p\n",
+	dev_vdbg(dmaengine_chan_dev(&dc->chan), "dostart %u %p\n",
 		 first->txd.cookie, first);
 	/* ASSERT:  channel is idle */
 	if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) {
-		dev_err(chan2dev(&dc->chan),
+		dev_err(dmaengine_chan_dev(&dc->chan),
 			"BUG: Attempted to start non-idle channel\n");
 		txx9dmac_dump_regs(dc);
 		/* The tasklet will hopefully advance the queue... */
@@ -402,7 +397,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
 	struct dmaengine_desc_callback cb;
 	struct dma_async_tx_descriptor *txd = &desc->txd;
 
-	dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n",
+	dev_vdbg(dmaengine_chan_dev(&dc->chan), "descriptor %u %p complete\n",
 		 txd->cookie, desc);
 
 	dma_cookie_complete(txd);
@@ -469,11 +464,11 @@ static void txx9dmac_dump_desc(struct txx9dmac_chan *dc,
 {
 	if (is_dmac64(dc)) {
 #ifdef TXX9_DMA_USE_SIMPLE_CHAIN
-		dev_crit(chan2dev(&dc->chan),
+		dev_crit(dmaengine_chan_dev(&dc->chan),
 			 "  desc: ch%#llx s%#llx d%#llx c%#x\n",
 			 (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR);
 #else
-		dev_crit(chan2dev(&dc->chan),
+		dev_crit(dmaengine_chan_dev(&dc->chan),
 			 "  desc: ch%#llx s%#llx d%#llx c%#x"
 			 " si%#x di%#x cc%#x cs%#x\n",
 			 (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR,
@@ -482,11 +477,11 @@ static void txx9dmac_dump_desc(struct txx9dmac_chan *dc,
 	} else {
 		struct txx9dmac_hwdesc32 *d = (struct txx9dmac_hwdesc32 *)desc;
 #ifdef TXX9_DMA_USE_SIMPLE_CHAIN
-		dev_crit(chan2dev(&dc->chan),
+		dev_crit(dmaengine_chan_dev(&dc->chan),
 			 "  desc: ch%#x s%#x d%#x c%#x\n",
 			 d->CHAR, d->SAR, d->DAR, d->CNTR);
 #else
-		dev_crit(chan2dev(&dc->chan),
+		dev_crit(dmaengine_chan_dev(&dc->chan),
 			 "  desc: ch%#x s%#x d%#x c%#x"
 			 " si%#x di%#x cc%#x cs%#x\n",
 			 d->CHAR, d->SAR, d->DAR, d->CNTR,
@@ -506,7 +501,7 @@ static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr)
 	 * borked. Since we don't have any way to report errors, we'll
 	 * just have to scream loudly and try to carry on.
 	 */
-	dev_crit(chan2dev(&dc->chan), "Abnormal Chain Completion\n");
+	dev_crit(dmaengine_chan_dev(&dc->chan), "Abnormal Chain Completion\n");
 	txx9dmac_dump_regs(dc);
 
 	bad_desc = txx9dmac_first_active(dc);
@@ -523,7 +518,7 @@ static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr)
 	if (!list_empty(&dc->active_list))
 		txx9dmac_dostart(dc, txx9dmac_first_active(dc));
 
-	dev_crit(chan2dev(&dc->chan),
+	dev_crit(dmaengine_chan_dev(&dc->chan),
 		 "Bad descriptor submitted for DMA! (cookie: %d)\n",
 		 bad_desc->txd.cookie);
 	txx9dmac_dump_desc(dc, &bad_desc->hwdesc);
@@ -558,7 +553,7 @@ static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc)
 	if (!(csr & TXX9_DMA_CSR_CHNEN))
 		chain = 0;	/* last descriptor of this chain */
 
-	dev_vdbg(chan2dev(&dc->chan), "scan_descriptors: char=%#llx\n",
+	dev_vdbg(dmaengine_chan_dev(&dc->chan), "scan_descriptors: char=%#llx\n",
 		 (u64)chain);
 
 	list_for_each_entry_safe(desc, _desc, &dc->active_list, desc_node) {
@@ -589,7 +584,7 @@ static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc)
 		return;
 	}
 
-	dev_err(chan2dev(&dc->chan),
+	dev_err(dmaengine_chan_dev(&dc->chan),
 		"BUG: All descriptors done, but channel not idle!\n");
 
 	/* Try to continue after resetting the channel... */
@@ -609,7 +604,7 @@ static void txx9dmac_chan_tasklet(struct tasklet_struct *t)
 
 	dc = from_tasklet(dc, t, tasklet);
 	csr = channel_readl(dc, CSR);
-	dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", csr);
+	dev_vdbg(dmaengine_chan_dev(&dc->chan), "tasklet: status=%x\n", csr);
 
 	spin_lock(&dc->lock);
 	if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC |
@@ -625,7 +620,7 @@ static irqreturn_t txx9dmac_chan_interrupt(int irq, void *dev_id)
 {
 	struct txx9dmac_chan *dc = dev_id;
 
-	dev_vdbg(chan2dev(&dc->chan), "interrupt: status=%#x\n",
+	dev_vdbg(dmaengine_chan_dev(&dc->chan), "interrupt: status=%#x\n",
 			channel_readl(dc, CSR));
 
 	tasklet_schedule(&dc->tasklet);
@@ -654,7 +649,7 @@ static void txx9dmac_tasklet(struct tasklet_struct *t)
 		if ((mcr >> (24 + i)) & 0x11) {
 			dc = ddev->chan[i];
 			csr = channel_readl(dc, CSR);
-			dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n",
+			dev_vdbg(dmaengine_chan_dev(&dc->chan), "tasklet: status=%x\n",
 				 csr);
 			spin_lock(&dc->lock);
 			if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC |
@@ -696,7 +691,7 @@ static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx)
 	spin_lock_bh(&dc->lock);
 	cookie = dma_cookie_assign(tx);
 
-	dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u %p\n",
+	dev_vdbg(dmaengine_chan_dev(tx->chan), "tx_submit: queued %u %p\n",
 		 desc->txd.cookie, desc);
 
 	list_add_tail(&desc->desc_node, &dc->queue);
@@ -717,11 +712,11 @@ txx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 	size_t xfer_count;
 	size_t offset;
 
-	dev_vdbg(chan2dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n",
+	dev_vdbg(dmaengine_chan_dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n",
 		 (u64)dest, (u64)src, len, flags);
 
 	if (unlikely(!len)) {
-		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
+		dev_dbg(dmaengine_chan_dev(chan), "prep_dma_memcpy: length is zero!\n");
 		return NULL;
 	}
 
@@ -812,7 +807,7 @@ txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 	unsigned int i;
 	struct scatterlist *sg;
 
-	dev_vdbg(chan2dev(chan), "prep_dma_slave\n");
+	dev_vdbg(dmaengine_chan_dev(chan), "prep_dma_slave\n");
 
 	BUG_ON(!ds || !ds->reg_width);
 	if (ds->tx_reg)
@@ -900,7 +895,7 @@ static int txx9dmac_terminate_all(struct dma_chan *chan)
 	struct txx9dmac_desc *desc, *_desc;
 	LIST_HEAD(list);
 
-	dev_vdbg(chan2dev(chan), "terminate_all\n");
+	dev_vdbg(dmaengine_chan_dev(chan), "terminate_all\n");
 	spin_lock_bh(&dc->lock);
 
 	txx9dmac_reset_chan(dc);
@@ -988,11 +983,11 @@ static int txx9dmac_alloc_chan_resources(struct dma_chan *chan)
 	struct txx9dmac_desc *desc;
 	int i;
 
-	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
+	dev_vdbg(dmaengine_chan_dev(chan), "alloc_chan_resources\n");
 
 	/* ASSERT:  channel is idle */
 	if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) {
-		dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
+		dev_dbg(dmaengine_chan_dev(chan), "DMA channel not idle?\n");
 		return -EIO;
 	}
 
@@ -1022,7 +1017,7 @@ static int txx9dmac_alloc_chan_resources(struct dma_chan *chan)
 
 		desc = txx9dmac_desc_alloc(dc, GFP_KERNEL);
 		if (!desc) {
-			dev_info(chan2dev(chan),
+			dev_info(dmaengine_chan_dev(chan),
 				"only allocated %d descriptors\n", i);
 			spin_lock_bh(&dc->lock);
 			break;
@@ -1034,7 +1029,7 @@ static int txx9dmac_alloc_chan_resources(struct dma_chan *chan)
 	}
 	spin_unlock_bh(&dc->lock);
 
-	dev_dbg(chan2dev(chan),
+	dev_dbg(dmaengine_chan_dev(chan),
 		"alloc_chan_resources allocated %d descriptors\n", i);
 
 	return i;
@@ -1047,7 +1042,7 @@ static void txx9dmac_free_chan_resources(struct dma_chan *chan)
 	struct txx9dmac_desc *desc, *_desc;
 	LIST_HEAD(list);
 
-	dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n",
+	dev_dbg(dmaengine_chan_dev(chan), "free_chan_resources (descs allocated=%u)\n",
 			dc->descs_allocated);
 
 	/* ASSERT:  channel is idle */
@@ -1061,13 +1056,13 @@ static void txx9dmac_free_chan_resources(struct dma_chan *chan)
 	spin_unlock_bh(&dc->lock);
 
 	list_for_each_entry_safe(desc, _desc, &list, desc_node) {
-		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
+		dev_vdbg(dmaengine_chan_dev(chan), "  freeing descriptor %p\n", desc);
 		dma_unmap_single(chan2parent(chan), desc->txd.phys,
 				 ddev->descsize, DMA_TO_DEVICE);
 		kfree(desc);
 	}
 
-	dev_vdbg(chan2dev(chan), "free_chan_resources done\n");
+	dev_vdbg(dmaengine_chan_dev(chan), "free_chan_resources done\n");
 }
 
 /*----------------------------------------------------------------------*/
@@ -1167,26 +1162,20 @@ static void txx9dmac_chan_remove(struct platform_device *pdev)
 static int __init txx9dmac_probe(struct platform_device *pdev)
 {
 	struct txx9dmac_platform_data *pdata = dev_get_platdata(&pdev->dev);
-	struct resource *io;
 	struct txx9dmac_dev *ddev;
+	void __iomem *regs;
 	u32 mcr;
 	int err;
 
-	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!io)
-		return -EINVAL;
+	regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
 
 	ddev = devm_kzalloc(&pdev->dev, sizeof(*ddev), GFP_KERNEL);
 	if (!ddev)
 		return -ENOMEM;
 
-	if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io),
-				     dev_name(&pdev->dev)))
-		return -EBUSY;
-
-	ddev->regs = devm_ioremap(&pdev->dev, io->start, resource_size(io));
-	if (!ddev->regs)
-		return -ENOMEM;
+	ddev->regs = regs;
 	ddev->have_64bit_regs = pdata->have_64bit_regs;
 	if (__is_dmac64(ddev))
 		ddev->descsize = sizeof(struct txx9dmac_hwdesc);
diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h
index 59d9eabc8b6744..abf2ae737c30cb 100644
--- a/drivers/dma/virt-dma.h
+++ b/drivers/dma/virt-dma.h
@@ -41,6 +41,11 @@ static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan)
 	return container_of(chan, struct virt_dma_chan, chan);
 }
 
+static inline struct device *vchan_chan_dev(struct virt_dma_chan *vc)
+{
+	return dmaengine_chan_dev(&vc->chan);
+}
+
 void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head);
 void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev);
 struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *, dma_cookie_t);
@@ -209,6 +214,17 @@ static inline void vchan_free_chan_resources(struct virt_dma_chan *vc)
 	vchan_dma_desc_free_list(vc, &head);
 }
 
+/**
+ * vchan_chan_name - Return vchan DMA channel device name
+ * @vc: virtual channel
+ *
+ * Return: The name of the DMA channel device
+ */
+static inline const char *vchan_chan_name(struct virt_dma_chan *vc)
+{
+	return dma_chan_name(&vc->chan);
+}
+
 /**
  * vchan_synchronize() - synchronize callback execution to the current context
  * @vc: virtual channel to synchronize
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index bef2b031dba194..0336941d409586 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -223,6 +223,7 @@
 #define XILINX_MCDMA_IRQ_ERR_MASK		BIT(7)
 #define XILINX_MCDMA_BD_EOP			BIT(30)
 #define XILINX_MCDMA_BD_SOP			BIT(31)
+#define XILINX_MCDMA_BD_HW_SIZE			64
 
 /**
  * struct xilinx_vdma_desc_hw - Hardware Descriptor
@@ -277,8 +278,10 @@ struct xilinx_axidma_desc_hw {
  * @buf_addr_msb: MSB of Buffer address @0x0C
  * @rsvd: Reserved field @0x10
  * @control: Control Information field @0x14
- * @status: Status field @0x18
- * @sideband_status: Status of sideband signals @0x1C
+ * @mm2s_ctrl_sideband: Sideband control info for mm2s @0x18
+ * @s2mm_status: Status field for s2mm @0x18
+ * @mm2s_status: Status field for mm2s @0x1C
+ * @s2mm_sideband_status: Sideband status for s2mm @0x1C
  * @app: APP Fields @0x20 - 0x30
  */
 struct xilinx_aximcdma_desc_hw {
@@ -288,10 +291,17 @@ struct xilinx_aximcdma_desc_hw {
 	u32 buf_addr_msb;
 	u32 rsvd;
 	u32 control;
-	u32 status;
-	u32 sideband_status;
+	union {
+		u32 mm2s_ctrl_sideband;
+		u32 s2mm_status;
+	};
+	union {
+		u32 mm2s_status;
+		u32 s2mm_sideband_status;
+	};
 	u32 app[XILINX_DMA_NUM_APP_WORDS];
 } __aligned(64);
+static_assert(sizeof(struct xilinx_aximcdma_desc_hw) == XILINX_MCDMA_BD_HW_SIZE);
 
 /**
  * struct xilinx_cdma_desc_hw - Hardware Descriptor
@@ -641,17 +651,51 @@ static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan,
  * @tx: async transaction descriptor
  * @payload_len: metadata payload length
  * @max_len: metadata max length
- * Return: The app field pointer.
+ *
+ * The metadata lives in the SOP descriptor for TX and the EOF descriptor for RX.
+ * Field order depends on dmatype and direction:
+ *
+ *   AXI DMA:         [0..] app
+ *   AXI MCDMA (TX):  [0] ctrl_sideband, [1] status, [2..] app
+ *   AXI MCDMA (RX):  [0] status, [1] sideband, [2..] app
+ *
+ * Return: Pointer to the first metadata word.
  */
 static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx,
 					 size_t *payload_len, size_t *max_len)
 {
 	struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx);
+	struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan);
+
+	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
+		struct xilinx_aximcdma_tx_segment *seg;
+
+		if (chan->direction == DMA_DEV_TO_MEM) {
+			seg = list_last_entry(&desc->segments,
+					      struct xilinx_aximcdma_tx_segment, node);
+			*max_len = *payload_len = sizeof(seg->hw.s2mm_status) +
+						  sizeof(seg->hw.s2mm_sideband_status) +
+						  sizeof(seg->hw.app);
+			return &seg->hw.s2mm_status;
+		}
+		seg = list_first_entry(&desc->segments,
+				       struct xilinx_aximcdma_tx_segment, node);
+		*max_len = *payload_len = sizeof(seg->hw.mm2s_ctrl_sideband) +
+					  sizeof(seg->hw.mm2s_status) +
+					  sizeof(seg->hw.app);
+		return &seg->hw.mm2s_ctrl_sideband;
+	}
+
 	struct xilinx_axidma_tx_segment *seg;
 
-	*max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS;
-	seg = list_first_entry(&desc->segments,
-			       struct xilinx_axidma_tx_segment, node);
+	if (chan->direction == DMA_DEV_TO_MEM)
+		seg = list_last_entry(&desc->segments,
+				      struct xilinx_axidma_tx_segment, node);
+	else
+		seg = list_first_entry(&desc->segments,
+				       struct xilinx_axidma_tx_segment, node);
+
+	*max_len = *payload_len = sizeof(seg->hw.app);
 	return seg->hw.app;
 }
 
@@ -756,15 +800,25 @@ xilinx_aximcdma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 	return segment;
 }
 
-static void xilinx_dma_clean_hw_desc(struct xilinx_axidma_desc_hw *hw)
+static void xilinx_dma_clean_hw_desc(struct xilinx_dma_chan *chan,
+				     struct xilinx_axidma_tx_segment *segment)
 {
-	u32 next_desc = hw->next_desc;
-	u32 next_desc_msb = hw->next_desc_msb;
+	dma_addr_t next;
+	u32 i;
 
-	memset(hw, 0, sizeof(struct xilinx_axidma_desc_hw));
+	/*
+	 * Restore the buffer descriptor's next descriptor pointer to the value
+	 * set up in xilinx_dma_alloc_chan_resources(). Otherwise using the DMA
+	 * in cyclic mode leaves the next descriptor pointer altered and
+	 * prevents subsequent non-cyclic transfers.
+	 */
+	i = segment - chan->seg_v;
+	next = chan->seg_p +
+	       sizeof(*chan->seg_v) * ((i + 1) % XILINX_DMA_NUM_DESCS);
 
-	hw->next_desc = next_desc;
-	hw->next_desc_msb = next_desc_msb;
+	memset(&segment->hw, 0, sizeof(segment->hw));
+	segment->hw.next_desc = lower_32_bits(next);
+	segment->hw.next_desc_msb = upper_32_bits(next);
 }
 
 static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw)
@@ -786,7 +840,7 @@ static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw)
 static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan,
 				struct xilinx_axidma_tx_segment *segment)
 {
-	xilinx_dma_clean_hw_desc(&segment->hw);
+	xilinx_dma_clean_hw_desc(chan, segment);
 
 	list_add_tail(&segment->node, &chan->free_seg_list);
 }
@@ -920,9 +974,9 @@ static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan)
 
 	spin_lock_irqsave(&chan->lock, flags);
 
-	xilinx_dma_free_desc_list(chan, &chan->pending_list);
 	xilinx_dma_free_desc_list(chan, &chan->done_list);
 	xilinx_dma_free_desc_list(chan, &chan->active_list);
+	xilinx_dma_free_desc_list(chan, &chan->pending_list);
 
 	spin_unlock_irqrestore(&chan->lock, flags);
 }
@@ -1015,9 +1069,11 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan,
 					   struct xilinx_aximcdma_tx_segment,
 					   node);
 			aximcdma_hw = &aximcdma_seg->hw;
-			residue +=
-				(aximcdma_hw->control & chan->xdev->max_buffer_len) -
-				(aximcdma_hw->status & chan->xdev->max_buffer_len);
+			residue += aximcdma_hw->control & chan->xdev->max_buffer_len;
+			if (chan->direction == DMA_DEV_TO_MEM)
+				residue -= aximcdma_hw->s2mm_status & chan->xdev->max_buffer_len;
+			else
+				residue -= aximcdma_hw->mm2s_status & chan->xdev->max_buffer_len;
 		}
 	}
 
@@ -1772,6 +1828,17 @@ static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan)
 					      struct xilinx_axidma_tx_segment, node);
 			if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg)
 				break;
+		} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
+			struct xilinx_aximcdma_tx_segment *seg;
+			u32 status;
+
+			seg = list_last_entry(&desc->segments,
+					      struct xilinx_aximcdma_tx_segment,
+					      node);
+			status = (chan->direction == DMA_DEV_TO_MEM) ?
+				seg->hw.s2mm_status : seg->hw.mm2s_status;
+			if (!(status & XILINX_DMA_BD_COMP_MASK))
+				break;
 		}
 		if (chan->has_sg && chan->xdev->dma_config->dmatype !=
 		    XDMA_TYPE_VDMA)
@@ -2616,6 +2683,9 @@ xilinx_mcdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 		segment->hw.control |= XILINX_MCDMA_BD_EOP;
 	}
 
+	if (chan->xdev->has_axistream_connected)
+		desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops;
+
 	return &desc->async_tx;
 
 error:
@@ -3264,7 +3334,8 @@ static int xilinx_dma_probe(struct platform_device *pdev)
 
 	dma_set_max_seg_size(xdev->dev, xdev->max_buffer_len);
 
-	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
+	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA ||
+	    xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
 		xdev->has_axistream_connected =
 			of_property_read_bool(node, "xlnx,axistream-connected");
 	}
diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c
index 2b03584102acf4..3b7dfa53f9725b 100644
--- a/drivers/dma/xilinx/zynqmp_dma.c
+++ b/drivers/dma/xilinx/zynqmp_dma.c
@@ -12,6 +12,7 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/of_dma.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
@@ -112,8 +113,7 @@
 #define ZYNQMP_DMA_INT_DONE	(ZYNQMP_DMA_DONE | ZYNQMP_DMA_DST_DSCR_DONE)
 #define ZYNQMP_DMA_INT_EN_DEFAULT_MASK	(ZYNQMP_DMA_INT_DONE | \
 					ZYNQMP_DMA_INT_ERR | \
-					ZYNQMP_DMA_INT_OVRFL | \
-					ZYNQMP_DMA_DST_DSCR_DONE)
+					ZYNQMP_DMA_INT_OVRFL)
 
 /* Max number of descriptors per channel */
 #define ZYNQMP_DMA_NUM_DESCS	32
@@ -128,8 +128,6 @@
 /* Reset values for data attributes */
 #define ZYNQMP_DMA_AXCACHE_VAL		0xF
 
-#define ZYNQMP_DMA_SRC_ISSUE_RST_VAL	0x1F
-
 #define ZYNQMP_DMA_IDS_DEFAULT_MASK	0xFFF
 
 /* Bus width in bits */
@@ -205,10 +203,10 @@ struct zynqmp_dma_desc_sw {
  * @desc_pool_p: Physical allocated descriptor base
  * @desc_free_cnt: Descriptor available count
  * @dev: The dma device
- * @irq: Channel IRQ
+ * @irq: Linux IRQ number, or -1 when not registered
  * @is_dmacoherent: Tells whether dma operations are coherent or not
  * @tasklet: Cleanup work after irq
- * @idle : Channel status;
+ * @idle: Channel status
  * @desc_size: Size of the low level descriptor
  * @err: Channel has errors
  * @bus_width: Bus width
@@ -435,7 +433,7 @@ zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan)
 }
 
 /**
- * zynqmp_dma_free_descriptor - Issue pending transactions
+ * zynqmp_dma_free_descriptor - Return a descriptor to the free pool
  * @chan: ZynqMP DMA channel pointer
  * @sdesc: Transaction descriptor pointer
  */
@@ -483,8 +481,10 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan)
 		return ret;
 
 	chan->sw_desc_pool = kzalloc_objs(*desc, ZYNQMP_DMA_NUM_DESCS);
-	if (!chan->sw_desc_pool)
-		return -ENOMEM;
+	if (!chan->sw_desc_pool) {
+		ret = -ENOMEM;
+		goto err_pm;
+	}
 
 	chan->idle = true;
 	chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS;
@@ -502,8 +502,10 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan)
 					       (2 * ZYNQMP_DMA_DESC_SIZE(chan) *
 					       ZYNQMP_DMA_NUM_DESCS),
 					       &chan->desc_pool_p, GFP_KERNEL);
-	if (!chan->desc_pool_v)
-		return -ENOMEM;
+	if (!chan->desc_pool_v) {
+		ret = -ENOMEM;
+		goto err_free_sw_desc_pool;
+	}
 
 	for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) {
 		desc = chan->sw_desc_pool + i;
@@ -516,6 +518,13 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan)
 	}
 
 	return ZYNQMP_DMA_NUM_DESCS;
+
+err_free_sw_desc_pool:
+	kfree(chan->sw_desc_pool);
+	chan->sw_desc_pool = NULL;
+err_pm:
+	pm_runtime_put_autosuspend(chan->dev);
+	return ret;
 }
 
 /**
@@ -606,7 +615,6 @@ static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan)
 	zynqmp_dma_start(chan);
 }
 
-
 /**
  * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors
  * @chan: ZynqMP DMA channel
@@ -826,7 +834,8 @@ static void zynqmp_dma_synchronize(struct dma_chan *dchan)
  * @len: Transfer length
  * @flags: transfer ack flags
  *
- * Return: Async transaction descriptor on success and NULL on failure
+ * Return: Async transaction descriptor on success and NULL on failure or
+ *	   zero length transfer
  */
 static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy(
 				struct dma_chan *dchan, dma_addr_t dma_dst,
@@ -841,6 +850,9 @@ static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy(
 
 	chan = to_chan(dchan);
 
+	if (!len)
+		return NULL;
+
 	desc_cnt = DIV_ROUND_UP(len, ZYNQMP_DMA_MAX_TRANS_LEN);
 
 	spin_lock_irqsave(&chan->lock, irqflags);
@@ -885,10 +897,11 @@ static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan)
 	if (!chan)
 		return;
 
-	if (chan->irq)
+	if (chan->irq >= 0)
 		devm_free_irq(chan->zdev->dev, chan->irq, chan);
 	tasklet_kill(&chan->tasklet);
-	list_del(&chan->common.device_node);
+	if (!list_empty(&chan->common.device_node))
+		list_del(&chan->common.device_node);
 }
 
 /**
@@ -896,7 +909,7 @@ static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan)
  * @zdev: Driver specific device structure
  * @pdev: Pointer to the platform_device structure
  *
- * Return: '0' on success and failure value on error
+ * Return: 0 on success and negative error code on failure
  */
 static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
 			   struct platform_device *pdev)
@@ -904,13 +917,14 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
 	struct zynqmp_dma_chan *chan;
 	struct device_node *node = pdev->dev.of_node;
 	const struct zynqmp_dma_config *match_data;
-	int err;
+	int err, ret;
 
 	chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL);
 	if (!chan)
 		return -ENOMEM;
 	chan->dev = zdev->dev;
 	chan->zdev = zdev;
+	chan->irq = -1;
 
 	chan->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(chan->regs))
@@ -935,7 +949,7 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
 	if (match_data)
 		chan->irq_offset = match_data->offset;
 
-	chan->is_dmacoherent =  of_property_read_bool(node, "dma-coherent");
+	chan->is_dmacoherent = of_dma_is_coherent(node);
 	zdev->chan = chan;
 	tasklet_setup(&chan->tasklet, zynqmp_dma_do_tasklet);
 	spin_lock_init(&chan->lock);
@@ -943,22 +957,27 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
 	INIT_LIST_HEAD(&chan->pending_list);
 	INIT_LIST_HEAD(&chan->done_list);
 	INIT_LIST_HEAD(&chan->free_list);
+	INIT_LIST_HEAD(&chan->common.device_node);
 
 	dma_cookie_init(&chan->common);
 	chan->common.device = &zdev->common;
-	list_add_tail(&chan->common.device_node, &zdev->common.channels);
 
 	zynqmp_dma_init(chan);
-	chan->irq = platform_get_irq(pdev, 0);
-	if (chan->irq < 0)
-		return -ENXIO;
-	err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0,
+	ret = platform_get_irq(pdev, 0);
+	if (ret < 0)
+		return ret;
+
+	err = devm_request_irq(&pdev->dev, ret, zynqmp_dma_irq_handler, 0,
 			       "zynqmp-dma", chan);
 	if (err)
 		return err;
 
+	chan->irq = ret;
+
 	chan->desc_size = sizeof(struct zynqmp_dma_desc_ll);
 	chan->idle = true;
+	list_add_tail(&chan->common.device_node, &zdev->common.channels);
+
 	return 0;
 }
 
@@ -1025,11 +1044,11 @@ static int __maybe_unused zynqmp_dma_runtime_suspend(struct device *dev)
 }
 
 /**
- * zynqmp_dma_runtime_resume - Runtime suspend method for the driver
+ * zynqmp_dma_runtime_resume - Runtime resume method for the driver
  * @dev:	Address of the device structure
  *
- * Put the driver into low power mode.
- * Return: 0 always
+ * Enable device clocks.
+ * Return: 0 on success and failure value on error
  */
 static int __maybe_unused zynqmp_dma_runtime_resume(struct device *dev)
 {
@@ -1123,7 +1142,7 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
 	ret = zynqmp_dma_chan_probe(zdev, pdev);
 	if (ret) {
 		dev_err_probe(&pdev->dev, ret, "Probing channel failed\n");
-		goto err_disable_pm;
+		goto free_chan_resources;
 	}
 
 	p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);
@@ -1149,7 +1168,6 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
 
 free_chan_resources:
 	zynqmp_dma_chan_remove(zdev->chan);
-err_disable_pm:
 	if (!pm_runtime_enabled(zdev->dev))
 		zynqmp_dma_runtime_suspend(zdev->dev);
 	pm_runtime_disable(zdev->dev);
diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
index 45365214fbefa2..e5380d95f238b3 100644
--- a/drivers/dpll/dpll_netlink.c
+++ b/drivers/dpll/dpll_netlink.c
@@ -608,7 +608,7 @@ dpll_msg_add_pin_ref_sync(struct sk_buff *msg, struct dpll_pin *pin,
 	return -EMSGSIZE;
 }
 
-static bool dpll_pin_is_freq_supported(struct dpll_pin *pin, u32 freq)
+static bool dpll_pin_is_freq_supported(struct dpll_pin *pin, u64 freq)
 {
 	int fs;
 
@@ -1210,8 +1210,7 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin,
 	struct dpll_device *dpll;
 	int ret;
 
-	ref_sync_pin = xa_find(&pin->ref_sync_pins, &ref_sync_pin_idx,
-			       ULONG_MAX, XA_PRESENT);
+	ref_sync_pin = xa_load(&pin->ref_sync_pins, ref_sync_pin_idx);
 	if (!ref_sync_pin) {
 		NL_SET_ERR_MSG(extack, "reference sync pin not found");
 		return -EINVAL;
diff --git a/drivers/dpll/dpll_nl.c b/drivers/dpll/dpll_nl.c
index b1ba490e72b055..fa09b524a4a0fa 100644
--- a/drivers/dpll/dpll_nl.c
+++ b/drivers/dpll/dpll_nl.c
@@ -65,12 +65,7 @@ static const struct nla_policy dpll_pin_id_get_nl_policy[DPLL_A_PIN_TYPE + 1] =
 };
 
 /* DPLL_CMD_PIN_GET - do */
-static const struct nla_policy dpll_pin_get_do_nl_policy[DPLL_A_PIN_ID + 1] = {
-	[DPLL_A_PIN_ID] = { .type = NLA_U32, },
-};
-
-/* DPLL_CMD_PIN_GET - dump */
-static const struct nla_policy dpll_pin_get_dump_nl_policy[DPLL_A_PIN_ID + 1] = {
+static const struct nla_policy dpll_pin_get_nl_policy[DPLL_A_PIN_ID + 1] = {
 	[DPLL_A_PIN_ID] = { .type = NLA_U32, },
 };
 
@@ -78,9 +73,6 @@ static const struct nla_policy dpll_pin_get_dump_nl_policy[DPLL_A_PIN_ID + 1] =
 static const struct nla_policy dpll_pin_set_nl_policy[DPLL_A_PIN_REFERENCE_SYNC + 1] = {
 	[DPLL_A_PIN_ID] = { .type = NLA_U32, },
 	[DPLL_A_PIN_FREQUENCY] = { .type = NLA_U64, },
-	[DPLL_A_PIN_DIRECTION] = NLA_POLICY_RANGE(NLA_U32, 1, 2),
-	[DPLL_A_PIN_PRIO] = { .type = NLA_U32, },
-	[DPLL_A_PIN_STATE] = NLA_POLICY_RANGE(NLA_U32, 1, 3),
 	[DPLL_A_PIN_PARENT_DEVICE] = NLA_POLICY_NESTED(dpll_pin_parent_device_nl_policy),
 	[DPLL_A_PIN_PARENT_PIN] = NLA_POLICY_NESTED(dpll_pin_parent_pin_nl_policy),
 	[DPLL_A_PIN_PHASE_ADJUST] = { .type = NLA_S32, },
@@ -136,16 +128,14 @@ static const struct genl_split_ops dpll_nl_ops[] = {
 		.pre_doit	= dpll_pin_pre_doit,
 		.doit		= dpll_nl_pin_get_doit,
 		.post_doit	= dpll_pin_post_doit,
-		.policy		= dpll_pin_get_do_nl_policy,
+		.policy		= dpll_pin_get_nl_policy,
 		.maxattr	= DPLL_A_PIN_ID,
 		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
 	},
 	{
-		.cmd		= DPLL_CMD_PIN_GET,
-		.dumpit		= dpll_nl_pin_get_dumpit,
-		.policy		= dpll_pin_get_dump_nl_policy,
-		.maxattr	= DPLL_A_PIN_ID,
-		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
+		.cmd	= DPLL_CMD_PIN_GET,
+		.dumpit	= dpll_nl_pin_get_dumpit,
+		.flags	= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
 	},
 	{
 		.cmd		= DPLL_CMD_PIN_SET,
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index a44b85c440cade..4c70729b0d3af3 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -280,6 +280,19 @@ config EDAC_IMH
 	  first used on the Diamond Rapids servers but may appear on
 	  others in the future.
 
+config EDAC_INTEL_BFF_RESET
+	tristate "Intel bitfix filter reset driver"
+	depends on X86_64 && X86_MCE_INTEL
+	help
+	  Support for Intel systems that implement bitfix filters to
+	  suppress repeated logging and signaling of the same corrected
+	  error. Those filters can become clogged with transient errors
+	  over time. Clearing the filter may make space for additional
+	  hard errors.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called intel_bff.
+
 config EDAC_PND2
 	tristate "Intel Pondicherry2"
 	depends on PCI && X86_64 && X86_MCE_INTEL
@@ -581,4 +594,13 @@ config EDAC_VERSALNET
 	  and other system errors from various IP subsystems like RPU, NOCs,
 	  HNICX, PL on the AMD Versal NET DDR memory controller.
 
+config EDAC_DUMMY
+	tristate "Dummy EDAC driver"
+	default n
+	help
+	  A dummy EDAC driver is useful for testing purposes in a VM when one
+	  doesn't have all the hardware needed to test aspects of the EDAC
+	  subsystem code.
+
+	  You definitely wanna say N here.
 endif # EDAC
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index a37534300ab921..4020ea60738a25 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -35,7 +35,7 @@ obj-$(CONFIG_EDAC_I7300)		+= i7300_edac.o
 obj-$(CONFIG_EDAC_I7CORE)		+= i7core_edac.o
 obj-$(CONFIG_EDAC_SBRIDGE)		+= sb_edac.o
 obj-$(CONFIG_EDAC_PND2)			+= pnd2_edac.o
-obj-$(CONFIG_EDAC_IGEN6)			+= igen6_edac.o
+obj-$(CONFIG_EDAC_IGEN6)		+= igen6_edac.o
 obj-$(CONFIG_EDAC_E7XXX)		+= e7xxx_edac.o
 obj-$(CONFIG_EDAC_E752X)		+= e752x_edac.o
 obj-$(CONFIG_EDAC_I82875P)		+= i82875p_edac.o
@@ -66,6 +66,8 @@ obj-$(CONFIG_EDAC_I10NM)		+= i10nm_edac.o skx_edac_common.o
 imh_edac-y				:= imh_base.o
 obj-$(CONFIG_EDAC_IMH)			+= imh_edac.o skx_edac_common.o
 
+obj-$(CONFIG_EDAC_INTEL_BFF_RESET)	+= intel-bff.o
+
 obj-$(CONFIG_EDAC_HIGHBANK_MC)		+= highbank_mc_edac.o
 obj-$(CONFIG_EDAC_HIGHBANK_L2)		+= highbank_l2_edac.o
 
@@ -91,3 +93,4 @@ obj-$(CONFIG_EDAC_VERSAL)		+= versal_edac.o
 obj-$(CONFIG_EDAC_LOONGSON)		+= loongson_edac.o
 obj-$(CONFIG_EDAC_VERSALNET)		+= versalnet_edac.o
 obj-$(CONFIG_EDAC_CORTEX_A72)		+= a72_edac.o
+obj-$(CONFIG_EDAC_DUMMY)		+= dummy_edac.o
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index 68846f583eeefe..bb95dab847b3c7 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -453,15 +453,6 @@ static int altr_sdram_probe(struct platform_device *pdev)
 	return res;
 }
 
-static void altr_sdram_remove(struct platform_device *pdev)
-{
-	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
-
-	edac_mc_del_mc(&pdev->dev);
-	edac_mc_free(mci);
-	platform_set_drvdata(pdev, NULL);
-}
-
 /*
  * If you want to suspend, need to disable EDAC by removing it
  * from the device tree or defconfig.
@@ -481,13 +472,13 @@ static const struct dev_pm_ops altr_sdram_pm_ops = {
 
 static struct platform_driver altr_sdram_edac_driver = {
 	.probe = altr_sdram_probe,
-	.remove = altr_sdram_remove,
 	.driver = {
 		.name = "altr_sdram_edac",
 #ifdef CONFIG_PM
 		.pm = &altr_sdram_pm_ops,
 #endif
 		.of_match_table = altr_sdram_ctrl_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
 
@@ -517,6 +508,7 @@ static struct platform_driver altr_edac_driver = {
 	.driver = {
 		.name = "socfpga_ecc_manager",
 		.of_match_table = altr_edac_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
 module_platform_driver(altr_edac_driver);
@@ -803,22 +795,12 @@ static int altr_edac_device_probe(struct platform_device *pdev)
 	return res;
 }
 
-static void altr_edac_device_remove(struct platform_device *pdev)
-{
-	struct edac_device_ctl_info *dci = platform_get_drvdata(pdev);
-	struct altr_edac_device_dev *drvdata = dci->pvt_info;
-
-	debugfs_remove_recursive(drvdata->debugfs_dir);
-	edac_device_del_device(&pdev->dev);
-	edac_device_free_ctl_info(dci);
-}
-
 static struct platform_driver altr_edac_device_driver = {
 	.probe =  altr_edac_device_probe,
-	.remove = altr_edac_device_remove,
 	.driver = {
 		.name = "altr_edac_device",
 		.of_match_table = altr_edac_device_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
 module_platform_driver(altr_edac_device_driver);
@@ -944,7 +926,7 @@ static int __maybe_unused altr_init_memory_port(void __iomem *ioaddr, int port)
 	return ret;
 }
 
-static __init int __maybe_unused
+static int __maybe_unused
 altr_init_a10_ecc_block(struct device_node *np, u32 irq_mask,
 			u32 ecc_ctrl_en_mask, bool dual_port)
 {
@@ -1019,7 +1001,7 @@ altr_init_a10_ecc_block(struct device_node *np, u32 irq_mask,
 
 static int validate_parent_available(struct device_node *np);
 static const struct of_device_id altr_edac_a10_device_of_match[];
-static int __init __maybe_unused altr_init_a10_ecc_device_type(char *compat)
+static int __maybe_unused altr_init_a10_ecc_device_type(char *compat)
 {
 	int irq;
 	struct device_node *child, *np;
@@ -1348,7 +1330,7 @@ static const struct edac_device_prv_data a10_l2ecc_data = {
 
 #ifdef CONFIG_EDAC_ALTERA_ETHERNET
 
-static int __init socfpga_init_ethernet_ecc(struct altr_edac_device_dev *dev)
+static int socfpga_init_ethernet_ecc(struct altr_edac_device_dev *dev)
 {
 	int ret;
 
@@ -1378,7 +1360,7 @@ static const struct edac_device_prv_data a10_enetecc_data = {
 
 #ifdef CONFIG_EDAC_ALTERA_NAND
 
-static int __init socfpga_init_nand_ecc(struct altr_edac_device_dev *device)
+static int socfpga_init_nand_ecc(struct altr_edac_device_dev *device)
 {
 	int ret;
 
@@ -1408,7 +1390,7 @@ static const struct edac_device_prv_data a10_nandecc_data = {
 
 #ifdef CONFIG_EDAC_ALTERA_DMA
 
-static int __init socfpga_init_dma_ecc(struct altr_edac_device_dev *device)
+static int socfpga_init_dma_ecc(struct altr_edac_device_dev *device)
 {
 	int ret;
 
@@ -1438,7 +1420,7 @@ static const struct edac_device_prv_data a10_dmaecc_data = {
 
 #ifdef CONFIG_EDAC_ALTERA_USB
 
-static int __init socfpga_init_usb_ecc(struct altr_edac_device_dev *device)
+static int socfpga_init_usb_ecc(struct altr_edac_device_dev *device)
 {
 	int ret;
 
@@ -1468,7 +1450,7 @@ static const struct edac_device_prv_data a10_usbecc_data = {
 
 #ifdef CONFIG_EDAC_ALTERA_QSPI
 
-static int __init socfpga_init_qspi_ecc(struct altr_edac_device_dev *device)
+static int socfpga_init_qspi_ecc(struct altr_edac_device_dev *device)
 {
 	int ret;
 
@@ -1524,6 +1506,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 	dci = edac_device_alloc_ctl_info(sizeof(*altdev), ecc_name, 1,
 					 ecc_name, 1, 0, edac_idx);
 	if (!dci) {
+		of_node_put(np);
 		edac_printk(KERN_ERR, EDAC_DEVICE,
 			    "%s: Unable to allocate PortB EDAC device\n",
 			    ecc_name);
@@ -1534,8 +1517,11 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 	altdev = dci->pvt_info;
 	*altdev = *device;
 
-	if (!devres_open_group(device->edac->dev, altr_portb_setup, GFP_KERNEL))
+	if (!devres_open_group(device->edac->dev, altr_portb_setup, GFP_KERNEL)) {
+		edac_device_free_ctl_info(dci);
+		of_node_put(np);
 		return -ENOMEM;
+	}
 
 	/* Update PortB specific values */
 	altdev->edac_dev_name = ecc_name;
@@ -1600,6 +1586,8 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 		rc = -ENOMEM;
 		goto err_release_group_1;
 	}
+	of_node_put(np);
+
 	altr_create_edacdev_dbgfs(dci, prv);
 
 	list_add(&altdev->next, &altdev->edac->a10_ecc_devices);
@@ -1609,14 +1597,20 @@ static int altr_portb_setup(struct altr_edac_device_dev *device)
 	return 0;
 
 err_release_group_1:
-	edac_device_free_ctl_info(dci);
+	/*
+	 * Release the devres group first so the managed IRQs are
+	 * unregistered before dci (which contains the IRQ handler's
+	 * data via dci->pvt_info) is freed, avoiding a use-after-free.
+	 */
 	devres_release_group(device->edac->dev, altr_portb_setup);
+	edac_device_free_ctl_info(dci);
+	of_node_put(np);
 	edac_printk(KERN_ERR, EDAC_DEVICE,
 		    "%s:Error setting up EDAC device: %d\n", ecc_name, rc);
 	return rc;
 }
 
-static int __init socfpga_init_sdmmc_ecc(struct altr_edac_device_dev *device)
+static int socfpga_init_sdmmc_ecc(struct altr_edac_device_dev *device)
 {
 	int rc = -ENODEV;
 	struct device_node *child;
@@ -1638,7 +1632,7 @@ static int __init socfpga_init_sdmmc_ecc(struct altr_edac_device_dev *device)
 		goto exit;
 
 	/* Setup portB */
-	return altr_portb_setup(device);
+	rc = altr_portb_setup(device);
 
 exit:
 	of_node_put(child);
@@ -2013,9 +2007,17 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac,
 	return 0;
 
 err_release_group1:
+	/*
+	 * Release the devres group first so the managed IRQs are
+	 * unregistered before dci (which contains the IRQ handler's
+	 * data via dci->pvt_info) is freed, avoiding a use-after-free.
+	 */
+	devres_release_group(edac->dev, NULL);
 	edac_device_free_ctl_info(dci);
+	goto err_print;
 err_release_group:
 	devres_release_group(edac->dev, NULL);
+err_print:
 	edac_printk(KERN_ERR, EDAC_DEVICE,
 		    "%s:Error setting up EDAC device: %d\n", ecc_name, rc);
 
@@ -2214,6 +2216,7 @@ static struct platform_driver altr_edac_a10_driver = {
 	.driver = {
 		.name = "socfpga_a10_ecc_manager",
 		.of_match_table = altr_edac_a10_of_match,
+		.suppress_bind_attrs = true,
 	},
 };
 module_platform_driver(altr_edac_a10_driver);
diff --git a/drivers/edac/bluefield_edac.c b/drivers/edac/bluefield_edac.c
index ae3bb7afa103eb..472fc876277880 100644
--- a/drivers/edac/bluefield_edac.c
+++ b/drivers/edac/bluefield_edac.c
@@ -472,8 +472,8 @@ static void bluefield_edac_mc_remove(struct platform_device *pdev)
 }
 
 static const struct acpi_device_id bluefield_mc_acpi_ids[] = {
-	{"MLNXBF08", 0},
-	{}
+	{ .id = "MLNXBF08" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(acpi, bluefield_mc_acpi_ids);
diff --git a/drivers/edac/dummy_edac.c b/drivers/edac/dummy_edac.c
new file mode 100644
index 00000000000000..193f561a9d513b
--- /dev/null
+++ b/drivers/edac/dummy_edac.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Dummy EDAC driver
+ *
+ * A minimal EDAC device driver that registers its own platform device
+ * unconditionally, so it probes successfully on any system -- including
+ * VMs and guests that have no real ECC-capable cache/memory hardware
+ * exposed to them. Useful for exercising the EDAC core, sysfs interface,
+ * and userspace tooling (edac-utils, mcelog, etc.) without needing the
+ * physical hardware the "real" drivers depend on.
+ *
+ * Modeled after drivers/edac/a72_edac.c (Cortex A72 EDAC L1/L2 cache
+ * error detection), but with all of the hardware-specific bits (system
+ * register reads, SMP cross-calls, devicetree compatible matching)
+ * replaced by a software-only error source that can be poked from
+ * debugfs or left to generate nothing at all.
+ */
+
+#include <linux/cpu.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "edac_module.h"
+
+#define DRVNAME		"dummy-edac"
+#define MESSAGE_SIZE	64
+
+/*
+ * Simulated "error syndrome". In real hardware drivers this would be
+ * read out of a per-CPU system/MMIO register. Here userspace (via
+ * debugfs) or nothing at all drives it, which is exactly the point:
+ * the driver has no dependency on any physical error source.
+ */
+struct dummy_edac_priv {
+	struct dentry *debugfs_dir;
+	atomic_t inject_ce;
+	atomic_t inject_ue;
+};
+
+static void dummy_edac_check(struct edac_device_ctl_info *edac_ctl)
+{
+	struct dummy_edac_priv *priv = edac_ctl->pvt_info;
+	int cpu = raw_smp_processor_id();
+	char msg[MESSAGE_SIZE];
+
+	if (atomic_xchg(&priv->inject_ce, 0)) {
+		snprintf(msg, MESSAGE_SIZE,
+			 "simulated correctable error on CPU %d", cpu);
+		edac_device_handle_ce(edac_ctl, cpu, 0, msg);
+	}
+
+	if (atomic_xchg(&priv->inject_ue, 0)) {
+		snprintf(msg, MESSAGE_SIZE,
+			 "simulated uncorrectable error on CPU %d", cpu);
+		edac_device_handle_ue(edac_ctl, cpu, 0, msg);
+	}
+}
+
+/*
+ * debugfs knobs so you can drive the driver from userspace inside the
+ * guest, e.g.:
+ *   echo 1 > /sys/kernel/debug/dummy-edac/inject_ce
+ *   echo 1 > /sys/kernel/debug/dummy-edac/inject_ue
+ */
+static int inject_ce_set(void *data, u64 val)
+{
+	struct dummy_edac_priv *priv = data;
+
+	if (val)
+		atomic_set(&priv->inject_ce, 1);
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(inject_ce_fops, NULL, inject_ce_set, "%llu\n");
+
+static int inject_ue_set(void *data, u64 val)
+{
+	struct dummy_edac_priv *priv = data;
+
+	if (val)
+		atomic_set(&priv->inject_ue, 1);
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(inject_ue_fops, NULL, inject_ue_set, "%llu\n");
+
+static int dummy_edac_probe(struct platform_device *pdev)
+{
+	struct edac_device_ctl_info *edac_ctl;
+	struct dummy_edac_priv *priv;
+	struct device *dev = &pdev->dev;
+	int rc;
+
+	edac_ctl = edac_device_alloc_ctl_info(sizeof(*priv), "cpu",
+					      num_possible_cpus(), "L", 1, 1,
+					      edac_device_alloc_index());
+	if (!edac_ctl)
+		return -ENOMEM;
+
+	priv = edac_ctl->pvt_info;
+	atomic_set(&priv->inject_ce, 0);
+	atomic_set(&priv->inject_ue, 0);
+
+	edac_ctl->edac_check = dummy_edac_check;
+	edac_ctl->dev = dev;
+	edac_ctl->mod_name = dev_name(dev);
+	edac_ctl->dev_name = dev_name(dev);
+	edac_ctl->ctl_name = DRVNAME;
+	/* Poll fairly slowly; this is a software source, not real hardware. */
+	edac_ctl->poll_msec = 1000;
+	dev_set_drvdata(dev, edac_ctl);
+
+	rc = edac_device_add_device(edac_ctl);
+	if (rc)
+		goto out_dev;
+
+	priv->debugfs_dir = debugfs_create_dir(DRVNAME, NULL);
+	debugfs_create_file_unsafe("inject_ce", 0200, priv->debugfs_dir,
+				    priv, &inject_ce_fops);
+	debugfs_create_file_unsafe("inject_ue", 0200, priv->debugfs_dir,
+				    priv, &inject_ue_fops);
+
+	return 0;
+
+out_dev:
+	edac_device_free_ctl_info(edac_ctl);
+
+	return rc;
+}
+
+static void dummy_edac_remove(struct platform_device *pdev)
+{
+	struct edac_device_ctl_info *edac_ctl = dev_get_drvdata(&pdev->dev);
+	struct dummy_edac_priv *priv = edac_ctl->pvt_info;
+
+	debugfs_remove_recursive(priv->debugfs_dir);
+	edac_device_del_device(edac_ctl->dev);
+	edac_device_free_ctl_info(edac_ctl);
+}
+
+static struct platform_driver dummy_edac_driver = {
+	.probe = dummy_edac_probe,
+	.remove = dummy_edac_remove,
+	.driver = {
+		.name = DRVNAME,
+	},
+};
+
+/*
+ * No devicetree/ACPI match table, and no scan of CPU nodes for a "compatible"
+ * + enable property: simply register a platform device unconditionally so
+ * probe() always runs. There is nothing here that depends on the underlying
+ * platform actually exposing the corresponding hardware, so it loads fine
+ * under QEMU/KVM, containers-with-a-kernel, or any other guest environment.
+ */
+static struct platform_device *dummy_pdev;
+
+static int __init dummy_edac_driver_init(void)
+{
+	int rc;
+
+	dummy_pdev = platform_device_register_simple(DRVNAME, -1, NULL, 0);
+	if (IS_ERR(dummy_pdev)) {
+		pr_err(DRVNAME ": failed to register dummy platform device\n");
+		return PTR_ERR(dummy_pdev);
+	}
+
+	rc = platform_driver_register(&dummy_edac_driver);
+	if (rc) {
+		platform_device_unregister(dummy_pdev);
+		return rc;
+	}
+
+	pr_info("Loading %s\n", DRVNAME);
+
+	return 0;
+}
+
+static void __exit dummy_edac_driver_exit(void)
+{
+	platform_driver_unregister(&dummy_edac_driver);
+	platform_device_unregister(dummy_pdev);
+
+	pr_info("Removing %s\n", DRVNAME);
+}
+
+module_init(dummy_edac_driver_init);
+module_exit(dummy_edac_driver_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Dummy EDAC driver for testing without real ECC hardware");
diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
index fa1853f252b068..87e541c3bf4bbf 100644
--- a/drivers/edac/i10nm_base.c
+++ b/drivers/edac/i10nm_base.c
@@ -479,6 +479,7 @@ static struct pci_dev *get_gnr_mdev(struct skx_dev *d, int logical_idx, int *phy
 		if (mdev) {
 			if (logical == logical_idx) {
 				*physical_idx = i;
+				skx_set_mc_mapping(d, i, logical_idx);
 				return mdev;
 			}
 
@@ -634,6 +635,7 @@ static int i10nm_get_ddr_munits(void)
 
 			mbase = ioremap(base + off, size);
 			if (!mbase) {
+				pci_dev_put(mdev);
 				i10nm_printk(KERN_ERR, "Failed to ioremap 0x%llx\n",
 					     base + off);
 				return -ENODEV;
diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c
index 6f5fdf43e7732c..88af77ee07fe36 100644
--- a/drivers/edac/ie31200_edac.c
+++ b/drivers/edac/ie31200_edac.c
@@ -660,7 +660,9 @@ static int ie31200_init_one(struct pci_dev *pdev,
 	if (pci_enable_device(pdev) < 0)
 		return -EIO;
 	rc = ie31200_probe1(pdev, (struct res_config *)ent->driver_data);
-	if (rc == 0 && !mci_pdev)
+	if (rc)
+		pci_disable_device(pdev);
+	else if (!mci_pdev)
 		mci_pdev = pci_dev_get(pdev);
 
 	return rc;
diff --git a/drivers/edac/intel-bff.c b/drivers/edac/intel-bff.c
new file mode 100644
index 00000000000000..adf2ec34312901
--- /dev/null
+++ b/drivers/edac/intel-bff.c
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright(c) 2026 Intel Corporation. */
+
+/*
+ * Intel driver to reset bitfix filters when they overflow.
+ *
+ * Each bitfix filter has limited slots to track corrected errors.
+ * These slots can be filled by transient corrected errors (e.g.,
+ * bit flips from particle strikes) that don't represent permanent
+ * hardware defects.
+ *
+ * When the filter overflows (yellow status), reset it to reclaim
+ * slots occupied by transient corrected errors. If the overflow
+ * repeats frequently after reset, it indicates persistent hardware
+ * defects that need attention: the system should be scheduled for
+ * servicing.
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cacheinfo.h>
+#include <linux/cleanup.h>
+#include <linux/cpufeature.h>
+#include <linux/cpuhplock.h>
+#include <linux/device-id/x86_cpu.h>
+#include <linux/errno.h>
+#include <linux/gfp_types.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
+#include <linux/limits.h>
+#include <linux/module.h>
+#include <linux/notifier.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/topology.h>
+#include <linux/types.h>
+#include <linux/xarray.h>
+
+#include <asm/cpu_device_id.h>
+#include <asm/cpufeatures.h>
+#include <asm/intel-family.h>
+#include <asm/mce.h>
+#include <asm/msr.h>
+#include <asm/msr-index.h>
+
+/*
+ * A 10-minute observation period helps distinguish between:
+ *
+ *  - A long-term accumulation of transient corrected errors
+ *    (filter stays clear after reset).
+ *
+ *  - Permanent defects (filter overflows again quickly).
+ */
+#define BFF_OVERFLOW_INTERVAL	secs_to_jiffies(10 * 60)
+
+/* Intel bitfix filter control register defines */
+#define MSR_MC0_BFF_CTL		0x000006c0
+#define MSR_MCx_BFF_CTL(x)	(MSR_MC0_BFF_CTL + (x))
+#define  MCI_BFF_RESET		BIT_ULL(0)
+
+/* Machine check bank hardware unit types */
+enum bff_bank_type {
+	BFF_BANK_NONE = 0,
+	BFF_BANK_DCU,
+	BFF_BANK_DTLB,
+	BFF_BANK_MLC,
+	BFF_BANK_CCF,
+	BFF_BANK_HSF,
+	BFF_BANK_IOCACHE,
+};
+
+/*
+ * Mapping from machine check bank numbers on Diamond Rapids CPU, that are
+ * supported by a bitfix filter, to hardware unit type.
+ */
+static const enum bff_bank_type dmr_mcbanks[MAX_NR_BANKS] = {
+	[1]	= BFF_BANK_DCU,
+	[2]	= BFF_BANK_DTLB,
+	[3]	= BFF_BANK_MLC,
+	[6]	= BFF_BANK_CCF,
+	[13]	= BFF_BANK_HSF,
+	[17]	= BFF_BANK_IOCACHE
+};
+
+static const struct x86_cpu_id bff_cpu_ids[] __initconst = {
+	X86_MATCH_VFM(INTEL_DIAMONDRAPIDS_X, dmr_mcbanks),
+	{}
+};
+MODULE_DEVICE_TABLE(x86cpu, bff_cpu_ids);
+
+static const enum bff_bank_type *bff_bank_types;
+
+static DEFINE_XARRAY(bff_bank_xa);
+
+/* Diamond Rapids maps APICID[2] to the IMH instance within a socket. */
+#define APICID_IMH_NUM		GENMASK(2, 2)
+#define IMH_NUM(apicid)		FIELD_GET(APICID_IMH_NUM, apicid)
+#define NUM_IMH_PER_SOCKET	2
+
+static void bff_set_imh_id(struct mce *mce, unsigned long *id)
+{
+	int imh_num;
+
+	imh_num = NUM_IMH_PER_SOCKET * topology_physical_package_id(mce->extcpu) +
+			IMH_NUM(mce->apicid);
+
+	*id |= imh_num;
+}
+
+static bool bff_set_cache_id(int cpu, int level, unsigned long *id)
+{
+	int cacheid;
+
+	guard(cpus_read_lock)();
+
+	cacheid = get_cpu_cacheinfo_id(cpu, level);
+	if (cacheid == -1) {
+		pr_warn("Could not get L%d cache id for CPU %d\n", level, cpu);
+		return false;
+	}
+
+	*id |= cacheid;
+
+	return true;
+}
+
+/*
+ * Cache IDs are only unique within a cache level.
+ * Include the MCA bank number so each BFF-capable hardware
+ * resource has a unique tracking ID.
+ */
+#define BFF_ID_BANK_FIELD	GENMASK(63, 32)
+
+static unsigned long bff_get_id(struct mce *mce)
+{
+	unsigned long id = FIELD_PREP(BFF_ID_BANK_FIELD, mce->bank);
+
+	switch (bff_bank_types[mce->bank]) {
+	case BFF_BANK_DCU:
+	case BFF_BANK_DTLB:
+		if (!bff_set_cache_id(mce->extcpu, 1, &id))
+			return ULONG_MAX;
+		break;
+
+	case BFF_BANK_MLC:
+		if (!bff_set_cache_id(mce->extcpu, 2, &id))
+			return ULONG_MAX;
+		break;
+
+	case BFF_BANK_CCF:
+		if (!bff_set_cache_id(mce->extcpu, 3, &id))
+			return ULONG_MAX;
+		break;
+
+	case BFF_BANK_HSF:
+	case BFF_BANK_IOCACHE:
+		bff_set_imh_id(mce, &id);
+		break;
+
+	default:
+		return ULONG_MAX;
+	}
+
+	return id;
+}
+
+/*
+ * Save current timestamp for bff_id. Return true if it is within
+ * BFF_OVERFLOW_INTERVAL of previous timestamp for this bff_id.
+ */
+static bool bff_overflow_is_frequent(unsigned long bff_id)
+{
+	unsigned long now = jiffies, interval_end;
+	unsigned long *ts;
+
+	if (bff_id == ULONG_MAX)
+		return false;
+
+	ts = xa_load(&bff_bank_xa, bff_id);
+	if (!ts) {
+		ts = kzalloc_obj(*ts);
+		if (!ts) {
+			pr_warn("Failed to allocate timestamp for bitfix filter 0x%lx\n", bff_id);
+			return false;
+		}
+		if (xa_is_err(xa_store(&bff_bank_xa, bff_id, ts, GFP_KERNEL))) {
+			kfree(ts);
+			pr_warn("Failed to record timestamp for bitfix filter 0x%lx\n", bff_id);
+			return false;
+		}
+		*ts = now;
+
+		return false;
+	}
+
+	interval_end = *ts + BFF_OVERFLOW_INTERVAL;
+	*ts = now;
+
+	return time_before(now, interval_end);
+}
+
+static void bff_reset_and_report(struct mce *mce)
+{
+	/* Reset bitfix filter using the CPU that logged the yellow status */
+	if (wrmsrq_on_cpu(mce->extcpu, MSR_MCx_BFF_CTL(mce->bank), MCI_BFF_RESET))
+		pr_warn("Failed to reset bitfix filter for CPU %d Bank %d\n",
+			mce->extcpu, mce->bank);
+
+	/*
+	 * Use the unique id for the bitfix filter instance that overflowed and
+	 * check if this is a repeat within the BFF_OVERFLOW_INTERVAL. If it
+	 * is, then report at WARN severity as the user may want to take action.
+	 */
+	if (bff_overflow_is_frequent(bff_get_id(mce))) {
+		pr_warn_ratelimited(HW_ERR "Socket %d CPU %d Bank %d bitfix filter overflowed frequently\n",
+				    mce->socketid, mce->extcpu, mce->bank);
+	} else {
+		pr_notice_ratelimited(HW_ERR "Socket %d CPU %d Bank %d bitfix filter overflowed\n",
+				      mce->socketid, mce->extcpu, mce->bank);
+	}
+}
+
+static int bff_mce_notify(struct notifier_block *nb, unsigned long val, void *data)
+{
+	struct mce *mce = data;
+
+	/* TES is undefined for uncorrected errors. */
+	if (mce->status & MCI_STATUS_UC)
+		return NOTIFY_DONE;
+
+	if (MCI_STATUS_TES(mce->status) != MCI_STATUS_TES_YELLOW)
+		return NOTIFY_DONE;
+
+	if (mce->bank >= MAX_NR_BANKS || bff_bank_types[mce->bank] == BFF_BANK_NONE)
+		return NOTIFY_DONE;
+
+	bff_reset_and_report(mce);
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block bff_notifier = {
+	.notifier_call	= bff_mce_notify,
+	.priority	= MCE_PRIO_EDAC,
+};
+
+static int __init bff_init(void)
+{
+	const struct x86_cpu_id *m;
+	u64 core_caps, mcg_cap;
+
+	if (!cpu_feature_enabled(X86_FEATURE_MCA))
+		return -ENODEV;
+	rdmsrq(MSR_IA32_MCG_CAP, mcg_cap);
+	if (!(mcg_cap & MCG_TES_P))
+		return -ENODEV;
+
+	if (!cpu_feature_enabled(X86_FEATURE_CORE_CAPABILITIES))
+		return -ENODEV;
+	rdmsrq(MSR_IA32_CORE_CAPS, core_caps);
+	if (!(core_caps & MSR_IA32_CORE_CAPS_BFF_RESET))
+		return -ENODEV;
+
+	m = x86_match_cpu(bff_cpu_ids);
+	if (!m) {
+		pr_info("CPU model not supported by the bitfix filter reset driver\n");
+		return -ENODEV;
+	}
+
+	bff_bank_types = (const enum bff_bank_type *)m->driver_data;
+
+	mce_register_decode_chain(&bff_notifier);
+
+	return 0;
+}
+
+static void __exit bff_exit(void)
+{
+	unsigned long bff_id;
+	unsigned long *ts;
+
+	mce_unregister_decode_chain(&bff_notifier);
+
+	xa_for_each(&bff_bank_xa, bff_id, ts)
+		kfree(ts);
+	xa_destroy(&bff_bank_xa);
+}
+
+module_init(bff_init);
+module_exit(bff_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Tony Luck");
+MODULE_DESCRIPTION("Intel bitfix filter reset driver");
diff --git a/drivers/edac/loongson_edac.c b/drivers/edac/loongson_edac.c
index 38745800ed01dd..3ed4d34a33680c 100644
--- a/drivers/edac/loongson_edac.c
+++ b/drivers/edac/loongson_edac.c
@@ -137,8 +137,8 @@ static void edac_remove(struct platform_device *pdev)
 }
 
 static const struct acpi_device_id loongson_edac_acpi_match[] = {
-	{"LOON0010", 0},
-	{}
+	{ .id = "LOON0010" },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, loongson_edac_acpi_match);
 
diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c
index de749413ff9a84..ae2fda006632c9 100644
--- a/drivers/edac/skx_base.c
+++ b/drivers/edac/skx_base.c
@@ -356,7 +356,7 @@ static bool skx_sad_decode(struct decoded_addr *res)
 			break;
 		case 2:
 			lchan = (addr >> shift) % 2;
-			lchan = (lchan << 1) | !lchan;
+			lchan = (lchan << 1) | (lchan ^ 1);
 			break;
 		case 3:
 			lchan = ((addr >> shift) % 2) << 1;
diff --git a/drivers/edac/versalnet_edac.c b/drivers/edac/versalnet_edac.c
index 9e65c4b1d99dd3..eae9fea60db156 100644
--- a/drivers/edac/versalnet_edac.c
+++ b/drivers/edac/versalnet_edac.c
@@ -953,6 +953,7 @@ static void mc_remove(struct platform_device *pdev)
 	remove_versalnet(priv);
 	rproc_shutdown(priv->mcdi->r5_rproc);
 	cdx_mcdi_finish(priv->mcdi);
+	rproc_put(priv->mcdi->r5_rproc);
 	kfree(priv->mcdi);
 }
 
diff --git a/drivers/firewire/.kunitconfig b/drivers/firewire/.kunitconfig
index 7406acb0047821..ee9129081ceffa 100644
--- a/drivers/firewire/.kunitconfig
+++ b/drivers/firewire/.kunitconfig
@@ -7,3 +7,4 @@ CONFIG_FIREWIRE_KUNIT_PACKET_SERDES_TEST=y
 CONFIG_FIREWIRE_KUNIT_SELF_ID_SEQUENCE_HELPER_TEST=y
 CONFIG_FIREWIRE_KUNIT_OHCI_SERDES_TEST=y
 CONFIG_FIREWIRE_KUNIT_NODE_TREE_TEST=y
+CONFIG_FIREWIRE_KUNIT_CONFIG_ROM_PARSER_AND_GENERATOR_TEST=y
diff --git a/drivers/firewire/Kconfig b/drivers/firewire/Kconfig
index b5abe00accc9dd..7e1d4b2c440b41 100644
--- a/drivers/firewire/Kconfig
+++ b/drivers/firewire/Kconfig
@@ -96,6 +96,22 @@ config FIREWIRE_KUNIT_NODE_TREE_TEST
 	  For more information on KUnit and unit tests in general, refer
 	  to the KUnit documentation in Documentation/dev-tools/kunit/.
 
+config FIREWIRE_KUNIT_CONFIG_ROM_PARSER_AND_GENERATOR_TEST
+	tristate "KUnit tests for config ROM parser and generator" if !KUNIT_ALL_TESTS
+	depends on FIREWIRE && KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This builds the KUnit tests to check parser and generator for
+	  configuration ROM content defined in IEEE 1212.
+
+	  KUnit tests run during boot and output the results to the debug
+	  log in TAP format (https://testanything.org/). Only useful for
+	  kernel devs running KUnit test harness and are not for inclusion
+	  into a production build.
+
+	  For more information on KUnit and unit tests in general, refer
+	  to the KUnit documentation in Documentation/dev-tools/kunit/.
+
 config FIREWIRE_OHCI
 	tristate "OHCI-1394 controllers"
 	depends on PCI && FIREWIRE
diff --git a/drivers/firewire/config-rom-generator-test.c b/drivers/firewire/config-rom-generator-test.c
new file mode 100644
index 00000000000000..dd2e71213209fc
--- /dev/null
+++ b/drivers/firewire/config-rom-generator-test.c
@@ -0,0 +1,407 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// config-rom-generator-test.c - An application of Kunit to test configuration ROM generator.
+//
+// Copyright (c) 2026 Takashi Sakamoto
+//
+// This file can not be built independently since it is intentionally included in core-card.c.
+
+#include <kunit/test.h>
+#include <kunit/device.h>
+
+static const __be32 config_rom_bare[] = {
+	cpu_to_be32(0x0404921b), // bus info
+	cpu_to_be32(0x31333934), // |
+	cpu_to_be32(0xf000b223), // |
+	cpu_to_be32(0x01234567), // |
+	cpu_to_be32(0x89abcdef), // v
+	cpu_to_be32(0x00051b70), // root directory
+	cpu_to_be32(0x0c0083c0), // |
+	cpu_to_be32(0x03001f11), // |
+	cpu_to_be32(0x81000003), // |
+	cpu_to_be32(0x17023901), // |
+	cpu_to_be32(0x81000008), // v
+	cpu_to_be32(0x00064cb7), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4c696e75), // |
+	cpu_to_be32(0x78204669), // |
+	cpu_to_be32(0x72657769), // |
+	cpu_to_be32(0x72650000), // v
+	cpu_to_be32(0x0003ff1c), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4a756a75), // v Juju is a code name at the development time of this stack.
+};
+
+// Following to Configuration ROM for AV/C Devices 1.0 (Dec. 2000. 1394 Trading Association,
+// Document 1999027).
+#define UNIT_SPEC_ID_1394TA		0x0000a02d
+#define UNIT_SW_VERSION_AVC		0x00010001
+#define UNIT_SW_VERSION_IIDC_0104	0x00000100
+
+static const u32 avc_unit_directory_and_leaf[] = {
+	0x00040000,		// Unit directory consists of below 4 quads.
+	(CSR_SPECIFIER_ID << 24) | UNIT_SPEC_ID_1394TA,
+	(CSR_VERSION << 24) | UNIT_SW_VERSION_AVC,
+	(CSR_MODEL << 24) | 0x00260827,	// Today.
+	((CSR_LEAF | CSR_DESCRIPTOR) << 24) | 0x00000001, // Point to next quadlet.
+	0x00030000,	// Text leaf consists of below 3 quads.
+	0x00000000,
+	0x00000000,
+	0x50756900,	// Pui is the name of a cat that the author takes care of.
+};
+
+static const __be32 config_rom_with_avc_unit[] = {
+	cpu_to_be32(0x0404c1e5), // bus info
+	cpu_to_be32(0x31333934), // |
+	cpu_to_be32(0xf000b233), // |
+	cpu_to_be32(0x01234567), // |
+	cpu_to_be32(0x89abcdef), // v
+	cpu_to_be32(0x0006a2d2), // root directory
+	cpu_to_be32(0x0c0083c0), // |
+	cpu_to_be32(0x03001f11), // |
+	cpu_to_be32(0x81000004), // |
+	cpu_to_be32(0x17023901), // |
+	cpu_to_be32(0x81000009), // |
+	cpu_to_be32(0xd100000c), // v
+	cpu_to_be32(0x00064cb7), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4c696e75), // |
+	cpu_to_be32(0x78204669), // |
+	cpu_to_be32(0x72657769), // |
+	cpu_to_be32(0x72650000), // v
+	cpu_to_be32(0x0003ff1c), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4a756a75), // v
+	cpu_to_be32(0x0004227b), // unit directory (from root)
+	cpu_to_be32(0x1200a02d), // |
+	cpu_to_be32(0x13010001), // |
+	cpu_to_be32(0x17260827), // |
+	cpu_to_be32(0x81000001), // v
+	cpu_to_be32(0x0003f771), // text descriptor leaf (from unit)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x50756900), // v
+};
+
+// Following to 1394-based Digital Camera Specification Version 1.04 (Aug. 1996. 1394 Trading
+// Association)
+#define IIDC_COMMAND_REGS_BASE	0x00
+#define IIDC_VENDOR_NAME_LEAF	0x01
+#define IIDC_MODEL_NAME_LEAF	0x02
+
+static const u32 iidc_unit_directories_and_leafs[] = {
+	0x00030000,
+	(CSR_SPECIFIER_ID << 24) | UNIT_SPEC_ID_1394TA,
+	(CSR_VERSION << 24) | UNIT_SW_VERSION_IIDC_0104,
+	((CSR_DIRECTORY | CSR_DEPENDENT_INFO) << 24) | 0x00000001,
+	0x00030000,
+	((CSR_OFFSET | IIDC_COMMAND_REGS_BASE) << 24) | 0x00012345,
+	((CSR_LEAF | IIDC_VENDOR_NAME_LEAF) << 24) | 0x00000002,
+	((CSR_LEAF | IIDC_MODEL_NAME_LEAF) << 24) | 0x00000008,
+	0x00060000,	// Text leaf consists of below 6 quads.
+	0x00000000,
+	0x00000000,
+	0x4c696e75,	// Use the same name in root directory.
+	0x78204669,
+	0x72657769,
+	0x72650000,
+	0x00040000,	// Text leaf consists of below 4 quads.
+	0x00000000,
+	0x00000000,
+	0x43686566,	// Chef Cat is a nick name when inventing IEEE 1394 itself.
+	0x20436174,
+};
+
+static const __be32 config_rom_with_iidc_unit[] = {
+	cpu_to_be32(0x04046a3e), // bus info
+	cpu_to_be32(0x31333934), // |
+	cpu_to_be32(0xf000b243), // |
+	cpu_to_be32(0x01234567), // |
+	cpu_to_be32(0x89abcdef), // v
+	cpu_to_be32(0x0006a2d2), // root directory
+	cpu_to_be32(0x0c0083c0), // |
+	cpu_to_be32(0x03001f11), // |
+	cpu_to_be32(0x81000004), // |
+	cpu_to_be32(0x17023901), // |
+	cpu_to_be32(0x81000009), // |
+	cpu_to_be32(0xd100000c), // v
+	cpu_to_be32(0x00064cb7), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4c696e75), // |
+	cpu_to_be32(0x78204669), // |
+	cpu_to_be32(0x72657769), // |
+	cpu_to_be32(0x72650000), // v
+	cpu_to_be32(0x0003ff1c), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4a756a75), // v
+	cpu_to_be32(0x0003d7fe), // unit directory (from root)
+	cpu_to_be32(0x1200a02d), // |
+	cpu_to_be32(0x13000100), // |
+	cpu_to_be32(0xd4000001), // v
+	cpu_to_be32(0x0003ea57), // dependent directory (from unit directory)
+	cpu_to_be32(0x40012345), // |
+	cpu_to_be32(0x81000002), // |
+	cpu_to_be32(0x82000008), // v
+	cpu_to_be32(0x00064cb7), // text descriptor leaf (from dependent directory)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4c696e75), // |
+	cpu_to_be32(0x78204669), // |
+	cpu_to_be32(0x72657769), // |
+	cpu_to_be32(0x72650000), // v
+	cpu_to_be32(0x0004a3e9), // text descriptor leaf (from dependent directory)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x43686566), // |
+	cpu_to_be32(0x20436174), // v
+};
+
+static const __be32 config_rom_with_avc_and_iidc_units[] = {
+	cpu_to_be32(0x040439c0), // bus info
+	cpu_to_be32(0x31333934), // |
+	cpu_to_be32(0xf000b253), // |
+	cpu_to_be32(0x01234567), // |
+	cpu_to_be32(0x89abcdef), // v
+	cpu_to_be32(0x0007073e), // root directory
+	cpu_to_be32(0x0c0083c0), // |
+	cpu_to_be32(0x03001f11), // |
+	cpu_to_be32(0x81000005), // |
+	cpu_to_be32(0x17023901), // |
+	cpu_to_be32(0x8100000a), // |
+	cpu_to_be32(0xd100000d), // |
+	cpu_to_be32(0xd1000015), // v
+	cpu_to_be32(0x00064cb7), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4c696e75), // |
+	cpu_to_be32(0x78204669), // |
+	cpu_to_be32(0x72657769), // |
+	cpu_to_be32(0x72650000), // v
+	cpu_to_be32(0x0003ff1c), // text descriptor leaf (from root)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4a756a75), // v
+	cpu_to_be32(0x0004227b), // unit directory (from root)
+	cpu_to_be32(0x1200a02d), // |
+	cpu_to_be32(0x13010001), // |
+	cpu_to_be32(0x17260827), // |
+	cpu_to_be32(0x81000001), // v
+	cpu_to_be32(0x0003f771), // text descriptor leaf (from unit)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x50756900), // v
+	cpu_to_be32(0x0003d7fe), // unit directory (from root)
+	cpu_to_be32(0x1200a02d), // |
+	cpu_to_be32(0x13000100), // |
+	cpu_to_be32(0xd4000001), // v
+	cpu_to_be32(0x0003ea57), // dependent directory (from unit directory)
+	cpu_to_be32(0x40012345), // |
+	cpu_to_be32(0x81000002), // |
+	cpu_to_be32(0x82000008), // v
+	cpu_to_be32(0x00064cb7), // text descriptor leaf (from dependent directory)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x4c696e75), // |
+	cpu_to_be32(0x78204669), // |
+	cpu_to_be32(0x72657769), // |
+	cpu_to_be32(0x72650000), // v
+	cpu_to_be32(0x0004a3e9), // text descriptor leaf (from dependent directory)
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x00000000), // |
+	cpu_to_be32(0x43686566), // |
+	cpu_to_be32(0x20436174), // v
+};
+
+static const struct generator_test_case {
+	const char *const name;
+	int config_rom_generation;
+	const __be32 *const expected;
+	unsigned int quadlet_length;
+} generator_test_cases[] = {
+	{
+		.name = "bare",
+		.config_rom_generation = 0,
+		.expected = config_rom_bare,
+		.quadlet_length = ARRAY_SIZE(config_rom_bare),
+	},
+	{
+		.name = "with_avc_unit",
+		.config_rom_generation = 1,
+		.expected = config_rom_with_avc_unit,
+		.quadlet_length = ARRAY_SIZE(config_rom_with_avc_unit),
+	},
+	{
+		.name = "with_iidc_unit",
+		.config_rom_generation = 2,
+		.expected = config_rom_with_iidc_unit,
+		.quadlet_length = ARRAY_SIZE(config_rom_with_iidc_unit),
+	},
+	{
+		.name = "with_avc_and_iidc_unit",
+		.config_rom_generation = 3,
+		.expected = config_rom_with_avc_and_iidc_units,
+		.quadlet_length = ARRAY_SIZE(config_rom_with_avc_and_iidc_units),
+	},
+};
+
+// Define generator_test_gen_params.
+KUNIT_ARRAY_PARAM_DESC(generator_test, generator_test_cases, name);
+
+struct state_data {
+	struct fw_card card;
+	__be32 config_rom[(CSR_CONFIG_ROM_END - CSR_CONFIG_ROM) / sizeof(__be32)];
+};
+
+static void test_config_rom_generator(struct kunit *test)
+{
+	// Use kernel stack since they should be mutable for doubly linked-list.
+	struct fw_descriptor avc_unit_entry = {
+		.length = ARRAY_SIZE(avc_unit_directory_and_leaf),
+		.key = (CSR_DIRECTORY | CSR_UNIT) << 24,
+		.data = avc_unit_directory_and_leaf,
+	};
+	struct fw_descriptor iidc_unit_entry = {
+		.length = ARRAY_SIZE(iidc_unit_directories_and_leafs),
+		.key = (CSR_DIRECTORY | CSR_UNIT) << 24,
+		.data = iidc_unit_directories_and_leafs,
+	};
+	const struct generator_test_case *test_case = test->param_value;
+	struct state_data *state = test->priv;
+	struct fw_card *card = &state->card;
+	__be32 *config_rom = state->config_rom;
+
+	card->config_rom_generation = test_case->config_rom_generation;
+	card->link_speed = SCODE_800;
+	card->max_receive = 11;
+	card->guid = 0x0123456789abcdefULL;
+
+	if (test_case->expected == config_rom_with_avc_unit ||
+	    test_case->expected == config_rom_with_avc_and_iidc_units)
+		KUNIT_EXPECT_EQ(test, fw_core_add_descriptor(&avc_unit_entry), 0);
+
+	if (test_case->expected == config_rom_with_iidc_unit ||
+	    test_case->expected == config_rom_with_avc_and_iidc_units)
+		KUNIT_EXPECT_EQ(test, fw_core_add_descriptor(&iidc_unit_entry), 0);
+
+	scoped_guard(mutex, &card_mutex) {
+		generate_config_rom(card, config_rom);
+		KUNIT_EXPECT_EQ(test, config_rom_length, test_case->quadlet_length);
+	}
+
+	KUNIT_EXPECT_MEMEQ(test, config_rom, test_case->expected,
+			   sizeof(*test_case->expected) * test_case->quadlet_length);
+
+	if (test_case->expected == config_rom_with_iidc_unit ||
+	    test_case->expected == config_rom_with_avc_and_iidc_units)
+		fw_core_remove_descriptor(&iidc_unit_entry);
+
+	if (test_case->expected == config_rom_with_avc_unit ||
+	    test_case->expected == config_rom_with_avc_and_iidc_units)
+		fw_core_remove_descriptor(&avc_unit_entry);
+}
+
+static void add_descriptor_with_invalid_length(struct kunit *test)
+{
+	// Use kernel stack since they should be mutable for doubly linked-list.
+	struct fw_descriptor entry_with_invalid_length = {
+		.length = 257,
+	};
+
+	KUNIT_EXPECT_EQ(test, fw_core_add_descriptor(&entry_with_invalid_length), -EINVAL);
+}
+
+static void add_descriptor_with_invalid_data(struct kunit *test)
+{
+	// Use vendor directory defined in Annex A of Configuration ROM for AV/C Devices 1.0 (Dec.
+	// 2000, 1394 Trading Association, TA Document 1999027).
+	static const u32 invalid_vendor_directory[] = {
+		0x00020000,
+		(CSR_MODEL << 24) | 0x00009402,
+		((CSR_LEAF | CSR_DESCRIPTOR) << 24) | 0x00000001,
+		0xffff0000,	// The length should be 6, invalid.
+		0x00000000,
+		0x00000000,
+		0x436f6e63,
+		0x6174204e,
+		0x6f746174,
+		0x696f6e00,
+	};
+	// Use kernel stack since they should be mutable for doubly linked-list.
+	struct fw_descriptor entry_with_invalid_data = {
+		.length = ARRAY_SIZE(invalid_vendor_directory),
+		.key = (CSR_DIRECTORY | CSR_VENDOR) << 24,
+		.data = invalid_vendor_directory,
+	};
+
+	KUNIT_EXPECT_EQ(test, fw_core_add_descriptor(&entry_with_invalid_data), -EINVAL);
+}
+
+static void add_descriptor_beyond_upper_limit(struct kunit *test)
+{
+	// Use kernel stack since they should be mutable for doubly linked-list.
+	struct fw_descriptor entry_beyond_upper_limit = {
+		.length = 255,
+		.key = (CSR_DIRECTORY | CSR_UNIT) << 24,
+		.data = NULL,
+	};
+	u32 *data;
+
+	data = kunit_kzalloc(test, entry_beyond_upper_limit.length, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, data);
+
+	data[0] = (entry_beyond_upper_limit.length - 1) << 16;
+	entry_beyond_upper_limit.data = data;
+	KUNIT_EXPECT_EQ(test, fw_core_add_descriptor(&entry_beyond_upper_limit), -EBUSY);
+
+	kunit_kfree(test, data);
+}
+
+static const struct fw_card_driver dummy_card_driver;
+
+static int config_rom_generator_test_init(struct kunit *test)
+{
+	struct state_data *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, state);
+
+	struct device *dev = kunit_device_register(test, "dummy-device");
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	fw_card_initialize(&state->card, &dummy_card_driver, dev);
+
+	test->priv = state;
+
+	return 0;
+}
+
+static void config_rom_generator_test_exit(struct kunit *test)
+{
+	struct state_data *state = test->priv;
+
+	kunit_device_unregister(test, state->card.device);
+	kunit_kfree(test, state);
+}
+
+
+static struct kunit_case config_rom_generator_test_cases[] = {
+	KUNIT_CASE_PARAM(test_config_rom_generator, generator_test_gen_params),
+	KUNIT_CASE(add_descriptor_with_invalid_length),
+	KUNIT_CASE(add_descriptor_with_invalid_data),
+	KUNIT_CASE(add_descriptor_beyond_upper_limit),
+	{}
+};
+
+static struct kunit_suite config_rom_generator_test_suite = {
+	.name = "firewire-config-rom-generator",
+	.init = config_rom_generator_test_init,
+	.exit = config_rom_generator_test_exit,
+	.test_cases = config_rom_generator_test_cases,
+};
+kunit_test_suite(config_rom_generator_test_suite);
diff --git a/drivers/firewire/config-rom-parser-test.c b/drivers/firewire/config-rom-parser-test.c
new file mode 100644
index 00000000000000..45f1ead326c50e
--- /dev/null
+++ b/drivers/firewire/config-rom-parser-test.c
@@ -0,0 +1,355 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// config-rom-parser-test.c - An application of Kunit to test configuration ROM parser.
+//
+// Copyright (c) 2026 Takashi Sakamoto
+//
+// This file can not be built independently since it is intentionally included in core-device.c.
+
+#include <kunit/test.h>
+#include <kunit/static_stub.h>
+#include <kunit/device.h>
+
+static const u32 sony_dcr_trv310k_config_rom[] = {
+	0x0404e552,
+	0x31333934,
+	0xe0648100,
+	0x00008500,
+	0x005eb597,
+	0x0007cdd0,
+	0x03000085,
+	0x8100000d,
+	0x17000002,
+	0x81000010,
+	0x0c0083c0,
+	0xd8000002,
+	0xd1000003,
+	0x0001ce96,
+	0xd1000001,
+	0x0004bbee,
+	0x1200a02d,
+	0x13010001,
+	0x17000002,
+	0x81000006,
+	0x00046dc8,
+	0x00000000,
+	0x00000000,
+	0x43616e6f,
+	0x6e000000,
+	0x000621ee,
+	0x00000000,
+	0x00000000,
+	0x4d563569,
+	0x204d4300,
+	0x00000000,
+	0x00000000,
+};
+
+static const u32 canon_mv5i_mc_config_rom[] = {
+	0x0404e552,
+	0x31333934,
+	0xe0648100,
+	0x00008500,
+	0x005eb597,
+	0x0007cdd0,
+	0x03000085,
+	0x8100000d,
+	0x17000002,
+	0x81000010,
+	0x0c0083c0,
+	0xd8000002,
+	0xd1000003,
+	0x0001ce96,
+	0xd1000001,
+	0x0004bbee,
+	0x1200a02d,
+	0x13010001,
+	0x17000002,
+	0x81000006,
+	0x00046dc8,
+	0x00000000,
+	0x00000000,
+	0x43616e6f,
+	0x6e000000,
+	0x000621ee,
+	0x00000000,
+	0x00000000,
+	0x4d563569,
+	0x204d4300,
+	0x00000000,
+	0x00000000,
+};
+
+static const u32 motu_audioexpress_config_rom[] = {
+	0x0410a756,
+	0x31333934,
+	0x20ff7000,
+	0x0001f200,
+	0x000a8a7b,
+	0x0004ef04,
+	0x030001f2,
+	0x0c0083c0,
+	0xd1000002,
+	0x8d000005,
+	0x00031680,
+	0x120001f2,
+	0x13000033,
+	0x17104800,
+	0x00025ef3,
+	0x0001f200,
+	0x000a8a7b,
+};
+
+static const u32 tascam_fw1884_config_rom[] = {
+	0x040f23c0,
+	0x31333934,
+	0x20ff7002,
+	0x00022eff,
+	0xfe800000,
+	0x0004bccc,
+	0x0300022e,
+	0x0c0083c0,
+	0x8d000006,
+	0xd1000001,
+	0x000347f5,
+	0x1200022e,
+	0x13800000,
+	0xd4000004,
+	0x000289aa,
+	0x00022eff,
+	0xfe800000,
+	0x0002ae47,
+	0x81000002,
+	0x82000006,
+	0x0004a79e,
+	0x00000000,
+	0x00000000,
+	0x54415343,
+	0x414d0000,
+	0x00045443,
+	0x00000000,
+	0x00000000,
+	0x46572d31,
+	0x38383400,
+};
+
+static const struct parser_test_case {
+	const char *name;
+	const u32 *quadlets;
+	size_t quadlet_length;
+	int phy_speed_in_self_id;
+	int expected_speed;
+	int expected_quirk;
+	unsigned int expected_max_rec;
+	bool expected_cmc;
+	bool expected_irmc;
+} parser_test_cases[] = {
+	{
+		.name = "detect_irm_is_1394_1995_only_quirk",
+		.quadlets = sony_dcr_trv310k_config_rom,
+		.quadlet_length = ARRAY_SIZE(sony_dcr_trv310k_config_rom),
+		.phy_speed_in_self_id = SCODE_100,
+		.expected_speed = SCODE_100,
+		.expected_quirk = FW_DEVICE_QUIRK_IRM_IS_1394_1995_ONLY,
+		.expected_max_rec = 8,
+		.expected_cmc = true,
+		.expected_irmc = true,
+	},
+	{
+		.name = "detect_irm_ignores_bus_manager_quirk",
+		.quadlets = canon_mv5i_mc_config_rom,
+		.quadlet_length = ARRAY_SIZE(canon_mv5i_mc_config_rom),
+		.phy_speed_in_self_id = SCODE_100,
+		.expected_speed = SCODE_100,
+		.expected_quirk = FW_DEVICE_QUIRK_IRM_IGNORES_BUS_MANAGER,
+		.expected_max_rec = 8,
+		.expected_cmc = true,
+		.expected_irmc = true,
+	},
+	{
+		.name = "detect_ack_packet_with_invalid_pending_code_quirk",
+		.quadlets = motu_audioexpress_config_rom,
+		.quadlet_length = ARRAY_SIZE(motu_audioexpress_config_rom),
+		.phy_speed_in_self_id = SCODE_400,
+		.expected_speed = SCODE_400,
+		.expected_quirk = FW_DEVICE_QUIRK_ACK_PACKET_WITH_INVALID_PENDING_CODE,
+		.expected_max_rec = 7,
+		.expected_cmc = false,
+		.expected_irmc = false,
+	},
+	{
+		.name = "detect_unstable_at_s400_quirk",
+		.quadlets = tascam_fw1884_config_rom,
+		.quadlet_length = ARRAY_SIZE(tascam_fw1884_config_rom),
+		.phy_speed_in_self_id = SCODE_400,
+		.expected_speed = SCODE_200,
+		.expected_quirk = FW_DEVICE_QUIRK_UNSTABLE_AT_S400,
+		.expected_max_rec = 7,
+		.expected_cmc = false,
+		.expected_irmc = false,
+	},
+};
+
+// Define parser_test_gen_params.
+KUNIT_ARRAY_PARAM_DESC(parser_test, parser_test_cases, name);
+
+static int stub_run_transaction_regular(struct fw_card *card, int tcode, int destination_id,
+					int generation, int speed, unsigned long long offset,
+					void *payload, size_t length)
+{
+	struct kunit *test = kunit_get_current_test();
+	const struct parser_test_case *param = test->param_value;
+
+	KUNIT_ASSERT_GE(test, offset, CSR_REGISTER_BASE | CSR_CONFIG_ROM);
+	KUNIT_ASSERT_LT(test, offset, CSR_REGISTER_BASE | CSR_CONFIG_ROM_END);
+	KUNIT_ASSERT_NOT_NULL(test, payload);
+	KUNIT_ASSERT_EQ(test, length, 4);
+
+	unsigned int index = (offset - (CSR_REGISTER_BASE | CSR_CONFIG_ROM)) / sizeof(u32);
+	__be32 *quadlet = payload;
+
+	KUNIT_EXPECT_LE(test, speed, param->expected_speed);
+	KUNIT_EXPECT_LT(test, index, param->quadlet_length);
+
+	*quadlet = cpu_to_be32(param->quadlets[index]);
+
+	return RCODE_COMPLETE;
+}
+
+static void test_parser_with_regular_cases(struct kunit *test)
+{
+	struct fw_device *device = test->priv;
+	const struct parser_test_case *param = test->param_value;
+
+	kunit_activate_static_stub(test, fw_run_transaction, stub_run_transaction_regular);
+
+	device->card->link_speed = SCODE_BETA;
+	device->node->max_speed = param->phy_speed_in_self_id;
+
+	KUNIT_EXPECT_EQ(test, read_config_rom(device, 0), RCODE_COMPLETE);
+
+	KUNIT_EXPECT_EQ(test, device->config_rom_length, param->quadlet_length);
+	KUNIT_EXPECT_MEMEQ(test, device->config_rom, param->quadlets, param->quadlet_length);
+
+	KUNIT_EXPECT_TRUE(test, device->quirks & param->expected_quirk);
+	KUNIT_EXPECT_EQ(test, device->max_speed, param->expected_speed);
+	KUNIT_EXPECT_EQ(test, (unsigned int)device->max_rec, param->expected_max_rec);
+	KUNIT_EXPECT_EQ(test, (bool)device->cmc, param->expected_cmc);
+	KUNIT_EXPECT_EQ(test, (bool)device->irmc, param->expected_irmc);
+
+	kunit_deactivate_static_stub(test, fw_run_transaction);
+}
+
+static int stub_run_transaction_malformed(struct fw_card *card, int tcode, int destination_id,
+					  int generation, int speed, unsigned long long offset,
+					  void *payload, size_t length)
+{
+	static const u32 config_rom_first_part[] = {
+		0x04000000,
+		0x31333934,
+		0x00008002,
+		0x00000000,
+		0x00000000,
+		0x00030000,
+		(CSR_VENDOR << 24) | 0x00123456,	// Regular entry.
+		((CSR_LEAF | CSR_DESCRIPTOR) << 24) | 0x000000f9,	// Beyond the upper limit.
+		((CSR_DIRECTORY | CSR_UNIT) << 24) | 0x00000001,
+		0xffff0000,	// Over the upper limit.
+	};
+	struct kunit *test = kunit_get_current_test();
+
+	KUNIT_ASSERT_GE(test, offset, CSR_REGISTER_BASE | CSR_CONFIG_ROM);
+	KUNIT_ASSERT_LT(test, offset, CSR_REGISTER_BASE | CSR_CONFIG_ROM_END);
+	KUNIT_ASSERT_NOT_NULL(test, payload);
+	KUNIT_ASSERT_EQ(test, length, 4);
+
+	unsigned int index = (offset - (CSR_REGISTER_BASE | CSR_CONFIG_ROM)) / sizeof(u32);
+	__be32 *quadlet = payload;
+
+	if (index < ARRAY_SIZE(config_rom_first_part))
+		*quadlet = cpu_to_be32(config_rom_first_part[index]);
+	else
+		*quadlet = 0;
+
+	return RCODE_COMPLETE;
+}
+
+static void test_parser_with_overflowed_case(struct kunit *test)
+{
+	static const u32 corrected_config_rom[] = {
+		0x04000000,
+		0x31333934,
+		0x00008002,
+		0x00000000,
+		0x00000000,
+		0x00030000,
+		0x03123456,
+		0x00000000,	// Sanitized.
+		0xd1000001,
+		0x00000000,	// Sanitized.
+	};
+	struct fw_device *device = test->priv;
+
+	kunit_activate_static_stub(test, fw_run_transaction, stub_run_transaction_malformed);
+
+	device->card->link_speed = SCODE_BETA;
+
+	KUNIT_EXPECT_EQ(test, read_config_rom(device, 0), RCODE_COMPLETE);
+
+	KUNIT_EXPECT_EQ(test, device->config_rom_length, ARRAY_SIZE(corrected_config_rom));
+	KUNIT_EXPECT_MEMEQ(test, device->config_rom, corrected_config_rom,
+			   sizeof(corrected_config_rom));
+
+	kunit_deactivate_static_stub(test, fw_run_transaction);
+}
+
+static const struct fw_card_driver dummy_card_driver;
+
+static int config_rom_parser_test_init(struct kunit *test)
+{
+	struct fw_device *device;
+	struct device *dev;
+
+	device = kunit_kzalloc(test, sizeof(*device), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, device);
+
+	device->node = kunit_kzalloc(test, sizeof(*device->node), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, device->node);
+	kref_init(&device->node->kref);
+
+	device->card = kunit_kzalloc(test, sizeof(*device->card), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, device->card);
+
+	dev = kunit_device_register(test, "dummy-device");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	fw_card_initialize(device->card, &dummy_card_driver, dev);
+
+	test->priv = device;
+
+	return 0;
+}
+
+static void config_rom_parser_test_exit(struct kunit *test)
+{
+	struct fw_device *device = test->priv;
+
+	kunit_device_unregister(test, device->card->device);
+	kunit_kfree(test, device->card);
+	kunit_kfree(test, device->node);
+	kunit_kfree(test, device);
+}
+
+static struct kunit_case config_rom_parser_test_cases[] = {
+	KUNIT_CASE_PARAM(test_parser_with_regular_cases, parser_test_gen_params),
+	KUNIT_CASE(test_parser_with_overflowed_case),
+	{}
+};
+
+static struct kunit_suite config_rom_parser_test_suite = {
+	.name = "firewire-config-rom-parser",
+	.init = config_rom_parser_test_init,
+	.exit = config_rom_parser_test_exit,
+	.test_cases = config_rom_parser_test_cases,
+};
+kunit_test_suite(config_rom_parser_test_suite);
diff --git a/drivers/firewire/core-card.c b/drivers/firewire/core-card.c
index 94992791f02e86..be6f18d67ece69 100644
--- a/drivers/firewire/core-card.c
+++ b/drivers/firewire/core-card.c
@@ -643,11 +643,12 @@ int fw_card_add(struct fw_card *card, u32 max_receive, u32 link_speed, u64 guid,
 	//  * == WQ_FREEZABLE		The target device would not be available when being freezed.
 	//  * == WQ_HIGHPRI		High priority to process semi-realtime timestamped data.
 	//  * == WQ_SYSFS		Parameters are available via sysfs.
-	//  * max_active == 4		A hardIRQ could notify events for a pair of requests and
-	//				response AR/AT contexts.
+	//  * max_active == 4 + 2	A hardIRQ could notify events for a pair of requests and
+	//				response AR/AT contexts. Additional 2 capacity are for the
+	//				internal handling of local AT request and response packets.
 	async_wq = alloc_workqueue("firewire-async-card%u",
 				   WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_HIGHPRI | WQ_SYSFS,
-				   4, card->index);
+				   6, card->index);
 	if (!async_wq)
 		return -ENOMEM;
 
@@ -851,3 +852,7 @@ int fw_card_read_cycle_time(struct fw_card *card, u32 *cycle_time)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(fw_card_read_cycle_time);
+
+#ifdef CONFIG_FIREWIRE_KUNIT_CONFIG_ROM_PARSER_AND_GENERATOR_TEST
+#include "config-rom-generator-test.c"
+#endif
diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c
index e49d8a58be09ed..60067c5b19736f 100644
--- a/drivers/firewire/core-cdev.c
+++ b/drivers/firewire/core-cdev.c
@@ -47,14 +47,14 @@
 #define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW	5
 #define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP	6
 
-static DEFINE_SPINLOCK(phy_receiver_list_lock);
+static DEFINE_MUTEX(phy_receiver_list_mutex);
 static LIST_HEAD(phy_receiver_list);
 
 struct client {
 	u32 version;
 	struct fw_device *device;
 
-	spinlock_t lock;
+	struct mutex mutex;
 	bool in_shutdown;
 	struct xarray resource_xa;
 	struct list_head event_list;
@@ -133,16 +133,17 @@ struct iso_resource_params {
 	s32 bandwidth;
 };
 
+enum {
+	ISO_RES_AUTO_ALLOC,
+	ISO_RES_AUTO_REALLOC,
+	ISO_RES_AUTO_DEALLOC,
+};
+
 struct iso_resource_auto {
 	struct client_resource resource;
 	struct client *client;
-	/* Schedule work and access todo only with client->lock held. */
 	struct delayed_work work;
-	enum {
-		ISO_RES_AUTO_ALLOC,
-		ISO_RES_AUTO_REALLOC,
-		ISO_RES_AUTO_DEALLOC,
-	} todo;
+	atomic_t todo; // one of ISO_RES_AUTO_XXX.
 	int generation;
 	struct iso_resource_params params;
 	struct iso_resource_event *e_alloc, *e_dealloc;
@@ -196,9 +197,7 @@ static int is_outbound_transaction_resource(const struct client_resource *resour
 
 static void schedule_iso_resource_auto(struct iso_resource_auto *r, unsigned long delay)
 {
-	client_get(r->client);
-	if (!queue_delayed_work(fw_workqueue, &r->work, delay))
-		client_put(r->client);
+	queue_delayed_work(fw_workqueue, &r->work, delay);
 }
 
 /*
@@ -316,7 +315,7 @@ static int fw_device_op_open(struct inode *inode, struct file *file)
 	}
 
 	client->device = device;
-	spin_lock_init(&client->lock);
+	mutex_init(&client->mutex);
 	xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH);
 	INIT_LIST_HEAD(&client->event_list);
 	init_waitqueue_head(&client->wait);
@@ -339,50 +338,61 @@ static void queue_event(struct client *client, struct event *event,
 	event->v[1].data = data1;
 	event->v[1].size = size1;
 
-	scoped_guard(spinlock_irqsave, &client->lock) {
-		if (client->in_shutdown)
+	scoped_guard(mutex, &client->mutex) {
+		if (client->in_shutdown) {
 			kfree(event);
-		else
+		} else {
 			list_add_tail(&event->link, &client->event_list);
-	}
 
-	wake_up_interruptible(&client->wait);
+			wake_up_interruptible(&client->wait);
+		}
+	}
 }
 
-static int dequeue_event(struct client *client,
-			 char __user *buffer, size_t count)
+static ssize_t dequeue_event(struct client *client, char __user *buffer, size_t count)
 {
 	struct event *event;
-	size_t size, total;
-	int i, ret;
 
-	ret = wait_event_interruptible(client->wait,
-			!list_empty(&client->event_list) ||
-			fw_device_is_shutdown(client->device));
-	if (ret < 0)
-		return ret;
+	// After the following block, the event pointer above is guaranteed to have a correct value.
+	{
+		mutex_lock(&client->mutex);
+
+		// This could be replaced with wait_var_event_any_lock() if poll_wait() alternative
+		// would be introduced.
+		int ret = ___wait_event(client->wait,
+			!list_empty(&client->event_list) || fw_device_is_shutdown(client->device),
+			TASK_INTERRUPTIBLE, 0, 0,
+			mutex_unlock(&client->mutex);
+			schedule();
+			mutex_lock(&client->mutex)
+		);
+		if (ret < 0) {
+			mutex_unlock(&client->mutex);
+			return ret;
+		}
 
-	if (list_empty(&client->event_list) &&
-		       fw_device_is_shutdown(client->device))
-		return -ENODEV;
+		if (fw_device_is_shutdown(client->device)) {
+			mutex_unlock(&client->mutex);
+			return -ENODEV;
+		}
 
-	scoped_guard(spinlock_irq, &client->lock) {
 		event = list_first_entry(&client->event_list, struct event, link);
 		list_del(&event->link);
+
+		mutex_unlock(&client->mutex);
 	}
 
-	total = 0;
-	for (i = 0; i < ARRAY_SIZE(event->v) && total < count; i++) {
-		size = min(event->v[i].size, count - total);
-		if (copy_to_user(buffer + total, event->v[i].data, size)) {
+	ssize_t ret = 0;
+
+	for (int i = 0; i < ARRAY_SIZE(event->v) && ret < count; i++) {
+		size_t size = min(event->v[i].size, count - ret);
+		if (copy_to_user(buffer + ret, event->v[i].data, size)) {
 			ret = -EFAULT;
-			goto out;
+			break;
 		}
-		total += size;
+		ret += size;
 	}
-	ret = total;
 
- out:
 	kfree(event);
 
 	return ret;
@@ -398,19 +408,27 @@ static ssize_t fw_device_op_read(struct file *file, char __user *buffer,
 
 static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event,
 				 struct client *client)
+__must_hold(&client->device->client_list_mutex)
 {
-	struct fw_card *card = client->device->card;
-
-	guard(spinlock_irq)(&card->lock);
+	lockdep_assert_held(&client->device->client_list_mutex);
 
+	// This member is related to the above mutex. In detail, see 93b37905f70 ("firewire: cdev:
+	// prevent race between first get_info ioctl and bus reset event queuing").
 	event->closure	     = client->bus_reset_closure;
 	event->type          = FW_CDEV_EVENT_BUS_RESET;
+
 	event->generation    = client->device->generation;
+	smp_rmb();
 	event->node_id       = client->device->node_id;
-	event->local_node_id = card->local_node->node_id;
-	event->bm_node_id    = card->bm_node_id;
-	event->irm_node_id   = card->irm_node->node_id;
-	event->root_node_id  = card->root_node->node_id;
+
+	struct fw_card *card = client->device->card;
+
+	scoped_guard(spinlock_irq, &card->lock) {
+		event->local_node_id = card->local_node->node_id;
+		event->bm_node_id    = card->bm_node_id;
+		event->irm_node_id   = card->irm_node->node_id;
+		event->root_node_id  = card->root_node->node_id;
+	}
 }
 
 static void for_each_client(struct fw_device *device,
@@ -439,11 +457,11 @@ static void queue_bus_reset_event(struct client *client)
 	queue_event(client, &e->event,
 		    &e->reset, sizeof(e->reset), NULL, 0);
 
-	guard(spinlock_irq)(&client->lock);
-
-	xa_for_each(&client->resource_xa, index, resource) {
-		if (is_iso_resource_auto(resource))
-			schedule_iso_resource_auto(to_iso_resource_auto(resource), 0);
+	scoped_guard(mutex, &client->mutex) {
+		xa_for_each(&client->resource_xa, index, resource) {
+			if (is_iso_resource_auto(resource))
+				schedule_iso_resource_auto(to_iso_resource_auto(resource), 0);
+		}
 	}
 }
 
@@ -488,8 +506,6 @@ union ioctl_arg {
 static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
 {
 	struct fw_cdev_get_info *a = &arg->get_info;
-	struct fw_cdev_event_bus_reset bus_reset;
-	unsigned long ret = 0;
 
 	client->version = a->version;
 	a->version = FW_CDEV_KERNEL_VERSION;
@@ -497,52 +513,58 @@ static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
 
 	scoped_guard(rwsem_read, &fw_device_rwsem) {
 		if (a->rom != 0) {
-			size_t want = a->rom_length;
-			size_t have = client->device->config_rom_length * 4;
+			size_t length = min_t(size_t, a->rom_length,
+					      client->device->config_rom_length * 4);
 
-			ret = copy_to_user(u64_to_uptr(a->rom), client->device->config_rom,
-					   min(want, have));
-			if (ret != 0)
+			if (copy_to_user(u64_to_uptr(a->rom), client->device->config_rom, length))
 				return -EFAULT;
 		}
 		a->rom_length = client->device->config_rom_length * 4;
 	}
 
-	guard(mutex)(&client->device->client_list_mutex);
+	scoped_guard(mutex, &client->device->client_list_mutex) {
+		// Coordinate concurrent access to this member with bus reset event handling, see
+		// 93b37905f70 ("firewire: cdev: prevent race between first get_info ioctl and bus
+		// reset event queuing").
+		client->bus_reset_closure = a->bus_reset_closure;
 
-	client->bus_reset_closure = a->bus_reset_closure;
-	if (a->bus_reset != 0) {
-		fill_bus_reset_event(&bus_reset, client);
-		/* unaligned size of bus_reset is 36 bytes */
-		ret = copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36);
+		if (a->bus_reset != 0) {
+			struct fw_cdev_event_bus_reset bus_reset;
+
+			memset(&bus_reset, 0, sizeof(bus_reset));
+			fill_bus_reset_event(&bus_reset, client);
+
+			// This structure has 4 bytes of trailing padding under the System V ABI
+			// on most architectures (due to 8-byte alignment of the long long type),
+			// except for Intel386 (where long long type is aligned to 4 bytes). In
+			// either case, the effective length is 36 bytes.
+			if (copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36))
+				return -EFAULT;
+		}
+
+		if (list_empty(&client->link))
+			list_add_tail(&client->link, &client->device->client_list);
 	}
-	if (ret == 0 && list_empty(&client->link))
-		list_add_tail(&client->link, &client->device->client_list);
 
-	return ret ? -EFAULT : 0;
+	return 0;
 }
 
 static int add_client_resource(struct client *client, struct client_resource *resource,
-			       gfp_t gfp_mask)
+			       client_resource_release_fn_t release)
 {
-	scoped_guard(spinlock_irqsave, &client->lock) {
+	scoped_guard(mutex, &client->mutex) {
 		u32 index;
 		int ret;
 
 		if (client->in_shutdown)
 			return  -ECANCELED;
 
-		if (gfpflags_allow_blocking(gfp_mask)) {
-			ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b,
-				       GFP_NOWAIT);
-		} else {
-			ret = xa_alloc_bh(&client->resource_xa, &index, resource,
-					  xa_limit_32b, GFP_NOWAIT);
-		}
+		ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b, GFP_KERNEL);
 		if (ret < 0)
 			return ret;
 
 		resource->handle = index;
+		resource->release = release;
 		client_get(client);
 	}
 
@@ -556,7 +578,7 @@ static int release_client_resource(struct client *client, u32 handle,
 	unsigned long index = handle;
 	struct client_resource *resource;
 
-	scoped_guard(spinlock_irq, &client->lock) {
+	scoped_guard(mutex, &client->mutex) {
 		if (client->in_shutdown)
 			return -EINVAL;
 
@@ -589,7 +611,7 @@ static void complete_transaction(struct fw_card *card, int rcode, u32 request_ts
 	struct client *client = e->client;
 	unsigned long index = e->r.resource.handle;
 
-	scoped_guard(spinlock_irqsave, &client->lock) {
+	scoped_guard(mutex, &client->mutex) {
 		xa_erase(&client->resource_xa, index);
 		if (client->in_shutdown)
 			wake_up(&client->tx_flush_wait);
@@ -661,8 +683,8 @@ static int init_request(struct client *client,
 	    request->length < 4)
 		return -EINVAL;
 
-	e = kmalloc(sizeof(*e) + request->length, GFP_KERNEL);
-	if (e == NULL)
+	e = kzalloc_flex(*e, rsp.with_tstamp.data, DIV_ROUND_UP(request->length, sizeof(u32)));
+	if (!e)
 		return -ENOMEM;
 	e->client = client;
 
@@ -687,8 +709,7 @@ static int init_request(struct client *client,
 		goto failed;
 	}
 
-	e->r.resource.release = release_transaction;
-	ret = add_client_resource(client, &e->r.resource, GFP_KERNEL);
+	ret = add_client_resource(client, &e->r.resource, release_transaction);
 	if (ret < 0)
 		goto failed;
 
@@ -760,8 +781,8 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
 	if (is_fcp)
 		fw_request_get(request);
 
-	r = kmalloc_obj(*r, GFP_ATOMIC);
-	e = kmalloc_obj(*e, GFP_ATOMIC);
+	r = kmalloc_obj(*r);
+	e = kmalloc_obj(*e);
 	if (r == NULL || e == NULL)
 		goto failed;
 
@@ -771,8 +792,7 @@ static void handle_request(struct fw_card *card, struct fw_request *request,
 	r->data    = payload;
 	r->length  = length;
 
-	r->resource.release = release_request;
-	ret = add_client_resource(handler->client, &r->resource, GFP_ATOMIC);
+	ret = add_client_resource(handler->client, &r->resource, release_request);
 	if (ret < 0)
 		goto failed;
 
@@ -875,8 +895,7 @@ static int ioctl_allocate(struct client *client, union ioctl_arg *arg)
 	}
 	a->offset = r->handler.offset;
 
-	r->resource.release = release_address_handler;
-	ret = add_client_resource(client, &r->resource, GFP_KERNEL);
+	ret = add_client_resource(client, &r->resource, release_address_handler);
 	if (ret < 0) {
 		release_address_handler(client, &r->resource);
 		return ret;
@@ -975,8 +994,7 @@ static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg)
 	if (ret < 0)
 		goto failed;
 
-	r->resource.release = release_descriptor;
-	ret = add_client_resource(client, &r->resource, GFP_KERNEL);
+	ret = add_client_resource(client, &r->resource, release_descriptor);
 	if (ret < 0) {
 		fw_core_remove_descriptor(&r->descriptor);
 		goto failed;
@@ -1002,7 +1020,7 @@ static void iso_callback(struct fw_iso_context *context, u32 cycle,
 	struct client *client = data;
 	struct iso_interrupt_event *e;
 
-	e = kmalloc(sizeof(*e) + header_length, GFP_KERNEL);
+	e = kzalloc_flex(*e, interrupt.header, header_length);
 	if (e == NULL)
 		return;
 
@@ -1327,30 +1345,27 @@ static void iso_resource_auto_work(struct work_struct *work)
 	struct iso_resource_auto *r = from_work(r, work, work.work);
 	struct client *client = r->client;
 	unsigned long index = r->resource.handle;
-	int current_generation, resource_generation, channel, bandwidth, todo;
-	u64 reset_jiffies;
+	int channel, bandwidth;
 	bool free;
 
-	scoped_guard(spinlock_irq, &client->lock) {
-		reset_jiffies = client->device->card->reset_jiffies;
-		current_generation = client->device->generation;
-		resource_generation = r->generation;
-		r->generation = current_generation;
-		todo = r->todo;
-	}
+	u64 reset_jiffies = client->device->card->reset_jiffies;
+	int current_generation = client->device->generation;
+
+	int resource_generation = xchg(&r->generation, current_generation); // But no need to be atomic.
+	int todo = atomic_read(&r->todo);
 
 	switch (todo) {
 	case ISO_RES_AUTO_ALLOC:
 		// Allow 1000ms grace period for other reallocations.
 		if (time_is_after_jiffies64(reset_jiffies + secs_to_jiffies(1))) {
 			schedule_iso_resource_auto(r, msecs_to_jiffies(333));
-			goto out;
+			return;
 		}
 		break;
 	case ISO_RES_AUTO_REALLOC:
 		// We could be called twice within the same generation.
 		if (resource_generation == current_generation)
-			goto out;
+			return;
 		break;
 	case ISO_RES_AUTO_DEALLOC:
 	default:
@@ -1364,23 +1379,23 @@ static void iso_resource_auto_work(struct work_struct *work)
 
 	if (todo == ISO_RES_AUTO_DEALLOC) {
 		free = true;
-		e = r->e_dealloc;
-		r->e_dealloc = NULL;
+		e = xchg(&r->e_dealloc, NULL); // But no need to be atomic.
 	} else {
 		free = false;
 
 		// Is this generation outdated already?  As long as this resource sticks in the
 		// xarray, it will be scheduled again for a newer generation or at shutdown.
 		if (channel == -EAGAIN)
-			goto out;
+			return;
 
 		bool success = channel >= 0 || bandwidth > 0;
 
 		if (!success) {
 			// Allocation or reallocation failure?  Pull this resource out of the
 			// xarray and prepare for deletion, unless the client is shutting down.
-			scoped_guard(spinlock_irq,  &client->lock) {
+			scoped_guard(mutex,  &client->mutex) {
 				if (!client->in_shutdown && xa_erase(&client->resource_xa, index)) {
+					// For the incrementation by add_client_resource().
 					client_put(client);
 					free = true;
 				}
@@ -1389,22 +1404,20 @@ static void iso_resource_auto_work(struct work_struct *work)
 
 		if (todo == ISO_RES_AUTO_REALLOC) {
 			if (success)
-				goto out;
+				return;
 
 			// Notify the userspace client of the failure through a deallocation event.
-			e = r->e_dealloc;
-			r->e_dealloc = NULL;
+			e = xchg(&r->e_dealloc, NULL); // But no need to be atomic.
 		} else {
-			// Transit from allocation to reallocation, except if the client requested
-			// deallocation in the meantime.
-			scoped_guard(spinlock_irq,  &client->lock)
-				r->todo = ISO_RES_AUTO_REALLOC;
+			// Transit from allocation to reallocation. Use compare-and-swap atomic
+			// operation because the todo member can be set with ISO_RES_AUTO_DEALLOC
+			// by release_iso_resource_auto() in parallel.
+			atomic_cmpxchg_relaxed(&r->todo, ISO_RES_AUTO_ALLOC, ISO_RES_AUTO_REALLOC);
 
 			if (channel >= 0)
 				r->params.channels_mask = BIT_ULL(channel);
 
-			e = r->e_alloc;
-			r->e_alloc = NULL;
+			e = xchg(&r->e_alloc, NULL); // But no need to be atomic.
 		}
 	}
 
@@ -1420,18 +1433,17 @@ static void iso_resource_auto_work(struct work_struct *work)
 		kfree(r->e_alloc);
 		kfree(r->e_dealloc);
 		kfree(r);
+
+		// For the incrementation by ioctl_allocate_iso_resource().
+		client_put(client);
 	}
- out:
-	client_put(client);
 }
 
 static void release_iso_resource_auto(struct client *client, struct client_resource *resource)
 {
 	struct iso_resource_auto *r = to_iso_resource_auto(resource);
 
-	guard(spinlock_irq)(&client->lock);
-
-	r->todo = ISO_RES_AUTO_DEALLOC;
+	atomic_set(&r->todo, ISO_RES_AUTO_DEALLOC);
 	schedule_iso_resource_auto(r, 0);
 }
 
@@ -1452,7 +1464,7 @@ static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *a
 
 	INIT_DELAYED_WORK(&r->work, iso_resource_auto_work);
 	r->client	= client;
-	r->todo		= ISO_RES_AUTO_ALLOC;
+	atomic_set(&r->todo, ISO_RES_AUTO_ALLOC);
 	r->e_alloc	= e1;
 	r->e_dealloc	= e2;
 
@@ -1461,11 +1473,11 @@ static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *a
 	e2->iso_resource.closure = request->closure;
 	e2->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
 
-	r->resource.release = release_iso_resource_auto;
-	err = add_client_resource(client, &r->resource, GFP_KERNEL);
+	err = add_client_resource(client, &r->resource, release_iso_resource_auto);
 	if (err < 0)
 		return err;
 	request->handle = r->resource.handle;
+	client_get(client);
 
 	retain_and_null_ptr(e1);
 	retain_and_null_ptr(e2);
@@ -1488,12 +1500,10 @@ static void iso_resource_once_work(struct work_struct *work)
 	struct iso_resource_once *r = from_work(r, work, work);
 	struct client *client = r->client;
 	struct iso_resource_event *e = r->event;
-	int generation, channel, bandwidth;
-
-	scoped_guard(spinlock_irq, &client->lock)
-		generation = client->device->generation;
+	int channel;
 
-	bandwidth = r->params.bandwidth;
+	int generation = client->device->generation;
+	int bandwidth = r->params.bandwidth;
 
 	fw_iso_resource_manage(client->device->card, generation, r->params.channels_mask, &channel,
 			       &bandwidth, r->todo == ISO_RES_ONCE_ALLOC);
@@ -1686,7 +1696,7 @@ static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
 	if (!client->device->is_local)
 		return -ENOSYS;
 
-	e = kzalloc(sizeof(*e) + sizeof(a->data), GFP_KERNEL);
+	e = kzalloc_flex(*e, phy_packet.with_tstamp.data, ARRAY_SIZE(a->data));
 	if (e == NULL)
 		return -ENOMEM;
 
@@ -1734,9 +1744,7 @@ static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg
 	if (!client->device->is_local)
 		return -ENOSYS;
 
-	// NOTE: This can be without irq when we can guarantee that __fw_send_request() for local
-	// destination never runs in any type of IRQ context.
-	scoped_guard(spinlock_irq, &phy_receiver_list_lock)
+	scoped_guard(mutex, &phy_receiver_list_mutex)
 		list_move_tail(&client->phy_receiver_link, &phy_receiver_list);
 
 	client->phy_receiver_closure = a->closure;
@@ -1748,18 +1756,15 @@ void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p)
 {
 	struct client *client;
 
-	// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for local
-	// destination never runs in any type of IRQ context.
-	guard(spinlock_irqsave)(&phy_receiver_list_lock);
+	guard(mutex)(&phy_receiver_list_mutex);
 
 	list_for_each_entry(client, &phy_receiver_list, phy_receiver_link) {
-		struct inbound_phy_packet_event *e;
-
 		if (client->device->card != card)
 			continue;
 
-		e = kmalloc(sizeof(*e) + 8, GFP_ATOMIC);
-		if (e == NULL)
+		struct inbound_phy_packet_event *e =
+			kzalloc_flex(*e, phy_packet.with_tstamp.data, 2);
+		if (!e)
 			break;
 
 		if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
@@ -1909,11 +1914,11 @@ static bool has_outbound_transactions(struct client *client)
 	struct client_resource *resource;
 	unsigned long index;
 
-	guard(spinlock_irq)(&client->lock);
-
-	xa_for_each(&client->resource_xa, index, resource) {
-		if (is_outbound_transaction_resource(resource))
-			return true;
+	scoped_guard(mutex, &client->mutex) {
+		xa_for_each(&client->resource_xa, index, resource) {
+			if (is_outbound_transaction_resource(resource))
+				return true;
+		}
 	}
 
 	return false;
@@ -1926,9 +1931,7 @@ static int fw_device_op_release(struct inode *inode, struct file *file)
 	struct client_resource *resource;
 	unsigned long index;
 
-	// NOTE: This can be without irq when we can guarantee that __fw_send_request() for local
-	// destination never runs in any type of IRQ context.
-	scoped_guard(spinlock_irq, &phy_receiver_list_lock)
+	scoped_guard(mutex, &phy_receiver_list_mutex)
 		list_del(&client->phy_receiver_link);
 
 	scoped_guard(mutex, &client->device->client_list_mutex)
@@ -1942,7 +1945,7 @@ static int fw_device_op_release(struct inode *inode, struct file *file)
 		fw_iso_buffer_destroy(&client->buffer, client->device->card);
 
 	// Freeze client->resource_xa and client->event_list.
-	scoped_guard(spinlock_irq, &client->lock)
+	scoped_guard(mutex, &client->mutex)
 		client->in_shutdown = true;
 
 	wait_event(client->tx_flush_wait, !has_outbound_transactions(client));
diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c
index cbac6691624087..4e79c57263ff2c 100644
--- a/drivers/firewire/core-device.c
+++ b/drivers/firewire/core-device.c
@@ -1442,3 +1442,7 @@ void fw_node_event(struct fw_card *card, struct fw_node *node, int event)
 #ifdef CONFIG_FIREWIRE_KUNIT_DEVICE_ATTRIBUTE_TEST
 #include "device-attribute-test.c"
 #endif
+
+#ifdef CONFIG_FIREWIRE_KUNIT_CONFIG_ROM_PARSER_AND_GENERATOR_TEST
+#include "config-rom-parser-test.c"
+#endif
diff --git a/drivers/firewire/core-transaction.c b/drivers/firewire/core-transaction.c
index 22ae387ae03ca6..b4fccabb8fb210 100644
--- a/drivers/firewire/core-transaction.c
+++ b/drivers/firewire/core-transaction.c
@@ -24,6 +24,7 @@
 #include <linux/timer.h>
 #include <linux/types.h>
 #include <linux/workqueue.h>
+#include <kunit/static_stub.h>
 
 #include <asm/byteorder.h>
 
@@ -39,7 +40,7 @@
 static int try_cancel_split_timeout(struct fw_transaction *t)
 {
 	if (t->is_split_transaction)
-		return timer_delete(&t->split_timeout_timer);
+		return timer_delete(&t->split_timeout_timer) || disable_work(&t->error_work);
 	else
 		return 1;
 }
@@ -51,6 +52,17 @@ static void remove_transaction_entry(struct fw_card *card, struct fw_transaction
 	card->transactions.tlabel_mask &= ~(1ULL << entry->tlabel);
 }
 
+static void invoke_callback(struct fw_transaction *t, int rcode, u32 response_tstamp, void *data,
+			    size_t data_length)
+{
+	if (!t->with_tstamp) {
+		t->callback.without_tstamp(t->card, rcode, data, data_length, t->callback_data);
+	} else {
+		t->callback.with_tstamp(t->card, rcode, t->packet.timestamp, response_tstamp,
+					data, data_length, t->callback_data);
+	}
+}
+
 // Must be called without holding card->transactions.lock.
 void fw_cancel_pending_transactions(struct fw_card *card)
 {
@@ -68,14 +80,7 @@ void fw_cancel_pending_transactions(struct fw_card *card)
 
 	list_for_each_entry_safe(t, tmp, &pending_list, link) {
 		list_del(&t->link);
-
-		if (!t->with_tstamp) {
-			t->callback.without_tstamp(card, RCODE_CANCELLED, NULL, 0,
-						   t->callback_data);
-		} else {
-			t->callback.with_tstamp(card, RCODE_CANCELLED, t->packet.timestamp, 0,
-						NULL, 0, t->callback_data);
-		}
+		invoke_callback(t, RCODE_CANCELLED, 0, NULL, 0);
 	}
 }
 
@@ -107,12 +112,7 @@ static int close_transaction(struct fw_transaction *transaction, struct fw_card
 			return -ENOENT;
 	}
 
-	if (!t->with_tstamp) {
-		t->callback.without_tstamp(card, rcode, NULL, 0, t->callback_data);
-	} else {
-		t->callback.with_tstamp(card, rcode, t->packet.timestamp, response_tstamp, NULL, 0,
-					t->callback_data);
-	}
+	invoke_callback(t, rcode, response_tstamp, NULL, 0);
 
 	return 0;
 }
@@ -135,24 +135,34 @@ int fw_cancel_transaction(struct fw_card *card,
 	if (card->driver->cancel_packet(card, &transaction->packet) == 0)
 		return 0;
 
+	u32 curr_cycle_time = 0;
+
+	// Timestamping on behalf of hardware.
+	(void)fw_card_read_cycle_time(card, &curr_cycle_time);
+	tstamp = cycle_time_to_ohci_tstamp(curr_cycle_time);
+
 	/*
 	 * If the request packet has already been sent, we need to see
 	 * if the transaction is still pending and remove it in that case.
 	 */
+	return close_transaction(transaction, card, RCODE_CANCELLED, tstamp);
+}
+EXPORT_SYMBOL(fw_cancel_transaction);
 
-	if (transaction->packet.ack == 0) {
-		// The timestamp is reused since it was just read now.
-		tstamp = transaction->packet.timestamp;
-	} else {
-		u32 curr_cycle_time = 0;
+static void error_callback_work(struct work_struct *work)
+{
+	struct fw_transaction *t = from_work(t, work, error_work);
 
-		(void)fw_card_read_cycle_time(card, &curr_cycle_time);
-		tstamp = cycle_time_to_ohci_tstamp(curr_cycle_time);
-	}
+	invoke_callback(t, t->rcode, t->response_timestamp, NULL, 0);
+}
 
-	return close_transaction(transaction, card, RCODE_CANCELLED, tstamp);
+static void schedule_error_callback(struct fw_transaction *t, int rcode, u32 response_timestamp)
+{
+	t->rcode = rcode;
+	t->response_timestamp = response_timestamp;
+
+	queue_work(t->card->async_wq, &t->error_work);
 }
-EXPORT_SYMBOL(fw_cancel_transaction);
 
 static void split_transaction_timeout_callback(struct timer_list *timer)
 {
@@ -165,12 +175,7 @@ static void split_transaction_timeout_callback(struct timer_list *timer)
 		remove_transaction_entry(card, t);
 	}
 
-	if (!t->with_tstamp) {
-		t->callback.without_tstamp(card, RCODE_CANCELLED, NULL, 0, t->callback_data);
-	} else {
-		t->callback.with_tstamp(card, RCODE_CANCELLED, t->packet.timestamp,
-					t->split_timeout_cycle, NULL, 0, t->callback_data);
-	}
+	schedule_error_callback(t, RCODE_CANCELLED, t->split_timeout_cycle);
 }
 
 // card->transactions.lock should be acquired in advance for the linked list.
@@ -203,9 +208,7 @@ static void transmit_complete_callback(struct fw_packet *packet,
 	{
 		unsigned int delta;
 
-		// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for
-		// local destination never runs in any type of IRQ context.
-		scoped_guard(spinlock_irqsave, &card->split_timeout.lock) {
+		scoped_guard(spinlock, &card->split_timeout.lock) {
 			t->split_timeout_cycle =
 				compute_split_timeout_timestamp(card, packet->timestamp) & 0xffff;
 			delta = card->split_timeout.jiffies;
@@ -366,8 +369,8 @@ __must_hold(&card->transactions.lock)
  *
  * In case of request types without payload, @data is NULL and @length is 0.
  *
- * After the transaction is completed successfully or unsuccessfully, the
- * @callback will be called.  Among its parameters is the response code which
+ * After the transaction is completed successfully or unsuccessfully, the @callback will be called
+ * in process context.  Among its parameters is the response code which
  * is either one of the rcodes per IEEE 1394 or, in case of internal errors,
  * the firewire-core specific %RCODE_SEND_ERROR.  The other firewire-core
  * specific rcodes (%RCODE_CANCELLED, %RCODE_BUSY, %RCODE_GENERATION,
@@ -386,6 +389,12 @@ void __fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode
 {
 	int tlabel;
 
+	t->card = card;
+	t->callback = callback;
+	t->with_tstamp = with_tstamp;
+	t->callback_data = callback_data;
+	INIT_WORK(&t->error_work, error_callback_work);
+
 	/*
 	 * Allocate tlabel from the bitmap and put the transaction on
 	 * the list while holding the card spinlock.
@@ -396,30 +405,23 @@ void __fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode
 	scoped_guard(spinlock_irqsave, &card->transactions.lock)
 		tlabel = allocate_tlabel(card);
 	if (tlabel < 0) {
-		if (!with_tstamp) {
-			callback.without_tstamp(card, RCODE_SEND_ERROR, NULL, 0, callback_data);
-		} else {
-			// Timestamping on behalf of hardware.
-			u32 curr_cycle_time = 0;
-			u32 tstamp;
+		// Timestamping on behalf of hardware.
+		u32 curr_cycle_time = 0;
+		u32 tstamp;
 
-			(void)fw_card_read_cycle_time(card, &curr_cycle_time);
-			tstamp = cycle_time_to_ohci_tstamp(curr_cycle_time);
+		(void)fw_card_read_cycle_time(card, &curr_cycle_time);
+		tstamp = cycle_time_to_ohci_tstamp(curr_cycle_time);
+
+		t->packet.timestamp = tstamp;
+		schedule_error_callback(t, RCODE_SEND_ERROR, tstamp);
 
-			callback.with_tstamp(card, RCODE_SEND_ERROR, tstamp, tstamp, NULL, 0,
-					     callback_data);
-		}
 		return;
 	}
 
 	t->node_id = destination_id;
 	t->tlabel = tlabel;
-	t->card = card;
 	t->is_split_transaction = false;
 	timer_setup(&t->split_timeout_timer, split_transaction_timeout_callback, 0);
-	t->callback = callback;
-	t->with_tstamp = with_tstamp;
-	t->callback_data = callback_data;
 	t->packet.callback = transmit_complete_callback;
 
 	// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for
@@ -481,6 +483,9 @@ int fw_run_transaction(struct fw_card *card, int tcode, int destination_id,
 		       int generation, int speed, unsigned long long offset,
 		       void *payload, size_t length)
 {
+	KUNIT_STATIC_STUB_REDIRECT(fw_run_transaction, card, tcode, destination_id, generation,
+				   speed, offset, payload, length);
+
 	struct transaction_callback_data d;
 	struct fw_transaction t;
 
@@ -635,16 +640,14 @@ static int put_address_handler(struct fw_address_handler *handler)
  *
  * When a request is received that falls within the specified address range, the specified callback
  * is invoked.  The parameters passed to the callback give the details of the particular request.
- * The callback is invoked in the workqueue context in most cases. However, if the request is
- * initiated by the local node, the callback is invoked in the initiator's context.
- *
- * To be called in process context.
- * Return value:  0 on success, non-zero otherwise.
  *
  * The start offset of the handler's address region is determined by
  * fw_core_add_address_handler() and is returned in handler->offset.
  *
  * Address allocations are exclusive, except for the FCP registers.
+ *
+ * Context: Process context.
+ * Returns: 0 on success, non-zero otherwise.
  */
 int fw_core_add_address_handler(struct fw_address_handler *handler,
 				const struct fw_address_region *region)
@@ -891,14 +894,12 @@ static struct fw_request *allocate_request(struct fw_card *card,
 		return NULL;
 	}
 
-	request = kmalloc(sizeof(*request) + length, GFP_ATOMIC);
-	if (request == NULL)
+	request = kzalloc_flex(*request, data, DIV_ROUND_UP(length, sizeof(u32)));
+	if (!request)
 		return NULL;
 	kref_init(&request->kref);
 
-	// NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for
-	// local destination never runs in any type of IRQ context.
-	scoped_guard(spinlock_irqsave, &card->split_timeout.lock)
+	scoped_guard(spinlock, &card->split_timeout.lock)
 		request->response.timestamp = compute_split_timeout_timestamp(card, p->timestamp);
 
 	request->response.speed = p->speed;
@@ -1198,12 +1199,7 @@ void fw_core_handle_response(struct fw_card *card, struct fw_packet *p)
 	 */
 	card->driver->cancel_packet(card, &t->packet);
 
-	if (!t->with_tstamp) {
-		t->callback.without_tstamp(card, rcode, data, data_length, t->callback_data);
-	} else {
-		t->callback.with_tstamp(card, rcode, t->packet.timestamp, p->timestamp, data,
-					data_length, t->callback_data);
-	}
+	invoke_callback(t, rcode, p->timestamp, data, data_length);
 }
 EXPORT_SYMBOL(fw_core_handle_response);
 
@@ -1256,9 +1252,7 @@ static void handle_topology_map(struct fw_card *card, struct fw_request *request
 
 	start = (offset - topology_map_region.start) / 4;
 
-	// NOTE: This can be without irqsave when we can guarantee that fw_send_request() for local
-	// destination never runs in any type of IRQ context.
-	scoped_guard(spinlock_irqsave, &card->topology_map.lock)
+	scoped_guard(spinlock, &card->topology_map.lock)
 		memcpy(payload, &card->topology_map.buffer[start], length);
 
 	fw_send_response(card, request, RCODE_COMPLETE);
@@ -1336,10 +1330,7 @@ static void handle_registers(struct fw_card *card, struct fw_request *request,
 		if (tcode == TCODE_READ_QUADLET_REQUEST) {
 			*data = cpu_to_be32(card->split_timeout.hi);
 		} else if (tcode == TCODE_WRITE_QUADLET_REQUEST) {
-			// NOTE: This can be without irqsave when we can guarantee that
-			// __fw_send_request() for local destination never runs in any type of IRQ
-			// context.
-			scoped_guard(spinlock_irqsave, &card->split_timeout.lock) {
+			scoped_guard(spinlock, &card->split_timeout.lock) {
 				card->split_timeout.hi = be32_to_cpu(*data) & 7;
 				update_split_timeout(card);
 			}
@@ -1352,10 +1343,7 @@ static void handle_registers(struct fw_card *card, struct fw_request *request,
 		if (tcode == TCODE_READ_QUADLET_REQUEST) {
 			*data = cpu_to_be32(card->split_timeout.lo);
 		} else if (tcode == TCODE_WRITE_QUADLET_REQUEST) {
-			// NOTE: This can be without irqsave when we can guarantee that
-			// __fw_send_request() for local destination never runs in any type of IRQ
-			// context.
-			scoped_guard(spinlock_irqsave, &card->split_timeout.lock) {
+			scoped_guard(spinlock, &card->split_timeout.lock) {
 				card->split_timeout.lo = be32_to_cpu(*data) & 0xfff80000;
 				update_split_timeout(card);
 			}
diff --git a/drivers/firewire/device-attribute-test.c b/drivers/firewire/device-attribute-test.c
index 97478a96d1c965..2c482afca8f11a 100644
--- a/drivers/firewire/device-attribute-test.c
+++ b/drivers/firewire/device-attribute-test.c
@@ -86,7 +86,7 @@ static void device_attr_simple_avc(struct kunit *test)
 			.type = &fw_device_type,
 		},
 		.config_rom = simple_avc_config_rom,
-		.config_rom_length = sizeof(simple_avc_config_rom),
+		.config_rom_length = ARRAY_SIZE(simple_avc_config_rom),
 	};
 	static const struct fw_unit unit0 = {
 		.device = {
@@ -168,7 +168,7 @@ static void device_attr_legacy_avc(struct kunit *test)
 			.type = &fw_device_type,
 		},
 		.config_rom = legacy_avc_config_rom,
-		.config_rom_length = sizeof(legacy_avc_config_rom),
+		.config_rom_length = ARRAY_SIZE(legacy_avc_config_rom),
 	};
 	static const struct fw_unit unit0 = {
 		.device = {
diff --git a/drivers/firewire/ohci-serdes-test.c b/drivers/firewire/ohci-serdes-test.c
index 258f668619ef1c..87b136e041de3b 100644
--- a/drivers/firewire/ohci-serdes-test.c
+++ b/drivers/firewire/ohci-serdes-test.c
@@ -43,7 +43,7 @@ static void test_self_id_receive_buffer_deserialization(struct kunit *test)
 static void test_at_data_serdes(struct kunit *test)
 {
 	static const __le32 expected[] = {
-		cpu_to_le32(0x00020e80),
+		cpu_to_le32(0x00820e80),
 		cpu_to_le32(0xffc2ffff),
 		cpu_to_le32(0xe0000000),
 	};
@@ -56,7 +56,7 @@ static void test_at_data_serdes(struct kunit *test)
 	unsigned int destination_id = ohci1394_at_data_get_destination_id(expected);
 	u64 destination_offset = ohci1394_at_data_get_destination_offset(expected);
 
-	KUNIT_EXPECT_FALSE(test, has_src_bus_id);
+	KUNIT_EXPECT_TRUE(test, has_src_bus_id);
 	KUNIT_EXPECT_EQ(test, 0x02, speed);
 	KUNIT_EXPECT_EQ(test, 0x03, tlabel);
 	KUNIT_EXPECT_EQ(test, 0x02, retry);
diff --git a/drivers/firewire/ohci.c b/drivers/firewire/ohci.c
index 46dd53ecedc72c..df53efec0b0a07 100644
--- a/drivers/firewire/ohci.c
+++ b/drivers/firewire/ohci.c
@@ -162,7 +162,13 @@ struct context {
 struct at_context {
 	struct context context;
 	struct work_struct work;
-	bool flushing;
+};
+
+// The local-to-local transaction is handled by the work item in the following structure.
+struct at_local {
+	struct list_head list;
+	spinlock_t lock;
+	struct work_struct work;
 };
 
 struct iso_context {
@@ -217,6 +223,9 @@ struct fw_ohci {
 	struct at_context at_request_ctx;
 	struct at_context at_response_ctx;
 
+	struct at_local at_request_local;
+	struct at_local at_response_local;
+
 	u32 it_context_support;
 	u32 it_context_mask;     /* unoccupied IT contexts */
 	struct iso_context *it_context_list;
@@ -1338,9 +1347,7 @@ static void at_context_flush(struct at_context *ctx)
 
 	disable_work_sync(&ctx->work);
 
-	WRITE_ONCE(ctx->flushing, true);
 	ohci_at_context_work(&ctx->work);
-	WRITE_ONCE(ctx->flushing, false);
 
 	enable_work(&ctx->work);
 }
@@ -1362,8 +1369,10 @@ static int handle_at_packet(struct context *context,
 	struct driver_data *driver_data;
 	struct fw_packet *packet;
 	int evt;
+	// Check whether this is called from at_context_flush().
+	bool in_flushing = current_work() != &ctx->work;
 
-	if (last->transfer_status == 0 && !READ_ONCE(ctx->flushing))
+	if (last->transfer_status == 0 && !in_flushing)
 		/* This descriptor isn't done yet, stop iteration. */
 		return 0;
 
@@ -1395,7 +1404,7 @@ static int handle_at_packet(struct context *context,
 		break;
 
 	case OHCI1394_evt_missing_ack:
-		if (READ_ONCE(ctx->flushing))
+		if (in_flushing)
 			packet->ack = RCODE_GENERATION;
 		else {
 			/*
@@ -1417,7 +1426,7 @@ static int handle_at_packet(struct context *context,
 		break;
 
 	case OHCI1394_evt_no_status:
-		if (READ_ONCE(ctx->flushing)) {
+		if (in_flushing) {
 			packet->ack = RCODE_GENERATION;
 			break;
 		}
@@ -1456,11 +1465,11 @@ static int handle_at_packet(struct context *context,
 
 static u32 get_cycle_time(struct fw_ohci *ohci);
 
-static void handle_local_rom(struct fw_ohci *ohci,
-			     struct fw_packet *packet, u32 csr)
+static void handle_local_rom(struct fw_ohci *ohci, struct fw_packet *packet, u32 csr,
+			     struct fw_packet *response)
 {
-	struct fw_packet response;
-	int tcode, length, i;
+	int tcode, rcode, length, i;
+	void *payload = NULL;
 
 	tcode = async_header_get_tcode(packet->header);
 	if (tcode_is_block_packet(tcode))
@@ -1470,25 +1479,22 @@ static void handle_local_rom(struct fw_ohci *ohci,
 
 	i = csr - CSR_CONFIG_ROM;
 	if (i + length > CONFIG_ROM_SIZE) {
-		fw_fill_response(&response, packet->header,
-				 RCODE_ADDRESS_ERROR, NULL, 0);
+		rcode = RCODE_ADDRESS_ERROR;
+		length = 0;
 	} else if (!tcode_is_read_request(tcode)) {
-		fw_fill_response(&response, packet->header,
-				 RCODE_TYPE_ERROR, NULL, 0);
+		rcode = RCODE_TYPE_ERROR;
+		length = 0;
 	} else {
-		fw_fill_response(&response, packet->header, RCODE_COMPLETE,
-				 (void *) ohci->config_rom + i, length);
+		rcode = RCODE_COMPLETE;
+		payload = (u8 *)ohci->config_rom + i;
 	}
 
-	// Timestamping on behalf of the hardware.
-	response.timestamp = cycle_time_to_ohci_tstamp(get_cycle_time(ohci));
-	fw_core_handle_response(&ohci->card, &response);
+	fw_fill_response(response, packet->header, rcode, payload, length);
 }
 
-static void handle_local_lock(struct fw_ohci *ohci,
-			      struct fw_packet *packet, u32 csr)
+static void handle_local_lock(struct fw_ohci *ohci, struct fw_packet *packet, u32 csr,
+			      struct fw_packet *response)
 {
-	struct fw_packet response;
 	int tcode, length, ext_tcode, sel, try;
 	__be32 *payload, lock_old;
 	u32 lock_arg, lock_data;
@@ -1506,9 +1512,8 @@ static void handle_local_lock(struct fw_ohci *ohci,
 		lock_arg = 0;
 		lock_data = 0;
 	} else {
-		fw_fill_response(&response, packet->header,
-				 RCODE_TYPE_ERROR, NULL, 0);
-		goto out;
+		fw_fill_response(response, packet->header, RCODE_TYPE_ERROR, NULL, 0);
+		return;
 	}
 
 	sel = (csr - CSR_BUS_MANAGER_ID) / 4;
@@ -1520,86 +1525,159 @@ static void handle_local_lock(struct fw_ohci *ohci,
 		if (reg_read(ohci, OHCI1394_CSRControl) & 0x80000000) {
 			lock_old = cpu_to_be32(reg_read(ohci,
 							OHCI1394_CSRData));
-			fw_fill_response(&response, packet->header,
-					 RCODE_COMPLETE,
-					 &lock_old, sizeof(lock_old));
-			goto out;
+			fw_fill_response(response, packet->header, RCODE_COMPLETE, &lock_old,
+					 sizeof(lock_old));
+			return;
 		}
 
 	ohci_err(ohci, "swap not done (CSR lock timeout)\n");
-	fw_fill_response(&response, packet->header, RCODE_BUSY, NULL, 0);
+	fw_fill_response(response, packet->header, RCODE_BUSY, NULL, 0);
+}
 
- out:
-	// Timestamping on behalf of the hardware.
-	response.timestamp = cycle_time_to_ohci_tstamp(get_cycle_time(ohci));
-	fw_core_handle_response(&ohci->card, &response);
+static bool in_config_rom_csr_registers(u64 offset)
+{
+	return in_range(offset, CSR_CONFIG_ROM, CONFIG_ROM_SIZE);
 }
 
-static void handle_local_request(struct at_context *ctx, struct fw_packet *packet)
+// 5.5.1 Bus Management CSR Registers.
+static bool in_bus_management_csr_registers(u64 offset)
 {
-	struct fw_ohci *ohci = ctx->context.ohci;
-	u64 offset, csr;
+	// 0x22c = CSR_MAINT_CONTROL following to CSR_CHANNELS_AVAILABLE_LO.
+	return in_range(offset, CSR_BUS_MANAGER_ID, 0x22c - CSR_BUS_MANAGER_ID);
+}
 
-	if (ctx == &ohci->at_request_ctx) {
-		packet->ack = ACK_PENDING;
-		packet->callback(packet, &ohci->card, packet->ack);
-	}
+static void handle_at_request_local_packet(struct fw_ohci *ohci, struct fw_packet *packet)
+{
+	// Emulate split transaction.
+	packet->ack = ACK_PENDING;
+	packet->callback(packet, &ohci->card, packet->ack);
 
-	offset = async_header_get_offset(packet->header);
-	csr = offset - CSR_REGISTER_BASE;
+	u64 csr_offset = async_header_get_offset(packet->header) - CSR_REGISTER_BASE;
 
-	/* Handle config rom reads. */
-	if (csr >= CSR_CONFIG_ROM && csr < CSR_CONFIG_ROM_END)
-		handle_local_rom(ohci, packet, csr);
-	else switch (csr) {
-	case CSR_BUS_MANAGER_ID:
-	case CSR_BANDWIDTH_AVAILABLE:
-	case CSR_CHANNELS_AVAILABLE_HI:
-	case CSR_CHANNELS_AVAILABLE_LO:
-		handle_local_lock(ohci, packet, csr);
-		break;
-	default:
-		if (ctx == &ohci->at_request_ctx)
-			fw_core_handle_request(&ohci->card, packet);
+	if (in_config_rom_csr_registers(csr_offset) || in_bus_management_csr_registers(csr_offset)) {
+		struct fw_packet response;
+
+		memset(&response, 0, sizeof(response));
+
+		if (in_bus_management_csr_registers(csr_offset))
+			handle_local_lock(ohci, packet, csr_offset, &response);
 		else
-			fw_core_handle_response(&ohci->card, packet);
-		break;
-	}
+			handle_local_rom(ohci, packet, csr_offset, &response);
 
-	if (ctx == &ohci->at_response_ctx) {
-		packet->ack = ACK_COMPLETE;
-		packet->callback(packet, &ohci->card, packet->ack);
+		// Timestamping on behalf of the hardware.
+		response.timestamp = cycle_time_to_ohci_tstamp(get_cycle_time(ohci));
+
+		// Finish the transaction immediately.
+		fw_core_handle_response(&ohci->card, &response);
+	} else {
+		fw_core_handle_request(&ohci->card, packet);
 	}
 }
 
-static void at_context_transmit(struct at_context *ctx, struct fw_packet *packet)
+static void handle_at_response_local_packet(struct fw_ohci *ohci, struct fw_packet *packet)
 {
-	struct fw_ohci *ohci = ctx->context.ohci;
-	unsigned long flags;
-	int ret;
+	u64 csr_offset = async_header_get_offset(packet->header) - CSR_REGISTER_BASE;
 
-	spin_lock_irqsave(&ohci->lock, flags);
+	// The transaction is already finished by handle_local_at_request_packet().
+	if (WARN_ON(in_config_rom_csr_registers(csr_offset)) ||
+	    WARN_ON(in_bus_management_csr_registers(csr_offset)))
+		return;
 
-	if (async_header_get_destination(packet->header) == ohci->node_id &&
-	    ohci->generation == packet->generation) {
-		spin_unlock_irqrestore(&ohci->lock, flags);
+	fw_core_handle_response(&ohci->card, packet);
 
-		// Timestamping on behalf of the hardware.
-		packet->timestamp = cycle_time_to_ohci_tstamp(get_cycle_time(ohci));
+	packet->ack = ACK_COMPLETE;
+	packet->callback(packet, &ohci->card, packet->ack);
+}
 
-		handle_local_request(ctx, packet);
-		return;
+static void handle_at_local_packets(struct at_local *local, struct fw_ohci *ohci,
+			void (*handle_at_local_packet)(struct fw_ohci *, struct fw_packet *))
+{
+	struct fw_packet *packet;
+
+	spin_lock(&local->lock);
+
+	while ((packet = list_first_entry_or_null(&local->list, typeof(*packet), link_for_local))) {
+		list_del(&packet->link_for_local);
+		spin_unlock(&local->lock);
+
+		if (unlikely(packet->ack != 0)) {
+			// This case is active when the call of at_context_queue_packet() returns
+			// error in at_context_transmit().
+			packet->callback(packet, &ohci->card, packet->ack);
+		} else {
+			handle_at_local_packet(ohci, packet);
+		}
+
+		spin_lock(&local->lock);
 	}
 
-	ret = at_context_queue_packet(ctx, packet);
-	spin_unlock_irqrestore(&ohci->lock, flags);
+	spin_unlock(&local->lock);
+}
+
+static void at_request_local_work(struct work_struct *work)
+{
+	struct at_local *local = from_work(local, work, work);
+	struct fw_ohci *ohci = container_of(local, struct fw_ohci, at_request_local);
 
-	if (ret < 0) {
-		// Timestamping on behalf of the hardware.
-		packet->timestamp = cycle_time_to_ohci_tstamp(get_cycle_time(ohci));
+	handle_at_local_packets(local, ohci, handle_at_request_local_packet);
+}
+
+static void at_response_local_work(struct work_struct *work)
+{
+	struct at_local *local = from_work(local, work, work);
+	struct fw_ohci *ohci = container_of(local, struct fw_ohci, at_response_local);
+
+	handle_at_local_packets(local, ohci, handle_at_response_local_packet);
+}
+
+static void at_local_init(struct at_local *local, work_func_t func)
+{
+	spin_lock_init(&local->lock);
+	INIT_LIST_HEAD(&local->list);
+	INIT_WORK(&local->work, func);
+}
+
+static bool destination_is_local(const struct fw_packet *packet, const struct fw_ohci *ohci)
+__must_hold(&ohci->lock)
+{
+	lockdep_assert_held(&ohci->lock);
+
+	return (async_header_get_destination(packet->header) == ohci->node_id &&
+		ohci->generation == packet->generation);
+}
+
+static void queue_work_for_at_local_packet(struct at_context *ctx, struct fw_packet *packet,
+					   struct fw_ohci *ohci)
+{
+	struct at_local *local;
+
+	if (ctx == &ohci->at_request_ctx)
+		local = &ohci->at_request_local;
+	else
+		local = &ohci->at_response_local;
+
+	// Timestamping on behalf of the hardware.
+	packet->timestamp = cycle_time_to_ohci_tstamp(get_cycle_time(ohci));
+
+	scoped_guard(spinlock_irqsave, &local->lock)
+		list_add_tail(&packet->link_for_local, &local->list);
+	queue_work(ohci->card.async_wq, &local->work);
+}
 
-		packet->callback(packet, &ohci->card, packet->ack);
+static void at_context_transmit(struct at_context *ctx, struct fw_packet *packet)
+{
+	struct fw_ohci *ohci = ctx->context.ohci;
+	bool use_work = true;
+
+	scoped_guard(spinlock_irqsave, &ohci->lock) {
+		if (!destination_is_local(packet, ohci)) {
+			if (!at_context_queue_packet(ctx, packet))
+				use_work = false;
+		}
 	}
+
+	if (use_work)
+		queue_work_for_at_local_packet(ctx, packet, ohci);
 }
 
 static void detect_dead_context(struct fw_ohci *ohci,
@@ -2456,6 +2534,9 @@ static void ohci_disable(struct fw_card *card)
 	flush_work(&ohci->at_request_ctx.work);
 	flush_work(&ohci->at_response_ctx.work);
 
+	flush_work(&ohci->at_request_local.work);
+	flush_work(&ohci->at_response_local.work);
+
 	for (i = 0; i < ohci->n_ir; ++i) {
 		if (!(ohci->ir_context_mask & BIT(i)))
 			flush_work(&ohci->ir_context_list[i].base.work);
@@ -2467,6 +2548,9 @@ static void ohci_disable(struct fw_card *card)
 
 	at_context_flush(&ohci->at_request_ctx);
 	at_context_flush(&ohci->at_response_ctx);
+
+	at_request_local_work(&ohci->at_request_local.work);
+	at_response_local_work(&ohci->at_response_local.work);
 }
 
 static int ohci_set_config_rom(struct fw_card *card,
@@ -2592,7 +2676,7 @@ static int ohci_cancel_packet(struct fw_card *card, struct fw_packet *packet)
 	packet->callback(packet, &ohci->card, packet->ack);
 	ret = 0;
  out:
-	enable_work(&ctx->work);
+	enable_and_queue_work(card->async_wq, &ctx->work);
 
 	return ret;
 }
@@ -3666,6 +3750,9 @@ static int pci_probe(struct pci_dev *dev,
 		return err;
 	INIT_WORK(&ohci->at_response_ctx.work, ohci_at_context_work);
 
+	at_local_init(&ohci->at_request_local, at_request_local_work);
+	at_local_init(&ohci->at_response_local, at_response_local_work);
+
 	reg_write(ohci, OHCI1394_IsoRecvIntMaskSet, ~0);
 	ohci->ir_context_channels = ~0ULL;
 	ohci->ir_context_support = reg_read(ohci, OHCI1394_IsoRecvIntMaskSet);
@@ -3761,7 +3848,7 @@ static void pci_remove(struct pci_dev *dev)
 	dev_notice(&dev->dev, "removing fw-ohci device\n");
 }
 
-static int __maybe_unused pci_suspend(struct device *dev)
+static int pci_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct fw_ohci *ohci = pci_get_drvdata(pdev);
@@ -3773,7 +3860,7 @@ static int __maybe_unused pci_suspend(struct device *dev)
 }
 
 
-static int __maybe_unused pci_resume(struct device *dev)
+static int pci_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct fw_ohci *ohci = pci_get_drvdata(pdev);
@@ -3804,14 +3891,14 @@ static const struct pci_device_id pci_table[] = {
 
 MODULE_DEVICE_TABLE(pci, pci_table);
 
-static SIMPLE_DEV_PM_OPS(pci_pm_ops, pci_suspend, pci_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(pci_pm_ops, pci_suspend, pci_resume);
 
 static struct pci_driver fw_ohci_pci_driver = {
 	.name		= ohci_driver_name,
 	.id_table	= pci_table,
 	.probe		= pci_probe,
 	.remove		= pci_remove,
-	.driver.pm	= &pci_pm_ops,
+	.driver.pm	= pm_sleep_ptr(&pci_pm_ops),
 };
 
 static int __init fw_ohci_init(void)
diff --git a/drivers/firewire/ohci.h b/drivers/firewire/ohci.h
index 218666cfe14a5e..63ccf2f4450458 100644
--- a/drivers/firewire/ohci.h
+++ b/drivers/firewire/ohci.h
@@ -178,7 +178,7 @@
 
 static inline bool ohci1394_at_data_get_src_bus_id(const __le32 *data)
 {
-	return !!((data[0] & OHCI1394_AT_DATA_Q0_srcBusID_MASK) >> OHCI1394_AT_DATA_Q0_srcBusID_SHIFT);
+	return !!((le32_to_cpu(data[0]) & OHCI1394_AT_DATA_Q0_srcBusID_MASK) >> OHCI1394_AT_DATA_Q0_srcBusID_SHIFT);
 }
 
 static inline void ohci1394_at_data_set_src_bus_id(__le32 *data, bool src_bus_id)
diff --git a/drivers/firewire/uapi-test.c b/drivers/firewire/uapi-test.c
index bc3f10a2e516e0..0bdacb1caeb830 100644
--- a/drivers/firewire/uapi-test.c
+++ b/drivers/firewire/uapi-test.c
@@ -72,11 +72,31 @@ static void structure_layout_event_phy_packet2(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, 24, offsetof(struct fw_cdev_event_phy_packet2, data));
 }
 
+static void structure_layout_event_bus_reset(struct kunit *test)
+{
+#if defined(CONFIG_X86_32)
+       // 4 bytes alignment for aggregate type including 8 bytes storage types.
+       KUNIT_EXPECT_EQ(test, 36, sizeof(struct fw_cdev_event_bus_reset));
+#else
+       KUNIT_EXPECT_EQ(test, 40, sizeof(struct fw_cdev_event_bus_reset));
+#endif
+
+       KUNIT_EXPECT_EQ(test, 0, offsetof(struct fw_cdev_event_bus_reset, closure));
+       KUNIT_EXPECT_EQ(test, 8, offsetof(struct fw_cdev_event_bus_reset, type));
+       KUNIT_EXPECT_EQ(test, 12, offsetof(struct fw_cdev_event_bus_reset, node_id));
+       KUNIT_EXPECT_EQ(test, 16, offsetof(struct fw_cdev_event_bus_reset, local_node_id));
+       KUNIT_EXPECT_EQ(test, 20, offsetof(struct fw_cdev_event_bus_reset, bm_node_id));
+       KUNIT_EXPECT_EQ(test, 24, offsetof(struct fw_cdev_event_bus_reset, irm_node_id));
+       KUNIT_EXPECT_EQ(test, 28, offsetof(struct fw_cdev_event_bus_reset, root_node_id));
+       KUNIT_EXPECT_EQ(test, 32, offsetof(struct fw_cdev_event_bus_reset, generation));
+}
+
 static struct kunit_case structure_layout_test_cases[] = {
 	KUNIT_CASE(structure_layout_event_response),
 	KUNIT_CASE(structure_layout_event_request3),
 	KUNIT_CASE(structure_layout_event_response2),
 	KUNIT_CASE(structure_layout_event_phy_packet2),
+	KUNIT_CASE(structure_layout_event_bus_reset),
 	{}
 };
 
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index b7cc11e4fbfa65..c183d98c1e8f5a 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -299,7 +299,7 @@ endif # TURRIS_MOX_RWTM
 source "drivers/firmware/arm_ffa/Kconfig"
 source "drivers/firmware/broadcom/Kconfig"
 source "drivers/firmware/cirrus/Kconfig"
-source "drivers/firmware/google/Kconfig"
+source "drivers/firmware/coreboot/Kconfig"
 source "drivers/firmware/efi/Kconfig"
 source "drivers/firmware/imx/Kconfig"
 source "drivers/firmware/meson/Kconfig"
@@ -310,5 +310,6 @@ source "drivers/firmware/samsung/Kconfig"
 source "drivers/firmware/smccc/Kconfig"
 source "drivers/firmware/tegra/Kconfig"
 source "drivers/firmware/xilinx/Kconfig"
+source "drivers/firmware/arm_rmm/Kconfig"
 
 endmenu
diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile
index be46f1e1dc77fa..a855d369617321 100644
--- a/drivers/firmware/Makefile
+++ b/drivers/firmware/Makefile
@@ -30,7 +30,7 @@ obj-y				+= broadcom/
 obj-y				+= cirrus/
 obj-y				+= meson/
 obj-y				+= microchip/
-obj-$(CONFIG_GOOGLE_FIRMWARE)	+= google/
+obj-$(CONFIG_COREBOOT_FIRMWARE)	+= coreboot/
 obj-y				+= efi/
 obj-y				+= imx/
 obj-y				+= psci/
@@ -39,3 +39,4 @@ obj-y				+= samsung/
 obj-y				+= smccc/
 obj-y				+= tegra/
 obj-y				+= xilinx/
+obj-y				+= arm_rmm/
diff --git a/drivers/firmware/arm_rmm/Kconfig b/drivers/firmware/arm_rmm/Kconfig
new file mode 100644
index 00000000000000..e981e12fdbe6a8
--- /dev/null
+++ b/drivers/firmware/arm_rmm/Kconfig
@@ -0,0 +1,17 @@
+
+config ARCH_SUPPORTS_RMM
+	bool
+
+config ARM_RMM_RSI
+	bool "Realm Service Interface (RSI) Support"
+	depends on ARCH_SUPPORTS_RMM
+	default y
+	help
+	  Enable support for running Linux as a Realm guest under a Realm
+	  Management Monitor (RMM), as defined by the Arm Confidential
+	  Compute Architecture.
+
+	  The RMM exposes the Realm Service Interface (RSI) to Realm guests.
+	  Linux uses RSI to negotiate the interface version, obtain the Realm
+	  configuration, and manage the protected or shared state of guest
+	  memory.
diff --git a/drivers/firmware/arm_rmm/Makefile b/drivers/firmware/arm_rmm/Makefile
new file mode 100644
index 00000000000000..3708b37a9aafc6
--- /dev/null
+++ b/drivers/firmware/arm_rmm/Makefile
@@ -0,0 +1,2 @@
+
+obj-$(CONFIG_ARM_RMM_RSI)	= rsi.o
diff --git a/drivers/firmware/arm_rmm/rsi.c b/drivers/firmware/arm_rmm/rsi.c
new file mode 100644
index 00000000000000..0fcc4e33bd92f6
--- /dev/null
+++ b/drivers/firmware/arm_rmm/rsi.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023 ARM Ltd.
+ */
+
+#include <linux/jump_label.h>
+#include <linux/memblock.h>
+#include <linux/psci.h>
+#include <linux/swiotlb.h>
+#include <linux/arm-rsi-cmds.h>
+#include <linux/kobject.h>
+#include <linux/sysfs.h>
+
+#include <asm/io.h>
+#include <asm/mem_encrypt.h>
+#include <asm/pgtable.h>
+
+static struct realm_config config;
+
+unsigned long prot_ns_shared;
+EXPORT_SYMBOL(prot_ns_shared);
+
+DEFINE_STATIC_KEY_FALSE_RO(rsi_present);
+EXPORT_SYMBOL(rsi_present);
+
+static bool rsi_version_matches(void)
+{
+	unsigned long ver_lower, ver_higher;
+	unsigned long ret = rsi_request_version(RSI_ABI_VERSION,
+						&ver_lower,
+						&ver_higher);
+
+	if (ret == SMCCC_RET_NOT_SUPPORTED)
+		return false;
+
+	if (ret != RSI_SUCCESS) {
+		pr_err("RME: RMM doesn't support RSI version %lu.%lu. Supported range: %lu.%lu-%lu.%lu\n",
+		       RSI_ABI_VERSION_MAJOR, RSI_ABI_VERSION_MINOR,
+		       RSI_ABI_VERSION_GET_MAJOR(ver_lower),
+		       RSI_ABI_VERSION_GET_MINOR(ver_lower),
+		       RSI_ABI_VERSION_GET_MAJOR(ver_higher),
+		       RSI_ABI_VERSION_GET_MINOR(ver_higher));
+		return false;
+	}
+
+	pr_info("RME: Using RSI version %lu.%lu\n",
+		RSI_ABI_VERSION_GET_MAJOR(ver_lower),
+		RSI_ABI_VERSION_GET_MINOR(ver_lower));
+
+	return true;
+}
+
+static void __init arm64_rsi_setup_memory(void)
+{
+	u64 i;
+	phys_addr_t start, end;
+
+	/*
+	 * Iterate over the available memory ranges and convert the state to
+	 * protected memory. We should take extra care to ensure that we DO NOT
+	 * permit any "DESTROYED" pages to be converted to "RAM".
+	 *
+	 * panic() is used because if the attempt to switch the memory to
+	 * protected has failed here, then future accesses to the memory are
+	 * simply going to be reflected as a SEA (Synchronous External Abort)
+	 * which we can't handle.  Bailing out early prevents the guest limping
+	 * on and dying later.
+	 */
+	for_each_mem_range(i, &start, &end) {
+		if (rsi_set_memory_range_protected_safe(start, end)) {
+			panic("Failed to set memory range to protected: %pa-%pa",
+			      &start, &end);
+		}
+	}
+}
+
+/*
+ * Check if a given PA range is Trusted (e.g., Protected memory, a Trusted Device
+ * mapping, or an MMIO emulated in the Realm world).
+ *
+ * We can rely on the RIPAS value of the region to detect if a given region is
+ * protected.
+ *
+ *  RIPAS_DEV - A trusted device memory or a trusted emulated MMIO (in the Realm
+ *		world
+ *  RIPAS_RAM - Memory (RAM), protected by the RMM guarantees. (e.g., Firmware
+ *		reserved regions for data sharing).
+ *
+ *  RIPAS_DESTROYED is a special case of one of the above, where the host did
+ *  something without our permission and as such we can't do anything about it.
+ *
+ * The only case where something is emulated by the untrusted hypervisor or is
+ * backed by shared memory is indicated by RSI_RIPAS_EMPTY.
+ */
+bool arm64_rsi_is_protected(phys_addr_t base, size_t size)
+{
+	enum ripas ripas;
+	phys_addr_t end, top;
+
+	/* Overflow ? */
+	if (WARN_ON(base + size <= base))
+		return false;
+
+	end = ALIGN(base + size, RSI_GRANULE_SIZE);
+	base = ALIGN_DOWN(base, RSI_GRANULE_SIZE);
+
+	while (base < end) {
+		if (WARN_ON(rsi_ipa_state_get(base, end, &ripas, &top)))
+			break;
+		if (WARN_ON(top <= base))
+			break;
+		if (ripas == RSI_RIPAS_EMPTY)
+			break;
+		base = top;
+	}
+
+	return base >= end;
+}
+EXPORT_SYMBOL(arm64_rsi_is_protected);
+
+static int realm_ioremap_hook(phys_addr_t phys, size_t size, pgprot_t *prot)
+{
+	if (arm64_rsi_is_protected(phys, size))
+		*prot = pgprot_encrypted(*prot);
+	else
+		*prot = pgprot_decrypted(*prot);
+
+	return 0;
+}
+
+static int realm_set_memory_encrypted(unsigned long addr, int numpages)
+{
+	int ret = __set_memory_enc_dec(addr, numpages, true);
+
+	/*
+	 * If the request to change state fails, then the only sensible cause
+	 * of action for the caller is to leak the memory
+	 */
+	WARN(ret, "Failed to encrypt memory, %d pages will be leaked",
+	     numpages);
+
+	return ret;
+}
+
+static int realm_set_memory_decrypted(unsigned long addr, int numpages)
+{
+	int ret = __set_memory_enc_dec(addr, numpages, false);
+
+	WARN(ret, "Failed to decrypt memory, %d pages will be leaked",
+	     numpages);
+
+	return ret;
+}
+
+static const struct arm64_mem_crypt_ops realm_crypt_ops = {
+	.encrypt = realm_set_memory_encrypted,
+	.decrypt = realm_set_memory_decrypted,
+};
+
+static int realm_register_memory_enc_ops(void)
+{
+	return arm64_mem_crypt_ops_register(&realm_crypt_ops);
+}
+
+void __init arm64_rsi_init(void)
+{
+	if (arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_SMC)
+		return;
+	if (!rsi_version_matches())
+		return;
+	if (WARN_ON(rsi_get_realm_config(lm_alias(&config))))
+		return;
+	prot_ns_shared = __phys_to_pte_val(BIT(config.ipa_bits - 1));
+
+	if (arm64_ioremap_prot_hook_register(realm_ioremap_hook))
+		return;
+
+	if (realm_register_memory_enc_ops())
+		return;
+
+	arm64_rsi_setup_memory();
+
+	static_branch_enable(&rsi_present);
+}
+
+static ssize_t realm_guest_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "1\n");
+}
+
+static struct kobj_attribute cca_realm_guest = __ATTR_RO(realm_guest);
+static const struct attribute *cca_realm_attrs[] = {
+	&cca_realm_guest.attr,
+	NULL
+};
+
+static int __init realm_sysfs_init(void)
+{
+	int ret;
+	struct kobject *cca_kobj;
+
+	if (!is_realm_world())
+		return 0;
+
+	cca_kobj = kobject_create_and_add("cca", firmware_kobj);
+	if (!cca_kobj)
+		return -ENOMEM;
+
+	ret = sysfs_create_files(cca_kobj, cca_realm_attrs);
+	if (!ret)
+		return 0;
+
+	kobject_put(cca_kobj);
+	return ret;
+}
+device_initcall(realm_sysfs_init);
diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c
index e060edbe7e832e..25197197db8e1a 100644
--- a/drivers/firmware/arm_scmi/bus.c
+++ b/drivers/firmware/arm_scmi/bus.c
@@ -9,14 +9,15 @@
 
 #include <linux/types.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 #include <linux/device.h>
 
 #include "common.h"
 
-#define SCMI_UEVENT_MODALIAS_FMT	"%s:%02x:%s"
+#define SCMI_UEVENT_MODALIAS_FMT	SCMI_MODULE_PREFIX "%02x:%s"
 
 BLOCKING_NOTIFIER_HEAD(scmi_requested_devices_nh);
 EXPORT_SYMBOL_GPL(scmi_requested_devices_nh);
@@ -185,7 +186,7 @@ static int scmi_protocol_table_register(const struct scmi_device_id *id_table)
 	const struct scmi_device_id *entry;
 	int ret;
 
-	for (entry = id_table; entry->name; entry++) {
+	for (entry = id_table; entry->name[0]; entry++) {
 		ret = scmi_protocol_device_request(entry);
 		if (ret)
 			goto err_unrequest;
@@ -205,7 +206,7 @@ scmi_protocol_table_unregister(const struct scmi_device_id *id_table)
 {
 	const struct scmi_device_id *entry;
 
-	for (entry = id_table; entry->name; entry++)
+	for (entry = id_table; entry->name[0]; entry++)
 		scmi_protocol_device_unrequest(entry);
 }
 
@@ -219,10 +220,10 @@ static int __scmi_dev_match_by_id_table(struct scmi_device *scmi_dev,
 					const struct scmi_device_id *id_table,
 					bool skip_transport)
 {
-	if (!id_table || !id_table->name)
+	if (!id_table || !id_table->name[0])
 		return 0;
 
-	for (; id_table->protocol_id && id_table->name; id_table++)
+	for (; id_table->protocol_id && id_table->name[0]; id_table++)
 		if (id_table->protocol_id == scmi_dev->protocol_id &&
 		    !(skip_transport && scmi_device_is_transport(scmi_dev)) &&
 		    !strcmp(id_table->name, scmi_dev->name))
@@ -266,7 +267,7 @@ scmi_child_dev_find_get(struct device *parent, int prot_id, const char *name)
 	struct device *dev;
 
 	id_table[0].protocol_id = prot_id;
-	id_table[0].name = name;
+	strscpy(id_table[0].name, name, sizeof(id_table[0].name));
 
 	dev = device_find_child(parent, &id_table, scmi_match_by_id_table);
 	if (!dev)
@@ -300,8 +301,7 @@ static int scmi_device_uevent(const struct device *dev, struct kobj_uevent_env *
 	const struct scmi_device *scmi_dev = to_scmi_dev(dev);
 
 	return add_uevent_var(env, "MODALIAS=" SCMI_UEVENT_MODALIAS_FMT,
-			      dev_name(&scmi_dev->dev), scmi_dev->protocol_id,
-			      scmi_dev->name);
+			      scmi_dev->protocol_id, scmi_dev->name);
 }
 
 static ssize_t modalias_show(struct device *dev,
@@ -309,9 +309,8 @@ static ssize_t modalias_show(struct device *dev,
 {
 	struct scmi_device *scmi_dev = to_scmi_dev(dev);
 
-	return sysfs_emit(buf, SCMI_UEVENT_MODALIAS_FMT,
-			  dev_name(&scmi_dev->dev), scmi_dev->protocol_id,
-			  scmi_dev->name);
+	return sysfs_emit(buf, SCMI_UEVENT_MODALIAS_FMT "\n",
+			  scmi_dev->protocol_id, scmi_dev->name);
 }
 static DEVICE_ATTR_RO(modalias);
 
@@ -429,17 +428,16 @@ static void scmi_device_release(struct device *dev)
 	struct scmi_device *scmi_dev = to_scmi_dev(dev);
 
 	scmi_device_release_resources(scmi_dev);
-	of_node_put(dev->of_node);
+	fwnode_handle_put(dev_fwnode(dev));
 	kfree_const(scmi_dev->name);
 	kfree(scmi_dev);
 }
 
 static void __scmi_device_destroy(struct scmi_device *scmi_dev)
 {
-	pr_debug("(%pOF) Destroying SCMI device '%s' for protocol 0x%x (%s)\n",
-		 scmi_dev->dev.parent->of_node,
-		 dev_name(&scmi_dev->dev), scmi_dev->protocol_id,
-		 scmi_dev->name);
+	pr_debug("(%pfwf) Destroying SCMI device '%s' for protocol 0x%x (%s)\n",
+		 dev_fwnode(&scmi_dev->dev), dev_name(&scmi_dev->dev),
+		 scmi_dev->protocol_id, scmi_dev->name);
 
 	device_del(&scmi_dev->dev);
 	scmi_device_release_resources(scmi_dev);
@@ -447,7 +445,7 @@ static void __scmi_device_destroy(struct scmi_device *scmi_dev)
 }
 
 static struct scmi_device *
-__scmi_device_create(struct device_node *np, struct device *parent,
+__scmi_device_create(struct fwnode_handle *fwnode, struct device *parent,
 		     int protocol, const char *name)
 {
 	int id, retval;
@@ -458,7 +456,7 @@ __scmi_device_create(struct device_node *np, struct device *parent,
 	 * If the same protocol/name device already exist under the same parent
 	 * (i.e. SCMI instance) just return the existent device.
 	 * This avoids any race between the SCMI driver, creating devices for
-	 * each DT defined protocol at probe time, and the concurrent
+	 * each fwnode defined protocol at probe time, and the concurrent
 	 * registration of SCMI drivers.
 	 */
 	scmi_dev = scmi_child_dev_find_get(parent, protocol, name);
@@ -495,7 +493,7 @@ __scmi_device_create(struct device_node *np, struct device *parent,
 
 	scmi_dev->id = id;
 	scmi_dev->dev.parent = parent;
-	device_set_node(&scmi_dev->dev, of_fwnode_handle(of_node_get(np)));
+	device_set_node(&scmi_dev->dev, fwnode_handle_get(fwnode));
 	scmi_dev->dev.bus = &scmi_bus_type;
 	scmi_dev->dev.release = scmi_device_release;
 	dev_set_name(&scmi_dev->dev, "scmi_dev.%d", id);
@@ -504,8 +502,8 @@ __scmi_device_create(struct device_node *np, struct device *parent,
 	if (retval)
 		goto put_dev;
 
-	pr_debug("(%pOF) Created SCMI device '%s' for protocol 0x%x (%s)\n",
-		 parent->of_node, dev_name(&scmi_dev->dev), protocol, name);
+	pr_debug("(%pfwf) Created SCMI device '%s' - protocol 0x%x (%s)\n",
+		 fwnode, dev_name(&scmi_dev->dev), protocol, name);
 
 	return scmi_dev;
 put_dev:
@@ -521,23 +519,51 @@ __scmi_device_create(struct device_node *np, struct device *parent,
 }
 
 static struct scmi_device *
-_scmi_device_create(struct device_node *np, struct device *parent,
+_scmi_device_create(struct fwnode_handle *fwnode, struct device *parent,
 		    int protocol, const char *name)
 {
 	struct scmi_device *sdev;
 
-	sdev = __scmi_device_create(np, parent, protocol, name);
+	sdev = __scmi_device_create(fwnode, parent, protocol, name);
 	if (!sdev)
-		pr_err("(%pOF) Failed to create device for protocol 0x%x (%s)\n",
-		       parent->of_node, protocol, name);
+		pr_err("(%pfwf) Failed to create device - protocol 0x%x (%s)\n",
+		       fwnode, protocol, name);
 
 	return sdev;
 }
 
+/* Standard protocols table */
+static const struct scmi_device_id scmi_std_id_table[] = {
+	{ SCMI_PROTOCOL_POWER, "genpd" },
+	{ SCMI_PROTOCOL_SYSTEM, "syspower" },
+	{ SCMI_PROTOCOL_PERF, "perf" },
+	{ SCMI_PROTOCOL_PERF, "cpufreq" },
+	{ SCMI_PROTOCOL_CLOCK, "clocks" },
+	{ SCMI_PROTOCOL_SENSOR, "hwmon" },
+	{ SCMI_PROTOCOL_SENSOR, "iiodev" },
+	{ SCMI_PROTOCOL_RESET, "reset" },
+	{ SCMI_PROTOCOL_VOLTAGE, "regulator" },
+	{ SCMI_PROTOCOL_POWERCAP, "powercap" },
+	{ SCMI_PROTOCOL_PINCTRL, "pinctrl" },
+	{ SCMI_PROTOCOL_PINCTRL, "pinctrl-imx" },
+	{ },
+};
+
+static bool scmi_device_id_in_std_id_table(const struct scmi_device_id *id)
+{
+	for (int i = 0; scmi_std_id_table[i].name[0]; i++) {
+		if (scmi_std_id_table[i].protocol_id == id->protocol_id &&
+		    !strcmp(scmi_std_id_table[i].name, id->name))
+			return true;
+	}
+
+	return false;
+}
+
 /**
  * scmi_device_create  - A method to create one or more SCMI devices
  *
- * @np: A reference to the device node to use for the new device(s)
+ * @fwnode: A reference to the device node to use for the new device(s)
  * @parent: The parent device to use identifying a specific SCMI instance
  * @protocol: The SCMI protocol to be associated with this device
  * @name: The requested-name of the device to be created; this is optional
@@ -557,30 +583,47 @@ _scmi_device_create(struct device_node *np, struct device *parent,
  *	   could have been potentially created for a whole protocol, unless no
  *	   device was found to have been requested for that specific protocol.
  */
-struct scmi_device *scmi_device_create(struct device_node *np,
+struct scmi_device *scmi_device_create(struct fwnode_handle *fwnode,
 				       struct device *parent, int protocol,
 				       const char *name)
 {
 	struct list_head *phead;
 	struct scmi_requested_dev *rdev;
-	struct scmi_device *scmi_dev = NULL;
+	struct scmi_device *sdev, *scmi_dev = NULL;
 
 	if (name)
-		return _scmi_device_create(np, parent, protocol, name);
+		return _scmi_device_create(fwnode, parent, protocol, name);
+
+	/*
+	 * Always create devices for standard protocols, even if the device-ids
+	 * have not been registered into scmi_requested_devices yet. This allows
+	 * auto-loading of SCMI protocol driver modules for standard protocols.
+	 */
+	for (int i = 0; scmi_std_id_table[i].name[0]; i++) {
+		if (scmi_std_id_table[i].protocol_id != protocol)
+			continue;
+
+		sdev = _scmi_device_create(fwnode, parent, protocol,
+					   scmi_std_id_table[i].name);
+		if (sdev)
+			scmi_dev = sdev;
+	}
 
 	mutex_lock(&scmi_requested_devices_mtx);
 	phead = idr_find(&scmi_requested_devices, protocol);
 	/* Nothing to do. */
 	if (!phead) {
 		mutex_unlock(&scmi_requested_devices_mtx);
-		return NULL;
+		return scmi_dev;
 	}
 
 	/* Walk the list of requested devices for protocol and create them */
 	list_for_each_entry(rdev, phead, node) {
-		struct scmi_device *sdev;
+		/* Standard proto matches already have their dev created above */
+		if (scmi_device_id_in_std_id_table(rdev->id_table))
+			continue;
 
-		sdev = _scmi_device_create(np, parent,
+		sdev = _scmi_device_create(fwnode, parent,
 					   rdev->id_table->protocol_id,
 					   rdev->id_table->name);
 		if (sdev)
diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h
index fe8c22cfb9f7d1..af5b025ee0723a 100644
--- a/drivers/firmware/arm_scmi/common.h
+++ b/drivers/firmware/arm_scmi/common.h
@@ -9,6 +9,7 @@
 #ifndef _SCMI_COMMON_H
 #define _SCMI_COMMON_H
 
+#include <linux/acpi.h>
 #include <linux/bitfield.h>
 #include <linux/completion.h>
 #include <linux/device.h>
@@ -152,7 +153,7 @@ extern const struct bus_type scmi_bus_type;
 #define SCMI_BUS_NOTIFY_DEVICE_UNREQUEST	1
 extern struct blocking_notifier_head scmi_requested_devices_nh;
 
-struct scmi_device *scmi_device_create(struct device_node *np,
+struct scmi_device *scmi_device_create(struct fwnode_handle *fwnode,
 				       struct device *parent, int protocol,
 				       const char *name);
 void scmi_device_destroy(struct device *parent, int protocol, const char *name);
@@ -206,7 +207,8 @@ struct scmi_chan_info {
  * @poll_done: Callback to poll transfer status
  */
 struct scmi_transport_ops {
-	bool (*chan_available)(struct device_node *of_node, int idx);
+	bool (*chan_available)(struct fwnode_handle *fwnode, int prot_id,
+			       int idx);
 	int (*chan_setup)(struct scmi_chan_info *cinfo, struct device *dev,
 			  bool tx);
 	int (*chan_free)(int id, void *p, void *data);
@@ -232,7 +234,7 @@ struct scmi_transport_ops {
  *	be pending simultaneously in the system. May be overridden by the
  *	get_max_msg op.
  * @max_msg_size: Maximum size of data payload per message that can be handled.
- * @atomic_threshold: Optional system wide DT-configured threshold, expressed
+ * @atomic_threshold: Optional system wide fwnode-configured threshold, expressed
  *		      in microseconds, for atomic operations.
  *		      Only SCMI synchronous commands reported by the platform
  *		      to have an execution latency lesser-equal to the threshold
@@ -240,7 +242,8 @@ struct scmi_transport_ops {
  *		      decision is finally left up to the SCMI drivers.
  * @no_completion_irq: Flag to indicate that this transport has no completion
  *		       interrupt and has to be polled. This is similar to the
- *		       force_polling below, except this is set via DT property.
+ *		       force_polling below, except this is set via fwnode
+ *		       property.
  * @force_polling: Flag to force this whole transport to use SCMI core polling
  *		   mechanism instead of completion interrupts even if available.
  * @sync_cmds_completed_on_ret: Flag to indicate that the transport assures
@@ -284,6 +287,28 @@ static inline bool is_polling_enabled(struct scmi_chan_info *cinfo,
 		is_transport_polling_capable(desc);
 }
 
+/**
+ * scmi_xfer_async_response_arm  - Arm the delayed response completion
+ *
+ * @xfer: A reference to the xfer to arm
+ * @async_done: The completion to signal upon reception of a delayed response,
+ *		or NULL to disarm @xfer.
+ */
+static inline void scmi_xfer_async_response_arm(struct scmi_xfer *xfer,
+						struct completion *async_done)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&xfer->lock, flags);
+	xfer->async_done = async_done;
+	spin_unlock_irqrestore(&xfer->lock, flags);
+}
+
+static inline void scmi_xfer_async_response_disarm(struct scmi_xfer *xfer)
+{
+	scmi_xfer_async_response_arm(xfer, NULL);
+}
+
 void scmi_xfer_raw_put(const struct scmi_handle *handle,
 		       struct scmi_xfer *xfer);
 struct scmi_xfer *scmi_xfer_raw_get(const struct scmi_handle *handle);
@@ -465,6 +490,17 @@ struct scmi_transport_core_operations {
 	const struct scmi_message_operations *msg;
 };
 
+struct scmi_dsd_info {
+	u32 protocol_id;
+	const char *const property_name;
+};
+
+static const struct scmi_dsd_info scmi_dsd_info_list[] __maybe_unused = {
+	{ SCMI_PROTOCOL_BASE, "arm-arml0001-transport-pcc" },
+	{ SCMI_PROTOCOL_POWERCAP, "arm-arml0001-protocol-pcap" },
+	{ SCMI_PROTOCOL_TELEMETRY, "arm-arml0001-protocol-telemetry" },
+};
+
 /**
  * struct scmi_transport_handle  - Transport instance handle
  * @supplier_get: A helper to retrieve the device descriptor, identifying the
@@ -615,7 +651,8 @@ struct scmi_transport_supplier __supplier = {			\
 	.th.supplier_put = scmi_transport_supplier_put,		\
 }
 
-#define DEFINE_SCMI_TRANSPORT_DRIVER(__tag, __drv, __desc, __match, __core_ops)\
+#define __DEFINE_SCMI_TRANSPORT_DRIVER(__tag, __drv, __desc, __of_match,       \
+					__acpi_match, __core_ops)	       \
 static void __tag##_dev_free(void *data)				       \
 {									       \
 	struct platform_device *spdev = data;				       \
@@ -649,7 +686,10 @@ static int __tag##_probe(struct platform_device *pdev)			       \
 		goto err_mem;						       \
 	}								       \
 									       \
-	device_set_of_node_from_dev(&spdev->dev, dev);			       \
+	if (dev_of_node(dev))						       \
+		platform_device_set_of_node_from_dev(spdev, dev);	       \
+	else								       \
+		platform_device_set_fwnode(spdev, dev_fwnode(dev));	       \
 									       \
 	strans.supplier = supplier;					       \
 	memcpy(&strans.desc, &(__desc), sizeof(strans.desc));		       \
@@ -678,11 +718,18 @@ err_mem:								       \
 static struct platform_driver __drv = {					       \
 	.driver = {							       \
 		   .name = #__tag "_transport",				       \
-		   .of_match_table = __match,				       \
+		   .of_match_table = __of_match,			       \
+		   .acpi_match_table = ACPI_PTR(__acpi_match),		       \
 		   },							       \
 	.probe = __tag##_probe,						       \
 }
 
+#define DEFINE_SCMI_TRANSPORT_DRIVER(__tag, __drv, __desc, __match, __core_ops)\
+	__DEFINE_SCMI_TRANSPORT_DRIVER(__tag, __drv, __desc, __match, NULL, __core_ops)
+
+#define DEFINE_SCMI_ACPI_TRANSPORT_DRIVER(__tag, __drv, __desc, __match, __core_ops)\
+	__DEFINE_SCMI_TRANSPORT_DRIVER(__tag, __drv, __desc, NULL, __match, __core_ops)
+
 void scmi_notification_instance_data_set(const struct scmi_handle *handle,
 					 void *priv);
 void *scmi_notification_instance_data_get(const struct scmi_handle *handle);
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index a666ad39acdf14..1e6f69b1432159 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -16,6 +16,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/acpi.h>
 #include <linux/bitmap.h>
 #include <linux/cleanup.h>
 #include <linux/debugfs.h>
@@ -30,7 +31,7 @@
 #include <linux/hashtable.h>
 #include <linux/list.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/platform_device.h>
 #include <linux/processor.h>
 #include <linux/rcupdate.h>
@@ -102,7 +103,7 @@ struct scmi_xfers_info {
  *	initialization code to identify this instance.
  *
  * Each protocol is initialized independently once for each SCMI platform in
- * which is defined by DT and implemented by the SCMI server fw.
+ * which is defined by fwnode and implemented by the SCMI server fw.
  */
 struct scmi_protocol_instance {
 	const struct scmi_handle	*handle;
@@ -137,8 +138,9 @@ struct scmi_protocol_instance {
  * @protocols_imp: List of protocols implemented, currently maximum of
  *		   scmi_base_info.num_protocols elements allocated by the
  *		   base protocol
- * @active_protocols: IDR storing device_nodes for protocols actually defined
- *		      in the DT and confirmed as implemented by fw.
+ * @active_protocols: IDR storing fwnodes for protocols actually defined
+ *		      in the firmware description and confirmed as implemented
+ *		      by fw.
  * @notify_priv: Pointer to private data structure specific to notifications.
  * @node: List head
  * @users: Number of users of this instance
@@ -416,19 +418,19 @@ EXPORT_SYMBOL_GPL(scmi_protocol_unregister);
  * scmi_create_protocol_devices  - Create devices for all pending requests for
  * this SCMI instance.
  *
- * @np: The device node describing the protocol
+ * @fwnode: The firmware node describing the protocol
  * @info: The SCMI instance descriptor
  * @prot_id: The protocol ID
  * @name: The optional name of the device to be created: if not provided this
  *	  call will lead to the creation of all the devices currently requested
  *	  for the specified protocol.
  */
-static void scmi_create_protocol_devices(struct device_node *np,
+static void scmi_create_protocol_devices(struct fwnode_handle *fwnode,
 					 struct scmi_info *info,
 					 int prot_id, const char *name)
 {
 	mutex_lock(&info->devreq_mtx);
-	scmi_device_create(np, info->dev, prot_id, name);
+	scmi_device_create(fwnode, info->dev, prot_id, name);
 	mutex_unlock(&info->devreq_mtx);
 }
 
@@ -718,6 +720,7 @@ static struct scmi_xfer *scmi_xfer_get(const struct scmi_handle *handle,
 
 	refcount_set(&xfer->users, 1);
 	atomic_set(&xfer->busy, SCMI_XFER_FREE);
+	xfer->async_done = NULL;
 	spin_unlock_irqrestore(&minfo->xfer_lock, flags);
 
 	return xfer;
@@ -752,9 +755,10 @@ struct scmi_xfer *scmi_xfer_raw_get(const struct scmi_handle *handle)
  * @protocol_id: Identifier of the protocol
  *
  * Note that in a regular SCMI stack, usually, a protocol has to be defined in
- * the DT to have an associated channel and be usable; but in Raw mode any
- * protocol in range is allowed, re-using the Base channel, so as to enable
- * fuzzing on any protocol without the need of a fully compiled DT.
+ * the firmware description to have an associated channel and be usable; but in
+ * Raw mode any protocol in range is allowed, re-using the Base channel, so as
+ * to enable fuzzing on any protocol without the need of a fully compiled
+ * firmware description.
  *
  * Return: A reference to the channel to use, or an ERR_PTR
  */
@@ -768,7 +772,7 @@ scmi_xfer_raw_channel_get(const struct scmi_handle *handle, u8 protocol_id)
 	if (!cinfo) {
 		if (protocol_id == SCMI_PROTOCOL_BASE)
 			return ERR_PTR(-EINVAL);
-		/* Use Base channel for protocols not defined for DT */
+		/* Use Base channel for protocols not defined for fwnode */
 		cinfo = idr_find(&info->tx_idr, SCMI_PROTOCOL_BASE);
 		if (!cinfo)
 			return ERR_PTR(-EINVAL);
@@ -1064,6 +1068,29 @@ static inline void scmi_xfer_command_release(struct scmi_info *info,
 	__scmi_xfer_put(&info->tx_minfo, xfer);
 }
 
+/**
+ * scmi_xfer_async_response_complete  - Signal a received delayed response
+ *
+ * @xfer: A reference to the xfer whose delayed response was received
+ *
+ * Return: True if a completion was still armed on @xfer and has been
+ *	   signalled, false if a waiter has already disarmed it due to
+ *	   timeout or other error.
+ */
+static bool scmi_xfer_async_response_complete(struct scmi_xfer *xfer)
+{
+	unsigned long flags;
+	struct completion *async_done;
+
+	spin_lock_irqsave(&xfer->lock, flags);
+	async_done = xfer->async_done;
+	if (async_done)
+		complete(async_done);
+	spin_unlock_irqrestore(&xfer->lock, flags);
+
+	return !!async_done;
+}
+
 static inline void scmi_clear_channel(struct scmi_info *info,
 				      struct scmi_chan_info *cinfo)
 {
@@ -1167,8 +1194,12 @@ static void scmi_handle_response(struct scmi_chan_info *cinfo,
 
 	if (xfer->hdr.type == MSG_TYPE_DELAYED_RESP) {
 		scmi_clear_channel(info, cinfo);
-		complete(xfer->async_done);
-		scmi_inc_count(info->dbg, DELAYED_RESPONSE_OK);
+		if (scmi_xfer_async_response_complete(xfer))
+			scmi_inc_count(info->dbg, DELAYED_RESPONSE_OK);
+		else
+			dev_err(cinfo->dev,
+				"Delayed Response for %d (protocol 0x%X msg 0x%X) received after waiter was disarmed, dropping\n",
+				xfer->hdr.seq, xfer->hdr.protocol_id, xfer->hdr.id);
 	} else {
 		complete(&xfer->done);
 		scmi_inc_count(info->dbg, RESPONSE_OK);
@@ -1510,7 +1541,7 @@ static int do_xfer_with_response(const struct scmi_protocol_handle *ph,
 	int ret, timeout = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT);
 	DECLARE_COMPLETION_ONSTACK(async_response);
 
-	xfer->async_done = &async_response;
+	scmi_xfer_async_response_arm(xfer, &async_response);
 
 	/*
 	 * Delayed responses should not be polled, so an async command should
@@ -1522,7 +1553,7 @@ static int do_xfer_with_response(const struct scmi_protocol_handle *ph,
 
 	ret = do_xfer(ph, xfer);
 	if (!ret) {
-		if (!wait_for_completion_timeout(xfer->async_done, timeout)) {
+		if (!wait_for_completion_timeout(&async_response, timeout)) {
 			dev_err(ph->dev,
 				"timed out in delayed resp(caller: %pS)\n",
 				(void *)_RET_IP_);
@@ -1532,7 +1563,7 @@ static int do_xfer_with_response(const struct scmi_protocol_handle *ph,
 		}
 	}
 
-	xfer->async_done = NULL;
+	scmi_xfer_async_response_disarm(xfer);
 	return ret;
 }
 
@@ -2749,7 +2780,7 @@ static int scmi_xfer_info_init(struct scmi_info *sinfo)
 	return ret;
 }
 
-static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
+static int scmi_chan_setup(struct scmi_info *info, struct fwnode_handle *fwnode,
 			   int prot_id, bool tx)
 {
 	int ret, idx;
@@ -2765,7 +2796,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 	if (idr_find(idr, prot_id))
 		return -EEXIST;
 
-	if (!info->desc->ops->chan_available(of_node, idx)) {
+	if (!info->desc->ops->chan_available(fwnode, prot_id, idx)) {
 		cinfo = idr_find(idr, SCMI_PROTOCOL_BASE);
 		if (unlikely(!cinfo)) /* Possible only if platform has no Rx */
 			return -EINVAL;
@@ -2785,7 +2816,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 	snprintf(name, sizeof(name), SCMI_TRANSPORT_DEVNAME_PREFIX "_%s_%02X",
 		 idx ? "rx" : "tx", prot_id);
 	/* Create a uniquely named, dedicated transport device for this chan */
-	tdev = scmi_device_create(of_node, info->dev, prot_id, name);
+	tdev = scmi_device_create(fwnode, info->dev, prot_id, name);
 	if (!tdev) {
 		dev_err(info->dev,
 			"failed to create transport device (%s)\n", name);
@@ -2831,14 +2862,14 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,
 }
 
 static inline int
-scmi_txrx_setup(struct scmi_info *info, struct device_node *of_node,
+scmi_txrx_setup(struct scmi_info *info, struct fwnode_handle *fwnode,
 		int prot_id)
 {
-	int ret = scmi_chan_setup(info, of_node, prot_id, true);
+	int ret = scmi_chan_setup(info, fwnode, prot_id, true);
 
 	if (!ret) {
 		/* Rx is optional, report only memory errors */
-		ret = scmi_chan_setup(info, of_node, prot_id, false);
+		ret = scmi_chan_setup(info, fwnode, prot_id, false);
 		if (ret && ret != -ENOMEM)
 			ret = 0;
 	}
@@ -2855,44 +2886,57 @@ scmi_txrx_setup(struct scmi_info *info, struct device_node *of_node,
  *
  * @info: The SCMI instance descriptor.
  *
- * Initialize all the channels found described in the DT against the underlying
- * configured transport using custom defined dedicated devices instead of
- * borrowing devices from the SCMI drivers; this way channels are initialized
+ * Initialize all the channels found described in the fwnode against the
+ * underlying configured transport using custom defined dedicated devices instead
+ * of borrowing devices from the SCMI drivers; this way channels are initialized
  * upfront during core SCMI stack probing and are no more coupled with SCMI
  * devices used by SCMI drivers.
  *
  * Note that, even though a pair of TX/RX channels is associated to each
- * protocol defined in the DT, a distinct freshly initialized channel is
- * created only if the DT node for the protocol at hand describes a dedicated
+ * protocol defined in the fwnode, a distinct freshly initialized channel is
+ * created only if the fwnode for the protocol at hand describes a dedicated
  * channel: in all the other cases the common BASE protocol channel is reused.
  *
  * Return: 0 on Success
  */
 static int scmi_channels_setup(struct scmi_info *info)
 {
-	int ret;
-	struct device_node *top_np = info->dev->of_node;
+	int ret, idx;
+	struct fwnode_handle *fwnode = dev_fwnode(info->dev);
 
 	/* Initialize a common generic channel at first */
-	ret = scmi_txrx_setup(info, top_np, SCMI_PROTOCOL_BASE);
+	ret = scmi_txrx_setup(info, fwnode, SCMI_PROTOCOL_BASE);
 	if (ret)
 		return ret;
 
-	for_each_available_child_of_node_scoped(top_np, child) {
-		u32 prot_id;
+	if (!is_acpi_node(fwnode)) {
+		fwnode_for_each_available_child_node_scoped(fwnode, child) {
+			u32 prot_id;
 
-		if (of_property_read_u32(child, "reg", &prot_id))
-			continue;
+			if (fwnode_property_read_u32(child, "reg", &prot_id))
+				continue;
+
+			if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) {
+				dev_err(info->dev,
+					"Out of range protocol %d\n", prot_id);
+				continue;
+			}
 
-		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) {
-			dev_err(info->dev,
-				"Out of range protocol %d\n", prot_id);
-			continue;
+			ret = scmi_txrx_setup(info, child, prot_id);
+			if (ret)
+				return ret;
 		}
+	} else {
+		for (idx = 0; idx < ARRAY_SIZE(scmi_dsd_info_list); idx++) {
+			int prot_id = scmi_dsd_info_list[idx].protocol_id;
 
-		ret = scmi_txrx_setup(info, child, prot_id);
-		if (ret)
-			return ret;
+			if (prot_id == SCMI_PROTOCOL_BASE)
+				continue;
+
+			ret = scmi_txrx_setup(info, fwnode, prot_id);
+			if (ret)
+				return ret;
+		}
 	}
 
 	return 0;
@@ -2969,14 +3013,14 @@ static int scmi_bus_notifier(struct notifier_block *nb,
 static int scmi_device_request_notifier(struct notifier_block *nb,
 					unsigned long action, void *data)
 {
-	struct device_node *np;
+	struct fwnode_handle *fwnode;
 	struct scmi_device_id *id_table = data;
 	struct scmi_info *info = req_nb_to_scmi_info(nb);
 
 	rcu_read_lock();
-	np = idr_find(&info->active_protocols, id_table->protocol_id);
+	fwnode = idr_find(&info->active_protocols, id_table->protocol_id);
 	rcu_read_unlock();
-	if (!np)
+	if (!fwnode)
 		return NOTIFY_DONE;
 
 	dev_dbg(info->dev, "%sRequested device (%s) for protocol 0x%x\n",
@@ -2985,7 +3029,7 @@ static int scmi_device_request_notifier(struct notifier_block *nb,
 
 	switch (action) {
 	case SCMI_BUS_NOTIFY_DEVICE_REQUEST:
-		scmi_create_protocol_devices(np, info, id_table->protocol_id,
+		scmi_create_protocol_devices(fwnode, info, id_table->protocol_id,
 					     id_table->name);
 		break;
 	case SCMI_BUS_NOTIFY_DEVICE_UNREQUEST:
@@ -3061,6 +3105,15 @@ static void scmi_debugfs_common_cleanup(void *d)
 	kfree(dbg->type);
 }
 
+static const char *scmi_acpi_device_hid(struct acpi_device *adev)
+{
+#ifdef CONFIG_ACPI
+	return adev ? acpi_device_hid(adev) : "unknown";
+#else
+	return "unknown";
+#endif
+}
+
 static struct scmi_debug_info *scmi_debugfs_common_setup(struct scmi_info *info)
 {
 	char top_dir[16];
@@ -3072,13 +3125,16 @@ static struct scmi_debug_info *scmi_debugfs_common_setup(struct scmi_info *info)
 	if (!dbg)
 		return NULL;
 
-	dbg->name = kstrdup(of_node_full_name(info->dev->of_node), GFP_KERNEL);
+	dbg->name = kstrdup(fwnode_get_name(dev_fwnode(info->dev)), GFP_KERNEL);
 	if (!dbg->name) {
 		devm_kfree(info->dev, dbg);
 		return NULL;
 	}
 
-	of_property_read_string(info->dev->of_node, "compatible", &c_ptr);
+	if (fwnode_property_read_string(dev_fwnode(info->dev), "compatible",
+					&c_ptr))
+		c_ptr = scmi_acpi_device_hid(ACPI_COMPANION(info->dev));
+
 	dbg->type = kstrdup(c_ptr, GFP_KERNEL);
 	if (!dbg->type) {
 		kfree(dbg->name);
@@ -3184,23 +3240,23 @@ static const struct scmi_desc *scmi_transport_setup(struct device *dev)
 
 	dev_info(dev, "Using %s\n", dev_driver_string(trans->supplier));
 
-	ret = of_property_read_u32(dev->of_node, "arm,max-rx-timeout-ms",
-				   &trans->desc.max_rx_timeout_ms);
+	ret = fwnode_property_read_u32(dev_fwnode(dev), "arm,max-rx-timeout-ms",
+				       &trans->desc.max_rx_timeout_ms);
 	if (ret && ret != -EINVAL)
-		dev_err(dev, "Malformed arm,max-rx-timeout-ms DT property.\n");
+		dev_err(dev, "Malformed arm,max-rx-timeout-ms property.\n");
 
-	ret = of_property_read_u32(dev->of_node, "arm,max-msg-size",
-				   &trans->desc.max_msg_size);
+	ret = fwnode_property_read_u32(dev_fwnode(dev), "arm,max-msg-size",
+				       &trans->desc.max_msg_size);
 	if (ret && ret != -EINVAL)
-		dev_err(dev, "Malformed arm,max-msg-size DT property.\n");
+		dev_err(dev, "Malformed arm,max-msg-size property.\n");
 
-	ret = of_property_read_u32(dev->of_node, "arm,max-msg",
-				   &trans->desc.max_msg);
+	ret = fwnode_property_read_u32(dev_fwnode(dev), "arm,max-msg",
+				       &trans->desc.max_msg);
 	if (ret && ret != -EINVAL)
-		dev_err(dev, "Malformed arm,max-msg DT property.\n");
+		dev_err(dev, "Malformed arm,max-msg property.\n");
 
-	trans->desc.no_completion_irq = of_property_read_bool(dev->of_node,
-							      "arm,no-completion-irq");
+	trans->desc.no_completion_irq =
+		fwnode_property_read_bool(dev_fwnode(dev), "arm,no-completion-irq");
 
 	dev_info(dev,
 		 "SCMI max-rx-timeout: %dms / max-msg-size: %dbytes / max-msg: %d\n",
@@ -3208,8 +3264,8 @@ static const struct scmi_desc *scmi_transport_setup(struct device *dev)
 		 trans->desc.max_msg);
 
 	/* System wide atomic threshold for atomic ops .. if any */
-	if (!of_property_read_u32(dev->of_node, "atomic-threshold-us",
-				  &trans->desc.atomic_threshold))
+	if (!fwnode_property_read_u32(dev_fwnode(dev), "atomic-threshold-us",
+				      &trans->desc.atomic_threshold))
 		dev_info(dev,
 			 "SCMI System wide atomic threshold set to %u us\n",
 			 trans->desc.atomic_threshold);
@@ -3229,16 +3285,52 @@ static void scmi_enable_matching_quirks(struct scmi_info *info)
 			   rev->sub_vendor_id, rev->impl_ver);
 }
 
-static int scmi_probe(struct platform_device *pdev)
+static void scmi_device_check_create(struct fwnode_handle *fwnode, int prot_id,
+				     struct scmi_info *info, bool report_missing)
 {
 	int ret;
+	struct device *dev = info->dev;
+	struct scmi_handle *handle = &info->handle;
+
+	if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) {
+		dev_err(dev, "Out of range protocol %d\n", prot_id);
+		return;
+	}
+
+	if (!scmi_is_protocol_implemented(handle, prot_id)) {
+		if (!report_missing)
+			return;
+
+		dev_err(dev, "SCMI protocol %d not implemented\n", prot_id);
+		return;
+	}
+
+	/*
+	 * Save this valid fwnode protocol descriptor amongst
+	 * @active_protocols for this SCMI instance.
+	 */
+	ret = idr_alloc(&info->active_protocols, fwnode,
+			prot_id, prot_id + 1, GFP_KERNEL);
+	if (ret != prot_id) {
+		dev_err(dev, "SCMI protocol %d already activated. Skip\n",
+			prot_id);
+		return;
+	}
+
+	scmi_create_protocol_devices(fwnode_handle_get(fwnode), info, prot_id,
+				     NULL);
+}
+
+static int scmi_probe(struct platform_device *pdev)
+{
+	int ret, idx;
 	char *err_str = "probe failure\n";
 	struct scmi_handle *handle;
 	const struct scmi_desc *desc;
 	struct scmi_info *info;
 	bool coex = IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT_COEX);
 	struct device *dev = &pdev->dev;
-	struct device_node *child, *np = dev->of_node;
+	struct fwnode_handle *child;
 
 	desc = scmi_transport_setup(dev);
 	if (!desc) {
@@ -3277,7 +3369,7 @@ static int scmi_probe(struct platform_device *pdev)
 	handle->devm_protocol_put = scmi_devm_protocol_put;
 	handle->is_transport_atomic = scmi_is_transport_atomic;
 
-	/* Setup all channels described in the DT at first */
+	/* Setup all channels described in the fwnode at first */
 	ret = scmi_channels_setup(info);
 	if (ret) {
 		err_str = "failed to setup channels\n";
@@ -3352,37 +3444,25 @@ static int scmi_probe(struct platform_device *pdev)
 
 	scmi_enable_matching_quirks(info);
 
-	for_each_available_child_of_node(np, child) {
-		u32 prot_id;
+	if (!is_acpi_node(dev_fwnode(dev))) {
+		fwnode_for_each_available_child_node(dev_fwnode(dev), child) {
+			u32 prot_id;
 
-		if (of_property_read_u32(child, "reg", &prot_id))
-			continue;
+			if (fwnode_property_read_u32(child, "reg", &prot_id))
+				continue;
 
-		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) {
-			dev_err(dev, "Out of range protocol %d\n", prot_id);
-			continue;
+			scmi_device_check_create(child, prot_id, info, true);
 		}
+	} else {
+		for (idx = 0; idx < ARRAY_SIZE(scmi_dsd_info_list); idx++) {
+			int prot_id = scmi_dsd_info_list[idx].protocol_id;
 
-		if (!scmi_is_protocol_implemented(handle, prot_id)) {
-			dev_err(dev, "SCMI protocol %d not implemented\n",
-				prot_id);
-			continue;
-		}
+			if (prot_id == SCMI_PROTOCOL_BASE)
+				continue;
 
-		/*
-		 * Save this valid DT protocol descriptor amongst
-		 * @active_protocols for this SCMI instance/
-		 */
-		ret = idr_alloc(&info->active_protocols, child,
-				prot_id, prot_id + 1, GFP_KERNEL);
-		if (ret != prot_id) {
-			dev_err(dev, "SCMI protocol %d already activated. Skip\n",
-				prot_id);
-			continue;
+			scmi_device_check_create(dev_fwnode(dev), prot_id,
+						 info, false);
 		}
-
-		of_node_get(child);
-		scmi_create_protocol_devices(child, info, prot_id, NULL);
 	}
 
 	return 0;
@@ -3409,7 +3489,7 @@ static void scmi_remove(struct platform_device *pdev)
 {
 	int id;
 	struct scmi_info *info = platform_get_drvdata(pdev);
-	struct device_node *child;
+	struct fwnode_handle *child;
 
 	if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT))
 		scmi_raw_mode_cleanup(info->raw);
@@ -3434,7 +3514,7 @@ static void scmi_remove(struct platform_device *pdev)
 	mutex_unlock(&info->protocols_mtx);
 
 	idr_for_each_entry(&info->active_protocols, child, id)
-		of_node_put(child);
+		fwnode_handle_put(child);
 	idr_destroy(&info->active_protocols);
 
 	bus_unregister_notifier(&scmi_bus_type, &info->bus_nb);
diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h
index 15ad5162e37a90..8583159059e6b0 100644
--- a/drivers/firmware/arm_scmi/protocols.h
+++ b/drivers/firmware/arm_scmi/protocols.h
@@ -100,7 +100,10 @@ struct scmi_msg_hdr {
  *	message. If request-ACK protocol is used, we can reuse the same
  *	buffer for the rx path as we use for the tx path.
  * @done: command message transmit completion event
- * @async_done: pointer to delayed response message received event completion
+ * @async_done: pointer to delayed response message received event completion,
+ *		or NULL when no delayed response is expected. Protected by
+ *		@lock, since the completion is owned by the waiter and can
+ *		vanish once the wait times out.
  * @pending: True for xfers added to @pending_xfers hashtable
  * @node: An hlist_node reference used to store this xfer, alternatively, on
  *	  the free list @free_xfers or in the @pending_xfers hashtable
@@ -121,7 +124,7 @@ struct scmi_msg_hdr {
  *	    - SCMI_XFER_SENT_OK -> SCMI_XFER_DRESP_OK
  *	      (Missing synchronous response is assumed OK and ignored)
  * @flags: Optional flags associated to this xfer.
- * @lock: A spinlock to protect state and busy fields.
+ * @lock: A spinlock to protect state, busy and async_done fields.
  * @priv: A pointer for transport private usage.
  */
 struct scmi_xfer {
@@ -147,7 +150,7 @@ struct scmi_xfer {
 #define SCMI_XFER_IS_CHAN_SET(x)	\
 	((x)->flags & SCMI_XFER_FLAG_CHAN_SET)
 	int flags;
-	/* A lock to protect state and busy fields */
+	/* A lock to protect state, busy and async_done fields */
 	spinlock_t lock;
 	void *priv;
 };
diff --git a/drivers/firmware/arm_scmi/raw_mode.c b/drivers/firmware/arm_scmi/raw_mode.c
index 1f6e51670208d9..f01bf2f6ce1d44 100644
--- a/drivers/firmware/arm_scmi/raw_mode.c
+++ b/drivers/firmware/arm_scmi/raw_mode.c
@@ -198,6 +198,8 @@ struct scmi_raw_mode_info {
  * @async_response: A completion to be, optionally, used for async waits: it
  *		    will be setup by @scmi_do_xfer_raw_start, if needed, to be
  *		    pointed at by xfer->async_done.
+ * @async: True if @async_response was armed on @xfer, i.e. if a delayed
+ *	   response has to be waited for.
  * @node: A list node.
  */
 struct scmi_xfer_raw_waiter {
@@ -205,6 +207,7 @@ struct scmi_xfer_raw_waiter {
 	struct scmi_chan_info *cinfo;
 	struct scmi_xfer *xfer;
 	struct completion async_response;
+	bool async;
 	struct list_head node;
 };
 
@@ -346,9 +349,10 @@ scmi_xfer_raw_waiter_get(struct scmi_raw_mode_info *raw, struct scmi_xfer *xfer,
 
 		if (async) {
 			reinit_completion(&rw->async_response);
-			xfer->async_done = &rw->async_response;
+			scmi_xfer_async_response_arm(xfer, &rw->async_response);
 		}
 
+		rw->async = async;
 		rw->cinfo = cinfo;
 		rw->xfer = xfer;
 	}
@@ -361,8 +365,16 @@ static void scmi_xfer_raw_waiter_put(struct scmi_raw_mode_info *raw,
 				     struct scmi_xfer_raw_waiter *rw)
 {
 	if (rw->xfer) {
-		rw->xfer->async_done = NULL;
+		/*
+		 * Disarm the delayed response before this waiter, and its
+		 * embedded completion, can be picked up again for a new
+		 * transaction: a delayed response received late, after the
+		 * related wait timed out, must not signal a completion which
+		 * has been in the meantime re-armed on a different xfer.
+		 */
+		scmi_xfer_async_response_disarm(rw->xfer);
 		rw->xfer = NULL;
+		rw->async = false;
 	}
 
 	mutex_lock(&raw->free_mtx);
@@ -479,18 +491,23 @@ static void scmi_xfer_raw_worker(struct work_struct *work)
 				    ret, scmi_inflight_count(raw->handle));
 
 		/* Wait also for an async delayed response if needed */
-		if (!ret && xfer->async_done) {
+		if (!ret && rw->async) {
 			unsigned long tmo = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT);
 
-			if (!wait_for_completion_timeout(xfer->async_done, tmo))
+			if (!wait_for_completion_timeout(&rw->async_response, tmo))
 				dev_err(dev,
 					"timed out in RAW delayed resp - HDR:%08X\n",
 					pack_scmi_header(&xfer->hdr));
 		}
 
-		/* Release waiter and xfer */
-		scmi_xfer_raw_put(raw->handle, xfer);
+		/*
+		 * Release the waiter first: this disarms the delayed response
+		 * while we still hold a reference on the xfer, so that the xfer
+		 * cannot be recycled by a new transaction while it still points
+		 * at this waiter's completion.
+		 */
 		scmi_xfer_raw_waiter_put(raw, rw);
+		scmi_xfer_raw_put(raw->handle, xfer);
 	} while (1);
 }
 
@@ -694,6 +711,7 @@ static int scmi_raw_message_send(struct scmi_raw_mode_info *raw,
 		} else {
 			dev_err(raw->handle->dev,
 				"Failed to send RAW message - Polling NOT supported\n");
+			scmi_xfer_raw_put(raw->handle, xfer);
 			return -EINVAL;
 		}
 	}
diff --git a/drivers/firmware/arm_scmi/reset.c b/drivers/firmware/arm_scmi/reset.c
index 4bc5c24c2d72d3..2cdddc28ba47a2 100644
--- a/drivers/firmware/arm_scmi/reset.c
+++ b/drivers/firmware/arm_scmi/reset.c
@@ -60,8 +60,7 @@ struct scmi_reset_issued_notify_payld {
 struct reset_dom_info {
 	bool async_reset;
 	bool reset_notify;
-	u32 latency_us;
-	char name[SCMI_MAX_STR_SIZE];
+	struct scmi_reset_domain_info info;
 };
 
 struct scmi_reset_info {
@@ -134,10 +133,10 @@ scmi_reset_domain_attributes_get(const struct scmi_protocol_handle *ph,
 		if (pinfo->notify_reset_cmd)
 			dom_info->reset_notify =
 				SUPPORTS_NOTIFY_RESET(attributes);
-		dom_info->latency_us = le32_to_cpu(attr->latency);
-		if (dom_info->latency_us == U32_MAX)
-			dom_info->latency_us = 0;
-		strscpy(dom_info->name, attr->name, SCMI_SHORT_NAME_MAX_SIZE);
+		dom_info->info.latency_us = le32_to_cpu(attr->latency);
+		if (dom_info->info.latency_us == U32_MAX)
+			dom_info->info.latency_us = 0;
+		strscpy(dom_info->info.name, attr->name, SCMI_SHORT_NAME_MAX_SIZE);
 	}
 
 	ph->xops->xfer_put(ph, t);
@@ -149,7 +148,7 @@ scmi_reset_domain_attributes_get(const struct scmi_protocol_handle *ph,
 	if (!ret && PROTOCOL_REV_MAJOR(ph->version) >= 0x3 &&
 	    SUPPORTS_EXTENDED_NAMES(attributes))
 		ph->hops->extended_name_get(ph, RESET_DOMAIN_NAME_GET, domain,
-					    NULL, dom_info->name,
+					    NULL, dom_info->info.name,
 					    SCMI_MAX_STR_SIZE);
 
 	return ret;
@@ -162,28 +161,16 @@ static int scmi_reset_num_domains_get(const struct scmi_protocol_handle *ph)
 	return pi->num_domains;
 }
 
-static const char *
-scmi_reset_name_get(const struct scmi_protocol_handle *ph, u32 domain)
+static const struct scmi_reset_domain_info *
+scmi_reset_info_get(const struct scmi_protocol_handle *ph, u32 domain)
 {
 	struct reset_dom_info *dom_info;
 
 	dom_info = scmi_reset_domain_lookup(ph, domain);
 	if (IS_ERR(dom_info))
-		return "unknown";
-
-	return dom_info->name;
-}
-
-static int scmi_reset_latency_get(const struct scmi_protocol_handle *ph,
-				  u32 domain)
-{
-	struct reset_dom_info *dom_info;
-
-	dom_info = scmi_reset_domain_lookup(ph, domain);
-	if (IS_ERR(dom_info))
-		return PTR_ERR(dom_info);
+		return NULL;
 
-	return dom_info->latency_us;
+	return &dom_info->info;
 }
 
 static int scmi_domain_reset(const struct scmi_protocol_handle *ph, u32 domain,
@@ -241,8 +228,7 @@ scmi_reset_domain_deassert(const struct scmi_protocol_handle *ph, u32 domain)
 
 static const struct scmi_reset_proto_ops reset_proto_ops = {
 	.num_domains_get = scmi_reset_num_domains_get,
-	.name_get = scmi_reset_name_get,
-	.latency_get = scmi_reset_latency_get,
+	.info_get = scmi_reset_info_get,
 	.reset = scmi_reset_domain_reset,
 	.assert = scmi_reset_domain_assert,
 	.deassert = scmi_reset_domain_deassert,
diff --git a/drivers/firmware/arm_scmi/transports/Kconfig b/drivers/firmware/arm_scmi/transports/Kconfig
index 57eccf316e26b3..0e34fbf2505323 100644
--- a/drivers/firmware/arm_scmi/transports/Kconfig
+++ b/drivers/firmware/arm_scmi/transports/Kconfig
@@ -77,6 +77,18 @@ config ARM_SCMI_TRANSPORT_OPTEE
 	  This driver can also be built as a module. If so, the module
 	  will be called scmi_transport_optee.
 
+config ARM_SCMI_TRANSPORT_PCC
+	tristate "SCMI transport based on ACPI PCC"
+	depends on PCC
+	select ARM_SCMI_HAVE_TRANSPORT
+	help
+	  This enables the ACPI PCC mailbox based transport for SCMI.
+
+	  If you want the ARM SCMI PROTOCOL stack to include support for a
+	  transport based on ACPI PCC mailboxes, answer Y.
+	  This driver can also be built as a module. If so, the module
+	  will be called scmi_transport_pcc.
+
 config ARM_SCMI_TRANSPORT_VIRTIO
 	tristate "SCMI transport based on VirtIO"
 	depends on VIRTIO
diff --git a/drivers/firmware/arm_scmi/transports/Makefile b/drivers/firmware/arm_scmi/transports/Makefile
index 3ba3d3bee1513b..e7c4b8de6251e9 100644
--- a/drivers/firmware/arm_scmi/transports/Makefile
+++ b/drivers/firmware/arm_scmi/transports/Makefile
@@ -7,6 +7,8 @@ scmi_transport_mailbox-objs := mailbox.o
 obj-$(CONFIG_ARM_SCMI_TRANSPORT_MAILBOX) += scmi_transport_mailbox.o
 scmi_transport_optee-objs := optee.o
 obj-$(CONFIG_ARM_SCMI_TRANSPORT_OPTEE) += scmi_transport_optee.o
+scmi_transport_pcc-objs := pcc.o
+obj-$(CONFIG_ARM_SCMI_TRANSPORT_PCC) += scmi_transport_pcc.o
 scmi_transport_virtio-objs := virtio.o
 obj-$(CONFIG_ARM_SCMI_TRANSPORT_VIRTIO) += scmi_transport_virtio.o
 
diff --git a/drivers/firmware/arm_scmi/transports/mailbox.c b/drivers/firmware/arm_scmi/transports/mailbox.c
index 308736c3ead9c5..56af2cb0f42473 100644
--- a/drivers/firmware/arm_scmi/transports/mailbox.c
+++ b/drivers/firmware/arm_scmi/transports/mailbox.c
@@ -77,9 +77,14 @@ static void rx_callback(struct mbox_client *cl, void *m)
 		      core->shmem->read_header(smbox->shmem), NULL);
 }
 
-static bool mailbox_chan_available(struct device_node *of_node, int idx)
+static bool
+mailbox_chan_available(struct fwnode_handle *fwnode, int prot_id, int idx)
 {
 	int num_mb;
+	struct device_node *of_node = to_of_node(fwnode);
+
+	if (!of_node)
+		return false;
 
 	/*
 	 * Just check if bidirrectional channels are involved, and check the
diff --git a/drivers/firmware/arm_scmi/transports/optee.c b/drivers/firmware/arm_scmi/transports/optee.c
index dbe32e141748b3..6022f74d28f323 100644
--- a/drivers/firmware/arm_scmi/transports/optee.c
+++ b/drivers/firmware/arm_scmi/transports/optee.c
@@ -314,9 +314,14 @@ static int invoke_process_msg_channel(struct scmi_optee_channel *channel, size_t
 	return 0;
 }
 
-static bool scmi_optee_chan_available(struct device_node *of_node, int idx)
+static bool
+scmi_optee_chan_available(struct fwnode_handle *fwnode, int prot_id, int idx)
 {
 	u32 channel_id;
+	struct device_node *of_node = to_of_node(fwnode);
+
+	if (!of_node)
+		return false;
 
 	return !of_property_read_u32_index(of_node, "linaro,optee-channel-id",
 					   idx, &channel_id);
diff --git a/drivers/firmware/arm_scmi/transports/pcc.c b/drivers/firmware/arm_scmi/transports/pcc.c
new file mode 100644
index 00000000000000..86d4a097d8ec5f
--- /dev/null
+++ b/drivers/firmware/arm_scmi/transports/pcc.c
@@ -0,0 +1,875 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * System Control and Management Interface (SCMI) Message ACPI PCC
+ * Transport Driver
+ *
+ * This transport uses ACPI PCC (PCCT Type 3/4) subspaces via the Linux
+ * PCC mailbox controller to exchange SCMI messages over the standard
+ * SCMI Shared Memory Transport (SMT) layout.
+ *
+ * PCC subspace selection is conveyed via ACPI SCMI namespace device.
+ * described in ACPI for System Control and Management Interface Platform
+ * Design Document[1]
+ *
+ * [1] https://support.arm.com/documentation/111115/latest
+ *
+ * Copyright (C) 2026
+ */
+
+#include <linux/acpi.h>
+#include <linux/cleanup.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/hashtable.h>
+#include <linux/io.h>
+#include <linux/limits.h>
+#include <linux/list.h>
+#include <linux/mailbox_client.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <acpi/pcc.h>
+
+#include "../common.h"
+
+#define SCMI_TRANSPORT_PACKAGE_MAX_VERSION	(1)
+#define SCMI_PROTOCOL_PACKAGE_MAX_VERSION	(1)
+#define SCMI_TRANSPORT_SHARED_CHANNEL		BIT_ULL(0)
+#define SCMI_TRANSPORT_P2A_CHANNEL		BIT_ULL(1)
+#define SCMI_TRANSPORT_FLAGS_MASK	(SCMI_TRANSPORT_SHARED_CHANNEL | \
+					 SCMI_TRANSPORT_P2A_CHANNEL)
+
+#define SCMI_DSD_GUID_ENTRY_COUNT	2
+#define SCMI_PROTOCOL_MAX_TRANSPORTS	2
+
+enum scmi_dsd_property_index {
+	SCMI_DSD_PROPERTY_NAME,
+	SCMI_DSD_PROPERTY_VALUE,
+	SCMI_DSD_PROPERTY_COUNT,
+};
+
+enum scmi_pcc_transport_index {
+	SCMI_PCC_TRANSPORT_REVISION,
+	SCMI_PCC_TRANSPORT_COUNT,
+	SCMI_PCC_TRANSPORT_DESCRIPTORS,
+};
+
+enum scmi_pcc_descriptor_index {
+	SCMI_PCC_DESCRIPTOR_PCCT_INDEX,
+	SCMI_PCC_DESCRIPTOR_TRANSPORT_UID,
+	SCMI_PCC_DESCRIPTOR_FLAGS,
+	SCMI_PCC_DESCRIPTOR_COUNT,
+};
+
+enum scmi_protocol_property_index {
+	SCMI_PROTOCOL_PROPERTY_REVISION,
+	SCMI_PROTOCOL_PROPERTY_TRANSPORTS,
+	SCMI_PROTOCOL_PROPERTY_ASSOCIATIONS,
+	SCMI_PROTOCOL_PROPERTY_COUNT,
+};
+
+enum scmi_protocol_transport_index {
+	SCMI_PROTOCOL_TRANSPORT_UID,
+	SCMI_PROTOCOL_TRANSPORT_FLAGS,
+	SCMI_PROTOCOL_TRANSPORT_COUNT,
+};
+
+/*
+ * SCMI specification requires all parameters, message headers, return
+ * arguments or any protocol data to be expressed in little endian
+ * format only.
+ */
+struct pcc_shared_mem {
+	struct acpi_pcct_ext_pcc_shared_memory header;
+	u8 msg_payload[];
+};
+
+#define SCMI_PCC_MSG_HEADER_SIZE	sizeof(u32)
+#define SCMI_PCC_STATUS_SIZE		sizeof(u32)
+#define SCMI_PCC_RESPONSE_OVERHEAD	(SCMI_PCC_MSG_HEADER_SIZE + \
+					 SCMI_PCC_STATUS_SIZE)
+#define SCMI_PCC_SHMEM_OVERHEAD		(sizeof(struct pcc_shared_mem) + \
+					 SCMI_PCC_STATUS_SIZE)
+
+/**
+ * struct scmi_pcc - Structure representing a SCMI mailbox transport
+ *
+ * @cl: Mailbox Client
+ * @pchan: Transmit/Receive PCC/mailbox channel
+ * @cinfo: SCMI channel info
+ */
+struct scmi_pcc {
+	struct mbox_client cl;
+	struct pcc_mbox_chan *pchan;
+	struct scmi_chan_info *cinfo;
+};
+
+struct pcc_transport {
+	u64 uid;
+	u64 flags;
+	u32 pcc_ss_id;
+	u32 protocol_id;
+	struct hlist_node hnode;
+};
+
+struct pcc_transport_map {
+	struct fwnode_handle *fwnode;
+	struct device *dev;
+	struct list_head node;
+	DECLARE_HASHTABLE(table, ilog2(MAX_PCC_SUBSPACES / 8));
+};
+
+#define client_to_scmi_pcc(c) container_of(c, struct scmi_pcc, cl)
+
+static struct scmi_transport_core_operations *core;
+static LIST_HEAD(pcc_transport_maps);
+static DEFINE_MUTEX(pcc_transport_maps_lock);
+
+static void acpi_scmi_clear_transport_map(struct pcc_transport_map *map)
+{
+	struct pcc_transport *p;
+	struct hlist_node *tmp;
+	int idx;
+
+	hash_for_each_safe(map->table, idx, tmp, p, hnode) {
+		hash_del(&p->hnode);
+		kfree(p);
+	}
+}
+
+static void acpi_scmi_destroy_transport_map(struct pcc_transport_map *map)
+{
+	acpi_scmi_clear_transport_map(map);
+	if (map->fwnode)
+		fwnode_handle_put(map->fwnode);
+	kfree(map);
+}
+
+static void pcc_transport_map_release(void *data)
+{
+	struct pcc_transport_map *map = data;
+
+	guard(mutex)(&pcc_transport_maps_lock);
+	list_del(&map->node);
+	acpi_scmi_destroy_transport_map(map);
+}
+
+static void acpi_scmi_clear_transport_maps(void)
+{
+	struct pcc_transport_map *map, *tmp;
+
+	guard(mutex)(&pcc_transport_maps_lock);
+	list_for_each_entry_safe(map, tmp, &pcc_transport_maps, node) {
+		list_del(&map->node);
+		acpi_scmi_destroy_transport_map(map);
+	}
+}
+
+static const union acpi_object *
+acpi_scmi_pkg_elements(const union acpi_object *obj, unsigned int count)
+{
+	if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != count)
+		return NULL;
+
+	return obj->package.elements;
+}
+
+static int
+acpi_scmi_pkg_u64(const union acpi_object *pkg, unsigned int idx, u64 *val)
+{
+	const union acpi_object *elem = &pkg->package.elements[idx];
+
+	if (elem->type != ACPI_TYPE_INTEGER)
+		return -EINVAL;
+
+	*val = elem->integer.value;
+
+	return 0;
+}
+
+static int acpi_scmi_property(const union acpi_object *properties,
+			      unsigned int idx, const char **name,
+			      const union acpi_object **value)
+{
+	const union acpi_object *property;
+	const union acpi_object *elems;
+
+	property = &properties->package.elements[idx];
+	elems = acpi_scmi_pkg_elements(property, SCMI_DSD_PROPERTY_COUNT);
+	if (!elems)
+		return -EINVAL;
+	if (elems[SCMI_DSD_PROPERTY_NAME].type != ACPI_TYPE_STRING ||
+	    !elems[SCMI_DSD_PROPERTY_NAME].string.pointer)
+		return -EINVAL;
+
+	*name = elems[SCMI_DSD_PROPERTY_NAME].string.pointer;
+	*value = &elems[SCMI_DSD_PROPERTY_VALUE];
+
+	return 0;
+}
+
+static int
+acpi_scmi_dsd_parse_transport_package(struct pcc_transport_map *map,
+				      const union acpi_object *obj)
+{
+	u64 revision, pkg_cnt;
+	bool common_a2p = false, common_p2a = false;
+
+	if (obj->type != ACPI_TYPE_PACKAGE ||
+	    obj->package.count < SCMI_PCC_TRANSPORT_DESCRIPTORS ||
+	    acpi_scmi_pkg_u64(obj, SCMI_PCC_TRANSPORT_REVISION, &revision) ||
+	    acpi_scmi_pkg_u64(obj, SCMI_PCC_TRANSPORT_COUNT, &pkg_cnt))
+		return -EINVAL;
+	if (revision != SCMI_TRANSPORT_PACKAGE_MAX_VERSION)
+		return -EINVAL;
+	if (pkg_cnt != obj->package.count - SCMI_PCC_TRANSPORT_DESCRIPTORS)
+		return -EINVAL;
+
+	for (u32 idx = 0; idx < pkg_cnt; idx++) {
+		struct pcc_transport *p, *tmp;
+		union acpi_object *pack;
+		u64 pcc_ss_id, uid, flags;
+
+		pack = &obj->package.elements[idx +
+					      SCMI_PCC_TRANSPORT_DESCRIPTORS];
+		if (!acpi_scmi_pkg_elements(pack, SCMI_PCC_DESCRIPTOR_COUNT)) {
+			pr_info("Invalid transport properties pkg %u\n", idx);
+			return -EINVAL;
+		}
+		if (acpi_scmi_pkg_u64(pack, SCMI_PCC_DESCRIPTOR_PCCT_INDEX,
+				      &pcc_ss_id) ||
+		    acpi_scmi_pkg_u64(pack, SCMI_PCC_DESCRIPTOR_TRANSPORT_UID,
+				      &uid) ||
+		    acpi_scmi_pkg_u64(pack, SCMI_PCC_DESCRIPTOR_FLAGS, &flags))
+			return -EINVAL;
+		if (pcc_ss_id > INT_MAX ||
+		    flags & ~SCMI_TRANSPORT_FLAGS_MASK)
+			return -EINVAL;
+
+		hash_for_each_possible(map->table, tmp, hnode, uid) {
+			if (tmp->uid == uid) {
+				pr_info("Duplicate UID %llu\n", uid);
+				return -EEXIST;
+			}
+		}
+
+		if (flags & SCMI_TRANSPORT_SHARED_CHANNEL) {
+			if (flags & SCMI_TRANSPORT_P2A_CHANNEL) {
+				if (common_p2a)
+					return -EINVAL;
+				common_p2a = true;
+			} else {
+				if (common_a2p)
+					return -EINVAL;
+				common_a2p = true;
+			}
+		}
+
+		p = kzalloc(sizeof(*p), GFP_KERNEL);
+		if (!p)
+			return -ENOMEM;
+
+		p->uid = uid;
+		p->pcc_ss_id = pcc_ss_id;
+		p->flags = flags;
+		if (p->flags & SCMI_TRANSPORT_SHARED_CHANNEL)
+			p->protocol_id = SCMI_PROTOCOL_BASE;
+
+		hash_add(map->table, &p->hnode, uid);
+	}
+
+	if (!common_a2p)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+acpi_scmi_dsd_parse_protocol_subpackage(struct pcc_transport_map *map,
+					const union acpi_object *obj,
+					int prot_id)
+{
+	struct pcc_transport *p;
+	unsigned int pkg_cnt = obj->package.count;
+	bool found, tx_found = false, rx_found = false;
+	u64 uid;
+
+	if (pkg_cnt > SCMI_PROTOCOL_MAX_TRANSPORTS) {
+		pr_warn("Found %u protocol channels, at most %u allowed\n",
+			pkg_cnt, SCMI_PROTOCOL_MAX_TRANSPORTS);
+		return -EINVAL;
+	}
+
+	for (u32 idx = 0; idx < pkg_cnt; idx++) {
+		union acpi_object *pack = &obj->package.elements[idx];
+		u64 flags;
+
+		if (!acpi_scmi_pkg_elements(pack, SCMI_PROTOCOL_TRANSPORT_COUNT) ||
+		    acpi_scmi_pkg_u64(pack, SCMI_PROTOCOL_TRANSPORT_UID, &uid) ||
+		    acpi_scmi_pkg_u64(pack, SCMI_PROTOCOL_TRANSPORT_FLAGS,
+				      &flags))
+			return -EINVAL;
+		if (flags)
+			return -EINVAL;
+
+		found = false;
+		hash_for_each_possible(map->table, p, hnode, uid) {
+			if (p->uid != uid)
+				continue;
+
+			found = true;
+			if (p->flags & SCMI_TRANSPORT_SHARED_CHANNEL) {
+				pr_info("Protocol package references common PCC subspace %u\n",
+					p->pcc_ss_id);
+				return -EINVAL;
+			}
+			if (p->protocol_id && p->protocol_id != prot_id)
+				return -EINVAL;
+
+			if (p->flags & SCMI_TRANSPORT_P2A_CHANNEL) {
+				if (rx_found)
+					return -EINVAL;
+				rx_found = true;
+			} else {
+				if (tx_found)
+					return -EINVAL;
+				tx_found = true;
+			}
+			p->protocol_id = prot_id;
+			break;
+		}
+
+		if (!found)
+			return -ENOENT;
+	}
+
+	return 0;
+}
+
+static int
+acpi_scmi_dsd_parse_protocol_package(struct pcc_transport_map *map,
+				     const union acpi_object *obj, int prot_id)
+{
+	const union acpi_object *elems;
+	const union acpi_object *pack;
+	u64 revision;
+	int ret;
+
+	elems = acpi_scmi_pkg_elements(obj, SCMI_PROTOCOL_PROPERTY_COUNT);
+	if (!elems ||
+	    acpi_scmi_pkg_u64(obj, SCMI_PROTOCOL_PROPERTY_REVISION, &revision))
+		return -EINVAL;
+
+	pack = &elems[SCMI_PROTOCOL_PROPERTY_TRANSPORTS];
+
+	if (revision != SCMI_PROTOCOL_PACKAGE_MAX_VERSION)
+		return -EINVAL;
+
+	if (pack->type != ACPI_TYPE_PACKAGE) {
+		pr_info("Invalid protocol transport package\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Document 111115A v1.0, Sections 2.2.2.1 and 2.2.3 permit an empty
+	 * protocol transport package when common transport channels exist.
+	 */
+	if (pack->package.count != 0) {
+		ret = acpi_scmi_dsd_parse_protocol_subpackage(map, pack, prot_id);
+		if (ret)
+			return ret;
+	}
+
+	pack = &elems[SCMI_PROTOCOL_PROPERTY_ASSOCIATIONS];
+	if (pack->type != ACPI_TYPE_PACKAGE) {
+		pr_info("Invalid protocol transport association package\n");
+		return -EINVAL;
+	}
+
+	if (pack->package.count != 0) {
+		pr_info("Non-empty association package not supported\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* ACPI SCMI _DSD UUID: "84a2d1c6-86b6-4199-8dac-9c17449d5e03" */
+static const guid_t acpi_scmi_uuid = GUID_INIT(0x84a2d1c6, 0x86b6, 0x4199,
+					       0x8d, 0xac, 0x9c, 0x17,
+					       0x44, 0x9d, 0x5e, 0x03);
+
+static int acpi_scmi_lookup_protocol_id(const char *name)
+{
+	for (size_t i = 0; i < ARRAY_SIZE(scmi_dsd_info_list); i++) {
+		if (!strcmp(name, scmi_dsd_info_list[i].property_name))
+			return scmi_dsd_info_list[i].protocol_id;
+	}
+	return -ENOENT;
+}
+
+static int acpi_scmi_parse_properties(struct pcc_transport_map *map,
+				      const union acpi_object *properties)
+{
+	bool transport_found = false;
+
+	if (properties->type != ACPI_TYPE_PACKAGE)
+		return -EINVAL;
+
+	/*
+	 * Protocol properties reference TransportUIDs declared by the transport
+	 * property. Parse the transport first because _DSD property ordering is
+	 * not defined.
+	 */
+	for (u32 i = 0; i < properties->package.count; i++) {
+		const union acpi_object *v;
+		const char *name;
+		int prot_id, ret;
+
+		ret = acpi_scmi_property(properties, i, &name, &v);
+		if (ret)
+			return ret;
+
+		prot_id = acpi_scmi_lookup_protocol_id(name);
+		if (prot_id < 0)
+			continue;
+		if (prot_id != SCMI_PROTOCOL_BASE)
+			continue;
+		if (v->type != ACPI_TYPE_PACKAGE)
+			return -EINVAL;
+		if (transport_found)
+			return -EEXIST;
+
+		ret = acpi_scmi_dsd_parse_transport_package(map, v);
+		if (ret)
+			return ret;
+		transport_found = true;
+	}
+
+	if (!transport_found)
+		return -ENOENT;
+
+	for (u32 i = 0; i < properties->package.count; i++) {
+		const union acpi_object *v;
+		const char *name;
+		int prot_id, ret;
+
+		ret = acpi_scmi_property(properties, i, &name, &v);
+		if (ret)
+			return ret;
+
+		prot_id = acpi_scmi_lookup_protocol_id(name);
+		if (prot_id < 0 || prot_id == SCMI_PROTOCOL_BASE)
+			continue;
+		if (v->type != ACPI_TYPE_PACKAGE)
+			return -EINVAL;
+
+		ret = acpi_scmi_dsd_parse_protocol_package(map, v, prot_id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int acpi_scmi_namespace_fwnode_parse(struct fwnode_handle *fwnode,
+					    struct pcc_transport_map *map)
+{
+	struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER, NULL };
+	struct acpi_device *adev = to_acpi_device_node(fwnode);
+	union acpi_object *desc;
+	acpi_status status;
+	int ret = -ENOENT;
+
+	if (!adev || !adev->handle)
+		return -EINVAL;
+
+	status = acpi_evaluate_object_typed(adev->handle, "_DSD", NULL, &buf,
+					    ACPI_TYPE_PACKAGE);
+	if (ACPI_FAILURE(status))
+		return -EINVAL;
+
+	desc = buf.pointer;
+	if (desc->package.count % SCMI_DSD_GUID_ENTRY_COUNT) {
+		ret = -EINVAL;
+		goto out_free;
+	}
+
+	/* Look for the device properties GUID. */
+	for (u32 i = 0; i < desc->package.count;
+	     i += SCMI_DSD_GUID_ENTRY_COUNT) {
+		const union acpi_object *guid;
+		const union acpi_object *properties;
+
+		guid = &desc->package.elements[i];
+		properties = &desc->package.elements[i + 1];
+
+		/*
+		 * The first element must be a GUID and the second one must be
+		 * a package.
+		 */
+		if (guid->type != ACPI_TYPE_BUFFER ||
+		    guid->buffer.length != UUID_SIZE ||
+		    properties->type != ACPI_TYPE_PACKAGE)
+			continue;
+
+		if (!guid_equal((guid_t *)guid->buffer.pointer,
+				&acpi_scmi_uuid))
+			continue;
+
+		ret = acpi_scmi_parse_properties(map, properties);
+		break;
+	}
+
+out_free:
+	ACPI_FREE(buf.pointer);
+	return ret;
+}
+
+static bool
+pcc_transport_map_has_ss_id(struct pcc_transport_map *map, u32 pcc_ss_id)
+{
+	struct pcc_transport *p;
+	int idx;
+
+	hash_for_each(map->table, idx, p, hnode) {
+		if (p->pcc_ss_id == pcc_ss_id)
+			return true;
+	}
+
+	return false;
+}
+
+static int pcc_transport_map_validate(struct pcc_transport_map *map)
+{
+	struct pcc_transport_map *iter;
+	struct pcc_transport *p;
+	int idx;
+
+	list_for_each_entry(iter, &pcc_transport_maps, node) {
+		hash_for_each(map->table, idx, p, hnode) {
+			if (pcc_transport_map_has_ss_id(iter, p->pcc_ss_id))
+				return -EEXIST;
+		}
+	}
+
+	return 0;
+}
+
+static
+struct pcc_transport_map *pcc_transport_map_find(struct fwnode_handle *fwnode)
+{
+	struct pcc_transport_map *map;
+
+	list_for_each_entry(map, &pcc_transport_maps, node) {
+		if (map->fwnode == fwnode)
+			return map;
+	}
+
+	return NULL;
+}
+
+static
+struct pcc_transport_map *pcc_transport_map_get(struct fwnode_handle *fwnode)
+{
+	struct pcc_transport_map *map;
+	int ret;
+
+	map = pcc_transport_map_find(fwnode);
+	if (map)
+		return map;
+
+	map = kzalloc_obj(*map, GFP_KERNEL);
+	if (!map)
+		return ERR_PTR(-ENOMEM);
+
+	hash_init(map->table);
+	ret = acpi_scmi_namespace_fwnode_parse(fwnode, map);
+	if (ret)
+		goto err_free_map;
+
+	ret = pcc_transport_map_validate(map);
+	if (ret)
+		goto err_free_map;
+
+	map->fwnode = fwnode_handle_get(fwnode);
+	list_add_tail(&map->node, &pcc_transport_maps);
+
+	return map;
+
+err_free_map:
+	acpi_scmi_clear_transport_map(map);
+	kfree(map);
+	return ERR_PTR(ret);
+}
+
+static int pcc_transport_map_manage(struct pcc_transport_map *map,
+				    struct device *dev)
+{
+	int ret;
+
+	if (!dev || map->dev == dev)
+		return 0;
+	if (map->dev)
+		return -EBUSY;
+
+	/*
+	 * Release the cached map after the SCMI remove path tears down channels.
+	 */
+	ret = devm_add_action(dev, pcc_transport_map_release, map);
+	if (ret)
+		return ret;
+
+	map->dev = dev;
+
+	return 0;
+}
+
+static int pcc_lookup_ss_id(struct pcc_transport_map *map, u32 prot_id, bool tx)
+{
+	struct pcc_transport *p;
+	int idx;
+
+	hash_for_each(map->table, idx, p, hnode) {
+		if (p->protocol_id != prot_id)
+			continue;
+
+		if ((!tx && (p->flags & SCMI_TRANSPORT_P2A_CHANNEL)) ||
+		    (tx && !(p->flags & SCMI_TRANSPORT_P2A_CHANNEL)))
+			return p->pcc_ss_id;
+	}
+
+	return -ENOENT;
+}
+
+static int pcc_get_ss_id(struct fwnode_handle *fwnode, u32 prot_id, bool tx,
+			 struct device *dev)
+{
+	struct pcc_transport_map *map;
+	int ret;
+
+	if (!fwnode)
+		return -EINVAL;
+
+	guard(mutex)(&pcc_transport_maps_lock);
+	map = pcc_transport_map_get(fwnode);
+	if (IS_ERR(map))
+		return PTR_ERR(map);
+
+	ret = pcc_transport_map_manage(map, dev);
+	if (ret)
+		return ret;
+
+	return pcc_lookup_ss_id(map, prot_id, tx);
+}
+
+static bool
+pcc_chan_available(struct fwnode_handle *fwnode, int prot_id, int idx)
+{
+	return pcc_get_ss_id(fwnode, prot_id, !idx, NULL) >= 0;
+}
+
+static void tx_prepare(struct mbox_client *cl, void *m)
+{
+	struct scmi_pcc *smbox = client_to_scmi_pcc(cl);
+	struct pcc_shared_mem __iomem *shmem = smbox->pchan->shmem;
+	struct scmi_xfer *xfer = m;
+
+	/*
+	 * PCC take cares not to call tx_prepare until last transmit is done.
+	 * Request platform interrupt notification if available.
+	 */
+	if (xfer->tx.buf)
+		memcpy_toio(shmem->msg_payload, xfer->tx.buf, xfer->tx.len);
+	iowrite32(SCMI_PCC_MSG_HEADER_SIZE + xfer->tx.len,
+		  &shmem->header.length);
+	iowrite32(pack_scmi_header(&xfer->hdr), &shmem->header.command);
+	iowrite32(smbox->pchan->mchan->mbox->txdone_irq ?
+		  PCC_CMD_COMPLETION_NOTIFY : 0, &shmem->header.flags);
+}
+
+static void rx_callback(struct mbox_client *cl, void *m)
+{
+	struct scmi_pcc *smbox = client_to_scmi_pcc(cl);
+	struct pcc_shared_mem __iomem *shmem = smbox->pchan->shmem;
+
+	core->rx_callback(smbox->cinfo, ioread32(&shmem->header.command), NULL);
+}
+
+static int pcc_chan_validate_shmem(struct scmi_chan_info *cinfo,
+				   struct scmi_pcc *smbox, int ss_id)
+{
+	struct pcc_shared_mem __iomem *shmem = smbox->pchan->shmem;
+	u32 valid_signature = ss_id + PCC_SIGNATURE;
+
+	if (smbox->pchan->shmem_size < SCMI_PCC_SHMEM_OVERHEAD ||
+	    smbox->pchan->shmem_size - SCMI_PCC_SHMEM_OVERHEAD <
+	    cinfo->max_msg_size) {
+		dev_err(cinfo->dev, "misconfigured SCMI PCC shared memory\n");
+		return -ENOSPC;
+	}
+
+	if (ioread32(&shmem->header.signature) != valid_signature) {
+		dev_err(cinfo->dev, "invalid PCC shared memory signature\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int pcc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
+			  bool tx)
+{
+	const char *desc = tx ? "Tx" : "Rx";
+	struct device *cdev = cinfo->dev;
+	struct scmi_pcc *smbox;
+	struct mbox_client *cl;
+	int ret, ss_id;
+
+	smbox = devm_kzalloc(dev, sizeof(*smbox), GFP_KERNEL);
+	if (!smbox)
+		return -ENOMEM;
+
+	cl = &smbox->cl;
+	cl->dev = cdev;
+	cl->tx_prepare = tx ? tx_prepare : NULL;
+	cl->rx_callback = rx_callback;
+	cl->tx_block = false;
+
+	ss_id = pcc_get_ss_id(dev_fwnode(cinfo->dev), cinfo->id, tx, dev);
+	if (ss_id < 0)
+		return ss_id;
+
+	smbox->pchan = pcc_mbox_request_channel(cl, ss_id);
+	if (IS_ERR(smbox->pchan)) {
+		ret = PTR_ERR(smbox->pchan);
+		if (ret != -EPROBE_DEFER)
+			dev_err(cdev,
+				"failed to request SCMI %s mailbox\n", desc);
+		return ret;
+	}
+
+	ret = pcc_chan_validate_shmem(cinfo, smbox, ss_id);
+	if (ret) {
+		pcc_mbox_free_channel(smbox->pchan);
+		return ret;
+	}
+
+	cinfo->transport_info = smbox;
+	smbox->cinfo = cinfo;
+
+	return 0;
+}
+
+static int pcc_chan_free(int id, void *p, void *data)
+{
+	struct scmi_chan_info *cinfo = p;
+	struct scmi_pcc *smbox = cinfo->transport_info;
+
+	if (!smbox || IS_ERR_OR_NULL(smbox->pchan))
+		return 0;
+
+	pcc_mbox_free_channel(smbox->pchan);
+	cinfo->transport_info = NULL;
+	smbox->pchan = NULL;
+	smbox->cinfo = NULL;
+
+	return 0;
+}
+
+static int
+pcc_send_message(struct scmi_chan_info *cinfo, struct scmi_xfer *xfer)
+{
+	struct scmi_pcc *smbox = cinfo->transport_info;
+	int ret;
+
+	/*
+	 * The mailbox layer has its own queue. However the mailbox queue
+	 * confuses the per message SCMI timeouts since the clock starts when
+	 * the message is submitted into the mailbox queue. So when multiple
+	 * messages are queued up the clock starts on all messages instead of
+	 * only the one inflight.
+	 */
+	ret = mbox_send_message(smbox->pchan->mchan, xfer);
+	/* mbox_send_message returns non-negative value on success */
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static void pcc_fetch_response(struct scmi_chan_info *cinfo,
+			       struct scmi_xfer *xfer)
+{
+	struct scmi_pcc *smbox = cinfo->transport_info;
+	struct pcc_shared_mem __iomem *shmem = smbox->pchan->shmem;
+	size_t len = ioread32(&shmem->header.length);
+	size_t payload_len;
+
+	xfer->hdr.status = ioread32(shmem->msg_payload);
+	payload_len = len > SCMI_PCC_RESPONSE_OVERHEAD ?
+		      len - SCMI_PCC_RESPONSE_OVERHEAD : 0;
+	xfer->rx.len = min(xfer->rx.len, payload_len);
+
+	/* msg_payload starts after the header, so skip only the status here. */
+	memcpy_fromio(xfer->rx.buf, shmem->msg_payload + SCMI_PCC_STATUS_SIZE,
+		      xfer->rx.len);
+}
+
+static void pcc_fetch_notification(struct scmi_chan_info *cinfo, size_t max_len,
+				   struct scmi_xfer *xfer)
+{
+	struct scmi_pcc *smbox = cinfo->transport_info;
+	struct pcc_shared_mem __iomem *shmem = smbox->pchan->shmem;
+	size_t len = ioread32(&shmem->header.length);
+	size_t payload_len;
+
+	payload_len = len > SCMI_PCC_MSG_HEADER_SIZE ?
+		      len - SCMI_PCC_MSG_HEADER_SIZE : 0;
+	xfer->rx.len = min(max_len, payload_len);
+	memcpy_fromio(xfer->rx.buf, shmem->msg_payload, xfer->rx.len);
+}
+
+static const struct scmi_transport_ops scmi_pcc_ops = {
+	.chan_available = pcc_chan_available,
+	.chan_setup = pcc_chan_setup,
+	.chan_free = pcc_chan_free,
+	.send_message = pcc_send_message,
+	.fetch_response = pcc_fetch_response,
+	.fetch_notification = pcc_fetch_notification,
+};
+
+static struct scmi_desc scmi_pcc_desc = {
+	.ops = &scmi_pcc_ops,
+	.max_rx_timeout_ms = 30,
+	.max_msg = MBOX_TX_QUEUE_LEN,
+	.max_msg_size = SCMI_SHMEM_MAX_PAYLOAD_SIZE,
+};
+
+static const struct acpi_device_id scmi_acpi_ids[] = {
+	{ .id = "ARML0001" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(acpi, scmi_acpi_ids);
+
+DEFINE_SCMI_ACPI_TRANSPORT_DRIVER(scmi_pcc, scmi_pcc_driver,
+				  scmi_pcc_desc, scmi_acpi_ids, core);
+
+static int __init scmi_pcc_init(void)
+{
+	return platform_driver_register(&scmi_pcc_driver);
+}
+
+static void __exit scmi_pcc_exit(void)
+{
+	platform_driver_unregister(&scmi_pcc_driver);
+	acpi_scmi_clear_transport_maps();
+}
+module_init(scmi_pcc_init);
+module_exit(scmi_pcc_exit);
+
+MODULE_AUTHOR("Sudeep Holla <sudeep.holla@kernel.org>");
+MODULE_DESCRIPTION("SCMI ACPI PCC Transport driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/arm_scmi/transports/smc.c b/drivers/firmware/arm_scmi/transports/smc.c
index 1fce3ccdeb7fcb..7fb2736247a32a 100644
--- a/drivers/firmware/arm_scmi/transports/smc.c
+++ b/drivers/firmware/arm_scmi/transports/smc.c
@@ -84,10 +84,16 @@ static irqreturn_t smc_msg_done_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static bool smc_chan_available(struct device_node *of_node, int idx)
+static bool
+smc_chan_available(struct fwnode_handle *fwnode, int prot_id, int idx)
 {
+	struct device_node *of_node = to_of_node(fwnode);
+
+	if (!of_node)
+		return false;
+
 	struct device_node *np __free(device_node) =
-					of_parse_phandle(of_node, "shmem", 0);
+		of_parse_phandle(of_node, "shmem", 0);
 	if (!np)
 		return false;
 
diff --git a/drivers/firmware/arm_scmi/transports/virtio.c b/drivers/firmware/arm_scmi/transports/virtio.c
index 3282d8271839df..3d608c8c4fe861 100644
--- a/drivers/firmware/arm_scmi/transports/virtio.c
+++ b/drivers/firmware/arm_scmi/transports/virtio.c
@@ -375,7 +375,8 @@ static unsigned int virtio_get_max_msg(struct scmi_chan_info *base_cinfo)
 	return vioch->max_msg;
 }
 
-static bool virtio_chan_available(struct device_node *of_node, int idx)
+static bool
+virtio_chan_available(struct fwnode_handle *fwnode, int prot_id, int idx)
 {
 	struct scmi_vio_channel *channels, *vioch = NULL;
 
diff --git a/drivers/firmware/cirrus/cs_dsp.c b/drivers/firmware/cirrus/cs_dsp.c
index bb93b24c659dd6..04015280ac5da3 100644
--- a/drivers/firmware/cirrus/cs_dsp.c
+++ b/drivers/firmware/cirrus/cs_dsp.c
@@ -3505,7 +3505,7 @@ EXPORT_SYMBOL_NS_GPL(cs_dsp_chunk_write, "FW_CS_DSP");
  *
  * As cs_dsp_chunk_write only writes data when a whole DSP word is ready to
  * be written out it is possible that some data will remain in the cache, this
- * function will pad that data with zeros upto a whole DSP word and write out.
+ * function will pad that data with zeros up to a whole DSP word and write out.
  *
  * Return: Zero for success, a negative number on error.
  */
diff --git a/drivers/firmware/coreboot/Kconfig b/drivers/firmware/coreboot/Kconfig
new file mode 100644
index 00000000000000..d273bd836245eb
--- /dev/null
+++ b/drivers/firmware/coreboot/Kconfig
@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: GPL-2.0-only
+menuconfig COREBOOT_FIRMWARE
+	bool "coreboot firmware drivers"
+	default GOOGLE_FIRMWARE
+	default n
+	help
+	  These firmware drivers are used by devices running coreboot
+	  firmware, including Chromebooks and some Google servers.
+	  They include shared support for coreboot table entries and
+	  drivers for CBMEM, framebuffer, firmware console and VPD records.
+	  If in doubt, say "N".
+
+if COREBOOT_FIRMWARE
+
+config GOOGLE_SMI
+	tristate "SMI interface for Google platforms"
+	depends on X86 && ACPI && DMI
+	help
+	  Say Y here if you want to enable SMI callbacks for Google
+	  platforms.  This provides an interface for writing to and
+	  clearing the event log.  If CONFIG_EFI is also enabled this
+	  driver provides an interface for reading and writing NVRAM
+	  variables.
+
+config COREBOOT_CBMEM
+	tristate "CBMEM entries in sysfs"
+	depends on COREBOOT_TABLE
+	default GOOGLE_CBMEM
+	help
+	  CBMEM is a downwards-growing memory region created by the
+	  Coreboot BIOS containing tagged data structures from the
+	  BIOS.  These data structures expose things like the verified
+	  boot firmware variables, flash layout, firmware event log,
+	  and more.
+
+	  This option enables the cbmem module, which causes the
+	  kernel to search for Coreboot CBMEM entries, and expose the
+	  memory for each entry in sysfs under
+	  /sys/bus/coreboot/devices/cbmem-<id>.
+
+config COREBOOT_TABLE
+	tristate "Coreboot Table Access"
+	depends on HAS_IOMEM && (ACPI || OF)
+	default GOOGLE_COREBOOT_TABLE
+	help
+	  This option enables the coreboot_table module, which provides other
+	  firmware modules access to the coreboot table. The coreboot table
+	  pointer is accessed through the ACPI "GOOGCB00" object or the
+	  device tree node /firmware/coreboot.
+	  If unsure say N.
+
+config COREBOOT_MEMCONSOLE_COMMON
+	tristate
+	depends on GOOGLE_MEMCONSOLE_X86_LEGACY || COREBOOT_MEMCONSOLE
+	default GOOGLE_MEMCONSOLE
+
+config GOOGLE_MEMCONSOLE_X86_LEGACY
+	tristate "Firmware Memory Console - X86 Legacy support"
+	depends on X86 && ACPI && DMI
+	select COREBOOT_MEMCONSOLE_COMMON
+	help
+	  This option enables the kernel to search for a firmware log in
+	  the EBDA on Google servers.  If found, this log is exported to
+	  userland in the file /sys/firmware/log.
+	  This driver is limited to x86 systems that expose the legacy
+	  Google EBDA console format.
+
+config COREBOOT_FRAMEBUFFER
+	tristate "Coreboot Framebuffer"
+	depends on COREBOOT_TABLE
+	default GOOGLE_FRAMEBUFFER_COREBOOT
+	help
+	  This option enables the kernel to search for a framebuffer in
+	  the coreboot table.  If found, it is registered with a platform
+	  device of type coreboot-framebuffer. Using the old device of
+	  type simple-framebuffer is deprecated.
+
+config COREBOOT_MEMCONSOLE
+	tristate "Firmware Memory Console"
+	depends on COREBOOT_TABLE
+	select COREBOOT_MEMCONSOLE_COMMON
+	default GOOGLE_MEMCONSOLE_COREBOOT
+	help
+	  This option enables the kernel to search for a firmware log in
+	  the coreboot table.  If found, this log is exported to userland
+	  in the file /sys/firmware/log.
+	  This is the coreboot-table backed variant of the firmware memory
+	  console driver.
+
+config GOOGLE_VPD
+	tristate "Vital Product Data"
+	depends on COREBOOT_TABLE
+	help
+	  This option enables the kernel to expose the content of Google VPD
+	  under /sys/firmware/vpd.
+	  VPD records carry firmware-provided key/value data used by
+	  Google devices.
+
+endif # COREBOOT_FIRMWARE
+
+config GOOGLE_FIRMWARE
+	bool
+	transitional
+
+config GOOGLE_CBMEM
+	tristate
+	transitional
+
+config GOOGLE_COREBOOT_TABLE
+	tristate
+	transitional
+
+config GOOGLE_MEMCONSOLE
+	tristate
+	transitional
+
+config GOOGLE_FRAMEBUFFER_COREBOOT
+	tristate
+	transitional
+
+config GOOGLE_MEMCONSOLE_COREBOOT
+	tristate
+	transitional
diff --git a/drivers/firmware/coreboot/Makefile b/drivers/firmware/coreboot/Makefile
new file mode 100644
index 00000000000000..e681f274417abb
--- /dev/null
+++ b/drivers/firmware/coreboot/Makefile
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_GOOGLE_SMI)		+= gsmi.o
+obj-$(CONFIG_COREBOOT_TABLE)               += coreboot_table.o
+obj-$(CONFIG_COREBOOT_FRAMEBUFFER)         += framebuffer-coreboot.o
+obj-$(CONFIG_COREBOOT_MEMCONSOLE_COMMON)   += memconsole.o
+obj-$(CONFIG_COREBOOT_MEMCONSOLE)          += memconsole-coreboot.o
+obj-$(CONFIG_GOOGLE_MEMCONSOLE_X86_LEGACY) += memconsole-x86-legacy.o
+
+# Must come after coreboot_table.o, as this driver depends on that bus type.
+obj-$(CONFIG_COREBOOT_CBMEM)		+= cbmem.o
+
+vpd-sysfs-y := vpd.o vpd_decode.o
+obj-$(CONFIG_GOOGLE_VPD)		+= vpd-sysfs.o
diff --git a/drivers/firmware/google/cbmem.c b/drivers/firmware/coreboot/cbmem.c
similarity index 100%
rename from drivers/firmware/google/cbmem.c
rename to drivers/firmware/coreboot/cbmem.c
diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/coreboot/coreboot_table.c
similarity index 99%
rename from drivers/firmware/google/coreboot_table.c
rename to drivers/firmware/coreboot/coreboot_table.c
index 25ee8cf534290c..2c8c928fbb33e2 100644
--- a/drivers/firmware/google/coreboot_table.c
+++ b/drivers/firmware/coreboot/coreboot_table.c
@@ -250,8 +250,8 @@ static void coreboot_table_remove(struct platform_device *pdev)
 
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id cros_coreboot_acpi_match[] = {
-	{ "GOOGCB00", 0 },
-	{ "BOOT0000", 0 },
+	{ .id = "GOOGCB00" },
+	{ .id = "BOOT0000" },
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, cros_coreboot_acpi_match);
diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/coreboot/coreboot_table.h
similarity index 100%
rename from drivers/firmware/google/coreboot_table.h
rename to drivers/firmware/coreboot/coreboot_table.h
diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/coreboot/framebuffer-coreboot.c
similarity index 100%
rename from drivers/firmware/google/framebuffer-coreboot.c
rename to drivers/firmware/coreboot/framebuffer-coreboot.c
diff --git a/drivers/firmware/google/gsmi.c b/drivers/firmware/coreboot/gsmi.c
similarity index 100%
rename from drivers/firmware/google/gsmi.c
rename to drivers/firmware/coreboot/gsmi.c
diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/coreboot/memconsole-coreboot.c
similarity index 100%
rename from drivers/firmware/google/memconsole-coreboot.c
rename to drivers/firmware/coreboot/memconsole-coreboot.c
diff --git a/drivers/firmware/google/memconsole-x86-legacy.c b/drivers/firmware/coreboot/memconsole-x86-legacy.c
similarity index 100%
rename from drivers/firmware/google/memconsole-x86-legacy.c
rename to drivers/firmware/coreboot/memconsole-x86-legacy.c
diff --git a/drivers/firmware/google/memconsole.c b/drivers/firmware/coreboot/memconsole.c
similarity index 100%
rename from drivers/firmware/google/memconsole.c
rename to drivers/firmware/coreboot/memconsole.c
diff --git a/drivers/firmware/google/memconsole.h b/drivers/firmware/coreboot/memconsole.h
similarity index 82%
rename from drivers/firmware/google/memconsole.h
rename to drivers/firmware/coreboot/memconsole.h
index aaff2b72b606e4..58899c2abaaaa6 100644
--- a/drivers/firmware/google/memconsole.h
+++ b/drivers/firmware/coreboot/memconsole.h
@@ -7,8 +7,8 @@
  * Copyright 2017 Google Inc.
  */
 
-#ifndef __FIRMWARE_GOOGLE_MEMCONSOLE_H
-#define __FIRMWARE_GOOGLE_MEMCONSOLE_H
+#ifndef __FIRMWARE_COREBOOT_MEMCONSOLE_H
+#define __FIRMWARE_COREBOOT_MEMCONSOLE_H
 
 #include <linux/types.h>
 
@@ -33,4 +33,4 @@ int memconsole_sysfs_init(void);
  */
 void memconsole_exit(void);
 
-#endif /* __FIRMWARE_GOOGLE_MEMCONSOLE_H */
+#endif /* __FIRMWARE_COREBOOT_MEMCONSOLE_H */
diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/coreboot/vpd.c
similarity index 100%
rename from drivers/firmware/google/vpd.c
rename to drivers/firmware/coreboot/vpd.c
diff --git a/drivers/firmware/google/vpd_decode.c b/drivers/firmware/coreboot/vpd_decode.c
similarity index 100%
rename from drivers/firmware/google/vpd_decode.c
rename to drivers/firmware/coreboot/vpd_decode.c
diff --git a/drivers/firmware/google/vpd_decode.h b/drivers/firmware/coreboot/vpd_decode.h
similarity index 100%
rename from drivers/firmware/google/vpd_decode.h
rename to drivers/firmware/coreboot/vpd_decode.h
diff --git a/drivers/firmware/efi/capsule-loader.c b/drivers/firmware/efi/capsule-loader.c
index 8e8f81f0a5a0af..e423df3438da12 100644
--- a/drivers/firmware/efi/capsule-loader.c
+++ b/drivers/firmware/efi/capsule-loader.c
@@ -197,7 +197,7 @@ static ssize_t efi_capsule_write(struct file *file, const char __user *buff,
 		page = cap_info->pages[cap_info->index - 1];
 	}
 
-	kbuff = kmap(page);
+	kbuff = kmap_local_page(page);
 	kbuff += PAGE_SIZE - cap_info->page_bytes_remain;
 
 	/* Copy capsule binary data from user space to kernel space buffer */
@@ -217,7 +217,7 @@ static ssize_t efi_capsule_write(struct file *file, const char __user *buff,
 	}
 
 	cap_info->count += write_byte;
-	kunmap(page);
+	kunmap_local(kbuff);
 
 	/* Submit the full binary to efi_capsule_update() API */
 	if (cap_info->header.headersize > 0 &&
@@ -236,7 +236,7 @@ static ssize_t efi_capsule_write(struct file *file, const char __user *buff,
 	return write_byte;
 
 fail_unmap:
-	kunmap(page);
+	kunmap_local(kbuff);
 failed:
 	efi_free_all_buff_pages(cap_info);
 	return ret;
diff --git a/drivers/firmware/efi/capsule.c b/drivers/firmware/efi/capsule.c
index dd62526380d242..417e8f13a014f2 100644
--- a/drivers/firmware/efi/capsule.c
+++ b/drivers/firmware/efi/capsule.c
@@ -245,7 +245,7 @@ int efi_capsule_update(efi_capsule_header_t *capsule, phys_addr_t *pages)
 	for (i = 0; i < sg_count; i++) {
 		efi_capsule_block_desc_t *sglist;
 
-		sglist = kmap_atomic(sg_pages[i]);
+		sglist = kmap_local_page(sg_pages[i]);
 
 		for (j = 0; j < SGLIST_PER_PAGE && count > 0; j++) {
 			u64 sz = min_t(u64, imagesize,
@@ -277,7 +277,7 @@ int efi_capsule_update(efi_capsule_header_t *capsule, phys_addr_t *pages)
 		 */
 		efi_capsule_flush_cache_range(sglist, PAGE_SIZE);
 #endif
-		kunmap_atomic(sglist);
+		kunmap_local(sglist);
 	}
 
 	mutex_lock(&capsule_mutex);
diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c
index 06b4fdb59917a1..13b3e72c3b62fa 100644
--- a/drivers/firmware/efi/cper.c
+++ b/drivers/firmware/efi/cper.c
@@ -389,10 +389,28 @@ void cper_mem_err_pack(const struct cper_sec_mem_err *mem,
 	cmem->requestor_id = mem->requestor_id;
 	cmem->responder_id = mem->responder_id;
 	cmem->target_id = mem->target_id;
-	cmem->extended = mem->extended;
-	cmem->rank = mem->rank;
-	cmem->mem_array_handle = mem->mem_array_handle;
-	cmem->mem_dev_handle = mem->mem_dev_handle;
+
+	/*
+	 * These four sit past the end of the UEFI 2.1/2.2 layout, which older
+	 * firmware still emits, so reading them unconditionally runs off a
+	 * short record. Every consumer of the compact record gates them on the
+	 * same validation bits, so leave them zero when firmware does not
+	 * claim them.
+	 */
+	cmem->extended = 0;
+	cmem->rank = 0;
+	cmem->mem_array_handle = 0;
+	cmem->mem_dev_handle = 0;
+
+	if (mem->validation_bits &
+	    (CPER_MEM_VALID_ROW_EXT | CPER_MEM_VALID_CHIP_ID))
+		cmem->extended = mem->extended;
+	if (mem->validation_bits & CPER_MEM_VALID_RANK_NUMBER)
+		cmem->rank = mem->rank;
+	if (mem->validation_bits & CPER_MEM_VALID_CARD_HANDLE)
+		cmem->mem_array_handle = mem->mem_array_handle;
+	if (mem->validation_bits & CPER_MEM_VALID_MODULE_HANDLE)
+		cmem->mem_dev_handle = mem->mem_dev_handle;
 }
 EXPORT_SYMBOL_GPL(cper_mem_err_pack);
 
@@ -745,6 +763,17 @@ int cper_estatus_check_header(const struct acpi_hest_generic_status *estatus)
 	    estatus->raw_data_offset < sizeof(*estatus) + estatus->data_length)
 		return -EINVAL;
 
+	/*
+	 * cper_estatus_len() sums these into a u32, and a wrapped sum reads
+	 * back smaller than the record. Reject a length that cannot be
+	 * expressed so no caller is handed the short value.
+	 */
+	if ((u64)sizeof(*estatus) + estatus->data_length > U32_MAX)
+		return -EINVAL;
+	if (estatus->raw_data_length &&
+	    (u64)estatus->raw_data_offset + estatus->raw_data_length > U32_MAX)
+		return -EINVAL;
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(cper_estatus_check_header);
@@ -752,7 +781,7 @@ EXPORT_SYMBOL_GPL(cper_estatus_check_header);
 int cper_estatus_check(const struct acpi_hest_generic_status *estatus)
 {
 	struct acpi_hest_generic_data *gdata;
-	unsigned int data_len, record_size;
+	unsigned int data_len;
 	int rc;
 
 	rc = cper_estatus_check_header(estatus);
@@ -762,10 +791,18 @@ int cper_estatus_check(const struct acpi_hest_generic_status *estatus)
 	data_len = estatus->data_length;
 
 	apei_estatus_for_each_section(estatus, gdata) {
-		if (acpi_hest_get_size(gdata) > data_len)
+		int record_size;
+
+		/*
+		 * The <acpi/ghes.h> helpers sum these as a signed int, so a
+		 * huge error_data_length wraps small rather than large and the
+		 * walk then advances by that wrapped value. Reject a size an
+		 * int cannot carry.
+		 */
+		if (check_add_overflow(acpi_hest_get_size(gdata),
+				       gdata->error_data_length, &record_size))
 			return -EINVAL;
 
-		record_size = acpi_hest_get_record_size(gdata);
 		if (record_size > data_len)
 			return -EINVAL;
 
diff --git a/drivers/firmware/efi/efi-init.c b/drivers/firmware/efi/efi-init.c
index 6103b1a082d247..9f09b0d2daa0b9 100644
--- a/drivers/firmware/efi/efi-init.c
+++ b/drivers/firmware/efi/efi-init.c
@@ -165,10 +165,10 @@ static __init void reserve_regions(void)
 		pr_info("Processing EFI memory map:\n");
 
 	/*
-	 * Discard memblocks discovered so far except for KHO scratch
+	 * Discard memblocks discovered so far except for KHO bootmem
 	 * regions. Most memblocks at this point originate from memory nodes
 	 * in the DT and UEFI uses its own memory map instead. However, if
-	 * KHO is enabled, scratch regions, which are good known memory
+	 * KHO is enabled, bootmem regions, which are good known memory
 	 * must be preserved.
 	 */
 	memblock_dump_all();
@@ -178,7 +178,7 @@ static __init void reserve_regions(void)
 
 		/* Remove all non-KHO regions */
 		for_each_mem_region(r) {
-			if (!memblock_is_kho_scratch(r)) {
+			if (!memblock_is_kho_noprsrv(r)) {
 				memblock_remove(r->base, r->size);
 				r--;
 			}
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index 0327a39d31fa5d..e08ce813a3d4a6 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -8,7 +8,7 @@
  *
  * This code registers /sys/firmware/efi{,/efivars} when EFI is supported,
  * allowing the efivarfs to be mounted or the efivars module to be loaded.
- * The existance of /sys/firmware/efi may also be used by userspace to
+ * The existence of /sys/firmware/efi may also be used by userspace to
  * determine that the system supports EFI.
  */
 
@@ -401,6 +401,32 @@ static void __init efi_debugfs_init(void)
 static inline void efi_debugfs_init(void) {}
 #endif
 
+static ssize_t efi_runtime_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", efi_enabled(EFI_RUNTIME_SERVICES));
+}
+
+static ssize_t efi_runtime_store(struct kobject *kobj, struct kobj_attribute *attr,
+				 const char *buf, size_t count)
+{
+	int ret;
+	bool enable;
+
+	ret = kstrtobool(buf, &enable);
+	if (ret)
+		return ret;
+
+	if (efi_runtime_set_enable_flag(enable) != EFI_SUCCESS) {
+		pr_warn("unable to enable/disable efi runtime service\n");
+		return -EAGAIN;
+	}
+
+	return count;
+}
+
+static struct kobj_attribute efi_runtime_attr =
+	__ATTR(runtime_enable, 0644, efi_runtime_show, efi_runtime_store);
+
 static int __init efipostcore_init(void)
 {
 	if (!efi_enabled(EFI_RUNTIME_SERVICES))
@@ -446,6 +472,11 @@ static int __init efisubsys_init(void)
 		goto err_destroy_wq;
 	}
 
+	if (IS_ENABLED(CONFIG_PREEMPT_RT) && efi.runtime_supported_mask) {
+		if (sysfs_create_file(efi_kobj, &efi_runtime_attr.attr))
+			pr_warn("unable to register efi dynamic sysfs interface\n");
+	}
+
 	if (efi_rt_services_supported(EFI_RT_SUPPORTED_GET_VARIABLE |
 				      EFI_RT_SUPPORTED_GET_NEXT_VARIABLE_NAME)) {
 		error = generic_ops_register();
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 77a2b2d74f3f62..663271bef90eee 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -40,7 +40,7 @@ KBUILD_CFLAGS			:= $(subst $(CC_FLAGS_FTRACE),,$(cflags-y)) \
 				   -D__NO_FORTIFY \
 				   -ffreestanding \
 				   -fno-stack-protector \
-				   $(call cc-option,-fno-addrsig) \
+				   $(CONFIG_CC_OPT_NO_ADDRSIG) \
 				   -D__DISABLE_EXPORTS
 
 #
@@ -66,7 +66,8 @@ KBUILD_AFLAGS			:= $(KBUILD_CFLAGS) -D__ASSEMBLY__
 lib-y				:= efi-stub-helper.o gop.o secureboot.o tpm.o \
 				   file.o mem.o random.o randomalloc.o pci.o \
 				   skip_spaces.o lib-cmdline.o lib-ctype.o \
-				   alignedmem.o printk.o vsprintf.o
+				   alignedmem.o printk.o vsprintf.o bli.o \
+				   lib-ucs2_string.o
 
 # include the stub's libfdt dependencies from lib/ when needed
 libfdt-deps			:= fdt_rw.c fdt_ro.c fdt_wip.c fdt.c \
diff --git a/drivers/firmware/efi/libstub/bli.c b/drivers/firmware/efi/libstub/bli.c
new file mode 100644
index 00000000000000..b2407f63b743d2
--- /dev/null
+++ b/drivers/firmware/efi/libstub/bli.c
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <generated/utsrelease.h>
+
+#include <linux/efi.h>
+#include <linux/errno.h>
+#include <linux/unaligned.h>
+
+#include "efistub.h"
+
+static efi_guid_t loader_entry_guid = LINUX_EFI_LOADER_ENTRY_GUID;
+
+static const struct efi_hd_dev_path *
+efi_bli_find_hd_node(const struct efi_dev_path *path)
+{
+	const struct efi_dev_path *node;
+	u16 node_len;
+
+	for (node = path;
+	     node->header.type != EFI_DEV_END_PATH &&
+	     node->header.type != EFI_DEV_END_PATH2;
+	     node = (const void *)node + node_len) {
+		node_len = get_unaligned_le16(&node->header.length);
+
+		if (node_len < sizeof(node->header))
+			return NULL;
+
+		if (node->header.type != EFI_DEV_MEDIA ||
+		    node->header.sub_type != EFI_DEV_MEDIA_HARD_DRIVE)
+			continue;
+
+		if (node_len < sizeof(node->hd))
+			return NULL;
+
+		if (node->hd.partition_format != EFI_HD_PARTITION_FORMAT_GPT ||
+		    node->hd.signature_type != EFI_HD_SIGNATURE_TYPE_GUID)
+			continue;
+
+		return &node->hd;
+	}
+
+	return NULL;
+}
+
+static void efi_bli_populate_loader_part_uuid(const efi_loaded_image_t *image)
+{
+	static efi_guid_t device_path_guid = EFI_DEVICE_PATH_PROTOCOL_GUID;
+	efi_char16_t partuuid[UUID_STRING_LEN + 1];
+	const struct efi_hd_dev_path *hd_node;
+	const struct efi_dev_path *path;
+
+	if (efi_bs_call(handle_protocol, efi_table_attr(image, device_handle),
+			&device_path_guid, (void **)&path) != EFI_SUCCESS)
+		return;
+
+	hd_node = efi_bli_find_hd_node(path);
+	if (!hd_node)
+		return;
+
+	if (efi_snprintf(partuuid, ARRAY_SIZE(partuuid), "%pUl",
+			 &hd_node->signature) != UUID_STRING_LEN)
+		return;
+
+	set_efi_var(L"LoaderDevicePartUUID", &loader_entry_guid,
+		    EFI_VARIABLE_BOOTSERVICE_ACCESS | EFI_VARIABLE_RUNTIME_ACCESS,
+		    sizeof(partuuid), partuuid);
+}
+
+void efi_bli_set_variables(const efi_loaded_image_t *image)
+{
+	static efi_char16_t loader_info[] = L"Linux EFI stub " UTS_RELEASE;
+	unsigned long size = 0;
+
+	if (!image)
+		return;
+
+	if (get_efi_var(L"LoaderInfo", &loader_entry_guid,
+			NULL, &size, NULL) != EFI_NOT_FOUND)
+		return;
+
+	if (set_efi_var(L"LoaderInfo", &loader_entry_guid,
+			EFI_VARIABLE_BOOTSERVICE_ACCESS | EFI_VARIABLE_RUNTIME_ACCESS,
+			sizeof(loader_info), loader_info) != EFI_SUCCESS)
+		return;
+
+	efi_bli_populate_loader_part_uuid(image);
+}
diff --git a/drivers/firmware/efi/libstub/efi-stub-entry.c b/drivers/firmware/efi/libstub/efi-stub-entry.c
index aa85e910fe595e..8ecd8cd72561f1 100644
--- a/drivers/firmware/efi/libstub/efi-stub-entry.c
+++ b/drivers/firmware/efi/libstub/efi-stub-entry.c
@@ -35,13 +35,13 @@ struct sysfb_display_info *alloc_primary_display(void)
 efi_status_t __efiapi efi_pe_entry(efi_handle_t handle,
 				   efi_system_table_t *systab)
 {
+	static efi_guid_t loaded_image_proto = LOADED_IMAGE_PROTOCOL_GUID;
 	efi_loaded_image_t *image;
 	efi_status_t status;
 	unsigned long image_addr;
 	unsigned long image_size = 0;
 	/* addr/point and size pairs for memory management*/
 	char *cmdline_ptr = NULL;
-	efi_guid_t loaded_image_proto = LOADED_IMAGE_PROTOCOL_GUID;
 	unsigned long reserve_addr = 0;
 	unsigned long reserve_size = 0;
 
diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c
index f27f2e1f001997..8e43eb3f418b13 100644
--- a/drivers/firmware/efi/libstub/efi-stub-helper.c
+++ b/drivers/firmware/efi/libstub/efi-stub-helper.c
@@ -12,6 +12,7 @@
 #include <linux/efi.h>
 #include <linux/kernel.h>
 #include <linux/overflow.h>
+#include <linux/ucs2_string.h>
 #include <asm/efi.h>
 #include <asm/setup.h>
 
@@ -248,6 +249,7 @@ static efi_status_t efi_measure_tagged_event(unsigned long load_addr,
 					     unsigned long load_size,
 					     enum efistub_event_type event)
 {
+	static efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID;
 	union {
 		efi_status_t
 		(__efiapi *hash_log_extend_event)(void *, u64, efi_physical_addr_t,
@@ -257,7 +259,6 @@ static efi_status_t efi_measure_tagged_event(unsigned long load_addr,
 	struct efistub_measured_event *evt __free(efi_pool) = NULL;
 	int size = struct_size(evt, tagged_event.tagged_event_data,
 			       events[event].event_data_len);
-	efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID;
 	efi_tcg2_protocol_t *tcg2 = NULL;
 	union efistub_event ev;
 	efi_status_t status;
@@ -276,7 +277,7 @@ static efi_status_t efi_measure_tagged_event(unsigned long load_addr,
 		method.hash_log_extend_event =
 			(void *)efi_table_attr(tcg2, hash_log_extend_event);
 	} else {
-		efi_guid_t cc_guid = EFI_CC_MEASUREMENT_PROTOCOL_GUID;
+		static efi_guid_t cc_guid = EFI_CC_MEASUREMENT_PROTOCOL_GUID;
 		efi_cc_protocol_t *cc = NULL;
 
 		efi_bs_call(locate_protocol, &cc_guid, NULL, (void **)&cc);
@@ -334,81 +335,56 @@ char *efi_convert_cmdline(efi_loaded_image_t *image)
 {
 	const efi_char16_t *options = efi_table_attr(image, load_options);
 	u32 options_size = efi_table_attr(image, load_options_size);
-	int options_bytes = 0, safe_options_bytes = 0;  /* UTF-8 bytes */
-	unsigned long cmdline_addr = 0;
-	const efi_char16_t *s2;
-	bool in_quote = false;
+	unsigned long options_chars = 0;
+	unsigned long cmdline_bytes;
 	efi_status_t status;
-	u32 options_chars;
+	char *cmdline_addr;
 
 	if (options_size > 0)
 		efi_measure_tagged_event((unsigned long)options, options_size,
 					 EFISTUB_EVT_LOAD_OPTIONS);
 
 	efi_apply_loadoptions_quirk((const void **)&options, &options_size);
-	options_chars = options_size / sizeof(efi_char16_t);
-
-	if (options) {
-		s2 = options;
-		while (options_bytes < COMMAND_LINE_SIZE && options_chars--) {
-			efi_char16_t c = *s2++;
-
-			if (c < 0x80) {
-				if (c == L'\0' || c == L'\n')
-					break;
-				if (c == L'"')
-					in_quote = !in_quote;
-				else if (!in_quote && isspace((char)c))
-					safe_options_bytes = options_bytes;
-
-				options_bytes++;
-				continue;
-			}
-
-			/*
-			 * Get the number of UTF-8 bytes corresponding to a
-			 * UTF-16 character.
-			 * The first part handles everything in the BMP.
-			 */
-			options_bytes += 2 + (c >= 0x800);
-			/*
-			 * Add one more byte for valid surrogate pairs. Invalid
-			 * surrogates will be replaced with 0xfffd and take up
-			 * only 3 bytes.
-			 */
-			if ((c & 0xfc00) == 0xd800) {
-				/*
-				 * If the very last word is a high surrogate,
-				 * we must ignore it since we can't access the
-				 * low surrogate.
-				 */
-				if (!options_chars) {
-					options_bytes -= 3;
-				} else if ((*s2 & 0xfc00) == 0xdc00) {
-					options_bytes++;
-					options_chars--;
-					s2++;
-				}
-			}
-		}
-		if (options_bytes >= COMMAND_LINE_SIZE) {
-			options_bytes = safe_options_bytes;
-			efi_err("Command line is too long: truncated to %d bytes\n",
-				options_bytes);
-		}
-	}
+	if (options)
+		options_chars = ucs2_strnlen(options,
+					     options_size / sizeof(efi_char16_t));
 
-	options_bytes++;	/* NUL termination */
+	/* Each UCS-2 char takes up at most 3 UTF-8 bytes */
+	cmdline_bytes = min(3 * options_chars, COMMAND_LINE_SIZE - 1) + 3;
 
-	status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, options_bytes,
+	status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, cmdline_bytes,
 			     (void **)&cmdline_addr);
 	if (status != EFI_SUCCESS)
 		return NULL;
 
-	snprintf((char *)cmdline_addr, options_bytes, "%.*ls",
-		 options_bytes - 1, options);
+	if (ucs2_as_utf8_l(cmdline_addr, options, options_chars,
+			   cmdline_bytes) >= COMMAND_LINE_SIZE) {
+		/*
+		 * The output fills up the entire buffer, and may have been
+		 * truncated. Work backwards through the buffer to find a safe
+		 * truncation point (i.e., a blank character not inside a
+		 * quoted string).
+		 */
+		int safe_pos[2] = {};
+		int in_quote = 0;
+
+		for (int i = COMMAND_LINE_SIZE - 1; i >= 0; i--) {
+			char c = cmdline_addr[i];
+
+			if (!c)
+				return cmdline_addr;
+			else if (c == '"')
+				in_quote ^= 1;
+			else if (!safe_pos[in_quote] && isspace(c))
+				safe_pos[in_quote] = i;
+		}
+
+		efi_err("Command line is too long: truncated to %d bytes\n",
+			safe_pos[in_quote]);
+		cmdline_addr[safe_pos[in_quote]] = '\0';
+	}
 
-	return (char *)cmdline_addr;
+	return cmdline_addr;
 }
 
 /**
@@ -552,7 +528,7 @@ static
 efi_status_t efi_load_initrd_dev_path(struct linux_efi_initrd *initrd,
 				      unsigned long max)
 {
-	efi_guid_t lf2_proto_guid = EFI_LOAD_FILE2_PROTOCOL_GUID;
+	static efi_guid_t lf2_proto_guid = EFI_LOAD_FILE2_PROTOCOL_GUID;
 	efi_device_path_protocol_t *dp;
 	efi_load_file2_protocol_t *lf2;
 	efi_handle_t handle;
@@ -614,7 +590,7 @@ efi_status_t efi_load_initrd(efi_loaded_image_t *image,
 			     unsigned long hard_limit,
 			     const struct linux_efi_initrd **out)
 {
-	efi_guid_t tbl_guid = LINUX_EFI_INITRD_MEDIA_GUID;
+	static efi_guid_t tbl_guid = LINUX_EFI_INITRD_MEDIA_GUID;
 	efi_status_t status = EFI_SUCCESS;
 	struct linux_efi_initrd initrd, *tbl;
 
@@ -725,7 +701,7 @@ efi_status_t efi_wait_for_key(unsigned long usec, efi_input_key_t *key)
 void efi_remap_image(unsigned long image_base, unsigned alloc_size,
 		     unsigned long code_size)
 {
-	efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID;
+	static efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID;
 	efi_memory_attribute_protocol_t *memattr;
 	efi_status_t status;
 	u64 attr;
diff --git a/drivers/firmware/efi/libstub/efi-stub.c b/drivers/firmware/efi/libstub/efi-stub.c
index 42d6073bcd062a..975148f986bf84 100644
--- a/drivers/firmware/efi/libstub/efi-stub.c
+++ b/drivers/firmware/efi/libstub/efi-stub.c
@@ -79,8 +79,8 @@ static struct sysfb_display_info *setup_primary_display(void)
 
 static void install_memreserve_table(void)
 {
+	static efi_guid_t memreserve_table_guid = LINUX_EFI_MEMRESERVE_TABLE_GUID;
 	struct linux_efi_memreserve *rsv;
-	efi_guid_t memreserve_table_guid = LINUX_EFI_MEMRESERVE_TABLE_GUID;
 	efi_status_t status;
 
 	status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, sizeof(*rsv),
@@ -165,6 +165,7 @@ efi_status_t efi_stub_common(efi_handle_t handle,
 	dpy = setup_primary_display();
 
 	efi_retrieve_eventlog();
+	efi_bli_set_variables(image);
 
 	/* Ask the firmware to clear memory on unclean shutdown */
 	efi_enable_reset_attack_mitigation();
diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h
index fd91fc15ec810b..f5eec8c12894c0 100644
--- a/drivers/firmware/efi/libstub/efistub.h
+++ b/drivers/firmware/efi/libstub/efistub.h
@@ -224,8 +224,8 @@ typedef void (__efiapi *efi_event_notify_t)(efi_event_t, void *);
  * efi_set_event_at() - add event to events array
  *
  * @events:	array of UEFI events
- * @ids:	index where to put the event in the array
- * @event:	event to add to the aray
+ * @idx:	index where to put the event in the array
+ * @event:	event to add to the array
  *
  * boottime->wait_for_event() takes an array of events as input.
  * Provide a helper to set it up correctly for mixed mode.
@@ -1072,17 +1072,21 @@ efi_status_t efi_random_alloc(unsigned long size, unsigned long align,
 			      int memory_type, unsigned long alloc_min,
 			      unsigned long alloc_max);
 
+void efi_bli_set_variables(const efi_loaded_image_t *image);
+
 efi_status_t efi_random_get_seed(void);
 
 efi_status_t check_platform_features(void);
 
 void *get_efi_config_table(efi_guid_t guid);
 
-/* NOTE: These functions do not print a trailing newline after the string */
 void efi_char16_puts(efi_char16_t *);
-void efi_puts(const char *str);
+
+int efi_vsnprintf(efi_char16_t *buf, size_t size, const char *fmt, va_list ap,
+		  bool crlf);
 
 __printf(1, 2) int efi_printk(char const *fmt, ...);
+__printf(3, 4) int efi_snprintf(efi_char16_t *buf, size_t size, const char *fmt, ...);
 
 void efi_free(unsigned long size, unsigned long addr);
 DEFINE_FREE(efi_pool, void *, if (_T) efi_bs_call(free_pool, _T));
diff --git a/drivers/firmware/efi/libstub/file.c b/drivers/firmware/efi/libstub/file.c
index bd626d55dcbc2f..b2601e28469593 100644
--- a/drivers/firmware/efi/libstub/file.c
+++ b/drivers/firmware/efi/libstub/file.c
@@ -29,6 +29,8 @@
  */
 #define EFI_READ_CHUNK_SIZE	SZ_1M
 
+static efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID;
+
 struct finfo {
 	efi_file_info_t info;
 	efi_char16_t	filename[MAX_FILENAME_SIZE];
@@ -39,7 +41,7 @@ static efi_status_t efi_open_file(efi_file_protocol_t *volume,
 				  efi_file_protocol_t **handle,
 				  unsigned long *file_size)
 {
-	efi_guid_t info_guid = EFI_FILE_INFO_ID;
+	static efi_guid_t info_guid = EFI_FILE_INFO_ID;
 	efi_file_protocol_t *fh;
 	unsigned long info_sz;
 	efi_status_t status;
@@ -74,7 +76,6 @@ static efi_status_t efi_open_file(efi_file_protocol_t *volume,
 static efi_status_t efi_open_volume(efi_loaded_image_t *image,
 				    efi_file_protocol_t **fh)
 {
-	efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID;
 	efi_simple_file_system_protocol_t *io;
 	efi_status_t status;
 
@@ -128,9 +129,8 @@ static int find_file_option(const efi_char16_t *cmdline, int cmdline_len,
 static efi_status_t efi_open_device_path(efi_file_protocol_t **volume,
 					 struct finfo *fi)
 {
-	efi_guid_t text_to_dp_guid = EFI_DEVICE_PATH_FROM_TEXT_PROTOCOL_GUID;
+	static efi_guid_t text_to_dp_guid = EFI_DEVICE_PATH_FROM_TEXT_PROTOCOL_GUID;
 	static efi_device_path_from_text_protocol_t *text_to_dp = NULL;
-	efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID;
 	efi_device_path_protocol_t *initrd_dp;
 	efi_simple_file_system_protocol_t *io;
 	struct efi_file_path_dev_path *fpath;
diff --git a/drivers/firmware/efi/libstub/gop.c b/drivers/firmware/efi/libstub/gop.c
index 80dc8cfeb33e96..b800a6c2290cfa 100644
--- a/drivers/firmware/efi/libstub/gop.c
+++ b/drivers/firmware/efi/libstub/gop.c
@@ -24,6 +24,8 @@ enum efi_cmdline_option {
 	EFI_CMDLINE_LIST
 };
 
+static efi_guid_t graphics_output_guid = EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID;
+
 static struct {
 	enum efi_cmdline_option option;
 	union {
@@ -309,12 +311,12 @@ static u32 choose_mode_list(efi_graphics_output_protocol_t *gop)
 	efi_status_t status;
 
 	efi_printk("Available graphics modes are 0-%u\n", max_mode-1);
-	efi_puts("  * = current mode\n"
-		 "  - = unusable mode\n");
+	efi_char16_puts(L"  * = current mode\r\n"
+			 "  - = unusable mode\r\n");
 
 	choose_mode(gop, match_list, (void *)cur_mode);
 
-	efi_puts("\nPress any key to continue (or wait 10 seconds)\n");
+	efi_char16_puts(L"\r\nPress any key to continue (or wait 10 seconds)\r\n");
 	status = efi_wait_for_key(10 * EFI_USEC_PER_SEC, &key);
 	if (status != EFI_SUCCESS && status != EFI_TIMEOUT) {
 		efi_err("Unable to read key, continuing in 10 seconds\n");
@@ -425,6 +427,7 @@ static void setup_edid_info(struct edid_info *edid, u32 gop_size_of_edid, u8 *go
 static efi_handle_t find_handle_with_primary_gop(unsigned long num, const efi_handle_t handles[],
 						 efi_graphics_output_protocol_t **found_gop)
 {
+	static efi_guid_t console_out_device_guid = EFI_CONSOLE_OUT_DEVICE_GUID;
 	efi_graphics_output_protocol_t *first_gop;
 	efi_handle_t h, first_gop_handle;
 
@@ -439,8 +442,7 @@ static efi_handle_t find_handle_with_primary_gop(unsigned long num, const efi_ha
 		efi_graphics_output_mode_info_t *info;
 		void *dummy = NULL;
 
-		status = efi_bs_call(handle_protocol, h,
-				     &EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID,
+		status = efi_bs_call(handle_protocol, h, &graphics_output_guid,
 				     (void **)&gop);
 		if (status != EFI_SUCCESS)
 			continue;
@@ -462,7 +464,7 @@ static efi_handle_t find_handle_with_primary_gop(unsigned long num, const efi_ha
 		 * don't bother looking any further.
 		 */
 		status = efi_bs_call(handle_protocol, h,
-				     &EFI_CONSOLE_OUT_DEVICE_GUID, &dummy);
+				     &console_out_device_guid, &dummy);
 		if (status == EFI_SUCCESS) {
 			if (found_gop)
 				*found_gop = gop;
@@ -480,6 +482,8 @@ static efi_handle_t find_handle_with_primary_gop(unsigned long num, const efi_ha
 
 efi_status_t efi_setup_graphics(struct screen_info *si, struct edid_info *edid)
 {
+	static efi_guid_t edid_active_guid = EFI_EDID_ACTIVE_PROTOCOL_GUID;
+	static efi_guid_t edid_discovered_guid = EFI_EDID_DISCOVERED_PROTOCOL_GUID;
 	efi_handle_t *handles __free(efi_pool) = NULL;
 	efi_handle_t handle;
 	efi_graphics_output_protocol_t *gop;
@@ -487,8 +491,7 @@ efi_status_t efi_setup_graphics(struct screen_info *si, struct edid_info *edid)
 	unsigned long num;
 
 	status = efi_bs_call(locate_handle_buffer, EFI_LOCATE_BY_PROTOCOL,
-			      &EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID, NULL, &num,
-			      &handles);
+			      &graphics_output_guid, NULL, &num, &handles);
 	if (status != EFI_SUCCESS)
 		return status;
 
@@ -510,14 +513,14 @@ efi_status_t efi_setup_graphics(struct screen_info *si, struct edid_info *edid)
 		u32 gop_size_of_edid = 0;
 		u8 *gop_edid = NULL;
 
-		status = efi_bs_call(handle_protocol, handle, &EFI_EDID_ACTIVE_PROTOCOL_GUID,
+		status = efi_bs_call(handle_protocol, handle, &edid_active_guid,
 				     (void **)&active_edid);
 		if (status == EFI_SUCCESS) {
 			gop_size_of_edid = efi_table_attr(active_edid, size_of_edid);
 			gop_edid = efi_table_attr(active_edid, edid);
 		} else {
 			status = efi_bs_call(handle_protocol, handle,
-					     &EFI_EDID_DISCOVERED_PROTOCOL_GUID,
+					     &edid_discovered_guid,
 					     (void **)&discovered_edid);
 			if (status == EFI_SUCCESS) {
 				gop_size_of_edid = efi_table_attr(discovered_edid, size_of_edid);
diff --git a/drivers/firmware/efi/libstub/kaslr.c b/drivers/firmware/efi/libstub/kaslr.c
index 4bc963e999eb97..8f8dc335b3b3fa 100644
--- a/drivers/firmware/efi/libstub/kaslr.c
+++ b/drivers/firmware/efi/libstub/kaslr.c
@@ -18,7 +18,7 @@
  */
 u32 efi_kaslr_get_phys_seed(efi_handle_t image_handle)
 {
-	efi_guid_t li_fixed_proto = LINUX_EFI_LOADED_IMAGE_FIXED_GUID;
+	static efi_guid_t li_fixed_proto = LINUX_EFI_LOADED_IMAGE_FIXED_GUID;
 	void *p;
 
 	if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
@@ -59,8 +59,7 @@ static bool check_image_region(u64 base, u64 size)
 {
 	struct efi_boot_memmap *map __free(efi_pool) = NULL;
 	efi_status_t status;
-	bool ret = false;
-	int map_offset;
+	unsigned long map_offset;
 
 	status = efi_get_memory_map(&map, false);
 	if (status != EFI_SUCCESS)
@@ -74,18 +73,16 @@ static bool check_image_region(u64 base, u64 size)
 		 * Find the region that covers base, and return whether
 		 * it covers base+size bytes.
 		 */
-		if (base >= md->phys_addr && base < end) {
-			ret = (base + size) <= end;
-			break;
-		}
+		if (base >= md->phys_addr && base < end)
+			return (base + size) <= end;
 	}
 
-	return ret;
+	return false;
 }
 
 /**
  * efi_kaslr_relocate_kernel() - Relocate the kernel (random if KASLR enabled)
- * @image_addr: Pointer to the current kernel location
+ * @image_addr:		Pointer to the current kernel location
  * @reserve_addr:	Pointer to the relocated kernel location
  * @reserve_size:	Size of the relocated kernel
  * @kernel_size:	Size of the text + data
@@ -94,8 +91,8 @@ static bool check_image_region(u64 base, u64 size)
  * @phys_seed:		Random seed used for the relocation
  *
  * If KASLR is not enabled, this function relocates the kernel to a fixed
- * address (or leave it as its current location). If KASLR is enabled, the
- * kernel physical location is randomized using the seed in parameter.
+ * address or leaves it at its current location. If KASLR is enabled, the
+ * kernel's physical location is randomized using the supplied seed.
  *
  * Return:	status code, EFI_SUCCESS if relocation is successful
  */
diff --git a/drivers/firmware/efi/libstub/mem.c b/drivers/firmware/efi/libstub/mem.c
index 59f3f83de50c2d..fec561e3a7924f 100644
--- a/drivers/firmware/efi/libstub/mem.c
+++ b/drivers/firmware/efi/libstub/mem.c
@@ -20,10 +20,10 @@
 efi_status_t efi_get_memory_map(struct efi_boot_memmap **map,
 				bool install_cfg_tbl)
 {
+	static efi_guid_t tbl_guid = LINUX_EFI_BOOT_MEMMAP_GUID;
 	struct efi_boot_memmap tmp, *m __free(efi_pool) = NULL;
 	int memtype = install_cfg_tbl ? EFI_ACPI_RECLAIM_MEMORY
 				      : EFI_LOADER_DATA;
-	efi_guid_t tbl_guid = LINUX_EFI_BOOT_MEMMAP_GUID;
 	efi_status_t status;
 	unsigned long size;
 
diff --git a/drivers/firmware/efi/libstub/pci.c b/drivers/firmware/efi/libstub/pci.c
index 1dccf77958d3a0..5daa7a0a0e87a4 100644
--- a/drivers/firmware/efi/libstub/pci.c
+++ b/drivers/firmware/efi/libstub/pci.c
@@ -15,7 +15,7 @@
 
 void efi_pci_disable_bridge_busmaster(void)
 {
-	efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
+	static efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
 	efi_handle_t *pci_handle __free(efi_pool) = NULL;
 	unsigned long pci_handle_num;
 	efi_handle_t handle;
diff --git a/drivers/firmware/efi/libstub/printk.c b/drivers/firmware/efi/libstub/printk.c
index bc599212c05dd7..0a18cfe325283e 100644
--- a/drivers/firmware/efi/libstub/printk.c
+++ b/drivers/firmware/efi/libstub/printk.c
@@ -23,98 +23,20 @@ void efi_char16_puts(efi_char16_t *str)
 		       output_string, str);
 }
 
-static
-u32 utf8_to_utf32(const u8 **s8)
-{
-	u32 c32;
-	u8 c0, cx;
-	size_t clen, i;
-
-	c0 = cx = *(*s8)++;
-	/*
-	 * The position of the most-significant 0 bit gives us the length of
-	 * a multi-octet encoding.
-	 */
-	for (clen = 0; cx & 0x80; ++clen)
-		cx <<= 1;
-	/*
-	 * If the 0 bit is in position 8, this is a valid single-octet
-	 * encoding. If the 0 bit is in position 7 or positions 1-3, the
-	 * encoding is invalid.
-	 * In either case, we just return the first octet.
-	 */
-	if (clen < 2 || clen > 4)
-		return c0;
-	/* Get the bits from the first octet. */
-	c32 = cx >> clen--;
-	for (i = 0; i < clen; ++i) {
-		/* Trailing octets must have 10 in most significant bits. */
-		cx = (*s8)[i] ^ 0x80;
-		if (cx & 0xc0)
-			return c0;
-		c32 = (c32 << 6) | cx;
-	}
-	/*
-	 * Check for validity:
-	 * - The character must be in the Unicode range.
-	 * - It must not be a surrogate.
-	 * - It must be encoded using the correct number of octets.
-	 */
-	if (c32 > 0x10ffff ||
-	    (c32 & 0xf800) == 0xd800 ||
-	    clen != (c32 >= 0x80) + (c32 >= 0x800) + (c32 >= 0x10000))
-		return c0;
-	*s8 += clen;
-	return c32;
-}
-
-/**
- * efi_puts() - Write a UTF-8 encoded string to the console
- * @str:	UTF-8 encoded string
- */
-void efi_puts(const char *str)
-{
-	efi_char16_t buf[128];
-	size_t pos = 0, lim = ARRAY_SIZE(buf);
-	const u8 *s8 = (const u8 *)str;
-	u32 c32;
-
-	while (*s8) {
-		if (*s8 == '\n')
-			buf[pos++] = L'\r';
-		c32 = utf8_to_utf32(&s8);
-		if (c32 < 0x10000) {
-			/* Characters in plane 0 use a single word. */
-			buf[pos++] = c32;
-		} else {
-			/*
-			 * Characters in other planes encode into a surrogate
-			 * pair.
-			 */
-			buf[pos++] = (0xd800 - (0x10000 >> 10)) + (c32 >> 10);
-			buf[pos++] = 0xdc00 + (c32 & 0x3ff);
-		}
-		if (*s8 == '\0' || pos >= lim - 2) {
-			buf[pos] = L'\0';
-			efi_char16_puts(buf);
-			pos = 0;
-		}
-	}
-}
-
 /**
  * efi_printk() - Print a kernel message
  * @fmt:	format string
  *
  * The first letter of the format string is used to determine the logging level
  * of the message. If the level is less then the current EFI logging level, the
- * message is suppressed. The message will be truncated to 255 bytes.
+ * message is suppressed. The message will be truncated to 255 characters
+ * (ignoring surrogates).
  *
  * Return:	number of printed characters
  */
 int efi_printk(const char *fmt, ...)
 {
-	char printf_buf[256];
+	efi_char16_t printf_buf[256];
 	va_list args;
 	int printed;
 	int loglevel = printk_get_level(fmt);
@@ -136,17 +58,18 @@ int efi_printk(const char *fmt, ...)
 		return 0;
 
 	if (loglevel >= 0)
-		efi_puts("EFI stub: ");
+		efi_char16_puts(L"EFI stub: ");
 
 	fmt = printk_skip_level(fmt);
 
 	va_start(args, fmt);
-	printed = vsnprintf(printf_buf, sizeof(printf_buf), fmt, args);
+	printed = efi_vsnprintf(printf_buf, ARRAY_SIZE(printf_buf), fmt, args,
+				true);
 	va_end(args);
 
-	efi_puts(printf_buf);
-	if (printed >= sizeof(printf_buf)) {
-		efi_puts("[Message truncated]\n");
+	efi_char16_puts(printf_buf);
+	if (printed >= ARRAY_SIZE(printf_buf)) {
+		efi_char16_puts(L"[Message truncated]\r\n");
 		return -1;
 	}
 
diff --git a/drivers/firmware/efi/libstub/random.c b/drivers/firmware/efi/libstub/random.c
index 7109b8a2dcba81..d63262d36e47a7 100644
--- a/drivers/firmware/efi/libstub/random.c
+++ b/drivers/firmware/efi/libstub/random.c
@@ -8,6 +8,8 @@
 
 #include "efistub.h"
 
+static efi_guid_t rng_proto = EFI_RNG_PROTOCOL_GUID;
+
 typedef union efi_rng_protocol efi_rng_protocol_t;
 
 union efi_rng_protocol {
@@ -38,7 +40,6 @@ union efi_rng_protocol {
  */
 efi_status_t efi_get_random_bytes(unsigned long size, u8 *out)
 {
-	efi_guid_t rng_proto = EFI_RNG_PROTOCOL_GUID;
 	efi_status_t status;
 	efi_rng_protocol_t *rng = NULL;
 
@@ -64,9 +65,8 @@ efi_status_t efi_get_random_bytes(unsigned long size, u8 *out)
  */
 efi_status_t efi_random_get_seed(void)
 {
-	efi_guid_t rng_proto = EFI_RNG_PROTOCOL_GUID;
-	efi_guid_t rng_algo_raw = EFI_RNG_ALGORITHM_RAW;
-	efi_guid_t rng_table_guid = LINUX_EFI_RANDOM_SEED_TABLE_GUID;
+	static efi_guid_t rng_algo_raw = EFI_RNG_ALGORITHM_RAW;
+	static efi_guid_t rng_table_guid = LINUX_EFI_RANDOM_SEED_TABLE_GUID;
 	struct linux_efi_random_seed *prev_seed, *seed = NULL;
 	int prev_seed_size = 0, seed_size = EFI_RANDOM_SEED_SIZE;
 	unsigned long nv_seed_size = 0, offset = 0;
diff --git a/drivers/firmware/efi/libstub/riscv.c b/drivers/firmware/efi/libstub/riscv.c
index f66f33ceb99e30..49db47c05bfe98 100644
--- a/drivers/firmware/efi/libstub/riscv.c
+++ b/drivers/firmware/efi/libstub/riscv.c
@@ -45,7 +45,7 @@ static int get_boot_hartid_from_fdt(void)
 
 static efi_status_t get_boot_hartid_from_efi(void)
 {
-	efi_guid_t boot_protocol_guid = RISCV_EFI_BOOT_PROTOCOL_GUID;
+	static efi_guid_t boot_protocol_guid = RISCV_EFI_BOOT_PROTOCOL_GUID;
 	struct riscv_efi_boot_protocol *boot_protocol;
 	efi_status_t status;
 
diff --git a/drivers/firmware/efi/libstub/smbios.c b/drivers/firmware/efi/libstub/smbios.c
index f31410d7e7e180..efbbfc3c2c0dc0 100644
--- a/drivers/firmware/efi/libstub/smbios.c
+++ b/drivers/firmware/efi/libstub/smbios.c
@@ -35,12 +35,13 @@ union efi_smbios_protocol {
 
 const struct efi_smbios_record *efi_get_smbios_record(u8 type)
 {
+	static efi_guid_t smbios_guid = EFI_SMBIOS_PROTOCOL_GUID;
 	struct efi_smbios_record *record;
 	efi_smbios_protocol_t *smbios;
 	efi_status_t status;
 	u16 handle = 0xfffe;
 
-	status = efi_bs_call(locate_protocol, &EFI_SMBIOS_PROTOCOL_GUID, NULL,
+	status = efi_bs_call(locate_protocol, &smbios_guid, NULL,
 			     (void **)&smbios) ?:
 		 efi_call_proto(smbios, get_next, &handle, &type, &record, NULL);
 	if (status != EFI_SUCCESS)
diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c
index a5c6c4f163fce4..73f0011147323d 100644
--- a/drivers/firmware/efi/libstub/tpm.c
+++ b/drivers/firmware/efi/libstub/tpm.c
@@ -28,8 +28,8 @@ static const efi_char16_t efi_MemoryOverWriteRequest_name[] =
  */
 void efi_enable_reset_attack_mitigation(void)
 {
+	static efi_guid_t var_guid = MEMORY_ONLY_RESET_CONTROL_GUID;
 	u8 val = 1;
-	efi_guid_t var_guid = MEMORY_ONLY_RESET_CONTROL_GUID;
 	efi_status_t status;
 	unsigned long datasize = 0;
 
@@ -52,7 +52,7 @@ static void efi_retrieve_tcg2_eventlog(int version, efi_physical_addr_t log_loca
 				       efi_bool_t truncated,
 				       struct efi_tcg2_final_events_table *final_events_table)
 {
-	efi_guid_t linux_eventlog_guid = LINUX_EFI_TPM_EVENT_LOG_GUID;
+	static efi_guid_t linux_eventlog_guid = LINUX_EFI_TPM_EVENT_LOG_GUID;
 	efi_status_t status;
 	struct linux_efi_tpm_eventlog *log_tbl = NULL;
 	unsigned long first_entry_addr, last_entry_addr;
@@ -150,9 +150,9 @@ static void efi_retrieve_tcg2_eventlog(int version, efi_physical_addr_t log_loca
 
 void efi_retrieve_eventlog(void)
 {
+	static efi_guid_t tpm2_guid = EFI_TCG2_PROTOCOL_GUID;
 	struct efi_tcg2_final_events_table *final_events_table = NULL;
 	efi_physical_addr_t log_location = 0, log_last_entry = 0;
-	efi_guid_t tpm2_guid = EFI_TCG2_PROTOCOL_GUID;
 	int version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_2;
 	efi_tcg2_protocol_t *tpm2 = NULL;
 	efi_bool_t truncated;
@@ -173,7 +173,7 @@ void efi_retrieve_eventlog(void)
 				get_efi_config_table(EFI_TCG2_FINAL_EVENTS_TABLE_GUID);
 		}
 	} else {
-		efi_guid_t cc_guid = EFI_CC_MEASUREMENT_PROTOCOL_GUID;
+		static efi_guid_t cc_guid = EFI_CC_MEASUREMENT_PROTOCOL_GUID;
 		efi_cc_protocol_t *cc = NULL;
 
 		status = efi_bs_call(locate_protocol, &cc_guid, NULL, (void **)&cc);
diff --git a/drivers/firmware/efi/libstub/unaccepted_memory.c b/drivers/firmware/efi/libstub/unaccepted_memory.c
index 757dbe734a47af..2a7eac7fef862a 100644
--- a/drivers/firmware/efi/libstub/unaccepted_memory.c
+++ b/drivers/firmware/efi/libstub/unaccepted_memory.c
@@ -9,7 +9,7 @@ struct efi_unaccepted_memory *unaccepted_table;
 efi_status_t allocate_unaccepted_bitmap(__u32 nr_desc,
 					struct efi_boot_memmap *map)
 {
-	efi_guid_t unaccepted_table_guid = LINUX_EFI_UNACCEPTED_MEM_TABLE_GUID;
+	static efi_guid_t unaccepted_table_guid = LINUX_EFI_UNACCEPTED_MEM_TABLE_GUID;
 	u64 unaccepted_start = ULLONG_MAX, unaccepted_end = 0, bitmap_size;
 	efi_status_t status;
 	int i;
diff --git a/drivers/firmware/efi/libstub/vsprintf.c b/drivers/firmware/efi/libstub/vsprintf.c
index 71c71c222346a4..7fd6589f44ef91 100644
--- a/drivers/firmware/efi/libstub/vsprintf.c
+++ b/drivers/firmware/efi/libstub/vsprintf.c
@@ -14,10 +14,14 @@
 
 #include <linux/compiler.h>
 #include <linux/ctype.h>
+#include <linux/efi.h>
 #include <linux/kernel.h>
 #include <linux/limits.h>
 #include <linux/string.h>
 #include <linux/types.h>
+#include <linux/ucs2_string.h>
+
+#include "efistub.h"
 
 static
 int skip_atoi(const char **s)
@@ -109,6 +113,9 @@ char *put_dec(char *end, unsigned long long n)
 	return p;
 }
 
+/* we are called with base 8, 10 or 16, only, thus don't need "G..."  */
+static const char digits[16] = "0123456789ABCDEF"; /* "GHIJKLMNOPQRSTUVWXYZ"; */
+
 static
 char *number(char *end, unsigned long long num, int base, char locase)
 {
@@ -117,9 +124,6 @@ char *number(char *end, unsigned long long num, int base, char locase)
 	 * produces same digits or (maybe lowercased) letters
 	 */
 
-	/* we are called with base 8, 10 or 16, only, thus don't need "G..."  */
-	static const char digits[16] = "0123456789ABCDEF"; /* "GHIJKLMNOPQRSTUVWXYZ"; */
-
 	switch (base) {
 	case 10:
 		if (num != 0)
@@ -140,6 +144,29 @@ char *number(char *end, unsigned long long num, int base, char locase)
 	return end;
 }
 
+static char *guid_to_str(const efi_guid_t *guid, char *out, char locase)
+{
+	static const u8 guid_index[UUID_SIZE] = {
+		3, 2, 1, 0, 5, 4, 7, 6, 8, 9, 10, 11, 12, 13, 14, 15,
+	};
+
+	for (int i = 0, p = 0; i < ARRAY_SIZE(guid_index); i++) {
+		u8 byte = guid->b[guid_index[i]];
+
+		out[p++] = locase | digits[byte >> 4];
+		out[p++] = locase | digits[byte & 0xf];
+
+		switch (i) {
+		case 3:
+		case 5:
+		case 7:
+		case 9:
+			out[p++] = '-';
+		}
+	}
+	return out;
+}
+
 #define ZEROPAD	1		/* pad with zero */
 #define SIGN	2		/* unsigned/signed long */
 #define PLUS	4		/* show plus */
@@ -239,58 +266,6 @@ char get_sign(long long *num, int flags)
 	return 0;
 }
 
-static
-size_t utf16s_utf8nlen(const u16 *s16, size_t maxlen)
-{
-	size_t len, clen;
-
-	for (len = 0; len < maxlen && *s16; len += clen) {
-		u16 c0 = *s16++;
-
-		/* First, get the length for a BMP character */
-		clen = 1 + (c0 >= 0x80) + (c0 >= 0x800);
-		if (len + clen > maxlen)
-			break;
-		/*
-		 * If this is a high surrogate, and we're already at maxlen, we
-		 * can't include the character if it's a valid surrogate pair.
-		 * Avoid accessing one extra word just to check if it's valid
-		 * or not.
-		 */
-		if ((c0 & 0xfc00) == 0xd800) {
-			if (len + clen == maxlen)
-				break;
-			if ((*s16 & 0xfc00) == 0xdc00) {
-				++s16;
-				++clen;
-			}
-		}
-	}
-
-	return len;
-}
-
-static
-u32 utf16_to_utf32(const u16 **s16)
-{
-	u16 c0, c1;
-
-	c0 = *(*s16)++;
-	/* not a surrogate */
-	if ((c0 & 0xf800) != 0xd800)
-		return c0;
-	/* invalid: low surrogate instead of high */
-	if (c0 & 0x0400)
-		return 0xfffd;
-	c1 = **s16;
-	/* invalid: missing low surrogate */
-	if ((c1 & 0xfc00) != 0xdc00)
-		return 0xfffd;
-	/* valid surrogate pair */
-	++(*s16);
-	return (0x10000 - (0xd800 << 10) - 0xdc00) + (c0 << 10) + c1;
-}
-
 #define PUTC(c) \
 do {				\
 	if (pos < size)		\
@@ -298,10 +273,10 @@ do {				\
 	++pos;			\
 } while (0);
 
-int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap)
+int efi_vsnprintf(efi_char16_t *buf, size_t size, const char *fmt, va_list ap,
+		  bool crlf)
 {
-	/* The maximum space required is to print a 64-bit number in octal */
-	char tmp[(sizeof(unsigned long long) * 8 + 2) / 3];
+	char tmp[UUID_STRING_LEN];
 	char *tmp_end = &tmp[ARRAY_SIZE(tmp)];
 	long long num;
 	int base;
@@ -336,6 +311,8 @@ int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap)
 
 	for (pos = 0; *fmt; ++fmt) {
 		if (*fmt != '%' || *++fmt == '%') {
+			if (crlf && *fmt == '\n')
+				PUTC('\r');
 			PUTC(*fmt);
 			continue;
 		}
@@ -400,7 +377,7 @@ int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap)
 			else if (qualifier == 'l') {
 		wstring:
 				flags |= WIDE;
-				precision = len = utf16s_utf8nlen((const u16 *)s, precision);
+				precision = len = ucs2_strnlen((const u16 *)s, precision);
 				goto output;
 			}
 			precision = len = strnlen(s, precision);
@@ -412,6 +389,14 @@ int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap)
 			break;
 
 		case 'p':
+			if (fmt[1] == 'U' && (fmt[2] | 0x20) == 'l') {
+				flags &= LEFT;
+				s = guid_to_str(va_arg(args, efi_guid_t *), tmp, fmt[2] & 0x20);
+				precision = len = UUID_STRING_LEN;
+				fmt += 2;
+				goto output;
+			}
+
 			if (precision < 0)
 				precision = 2 * sizeof(void *);
 			fallthrough;
@@ -505,36 +490,9 @@ int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap)
 		if (flags & WIDE) {
 			const u16 *ws = (const u16 *)s;
 
-			while (len-- > 0) {
-				u32 c32 = utf16_to_utf32(&ws);
-				u8 *s8;
-				size_t clen;
-
-				if (c32 < 0x80) {
-					PUTC(c32);
-					continue;
-				}
-
-				/* Number of trailing octets */
-				clen = 1 + (c32 >= 0x800) + (c32 >= 0x10000);
-
-				len -= clen;
-				s8 = (u8 *)&buf[pos];
-
-				/* Avoid writing partial character */
-				PUTC('\0');
-				pos += clen;
-				if (pos >= size)
-					continue;
-
-				/* Set high bits of leading octet */
-				*s8 = (0xf00 >> 1) >> clen;
-				/* Write trailing octets in reverse order */
-				for (s8 += clen; clen; --clen, c32 >>= 6)
-					*s8-- = 0x80 | (c32 & 0x3f);
-				/* Set low bits of leading octet */
-				*s8 |= c32;
-			}
+			if (pos < size)
+				memcpy(&buf[pos], ws, min(len, size - pos) * sizeof(*ws));
+			pos += len;
 		} else {
 			while (len-- > 0)
 				PUTC(*s++);
@@ -552,13 +510,13 @@ int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap)
 	return pos;
 }
 
-int snprintf(char *buf, size_t size, const char *fmt, ...)
+int efi_snprintf(efi_char16_t *buf, size_t size, const char *fmt, ...)
 {
 	va_list args;
 	int i;
 
 	va_start(args, fmt);
-	i = vsnprintf(buf, size, fmt, args);
+	i = efi_vsnprintf(buf, size, fmt, args, false);
 	va_end(args);
 	return i;
 }
diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c
index cef32e2c82d8fd..f4799e29f4cf5a 100644
--- a/drivers/firmware/efi/libstub/x86-stub.c
+++ b/drivers/firmware/efi/libstub/x86-stub.c
@@ -10,6 +10,7 @@
 #include <linux/pci.h>
 #include <linux/stddef.h>
 
+#include <asm/cpuid/api.h>
 #include <asm/efi.h>
 #include <asm/e820/types.h>
 #include <asm/setup.h>
@@ -17,6 +18,7 @@
 #include <asm/boot.h>
 #include <asm/kaslr.h>
 #include <asm/sev.h>
+#include <asm/shared/tdx.h>
 
 #include "efistub.h"
 #include "x86-stub.h"
@@ -114,9 +116,9 @@ preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom)
  */
 static void setup_efi_pci(struct boot_params *params)
 {
+	static efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
 	efi_status_t status;
 	efi_handle_t *pci_handle __free(efi_pool) = NULL;
-	efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID;
 	struct setup_data *data;
 	unsigned long num;
 	efi_handle_t h;
@@ -155,7 +157,7 @@ static void setup_efi_pci(struct boot_params *params)
 
 static void retrieve_apple_device_properties(struct boot_params *boot_params)
 {
-	efi_guid_t guid = APPLE_PROPERTIES_PROTOCOL_GUID;
+	static efi_guid_t guid = APPLE_PROPERTIES_PROTOCOL_GUID;
 	struct setup_data *data, *new;
 	efi_status_t status;
 	u32 size = 0;
@@ -335,6 +337,7 @@ static bool apple_match_product_name(void)
 
 static void apple_set_os(void)
 {
+	static efi_guid_t apple_set_os_guid = APPLE_SET_OS_PROTOCOL_GUID;
 	struct {
 		unsigned long version;
 		efi_status_t (__efiapi *set_os_version)(const char *);
@@ -345,7 +348,7 @@ static void apple_set_os(void)
 	if (!efi_is_64bit() || !apple_match_product_name())
 		return;
 
-	status = efi_bs_call(locate_protocol, &APPLE_SET_OS_PROTOCOL_GUID, NULL,
+	status = efi_bs_call(locate_protocol, &apple_set_os_guid, NULL,
 			     (void **)&set_os);
 	if (status != EFI_SUCCESS)
 		return;
@@ -444,7 +447,7 @@ efi_status_t efi_adjust_memory_range_protection(unsigned long start,
 
 static void setup_unaccepted_memory(void)
 {
-	efi_guid_t mem_acceptance_proto = OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID;
+	static efi_guid_t mem_acceptance_proto = OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID;
 	sev_memory_acceptance_protocol_t *proto;
 	efi_status_t status;
 
@@ -507,7 +510,7 @@ static void __noreturn efi_exit(efi_handle_t handle, efi_status_t status)
 static efi_status_t efi_allocate_bootparams(efi_handle_t handle,
 					    struct boot_params **bp)
 {
-	efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
+	static efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
 	struct boot_params *boot_params;
 	struct setup_header *hdr;
 	efi_status_t status;
@@ -648,7 +651,7 @@ setup_e820(struct boot_params *params, struct setup_data *e820ext, u32 e820ext_s
 		}
 
 		if (nr_entries == ARRAY_SIZE(params->e820_table)) {
-			u32 need = (nr_desc - i) * sizeof(struct e820_entry) +
+			u32 need = (nr_desc - i) * sizeof(struct boot_e820_entry) +
 				   sizeof(struct setup_data);
 
 			if (!e820ext || e820ext_size < need)
@@ -684,7 +687,7 @@ static efi_status_t alloc_e820ext(u32 nr_desc, struct setup_data **e820ext,
 	unsigned long size;
 
 	size = sizeof(struct setup_data) +
-		sizeof(struct e820_entry) * nr_desc;
+		sizeof(struct boot_e820_entry) * nr_desc;
 
 	if (*e820ext) {
 		efi_bs_call(free_pool, *e820ext);
@@ -914,7 +917,7 @@ void __noreturn efi_stub_entry(efi_handle_t handle,
 			       struct boot_params *boot_params)
 
 {
-	efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID;
+	static efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID;
 	const struct linux_efi_initrd *initrd = NULL;
 	unsigned long kernel_entry;
 	struct setup_header *hdr;
@@ -1014,6 +1017,7 @@ void __noreturn efi_stub_entry(efi_handle_t handle,
 	efi_random_get_seed();
 
 	efi_retrieve_eventlog();
+	efi_bli_set_variables(image);
 
 	setup_graphics(boot_params);
 
@@ -1068,3 +1072,40 @@ void efi64_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
 		      struct boot_params *boot_params);
 #endif
 #endif
+
+#ifdef CONFIG_UNACCEPTED_MEMORY
+/*
+ * process_unaccepted_memory() is called after ExitBootServices(), and so these
+ * memory acceptance routines cannot rely on EFI protocols for detecting the
+ * presence of TDX or SEV-SNP, or emit any kind of output if any error
+ * conditions are detected.
+ */
+static bool early_is_tdx_guest(void)
+{
+	static bool once;
+	static bool is_tdx;
+
+	if (!IS_ENABLED(CONFIG_INTEL_TDX_GUEST))
+		return false;
+
+	if (!once) {
+		u32 eax = TDX_CPUID_LEAF_ID, sig[3] = {};
+
+		native_cpuid(&eax, &sig[0], &sig[2], &sig[1]);
+		is_tdx = !memcmp(TDX_IDENT, sig, sizeof(sig));
+		once = true;
+	}
+
+	return is_tdx;
+}
+
+void arch_accept_memory(phys_addr_t start, phys_addr_t end)
+{
+	if (early_is_tdx_guest()) {
+		if (!tdx_accept_memory(start, end))
+			tdx_panic("Failed to accept memory");
+	} else if (early_is_sevsnp_guest()) {
+		snp_accept_memory(start, end);
+	}
+}
+#endif
diff --git a/drivers/firmware/efi/libstub/zboot.c b/drivers/firmware/efi/libstub/zboot.c
index 4b76f74c56dae0..4a5ffe164873ea 100644
--- a/drivers/firmware/efi/libstub/zboot.c
+++ b/drivers/firmware/efi/libstub/zboot.c
@@ -34,6 +34,7 @@ struct sysfb_display_info *alloc_primary_display(void)
 asmlinkage efi_status_t __efiapi
 efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab)
 {
+	static efi_guid_t loaded_image_guid = LOADED_IMAGE_PROTOCOL_GUID;
 	char *cmdline_ptr __free(efi_pool) = NULL;
 	unsigned long image_base, alloc_size;
 	efi_loaded_image_t *image;
@@ -42,7 +43,7 @@ efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab)
 	WRITE_ONCE(efi_system_table, systab);
 
 	status = efi_bs_call(handle_protocol, handle,
-			     &LOADED_IMAGE_PROTOCOL_GUID, (void **)&image);
+			     &loaded_image_guid, (void **)&image);
 	if (status != EFI_SUCCESS) {
 		efi_err("Failed to locate parent's loaded image protocol\n");
 		return status;
diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c
index 2344b9d1e81f8e..740f422df337a4 100644
--- a/drivers/firmware/efi/runtime-wrappers.c
+++ b/drivers/firmware/efi/runtime-wrappers.c
@@ -652,3 +652,31 @@ void efi_runtime_assert_lock_held(void)
 {
 	WARN_ON(efi_runtime_lock_owner != current);
 }
+
+efi_status_t efi_runtime_set_enable_flag(bool enable)
+{
+	static bool runtime_disabled;
+	efi_status_t ret = EFI_NOT_READY;
+
+	if (down_interruptible(&efi_runtime_lock))
+		return EFI_ABORTED;
+
+	if (enable) {
+		/* It could be enabled only if it is disabled here */
+		if (runtime_disabled) {
+			set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
+			runtime_disabled = false;
+			ret = EFI_SUCCESS;
+		}
+	} else {
+		if (efi_enabled(EFI_RUNTIME_SERVICES)) {
+			clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
+			runtime_disabled = true;
+			ret = EFI_SUCCESS;
+		}
+	}
+
+	up(&efi_runtime_lock);
+
+	return ret;
+}
diff --git a/drivers/firmware/efi/sysfb_efi.c b/drivers/firmware/efi/sysfb_efi.c
index 685283bb7327ad..2338de73c62c9d 100644
--- a/drivers/firmware/efi/sysfb_efi.c
+++ b/drivers/firmware/efi/sysfb_efi.c
@@ -301,15 +301,6 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
 		},
 		.callback = efifb_swap_width_height,
 	},
-	{
-		/* Lenovo D330 with 800x1280 or 1200x1920 portrait screen */
-		.matches = {
-			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
-			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
-					"Lenovo ideapad D330-10IGM"),
-		},
-		.callback = efifb_swap_width_height,
-	},
 	{
 		/*
 		 * Lenovo IdeaPad Duet 3 10IGL5 and 10IGL5-LTE with
diff --git a/drivers/firmware/google/Kconfig b/drivers/firmware/google/Kconfig
deleted file mode 100644
index b78c644fa2539a..00000000000000
--- a/drivers/firmware/google/Kconfig
+++ /dev/null
@@ -1,85 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-menuconfig GOOGLE_FIRMWARE
-	bool "Google Firmware Drivers"
-	default n
-	help
-	  These firmware drivers are used by Google servers,
-	  Chromebooks and other devices using coreboot firmware.
-	  If in doubt, say "N".
-
-if GOOGLE_FIRMWARE
-
-config GOOGLE_SMI
-	tristate "SMI interface for Google platforms"
-	depends on X86 && ACPI && DMI
-	help
-	  Say Y here if you want to enable SMI callbacks for Google
-	  platforms.  This provides an interface for writing to and
-	  clearing the event log.  If CONFIG_EFI is also enabled this
-	  driver provides an interface for reading and writing NVRAM
-	  variables.
-
-config GOOGLE_CBMEM
-	tristate "CBMEM entries in sysfs"
-	depends on GOOGLE_COREBOOT_TABLE
-	help
-	  CBMEM is a downwards-growing memory region created by the
-	  Coreboot BIOS containing tagged data structures from the
-	  BIOS.  These data structures expose things like the verified
-	  boot firmware variables, flash layout, firmware event log,
-	  and more.
-
-	  This option enables the cbmem module, which causes the
-	  kernel to search for Coreboot CBMEM entries, and expose the
-	  memory for each entry in sysfs under
-	  /sys/bus/coreboot/devices/cbmem-<id>.
-
-config GOOGLE_COREBOOT_TABLE
-	tristate "Coreboot Table Access"
-	depends on HAS_IOMEM && (ACPI || OF)
-	help
-	  This option enables the coreboot_table module, which provides other
-	  firmware modules access to the coreboot table. The coreboot table
-	  pointer is accessed through the ACPI "GOOGCB00" object or the
-	  device tree node /firmware/coreboot.
-	  If unsure say N.
-
-config GOOGLE_MEMCONSOLE
-	tristate
-	depends on GOOGLE_MEMCONSOLE_X86_LEGACY || GOOGLE_MEMCONSOLE_COREBOOT
-
-config GOOGLE_MEMCONSOLE_X86_LEGACY
-	tristate "Firmware Memory Console - X86 Legacy support"
-	depends on X86 && ACPI && DMI
-	select GOOGLE_MEMCONSOLE
-	help
-	  This option enables the kernel to search for a firmware log in
-	  the EBDA on Google servers.  If found, this log is exported to
-	  userland in the file /sys/firmware/log.
-
-config GOOGLE_FRAMEBUFFER_COREBOOT
-	tristate "Coreboot Framebuffer"
-	depends on GOOGLE_COREBOOT_TABLE
-	help
-	  This option enables the kernel to search for a framebuffer in
-	  the coreboot table.  If found, it is registered with a platform
-	  device of type coreboot-framebuffer. Using the old device of
-	  type simple-framebuffer is deprecated.
-
-config GOOGLE_MEMCONSOLE_COREBOOT
-	tristate "Firmware Memory Console"
-	depends on GOOGLE_COREBOOT_TABLE
-	select GOOGLE_MEMCONSOLE
-	help
-	  This option enables the kernel to search for a firmware log in
-	  the coreboot table.  If found, this log is exported to userland
-	  in the file /sys/firmware/log.
-
-config GOOGLE_VPD
-	tristate "Vital Product Data"
-	depends on GOOGLE_COREBOOT_TABLE
-	help
-	  This option enables the kernel to expose the content of Google VPD
-	  under /sys/firmware/vpd.
-
-endif # GOOGLE_FIRMWARE
diff --git a/drivers/firmware/google/Makefile b/drivers/firmware/google/Makefile
deleted file mode 100644
index 8151e323cc4346..00000000000000
--- a/drivers/firmware/google/Makefile
+++ /dev/null
@@ -1,14 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-obj-$(CONFIG_GOOGLE_SMI)		+= gsmi.o
-obj-$(CONFIG_GOOGLE_COREBOOT_TABLE)        += coreboot_table.o
-obj-$(CONFIG_GOOGLE_FRAMEBUFFER_COREBOOT)  += framebuffer-coreboot.o
-obj-$(CONFIG_GOOGLE_MEMCONSOLE)            += memconsole.o
-obj-$(CONFIG_GOOGLE_MEMCONSOLE_COREBOOT)   += memconsole-coreboot.o
-obj-$(CONFIG_GOOGLE_MEMCONSOLE_X86_LEGACY) += memconsole-x86-legacy.o
-
-# Must come after coreboot_table.o, as this driver depends on that bus type.
-obj-$(CONFIG_GOOGLE_CBMEM)		+= cbmem.o
-
-vpd-sysfs-y := vpd.o vpd_decode.o
-obj-$(CONFIG_GOOGLE_VPD)		+= vpd-sysfs.o
diff --git a/drivers/firmware/imx/Kconfig b/drivers/firmware/imx/Kconfig
index 127ad752acf86d..93ac339800e22f 100644
--- a/drivers/firmware/imx/Kconfig
+++ b/drivers/firmware/imx/Kconfig
@@ -55,3 +55,15 @@ config IMX_SCMI_MISC_DRV
 	  core that could provide misc functions such as board control.
 
 	  This driver can also be built as a module.
+
+config IMX_SEC_ENCLAVE
+	tristate "i.MX Embedded Secure Enclave - EdgeLock Enclave Firmware driver."
+	depends on MAILBOX && ((IMX_MBOX && ARCH_MXC && ARM64) || COMPILE_TEST)
+	select FW_LOADER
+	default m if ARCH_MXC
+
+	help
+	  Exposes APIs supported by the iMX Secure Enclave HW IP called:
+	  - EdgeLock Enclave Firmware (for i.MX8ULP, i.MX93),
+	    like base, HSM, V2X & SHE using the SAB protocol via the shared Messaging
+	    Unit.
diff --git a/drivers/firmware/imx/Makefile b/drivers/firmware/imx/Makefile
index 3bbaffa6e34781..33f30eaedad5e8 100644
--- a/drivers/firmware/imx/Makefile
+++ b/drivers/firmware/imx/Makefile
@@ -4,3 +4,5 @@ obj-$(CONFIG_IMX_SCU)		+= imx-scu.o misc.o imx-scu-irq.o rm.o imx-scu-soc.o
 obj-${CONFIG_IMX_SCMI_CPU_DRV}	+= sm-cpu.o
 obj-${CONFIG_IMX_SCMI_MISC_DRV}	+= sm-misc.o
 obj-${CONFIG_IMX_SCMI_LMM_DRV}	+= sm-lmm.o
+sec_enclave-objs		= se_ctrl.o ele_common.o ele_base_msg.o ele_fw_api.o ele_msg_addr_field.o
+obj-${CONFIG_IMX_SEC_ENCLAVE}	+= sec_enclave.o
diff --git a/drivers/firmware/imx/ele_base_msg.c b/drivers/firmware/imx/ele_base_msg.c
new file mode 100644
index 00000000000000..b60665120db2f4
--- /dev/null
+++ b/drivers/firmware/imx/ele_base_msg.c
@@ -0,0 +1,395 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2025 NXP
+ */
+
+#include <linux/types.h>
+
+#include <linux/cleanup.h>
+#include <linux/completion.h>
+#include <linux/dma-mapping.h>
+#include <linux/genalloc.h>
+
+#include "ele_base_msg.h"
+#include "ele_common.h"
+
+#define FW_DBG_DUMP_FIXED_STR		"ELE"
+
+int ele_uapi_allowed_base_cmd(struct se_if_device_ctx *dev_ctx,
+			      struct se_msg_hdr *header, u32 tx_msg_sz)
+{
+	struct se_api_msg *msg = container_of(header, struct se_api_msg, header);
+	const struct se_cmd_addr_field *fields;
+	size_t count;
+
+	/*
+	 * Identify the command first. Only commands in this allow-list may be
+	 * issued from userspace; everything else is rejected. Once a command is
+	 * known to be supported, decide whether it needs a DMA-address boundary
+	 * check and, if so, run it before returning.
+	 */
+	switch (header->command) {
+	case ELE_PING_REQ:
+	case ELE_DEBUG_DUMP_REQ:
+	case ELE_OEM_VERIFY_IMAGE_REQ:
+	case ELE_OEM_REL_CONTAINER_REQ:
+	case ELE_FW_LIFE_CYCLE_REQ:
+	case ELE_READ_FUSE_REQ:
+	case ELE_GET_FW_VERS_REQ:
+	case ELE_RETURN_LIFE_CYCLE_REQ:
+	case ELE_GET_EVENT_REQ:
+	case ELE_COMMIT_REQ:
+	case ELE_GET_FW_STATUS_REQ:
+	case ELE_WRITE_FUSE:
+	case ELE_WRITE_SHADOW_FUSE_REQ:
+	case ELE_READ_SHADOW_FUSE_REQ:
+		return 0;
+	default:
+		/* Base commands that embed DMA addresses. */
+		fields = ele_base_cmd_addr_fields(header->command, &count);
+		if (!count)
+			return -EOPNOTSUPP;
+		return se_val_cmd_addrs(dev_ctx, msg, tx_msg_sz, fields, count);
+	}
+}
+
+static void ele_get_info_cleanup(struct se_if_priv *priv)
+{
+	/* For the case when priv->mem_pool != NULL:
+	 *
+	 *   If this probe-time transaction timed out, the firmware may
+	 *   still write into the SRAM buffer after this function returns.
+	 *   Do not release it back to the pool while the firmware-busy
+	 *   circuit breaker still marks this context as owning an
+	 *   outstanding transaction. The buffer is reclaimed with the
+	 *   device on unbind; leaking this fixed-size probe buffer is
+	 *   preferable to letting the firmware corrupt reused pool memory.
+	 *   This mirrors the guard already applied on the shared-memory
+	 *   cleanup path below.
+	 */
+
+	if (priv->mem_pool) {
+		if (se_is_fw_busy_ctx(priv->priv_dev_ctx))
+			return;
+		se_cleanup_mem_pool_buf(priv->priv_dev_ctx, true);
+	} else {
+		se_dev_ctx_shared_mem_cleanup(priv->priv_dev_ctx);
+	}
+}
+
+/**
+ * ele_get_info() - retrieve SoC and firmware information from the ELE.
+ * @priv: pointer to the SE interface private data.
+ * @s_info: output buffer; filled with device info on success.
+ *
+ * Allocates a DMA-coherent bounce buffer (from the gen_pool if available,
+ * otherwise from the DMA API), sends an ELE_GET_INFO_REQ command, and copies
+ * the result into @s_info.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int ele_get_info(struct se_if_priv *priv, struct ele_dev_info *s_info)
+{
+	dma_addr_t get_info_addr = 0;
+	void *get_info_data = NULL;
+	u32 get_info_len;
+	int ret;
+
+	if (!priv)
+		return -EINVAL;
+
+	guard(mutex)(&priv->priv_dev_ctx->fops_lock);
+	memset(s_info, 0x0, sizeof(*s_info));
+
+	struct se_api_msg *tx_msg __free(kfree) =
+		kzalloc(ELE_GET_INFO_REQ_MSG_SZ, GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_GET_INFO_RSP_MSG_SZ, GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	get_info_len = ELE_GET_INFO_BUFF_SZ;
+	if (priv->mem_pool) {
+		ret = se_get_mem_pool_buf(priv->priv_dev_ctx, &get_info_data,
+					  &get_info_addr, get_info_len);
+		if (ret) {
+			dev_err(priv->dev, "Failed[0x%x] to alloc from gen_pool.\n", ret);
+			return -ENOMEM;
+		}
+	} else {
+		ret = get_shared_mem_slot(priv->priv_dev_ctx,
+					  &get_info_len, &get_info_addr,
+					  &get_info_data);
+		if (ret) {
+			dev_err(priv->dev, "Failed to allocate buffer.\n");
+			return -ENOMEM;
+		}
+	}
+
+	se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+			    ELE_GET_INFO_REQ, ELE_GET_INFO_REQ_MSG_SZ, true);
+
+	tx_msg->data[0] = upper_32_bits(get_info_addr);
+	tx_msg->data[1] = lower_32_bits(get_info_addr);
+	tx_msg->data[2] = sizeof(*s_info);
+
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_GET_INFO_REQ_MSG_SZ,
+			       rx_msg, ELE_GET_INFO_RSP_MSG_SZ, NULL);
+	if (ret < 0) {
+		ele_get_info_cleanup(priv);
+		return ret;
+	}
+
+	ret = se_val_rsp_hdr_n_status(priv->priv_dev_ctx, rx_msg, ELE_GET_INFO_REQ,
+				      ELE_GET_INFO_RSP_MSG_SZ,
+				      priv->if_defs->base_api_ver);
+	if (ret < 0) {
+		ele_get_info_cleanup(priv);
+		return ret;
+	}
+
+	memcpy(s_info, get_info_data, sizeof(*s_info));
+
+	ele_get_info_cleanup(priv);
+
+	return ret;
+}
+
+/**
+ * ele_fetch_soc_info() - wrapper around ele_get_info() for generic callers.
+ * @priv: pointer to the SE interface private data.
+ * @data: output buffer of at least sizeof(struct ele_dev_info) bytes.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int ele_fetch_soc_info(struct se_if_priv *priv, void *data)
+{
+	return ele_get_info(priv, (struct ele_dev_info *)data);
+}
+
+/**
+ * ele_ping() - send a ping command to the secure enclave.
+ * @priv: pointer to the SE interface private data.
+ *
+ * Verifies that the secure enclave is alive and responsive.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int ele_ping(struct se_if_priv *priv)
+{
+	int ret;
+
+	if (!priv)
+		return -EINVAL;
+
+	struct se_api_msg *tx_msg __free(kfree) = kzalloc(ELE_PING_REQ_SZ,
+							  GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) = kzalloc(ELE_PING_RSP_SZ,
+							  GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+			    ELE_PING_REQ, ELE_PING_REQ_SZ, true);
+
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_PING_REQ_SZ,
+			       rx_msg, ELE_PING_RSP_SZ, NULL);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(priv->priv_dev_ctx, rx_msg, ELE_PING_REQ,
+				      ELE_PING_RSP_SZ,
+				      priv->if_defs->base_api_ver);
+
+	return ret;
+}
+
+/**
+ * ele_service_swap() - issue an ELE service-swap (IMEM export/import) command.
+ * @priv: pointer to the SE interface private data.
+ * @addr: DMA address of the IMEM buffer; must fit in 32 bits.
+ * @addr_size: size of the buffer at @addr in bytes.
+ * @flag: ELE_IMEM_EXPORT or ELE_IMEM_IMPORT.
+ *
+ * Return: exported size in bytes (ELE_IMEM_EXPORT), 0 (ELE_IMEM_IMPORT),
+ * or negative errno on failure.
+ */
+int ele_service_swap(struct se_if_priv *priv,
+		     dma_addr_t addr,
+		     u32 addr_size, u16 flag)
+{
+	int ret;
+
+	if (!priv)
+		return -EINVAL;
+
+	if (upper_32_bits(addr)) {
+		dev_err(priv->dev,
+			"ELE service-swap address exceeds 32-bit range: %pad\n",
+			&addr);
+		return -ERANGE;
+	}
+
+	struct se_api_msg *tx_msg __free(kfree)	=
+		kzalloc(ELE_SERVICE_SWAP_REQ_MSG_SZ, GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_SERVICE_SWAP_RSP_MSG_SZ, GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+			    ELE_SERVICE_SWAP_REQ, ELE_SERVICE_SWAP_REQ_MSG_SZ, true);
+
+	tx_msg->data[0] = flag;
+	tx_msg->data[1] = addr_size;
+	tx_msg->data[2] = ELE_NONE_VAL;
+	tx_msg->data[3] = lower_32_bits(addr);
+	ret = se_update_msg_chksum((u32 *)&tx_msg[0], ELE_SERVICE_SWAP_REQ_MSG_SZ);
+	if (ret)
+		return -EINVAL;
+
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_SERVICE_SWAP_REQ_MSG_SZ,
+			       rx_msg, ELE_SERVICE_SWAP_RSP_MSG_SZ, NULL);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(priv->priv_dev_ctx, rx_msg, ELE_SERVICE_SWAP_REQ,
+				      ELE_SERVICE_SWAP_RSP_MSG_SZ,
+				      priv->if_defs->base_api_ver);
+	if (ret)
+		return ret;
+
+	if (flag == ELE_IMEM_EXPORT)
+		ret = rx_msg->data[1];
+	else
+		ret = 0;
+
+	return ret;
+}
+
+/**
+ * ele_fw_authenticate() - authenticate a firmware container via the ELE.
+ * @priv: pointer to the SE interface private data.
+ * @contnr_addr: DMA address of the firmware container; must fit in 32 bits.
+ * @img_addr: DMA address of the firmware image; must fit in 32 bits.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int ele_fw_authenticate(struct se_if_priv *priv, dma_addr_t contnr_addr,
+			dma_addr_t img_addr)
+{
+	int ret;
+
+	if (!priv)
+		return -EINVAL;
+
+	if (upper_32_bits(contnr_addr) || upper_32_bits(img_addr)) {
+		dev_err(priv->dev, "Wrong address: %pad %pad\n", &contnr_addr, &img_addr);
+		return -EINVAL;
+	}
+
+	struct se_api_msg *tx_msg __free(kfree)	=
+		kzalloc(ELE_FW_AUTH_REQ_SZ, GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_FW_AUTH_RSP_MSG_SZ, GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+			    ELE_FW_AUTH_REQ, ELE_FW_AUTH_REQ_SZ, true);
+
+	tx_msg->data[0] = lower_32_bits(contnr_addr);
+	tx_msg->data[1] = 0;
+	tx_msg->data[2] = lower_32_bits(img_addr);
+
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_FW_AUTH_REQ_SZ, rx_msg,
+			       ELE_FW_AUTH_RSP_MSG_SZ, NULL);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(priv->priv_dev_ctx, rx_msg, ELE_FW_AUTH_REQ,
+				      ELE_FW_AUTH_RSP_MSG_SZ,
+				      priv->if_defs->base_api_ver);
+
+	return ret;
+}
+
+/**
+ * ele_debug_dump() - retrieve and log the ELE debug dump buffer.
+ * @priv: pointer to the SE interface private data.
+ *
+ * Repeatedly issues ELE_DEBUG_DUMP_REQ commands and logs the responses via
+ * dev_info() until no more data is available or the maximum packet count is
+ * reached.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int ele_debug_dump(struct se_if_priv *priv)
+{
+	bool keep_logging;
+	int msg_ex_cnt;
+	int ret;
+	int i;
+
+	if (!priv)
+		return -EINVAL;
+
+	struct se_api_msg *tx_msg __free(kfree) = kzalloc(ELE_DEBUG_DUMP_REQ_SZ,
+							  GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree)	= kzalloc(ELE_DEBUG_DUMP_RSP_SZ,
+							  GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	se_fill_cmd_msg_hdr(priv, &tx_msg->header, ELE_DEBUG_DUMP_REQ,
+			    ELE_DEBUG_DUMP_REQ_SZ, true);
+
+	msg_ex_cnt = 0;
+	do {
+		memset(rx_msg, 0x0, ELE_DEBUG_DUMP_RSP_SZ);
+
+		ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_DEBUG_DUMP_REQ_SZ,
+				       rx_msg, ELE_DEBUG_DUMP_RSP_SZ, NULL);
+		if (ret < 0)
+			return ret;
+
+		ret = se_val_rsp_hdr_n_status(priv->priv_dev_ctx, rx_msg, ELE_DEBUG_DUMP_REQ,
+					      ELE_DEBUG_DUMP_RSP_SZ,
+					      priv->if_defs->base_api_ver);
+		if (ret) {
+			dev_err(priv->dev, "Dump_Debug_Buffer Error: %x.\n", ret);
+			break;
+		}
+		keep_logging = (rx_msg->header.size >= (ELE_DEBUG_DUMP_RSP_SZ >> 2) &&
+				msg_ex_cnt < ELE_MAX_DBG_DMP_PKT);
+
+		rx_msg->header.size -= 2;
+
+		if (rx_msg->header.size > 2)
+			rx_msg->header.size--;
+
+		for (i = 0; i < rx_msg->header.size; i += 2)
+			dev_info(priv->dev, "%s%02x_%02x: 0x%08x 0x%08x\n",
+				 FW_DBG_DUMP_FIXED_STR, msg_ex_cnt, i,
+				 rx_msg->data[i + 1], rx_msg->data[i + 2]);
+
+		msg_ex_cnt++;
+	} while (keep_logging);
+
+	return ret;
+}
diff --git a/drivers/firmware/imx/ele_base_msg.h b/drivers/firmware/imx/ele_base_msg.h
new file mode 100644
index 00000000000000..50e2a1a7571662
--- /dev/null
+++ b/drivers/firmware/imx/ele_base_msg.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2025 NXP
+ *
+ * Header file for the EdgeLock Enclave Base API(s).
+ */
+
+#ifndef ELE_BASE_MSG_H
+#define ELE_BASE_MSG_H
+
+#include <linux/unaligned.h>
+#include <linux/device.h>
+#include <linux/types.h>
+
+#include "se_ctrl.h"
+
+#define ELE_NONE_VAL			0x0
+#define ELE_MAX_DBG_DMP_PKT		50
+
+#define ELE_PING_REQ			0x01
+#define ELE_PING_REQ_SZ			0x04
+#define ELE_PING_RSP_SZ			0x08
+
+#define ELE_FW_AUTH_REQ			0x02
+#define ELE_FW_AUTH_REQ_SZ		0x10
+#define ELE_FW_AUTH_RSP_MSG_SZ		0x08
+
+#define ELE_DEBUG_DUMP_REQ		0x21
+#define ELE_DEBUG_DUMP_REQ_SZ		0x4
+#define ELE_DEBUG_DUMP_RSP_SZ		0x5c
+
+#define ELE_OEM_AUTH_CONTAINER_REQ	0x87
+#define ELE_OEM_VERIFY_IMAGE_REQ	0x88
+#define ELE_OEM_REL_CONTAINER_REQ	0x89
+#define ELE_FW_LIFE_CYCLE_REQ		0x95
+#define ELE_READ_FUSE_REQ		0x97
+#define ELE_GET_FW_VERS_REQ		0x9d
+#define ELE_RETURN_LIFE_CYCLE_REQ	0xa0
+#define ELE_GET_EVENT_REQ		0xa2
+#define ELE_COMMIT_REQ			0xa8
+#define ELE_GEN_KEY_BLOB_REQ		0xaf
+#define ELE_GET_FW_STATUS_REQ		0xc5
+#define ELE_WRITE_FUSE                  0xd6
+
+#define ELE_GET_INFO_REQ		0xda
+#define ELE_GET_INFO_REQ_MSG_SZ		0x10
+#define ELE_GET_INFO_RSP_MSG_SZ		0x08
+
+#define MAX_UID_SIZE                     (16)
+#define DEV_GETINFO_ROM_PATCH_SHA_SZ     (32)
+#define DEV_GETINFO_FW_SHA_SZ            (32)
+#define DEV_GETINFO_OEM_SRKH_SZ          (64)
+#define DEV_GETINFO_MIN_VER_MASK	0xff
+#define DEV_GETINFO_MAJ_VER_MASK	0xff00
+#define ELE_DEV_INFO_EXTRA_SZ		0x60
+
+struct dev_info {
+	u8  cmd;
+	u8  ver;
+	u16 length;
+	u16 soc_id;
+	u16 soc_rev;
+	u16 lmda_val;
+	u8  ssm_state;
+	u8  dev_atts_api_ver;
+	u8  uid[MAX_UID_SIZE];
+	u8  sha_rom_patch[DEV_GETINFO_ROM_PATCH_SHA_SZ];
+	u8  sha_fw[DEV_GETINFO_FW_SHA_SZ];
+};
+
+struct dev_addn_info {
+	u8  oem_srkh[DEV_GETINFO_OEM_SRKH_SZ];
+	u8  trng_state;
+	u8  csal_state;
+	u8  imem_state;
+	u8  reserved2;
+};
+
+struct ele_dev_info {
+	struct dev_info d_info;
+	struct dev_addn_info d_addn_info;
+};
+
+#define ELE_GET_INFO_BUFF_SZ		(sizeof(struct ele_dev_info) \
+						+ ELE_DEV_INFO_EXTRA_SZ)
+
+#define ELE_DEV_ATTEST_REQ              0xdb
+#define ELE_WRITE_SHADOW_FUSE_REQ       0xf2
+#define ELE_READ_SHADOW_FUSE_REQ        0xf3
+
+#define ELE_SERVICE_SWAP_REQ		0xdf
+#define ELE_SERVICE_SWAP_REQ_MSG_SZ	0x18
+#define ELE_SERVICE_SWAP_RSP_MSG_SZ	0x0c
+#define ELE_IMEM_SIZE			0x10000
+#define ELE_IMEM_STATE_OK		0xca
+#define ELE_IMEM_STATE_BAD		0xfe
+#define ELE_IMEM_STATE_WORD		0x27
+#define ELE_IMEM_STATE_MASK		0x00ff0000
+#define ELE_IMEM_EXPORT			0x1
+#define ELE_IMEM_IMPORT			0x2
+
+#define GET_SERIAL_NUM_FROM_UID(x, uid_word_sz) ({\
+	const u8 *__x = (const u8 *)(x); \
+	size_t __sz = (uid_word_sz); \
+	((u64)get_unaligned_le32(__x + (__sz - 1) * sizeof(u32)) << 32) | \
+		get_unaligned_le32(__x); \
+	})
+
+int ele_get_info(struct se_if_priv *priv, struct ele_dev_info *s_info);
+int ele_fetch_soc_info(struct se_if_priv *priv, void *data);
+int ele_ping(struct se_if_priv *priv);
+int ele_service_swap(struct se_if_priv *priv, dma_addr_t addr,
+		     u32 addr_size, u16 flag);
+int ele_fw_authenticate(struct se_if_priv *priv, dma_addr_t contnr_addr,
+			dma_addr_t img_addr);
+int ele_debug_dump(struct se_if_priv *priv);
+int ele_uapi_allowed_base_cmd(struct se_if_device_ctx *dev_ctx,
+			      struct se_msg_hdr *header, u32 tx_msg_sz);
+#endif
diff --git a/drivers/firmware/imx/ele_common.c b/drivers/firmware/imx/ele_common.c
new file mode 100644
index 00000000000000..3a0cd62f2ba581
--- /dev/null
+++ b/drivers/firmware/imx/ele_common.c
@@ -0,0 +1,1012 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2025 NXP
+ */
+
+#include "ele_base_msg.h"
+#include "ele_common.h"
+#include "ele_fw_api.h"
+#include "se_ctrl.h"
+
+int se_chk_tx_rsp_msg_hdr(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			  u32 tx_msg_sz)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	if (!header->size || header->size > MAX_WORD_SIZE)
+		return -EINVAL;
+
+	if (header->tag != priv->if_defs->rsp_tag)
+		return -EINVAL;
+
+	if (header->ver == priv->if_defs->base_api_ver)
+		return -EINVAL;
+
+	else if (header->ver == priv->if_defs->fw_api_ver)
+		return ele_uapi_allowed_fw_rsp(dev_ctx, header, tx_msg_sz);
+
+	return -EINVAL;
+}
+
+int se_chk_tx_cmd_msg_hdr(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			  u32 tx_msg_sz, u32 rx_msg_sz)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	if (!header->size || header->size > MAX_WORD_SIZE)
+		return -EINVAL;
+
+	if (header->tag != priv->if_defs->cmd_tag)
+		return -EINVAL;
+
+	if (header->ver == priv->if_defs->base_api_ver)
+		return ele_uapi_allowed_base_cmd(dev_ctx, header, tx_msg_sz);
+	else if (header->ver == priv->if_defs->fw_api_ver)
+		return ele_uapi_allowed_fw_cmd(dev_ctx, header, tx_msg_sz, rx_msg_sz);
+
+	return -EINVAL;
+}
+
+/*
+ * Reject a command that embeds a DMA physical address which does not point
+ * inside this context's shared-memory window. The userspace library stages all
+ * command buffers in that coherent region (see get_shared_mem_slot), so any
+ * address outside [dma_addr, dma_addr + size) is not one the driver handed out
+ * and must not be forwarded to firmware. Absent optional buffers are encoded as
+ * a zero address and skipped; polymorphic key words are only range-checked when
+ * their gating flag marks them as a plaintext-key buffer rather than an integer
+ * key identifier. An address may occupy one word (FW-API, low 32 bits only) or
+ * two words (some base-API commands split it into low and high halves).
+ *
+ * When a field also names a size word, the buffer length carried there is
+ * validated too: the whole buffer [addr, addr + len) must fit inside the
+ * window, not just its start address. The check is written as len > end - addr
+ * (addr is already known to be < end) so it cannot overflow.
+ */
+int se_val_cmd_addrs(struct se_if_device_ctx *dev_ctx, struct se_api_msg *msg,
+		     u32 tx_msg_sz, const struct se_cmd_addr_field *fields,
+		     size_t count)
+{
+	const struct se_shared_mem *mem = &dev_ctx->se_shared_mem_mgmt.non_secure_mem;
+	u32 payload_words;
+	size_t i;
+	u64 base, end;
+
+	if (!fields || !count)
+		return 0;
+
+	if (!msg)
+		return -EINVAL;
+
+	/* Number of complete u32 payload words present after the header. */
+	if (tx_msg_sz < SE_MU_HDR_SZ)
+		return -EINVAL;
+	/*
+	 * The caller-supplied byte count must agree with the size the firmware
+	 * will act on (header word-size field, in 32-bit words), so a lying
+	 * header cannot make us validate fewer words than are actually sent.
+	 */
+	if (tx_msg_sz != (u32)msg->header.size * sizeof(u32))
+		return -EINVAL;
+	payload_words = (tx_msg_sz - SE_MU_HDR_SZ) / sizeof(u32);
+
+	base = (u64)mem->dma_addr;
+	end = base + mem->size;
+
+	/* A zero-sized or wrapping window can never contain a valid buffer. */
+	if (end <= base)
+		return -EINVAL;
+
+	for (i = 0; i < count; i++) {
+		const struct se_cmd_addr_field *f = &fields[i];
+		u64 addr;
+
+		/* Every word the field references must lie within the message. */
+		if (f->lsb_idx >= payload_words)
+			return -EINVAL;
+		if (f->has_msb && f->msb_idx >= payload_words)
+			return -EINVAL;
+
+		if (f->flag_idx != SE_CMD_ADDR_ALWAYS) {
+			bool flag_set;
+
+			if (f->flag_idx >= payload_words)
+				return -EINVAL;
+
+			flag_set = !!(msg->data[f->flag_idx] & f->flag_mask);
+			/*
+			 * When the flag does not select DMA-address mode the
+			 * word holds an integer key identifier; leave it alone.
+			 */
+			if (flag_set != f->is_addr_when_set)
+				continue;
+		}
+
+		addr = msg->data[f->lsb_idx];
+		if (f->has_msb)
+			addr |= (u64)msg->data[f->msb_idx] << 32;
+
+		/* Zero marks an absent optional buffer. */
+		if (!addr)
+			continue;
+
+		if (addr < base || addr >= end)
+			return -EACCES;
+
+		/*
+		 * The whole buffer [addr, addr + len) must fit inside the
+		 * window, not just its start. addr is already >= base and
+		 * < end here, so end - addr is a positive value and every
+		 * "len > end - addr" comparison below cannot overflow. Where
+		 * the length comes from depends on f->size_idx.
+		 */
+		switch (f->size_idx) {
+		case SE_CMD_ADDR_DEDUCE_SZ: {
+			/*
+			 * No length word is carried in the payload. Instead the
+			 * buffer begins with a struct fw_tag_len_vers_info
+			 * header whose length field gives the total buffer byte
+			 * count.
+			 */
+			const struct fw_tag_len_vers_info *hdr;
+			u64 len;
+
+			/*
+			 * The header itself must lie inside the window before it
+			 * can be read. addr is already >= base and < end, so
+			 * end - addr is positive and cannot overflow.
+			 */
+			if (sizeof(*hdr) > end - addr)
+				return -EACCES;
+
+			/*
+			 * Translate the validated DMA address to its kernel
+			 * virtual alias inside the coherent shared-memory
+			 * mapping before dereferencing it; a raw DMA address
+			 * must never be dereferenced directly.
+			 */
+			hdr = (const struct fw_tag_len_vers_info *)
+				(mem->ptr + (addr - base));
+			len = le16_to_cpu(hdr->length);
+			if (!len || len > end - addr)
+				return -EACCES;
+			break;
+		}
+		case SE_CMD_ADDR_FIXED_SIZE:
+			/* buf_size: literal byte count (FW-defined constant). */
+			if (!f->buf_size)
+				return -EINVAL;
+			if ((u64)f->buf_size > end - addr)
+				return -EACCES;
+			break;
+		case SE_CMD_RCVR_ADDR_VAR_SIZE: {
+			struct cmd_rcvr_data_info *crcvr_info =
+							&dev_ctx->priv->crcvr_info;
+			/*
+			 * Export-response buffer: the size was supplied by FW
+			 * in the preceding export command and stored per SE
+			 * interface in cmd_rcvr_var_size.
+			 */
+			if ((u64)crcvr_info->cmd_rcvr_var_size > end - addr)
+				return -EACCES;
+			break;
+		}
+		default: {
+			/* size_idx names a payload word carrying the length. */
+			u64 len;
+
+			if (f->size_idx >= payload_words)
+				return -EINVAL;
+
+			/* size_mask == 0 with a valid size_idx is a descriptor bug. */
+			if (!f->size_mask)
+				return -EINVAL;
+
+			/*
+			 * Widen to u64 before shifting: size_shift is u8 and
+			 * shifting a u32 by >= 32 is undefined behaviour.
+			 */
+			len = ((u64)msg->data[f->size_idx] >> f->size_shift) & f->size_mask;
+			if (len > end - addr)
+				return -EACCES;
+			break;
+		}
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * se_update_msg_chksum() - calculate and update message checksum word.
+ * @msg: message buffer.
+ * @msg_len: message length in bytes.
+ *
+ * The message length must be 4-byte aligned. The last word is treated as the
+ * checksum field and is not included in the checksum calculation.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int se_update_msg_chksum(u32 *msg, u32 msg_len)
+{
+	u32 nb_words;
+	u32 chksum = 0;
+	u32 i;
+
+	if (!msg)
+		return -EINVAL;
+
+	if (msg_len % SE_MSG_WORD_SZ) {
+		pr_err("Msg-len is not 4-byte aligned.\n");
+		return -EINVAL;
+	}
+
+	nb_words = msg_len / sizeof(*msg);
+	if (nb_words < 5)
+		return -EINVAL;
+
+	/* Last word is the checksum word, so skip it. */
+	nb_words--;
+
+	for (i = 0; i < nb_words; i++)
+		chksum ^= msg[i];
+
+	msg[nb_words] = chksum;
+
+	return 0;
+}
+
+static void se_mark_fw_busy(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct fw_busy_info *fbusy_info = &priv->fw_busy_info;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fbusy_info->fw_busy_lock, flags);
+	if (!fbusy_info->fw_busy_dev_ctx) {
+		kref_get(&dev_ctx->refcount);
+		fbusy_info->fw_busy_dev_ctx = dev_ctx;
+		/*
+		 * Snapshot the devname now, while it is still valid. If the
+		 * userspace fd is closed while the breaker is armed,
+		 * cleanup_dev_ctx() frees dev_ctx->devname and sets it to NULL,
+		 * so se_clear_fw_busy() cannot rely on it for its late-response
+		 * diagnostics. devname is short ("%s0_ch%d"); strscpy() safely
+		 * truncates into the fixed buffer.
+		 */
+		if (dev_ctx->devname)
+			strscpy(fbusy_info->devname, dev_ctx->devname,
+				sizeof(fbusy_info->devname));
+		else
+			fbusy_info->devname[0] = '\0';
+		atomic_set(&fbusy_info->fw_busy, 1);
+	}
+	spin_unlock_irqrestore(&fbusy_info->fw_busy_lock, flags);
+}
+
+void set_se_rcv_msg_timeout(struct se_if_device_ctx *dev_ctx, u32 timeout_ms)
+{
+	dev_ctx->rcv_msg_timeout_jiffies = msecs_to_jiffies(timeout_ms);
+}
+
+/**
+ * ele_msg_rcv() - wait for a response from the secure enclave.
+ * @dev_ctx: pointer to the SE dev context data.
+ * @se_clbk_hdl: callback handle whose completion will be signaled when the
+ *               response arrives.
+ *
+ * Blocks until the firmware delivers a response into the buffer registered
+ * in @se_clbk_hdl, or until the per-interface timeout expires.  When waiting
+ * on the response path a deadline is enforced; on timeout the firmware-busy
+ * circuit breaker is armed to prevent further transactions until the delayed
+ * response arrives and clears it.
+ *
+ * Return: number of bytes received on success, negative errno on error
+ * (e.g. -ETIMEDOUT, -ERESTARTSYS).
+ */
+int ele_msg_rcv(struct se_if_device_ctx *dev_ctx, struct se_clbk_handle *se_clbk_hdl)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	bool is_rsp_wait_with_timeout = false;
+	bool wait_uninterruptible = false;
+	bool wait_killable = false;
+	unsigned long remaining_jiffies;
+	unsigned long deadline_jiffies;
+	unsigned long flags;
+	int ret;
+
+	remaining_jiffies = dev_ctx->rcv_msg_timeout_jiffies;
+	if (se_clbk_hdl == &priv->waiting_rsp_clbk_hdl) {
+		is_rsp_wait_with_timeout = true;
+		deadline_jiffies = jiffies + remaining_jiffies;
+
+		/*
+		 * Internal kernel transactions run on priv_dev_ctx (probe
+		 * get_info/ping, FW auth, PM IMEM swap). They are not tied to a
+		 * restartable syscall, so wait uninterruptibly: PM freezer fake
+		 * signals must not abort them with -ERESTARTSYS. Userspace
+		 * waiters stay interruptible via the deferred-signal path below.
+		 */
+		if (se_clbk_hdl->dev_ctx == priv->priv_dev_ctx)
+			wait_uninterruptible = true;
+	}
+
+	do {
+		if (is_rsp_wait_with_timeout) {
+			unsigned long now = jiffies;
+
+			if (time_after_eq(now, deadline_jiffies)) {
+				/* Deadline hit: fence hung FW, like the ret==0 path. */
+				spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+				se_clbk_hdl->rx_msg = NULL;
+				/* rx_delivered is set only after a real response has
+				 * been copied under clbk_rx_lock, so it correctly
+				 * distinguishes a genuine timeout (no response → mark
+				 * busy) from a spurious teardown-forced wakeup where
+				 * the data is not yet safe to free.
+				 */
+				if (!se_clbk_hdl->rx_delivered)
+					se_mark_fw_busy(dev_ctx);
+				spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+				ret = -ETIMEDOUT;
+				break;
+			}
+			remaining_jiffies = deadline_jiffies - now;
+		}
+
+		if (wait_uninterruptible)
+			ret = wait_for_completion_timeout(&se_clbk_hdl->done,
+							  remaining_jiffies);
+		else if (wait_killable)
+			ret = wait_for_completion_killable_timeout(&se_clbk_hdl->done,
+								   remaining_jiffies);
+		else
+			ret = wait_for_completion_interruptible_timeout(&se_clbk_hdl->done,
+									remaining_jiffies);
+		if (ret == -ERESTARTSYS) {
+			/*
+			 * First, non-fatal signal on the interruptible userspace
+			 * path: defer it. Record that a signal was observed and keep
+			 * waiting - now only killably - until the response arrives or
+			 * the timeout expires. ele_msg_send_rcv() then surfaces the
+			 * interruption to userspace as -ERESTARTSYS once the protocol
+			 * transaction has resynchronised, so the in-flight command is
+			 * neither abandoned nor re-sent.
+			 *
+			 * Waiting killably rather than fully uninterruptibly is what
+			 * keeps a fatal signal (SIGKILL) able to terminate the task:
+			 * a non-fatal signal no longer aborts the wait, but the task
+			 * can never get stuck for the multi-thousand-second long
+			 * timeout and trip the hung-task watchdog.
+			 */
+			if (is_rsp_wait_with_timeout && !wait_killable &&
+			    READ_ONCE(se_clbk_hdl->rx_msg)) {
+				WRITE_ONCE(se_clbk_hdl->signal_rcvd, true);
+				wait_killable = true;
+				continue;
+			}
+
+			/*
+			 * Command-receiver path, or a fatal signal on the
+			 * killable path: the task is dying but the enclave may
+			 * still DMA into the soon-to-be-freed buffer. Under
+			 * clbk_rx_lock, drop rx_msg and arm fw_busy so a late
+			 * callback cannot write freed memory. Exception: if
+			 * rx_delivered is set a real response already landed, so
+			 * report its size and keep the handle.
+			 */
+			if (is_rsp_wait_with_timeout) {
+				spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+				if (se_clbk_hdl->rx_delivered) {
+					ret = se_clbk_hdl->rx_msg_sz;
+					spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+					break;
+				}
+				if (se_clbk_hdl->rx_msg) {
+					se_clbk_hdl->rx_msg = NULL;
+					se_mark_fw_busy(dev_ctx);
+				}
+				spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+			}
+			break;
+		}
+
+		if (ret == 0) {
+			/*
+			 * The response buffer belongs to the caller of ele_msg_send_rcv()
+			 * and may be freed as soon as this function returns. Clear rx_msg
+			 * under clbk_rx_lock so that a late se_if_rx_callback() can
+			 * observe that the waiter has timed out and must not copy into
+			 * the stale buffer.
+			 *
+			 * If the completion has not yet been signaled, mark the firmware
+			 * path busy. This acts as a circuit breaker: reject new
+			 * command/response transactions until the delayed response
+			 * arrives and the callback closes the breaker.
+			 */
+
+			spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+			se_clbk_hdl->rx_msg = NULL;
+			/*
+			 * rx_delivered helps to decide if the circuit breaker is armed
+			 * or not. rx_delivered is set only after a real response has
+			 * been copied under clbk_rx_lock, so it correctly distinguishes
+			 * a genuine timeout (no response → mark busy) from a spurious
+			 * teardown-forced wakeup where the data is not yet safe to free.
+			 */
+			if (!se_clbk_hdl->rx_delivered)
+				se_mark_fw_busy(dev_ctx);
+
+			spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+			ret = -ETIMEDOUT;
+			dev_err(priv->dev,
+				"Fatal Error: SE interface %s0, hangs indefinitely.\n",
+				get_se_if_name(priv->if_defs->se_if_type));
+			break;
+		}
+
+		/*
+		 * A positive wait return normally means a real response. During
+		 * teardown, se_if_probe_cleanup() forces this wait to return via
+		 * complete_all() with no response, while the enclave may still
+		 * DMA into the shared buffer. Treat that as a failed transaction
+		 * and arm the circuit breaker so the buffer is quarantined, not
+		 * freed.
+		 *
+		 * rx_delivered tells the two apart: se_if_rx_callback() sets it
+		 * under clbk_rx_lock only after copying a real response. This
+		 * keeps teardown-time session/storage close responses from being
+		 * mistaken for the forced abort, which would fail the close and
+		 * leak its DMA buffer.
+		 */
+		spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+		if (is_rsp_wait_with_timeout && atomic_read(&priv->going_away) &&
+		    !se_clbk_hdl->rx_delivered) {
+			se_clbk_hdl->rx_msg = NULL;
+			se_mark_fw_busy(dev_ctx);
+			spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+			ret = -ENODEV;
+			break;
+		}
+		spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+
+		ret = se_clbk_hdl->rx_msg_sz;
+		break;
+
+	} while (ret < 0);
+
+	return ret;
+}
+
+/**
+ * ele_msg_send() - send a message to the secure enclave over the mailbox.
+ * @dev_ctx: pointer to the SE device context.
+ * @tx_msg: buffer containing the message to send.
+ * @tx_msg_sz: size of @tx_msg in bytes; must match the size field in the
+ *             message header.
+ *
+ * Copies the message into the MU TX registers via the mailbox framework.
+ * The MU controller does not retain the caller's buffer after this call
+ * returns, so the caller may free @tx_msg immediately on success.
+ *
+ * Return: @tx_msg_sz on success, negative errno on error.
+ */
+int ele_msg_send(struct se_if_device_ctx *dev_ctx,
+		 void *tx_msg,
+		 int tx_msg_sz)
+{
+	struct se_msg_hdr *header = tx_msg;
+	int err;
+
+	/*
+	 * Check that the size passed as argument matches the size
+	 * carried in the message.
+	 */
+	if (header->size << 2 != tx_msg_sz) {
+		dev_err(dev_ctx->priv->dev,
+			"%s: User buf hdr: 0x%x, sz mismatched with input-sz (%d != %d).\n",
+			dev_ctx->devname, *(u32 *)header, header->size << 2, tx_msg_sz);
+		return -EINVAL;
+	}
+
+	/*
+	 * The i.MX MU mailbox controller copies the payload words into MU
+	 * registers synchronously from its send path. It does not retain the
+	 * caller-provided tx_msg pointer after mbox_send_message() returns, so
+	 * the caller-owned buffer may be released after a successful send.
+	 */
+	err = mbox_send_message(dev_ctx->priv->tx_chan, tx_msg);
+	if (err < 0) {
+		dev_err(dev_ctx->priv->dev,
+			"%s: Error: mbox_send_message failure.\n", dev_ctx->devname);
+		return err;
+	}
+
+	return tx_msg_sz;
+}
+
+static void ele_msg_send_rcv_cleanup(struct se_if_priv *priv, int *act_rx_msg_sz)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+	priv->waiting_rsp_clbk_hdl.dev_ctx = NULL;
+	priv->waiting_rsp_clbk_hdl.rx_msg = NULL;
+	if (act_rx_msg_sz)
+		*act_rx_msg_sz = priv->waiting_rsp_clbk_hdl.rx_msg_sz;
+	priv->waiting_rsp_clbk_hdl.rx_msg_sz = 0;
+	spin_unlock_irqrestore(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+}
+
+/**
+ * ele_msg_send_rcv() - send a command and wait for the response.
+ * @dev_ctx: pointer to the dev_ctx data.
+ * @tx_msg: buffer containing the command message to send.
+ * @tx_msg_sz: size of @tx_msg in bytes.
+ * @rx_msg: caller-provided buffer to receive the response into.
+ * @exp_rx_msg_sz: expected response size in bytes.
+ * @act_rx_msg_sz: optional output pointer; if non-NULL, receives the actual
+ *                 number of bytes copied into @rx_msg (min of FW-declared
+ *                 size and @exp_rx_msg_sz). Pass NULL when the caller does not
+ *                 need this value.
+ *
+ * Holds the SE command lock for the duration of the exchange to prevent
+ * concurrent transactions.  Signals are deferred until the protocol
+ * resynchronizes; -ERESTARTSYS is returned to the caller after a clean
+ * response is received if a signal arrived during the wait.
+ *
+ * Return: number of bytes received on success, negative errno on error.
+ */
+int ele_msg_send_rcv(struct se_if_device_ctx *dev_ctx, void *tx_msg,
+		     int tx_msg_sz, void *rx_msg, int exp_rx_msg_sz, int *act_rx_msg_sz)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct fw_busy_info *fbusy_info = &priv->fw_busy_info;
+	struct task_struct *msg_excl_owner;
+	unsigned long flags;
+	int err;
+
+	guard(mutex)(&priv->se_if_cmd_lock);
+
+	/*
+	 * Arm under clbk_rx_lock so the going_away check and arming are atomic
+	 * against teardown (closes the lost-wakeup window); priv_dev_ctx close
+	 * commands still pass. Check going_away before fw_busy so a caller
+	 * racing unbind gets the permanent -ENODEV, not the retryable -EBUSY -
+	 * these are deliberately distinct from the transient msg_if reservation.
+	 */
+
+	spin_lock_irqsave(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+	msg_excl_owner = READ_ONCE(priv->msg_excl_flow.msg_excl_owner);
+	if (atomic_read(&priv->going_away) && msg_excl_owner != current) {
+		spin_unlock_irqrestore(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+		return -ENODEV;
+	}
+	/*
+	 * fw_busy is the circuit breaker: while it is set, reject new
+	 * transactions with -EBUSY. The one exception is a flow that has
+	 * reserved this interface exclusively for itself via se_reserve_msg_if()
+	 * by publishing its task in priv->msg_excl_flow.msg_excl_owner (e.g.
+	 * the recovery flow in se_clear_fw_busy()). Only that owning task is let
+	 * through here to issue its teardown-close messages; every other caller
+	 * still gets -EBUSY. When the flow calls se_release_msg_if() the owner
+	 * is cleared and the interface returns to general se_if_cmd_lock message
+	 * exchange. The owner is only compared against current, so a stale read
+	 * is harmless (a non-owner can never match) and no msg_excl_lock is
+	 * needed here; there is no deadlock.
+	 */
+	if (atomic_read(&fbusy_info->fw_busy) && msg_excl_owner != current) {
+		spin_unlock_irqrestore(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+		return -EBUSY;
+	}
+
+	reinit_completion(&priv->waiting_rsp_clbk_hdl.done);
+	priv->waiting_rsp_clbk_hdl.dev_ctx = dev_ctx;
+	priv->waiting_rsp_clbk_hdl.rx_msg_sz = exp_rx_msg_sz;
+	priv->waiting_rsp_clbk_hdl.rx_msg = rx_msg;
+	/*
+	 * Arm a fresh transaction: clear the delivered flag so a stale value
+	 * from a previous response cannot make ele_msg_rcv() mistake a
+	 * teardown-forced complete_all() for a genuine firmware response.
+	 */
+	priv->waiting_rsp_clbk_hdl.rx_delivered = false;
+	spin_unlock_irqrestore(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+
+	err = ele_msg_send(dev_ctx, tx_msg, tx_msg_sz);
+	if (err < 0) {
+		ele_msg_send_rcv_cleanup(priv, NULL);
+		return err;
+	}
+
+	err = ele_msg_rcv(dev_ctx, &priv->waiting_rsp_clbk_hdl);
+
+	if (priv->waiting_rsp_clbk_hdl.signal_rcvd) {
+		/*
+		 * Signal was deferred until the FW/kernel protocol resynchronized.
+		 * On success report -ERESTARTSYS for the interrupted wait; the
+		 * command is not re-sent. Keep real errors like -ETIMEDOUT.
+		 */
+		if (err > 0)
+			err = -ERESTARTSYS;
+		priv->waiting_rsp_clbk_hdl.signal_rcvd = false;
+		dev_dbg(priv->dev, "%s: Err[0x%x]:Interrupted by signal.\n",
+			dev_ctx->devname, err);
+	}
+
+	ele_msg_send_rcv_cleanup(priv, act_rx_msg_sz);
+
+	return err;
+}
+
+static bool check_hdr_exception_for_sz(struct se_if_priv *priv,
+				       struct se_msg_hdr *header)
+{
+	/*
+	 * List of API headers that can accept a variable length response buffer.
+	 */
+	if (header->command == ELE_DEBUG_DUMP_REQ &&
+	    header->ver == priv->if_defs->base_api_ver &&
+	    header->size >= 2 && header->size <= (ELE_DEBUG_DUMP_RSP_SZ / 4))
+		return true;
+
+	return false;
+}
+
+/**
+ * se_if_rx_callback() - mailbox RX callback for secure enclave messages.
+ * @mbox_cl: mailbox client registered for this SE interface.
+ * @msg: pointer to the received message buffer; may be NULL or an ERR_PTR.
+ *
+ * Dispatches the incoming message to either the command receiver (cmd_tag)
+ * or the synchronous response waiter (rsp_tag).  Called from mailbox IRQ
+ * context; must not sleep.
+ */
+void se_if_rx_callback(struct mbox_client *mbox_cl, void *msg)
+{
+	struct se_clbk_handle *se_clbk_hdl;
+	struct device *dev = mbox_cl->dev;
+	struct fw_busy_info *fbusy_info;
+	struct se_msg_hdr *header;
+	bool sz_mismatch = false;
+	struct se_if_priv *priv;
+	/*
+	 * devname_snap: a local copy of dev_ctx->devname taken while
+	 * clbk_rx_lock is held.
+	 */
+	char devname_snap[32];
+	unsigned long flags;
+	u32 rx_msg_sz;
+
+	priv = dev_get_drvdata(dev);
+	if (!priv)
+		return;
+
+	fbusy_info = &priv->fw_busy_info;
+
+	/* The function can be called with NULL msg */
+	if (IS_ERR_OR_NULL(msg)) {
+		dev_err(dev, "Message is invalid\n");
+		return;
+	}
+
+	header = msg;
+	rx_msg_sz = header->size << 2;
+
+	/* Incoming command: wake up the receiver if any. */
+	if (header->tag == priv->if_defs->cmd_tag) {
+		se_clbk_hdl = &priv->cmd_receiver_clbk_hdl;
+		spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+		if (!se_clbk_hdl->dev_ctx || !se_clbk_hdl->rx_msg) {
+			spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+			dev_warn(dev, "No command receiver registered for message: %.8x\n",
+				 *((u32 *)header));
+			return;
+		}
+
+		/*
+		 * cmd_tag messages are delivered only to the explicitly registered
+		 * command receiver. Unlike the synchronous response waiter path, the
+		 * command receiver uses a dedicated long-lived buffer installed by
+		 * SE_IOCTL_ENABLE_CMD_RCV and is not subject to the timeout/circuit-
+		 * breaker handling used for rsp_tag messages.
+		 */
+		dev_dbg(dev, "Selecting cmd receiver:%s for mesg header:0x%x.\n",
+			se_clbk_hdl->dev_ctx->devname,  *(u32 *)header);
+
+		/*
+		 * Pre-allocated buffer of MAX_NVM_MSG_LEN
+		 * as the NVM command are initiated by FW.
+		 * Size is revealed as part of this call function.
+		 */
+
+		if (rx_msg_sz > MAX_NVM_MSG_LEN)
+			sz_mismatch = true;
+
+		/*
+		 * Clamp the copy length to the pre-allocated receiver buffer (MAX_NVM_MSG_LEN).
+		 */
+		se_clbk_hdl->rx_msg_sz = min(rx_msg_sz, MAX_NVM_MSG_LEN);
+		strscpy(devname_snap, se_clbk_hdl->dev_ctx->devname,
+			sizeof(devname_snap));
+		memcpy(se_clbk_hdl->rx_msg, msg, se_clbk_hdl->rx_msg_sz);
+		complete(&se_clbk_hdl->done);
+		spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+		if (sz_mismatch)
+			dev_err(dev,
+				"%s: CMD-RCVER NVM: hdr(0x%x) with different sz(%d != %d).\n",
+				devname_snap, *(u32 *)header,
+				(header->size << 2), rx_msg_sz);
+	} else if (header->tag == priv->if_defs->rsp_tag) {
+		bool exception_for_sz_mismatch = check_hdr_exception_for_sz(priv, header);
+		u32 exp_rx_msg_sz;
+
+		/*
+		 * rx_msg and rx_msg_sz are owned by the sender under clbk_rx_lock.
+		 * Read both under the lock: drop a late response instead of copying
+		 * into freed memory, and avoid a stale size. A late response also
+		 * closes the firmware-busy circuit breaker.
+		 */
+		se_clbk_hdl = &priv->waiting_rsp_clbk_hdl;
+		spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+		if (!se_clbk_hdl->rx_msg) {
+			/*
+			 * Schedule fw_busy_work only while going_away is clear:
+			 * teardown sets going_away under this lock before
+			 * cancel_work_sync(), so scheduling after that would
+			 * re-queue against freed priv (UAF). schedule_work()
+			 * only enqueues (the handler runs later in process
+			 * context, so se_clear_fw_busy()'s mutex is not taken
+			 * under this spinlock). se_clear_fw_busy() also runs
+			 * from teardown; both take fw_busy_lock first, so the
+			 * second caller sees NULL and returns.
+			 */
+			if (atomic_read(&fbusy_info->fw_busy)) {
+				/*
+				 * Snapshot the late response before scheduling
+				 * fw_busy_work. se_clear_fw_busy() will parse
+				 * this buffer to detect a session-open or
+				 * storage-open response and immediately close
+				 * the leaked firmware handle via
+				 * fw_api_specific_ops(). The buffer is sized
+				 * to MAX_ALLOWED_RX_MSG_SZ; clamp the copy
+				 * length so an oversized FW message cannot
+				 * overflow it.
+				 */
+				memcpy(fbusy_info->orphan_fw_rx_msg, msg,
+				       min(rx_msg_sz,
+					   (u32)MAX_ALLOWED_RX_MSG_SZ));
+				if (!atomic_read(&priv->going_away))
+					schedule_work(&fbusy_info->fw_busy_work);
+			}
+			spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+
+			dev_info(dev, "ELE responded (late), recovery FW available.\n");
+			return;
+		}
+		exp_rx_msg_sz = se_clbk_hdl->rx_msg_sz;
+		dev_dbg(dev, "Selecting resp waiter:%s for mesg header:0x%x.\n",
+			se_clbk_hdl->dev_ctx->devname, *(u32 *)header);
+
+		/*
+		 * For rsp_tag traffic, the sender provides the expected response
+		 * buffer size. If firmware returns a different size, clamp the copy
+		 * length to the caller's buffer capacity before memcpy() and report the
+		 * mismatch after dropping the spinlock.
+		 */
+		if (rx_msg_sz != exp_rx_msg_sz && !exception_for_sz_mismatch)
+			sz_mismatch = true;
+
+		se_clbk_hdl->rx_msg_sz = min(rx_msg_sz, exp_rx_msg_sz);
+		/* Snapshot devname before complete() can free the context. */
+		strscpy(devname_snap, se_clbk_hdl->dev_ctx->devname,
+			sizeof(devname_snap));
+		memcpy(se_clbk_hdl->rx_msg, msg, se_clbk_hdl->rx_msg_sz);
+		/*
+		 * Mark that a genuine firmware response was delivered. ele_msg_rcv()
+		 * reads this under clbk_rx_lock to avoid mistaking this response for
+		 * a teardown-forced complete_all() wakeup.
+		 */
+		se_clbk_hdl->rx_delivered = true;
+		complete(&se_clbk_hdl->done);
+		spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+
+		if (sz_mismatch)
+			dev_err(dev,
+				"%s: Rsp to CMD: hdr(0x%x) with different sz(%d != %d).\n",
+				devname_snap, *(u32 *)header,
+				(header->size << 2), exp_rx_msg_sz);
+	} else {
+		dev_err(dev, "Failed to select a device for message: %.8x\n",
+			*((u32 *)header));
+	}
+}
+
+/**
+ * se_val_rsp_hdr_n_status() - validate a response message header and status.
+ * @dev_ctx: pointer to the SE device context.
+ * @msg: response message buffer to validate.
+ * @msg_id: expected command identifier.
+ * @sz: expected message size in bytes.
+ * @version: expected API version byte from the command message header
+ *           (tx_msg->header.ver); the response header->ver must match this
+ *           value exactly.
+ *
+ * Checks the response tag, command id, API version, status word, and
+ * response size. The size check is performed after the status check so that
+ * a well-formed response whose FW-declared size differs from the userspace
+ * buffer size still has its status extracted and its FW handle recorded.
+ * Size mismatch returns -ENOSPC (not -EINVAL) so callers can distinguish it
+ * from a header field mismatch and still run fw_api_specific_ops().
+ *
+ * Return: 0 on success, -EINVAL if tag/command/version mismatch, -EPERM if
+ * the firmware status indicates a command failure, -ENOSPC if the response
+ * size does not match @sz.
+ */
+int se_val_rsp_hdr_n_status(struct se_if_device_ctx *dev_ctx, struct se_api_msg *msg,
+			    u8 msg_id, u8 sz, u8 version)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_msg_hdr *header = &msg->header;
+	u32 status;
+
+	if (header->tag != priv->if_defs->rsp_tag) {
+		dev_dbg(priv->dev, "MSG[0x%x] Hdr: Resp tag mismatch. (0x%x != 0x%x)\n",
+			msg_id, header->tag, priv->if_defs->rsp_tag);
+		return -EINVAL;
+	}
+
+	if (header->command != msg_id) {
+		dev_dbg(priv->dev, "MSG Header: Cmd id mismatch. (0x%x != 0x%x)\n",
+			header->command, msg_id);
+		return -EINVAL;
+	}
+
+	if (header->ver != version) {
+		dev_dbg(priv->dev,
+			"MSG[0x%x] Hdr: API Vers mismatch. (0x%x != 0x%x)\n",
+			msg_id, header->ver, version);
+		return -EINVAL;
+	}
+
+	if (header->size > SE_MU_HDR_WORD_SZ && (sz >> 2) > SE_MU_HDR_WORD_SZ) {
+		status = RES_STATUS(msg->data[0]);
+		if (status != priv->if_defs->success_tag) {
+			dev_dbg(priv->dev, "Command Id[%x], Response Failure = 0x%x\n",
+				header->command, status);
+			return -EPERM;
+		}
+	}
+
+	if ((sz % 4) || (header->size != (sz >> 2) &&
+			 !check_hdr_exception_for_sz(priv, header))) {
+		dev_dbg(priv->dev, "MSG[0x%x] Hdr: Cmd size mismatch. (0x%x != 0x%x)\n",
+			msg_id, header->size, (sz >> 2));
+		return -ENOSPC;
+	}
+
+	return 0;
+}
+
+/**
+ * se_save_imem_state() - export and save the encrypted IMEM state.
+ * @priv: pointer to the SE interface private data.
+ * @imem: IMEM buffer descriptor; @imem->daddr must point to a DMA-coherent
+ *        buffer of at least ELE_IMEM_SIZE bytes.
+ *
+ * Issues an ELE_IMEM_EXPORT service-swap command to save the current IMEM
+ * content into the pre-allocated DMA buffer.  Intended to be called during
+ * system suspend.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+int se_save_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem)
+{
+	struct ele_dev_info s_info = {0};
+	int ret;
+
+	ret = ele_get_info(priv, &s_info);
+	if (ret) {
+		dev_err(priv->dev, "Failed to get info from ELE.\n");
+		return ret;
+	}
+
+	/* Check for the imem-state before continue to save imem state. */
+	if (s_info.d_addn_info.imem_state == ELE_IMEM_STATE_BAD)
+		return 0;
+
+	/*
+	 * EXPORT command will save encrypted IMEM to given address,
+	 * so later in resume, IMEM can be restored from the given
+	 * address.
+	 *
+	 * Size must be at least 64 kB.
+	 */
+	ret = ele_service_swap(priv, imem->daddr, ELE_IMEM_SIZE, ELE_IMEM_EXPORT);
+	if (ret < 0) {
+		dev_err(priv->dev, "Failed to export IMEM.\n");
+		imem->size = 0;
+	} else if (ret > ELE_IMEM_SIZE) {
+		dev_err(priv->dev, "Invalid exported IMEM size %d.\n", ret);
+		imem->size = 0;
+		ret = -EIO;
+	} else {
+		dev_dbg(priv->dev,
+			"Exported %d bytes of encrypted IMEM.\n",
+			ret);
+		imem->size = ret;
+	}
+
+	return ret > 0 ? 0 : ret;
+}
+
+/**
+ * se_restore_imem_state() - restore the encrypted IMEM state after resume.
+ * @priv: pointer to the SE interface private data.
+ * @imem: IMEM buffer descriptor populated by a prior se_save_imem_state()
+ *        call; @imem->size must be non-zero.
+ *
+ * Issues an ELE_IMEM_IMPORT service-swap command to restore IMEM from the
+ * saved DMA buffer, then verifies that the enclave reports
+ * ELE_IMEM_STATE_OK.  Intended to be called during system resume.
+ *
+ * Return: 0 on success, -EIO if IMEM state is not OK after import, or
+ * another negative errno on communication failure.
+ */
+int se_restore_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem)
+{
+	struct ele_dev_info s_info;
+	int ret;
+
+	/* get info from ELE */
+	ret = ele_get_info(priv, &s_info);
+	if (ret) {
+		dev_err(priv->dev, "Failed to get info from ELE.\n");
+		return ret;
+	}
+	imem->state = s_info.d_addn_info.imem_state;
+
+	/* Check for the imem-state and imem-size before continue to
+	 * restore imem state.
+	 */
+	if (s_info.d_addn_info.imem_state != ELE_IMEM_STATE_BAD || !imem->size)
+		return 0;
+
+	/*
+	 * IMPORT command will restore IMEM from the given
+	 * address, here size is the actual size returned by ELE
+	 * during the export operation
+	 */
+	ret = ele_service_swap(priv, imem->daddr, imem->size, ELE_IMEM_IMPORT);
+	if (ret) {
+		dev_err(priv->dev, "Failed to import IMEM\n");
+		return ret;
+	}
+
+	/*
+	 * After importing IMEM, check if IMEM state is equal to 0xCA
+	 * to ensure IMEM is fully loaded and
+	 * ELE functionality can be used.
+	 */
+	ret = ele_get_info(priv, &s_info);
+	if (ret) {
+		dev_err(priv->dev, "Failed to get info from ELE.\n");
+		return ret;
+	}
+	imem->state = s_info.d_addn_info.imem_state;
+
+	if (s_info.d_addn_info.imem_state == ELE_IMEM_STATE_OK) {
+		dev_dbg(priv->dev, "Successfully restored IMEM.\n");
+	} else {
+		dev_err(priv->dev, "Failed to restore IMEM: state=0x%02x, expected 0x%02x.\n",
+			s_info.d_addn_info.imem_state, ELE_IMEM_STATE_OK);
+		/*
+		 * ele_get_info() succeeded (ret == 0) but the IMEM state
+		 * reported by the hardware is not ELE_IMEM_STATE_OK. Return
+		 * -EIO so the PM subsystem knows the enclave is non-functional
+		 * after resume, instead of silently continuing with bad state.
+		 */
+		ret = -EIO;
+	}
+
+	return ret;
+}
diff --git a/drivers/firmware/imx/ele_common.h b/drivers/firmware/imx/ele_common.h
new file mode 100644
index 00000000000000..332b35eb7db243
--- /dev/null
+++ b/drivers/firmware/imx/ele_common.h
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2025 NXP
+ */
+
+#ifndef __ELE_COMMON_H__
+#define __ELE_COMMON_H__
+
+#include "se_ctrl.h"
+
+#define SE_RCV_MSG_DEFAULT_TIMEOUT_MS	3000
+#define SE_RCV_MSG_LONG_TIMEOUT_MS	5000000
+
+#define ELE_SUCCESS_IND			0xD6
+
+#define IMX_ELE_FW_DIR                 "imx/ele/"
+
+#define MAX_WORD_SIZE			0x20
+
+void set_se_rcv_msg_timeout(struct se_if_device_ctx *dev_ctx, u32 val);
+int se_update_msg_chksum(u32 *msg, u32 msg_len);
+
+int ele_msg_rcv(struct se_if_device_ctx *dev_ctx, struct se_clbk_handle *se_clbk_hdl);
+
+int ele_msg_send(struct se_if_device_ctx *dev_ctx, void *tx_msg, int tx_msg_sz);
+
+int ele_msg_send_rcv(struct se_if_device_ctx *dev_ctx, void *tx_msg,
+		     int tx_msg_sz, void *rx_msg, int exp_rx_msg_sz,
+		     int *act_rx_msg_sz);
+
+void se_if_rx_callback(struct mbox_client *mbox_cl, void *msg);
+
+int se_val_rsp_hdr_n_status(struct se_if_device_ctx *dev_ctx, struct se_api_msg *msg,
+			    u8 msg_id, u8 sz, u8 version);
+
+/*
+ * Header that prefixes the input buffer of commands whose descriptor uses
+ * SE_CMD_ADDR_DEDUCE_SZ (e.g. ELE_OEM_AUTH_CONTAINER_REQ,
+ * ELE_KEYSTORE_REPROV_ENABLE_REQ). The command payload carries only the start
+ * address of that buffer, but its first four bytes follow this fixed layout:
+ * the 16-bit length field gives the total byte count of the buffer, so
+ * se_val_cmd_addrs() can bound-check the whole buffer, not just its start. The
+ * length is little-endian on the wire; read it with le16_to_cpu().
+ */
+struct fw_tag_len_vers_info {
+	u8 version;
+	__le16 length;
+	u8 tag;
+} __packed;
+
+/**
+ * struct se_cmd_addr_field - One DMA address embedded in an ELE command
+ *                            message payload.
+ *
+ * A number of ELE commands carry DMA physical addresses inside their message
+ * payload. se_val_cmd_addrs() range-checks each such address against the
+ * calling context's shared-memory window before the message reaches firmware.
+ *
+ * data[] index = message WORD index - 1, because the 4-byte se_msg_hdr is
+ * message WORD 0 and se_api_msg.data[0] is message WORD 1.
+ *
+ * @lsb_idx: data[] index of the low 32 bits of the address.
+ * @msb_idx: data[] index of the high 32 bits; valid only when @has_msb is
+ *           set. Some base-API commands split the address into two words;
+ *           FW-API commands do not.
+ * @has_msb: true when the address occupies two words (@lsb_idx + @msb_idx).
+ * @flag_idx: data[] index of the flag word that selects whether
+ *            data[@lsb_idx] is a DMA address or an integer key identifier;
+ *            %SE_CMD_ADDR_ALWAYS when the word is always a DMA address.
+ * @flag_mask: the selecting flag bit, already shifted to its position inside
+ *             the 32-bit little-endian flag word.
+ * @is_addr_when_set: true when the word is a DMA address if the flag bit is
+ *                    set; false when it is an address if the bit is clear
+ *                    (inverse polarity, e.g. VERIFY_SIGN OPAQUE_KEY).
+ * @size_idx: data[] index of the word carrying the length in bytes of the
+ *            buffer at this address, or one of the size-source sentinels
+ *            below. se_val_cmd_addrs() uses the resulting length to confirm
+ *            the whole buffer [addr, addr + len) fits inside the
+ *            shared-memory window, not just its start.
+ *            %SE_CMD_ADDR_FIXED_SIZE when the length is a firmware-defined
+ *            literal supplied in @buf_size.
+ *            %SE_CMD_RCVR_ADDR_VAR_SIZE when the length is taken from
+ *            se_if_priv.cmd_rcvr_var_size (command-receiver responses).
+ *            %SE_CMD_ADDR_DEDUCE_SZ when no length word is carried in the
+ *            payload, but the buffer starts with a struct fw_tag_len_vers_info
+ *            header whose length field gives the total buffer byte count;
+ *            se_val_cmd_addrs() reads that header to range-check the whole
+ *            buffer.
+ * @size_shift: right shift applied to the size word before masking, for a
+ *              length packed into the high half of a word.
+ * @size_mask: bitmask applied after @size_shift to extract the length from
+ *             the message word (0xFFFFFFFF for a full 32-bit length, 0xFFFF
+ *             for a u16, 0xFF for a u8). Used only when @size_idx names a
+ *             payload word; zero for the sentinel values.
+ * @buf_size: firmware-defined literal byte count. Used only when
+ *            @size_idx == %SE_CMD_ADDR_FIXED_SIZE, where the whole buffer
+ *            [addr, addr + @buf_size) must fit inside the shared-memory
+ *            window. Left zero for every other @size_idx value.
+ */
+struct se_cmd_addr_field {
+	u8 lsb_idx;
+	u8 msb_idx;
+	bool has_msb;
+	u8 flag_idx;
+	u32 flag_mask;
+	bool is_addr_when_set;
+	u8 size_idx;
+	u8 size_shift;
+	u32 size_mask;
+	u32 buf_size;
+};
+
+#define SE_CMD_ADDR_ALWAYS		0xEFu
+/* size is the literal in buf_size */
+#define SE_CMD_ADDR_FIXED_SIZE		0xFDu
+/* size from se_if_priv.cmd_rcvr_var_size */
+#define SE_CMD_RCVR_ADDR_VAR_SIZE	0xFEu
+/* size read from the buffer's fw_tag_len_vers_info header */
+#define SE_CMD_ADDR_DEDUCE_SZ		0xFFu
+
+int se_val_cmd_addrs(struct se_if_device_ctx *dev_ctx, struct se_api_msg *msg,
+		     u32 tx_msg_sz, const struct se_cmd_addr_field *fields,
+		     size_t count);
+
+const struct se_cmd_addr_field *ele_fw_cmd_addr_fields(u8 cmd, size_t *count);
+const struct se_cmd_addr_field *ele_fw_rsp_addr_fields(u8 cmd, size_t *count);
+const struct se_cmd_addr_field *ele_base_cmd_addr_fields(u8 cmd, size_t *count);
+/* Fill a command message header with a given command ID and length in bytes. */
+static inline void se_fill_cmd_msg_hdr(struct se_if_priv *priv, struct se_msg_hdr *hdr,
+				       u8 cmd, u32 len, bool is_base_api)
+{
+	hdr->tag = priv->if_defs->cmd_tag;
+	hdr->ver = (is_base_api) ? priv->if_defs->base_api_ver : priv->if_defs->fw_api_ver;
+	hdr->command = cmd;
+	hdr->size = len >> 2;
+}
+
+int se_save_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem);
+
+int se_restore_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem);
+
+int se_chk_tx_rsp_msg_hdr(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			  u32 tx_msg_sz);
+int se_chk_tx_cmd_msg_hdr(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			  u32 tx_msg_sz, u32 rx_msg_sz);
+
+#endif /*__ELE_COMMON_H__ */
diff --git a/drivers/firmware/imx/ele_fw_api.c b/drivers/firmware/imx/ele_fw_api.c
new file mode 100644
index 00000000000000..a3a68234bcb1a1
--- /dev/null
+++ b/drivers/firmware/imx/ele_fw_api.c
@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2026 NXP
+ */
+
+#include <linux/slab.h>
+
+#include "se_ctrl.h"
+#include "ele_common.h"
+#include "ele_fw_api.h"
+
+static int se_cmd_receiver_allowed_cmd(struct se_if_device_ctx *dev_ctx,
+				       struct se_api_msg *msg, u32 tx_msg_sz)
+{
+	u8 cmd = msg->header.command;
+
+	switch (cmd) {
+	case ELE_SESSION_CLOSE_REQ:
+		if (tx_msg_sz < ELE_SESSION_CLOSE_REQ_SZ ||
+		    msg->data[0] != dev_ctx->sess_hdl)
+			return -EINVAL;
+		return 0;
+	case ELE_STORAGE_CLOSE_REQ:
+		if (tx_msg_sz < ELE_STORAGE_CLOSE_REQ_SZ ||
+		    msg->data[0] != dev_ctx->strg_hdl)
+			return -EINVAL;
+		return 0;
+	case ELE_STORAGE_MASTER_IMPORT_REQ: {
+		const struct se_cmd_addr_field *fields;
+		size_t count;
+
+		fields = ele_fw_cmd_addr_fields(cmd, &count);
+		return se_val_cmd_addrs(dev_ctx, msg, tx_msg_sz, fields, count);
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int se_cmd_receiver_allowed_rsp(struct se_if_device_ctx *dev_ctx,
+				       struct se_api_msg *msg, u32 tx_msg_sz)
+{
+	struct cmd_rcvr_data_info *crcvr_info = &dev_ctx->priv->crcvr_info;
+	const struct se_cmd_addr_field *fields;
+	u8 cmd = msg->header.command;
+	size_t count;
+
+	switch (cmd) {
+	case ELE_STORAGE_EXPORT_FINISH_REQ:
+	case ELE_STORAGE_CHUNK_GET_DONE_REQ:
+	case ELE_STORAGE_CHUNK_DELETE_REQ:
+		return 0;
+	default:
+		/*
+		 * These responses supply a kernel buffer address to firmware.
+		 * Range-check the embedded DMA address against the calling
+		 * context's shared-memory window before the message is sent.
+		 */
+		if (crcvr_info->cmd_rcvr_last_rcvd_cmd_id != cmd)
+			return -EINVAL;
+
+		fields = ele_fw_rsp_addr_fields(cmd, &count);
+		if (!count)
+			return -EOPNOTSUPP;
+
+		return se_val_cmd_addrs(dev_ctx, msg, tx_msg_sz, fields, count);
+	}
+}
+
+int ele_uapi_allowed_fw_rsp(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			    u32 tx_msg_sz)
+{
+	struct se_api_msg *msg = container_of(header, struct se_api_msg, header);
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	scoped_guard(mutex, &priv->modify_lock)
+		if (dev_ctx != priv->cmd_receiver_clbk_hdl.dev_ctx)
+			return -EINVAL;
+
+	return se_cmd_receiver_allowed_rsp(dev_ctx, msg, tx_msg_sz);
+}
+
+int ele_uapi_allowed_fw_cmd(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			    u32 tx_msg_sz, u32 rx_msg_sz)
+{
+	struct se_api_msg *msg = container_of(header, struct se_api_msg, header);
+	struct se_if_priv *priv = dev_ctx->priv;
+	const struct se_cmd_addr_field *fields;
+	bool receiver_exists = false;
+	bool is_cmd_receiver = false;
+	size_t count;
+	int ret = 0;
+
+	scoped_guard(mutex, &priv->modify_lock) {
+		if (priv->cmd_receiver_clbk_hdl.dev_ctx)
+			receiver_exists = true;
+		if (dev_ctx == priv->cmd_receiver_clbk_hdl.dev_ctx)
+			is_cmd_receiver = true;
+	}
+
+	if (is_cmd_receiver && header->tag == priv->if_defs->cmd_tag)
+		return se_cmd_receiver_allowed_cmd(dev_ctx, msg, tx_msg_sz);
+
+	/* Reject any response message with non-command receiver */
+	if (header->tag == priv->if_defs->rsp_tag)
+		return -EOPNOTSUPP;
+
+	/* Reject any other tag */
+	if (header->tag != priv->if_defs->cmd_tag)
+		return -EOPNOTSUPP;
+
+	/*
+	 * Identify the command first. Session/storage commands enforce their
+	 * own-handle checks; crypto commands that embed DMA addresses defer to
+	 * the shared range check below. Any command not named here is left with
+	 * ret == 0 (permitted) as before.
+	 */
+	switch (header->command) {
+	case ELE_SESSION_OPEN_REQ:
+		/* Might be cleared as part of tear down. */
+		ret = dev_ctx->sess_hdl ? -EEXIST : 0;
+		if (rx_msg_sz < ELE_SESSION_OPEN_RSP_SZ)
+			ret = -EINVAL;
+		break;
+	case ELE_SESSION_CLOSE_REQ:
+		/* Might be cleared as part of tear down. */
+		if (!dev_ctx->sess_hdl) {
+			ret = -ENXIO;
+			break;
+		}
+		/*
+		 * A close request must target this context's own session. The
+		 * handle to close is carried in the payload (data[0]); reject a
+		 * request whose buffer is too short to hold it, or whose handle
+		 * does not match this context. Checking the buffer size first
+		 * also keeps the data[0] read in bounds. This stops one process
+		 * from closing - and leaking - another process's session with a
+		 * spoofed handle.
+		 */
+		if (tx_msg_sz < ELE_SESSION_CLOSE_REQ_SZ ||
+		    msg->data[0] != dev_ctx->sess_hdl)
+			ret = -EINVAL;
+		break;
+	case ELE_FW_GET_INFO_REQ:
+	case ELE_KEY_STORE_OPEN_REQ:
+	case ELE_KEY_STORE_CLOSE_REQ:
+	case ELE_KEY_MGMT_OPEN_REQ:
+	case ELE_KEY_MGMT_CLOSE_REQ:
+	case ELE_MANAGE_KEY_GROUP_REQ:
+	case ELE_GET_KEY_ATTR_REQ:
+	case ELE_KEY_DELETE_REQ:
+	case ELE_MAC_OPEN_REQ:
+	case ELE_MAC_CLOSE_REQ:
+	case ELE_CIPHER_OPEN_REQ:
+	case ELE_CIPHER_CLOSE_REQ:
+	case ELE_SIGNATURE_GENERATE_OPEN_REQ:
+	case ELE_SIGNATURE_GENERATE_CLOSE_REQ:
+	case ELE_SIGNATURE_VERIFY_OPEN_REQ:
+	case ELE_SIGNATURE_VERIFY_CLOSE_REQ:
+	case ELE_DATA_STORAGE_OPEN_REQ:
+	case ELE_DATA_STORAGE_CLOSE_REQ:
+	case ELE_DATA_DELETE_REQ:
+		ret = 0;
+		break;
+	case ELE_STORAGE_OPEN_REQ:
+		/* Might be cleared as part of tear down. */
+		if (dev_ctx->strg_hdl) {
+			ret = -EEXIST;
+			break;
+		}
+		/*
+		 * NOTE: this advisory early check is intentionally not the
+		 * definitive exclusivity gate. modify_lock is dropped before the
+		 * command is sent, creating a TOCTOU window. That window is
+		 * closed by two additional layers:
+		 *   1. se_if_cmd_lock, held for the full send+receive cycle, so
+		 *      only one ELE_STORAGE_OPEN_REQ is in flight per MU at a
+		 *      time.
+		 *   2. set_dev_ctx_as_command_receiver(), which re-checks under
+		 *      modify_lock after the response arrives. If two callers
+		 *      race past this check, FW itself rejects the second
+		 *      ELE_STORAGE_OPEN_REQ before any handle is allocated.
+		 * See Documentation/driver-api/firmware/other_interfaces.rst,
+		 * section "ELE_STORAGE_OPEN_REQ concurrency and
+		 * command-receiver exclusivity".
+		 */
+		if (receiver_exists && !is_cmd_receiver)
+			ret = -EBUSY;
+		if (rx_msg_sz < ELE_STORAGE_OPEN_RSP_SZ)
+			ret = -EINVAL;
+		break;
+	case ELE_STORAGE_CLOSE_REQ:
+		/* Might be cleared as part of tear down. */
+		if (!dev_ctx->strg_hdl) {
+			ret = -ENXIO;
+			break;
+		}
+		/* Same self-ownership check as the session close above. */
+		if (tx_msg_sz < ELE_STORAGE_CLOSE_REQ_SZ ||
+		    msg->data[0] != dev_ctx->strg_hdl)
+			ret = -EINVAL;
+		break;
+	case ELE_STORAGE_STATUS_REQ:
+		ret = 0;
+		break;
+	default:
+		/* FW commands that embed DMA addresses. */
+		fields = ele_fw_cmd_addr_fields(header->command, &count);
+		if (!count) {
+			ret = -EOPNOTSUPP;
+			break;
+		}
+
+		ret = se_val_cmd_addrs(dev_ctx, msg, tx_msg_sz, fields, count);
+		break;
+	}
+
+	return ret;
+}
+
+void cmd_receiver_specific_ops(struct se_if_device_ctx *dev_ctx,
+			       struct se_api_msg *rx_msg)
+{
+	struct cmd_rcvr_data_info *crcvr_info = &dev_ctx->priv->crcvr_info;
+	struct se_msg_hdr *header = &rx_msg->header;
+
+	crcvr_info->cmd_rcvr_last_rcvd_cmd_id = 0;
+	crcvr_info->cmd_rcvr_var_size = 0;
+	switch (header->command) {
+	case ELE_STORAGE_MASTER_EXPORT_REQ:
+		/*
+		 * FW sent an export-start command with key_store_size at
+		 * data[1]. Save it so se_val_cmd_addrs() can range-check the
+		 * response buffer when the cmd_receiver sends back the address.
+		 */
+		crcvr_info->cmd_rcvr_last_rcvd_cmd_id = ELE_STORAGE_MASTER_EXPORT_REQ;
+		crcvr_info->cmd_rcvr_var_size = rx_msg->data[1];
+		break;
+	case ELE_STORAGE_CHUNK_EXPORT_REQ:
+		/*
+		 * FW sent a chunk-export command with chunk_size at data[1].
+		 * Save it so se_val_cmd_addrs() can range-check the response
+		 * buffer when the cmd_receiver sends back the address.
+		 */
+		crcvr_info->cmd_rcvr_last_rcvd_cmd_id = ELE_STORAGE_CHUNK_EXPORT_REQ;
+		crcvr_info->cmd_rcvr_var_size = rx_msg->data[1];
+		break;
+	case ELE_STORAGE_CHUNK_GET_REQ:
+		crcvr_info->cmd_rcvr_last_rcvd_cmd_id = ELE_STORAGE_CHUNK_GET_REQ;
+		break;
+	}
+}
+
+int fw_api_specific_ops(struct se_if_device_ctx *dev_ctx, struct se_api_msg *rx_msg,
+			bool is_cmd_interrupted)
+{
+	struct se_msg_hdr *header = &rx_msg->header;
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	switch (header->command) {
+	case ELE_SESSION_OPEN_REQ:
+		dev_ctx->sess_hdl = rx_msg->data[1];
+		if (is_cmd_interrupted) {
+			if (se_close_session(dev_ctx, dev_ctx->sess_hdl))
+				dev_err(dev_ctx->priv->dev, "failed to close session.\n");
+			dev_ctx->sess_hdl = 0;
+		}
+		break;
+	case ELE_SESSION_CLOSE_REQ:
+		dev_ctx->sess_hdl = 0;
+		break;
+	case ELE_STORAGE_OPEN_REQ: {
+		int rc;
+
+		/*
+		 * Record the storage handle before registering as command
+		 * receiver. FW has already allocated the handle; if we assigned
+		 * it only after a successful registration, a failing
+		 * set_dev_ctx_as_command_receiver() (e.g. -EBUSY) would leave
+		 * strg_hdl at 0 while the ioctl still returns success to
+		 * userspace. The kernel would then never close the handle on
+		 * teardown, leaking it in FW. Storing it first guarantees
+		 * cleanup_dev_ctx() closes it on the next close(), regardless
+		 * of whether registration succeeded.
+		 */
+		dev_ctx->strg_hdl = rx_msg->data[1];
+
+		rc = is_cmd_interrupted ? 0 : set_dev_ctx_as_command_receiver(dev_ctx);
+		if (is_cmd_interrupted || rc) {
+			if (se_close_storage(dev_ctx, dev_ctx->strg_hdl))
+				dev_err(dev_ctx->priv->dev, "failed to close storage.\n");
+			dev_ctx->strg_hdl = 0;
+			if (rc)
+				dev_err(priv->dev,
+					"Failed to register %s as CMD-Receiver: %d\n",
+					dev_ctx->devname, rc);
+			return rc;
+		}
+		break;
+	}
+	case ELE_STORAGE_CLOSE_REQ:
+		scoped_guard(mutex, &priv->modify_lock)
+			unset_dev_ctx_as_command_receiver(dev_ctx);
+		dev_ctx->strg_hdl = 0;
+		break;
+	}
+
+	return 0;
+}
+
+int se_close_session(struct se_if_device_ctx *dev_ctx, u32 session_hdl)
+{
+	struct se_api_msg *tx_msg __free(kfree) =
+		kzalloc(ELE_SESSION_CLOSE_REQ_SZ, GFP_KERNEL);
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_SESSION_CLOSE_RSP_SZ, GFP_KERNEL);
+	struct se_if_priv *priv;
+	int ret;
+
+	if (!dev_ctx || !dev_ctx->priv)
+		return -EINVAL;
+
+	if (!tx_msg || !rx_msg)
+		return -ENOMEM;
+
+	priv = dev_ctx->priv;
+
+	/*
+	 * Session close is a FW-API command; pass is_base_api=false so the
+	 * header carries fw_api_ver.
+	 */
+	se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+			    ELE_SESSION_CLOSE_REQ, ELE_SESSION_CLOSE_REQ_SZ, false);
+
+	tx_msg->data[0] = session_hdl;
+
+	/*
+	 * Transmit on the caller's own context. Using dev_ctx (rather than
+	 * hardcoding priv->priv_dev_ctx) keeps a userspace close() subject to
+	 * the going_away check in ele_msg_send_rcv(): if unbind has begun and
+	 * freed priv->tx_chan, the send is rejected with -ENODEV instead of
+	 * touching the freed mailbox channel.
+	 *
+	 * The teardown and fw_busy-recovery paths instead reserve the messaging
+	 * interface for their own task via se_reserve_msg_if() before calling
+	 * this. That reservation (msg_excl_owner == current) is what makes
+	 * ele_msg_send_rcv() let their resync closes through the going_away and
+	 * fw_busy gates; they also pass priv_dev_ctx so the wait is the
+	 * uninterruptible internal-context wait.
+	 */
+
+	ret = ele_msg_send_rcv(dev_ctx,
+			       tx_msg,
+			       ELE_SESSION_CLOSE_REQ_SZ,
+			       rx_msg,
+			       ELE_SESSION_CLOSE_RSP_SZ, NULL);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(dev_ctx,
+				      rx_msg,
+				      ELE_SESSION_CLOSE_REQ,
+				      ELE_SESSION_CLOSE_RSP_SZ,
+				      priv->if_defs->fw_api_ver);
+	return ret;
+}
+
+int se_close_storage(struct se_if_device_ctx *dev_ctx, u32 storage_hdl)
+{
+	struct se_api_msg *tx_msg __free(kfree) =
+		kzalloc(ELE_STORAGE_CLOSE_REQ_SZ, GFP_KERNEL);
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_STORAGE_CLOSE_RSP_SZ, GFP_KERNEL);
+	struct se_if_priv *priv;
+	int ret;
+
+	if (!dev_ctx || !dev_ctx->priv)
+		return -EINVAL;
+
+	if (!tx_msg || !rx_msg)
+		return -ENOMEM;
+
+	priv = dev_ctx->priv;
+
+	/* Same FW-API version handling as se_close_session() above. */
+	se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+			    ELE_STORAGE_CLOSE_REQ, ELE_STORAGE_CLOSE_REQ_SZ, false);
+
+	tx_msg->data[0] = storage_hdl;
+
+	/* Transmit on the caller's own context; see se_close_session(). */
+	ret = ele_msg_send_rcv(dev_ctx,
+			       tx_msg,
+			       ELE_STORAGE_CLOSE_REQ_SZ,
+			       rx_msg,
+			       ELE_STORAGE_CLOSE_RSP_SZ, NULL);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(dev_ctx,
+				      rx_msg,
+				      ELE_STORAGE_CLOSE_REQ,
+				      ELE_STORAGE_CLOSE_RSP_SZ,
+				      priv->if_defs->fw_api_ver);
+	return ret;
+}
diff --git a/drivers/firmware/imx/ele_fw_api.h b/drivers/firmware/imx/ele_fw_api.h
new file mode 100644
index 00000000000000..7a6d7dab84ff3a
--- /dev/null
+++ b/drivers/firmware/imx/ele_fw_api.h
@@ -0,0 +1,104 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2026 NXP
+ */
+
+#ifndef ELE_FW_API_H
+#define ELE_FW_API_H
+#include "se_ctrl.h"
+
+#define ELE_SESSION_OPEN_REQ            0x10u
+#define ELE_SESSION_OPEN_RSP_SZ         0x0Cu
+
+#define ELE_SESSION_CLOSE_REQ_SZ	0x08u
+#define ELE_SESSION_CLOSE_RSP_SZ	0x08u
+#define ELE_SESSION_CLOSE_REQ           0x11u
+
+/*
+ * Session-scoped FW-API service, key-management and close command opcodes
+ * (SAB command IDs, PSA_COMPLIANT message layout). These commands do not
+ * embed DMA staging-buffer addresses that require range-checking; they are
+ * defined here for completeness and for use by the command allow-list.
+ * ELE_FW_GET_INFO_REQ is the FW-API get-info opcode and is intentionally
+ * distinct from the base-API ELE_GET_INFO_REQ (0xda) in ele_base_msg.h.
+ */
+#define ELE_FW_GET_INFO_REQ             0x16u
+#define ELE_KEY_STORE_OPEN_REQ          0x30u
+#define ELE_KEY_STORE_CLOSE_REQ         0x31u
+#define ELE_KEY_MGMT_OPEN_REQ           0x40u
+#define ELE_KEY_MGMT_CLOSE_REQ          0x41u
+#define ELE_MANAGE_KEY_GROUP_REQ        0x45u
+#define ELE_GET_KEY_ATTR_REQ            0x4Cu
+#define ELE_KEY_DELETE_REQ              0x4Eu
+#define ELE_MAC_OPEN_REQ                0x50u
+#define ELE_MAC_CLOSE_REQ               0x51u
+#define ELE_CIPHER_OPEN_REQ             0x60u
+#define ELE_CIPHER_CLOSE_REQ            0x61u
+#define ELE_SIGNATURE_GENERATE_OPEN_REQ 0x70u
+#define ELE_SIGNATURE_GENERATE_CLOSE_REQ 0x71u
+#define ELE_SIGNATURE_VERIFY_OPEN_REQ   0x80u
+#define ELE_SIGNATURE_VERIFY_CLOSE_REQ  0x81u
+#define ELE_DATA_STORAGE_OPEN_REQ       0xA0u
+#define ELE_DATA_STORAGE_CLOSE_REQ      0xA1u
+#define ELE_DATA_DELETE_REQ             0xA4u
+
+/*
+ * FW-API crypto command opcodes that embed one or more DMA physical addresses
+ * in their message payload. ele_uapi_allowed_fw_cmd() range-checks those
+ * addresses against the calling context's shared-memory window before the
+ * message is handed to firmware. Opcodes match the SAB command IDs emitted by
+ * the userspace library (PSA_COMPLIANT message layout).
+ */
+#define ELE_PUB_KEY_EXPORT_REQ          0x32u
+#define ELE_KEYSTORE_REPROV_ENABLE_REQ  0x3Fu
+#define ELE_KEYGEN_REQ                  0x42u
+#define ELE_KEY_EXCHANGE_REQ            0x47u
+#define ELE_KEY_IMPORT_REQ              0x4Fu
+#define ELE_KEY_IMPORT                  0x4Fu
+#define ELE_MAC_REQ                     0x52u
+#define ELE_CIPHER_REQ                  0x62u
+#define ELE_AUTH_ENC_REQ                0x64u
+#define ELE_AUTH_ENC_NEW_REQ            0x65u
+#define ELE_SIGNATURE_GENERATE_REQ      0x72u
+#define ELE_PUB_KEY_ATTEST_REQ          0x74u
+#define ELE_SIGNATURE_VERIFY_REQ        0x82u
+#define ELE_DATA_STORAGE_REQ            0xA2u
+#define ELE_ENC_DATA_STORAGE_REQ        0xA3u
+#define ELE_ASYMMETRIC_ENC_REQ          0x92u
+
+#define ELE_KEY_GENERIC_CRYPTO_REQ      0xC2u
+#define ELE_GC_CIPHER_REQ               0xC8u
+#define ELE_GC_AEAD_REQ                 0xC9u
+#define ELE_GC_ACRYPTO_REQ              0xCAu
+#define ELE_GC_AKEY_GEN_REQ             0xCBu
+#define ELE_HASH_ONE_GO_REQ             0xCCu
+#define ELE_RNG_GET_RANDOM_REQ          0xCDu
+
+#define ELE_STORAGE_OPEN_REQ            0xE0u
+#define ELE_STORAGE_OPEN_RSP_SZ         0x0Cu
+
+#define ELE_STORAGE_CLOSE_REQ_SZ	0x08u
+#define ELE_STORAGE_CLOSE_RSP_SZ	0x08u
+#define ELE_STORAGE_CLOSE_REQ           0xE1u
+
+#define ELE_STORAGE_MASTER_IMPORT_REQ   0xE2u
+#define ELE_STORAGE_MASTER_EXPORT_REQ   0xE3u
+#define ELE_STORAGE_EXPORT_FINISH_REQ   0xE4u
+#define ELE_STORAGE_CHUNK_EXPORT_REQ    0xE5u
+#define ELE_STORAGE_CHUNK_GET_REQ       0xE6u
+#define ELE_STORAGE_CHUNK_GET_DONE_REQ  0xE7u
+#define ELE_STORAGE_CHUNK_DELETE_REQ    0xE9u
+#define ELE_STORAGE_STATUS_REQ          0xEAu
+
+int ele_uapi_allowed_fw_rsp(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			    u32 tx_msg_sz);
+int ele_uapi_allowed_fw_cmd(struct se_if_device_ctx *dev_ctx, struct se_msg_hdr *header,
+			    u32 tx_msg_sz, u32 rx_msg_sz);
+int fw_api_specific_ops(struct se_if_device_ctx *dev_ctx, struct se_api_msg *rx_msg,
+			bool is_cmd_interrupted);
+void cmd_receiver_specific_ops(struct se_if_device_ctx *dev_ctx,
+			       struct se_api_msg *rx_msg);
+int se_close_session(struct se_if_device_ctx *dev_ctx, u32 session_hdl);
+int se_close_storage(struct se_if_device_ctx *dev_ctx, u32 storage_hdl);
+
+#endif /* ELE_FW_API_H */
diff --git a/drivers/firmware/imx/ele_msg_addr_field.c b/drivers/firmware/imx/ele_msg_addr_field.c
new file mode 100644
index 00000000000000..ac49ac78da687a
--- /dev/null
+++ b/drivers/firmware/imx/ele_msg_addr_field.c
@@ -0,0 +1,619 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2026 NXP
+ */
+
+#include <linux/types.h>
+
+#include "ele_common.h"
+#include "ele_base_msg.h"
+#include "ele_fw_api.h"
+
+/*
+ * Base-API commands that embed one or more DMA physical addresses in their
+ * payload. Unlike the FW-API crypto commands, GET_INFO and DEV_ATTEST split
+ * their response-buffer address across two words: the high half is written
+ * first (lower word index) and the low half next, so has_msb is set and the
+ * msb_idx precedes the lsb_idx. GEN_KEY_BLOB uses single-word LSB addresses.
+ * See struct se_cmd_addr_field in ele_common.h for the field semantics.
+ */
+static const struct se_cmd_addr_field ele_get_info_addr_fields[] = {
+	/*
+	 * rsp_data_addr_hi @ data[0], rsp_data_addr_lo @ data[1];
+	 * buf_sz is a u16 in the low half of data[2].
+	 */
+	{ .lsb_idx = 1, .msb_idx = 0, .has_msb = true, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFu },
+};
+
+static const struct se_cmd_addr_field ele_dev_attest_addr_fields[] = {
+	/*
+	 * rsp_data_addr_hi @ data[0], rsp_data_addr_lo @ data[1];
+	 * buf_sz is a u16 in the low half of data[2].
+	 */
+	{ .lsb_idx = 1, .msb_idx = 0, .has_msb = true, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFu },
+};
+
+static const struct se_cmd_addr_field ele_oem_auth_cntr_addr_fields[] = {
+	/*
+	 * Container Header address: a 64-bit physical address split across two
+	 * words. data[0] holds the 32-bit MSB and data[1] holds the 32-bit LSB
+	 * (ELE API spec Table 27, word size = 0x3, so the command is header +
+	 * MSB + LSB only). No length word is carried in the payload; the buffer
+	 * length is read from the fw_tag_len_vers_info header at its start
+	 * (SE_CMD_ADDR_DEDUCE_SZ) so the whole buffer is range-checked.
+	 */
+	{ .lsb_idx = 1, .msb_idx = 0, .has_msb = true, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = SE_CMD_ADDR_DEDUCE_SZ },			/* container_hdr_addr */
+};
+
+/*
+ * GENERATE ELE KEY BLOB command (ELE_GEN_KEY_BLOB_REQ, 0xAF).
+ * ELE API spec Table 73, word size = 0x8 (header + 7 data words):
+ *   data[0] = key_identifier
+ *   data[1] = Reserved
+ *   data[2] = load_address  (32-bit; must be 64-bit aligned)
+ *   data[3] = Reserved
+ *   data[4] = store_address (32-bit; must be 64-bit aligned)
+ *   data[5] = Reserved[31:16] | max_export_size[15:0]
+ *   data[6] = CRC
+ *
+ * load_addr points to the input: a blob header (8 bytes, Table 77) followed
+ * by the plaintext payload. No size word is present in the message for this
+ * input buffer. The maximum input size is determined by the largest supported
+ * payload type: OTFAD key configuration (0x28 bytes per Table 79) plus the
+ * 8-byte header gives 0x30 bytes. That is the literal upper bound placed in
+ * buf_size and selected with SE_CMD_ADDR_FIXED_SIZE, so se_val_cmd_addrs()
+ * can verify [load_addr, load_addr+0x30) lies within the shared-memory
+ * window.
+ */
+#define OP_GEN_ELE_KEY_BLOB_INPUT_MAX_SZ	0x30  /* blob hdr (8) + OTFAD payload (0x28) */
+static const struct se_cmd_addr_field ele_gen_key_blob_addr_fields[] = {
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = SE_CMD_ADDR_FIXED_SIZE,
+	  .buf_size = OP_GEN_ELE_KEY_BLOB_INPUT_MAX_SZ },	/* load_address */
+	/* store_address @ data[4]; max_export_size is u16 in low half of data[5] */
+	{ .lsb_idx = 4, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFu },		/* store_address */
+};
+
+/*
+ * Return the address-field descriptor table for a base-API command, or NULL
+ * when the command embeds no DMA addresses. count is set to the number of
+ * entries.
+ */
+const struct se_cmd_addr_field *ele_base_cmd_addr_fields(u8 cmd, size_t *count)
+{
+	switch (cmd) {
+	case ELE_OEM_AUTH_CONTAINER_REQ:
+		*count = ARRAY_SIZE(ele_oem_auth_cntr_addr_fields);
+		return ele_oem_auth_cntr_addr_fields;
+	case ELE_GEN_KEY_BLOB_REQ:
+		*count = ARRAY_SIZE(ele_gen_key_blob_addr_fields);
+		return ele_gen_key_blob_addr_fields;
+	case ELE_GET_INFO_REQ:
+		*count = ARRAY_SIZE(ele_get_info_addr_fields);
+		return ele_get_info_addr_fields;
+	case ELE_DEV_ATTEST_REQ:
+		*count = ARRAY_SIZE(ele_dev_attest_addr_fields);
+		return ele_dev_attest_addr_fields;
+	default:
+		*count = 0;
+		return NULL;
+	}
+}
+
+/*
+ * FW-API crypto commands that embed one or more DMA physical addresses in
+ * their payload. On the PSA_COMPLIANT ABI most addresses are written by the
+ * userspace library as a single little-endian 32-bit LSB word (the high half
+ * is always zero), so has_msb is left false for those entries. A few commands
+ * (pub-key-export 0x32, keystore reprov-enable 0x3F) carry an explicit ext/MSB
+ * word ahead of the LSB word, matching the base-API two-word address layout;
+ * their entries set has_msb = true so the MSB word is validated too. See
+ * struct se_cmd_addr_field in ele_common.h for the field semantics.
+ */
+static const struct se_cmd_addr_field ele_pub_key_export_addr_fields[] = {
+	/*
+	 * out_key_addr: the recovered public key output buffer. Its high half
+	 * out_key_addr_ext is data[2] and its low half out_key_addr is data[3];
+	 * the library always writes it via set_phy_addr_to_words(), so it is
+	 * always a DMA address. Its length is out_key_size, the u16 in the low
+	 * half of data[4]. key_identifier (data[1]) is an integer, not an
+	 * address.
+	 */
+	{ .lsb_idx = 3, .msb_idx = 2, .has_msb = true, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFu },		/* out_key_addr */
+};
+
+static const struct se_cmd_addr_field ele_keystore_reprov_en_addr_fields[] = {
+	/*
+	 * Signed message address: a 64-bit physical address split across two
+	 * words. data[0] holds the 32-bit MSB and data[1] holds the 32-bit LSB
+	 * (ELE API spec Table 202, word size = 0x3, so the command is header +
+	 * MSB + LSB only). No length word is carried in the payload; the buffer
+	 * length is read from the fw_tag_len_vers_info header at its start
+	 * (SE_CMD_ADDR_DEDUCE_SZ) so the whole buffer is range-checked.
+	 */
+	{ .lsb_idx = 1, .msb_idx = 0, .has_msb = true, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = SE_CMD_ADDR_DEDUCE_SZ },			/* signed_msg_addr */
+};
+
+static const struct se_cmd_addr_field ele_keygen_addr_fields[] = {
+	/*
+	 * key @ data[1]: a plaintext private-key output buffer only when the
+	 * KEY_GENERATION PLAINTEXT_KEY flag (bit 3 of the flags byte in the low
+	 * 8 bits of data[8]) is set; otherwise it is an integer key identifier.
+	 * Its length is priv_key_sz, the u16 in the high half of data[8].
+	 */
+	{ .lsb_idx = 1, .flag_idx = 8, .flag_mask = 0x00000008u, .is_addr_when_set = true,
+	  .size_idx = 8, .size_shift = 16, .size_mask = 0xFFFFu },	/* priv_key_addr */
+	/*
+	 * pub_key_addr @ data[9]: always a DMA address. Its length is
+	 * pub_key_sz, the u16 in the low half of data[2].
+	 */
+	{ .lsb_idx = 9, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFu },		/* pub_key_addr */
+};
+
+static const struct se_cmd_addr_field ele_key_exchange_addr_fields[] = {
+	/*
+	 * PSA_COMPLIANT key-exchange payload. key_management_handle is data[0]
+	 * and flags/reserved is data[1]; the four buffer addresses that follow
+	 * are each written unconditionally via set_phy_addr_to_words() (single
+	 * LSB word, high half always zero), so all are always DMA addresses.
+	 * Each address is immediately followed by its full u32 byte length.
+	 */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 3, .size_mask = 0xFFFFFFFFu },		/* in_content_addr */
+	{ .lsb_idx = 4, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFFFFFu },		/* in_pub_buffer_addr */
+	{ .lsb_idx = 6, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 7, .size_mask = 0xFFFFFFFFu },		/* user_fixed_info_addr */
+	{ .lsb_idx = 8, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_mask = 0xFFFFFFFFu },		/* output_addr */
+};
+
+static const struct se_cmd_addr_field ele_key_import_addr_fields[] = {
+	/*
+	 * PSA_COMPLIANT key-import payload. key_management_handle is data[0]
+	 * and flags/reserved is data[1]; the single input buffer address that
+	 * follows is written unconditionally via set_phy_addr_to_words()
+	 * (single LSB word, high half always zero), so it is always a DMA
+	 * address. Its length is the full u32 input_size in data[3].
+	 */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 3, .size_mask = 0xFFFFFFFFu },		/* input_address */
+};
+
+/* ELE_MAC_REQ: MAC one-go operation. */
+static const struct se_cmd_addr_field ele_mac_addr_fields[] = {
+	/* key: plaintext-key buffer only when the MAC PLAINTEXT_KEY flag is set */
+	{ .lsb_idx = 1, .flag_idx = 5, .flag_mask = 0x00080000u, .is_addr_when_set = true,
+	  .size_idx = 7, .size_mask = 0xFFFFu },		/* key_size */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFFFFFu },		/* payload_address */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFu },		/* mac_address */
+	{ .lsb_idx = 8, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_mask = 0xFFFFu },		/* context_address */
+};
+
+/* ELE_CIPHER_REQ: symmetric cipher one-go operation. */
+static const struct se_cmd_addr_field ele_cipher_addr_fields[] = {
+	/* key: plaintext-key buffer only when the CIPHER PLAINTEXT_KEY flag is set */
+	{ .lsb_idx = 1, .flag_idx = 3, .flag_mask = 0x00080000u, .is_addr_when_set = true,
+	  .size_idx = 9, .size_mask = 0xFFFFu },		/* key_size */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 3, .size_mask = 0xFFFFu },		/* iv_address */
+	{ .lsb_idx = 5, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 7, .size_mask = 0xFFFFFFFFu },		/* input_address */
+	{ .lsb_idx = 6, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 8, .size_mask = 0xFFFFFFFFu },		/* output_address */
+	{ .lsb_idx = 10, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 11, .size_mask = 0xFFFFu },		/* context_address */
+};
+
+/*
+ * AEAD encrypt/decrypt legacy command (ELE_AUTH_ENC_REQ, 0x64).
+ * ELE API spec Table 287, word size = 0xD (header + 12 data words):
+ *   data[0]  = cipher_handle
+ *   data[1]  = key_identifier
+ *   data[2]  = IV LSB address
+ *   data[3]  = Reserved[31:24] | Flags[23:16] | IV_size[15:0]
+ *   data[4]  = algorithm
+ *   data[5]  = AAD LSB address
+ *   data[6]  = Reserved[31:16] | AAD_size[15:0]
+ *   data[7]  = Input LSB address
+ *   data[8]  = Output LSB address
+ *   data[9]  = Input size (u32)
+ *   data[10] = Output size (u32)
+ *   data[11] = CRC
+ * IV size is the 16-bit low half of data[3]; AAD size is the 16-bit low half
+ * of data[6]; input and output sizes are full u32 words.
+ */
+static const struct se_cmd_addr_field ele_auth_enc_addr_fields[] = {
+	/* iv_address    (size[15:0]  @ data[3]) */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 3, .size_mask = 0xFFFFu },
+	/* aad_address   (size[15:0]  @ data[6]) */
+	{ .lsb_idx = 5, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 6, .size_mask = 0xFFFFu },
+	/* input_address (size[31:0]  @ data[9]) */
+	{ .lsb_idx = 7, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_mask = 0xFFFFFFFFu },
+	/* output_address (size[31:0] @ data[10]) */
+	{ .lsb_idx = 8, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 10, .size_mask = 0xFFFFFFFFu },
+};
+
+/* ELE_AUTH_ENC_NEW_REQ: AEAD encrypt/decrypt with internally-generated IV output. */
+#define ELE_AUTH_ENC_IV_OUT_SIZE      12  /* firmware always writes exactly 12 bytes */
+static const struct se_cmd_addr_field ele_auth_enc_new_addr_fields[] = {
+	/* iv_address_in length is packed in the high half of the iv-size word */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_shift = 16, .size_mask = 0xFFFFu },	/* iv_address_in */
+	/* iv_address_out has a fixed firmware-defined length, not carried in msg */
+	{ .lsb_idx = 4, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = SE_CMD_ADDR_FIXED_SIZE,
+	  .buf_size = ELE_AUTH_ENC_IV_OUT_SIZE },		/* iv_address_out */
+	/* key: plaintext-key buffer only when the PLAINTEXT_KEY flag is set */
+	{ .lsb_idx = 5, .flag_idx = 2, .flag_mask = 0x00000008u, .is_addr_when_set = true,
+	  .size_idx = 7, .size_shift = 16, .size_mask = 0xFFFFu },	/* key_size */
+	{ .lsb_idx = 6, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 7, .size_mask = 0xFFFFu },		/* tag_address */
+	{ .lsb_idx = 8, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_mask = 0xFFFFFFFFu },		/* aad_address */
+	{ .lsb_idx = 10, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 11, .size_mask = 0xFFFFFFFFu },		/* input_address */
+	{ .lsb_idx = 12, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 13, .size_mask = 0xFFFFFFFFu },		/* output_address */
+	{ .lsb_idx = 14, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 15, .size_mask = 0xFFFFu },		/* context_address */
+};
+
+/* ELE_SIGNATURE_GENERATE_REQ: digital signature generation. */
+static const struct se_cmd_addr_field ele_sign_gen_addr_fields[] = {
+	/* key: plaintext-key buffer only when GENERATE_SIGN PLAINTEXT_KEY is set */
+	{ .lsb_idx = 1, .flag_idx = 5, .flag_mask = 0x00080000u, .is_addr_when_set = true,
+	  .size_idx = 8, .size_mask = 0xFFFFu },		/* priv_key_size */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFFFFFu },		/* message_addr */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFu },		/* signature_addr */
+	{ .lsb_idx = 9, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 11, .size_mask = 0xFFFFu },		/* sm2_pub_key_addr */
+	{ .lsb_idx = 10, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 11, .size_shift = 16,
+	  .size_mask = 0xFFFFu },			/* sm2_id / ml_dsa_ctx addr */
+};
+
+static const struct se_cmd_addr_field ele_pub_key_attest_addr_fields[] = {
+	/*
+	 * PSA_COMPLIANT public-key-attestation payload. sig_gen_hdl is data[0],
+	 * key_identifier data[1], key_attestation_id data[2], attest_algo
+	 * data[3]. The two buffer addresses that follow are each written
+	 * unconditionally via set_phy_addr_to_words() (single LSB word, high
+	 * half always zero), so both are always DMA addresses. Each address is
+	 * immediately followed by its full u32 byte length.
+	 */
+	{ .lsb_idx = 4, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFFFFFu },		/* auth_challenge_addr */
+	{ .lsb_idx = 6, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 7, .size_mask = 0xFFFFFFFFu },		/* certificate_addr */
+};
+
+/* ELE_SIGNATURE_VERIFY_REQ: digital signature verification. */
+static const struct se_cmd_addr_field ele_verify_sign_addr_fields[] = {
+	/* key: plaintext-key buffer unless the VERIFY_SIGN OPAQUE_KEY flag is set */
+	{ .lsb_idx = 1, .flag_idx = 7, .flag_mask = 0x00000008u, .is_addr_when_set = false,
+	  .size_idx = 5, .size_shift = 16, .size_mask = 0xFFFFu },	/* key_size */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFFFFFu },		/* msg_addr */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFu },		/* sig_addr */
+	{ .lsb_idx = 10, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 11, .size_mask = 0xFFFFu },		/* sm2_id / ml_dsa_ctx addr */
+};
+
+static const struct se_cmd_addr_field ele_data_storage_addr_fields[] = {
+	/*
+	 * PSA_COMPLIANT data-storage payload. data_storage_handle is data[0],
+	 * flags/reserved is data[1], data_id is data[2]. data_address (data[3])
+	 * is the plaintext data buffer, written unconditionally via
+	 * set_phy_addr_to_words() (single LSB word, high half always zero), so
+	 * it is always a DMA address. Its length is the full u32 data_size in
+	 * data[4].
+	 */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFFFFFu },		/* data_address */
+};
+
+/* ELE_ASYMMETRIC_ENC_REQ: asymmetric encryption/decryption. */
+static const struct se_cmd_addr_field ele_asym_enc_addr_fields[] = {
+	/* key_id_addr: plaintext-key buffer only when the PLAINTEXT_KEY flag is set */
+	{ .lsb_idx = 1, .flag_idx = 8, .flag_mask = 0x00000008u, .is_addr_when_set = true,
+	  .size_idx = 10, .size_mask = 0xFFFFFFFFu },		/* input_plainkey_size */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFFFFFu },		/* plaintext_addr */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 6, .size_mask = 0xFFFFFFFFu },		/* ciphertext_addr */
+	{ .lsb_idx = 4, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 7, .size_mask = 0xFFFFFFFFu },		/* label_addr */
+};
+
+/* ELE_KEY_GENERIC_CRYPTO_REQ: generic crypto operation with a raw key. */
+static const struct se_cmd_addr_field ele_key_generic_crypto_addr_fields[] = {
+	/* key_address length is the u8 key_size in the third byte of the iv-size word */
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 3, .size_shift = 16, .size_mask = 0xFFu },	/* key_address */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 3, .size_mask = 0xFFFFu },		/* iv_address */
+	{ .lsb_idx = 4, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFu },		/* aad_address */
+	{ .lsb_idx = 6, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 8, .size_mask = 0xFFFFFFFFu },		/* input_address */
+	{ .lsb_idx = 7, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_mask = 0xFFFFFFFFu },		/* output_address */
+};
+
+/* ELE_GC_CIPHER_REQ: GC symmetric cipher operation. */
+static const struct se_cmd_addr_field ele_gc_cipher_addr_fields[] = {
+	{ .lsb_idx = 0, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFFFFFu },		/* in_addr */
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFFFFFu },		/* out_addr (shares data_size) */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFFFFFu },		/* key_addr */
+	{ .lsb_idx = 5, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 6, .size_mask = 0xFFFFFFFFu },		/* iv_addr */
+};
+
+/* ELE_GC_AEAD_REQ: GC AEAD operation. */
+static const struct se_cmd_addr_field ele_gc_aead_addr_fields[] = {
+	{ .lsb_idx = 0, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFFFFFu },		/* in_addr */
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFFFFFu },		/* out_addr (shares data_size) */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFFFFFu },		/* key_addr */
+	{ .lsb_idx = 5, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 6, .size_mask = 0xFFFFFFFFu },		/* nonce_addr */
+	{ .lsb_idx = 7, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 8, .size_mask = 0xFFFFFFFFu },		/* aad_addr */
+	{ .lsb_idx = 9, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 10, .size_mask = 0xFFFFFFFFu },		/* tag_addr */
+};
+
+/* ELE_GC_ACRYPTO_REQ: GC asymmetric crypto operation. */
+static const struct se_cmd_addr_field ele_gc_acrypto_addr_fields[] = {
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFFFFFu },		/* data_buff1_addr */
+	{ .lsb_idx = 4, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 6, .size_mask = 0xFFFFFFFFu },		/* data_buff2_addr */
+	{ .lsb_idx = 7, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_mask = 0xFFFFu },		/* key_buff1_addr */
+	{ .lsb_idx = 8, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_shift = 16, .size_mask = 0xFFFFu },	/* key_buff2_addr */
+	{ .lsb_idx = 12, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 13, .size_mask = 0xFFFFu },		/* rsa_label_addr */
+};
+
+/* ELE_GC_AKEY_GEN_REQ: GC asymmetric key generation. */
+static const struct se_cmd_addr_field ele_gc_akey_gen_addr_fields[] = {
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFu },		/* modulus_addr */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_shift = 16, .size_mask = 0xFFFFu },	/* priv_buff_addr */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFu },		/* pub_buff_addr */
+};
+
+/* ELE_HASH_ONE_GO_REQ: hash one-go operation. */
+static const struct se_cmd_addr_field ele_hash_one_go_addr_fields[] = {
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 6, .size_shift = 16, .size_mask = 0xFFFFu },	/* ctx_addr */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 4, .size_mask = 0xFFFFFFFFu },		/* input_addr */
+	{ .lsb_idx = 3, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 5, .size_mask = 0xFFFFFFFFu },		/* output_addr */
+};
+
+static const struct se_cmd_addr_field ele_enc_data_storage_addr_fields[] = {
+	/*
+	 * PSA_COMPLIANT encrypted-data-storage payload. data_storage_handle is
+	 * data[0] and data_id is data[1]. data_address (data[2]) is written
+	 * unconditionally via set_phy_addr_to_words() (single LSB word, high
+	 * half always zero), so it is always a DMA address; its length is the
+	 * full u32 data_size in data[3]. iv_address (data[8]) is only written
+	 * when an IV is supplied and is left zero otherwise, so it is an
+	 * optional always-address handled by the zero-address skip; its length
+	 * is the u16 iv_size in the low half of data[9].
+	 */
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 3, .size_mask = 0xFFFFFFFFu },		/* data_address */
+	{ .lsb_idx = 8, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 9, .size_mask = 0xFFFFu },		/* iv_address */
+};
+
+static const struct se_cmd_addr_field ele_rng_get_random_addr_fields[] = {
+	/*
+	 * PSA_COMPLIANT get-random payload. reserved/flags is data[0]; rnd_addr
+	 * (data[1]) is the output buffer, written unconditionally via
+	 * set_phy_addr_to_words() (single LSB word, high half always zero), so
+	 * it is always a DMA address. Its length is the full u32 rnd_size in
+	 * data[2].
+	 */
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFFFFFu },		/* rnd_addr */
+};
+
+static const struct se_cmd_addr_field ele_storage_master_import_addr_fields[] = {
+	/*
+	 * Storage master-import command (ELE_STORAGE_MASTER_IMPORT_REQ).
+	 * Payload layout (data[] = message word minus header word 0):
+	 *   data[0] = storage_handle
+	 *   data[1] = key_store_address  (LSB; high half always zero)
+	 *   data[2] = key_store_size
+	 * The address is set unconditionally via set_phy_addr_to_words() and
+	 * its length is the full u32 key_store_size.
+	 */
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 2, .size_mask = 0xFFFFFFFFu },		/* key_store_address */
+};
+
+const struct se_cmd_addr_field *ele_fw_cmd_addr_fields(u8 cmd, size_t *count)
+{
+	switch (cmd) {
+	case ELE_PUB_KEY_EXPORT_REQ:
+		*count = ARRAY_SIZE(ele_pub_key_export_addr_fields);
+		return ele_pub_key_export_addr_fields;
+	case ELE_KEYSTORE_REPROV_ENABLE_REQ:
+		*count = ARRAY_SIZE(ele_keystore_reprov_en_addr_fields);
+		return ele_keystore_reprov_en_addr_fields;
+	case ELE_KEYGEN_REQ:
+		*count = ARRAY_SIZE(ele_keygen_addr_fields);
+		return ele_keygen_addr_fields;
+	case ELE_KEY_EXCHANGE_REQ:
+		*count = ARRAY_SIZE(ele_key_exchange_addr_fields);
+		return ele_key_exchange_addr_fields;
+	case ELE_KEY_IMPORT_REQ:
+		*count = ARRAY_SIZE(ele_key_import_addr_fields);
+		return ele_key_import_addr_fields;
+	case ELE_MAC_REQ:
+		*count = ARRAY_SIZE(ele_mac_addr_fields);
+		return ele_mac_addr_fields;
+	case ELE_CIPHER_REQ:
+		*count = ARRAY_SIZE(ele_cipher_addr_fields);
+		return ele_cipher_addr_fields;
+	case ELE_AUTH_ENC_REQ:
+		*count = ARRAY_SIZE(ele_auth_enc_addr_fields);
+		return ele_auth_enc_addr_fields;
+	case ELE_AUTH_ENC_NEW_REQ:
+		*count = ARRAY_SIZE(ele_auth_enc_new_addr_fields);
+		return ele_auth_enc_new_addr_fields;
+	case ELE_SIGNATURE_GENERATE_REQ:
+		*count = ARRAY_SIZE(ele_sign_gen_addr_fields);
+		return ele_sign_gen_addr_fields;
+	case ELE_PUB_KEY_ATTEST_REQ:
+		*count = ARRAY_SIZE(ele_pub_key_attest_addr_fields);
+		return ele_pub_key_attest_addr_fields;
+	case ELE_SIGNATURE_VERIFY_REQ:
+		*count = ARRAY_SIZE(ele_verify_sign_addr_fields);
+		return ele_verify_sign_addr_fields;
+	case ELE_DATA_STORAGE_REQ:
+		*count = ARRAY_SIZE(ele_data_storage_addr_fields);
+		return ele_data_storage_addr_fields;
+	case ELE_ENC_DATA_STORAGE_REQ:
+		*count = ARRAY_SIZE(ele_enc_data_storage_addr_fields);
+		return ele_enc_data_storage_addr_fields;
+	case ELE_ASYMMETRIC_ENC_REQ:
+		*count = ARRAY_SIZE(ele_asym_enc_addr_fields);
+		return ele_asym_enc_addr_fields;
+	case ELE_KEY_GENERIC_CRYPTO_REQ:
+		*count = ARRAY_SIZE(ele_key_generic_crypto_addr_fields);
+		return ele_key_generic_crypto_addr_fields;
+	case ELE_GC_CIPHER_REQ:
+		*count = ARRAY_SIZE(ele_gc_cipher_addr_fields);
+		return ele_gc_cipher_addr_fields;
+	case ELE_GC_AEAD_REQ:
+		*count = ARRAY_SIZE(ele_gc_aead_addr_fields);
+		return ele_gc_aead_addr_fields;
+	case ELE_GC_ACRYPTO_REQ:
+		*count = ARRAY_SIZE(ele_gc_acrypto_addr_fields);
+		return ele_gc_acrypto_addr_fields;
+	case ELE_GC_AKEY_GEN_REQ:
+		*count = ARRAY_SIZE(ele_gc_akey_gen_addr_fields);
+		return ele_gc_akey_gen_addr_fields;
+	case ELE_HASH_ONE_GO_REQ:
+		*count = ARRAY_SIZE(ele_hash_one_go_addr_fields);
+		return ele_hash_one_go_addr_fields;
+	case ELE_RNG_GET_RANDOM_REQ:
+		*count = ARRAY_SIZE(ele_rng_get_random_addr_fields);
+		return ele_rng_get_random_addr_fields;
+	case ELE_STORAGE_MASTER_IMPORT_REQ:
+		*count = ARRAY_SIZE(ele_storage_master_import_addr_fields);
+		return ele_storage_master_import_addr_fields;
+	default:
+		*count = 0;
+		return NULL;
+	}
+}
+
+/*
+ * FW API for Command Receiver.
+ *
+ * Storage master-export response (ELE_STORAGE_MASTER_EXPORT_REQ).
+ * The cmd_receiver sends this response to firmware to supply the
+ * output buffer address. Payload layout:
+ *   data[0] = storage_handle
+ *   data[1] = rsp_code
+ *   data[2] = key_store_export_address  (LSB; high half always zero)
+ * No length word is present in the response itself; the export size is
+ * taken from the FW command received earlier (key_store_size) and stored
+ * in se_if_priv.cmd_rcvr_var_size by cmd_receiver_specific_ops().
+ * se_val_cmd_addrs() reads it when size_idx == SE_CMD_RCVR_ADDR_VAR_SIZE
+ * to range-check the full response buffer before it is forwarded to FW.
+ */
+static const struct se_cmd_addr_field ele_storage_master_export_addr_fields[] = {
+	{ .lsb_idx = 2, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = SE_CMD_RCVR_ADDR_VAR_SIZE },		/* key_store_export_address */
+};
+
+/*
+ * Storage chunk-get response (ELE_STORAGE_CHUNK_GET_REQ).
+ * The cmd_receiver fills in the chunk buffer address and its size so
+ * firmware can DMA the chunk data into the kernel's coherent buffer.
+ * Payload layout:
+ *   data[0] = chunk_size
+ *   data[1] = chunk_addr  (LSB; high half always zero)
+ *   data[2] = rsp_code
+ * The size word precedes the address in the message.
+ */
+static const struct se_cmd_addr_field ele_storage_chunk_get_addr_fields[] = {
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = 0, .size_mask = 0xFFFFFFFFu },		/* chunk_addr */
+};
+
+/*
+ * Storage chunk-export response (ELE_STORAGE_CHUNK_EXPORT_REQ).
+ * The cmd_receiver supplies the output buffer address. Payload layout:
+ *   data[0] = rsp_code
+ *   data[1] = chunk_export_address  (LSB; high half always zero)
+ * No length word is present in the response itself; the export size is
+ * taken from the FW command received earlier (chunk_size) and stored
+ * in se_if_priv.cmd_rcvr_var_size by cmd_receiver_specific_ops().
+ * se_val_cmd_addrs() reads it when size_idx == SE_CMD_RCVR_ADDR_VAR_SIZE
+ * to range-check the full response buffer before it is forwarded to FW.
+ */
+static const struct se_cmd_addr_field ele_storage_chunk_export_addr_fields[] = {
+	{ .lsb_idx = 1, .flag_idx = SE_CMD_ADDR_ALWAYS,
+	  .size_idx = SE_CMD_RCVR_ADDR_VAR_SIZE },		/* chunk_export_address */
+};
+
+/*
+ * Return the address-field descriptor table for a cmd_receiver response
+ * message (rsp_tag), or NULL when the response embeds no DMA addresses.
+ * count is set to the number of entries. Only the three storage responses
+ * that supply a kernel buffer address to firmware are covered here;
+ * ELE_STORAGE_EXPORT_FINISH_REQ, ELE_STORAGE_CHUNK_GET_DONE_REQ, and
+ * ELE_STORAGE_CHUNK_DELETE_REQ carry no DMA addresses and return NULL.
+ */
+const struct se_cmd_addr_field *ele_fw_rsp_addr_fields(u8 cmd, size_t *count)
+{
+	switch (cmd) {
+	case ELE_STORAGE_MASTER_EXPORT_REQ:
+		*count = ARRAY_SIZE(ele_storage_master_export_addr_fields);
+		return ele_storage_master_export_addr_fields;
+	case ELE_STORAGE_CHUNK_GET_REQ:
+		*count = ARRAY_SIZE(ele_storage_chunk_get_addr_fields);
+		return ele_storage_chunk_get_addr_fields;
+	case ELE_STORAGE_CHUNK_EXPORT_REQ:
+		*count = ARRAY_SIZE(ele_storage_chunk_export_addr_fields);
+		return ele_storage_chunk_export_addr_fields;
+	default:
+		*count = 0;
+		return NULL;
+	}
+}
diff --git a/drivers/firmware/imx/imx-dsp.c b/drivers/firmware/imx/imx-dsp.c
index ed79e823157afb..7e3f45ddf0c18f 100644
--- a/drivers/firmware/imx/imx-dsp.c
+++ b/drivers/firmware/imx/imx-dsp.c
@@ -99,8 +99,10 @@ static int imx_dsp_setup_channels(struct imx_dsp_ipc *dsp_ipc)
 		else
 			chan_name = kasprintf(GFP_KERNEL, "rxdb%d", i - 2);
 
-		if (!chan_name)
-			return -ENOMEM;
+		if (!chan_name) {
+			ret = -ENOMEM;
+			goto out;
+		}
 
 		dsp_chan = &dsp_ipc->chans[i];
 		dsp_chan->name = chan_name;
diff --git a/drivers/firmware/imx/se_ctrl.c b/drivers/firmware/imx/se_ctrl.c
new file mode 100644
index 00000000000000..519f24d24a8599
--- /dev/null
+++ b/drivers/firmware/imx/se_ctrl.c
@@ -0,0 +1,2434 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2026 NXP
+ */
+
+#include <linux/bitfield.h>
+#include <linux/cleanup.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/export.h>
+#include <linux/firmware.h>
+#include <linux/firmware/imx/se_api.h>
+#include <linux/genalloc.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kref.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/sched/mm.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/sys_soc.h>
+#include <uapi/linux/se_ioctl.h>
+
+#include "ele_base_msg.h"
+#include "ele_common.h"
+#include "ele_fw_api.h"
+#include "se_ctrl.h"
+
+/* Maximum response buffer size in bytes for debug-dump replies. */
+#define MAX_ALLOWED_TX_MSG_SZ		SZ_4K
+
+#define MAX_SOC_INFO_DATA_SZ		256
+
+struct se_soc_dev_regn {
+	bool soc_dev_registered;
+	struct soc_device *soc_dev;
+	struct soc_device_attribute *soc_dev_attr;
+};
+
+struct se_var_info {
+	u16 soc_rev;
+	struct se_soc_dev_regn soc_dev_regn;
+	/* To serialize populating common SoC level info. */
+	struct mutex se_var_info_lock;
+};
+
+/* contains fixed information */
+struct se_soc_info {
+	const u16 soc_id;
+	const char *soc_name;
+	const struct se_fw_img_name se_fw_img_nm;
+	bool imem_state_mgmt;
+};
+
+struct se_if_node {
+	struct se_soc_info *se_info;
+	u8 *pool_name;
+	bool reserved_dma_ranges;
+	struct se_if_defines if_defs;
+};
+
+/* common for all the SoC. */
+static struct se_var_info var_se_info = {
+	.soc_rev = 0,
+	.se_var_info_lock = __MUTEX_INITIALIZER(var_se_info.se_var_info_lock)
+};
+
+static struct se_soc_info se_imx8ulp_info = {
+	.soc_id = SOC_ID_OF_IMX8ULP,
+	.soc_name = "i.MX8ULP",
+	.se_fw_img_nm = {
+		.prim_fw_nm_in_rfs = IMX_ELE_FW_DIR
+			"mx8ulpa2-ahab-container.img",
+		.seco_fw_nm_in_rfs = IMX_ELE_FW_DIR
+			"mx8ulpa2ext-ahab-container.img",
+	},
+	.imem_state_mgmt = true,
+};
+
+static struct se_if_node imx8ulp_se_ele_hsm = {
+	.se_info = &se_imx8ulp_info,
+	.pool_name = "sram",
+	.reserved_dma_ranges = true,
+	.if_defs = {
+		.se_if_type = SE_TYPE_ID_HSM,
+		.cmd_tag = 0x17,
+		.rsp_tag = 0xe1,
+		.success_tag = ELE_SUCCESS_IND,
+		.base_api_ver = MESSAGING_VERSION_6,
+		.fw_api_ver = MESSAGING_VERSION_7,
+	},
+};
+
+static struct se_soc_info se_imx93_info = {
+	.soc_id = SOC_ID_OF_IMX93,
+};
+
+static struct se_if_node imx93_se_ele_hsm = {
+	.se_info = &se_imx93_info,
+	.reserved_dma_ranges = true,
+	.if_defs = {
+		.se_if_type = SE_TYPE_ID_HSM,
+		.cmd_tag = 0x17,
+		.rsp_tag = 0xe1,
+		.success_tag = ELE_SUCCESS_IND,
+		.base_api_ver = MESSAGING_VERSION_6,
+		.fw_api_ver = MESSAGING_VERSION_7,
+	},
+};
+
+static const struct of_device_id se_match[] = {
+	{ .compatible = "fsl,imx8ulp-se-ele-hsm", .data = &imx8ulp_se_ele_hsm },
+	{ .compatible = "fsl,imx93-se-ele-hsm", .data = &imx93_se_ele_hsm },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, se_match);
+
+/**
+ * get_se_if_name() - return a human-readable string for a SE interface type.
+ * @se_if_id: SE interface type identifier (e.g. SE_TYPE_ID_HSM).
+ *
+ * Return: pointer to a constant string naming the interface type, or "unknown"
+ * if @se_if_id does not match any known type.
+ */
+char *get_se_if_name(u8 se_if_id)
+{
+	switch (se_if_id) {
+	case SE_TYPE_ID_DBG: return SE_TYPE_STR_DBG;
+	case SE_TYPE_ID_HSM: return SE_TYPE_STR_HSM;
+	}
+
+	return "unknown";
+}
+
+static u32 get_se_soc_id(struct se_if_priv *priv)
+{
+	const struct se_if_node *if_node = device_get_match_data(priv->dev);
+
+	return if_node->se_info->soc_id;
+}
+
+static struct se_fw_load_info *get_load_fw_instance(struct se_if_priv *priv)
+{
+	return &priv->load_fw;
+}
+
+static void se_soc_device_unregister(struct se_soc_dev_regn *soc_dev_regn)
+{
+	guard(mutex)(&var_se_info.se_var_info_lock);
+
+	if (soc_dev_regn->soc_dev) {
+		soc_device_unregister(soc_dev_regn->soc_dev);
+		soc_dev_regn->soc_dev = NULL;
+	}
+
+	if (soc_dev_regn->soc_dev_attr) {
+		/*
+		 * revision and serial_number are the only kasprintf()-allocated
+		 * strings. machine points into the DT, and soc_id/family are
+		 * constants, so they must not be freed.
+		 */
+		kfree(soc_dev_regn->soc_dev_attr->revision);
+		kfree(soc_dev_regn->soc_dev_attr->serial_number);
+		kfree(soc_dev_regn->soc_dev_attr);
+		soc_dev_regn->soc_dev_attr = NULL;
+	}
+
+	soc_dev_regn->soc_dev_registered = false;
+}
+
+/*
+ * Build and register a soc_device entry for this SoC. Separated from
+ * get_se_soc_info() so that the firmware-fetch path and the sysfs
+ * registration path can be reasoned about independently.
+ */
+static int se_soc_dev_register(struct se_if_priv *priv, u16 soc_rev,
+			       const char *soc_name, const u8 *uid)
+{
+	struct soc_device_attribute *attr;
+	struct soc_device *sdev;
+	int err;
+
+	if (!soc_rev || !soc_name || !uid)
+		return -EINVAL;
+
+	attr = kzalloc_obj(*attr);
+	if (!attr)
+		return -ENOMEM;
+
+	if (FIELD_GET(DEV_GETINFO_MIN_VER_MASK, soc_rev))
+		attr->revision = kasprintf(GFP_KERNEL, "%x.%x",
+					   FIELD_GET(DEV_GETINFO_MAJ_VER_MASK, soc_rev),
+					   FIELD_GET(DEV_GETINFO_MIN_VER_MASK, soc_rev));
+	else
+		attr->revision = kasprintf(GFP_KERNEL, "%x",
+					   FIELD_GET(DEV_GETINFO_MAJ_VER_MASK, soc_rev));
+
+	if (!attr->revision) {
+		err = -ENOMEM;
+		goto err_free_attr;
+	}
+
+	attr->soc_id = soc_name;
+
+	err = of_property_read_string(of_root, "model", &attr->machine);
+	if (err) {
+		err = -EINVAL;
+		goto err_free_rev;
+	}
+
+	attr->family = "Freescale i.MX";
+
+	attr->serial_number = kasprintf(GFP_KERNEL, "%016llX",
+					GET_SERIAL_NUM_FROM_UID(uid, MAX_UID_SIZE >> 2));
+	if (!attr->serial_number) {
+		err = -ENOMEM;
+		goto err_free_rev;
+	}
+
+	sdev = soc_device_register(attr);
+	if (IS_ERR(sdev)) {
+		err = PTR_ERR(sdev);
+		goto err_free_serial;
+	}
+
+	/*
+	 * Publish the singleton. Freed once, at module unload, by
+	 * se_soc_device_unregister(). Caller holds se_var_info_lock.
+	 */
+	var_se_info.soc_dev_regn.soc_dev = sdev;
+	var_se_info.soc_dev_regn.soc_dev_attr = attr;
+
+	/* Mark registration complete so get_se_soc_info() skips this path on retry. */
+	var_se_info.soc_dev_regn.soc_dev_registered = true;
+
+	return 0;
+
+err_free_serial:
+	kfree(attr->serial_number);
+err_free_rev:
+	kfree(attr->revision);
+err_free_attr:
+	kfree(attr);
+
+	return err;
+}
+
+static int get_se_soc_info(struct se_if_priv *priv, const struct se_soc_info *se_info)
+{
+	struct se_fw_load_info *load_fw = get_load_fw_instance(priv);
+	u8 data[MAX_SOC_INFO_DATA_SZ];
+	struct ele_dev_info *s_info;
+	int err;
+
+	guard(mutex)(&var_se_info.se_var_info_lock);
+
+	/*
+	 * Early exit: both objectives already complete, nothing to do.
+	 * Do not exit early when imem_mgmt is active: load_fw is per-probe
+	 * (embedded in priv) and starts zeroed on every probe, so imem.state
+	 * must be refreshed from firmware on each probe even when soc_rev is
+	 * already cached in the module-lifetime var_se_info.
+	 */
+	if (var_se_info.soc_rev &&
+	    (!se_info->soc_name || var_se_info.soc_dev_regn.soc_dev_registered) &&
+	    !load_fw->imem_mgmt)
+		return 0;
+
+	err = ele_fetch_soc_info(priv, &data);
+	if (err < 0)
+		return dev_err_probe(priv->dev, err, "Failed to fetch SoC Info.\n");
+
+	s_info = (struct ele_dev_info *)data;
+
+	if (!var_se_info.soc_rev)
+		var_se_info.soc_rev = s_info->d_info.soc_rev;
+
+	/*
+	 * imem.state is per-probe state (lives in priv->load_fw which is
+	 * zeroed on every probe). Update it unconditionally whenever the
+	 * IMEM management path is active, regardless of whether soc_rev was
+	 * already cached from a previous probe or a sibling interface.
+	 */
+	if (load_fw->imem_mgmt)
+		load_fw->imem.state = s_info->d_addn_info.imem_state;
+
+	if (se_info->soc_name && !var_se_info.soc_dev_regn.soc_dev_registered) {
+		err = se_soc_dev_register(priv, var_se_info.soc_rev,
+					  se_info->soc_name, s_info->d_info.uid);
+		if (err < 0)
+			return dev_err_probe(priv->dev, err,
+					     "Failed to register SE SoC device.\n");
+	}
+
+	return 0;
+}
+
+static int load_firmware(struct se_if_priv *priv, const u8 *se_img_file_to_load)
+{
+	const struct firmware *fw = NULL;
+	dma_addr_t se_fw_dma_addr;
+	u32 se_fw_buf_len;
+	void *se_fw_buf;
+	int ret;
+
+	if (!se_img_file_to_load) {
+		dev_err(priv->dev, "FW image is not provided.\n");
+		return -EINVAL;
+	}
+	ret = request_firmware(&fw, se_img_file_to_load, priv->dev);
+	if (ret)
+		return ret;
+
+	if (fw->size > U32_MAX) {
+		ret = -EFBIG;
+		release_firmware(fw);
+		return ret;
+	}
+	dev_info(priv->dev, "loading firmware %s.\n", se_img_file_to_load);
+
+	/*
+	 * Serialize access to priv_dev_ctx shared memory to prevent pos
+	 * corruption if two driver-internal callers run concurrently (e.g.
+	 * ele_get_info() racing with load_firmware()).
+	 */
+	scoped_guard(mutex, &priv->priv_dev_ctx->fops_lock) {
+		se_fw_buf_len = fw->size;
+		ret = get_shared_mem_slot(priv->priv_dev_ctx,
+					  &se_fw_buf_len, &se_fw_dma_addr,
+					  &se_fw_buf);
+		if (ret) {
+			dev_err(priv->dev, "Failed to allocate firmware shared buffer: %d\n",
+				ret);
+			release_firmware(fw);
+			return ret;
+		}
+
+		memcpy(se_fw_buf, fw->data, fw->size);
+		ret = ele_fw_authenticate(priv, se_fw_dma_addr, se_fw_dma_addr);
+		if (ret < 0) {
+			dev_err(priv->dev,
+				"Error %pe: Authenticate & load SE firmware %s.",
+				ERR_PTR(ret), se_img_file_to_load);
+			ret = -EPERM;
+		}
+		if (!se_is_fw_busy_ctx(priv->priv_dev_ctx))
+			se_dev_ctx_shared_mem_cleanup(priv->priv_dev_ctx);
+	}
+
+	release_firmware(fw);
+
+	return ret;
+}
+
+static int se_load_firmware(struct se_if_priv *priv)
+{
+	struct se_fw_load_info *load_fw = get_load_fw_instance(priv);
+	int ret = 0;
+
+	guard(mutex)(&load_fw->load_fw_lock);
+	if (!load_fw->is_fw_tobe_loaded)
+		return 0;
+
+	if (load_fw->imem.state == ELE_IMEM_STATE_BAD) {
+		ret = load_firmware(priv, load_fw->se_fw_img_nm->prim_fw_nm_in_rfs);
+		if (ret) {
+			dev_err(priv->dev, "Failed to load boot firmware.\n");
+			return -EPERM;
+		}
+	}
+
+	ret = load_firmware(priv, load_fw->se_fw_img_nm->seco_fw_nm_in_rfs);
+	if (ret) {
+		dev_err(priv->dev, "Failed to load runtime firmware.\n");
+		return -EPERM;
+	}
+
+	load_fw->is_fw_tobe_loaded = false;
+
+	return ret;
+}
+
+static int init_se_shared_mem(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	INIT_LIST_HEAD(&se_shared_mem_mgmt->pending_out);
+	INIT_LIST_HEAD(&se_shared_mem_mgmt->pending_in);
+
+	if (priv->mem_pool)
+		INIT_LIST_HEAD(&se_shared_mem_mgmt->mem_pool_buf_list);
+
+	se_shared_mem_mgmt->non_secure_mem.ptr =
+			dma_alloc_coherent(priv->dev, MAX_DATA_SIZE_PER_USER,
+					   &se_shared_mem_mgmt->non_secure_mem.dma_addr,
+					   GFP_KERNEL);
+	if (!se_shared_mem_mgmt->non_secure_mem.ptr)
+		return -ENOMEM;
+
+	se_shared_mem_mgmt->non_secure_mem.size = MAX_DATA_SIZE_PER_USER;
+	se_shared_mem_mgmt->non_secure_mem.pos = 0;
+
+	return 0;
+}
+
+static void cleanup_se_shared_mem(struct se_if_device_ctx *dev_ctx, bool reclaim)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+	bool free_dma_buf;
+
+	/*
+	 * mem_pool_buf_list is only initialised for interfaces that own a
+	 * gen_pool (priv->mem_pool != NULL). On interfaces without a pool
+	 * (e.g. imx93, which has no pool_name) the list head is left
+	 * zero-filled, so se_cleanup_mem_pool_buf() must not walk it here or
+	 * list_for_each_entry_safe() would dereference a NULL head and panic
+	 * the kernel on close/teardown. Skip the pool cleanup entirely when
+	 * there is no pool; there is nothing to reclaim in that case.
+	 */
+	if (priv->mem_pool)
+		se_cleanup_mem_pool_buf(dev_ctx, reclaim);
+
+	/* Guard against being called before shared memory was ever allocated
+	 * (e.g. probe failure before dma_alloc_coherent succeeded).
+	 */
+	if (!se_shared_mem_mgmt->non_secure_mem.ptr)
+		return;
+
+	/*
+	 * Decide whether the DMA buffer can be released before touching the
+	 * pending lists. se_dev_ctx_shared_mem_cleanup() resets
+	 * non_secure_mem.pos, so the "nothing staged" test must be sampled
+	 * here first. When reclaim is false the buffer is released only if no
+	 * data is still staged for the firmware; otherwise the enclave may
+	 * still be DMA-ing into it and the buffer is deliberately leaked to
+	 * avoid a DMA-after-free.
+	 */
+	free_dma_buf = reclaim || !se_shared_mem_mgmt->non_secure_mem.pos;
+
+	/*
+	 * Free any se_buf_desc items that were never consumed (e.g. when the
+	 * fd is closed while pending I/O buffers are still listed). This must
+	 * happen before the DMA backing memory is released to avoid a leak.
+	 */
+	se_dev_ctx_shared_mem_cleanup(dev_ctx);
+
+	if (free_dma_buf) {
+		dma_free_coherent(priv->dev, MAX_DATA_SIZE_PER_USER,
+				  se_shared_mem_mgmt->non_secure_mem.ptr,
+				  se_shared_mem_mgmt->non_secure_mem.dma_addr);
+	}
+
+	/*
+	 * Drop the host-side tracking unconditionally. On the reclaim path the
+	 * buffer has been freed. On the deliberate-leak path the buffer is
+	 * abandoned on purpose, so clearing the pointer here guarantees a later
+	 * cleanup pass (e.g. se_if_priv_release()) cannot double-free it.
+	 */
+	se_shared_mem_mgmt->non_secure_mem.ptr = NULL;
+	se_shared_mem_mgmt->non_secure_mem.dma_addr = 0;
+	se_shared_mem_mgmt->non_secure_mem.size = 0;
+	se_shared_mem_mgmt->non_secure_mem.pos = 0;
+}
+
+static int se_dev_ctx_cpy_out_data(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_buf_desc *b_desc, *temp;
+	bool do_cpy = true;
+
+	list_for_each_entry_safe(b_desc, temp, &se_shared_mem_mgmt->pending_out, link) {
+		if (b_desc->usr_buf_ptr && b_desc->shared_buf_ptr && do_cpy) {
+			dev_dbg(priv->dev, "Copying output data to user.\n");
+			if (do_cpy && copy_to_user(b_desc->usr_buf_ptr,
+						   b_desc->shared_buf_ptr,
+						   b_desc->size)) {
+				dev_err(priv->dev, "Failure copying output data to user.\n");
+				do_cpy = false;
+			}
+		}
+
+		if (b_desc->shared_buf_ptr)
+			memset(b_desc->shared_buf_ptr, 0, b_desc->size);
+
+		list_del(&b_desc->link);
+		kfree(b_desc);
+	}
+
+	return do_cpy ? 0 : -EFAULT;
+}
+
+/*
+ * Clean the used Shared Memory space,
+ * whether its Input Data copied from user buffers, or
+ * Data received from FW.
+ */
+void se_dev_ctx_shared_mem_cleanup(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct list_head *pending_lists[] = {&se_shared_mem_mgmt->pending_in,
+						&se_shared_mem_mgmt->pending_out};
+	struct se_buf_desc *b_desc, *temp;
+	bool is_fw_busy_dev_ctx;
+	int i;
+
+	/*
+	 * If this context is the one that caused a firmware timeout the shared
+	 * DMA buffers may still be actively read/written by the firmware.
+	 */
+	is_fw_busy_dev_ctx = se_is_fw_busy_ctx(dev_ctx);
+
+	for (i = 0; i < ARRAY_SIZE(pending_lists); i++) {
+		list_for_each_entry_safe(b_desc, temp, pending_lists[i], link) {
+			if (!is_fw_busy_dev_ctx && b_desc->shared_buf_ptr)
+				memset(b_desc->shared_buf_ptr, 0, b_desc->size);
+
+			list_del(&b_desc->link);
+			kfree(b_desc);
+		}
+	}
+
+	/*
+	 * Keep non_secure_mem.pos non-zero while this context still owns an
+	 * outstanding firmware transaction. A non-zero pos is the marker that
+	 * data is still staged for the enclave, which cleanup_se_shared_mem()
+	 * uses to decide the buffer must be leaked rather than freed. Resetting
+	 * it here would let a later teardown pass free a buffer the enclave may
+	 * still be DMA-ing into.
+	 */
+	if (!is_fw_busy_dev_ctx)
+		se_shared_mem_mgmt->non_secure_mem.pos = 0;
+}
+
+static struct se_buf_desc *add_b_desc_to_pending_list(void *shared_ptr_with_pos,
+						      struct se_ioctl_setup_iobuf *io,
+						      struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_buf_desc *b_desc = NULL;
+
+	b_desc = kzalloc_obj(*b_desc);
+	if (!b_desc)
+		return ERR_PTR(-ENOMEM);
+
+	b_desc->shared_buf_ptr = shared_ptr_with_pos;
+	b_desc->usr_buf_ptr = u64_to_user_ptr(io->user_buf);
+	b_desc->size = io->length;
+
+	if (io->flags & SE_IO_BUF_FLAGS_IS_INPUT) {
+		/*
+		 * buffer is input:
+		 * add an entry in the "pending input buffers" list so
+		 * that copied data can be cleaned from shared memory
+		 * later.
+		 */
+		list_add_tail(&b_desc->link, &se_shared_mem_mgmt->pending_in);
+	} else {
+		/*
+		 * buffer is output:
+		 * add an entry in the "pending out buffers" list so data
+		 * can be copied to user space when receiving Secure-Enclave
+		 * response.
+		 */
+		list_add_tail(&b_desc->link, &se_shared_mem_mgmt->pending_out);
+	}
+
+	return b_desc;
+}
+
+static void se_if_open_gate_release(struct kref *kref)
+{
+	struct se_if_open_gate *gate =
+		container_of(kref, struct se_if_open_gate, refcount);
+
+	kfree(gate);
+}
+
+static bool se_if_open_gate_get(struct se_if_open_gate *gate)
+{
+	if (!gate)
+		return false;
+
+	return kref_get_unless_zero(&gate->refcount);
+}
+
+static void se_if_open_gate_put(struct se_if_open_gate *gate)
+{
+	if (gate)
+		kref_put(&gate->refcount, se_if_open_gate_release);
+}
+
+/*
+ * Distinct lockdep class for the internal priv_dev_ctx fops_lock. Taking it
+ * while an open context's fops_lock is held (for example a firmware load
+ * triggered from an ioctl) is valid hierarchical locking, but shares the same
+ * class as the per-open fops_lock and would otherwise be misreported as
+ * recursive locking by lockdep.
+ */
+static struct lock_class_key se_priv_ctx_fops_key;
+
+static int init_misc_device_context(struct se_if_priv *priv, int ch_id,
+				    struct se_if_device_ctx **new_dev_ctx,
+				    const struct file_operations *se_if_fops)
+{
+	struct se_if_open_gate *gate = NULL;
+	struct se_if_device_ctx *dev_ctx;
+	int ret = -ENOMEM;
+
+	dev_ctx = kzalloc_obj(*dev_ctx);
+	if (!dev_ctx)
+		return -ENOMEM;
+
+	dev_ctx->priv = priv;
+	dev_ctx->devname = kasprintf(GFP_KERNEL, "%s0_ch%d",
+				     get_se_if_name(priv->if_defs->se_if_type),
+				     ch_id);
+	if (!dev_ctx->devname) {
+		kfree(dev_ctx);
+		return -ENOMEM;
+	}
+
+	mutex_init(&dev_ctx->fops_lock);
+	lockdep_set_class(&dev_ctx->fops_lock, &se_priv_ctx_fops_key);
+
+	kref_init(&dev_ctx->refcount);
+	dev_ctx->cleanup_done = false;
+	*new_dev_ctx = dev_ctx;
+	set_se_rcv_msg_timeout(dev_ctx, SE_RCV_MSG_DEFAULT_TIMEOUT_MS);
+
+	ret = init_se_shared_mem(dev_ctx);
+	if (ret < 0)
+		goto exit;
+
+	gate = kzalloc_obj(*gate);
+	if (!gate) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	mutex_init(&gate->lock);
+	kref_init(&gate->refcount);    /* device-owned reference */
+	gate->priv = priv;
+	gate->dying = false;
+	priv->open_gate = gate;
+
+	/*
+	 * The miscdevice storage is now owned by the open gate object.
+	 * priv->priv_dev_ctx still keeps a pointer to that miscdevice.
+	 */
+	dev_ctx->miscdev = &gate->miscdev;
+
+	dev_ctx->miscdev->name = dev_ctx->devname;
+	dev_ctx->miscdev->minor = MISC_DYNAMIC_MINOR;
+	dev_ctx->miscdev->fops = se_if_fops;
+	dev_ctx->miscdev->parent = priv->dev;
+
+	return 0;
+exit:
+	*new_dev_ctx = NULL;
+
+	if (gate) {
+		priv->open_gate = NULL;
+		se_if_open_gate_put(gate);
+	}
+	cleanup_se_shared_mem(dev_ctx, true);
+	kfree(dev_ctx->devname);
+	kfree(dev_ctx);
+	return ret;
+}
+
+static int se_if_request_channel(struct device *dev, struct mbox_chan **chan,
+				 struct mbox_client *cl, const char *name)
+{
+	struct mbox_chan *t_chan;
+
+	t_chan = mbox_request_channel_byname(cl, name);
+	if (IS_ERR(t_chan))
+		return dev_err_probe(dev, PTR_ERR(t_chan),
+				     "Failed to request %s channel.\n", name);
+
+	*chan = t_chan;
+
+	return 0;
+}
+
+/*
+ * Forward declarations. se_if_probe_cleanup() and se_if_probe() are kept
+ * together as the teardown/probe pair, but several helpers, the file
+ * operations table and the firmware-busy work handler they reference are
+ * defined further down in this file.
+ */
+static void dlink_dev_ctx(struct se_if_device_ctx *dev_ctx);
+static void cleanup_dev_ctx(struct se_if_device_ctx *dev_ctx, bool is_fclose);
+static void se_clear_fw_busy(struct se_if_priv *priv);
+static void se_if_dev_ctx_release(struct kref *kref);
+static void se_if_priv_release(struct kref *kref);
+static int se_if_misc_register(struct se_if_priv *priv);
+static void se_fw_busy_work(struct work_struct *work);
+static const struct file_operations se_if_fops;
+
+static void se_if_probe_cleanup(void *plat_dev)
+{
+	struct platform_device *pdev = plat_dev;
+	struct se_if_device_ctx *dev_ctx;
+	struct device *dev = &pdev->dev;
+	struct fw_busy_info *fbusy_info;
+	struct se_if_priv *priv;
+
+	priv = dev_get_drvdata(dev);
+	if (!priv)
+		return;
+
+	fbusy_info = &priv->fw_busy_info;
+
+	/*
+	 * Announce teardown, then wake any in-flight waiter. going_away makes
+	 * ele_msg_send_rcv() bail out instead of arming a new transaction and
+	 * lets ele_msg_rcv() tell a teardown-forced completion apart from a
+	 * real response; it must be set before complete_all().
+	 *
+	 * Set it under clbk_rx_lock, not se_if_cmd_lock: se_if_cmd_lock is held
+	 * across the whole blocking transaction, so taking it here would stall
+	 * unbind for a full receive-timeout. clbk_rx_lock is the short spinlock
+	 * ele_msg_send_rcv() holds while arming, so this closes the lost-wakeup
+	 * window - the sender either sees going_away and bails before arming, or
+	 * armed first and this store (and complete_all()) is ordered after its
+	 * reinit_completion() - and supplies the ordering the relaxed atomics do
+	 * not.
+	 */
+	scoped_guard(spinlock_irqsave, &priv->waiting_rsp_clbk_hdl.clbk_rx_lock)
+		atomic_set(&priv->going_away, 1);
+	/*
+	 * Wake the waiter before iterating the device-context list. It sleeps on
+	 * this completion holding dev_ctx->fops_lock, which cleanup_dev_ctx()
+	 * below also takes, so completing first avoids an unbind hang. Runs
+	 * outside clbk_rx_lock; the going_away store above already orders it
+	 * against the arming path.
+	 */
+	complete_all(&priv->waiting_rsp_clbk_hdl.done);
+
+	/*
+	 * Only now reserve the messaging interface for this teardown flow.
+	 *
+	 * se_reserve_msg_if() blocks on msg_excl_flow_lock, and the fw_busy
+	 * recovery worker (se_fw_busy_work() -> se_clear_fw_busy()) may already
+	 * hold that reservation while parked uninterruptibly in ele_msg_rcv()
+	 * waiting on a possibly hung firmware for up to the full receive
+	 * timeout. The only thing that cuts that wait short is the complete_all()
+	 * above, so it MUST run before this reserve: otherwise teardown would
+	 * sleep on the reservation the worker holds, the worker would stay
+	 * blocked on firmware, and unbind would stall for the entire multi-
+	 * thousand-second timeout (an unbind hang / hung-task).
+	 *
+	 * With going_away already set and the in-flight waiter already forced to
+	 * unwind, the worker returns promptly (its send is failed with -ENODEV),
+	 * drops the reservation, and this call acquires it without waiting on
+	 * anything firmware-related. From here on teardown is the exclusive
+	 * owner: ele_msg_send_rcv() lets only this task's priv_dev_ctx close
+	 * traffic through and rejects every other caller.
+	 */
+	se_reserve_msg_if(priv);
+
+	/*
+	 * Mark the private device context as cleanup_done first.
+	 * This prevents new device contexts from being created in open().
+	 */
+	if (priv->priv_dev_ctx) {
+		/*
+		 * Mark cleanup_done under fops_lock so that se_if_fops_open(),
+		 * which checks cleanup_done while holding fops_lock, cannot
+		 * race past this and add a new device context after teardown.
+		 */
+		scoped_guard(mutex, &priv->priv_dev_ctx->fops_lock)
+			priv->priv_dev_ctx->cleanup_done = true;
+
+		if (priv->open_gate) {
+			scoped_guard(mutex, &priv->open_gate->lock) {
+				priv->open_gate->dying = true;
+				priv->open_gate->priv = NULL;
+			}
+		}
+
+		/*
+		 * misc_register() is deferred to the end of probe, so the
+		 * device may have a miscdev set up but never registered if
+		 * probe failed before se_if_misc_register(). Only deregister
+		 * when registration actually succeeded.
+		 */
+		if (priv->open_gate && priv->open_gate->registered &&
+		    priv->priv_dev_ctx->miscdev)
+			misc_deregister(priv->priv_dev_ctx->miscdev);
+	}
+
+	while (true) {
+		bool list_was_empty = false;
+
+		dev_ctx = NULL;
+
+		scoped_guard(mutex, &priv->modify_lock) {
+			if (list_empty(&priv->dev_ctx_list)) {
+				list_was_empty = true;
+			} else {
+				dev_ctx = list_first_entry(&priv->dev_ctx_list,
+							   struct se_if_device_ctx, link);
+
+				/* pin this context so close() cannot free it under us */
+				kref_get(&dev_ctx->refcount);
+				dlink_dev_ctx(dev_ctx);
+			}
+		}
+
+		if (list_was_empty)
+			break;
+
+		/*
+		 * Local cleanup outside the global lock avoids ABBA deadlock
+		 * with paths that already take dev_ctx->fops_lock first.
+		 */
+		cleanup_dev_ctx(dev_ctx, false);
+		kref_put(&dev_ctx->refcount, se_if_dev_ctx_release);
+	}
+
+	se_release_msg_if(priv);
+	/*
+	 * Release any dev_ctx retained by the firmware-busy circuit breaker.
+	 * A synchronous command that timed out parks its dev_ctx in
+	 * fbusy_info->fw_busy_dev_ctx so that a late firmware response can still be
+	 * routed back. If no such response arrived before teardown, that
+	 * retained reference must be dropped here to avoid a leak.
+	 * se_clear_fw_busy() is safe to call unconditionally: it checks
+	 * fbusy_info->fw_busy_dev_ctx under fw_busy_lock and is a no-op when
+	 * nothing is parked.
+	 */
+	se_clear_fw_busy(priv);
+
+	/*
+	 * Free the mailbox channels under se_if_cmd_lock. ele_msg_send_rcv()
+	 * holds se_if_cmd_lock for the full duration of a synchronous
+	 * transaction, including the mbox_send_message() call on priv->tx_chan.
+	 * going_away was set above and complete_all() has already woken any
+	 * in-flight waiter, so any transaction in progress will unwind to
+	 * -ENODEV and release the lock quickly. Acquiring se_if_cmd_lock here
+	 * guarantees no caller is still touching the channels when they are
+	 * freed, and nulling the pointers under the lock prevents any sender
+	 * that races past the going_away check from accessing a freed channel.
+	 */
+	scoped_guard(mutex, &priv->se_if_cmd_lock) {
+		if (priv->rx_chan) {
+			mbox_free_channel(priv->rx_chan);
+			priv->rx_chan = NULL;
+		}
+		if (priv->tx_chan) {
+			mbox_free_channel(priv->tx_chan);
+			priv->tx_chan = NULL;
+		}
+	}
+
+	/*
+	 * Cancel any pending fw_busy_work before dropping the initial priv
+	 * reference. going_away was set above, so no new work can be scheduled
+	 * after this point. Canceling here while probe_cleanup still holds its
+	 * own priv reference prevents two races:
+	 *
+	 * 1. UAF: if fw_busy_work has dev_ctx == priv_dev_ctx, letting it run
+	 *    past this point while se_if_priv_release() frees priv_dev_ctx
+	 *    causes a use-after-free of dev_ctx->fops_lock in se_clear_fw_busy().
+	 *
+	 * 2. Deadlock: if fw_busy_work drops the last priv reference,
+	 *    se_if_dev_ctx_release() -> se_if_priv_release() would call
+	 *    cancel_work_sync() from inside the worker, causing the worker to
+	 *    wait for its own completion.
+	 *
+	 * Both are avoided by canceling here: probe_cleanup still holds a priv
+	 * reference so the worker cannot invoke se_if_priv_release(), and the
+	 * cancel runs from a non-worker context.
+	 */
+	cancel_work_sync(&fbusy_info->fw_busy_work);
+
+	/*
+	 * Reclaim priv_dev_ctx shared memory before of_reserved_mem_device_release():
+	 * cleanup_se_shared_mem() calls dma_free_coherent(), which must run while
+	 * the DMA config is still active. fw_busy_work was canceled above, so no
+	 * concurrent caller holds priv_dev_ctx->fops_lock.
+	 *
+	 * reclaim=true is safe even if FW hung at teardown (command timed out, pos
+	 * still non-zero): the ELE region is no-map/shared-dma-pool, so freeing only
+	 * drops the kernel VA/bitmap while the physical pages stay reserved (no
+	 * DMA-after-free). The next probe also sends ELE_GET_INFO into a fresh
+	 * buffer before accepting commands, so a stale FW write to the old buffer is
+	 * never observed by the new driver instance.
+	 */
+	if (priv->priv_dev_ctx) {
+		scoped_guard(mutex, &priv->priv_dev_ctx->fops_lock)
+			cleanup_se_shared_mem(priv->priv_dev_ctx, true);
+	}
+
+	/*
+	 * Release the reserved DMA memory configuration at unbind time, paired
+	 * with of_reserved_mem_device_init() in se_if_probe(). This must not be
+	 * deferred to se_if_priv_release(): that runs when the last file
+	 * descriptor closes, which may be after a new driver instance has already
+	 * called of_reserved_mem_device_init() on the same struct device. Calling
+	 * the release at that point would corrupt the new instance's DMA setup.
+	 */
+	of_reserved_mem_device_release(dev);
+
+	/*
+	 * Being device managed buffer, no need to free the buffer allocated
+	 * in se probe to store encrypted IMEM.
+	 */
+
+	dev_set_drvdata(dev, NULL);
+
+	/* Drop the initial reference - priv will be freed when last fd closes */
+	kref_put(&priv->refcount, se_if_priv_release);
+}
+
+static int se_if_probe(struct platform_device *pdev)
+{
+	const struct se_soc_info *se_info;
+	const struct se_if_node *if_node;
+	struct device *dev = &pdev->dev;
+	struct se_fw_load_info *load_fw;
+	struct se_if_priv *priv;
+	int ret;
+
+	if_node = device_get_match_data(dev);
+	if (!if_node)
+		return -EINVAL;
+
+	se_info = if_node->se_info;
+
+	priv = kzalloc_obj(*priv);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	/*
+	 * Pin the parent device for the lifetime of priv. A file descriptor may
+	 * stay open after the device is unbound; close() then still passes
+	 * priv->dev to dma_free_coherent()/dev_warn(). Without this reference
+	 * the struct device could be freed while priv->dev still points at it,
+	 * so the reference is dropped in se_if_priv_release() via put_device().
+	 */
+	get_device(priv->dev);
+	kref_init(&priv->refcount);
+	priv->if_defs = &if_node->if_defs;
+	dev_set_drvdata(dev, priv);
+
+	spin_lock_init(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock);
+	spin_lock_init(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock);
+	priv->msg_excl_flow.msg_excl_owner = NULL;
+	spin_lock_init(&priv->msg_excl_flow.msg_excl_lock);
+	mutex_init(&priv->msg_excl_flow.msg_excl_flow_lock);
+	struct fw_busy_info *fbusy_info = &priv->fw_busy_info;
+
+	atomic_set(&fbusy_info->fw_busy, 0);
+
+	spin_lock_init(&fbusy_info->fw_busy_lock);
+	fbusy_info->fw_busy_dev_ctx = NULL;
+	INIT_WORK(&fbusy_info->fw_busy_work, se_fw_busy_work);
+
+	init_completion(&priv->waiting_rsp_clbk_hdl.done);
+	init_completion(&priv->cmd_receiver_clbk_hdl.done);
+	INIT_LIST_HEAD(&priv->dev_ctx_list);
+
+	mutex_init(&priv->se_if_cmd_lock);
+	mutex_init(&priv->modify_lock);
+
+	load_fw = get_load_fw_instance(priv);
+	mutex_init(&load_fw->load_fw_lock);
+	if (se_info->se_fw_img_nm.seco_fw_nm_in_rfs) {
+		load_fw->se_fw_img_nm = &se_info->se_fw_img_nm;
+		load_fw->is_fw_tobe_loaded = true;
+	}
+	ret = devm_add_action_or_reset(dev, se_if_probe_cleanup, pdev);
+	if (ret)
+		return ret;
+
+	/* Mailbox client configuration */
+	priv->se_mb_cl.dev		= dev;
+	priv->se_mb_cl.tx_block		= false;
+	priv->se_mb_cl.knows_txdone	= false;
+	priv->se_mb_cl.rx_callback	= se_if_rx_callback;
+
+	ret = se_if_request_channel(dev, &priv->tx_chan, &priv->se_mb_cl, "tx");
+	if (ret)
+		return ret;
+
+	ret = se_if_request_channel(dev, &priv->rx_chan, &priv->se_mb_cl, "rx");
+	if (ret)
+		return ret;
+
+	if (if_node->pool_name) {
+		priv->mem_pool = of_gen_pool_get(dev->of_node, if_node->pool_name, 0);
+		if (!priv->mem_pool)
+			return dev_err_probe(dev, -ENOMEM,
+					     "Unable to get sram pool = %s.\n",
+					     if_node->pool_name);
+	}
+
+	if (if_node->reserved_dma_ranges) {
+		ret = of_reserved_mem_device_init(dev);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to init reserved memory region.\n");
+	}
+
+	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+
+	/* By default, there is no pending FW to be loaded.*/
+	if (se_info->imem_state_mgmt) {
+		/* allocate buffer where SE store encrypted IMEM */
+		load_fw->imem.buf = dmam_alloc_coherent(priv->dev, ELE_IMEM_SIZE,
+							&load_fw->imem.daddr,
+							GFP_KERNEL);
+		if (!load_fw->imem.buf)
+			return dev_err_probe(dev, -ENOMEM,
+					     "dmam-alloc-failed: To store encr-IMEM.\n");
+		load_fw->imem_mgmt = true;
+	}
+
+	ret = init_misc_device_context(priv, 0, &priv->priv_dev_ctx, &se_if_fops);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to create device contexts.\n");
+
+	if (if_node->if_defs.se_if_type == SE_TYPE_ID_HSM) {
+		ret = get_se_soc_info(priv, se_info);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to fetch SoC Info.\n");
+	}
+
+	/*
+	 * All probe-time initialization is complete; expose the
+	 * interface to userspace last so that an open()/ioctl cannot
+	 * race against a not-yet-initialized device.
+	 */
+	ret = se_if_misc_register(priv);
+	if (ret)
+		return ret;
+
+	dev_info(dev, "i.MX secure-enclave: %s0 interface to firmware, configured.\n",
+		 get_se_if_name(priv->if_defs->se_if_type));
+
+	return ret;
+}
+
+/*
+ * Expose the interface to userspace. Deferred until the end of probe so
+ * the device node only becomes openable after SoC info has been fetched
+ * and, on SoCs with IMEM management, the encrypted-IMEM buffer has been
+ * allocated. This prevents userspace from opening the node and issuing
+ * commands against a partially initialized interface.
+ */
+static int se_if_misc_register(struct se_if_priv *priv)
+{
+	int ret;
+
+	ret = misc_register(priv->priv_dev_ctx->miscdev);
+	if (ret)
+		return dev_err_probe(priv->dev, ret,
+				     "Failed to register misc device.");
+
+	priv->open_gate->registered = true;
+
+	return 0;
+}
+
+static void se_if_priv_release(struct kref *kref)
+{
+	struct se_if_priv *priv = container_of(kref, struct se_if_priv, refcount);
+
+	/*
+	 * Free priv_dev_ctx if it still exists. se_if_priv_release() always
+	 * runs after se_if_probe_cleanup() has completed: the initial kref
+	 * held by probe_cleanup is the last one dropped by probe_cleanup
+	 * itself, so no other kref_put() can reach zero -- and therefore
+	 * trigger se_if_priv_release() -- until probe_cleanup's own
+	 * kref_put() fires. By that time cleanup_se_shared_mem() and
+	 * of_reserved_mem_device_release() have already run in
+	 * probe_cleanup, so only the struct itself and its devname string
+	 * need to be freed here. Calling cleanup_se_shared_mem() again
+	 * would be a use-after-free of already-freed DMA memory.
+	 */
+	if (priv->priv_dev_ctx) {
+		kfree(priv->priv_dev_ctx->devname);
+		kfree(priv->priv_dev_ctx);
+		priv->priv_dev_ctx = NULL;
+	}
+	/*
+	 * Be defensive: if teardown did not already drop the device-owned
+	 * gate reference for some reason, release it here.
+	 */
+	if (priv->open_gate) {
+		se_if_open_gate_put(priv->open_gate);
+		priv->open_gate = NULL;
+	}
+
+	/*
+	 * Drop the reference on priv->dev taken in se_if_probe(). The device was
+	 * pinned so that a file descriptor closed after device unbind can still
+	 * safely pass priv->dev to dma_free_coherent()/dev_warn().
+	 */
+	put_device(priv->dev);
+	mutex_destroy(&priv->load_fw.load_fw_lock);
+	mutex_destroy(&priv->modify_lock);
+	mutex_destroy(&priv->se_if_cmd_lock);
+	mutex_destroy(&priv->msg_excl_flow.msg_excl_flow_lock);
+
+	/* Free any remaining resources that weren't devm-managed */
+	kfree(priv);
+}
+
+static void se_if_dev_ctx_release(struct kref *kref)
+{
+	struct se_if_device_ctx *dev_ctx =
+		container_of(kref, struct se_if_device_ctx, refcount);
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	kfree(dev_ctx);
+
+	/* drop the priv reference owned by this device context */
+	kref_put(&priv->refcount, se_if_priv_release);
+}
+
+/*
+ * se_reserve_msg_if() - reserve the SE messaging interface for the current task.
+ *
+ * Blocks on msg_excl_flow_lock until this task owns the reservation, then
+ * publishes current as msg_excl_owner under msg_excl_lock. While a reservation
+ * is held, ele_msg_send_rcv() lets only the owning task issue transactions and
+ * rejects every other caller with -EBUSY. Used by the fw_busy recovery flow in
+ * se_clear_fw_busy() to drive its teardown-close messages through the otherwise
+ * closed circuit breaker.
+ *
+ * If a second flow tries to reserve while the interface is already reserved,
+ * it sleeps on msg_excl_flow_lock until the current owner calls
+ * se_release_msg_if(). Must be called from process/workqueue context (it may
+ * sleep) and every successful call must be balanced by se_release_msg_if().
+ *
+ * Return: 0 (the reservation is always acquired once this returns).
+ */
+int se_reserve_msg_if(struct se_if_priv *priv)
+{
+	unsigned long flags;
+
+	mutex_lock(&priv->msg_excl_flow.msg_excl_flow_lock);
+	/*
+	 * The mutex guarantees this task is now the sole reserver, so
+	 * msg_excl_owner is either NULL or already current. Publish current
+	 * under msg_excl_lock so the lockless READ_ONCE in ele_msg_send_rcv()
+	 * observes a consistent pointer.
+	 */
+	spin_lock_irqsave(&priv->msg_excl_flow.msg_excl_lock, flags);
+	priv->msg_excl_flow.msg_excl_owner = current;
+	spin_unlock_irqrestore(&priv->msg_excl_flow.msg_excl_lock, flags);
+
+	return 0;
+}
+
+/* se_release_msg_if() - release a reservation taken by se_reserve_msg_if(). */
+void se_release_msg_if(struct se_if_priv *priv)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->msg_excl_flow.msg_excl_lock, flags);
+	priv->msg_excl_flow.msg_excl_owner = NULL;
+	spin_unlock_irqrestore(&priv->msg_excl_flow.msg_excl_lock, flags);
+	mutex_unlock(&priv->msg_excl_flow.msg_excl_flow_lock);
+}
+
+/* se_clear_fw_busy() - atomically clear fw_busy and reclaim the parked dev_ctx. */
+static void se_clear_fw_busy(struct se_if_priv *priv)
+{
+	struct fw_busy_info *fbusy_info = &priv->fw_busy_info;
+	struct se_if_device_ctx *dev_ctx = NULL;
+	unsigned long flags;
+
+	scoped_guard(spinlock_irqsave, &fbusy_info->fw_busy_lock) {
+		dev_ctx = fbusy_info->fw_busy_dev_ctx;
+		fbusy_info->fw_busy_dev_ctx = NULL;
+
+		if (!dev_ctx) {
+			/*
+			 * No parked context: nothing to recover. Clear fw_busy
+			 * and return without reserving the interface, so the
+			 * no-op path never leaves a dangling recovery
+			 * reservation. The scoped_guard releases fw_busy_lock
+			 * on this return.
+			 */
+			atomic_set(&fbusy_info->fw_busy, 0);
+			return;
+		}
+	}
+
+	/*
+	 * A context is parked and its handles must be recovered. Keep
+	 * fw_busy set (breaker stays closed to all third parties) and
+	 * reserve the SE interface exclusively for this recovery flow by
+	 * publishing the current task as msg_excl_owner. ele_msg_send_rcv()
+	 * then lets only this task's teardown-close messages through and
+	 * rejects everyone else with -EBUSY. The reservation is assigned here,
+	 * from outside ele_msg_send_rcv(), and released with se_release_msg_if()
+	 * at the end, after which the interface is available for general
+	 * se_if_cmd_lock message exchange.
+	 *
+	 * The scoped_guard above has already dropped fw_busy_lock before this
+	 * se_reserve_msg_if() call: the reserve helper takes msg_excl_lock, and
+	 * taking it while still holding fw_busy_lock would introduce a new
+	 * fw_busy_lock -> msg_excl_lock nesting. The brief fw_busy == 1 /
+	 * owner == NULL window that this opens is harmless - the breaker is
+	 * fully closed, so every caller (including a would-be re-arm) is
+	 * rejected with -EBUSY.
+	 */
+	se_reserve_msg_if(priv);
+
+	scoped_guard(mutex, &dev_ctx->fops_lock) {
+		/*
+		 * Snapshot any orphaned late FW response. On the teardown
+		 * path se_if_probe_cleanup calls se_clear_fw_busy before
+		 * cancel_work_sync, so fw_busy is still 1 here and a late
+		 * IRQ can write orphan_fw_rx_msg concurrently - take
+		 * clbk_rx_lock. On the workqueue path the IRQ writer has
+		 * already finished; the lock is a no-contention formality.
+		 * Call fw_api_specific_ops() outside the spinlock since it
+		 * may sleep.
+		 */
+		u8 late_rx_snap[MAX_ALLOWED_RX_MSG_SZ];
+		bool have_snap;
+
+		spin_lock_irqsave(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+		have_snap = fbusy_info->orphan_fw_rx_msg[0] != 0;
+		if (have_snap) {
+			memcpy(late_rx_snap, fbusy_info->orphan_fw_rx_msg,
+			       sizeof(late_rx_snap));
+			memset(fbusy_info->orphan_fw_rx_msg, 0,
+			       sizeof(fbusy_info->orphan_fw_rx_msg));
+		}
+		spin_unlock_irqrestore(&priv->waiting_rsp_clbk_hdl.clbk_rx_lock, flags);
+
+		if (have_snap) {
+			/*
+			 * FW responded late. DMA staging buffer is no longer
+			 * being written - safe to reclaim. Close any firmware
+			 * resource handle carried in the response.
+			 */
+			fw_api_specific_ops(priv->priv_dev_ctx,
+					    (struct se_api_msg *)late_rx_snap, true);
+
+			if (dev_ctx == priv->priv_dev_ctx) {
+				/*
+				 * Internal context: probe-time static DMA buf;
+				 * se_if_probe_cleanup reclaims it explicitly.
+				 * Just reset logical pos and return gen_pool
+				 * loan buffers for reuse.
+				 */
+				if (priv->mem_pool)
+					se_cleanup_mem_pool_buf(dev_ctx, true);
+				se_dev_ctx_shared_mem_cleanup(dev_ctx);
+			} else if (dev_ctx->cleanup_done) {
+				/*
+				 * Userspace fd already closed while fw_busy was
+				 * armed (e.g. SIGKILL). FW has now responded;
+				 * close deferred handles and free the DMA buf.
+				 *
+				 * cleanup_dev_ctx() already freed dev_ctx->devname
+				 * and set it to NULL, so use the stable snapshot
+				 * captured at arm time (fbusy_info->devname) for
+				 * these diagnostics rather than dev_ctx->devname.
+				 */
+				if (dev_ctx->strg_hdl &&
+				    se_close_storage(priv->priv_dev_ctx,
+						     dev_ctx->strg_hdl))
+					dev_err(priv->dev,
+						"%s: failed to close deferred storage handle\n",
+						fbusy_info->devname);
+				if (dev_ctx->sess_hdl &&
+				    se_close_session(priv->priv_dev_ctx,
+						     dev_ctx->sess_hdl))
+					dev_err(priv->dev,
+						"%s: failed to close deferred session handle\n",
+						fbusy_info->devname);
+				dev_ctx->strg_hdl = 0;
+				dev_ctx->sess_hdl = 0;
+				cleanup_se_shared_mem(dev_ctx, true);
+			} else {
+				/* Pure timeout, fd still open: reset pos only. */
+				se_dev_ctx_shared_mem_cleanup(dev_ctx);
+			}
+		} else {
+			/*
+			 * have_snap=false only on teardown (FW never responded,
+			 * or teardown beat the late IRQ, which going_away then
+			 * drops). priv_dev_ctx is handled by probe_cleanup's
+			 * single cleanup_se_shared_mem(reclaim=true) after
+			 * cancel_work_sync, so nothing to do here. For a
+			 * userspace dev_ctx (cleanup_done already true), use
+			 * reclaim=false: the pos gate leaks the buffer if FW may
+			 * still be writing, else frees it. reclaim=true would
+			 * also be safe here since the region is no-map.
+			 */
+			if (dev_ctx != priv->priv_dev_ctx && dev_ctx->cleanup_done)
+				cleanup_se_shared_mem(dev_ctx, false);
+		}
+	}
+
+	/*
+	 * Recovery flow is done: release the exclusive reservation, then clear
+	 * the breaker. se_release_msg_if() drops msg_excl_owner; the fw_busy
+	 * clear below reopens the interface. Ordering is safe either way: while
+	 * fw_busy is still 1 a third party is rejected regardless of owner, and
+	 * once fw_busy is 0 the owner is no longer consulted.
+	 */
+	se_release_msg_if(priv);
+
+	spin_lock_irqsave(&fbusy_info->fw_busy_lock, flags);
+	atomic_set(&fbusy_info->fw_busy, 0);
+	spin_unlock_irqrestore(&fbusy_info->fw_busy_lock, flags);
+	kref_put(&dev_ctx->refcount, se_if_dev_ctx_release);
+}
+
+void unset_dev_ctx_as_command_receiver(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_api_msg *old_rx_msg = NULL;
+	struct se_clbk_handle *se_clbk_hdl;
+	unsigned long flags;
+
+	lockdep_assert_held(&priv->modify_lock);
+
+	se_clbk_hdl = &priv->cmd_receiver_clbk_hdl;
+
+	if (se_clbk_hdl->dev_ctx == dev_ctx) {
+		spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+		old_rx_msg = se_clbk_hdl->rx_msg;
+		se_clbk_hdl->dev_ctx = NULL;
+		se_clbk_hdl->rx_msg = NULL;
+		se_clbk_hdl->rx_msg_sz = 0;
+		spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+
+		kfree(old_rx_msg);
+		complete_all(&se_clbk_hdl->done);
+	}
+}
+
+/*
+ * check_cmd_rcvr_status() - check whether dev_ctx can become the command
+ * receiver or is already become the command receiver.
+ *
+ * Returns:
+ *   0        - dev_ctx is already the registered receiver
+ *   -EBUSY   - another context is already the receiver
+ *   -EINVAL  - dev_ctx has no storage handle
+ *   -ENXIO   - ready to proceed: no receiver set, strg_hdl present
+ *
+ * Caller must hold priv->modify_lock.
+ */
+static int check_cmd_rcvr_status(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_clbk_handle *se_clbk_hdl = &priv->cmd_receiver_clbk_hdl;
+
+	lockdep_assert_held(&priv->modify_lock);
+
+	if (se_clbk_hdl->dev_ctx == dev_ctx)
+		return 0;
+
+	if (se_clbk_hdl->dev_ctx)
+		return -EBUSY;
+
+	if (!dev_ctx->strg_hdl)
+		return -EINVAL;
+
+	/* Reaching here means, with a valid storage handle and command-receiver as NULL,
+	 * either the registration process is to be done or failed.
+	 */
+	return -ENXIO;
+}
+
+int set_dev_ctx_as_command_receiver(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_clbk_handle *se_clbk_hdl = &priv->cmd_receiver_clbk_hdl;
+	struct se_api_msg *new_rx_msg = NULL;
+	unsigned long flags;
+	int ret;
+
+	guard(mutex)(&priv->modify_lock);
+
+	/*
+	 * All state checks happen inside modify_lock so the result cannot
+	 * go stale between the check and the arming below.
+	 */
+	ret = check_cmd_rcvr_status(dev_ctx);
+	if (ret != -ENXIO)
+		return ret;
+
+	if (!se_clbk_hdl->rx_msg) {
+		new_rx_msg = kzalloc(MAX_NVM_MSG_LEN, GFP_KERNEL);
+		if (!new_rx_msg)
+			return -ENOMEM;
+	}
+	spin_lock_irqsave(&se_clbk_hdl->clbk_rx_lock, flags);
+	if (new_rx_msg)
+		se_clbk_hdl->rx_msg = new_rx_msg;
+	reinit_completion(&se_clbk_hdl->done);
+	se_clbk_hdl->rx_msg_sz = MAX_NVM_MSG_LEN;
+	se_clbk_hdl->dev_ctx = dev_ctx;
+	dev_ctx->rcv_msg_timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
+	spin_unlock_irqrestore(&se_clbk_hdl->clbk_rx_lock, flags);
+
+	return 0;
+}
+
+static void dlink_dev_ctx(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	unset_dev_ctx_as_command_receiver(dev_ctx);
+
+	if (!list_empty(&dev_ctx->link)) {
+		list_del_init(&dev_ctx->link);
+		priv->active_devctx_count--;
+	}
+}
+
+bool se_is_fw_busy_ctx(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct fw_busy_info *fbusy_info = &priv->fw_busy_info;
+	unsigned long flags;
+	bool match;
+
+	spin_lock_irqsave(&fbusy_info->fw_busy_lock, flags);
+	match = fbusy_info->fw_busy_dev_ctx == dev_ctx;
+	spin_unlock_irqrestore(&fbusy_info->fw_busy_lock, flags);
+
+	return match;
+}
+
+static void cleanup_dev_ctx(struct se_if_device_ctx *dev_ctx, bool is_fclose)
+{
+	struct fw_busy_info *fbusy_info = &dev_ctx->priv->fw_busy_info;
+	bool already_done;
+
+	scoped_guard(mutex, &dev_ctx->fops_lock) {
+		already_done = dev_ctx->cleanup_done;
+		if (!already_done) {
+			/*
+			 * Ask FW to drop this context's session and storage so
+			 * the kernel and FW stay in sync. Done here, under this
+			 * context's fops_lock only (not the global modify_lock),
+			 * because both close requests block on a firmware
+			 * round-trip; issuing them while modify_lock was held
+			 * would stall every other context for the FW timeout.
+			 *
+			 * Skip the round-trips once the FW path is marked busy.
+			 * fw_busy is armed when a synchronous transaction times
+			 * out; while it is set ele_msg_send_rcv() rejects further
+			 * commands with -EBUSY without waiting. It is only cleared
+			 * by se_clear_fw_busy(), which during unbind runs once
+			 * after this loop (or earlier from fw_busy_work only if a
+			 * genuine late FW response arrives). On a hung FW no late
+			 * response comes, so the breaker stays set for the rest of
+			 * the loop and the remaining closes would just return
+			 * -EBUSY and log spurious "failed to close" errors. Skip
+			 * them and emit a single warning instead.
+			 */
+			if (atomic_read(&fbusy_info->fw_busy)) {
+				if (dev_ctx->strg_hdl || dev_ctx->sess_hdl)
+					dev_warn(dev_ctx->priv->dev,
+						 "%s: skipping session/storage close, FW is busy\n",
+						 dev_ctx->devname);
+			} else {
+				/*
+				 * Choose which dev_ctx sends the close messages.
+				 * fclose: use the caller's own dev_ctx so a race with
+				 * unbind is rejected with -ENODEV instead of hitting a
+				 * freed tx_chan. Teardown: use priv_dev_ctx; going_away
+				 * is set but the reservation (msg_excl_owner == current)
+				 * lets these closes through while tx_chan is still live.
+				 */
+				struct se_if_device_ctx *tx_ctx = is_fclose ? dev_ctx :
+							dev_ctx->priv->priv_dev_ctx;
+
+				if (dev_ctx->strg_hdl &&
+				    se_close_storage(tx_ctx, dev_ctx->strg_hdl))
+					dev_err(dev_ctx->priv->dev, "failed to close storage.\n");
+				if (dev_ctx->sess_hdl &&
+				    se_close_session(tx_ctx, dev_ctx->sess_hdl))
+					dev_err(dev_ctx->priv->dev, "failed to close session.\n");
+			}
+			/*
+			 * fw_busy is caused by one timed-out synchronous transaction.
+			 * Only that transaction's dev_ctx may still have coherent
+			 * memory referenced by FW. Do not skip cleanup for unrelated
+			 * contexts while fw_busy is set.
+			 */
+			if (se_is_fw_busy_ctx(dev_ctx))
+				dev_warn(dev_ctx->priv->dev,
+					 "%s: deferring shared memory cleanup while FW is busy\n",
+					 dev_ctx->devname);
+			else
+				cleanup_se_shared_mem(dev_ctx, true);
+
+			kfree(dev_ctx->devname);
+			dev_ctx->devname = NULL;
+			dev_ctx->cleanup_done = true;
+		}
+	}
+
+	if (is_fclose)
+		kref_put(&dev_ctx->refcount, se_if_dev_ctx_release);
+}
+
+static void dlink_n_cleanup_dev_ctx(struct se_if_device_ctx *dev_ctx, bool is_fclose)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	if (is_fclose) {
+		scoped_guard(mutex, &priv->modify_lock)
+			dlink_dev_ctx(dev_ctx);
+	}
+
+	cleanup_dev_ctx(dev_ctx, is_fclose);
+}
+
+static int init_device_context(struct se_if_priv *priv, int ch_id,
+			       struct se_if_device_ctx **new_dev_ctx)
+{
+	struct se_if_device_ctx *dev_ctx;
+	int ret = 0;
+
+	dev_ctx = kzalloc_obj(*dev_ctx);
+
+	if (!dev_ctx)
+		return -ENOMEM;
+
+	dev_ctx->devname = kasprintf(GFP_KERNEL, "%s0_ch%d",
+				     get_se_if_name(priv->if_defs->se_if_type),
+				     ch_id);
+	if (!dev_ctx->devname) {
+		kfree(dev_ctx);
+		return -ENOMEM;
+	}
+
+	mutex_init(&dev_ctx->fops_lock);
+	kref_init(&dev_ctx->refcount);
+	dev_ctx->priv = priv;
+	dev_ctx->cleanup_done = false;
+	INIT_LIST_HEAD(&dev_ctx->link);
+	set_se_rcv_msg_timeout(dev_ctx, SE_RCV_MSG_LONG_TIMEOUT_MS);
+	*new_dev_ctx = dev_ctx;
+
+	ret = init_se_shared_mem(dev_ctx);
+	if (ret < 0) {
+		kfree(dev_ctx->devname);
+		kfree(dev_ctx);
+		*new_dev_ctx = NULL;
+
+		return ret;
+	}
+
+	/* Take a reference to priv for this device context */
+	kref_get(&priv->refcount);
+
+	scoped_guard(mutex, &priv->modify_lock) {
+		list_add_tail(&dev_ctx->link, &priv->dev_ctx_list);
+		priv->active_devctx_count++;
+	}
+
+	return ret;
+}
+
+static int se_ioctl_cmd_snd_rcv_cleanup(struct se_if_device_ctx *dev_ctx, void __user *uarg,
+					struct se_ioctl_cmd_snd_rcv_rsp_info *cmd_snd_rcv_rsp_info)
+{
+	/* shared memory is allocated before this IOCTL */
+	se_dev_ctx_shared_mem_cleanup(dev_ctx);
+
+	if (cmd_snd_rcv_rsp_info->rx_buf_sz &&
+	    copy_to_user(uarg, cmd_snd_rcv_rsp_info, sizeof(*cmd_snd_rcv_rsp_info))) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed to copy cmd_snd_rcv_rsp_info to user.\n",
+			dev_ctx->devname);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static int se_ioctl_cmd_snd_rcv_rsp_handler(struct se_if_device_ctx *dev_ctx,
+					    void __user *uarg)
+{
+	struct se_ioctl_cmd_snd_rcv_rsp_info cmd_snd_rcv_rsp_info = {0};
+	struct se_if_priv *priv = dev_ctx->priv;
+	int rsp_status_err = 0;
+	int act_rx_msg_sz = 0;
+	int cleanup_err = 0;
+	int err = 0;
+
+	if (copy_from_user(&cmd_snd_rcv_rsp_info, uarg,
+			   sizeof(cmd_snd_rcv_rsp_info))) {
+		dev_err(priv->dev,
+			"%s: Failed to copy cmd_snd_rcv_rsp_info from user.",
+			dev_ctx->devname);
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -EFAULT;
+	}
+
+	if (cmd_snd_rcv_rsp_info.tx_buf_sz < SE_MU_HDR_SZ ||
+	    cmd_snd_rcv_rsp_info.tx_buf_sz > MAX_ALLOWED_TX_MSG_SZ) {
+		dev_err(priv->dev, "%s: User buffer too small/large(%d < %d)\n",
+			dev_ctx->devname, cmd_snd_rcv_rsp_info.tx_buf_sz,
+			cmd_snd_rcv_rsp_info.tx_buf_sz < SE_MU_HDR_SZ ? SE_MU_HDR_SZ :
+								MAX_ALLOWED_TX_MSG_SZ);
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -ENOSPC;
+	}
+
+	struct se_api_msg *tx_msg __free(kfree) =
+		memdup_user(u64_to_user_ptr(cmd_snd_rcv_rsp_info.tx_buf),
+			    cmd_snd_rcv_rsp_info.tx_buf_sz);
+	if (IS_ERR(tx_msg)) {
+		err = PTR_ERR(tx_msg);
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return err;
+	}
+
+	err = se_chk_tx_cmd_msg_hdr(dev_ctx, &tx_msg->header,
+				    cmd_snd_rcv_rsp_info.tx_buf_sz,
+				    cmd_snd_rcv_rsp_info.rx_buf_sz);
+	if (err) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return err;
+	}
+
+	if (cmd_snd_rcv_rsp_info.rx_buf_sz < SE_MU_HDR_SZ ||
+	    cmd_snd_rcv_rsp_info.rx_buf_sz > MAX_ALLOWED_RX_MSG_SZ) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -EINVAL;
+	}
+
+	if (tx_msg->header.tag != priv->if_defs->cmd_tag) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -EINVAL;
+	}
+
+	if (tx_msg->header.ver == priv->if_defs->fw_api_ver &&
+	    get_load_fw_instance(priv)->is_fw_tobe_loaded) {
+		err = se_load_firmware(priv);
+		if (err) {
+			dev_err(priv->dev, "Could not send msg as FW is not loaded.\n");
+			se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+			return -EPERM;
+		}
+	}
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(cmd_snd_rcv_rsp_info.rx_buf_sz, GFP_KERNEL);
+	if (!rx_msg) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -ENOMEM;
+	}
+
+	err = ele_msg_send_rcv(dev_ctx, tx_msg, cmd_snd_rcv_rsp_info.tx_buf_sz,
+			       rx_msg, cmd_snd_rcv_rsp_info.rx_buf_sz, &act_rx_msg_sz);
+	if (err < 0) {
+		/*
+		 * -ERESTARTSYS here means the wait was interrupted by a signal
+		 * after the command had already been handed to - and executed
+		 * by - the firmware, with its response delivered into rx_msg
+		 * (ele_msg_send_rcv() converts only a positive, i.e. successfully
+		 * received, result to -ERESTARTSYS). If that response carried a
+		 * freshly allocated session/storage handle, record it now via
+		 * fw_api_specific_ops(): the handle is already live in firmware,
+		 * so leaving it untracked would stop cleanup_dev_ctx() from ever
+		 * closing it and leak the firmware resource. Validate the
+		 * delivered response first, using its own declared length bounded
+		 * by the caller's buffer, so a truncated or malformed reply is
+		 * not acted upon.
+		 */
+		if (err == -ERESTARTSYS) {
+			u32 rsp_sz = rx_msg->header.size << 2;
+
+			if (rsp_sz && rsp_sz <= cmd_snd_rcv_rsp_info.rx_buf_sz &&
+			    !se_val_rsp_hdr_n_status(dev_ctx, rx_msg,
+						     tx_msg->header.command, act_rx_msg_sz,
+						     tx_msg->header.ver)) {
+				se_dev_ctx_cpy_out_data(dev_ctx);
+				fw_api_specific_ops(dev_ctx, rx_msg, true);
+			}
+			/*
+			 * NOTE: se_dev_ctx_cpy_out_data() above has already
+			 * copied the firmware response payload to userspace before
+			 * this point. Returning -EINTR here is intentional, not
+			 * -ERESTARTSYS: the VFS would transparently restart the
+			 * ioctl on -ERESTARTSYS, re-issuing the command with
+			 * already-zeroed shared input buffers. -EINTR prevents
+			 * auto-restart and lets userspace enter its signal handler
+			 * to decide whether to reissue the command.
+			 * See Documentation/driver-api/firmware/other_interfaces.rst,
+			 * section "Signal handling after a completed hardware
+			 * operation".
+			 */
+			err = -EINTR;
+		}
+
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+
+		return err;
+	}
+
+	/*
+	 * ele_msg_send_rcv() returns a positive received-message size on
+	 * success. Returning that raw size as the ioctl result would make a
+	 * successful transaction look like a positive (non-zero) return value
+	 * to userspace. Record the actual received size in rx_buf_sz for the
+	 * response copied back to userspace, then normalise err to 0 so the
+	 * ioctl reports plain success; the firmware status is conveyed to
+	 * userspace inside the response buffer itself.
+	 */
+	cmd_snd_rcv_rsp_info.rx_buf_sz = act_rx_msg_sz;
+	err = 0;
+
+	dev_dbg(priv->dev, "%s: %s %s.\n", dev_ctx->devname, __func__,
+		"message received, start transmit to user");
+
+	rsp_status_err =
+		se_val_rsp_hdr_n_status(dev_ctx, rx_msg, tx_msg->header.command,
+					act_rx_msg_sz, tx_msg->header.ver);
+
+	if (!rsp_status_err) {
+		/*
+		 * For msg IDs handled by fw_api_specific_ops(), the exact
+		 * response size was already ensured in ele_uapi_allowed_fw_cmd().
+		 */
+		err = fw_api_specific_ops(dev_ctx, rx_msg, false);
+		if (err) {
+			se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+			return err;
+		}
+
+		err = se_dev_ctx_cpy_out_data(dev_ctx);
+		if (err < 0) {
+			se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+			return err;
+		}
+	}
+
+	/* Copy data from the buffer */
+	print_hex_dump_debug("to user ", DUMP_PREFIX_OFFSET, 4, 4, rx_msg,
+			     cmd_snd_rcv_rsp_info.rx_buf_sz, false);
+
+	if (copy_to_user(u64_to_user_ptr(cmd_snd_rcv_rsp_info.rx_buf), rx_msg,
+			 cmd_snd_rcv_rsp_info.rx_buf_sz)) {
+		dev_err(priv->dev, "%s: Failed to copy to user.\n", dev_ctx->devname);
+		err = -EFAULT;
+	}
+
+	cleanup_err = se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+
+	if (cleanup_err && !err)
+		err = cleanup_err;
+
+	return err;
+}
+
+static int se_ioctl_get_mu_info(struct se_if_device_ctx *dev_ctx,
+				void __user *uarg)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_ioctl_get_if_info if_info;
+	struct se_if_node *if_node;
+	int err = 0;
+
+	if_node = container_of(priv->if_defs, typeof(*if_node), if_defs);
+
+	if_info.se_if_id = 0;
+	if_info.interrupt_idx = 0;
+	if_info.tz = 0;
+	if_info.did = 0;
+	if_info.cmd_tag = priv->if_defs->cmd_tag;
+	if_info.rsp_tag = priv->if_defs->rsp_tag;
+	if_info.success_tag = priv->if_defs->success_tag;
+	if_info.base_api_ver = priv->if_defs->base_api_ver;
+	if_info.fw_api_ver = priv->if_defs->fw_api_ver;
+
+	dev_dbg(priv->dev, "%s: info [se_if_id: %d, irq_idx: %d, tz: 0x%x, did: 0x%x].\n",
+		dev_ctx->devname, if_info.se_if_id, if_info.interrupt_idx, if_info.tz,
+		if_info.did);
+
+	if (copy_to_user(uarg, &if_info, sizeof(if_info))) {
+		dev_err(priv->dev, "%s: Failed to copy mu info to user.\n",
+			dev_ctx->devname);
+		err = -EFAULT;
+	}
+
+	return err;
+}
+
+static void rollback_shared_mem_pos(struct se_if_device_ctx *dev_ctx, u32 length)
+{
+	struct se_shared_mem *shared_mem = NULL;
+
+	shared_mem = &dev_ctx->se_shared_mem_mgmt.non_secure_mem;
+
+	if (WARN_ON_ONCE(length > shared_mem->pos)) {
+		shared_mem->pos = 0;
+		return;
+	}
+
+	shared_mem->pos -= length;
+}
+
+int get_shared_mem_slot(struct se_if_device_ctx *dev_ctx,
+			u32 *length, dma_addr_t *ele_dma_addr, void **ptr)
+{
+	struct se_shared_mem *shared_mem = NULL;
+	bool is_fw_busy_dev_ctx;
+	size_t aligned_len = 0;
+	u32 pos;
+
+	/*
+	 * If this context is the one that caused a firmware timeout the shared
+	 * DMA buffers may still be actively read/written by the firmware.
+	 */
+	is_fw_busy_dev_ctx = se_is_fw_busy_ctx(dev_ctx);
+	if (is_fw_busy_dev_ctx)
+		return -EBUSY;
+
+	aligned_len = round_up((size_t)*length, 8);
+	if (aligned_len < *length) {
+		dev_err(dev_ctx->priv->dev, "%s: Invalid buffer length.\n",
+			dev_ctx->devname);
+		return -EINVAL;
+	}
+
+	/* No specific requirement for this buffer. */
+	shared_mem = &dev_ctx->se_shared_mem_mgmt.non_secure_mem;
+
+	/* Check there is enough space in the shared memory. */
+	dev_dbg(dev_ctx->priv->dev, "%s: req_size = %zd, max_size= %d, curr_pos = %d\n",
+		dev_ctx->devname, aligned_len, shared_mem->size,
+		shared_mem->pos);
+
+	if (shared_mem->size < shared_mem->pos ||
+	    aligned_len > (shared_mem->size - shared_mem->pos)) {
+		dev_err(dev_ctx->priv->dev, "%s: Not enough space in shared memory.\n",
+			dev_ctx->devname);
+		return -ENOMEM;
+	}
+
+	/* Allocate space in shared memory. 8 bytes aligned. */
+	pos = shared_mem->pos;
+	shared_mem->pos += aligned_len;
+	*ele_dma_addr = (u64)shared_mem->dma_addr + pos;
+	*ptr = shared_mem->ptr + pos;
+	*length = aligned_len;
+
+	memset(shared_mem->ptr + pos, 0, aligned_len);
+
+	return 0;
+}
+
+/*
+ * Copy a buffer of data to/from the user and return the address to use in
+ * messages
+ */
+static int se_ioctl_setup_iobuf_handler(struct se_if_device_ctx *dev_ctx,
+					void __user *uarg)
+{
+	struct se_ioctl_setup_iobuf io = {0};
+	struct se_buf_desc *b_desc = NULL;
+	void *dma_buf_ptr = NULL;
+	dma_addr_t ele_dma_addr;
+	u32 aligned_len = 0;
+	int err = 0;
+
+	if (copy_from_user(&io, uarg, sizeof(io))) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed copy iobuf config from user.\n",
+			dev_ctx->devname);
+		return -EFAULT;
+	}
+
+	dev_dbg(dev_ctx->priv->dev, "%s: io [buf: %p(%d) flag: %x].\n", dev_ctx->devname,
+		u64_to_user_ptr(io.user_buf), io.length, io.flags);
+
+	if (io.length == 0 || !io.user_buf) {
+		/*
+		 * Accept NULL pointers since some buffers are optional
+		 * in FW commands. In this case we should return 0 as
+		 * pointer to be embedded into the message.
+		 * Skip all data copy part of code below.
+		 */
+		io.ele_addr = 0;
+		goto copy;
+	}
+
+	aligned_len = io.length;
+	err = get_shared_mem_slot(dev_ctx, &aligned_len, &ele_dma_addr, &dma_buf_ptr);
+	if (err)
+		return err;
+
+	io.ele_addr = ele_dma_addr;
+	if ((io.flags & SE_IO_BUF_FLAGS_IS_INPUT) ||
+	    (io.flags & SE_IO_BUF_FLAGS_IS_IN_OUT)) {
+		/*
+		 * buffer is input:
+		 * copy data from user space to this allocated buffer.
+		 */
+		if (copy_from_user(dma_buf_ptr, u64_to_user_ptr(io.user_buf),
+				   io.length)) {
+			dev_err(dev_ctx->priv->dev,
+				"%s: Failed copy data to shared memory.",
+				dev_ctx->devname);
+			err = -EFAULT;
+			goto rollback;
+		}
+	}
+
+	b_desc = add_b_desc_to_pending_list(dma_buf_ptr, &io, dev_ctx);
+	if (IS_ERR(b_desc)) {
+		err = PTR_ERR(b_desc);
+		dev_err(dev_ctx->priv->dev, "%s: Failed to allocate/link b_desc.\n",
+			dev_ctx->devname);
+		goto rollback;
+	}
+
+copy:
+	/* Provide the EdgeLock Enclave address to user space only if success.*/
+	if (copy_to_user(uarg, &io, sizeof(io))) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed to copy iobuff setup to user.\n",
+			dev_ctx->devname);
+		err = -EFAULT;
+		goto rollback;
+	}
+	return err;
+
+rollback:
+	if (!IS_ERR_OR_NULL(b_desc)) {
+		list_del(&b_desc->link);
+		kfree(b_desc);
+	}
+
+	if (dma_buf_ptr && aligned_len) {
+		memset(dma_buf_ptr, 0, aligned_len);
+		rollback_shared_mem_pos(dev_ctx, aligned_len);
+	}
+
+	return err;
+}
+
+/* IOCTL to provide SoC information */
+static int se_ioctl_get_se_soc_info_handler(struct se_if_device_ctx *dev_ctx,
+					    void __user *uarg)
+{
+	struct se_ioctl_get_soc_info soc_info;
+	int err = -EINVAL;
+
+	soc_info.soc_id = get_se_soc_id(dev_ctx->priv);
+	soc_info.soc_rev = var_se_info.soc_rev;
+
+	err = copy_to_user(uarg, (u8 *)(&soc_info), sizeof(soc_info));
+	if (err) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed to copy soc info to user.\n",
+			dev_ctx->devname);
+		err = -EFAULT;
+	}
+
+	return err;
+}
+
+/*
+ * File operations for user-space
+ */
+
+/* Write a message to the MU. */
+static ssize_t se_if_fops_write(struct file *fp, const char __user *buf,
+				size_t size, loff_t *ppos)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+	struct se_if_priv *priv;
+	int err;
+
+	scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &dev_ctx->fops_lock) {
+		if (dev_ctx->cleanup_done)
+			return -ENODEV;
+
+		priv = dev_ctx->priv;
+
+		dev_dbg(priv->dev, "%s: write from buf (%p)%zu, ppos=%lld.\n", dev_ctx->devname,
+			buf, size, ((ppos) ? *ppos : 0));
+
+		if (dev_ctx != priv->cmd_receiver_clbk_hdl.dev_ctx) {
+			se_dev_ctx_shared_mem_cleanup(dev_ctx);
+			return -EINVAL;
+		}
+
+		if (size < SE_MU_HDR_SZ || size > MAX_ALLOWED_TX_MSG_SZ) {
+			dev_err(priv->dev, "%s: User buffer too small/large(%zu < %d)\n",
+				dev_ctx->devname, size,
+				size < SE_MU_HDR_SZ ? SE_MU_HDR_SZ :
+								MAX_ALLOWED_TX_MSG_SZ);
+			return -ENOSPC;
+		}
+
+		struct se_api_msg *tx_msg __free(kfree) = memdup_user(buf, size);
+		if (IS_ERR(tx_msg))
+			return PTR_ERR(tx_msg);
+
+		err = se_chk_tx_rsp_msg_hdr(dev_ctx, &tx_msg->header, size);
+		if (err)
+			return err;
+
+		print_hex_dump_debug("from user ", DUMP_PREFIX_OFFSET, 4, 4,
+				     tx_msg, size, false);
+
+		err = ele_msg_send(dev_ctx, tx_msg, size);
+
+		return err;
+	}
+}
+
+/*
+ * Read a message from the MU.
+ * Blocking until a message is available.
+ */
+static ssize_t se_if_fops_read(struct file *fp, char __user *buf, size_t size,
+			       loff_t *ppos)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+	u8 rx_msg_snap[MAX_NVM_MSG_LEN] = {};
+	char devname_snap[32] = {};
+	struct se_if_priv *priv;
+	unsigned long flags;
+	size_t copy_len;
+	int err;
+
+	scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &dev_ctx->fops_lock) {
+		priv = dev_ctx->priv;
+
+		if (dev_ctx->cleanup_done)
+			return -ENODEV;
+
+		/*
+		 * Snapshot devname once while fops_lock is held. After the
+		 * scoped guard releases the lock, a concurrent cleanup_dev_ctx()
+		 * could free dev_ctx->devname before the error path below runs.
+		 */
+		strscpy(devname_snap, dev_ctx->devname, sizeof(devname_snap));
+
+		dev_dbg(priv->dev, "%s: read to buf %p(%zu), ppos=%lld.\n", devname_snap,
+			buf, size, ((ppos) ? *ppos : 0));
+
+		mutex_lock(&priv->modify_lock);
+		if (dev_ctx != priv->cmd_receiver_clbk_hdl.dev_ctx) {
+			mutex_unlock(&priv->modify_lock);
+			se_dev_ctx_shared_mem_cleanup(dev_ctx);
+			return -EINVAL;
+		}
+		mutex_unlock(&priv->modify_lock);
+	}
+
+	err = ele_msg_rcv(dev_ctx, &priv->cmd_receiver_clbk_hdl);
+	if (err < 0) {
+		if (err != -ERESTARTSYS)
+			dev_err(priv->dev,
+				"%s: Er[0x%x]: Signal Interrupted. Current act-dev-ctx count: %d.",
+				devname_snap, err, dev_ctx->priv->active_devctx_count);
+		return err;
+	}
+
+	/*
+	 * Reacquire fops_lock before touching any dev_ctx state (pending lists,
+	 * rx_msg) after the blocking wait. fops_lock was dropped before calling
+	 * ele_msg_rcv(). If cleanup_dev_ctx() ran concurrently it could have
+	 * freed the DMA buffers and the pending lists, leading to UAF and list
+	 * corruption. Re-checking cleanup_done under fops_lock prevents that.
+	 */
+	mutex_lock(&dev_ctx->fops_lock);
+
+	if (dev_ctx->cleanup_done) {
+		mutex_unlock(&dev_ctx->fops_lock);
+		return -ENODEV;
+	}
+
+	/*
+	 * Snapshot the whole rx_msg under modify_lock + clbk_rx_lock, not just
+	 * copy_len bytes: fw_api_specific_ops() reads data words (e.g. strg_hdl
+	 * at data[1]) beyond the userspace read size; truncating would record a
+	 * zero handle. Run fw_api_specific_ops() OUTSIDE modify_lock
+	 * (ELE_STORAGE_OPEN_REQ re-takes it, else deadlock).
+	 */
+	scoped_guard(mutex, &priv->modify_lock) {
+		spin_lock_irqsave(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock, flags);
+		if (priv->cmd_receiver_clbk_hdl.dev_ctx != dev_ctx ||
+		    !priv->cmd_receiver_clbk_hdl.rx_msg ||
+		    !priv->cmd_receiver_clbk_hdl.rx_msg_sz) {
+			spin_unlock_irqrestore(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock, flags);
+			mutex_unlock(&dev_ctx->fops_lock);
+			return -ENODEV;
+		}
+		copy_len = min(size, (size_t)priv->cmd_receiver_clbk_hdl.rx_msg_sz);
+		memcpy(rx_msg_snap, priv->cmd_receiver_clbk_hdl.rx_msg,
+		       priv->cmd_receiver_clbk_hdl.rx_msg_sz);
+		priv->cmd_receiver_clbk_hdl.rx_msg_sz = 0;
+		spin_unlock_irqrestore(&priv->cmd_receiver_clbk_hdl.clbk_rx_lock, flags);
+
+		/* We may need to copy the output data to user before
+		 * delivering the completion message.
+		 */
+		err = se_dev_ctx_cpy_out_data(dev_ctx);
+		if (err < 0) {
+			se_dev_ctx_shared_mem_cleanup(dev_ctx);
+			mutex_unlock(&dev_ctx->fops_lock);
+			return err;
+		}
+	}
+
+	/* fw_api_specific_ops() runs outside modify_lock; see comment above. */
+	print_hex_dump_debug("to user ", DUMP_PREFIX_OFFSET, 4, 4,
+			     rx_msg_snap, copy_len, false);
+
+	cmd_receiver_specific_ops(dev_ctx, (struct se_api_msg *)rx_msg_snap);
+	err = copy_len;
+	if (copy_to_user(buf, rx_msg_snap, copy_len))
+		err = -EFAULT;
+
+	se_dev_ctx_shared_mem_cleanup(dev_ctx);
+	mutex_unlock(&dev_ctx->fops_lock);
+
+	return err;
+}
+
+/* Open a character device. */
+static int se_if_fops_open(struct inode *nd, struct file *fp)
+{
+	struct miscdevice *miscdev = fp->private_data;
+	struct se_if_open_gate *gate;
+	struct se_if_device_ctx *misc_dev_ctx;
+	struct se_if_device_ctx *dev_ctx;
+	struct se_if_priv *priv;
+	int err = 0;
+
+	gate = container_of(miscdev, struct se_if_open_gate, miscdev);
+
+	if (!se_if_open_gate_get(gate))
+		return -ENODEV;
+
+	if (mutex_lock_interruptible(&gate->lock)) {
+		se_if_open_gate_put(gate);
+		return -ERESTARTSYS;
+	}
+
+	if (gate->dying || !gate->priv ||
+	    !kref_get_unless_zero(&gate->priv->refcount)) {
+		mutex_unlock(&gate->lock);
+		se_if_open_gate_put(gate);
+		return -ENODEV;
+	}
+
+	priv = gate->priv;
+	mutex_unlock(&gate->lock);
+
+	misc_dev_ctx = priv->priv_dev_ctx;
+
+	if (mutex_lock_interruptible(&misc_dev_ctx->fops_lock)) {
+		err = -ERESTARTSYS;
+		goto out_put_priv;
+	}
+
+	if (misc_dev_ctx->cleanup_done) {
+		err = -ENODEV;
+		goto out_unlock_misc;
+	}
+
+	priv->dev_ctx_mono_count++;
+	err = init_device_context(priv, priv->dev_ctx_mono_count, &dev_ctx);
+	if (err) {
+		dev_err(priv->dev, "Failed to create dev-ctx.\n");
+		goto out_unlock_misc;
+	}
+
+	fp->private_data = dev_ctx;
+
+out_unlock_misc:
+	mutex_unlock(&misc_dev_ctx->fops_lock);
+out_put_priv:
+	kref_put(&priv->refcount, se_if_priv_release);
+	se_if_open_gate_put(gate);
+	return err;
+}
+
+/* Close a character device. */
+static int se_if_fops_close(struct inode *nd, struct file *fp)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+
+	dlink_n_cleanup_dev_ctx(dev_ctx, true);
+
+	return 0;
+}
+
+/* IOCTL entry point of a character device */
+static long se_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+	struct se_if_priv *priv;
+	void __user *uarg = (void __user *)arg;
+	long err;
+
+	/* Prevent race during change of device context */
+	scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &dev_ctx->fops_lock) {
+		if (dev_ctx->cleanup_done)
+			return -ENODEV;
+
+		priv = dev_ctx->priv;
+
+		switch (cmd) {
+		case SE_IOCTL_CHECK_CMD_RCV_REG_STATUS: {
+			guard(mutex)(&priv->modify_lock);
+			err = check_cmd_rcvr_status(dev_ctx);
+		break;
+		}
+		case SE_IOCTL_GET_MU_INFO:
+			err = se_ioctl_get_mu_info(dev_ctx, uarg);
+			break;
+		case SE_IOCTL_SETUP_IOBUF:
+			err = se_ioctl_setup_iobuf_handler(dev_ctx, uarg);
+			break;
+		case SE_IOCTL_GET_SOC_INFO:
+			err = se_ioctl_get_se_soc_info_handler(dev_ctx, uarg);
+			break;
+		case SE_IOCTL_CMD_SEND_RCV_RSP:
+			err = se_ioctl_cmd_snd_rcv_rsp_handler(dev_ctx, uarg);
+			break;
+		default:
+			err = -ENOTTY;
+			dev_dbg(priv->dev, "%s: IOCTL %.8x not supported.\n",
+				dev_ctx->devname, cmd);
+		}
+	}
+
+	return err;
+}
+
+/* Char driver setup */
+static const struct file_operations se_if_fops = {
+	.open		= se_if_fops_open,
+	.owner		= THIS_MODULE,
+	.release	= se_if_fops_close,
+	.unlocked_ioctl = se_ioctl,
+	.compat_ioctl   = compat_ptr_ioctl,
+	.read		= se_if_fops_read,
+	.write		= se_if_fops_write,
+};
+
+int se_get_mem_pool_buf(struct se_if_device_ctx *dev_ctx, void **buf,
+			dma_addr_t *daddr, u32 len)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_buf_desc *b_desc = NULL;
+
+	lockdep_assert_held(&dev_ctx->fops_lock);
+
+	if (se_is_fw_busy_ctx(dev_ctx))
+		return -EBUSY;
+
+	b_desc = kzalloc_obj(*b_desc);
+	if (!b_desc)
+		return -ENOMEM;
+
+	/*
+	 * gen_pool is internally thread-safe, so contexts may allocate
+	 * concurrently. The buffer is tracked on this context's own
+	 * mem_pool_buf_list and released on its cleanup path.
+	 */
+	*buf = gen_pool_dma_alloc(priv->mem_pool, len, daddr);
+	if (!*buf) {
+		dev_err(priv->dev, "Failed to alloc from gen_pool.\n");
+		kfree(b_desc);
+		return -ENOMEM;
+	}
+
+	/* gen_pool_dma_alloc() does not zero the buffer. */
+	memset(*buf, 0, len);
+	b_desc->shared_buf_ptr = *buf;
+	b_desc->size = len;
+
+	list_add_tail(&b_desc->link, &se_shared_mem_mgmt->mem_pool_buf_list);
+
+	return 0;
+}
+
+void se_cleanup_mem_pool_buf(struct se_if_device_ctx *dev_ctx, bool reclaim)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_buf_desc *b_desc, *temp;
+
+	/*
+	 * Free only the buffers this context allocated. A context that never
+	 * used the pool has an empty list, so this is a no-op for it.
+	 *
+	 * Unlike the coherent staging buffer, the pool path needs no
+	 * "nothing staged" (pos) gate on the reclaim=false leg. Pool buffers
+	 * are ephemeral, per-transaction allocations: se_get_mem_pool_buf()
+	 * refuses to allocate once the context is fw_busy, ele_msg_send_rcv()
+	 * refuses to start a new command while fw_busy, and the success path
+	 * frees the whole list via se_cleanup_mem_pool_buf(reclaim=true)
+	 * before returning. se_if_cmd_lock serialises synchronous commands, so
+	 * at most one transaction is outstanding. The only way to reach here
+	 * with reclaim=false and a non-empty list is the single fw_busy
+	 * context still owning the buffer(s) from the one timed-out
+	 * transaction. Those buffers are exactly the in-flight ones the
+	 * enclave may still be DMA-ing into, so leaving them on the list (no
+	 * gen_pool_free) deliberately leaks them to avoid a DMA-after-free -
+	 * there are no already-consumed pool buffers to reclaim on this leg.
+	 */
+	list_for_each_entry_safe(b_desc, temp, &se_shared_mem_mgmt->mem_pool_buf_list, link) {
+		if (reclaim)
+			gen_pool_free(priv->mem_pool,
+				      (unsigned long)b_desc->shared_buf_ptr,
+				      b_desc->size);
+		list_del(&b_desc->link);
+		kfree(b_desc);
+	}
+}
+
+static void se_fw_busy_work(struct work_struct *work)
+{
+	struct fw_busy_info *fbusy_info =
+		container_of(work, struct fw_busy_info, fw_busy_work);
+	struct se_if_priv *priv =
+		container_of(fbusy_info, struct se_if_priv, fw_busy_info);
+
+	se_clear_fw_busy(priv);
+}
+
+static int se_suspend(struct device *dev)
+{
+	struct se_if_priv *priv = dev_get_drvdata(dev);
+	struct se_fw_load_info *load_fw;
+	unsigned int noio_flag;
+	int ret = 0;
+
+	load_fw = get_load_fw_instance(priv);
+
+	if (load_fw->imem_mgmt) {
+		/*
+		 * Set PF_MEMALLOC_NOIO for the duration of the suspend
+		 * callbacks. This covers all allocations in the call chain
+		 * (ele_get_info, se_service_swap, se_get_mem_pool_buf) without
+		 * requiring each site to pass GFP_NOIO explicitly. Without this,
+		 * GFP_KERNEL allocations in those paths could trigger direct
+		 * reclaim and attempt I/O to a storage device that is already
+		 * suspended, causing a deadlock.
+		 */
+		noio_flag = memalloc_noio_save();
+		ret = se_save_imem_state(priv, &load_fw->imem);
+		memalloc_noio_restore(noio_flag);
+		if (ret)
+			dev_err(dev, "Failure saving IMEM state[0x%x]\n", ret);
+	}
+
+	return ret;
+}
+
+static int se_resume(struct device *dev)
+{
+	struct se_if_priv *priv = dev_get_drvdata(dev);
+	struct se_fw_load_info *load_fw;
+	unsigned int noio_flag;
+	int ret = 0;
+
+	load_fw = get_load_fw_instance(priv);
+
+	if (load_fw->imem_mgmt) {
+		noio_flag = memalloc_noio_save();
+		ret = se_restore_imem_state(priv, &load_fw->imem);
+		memalloc_noio_restore(noio_flag);
+		if (ret)
+			dev_err(dev, "Failure restoring IMEM state[0x%x]\n", ret);
+	}
+
+	return ret;
+}
+
+DEFINE_SIMPLE_DEV_PM_OPS(se_pm, se_suspend, se_resume);
+
+static struct platform_driver se_driver = {
+	.driver = {
+		.name = "fsl-se",
+		.of_match_table = se_match,
+		.pm = pm_sleep_ptr(&se_pm),
+	},
+	.probe = se_if_probe,
+};
+
+static int __init se_init(void)
+{
+	return platform_driver_register(&se_driver);
+}
+module_init(se_init);
+
+static void __exit se_exit(void)
+{
+	platform_driver_unregister(&se_driver);
+
+	/*
+	 * The soc_device is a module-scoped singleton that outlives any single
+	 * MU interface bind/unbind. Release it here, once, after every interface
+	 * has been unbound, so its lifetime is tied to the module rather than to
+	 * the first-probed interface.
+	 */
+	se_soc_device_unregister(&var_se_info.soc_dev_regn);
+}
+module_exit(se_exit);
+
+MODULE_AUTHOR("Pankaj Gupta <pankaj.gupta@nxp.com>");
+MODULE_DESCRIPTION("iMX Secure Enclave Driver.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/imx/se_ctrl.h b/drivers/firmware/imx/se_ctrl.h
new file mode 100644
index 00000000000000..c05328f3b408ab
--- /dev/null
+++ b/drivers/firmware/imx/se_ctrl.h
@@ -0,0 +1,309 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2026 NXP
+ */
+
+#ifndef SE_CTRL_H
+#define SE_CTRL_H
+
+#include <linux/bitfield.h>
+#include <linux/miscdevice.h>
+#include <linux/mailbox_client.h>
+#include <linux/semaphore.h>
+#include <linux/workqueue.h>
+
+#define MAX_DEVNAME_SZ			64
+#define MAX_FW_LOAD_RETRIES		50
+#define SE_MSG_WORD_SZ			0x4
+
+#define RES_STATUS(x)			FIELD_GET(0x000000ff, x)
+#define MAX_DATA_SIZE_PER_USER		(128 * 1024)
+#define MAX_NVM_MSG_LEN			(256)
+/* Largest firmware response buffer size in bytes; equals ELE_DEBUG_DUMP_RSP_SZ (0x5c). */
+#define MAX_ALLOWED_RX_MSG_SZ		0x5c
+#define MESSAGING_VERSION_6		0x6
+#define MESSAGING_VERSION_7		0x7
+
+struct se_if_open_gate {
+	struct miscdevice miscdev;
+	struct se_if_priv *priv;
+	/* to lock to update the structure */
+	struct mutex lock;
+	struct kref refcount;
+	bool dying;
+	/* set once misc_register() has succeeded (deferred to probe end) */
+	bool registered;
+};
+
+struct se_clbk_handle {
+	struct se_if_device_ctx *dev_ctx;
+	struct completion done;
+	bool signal_rcvd;
+	/*
+	 * Set under clbk_rx_lock once a real response is copied into rx_msg,
+	 * cleared when a new transaction is armed. Lets ele_msg_rcv() tell a
+	 * genuine response from a teardown-forced complete_all() with no data.
+	 */
+	bool rx_delivered;
+	u32 rx_msg_sz;
+
+	/*
+	 * Assignment of the rx_msg buffer to held till the
+	 * received content as part callback function, is copied.
+	 */
+	struct se_api_msg *rx_msg;
+	/*
+	 * Serialise the timeout path in ele_msg_rcv() against
+	 * se_if_rx_callback() so that the callback can never
+	 * memcpy into a buffer that the timeout path has already
+	 * freed.
+	 */
+	spinlock_t clbk_rx_lock;
+};
+
+struct se_imem_buf {
+	u8 *buf;
+	dma_addr_t daddr;
+	u32 size;
+	u32 state;
+};
+
+struct se_buf_desc {
+	u8 *shared_buf_ptr;
+	void __user *usr_buf_ptr;
+	u32 size;
+	struct list_head link;
+};
+
+struct se_shared_mem {
+	dma_addr_t dma_addr;
+	u32 size;
+	u32 pos;
+	u8 *ptr;
+};
+
+struct se_shared_mem_mgmt_info {
+	struct list_head mem_pool_buf_list;
+	struct list_head pending_in;
+	struct list_head pending_out;
+
+	struct se_shared_mem non_secure_mem;
+};
+
+/* Private struct for each char device instance. */
+struct se_if_device_ctx {
+	struct se_if_priv *priv;
+	struct miscdevice *miscdev;
+	const char *devname;
+	u32 sess_hdl;
+	u32 strg_hdl;
+	bool cleanup_done;
+	unsigned long rcv_msg_timeout_jiffies;
+
+	/* process one file operation at a time. */
+	struct mutex fops_lock;
+
+	struct se_shared_mem_mgmt_info se_shared_mem_mgmt;
+	struct list_head link;
+
+	/* Add reference counting */
+	struct kref refcount;
+};
+
+/* Header of the messages exchange with the EdgeLock Enclave */
+struct se_msg_hdr {
+	u8 ver;
+	u8 size;
+	u8 command;
+	u8 tag;
+}  __packed;
+
+#define SE_MU_HDR_SZ		4
+#define SE_MU_HDR_WORD_SZ	1
+
+struct se_api_msg {
+	struct se_msg_hdr header;
+	u32 data[];
+};
+
+struct se_if_defines {
+	const u8 se_if_type;
+	u8 cmd_tag;
+	u8 rsp_tag;
+	u8 success_tag;
+	u8 base_api_ver;
+	u8 fw_api_ver;
+};
+
+struct se_fw_img_name {
+	const char *prim_fw_nm_in_rfs;
+	const char *seco_fw_nm_in_rfs;
+};
+
+struct se_fw_load_info {
+	const struct se_fw_img_name *se_fw_img_nm;
+	bool is_fw_tobe_loaded;
+	bool imem_mgmt;
+	struct se_imem_buf imem;
+	/* to serialize the fw load state */
+	struct mutex load_fw_lock;
+};
+
+struct cmd_rcvr_data_info {
+	/*
+	 * Tracks the last FW export command received by the command receiver
+	 * (ELE_STORAGE_MASTER_EXPORT_REQ or ELE_STORAGE_CHUNK_EXPORT_REQ).
+	 * Set by cmd_receiver_specific_ops() when the FW command arrives via
+	 * read(), cleared at entry. se_cmd_receiver_allowed_rsp() checks it to
+	 * ensure write() can only follow a matching read() for export responses.
+	 * Stored per SE interface (not file-scope static) to prevent a race when
+	 * multiple SE interfaces (e.g. ELE and V2X) run concurrent export flows.
+	 */
+	u8 cmd_rcvr_last_rcvd_cmd_id;
+
+	/*
+	 * Export buffer size communicated by the FW in the preceding
+	 * ELE_STORAGE_MASTER_EXPORT_REQ or ELE_STORAGE_CHUNK_EXPORT_REQ
+	 * command. cmd_receiver_specific_ops() stores it here; se_val_cmd_addrs()
+	 * reads it when size_idx == SE_CMD_RCVR_ADDR_VAR_SIZE to range-check
+	 * the response buffer. Stored per SE interface so concurrent ELE and V2X
+	 * export flows cannot corrupt each other's size.
+	 */
+	u32 cmd_rcvr_var_size;
+};
+
+struct fw_busy_info {
+	/*
+	 * fw_busy acts as a circuit breaker: set when a synchronous
+	 * transaction times out, cleared when the late FW response
+	 * arrives and se_clear_fw_busy() has reclaimed the parked
+	 * dev_ctx.
+	 */
+	atomic_t fw_busy;
+
+	/*
+	 * Serialise fw_busy, fw_busy_dev_ctx state updates between the
+	 * timeout path, late-response callback/work, and teardown.
+	 */
+	spinlock_t fw_busy_lock;
+
+	/*
+	 * dev_ctx whose synchronous transaction timed out; parked here
+	 * so a late FW response can still be routed to it by
+	 * se_clear_fw_busy().
+	 */
+	struct se_if_device_ctx *fw_busy_dev_ctx;
+
+	/*
+	 * Snapshot of fw_busy_dev_ctx->devname, captured under
+	 * fw_busy_lock at arm time in se_mark_fw_busy(). cleanup_dev_ctx()
+	 * frees dev_ctx->devname (and sets it to NULL) when a userspace fd
+	 * is closed while the breaker is armed, so the parked dev_ctx may
+	 * carry a NULL devname by the time se_clear_fw_busy() runs on a late
+	 * FW response. Use this stable copy for diagnostics instead. Sized to
+	 * match the "%s0_ch%d" devname and the char devname_snap[32] buffers
+	 * used elsewhere in this driver.
+	 */
+	char devname[MAX_DEVNAME_SZ];
+
+	/* work item scheduled by se_if_rx_callback() on late response. */
+	struct work_struct fw_busy_work;
+
+	/*
+	 * Snapshot of the orphaned firmware response that triggered
+	 * fw_busy_work. Written once (under clbk_rx_lock, before
+	 * schedule_work()) and read once (in se_clear_fw_busy(), under
+	 * clbk_rx_lock). Sized to MAX_ALLOWED_RX_MSG_SZ (= ELE_DEBUG_DUMP_RSP_SZ,
+	 * the largest firmware response) to accommodate any FW reply.
+	 */
+	u8 orphan_fw_rx_msg[MAX_ALLOWED_RX_MSG_SZ];
+};
+
+struct msg_excl_flow_info {
+	/*
+	 * Optional exclusive reservation of the SE messaging interface for a
+	 * single flow. A flow that needs ele_msg_send_rcv() reserved to itself
+	 * (e.g. the fw_busy recovery in se_clear_fw_busy()) publishes its own
+	 * task here via se_reserve_msg_if() from OUTSIDE ele_msg_send_rcv(), and
+	 * releases it with se_release_msg_if() when done. While non-NULL,
+	 * ele_msg_send_rcv() lets only this owning task through and rejects every
+	 * other caller with -EBUSY; while NULL the interface is open to general
+	 * se_if_cmd_lock-based message exchange. Written under msg_excl_lock,
+	 * read locklessly in ele_msg_send_rcv() via READ_ONCE and only ever
+	 * compared against current, so a stale read is harmless (a non-owner can
+	 * never match).
+	 */
+	struct task_struct *msg_excl_owner;
+	/* Serialise updates to msg_excl_owner. */
+	spinlock_t msg_excl_lock;
+	/*
+	 * Serialises competing reservation flows. se_reserve_msg_if() holds
+	 * this mutex for the whole duration of a reservation, so a second flow
+	 * that wants exclusive ownership of the messaging interface sleeps here
+	 * until the current owner calls se_release_msg_if(). Held only from
+	 * process/workqueue context.
+	 */
+	struct mutex msg_excl_flow_lock;
+};
+
+struct se_if_priv {
+	struct device *dev;
+
+	struct se_clbk_handle cmd_receiver_clbk_hdl;
+	/*
+	 * Update to the waiting_rsp_dev, to be protected
+	 * under se_if_cmd_lock.
+	 */
+	struct se_clbk_handle waiting_rsp_clbk_hdl;
+
+	/*
+	 * prevent new command to be sent on the se interface while previous
+	 * command is still processing. (response is awaited)
+	 */
+	struct mutex se_if_cmd_lock;
+
+	struct msg_excl_flow_info msg_excl_flow;
+	struct mbox_client se_mb_cl;
+	struct mbox_chan *tx_chan, *rx_chan;
+
+	struct gen_pool *mem_pool;
+	const struct se_if_defines *if_defs;
+	struct se_fw_load_info load_fw;
+
+	/*
+	 * Set once teardown begins. New synchronous transactions are rejected
+	 * and a teardown-forced completion is not mistaken for a real firmware
+	 * response.
+	 */
+	atomic_t going_away;
+	struct fw_busy_info fw_busy_info;
+
+	struct se_if_device_ctx *priv_dev_ctx;
+	struct list_head dev_ctx_list;
+
+	/* prevent modifying priv member variable in parallel. */
+	struct mutex modify_lock;
+	u32 active_devctx_count;
+	u32 dev_ctx_mono_count;
+
+	/* Add reference counting */
+	struct kref refcount;
+
+	/* stable gate used by .open() */
+	struct se_if_open_gate *open_gate;
+	struct cmd_rcvr_data_info crcvr_info;
+};
+
+char *get_se_if_name(u8 se_if_id);
+void unset_dev_ctx_as_command_receiver(struct se_if_device_ctx *dev_ctx);
+int set_dev_ctx_as_command_receiver(struct se_if_device_ctx *dev_ctx);
+bool se_is_fw_busy_ctx(struct se_if_device_ctx *dev_ctx);
+void se_dev_ctx_shared_mem_cleanup(struct se_if_device_ctx *dev_ctx);
+int get_shared_mem_slot(struct se_if_device_ctx *dev_ctx,
+			u32 *length, dma_addr_t *ele_dma_addr, void **ptr);
+int se_get_mem_pool_buf(struct se_if_device_ctx *dev_ctx, void **buf,
+			dma_addr_t *daddr, u32 len);
+void se_cleanup_mem_pool_buf(struct se_if_device_ctx *dev_ctx, bool reclaim);
+int se_reserve_msg_if(struct se_if_priv *priv);
+void se_release_msg_if(struct se_if_priv *priv);
+#endif
diff --git a/drivers/firmware/microchip/mpfs-auto-update.c b/drivers/firmware/microchip/mpfs-auto-update.c
index 1211fd8d046330..b97d34bd2354ba 100644
--- a/drivers/firmware/microchip/mpfs-auto-update.c
+++ b/drivers/firmware/microchip/mpfs-auto-update.c
@@ -156,7 +156,7 @@ static int mpfs_auto_update_verify_image(struct fw_upload *fw_uploader)
 {
 	struct mpfs_auto_update_priv *priv = fw_uploader->dd_handle;
 	u32 *response_msg __free(kfree) =
-		kzalloc(AUTO_UPDATE_FEATURE_RESP_SIZE * sizeof(*response_msg), GFP_KERNEL);
+		kzalloc_objs(*response_msg, AUTO_UPDATE_FEATURE_RESP_SIZE);
 	struct mpfs_mss_response *response __free(kfree) =
 		kzalloc_obj(struct mpfs_mss_response);
 	struct mpfs_mss_msg *message __free(kfree) =
@@ -358,7 +358,7 @@ static const struct fw_upload_ops mpfs_auto_update_ops = {
 static int mpfs_auto_update_available(struct mpfs_auto_update_priv *priv)
 {
 	u32 *response_msg __free(kfree) =
-		kzalloc(AUTO_UPDATE_FEATURE_RESP_SIZE * sizeof(*response_msg), GFP_KERNEL);
+		kzalloc_objs(*response_msg, AUTO_UPDATE_FEATURE_RESP_SIZE);
 	struct mpfs_mss_response *response __free(kfree) =
 		kzalloc_obj(struct mpfs_mss_response);
 	struct mpfs_mss_msg *message __free(kfree) =
diff --git a/drivers/firmware/qcom/Kconfig b/drivers/firmware/qcom/Kconfig
index c7f8413ab996cf..3ad22f8fc3e5f4 100644
--- a/drivers/firmware/qcom/Kconfig
+++ b/drivers/firmware/qcom/Kconfig
@@ -7,7 +7,7 @@
 menu "Qualcomm firmware drivers"
 
 config QCOM_PAS
-	tristate "Qualcomm generic PAS interface driver"
+	tristate "Qualcomm generic PAS interface driver" if COMPILE_TEST
 	help
 	  Enable the generic Peripheral Authentication Service (PAS) provided
 	  by the firmware. It acts as the common layer with different TZ
@@ -16,6 +16,7 @@ config QCOM_PAS
 
 config QCOM_PAS_TEE
 	tristate "Qualcomm PAS TEE interface driver"
+	depends on ARCH_QCOM || COMPILE_TEST
 	select QCOM_PAS
 	depends on TEE
 	depends on !CPU_BIG_ENDIAN
@@ -25,7 +26,7 @@ config QCOM_PAS_TEE
 	  by the firmware TEE implementation as the backend.
 
 config QCOM_SCM
-	tristate "Qualcomm PAS SCM interface driver"
+	tristate "Qualcomm PAS SCM interface driver" if ARCH_QCOM || COMPILE_TEST
 	select QCOM_PAS
 	select QCOM_TZMEM
 	default y if ARCH_QCOM
@@ -34,33 +35,17 @@ config QCOM_TZMEM
 	tristate
 	select GENERIC_ALLOCATOR
 
-choice
-	prompt "TrustZone interface memory allocator mode"
-	depends on QCOM_TZMEM
-	default QCOM_TZMEM_MODE_GENERIC
-	help
-	  Selects the mode of the memory allocator providing memory buffers of
-	  suitable format for sharing with the TrustZone. If in doubt, select
-	  'Generic'.
-
-config QCOM_TZMEM_MODE_GENERIC
-	bool "Generic"
-	help
-	  Use the generic allocator mode. The memory is page-aligned, non-cachable
-	  and physically contiguous.
-
 config QCOM_TZMEM_MODE_SHMBRIDGE
-	bool "SHM Bridge"
+	bool "TrustZone interface memory allocator: SHM Bridge"
+	depends on QCOM_TZMEM
 	help
-	  Use Qualcomm Shared Memory Bridge. The memory has the same alignment as
-	  in the 'Generic' allocator but is also explicitly marked as an SHM Bridge
-	  buffer.
+	  Use Qualcomm Shared Memory Bridge as memory allocator. The memory has the
+	  same alignment as in the 'Generic' allocator, which is used when this option
+	  is disabled, but is also explicitly marked as an SHM Bridge buffer.
 
 	  With this selected, all buffers passed to the TrustZone must be allocated
 	  using the TZMem allocator or else the TrustZone will refuse to use them.
 
-endchoice
-
 config QCOM_QSEECOM
 	bool "Qualcomm QSEECOM interface driver"
 	depends on QCOM_SCM=y
diff --git a/drivers/firmware/qcom/qcom_scm-legacy.c b/drivers/firmware/qcom/qcom_scm-legacy.c
index 029e6d117cb8d2..fe751ee512da35 100644
--- a/drivers/firmware/qcom/qcom_scm-legacy.c
+++ b/drivers/firmware/qcom/qcom_scm-legacy.c
@@ -219,7 +219,7 @@ int scm_legacy_call(struct device *dev, const struct qcom_scm_desc *desc,
  * @res:  SCM call return values
  *
  * This shall only be used with commands that are guaranteed to be
- * uninterruptable, atomic and SMP safe.
+ * uninterruptible, atomic and SMP safe.
  */
 int scm_legacy_call_atomic(struct device *unused,
 			   const struct qcom_scm_desc *desc,
diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c
index 3eaa4c9ccf3cc7..43e250faea75ab 100644
--- a/drivers/firmware/qcom/qcom_scm.c
+++ b/drivers/firmware/qcom/qcom_scm.c
@@ -576,26 +576,6 @@ static void qcom_scm_set_download_mode(struct qcom_scm *scm, u32 dload_mode)
 		writel_relaxed(minidump_dest, scm->minidump_sram);
 }
 
-struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev,
-							     u32 pas_id,
-							     phys_addr_t mem_phys,
-							     size_t mem_size)
-{
-	struct qcom_pas_context *ctx;
-
-	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
-	if (!ctx)
-		return ERR_PTR(-ENOMEM);
-
-	ctx->dev = dev;
-	ctx->pas_id = pas_id;
-	ctx->mem_phys = mem_phys;
-	ctx->mem_size = mem_size;
-
-	return (struct qcom_scm_pas_context *)ctx;
-}
-EXPORT_SYMBOL_GPL(devm_qcom_scm_pas_context_alloc);
-
 static int __qcom_scm_pas_init_image(struct device *dev, u32 pas_id,
 				     dma_addr_t mdata_phys,
 				     struct qcom_scm_res *res)
@@ -653,9 +633,9 @@ static int qcom_scm_pas_prep_and_init_image(struct device *dev,
 	return ret ? : res.result[0];
 }
 
-static int __qcom_scm_pas_init_image2(struct device *dev, u32 pas_id,
-				      const void *metadata, size_t size,
-				      struct qcom_pas_context *ctx)
+static int qcom_scm_pas_init_image(struct device *dev, u32 pas_id,
+				   const void *metadata, size_t size,
+				   struct qcom_pas_context *ctx)
 {
 	struct qcom_scm_res res;
 	dma_addr_t mdata_phys;
@@ -695,16 +675,8 @@ static int __qcom_scm_pas_init_image2(struct device *dev, u32 pas_id,
 	return ret ? : res.result[0];
 }
 
-int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size,
-			    struct qcom_scm_pas_context *ctx)
-{
-	return __qcom_scm_pas_init_image2(__scm->dev, pas_id, metadata, size,
-					  (struct qcom_pas_context *)ctx);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_init_image);
-
-static void __qcom_scm_pas_metadata_release(struct device *dev,
-					    struct qcom_pas_context *ctx)
+static void qcom_scm_pas_metadata_release(struct device *dev,
+					  struct qcom_pas_context *ctx)
 {
 	if (ctx->use_tzmem)
 		qcom_tzmem_free(ctx->ptr);
@@ -714,15 +686,8 @@ static void __qcom_scm_pas_metadata_release(struct device *dev,
 	ctx->ptr = NULL;
 }
 
-void qcom_scm_pas_metadata_release(struct qcom_scm_pas_context *ctx)
-{
-	__qcom_scm_pas_metadata_release(__scm->dev,
-					(struct qcom_pas_context *)ctx);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_metadata_release);
-
-static int __qcom_scm_pas_mem_setup(struct device *dev, u32 pas_id,
-				    phys_addr_t addr, phys_addr_t size)
+static int qcom_scm_pas_mem_setup(struct device *dev, u32 pas_id,
+				  phys_addr_t addr, phys_addr_t size)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -753,12 +718,6 @@ static int __qcom_scm_pas_mem_setup(struct device *dev, u32 pas_id,
 	return ret ? : res.result[0];
 }
 
-int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size)
-{
-	return __qcom_scm_pas_mem_setup(__scm->dev, pas_id, addr, size);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_mem_setup);
-
 static void *__qcom_scm_pas_get_rsc_table(struct device *dev, u32 pas_id,
 					  void *input_rt_tzm,
 					  size_t input_rt_size,
@@ -812,11 +771,10 @@ static void *__qcom_scm_pas_get_rsc_table(struct device *dev, u32 pas_id,
 	return ret ? ERR_PTR(ret) : output_rt_tzm;
 }
 
-static void *__qcom_scm_pas_get_rsc_table2(struct device *dev,
-					   struct qcom_pas_context *ctx,
-					   void *input_rt,
-					   size_t input_rt_size,
-					   size_t *output_rt_size)
+static void *qcom_scm_pas_get_rsc_table(struct device *dev,
+					struct qcom_pas_context *ctx,
+					void *input_rt, size_t input_rt_size,
+					size_t *output_rt_size)
 {
 	struct resource_table empty_rsc = {};
 	size_t size = SZ_16K;
@@ -854,17 +812,27 @@ static void *__qcom_scm_pas_get_rsc_table2(struct device *dev,
 	output_rt_tzm = __qcom_scm_pas_get_rsc_table(dev, ctx->pas_id,
 						     input_rt_tzm,
 						     input_rt_size, &size);
-	if (PTR_ERR(output_rt_tzm) == -EOVERFLOW)
-		/* Try again with the size requested by the TZ */
+	if (PTR_ERR(output_rt_tzm) == -EOVERFLOW) {
+		/* Try again with a page-aligned size requested by the TZ. */
+		size = PAGE_ALIGN(size);
 		output_rt_tzm = __qcom_scm_pas_get_rsc_table(dev, ctx->pas_id,
 							     input_rt_tzm,
 							     input_rt_size,
 							     &size);
+	}
 	if (IS_ERR(output_rt_tzm)) {
 		ret = PTR_ERR(output_rt_tzm);
 		goto free_input_rt;
 	}
 
+	*output_rt_size = size;
+
+	/* A zero-sized table means that TrustZone has no resource table. */
+	if (!size) {
+		tbl_ptr = NULL;
+		goto free_output_rt;
+	}
+
 	tbl_ptr = kmemdup(output_rt_tzm, size, GFP_KERNEL);
 	if (!tbl_ptr) {
 		qcom_tzmem_free(output_rt_tzm);
@@ -872,7 +840,7 @@ static void *__qcom_scm_pas_get_rsc_table2(struct device *dev,
 		goto free_input_rt;
 	}
 
-	*output_rt_size = size;
+free_output_rt:
 	qcom_tzmem_free(output_rt_tzm);
 
 free_input_rt:
@@ -887,19 +855,7 @@ static void *__qcom_scm_pas_get_rsc_table2(struct device *dev,
 	return ret ? ERR_PTR(ret) : tbl_ptr;
 }
 
-struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *ctx,
-						  void *input_rt,
-						  size_t input_rt_size,
-						  size_t *output_rt_size)
-{
-	return __qcom_scm_pas_get_rsc_table2(__scm->dev,
-					     (struct qcom_pas_context *)ctx,
-					     input_rt, input_rt_size,
-					     output_rt_size);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_get_rsc_table);
-
-static int __qcom_scm_pas_auth_and_reset(struct device *dev, u32 pas_id)
+static int qcom_scm_pas_auth_and_reset(struct device *dev, u32 pas_id)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -928,14 +884,8 @@ static int __qcom_scm_pas_auth_and_reset(struct device *dev, u32 pas_id)
 	return ret ? : res.result[0];
 }
 
-int qcom_scm_pas_auth_and_reset(u32 pas_id)
-{
-	return __qcom_scm_pas_auth_and_reset(__scm->dev, pas_id);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_auth_and_reset);
-
-static int __qcom_scm_pas_prepare_and_auth_reset(struct device *dev,
-						 struct qcom_pas_context *ctx)
+static int qcom_scm_pas_prepare_and_auth_reset(struct device *dev,
+					       struct qcom_pas_context *ctx)
 {
 	u64 handle;
 	int ret;
@@ -946,7 +896,7 @@ static int __qcom_scm_pas_prepare_and_auth_reset(struct device *dev,
 	 * memory region and then invokes a call to TrustZone to authenticate.
 	 */
 	if (!ctx->use_tzmem)
-		return __qcom_scm_pas_auth_and_reset(dev, ctx->pas_id);
+		return qcom_scm_pas_auth_and_reset(dev, ctx->pas_id);
 
 	/*
 	 * When Linux runs @ EL2 Linux must create the shmbridge itself and then
@@ -956,21 +906,14 @@ static int __qcom_scm_pas_prepare_and_auth_reset(struct device *dev,
 	if (ret)
 		return ret;
 
-	ret = __qcom_scm_pas_auth_and_reset(dev, ctx->pas_id);
+	ret = qcom_scm_pas_auth_and_reset(dev, ctx->pas_id);
 	qcom_tzmem_shm_bridge_delete(handle);
 
 	return ret;
 }
 
-int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx)
-{
-	return __qcom_scm_pas_prepare_and_auth_reset(__scm->dev,
-						     (struct qcom_pas_context *)ctx);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_prepare_and_auth_reset);
-
-static int __qcom_scm_pas_set_remote_state(struct device *dev, u32 state,
-					   u32 pas_id)
+static int qcom_scm_pas_set_remote_state(struct device *dev, u32 state,
+					 u32 pas_id)
 {
 	struct qcom_scm_desc desc = {
 		.svc = QCOM_SCM_SVC_BOOT,
@@ -988,13 +931,7 @@ static int __qcom_scm_pas_set_remote_state(struct device *dev, u32 state,
 	return ret ? : res.result[0];
 }
 
-int qcom_scm_set_remote_state(u32 state, u32 id)
-{
-	return __qcom_scm_pas_set_remote_state(__scm->dev, state, id);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_set_remote_state);
-
-static int __qcom_scm_pas_shutdown(struct device *dev, u32 pas_id)
+static int qcom_scm_pas_shutdown(struct device *dev, u32 pas_id)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -1023,13 +960,7 @@ static int __qcom_scm_pas_shutdown(struct device *dev, u32 pas_id)
 	return ret ? : res.result[0];
 }
 
-int qcom_scm_pas_shutdown(u32 pas_id)
-{
-	return __qcom_scm_pas_shutdown(__scm->dev, pas_id);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_shutdown);
-
-static bool __qcom_scm_pas_supported(struct device *dev, u32 pas_id)
+static bool qcom_scm_pas_supported(struct device *dev, u32 pas_id)
 {
 	int ret;
 	struct qcom_scm_desc desc = {
@@ -1050,23 +981,17 @@ static bool __qcom_scm_pas_supported(struct device *dev, u32 pas_id)
 	return ret ? false : !!res.result[0];
 }
 
-bool qcom_scm_pas_supported(u32 pas_id)
-{
-	return __qcom_scm_pas_supported(__scm->dev, pas_id);
-}
-EXPORT_SYMBOL_GPL(qcom_scm_pas_supported);
-
 static struct qcom_pas_ops qcom_pas_ops_scm = {
 	.drv_name		= "qcom_scm",
-	.supported		= __qcom_scm_pas_supported,
-	.init_image		= __qcom_scm_pas_init_image2,
-	.mem_setup		= __qcom_scm_pas_mem_setup,
-	.get_rsc_table		= __qcom_scm_pas_get_rsc_table2,
-	.auth_and_reset		= __qcom_scm_pas_auth_and_reset,
-	.prepare_and_auth_reset	= __qcom_scm_pas_prepare_and_auth_reset,
-	.set_remote_state	= __qcom_scm_pas_set_remote_state,
-	.shutdown		= __qcom_scm_pas_shutdown,
-	.metadata_release	= __qcom_scm_pas_metadata_release,
+	.supported		= qcom_scm_pas_supported,
+	.init_image		= qcom_scm_pas_init_image,
+	.mem_setup		= qcom_scm_pas_mem_setup,
+	.get_rsc_table		= qcom_scm_pas_get_rsc_table,
+	.auth_and_reset		= qcom_scm_pas_auth_and_reset,
+	.prepare_and_auth_reset	= qcom_scm_pas_prepare_and_auth_reset,
+	.set_remote_state	= qcom_scm_pas_set_remote_state,
+	.shutdown		= qcom_scm_pas_shutdown,
+	.metadata_release	= qcom_scm_pas_metadata_release,
 };
 
 /**
@@ -2302,6 +2227,7 @@ EXPORT_SYMBOL_GPL(qcom_scm_qseecom_app_send);
 static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "asus,vivobook-s15" },
 	{ .compatible = "asus,vivobook-s15-x1p4" },
+	{ .compatible = "asus,zenbook-a14-ux3407na" },
 	{ .compatible = "asus,zenbook-a14-ux3407qa" },
 	{ .compatible = "asus,zenbook-a14-ux3407ra" },
 	{ .compatible = "asus,zenbook-a16-ux3607oa" },
@@ -2311,14 +2237,17 @@ static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "ecs,liva-qc710" },
 	{ .compatible = "honor,magicbook-art-14-snapdragon" },
 	{ .compatible = "hp,elitebook-ultra-g1q" },
+	{ .compatible = "hp,elitebook-x-g2q" },
 	{ .compatible = "hp,omnibook-x14" },
 	{ .compatible = "huawei,gaokun3" },
 	{ .compatible = "lenovo,flex-5g" },
 	{ .compatible = "lenovo,ideacentre-mini-01q8x10" },
 	{ .compatible = "lenovo,thinkbook-16" },
 	{ .compatible = "lenovo,thinkpad-t14s" },
+	{ .compatible = "lenovo,thinkpad-t14s-gen7" },
 	{ .compatible = "lenovo,thinkpad-x13s", },
 	{ .compatible = "lenovo,yoga-slim7x" },
+	{ .compatible = "lenovo,yoga-slim7x-gen11" },
 	{ .compatible = "medion,sprchrgd14s1" },
 	{ .compatible = "microsoft,arcata", },
 	{ .compatible = "microsoft,surface-pro-12in", },
@@ -2327,7 +2256,9 @@ static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "microsoft,romulus13", },
 	{ .compatible = "microsoft,romulus15", },
 	{ .compatible = "qcom,glymur-crd" },
+	{ .compatible = "qcom,glymur-qcb" },
 	{ .compatible = "qcom,hamoa-iot-evk" },
+	{ .compatible = "qcom,kalambo-crd" },
 	{ .compatible = "qcom,mahua-crd" },
 	{ .compatible = "qcom,purwa-iot-evk" },
 	{ .compatible = "qcom,sc8180x-primus" },
@@ -2844,17 +2775,15 @@ static int qcom_scm_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	ret = of_reserved_mem_device_init(scm->dev);
+	ret = devm_of_reserved_mem_device_init(scm->dev);
 	if (ret && ret != -ENODEV)
 		return dev_err_probe(scm->dev, ret,
 				     "Failed to setup the reserved memory region for TZ mem\n");
 
 	ret = qcom_tzmem_enable(scm->dev);
-	if (ret) {
-		ret = dev_err_probe(scm->dev, ret,
-				    "Failed to enable the TrustZone memory allocator\n");
-		goto err_rmem;
-	}
+	if (ret)
+		return dev_err_probe(scm->dev, ret,
+				     "Failed to enable the TrustZone memory allocator\n");
 
 	memset(&pool_config, 0, sizeof(pool_config));
 	pool_config.initial_size = 0;
@@ -2862,11 +2791,9 @@ static int qcom_scm_probe(struct platform_device *pdev)
 	pool_config.max_size = SZ_256K;
 
 	scm->mempool = devm_qcom_tzmem_pool_new(scm->dev, &pool_config);
-	if (IS_ERR(scm->mempool)) {
-		ret = dev_err_probe(scm->dev, PTR_ERR(scm->mempool),
-				    "Failed to create the SCM memory pool\n");
-		goto err_rmem;
-	}
+	if (IS_ERR(scm->mempool))
+		return dev_err_probe(scm->dev, PTR_ERR(scm->mempool),
+				     "Failed to create the SCM memory pool\n");
 
 	ret = qcom_scm_query_waitq_count(scm);
 	scm->wq_cnt = ret < 0 ? QCOM_SCM_DEFAULT_WAITQ_COUNT : ret;
@@ -2942,10 +2869,6 @@ static int qcom_scm_probe(struct platform_device *pdev)
 	qcom_scm_gunyah_wdt_init(scm);
 
 	return 0;
-
-err_rmem:
-	of_reserved_mem_device_release(scm->dev);
-	return ret;
 }
 
 static void qcom_scm_shutdown(struct platform_device *pdev)
diff --git a/drivers/firmware/qcom/qcom_tzmem.c b/drivers/firmware/qcom/qcom_tzmem.c
index f926c1f648221f..e1d944d0eb63fd 100644
--- a/drivers/firmware/qcom/qcom_tzmem.c
+++ b/drivers/firmware/qcom/qcom_tzmem.c
@@ -50,7 +50,7 @@ static struct device *qcom_tzmem_dev;
 static RADIX_TREE(qcom_tzmem_chunks, GFP_ATOMIC);
 static DEFINE_SPINLOCK(qcom_tzmem_chunks_lock);
 
-#if IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_GENERIC)
+#ifndef CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE
 
 static int qcom_tzmem_init(void)
 {
@@ -67,7 +67,7 @@ static void qcom_tzmem_cleanup_area(struct qcom_tzmem_area *area)
 
 }
 
-#elif IS_ENABLED(CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE)
+#else
 
 #include <linux/firmware/qcom/qcom_scm.h>
 #include <linux/of.h>
@@ -82,6 +82,7 @@ static const char *const qcom_tzmem_blacklist[] = {
 	"qcom,sc8180x",
 	"qcom,sdm670", /* failure in GPU firmware loading */
 	"qcom,sdm845", /* reset in rmtfs memory assignment */
+	"qcom,sm7125", /* hang in rmtfs memory assignment */
 	"qcom,sm7150", /* reset in rmtfs memory assignment */
 	"qcom,sm8150", /* reset in rmtfs memory assignment */
 	NULL
@@ -516,7 +517,7 @@ int qcom_tzmem_enable(struct device *dev)
 	static int result;
 
 	qcom_tzmem_dev = dev;
-	DO_ONCE(qcom_tzmem_do_init, &result);
+	DO_ONCE_SLEEPABLE(qcom_tzmem_do_init, &result);
 	return result;
 }
 EXPORT_SYMBOL_GPL(qcom_tzmem_enable);
diff --git a/drivers/firmware/smccc/Makefile b/drivers/firmware/smccc/Makefile
index 40d19144a86072..68bbff1407b8b4 100644
--- a/drivers/firmware/smccc/Makefile
+++ b/drivers/firmware/smccc/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
 #
-obj-$(CONFIG_HAVE_ARM_SMCCC_DISCOVERY)	+= smccc.o kvm_guest.o
+obj-$(CONFIG_HAVE_ARM_SMCCC_DISCOVERY)	+= bus.o smccc.o kvm_guest.o
 obj-$(CONFIG_ARM_SMCCC_SOC_ID)	+= soc_id.o
diff --git a/drivers/firmware/smccc/bus.c b/drivers/firmware/smccc/bus.c
new file mode 100644
index 00000000000000..0357a7c48af0ba
--- /dev/null
+++ b/drivers/firmware/smccc/bus.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Arm Limited
+ */
+
+#include <linux/arm-smccc-bus.h>
+#include <linux/slab.h>
+
+static int arm_smccc_bus_match(struct device *dev,
+		const struct device_driver *drv)
+{
+	const struct arm_smccc_device_id *id_table;
+	const struct arm_smccc_device *smccc_dev = to_arm_smccc_device(dev);
+
+	id_table = to_arm_smccc_driver(drv)->id_table;
+	if (!id_table)
+		return 0;
+
+	while (id_table->func_id) {
+		if (smccc_dev->func_id == id_table->func_id)
+			return 1;
+		id_table++;
+	}
+
+	return 0;
+}
+
+static int arm_smccc_bus_probe(struct device *dev)
+{
+	struct arm_smccc_driver *smccc_drv = to_arm_smccc_driver(dev->driver);
+
+	return smccc_drv->probe(to_arm_smccc_device(dev));
+}
+
+static void arm_smccc_bus_remove(struct device *dev)
+{
+	struct arm_smccc_driver *smcc_drv = to_arm_smccc_driver(dev->driver);
+
+	if (smcc_drv->remove)
+		smcc_drv->remove(to_arm_smccc_device(dev));
+}
+
+static int arm_smccc_bus_uevent(const struct device *dev,
+		struct kobj_uevent_env *env)
+{
+	const struct arm_smccc_device *smccc_dev = to_arm_smccc_device(dev);
+
+	return add_uevent_var(env, "MODALIAS=" ARM_SMCCC_MODULE_PREFIX "f%08X",
+			      smccc_dev->func_id);
+}
+
+static ssize_t modalias_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct arm_smccc_device *smccc_dev = to_arm_smccc_device(dev);
+
+	return sysfs_emit(buf, ARM_SMCCC_MODULE_PREFIX "f%08X\n",
+			  smccc_dev->func_id);
+}
+static DEVICE_ATTR_RO(modalias);
+
+static struct attribute *arm_smccc_device_attrs[] = {
+	&dev_attr_modalias.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(arm_smccc_device);
+
+const struct bus_type arm_smccc_bus_type = {
+	.name = "arm_smccc",
+	.match = arm_smccc_bus_match,
+	.probe = arm_smccc_bus_probe,
+	.remove = arm_smccc_bus_remove,
+	.uevent = arm_smccc_bus_uevent,
+	.dev_groups = arm_smccc_device_groups,
+};
+EXPORT_SYMBOL_GPL(arm_smccc_bus_type);
+
+int arm_smccc_driver_register(struct arm_smccc_driver *driver,
+		struct module *owner, const char *mod_name)
+{
+	if (!driver->probe || !driver->id_table)
+		return -EINVAL;
+
+	driver->driver.bus = &arm_smccc_bus_type;
+	driver->driver.name = driver->name;
+	driver->driver.owner = owner;
+	driver->driver.mod_name = mod_name;
+
+	return driver_register(&driver->driver);
+}
+EXPORT_SYMBOL_GPL(arm_smccc_driver_register);
+
+void arm_smccc_driver_unregister(struct arm_smccc_driver *driver)
+{
+	driver_unregister(&driver->driver);
+}
+EXPORT_SYMBOL_GPL(arm_smccc_driver_unregister);
+
+static void arm_smccc_release_device(struct device *dev)
+{
+	struct arm_smccc_device *smccc_dev = to_arm_smccc_device(dev);
+
+	kfree(smccc_dev);
+}
+
+static struct device *arm_smccc_device_alloc(u32 func_id)
+{
+	struct arm_smccc_device *smccc_dev;
+
+	smccc_dev = kzalloc_obj(*smccc_dev);
+	if (!smccc_dev)
+		return NULL;
+
+	smccc_dev->func_id = func_id;
+	smccc_dev->dev.bus = &arm_smccc_bus_type;
+	smccc_dev->dev.release = arm_smccc_release_device;
+	device_initialize(&smccc_dev->dev);
+
+	return &smccc_dev->dev;
+}
+
+struct arm_smccc_device *arm_smccc_device_register(const char *name, u32 func_id)
+{
+	int ret;
+
+	if (!name)
+		return ERR_PTR(-EINVAL);
+
+	struct device *dev __free(put_device) = arm_smccc_device_alloc(func_id);
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	ret = dev_set_name(dev, "%s", name);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = device_add(dev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return to_arm_smccc_device(no_free_ptr(dev));
+}
+EXPORT_SYMBOL_GPL(arm_smccc_device_register);
+
+void arm_smccc_device_unregister(struct arm_smccc_device *smccc_dev)
+{
+	if (!smccc_dev)
+		return;
+
+	device_unregister(&smccc_dev->dev);
+}
+EXPORT_SYMBOL_GPL(arm_smccc_device_unregister);
+
+static int __init arm_smccc_bus_init(void)
+{
+	return bus_register(&arm_smccc_bus_type);
+}
+subsys_initcall(arm_smccc_bus_init);
diff --git a/drivers/firmware/smccc/smccc.c b/drivers/firmware/smccc/smccc.c
index bdee057db2fd3b..afc2e4d602ba62 100644
--- a/drivers/firmware/smccc/smccc.c
+++ b/drivers/firmware/smccc/smccc.c
@@ -9,7 +9,9 @@
 #include <linux/init.h>
 #include <linux/arm-smccc.h>
 #include <linux/kernel.h>
-#include <linux/platform_device.h>
+#include <linux/arm-smccc-bus.h>
+#include <linux/arm-smccc-rsi.h>
+
 #include <asm/archrandom.h>
 
 static u32 smccc_version = ARM_SMCCC_VERSION_1_0;
@@ -81,16 +83,64 @@ bool arm_smccc_hypervisor_has_uuid(const uuid_t *hyp_uuid)
 }
 EXPORT_SYMBOL_GPL(arm_smccc_hypervisor_has_uuid);
 
+struct smccc_device_info {
+	u32 func_id;
+	bool requires_smc;
+	const char *device_name;
+};
+
+static const struct smccc_device_info smccc_devices[] __initconst = {
+	{
+		.func_id        = ARM_SMCCC_TRNG_VERSION,
+		.requires_smc   = false,
+		.device_name    = "arm-smccc-trng",
+	},
+	{
+		.func_id        = SMC_RSI_ABI_VERSION,
+		.requires_smc   = true,
+		.device_name    = "arm-rsi",
+	},
+};
+
+static bool __init smccc_probe_smccc_device(const struct smccc_device_info *smccc_dev)
+{
+	int ret;
+	struct arm_smccc_res res = {};
+
+	if (smccc_conduit == SMCCC_CONDUIT_NONE)
+		return false;
+
+	if (smccc_dev->requires_smc && smccc_conduit != SMCCC_CONDUIT_SMC)
+		return false;
+
+	if (IS_ENABLED(CONFIG_ARM) && ARM_SMCCC_IS_64(smccc_dev->func_id))
+		return false;
+
+	arm_smccc_1_1_invoke(smccc_dev->func_id,
+			     0, 0, 0, 0, 0, 0, 0, &res);
+	ret = res.a0;
+
+	if (ret == SMCCC_RET_NOT_SUPPORTED)
+		return false;
+
+	return true;
+}
+
 static int __init smccc_devices_init(void)
 {
-	struct platform_device *pdev;
-
-	if (smccc_trng_available) {
-		pdev = platform_device_register_simple("smccc_trng", -1,
-						       NULL, 0);
-		if (IS_ERR(pdev))
-			pr_err("smccc_trng: could not register device: %ld\n",
-			       PTR_ERR(pdev));
+	struct arm_smccc_device *sdev;
+	const struct smccc_device_info *smccc_dev;
+
+	for (int i = 0; i < ARRAY_SIZE(smccc_devices); i++) {
+		smccc_dev = &smccc_devices[i];
+
+		if (!smccc_probe_smccc_device(smccc_dev))
+			continue;
+
+		sdev = arm_smccc_device_register(smccc_dev->device_name, smccc_dev->func_id);
+		if (IS_ERR(sdev))
+			pr_err("%s: could not register device: %pe\n",
+			       smccc_dev->device_name, sdev);
 	}
 
 	return 0;
diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c
index 33d7c4a4181e6e..5fa63ff7c74a6d 100644
--- a/drivers/firmware/ti_sci.c
+++ b/drivers/firmware/ti_sci.c
@@ -4050,7 +4050,7 @@ static const struct ti_sci_desc ti_sci_pmmc_k2g_desc = {
 	.default_host_id = 2,
 	/* Conservative duration */
 	.max_rx_timeout_ms = 1000,
-	/* Limited by MBOX_TX_QUEUE_LEN. K2G can handle upto 128 messages! */
+	/* Limited by MBOX_TX_QUEUE_LEN. K2G can handle up to 128 messages! */
 	.max_msgs = 20,
 	.max_msg_size = 64,
 };
@@ -4060,7 +4060,7 @@ static const struct ti_sci_desc ti_sci_pmmc_am654_desc = {
 	.default_host_id = 12,
 	/* Conservative duration */
 	.max_rx_timeout_ms = 10000,
-	/* Limited by MBOX_TX_QUEUE_LEN. K2G can handle upto 128 messages! */
+	/* Limited by MBOX_TX_QUEUE_LEN. K2G can handle up to 128 messages! */
 	.max_msgs = 20,
 	.max_msg_size = 60,
 };
@@ -4132,7 +4132,7 @@ static int ti_sci_probe(struct platform_device *pdev)
 
 	/* Pre-initialize the buffer pointer to pre-allocated buffers */
 	for (i = 0, xfer = minfo->xfer_block; i < desc->max_msgs; i++, xfer++) {
-		xfer->xfer_buf = devm_kcalloc(dev, 1, desc->max_msg_size,
+		xfer->xfer_buf = devm_kzalloc(dev, desc->max_msg_size,
 					      GFP_KERNEL);
 		if (!xfer->xfer_buf)
 			return -ENOMEM;
diff --git a/drivers/firmware/xilinx/zynqmp-ufs.c b/drivers/firmware/xilinx/zynqmp-ufs.c
index 85da8a822f3a58..81ccf61a037ce6 100644
--- a/drivers/firmware/xilinx/zynqmp-ufs.c
+++ b/drivers/firmware/xilinx/zynqmp-ufs.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2025 Advanced Micro Devices, Inc.
  */
 
+#include <linux/delay.h>
 #include <linux/firmware/xlnx-zynqmp.h>
 #include <linux/module.h>
 
@@ -33,7 +34,7 @@
  *
  * Return:	Returns 0 on success or error value on failure.
  */
-int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready)
+static int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready)
 {
 	u32 regval;
 	int ret;
@@ -53,7 +54,6 @@ int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready)
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(zynqmp_pm_is_mphy_tx_rx_config_ready);
 
 /**
  * zynqmp_pm_is_sram_init_done - check SRAM initialization
@@ -61,7 +61,7 @@ EXPORT_SYMBOL_GPL(zynqmp_pm_is_mphy_tx_rx_config_ready);
  *
  * Return:	Returns 0 on success or error value on failure.
  */
-int zynqmp_pm_is_sram_init_done(bool *is_done)
+static int zynqmp_pm_is_sram_init_done(bool *is_done)
 {
 	u32 regval;
 	int ret;
@@ -81,7 +81,69 @@ int zynqmp_pm_is_sram_init_done(bool *is_done)
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(zynqmp_pm_is_sram_init_done);
+
+/**
+ * zynqmp_pm_wait_mphy_tx_rx_config_ready - wait for M-PHY TX-RX config ready
+ * @timeout_us:	Caller-supplied timeout budget in microseconds
+ *
+ * Poll the M-PHY TX-RX configuration-ready status until it settles or the
+ * timeout elapses. The poll loop is EEMI-specific (legacy firmware only offers
+ * the per-read status primitive), so it lives here in the firmware driver
+ * rather than in the UFS controller driver; an SCMI-based backend can instead
+ * offload the wait to the platform in a single call. The timeout budget is
+ * owned by the caller (UFS driver), keeping the policy with the consumer.
+ *
+ * Return:	Returns 0 once ready, -ETIMEDOUT on timeout, or error value.
+ */
+int zynqmp_pm_wait_mphy_tx_rx_config_ready(u32 timeout_us)
+{
+	bool is_ready;
+	int ret;
+
+	while (timeout_us--) {
+		ret = zynqmp_pm_is_mphy_tx_rx_config_ready(&is_ready);
+		if (ret)
+			return ret;
+
+		if (!is_ready)
+			return 0;
+
+		usleep_range(1, 5);
+	}
+
+	return -ETIMEDOUT;
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_wait_mphy_tx_rx_config_ready);
+
+/**
+ * zynqmp_pm_wait_sram_init_done - wait for SRAM initialization to complete
+ * @timeout_us:	Caller-supplied timeout budget in microseconds
+ *
+ * Poll the SRAM initialization-done status until it is set or the timeout
+ * elapses. As with the M-PHY wait, the poll loop is EEMI-specific and kept in
+ * the firmware driver so the UFS controller driver stays backend-agnostic.
+ *
+ * Return:	Returns 0 once done, -ETIMEDOUT on timeout, or error value.
+ */
+int zynqmp_pm_wait_sram_init_done(u32 timeout_us)
+{
+	bool is_done;
+	int ret;
+
+	while (timeout_us--) {
+		ret = zynqmp_pm_is_sram_init_done(&is_done);
+		if (ret)
+			return ret;
+
+		if (is_done)
+			return 0;
+
+		usleep_range(1, 5);
+	}
+
+	return -ETIMEDOUT;
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_wait_sram_init_done);
 
 /**
  * zynqmp_pm_set_sram_bypass - Set SRAM bypass Control
diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index fe650747ac91c9..d76660c553542d 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -1527,7 +1527,6 @@ EXPORT_SYMBOL_GPL(zynqmp_pm_request_wake);
  */
 int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr)
 {
-	enum rpu_boot_mem bootmem;
 	int ret;
 
 	/*
@@ -1536,22 +1535,55 @@ int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr)
 	 * starts at the base-address and subsequent vectors are on 4-byte
 	 * boundaries.
 	 *
+	 * The Cortex-R5 cores can boot from TCM and OCM memories.
 	 * Exception vectors can start either from 0x0000_0000 (LOVEC) or
 	 * from 0xFFFF_0000 (HIVEC) which is mapped in the OCM (On-Chip Memory)
 	 *
 	 * Usually firmware will put Exception vectors at LOVEC.
 	 *
+	 * The Cortex-R52 cores can boot from DDR and TCM.
+	 * That means vector table can be at address 0x0 or at any DDR address
+	 * that is in the 32-bit address range. Note that booting from DDR
+	 * address 0x0 is not supported and the user must always assume that if
+	 * 0x0 address is passed, then it will be TCM boot.
+	 *
 	 * It is not recommend that you change the exception vector.
 	 * Changing the EVP to HIVEC will result in increased interrupt latency
 	 * and jitter. Also, if the OCM is secured and the Cortex-R5F processor
 	 * is non-secured, then the Cortex-R5F processor cannot access the
 	 * HIVEC exception vectors in the OCM.
 	 */
-	bootmem = (bootaddr >= 0xFFFC0000) ?
-		   PM_RPU_BOOTMEM_HIVEC : PM_RPU_BOOTMEM_LOVEC;
+	if (upper_32_bits(bootaddr)) {
+		pr_err("invalid bootaddr = 0x%llx\n", bootaddr);
+		return -EINVAL;
+	}
 
-	pr_debug("RPU boot addr 0x%llx from %s.", bootaddr,
-		 bootmem == PM_RPU_BOOTMEM_HIVEC ? "OCM" : "TCM");
+	/*
+	 * New platforms can boot from DDR and require the DDR address
+	 * to be configured explicitly. If the correct version of the
+	 * IOCTL_RPU_BOOT_ADDR_CONFIG ioctl is not supported, do not
+	 * treat it as a failure. If a DDR address is passed on other
+	 * platforms, the PM request wake EEMI call will still fail and
+	 * the RPU won't boot.
+	 */
+	ret = zynqmp_pm_is_function_supported(PM_IOCTL,
+					      IOCTL_RPU_BOOT_ADDR_CONFIG);
+	if (!ret) {
+		/* config ddr boot address */
+		ret = zynqmp_pm_invoke_fn(PM_IOCTL, NULL, 3,
+					  node,
+					  IOCTL_RPU_BOOT_ADDR_CONFIG,
+					  bootaddr);
+		if (ret < 0) {
+			pr_err("failed to set RPU Boot address 0x%llx\n",
+			       bootaddr);
+			return ret;
+		}
+	} else if (ret != -EOPNOTSUPP && ret != -ENODATA) {
+		pr_err("ioctl rpu boot addr config ver check failed %d\n",
+		       ret);
+		return ret;
+	}
 
 	/* Request node before starting RPU core if new version of API is supported */
 	if (zynqmp_pm_feature(PM_REQUEST_NODE) > PM_API_VERSION_1) {
@@ -1565,7 +1597,7 @@ int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr)
 	}
 
 	ret = zynqmp_pm_request_wake(node, true,
-				     bootmem, ZYNQMP_PM_REQUEST_ACK_NO);
+				     bootaddr, ZYNQMP_PM_REQUEST_ACK_NO);
 	if (ret)
 		pr_err("failed to start RPU = 0x%x\n", node);
 	return ret;
diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c
index 070c025ee21549..507e29d3a79fff 100644
--- a/drivers/fpga/altera-cvp.c
+++ b/drivers/fpga/altera-cvp.c
@@ -280,6 +280,16 @@ static int altera_cvp_send_block(struct altera_cvp_conf *conf,
 	return 0;
 }
 
+static void altera_cvp_disable_cvp_mode(struct altera_cvp_conf *conf)
+{
+	u32 val;
+
+	altera_read_config_dword(conf, VSE_CVP_MODE_CTRL, &val);
+	val &= ~VSE_CVP_MODE_CTRL_HIP_CLK_SEL;
+	val &= ~VSE_CVP_MODE_CTRL_CVP_MODE;
+	altera_write_config_dword(conf, VSE_CVP_MODE_CTRL, val);
+}
+
 static int altera_cvp_teardown(struct fpga_manager *mgr,
 			       struct fpga_image_info *info)
 {
@@ -308,7 +318,29 @@ static int altera_cvp_teardown(struct fpga_manager *mgr,
 	ret = altera_cvp_wait_status(conf, VSE_CVP_STATUS_CFG_RDY, 0,
 				     conf->priv->poll_time_us);
 	if (ret)
-		dev_err(&mgr->dev, "CFG_RDY == 0 timeout\n");
+		dev_warn(&mgr->dev, "CFG_RDY == 0 timeout\n");
+
+	return ret;
+}
+
+static int altera_cvp_recovery(struct fpga_manager *mgr,
+			       struct fpga_image_info *info)
+{
+	struct altera_cvp_conf *conf = mgr->priv;
+	int ret;
+
+	ret = altera_cvp_teardown(mgr, info);
+	if (ret) {
+		/*
+		 * CVP_STATUS may get stuck if the device left user mode
+		 * before the poll. Exit CvP mode to switch back the clock
+		 * feeding CVP_STATUS, then try the teardown again.
+		 */
+		altera_cvp_disable_cvp_mode(conf);
+		ret = altera_cvp_teardown(mgr, info);
+		if (ret)
+			dev_err(&mgr->dev, "Tear-down failed\n");
+	}
 
 	return ret;
 }
@@ -345,7 +377,7 @@ static int altera_cvp_write_init(struct fpga_manager *mgr,
 
 	if (val & VSE_CVP_STATUS_CFG_RDY) {
 		dev_warn(&mgr->dev, "CvP already started, tear down first\n");
-		ret = altera_cvp_teardown(mgr, info);
+		ret = altera_cvp_recovery(mgr, info);
 		if (ret)
 			return ret;
 	}
@@ -486,7 +518,7 @@ static int altera_cvp_write_complete(struct fpga_manager *mgr,
 	u32 mask, val;
 	int ret;
 
-	ret = altera_cvp_teardown(mgr, info);
+	ret = altera_cvp_recovery(mgr, info);
 	if (ret)
 		return ret;
 
@@ -498,10 +530,7 @@ static int altera_cvp_write_complete(struct fpga_manager *mgr,
 	}
 
 	/* STEP 17 - reset CVP_MODE and HIP_CLK_SEL bit */
-	altera_read_config_dword(conf, VSE_CVP_MODE_CTRL, &val);
-	val &= ~VSE_CVP_MODE_CTRL_HIP_CLK_SEL;
-	val &= ~VSE_CVP_MODE_CTRL_CVP_MODE;
-	altera_write_config_dword(conf, VSE_CVP_MODE_CTRL, val);
+	altera_cvp_disable_cvp_mode(conf);
 
 	/* STEP 18 - poll PLD_CLK_IN_USE and USER_MODE bits */
 	mask = VSE_CVP_STATUS_PLD_CLK_IN_USE | VSE_CVP_STATUS_USERMODE;
diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h
index 9549f63b00ebcb..9a65617bb87021 100644
--- a/drivers/fpga/dfl.h
+++ b/drivers/fpga/dfl.h
@@ -23,7 +23,6 @@
 #include <linux/interrupt.h>
 #include <linux/iopoll.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
-#include <linux/mod_devicetable.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/uuid.h>
diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c
index 2672070ced026b..5a0b7780b33588 100644
--- a/drivers/fpga/fpga-mgr.c
+++ b/drivers/fpga/fpga-mgr.c
@@ -744,7 +744,7 @@ EXPORT_SYMBOL_GPL(fpga_mgr_put);
  * @mgr:	fpga manager
  *
  * Given a pointer to FPGA Manager (from fpga_mgr_get() or
- * of_fpga_mgr_put()) attempt to get the mutex. The user should call
+ * of_fpga_mgr_get()) attempt to get the mutex. The user should call
  * fpga_mgr_lock() and verify that it returns 0 before attempting to
  * program the FPGA.  Likewise, the user should call fpga_mgr_unlock
  * when done programming the FPGA.
diff --git a/drivers/fpga/stratix10-soc.c b/drivers/fpga/stratix10-soc.c
index b8ec2e6f615fad..5aed560050c27b 100644
--- a/drivers/fpga/stratix10-soc.c
+++ b/drivers/fpga/stratix10-soc.c
@@ -286,7 +286,7 @@ static int s10_ops_write(struct fpga_manager *mgr, const char *buf,
 
 	/*
 	 * Loop waiting for buffers to be returned.  When a buffer is returned,
-	 * reuse it to send more data or free if if all data has been sent.
+	 * reuse it to send more data or free it if all data has been sent.
 	 */
 	while (count > 0 || s10_free_buffer_count(mgr) != NUM_SVC_BUFS) {
 		reinit_completion(&priv->status_return_completion);
diff --git a/drivers/fpga/xilinx-selectmap.c b/drivers/fpga/xilinx-selectmap.c
index d0cbb5fdfe3acf..ff167b23fdeda9 100644
--- a/drivers/fpga/xilinx-selectmap.c
+++ b/drivers/fpga/xilinx-selectmap.c
@@ -19,6 +19,8 @@
 struct xilinx_selectmap_conf {
 	struct xilinx_fpga_core core;
 	void __iomem *base;
+	struct gpio_desc *csi_b;
+	struct gpio_desc *rdwr_b;
 };
 
 #define to_xilinx_selectmap_conf(obj) \
@@ -30,16 +32,28 @@ static int xilinx_selectmap_write(struct xilinx_fpga_core *core,
 	struct xilinx_selectmap_conf *conf = to_xilinx_selectmap_conf(core);
 	size_t i;
 
+	/*
+	 * Select this device and put its SelectMAP port into write mode.
+	 *
+	 * RDWR_B is set before CSI_B. UG470 describes in chapter
+	 * "SelectMAP ABORT" that changing RDWR_B while the device is
+	 * selected aborts the configuration on the next CCLK.
+	 */
+	gpiod_set_value(conf->rdwr_b, 1);
+	gpiod_set_value(conf->csi_b, 1);
+
 	for (i = 0; i < count; ++i)
 		writeb(buf[i], conf->base);
 
+	gpiod_set_value(conf->csi_b, 0);
+	gpiod_set_value(conf->rdwr_b, 0);
+
 	return 0;
 }
 
 static int xilinx_selectmap_probe(struct platform_device *pdev)
 {
 	struct xilinx_selectmap_conf *conf;
-	struct gpio_desc *gpio;
 	void __iomem *base;
 
 	conf = devm_kzalloc(&pdev->dev, sizeof(*conf), GFP_KERNEL);
@@ -55,16 +69,23 @@ static int xilinx_selectmap_probe(struct platform_device *pdev)
 				     "ioremap error\n");
 	conf->base = base;
 
-	/* CSI_B is active low */
-	gpio = devm_gpiod_get_optional(&pdev->dev, "csi", GPIOD_OUT_HIGH);
-	if (IS_ERR(gpio))
-		return dev_err_probe(&pdev->dev, PTR_ERR(gpio),
+	/*
+	 * Request both signals deasserted, so a device sharing the SelectMAP
+	 * bus with others stays off that bus until its bitstream is written.
+	 *
+	 * The value in the gpiod flags is logical, gpiolib drives the line
+	 * high for GPIOD_OUT_LOW when the firmware describes it active low.
+	 */
+	conf->csi_b = devm_gpiod_get_optional(&pdev->dev, "csi",
+					      GPIOD_OUT_LOW);
+	if (IS_ERR(conf->csi_b))
+		return dev_err_probe(&pdev->dev, PTR_ERR(conf->csi_b),
 				     "Failed to get CSI_B gpio\n");
 
-	/* RDWR_B is active low */
-	gpio = devm_gpiod_get_optional(&pdev->dev, "rdwr", GPIOD_OUT_HIGH);
-	if (IS_ERR(gpio))
-		return dev_err_probe(&pdev->dev, PTR_ERR(gpio),
+	conf->rdwr_b = devm_gpiod_get_optional(&pdev->dev, "rdwr",
+					       GPIOD_OUT_LOW);
+	if (IS_ERR(conf->rdwr_b))
+		return dev_err_probe(&pdev->dev, PTR_ERR(conf->rdwr_b),
 				     "Failed to get RDWR_B gpio\n");
 
 	return xilinx_core_probe(&conf->core);
diff --git a/drivers/gpib/fmh_gpib/fmh_gpib.c b/drivers/gpib/fmh_gpib/fmh_gpib.c
index 5e10e9353fedd6..4ab9b0a317fdb8 100644
--- a/drivers/gpib/fmh_gpib/fmh_gpib.c
+++ b/drivers/gpib/fmh_gpib/fmh_gpib.c
@@ -1458,11 +1458,15 @@ static int fmh_gpib_attach_impl(struct gpib_board *board, const struct gpib_boar
 	e_priv->irq = irq;
 
 	if (acquire_dma) {
-		e_priv->dma_channel = dma_request_slave_channel(board->dev, "rxtx");
-		if (!e_priv->dma_channel) {
+		struct dma_chan *tmp_chan;
+
+		tmp_chan = dma_request_chan(board->dev, "rxtx");
+		retval = PTR_ERR_OR_ZERO(tmp_chan);
+		if (retval) {
 			dev_err(board->dev, "failed to acquire dma channel \"rxtx\".\n");
-			return -EIO;
+			return retval;
 		}
+		e_priv->dma_channel = tmp_chan;
 	}
 	/*
 	 * in the future we might want to know the half-fifo size
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index a48586bb8edbae..dfdc9ee1a6b38c 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -204,6 +204,27 @@ config GPIO_ATH79
 	  Select this option to enable GPIO driver for
 	  Atheros AR71XX/AR724X/AR913X SoC devices.
 
+config GPIO_AXIADO_SGPIO
+	tristate "Axiado AX3005 SGPIO support"
+	depends on ARCH_AXIADO || COMPILE_TEST
+	select GPIOLIB_IRQCHIP
+	select GPIO_REGMAP
+	select REGMAP_MMIO
+	help
+	  Enable support for the Axiado AX3005 Serial GPIO (SGPIO)
+	  controller.
+
+	  The SGPIO controller provides a serialized interface for
+	  controlling multiple GPIO signals over a limited number of
+	  physical lines. Each signal is exposed as one input and one
+	  output line, and input lines can raise interrupts.
+
+	  This driver integrates with the Linux GPIO subsystem and
+	  exposes the controller as a standard GPIO provider.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called gpio-axiado-sgpio.
+
 config GPIO_RASPBERRYPI_EXP
 	tristate "Raspberry Pi 3 GPIO Expander"
 	default RASPBERRYPI_FIRMWARE
@@ -1385,6 +1406,18 @@ config GPIO_BD72720
 	  be configured to GPO on the ROHM PMIC. The pin configuration is done
 	  on OTP at manufacturing.
 
+config GPIO_BD73800
+	tristate "ROHM BD73800 GPIO support"
+	depends on MFD_ROHM_BD71828
+	select GPIO_REGMAP
+	help
+	  Support for GPIOs on ROHM BD73800 PMIC. There can be up to 4
+	  GPI or GPO pins available on the PMIC in total. The purpose of
+	  the pins is decided at the device manufacturing by OTP
+	  configuration and can't be reconfigured later. Enable this
+	  if your PMIC has pins set as GPIs or GPOs and if you wish to
+	  control the pins via the GPIO framework.
+
 config GPIO_BD9571MWV
 	tristate "ROHM BD9571 GPIO support"
 	depends on MFD_BD9571MWV
@@ -1818,6 +1851,18 @@ endmenu
 menu "Auxiliary Bus GPIO drivers"
 	depends on AUXILIARY_BUS
 
+config GPIO_AD7768
+	tristate "Analog Devices AD7768 GPIO support"
+	depends on AD7768
+	depends on GPIOLIB
+	select GPIO_REGMAP
+	help
+	  Say yes here to expose the AD7768 utility pins as GPIOs when the
+	  device tree node is marked as a GPIO controller.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called gpio-ad7768.
+
 config GPIO_LTC4283
 	tristate "Analog Devices LTC4283 GPIO support"
 	depends on SENSORS_LTC4283
@@ -1830,6 +1875,17 @@ config GPIO_LTC4283
 
 endmenu
 
+config GPIO_TC9563
+	tristate "Toshiba TC9563 GPIO support"
+	default m if ARCH_QCOM
+	select AUXILIARY_BUS
+	select GPIO_REGMAP
+	help
+	  This enables support for the GPIO controller embedded in the Toshiba
+	  TC9563 (and Qualcomm QPS615).  This device connects to the host
+	  via PCIe port, which is the upstream port on an internal PCIe
+	  switch.
+
 menu "PCI GPIO expanders"
 	depends on PCI
 
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index dc9e6d643b5bca..88f748846e6051 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_GPIO_104_IDI_48)		+= gpio-104-idi-48.o
 obj-$(CONFIG_GPIO_104_IDIO_16)		+= gpio-104-idio-16.o
 obj-$(CONFIG_GPIO_74X164)		+= gpio-74x164.o
 obj-$(CONFIG_GPIO_74XX_MMIO)		+= gpio-74xx-mmio.o
+obj-$(CONFIG_GPIO_AD7768)		+= gpio-ad7768.o
 obj-$(CONFIG_GPIO_ADNP)			+= gpio-adnp.o
 obj-$(CONFIG_GPIO_ADP5520)		+= gpio-adp5520.o
 obj-$(CONFIG_GPIO_ADP5585)		+= gpio-adp5585.o
@@ -40,11 +41,13 @@ obj-$(CONFIG_GPIO_ARIZONA)		+= gpio-arizona.o
 obj-$(CONFIG_GPIO_ASPEED)		+= gpio-aspeed.o
 obj-$(CONFIG_GPIO_ASPEED_SGPIO)		+= gpio-aspeed-sgpio.o
 obj-$(CONFIG_GPIO_ATH79)		+= gpio-ath79.o
+obj-$(CONFIG_GPIO_AXIADO_SGPIO)		+= gpio-axiado-sgpio.o
 obj-$(CONFIG_GPIO_BCM_KONA)		+= gpio-bcm-kona.o
 obj-$(CONFIG_GPIO_BCM_XGS_IPROC)	+= gpio-xgs-iproc.o
 obj-$(CONFIG_GPIO_BD71815)		+= gpio-bd71815.o
 obj-$(CONFIG_GPIO_BD71828)		+= gpio-bd71828.o
 obj-$(CONFIG_GPIO_BD72720)		+= gpio-bd72720.o
+obj-$(CONFIG_GPIO_BD73800)		+= gpio-bd73800.o
 obj-$(CONFIG_GPIO_BD9571MWV)		+= gpio-bd9571mwv.o
 obj-$(CONFIG_GPIO_BLZP1600)		+= gpio-blzp1600.o
 obj-$(CONFIG_GPIO_BRCMSTB)		+= gpio-brcmstb.o
@@ -182,6 +185,7 @@ obj-$(CONFIG_GPIO_SYSCON)		+= gpio-syscon.o
 obj-$(CONFIG_GPIO_TANGIER)		+= gpio-tangier.o
 obj-$(CONFIG_GPIO_TB10X)		+= gpio-tb10x.o
 obj-$(CONFIG_GPIO_TC3589X)		+= gpio-tc3589x.o
+obj-$(CONFIG_GPIO_TC9563)		+= gpio-tc9563.o
 obj-$(CONFIG_GPIO_TEGRA186)		+= gpio-tegra186.o
 obj-$(CONFIG_GPIO_TEGRA)		+= gpio-tegra.o
 obj-$(CONFIG_GPIO_THUNDERX)		+= gpio-thunderx.o
diff --git a/drivers/gpio/gpio-ad7768.c b/drivers/gpio/gpio-ad7768.c
new file mode 100644
index 00000000000000..c37e1ac306126b
--- /dev/null
+++ b/drivers/gpio/gpio-ad7768.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Analog Devices AD7768 GPIO auxiliary driver
+ *
+ * Copyright 2026 Analog Devices Inc.
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+
+#define AD7768_REG_GPIO_CONTROL		0x0E
+#define   AD7768_GPIO_UGPIO_ENABLE	BIT(7)
+
+#define AD7768_REG_GPIO_WRITE		0x0F
+#define AD7768_REG_GPIO_READ		0x10
+
+#define AD7768_NUM_GPIOS		5
+#define AD7768_FILTER_GPIO		4
+
+static int ad7768_gpio_init_valid_mask(struct gpio_chip *gc,
+				       unsigned long *valid_mask,
+				       unsigned int ngpios)
+{
+	if (ngpios > AD7768_FILTER_GPIO &&
+	    fwnode_property_match_string(gc->fwnode, "clock-names", "mclk") < 0)
+		__clear_bit(AD7768_FILTER_GPIO, valid_mask);
+
+	return 0;
+}
+
+static int ad7768_gpio_reg_mask_xlate(struct gpio_regmap *gpio,
+				      enum gpio_regmap_operation op,
+				      unsigned int base, unsigned int offset,
+				      unsigned int *reg, unsigned int *mask)
+{
+	struct regmap *regmap = gpio_regmap_get_drvdata(gpio);
+	int ret;
+
+	*reg = base;
+	*mask = BIT(offset);
+
+	if (op != GPIO_REGMAP_GET_OP)
+		return 0;
+
+	/*
+	 * AD7768 has separate input-state and output-latch registers. For an
+	 * output line, report the programmed value from the output latch;
+	 * input lines continue to use the input-state register.
+	 */
+	ret = regmap_test_bits(regmap, AD7768_REG_GPIO_CONTROL, *mask);
+	if (ret < 0)
+		return ret;
+	if (ret)
+		*reg = AD7768_REG_GPIO_WRITE;
+
+	return 0;
+}
+
+static int ad7768_gpio_probe(struct auxiliary_device *adev,
+			     const struct auxiliary_device_id *id)
+{
+	struct gpio_regmap_config config;
+	struct device *dev = &adev->dev;
+	struct device *parent = dev->parent;
+	struct regmap *map;
+	int ret;
+
+	map = dev_get_regmap(parent, NULL);
+	if (!map)
+		return -ENODEV;
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(parent, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = regmap_set_bits(map, AD7768_REG_GPIO_CONTROL, AD7768_GPIO_UGPIO_ENABLE);
+	if (ret)
+		return ret;
+
+	config = (struct gpio_regmap_config) {
+		.parent = dev,
+		.regmap = map,
+		.label = dev_name(parent),
+		.ngpio = AD7768_NUM_GPIOS,
+		.reg_dat_base = AD7768_REG_GPIO_READ,
+		.reg_set_base = AD7768_REG_GPIO_WRITE,
+		.reg_dir_out_base = AD7768_REG_GPIO_CONTROL,
+		.pm_dev = parent,
+		.fwnode = dev_fwnode(parent),
+		.reg_mask_xlate = ad7768_gpio_reg_mask_xlate,
+		.init_valid_mask = ad7768_gpio_init_valid_mask,
+		.drvdata = map,
+	};
+
+	return PTR_ERR_OR_ZERO(devm_gpio_regmap_register(dev, &config));
+}
+
+static const struct auxiliary_device_id ad7768_gpio_ids[] = {
+	{ .name = "ad7768.gpio" },
+	{ }
+};
+MODULE_DEVICE_TABLE(auxiliary, ad7768_gpio_ids);
+
+static struct auxiliary_driver ad7768_gpio_driver = {
+	.probe = ad7768_gpio_probe,
+	.id_table = ad7768_gpio_ids,
+};
+module_auxiliary_driver(ad7768_gpio_driver);
+
+MODULE_AUTHOR("Janani Sunil <janani.sunil@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD7768 GPIO auxiliary driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-adp5585.c b/drivers/gpio/gpio-adp5585.c
index 6f10fc64600805..7c04a7e86c8f8c 100644
--- a/drivers/gpio/gpio-adp5585.c
+++ b/drivers/gpio/gpio-adp5585.c
@@ -390,16 +390,6 @@ static const struct irq_chip adp5585_irq_chip = {
 	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
-static void adp5585_gpio_unreg_notifier(void *data)
-{
-	struct adp5585_gpio_dev *adp5585_gpio = data;
-	struct device *dev = adp5585_gpio->gpio_chip.parent;
-	struct adp5585_dev *adp5585 = dev_get_drvdata(dev->parent);
-
-	blocking_notifier_chain_unregister(&adp5585->event_notifier,
-					   &adp5585_gpio->nb);
-}
-
 static int adp5585_gpio_probe(struct platform_device *pdev)
 {
 	struct adp5585_dev *adp5585 = dev_get_drvdata(pdev->dev.parent);
@@ -450,13 +440,9 @@ static int adp5585_gpio_probe(struct platform_device *pdev)
 		girq->threaded = true;
 
 		adp5585_gpio->nb.notifier_call = adp5585_gpio_key_event;
-		ret = blocking_notifier_chain_register(&adp5585->event_notifier,
-						       &adp5585_gpio->nb);
-		if (ret)
-			return ret;
-
-		ret = devm_add_action_or_reset(dev, adp5585_gpio_unreg_notifier,
-					       adp5585_gpio);
+		ret = devm_blocking_notifier_chain_register(dev,
+							    &adp5585->event_notifier,
+							    &adp5585_gpio->nb);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index 532e3360b70ea0..a6ac9356229cdc 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -220,8 +220,11 @@ static int altera_gpio_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	mapped_irq = platform_get_irq_optional(pdev, 0);
-	if (mapped_irq < 0)
+	if (mapped_irq < 0) {
+		if (mapped_irq != -ENXIO)
+			return mapped_irq;
 		goto skip_irq;
+	}
 
 	if (device_property_read_u32(dev, "altr,interrupt-type", &reg)) {
 		dev_err(&pdev->dev,
diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c
index a7e98d395d8e5c..88cf013f0d90de 100644
--- a/drivers/gpio/gpio-arizona.c
+++ b/drivers/gpio/gpio-arizona.c
@@ -115,8 +115,12 @@ static int arizona_gpio_direction_out(struct gpio_chip *chip,
 	if (value)
 		value = ARIZONA_GPN_LVL;
 
-	return regmap_update_bits(arizona->regmap, ARIZONA_GPIO1_CTRL + offset,
-				  ARIZONA_GPN_DIR | ARIZONA_GPN_LVL, value);
+	ret = regmap_update_bits(arizona->regmap, ARIZONA_GPIO1_CTRL + offset,
+				 ARIZONA_GPN_DIR | ARIZONA_GPN_LVL, value);
+	if (ret < 0 && (val & ARIZONA_GPN_DIR) && persistent)
+		pm_runtime_put_autosuspend(chip->parent);
+
+	return ret;
 }
 
 static int arizona_gpio_set(struct gpio_chip *chip, unsigned int offset,
diff --git a/drivers/gpio/gpio-axiado-sgpio.c b/drivers/gpio/gpio-axiado-sgpio.c
new file mode 100644
index 00000000000000..b57a5074ccef39
--- /dev/null
+++ b/drivers/gpio/gpio-axiado-sgpio.c
@@ -0,0 +1,736 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2022-2026 Axiado Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/bitops.h>
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdesc.h>
+#include <linux/irqdomain.h>
+#include <linux/math.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+struct axiado_sgpio_reg_offsets {
+	u32 mux_0;
+	u32 preset_0;
+	u32 count_0;
+	u32 pos_0;
+
+	u32 mux_1;
+	u32 ld;
+	u32 ld_ss;
+
+	u32 preset_1;
+	u32 count_1;
+	u32 pos_1;
+
+	u32 mux_2;
+	u32 dout;
+	u32 dout_ss;
+
+	u32 preset_2;
+	u32 count_2;
+	u32 pos_2;
+
+	u32 mux_3;
+	u32 preset_3;
+	u32 count_3;
+	u32 pos_3;
+
+	u32 mux_4;
+	u32 oe;
+	u32 oe_ss;
+
+	u32 preset_4;
+	u32 count_4;
+	u32 pos_4;
+
+	u32 mask;
+	u32 ctrl_en;
+	u32 ctrl_en_pos;
+
+	u32 din_ss;
+	u32 status;
+};
+
+static const struct axiado_sgpio_reg_offsets axiado_sgpio_offsets = {
+	.mux_0 = 0x000,
+	.preset_0 = 0x1dc,
+	.count_0 = 0x1f0,
+	.pos_0 = 0x204,
+
+	.mux_1 = 0x004,
+	.ld = 0x014,
+	.ld_ss = 0x0d8,
+
+	.preset_1 = 0x1e0,
+	.count_1 = 0x1f4,
+	.pos_1 = 0x208,
+
+	.mux_2 = 0x008,
+	.dout = 0x054,
+	.dout_ss = 0x158,
+
+	.preset_2 = 0x1e4,
+	.count_2 = 0x1f8,
+	.pos_2 = 0x20c,
+
+	.mux_3 = 0x00c,
+	.preset_3 = 0x1e8,
+	.count_3 = 0x1fc,
+	.pos_3 = 0x210,
+
+	.mux_4 = 0x010,
+	.oe = 0x0d4,
+	.oe_ss = 0x1d8,
+
+	.preset_4 = 0x1ec,
+	.count_4 = 0x200,
+	.pos_4 = 0x214,
+
+	.mask = 0x224,
+	.ctrl_en = 0x218,
+	.ctrl_en_pos = 0x21c,
+
+	.din_ss = 0x198,
+	.status = 0x228,
+};
+
+/* Number of SGPIO positions held by one bank register. */
+#define SGPIO_BANK_SIZE			32
+#define SGPIO_MAX_SIGNALS		512
+#define SGPIO_NUM_BANKS			16
+
+/*
+ * Slice mux encodings taken from the AX3005 SGPIO programming sequence.
+ */
+#define SGPIO_MUX_CLK			0x306
+#define SGPIO_MUX_DATA			0x30c
+#define SGPIO_MUX_DIN			0x14c
+#define SGPIO_MUX_OE			0x10c
+
+/* A POS register holds the first and the last position driven by the slice. */
+#define SGPIO_POS_FIRST			GENMASK(31, 16)
+#define SGPIO_POS_LAST			GENMASK(15, 0)
+#define SGPIO_POS(pos)			(FIELD_PREP(SGPIO_POS_FIRST, (pos)) | \
+					 FIELD_PREP(SGPIO_POS_LAST, (pos)))
+
+/* The clock and output enable slices always cover a full bank. */
+#define SGPIO_POS_FULL_BANK		SGPIO_POS(SGPIO_BANK_SIZE - 1)
+
+/* The enable and interrupt registers hold one bit per slice. */
+#define SGPIO_SLICE_MASK		GENMASK(15, 0)
+
+/*
+ * Interrupt sources enabled by the programming sequence. Bit 13 is left
+ * masked, the remaining slice events are all reported.
+ */
+#define SGPIO_IRQ_ENABLE		0xdfff
+
+/* The upper half of the status register reports one bit per data bank. */
+#define SGPIO_STATUS_EXCHANGE		GENMASK(31, 16)
+
+struct axiado_sgpio {
+	u32 preset_value;
+	u32 count_value;
+	u32 pos_reg;
+	/* Last sampled input word of each bank, for edge detection. */
+	u32 *din_shadow;
+	unsigned long *irq_unmasked;
+	unsigned long *irq_rising;
+	unsigned long *irq_falling;
+	u32 nsignals;
+	u32 din_bank_shift;
+	unsigned long *dir_out;
+	struct gpio_regmap *gpio;
+	struct irq_domain *domain;
+	struct regmap *regmap;
+	const struct axiado_sgpio_reg_offsets *regs;
+};
+
+/*
+ * Tell lockdep that these interrupts are in a different category than the
+ * parent they are handled from, so that the nested handling is not reported
+ * as recursion.
+ */
+static struct lock_class_key axiado_sgpio_irq_lock_class;
+static struct lock_class_key axiado_sgpio_irq_request_class;
+
+/*
+ * Each SGPIO signal is exposed as two GPIO lines: an even line for the serial
+ * input and the following odd line for the serial output. Signal n therefore
+ * lives in bit (n % 32) of bank (n / 32) of either the input or the output
+ * register.
+ */
+static int axiado_sgpio_reg_mask_xlate(struct gpio_regmap *gpio,
+				       enum gpio_regmap_operation op,
+				       unsigned int base, unsigned int offset,
+				       unsigned int *reg, unsigned int *mask)
+{
+	struct axiado_sgpio *sgpio = gpio_regmap_get_drvdata(gpio);
+	unsigned int signal = offset / 2;
+	unsigned int bank = signal / SGPIO_BANK_SIZE;
+
+	/*
+	 * An output line lives in the output register for both reads and
+	 * writes, so the register follows from the line rather than from the
+	 * operation. The AX3005 input banks are right aligned, hence the
+	 * shift.
+	 */
+	if (offset % 2)
+		base = sgpio->regs->dout_ss;
+	else
+		bank += sgpio->din_bank_shift;
+
+	*reg = base + bank * sizeof(u32);
+	*mask = BIT(signal % SGPIO_BANK_SIZE);
+
+	return 0;
+}
+
+static irqreturn_t axiado_sgpio_irq_handler(int irq, void *arg)
+{
+	struct axiado_sgpio *sgpio = arg;
+	unsigned int nbanks = DIV_ROUND_UP(sgpio->nsignals, SGPIO_BANK_SIZE);
+	u32 status, new_value, changed_value;
+	unsigned int bit, reg_ptr, i;
+	int ret;
+
+	/* Read-on-clear (ACK) parent cause */
+	ret = regmap_read(sgpio->regmap, sgpio->regs->status, &status);
+	if (ret)
+		return IRQ_NONE;
+
+	/* Nothing pending: the interrupt came from somebody else. */
+	if (!status)
+		return IRQ_NONE;
+
+	status = FIELD_GET(SGPIO_STATUS_EXCHANGE, status);
+
+	reg_ptr = sgpio->din_bank_shift;
+
+	for (i = 0; i < nbanks; i++, reg_ptr++) {
+		if (status & BIT(reg_ptr)) {
+			ret = regmap_read(sgpio->regmap,
+					  sgpio->regs->din_ss + reg_ptr * sizeof(u32),
+					  &new_value);
+			if (ret)
+				continue;
+
+			/*
+			 * Track every signal so that the shadow always holds
+			 * the last sampled level, then report only the ones
+			 * whose interrupt is unmasked. Transitions of a masked
+			 * signal are dropped rather than delivered later, the
+			 * hardware has no per signal latch to replay them
+			 * from. Only this handler touches the shadow and it is
+			 * not reentrant.
+			 */
+			changed_value = sgpio->din_shadow[i] ^ new_value;
+			sgpio->din_shadow[i] = new_value;
+			changed_value &= bitmap_read(sgpio->irq_unmasked,
+						     i * SGPIO_BANK_SIZE,
+						     SGPIO_BANK_SIZE);
+
+			while (changed_value) {
+				unsigned int child_irq;
+				irq_hw_number_t hwirq;
+				unsigned int position;
+				bool rising;
+
+				bit = __ffs(changed_value);
+				changed_value &= ~BIT(bit);
+
+				position = i * SGPIO_BANK_SIZE + bit;
+				hwirq = position * 2;
+
+				rising = !!(new_value & BIT(bit));
+
+				if (!test_bit(position, rising ? sgpio->irq_rising
+							       : sgpio->irq_falling))
+					continue;
+
+				child_irq = irq_find_mapping(sgpio->domain, hwirq);
+				if (child_irq)
+					handle_nested_irq(child_irq);
+			}
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int axiado_sgpio_hw_init(struct axiado_sgpio *sgpio)
+{
+	const struct axiado_sgpio_reg_offsets *regs = sgpio->regs;
+	unsigned int nbanks = DIV_ROUND_UP(sgpio->nsignals, SGPIO_BANK_SIZE);
+	unsigned int last_bank = (sgpio->nsignals - 1) / SGPIO_BANK_SIZE;
+	unsigned int last_bit = (sgpio->nsignals - 1) % SGPIO_BANK_SIZE;
+	unsigned int i;
+	int ret;
+
+	/* Slice A0 is the shift clock, slice B1 the data load. */
+	const struct reg_sequence pre_load[] = {
+		{ regs->mask, 0 },
+		{ regs->mux_0, SGPIO_MUX_CLK },
+		{ regs->preset_0, sgpio->preset_value },
+		{ regs->count_0, sgpio->count_value },
+		{ regs->pos_0, SGPIO_POS_FULL_BANK },
+		{ regs->mux_1, SGPIO_MUX_DATA },
+	};
+	/* Slice C2 is the data output. */
+	const struct reg_sequence pre_dout[] = {
+		{ regs->preset_1, sgpio->preset_value },
+		{ regs->count_1, sgpio->count_value },
+		{ regs->pos_1, sgpio->pos_reg },
+		{ regs->mux_2, SGPIO_MUX_DATA },
+	};
+	/* Slice D3 is the data input, slice E4 the output enable. */
+	const struct reg_sequence post_dout[] = {
+		{ regs->preset_2, sgpio->preset_value },
+		{ regs->count_2, sgpio->count_value },
+		{ regs->pos_2, sgpio->pos_reg },
+		{ regs->mux_3, SGPIO_MUX_DIN },
+		{ regs->preset_3, sgpio->preset_value },
+		{ regs->count_3, sgpio->count_value },
+		{ regs->pos_3, sgpio->pos_reg },
+		{ regs->mux_4, SGPIO_MUX_OE },
+		{ regs->oe, GENMASK(31, 0) },
+		{ regs->oe_ss, GENMASK(31, 0) },
+		{ regs->preset_4, sgpio->preset_value },
+		{ regs->count_4, sgpio->count_value },
+		{ regs->pos_4, SGPIO_POS_FULL_BANK },
+		{ regs->ctrl_en, SGPIO_SLICE_MASK },
+		{ regs->ctrl_en_pos, SGPIO_SLICE_MASK },
+	};
+
+	ret = regmap_multi_reg_write(sgpio->regmap, pre_load,
+				     ARRAY_SIZE(pre_load));
+	if (ret)
+		return ret;
+
+	/*
+	 * Latch the signals that are present. Only the last bank may be
+	 * partially populated; a last bit of zero still describes one signal.
+	 */
+	for (i = 0; i < nbanks; i++) {
+		u32 val = i < last_bank ? GENMASK(SGPIO_BANK_SIZE - 1, 0) :
+					  GENMASK(last_bit, 0);
+
+		ret = regmap_write(sgpio->regmap, regs->ld + i * 4, val);
+		if (ret)
+			return ret;
+
+		ret = regmap_write(sgpio->regmap, regs->ld_ss + i * 4, val);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_multi_reg_write(sgpio->regmap, pre_dout,
+				     ARRAY_SIZE(pre_dout));
+	if (ret)
+		return ret;
+
+	/* All outputs start out driving zero. */
+	for (i = 0; i < nbanks; i++) {
+		ret = regmap_write(sgpio->regmap, regs->dout + i * 4, 0);
+		if (ret)
+			return ret;
+
+		ret = regmap_write(sgpio->regmap, regs->dout_ss + i * 4, 0);
+		if (ret)
+			return ret;
+	}
+
+	return regmap_multi_reg_write(sgpio->regmap, post_dout,
+				      ARRAY_SIZE(post_dout));
+}
+
+static int axiado_sgpio_irq_enable(struct axiado_sgpio *sgpio)
+{
+	return regmap_write(sgpio->regmap, sgpio->regs->mask,
+			    SGPIO_IRQ_ENABLE);
+}
+
+static void axiado_sgpio_disable(void *data)
+{
+	struct axiado_sgpio *sgpio = data;
+
+	/* Mask the interrupts and stop the shift engine. */
+	regmap_write(sgpio->regmap, sgpio->regs->mask, 0);
+	regmap_write(sgpio->regmap, sgpio->regs->ctrl_en, 0);
+}
+
+static int axiado_sgpio_set_irq_type(struct irq_data *d, unsigned int type)
+{
+	struct axiado_sgpio *sgpio = irq_data_get_irq_chip_data(d);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	unsigned int position;
+
+	/* Only even GPIO offsets represent SGPIO inputs. */
+	if (hwirq & 1)
+		return -EINVAL;
+
+	position = hwirq / 2;
+	if (position >= sgpio->nsignals)
+		return -EINVAL;
+
+	type &= IRQ_TYPE_SENSE_MASK;
+
+	switch (type) {
+	case IRQ_TYPE_EDGE_BOTH:
+	case IRQ_TYPE_EDGE_RISING:
+	case IRQ_TYPE_EDGE_FALLING:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	assign_bit(position, sgpio->irq_rising, type & IRQ_TYPE_EDGE_RISING);
+	assign_bit(position, sgpio->irq_falling, type & IRQ_TYPE_EDGE_FALLING);
+
+	return 0;
+}
+
+static void axiado_sgpio_mask_irq(struct irq_data *d)
+{
+	struct axiado_sgpio *sgpio = irq_data_get_irq_chip_data(d);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+
+	clear_bit(hwirq / 2, sgpio->irq_unmasked);
+	gpio_regmap_disable_irq(sgpio->gpio, hwirq);
+}
+
+static void axiado_sgpio_unmask_irq(struct irq_data *d)
+{
+	struct axiado_sgpio *sgpio = irq_data_get_irq_chip_data(d);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+
+	gpio_regmap_enable_irq(sgpio->gpio, hwirq);
+	set_bit(hwirq / 2, sgpio->irq_unmasked);
+}
+
+static int axiado_sgpio_irq_request_resources(struct irq_data *d)
+{
+	struct axiado_sgpio *sgpio = irq_data_get_irq_chip_data(d);
+
+	return gpio_regmap_reqres_irq(sgpio->gpio, d->hwirq);
+}
+
+static void axiado_sgpio_irq_release_resources(struct irq_data *d)
+{
+	struct axiado_sgpio *sgpio = irq_data_get_irq_chip_data(d);
+
+	gpio_regmap_relres_irq(sgpio->gpio, d->hwirq);
+}
+
+static const struct irq_chip axiado_sgpio_irqchip = {
+	.name = "axiado-sgpio",
+	.irq_mask = axiado_sgpio_mask_irq,
+	.irq_unmask = axiado_sgpio_unmask_irq,
+	.irq_set_type = axiado_sgpio_set_irq_type,
+	.flags = IRQCHIP_IMMUTABLE | IRQCHIP_MASK_ON_SUSPEND,
+	.irq_request_resources = axiado_sgpio_irq_request_resources,
+	.irq_release_resources = axiado_sgpio_irq_release_resources,
+};
+
+static int axiado_sgpio_irq_map(struct irq_domain *domain, unsigned int virq,
+				irq_hw_number_t hwirq)
+{
+	struct axiado_sgpio *sgpio = domain->host_data;
+
+	/* Only even offsets represent input GPIOs. */
+	if (hwirq % 2)
+		return -EINVAL;
+
+	irq_set_chip_data(virq, sgpio);
+	irq_set_lockdep_class(virq, &axiado_sgpio_irq_lock_class,
+			      &axiado_sgpio_irq_request_class);
+	irq_set_chip_and_handler(virq, &axiado_sgpio_irqchip, handle_simple_irq);
+	irq_set_nested_thread(virq, true);
+	irq_set_noprobe(virq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops axiado_sgpio_domain_ops = {
+	.map = axiado_sgpio_irq_map,
+	.xlate = irq_domain_xlate_twocell,
+};
+
+static int axiado_sgpio_init_input_cache(struct axiado_sgpio *sgpio)
+{
+	unsigned int num_banks = DIV_ROUND_UP(sgpio->nsignals,
+					      SGPIO_BANK_SIZE);
+	unsigned int reg_ptr = sgpio->din_bank_shift;
+	unsigned int i;
+	int ret;
+
+	for (i = 0; i < num_banks; i++, reg_ptr++) {
+		ret = regmap_read(sgpio->regmap,
+				  sgpio->regs->din_ss + reg_ptr * sizeof(u32),
+				  &sgpio->din_shadow[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static bool axiado_sgpio_dout_reg(const struct axiado_sgpio *sgpio,
+				  unsigned int reg)
+{
+	unsigned int span = SGPIO_NUM_BANKS * sizeof(u32);
+
+	return reg >= sgpio->regs->dout_ss &&
+	       reg < sgpio->regs->dout_ss + span;
+}
+
+static bool axiado_sgpio_readable_reg(struct device *dev, unsigned int reg)
+{
+	const struct axiado_sgpio *sgpio = dev_get_drvdata(dev);
+
+	/* The status register is the last one of the block. */
+	if (reg > sgpio->regs->status)
+		return false;
+
+	/*
+	 * The output shadow feeds the shift register and cannot be read back,
+	 * the driven value is kept in the register cache instead.
+	 */
+	return !axiado_sgpio_dout_reg(sgpio, reg);
+}
+
+static bool axiado_sgpio_volatile_reg(struct device *dev, unsigned int reg)
+{
+	const struct axiado_sgpio *sgpio = dev_get_drvdata(dev);
+
+	/*
+	 * Everything but the output shadow is live hardware state: shifted in
+	 * data, running counters and the interrupt status. Caching any of it
+	 * would hand out stale values, so the cache holds the output banks
+	 * alone.
+	 */
+	return !axiado_sgpio_dout_reg(sgpio, reg);
+}
+
+static bool axiado_sgpio_precious_reg(struct device *dev, unsigned int reg)
+{
+	const struct axiado_sgpio *sgpio = dev_get_drvdata(dev);
+
+	/* Reading the status register acknowledges the interrupts. */
+	return reg == sgpio->regs->status;
+}
+
+static const struct regmap_config axiado_sgpio_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.readable_reg = axiado_sgpio_readable_reg,
+	.volatile_reg = axiado_sgpio_volatile_reg,
+	.precious_reg = axiado_sgpio_precious_reg,
+	.cache_type = REGCACHE_MAPLE,
+};
+
+static int axiado_sgpio_probe(struct platform_device *pdev)
+{
+	struct gpio_regmap_config config = { };
+	struct irq_domain_info d_info = { };
+	struct axiado_sgpio *sgpio;
+	unsigned int ngpio, i;
+	unsigned long apb_freq;
+	struct clk *apb_clk;
+	void __iomem *base;
+	u32 sgpio_freq;
+	int irq, rc;
+
+	sgpio = devm_kzalloc(&pdev->dev, sizeof(*sgpio), GFP_KERNEL);
+	if (!sgpio)
+		return -ENOMEM;
+
+	/* The regmap callbacks below need both of these. */
+	platform_set_drvdata(pdev, sgpio);
+
+	sgpio->regs = &axiado_sgpio_offsets;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	sgpio->regmap = devm_regmap_init_mmio(&pdev->dev, base,
+					      &axiado_sgpio_regmap_config);
+	if (IS_ERR(sgpio->regmap))
+		return dev_err_probe(&pdev->dev, PTR_ERR(sgpio->regmap),
+				     "Failed to init regmap\n");
+
+	rc = device_property_read_u32(&pdev->dev, "ngpios", &ngpio);
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc,
+				     "Failed to read ngpios property\n");
+
+	/*
+	 * Each SGPIO signal is exposed as one input and one output line, so
+	 * the number of lines is twice the number of signals.
+	 */
+	if (!ngpio || ngpio % 2 || ngpio > SGPIO_MAX_SIGNALS * 2)
+		return dev_err_probe(&pdev->dev, -EINVAL,
+				     "Invalid ngpios value: %u (even, max %u)\n",
+				     ngpio, SGPIO_MAX_SIGNALS * 2);
+
+	sgpio->nsignals = ngpio / 2;
+
+	sgpio->din_shadow = devm_kcalloc(&pdev->dev,
+					 DIV_ROUND_UP(sgpio->nsignals,
+						      SGPIO_BANK_SIZE),
+					 sizeof(*sgpio->din_shadow), GFP_KERNEL);
+	sgpio->irq_unmasked = devm_bitmap_zalloc(&pdev->dev, sgpio->nsignals,
+						 GFP_KERNEL);
+	sgpio->irq_rising = devm_bitmap_zalloc(&pdev->dev, sgpio->nsignals,
+					       GFP_KERNEL);
+	sgpio->irq_falling = devm_bitmap_zalloc(&pdev->dev, sgpio->nsignals,
+						GFP_KERNEL);
+	if (!sgpio->din_shadow || !sgpio->irq_unmasked ||
+	    !sgpio->irq_rising || !sgpio->irq_falling)
+		return -ENOMEM;
+
+	apb_clk = devm_clk_get_enabled(&pdev->dev, NULL);
+	if (IS_ERR(apb_clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(apb_clk),
+				     "Failed to get and enable APB clock\n");
+
+	rc = device_property_read_u32(&pdev->dev, "bus-frequency",
+				      &sgpio_freq);
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc,
+				     "Failed to read bus-frequency\n");
+
+	apb_freq = clk_get_rate(apb_clk);
+
+	if (!apb_freq || !sgpio_freq || sgpio_freq > apb_freq)
+		return dev_err_probe(&pdev->dev, -EINVAL,
+				     "Invalid SGPIO bus frequency\n");
+
+	sgpio->preset_value = (apb_freq / sgpio_freq) - 1;
+	sgpio->count_value = sgpio->preset_value;
+
+	sgpio->pos_reg = SGPIO_POS(sgpio->nsignals - 1);
+
+	sgpio->din_bank_shift =	SGPIO_NUM_BANKS -
+				DIV_ROUND_UP(sgpio->nsignals, SGPIO_BANK_SIZE);
+
+	/*
+	 * From here on the hardware may be running, so the shutdown action has
+	 * to be in place before the first register is programmed. It only ever
+	 * writes zeroes, which is harmless if initialisation never got that
+	 * far, and it runs before the APB clock is disabled because that was
+	 * requested earlier.
+	 */
+	rc = devm_add_action_or_reset(&pdev->dev, axiado_sgpio_disable, sgpio);
+	if (rc)
+		return rc;
+
+	rc = axiado_sgpio_hw_init(sgpio);
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc,
+				     "Failed to initialize hardware\n");
+
+	rc = axiado_sgpio_init_input_cache(sgpio);
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc,
+				     "Failed to initialize input cache\n");
+
+	sgpio->dir_out = devm_bitmap_zalloc(&pdev->dev, ngpio, GFP_KERNEL);
+	if (!sgpio->dir_out)
+		return -ENOMEM;
+
+	/* Even lines are the serial inputs, odd lines the serial outputs. */
+	for (i = 1; i < ngpio; i += 2)
+		__set_bit(i, sgpio->dir_out);
+
+	d_info.fwnode = dev_fwnode(&pdev->dev);
+	d_info.size = ngpio;
+	d_info.hwirq_max = ngpio;
+	d_info.ops = &axiado_sgpio_domain_ops;
+	d_info.host_data = sgpio;
+
+	sgpio->domain = devm_irq_domain_instantiate(&pdev->dev, &d_info);
+	if (IS_ERR(sgpio->domain))
+		return PTR_ERR(sgpio->domain);
+
+	config.parent = &pdev->dev;
+	config.regmap = sgpio->regmap;
+	config.ngpio = ngpio;
+	config.reg_dat_base = GPIO_REGMAP_ADDR(sgpio->regs->din_ss);
+	config.reg_set_base = GPIO_REGMAP_ADDR(sgpio->regs->dout_ss);
+	config.reg_mask_xlate = axiado_sgpio_reg_mask_xlate;
+	config.fixed_direction_output = sgpio->dir_out;
+	config.irq_domain = sgpio->domain;
+	config.drvdata = sgpio;
+
+	sgpio->gpio = devm_gpio_regmap_register(&pdev->dev, &config);
+	if (IS_ERR(sgpio->gpio))
+		return dev_err_probe(&pdev->dev, PTR_ERR(sgpio->gpio),
+				     "Could not register gpiochip\n");
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	rc = devm_request_threaded_irq(&pdev->dev, irq, NULL, axiado_sgpio_irq_handler,
+				       IRQF_ONESHOT, dev_name(&pdev->dev), sgpio);
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc, "Failed to request IRQ\n");
+
+	rc = axiado_sgpio_irq_enable(sgpio);
+	if (rc)
+		return dev_err_probe(&pdev->dev, rc,
+				     "Failed to enable interrupts\n");
+
+	return 0;
+}
+
+static void axiado_sgpio_remove(struct platform_device *pdev)
+{
+	struct axiado_sgpio *sgpio = platform_get_drvdata(pdev);
+
+	/*
+	 * Runs before the devres unwind, so the interrupts are masked while the
+	 * parent interrupt is still requested. The shift engine is stopped by
+	 * axiado_sgpio_disable(), registered before the hardware was programmed
+	 * so that it also covers failures during probe.
+	 */
+	regmap_write(sgpio->regmap, sgpio->regs->mask, 0);
+}
+
+static const struct of_device_id axiado_sgpio_of_match[] = {
+	{ .compatible = "axiado,ax3005-sgpio" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, axiado_sgpio_of_match);
+
+static struct platform_driver axiado_sgpio_driver = {
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = axiado_sgpio_of_match,
+	},
+	.probe = axiado_sgpio_probe,
+	.remove = axiado_sgpio_remove,
+};
+module_platform_driver(axiado_sgpio_driver);
+
+MODULE_DESCRIPTION("Axiado AX3005 Serial GPIO Driver");
+MODULE_AUTHOR("Axiado Corporation");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-bd73800.c b/drivers/gpio/gpio-bd73800.c
new file mode 100644
index 00000000000000..2be2a885820e25
--- /dev/null
+++ b/drivers/gpio/gpio-bd73800.c
@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Support to GPIOs on ROHM BD73800
+ * Copyright 2024 ROHM Semiconductors.
+ * Author: Matti Vaittinen <mazziesaccount@gmail.com>
+ */
+
+#include <linux/bitmap.h>
+#include <linux/bitops.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/mfd/rohm-bd73800.h>
+
+#define BD73800_GPIO_MAX_PINS 4
+/*
+ * The BD73800 has several "one time programmable" (OTP) configurations which
+ * can be set at manufacturing phase. Some of these options allow using
+ * individual pins as GPI or GPO (not both at the same time). The OTP
+ * configuration can't be read at run-time, so drivers rely on device-tree to
+ * advertise the OTP programmed in manufacturing.
+ *
+ * The pins which can be used as GPIO are:
+ * GPIO1, CLKOUT (GPIO2), FAULT_B, EXTEN_OUT.
+ *
+ * The OTP options 2 and 3 state for all the pins:
+ *  - OTP2: GPI (also IRQ source)
+ *  - OTP3: GPO (NOTE: This is actually 2 different OTP options. Either a
+ *    register controllable output or a power-sequence controlled output.
+ *    The "gpo" referred here means only the register controllable output.)
+ *    The datasheet refers to this as: "<pin> output is controlled by
+ *    GPIO<N>_OUT or power on/off sequencer to control external VRs. ON/OFF
+ *    sequence timing is configurable."
+ *
+ * The data-sheet further says that the GPI/GPO is not a default OTP
+ * configuration for any of the pins. Hence the GPIO driver defaults to a pin
+ * not being a GPI or GPO, but requires the pin to be explicitly marked as a
+ * GPI or GPO in the device-tree.
+ *
+ * DT properties:
+ * "rohm,pin-gpio1", "rohm,pin-clkout", "rohm,pin-fault_b", "rohm,pin-exten"
+ * can be set to one of the values "gpi" or "gpo" to enable them to be used as
+ * GPIO.
+ *
+ * The amount of GPIO lines the chip exposes to the user-space (chip.ngpio) is
+ * always the same, regardless of the OTP variant in use. The lines which are
+ * not usable as GPIO on a given OTP variant are marked invalid via the GPIO
+ * valid_mask. This way the user-space always sees a constant amount of GPIO
+ * lines, where numbering (relative to the chip's base) stays also the same.
+ * Users can then use the valid_mask to find out which of the lines are usable.
+ */
+
+struct bd73800_gpio {
+	/* dev points to the platform device for devm and prints */
+	struct device *dev;
+	struct regmap *regmap;
+	/* Pins which have been OTP configured as GPI or GPO */
+	DECLARE_BITMAP(valid_mask, BD73800_GPIO_MAX_PINS);
+	/* Subset of valid_mask - pins which have been OTP configured as GPO */
+	DECLARE_BITMAP(output_mask, BD73800_GPIO_MAX_PINS);
+};
+
+static const char * const bd73800_gpio_properties[BD73800_GPIO_MAX_PINS] = {
+	"rohm,pin-gpio1",
+	"rohm,pin-clkout",
+	"rohm,pin-fault_b",
+	"rohm,pin-exten"
+};
+
+static int bd73800_gpio_get_pins(struct bd73800_gpio *data)
+{
+	struct device *parent = data->dev->parent;
+	const char *val;
+	int i, ret;
+
+	for (i = 0; i < BD73800_GPIO_MAX_PINS; i++) {
+		ret = device_property_read_string(parent,
+						  bd73800_gpio_properties[i],
+						  &val);
+		if (ret) {
+			if (ret == -EINVAL)
+				continue;
+
+			return dev_err_probe(data->dev, ret,
+					"pin %d (%s), bad configuration\n", i,
+					bd73800_gpio_properties[i]);
+		}
+
+		if (!strcmp(val, "gpi")) {
+			__set_bit(i, data->valid_mask);
+		} else if (!strcmp(val, "gpo")) {
+			__set_bit(i, data->valid_mask);
+			__set_bit(i, data->output_mask);
+		} else {
+			dev_warn(data->dev,
+				 "pin %d (%s), unknown value '%s' ignored\n", i,
+				 bd73800_gpio_properties[i], val);
+		}
+	}
+
+	return 0;
+}
+
+static int bd73800_gpio_init_valid_mask(struct gpio_chip *gc,
+					unsigned long *valid_mask,
+					unsigned int ngpios)
+{
+	struct gpio_regmap *gpio = gpiochip_get_data(gc);
+	struct bd73800_gpio *data = gpio_regmap_get_drvdata(gpio);
+
+	bitmap_copy(valid_mask, data->valid_mask, ngpios);
+
+	return 0;
+}
+
+/*
+ * The used register depends on OTP:
+ *  - If OTP has set pin as GPO, only the GPO_OUT register is valid.
+ *  - If OTP has set pin as GPI, only the INT_5_SRC register is valid.
+ */
+static int bd73800_gpio_reg_mask_xlate(struct gpio_regmap *gpio,
+				       enum gpio_regmap_operation op,
+				       unsigned int base, unsigned int offset,
+				       unsigned int *reg, unsigned int *mask)
+{
+	struct bd73800_gpio *data = gpio_regmap_get_drvdata(gpio);
+	bool is_output = test_bit(offset, data->output_mask);
+
+	if (is_output)
+		*reg = BD73800_REG_GPO_OUT;
+	else
+		*reg = base;
+
+	*mask = BIT(offset);
+
+	return 0;
+}
+
+static int gpo_bd73800_probe(struct platform_device *pdev)
+{
+	struct gpio_regmap_config config = { };
+	struct bd73800_gpio *data;
+	struct device *parent, *dev;
+	int ret;
+
+	dev = &pdev->dev;
+	/* The device-tree and regmap come from MFD => use parent for that */
+	parent = dev->parent;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->dev = dev;
+	data->regmap = dev_get_regmap(parent, NULL);
+	if (!data->regmap)
+		return dev_err_probe(dev, -ENODEV, "no parent regmap\n");
+
+	ret = bd73800_gpio_get_pins(data);
+	if (ret)
+		return ret;
+
+	if (bitmap_empty(data->valid_mask, BD73800_GPIO_MAX_PINS)) {
+		/*
+		 * The BD73800 may or may not have pins allocated for GPIO
+		 * depending on the OTP used at manufacturing.
+		 * If there are no pins, then we have nothing to do.
+		 */
+		dev_dbg(dev, "no GPIO pins\n");
+		return -ENODEV;
+	}
+
+	config.parent = parent;
+	config.regmap = data->regmap;
+	config.label = "bd73800";
+	config.ngpio = BD73800_GPIO_MAX_PINS;
+	config.reg_dat_base = BD73800_REG_INT_5_SRC;
+	config.reg_set_base = BD73800_REG_GPO_OUT;
+	config.reg_mask_xlate = bd73800_gpio_reg_mask_xlate;
+	config.init_valid_mask = bd73800_gpio_init_valid_mask;
+	/* All pins that are valid GPIO lines also have a fixed direction */
+	config.fixed_direction_mask = data->valid_mask;
+	config.fixed_direction_output = data->output_mask;
+	config.drvdata = data;
+
+	return PTR_ERR_OR_ZERO(devm_gpio_regmap_register(dev, &config));
+}
+
+static const struct platform_device_id bd73800_gpio_id[] = {
+	{ "bd73800-gpio" },
+	{ },
+};
+MODULE_DEVICE_TABLE(platform, bd73800_gpio_id);
+
+static struct platform_driver gpo_bd73800_driver = {
+	.driver = {
+		.name = "bd73800-gpio",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
+	.probe = gpo_bd73800_probe,
+	.id_table = bd73800_gpio_id,
+};
+module_platform_driver(gpo_bd73800_driver);
+
+MODULE_AUTHOR("Matti Vaittinen <mazziesaccount@gmail.com>");
+MODULE_DESCRIPTION("GPIO interface for BD73800");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-crystalcove.c b/drivers/gpio/gpio-crystalcove.c
index 0fb5c06d088635..ab7fab5b975459 100644
--- a/drivers/gpio/gpio-crystalcove.c
+++ b/drivers/gpio/gpio-crystalcove.c
@@ -396,3 +396,4 @@ module_platform_driver(crystalcove_gpio_driver);
 MODULE_AUTHOR("Yang, Bin <bin.yang@intel.com>");
 MODULE_DESCRIPTION("Intel Crystal Cove GPIO Driver");
 MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:crystal_cove_gpio");
diff --git a/drivers/gpio/gpio-dln2.c b/drivers/gpio/gpio-dln2.c
index 4670ffd7ea7f00..aef4345d265e2c 100644
--- a/drivers/gpio/gpio-dln2.c
+++ b/drivers/gpio/gpio-dln2.c
@@ -243,10 +243,8 @@ static int dln2_gpio_set_direction(struct gpio_chip *chip, unsigned offset,
 	if (ret < 0)
 		return ret;
 
-	if (dir == DLN2_GPIO_DIRECTION_OUT)
-		set_bit(offset, dln2->output_enabled);
-	else
-		clear_bit(offset, dln2->output_enabled);
+	assign_bit(offset, dln2->output_enabled,
+		   dir == DLN2_GPIO_DIRECTION_OUT);
 
 	return ret;
 }
diff --git a/drivers/gpio/gpio-eic-sprd.c b/drivers/gpio/gpio-eic-sprd.c
index 3b7ebcf12fe7f6..86eb9624d38acc 100644
--- a/drivers/gpio/gpio-eic-sprd.c
+++ b/drivers/gpio/gpio-eic-sprd.c
@@ -601,13 +601,6 @@ static const struct irq_chip sprd_eic_irq = {
 	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
-static void sprd_eic_unregister_notifier(void *data)
-{
-	struct notifier_block *nb = data;
-
-	atomic_notifier_chain_unregister(&sprd_eic_irq_notifier, nb);
-}
-
 static int sprd_eic_probe(struct platform_device *pdev)
 {
 	const struct sprd_eic_variant_data *pdata;
@@ -690,14 +683,8 @@ static int sprd_eic_probe(struct platform_device *pdev)
 	}
 
 	sprd_eic->irq_nb.notifier_call = sprd_eic_irq_notify;
-	ret = atomic_notifier_chain_register(&sprd_eic_irq_notifier,
-					     &sprd_eic->irq_nb);
-	if (ret)
-		return dev_err_probe(dev, ret,
-				     "Failed to register with the interrupt notifier");
-
-	return devm_add_action_or_reset(dev, sprd_eic_unregister_notifier,
-					&sprd_eic->irq_nb);
+	return devm_atomic_notifier_chain_register(dev, &sprd_eic_irq_notifier,
+						   &sprd_eic->irq_nb);
 }
 
 static const struct of_device_id sprd_eic_of_match[] = {
diff --git a/drivers/gpio/gpio-it87.c b/drivers/gpio/gpio-it87.c
index 2ad3c239367bcc..699325129a7de4 100644
--- a/drivers/gpio/gpio-it87.c
+++ b/drivers/gpio/gpio-it87.c
@@ -362,7 +362,7 @@ static int __init it87_gpio_init(void)
 	 */
 	labels = kcalloc(it87_gpio->chip.ngpio, sizeof("it87_gpXY"),
 								GFP_KERNEL);
-	labels_table = kcalloc(it87_gpio->chip.ngpio, sizeof(const char *),
+	labels_table = kcalloc(it87_gpio->chip.ngpio, sizeof(*labels_table),
 								GFP_KERNEL);
 
 	if (!labels || !labels_table) {
diff --git a/drivers/gpio/gpio-mlxbf2.c b/drivers/gpio/gpio-mlxbf2.c
index 4e2f3381d82b88..4e5c5d196a5003 100644
--- a/drivers/gpio/gpio-mlxbf2.c
+++ b/drivers/gpio/gpio-mlxbf2.c
@@ -394,7 +394,9 @@ mlxbf2_gpio_probe(struct platform_device *pdev)
 	gc->owner = THIS_MODULE;
 
 	irq = platform_get_irq_optional(pdev, 0);
-	if (irq >= 0) {
+	if (irq < 0 && irq != -ENXIO)
+		return irq;
+	if (irq > 0) {
 		girq = &gs->chip.gc.irq;
 		gpio_irq_chip_set_chip(girq, &mlxbf2_gpio_irq_chip);
 		girq->handler = handle_simple_irq;
diff --git a/drivers/gpio/gpio-mlxbf3.c b/drivers/gpio/gpio-mlxbf3.c
index 566326644a2c12..929f64c5140a54 100644
--- a/drivers/gpio/gpio-mlxbf3.c
+++ b/drivers/gpio/gpio-mlxbf3.c
@@ -231,7 +231,9 @@ static int mlxbf3_gpio_probe(struct platform_device *pdev)
 	gc->add_pin_ranges = mlxbf3_gpio_add_pin_ranges;
 
 	irq = platform_get_irq_optional(pdev, 0);
-	if (irq >= 0) {
+	if (irq < 0 && irq != -ENXIO)
+		return irq;
+	if (irq > 0) {
 		girq = &gs->chip.gc.irq;
 		gpio_irq_chip_set_chip(girq, &gpio_mlxbf3_irqchip);
 		girq->default_type = IRQ_TYPE_NONE;
diff --git a/drivers/gpio/gpio-mmio.c b/drivers/gpio/gpio-mmio.c
index e9c531eef4521e..0709ffaab4dec3 100644
--- a/drivers/gpio/gpio-mmio.c
+++ b/drivers/gpio/gpio-mmio.c
@@ -49,6 +49,7 @@ o        `                     ~~~~\___/~~~~    ` controller in FPGA is ,.`
 #include <linux/log2.h>
 #include <linux/module.h>
 #include <linux/pinctrl/consumer.h>
+#include <linux/pinctrl/pinconf-generic.h>
 #include <linux/platform_device.h>
 #include <linux/property.h>
 #include <linux/spinlock.h>
@@ -354,6 +355,11 @@ static int gpio_mmio_dir_return(struct gpio_chip *gc, unsigned int gpio,
 	if (!chip->pinctrl)
 		return 0;
 
+#ifdef CONFIG_PINCTRL
+	if (list_empty(&gc->gpiodev->pin_ranges))
+		return 0;
+#endif
+
 	if (dir_out)
 		return pinctrl_gpio_direction_output(gc, gpio);
 	else
@@ -367,7 +373,17 @@ static int gpio_mmio_dir_in_err(struct gpio_chip *gc, unsigned int gpio)
 
 static int gpio_mmio_simple_dir_in(struct gpio_chip *gc, unsigned int gpio)
 {
-	return gpio_mmio_dir_return(gc, gpio, false);
+	struct gpio_generic_chip *chip = to_gpio_generic_chip(gc);
+	int ret;
+
+	ret = gpio_mmio_dir_return(gc, gpio, false);
+	if (ret)
+		return ret;
+
+	guard(raw_spinlock_irqsave)(&chip->lock);
+	chip->sdir &= ~gpio_mmio_line2mask(gc, gpio);
+
+	return 0;
 }
 
 static int gpio_mmio_dir_out_err(struct gpio_chip *gc, unsigned int gpio,
@@ -379,9 +395,19 @@ static int gpio_mmio_dir_out_err(struct gpio_chip *gc, unsigned int gpio,
 static int gpio_mmio_simple_dir_out(struct gpio_chip *gc, unsigned int gpio,
 				    int val)
 {
+	struct gpio_generic_chip *chip = to_gpio_generic_chip(gc);
+	int ret;
+
 	gc->set(gc, gpio, val);
 
-	return gpio_mmio_dir_return(gc, gpio, true);
+	ret = gpio_mmio_dir_return(gc, gpio, true);
+	if (ret)
+		return ret;
+
+	guard(raw_spinlock_irqsave)(&chip->lock);
+	chip->sdir |= gpio_mmio_line2mask(gc, gpio);
+
+	return 0;
 }
 
 static int gpio_mmio_dir_in(struct gpio_chip *gc, unsigned int gpio)
@@ -596,20 +622,51 @@ static int gpio_mmio_setup_direction(struct gpio_generic_chip *chip,
 			gc->direction_input = gpio_mmio_dir_in_err;
 		else
 			gc->direction_input = gpio_mmio_simple_dir_in;
+
+		if (cfg->flags & GPIO_GENERIC_PINCTRL_BACKEND) {
+			chip->dir_unreadable = true;
+			gc->get_direction = gpio_mmio_get_dir;
+			gc->get_config = gpiochip_generic_get_config;
+		}
 	}
 
 	return 0;
 }
 
+static void gpio_mmio_seed_dir_from_pinctrl(struct gpio_chip *gc,
+					    unsigned int gpio)
+{
+	struct gpio_generic_chip *chip = to_gpio_generic_chip(gc);
+	unsigned long config;
+
+	if (!gc->get_config || chip->reg_dir_out || chip->reg_dir_in)
+		return;
+
+	config = pinconf_to_config_packed(PIN_CONFIG_OUTPUT_ENABLE, 0);
+	if (gc->get_config(gc, gpio, &config))
+		return;
+
+	guard(raw_spinlock_irqsave)(&chip->lock);
+	if (config)
+		chip->sdir |= gpio_mmio_line2mask(gc, gpio);
+	else
+		chip->sdir &= ~gpio_mmio_line2mask(gc, gpio);
+}
+
 static int gpio_mmio_request(struct gpio_chip *gc, unsigned int gpio_pin)
 {
 	struct gpio_generic_chip *chip = to_gpio_generic_chip(gc);
+	int ret;
 
 	if (gpio_pin >= gc->ngpio)
 		return -EINVAL;
 
-	if (chip->pinctrl)
-		return gpiochip_generic_request(gc, gpio_pin);
+	if (chip->pinctrl) {
+		ret = gpiochip_generic_request(gc, gpio_pin);
+		if (ret)
+			return ret;
+		gpio_mmio_seed_dir_from_pinctrl(gc, gpio_pin);
+	}
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index 93b8a08b04b9f9..267e6d5bacd0d1 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -599,6 +599,21 @@ static void mvebu_gpio_irq_handler(struct irq_desc *desc)
 }
 
 static const struct regmap_config mvebu_gpio_regmap_config = {
+	.name = "gpio",
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+};
+
+static const struct regmap_config mvebu_gpio_percpu_regmap_config = {
+	.name = "percpu",
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+};
+
+static const struct regmap_config mvebu_pwm_regmap_config = {
+	.name = "pwm",
 	.reg_bits = 32,
 	.reg_stride = 4,
 	.val_bits = 32,
@@ -868,7 +883,7 @@ static int mvebu_pwm_probe(struct platform_device *pdev,
 			return PTR_ERR(base);
 
 		mvpwm->regs = devm_regmap_init_mmio(&pdev->dev, base,
-						    &mvebu_gpio_regmap_config);
+						    &mvebu_pwm_regmap_config);
 		if (IS_ERR(mvpwm->regs))
 			return PTR_ERR(mvpwm->regs);
 
@@ -1034,6 +1049,8 @@ static int mvebu_gpio_suspend(struct platform_device *pdev, pm_message_t state)
 static int mvebu_gpio_resume(struct platform_device *pdev)
 {
 	struct mvebu_gpio_chip *mvchip = platform_get_drvdata(pdev);
+	u32 edge_cache = ~0U, level_cache = ~0U;
+	unsigned long flags;
 	int i;
 
 	regmap_write(mvchip->regs, GPIO_OUT_OFF + mvchip->offset,
@@ -1045,32 +1062,51 @@ static int mvebu_gpio_resume(struct platform_device *pdev)
 	regmap_write(mvchip->regs, GPIO_IN_POL_OFF + mvchip->offset,
 		     mvchip->in_pol_reg);
 
+	/*
+	 * genirq skips mask_irq() for a line it already considers masked, so
+	 * unmasking one behind its back leaves an asserted level line that
+	 * nobody masks. Restore only bits the irqchip cache still has set.
+	 *
+	 * Snapshot the caches under the raw spinlock, but release it before
+	 * the regmap writes below: regmap_write() takes a sleepable lock on
+	 * PREEMPT_RT.
+	 */
+	if (mvchip->domain) {
+		struct irq_chip_generic *gc;
+
+		gc = irq_get_domain_generic_chip(mvchip->domain, 0);
+		raw_spin_lock_irqsave(&gc->lock, flags);
+		level_cache = gc->chip_types[0].mask_cache_priv;
+		edge_cache = gc->chip_types[1].mask_cache_priv;
+		raw_spin_unlock_irqrestore(&gc->lock, flags);
+	}
+
 	switch (mvchip->soc_variant) {
 	case MVEBU_GPIO_SOC_VARIANT_ORION:
 	case MVEBU_GPIO_SOC_VARIANT_A8K:
 		regmap_write(mvchip->regs, GPIO_EDGE_MASK_OFF + mvchip->offset,
-			     mvchip->edge_mask_regs[0]);
+			     mvchip->edge_mask_regs[0] & edge_cache);
 		regmap_write(mvchip->regs, GPIO_LEVEL_MASK_OFF + mvchip->offset,
-			     mvchip->level_mask_regs[0]);
+			     mvchip->level_mask_regs[0] & level_cache);
 		break;
 	case MVEBU_GPIO_SOC_VARIANT_MV78200:
 		for (i = 0; i < 2; i++) {
 			regmap_write(mvchip->regs,
 				     GPIO_EDGE_MASK_MV78200_OFF(i),
-				     mvchip->edge_mask_regs[i]);
+				     mvchip->edge_mask_regs[i] & edge_cache);
 			regmap_write(mvchip->regs,
 				     GPIO_LEVEL_MASK_MV78200_OFF(i),
-				     mvchip->level_mask_regs[i]);
+				     mvchip->level_mask_regs[i] & level_cache);
 		}
 		break;
 	case MVEBU_GPIO_SOC_VARIANT_ARMADAXP:
 		for (i = 0; i < 4; i++) {
 			regmap_write(mvchip->regs,
 				     GPIO_EDGE_MASK_ARMADAXP_OFF(i),
-				     mvchip->edge_mask_regs[i]);
+				     mvchip->edge_mask_regs[i] & edge_cache);
 			regmap_write(mvchip->regs,
 				     GPIO_LEVEL_MASK_ARMADAXP_OFF(i),
-				     mvchip->level_mask_regs[i]);
+				     mvchip->level_mask_regs[i] & level_cache);
 		}
 		break;
 	default:
@@ -1114,7 +1150,7 @@ static int mvebu_gpio_probe_raw(struct platform_device *pdev,
 
 		mvchip->percpu_regs =
 			devm_regmap_init_mmio(&pdev->dev, base,
-					      &mvebu_gpio_regmap_config);
+					      &mvebu_gpio_percpu_regmap_config);
 		if (IS_ERR(mvchip->percpu_regs))
 			return PTR_ERR(mvchip->percpu_regs);
 	}
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index 005420baecf4d1..a9b9a3d9631e29 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -1032,8 +1032,6 @@ static void omap_gpio_mod_init(struct gpio_bank *bank)
 static int omap_gpio_chip_init(struct gpio_bank *bank, struct device *pm_dev)
 {
 	struct gpio_irq_chip *irq;
-	static int gpio;
-	const char *label;
 	int ret;
 
 	/*
@@ -1055,11 +1053,7 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct device *pm_dev)
 		if (bank->regs->wkup_en)
 			bank->chip.parent = &omap_mpuio_device.dev;
 	} else {
-		label = devm_kasprintf(bank->chip.parent, GFP_KERNEL, "gpio-%d-%d",
-				       gpio, gpio + bank->width - 1);
-		if (!label)
-			return -ENOMEM;
-		bank->chip.label = label;
+		bank->chip.label = dev_name(bank->chip.parent);
 	}
 	bank->chip.base = -1;
 	bank->chip.ngpio = bank->width;
@@ -1086,9 +1080,6 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct device *pm_dev)
 	if (ret)
 		gpiochip_remove(&bank->chip);
 
-	if (!bank->is_mpuio)
-		gpio += bank->width;
-
 	return ret;
 }
 
diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c
index c1f5e10a3c2026..1ba6a09542f3c6 100644
--- a/drivers/gpio/gpio-pcf857x.c
+++ b/drivers/gpio/gpio-pcf857x.c
@@ -71,8 +71,9 @@ MODULE_DEVICE_TABLE(of, pcf857x_of_table);
 struct pcf857x {
 	struct gpio_chip	chip;
 	struct i2c_client	*client;
-	struct mutex		lock;		/* protect 'out' */
+	struct mutex		lock;		/* protect 'out' and 'dir' */
 	unsigned int		out;		/* software latch */
+	unsigned int		dir;		/* direction latch (1 = input) */
 	unsigned int		status;		/* current status */
 	unsigned int		irq_enabled;	/* enabled irqs */
 
@@ -126,12 +127,21 @@ static int pcf857x_input(struct gpio_chip *chip, unsigned int offset)
 
 	mutex_lock(&gpio->lock);
 	gpio->out |= (1 << offset);
+	gpio->dir |= (1 << offset);
 	status = gpio->write(gpio->client, gpio->out);
 	mutex_unlock(&gpio->lock);
 
 	return status;
 }
 
+static int pcf857x_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	struct pcf857x *gpio = gpiochip_get_data(chip);
+
+	return (gpio->dir & (1 << offset)) ? GPIO_LINE_DIRECTION_IN
+					   : GPIO_LINE_DIRECTION_OUT;
+}
+
 static int pcf857x_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct pcf857x *gpio = gpiochip_get_data(chip);
@@ -167,6 +177,7 @@ static int pcf857x_output(struct gpio_chip *chip, unsigned int offset, int value
 		gpio->out |= bit;
 	else
 		gpio->out &= ~bit;
+	gpio->dir &= ~bit;
 	status = gpio->write(gpio->client, gpio->out);
 	mutex_unlock(&gpio->lock);
 
@@ -187,6 +198,7 @@ static int pcf857x_set_multiple(struct gpio_chip *chip, unsigned long *mask,
 	mutex_lock(&gpio->lock);
 	gpio->out &= ~*mask;
 	gpio->out |= *bits & *mask;
+	gpio->dir &= ~*mask;
 	status = gpio->write(gpio->client, gpio->out);
 	mutex_unlock(&gpio->lock);
 
@@ -301,6 +313,7 @@ static int pcf857x_probe(struct i2c_client *client)
 	gpio->chip.set_multiple		= pcf857x_set_multiple;
 	gpio->chip.direction_input	= pcf857x_input;
 	gpio->chip.direction_output	= pcf857x_output;
+	gpio->chip.get_direction	= pcf857x_get_direction;
 	gpio->chip.ngpio		= (uintptr_t)i2c_get_match_data(client);
 
 	reset_gpio = devm_gpiod_get_optional(&client->dev, "reset", GPIOD_OUT_HIGH);
@@ -396,6 +409,7 @@ static int pcf857x_probe(struct i2c_client *client)
 	 * reset state.  Otherwise it flags pins to be driven low.
 	 */
 	gpio->out = ~n_latch;
+	gpio->dir = ~n_latch;
 	gpio->status = gpio->read(gpio->client);
 
 	/* Enable irqchip if we have an interrupt */
diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c
index 2b5da973f51fea..3ff21e8992788f 100644
--- a/drivers/gpio/gpio-pxa.c
+++ b/drivers/gpio/gpio-pxa.c
@@ -268,7 +268,7 @@ static int pxa_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
 	spin_lock_irqsave(&gpio_lock, flags);
 
 	value = readl_relaxed(base + GPDR_OFFSET);
-	if (__gpio_is_inverted(chip->base + offset))
+	if (__gpio_is_inverted(offset))
 		value |= mask;
 	else
 		value &= ~mask;
@@ -297,7 +297,7 @@ static int pxa_gpio_direction_output(struct gpio_chip *chip,
 	spin_lock_irqsave(&gpio_lock, flags);
 
 	tmp = readl_relaxed(base + GPDR_OFFSET);
-	if (__gpio_is_inverted(chip->base + offset))
+	if (__gpio_is_inverted(offset))
 		tmp &= ~mask;
 	else
 		tmp |= mask;
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index 09bebde5c4260a..54dc47d45de7e8 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -36,6 +36,7 @@ struct gpio_rcar_info {
 	bool has_both_edge_trigger;
 	bool has_always_in;
 	bool has_inen;
+	const u8 *offsetmap;
 };
 
 struct gpio_rcar_priv {
@@ -49,34 +50,68 @@ struct gpio_rcar_priv {
 	struct gpio_rcar_bank_info bank_info;
 };
 
-#define IOINTSEL	0x00	/* General IO/Interrupt Switching Register */
-#define INOUTSEL	0x04	/* General Input/Output Switching Register */
-#define OUTDT		0x08	/* General Output Register */
-#define INDT		0x0c	/* General Input Register */
-#define INTDT		0x10	/* Interrupt Display Register */
-#define INTCLR		0x14	/* Interrupt Clear Register */
-#define INTMSK		0x18	/* Interrupt Mask Register */
-#define MSKCLR		0x1c	/* Interrupt Mask Clear Register */
-#define POSNEG		0x20	/* Positive/Negative Logic Select Register */
-#define EDGLEVEL	0x24	/* Edge/level Select Register */
-#define OUTDTSEL	0x40	/* Output Data Select Register */
-#define BOTHEDGE	0x4c	/* One Edge/Both Edge Select Register */
-#define INEN		0x50	/* General Input Enable Register */
+enum gpio_rcar_reg {
+	IOINTSEL,	/* General IO/Interrupt Switching Register */
+	INOUTSEL,	/* General Input/Output Switching Register */
+	OUTDT,		/* General Output Register */
+	INDT,		/* General Input Register */
+	INTDT,		/* Interrupt Display Register */
+	INTCLR,		/* Interrupt Clear Register */
+	INTMSK,		/* Interrupt Mask Register */
+	MSKCLR,		/* Interrupt Mask Clear Register */
+	POSNEG,		/* Positive/Negative Logic Select Register */
+	EDGLEVEL,	/* Edge/level Select Register */
+	OUTDTSEL,	/* Output Data Select Register */
+	BOTHEDGE,	/* One Edge/Both Edge Select Register */
+	INEN,		/* General Input Enable Register */
+};
+
+static const u8 gpio_rcar_offsetmap_gen1[] = {
+	[IOINTSEL]	= 0x00,
+	[INOUTSEL]	= 0x04,
+	[OUTDT]		= 0x08,
+	[INDT]		= 0x0c,
+	[INTDT]		= 0x10,
+	[INTCLR]	= 0x14,
+	[INTMSK]	= 0x18,
+	[MSKCLR]	= 0x1c,
+	[POSNEG]	= 0x20,
+	[EDGLEVEL]	= 0x24,
+	[OUTDTSEL]	= 0x40,
+	[BOTHEDGE]	= 0x4c,
+	[INEN]		= 0x50,
+};
+
+static const u8 gpio_rcar_offsetmap_gen5[] = {
+	[IOINTSEL]	= 0x00,
+	[INOUTSEL]	= 0x04,
+	[OUTDT]		= 0x08,
+	[INDT]		= 0x1c,
+	[INTDT]		= 0x80,
+	[INTCLR]	= 0x84,
+	[INTMSK]	= 0x88,
+	[MSKCLR]	= 0x8c,
+	[POSNEG]	= 0x90,
+	[EDGLEVEL]	= 0x94,
+	[OUTDTSEL]	= 0x0c,
+	[BOTHEDGE]	= 0xbc,
+	[INEN]		= 0x18,
+};
 
 #define RCAR_MAX_GPIO_PER_BANK		32
 
-static inline u32 gpio_rcar_read(struct gpio_rcar_priv *p, int offs)
+static inline u32 gpio_rcar_read(struct gpio_rcar_priv *p, enum gpio_rcar_reg offs)
 {
-	return ioread32(p->base + offs);
+	return ioread32(p->base + p->info.offsetmap[offs]);
 }
 
-static inline void gpio_rcar_write(struct gpio_rcar_priv *p, int offs,
+static inline void gpio_rcar_write(struct gpio_rcar_priv *p, enum gpio_rcar_reg offs,
 				   u32 value)
 {
-	iowrite32(value, p->base + offs);
+	iowrite32(value, p->base + p->info.offsetmap[offs]);
 }
 
-static void gpio_rcar_modify_bit(struct gpio_rcar_priv *p, int offs,
+static void gpio_rcar_modify_bit(struct gpio_rcar_priv *p, enum gpio_rcar_reg offs,
 				 int bit, bool value)
 {
 	u32 tmp = gpio_rcar_read(p, offs);
@@ -399,6 +434,7 @@ static const struct gpio_rcar_info gpio_rcar_info_gen1 = {
 	.has_both_edge_trigger = false,
 	.has_always_in = false,
 	.has_inen = false,
+	.offsetmap = gpio_rcar_offsetmap_gen1,
 };
 
 static const struct gpio_rcar_info gpio_rcar_info_gen2 = {
@@ -406,6 +442,7 @@ static const struct gpio_rcar_info gpio_rcar_info_gen2 = {
 	.has_both_edge_trigger = true,
 	.has_always_in = false,
 	.has_inen = false,
+	.offsetmap = gpio_rcar_offsetmap_gen1,
 };
 
 static const struct gpio_rcar_info gpio_rcar_info_gen3 = {
@@ -413,6 +450,7 @@ static const struct gpio_rcar_info gpio_rcar_info_gen3 = {
 	.has_both_edge_trigger = true,
 	.has_always_in = true,
 	.has_inen = false,
+	.offsetmap = gpio_rcar_offsetmap_gen1,
 };
 
 static const struct gpio_rcar_info gpio_rcar_info_gen4 = {
@@ -420,6 +458,15 @@ static const struct gpio_rcar_info gpio_rcar_info_gen4 = {
 	.has_both_edge_trigger = true,
 	.has_always_in = true,
 	.has_inen = true,
+	.offsetmap = gpio_rcar_offsetmap_gen1,
+};
+
+static const struct gpio_rcar_info gpio_rcar_info_gen5 = {
+	.has_outdtsel = true,
+	.has_both_edge_trigger = true,
+	.has_always_in = true,
+	.has_inen = true,
+	.offsetmap = gpio_rcar_offsetmap_gen5,
 };
 
 static const struct of_device_id gpio_rcar_of_table[] = {
@@ -438,6 +485,9 @@ static const struct of_device_id gpio_rcar_of_table[] = {
 	}, {
 		.compatible = "renesas,rcar-gen4-gpio",
 		.data = &gpio_rcar_info_gen4,
+	}, {
+		.compatible = "renesas,rcar-gen5-gpio",
+		.data = &gpio_rcar_info_gen5,
 	}, {
 		.compatible = "renesas,gpio-rcar",
 		.data = &gpio_rcar_info_gen1,
diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c
index 4a606bad584868..5d98ad597d9e27 100644
--- a/drivers/gpio/gpio-realtek-otto.c
+++ b/drivers/gpio/gpio-realtek-otto.c
@@ -176,10 +176,10 @@ static void realtek_gpio_update_line_imr(struct realtek_gpio_ctrl *ctrl, unsigne
 	u32 reg_val;
 
 	reg += 4 * (line_shift / 32);
-	reg_val = ioread32(reg);
+	reg_val = __raw_readl(reg);
 	reg_val &= ~(REALTEK_GPIO_IMR_LINE_MASK << shift);
 	reg_val |= (irq_type & irq_mask & REALTEK_GPIO_IMR_LINE_MASK) << shift;
-	iowrite32(reg_val, reg);
+	__raw_writel(reg_val, reg);
 }
 
 static void realtek_gpio_irq_ack(struct irq_data *data)
@@ -393,18 +393,29 @@ static int realtek_gpio_probe(struct platform_device *pdev)
 
 	raw_spin_lock_init(&ctrl->lock);
 
+	gen_gc_flags = GPIO_GENERIC_PINCTRL_BACKEND;
+
 	if (dev_flags & GPIO_PORTS_REVERSED) {
-		gen_gc_flags = 0;
 		ctrl->bank_read = realtek_gpio_bank_read;
 		ctrl->bank_write = realtek_gpio_bank_write;
 		ctrl->line_imr_pos = realtek_gpio_line_imr_pos;
 	} else {
-		gen_gc_flags = GPIO_GENERIC_BIG_ENDIAN_BYTE_ORDER;
+		gen_gc_flags |= GPIO_GENERIC_BIG_ENDIAN_BYTE_ORDER;
 		ctrl->bank_read = realtek_gpio_bank_read_swapped;
 		ctrl->bank_write = realtek_gpio_bank_write_swapped;
 		ctrl->line_imr_pos = realtek_gpio_line_imr_pos_swapped;
 	}
 
+	if (device_property_read_bool(dev, "little-endian")) {
+		gen_gc_flags = 0;
+		ctrl->bank_read = realtek_gpio_bank_read;
+		ctrl->bank_write = realtek_gpio_bank_write;
+	} else if (device_is_big_endian(dev)) {
+		gen_gc_flags = GPIO_GENERIC_BIG_ENDIAN_BYTE_ORDER;
+		ctrl->bank_read = realtek_gpio_bank_read_swapped;
+		ctrl->bank_write = realtek_gpio_bank_write_swapped;
+	}
+
 	config = (struct gpio_generic_chip_config) {
 		.dev = dev,
 		.sz = 4,
@@ -423,6 +434,8 @@ static int realtek_gpio_probe(struct platform_device *pdev)
 	ctrl->chip.gc.owner = THIS_MODULE;
 
 	irq = platform_get_irq_optional(pdev, 0);
+	if (irq < 0 && irq != -ENXIO)
+		return irq;
 	if (!(dev_flags & GPIO_INTERRUPTS_DISABLED) && irq > 0) {
 		girq = &ctrl->chip.gc.irq;
 		gpio_irq_chip_set_chip(girq, &realtek_gpio_irq_chip);
diff --git a/drivers/gpio/gpio-regmap.c b/drivers/gpio/gpio-regmap.c
index fb21396e1d024e..2f5f45b84cc94f 100644
--- a/drivers/gpio/gpio-regmap.c
+++ b/drivers/gpio/gpio-regmap.c
@@ -6,11 +6,13 @@
  */
 
 #include <linux/bits.h>
+#include <linux/cleanup.h>
 #include <linux/compiler_attributes.h>
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
 #include <linux/types.h>
@@ -23,6 +25,8 @@
 struct gpio_regmap {
 	struct device *parent;
 	struct regmap *regmap;
+	struct device *pm_dev;
+
 	struct gpio_chip gpio_chip;
 
 	int reg_stride;
@@ -79,6 +83,33 @@ static int gpio_regmap_simple_xlate(struct gpio_regmap *gpio,
 	return 0;
 }
 
+static int gpio_regmap_runtime_get(struct gpio_regmap *gpio)
+{
+	if (!gpio->pm_dev)
+		return 0;
+
+	return pm_runtime_get_active(gpio->pm_dev, RPM_TRANSPARENT);
+}
+
+static void gpio_regmap_runtime_put(struct gpio_regmap *gpio)
+{
+	if (!gpio->pm_dev)
+		return;
+
+	pm_runtime_put_autosuspend(gpio->pm_dev);
+}
+
+DEFINE_GUARD(gpio_regmap_runtime, struct gpio_regmap *,
+	     gpio_regmap_runtime_get(_T), gpio_regmap_runtime_put(_T))
+DEFINE_GUARD_COND(gpio_regmap_runtime, _try,
+		  gpio_regmap_runtime_get(_T), _RET == 0)
+
+#define GPIO_REGMAP_RUNTIME_ACQUIRE(_gpio, _var) \
+	ACQUIRE(gpio_regmap_runtime_try, _var)(_gpio)
+
+#define GPIO_REGMAP_RUNTIME_ACQUIRE_ERR(_var_ptr) \
+	ACQUIRE_ERR(gpio_regmap_runtime, _var_ptr)
+
 static int gpio_regmap_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct gpio_regmap *gpio = gpiochip_get_data(chip);
@@ -91,15 +122,21 @@ static int gpio_regmap_get(struct gpio_chip *chip, unsigned int offset)
 	else
 		base = gpio_regmap_addr(gpio->reg_set_base);
 
-	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_GET_OP, base, offset, &reg, &mask);
+	GPIO_REGMAP_RUNTIME_ACQUIRE(gpio, pm);
+	ret = GPIO_REGMAP_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_GET_OP, base, offset,
+				   &reg, &mask);
 	if (ret)
 		return ret;
 
+	if (gpio->reg_dat_base != gpio->reg_set_base)
+		return regmap_test_bits(gpio->regmap, reg, mask);
+
 	/* ensure we don't spoil any register cache with pin input values */
-	if (gpio->reg_dat_base == gpio->reg_set_base)
-		ret = regmap_read_bypassed(gpio->regmap, reg, &val);
-	else
-		ret = regmap_read(gpio->regmap, reg, &val);
+	ret = regmap_read_bypassed(gpio->regmap, reg, &val);
 	if (ret)
 		return ret;
 
@@ -114,7 +151,13 @@ static int gpio_regmap_set(struct gpio_chip *chip, unsigned int offset,
 	unsigned int reg, mask, mask_val;
 	int ret;
 
-	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_SET_OP, base, offset, &reg, &mask);
+	GPIO_REGMAP_RUNTIME_ACQUIRE(gpio, pm);
+	ret = GPIO_REGMAP_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_SET_OP, base, offset,
+				   &reg, &mask);
 	if (ret)
 		return ret;
 
@@ -146,6 +189,11 @@ static int gpio_regmap_set_with_clear(struct gpio_chip *chip,
 	unsigned int base, reg, mask, value = 0;
 	int ret;
 
+	GPIO_REGMAP_RUNTIME_ACQUIRE(gpio, pm);
+	ret = GPIO_REGMAP_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
 	if (val)
 		base = gpio_regmap_addr(gpio->reg_set_base);
 	else
@@ -183,7 +231,7 @@ static int gpio_regmap_get_direction(struct gpio_chip *chip,
 				     unsigned int offset)
 {
 	struct gpio_regmap *gpio = gpiochip_get_data(chip);
-	unsigned int base, val, reg, mask;
+	unsigned int base, reg, mask;
 	int invert, ret;
 
 	if (gpio_regmap_fixed_direction(gpio, offset)) {
@@ -208,18 +256,24 @@ static int gpio_regmap_get_direction(struct gpio_chip *chip,
 		return -ENOTSUPP;
 	}
 
-	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_GET_DIR_OP, base, offset, &reg, &mask);
+	GPIO_REGMAP_RUNTIME_ACQUIRE(gpio, pm);
+	ret = GPIO_REGMAP_RUNTIME_ACQUIRE_ERR(&pm);
 	if (ret)
 		return ret;
 
-	ret = regmap_read(gpio->regmap, reg, &val);
+	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_GET_DIR_OP, base, offset,
+				   &reg, &mask);
 	if (ret)
 		return ret;
 
-	if (!!(val & mask) ^ invert)
+	ret = regmap_test_bits(gpio->regmap, reg, mask);
+	if (ret < 0)
+		return ret;
+
+	if (ret ^ invert)
 		return GPIO_LINE_DIRECTION_OUT;
-	else
-		return GPIO_LINE_DIRECTION_IN;
+
+	return GPIO_LINE_DIRECTION_IN;
 }
 
 static int gpio_regmap_try_direction_fixed(struct gpio_regmap *gpio,
@@ -262,7 +316,13 @@ static int gpio_regmap_set_direction(struct gpio_chip *chip,
 		return -ENOTSUPP;
 	}
 
-	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_SET_DIR_OP, base, offset, &reg, &mask);
+	GPIO_REGMAP_RUNTIME_ACQUIRE(gpio, pm);
+	ret = GPIO_REGMAP_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = gpio->reg_mask_xlate(gpio, GPIO_REGMAP_SET_DIR_OP, base, offset,
+				   &reg, &mask);
 	if (ret)
 		return ret;
 
@@ -293,6 +353,11 @@ static int gpio_regmap_direction_output(struct gpio_chip *chip,
 	struct gpio_regmap *gpio = gpiochip_get_data(chip);
 	int ret;
 
+	GPIO_REGMAP_RUNTIME_ACQUIRE(gpio, pm);
+	ret = GPIO_REGMAP_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
 	/*
 	 * First check if this is gonna work on a fixed direction line,
 	 * if it doesn't (i.e. this is a fixed input line), then do not
@@ -304,7 +369,9 @@ static int gpio_regmap_direction_output(struct gpio_chip *chip,
 			return ret;
 	}
 
-	gpio_regmap_set(chip, offset, value);
+	ret = gpio_regmap_set(chip, offset, value);
+	if (ret)
+		return ret;
 
 	return gpio_regmap_set_direction(chip, offset, true);
 }
@@ -399,6 +466,7 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config
 	gpio->reg_clr_base = config->reg_clr_base;
 	gpio->reg_dir_in_base = config->reg_dir_in_base;
 	gpio->reg_dir_out_base = config->reg_dir_out_base;
+	gpio->pm_dev = config->pm_dev;
 
 	chip = &gpio->gpio_chip;
 	chip->parent = config->parent;
@@ -406,7 +474,7 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config
 	chip->base = -1;
 	chip->names = config->names;
 	chip->label = config->label ?: dev_name(config->parent);
-	chip->can_sleep = regmap_might_sleep(config->regmap);
+	chip->can_sleep = config->pm_dev || regmap_might_sleep(config->regmap);
 	chip->init_valid_mask = config->init_valid_mask;
 
 	chip->request = gpiochip_generic_request;
diff --git a/drivers/gpio/gpio-rtd1625.c b/drivers/gpio/gpio-rtd1625.c
index 483e44cf5abc2f..bebd1b8ede4839 100644
--- a/drivers/gpio/gpio-rtd1625.c
+++ b/drivers/gpio/gpio-rtd1625.c
@@ -107,10 +107,11 @@ static int rtd1625_reg_mask_xlate(struct gpio_regmap *gpio, enum gpio_regmap_ope
 				  unsigned int base, unsigned int offset, unsigned int *reg,
 				  unsigned int *mask)
 {
-	/* Each GPIO has its own dedicated 32-bit register */
 	struct rtd1625_gpio *data = gpio_regmap_get_drvdata(gpio);
-	int val = 0, ret = 0;
+	/* Each GPIO has its own dedicated 32-bit register */
 	*reg = base + offset * 4;
+	unsigned int val;
+	int ret;
 
 	switch (op) {
 	case GPIO_REGMAP_SET_OP:
@@ -220,11 +221,11 @@ static void rtd1625_gpio_irq_handle(struct irq_desc *desc)
 {
 	unsigned int (*get_reg_offset)(struct rtd1625_gpio *gpio, unsigned int offset);
 	struct rtd1625_gpio *data = irq_desc_get_handler_data(desc);
+	struct device *dev = regmap_get_device(data->regmap);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
 	unsigned int irq = irq_desc_get_irq(desc);
 	struct irq_domain *domain = data->domain;
-	unsigned int reg_offset, i, j, val;
-	irq_hw_number_t hwirq;
+	unsigned int reg_offset, j, val;
 	unsigned long status;
 	u32 irq_type;
 	int ret;
@@ -240,11 +241,12 @@ static void rtd1625_gpio_irq_handle(struct irq_desc *desc)
 
 	chained_irq_enter(chip, desc);
 
-	for (i = 0; i < data->info->num_gpios; i += 32) {
+	for (unsigned int i = 0; i < data->info->num_gpios; i += 32) {
 		reg_offset = get_reg_offset(data, i);
 		ret = regmap_read(data->regmap, reg_offset, &val);
 		if (ret) {
-			pr_err_ratelimited("Failed to read IRQ status for GPIO %u: %d\n", i, ret);
+			dev_err_ratelimited(dev, "Failed to read IRQ status for GPIO %u: %d\n",
+					    i, ret);
 			continue;
 		}
 
@@ -261,12 +263,14 @@ static void rtd1625_gpio_irq_handle(struct irq_desc *desc)
 		if (irq != data->irqs[RTD1625_IRQ_LEVEL]) {
 			ret = regmap_write(data->regmap, reg_offset, status);
 			if (ret)
-				pr_err_ratelimited("Failed to clear edge IRQ for GPIO %u: %d\n",
-						   i, ret);
+				dev_err_ratelimited(dev,
+						    "Failed to clear edge IRQ for GPIO %u: %d\n",
+						    i, ret);
 		}
 
 		for_each_set_bit(j, &status, 32) {
-			hwirq = i + j;
+			irq_hw_number_t hwirq = i + j;
+
 			irq_type = irq_get_trigger_type(irq_find_mapping(domain, hwirq));
 
 			/*
@@ -486,7 +490,6 @@ static int rtd1625_gpio_setup_irq(struct platform_device *pdev, struct rtd1625_g
 		return irq;
 
 	num_irqs = (data->info->irq_type_support & IRQ_TYPE_LEVEL_MASK) ? 3 : 2;
-
 	for (unsigned int i = 0; i < num_irqs; i++) {
 		irq = platform_get_irq(pdev, i);
 		if (irq < 0)
@@ -612,8 +615,7 @@ static const struct rtd1625_gpio_info rtd1625_iso_gpio_info = {
 
 static const struct rtd1625_gpio_info rtd1625_isom_gpio_info = {
 	.num_gpios        = 4,
-	.irq_type_support = IRQ_TYPE_EDGE_BOTH | IRQ_TYPE_LEVEL_LOW |
-			    IRQ_TYPE_LEVEL_HIGH,
+	.irq_type_support = IRQ_TYPE_DEFAULT,
 	.base_offset      = 0x20,
 	.gpa_offset       = 0x00,
 	.gpda_offset      = 0x04,
diff --git a/drivers/gpio/gpio-siox.c b/drivers/gpio/gpio-siox.c
index 958034b9f3f33a..d60680ace144df 100644
--- a/drivers/gpio/gpio-siox.c
+++ b/drivers/gpio/gpio-siox.c
@@ -148,11 +148,11 @@ static int gpio_siox_get(struct gpio_chip *chip, unsigned int offset)
 	if (offset >= 12) {
 		unsigned int bitpos = 19 - offset;
 
-		ret = ddata->setdata[0] & (1 << bitpos);
+		ret = !!(ddata->setdata[0] & (1 << bitpos));
 	} else {
 		unsigned int bitpos = 11 - offset;
 
-		ret = ddata->getdata[bitpos / 8] & (1 << (bitpos % 8));
+		ret = !!(ddata->getdata[bitpos / 8] & (1 << (bitpos % 8)));
 	}
 
 	mutex_unlock(&ddata->lock);
diff --git a/drivers/gpio/gpio-sloppy-logic-analyzer.c b/drivers/gpio/gpio-sloppy-logic-analyzer.c
index 044d81e7cafbda..0717649a6820c0 100644
--- a/drivers/gpio/gpio-sloppy-logic-analyzer.c
+++ b/drivers/gpio/gpio-sloppy-logic-analyzer.c
@@ -299,8 +299,8 @@ static int gpio_la_poll_probe(struct platform_device *pdev)
 	debugfs_create_blob("meta_data", 0400, priv->debug_dir, &priv->meta);
 	debugfs_create_ulong("delay_ns", 0600, priv->debug_dir, &priv->delay_ns);
 	debugfs_create_ulong("delay_ns_acquisition", 0400, priv->debug_dir, &priv->acq_delay);
-	debugfs_create_file_unsafe("buf_size", 0600, priv->debug_dir, priv, &fops_buf_size);
-	debugfs_create_file_unsafe("capture", 0200, priv->debug_dir, priv, &fops_capture);
+	debugfs_create_file("buf_size", 0600, priv->debug_dir, priv, &fops_buf_size);
+	debugfs_create_file("capture", 0200, priv->debug_dir, priv, &fops_capture);
 	debugfs_create_file("trigger", 0200, priv->debug_dir, priv, &fops_trigger);
 
 	return 0;
diff --git a/drivers/gpio/gpio-tc9563.c b/drivers/gpio/gpio-tc9563.c
new file mode 100644
index 00000000000000..68c20c8cba7994
--- /dev/null
+++ b/drivers/gpio/gpio-tc9563.c
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 by RISCstar Solutions Corporation.  All rights reserved.
+ */
+
+/*
+ * The Toshiba TC9563 implements a PCIe Gen 3 switch that connects an
+ * upstream x4 port to two downstream PCIe x2 ports. It incorporates
+ * an internal endpoint on a internal PCIe port that implements two
+ * Synopsys XGMAC Ethernet interfaces.
+ *
+ * 37 GPIOs are also implemented by an embedded GPIO controller.  Three
+ * registers control the first 32 GPIOs (other than 20 and 21, which are
+ * reserved). Three other registers control GPIOs 32 through 36. GPIOs
+ * 22-24, 27-28, 31, and 34 are treated as "input only".
+ *
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/soc/qcom/tc9563.h>
+
+/*
+ * There are two sets of registers, each representing (up to) 32 GPIOs with a
+ * stride of 4 bytes (IN1 is 4 bytes past IN0, EN1 is 4 bytes past EN0, etc.).
+ */
+#define TC9563_GPIO_COUNT		37
+#define TC9563_GPIO_PER_REG		32
+#define TC9563_GPIO_REG_STRIDE		4
+
+static int tc9563_gpio_init_valid_mask(struct gpio_chip *gc,
+				       unsigned long *valid_mask,
+				       unsigned int ngpios)
+{
+	/* GPIOs 20 and 21 are reserved */
+	bitmap_fill(valid_mask, ngpios);
+	bitmap_clear(valid_mask, 20, 2);
+
+	return 0;
+}
+
+static int tc9563_gpio_probe(struct auxiliary_device *adev,
+			     const struct auxiliary_device_id *id)
+{
+	struct gpio_regmap_config config = {
+		.parent = &adev->dev,
+		.ngpio = TC9563_GPIO_COUNT,
+		.reg_stride = TC9563_GPIO_REG_STRIDE,
+		.ngpio_per_reg = TC9563_GPIO_PER_REG,
+		.reg_dat_base = GPIO_REGMAP_ADDR(TC9563_GPIO_IN0_OFFSET),
+		.reg_set_base = GPIO_REGMAP_ADDR(TC9563_GPIO_OUT0_OFFSET),
+		.reg_dir_in_base = GPIO_REGMAP_ADDR(TC9563_GPIO_EN0_OFFSET),
+		.init_valid_mask = tc9563_gpio_init_valid_mask,
+	};
+	DECLARE_BITMAP(fixed_dir_mask, TC9563_GPIO_COUNT);
+	DECLARE_BITMAP(fixed_dir_out, TC9563_GPIO_COUNT);
+
+	config.regmap = dev_get_platdata(&adev->dev);
+	if (!config.regmap)
+		return -EINVAL;
+
+	/*
+	 * Only some of our GPIOs are fixed direction:
+	 * 22, 23, 24, 27, 28, 31, and 34 are input-only.
+	 */
+	bitmap_zero(fixed_dir_mask, TC9563_GPIO_COUNT);
+	bitmap_set(fixed_dir_mask, 22, 3);
+	bitmap_set(fixed_dir_mask, 27, 2);
+	set_bit(31, fixed_dir_mask);
+	set_bit(34, fixed_dir_mask);
+	config.fixed_direction_mask = fixed_dir_mask;
+
+	bitmap_zero(fixed_dir_out, TC9563_GPIO_COUNT);
+	config.fixed_direction_output = fixed_dir_out;
+
+	return PTR_ERR_OR_ZERO(devm_gpio_regmap_register(&adev->dev, &config));
+};
+
+static const struct auxiliary_device_id tc9563_gpio_ids[] = {
+	{ "pci_pwrctrl_tc9563." TC9563_GPIO_DEV_NAME },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(auxiliary, tc9563_gpio_ids);
+
+static struct auxiliary_driver tc9563_gpio_driver = {
+	.name		= TC9563_GPIO_DEV_NAME,
+	.probe		= tc9563_gpio_probe,
+	.id_table	= tc9563_gpio_ids,
+};
+module_auxiliary_driver(tc9563_gpio_driver);
+
+MODULE_AUTHOR("Alex Elder <elder@riscstar.com>");
+MODULE_AUTHOR("Daniel Thompson <daniel@riscstar.com>");
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi@oss.qualcomm.com>");
+MODULE_DESCRIPTION("Toshiba TC9563 GPIO Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-tegra186.c b/drivers/gpio/gpio-tegra186.c
index d9a2dedf50eae2..34e084c95d7a91 100644
--- a/drivers/gpio/gpio-tegra186.c
+++ b/drivers/gpio/gpio-tegra186.c
@@ -1419,6 +1419,71 @@ static const struct tegra_gpio_soc tegra264_uphy_soc = {
 	.has_vm_support = true,
 };
 
+static const struct tegra_gpio_port tegra268_main_ports[] = {
+	TEGRA264_MAIN_GPIO_PORT(F, 3, 0, 8),
+	TEGRA264_MAIN_GPIO_PORT(G, 3, 1, 5),
+	TEGRA264_MAIN_GPIO_PORT(H, 1, 0, 8),
+	TEGRA264_MAIN_GPIO_PORT(J, 1, 1, 8),
+	TEGRA264_MAIN_GPIO_PORT(K, 1, 2, 8),
+	TEGRA264_MAIN_GPIO_PORT(L, 1, 3, 8),
+	TEGRA264_MAIN_GPIO_PORT(M, 1, 4, 2),
+	TEGRA264_MAIN_GPIO_PORT(P, 2, 0, 8),
+	TEGRA264_MAIN_GPIO_PORT(Q, 2, 1, 8),
+	TEGRA264_MAIN_GPIO_PORT(R, 2, 2, 8),
+	TEGRA264_MAIN_GPIO_PORT(S, 2, 3, 2),
+	TEGRA264_MAIN_GPIO_PORT(T, 0, 0, 7),
+	TEGRA264_MAIN_GPIO_PORT(U, 0, 1, 8),
+	TEGRA264_MAIN_GPIO_PORT(V, 0, 2, 8),
+	TEGRA264_MAIN_GPIO_PORT(W, 0, 3, 8),
+	TEGRA264_MAIN_GPIO_PORT(Y, 0, 5, 8),
+	TEGRA264_MAIN_GPIO_PORT(Z, 0, 6, 8),
+	TEGRA264_MAIN_GPIO_PORT(AL, 0, 4, 3),
+};
+
+static const struct tegra_gpio_soc tegra268_main_soc = {
+	.num_ports = ARRAY_SIZE(tegra268_main_ports),
+	.ports = tegra268_main_ports,
+	.name = "tegra268-gpio",
+	.instance = 0,
+	.num_irqs_per_bank = 8,
+	.has_vm_support = true,
+};
+
+static const struct tegra_gpio_port tegra268_aon_ports[] = {
+	TEGRA264_AON_GPIO_PORT(AA, 0, 0, 8),
+	TEGRA264_AON_GPIO_PORT(BB, 0, 1, 2),
+	TEGRA264_AON_GPIO_PORT(CC, 0, 2, 8),
+	TEGRA264_AON_GPIO_PORT(DD, 0, 3, 8),
+	TEGRA264_AON_GPIO_PORT(EE, 0, 4, 2)
+};
+
+static const struct tegra_gpio_soc tegra268_aon_soc = {
+	.num_ports = ARRAY_SIZE(tegra268_aon_ports),
+	.ports = tegra268_aon_ports,
+	.name = "tegra268-gpio-aon",
+	.instance = 1,
+	.num_irqs_per_bank = 8,
+	.has_gte = true,
+	.has_vm_support = true,
+};
+
+static const struct tegra_gpio_port tegra268_uphy_ports[] = {
+	TEGRA264_UPHY_GPIO_PORT(A, 0, 0, 6),
+	TEGRA264_UPHY_GPIO_PORT(B, 0, 1, 8),
+	TEGRA264_UPHY_GPIO_PORT(C, 0, 2, 3),
+	TEGRA264_UPHY_GPIO_PORT(D, 1, 0, 6),
+	TEGRA264_UPHY_GPIO_PORT(E, 1, 1, 4),
+};
+
+static const struct tegra_gpio_soc tegra268_uphy_soc = {
+	.num_ports = ARRAY_SIZE(tegra268_uphy_ports),
+	.ports = tegra268_uphy_ports,
+	.name = "tegra268-gpio-uphy",
+	.instance = 2,
+	.num_irqs_per_bank = 8,
+	.has_vm_support = true,
+};
+
 #define TEGRA256_MAIN_GPIO_PORT(_name, _bank, _port, _pins) \
 	TEGRA_GPIO_PORT(TEGRA256_MAIN, _name, _bank, _port, _pins)
 
@@ -1528,6 +1593,15 @@ static const struct of_device_id tegra186_gpio_of_match[] = {
 	}, {
 		.compatible = "nvidia,tegra264-gpio-uphy",
 		.data = &tegra264_uphy_soc
+	}, {
+		.compatible = "nvidia,tegra268-gpio",
+		.data = &tegra268_main_soc
+	}, {
+		.compatible = "nvidia,tegra268-gpio-aon",
+		.data = &tegra268_aon_soc
+	}, {
+		.compatible = "nvidia,tegra268-gpio-uphy",
+		.data = &tegra268_uphy_soc
 	}, {
 		/* sentinel */
 	}
diff --git a/drivers/gpio/gpio-tps65219.c b/drivers/gpio/gpio-tps65219.c
index 457fd8a589e892..6958466455d04c 100644
--- a/drivers/gpio/gpio-tps65219.c
+++ b/drivers/gpio/gpio-tps65219.c
@@ -79,7 +79,7 @@ static int tps65219_gpio_get(struct gpio_chip *gc, unsigned int offset)
 	if (ret)
 		return ret;
 
-	ret = !!(val & BIT(TPS65219_MFP_GPIO_STATUS_MASK));
+	ret = !!(val & TPS65219_MFP_GPIO_STATUS_MASK);
 	dev_warn(dev, "GPIO%d = %d, MULTI_DEVICE_ENABLE, not a standard GPIO\n", offset, ret);
 
 	/*
@@ -87,7 +87,7 @@ static int tps65219_gpio_get(struct gpio_chip *gc, unsigned int offset)
 	 * status bit.
 	 */
 
-	if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT)
+	if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT)
 		return -ENOTSUPP;
 
 	return ret;
@@ -158,8 +158,10 @@ static int tps65214_gpio_change_direction(struct gpio_chip *gc, unsigned int off
 	if (ret)
 		dev_err(dev, "GPIO%d configured as VSEL, not GPIO\n", offset);
 
+	val = direction == GPIO_LINE_DIRECTION_OUT ?
+		TPS65214_GPIO0_DIR_MASK : 0;
 	ret = regmap_update_bits(gpio->tps->regmap, TPS65219_REG_GENERAL_CONFIG,
-				 TPS65214_GPIO0_DIR_MASK, direction);
+				 TPS65214_GPIO0_DIR_MASK, val);
 	if (ret)
 		dev_err(dev, "Fail to change direction to %u for GPIO%d.\n", direction, offset);
 
@@ -176,7 +178,7 @@ static int tps65219_gpio_direction_input(struct gpio_chip *gc, unsigned int offs
 		return -ENOTSUPP;
 	}
 
-	if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_IN)
+	if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_IN)
 		return 0;
 
 	return gpio->change_dir(gc, offset, GPIO_LINE_DIRECTION_IN);
@@ -190,7 +192,7 @@ static int tps65219_gpio_direction_output(struct gpio_chip *gc, unsigned int off
 	if (offset != TPS6521X_GPIO0_IDX)
 		return 0;
 
-	if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT)
+	if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT)
 		return 0;
 
 	return gpio->change_dir(gc, offset, GPIO_LINE_DIRECTION_OUT);
diff --git a/drivers/gpio/gpio-wcove.c b/drivers/gpio/gpio-wcove.c
index 4a5e20e936a95a..937c5db7e6e49c 100644
--- a/drivers/gpio/gpio-wcove.c
+++ b/drivers/gpio/gpio-wcove.c
@@ -129,10 +129,7 @@ static void wcove_update_irq_mask(struct wcove_gpio *wg, irq_hw_number_t gpio)
 {
 	unsigned int mask, reg = to_ireg(gpio, IRQ_MASK, &mask);
 
-	if (wg->set_irq_mask)
-		regmap_set_bits(wg->regmap, reg, mask);
-	else
-		regmap_clear_bits(wg->regmap, reg, mask);
+	regmap_assign_bits(wg->regmap, reg, mask, wg->set_irq_mask);
 }
 
 static void wcove_update_irq_ctrl(struct wcove_gpio *wg, irq_hw_number_t gpio)
diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c
index 532205175827a0..9406ce85f93308 100644
--- a/drivers/gpio/gpio-xilinx.c
+++ b/drivers/gpio/gpio-xilinx.c
@@ -271,14 +271,7 @@ static void xgpio_save_regs(struct xgpio_instance *chip)
 
 static int xgpio_request(struct gpio_chip *chip, unsigned int offset)
 {
-	int ret;
-
-	ret = pm_runtime_get_sync(chip->parent);
-	/*
-	 * If the device is already active pm_runtime_get() will return 1 on
-	 * success, but gpio_request still needs to return 0.
-	 */
-	return ret < 0 ? ret : 0;
+	return pm_runtime_resume_and_get(chip->parent);
 }
 
 static void xgpio_free(struct gpio_chip *chip, unsigned int offset)
diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c
index 15a79d9a2e9eb4..13e4c5f0e297ce 100644
--- a/drivers/gpio/gpio-zynq.c
+++ b/drivers/gpio/gpio-zynq.c
@@ -798,15 +798,7 @@ static int zynq_gpio_runtime_resume(struct device *dev)
 
 static int zynq_gpio_request(struct gpio_chip *chip, unsigned int offset)
 {
-	int ret;
-
-	ret = pm_runtime_get_sync(chip->parent);
-
-	/*
-	 * If the device is already active pm_runtime_get() will return 1 on
-	 * success, but gpio_request still needs to return 0.
-	 */
-	return ret < 0 ? ret : 0;
+	return pm_runtime_resume_and_get(chip->parent);
 }
 
 static void zynq_gpio_free(struct gpio_chip *chip, unsigned int offset)
diff --git a/drivers/gpio/gpiolib-acpi-quirks.c b/drivers/gpio/gpiolib-acpi-quirks.c
index a0116f004975ae..54150b161a4b28 100644
--- a/drivers/gpio/gpiolib-acpi-quirks.c
+++ b/drivers/gpio/gpiolib-acpi-quirks.c
@@ -392,6 +392,32 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] __initconst = {
 			.ignore_wake = "VEN_0488:00@355",
 		},
 	},
+	{
+		/*
+		 * Lenovo IdeaPad Slim 3 15ABR8: the touchscreen wake IRQ fires as soon
+		 * as the system enters suspend, causing an immediate spurious resume.
+		 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "82XM"),
+		},
+		.driver_data = &(struct acpi_gpiolib_dmi_quirk) {
+			.ignore_wake = "GTCH7503:00@10",
+		},
+	},
+	{
+		/*
+		 * Spurious wakeups from GPP3 PCIe bridge interrupt
+		 * Found in BIOS UX425UA.301
+		 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX425UA_UM425UA"),
+		},
+		.driver_data = &(struct acpi_gpiolib_dmi_quirk) {
+			.ignore_wake = "AMDI0030:00@24",
+		},
+	},
 	{} /* Terminating entry */
 };
 
diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c
index d1105b7ae43777..a30564d1525adf 100644
--- a/drivers/gpio/gpiolib-cdev.c
+++ b/drivers/gpio/gpiolib-cdev.c
@@ -377,11 +377,11 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip)
 	FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC,
 		   anon_inode_getfile("gpio-linehandle", &linehandle_fileops,
 				      lh, O_RDONLY | O_CLOEXEC));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 	retain_and_null_ptr(lh);
 
-	handlereq.fd = fd_prepare_fd(fdf);
+	handlereq.fd = fdf->fd;
 	if (copy_to_user(ip, &handlereq, sizeof(handlereq)))
 		return -EFAULT;
 
@@ -715,6 +715,8 @@ static int hte_edge_setup(struct line *line, u64 eflags)
 			   line->desc);
 
 	ret = hte_ts_get(NULL, hdesc, 0);
+	if (ret == -ENODEV)
+		return -EOPNOTSUPP;
 	if (ret)
 		return ret;
 
@@ -1715,11 +1717,11 @@ static int linereq_create(struct gpio_device *gdev, void __user *ip)
 	FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC,
 		   anon_inode_getfile("gpio-line", &line_fileops, lr,
 				      O_RDONLY | O_CLOEXEC));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 	retain_and_null_ptr(lr);
 
-	ulr.fd = fd_prepare_fd(fdf);
+	ulr.fd = fdf->fd;
 	if (copy_to_user(ip, &ulr, sizeof(ulr)))
 		return -EFAULT;
 
@@ -2115,11 +2117,11 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 	FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC,
 		   anon_inode_getfile("gpio-event", &lineevent_fileops, le,
 				      O_RDONLY | O_CLOEXEC));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 	retain_and_null_ptr(le);
 
-	eventreq.fd = fd_prepare_fd(fdf);
+	eventreq.fd = fdf->fd;
 	if (copy_to_user(ip, &eventreq, sizeof(eventreq)))
 		return -EFAULT;
 
@@ -2172,18 +2174,19 @@ static void gpio_v2_line_info_changed_to_v1(
 
 #endif /* CONFIG_GPIO_CDEV_V1 */
 
-static void gpio_desc_to_lineinfo(struct gpio_desc *desc,
-				  struct gpio_v2_line_info *info, bool atomic)
+static int gpio_desc_to_lineinfo(struct gpio_desc *desc,
+				 struct gpio_v2_line_info *info, bool atomic)
 {
 	u32 debounce_period_us;
 	unsigned long dflags;
 	const char *label;
 
+	memset(info, 0, sizeof(*info));
+
 	CLASS(gpio_chip_guard, guard)(desc);
 	if (!guard.gc)
-		return;
+		return -ENODEV;
 
-	memset(info, 0, sizeof(*info));
 	info->offset = gpiod_hwgpio(desc);
 
 	if (desc->name)
@@ -2258,6 +2261,8 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc,
 							debounce_period_us;
 		info->num_attrs++;
 	}
+
+	return 0;
 }
 
 struct gpio_chardev_data {
@@ -2309,6 +2314,7 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip,
 	struct gpio_desc *desc;
 	struct gpioline_info lineinfo;
 	struct gpio_v2_line_info lineinfo_v2;
+	int ret;
 
 	if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
 		return -EFAULT;
@@ -2326,7 +2332,10 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip,
 			return -EBUSY;
 	}
 
-	gpio_desc_to_lineinfo(desc, &lineinfo_v2, false);
+	ret = gpio_desc_to_lineinfo(desc, &lineinfo_v2, false);
+	if (ret)
+		return ret;
+
 	gpio_v2_line_info_to_v1(&lineinfo_v2, &lineinfo);
 
 	if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) {
@@ -2344,6 +2353,7 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip,
 {
 	struct gpio_desc *desc;
 	struct gpio_v2_line_info lineinfo;
+	int ret;
 
 	if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
 		return -EFAULT;
@@ -2363,7 +2373,10 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip,
 		if (test_and_set_bit(lineinfo.offset, cdev->watched_lines))
 			return -EBUSY;
 	}
-	gpio_desc_to_lineinfo(desc, &lineinfo, false);
+
+	ret = gpio_desc_to_lineinfo(desc, &lineinfo, false);
+	if (ret)
+		return ret;
 
 	if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) {
 		if (watch)
@@ -2489,6 +2502,7 @@ static int lineinfo_changed_notify(struct notifier_block *nb,
 	struct lineinfo_changed_ctx *ctx;
 	struct gpio_desc *desc = data;
 	struct file *fp;
+	int ret;
 
 	if (!test_bit(gpiod_hwgpio(desc), cdev->watched_lines))
 		return NOTIFY_DONE;
@@ -2519,7 +2533,13 @@ static int lineinfo_changed_notify(struct notifier_block *nb,
 
 	ctx->chg.event_type = action;
 	ctx->chg.timestamp_ns = ktime_get_ns();
-	gpio_desc_to_lineinfo(desc, &ctx->chg.info, true);
+
+	ret = gpio_desc_to_lineinfo(desc, &ctx->chg.info, true);
+	if (ret) {
+		fput(fp);
+		return NOTIFY_DONE;
+	}
+
 	/* Keep the GPIO device alive until we emit the event. */
 	ctx->gdev = gpio_device_get(desc->gdev);
 	ctx->cdev = cdev;
diff --git a/drivers/gpio/gpiolib-kunit.c b/drivers/gpio/gpiolib-kunit.c
index e6cb43a8df5b8e..90b7e77ef0b51f 100644
--- a/drivers/gpio/gpiolib-kunit.c
+++ b/drivers/gpio/gpiolib-kunit.c
@@ -564,13 +564,6 @@ static int gpio_unbind_notify(struct notifier_block *nb, unsigned long action,
 	return NOTIFY_OK;
 }
 
-static void gpio_unbind_unregister_notifier(void *data)
-{
-	struct notifier_block *nb = data;
-
-	blocking_notifier_chain_unregister(&gpio_unbind_notifier, nb);
-}
-
 static int gpio_unbind_consumer_probe(struct platform_device *pdev)
 {
 	struct gpio_unbind_consumer_drvdata *data;
@@ -588,11 +581,8 @@ static int gpio_unbind_consumer_probe(struct platform_device *pdev)
 		return PTR_ERR(data->desc);
 
 	data->nb.notifier_call = gpio_unbind_notify;
-	ret = blocking_notifier_chain_register(&gpio_unbind_notifier, &data->nb);
-	if (ret)
-		return ret;
-
-	ret = devm_add_action_or_reset(dev, gpio_unbind_unregister_notifier, &data->nb);
+	ret = devm_blocking_notifier_chain_register(dev, &gpio_unbind_notifier,
+						    &data->nb);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 66d2325bfae873..f9173560812d00 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -1029,6 +1029,13 @@ int gpiochip_add_hog(struct gpio_chip *gc, struct fwnode_handle *fwnode)
 			ret = of_gpiochip_get_lflags(gc, &gpiospec, &lflags);
 			if (ret)
 				return ret;
+
+			/*
+			 * If no line-name property is present, fall back to the OF
+			 * node name as in the previous implementation.
+			 */
+			if (!name)
+				name = to_of_node(fwnode)->name;
 		} else {
 			/*
 			 * GPIO_ACTIVE_LOW is currently the only lookup flag
@@ -2440,6 +2447,27 @@ int gpiochip_generic_config(struct gpio_chip *gc, unsigned int offset,
 }
 EXPORT_SYMBOL_GPL(gpiochip_generic_config);
 
+/**
+ * gpiochip_generic_get_config() - read back the configuration of a pin
+ * @gc: the gpiochip owning the GPIO
+ * @offset: the offset of the GPIO to query
+ * @config: the packed parameter to query, replaced by its bare argument
+ *
+ * Returns:
+ * 0 on success, or negative errno on failure.
+ */
+int gpiochip_generic_get_config(struct gpio_chip *gc, unsigned int offset,
+				unsigned long *config)
+{
+#ifdef CONFIG_PINCTRL
+	if (list_empty(&gc->gpiodev->pin_ranges))
+		return -ENOTSUPP;
+#endif
+
+	return pinctrl_gpio_get_config(gc, offset, config);
+}
+EXPORT_SYMBOL_GPL(gpiochip_generic_get_config);
+
 #ifdef CONFIG_PINCTRL
 
 /**
diff --git a/drivers/gpu/buddy.c b/drivers/gpu/buddy.c
index 26e7a48b55f332..2f2aaadafe3511 100644
--- a/drivers/gpu/buddy.c
+++ b/drivers/gpu/buddy.c
@@ -3,11 +3,14 @@
  * Copyright © 2021 Intel Corporation
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/bug.h>
 #include <linux/export.h>
 #include <linux/kmemleak.h>
 #include <linux/module.h>
 #include <linux/sizes.h>
+#include <linux/slab.h>
 
 #include <linux/gpu_buddy.h>
 
@@ -34,6 +37,430 @@
 #endif
 
 static struct kmem_cache *slab_blocks;
+static struct kmem_cache *slab_extents;
+
+/*
+ * Dirty tracker
+ * -------------
+ *
+ * The dirty tracker maintains an augmented interval rbtree of contiguous
+ * dirty address ranges, decoupled from the buddy free trees.
+ * Each node covers a maximal coalesced run; adjacent extents are merged
+ * on insertion so the tree always holds the smallest possible number of
+ * extents.  The augmentation field @subtree_max_size lets the allocator
+ * locate the largest dirty extent in O(log E).
+ *
+ * Free trees (mm->free_tree[])
+ * ----------------------------
+ *
+ * Per-order augmented rbtrees of FREE buddy blocks, keyed by offset.
+ * Every node carries:
+ *   - subtree_max_alignment: largest natural alignment in the subtree,
+ *     used by aligned/range allocations to skip unsuitable subtrees in
+ *     O(log N).
+ *   - subtree_block_state: the highest clear class (DIRTY < MIXED < CLEAR)
+ *     of any block in the subtree, maintained as a max augment. A value of
+ *     >= MIXED means a clear-or-mixed block exists; == CLEAR means a
+ *     fully-clear block exists.
+ *
+ * Block classes
+ * -------------
+ *
+ * Each FREE block falls into one of three classes, determined in
+ * mark_free() by querying the dirty tracker for the block's range:
+ *
+ *   clear   -- HEADER_CLEAR set; no dirty extent overlaps the range.
+ *   mixed   -- HEADER_CLEAR unset; range has both dirty and clear bytes.
+ *   dirty   -- HEADER_CLEAR unset; range is fully dirty.
+ *
+ * Clear allocation
+ * ----------------
+ *
+ * A clear (CLEAR_ALLOCATION) request prefers clear -> mixed -> dirty.
+ * Climbing from the requested order up to max_order, rbtree_last_clear_free_block()
+ * returns, in one O(log N) descent per order, the right-most clear-or-mixed block
+ * (fully-clear preferred over mixed) at the lowest order whose free tree contains
+ * such a block. Only if no clear-or-mixed block exists at any order >= the
+ * requested one does it fall back to a dirty block.
+ *
+ * Clear state is reported to the driver per whole block via HEADER_CLEAR, so a
+ * fully-clear block of the requested order lets the driver skip the clear pass.
+ *
+ * The effective allocation preference depends on how the driver handles
+ * freed blocks:
+ *
+ *   1) Never clear on free:
+ *      No free block contains clear bytes, so clear allocations always
+ *      fall back to dirty blocks.
+ *
+ *   2) Always clear on free:
+ *      Freed blocks become clear while untouched blocks remain dirty.
+ *      Merging clear and dirty buddies produces mixed blocks, which are
+ *      reclassified when split. Over time, clear blocks become dominant,
+ *      so clear allocations are typically satisfied from clear blocks,
+ *      following a clear -> mixed -> dirty preference.
+ *
+ *   3) Selective clear on free:
+ *      For each order examined, fully-clear blocks are preferred over
+ *      mixed blocks, and mixed blocks are preferred over dirty blocks.
+ *      If a clear or mixed block is found at an order, it is selected
+ *      without searching higher orders. Dirty blocks are used only when
+ *      no clear or mixed block exists at any eligible order.
+ */
+
+static u64 extent_size(struct gpu_dirty_extent *dirty_extent)
+{
+	return dirty_extent->end - dirty_extent->start;
+}
+
+RB_DECLARE_CALLBACKS_MAX(static, gpu_dirty_augment_cb,
+			 struct gpu_dirty_extent, rb,
+			 u64, subtree_max_size,
+			 extent_size)
+
+static struct gpu_dirty_extent *extent_alloc(struct gpu_dirty_tracker *dirty_tracker)
+{
+	return kmem_cache_alloc(slab_extents, GFP_KERNEL);
+}
+
+static void extent_free(struct gpu_dirty_tracker *dirty_tracker,
+			struct gpu_dirty_extent *dirty_extent)
+{
+	kmem_cache_free(slab_extents, dirty_extent);
+}
+
+/* Return the rightmost extent whose start is strictly below @offset. */
+static struct gpu_dirty_extent *
+prev_extent(struct gpu_dirty_tracker *dirty_tracker, u64 offset)
+{
+	struct rb_node *rb = dirty_tracker->root.rb_node;
+	struct gpu_dirty_extent *dirty_extent = NULL;
+
+	while (rb) {
+		struct gpu_dirty_extent *tmp_extent =
+			rb_entry(rb, struct gpu_dirty_extent, rb);
+
+		if (tmp_extent->start < offset) {
+			dirty_extent = tmp_extent;
+			rb = rb->rb_right;
+		} else {
+			rb = rb->rb_left;
+		}
+	}
+
+	return dirty_extent;
+}
+
+/* Return the leftmost extent whose start is at or above @offset. */
+static struct gpu_dirty_extent *
+next_extent(struct gpu_dirty_tracker *dirty_tracker, u64 offset)
+{
+	struct rb_node *rb = dirty_tracker->root.rb_node;
+	struct gpu_dirty_extent *dirty_extent = NULL;
+
+	while (rb) {
+		struct gpu_dirty_extent *tmp_extent =
+			rb_entry(rb, struct gpu_dirty_extent, rb);
+
+		if (tmp_extent->start >= offset) {
+			dirty_extent = tmp_extent;
+			rb = rb->rb_left;
+		} else {
+			rb = rb->rb_right;
+		}
+	}
+
+	return dirty_extent;
+}
+
+static void insert_extent(struct gpu_dirty_tracker *dirty_tracker,
+			  struct gpu_dirty_extent *dirty_extent)
+{
+	struct rb_node **link = &dirty_tracker->root.rb_node;
+	struct rb_node *parent = NULL;
+	u64 size = extent_size(dirty_extent);
+
+	while (*link) {
+		struct gpu_dirty_extent *tmp_extent;
+
+		parent = *link;
+		tmp_extent = rb_entry(parent, struct gpu_dirty_extent, rb);
+
+		if (tmp_extent->subtree_max_size < size)
+			tmp_extent->subtree_max_size = size;
+
+		if (dirty_extent->start < tmp_extent->start)
+			link = &parent->rb_left;
+		else
+			link = &parent->rb_right;
+	}
+
+	dirty_extent->subtree_max_size = size;
+	rb_link_node(&dirty_extent->rb, parent, link);
+	rb_insert_augmented(&dirty_extent->rb, &dirty_tracker->root, &gpu_dirty_augment_cb);
+}
+
+static void remove_extent(struct gpu_dirty_tracker *dirty_tracker,
+			  struct gpu_dirty_extent *dirty_extent)
+{
+	rb_erase_augmented(&dirty_extent->rb, &dirty_tracker->root, &gpu_dirty_augment_cb);
+	RB_CLEAR_NODE(&dirty_extent->rb);
+}
+
+static void gpu_dirty_tracker_init(struct gpu_dirty_tracker *dirty_tracker)
+{
+	dirty_tracker->root = RB_ROOT;
+	dirty_tracker->total_dirty = 0;
+}
+
+static void gpu_dirty_tracker_empty(struct gpu_dirty_tracker *dirty_tracker)
+{
+	struct rb_node *rb;
+
+	while ((rb = rb_first(&dirty_tracker->root))) {
+		struct gpu_dirty_extent *dirty_extent =
+			rb_entry(rb, struct gpu_dirty_extent, rb);
+
+		remove_extent(dirty_tracker, dirty_extent);
+		extent_free(dirty_tracker, dirty_extent);
+	}
+
+	dirty_tracker->total_dirty = 0;
+}
+
+static void gpu_dirty_tracker_fini(struct gpu_dirty_tracker *dirty_tracker)
+{
+	gpu_dirty_tracker_empty(dirty_tracker);
+}
+
+/*
+ * Mark the range [start, start + size] as dirty. Merge with the neighbour on
+ * each side if they are contiguous, so the tree never holds two adjacent ranges.
+ */
+static void gpu_dirty_tracker_mark_dirty(struct gpu_dirty_tracker *dirty_tracker,
+					 u64 start, u64 size)
+{
+	struct gpu_dirty_extent *left, *right, *dirty_extent;
+	u64 end = start + size;
+
+	gpu_buddy_assert(size);
+
+	/* Find contiguous neighbours, if any. */
+	left = prev_extent(dirty_tracker, start);
+	if (left && left->end != start)
+		left = NULL;
+
+	right = next_extent(dirty_tracker, end);
+	if (right && right->start != end)
+		right = NULL;
+
+	if (left && right) {
+		/* Merge left + new + right into a single extent. */
+		remove_extent(dirty_tracker, left);
+		remove_extent(dirty_tracker, right);
+		left->end = right->end;
+		extent_free(dirty_tracker, right);
+		insert_extent(dirty_tracker, left);
+	} else if (left) {
+		/* Extend left neighbour rightwards. */
+		remove_extent(dirty_tracker, left);
+		left->end = end;
+		insert_extent(dirty_tracker, left);
+	} else if (right) {
+		/* Extend right neighbour leftwards. */
+		remove_extent(dirty_tracker, right);
+		right->start = start;
+		insert_extent(dirty_tracker, right);
+	} else {
+		/* Standalone extent. */
+		dirty_extent = extent_alloc(dirty_tracker);
+		if (!dirty_extent) {
+			pr_warn_once("dirty extent allocation failed, skipping tracker update\n");
+			return;
+		}
+		dirty_extent->start = start;
+		dirty_extent->end   = end;
+		insert_extent(dirty_tracker, dirty_extent);
+	}
+
+	dirty_tracker->total_dirty += size;
+}
+
+/*
+ * Remove the range [start, start + size] from the dirty tracker. Punch the
+ * range out of every overlapping dirty extent, splitting one extent in two if
+ * the removed range falls strictly inside it.
+ */
+static void gpu_dirty_tracker_remove_range(struct gpu_dirty_tracker *dirty_tracker,
+					   u64 start, u64 size)
+{
+	struct gpu_dirty_extent *dirty_extent, *next;
+	u64 end = start + size;
+
+	gpu_buddy_assert(size);
+
+	dirty_extent = prev_extent(dirty_tracker, start + 1);
+	if (!dirty_extent)
+		dirty_extent = next_extent(dirty_tracker, start);
+
+	while (dirty_extent && dirty_extent->start < end) {
+		struct rb_node *next_node = rb_next(&dirty_extent->rb);
+		u64 extent_start = dirty_extent->start;
+		u64 extent_end = dirty_extent->end;
+
+		if (next_node)
+			next = rb_entry(next_node, struct gpu_dirty_extent, rb);
+		else
+			next = NULL;
+
+		/* Skip a non-overlapping neighbour returned by prev_extent(). */
+		if (extent_end <= start) {
+			dirty_extent = next;
+			continue;
+		}
+
+		if (extent_start < start && extent_end > end) {
+			/*
+			 * Removed range lies strictly inside this dirty extent:
+			 * split it into the dirty left and right halves.
+			 */
+			struct gpu_dirty_extent *right = extent_alloc(dirty_tracker);
+
+			if (!right) {
+				pr_warn_once("dirty extent allocation failed, skipping tracker update\n");
+				dirty_extent = next;
+				continue;
+			}
+
+			remove_extent(dirty_tracker, dirty_extent);
+
+			dirty_extent->end = start;
+			right->start = end;
+			right->end   = extent_end;
+
+			insert_extent(dirty_tracker, dirty_extent);
+			insert_extent(dirty_tracker, right);
+
+			dirty_tracker->total_dirty -= size;
+		} else if (extent_start >= start && extent_end <= end) {
+			/* Extent fully covered: drop it. */
+			remove_extent(dirty_tracker, dirty_extent);
+			extent_free(dirty_tracker, dirty_extent);
+
+			dirty_tracker->total_dirty -= (extent_end - extent_start);
+		} else if (extent_start < start) {
+			/* Extent overlaps from the left: trim its right end. */
+			remove_extent(dirty_tracker, dirty_extent);
+			dirty_extent->end = start;
+			insert_extent(dirty_tracker, dirty_extent);
+
+			dirty_tracker->total_dirty -= (extent_end - start);
+		} else {
+			/* Extent overlaps from the right: trim its left end. */
+			remove_extent(dirty_tracker, dirty_extent);
+			dirty_extent->start = end;
+			insert_extent(dirty_tracker, dirty_extent);
+
+			dirty_tracker->total_dirty -= (end - extent_start);
+		}
+
+		dirty_extent = next;
+	}
+}
+
+static enum gpu_block_state
+gpu_dirty_range_state(struct gpu_dirty_tracker *dirty_tracker,
+		      u64 start, u64 size)
+{
+	struct gpu_dirty_extent *dirty_extent;
+	u64 end = start + size;
+
+	dirty_extent = prev_extent(dirty_tracker, start + 1);
+	if (dirty_extent) {
+		if (dirty_extent->start <= start && dirty_extent->end >= end)
+			return GPU_BLOCK_DIRTY;
+		if (dirty_extent->start < end && dirty_extent->end > start)
+			return GPU_BLOCK_MIXED;
+	}
+
+	dirty_extent = next_extent(dirty_tracker, start);
+	if (dirty_extent && dirty_extent->start < end)
+		return GPU_BLOCK_MIXED;
+
+	return GPU_BLOCK_CLEAR;
+}
+
+static struct rb_node *
+dirty_tracker_descend_right(struct rb_node *node, u64 min_size)
+{
+	while (node->rb_right) {
+		struct gpu_dirty_extent *tmp_extent;
+
+		tmp_extent = rb_entry(node->rb_right, struct gpu_dirty_extent, rb);
+
+		if (tmp_extent->subtree_max_size < min_size)
+			break;
+		node = node->rb_right;
+	}
+
+	return node;
+}
+
+static struct gpu_dirty_extent *
+gpu_dirty_tracker_find(struct gpu_dirty_tracker *dirty_tracker,
+		       u64 min_size, u64 *aligned_start_out)
+{
+	struct rb_node *rb = dirty_tracker->root.rb_node;
+	struct gpu_dirty_extent *root_extent;
+	struct rb_node *parent;
+
+	if (!min_size || !is_power_of_2(min_size))
+		return NULL;
+
+	if (!rb)
+		return NULL;
+
+	root_extent = rb_entry(rb, struct gpu_dirty_extent, rb);
+	if (root_extent->subtree_max_size < min_size)
+		return NULL;
+
+	rb = dirty_tracker_descend_right(rb, min_size);
+
+	while (rb) {
+		struct gpu_dirty_extent *dirty_extent;
+		u64 aligned_start;
+
+		dirty_extent = rb_entry(rb, struct gpu_dirty_extent, rb);
+		aligned_start = ALIGN(dirty_extent->start, min_size);
+
+		/* Check if a min_size block fits after the alignment skip. */
+		if (aligned_start <= dirty_extent->end &&
+		    dirty_extent->end - aligned_start >= min_size) {
+			*aligned_start_out = aligned_start;
+			return dirty_extent;
+		}
+
+		if (rb->rb_left) {
+			struct gpu_dirty_extent *tmp_extent;
+
+			tmp_extent = rb_entry(rb->rb_left, struct gpu_dirty_extent, rb);
+			if (tmp_extent->subtree_max_size >= min_size) {
+				rb = dirty_tracker_descend_right(rb->rb_left, min_size);
+				continue;
+			}
+		}
+
+		/* Walk up until we exit a node via its right child. */
+		parent = rb_parent(rb);
+		while (parent && parent->rb_right != rb) {
+			rb = parent;
+			parent = rb_parent(rb);
+		}
+		rb = parent;
+	}
+
+	return NULL;
+}
 
 static unsigned int
 gpu_buddy_block_state(struct gpu_buddy_block *block)
@@ -67,10 +494,97 @@ static unsigned int gpu_buddy_block_offset_alignment(struct gpu_buddy_block *blo
 	return __ffs64(offset);
 }
 
-RB_DECLARE_CALLBACKS_MAX(static, gpu_buddy_augment_cb,
-			 struct gpu_buddy_block, rb,
-			 unsigned int, subtree_max_alignment,
-			 gpu_buddy_block_offset_alignment);
+static inline enum gpu_block_state
+gpu_block_cached_state(struct gpu_buddy_block *block)
+{
+	if (gpu_buddy_block_is_clear(block))
+		return GPU_BLOCK_CLEAR;
+	if (block->has_clear)
+		return GPU_BLOCK_MIXED;
+	return GPU_BLOCK_DIRTY;
+}
+
+static inline void gpu_buddy_augment_compute(struct gpu_buddy_block *block)
+{
+	enum gpu_block_state block_state;
+	struct gpu_buddy_block *right;
+	struct gpu_buddy_block *left;
+	unsigned int max_align;
+
+	max_align = gpu_buddy_block_offset_alignment(block);
+	block_state = gpu_block_cached_state(block);
+
+	left = rb_entry_safe(block->rb.rb_left, struct gpu_buddy_block, rb);
+	if (left) {
+		if (left->subtree_max_alignment > max_align)
+			max_align = left->subtree_max_alignment;
+
+		block_state = max(block_state, left->subtree_block_state);
+	}
+
+	right = rb_entry_safe(block->rb.rb_right, struct gpu_buddy_block, rb);
+	if (right) {
+		if (right->subtree_max_alignment > max_align)
+			max_align = right->subtree_max_alignment;
+
+		block_state = max(block_state, right->subtree_block_state);
+	}
+
+	block->subtree_max_alignment = max_align;
+	block->subtree_block_state = block_state;
+}
+
+static void gpu_buddy_augment_propagate(struct rb_node *rb, struct rb_node *stop)
+{
+	while (rb != stop) {
+		struct gpu_buddy_block *block;
+		unsigned int old_align;
+		enum gpu_block_state old_block_state;
+
+		block = rb_entry(rb, struct gpu_buddy_block, rb);
+		old_align = block->subtree_max_alignment;
+		old_block_state = block->subtree_block_state;
+
+		gpu_buddy_augment_compute(block);
+		if (block->subtree_max_alignment == old_align &&
+		    block->subtree_block_state == old_block_state)
+			break;
+
+		rb = rb_parent(&block->rb);
+	}
+}
+
+static void gpu_buddy_augment_copy(struct rb_node *rb_old, struct rb_node *rb_new)
+{
+	struct gpu_buddy_block *old;
+	struct gpu_buddy_block *new;
+
+	old = rb_entry(rb_old, struct gpu_buddy_block, rb);
+	new = rb_entry(rb_new, struct gpu_buddy_block, rb);
+
+	new->subtree_max_alignment = old->subtree_max_alignment;
+	new->subtree_block_state = old->subtree_block_state;
+}
+
+static void gpu_buddy_augment_rotate(struct rb_node *rb_old, struct rb_node *rb_new)
+{
+	struct gpu_buddy_block *old;
+	struct gpu_buddy_block *new;
+
+	old = rb_entry(rb_old, struct gpu_buddy_block, rb);
+	new = rb_entry(rb_new, struct gpu_buddy_block, rb);
+
+	new->subtree_max_alignment = old->subtree_max_alignment;
+	new->subtree_block_state = old->subtree_block_state;
+
+	gpu_buddy_augment_compute(old);
+}
+
+static const struct rb_augment_callbacks gpu_buddy_augment_cb = {
+	.propagate = gpu_buddy_augment_propagate,
+	.copy      = gpu_buddy_augment_copy,
+	.rotate    = gpu_buddy_augment_rotate,
+};
 
 static struct gpu_buddy_block *gpu_block_alloc(struct gpu_buddy *mm,
 					       struct gpu_buddy_block *parent,
@@ -101,13 +615,6 @@ static void gpu_block_free(struct gpu_buddy *mm,
 	kmem_cache_free(slab_blocks, block);
 }
 
-static enum gpu_buddy_free_tree
-get_block_tree(struct gpu_buddy_block *block)
-{
-	return gpu_buddy_block_is_clear(block) ?
-	       GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
-}
-
 static struct gpu_buddy_block *
 rbtree_get_free_block(const struct rb_node *node)
 {
@@ -120,24 +627,59 @@ rbtree_last_free_block(struct rb_root *root)
 	return rbtree_get_free_block(rb_last(root));
 }
 
-static bool rbtree_is_empty(struct rb_root *root)
+static struct gpu_buddy_block *
+rbtree_last_clear_free_block(struct rb_root *root,
+			     enum gpu_block_state min_block_state)
 {
-	return RB_EMPTY_ROOT(root);
+	struct rb_node *node = root->rb_node;
+	struct gpu_buddy_block *block = NULL;
+	struct gpu_buddy_block *root_block;
+	enum gpu_block_state target_state;
+
+	root_block = rbtree_get_free_block(node);
+	if (!root_block || root_block->subtree_block_state < min_block_state)
+		return NULL;
+
+	target_state = root_block->subtree_block_state;
+
+	while (node) {
+		struct gpu_buddy_block *right_block;
+		struct gpu_buddy_block *node_block;
+
+		node_block = rbtree_get_free_block(node);
+		right_block = rbtree_get_free_block(node->rb_right);
+
+		if (right_block && right_block->subtree_block_state >= target_state) {
+			node = node->rb_right;
+			continue;
+		}
+
+		if (gpu_block_cached_state(node_block) == target_state) {
+			block = node_block;
+			break;
+		}
+
+		node = node->rb_left;
+	}
+
+	return block;
 }
 
 static void rbtree_insert(struct gpu_buddy *mm,
-			  struct gpu_buddy_block *block,
-			  enum gpu_buddy_free_tree tree)
+			  struct gpu_buddy_block *block)
 {
 	struct rb_node **link, *parent = NULL;
-	unsigned int block_alignment, order;
+	enum gpu_block_state block_state;
 	struct gpu_buddy_block *node;
+	unsigned int block_alignment;
 	struct rb_root *root;
+	unsigned int order;
 
 	order = gpu_buddy_block_order(block);
 	block_alignment = gpu_buddy_block_offset_alignment(block);
+	block_state = gpu_block_cached_state(block);
 
-	root = &mm->free_trees[tree][order];
+	root = &mm->free_tree[order];
 	link = &root->rb_node;
 
 	while (*link) {
@@ -147,10 +689,12 @@ static void rbtree_insert(struct gpu_buddy *mm,
 		 * Manual augmentation update during insertion traversal. Required
 		 * because rb_insert_augmented() only calls rotate callback during
 		 * rotations. This ensures all ancestors on the insertion path have
-		 * correct subtree_max_alignment values.
+		 * correct subtree_max_alignment / subtree_block_state values.
 		 */
 		if (node->subtree_max_alignment < block_alignment)
 			node->subtree_max_alignment = block_alignment;
+		if (node->subtree_block_state < block_state)
+			node->subtree_block_state = block_state;
 
 		if (gpu_buddy_block_offset(block) < gpu_buddy_block_offset(node))
 			link = &parent->rb_left;
@@ -159,6 +703,7 @@ static void rbtree_insert(struct gpu_buddy *mm,
 	}
 
 	block->subtree_max_alignment = block_alignment;
+	block->subtree_block_state = block_state;
 	rb_link_node(&block->rb, parent, link);
 	rb_insert_augmented(&block->rb, root, &gpu_buddy_augment_cb);
 }
@@ -167,53 +712,55 @@ static void rbtree_remove(struct gpu_buddy *mm,
 			  struct gpu_buddy_block *block)
 {
 	unsigned int order = gpu_buddy_block_order(block);
-	enum gpu_buddy_free_tree tree;
-	struct rb_root *root;
-
-	tree = get_block_tree(block);
-	root = &mm->free_trees[tree][order];
 
-	rb_erase_augmented(&block->rb, root, &gpu_buddy_augment_cb);
+	rb_erase_augmented(&block->rb, &mm->free_tree[order], &gpu_buddy_augment_cb);
 	RB_CLEAR_NODE(&block->rb);
 }
 
-static void clear_reset(struct gpu_buddy_block *block)
-{
-	block->header &= ~GPU_BUDDY_HEADER_CLEAR;
-}
-
-static void mark_cleared(struct gpu_buddy_block *block)
-{
-	block->header |= GPU_BUDDY_HEADER_CLEAR;
-}
-
 static void mark_allocated(struct gpu_buddy *mm,
 			   struct gpu_buddy_block *block)
 {
 	block->header &= ~GPU_BUDDY_HEADER_STATE;
 	block->header |= GPU_BUDDY_ALLOCATED;
 
+	block->has_clear = false;
+
 	mm->free_scoreboard[gpu_buddy_block_order(block)]--;
 	mm->used_scoreboard[gpu_buddy_block_order(block)]++;
 
 	rbtree_remove(mm, block);
 }
 
-static void mark_free(struct gpu_buddy *mm,
-		      struct gpu_buddy_block *block)
+static void __mark_free(struct gpu_buddy *mm,
+			struct gpu_buddy_block *block,
+			enum gpu_block_state block_state)
 {
-	enum gpu_buddy_free_tree tree;
-
 	if (gpu_buddy_block_is_allocated(block))
 		mm->used_scoreboard[gpu_buddy_block_order(block)]--;
 
 	block->header &= ~GPU_BUDDY_HEADER_STATE;
 	block->header |= GPU_BUDDY_FREE;
 
+	block->header &= ~GPU_BUDDY_HEADER_CLEAR;
+
+	block->has_clear = (block_state != GPU_BLOCK_DIRTY);
+	if (block_state == GPU_BLOCK_CLEAR)
+		block->header |= GPU_BUDDY_HEADER_CLEAR;
+
 	mm->free_scoreboard[gpu_buddy_block_order(block)]++;
 
-	tree = get_block_tree(block);
-	rbtree_insert(mm, block, tree);
+	rbtree_insert(mm, block);
+}
+
+static void mark_free(struct gpu_buddy *mm,
+		      struct gpu_buddy_block *block)
+{
+	enum gpu_block_state block_state;
+
+	block_state = gpu_dirty_range_state(&mm->dirty,
+					    gpu_buddy_block_offset(block),
+					    gpu_buddy_block_size(mm, block));
+	__mark_free(mm, block, block_state);
 }
 
 static void mark_split(struct gpu_buddy *mm,
@@ -253,37 +800,31 @@ __get_buddy(struct gpu_buddy_block *block)
 }
 
 static unsigned int __gpu_buddy_free(struct gpu_buddy *mm,
-				     struct gpu_buddy_block *block,
-				     bool force_merge)
+				     struct gpu_buddy_block *block)
 {
+	enum gpu_block_state block_state;
 	struct gpu_buddy_block *parent;
 	unsigned int order;
 
-	while ((parent = block->parent)) {
-		struct gpu_buddy_block *buddy;
+	block_state = gpu_block_cached_state(block);
 
-		buddy = __get_buddy(block);
+	while ((parent = block->parent)) {
+		struct gpu_buddy_block *buddy = __get_buddy(block);
 
 		if (!gpu_buddy_block_is_free(buddy))
 			break;
 
-		if (!force_merge) {
-			/*
-			 * Check the block and its buddy clear state and exit
-			 * the loop if they both have the dissimilar state.
-			 */
-			if (gpu_buddy_block_is_clear(block) !=
-			    gpu_buddy_block_is_clear(buddy))
-				break;
+		if (block_state != GPU_BLOCK_MIXED) {
+			enum gpu_block_state buddy_state;
 
-			if (gpu_buddy_block_is_clear(block))
-				mark_cleared(parent);
+			buddy_state = gpu_block_cached_state(buddy);
+
+			if (buddy_state != block_state)
+				block_state = GPU_BLOCK_MIXED;
 		}
 
 		rbtree_remove(mm, buddy);
 		mm->free_scoreboard[gpu_buddy_block_order(buddy)]--;
-		if (force_merge && gpu_buddy_block_is_clear(buddy))
-			mm->clear_avail -= gpu_buddy_block_size(mm, buddy);
 
 		if (gpu_buddy_block_is_allocated(block))
 			mm->used_scoreboard[gpu_buddy_block_order(block)]--;
@@ -295,74 +836,11 @@ static unsigned int __gpu_buddy_free(struct gpu_buddy *mm,
 	}
 
 	order = gpu_buddy_block_order(block);
-	mark_free(mm, block);
+	__mark_free(mm, block, block_state);
 
 	return order;
 }
 
-static int __force_merge(struct gpu_buddy *mm,
-			 u64 start,
-			 u64 end,
-			 unsigned int min_order)
-{
-	unsigned int tree, order;
-	int i;
-
-	if (!min_order)
-		return -ENOMEM;
-
-	if (min_order > mm->max_order)
-		return -EINVAL;
-
-	for_each_free_tree(tree) {
-		for (i = min_order - 1; i >= 0; i--) {
-			struct rb_node *iter = rb_last(&mm->free_trees[tree][i]);
-
-			while (iter) {
-				struct gpu_buddy_block *block, *buddy;
-				u64 block_start, block_end;
-
-				block = rbtree_get_free_block(iter);
-				iter = rb_prev(iter);
-
-				if (!block || !block->parent)
-					continue;
-
-				block_start = gpu_buddy_block_offset(block);
-				block_end = block_start + gpu_buddy_block_size(mm, block) - 1;
-
-				if (!contains(start, end, block_start, block_end))
-					continue;
-
-				buddy = __get_buddy(block);
-				if (!gpu_buddy_block_is_free(buddy))
-					continue;
-
-				gpu_buddy_assert(gpu_buddy_block_is_clear(block) !=
-						 gpu_buddy_block_is_clear(buddy));
-
-				/*
-				 * Advance to the next node when the current node is the buddy,
-				 * as freeing the block will also remove its buddy from the tree.
-				 */
-				if (iter == &buddy->rb)
-					iter = rb_prev(iter);
-
-				rbtree_remove(mm, block);
-				mm->free_scoreboard[gpu_buddy_block_order(block)]--;
-				if (gpu_buddy_block_is_clear(block))
-					mm->clear_avail -= gpu_buddy_block_size(mm, block);
-
-				order = __gpu_buddy_free(mm, block, true);
-				if (order >= min_order)
-					return 0;
-			}
-		}
-	}
-
-	return -ENOMEM;
-}
-
 /**
  * gpu_buddy_init - init memory manager
  *
@@ -377,7 +855,7 @@ static int __force_merge(struct gpu_buddy *mm,
  */
 int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 {
-	unsigned int i, j, root_count = 0;
+	unsigned int root_count = 0;
 	u64 offset = 0;
 
 	if (size < chunk_size)
@@ -393,7 +871,6 @@ int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 
 	mm->size = size;
 	mm->avail = size;
-	mm->clear_avail = 0;
 	mm->chunk_size = chunk_size;
 	mm->max_order = ilog2(size) - ilog2(chunk_size);
 
@@ -411,19 +888,13 @@ int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 	if (!mm->used_scoreboard)
 		goto out_free_free_scoreboard;
 
-	mm->free_trees = kmalloc_objs(*mm->free_trees, GPU_BUDDY_MAX_FREE_TREES);
-	if (!mm->free_trees)
+	mm->free_tree = kcalloc(mm->max_order + 1,
+				sizeof(struct rb_root),
+				GFP_KERNEL);
+	if (!mm->free_tree)
 		goto out_free_used_scoreboard;
 
-	for_each_free_tree(i) {
-		mm->free_trees[i] = kmalloc_objs(struct rb_root,
-						 mm->max_order + 1);
-		if (!mm->free_trees[i])
-			goto out_free_tree;
-
-		for (j = 0; j <= mm->max_order; ++j)
-			mm->free_trees[i][j] = RB_ROOT;
-	}
+	gpu_dirty_tracker_init(&mm->dirty);
 
 	mm->n_roots = hweight64(size);
 
@@ -447,7 +918,8 @@ int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 		if (!root)
 			goto out_free_roots;
 
-		mark_free(mm, root);
+		gpu_dirty_tracker_mark_dirty(&mm->dirty, offset, root_size);
+		__mark_free(mm, root, GPU_BLOCK_DIRTY);
 
 		BUG_ON(root_count > mm->max_order);
 		BUG_ON(gpu_buddy_block_size(mm, root) < chunk_size);
@@ -469,9 +941,8 @@ int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 		gpu_block_free(mm, mm->roots[root_count]);
 	kfree(mm->roots);
 out_free_tree:
-	while (i--)
-		kfree(mm->free_trees[i]);
-	kfree(mm->free_trees);
+	gpu_dirty_tracker_fini(&mm->dirty);
+	kfree(mm->free_tree);
 out_free_used_scoreboard:
 	kfree(mm->used_scoreboard);
 out_free_free_scoreboard:
@@ -489,7 +960,7 @@ EXPORT_SYMBOL(gpu_buddy_init);
  */
 void gpu_buddy_fini(struct gpu_buddy *mm)
 {
-	u64 root_size, size, start;
+	u64 root_size, size;
 	unsigned int order;
 	int i;
 
@@ -497,14 +968,10 @@ void gpu_buddy_fini(struct gpu_buddy *mm)
 
 	for (i = 0; i < mm->n_roots; ++i) {
 		order = ilog2(size) - ilog2(mm->chunk_size);
-		start = gpu_buddy_block_offset(mm->roots[i]);
-		__force_merge(mm, start, start + size, order);
+		root_size = mm->chunk_size << order;
 
 		gpu_buddy_assert(gpu_buddy_block_is_free(mm->roots[i]));
-
 		gpu_block_free(mm, mm->roots[i]);
-
-		root_size = mm->chunk_size << order;
 		size -= root_size;
 	}
 
@@ -513,9 +980,8 @@ void gpu_buddy_fini(struct gpu_buddy *mm)
 	for (i = 0; i <= mm->max_order; ++i)
 		gpu_buddy_assert(!mm->used_scoreboard[i]);
 
-	for_each_free_tree(i)
-		kfree(mm->free_trees[i]);
-	kfree(mm->free_trees);
+	gpu_dirty_tracker_fini(&mm->dirty);
+	kfree(mm->free_tree);
 	kfree(mm->roots);
 	kfree(mm->free_scoreboard);
 	kfree(mm->used_scoreboard);
@@ -527,6 +993,7 @@ static int split_block(struct gpu_buddy *mm,
 {
 	unsigned int block_order = gpu_buddy_block_order(block) - 1;
 	u64 offset = gpu_buddy_block_offset(block);
+	enum gpu_block_state parent_state;
 
 	BUG_ON(!gpu_buddy_block_is_free(block));
 	BUG_ON(!gpu_buddy_block_order(block));
@@ -542,17 +1009,18 @@ static int split_block(struct gpu_buddy *mm,
 		return -ENOMEM;
 	}
 
+	parent_state = gpu_block_cached_state(block);
+
 	mark_split(mm, block);
 
-	if (gpu_buddy_block_is_clear(block)) {
-		mark_cleared(block->left);
-		mark_cleared(block->right);
-		clear_reset(block);
+	if (parent_state == GPU_BLOCK_MIXED) {
+		mark_free(mm, block->left);
+		mark_free(mm, block->right);
+	} else {
+		__mark_free(mm, block->left, parent_state);
+		__mark_free(mm, block->right, parent_state);
 	}
 
-	mark_free(mm, block->left);
-	mark_free(mm, block->right);
-
 	return 0;
 }
 
@@ -567,45 +1035,47 @@ static int split_block(struct gpu_buddy *mm,
  */
 void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear)
 {
-	enum gpu_buddy_free_tree src_tree, dst_tree;
-	u64 root_size, size, start;
-	unsigned int order;
-	int i;
+	unsigned int i;
 
 	gpu_buddy_driver_lock_held(mm);
-	size = mm->size;
-	for (i = 0; i < mm->n_roots; ++i) {
-		order = ilog2(size) - ilog2(mm->chunk_size);
-		start = gpu_buddy_block_offset(mm->roots[i]);
-		__force_merge(mm, start, start + size, order);
-
-		root_size = mm->chunk_size << order;
-		size -= root_size;
-	}
 
-	src_tree = is_clear ? GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
-	dst_tree = is_clear ? GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
+	gpu_dirty_tracker_empty(&mm->dirty);
 
 	for (i = 0; i <= mm->max_order; ++i) {
-		struct rb_root *root = &mm->free_trees[src_tree][i];
 		struct gpu_buddy_block *block, *tmp;
 
-		rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) {
-			rbtree_remove(mm, block);
+		rbtree_postorder_for_each_entry_safe(block, tmp,
+						     &mm->free_tree[i], rb) {
 			if (is_clear) {
-				mark_cleared(block);
-				mm->clear_avail += gpu_buddy_block_size(mm, block);
+				if (!gpu_buddy_block_is_clear(block))
+					block->header |= GPU_BUDDY_HEADER_CLEAR;
+				block->has_clear = true;
 			} else {
-				clear_reset(block);
-				mm->clear_avail -= gpu_buddy_block_size(mm, block);
+				block->header &= ~GPU_BUDDY_HEADER_CLEAR;
+				block->has_clear = false;
+				gpu_dirty_tracker_mark_dirty(&mm->dirty,
+							     gpu_buddy_block_offset(block),
+							     gpu_buddy_block_size(mm, block));
 			}
 
-			rbtree_insert(mm, block, dst_tree);
+			gpu_buddy_augment_compute(block);
 		}
 	}
 }
 EXPORT_SYMBOL(gpu_buddy_reset_clear);
 
+static void __gpu_buddy_free_block_internal(struct gpu_buddy *mm,
+					    struct gpu_buddy_block *block)
+{
+	u64 size = gpu_buddy_block_size(mm, block);
+
+	gpu_buddy_driver_lock_held(mm);
+	BUG_ON(!gpu_buddy_block_is_allocated(block));
+
+	mm->avail += size;
+	__gpu_buddy_free(mm, block);
+}
+
 /**
  * gpu_buddy_free_block - free a block
  *
@@ -615,13 +1085,12 @@ EXPORT_SYMBOL(gpu_buddy_reset_clear);
 void gpu_buddy_free_block(struct gpu_buddy *mm,
 			  struct gpu_buddy_block *block)
 {
-	gpu_buddy_driver_lock_held(mm);
-	BUG_ON(!gpu_buddy_block_is_allocated(block));
-	mm->avail += gpu_buddy_block_size(mm, block);
-	if (gpu_buddy_block_is_clear(block))
-		mm->clear_avail += gpu_buddy_block_size(mm, block);
+	if (!gpu_buddy_block_is_clear(block))
+		gpu_dirty_tracker_mark_dirty(&mm->dirty,
+					     gpu_buddy_block_offset(block),
+					     gpu_buddy_block_size(mm, block));
 
-	__gpu_buddy_free(mm, block, false);
+	__gpu_buddy_free_block_internal(mm, block);
 }
 EXPORT_SYMBOL(gpu_buddy_free_block);
 
@@ -684,17 +1153,47 @@ static void __gpu_buddy_free_list(struct gpu_buddy *mm,
 				  bool mark_dirty)
 {
 	struct gpu_buddy_block *block, *on;
+	u64 dirty_start = 0, dirty_size = 0;
 
 	gpu_buddy_assert(!(mark_dirty && mark_clear));
 
 	list_for_each_entry_safe(block, on, objects, link) {
+		u64 offset = gpu_buddy_block_offset(block);
+		u64 size = gpu_buddy_block_size(mm, block);
+
 		if (mark_clear)
-			mark_cleared(block);
+			block->header |= GPU_BUDDY_HEADER_CLEAR;
 		else if (mark_dirty)
-			clear_reset(block);
-		gpu_buddy_free_block(mm, block);
+			block->header &= ~GPU_BUDDY_HEADER_CLEAR;
+
+		/*
+		 * Coalesce contiguous dirty blocks into one extent update so
+		 * a multi-block contiguous free costs a single mark_dirty().
+		 * Flush the pending extent and start over on a gap.
+		 */
+		if (!gpu_buddy_block_is_clear(block)) {
+			if (dirty_size &&
+			    (dirty_start + dirty_size == offset ||
+			     offset + size == dirty_start)) {
+				dirty_start = min(dirty_start, offset);
+				dirty_size += size;
+			} else {
+				if (dirty_size)
+					gpu_dirty_tracker_mark_dirty(&mm->dirty,
+								     dirty_start,
+								     dirty_size);
+				dirty_start = offset;
+				dirty_size = size;
+			}
+		}
+
+		__gpu_buddy_free_block_internal(mm, block);
 		cond_resched();
 	}
+
+	if (dirty_size)
+		gpu_dirty_tracker_mark_dirty(&mm->dirty, dirty_start, dirty_size);
+
 	INIT_LIST_HEAD(objects);
 }
 
@@ -727,13 +1226,6 @@ void gpu_buddy_free_list(struct gpu_buddy *mm,
 }
 EXPORT_SYMBOL(gpu_buddy_free_list);
 
-static bool block_incompatible(struct gpu_buddy_block *block, unsigned int flags)
-{
-	bool needs_clear = flags & GPU_BUDDY_CLEAR_ALLOCATION;
-
-	return needs_clear != gpu_buddy_block_is_clear(block);
-}
-
 static void __gpu_buddy_undo_splits(struct gpu_buddy *mm,
 				    struct gpu_buddy_block *block)
 {
@@ -744,7 +1236,7 @@ static void __gpu_buddy_undo_splits(struct gpu_buddy *mm,
 	     gpu_buddy_block_is_free(buddy))) {
 		rbtree_remove(mm, block);
 		mm->free_scoreboard[gpu_buddy_block_order(block)]--;
-		__gpu_buddy_free(mm, block, false);
+		__gpu_buddy_free(mm, block);
 	}
 }
 
@@ -752,8 +1244,7 @@ static struct gpu_buddy_block *
 __alloc_range_bias(struct gpu_buddy *mm,
 		   u64 start, u64 end,
 		   unsigned int order,
-		   unsigned long flags,
-		   bool fallback)
+		   unsigned long flags)
 {
 	u64 req_size = mm->chunk_size << order;
 	struct gpu_buddy_block *block;
@@ -763,7 +1254,15 @@ __alloc_range_bias(struct gpu_buddy *mm,
 
 	end = end - 1;
 
-	for (i = 0; i < mm->n_roots; ++i)
+	/*
+	 * This range-constrained search hands back the highest/right-most
+	 * address that satisfies the request: the roots are seeded high-to-low
+	 * and the right (higher-address) child is descended first, making
+	 * top-down the default placement here. A non-top-down clear request is
+	 * the only exception, where the descent is biased towards clear or
+	 * clear-containing subtrees to satisfy the clear preference.
+	 */
+	for (i = mm->n_roots - 1; i >= 0; --i)
 		list_add_tail(&mm->roots[i]->tmp_link, &dfs);
 
 	do {
@@ -799,9 +1298,6 @@ __alloc_range_bias(struct gpu_buddy *mm,
 				continue;
 		}
 
-		if (!fallback && block_incompatible(block, flags))
-			continue;
-
 		if (contains(start, end, block_start, block_end) &&
 		    order == gpu_buddy_block_order(block)) {
 			/*
@@ -819,8 +1315,38 @@ __alloc_range_bias(struct gpu_buddy *mm,
 				goto err_undo;
 		}
 
-		list_add(&block->right->tmp_link, &dfs);
-		list_add(&block->left->tmp_link, &dfs);
+		/*
+		 * Top-down is a strict address-placement policy, so when it is
+		 * requested we ignore clear steering and simply descend the
+		 * right (higher-address) child first. Only a non-top-down clear
+		 * request biases the descent towards clear/has_clear subtrees.
+		 */
+		if ((flags & GPU_BUDDY_CLEAR_ALLOCATION) &&
+		    !(flags & GPU_BUDDY_TOPDOWN_ALLOCATION)) {
+			struct gpu_buddy_block *prefer;
+
+			if (gpu_buddy_block_is_clear(block->right))
+				prefer = block->right;
+			else if (gpu_buddy_block_is_clear(block->left))
+				prefer = block->left;
+			else if (block->right->has_clear)
+				prefer = block->right;
+			else if (block->left->has_clear)
+				prefer = block->left;
+			else
+				prefer = block->right;
+
+			if (prefer == block->right) {
+				list_add(&block->left->tmp_link, &dfs);
+				list_add(&block->right->tmp_link, &dfs);
+			} else {
+				list_add(&block->right->tmp_link, &dfs);
+				list_add(&block->left->tmp_link, &dfs);
+			}
+		} else {
+			list_add(&block->left->tmp_link, &dfs);
+			list_add(&block->right->tmp_link, &dfs);
+		}
 	} while (1);
 
 	return ERR_PTR(-ENOSPC);
@@ -835,48 +1361,32 @@ __alloc_range_bias(struct gpu_buddy *mm,
 	return ERR_PTR(err);
 }
 
-static struct gpu_buddy_block *
-__gpu_buddy_alloc_range_bias(struct gpu_buddy *mm,
-			     u64 start, u64 end,
-			     unsigned int order,
-			     unsigned long flags)
-{
-	struct gpu_buddy_block *block;
-	bool fallback = false;
-
-	block = __alloc_range_bias(mm, start, end, order,
-				   flags, fallback);
-	if (IS_ERR(block))
-		return __alloc_range_bias(mm, start, end, order,
-					  flags, !fallback);
-
-	return block;
-}
-
+/* Return the highest-address free block of at least @order. */
 static struct gpu_buddy_block *
 get_maxblock(struct gpu_buddy *mm,
-	     unsigned int order,
-	     enum gpu_buddy_free_tree tree)
+	     unsigned int order)
 {
-	struct gpu_buddy_block *max_block = NULL, *block = NULL;
-	struct rb_root *root;
+	struct gpu_buddy_block *max_block;
+	struct gpu_buddy_block *block;
 	unsigned int i;
 
+	/*
+	 * Top-down allocation is a strict address-placement policy: the block
+	 * is chosen purely by offset, regardless of its clear/dirty state.
+	 * Clear state is re-derived from the dirty tracker once the allocation
+	 * completes, and the driver is responsible for issuing the clear pass
+	 * if a clear region is required.
+	 */
+	max_block = NULL;
+
 	for (i = order; i <= mm->max_order; ++i) {
-		root = &mm->free_trees[tree][i];
-		block = rbtree_last_free_block(root);
+		block = rbtree_last_free_block(&mm->free_tree[i]);
 		if (!block)
 			continue;
 
-		if (!max_block) {
+		if (!max_block ||
+		    gpu_buddy_block_offset(block) > gpu_buddy_block_offset(max_block))
 			max_block = block;
-			continue;
-		}
-
-		if (gpu_buddy_block_offset(block) >
-		    gpu_buddy_block_offset(max_block)) {
-			max_block = block;
-		}
 	}
 
 	return max_block;
@@ -888,45 +1398,34 @@ alloc_from_freetree(struct gpu_buddy *mm,
 		    unsigned long flags)
 {
 	struct gpu_buddy_block *block = NULL;
-	struct rb_root *root;
-	enum gpu_buddy_free_tree tree;
 	unsigned int tmp;
 	int err;
 
-	tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ?
-		GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
-
 	if (flags & GPU_BUDDY_TOPDOWN_ALLOCATION) {
-		block = get_maxblock(mm, order, tree);
+		block = get_maxblock(mm, order);
 		if (block)
-			/* Store the obtained block order */
 			tmp = gpu_buddy_block_order(block);
 	} else {
-		for (tmp = order; tmp <= mm->max_order; ++tmp) {
-			/* Get RB tree root for this order and tree */
-			root = &mm->free_trees[tree][tmp];
-			block = rbtree_last_free_block(root);
-			if (block)
-				break;
+		if (flags & GPU_BUDDY_CLEAR_ALLOCATION) {
+			for (tmp = order; tmp <= mm->max_order; ++tmp) {
+				block = rbtree_last_clear_free_block(&mm->free_tree[tmp],
+								     GPU_BLOCK_MIXED);
+				if (block)
+					break;
+			}
 		}
-	}
-
-	if (!block) {
-		/* Try allocating from the other tree */
-		tree = (tree == GPU_BUDDY_CLEAR_TREE) ?
-			GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
-
-		for (tmp = order; tmp <= mm->max_order; ++tmp) {
-			root = &mm->free_trees[tree][tmp];
-			block = rbtree_last_free_block(root);
-			if (block)
-				break;
+		if (!block) {
+			for (tmp = order; tmp <= mm->max_order; ++tmp) {
+				block = rbtree_last_free_block(&mm->free_tree[tmp]);
+				if (block)
+					break;
+			}
 		}
-
-		if (!block)
-			return ERR_PTR(-ENOSPC);
 	}
 
+	if (!block)
+		return ERR_PTR(-ENOSPC);
+
 	BUG_ON(!gpu_buddy_block_is_free(block));
 
 	while (tmp != order) {
@@ -934,7 +1433,26 @@ alloc_from_freetree(struct gpu_buddy *mm,
 		if (unlikely(err))
 			goto err_undo;
 
-		block = block->right;
+		if ((flags & GPU_BUDDY_CLEAR_ALLOCATION) &&
+		    !(flags & GPU_BUDDY_TOPDOWN_ALLOCATION)) {
+			bool right_clear, left_clear;
+
+			right_clear = gpu_buddy_block_is_clear(block->right);
+			left_clear = gpu_buddy_block_is_clear(block->left);
+
+			if (right_clear)
+				block = block->right;
+			else if (left_clear)
+				block = block->left;
+			else if (block->right->has_clear)
+				block = block->right;
+			else if (block->left->has_clear)
+				block = block->left;
+			else
+				block = block->right;
+		} else {
+			block = block->right;
+		}
 		tmp--;
 	}
 	return block;
@@ -961,12 +1479,10 @@ static bool gpu_buddy_subtree_can_satisfy(struct rb_node *node,
 
 static struct gpu_buddy_block *
 gpu_buddy_find_block_aligned(struct gpu_buddy *mm,
-			     enum gpu_buddy_free_tree tree,
 			     unsigned int order,
-			     unsigned int alignment,
-			     unsigned long flags)
+			     unsigned int alignment)
 {
-	struct rb_root *root = &mm->free_trees[tree][order];
+	struct rb_root *root = &mm->free_tree[order];
 	struct rb_node *rb = root->rb_node;
 
 	while (rb) {
@@ -999,12 +1515,10 @@ gpu_buddy_find_block_aligned(struct gpu_buddy *mm,
 static struct gpu_buddy_block *
 gpu_buddy_offset_aligned_allocation(struct gpu_buddy *mm,
 				    u64 size,
-				    u64 min_block_size,
-				    unsigned long flags)
+				    u64 min_block_size)
 {
 	struct gpu_buddy_block *block = NULL;
 	unsigned int order, tmp, alignment;
-	enum gpu_buddy_free_tree tree;
 	unsigned long pages;
 	int err;
 
@@ -1012,19 +1526,15 @@ gpu_buddy_offset_aligned_allocation(struct gpu_buddy *mm,
 	pages = size >> ilog2(mm->chunk_size);
 	order = fls(pages) - 1;
 
-	tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ?
-		GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE;
-
+	/*
+	 * Offset-aligned allocation is a strict address-placement policy: the
+	 * block is chosen purely by its offset alignment, regardless of its
+	 * clear/dirty state. Clear state is re-derived from the dirty tracker
+	 * once the allocation completes, and the driver is responsible for
+	 * issuing the clear pass if a clear region is required.
+	 */
 	for (tmp = order; tmp <= mm->max_order; ++tmp) {
-		block = gpu_buddy_find_block_aligned(mm, tree, tmp,
-						     alignment, flags);
-		if (!block) {
-			tree = (tree == GPU_BUDDY_CLEAR_TREE) ?
-				GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE;
-			block = gpu_buddy_find_block_aligned(mm, tree, tmp,
-							     alignment, flags);
-		}
-
+		block = gpu_buddy_find_block_aligned(mm, tmp, alignment);
 		if (block)
 			break;
 	}
@@ -1063,6 +1573,7 @@ gpu_buddy_offset_aligned_allocation(struct gpu_buddy *mm,
 static int __alloc_range(struct gpu_buddy *mm,
 			 struct list_head *dfs,
 			 u64 start, u64 size,
+			 unsigned long flags,
 			 struct list_head *blocks,
 			 u64 *total_allocated_on_err)
 {
@@ -1099,16 +1610,23 @@ static int __alloc_range(struct gpu_buddy *mm,
 
 		if (contains(start, end, block_start, block_end)) {
 			if (gpu_buddy_block_is_free(block)) {
+				u64 block_offset;
+				u64 block_size;
+
+				block_size = gpu_buddy_block_size(mm, block);
+				block_offset = gpu_buddy_block_offset(block);
+
+				if (!gpu_buddy_block_is_clear(block))
+					gpu_dirty_tracker_remove_range(&mm->dirty,
+								       block_offset,
+								       block_size);
+
 				mark_allocated(mm, block);
-				total_allocated += gpu_buddy_block_size(mm, block);
-				mm->avail -= gpu_buddy_block_size(mm, block);
-				if (gpu_buddy_block_is_clear(block))
-					mm->clear_avail -= gpu_buddy_block_size(mm, block);
+				total_allocated += block_size;
+				mm->avail -= block_size;
+
 				list_add_tail(&block->link, &allocated);
 				continue;
-			} else if (!mm->clear_avail) {
-				err = -ENOSPC;
-				goto err_free;
 			}
 		}
 
@@ -1153,6 +1671,7 @@ static int __alloc_range(struct gpu_buddy *mm,
 static int __gpu_buddy_alloc_range(struct gpu_buddy *mm,
 				   u64 start,
 				   u64 size,
+				   unsigned long flags,
 				   u64 *total_allocated_on_err,
 				   struct list_head *blocks)
 {
@@ -1162,7 +1681,7 @@ static int __gpu_buddy_alloc_range(struct gpu_buddy *mm,
 	for (i = 0; i < mm->n_roots; ++i)
 		list_add_tail(&mm->roots[i]->tmp_link, &dfs);
 
-	return __alloc_range(mm, &dfs, start, size,
+	return __alloc_range(mm, &dfs, start, size, flags,
 			     blocks, total_allocated_on_err);
 }
 
@@ -1170,85 +1689,89 @@ static int __alloc_contig_aligned_retry(struct gpu_buddy *mm,
 					u64 unaligned_offset,
 					u64 size,
 					u64 min_block_size,
+					unsigned long flags,
 					struct list_head *blocks)
 {
 	u64 aligned_offset = round_down(unaligned_offset, min_block_size);
 
-	return __gpu_buddy_alloc_range(mm, aligned_offset, size, NULL, blocks);
+	return __gpu_buddy_alloc_range(mm, aligned_offset, size, flags,
+				       NULL, blocks);
 }
 
 static int __alloc_contig_try_harder(struct gpu_buddy *mm,
 				     u64 size,
 				     u64 min_block_size,
+				     unsigned long flags,
 				     struct list_head *blocks)
 {
 	u64 rhs_offset, lhs_offset, filled;
 	struct gpu_buddy_block *block;
-	unsigned int tree, order;
+	struct rb_root *root;
+	struct rb_node *iter;
+	unsigned long pages;
+	unsigned int order;
 	u64 modify_size;
 	int err;
 
 	modify_size = rounddown_pow_of_two(size);
-	order = ilog2(modify_size) - ilog2(mm->chunk_size);
+	pages = modify_size >> ilog2(mm->chunk_size);
+	order = fls(pages) - 1;
 	if (order == 0)
 		return -ENOSPC;
 
-	for_each_free_tree(tree) {
-		struct rb_root *root;
-		struct rb_node *iter;
-
-		root = &mm->free_trees[tree][order];
-		if (rbtree_is_empty(root))
-			continue;
-
-		iter = rb_last(root);
-		while (iter) {
-			block = rbtree_get_free_block(iter);
-
-			rhs_offset = gpu_buddy_block_offset(block);
-
-			/* Allocate blocks traversing RHS */
-			err =  __gpu_buddy_alloc_range(mm, rhs_offset, size,
-						       &filled, blocks);
-			if (err && err != -ENOSPC)
-				return err;
-			if (!err && IS_ALIGNED(rhs_offset, min_block_size))
-				return 0;
-			if (!err) {
-				/* Allocate the unaligned RHS offset using round_down */
-				gpu_buddy_free_list_internal(mm, blocks);
-				err = __alloc_contig_aligned_retry(mm, rhs_offset,
-								   size,
-								   min_block_size,
-								   blocks);
-				if (!err)
-					return 0;
-				if (err != -ENOSPC) {
-					gpu_buddy_free_list_internal(mm, blocks);
-					return err;
-				}
-				goto next;
-			}
-
-			if (size - filled > rhs_offset)
-				goto next;
-
-			lhs_offset = rhs_offset - (size - filled);
+	root = &mm->free_tree[order];
+	if (RB_EMPTY_ROOT(root))
+		return -ENOSPC;
 
-			/* Allocate the unaligned LHS offset using round_down */
+	iter = rb_last(root);
+	while (iter) {
+		block = rbtree_get_free_block(iter);
+
+		rhs_offset = gpu_buddy_block_offset(block);
+
+		/* Allocate blocks traversing RHS */
+		err =  __gpu_buddy_alloc_range(mm, rhs_offset, size,
+					       flags, &filled, blocks);
+		if (err && err != -ENOSPC)
+			return err;
+		if (!err && IS_ALIGNED(rhs_offset, min_block_size))
+			return 0;
+		if (!err) {
+			/* Allocate the unaligned RHS offset using round_down */
 			gpu_buddy_free_list_internal(mm, blocks);
-			err = __alloc_contig_aligned_retry(mm, lhs_offset, size,
-							   min_block_size, blocks);
+			err = __alloc_contig_aligned_retry(mm, rhs_offset,
+							   size,
+							   min_block_size,
+							   flags, blocks);
 			if (!err)
 				return 0;
 			if (err != -ENOSPC) {
 				gpu_buddy_free_list_internal(mm, blocks);
 				return err;
 			}
-next:
+			goto next;
+		}
+
+		if (size - filled > rhs_offset)
+			goto next;
+
+		lhs_offset = rhs_offset - (size - filled);
+
+		/* Allocate the unaligned LHS offset using round_down */
+		gpu_buddy_free_list_internal(mm, blocks);
+		err = __alloc_contig_aligned_retry(mm, lhs_offset,
+						   size,
+						   min_block_size,
+						   flags, blocks);
+		if (!err)
+			return 0;
+		if (err != -ENOSPC) {
 			gpu_buddy_free_list_internal(mm, blocks);
-			iter = rb_prev(iter);
+			return err;
 		}
+next:
+		gpu_buddy_free_list_internal(mm, blocks);
+		iter = rb_prev(iter);
 	}
 
 	return -ENOSPC;
@@ -1282,6 +1805,7 @@ int gpu_buddy_block_trim(struct gpu_buddy *mm,
 	struct gpu_buddy_block *block;
 	u64 block_start, block_end;
 	LIST_HEAD(dfs);
+	bool was_clear;
 	u64 new_start;
 	int err;
 
@@ -1324,22 +1848,35 @@ int gpu_buddy_block_trim(struct gpu_buddy *mm,
 	}
 
 	list_del(&block->link);
-	mark_free(mm, block);
+
+	was_clear = gpu_buddy_block_is_clear(block);
+
+	if (!was_clear)
+		gpu_dirty_tracker_mark_dirty(&mm->dirty,
+					     gpu_buddy_block_offset(block),
+					     gpu_buddy_block_size(mm, block));
+
+	__mark_free(mm, block, was_clear ? GPU_BLOCK_CLEAR : GPU_BLOCK_DIRTY);
 	mm->avail += gpu_buddy_block_size(mm, block);
-	if (gpu_buddy_block_is_clear(block))
-		mm->clear_avail += gpu_buddy_block_size(mm, block);
 
 	/* Prevent recursively freeing this node */
 	parent = block->parent;
 	block->parent = NULL;
 
 	list_add(&block->tmp_link, &dfs);
-	err =  __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL);
+	err =  __alloc_range(mm, &dfs, new_start, new_size,
+			     was_clear ? GPU_BUDDY_CLEAR_ALLOCATION : 0,
+			     blocks, NULL);
 	if (err) {
 		mark_allocated(mm, block);
 		mm->avail -= gpu_buddy_block_size(mm, block);
-		if (gpu_buddy_block_is_clear(block))
-			mm->clear_avail -= gpu_buddy_block_size(mm, block);
+		if (!was_clear) {
+			gpu_dirty_tracker_remove_range(&mm->dirty,
+						       gpu_buddy_block_offset(block),
+						       gpu_buddy_block_size(mm, block));
+		}
+		if (was_clear)
+			block->header |= GPU_BUDDY_HEADER_CLEAR;
 		list_add(&block->link, blocks);
 	}
 
@@ -1348,6 +1885,22 @@ int gpu_buddy_block_trim(struct gpu_buddy *mm,
 }
 EXPORT_SYMBOL(gpu_buddy_block_trim);
 
+static bool dirty_steer_window(struct gpu_buddy *mm, u64 req_size,
+			       u64 *start, u64 *end, unsigned long *flags)
+{
+	u64 aligned_start;
+	struct gpu_dirty_extent *ext =
+		gpu_dirty_tracker_find(&mm->dirty, req_size, &aligned_start);
+
+	if (!ext)
+		return false;
+
+	*start  = aligned_start;
+	*end    = ext->end;
+	*flags |= GPU_BUDDY_RANGE_ALLOCATION;
+	return true;
+}
+
 static struct gpu_buddy_block *
 __gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
 			 u64 start, u64 end,
@@ -1355,18 +1908,36 @@ __gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
 			 unsigned int order,
 			 unsigned long flags)
 {
-	if (flags & GPU_BUDDY_RANGE_ALLOCATION)
+	struct gpu_buddy_block *block;
+	bool steered = false;
+
+	/* Allocate from dirty tracker */
+	if (!(flags & GPU_BUDDY_RANGE_ALLOCATION) &&
+	    !(flags & GPU_BUDDY_CLEAR_ALLOCATION) &&
+	    size >= min_block_size &&
+	    gpu_buddy_clear_avail(mm) && mm->dirty.total_dirty) {
+		u64 block_size = mm->chunk_size << order;
+
+		steered = dirty_steer_window(mm, block_size,
+					     &start, &end, &flags);
+	}
+
+	if (flags & GPU_BUDDY_RANGE_ALLOCATION) {
 		/* Allocate traversing within the range */
-		return  __gpu_buddy_alloc_range_bias(mm, start, end,
-						     order, flags);
-	else if (size < min_block_size)
+		block = __alloc_range_bias(mm, start, end, order, flags);
+		if (!IS_ERR(block) || !steered)
+			return block;
+
+		flags &= ~GPU_BUDDY_RANGE_ALLOCATION;
+	}
+
+	if (size < min_block_size)
 		/* Allocate from an offset-aligned region without size rounding */
 		return gpu_buddy_offset_aligned_allocation(mm, size,
-							   min_block_size,
-							   flags);
-	else
-		/* Allocate from freetree */
-		return alloc_from_freetree(mm, order, flags);
+							   min_block_size);
+
+	/* Allocate from freetree */
+	return alloc_from_freetree(mm, order, flags);
 }
 
 /**
@@ -1427,7 +1998,7 @@ int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
 		if (!IS_ALIGNED(start | end, min_block_size))
 			return -EINVAL;
 
-		return __gpu_buddy_alloc_range(mm, start, size, NULL, blocks);
+		return __gpu_buddy_alloc_range(mm, start, size, flags, NULL, blocks);
 	}
 
 	original_size = size;
@@ -1453,7 +2024,8 @@ int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
 		if ((flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) &&
 		    !(flags & GPU_BUDDY_RANGE_ALLOCATION))
 			return __alloc_contig_try_harder(mm, original_size,
-							 original_min_size, blocks);
+							 original_min_size,
+							 flags, blocks);
 
 		return -EINVAL;
 	}
@@ -1468,8 +2040,6 @@ int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
 		BUG_ON(size >= min_block_size && order < min_order);
 
 		do {
-			unsigned int fallback_order;
-
 			block = __gpu_buddy_alloc_blocks(mm, start,
 							 end,
 							 size,
@@ -1479,48 +2049,40 @@ int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
 			if (!IS_ERR(block))
 				break;
 
-			if (size < min_block_size) {
-				fallback_order = order;
-			} else if (order == min_order) {
-				fallback_order = min_order;
-			} else {
+			if (size >= min_block_size && order > min_order) {
 				order--;
 				continue;
 			}
 
-			/* Try allocation through force merge method */
-			if (mm->clear_avail &&
-			    !__force_merge(mm, start, end, fallback_order)) {
-				block = __gpu_buddy_alloc_blocks(mm, start,
-								 end,
-								 size,
-								 min_block_size,
-								 fallback_order,
-								 flags);
-				if (!IS_ERR(block)) {
-					order = fallback_order;
-					break;
-				}
-			}
-
 			/*
 			 * Try contiguous block allocation through
 			 * try harder method.
 			 */
 			if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION &&
-			    !(flags & GPU_BUDDY_RANGE_ALLOCATION))
-				return __alloc_contig_try_harder(mm,
-								 original_size,
-								 original_min_size,
-								 blocks);
+			    !(flags & GPU_BUDDY_RANGE_ALLOCATION)) {
+				err = __alloc_contig_try_harder(mm,
+								original_size,
+								original_min_size,
+								flags,
+								blocks);
+				if (!err)
+					return 0;
+				if (err != -ENOSPC)
+					return err;
+				goto err_free;
+			}
 			err = -ENOSPC;
 			goto err_free;
 		} while (1);
 
+		if (!gpu_buddy_block_is_clear(block))
+			gpu_dirty_tracker_remove_range(&mm->dirty,
+						       gpu_buddy_block_offset(block),
+						       gpu_buddy_block_size(mm, block));
+
 		mark_allocated(mm, block);
 		mm->avail -= gpu_buddy_block_size(mm, block);
-		if (gpu_buddy_block_is_clear(block))
-			mm->clear_avail -= gpu_buddy_block_size(mm, block);
+
 		kmemleak_update_trace(block);
 		list_add_tail(&block->link, &allocated);
 
@@ -1594,7 +2156,8 @@ void gpu_buddy_print(struct gpu_buddy *mm)
 
 	gpu_buddy_driver_lock_held(mm);
 	pr_info("chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n",
-		mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
+		mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20,
+		gpu_buddy_clear_avail(mm) >> 20);
 
 	for (order = mm->max_order; order >= 0; order--) {
 		u64 free_count = mm->free_scoreboard[order];
@@ -1615,6 +2178,7 @@ EXPORT_SYMBOL(gpu_buddy_print);
 
 static void gpu_buddy_module_exit(void)
 {
+	kmem_cache_destroy(slab_extents);
 	kmem_cache_destroy(slab_blocks);
 }
 
@@ -1624,7 +2188,15 @@ static int __init gpu_buddy_module_init(void)
 	if (!slab_blocks)
 		return -ENOMEM;
 
+	slab_extents = KMEM_CACHE(gpu_dirty_extent, 0);
+	if (!slab_extents)
+		goto err_destroy_blocks;
+
 	return 0;
+
+err_destroy_blocks:
+	kmem_cache_destroy(slab_blocks);
+	return -ENOMEM;
 }
 
 module_init(gpu_buddy_module_init);
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 323422861e8f68..65d46dfa72661b 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -45,6 +45,7 @@ config DRM_MIPI_DSI
 config DRM_KMS_HELPER
 	tristate
 	depends on DRM
+	select DRM_PANIC_HELPER if DRM_PANIC
 	select FB_CORE if DRM_FBDEV_EMULATION
 	help
 	  CRTC helpers for KMS drivers.
@@ -56,8 +57,6 @@ config DRM_DRAW
 config DRM_PANIC
 	bool "Display a user-friendly message when a kernel panic occurs"
 	depends on DRM && PRINTK
-	select FONT_SUPPORT
-	select DRM_DRAW
 	help
 	  Enable a drm panic handler, which will display a user-friendly message
 	  when a kernel panic occurs. It's useful when using a user-space
@@ -130,6 +129,12 @@ config DRM_PANIC_SCREEN_QR_VERSION
 	  Smaller QR code are easier to read, but will contain less debugging
 	  data. Default is 40.
 
+config DRM_PANIC_HELPER
+	bool
+	depends on DRM
+	select DRM_DRAW
+	select FONT_SUPPORT
+
 config DRM_RAS
 	bool "DRM RAS support"
 	depends on DRM
diff --git a/drivers/gpu/drm/Kconfig.debug b/drivers/gpu/drm/Kconfig.debug
index 1f4c98cbf0906c..b909cd0a619fec 100644
--- a/drivers/gpu/drm/Kconfig.debug
+++ b/drivers/gpu/drm/Kconfig.debug
@@ -69,6 +69,7 @@ config DRM_KUNIT_TEST
 	select DRM_EXPORT_FOR_TESTS if m
 	select DRM_GEM_SHMEM_HELPER
 	select DRM_KUNIT_TEST_HELPERS
+	select DRM_PANIC_HELPER
 	select DRM_SYSFB_HELPER
 	select PRIME_NUMBERS
 	default KUNIT_ALL_TESTS
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index e635fcffd37909..cadc8529c99551 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -79,7 +79,6 @@ drm-$(CONFIG_DRM_CLIENT) += \
 	drm_client_modeset.o \
 	drm_client_sysrq.o
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
-drm-$(CONFIG_DRM_PANEL) += drm_panel.o
 drm-$(CONFIG_OF) += drm_of.o
 drm-$(CONFIG_PCI) += drm_pci.o
 drm-$(CONFIG_DEBUG_FS) += \
@@ -92,10 +91,10 @@ drm-$(CONFIG_DRM_PRIVACY_SCREEN) += \
 drm-$(CONFIG_DRM_ACCEL) += ../../accel/drm_accel.o
 drm-$(CONFIG_DRM_PANIC) += drm_panic.o
 drm-$(CONFIG_DRM_DRAW) += drm_draw.o
-drm-$(CONFIG_DRM_PANIC_SCREEN_QR_CODE) += drm_panic_qr.o
 drm-$(CONFIG_DRM_RAS) += drm_ras.o drm_ras_nl.o drm_ras_genl_family.o
 obj-$(CONFIG_DRM)	+= drm.o
 
+obj-$(CONFIG_DRM_PANEL) += drm_panel.o
 obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
 obj-$(CONFIG_DRM_PANEL_BACKLIGHT_QUIRKS) += drm_panel_backlight_quirks.o
 
@@ -156,7 +155,8 @@ drm_kms_helper-y := \
 	drm_self_refresh_helper.o \
 	drm_simple_kms_helper.o \
 	drm_vblank_helper.o
-drm_kms_helper-$(CONFIG_DRM_PANEL_BRIDGE) += bridge/panel.o
+drm_kms_helper-$(CONFIG_DRM_PANIC_HELPER) += drm_panic_helper.o
+drm_kms_helper-$(CONFIG_DRM_PANIC_SCREEN_QR_CODE) += drm_panic_helper_qr.o
 drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o
 obj-$(CONFIG_DRM_KMS_HELPER) += drm_kms_helper.o
 
diff --git a/drivers/gpu/drm/adp/adp-mipi.c b/drivers/gpu/drm/adp/adp-mipi.c
index 232bb95394237e..95653387078196 100644
--- a/drivers/gpu/drm/adp/adp-mipi.c
+++ b/drivers/gpu/drm/adp/adp-mipi.c
@@ -8,6 +8,7 @@
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
 
 #define DSI_GEN_HDR 0x6c
 #define DSI_GEN_PLD_DATA 0x70
diff --git a/drivers/gpu/drm/adp/adp_drv.c b/drivers/gpu/drm/adp/adp_drv.c
index ce9dd6222634df..fdf0e631617564 100644
--- a/drivers/gpu/drm/adp/adp_drv.c
+++ b/drivers/gpu/drm/adp/adp_drv.c
@@ -351,7 +351,7 @@ static const struct drm_crtc_funcs adp_crtc_funcs = {
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = adp_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 105b9dcc19a2b8..7b08814a24da4d 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -91,7 +91,7 @@ amdgpu-y += \
 	nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o soc24.o \
 	sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \
 	nbio_v7_9.o aqua_vanjaram.o nbio_v7_11.o lsdma_v7_0.o hdp_v7_0.o nbif_v6_3_1.o \
-	cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o nbio_v6_3_2.o
+	cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o nbio_v6_3_2.o nbio_v7_11_5.o
 
 # add DF block
 amdgpu-y += \
@@ -278,6 +278,9 @@ amdgpu-y += amdgpu_amdkfd.o
 # add gfx usermode queue
 amdgpu-y += amdgpu_userq.o
 
+# add UALink manager support
+amdgpu-y += amdgpu_ualink.o ualink_v1_0.o
+
 ifneq ($(CONFIG_HSA_AMD),)
 AMDKFD_PATH := ../amdkfd
 include $(FULL_AMD_PATH)/amdkfd/Makefile
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index a9c6f5d4a6397c..3f8f8ebbf356d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -79,7 +79,7 @@
 #include "amdgpu_umsch_mm.h"
 #include "amdgpu_gmc.h"
 #include "amdgpu_gfx.h"
-#include "amdgpu_sdma.h"
+#include "amdgpu_sdma_types.h"
 #include "amdgpu_lsdma.h"
 #include "amdgpu_nbio.h"
 #include "amdgpu_reg_access.h"
@@ -116,6 +116,7 @@
 #include "amdgpu_mes.h"
 #include "amdgpu_sa.h"
 #include "amdgpu_acpi.h"
+#include "amdgpu_ualink.h"
 #if defined(CONFIG_DRM_AMD_ISP)
 #include "amdgpu_isp.h"
 #endif
@@ -156,6 +157,7 @@ struct amdgpu_watchdog_timer {
  * Modules parameters.
  */
 extern int amdgpu_modeset;
+extern int amdgpu_iommu_perfopt;
 extern unsigned int amdgpu_vram_limit;
 extern int amdgpu_vis_vram_limit;
 extern int amdgpu_gart_size;
@@ -208,7 +210,7 @@ extern int amdgpu_backlight;
 extern int amdgpu_damage_clips;
 extern struct amdgpu_mgpu_info mgpu_info;
 extern int amdgpu_ras_enable;
-extern uint amdgpu_ras_mask;
+extern u64 amdgpu_ras_mask;
 extern int amdgpu_bad_page_threshold;
 extern bool amdgpu_ignore_bad_page_threshold;
 extern struct amdgpu_watchdog_timer amdgpu_watchdog_timer;
@@ -222,6 +224,7 @@ extern int amdgpu_force_asic_type;
 extern int amdgpu_smartshift_bias;
 extern int amdgpu_use_xgmi_p2p;
 extern int amdgpu_mtype_local;
+extern int amdgpu_mtype_remote;
 extern int amdgpu_enforce_isolation;
 extern uint amdgpu_debug_mask;
 #ifdef CONFIG_HSA_AMD
@@ -482,6 +485,9 @@ struct amdgpu_asic_funcs {
 	ssize_t (*get_reg_state)(struct amdgpu_device *adev,
 				 enum amdgpu_reg_state reg_state, void *buf,
 				 size_t max_size);
+	/* query FW reserved region (size/offset) from discovery */
+	void (*get_fw_reserved_info)(struct amdgpu_device *adev,
+				     u64 *reserve_size, u64 *offset);
 };
 
 /*
@@ -799,6 +805,9 @@ struct amdgpu_device {
 	/* display related functionality */
 	struct amdgpu_display_manager dm;
 
+	/* UALink manager */
+	struct amdgpu_ualink_mgr	ualink;
+
 #if defined(CONFIG_DRM_AMD_ISP)
 	/* isp */
 	struct amdgpu_isp		isp;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index 9014678d75abff..e475bf2bd1a07a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -588,11 +588,6 @@ static int acp_resume(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool acp_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int acp_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 				     enum amd_clockgating_state state)
 {
@@ -618,7 +613,6 @@ static const struct amd_ip_funcs acp_ip_funcs = {
 	.hw_fini = acp_hw_fini,
 	.suspend = acp_suspend,
 	.resume = acp_resume,
-	.is_idle = acp_is_idle,
 	.set_clockgating_state = acp_set_clockgating_state,
 	.set_powergating_state = acp_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 7f5abb03be1b39..8b8acf98fdfe20 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -1167,8 +1167,10 @@ int amdgpu_acpi_enumerate_xcc(void)
 		}
 
 		xcc_info = kzalloc_obj(struct amdgpu_acpi_xcc_info);
-		if (!xcc_info)
+		if (!xcc_info) {
+			acpi_dev_put(acpi_dev);
 			return -ENOMEM;
+		}
 
 		INIT_LIST_HEAD(&xcc_info->list);
 		xcc_info->handle = acpi_device_handle(acpi_dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 054870e9078d73..afcfb92fb9f5fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -245,6 +245,25 @@ void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev)
 	}
 }
 
+int amdgpu_amdkfd_prepare_partition_switch(struct amdgpu_device *adev)
+{
+	struct kfd_dev *kfd = adev->kfd.dev;
+	int r;
+
+	if (!kfd)
+		return 0;
+	r = kgd2kfd_check_and_lock_kfd(kfd);
+	if (r)
+		return r;
+
+	adev->kfd.init_complete = false;
+	kgd2kfd_device_fini(kfd);
+	adev->kfd.dev = NULL;
+	amdgpu_amdkfd_total_mem_size -= adev->gmc.real_vram_size;
+
+	return 0;
+}
+
 void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
 		const void *ih_ring_entry)
 {
@@ -727,7 +746,7 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
 	job->vmid = vmid;
 	job->num_ibs = 1;
 
-	ret = amdgpu_ib_schedule(ring, 1, ib, job, &f);
+	ret = amdgpu_ib_schedule(ring, job, &f);
 
 	if (ret) {
 		drm_err(adev_to_drm(adev), "failed to schedule IB.\n");
@@ -817,16 +836,6 @@ int amdgpu_amdkfd_send_close_event_drain_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
-int amdgpu_amdkfd_check_and_lock_kfd(struct amdgpu_device *adev)
-{
-	return kgd2kfd_check_and_lock_kfd(adev->kfd.dev);
-}
-
-void amdgpu_amdkfd_unlock_kfd(struct amdgpu_device *adev)
-{
-	kgd2kfd_unlock_kfd(adev->kfd.dev);
-}
-
 
 u64 amdgpu_amdkfd_xcp_memory_size(struct amdgpu_device *adev, int xcp_id)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 1b7dc0d3963b80..e3da382ceb3c0b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -170,8 +170,7 @@ void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
 void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
 void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev);
-int amdgpu_amdkfd_check_and_lock_kfd(struct amdgpu_device *adev);
-void amdgpu_amdkfd_unlock_kfd(struct amdgpu_device *adev);
+int amdgpu_amdkfd_prepare_partition_switch(struct amdgpu_device *adev);
 int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
 				enum kgd_engine_type engine,
 				uint32_t vmid, uint64_t gpu_addr,
@@ -431,6 +430,7 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf);
 bool kgd2kfd_device_init(struct kfd_dev *kfd,
 			 const struct kgd2kfd_shared_resources *gpu_resources);
 void kgd2kfd_device_exit(struct kfd_dev *kfd);
+void kgd2kfd_device_fini(struct kfd_dev *kfd);
 void kgd2kfd_suspend(struct kfd_dev *kfd, bool suspend_proc);
 int kgd2kfd_resume(struct kfd_dev *kfd, bool resume_proc);
 void kgd2kfd_suspend_process(struct kfd_dev *kfd);
@@ -485,6 +485,10 @@ static inline void kgd2kfd_device_exit(struct kfd_dev *kfd)
 {
 }
 
+static inline void kgd2kfd_device_fini(struct kfd_dev *kfd)
+{
+}
+
 static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool suspend_proc)
 {
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
index 7e9f7a280c1b98..98cd0435ced53e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
@@ -193,5 +193,6 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = {
 	.program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,
 	.hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v9_hqd_reset,
-	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell
+	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell,
+	.hqd_gfx_clean_fault = kgd_gfx_v9_clean_fault
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
index bc079b95fc529f..4626e0b139098d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_amdkfd_gfx_v9.h"
 #include "amdgpu_amdkfd_aldebaran.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
index 88acf75f0edd6e..1550e275f0ee0f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
@@ -1026,7 +1026,8 @@ void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *ade
 						uint32_t sch_wave,
 						uint32_t que_sleep,
 						uint32_t *reg_offset,
-						uint32_t *reg_data)
+						uint32_t *reg_data,
+						uint32_t inst)
 {
 	*reg_data = wait_times;
 
@@ -1090,6 +1091,153 @@ uint32_t kgd_gfx_v10_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void lock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+	mutex_lock(&adev->srbm_mutex);
+	mutex_lock(&adev->grbm_idx_mutex);
+
+}
+
+static void unlock_spi_csq_mutexes(struct amdgpu_device *adev)
+{
+	mutex_unlock(&adev->grbm_idx_mutex);
+	mutex_unlock(&adev->srbm_mutex);
+}
+
+/**
+ * get_wave_count: Read device registers to get number of waves in flight for
+ * a particular queue. The method also returns the doorbell offset associated
+ * with the queue.
+ *
+ * @adev: Handle of device whose registers are to be read
+ * @queue_idx: Index of queue in the queue-map bit-field
+ * @queue_cnt: Stores the wave count and doorbell offset for an active queue
+ * @inst: xcc's instance number on a multi-XCC setup
+ */
+static void get_wave_count(struct amdgpu_device *adev, int queue_idx,
+		struct kfd_cu_occupancy *queue_cnt, uint32_t inst)
+{
+	int pipe_idx;
+	int queue_slot;
+	unsigned int reg_val;
+	unsigned int wave_cnt;
+	/*
+	 * Program GRBM with appropriate MEID, PIPEID, QUEUEID and VMID
+	 * parameters to read out waves in flight. Get doorbell offset if there are
+	 * non-zero waves in flight.
+	 */
+	pipe_idx = queue_idx / adev->gfx.mec.num_queue_per_pipe;
+	queue_slot = queue_idx % adev->gfx.mec.num_queue_per_pipe;
+	nv_grbm_select(adev, 1, pipe_idx, queue_slot, 0);
+	reg_val = RREG32_SOC15_IP(GC, SOC15_REG_OFFSET(GC, 0,
+				  mmSPI_CSQ_WF_ACTIVE_COUNT_0) + queue_slot);
+	wave_cnt = reg_val & SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK;
+	if (wave_cnt != 0) {
+		queue_cnt->wave_cnt += wave_cnt;
+		queue_cnt->doorbell_off =
+			(RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL) &
+			 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >>
+			 CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	}
+}
+
+/**
+ * kgd_gfx_v10_get_cu_occupancy: Reads relevant registers associated with each
+ * shader engine and aggregates the number of waves that are in flight for the
+ * process whose pasid is provided as a parameter. The process could have ZERO
+ * or more queues running and submitting waves to compute units.
+ *
+ * @adev: Handle of device from which to get number of waves in flight
+ * @cu_occupancy: Array that gets filled with wave_cnt and doorbell offset
+ *		  for comparison later.
+ * @max_waves_per_cu: Output parameter updated with maximum number of waves
+ *                    possible per Compute Unit
+ * @inst: xcc's instance number on a multi-XCC setup
+ *
+ * Note: It's possible that the device has too many queues (oversubscription)
+ * in which case a VMID could be remapped to a different PASID. This could lead
+ * to an inaccurate wave count. Following is a high-level sequence:
+ *    Time T1: vmid = getVmid(); vmid is associated with Pasid P1
+ *    Time T2: passId = getPasId(vmid); vmid is associated with Pasid P2
+ * In the sequence above wave count obtained from time T1 will be incorrectly
+ * lost or added to total wave count.
+ *
+ * The registers that provide the waves in flight are:
+ *
+ *  SPI_CSQ_WF_ACTIVE_STATUS - bit-map of queues per pipe. The bit is ON if a
+ *  queue is slotted, OFF if there is no queue. A process could have ZERO or
+ *  more queues slotted and submitting waves to be run on compute units. Even
+ *  when there is a queue it is possible there could be zero wave fronts, this
+ *  can happen when queue is waiting on top-of-pipe events - e.g. waitRegMem
+ *  command
+ *
+ *  For each bit that is ON from above:
+ *
+ *    Read (SPI_CSQ_WF_ACTIVE_COUNT_0 + queue_idx) register. It provides the
+ *    number of waves that are in flight for the queue at specified index. The
+ *    index ranges from 0 to 7.
+ *
+ *    If non-zero waves are in flight, store the corresponding doorbell offset
+ *    of the queue, along with the wave count.
+ *
+ *    Determine if the queue belongs to the process by comparing the doorbell
+ *    offset against the process's queues. If it matches, aggregate the wave
+ *    count for the process.
+ *
+ *  Reading registers referenced above involves programming GRBM appropriately
+ */
+void kgd_gfx_v10_get_cu_occupancy(struct amdgpu_device *adev,
+				struct kfd_cu_occupancy *cu_occupancy,
+				int *max_waves_per_cu, uint32_t inst)
+{
+	int qidx;
+	int se_idx;
+	int se_cnt;
+	int queue_map;
+	int max_queue_cnt;
+	DECLARE_BITMAP(cp_queue_bitmap, AMDGPU_MAX_QUEUES);
+
+	lock_spi_csq_mutexes(adev);
+	nv_grbm_select(adev, 1, 0, 0, 0);
+
+	/*
+	 * Iterate through the shader engines and arrays of the device
+	 * to get number of waves in flight
+	 */
+	bitmap_complement(cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap,
+			  AMDGPU_MAX_QUEUES);
+	max_queue_cnt = adev->gfx.mec.num_pipe_per_mec *
+			adev->gfx.mec.num_queue_per_pipe;
+	se_cnt = adev->gfx.config.max_shader_engines;
+	for (se_idx = 0; se_idx < se_cnt; se_idx++) {
+		amdgpu_gfx_select_se_sh(adev, se_idx, 0, 0xffffffff, inst);
+		queue_map = RREG32_SOC15(GC, 0,
+					 mmSPI_CSQ_WF_ACTIVE_STATUS);
+
+		for (qidx = 0; qidx < max_queue_cnt; qidx++) {
+			/* Skip queues that are not associated with
+			 * compute functions
+			 */
+			if (!test_bit(qidx, cp_queue_bitmap))
+				continue;
+
+			if (!(queue_map & (1 << qidx)))
+				continue;
+
+			/* Get number of waves in flight and aggregate them */
+			get_wave_count(adev, qidx, &cu_occupancy[qidx], inst);
+		}
+	}
+
+	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, inst);
+	nv_grbm_select(adev, 0, 0, 0, 0);
+	unlock_spi_csq_mutexes(adev);
+
+	/* Update the output parameters and return */
+	*max_waves_per_cu = adev->gfx.cu_info.simd_per_cu *
+				adev->gfx.cu_info.max_waves_per_simd;
+}
+
 const struct kfd2kgd_calls gfx_v10_kfd2kgd = {
 	.program_sh_mem_settings = kgd_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
@@ -1119,5 +1267,6 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = {
 	.program_trap_handler_settings = program_trap_handler_settings,
 	.hqd_get_pq_addr = kgd_gfx_v10_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v10_hqd_reset,
+	.get_cu_occupancy = kgd_gfx_v10_get_cu_occupancy,
 	.hqd_sdma_get_doorbell = kgd_gfx_v10_hqd_sdma_get_doorbell
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
index a4c607c881783a..a91b638c6bcde6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
@@ -56,7 +56,8 @@ void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *ade
 					       uint32_t sch_wave,
 					       uint32_t que_sleep,
 					       uint32_t *reg_offset,
-					       uint32_t *reg_data);
+					       uint32_t *reg_data,
+					       uint32_t inst);
 uint64_t kgd_gfx_v10_hqd_get_pq_addr(struct amdgpu_device *adev,
 				    uint32_t pipe_id,
 				    uint32_t queue_id,
@@ -68,3 +69,6 @@ uint64_t kgd_gfx_v10_hqd_reset(struct amdgpu_device *adev,
 			      unsigned int utimeout);
 uint32_t kgd_gfx_v10_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
 					   int engine, int queue);
+void kgd_gfx_v10_get_cu_occupancy(struct amdgpu_device *adev,
+				struct kfd_cu_occupancy *cu_occupancy,
+				int *max_waves_per_cu, uint32_t inst);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c
index e31afee429793a..50535563efa86d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c
@@ -683,5 +683,6 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = {
 	.clear_address_watch = kgd_gfx_v10_clear_address_watch,
 	.hqd_get_pq_addr = kgd_gfx_v10_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v10_hqd_reset,
+	.get_cu_occupancy = kgd_gfx_v10_get_cu_occupancy,
 	.hqd_sdma_get_doorbell = kgd_gfx_v10_hqd_sdma_get_doorbell
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
index 070001fd34b0f5..0694bff871d2dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v12_1.c
@@ -518,6 +518,37 @@ static void kgd_gfx_v12_1_get_cu_occupancy(struct amdgpu_device *adev,
 				adev->gfx.cu_info.max_waves_per_simd;
 }
 
+static void kgd_gfx_v12_1_get_iq_wait_times(struct amdgpu_device *adev,
+					uint32_t *wait_times,
+					uint32_t inst)
+{
+	*wait_times = RREG32_SOC15(GC, GET_INST(GC, inst), regCP_IQ_WAIT_TIME2);
+}
+
+static void kgd_gfx_v12_1_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,
+					uint32_t wait_times,
+					uint32_t sch_wave,
+					uint32_t que_sleep,
+					uint32_t *reg_offset,
+					uint32_t *reg_data,
+					uint32_t inst)
+{
+	*reg_data = wait_times;
+
+	if (sch_wave)
+		*reg_data = REG_SET_FIELD(*reg_data,
+				CP_IQ_WAIT_TIME2,
+				SCH_WAVE,
+				sch_wave);
+	if (que_sleep)
+		*reg_data = REG_SET_FIELD(*reg_data,
+				CP_IQ_WAIT_TIME2,
+				QUE_SLEEP,
+				que_sleep);
+
+	*reg_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, inst), regCP_IQ_WAIT_TIME2);
+}
+
 const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = {
 	.init_interrupts = init_interrupts_v12_1,
 	.hqd_dump = hqd_dump_v12_1,
@@ -532,5 +563,8 @@ const struct kfd2kgd_calls gfx_v12_1_kfd2kgd = {
 	.set_address_watch = kgd_gfx_v12_1_set_address_watch,
 	.clear_address_watch = kgd_gfx_v12_1_clear_address_watch,
 	.hqd_sdma_get_doorbell = kgd_gfx_v12_1_hqd_sdma_get_doorbell,
-	.get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy
+	.get_cu_occupancy = kgd_gfx_v12_1_get_cu_occupancy,
+	.get_iq_wait_times = kgd_gfx_v12_1_get_iq_wait_times,
+	.build_dequeue_wait_counts_packet_info =
+				kgd_gfx_v12_1_build_dequeue_wait_counts_packet_info,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index 2e116c06d5be32..0d62cf475f4075 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_amdkfd.h"
 #include "gc/gc_9_0_offset.h"
 #include "gc/gc_9_0_sh_mask.h"
@@ -37,6 +38,7 @@
 #include "soc15.h"
 #include "soc15d.h"
 #include "gfx_v9_0.h"
+#include "gfx_v9_4_2.h"
 #include "amdgpu_amdkfd_gfx_v9.h"
 #include <uapi/linux/kfd_ioctl.h>
 
@@ -1082,7 +1084,8 @@ void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev
 		uint32_t sch_wave,
 		uint32_t que_sleep,
 		uint32_t *reg_offset,
-		uint32_t *reg_data)
+		uint32_t *reg_data,
+		uint32_t inst)
 {
 	*reg_data = wait_times;
 
@@ -1225,11 +1228,15 @@ uint64_t kgd_gfx_v9_hqd_reset(struct amdgpu_device *adev,
 
 uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
 					  int engine, int queue)
-
 {
 	return 0;
 }
 
+void kgd_gfx_v9_clean_fault(struct amdgpu_device *adev)
+{
+	gfx_v9_4_2_clean_fault(adev);
+}
+
 const struct kfd2kgd_calls gfx_v9_kfd2kgd = {
 	.program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping,
@@ -1260,5 +1267,6 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = {
 	.program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,
 	.hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v9_hqd_reset,
-	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell
+	.hqd_sdma_get_doorbell = kgd_gfx_v9_hqd_sdma_get_doorbell,
+	.hqd_gfx_clean_fault = kgd_gfx_v9_clean_fault
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
index 704452ca62f8ee..dba7ace872deb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
@@ -102,7 +102,8 @@ void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev
 					       uint32_t sch_wave,
 					       uint32_t que_sleep,
 					       uint32_t *reg_offset,
-					       uint32_t *reg_data);
+					       uint32_t *reg_data,
+					       uint32_t inst);
 uint64_t kgd_gfx_v9_hqd_get_pq_addr(struct amdgpu_device *adev,
 				    uint32_t pipe_id,
 				    uint32_t queue_id,
@@ -114,3 +115,4 @@ uint64_t kgd_gfx_v9_hqd_reset(struct amdgpu_device *adev,
 			      unsigned int utimeout);
 uint32_t kgd_gfx_v9_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
 					  int engine, int queue);
+void kgd_gfx_v9_clean_fault(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index d66881684ee564..057a4cf8834482 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -22,7 +22,6 @@
  */
 #include <linux/dma-buf.h>
 #include <linux/list.h>
-#include <linux/pagemap.h>
 #include <linux/sched/mm.h>
 #include <linux/sched/task.h>
 #include <drm/ttm/ttm_tt.h>
@@ -1954,7 +1953,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
 		return ret;
 
 	/* Cleanup user pages and MMU notifiers */
-	if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
+	if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
 		amdgpu_hmm_unregister(mem->bo);
 		amdgpu_hmm_range_free(mem->range);
 		mem->range = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 03b41f80352066..31a8e194178f5a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -26,7 +26,6 @@
  */
 
 #include <linux/file.h>
-#include <linux/pagemap.h>
 #include <linux/sync_file.h>
 #include <linux/dma-buf.h>
 
@@ -1146,8 +1145,6 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 
 	if (fpriv->csa_va) {
 		bo_va = fpriv->csa_va;
-		if (!bo_va)
-			return -ENOMEM;
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
@@ -1804,8 +1801,6 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 	struct amdgpu_bo_va_mapping *mapping;
 	int i, r;
 
-	addr /= AMDGPU_GPU_PAGE_SIZE;
-
 	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
 	if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index d53259a5b82f6b..acafa8c59cc909 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -532,7 +532,7 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
 	out->state.flags = 0x0;
 	out->state.hangs = 0x0;
 
-	/* determine if a GPU reset has occured since the last call */
+	/* determine if a GPU reset has occurred since the last call */
 	reset_counter = atomic_read(&adev->gpu_reset_counter);
 	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
 	if (ctx->reset_counter_query == reset_counter)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 132d054900b5b4..1b7673d239dbba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -36,6 +36,7 @@
 #include "amdgpu_rap.h"
 #include "amdgpu_securedisplay.h"
 #include "amdgpu_fw_attestation.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_umr.h"
 
 #include "amdgpu_reset.h"
@@ -1780,8 +1781,10 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
 
 	/* Avoid accidently unparking the sched thread during GPU reset */
 	r = down_write_killable(&adev->reset_domain->sem);
-	if (r)
+	if (r) {
+		pm_runtime_put_autosuspend(dev->dev);
 		return r;
+	}
 
 	/* hold on the scheduler */
 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
@@ -2229,6 +2232,20 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
 	debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
 			    &amdgpu_benchmark_fops);
 
+	/* Debug-only: bitmap of incoming UALink protocol messages to drop.
+	 * Each bit position corresponds to an enum AMDGPU_UALINK_PROTOCOL_MESSAGES
+	 * value. Setting a bit causes exactly one matching incoming packet to be
+	 * dropped, after which the bit auto-clears and traffic resumes. Used to
+	 * exercise the connection reset paths.
+	 *
+	 * Examples (drop one NPA-REQ):
+	 *   echo 0x8  > /sys/kernel/debug/dri/0/amdgpu_ualink_drop_msg_bitmap
+	 * (drop one NPA-RSP):
+	 *   echo 0x10 > /sys/kernel/debug/dri/0/amdgpu_ualink_drop_msg_bitmap
+	 */
+	debugfs_create_ulong("amdgpu_ualink_drop_msg_bitmap", 0600, root,
+			     &adev->ualink.drop_msg_bitmap);
+
 	adev->debugfs_vbios_blob.data = adev->bios;
 	adev->debugfs_vbios_blob.size = adev->bios_size;
 	debugfs_create_blob("amdgpu_vbios", 0444, root,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
index 7fd2a8bc7d0874..30a8b4c5591dd3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c
@@ -256,10 +256,9 @@ amdgpu_devcoredump_print_ibs(struct drm_printer *p,
 			goto unlock;
 
 		for (int i = 0; i < coredump->num_ibs; i++) {
-			u64 pfn = (coredump->ibs[i].gpu_addr &
-				   AMDGPU_GMC_HOLE_MASK) / AMDGPU_GPU_PAGE_SIZE;
+			u64 addr = coredump->ibs[i].gpu_addr & AMDGPU_GMC_HOLE_MASK;
 
-			mapping = amdgpu_vm_bo_lookup_mapping(vm, pfn);
+			mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
 			if (!mapping)
 				continue;
 
@@ -280,8 +279,7 @@ amdgpu_devcoredump_print_ibs(struct drm_printer *p,
 			continue;
 
 		va_start = coredump->ibs[i].gpu_addr & AMDGPU_GMC_HOLE_MASK;
-		mapping = amdgpu_vm_bo_lookup_mapping(vm,
-						      va_start / AMDGPU_GPU_PAGE_SIZE);
+		mapping = amdgpu_vm_bo_lookup_mapping(vm, va_start);
 		if (!mapping)
 			goto output_ib_content;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 9269e780feb730..e6ae48a1974766 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -33,6 +33,7 @@
 #include <linux/console.h>
 #include <linux/slab.h>
 #include <linux/iommu.h>
+#include <linux/amd-iommu.h>
 #include <linux/pci.h>
 #include <linux/pci-p2pdma.h>
 #include <linux/apple-gmux.h>
@@ -68,6 +69,7 @@
 #include "amdgpu_vf_error.h"
 
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_pm.h"
 
 #include "amdgpu_xgmi.h"
@@ -247,6 +249,9 @@ static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev)
 		ret = sysfs_create_file(&adev->dev->kobj,
 					&dev_attr_pcie_replay_count.attr);
 
+	if (!ret)
+		ret = amdgpu_discovery_mem_reserved_info_sysfs_init(adev);
+
 	return ret;
 }
 
@@ -255,6 +260,8 @@ static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev)
 	if (amdgpu_nbio_is_replay_cnt_supported(adev))
 		sysfs_remove_file(&adev->dev->kobj,
 				  &dev_attr_pcie_replay_count.attr);
+
+	amdgpu_discovery_mem_reserved_info_sysfs_fini(adev);
 }
 
 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
@@ -1348,31 +1355,6 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev)
 #endif
 }
 
-/*
- * Some dGPUs expose their display endpoint below an internal PCIe switch.
- * Use the switch upstream port to query the host-facing link.
- */
-static struct pci_dev *amdgpu_device_get_aspm_pdev(struct amdgpu_device *adev)
-{
-	struct pci_dev *swds, *swus;
-
-	swds = pci_upstream_bridge(adev->pdev);
-	if (!swds ||
-	    (swds->vendor != PCI_VENDOR_ID_ATI &&
-	     swds->vendor != PCI_VENDOR_ID_AMD) ||
-	    pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM)
-		return adev->pdev;
-
-	swus = pci_upstream_bridge(swds);
-	if (!swus ||
-	    (swus->vendor != PCI_VENDOR_ID_ATI &&
-	     swus->vendor != PCI_VENDOR_ID_AMD) ||
-	    pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM)
-		return adev->pdev;
-
-	return swus;
-}
-
 /**
  * amdgpu_device_should_use_aspm - check if the device should program ASPM
  *
@@ -1408,7 +1390,7 @@ bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
 	 * the immediate upstream bridge. Use SWUS for dGPUs with an
 	 * internal switch so that this is the host-facing link.
 	 */
-	aspm_pdev = amdgpu_device_get_aspm_pdev(adev);
+	aspm_pdev = adev->link_dev;
 	parent = pci_upstream_bridge(aspm_pdev);
 	if (!parent) {
 		dev_dbg(adev->dev, "ASPM: no upstream PCIe link for %s\n",
@@ -1734,10 +1716,17 @@ static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
 	return atomic_read(&dev->open_count) == 0;
 }
 
+static void amdgpu_switcheroo_pre_switch(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+
+	drm_client_dev_acquire_outputs(dev);
+}
+
 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
 	.set_gpu_state = amdgpu_switcheroo_set_state,
-	.reprobe = NULL,
 	.can_switch = amdgpu_switcheroo_can_switch,
+	.pre_switch = amdgpu_switcheroo_pre_switch,
 };
 
 /**
@@ -1956,15 +1945,26 @@ static void amdgpu_uid_fini(struct amdgpu_device *adev)
 
 static void amdgpu_device_init_pcie_links(struct amdgpu_device *adev)
 {
+	struct pci_dev *swds, *swus;
+
 	adev->link_dev = adev->pdev;
 	adev->link_partner = pci_upstream_bridge(adev->link_dev);
 
-	/* Skip upstream/downstream switches internal to the dGPU. */
-	while (adev->link_partner &&
-	       adev->link_partner->vendor == PCI_VENDOR_ID_ATI) {
-		adev->link_dev = adev->link_partner;
-		adev->link_partner = pci_upstream_bridge(adev->link_dev);
-	}
+	/* Skip an upstream/downstream switch internal to the dGPU. */
+	swds = adev->link_partner;
+	if (!swds || swds->vendor != PCI_VENDOR_ID_ATI ||
+	    pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM)
+		return;
+
+	swus = pci_upstream_bridge(swds);
+	if (!swus ||
+	    (swus->vendor != PCI_VENDOR_ID_ATI &&
+	     swus->vendor != PCI_VENDOR_ID_AMD) ||
+	    pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM)
+		return;
+
+	adev->link_dev = swus;
+	adev->link_partner = pci_upstream_bridge(swus);
 }
 
 /**
@@ -2501,7 +2501,8 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 	/**
 	 * In case of XGMI grab extra reference for reset domain for this device
 	 */
-	if (adev->gmc.xgmi.num_physical_nodes > 1) {
+	if (adev->gmc.xgmi.num_physical_nodes > 1 &&
+	    adev->gmc.xgmi.supported) {
 		if (amdgpu_xgmi_add_device(adev) == 0) {
 			if (!amdgpu_sriov_vf(adev)) {
 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
@@ -2784,7 +2785,7 @@ static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
 	     adev->asic_type == CHIP_ALDEBARAN))
 		amdgpu_dpm_handle_passthrough_sbr(adev, true);
 
-	if (adev->gmc.xgmi.num_physical_nodes > 1) {
+	if (adev->gmc.xgmi.num_physical_nodes > 1 && adev->gmc.xgmi.supported) {
 		mutex_lock(&mgpu_info.mutex);
 
 		/*
@@ -2947,7 +2948,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
 		amdgpu_virt_release_ras_err_handler_data(adev);
 
-	if (adev->gmc.xgmi.num_physical_nodes > 1)
+	if (adev->gmc.xgmi.num_physical_nodes > 1 && adev->gmc.xgmi.supported)
 		amdgpu_xgmi_remove_device(adev);
 
 	amdgpu_amdkfd_device_fini_sw(adev);
@@ -3721,6 +3722,7 @@ static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev)
 	amdgpu_xcp_sysfs_init(adev);
 	amdgpu_uma_sysfs_init(adev);
 	amdgpu_ptl_sysfs_init(adev);
+	amdgpu_ualink_sysfs_init(adev);
 
 	return r;
 }
@@ -3742,6 +3744,7 @@ static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev)
 	amdgpu_xcp_sysfs_fini(adev);
 	amdgpu_uma_sysfs_fini(adev);
 	amdgpu_ptl_sysfs_fini(adev);
+	amdgpu_ualink_sysfs_fini(adev);
 }
 
 static bool
@@ -3752,6 +3755,28 @@ amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px)
 		       apple_gmux_detect(NULL, NULL)));
 }
 
+static inline bool amdgpu_device_identity(struct amdgpu_device *adev)
+{
+	struct pci_dev *pdev = adev->pdev;
+	struct iommu_domain *domain = iommu_get_domain_for_dev(&pdev->dev);
+
+	if (!domain)
+		return false;
+
+	return domain->type == IOMMU_DOMAIN_IDENTITY;
+}
+
+static bool amdgpu_device_use_perfopt(struct amdgpu_device *adev)
+{
+	if (amdgpu_iommu_perfopt == 0)
+		return false;
+
+	if (!(adev->flags & AMD_IS_APU))
+		return false;
+
+	return amdgpu_device_identity(adev);
+}
+
 /**
  * amdgpu_device_init - initialize the driver
  *
@@ -3840,6 +3865,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 
 	spin_lock_init(&adev->irq.lock);
 
+	amdgpu_ip_map_init(adev);
 	amdgpu_early_init_rlc_reg_funcs(adev);
 	amdgpu_device_init_apu_flags(adev);
 
@@ -3851,6 +3877,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	spin_lock_init(&adev->mm_stats.lock);
 	spin_lock_init(&adev->virt.rlcg_reg_lock);
 	spin_lock_init(&adev->wb.lock);
+	mutex_init(&adev->lsdma.lock);
 
 	INIT_LIST_HEAD(&adev->reset_list);
 
@@ -3959,6 +3986,16 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	if (r)
 		return r;
 
+	if (amdgpu_device_use_perfopt(adev)) {
+		int perfopt_ret = amd_iommu_enable_perfopt(pdev);
+
+		/* Optional optimization; a failure to arm it must not abort probe. */
+		if (perfopt_ret)
+			dev_warn(adev->dev,
+				 "Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
+				 perfopt_ret);
+	}
+
 	/*
 	 * No need to remove conflicting FBs for non-display class devices.
 	 * This prevents the sysfb from being freed accidently.
@@ -4002,24 +4039,21 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	}
 
 	/* enable PCIE atomic ops */
-	if (amdgpu_sriov_vf(adev)) {
-		if (adev->virt.fw_reserve.p_pf2vf)
-			adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
-						      adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
-				(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
-	/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
-	 * internal path natively support atomics, set have_atomics_support to true.
-	 */
-	} else if ((adev->flags & AMD_IS_APU &&
-		   amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) ||
-		   (adev->gmc.xgmi.connected_to_cpu &&
-		   amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0))) {
-		adev->have_atomics_support = true;
-	} else {
-		adev->have_atomics_support =
-			!pci_enable_atomic_ops_to_root(adev->pdev,
-					  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
-					  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
+	if (!amdgpu_sriov_vf(adev)) {
+		/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
+		 * internal path natively support atomics, set have_atomics_support to true.
+		 */
+		if ((adev->flags & AMD_IS_APU &&
+		     amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) ||
+		    (adev->gmc.xgmi.connected_to_cpu &&
+		     amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0))) {
+			adev->have_atomics_support = true;
+		} else {
+			adev->have_atomics_support =
+				!pci_enable_atomic_ops_to_root(adev->pdev,
+						  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
+						  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
+		}
 	}
 
 	if (!adev->have_atomics_support)
@@ -4328,6 +4362,9 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 
 	amdgpu_gart_dummy_page_fini(adev);
 
+	if (amdgpu_device_use_perfopt(adev))
+		amd_iommu_disable_perfopt(adev->pdev);
+
 	if (pci_dev_is_disconnected(adev->pdev))
 		amdgpu_device_unmap_mmio(adev);
 
@@ -4691,6 +4728,20 @@ int amdgpu_device_resume(struct drm_device *dev, bool notify_clients)
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
+	if (amdgpu_device_use_perfopt(adev)) {
+		int perfopt_ret = amd_iommu_enable_perfopt(adev->pdev);
+
+		/*
+		 * Must not return on failure: a bare return would leak the
+		 * SR-IOV VF exclusive-mode acquisition taken above (released
+		 * via the exit: path).
+		 */
+		if (perfopt_ret)
+			dev_warn(adev->dev,
+				 "Failed to enable IOMMU PerfOpt (%d); continuing without it\n",
+				 perfopt_ret);
+	}
+
 	if (adev->in_s0ix)
 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
 
@@ -6418,16 +6469,10 @@ static void amdgpu_device_cache_switch_state(struct amdgpu_device *adev)
 	struct pci_dev *swus, *swds;
 	int r;
 
-	swds = pci_upstream_bridge(adev->pdev);
-	if (!swds || swds->vendor != PCI_VENDOR_ID_ATI ||
-	    pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM)
-		return;
-	swus = pci_upstream_bridge(swds);
-	if (!swus ||
-	    (swus->vendor != PCI_VENDOR_ID_ATI &&
-	     swus->vendor != PCI_VENDOR_ID_AMD) ||
-	    pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM)
+	swus = adev->link_dev != adev->pdev ? adev->link_dev : NULL;
+	if (!swus)
 		return;
+	swds = pci_upstream_bridge(adev->pdev);
 
 	/* If already saved, return */
 	if (adev->pcie_reset_ctx.swus)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index 62f5c5cbd7f946..ad21986d7b1b3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -26,6 +26,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_discovery.h"
+#include "amdgpu_ualink.h"
 #include "soc15_hw_ip.h"
 #include "discovery.h"
 #include "amdgpu_ras.h"
@@ -44,6 +45,7 @@
 #include "nbio_v7_4.h"
 #include "nbio_v7_9.h"
 #include "nbio_v7_11.h"
+#include "nbio_v7_11_5.h"
 #include "hdp_v4_0.h"
 #include "vega10_ih.h"
 #include "vega20_ih.h"
@@ -356,6 +358,7 @@ static int amdgpu_discovery_get_tmr_info(struct amdgpu_device *adev,
 			tmr_offset_hi = RREG32(mmDRIVER_SCRATCH_1);
 			adev->discovery.offset = ((u64)le32_to_cpu(tmr_offset_hi) << 32 |
 						  le32_to_cpu(tmr_offset_lo));
+			adev->discovery.reserve_tmr = true;
 		} else if (!vram_size) {
 			/* fall back to apci approach to query tmr offset if vram_size is 0 */
 			ret = amdgpu_acpi_get_tmr_info(adev, &tmr_offset, &tmr_size);
@@ -678,6 +681,16 @@ static int amdgpu_discovery_table_check(struct amdgpu_device *adev,
 		check_table = false;
 		break;
 	}
+	case NPS_INFO: {
+		table_name = "nps table";
+		check_table = false;
+		break;
+	}
+	case MEM_RESERVED_INFO:
+		/* Optional table; signature is validated in get_mem_reserved_info_table(). */
+		table_name = "mem_reserved table";
+		check_table = false;
+		break;
 	default:
 		dev_err(adev->dev, "invalid ip discovery table id %d specified\n", table_id);
 		check_table = false;
@@ -700,6 +713,21 @@ static int amdgpu_discovery_table_check(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void amdgpu_discovery_dump_mem_reserved_info_table(struct amdgpu_device *adev,
+		struct mem_reserved_info_table_v1_0 *table);
+
+static void amdgpu_discovery_log_bad_signature(struct amdgpu_device *adev,
+					       uint8_t *discovery_bin,
+					       const char *fw_name)
+{
+	dev_err(adev->dev,
+		"get invalid ip discovery binary signature: got 0x%08x, expected 0x%08x (source=%s)\n",
+		le32_to_cpu(((struct binary_header *)discovery_bin)->binary_signature),
+		BINARY_SIGNATURE, fw_name ? "file" : "mem");
+	print_hex_dump(KERN_ERR, "ip_discovery head: ", DUMP_PREFIX_OFFSET,
+		       16, 1, discovery_bin, 64, false);
+}
+
 static int amdgpu_discovery_init(struct amdgpu_device *adev)
 {
 	struct binary_header *bhdr;
@@ -735,8 +763,7 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev)
 
 	/* check the ip discovery binary signature */
 	if (!amdgpu_discovery_verify_binary_signature(discovery_bin)) {
-		dev_err(adev->dev,
-			"get invalid ip discovery binary signature\n");
+		amdgpu_discovery_log_bad_signature(adev, discovery_bin, fw_name);
 		r = -EINVAL;
 		goto out;
 	}
@@ -755,12 +782,20 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev)
 		goto out;
 	}
 
-	for (table_id = 0; table_id <= MALL_INFO; table_id++) {
+	for (table_id = 0; table_id < TOTAL_TABLES; table_id++) {
 		r = amdgpu_discovery_table_check(adev, discovery_bin, table_id);
 		if (r)
 			goto out;
 	}
 
+	/*
+	 * Resolve and dump the MEM_RESERVED_INFO table once at init time.
+	 * The table is optional, so missing it is not a fatal error.
+	 */
+	if (!amdgpu_discovery_get_mem_reserved_info_table(adev))
+		amdgpu_discovery_dump_mem_reserved_info_table(adev,
+				adev->discovery.mem_reserved_table);
+
 	return 0;
 
 out:
@@ -779,6 +814,8 @@ void amdgpu_discovery_fini(struct amdgpu_device *adev)
 
 	kfree(adev->discovery.bin);
 	adev->discovery.bin = NULL;
+	/* Cached pointer lives inside discovery.bin; drop it to avoid UAF. */
+	adev->discovery.mem_reserved_table = NULL;
 }
 
 static int amdgpu_discovery_validate_ip(struct amdgpu_device *adev,
@@ -925,6 +962,12 @@ static void amdgpu_discovery_read_from_harvest_table(struct amdgpu_device *adev,
 		case SDMA0_HWID:
 			adev->sdma.sdma_mask &= ~BIT(inst);
 			break;
+		case MMHUB_HWID:
+			adev->mmhub.inst_mask &= ~BIT(inst);
+			break;
+		case LSDMA_HWID:
+			adev->lsdma.inst_mask &= ~BIT(inst);
+			break;
 #if defined(CONFIG_DRM_AMD_ISP)
 		case ISP_HWID:
 			adev->isp.harvest_config |= ~BIT(inst);
@@ -1215,6 +1258,12 @@ static uint8_t amdgpu_discovery_get_harvest_info(struct amdgpu_device *adev,
 	case SDMA0_HWID:
 		harvest = ((1 << inst) & adev->sdma.sdma_mask) == 0;
 		break;
+	case MMHUB_HWID:
+		harvest = (BIT(inst) & adev->mmhub.inst_mask) == 0;
+		break;
+	case LSDMA_HWID:
+		harvest = (BIT(inst) & adev->lsdma.inst_mask) == 0;
+		break;
 	default:
 		break;
 	}
@@ -1759,6 +1808,8 @@ static int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev)
 	adev->sdma.sdma_mask = 0;
 	adev->vcn.inst_mask = 0;
 	adev->jpeg.inst_mask = 0;
+	adev->mmhub.inst_mask = 0;
+	adev->lsdma.inst_mask = 0;
 	r = amdgpu_discovery_get_table_info(adev, &info, IP_DISCOVERY);
 	if (r)
 		return r;
@@ -1824,6 +1875,25 @@ static int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev)
 				}
 				ip->revision &= ~0xc0;
 			}
+
+			if (le16_to_cpu(ip->hw_id) == MMHUB_HWID) {
+				if (inst < AMDGPU_MAX_MMHUB_INSTANCES)
+					adev->mmhub.inst_mask |= BIT(inst);
+				else
+					dev_err(adev->dev, "Too many MMHUB instances: %d vs %d\n",
+						inst + 1,
+						AMDGPU_MAX_MMHUB_INSTANCES);
+			}
+
+			if (le16_to_cpu(ip->hw_id) == LSDMA_HWID) {
+				if (inst < AMDGPU_MAX_LSDMA_INSTANCES)
+					adev->lsdma.inst_mask |= BIT(inst);
+				else
+					dev_err(adev->dev, "Too many LSDMA instances: %d vs %d\n",
+						inst + 1,
+						AMDGPU_MAX_LSDMA_INSTANCES);
+			}
+
 			if (le16_to_cpu(ip->hw_id) == SDMA0_HWID ||
 			    le16_to_cpu(ip->hw_id) == SDMA1_HWID ||
 			    le16_to_cpu(ip->hw_id) == SDMA2_HWID ||
@@ -1986,6 +2056,7 @@ union gc_info {
 	struct gc_info_v1_1 v1_1;
 	struct gc_info_v1_2 v1_2;
 	struct gc_info_v1_3 v1_3;
+	struct gc_info_v1_5 v1_5;
 	struct gc_info_v2_0 v2;
 	struct gc_info_v2_1 v2_1;
 };
@@ -1995,6 +2066,7 @@ static int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
 	uint8_t *discovery_bin = adev->discovery.bin;
 	struct table_info *info;
 	union gc_info *gc_info;
+	uint32_t num_wgps_per_sa, num_wgps_per_sa1;
 	u16 offset;
 
 	if (!discovery_bin) {
@@ -2015,7 +2087,7 @@ static int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
 	case 1:
 		adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->v1.gc_num_se);
 		adev->gfx.config.max_cu_per_sh = 2 * (le32_to_cpu(gc_info->v1.gc_num_wgp0_per_sa) +
-						      le32_to_cpu(gc_info->v1.gc_num_wgp1_per_sa));
+					      le32_to_cpu(gc_info->v1.gc_num_wgp1_per_sa));
 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gc_info->v1.gc_num_sa_per_se);
 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gc_info->v1.gc_num_rb_per_se);
 		adev->gfx.config.max_texture_channel_caches = le32_to_cpu(gc_info->v1.gc_num_gl2c);
@@ -2056,6 +2128,26 @@ static int amdgpu_discovery_get_gfx_info(struct amdgpu_device *adev)
 			adev->gfx.config.gc_tcc_size = le32_to_cpu(gc_info->v1_3.gc_tcc_size);
 			adev->gfx.config.gc_tcc_cache_line_size = le32_to_cpu(gc_info->v1_3.gc_tcc_cache_line_size);
 		}
+		if (le16_to_cpu(gc_info->v1.header.version_minor) == 4 ||
+		    le16_to_cpu(gc_info->v1.header.version_minor) > 5) {
+			dev_err(adev->dev,
+				"Unsupported GC info table %d.%d\n",
+				le16_to_cpu(gc_info->v1.header.version_major),
+				le16_to_cpu(gc_info->v1.header.version_minor));
+			return -EINVAL;
+		}
+		if (le16_to_cpu(gc_info->v1.header.version_minor) == 5) {
+			adev->gfx.config.gc_max_num_residency_ways = le32_to_cpu(gc_info->v1_5.gc_max_num_residency_ways);
+			adev->gfx.config.gc_cache_ways_size_in_bytes = le32_to_cpu(gc_info->v1_5.gc_cache_ways_size_in_bytes);
+			adev->gfx.config.gc_reserved = le32_to_cpu(gc_info->v1_5.gc_reserved);
+
+			num_wgps_per_sa = le32_to_cpu(gc_info->v1.gc_num_wgp0_per_sa) +
+					  le32_to_cpu(gc_info->v1.gc_num_wgp1_per_sa);
+			num_wgps_per_sa1 = le32_to_cpu(gc_info->v1_5.gc_num_wgp0_per_sa1) +
+					   le32_to_cpu(gc_info->v1_5.gc_num_wgp1_per_sa1);
+			adev->gfx.config.max_cu_per_sh = num_wgps_per_sa >= num_wgps_per_sa1 ?
+							 num_wgps_per_sa : num_wgps_per_sa1;
+		}
 		break;
 	case 2:
 		adev->gfx.config.max_shader_engines = le32_to_cpu(gc_info->v2.gc_num_se);
@@ -2325,6 +2417,193 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 	return 0;
 }
 
+static const char *amdgpu_discovery_mem_reserved_region_name(u32 id)
+{
+	switch (id) {
+	case NO_RESERVED_REGION_ID:		return "NO_RESERVED_REGION";
+	case PRE_OS_DISP_FW_REGION_ID:		return "PRE_OS_DISP_FW";
+	case MASTER_DIE_UMF_REGION_ID:		return "MASTER_DIE_UMF";
+	case DCC_META_DATA_REGION_ID:		return "DCC_META_DATA";
+	case VM_PAGE_FAULT_REGION_ID:		return "VM_PAGE_FAULT";
+	case G7_PSTATE_APERTURE1_ID:		return "G7_PSTATE_APERTURE1";
+	case G7_PSTATE_APERTURE2_ID:		return "G7_PSTATE_APERTURE2";
+	case G7_PSTATE_MIRROR_REGION_ID:	return "G7_PSTATE_MIRROR";
+	case G7_TRAINING_DATA_REGION_ID:	return "G7_TRAINING_DATA";
+	case SPECIFIC_PURPOSE_REGION_ID:	return "SPECIFIC_PURPOSE";
+	default:				return "UNKNOWN";
+	}
+}
+
+/* Dump all the mem_reserved_info table entries for debugging */
+static void amdgpu_discovery_dump_mem_reserved_info_table(struct amdgpu_device *adev,
+		struct mem_reserved_info_table_v1_0 *table)
+{
+	u32 list_num = min_t(u32, le32_to_cpu(table->list_num), MAX_MEM_REGION_NUM);
+	u32 id, i;
+
+	dev_info(adev->dev, "MEM_RESERVED_INFO: list_num=%u\n", list_num);
+	for (i = 0; i < list_num; i++) {
+		id = le32_to_cpu(table->list[i].reserved_region_id);
+		dev_info(adev->dev, "  [%u] id=%u (%s) size=0x%llx start=0x%llx\n",
+			 i, id,
+			 amdgpu_discovery_mem_reserved_region_name(id),
+			 le64_to_cpu(table->list[i].reserved_region_size),
+			 le64_to_cpu(table->list[i].reserved_region_start));
+	}
+}
+
+/**
+ * Add READ-ONLY mem_reserved_info sysfs entry.
+ * MEM_RESERVED_INFO table is parsed from the IP discovery binary.
+ * The file mem_reserved_info dumps every entry as:
+ *   list_num=<N>
+ *   [<i>] id=<id> (<name>) size=<size> start=<start>
+ */
+static ssize_t mem_reserved_info_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	struct mem_reserved_info_table_v1_0 *table;
+	u32 list_num, id, i;
+	ssize_t len = 0;
+
+	if (amdgpu_discovery_get_mem_reserved_info_table(adev))
+		return sysfs_emit(buf, "MEM_RESERVED_INFO table not available\n");
+
+	table = adev->discovery.mem_reserved_table;
+	list_num = min_t(u32, le32_to_cpu(table->list_num), MAX_MEM_REGION_NUM);
+
+	len += sysfs_emit_at(buf, len, "list_num=%u\n", list_num);
+	for (i = 0; i < list_num; i++) {
+		id = le32_to_cpu(table->list[i].reserved_region_id);
+		len += sysfs_emit_at(buf, len,
+			"[%u] id=%u (%s) size=0x%llx start=0x%llx\n",
+			i, id,
+			amdgpu_discovery_mem_reserved_region_name(id),
+			le64_to_cpu(table->list[i].reserved_region_size),
+			le64_to_cpu(table->list[i].reserved_region_start));
+	}
+
+	return len;
+}
+
+static DEVICE_ATTR_RO(mem_reserved_info);
+
+int amdgpu_discovery_mem_reserved_info_sysfs_init(struct amdgpu_device *adev)
+{
+	if (!adev->discovery.mem_reserved_table)
+		return 0;
+
+	return sysfs_create_file(&adev->dev->kobj,
+				 &dev_attr_mem_reserved_info.attr);
+}
+
+void amdgpu_discovery_mem_reserved_info_sysfs_fini(struct amdgpu_device *adev)
+{
+	if (!adev->discovery.mem_reserved_table)
+		return;
+
+	sysfs_remove_file(&adev->dev->kobj,
+			  &dev_attr_mem_reserved_info.attr);
+}
+
+/*
+ * Resolve the MEM_RESERVED_INFO table from the IP discovery binary and
+ * cache it in adev->discovery.mem_reserved_table.
+ * Return: 0 on success, -ENOENT/-EINVAL if the table unavailable.
+ */
+int amdgpu_discovery_get_mem_reserved_info_table(struct amdgpu_device *adev)
+{
+	uint8_t *discovery_bin = adev->discovery.bin;
+	struct mem_reserved_info_header *mrhdr;
+	struct binary_header *bhdr;
+	struct table_info *info;
+	u16 offset;
+
+	/* If already queried, do not query again. */
+	if (adev->discovery.mem_reserved_table)
+		return 0;
+
+	if (!discovery_bin) {
+		dev_err(adev->dev, "ip discovery uninitialized\n");
+		return -ENOENT;
+	}
+
+	/* MEM_RESERVED_INFO only exists in binary_header >= v2. Skip populating it on v1 */
+	bhdr = (struct binary_header *)discovery_bin;
+	if (le16_to_cpu(bhdr->version_major) < 2) {
+		dev_dbg(adev->dev, "header version = %d\n", le16_to_cpu(bhdr->version_major));
+		return -ENOENT;
+	}
+
+	if (amdgpu_discovery_get_table_info(adev, &info, MEM_RESERVED_INFO)) {
+		dev_dbg(adev->dev, "MEM_RESERVED_INFO table entry not present\n");
+		return -EINVAL;
+	}
+
+	offset = le16_to_cpu(info->offset);
+	if (!offset) {
+		dev_dbg(adev->dev, "MEM_RESERVED_INFO table offset is 0, invalid!\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Only populated when the signature matches; skip ASICs whose
+	 * slot points to unrelated data.
+	 */
+	mrhdr = (struct mem_reserved_info_header *)(discovery_bin + offset);
+	if (le32_to_cpu(mrhdr->signature) != MEM_RSV_TABLE_SIGNATURE) {
+		dev_dbg(adev->dev, "MEM_RESERVED_INFO table signature mismatch, skipping\n");
+		return -ENOENT;
+	}
+
+	/* Cache for subsequent lookups. */
+	adev->discovery.mem_reserved_table =
+		(struct mem_reserved_info_table_v1_0 *)mrhdr;
+
+	dev_dbg(adev->dev, "MEM_RESERVED_INFO table exist\n");
+	return 0;
+}
+
+/**
+ * Query mem_reserved_info by region ID
+ *
+ * @reserved_region_id: MEM_RESERVED_REGION_ID to look up
+ * @info:  pass NULL to skip
+ */
+int amdgpu_discovery_get_mem_reserved_region_by_id(struct amdgpu_device *adev,
+						   u32 reserved_region_id,
+						   struct mem_reserved_info *info)
+{
+	struct mem_reserved_info_table_v1_0 *table;
+	u32 list_num, i;
+
+	if (!adev->discovery.mem_reserved_table)
+		return -ENOENT;
+
+	table = adev->discovery.mem_reserved_table;
+	list_num = min_t(u32, le32_to_cpu(table->list_num), MAX_MEM_REGION_NUM);
+
+	for (i = 0; i < list_num; i++) {
+		if (le32_to_cpu(table->list[i].reserved_region_id) != reserved_region_id)
+			continue;
+
+		if (info) {
+			info->reserved_region_id = reserved_region_id;
+			info->reserved_region_size =
+				le64_to_cpu(table->list[i].reserved_region_size);
+			info->reserved_region_start =
+				le64_to_cpu(table->list[i].reserved_region_start);
+		}
+		dev_dbg(adev->dev, "Found reserved region: %s (id=%u)\n",
+			amdgpu_discovery_mem_reserved_region_name(reserved_region_id),
+			reserved_region_id);
+		return 0;
+	}
+	return -ENOENT;
+}
+
 static int amdgpu_discovery_set_common_ip_blocks(struct amdgpu_device *adev)
 {
 	/* what IP to use for this? */
@@ -2851,6 +3130,7 @@ static int amdgpu_discovery_set_ras_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 6):
 	case IP_VERSION(13, 0, 12):
 	case IP_VERSION(13, 0, 14):
+	case IP_VERSION(15, 0, 8):
 		amdgpu_device_ip_block_add(adev, &ras_v1_0_ip_block);
 		break;
 	default:
@@ -3001,8 +3281,7 @@ static int amdgpu_discovery_set_mes_ip_blocks(struct amdgpu_device *adev)
 		amdgpu_device_ip_block_add(adev, &mes_v12_1_ip_block);
 		adev->enable_mes = true;
 		adev->enable_mes_kiq = true;
-		if (amdgpu_uni_mes)
-			adev->enable_uni_mes = true;
+		adev->enable_uni_mes = true;
 		break;
 	default:
 		break;
@@ -3080,6 +3359,22 @@ static int amdgpu_discovery_set_isp_ip_blocks(struct amdgpu_device *adev)
 	return 0;
 }
 
+static int amdgpu_discovery_set_ualink_ip_blocks(struct amdgpu_device *adev)
+{
+	/* TODO: no dedicated IP discovery version for UALink yet; key off the
+	 * GC IP version for now.
+	 */
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(12, 1, 0):
+		amdgpu_device_ip_block_add(adev, &ualink_v1_0_ip_block);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
 int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 {
 	int r;
@@ -3499,10 +3794,13 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		break;
 	case IP_VERSION(6, 3, 1):
 	case IP_VERSION(7, 11, 4):
-	case IP_VERSION(7, 11, 5):
 		adev->nbio.funcs = &nbif_v6_3_1_funcs;
 		adev->nbio.hdp_flush_reg = &nbif_v6_3_1_hdp_flush_reg;
 		break;
+	case IP_VERSION(7, 11, 5):
+		adev->nbio.funcs = &nbio_v7_11_5_funcs;
+		adev->nbio.hdp_flush_reg = &nbio_v7_11_5_hdp_flush_reg;
+		break;
 	case IP_VERSION(6, 3, 2):
 		adev->nbio.funcs = &nbio_v6_3_2_funcs;
 		break;
@@ -3726,6 +4024,10 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		return r;
 
 	r = amdgpu_discovery_set_isp_ip_blocks(adev);
+	if (r)
+		return r;
+
+	r = amdgpu_discovery_set_ualink_ip_blocks(adev);
 	if (r)
 		return r;
 	return 0;
@@ -3756,3 +4058,27 @@ int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev,
 		*minor = le16_to_cpu(gc_info->v1.header.version_minor);
 	return 0;
 }
+
+int amdgpu_discovery_get_die_rev_id(struct amdgpu_device *adev,
+				    uint16_t *die_rev_id)
+{
+	uint8_t *discovery_bin = adev->discovery.bin;
+	struct ip_discovery_header *ihdr;
+	struct table_info *info;
+	u16 offset;
+
+	if (!discovery_bin)
+		return -EINVAL;
+	if (amdgpu_discovery_get_table_info(adev, &info, IP_DISCOVERY))
+		return -EINVAL;
+
+	offset = le16_to_cpu(info->offset);
+	if (!offset)
+		return -EINVAL;
+
+	ihdr = (struct ip_discovery_header *)(discovery_bin + offset);
+
+	if (die_rev_id)
+		*die_rev_id = le16_to_cpu(ihdr->die_info[0].die_id);
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
index 5b2b16f68576cd..6e1d55e78790a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
@@ -25,6 +25,7 @@
 #define __AMDGPU_DISCOVERY__
 
 #include <linux/debugfs.h>
+#include "discovery.h"
 
 #define DISCOVERY_TMR_SIZE      (10 << 10)
 #define DISCOVERY_TMR_OFFSET    (64 << 10)
@@ -39,6 +40,7 @@ struct amdgpu_discovery_info {
 	uint32_t size;
 	uint8_t *bin;
 	bool reserve_tmr;
+	struct mem_reserved_info_table_v1_0 *mem_reserved_table;
 };
 
 void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev);
@@ -51,6 +53,8 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 				  int *range_cnt, bool refresh);
 int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev,
 						uint16_t *major, uint16_t *minor);
+int amdgpu_discovery_get_die_rev_id(struct amdgpu_device *adev,
+				    uint16_t *die_rev_id);
 
 void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p);
 
@@ -59,4 +63,13 @@ int amdgpu_discovery_sysfs_early_init(struct amdgpu_device *adev,
 				       struct pci_dev *pdev);
 void amdgpu_discovery_sysfs_early_fini(struct pci_dev *pdev);
 
+int amdgpu_discovery_get_mem_reserved_info_table(struct amdgpu_device *adev);
+
+int amdgpu_discovery_get_mem_reserved_region_by_id(struct amdgpu_device *adev,
+						   u32 reserved_region_id,
+						   struct mem_reserved_info *info);
+
+int amdgpu_discovery_mem_reserved_info_sysfs_init(struct amdgpu_device *adev);
+void amdgpu_discovery_mem_reserved_info_sysfs_fini(struct amdgpu_device *adev);
+
 #endif /* __AMDGPU_DISCOVERY__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index 9adf3eed8822c7..4031ebe075d81a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -599,6 +599,11 @@ struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev,
 			 */
 			drm_gem_object_get(obj);
 			return obj;
+		} else {
+			struct ttm_resource *mem = gem_to_amdgpu_bo(obj)->tbo.resource;
+			/* NPA DMA-buf can only be imported on the same device. */
+			if (mem && mem->mem_type == AMDGPU_PL_NPA)
+				return ERR_PTR(-EPERM);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 5c33c19fd9bc5d..75e35c81250b0b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -54,6 +54,7 @@
 #include "amdgpu_userq.h"
 #include "amdgpu_userq_fence.h"
 #include "../amdxcp/amdgpu_xcp_drv.h"
+#include "amdgpu_ualink.h"
 
 /*
  * KMS wrapper.
@@ -126,9 +127,10 @@
  * - 3.62.0 - Add AMDGPU_IDS_FLAGS_MODE_PF, AMDGPU_IDS_FLAGS_MODE_VF & AMDGPU_IDS_FLAGS_MODE_PT
  * - 3.63.0 - GFX12 display DCC supports 256B max compressed block size
  * - 3.64.0 - Userq IP support query
+ * - 3.65.0 - Add userq syncobj timeline signaling support
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	64
+#define KMS_DRIVER_MINOR	65
 #define KMS_DRIVER_PATCHLEVEL	0
 
 /*
@@ -147,6 +149,7 @@ enum AMDGPU_DEBUG_MASK {
 	AMDGPU_DEBUG_ENABLE_CE_CS = BIT(10),
 	AMDGPU_DEBUG_HIBERNATION_THAW_RESUME_GPU = BIT(11),
 	AMDGPU_DEBUG_DISABLE_IP_BLOCK_SOFT_RESET = BIT(12),
+	AMDGPU_DEBUG_SDMA_RB_CMD = BIT(13),
 };
 
 unsigned int amdgpu_vram_limit = UINT_MAX;
@@ -185,6 +188,7 @@ char *amdgpu_disable_cu;
 char *amdgpu_virtual_display;
 int amdgpu_enforce_isolation = -1;
 int amdgpu_modeset = -1;
+int amdgpu_iommu_perfopt = -1;
 
 /* Specifies the default granularity for SVM, used in buffer
  * migration and restoration of backing memory when handling
@@ -216,8 +220,9 @@ int amdgpu_smu_pptable_id = -1;
  * DISABLE_FRACTIONAL_PWM (bit 2) disabled by default
  * PSR (bit 3) disabled by default
  * EDP NO POWER SEQUENCING (bit 4) disabled by default
+ * FRL (bit 10) enabled by default
  */
-uint amdgpu_dc_feature_mask = 2;
+uint amdgpu_dc_feature_mask = DC_MULTI_MON_PP_MCLK_SWITCH_MASK | DC_FRL_MASK;
 uint amdgpu_dc_debug_mask;
 uint amdgpu_dc_visual_confirm;
 int amdgpu_async_gfx_ring = 1;
@@ -264,7 +269,7 @@ struct amdgpu_mgpu_info mgpu_info = {
 	.mutex = __MUTEX_INITIALIZER(mgpu_info.mutex),
 };
 int amdgpu_ras_enable = -1;
-uint amdgpu_ras_mask = 0xffffffff;
+u64 amdgpu_ras_mask = U64_MAX;
 int amdgpu_bad_page_threshold = -1;
 struct amdgpu_watchdog_timer amdgpu_watchdog_timer = {
 	.timeout_fatal_disable = false,
@@ -392,6 +397,17 @@ module_param_named(fw_load_type, amdgpu_fw_load_type, int, 0444);
 MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(aspm, amdgpu_aspm, int, 0444);
 
+/**
+ * DOC: iommu_perfopt (int)
+ * Control the AMD IOMMU PerfOpt DMA-latency optimization
+ * (0 = disable; -1 = enable on supported devices).
+ * This arms the IOMMU PerfOpt control (IOMMU spec, MMIO Offset 016Ch, EFR PerfOptSup / PerfOptEn)
+ * Arming it disables ATS, PRI, PASID and SVA for the GPU and removes IOMMU DMA containment for it,
+ * trading isolation for lower DMA latency.
+ */
+MODULE_PARM_DESC(iommu_perfopt, "Control IOMMU PerfOpt DMA-latency optimization (-1 = enable on supported devices, 0 = disable)");
+module_param_named(iommu_perfopt, amdgpu_iommu_perfopt, int, 0444);
+
 /**
  * DOC: runpm (int)
  * Override for runtime power management control for dGPUs. The amdgpu driver can dynamically power down
@@ -596,12 +612,12 @@ MODULE_PARM_DESC(ras_enable, "Enable RAS features on the GPU (0 = disable, 1 = e
 module_param_named(ras_enable, amdgpu_ras_enable, int, 0444);
 
 /**
- * DOC: ras_mask (uint)
- * Mask of RAS features to enable (default 0xffffffff), only valid when ras_enable == 1
+ * DOC: ras_mask (ullong)
+ * Mask of RAS features to enable (default 0xffffffffffffffff), only valid when ras_enable == 1
  * See the flags in drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
  */
-MODULE_PARM_DESC(ras_mask, "Mask of RAS features to enable (default 0xffffffff), only valid when ras_enable == 1");
-module_param_named(ras_mask, amdgpu_ras_mask, uint, 0444);
+MODULE_PARM_DESC(ras_mask, "Mask of RAS features to enable (default 0xffffffffffffffff), only valid when ras_enable == 1");
+module_param_named(ras_mask, amdgpu_ras_mask, ullong, 0444);
 
 /**
  * DOC: timeout_fatal_disable (bool)
@@ -842,6 +858,13 @@ int amdgpu_mtype_local = -1;
 MODULE_PARM_DESC(mtype_local, "MTYPE for local memory (default: ASIC dependent, 0 = MTYPE_RW, 1 = MTYPE_NC, 2 = MTYPE_CC)");
 module_param_named_unsafe(mtype_local, amdgpu_mtype_local, int, 0444);
 
+/**
+ * DOC: mtype_remote (int)
+ */
+int amdgpu_mtype_remote = -1;
+MODULE_PARM_DESC(mtype_remote, "MTYPE for remote memory (default: ASIC dependent, 0 = MTYPE_NC, 1 = MTYPE_UC)");
+module_param_named_unsafe(mtype_remote, amdgpu_mtype_remote, int, 0444);
+
 /**
  * DOC: pcie_p2p (bool)
  * Enable PCIe P2P (requires large-BAR). Default value: true (on)
@@ -2300,6 +2323,11 @@ static void amdgpu_init_debug_options(struct amdgpu_device *adev)
 		pr_info("debug: IP block soft reset disabled\n");
 		adev->debug_disable_ip_block_soft_reset = true;
 	}
+
+	if (amdgpu_debug_mask & AMDGPU_DEBUG_SDMA_RB_CMD) {
+		pr_info("debug: enable SDMA RB command switch\n");
+		adev->sdma.sdma_debug = true;
+	}
 }
 
 static unsigned long amdgpu_fix_asic_type(struct pci_dev *pdev, unsigned long flags)
@@ -3105,6 +3133,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
 	DRM_IOCTL_DEF_DRV(AMDGPU_USERQ_WAIT, amdgpu_userq_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_GEM_LIST_HANDLES, amdgpu_gem_list_handles_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(AMDGPU_PROC_OPTIONS, amdgpu_proc_options_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(AMDGPU_UALINK_HANDLE, amdgpu_gem_ualink_handle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW)
 };
 
 static const struct drm_driver amdgpu_kms_driver = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
index f6b7522c3c82ff..f8652fd0525df7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c
@@ -68,6 +68,9 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
 
 	mutex_lock(&uq_mgr->userq_mutex);
 
+	/* Fence waits are not allowed in a fence signalling critical section. */
+	amdgpu_userq_wait_for_signal(uq_mgr);
+
 	/*
 	 * This is intentionally after taking the userq_mutex since we do
 	 * allocate memory while holding this lock, but only after ensuring that
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 165344f7197fb5..88d522de93151a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -28,7 +28,6 @@
 #include <linux/ktime.h>
 #include <linux/module.h>
 #include <linux/overflow.h>
-#include <linux/pagemap.h>
 #include <linux/pci.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-fence-unwrap.h>
@@ -386,6 +385,14 @@ static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_str
 	return drm_gem_ttm_mmap(obj, vma);
 }
 
+static void amdgpu_gem_object_handle_free(struct drm_gem_object *gobj)
+{
+	struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
+
+	amdgpu_ualink_revoke_exported_memory(aobj);
+
+}
+
 const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
 	.free = amdgpu_gem_object_free,
 	.open = amdgpu_gem_object_open,
@@ -395,6 +402,7 @@ const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
 	.vunmap = drm_gem_ttm_vunmap,
 	.mmap = amdgpu_gem_object_mmap,
 	.vm_ops = &amdgpu_gem_vm_ops,
+	.handle_free = amdgpu_gem_object_handle_free
 };
 
 static bool amdgpu_gem_are_domains_valid(u32 domains)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index a6f95ff47d24e6..7a132e6d7e4294 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -1235,7 +1235,8 @@ uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg, uint32_t xcc_
 	if (adev->mes.ring[0].sched.ready)
 		return amdgpu_mes_rreg(adev, reg, xcc_id);
 
-	BUG_ON(!ring->funcs->emit_rreg);
+	if (!ring->funcs || !ring->funcs->emit_rreg)
+		goto failed_kiq_read;
 
 	spin_lock_irqsave(&kiq->ring_lock, flags);
 	if (amdgpu_wb_get(adev, &reg_val_offs)) {
@@ -1303,8 +1304,6 @@ void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint3
 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
 	struct amdgpu_ring *ring = &kiq->ring;
 
-	BUG_ON(!ring->funcs->emit_wreg);
-
 	if (amdgpu_device_skip_hw_access(adev))
 		return;
 
@@ -1313,6 +1312,9 @@ void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, uint3
 		return;
 	}
 
+	if (!ring->funcs || !ring->funcs->emit_wreg)
+		goto failed_kiq_write;
+
 	spin_lock_irqsave(&kiq->ring_lock, flags);
 	r = amdgpu_ring_alloc(ring, 32);
 	if (r)
@@ -1409,9 +1411,8 @@ int amdgpu_kiq_hdp_flush(struct amdgpu_device *adev)
 	if (adev->enable_mes_kiq && adev->mes.ring[0].sched.ready)
 		return amdgpu_mes_hdp_flush(adev);
 
-	if (!ring->funcs->emit_hdp_flush) {
+	if (!ring->funcs || !ring->funcs->emit_hdp_flush)
 		return -EOPNOTSUPP;
-	}
 
 	spin_lock_irqsave(&kiq->ring_lock, flags);
 	r = amdgpu_ring_alloc(ring, 32);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 1023719c18bb66..f5fb2e585e8ae8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -307,6 +307,9 @@ struct amdgpu_gfx_config {
 	uint32_t gc_scalar_data_cache_size_per_sqc;
 	uint32_t gc_scalar_data_cache_line_size;
 	uint32_t gc_tcc_cache_line_size;
+	uint32_t gc_max_num_residency_ways;
+	uint32_t gc_cache_ways_size_in_bytes;
+	uint32_t gc_reserved;
 };
 
 struct amdgpu_cu_info {
@@ -475,6 +478,7 @@ struct amdgpu_gfx {
 	struct amdgpu_irq_src		sq_irq;
 	struct amdgpu_irq_src		rlc_gc_fed_irq;
 	struct amdgpu_irq_src		rlc_poison_irq;
+	struct amdgpu_irq_src		pmr_ea_irq;
 	struct sq_work			sq_work;
 
 	/* gfx status */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 3dc8faa091d7e5..1bf2a42fa63cee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -33,6 +33,8 @@
 #include "amdgpu_gmc.h"
 #include "amdgpu_ras.h"
 #include "amdgpu_reset.h"
+#include "amdgpu_ring.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_xgmi.h"
 #include "amdgpu_atomfirmware.h"
 
@@ -672,7 +674,8 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
 			continue;
 
 		/* Skip if the ring is a shared ring */
-		if (amdgpu_sdma_is_shared_inv_eng(adev, ring))
+		if (amdgpu_sdma_is_shared_inv_eng(adev, ring) ||
+		    amdgpu_jpeg_is_shared_inv_eng(adev, ring))
 			continue;
 
 		inv_eng = ffs(vm_inv_engs[vmhub]);
@@ -702,6 +705,12 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
 					ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub);
 			continue;
 		}
+
+		/* All decode rings within one JPEG instance share one VM
+		 * invalidation engine (no-op unless ring is a JPEG instance's
+		 * first ring).
+		 */
+		amdgpu_jpeg_set_shared_inv_eng(adev, ring);
 	}
 
 	return 0;
@@ -1010,9 +1019,6 @@ void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)
 				gc_ver == IP_VERSION(9, 5, 0) ||
 				gc_ver >= IP_VERSION(10, 1, 0));
 
-	/* For GFX12.1 B0, set xnack (retry) on as default */
-	if (gc_ver == IP_VERSION(12, 1, 0) && (adev->rev_id & 0xf) == 0x1)
-		noretry_default = false;
 	if (!amdgpu_sriov_xnack_support(adev))
 		gmc->noretry = 1;
 	else
@@ -1360,7 +1366,8 @@ int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev)
 	if (!adev->gmc.gmc_funcs->query_mem_partition_mode)
 		return 0;
 
-	nps_switch_support = (hweight32(adev->gmc.supported_nps_modes &
+	nps_switch_support = !adev->gmc.xgmi.connected_to_cpu &&
+			     (hweight32(adev->gmc.supported_nps_modes &
 					AMDGPU_ALL_NPS_MASK) > 1);
 	if (!nps_switch_support)
 		dev_attr_current_memory_partition.attr.mode &=
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index c0884797dd544b..cffd5ca3fc66b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -114,6 +114,8 @@ struct amdgpu_gmc_fault {
 struct amdgpu_vmhub_funcs {
 	void (*print_l2_protection_fault_status)(struct amdgpu_device *adev,
 						 uint32_t status);
+	void (*print_l2_protection_fault_status_hi)(struct amdgpu_device *adev,
+						    uint32_t status);
 	uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t flush_type);
 };
 
@@ -125,6 +127,7 @@ struct amdgpu_vmhub {
 	uint32_t	vm_inv_eng0_ack;
 	uint32_t	vm_context0_cntl;
 	uint32_t	vm_l2_pro_fault_status;
+	uint32_t	vm_l2_pro_fault_status_hi;
 	uint32_t	vm_l2_pro_fault_cntl;
 
 	/*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 360e6f00cb7c0a..0f8b097001e3ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -27,8 +27,6 @@
  *          Christian König
  */
 #include <linux/seq_file.h>
-#include <linux/slab.h>
-
 #include <drm/amdgpu_drm.h>
 
 #include "amdgpu.h"
@@ -102,11 +100,9 @@ void amdgpu_ib_free(struct amdgpu_ib *ib, struct dma_fence *f)
 /**
  * amdgpu_ib_schedule - schedule an IB (Indirect Buffer) on the ring
  *
- * @ring: ring index the IB is associated with
- * @num_ibs: number of IBs to schedule
- * @ibs: IB objects to schedule
+ * @ring: ring structure the IB is associated with
  * @job: job to schedule
- * @f: fence created during this submission
+ * @f: fence referenced during this submission
  *
  * Schedule an IB on the associated ring (all asics).
  * Returns 0 on success, error on failure.
@@ -121,92 +117,71 @@ void amdgpu_ib_free(struct amdgpu_ib *ib, struct dma_fence *f)
  * a CONST_IB), it will be put on the ring prior to the DE IB.  Prior
  * to SI there was just a DE IB.
  */
-int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
-		       struct amdgpu_ib *ibs, struct amdgpu_job *job,
+int amdgpu_ib_schedule(struct amdgpu_ring *ring, struct amdgpu_job *job,
 		       struct dma_fence **f)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib *ib = &ibs[0];
+	struct amdgpu_ib *ib;
 	struct dma_fence *tmp = NULL;
 	struct amdgpu_fence *af;
 	struct amdgpu_fence *vm_af;
 	bool need_ctx_switch;
 	bool emit_spm_needed = false;
 	bool emit_gds_needed = false;
-	struct amdgpu_vm *vm;
 	uint64_t fence_ctx;
 	uint32_t status = 0, alloc_size;
 	unsigned int fence_flags = 0;
-	bool secure, init_shadow;
-	u64 shadow_va, csa_va, gds_va;
-	int vmid = AMDGPU_JOB_GET_VMID(job);
+	bool secure;
+	unsigned int vmid;
 	bool need_pipe_sync = false;
 	unsigned int cond_exec;
 	unsigned int i;
-	int r = 0;
+	int r;
 
-	if (num_ibs == 0)
+	if (!job || !job->num_ibs)
 		return -EINVAL;
 
-	/* ring tests don't use a job */
-	if (job) {
-		vm = job->vm;
-		fence_ctx = job->base.s_fence ?
-			job->base.s_fence->finished.context : 0;
-		shadow_va = job->shadow_va;
-		csa_va = job->csa_va;
-		gds_va = job->gds_va;
-		init_shadow = job->init_shadow;
-		af = job->hw_fence;
-		/* Save the context of the job for reset handling.
-		 * The driver needs this so it can skip the ring
-		 * contents for guilty contexts.
-		 */
-		af->context = fence_ctx;
-		/* the vm fence is also part of the job's context */
-		job->hw_vm_fence->context = fence_ctx;
-	} else {
-		vm = NULL;
-		fence_ctx = 0;
-		shadow_va = 0;
-		csa_va = 0;
-		gds_va = 0;
-		init_shadow = false;
-		af = kzalloc_obj(*af, GFP_ATOMIC);
-		if (!af)
-			return -ENOMEM;
-	}
+	vmid = AMDGPU_JOB_GET_VMID(job);
+	ib = &job->ibs[0];
+	fence_ctx = job->base.s_fence ?
+		job->base.s_fence->finished.context : 0;
+	af = job->hw_fence;
+	vm_af = job->hw_vm_fence;
+	/* Save the context of the job for reset handling.
+	 * The driver needs this so it can skip the ring
+	 * contents for guilty contexts.
+	 */
+	af->context = fence_ctx;
+	/* the vm fence is also part of the job's context */
+	vm_af->context = fence_ctx;
 
 	if (!ring->sched.ready) {
 		dev_err(adev->dev, "couldn't schedule ib on ring <%s>\n", ring->name);
-		r = -EINVAL;
-		goto free_fence;
+		return -EINVAL;
 	}
 
-	if (vm && !job->vmid) {
+	if (job->vm && !job->vmid) {
 		dev_err(adev->dev, "VM IB without ID\n");
-		r = -EINVAL;
-		goto free_fence;
+		return -EINVAL;
 	}
 
 	if ((ib->flags & AMDGPU_IB_FLAGS_SECURE) &&
 	    (!ring->funcs->secure_submission_supported)) {
 		dev_err(adev->dev, "secure submissions not supported on ring <%s>\n", ring->name);
-		r = -EINVAL;
-		goto free_fence;
+		return -EINVAL;
 	}
 
-	alloc_size = ring->funcs->emit_frame_size + num_ibs *
+	alloc_size = ring->funcs->emit_frame_size + job->num_ibs *
 		ring->funcs->emit_ib_size;
 
 	r = amdgpu_ring_alloc(ring, alloc_size);
 	if (r) {
 		dev_err(adev->dev, "scheduling IB failed (%d).\n", r);
-		goto free_fence;
+		return r;
 	}
 
 	need_ctx_switch = ring->current_ctx != fence_ctx;
-	if (ring->funcs->emit_pipeline_sync && job &&
+	if (ring->funcs->emit_pipeline_sync &&
 	    ((tmp = amdgpu_sync_get_fence(&job->explicit_sync)) ||
 	     need_ctx_switch || amdgpu_vm_need_pipeline_sync(ring, job))) {
 
@@ -218,15 +193,12 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		dma_fence_put(tmp);
 	}
 
-	if (job) {
-		vm_af = job->hw_vm_fence;
-		/* VM sequence */
-		vm_af->ib_wptr = ring->wptr;
-		amdgpu_vm_flush(ring, job, &need_pipe_sync, &emit_spm_needed,
-				&emit_gds_needed);
-		vm_af->ib_dw_size =
-			amdgpu_ring_get_dw_distance(ring, vm_af->ib_wptr, ring->wptr);
-	}
+	/* VM sequence */
+	vm_af->ib_wptr = ring->wptr;
+	amdgpu_vm_flush(ring, job, &need_pipe_sync, &emit_spm_needed,
+			&emit_gds_needed);
+	vm_af->ib_dw_size =
+		amdgpu_ring_get_dw_distance(ring, vm_af->ib_wptr, ring->wptr);
 
 	/* IB sequence */
 	af->ib_wptr = ring->wptr;
@@ -256,8 +228,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		ring->funcs->emit_wave_limit(ring, true);
 
 	if (ring->funcs->emit_gfx_shadow && adev->gfx.cp_gfx_shadow)
-		amdgpu_ring_emit_gfx_shadow(ring, shadow_va, csa_va, gds_va,
-					    init_shadow, vmid);
+		amdgpu_ring_emit_gfx_shadow(ring, job->shadow_va, job->csa_va, job->gds_va,
+					    job->init_shadow, vmid);
 
 	if (ring->funcs->init_cond_exec)
 		cond_exec = amdgpu_ring_init_cond_exec(ring,
@@ -271,7 +243,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 	if (need_ctx_switch)
 		status |= AMDGPU_HAVE_CTX_SWITCH;
 
-	if (job && ring->funcs->emit_cntxcntl) {
+	if (ring->funcs->emit_cntxcntl) {
 		status |= job->preamble_status;
 		status |= job->preemption_status;
 		amdgpu_ring_emit_cntxcntl(ring, status);
@@ -280,15 +252,15 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 	/* Setup initial TMZiness and send it off.
 	 */
 	secure = false;
-	if (job && ring->funcs->emit_frame_cntl) {
+	if (ring->funcs->emit_frame_cntl) {
 		secure = ib->flags & AMDGPU_IB_FLAGS_SECURE;
 		amdgpu_ring_emit_frame_cntl(ring, true, secure);
 	}
 
-	for (i = 0; i < num_ibs; ++i) {
-		ib = &ibs[i];
+	for (i = 0; i < job->num_ibs; ++i) {
+		ib = &job->ibs[i];
 
-		if (job && ring->funcs->emit_frame_cntl) {
+		if (ring->funcs->emit_frame_cntl) {
 			if (secure != !!(ib->flags & AMDGPU_IB_FLAGS_SECURE)) {
 				amdgpu_ring_emit_frame_cntl(ring, false, secure);
 				secure = !secure;
@@ -300,7 +272,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		status &= ~AMDGPU_HAVE_CTX_SWITCH;
 	}
 
-	if (job && ring->funcs->emit_frame_cntl)
+	if (ring->funcs->emit_frame_cntl)
 		amdgpu_ring_emit_frame_cntl(ring, false, secure);
 
 	amdgpu_device_invalidate_hdp(adev, ring);
@@ -312,7 +284,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 		fence_flags |= AMDGPU_FENCE_FLAG_TC_WB_ONLY;
 
 	/* wrap the last IB with fence */
-	if (job && job->uf_addr) {
+	if (job->uf_addr) {
 		amdgpu_ring_emit_fence(ring, job->uf_addr, job->uf_sequence,
 				       fence_flags | AMDGPU_FENCE_FLAG_64BIT);
 	}
@@ -326,8 +298,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 	amdgpu_fence_emit(ring, af, fence_flags);
 	*f = &af->base;
 	/* get a ref for the job */
-	if (job)
-		dma_fence_get(*f);
+	dma_fence_get(*f);
 
 	if (ring->funcs->insert_end)
 		ring->funcs->insert_end(ring);
@@ -335,7 +306,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 	amdgpu_ring_patch_cond_exec(ring, cond_exec);
 
 	ring->current_ctx = fence_ctx;
-	if (job && ring->funcs->emit_switch_buffer)
+	if (ring->funcs->emit_switch_buffer)
 		amdgpu_ring_emit_switch_buffer(ring);
 
 	if (ring->funcs->emit_wave_limit &&
@@ -349,11 +320,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs,
 	amdgpu_ring_commit(ring);
 
 	return 0;
-
-free_fence:
-	if (!job)
-		kfree(af);
-	return r;
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
index 8a2d64f0ebc3fc..5e424e3b6e71f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -228,12 +228,12 @@ void amdgpu_pasid_free_delayed(struct dma_resv *resv,
  */
 
 /**
- * amdgpu_vmid_had_gpu_reset - check if reset occured since last use
+ * amdgpu_vmid_had_gpu_reset - check if reset occurred since last use
  *
  * @adev: amdgpu_device pointer
  * @id: VMID structure
  *
- * Check if GPU reset occured since last use of the VMID.
+ * Check if GPU reset occurred since last use of the VMID.
  */
 bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev,
 			       struct amdgpu_vmid *id)
@@ -625,12 +625,27 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev)
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
 		struct amdgpu_vmid_mgr *id_mgr =
 			&adev->vm_manager.id_mgr[i];
-
-		for (j = 1; j < id_mgr->num_ids; ++j)
+		for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID)
 			amdgpu_vmid_reset(adev, i, j);
 	}
 }
 
+void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev,
+				   unsigned long vmid_mask, bool for_mmhub)
+{
+	unsigned int i;
+
+	BUILD_BUG_ON(AMDGPU_NUM_VMID > BITS_PER_LONG);
+
+	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) {
+		bool is_mmhub = AMDGPU_IS_MMHUB0(i) || AMDGPU_IS_MMHUB1(i);
+
+		if (is_mmhub == for_mmhub)
+			bitmap_copy(adev->vm_manager.id_mgr[i].vmid_mask,
+				    &vmid_mask, AMDGPU_NUM_VMID);
+	}
+}
+
 /**
  * amdgpu_vmid_mgr_init - init the VMID manager
  *
@@ -649,19 +664,8 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev)
 		mutex_init(&id_mgr->lock);
 		INIT_LIST_HEAD(&id_mgr->ids_lru);
 
-		/* for GC <10, SDMA uses MMHUB so use first_kfd_vmid for both GC and MM */
-		if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 0, 0))
-			/* manage only VMIDs not used by KFD */
-			id_mgr->num_ids = adev->vm_manager.first_kfd_vmid;
-		else if (AMDGPU_IS_MMHUB0(i) ||
-			 AMDGPU_IS_MMHUB1(i))
-			id_mgr->num_ids = 16;
-		else
-			/* manage only VMIDs not used by KFD */
-			id_mgr->num_ids = adev->vm_manager.first_kfd_vmid;
-
 		/* skip over VMID 0, since it is the system VM */
-		for (j = 1; j < id_mgr->num_ids; ++j) {
+		for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID) {
 			amdgpu_vmid_reset(adev, i, j);
 			amdgpu_sync_create(&id_mgr->ids[j].active);
 			list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
@@ -685,6 +689,7 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev)
 			&adev->vm_manager.id_mgr[i];
 
 		mutex_destroy(&id_mgr->lock);
+
 		for (j = 0; j < AMDGPU_NUM_VMID; ++j) {
 			struct amdgpu_vmid *id = &id_mgr->ids[j];
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
index 4b55d0d9703a33..46b2c216012633 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
@@ -26,6 +26,8 @@
 #include <linux/types.h>
 #include <linux/mutex.h>
 #include <linux/list.h>
+#include <linux/bitops.h>
+#include <linux/find.h>
 #include <linux/dma-fence.h>
 
 #include "amdgpu_sync.h"
@@ -33,6 +35,27 @@
 /* maximum number of VMIDs */
 #define AMDGPU_NUM_VMID	16
 
+/**
+ * for_each_vmid_and_zero - iterate over VMID 0 plus every VMID set in a hub's
+ *                          vmid_mask
+ * @vmid:	loop cursor (int / unsigned int)
+ * @adev:	struct amdgpu_device *
+ * @hub:	vmhub index into adev->vm_manager.id_mgr[]
+ *
+ * Equivalent to OR-ing BIT(0) into id_mgr[hub].vmid_mask and walking the
+ * result with for_each_set_bit(). Used by the gfx_v*_constants_init() paths
+ * which must always touch VMID 0 (system) in addition to the regular VMIDs
+ * owned by the hub.
+ *
+ * Relies on AMDGPU_NUM_VMID (16) fitting in a single unsigned long, i.e.
+ * vmid_mask[] being a 1-element bitmap.
+ */
+#define for_each_vmid_and_zero(vmid, adev, hub)				\
+	for (unsigned long __vmid_mask =				\
+		(adev)->vm_manager.id_mgr[(hub)].vmid_mask[0] | BIT(0);	\
+	     __vmid_mask; __vmid_mask = 0)				\
+		for_each_set_bit((vmid), &__vmid_mask, AMDGPU_NUM_VMID)
+
 struct amdgpu_device;
 struct amdgpu_fpriv;
 struct amdgpu_vm;
@@ -65,10 +88,10 @@ struct amdgpu_vmid {
 
 struct amdgpu_vmid_mgr {
 	struct mutex		lock;
-	unsigned		num_ids;
 	struct list_head	ids_lru;
 	struct amdgpu_vmid	ids[AMDGPU_NUM_VMID];
 	bool			reserved_vmid;
+	DECLARE_BITMAP(vmid_mask, AMDGPU_NUM_VMID);
 };
 
 int amdgpu_pasid_alloc(unsigned int bits, struct amdgpu_fpriv *fpriv);
@@ -95,5 +118,7 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev);
 
 void amdgpu_vmid_mgr_init(struct amdgpu_device *adev);
 void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev);
+void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev,
+				   unsigned long vmid_mask, bool for_mmhub);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index 444437c3008868..8d74a89b40cf84 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -29,6 +29,8 @@
 
 #define IH_RING_SIZE	(256 * 1024)
 #define IH_SW_RING_SIZE	(16 * 1024)	/* enough for 512 CAM entries */
+#define IH_PSP_RING_SIZE (SZ_4K)
+#define IH_UALINK_RING_SIZE    (256 * 1024)
 
 struct amdgpu_device;
 struct amdgpu_iv_entry;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
index eb8537558f5fb6..9c56be725ff3d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
@@ -39,6 +39,7 @@ struct amdgpu_imu_funcs {
     int (*switch_compute_partition)(struct amdgpu_device *adev,
 				    int num_xccs_per_xcp,
 				    int compute_partition_mode);
+    void (*init_mcm_addr_lut)(struct amdgpu_device *adev);
 };
 
 struct imu_rlc_ram_golden {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
index 922f4b15619d55..734f2a0bc79aff 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.c
@@ -24,39 +24,16 @@
 #include "amdgpu.h"
 #include "amdgpu_ip.h"
 
-static int8_t amdgpu_logical_to_dev_inst(struct amdgpu_device *adev,
-					 enum amd_hw_ip_block_type block,
-					 int8_t inst)
+u32 amdgpu_ip_map_logical_to_dev_mask(struct amdgpu_ip_map_info *ip_map,
+				      enum amd_hw_ip_block_type block,
+				      u32 mask)
 {
-	int8_t dev_inst;
-
-	switch (block) {
-	case GC_HWIP:
-	case SDMA0_HWIP:
-	/* Both JPEG and VCN as JPEG is only alias of VCN */
-	case VCN_HWIP:
-		dev_inst = adev->ip_map.dev_inst[block][inst];
-		break;
-	default:
-		/* For rest of the IPs, no look up required.
-		 * Assume 'logical instance == physical instance' for all configs. */
-		dev_inst = inst;
-		break;
-	}
-
-	return dev_inst;
-}
-
-static uint32_t amdgpu_logical_to_dev_mask(struct amdgpu_device *adev,
-					   enum amd_hw_ip_block_type block,
-					   uint32_t mask)
-{
-	uint32_t dev_mask = 0;
-	int8_t log_inst, dev_inst;
+	unsigned int log_inst, dev_inst;
+	u32 dev_mask = 0;
 
 	while (mask) {
 		log_inst = ffs(mask) - 1;
-		dev_inst = amdgpu_logical_to_dev_inst(adev, block, log_inst);
+		dev_inst = ip_map->dev_inst[block][log_inst];
 		dev_mask |= (1 << dev_inst);
 		mask &= ~(1 << log_inst);
 	}
@@ -64,35 +41,15 @@ static uint32_t amdgpu_logical_to_dev_mask(struct amdgpu_device *adev,
 	return dev_mask;
 }
 
-static void amdgpu_populate_ip_map(struct amdgpu_device *adev,
-				   enum amd_hw_ip_block_type ip_block,
-				   uint32_t inst_mask)
-{
-	int l = 0, i;
-
-	while (inst_mask) {
-		i = ffs(inst_mask) - 1;
-		adev->ip_map.dev_inst[ip_block][l++] = i;
-		inst_mask &= ~(1 << i);
-	}
-	for (; l < HWIP_MAX_INSTANCE; l++)
-		adev->ip_map.dev_inst[ip_block][l] = -1;
-}
-
 void amdgpu_ip_map_init(struct amdgpu_device *adev)
 {
-	u32 ip_map[][2] = {
-		{ GC_HWIP, adev->gfx.xcc_mask },
-		{ SDMA0_HWIP, adev->sdma.sdma_mask },
-		{ VCN_HWIP, adev->vcn.inst_mask },
-	};
-	int i;
+	int i, j;
 
-	for (i = 0; i < ARRAY_SIZE(ip_map); ++i)
-		amdgpu_populate_ip_map(adev, ip_map[i][0], ip_map[i][1]);
-
-	adev->ip_map.logical_to_dev_inst = amdgpu_logical_to_dev_inst;
-	adev->ip_map.logical_to_dev_mask = amdgpu_logical_to_dev_mask;
+	/* Assume 'logical instance == physical instance' unless overridden. */
+	for (i = 0; i < MAX_HWIP; i++) {
+		for (j = 0; j < HWIP_MAX_INSTANCE; j++)
+			adev->ip_map.dev_inst[i][j] = j;
+	}
 }
 
 int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block)
@@ -198,6 +155,7 @@ static const char *const ip_block_names[] = {
 	[AMD_IP_BLOCK_TYPE_UMSCH_MM] = "umsch_mm",
 	[AMD_IP_BLOCK_TYPE_ISP] = "isp",
 	[AMD_IP_BLOCK_TYPE_RAS] = "ras",
+	[AMD_IP_BLOCK_TYPE_UALINK] = "ualink",
 };
 
 static const char *ip_block_name(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
index 70fc4e5db51f3f..5dd6273982c42f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ip.h
@@ -90,13 +90,7 @@ struct amdgpu_fence;
 
 struct amdgpu_ip_map_info {
 	/* Map of logical to actual dev instances/mask */
-	uint32_t dev_inst[MAX_HWIP][HWIP_MAX_INSTANCE];
-	int8_t (*logical_to_dev_inst)(struct amdgpu_device *adev,
-				      enum amd_hw_ip_block_type block,
-				      int8_t inst);
-	uint32_t (*logical_to_dev_mask)(struct amdgpu_device *adev,
-					enum amd_hw_ip_block_type block,
-					uint32_t mask);
+	unsigned int dev_inst[MAX_HWIP][HWIP_MAX_INSTANCE];
 };
 
 #define AMDGPU_MAX_IP_NUM AMD_IP_BLOCK_TYPE_NUM
@@ -125,6 +119,18 @@ struct amdgpu_ip_block {
 
 void amdgpu_ip_map_init(struct amdgpu_device *adev);
 
+u32 amdgpu_ip_map_logical_to_dev_mask(struct amdgpu_ip_map_info *ip_map,
+				      enum amd_hw_ip_block_type block,
+				      u32 mask);
+
+static inline unsigned int
+amdgpu_ip_map_logical_to_dev_inst(struct amdgpu_ip_map_info *ip_map,
+				  enum amd_hw_ip_block_type block,
+				  unsigned int inst)
+{
+	return ip_map->dev_inst[block][inst];
+}
+
 int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block);
 int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 95cceed4e971a5..dcf90a974326de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -257,6 +257,22 @@ static void amdgpu_irq_handle_ih_soft(struct work_struct *work)
 	amdgpu_ih_process(adev, &adev->irq.ih_soft);
 }
 
+static void amdgpu_irq_handle_ih_psp(struct work_struct *work)
+{
+	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
+						  irq.ih_psp_work);
+
+	amdgpu_ih_process(adev, &adev->irq.ih_psp);
+}
+	
+static void amdgpu_irq_handle_ih_ualink(struct work_struct *work)
+{
+	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
+						  irq.ih_ualink_work);
+
+	amdgpu_ih_process(adev, &adev->irq.ih_ualink);
+}
+
 /**
  * amdgpu_msi_ok - check whether MSI functionality is enabled
  *
@@ -332,6 +348,8 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
 	INIT_WORK(&adev->irq.ih1_work, amdgpu_irq_handle_ih1);
 	INIT_WORK(&adev->irq.ih2_work, amdgpu_irq_handle_ih2);
 	INIT_WORK(&adev->irq.ih_soft_work, amdgpu_irq_handle_ih_soft);
+	INIT_WORK(&adev->irq.ih_psp_work, amdgpu_irq_handle_ih_psp);
+	INIT_WORK(&adev->irq.ih_ualink_work, amdgpu_irq_handle_ih_ualink);
 
 	/* Use vector 0 for MSI-X. */
 	r = pci_irq_vector(adev->pdev, 0);
@@ -370,6 +388,7 @@ void amdgpu_irq_fini_hw(struct amdgpu_device *adev)
 	}
 
 	amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft);
+	amdgpu_ih_ring_fini(adev, &adev->irq.ih_psp);
 	amdgpu_ih_ring_fini(adev, &adev->irq.ih);
 	amdgpu_ih_ring_fini(adev, &adev->irq.ih1);
 	amdgpu_ih_ring_fini(adev, &adev->irq.ih2);
@@ -548,6 +567,22 @@ void amdgpu_irq_delegate(struct amdgpu_device *adev,
 	queue_work(system_dfl_wq, &adev->irq.ih_soft_work);
 }
 
+void amdgpu_irq_psp_delegate(struct amdgpu_device *adev,
+			     struct amdgpu_iv_entry *entry,
+			     unsigned int num_dw)
+{
+	amdgpu_ih_ring_write(adev, &adev->irq.ih_psp, entry->iv_entry, num_dw);
+	schedule_work(&adev->irq.ih_psp_work);
+}
+
+void amdgpu_irq_ualink_delegate(struct amdgpu_device *adev,
+				struct amdgpu_iv_entry *entry,
+				unsigned int num_dw)
+{
+	amdgpu_ih_ring_write(adev, &adev->irq.ih_ualink, entry->iv_entry, num_dw);
+	schedule_work(&adev->irq.ih_ualink_work);
+}
+
 /**
  * amdgpu_irq_update - update hardware interrupt state
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index af72405a722626..c88431471c3034 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -90,9 +90,10 @@ struct amdgpu_irq {
 	bool				msi_enabled; /* msi enabled */
 
 	/* interrupt rings */
-	struct amdgpu_ih_ring		ih, ih1, ih2, ih_soft;
+	struct amdgpu_ih_ring		ih, ih1, ih2, ih_soft, ih_psp, ih_ualink;
 	const struct amdgpu_ih_funcs    *ih_funcs;
-	struct work_struct		ih1_work, ih2_work, ih_soft_work;
+	struct work_struct		ih1_work, ih2_work, ih_soft_work,
+					ih_psp_work, ih_ualink_work;
 	struct amdgpu_irq_src		self_irq;
 
 	/* gen irq stuff */
@@ -134,6 +135,12 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev,
 void amdgpu_irq_delegate(struct amdgpu_device *adev,
 			 struct amdgpu_iv_entry *entry,
 			 unsigned int num_dw);
+void amdgpu_irq_psp_delegate(struct amdgpu_device *adev,
+			     struct amdgpu_iv_entry *entry,
+			     unsigned int num_dw);
+void amdgpu_irq_ualink_delegate(struct amdgpu_device *adev,
+				 struct amdgpu_iv_entry *entry,
+				 unsigned int num_dw);
 int amdgpu_irq_update(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
 		      unsigned type);
 int amdgpu_irq_get(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_isp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_isp.c
index 532f83d783d127..976b65f90bc12e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_isp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_isp.c
@@ -126,11 +126,6 @@ static int isp_early_init(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool isp_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int isp_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 				     enum amd_clockgating_state state)
 {
@@ -345,7 +340,6 @@ static const struct amd_ip_funcs isp_ip_funcs = {
 	.early_init = isp_early_init,
 	.hw_init = isp_hw_init,
 	.hw_fini = isp_hw_fini,
-	.is_idle = isp_is_idle,
 	.suspend = isp_suspend,
 	.resume = isp_resume,
 	.set_clockgating_state = isp_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 1dedaf1698dc49..9cc210198fa45a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -387,7 +387,7 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring,
 	int r;
 
 	job->base.sched = &ring->sched;
-	r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job, fence);
+	r = amdgpu_ib_schedule(ring, job, fence);
 
 	if (r)
 		return r;
@@ -457,8 +457,7 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 		dev_dbg(adev->dev, "Skip scheduling IBs in ring(%s)",
 			ring->name);
 	} else {
-		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job,
-				       &fence);
+		r = amdgpu_ib_schedule(ring, job, &fence);
 		if (r)
 			dev_err(adev->dev,
 				"Error scheduling IBs (%d) in ring(%s)", r,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index 44fe40f9e8dfac..57bd34981be0ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -31,7 +31,7 @@
 #define AMDGPU_PREAMBLE_IB_PRESENT          (1 << 0)
 /* bit set means preamble IB is first presented in belonging context */
 #define AMDGPU_PREAMBLE_IB_PRESENT_FIRST    (1 << 1)
-/* bit set means context switch occured */
+/* bit set means context switch occurred */
 #define AMDGPU_HAVE_CTX_SWITCH              (1 << 2)
 /* bit set means IB is preempted */
 #define AMDGPU_IB_PREEMPTED                 (1 << 3)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
index 363b9916847880..9887a4f99aed1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.c
@@ -25,6 +25,7 @@
  */
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_jpeg.h"
 #include "amdgpu_pm.h"
 #include "soc15d.h"
@@ -607,3 +608,41 @@ int amdgpu_jpeg_dec_parse_cs(struct amdgpu_cs_parser *parser,
 
 	return 0;
 }
+
+/**
+ * amdgpu_jpeg_is_shared_inv_eng - Check if a ring is a JPEG ring that shares a VM invalidation
+ * engine
+ * @adev: Pointer to the AMDGPU device structure
+ * @ring: Pointer to the ring structure to check
+ *
+ * All decode rings within one JPEG instance share a single VM invalidation
+ * engine (see amdgpu_jpeg_set_shared_inv_eng()). This returns true for every
+ * ring in the instance except the first (ring->pipe == 0), which owns the
+ * engine allocated by amdgpu_gmc_allocate_vm_inv_eng().
+ */
+bool amdgpu_jpeg_is_shared_inv_eng(struct amdgpu_device *adev, struct amdgpu_ring *ring)
+{
+	return ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG && ring->pipe != 0;
+}
+
+/**
+ * amdgpu_jpeg_set_shared_inv_eng - Propagate a VM invalidation engine to the rest of a JPEG
+ * instance
+ * @adev: Pointer to the AMDGPU device structure
+ * @ring: Pointer to the ring that just had a VM invalidation engine allocated
+ *
+ * No-op unless @ring is the owner (ring->pipe == 0) of a JPEG instance.
+ * When it is, mirrors its freshly allocated vm_inv_eng onto the rest of the
+ * rings in that instance, so the whole instance shares one engine instead of
+ * consuming one per ring.
+ */
+void amdgpu_jpeg_set_shared_inv_eng(struct amdgpu_device *adev, struct amdgpu_ring *ring)
+{
+	int j;
+
+	if (ring->funcs->type != AMDGPU_RING_TYPE_VCN_JPEG || ring->pipe != 0)
+		return;
+
+	for (j = 1; j < adev->jpeg.num_jpeg_rings; j++)
+		adev->jpeg.inst[ring->me].ring_dec[j].vm_inv_eng = ring->vm_inv_eng;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
index fe95d9188713b4..05d58e6bbfabcf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_jpeg.h
@@ -183,5 +183,7 @@ void amdgpu_jpeg_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_pri
 int amdgpu_jpeg_dec_parse_cs(struct amdgpu_cs_parser *parser,
 			     struct amdgpu_job *job,
 			     struct amdgpu_ib *ib);
+bool amdgpu_jpeg_is_shared_inv_eng(struct amdgpu_device *adev, struct amdgpu_ring *ring);
+void amdgpu_jpeg_set_shared_inv_eng(struct amdgpu_device *adev, struct amdgpu_ring *ring);
 
 #endif /*__AMDGPU_JPEG_H__*/
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 242c48e85912f2..f2f179421c4da4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -47,6 +47,7 @@
 #include "amd_pcie.h"
 #include "amdgpu_userq.h"
 #include "amdgpu_video_codecs.h"
+#include "amdgpu_ras_mgr.h"
 
 void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
 {
@@ -348,8 +349,8 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
 		fw_info->feature = adev->psp.toc.feature_version;
 		break;
 	case AMDGPU_INFO_FW_CAP:
-		fw_info->ver = adev->psp.cap_fw_version;
-		fw_info->feature = adev->psp.cap_feature_version;
+		fw_info->ver = adev->psp.cap.fw_version;
+		fw_info->feature = adev->psp.cap.feature_version;
 		break;
 	case AMDGPU_INFO_FW_MES_KIQ:
 		fw_info->ver = adev->mes.kiq_version & AMDGPU_MES_VERSION_MASK;
@@ -1290,7 +1291,11 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 
 		if (!ras)
 			return -EINVAL;
-		ras_mask = (uint64_t)adev->ras_enabled << 32 | ras->features;
+
+		if (amdgpu_uniras_enabled(adev))
+			ras_mask = amdgpu_uniras_get_ras_caps(adev);
+		else
+			ras_mask = (uint64_t)adev->ras_enabled << 32 | ras->features;
 
 		return copy_to_user(out, &ras_mask,
 				min_t(u64, size, sizeof(ras_mask))) ?
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.c
index 4d1d4994ea3fe7..05bc281216bfc6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.c
@@ -48,23 +48,27 @@ int amdgpu_lsdma_copy_mem(struct amdgpu_device *adev,
 			  uint64_t dst_addr,
 			  uint64_t mem_size)
 {
-	int ret;
+	int ret = 0;
 
 	if (mem_size == 0)
 		return -EINVAL;
 
+	mutex_lock(&adev->lsdma.lock);
+
 	while (mem_size > 0) {
 		uint64_t current_copy_size = min(mem_size, AMDGPU_LSDMA_MAX_SIZE);
 
 		ret = adev->lsdma.funcs->copy_mem(adev, src_addr, dst_addr, current_copy_size);
 		if (ret)
-			return ret;
+			goto out_unlock;
 		src_addr += current_copy_size;
 		dst_addr += current_copy_size;
 		mem_size -= current_copy_size;
 	}
 
-	return 0;
+out_unlock:
+	mutex_unlock(&adev->lsdma.lock);
+	return ret;
 }
 
 int amdgpu_lsdma_fill_mem(struct amdgpu_device *adev,
@@ -72,20 +76,24 @@ int amdgpu_lsdma_fill_mem(struct amdgpu_device *adev,
 			  uint32_t data,
 			  uint64_t mem_size)
 {
-	int ret;
+	int ret = 0;
 
 	if (mem_size == 0)
 		return -EINVAL;
 
+	mutex_lock(&adev->lsdma.lock);
+
 	while (mem_size > 0) {
 		uint64_t current_fill_size = min(mem_size, AMDGPU_LSDMA_MAX_SIZE);
 
 		ret = adev->lsdma.funcs->fill_mem(adev, dst_addr, data, current_fill_size);
 		if (ret)
-			return ret;
+			goto out_unlock;
 		dst_addr += current_fill_size;
 		mem_size -= current_fill_size;
 	}
 
-	return 0;
+out_unlock:
+	mutex_unlock(&adev->lsdma.lock);
+	return ret;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.h
index c61ba58c5ee0dd..34e92aaa4617cf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_lsdma.h
@@ -24,8 +24,12 @@
 #ifndef __AMDGPU_LSDMA_H__
 #define __AMDGPU_LSDMA_H__
 
+#define AMDGPU_MAX_LSDMA_INSTANCES 2
+
 struct amdgpu_lsdma {
+	struct mutex lock;
 	const struct amdgpu_lsdma_funcs      *funcs;
+	uint32_t inst_mask;
 };
 
 struct amdgpu_lsdma_funcs {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
index 9a7f7d2b2767a5..42a05b5a2d3030 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mca.c
@@ -87,7 +87,7 @@ int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev)
 		return err;
 	}
 
-	strcpy(ras->ras_block.ras_comm.name, "mca.mp0");
+	strscpy(ras->ras_block.ras_comm.name, "mca.mp0");
 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
 	ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP0;
 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
@@ -112,7 +112,7 @@ int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev)
 		return err;
 	}
 
-	strcpy(ras->ras_block.ras_comm.name, "mca.mp1");
+	strscpy(ras->ras_block.ras_comm.name, "mca.mp1");
 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
 	ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MP1;
 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
@@ -137,7 +137,7 @@ int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev)
 		return err;
 	}
 
-	strcpy(ras->ras_block.ras_comm.name, "mca.mpio");
+	strscpy(ras->ras_block.ras_comm.name, "mca.mpio");
 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__MCA;
 	ras->ras_block.ras_comm.sub_block_index = AMDGPU_RAS_MCA_BLOCK__MPIO;
 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 1a86a47406b1d4..4cb37cdbf8219c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -101,6 +101,8 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
 	return (total_hqd_mask & ~reserved_hqd_mask);
 }
 
+static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work);
+
 int amdgpu_mes_init(struct amdgpu_device *adev)
 {
 	int i, r, num_pipes, num_queues = 0;
@@ -123,6 +125,9 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 		spin_lock_init(&adev->mes.ring_lock[i]);
 
 	adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK;
+	atomic_set(&adev->mes.userq_hw_queue_count, 0);
+	INIT_DELAYED_WORK(&adev->mes.userq_notify_unmap_work,
+			  amdgpu_mes_userq_notify_unmap_work_handler);
 	total_vmid_mask = (u32)((1UL << 16) - 1);
 	reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1);
 
@@ -288,6 +293,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
 	int i;
 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 
+	cancel_delayed_work_sync(&adev->mes.userq_notify_unmap_work);
+
 	kfree(adev->gfx.mec.mes_hung_db_array);
 
 	amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj,
@@ -1140,6 +1147,98 @@ void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
 	amdgpu_mes_unlock(mes);
 }
 
+int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev)
+{
+	struct mes_misc_op_input op_input = {0};
+	int r;
+
+	op_input.op = MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE;
+
+	if (!adev->mes.funcs->misc_op) {
+		dev_err(adev->dev, "mes notify unmap queue is not supported!\n");
+		r = -EINVAL;
+		goto error;
+	}
+
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->misc_op(&adev->mes, &op_input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		dev_err(adev->dev, "failed to notify unmap queue.\n");
+
+error:
+	return r;
+}
+
+/* Interval for notifying MES of work on unmapped queues during oversubscription */
+#define AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_US 50
+
+static unsigned int amdgpu_mes_userq_hw_queue_num(struct amdgpu_device *adev)
+{
+	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
+	unsigned int n = bitmap_weight(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
+	int i;
+
+	for (i = 0; i < num_xcc; i++)
+		n += bitmap_weight(adev->gfx.mec_bitmap[i].queue_bitmap,
+				    AMDGPU_MAX_COMPUTE_QUEUES);
+
+	return n;
+}
+
+static void amdgpu_mes_userq_notify_unmap_work_handler(struct work_struct *work)
+{
+	struct amdgpu_mes *mes = container_of(work, struct amdgpu_mes,
+					       userq_notify_unmap_work.work);
+	struct amdgpu_device *adev = mes->adev;
+
+	amdgpu_mes_notify_unmap_queue(adev);
+
+	/* Re-arm if still oversubscribed */
+	if (atomic_read(&mes->userq_hw_queue_count) >
+	    amdgpu_mes_userq_hw_queue_num(adev))
+		queue_delayed_work(system_wq, &mes->userq_notify_unmap_work,
+				   usecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_US));
+}
+
+/*
+ * Called after a GFX11 usermode queue is successfully mapped to MES.
+ * Starts the periodic unmap-notify timer if this pushed the device into
+ * HW queue oversubscription.
+ */
+void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
+	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
+		return;
+
+	if (atomic_inc_return(&adev->mes.userq_hw_queue_count) >
+	    amdgpu_mes_userq_hw_queue_num(adev))
+		queue_delayed_work(system_wq, &adev->mes.userq_notify_unmap_work,
+				   usecs_to_jiffies(AMDGPU_USERQ_UNMAP_NOTIFY_DELAY_US));
+}
+
+/*
+ * Called after a GFX11 usermode queue is unmapped from MES. Stops the
+ * periodic unmap-notify timer once oversubscription clears.
+ */
+void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	if (!(amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0) &&
+	      amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 0, 0)))
+		return;
+
+	if (atomic_dec_return(&adev->mes.userq_hw_queue_count) <=
+	    amdgpu_mes_userq_hw_queue_num(adev))
+		cancel_delayed_work(&adev->mes.userq_notify_unmap_work);
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 static int amdgpu_debugfs_mes_event_log_show(struct seq_file *m, void *unused)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 977c057dcce8ad..5160b227943b8f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -29,6 +29,7 @@
 #include "amdgpu_gfx.h"
 #include "amdgpu_doorbell.h"
 #include <linux/sched/mm.h>
+#include <linux/workqueue.h>
 
 #define AMDGPU_MES_MAX_COMPUTE_PIPES        8
 #define AMDGPU_MES_MAX_GFX_PIPES            2
@@ -90,6 +91,10 @@ struct amdgpu_mes {
 	uint32_t                        total_max_queue;
 	uint32_t                        max_doorbell_slices;
 
+	/* GFX11 usermode queue oversubscription notify timer */
+	atomic_t                        userq_hw_queue_count;
+	struct delayed_work             userq_notify_unmap_work;
+
 	uint64_t                        default_process_quantum;
 	uint64_t                        default_gang_quantum;
 
@@ -366,6 +371,7 @@ enum mes_misc_opcode {
 	MES_MISC_OP_WRM_REG_WR_WAIT,
 	MES_MISC_OP_SET_SHADER_DEBUGGER,
 	MES_MISC_OP_CHANGE_CONFIG,
+	MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE
 };
 
 struct mes_misc_op_input {
@@ -643,4 +649,9 @@ int amdgpu_mes_alloc_gang_ctx_index(struct amdgpu_mes *mes,
 				    uint32_t *index);
 void amdgpu_mes_free_gang_ctx_index(struct amdgpu_mes *mes,
 				    uint32_t index);
+
+int amdgpu_mes_notify_unmap_queue(struct amdgpu_device *adev);
+void amdgpu_mes_userq_queue_mapped(struct amdgpu_device *adev);
+void amdgpu_mes_userq_queue_unmapped(struct amdgpu_device *adev);
+
 #endif /* __AMDGPU_MES_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
index c5120ba51e248d..bc25a131806902 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h
@@ -21,6 +21,8 @@
 #ifndef __AMDGPU_MMHUB_H__
 #define __AMDGPU_MMHUB_H__
 
+#define AMDGPU_MAX_MMHUB_INSTANCES 4
+
 struct amdgpu_mmhub_ras {
 	struct amdgpu_ras_block_object ras_block;
 };
@@ -53,6 +55,7 @@ struct amdgpu_mmhub {
 	const struct amdgpu_mmhub_funcs *funcs;
 	struct amdgpu_mmhub_ras  *ras;
 	struct amdgpu_mmhub_client_ids client_ids;
+	uint32_t inst_mask;
 };
 
 static inline void
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 1b6c32a177fb6a..d6d84ff110a0f2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -204,6 +204,14 @@ void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 		c++;
 	}
 
+	if (domain & AMDGPU_GEM_DOMAIN_NPA) {
+		places[c].fpfn = 0;
+		places[c].lpfn = 0;
+		places[c].mem_type = AMDGPU_PL_NPA;
+		places[c].flags = 0;
+		c++;
+	}
+
 	if (!c) {
 		places[c].fpfn = 0;
 		places[c].lpfn = 0;
@@ -1663,6 +1671,9 @@ u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
 			case AMDGPU_PL_MMIO_REMAP:
 				placement = "MMIO REMAP";
 				break;
+			case AMDGPU_PL_NPA:
+				placement = "NPA";
+				break;
 			case TTM_PL_SYSTEM:
 			default:
 				placement = "CPU";
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index ff11a09034997e..2e6a8e6f577a72 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -125,6 +125,9 @@ struct amdgpu_bo {
 	 * for memory accounting.
 	 */
 	int8_t				xcp_id;
+
+	/* UALink Handle low 64 bits*/
+	u64				ualink_handle_lo;
 };
 
 struct amdgpu_bo_user {
@@ -169,6 +172,8 @@ static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type)
 		return AMDGPU_GEM_DOMAIN_OA;
 	case AMDGPU_PL_DOORBELL:
 		return AMDGPU_GEM_DOMAIN_DOORBELL;
+	case AMDGPU_PL_NPA:
+		return AMDGPU_GEM_DOMAIN_NPA;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 42adc8e738d893..66eb2f34cc0d94 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -27,6 +27,7 @@
 #include <drm/drm_drv.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_psp.h"
 #include "amdgpu_ucode.h"
 #include "amdgpu_xgmi.h"
@@ -48,6 +49,8 @@
 
 #define AMD_VBIOS_FILE_MAX_SIZE_B      (1024*1024*16)
 
+#define PSP_IRQ_DEFAULT_HANDLER_ID 0xFFFFFFFF
+
 static int psp_load_smu_fw(struct psp_context *psp);
 static int psp_rap_terminate(struct psp_context *psp);
 static int psp_securedisplay_terminate(struct psp_context *psp);
@@ -467,6 +470,128 @@ static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 	return ret;
 }
 
+static int psp_register_irq_handler(struct amdgpu_psp_irq_mgr *mgr)
+{
+	struct psp_context *psp = mgr->psp;
+
+	if (!psp->funcs->register_irq_handler)
+		return 0;
+	return psp->funcs->register_irq_handler(mgr, &mgr->irq_src);
+}
+
+static int psp_irq_enable(struct psp_context *psp)
+{
+	if (!psp->irq_mgr.irq_src.funcs)
+		return 0;
+	dev_info(psp->adev->dev, "psp interrupt enabled");
+	return amdgpu_irq_get(psp->adev, &psp->irq_mgr.irq_src, 0);
+}
+
+static int psp_irq_disable(struct psp_context *psp)
+{
+	if (!psp->irq_mgr.irq_src.funcs)
+		return 0;
+	return amdgpu_irq_put(psp->adev, &psp->irq_mgr.irq_src, 0);
+}
+
+/**
+ * amdgpu_psp_irq_mgr_dispatch() - run the registered handler for a PSP event
+ * @mgr: IRQ manager
+ * @entry: IH entry from the PSP soft ring
+ *
+ * Invokes the handler for the entry's event id (or the default handler) in
+ * process context. Call from the .process callback for PSP soft ring entries.
+ */
+void amdgpu_psp_irq_mgr_dispatch(struct amdgpu_psp_irq_mgr *mgr,
+				 struct amdgpu_iv_entry *entry)
+{
+	struct amdgpu_psp_irq_handler *h;
+	amdgpu_psp_irq_handler_fn cb;
+	u32 event_id = entry->src_data[0];
+
+	cb = NULL;
+	xa_lock(&mgr->irq_bh_handlers);
+	h = xa_load(&mgr->irq_bh_handlers, (unsigned long)event_id);
+	if (h) {
+		cb = h->callback;
+	} else {
+		h = xa_load(&mgr->irq_bh_handlers, PSP_IRQ_DEFAULT_HANDLER_ID);
+		if (h) {
+			cb = h->callback;
+		} else {
+			dev_dbg(mgr->psp->adev->dev,
+				"PSP event: no handler for event_id:%x\n",
+				event_id);
+			xa_unlock(&mgr->irq_bh_handlers);
+			return;
+		}
+	}
+	xa_unlock(&mgr->irq_bh_handlers);
+
+	if (cb)
+		cb(mgr, event_id, entry);
+}
+
+static void psp_irq_mgr_init(struct psp_context *psp)
+{
+	struct amdgpu_psp_irq_mgr *mgr = &psp->irq_mgr;
+	int ret;
+
+	mgr->psp = psp;
+	xa_init(&mgr->irq_bh_handlers);
+
+	ret = psp_register_irq_handler(mgr);
+	if (ret)
+		dev_dbg(psp->adev->dev, "Failed to register IRQ handler!\n");
+}
+
+static void psp_irq_mgr_fini(struct amdgpu_psp_irq_mgr *mgr)
+{
+	xa_destroy(&mgr->irq_bh_handlers);
+}
+
+/**
+ * amdgpu_psp_irq_mgr_register() - register PSP interrupt handlers by event id
+ * @mgr: PSP interrupt manager
+ * @handlers: per-event handlers, or NULL
+ * @count: number of @handlers
+ * @default_handler: fallback for unregistered event ids, or NULL
+ *
+ * Handlers must be statically allocated and stay valid until psp_irq_mgr_fini().
+ *
+ * Return: 0 on success, negative error from xa_insert (e.g. duplicate event_id).
+ */
+int amdgpu_psp_irq_mgr_register(
+	struct amdgpu_psp_irq_mgr *mgr,
+	const struct amdgpu_psp_irq_handler *handlers, int count,
+	const struct amdgpu_psp_irq_handler *default_handler)
+{
+	int ret, i;
+
+	if (handlers) {
+		for (i = 0; i < count; i++) {
+			ret = xa_insert(&mgr->irq_bh_handlers,
+					(unsigned long)handlers[i].event_id,
+					(void *)&handlers[i], GFP_KERNEL);
+			if (ret)
+				dev_dbg(mgr->psp->adev->dev,
+					"PSP IRQ: handler already registered for event_id:%x\n",
+					handlers[i].event_id);
+		}
+	}
+
+	if (default_handler) {
+		ret = xa_insert(&mgr->irq_bh_handlers,
+				PSP_IRQ_DEFAULT_HANDLER_ID,
+				(void *)default_handler, GFP_KERNEL);
+		if (ret)
+			dev_dbg(mgr->psp->adev->dev,
+				"PSP IRQ: handler already registered for default handler");
+	}
+
+	return ret;
+}
+
 static int psp_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -557,11 +682,15 @@ static int psp_sw_init(struct amdgpu_ip_block *ip_block)
 	ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 				      AMDGPU_GEM_DOMAIN_VRAM |
 				      AMDGPU_GEM_DOMAIN_GTT,
-				      &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
-				      (void **)&psp->cmd_buf_mem);
+				      &psp->cmd_resp_buf_bo, &psp->cmd_resp_buf_mc_addr,
+				      (void **)&psp->cmd_resp_buf_mem);
 	if (ret)
 		goto failed2;
 
+	/* Space for extended data in the tail of the cmd_buf allocation */
+	psp->cmd_ext_resp_mc_addr = psp->cmd_resp_buf_mc_addr + sizeof(struct psp_gfx_cmd_resp);
+	psp->cmd_ext_resp_mem = psp->cmd_resp_buf_mem + 1;
+	psp_irq_mgr_init(psp);
 	return 0;
 
 failed2:
@@ -578,6 +707,8 @@ static int psp_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	struct psp_context *psp = &adev->psp;
 
+	psp_irq_mgr_fini(&psp->irq_mgr);
+
 	psp_memory_training_fini(psp);
 
 	amdgpu_ucode_release(&psp->sos_fw);
@@ -600,8 +731,8 @@ static int psp_sw_fini(struct amdgpu_ip_block *ip_block)
 			      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 	amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 			      &psp->fence_buf_mc_addr, &psp->fence_buf);
-	amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
-			      (void **)&psp->cmd_buf_mem);
+	amdgpu_bo_free_kernel(&psp->cmd_resp_buf_bo, &psp->cmd_resp_buf_mc_addr,
+			      (void **)&psp->cmd_resp_buf_mem);
 
 	return 0;
 }
@@ -703,6 +834,18 @@ static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 		return "NPS_MODE_CHANGE";
 	case GFX_CMD_ID_PERF_HW:
 		return "PERF MONITORING HW";
+	case GFX_CMD_ID_UAL_GET_INTERFACE_VER:
+		return "UAL_GET_INTERFACE_VER";
+	case GFX_CMD_ID_UAL_GET_CONFIG:
+		return "UAL_GET_CONFIG";
+	case GFX_CMD_ID_UAL_SET_PPOD_CONFIG:
+		return "UAL_SET_PPOD_CONFIG";
+	case GFX_CMD_ID_UAL_SET_VPOD_CONFIG:
+		return "UAL_SET_VPOD_CONFIG";
+	case GFX_CMD_ID_UAL_SET_STATION_CONFIG:
+		return "UAL_SET_STATION_CONFIG";
+	case GFX_CMD_ID_UAL_SET_NPA_CONFIG:
+		return "UAL_SET_NPA_CONFIG";
 	default:
 		return "UNKNOWN CMD";
 	}
@@ -710,7 +853,7 @@ static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 
 static bool psp_err_warn(struct psp_context *psp)
 {
-	struct psp_gfx_cmd_resp *cmd = psp->cmd_buf_mem;
+	struct psp_gfx_cmd_resp *cmd = psp->cmd_resp_buf_mem;
 
 	/* This response indicates reg list is already loaded */
 	if (amdgpu_ip_version(psp->adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 2) &&
@@ -736,12 +879,12 @@ psp_cmd_submit_buf(struct psp_context *psp,
 	if (psp->adev->no_hw_access)
 		return 0;
 
-	memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
+	memset(psp->cmd_resp_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 
-	memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
+	memcpy(psp->cmd_resp_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 
 	index = atomic_inc_return(&psp->fence_value);
-	ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
+	ret = psp_ring_cmd_submit(psp, psp->cmd_resp_buf_mc_addr, fence_mc_addr, index);
 	if (ret) {
 		atomic_dec(&psp->fence_value);
 		goto exit;
@@ -759,15 +902,16 @@ psp_cmd_submit_buf(struct psp_context *psp,
 		ras_intr = amdgpu_ras_intr_triggered();
 		if (ras_intr)
 			break;
-		usleep_range(60, 100);
+		usleep_range(60, 150);
 		amdgpu_device_invalidate_hdp(psp->adev, NULL);
 	}
 
 	/* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
-	skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
-		psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
+	skip_unsupport = (psp->cmd_resp_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
+			  psp->cmd_resp_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) &&
+			 amdgpu_sriov_vf(psp->adev);
 
-	memcpy(&cmd->resp, &psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
+	memcpy(&cmd->resp, &psp->cmd_resp_buf_mem->resp, sizeof(struct psp_gfx_resp));
 
 	/* In some cases, psp response status is not 0 even there is no
 	 * problem while the command is submitted. Some version of PSP FW
@@ -776,7 +920,7 @@ psp_cmd_submit_buf(struct psp_context *psp,
 	 * during psp initialization to avoid breaking hw_init and it doesn't
 	 * return -EINVAL.
 	 */
-	if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
+	if (!skip_unsupport && (psp->cmd_resp_buf_mem->resp.status || !timeout) && !ras_intr) {
 		if (ucode)
 			dev_warn(psp->adev->dev,
 				 "failed to load ucode %s(0x%X) ",
@@ -785,9 +929,9 @@ psp_cmd_submit_buf(struct psp_context *psp,
 			dev_warn(
 				psp->adev->dev,
 				"psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
-				psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id),
-				psp->cmd_buf_mem->cmd_id,
-				psp->cmd_buf_mem->resp.status);
+				psp_gfx_cmd_name(psp->cmd_resp_buf_mem->cmd_id),
+				psp->cmd_resp_buf_mem->cmd_id,
+				psp->cmd_resp_buf_mem->resp.status);
 		/* If any firmware (including CAP) load fails under SRIOV, it should
 		 * return failure to stop the VF from initializing.
 		 * Also return failure in case of timeout
@@ -799,8 +943,8 @@ psp_cmd_submit_buf(struct psp_context *psp,
 	}
 
 	if (ucode) {
-		ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
-		ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
+		ucode->tmr_mc_addr_lo = psp->cmd_resp_buf_mem->resp.fw_addr_lo;
+		ucode->tmr_mc_addr_hi = psp->cmd_resp_buf_mem->resp.fw_addr_hi;
 	}
 
 exit:
@@ -876,7 +1020,7 @@ static int psp_load_toc(struct psp_context *psp,
 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
 				 psp->fence_buf_mc_addr);
 	if (!ret)
-		*tmr_size = psp->cmd_buf_mem->resp.tmr_size;
+		*tmr_size = psp->cmd_resp_buf_mem->resp.tmr_size;
 
 	release_psp_cmd_buf(psp);
 
@@ -1062,6 +1206,261 @@ static int psp_get_fw_reservation_info(struct psp_context *psp,
 	return 0;
 }
 
+int psp_ual_get_interface_version(struct psp_context *psp, uint32_t *intf_ver)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_UAL_GET_INTERFACE_VER;
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+	if (!ret && !cmd->resp.status) {
+		*intf_ver = cmd->resp.uresp.get_intf_ver_ual.intf_ver;
+	} else if (!ret) {
+		pr_debug("ual_get_if_ver: PSP status %x\n", cmd->resp.status);
+		ret = -EINVAL;
+	}
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
+int psp_ual_query_info(struct psp_context *psp, uint32_t intf_ver,
+		       struct amdgpu_ualink_info *info,
+		       enum psp_gfx_ual_config_state *cfg_state)
+{
+	struct psp_gfx_get_config_ual_v1 *ual_config;
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	/* TBD check interface version 1.x */
+	if (intf_ver > 0x1ffff) {
+		pr_warn("PSP UAL interface version mismatch: 0x%x\n", intf_ver);
+		return -EOPNOTSUPP;
+	}
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	ual_config = psp->cmd_ext_resp_mem;
+
+	cmd->cmd_id = GFX_CMD_ID_UAL_GET_CONFIG;
+	cmd->cmd.cmd_get_config_ual.ual_cfg_addr_lo = lower_32_bits(psp->cmd_ext_resp_mc_addr);
+	cmd->cmd.cmd_get_config_ual.ual_cfg_addr_hi = upper_32_bits(psp->cmd_ext_resp_mc_addr);
+	cmd->cmd.cmd_get_config_ual.ual_cfg_size = sizeof(*ual_config);
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+	if (!ret && !cmd->resp.status) {
+		WARN_ON(cmd->resp.uresp.get_config_ual.resp_size < sizeof(*ual_config));
+
+		info->link_type = (enum amdgpu_ualink_type)ual_config->link_type;
+
+		info->ppod.accel_id = ual_config->accelerator_id;
+		info->ppod.bandwidth = ual_config->bandwidth;
+		info->ppod.latency = ual_config->latency;
+		info->ppod.size = ual_config->ppod_size;
+		memcpy(&info->ppod.id, ual_config->ppod_id, sizeof(info->ppod.id));
+
+		info->vpod.id = ual_config->vpod_id;
+		info->vpod.size = ual_config->vpod_size;
+		info->vpod.addr_mode = ual_config->addr_mode;
+		bitmap_from_arr32(info->vpod.active_accel_bits,
+				  ual_config->vpod_active_accelerators,
+				  min(AMDGPU_UALINK_ACCEL_MAX, PSP_GFX_UAL_MAX_ACC_BIT_MASK*32));
+		/* Ensure no uninitialized data in the bitmap, even if these
+		 * constants change in the future.
+		 */
+		if (AMDGPU_UALINK_ACCEL_MAX > PSP_GFX_UAL_MAX_ACC_BIT_MASK*32)
+			bitmap_clear(info->vpod.active_accel_bits, PSP_GFX_UAL_MAX_ACC_BIT_MASK*32,
+				     AMDGPU_UALINK_ACCEL_MAX - PSP_GFX_UAL_MAX_ACC_BIT_MASK*32);
+
+		if (cfg_state)
+			*cfg_state = ual_config->config_state;
+	} else if (!ret) {
+		ret = -EINVAL;
+	}
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
+int psp_ual_set_ppod_config(struct psp_context *psp, uint32_t intf_ver,
+			    const struct amdgpu_ualink_ppod_setup *setup)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	/* TBD check interface version 1.x */
+	if (intf_ver > 0x1ffff) {
+		pr_warn("PSP UAL interface version mismatch: 0x%x\n", intf_ver);
+		return -EOPNOTSUPP;
+	}
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_UAL_SET_PPOD_CONFIG;
+	cmd->cmd.cmd_set_ppod_config_ual.accelerator_id = setup->ppod.accel_id;
+	memcpy(cmd->cmd.cmd_set_ppod_config_ual.ppod_id, &setup->ppod.id,
+	       sizeof(cmd->cmd.cmd_set_ppod_config_ual.ppod_id));
+	cmd->cmd.cmd_set_ppod_config_ual.ppod_size = setup->ppod.size;
+	cmd->cmd.cmd_set_ppod_config_ual.bandwidth = setup->ppod.bandwidth;
+	cmd->cmd.cmd_set_ppod_config_ual.latency = setup->ppod.latency;
+
+	memcpy(cmd->cmd.cmd_set_ppod_config_ual.local_accelerators,
+	       setup->local_accels,
+	       min(sizeof(cmd->cmd.cmd_set_ppod_config_ual.local_accelerators),
+		   setup->n_local_accels * sizeof(u32)));
+	/* Fill the remainder of the array with invalid accelerator IDs */
+	if (sizeof(cmd->cmd.cmd_set_ppod_config_ual.local_accelerators) >
+	    setup->n_local_accels * sizeof(u32))
+		memset(&cmd->cmd.cmd_set_ppod_config_ual.local_accelerators[setup->n_local_accels],
+		       0xff, sizeof(cmd->cmd.cmd_set_ppod_config_ual.local_accelerators) -
+		       setup->n_local_accels * sizeof(u32));
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+	if (!ret && cmd->resp.status)
+		ret = -EINVAL;
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
+int psp_ual_set_vpod_config(struct psp_context *psp, uint32_t intf_ver,
+			    const struct amdgpu_ualink_vpod_config *config)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	/* TBD check interface version 1.x */
+	if (intf_ver > 0x1ffff) {
+		pr_warn("PSP UAL interface version mismatch: 0x%x\n", intf_ver);
+		return -EOPNOTSUPP;
+	}
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_UAL_SET_VPOD_CONFIG;
+	cmd->cmd.cmd_set_vpod_config_ual.addr_mode =
+		(enum psp_gfx_ual_npa_address_mode)config->vpod.addr_mode;
+	cmd->cmd.cmd_set_vpod_config_ual.vpod_id = config->vpod.id;
+	cmd->cmd.cmd_set_vpod_config_ual.vpod_size = config->vpod.size;
+
+	bitmap_to_arr32(cmd->cmd.cmd_set_vpod_config_ual.vpod_active_accelerators,
+			config->vpod.active_accel_bits,
+			min(AMDGPU_UALINK_ACCEL_MAX, PSP_GFX_UAL_MAX_ACC_BIT_MASK*32));
+	/* Clear any remaining accelerator bits */
+	if (PSP_GFX_UAL_MAX_ACC_BIT_MASK > DIV_ROUND_UP(AMDGPU_UALINK_ACCEL_MAX, 32))
+		memset(cmd->cmd.cmd_set_vpod_config_ual.vpod_active_accelerators +
+		       DIV_ROUND_UP(AMDGPU_UALINK_ACCEL_MAX, 32), 0,
+		       (PSP_GFX_UAL_MAX_ACC_BIT_MASK -
+			DIV_ROUND_UP(AMDGPU_UALINK_ACCEL_MAX, 32)) * sizeof(u32));
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+	if (!ret && cmd->resp.status)
+		ret = -EINVAL;
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
+int psp_ual_set_station_config(struct psp_context *psp, uint32_t intf_ver,
+			       const struct amdgpu_ualink_station_config *stations)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	/* TBD check interface version 1.x */
+	if (intf_ver > 0x1ffff) {
+		pr_warn("PSP UAL interface version mismatch: 0x%x\n", intf_ver);
+		return -EOPNOTSUPP;
+	}
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_UAL_SET_STATION_CONFIG;
+	cmd->cmd.cmd_set_station_config_ual.num_stations = stations->n_stations;
+	cmd->cmd.cmd_set_station_config_ual.station_flag = stations->flags;
+	memcpy(cmd->cmd.cmd_set_station_config_ual.lane_en_bitmap,
+	       stations->lane_en_bitmap,
+	       min(stations->n_stations,
+		   sizeof(cmd->cmd.cmd_set_station_config_ual.lane_en_bitmap)));
+	/* The remainder is already 0-initialized by acquire_psp_cmd_buf */
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+	if (!ret && cmd->resp.status)
+		ret = -EINVAL;
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
+int psp_ual_set_npa_config(struct psp_context *psp, uint32_t intf_ver,
+			   unsigned int vmid, bool enable)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	/* TBD check interface version 1.x */
+	if (intf_ver > 0x1ffff) {
+		pr_warn("PSP UAL interface version mismatch: 0x%x\n", intf_ver);
+		return -EOPNOTSUPP;
+	}
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_UAL_SET_NPA_CONFIG;
+	cmd->cmd.cmd_set_npa_config_ual.vmid = vmid;
+	cmd->cmd.cmd_set_npa_config_ual.enable_npa_translation = enable;
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+	if (!ret && cmd->resp.status)
+		ret = -EINVAL;
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
+int psp_ual_send_completion(struct psp_context *psp, uint32_t intf_ver,
+			    uint32_t cmd_id, uint32_t status)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	int ret;
+
+	/* TBD check interface version 1.x */
+	if (intf_ver > 0x1ffff) {
+		pr_warn("PSP UAL interface version mismatch: 0x%x\n", intf_ver);
+		return -EOPNOTSUPP;
+	}
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id = GFX_CMD_ID_UAL_SEND_COMPLETION;
+	cmd->cmd.cmd_send_completion_ual.cmd_id = cmd_id;
+	cmd->cmd.cmd_send_completion_ual.status = status;
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+
+	if (!ret && cmd->resp.status)
+		ret = -EINVAL;
+
+	release_psp_cmd_buf(psp);
+
+	return ret;
+}
+
 int psp_update_fw_reservation(struct psp_context *psp)
 {
 	int ret;
@@ -1176,6 +1575,7 @@ static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 
 static int psp_rl_load(struct amdgpu_device *adev)
 {
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	int ret;
 	struct psp_context *psp = &adev->psp;
 	struct psp_gfx_cmd_resp *cmd;
@@ -1183,6 +1583,9 @@ static int psp_rl_load(struct amdgpu_device *adev)
 	if (!is_psp_fw_valid(psp->rl))
 		return 0;
 
+	if (con && con->uniras_load_ras_fw)
+		return 0;
+
 	cmd = acquire_psp_cmd_buf(psp);
 
 	ret = psp_copy_fw(psp, psp->rl.start_addr, psp->rl.size_bytes);
@@ -1893,14 +2296,19 @@ int psp_xgmi_terminate(struct psp_context *psp)
 	return ret;
 }
 
+bool psp_is_xgmi_ta_supported(struct psp_context *psp)
+{
+	return psp->ta_fw &&
+	       psp->xgmi_context.context.bin_desc.size_bytes &&
+	       psp->xgmi_context.context.bin_desc.start_addr;
+}
+
 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
 {
 	struct ta_xgmi_shared_memory *xgmi_cmd;
 	int ret;
 
-	if (!psp->ta_fw ||
-	    !psp->xgmi_context.context.bin_desc.size_bytes ||
-	    !psp->xgmi_context.context.bin_desc.start_addr)
+	if (!psp_is_xgmi_ta_supported(psp))
 		return -ENOENT;
 
 	if (!load_ta)
@@ -2270,7 +2678,7 @@ static int psp_ras_send_cmd(struct psp_context *psp,
 			memcpy(out, &ras_cmd->ras_status, sizeof(ras_cmd->ras_status));
 		break;
 	case TA_RAS_COMMAND__QUERY_ADDRESS:
-		if (ret || ras_cmd->ras_status || psp->cmd_buf_mem->resp.status)
+		if (ret || ras_cmd->ras_status || psp->cmd_resp_buf_mem->resp.status)
 			ret = -EINVAL;
 		else if (out)
 			memcpy(out,
@@ -2328,9 +2736,13 @@ int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 int psp_ras_enable_features(struct psp_context *psp,
 		union ta_ras_cmd_input *info, bool enable)
 {
+	struct amdgpu_ras *con = amdgpu_ras_get_context(psp->adev);
 	enum ras_command cmd_id;
 	int ret;
 
+	if (con && con->uniras_load_ras_fw)
+		return 0;
+
 	if (!psp->ras_context.context.initialized || !info)
 		return -EINVAL;
 
@@ -2370,6 +2782,7 @@ int psp_ras_initialize(struct psp_context *psp)
 	int ret;
 	uint32_t boot_cfg = 0xFF;
 	struct amdgpu_device *adev = psp->adev;
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct ta_ras_shared_memory *ras_cmd;
 
 	/*
@@ -2437,6 +2850,9 @@ int psp_ras_initialize(struct psp_context *psp)
 		}
 	}
 
+	if (con && con->uniras_load_ras_fw)
+		return 0;
+
 	psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
 	psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
 
@@ -2459,8 +2875,9 @@ int psp_ras_initialize(struct psp_context *psp)
 	if (adev->gmc.gmc_funcs->query_mem_partition_mode)
 		ras_cmd->ras_in_message.init_flags.nps_mode =
 			adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
-	ras_cmd->ras_in_message.init_flags.active_umc_mask = adev->umc.active_mask;
+	ras_cmd->ras_in_message.init_flags.active_umc_mask = lower_32_bits(adev->umc.active_mask);
 	ras_cmd->ras_in_message.init_flags.vram_type = (uint8_t)adev->gmc.vram_type;
+	ras_cmd->ras_in_message.init_flags.ext_umc_mask = upper_32_bits(adev->umc.active_mask);
 
 	ret = psp_ta_load(psp, &psp->ras_context.context);
 
@@ -2940,10 +3357,12 @@ static void psp_update_gpu_addresses(struct amdgpu_device *adev)
 {
 	struct psp_context *psp = &adev->psp;
 
-	if (psp->cmd_buf_bo && psp->cmd_buf_mem) {
+	if (psp->cmd_resp_buf_bo && psp->cmd_resp_buf_mem) {
 		psp->fw_pri_mc_addr = amdgpu_bo_fb_aper_addr(psp->fw_pri_bo);
 		psp->fence_buf_mc_addr = amdgpu_bo_fb_aper_addr(psp->fence_buf_bo);
-		psp->cmd_buf_mc_addr = amdgpu_bo_fb_aper_addr(psp->cmd_buf_bo);
+		psp->cmd_resp_buf_mc_addr = amdgpu_bo_fb_aper_addr(psp->cmd_resp_buf_bo);
+		psp->cmd_ext_resp_mc_addr = psp->cmd_resp_buf_mc_addr +
+					    sizeof(struct psp_gfx_cmd_resp);
 	}
 	if (adev->firmware.rbuf && psp->km_ring.ring_mem)
 		psp->km_ring.ring_mem_mc_addr = amdgpu_bo_fb_aper_addr(adev->firmware.rbuf);
@@ -3049,6 +3468,7 @@ static int psp_hw_start(struct psp_context *psp)
 		}
 	}
 
+	psp_irq_enable(psp);
 	ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 	if (ret) {
 		dev_err(adev->dev, "PSP create ring failed!\n");
@@ -3728,6 +4148,7 @@ static int psp_hw_fini(struct amdgpu_ip_block *ip_block)
 	psp_asd_terminate(psp);
 	psp_tmr_terminate(psp);
 
+	psp_irq_disable(psp);
 	psp_ring_destroy(psp, PSP_RING_TYPE__KM);
 
 	return 0;
@@ -3787,7 +4208,7 @@ static int psp_suspend(struct amdgpu_ip_block *ip_block)
 		dev_err(adev->dev, "Failed to terminate tmr\n");
 		goto out;
 	}
-
+	psp_irq_disable(psp);
 	ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
 	if (ret)
 		dev_err(adev->dev, "PSP ring stop failed\n");
@@ -3966,58 +4387,92 @@ int psp_ring_cmd_submit(struct psp_context *psp,
 	return 0;
 }
 
+/**
+ * parse_psp_v1_bin_descriptor - Populate struct psp_bin_desc from a
+ * v1.0 PSP firmware header (standalone images such as ASD or TOC).
+ */
+static int parse_psp_v1_bin_descriptor(struct psp_bin_desc *desc,
+					const struct psp_firmware_header_v1_0 *hdr)
+{
+	if (!desc || !hdr)
+		return -EINVAL;
+
+	desc->fw_version = le32_to_cpu(hdr->header.ucode_version);
+	desc->feature_version = le32_to_cpu(hdr->sos.fw_version);
+	desc->size_bytes = le32_to_cpu(hdr->header.ucode_size_bytes);
+	desc->start_addr = (uint8_t *)hdr +
+			   le32_to_cpu(hdr->header.ucode_array_offset_bytes);
+
+	return 0;
+}
+
 int psp_init_asd_microcode(struct psp_context *psp, const char *chip_name)
 {
-	struct amdgpu_device *adev = psp->adev;
-	const struct psp_firmware_header_v1_0 *asd_hdr;
-	int err = 0;
+	int err;
 
-	err = amdgpu_ucode_request(adev, &adev->psp.asd_fw, AMDGPU_UCODE_REQUIRED,
+	err = amdgpu_ucode_request(psp->adev, &psp->asd_fw, AMDGPU_UCODE_REQUIRED,
 				   "amdgpu/%s_asd.bin", chip_name);
 	if (err)
 		goto out;
 
-	asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
-	adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
-	adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
-	adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
-	adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
-				le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
+	err = parse_psp_v1_bin_descriptor(&psp->asd_context.bin_desc,
+		    (const struct psp_firmware_header_v1_0 *)psp->asd_fw->data);
+	if (err)
+		goto out;
+
 	return 0;
 out:
-	amdgpu_ucode_release(&adev->psp.asd_fw);
+	amdgpu_ucode_release(&psp->asd_fw);
 	return err;
 }
 
 int psp_init_toc_microcode(struct psp_context *psp, const char *chip_name)
 {
-	struct amdgpu_device *adev = psp->adev;
-	const struct psp_firmware_header_v1_0 *toc_hdr;
-	int err = 0;
+	int err;
 
-	if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 8) &&
-	    adev->rev_id == 0)
-		err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
+	if (amdgpu_ip_version(psp->adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 8) &&
+	    psp->adev->rev_id == 0 &&
+	    !amdgpu_emu_mode)
+		err = amdgpu_ucode_request(psp->adev, &psp->toc_fw, AMDGPU_UCODE_REQUIRED,
 				   "amdgpu/%s_toc_1.bin", chip_name);
 	else
-		err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
+		err = amdgpu_ucode_request(psp->adev, &psp->toc_fw, AMDGPU_UCODE_REQUIRED,
 				   "amdgpu/%s_toc.bin", chip_name);
 	if (err)
 		goto out;
 
-	toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
-	adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
-	adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
-	adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
-	adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
-				le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
+	err = parse_psp_v1_bin_descriptor(&psp->toc,
+		    (const struct psp_firmware_header_v1_0 *)psp->toc_fw->data);
+	if (err)
+		goto out;
+
 	return 0;
 out:
-	amdgpu_ucode_release(&adev->psp.toc_fw);
+	amdgpu_ucode_release(&psp->toc_fw);
 	return err;
 }
 
-static int parse_sos_bin_descriptor(struct psp_context *psp,
+int psp_init_rl_microcode(struct psp_context *psp, const char *chip_name)
+{
+	int err;
+
+	err = amdgpu_ucode_request(psp->adev, &psp->rl_fw, AMDGPU_UCODE_REQUIRED,
+				   "amdgpu/%s_rl.bin", chip_name);
+	if (err)
+		goto out;
+
+	err = parse_psp_v1_bin_descriptor(&psp->rl,
+		    (const struct psp_firmware_header_v1_0 *)psp->rl_fw->data);
+	if (err)
+		goto out;
+
+	return 0;
+out:
+	amdgpu_ucode_release(&psp->rl_fw);
+	return err;
+}
+
+static int parse_psp_v2_bin_descriptor(struct psp_context *psp,
 				   const struct psp_fw_bin_desc *desc,
 				   const struct psp_firmware_header_v2_0 *sos_hdr)
 {
@@ -4247,7 +4702,7 @@ int psp_init_sos_microcode(struct psp_context *psp, const char *chip_name)
 		}
 
 		for (fw_index = start_index; fw_index < fw_bin_count; fw_index++) {
-			err = parse_sos_bin_descriptor(psp, fw_bin + fw_index,
+			err = parse_psp_v2_bin_descriptor(psp, fw_bin + fw_index,
 						       sos_hdr_v2_0);
 			if (err)
 				goto out;
@@ -4482,16 +4937,15 @@ int psp_init_ta_microcode(struct psp_context *psp, const char *chip_name)
 int psp_init_cap_microcode(struct psp_context *psp, const char *chip_name)
 {
 	struct amdgpu_device *adev = psp->adev;
-	const struct psp_firmware_header_v1_0 *cap_hdr_v1_0;
-	struct amdgpu_firmware_info *info = NULL;
-	int err = 0;
+	struct amdgpu_firmware_info *info;
+	int err;
 
 	if (!amdgpu_sriov_vf(adev)) {
 		dev_err(adev->dev, "cap microcode should only be loaded under SRIOV\n");
 		return -EINVAL;
 	}
 
-	err = amdgpu_ucode_request(adev, &adev->psp.cap_fw, AMDGPU_UCODE_OPTIONAL,
+	err = amdgpu_ucode_request(adev, &psp->cap_fw, AMDGPU_UCODE_OPTIONAL,
 				   "amdgpu/%s_cap.bin", chip_name);
 	if (err) {
 		if (err == -ENODEV) {
@@ -4503,21 +4957,19 @@ int psp_init_cap_microcode(struct psp_context *psp, const char *chip_name)
 		goto out;
 	}
 
+	err = parse_psp_v1_bin_descriptor(&psp->cap,
+		(const struct psp_firmware_header_v1_0 *)psp->cap_fw->data);
+	if (err)
+		goto out;
+
 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CAP];
 	info->ucode_id = AMDGPU_UCODE_ID_CAP;
-	info->fw = adev->psp.cap_fw;
-	cap_hdr_v1_0 = (const struct psp_firmware_header_v1_0 *)
-		adev->psp.cap_fw->data;
-	adev->firmware.fw_size += ALIGN(
-			le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes), PAGE_SIZE);
-	adev->psp.cap_fw_version = le32_to_cpu(cap_hdr_v1_0->header.ucode_version);
-	adev->psp.cap_feature_version = le32_to_cpu(cap_hdr_v1_0->sos.fw_version);
-	adev->psp.cap_ucode_size = le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes);
-
+	info->fw = psp->cap_fw;
+	adev->firmware.fw_size += ALIGN(psp->cap.size_bytes, PAGE_SIZE);
 	return 0;
 
 out:
-	amdgpu_ucode_release(&adev->psp.cap_fw);
+	amdgpu_ucode_release(&psp->cap_fw);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index d80c85793e3b95..35839f42b8a08a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -139,6 +139,8 @@ enum psp_reg_prog_id {
 #define PSP_WAITREG_CHANGED BIT(0) /* check if the value has changed */
 #define PSP_WAITREG_NOVERBOSE BIT(1) /* No error verbose */
 
+struct amdgpu_psp_irq_mgr;
+
 struct psp_funcs {
 	int (*init_microcode)(struct psp_context *psp);
 	int (*wait_for_bootloader)(struct psp_context *psp);
@@ -176,6 +178,8 @@ struct psp_funcs {
 				   enum psp_reg_prog_id id);
 	int (*get_fw_type)(struct amdgpu_firmware_info *ucode,
 			enum psp_gfx_fw_type *type);
+	int (*register_irq_handler)(struct amdgpu_psp_irq_mgr *mgr,
+				    struct amdgpu_irq_src *irq_src);
 };
 
 struct ta_funcs {
@@ -384,6 +388,37 @@ struct psp_ptl_perf_req {
 	uint32_t pref_format2;
 };
 
+/**
+ * typedef amdgpu_psp_irq_handler_fn - callback for a registered PSP event id
+ * @mgr: IRQ manager
+ * @entry: copy of the IH entry that triggered this event
+ */
+typedef void (*amdgpu_psp_irq_handler_fn)(struct amdgpu_psp_irq_mgr *mgr,
+					  u32 event_id,
+					  struct amdgpu_iv_entry *entry);
+
+/**
+ * struct amdgpu_psp_irq_handler - statically allocated handler node
+ *
+ * Callers define these statically; a single instance can be shared across
+ * multiple devices.  Indexed in &amdgpu_psp_irq_mgr.irq_bh_handlers by @event_id.
+ */
+struct amdgpu_psp_irq_handler {
+	u32				event_id;
+	amdgpu_psp_irq_handler_fn	callback;
+};
+
+/**
+ * struct amdgpu_psp_irq_mgr - deferred dispatch for PSP-driven IH events
+ *
+ * The xarray's own lock serializes handler lookup against registration.
+ */
+struct amdgpu_psp_irq_mgr {
+	struct psp_context		*psp;
+	struct amdgpu_irq_src		irq_src;
+	struct xarray			irq_bh_handlers;
+};
+
 struct psp_context {
 	struct amdgpu_device		*adev;
 	struct psp_ring			km_ring;
@@ -425,15 +460,20 @@ struct psp_context {
 	/* cap firmware */
 	const struct firmware		*cap_fw;
 
+	/* rl firmware */
+	const struct firmware		*rl_fw;
+
 	/* fence buffer */
 	struct amdgpu_bo		*fence_buf_bo;
 	uint64_t			fence_buf_mc_addr;
 	void				*fence_buf;
 
 	/* cmd buffer */
-	struct amdgpu_bo		*cmd_buf_bo;
-	uint64_t			cmd_buf_mc_addr;
-	struct psp_gfx_cmd_resp		*cmd_buf_mem;
+	struct amdgpu_bo		*cmd_resp_buf_bo;
+	uint64_t			cmd_resp_buf_mc_addr;
+	struct psp_gfx_cmd_resp		*cmd_resp_buf_mem;
+	uint64_t			cmd_ext_resp_mc_addr;
+	void				*cmd_ext_resp_mem;
 
 	/* fence value associated with cmd buffer */
 	atomic_t			fence_value;
@@ -448,9 +488,7 @@ struct psp_context {
 	const struct firmware		*ta_fw;
 	uint32_t			ta_fw_version;
 
-	uint32_t			cap_fw_version;
-	uint32_t			cap_feature_version;
-	uint32_t			cap_ucode_size;
+	struct psp_bin_desc		cap;
 
 	struct ta_context		asd_context;
 	struct psp_xgmi_context		xgmi_context;
@@ -471,6 +509,7 @@ struct psp_context {
 	char				*vbflash_tmp_buf;
 	size_t				vbflash_image_size;
 	bool				vbflash_done;
+	struct amdgpu_psp_irq_mgr	irq_mgr;
 #if defined(CONFIG_DEBUG_FS)
 	struct spirom_bo *spirom_dump_trip;
 #endif
@@ -589,6 +628,7 @@ int psp_ta_invoke(struct psp_context *psp,
 			struct ta_context *context);
 
 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta);
+bool psp_is_xgmi_ta_supported(struct psp_context *psp);
 int psp_xgmi_terminate(struct psp_context *psp);
 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id);
 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id);
@@ -658,5 +698,39 @@ void amdgpu_psp_debugfs_init(struct amdgpu_device *adev);
 int amdgpu_psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 			   enum psp_gfx_fw_type *type);
 int psp_set_mmhub_eco_sec_level(struct amdgpu_device *adev);
+int psp_init_rl_microcode(struct psp_context *psp, const char *chip_name);
+
+void amdgpu_psp_irq_mgr_dispatch(struct amdgpu_psp_irq_mgr *mgr,
+				 struct amdgpu_iv_entry *entry);
+int amdgpu_psp_irq_mgr_register(
+	struct amdgpu_psp_irq_mgr *mgr,
+	const struct amdgpu_psp_irq_handler *handlers, int count,
+	const struct amdgpu_psp_irq_handler *default_handler);
+
+struct amdgpu_ualink_info;
+struct amdgpu_ualink_ppod_setup;
+struct amdgpu_ualink_vpod_config;
+struct amdgpu_ualink_station_config;
+
+int psp_ual_get_interface_version(struct psp_context *psp, uint32_t *intf_ver);
+
+int psp_ual_query_info(struct psp_context *psp, uint32_t intf_ver,
+		       struct amdgpu_ualink_info *info,
+		       enum psp_gfx_ual_config_state *cfg_state);
+
+int psp_ual_set_ppod_config(struct psp_context *psp, uint32_t intf_ver,
+			    const struct amdgpu_ualink_ppod_setup *setup);
+
+int psp_ual_set_vpod_config(struct psp_context *psp, uint32_t intf_ver,
+			    const struct amdgpu_ualink_vpod_config *config);
+
+int psp_ual_set_station_config(struct psp_context *psp, uint32_t intf_ver,
+			       const struct amdgpu_ualink_station_config *stations);
+
+int psp_ual_set_npa_config(struct psp_context *psp, uint32_t intf_ver,
+			   unsigned int vmid, bool enable);
+
+int psp_ual_send_completion(struct psp_context *psp, uint32_t intf_ver,
+			    uint32_t cmd_id, uint32_t status);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 35eb87591740ac..6b2b8acdbf9b25 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -219,8 +219,9 @@ static int amdgpu_check_address_validity(struct amdgpu_device *adev,
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct amdgpu_vram_block_info blk_info;
-	uint64_t page_pfns[32] = {0};
-	int i, ret, count;
+	uint64_t *page_pfns = NULL;
+	u32 nr_page_pfns = 256;
+	int i, ret = 0, count;
 	bool hit = false;
 
 	if (amdgpu_ip_version(adev, UMC_HWIP, 0) < IP_VERSION(12, 0, 0))
@@ -243,15 +244,21 @@ static int amdgpu_check_address_validity(struct amdgpu_device *adev,
 	    (address >= RAS_UMC_INJECT_ADDR_LIMIT))
 		return -EFAULT;
 
+	page_pfns = kcalloc(nr_page_pfns, sizeof(*page_pfns), GFP_KERNEL);
+	if (!page_pfns)
+		return -ENOMEM;
+
 	if (amdgpu_sriov_vf(adev))
 		count = amdgpu_virt_ras_convert_retired_address(adev, address,
-			page_pfns, ARRAY_SIZE(page_pfns));
+			page_pfns, nr_page_pfns);
 	else
 		count = amdgpu_ras_mgr_lookup_bad_pages_in_a_row(adev, address,
-			page_pfns, ARRAY_SIZE(page_pfns));
+			page_pfns, nr_page_pfns);
 
-	if (count <= 0)
-		return -EPERM;
+	if (count <= 0) {
+		ret = -EPERM;
+		goto out;
+	}
 
 	for (i = 0; i < count; i++) {
 		memset(&blk_info, 0, sizeof(blk_info));
@@ -264,16 +271,23 @@ static int amdgpu_check_address_validity(struct amdgpu_device *adev,
 			 */
 			if ((flags == BYPASS_ALLOCATED_ADDRESS) &&
 			    ((blk_info.task.pid != task_pid_nr(current)) ||
-				strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN)))
-				return -EACCES;
-			else if ((flags == BYPASS_INITIALIZATION_ADDRESS) &&
+				strncmp(blk_info.task.comm, current->comm, TASK_COMM_LEN))) {
+				ret = -EACCES;
+				goto out;
+			} else if ((flags == BYPASS_INITIALIZATION_ADDRESS) &&
 				(blk_info.task.pid == con->init_task_pid) &&
-				!strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN))
-				return -EACCES;
+				!strncmp(blk_info.task.comm, con->init_task_comm, TASK_COMM_LEN)) {
+				ret = -EACCES;
+				goto out;
+			}
 		}
 	}
 
-	return 0;
+	ret = 0;
+
+out:
+	kfree(page_pfns);
+	return ret;
 }
 
 static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
@@ -919,7 +933,14 @@ int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
 			};
 		}
 
-		ret = psp_ras_enable_features(&adev->psp, info, enable);
+		if (amdgpu_uniras_enabled(adev))
+			ret = amdgpu_ras_mgr_enable_feature(adev,
+					amdgpu_ras_block_to_ta(head->block),
+					amdgpu_ras_error_to_ta(head->type),
+					enable);
+		else
+			ret = psp_ras_enable_features(&adev->psp, info, enable);
+
 		if (ret) {
 			dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
 				enable ? "enable":"disable",
@@ -1820,8 +1841,12 @@ static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
 {
 	struct amdgpu_ras *con =
 		container_of(attr, struct amdgpu_ras, features_attr);
+	u64 ras_features;
+
+	ras_features = amdgpu_uniras_enabled(con->adev) ?
+		amdgpu_uniras_get_ras_caps(con->adev) : con->features;
 
-	return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
+	return sysfs_emit(buf, "feature mask: 0x%llx\n", ras_features);
 }
 
 static bool amdgpu_ras_get_version_info(struct amdgpu_device *adev, u32 *major,
@@ -2186,6 +2211,9 @@ static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct ras_manager *con_obj, *ip_obj, *tmp;
 
+	if (!con)
+		return 0;
+
 	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
 		list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
 			ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
@@ -2393,9 +2421,7 @@ int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
 
 		memset(&ih_info, 0, sizeof(ih_info));
 		ih_info.block = info->head.block;
-		memcpy(&ih_info.iv_entry, info->entry, sizeof(struct amdgpu_iv_entry));
-
-		return amdgpu_ras_mgr_handle_controller_interrupt(adev, &ih_info);
+		return amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info);
 	}
 
 	obj = amdgpu_ras_find_obj(adev, &info->head);
@@ -3303,6 +3329,7 @@ static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
 		case IP_VERSION(13, 0, 14):
 		case IP_VERSION(13, 0, 15):
 		case IP_VERSION(14, 0, 3):
+		case IP_VERSION(15, 0, 8):
 			return true;
 		default:
 			return false;
@@ -3464,7 +3491,7 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
 	adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
 
 	adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
-		adev->ras_hw_enabled & amdgpu_ras_mask;
+		(uint32_t)(adev->ras_hw_enabled & amdgpu_ras_mask);
 
 	/* bad page feature is not applicable to specific app platform */
 	if (adev->gmc.is_app_apu &&
@@ -3565,6 +3592,7 @@ static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
 int amdgpu_ras_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_module_param param = {0};
 	int r;
 
 	if (con)
@@ -3588,7 +3616,13 @@ int amdgpu_ras_init(struct amdgpu_device *adev)
 
 	amdgpu_ras_check_supported(adev);
 
-	if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
+	param.ras_feature_enable = amdgpu_ras_enable;
+	param.ras_feature_mask = amdgpu_ras_mask;
+	param.ras_bad_page_threshold = amdgpu_bad_page_threshold;
+	amdgpu_ras_mgr_sw_init(adev, &param);
+
+	if (!con->uniras_enabled &&
+	    (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10)) {
 		/* set gfx block ras context feature for VEGA20 Gaming
 		 * send ras disable cmd to ras ta during ras late init.
 		 */
@@ -3860,6 +3894,8 @@ void amdgpu_ras_resume(struct amdgpu_device *adev)
 		return;
 	}
 
+	amdgpu_ras_mgr_set_debug_mode(adev, false);
+
 	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
 		/* Set up all other IPs which are not implemented. There is a
 		 * tricky thing that IP's actual ras error type should be
@@ -3920,9 +3956,6 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev)
 	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
 		return 0;
 
-	if (amdgpu_uniras_enabled(adev))
-		amdgpu_ras_mgr_set_debug_mode(adev, false);
-
 	list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
 		obj = node->ras_obj;
 		if (!obj) {
@@ -3978,6 +4011,10 @@ int amdgpu_ras_fini(struct amdgpu_device *adev)
 	struct amdgpu_ras_block_object *obj = NULL;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 
+	amdgpu_ras_mgr_sw_fini(adev);
+
+	amdgpu_ras_fs_fini(adev);
+
 	if (!adev->ras_enabled || !con)
 		return 0;
 
@@ -4000,7 +4037,6 @@ int amdgpu_ras_fini(struct amdgpu_device *adev)
 		kfree(ras_node);
 	}
 
-	amdgpu_ras_fs_fini(adev);
 	amdgpu_ras_interrupt_remove_all(adev);
 
 	WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
@@ -4369,7 +4405,7 @@ int amdgpu_ras_is_supported(struct amdgpu_device *adev,
 	     block == AMDGPU_RAS_BLOCK__SDMA ||
 	     block == AMDGPU_RAS_BLOCK__VCN ||
 	     block == AMDGPU_RAS_BLOCK__JPEG) &&
-		(amdgpu_ras_mask & (1 << block)) &&
+		(amdgpu_ras_mask & BIT_ULL(block)) &&
 	    amdgpu_ras_is_poison_mode_supported(adev) &&
 	    amdgpu_ras_get_ras_block(adev, block, 0))
 		ret = 1;
@@ -5005,3 +5041,29 @@ int amdgpu_ras_resume_after_reset(struct amdgpu_device *adev)
 
 	return amdgpu_cper_deferred_init(adev);
 }
+
+uint64_t amdgpu_uniras_get_ras_caps(struct amdgpu_device *adev)
+{
+	struct ras_cmd_get_ras_cap_req req = {0};
+	struct ras_cmd_get_ras_cap_rsp rsp = {0};
+	union ras_feature feature = {0};
+	u32 socket_id = 0;
+
+	if (amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__GET_RAS_CAP,
+			&req, sizeof(struct ras_cmd_get_ras_cap_req),
+			&rsp, sizeof(struct ras_cmd_get_ras_cap_rsp)))
+		return 0;
+
+	if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
+		socket_id = adev->smuio.funcs->get_socket_id(adev);
+
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(12, 1, 0))
+		return AMDGPU_RAS_GET_FEATURES(lower_32_bits(rsp.ras_block_mask)) |
+		       (socket_id << AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
+
+	feature.block_mask = rsp.ras_block_mask;
+	feature.en = !!adev->ras_enabled;
+	feature.tag = socket_id;
+
+	return feature.value;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
index c53f911d5729fd..84799c39ef81c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
@@ -572,6 +572,8 @@ struct amdgpu_ras {
 
 	/* Disable/Enable uniras switch */
 	bool uniras_enabled;
+	bool uniras_load_ras_fw;
+	const struct ras_smu_drv *ras_smu_drv;
 };
 
 struct ras_fs_data {
@@ -981,4 +983,5 @@ void amdgpu_ras_pre_reset(struct amdgpu_device *adev,
 void amdgpu_ras_post_reset(struct amdgpu_device *adev,
 					  struct list_head *device_list);
 int amdgpu_ras_resume_after_reset(struct amdgpu_device *adev);
+uint64_t amdgpu_uniras_get_ras_caps(struct amdgpu_device *adev);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index 78adfc839ef80a..7779af085d1a47 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -1279,11 +1279,11 @@ amdgpu_ras_debugfs_table_read_uniras(struct amdgpu_device *adev,
 		return 0;
 
 	/* pmfw manages eeprom data by itself */
-	if (ras_fw_eeprom_supported(ras_core))
+	if (ras_eeprom_mgr_fw_record_enabled(ras_core))
 		return 0;
 
-	control = &ras_core->ras_eeprom;
-	num_recs = ras_eeprom_get_record_count(ras_core);
+	control = ras_core->eeprom_mgr.ras_eeprom;
+	num_recs = ras_eeprom_mgr_get_record_count(ras_core);
 
 	bufsz = strlen(tbl_hdr_str) + tbl_hdr_fmt_size +
 		strlen(rec_hdr_str) + (size_t)rec_hdr_fmt_size * num_recs + 1;
@@ -1299,7 +1299,7 @@ amdgpu_ras_debugfs_table_read_uniras(struct amdgpu_device *adev,
 			goto out;
 		}
 
-		res = ras_eeprom_read(ras_core, records, num_recs);
+		res = ras_eeprom_mgr_get_records(ras_core, 0, records, num_recs);
 		if (res)
 			goto out;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
index 8908d9e08a3043..6650b1194c0f2c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h
@@ -92,6 +92,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res,
 	case TTM_PL_TT:
 	case AMDGPU_PL_DOORBELL:
 	case AMDGPU_PL_MMIO_REMAP:
+	case AMDGPU_PL_NPA:
 		node = to_ttm_range_mgr_node(res)->mm_nodes;
 		while (start >= node->size << PAGE_SHIFT)
 			start -= node++->size << PAGE_SHIFT;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
index 428c3cbc4a401b..c8016fa55a92fb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
@@ -25,6 +25,7 @@
 #include "aldebaran.h"
 #include "sienna_cichlid.h"
 #include "smu_v13_0_10.h"
+#include "soc_v1_0.h"
 
 static int amdgpu_reset_xgmi_reset_on_init_suspend(struct amdgpu_device *adev)
 {
@@ -194,6 +195,9 @@ int amdgpu_reset_init(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 10):
 		ret = smu_v13_0_10_reset_init(adev);
 		break;
+	case IP_VERSION(15, 0, 8):
+		ret = soc_v1_0_reset_init(adev);
+		break;
 	default:
 		break;
 	}
@@ -218,6 +222,9 @@ int amdgpu_reset_fini(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 10):
 		ret = smu_v13_0_10_reset_fini(adev);
 		break;
+	case IP_VERSION(15, 0, 8):
+		ret = soc_v1_0_reset_fini(adev);
+		break;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 686c92e9602543..22773454131650 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -117,22 +117,7 @@ int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned int ndw)
  */
 void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
-	uint32_t occupied, chunk1, chunk2;
-
-	occupied = ring->wptr & ring->buf_mask;
-	chunk1 = ring->buf_mask + 1 - occupied;
-	chunk1 = (chunk1 >= count) ? count : chunk1;
-	chunk2 = count - chunk1;
-
-	if (chunk1)
-		memset32(&ring->ring[occupied], ring->funcs->nop, chunk1);
-
-	if (chunk2)
-		memset32(ring->ring, ring->funcs->nop, chunk2);
-
-	ring->wptr += count;
-	ring->wptr &= ring->ptr_mask;
-	ring->count_dw -= count;
+	amdgpu_ring_fill(ring, ring->funcs->nop, count);
 }
 
 /**
@@ -254,7 +239,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 		ring->adev = adev;
 		ring->num_hw_submission = sched_hw_submission;
 		ring->sched_score = sched_score;
-		ring->vmid_wait = dma_fence_get_stub();
 
 		ring->idx = adev->num_rings++;
 		adev->rings[ring->idx] = ring;
@@ -347,8 +331,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 	ring->buf_mask = (ring->ring_size / 4) - 1;
 	ring->ptr_mask = ring->funcs->support_64bit_ptrs ?
 		0xffffffffffffffff : ring->buf_mask;
-	/*  Initialize cached_rptr to 0 */
-	ring->cached_rptr = 0;
 
 	if (!ring->ring_backup) {
 		ring->ring_backup = kvzalloc(ring->ring_size, GFP_KERNEL);
@@ -374,6 +356,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 
 	ring->max_dw = max_dw;
 	ring->hw_prio = hw_prio;
+	ring->vmid_wait = dma_fence_get_stub();
 
 	if (!ring->no_scheduler && ring->funcs->type < AMDGPU_HW_IP_NUM) {
 		hw_ip = ring->funcs->type;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 015623b8fd0512..b2040f80b7f29d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -423,8 +423,6 @@ struct amdgpu_ring {
 
 	bool            is_sw_ring;
 	unsigned int    entry_index;
-	/* store the cached rptr to restore after reset */
-	uint64_t cached_rptr;
 };
 
 #define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))
@@ -490,6 +488,25 @@ static inline void amdgpu_ring_clear_ring(struct amdgpu_ring *ring)
 	memset32(ring->ring, ring->funcs->nop, ring->buf_mask + 1);
 }
 
+static inline void amdgpu_ring_clear_ring_and_ptrs(struct amdgpu_ring *ring)
+{
+	/* Clear the contents of the ring. */
+	amdgpu_ring_clear_ring(ring);
+
+	/* Clear the ring pointers on the CPU. */
+	ring->wptr = 0;
+
+	/*
+	 * Clear the ring pointers allocated in writeback.
+	 *
+	 * Note: we always allocate 64 bits for these pointers,
+	 * but older HW generations only use the lower 32 bits.
+	 * Use 64-bit atomics for simplicity.
+	 */
+	atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
+	atomic64_set((atomic64_t *)ring->rptr_cpu_addr, 0);
+}
+
 static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v)
 {
 	ring->ring[ring->wptr++ & ring->buf_mask] = v;
@@ -522,6 +539,32 @@ static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring,
 	ring->count_dw -= count_dw;
 }
 
+static inline void amdgpu_ring_fill(struct amdgpu_ring *ring,
+				    u32 val, u32 count)
+{
+	const u32 buf_mask = ring->buf_mask;
+	u32 occupied, chunk1, chunk2;
+	u64 wptr = ring->wptr;
+
+	if (count == 0)
+		return;
+
+	occupied = wptr & buf_mask;
+	chunk1 = buf_mask + 1 - occupied;
+	chunk1 = (chunk1 >= count) ? count : chunk1;
+	chunk2 = count - chunk1;
+
+	if (chunk1)
+		memset32(&ring->ring[occupied], val, chunk1);
+
+	if (chunk2)
+		memset32(ring->ring, val, chunk2);
+
+	wptr += count;
+	ring->wptr = wptr & ring->ptr_mask;
+	ring->count_dw -= count;
+}
+
 static inline unsigned int amdgpu_ring_get_dw_distance(struct amdgpu_ring *ring,
 						       u64 start_wptr, u64 end_wptr)
 {
@@ -578,8 +621,7 @@ int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		  enum amdgpu_ib_pool_type pool,
 		  struct amdgpu_ib *ib);
 void amdgpu_ib_free(struct amdgpu_ib *ib, struct dma_fence *f);
-int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
-		       struct amdgpu_ib *ibs, struct amdgpu_job *job,
+int amdgpu_ib_schedule(struct amdgpu_ring *ring, struct amdgpu_job *job,
 		       struct dma_fence **f);
 int amdgpu_ib_pool_init(struct amdgpu_device *adev);
 void amdgpu_ib_pool_fini(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
index fbac732f3e01a6..87ecbd429f9516 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
@@ -22,6 +22,7 @@
  */
 
 #include <linux/firmware.h>
+#include <linux/stdarg.h>
 #include "amdgpu.h"
 #include "amdgpu_sdma.h"
 #include "amdgpu_ras.h"
@@ -37,57 +38,35 @@
  * GPU SDMA IP block helpers function.
  */
 
-struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
+int __printf(4, 5)
+amdgpu_sdma_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
+		      unsigned int instance, const char *fmt, ...)
 {
-	struct amdgpu_device *adev = ring->adev;
-	int i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++)
-		if (ring == &adev->sdma.instance[i].ring ||
-		    ring == &adev->sdma.instance[i].page)
-			return &adev->sdma.instance[i];
-
-	return NULL;
-}
-
-int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
-{
-	struct amdgpu_device *adev = ring->adev;
-	int i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		if (ring == &adev->sdma.instance[i].ring ||
-			ring == &adev->sdma.instance[i].page) {
-			*index = i;
-			return 0;
-		}
-	}
-
-	return -EINVAL;
-}
-
-uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
-				     unsigned int vmid)
-{
-	struct amdgpu_device *adev = ring->adev;
-	uint64_t csa_mc_addr;
-	uint32_t index = 0;
+	struct amdgpu_sdma_instance *sdma;
+	va_list args;
 	int r;
 
-	/* don't enable OS preemption on SDMA under SRIOV */
-	if (amdgpu_sriov_vf(adev) || vmid == 0 || !adev->gfx.mcbp)
-		return 0;
+	ring->ring_obj = NULL;
 
-	r = amdgpu_sdma_get_index_from_ring(ring, &index);
+	va_start(args, fmt);
+	vsnprintf(ring->name, sizeof(ring->name), fmt, args);
+	va_end(args);
+
+	r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
+			     AMDGPU_SDMA_IRQ_INSTANCE0 + instance,
+			     AMDGPU_RING_PRIO_DEFAULT, NULL);
+	if (r)
+		return r;
 
-	if (r || index > 31)
-		csa_mc_addr = 0;
+	sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	if (amdgpu_sriov_vf(adev) || !adev->gfx.mcbp)
+		sdma->csa_addr = 0;
 	else
-		csa_mc_addr = amdgpu_csa_vaddr(adev) +
-			AMDGPU_CSA_SDMA_OFFSET +
-			index * AMDGPU_CSA_SDMA_SIZE;
+		sdma->csa_addr = amdgpu_csa_vaddr(adev) +
+				 AMDGPU_CSA_SDMA_OFFSET +
+				 ring->me * AMDGPU_CSA_SDMA_SIZE;
 
-	return csa_mc_addr;
+	return r;
 }
 
 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
@@ -542,16 +521,14 @@ static int amdgpu_sdma_soft_reset(struct amdgpu_device *adev, u32 instance_id)
 }
 
 /**
- * amdgpu_sdma_reset_engine - Reset a specific SDMA engine
+ * amdgpu_sdma_reset_engine() - Reset a specific SDMA engine instance.
+ *
  * @adev: Pointer to the AMDGPU device
  * @instance_id: Logical ID of the SDMA engine instance to reset
- * @caller_handles_kernel_queues: Skip kernel queue processing. Caller
- * will handle it.
  *
  * Returns: 0 on success, or a negative error code on failure.
  */
-int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
-			     bool caller_handles_kernel_queues)
+int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id)
 {
 	struct amdgpu_sdma_instance *sdma_instance = &adev->sdma.instance[instance_id];
 	struct amdgpu_ring *gfx_ring = &sdma_instance->ring;
@@ -564,20 +541,23 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 
 	mutex_lock(&sdma_instance->engine_reset_mutex);
 
-	if (!caller_handles_kernel_queues) {
-		/* Stop the scheduler's work queue for the GFX and page rings if they are running.
-		 * This ensures that no new tasks are submitted to the queues while
-		 * the reset is in progress.
-		 */
+	/*
+	 * Stop the scheduler's work queue for the GFX and page rings if they are running.
+	 * This ensures that no new tasks are submitted to the queues while
+	 * the reset is in progress.
+	 */
+	if (amdgpu_ring_sched_ready(gfx_ring) && !drm_sched_is_stopped(&gfx_ring->sched))
 		drm_sched_wqueue_stop(&gfx_ring->sched);
-		gfx_fence = amdgpu_ring_find_guilty_fence(gfx_ring);
-		amdgpu_ring_reset_helper_begin(gfx_ring, gfx_fence);
 
-		if (adev->sdma.has_page_queue) {
+	gfx_fence = amdgpu_ring_find_guilty_fence(gfx_ring);
+	amdgpu_ring_reset_helper_begin(gfx_ring, gfx_fence);
+
+	if (adev->sdma.has_page_queue) {
+		if (amdgpu_ring_sched_ready(page_ring) && !drm_sched_is_stopped(&page_ring->sched))
 			drm_sched_wqueue_stop(&page_ring->sched);
-			page_fence = amdgpu_ring_find_guilty_fence(page_ring);
-			amdgpu_ring_reset_helper_begin(page_ring, page_fence);
-		}
+
+		page_fence = amdgpu_ring_find_guilty_fence(page_ring);
+		amdgpu_ring_reset_helper_begin(page_ring, page_fence);
 	}
 
 	if (sdma_instance->funcs->stop_kernel_queue) {
@@ -593,6 +573,11 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 		goto exit;
 	}
 
+	amdgpu_ring_clear_ring_and_ptrs(gfx_ring);
+
+	if (adev->sdma.has_page_queue)
+		amdgpu_ring_clear_ring_and_ptrs(page_ring);
+
 	if (sdma_instance->funcs->start_kernel_queue) {
 		sdma_instance->funcs->start_kernel_queue(gfx_ring);
 		if (adev->sdma.has_page_queue)
@@ -600,22 +585,25 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 	}
 
 exit:
-	if (!caller_handles_kernel_queues) {
-		/* Restart the scheduler's work queue for the GFX and page rings
-		 * if they were stopped by this function. This allows new tasks
-		 * to be submitted to the queues after the reset is complete.
-		 */
-		if (!ret) {
-			ret = amdgpu_ring_reset_helper_end(gfx_ring, gfx_fence);
+	/* Restart the scheduler's work queue for the GFX and page rings
+	 * if they were stopped by this function. This allows new tasks
+	 * to be submitted to the queues after the reset is complete.
+	 */
+	if (!ret) {
+		ret = amdgpu_ring_reset_helper_end(gfx_ring, gfx_fence);
+		if (ret)
+			goto unlock;
+
+		if (amdgpu_ring_sched_ready(gfx_ring))
+			drm_sched_wqueue_start(&gfx_ring->sched);
+
+		if (adev->sdma.has_page_queue) {
+			ret = amdgpu_ring_reset_helper_end(page_ring, page_fence);
 			if (ret)
 				goto unlock;
-			drm_sched_wqueue_start(&gfx_ring->sched);
-			if (adev->sdma.has_page_queue) {
-				ret = amdgpu_ring_reset_helper_end(page_ring, page_fence);
-				if (ret)
-					goto unlock;
+
+			if (amdgpu_ring_sched_ready(page_ring))
 				drm_sched_wqueue_start(&page_ring->sched);
-			}
 		}
 	}
 unlock:
@@ -623,3 +611,38 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 
 	return ret;
 }
+
+/**
+ * amdgpu_sdma_reset_queue_legacy() - Reset legacy SDMA queue after timeout (without MES)
+ *
+ * @ring: Pointer to the ring of the SDMA queue
+ * @vmid: VMID of the timed out job
+ * @timedout_fence: Fence of the timed out job
+ *
+ * Common implementation for resetting SDMA queues without MES (legacy).
+ * This relies on the proper amdgpu_sdma_funcs to be set up
+ * for the given ring.
+ *
+ * Applicable to SDMA versions that don't rely on the MES yet,
+ * that is all versions up to SDMA v5.x and older.
+ */
+int amdgpu_sdma_reset_queue_legacy(struct amdgpu_ring *ring,
+				   unsigned int vmid,
+				   struct amdgpu_fence *timedout_fence)
+{
+	struct amdgpu_device *adev = ring->adev;
+	int r;
+
+	if (ring->me >= adev->sdma.num_instances) {
+		dev_err(adev->dev, "sdma instance not found\n");
+		return -EINVAL;
+	}
+
+	amdgpu_amdkfd_suspend(adev, true);
+	r = amdgpu_sdma_reset_engine(adev, ring->me);
+	amdgpu_amdkfd_resume(adev, true);
+	if (r)
+		return r;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 4f4e56022c970d..51dc52757f4894 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -23,150 +23,49 @@
 
 #ifndef __AMDGPU_SDMA_H__
 #define __AMDGPU_SDMA_H__
-#include "amdgpu_ras.h"
 
-/* max number of IP instances */
-#define AMDGPU_MAX_SDMA_INSTANCES		16
+#include "amdgpu_sdma_types.h"
+#include "amdgpu.h"
+#include "amdgpu_ring.h"
 
-enum amdgpu_sdma_irq {
-	AMDGPU_SDMA_IRQ_INSTANCE0  = 0,
-	AMDGPU_SDMA_IRQ_INSTANCE1,
-	AMDGPU_SDMA_IRQ_INSTANCE2,
-	AMDGPU_SDMA_IRQ_INSTANCE3,
-	AMDGPU_SDMA_IRQ_INSTANCE4,
-	AMDGPU_SDMA_IRQ_INSTANCE5,
-	AMDGPU_SDMA_IRQ_INSTANCE6,
-	AMDGPU_SDMA_IRQ_INSTANCE7,
-	AMDGPU_SDMA_IRQ_INSTANCE8,
-	AMDGPU_SDMA_IRQ_INSTANCE9,
-	AMDGPU_SDMA_IRQ_INSTANCE10,
-	AMDGPU_SDMA_IRQ_INSTANCE11,
-	AMDGPU_SDMA_IRQ_INSTANCE12,
-	AMDGPU_SDMA_IRQ_INSTANCE13,
-	AMDGPU_SDMA_IRQ_INSTANCE14,
-	AMDGPU_SDMA_IRQ_INSTANCE15,
-	AMDGPU_SDMA_IRQ_LAST
-};
+#include <linux/types.h>
 
-#define NUM_SDMA(x) hweight32(x)
-
-struct amdgpu_sdma_csa_info {
-	u32 size;
-	u32 alignment;
-};
-
-struct amdgpu_sdma_funcs {
-	int (*stop_kernel_queue)(struct amdgpu_ring *ring);
-	int (*start_kernel_queue)(struct amdgpu_ring *ring);
-	int (*soft_reset_kernel_queue)(struct amdgpu_device *adev, u32 instance_id);
-};
-
-struct amdgpu_sdma_instance {
-	/* SDMA firmware */
-	const struct firmware	*fw;
-	uint32_t		fw_version;
-	uint32_t		feature_version;
-
-	struct amdgpu_ring	ring;
-	struct amdgpu_ring	page;
-	bool			burst_nop;
-	union {
-	    uint32_t		aid_id;
-	    uint32_t		xcc_id;
-	};
-
-	struct amdgpu_bo	*sdma_fw_obj;
-	uint64_t		sdma_fw_gpu_addr;
-	uint32_t		*sdma_fw_ptr;
-	struct mutex		engine_reset_mutex;
-	/* track guilty state of GFX and PAGE queues */
-	bool			gfx_guilty;
-	bool			page_guilty;
-	const struct amdgpu_sdma_funcs   *funcs;
-};
-
-struct amdgpu_sdma_ras {
-	struct amdgpu_ras_block_object ras_block;
-};
-
-struct amdgpu_sdma {
-	struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES];
-	struct amdgpu_irq_src	trap_irq;
-	struct amdgpu_irq_src	illegal_inst_irq;
-	struct amdgpu_irq_src	fence_irq;
-	struct amdgpu_irq_src	ecc_irq;
-	struct amdgpu_irq_src	vm_hole_irq;
-	struct amdgpu_irq_src	doorbell_invalid_irq;
-	struct amdgpu_irq_src	pool_timeout_irq;
-	struct amdgpu_irq_src	srbm_write_irq;
-	struct amdgpu_irq_src	ctxt_empty_irq;
+struct amdgpu_device;
+struct amdgpu_iv_entry;
+struct amdgpu_irq_src;
+struct amdgpu_ring;
+struct ras_common_if;
 
-	int			num_instances;
-	uint32_t 		sdma_mask;
-	union {
-	    int			num_inst_per_aid;
-	    int			num_inst_per_xcc;
-	};
-	uint32_t                    srbm_soft_reset;
-	bool			has_page_queue;
-	struct ras_common_if	*ras_if;
-	struct amdgpu_sdma_ras	*ras;
-	uint32_t		*ip_dump;
-	uint32_t 		supported_reset;
-	struct list_head	reset_callback_list;
-	bool			no_user_submission;
-	bool			disable_uq;
-	void (*get_csa_info)(struct amdgpu_device *adev,
-			     struct amdgpu_sdma_csa_info *csa_info);
-};
-
-/*
- * Provided by hw blocks that can move/clear data.  e.g., gfx or sdma
- * But currently, we use sdma to move data.
- */
-struct amdgpu_buffer_funcs {
-	/* maximum bytes in a single operation */
-	uint32_t	copy_max_bytes;
-
-	/* number of dw to reserve per operation */
-	unsigned	copy_num_dw;
-
-	/* used for buffer migration */
-	void (*emit_copy_buffer)(struct amdgpu_ib *ib,
-				 /* src addr in bytes */
-				 uint64_t src_offset,
-				 /* dst addr in bytes */
-				 uint64_t dst_offset,
-				 /* number of byte to transfer */
-				 uint32_t byte_count,
-				 uint32_t copy_flags);
-
-	/* maximum bytes in a single operation */
-	uint32_t	fill_max_bytes;
+#define NUM_SDMA(x) hweight32(x)
 
-	/* number of dw to reserve per operation */
-	unsigned	fill_num_dw;
+int __printf(4, 5)
+amdgpu_sdma_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
+		      unsigned int instance, const char *fmt, ...);
 
-	/* used for buffer clearing */
-	void (*emit_fill_buffer)(struct amdgpu_ib *ib,
-				 /* value to write to memory */
-				 uint32_t src_data,
-				 /* dst addr in bytes */
-				 uint64_t dst_offset,
-				 /* number of byte to fill */
-				 uint32_t byte_count);
-};
+int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id);
 
-int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
-			     bool caller_handles_kernel_queues);
+int amdgpu_sdma_reset_queue_legacy(struct amdgpu_ring *ring,
+				   unsigned int vmid,
+				   struct amdgpu_fence *timedout_fence);
 
 #define amdgpu_emit_copy_buffer(adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib),  (s), (d), (b), (t))
 #define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b))
 
-struct amdgpu_sdma_instance *
-amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring);
-int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index);
-uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring, unsigned vmid);
+static inline struct amdgpu_sdma_instance *
+amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
+{
+	return &ring->adev->sdma.instance[ring->me];
+}
+
+static inline uint64_t
+amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring, unsigned vmid)
+{
+	if (vmid == 0)
+		return 0;
+	else
+		return amdgpu_sdma_get_instance_from_ring(ring)->csa_addr;
+}
+
 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
 			      struct ras_common_if *ras_block);
 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma_types.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma_types.h
new file mode 100644
index 00000000000000..1360db74d8b779
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma_types.h
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __AMDGPU_SDMA_TYPES_H__
+#define __AMDGPU_SDMA_TYPES_H__
+#include "amdgpu_ras.h"
+
+/* max number of IP instances */
+#define AMDGPU_MAX_SDMA_INSTANCES		16
+
+enum amdgpu_sdma_irq {
+	AMDGPU_SDMA_IRQ_INSTANCE0  = 0,
+	AMDGPU_SDMA_IRQ_INSTANCE1,
+	AMDGPU_SDMA_IRQ_INSTANCE2,
+	AMDGPU_SDMA_IRQ_INSTANCE3,
+	AMDGPU_SDMA_IRQ_INSTANCE4,
+	AMDGPU_SDMA_IRQ_INSTANCE5,
+	AMDGPU_SDMA_IRQ_INSTANCE6,
+	AMDGPU_SDMA_IRQ_INSTANCE7,
+	AMDGPU_SDMA_IRQ_INSTANCE8,
+	AMDGPU_SDMA_IRQ_INSTANCE9,
+	AMDGPU_SDMA_IRQ_INSTANCE10,
+	AMDGPU_SDMA_IRQ_INSTANCE11,
+	AMDGPU_SDMA_IRQ_INSTANCE12,
+	AMDGPU_SDMA_IRQ_INSTANCE13,
+	AMDGPU_SDMA_IRQ_INSTANCE14,
+	AMDGPU_SDMA_IRQ_INSTANCE15,
+	AMDGPU_SDMA_IRQ_LAST
+};
+
+struct amdgpu_sdma_csa_info {
+	u32 size;
+	u32 alignment;
+};
+
+struct amdgpu_sdma_funcs {
+	int (*stop_kernel_queue)(struct amdgpu_ring *ring);
+	int (*start_kernel_queue)(struct amdgpu_ring *ring);
+	int (*soft_reset_kernel_queue)(struct amdgpu_device *adev, u32 instance_id);
+	bool (*detect_hung_queue)(struct amdgpu_device *adev, u32 doorbell_index,
+				  u32 *instance_id, u32 *queue_id);
+};
+
+struct amdgpu_sdma_instance {
+	/* SDMA firmware */
+	const struct firmware	*fw;
+	uint32_t		fw_version;
+	uint32_t		feature_version;
+
+	struct amdgpu_ring	ring;
+	struct amdgpu_ring	page;
+	bool			burst_nop;
+	union {
+	    uint32_t		aid_id;
+	    uint32_t		xcc_id;
+	};
+
+	uint64_t		csa_addr;
+
+	struct amdgpu_bo	*sdma_fw_obj;
+	uint64_t		sdma_fw_gpu_addr;
+	uint32_t		*sdma_fw_ptr;
+	struct mutex		engine_reset_mutex;
+	/* track guilty state of GFX and PAGE queues */
+	bool			gfx_guilty;
+	bool			page_guilty;
+	const struct amdgpu_sdma_funcs   *funcs;
+};
+
+struct amdgpu_sdma_ras {
+	struct amdgpu_ras_block_object ras_block;
+};
+
+struct amdgpu_sdma {
+	struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES];
+	struct amdgpu_irq_src	trap_irq;
+	struct amdgpu_irq_src	illegal_inst_irq;
+	struct amdgpu_irq_src	fence_irq;
+	struct amdgpu_irq_src	ecc_irq;
+	struct amdgpu_irq_src	vm_hole_irq;
+	struct amdgpu_irq_src	doorbell_invalid_irq;
+	struct amdgpu_irq_src	pool_timeout_irq;
+	struct amdgpu_irq_src	srbm_write_irq;
+	struct amdgpu_irq_src	ctxt_empty_irq;
+
+	int			num_instances;
+	uint32_t 		sdma_mask;
+	bool    		sdma_debug;
+	union {
+	    int			num_inst_per_aid;
+	    int			num_inst_per_xcc;
+	};
+	uint32_t                    srbm_soft_reset;
+	bool			has_page_queue;
+	struct ras_common_if	*ras_if;
+	struct amdgpu_sdma_ras	*ras;
+	uint32_t		*ip_dump;
+	uint32_t 		supported_reset;
+	struct list_head	reset_callback_list;
+	bool			no_user_submission;
+	bool			disable_uq;
+	void (*get_csa_info)(struct amdgpu_device *adev,
+			     struct amdgpu_sdma_csa_info *csa_info);
+};
+
+/*
+ * Provided by hw blocks that can move/clear data.  e.g., gfx or sdma
+ * But currently, we use sdma to move data.
+ */
+struct amdgpu_buffer_funcs {
+	/* maximum bytes in a single operation */
+	uint32_t	copy_max_bytes;
+
+	/* number of dw to reserve per operation */
+	unsigned	copy_num_dw;
+
+	/* used for buffer migration */
+	void (*emit_copy_buffer)(struct amdgpu_ib *ib,
+				 /* src addr in bytes */
+				 uint64_t src_offset,
+				 /* dst addr in bytes */
+				 uint64_t dst_offset,
+				 /* number of byte to transfer */
+				 uint32_t byte_count,
+				 uint32_t copy_flags);
+
+	/* maximum bytes in a single operation */
+	uint32_t	fill_max_bytes;
+
+	/* number of dw to reserve per operation */
+	unsigned	fill_num_dw;
+
+	/* used for buffer clearing */
+	void (*emit_fill_buffer)(struct amdgpu_ib *ib,
+				 /* value to write to memory */
+				 uint32_t src_data,
+				 /* dst addr in bytes */
+				 uint64_t dst_offset,
+				 /* number of byte to fill */
+				 uint32_t byte_count);
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index fcd4cf2f4cd887..66f4a187291385 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -37,6 +37,8 @@
 #define AMDGPU_JOB_GET_TIMELINE_NAME(job) \
 	 job->base.s_fence->finished.ops->get_timeline_name(&job->base.s_fence->finished)
 
+struct amdgpu_vm_update_params;
+
 TRACE_EVENT(amdgpu_device_rreg,
 	    TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
 	    TP_ARGS(did, reg, value),
@@ -325,9 +327,8 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
 	    TP_PROTO(struct amdgpu_vm_update_params *p,
 		     uint64_t start, uint64_t end,
 		     unsigned int nptes, uint64_t dst,
-		     uint64_t incr, uint64_t flags,
-		     pid_t pid, uint64_t vm_ctx),
-	TP_ARGS(p, start, end, nptes, dst, incr, flags, pid, vm_ctx),
+		     uint64_t incr, uint64_t flags),
+	TP_ARGS(p, start, end, nptes, dst, incr, flags),
 	TP_STRUCT__entry(
 			 __field(u64, start)
 			 __field(u64, end)
@@ -347,8 +348,9 @@ TRACE_EVENT(amdgpu_vm_update_ptes,
 			__entry->flags = flags;
 			__entry->incr = incr;
 			__entry->nptes = nptes;
-			__entry->pid = pid;
-			__entry->vm_ctx = vm_ctx;
+			__entry->pid = p->vm->task_info ?
+				p->vm->task_info->tgid : 0;
+			__entry->vm_ctx = p->vm->immediate.fence_context;
 			for (i = 0; i < nptes; ++i) {
 				u64 addr = p->pages_addr ? amdgpu_vm_map_gart(
 					p->pages_addr, dst) : dst;
@@ -728,6 +730,39 @@ TRACE_EVENT(amdgpu_userq_state_changed,
 		      __entry->client_id, __entry->queue_type, __entry->doorbell_index, __entry->to)
 );
 
+TRACE_EVENT(amdgpu_register_pid,
+	    TP_PROTO(uint32_t pid, int major, int minor),
+	    TP_ARGS(pid, major, minor),
+	    TP_STRUCT__entry(
+			__field(u32, pid)
+			__field(int, major)
+			__field(int, minor)
+	    ),
+	    TP_fast_assign(
+			__entry->pid = pid;
+			__entry->major = major;
+			__entry->minor = minor;
+	    ),
+	    TP_printk("amdgpu register pid %d render %d:%d",
+		      __entry->pid, __entry->major, __entry->minor)
+);
+TRACE_EVENT(amdgpu_deregister_pid,
+	    TP_PROTO(uint32_t pid, int major, int minor),
+	    TP_ARGS(pid, major, minor),
+	    TP_STRUCT__entry(
+			__field(u32, pid)
+			__field(int, major)
+			__field(int, minor)
+	    ),
+	    TP_fast_assign(
+			__entry->pid = pid;
+			__entry->major = major;
+			__entry->minor = minor;
+	    ),
+	    TP_printk("amdgpu deregister pid %d render %d:%d",
+		      __entry->pid, __entry->major, __entry->minor)
+);
+
 #undef AMDGPU_JOB_GET_TIMELINE_NAME
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
index b96d885f6e3316..fe61ed5b487e7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace_points.c
@@ -25,6 +25,7 @@
 #include <drm/amdgpu_drm.h>
 #include "amdgpu_cs.h"
 #include "amdgpu.h"
+#include "amdgpu_vm_internal.h"
 
 #define CREATE_TRACE_POINTS
 #include "amdgpu_trace.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 016957cac1f29b..0ca9b1669e0fea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -31,13 +31,12 @@
  */
 
 #include <linux/dma-mapping.h>
+#include <linux/highmem.h>
 #include <linux/iommu.h>
-#include <linux/pagemap.h>
 #include <linux/sched/task.h>
 #include <linux/sched/mm.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
-#include <linux/swap.h>
 #include <linux/dma-buf.h>
 #include <linux/sizes.h>
 #include <linux/module.h>
@@ -56,6 +55,7 @@
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_sdma.h"
 #include "amdgpu_ras.h"
+#include "amdgpu_ras_mgr.h"
 #include "amdgpu_hmm.h"
 #include "amdgpu_atomfirmware.h"
 #include "amdgpu_res_cursor.h"
@@ -128,6 +128,7 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
 	case AMDGPU_PL_OA:
 	case AMDGPU_PL_DOORBELL:
 	case AMDGPU_PL_MMIO_REMAP:
+	case AMDGPU_PL_NPA:
 		placement->num_placement = 0;
 		return;
 
@@ -203,7 +204,8 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
 				 struct amdgpu_res_cursor *mm_cur,
 				 unsigned int window,
 				 bool tmz, uint64_t *size, uint64_t *addr,
-				 bool *vm_needs_flush)
+				 bool *vm_needs_flush,
+				 bool readonly)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
 	unsigned int offset, num_pages, num_dw, num_bytes;
@@ -268,6 +270,8 @@ static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
 	flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem);
 	if (tmz)
 		flags |= AMDGPU_PTE_TMZ;
+	if (readonly)
+		flags &= ~AMDGPU_PTE_WRITEABLE;
 
 	cpu_addr = &job->ibs[0].ptr[num_dw];
 
@@ -338,12 +342,12 @@ static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 
 		/* Map src to window 0 and dst to window 1. */
 		r = amdgpu_ttm_map_buffer(entity, src->bo, src->mem, &src_mm,
-					  0, tmz, &cur_size, &from, &src_vm_flush);
+					  0, tmz, &cur_size, &from, &src_vm_flush, true);
 		if (r)
 			goto error;
 
 		r = amdgpu_ttm_map_buffer(entity, dst->bo, dst->mem, &dst_mm,
-					  1, tmz, &cur_size, &to, &dst_vm_flush);
+					  1, tmz, &cur_size, &to, &dst_vm_flush, false);
 		if (r)
 			goto error;
 
@@ -583,11 +587,13 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 	    old_mem->mem_type == AMDGPU_PL_OA ||
 	    old_mem->mem_type == AMDGPU_PL_DOORBELL ||
 	    old_mem->mem_type == AMDGPU_PL_MMIO_REMAP ||
+	    old_mem->mem_type == AMDGPU_PL_NPA ||
 	    new_mem->mem_type == AMDGPU_PL_GDS ||
 	    new_mem->mem_type == AMDGPU_PL_GWS ||
 	    new_mem->mem_type == AMDGPU_PL_OA ||
 	    new_mem->mem_type == AMDGPU_PL_DOORBELL ||
-	    new_mem->mem_type == AMDGPU_PL_MMIO_REMAP) {
+	    new_mem->mem_type == AMDGPU_PL_MMIO_REMAP ||
+	    new_mem->mem_type == AMDGPU_PL_NPA) {
 		/* Nothing to save here */
 		amdgpu_bo_move_notify(bo, evict, new_mem);
 		ttm_bo_move_null(bo, new_mem);
@@ -1740,7 +1746,7 @@ void amdgpu_ttm_init_vram_resv(struct amdgpu_device *adev,
 
 static void amdgpu_ttm_init_fw_resv_region(struct amdgpu_device *adev)
 {
-	uint32_t reserve_size = 0;
+	u64 reserve_size = 0, offset = 0;
 
 	if (!adev->discovery.reserve_tmr)
 		return;
@@ -1752,24 +1758,28 @@ static void amdgpu_ttm_init_fw_resv_region(struct amdgpu_device *adev)
 	 * Otherwise, fallback to legacy approach to check and reserve tmr block for ip
 	 * discovery data and G6 memory training data respectively
 	 */
-	if (adev->bios)
+	if (adev->bios) {
 		reserve_size =
 			amdgpu_atomfirmware_get_fw_reserved_fb_size(adev);
+		if (reserve_size)
+			offset = adev->gmc.real_vram_size - reserve_size;
+	}
 
 	if (!adev->bios &&
 	    (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
 	     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
-	     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)))
-		reserve_size = max(reserve_size, (uint32_t)280 << 20);
-	else if (!adev->bios &&
-		 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
-		reserve_size = max(reserve_size, (uint32_t)150 << 20);
-	} else if (!reserve_size)
+	     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0))) {
+		reserve_size = (u64)280 << 20;
+		offset = adev->gmc.real_vram_size - reserve_size;
+	} else if (adev->asic_funcs && adev->asic_funcs->get_fw_reserved_info) {
+		dev_dbg(adev->dev, "Querying FW reserved region info through get_fw_reserved_info\n");
+		adev->asic_funcs->get_fw_reserved_info(adev, &reserve_size, &offset);
+	} else if (!reserve_size) {
 		reserve_size = DISCOVERY_TMR_OFFSET;
-
+		offset = adev->gmc.real_vram_size - reserve_size;
+	}
 	amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_FW,
-				  adev->gmc.real_vram_size - reserve_size,
-				  reserve_size, false);
+				offset,	reserve_size, false);
 }
 
 static void amdgpu_ttm_init_mem_train_resv_region(struct amdgpu_device *adev)
@@ -2168,6 +2178,9 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
+	/* RAS loads and reserves bad pages */
+	amdgpu_ras_mgr_early_init_service(adev);
+
 	if (adev->mman.resv_region[AMDGPU_RESV_MEM_TRAIN].size) {
 		struct psp_memory_training_context *ctx =
 					&adev->psp.mem_train_ctx;
@@ -2273,6 +2286,16 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 		dev_err(adev->dev, "Failed initializing oa heap.\n");
 		return r;
 	}
+
+	if (adev->ualink.npa_size) {
+		r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_NPA,
+					    adev->ualink.npa_size);
+		if (r) {
+			dev_err(adev->dev, "Failed initializing NPA heap.\n");
+			return r;
+		}
+	}
+
 	if (amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
 				AMDGPU_GEM_DOMAIN_GTT,
 				&adev->mman.sdma_access_bo, NULL,
@@ -2326,6 +2349,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA);
 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_DOORBELL);
 	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_MMIO_REMAP);
+	ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_NPA);
 	ttm_device_fini(&adev->mman.bdev);
 	adev->mman.initialized = false;
 	dev_info(adev->dev, " ttm finalized\n");
@@ -2653,7 +2677,7 @@ int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity,
 		cur_size = min(dst.size, 256ULL << 20);
 
 		r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &dst,
-					  0, false, &cur_size, &to, &vm_needs_flush);
+					  0, false, &cur_size, &to, &vm_needs_flush, false);
 		if (r)
 			goto error;
 
@@ -2708,6 +2732,7 @@ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
 	case AMDGPU_PL_GWS:
 	case AMDGPU_PL_GDS:
 	case AMDGPU_PL_OA:
+	case AMDGPU_PL_NPA:
 		man = ttm_manager_type(&adev->mman.bdev, mem_type);
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index af1e7fcc717566..adb6cfae50527d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -37,7 +37,9 @@
 #define AMDGPU_PL_PREEMPT	(TTM_PL_PRIV + 3)
 #define AMDGPU_PL_DOORBELL	(TTM_PL_PRIV + 4)
 #define AMDGPU_PL_MMIO_REMAP	(TTM_PL_PRIV + 5)
-#define __AMDGPU_PL_NUM	(TTM_PL_PRIV + 6)
+#define AMDGPU_PL_NPA		(TTM_PL_PRIV + 6)
+#define __AMDGPU_PL_NUM		(TTM_PL_PRIV + 7)
+
 
 #define AMDGPU_GTT_MAX_TRANSFER_SIZE	(1ULL << 22)
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
new file mode 100644
index 00000000000000..8411ea17172fb7
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.c
@@ -0,0 +1,6539 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/xarray.h>
+#include <drm/drm_mm.h>
+#include <linux/random.h>
+#include "amdgpu.h"
+#include "amdgpu_sdma.h"
+#include "amdgpu_ualink.h"
+#include "amdgpu_xgmi.h"
+#include "amdgpu_dma_buf.h"
+#include "psp_gfx_if.h"
+#include <linux/math.h>
+#include <linux/sysfs.h>
+#include <linux/string.h>
+
+static void deactivate_accelerator(struct amdgpu_device *adev);
+static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev,
+				u32 remote_accel_id, u32 dw0, u32 dw1,
+				u32 dw2, u32 dw3);
+static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev,
+				    u32 flush_type);
+static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev,
+						struct amdgpu_bo *bos[], u32 n_bos,
+						struct drm_exec *exec,
+						bool interruptible);
+static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev,
+						   struct drm_exec *exec);
+static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev,
+						  u32 remote_accel_id, u32 state,
+						  u32 generation_count);
+static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo);
+static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev,
+					  u32 remote_accel_id, u64 addr,
+					  u32 size_in_pages, u32 flush_type);
+static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev);
+static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev);
+static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev,
+					   u32 remote_accel_id);
+static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev,
+					  u32 remote_accel_id);
+
+#define STRIP_NPA(addr)						\
+	(((u64)(addr) & ~AMDGPU_UALINK_NPA_ADDR_GPUID_MASK))
+
+#define GENERATE_NPA(addr, remote_acc_id)			\
+		((u64)(((u64)(addr)) |				\
+		 ((u64)(remote_acc_id) << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT)))
+
+static const struct drm_client_funcs ualink_client_funcs = {
+	.unregister	= drm_client_release,
+};
+
+static int amdgpu_ualink_drm_client_create(struct amdgpu_device *adev)
+{
+	int ret;
+
+	ret = drm_client_init(&adev->ddev, &adev->ualink.client, "ualink",
+			      &ualink_client_funcs);
+	if (ret) {
+		dev_err(adev->dev, "Failed to init UALink DRM client: %d\n",
+			ret);
+		return ret;
+	}
+
+	drm_client_register(&adev->ualink.client);
+
+	return 0;
+}
+
+static void amdgpu_ualink_object_fini(struct amdgpu_device *adev)
+{
+	if (!adev->ualink.info)
+		return;
+
+	kobject_put(&adev->ualink.stations->kobj);
+	kobject_put(&adev->ualink.config->kobj);
+	kobject_put(&adev->ualink.setup->kobj);
+	kobject_put(&adev->ualink.info->kobj);
+	adev->ualink.stations = NULL;
+	adev->ualink.config = NULL;
+	adev->ualink.setup = NULL;
+	adev->ualink.info = NULL;
+}
+
+static bool __check_ppod_info(struct amdgpu_device *adev,
+			      const struct amdgpu_ualink_info *info)
+{
+	const struct amdgpu_ualink_ppod_info *ppod = &info->ppod;
+
+	if (ppod->size <= 0 || ppod->size > AMDGPU_UALINK_ACCEL_MAX) {
+		dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n",
+			ppod->size, AMDGPU_UALINK_ACCEL_MAX);
+		return false;
+	}
+	if (ppod->accel_id >= ppod->size) {
+		dev_dbg(adev->dev,
+			"Accelerator ID %u greater or equal pPod size %u\n",
+			ppod->accel_id, ppod->size);
+		return false;
+	}
+
+	return true;
+}
+
+static bool __check_vpod_info(struct amdgpu_device *adev,
+			      const struct amdgpu_ualink_info *info)
+{
+	const struct amdgpu_ualink_ppod_info *ppod = &info->ppod;
+	const struct amdgpu_ualink_vpod_info *vpod = &info->vpod;
+	unsigned int weight;
+
+	if (vpod->size == 0 || vpod->size > ppod->size) {
+		dev_warn(adev->dev,
+			 "UALINK: vPod size %u out of range [1..%u]\n",
+			 vpod->size, ppod->size);
+		return false;
+	}
+	if (vpod->addr_mode >= AMDGPU_UALINK_ADDR_MODE_MAX) {
+		dev_warn(adev->dev,
+			 "UALINK: invalid addr mode %u\n",
+			 vpod->addr_mode);
+		return false;
+	}
+	weight =
+		bitmap_weight(vpod->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
+	if (weight != vpod->size) {
+		dev_warn(adev->dev,
+			 "UALINK: vPod size doesn't match vpod_active_accels list: %u != %u\n",
+			 vpod->size, weight);
+		return false;
+	}
+	if (!test_bit(ppod->accel_id, vpod->active_accel_bits)) {
+		dev_warn(adev->dev,
+			 "UALINK: accelerator ID %u not listed in vpod_active_accels\n",
+			 ppod->accel_id);
+		return false;
+	}
+
+	return true;
+}
+
+static void
+amdgpu_ualink_info_set_accel_state(struct amdgpu_device *adev,
+				   struct amdgpu_ualink_info *info,
+				   enum psp_gfx_ual_config_state cfg_state)
+{
+	enum amdgpu_ualink_accel_state cur;
+	enum amdgpu_ualink_accel_state target;
+	bool ppod_validated;
+	bool vpod_validated;
+
+	if (!info)
+		return;
+
+	cur = info->accel_state;
+
+	ppod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED &&
+			 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
+	vpod_validated = cur >= AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED &&
+			 cur <= AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
+
+	switch (cfg_state) {
+	case UAL_CFG_IDLE:
+		return;
+	case UAL_CFG_PPOD:
+		target = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
+		break;
+	case UAL_CFG_VPOD:
+	case UAL_CFG_STATION:
+		target = AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED;
+		break;
+	case UAL_CFG_COMPLETE:
+		target = AMDGPU_UALINK_ACCEL_STATE_READY;
+		break;
+	default:
+		dev_dbg(adev->dev, "invalid configuration state %u", cfg_state);
+		return;
+	}
+
+	/* ppod stage: should be part of a ppod first */
+	if (!ppod_validated && !__check_ppod_info(adev, info)) {
+		if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
+			info->accel_state =
+				AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
+		} else {
+			info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR;
+			dev_err(adev->dev,
+				"ppod configuration is invalid, setting to error state");
+		}
+		return;
+	}
+	if (target == AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
+		if (!ppod_validated)
+			info->accel_state =
+				AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
+		return;
+	}
+
+	/* A vpod_id of 0 means the GPU is not part of any vPod */
+	if (info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) {
+		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
+		return;
+	}
+
+	/* vpod stage: required to reach vpod_configured or ready */
+	if (!vpod_validated && !__check_vpod_info(adev, info)) {
+		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ERROR;
+		dev_err(adev->dev,
+			"vpod configuration is invalid, setting to error state");
+		return;
+	}
+	if (target == AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) {
+		if (!vpod_validated)
+			info->accel_state =
+				AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED;
+		return;
+	}
+
+	/* complete stage: advance to ready unless already ready/active */
+	if (cur < AMDGPU_UALINK_ACCEL_STATE_READY ||
+	    cur > AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
+		info->accel_state = AMDGPU_UALINK_ACCEL_STATE_READY;
+}
+
+static int amdgpu_ualink_update_vpod_config(struct amdgpu_device *adev)
+{
+	/* TBD: Do updates/cleanup based on updated vpod configuration */
+	return 0;
+}
+
+static int amdgpu_ualink_update_accel_state(
+	struct amdgpu_device *adev, enum amdgpu_ualink_accel_state prev_state,
+	u32 prev_vpod_id, enum psp_gfx_ual_config_state cfg_state)
+{
+	dev_info(adev->dev,
+		 "UALINK: update_accel_state: prev_state=%d prev_vpod_id=%u new_vpod_id=%u cfg_state=%d\n",
+		 prev_state, prev_vpod_id, adev->ualink.info->vpod.id,
+		 cfg_state);
+
+	/* If the device is already active and its vpod_id is unchanged, the
+	 * update does not affect vpod membership. Skip the local vpod
+	 * integrity check and re-activation.
+	 */
+	if (prev_state == AMDGPU_UALINK_ACCEL_STATE_ACTIVE &&
+	    adev->ualink.info->vpod.id == prev_vpod_id) {
+		amdgpu_ualink_update_vpod_config(adev);
+
+		/* Same vpod_id, but the member set may have changed (vPod grown
+		 * or shrunk while this GPU stayed ACTIVE). If so, run the local
+		 * vpod activation path: once every local peer has committed the
+		 * new config (integrity passes), it bounces the affected ACTIVE
+		 * peers to rebuild links/GART for the new member set.
+		 */
+		if (amdgpu_ualink_vpod_membership_changed(adev)) {
+			dev_info(adev->dev,
+				 "UALINK: update_accel_state: ACTIVE vpod_id=%u membership changed, reconfiguring\n",
+				 adev->ualink.info->vpod.id);
+			scoped_guard(mutex, &mgpu_info.mutex)
+				__amdgpu_ualink_activate_vpod_locked(adev);
+		} else {
+			dev_info(adev->dev,
+				 "UALINK: update_accel_state: already ACTIVE, vpod_id/membership unchanged\n");
+		}
+		return 0;
+	}
+
+	/* A new vpod_id of 0 means this GPU was removed from the vPod. */
+	if (adev->ualink.info->vpod.id == AMDGPU_UALINK_VPOD_ID_INVALID) {
+		dev_info(adev->dev,
+			 "UALINK: update_accel_state: vpod_id=0, removing accelerator from vPod\n");
+		amdgpu_ualink_update_vpod_config(adev);
+		scoped_guard(mutex, &mgpu_info.mutex)
+			deactivate_accelerator(adev);
+		return 0;
+	}
+
+	/* GPU joining a new vpod should be with invalid vpod id*/
+	scoped_guard(mutex, &mgpu_info.mutex) {
+		if (prev_vpod_id == AMDGPU_UALINK_VPOD_ID_INVALID) {
+			dev_info(adev->dev,
+				 "UALINK: update_accel_state: joining vpod_id=%u\n",
+				 adev->ualink.info->vpod.id);
+			amdgpu_ualink_info_set_accel_state(
+				adev, adev->ualink.info, cfg_state);
+			__amdgpu_ualink_activate_vpod_locked(adev);
+		} else {
+			/* GPU should first get removal which will set invalid vpod_id
+			* and then join a new vpod
+			*/
+			dev_err(adev->dev,
+				"Invalid vpod transition from %u to %u\n",
+				prev_vpod_id, adev->ualink.info->vpod.id);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev)
+{
+	enum amdgpu_ualink_accel_state prev_state;
+	enum psp_gfx_ual_config_state cfg_state;
+	u32 prev_vpod_id;
+	int r, qerr;
+
+	/* TBD: Stop ASP interrupts if driver faced an issue */
+	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
+		u32 status;
+
+		dev_dbg(adev->dev,
+			"UALink not initialized, skipping config update\n");
+		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
+		return psp_ual_send_completion(
+			&adev->psp, adev->ualink.psp_if_ver,
+			PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, status);
+	}
+
+	prev_state = adev->ualink.info->accel_state;
+	prev_vpod_id = adev->ualink.info->vpod.id;
+
+	qerr = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver,
+				  adev->ualink.info, &cfg_state);
+
+	/*TBD: find the right value of status to be sent to ASP*/
+	r = psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
+				    PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID, 0);
+	if (r || qerr)
+		goto err;
+
+	r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id,
+					     cfg_state);
+	if (r)
+		goto err;
+
+	return 0;
+
+err:
+	scoped_guard(mutex, &mgpu_info.mutex) {
+		deactivate_accelerator(adev);
+		adev->ualink.info->accel_state =
+			AMDGPU_UALINK_ACCEL_STATE_ERROR;
+	}
+	dev_err(adev->dev,
+		"UALink config update failed, setting to error state");
+
+	return r;
+}
+
+int amdgpu_ualink_pause_handler(struct amdgpu_device *adev)
+{
+	u32 status = 0;
+
+	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
+		dev_dbg(adev->dev,
+			"UALink not initialized, skipping pause update\n");
+		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
+		goto out;
+	}
+
+	dev_dbg(adev->dev, "UALink pause command is not handled\n");
+
+out:
+	return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
+				       PSP_GFX_INT_CTXT_UAL_CMD_PAUSE_ID,
+				       status);
+}
+
+int amdgpu_ualink_resume_handler(struct amdgpu_device *adev)
+{
+	u32 status = 0;
+
+	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE) {
+		dev_dbg(adev->dev,
+			"UALink not initialized, skipping pause update\n");
+		status = !!(adev->ualink.mgr_state == AMDGPU_UALINK_INIT_ERROR);
+		goto out;
+	}
+
+	dev_dbg(adev->dev, "UALink resume command is not handled\n");
+
+out:
+	return psp_ual_send_completion(&adev->psp, adev->ualink.psp_if_ver,
+				       PSP_GFX_INT_CTXT_UAL_CMD_RESUME_ID,
+				       status);
+}
+
+int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	if (!adev->ualink.info)
+		return 0;
+
+	adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
+	r = psp_ual_get_interface_version(&adev->psp, &adev->ualink.psp_if_ver);
+	if (r) {
+		adev->ualink.psp_if_ver = 0xffffffff;
+		goto disable;
+	}
+	dev_info(adev->dev, "Found UALink interface version 0x%x\n",
+		 adev->ualink.psp_if_ver);
+
+	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_HW;
+
+	return 0;
+disable:
+	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR;
+	return 0;
+}
+
+int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)
+		return 0;
+
+	scoped_guard(mutex, &mgpu_info.mutex)
+		deactivate_accelerator(adev);
+
+	return 0;
+}
+
+int ualink_ip_late_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	enum psp_gfx_ual_config_state cfg_state;
+	int r;
+
+	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_HW)
+		return 0;
+
+	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver,
+			       adev->ualink.info, &cfg_state);
+	if (r)
+		return r;
+
+	scoped_guard(mutex, &mgpu_info.mutex) {
+		amdgpu_ualink_info_set_accel_state(adev, adev->ualink.info,
+						   cfg_state);
+		__amdgpu_ualink_activate_vpod_locked(adev);
+	}
+
+	r = amdgpu_ualink_drm_client_create(adev);
+	if (r) {
+		dev_err(adev->dev, "Failed to create UALink DRM client: %d\n",
+			r);
+		goto error;
+	}
+
+	/* Consider UALink initialized only at this stage */
+	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_COMPLETE;
+
+	return 0;
+
+error:
+	adev->ualink.mgr_state = AMDGPU_UALINK_INIT_ERROR;
+	return r;
+}
+
+/****************************************************************************
+ * UALink info and configuration in sysfs
+ *
+ * Using kobj_attribute and not device_attribute here because the UALink
+ * attributes are part of their own nested kobjects and not the amdgpu
+ * device itself or a subdevice.
+ ****************************************************************************/
+
+/* UALink configuration APIs are being deprecated before upstreaming and are
+ * not enabled by default. They are replaced by a unified in-band/side-band
+ * approach through the IFoE driver. Querying of the current configuration
+ * will continue to be supported in sysfs.
+ *
+ * Uncomment the following line to enable the deprecated configuration APIs
+ * for testing purposes only. The code to support these APIs will be removed
+ * in a future patch.
+ */
+/* #define UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS */
+
+#define UALINK_VALUE_SHOW(prefix, name, field, format)			\
+static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
+					       struct kobj_attribute *attr,\
+					       char *buf)		\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+									\
+	return sysfs_emit(buf, format"\n", info->field);		\
+}
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+#define UALINK_VALUE_STORE(prefix, name, field, type, base)		\
+static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
+						struct kobj_attribute *attr,\
+						const char *buf, size_t count)\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+	int r;								\
+									\
+	r = kstrto##type(buf, base, &info->field);			\
+	if (r < 0)							\
+		return r;						\
+	return count;							\
+}
+#endif
+
+static ssize_t show_idbits(unsigned long *bits, unsigned int nbits, char *buf)
+{
+	ssize_t len = 0;
+	unsigned int id;
+
+	for_each_set_bit(id, bits, nbits)
+		len += sysfs_emit_at(buf, len, "%u ", id);
+	if (len)
+		len--;
+	return len + sysfs_emit_at(buf, len, "\n");
+}
+#define UALINK_IDBITS_SHOW(prefix, name, field)				\
+static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
+					       struct kobj_attribute *attr,\
+					       char *buf)		\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+									\
+	return show_idbits(info->field, sizeof(info->field)*8, buf);	\
+}
+
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+static ssize_t store_idbits(unsigned long *bits, unsigned int nbits,
+			    const char *buf, size_t count)
+{
+	char *dup_buf, *str, *tok;
+	ssize_t r;
+	u32 id;
+
+	bitmap_zero(bits, nbits);
+	if (!count)
+		return 0;
+
+	dup_buf = kstrndup(buf, count, GFP_KERNEL);
+	if (unlikely(!dup_buf))
+		return -ENOMEM;
+	str = dup_buf;
+	do {
+		str += strspn(str, " ");
+		tok = strsep(&str, ", ");
+		r = kstrtou32(tok, 10, &id);
+		if (r < 0)
+			goto err;
+		if (id >= nbits) {
+			r = -ERANGE;
+			goto err;
+		}
+		set_bit(id, bits);
+	} while (str);
+	kfree(dup_buf);
+
+	return count;
+
+err:
+	kfree(dup_buf);
+	return r;
+}
+#define UALINK_IDBITS_STORE(prefix, name, field, nbits)			\
+static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
+						struct kobj_attribute *attr,\
+						const char *buf, size_t count)\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+	DECLARE_BITMAP(bits, nbits);					\
+	ssize_t r = store_idbits(bits, nbits, buf, count);		\
+									\
+	if (r >= 0)							\
+		bitmap_copy(info->field, bits, nbits);			\
+	return r;							\
+}
+#endif
+
+static ssize_t show_idarray(const u32 *array, u32 size, char *buf)
+{
+	ssize_t len = 0;
+	unsigned int i;
+
+	for (i = 0; i < size; i++)
+		len += sysfs_emit_at(buf, len, "%u ", array[i]);
+	if (len)
+		len--;
+	return len + sysfs_emit_at(buf, len, "\n");
+}
+#define UALINK_IDARRAY_SHOW(prefix, name, field, size)			\
+static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
+					       struct kobj_attribute *attr,\
+					       char *buf)		\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+									\
+	return show_idarray(info->field, info->size, buf);		\
+}
+
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+static ssize_t store_idarray(u32 *array, u32 *size, u32 max_id, u32 max_size,
+			     const char *buf, size_t count)
+{
+	char *dup_buf, *str, *tok;
+	unsigned int index = 0;
+	ssize_t r;
+	u32 id;
+
+	*size = 0;
+	if (!count)
+		return 0;
+
+	dup_buf = kstrndup(buf, count, GFP_KERNEL);
+	if (unlikely(!dup_buf))
+		return -ENOMEM;
+	str = dup_buf;
+	do {
+		str += strspn(str, " ");
+		tok = strsep(&str, ", ");
+		r = kstrtou32(tok, 10, &id);
+		if (r < 0)
+			goto err;
+		if (index >= max_size) {
+			r = -ENOSPC;
+			goto err;
+		}
+		if (id >= max_id) {
+			r = -ERANGE;
+			goto err;
+		}
+		array[index++] = id;
+	} while (str);
+	kfree(dup_buf);
+
+	*size = index;
+	return count;
+
+err:
+	kfree(dup_buf);
+	return r;
+}
+#define UALINK_IDARRAY_STORE(prefix, name, field, size, max_id)		\
+static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
+						struct kobj_attribute *attr,\
+						const char *buf, size_t count)\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+	u32 new_array[ARRAY_SIZE(info->field)];				\
+	u32 new_size;							\
+	ssize_t r = store_idarray(new_array, &new_size, max_id,		\
+				  ARRAY_SIZE(new_array), buf, count);	\
+									\
+	if (r >= 0) {							\
+		memcpy(info->field, new_array, sizeof(new_array));	\
+		info->size = new_size;					\
+	}								\
+	return r;							\
+}
+#endif
+
+#define UALINK_UUID_SHOW(prefix, name, field)				\
+static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
+					       struct kobj_attribute *attr,\
+					       char *buf)		\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+									\
+	return sysfs_emit(buf, "%pU\n", &info->field);			\
+}
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+#define UALINK_UUID_STORE(prefix, name, field)				\
+static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
+						struct kobj_attribute *attr,\
+						const char *buf, size_t count)\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+	int r;								\
+									\
+	r = uuid_parse(buf, &info->field);				\
+	return r ? r : count;						\
+}
+#endif
+
+static ssize_t show_enum(u32 x, const char * const values[], unsigned int n,
+			 char *buf)
+{
+	return sysfs_emit(buf, "%s\n", x < n && values[x] ? values[x] : "invalid");
+}
+#define UALINK_ENUM_SHOW(prefix, name, field)				\
+static ssize_t ualink_##prefix##_##name##_show(struct kobject *kobj,	\
+					       struct kobj_attribute *attr,\
+					       char *buf)		\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+									\
+	return show_enum(info->field, ualink_##name##_values,		\
+			 ARRAY_SIZE(ualink_##name##_values), buf);	\
+}
+
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+static ssize_t store_enum(u32 *x, const char * const values[], unsigned int n,
+			  const char *buf, size_t count)
+{
+	unsigned int i;
+
+	for (i = 0; i < n; i++) {
+		if (values[i] && sysfs_streq(buf, values[i])) {
+			*x = i;
+			return count;
+		}
+	}
+	return -EINVAL;
+}
+#define UALINK_ENUM_STORE(prefix, name, field)				\
+static ssize_t ualink_##prefix##_##name##_store(struct kobject *kobj,	\
+						struct kobj_attribute *attr,\
+						const char *buf, size_t count)\
+{									\
+	struct amdgpu_ualink_##prefix *info = to_ualink_##prefix(kobj);	\
+									\
+	return store_enum(&info->field, ualink_##name##_values,		\
+			 ARRAY_SIZE(ualink_##name##_values),		\
+			 buf, count);					\
+}
+#endif
+static const char * const ualink_link_type_values[] = {
+	"UALoE", "UALink"
+};
+static const char * const ualink_addr_mode_values[] = {
+	"source-aliasing", "source-identification"
+};
+static const char * const ualink_accel_state_values[] = {
+	[AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED]	= "unconfigured",
+	[AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED]	= "ppod_configured",
+	[AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED]	= "vpod_configured",
+	[AMDGPU_UALINK_ACCEL_STATE_READY]		= "ready",
+	[AMDGPU_UALINK_ACCEL_STATE_ACTIVE]		= "active",
+	[AMDGPU_UALINK_ACCEL_STATE_ERROR]		= "error",
+};
+static_assert(ARRAY_SIZE(ualink_accel_state_values) ==
+	      AMDGPU_UALINK_ACCEL_STATE_MAX);
+
+UALINK_ENUM_SHOW(info, link_type, link_type);
+UALINK_VALUE_SHOW(info, accel_id,  ppod.accel_id,  "%u");
+UALINK_VALUE_SHOW(info, bandwidth, ppod.bandwidth, "%u");
+UALINK_VALUE_SHOW(info, latency,   ppod.latency,   "%u");
+UALINK_UUID_SHOW(info, ppod_id, ppod.id);
+UALINK_VALUE_SHOW(info, ppod_size, ppod.size,      "%u");
+UALINK_VALUE_SHOW(info, vpod_id,   vpod.id,        "%u");
+UALINK_VALUE_SHOW(info, vpod_size, vpod.size,      "%u");
+UALINK_IDBITS_SHOW(info, vpod_active_accels, vpod.active_accel_bits);
+UALINK_ENUM_SHOW(info, addr_mode, vpod.addr_mode);
+UALINK_ENUM_SHOW(info, accel_state, accel_state);
+UALINK_IDARRAY_SHOW(info, local_accels, local_accels, n_local_accels);
+
+static struct amdgpu_device *ualink_xcp_kobj_to_adev(struct kobject *kobj)
+{
+	struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp,
+					      ualink.kobj);
+
+	return xcp->xcp_mgr->adev;
+}
+
+#define UALINK_XCP_INFO_SHOW(name)					\
+static ssize_t ualink_xcp_info_##name##_show(struct kobject *kobj,	\
+		struct kobj_attribute *attr, char *buf)			\
+{									\
+	struct amdgpu_device *adev = ualink_xcp_kobj_to_adev(kobj);	\
+									\
+	return ualink_info_##name##_show(&adev->ualink.info->kobj,	\
+					 attr, buf);			\
+}
+
+UALINK_XCP_INFO_SHOW(link_type)
+UALINK_XCP_INFO_SHOW(accel_id)
+UALINK_XCP_INFO_SHOW(bandwidth)
+UALINK_XCP_INFO_SHOW(latency)
+UALINK_XCP_INFO_SHOW(ppod_id)
+UALINK_XCP_INFO_SHOW(ppod_size)
+UALINK_XCP_INFO_SHOW(vpod_id)
+UALINK_XCP_INFO_SHOW(vpod_size)
+UALINK_XCP_INFO_SHOW(vpod_active_accels)
+UALINK_XCP_INFO_SHOW(addr_mode)
+UALINK_XCP_INFO_SHOW(accel_state)
+UALINK_XCP_INFO_SHOW(local_accels)
+
+#define UALINK_INFO_ATTR(name) __ATTR(name, 0444, ualink_info_##name##_show, NULL)
+static struct kobj_attribute ualink_info_link_type = UALINK_INFO_ATTR(link_type);
+static struct kobj_attribute ualink_info_accel_id  = UALINK_INFO_ATTR(accel_id);
+static struct kobj_attribute ualink_info_bandwidth = UALINK_INFO_ATTR(bandwidth);
+static struct kobj_attribute ualink_info_latency   = UALINK_INFO_ATTR(latency);
+static struct kobj_attribute ualink_info_ppod_id   = UALINK_INFO_ATTR(ppod_id);
+static struct kobj_attribute ualink_info_ppod_size = UALINK_INFO_ATTR(ppod_size);
+static struct kobj_attribute ualink_info_vpod_id   = UALINK_INFO_ATTR(vpod_id);
+static struct kobj_attribute ualink_info_vpod_size = UALINK_INFO_ATTR(vpod_size);
+static struct kobj_attribute ualink_info_vpod_active_accels = UALINK_INFO_ATTR(vpod_active_accels);
+static struct kobj_attribute ualink_info_addr_mode = UALINK_INFO_ATTR(addr_mode);
+static struct kobj_attribute ualink_info_accel_state = UALINK_INFO_ATTR(accel_state);
+static struct kobj_attribute ualink_info_local_accels = UALINK_INFO_ATTR(local_accels);
+
+static const struct attribute *ualink_info_attrs[] = {
+	&ualink_info_link_type.attr,
+	&ualink_info_accel_id.attr,
+	&ualink_info_bandwidth.attr,
+	&ualink_info_latency.attr,
+	&ualink_info_ppod_id.attr,
+	&ualink_info_ppod_size.attr,
+	&ualink_info_vpod_id.attr,
+	&ualink_info_vpod_size.attr,
+	&ualink_info_vpod_active_accels.attr,
+	&ualink_info_addr_mode.attr,
+	&ualink_info_accel_state.attr,
+	&ualink_info_local_accels.attr,
+	NULL
+};
+
+#define UALINK_XCP_INFO_ATTR(name) \
+	__ATTR(name, 0444, ualink_xcp_info_##name##_show, NULL)
+static struct kobj_attribute ualink_xcp_info_link_type = UALINK_XCP_INFO_ATTR(link_type);
+static struct kobj_attribute ualink_xcp_info_accel_id  = UALINK_XCP_INFO_ATTR(accel_id);
+static struct kobj_attribute ualink_xcp_info_bandwidth = UALINK_XCP_INFO_ATTR(bandwidth);
+static struct kobj_attribute ualink_xcp_info_latency   = UALINK_XCP_INFO_ATTR(latency);
+static struct kobj_attribute ualink_xcp_info_ppod_id   = UALINK_XCP_INFO_ATTR(ppod_id);
+static struct kobj_attribute ualink_xcp_info_ppod_size = UALINK_XCP_INFO_ATTR(ppod_size);
+static struct kobj_attribute ualink_xcp_info_vpod_id   = UALINK_XCP_INFO_ATTR(vpod_id);
+static struct kobj_attribute ualink_xcp_info_vpod_size = UALINK_XCP_INFO_ATTR(vpod_size);
+static struct kobj_attribute ualink_xcp_info_vpod_active_accels = UALINK_XCP_INFO_ATTR(vpod_active_accels);
+static struct kobj_attribute ualink_xcp_info_addr_mode = UALINK_XCP_INFO_ATTR(addr_mode);
+static struct kobj_attribute ualink_xcp_info_accel_state = UALINK_XCP_INFO_ATTR(accel_state);
+static struct kobj_attribute ualink_xcp_info_local_accels = UALINK_XCP_INFO_ATTR(local_accels);
+
+static struct attribute *ualink_xcp_info_attrs[] = {
+	&ualink_xcp_info_link_type.attr,
+	&ualink_xcp_info_accel_id.attr,
+	&ualink_xcp_info_bandwidth.attr,
+	&ualink_xcp_info_latency.attr,
+	&ualink_xcp_info_ppod_id.attr,
+	&ualink_xcp_info_ppod_size.attr,
+	&ualink_xcp_info_vpod_id.attr,
+	&ualink_xcp_info_vpod_size.attr,
+	&ualink_xcp_info_vpod_active_accels.attr,
+	&ualink_xcp_info_addr_mode.attr,
+	&ualink_xcp_info_accel_state.attr,
+	&ualink_xcp_info_local_accels.attr,
+	NULL
+};
+
+static umode_t ualink_xcp_info_is_visible(struct kobject *kobj,
+					  struct attribute *attr, int n)
+{
+	struct amdgpu_xcp *xcp = container_of(kobj, struct amdgpu_xcp,
+					      ualink.kobj);
+
+	if (!xcp->valid)
+		return 0;
+
+	return attr->mode;
+}
+
+static const struct attribute_group ualink_xcp_info_group = {
+	.attrs = ualink_xcp_info_attrs,
+	.is_visible = ualink_xcp_info_is_visible,
+};
+
+static const struct kobj_type ualink_xcp_info_ktype = {
+	.sysfs_ops = &kobj_sysfs_ops
+};
+
+static void ualink_info_release(struct kobject *kobj)
+{
+	kfree(to_ualink_info(kobj));
+}
+
+static const struct kobj_type ualink_info_ktype = {
+	.release = ualink_info_release,
+	.sysfs_ops = &kobj_sysfs_ops
+};
+
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+UALINK_VALUE_SHOW(ppod_setup, accel_id,  ppod.accel_id,  "%u");
+UALINK_VALUE_SHOW(ppod_setup, bandwidth, ppod.bandwidth, "%u");
+UALINK_VALUE_SHOW(ppod_setup, latency,   ppod.latency,   "%u");
+UALINK_UUID_SHOW(ppod_setup, ppod_id, ppod.id);
+UALINK_VALUE_SHOW(ppod_setup, ppod_size, ppod.size,      "%u");
+UALINK_IDARRAY_SHOW(ppod_setup, local_accels, local_accels, n_local_accels);
+
+UALINK_VALUE_STORE(ppod_setup, accel_id,  ppod.accel_id,  u32, 10);
+UALINK_VALUE_STORE(ppod_setup, bandwidth, ppod.bandwidth, u32, 10);
+UALINK_VALUE_STORE(ppod_setup, latency,   ppod.latency,   u32, 10);
+UALINK_UUID_STORE(ppod_setup, ppod_id,   ppod.id);
+UALINK_VALUE_STORE(ppod_setup, ppod_size, ppod.size,      u32, 10);
+UALINK_IDARRAY_STORE(ppod_setup, local_accels, local_accels, n_local_accels,
+		     AMDGPU_UALINK_ACCEL_MAX);
+
+static enum amdgpu_ualink_accel_state
+check_ppod_state(struct amdgpu_device *adev,
+		 const struct amdgpu_ualink_ppod_setup *setup)
+{
+	if (setup->ppod.size <= 0 || setup->ppod.size > AMDGPU_UALINK_ACCEL_MAX) {
+		dev_dbg(adev->dev, "pPod size %u out of range [1..%u]\n",
+			setup->ppod.size, AMDGPU_UALINK_ACCEL_MAX);
+		return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
+	}
+	if (setup->ppod.accel_id >= setup->ppod.size) {
+		dev_dbg(adev->dev, "Accelerator ID %u greater or equal pPod size %u\n",
+			setup->ppod.accel_id, setup->ppod.size);
+		return AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED;
+	}
+	return AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
+}
+
+static ssize_t ualink_ppod_setup_commit_store(struct kobject *kobj,
+					      struct kobj_attribute *attr,
+					      const char *buf, size_t count)
+{
+	struct amdgpu_ualink_ppod_setup *setup = to_ualink_ppod_setup(kobj);
+	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
+	struct device *dev = kobj_to_dev(info->kobj.parent);
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	int r;
+
+	if (!sysfs_streq(buf, "true"))
+		return -EINVAL;
+
+	r = psp_ual_set_ppod_config(&adev->psp, adev->ualink.psp_if_ver,
+				    setup);
+	if (r)
+		return r;
+	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL);
+	if (r)
+		return r;
+
+	/*
+	 * Hold mgpu_info.mutex to serialize with activate_local_vpod() in
+	 * vpod_config_commit_store which also holds this lock. Without it,
+	 * deactivate_accelerator() can race with activate_accelerator(),
+	 * causing concurrent vm_fini / vm_init on the same NPA VM.
+	 */
+	mutex_lock(&mgpu_info.mutex);
+	if (info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_READY)
+		deactivate_accelerator(adev);
+	info->accel_state = check_ppod_state(adev, setup);
+	/* PPOD is expected to be configured first */
+	info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID;
+	mutex_unlock(&mgpu_info.mutex);
+
+	/* TODO: If accel_state was ACTIVE, reset all connections */
+
+	return count;
+}
+
+#define UALINK_PPOD_SETUP_ATTR(name) __ATTR(name, 0600,			\
+		ualink_ppod_setup_##name##_show,			\
+		ualink_ppod_setup_##name##_store)
+static struct kobj_attribute ualink_ppod_setup_accel_id  = UALINK_PPOD_SETUP_ATTR(accel_id);
+static struct kobj_attribute ualink_ppod_setup_bandwidth = UALINK_PPOD_SETUP_ATTR(bandwidth);
+static struct kobj_attribute ualink_ppod_setup_latency   = UALINK_PPOD_SETUP_ATTR(latency);
+static struct kobj_attribute ualink_ppod_setup_ppod_id   = UALINK_PPOD_SETUP_ATTR(ppod_id);
+static struct kobj_attribute ualink_ppod_setup_ppod_size = UALINK_PPOD_SETUP_ATTR(ppod_size);
+static struct kobj_attribute ualink_ppod_setup_local_accels = UALINK_PPOD_SETUP_ATTR(local_accels);
+static struct kobj_attribute ualink_ppod_setup_commit = __ATTR(commit, 0200, NULL,
+							       ualink_ppod_setup_commit_store);
+
+static const struct attribute *ualink_ppod_setup_attrs[] = {
+	&ualink_ppod_setup_accel_id.attr,
+	&ualink_ppod_setup_bandwidth.attr,
+	&ualink_ppod_setup_latency.attr,
+	&ualink_ppod_setup_ppod_id.attr,
+	&ualink_ppod_setup_ppod_size.attr,
+	&ualink_ppod_setup_local_accels.attr,
+	&ualink_ppod_setup_commit.attr,
+	NULL
+};
+#endif
+
+static void ualink_ppod_setup_release(struct kobject *kobj)
+{
+	kfree(to_ualink_ppod_setup(kobj));
+}
+
+static const struct kobj_type ualink_ppod_setup_ktype = {
+	.release = ualink_ppod_setup_release,
+	.sysfs_ops = &kobj_sysfs_ops
+};
+
+static struct amdgpu_device *find_peer_adev(unsigned int accel_id)
+{
+	unsigned int i;
+
+	for (i = 0; i < mgpu_info.num_gpu; i++) {
+		struct amdgpu_device *peer_adev = mgpu_info.gpu_ins[i].adev;
+
+		if (peer_adev->ualink.info &&
+		    peer_adev->ualink.info->ppod.accel_id == accel_id)
+			return peer_adev;
+	}
+
+	return NULL;
+}
+
+static bool amdgpu_ualink_is_local_accel(struct amdgpu_device *adev,
+					 u32 accel_id)
+{
+	struct amdgpu_ualink_info *info = adev->ualink.info;
+	unsigned int i;
+
+	if (!info)
+		return false;
+
+	for (i = 0; i < info->n_local_accels; i++) {
+		if (info->local_accels[i] == accel_id)
+			return true;
+	}
+	return false;
+}
+
+static void activate_accelerator(struct amdgpu_device *adev)
+{
+	int r;
+
+	if (adev->ualink.info->accel_state >= AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
+		return;
+
+	dev_info(adev->dev,
+		 "UALINK: activating accelerator accel_id=%u (accel_state=%d)\n",
+		 adev->ualink.info->ppod.accel_id,
+		 adev->ualink.info->accel_state);
+
+	/* Enable incoming NPA address translation with NPA VMID */
+	r = psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver,
+				   adev->vm_manager.npa_vmid, true);
+	if (r) {
+		dev_err(adev->dev, "Failed to set NPA config\n");
+		return;
+	}
+
+	r = amdgpu_ualink_manager_start(adev);
+	if (r) {
+		dev_err(adev->dev, "Failed to start UALink manager\n");
+		return;
+	}
+
+	r = amdgpu_ualink_sw_init(adev);
+	if (r) {
+		dev_err(adev->dev, "Failed to init UALink sw r=%d\n", r);
+		amdgpu_ualink_manager_stop(adev);
+		return;
+	}
+
+	adev->ualink.info->accel_state = AMDGPU_UALINK_ACCEL_STATE_ACTIVE;
+	dev_info(adev->dev,
+		 "UALINK: accelerator accel_id=%u is now ACTIVE\n",
+		 adev->ualink.info->ppod.accel_id);
+}
+
+static void deactivate_accelerator(struct amdgpu_device *adev)
+{
+	if (adev->ualink.info->accel_state < AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
+		return;
+
+	dev_info(adev->dev,
+		 "UALINK: deactivating accelerator accel_id=%u (accel_state=%d -> PPOD_CONFIGURED)\n",
+		 adev->ualink.info->ppod.accel_id,
+		 adev->ualink.info->accel_state);
+
+	/* Disable incoming NPA address translation with NPA VMID */
+	psp_ual_set_npa_config(&adev->psp, adev->ualink.psp_if_ver,
+			       adev->vm_manager.npa_vmid, false);
+	/* ignore return value */
+	adev->ualink.info->accel_state =
+		AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED;
+
+	amdgpu_ualink_sw_fini(adev);
+	amdgpu_ualink_manager_stop(adev);
+}
+
+static void activate_local_vpod(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_info *info = adev->ualink.info;
+	struct amdgpu_device *peer_adev;
+	unsigned int accel_id;
+	unsigned int i;
+
+	for (i = 0; i < info->n_local_accels; i++) {
+		accel_id = info->local_accels[i];
+
+		peer_adev = find_peer_adev(accel_id);
+		if (WARN_ON(!peer_adev || !peer_adev->ualink.info))
+			/* info->local_accels is corrupted? */
+			continue;
+
+		/* Bring the peer up to match the current vPod membership.
+		 *
+		 * A peer that is not yet ACTIVE (or whose ACTIVE membership
+		 * changed on a grow/shrink) has its remote metadata, links and
+		 * GART mappings built for a stale member set (or none at all).
+		 * Firmware only accepts a full metadata reload while halted, so
+		 * surgical per-peer deltas are not possible: fully bounce the
+		 * accelerator. deactivate_accelerator() is a no-op when the peer
+		 * is not ACTIVE, so this handles first-time bring-up too.
+		 * Unchanged ACTIVE peers are left untouched.
+		 */
+		if (amdgpu_ualink_vpod_membership_changed(peer_adev)) {
+			dev_info(peer_adev->dev,
+				 "UALINK: (re)configuring vpod for accel_id=%u\n",
+				 peer_adev->ualink.info->ppod.accel_id);
+			deactivate_accelerator(peer_adev);
+			activate_accelerator(peer_adev);
+		}
+	}
+}
+
+static inline bool __is_vpod_peer(struct amdgpu_ualink_info *info,
+				  struct amdgpu_ualink_info *peer_info)
+{
+	return peer_info->vpod.id == info->vpod.id &&
+	       uuid_equal(&peer_info->ppod.id, &info->ppod.id);
+}
+
+static int __check_local_vpod_integrity(struct amdgpu_device *adev)
+{
+	DECLARE_BITMAP(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX);
+	struct amdgpu_ualink_info *info = adev->ualink.info;
+	u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX];
+	struct amdgpu_ualink_info *peer_info;
+	struct amdgpu_device *peer_adev;
+	unsigned int i, n_local_accels;
+	unsigned int accel_id;
+	/* Check that all local accelerators listed in vpod_active_accels have
+	 * matching pod IDs
+	 */
+	bitmap_zero(local_accel_ids, AMDGPU_UALINK_ACCEL_MAX);
+	n_local_accels = 0;
+	__set_bit(info->ppod.accel_id, local_accel_ids);
+	local_accels[n_local_accels++] = info->ppod.accel_id;
+
+	for (i = 0; i < mgpu_info.num_gpu; i++) {
+		peer_adev = mgpu_info.gpu_ins[i].adev;
+		if (peer_adev == adev || !peer_adev->ualink.info)
+			continue;
+
+		peer_info = peer_adev->ualink.info;
+		/* peer device ppod not configured */
+		if (peer_info->accel_state <
+		    AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
+			dev_info(adev->dev,
+				 "UALINK: integrity EAGAIN: peer accel_id=%u not ppod-configured (accel_state=%d)\n",
+				 peer_info->ppod.accel_id,
+				 peer_info->accel_state);
+			return -EAGAIN;
+		}
+
+		accel_id = peer_info->ppod.accel_id;
+		if (!test_bit(accel_id, info->vpod.active_accel_bits))
+			continue;
+		/* peer device vpod not configured */
+		if (peer_info->accel_state <
+		    AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED) {
+			dev_info(adev->dev,
+				 "UALINK: integrity EAGAIN: vpod peer accel_id=%u not vpod-configured (accel_state=%d)\n",
+				 accel_id, peer_info->accel_state);
+			return -EAGAIN;
+		}
+
+		if (!uuid_equal(&peer_info->ppod.id, &info->ppod.id)) {
+			dev_warn(adev->dev,
+				 "UALINK: integrity fail: peer %u ppod_id doesn't match: %pU != %pU\n",
+				 peer_info->ppod.accel_id, &peer_info->ppod.id,
+				 &info->ppod.id);
+			return -EINVAL;
+		}
+
+		if (peer_info->ppod.size != info->ppod.size) {
+			dev_warn(adev->dev,
+				 "UALINK: integrity fail: peer %u ppod_size doesn't match: %u != %u\n",
+				 accel_id, peer_info->ppod.size,
+				 info->ppod.size);
+			return -EINVAL;
+		}
+
+		if (peer_info->vpod.id != info->vpod.id) {
+			dev_warn(adev->dev,
+				 "UALINK: integrity fail: peer %u vpod_id doesn't match: %u != %u\n",
+				 accel_id, peer_info->vpod.id, info->vpod.id);
+			return -EINVAL;
+		}
+		if (peer_info->vpod.size != info->vpod.size) {
+			dev_warn(adev->dev,
+				 "UALINK: integrity fail: peer %u vpod_size doesn't match: %u != %u\n",
+				 accel_id, peer_info->vpod.size,
+				 info->vpod.size);
+			return -EINVAL;
+		}
+		if (peer_info->vpod.addr_mode != info->vpod.addr_mode) {
+			dev_warn(adev->dev,
+				 "UALINK: integrity fail: peer %u addr_mode doesn't match: %u != %u\n",
+				 accel_id, peer_info->vpod.addr_mode,
+				 info->vpod.addr_mode);
+			return -EINVAL;
+		}
+		if (!bitmap_equal(peer_info->vpod.active_accel_bits,
+				  info->vpod.active_accel_bits,
+				  AMDGPU_UALINK_ACCEL_MAX)) {
+			dev_warn(adev->dev,
+				 "UALINK: integrity fail: peer %u vpod_active_accels don't match\n",
+				 accel_id);
+			return -EINVAL;
+		}
+
+		if (__test_and_set_bit(accel_id, local_accel_ids)) {
+			dev_warn(adev->dev,
+				 "UALINK: integrity fail: duplicate accel_id %u among local vpod peers\n",
+				 accel_id);
+			return -EINVAL;
+		}
+		local_accels[n_local_accels++] = accel_id;
+	}
+
+	for (i = 0; i < mgpu_info.num_gpu; i++) {
+		peer_adev = mgpu_info.gpu_ins[i].adev;
+		peer_info = peer_adev->ualink.info;
+
+		if (!peer_info)
+			continue;
+		if (peer_adev != adev && !__is_vpod_peer(info, peer_info))
+			continue;
+
+		peer_info->n_local_accels = n_local_accels;
+		memcpy(peer_info->local_accels, local_accels,
+		       sizeof(local_accels));
+	}
+
+	return 0;
+}
+
+static void __amdgpu_ualink_activate_vpod_locked(struct amdgpu_device *adev)
+{
+	int ret;
+
+	if (adev->ualink.info->accel_state <
+		    AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED ||
+	    adev->ualink.info->accel_state ==
+		    AMDGPU_UALINK_ACCEL_STATE_ERROR)
+		return;
+
+	ret = __check_local_vpod_integrity(adev);
+	if (ret && ret != -EAGAIN) {
+		dev_err(adev->dev,
+			"Local vpod integrity check failed: %d\n", ret);
+		return;
+	}
+	if (ret == -EAGAIN) {
+		dev_info(adev->dev,
+			 "UALINK: activate deferred, waiting for local vpod peers to reach VPOD_CONFIGURED\n");
+		return;
+	}
+
+	dev_info(adev->dev,
+		 "UALINK: integrity OK, applying local vpod (%u local accels)\n",
+		 adev->ualink.info->n_local_accels);
+	activate_local_vpod(adev);
+}
+
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+UALINK_VALUE_SHOW(vpod_config, vpod_id,   vpod.id,   "%u");
+UALINK_VALUE_SHOW(vpod_config, vpod_size, vpod.size, "%u");
+UALINK_IDBITS_SHOW(vpod_config, vpod_active_accels, vpod.active_accel_bits);
+UALINK_ENUM_SHOW(vpod_config, addr_mode, vpod.addr_mode);
+
+UALINK_VALUE_STORE(vpod_config, vpod_id,   vpod.id,   u32, 10);
+UALINK_VALUE_STORE(vpod_config, vpod_size, vpod.size, u32, 10);
+UALINK_IDBITS_STORE(vpod_config, vpod_active_accels, vpod.active_accel_bits,
+		    AMDGPU_UALINK_ACCEL_MAX);
+UALINK_ENUM_STORE(vpod_config, addr_mode, vpod.addr_mode);
+
+static ssize_t ualink_vpod_config_commit_store(struct kobject *kobj,
+					       struct kobj_attribute *attr,
+					       const char *buf, size_t count)
+{
+	struct amdgpu_ualink_vpod_config *config = to_ualink_vpod_config(kobj);
+	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
+	struct device *dev = kobj_to_dev(info->kobj.parent);
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	enum amdgpu_ualink_accel_state prev_state;
+	u32 prev_vpod_id;
+	int r;
+
+	if (!sysfs_streq(buf, "true"))
+		return -EINVAL;
+
+	dev_dbg(adev->dev,
+		"UALINK: vpod-commit enter: accel_state=%d staged vpod_id=%u vpod_size=%u addr_mode=%u\n",
+		info->accel_state, config->vpod.id, config->vpod.size,
+		config->vpod.addr_mode);
+
+	if (info->accel_state < AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED) {
+		dev_warn(adev->dev,
+			 "UALINK: vpod-commit rejected, ppod not configured (accel_state=%d)\n",
+			 info->accel_state);
+		return -EINVAL;
+	}
+
+	prev_state = info->accel_state;
+	prev_vpod_id = info->vpod.id;
+	r = psp_ual_set_vpod_config(&adev->psp, adev->ualink.psp_if_ver,
+				    config);
+	if (r) {
+		dev_warn(adev->dev,
+			 "UALINK: vpod-commit psp_ual_set_vpod_config failed: %d\n",
+			 r);
+		return r;
+	}
+
+	r = psp_ual_query_info(&adev->psp, adev->ualink.psp_if_ver, info, NULL);
+	if (r) {
+		dev_warn(adev->dev,
+			 "UALINK: vpod-commit psp_ual_query_info failed: %d\n",
+			 r);
+		return r;
+	}
+
+	dev_dbg(adev->dev,
+		"UALINK: vpod-commit fw read-back: vpod_id=%u vpod_size=%u accel_id=%u ppod.size=%u addr_mode=%u\n",
+		info->vpod.id, info->vpod.size, info->ppod.accel_id,
+		info->ppod.size, info->vpod.addr_mode);
+
+	if (info->vpod.id != AMDGPU_UALINK_VPOD_ID_INVALID &&
+	    !__check_vpod_info(adev, info)) {
+		dev_err(adev->dev,
+			"UALINK: vpod-commit __check_vpod_info() failed: %d\n",
+			r);
+		return -EINVAL;
+	}
+
+	/* The integrity check makes sure each new GPU is consistent with the
+	 * other GPUs already in the vPod. All known local GPUs can become
+	 * "ready" at the same time.
+	 *
+	 * Misconfiguration of one GPU does not reduce the state of other GPUs
+	 * already in the vPod. GPU is intentionally not put to ERROR state if
+	 * misconfiguration occurs.
+	 */
+	r = amdgpu_ualink_update_accel_state(adev, prev_state, prev_vpod_id,
+					     UAL_CFG_VPOD);
+	if (r)
+		return r;
+
+	return count;
+}
+
+#define UALINK_VPOD_CONFIG_ATTR(name) __ATTR(name, 0600,		\
+		ualink_vpod_config_##name##_show,			\
+		ualink_vpod_config_##name##_store)
+static struct kobj_attribute ualink_vpod_config_vpod_id   = UALINK_VPOD_CONFIG_ATTR(vpod_id);
+static struct kobj_attribute ualink_vpod_config_vpod_size = UALINK_VPOD_CONFIG_ATTR(vpod_size);
+static struct kobj_attribute ualink_vpod_config_vpod_active_accels =
+						UALINK_VPOD_CONFIG_ATTR(vpod_active_accels);
+static struct kobj_attribute ualink_vpod_config_addr_mode = UALINK_VPOD_CONFIG_ATTR(addr_mode);
+static struct kobj_attribute ualink_vpod_config_commit = __ATTR(commit, 0200, NULL,
+								ualink_vpod_config_commit_store);
+
+static const struct attribute *ualink_vpod_config_attrs[] = {
+	&ualink_vpod_config_vpod_id.attr,
+	&ualink_vpod_config_vpod_size.attr,
+	&ualink_vpod_config_vpod_active_accels.attr,
+	&ualink_vpod_config_addr_mode.attr,
+	&ualink_vpod_config_commit.attr,
+	NULL
+};
+#endif
+
+static void ualink_vpod_config_release(struct kobject *kobj)
+{
+	kfree(to_ualink_vpod_config(kobj));
+}
+
+static const struct kobj_type ualink_vpod_config_ktype = {
+	.release = ualink_vpod_config_release,
+	.sysfs_ops = &kobj_sysfs_ops
+};
+
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+UALINK_VALUE_SHOW(station_config, flags, flags, "0x%x");
+
+UALINK_VALUE_STORE(station_config, flags, flags, u8, 16);
+
+static size_t lane_bitmap_to_string(char *str,
+		const struct amdgpu_ualink_station_config *stations)
+{
+	size_t i;
+
+	for (i = 0; i < stations->n_stations; i++)
+		str[i] = stations->lane_en_bitmap[i] +
+			(stations->lane_en_bitmap[i] < 10 ? '0' : 'a' - 10);
+	str[i] = '\0';
+
+	return i;
+}
+static ssize_t
+ualink_station_config_lane_en_bitmap_show(struct kobject *kobj,
+					  struct kobj_attribute *attr,
+					  char *buf)
+{
+	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
+	char str[AMDGPU_UALINK_STATIONS_MAX + 1];
+
+	lane_bitmap_to_string(str, stations);
+	return sysfs_emit(buf, "%s\n", str);
+}
+static ssize_t
+ualink_station_config_lane_en_bitmap_store(struct kobject *kobj,
+					   struct kobj_attribute *attr,
+					   const char *buf, size_t count)
+{
+	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
+	u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX];
+	u32 n_stations = 0;
+	bool end = false;
+	size_t i;
+
+	for (i = 0; i < count; i++) {
+		/* Accept any number of \n in the end */
+		if (buf[i] == '\n') {
+			end = true;
+			continue;
+		}
+		/* Don't accept any more data after \n */
+		if (end)
+			return -EINVAL;
+		/* Don't accept more data than the array size */
+		if (i >= sizeof(lane_en_bitmap))
+			return -ENOSPC;
+		/* Accept hex digits */
+		if (buf[i] >= '0' && buf[i] <= '9')
+			lane_en_bitmap[n_stations++] = buf[i] - '0';
+		else if (buf[i] >= 'a' && buf[i] <= 'f')
+			lane_en_bitmap[n_stations++] = buf[i] + 10 - 'a';
+		else if (buf[i] >= 'A' && buf[i] <= 'F')
+			lane_en_bitmap[n_stations++] = buf[i] + 10 - 'A';
+		else
+			return -EINVAL;
+	}
+
+	memcpy(stations->lane_en_bitmap, lane_en_bitmap, sizeof(lane_en_bitmap));
+	stations->n_stations = n_stations;
+	return i;
+}
+
+static ssize_t ualink_station_config_commit_store(struct kobject *kobj,
+						  struct kobj_attribute *attr,
+						  const char *buf, size_t count)
+{
+	struct amdgpu_ualink_station_config *stations = to_ualink_station_config(kobj);
+	struct amdgpu_ualink_info *info = to_ualink_info(kobj->parent);
+	struct device *dev = kobj_to_dev(info->kobj.parent);
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	int r;
+
+	/* DF reconfiguration does not interact with accelerator state */
+
+	if (!sysfs_streq(buf, "true"))
+		return -EINVAL;
+
+	r = psp_ual_set_station_config(&adev->psp, adev->ualink.psp_if_ver, stations);
+	if (r)
+		return r;
+
+	return count;
+}
+
+#define UALINK_STATION_CONFIG_ATTR(name) __ATTR(name, 0600,		\
+		ualink_station_config_##name##_show,			\
+		ualink_station_config_##name##_store)
+static struct kobj_attribute ualink_station_config_flags  = UALINK_STATION_CONFIG_ATTR(flags);
+static struct kobj_attribute ualink_station_config_lane_en_bitmap =
+							UALINK_STATION_CONFIG_ATTR(lane_en_bitmap);
+static struct kobj_attribute ualink_station_config_commit =
+				__ATTR(commit, 0200, NULL, ualink_station_config_commit_store);
+
+static const struct attribute *ualink_station_config_attrs[] = {
+	&ualink_station_config_flags.attr,
+	&ualink_station_config_lane_en_bitmap.attr,
+	&ualink_station_config_commit.attr,
+	NULL
+};
+#endif
+
+static void ualink_station_config_release(struct kobject *kobj)
+{
+	kfree(to_ualink_station_config(kobj));
+}
+
+static const struct kobj_type ualink_station_config_ktype = {
+	.release = ualink_station_config_release,
+	.sysfs_ops = &kobj_sysfs_ops
+};
+
+int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_ualink_station_config *stations;
+	struct amdgpu_ualink_vpod_config *vpod_config;
+	struct amdgpu_ualink_ppod_setup *ppod_setup;
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_ualink_info *info;
+
+	info = kzalloc(sizeof(*info), GFP_KERNEL);
+	ppod_setup = kzalloc(sizeof(*ppod_setup), GFP_KERNEL);
+	vpod_config = kzalloc(sizeof(*vpod_config), GFP_KERNEL);
+	stations = kzalloc(sizeof(*stations), GFP_KERNEL);
+	if (!info || !ppod_setup || !vpod_config || !stations) {
+		kfree(info);
+		kfree(ppod_setup);
+		kfree(vpod_config);
+		kfree(stations);
+		return -ENOMEM;
+	}
+
+	info->ppod.accel_id = 0xffffffff;
+	info->ppod.bandwidth = 0xffffffff;
+	info->ppod.latency = 0xffffffff;
+	info->vpod.id = AMDGPU_UALINK_VPOD_ID_INVALID;
+	info->vpod.addr_mode = AMDGPU_UALINK_ADDR_MODE_MAX;
+
+	/*
+	 * Initialize the kobjects here so their lifetime is tied to the UALink
+	 * manager software state. amdgpu_ualink_object_fini() drops the final
+	 * reference via kobject_put().
+	 */
+	kobject_init(&info->kobj, &ualink_info_ktype);
+	kobject_init(&ppod_setup->kobj, &ualink_ppod_setup_ktype);
+	kobject_init(&vpod_config->kobj, &ualink_vpod_config_ktype);
+	kobject_init(&stations->kobj, &ualink_station_config_ktype);
+
+	adev->ualink.info = info;
+	adev->ualink.setup = ppod_setup;
+	adev->ualink.config = vpod_config;
+	adev->ualink.stations = stations;
+
+	return 0;
+}
+
+int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block)
+{
+	amdgpu_ualink_object_fini(ip_block->adev);
+	return 0;
+}
+
+static int ualink_kobj_add(struct kobject *kobj, struct kobject *parent,
+			   const char *name, const struct attribute **attrs)
+{
+	int r;
+
+	r = kobject_add(kobj, parent, "%s", name);
+	if (r)
+		return r;
+	r = sysfs_create_files(kobj, attrs);
+	if (r)
+		kobject_del(kobj);
+
+	return r;
+}
+
+static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev);
+static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev);
+
+int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_info *info = adev->ualink.info;
+	int r;
+
+	if (adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)
+		return 0;
+
+	/* ualink parent node */
+	r = ualink_kobj_add(&info->kobj, &adev->dev->kobj, "ualink",
+			    ualink_info_attrs);
+	if (r)
+		goto err;
+
+#ifdef UALINK_ENABLE_DEPRECATED_CONFIG_SYSFS
+	r = ualink_kobj_add(&adev->ualink.setup->kobj, &info->kobj, "setup",
+			    ualink_ppod_setup_attrs);
+	if (r)
+		goto err_info;
+
+	r = ualink_kobj_add(&adev->ualink.config->kobj, &info->kobj, "config",
+			    ualink_vpod_config_attrs);
+	if (r)
+		goto err_setup;
+
+	r = ualink_kobj_add(&adev->ualink.stations->kobj, &info->kobj,
+			    "stations", ualink_station_config_attrs);
+	if (r)
+		goto err_config;
+#else
+	r = kobject_add(&adev->ualink.setup->kobj, &info->kobj, "%s",
+			"setup");
+	if (r)
+		goto err_info;
+
+	r = kobject_add(&adev->ualink.config->kobj, &info->kobj, "%s",
+			"config");
+	if (r)
+		goto err_setup;
+
+	r = kobject_add(&adev->ualink.stations->kobj, &info->kobj, "%s",
+			"stations");
+	if (r)
+		goto err_config;
+#endif
+
+	amdgpu_ualink_xcp_sysfs_init(adev);
+	adev->ualink.sysfs_init = true;
+
+	return 0;
+
+err_config:
+	kobject_del(&adev->ualink.config->kobj);
+err_setup:
+	kobject_del(&adev->ualink.setup->kobj);
+err_info:
+	kobject_del(&info->kobj);
+err:
+	dev_warn(adev->dev, "Failed to create UALink sysfs: %d\n", r);
+	return 0;
+}
+
+void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev)
+{
+	if (!adev->ualink.sysfs_init)
+		return;
+
+	amdgpu_ualink_xcp_sysfs_fini(adev);
+	kobject_del(&adev->ualink.stations->kobj);
+	kobject_del(&adev->ualink.config->kobj);
+	kobject_del(&adev->ualink.setup->kobj);
+	kobject_del(&adev->ualink.info->kobj);
+	adev->ualink.sysfs_init = false;
+}
+
+static int amdgpu_ualink_xcp_sysfs_add(struct amdgpu_xcp *xcp)
+{
+	struct amdgpu_device *adev = xcp->xcp_mgr->adev;
+	int r;
+
+	r = kobject_init_and_add(&xcp->ualink.kobj, &ualink_xcp_info_ktype,
+				 &xcp->ddev->dev->kobj, "ualink");
+	if (r)
+		goto err;
+
+	r = sysfs_create_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
+	if (r)
+		goto err;
+
+	/* pin info so it outlives this node regardless of teardown order */
+	kobject_get(&adev->ualink.info->kobj);
+	xcp->ualink.sysfs = true;
+	return 0;
+err:
+	kobject_put(&xcp->ualink.kobj);
+	return r;
+}
+
+void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp)
+{
+	if (!xcp->ualink.sysfs)
+		return;
+
+	sysfs_update_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
+}
+
+static void amdgpu_ualink_xcp_sysfs_remove(struct amdgpu_xcp *xcp)
+{
+	struct amdgpu_device *adev = xcp->xcp_mgr->adev;
+
+	if (!xcp->ualink.sysfs)
+		return;
+
+	/* group is only populated for valid partitions */
+	if (xcp->valid)
+		sysfs_remove_group(&xcp->ualink.kobj, &ualink_xcp_info_group);
+	kobject_put(&adev->ualink.info->kobj);
+	kobject_put(&xcp->ualink.kobj);
+	xcp->ualink.sysfs = false;
+}
+
+static void amdgpu_ualink_xcp_sysfs_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_xcp *xcp;
+	int i;
+
+	if (!adev->xcp_mgr)
+		return;
+
+	for (i = 0; i < MAX_XCP; i++) {
+		xcp = &adev->xcp_mgr->xcp[i];
+		if (!xcp->ddev || amdgpu_xcp_is_primary(xcp))
+			continue;
+		amdgpu_ualink_xcp_sysfs_add(xcp);
+	}
+}
+
+static void amdgpu_ualink_xcp_sysfs_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_xcp *xcp;
+	int i;
+
+	if (!adev->xcp_mgr)
+		return;
+
+	for (i = 0; i < MAX_XCP; i++) {
+		xcp = &adev->xcp_mgr->xcp[i];
+		amdgpu_ualink_xcp_sysfs_remove(xcp);
+	}
+}
+
+static int amdgpu_ualink_npa_alloc_va(struct amdgpu_device *adev,
+			       struct drm_mm_node *mm_node,
+			       u64 va, u64 range_start,
+			       u64 range_end, int size)
+{
+	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
+	u64 alignment = 0;
+	int ret;
+
+	mutex_lock(&npa_mm->mm_lock);
+	if (va) {
+		mm_node->start = va;
+		mm_node->size = size;
+		ret = drm_mm_reserve_node(&npa_mm->mm, mm_node);
+	} else {
+		if (!range_start && !range_end) {
+			range_start = npa_mm->va_start;
+			range_end = npa_mm->va_start + npa_mm->va_size;
+		}
+
+		/* If size is greater than 2MB (or 512 pages),
+		 * align it to 2MB (or 512 pages) granularity.
+		 */
+		if (size >= 0x200)
+			alignment = 0x200;
+
+		ret = drm_mm_insert_node_in_range(&npa_mm->mm, mm_node, size,
+						  alignment, 0, range_start,
+						  range_end, 0);
+	}
+	mutex_unlock(&npa_mm->mm_lock);
+
+	if (ret)
+		dev_err(adev->dev, "Failed to allocate NPA address\n");
+
+	return ret;
+}
+
+static void amdgpu_ualink_npa_free_va(struct amdgpu_device *adev,
+			       struct drm_mm_node *mm_node)
+{
+	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
+
+	mutex_lock(&npa_mm->mm_lock);
+	drm_mm_remove_node(mm_node);
+	mutex_unlock(&npa_mm->mm_lock);
+}
+
+static void amdgpu_ualink_npa_mm_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_npa_mm *npa_mm = &adev->ualink.npa_mm;
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	u64 va_size;
+
+	mutex_init(&npa_mm->mm_lock);
+
+	if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
+		/* 51 bit address space in Source-Identification mode.
+		 * 39 bits in page granularity.
+		 */
+		npa_mm->va_start = 0;
+		va_size = GENMASK_ULL(38, 0);
+	} else {
+		/* 41 bit address space in Source-Alias mode.
+		 * 29 bits in page granularity.
+		 */
+		/* First 8MB is reserved for Metadata NPAs */
+		npa_mm->va_start = 0x800;
+		va_size = GENMASK_ULL(28, 0);
+	}
+
+	npa_mm->va_size = va_size - npa_mm->va_start;
+	drm_mm_init(&npa_mm->mm, npa_mm->va_start, npa_mm->va_size);
+}
+
+static void amdgpu_ualink_npa_mm_fini(struct amdgpu_device *adev)
+{
+	mutex_destroy(&adev->ualink.npa_mm.mm_lock);
+	drm_mm_takedown(&adev->ualink.npa_mm.mm);
+}
+
+/* The caller of this function is expected to hold the XA lock when calling
+ * this function.
+ */
+static void amdgpu_generate_ualink_handle(struct amdgpu_device *adev,
+				   struct amdgpu_ualink_handle *handle)
+{
+	bool unique = false;
+
+	do {
+		handle->handle_lo = get_random_u64();
+		/* Replace bottom 10 bits in handle_lo with accId */
+		handle->handle_lo &= ~AMDGPU_UALINK_HANDLE_ACCID_MASK;
+		handle->handle_lo |= adev->ualink.info->ppod.accel_id;
+
+		/* Don't generate/store a Handle with value 0. */
+		if (!handle->handle_lo)
+			continue;
+
+		/* Find if the handle already exists in the exporter xarray.
+		 * If it already exists, then regenerate the handle since we
+		 * want the handle to be unique.
+		 */
+		unique = !xa_load(&adev->ualink.exp_xa, handle->handle_lo);
+	} while (!unique);
+
+	handle->handle_hi = get_random_u64();
+	dev_dbg(adev->dev, "GENERATE-HANDLE: generated handle: %llx:%llx\n",
+		handle->handle_hi, handle->handle_lo);
+}
+
+static void amdgpu_ualink_cleanup_exp_xa_node(struct kref *ref)
+{
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	struct amdgpu_device *adev;
+
+	exp_xa_node = container_of(ref, struct amdgpu_ualink_exp_xa_node,
+				   refcount);
+	adev = amdgpu_ttm_adev(exp_xa_node->bo->tbo.bdev);
+	queue_work(adev->ualink.npa_wq, &exp_xa_node->cleanup_work);
+}
+
+static void amdgpu_ualink_cleanup_imp_xa_node(struct kref *ref)
+{
+	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
+	struct amdgpu_device *adev;
+
+	imp_xa_node = container_of(ref, struct amdgpu_ualink_imp_xa_node,
+				   refcount);
+	adev = imp_xa_node->adev;
+
+	dev_dbg(adev->dev, "IMP-CLEANUP: Remove entry from XA for handle:%llx:%llx\n",
+		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
+
+	/* Remove node from the Xarray */
+	if (!xa_erase(&adev->ualink.imp_xa, imp_xa_node->handle.handle_lo))
+		dev_err(adev->dev, "IMP-CLEANUP: Failed to find entry in XA for handle:%llx:%llx\n",
+			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
+
+	if (imp_xa_node->npa_addr && imp_xa_node->size && imp_xa_node->dmabuf) {
+		dev_dbg(adev->dev,
+			"IMP-CLEANUP: dmabuf free for NPA:%llx handle:%llx:%llx, fc:%lu\n",
+			imp_xa_node->npa_addr, imp_xa_node->handle.handle_hi,
+			imp_xa_node->handle.handle_lo,
+			file_count(imp_xa_node->dmabuf->file));
+		dma_buf_put(imp_xa_node->dmabuf);
+		drm_gem_handle_delete(adev->ualink.client.file,
+				      imp_xa_node->gem_handle);
+
+	}
+
+	dev_dbg(adev->dev, "IMP-CLEANUP: Freeing XA entry for handle:%llx:%llx\n",
+		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo);
+
+	kfree(imp_xa_node);
+}
+
+static int amdgpu_ualink_exp_xa_entry_get(struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	return kref_get_unless_zero(&exp_xa_node->refcount);
+}
+
+static void amdgpu_ualink_exp_xa_entry_put(struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	kref_put(&exp_xa_node->refcount, amdgpu_ualink_cleanup_exp_xa_node);
+}
+
+static int amdgpu_ualink_imp_xa_entry_get(struct amdgpu_ualink_imp_xa_node *imp_xa_node)
+{
+	return kref_get_unless_zero(&imp_xa_node->refcount);
+}
+
+static void amdgpu_ualink_imp_xa_entry_put(struct amdgpu_ualink_imp_xa_node *imp_xa_node)
+{
+	kref_put(&imp_xa_node->refcount, amdgpu_ualink_cleanup_imp_xa_node);
+}
+
+static int amdgpu_ualink_send_npa_release_msg(struct amdgpu_device *adev,
+				    u32 remote_acc_id,
+				    struct amdgpu_ualink_handle handle)
+{
+	u32 dw0, dw1, dw2, dw3;
+
+	dw0 = lower_32_bits(handle.handle_lo);
+	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
+	dw0 |= AMDGPU_UALINK_NPA_RELEASE_MSG;
+
+	dw1 = upper_32_bits(handle.handle_lo);
+	dw2 = lower_32_bits(handle.handle_hi);
+	dw3 = upper_32_bits(handle.handle_hi);
+
+	dev_dbg(adev->dev, "SEND NPA-RELEASE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
+		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
+
+	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
+					      dw2, dw3);
+}
+
+static int amdgpu_ualink_send_npa_revoke_msg(struct amdgpu_device *adev,
+					u32 remote_acc_id,
+					struct amdgpu_ualink_handle handle)
+{
+	u32 dw0, dw1, dw2, dw3;
+
+	dw0 = lower_32_bits(handle.handle_lo);
+	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
+	dw0 |= AMDGPU_UALINK_NPA_REVOKE_MSG;
+
+	dw1 = upper_32_bits(handle.handle_lo);
+	dw2 = lower_32_bits(handle.handle_hi);
+	dw3 = upper_32_bits(handle.handle_hi);
+
+	dev_dbg(adev->dev, "SEND NPA-REVOKE: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
+		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
+
+	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
+					      dw2, dw3);
+}
+
+static int amdgpu_ualink_send_npa_fail_msg(struct amdgpu_device *adev,
+				    u32 remote_acc_id,
+				    struct amdgpu_ualink_handle handle,
+				    u32 fail_reason)
+{
+	u32 dw0, dw1, dw2, dw3;
+
+	dw0 = lower_32_bits(handle.handle_lo);
+	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
+	dw0 |= AMDGPU_UALINK_NPA_FAIL_MSG;
+
+	dw1 = upper_32_bits(handle.handle_lo);
+	dw2 = fail_reason & 0xFF;
+	dw3 = 0;
+
+	dev_dbg(adev->dev, "SEND NPA-FAIL: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
+		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
+
+	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
+					      dw2, dw3);
+}
+
+static int amdgpu_ualink_send_npa_rsp_msg(struct amdgpu_device *adev,
+					  u32 remote_acc_id,
+					  struct amdgpu_ualink_handle handle,
+					  u32 npa_addr, u32 size)
+{
+	u32 dw0, dw1, dw2, dw3;
+
+	dw0 = lower_32_bits(handle.handle_lo);
+	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
+	dw0 |= AMDGPU_UALINK_NPA_RSP_MSG;
+
+	dw1 = upper_32_bits(handle.handle_lo);
+	dw2 = size;
+	dw3 = npa_addr;
+
+	dev_dbg(adev->dev, "SEND NPA-RSP: remote_acc_id %u handle %llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
+		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
+
+	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
+					      dw2, dw3);
+}
+
+static int amdgpu_ualink_send_npa_req_msg(struct amdgpu_device *adev,
+					  u32 remote_acc_id,
+					  struct amdgpu_ualink_handle handle)
+{
+	u32 dw0, dw1, dw2, dw3;
+
+	dw0 =  lower_32_bits(handle.handle_lo);
+	dw0 &= ~AMDGPU_UALINK_MESSAGE_HEADER_MASK;
+	dw0 |= AMDGPU_UALINK_NPA_REQ_MSG;
+
+	dw1 = upper_32_bits(handle.handle_lo);
+	dw2 = lower_32_bits(handle.handle_hi);
+	dw3 = upper_32_bits(handle.handle_hi);
+
+	dev_dbg(adev->dev, "SEND NPA-REQ: remote_acc_id %u handle 0x%llx:%llx dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
+		remote_acc_id, handle.handle_hi, handle.handle_lo, dw0, dw1, dw2, dw3);
+
+	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, dw1,
+					      dw2, dw3);
+}
+
+static u64 amdgpu_ualink_get_export_pte_flags(struct amdgpu_device *adev,
+				       struct amdgpu_bo *bo,
+				       u64 mapping_flags)
+{
+	u64 pte_flags = adev->gmc.init_pte_flags;
+
+	pte_flags |= (AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE |
+		      AMDGPU_PTE_WRITEABLE);
+	mapping_flags |= AMDGPU_VM_MTYPE_DEFAULT;
+
+	amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, bo, mapping_flags,
+			      &pte_flags);
+
+	return pte_flags;
+}
+
+static int amdgpu_ualink_unmap_npa_addr(struct amdgpu_device *adev,
+					struct amdgpu_bo *bo,
+					u64 npa_addr, u64 size)
+{
+	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
+	uint64_t pte_value = adev->gmc.noretry_flags;
+	struct amdgpu_bo *bos[] = { bo };
+	struct dma_fence *fence = NULL;
+	struct drm_exec exec;
+	int r;
+
+	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false);
+	if (unlikely(r)) {
+		dev_err(adev->dev, "Failed to reserve VM and BO in unmap_npa_addr\n");
+		return r;
+	}
+
+	r = amdgpu_vm_update_range(adev, vm, false, false, true,
+				false, NULL, npa_addr, npa_addr + size - 1,
+				pte_value, 0, 0, NULL, NULL, &vm->last_update);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed to unmap NPA addr (%llx) from NPA VM\n", npa_addr);
+		goto out;
+	}
+
+	r = amdgpu_vm_update_pdes(adev, vm, false);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed %d to update page directories during unmapping NPA: 0x%llx\n",
+			r, npa_addr);
+		goto out;
+	}
+
+	fence = dma_fence_get(vm->last_update);
+	if (fence) {
+		r = dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		fence = NULL;
+		if (r) {
+			dev_dbg(adev->dev,
+				"UNMAP-NPA: dma fence wait failed, error: %d\n", r);
+			goto out;
+		}
+	}
+
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+out:
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+
+	return r;
+}
+
+static int amdgpu_ualink_map_npa_addr(struct amdgpu_device *adev, u64 npa_addr,
+				u64 size, struct amdgpu_bo *bo, u64 offset,
+				u64 pte_flags)
+{
+	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
+	struct amdgpu_bo *bos[] = { bo };
+	struct dma_fence *fence = NULL;
+	struct drm_exec exec;
+	int r;
+
+	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, ARRAY_SIZE(bos), &exec, false);
+	if (unlikely(r)) {
+		dev_err(adev->dev, "Failed to reserve VM and BO in map_npa_addr\n");
+		return r;
+	}
+
+	r = amdgpu_vm_update_range(adev, vm, false, false, true,
+				false, NULL, npa_addr, npa_addr + size - 1,
+				pte_flags, offset, adev->vm_manager.vram_base_offset,
+				bo->tbo.resource, NULL, &vm->last_update);
+	if (r) {
+		dev_warn(adev->dev,
+			"Failed to map NPA addr (%llx) into NPA VM\n", npa_addr);
+		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+		goto out;
+	}
+
+	r = amdgpu_vm_update_pdes(adev, vm, false);
+	if (r) {
+		dev_err(adev->dev,
+			"failed %d to update page directories for NPA: 0x%llx\n",
+			r, npa_addr);
+		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+		goto unmap_npa;
+	}
+
+	fence = dma_fence_get(vm->last_update);
+	if (fence) {
+		r = dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		fence = NULL;
+		if (r) {
+			pr_debug("failed %d to dma fence wait\n", r);
+			amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+			goto unmap_npa;
+		}
+	}
+
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+
+	/* TLB flush may be needed after updated page directories */
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+
+	return 0;
+
+unmap_npa:
+	amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size);
+out:
+	return r;
+}
+
+static int amdgpu_ualink_send_hello_ack_msg(struct amdgpu_device *adev,
+					    u32 remote_acc_id)
+{
+	dev_dbg(adev->dev, "SEND HELLO-ACK: HELLO-ACK message to remote AccId:%u\n",
+		remote_acc_id);
+	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id,
+		AMDGPU_UALINK_HELLO_ACK_MSG,
+		0, 0, 0);
+}
+
+static int amdgpu_ualink_send_hello_msg(struct amdgpu_device *adev,
+					u32 remote_acc_id)
+{
+	u32 dw0;
+
+	dw0 = AMDGPU_UALINK_HELLO_MSG;
+	dw0 |= (remote_acc_id << AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT);
+	dw0 |= (adev->ualink.info->ppod.accel_id <<
+		AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT);
+
+	dev_dbg(adev->dev, "SEND HELLO: HELLO message to remote AccId:%u\n",
+		remote_acc_id);
+	return amdgpu_ualink_remote_interrupt(adev, remote_acc_id, dw0, 0,
+					      0, 0);
+}
+
+static u32 amdgpu_ualink_check_conn_ready(struct amdgpu_device *adev,
+					  u32 remote_acc_id, u32 gen_count)
+{
+	struct amdgpu_ualink_connection *conn_state;
+	u32 current_gen_count = 0;
+
+	conn_state = &adev->ualink.conn_state[remote_acc_id];
+
+	/* Check if the connection is established. */
+	mutex_lock(&conn_state->lock);
+	if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) &&
+	    (!gen_count || conn_state->generation_count == gen_count))
+		current_gen_count = conn_state->generation_count;
+	mutex_unlock(&conn_state->lock);
+
+	return current_gen_count;
+}
+
+static void amdgpu_ualink_process_hello_ack_msg(struct amdgpu_device *adev,
+					       u32 sender_acc_id)
+{
+	struct amdgpu_ualink_connection *conn_state;
+
+	if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
+		dev_err(adev->dev,
+			"HELLO-ACK: sender AccId out of range:%u\n",
+			sender_acc_id);
+		return;
+	}
+
+	conn_state = &adev->ualink.conn_state[sender_acc_id];
+
+	/* If we are in IN_PROGRESS state, then transition the connection
+	 * state to established and signal that the HELLO ACK is received.
+	 * Otherwise, ignore the HELLO ACK.
+	 */
+	mutex_lock(&conn_state->lock);
+
+	if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS) {
+		conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED;
+		conn_state->generation_count++;
+		complete(&conn_state->hello_done);
+	} else {
+		dev_dbg(adev->dev,
+			"HELLO-ACK: already connected, ignoring from AccId:%u\n",
+			sender_acc_id);
+	}
+	mutex_unlock(&conn_state->lock);
+}
+
+static void amdgpu_ualink_process_hello_msg(struct amdgpu_device *adev,
+					   u32 receiver_acc_id,
+					   u32 sender_acc_id,
+					   u32 src_acc_id)
+{
+	struct amdgpu_ualink_connection *conn_state;
+	u32 generation_count;
+	int r;
+
+	if (receiver_acc_id != adev->ualink.info->ppod.accel_id) {
+		dev_err(adev->dev,
+			"HELLO: receiver AccId mismatch got:%u self:%u\n",
+			receiver_acc_id, adev->ualink.info->ppod.accel_id);
+		return;
+	}
+
+	/* src_acc_id is the AccId received in IH cookie. Confirm it
+	 * matches with the sender AccId.
+	 */
+	if (sender_acc_id != src_acc_id) {
+		dev_err(adev->dev,
+			"HELLO: sender AccId mismatch sender:%u IH cookie:%u\n",
+			sender_acc_id, src_acc_id);
+		return;
+	}
+
+	if (sender_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
+		dev_err(adev->dev,
+			"HELLO: sender AccId out of range:%u\n",
+			sender_acc_id);
+		return;
+	}
+
+	conn_state = &adev->ualink.conn_state[sender_acc_id];
+
+	/* Check if connection is already established. If yes, then receiving HELLO msg
+	 * triggers a reset handling scenario.
+	 * If the connection is not ready, and we receive a HELLO msg, then
+	 * transition the state to PENDING.
+	 * Otherwise, leave it IN_PROGRESS.
+	 */
+	mutex_lock(&conn_state->lock);
+	if (conn_state->state != AMDGPU_UALINK_CONN_ESTABLISHED) {
+		if (conn_state->state == AMDGPU_UALINK_CONN_NOT_READY)
+			conn_state->state = AMDGPU_UALINK_CONN_PENDING;
+		/* otherwise, leave it IN_PROGRESS to signal the completion below */
+		mutex_unlock(&conn_state->lock);
+	} else {
+		/* Leave state ESTABLISHED so handle_connection_reset()
+		 * performs the transition and imp/exp XA cleanup.
+		 */
+		generation_count = conn_state->generation_count;
+		mutex_unlock(&conn_state->lock);
+
+		amdgpu_ualink_handle_connection_reset(adev, sender_acc_id,
+						AMDGPU_UALINK_CONN_PENDING,
+						generation_count);
+	}
+
+	/* Peer may have rebooted. */
+	r = amdgpu_ualink_ring_resync_ptrs(adev, sender_acc_id);
+	if (r < 0) {
+		dev_err(adev->dev,
+			"HELLO: resync sender AccId %u ptrs failed %d\n",
+			sender_acc_id, r);
+		goto out_set_state;
+	}
+	if (r)
+		amdgpu_ualink_reset_peer_rings(adev, sender_acc_id);
+
+	r = amdgpu_ualink_send_hello_ack_msg(adev, sender_acc_id);
+	if (r)
+		dev_err(adev->dev, "HELLO-ACK: send failed to remote AccId:%u\n",
+			sender_acc_id);
+
+out_set_state:
+	mutex_lock(&conn_state->lock);
+	if (r) {
+		conn_state->state = AMDGPU_UALINK_CONN_NOT_READY;
+	} else {
+		/* If we are in IN_PROGRESS state and we receivied the HELLO message,
+		 * upon receiving the HELLO message, transition the state to ESTABLISHED
+		 * and signal the completion.
+		 */
+		if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS)
+			complete(&conn_state->hello_done);
+		conn_state->state = AMDGPU_UALINK_CONN_ESTABLISHED;
+		conn_state->generation_count++;
+	}
+	mutex_unlock(&conn_state->lock);
+}
+
+/* True if nothing is exported to, or imported from, this remote accelerator. */
+static bool amdgpu_ualink_peer_idle(struct amdgpu_device *adev, u32 remote_accel_id)
+{
+	/* Unlocked: a racing import at worst costs one redundant ring resync. */
+	return list_empty(&adev->ualink.imp_handles_list[remote_accel_id]) &&
+	       list_empty(&adev->ualink.exp_handles_list[remote_accel_id]);
+}
+
+static int amdgpu_ualink_setup_connection(struct amdgpu_device *adev,
+					  u32 remote_acc_id)
+{
+	struct amdgpu_ualink_connection *conn_state;
+	int r;
+
+	conn_state = &adev->ualink.conn_state[remote_acc_id];
+
+	/* Connection state management goes through different states.
+	 * The states are:
+	 * - NOT_READY: The connection is not ready.
+	 * - IN_PROGRESS: GPU sent HELLO message and is waiting for the HELLO_ACK.
+	 * - PENDING: GPU received HELLO message and is in the process of sending
+	 *            the HELLO_ACK.
+	 * - ESTABLISHED: The connection is established.
+	 */
+
+	/* Grab the lock and check if connection establishment was
+	 * already done by another thread.
+	 */
+	mutex_lock(&conn_state->lock);
+	if (conn_state->state == AMDGPU_UALINK_CONN_IN_PROGRESS ||
+	    conn_state->state == AMDGPU_UALINK_CONN_PENDING) {
+		r = -EAGAIN;
+		goto out;
+	}
+
+	if (conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED &&
+	    !amdgpu_ualink_peer_idle(adev, remote_acc_id)) {
+		r = 0;
+		goto out;
+	}
+
+	/* Peer is idle or not connected, it may have rebooted. Skip the
+	 * handshake if the rings are still in sync, it would reset the remote.
+	 */
+	r = amdgpu_ualink_ring_resync_ptrs(adev, remote_acc_id);
+	if (r < 0)
+		goto out;
+	if (!r && conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED)
+		goto out;
+
+	amdgpu_ualink_reset_peer_rings(adev, remote_acc_id);
+
+	conn_state->state = AMDGPU_UALINK_CONN_IN_PROGRESS;
+
+	mutex_unlock(&conn_state->lock);
+
+	/* Send HELLO message */
+	r = amdgpu_ualink_send_hello_msg(adev, remote_acc_id);
+	if (r) {
+		dev_warn(adev->dev,
+			"HELLO: Send failed to remote AccId:%u\n",
+			remote_acc_id);
+		goto reset_state;
+	}
+
+	/* Wait for the HELLO_ACK to come back */
+	/* complete(conn_state->hello_done) should be called from the IRQ
+	 * handler when the HELLO_ACK is received.
+	 */
+	r = wait_for_completion_interruptible_timeout(&conn_state->hello_done,
+				msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
+	if (r == -ERESTARTSYS) {
+		dev_err_ratelimited(adev->dev,
+			"HELLO-ACK: interrupted by signal\n");
+		goto reset_state;
+	} else if (r == 0) {
+		dev_warn(adev->dev,
+			"HELLO-ACK: Timeout from remote AccId:%u\n",
+			remote_acc_id);
+		r = -ETIMEDOUT;
+		goto reset_state;
+	}
+
+	return 0;
+
+reset_state:
+	mutex_lock(&conn_state->lock);
+	conn_state->state = AMDGPU_UALINK_CONN_NOT_READY;
+out:
+	mutex_unlock(&conn_state->lock);
+
+	return r;
+}
+
+/* Invalidate an importer node's mappings and drop its last ref (frees the
+ * node, dma-buf and GEM handle). Caller must have unlinked it from the
+ * per-remote list and must not hold the imp_xa lock.
+ */
+static void amdgpu_ualink_release_imp_xa_node(struct amdgpu_device *adev,
+				struct amdgpu_ualink_imp_xa_node *imp_xa_node)
+{
+	struct amdgpu_bo *bo;
+
+	dev_dbg(adev->dev,
+		"IMP-CLEANUP: handle:%llx:%llx npa:%llx size:%llx\n",
+		imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
+		imp_xa_node->npa_addr, imp_xa_node->size);
+
+	if (imp_xa_node->dmabuf) {
+		bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv);
+		amdgpu_ualink_invalidate_import_mappings(bo);
+	}
+
+	amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
+}
+
+static void amdgpu_ualink_cleanup_imp_xa_entries(struct amdgpu_device *adev,
+						 u32 remote_acc_id)
+{
+	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
+	struct list_head *imp_handles_list;
+
+	dev_dbg(adev->dev,
+		"IMP-RESET: Cleaning up all XA entries for remote:%u\n",
+		remote_acc_id);
+
+	imp_handles_list = &adev->ualink.imp_handles_list[remote_acc_id];
+
+	xa_lock(&adev->ualink.imp_xa);
+	while (!list_empty(imp_handles_list)) {
+		imp_xa_node = list_first_entry(imp_handles_list,
+					struct amdgpu_ualink_imp_xa_node, list);
+		list_del_init(&imp_xa_node->list);
+		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
+		xa_unlock(&adev->ualink.imp_xa);
+
+		amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node);
+
+		xa_lock(&adev->ualink.imp_xa);
+	}
+	xa_unlock(&adev->ualink.imp_xa);
+}
+
+static void amdgpu_ualink_cleanup_exp_xa_entries(struct amdgpu_device *adev,
+						 u32 remote_acc_id)
+{
+	struct amdgpu_ualink_importer_entry *importer_entry;
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	struct list_head *exp_handles_list;
+	struct drm_mm_node *mm_node;
+	u64 npa_addr, size;
+
+	/* Get the list head for the list containing all the handles
+	 * exported to this remote GPU.
+	 */
+	exp_handles_list = &adev->ualink.exp_handles_list[remote_acc_id];
+
+	dev_dbg(adev->dev,
+		"EXP-RESET: Cleaning up all XA entries for remote:%u\n",
+		remote_acc_id);
+
+	xa_lock(&adev->ualink.exp_xa);
+	while (!list_empty(exp_handles_list)) {
+		importer_entry = list_first_entry(exp_handles_list,
+					struct amdgpu_ualink_importer_entry, list);
+		list_del_init(&importer_entry->list);
+
+		exp_xa_node = importer_entry->parent;
+		if (!amdgpu_ualink_exp_xa_entry_get(exp_xa_node))
+			continue;
+
+		xa_unlock(&adev->ualink.exp_xa);
+		/* Clear the bit corresponding to this remote GPU in
+		 * the importer bitmap.
+		 */
+		if (!test_and_clear_bit(remote_acc_id,
+					exp_xa_node->importers_bitmap)) {
+			amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
+			xa_lock(&adev->ualink.exp_xa);
+			continue;
+		}
+
+		mutex_lock(&exp_xa_node->node_lock);
+		size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
+		/* Unpin the BO */
+		if (likely(!amdgpu_bo_reserve(exp_xa_node->bo, true))) {
+			amdgpu_bo_unpin(exp_xa_node->bo);
+			amdgpu_bo_unreserve(exp_xa_node->bo);
+		} else {
+			dev_warn(adev->dev,
+				"EXP-RESET: BO reserve to unpin failed handle:%llx:%llx\n",
+				exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo);
+		}
+
+		dev_dbg(adev->dev,
+			"EXP-RESET: handle:%llx:%llx pin_count:%d, importers:%d\n",
+			exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
+			exp_xa_node->bo->tbo.pin_count,
+			bitmap_weight(exp_xa_node->importers_bitmap,
+			AMDGPU_UALINK_ACCEL_MAX));
+
+		WARN_ON(exp_xa_node->bo->tbo.pin_count <
+			bitmap_weight(exp_xa_node->importers_bitmap,
+			AMDGPU_UALINK_ACCEL_MAX));
+
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
+			npa_addr = importer_entry->npa_addr;
+			mm_node = importer_entry->mm_node;
+			mutex_unlock(&exp_xa_node->node_lock);
+
+			dev_dbg(adev->dev,
+				"EXP-RESET: Unmap NPA:%llx size: %llx remote:%u handle:%llx:%llx\n",
+				npa_addr, size, remote_acc_id,
+				exp_xa_node->handle.handle_hi,
+				exp_xa_node->handle.handle_lo);
+			amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo,
+						     npa_addr, size);
+			amdgpu_ualink_npa_free_va(adev, mm_node);
+			kfree(mm_node);
+
+			/* Reset the NPA addr and mm_node */
+			mutex_lock(&exp_xa_node->node_lock);
+			importer_entry->npa_addr = 0;
+			importer_entry->mm_node = NULL;
+		} else if (bitmap_empty(exp_xa_node->importers_bitmap,
+				AMDGPU_UALINK_ACCEL_MAX)) {
+			/* In Source-Aliasing mode, if there are no importers
+			 * for this handle, then we can unmap and free the
+			 * NPA address.
+			 */
+			mm_node = exp_xa_node->importer_entries[0].mm_node;
+			npa_addr = exp_xa_node->importer_entries[0].npa_addr;
+			mutex_unlock(&exp_xa_node->node_lock);
+
+			dev_dbg(adev->dev,
+				"EXP-RESET: Unmap NPA:%llx size: %llx handle:%llx:%llx\n",
+				npa_addr, size, exp_xa_node->handle.handle_hi,
+				exp_xa_node->handle.handle_lo);
+			amdgpu_ualink_unmap_npa_addr(adev, exp_xa_node->bo,
+						     npa_addr, size);
+			amdgpu_ualink_npa_free_va(adev, mm_node);
+			kfree(mm_node);
+			mutex_lock(&exp_xa_node->node_lock);
+			exp_xa_node->importer_entries[0].npa_addr = 0;
+			exp_xa_node->importer_entries[0].mm_node = NULL;
+		}
+
+		mutex_unlock(&exp_xa_node->node_lock);
+		xa_lock(&adev->ualink.exp_xa);
+		amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
+	}
+	xa_unlock(&adev->ualink.exp_xa);
+}
+
+static void amdgpu_ualink_handle_connection_reset(struct amdgpu_device *adev,
+						u32 remote_acc_id, u32 state,
+						u32 generation_count)
+{
+	struct amdgpu_ualink_connection *conn_state;
+
+	conn_state = &adev->ualink.conn_state[remote_acc_id];
+
+	mutex_lock(&conn_state->lock);
+	if ((conn_state->state == AMDGPU_UALINK_CONN_ESTABLISHED) &&
+	     (conn_state->generation_count == generation_count)) {
+		conn_state->state = state;
+		mutex_unlock(&conn_state->lock);
+
+		amdgpu_ualink_cleanup_imp_xa_entries(adev, remote_acc_id);
+		amdgpu_ualink_cleanup_exp_xa_entries(adev, remote_acc_id);
+	} else {
+		mutex_unlock(&conn_state->lock);
+	}
+}
+
+/* Set PTE.X = 1 for all importer entries to retry RPCs. */
+static void amdgpu_ualink_force_retry_rpcs(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
+	u64 pte_flags, npa_addr, size;
+	struct dma_fence *fence = NULL;
+	struct amdgpu_bo *bo;
+	struct drm_exec exec;
+	u32 remote_acc_id;
+	int r;
+
+	bo = exp_xa_node->bo;
+	size = amdgpu_bo_ngpu_pages(bo);
+
+	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &bo, 1, &exec, false);
+	if (unlikely(r)) {
+		dev_err(adev->dev, "Failed to reserve VM and BO in force_retry_rpcs\n");
+		return;
+	}
+
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			imp_entry = &exp_xa_node->importer_entries[0];
+		else
+			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		npa_addr = imp_entry->npa_addr;
+
+		/* If the connection state changed while we are freeing
+		 * the BO, then ignore this importer. We will unmap this
+		 * address eventually in amdgpu_ualink_unmap_all_npa_addr()
+		 * function. Also, we will clear the corresponding bit in
+		 * the importer_bitmap in the same function.
+		 */
+		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
+					imp_entry->generation_count))
+			continue;
+
+		/* Set PTE.X = 1 */
+		pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo,
+					AMDGPU_VM_PAGE_EXECUTABLE);
+		dev_dbg(adev->dev,
+			"RETRY-RPC: setting PTE.X=1 for NPA:%llx remote:%u pte:0x%llx\n",
+			npa_addr, remote_acc_id, pte_flags);
+
+		r = amdgpu_vm_update_range(adev, vm, false, false, true,
+					false, NULL, npa_addr, npa_addr + size - 1,
+					pte_flags, 0, adev->vm_manager.vram_base_offset,
+					bo->tbo.resource, NULL, &vm->last_update);
+
+		if (r)
+			dev_warn(adev->dev,
+				"RETRY-RPC: PTE.X update failed for NPA:%llx remote:%u, r: %d\n",
+				npa_addr, remote_acc_id, r);
+
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			break;
+	}
+
+	r = amdgpu_vm_update_pdes(adev, vm, false);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed %d to update page directories during force retry rpcs\n",
+			r);
+		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+		return;
+	}
+
+	fence = dma_fence_get(vm->last_update);
+	if (fence) {
+		r = dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		fence = NULL;
+		if (r)
+			dev_dbg(adev->dev, "RETRY-RPC: dma fence wait failed err:%d\n", r);
+	}
+
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+}
+
+static void amdgpu_ualink_send_tlb_shootdown(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	u32 accel_id = adev->ualink.info->ppod.accel_id;
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	u64 npa_addr, size;
+	u32 remote_acc_id;
+	int r;
+
+	size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
+
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		/*
+		 * Replace the GPU-id bits with local accel_id
+		 */
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			imp_entry = &exp_xa_node->importer_entries[0];
+		else
+			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+
+		npa_addr = STRIP_NPA(imp_entry->npa_addr);
+		npa_addr = GENERATE_NPA(npa_addr, accel_id);
+
+		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
+					imp_entry->generation_count)) {
+			clear_bit(remote_acc_id,
+				  exp_xa_node->importers_bitmap);
+			continue;
+		}
+
+		dev_dbg(adev->dev,
+			"EXP-CLEANUP: Sending TLB-shootdown to remote:%u\n",
+			remote_acc_id);
+		r = amdgpu_ualink_remote_shootdown(adev, remote_acc_id,
+					npa_addr, size,
+					AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN);
+		if (r) {
+			dev_err(adev->dev,
+				"EXP-CLEANUP: TLB shootdown send failed to remote:%u\n",
+				remote_acc_id);
+
+			/* Send failed: mark the connection NOT_READY so the
+			 * rest of this cleanup (and other cleanups) skip the
+			 * unreachable peer via amdgpu_ualink_check_conn_ready().
+			 */
+			amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
+						AMDGPU_UALINK_CONN_NOT_READY,
+						imp_entry->generation_count);
+		}
+	}
+}
+
+/* Unmap all NPA addresses associated with a BO (UALink handle). This function is used
+ * only in Source Identification mode.
+ */
+static void amdgpu_ualink_unmap_all_npa_addr(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	struct amdgpu_vm *vm = &adev->ualink.npa_vm;
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	u64 pte_value = adev->gmc.noretry_flags;
+	struct dma_fence *fence = NULL;
+	struct drm_exec exec;
+	u64 npa_addr, size;
+	u32 remote_acc_id;
+	int r;
+
+	size = amdgpu_bo_ngpu_pages(exp_xa_node->bo);
+
+	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, &exp_xa_node->bo, 1,
+						 &exec, false);
+	if (unlikely(r)) {
+		dev_err(adev->dev, "Failed to reserve VM and BO in unmap_all_npa_addr\n");
+		return;
+	}
+
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			imp_entry = &exp_xa_node->importer_entries[0];
+		else
+			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		npa_addr = imp_entry->npa_addr;
+
+		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
+					imp_entry->generation_count)) {
+			clear_bit(remote_acc_id,
+				  exp_xa_node->importers_bitmap);
+			continue;
+		}
+
+		dev_dbg(adev->dev,
+			"UNMAP-NPA: Unmapping NPA:%llx, handle:%llx:%llx remote:%u pte:0x%llx\n",
+			npa_addr, exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
+			remote_acc_id, pte_value);
+
+		r = amdgpu_vm_update_range(adev, vm, false,
+					   false, true, false, NULL, npa_addr,
+					   npa_addr + size - 1, pte_value, 0,
+					   0, NULL, NULL, &vm->last_update);
+
+		if (r)
+			dev_err(adev->dev,
+				"UNMAP-NPA: Unmap failed NPA:%llx, handle:%llx:%llx remote:%u\n",
+				npa_addr, exp_xa_node->handle.handle_hi,
+				exp_xa_node->handle.handle_lo, remote_acc_id);
+
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			break;
+	}
+
+	r = amdgpu_vm_update_pdes(adev, vm, false);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed %d to update page directories during all NPA addresses unmapping\n",
+			r);
+		amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+		return;
+	}
+
+	fence = dma_fence_get(vm->last_update);
+	if (fence) {
+		r = dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		fence = NULL;
+		if (r)
+			dev_err(adev->dev,
+				"UNMAP-NPA: dma fence wait failed\n");
+	}
+
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+}
+
+static void amdgpu_ualink_free_all_npa_va(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node,
+				unsigned long *importers_bitmap)
+{
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct drm_mm_node *mm_node;
+	u32 remote_acc_id;
+
+	for_each_set_bit(remote_acc_id, importers_bitmap, AMDGPU_UALINK_ACCEL_MAX) {
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			mm_node = exp_xa_node->importer_entries[0].mm_node;
+		else
+			mm_node = exp_xa_node->importer_entries[remote_acc_id].mm_node;
+
+		if (!mm_node) {
+			if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+				break;
+			continue;
+		}
+
+		dev_dbg(adev->dev,
+			"FREE-NPA: freeing NPA address:%llx, handle:%llx:%llx remote:%u\n",
+			mm_node->start, exp_xa_node->handle.handle_hi,
+			exp_xa_node->handle.handle_lo, remote_acc_id);
+
+		amdgpu_ualink_npa_free_va(adev, mm_node);
+		kfree(mm_node);
+
+		if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+			break;
+	}
+}
+
+static void amdgpu_ualink_exp_cleanup_worker(struct work_struct *work)
+{
+	DECLARE_BITMAP(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	struct amdgpu_ualink_handle handle;
+	struct amdgpu_device *adev;
+	struct amdgpu_bo *bo;
+	u32 remote_acc_id;
+	int r;
+
+	exp_xa_node = container_of(work, struct amdgpu_ualink_exp_xa_node,
+				   cleanup_work);
+	bo = exp_xa_node->bo;
+	adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	handle = exp_xa_node->handle;
+
+	/* Revoking access to an exported memory follows the steps:
+	 * 1. Set PTE.X = 1 to retry for RPCs.
+	 * 2. Send Remote TLB Shootdowns to all importers.
+	 * 3. Unmap the NPA address from NPA VM.
+	 * 4. Drop the ref count for the BO.
+	 * 5. Send NPA_REVOKE to all importers.
+	 * 6. Wait for NPA_RELEASE from all importers.
+	 * 7. Free the NPA address once all responses are received.
+	 */
+	/* If there are no importers for this BO/handle */
+	if (bitmap_empty(exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX)) {
+		/* Release the dma_buf created at export time (used for the
+		 * local-import shortcut). The has-importers path below drops
+		 * it too; this branch must not skip it or the dma_buf and the
+		 * BO it pins are leaked.
+		 */
+		dma_buf_put(exp_xa_node->dmabuf);
+		/* Drop the BO reference so it can be freed. */
+		amdgpu_bo_unref(&bo);
+		exp_xa_node->bo = NULL;
+		goto free_node;
+	}
+
+	dev_dbg(adev->dev,
+		"EXP-CLEANUP: handle:%llx:%llx importers bitmap: %*pbl\n",
+		handle.handle_hi, handle.handle_lo,
+		AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->importers_bitmap);
+
+	bitmap_copy(orig_importers_bitmap, exp_xa_node->importers_bitmap,
+		    AMDGPU_UALINK_ACCEL_MAX);
+
+	/* Set PTE.X = 1 for NPA addresses from all importers*/
+	amdgpu_ualink_force_retry_rpcs(adev, exp_xa_node);
+
+	/* Send TLB-shootdown to all importer GPUs */
+	amdgpu_ualink_send_tlb_shootdown(adev, exp_xa_node);
+
+	/* Unmap all NPA addresses for this BO from NPA VM */
+	amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node);
+
+	dev_dbg(adev->dev, "EXP-CLEANUP: handle:%llx:%llx Unpin BO, pin_count:%u, importers:%u\n",
+		handle.handle_hi, handle.handle_lo, bo->tbo.pin_count,
+		bitmap_weight(orig_importers_bitmap, AMDGPU_UALINK_ACCEL_MAX));
+	WARN_ON(bo->tbo.pin_count < bitmap_weight(orig_importers_bitmap,
+						AMDGPU_UALINK_ACCEL_MAX));
+
+	/* Unpin the BO */
+	if (likely(!amdgpu_bo_reserve(bo, true))) {
+		bo->ualink_handle_lo = 0ULL;
+		for_each_set_bit(remote_acc_id, orig_importers_bitmap,
+				 AMDGPU_UALINK_ACCEL_MAX)
+			amdgpu_bo_unpin(bo);
+		amdgpu_bo_unreserve(bo);
+	} else {
+		dev_warn(adev->dev,
+			"EXP-CLEANUP: BO reserve to unpin failed for handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+	}
+
+	/* Free the DMABuf */
+	dma_buf_put(exp_xa_node->dmabuf);
+	/* Drop the reference to the BO so it can be freed. */
+	amdgpu_bo_unref(&bo);
+	exp_xa_node->bo = NULL;
+
+	/* Build the full npa_release_bitmap before sending any NPA-REVOKE. */
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+				 AMDGPU_UALINK_ACCEL_MAX) {
+		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id,
+					imp_entry->generation_count)) {
+			clear_bit(remote_acc_id,
+				  exp_xa_node->importers_bitmap);
+			continue;
+		}
+
+		set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap);
+	}
+
+	/* Send NPA-REVOKE to all importers which have imported this memory.
+	 * On send failure clear the bit (no response will arrive) and mark the
+	 * connection NOT_READY so other cleanups skip this unreachable peer.
+	 */
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+				 AMDGPU_UALINK_ACCEL_MAX) {
+		dev_dbg(adev->dev,
+			"EXP-CLEANUP: Sending NPA-REVOKE to remote:%u\n",
+			remote_acc_id);
+		r = amdgpu_ualink_send_npa_revoke_msg(adev, remote_acc_id, handle);
+		if (r) {
+			dev_err(adev->dev,
+				"EXP-CLEANUP: NPA-REVOKE send failed to remote:%u\n",
+				remote_acc_id);
+			clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap);
+
+			imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+			amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
+						AMDGPU_UALINK_CONN_NOT_READY,
+						imp_entry->generation_count);
+		}
+	}
+
+	/* Wait for the NPA_RELEASE to come back from all importers */
+	if (!bitmap_empty(exp_xa_node->npa_release_bitmap,
+			AMDGPU_UALINK_ACCEL_MAX)) {
+		dev_dbg(adev->dev,
+			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE bitmap: %*pbl\n",
+			handle.handle_hi, handle.handle_lo,
+			AMDGPU_UALINK_ACCEL_MAX, exp_xa_node->npa_release_bitmap);
+
+		/* Wait for the NPA_RELEASE to come back from all importers */
+		r = wait_for_completion_timeout(&exp_xa_node->npa_done,
+					msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
+
+		if (r == 0)
+			dev_warn(adev->dev,
+				"EXP-CLEANUP: NPA-RELEASE timeout for handle:%llx:%llx\n",
+				handle.handle_hi, handle.handle_lo);
+	}
+
+	/* Free the NPA addresses given to all the importers */
+	amdgpu_ualink_free_all_npa_va(adev, exp_xa_node,
+				      orig_importers_bitmap);
+
+	/* Warn about all importers that didn't respond back with
+	 * NPA-RELEASE message and trigger connection timeout handling.
+	 */
+	for_each_set_bit(remote_acc_id, exp_xa_node->npa_release_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		dev_warn(adev->dev,
+			"EXP-CLEANUP: handle:%llx:%llx NPA-RELEASE timeout from remote:%u\n",
+			handle.handle_hi, handle.handle_lo, remote_acc_id);
+		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
+						AMDGPU_UALINK_CONN_NOT_READY,
+						imp_entry->generation_count);
+	}
+
+free_node:
+	xa_erase(&adev->ualink.handle_invalid_xa, handle.handle_lo);
+	mutex_destroy(&exp_xa_node->node_lock);
+	kfree(exp_xa_node);
+}
+
+/* This function is a copy of amdgpu_dma_buf_move_notify() function.
+ * amdgpu_dma_buf_move_notify is only called for import attachments.
+ * But NPA DMABufs don't use attachments because they are imported
+ * on the same device. So we need to invalidate the GPUVM mappings
+ * manually.
+ */
+static void amdgpu_ualink_invalidate_import_mappings(struct amdgpu_bo *bo)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	struct ttm_operation_ctx ctx = { false, false };
+	struct ttm_placement placement = {};
+	struct amdgpu_vm_bo_base *bo_base;
+	struct drm_exec exec;
+	int r;
+
+	/*
+	 * Lock the BO together with every client VM page directory it is
+	 * mapped into in a single drm_exec transaction.
+	 */
+	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
+	drm_exec_until_all_locked(&exec) {
+		r = drm_exec_lock_obj(&exec, &bo->tbo.base);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(r))
+			goto fini;
+
+		for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
+			r = amdgpu_vm_lock_pd(bo_base->vm, &exec, 0);
+			drm_exec_retry_on_contention(&exec);
+			if (unlikely(r))
+				goto fini;
+		}
+	}
+
+	/* FIXME: This should be after the "if", but needs a fix to make sure
+	 * DMABuf imports are initialized in the right VM list.
+	 */
+	amdgpu_vm_bo_invalidate(bo, false);
+	if (!bo->tbo.resource || bo->tbo.resource->mem_type == TTM_PL_SYSTEM)
+		goto fini;
+
+	r = ttm_bo_validate(&bo->tbo, &placement, &ctx);
+	if (r) {
+		dev_err(adev->dev, "Failed to invalidate NPA DMA-buf import (%d)\n",
+			r);
+		goto fini;
+	}
+
+	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
+		struct amdgpu_vm *vm = bo_base->vm;
+
+		/*
+		 * Fences for the two SDMA page table updates were already
+		 * reserved by amdgpu_vm_lock_pd() above (it reserves 2 +
+		 * num_fences on the same VM root PD dma_resv).
+		 */
+		r = amdgpu_vm_clear_freed(adev, vm, NULL);
+		if (!r)
+			r = amdgpu_vm_handle_moved(adev, vm, &exec.ticket);
+
+		if (r && r != -EBUSY)
+			dev_err(adev->dev, "Failed to invalidate VM page tables (%d))\n",
+				r);
+	}
+
+fini:
+	drm_exec_fini(&exec);
+}
+
+static int amdgpu_ualink_map_npa_to_dmabuf(struct amdgpu_device *adev,
+				struct amdgpu_ualink_imp_xa_node *imp_xa_node)
+{
+	struct ttm_operation_ctx ctx = { false, false };
+	u32 initial_domain = AMDGPU_GEM_DOMAIN_NPA;
+	struct drm_gem_object *gobj = NULL;
+	u64 alloc_flags, npa_addr, size;
+	struct dma_buf *dmabuf;
+	struct amdgpu_bo *bo;
+	u32 handle;
+	int r;
+
+	npa_addr = imp_xa_node->npa_addr;
+	size = imp_xa_node->size;
+
+	dev_dbg(adev->dev, "Create NPA BO addr 0x%llx size in pages 0x%llx\n",
+		npa_addr, size);
+
+	/*
+	 * Create the BO directly in the NPA domain.
+	 */
+	alloc_flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+	r = amdgpu_gem_object_create(adev, size * AMDGPU_GPU_PAGE_SIZE, 1,
+				     initial_domain, alloc_flags,
+				     ttm_bo_type_device, NULL, &gobj, 0);
+	if (r) {
+		dev_err(adev->dev, "Failed to create NPA BO. ret %d\n", r);
+		return r;
+	}
+
+	bo = gem_to_amdgpu_bo(gobj);
+
+	r = amdgpu_bo_reserve(bo, false);
+	if (unlikely(r != 0)) {
+		dev_err(adev->dev, "Failed to reserve NPA BO, r: %d\n", r);
+		goto err_reserve_failed;
+	}
+
+	/*
+	 * Drop the arbitrarily-placed NPA node and re-create it at the exact
+	 * remote window, mirroring amdgpu_bo_create_kernel_at().
+	 */
+	ttm_resource_free(&bo->tbo, &bo->tbo.resource);
+
+	bo->placements[0].fpfn = npa_addr;
+	bo->placements[0].lpfn = npa_addr + size;
+
+	r = ttm_bo_mem_space(&bo->tbo, &bo->placement, &bo->tbo.resource, &ctx);
+	amdgpu_bo_unreserve(bo);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed to place NPA BO at 0x%llx, r: %d\n", npa_addr, r);
+		goto err_validate_failed;
+	}
+
+	r = drm_gem_handle_create(adev->ualink.client.file, gobj, &handle);
+	if (r) {
+		dev_err(adev->dev,
+			"Failed to get handle for NPA GEM object, r: %d\n", r);
+		goto err_validate_failed;
+	}
+	drm_gem_object_put(gobj);
+
+	dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, adev->ualink.client.file,
+						handle, DRM_CLOEXEC | DRM_RDWR);
+	if (IS_ERR(dmabuf)) {
+		r = PTR_ERR(dmabuf);
+		dev_err(adev->dev,
+			"Failed to generate DMABuf for NPA GEM object\n");
+		goto err_dmabuf_failed;
+	}
+
+	imp_xa_node->dmabuf = dmabuf;
+	imp_xa_node->gem_handle = handle;
+
+	return 0;
+
+err_dmabuf_failed:
+	drm_gem_handle_delete(adev->ualink.client.file, handle);
+	return r;
+err_validate_failed:
+err_reserve_failed:
+	drm_gem_object_put(gobj);
+
+	return r;
+}
+
+void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo)
+{
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	struct amdgpu_ualink_importer_entry *imp_entry;
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	u32 remote_acc_id;
+	int r;
+
+	if (!bo->ualink_handle_lo)
+		return;
+
+	/* Remove the entry from the Xarray. */
+	xa_lock(&adev->ualink.exp_xa);
+	exp_xa_node = __xa_erase(&adev->ualink.exp_xa,
+				 bo->ualink_handle_lo);
+	if (!exp_xa_node) {
+		xa_unlock(&adev->ualink.exp_xa);
+		dev_warn(adev->dev,
+			 "Exp XA: handle_lo:%llx not found\n",
+			 bo->ualink_handle_lo);
+		return;
+	}
+
+	for_each_set_bit(remote_acc_id, exp_xa_node->importers_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX) {
+		imp_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		list_del_init(&imp_entry->list);
+	}
+	xa_unlock(&adev->ualink.exp_xa);
+
+	/* Add it to the Handle_Invalid xarray */
+	r = xa_err(xa_store(&adev->ualink.handle_invalid_xa,
+			    bo->ualink_handle_lo,
+			    exp_xa_node, GFP_KERNEL));
+	if (r)
+		dev_err(adev->dev,
+			"Handle_Invalid XA store failed handle:%llx:%llx error:%d\n",
+			exp_xa_node->handle.handle_hi,
+			exp_xa_node->handle.handle_lo, r);
+
+	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
+}
+
+static void amdgpu_ualink_process_npa_release_msg(struct amdgpu_device *adev,
+						 u32 remote_acc_id,
+						 struct amdgpu_ualink_handle handle)
+{
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+
+	xa_lock(&adev->ualink.handle_invalid_xa);
+	exp_xa_node = xa_load(&adev->ualink.handle_invalid_xa, handle.handle_lo);
+	if (!exp_xa_node) {
+		dev_warn(adev->dev,
+			 "NPA-RELEASE: Handle (%llx:%llx) not found\n",
+			 handle.handle_hi, handle.handle_lo);
+		goto out;
+	}
+
+	/* Confirm that the complete handle matches. */
+	if (handle.handle_hi != exp_xa_node->handle.handle_hi) {
+		dev_warn(adev->dev,
+			 "NPA-RELEASE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n",
+			 exp_xa_node->handle.handle_hi,
+			 handle.handle_hi, handle.handle_lo, remote_acc_id);
+		goto out;
+	}
+
+	/* Test and clear the bit corresponding to the remote GPU id to signal
+	 * the arrival of NPA_RELEASE message from it.
+	 * If the corresponding bit wasn't set, then raise a warning and ignore
+	 * the NPA-Release message from the remote GPU.
+	 */
+	if (!test_and_clear_bit(remote_acc_id, exp_xa_node->npa_release_bitmap)) {
+		dev_warn(adev->dev,
+			 "NPA-RELEASE: unexpected from remote:%u handle:%llx:%llx\n",
+			 remote_acc_id, handle.handle_hi, handle.handle_lo);
+		goto out;
+	}
+
+	/* Signal completion if NPA_Release received from all importers */
+	if (bitmap_empty(exp_xa_node->npa_release_bitmap,
+			 AMDGPU_UALINK_ACCEL_MAX))
+		complete(&exp_xa_node->npa_done);
+
+out:
+	xa_unlock(&adev->ualink.handle_invalid_xa);
+}
+
+static void amdgpu_ualink_process_npa_revoke_msg(struct amdgpu_device *adev,
+						u32 remote_acc_id,
+						struct amdgpu_ualink_handle handle)
+{
+	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
+	struct amdgpu_bo *bo;
+	int r = 0;
+
+	/* Remove the entry from the Xarray. */
+	xa_lock(&adev->ualink.imp_xa);
+	imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo);
+	if (!imp_xa_node) {
+		xa_unlock(&adev->ualink.imp_xa);
+		dev_warn(adev->dev,
+			 "NPA-REVOKE: Handle (%llx:%llx) not found\n",
+			 handle.handle_hi, handle.handle_lo);
+		return;
+	}
+
+	/* Confirm that the complete handle matches. */
+	if (handle.handle_hi != imp_xa_node->handle.handle_hi) {
+		xa_unlock(&adev->ualink.imp_xa);
+		dev_warn(adev->dev,
+			 "NPA-REVOKE: handle_hi mismatch exp:%llx got:%llx:%llx remote:%u\n",
+			 imp_xa_node->handle.handle_hi,
+			 handle.handle_hi, handle.handle_lo, remote_acc_id);
+		return;
+	}
+
+	WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
+	list_del_init(&imp_xa_node->list);
+	xa_unlock(&adev->ualink.imp_xa);
+
+	/* Invalidate the GPUVM mappings */
+	bo = gem_to_amdgpu_bo(imp_xa_node->dmabuf->priv);
+	amdgpu_ualink_invalidate_import_mappings(bo);
+
+	/* Drop the refcount for the node */
+	amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
+
+	r = amdgpu_ualink_send_npa_release_msg(adev, remote_acc_id, handle);
+	if (r)
+		dev_err(adev->dev,
+			"NPA-Release send failed remote:%u handle:%llx:%llx error:%d\n",
+			remote_acc_id, handle.handle_hi, handle.handle_lo, r);
+}
+
+static void amdgpu_ualink_process_npa_fail_msg(struct amdgpu_device *adev,
+				       u32 remote_acc_id, u64 partial_handle,
+				       u32 fail_reason)
+{
+	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
+	int r = 0;
+
+	if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) {
+		dev_warn(adev->dev,
+			"NPA-FAIL: no connection with remote AccId:%u\n",
+			remote_acc_id);
+		goto conn_setup;
+	}
+
+	xa_lock(&adev->ualink.imp_xa);
+	imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle);
+	if (!imp_xa_node) {
+		xa_unlock(&adev->ualink.imp_xa);
+		dev_warn(adev->dev,
+			"NPA-FAIL: imp XA handle not found:%llx\n",
+			partial_handle);
+		return;
+	}
+
+	imp_xa_node->fail_reason = fail_reason;
+	/* Signal completion done to signal response received for NPA-REQ
+	 * message.
+	 * If the node is in NOT_READY state, then set the node state to
+	 * PENDING and signal the completion. If the node is not in NOT_READY
+	 * state, then it is an unsolicited NPA-FAIL message and we
+	 * log a debug message.
+	 */
+	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) {
+		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING);
+		complete(&imp_xa_node->npa_done);
+	} else {
+		dev_dbg(adev->dev,
+			"NPA-FAIL: unsolicited for handle:%llx:%llx from AccId:%u\n",
+			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
+			remote_acc_id);
+	}
+	xa_unlock(&adev->ualink.imp_xa);
+
+	return;
+
+conn_setup:
+	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
+	if (r)
+		dev_warn(adev->dev,
+			"NPA-FAIL: connection setup failed with remote AccId:%u\n",
+			remote_acc_id);
+}
+
+static void amdgpu_ualink_process_npa_rsp_msg(struct amdgpu_device *adev,
+				      u32 remote_acc_id, u64 partial_handle,
+				      u64 npa_addr, u64 size)
+{
+	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
+	int r = 0;
+
+	/* Check if the connection is established. If it is not, then start
+	 * connection setup.
+	 */
+	if (!amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0)) {
+		dev_warn(adev->dev,
+			"NPA-RSP: no connection with remote AccId:%u\n",
+			remote_acc_id);
+		goto conn_setup;
+	}
+
+	xa_lock(&adev->ualink.imp_xa);
+	imp_xa_node = xa_load(&adev->ualink.imp_xa, partial_handle);
+	if (!imp_xa_node) {
+		xa_unlock(&adev->ualink.imp_xa);
+		dev_warn(adev->dev,
+			"NPA-RSP: imp XA handle not found:%llx\n", partial_handle);
+		return;
+	}
+
+	/* NPA addr received in NPA-RSP is page aligned and without the remote
+	 * GPU-id in Bits 41-50. Assemble back the NPA address before storing
+	 * it.
+	 */
+	imp_xa_node->npa_addr = GENERATE_NPA(npa_addr, remote_acc_id);
+	/* Size is in number of GPU pages granularity. */
+	imp_xa_node->size = size;
+
+	/* Signal completion done to signal NPA_RSP received.
+	 * If the node is in NOT_READY state, then set the node state to
+	 * PENDING and signal the completion. If the node is not in NOT_READY
+	 * state, then it is an unsolicited NPA-RSP message and we
+	 * log a debug message.
+	 */
+	if (READ_ONCE(imp_xa_node->node_state) == AMDGPU_UALINK_NODE_NOT_READY) {
+		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_PENDING);
+		complete(&imp_xa_node->npa_done);
+	} else {
+		dev_dbg(adev->dev,
+			"NPA-RSP: unsolicited for handle:%llx:%llx from AccId:%u\n",
+			imp_xa_node->handle.handle_hi, imp_xa_node->handle.handle_lo,
+			remote_acc_id);
+	}
+	xa_unlock(&adev->ualink.imp_xa);
+
+	return;
+
+conn_setup:
+	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
+	if (r)
+		dev_warn(adev->dev,
+			"NPA-RSP: connection setup failed with remote AccId:%u\n",
+			remote_acc_id);
+}
+
+static void amdgpu_ualink_process_npa_req_msg(struct amdgpu_device *adev,
+				      u32 remote_acc_id,
+				      struct amdgpu_ualink_handle handle)
+{
+	struct amdgpu_ualink_importer_entry *importer_entry, *npa_addr_entry;
+	u32 addr_mode = adev->ualink.info->vpod.addr_mode;
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	u64 range_start, range_end, pte_flags;
+	struct drm_mm_node *mm_node = NULL;
+	int r = 0, fail_reason = 0;
+	bool send_npa_fail = true;
+	u64 npa_addr = 0, size;
+	struct amdgpu_bo *bo;
+	u32 gen_count;
+
+	/* Check if the connection is established. If it is not, then start
+	 * connection setup.
+	 */
+	gen_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0);
+	if (!gen_count) {
+		dev_warn(adev->dev,
+			"NPA-REQ: no connection with remote AccId:%u\n",
+			remote_acc_id);
+		goto conn_setup;
+	}
+
+	/* Check entry exists in Exporter XA. If yes, increase the refcount
+	 * for the node.
+	 */
+	xa_lock(&adev->ualink.exp_xa);
+	exp_xa_node = xa_load(&adev->ualink.exp_xa, handle.handle_lo);
+	if (!exp_xa_node  || (handle.handle_hi != exp_xa_node->handle.handle_hi) ||
+	    !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) {
+		xa_unlock(&adev->ualink.exp_xa);
+		dev_warn(adev->dev,
+			"NPA-REQ: exp XA handle not found handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+		fail_reason = AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE;
+		goto handle_invalid_fail;
+	}
+	xa_unlock(&adev->ualink.exp_xa);
+
+	bo = exp_xa_node->bo;
+	size = amdgpu_bo_ngpu_pages(bo);
+
+	/* Pin the BO */
+	r = amdgpu_bo_reserve(bo, true);
+	if (unlikely(r)) {
+		dev_warn(adev->dev,
+			"NPA-REQ: BO reserve failed handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
+		goto bo_reserve_fail;
+	}
+	r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_VRAM);
+	amdgpu_bo_unreserve(bo);
+	if (r) {
+		dev_warn(adev->dev,
+			"NPA-REQ: BO pin failed handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
+		goto bo_pin_fail;
+	}
+
+	if (addr_mode == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
+		mutex_lock(&exp_xa_node->node_lock);
+		importer_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		npa_addr_entry = importer_entry;
+		mutex_unlock(&exp_xa_node->node_lock);
+		/* Check if NPA address is already allocated for this importer.
+		 * If yes, then send the NPA-FAIL message back to the remote GPU.
+		 */
+		if (importer_entry->npa_addr) {
+			fail_reason = AMDGPU_UALINK_NPA_FAIL_DUPLICATE;
+			goto npa_duplicate_fail;
+		}
+
+		range_start = ((u64)remote_acc_id << AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT) |
+				AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED;
+		range_end = range_start | AMDGPU_UALINK_NPA_ADDR_RANGE_MASK;
+	} else {
+		/* We store NPA-address in importer_entries[0] in
+		 * Source-Aliasing mode.
+		 */
+		mutex_lock(&exp_xa_node->node_lock);
+		npa_addr_entry = &exp_xa_node->importer_entries[0];
+		importer_entry = &exp_xa_node->importer_entries[remote_acc_id];
+		/* Check if NPA address is already allocated for this importer.
+		 * If yes, then set the corresponding bit in the importers_bitmap,
+		 * set the generation count and send the NPA-RSP back to the remote GPU.
+		 */
+		if (npa_addr_entry->npa_addr) {
+			npa_addr = npa_addr_entry->npa_addr;
+			set_bit(remote_acc_id, exp_xa_node->importers_bitmap);
+			importer_entry->generation_count = gen_count;
+			mutex_unlock(&exp_xa_node->node_lock);
+			dev_dbg(adev->dev,
+				"NPA-REQ: NPA:%llx size:%llx handle:%llx:%llx\n",
+				npa_addr, size, handle.handle_hi, handle.handle_lo);
+
+			goto send_npa_rsp;
+		}
+		mutex_unlock(&exp_xa_node->node_lock);
+
+		range_start = 0;
+		range_end = 0;
+	}
+
+	mm_node = kzalloc(sizeof(*mm_node), GFP_KERNEL);
+	if (!mm_node) {
+		dev_warn(adev->dev,
+			"NPA-REQ: mm_node alloc failed handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+		fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE;
+		goto mem_alloc_fail;
+	}
+
+	/* Allocate NPA address */
+	r = amdgpu_ualink_npa_alloc_va(adev, mm_node, 0, range_start,
+					range_end, size);
+	if (r) {
+		dev_warn(adev->dev,
+			"NPA-REQ: NPA addr alloc failed handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+		fail_reason = AMDGPU_UALINK_NPA_FAIL_NOSPACE;
+		goto npa_alloc_fail;
+	}
+	npa_addr = mm_node->start;
+
+	pte_flags = amdgpu_ualink_get_export_pte_flags(adev, bo, 0);
+	dev_dbg(adev->dev,
+		"NPA-REQ: Allocated NPA:%llx size:%llx PTE:%llx handle:%llx:%llx\n",
+		npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo);
+
+	/* Map the NPA address into NPA VM*/
+	r = amdgpu_ualink_map_npa_addr(adev, npa_addr, size, bo, 0, pte_flags);
+	if (r) {
+		fail_reason = AMDGPU_UALINK_NPA_FAIL_ERROR;
+		dev_warn(adev->dev,
+			"NPA-REQ: NPA addr (%llx) map failed handle:%llx:%llx\n",
+			npa_addr, handle.handle_hi, handle.handle_lo);
+		goto map_npa_fail;
+	}
+
+	dev_dbg(adev->dev,
+		"NPA-REQ: Mapped NPA:%llx size:%llx pte:%llx handle:%llx:%llx\n",
+		npa_addr, size, pte_flags, handle.handle_hi, handle.handle_lo);
+send_npa_rsp:
+	/* Send NPA-RSP back to the remote GPU */
+	r = amdgpu_ualink_send_npa_rsp_msg(adev, remote_acc_id, handle,
+					   STRIP_NPA(npa_addr), size);
+	if (r) {
+		dev_warn(adev->dev,
+			"NPA-REQ: send NPA-RSP failed remote:%u handle:%llx:%llx\n",
+			remote_acc_id, handle.handle_hi, handle.handle_lo);
+		send_npa_fail = false;
+		goto send_npa_rsp_fail;
+	}
+
+	dev_dbg(adev->dev,
+		"NPA-REQ: Sent NPA-RSP with NPA:%llx size:%llx handle:%llx:%llx\n",
+		npa_addr, size, handle.handle_hi, handle.handle_lo);
+
+	/* If this is the first time we are setting the bit for this importer,
+	 * then store the NPA address, mm_node and generation count.
+	 */
+	mutex_lock(&exp_xa_node->node_lock);
+	if (!test_and_set_bit(remote_acc_id, exp_xa_node->importers_bitmap)) {
+		npa_addr_entry->npa_addr = npa_addr;
+		npa_addr_entry->mm_node = mm_node;
+		importer_entry->generation_count = gen_count;
+	}
+	mutex_unlock(&exp_xa_node->node_lock);
+
+	dev_dbg(adev->dev,
+		"NPA-REQ: BO pin_count:%d, importers:%d, handle:%llx:%llx\n",
+		bo->tbo.pin_count, bitmap_weight(exp_xa_node->importers_bitmap,
+		AMDGPU_UALINK_ACCEL_MAX), handle.handle_hi, handle.handle_lo);
+	WARN_ON(bo->tbo.pin_count < bitmap_weight(exp_xa_node->importers_bitmap,
+						   AMDGPU_UALINK_ACCEL_MAX));
+
+	/* Add this node to the exported handles list for the remote GPU,
+	 * but only if the node is still in exp_xa. If revoke already erased
+	 * it, skip the list_add to avoid a dangling list entry. The cleanup
+	 * worker is guaranteed to run after we drop our ref, so it will see
+	 * this importer in the bitmap and send NPA-REVOKE.
+	 */
+	xa_lock(&adev->ualink.exp_xa);
+	if (xa_load(&adev->ualink.exp_xa, exp_xa_node->handle.handle_lo) == exp_xa_node)
+		list_add(&importer_entry->list, &adev->ualink.exp_handles_list[remote_acc_id]);
+	xa_unlock(&adev->ualink.exp_xa);
+
+	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
+
+	return;
+
+send_npa_rsp_fail:
+	mutex_lock(&exp_xa_node->node_lock);
+	clear_bit(remote_acc_id, exp_xa_node->importers_bitmap);
+	mutex_unlock(&exp_xa_node->node_lock);
+	if (mm_node)
+		amdgpu_ualink_unmap_npa_addr(adev, bo, npa_addr, size);
+
+map_npa_fail:
+	if (mm_node)
+		amdgpu_ualink_npa_free_va(adev, mm_node);
+
+npa_alloc_fail:
+	kfree(mm_node);
+mem_alloc_fail:
+npa_duplicate_fail:
+	r = amdgpu_bo_reserve(bo, true);
+	if (likely(!r)) {
+		amdgpu_bo_unpin(bo);
+		amdgpu_bo_unreserve(bo);
+	} else {
+		dev_warn(adev->dev,
+			"NPA-REQ: BO reserve to unpin failed for handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+	}
+
+bo_pin_fail:
+bo_reserve_fail:
+	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
+
+handle_invalid_fail:
+	if (send_npa_fail) {
+		r = amdgpu_ualink_send_npa_fail_msg(adev, remote_acc_id,
+						    handle, fail_reason);
+		if (r)
+			dev_warn(adev->dev,
+				"NPA-REQ: send NPA-FAIL failed remote:%u handle:%llx:%llx\n",
+				remote_acc_id, handle.handle_hi, handle.handle_lo);
+	}
+	return;
+conn_setup:
+	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
+	if (r)
+		dev_warn(adev->dev,
+			"NPA-REQ: connection setup failed with remote AccId:%u\n",
+			remote_acc_id);
+}
+
+static int amdgpu_ualink_translate_npa_fail_reason(struct amdgpu_device *adev,
+						   u32 fail_reason)
+{
+	switch (fail_reason) {
+	case AMDGPU_UALINK_NPA_FAIL_NOSPACE:
+		return -ENOSPC;
+	case AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE:
+	case AMDGPU_UALINK_NPA_FAIL_DUPLICATE:
+	case AMDGPU_UALINK_NPA_FAIL_ERROR:
+		return -EINVAL;
+	default:
+		dev_err(adev->dev,
+			"IMPORT: invalid NPA-FAIL reason:%u\n",
+			fail_reason);
+		return -EINVAL;
+	}
+}
+
+static int amdgpu_ualink_do_import_handle(struct amdgpu_device *adev,
+				struct amdgpu_ualink_imp_xa_node *imp_xa_node,
+				u32 remote_acc_id)
+{
+	struct amdgpu_ualink_handle handle = imp_xa_node->handle;
+	u32 generation_count;
+	int r;
+
+	/* First check if the connection is setup with the
+	 * remote GPU. If yes, then initiate the NPA protocol to
+	 * get the NPA address.
+	 * If not, then initiate the HELLO protocol to first setup
+	 * the connection and once the connection is setup, then
+	 * initiate the NPA protocol.
+	 */
+	r = amdgpu_ualink_setup_connection(adev, remote_acc_id);
+	if (r) {
+		if (r != -EAGAIN)
+			dev_warn(adev->dev,
+			"IMPORT: connection setup failed with remote AccId:%u\n",
+			remote_acc_id);
+		return r;
+	}
+
+	/* Send NPA_REQ message */
+	r = amdgpu_ualink_send_npa_req_msg(adev, remote_acc_id, handle);
+	if (r) {
+		dev_warn(adev->dev,
+			"IMPORT: NPA-REQ send failed to remote AccId:%u\n",
+			remote_acc_id);
+		goto reset_conn;
+	}
+
+	/* Wait for the NPA_RSP to come back */
+	r = wait_for_completion_interruptible_timeout(&imp_xa_node->npa_done,
+				msecs_to_jiffies(AMDGPU_UALINK_RESP_TIMEOUT));
+	if (r == -ERESTARTSYS) {
+		dev_err_ratelimited(adev->dev,
+			"IMPORT: NPA-RSP wait interrupted by signal\n");
+		return r;
+	} else if (r == 0) {
+		dev_warn(adev->dev,
+			"IMPORT: NPA-RSP timeout from remote AccId:%u\n",
+			remote_acc_id);
+		r = -ETIMEDOUT;
+		goto reset_conn;
+	}
+
+	/* If the NPA addr/size isn't filled with valid values, then either
+	 * we got a NPA_FAIL or something bad happened. In either case, we
+	 * return the error back to user-space.
+	 */
+	if (imp_xa_node->fail_reason) {
+		dev_warn(adev->dev,
+			"IMPORT: NPA-REQ failed with fail_reason:%d handle:%llx:%llx\n",
+			imp_xa_node->fail_reason, handle.handle_hi, handle.handle_lo);
+		return amdgpu_ualink_translate_npa_fail_reason(adev,
+						imp_xa_node->fail_reason);
+	}
+
+	if (!imp_xa_node->npa_addr || !imp_xa_node->size) {
+		dev_warn(adev->dev,
+			"IMPORT: invalid npa:%llx or size:%llx\n",
+			imp_xa_node->npa_addr, imp_xa_node->size);
+		imp_xa_node->npa_addr = 0;
+		imp_xa_node->size = 0;
+		return -EINVAL;
+	}
+
+	r = amdgpu_ualink_map_npa_to_dmabuf(adev, imp_xa_node);
+	if (r) {
+		dev_warn(adev->dev,
+			"IMPORT: dmabuf creation failed npa:%llx size:%llx\n",
+			imp_xa_node->npa_addr, imp_xa_node->size);
+		imp_xa_node->npa_addr = 0;
+		imp_xa_node->size = 0;
+		return r;
+	}
+
+	/* Add this node to the imported handles list for the remote GPU */
+	xa_lock(&adev->ualink.imp_xa);
+	list_add(&imp_xa_node->list, &adev->ualink.imp_handles_list[remote_acc_id]);
+	xa_unlock(&adev->ualink.imp_xa);
+
+	return 0;
+
+reset_conn:
+	dev_dbg(adev->dev,
+		"IMPORT: Resetting connection for remote:%u\n", remote_acc_id);
+	generation_count = amdgpu_ualink_check_conn_ready(adev, remote_acc_id, 0);
+	amdgpu_ualink_handle_connection_reset(adev, remote_acc_id,
+					      AMDGPU_UALINK_CONN_NOT_READY,
+					      generation_count);
+	return r;
+}
+
+static int amdgpu_ualink_local_import(struct amdgpu_device *adev,
+				      u32 remote_acc_id,
+				      struct amdgpu_ualink_handle *handle,
+				      int *fd_out)
+{
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	struct amdgpu_device *peer_adev;
+	int fd;
+
+	mutex_lock(&mgpu_info.mutex);
+	peer_adev = find_peer_adev(remote_acc_id);
+	mutex_unlock(&mgpu_info.mutex);
+	if (!peer_adev) {
+		dev_err(adev->dev,
+			"IMPORT LOCAL: peer adev not found for AccId:%u\n",
+			remote_acc_id);
+		return -ENODEV;
+	}
+
+	xa_lock(&peer_adev->ualink.exp_xa);
+	exp_xa_node = xa_load(&peer_adev->ualink.exp_xa, handle->handle_lo);
+	if (!exp_xa_node ||
+	    exp_xa_node->handle.handle_hi != handle->handle_hi ||
+	    !amdgpu_ualink_exp_xa_entry_get(exp_xa_node)) {
+		xa_unlock(&peer_adev->ualink.exp_xa);
+		dev_err(adev->dev,
+			"IMPORT LOCAL: handle:%llx:%llx not found in peer exp_xa\n",
+			handle->handle_hi, handle->handle_lo);
+		return -EINVAL;
+	}
+	xa_unlock(&peer_adev->ualink.exp_xa);
+
+	get_dma_buf(exp_xa_node->dmabuf);
+	/* Get a new fd for the DMABuf */
+	fd = dma_buf_fd(exp_xa_node->dmabuf, O_CLOEXEC | O_RDWR);
+	if (fd < 0) {
+		dev_err(adev->dev,
+			"IMPORT LOCAL: dma-buf fd failed handle:%llx:%llx\n",
+			handle->handle_hi, handle->handle_lo);
+		dma_buf_put(exp_xa_node->dmabuf);
+		amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
+		return fd;
+	}
+
+	amdgpu_ualink_exp_xa_entry_put(exp_xa_node);
+	*fd_out = fd;
+	return 0;
+}
+
+int amdgpu_ualink_import_handle(struct drm_device *dev,
+				const struct amdgpu_ualink_handle *ualink_handle,
+				int *fd_out)
+{
+	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_ualink_handle handle = *ualink_handle;
+	u32 remote_acc_id, node_state;
+	int r = 0, fd;
+
+	remote_acc_id = (handle.handle_lo &
+			AMDGPU_UALINK_HANDLE_ACCID_MASK);
+
+	if (remote_acc_id >= AMDGPU_UALINK_ACCEL_MAX) {
+		dev_err(adev->dev,
+			"IMPORT: invalid remote AccId:%u\n", remote_acc_id);
+		return -EINVAL;
+	}
+
+	if (amdgpu_ualink_is_local_accel(adev, remote_acc_id))
+		return amdgpu_ualink_local_import(adev, remote_acc_id,
+						  &handle, fd_out);
+
+	xa_lock(&adev->ualink.imp_xa);
+	imp_xa_node = xa_load(&adev->ualink.imp_xa, handle.handle_lo);
+
+	if (imp_xa_node) {
+		/* If node state is Not_ready/Pending, then some other
+		 * thread is already trying the NPA protocol for the same
+		 * ualink handle. Back off and let the thread finish.
+		 * If the node state is Teardown, then it means this
+		 * node is about to be removed. So let user-space know
+		 * that this handle is invalid.
+		 */
+		node_state = READ_ONCE(imp_xa_node->node_state);
+		if (node_state == AMDGPU_UALINK_NODE_NOT_READY ||
+		    node_state == AMDGPU_UALINK_NODE_PENDING) {
+			xa_unlock(&adev->ualink.imp_xa);
+			r = -EAGAIN;
+			goto out;
+		} else if (node_state == AMDGPU_UALINK_NODE_TEARDOWN) {
+			xa_unlock(&adev->ualink.imp_xa);
+			r = -EINVAL;
+			goto out;
+		}
+
+		/* Increase the refcount while we are processing the request */
+		r = amdgpu_ualink_imp_xa_entry_get(imp_xa_node) ? 0 : -EINVAL;
+		xa_unlock(&adev->ualink.imp_xa);
+
+		/* If the refcount has become 0 but the entry is not yet
+		 * removed from the Xarray, then return error to user-space.
+		 */
+		if (r)
+			goto out;
+	} else {
+		xa_unlock(&adev->ualink.imp_xa);
+		/* if the partial handle doesn't exist in the Importer xarray then
+		 * initiate the NPA protocol and generate the DMABuf corresponding
+		 * to the NPA address.
+		 * First store the entry in the Xarray.
+		 */
+		imp_xa_node = kzalloc(sizeof(*imp_xa_node), GFP_KERNEL);
+		if (!imp_xa_node) {
+			r = -ENOMEM;
+			goto out;
+		}
+		imp_xa_node->adev = adev;
+		imp_xa_node->node_state = AMDGPU_UALINK_NODE_NOT_READY;
+		imp_xa_node->handle = handle;
+		init_completion(&imp_xa_node->npa_done);
+		INIT_LIST_HEAD(&imp_xa_node->list);
+		kref_init(&imp_xa_node->refcount);
+
+		/* Take an extra reference to store in the Xarray. The error
+		 * handling paths will drop both these references, while a
+		 * successful path will drop only one reference to the Xarray
+		 * entry.
+		 */
+		amdgpu_ualink_imp_xa_entry_get(imp_xa_node);
+
+		/* Check if another thread created a node for the same handle while
+		 * we were trying to create and initialize the node.
+		 */
+		r = xa_insert(&adev->ualink.imp_xa, handle.handle_lo,
+			      imp_xa_node, GFP_KERNEL);
+		if (r) {
+			kfree(imp_xa_node);
+			/* -EBUSY means another thread raced us and inserted a
+			 * node for the same handle. Ask user-space to retry.
+			 */
+			if (r == -EBUSY)
+				r = -EAGAIN;
+			else
+				dev_err(adev->dev,
+					"IMPORT: XA insert failed for handle:%llx:%llx err:%d\n",
+					handle.handle_hi, handle.handle_lo, r);
+			goto out;
+		}
+
+		r = amdgpu_ualink_do_import_handle(adev, imp_xa_node, remote_acc_id);
+
+		/* If error is returned, then cleanup the xarray entry before returning
+		 * the error back to user-space
+		 */
+		if (r) {
+			amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
+			if (r != -EAGAIN)
+				dev_err(adev->dev,
+					"IMPORT: XA import failed for handle:%llx:%llx\n",
+					handle.handle_hi, handle.handle_lo);
+			goto cleanup;
+		} else {
+			WRITE_ONCE(imp_xa_node->node_state,
+				   AMDGPU_UALINK_NODE_READY);
+		}
+	}
+
+	/* dma_buf_fd consumes a reference and assigns it to the fd.
+	 * Therefore take an extra reference to be consumed. It will be
+	 * released when user mode closes the fd.
+	 */
+	get_dma_buf(imp_xa_node->dmabuf);
+
+	fd = dma_buf_fd(imp_xa_node->dmabuf, O_CLOEXEC | O_RDWR);
+	if (fd >= 0) {
+		*fd_out = fd;
+	} else {
+		dma_buf_put(imp_xa_node->dmabuf);
+		r = fd;
+		dev_err(adev->dev,
+			"IMPORT: dma-buf fd creation failed handle:%llx:%llx\n",
+			handle.handle_hi, handle.handle_lo);
+	}
+
+cleanup:
+	amdgpu_ualink_imp_xa_entry_put(imp_xa_node);
+out:
+	return r;
+}
+
+int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp,
+				u32 gem_handle,
+				struct amdgpu_ualink_handle *handle_out)
+{
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	struct amdgpu_ualink_handle handle;
+	struct drm_gem_object *gobj;
+	struct amdgpu_device *adev;
+	struct amdgpu_bo *robj;
+	int r = 0, i;
+
+	gobj = drm_gem_object_lookup(filp, gem_handle);
+	if (!gobj)
+		return -ENOENT;
+
+	robj = gem_to_amdgpu_bo(gobj);
+	adev = amdgpu_ttm_adev(robj->tbo.bdev);
+
+	if (!(robj->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
+		dev_err(adev->dev, "Only VRAM BOs can be exported\n");
+		r = -EOPNOTSUPP;
+		goto out;
+	}
+
+	if (!robj->ualink_handle_lo) {
+		/* If no ualink handle generated for BO, then generate one and
+		 * add it to the exporter Xarray.
+		 */
+		exp_xa_node = kzalloc(sizeof(*exp_xa_node), GFP_KERNEL);
+		if (!exp_xa_node) {
+			dev_err(adev->dev, "Failed to allocate exp_xa_node\n");
+			r = -ENOMEM;
+			goto out;
+		}
+
+		amdgpu_bo_ref(robj);
+		exp_xa_node->bo = robj;
+		init_completion(&exp_xa_node->npa_done);
+		bitmap_zero(exp_xa_node->importers_bitmap,
+			    AMDGPU_UALINK_ACCEL_MAX);
+		bitmap_zero(exp_xa_node->npa_release_bitmap,
+			AMDGPU_UALINK_ACCEL_MAX);
+		kref_init(&exp_xa_node->refcount);
+		mutex_init(&exp_xa_node->node_lock);
+		INIT_WORK(&exp_xa_node->cleanup_work,
+			  amdgpu_ualink_exp_cleanup_worker);
+		for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
+			INIT_LIST_HEAD(&exp_xa_node->importer_entries[i].list);
+			exp_xa_node->importer_entries[i].parent = exp_xa_node;
+		}
+		/* DMABuf handle for local import of fabric handles */
+		exp_xa_node->dmabuf = drm_gem_prime_handle_to_dmabuf(&adev->ddev, filp,
+						gem_handle, DRM_CLOEXEC | DRM_RDWR);
+		if (IS_ERR(exp_xa_node->dmabuf)) {
+			r = PTR_ERR(exp_xa_node->dmabuf);
+			dev_err(adev->dev, "Failed to generate DMABuf for the BO\n");
+			kfree(exp_xa_node);
+			goto out;
+		}
+
+		xa_lock(&adev->ualink.exp_xa);
+		amdgpu_generate_ualink_handle(adev, &handle);
+		exp_xa_node->handle = handle;
+		r = __xa_insert(&adev->ualink.exp_xa, handle.handle_lo,
+				exp_xa_node, GFP_KERNEL);
+		xa_unlock(&adev->ualink.exp_xa);
+		if (r) {
+			dev_err(adev->dev, "Failed to insert exp_xa_node into XA: %d\n", r);
+			dma_buf_put(exp_xa_node->dmabuf);
+			amdgpu_bo_unref(&robj);
+			kfree(exp_xa_node);
+			goto out;
+		}
+
+		robj->ualink_handle_lo = handle.handle_lo;
+		/* Return the generated handle back to the caller */
+		*handle_out = handle;
+	} else {
+		handle_out->handle_lo = robj->ualink_handle_lo;
+
+		/* Do a sanity check to ensure the handle exists in the XA */
+		xa_lock(&adev->ualink.exp_xa);
+		exp_xa_node = xa_load(&adev->ualink.exp_xa,
+				      robj->ualink_handle_lo);
+		xa_unlock(&adev->ualink.exp_xa);
+		WARN(!exp_xa_node, "Exp XA: Handle_Lo: %llx not found",
+		     robj->ualink_handle_lo);
+		if (exp_xa_node)
+			handle_out->handle_hi = exp_xa_node->handle.handle_hi;
+	}
+
+out:
+	drm_gem_object_put(gobj);
+	return r;
+}
+
+int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data,
+				   struct drm_file *filp)
+{
+	union drm_amdgpu_ualink_handle *args = data;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_ualink_handle handle = {};
+	u32 gem_handle;
+	int r, fd = -1;
+
+	if (adev->ualink.info->accel_state !=
+	    AMDGPU_UALINK_ACCEL_STATE_ACTIVE) {
+		dev_err(adev->dev,
+			"ualink device is not in active state in vpod\n");
+		return -EOPNOTSUPP;
+	}
+
+	/* The input and output members of the ioctl argument alias each other.
+	 * Latch every input field before invoking the handlers, and only write
+	 * the output fields afterwards.
+	 */
+	switch (args->in.op) {
+	case DRM_AMDGPU_UALINK_HANDLE_OP_EXPORT:
+		gem_handle = args->in.gem_handle;
+		r = amdgpu_ualink_export_handle(dev, filp, gem_handle, &handle);
+		if (!r) {
+			args->out.export_ualink_handle[0] = handle.handle_lo;
+			args->out.export_ualink_handle[1] = handle.handle_hi;
+		}
+		break;
+	case DRM_AMDGPU_UALINK_HANDLE_OP_IMPORT:
+		handle.handle_lo = args->in.import_ualink_handle[0];
+		handle.handle_hi = args->in.import_ualink_handle[1];
+		r = amdgpu_ualink_import_handle(dev, &handle, &fd);
+		if (!r)
+			args->out.import_dmabuf_handle = fd;
+		break;
+	default:
+		r = -EINVAL;
+		break;
+	}
+
+	return r;
+}
+
+int amdgpu_ualink_manager_start(struct amdgpu_device *adev)
+{
+	int i, r;
+
+	r = amdgpu_vm_init(adev, &adev->ualink.npa_vm, 0);
+	if (r)
+		goto out;
+
+	/* For pinning page tables and using CPU for page table updates. */
+	r = amdgpu_vm_make_npa(adev, &adev->ualink.npa_vm);
+	if (r)
+		goto uninit_vm;
+
+	adev->ualink.npa_wq = alloc_workqueue("NPA WQ", WQ_UNBOUND, 0);
+	if (unlikely(!adev->ualink.npa_wq)) {
+		dev_err(adev->dev, "Failed to allocate NPA WQ\n");
+		r = -ENOMEM;
+		goto uninit_vm;
+	}
+
+	/* Map this VM to NPA VMID */
+	adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid,
+			amdgpu_gmc_pd_addr(adev->ualink.npa_vm.root.bo));
+
+	xa_init(&adev->ualink.exp_xa);
+	xa_init(&adev->ualink.imp_xa);
+	xa_init(&adev->ualink.handle_invalid_xa);
+
+	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
+		init_completion(&adev->ualink.conn_state[i].hello_done);
+		mutex_init(&adev->ualink.conn_state[i].lock);
+		adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY;
+		INIT_LIST_HEAD(&adev->ualink.exp_handles_list[i]);
+		INIT_LIST_HEAD(&adev->ualink.imp_handles_list[i]);
+	}
+
+	amdgpu_ualink_npa_mm_init(adev);
+
+	return 0;
+
+uninit_vm:
+	amdgpu_vm_fini(adev, &adev->ualink.npa_vm);
+out:
+	return r;
+}
+
+/* Free every importer entry left in imp_xa at manager stop. */
+static void amdgpu_ualink_teardown_imp_xa_entries(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_imp_xa_node *imp_xa_node;
+	unsigned long index;
+
+	xa_for_each(&adev->ualink.imp_xa, index, imp_xa_node) {
+		list_del_init(&imp_xa_node->list);
+		WRITE_ONCE(imp_xa_node->node_state, AMDGPU_UALINK_NODE_TEARDOWN);
+		amdgpu_ualink_release_imp_xa_node(adev, imp_xa_node);
+	}
+}
+
+/* Free a single exporter node at manager stop. This is the message-free
+ * portion of amdgpu_ualink_exp_cleanup_worker().
+ */
+static void amdgpu_ualink_teardown_exp_xa_node(struct amdgpu_device *adev,
+				struct amdgpu_ualink_exp_xa_node *exp_xa_node)
+{
+	DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
+	struct amdgpu_bo *bo = exp_xa_node->bo;
+	u32 remote_acc_id;
+
+	bitmap_copy(importers_bitmap, exp_xa_node->importers_bitmap,
+		    AMDGPU_UALINK_ACCEL_MAX);
+
+	dev_dbg(adev->dev,
+		"EXP-STOP: handle:%llx:%llx importers bitmap: %*pbl\n",
+		exp_xa_node->handle.handle_hi, exp_xa_node->handle.handle_lo,
+		AMDGPU_UALINK_ACCEL_MAX, importers_bitmap);
+
+	if (!bitmap_empty(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX)) {
+		amdgpu_ualink_unmap_all_npa_addr(adev, exp_xa_node);
+
+		/* Unlink importers and unpin the BO once per importer. */
+		if (likely(!amdgpu_bo_reserve(bo, true))) {
+			bo->ualink_handle_lo = 0ULL;
+			for_each_set_bit(remote_acc_id, importers_bitmap,
+					 AMDGPU_UALINK_ACCEL_MAX) {
+				list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list);
+				amdgpu_bo_unpin(bo);
+			}
+			amdgpu_bo_unreserve(bo);
+		} else {
+			dev_warn(adev->dev,
+				"EXP-STOP: BO reserve to unpin failed handle:%llx:%llx\n",
+				exp_xa_node->handle.handle_hi,
+				exp_xa_node->handle.handle_lo);
+			for_each_set_bit(remote_acc_id, importers_bitmap,
+					 AMDGPU_UALINK_ACCEL_MAX)
+				list_del_init(&exp_xa_node->importer_entries[remote_acc_id].list);
+		}
+
+		amdgpu_ualink_free_all_npa_va(adev, exp_xa_node, importers_bitmap);
+	}
+
+	/* Release the export dma-buf and drop the BO ref. */
+	dma_buf_put(exp_xa_node->dmabuf);
+	amdgpu_bo_unref(&bo);
+	exp_xa_node->bo = NULL;
+
+	mutex_destroy(&exp_xa_node->node_lock);
+	kfree(exp_xa_node);
+}
+
+/* Free every exporter entry left in exp_xa at manager stop. These were
+ * exported but never revoked, so no cleanup worker was queued for them.
+ */
+static void amdgpu_ualink_teardown_exp_xa_entries(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_exp_xa_node *exp_xa_node;
+	unsigned long index;
+
+	xa_for_each(&adev->ualink.exp_xa, index, exp_xa_node) {
+		xa_erase(&adev->ualink.exp_xa, index);
+		amdgpu_ualink_teardown_exp_xa_node(adev, exp_xa_node);
+	}
+}
+
+void amdgpu_ualink_manager_stop(struct amdgpu_device *adev)
+{
+	int i;
+
+	adev->mmhub.funcs->setup_vm_pt_regs(adev, adev->vm_manager.npa_vmid, 0);
+
+	/* Mark connections down so the drained workers and the teardown below
+	 * take the message-free path in amdgpu_ualink_check_conn_ready().
+	 */
+	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++) {
+		mutex_lock(&adev->ualink.conn_state[i].lock);
+		adev->ualink.conn_state[i].state = AMDGPU_UALINK_CONN_NOT_READY;
+		mutex_unlock(&adev->ualink.conn_state[i].lock);
+	}
+
+	/* Drain in-flight exporter cleanup work so revoked nodes in
+	 * handle_invalid_xa free themselves and empty that xarray.
+	 */
+	drain_workqueue(adev->ualink.npa_wq);
+
+	/* Free live entries before the NPA allocator is torn down, so NPA
+	 * addresses can still be unmapped and freed.
+	 */
+	amdgpu_ualink_teardown_imp_xa_entries(adev);
+	amdgpu_ualink_teardown_exp_xa_entries(adev);
+
+	amdgpu_ualink_npa_mm_fini(adev);
+
+	/* All three xarrays are empty by now. */
+	xa_destroy(&adev->ualink.exp_xa);
+	xa_destroy(&adev->ualink.imp_xa);
+	xa_destroy(&adev->ualink.handle_invalid_xa);
+
+	for (i = 0; i < AMDGPU_UALINK_ACCEL_MAX; i++)
+		mutex_destroy(&adev->ualink.conn_state[i].lock);
+
+	destroy_workqueue(adev->ualink.npa_wq);
+	amdgpu_vm_fini(adev, &adev->ualink.npa_vm);
+}
+
+/*
+ * UALink remote interrupt and shootdown
+ */
+
+/* UALINK ring buffer size, same for both remote shootdown and interrupt ring */
+#define AMDGPU_UALINK_RB_SIZE	4096
+
+#define AMDGPU_UALINK_METADATA_HEADER	0x4E485446
+
+/* 2MB NPA start address for 2MB page mapping */
+#define AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET SZ_2M
+
+struct amdgpu_ualink_metadata {
+	u32 header;
+
+	/*
+	 * "ring entries" as unit. So the ring-size-in-bytes could be calculated
+	 * as entry-size * 2^RBsize. This would make the minimum size a single
+	 * entry and the maximum size 32786 entries.
+	 *
+	 * u32 rb_size:4;
+	 * u32 reserved0:4;
+	 * u32 vmid:4;
+	 * u32 reserved1:20;
+	 */
+	u32 rb_size;
+
+	/* ring buffer base address for remote interrupt */
+	u64 ri_rb;
+
+	/* tlb invalidate ring buffer base address for remote shootdown */
+	u64 tlb_inv_rb;
+
+	/* remote interrupt Tail pointer, write pointer address */
+	u64 tailptr_ri;
+
+	/* TLB invalidate ring buffer's Tail pointer, write pointer address */
+	u64 tailptr_tlb_inv;
+};
+
+struct amdgpu_ualink_wb {
+	/* last finished command seq number */
+	u32 data;
+
+	/* command complete error code */
+	u32 status;
+
+	/* ring rptr updated by FW */
+	u64 rptr;
+};
+
+/*
+ * For remote interrupt and shootdown
+ */
+struct amdgpu_ualink_ring {
+	u32 rb_size;
+
+	/* writeback data */
+	u32 seq;
+
+	/* local copy */
+	u64 wptr, rptr;
+
+	/* NPA gart mapping for SDMA */
+	u64 rb_npa_gart;
+	u64 wptr_npa_gart;
+	u64 doorbell_npa_gart;
+
+	/* gart mapping node */
+	struct drm_mm_node mm_node_rb;
+	struct drm_mm_node mm_node_wptr;
+	struct drm_mm_node mm_node_doorbell;
+
+	/* seq, complete status write back cpu address */
+	struct amdgpu_ualink_wb *wb_cpu;
+
+	/* true if fw write back address updated successfully */
+	bool ready;
+};
+
+struct amdgpu_ualink_peer {
+	/* remote interrupt and shootdown uses same SDMA entity */
+	struct mutex lock;
+
+	/* SDMA engine to send remote command via NPA */
+	struct drm_sched_entity entity;
+
+	/* to select different DXS ports, cycles through different values */
+	u32 dxs_port;
+
+	struct amdgpu_ualink_ring interrupt;
+	struct amdgpu_ualink_ring shootdown;
+};
+
+struct amdgpu_ualink_remote {
+	/* ualink metadata passed to MPNHT FW */
+	struct amdgpu_bo		*metadata_bo;
+	u64				metadata_gpu_addr;
+	void				*metadata_cpu_addr;
+
+	/* ualink ring, tlb ring buffer, wptr */
+	struct amdgpu_bo		*ring_bo;
+	u64				rb_gpu_addr;
+	void				*rb_cpu_addr;
+
+	/* ualink rptr, wb data, statuss for address alias mode */
+	struct amdgpu_bo		*rptr_bo;
+	u64				rptr_gpu_addr;
+	void				*rptr_cpu_addr;
+	u64				rptr_npa;
+
+	/* address alias mode alloc npa address for shared wb */
+	struct drm_mm_node		rptr_mm_node;
+
+	/*
+	 * Owned snapshot of the vPod's active accelerator bitmap, taken at
+	 * sw_init time. Must NOT alias info->vpod.active_accel_bits, which
+	 * psp_ual_query_info() refreshes on every commit: aliasing it would let
+	 * teardown unmap a different set than setup mapped, leaking drm_mm nodes
+	 * in the shared GTT manager.
+	 */
+	DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
+	u32				num_accel;
+
+	/* remote GPUs ring buffer, read, write pointer local copy and gart mapping */
+	struct amdgpu_ualink_peer	peer[AMDGPU_UALINK_ACCEL_MAX];
+
+	/* use lsdma write to NPA address for remote interrupt */
+	bool			       use_lsdma;
+};
+
+static inline struct amdgpu_ualink_remote *to_remote(struct amdgpu_device *adev)
+{
+	return adev->ualink.remote;
+}
+
+/*
+ * Returns true if the vPod membership most recently reported by firmware
+ * (info->vpod.active_accel_bits, read back via psp_ual_query_info() on the
+ * committing GPU) differs from the owned snapshot captured when this
+ * accelerator was last activated (remote->active_accel_bits). Used to
+ * decide whether an accelerator must be (re)built to match the new member
+ * set. A NULL remote means the accelerator has never been activated (no
+ * snapshot yet), which also counts as "changed" so it gets brought up.
+ */
+static bool amdgpu_ualink_vpod_membership_changed(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+
+	if (!remote)
+		return true;
+
+	return !bitmap_equal(remote->active_accel_bits,
+			     adev->ualink.info->vpod.active_accel_bits,
+			     AMDGPU_UALINK_ACCEL_MAX);
+}
+
+static inline u32 ualink_accel_id(struct amdgpu_device *adev)
+{
+	return adev->ualink.info->ppod.accel_id;
+}
+
+static inline enum amdgpu_ualink_addr_mode ualink_addr_mode(struct amdgpu_device *adev)
+{
+	return adev->ualink.info->vpod.addr_mode;
+}
+
+/*
+ * address mode and NPA address for ring wptr, rptr, wb status, wb data
+ *
+ * source identification address mode
+ *
+ * we need to export different tail pointers for different remote GPU accel_id,
+ * hence we need to allocate a whole page for each remote GPU separately.
+ *
+ * rptr, wb data, status share the whole page with wptr.
+ *
+ *     1 page reserved NPA address for 1 active remote GPU, max 255 pages
+ *     starting address: wptr NPA address + accel_id * 4K
+ *     offset:
+ *         remote interrupt ring wptr 0
+ *         remote shootdown ring wptr 8
+ *         remote interrupt wb status 16
+ *         remote interrupt wb data   20
+ *         remote interrupt ring rptr 24
+ *         remote shootdown wb status 32
+ *         remote shootdown wb data   36
+ *         remote shootdown ring rptr 40
+ *
+ * source aliasing address mode
+ *
+ * wptr for different remote GPU can share same reserved 1 page NPA address page.
+ *
+ * to save reserved NPA address space, rptr, wb data, status for different
+ * remote GPU share same allocated 2 pages NPA address.
+ *
+ *     1 page reserved NPA address for wptr of all active remote GPUs
+ *     offset:
+ *         remote interrupt ring wptr accel_id * 16
+ *         remote shootdown ring wptr accel_id * 16 + 8
+ *
+ *     2 page allocated NPA address for rptr, writeback of all active remote GPUs
+ *     starting address: at accel_id * 32 for each remote GPU
+ *     offset:
+ *         remote interrupt wb status 0
+ *         remote interrupt wb data   4
+ *         remote interrupt ring rptr 8
+ *         remote shootdown wb status 16
+ *         remote shootdown wb data   20
+ *         remote shootdown ring rptr 24
+ */
+static inline u32 ualink_wptr_size(struct amdgpu_device *adev)
+{
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
+		return AMDGPU_GPU_PAGE_SIZE;
+	else
+		return 0;	/* wptr uses the ring buffer for itself */
+}
+
+static inline u32 ualink_wb_size(struct amdgpu_device *adev)
+{
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
+		return 0;	/* share same page with wptr */
+	else
+		return 2 * sizeof(struct amdgpu_ualink_wb);
+}
+
+static inline u32 ualink_wptr_offset(struct amdgpu_device *adev, u32 accel_id)
+{
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
+		return 0;	/* one separate page per GPU */
+	else
+		return accel_id * 2 * sizeof(u64);	/* two pointers per GPU */
+}
+
+static inline u32 ualink_tlb_wptr_offset(struct amdgpu_device *adev, u32 accel_id)
+{
+	return ualink_wptr_offset(adev, accel_id) + sizeof(u64);
+}
+
+static inline u32 ualink_wb_offset(struct amdgpu_device *adev, u32 accel_id)
+{
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT)
+		return 2 * sizeof(u64);	/* after two wptr pointers */
+	else
+		return accel_id * ualink_wb_size(adev);
+}
+
+static inline u32 ualink_tlb_wb_offset(struct amdgpu_device *adev, u32 accel_id)
+{
+	return ualink_wb_offset(adev, accel_id) + sizeof(struct amdgpu_ualink_wb);
+}
+
+static void amdgpu_ualink_flush_tlb(struct amdgpu_device *adev, u32 flush_type)
+{
+	u32 bit;
+
+	bit = AMDGPU_MMHUB0_START;
+
+	for_each_set_bit_from(bit, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
+		amdgpu_gmc_flush_gpu_tlb(adev, adev->vm_manager.npa_vmid,
+					bit, flush_type);
+}
+
+/**
+ * amdgpu_ualink_npa_vm_map_range - Map a range in the NPA VM
+ * @adev: amdgpu device pointer
+ * @bo: buffer object backing the mapping
+ * @pte_flags: page table entry flags
+ * @offset: offset into the buffer object in bytes
+ * @size_in_pages: size of the range to map in pages
+ * @npa_in_pages: NPA target address in pages
+ *
+ * Maps a buffer object range into the NPA VM page table at the specified
+ * NPA address.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_npa_vm_map_range(struct amdgpu_device *adev, struct amdgpu_bo *bo,
+				   u64 pte_flags, u64 offset, u64 size_in_pages,
+				   u64 npa_in_pages)
+{
+	struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm;
+	int r;
+
+	dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n",
+		offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags,
+		npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
+
+	r = amdgpu_vm_update_range(adev, npa_vm, false, false, true,
+				   false, NULL, npa_in_pages,
+				   npa_in_pages + size_in_pages - 1,
+				   pte_flags, offset, adev->vm_manager.vram_base_offset,
+				   bo->tbo.resource, NULL, &npa_vm->last_update);
+	if (r)
+		dev_dbg(adev->dev, "failed %d to map npa 0x%llx to NPA VM\n", r,
+			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
+	return r;
+}
+
+/**
+ * amdgpu_npa_vm_unmap_range - Unmap a range from the NPA VM
+ * @adev: amdgpu device pointer
+ * @bo: buffer object backing the mapping
+ * @pte_flags: page table entry flags
+ * @offset: offset into the buffer object in bytes
+ * @size_in_pages: size of the range to unmap in pages
+ * @npa_in_pages: NPA target address in pages
+ *
+ * Removes a previously established mapping from the NPA VM page table.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_npa_vm_unmap_range(struct amdgpu_device *adev, struct amdgpu_bo *bo,
+				     u64 pte_flags, u64 offset, u64 size_in_pages,
+				     u64 npa_in_pages)
+{
+	struct amdgpu_vm *npa_vm = &adev->ualink.npa_vm;
+	int r;
+
+	dev_dbg(adev->dev, "offset 0x%llx size 0x%llx flags 0x%llx npa 0x%llx\n",
+		offset, size_in_pages << AMDGPU_GPU_PAGE_SHIFT, pte_flags,
+		npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
+
+	r = amdgpu_vm_update_range(adev, npa_vm, false, false, true,
+				   false, NULL, npa_in_pages,
+				   npa_in_pages + size_in_pages - 1,
+				   pte_flags, offset, 0, bo->tbo.resource, NULL,
+				   &npa_vm->last_update);
+	if (r)
+		dev_dbg(adev->dev, "failed %d to unmap npa 0x%llx from NPA VM\n", r,
+			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
+	return r;
+}
+
+/*
+ * Reserved NPA space for remote shootdown and interrupt ring buffer,
+ * write pointers, read pointers and writeback buffers
+ *
+ * NPA ring type: 0 = shootdown, 1 = interrupt, 2 = tailptr (wptr)
+ */
+enum ring_buffer_type {
+	RB_TYPE_TLB_INV			= 0,
+	RB_TYPE_REMOTE_INTERRUPT	= 1,
+	RB_TYPE_TAILPTR			= 2
+};
+
+/**
+ * amdgpu_ualink_npa_addr - Get reserved NPA address for ring buffer
+ * @adev: amdgpu device pointer
+ * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR)
+ * @src_accel_id: Source accelerator ID
+ * @dst_accel_id: Destination accelerator ID
+ *
+ * Return: NPA address for the specified ring buffer type and GPUs
+ */
+static u64 amdgpu_ualink_npa_addr(struct amdgpu_device *adev, u32 type,
+				  u32 src_accel_id, u32 dst_accel_id)
+{
+	u32 addr_mode = ualink_addr_mode(adev);
+	u64 npa;
+
+	WARN_ON_ONCE(src_accel_id >= AMDGPU_UALINK_ACCEL_MAX ||
+		     dst_accel_id >= AMDGPU_UALINK_ACCEL_MAX ||
+		     type > RB_TYPE_TAILPTR ||
+		     (addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT &&
+		     addr_mode != AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS));
+
+	switch (addr_mode) {
+	case AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT:
+		npa = (u64)src_accel_id << 41 | type << 12;
+		break;
+
+	case AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS:
+	default:
+		if (type == RB_TYPE_TAILPTR)
+			npa = (u64)dst_accel_id << 13;
+		else
+			npa = (u64)src_accel_id << 13 | type << 12;
+
+		/*
+		 * In order to avoid address conflicts between source-identification
+		 * and source-aliasing mode, add 2MB to the buffer addresses in
+		 * source-aliasing mode. This way in a misconfigured cluster, a GPU
+		 * configured in the wrong address mode will access unmapped NPA addresses
+		 * rather than the wrong buffer mappings.
+		 */
+		npa += AMDGPU_UALINK_SOURCE_ALIAS_NPA_OFFSET;
+		break;
+	}
+
+	dev_dbg(adev->dev, "addr mode %d from accel %d to accel %d type %d NPA 0x%llx\n",
+		addr_mode, src_accel_id, dst_accel_id, type, npa);
+
+	return npa;
+}
+
+/**
+ * amdgpu_ualink_gart_npa_addr - Get GART-mapped NPA address for ring buffer
+ * @adev: amdgpu device pointer
+ * @type: Ring buffer type (TLB_INV, REMOTE_INTERRUPT, or TAILPTR)
+ * @src_accel_id: Source accelerator ID
+ * @dst_accel_id: Destination accelerator ID
+ *
+ * Computes the NPA address suitable for GART mapping by adding dst_accid_id to
+ * NPA address to access remote by SDMA.
+
+ * clears the source GPU ID bits from the NPA address which is set for
+ * address-identification mode.
+ *
+ * Return: NPA address adjusted for GART mapping
+ */
+
+#define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT		41
+#define AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK		GENMASK_ULL(50, 41)
+
+static u64 amdgpu_ualink_gart_npa_addr(struct amdgpu_device *adev, u32 type,
+					u32 src_accel_id, u32 dst_accel_id)
+{
+	u64 npa;
+
+	npa = amdgpu_ualink_npa_addr(adev, type, src_accel_id, dst_accel_id);
+
+	/* wiping out source accelerator id */
+	npa &= ~AMDGPU_UALINK_GART_NPA_ADDR_GPUID_MASK;
+
+	return npa | ((u64)dst_accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT);
+}
+
+/**
+ * amdgpu_ualink_reserve_npa_vm_and_bos - Reserve the NPA VM page directory and BOs
+ * @adev: amdgpu device pointer
+ * @bos: array of buffer objects to reserve
+ * @n_bos: number of entries in @bos
+ * @exec: drm_exec context to initialize and use for locking
+ * @interruptible: true to use the interruptible dma_resv_lock
+ *
+ * Initializes @exec and uses it to lock all BOs in @bos together with the
+ * NPA VM page directory, retrying on contention. On failure, @exec is
+ * finalized and the error code is returned.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+static int amdgpu_ualink_reserve_npa_vm_and_bos(struct amdgpu_device *adev,
+						struct amdgpu_bo *bos[], u32 n_bos,
+						struct drm_exec *exec,
+						bool interruptible)
+{
+	u32 flags = DRM_EXEC_IGNORE_DUPLICATES;
+	int i, r = 0;
+
+	if (interruptible)
+		flags |= DRM_EXEC_INTERRUPTIBLE_WAIT;
+
+	dev_dbg(adev->dev, "reserve NPA vm and %d bos\n", n_bos);
+
+	drm_exec_init(exec, flags, 0);
+
+	drm_exec_until_all_locked(exec) {
+		for (i = 0; i < n_bos; i++) {
+			r = drm_exec_lock_obj(exec, &bos[i]->tbo.base);
+			drm_exec_retry_on_contention(exec);
+			if (unlikely(r))
+				goto out;
+		}
+
+		r = amdgpu_vm_lock_pd(&adev->ualink.npa_vm, exec, 0);
+		drm_exec_retry_on_contention(exec);
+		if (unlikely(r))
+			goto out;
+	}
+
+out:
+	if (r)
+		drm_exec_fini(exec);
+	return r;
+
+}
+
+/**
+ * amdgpu_ualink_unreserve_npa_vm_and_bos - Release the NPA VM page directory and BOs
+ * @adev: amdgpu device pointer
+ * @exec: drm_exec context previously initialized by
+ *        amdgpu_ualink_reserve_npa_vm_and_bos()
+ *
+ * Finalizes @exec, releasing all locks on the NPA VM page directory and
+ * the associated BOs acquired during reservation.
+ */
+static void amdgpu_ualink_unreserve_npa_vm_and_bos(struct amdgpu_device *adev,
+						   struct drm_exec *exec)
+{
+	dev_dbg(adev->dev, "unreserve NPA vm and bos\n");
+	drm_exec_fini(exec);
+}
+
+/**
+ * amdgpu_ualink_metadata_npa_unmapping - Tear down NPA address mappings
+ * @adev: amdgpu device pointer
+ *
+ * Unmaps all NPA address mappings from the NPA VM for ring buffers,
+ * write pointers, and read pointers. Waits for outstanding DMA fences
+ * and flushes the TLB.
+ */
+static void amdgpu_ualink_metadata_npa_unmapping(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	u64 timeout = msecs_to_jiffies(2000);
+	u32 dst_accel_id = ualink_accel_id(adev);
+	struct amdgpu_bo *bos[2];
+	u32 n_bos;
+	struct dma_fence *fence;
+	struct drm_exec exec;
+	u32 rptr_size, rptr_size_in_pages;
+	u32 rb_size, rb_size_in_pages;
+	u64 pte_flags = adev->gmc.noretry_flags;
+	u64 npa;
+	int r;
+
+	if (!remote->ring_bo)
+		return;
+
+	rb_size = AMDGPU_UALINK_RB_SIZE;
+	rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT;
+
+	bos[0] = remote->ring_bo;
+	n_bos = 1;
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
+		bos[1] = remote->rptr_bo;
+		n_bos = 2;
+	}
+
+	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false);
+	if (unlikely(r))
+		return;
+
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
+		/* rptr npa mapping, up to allocated 2 pages npa address */
+		rptr_size = ualink_wb_size(adev);
+		rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel);
+		rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT;
+
+		npa = remote->rptr_npa;
+
+		amdgpu_ualink_npa_vm_unmap_range(adev, remote->rptr_bo,
+						 pte_flags, 0, rptr_size_in_pages,
+						 npa >> AMDGPU_GPU_PAGE_SHIFT);
+		amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node);
+
+		/*
+		 * unmap wptr, remote interrupt, shootdown ring.
+		 * wptr page is part of the single 2MB mapping for remote GPUs
+		 * interrupt and shootdown ring buffer
+		 */
+		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV,
+					     0, dst_accel_id);
+		amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo, pte_flags,
+						 0,
+						 2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX,
+						 npa >> AMDGPU_GPU_PAGE_SHIFT);
+	} else {
+		u32 wptr_offset = 2 * rb_size * remote->num_accel;
+		u32 idx = 0;
+		u32 accel_id;
+
+		/* source identification mode */
+		for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
+			if (accel_id == dst_accel_id)
+				continue;
+
+			/* remote interrupt ring npa mapping */
+			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
+						     accel_id, dst_accel_id);
+			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
+							 pte_flags, idx * 2 * rb_size,
+							 rb_size_in_pages,
+							 npa >> AMDGPU_GPU_PAGE_SHIFT);
+
+			/* remote shootdown ring npa mapping */
+			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id,
+						     dst_accel_id);
+			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
+							 pte_flags, (idx * 2 + 1) * rb_size,
+							 rb_size_in_pages,
+							 npa >> AMDGPU_GPU_PAGE_SHIFT);
+
+			/* wptr, rptr npa mapping */
+			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id,
+						      dst_accel_id);
+			amdgpu_ualink_npa_vm_unmap_range(adev, remote->ring_bo,
+						  pte_flags, wptr_offset + idx * PAGE_SIZE,
+						  1, npa >> AMDGPU_GPU_PAGE_SHIFT);
+			idx++;
+		}
+	}
+
+	r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false);
+	if (r) {
+		dev_dbg(adev->dev, "failed %d to update directories\n", r);
+		goto out_unreserve;
+	}
+
+	fence = dma_fence_get(adev->ualink.npa_vm.last_update);
+	if (fence) {
+		r = dma_fence_wait_timeout(fence, true, timeout);
+		dma_fence_put(fence);
+		if (r <= 0)
+			dev_dbg(adev->dev, "failed %d to dma fence wait\n", r);
+	}
+
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+out_unreserve:
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+}
+
+/**
+ * amdgpu_ualink_metadata_npa_mapping - Setup NPA address mapping in VM
+ * @adev: amdgpu device pointer
+ *
+ * Maps NPA addresses to GPA for metadata ring buffers, write pointers,
+ * on NPA VMID 15.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_metadata_npa_mapping(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	u32 dst_accel_id = ualink_accel_id(adev);
+	u64 timeout = msecs_to_jiffies(2000);
+	struct amdgpu_bo *bos[2];
+	u32 n_bos;
+	struct dma_fence *fence;
+	struct drm_exec exec;
+	u32 rb_size, rb_size_in_pages;
+	u64 npa, npa_in_pages, pte_flags;
+	int r;
+
+	rb_size = AMDGPU_UALINK_RB_SIZE;
+	rb_size_in_pages = rb_size >> AMDGPU_GPU_PAGE_SHIFT;
+
+	bos[0] = remote->ring_bo;
+	n_bos = 1;
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
+		bos[1] = remote->rptr_bo;
+		n_bos = 2;
+	}
+
+	r = amdgpu_ualink_reserve_npa_vm_and_bos(adev, bos, n_bos, &exec, false);
+	if (unlikely(r))
+		return r;
+
+	pte_flags = amdgpu_ttm_tt_pte_flags(adev, remote->ring_bo->tbo.ttm,
+					    remote->ring_bo->tbo.resource);
+	dev_dbg(adev->dev, "init pte_flags 0x%llx\n", pte_flags);
+
+	amdgpu_gmc_get_vm_pte(adev, &adev->ualink.npa_vm, remote->ring_bo,
+			      AMDGPU_VM_MTYPE_DEFAULT, &pte_flags);
+	dev_dbg(adev->dev, "after get coherent pte_flags 0x%llx\n", pte_flags);
+
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
+		u32 rptr_size, rptr_size_in_pages;
+
+		/* rptr npa mapping, up to allocated 2 pages npa address */
+		rptr_size = ualink_wb_size(adev);
+		rptr_size = AMDGPU_GPU_PAGE_ALIGN(rptr_size * remote->num_accel);
+		rptr_size_in_pages = rptr_size >> AMDGPU_GPU_PAGE_SHIFT;
+
+		r = amdgpu_ualink_npa_alloc_va(adev, &remote->rptr_mm_node,
+					       0, 0, 0,
+					       rptr_size_in_pages);
+		if (r)
+			goto out;
+
+		npa_in_pages = remote->rptr_mm_node.start;
+
+		dev_dbg(adev->dev, "source aliasing rptr alloc 0x%llx and map to npa vm\n",
+			npa_in_pages << AMDGPU_GPU_PAGE_SHIFT);
+
+		r = amdgpu_ualink_npa_vm_map_range(adev, remote->rptr_bo, pte_flags, 0,
+						   rptr_size_in_pages,
+						   npa_in_pages);
+		if (r)
+			goto error_npa_mapping;
+
+		remote->rptr_npa = npa_in_pages << AMDGPU_GPU_PAGE_SHIFT;
+
+		/* wptr, remote interrupt, shootdown ring, single big 2MB mapping */
+		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV,
+					     0, dst_accel_id);
+		r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
+						   0,
+						   2 * rb_size_in_pages * AMDGPU_UALINK_ACCEL_MAX,
+						   npa >> AMDGPU_GPU_PAGE_SHIFT);
+		if (r)
+			goto error_npa_mapping;
+
+	} else {
+		u32 wptr_offset = 2 * rb_size * remote->num_accel;
+		u32 accel_id, idx = 0;
+
+		/* Source identification mode */
+		for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
+			if (accel_id == dst_accel_id)
+				continue;
+
+			/* remote interrupt ring npa mapping */
+			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
+						      accel_id, dst_accel_id);
+			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
+							   idx * 2 * rb_size,
+							   rb_size_in_pages,
+							   npa >> AMDGPU_GPU_PAGE_SHIFT);
+			if (r)
+				goto error_npa_mapping;
+
+			/* remote shootdown ring npa mapping */
+			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TLB_INV, accel_id,
+						     dst_accel_id);
+			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo, pte_flags,
+							   (idx * 2 + 1) * rb_size,
+							   rb_size_in_pages,
+							   npa >> AMDGPU_GPU_PAGE_SHIFT);
+			if (r)
+				goto error_npa_mapping;
+
+			/* wptr, rptr npa mapping */
+			npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR, accel_id,
+						     dst_accel_id);
+			r = amdgpu_ualink_npa_vm_map_range(adev, remote->ring_bo,
+							   pte_flags,
+							   wptr_offset + idx * PAGE_SIZE,
+							   1, npa >> AMDGPU_GPU_PAGE_SHIFT);
+			if (r)
+				goto error_npa_mapping;
+
+			idx++;
+		}
+	}
+
+	r = amdgpu_vm_update_pdes(adev, &adev->ualink.npa_vm, false);
+	if (r) {
+		dev_dbg(adev->dev, "failed %d to update directories\n", r);
+		goto error_npa_mapping;
+	}
+
+	/* TODO: only wait the last fence, then flush TLB */
+	fence = dma_fence_get(adev->ualink.npa_vm.last_update);
+	if (fence) {
+		r = dma_fence_wait_timeout(fence, true, timeout);
+		dma_fence_put(fence);
+		if (r <= 0)
+			dev_dbg(adev->dev, "failed %d to dma fence wait\n", r);
+	}
+
+	amdgpu_ualink_flush_tlb(adev, TLB_FLUSH_HEAVYWEIGHT);
+
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+	return 0;
+
+error_npa_mapping:
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+		amdgpu_ualink_npa_free_va(adev, &remote->rptr_mm_node);
+	if (r)
+		dev_dbg(adev->dev, "failed %d to map NPA vm\n", r);
+
+out:
+	amdgpu_ualink_unreserve_npa_vm_and_bos(adev, &exec);
+
+	return r;
+}
+
+/**
+ * amdgpu_ualink_sdma_entities_init - Initialize SDMA job scheduler entities
+ * @adev: amdgpu device pointer
+ *
+ * Sets up SDMA job scheduler entities for remote interrupt and shootdown.
+ * Each entity uses one SDMA scheduler/ring.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_sdma_entities_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct drm_gpu_scheduler *sched;
+	u32 accel_id;
+	int i = 0, r;
+
+	dev_dbg(adev->dev, "enter\n");
+
+	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
+		sched = &adev->sdma.instance[accel_id % adev->sdma.num_instances].ring.sched;
+
+		r = drm_sched_entity_init(&remote->peer[accel_id].entity, DRM_SCHED_PRIORITY_HIGH,
+					  &sched, 1, NULL);
+		if (r)
+			goto out_free;
+		i++;
+	}
+
+	remote->use_lsdma = true;
+
+	dev_dbg(adev->dev, "exit\n");
+	return 0;
+
+out_free:
+	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
+		if (i--)
+			drm_sched_entity_destroy(&remote->peer[accel_id].entity);
+		else
+			break;
+	}
+
+	return r;
+}
+
+static void amdgpu_ualink_sdma_entities_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	u32 accel_id;
+
+	dev_dbg(adev->dev, "enter\n");
+	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX)
+		drm_sched_entity_destroy(&remote->peer[accel_id].entity);
+	dev_dbg(adev->dev, "exit\n");
+}
+
+/**
+ * amdgpu_ualink_gart_map - Allocate GART entry and map NPA address
+ * @adev: amdgpu device pointer
+ * @npages: Number of pages to map
+ * @npa: NPA address to map
+ * @mm_node: DRM memory manager node for GART allocation
+ * @pte_flags: the GART mapping flags
+ *
+ * Allocates GART entries and sets up NPA address mapping without TTM BO.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_gart_map(struct amdgpu_device *adev, u64 npages,
+				   u64 npa, struct drm_mm_node *mm_node,
+				   u64 pte_flags)
+{
+	struct ttm_resource_manager *man =
+				ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
+	struct amdgpu_gtt_mgr *mgr =
+				container_of(man, struct amdgpu_gtt_mgr, manager);
+	dma_addr_t *dma_addr;
+	int i, r;
+
+	dev_dbg(adev->dev, "npa 0x%llx npages 0x%llx\n", npa, npages);
+
+	if (npages > 1) {
+		dma_addr = kmalloc_array(npages, sizeof(*dma_addr), GFP_KERNEL);
+		if (!dma_addr)
+			return -ENOMEM;
+
+		for (i = 0; i < npages; i++)
+			dma_addr[i] = npa + i * AMDGPU_GPU_PAGE_SIZE;
+	} else {
+		dma_addr = (dma_addr_t *)&npa;
+	}
+
+	r = amdgpu_gtt_mgr_alloc_entries(mgr, mm_node, npages, DRM_MM_INSERT_BEST);
+	if (r)
+		goto out;
+	amdgpu_gart_bind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, dma_addr,
+			 pte_flags);
+
+out:
+	dev_dbg(adev->dev, "npa 0x%llx mapped to 0x%llx npages 0x%llx r=%d\n",
+		npa, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages, r);
+
+	if (npages > 1)
+		kfree(dma_addr);
+	return r;
+}
+
+/**
+ * amdgpu_ualink_gart_unmap - Unmap and free GART entry
+ * @adev: amdgpu device pointer
+ * @npages: Number of pages to unmap
+ * @mm_node: DRM memory manager node for GART allocation
+ *
+ * Unbinds GART mapping and frees allocated entries.
+ */
+static void amdgpu_ualink_gart_unmap(struct amdgpu_device *adev, u64 npages,
+				      struct drm_mm_node *mm_node)
+{
+	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
+	struct amdgpu_gtt_mgr *mgr =
+				container_of(man, struct amdgpu_gtt_mgr, manager);
+
+	dev_dbg(adev->dev, "0x%llx npages 0x%llx\n", mm_node->start << AMDGPU_GPU_PAGE_SHIFT,
+		npages);
+
+	if (!drm_mm_node_allocated(mm_node))
+		return;
+	amdgpu_gart_unbind(adev, mm_node->start << AMDGPU_GPU_PAGE_SHIFT, npages);
+	amdgpu_gtt_mgr_free_entries(mgr, mm_node);
+}
+
+/**
+ * amdgpu_ualink_metadata_fini - Clean up ualink metadata structures
+ * @adev: amdgpu device pointer
+ *
+ * Unmaps NPA addresses and frees all allocated buffers for metadata,
+ * ring buffers, and pointers.
+ */
+static void amdgpu_ualink_metadata_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+
+	dev_dbg(adev->dev, "accel_id %u\n", ualink_accel_id(adev));
+	amdgpu_bo_free_kernel(&remote->rptr_bo, &remote->rptr_gpu_addr,
+				      &remote->rptr_cpu_addr);
+	amdgpu_bo_free_kernel(&remote->ring_bo, &remote->rb_gpu_addr,
+			      &remote->rb_cpu_addr);
+	amdgpu_bo_free_kernel(&remote->metadata_bo, &remote->metadata_gpu_addr,
+			      &remote->metadata_cpu_addr);
+}
+
+/**
+ * amdgpu_ualink_metadata_init - Initialize ualink metadata structures
+ * @adev: amdgpu device pointer
+ *
+ * Allocates and initializes metadata structures, ring buffers, and write/read
+ * pointers for multi-GPU ualink. Communicates with firmware to load metadata.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_metadata_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_ualink_metadata *metadata;
+	u32 size, rb_size, wptr_size, rptr_size, metadata_size;
+	u64 rb_gpu_addr, wptr_gpu_addr;
+	u32 status, accel_id;
+	int r;
+
+	bitmap_copy(remote->active_accel_bits,
+		    adev->ualink.info->vpod.active_accel_bits,
+		    AMDGPU_UALINK_ACCEL_MAX);
+	dev_dbg(adev->dev, "%d active accelerators config in vpod\n",
+		bitmap_weight(remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX));
+
+	/*
+	 * allocate metadata entries and ring buffer for all remote GPUs,
+	 * to get 2MB page ring buffer NPA mapping for remote access.
+	 */
+	remote->num_accel = AMDGPU_UALINK_ACCEL_MAX;
+
+	status = adev->ualink.msg_ctl->check_status(adev);
+	if (status != AMDGPU_NHT_FW_ST_PREINIT &&
+	    status != AMDGPU_NHT_FW_ST_HALT) {
+		dev_dbg(adev->dev, "fw status 0x%x not preinit or halt\n", status);
+		return -ENODEV;
+	}
+
+	dev_dbg(adev->dev, "accel_id %u addr_mode %d fw status 0x%x\n",
+		ualink_accel_id(adev), ualink_addr_mode(adev), status);
+
+	/* Alloc metadata structure for all GPUs */
+	metadata_size = sizeof(struct amdgpu_ualink_metadata) * AMDGPU_UALINK_ACCEL_MAX;
+	metadata_size = AMDGPU_GPU_PAGE_ALIGN(metadata_size);
+
+	dev_dbg(adev->dev, "metadata size 0x%x\n", metadata_size);
+
+	/* pinned system memory */
+	r = amdgpu_bo_create_kernel(adev, metadata_size, PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_GTT,
+				    &remote->metadata_bo, &remote->metadata_gpu_addr,
+				    &remote->metadata_cpu_addr);
+	if (r)
+		goto out;
+
+	memset(remote->metadata_cpu_addr, 0, metadata_size);
+	metadata = remote->metadata_cpu_addr;
+
+	dev_dbg(adev->dev, "metadata gpu address 0x%llx\n", remote->metadata_gpu_addr);
+
+	/* Allocate ring buffers, wptr for remote interrupt and shootdown */
+	rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE);
+	wptr_size = ualink_wptr_size(adev);
+	size = (rb_size + wptr_size) * remote->num_accel;
+	size = AMDGPU_GPU_PAGE_ALIGN(size);
+
+	dev_dbg(adev->dev, "rb_size 0x%x wptr_size 0x%x total alloc size 0x%x\n",
+		rb_size, wptr_size, size);
+
+	/* pinned VRAM */
+	r = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
+				    AMDGPU_GEM_DOMAIN_VRAM,
+				    &remote->ring_bo,
+				    &remote->rb_gpu_addr,
+				    &remote->rb_cpu_addr);
+	if (r)
+		goto out;
+
+	memset(remote->rb_cpu_addr, 0, size);
+
+	dev_dbg(adev->dev, "rb gpu addr 0x%llx cpu addr 0x%p vram_start 0x%llx vram_base 0x%llx\n",
+		remote->rb_gpu_addr, remote->rb_cpu_addr, adev->gmc.vram_start,
+		adev->vm_manager.vram_base_offset);
+
+	/*
+	 * address aliasing mode, shared wptr pagee left is not enough for wb data,
+	 * wb status and rptr, alloc another BO, and then alloc npa address and map
+	 * to npa vm, for remote to access.
+	 *
+	 * No gart mapping required for wb data, status and rptr because this is
+	 * updated by firmware.
+	 */
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS) {
+		rptr_size = AMDGPU_GPU_PAGE_ALIGN(ualink_wb_size(adev) * remote->num_accel);
+		dev_dbg(adev->dev, "source aliasing mode rptr_size 0x%x\n", rptr_size);
+
+		/* pinned VRAM */
+		r = amdgpu_bo_create_kernel(adev, rptr_size, PAGE_SIZE,
+					    AMDGPU_GEM_DOMAIN_VRAM,
+					    &remote->rptr_bo,
+					    &remote->rptr_gpu_addr,
+					    &remote->rptr_cpu_addr);
+		if (r)
+			goto out;
+
+		memset(remote->rptr_cpu_addr, 0, rptr_size);
+
+		dev_dbg(adev->dev, "source aliasing rptr gpu addr 0x%llx cpu addr 0x%p\n",
+			remote->rptr_gpu_addr, remote->rptr_cpu_addr);
+	}
+
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS)
+		wptr_gpu_addr = remote->rb_gpu_addr + rb_size * ualink_accel_id(adev);
+
+	for_each_set_bit(accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
+		dev_dbg(adev->dev, "init for accel_id %u\n", accel_id);
+
+		if (accel_id == ualink_accel_id(adev))
+			continue;
+
+		rb_gpu_addr = remote->rb_gpu_addr + rb_size * accel_id;
+
+		if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
+			wptr_gpu_addr = remote->rb_gpu_addr + rb_size * remote->num_accel;
+			wptr_gpu_addr += wptr_size * accel_id;
+		}
+
+		metadata[accel_id].header = AMDGPU_UALINK_METADATA_HEADER;
+		metadata[accel_id].rb_size = fls(AMDGPU_UALINK_RB_SIZE / 64) - 1;
+
+		/*
+		 * remote shootdown type is 0, remote command type is 1
+		 * with 2MB ring buffer mapping, remote shootdown ring NPA is before interrupt
+		 */
+		metadata[accel_id].ri_rb = rb_gpu_addr + AMDGPU_UALINK_RB_SIZE;
+		metadata[accel_id].tlb_inv_rb = rb_gpu_addr;
+
+		metadata[accel_id].tailptr_ri = wptr_gpu_addr +
+						ualink_wptr_offset(adev, accel_id);
+		metadata[accel_id].tailptr_tlb_inv = wptr_gpu_addr +
+						     ualink_tlb_wptr_offset(adev, accel_id);
+
+		dev_dbg(adev->dev, "init from accel_id %u to accel_id %u, rb_size 0x%x\n",
+			accel_id, ualink_accel_id(adev), metadata[accel_id].rb_size);
+		dev_dbg(adev->dev, "rb 0x%llx tlb rb 0x%llx\n",
+			metadata[accel_id].ri_rb, metadata[accel_id].tlb_inv_rb);
+		dev_dbg(adev->dev, "rb wptr at 0x%llx tlb wptr at 0x%llx\n",
+			metadata[accel_id].tailptr_ri, metadata[accel_id].tailptr_tlb_inv);
+	}
+
+	r = adev->ualink.msg_ctl->send_metadata(adev,
+						    remote->metadata_gpu_addr,
+						    AMDGPU_UALINK_ACCEL_MAX << 8);
+
+out:
+	if (r)
+		amdgpu_ualink_metadata_fini(adev);
+
+	dev_dbg(adev->dev, "ret 0x%x\n", r);
+	return r;
+}
+
+/*
+ * nHT Firmware Error code
+ *
+ * Success				0x0
+ * FIFO overflow			0x1
+ * Invalid RB command			0x2
+ * Timeout				0x3
+ * Invalid metadata entry		0x4
+ * RB overflow				0x5
+ * Invalid wptr address			0x6
+ * Interrupt cookie send failure	0x7
+ * Invalid RB address			0x8
+ * Invalid writeback address		0x9
+ */
+static void amdgpu_ualink_remote_error(struct amdgpu_device *adev, u32 status)
+{
+	const char *fw_err_code_msg[] = {
+		"Unknown FW error status",
+		"FIFO overflow",		/* error code 1 */
+		"Invalid RB command",
+		"Timeout",
+		"Invalid metadata entry",
+		"RB overflow",
+		"Invalid wptr address",
+		"Interrupt cookie send failure",
+		"Invalid RB address",
+		"Invalid writeback address"	/* error code 9 */
+		};
+
+	if (status >= ARRAY_SIZE(fw_err_code_msg))
+		status = 0;
+
+	dev_err(adev->dev, "remote error %s\n", fw_err_code_msg[status]);
+	return;
+}
+
+/**
+ * amdgpu_ualink_remote_wait_timeout - Wait for remote operation completion
+ * @remote_accel_id: remote accelator id to wait for reply
+ * @adev: amdgpu device pointer
+ * @wb_cpu: CPU virtual address of writeback buffer
+ * @seq: Sequence number to wait for
+ *
+ * Polls the writeback buffer waiting for the sequence number to reach or
+ * exceed the expected value. The writeback buffer contains status and data
+ * fields updated by firmware to indicate completion and error conditions.
+ *
+ * Return: 0 on success
+ *         -ETIME on timeout
+ *         -ECOMM on firmware return error status
+ */
+static long amdgpu_ualink_remote_wait_timeout(struct amdgpu_device *adev,
+					      u32 remote_accel_id,
+					      struct amdgpu_ualink_wb *wb_cpu,
+					      u32 seq)
+{
+	/* 2 seconds timeout, long enough for FW to reply */
+	ktime_t timeout = ktime_add_us(ktime_get(), 2 * USEC_PER_SEC);
+	u32 status, data;
+	u64 rptr;
+
+	while (true) {
+		data = READ_ONCE(wb_cpu->data);
+		if (data >= seq) {
+			status = READ_ONCE(wb_cpu->status);
+			rptr = READ_ONCE(wb_cpu->rptr);
+			break;
+		}
+
+		if (ktime_after(ktime_get(), timeout)) {
+			dev_dbg(adev->dev, "remote %d wait timeout\n", remote_accel_id);
+			return -ETIME;
+		}
+
+		usleep_range(10, 50);
+	}
+
+	if (status) {
+		dev_dbg(adev->dev, "remote %u status 0x%x wb data 0x%x seq 0x%x rptr 0x%llx\n",
+			remote_accel_id, status, data, seq, rptr);
+
+		amdgpu_ualink_remote_error(adev, status);
+		return -ECOMM;
+	}
+
+	dev_dbg(adev->dev, "remote %u succeed seq 0x%x wb data 0x%x rptr 0x%llx\n",
+		remote_accel_id, seq, data, rptr);
+	return 0;
+}
+
+#define AMDGPU_UALINK_REMOTE_OP_TLB_INV		0x1
+#define AMDGPU_UALINK_REMOTE_OP_INT		0x2
+#define AMDGPU_UALINK_REMOTE_OP_UPDATE_WB	0x3
+
+static void amdgpu_ualink_emit_shootdown(u32 **cpu_addr_p, u32 wb_data,
+					 u32 dw0,  u32 dw1, u32 dw2, u32 dw3)
+{
+	u64 addr = (((u64)dw1 << 32) | (u64)dw2) << AMDGPU_GPU_PAGE_SHIFT;
+	u32 *cpu_addr = *cpu_addr_p;
+	u32 flush_type = dw0;
+	u32 size_in_pages = dw3;
+	u32 cmd;
+
+	cmd = AMDGPU_UALINK_REMOTE_OP_TLB_INV;
+	cmd |= 1 << 28;	/* headptr update */
+	cmd |= 1 << 29;	/* wb enable */
+
+	*cpu_addr++ = cmd;
+	*cpu_addr++ = wb_data;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0; /* metadata */
+
+	pr_debug("NPA addr 0x%llx npages 0x%x\n", addr, size_in_pages);
+
+	/* Always shootdown everything with full address size s-field coding */
+	addr = GENMASK_ULL(51, 11);
+
+	*cpu_addr++ = lower_32_bits(addr) | flush_type;
+	*cpu_addr++ = upper_32_bits(addr);
+
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr_p = cpu_addr;
+}
+
+static void amdgpu_ualink_emit_interrupt(u32 **cpu_addr_p, u32 wb_data,
+					 u32 dw0, u32 dw1, u32 dw2, u32 dw3)
+{
+	u32 *cpu_addr = *cpu_addr_p;
+	u32 cmd;
+
+	cmd = AMDGPU_UALINK_REMOTE_OP_INT;
+	cmd |= 1 << 28;	/* headptr_update */
+	cmd |= 1 << 29; /* wb enable */
+
+	*cpu_addr++ = cmd;
+	*cpu_addr++ = wb_data;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0; /* metadata */
+	/* updated by f/w, overwrite PASID with GPU ID */
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	/* IH ContextID 4 dwords */
+	*cpu_addr++ = dw0;
+	*cpu_addr++ = dw1;
+	*cpu_addr++ = dw2;
+	*cpu_addr++ = dw3;
+	*cpu_addr_p = cpu_addr;
+}
+
+static void amdgpu_ualink_emit_update_wb_addr(u32 **cpu_addr_p, u32 wb_data,
+					      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
+{
+	u32 *cpu_addr = *cpu_addr_p;
+	u32 cmd;
+
+	cmd = AMDGPU_UALINK_REMOTE_OP_UPDATE_WB;
+	cmd |= 1 << 28;	/* headptr update */
+	cmd |= 1 << 29;	/* wb enable */
+
+	*cpu_addr++ = cmd;
+	*cpu_addr++ = wb_data;
+	*cpu_addr++ = dw1 & 0xFFFFFFFC;	/* lower32 wb_npa */
+	*cpu_addr++ = dw0 & 0xFFFFF;	/* upper32 wb_npa */
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0; /* metadata */
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr++ = 0;
+	*cpu_addr_p = cpu_addr;
+}
+
+/*
+ * Function proto to generate packet using above different emit_ functions
+ */
+typedef void (*ualink_emit_packet)(u32 **cpu_addr_p, u32 wb, u32 dw0,
+				   u32 dw1, u32 dw2, u32 dw3);
+
+/*
+ * Refresh rptr from the writeback and check the ring has room for one more
+ * command. The caller must hold the peer lock.
+ *
+ * Return 0 on success or a negative error code.
+ */
+static int amdgpu_ualink_ring_check_space(struct amdgpu_device *adev,
+					  u32 remote_accel_id,
+					  struct amdgpu_ualink_ring *ring,
+					  struct amdgpu_ualink_wb *wb_cpu)
+{
+	ring->rptr = READ_ONCE(wb_cpu->rptr);
+
+	if (WARN_ON_ONCE(ring->rptr > ring->wptr)) {
+		dev_err(adev->dev, "accel_id %u ring overflow wptr 0x%llx rptr 0x%llx\n",
+			remote_accel_id, ring->wptr, ring->rptr);
+		return -EFAULT;
+	}
+
+	if ((ring->wptr + 1 - ring->rptr) >= ring->rb_size) {
+		dev_err(adev->dev, "accel_id %u command ring full wptr 0x%llx rptr 0x%llx\n",
+			remote_accel_id, ring->wptr, ring->rptr);
+		return -ENOSPC;
+	}
+
+	return 0;
+}
+
+static int amdgpu_ualink_send_command(struct amdgpu_device *adev,
+				      u32 remote_accel_id,
+				      struct amdgpu_ualink_ring *ring,
+				      struct amdgpu_ualink_wb *wb_cpu,
+				      ualink_emit_packet emit_func,
+				      u32 dw0, u32 dw1, u32 dw2, u32 dw3)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_ualink_peer *peer;
+	struct amdgpu_ring *sdma_ring;
+	struct dma_fence *fence;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
+	u32 seq, ndw, ndw_copy_cmd, rem;
+	u64 doorbell_npa_gart;
+	u32 *src_cpu;
+	u64 src;
+	int r;
+
+	/* Refuse sends unless active */
+	if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
+		return -ESHUTDOWN;
+
+	peer = &remote->peer[remote_accel_id];
+
+	r = amdgpu_ualink_ring_check_space(adev, remote_accel_id, ring, wb_cpu);
+	if (r)
+		goto out;
+
+	/*
+	 * 3 sdma copy commands: write data to ring buffer, update wptr, ring doorbell
+	 *
+	 * insert 2 NOP commands to break SDMA back-to-back copy command overlap, to
+	 * ensure the ordering of 3 sdma copy commands execution.
+	 */
+
+	/* buffer after sdma commands, starting at 8 dwords boundary */
+	ndw_copy_cmd = ALIGN(3 * adev->mman.buffer_funcs->copy_num_dw + 2, 8);
+
+	 /* remote command 16 dwords, wptr 2 dwords, doorbell 1 dword */
+	ndw = ndw_copy_cmd + 16 + 2 + 1;
+	r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM,
+				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
+				     AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job);
+	if (r)
+		goto out;
+
+	ib = &job->ibs[0];
+	src = ib->gpu_addr + ndw_copy_cmd * 4;
+	src_cpu = ib->ptr + ndw_copy_cmd;
+
+	seq = ++ring->seq;
+
+	div_u64_rem(ring->wptr, ring->rb_size, &rem);
+	dev_dbg(adev->dev, "src 0x%llx to npa gart rb 0x%llx wptr 0x%llx doorbell 0x%llx seq 0x%x\n",
+		src, ring->rb_npa_gart + rem * 64,
+		ring->wptr_npa_gart, ring->doorbell_npa_gart, seq);
+
+	dev_dbg(adev->dev, "ring wptr 0x%llx rptr 0x%llx\n", ring->wptr, ring->rptr);
+
+	/* remote command packet */
+	emit_func(&src_cpu, seq, dw0, dw1, dw2, dw3);
+
+	/* wptr value */
+	*src_cpu++ = lower_32_bits(ring->wptr + 1);
+	*src_cpu++ = upper_32_bits(ring->wptr + 1);
+
+	/* doorbell value */
+	*src_cpu++ = lower_32_bits(ring->wptr + 1);
+
+	/*
+	 * Add an offset to the doorbell address that cycles through different
+	 * values of bit 11:8 to use different DXS ports.
+	 */
+	peer->dxs_port = (peer->dxs_port + 1) & 0xF;
+	doorbell_npa_gart = ring->doorbell_npa_gart | (peer->dxs_port << 8);
+	dev_dbg(adev->dev, "dxs_port 0x%x, doorbell_npa_gart 0x%llx -> 0x%llx\n",
+		peer->dxs_port, ring->doorbell_npa_gart, doorbell_npa_gart);
+
+	dev_dbg(adev->dev, "use %s\n", remote->use_lsdma ? "lsdma" : "sdma");
+
+	if (remote->use_lsdma) {
+		div_u64_rem(ring->wptr, ring->rb_size, &rem);
+		r = amdgpu_lsdma_copy_mem(adev, src,
+					  ring->rb_npa_gart + rem * 64,
+					  64);
+		if (!r)
+			r = amdgpu_lsdma_copy_mem(adev, src + 64,
+						  ring->wptr_npa_gart, 8);
+		if (!r)
+			r = amdgpu_lsdma_copy_mem(adev, src + 72,
+						  doorbell_npa_gart, 4);
+
+		amdgpu_job_free(job);
+
+		/* Copy failed: command never reached the remote, so no
+		 * writeback will arrive. Return now instead of polling the
+		 * completion for the full remote timeout.
+		 */
+		if (r) {
+			dev_dbg(adev->dev,
+				"remote %u lsdma copy failed (r %d), skip completion wait\n",
+				remote_accel_id, r);
+			return r;
+		}
+
+		goto out_wait_complete;
+	}
+
+	sdma_ring = &adev->sdma.instance[0].ring;
+
+	/*
+	 * SDMA commands copy from ib to NPA, insert NOPs to ensure SDMA copy commands
+	 * execution doesn't overlap, doorbell update is after data and wptr update
+	 * finished.
+	 */
+	div_u64_rem(ring->wptr, ring->rb_size, &rem);
+	amdgpu_emit_copy_buffer(adev, ib, src,
+				ring->rb_npa_gart + rem * 64,
+				64, 0);
+
+	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
+
+	amdgpu_emit_copy_buffer(adev, ib, src + 64, ring->wptr_npa_gart, 8, 0);
+
+	ib->ptr[ib->length_dw++] = sdma_ring->funcs->nop;
+
+	amdgpu_emit_copy_buffer(adev, ib, src + 72, doorbell_npa_gart, 4, 0);
+
+	amdgpu_ring_pad_ib(sdma_ring, ib);
+	WARN_ON(ib->length_dw > ndw_copy_cmd);
+
+	fence = amdgpu_job_submit(job);
+	r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT);
+	dma_fence_put(fence);
+	if (r <= 0) {
+		dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n",
+			remote_accel_id, r);
+		if (r == 0)
+			r = -ETIME;
+		goto out;
+	}
+
+out_wait_complete:
+	/*
+	 * poll remote completion writeback data
+	 */
+	r = amdgpu_ualink_remote_wait_timeout(adev, remote_accel_id, wb_cpu, seq);
+
+	/*
+	 * Advance wptr unless the remote timed out, even if the FW returned an
+	 * error status. On timeout the caller resets the connection, which
+	 * resyncs wptr from the remote in amdgpu_ualink_setup_connection.
+	 */
+	if (r != -ETIME)
+		ring->wptr++;
+
+out:
+	dev_dbg(adev->dev, "ret r = %d\n", r);
+	return r;
+}
+
+static void amdgpu_ualink_get_wb_addr(struct amdgpu_device *adev,
+				      u32 remote_accel_id,
+				      struct amdgpu_ualink_wb **wb_cpu,
+				      u64 *wb_npa, u32 type)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	u32 accel_id = ualink_accel_id(adev);
+	u32 rb_size = AMDGPU_GPU_PAGE_ALIGN(2 * AMDGPU_UALINK_RB_SIZE);
+	uintptr_t wb;
+	u64 npa;
+	u32 offset = 0;
+
+	if (ualink_addr_mode(adev) == AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT) {
+		wb = (uintptr_t)remote->rb_cpu_addr + rb_size * remote->num_accel;
+		npa = amdgpu_ualink_npa_addr(adev, RB_TYPE_TAILPTR,
+					     remote_accel_id, accel_id);
+	} else {
+		wb = (uintptr_t)remote->rptr_cpu_addr;
+		npa = remote->rptr_npa;
+		npa |= (u64)accel_id << AMDGPU_UALINK_GART_NPA_ADDR_GPUID_SHIFT;
+	}
+
+	if (type == RB_TYPE_TLB_INV)
+		offset = ualink_tlb_wb_offset(adev, remote_accel_id);
+	else if (type == RB_TYPE_REMOTE_INTERRUPT)
+		offset = ualink_wb_offset(adev, remote_accel_id);
+
+	*wb_cpu = (struct amdgpu_ualink_wb *)(wb + offset);
+	if (wb_npa)
+		*wb_npa = npa + offset;
+
+	dev_dbg(adev->dev, "source %d remote %d wb npa 0x%llx\n", accel_id,
+		remote_accel_id, npa + offset);
+}
+
+struct amdgpu_ualink_wptr_fence_cb {
+	struct dma_fence_cb base;
+	u64 *wptr_cpu;
+	u64 result;
+};
+
+static void amdgpu_ualink_read_wptr_fence_cb(struct dma_fence *fence,
+					     struct dma_fence_cb *cb)
+{
+	struct amdgpu_ualink_wptr_fence_cb *wptr_cb;
+
+	wptr_cb = container_of(cb, typeof(*wptr_cb), base);
+	wptr_cb->result = *wptr_cb->wptr_cpu;
+}
+
+static int amdgpu_ualink_read_remote_wptr(struct amdgpu_device *adev,
+					  u32 remote_accel_id,
+					  struct amdgpu_ualink_ring *ring,
+					  u64 *wptr)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_ualink_peer *peer = &remote->peer[remote_accel_id];
+	struct amdgpu_ualink_wptr_fence_cb wptr_cb = { };
+	struct amdgpu_ring *sdma_ring;
+	struct dma_fence *fence;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
+	u32 ndw, ndw_copy_cmd;
+	u64 wptr_gpu, *wptr_cpu;
+	int r;
+
+	*wptr = 0;
+
+	/* Refuse remote access unless active */
+	if (adev->ualink.info->accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE)
+		return -ESHUTDOWN;
+
+	/* 1 sdma copy command to read wptr */
+	ndw_copy_cmd = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
+
+	/* wptr 2 dwords */
+	ndw = ndw_copy_cmd + 2;
+	r = amdgpu_job_alloc_with_ib(adev, &peer->entity, AMDGPU_FENCE_OWNER_VM,
+				     ndw * 4, AMDGPU_IB_POOL_IMMEDIATE,
+				     AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER, &job);
+	if (r)
+		return r;
+
+	ib = &job->ibs[0];
+	wptr_gpu = ib->gpu_addr + ndw_copy_cmd * 4;
+	wptr_cpu = (u64 *)(ib->ptr + ndw_copy_cmd);
+
+	dev_dbg(adev->dev, "read wptr from remote %u npa gart wptr 0x%llx use %s\n",
+		remote_accel_id, ring->wptr_npa_gart,
+		remote->use_lsdma ? "lsdma" : "sdma");
+
+	if (remote->use_lsdma) {
+		r = amdgpu_lsdma_copy_mem(adev, ring->wptr_npa_gart, wptr_gpu, 8);
+		if (!r)
+			*wptr = *wptr_cpu;
+		amdgpu_job_free(job);
+		goto out;
+	}
+
+	wptr_cb.wptr_cpu = wptr_cpu;
+
+	amdgpu_emit_copy_buffer(adev, ib, ring->wptr_npa_gart, wptr_gpu, 8, 0);
+
+	sdma_ring = &adev->sdma.instance[0].ring;
+	amdgpu_ring_pad_ib(sdma_ring, ib);
+	WARN_ON(ib->length_dw > ndw_copy_cmd);
+
+	fence = amdgpu_job_submit(job);
+
+	r = dma_fence_add_callback(fence, &wptr_cb.base,
+				   amdgpu_ualink_read_wptr_fence_cb);
+	if (r == -ENOENT) {
+		/* Fence already signaled; the callback won't run, do it inline. */
+		amdgpu_ualink_read_wptr_fence_cb(fence, &wptr_cb.base);
+		r = 0;
+		goto out_put_fence;
+	} else if (r) {
+		/* -EINVAL: NULL fence or func, not expected here. */
+		goto out_put_fence;
+	}
+
+	r = dma_fence_wait_timeout(fence, false, AMDGPU_FENCE_JIFFIES_TIMEOUT);
+	if (r > 0) {
+		r = 0;	/* read wptr successfully */
+		goto out_put_fence;
+	}
+
+	dev_dbg(adev->dev, "remote %u sdma fence wait return r %d\n",
+		remote_accel_id, r);
+
+	if (r == 0)
+		r = -ETIME;
+
+	/*
+	 * Timed out: the callback is still armed and wptr_cb is on our stack.
+	 * Cancel it so it can't fire after we return and write into the freed
+	 * stack frame.
+	 */
+	dma_fence_remove_callback(fence, &wptr_cb.base);
+
+out_put_fence:
+	if (!r)
+		*wptr = wptr_cb.result;
+	dma_fence_put(fence);
+
+out:
+	dev_dbg(adev->dev, "remote %u wptr 0x%llx return %d\n",
+		remote_accel_id, *wptr, r);
+	return r;
+}
+
+/**
+ * amdgpu_ualink_ring_resync_ptrs - sync local ring pointers from remote
+ * @adev: amdgpu device pointer
+ * @remote_accel_id: remote accelerator ID
+ *
+ * After a reboot on either side the two ends disagree on wptr. Read the remote
+ * wptr of both peer rings and adopt it as the local wptr and rptr.
+ *
+ * Return 0 if both rings already match, 1 if they were resynced, or a negative
+ * error code if a remote read failed, leaving the rings untouched.
+ */
+static int amdgpu_ualink_ring_resync_ptrs(struct amdgpu_device *adev,
+					  u32 remote_accel_id)
+{
+	struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id];
+	struct amdgpu_ualink_wb *int_wb, *tlb_wb;
+	u64 int_wptr, tlb_wptr;
+	int r;
+
+	/* ring->wptr and ring->rptr are owned by the send path. */
+	guard(mutex)(&peer->lock);
+
+	/* Read both before updating either, so a failure changes nothing. */
+	r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id,
+					   &peer->interrupt, &int_wptr);
+	if (!r)
+		r = amdgpu_ualink_read_remote_wptr(adev, remote_accel_id,
+						   &peer->shootdown, &tlb_wptr);
+	if (r) {
+		dev_dbg(adev->dev, "accel_id %u read remote %u wptr failed %d\n",
+			ualink_accel_id(adev), remote_accel_id, r);
+		return r;
+	}
+
+	if (peer->interrupt.wptr == int_wptr && peer->shootdown.wptr == tlb_wptr) {
+		dev_dbg(adev->dev, "accel_id %u wptr 0x%llx 0x%llx is same as remote %u\n",
+			ualink_accel_id(adev), int_wptr, tlb_wptr, remote_accel_id);
+		return 0;
+	}
+
+	dev_dbg(adev->dev,
+		"accel_id %u wptr 0x%llx 0x%llx sync to remote %u wptr 0x%llx 0x%llx\n",
+		ualink_accel_id(adev), peer->interrupt.wptr, peer->shootdown.wptr,
+		remote_accel_id, int_wptr, tlb_wptr);
+
+	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &int_wb, NULL,
+				  RB_TYPE_REMOTE_INTERRUPT);
+	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &tlb_wb, NULL,
+				  RB_TYPE_TLB_INV);
+
+	peer->interrupt.wptr = int_wptr;
+	peer->interrupt.rptr = int_wptr;
+	peer->shootdown.wptr = tlb_wptr;
+	peer->shootdown.rptr = tlb_wptr;
+
+	/* Drop the stale writeback rptr, it disagrees with the resynced wptr. */
+	WRITE_ONCE(int_wb->rptr, int_wptr);
+	WRITE_ONCE(tlb_wb->rptr, tlb_wptr);
+	return 1;
+}
+
+static void amdgpu_ualink_reset_peer_rings(struct amdgpu_device *adev,
+					   u32 remote_accel_id)
+{
+	struct amdgpu_ualink_peer *peer = &to_remote(adev)->peer[remote_accel_id];
+
+	dev_dbg(adev->dev, "remote accel_id %u\n", remote_accel_id);
+
+	guard(mutex)(&peer->lock);
+
+	peer->interrupt.ready = false;
+	peer->shootdown.ready = false;
+}
+
+static int amdgpu_ualink_update_wb_address(struct amdgpu_device *adev,
+					   u32 remote_accel_id, u32 ring_type)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_ualink_peer *peer;
+	struct amdgpu_ualink_ring *ring;
+	struct amdgpu_ualink_wb *wb_cpu;
+	u64 wb_npa;
+	int r;
+
+	peer = &remote->peer[remote_accel_id];
+	if (ring_type == RB_TYPE_REMOTE_INTERRUPT)
+		ring = &peer->interrupt;
+	else if (ring_type == RB_TYPE_TLB_INV)
+		ring = &peer->shootdown;
+	else
+		return -EINVAL;
+
+	amdgpu_ualink_get_wb_addr(adev, remote_accel_id, &wb_cpu, &wb_npa,
+				  ring_type);
+
+	r = amdgpu_ualink_send_command(adev, remote_accel_id, ring, wb_cpu,
+				       amdgpu_ualink_emit_update_wb_addr,
+				       upper_32_bits(wb_npa),
+				       lower_32_bits(wb_npa),
+				       0, 0);
+	if (!r) {
+		ring->wb_cpu = wb_cpu;
+		ring->ready = true;
+	}
+	return r;
+}
+
+/**
+ * amdgpu_ualink_remote_shootdown - Send a remote TLB invalidation request
+ * @adev: amdgpu device pointer
+ * @remote_accel_id: accelerator ID of the remote GPU to shootdown
+ * @addr: page-aligned address to invalidate
+ * @size_in_pages: number of pages to invalidate, 0 to invalidate entire TLB
+ * @flush_type: type of TLB flush to perform
+ *
+ * Sends a remote TLB shootdown command to the specified peer GPU via the
+ * UALink ring buffer. The address and size are translated using the S-field
+ * encoding before being written to the ring.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_remote_shootdown(struct amdgpu_device *adev,
+				   u32 remote_accel_id, u64 addr,
+				   u32 size_in_pages, u32 flush_type)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_ualink_peer *peer;
+	struct amdgpu_ualink_ring *ring;
+	int r;
+
+	dev_dbg(adev->dev, "skip remote shootdown to remote_accel_id %u\n",
+		remote_accel_id);
+	return 0;
+
+	peer = &remote->peer[remote_accel_id];
+	ring = &peer->shootdown;
+
+	guard(mutex)(&peer->lock);
+
+	if (!ring->ready) {
+		dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id);
+		r = amdgpu_ualink_update_wb_address(adev, remote_accel_id,
+						    RB_TYPE_TLB_INV);
+		if (r)
+			return r;
+	}
+
+	return amdgpu_ualink_send_command(adev, remote_accel_id, ring,
+					  ring->wb_cpu,
+					  amdgpu_ualink_emit_shootdown,
+					  flush_type, upper_32_bits(addr),
+					  lower_32_bits(addr), size_in_pages);
+}
+
+/**
+ * amdgpu_ualink_remote_interrupt - Send a remote interrupt to a peer GPU
+ * @adev: amdgpu device pointer
+ * @remote_accel_id: accelerator ID of the remote GPU to interrupt
+ * @dw0: IH context ID dword 0
+ * @dw1: IH context ID dword 1
+ * @dw2: IH context ID dword 2
+ * @dw3: IH context ID dword 3
+ *
+ * Sends a remote interrupt command to the specified peer GPU via the
+ * UALink ring buffer. The four context ID dwords are delivered to the
+ * remote GPU's interrupt handler.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_remote_interrupt(struct amdgpu_device *adev,
+				   u32 remote_accel_id, u32 dw0, u32 dw1,
+				   u32 dw2, u32 dw3)
+{
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_ualink_peer *peer;
+	struct amdgpu_ualink_ring *ring;
+	int r;
+
+	peer = &remote->peer[remote_accel_id];
+	ring = &peer->interrupt;
+
+	guard(mutex)(&peer->lock);
+
+	if (!ring->ready) {
+		dev_dbg(adev->dev, "accel_id %u ring not ready\n", remote_accel_id);
+		r = amdgpu_ualink_update_wb_address(adev, remote_accel_id,
+						    RB_TYPE_REMOTE_INTERRUPT);
+		if (r)
+			return r;
+	}
+
+	return amdgpu_ualink_send_command(adev, remote_accel_id, ring,
+					  ring->wb_cpu,
+					  amdgpu_ualink_emit_interrupt,
+					  dw0, dw1, dw2, dw3);
+}
+
+/**
+ * amdgpu_ualink_peer_remote_init - Initialize remote peer GPU connections
+ * @adev: amdgpu device pointer
+ *
+ * Sets up ring buffers, GART mappings, and control structures for
+ * communicating with remote GPUs in the fabric.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int amdgpu_ualink_peer_remote_init(struct amdgpu_device *adev)
+{
+	u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	struct amdgpu_bo *bo = remote->ring_bo;
+	struct amdgpu_ualink_peer *peer;
+	struct amdgpu_ualink_ring *interrupt;
+	struct amdgpu_ualink_ring *shootdown;
+	u32 src_accel_id = ualink_accel_id(adev);
+	u32 dst_accel_id;
+	u64 npa, flags;
+	int r = 0;
+
+	/* NPA mapping PTE flags VSCT = 0011 */
+	flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, bo->tbo.resource);
+	flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 | AMDGPU_PTE_BUS_ATOMICS;
+	flags &= ~AMDGPU_PTE_VALID;
+	/* PTE.X=0 turn off RPC checks for RBs, wptr and doorbell NPA address */
+	flags &= ~AMDGPU_PTE_EXECUTABLE;
+
+	dev_dbg(adev->dev, "src_accel_id %u gart mapping flags 0x%llx\n",
+		src_accel_id, flags);
+
+	for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
+		if (dst_accel_id == src_accel_id)
+			continue;
+
+		dev_dbg(adev->dev, "peer_remote to dst_accel_id %u\n", dst_accel_id);
+
+		peer = &remote->peer[dst_accel_id];
+		mutex_init(&peer->lock);
+
+		interrupt = &peer->interrupt;
+		shootdown = &peer->shootdown;
+
+		/* ring buffer npa gart mapping */
+		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_REMOTE_INTERRUPT,
+						   src_accel_id, dst_accel_id);
+		r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &interrupt->mm_node_rb, flags);
+		if (r)
+			break;
+		interrupt->rb_npa_gart = adev->gmc.gart_start +
+					 (interrupt->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT);
+
+		dev_dbg(adev->dev, "rb npa 0x%llx mapped on gart 0x%llx\n",
+			npa, interrupt->rb_npa_gart);
+
+		/* tlb invalidate ring buffer npa gart mapping */
+		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TLB_INV, src_accel_id,
+						  dst_accel_id);
+		r = amdgpu_ualink_gart_map(adev, rb_pages, npa, &shootdown->mm_node_rb, flags);
+		if (r)
+			break;
+		shootdown->rb_npa_gart = adev->gmc.gart_start +
+					 (shootdown->mm_node_rb.start << AMDGPU_GPU_PAGE_SHIFT);
+
+		dev_dbg(adev->dev, "tlb rb npa 0x%llx mapped on gart 0x%llx\n",
+			npa, shootdown->rb_npa_gart);
+
+		/* wptr npa gart mapping */
+		npa = amdgpu_ualink_gart_npa_addr(adev, RB_TYPE_TAILPTR, src_accel_id,
+						  dst_accel_id);
+		r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_wptr, flags);
+		if (r)
+			break;
+
+		interrupt->wptr_npa_gart = adev->gmc.gart_start +
+					   (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT)
+					    + ualink_wptr_offset(adev, src_accel_id);
+		shootdown->wptr_npa_gart = adev->gmc.gart_start +
+					   (interrupt->mm_node_wptr.start << AMDGPU_GPU_PAGE_SHIFT)
+					    + ualink_tlb_wptr_offset(adev, src_accel_id);
+
+		dev_dbg(adev->dev, "rb wptr npa 0x%llx on gart 0x%llx, tlb wptr on gart 0x%llx\n",
+			npa, interrupt->wptr_npa_gart, shootdown->wptr_npa_gart);
+
+		/*
+		 * doorbell npa gart mapping, doorbell NPA address coding
+		 *
+		 * ReqAddr[51:40] = 0xFFF
+		 * ReqAddr[39:30] = GPU accel_id
+		 * ReqAddr[29:12] = 18'h00000
+		 * ReqAddr[11:8]  = Random value
+		 * ReqAddr[7:6]   = PMI Index
+		 *     0 - PMI0 Remote Interrupt
+		 *     1 - PMI1 Remote TLB Shootdown
+		 * ReqAddr[5:0]   = 6'h0
+		 */
+		npa = 0xFFFULL << 40 | (u64)dst_accel_id << 30;
+		r = amdgpu_ualink_gart_map(adev, 1, npa, &interrupt->mm_node_doorbell, flags);
+		if (r)
+			break;
+
+		interrupt->doorbell_npa_gart = adev->gmc.gart_start +
+			(interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT);
+
+		dev_dbg(adev->dev, "rb doorbell npa 0x%llx mapped on gart 0x%llx\n",
+			npa, interrupt->doorbell_npa_gart);
+
+		shootdown->doorbell_npa_gart = adev->gmc.gart_start+
+			(interrupt->mm_node_doorbell.start << AMDGPU_GPU_PAGE_SHIFT) + (1 << 6);
+
+		dev_dbg(adev->dev, "tlb doorbell npa 0x%llx mapped on gart 0x%llx\n",
+			npa, shootdown->doorbell_npa_gart);
+
+		interrupt->rb_size = AMDGPU_UALINK_RB_SIZE / 64;
+		interrupt->wptr = 0;
+		interrupt->rptr = 0;
+		interrupt->seq = 0;
+
+		shootdown->rb_size = AMDGPU_UALINK_RB_SIZE / 64;
+		shootdown->wptr = 0;
+		shootdown->rptr = 0;
+		shootdown->seq = 0;
+	}
+
+	dev_dbg(adev->dev, "peer remote init done r=%d\n", r);
+	return r;
+}
+
+/**
+ * amdgpu_ualink_peer_remote_fini - Clean up remote peer GPU connections
+ * @adev: amdgpu device pointer
+ *
+ * Unmaps GART entries and frees resources for remote GPU communication.
+ */
+static void amdgpu_ualink_peer_remote_fini(struct amdgpu_device *adev)
+{
+	u32 rb_pages = AMDGPU_UALINK_RB_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
+	struct amdgpu_ualink_remote *remote = to_remote(adev);
+	u32 src_accel_id = ualink_accel_id(adev);
+	u32 dst_accel_id;
+	struct amdgpu_ualink_peer *peer;
+	struct amdgpu_ualink_ring *ring;
+
+	dev_dbg(adev->dev, "src_accel_id %u\n", src_accel_id);
+
+	for_each_set_bit(dst_accel_id, remote->active_accel_bits, AMDGPU_UALINK_ACCEL_MAX) {
+		dev_dbg(adev->dev, "dst_accel_id %u\n", dst_accel_id);
+		if (dst_accel_id == src_accel_id)
+			continue;
+
+		peer = &remote->peer[dst_accel_id];
+		mutex_destroy(&peer->lock);
+
+		ring = &peer->interrupt;
+		amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb);
+		amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_wptr);
+		amdgpu_ualink_gart_unmap(adev, 1, &ring->mm_node_doorbell);
+
+		ring = &peer->shootdown;
+		amdgpu_ualink_gart_unmap(adev, rb_pages, &ring->mm_node_rb);
+	}
+}
+
+/**
+ * amdgpu_ualink_sw_init - Initialize UALink software resources
+ * @adev: amdgpu device pointer
+ *
+ * Performs full UALink software initialization including metadata allocation,
+ * NPA address mapping, SDMA scheduler entity setup, and remote peer GPU
+ * connection initialization. On failure, all previously initialized resources
+ * are cleaned up.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+int amdgpu_ualink_sw_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ualink_remote *remote;
+	int r;
+
+	remote = kzalloc(sizeof(*remote), GFP_KERNEL);
+	if (!remote)
+		return -ENOMEM;
+	adev->ualink.remote = remote;
+
+	r = amdgpu_ualink_metadata_init(adev);
+	if (r)
+		goto out;
+
+	r = amdgpu_ualink_metadata_npa_mapping(adev);
+	if (r)
+		goto out_metadata_fini;
+
+	r = amdgpu_ualink_sdma_entities_init(adev);
+	if (r)
+		goto out_npa_unmap;
+
+	r = amdgpu_ualink_peer_remote_init(adev);
+	if (r)
+		goto out_sdma_entities_fini;
+
+	dev_dbg(adev->dev, "ualink sw init succeed\n");
+	return 0;
+
+out_sdma_entities_fini:
+	amdgpu_ualink_sdma_entities_fini(adev);
+out_npa_unmap:
+	amdgpu_ualink_metadata_npa_unmapping(adev);
+out_metadata_fini:
+	amdgpu_ualink_metadata_fini(adev);
+out:
+	kfree(adev->ualink.remote);
+	adev->ualink.remote = NULL;
+	dev_dbg(adev->dev, "ualink sw init failed %d\n", r);
+	return r;
+}
+
+/**
+ * amdgpu_ualink_sw_fini - Tear down UALink software resources
+ * @adev: amdgpu device pointer
+ *
+ * Cleans up all UALink software resources in reverse order of initialization:
+ * remote peer connections, SDMA entities, NPA mappings, and metadata.
+ */
+void amdgpu_ualink_sw_fini(struct amdgpu_device *adev)
+{
+	dev_dbg(adev->dev, "halt accel_id %u addr_mode %d\n", ualink_accel_id(adev),
+		ualink_addr_mode(adev));
+
+	adev->ualink.msg_ctl->send_halt(adev);
+
+	amdgpu_ualink_peer_remote_fini(adev);
+	amdgpu_ualink_sdma_entities_fini(adev);
+	amdgpu_ualink_metadata_npa_unmapping(adev);
+	amdgpu_ualink_metadata_fini(adev);
+	kfree(adev->ualink.remote);
+	adev->ualink.remote = NULL;
+}
+
+int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id)
+{
+	return amdgpu_ualink_send_hello_msg(adev, remote_accel_id);
+}
+
+/*
+ * The low bits of handle_lo carry the message-header field on the wire.
+ * Strip those bits and splice in @acc_id to recover the
+ * fully-qualified handle_lo value.
+ */
+static inline u64 amdgpu_ualink_reassemble_handle(u64 handle_lo, u32 acc_id)
+{
+	return (handle_lo & ~AMDGPU_UALINK_MESSAGE_HEADER_MASK) |
+	       (acc_id & AMDGPU_UALINK_HANDLE_ACCID_MASK);
+}
+
+static int amdgpu_ualink_process_irq(struct amdgpu_device *adev,
+				     struct amdgpu_irq_src *source,
+				     struct amdgpu_iv_entry *entry)
+{
+	u32 sender_acc_id, receiver_acc_id, msg_type, src_acc_id;
+	enum amdgpu_ualink_accel_state accel_state;
+	struct amdgpu_ualink_handle handle;
+	u32 size, npa_addr, fail_reason;
+	u32 dw0, dw1, dw2, dw3;
+	u32 local_acc_id;
+	int handled = 1;
+
+	if (unlikely(adev->ualink.mgr_state != AMDGPU_UALINK_INIT_COMPLETE)) {
+		dev_dbg(adev->dev,
+			"UALink manager not initialized, dropping irq\n");
+		return handled;
+	}
+
+	/* Only handle interrupts while ACTIVE. Otherwise (e.g. during teardown)
+	 * consume stale ring entries but drop them, so they don't touch state
+	 * being freed.
+	 */
+	accel_state = adev->ualink.info->accel_state;
+	if (accel_state != AMDGPU_UALINK_ACCEL_STATE_ACTIVE) {
+		dev_dbg(adev->dev,
+			"Dropping stale UALink interrupt, accel_state %d\n",
+			accel_state);
+		return handled;
+	}
+
+	dev_dbg(adev->dev, "%s client_id 0x%x src_id 0x%x ih\n",
+		entry->ih == &adev->irq.ih ? "ring" : "ualink soft ring",
+		entry->client_id, entry->src_id);
+
+	/* ContextID 4 dwords */
+	src_acc_id = entry->pasid;
+	dw0 = entry->src_data[0];
+	dw1 = entry->src_data[1];
+	dw2 = entry->src_data[2];
+	dw3 = entry->src_data[3];
+
+	/* Copy IH entry into ualink soft ring. */
+	if (entry->ih == &adev->irq.ih) {
+		dev_dbg(adev->dev, "src accel_id %u context id 0x%x 0x%x 0x%x 0x%x\n",
+			src_acc_id, dw0, dw1, dw2, dw3);
+		dev_dbg(adev->dev, "delegate to ualink irq soft ring\n");
+		amdgpu_irq_ualink_delegate(adev, entry, 8);
+		return handled;
+	}
+
+	msg_type = dw0 & AMDGPU_UALINK_MESSAGE_HEADER_MASK;
+	local_acc_id = adev->ualink.info->ppod.accel_id;
+	dev_dbg(adev->dev, "Got MSG: remote acc_id %u msg_type %u\n",
+		src_acc_id, msg_type);
+
+	/* Debug hook: if the bit corresponding to this msg_type is set in
+	 * drop_msg_bitmap, drop this single packet and clear the bit so that
+	 * any subsequent incoming messages are processed normally. Used to
+	 * exercise the connection reset/recovery paths via debugfs.
+	 */
+	if (msg_type < BITS_PER_LONG &&
+	    test_and_clear_bit(msg_type, &adev->ualink.drop_msg_bitmap)) {
+		dev_warn(adev->dev,
+			 "DROP MSG (debugfs): src acc_id %u msg_type %u dw[0-3] 0x%x 0x%x 0x%x 0x%x\n",
+			 src_acc_id, msg_type, dw0, dw1, dw2, dw3);
+		return handled;
+	}
+
+	switch (msg_type) {
+	case AMDGPU_UALINK_HELLO_MSG:
+		receiver_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT) &
+					AMDGPU_UALINK_HELLO_MSG_ACCID_MASK;
+		sender_acc_id = (dw0 >> AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT) &
+					AMDGPU_UALINK_HELLO_MSG_ACCID_MASK;
+		dev_dbg(adev->dev,
+			"Got HELLO MSG: src acc_id %u receiver_acc_id %u sender_acc_id %u\n",
+			src_acc_id, receiver_acc_id, sender_acc_id);
+		amdgpu_ualink_process_hello_msg(adev, receiver_acc_id, sender_acc_id,
+						src_acc_id);
+		break;
+	case AMDGPU_UALINK_HELLO_ACK_MSG:
+		dev_dbg(adev->dev,
+			"Got HELLO-ACK MSG: remote acc_id %u\n", src_acc_id);
+		amdgpu_ualink_process_hello_ack_msg(adev, src_acc_id);
+		break;
+	case AMDGPU_UALINK_NPA_REQ_MSG:
+		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
+		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
+								   local_acc_id);
+		dev_dbg(adev->dev,
+			"Got NPA-REQ MSG: remote acc_id %u handle %llx:%llx\n",
+			src_acc_id, handle.handle_hi, handle.handle_lo);
+		amdgpu_ualink_process_npa_req_msg(adev, src_acc_id, handle);
+		break;
+	case AMDGPU_UALINK_NPA_RSP_MSG:
+		memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo));
+		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
+								   src_acc_id);
+		size = entry->src_data[2];
+		npa_addr = entry->src_data[3];
+		dev_dbg(adev->dev,
+			"Got NPA-RSP MSG: remote acc_id %u handle_lo %llx size 0x%x npa_addr 0x%x\n",
+			src_acc_id, handle.handle_lo, size, npa_addr);
+		amdgpu_ualink_process_npa_rsp_msg(adev, src_acc_id,
+						  handle.handle_lo,
+						  npa_addr, size);
+		break;
+	case AMDGPU_UALINK_NPA_FAIL_MSG:
+		memcpy(&handle.handle_lo, entry->src_data, sizeof(handle.handle_lo));
+		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
+								   src_acc_id);
+		fail_reason = entry->src_data[2] &
+				AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK;
+		dev_dbg(adev->dev,
+			"Got NPA-FAIL MSG: remote acc_id %u handle_lo %llx fail_reason %u\n",
+			src_acc_id, handle.handle_lo, fail_reason);
+		amdgpu_ualink_process_npa_fail_msg(adev, src_acc_id,
+						   handle.handle_lo,
+						   fail_reason);
+		break;
+	case AMDGPU_UALINK_NPA_REVOKE_MSG:
+		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
+		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
+								   src_acc_id);
+		dev_dbg(adev->dev,
+			"Got NPA-REVOKE MSG: remote acc_id %u handle %llx:%llx\n",
+			src_acc_id, handle.handle_hi, handle.handle_lo);
+		amdgpu_ualink_process_npa_revoke_msg(adev, src_acc_id, handle);
+		break;
+	case AMDGPU_UALINK_NPA_RELEASE_MSG:
+		memcpy(&handle, entry->src_data, sizeof(struct amdgpu_ualink_handle));
+		handle.handle_lo = amdgpu_ualink_reassemble_handle(handle.handle_lo,
+								   local_acc_id);
+		dev_dbg(adev->dev,
+			"Got NPA-RELEASE MSG: remote acc_id %u handle %llx:%llx\n",
+			src_acc_id, handle.handle_hi, handle.handle_lo);
+		amdgpu_ualink_process_npa_release_msg(adev, src_acc_id, handle);
+		break;
+	default:
+		dev_err(adev->dev, "Unknown message type (%u)\n", msg_type);
+		break;
+	}
+
+	return handled;
+}
+
+static int amdgpu_ualink_set_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					u32 type,
+					enum amdgpu_interrupt_state state)
+{
+	/*
+	 * Don't set register to enable/disable nHT controller interrupt.
+	 *
+	 * F/W running on MP2, which can always send cookie to IH block to
+	 * interrupt driver.
+	 */
+	dev_dbg(adev->dev, "ualink interrupt %s\n",
+		state == AMDGPU_IRQ_STATE_ENABLE ? "enable" : "disable");
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs ualink_irq_funcs = {
+	.set = amdgpu_ualink_set_irq_state,
+	.process = amdgpu_ualink_process_irq,
+};
+
+/**
+ * amdgpu_ualink_init_interrupt - register the UALink IRQ source
+ * @adev: amdgpu device pointer
+ * @client_id: IH client ID for the interrupt source
+ * @src_id: source ID for the interrupt source
+ *
+ * Registers the UALink interrupt source with the IH subsystem.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev,
+				 unsigned int client_id, unsigned int src_id)
+{
+	int r;
+
+	dev_dbg(adev->dev, "init ualink irq client_id 0x%x src_id 0x%x\n",
+		client_id, src_id);
+
+	adev->ualink.irq.num_types = 1;
+	adev->ualink.irq.funcs = &ualink_irq_funcs;
+
+	r = amdgpu_irq_add_id(adev, client_id, src_id, &adev->ualink.irq);
+	return r;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.h
new file mode 100644
index 00000000000000..85c9a9634a4b06
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ualink.h
@@ -0,0 +1,427 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef AMDGPU_UALINK_H
+#define AMDGPU_UALINK_H
+
+#include <linux/uuid.h>
+#include <linux/bitmap.h>
+#include <linux/kobject.h>
+
+struct amdgpu_device;
+
+struct amdgpu_xcp;
+
+#define AMDGPU_UALINK_ACCEL_MAX 256
+#define AMDGPU_UALINK_LOCAL_ACCELS_MAX 8
+#define AMDGPU_UALINK_STATIONS_MAX 64
+
+/* A vpod_id of 0 is reserved and treated as invalid/no vPod */
+#define AMDGPU_UALINK_VPOD_ID_INVALID 0
+
+/* nHT firmware status */
+#define AMDGPU_NHT_FW_ST_PREINIT	0xA0
+#define AMDGPU_NHT_FW_ST_READY	0xA1
+#define AMDGPU_NHT_FW_ST_HALT	0xA2
+#define AMDGPU_NHT_FW_ST_ERROR	0xA3
+#define AMDGPU_NHT_FW_ST_FATAL	0xF0
+
+#define AMDGPU_UALINK_RESP_TIMEOUT			5000 /* 5s timeout */
+
+#define AMDGPU_UALINK_HANDLE_ACCID_MASK			GENMASK_ULL(9, 0)
+#define AMDGPU_UALINK_MESSAGE_HEADER_MASK		GENMASK_ULL(9, 0)
+#define AMDGPU_UALINK_HELLO_MSG_ACCID_MASK		GENMASK_U32(9, 0)
+#define AMDGPU_UALINK_HELLO_MSG_RECV_ACCID_SHIFT	10
+#define AMDGPU_UALINK_HELLO_MSG_SENDER_ACCID_SHIFT	20
+#define AMDGPU_UALINK_NPA_FAIL_MSG_FAIL_REASON_MASK	GENMASK_U32(7, 0)
+
+/* GPU-ID is stored in bits 41-50 of the NPA address. However, we
+ * store NPA address is GPU PAGE aligned so bottom 12 bits are not used.
+ * As a result, we need the GPU-ID shift to be 41 - 12 = 29.
+ */
+#define AMDGPU_UALINK_NPA_ADDR_GPUID_SHIFT		29
+#define AMDGPU_UALINK_NPA_ADDR_GPUID_MASK		GENMASK_ULL(38, 29)
+/* Reserve 2M in each 2TB range for ring buffer allocations for
+ * remote interrupts. In terms of GPU pages, this is 2M / 4K = 512 pages.
+ * So we reserve 512 pages in each 2TB range.
+ */
+#define AMDGPU_UALINK_NPA_ADDR_RANGE_RESERVED		(1U << 9)
+#define AMDGPU_UALINK_NPA_ADDR_RANGE_MASK		GENMASK_ULL(28, 0)
+
+enum AMDGPU_UALINK_NPA_FAIL_REASON {
+	AMDGPU_UALINK_NPA_FAIL_NOSPACE			= 1,
+	AMDGPU_UALINK_NPA_FAIL_INVALID_HANDLE		= 2,
+	AMDGPU_UALINK_NPA_FAIL_DUPLICATE		= 3,
+	AMDGPU_UALINK_NPA_FAIL_ERROR			= 4,
+};
+
+enum AMDGPU_UALINK_NODE_STATE {
+	AMDGPU_UALINK_NODE_NOT_READY			= 0,
+	AMDGPU_UALINK_NODE_PENDING			= 1,
+	AMDGPU_UALINK_NODE_READY			= 2,
+	AMDGPU_UALINK_NODE_TEARDOWN			= 3
+};
+
+enum AMDGPU_UALINK_PROTOCOL_MESSAGES {
+	AMDGPU_UALINK_HELLO_MSG				= 1,
+	AMDGPU_UALINK_HELLO_ACK_MSG			= 2,
+	AMDGPU_UALINK_NPA_REQ_MSG			= 3,
+	AMDGPU_UALINK_NPA_RSP_MSG			= 4,
+	AMDGPU_UALINK_NPA_FAIL_MSG			= 5,
+	AMDGPU_UALINK_NPA_REVOKE_MSG			= 6,
+	AMDGPU_UALINK_NPA_RELEASE_MSG			= 7,
+	AMDGPU_UALINK_MAX_PROTOCOL_MSG
+};
+
+#define AMDGPU_UALINK_LIGHTWEIGHT_TLB_SHOOTDOWN		0x1
+#define AMDGPU_UALINK_HEAVYWEIGHT_TLB_SHOOTDOWN		0x2
+
+enum amdgpu_ualink_conn_state {
+	AMDGPU_UALINK_CONN_NOT_READY			= 0,
+	AMDGPU_UALINK_CONN_IN_PROGRESS			= 1,
+	AMDGPU_UALINK_CONN_PENDING			= 2,
+	AMDGPU_UALINK_CONN_ESTABLISHED			= 3
+};
+
+enum amdgpu_ualink_type {
+	AMDGPU_UALINK_TYPE_UALOE = 0,
+	AMDGPU_UALINK_TYPE_UALINK = 1,
+	AMDGPU_UALINK_TYPE_MAX
+};
+
+enum amdgpu_ualink_accel_state {
+	AMDGPU_UALINK_ACCEL_STATE_UNCONFIGURED = 0,
+	AMDGPU_UALINK_ACCEL_STATE_PPOD_CONFIGURED,
+	AMDGPU_UALINK_ACCEL_STATE_VPOD_CONFIGURED,
+	AMDGPU_UALINK_ACCEL_STATE_READY,
+	AMDGPU_UALINK_ACCEL_STATE_ACTIVE,
+	AMDGPU_UALINK_ACCEL_STATE_ERROR,
+	AMDGPU_UALINK_ACCEL_STATE_MAX
+};
+
+enum amdgpu_ualink_addr_mode {
+	AMDGPU_UALINK_ADDR_MODE_SOURCE_ALIAS = 0,
+	AMDGPU_UALINK_ADDR_MODE_SOURCE_IDENT = 1,
+	AMDGPU_UALINK_ADDR_MODE_MAX
+};
+
+enum amdgpu_ualink_mgr_state {
+	AMDGPU_UALINK_INIT_NONE = 0,
+	AMDGPU_UALINK_INIT_HW,
+	AMDGPU_UALINK_INIT_COMPLETE,
+	AMDGPU_UALINK_INIT_ERROR
+};
+
+/* Physical pod info shared between query and setup API */
+struct amdgpu_ualink_ppod_info {
+	u32 accel_id;
+	u32 bandwidth;
+	u32 latency;
+	u32 size;
+	uuid_t id;
+};
+
+/* Virtual pod info shared between query and config API */
+struct amdgpu_ualink_vpod_info {
+	u32 id;
+	u32 size;
+	u32 addr_mode;
+	DECLARE_BITMAP(active_accel_bits, AMDGPU_UALINK_ACCEL_MAX);
+};
+
+/* Query current UALink physical and virtual pod info */
+struct amdgpu_ualink_info {
+	struct kobject kobj;
+	enum amdgpu_ualink_type link_type;
+	enum amdgpu_ualink_accel_state accel_state;
+	struct amdgpu_ualink_ppod_info ppod;
+	struct amdgpu_ualink_vpod_info vpod;
+	/* Local accelerator array in the vpod derived from
+	 * vpod.active_accel_bits, in arbitrary order
+	 */
+	u32 n_local_accels;
+	u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX];
+};
+#define to_ualink_info(ko) container_of(ko, struct amdgpu_ualink_info, kobj)
+
+/* UALink physical pod setup */
+struct amdgpu_ualink_ppod_setup {
+	struct kobject kobj;
+	struct amdgpu_ualink_ppod_info ppod;
+	/* Local accelerator array indexed by socket ID */
+	u32 n_local_accels;
+	u32 local_accels[AMDGPU_UALINK_LOCAL_ACCELS_MAX];
+};
+#define to_ualink_ppod_setup(ko) container_of(ko, struct amdgpu_ualink_ppod_setup, kobj)
+
+/* UAlink virtual pod config */
+struct amdgpu_ualink_vpod_config {
+	struct kobject kobj;
+	struct amdgpu_ualink_vpod_info vpod;
+};
+#define to_ualink_vpod_config(ko) container_of(ko, struct amdgpu_ualink_vpod_config, kobj)
+
+/* UALink station configuration */
+struct amdgpu_ualink_station_config {
+	struct kobject kobj;
+	/* Station configuration flags
+	 * bits [3:0]: PortPerStation (PPS) - 1, 2, or 4
+	 * bits [7:4]: Reserved
+	 */
+	u8 flags;
+	u8 n_stations;
+	/* bitmap or enabled lanes for each station in logical station order */
+	u8 lane_en_bitmap[AMDGPU_UALINK_STATIONS_MAX];
+};
+#define to_ualink_station_config(ko) container_of(ko, struct amdgpu_ualink_station_config, kobj)
+
+struct amdgpu_ualink_handle {
+	union {
+		struct {
+			u64 handle_lo;
+			u64 handle_hi;
+		};
+		u64 handle[2];
+	};
+};
+
+struct amdgpu_ualink_imp_xa_node {
+	struct amdgpu_device		*adev;
+
+	/* 128-bit handle for the BO */
+	struct amdgpu_ualink_handle	handle;
+
+	/* Use to signal NPA-RSP arrival */
+	struct completion		npa_done;
+
+	/* NPA address received in the NPA-RSP message */
+	u64				npa_addr;
+	u64				size;
+
+	/* GEM handle for the NPA BO */
+	u32				gem_handle;
+
+	/* Fail reason received in NPA-FAIL message */
+	int				fail_reason;
+
+	/* Node state to signal if node setup is in progress
+	 * or is already completed. Node state goes back to
+	 * in progress if a HELLO message is received in
+	 * response to NPA-REQ message.
+	 */
+	enum AMDGPU_UALINK_NODE_STATE	node_state;
+
+	/* Used to connect all importer XA nodes from a particular
+	 * exporter.
+	 */
+	struct list_head		list;
+
+	/* Dmabuf corresponding to the NPA BO */
+	struct dma_buf			*dmabuf;
+
+	/* Refcount to track lifetime of this node */
+	struct kref			refcount;
+};
+
+struct amdgpu_ualink_npa_mm {
+	struct drm_mm			mm;
+	u64				va_start;
+	u64				va_size;
+	struct mutex			mm_lock;
+};
+
+struct amdgpu_ualink_importer_entry {
+	struct drm_mm_node			*mm_node;
+	u64					npa_addr;
+
+	/* Keep track of if the connection got reset */
+	u32					generation_count;
+
+	/* Used to connect all handles exported to a particular importer */
+	struct list_head			list;
+
+	/* Pointer to the parent XA node. */
+	struct amdgpu_ualink_exp_xa_node	*parent;
+};
+
+struct amdgpu_ualink_exp_xa_node {
+	/* 128-bit handle for the BO */
+	struct amdgpu_ualink_handle		handle;
+
+	/* Pointer to the BO thats exported.*/
+	struct amdgpu_bo			*bo;
+
+	/* Dmabuf corresponding to the BO */
+	struct dma_buf				*dmabuf;
+
+	/* Mutex to protect the node from concurrent access */
+	struct mutex				node_lock;
+
+	/* Used for storing importer info in source identification mode */
+	struct amdgpu_ualink_importer_entry	importer_entries[AMDGPU_UALINK_ACCEL_MAX];
+
+	/* Used to track all importers of this BO. This is set when the
+	 * exporter sends back the NPA-RSP message.
+	 */
+	DECLARE_BITMAP(importers_bitmap, AMDGPU_UALINK_ACCEL_MAX);
+
+	/* This bitmap is used to send NPA-REVOKE messages to all the importers.
+	 * And to track the NPA-RELEASE response for each NPA-REVOKE message sent.
+	 * A bit is cleared in this bitmap when the NPA RELEASE message is
+	 * received in response to the NPA-REVOKE message.
+	 */
+	DECLARE_BITMAP(npa_release_bitmap, AMDGPU_UALINK_ACCEL_MAX);
+
+	/* Use to signal responses received from all importers */
+	struct completion			npa_done;
+
+	/* Refcount to track lifetime of this node */
+	struct kref				refcount;
+
+	/* Work to cleanup the node. */
+	struct work_struct			cleanup_work;
+};
+
+struct amdgpu_ualink_connection {
+	struct completion hello_done;
+	struct mutex lock;
+	u32 generation_count;
+	enum amdgpu_ualink_conn_state state;
+};
+
+struct amdgpu_ualink_remote;
+
+struct amdgpu_ualink_msg_ctl {
+	u32 (*check_status)(struct amdgpu_device *adev);
+	int (*send_metadata)(struct amdgpu_device *adev, u64 metadata_gpu_addr,
+			     u32 accel_data);
+	int (*send_halt)(struct amdgpu_device *adev);
+};
+
+struct amdgpu_ualink_mgr {
+	u64 npa_size;
+	u32 psp_if_ver;
+	const struct amdgpu_ualink_msg_ctl *msg_ctl;
+	struct amdgpu_ualink_info *info;
+	struct amdgpu_ualink_ppod_setup *setup;
+	struct amdgpu_ualink_vpod_config *config;
+	struct amdgpu_ualink_station_config *stations;
+	bool sysfs_init;
+	enum amdgpu_ualink_mgr_state mgr_state;
+
+	/* For remote interrupt and shootdown */
+	struct amdgpu_ualink_remote *remote;
+
+	/* handle irq from ualink client of remote GPUs */
+	struct amdgpu_irq_src irq;
+
+	/* Xarray to store info about exported BOs */
+	struct xarray	exp_xa;
+
+	/* Xarray to store info about imported ualink handles */
+	struct xarray	imp_xa;
+
+	/* Xarray to store handles to be deleted */
+	struct xarray	handle_invalid_xa;
+
+	/* Array to store connection state per remote GPU */
+	struct amdgpu_ualink_connection conn_state[AMDGPU_UALINK_ACCEL_MAX];
+
+	/* List of exported ualink handles per GPU. Used to quickly traverse
+	 * the list of all handles exported to a remote GPU.
+	 */
+	struct list_head exp_handles_list[AMDGPU_UALINK_ACCEL_MAX];
+
+	/* List of imported ualink handles per GPU. Used to quickly traverse
+	 * the list of all handles imported from a remote GPU.
+	 */
+	struct list_head imp_handles_list[AMDGPU_UALINK_ACCEL_MAX];
+
+	/* WQ to manage revocation of exported memory. */
+	struct workqueue_struct *npa_wq;
+
+	/* Memory-manager for managing NPA address space. */
+	struct amdgpu_ualink_npa_mm npa_mm;
+
+	/* NPA-VM used on the exporter.*/
+	struct amdgpu_vm npa_vm;
+
+	/* DRM client for UALink to manage GEM handles for NPA BOs */
+	struct drm_client_dev client;
+
+	/* Sequence number to track the need for TLB flushes */
+	atomic64_t last_flushed_tlb_seq;
+
+	/* Debug-only: bitmap of incoming UALink protocol messages to drop.
+	 *
+	 * Each bit position corresponds to a value from
+	 * enum AMDGPU_UALINK_PROTOCOL_MESSAGES:
+	 *   BIT(AMDGPU_UALINK_NPA_REQ_MSG) drops one incoming NPA-REQ.
+	 *   BIT(AMDGPU_UALINK_NPA_RSP_MSG) drops one incoming NPA-RSP.
+	 *
+	 *   On reception of a message whose corresponding bit is set, the bit
+	 *   is atomically cleared and the message is silently dropped. This
+	 *   means at most one packet per set bit is dropped; any further
+	 *   incoming packets of the same type are processed normally. This is
+	 *   intended to exercise the connection reset / recovery paths from a
+	 *   debugfs handle.
+	 */
+	unsigned long drop_msg_bitmap;
+};
+
+int amdgpu_ualink_init_interrupt(struct amdgpu_device *adev,
+				 unsigned int client_id, unsigned int src_id);
+
+int amdgpu_ualink_sw_init(struct amdgpu_device *adev);
+void amdgpu_ualink_sw_fini(struct amdgpu_device *adev);
+
+int ualink_send_hello(struct amdgpu_device *adev, u32 remote_accel_id);
+
+int amdgpu_ualink_config_update_handler(struct amdgpu_device *adev);
+int amdgpu_ualink_pause_handler(struct amdgpu_device *adev);
+int amdgpu_ualink_resume_handler(struct amdgpu_device *adev);
+
+int amdgpu_ualink_sysfs_init(struct amdgpu_device *adev);
+void amdgpu_ualink_sysfs_fini(struct amdgpu_device *adev);
+void amdgpu_ualink_xcp_sysfs_update(struct amdgpu_xcp *xcp);
+int amdgpu_ualink_manager_start(struct amdgpu_device *adev);
+void amdgpu_ualink_manager_stop(struct amdgpu_device *adev);
+int amdgpu_ualink_export_handle(struct drm_device *dev, struct drm_file *filp,
+				u32 gem_handle,
+				struct amdgpu_ualink_handle *handle_out);
+int amdgpu_ualink_import_handle(struct drm_device *dev,
+				const struct amdgpu_ualink_handle *ualink_handle,
+				int *fd_out);
+int amdgpu_gem_ualink_handle_ioctl(struct drm_device *dev, void *data,
+				   struct drm_file *filp);
+void amdgpu_ualink_revoke_exported_memory(struct amdgpu_bo *bo);
+
+int ualink_ip_hw_init(struct amdgpu_ip_block *ip_block);
+int ualink_ip_hw_fini(struct amdgpu_ip_block *ip_block);
+int ualink_ip_late_init(struct amdgpu_ip_block *ip_block);
+int ualink_ip_sw_init(struct amdgpu_ip_block *ip_block);
+int ualink_ip_sw_fini(struct amdgpu_ip_block *ip_block);
+
+extern const struct amdgpu_ip_block_version ualink_v1_0_ip_block;
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index 71d34fd093858c..31267be2bc7e34 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -21,6 +21,7 @@
  *
  */
 
+#include <linux/bitmap.h>
 #include <linux/sort.h>
 #include "amdgpu.h"
 #include "umc_v6_7.h"
@@ -230,6 +231,13 @@ int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
 			pasid_notify pasid_fn, void *data, uint32_t reset)
 {
 	int ret = AMDGPU_RAS_SUCCESS;
+	struct ras_ih_info ih_info = {
+		.block = block,
+		.pasid = pasid,
+		.reset = reset,
+		.pasid_fn = pasid_fn,
+		.data = data,
+	};
 
 	if (adev->gmc.xgmi.connected_to_cpu ||
 		adev->gmc.is_app_apu) {
@@ -264,17 +272,13 @@ int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev,
 
 			amdgpu_ras_error_data_fini(&err_data);
 		} else {
-			struct ras_ih_info ih_info = {0};
-
-			ih_info.block = block;
-			ih_info.pasid = pasid;
-			ih_info.reset = reset;
-			ih_info.pasid_fn = pasid_fn;
-			ih_info.data = data;
-			amdgpu_ras_mgr_handle_consumer_interrupt(adev, &ih_info);
+			amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info);
 		}
 	} else {
-		if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
+		if (amdgpu_uniras_enabled(adev) &&
+		    (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)))
+			amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info);
+		else if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
 			adev->virt.ops->ras_poison_handler(adev, block);
 		else
 			dev_warn(adev->dev,
@@ -423,6 +427,7 @@ static int amdgpu_umc_loop_all_aid(struct amdgpu_device *adev, umc_func func,
 	uint32_t node_inst;
 	uint32_t umc_inst;
 	uint32_t ch_inst;
+	DECLARE_BITMAP(umc_bitmap, 64);
 	int ret;
 
 	/*
@@ -432,9 +437,10 @@ static int amdgpu_umc_loop_all_aid(struct amdgpu_device *adev, umc_func func,
 	 * umc.node_inst_num = maximum number of node instances
 	 * Channel instances are not assumed to be harvested.
 	 */
-	dev_dbg(adev->dev, "active umcs :%lx umc_inst per node: %d",
+	dev_dbg(adev->dev, "active umcs :%llx umc_inst per node: %d",
 		adev->umc.active_mask, adev->umc.umc_inst_num);
-	for_each_set_bit(umc_node_inst, &(adev->umc.active_mask),
+	bitmap_from_u64(umc_bitmap, adev->umc.active_mask);
+	for_each_set_bit(umc_node_inst, umc_bitmap,
 			 adev->umc.node_inst_num * adev->umc.umc_inst_num) {
 		node_inst = umc_node_inst / adev->umc.umc_inst_num;
 		umc_inst = umc_node_inst % adev->umc.umc_inst_num;
@@ -467,12 +473,18 @@ int amdgpu_umc_loop_channels(struct amdgpu_device *adev,
 		return amdgpu_umc_loop_all_aid(adev, func, data);
 
 	if (adev->umc.node_inst_num) {
-		LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) {
-			ret = func(adev, node_inst, umc_inst, ch_inst, data);
-			if (ret) {
-				dev_err(adev->dev, "Node %d umc %d ch %d func returns %d\n",
-					node_inst, umc_inst, ch_inst, ret);
-				return ret;
+		DECLARE_BITMAP(umc_bitmap, 64);
+
+		bitmap_from_u64(umc_bitmap, adev->umc.active_mask);
+		for_each_set_bit(node_inst, umc_bitmap, adev->umc.node_inst_num) {
+			LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) {
+				ret = func(adev, node_inst, umc_inst, ch_inst, data);
+				if (ret) {
+					dev_err(adev->dev,
+						"Node %d umc %d ch %d func returns %d\n",
+						node_inst, umc_inst, ch_inst, ret);
+					return ret;
+				}
 			}
 		}
 	} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
index b2a3db60d231ed..9e6d8681b0731c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
@@ -45,12 +45,6 @@
 #define LOOP_UMC_CH_INST(ch_inst) for ((ch_inst) = 0; (ch_inst) < adev->umc.channel_inst_num; (ch_inst)++)
 #define LOOP_UMC_INST_AND_CH(umc_inst, ch_inst) LOOP_UMC_INST((umc_inst)) LOOP_UMC_CH_INST((ch_inst))
 
-#define LOOP_UMC_NODE_INST(node_inst) \
-		for_each_set_bit((node_inst), &(adev->umc.active_mask), adev->umc.node_inst_num)
-
-#define LOOP_UMC_EACH_NODE_INST_AND_CH(node_inst, umc_inst, ch_inst) \
-		LOOP_UMC_NODE_INST((node_inst)) LOOP_UMC_INST_AND_CH((umc_inst), (ch_inst))
-
 /* Page retirement tag */
 #define UMC_ECC_NEW_DETECTED_TAG       0x1
 /*
@@ -117,7 +111,7 @@ struct amdgpu_umc {
 	struct amdgpu_umc_ras *ras;
 
 	/* active mask for umc node instance */
-	unsigned long active_mask;
+	u64 active_mask;
 
 	unsigned long err_addr_cnt;
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index e43bda0cab3fbb..75d5ceaa53c65c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -184,27 +184,27 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
 {
 	struct amdgpu_device *adev;
-	unsigned long timeout_ms;
+	unsigned long timeout_jiffies;
 
 	adev = queue->userq_mgr->adev;
 	/* Determine timeout based on queue type */
 	switch (queue->queue_type) {
 	case AMDGPU_RING_TYPE_GFX:
-		timeout_ms = adev->gfx_timeout;
+		timeout_jiffies = adev->gfx_timeout;
 		break;
 	case AMDGPU_RING_TYPE_COMPUTE:
-		timeout_ms = adev->compute_timeout;
+		timeout_jiffies = adev->compute_timeout;
 		break;
 	case AMDGPU_RING_TYPE_SDMA:
-		timeout_ms = adev->sdma_timeout;
+		timeout_jiffies = adev->sdma_timeout;
 		break;
 	default:
-		timeout_ms = adev->gfx_timeout;
+		timeout_jiffies = adev->gfx_timeout;
 		break;
 	}
 
 	queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
-			   msecs_to_jiffies(timeout_ms));
+			   timeout_jiffies);
 }
 
 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
@@ -241,7 +241,6 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
 	struct amdgpu_vm *vm = queue->vm;
 	u64 start_addr;
 	u64 end_addr;
-	u64 start_page;
 
 	/* Caller must hold vm->root.bo reservation */
 	dma_resv_assert_held(queue->vm->root.bo->tbo.base.resv);
@@ -253,16 +252,14 @@ int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
 	if (check_add_overflow(start_addr, expected_size - 1, &end_addr))
 		return -EINVAL;
 
-	start_page = start_addr >> AMDGPU_GPU_PAGE_SHIFT;
-
-	va_map = amdgpu_vm_bo_lookup_mapping(vm, start_page);
+	va_map = amdgpu_vm_bo_lookup_mapping(vm, start_addr);
 	if (!va_map)
 		return -EINVAL;
 
-	/* Lookup guarantees start_page is mapped; ensure full span is covered. */
+	/* Lookup guarantees start_addr is mapped; ensure full span is covered. */
 	if ((end_addr >> AMDGPU_GPU_PAGE_SHIFT) <= va_map->last) {
 		va_map->bo_va->userq_va_mapped = true;
-		*va_out = start_page;
+		*va_out = start_addr;
 		return 0;
 	}
 
@@ -543,12 +540,15 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
 	trace_amdgpu_userq_destroy_start(queue);
 
 	cancel_delayed_work_sync(&uq_mgr->resume_work);
+	/* Cancel before taking userq_mutex: cancel_delayed_work_sync() waits
+	 * for any running instance, which itself takes userq_mutex.
+	 */
+	cancel_delayed_work_sync(&queue->hang_detect_work);
 
 	mutex_lock(&uq_mgr->userq_mutex);
 	amdgpu_userq_wait_for_last_fence(queue);
 
 	amdgpu_userq_detach_doorbell(queue);
-	cancel_delayed_work_sync(&queue->hang_detect_work);
 
 #if defined(CONFIG_DEBUG_FS)
 	debugfs_remove_recursive(queue->debugfs_queue);
@@ -659,6 +659,24 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 
 	uq_funcs = adev->userq_funcs[args->in.ip_type];
 	if (!uq_funcs) {
+		switch (args->in.ip_type) {
+		case AMDGPU_HW_IP_GFX:
+		case AMDGPU_HW_IP_COMPUTE:
+			dev_warn_once(adev->dev,
+				      "Usermode queues for GFX/COMPUTE is not supported by the fw "
+				      "on this ASIC (me: %u, pfp: %u, mec: %u, mes: %u)\n",
+				      adev->gfx.me_fw_version, adev->gfx.pfp_fw_version,
+				      adev->gfx.mec_fw_version, adev->mes.fw_version[0]);
+			break;
+		case AMDGPU_HW_IP_DMA:
+			dev_warn_once(adev->dev,
+				      "Usermode queues for SDMA is not supported by the fw "
+				      "on this ASIC (sdma: %u)\n",
+				      adev->sdma.instance[0].fw_version);
+			break;
+		default:
+			break;
+		}
 		r = -EINVAL;
 		goto err_pm_runtime;
 	}
@@ -1025,7 +1043,11 @@ amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr)
 retry_lock:
 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
 	drm_exec_until_all_locked(&exec) {
-		ret = amdgpu_vm_lock_pd(vm, &exec, 1);
+		/*
+		 * Rearm adds one BOOKKEEP fence and validation may queue move fences
+		 * on the root PD BO, so reserve caller-side slots accordingly.
+		 */
+		ret = amdgpu_vm_lock_pd(vm, &exec, TTM_NUM_MOVE_FENCES + 1);
 		drm_exec_retry_on_contention(&exec);
 		if (unlikely(ret))
 			goto unlock_all;
@@ -1272,7 +1294,7 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
 	return ret;
 }
 
-static void
+void
 amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
 {
 	struct amdgpu_usermode_queue *queue;
@@ -1291,8 +1313,6 @@ amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
 void
 amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr)
 {
-	/* Wait for any pending userqueue fence work to finish */
-	amdgpu_userq_wait_for_signal(uq_mgr);
 	amdgpu_userq_evict_all(uq_mgr);
 }
 
@@ -1373,15 +1393,11 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
 
 int amdgpu_userq_suspend(struct amdgpu_device *adev)
 {
-	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
 	struct amdgpu_usermode_queue *queue;
 	struct amdgpu_userq_mgr *uqm;
 	unsigned long queue_id;
 	int r;
 
-	if (!ip_mask)
-		return 0;
-
 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
 		uqm = queue->userq_mgr;
 		cancel_delayed_work_sync(&uqm->resume_work);
@@ -1398,15 +1414,11 @@ int amdgpu_userq_suspend(struct amdgpu_device *adev)
 
 int amdgpu_userq_resume(struct amdgpu_device *adev)
 {
-	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
 	struct amdgpu_usermode_queue *queue;
 	struct amdgpu_userq_mgr *uqm;
 	unsigned long queue_id;
 	int r;
 
-	if (!ip_mask)
-		return 0;
-
 	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
 		uqm = queue->userq_mgr;
 		guard(mutex)(&uqm->userq_mutex);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 6412a7f7b6ef03..488dc21d7c81bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -162,6 +162,7 @@ void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
 
+void amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr);
 void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr);
 
 void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index 743b41db5b7cb5..6f1dc813ec5bfe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -148,13 +148,13 @@ amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
 	LIST_HEAD(to_be_signaled);
 	struct dma_fence *fence;
 	unsigned long flags;
-	u64 rptr;
+	u64 fence_val;
 
 	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
-	rptr = amdgpu_userq_fence_read(fence_drv);
+	fence_val = amdgpu_userq_fence_read(fence_drv);
 
 	list_for_each_entry(userq_fence, &fence_drv->fences, link) {
-		if (rptr < userq_fence->base.seqno)
+		if (fence_val < userq_fence->base.seqno)
 			break;
 	}
 
@@ -314,12 +314,12 @@ static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
 {
 	struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
 	struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
-	u64 rptr, wptr;
+	u64 fence_val, wptr;
 
-	rptr = amdgpu_userq_fence_read(fence_drv);
+	fence_val = amdgpu_userq_fence_read(fence_drv);
 	wptr = fence->base.seqno;
 
-	if (rptr >= wptr)
+	if (fence_val >= wptr)
 		return true;
 
 	return false;
@@ -359,7 +359,7 @@ static const struct dma_fence_ops amdgpu_userq_fence_ops = {
  *
  * Read the wptr value from userq's MQD. The userq signal IOCTL
  * creates a dma_fence for the shared buffers that expects the
- * RPTR value written to seq64 memory >= WPTR.
+ * fence_val written to seq64 fence address >= WPTR.
  *
  * Returns wptr value on success, error on failure.
  */
@@ -383,7 +383,7 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
 		if (unlikely(ret))
 			goto lock_error;
 
-		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT);
+		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr);
 		if (!mapping) {
 			ret = -EINVAL;
 			goto lock_error;
@@ -464,9 +464,13 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 
 	struct drm_gem_object **gobj_write, **gobj_read;
 	u32 *syncobj_handles, num_syncobj_handles;
+	u64 *syncobj_points;
 	struct amdgpu_usermode_queue *queue;
 	struct amdgpu_userq_fence *fence;
-	struct drm_syncobj **syncobj;
+	struct {
+		struct drm_syncobj *syncobj;
+		struct dma_fence_chain *chain;
+	} *syncobj;
 	struct drm_exec exec;
 	void __user *ptr;
 	int r, i, entry;
@@ -486,19 +490,40 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	if (IS_ERR(syncobj_handles))
 		return PTR_ERR(syncobj_handles);
 
+	if (args->syncobj_points) {
+		ptr = u64_to_user_ptr(args->syncobj_points);
+		syncobj_points = memdup_array_user(ptr, num_syncobj_handles,
+					    sizeof(u64));
+		if (IS_ERR(syncobj_points)) {
+			r = PTR_ERR(syncobj_points);
+			goto free_syncobj_handles;
+		}
+	} else {
+		syncobj_points = NULL;
+	}
+
 	syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
 				GFP_KERNEL);
 	if (!syncobj) {
 		r = -ENOMEM;
-		goto free_syncobj_handles;
+		goto free_syncobj_points;
 	}
 
 	for (entry = 0; entry < num_syncobj_handles; entry++) {
-		syncobj[entry] = drm_syncobj_find(filp, syncobj_handles[entry]);
-		if (!syncobj[entry]) {
+		syncobj[entry].chain = NULL;
+		syncobj[entry].syncobj = drm_syncobj_find(filp, syncobj_handles[entry]);
+		if (!syncobj[entry].syncobj) {
 			r = -ENOENT;
 			goto free_syncobj;
 		}
+		if (syncobj_points && syncobj_points[entry]) {
+			syncobj[entry].chain = dma_fence_chain_alloc();
+			if (!syncobj[entry].chain) {
+				drm_syncobj_put(syncobj[entry].syncobj);
+				r = -ENOMEM;
+				goto free_syncobj;
+			}
+		}
 	}
 
 	ptr = u64_to_user_ptr(args->bo_read_handles);
@@ -567,8 +592,15 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 		dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
 				   DMA_RESV_USAGE_WRITE);
 
-	for (i = 0; i < num_syncobj_handles; i++)
-		drm_syncobj_replace_fence(syncobj[i], &fence->base);
+	for (i = 0; i < num_syncobj_handles; i++) {
+		if (syncobj[i].chain) {
+			drm_syncobj_add_point(syncobj[i].syncobj, syncobj[i].chain,
+					   &fence->base, syncobj_points[i]);
+			syncobj[i].chain = NULL;
+		} else {
+			drm_syncobj_replace_fence(syncobj[i].syncobj, &fence->base);
+		}
+	}
 
 exec_fini:
 	/* drop the reference acquired in fence creation function */
@@ -586,9 +618,13 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 		drm_gem_object_put(gobj_read[i]);
 	kvfree(gobj_read);
 free_syncobj:
-	while (entry-- > 0)
-		drm_syncobj_put(syncobj[entry]);
+	while (entry-- > 0) {
+		drm_syncobj_put(syncobj[entry].syncobj);
+		dma_fence_chain_free(syncobj[entry].chain);
+	}
 	kfree(syncobj);
+free_syncobj_points:
+	kfree(syncobj_points);
 free_syncobj_handles:
 	kfree(syncobj_handles);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 7a007f4916bcc7..a58a352b4a3578 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -339,6 +339,41 @@ static int amdgpu_virt_ras_realloc_eh_data_space(struct amdgpu_device *adev,
 	return 0;
 }
 
+int amdgpu_virt_get_uniras_ras_caps(struct amdgpu_device *adev,
+				    struct amd_sriov_uniras_caps *caps)
+{
+	struct amd_sriov_msg_pf2vf_info *pf2vf_msg;
+
+	if (!adev || !caps)
+		return -EINVAL;
+
+	if (!adev->virt.fw_reserve.p_pf2vf)
+		return -EINVAL;
+
+	if (adev->virt.fw_reserve.p_pf2vf->version != AMD_SRIOV_MSG_FW_VRAM_PF2VF_VER)
+		return -EINVAL;
+
+	pf2vf_msg = (struct amd_sriov_msg_pf2vf_info *)adev->virt.fw_reserve.p_pf2vf;
+
+	if (pf2vf_msg->header.size > (AMD_SRIOV_MSG_SIZE_KB << 10) ||
+	    pf2vf_msg->header.size <
+	    offsetof(struct amd_sriov_msg_pf2vf_info, pf2vf_ras_caps) +
+	    sizeof(pf2vf_msg->pf2vf_ras_caps))
+		return -EINVAL;
+
+	if (!pf2vf_msg->feature_flags.flags.uniras_support)
+		return -EOPNOTSUPP;
+
+	*caps = pf2vf_msg->pf2vf_ras_caps.uniras_caps;
+
+	dev_dbg(adev->dev,
+		"uniras caps: ras_ext_ecc_type=0x%x ras_int_ecc_attributes=0x%x ras_en_block_mask=0x%llx\n",
+		caps->ras_ext_ecc_type, caps->ras_int_ecc_attributes,
+		caps->ras_en_block_mask);
+
+	return 0;
+}
+
 static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev)
 {
 	struct amdgpu_virt *virt = &adev->virt;
@@ -614,9 +649,12 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 		if (amdgpu_sriov_is_unitid_support(adev))
 			adev->unitid = pf2vf->unitid;
 
-		adev->virt.ras_en_caps.all = pf2vf->ras_en_caps.all;
+		adev->virt.ras_en_caps.all = pf2vf->pf2vf_ras_caps.ras_en_caps.all;
 		adev->virt.ras_telemetry_en_caps.all =
-			pf2vf->ras_telemetry_en_caps.all;
+			pf2vf->pf2vf_ras_caps.ras_telemetry_en_caps.all;
+
+		adev->have_atomics_support = pf2vf->pcie_atomic_ops_support_flags ==
+			(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
 		break;
 	default:
 		dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index bcf7156a4a9e00..9957241bdf592a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -506,6 +506,8 @@ bool amdgpu_virt_get_rlcg_reg_access_flag(struct amdgpu_device *adev,
 					  u32 acc_flags, u32 hwip,
 					  bool write, u32 *rlcg_flag);
 u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id);
+int amdgpu_virt_get_uniras_ras_caps(struct amdgpu_device *adev,
+				    struct amd_sriov_uniras_caps *caps);
 bool amdgpu_virt_get_ras_capability(struct amdgpu_device *adev);
 int amdgpu_virt_req_ras_err_count(struct amdgpu_device *adev, enum amdgpu_ras_block block,
 				  struct ras_err_data *err_data);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
index c835504fdf2b00..5c5fdbec93c2c5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
@@ -47,7 +47,7 @@ static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = {
 	.set_config             = drm_atomic_helper_set_config,
 	.destroy                = drm_crtc_cleanup,
 	.page_flip              = drm_atomic_helper_page_flip,
-	.reset                  = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state   = drm_atomic_helper_crtc_destroy_state,
 	DRM_CRTC_VBLANK_TIMER_FUNCS,
@@ -144,7 +144,7 @@ static const struct drm_plane_funcs amdgpu_vkms_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 };
@@ -508,11 +508,6 @@ static int amdgpu_vkms_resume(struct amdgpu_ip_block *ip_block)
 	return drm_mode_config_helper_resume(adev_to_drm(ip_block->adev));
 }
 
-static bool amdgpu_vkms_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int amdgpu_vkms_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 					  enum amd_clockgating_state state)
 {
@@ -533,7 +528,6 @@ static const struct amd_ip_funcs amdgpu_vkms_ip_funcs = {
 	.hw_fini = amdgpu_vkms_hw_fini,
 	.suspend = amdgpu_vkms_suspend,
 	.resume = amdgpu_vkms_resume,
-	.is_idle = amdgpu_vkms_is_idle,
 	.set_clockgating_state = amdgpu_vkms_set_clockgating_state,
 	.set_powergating_state = amdgpu_vkms_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index bb04101b0fb507..0b8a8f82ecabf8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -33,10 +33,12 @@
 
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_ioctl.h>
 #include <drm/ttm/ttm_tt.h>
 #include <drm/drm_exec.h>
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_gmc.h"
@@ -615,9 +617,8 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	 * As soon as all page tables are in place we can start updating them
 	 * again.
 	 */
-	amdgpu_vm_eviction_lock(vm);
-	vm->evicting = false;
-	amdgpu_vm_eviction_unlock(vm);
+	scoped_guard(mutex, &vm->eviction_lock)
+		vm->evicting = false;
 
 	list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
 				 vm_status) {
@@ -677,9 +678,8 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 
 	amdgpu_vm_assert_locked(vm);
 
-	amdgpu_vm_eviction_lock(vm);
-	ret = !vm->evicting;
-	amdgpu_vm_eviction_unlock(vm);
+	scoped_guard(mutex, &vm->eviction_lock)
+		ret = !vm->evicting;
 
 	ret &= list_empty(&vm->kernel.evicted);
 
@@ -1195,7 +1195,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		uint64_t tmp, num_entries, addr;
 
 		num_entries = cursor.size >> AMDGPU_GPU_PAGE_SHIFT;
-		if (pages_addr) {
+		if (res && res->mem_type == AMDGPU_PL_NPA) {
+			addr = cursor.start;
+		} else if (pages_addr) {
 			bool contiguous = true;
 
 			if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) {
@@ -2165,6 +2167,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
 							 uint64_t addr)
 {
+	addr /= AMDGPU_GPU_PAGE_SIZE;
 	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
 }
 
@@ -2270,6 +2273,7 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
 {
 	struct amdgpu_vm_bo_base *bo_base = bo->vm_bo;
+	struct amdgpu_vm *vm;
 
 	/* Page tables of a destroyed VM can go away immediately */
 	if (!bo_base || !bo_base->vm)
@@ -2280,17 +2284,15 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo)
 		return false;
 
 	/* Try to block ongoing updates */
-	if (!amdgpu_vm_eviction_trylock(bo_base->vm))
-		return false;
+	vm = bo_base->vm;
+	scoped_cond_guard(mutex_try, return false, &vm->eviction_lock) {
 
-	/* Don't evict VM page tables while they are updated */
-	if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) {
-		amdgpu_vm_eviction_unlock(bo_base->vm);
-		return false;
-	}
+		/* Don't evict VM page tables while they are updated */
+		if (!dma_fence_is_signaled(vm->last_unlocked))
+			return false;
 
-	bo_base->vm->evicting = true;
-	amdgpu_vm_eviction_unlock(bo_base->vm);
+		vm->evicting = true;
+	}
 	return true;
 }
 
@@ -2559,6 +2561,12 @@ static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm)
 	return 0;
 }
 
+static void amdgpu_vm_render_devt(struct amdgpu_device *adev, int *major, int *minor)
+{
+	*major = DRM_MAJOR;
+	*minor = adev_to_drm(adev)->render->index;
+}
+
 /**
  * amdgpu_vm_set_task_info - Sets VMs task info.
  *
@@ -2572,6 +2580,18 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
 	if (vm->task_info->task.pid == current->pid)
 		return;
 
+	if (vm->root.bo) {
+		struct amdgpu_device *adev = amdgpu_ttm_adev(vm->root.bo->tbo.bdev);
+		int major, minor;
+
+		amdgpu_vm_render_devt(adev, &major, &minor);
+
+		if (vm->task_info->task.pid)
+			trace_amdgpu_deregister_pid(vm->task_info->task.pid,
+						    major, minor);
+		trace_amdgpu_register_pid(current->pid, major, minor);
+	}
+
 	vm->task_info->task.pid = current->pid;
 	get_task_comm(vm->task_info->task.comm, current);
 
@@ -2678,6 +2698,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	amdgpu_bo_unref(&root_bo);
 
 error_free_delayed:
+	dma_fence_put(vm->last_update);
 	dma_fence_put(vm->last_tlb_flush);
 	dma_fence_put(vm->last_unlocked);
 	ttm_lru_bulk_move_fini(&adev->mman.bdev, &vm->lru_bulk_move);
@@ -2748,6 +2769,46 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	return r;
 }
 
+/**
+ * amdgpu_vm_make_npa - Turn a GFX VM into an NPA VM
+ *
+ * @adev: amdgpu_device pointer
+ * @vm: requested vm
+ *
+ * This only works on GFX VMs that don't have any BOs added and no
+ * page tables allocated yet.
+ *
+ * Changes the following VM parameters:
+ * - use_cpu_for_update
+ * - pins page tables
+ * - initializes PTEs to no-retry encoding
+ *
+ * Reinitializes the page directory to reflect the changed ATS
+ * setting.
+ *
+ * Returns:
+ * 0 for success, -errno for errors.
+ */
+int amdgpu_vm_make_npa(struct amdgpu_device *adev, struct amdgpu_vm *vm)
+{
+	int r = amdgpu_vm_make_compute(adev, vm);
+
+	if (r)
+		return r;
+	vm->is_npa = true;
+	r = amdgpu_bo_reserve(vm->root.bo, false);
+	if (r)
+		return r;
+	r = amdgpu_bo_pin(vm->root.bo, AMDGPU_GEM_DOMAIN_VRAM);
+	amdgpu_bo_unreserve(vm->root.bo);
+	if (r)
+		return r;
+
+	vm->is_npa = true;
+
+	return 0;
+}
+
 static int amdgpu_vm_stats_is_zero(struct amdgpu_vm *vm)
 {
 	for (int i = 0; i < __AMDGPU_PL_NUM; ++i) {
@@ -2832,6 +2893,13 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 			 ti->process_name, ti->task.pid, ti->task.comm, ti->tgid);
 	}
 
+	if (vm->task_info && vm->task_info->task.pid) {
+		int major, minor;
+
+		amdgpu_vm_render_devt(adev, &major, &minor);
+		trace_amdgpu_deregister_pid(vm->task_info->task.pid, major, minor);
+	}
+
 	amdgpu_vm_put_task_info(vm->task_info);
 }
 
@@ -3052,7 +3120,8 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 	}
 
 	addr /= AMDGPU_GPU_PAGE_SIZE;
-	flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
+	flags = adev->gmc.init_pte_flags |
+		AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED |
 		AMDGPU_PTE_SYSTEM;
 
 	if (is_compute_context) {
@@ -3065,8 +3134,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 		/* Redirect the access to the dummy page */
 		value = adev->dummy_page_addr;
 		flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE |
-			AMDGPU_PTE_WRITEABLE;
-
+			 AMDGPU_PTE_WRITEABLE;
 	} else {
 		/* Let the hw retry silently on the PTE */
 		value = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index d32183cd9e0fcd..0f6634b749746f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -141,6 +141,8 @@ struct amdgpu_bo_vm;
 #define AMDGPU_PDE_PTE_FLAG(adev)	\
 	((amdgpu_ip_version((adev), GC_HWIP, 0) >= IP_VERSION(12, 0, 0)) ? AMDGPU_PDE_PTE_GFX12 : AMDGPU_PDE_PTE)
 
+/* Flag combination to set no-retry on GFX 12 */
+#define AMDGPU_VM_NORETRY_FLAGS_GFX12	(AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED)
 /* How to program VM fault handling */
 #define AMDGPU_VM_FAULT_STOP_NEVER	0
 #define AMDGPU_VM_FAULT_STOP_FIRST	1
@@ -262,80 +264,6 @@ struct amdgpu_task_info {
 	struct kref	refcount;
 };
 
-/**
- * struct amdgpu_vm_update_params
- *
- * Encapsulate some VM table update parameters to reduce
- * the number of function parameters
- *
- */
-struct amdgpu_vm_update_params {
-
-	/**
-	 * @adev: amdgpu device we do this update for
-	 */
-	struct amdgpu_device *adev;
-
-	/**
-	 * @vm: optional amdgpu_vm we do this update for
-	 */
-	struct amdgpu_vm *vm;
-
-	/**
-	 * @immediate: if changes should be made immediately
-	 */
-	bool immediate;
-
-	/**
-	 * @unlocked: true if the root BO is not locked
-	 */
-	bool unlocked;
-
-	/**
-	 * @pages_addr:
-	 *
-	 * DMA addresses to use for mapping
-	 */
-	dma_addr_t *pages_addr;
-
-	/**
-	 * @job: job to used for hw submission
-	 */
-	struct amdgpu_job *job;
-
-	/**
-	 * @num_dw_left: number of dw left for the IB
-	 */
-	unsigned int num_dw_left;
-
-	/**
-	 * @needs_flush: true whenever we need to invalidate the TLB
-	 */
-	bool needs_flush;
-
-	/**
-	 * @override_pte: true for memory that is not uncached and gmc override function is
-	 * implemented to allow MTYPE to be overridden for NUMA local memory.
-	 */
-	bool override_pte;
-
-	/**
-	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
-	 */
-	struct list_head tlb_flush_waitlist;
-};
-
-struct amdgpu_vm_update_funcs {
-	int (*map_table)(struct amdgpu_bo_vm *bo);
-	int (*prepare)(struct amdgpu_vm_update_params *p,
-		       struct amdgpu_sync *sync, u64 k_job_id);
-	int (*update)(struct amdgpu_vm_update_params *p,
-		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
-		      unsigned count, uint32_t incr, uint64_t flags);
-	int (*commit)(struct amdgpu_vm_update_params *p,
-		      struct dma_fence **fence);
-};
-
 struct amdgpu_vm_fault_info {
 	/* fault address */
 	uint64_t	addr;
@@ -441,6 +369,8 @@ struct amdgpu_vm {
 	struct ttm_lru_bulk_move lru_bulk_move;
 	/* Flag to indicate if VM is used for compute */
 	bool			is_compute_context;
+	/* Flag to indicate that page tables are for NPA mappings */
+	bool			is_npa;
 	/* Flag to indicate if VM needs a TLB fence (KFD or KGD) */
 	bool			need_tlb_fence;
 
@@ -483,6 +413,7 @@ struct amdgpu_vm_manager {
 
 	/* Global registration of recent page fault information */
 	struct amdgpu_vm_fault_info	fault_info;
+	unsigned int				npa_vmid;
 };
 
 struct amdgpu_bo_va_mapping;
@@ -500,6 +431,7 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, int32_t xcp_id);
 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+int amdgpu_vm_make_npa(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
 		      unsigned int num_fences);
@@ -600,22 +532,6 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM]);
 
-int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-		       struct amdgpu_bo_vm *vmbo, bool immediate);
-int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
-			int32_t xcp_id);
-void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
-
-int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
-			 struct amdgpu_vm_bo_base *entry);
-int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
-			  uint64_t start, uint64_t end,
-			  uint64_t dst, uint64_t flags);
-void amdgpu_vm_pt_free_work(struct work_struct *work);
-void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
-			    struct amdgpu_vm_update_params *params);
-
 #if defined(CONFIG_DEBUG_FS)
 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
 #endif
@@ -651,32 +567,6 @@ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm)
 	return atomic64_read(&vm->tlb_seq);
 }
 
-/*
- * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
- * happens while holding this lock anywhere to prevent deadlocks when
- * an MMU notifier runs in reclaim-FS context.
- */
-static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
-{
-	mutex_lock(&vm->eviction_lock);
-	vm->saved_flags = memalloc_noreclaim_save();
-}
-
-static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm)
-{
-	if (mutex_trylock(&vm->eviction_lock)) {
-		vm->saved_flags = memalloc_noreclaim_save();
-		return true;
-	}
-	return false;
-}
-
-static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
-{
-	memalloc_noreclaim_restore(vm->saved_flags);
-	mutex_unlock(&vm->eviction_lock);
-}
-
 void amdgpu_vm_update_fault_cache(struct amdgpu_device *adev,
 				  unsigned int pasid,
 				  uint64_t addr,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
index b31ff6f56f0d61..d03b9bbe20c7cb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu.h"
 #include "amdgpu_reset.h"
 #include "amdgpu_object.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
new file mode 100644
index 00000000000000..ca86eaac752352
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_internal.h
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Christian König
+ */
+#ifndef __AMDGPU_VM_INTERNAL_H__
+#define __AMDGPU_VM_INTERNAL_H__
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include "amdgpu_vm.h"
+
+struct amdgpu_device;
+struct amdgpu_vm;
+struct amdgpu_job;
+struct amdgpu_sync;
+struct amdgpu_bo_vm;
+struct amdgpu_vm_bo_base;
+struct dma_fence;
+
+/**
+ * struct amdgpu_vm_update_params
+ *
+ * Encapsulate some VM table update parameters to reduce
+ * the number of function parameters
+ *
+ */
+struct amdgpu_vm_update_params {
+
+	/**
+	 * @adev: amdgpu device we do this update for
+	 */
+	struct amdgpu_device *adev;
+
+	/**
+	 * @vm: optional amdgpu_vm we do this update for
+	 */
+	struct amdgpu_vm *vm;
+
+	/**
+	 * @immediate: if changes should be made immediately
+	 */
+	bool immediate;
+
+	/**
+	 * @unlocked: true if the root BO is not locked
+	 */
+	bool unlocked;
+
+	/**
+	 * @pages_addr:
+	 *
+	 * DMA addresses to use for mapping
+	 */
+	dma_addr_t *pages_addr;
+
+	/**
+	 * @job: job to used for hw submission
+	 */
+	struct amdgpu_job *job;
+
+	/**
+	 * @num_dw_left: number of dw left for the IB
+	 */
+	unsigned int num_dw_left;
+
+	/**
+	 * @needs_flush: true whenever we need to invalidate the TLB
+	 */
+	bool needs_flush;
+
+	/**
+	 * @override_pte: true for memory that is not uncached and gmc override function is
+	 * implemented to allow MTYPE to be overridden for NUMA local memory.
+	 */
+	bool override_pte;
+
+	/**
+	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
+	 */
+	struct list_head tlb_flush_waitlist;
+};
+
+struct amdgpu_vm_update_funcs {
+	int (*map_table)(struct amdgpu_bo_vm *bo);
+	int (*prepare)(struct amdgpu_vm_update_params *p,
+		       struct amdgpu_sync *sync, u64 k_job_id);
+	int (*update)(struct amdgpu_vm_update_params *p,
+		      struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
+		      unsigned count, uint32_t incr, uint64_t flags);
+	int (*commit)(struct amdgpu_vm_update_params *p,
+		      struct dma_fence **fence);
+};
+
+int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+		       struct amdgpu_bo_vm *vmbo, bool immediate);
+int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+			int level, bool immediate, struct amdgpu_bo_vm **vmbo,
+			int32_t xcp_id);
+void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
+			 struct amdgpu_vm_bo_base *entry);
+int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
+			  uint64_t start, uint64_t end,
+			  uint64_t dst, uint64_t flags);
+void amdgpu_vm_pt_free_work(struct work_struct *work);
+void amdgpu_vm_pt_free_list(struct amdgpu_device *adev,
+			    struct amdgpu_vm_update_params *params);
+int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm);
+
+/*
+ * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS
+ * happens while holding this lock anywhere to prevent deadlocks when
+ * an MMU notifier runs in reclaim-FS context.
+ */
+static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm)
+{
+	mutex_lock(&vm->eviction_lock);
+	vm->saved_flags = memalloc_noreclaim_save();
+}
+
+static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm)
+{
+	memalloc_noreclaim_restore(vm->saved_flags);
+	mutex_unlock(&vm->eviction_lock);
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index e43a60d0980828..1cdf2b854f2618 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -26,6 +26,7 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_job.h"
 
 /*
@@ -407,10 +408,14 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	if (adev->asic_type >= CHIP_VEGA10) {
 		if (level != AMDGPU_VM_PTB) {
+			if (vm->is_npa)
+				flags = adev->gmc.noretry_flags;
 			/* Handle leaf PDEs as PTEs */
 			flags |= AMDGPU_PDE_PTE_FLAG(adev);
 			amdgpu_gmc_get_vm_pde(adev, level,
 					      &value, &flags);
+		} else if (vm->is_npa) {
+			flags = adev->gmc.noretry_flags;
 		} else {
 			/* Workaround for fault priority problem on GMC9 */
 			flags = AMDGPU_PTE_EXECUTABLE | adev->gmc.init_pte_flags;
@@ -444,6 +449,7 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 {
 	struct amdgpu_bo_param bp;
 	unsigned int num_entries;
+	int r;
 
 	memset(&bp, 0, sizeof(bp));
 
@@ -476,7 +482,24 @@ int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (vm->root.bo)
 		bp.resv = vm->root.bo->tbo.base.resv;
 
-	return amdgpu_bo_create_vm(adev, &bp, vmbo);
+	r = amdgpu_bo_create_vm(adev, &bp, vmbo);
+	if (r)
+		return r;
+
+	/* Assumes that reservation is shared with the VM root and that the
+	 * reservation is locked
+	 */
+	if (vm->root.bo && vm->is_npa) {
+		struct amdgpu_bo *pt_bo = &(*vmbo)->bo;
+
+		r = amdgpu_bo_pin(pt_bo, AMDGPU_GEM_DOMAIN_VRAM);
+		if (r) {
+			amdgpu_bo_unref(&pt_bo);
+			return r;
+		}
+	}
+
+	return 0;
 }
 
 /**
@@ -526,6 +549,8 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev,
 	return 0;
 
 error_free_pt:
+	if (vm->is_npa)
+		amdgpu_bo_unpin(pt_bo);
 	amdgpu_bo_unref(&pt_bo);
 	return r;
 }
@@ -540,6 +565,9 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
 	if (!entry->bo)
 		return;
 
+	if (entry->vm->is_npa)
+		amdgpu_bo_unpin(entry->bo);
+
 	amdgpu_vm_update_stats(entry, entry->bo->tbo.resource, -1);
 	entry->bo->vm_bo = NULL;
 	ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
@@ -881,7 +909,6 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 		entry_end = min(entry_end, end);
 
 		do {
-			struct amdgpu_vm *vm = params->vm;
 			uint64_t upd_end = min(entry_end, frag_end);
 			unsigned int nptes = (upd_end - frag_start) >> shift;
 			uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag);
@@ -893,9 +920,7 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 
 			trace_amdgpu_vm_update_ptes(params, frag_start, upd_end,
 						    min(nptes, 32u), dst, incr,
-						    upd_flags,
-						    vm->task_info ? vm->task_info->tgid : 0,
-						    vm->immediate.fence_context);
+						    upd_flags);
 			amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt),
 						   cursor.level, pe_start, dst,
 						   nptes, incr, upd_flags);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 50cc0779c340e9..6dc71687c39836 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_vm.h"
+#include "amdgpu_vm_internal.h"
 #include "amdgpu_job.h"
 #include "amdgpu_object.h"
 #include "amdgpu_trace.h"
@@ -254,6 +255,11 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
 						     AMDGPU_KERNEL_JOB_ID_VM_UPDATE);
 			if (r)
 				return r;
+
+			/* The estimate above describes the job which was just
+			 * submitted, take the budget of the newly allocated one.
+			 */
+			ndw = p->num_dw_left;
 		}
 
 		if (!p->pages_addr) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c
index 82b8badde45300..b0002c8983861b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c
@@ -40,12 +40,12 @@ struct amdgpu_tlb_fence {
 
 static const char *amdgpu_tlb_fence_get_driver_name(struct dma_fence *fence)
 {
-	return "amdgpu tlb fence";
+	return "amdgpu_tlb_fence";
 }
 
 static const char *amdgpu_tlb_fence_get_timeline_name(struct dma_fence *f)
 {
-	return "amdgpu tlb timeline";
+	return "amdgpu_tlb_timeline";
 }
 
 static void amdgpu_tlb_fence_work(struct work_struct *work)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
index 2b45010b7ebedd..bea4984a08d187 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
@@ -827,7 +827,8 @@ static int vpe_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	const uint32_t test_pattern = 0xdeadbeef;
-	struct amdgpu_ib ib = {};
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	uint32_t index;
 	uint64_t wb_addr;
@@ -842,23 +843,29 @@ static int vpe_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	adev->wb.wb[index] = 0;
 	wb_addr = adev->wb.gpu_addr + (index * 4);
 
-	ret = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+	ret = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				       AMDGPU_IB_POOL_DIRECT,
+				       AMDGPU_KERNEL_JOB_ID_VPE_RING_TEST,
+				       &job);
 	if (ret)
 		goto err0;
-
-	ib.ptr[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0);
-	ib.ptr[1] = lower_32_bits(wb_addr);
-	ib.ptr[2] = upper_32_bits(wb_addr);
-	ib.ptr[3] = test_pattern;
-	ib.ptr[4] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
-	ib.ptr[5] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
-	ib.ptr[6] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
-	ib.ptr[7] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
-	ib.length_dw = 8;
-
-	ret = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (ret)
+	ib = &job->ibs[0];
+
+	ib->ptr[0] = VPE_CMD_HEADER(VPE_CMD_OPCODE_FENCE, 0);
+	ib->ptr[1] = lower_32_bits(wb_addr);
+	ib->ptr[2] = upper_32_bits(wb_addr);
+	ib->ptr[3] = test_pattern;
+	ib->ptr[4] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
+	ib->ptr[5] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
+	ib->ptr[6] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
+	ib->ptr[7] = VPE_CMD_HEADER(VPE_CMD_OPCODE_NOP, 0);
+	ib->length_dw = 8;
+
+	ret = amdgpu_job_submit_direct(job, ring, &f);
+	if (ret) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	ret = dma_fence_wait_timeout(f, false, timeout);
 	if (ret <= 0) {
@@ -869,7 +876,6 @@ static int vpe_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	ret = (le32_to_cpu(adev->wb.wb[index]) == test_pattern) ? 0 : -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index b15af318784b50..ab14079d1c4222 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -23,6 +23,7 @@
 #include "amdgpu.h"
 #include "amdgpu_xcp.h"
 #include "amdgpu_drv.h"
+#include "amdgpu_ip.h"
 
 #include <drm/drm_drv.h>
 #include "../amdxcp/amdgpu_xcp_drv.h"
@@ -667,6 +668,11 @@ void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr)
 		xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) |
 					  BIT(AMDGPU_TPX_PARTITION_MODE) |
 					  BIT(AMDGPU_CPX_PARTITION_MODE);
+
+		if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 3) &&
+			amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 4) &&
+			amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 5, 0))
+			xcp_mgr->supp_xcp_modes |= BIT(AMDGPU_DPX_PARTITION_MODE);
 		break;
 	case 4:
 		xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) |
@@ -689,14 +695,16 @@ void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr)
 
 int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
 {
+	int ret = 0;
+
 	/* TODO:
 	 * Stop user queues and threads, and make sure GPU is empty of work.
 	 */
 
 	if (flags & AMDGPU_XCP_OPS_KFD)
-		amdgpu_amdkfd_device_fini_sw(xcp_mgr->adev);
+		ret = amdgpu_amdkfd_prepare_partition_switch(xcp_mgr->adev);
 
-	return 0;
+	return ret;
 }
 
 int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
@@ -1056,6 +1064,11 @@ static const struct kobj_type xcp_sysfs_ktype = {
 	.sysfs_ops = &kobj_sysfs_ops,
 };
 
+bool amdgpu_xcp_is_primary(struct amdgpu_xcp *xcp)
+{
+	return xcp->ddev == adev_to_drm(xcp->xcp_mgr->adev);
+}
+
 static void amdgpu_xcp_sysfs_entries_fini(struct amdgpu_xcp_mgr *xcp_mgr, int n)
 {
 	struct amdgpu_xcp *xcp;
@@ -1105,6 +1118,7 @@ static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr)
 		if (!xcp->ddev)
 			continue;
 		sysfs_update_group(&xcp->kobj, &amdgpu_xcp_attrs_group);
+		amdgpu_ualink_xcp_sysfs_update(xcp);
 	}
 
 	return;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
index 878c1c422893cf..33157409eda0c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h
@@ -100,6 +100,11 @@ struct amdgpu_xcp_ip {
 	bool valid;
 };
 
+struct amdgpu_xcp_ualink {
+	struct kobject kobj;
+	bool sysfs;
+};
+
 struct amdgpu_xcp {
 	struct amdgpu_xcp_ip ip[AMDGPU_XCP_MAX_BLOCKS];
 
@@ -115,6 +120,7 @@ struct amdgpu_xcp {
 	struct amdgpu_sched	gpu_sched[AMDGPU_HW_IP_NUM][AMDGPU_RING_PRIO_MAX];
 	struct amdgpu_xcp_mgr *xcp_mgr;
 	struct kobject kobj;
+	struct amdgpu_xcp_ualink ualink;
 	uint64_t unique_id;
 };
 
@@ -191,6 +197,7 @@ int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags);
 int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags);
 void amdgpu_xcp_sysfs_init(struct amdgpu_device *adev);
 void amdgpu_xcp_sysfs_fini(struct amdgpu_device *adev);
+bool amdgpu_xcp_is_primary(struct amdgpu_xcp *xcp);
 
 static inline int amdgpu_xcp_get_num_xcp(struct amdgpu_xcp_mgr *xcp_mgr)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
index 33421ccdff04d7..5a253d50faa76c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
@@ -164,7 +164,8 @@ union amd_sriov_msg_feature_flags {
 		uint32_t xgmi_connected_to_cpu  : 1;
 		uint32_t ptl_support		: 1;
 		uint32_t unitid_support		: 1;
-		uint32_t reserved		: 16;
+		uint32_t uniras_support		: 1;
+		uint32_t reserved		: 15;
 	} flags;
 	uint32_t all;
 };
@@ -210,6 +211,27 @@ union amd_sriov_ras_caps {
 	uint64_t all;
 };
 
+struct amd_sriov_uniras_caps {
+	uint32_t ras_ext_ecc_type;
+	uint32_t ras_int_ecc_attributes;
+	uint64_t ras_en_block_mask;
+};
+
+/*
+ * PF2VF RAS capability words (16 bytes, layout matches legacy fields):
+ * ras_ext_ecc_type and ras_int_ecc_attributes are aliases ras_en_caps,
+ * ras_en_block_mask aliases ras_telemetry_en_caps.
+ */
+union amd_sriov_msg_pf2vf_ras_caps {
+	/* ras caps for uniras enabled case */
+	struct amd_sriov_uniras_caps uniras_caps;
+	/* ras caps for uniras disabled case*/
+	struct {
+		union amd_sriov_ras_caps ras_en_caps;
+		union amd_sriov_ras_caps ras_telemetry_en_caps;
+	};
+};
+
 union amd_sriov_msg_os_info {
 	struct {
 		uint32_t windows  : 1;
@@ -314,8 +336,7 @@ struct amd_sriov_msg_pf2vf_info {
 	/* vf bdf on host pci tree for debug only */
 	uint32_t bdf_on_host;
 	uint32_t more_bp;	//Reserved for future use.
-	union amd_sriov_ras_caps ras_en_caps;
-	union amd_sriov_ras_caps ras_telemetry_en_caps;
+	union amd_sriov_msg_pf2vf_ras_caps pf2vf_ras_caps;
 	/* PTL status response for guest */
 	uint32_t ptl_enabled;        // PTL enable status: 0=disabled, 1=enabled
 	uint32_t ptl_pref_format1;   // Current preferred format 1
@@ -507,12 +528,6 @@ struct amd_sriov_ras_chk_criti {
 	uint32_t hit;
 };
 
-union amd_sriov_ras_host_push {
-	struct amd_sriov_ras_telemetry_error_count error_count;
-	struct amd_sriov_ras_cper_dump cper_dump;
-	struct amd_sriov_ras_chk_criti chk_criti;
-};
-
 #define AMD_SRIOV_UNIRAS_BLOCKS_BUF_SIZE 4096
 #define AMD_SRIOV_UNIRAS_CMD_MAX_SIZE (4096 * 13)
 struct amd_sriov_uniras_shared_mem {
@@ -521,9 +536,19 @@ struct amd_sriov_uniras_shared_mem {
 };
 
 struct amdsriov_ras_telemetry {
-	struct amd_sriov_ras_telemetry_header header;
-	union amd_sriov_ras_host_push body;
-	struct amd_sriov_uniras_shared_mem uniras_shared_mem;
+	union {
+		struct {
+			struct amd_sriov_ras_telemetry_header header;
+
+			union {
+				struct amd_sriov_ras_telemetry_error_count error_count;
+				struct amd_sriov_ras_cper_dump cper_dump;
+				struct amd_sriov_ras_chk_criti chk_criti;
+			} body;
+		};
+
+		struct amd_sriov_uniras_shared_mem uniras_shared_mem;
+	};
 };
 
 /* version data stored in MAILBOX_MSGBUF_RCV_DW1 for future expansion */
diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
index cddfe4015f5313..05e4edf72e4d54 100644
--- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
+++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c
@@ -30,6 +30,7 @@
 #include "gfxhub_v1_2.h"
 #include "sdma_v4_4_2.h"
 #include "amdgpu_ip.h"
+#include "amdgpu_sdma.h"
 
 void aqua_vanjaram_doorbell_index_init(struct amdgpu_device *adev)
 {
@@ -391,15 +392,9 @@ static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 		!adev->in_suspend)
 		flags |= AMDGPU_XCP_OPS_KFD;
 
-	if (flags & AMDGPU_XCP_OPS_KFD) {
-		ret = amdgpu_amdkfd_check_and_lock_kfd(adev);
-		if (ret)
-			goto out;
-	}
-
 	ret = amdgpu_xcp_pre_partition_switch(xcp_mgr, flags);
 	if (ret)
-		goto unlock;
+		goto out;
 
 	num_xcc_per_xcp = __aqua_vanjaram_get_xcc_per_xcp(xcp_mgr, mode);
 	if (adev->gfx.funcs->switch_partition_mode)
@@ -413,9 +408,6 @@ static int aqua_vanjaram_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 	ret = amdgpu_xcp_post_partition_switch(xcp_mgr, flags);
 	if (!ret)
 		__aqua_vanjaram_update_available_partition_mode(xcp_mgr);
-unlock:
-	if (flags & AMDGPU_XCP_OPS_KFD)
-		amdgpu_amdkfd_unlock_kfd(adev);
 out:
 	return ret;
 }
@@ -513,6 +505,34 @@ static int aqua_vanjaram_xcp_mgr_init(struct amdgpu_device *adev)
 	return ret;
 }
 
+static void amdgpu_populate_ip_map(struct amdgpu_device *adev,
+				   enum amd_hw_ip_block_type ip_block,
+				   uint32_t inst_mask)
+{
+	int l = 0, i;
+
+	while (inst_mask) {
+		i = ffs(inst_mask) - 1;
+		adev->ip_map.dev_inst[ip_block][l++] = i;
+		inst_mask &= ~(1 << i);
+	}
+	for (; l < HWIP_MAX_INSTANCE; l++)
+		adev->ip_map.dev_inst[ip_block][l] = -1;
+}
+
+static void amdgpu_ip_map_aqua_vanjaram_override(struct amdgpu_device *adev)
+{
+	u32 ip_map[][2] = {
+		{ GC_HWIP, adev->gfx.xcc_mask },
+		{ SDMA0_HWIP, adev->sdma.sdma_mask },
+		{ VCN_HWIP, adev->vcn.inst_mask },
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ip_map); ++i)
+		amdgpu_populate_ip_map(adev, ip_map[i][0], ip_map[i][1]);
+}
+
 int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)
 {
 	u32 mask, avail_inst, inst_mask = adev->sdma.sdma_mask;
@@ -547,7 +567,7 @@ int aqua_vanjaram_init_soc_config(struct amdgpu_device *adev)
 	if (ret)
 		return ret;
 
-	amdgpu_ip_map_init(adev);
+	amdgpu_ip_map_aqua_vanjaram_override(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index e0e585f280e218..1a599d5cf6a5cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -1420,6 +1420,11 @@ static void atom_get_vbios_date(struct atom_context *ctx)
 
 	p_rom = ctx->bios;
 
+	if (ctx->bios_size < OFFSET_TO_VBIOS_DATE + 14) {
+		ctx->date[0] = '\0';
+		return;
+	}
+
 	date_in_rom = p_rom + OFFSET_TO_VBIOS_DATE;
 
 	ctx->date[0] = '2';
@@ -1483,10 +1488,15 @@ static void atom_get_vbios_pn(struct atom_context *ctx)
 		vbios_str = p_rom + OFFSET_TO_VBIOS_PART_NUMBER;
 	}
 
-	if (*vbios_str == 0) {
+	if (vbios_str >= p_rom + ctx->bios_size)
+		vbios_str = NULL;
+
+	if (vbios_str == NULL || *vbios_str == 0) {
 		vbios_str = atom_find_str_in_rom(ctx, BIOS_ATOM_PREFIX, 3, 1024, 64);
 		if (vbios_str == NULL)
 			vbios_str += sizeof(BIOS_ATOM_PREFIX) - 1;
+		if (vbios_str >= p_rom + ctx->bios_size)
+			vbios_str = NULL;
 	}
 	OPTIMIZER_HIDE_VAR(vbios_str);
 	if (vbios_str != NULL && *vbios_str == 0)
@@ -1494,7 +1504,9 @@ static void atom_get_vbios_pn(struct atom_context *ctx)
 
 	if (vbios_str != NULL) {
 		count = 0;
-		while ((count < BIOS_STRING_LENGTH) && vbios_str[count] >= ' ' &&
+		while ((count < BIOS_STRING_LENGTH) &&
+		       vbios_str + count < p_rom + ctx->bios_size &&
+		       vbios_str[count] >= ' ' &&
 		       vbios_str[count] <= 'z') {
 			ctx->vbios_pn[count] = vbios_str[count];
 			count++;
@@ -1527,9 +1539,10 @@ static void atom_get_vbios_version(struct atom_context *ctx)
 	/* find anchor ATOMBIOSBK-AMD */
 	vbios_ver =
 		atom_find_str_in_rom(ctx, BIOS_VERSION_PREFIX, start, end, 64);
-	if (vbios_ver != NULL) {
-		/* skip ATOMBIOSBK-AMD VER */
-		vbios_ver += 18;
+	if (vbios_ver != NULL)
+		vbios_ver += 18; /* skip ATOMBIOSBK-AMD VER */
+
+	if (vbios_ver != NULL && vbios_ver + STRLEN_NORMAL <= p_rom + ctx->bios_size) {
 		memcpy(ctx->vbios_ver_str, vbios_ver, STRLEN_NORMAL);
 	} else {
 		ctx->vbios_ver_str[0] = '\0';
@@ -1545,7 +1558,13 @@ static void atom_get_vbios_build(struct atom_context *ctx)
 	base = CU16(ATOM_ROM_TABLE_PTR);
 	atom_rom_hdr = CSTR(base);
 
+	if ((uint32_t)base + ATOM_ROM_CFG_PTR + sizeof(uint16_t) > ctx->bios_size)
+		return;
+
 	str = CSTR(CU16(base + ATOM_ROM_CFG_PTR));
+	if (str >= atom_rom_hdr)
+		return;
+
 	/* Skip config string */
 	while (str < atom_rom_hdr && *str++)
 		;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index e037c08c854799..d29991a4af4c54 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -2137,12 +2137,6 @@ static int cik_common_resume(struct amdgpu_ip_block *ip_block)
 	return cik_common_hw_init(ip_block);
 }
 
-static bool cik_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
-
 
 static int cik_common_soft_reset(struct amdgpu_ip_block *ip_block)
 {
@@ -2168,7 +2162,6 @@ static const struct amd_ip_funcs cik_common_ip_funcs = {
 	.hw_init = cik_common_hw_init,
 	.hw_fini = cik_common_hw_fini,
 	.resume = cik_common_resume,
-	.is_idle = cik_common_is_idle,
 	.soft_reset = cik_common_soft_reset,
 	.set_clockgating_state = cik_common_set_clockgating_state,
 	.set_powergating_state = cik_common_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index 876a3256dba41c..314d8f1d64cc44 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -357,17 +357,6 @@ static int cik_ih_resume(struct amdgpu_ip_block *ip_block)
 	return cik_ih_hw_init(ip_block);
 }
 
-static bool cik_ih_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
-		return false;
-
-	return true;
-}
-
 static int cik_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -435,7 +424,6 @@ static const struct amd_ip_funcs cik_ih_ip_funcs = {
 	.hw_fini = cik_ih_hw_fini,
 	.suspend = cik_ih_suspend,
 	.resume = cik_ih_resume,
-	.is_idle = cik_ih_is_idle,
 	.wait_for_idle = cik_ih_wait_for_idle,
 	.soft_reset = cik_ih_soft_reset,
 	.set_clockgating_state = cik_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index b951328d94cfb7..8a09fa91293180 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -27,6 +27,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 #include "cikd.h"
 #include "cik.h"
@@ -55,8 +56,6 @@ static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev);
 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev);
 static int cik_sdma_soft_reset(struct amdgpu_ip_block *ip_block);
 
-u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev);
-
 MODULE_FIRMWARE("amdgpu/bonaire_sdma.bin");
 MODULE_FIRMWARE("amdgpu/bonaire_sdma1.bin");
 MODULE_FIRMWARE("amdgpu/hawaii_sdma.bin");
@@ -318,18 +317,6 @@ static void cik_sdma_gfx_stop(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * cik_sdma_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues (CIK).
- */
-static void cik_sdma_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * cik_ctx_switch_enable - stop the async dma engines context switch
  *
@@ -402,7 +389,6 @@ static void cik_sdma_enable(struct amdgpu_device *adev, bool enable)
 
 	if (!enable) {
 		cik_sdma_gfx_stop(adev);
-		cik_sdma_rlc_stop(adev);
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -503,20 +489,6 @@ static int cik_sdma_gfx_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-/**
- * cik_sdma_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them (CIK).
- * Returns 0 for success, error for failure.
- */
-static int cik_sdma_rlc_resume(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-	return 0;
-}
-
 /**
  * cik_sdma_load_microcode - load the sDMA ME ucode
  *
@@ -579,9 +551,6 @@ static int cik_sdma_start(struct amdgpu_device *adev)
 
 	/* start the gfx rings and rlc compute queues */
 	r = cik_sdma_gfx_resume(adev);
-	if (r)
-		return r;
-	r = cik_sdma_rlc_resume(adev);
 	if (r)
 		return r;
 
@@ -652,7 +621,8 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
 static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
@@ -666,22 +636,28 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	tmp = 0xCAFEDEAD;
 	adev->wb.wb[index] = cpu_to_le32(tmp);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256,
-					AMDGPU_IB_POOL_DIRECT, &ib);
-	if (r)
-		goto err0;
 
-	ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE,
-				SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = 1;
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r)
+		goto err0;
+	ib = &job->ibs[0];
+
+	ib->ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE,
+				 SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = 1;
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -697,7 +673,6 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1225,7 +1200,6 @@ static const struct amd_ip_funcs cik_sdma_ip_funcs = {
 	.hw_fini = cik_sdma_hw_fini,
 	.suspend = cik_sdma_suspend,
 	.resume = cik_sdma_resume,
-	.is_idle = cik_sdma_is_idle,
 	.wait_for_idle = cik_sdma_wait_for_idle,
 	.soft_reset = cik_sdma_soft_reset,
 	.set_clockgating_state = cik_sdma_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index bc7a2e06ab5fae..8d02602f84337a 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -351,17 +351,6 @@ static int cz_ih_resume(struct amdgpu_ip_block *ip_block)
 	return cz_ih_hw_init(ip_block);
 }
 
-static bool cz_ih_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
-		return false;
-
-	return true;
-}
-
 static int cz_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -431,7 +420,6 @@ static const struct amd_ip_funcs cz_ih_ip_funcs = {
 	.hw_fini = cz_ih_hw_fini,
 	.suspend = cz_ih_suspend,
 	.resume = cz_ih_resume,
-	.is_idle = cz_ih_is_idle,
 	.wait_for_idle = cz_ih_wait_for_idle,
 	.soft_reset = cz_ih_soft_reset,
 	.set_clockgating_state = cz_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index f2977fe6d82425..7de7adc8f225ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -2921,11 +2921,6 @@ static int dce_v10_0_resume(struct amdgpu_ip_block *ip_block)
 	return amdgpu_display_resume_helper(adev);
 }
 
-static bool dce_v10_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static void dce_v10_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
 						     int crtc,
 						     enum amdgpu_interrupt_state state)
@@ -3267,7 +3262,6 @@ static const struct amd_ip_funcs dce_v10_0_ip_funcs = {
 	.hw_fini = dce_v10_0_hw_fini,
 	.suspend = dce_v10_0_suspend,
 	.resume = dce_v10_0_resume,
-	.is_idle = dce_v10_0_is_idle,
 	.set_clockgating_state = dce_v10_0_set_clockgating_state,
 	.set_powergating_state = dce_v10_0_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index c68de0fe1d7d38..3c1d6901127d1a 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -2867,11 +2867,6 @@ static int dce_v6_0_resume(struct amdgpu_ip_block *ip_block)
 	return amdgpu_display_resume_helper(adev);
 }
 
-static bool dce_v6_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static void dce_v6_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
 						     int crtc,
 						     enum amdgpu_interrupt_state state)
@@ -3167,7 +3162,6 @@ static const struct amd_ip_funcs dce_v6_0_ip_funcs = {
 	.hw_fini = dce_v6_0_hw_fini,
 	.suspend = dce_v6_0_suspend,
 	.resume = dce_v6_0_resume,
-	.is_idle = dce_v6_0_is_idle,
 	.set_clockgating_state = dce_v6_0_set_clockgating_state,
 	.set_powergating_state = dce_v6_0_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index c3906270f25efd..a17944d5e84343 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -2839,11 +2839,6 @@ static int dce_v8_0_resume(struct amdgpu_ip_block *ip_block)
 	return amdgpu_display_resume_helper(adev);
 }
 
-static bool dce_v8_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static void dce_v8_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
 						     int crtc,
 						     enum amdgpu_interrupt_state state)
@@ -3184,7 +3179,6 @@ static const struct amd_ip_funcs dce_v8_0_ip_funcs = {
 	.hw_fini = dce_v8_0_hw_fini,
 	.suspend = dce_v8_0_suspend,
 	.resume = dce_v8_0_resume,
-	.is_idle = dce_v8_0_is_idle,
 	.set_clockgating_state = dce_v8_0_set_clockgating_state,
 	.set_powergating_state = dce_v8_0_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index 00798d80479f36..045b9c7d2a835f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -4071,15 +4071,14 @@ static int gfx_v10_0_ring_test_ring(struct amdgpu_ring *ring)
 static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned int index;
 	uint64_t gpu_addr;
 	uint32_t *cpu_ptr;
 	long r;
 
-	memset(&ib, 0, sizeof(ib));
-
 	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
@@ -4088,22 +4087,28 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 	cpu_ptr = &adev->wb.wb[index];
 
-	r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 20,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
 		goto err1;
 	}
+	ib = &job->ibs[0];
 
-	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
-	ib.ptr[2] = lower_32_bits(gpu_addr);
-	ib.ptr[3] = upper_32_bits(gpu_addr);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
+	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib->ptr[2] = lower_32_bits(gpu_addr);
+	ib->ptr[3] = upper_32_bits(gpu_addr);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err2;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -4118,7 +4123,6 @@ static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	else
 		r = -EINVAL;
 err2:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
 	amdgpu_wb_free(adev, index);
@@ -5364,7 +5368,7 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev)
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) {
 		nv_grbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
@@ -7611,17 +7615,6 @@ static int gfx_v10_0_resume(struct amdgpu_ip_block *ip_block)
 	return gfx_v10_0_hw_init(ip_block);
 }
 
-static bool gfx_v10_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
-				GRBM_STATUS, GUI_ACTIVE))
-		return false;
-	else
-		return true;
-}
-
 static int gfx_v10_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned int i;
@@ -9821,7 +9814,6 @@ static const struct amd_ip_funcs gfx_v10_0_ip_funcs = {
 	.hw_fini = gfx_v10_0_hw_fini,
 	.suspend = gfx_v10_0_suspend,
 	.resume = gfx_v10_0_resume,
-	.is_idle = gfx_v10_0_is_idle,
 	.wait_for_idle = gfx_v10_0_wait_for_idle,
 	.soft_reset = gfx_v10_0_soft_reset,
 	.set_clockgating_state = gfx_v10_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 0ff5a80aa91833..a7ed4923857bb6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -622,7 +622,8 @@ static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring)
 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	uint64_t gpu_addr;
@@ -634,8 +635,6 @@ static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	    ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 		return 0;
 
-	memset(&ib, 0, sizeof(ib));
-
 	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
@@ -644,22 +643,28 @@ static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 	cpu_ptr = &adev->wb.wb[index];
 
-	r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 20,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
 		goto err1;
 	}
+	ib = &job->ibs[0];
 
-	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
-	ib.ptr[2] = lower_32_bits(gpu_addr);
-	ib.ptr[3] = upper_32_bits(gpu_addr);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
+	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib->ptr[2] = lower_32_bits(gpu_addr);
+	ib->ptr[3] = upper_32_bits(gpu_addr);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err2;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -674,7 +679,6 @@ static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	else
 		r = -EINVAL;
 err2:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
 	amdgpu_wb_free(adev, index);
@@ -2213,7 +2217,7 @@ static void gfx_v11_0_constants_init(struct amdgpu_device *adev)
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) {
 		soc21_grbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
@@ -5099,17 +5103,6 @@ static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block)
 	return gfx_v11_0_hw_init(ip_block);
 }
 
-static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
-				GRBM_STATUS, GUI_ACTIVE))
-		return false;
-	else
-		return true;
-}
-
 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -7057,7 +7050,6 @@ static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
 	.hw_fini = gfx_v11_0_hw_fini,
 	.suspend = gfx_v11_0_suspend,
 	.resume = gfx_v11_0_resume,
-	.is_idle = gfx_v11_0_is_idle,
 	.wait_for_idle = gfx_v11_0_wait_for_idle,
 	.soft_reset = gfx_v11_0_soft_reset,
 	.set_clockgating_state = gfx_v11_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index e2a81a55c63b36..ec1e142549c52d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -494,7 +494,8 @@ static int gfx_v12_0_ring_test_ring(struct amdgpu_ring *ring)
 static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	uint64_t gpu_addr;
@@ -506,8 +507,6 @@ static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	    ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 		return 0;
 
-	memset(&ib, 0, sizeof(ib));
-
 	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
@@ -516,22 +515,28 @@ static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 	cpu_ptr = &adev->wb.wb[index];
 
-	r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 16,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
 		goto err1;
 	}
+	ib = &job->ibs[0];
 
-	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
-	ib.ptr[2] = lower_32_bits(gpu_addr);
-	ib.ptr[3] = upper_32_bits(gpu_addr);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
+	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib->ptr[2] = lower_32_bits(gpu_addr);
+	ib->ptr[3] = upper_32_bits(gpu_addr);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err2;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -546,7 +551,6 @@ static int gfx_v12_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	else
 		r = -EINVAL;
 err2:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
 	amdgpu_wb_free(adev, index);
@@ -1852,7 +1856,7 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev)
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) {
 		soc24_grbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
@@ -3929,17 +3933,6 @@ static int gfx_v12_0_resume(struct amdgpu_ip_block *ip_block)
 	return gfx_v12_0_hw_init(ip_block);
 }
 
-static bool gfx_v12_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS),
-				GRBM_STATUS, GUI_ACTIVE))
-		return false;
-	else
-		return true;
-}
-
 static int gfx_v12_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -5343,7 +5336,6 @@ static const struct amd_ip_funcs gfx_v12_0_ip_funcs = {
 	.hw_fini = gfx_v12_0_hw_fini,
 	.suspend = gfx_v12_0_suspend,
 	.resume = gfx_v12_0_resume,
-	.is_idle = gfx_v12_0_is_idle,
 	.wait_for_idle = gfx_v12_0_wait_for_idle,
 	.set_clockgating_state = gfx_v12_0_set_clockgating_state,
 	.set_powergating_state = gfx_v12_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index e49d8a79045cc2..db541939555b7d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -61,6 +61,8 @@
 #define regCP_HQD_IB_CONTROL_DEFAULT                                              0x00100000
 
 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin");
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc.bin");
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec_1.bin");
 MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc_1.bin");
 
 #define SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0	0x00000001
@@ -201,12 +203,7 @@ static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev,
 static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev);
 static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num,
 				       u32 sh_num, u32 instance, int xcc_id);
-static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
-				     uint32_t val);
 static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
-static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
-					   uint16_t pasid, uint32_t flush_type,
-					   bool all_hub, uint8_t dst_sel);
 static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id);
 static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id);
 static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
@@ -214,142 +211,10 @@ static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev,
 static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev,
 					 bool enable, int xcc_id);
 static int gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device *adev);
-
-static void gfx_v12_1_kiq_set_resources(struct amdgpu_ring *kiq_ring,
-					uint64_t queue_mask)
-{
-	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
-	amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
-			  PACKET3_SET_RESOURCES_QUEUE_TYPE(0));	/* vmid_mask:0 queue_type:0 (KIQ) */
-	amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask));	/* queue mask lo */
-	amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask));	/* queue mask hi */
-	amdgpu_ring_write(kiq_ring, 0);	/* gws mask lo */
-	amdgpu_ring_write(kiq_ring, 0);	/* gws mask hi */
-	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */
-	amdgpu_ring_write(kiq_ring, 0);
-}
-
-static void gfx_v12_1_kiq_map_queues(struct amdgpu_ring *kiq_ring,
-				     struct amdgpu_ring *ring)
-{
-	uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
-	uint64_t wptr_addr = ring->wptr_gpu_addr;
-	uint32_t me = 0, eng_sel = 0;
-
-	switch (ring->funcs->type) {
-	case AMDGPU_RING_TYPE_COMPUTE:
-		me = 1;
-		eng_sel = 0;
-		break;
-	case AMDGPU_RING_TYPE_MES:
-		me = 2;
-		eng_sel = 5;
-		break;
-	default:
-		WARN_ON(1);
-	}
-
-	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
-	/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
-	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
-			  PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
-			  PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
-			  PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
-			  PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
-			  PACKET3_MAP_QUEUES_ME((me)) |
-			  PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */
-			  PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
-			  PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */
-	amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
-	amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
-	amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
-	amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
-	amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
-}
-
-static void gfx_v12_1_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
-				       struct amdgpu_ring *ring,
-				       enum amdgpu_unmap_queues_action action,
-				       u64 gpu_addr, u64 seq)
-{
-	struct amdgpu_device *adev = kiq_ring->adev;
-	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
-
-	if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) {
-		amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr,
-					      seq, kiq_ring->xcc_id);
-		return;
-	}
-
-	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
-	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
-			  PACKET3_UNMAP_QUEUES_ACTION(action) |
-			  PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
-			  PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
-			  PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
-	amdgpu_ring_write(kiq_ring,
-		  PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
-
-	if (action == PREEMPT_QUEUES_NO_UNMAP) {
-		amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
-		amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
-		amdgpu_ring_write(kiq_ring, seq);
-	} else {
-		amdgpu_ring_write(kiq_ring, 0);
-		amdgpu_ring_write(kiq_ring, 0);
-		amdgpu_ring_write(kiq_ring, 0);
-	}
-}
-
-static void gfx_v12_1_kiq_query_status(struct amdgpu_ring *kiq_ring,
-				       struct amdgpu_ring *ring,
-				       u64 addr, u64 seq)
-{
-	uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
-
-	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
-	amdgpu_ring_write(kiq_ring,
-			  PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
-			  PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
-			  PACKET3_QUERY_STATUS_COMMAND(2));
-	amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
-			  PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
-			  PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
-	amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
-	amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
-	amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
-	amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
-}
-
-static void gfx_v12_1_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
-					  uint16_t pasid,
-					  uint32_t flush_type,
-					  bool all_hub)
-{
-	gfx_v12_1_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1);
-}
-
-static const struct kiq_pm4_funcs gfx_v12_1_kiq_pm4_funcs = {
-	.kiq_set_resources = gfx_v12_1_kiq_set_resources,
-	.kiq_map_queues = gfx_v12_1_kiq_map_queues,
-	.kiq_unmap_queues = gfx_v12_1_kiq_unmap_queues,
-	.kiq_query_status = gfx_v12_1_kiq_query_status,
-	.kiq_invalidate_tlbs = gfx_v12_1_kiq_invalidate_tlbs,
-	.set_resources_size = 8,
-	.map_queues_size = 7,
-	.unmap_queues_size = 6,
-	.query_status_size = 7,
-	.invalidate_tlbs_size = 2,
-};
-
-static void gfx_v12_1_set_kiq_pm4_funcs(struct amdgpu_device *adev)
-{
-	int i, num_xcc;
-
-	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
-	for (i =0; i < num_xcc; i++)
-		adev->gfx.kiq[i].pmf = &gfx_v12_1_kiq_pm4_funcs;
-}
+static void gfx_v12_1_xcc_update_medium_grain_clock_gating(
+	struct amdgpu_device *adev, bool enable, int xcc_id, bool force);
+static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
+				      struct amdgpu_ring *ring, unsigned vmid);
 
 static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 				   int mem_space, int opt, uint32_t addr0,
@@ -399,14 +264,9 @@ static int gfx_v12_1_ring_test_ring(struct amdgpu_ring *ring)
 		return r;
 	}
 
-	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) {
-		gfx_v12_1_ring_emit_wreg(ring, xcc_offset, 0xDEADBEEF);
-	} else {
-		amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
-		amdgpu_ring_write(ring, xcc_offset -
-				  PACKET3_SET_UCONFIG_REG_START);
-		amdgpu_ring_write(ring, 0xDEADBEEF);
-	}
+	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
+	amdgpu_ring_write(ring, xcc_offset - PACKET3_SET_UCONFIG_REG_START);
+	amdgpu_ring_write(ring, 0xDEADBEEF);
 	amdgpu_ring_commit(ring);
 
 	for (i = 0; i < adev->usec_timeout; i++) {
@@ -427,20 +287,14 @@ static int gfx_v12_1_ring_test_ring(struct amdgpu_ring *ring)
 static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	uint64_t gpu_addr;
 	volatile uint32_t *cpu_ptr;
 	long r;
 
-	/* MES KIQ fw hasn't indirect buffer support for now */
-	if (adev->enable_mes_kiq &&
-	    ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
-		return 0;
-
-	memset(&ib, 0, sizeof(ib));
-
 	r = amdgpu_wb_get(adev, &index);
 	if (r)
 		return r;
@@ -449,22 +303,28 @@ static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
 	cpu_ptr = &adev->wb.wb[index];
 
-	r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 16,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r) {
 		dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r);
 		goto err1;
 	}
+	ib = &job->ibs[0];
 
-	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1);
-	ib.ptr[2] = lower_32_bits(gpu_addr);
-	ib.ptr[3] = upper_32_bits(gpu_addr);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
+	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib->ptr[1] = PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1);
+	ib->ptr[2] = lower_32_bits(gpu_addr);
+	ib->ptr[3] = upper_32_bits(gpu_addr);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err2;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -479,7 +339,6 @@ static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	else
 		r = -EINVAL;
 err2:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
 	amdgpu_wb_free(adev, index);
@@ -549,9 +408,15 @@ static int gfx_v12_1_init_microcode(struct amdgpu_device *adev)
 			goto out;
 	}
 
-	err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
-				   AMDGPU_UCODE_REQUIRED,
-				   "amdgpu/%s_mec.bin", ucode_prefix);
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) &&
+	    adev->rev_id == 0)
+		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
+					   AMDGPU_UCODE_REQUIRED,
+					   "amdgpu/%s_mec_1.bin", ucode_prefix);
+	else
+		err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
+					   AMDGPU_UCODE_REQUIRED,
+					   "amdgpu/%s_mec.bin", ucode_prefix);
 	if (err)
 		goto out;
 	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
@@ -567,7 +432,8 @@ static int gfx_v12_1_init_microcode(struct amdgpu_device *adev)
 	adev->gfx.mec2_fw = NULL;
 
 	if (adev->gfx.imu.funcs) {
-		if (adev->gfx.imu.funcs->init_microcode) {
+		if (adev->gfx.imu.funcs->init_microcode &&
+		    adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
 			err = adev->gfx.imu.funcs->init_microcode(adev);
 			if (err)
 				dev_err(adev->dev, "Failed to load imu firmware!\n");
@@ -680,7 +546,7 @@ static void gfx_v12_1_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev)
 static int gfx_v12_1_rlc_init(struct amdgpu_device *adev)
 {
 	const struct cs_section_def *cs_data;
-	int r, i, num_xcc;
+	int r;
 
 	adev->gfx.rlc.cs_data = gfx12_cs_data;
 
@@ -693,13 +559,6 @@ static int gfx_v12_1_rlc_init(struct amdgpu_device *adev)
 			return r;
 	}
 
-	/* init spm vmid with 0xf */
-	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
-	for (i = 0; i < num_xcc; i++) {
-		if (adev->gfx.rlc.funcs->update_spm_vmid)
-			adev->gfx.rlc.funcs->update_spm_vmid(adev, i, NULL, 0xf);
-	}
-
 	return 0;
 }
 
@@ -839,19 +698,16 @@ static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev,
 	soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
 }
 
-#define regGFX_IMU_PARTITION_SWITCH		0x5f8c
-#define regGFX_IMU_PARTITION_SWITCH_BASE_IDX	1
-#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP__SHIFT	0x2
-#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP_MASK		0x0000003CL
-
 static int gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device *adev)
 {
 	u32 reg_data;
 
 	/* the register data is expected to be the same on all instances */
-	reg_data = RREG32_SOC15(GC, GET_INST(GC, 0), regGFX_IMU_PARTITION_SWITCH);
+	reg_data = RREG32_SOC15(GC, GET_INST(GC, 0),
+				regGFX_IMU_PARTITION_SWITCH_SHADOW);
 
-	return REG_GET_FIELD(reg_data, GFX_IMU_PARTITION_SWITCH, TOTAL_XCCS_IN_XCP);
+	return REG_GET_FIELD(reg_data, GFX_IMU_PARTITION_SWITCH_SHADOW,
+			     TOTAL_XCCS_IN_XCP);
 }
 
 static int gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
@@ -1373,7 +1229,6 @@ static void gfx_v12_1_ip_dump(struct amdgpu_ip_block *ip_block)
 
 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
 
-	amdgpu_gfx_off_ctrl(adev, false);
 	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
 		xcc_offset = xcc_id * reg_count;
 		for (i = 0; i < reg_count; i++)
@@ -1381,7 +1236,6 @@ static void gfx_v12_1_ip_dump(struct amdgpu_ip_block *ip_block)
 				RREG32(SOC15_REG_ENTRY_OFFSET_INST(gc_reg_list_12_1[i],
 								   GET_INST(GC, xcc_id)));
 	}
-	amdgpu_gfx_off_ctrl(adev, true);
 
 	/* dump compute queue registers for all instances */
 	if (!adev->gfx.ip_dump_compute_queues)
@@ -1390,7 +1244,7 @@ static void gfx_v12_1_ip_dump(struct amdgpu_ip_block *ip_block)
 	num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec *
 		adev->gfx.mec.num_queue_per_pipe;
 	reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1);
-	amdgpu_gfx_off_ctrl(adev, false);
+
 	mutex_lock(&adev->srbm_mutex);
 	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
 		xcc_offset = xcc_id * reg_count * num_inst;
@@ -1417,7 +1271,6 @@ static void gfx_v12_1_ip_dump(struct amdgpu_ip_block *ip_block)
 	}
 	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, 0);
 	mutex_unlock(&adev->srbm_mutex);
-	amdgpu_gfx_off_ctrl(adev, true);
 }
 
 static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
@@ -1488,6 +1341,13 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	/* pmr ea error */
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GFX,
+				GFX_12_1_0__SRCID__PMR_EA_ERROR_INTERRUPT,
+				&adev->gfx.pmr_ea_irq);
+	if (r)
+		return r;
+
 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
 
 	r = gfx_v12_1_rlc_init(adev);
@@ -1522,18 +1382,6 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 			}
 		}
 
-		if (!adev->enable_mes_kiq) {
-			r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, xcc_id);
-			if (r) {
-				dev_err(adev->dev, "Failed to init KIQ BOs!\n");
-				return r;
-			}
-
-			r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
-			if (r)
-				return r;
-		}
-
 		r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_1_compute_mqd), xcc_id);
 		if (r)
 			return r;
@@ -1560,6 +1408,9 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 
 	gfx_v12_1_alloc_ip_dump(adev);
 
+	adev->gfx.me.use_mmio_for_reset = false;
+	adev->gfx.mec.use_mmio_for_reset = false;
+
 	mutex_init(&adev->gfx.mec.reset_mutex);
 
 	return 0;
@@ -1581,15 +1432,9 @@ static int gfx_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
 	for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++)
 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
 
-	for (i = 0; i < num_xcc; i++) {
+	for (i = 0; i < num_xcc; i++)
 		amdgpu_gfx_mqd_sw_fini(adev, i);
 
-		if (!adev->enable_mes_kiq) {
-			amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[i].ring);
-			amdgpu_gfx_kiq_fini(adev, i);
-		}
-	}
-
 	gfx_v12_1_rlc_fini(adev);
 	gfx_v12_1_mec_fini(adev);
 
@@ -1651,61 +1496,6 @@ static u32 gfx_v12_1_get_sa_active_bitmap(struct amdgpu_device *adev,
 	return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask));
 }
 
-static u32 gfx_v12_1_get_rb_active_bitmap(struct amdgpu_device *adev,
-					  int xcc_id)
-{
-	u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask;
-	u32 rb_mask;
-
-	gc_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
-					   regCC_RB_BACKEND_DISABLE);
-	gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask,
-					    CC_RB_BACKEND_DISABLE,
-					    BACKEND_DISABLE);
-	gc_user_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
-						regGC_USER_RB_BACKEND_DISABLE);
-	gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask,
-						 GC_USER_RB_BACKEND_DISABLE,
-						 BACKEND_DISABLE);
-	rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se *
-					    adev->gfx.config.max_shader_engines);
-
-	return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask));
-}
-
-static void gfx_v12_1_setup_rb(struct amdgpu_device *adev)
-{
-	u32 rb_bitmap_width_per_sa;
-	u32 max_sa;
-	u32 active_sa_bitmap;
-	u32 global_active_rb_bitmap;
-	u32 active_rb_bitmap = 0;
-	u32 i;
-	int xcc_id;
-
-	for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) {
-		/* query sa bitmap from SA_UNIT_DISABLE registers */
-		active_sa_bitmap = gfx_v12_1_get_sa_active_bitmap(adev, xcc_id);
-		/* query rb bitmap from RB_BACKEND_DISABLE registers */
-		global_active_rb_bitmap = gfx_v12_1_get_rb_active_bitmap(adev, xcc_id);
-
-		/* generate active rb bitmap according to active sa bitmap */
-		max_sa = adev->gfx.config.max_shader_engines *
-			 adev->gfx.config.max_sh_per_se;
-		rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se /
-					 adev->gfx.config.max_sh_per_se;
-		for (i = 0; i < max_sa; i++) {
-			if (active_sa_bitmap & (1 << i))
-				active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa));
-		}
-
-		active_rb_bitmap |= global_active_rb_bitmap;
-	}
-
-	adev->gfx.config.backend_enable_mask = active_rb_bitmap;
-	adev->gfx.config.num_rbs = hweight32(active_rb_bitmap);
-}
-
 static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev,
 					    int xcc_id)
 {
@@ -1744,15 +1534,6 @@ static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev,
 	mutex_unlock(&adev->srbm_mutex);
 }
 
-static void gfx_v12_1_tcp_harvest(struct amdgpu_device *adev)
-{
-	/* TODO: harvest feature to be added later. */
-}
-
-static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev)
-{
-}
-
 static void gfx_v12_1_xcc_xnack_set_chicken_bits(struct amdgpu_device *adev, int xcc_id)
 {
 	/* NOTE: COMPRESSION_ENABLE is used a chicken bit to enable/disable xcc xnack */
@@ -1775,7 +1556,7 @@ static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) {
 		soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
 		/* CP and shaders */
 		WREG32_SOC15(GC, GET_INST(GC, xcc_id),
@@ -1803,9 +1584,7 @@ static void gfx_v12_1_constants_init(struct amdgpu_device *adev)
 
 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
 
-	gfx_v12_1_setup_rb(adev);
 	gfx_v12_1_get_cu_info(adev, &adev->gfx.cu_info);
-	gfx_v12_1_get_tcc_info(adev);
 	adev->gfx.config.pa_sc_tile_steering_override = 0;
 
 	for (i = 0; i < num_xcc; i++)
@@ -2052,6 +1831,9 @@ static int gfx_v12_1_xcc_rlc_resume(struct amdgpu_device *adev,
 {
 	int r;
 
+	/* init spm vmid with 0xf */
+	gfx_v12_1_update_spm_vmid(adev, xcc_id, NULL, 0xf);
+
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 		gfx_v12_1_xcc_init_csb(adev, xcc_id);
 
@@ -2240,8 +2022,6 @@ static void gfx_v12_1_xcc_cp_compute_enable(struct amdgpu_device *adev,
 						 enable ? 0 : 1);
 	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, data);
 
-	adev->gfx.kiq[xcc_id].ring.sched.ready = enable;
-
 	udelay(50);
 }
 
@@ -2404,21 +2184,6 @@ static int gfx_v12_1_xcc_cp_compute_load_microcode_rs64(struct amdgpu_device *ad
 	return 0;
 }
 
-static void gfx_v12_1_xcc_kiq_setting(struct amdgpu_ring *ring,
-				      int xcc_id)
-{
-	uint32_t tmp;
-	struct amdgpu_device *adev = ring->adev;
-
-	/* tell RLC which is KIQ queue */
-	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS);
-	tmp &= 0xffffff00;
-	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
-	tmp |= 0x80;
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp);
-}
-
 static void gfx_v12_1_xcc_cp_set_doorbell_range(struct amdgpu_device *adev,
 						int xcc_id)
 {
@@ -2509,6 +2274,7 @@ static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
 			    (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, SCOPE, 3);
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
@@ -2574,149 +2340,6 @@ static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	return 0;
 }
 
-static int gfx_v12_1_xcc_kiq_init_register(struct amdgpu_ring *ring,
-					   int xcc_id)
-{
-	struct amdgpu_device *adev = ring->adev;
-	struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
-	int j;
-
-	/* inactivate the queue */
-	if (amdgpu_sriov_vf(adev))
-		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 0);
-
-	/* disable wptr polling */
-	WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0);
-
-	/* write the EOP addr */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR,
-	       mqd->cp_hqd_eop_base_addr_lo);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR_HI,
-	       mqd->cp_hqd_eop_base_addr_hi);
-
-	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL,
-	       mqd->cp_hqd_eop_control);
-
-	/* enable doorbell? */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
-	       mqd->cp_hqd_pq_doorbell_control);
-
-	/* disable the queue if it's active */
-	if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) {
-		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1);
-		for (j = 0; j < adev->usec_timeout; j++) {
-			if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1))
-				break;
-			udelay(1);
-		}
-		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST,
-		       mqd->cp_hqd_dequeue_request);
-		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR,
-		       mqd->cp_hqd_pq_rptr);
-		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
-		       mqd->cp_hqd_pq_wptr_lo);
-		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
-		       mqd->cp_hqd_pq_wptr_hi);
-	}
-
-	/* set the pointer to the MQD */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR,
-	       mqd->cp_mqd_base_addr_lo);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI,
-	       mqd->cp_mqd_base_addr_hi);
-
-	/* set MQD vmid to 0 */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL,
-	       mqd->cp_mqd_control);
-
-	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE,
-	       mqd->cp_hqd_pq_base_lo);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI,
-	       mqd->cp_hqd_pq_base_hi);
-
-	/* set up the HQD, this is similar to CP_RB0_CNTL */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL,
-	       mqd->cp_hqd_pq_control);
-
-	/* set the wb address whether it's enabled or not */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR,
-		mqd->cp_hqd_pq_rptr_report_addr_lo);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
-		mqd->cp_hqd_pq_rptr_report_addr_hi);
-
-	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR,
-	       mqd->cp_hqd_pq_wptr_poll_addr_lo);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
-	       mqd->cp_hqd_pq_wptr_poll_addr_hi);
-
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL,
-	       mqd->cp_hqd_pq_doorbell_control);
-
-	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO,
-	       mqd->cp_hqd_pq_wptr_lo);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI,
-	       mqd->cp_hqd_pq_wptr_hi);
-
-	/* set the vmid for the queue */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, mqd->cp_hqd_vmid);
-
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE,
-	       mqd->cp_hqd_persistent_state);
-
-	/* activate the queue */
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE,
-	       mqd->cp_hqd_active);
-
-	if (ring->use_doorbell)
-		WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_STATUS, DOORBELL_ENABLE, 1);
-
-	return 0;
-}
-
-static int gfx_v12_1_xcc_kiq_init_queue(struct amdgpu_ring *ring,
-					int xcc_id)
-{
-	struct amdgpu_device *adev = ring->adev;
-	struct v12_1_compute_mqd *mqd = ring->mqd_ptr;
-
-	gfx_v12_1_xcc_kiq_setting(ring, xcc_id);
-
-	if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
-		/* reset MQD to a clean status */
-		if (adev->gfx.kiq[xcc_id].mqd_backup)
-			memcpy(mqd, adev->gfx.kiq[xcc_id].mqd_backup, sizeof(*mqd));
-
-		/* reset ring buffer */
-		ring->wptr = 0;
-		amdgpu_ring_clear_ring(ring);
-
-		mutex_lock(&adev->srbm_mutex);
-		soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
-		gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
-		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
-		mutex_unlock(&adev->srbm_mutex);
-	} else {
-		memset((void *)mqd, 0, sizeof(*mqd));
-		if (amdgpu_sriov_vf(adev) && adev->in_suspend)
-			amdgpu_ring_clear_ring(ring);
-		mutex_lock(&adev->srbm_mutex);
-		soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id));
-		amdgpu_ring_init_mqd(ring);
-		gfx_v12_1_xcc_kiq_init_register(ring, xcc_id);
-		soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
-		mutex_unlock(&adev->srbm_mutex);
-
-		if (adev->gfx.kiq[xcc_id].mqd_backup)
-			memcpy(adev->gfx.kiq[xcc_id].mqd_backup, mqd, sizeof(*mqd));
-	}
-
-	return 0;
-}
-
 static int gfx_v12_1_xcc_kcq_init_queue(struct amdgpu_ring *ring,
 					int xcc_id)
 {
@@ -2747,32 +2370,6 @@ static int gfx_v12_1_xcc_kcq_init_queue(struct amdgpu_ring *ring,
 	return 0;
 }
 
-static int gfx_v12_1_xcc_kiq_resume(struct amdgpu_device *adev,
-				    int xcc_id)
-{
-	struct amdgpu_ring *ring;
-	int r;
-
-	ring = &adev->gfx.kiq[xcc_id].ring;
-
-	r = amdgpu_bo_reserve(ring->mqd_obj, false);
-	if (unlikely(r != 0))
-		return r;
-
-	r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
-	if (unlikely(r != 0)) {
-		amdgpu_bo_unreserve(ring->mqd_obj);
-		return r;
-	}
-
-	gfx_v12_1_xcc_kiq_init_queue(ring, xcc_id);
-	amdgpu_bo_kunmap(ring->mqd_obj);
-	ring->mqd_ptr = NULL;
-	amdgpu_bo_unreserve(ring->mqd_obj);
-	ring->sched.ready = true;
-	return 0;
-}
-
 static int gfx_v12_1_xcc_kcq_resume(struct amdgpu_device *adev,
 				    int xcc_id)
 {
@@ -2809,6 +2406,9 @@ static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask
 	int r, i, xcc_id;
 	struct amdgpu_ring *ring;
 
+	if (!adev->mes.kiq_hw_init)
+		return -EINVAL;
+
 	for_each_inst(xcc_id, xcc_mask) {
 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 			/* legacy firmware loading */
@@ -2817,27 +2417,16 @@ static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask
 				return r;
 		}
 
-		/* GFX CGCG and LS is set by default */
-		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
-			gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, true, xcc_id);
+		gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, false,
+							       xcc_id, true);
+		/* GFX CGCG and LS is disabled by rlc fw */
+		gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
 
 		gfx_v12_1_xcc_cp_set_doorbell_range(adev, xcc_id);
 
 		gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id);
 
-		if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) {
-			r = amdgpu_mes_kiq_hw_init(adev, xcc_id);
-			/*
-			 * With MES, GFX KIQ ring is owned by the MES and is never
-			 * initialized/used directly by the driver, so it must
-			 * not be left flagged as ready. mes_v12_0_hw_init() clears
-			 * but clear here if MES init fails
-			 */
-			if (r)
-				adev->gfx.kiq[xcc_id].ring.sched.ready = false;
-		} else {
-			r = gfx_v12_1_xcc_kiq_resume(adev, xcc_id);
-		}
+		r = amdgpu_mes_kiq_hw_init(adev, xcc_id);
 		if (r)
 			return r;
 
@@ -2851,6 +2440,8 @@ static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask
 			if (r)
 				return r;
 		}
+		gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, true,
+							       xcc_id, true);
 	}
 
 	return 0;
@@ -2919,42 +2510,6 @@ static int gfx_v12_1_gfxhub_enable(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int get_gb_addr_config(struct amdgpu_device *adev)
-{
-	u32 gb_addr_config;
-
-	gb_addr_config = RREG32_SOC15(GC, GET_INST(GC, 0), regGB_ADDR_CONFIG_READ);
-	if (gb_addr_config == 0)
-		return -EINVAL;
-
-	adev->gfx.config.gb_addr_config_fields.num_pkrs =
-		1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG_READ, NUM_PKRS);
-
-	adev->gfx.config.gb_addr_config = gb_addr_config;
-
-	adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
-			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
-				      GB_ADDR_CONFIG_READ, NUM_PIPES);
-
-	adev->gfx.config.max_tile_pipes =
-		adev->gfx.config.gb_addr_config_fields.num_pipes;
-
-	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
-			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
-				      GB_ADDR_CONFIG_READ, MAX_COMPRESSED_FRAGS);
-	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
-			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
-				      GB_ADDR_CONFIG_READ, NUM_RB_PER_SE);
-	adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
-			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
-				      GB_ADDR_CONFIG_READ, NUM_SHADER_ENGINES);
-	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
-			REG_GET_FIELD(adev->gfx.config.gb_addr_config,
-				      GB_ADDR_CONFIG_READ, PIPE_INTERLEAVE_SIZE));
-
-	return 0;
-}
-
 static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev,
 					   int xcc_id)
 {
@@ -3103,9 +2658,6 @@ static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
 
 	adev->gfx.is_poweron = true;
 
-	if (get_gb_addr_config(adev))
-		DRM_WARN("Invalid gb_addr_config !\n");
-
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)
 		gfx_v12_1_config_gfx_rs64(adev);
 
@@ -3124,12 +2676,6 @@ static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	/*
-	 * init golden registers and rlc resume may override some registers,
-	 * reconfig them here
-	 */
-	gfx_v12_1_tcp_harvest(adev);
-
 	r = gfx_v12_1_cp_resume(adev);
 	if (r)
 		return r;
@@ -3273,7 +2819,6 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
 		adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
 						  AMDGPU_MAX_COMPUTE_RINGS);
 
-	gfx_v12_1_set_kiq_pm4_funcs(adev);
 	gfx_v12_1_set_ring_funcs(adev);
 	gfx_v12_1_set_irq_funcs(adev);
 	gfx_v12_1_set_rlc_funcs(adev);
@@ -3284,6 +2829,11 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
 
 	amdgpu_init_rlc_reg_funcs(adev);
 
+	/* Set NPA address size here as its needed in amdgpu_ttm_init().
+	 * NPA address space is 52 bits or 2^40 pages long.
+	 */
+	adev->ualink.npa_size = 1ULL << 40;
+
 	return gfx_v12_1_init_microcode(adev);
 }
 
@@ -3392,20 +2942,7 @@ static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable)
 static int gfx_v12_1_set_powergating_state(struct amdgpu_ip_block *ip_block,
 					   enum amd_powergating_state state)
 {
-	struct amdgpu_device *adev = ip_block->adev;
-	bool enable = (state == AMD_PG_STATE_GATE);
-
-	if (amdgpu_sriov_vf(adev))
-		return 0;
-
-	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
-	case IP_VERSION(12, 1, 0):
-		amdgpu_gfx_off_ctrl(adev, enable);
-		break;
-	default:
-		break;
-	}
-
+	/* No powergating state for GC v12.1 */
 	return 0;
 }
 
@@ -3490,37 +3027,41 @@ static void gfx_v12_1_xcc_update_coarse_grain_clock_gating(struct amdgpu_device
 	}
 }
 
-static void gfx_v12_1_xcc_update_medium_grain_clock_gating(struct amdgpu_device *adev,
-							   bool enable, int xcc_id)
+static void gfx_v12_1_xcc_update_medium_grain_clock_gating(
+	struct amdgpu_device *adev, bool enable, int xcc_id, bool force)
 {
 	uint32_t data, def;
-	if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)))
+	bool support_mgcg;
+
+	support_mgcg = force || (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG);
+
+	if (!support_mgcg)
 		return;
 
 	/* It is disabled by HW by default */
 	if (enable) {
-		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
-			/* 1 - RLC_CGTT_MGCG_OVERRIDE */
-			def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
+		/* 1 - RLC_CGTT_MGCG_OVERRIDE */
+		def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+					  regRLC_CGTT_MGCG_OVERRIDE);
 
-			data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
-				  RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
-				  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
+		data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
+			  RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
+			  RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
 
-			if (def != data)
-				WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
-		}
+		if (def != data)
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+				     regRLC_CGTT_MGCG_OVERRIDE, data);
 	} else {
-		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
-			def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE);
+		def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id),
+					  regRLC_CGTT_MGCG_OVERRIDE);
 
-			data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
-				 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
-				 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
+		data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
+			 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
+			 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK);
 
-			if (def != data)
-				WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data);
-		}
+		if (def != data)
+			WREG32_SOC15(GC, GET_INST(GC, xcc_id),
+				     regRLC_CGTT_MGCG_OVERRIDE, data);
 	}
 }
 
@@ -3590,7 +3131,8 @@ static int gfx_v12_1_xcc_update_gfx_clock_gating(struct amdgpu_device *adev,
 
 	gfx_v12_1_xcc_update_coarse_grain_clock_gating(adev, enable, xcc_id);
 
-	gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, enable, xcc_id);
+	gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, enable, xcc_id,
+						       false);
 
 	gfx_v12_1_xcc_update_repeater_fgcg(adev, enable, xcc_id);
 
@@ -3777,90 +3319,22 @@ static void gfx_v12_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 			       upper_32_bits(addr), seq, 0xffffffff, 4);
 }
 
-static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring,
-				   uint16_t pasid, uint32_t flush_type,
-				   bool all_hub, uint8_t dst_sel)
-{
-	amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
-	amdgpu_ring_write(ring,
-			  PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) |
-			  PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
-			  PACKET3_INVALIDATE_TLBS_PASID(pasid) |
-			  PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
-}
-
 static void gfx_v12_1_ring_emit_vm_flush(struct amdgpu_ring *ring,
 					 unsigned vmid, uint64_t pd_addr)
 {
 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
 
 	/* compute doesn't have PFP */
-	if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
-		/* sync PFP to ME, otherwise we might get invalid PFP reads */
-		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
-		amdgpu_ring_write(ring, 0x0);
-	}
-}
-
-static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
-					  u64 seq, unsigned int flags)
-{
-	struct amdgpu_device *adev = ring->adev;
-
-	/* write fence seq to the "addr" */
-	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-	amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
-	amdgpu_ring_write(ring, lower_32_bits(addr));
-	amdgpu_ring_write(ring, upper_32_bits(addr));
-	amdgpu_ring_write(ring, lower_32_bits(seq));
-
-	if (flags & AMDGPU_FENCE_FLAG_INT) {
-		/* set register to trigger INT */
-		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-		amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(0) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
-		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
-		amdgpu_ring_write(ring, 0);
-		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
-	}
-}
-
-static void gfx_v12_1_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
-				     uint32_t reg_val_offs)
-{
-	struct amdgpu_device *adev = ring->adev;
-
-	reg = soc_v1_0_normalize_xcc_reg_offset(reg);
-
-	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
-	amdgpu_ring_write(ring, 0 |	/* src: register*/
-				(5 << 8) |	/* dst: memory */
-				(1 << 20));	/* write confirm */
-	amdgpu_ring_write(ring, reg);
-	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
-				reg_val_offs * 4));
-	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
-				reg_val_offs * 4));
 }
 
 static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring,
 				     uint32_t reg,
 				     uint32_t val)
 {
-	uint32_t cmd = 0;
-
 	reg = soc_v1_0_normalize_xcc_reg_offset(reg);
 
-	switch (ring->funcs->type) {
-	case AMDGPU_RING_TYPE_KIQ:
-		cmd = (1 << 16); /* no inc addr */
-		break;
-	default:
-		cmd = PACKET3_WRITE_DATA__WR_CONFIRM(1);
-		break;
-	}
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-	amdgpu_ring_write(ring, cmd);
+	amdgpu_ring_write(ring, PACKET3_WRITE_DATA__WR_CONFIRM(1));
 	amdgpu_ring_write(ring, reg);
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, val);
@@ -4156,6 +3630,36 @@ static int gfx_v12_1_rlc_poison_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int gfx_v12_1_pmr_ea_irq(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry)
+{
+	uint32_t ras_blk = RAS_BLOCK_ID__GFX;
+	struct ras_ih_info ih_info = {0};
+	uint32_t ctx_id = entry->src_data[0];
+	int i, num_xcc;
+
+	if (ctx_id & 0x1) {
+		/* request RLC MCA logging here
+		 * nHT CTO status clearing is handled by RLC firmware
+		 */
+
+		ih_info.block = ras_blk;
+		ih_info.reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
+		amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info);
+	}
+
+	if (ctx_id & 0x2) {
+		num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+		/* ea_ill_op_status will be cleared in a future update */
+		for (i = 0; i < num_xcc; i++)
+			dev_warn(adev->dev,
+				"an EA illegal-op is detected and need to clear ea_ill_op_status\n");
+	}
+
+	return 0;
+}
+
 static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
 {
 	const unsigned int gcr_cntl =
@@ -4186,7 +3690,6 @@ static const struct amd_ip_funcs gfx_v12_1_ip_funcs = {
 	.hw_fini = gfx_v12_1_hw_fini,
 	.suspend = gfx_v12_1_suspend,
 	.resume = gfx_v12_1_resume,
-	.is_idle = gfx_v12_1_is_idle,
 	.wait_for_idle = gfx_v12_1_wait_for_idle,
 	.set_clockgating_state = gfx_v12_1_set_clockgating_state,
 	.set_powergating_state = gfx_v12_1_set_powergating_state,
@@ -4225,41 +3728,12 @@ static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_compute = {
 	.emit_mem_sync = gfx_v12_1_emit_mem_sync,
 };
 
-static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_kiq = {
-	.type = AMDGPU_RING_TYPE_KIQ,
-	.align_mask = 0xff,
-	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
-	.support_64bit_ptrs = true,
-	.get_rptr = gfx_v12_1_ring_get_rptr_compute,
-	.get_wptr = gfx_v12_1_ring_get_wptr_compute,
-	.set_wptr = gfx_v12_1_ring_set_wptr_compute,
-	.emit_frame_size =
-		7 + /* gfx_v12_1_ring_emit_pipeline_sync */
-		SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
-		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
-		2 + /* gfx_v12_1_ring_emit_vm_flush */
-		8 + 8 + 8, /* gfx_v12_1_ring_emit_fence_kiq x3 for user fence, vm fence */
-	.emit_ib_size =	7, /* gfx_v12_1_ring_emit_ib_compute */
-	.emit_ib = gfx_v12_1_ring_emit_ib_compute,
-	.emit_fence = gfx_v12_1_ring_emit_fence_kiq,
-	.test_ring = gfx_v12_1_ring_test_ring,
-	.test_ib = gfx_v12_1_ring_test_ib,
-	.insert_nop = amdgpu_ring_insert_nop,
-	.pad_ib = amdgpu_ring_generic_pad_ib,
-	.emit_rreg = gfx_v12_1_ring_emit_rreg,
-	.emit_wreg = gfx_v12_1_ring_emit_wreg,
-	.emit_reg_wait = gfx_v12_1_ring_emit_reg_wait,
-	.emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait,
-};
-
 static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i, j, num_xcc;
 
 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
 	for (i = 0; i < num_xcc; i++) {
-		adev->gfx.kiq[i].ring.funcs = &gfx_v12_1_ring_funcs_kiq;
-
 		for (j = 0; j < adev->gfx.num_compute_rings; j++)
 			adev->gfx.compute_ring[j + i * adev->gfx.num_compute_rings].funcs =
 						&gfx_v12_1_ring_funcs_compute;
@@ -4285,6 +3759,10 @@ static const struct amdgpu_irq_src_funcs gfx_v12_1_rlc_poison_irq_funcs = {
 	.process = gfx_v12_1_rlc_poison_irq,
 };
 
+static const struct amdgpu_irq_src_funcs gfx_v12_1_pmr_ea_irq_funcs = {
+	.process = gfx_v12_1_pmr_ea_irq,
+};
+
 static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
@@ -4298,6 +3776,9 @@ static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev)
 
 	adev->gfx.rlc_poison_irq.num_types = 1;
 	adev->gfx.rlc_poison_irq.funcs = &gfx_v12_1_rlc_poison_irq_funcs;
+
+	adev->gfx.pmr_ea_irq.num_types = 1;
+	adev->gfx.pmr_ea_irq.funcs = &gfx_v12_1_pmr_ea_irq_funcs;
 }
 
 static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
index 77b4f6ea532f5e..349c28685c7c0c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
@@ -53,8 +53,6 @@
 
 /* Packet 3 types */
 #define	PACKET3_NOP					0x10
-#define	PACKET3_CLEAR_STATE				0x12
-#define	PACKET3_INDEX_BUFFER_SIZE			0x13
 #define	PACKET3_DISPATCH_DIRECT				0x15
 #define	PACKET3_DISPATCH_INDIRECT			0x16
 #define	PACKET3_ATOMIC_MEM				0x1E
@@ -81,23 +79,10 @@
 #define 	PACKET3_ATOMIC_MEM__TEMPORAL__NT 1
 #define 	PACKET3_ATOMIC_MEM__TEMPORAL__FW 2
 #define 	PACKET3_ATOMIC_MEM__TEMPORAL__UC 3
-#define	PACKET3_OCCLUSION_QUERY				0x1F
-#define	PACKET3_SET_PREDICATION				0x20
-#define	PACKET3_REG_RMW					0x21
 #define	PACKET3_COND_EXEC				0x22
 #define	PACKET3_PRED_EXEC				0x23
 #define 	PACKET3_PRED_EXEC__EXEC_COUNT(x) ((((unsigned)(x)) & 0x3FFF) << 0)
 #define 	PACKET3_PRED_EXEC__VIRTUALXCCID_SELECT(x) ((((unsigned)(x)) & 0xFF) << 24)
-#define	PACKET3_DRAW_INDIRECT				0x24
-#define	PACKET3_DRAW_INDEX_INDIRECT			0x25
-#define	PACKET3_INDEX_BASE				0x26
-#define	PACKET3_DRAW_INDEX_2				0x27
-#define	PACKET3_CONTEXT_CONTROL				0x28
-#define	PACKET3_DRAW_INDIRECT_MULTI			0x2C
-#define	PACKET3_DRAW_INDEX_AUTO				0x2D
-#define	PACKET3_NUM_INSTANCES				0x2F
-#define	PACKET3_DRAW_INDEX_MULTI_AUTO			0x30
-#define	PACKET3_DRAW_INDEX_OFFSET_2			0x35
 #define	PACKET3_WRITE_DATA				0x37
 #define 	PACKET3_WRITE_DATA__DST_SEL(x) ((((unsigned)(x)) & 0xF) << 8)
 #define 	PACKET3_WRITE_DATA__DST_SEL__MEM_MAPPED_REGISTER 0
@@ -148,7 +133,6 @@
 #define 	PACKET3_WRITE_DATA__DST_MMREG_ADDR_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
 #define 	PACKET3_WRITE_DATA__DST_MEM_ADDR_HI(x) ((unsigned)(x))
 #define 	PACKET3_WRITE_DATA__DATA(x) ((unsigned)(x))
-#define	PACKET3_DRAW_INDEX_INDIRECT_MULTI		0x38
 #define	PACKET3_WAIT_REG_MEM				0x3C
 #define		PACKET3_WAIT_REG_MEM__FUNCTION(x) ((((unsigned)(x)) & 0x7) << 0)
 #define		PACKET3_WAIT_REG_MEM__FUNCTION__ALWAYS_PASS 0
@@ -298,7 +282,6 @@
 #define 	PACKET3_COPY_DATA__DST_SCOPE__SYSTEM 3
 #define 	PACKET3_COPY_DATA__PQ_EXE_STATUS__DEFAULT 0
 #define 	PACKET3_COPY_DATA__PQ_EXE_STATUS__PHASE_UPDATE 1
-#define	PACKET3_PFP_SYNC_ME				0x42
 #define	PACKET3_COND_WRITE				0x45
 #define	PACKET3_EVENT_WRITE				0x46
 #define		PACKET3_EVENT_WRITE__EVENT_TYPE(x) ((((unsigned)(x)) & 0x3F) << 0)
@@ -309,8 +292,6 @@
 #define 	PACKET3_EVENT_WRITE__EVENT_INDEX__OTHER 0
 #define 	PACKET3_EVENT_WRITE__EVENT_INDEX__SAMPLE_PIPELINESTAT 2
 #define 	PACKET3_EVENT_WRITE__EVENT_INDEX__CS_PARTIAL_FLUSH 4
-#define	PACKET3_EVENT_WRITE_EOP				0x47
-#define	PACKET3_EVENT_WRITE_EOS				0x48
 #define	PACKET3_RELEASE_MEM				0x49
 #define		PACKET3_RELEASE_MEM__EVENT_TYPE(x)  ((((unsigned)(x)) & 0x3F) << 0)
 #define		PACKET3_RELEASE_MEM__WAIT_SYNC(x)  ((((unsigned)(x)) & 0x1) << 7)
@@ -383,9 +364,6 @@
 #define		PACKET3_RELEASE_MEM__CMP_DATA_HI(x)     ((unsigned)(x))
 #define		PACKET3_RELEASE_MEM__INT_CTXID(x)     ((unsigned)(x))
 
-#define	PACKET3_PREAMBLE_CNTL				0x4A
-#              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
-#              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
 #define	PACKET3_DMA_DATA				0x50
 /* 1. header
  * 2. CONTROL
@@ -436,7 +414,6 @@
 #              define PACKET3_DMA_DATA_CMD_DAIC     (1 << 29)
 #              define PACKET3_DMA_DATA_CMD_RAW_WAIT (1 << 30)
 #              define PACKET3_DMA_DATA_CMD_DIS_WC   (1 << 30)
-#define	PACKET3_CONTEXT_REG_RMW				0x51
 #define	PACKET3_ACQUIRE_MEM				0x58
 /* 1.  HEADER */
 #define		PACKET3_ACQUIRE_MEM__COHER_SIZE(x) ((unsigned)(x))
@@ -510,20 +487,11 @@
 		 * 2: REVERSE
 		 */
 #define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GCR_RANGE_IS_PA  (1 << 18)
-#define	PACKET3_GEN_PDEPTE				0x5B
 #define	PACKET3_PRIME_UTCL2				0x5D
-#define	PACKET3_LOAD_UCONFIG_REG			0x5E
-#define	PACKET3_LOAD_SH_REG				0x5F
-#define	PACKET3_LOAD_CONFIG_REG				0x60
-#define	PACKET3_LOAD_CONTEXT_REG			0x61
-#define	PACKET3_LOAD_COMPUTE_STATE			0x62
 #define	PACKET3_LOAD_SH_REG_INDEX			0x63
 #define	PACKET3_SET_CONFIG_REG				0x68
 #define		PACKET3_SET_CONFIG_REG_START			0x00002000
 #define		PACKET3_SET_CONFIG_REG_END			0x00002c00
-#define	PACKET3_SET_CONTEXT_REG				0x69
-#define		PACKET3_SET_CONTEXT_REG_START			0x0000a000
-#define		PACKET3_SET_CONTEXT_REG_END			0x0000a400
 #define	PACKET3_SET_SH_REG				0x76
 #define		PACKET3_SET_SH_REG_START			0x00002c00
 #define		PACKET3_SET_SH_REG_END				0x00003000
@@ -533,17 +501,12 @@
 #define 	PACKET3_SET_SH_REG__REG_DATA(x) (((unsigned)(x)))
 #define 	PACKET3_SET_SH_REG__INDEX__DEFAULT 0
 #define 	PACKET3_SET_SH_REG__INDEX__INSERT_VMID 1
-#define	PACKET3_SET_SH_REG_OFFSET			0x77
 #define	PACKET3_SET_QUEUE_REG				0x78
 #define	PACKET3_SET_UCONFIG_REG				0x79
 #define		PACKET3_SET_UCONFIG_REG_START			0x0000c000
 #define		PACKET3_SET_UCONFIG_REG_END			0x0000c400
 #define 	PACKET3_SET_UCONFIG_REG__REG_OFFSET(x) ((((unsigned)(x)) & 0xFFFF) << 0)
 #define 	PACKET3_SET_UCONFIG_REG__REG_DATA(x) (((unsigned)(x)))
-#define	PACKET3_SET_UCONFIG_REG_INDEX			0x7A
-#define	PACKET3_DISPATCH_DRAW_PREAMBLE			0x8C
-#define	PACKET3_DISPATCH_DRAW				0x8D
-#define	PACKET3_INDEX_ATTRIBUTES_INDIRECT		0x91
 #define	PACKET3_WAIT_REG_MEM64				0x93
 #define	PACKET3_HDP_FLUSH				0x95
 #define	PACKET3_INVALIDATE_TLBS				0x98
@@ -552,9 +515,7 @@
 #define PACKET3_INVALIDATE_TLBS_PASID(x)       ((x) << 5)
 #define PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(x)  ((x) << 29)
 
-#define	PACKET3_DMA_DATA_FILL_MULTI			0x9A
 #define	PACKET3_SET_SH_REG_INDEX			0x9B
-#define	PACKET3_LOAD_CONTEXT_REG_INDEX			0x9F
 #define	PACKET3_SET_RESOURCES				0xA0
 /* 1. header
  * 2. CONTROL
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 563abcf718b715..95e939363db62e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -1965,24 +1965,30 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct dma_fence *f = NULL;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	uint32_t tmp = 0;
 	long r;
 
 	WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
-	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
-	ib.ptr[1] = mmSCRATCH_REG0 - PACKET3_SET_CONFIG_REG_START;
-	ib.ptr[2] = 0xDEADBEEF;
-	ib.length_dw = 3;
+	ib = &job->ibs[0];
+	ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
+	ib->ptr[1] = mmSCRATCH_REG0 - PACKET3_SET_CONFIG_REG_START;
+	ib->ptr[2] = 0xDEADBEEF;
+	ib->length_dw = 3;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto error;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1998,7 +2004,6 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 error:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 	return r;
 }
@@ -3665,7 +3670,6 @@ static const struct amd_ip_funcs gfx_v6_0_ip_funcs = {
 	.hw_fini = gfx_v6_0_hw_fini,
 	.suspend = gfx_v6_0_suspend,
 	.resume = gfx_v6_0_resume,
-	.is_idle = gfx_v6_0_is_idle,
 	.wait_for_idle = gfx_v6_0_wait_for_idle,
 	.soft_reset = gfx_v6_0_soft_reset,
 	.set_clockgating_state = gfx_v6_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 5fc19fefca81a8..549d1239e326b4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1938,7 +1938,7 @@ static void gfx_v7_0_constants_init(struct amdgpu_device *adev)
 	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, 0) {
 		if (i == 0)
 			sh_mem_base = 0;
 		else
@@ -2292,25 +2292,32 @@ static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
 static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	uint32_t tmp = 0;
 	long r;
 
 	WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r)
 		return r;
 
-	ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
-	ib.ptr[1] = mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START;
-	ib.ptr[2] = 0xDEADBEEF;
-	ib.length_dw = 3;
+	ib = &job->ibs[0];
+	ib->ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
+	ib->ptr[1] = mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START;
+	ib->ptr[2] = 0xDEADBEEF;
+	ib->length_dw = 3;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto error;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -2326,7 +2333,6 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 error:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 	return r;
 }
@@ -4511,16 +4517,6 @@ static int gfx_v7_0_resume(struct amdgpu_ip_block *ip_block)
 	return gfx_v7_0_hw_init(ip_block);
 }
 
-static bool gfx_v7_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	if (RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK)
-		return false;
-	else
-		return true;
-}
-
 static int gfx_v7_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -4951,7 +4947,6 @@ static const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
 	.hw_fini = gfx_v7_0_hw_fini,
 	.suspend = gfx_v7_0_suspend,
 	.resume = gfx_v7_0_resume,
-	.is_idle = gfx_v7_0_is_idle,
 	.wait_for_idle = gfx_v7_0_wait_for_idle,
 	.soft_reset = gfx_v7_0_soft_reset,
 	.set_clockgating_state = gfx_v7_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 7f91186ef1d1e1..ab4123ddbb9426 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -868,9 +868,9 @@ static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
-
 	unsigned int index;
 	uint64_t gpu_addr;
 	uint32_t tmp;
@@ -882,22 +882,27 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
-	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 20,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r)
 		goto err1;
 
-	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
-	ib.ptr[2] = lower_32_bits(gpu_addr);
-	ib.ptr[3] = upper_32_bits(gpu_addr);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
+	ib = &job->ibs[0];
+	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib->ptr[2] = lower_32_bits(gpu_addr);
+	ib->ptr[3] = upper_32_bits(gpu_addr);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err2;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -914,7 +919,6 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err2:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
 	amdgpu_wb_free(adev, index);
@@ -1211,12 +1215,7 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
 out:
 	if (err) {
 		dev_err(adev->dev, "gfx8: Failed to load firmware %s gfx firmware\n", chip_name);
-		amdgpu_ucode_release(&adev->gfx.pfp_fw);
-		amdgpu_ucode_release(&adev->gfx.me_fw);
-		amdgpu_ucode_release(&adev->gfx.ce_fw);
-		amdgpu_ucode_release(&adev->gfx.rlc_fw);
-		amdgpu_ucode_release(&adev->gfx.mec_fw);
-		amdgpu_ucode_release(&adev->gfx.mec2_fw);
+		gfx_v8_0_free_microcode(adev);
 	}
 	return err;
 }
@@ -1475,7 +1474,8 @@ static const u32 sec_ded_counter_registers[] =
 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 {
 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	int r, i;
 	u32 tmp;
@@ -1513,106 +1513,109 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 	total_size += sizeof(sgpr_init_compute_shader);
 
 	/* allocate an indirect buffer to put the commands in */
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, total_size,
-					AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, total_size,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_RUN_SHADER,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r);
 		return r;
 	}
+	ib = &job->ibs[0];
 
 	/* load the compute shaders */
 	for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
-		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
+		ib->ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
 
 	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
-		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
+		ib->ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
 
 	/* init the ib length to 0 */
-	ib.length_dw = 0;
+	ib->length_dw = 0;
 
 	/* VGPR */
 	/* write the register state for the compute dispatch */
 	for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
-		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
-		ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
-		ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
+		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib->ptr[ib->length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
+		ib->ptr[ib->length_dw++] = vgpr_init_regs[i + 1];
 	}
 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
-	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
-	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
-	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
-	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+	gpu_addr = (ib->gpu_addr + (u64)vgpr_offset) >> 8;
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib->ptr[ib->length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
+	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
 
 	/* write dispatch packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
-	ib.ptr[ib.length_dw++] = 8; /* x */
-	ib.ptr[ib.length_dw++] = 1; /* y */
-	ib.ptr[ib.length_dw++] = 1; /* z */
-	ib.ptr[ib.length_dw++] =
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib->ptr[ib->length_dw++] = 8; /* x */
+	ib->ptr[ib->length_dw++] = 1; /* y */
+	ib->ptr[ib->length_dw++] = 1; /* z */
+	ib->ptr[ib->length_dw++] =
 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
 
 	/* write CS partial flush packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
-	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
 
 	/* SGPR1 */
 	/* write the register state for the compute dispatch */
 	for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
-		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
-		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
-		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
+		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib->ptr[ib->length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
+		ib->ptr[ib->length_dw++] = sgpr1_init_regs[i + 1];
 	}
 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
-	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
-	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
-	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
-	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+	gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8;
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib->ptr[ib->length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
+	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
 
 	/* write dispatch packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
-	ib.ptr[ib.length_dw++] = 8; /* x */
-	ib.ptr[ib.length_dw++] = 1; /* y */
-	ib.ptr[ib.length_dw++] = 1; /* z */
-	ib.ptr[ib.length_dw++] =
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib->ptr[ib->length_dw++] = 8; /* x */
+	ib->ptr[ib->length_dw++] = 1; /* y */
+	ib->ptr[ib->length_dw++] = 1; /* z */
+	ib->ptr[ib->length_dw++] =
 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
 
 	/* write CS partial flush packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
-	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
 
 	/* SGPR2 */
 	/* write the register state for the compute dispatch */
 	for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
-		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
-		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
-		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
+		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib->ptr[ib->length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
+		ib->ptr[ib->length_dw++] = sgpr2_init_regs[i + 1];
 	}
 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
-	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
-	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
-	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
-	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+	gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8;
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib->ptr[ib->length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
+	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
 
 	/* write dispatch packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
-	ib.ptr[ib.length_dw++] = 8; /* x */
-	ib.ptr[ib.length_dw++] = 1; /* y */
-	ib.ptr[ib.length_dw++] = 1; /* z */
-	ib.ptr[ib.length_dw++] =
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib->ptr[ib->length_dw++] = 8; /* x */
+	ib->ptr[ib->length_dw++] = 1; /* y */
+	ib->ptr[ib->length_dw++] = 1; /* z */
+	ib->ptr[ib->length_dw++] =
 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
 
 	/* write CS partial flush packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
-	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
 
-	/* shedule the ib on the ring */
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+	/* schedule the ib on the ring */
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r) {
 		drm_err(adev_to_drm(adev), "ib submit failed (%d).\n", r);
+		amdgpu_job_free(job);
 		goto fail;
 	}
 
@@ -1637,7 +1640,6 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 		RREG32(sec_ded_counter_registers[i]);
 
 fail:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 
 	return r;
@@ -3758,7 +3760,7 @@ static void gfx_v8_0_constants_init(struct amdgpu_device *adev)
 	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
 
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, 0) {
 		vi_srbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
 		if (i == 0) {
@@ -6764,7 +6766,6 @@ static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
 	.hw_fini = gfx_v8_0_hw_fini,
 	.suspend = gfx_v8_0_suspend,
 	.resume = gfx_v8_0_resume,
-	.is_idle = gfx_v8_0_is_idle,
 	.wait_for_idle = gfx_v8_0_wait_for_idle,
 	.soft_reset = gfx_v8_0_soft_reset,
 	.set_clockgating_state = gfx_v8_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index c3322058720f57..3ba2584e48c6ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -1224,9 +1224,9 @@ static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
-
 	unsigned index;
 	uint64_t gpu_addr;
 	uint32_t tmp;
@@ -1238,22 +1238,27 @@ static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
-	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 20,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r)
 		goto err1;
 
-	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
-	ib.ptr[2] = lower_32_bits(gpu_addr);
-	ib.ptr[3] = upper_32_bits(gpu_addr);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
+	ib = &job->ibs[0];
+	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib->ptr[2] = lower_32_bits(gpu_addr);
+	ib->ptr[3] = upper_32_bits(gpu_addr);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err2;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1270,7 +1275,6 @@ static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err2:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
 	amdgpu_wb_free(adev, index);
@@ -2682,7 +2686,7 @@ static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) {
 		soc15_grbm_select(adev, 0, 0, 0, i, 0);
 		/* CP and shaders */
 		if (i == 0) {
@@ -4684,7 +4688,8 @@ static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 {
 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	int r, i;
 	unsigned total_size, vgpr_offset, sgpr_offset;
@@ -4730,112 +4735,115 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 	total_size += sizeof(sgpr_init_compute_shader);
 
 	/* allocate an indirect buffer to put the commands in */
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, total_size,
-					AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, total_size,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_RUN_SHADER,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r);
 		return r;
 	}
+	ib = &job->ibs[0];
 
 	/* load the compute shaders */
 	for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
-		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
+		ib->ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
 
 	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
-		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
+		ib->ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
 
 	/* init the ib length to 0 */
-	ib.length_dw = 0;
+	ib->length_dw = 0;
 
 	/* VGPR */
 	/* write the register state for the compute dispatch */
 	for (i = 0; i < gpr_reg_size; i++) {
-		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
-		ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
+		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
 								- PACKET3_SET_SH_REG_START;
-		ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
+		ib->ptr[ib->length_dw++] = vgpr_init_regs_ptr[i].reg_value;
 	}
 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
-	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
-	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
+	gpu_addr = (ib->gpu_addr + (u64)vgpr_offset) >> 8;
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
 							- PACKET3_SET_SH_REG_START;
-	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
-	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
 
 	/* write dispatch packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
-	ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
-	ib.ptr[ib.length_dw++] = 1; /* y */
-	ib.ptr[ib.length_dw++] = 1; /* z */
-	ib.ptr[ib.length_dw++] =
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib->ptr[ib->length_dw++] = compute_dim_x * 2; /* x */
+	ib->ptr[ib->length_dw++] = 1; /* y */
+	ib->ptr[ib->length_dw++] = 1; /* z */
+	ib->ptr[ib->length_dw++] =
 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
 
 	/* write CS partial flush packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
-	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
 
 	/* SGPR1 */
 	/* write the register state for the compute dispatch */
 	for (i = 0; i < gpr_reg_size; i++) {
-		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
-		ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
+		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
 								- PACKET3_SET_SH_REG_START;
-		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
+		ib->ptr[ib->length_dw++] = sgpr1_init_regs[i].reg_value;
 	}
 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
-	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
-	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
+	gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8;
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
 							- PACKET3_SET_SH_REG_START;
-	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
-	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
 
 	/* write dispatch packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
-	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
-	ib.ptr[ib.length_dw++] = 1; /* y */
-	ib.ptr[ib.length_dw++] = 1; /* z */
-	ib.ptr[ib.length_dw++] =
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib->ptr[ib->length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
+	ib->ptr[ib->length_dw++] = 1; /* y */
+	ib->ptr[ib->length_dw++] = 1; /* z */
+	ib->ptr[ib->length_dw++] =
 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
 
 	/* write CS partial flush packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
-	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
 
 	/* SGPR2 */
 	/* write the register state for the compute dispatch */
 	for (i = 0; i < gpr_reg_size; i++) {
-		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
-		ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
+		ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
+		ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
 								- PACKET3_SET_SH_REG_START;
-		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
+		ib->ptr[ib->length_dw++] = sgpr2_init_regs[i].reg_value;
 	}
 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
-	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
-	ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
+	gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8;
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
+	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
 							- PACKET3_SET_SH_REG_START;
-	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
-	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
+	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
 
 	/* write dispatch packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
-	ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
-	ib.ptr[ib.length_dw++] = 1; /* y */
-	ib.ptr[ib.length_dw++] = 1; /* z */
-	ib.ptr[ib.length_dw++] =
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
+	ib->ptr[ib->length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
+	ib->ptr[ib->length_dw++] = 1; /* y */
+	ib->ptr[ib->length_dw++] = 1; /* z */
+	ib->ptr[ib->length_dw++] =
 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
 
 	/* write CS partial flush packet */
-	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
-	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
+	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
+	ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
 
-	/* shedule the ib on the ring */
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
+	/* schedule the ib on the ring */
+	r = amdgpu_job_submit_direct(job, ring, &f);
 	if (r) {
 		drm_err(adev_to_drm(adev), "ib schedule failed (%d).\n", r);
+		amdgpu_job_free(job);
 		goto fail;
 	}
 
@@ -4847,7 +4855,6 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 	}
 
 fail:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 
 	return r;
@@ -7528,7 +7535,6 @@ static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
 	.hw_fini = gfx_v9_0_hw_fini,
 	.suspend = gfx_v9_0_suspend,
 	.resume = gfx_v9_0_resume,
-	.is_idle = gfx_v9_0_is_idle,
 	.wait_for_idle = gfx_v9_0_wait_for_idle,
 	.soft_reset = gfx_v9_0_soft_reset,
 	.set_clockgating_state = gfx_v9_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
index 8058ea91ecafd8..344156bf20caf4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c
@@ -343,14 +343,37 @@ const struct soc15_reg_entry sgpr64_init_regs_aldebaran[] = {
 	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
 };
 
+static const u32 fault_cleaner_compute_shader_aldebaran[] = {
+	0xbe8000ff, 0x00001000, 0x80808100, 0x86000000, 0xbf85fffd, 0xbf810000
+};
+
+const struct soc15_reg_entry fault_cleaner_init_regs_aldebaran[] = {
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 1 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0xaf0000 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0xc8 },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0xea4fac },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
+	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
+};
+
 static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev,
 				 struct amdgpu_ring *ring,
-				 struct amdgpu_ib *ib,
 				 const u32 *shader_ptr, u32 shader_size,
 				 const struct soc15_reg_entry *init_regs, u32 regs_size,
 				 u32 compute_dim_x, u64 wb_gpu_addr, u32 pattern,
 				 struct dma_fence **fence_ptr)
 {
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	int r, i;
 	uint32_t total_size, shader_offset;
 	u64 gpu_addr;
@@ -360,14 +383,15 @@ static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev,
 	shader_offset = total_size;
 	total_size += ALIGN(shader_size, 256);
 
-	/* allocate an indirect buffer to put the commands in */
-	memset(ib, 0, sizeof(*ib));
-	r = amdgpu_ib_get(adev, NULL, total_size,
-					AMDGPU_IB_POOL_DIRECT, ib);
+	r = amdgpu_job_alloc_with_ib(adev, NULL, NULL, total_size,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_RUN_SHADER,
+				     &job);
 	if (r) {
 		dev_err(adev->dev, "failed to get ib (%d).\n", r);
 		return r;
 	}
+	ib = &job->ibs[0];
 
 	/* load the compute shaders */
 	for (i = 0; i < shader_size/sizeof(u32); i++)
@@ -408,11 +432,11 @@ static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev,
 	ib->ptr[ib->length_dw++] =
 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
 
-	/* shedule the ib on the ring */
-	r = amdgpu_ib_schedule(ring, 1, ib, NULL, fence_ptr);
+	/* schedule the ib on the ring */
+	r = amdgpu_job_submit_direct(job, ring, fence_ptr);
 	if (r) {
 		dev_err(adev->dev, "ib submit failed (%d).\n", r);
-		amdgpu_ib_free(ib, NULL);
+		amdgpu_job_free(job);
 	}
 	return r;
 }
@@ -493,7 +517,6 @@ static int gfx_v9_4_2_do_sgprs_init(struct amdgpu_device *adev)
 	int wb_size = adev->gfx.config.max_shader_engines *
 			 CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
 	struct amdgpu_ib wb_ib;
-	struct amdgpu_ib disp_ibs[3];
 	struct dma_fence *fences[3];
 	u32 pattern[3] = { 0x1, 0x5, 0xa };
 
@@ -514,7 +537,6 @@ static int gfx_v9_4_2_do_sgprs_init(struct amdgpu_device *adev)
 
 	r = gfx_v9_4_2_run_shader(adev,
 			&adev->gfx.compute_ring[0],
-			&disp_ibs[0],
 			sgpr112_init_compute_shader_aldebaran,
 			sizeof(sgpr112_init_compute_shader_aldebaran),
 			sgpr112_init_regs_aldebaran,
@@ -539,7 +561,6 @@ static int gfx_v9_4_2_do_sgprs_init(struct amdgpu_device *adev)
 
 	r = gfx_v9_4_2_run_shader(adev,
 			&adev->gfx.compute_ring[1],
-			&disp_ibs[1],
 			sgpr96_init_compute_shader_aldebaran,
 			sizeof(sgpr96_init_compute_shader_aldebaran),
 			sgpr96_init_regs_aldebaran,
@@ -579,7 +600,6 @@ static int gfx_v9_4_2_do_sgprs_init(struct amdgpu_device *adev)
 	memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
 	r = gfx_v9_4_2_run_shader(adev,
 			&adev->gfx.compute_ring[0],
-			&disp_ibs[2],
 			sgpr64_init_compute_shader_aldebaran,
 			sizeof(sgpr64_init_compute_shader_aldebaran),
 			sgpr64_init_regs_aldebaran,
@@ -611,13 +631,10 @@ static int gfx_v9_4_2_do_sgprs_init(struct amdgpu_device *adev)
 	}
 
 disp2_failed:
-	amdgpu_ib_free(&disp_ibs[2], NULL);
 	dma_fence_put(fences[2]);
 disp1_failed:
-	amdgpu_ib_free(&disp_ibs[1], NULL);
 	dma_fence_put(fences[1]);
 disp0_failed:
-	amdgpu_ib_free(&disp_ibs[0], NULL);
 	dma_fence_put(fences[0]);
 pro_end:
 	amdgpu_ib_free(&wb_ib, NULL);
@@ -637,7 +654,6 @@ static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev)
 	int wb_size = adev->gfx.config.max_shader_engines *
 			 CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
 	struct amdgpu_ib wb_ib;
-	struct amdgpu_ib disp_ib;
 	struct dma_fence *fence;
 	u32 pattern = 0xa;
 
@@ -657,7 +673,6 @@ static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev)
 
 	r = gfx_v9_4_2_run_shader(adev,
 			&adev->gfx.compute_ring[0],
-			&disp_ib,
 			vgpr_init_compute_shader_aldebaran,
 			sizeof(vgpr_init_compute_shader_aldebaran),
 			vgpr_init_regs_aldebaran,
@@ -687,7 +702,6 @@ static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev)
 	}
 
 disp_failed:
-	amdgpu_ib_free(&disp_ib, NULL);
 	dma_fence_put(fence);
 pro_end:
 	amdgpu_ib_free(&wb_ib, NULL);
@@ -700,6 +714,67 @@ static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev)
 	return r;
 }
 
+static int gfx_v9_4_2_clean_shaders(struct amdgpu_device *adev)
+{
+	int r;
+	/* CU_ID: 0~15, SIMD_ID: 0~3, WAVE_ID: 0 ~ 9 */
+	int wb_size = adev->gfx.config.max_shader_engines *
+			 CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
+	struct amdgpu_ib wb_ib;
+	struct dma_fence *fence = NULL;
+	u32 pattern = 0xa;
+
+	/* bail if the compute ring is not ready */
+	if (!adev->gfx.compute_ring[0].sched.ready)
+		return 0;
+
+	/* allocate the write-back buffer from IB */
+	memset(&wb_ib, 0, sizeof(wb_ib));
+	r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t),
+			  AMDGPU_IB_POOL_DIRECT, &wb_ib);
+	if (r) {
+		dev_err(adev->dev, "failed to get ib (%d) for wb.\n", r);
+		return r;
+	}
+	memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
+
+	r = gfx_v9_4_2_run_shader(adev,
+			&adev->gfx.compute_ring[0],
+			fault_cleaner_compute_shader_aldebaran,
+			sizeof(fault_cleaner_compute_shader_aldebaran),
+			fault_cleaner_init_regs_aldebaran,
+			ARRAY_SIZE(fault_cleaner_init_regs_aldebaran),
+			adev->gfx.cu_info.number * 32,
+			wb_ib.gpu_addr, pattern, &fence);
+
+	if (r) {
+		dev_err(adev->dev, "failed to clear MI200\n");
+		goto pro_end;
+	}
+
+	/* wait for the GPU to finish processing the IB */
+	r = dma_fence_wait(fence, false);
+	if (r)
+		dev_err(adev->dev, "timeout to clear MI200\n");
+
+pro_end:
+	if (fence)
+		dma_fence_put(fence);
+	amdgpu_ib_free(&wb_ib, NULL);
+
+	if (r)
+		dev_dbg(adev->dev, "Clean MI200 Failed\n");
+	else
+		dev_dbg(adev->dev, "Clean MI200 Successfully\n");
+
+	return r;
+}
+
+void gfx_v9_4_2_clean_fault(struct amdgpu_device *adev)
+{
+	gfx_v9_4_2_clean_shaders(adev);
+}
+
 int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev)
 {
 	/* only support when RAS is enabled */
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
index a603724c1dfc51..1cbade5ed65753 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h
@@ -30,6 +30,7 @@ void gfx_v9_4_2_init_golden_registers(struct amdgpu_device *adev,
 				      uint32_t die_id);
 void gfx_v9_4_2_init_sq(struct amdgpu_device *adev);
 void gfx_v9_4_2_set_power_brake_sequence(struct amdgpu_device *adev);
+void gfx_v9_4_2_clean_fault(struct amdgpu_device *adev);
 int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev);
 
 extern struct amdgpu_gfx_ras gfx_v9_4_2_ras;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index 522981fa182db6..484ecde55ac9e2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -23,6 +23,7 @@
 #include <linux/firmware.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_gfx.h"
 #include "soc15.h"
 #include "soc15d.h"
@@ -450,9 +451,9 @@ static int gfx_v9_4_3_ring_test_ring(struct amdgpu_ring *ring)
 static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
-
 	unsigned index;
 	uint64_t gpu_addr;
 	uint32_t tmp;
@@ -464,22 +465,27 @@ static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
-	memset(&ib, 0, sizeof(ib));
 
-	r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 20,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST,
+				     &job);
 	if (r)
 		goto err1;
 
-	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
-	ib.ptr[2] = lower_32_bits(gpu_addr);
-	ib.ptr[3] = upper_32_bits(gpu_addr);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.length_dw = 5;
+	ib = &job->ibs[0];
+	ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
+	ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib->ptr[2] = lower_32_bits(gpu_addr);
+	ib->ptr[3] = upper_32_bits(gpu_addr);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->length_dw = 5;
 
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err2;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -496,7 +502,6 @@ static int gfx_v9_4_3_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err2:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err1:
 	amdgpu_wb_free(adev, index);
@@ -1237,7 +1242,7 @@ static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev,
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) {
+	for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) {
 		soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id));
 		/* CP and shaders */
 		if (i == 0) {
@@ -1642,10 +1647,9 @@ static bool gfx_v9_4_3_check_rlcg_range(struct amdgpu_device *adev,
 		const struct soc15_reg_rlcg *entry;
 
 		entry = &entries[i];
-		inst = adev->ip_map.logical_to_dev_inst ?
-			       adev->ip_map.logical_to_dev_inst(
-				       adev, entry->hwip, entry->instance) :
-			       entry->instance;
+		inst = amdgpu_ip_map_logical_to_dev_inst(&adev->ip_map,
+							 entry->hwip,
+							 entry->instance);
 		reg = adev->reg_offset[entry->hwip][inst][entry->segment] +
 		      entry->reg;
 		if (offset == reg)
@@ -3944,7 +3948,6 @@ static const struct amd_ip_funcs gfx_v9_4_3_ip_funcs = {
 	.hw_fini = gfx_v9_4_3_hw_fini,
 	.suspend = gfx_v9_4_3_suspend,
 	.resume = gfx_v9_4_3_resume,
-	.is_idle = gfx_v9_4_3_is_idle,
 	.wait_for_idle = gfx_v9_4_3_wait_for_idle,
 	.soft_reset = gfx_v9_4_3_soft_reset,
 	.set_clockgating_state = gfx_v9_4_3_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
index 652eea6eae4aca..8be912d506019c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v11_5_0.c
@@ -155,6 +155,7 @@ static void gfxhub_v11_5_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 static void gfxhub_v11_5_0_init_system_aperture_regs(struct amdgpu_device *adev)
 {
 	uint64_t value;
+	u32 tmp;
 
 	WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BASE, 0);
 	WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
@@ -180,8 +181,12 @@ static void gfxhub_v11_5_0_init_system_aperture_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 		     (u32)((u64)adev->dummy_page_addr >> 44));
 
-	WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
-		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ENABLE_RETRY_FAULT_INTERRUPT, 1);
+	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 }
 
 static void gfxhub_v11_5_0_init_tlb_regs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
index 6cbf837d50ddc3..4144c1c2ee13c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_0.c
@@ -158,6 +158,7 @@ static void gfxhub_v12_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 static void gfxhub_v12_0_init_system_aperture_regs(struct amdgpu_device *adev)
 {
 	uint64_t value;
+	u32 tmp;
 
 	/* Program the AGP BAR */
 	WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BASE, 0);
@@ -184,8 +185,12 @@ static void gfxhub_v12_0_init_system_aperture_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 		     (u32)((u64)adev->dummy_page_addr >> 44));
 
-	WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
-		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ENABLE_RETRY_FAULT_INTERRUPT, 1);
+	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
index 4c2fd1e6616e4f..7fa451b373fcca 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v12_1.c
@@ -272,9 +272,6 @@ static void gfxhub_v12_1_xcc_init_tlb_regs(struct amdgpu_device *adev,
 		tmp = REG_SET_FIELD(tmp,
 				    GCMC_VM_MX_L1_TLB_CNTL,
 				    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
-		tmp = REG_SET_FIELD(tmp,
-				    GCMC_VM_MX_L1_TLB_CNTL,
-				    ECO_BITS, 0);
 		tmp = REG_SET_FIELD(tmp,
 				    GCMC_VM_MX_L1_TLB_CNTL,
 				    MTYPE, MTYPE_UC);
@@ -295,9 +292,6 @@ static void gfxhub_v12_1_xcc_init_cache_regs(struct amdgpu_device *adev,
 		tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL);
 		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
 				    ENABLE_L2_CACHE, 1);
-		/*TODO: set ENABLE_L2_FRAGMENT_PROCESSING to 1? */
-		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
-				    ENABLE_L2_FRAGMENT_PROCESSING, 0);
 		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
 				    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
 		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
@@ -311,14 +305,6 @@ static void gfxhub_v12_1_xcc_init_cache_regs(struct amdgpu_device *adev,
 		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
 				 regGCVM_L2_CNTL, tmp);
 
-		tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL2);
-		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2,
-				    INVALIDATE_ALL_L1_TLBS, 1);
-		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2,
-				    INVALIDATE_L2_CACHE, 1);
-		WREG32_SOC15_RLC(GC, GET_INST(GC, i),
-				 regGCVM_L2_CNTL2, tmp);
-
 		tmp = regGCVM_L2_CNTL3_DEFAULT;
 		if (adev->gmc.translate_further) {
 			tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12);
@@ -416,7 +402,10 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
 
 	num_level = adev->vm_manager.num_level;
 	block_size = adev->vm_manager.block_size;
-	block_size -= 9;
+	if (adev->gmc.translate_further)
+		num_level -= 1;
+	else
+		block_size -= 9;
 
 	for_each_inst(j, xcc_mask) {
 		hub = &adev->vmhub[AMDGPU_GFXHUB(j)];
@@ -447,7 +436,7 @@ static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev,
 			/* Send no-retry XNACK on fault to suppress VM fault storm */
 			tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
 					    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
-					    1);
+					    !adev->gmc.noretry);
 			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regGCVM_CONTEXT1_CNTL,
 					    i * hub->ctx_distance, tmp);
 			WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
@@ -716,8 +705,18 @@ static const char *gfxhub_v12_1_client_ids[] = {
 	"PA"
 };
 
-/*TODO: l2 protection fault status is increased to 64bits.
- * some critical fields like FED are moved to STATUS_HI32 */
+static void
+gfxhub_v12_1_print_l2_protection_fault_status_hi(struct amdgpu_device *adev,
+						 uint32_t status)
+{
+	dev_err(adev->dev,
+		"GCVM_L2_PROTECTION_FAULT_STATUS_HI32:0x%08X\n",
+		status);
+	dev_err(adev->dev, "\t FED: 0x%lx\n",
+		REG_GET_FIELD(status,
+			      GCVM_L2_PROTECTION_FAULT_STATUS_HI32, FED));
+}
+
 static void gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device *adev,
 							  uint32_t status)
 {
@@ -750,6 +749,8 @@ static void gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device *
 
 static const struct amdgpu_vmhub_funcs gfxhub_v12_1_vmhub_funcs = {
 	.print_l2_protection_fault_status = gfxhub_v12_1_print_l2_protection_fault_status,
+	.print_l2_protection_fault_status_hi =
+		gfxhub_v12_1_print_l2_protection_fault_status_hi,
 	.get_invalidate_req = gfxhub_v12_1_get_invalidate_req,
 };
 
@@ -779,10 +780,12 @@ static void gfxhub_v12_1_xcc_init(struct amdgpu_device *adev, uint32_t xcc_mask)
 		hub->vm_context0_cntl =
 			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
 				regGCVM_CONTEXT0_CNTL);
-		/* TODO: add a new member to accomandate additional fault status/cntl reg */
 		hub->vm_l2_pro_fault_status =
 			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
 				regGCVM_L2_PROTECTION_FAULT_STATUS_LO32);
+		hub->vm_l2_pro_fault_status_hi =
+			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
+					 regGCVM_L2_PROTECTION_FAULT_STATUS_HI32);
 		hub->vm_l2_pro_fault_cntl =
 			SOC15_REG_OFFSET(GC, GET_INST(GC, i),
 				regGCVM_L2_PROTECTION_FAULT_CNTL_LO32);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index bfe247b1a333c8..c35856180c3c44 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -91,6 +91,7 @@ static void gfxhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
 {
 	uint64_t value;
+	u32 tmp;
 
 	if (!amdgpu_sriov_vf(adev) || adev->asic_type <= CHIP_VEGA10) {
 		/* Program the AGP BAR */
@@ -134,8 +135,12 @@ static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
 		WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 			     (u32)((u64)adev->dummy_page_addr >> 44));
 
-		WREG32_FIELD15(GC, 0, VM_L2_PROTECTION_FAULT_CNTL2,
-			       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+		tmp = RREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL2);
+		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
+					 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+		tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
+					 ENABLE_RETRY_FAULT_INTERRUPT, 1);
+		WREG32_SOC15(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 	}
 
 	/* In the case squeezing vram into GART aperture, we don't use
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
index fbdf46070b38b7..e8981359053980 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_2.c
@@ -21,6 +21,7 @@
  *
  */
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_xcp.h"
 #include "gfxhub_v1_2.h"
 #include "gfxhub_v1_1.h"
@@ -176,6 +177,8 @@ gfxhub_v1_2_xcc_init_system_aperture_regs(struct amdgpu_device *adev,
 			tmp = RREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_CNTL2);
 			tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
 					    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+			tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
+					    ENABLE_RETRY_FAULT_INTERRUPT, 1);
 			WREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 		}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
index 9ea593e2c7199d..04778f9f95bd79 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_0.c
@@ -151,6 +151,7 @@ static void gfxhub_v2_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 static void gfxhub_v2_0_init_system_aperture_regs(struct amdgpu_device *adev)
 {
 	uint64_t value;
+	u32 tmp;
 
 	if (!amdgpu_sriov_vf(adev)) {
 		/* Program the AGP BAR */
@@ -178,8 +179,12 @@ static void gfxhub_v2_0_init_system_aperture_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 		     (u32)((u64)adev->dummy_page_addr >> 44));
 
-	WREG32_FIELD15(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
-		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ENABLE_RETRY_FAULT_INTERRUPT, 1);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
index 30b90d35abd018..6c77c17878f841 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
@@ -154,6 +154,7 @@ static void gfxhub_v2_1_init_gart_aperture_regs(struct amdgpu_device *adev)
 static void gfxhub_v2_1_init_system_aperture_regs(struct amdgpu_device *adev)
 {
 	uint64_t value;
+	u32 tmp;
 
 	if (amdgpu_sriov_vf(adev))
 		return;
@@ -182,8 +183,12 @@ static void gfxhub_v2_1_init_system_aperture_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 		     (u32)((u64)adev->dummy_page_addr >> 44));
 
-	WREG32_FIELD15(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
-		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ENABLE_RETRY_FAULT_INTERRUPT, 1);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
index 9e6a6e13dec0e2..e2b9fd0c8af05b 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0.c
@@ -150,6 +150,7 @@ static void gfxhub_v3_0_init_gart_aperture_regs(struct amdgpu_device *adev)
 static void gfxhub_v3_0_init_system_aperture_regs(struct amdgpu_device *adev)
 {
 	uint64_t value;
+	u32 tmp;
 
 	/* Program the AGP BAR */
 	WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BASE, 0);
@@ -176,8 +177,12 @@ static void gfxhub_v3_0_init_system_aperture_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 		     (u32)((u64)adev->dummy_page_addr >> 44));
 
-	WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
-		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ENABLE_RETRY_FAULT_INTERRUPT, 1);
+	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
index b3b1085c7cd3df..fa8b966b7ea12a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
@@ -153,6 +153,7 @@ static void gfxhub_v3_0_3_init_gart_aperture_regs(struct amdgpu_device *adev)
 static void gfxhub_v3_0_3_init_system_aperture_regs(struct amdgpu_device *adev)
 {
 	uint64_t value;
+	u32 tmp;
 
 	if (amdgpu_sriov_vf(adev))
 		return;
@@ -181,8 +182,12 @@ static void gfxhub_v3_0_3_init_system_aperture_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 		     (u32)((u64)adev->dummy_page_addr >> 44));
 
-	WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
-		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2,
+				 ENABLE_RETRY_FAULT_INTERRUPT, 1);
+	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index c70657bc089b53..490b4c4eb662af 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -26,6 +26,7 @@
 #include <drm/drm_cache.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_atomfirmware.h"
 #include "gmc_v10_0.h"
 #include "umc_v8_7.h"
@@ -869,6 +870,10 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdkfd will use VMIDs 8-15
 	 */
 	adev->vm_manager.first_kfd_vmid = 8;
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      false);
+	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true);
 
 	amdgpu_vm_manager_init(adev);
 
@@ -1043,12 +1048,6 @@ static int gmc_v10_0_resume(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool gmc_v10_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* MC is always ready in GMC v10.*/
-	return true;
-}
-
 static int gmc_v10_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* There is no need to wait for MC idle in GMC v10.*/
@@ -1114,7 +1113,6 @@ const struct amd_ip_funcs gmc_v10_0_ip_funcs = {
 	.hw_fini = gmc_v10_0_hw_fini,
 	.suspend = gmc_v10_0_suspend,
 	.resume = gmc_v10_0_resume,
-	.is_idle = gmc_v10_0_is_idle,
 	.wait_for_idle = gmc_v10_0_wait_for_idle,
 	.set_clockgating_state = gmc_v10_0_set_clockgating_state,
 	.set_powergating_state = gmc_v10_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index 7e612f947349f0..3cdc7893587a11 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -26,6 +26,7 @@
 #include <drm/drm_cache.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_atomfirmware.h"
 #include "gmc_v11_0.h"
 #include "umc_v8_10.h"
@@ -108,13 +109,16 @@ static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev,
 	bool write_fault = !!(entry->src_data[1] &
 			      AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE);
 	uint32_t status = 0;
+	uint32_t cam_index;
 	u64 addr;
 
 	addr = (u64)entry->src_data[0] << 12;
 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 
 	if (retry_fault) {
-		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, 0, 0,
+		cam_index = entry->src_data[2] & 0x3ff;
+
+		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, cam_index, 0,
 							write_fault);
 		/* Returning 1 here also prevents sending the IV to the KFD */
 		if (ret == 1)
@@ -852,6 +856,11 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 */
 	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
 
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      false);
+	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true);
+
 	amdgpu_vm_manager_init(adev);
 
 	r = amdgpu_gmc_ras_sw_init(adev);
@@ -1004,12 +1013,6 @@ static int gmc_v11_0_resume(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool gmc_v11_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* MC is always ready in GMC v11.*/
-	return true;
-}
-
 static int gmc_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* There is no need to wait for MC idle in GMC v11.*/
@@ -1054,7 +1057,6 @@ const struct amd_ip_funcs gmc_v11_0_ip_funcs = {
 	.hw_fini = gmc_v11_0_hw_fini,
 	.suspend = gmc_v11_0_suspend,
 	.resume = gmc_v11_0_resume,
-	.is_idle = gmc_v11_0_is_idle,
 	.wait_for_idle = gmc_v11_0_wait_for_idle,
 	.set_clockgating_state = gmc_v11_0_set_clockgating_state,
 	.set_powergating_state = gmc_v11_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index b05465dc30b4bd..d7f56bbcfd4409 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -27,6 +27,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_atomfirmware.h"
+#include "amdgpu_ip.h"
 #include "gmc_v12_0.h"
 #include "gmc_v12_1.h"
 #include "athub/athub_4_1_0_sh_mask.h"
@@ -99,6 +100,7 @@ static int gmc_v12_0_process_interrupt(struct amdgpu_device *adev,
 	bool write_fault = !!(entry->src_data[1] &
 			      AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE);
 	uint32_t status = 0;
+	uint32_t cam_index;
 	u64 addr;
 
 	addr = (u64)entry->src_data[0] << 12;
@@ -110,7 +112,9 @@ static int gmc_v12_0_process_interrupt(struct amdgpu_device *adev,
 		hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
 
 	if (retry_fault) {
-		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, 0, 0,
+		cam_index = entry->src_data[2] & 0x3ff;
+
+		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, cam_index, 0,
 							write_fault);
 		/* Returning 1 here also prevents sending the IV to the KFD */
 		if (ret == 1)
@@ -641,11 +645,12 @@ static int gmc_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 		adev->gmc.xgmi.connected_to_cpu =
 			adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
 
+	adev->gmc.init_pte_flags = AMDGPU_PTE_IS_PTE;
+
 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 	case IP_VERSION(12, 1, 0):
 		gmc_v12_1_set_gmc_funcs(adev);
 		gmc_v12_1_set_irq_funcs(adev);
-		adev->gmc.init_pte_flags = AMDGPU_PTE_IS_PTE;
 		break;
 	default:
 		gmc_v12_0_set_gmc_funcs(adev);
@@ -668,7 +673,7 @@ static int gmc_v12_0_early_init(struct amdgpu_ip_block *ip_block)
 		adev->gmc.private_aperture_end =
 			adev->gmc.private_aperture_start + (4ULL << 30) - 1;
 
-	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
+	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_GFX12;
 
 	return 0;
 }
@@ -810,6 +815,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
 	struct amdgpu_device *adev = ip_block->adev;
 	uint64_t pte_addr_mask = 0;
+	u32 mmhub_vmid_mask;
 	int i;
 
 	adev->mmhub.funcs->init(adev);
@@ -845,7 +851,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	case IP_VERSION(12, 1, 0):
 		bitmap_set(adev->vmhubs_mask, AMDGPU_GFXHUB(0),
 				NUM_XCC(adev->gfx.xcc_mask));
-		for (i = 0; i < hweight32(adev->aid_mask); i++)
+		for_each_inst(i, adev->mmhub.inst_mask)
 			set_bit(AMDGPU_MMHUB0(i), adev->vmhubs_mask);
 		/*
 		 * To fulfill 5-level page support,
@@ -855,6 +861,7 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 		amdgpu_vm_adjust_size(adev, 128 * 1024 * 1024, 9, 4, 57);
 		pte_addr_mask = 0x000FFFFFFFFFF000ULL; /* 52 bit PA */
 		dma_addr_bits = 52;
+		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
 		break;
 	default:
 		dev_warn(adev->dev, "Unrecognized GC IP version: 0x%08x\n",
@@ -944,17 +951,33 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0))
+		gmc_v12_1_init_nps_details(adev);
+
 	/*
 	 * number of VMs
 	 * VMID 0 is reserved for System
 	 * amdgpu graphics/compute will use VMIDs 1-7
-	 * amdkfd will use VMIDs 8-15
+	 * amdkfd will use VMIDs 8-15.
+	 * On GFX 12.1, amdkfd will use VMIDs 3-15, with VMID 15 reserved
+	 * for NPA and therefore excluded from the MMHUB VMID pool.
 	 */
-	adev->vm_manager.first_kfd_vmid =
-		 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ?
-		3 : 8;
-	adev->vm_manager.first_kfd_vmid =
-		adev->gfx.disable_kq ? 1 : (adev->vm_manager.first_kfd_vmid);
+	mmhub_vmid_mask = GENMASK(AMDGPU_NUM_VMID - 1, 1);
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
+		adev->vm_manager.first_kfd_vmid = 3;
+		adev->vm_manager.npa_vmid = 15;
+		mmhub_vmid_mask &= ~BIT(adev->vm_manager.npa_vmid);
+	} else {
+		adev->vm_manager.first_kfd_vmid = 8;
+	}
+
+	if (adev->gfx.disable_kq)
+		adev->vm_manager.first_kfd_vmid = 1;
+
+	amdgpu_vmid_mgr_set_vmid_mask(adev, mmhub_vmid_mask, true);
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      false);
 
 	amdgpu_vm_manager_init(adev);
 
@@ -962,6 +985,9 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0))
+		amdgpu_gmc_sysfs_init(adev);
+
 	return 0;
 }
 
@@ -981,12 +1007,18 @@ static int gmc_v12_0_sw_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0))
+		amdgpu_gmc_sysfs_fini(adev);
+
 	amdgpu_vm_manager_fini(adev);
 	gmc_v12_0_gart_fini(adev);
 	amdgpu_gem_force_release(adev);
 	amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
 	amdgpu_bo_fini(adev);
 
+	adev->gmc.num_mem_partitions = 0;
+	kfree(adev->gmc.mem_partitions);
+
 	return 0;
 }
 
@@ -1105,12 +1137,6 @@ static int gmc_v12_0_resume(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool gmc_v12_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* MC is always ready in GMC v11.*/
-	return true;
-}
-
 static int gmc_v12_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* There is no need to wait for MC idle in GMC v11.*/
@@ -1159,7 +1185,6 @@ const struct amd_ip_funcs gmc_v12_0_ip_funcs = {
 	.hw_fini = gmc_v12_0_hw_fini,
 	.suspend = gmc_v12_0_suspend,
 	.resume = gmc_v12_0_resume,
-	.is_idle = gmc_v12_0_is_idle,
 	.wait_for_idle = gmc_v12_0_wait_for_idle,
 	.set_clockgating_state = gmc_v12_0_set_clockgating_state,
 	.set_powergating_state = gmc_v12_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
index 5fe43f7eab29d3..3349e8bc1c2061 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -28,6 +28,10 @@
 #include "oss/osssys_7_1_0_sh_mask.h"
 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
 
+static void gmc_v12_1_get_mtypes(struct amdgpu_device *adev,
+				 unsigned int *mtype_local,
+				 unsigned int *mtype_remote);
+
 static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev,
 					      struct amdgpu_irq_src *src,
 					      unsigned int type,
@@ -100,6 +104,23 @@ static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
+/*
+ * NodeID definition in interrupt cookie for gmc v12.1.0
+ *
+ * DIE		IH Cookie NodeID
+ * MID0		0x0
+ * AID0		0x1
+ * AID0.XCD0	0x2
+ * AID0.XCD1	0x3
+ * AID0.XCD2	0x4
+ * AID0.XCD3	0x5
+ * MID1		0x8
+ * AID1		0x9
+ * AID1.XCD0	0xA
+ * AID1.XCD1	0xB
+ * AID1.XCD2	0xC
+ * AID1.XCD3	0xD
+ */
 static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
 				       struct amdgpu_irq_src *source,
 				       struct amdgpu_iv_entry *entry)
@@ -111,7 +132,7 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
 	uint32_t cam_index = 0;
 	const char *hub_name;
 	int ret, xcc_id = 0;
-	uint32_t status = 0;
+	uint32_t status = 0, status_hi = 0;
 	const char *die_name;
 	char die_name_buf[32];
 	u64 addr;
@@ -128,7 +149,7 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
 
 	if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) {
 		hub_name = "mmhub0";
-		vmhub = AMDGPU_MMHUB0(node_id / 4);
+		vmhub = AMDGPU_MMHUB0(node_id / 8);
 	} else {
 		hub_name = "gfxhub0";
 		if (adev->gfx.funcs->ih_node_to_logical_xcc) {
@@ -226,18 +247,29 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
 		RREG32(hub->vm_l2_pro_fault_status);
 
 	status = RREG32(hub->vm_l2_pro_fault_status);
+	if (hub->vmhub_funcs &&
+	    hub->vmhub_funcs->print_l2_protection_fault_status_hi)
+		status_hi = RREG32(hub->vm_l2_pro_fault_status_hi);
 
 	/* Only print L2 fault status if the status register could be read and
 	 * contains useful information
 	 */
-	if (!status)
+	if (!status && !status_hi)
 		return 0;
 
 	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
 
 	amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);
 
+	if (!hub->vmhub_funcs || !hub->vmhub_funcs->print_l2_protection_fault_status) {
+		dev_warn_once(adev->dev, "vmhub %d print fault status func not defined\n", vmhub);
+		return 0;
+	}
+
 	hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
+	if (hub->vmhub_funcs->print_l2_protection_fault_status_hi)
+		hub->vmhub_funcs->print_l2_protection_fault_status_hi(adev,
+								     status_hi);
 
 	return 0;
 }
@@ -375,9 +407,8 @@ static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	/* This is necessary for SRIOV as well as for GFXOFF to function
 	 * properly under bare metal
 	 */
-	if (((adev->gfx.kiq[inst].ring.sched.ready ||
-	      adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready) &&
-	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) {
+	if (adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready &&
+	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
 		struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
 		const unsigned eng = 17;
 		u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
@@ -400,7 +431,7 @@ static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
  * @pasid: pasid to be flush
  * @flush_type: the flush type
  * @all_hub: flush all hubs
- * @inst: is used to select which instance of KIQ to use for the invalidation
+ * @inst: XCC instance to use for the invalidation
  *
  * Flush the TLB for the requested pasid.
  */
@@ -411,7 +442,7 @@ static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 	uint16_t queried;
 	int vmid, i;
 
-	if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready &&
+	if (adev->mes.ring[0].sched.ready &&
 	    (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) {
 		struct mes_inv_tlbs_pasid_input input = {0};
 		input.xcc_id = inst;
@@ -553,7 +584,12 @@ static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level,
 	if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM))
 		*addr = adev->vm_manager.vram_base_offset + *addr -
 			adev->gmc.vram_start;
-	BUG_ON(*addr & 0xFFFF00000000003FULL);
+
+	if (*addr & ~adev->gmc.pte_addr_mask) {
+		dev_err(adev->dev, "Invalid PDE address: %llx\n", *addr);
+		*addr = 0;
+		*flags &= ~AMDGPU_PTE_VALID;
+	}
 
 	*flags |= AMDGPU_PTE_SNOOPED;
 
@@ -564,13 +600,70 @@ static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level,
 		/* Set the block fragment size */
 		if (!(*flags & AMDGPU_PDE_PTE_GFX12))
 			*flags |= AMDGPU_PDE_BFS_GFX12(0x9);
-
-	} else if (level == AMDGPU_VM_PDB0) {
-		if (*flags & AMDGPU_PDE_PTE_GFX12)
-			*flags &= ~AMDGPU_PDE_PTE_GFX12;
 	}
 }
 
+static void gmc_v12_1_get_npa_flags(struct amdgpu_device *adev,
+				    uint64_t *flags)
+{
+	unsigned int mtype_local, mtype_remote;
+
+	gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);
+
+	*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype_remote);
+	/* VSCT = 0011 to identify NPA. Additionally PTE.B = 1 */
+	*flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT_GFX12 |
+		   AMDGPU_PTE_BUS_ATOMICS;
+	*flags &= ~AMDGPU_PTE_VALID;
+	*flags &= ~AMDGPU_PTE_EXECUTABLE;
+}
+
+/*
+ * Resolve the MTYPEs used for local and remote memory accesses on GFX 12.1.
+ * Both default to an ASIC-dependent value that can be overridden through the
+ * amdgpu_mtype_local and amdgpu_mtype_remote module parameters.
+ */
+static void gmc_v12_1_get_mtypes(struct amdgpu_device *adev,
+				 unsigned int *mtype_local,
+				 unsigned int *mtype_remote)
+{
+	bool is_aid_a1 = (adev->rev_id & 0x10);
+
+	/* Local memory: ASIC default depends on the AID stepping. */
+	*mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC;
+	if (amdgpu_mtype_local == 0)
+		*mtype_local = MTYPE_RW;
+	else if (amdgpu_mtype_local == 1)
+		*mtype_local = MTYPE_NC;
+	else if (amdgpu_mtype_local == 2)
+		DRM_INFO_ONCE("MTYPE_CC not supported for local memory\n");
+
+	/* Remote memory defaults to MTYPE_UC on GFX 12.1. */
+	*mtype_remote = MTYPE_UC;
+	if (amdgpu_mtype_remote == 0)
+		*mtype_remote = MTYPE_NC;
+	else if (amdgpu_mtype_remote == 1)
+		*mtype_remote = MTYPE_UC;
+
+	DRM_INFO_ONCE("Using %s for local memory and %s for remote memory\n",
+		      *mtype_local == MTYPE_RW ? "MTYPE_RW" : "MTYPE_NC",
+		      *mtype_remote == MTYPE_NC ? "MTYPE_NC" : "MTYPE_UC");
+}
+
+/*
+ * The compute MQD coherent_aql_mtype field (offset 509) must be programmed to
+ * 0 whenever the driver maps local or remote memory as MTYPE_NC, and to 1 in
+ * all other cases.
+ */
+u32 gmc_v12_1_get_coherent_aql_mtype(struct amdgpu_device *adev)
+{
+	unsigned int mtype_local, mtype_remote;
+
+	gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);
+
+	return (mtype_local == MTYPE_NC || mtype_remote == MTYPE_NC) ? 0 : 1;
+}
+
 static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
 					  struct amdgpu_bo *bo,
 					  uint64_t *flags)
@@ -586,25 +679,10 @@ static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
 	unsigned int mtype, mtype_local, mtype_remote;
 	bool snoop = false;
 	bool is_local = false;
-	bool is_aid_a1;
 
 	switch (gc_ip_version) {
 	case IP_VERSION(12, 1, 0):
-		is_aid_a1 = (adev->rev_id & 0x10);
-
-		mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC;
-		mtype_remote = is_aid_a1 ? MTYPE_NC : MTYPE_UC;
-		if (amdgpu_mtype_local == 0) {
-			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
-			mtype_local = MTYPE_RW;
-		} else if (amdgpu_mtype_local == 1) {
-			DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
-			mtype_local = MTYPE_NC;
-		} else if (amdgpu_mtype_local == 2) {
-			DRM_INFO_ONCE("MTYPE_CC not supported, using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
-		} else {
-			DRM_INFO_ONCE("Using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
-		}
+		gmc_v12_1_get_mtypes(adev, &mtype_local, &mtype_remote);
 
 		is_local = (is_vram && adev == bo_adev);
 		snoop = true;
@@ -638,6 +716,8 @@ static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev,
 				 uint32_t vm_flags,
 				 uint64_t *flags)
 {
+	struct ttm_resource *mem;
+
 	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
 		*flags |= AMDGPU_PTE_EXECUTABLE;
 	else
@@ -659,8 +739,17 @@ static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev,
 		break;
 	}
 
-	if ((*flags & AMDGPU_PTE_VALID) && bo)
-		gmc_v12_1_get_coherence_flags(adev, bo, flags);
+	if (bo) {
+		mem = bo->tbo.resource;
+		if (mem && mem->mem_type == AMDGPU_PL_NPA) {
+			dev_dbg(adev->dev,
+				"Setting PTE for NPA BO, mem->type: %d, mem->start: %lx, mem->size: %u, cur_flags: %llx\n",
+				mem->mem_type, mem->start, (u32)mem->size, *flags);
+			gmc_v12_1_get_npa_flags(adev, flags);
+		} else if (*flags & AMDGPU_PTE_VALID) {
+			gmc_v12_1_get_coherence_flags(adev, bo, flags);
+		}
+	}
 }
 
 static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = {
@@ -703,3 +792,36 @@ void gmc_v12_1_init_vram_info(struct amdgpu_device *adev)
 	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4;
 	adev->gmc.vram_width = 384 * 64;
 }
+
+void gmc_v12_1_init_nps_details(struct amdgpu_device *adev)
+{
+	enum amdgpu_memory_partition mode;
+	uint32_t supp_modes;
+	int i;
+
+	adev->gmc.supported_nps_modes = 0;
+
+	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
+		return;
+
+	mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes);
+
+	/* Mode detected by hardware and supported modes available */
+	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && supp_modes) {
+		while ((i = ffs(supp_modes))) {
+			if (AMDGPU_ALL_NPS_MASK & BIT(i))
+				adev->gmc.supported_nps_modes |= BIT(i);
+			supp_modes &= supp_modes - 1;
+		}
+	} else {
+		switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+		case IP_VERSION(12, 1, 0):
+			adev->gmc.supported_nps_modes =
+				BIT(AMDGPU_NPS1_PARTITION_MODE) |
+				BIT(AMDGPU_NPS2_PARTITION_MODE);
+			break;
+		default:
+			break;
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
index 2f08f4b8cd99e2..1c7aad7a1791d4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.h
@@ -27,4 +27,6 @@
 void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev);
 void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev);
 void gmc_v12_1_init_vram_info(struct amdgpu_device *adev);
+void gmc_v12_1_init_nps_details(struct amdgpu_device *adev);
+u32 gmc_v12_1_get_coherent_aql_mtype(struct amdgpu_device *adev);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index b49098931d1971..cdd11587b12ae5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -864,6 +864,12 @@ static int gmc_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdkfd will use VMIDs 8-15
 	 */
 	adev->vm_manager.first_kfd_vmid = 8;
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      false);
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      true);
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
@@ -1132,7 +1138,6 @@ static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
 	.hw_fini = gmc_v6_0_hw_fini,
 	.suspend = gmc_v6_0_suspend,
 	.resume = gmc_v6_0_resume,
-	.is_idle = gmc_v6_0_is_idle,
 	.wait_for_idle = gmc_v6_0_wait_for_idle,
 	.soft_reset = gmc_v6_0_soft_reset,
 	.set_clockgating_state = gmc_v6_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 98db62cc87186d..c9cc55e18893be 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -1045,6 +1045,12 @@ static int gmc_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdkfd will use VMIDs 8-15
 	 */
 	adev->vm_manager.first_kfd_vmid = 8;
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      false);
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      true);
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
@@ -1346,7 +1352,6 @@ static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
 	.hw_fini = gmc_v7_0_hw_fini,
 	.suspend = gmc_v7_0_suspend,
 	.resume = gmc_v7_0_resume,
-	.is_idle = gmc_v7_0_is_idle,
 	.wait_for_idle = gmc_v7_0_wait_for_idle,
 	.soft_reset = gmc_v7_0_soft_reset,
 	.set_clockgating_state = gmc_v7_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 64ebedc595b500..8d8a0775878ff5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -1122,6 +1122,12 @@ static int gmc_v8_0_sw_init(struct amdgpu_ip_block *ip_block)
 	 * amdkfd will use VMIDs 8-15
 	 */
 	adev->vm_manager.first_kfd_vmid = 8;
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      false);
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      true);
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
@@ -1221,18 +1227,6 @@ static int gmc_v8_0_resume(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool gmc_v8_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
-		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
-		return false;
-
-	return true;
-}
-
 static int gmc_v8_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned int i;
@@ -1592,7 +1586,6 @@ static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
 	.hw_fini = gmc_v8_0_hw_fini,
 	.suspend = gmc_v8_0_suspend,
 	.resume = gmc_v8_0_resume,
-	.is_idle = gmc_v8_0_is_idle,
 	.wait_for_idle = gmc_v8_0_wait_for_idle,
 	.set_clockgating_state = gmc_v8_0_set_clockgating_state,
 	.set_powergating_state = gmc_v8_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 1192682735300d..5245d8850a421c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -63,6 +63,7 @@
 
 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
 
+#include "amdgpu_ip.h"
 #include "amdgpu_ras.h"
 #include "amdgpu_xgmi.h"
 
@@ -2038,6 +2039,12 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
 			3 :
 			8;
 
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      false);
+	amdgpu_vmid_mgr_set_vmid_mask(adev,
+				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
+				      true);
 	amdgpu_vm_manager_init(adev);
 
 	gmc_v9_0_save_registers(adev);
@@ -2300,12 +2307,6 @@ static int gmc_v9_0_resume(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static bool gmc_v9_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* MC is always ready in GMC v9.*/
-	return true;
-}
-
 static int gmc_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* There is no need to wait for MC idle in GMC v9.*/
@@ -2355,7 +2356,6 @@ const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
 	.hw_fini = gmc_v9_0_hw_fini,
 	.suspend = gmc_v9_0_suspend,
 	.resume = gmc_v9_0_resume,
-	.is_idle = gmc_v9_0_is_idle,
 	.wait_for_idle = gmc_v9_0_wait_for_idle,
 	.soft_reset = gmc_v9_0_soft_reset,
 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 01cadf898c00b5..49002c80dc712e 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -345,17 +345,6 @@ static int iceland_ih_resume(struct amdgpu_ip_block *ip_block)
 	return iceland_ih_hw_init(ip_block);
 }
 
-static bool iceland_ih_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
-		return false;
-
-	return true;
-}
-
 static int iceland_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -423,7 +412,6 @@ static const struct amd_ip_funcs iceland_ih_ip_funcs = {
 	.hw_fini = iceland_ih_hw_fini,
 	.suspend = iceland_ih_suspend,
 	.resume = iceland_ih_resume,
-	.is_idle = iceland_ih_is_idle,
 	.wait_for_idle = iceland_ih_wait_for_idle,
 	.soft_reset = iceland_ih_soft_reset,
 	.set_clockgating_state = iceland_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
index 333e9c30c091d7..d6554add485cd4 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
@@ -307,6 +307,16 @@ static int ih_v6_0_enable_ring(struct amdgpu_device *adev,
 	return 0;
 }
 
+static uint32_t ih_v6_0_setup_retry_doorbell(u32 doorbell_index)
+{
+	u32 val = 0;
+
+	val = REG_SET_FIELD(val, IH_DOORBELL_RPTR, OFFSET, doorbell_index);
+	val = REG_SET_FIELD(val, IH_DOORBELL_RPTR, ENABLE, 1);
+
+	return val;
+}
+
 /**
  * ih_v6_0_irq_init - init and enable the interrupt ring
  *
@@ -392,6 +402,21 @@ static int ih_v6_0_irq_init(struct amdgpu_device *adev)
 
 	pci_set_master(adev->pdev);
 
+	if (!(adev->flags & AMD_IS_APU)) {
+		/* Allocate the doorbell for IH Retry CAM */
+		adev->irq.retry_cam_doorbell_index = (adev->doorbell_index.ih + 2) << 1;
+		WREG32_SOC15(OSSSYS, 0, regIH_DOORBELL_RETRY_CAM,
+			     ih_v6_0_setup_retry_doorbell(adev->irq.retry_cam_doorbell_index));
+
+		/* Enable IH Retry CAM */
+		tmp = RREG32_SOC15(OSSSYS, 0, regIH_RETRY_INT_CAM_CNTL);
+		tmp = REG_SET_FIELD(tmp, IH_RETRY_INT_CAM_CNTL, ENABLE, 1);
+		tmp = REG_SET_FIELD(tmp, IH_RETRY_INT_CAM_CNTL, CAM_SIZE, 0xF);
+		WREG32_SOC15(OSSSYS, 0, regIH_RETRY_INT_CAM_CNTL, tmp);
+
+		adev->irq.retry_cam_enabled = true;
+	}
+
 	/* enable interrupts */
 	ret = ih_v6_0_toggle_interrupts(adev, true);
 	if (ret)
@@ -439,6 +464,10 @@ static u32 ih_v6_0_get_wptr(struct amdgpu_device *adev,
 	struct amdgpu_ih_regs *ih_regs;
 
 	wptr = le32_to_cpu(*ih->wptr_cpu);
+
+	if (ih == &adev->irq.ih_soft)
+		goto out;
+
 	ih_regs = &ih->ih_regs;
 
 	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
@@ -514,6 +543,9 @@ static void ih_v6_0_set_rptr(struct amdgpu_device *adev,
 {
 	struct amdgpu_ih_regs *ih_regs;
 
+	if (ih == &adev->irq.ih_soft)
+		return;
+
 	if (ih->use_doorbell) {
 		/* XXX check if swapping is necessary on BE */
 		*ih->rptr_cpu = ih->rptr;
@@ -655,12 +687,6 @@ static int ih_v6_0_resume(struct amdgpu_ip_block *ip_block)
 	return ih_v6_0_hw_init(ip_block);
 }
 
-static bool ih_v6_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-	return true;
-}
-
 static int ih_v6_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* todo */
@@ -788,7 +814,6 @@ static const struct amd_ip_funcs ih_v6_0_ip_funcs = {
 	.hw_fini = ih_v6_0_hw_fini,
 	.suspend = ih_v6_0_suspend,
 	.resume = ih_v6_0_resume,
-	.is_idle = ih_v6_0_is_idle,
 	.wait_for_idle = ih_v6_0_wait_for_idle,
 	.soft_reset = ih_v6_0_soft_reset,
 	.set_clockgating_state = ih_v6_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c
index 699c274d357ec3..0fc348010ac76a 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_1.c
@@ -410,6 +410,10 @@ static u32 ih_v6_1_get_wptr(struct amdgpu_device *adev,
 	struct amdgpu_ih_regs *ih_regs;
 
 	wptr = le32_to_cpu(*ih->wptr_cpu);
+
+	if (ih == &adev->irq.ih_soft)
+		goto out;
+
 	ih_regs = &ih->ih_regs;
 
 	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
@@ -481,6 +485,9 @@ static void ih_v6_1_irq_rearm(struct amdgpu_device *adev,
 static void ih_v6_1_set_rptr(struct amdgpu_device *adev,
 			       struct amdgpu_ih_ring *ih)
 {
+	if (ih == &adev->irq.ih_soft)
+		return;
+
 	struct amdgpu_ih_regs *ih_regs;
 
 	if (ih->use_doorbell) {
@@ -583,7 +590,7 @@ static int ih_v6_1_sw_init(struct amdgpu_ip_block *ip_block)
 	/* initialize ih control register offset */
 	ih_v6_1_init_register_offset(adev);
 
-	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true);
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, IH_SW_RING_SIZE, true);
 	if (r)
 		return r;
 
@@ -630,12 +637,6 @@ static int ih_v6_1_resume(struct amdgpu_ip_block *ip_block)
 	return ih_v6_1_hw_init(ip_block);
 }
 
-static bool ih_v6_1_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-	return true;
-}
-
 static int ih_v6_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* todo */
@@ -767,7 +768,6 @@ static const struct amd_ip_funcs ih_v6_1_ip_funcs = {
 	.hw_fini = ih_v6_1_hw_fini,
 	.suspend = ih_v6_1_suspend,
 	.resume = ih_v6_1_resume,
-	.is_idle = ih_v6_1_is_idle,
 	.wait_for_idle = ih_v6_1_wait_for_idle,
 	.soft_reset = ih_v6_1_soft_reset,
 	.set_clockgating_state = ih_v6_1_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
index 6de9e87e04e1ac..3745439b86b368 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
@@ -395,7 +395,8 @@ static int ih_v7_0_irq_init(struct amdgpu_device *adev)
 
 	pci_set_master(adev->pdev);
 
-	if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0)) {
+	if (!(adev->flags & AMD_IS_APU) ||
+	    amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0)) {
 		/* Allocate the doorbell for IH Retry CAM */
 		adev->irq.retry_cam_doorbell_index = (adev->doorbell_index.ih + 2) << 1;
 		WREG32_SOC15(OSSSYS, 0, regIH_DOORBELL_RETRY_CAM,
@@ -420,6 +421,12 @@ static int ih_v7_0_irq_init(struct amdgpu_device *adev)
 	if (adev->irq.ih_soft.ring_size)
 		adev->irq.ih_soft.enabled = true;
 
+	if (adev->irq.ih_psp.ring_size)
+		adev->irq.ih_psp.enabled = true;
+
+	if (adev->irq.ih_ualink.ring_size)
+		adev->irq.ih_ualink.enabled = true;
+
 	return 0;
 }
 
@@ -457,6 +464,10 @@ static u32 ih_v7_0_get_wptr(struct amdgpu_device *adev,
 	struct amdgpu_ih_regs *ih_regs;
 
 	wptr = le32_to_cpu(*ih->wptr_cpu);
+
+	if (ih == &adev->irq.ih_soft)
+		goto out;
+
 	ih_regs = &ih->ih_regs;
 
 	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
@@ -527,6 +538,9 @@ static void ih_v7_0_set_rptr(struct amdgpu_device *adev,
 {
 	struct amdgpu_ih_regs *ih_regs;
 
+	if (ih == &adev->irq.ih_soft)
+		return;
+
 	if (ih->use_doorbell) {
 		/* XXX check if swapping is necessary on BE */
 		*ih->rptr_cpu = ih->rptr;
@@ -589,7 +603,6 @@ static int ih_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 	int r;
 	struct amdgpu_device *adev = ip_block->adev;
 	bool use_bus_addr;
-	unsigned int sw_ring_size;
 
 	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_IH, 0,
 			      &adev->irq.self_irq);
@@ -621,9 +634,16 @@ static int ih_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 	/* initialize ih control register offset */
 	ih_v7_0_init_register_offset(adev);
 
-	sw_ring_size = (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) == IP_VERSION(7, 1, 0)) ?
-			IH_SW_RING_SIZE : PAGE_SIZE;
-	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, sw_ring_size, true);
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, IH_SW_RING_SIZE, true);
+	if (r)
+		return r;
+
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_psp, IH_PSP_RING_SIZE, true);
+	if (r)
+		return r;
+
+	dev_dbg(adev->dev, "ualink init ih_ualink\n");
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_ualink, IH_UALINK_RING_SIZE, true);
 	if (r)
 		return r;
 
@@ -655,7 +675,11 @@ static int ih_v7_0_hw_init(struct amdgpu_ip_block *ip_block)
 
 static int ih_v7_0_hw_fini(struct amdgpu_ip_block *ip_block)
 {
-	ih_v7_0_irq_disable(ip_block->adev);
+	struct amdgpu_device *adev = ip_block->adev;
+
+	ih_v7_0_irq_disable(adev);
+
+	cancel_work_sync(&adev->irq.ih_psp_work);
 
 	return 0;
 }
@@ -670,12 +694,6 @@ static int ih_v7_0_resume(struct amdgpu_ip_block *ip_block)
 	return ih_v7_0_hw_init(ip_block);
 }
 
-static bool ih_v7_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-	return true;
-}
-
 static int ih_v7_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* todo */
@@ -844,7 +862,6 @@ static const struct amd_ip_funcs ih_v7_0_ip_funcs = {
 	.hw_fini = ih_v7_0_hw_fini,
 	.suspend = ih_v7_0_suspend,
 	.resume = ih_v7_0_resume,
-	.is_idle = ih_v7_0_is_idle,
 	.wait_for_idle = ih_v7_0_wait_for_idle,
 	.soft_reset = ih_v7_0_soft_reset,
 	.set_clockgating_state = ih_v7_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
index fe20cf2b945452..313c09b48e6332 100644
--- a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
@@ -32,8 +32,6 @@
 #include "gc/gc_12_1_0_sh_mask.h"
 #include "mmhub/mmhub_4_2_0_offset.h"
 
-MODULE_FIRMWARE("amdgpu/gc_12_1_0_imu.bin");
-
 #define TRANSFER_RAM_MASK	0x001c0000
 
 static int imu_v12_1_init_microcode(struct amdgpu_device *adev)
@@ -133,24 +131,111 @@ static int imu_v12_1_load_microcode(struct amdgpu_device *adev)
 	return 0;
 }
 
+#define regGFX_IMU_PARTITION_SWITCH		0x5f8c
+#define regGFX_IMU_PARTITION_SWITCH_BASE_IDX	1
+#define GFX_IMU_PARTITION_SWITCH__PARTITION_MODE__SHIFT		0x0
+#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP__SHIFT	0x2
+#define GFX_IMU_PARTITION_SWITCH__VIRTUAL_XCC_ID__SHIFT		0x9
+#define GFX_IMU_PARTITION_SWITCH__PHYSICAL_XCC_MASK__SHIFT	0x10
+#define GFX_IMU_PARTITION_SWITCH__PARTITION_MODE_MASK		0x00000003L
+#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP_MASK	0x0000003CL
+#define GFX_IMU_PARTITION_SWITCH__VIRTUAL_XCC_ID_MASK		0x00000E00L
+#define GFX_IMU_PARTITION_SWITCH__PHYSICAL_XCC_MASK_MASK	0x00FF0000L
+
 static int imu_v12_1_switch_compute_partition(struct amdgpu_device *adev,
 					      int num_xccs_per_xcp,
 					      int compute_partition_mode)
 {
-	int ret;
+	int ret, i, num_xcc;
+	u32 tmp = 0, physical_xcc_mask = 0;
 
 	if (adev->psp.funcs) {
 		ret = psp_spatial_partition(&adev->psp, compute_partition_mode);
 		if (ret)
 			return ret;
+	} else {
+		num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+		/* 00 - SPX; 01 - DPX; 10 - QPX; 11 - CPX */
+		switch (compute_partition_mode) {
+		case AMDGPU_SPX_PARTITION_MODE:
+			compute_partition_mode = 0;
+			break;
+		case AMDGPU_DPX_PARTITION_MODE:
+			compute_partition_mode = 1;
+			break;
+		case AMDGPU_QPX_PARTITION_MODE:
+			compute_partition_mode = 2;
+			break;
+		case AMDGPU_CPX_PARTITION_MODE:
+			compute_partition_mode = 3;
+			break;
+		default:
+			dev_err(adev->dev, "Invalid compute partition mode\n");
+			return -EINVAL;
+		}
+
+		num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+		for(i = 0; i < num_xcc; i++) {
+			if (i % num_xccs_per_xcp == 0) {
+				physical_xcc_mask =
+					adev->gfx.xcc_mask &
+					(GENMASK(GET_INST(GC, i + num_xccs_per_xcp - 1),
+						 GET_INST(GC, i)));
+			}
+			tmp = REG_SET_FIELD(tmp, GFX_IMU_PARTITION_SWITCH,
+					    PARTITION_MODE, compute_partition_mode);
+			tmp = REG_SET_FIELD(tmp, GFX_IMU_PARTITION_SWITCH,
+					    TOTAL_XCCS_IN_XCP, num_xccs_per_xcp);
+			tmp = REG_SET_FIELD(tmp, GFX_IMU_PARTITION_SWITCH,
+					    VIRTUAL_XCC_ID, i % num_xccs_per_xcp);
+			tmp = REG_SET_FIELD(tmp, GFX_IMU_PARTITION_SWITCH,
+					    PHYSICAL_XCC_MASK, physical_xcc_mask);
+			WREG32_SOC15(GC, GET_INST(GC, i), regGFX_IMU_PARTITION_SWITCH, tmp);
+		}
+		ret = 0;
 	}
 
 	adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;
 	return 0;
 }
 
+#define regGFX_IMU_MCM_ADDR_LUT_PF		0x5f8d
+#define regGFX_IMU_MCM_ADDR_LUT_PF_BASE_IDX	1
+
+static void imu_v12_1_init_mcm_addr_lut(struct amdgpu_device *adev)
+{
+	/* Encoding mcm_addr_lut in SPX mode for now
+	 * I've noticed variations in encoding
+	 * Need to revisit the configuration later
+	 * XCD0:: Bits-[3:0 ] AID-01 XCD-00
+	 * XCD1:: Bits-[7:4 ] AID-01 XCD-01
+	 * XCD2:: Bits-[11:8 ] AID-01 XCD-10
+	 * XCD3:: Bits-[15:12] AID-01 XCD-11
+	 * XCD4:: Bits-[19:16] AID-10 XCD-00
+	 * XCD5:: Bits-[23:20] AID-10 XCD-01
+	 * XCD6:: Bits-[27:24] AID-10 XCD-10
+	 * XCD7:: Bits-[31:28] AID-10 XCD-11
+	 */
+	int data, encode_data;
+	int i, j, num_xcc;
+
+	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	for (i = 0; i < (num_xcc / adev->gfx.num_xcc_per_xcp); i++) {
+		encode_data = 0;
+		for (j = 0; j < adev->gfx.num_xcc_per_xcp; j++) {
+			data = ((GET_INST(GC, i * adev->gfx.num_xcc_per_xcp + j) / 4 + 1) << 2) |
+			        (GET_INST(GC, i * adev->gfx.num_xcc_per_xcp + j) % 4);
+			encode_data |= data << (j * 4);
+		}
+		for (j = 0; j < adev->gfx.num_xcc_per_xcp; j++)
+			WREG32_SOC15(GC, GET_INST(GC, i * adev->gfx.num_xcc_per_xcp + j),
+					regGFX_IMU_MCM_ADDR_LUT_PF, encode_data);
+	}
+}
+
 const struct amdgpu_imu_funcs gfx_v12_1_imu_funcs = {
 	.init_microcode = imu_v12_1_init_microcode,
 	.load_microcode = imu_v12_1_load_microcode,
 	.switch_compute_partition = imu_v12_1_switch_compute_partition,
+	.init_mcm_addr_lut = imu_v12_1_init_mcm_addr_lut,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
index a9c00adcfe4107..72069b77f6f1fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c
@@ -791,7 +791,6 @@ static const struct amd_ip_funcs jpeg_v2_0_ip_funcs = {
 	.hw_fini = jpeg_v2_0_hw_fini,
 	.suspend = jpeg_v2_0_suspend,
 	.resume = jpeg_v2_0_resume,
-	.is_idle = jpeg_v2_0_is_idle,
 	.wait_for_idle = jpeg_v2_0_wait_for_idle,
 	.set_clockgating_state = jpeg_v2_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v2_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
index 5208312e7017c1..2968256d4cd741 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
@@ -665,7 +665,6 @@ static const struct amd_ip_funcs jpeg_v2_5_ip_funcs = {
 	.hw_fini = jpeg_v2_5_hw_fini,
 	.suspend = jpeg_v2_5_suspend,
 	.resume = jpeg_v2_5_resume,
-	.is_idle = jpeg_v2_5_is_idle,
 	.wait_for_idle = jpeg_v2_5_wait_for_idle,
 	.set_clockgating_state = jpeg_v2_5_set_clockgating_state,
 	.set_powergating_state = jpeg_v2_5_set_powergating_state,
@@ -682,7 +681,6 @@ static const struct amd_ip_funcs jpeg_v2_6_ip_funcs = {
 	.hw_fini = jpeg_v2_5_hw_fini,
 	.suspend = jpeg_v2_5_suspend,
 	.resume = jpeg_v2_5_resume,
-	.is_idle = jpeg_v2_5_is_idle,
 	.wait_for_idle = jpeg_v2_5_wait_for_idle,
 	.set_clockgating_state = jpeg_v2_5_set_clockgating_state,
 	.set_powergating_state = jpeg_v2_5_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
index e9ccadeb76961e..564e40899df22c 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c
@@ -564,7 +564,6 @@ static const struct amd_ip_funcs jpeg_v3_0_ip_funcs = {
 	.hw_fini = jpeg_v3_0_hw_fini,
 	.suspend = jpeg_v3_0_suspend,
 	.resume = jpeg_v3_0_resume,
-	.is_idle = jpeg_v3_0_is_idle,
 	.wait_for_idle = jpeg_v3_0_wait_for_idle,
 	.set_clockgating_state = jpeg_v3_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v3_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
index 6fd4238a8471a9..b29ca3a49939b2 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c
@@ -748,7 +748,6 @@ static const struct amd_ip_funcs jpeg_v4_0_ip_funcs = {
 	.hw_fini = jpeg_v4_0_hw_fini,
 	.suspend = jpeg_v4_0_suspend,
 	.resume = jpeg_v4_0_resume,
-	.is_idle = jpeg_v4_0_is_idle,
 	.wait_for_idle = jpeg_v4_0_wait_for_idle,
 	.set_clockgating_state = jpeg_v4_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v4_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
index 0fdc32b3ae91b6..0343a64595f2c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
@@ -22,6 +22,7 @@
  */
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_jpeg.h"
 #include "soc15.h"
 #include "soc15d.h"
@@ -1225,7 +1226,6 @@ static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = {
 	.hw_fini = jpeg_v4_0_3_hw_fini,
 	.suspend = jpeg_v4_0_3_suspend,
 	.resume = jpeg_v4_0_3_resume,
-	.is_idle = jpeg_v4_0_3_is_idle,
 	.wait_for_idle = jpeg_v4_0_3_wait_for_idle,
 	.set_clockgating_state = jpeg_v4_0_3_set_clockgating_state,
 	.set_powergating_state = jpeg_v4_0_3_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
index 3f61d5367d9469..4da3f9c995c11f 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c
@@ -22,6 +22,7 @@
  */
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_jpeg.h"
 #include "amdgpu_pm.h"
 #include "soc15.h"
@@ -806,7 +807,6 @@ static const struct amd_ip_funcs jpeg_v4_0_5_ip_funcs = {
 	.hw_fini = jpeg_v4_0_5_hw_fini,
 	.suspend = jpeg_v4_0_5_suspend,
 	.resume = jpeg_v4_0_5_resume,
-	.is_idle = jpeg_v4_0_5_is_idle,
 	.wait_for_idle = jpeg_v4_0_5_wait_for_idle,
 	.set_clockgating_state = jpeg_v4_0_5_set_clockgating_state,
 	.set_powergating_state = jpeg_v4_0_5_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
index e6e99f7e0a8a95..dcdb089b43083d 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c
@@ -22,6 +22,7 @@
  */
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_jpeg.h"
 #include "amdgpu_pm.h"
 #include "soc15.h"
@@ -682,7 +683,6 @@ static const struct amd_ip_funcs jpeg_v5_0_0_ip_funcs = {
 	.hw_fini = jpeg_v5_0_0_hw_fini,
 	.suspend = jpeg_v5_0_0_suspend,
 	.resume = jpeg_v5_0_0_resume,
-	.is_idle = jpeg_v5_0_0_is_idle,
 	.wait_for_idle = jpeg_v5_0_0_wait_for_idle,
 	.set_clockgating_state = jpeg_v5_0_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v5_0_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
index db3fbe4eabb991..f2debf6b6bd070 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c
@@ -22,6 +22,7 @@
  */
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_jpeg.h"
 #include "amdgpu_pm.h"
 #include "soc15.h"
@@ -772,7 +773,7 @@ static int jpeg_v5_0_1_set_ras_interrupt_state(struct amdgpu_device *adev,
 
 
 
-int jpeg_v5_0_1_process_interrupt(struct amdgpu_device *adev,
+static int jpeg_v5_0_1_process_interrupt(struct amdgpu_device *adev,
 					 struct amdgpu_irq_src *source,
 					 struct amdgpu_iv_entry *entry)
 {
@@ -886,7 +887,6 @@ static const struct amd_ip_funcs jpeg_v5_0_1_ip_funcs = {
 	.hw_fini = jpeg_v5_0_1_hw_fini,
 	.suspend = jpeg_v5_0_1_suspend,
 	.resume = jpeg_v5_0_1_resume,
-	.is_idle = jpeg_v5_0_1_is_idle,
 	.wait_for_idle = jpeg_v5_0_1_wait_for_idle,
 	.soft_reset = NULL,
 	.set_clockgating_state = jpeg_v5_0_1_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h
index 67346faecb4730..a7e58d5fb2465e 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.h
@@ -108,12 +108,4 @@ enum amdgpu_jpeg_v5_0_1_sub_block {
 	AMDGPU_JPEG_V5_0_1_MAX_SUB_BLOCK,
 };
 
-struct amdgpu_irq_src;
-struct amdgpu_iv_entry;
-struct amdgpu_device;
-
-int jpeg_v5_0_1_process_interrupt(struct amdgpu_device *adev,
-					struct amdgpu_irq_src *source,
-					struct amdgpu_iv_entry *entry);
-
 #endif /* __JPEG_V5_0_1_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
index ea54a2b6eddb8e..dd0abce73687c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_2.c
@@ -34,8 +34,7 @@
 #include "vcn/vcn_5_0_0_sh_mask.h"
 #include "ivsrcid/vcn/irqsrcs_vcn_5_0.h"
 
-#include "jpeg_v5_0_1.h"
-
+static int jpeg_v5_0_2_start_sriov(struct amdgpu_device *adev);
 static void jpeg_v5_0_2_set_dec_ring_funcs(struct amdgpu_device *adev);
 static void jpeg_v5_0_2_set_irq_funcs(struct amdgpu_device *adev);
 static int jpeg_v5_0_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
@@ -140,7 +139,12 @@ static int jpeg_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 		/* JPEG TRAP */
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
+		r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_VCN,
+				      amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
+		if (r)
+			return r;
+
+		r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_VCN1,
 				      amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
 		if (r)
 			return r;
@@ -159,12 +163,18 @@ static int jpeg_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
 
 		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 			ring = &adev->jpeg.inst[i].ring_dec[j];
-			ring->use_doorbell = false;
+			ring->use_doorbell = true;
 			ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id);
-			ring->doorbell_index =
-				(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
-				1 + j + 11 * jpeg_inst;
-			sprintf(ring->name, "jpeg_dec_%d.%d", adev->jpeg.inst[i].aid_id, j);
+			if (amdgpu_sriov_vf(adev)) {
+				ring->doorbell_index =
+					(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
+					2 + j + 32 * jpeg_inst;
+			} else {
+				ring->doorbell_index =
+					(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
+					1 + j + 11 * jpeg_inst;
+			}
+			sprintf(ring->name, "jpeg_dec_%d.%d", i, j);
 			r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst->irq, 0,
 					     AMDGPU_RING_PRIO_DEFAULT, NULL);
 			if (r)
@@ -184,7 +194,8 @@ static int jpeg_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
 
 	adev->jpeg.supported_reset =
 		amdgpu_get_soft_full_reset_mask(&adev->jpeg.inst[0].ring_dec[0]);
-	adev->jpeg.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
+	if (!amdgpu_sriov_vf(adev))
+		adev->jpeg.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE;
 	r = amdgpu_jpeg_sysfs_reset_mask_init(adev);
 
 	return r;
@@ -225,6 +236,23 @@ static int jpeg_v5_0_2_hw_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_ring *ring;
 	int i, j, r, jpeg_inst, tmp;
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = jpeg_v5_0_2_start_sriov(adev);
+		if (r)
+			return r;
+
+		for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
+			for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
+				ring = &adev->jpeg.inst[i].ring_dec[j];
+				ring->wptr = 0;
+				ring->wptr_old = 0;
+				jpeg_v5_0_2_dec_ring_set_wptr(ring);
+				ring->sched.ready = true;
+			}
+		}
+		return 0;
+	}
+
 	if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x100)
 		adev->jpeg.caps |= AMDGPU_JPEG_CAPS(RRMT_ENABLED);
 
@@ -240,7 +268,7 @@ static int jpeg_v5_0_2_hw_init(struct amdgpu_ip_block *ip_block)
 		if (ring->use_doorbell)
 			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
 				 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 11 * jpeg_inst,
-				 adev->jpeg.inst[i].aid_id);
+				 jpeg_inst);
 
 		for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 			ring = &adev->jpeg.inst[i].ring_dec[j];
@@ -273,8 +301,10 @@ static int jpeg_v5_0_2_hw_fini(struct amdgpu_ip_block *ip_block)
 
 	cancel_delayed_work_sync(&adev->jpeg.idle_work);
 
+	if (!amdgpu_sriov_vf(adev)) {
 		if (adev->jpeg.cur_state != AMD_PG_STATE_GATE)
 			ret = jpeg_v5_0_2_set_powergating_state(ip_block, AMD_PG_STATE_GATE);
+	}
 
 	return ret;
 }
@@ -404,6 +434,122 @@ static void jpeg_v5_0_2_init_jrbc(struct amdgpu_ring *ring)
 					 reg_offset);
 }
 
+/**
+ * jpeg_v5_0_2_start_sriov - start JPEG block under SRIOV env
+ */
+static int jpeg_v5_0_2_start_sriov(struct amdgpu_device *adev)
+{
+	struct amdgpu_ring *ring;
+	uint64_t ctx_addr;
+	uint32_t param, resp, expected;
+	uint32_t tmp, timeout;
+
+	struct amdgpu_mm_table *table = &adev->virt.mm_table;
+	uint32_t *table_loc;
+	uint32_t table_size;
+	uint32_t size, size_dw, item_offset;
+	uint32_t init_status;
+	int i, j, jpeg_inst;
+
+	struct mmsch_v5_0_cmd_direct_write
+		direct_wt = { {0} };
+	struct mmsch_v5_0_cmd_end end = { {0} };
+	struct mmsch_v5_0_init_header header;
+
+	direct_wt.cmd_header.command_type =
+		MMSCH_COMMAND__DIRECT_REG_WRITE;
+	end.cmd_header.command_type =
+		MMSCH_COMMAND__END;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
+		jpeg_inst = GET_INST(JPEG, i);
+
+		memset(&header, 0, sizeof(struct mmsch_v5_0_init_header));
+		header.version = MMSCH_VERSION;
+		header.total_size = sizeof(struct mmsch_v5_0_init_header) >> 2;
+
+		table_loc = (uint32_t *)table->cpu_addr;
+		table_loc += header.total_size;
+
+		item_offset = header.total_size;
+
+		for (j = 0; j < adev->jpeg.num_jpeg_rings; j++) {
+			ring = &adev->jpeg.inst[i].ring_dec[j];
+			table_size = 0;
+
+			tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_LMI_JRBC_RB_64BIT_BAR_LOW);
+			MMSCH_V5_0_INSERT_DIRECT_WT(tmp, lower_32_bits(ring->gpu_addr));
+			tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_LMI_JRBC_RB_64BIT_BAR_HIGH);
+			MMSCH_V5_0_INSERT_DIRECT_WT(tmp, upper_32_bits(ring->gpu_addr));
+			tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JRBC_RB_SIZE);
+			MMSCH_V5_0_INSERT_DIRECT_WT(tmp, ring->ring_size / 4);
+
+			if (j < 5) {
+				header.mjpegdec0[j].table_offset = item_offset;
+				header.mjpegdec0[j].init_status = 0;
+				header.mjpegdec0[j].table_size = table_size;
+			} else {
+				header.mjpegdec1[j - 5].table_offset = item_offset;
+				header.mjpegdec1[j - 5].init_status = 0;
+				header.mjpegdec1[j - 5].table_size = table_size;
+			}
+			header.total_size += table_size;
+			item_offset += table_size;
+		}
+
+		MMSCH_V5_0_INSERT_END();
+
+		/* send init table to MMSCH */
+		size = sizeof(struct mmsch_v5_0_init_header);
+		table_loc = (uint32_t *)table->cpu_addr;
+		memcpy((void *)table_loc, &header, size);
+
+		ctx_addr = table->gpu_addr;
+		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
+		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
+
+		tmp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID);
+		tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
+		tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
+		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_VMID, tmp);
+
+		size = header.total_size;
+		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_CTX_SIZE, size);
+
+		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP, 0);
+
+		param = 0x00000001;
+		WREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_HOST, param);
+		tmp = 0;
+		timeout = amdgpu_emu_mode ? 1000000 : 1000;
+		resp = 0;
+		expected = MMSCH_VF_MAILBOX_RESP__OK;
+		init_status =
+			((struct mmsch_v5_0_init_header *)(table_loc))->mjpegdec0[i].init_status;
+		while (resp != expected) {
+			resp = RREG32_SOC15(VCN, jpeg_inst, regMMSCH_VF_MAILBOX_RESP);
+
+			if (resp != 0)
+				break;
+			udelay(10);
+			tmp = tmp + 10;
+			if (tmp >= timeout) {
+				DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
+					" waiting for regMMSCH_VF_MAILBOX_RESP "\
+					"(expected=0x%08x, readback=0x%08x)\n",
+					tmp, expected, resp);
+				return -EBUSY;
+			}
+		}
+		if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE &&
+				init_status != MMSCH_VF_ENGINE_STATUS__PASS)
+			DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init status for jpeg: %x\n",
+					resp, init_status);
+
+	}
+	return 0;
+}
+
 /**
  * jpeg_v5_0_2_start - start JPEG block
  *
@@ -502,7 +648,7 @@ static void jpeg_v5_0_2_dec_ring_set_wptr(struct amdgpu_ring *ring)
 static bool jpeg_v5_0_2_is_idle(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
-	bool ret = false;
+	bool ret = true;
 	int i, j;
 
 	for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) {
@@ -563,6 +709,11 @@ static int jpeg_v5_0_2_set_powergating_state(struct amdgpu_ip_block *ip_block,
 	struct amdgpu_device *adev = ip_block->adev;
 	int ret;
 
+	if (amdgpu_sriov_vf(adev)) {
+		adev->jpeg.cur_state = AMD_PG_STATE_UNGATE;
+		return 0;
+	}
+
 	if (state == adev->jpeg.cur_state)
 		return 0;
 
@@ -585,6 +736,95 @@ static int jpeg_v5_0_2_set_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int jpeg_v5_0_2_process_interrupt(struct amdgpu_device *adev,
+					 struct amdgpu_irq_src *source,
+					 struct amdgpu_iv_entry *entry)
+{
+	uint32_t mid, cid;
+	int i, jpeg_inst;
+
+	DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
+
+	switch (entry->client_id) {
+	case SOC_V1_0_IH_CLIENTID_VCN:
+		cid = 0;
+		break;
+	case SOC_V1_0_IH_CLIENTID_VCN1:
+		cid = 1;
+		break;
+	default:
+		dev_WARN_ONCE(adev->dev, 1,
+			      "Interrupt received for unknown JPEG client_id %d",
+			      entry->client_id);
+		return 0;
+	}
+
+	switch (entry->node_id) {
+	case 0:
+		mid = 0;
+		break;
+	case 8:
+		mid = 1;
+		break;
+	default:
+		dev_WARN_ONCE(adev->dev, 1,
+			      "Interrupt received for unknown JPEG node_id %d",
+			      entry->node_id);
+		return 0;
+	}
+
+	jpeg_inst = mid * adev->jpeg.num_inst_per_aid + cid;
+
+	for (i = 0; i < adev->jpeg.num_jpeg_inst; i++)
+		if (GET_INST(JPEG, i) == jpeg_inst)
+			break;
+
+	if (i >= adev->jpeg.num_jpeg_inst) {
+		dev_WARN_ONCE(adev->dev, 1,
+			      "Interrupt received for unknown JPEG inst %d",
+			      jpeg_inst);
+		return 0;
+	}
+
+	switch (entry->src_id) {
+	case VCN_5_0__SRCID__JPEG_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[0]);
+		break;
+	case VCN_5_0__SRCID__JPEG1_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[1]);
+		break;
+	case VCN_5_0__SRCID__JPEG2_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[2]);
+		break;
+	case VCN_5_0__SRCID__JPEG3_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[3]);
+		break;
+	case VCN_5_0__SRCID__JPEG4_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[4]);
+		break;
+	case VCN_5_0__SRCID__JPEG5_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[5]);
+		break;
+	case VCN_5_0__SRCID__JPEG6_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[6]);
+		break;
+	case VCN_5_0__SRCID__JPEG7_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[7]);
+		break;
+	case VCN_5_0__SRCID__JPEG8_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[8]);
+		break;
+	case VCN_5_0__SRCID__JPEG9_DECODE:
+		amdgpu_fence_process(&adev->jpeg.inst[i].ring_dec[9]);
+		break;
+	default:
+		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
+			      entry->src_id, entry->src_data[0]);
+		break;
+	}
+
+	return 0;
+}
 
 static void jpeg_v5_0_2_core_stall_reset(struct amdgpu_ring *ring)
 {
@@ -631,7 +871,6 @@ static const struct amd_ip_funcs jpeg_v5_0_2_ip_funcs = {
 	.hw_fini = jpeg_v5_0_2_hw_fini,
 	.suspend = jpeg_v5_0_2_suspend,
 	.resume = jpeg_v5_0_2_resume,
-	.is_idle = jpeg_v5_0_2_is_idle,
 	.wait_for_idle = jpeg_v5_0_2_wait_for_idle,
 	.soft_reset = NULL,
 	.set_clockgating_state = jpeg_v5_0_2_set_clockgating_state,
@@ -690,7 +929,7 @@ static void jpeg_v5_0_2_set_dec_ring_funcs(struct amdgpu_device *adev)
 
 static const struct amdgpu_irq_src_funcs jpeg_v5_0_2_irq_funcs = {
 	.set = jpeg_v5_0_2_set_interrupt_state,
-	.process = jpeg_v5_0_1_process_interrupt,
+	.process = jpeg_v5_0_2_process_interrupt,
 };
 
 static void jpeg_v5_0_2_set_irq_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
index c929d6f83bac6b..8ddd12740a040d 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c
@@ -671,7 +671,6 @@ static const struct amd_ip_funcs jpeg_v5_3_0_ip_funcs = {
 	.hw_fini = jpeg_v5_3_0_hw_fini,
 	.suspend = jpeg_v5_3_0_suspend,
 	.resume = jpeg_v5_3_0_resume,
-	.is_idle = jpeg_v5_3_0_is_idle,
 	.wait_for_idle = jpeg_v5_3_0_wait_for_idle,
 	.set_clockgating_state = jpeg_v5_3_0_set_clockgating_state,
 	.set_powergating_state = jpeg_v5_3_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c
index d93a0e65ce7de9..fa739b0e9ddfba 100644
--- a/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/lsdma_v7_1.c
@@ -31,9 +31,34 @@
 
 static int lsdma_v7_1_wait_pio_status(struct amdgpu_device *adev)
 {
-	return amdgpu_lsdma_wait_for(adev, SOC15_REG_OFFSET(LSDMA, 0, regLSDMA_PIO_STATUS),
+	u32 error_mask;
+	u32 reg;
+	int r;
+
+	r = amdgpu_lsdma_wait_for(adev, SOC15_REG_OFFSET(LSDMA, 0, regLSDMA_PIO_STATUS),
 			LSDMA_PIO_STATUS__PIO_IDLE_MASK | LSDMA_PIO_STATUS__PIO_FIFO_EMPTY_MASK,
 			LSDMA_PIO_STATUS__PIO_IDLE_MASK | LSDMA_PIO_STATUS__PIO_FIFO_EMPTY_MASK);
+	if (r)
+		return r;
+
+	error_mask = LSDMA_PIO_STATUS__ERROR_WRRET_NACK_GEN_ERR_MASK |
+		     LSDMA_PIO_STATUS__ERROR_RDRET_NACK_GEN_ERR_MASK |
+		     LSDMA_PIO_STATUS__ERROR_INVALID_ADDR_MASK |
+		     LSDMA_PIO_STATUS__ERROR_ZERO_COUNT_MASK |
+		     LSDMA_PIO_STATUS__ERROR_DRAM_ECC_MASK;
+
+	reg = RREG32(SOC15_REG_OFFSET(LSDMA, 0, regLSDMA_PIO_STATUS));
+	if (reg & error_mask) {
+		dev_warn(adev->dev, "LSDMA PIO error status bits 0x%x\n", reg & error_mask);
+
+		/*
+		 * write 1 to clear only the set lsdma error status bits
+		 */
+		WREG32(SOC15_REG_OFFSET(LSDMA, 0, regLSDMA_PIO_STATUS), reg & error_mask);
+		return -EIO;
+	}
+
+	return 0;
 }
 
 static int lsdma_v7_1_copy_mem(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 14a5abe42d1ffc..fe49aa4dc5ff37 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -53,14 +53,15 @@ mes_userq_create_wptr_mapping(struct amdgpu_device *adev,
 		if (unlikely(ret))
 			goto fail_lock;
 
-		wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT);
+		wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr);
 		if (!wptr_mapping) {
 			ret = -EINVAL;
 			goto fail_lock;
 		}
 
 		obj = wptr_mapping->bo_va->base.bo;
-		ret = drm_exec_lock_obj(&exec, &obj->tbo.base);
+		ret = drm_exec_prepare_obj(&exec, &obj->tbo.base,
+					   TTM_NUM_MOVE_FENCES + 1);
 		drm_exec_retry_on_contention(&exec);
 		if (unlikely(ret))
 			goto fail_lock;
@@ -171,6 +172,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
 		return r;
 	}
 
+	amdgpu_mes_userq_queue_mapped(adev);
+
 	DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
 	return 0;
 }
@@ -195,9 +198,13 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue)
 	amdgpu_mes_unlock(&adev->mes);
 	if (mes->use_rs64mem)
 		amdgpu_mes_free_gang_ctx_index(mes, queue->gang_ctx_array_index);
-	if (r)
+	if (r) {
 		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
-	return r;
+		return r;
+	}
+
+	amdgpu_mes_userq_queue_unmapped(adev);
+	return 0;
 }
 
 int mes_userq_reset(struct amdgpu_usermode_queue *queue)
@@ -207,26 +214,55 @@ int mes_userq_reset(struct amdgpu_usermode_queue *queue)
 	struct mes_reset_queue_input queue_input;
 	int r;
 
-	/* XXX: add a FW version check for SDMA per queue reset */
+	/* already reset by an earlier job's hang-detect; just signal and bail */
+	if (queue->state == AMDGPU_USERQ_STATE_HUNG)
+		return 0;
+
 	memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input));
 	queue_input.doorbell_offset = queue->doorbell_index;
 	queue_input.queue_type = queue->queue_type;
 
+	/*
+	 * The MES packet reset fails once the hung queue wedges MES. For SDMA the
+	 * queue is still on its HW slot, so reset it over MMIO instead: recover
+	 * the (instance, queue_id) slot from the doorbell.
+	 */
+	if (queue->queue_type == AMDGPU_HW_IP_DMA &&
+	    adev->sdma.instance[0].funcs &&
+	    adev->sdma.instance[0].funcs->detect_hung_queue) {
+		u32 instance, hw_queue_id;
+
+		if (adev->sdma.instance[0].funcs->detect_hung_queue(adev,
+				queue->doorbell_index, &instance, &hw_queue_id)) {
+			queue_input.use_mmio = true;
+			queue_input.me_id = instance;
+			queue_input.queue_id = hw_queue_id;
+		} else {
+			dev_warn(adev->dev,
+				 "SDMA userq (doorbell %llu) not on any HW slot; falling back to MES reset\n",
+				 queue->doorbell_index);
+		}
+	}
+
 	amdgpu_mes_lock(&adev->mes);
 	r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input);
 	amdgpu_mes_unlock(&adev->mes);
 	if (r)
 		return r;
 
-	/* mes_userq_unmap() does not update queue->state; mark it UNMAPPED so the
-	 * destroy path does not issue a second REMOVE_QUEUE for the removed queue.
-	 */
+	/* drop the queue from MES */
 	r = mes_userq_unmap(queue);
-	if (!r) {
-		trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED);
-		queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
-	}
-	return r;
+	if (r)
+		return r;
+
+	/*
+	 * HUNG, not UNMAPPED: the guilty job is still in the ring, so the
+	 * restore worker must not re-map and re-run it.
+	 */
+	trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG);
+	queue->state = AMDGPU_USERQ_STATE_HUNG;
+
+	return 0;
 }
 
 int mes_userq_reset_queue(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index c57bcb9a98b197..51bfb672bed7ea 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -257,14 +257,17 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 	if (r < 1 || !*status_ptr) {
 
 		if (misc_op_str)
-			dev_err(adev->dev, "MES failed to respond to msg=%s (%s)\n",
-				op_str, misc_op_str);
+			dev_err(adev->dev,
+				"MES failed to respond to msg=%s (%s) fence_wait_ret=%ld status=0x%llx\n",
+				op_str, misc_op_str, r, *status_ptr);
 		else if (op_str)
-			dev_err(adev->dev, "MES failed to respond to msg=%s\n",
-				op_str);
+			dev_err(adev->dev,
+				"MES failed to respond to msg=%s fence_wait_ret=%ld status=0x%llx\n",
+				op_str, r, *status_ptr);
 		else
-			dev_err(adev->dev, "MES failed to respond to msg=%d\n",
-				x_pkt->header.opcode);
+			dev_err(adev->dev,
+				"MES failed to respond to msg=%d fence_wait_ret=%ld status=0x%llx\n",
+				x_pkt->header.opcode, r, *status_ptr);
 
 		while (halt_if_hws_hang)
 			schedule();
@@ -957,7 +960,15 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 		misc_pkt.change_config.option.bits.limit_single_process =
 				input->change_config.option.limit_single_process;
 		break;
-
+	case MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE:
+		if ((mes->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) < 0x70) {
+			dev_warn_once(mes->adev->dev,
+				      "MES FW version must be larger than 0x70 to support notify work on unmapped queue.\n");
+			return 0;
+		}
+		misc_pkt.opcode = MESAPI_MISC__NOTIFY_WORK_ON_UNMAPPED_QUEUE;
+		misc_pkt.queue_sch_level = AMD_PRIORITY_LEVEL_NORMAL;
+		break;
 	default:
 		drm_err(adev_to_drm(mes->adev), "unsupported misc op (%d)\n", input->op);
 		return -EINVAL;
@@ -1016,7 +1027,15 @@ static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
 	mes_set_hw_res_pkt.use_different_vmid_compute = 1;
 	mes_set_hw_res_pkt.enable_reg_active_poll = 1;
 	mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
-	mes_set_hw_res_pkt.oversubscription_timer = 50;
+	/*
+	 * Bare metal: disabled here, KFD arms a software replacement only
+	 * while oversubscribed (see kfd_device_queue_manager.c). SR-IOV
+	 * guests keep the firmware timer instead of the KFD-side one, since
+	 * the notify round-trip is much slower under SR-IOV and would
+	 * otherwise stall remapping queues.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		mes_set_hw_res_pkt.oversubscription_timer = 50;
 	if (adev->mes.use_rs64mem)
 		mes_set_hw_res_pkt.use_rs64mem_for_proc_gang_ctx = 1;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index ee72afdb0a1d37..9d220bff0f6f1a 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -240,14 +240,20 @@ static int mes_v12_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 	if (r < 1 || !(lower_32_bits(*status_ptr))) {
 
 		if (misc_op_str)
-			dev_err(adev->dev, "MES(%d) failed to respond to msg=%s (%s)\n",
-				pipe, op_str, misc_op_str);
+			dev_err(adev->dev,
+				"MES(%d) failed to respond to msg=%s (%s) fence_wait_ret=%ld status=0x%x_%x\n",
+				pipe, op_str, misc_op_str, r,
+				upper_32_bits(*status_ptr), lower_32_bits(*status_ptr));
 		else if (op_str)
-			dev_err(adev->dev, "MES(%d) failed to respond to msg=%s\n",
-				pipe, op_str);
+			dev_err(adev->dev,
+				"MES(%d) failed to respond to msg=%s fence_wait_ret=%ld status=0x%x_%x\n",
+				pipe, op_str, r,
+				upper_32_bits(*status_ptr), lower_32_bits(*status_ptr));
 		else
-			dev_err(adev->dev, "MES(%d) failed to respond to msg=%d\n",
-				pipe, x_pkt->header.opcode);
+			dev_err(adev->dev,
+				"MES(%d) failed to respond to msg=%d fence_wait_ret=%ld status=0x%x_%x\n",
+				pipe, x_pkt->header.opcode, r,
+				upper_32_bits(*status_ptr), lower_32_bits(*status_ptr));
 
 		while (halt_if_hws_hang)
 			schedule();
@@ -1306,6 +1312,17 @@ static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable)
 	uint32_t pipe, data = 0;
 
 	if (enable) {
+		/*
+		 * The event-log/MSCRATCH region is only zeroed once at sw_init.
+		 * On the resume path (hw_init without sw_init) it still holds the
+		 * previous session's data, so the MES firmware walks stale
+		 * pointers on activation and triggers a CPC page fault. Re-clear
+		 * it before unhalting MES.
+		 */
+		if (amdgpu_mes_log_enable && adev->mes.event_log_cpu_addr)
+			memset(adev->mes.event_log_cpu_addr, 0,
+			       adev->mes.event_log_size);
+
 		mutex_lock(&adev->srbm_mutex);
 		for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
 			soc24_grbm_select(adev, 3, pipe, 0, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
index f1098118d5c05d..20159fb9d6ea3a 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
@@ -34,8 +34,6 @@
 #include "gfx_v12_1_pkt.h"
 #include "sdma_v7_1_0_pkt_open.h"
 
-MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes.bin");
-MODULE_FIRMWARE("amdgpu/gc_12_1_0_mes1.bin");
 MODULE_FIRMWARE("amdgpu/gc_12_1_0_uni_mes.bin");
 
 static int mes_v12_1_hw_init(struct amdgpu_ip_block *ip_block);
@@ -167,7 +165,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 	unsigned long flags;
 	u64 status_gpu_addr;
 	u32 seq, status_offset;
-	u64 *status_ptr;
+	u32 *status_ptr;
 	signed long r;
 	int ret;
 
@@ -186,7 +184,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 		return ret;
 
 	status_gpu_addr = adev->wb.gpu_addr + (status_offset * 4);
-	status_ptr = (u64 *)&adev->wb.wb[status_offset];
+	status_ptr = &adev->wb.wb[status_offset];
 	*status_ptr = 0;
 
 	spin_lock_irqsave(ring_lock, flags);
@@ -235,19 +233,19 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 			xcc_id, pipe, x_pkt->header.opcode);
 
 	r = amdgpu_fence_wait_polling(ring, seq, timeout);
-	if (r < 1 || !lower_32_bits(*status_ptr)) {
+	if (r < 1 || !*status_ptr) {
 		if (misc_op_str)
 			dev_err(adev->dev,
-				"MES(%d, %d) failed to respond to msg=%s (%s)\n",
-				xcc_id, pipe, op_str, misc_op_str);
+				"MES(%d, %d) failed to respond to msg=%s (%s) fence_wait_ret=%ld status=0x%x\n",
+				xcc_id, pipe, op_str, misc_op_str, r, *status_ptr);
 		else if (op_str)
 			dev_err(adev->dev,
-				"MES(%d, %d) failed to respond to msg=%s\n",
-				xcc_id, pipe, op_str);
+				"MES(%d, %d) failed to respond to msg=%s fence_wait_ret=%ld status=0x%x\n",
+				xcc_id, pipe, op_str, r, *status_ptr);
 		else
 			dev_err(adev->dev,
-				"MES(%d, %d) failed to respond to msg=%d\n",
-				xcc_id, pipe, x_pkt->header.opcode);
+				"MES(%d, %d) failed to respond to msg=%d fence_wait_ret=%ld status=0x%x\n",
+				xcc_id, pipe, x_pkt->header.opcode, r, *status_ptr);
 
 		while (halt_if_hws_hang)
 			schedule();
@@ -291,10 +289,9 @@ static int mes_v12_1_add_hw_queue(struct amdgpu_mes *mes,
 {
 	union MESAPI__ADD_QUEUE mes_add_queue_pkt;
 	int xcc_id = input->xcc_id;
-	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
 
 	if (mes->enable_coop_mode)
-		xcc_id = mes->master_xcc_ids[inst];
+		xcc_id = mes->master_xcc_ids[xcc_id];
 
 	memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
 
@@ -352,10 +349,9 @@ static int mes_v12_1_remove_hw_queue(struct amdgpu_mes *mes,
 {
 	union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
 	int xcc_id = input->xcc_id;
-	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
 
 	if (mes->enable_coop_mode)
-		xcc_id = mes->master_xcc_ids[inst];
+		xcc_id = mes->master_xcc_ids[xcc_id];
 
 	memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
 
@@ -378,7 +374,6 @@ static int mes_v12_1_reset_hw_queue(struct amdgpu_mes *mes,
 				    struct mes_reset_queue_input *input)
 {
 	union MESAPI__RESET mes_reset_queue_pkt;
-	int pipe;
 
 	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
 
@@ -388,17 +383,12 @@ static int mes_v12_1_reset_hw_queue(struct amdgpu_mes *mes,
 
 	mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 	/* mes_reset_queue_pkt.gang_context_addr = input->gang_context_addr; */
-	/*mes_reset_queue_pkt.reset_queue_only = 1;*/
-
-	if (mes->adev->enable_uni_mes)
-		pipe = AMDGPU_MES_KIQ_PIPE;
-	else
-		pipe = AMDGPU_MES_SCHED_PIPE;
+	mes_reset_queue_pkt.reset_queue_only = 1;
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes,
-			input->xcc_id, pipe,
+			input->xcc_id, AMDGPU_MES_SCHED_PIPE,
 			&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
-			offsetof(union MESAPI__REMOVE_QUEUE, api_status));
+			offsetof(union MESAPI__RESET, api_status));
 }
 
 static int mes_v12_1_map_legacy_queue(struct amdgpu_mes *mes,
@@ -422,15 +412,11 @@ static int mes_v12_1_map_legacy_queue(struct amdgpu_mes *mes,
 		convert_to_mes_queue_type(input->queue_type);
 	mes_add_queue_pkt.map_legacy_kq = 1;
 
-	if (mes->adev->enable_uni_mes) {
-		/* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
-		if (input->queue_type == AMDGPU_RING_TYPE_MES)
-			pipe = AMDGPU_MES_KIQ_PIPE;
-		else
-			pipe = AMDGPU_MES_SCHED_PIPE;
-	} else {
+	/* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
+	if (input->queue_type == AMDGPU_RING_TYPE_MES)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
 		pipe = AMDGPU_MES_SCHED_PIPE;
-	}
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes,
 			input->xcc_id, pipe,
@@ -467,15 +453,11 @@ static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes,
 			convert_to_mes_queue_type(input->queue_type);
 	}
 
-	if (mes->adev->enable_uni_mes) {
-		/* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
-		if (input->queue_type == AMDGPU_RING_TYPE_MES)
-			pipe = AMDGPU_MES_KIQ_PIPE;
-		else
-			pipe = AMDGPU_MES_SCHED_PIPE;
-	} else {
+	/* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */
+	if (input->queue_type == AMDGPU_RING_TYPE_MES)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
 		pipe = AMDGPU_MES_SCHED_PIPE;
-	}
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes,
 			input->xcc_id, pipe,
@@ -574,7 +556,8 @@ static int mes_v12_1_misc_op(struct amdgpu_mes *mes,
 	union MESAPI__MISC misc_pkt;
 	int pipe;
 
-	if (mes->adev->enable_uni_mes)
+	/*OP_WRM_REG_WR_WAIT is used to do tlb invalidation which need to be handled in sched pipe for gfx_12_1.*/
+	if (input->op != MES_MISC_OP_WRM_REG_WR_WAIT)
 		pipe = AMDGPU_MES_KIQ_PIPE;
 	else
 		pipe = AMDGPU_MES_SCHED_PIPE;
@@ -663,7 +646,7 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
 					  int pipe, int xcc_id)
 {
 	union MESAPI_SET_HW_RESOURCES_1 mes_set_hw_res_1_pkt;
-	int master_xcc_id, inst = MES_PIPE_INST(xcc_id, pipe);
+	int master_xcc_id;
 
 	memset(&mes_set_hw_res_1_pkt, 0, sizeof(mes_set_hw_res_1_pkt));
 
@@ -673,14 +656,14 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
 	mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 100;
 
 	/* From version 0x74 above, pipe1 support use shared command buffer
-	   to distribute some tasks on individual XCCs*/
+ 	   to distribute some tasks on individual XCCs*/
 	if (mes->enable_coop_mode &&
 	    ((pipe == AMDGPU_MES_SCHED_PIPE) ||
 	    ((mes->kiq_version & AMDGPU_MES_VERSION_MASK) >= 0x74))) {
-		master_xcc_id = mes->master_xcc_ids[inst];
+		master_xcc_id = mes->master_xcc_ids[xcc_id];
 		mes_set_hw_res_1_pkt.mes_coop_mode = 1;
 		mes_set_hw_res_1_pkt.coop_sch_shared_mc_addr =
-			mes->shared_cmd_buf_gpu_addr[master_xcc_id + pipe];
+			mes->shared_cmd_buf_gpu_addr[MES_PIPE_INST(master_xcc_id, pipe)];
 	}
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
@@ -743,6 +726,8 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
 	mes_set_hw_res_pkt.use_different_vmid_compute = 1;
 	mes_set_hw_res_pkt.enable_reg_active_poll = 1;
 	mes_set_hw_res_pkt.enable_level_process_quantum_check = 1;
+	/* proceeds pipe reset if queue reset fails */
+	mes_set_hw_res_pkt.enable_compute_pipe_reset = 1;
 
 	/*
 	 * Keep oversubscribe timer for sdma . When we have unmapped doorbell
@@ -771,7 +756,7 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
 
 	if (pipe == AMDGPU_MES_SCHED_PIPE)
 		adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
-	else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
+	else if (pipe == AMDGPU_MES_KIQ_PIPE)
 		adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
 
 	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
@@ -885,10 +870,7 @@ static int mes_v12_1_reset_legacy_queue(struct amdgpu_mes *mes,
 		mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset;
 	}
 
-	if (mes->adev->enable_uni_mes)
-		pipe = AMDGPU_MES_KIQ_PIPE;
-	else
-		pipe = AMDGPU_MES_SCHED_PIPE;
+	pipe = AMDGPU_MES_KIQ_PIPE;
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes,
 			input->xcc_id, pipe,
@@ -945,11 +927,11 @@ static int mes_v12_1_inv_tlbs_pasid(struct amdgpu_mes *mes,
 {
 	union MESAPI__INV_TLBS mes_inv_tlbs;
 	int xcc_id = input->xcc_id;
-	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
 	int ret;
 
+	/* See mes_v12_1_add_hw_queue() for the indexing rationale. */
 	if (mes->enable_coop_mode)
-		xcc_id = mes->master_xcc_ids[inst];
+		xcc_id = mes->master_xcc_ids[xcc_id];
 
 	memset(&mes_inv_tlbs, 0, sizeof(mes_inv_tlbs));
 
@@ -966,7 +948,7 @@ static int mes_v12_1_inv_tlbs_pasid(struct amdgpu_mes *mes,
 	if (ret < 0)
 		return -EINVAL;
 	mes_inv_tlbs.invalidate_tlbs.hub_id = ret;
-	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, AMDGPU_MES_KIQ_PIPE,
+	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, AMDGPU_MES_SCHED_PIPE,
 			&mes_inv_tlbs, sizeof(mes_inv_tlbs),
 			offsetof(union MESAPI__INV_TLBS, api_status));
 
@@ -1105,10 +1087,8 @@ static void mes_v12_1_enable(struct amdgpu_device *adev,
 
 		if (amdgpu_emu_mode)
 			msleep(500);
-		else if (adev->enable_uni_mes)
-			udelay(500);
 		else
-			udelay(50);
+			udelay(500);
 	} else {
 		data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_CNTL);
 		data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
@@ -1322,6 +1302,7 @@ static int mes_v12_1_mqd_init(struct amdgpu_ring *ring)
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
 			    ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, SCOPE, 3);
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
@@ -1431,31 +1412,6 @@ static void mes_v12_1_queue_init_register(struct amdgpu_ring *ring,
 	mutex_unlock(&adev->srbm_mutex);
 }
 
-static int mes_v12_1_kiq_enable_queue(struct amdgpu_device *adev, int xcc_id)
-{
-	struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_id];
-	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[xcc_id].ring;
-	int r, inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
-
-	if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
-		return -EINVAL;
-
-	r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
-	if (r) {
-		DRM_ERROR("Failed to lock KIQ (%d).\n", r);
-		return r;
-	}
-
-	kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring[inst]);
-
-	r = amdgpu_ring_test_ring(kiq_ring);
-	if (r) {
-		DRM_ERROR("kfq enable failed\n");
-		kiq_ring->sched.ready = false;
-	}
-	return r;
-}
-
 static int mes_v12_1_queue_init(struct amdgpu_device *adev,
 				  enum amdgpu_mes_pipe pipe,
 				  int xcc_id)
@@ -1463,13 +1419,9 @@ static int mes_v12_1_queue_init(struct amdgpu_device *adev,
 	struct amdgpu_ring *ring;
 	int r;
 
-	if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
-		ring = &adev->gfx.kiq[xcc_id].ring;
-	else
-		ring = &adev->mes.ring[MES_PIPE_INST(xcc_id, pipe)];
+	ring = &adev->mes.ring[MES_PIPE_INST(xcc_id, pipe)];
 
-	if ((adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) &&
-	    (amdgpu_in_reset(adev) || adev->in_suspend)) {
+	if (amdgpu_in_reset(adev) || adev->in_suspend) {
 		*(ring->wptr_cpu_addr) = 0;
 		*(ring->rptr_cpu_addr) = 0;
 		amdgpu_ring_clear_ring(ring);
@@ -1480,10 +1432,7 @@ static int mes_v12_1_queue_init(struct amdgpu_device *adev,
 		return r;
 
 	if (pipe == AMDGPU_MES_SCHED_PIPE) {
-		if (adev->enable_uni_mes)
-			r = amdgpu_mes_map_legacy_queue(adev, ring, xcc_id);
-		else
-			r = mes_v12_1_kiq_enable_queue(adev, xcc_id);
+		r = amdgpu_mes_map_legacy_queue(adev, ring, xcc_id);
 		if (r)
 			return r;
 	} else {
@@ -1532,39 +1481,6 @@ static int mes_v12_1_ring_init(struct amdgpu_device *adev,
 				AMDGPU_RING_PRIO_DEFAULT, NULL);
 }
 
-static int mes_v12_1_kiq_ring_init(struct amdgpu_device *adev, int xcc_id)
-{
-	struct amdgpu_ring *ring;
-	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_KIQ_PIPE);
-
-	spin_lock_init(&adev->gfx.kiq[xcc_id].ring_lock);
-
-	ring = &adev->gfx.kiq[xcc_id].ring;
-
-	ring->me = 3;
-	ring->pipe = 1;
-	ring->queue = 0;
-	ring->xcc_id = xcc_id;
-	ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
-
-	ring->adev = NULL;
-	ring->ring_obj = NULL;
-	ring->use_doorbell = true;
-	ring->eop_gpu_addr = adev->mes.eop_gpu_addr[inst];
-	ring->no_scheduler = true;
-	ring->doorbell_index =
-		(adev->doorbell_index.mes_ring1 +
-		 xcc_id * adev->doorbell_index.xcc_doorbell_range)
-		<< 1;
-
-	snprintf(ring->name, sizeof(ring->name), "mes_kiq_%hhu.%hhu.%hhu.%hhu",
-		 (unsigned char)xcc_id, (unsigned char)ring->me,
-		 (unsigned char)ring->pipe, (unsigned char)ring->queue);
-
-	return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
-				AMDGPU_RING_PRIO_DEFAULT, NULL);
-}
-
 static int mes_v12_1_mqd_sw_init(struct amdgpu_device *adev,
 				   enum amdgpu_mes_pipe pipe,
 				   int xcc_id)
@@ -1573,10 +1489,7 @@ static int mes_v12_1_mqd_sw_init(struct amdgpu_device *adev,
 	struct amdgpu_ring *ring;
 	int inst = MES_PIPE_INST(xcc_id, pipe);
 
-	if (!adev->enable_uni_mes && pipe == AMDGPU_MES_KIQ_PIPE)
-		ring = &adev->gfx.kiq[xcc_id].ring;
-	else
-		ring = &adev->mes.ring[inst];
+	ring = &adev->mes.ring[inst];
 
 	if (ring->mqd_obj)
 		return 0;
@@ -1611,8 +1524,8 @@ static int mes_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 	adev->mes.kiq_hw_fini = &mes_v12_1_kiq_hw_fini;
 	adev->mes.enable_legacy_queue_map = true;
 
-	adev->mes.event_log_size =
-		adev->enable_uni_mes ? (AMDGPU_MAX_MES_PIPES * AMDGPU_MES_LOG_BUFFER_SIZE * num_xcc) : AMDGPU_MES_LOG_BUFFER_SIZE;
+	adev->mes.event_log_size = AMDGPU_MAX_MES_PIPES *
+				   AMDGPU_MES_LOG_BUFFER_SIZE * num_xcc;
 
 	r = amdgpu_mes_init(adev);
 	if (r)
@@ -1628,15 +1541,11 @@ static int mes_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 			if (r)
 				return r;
 
-			if (!adev->enable_uni_mes && pipe ==
-			    AMDGPU_MES_KIQ_PIPE)
-				r = mes_v12_1_kiq_ring_init(adev, xcc_id);
-			else
-				r = mes_v12_1_ring_init(adev, xcc_id, pipe);
+			r = mes_v12_1_ring_init(adev, xcc_id, pipe);
 			if (r)
 				return r;
 
-			if (adev->enable_uni_mes && num_xcc > 1) {
+			if (num_xcc > 1) {
 				r = mes_v12_1_allocate_shared_cmd_buf(adev,
 							      pipe, xcc_id);
 				if (r)
@@ -1667,12 +1576,10 @@ static int mes_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
 					      &adev->mes.eop_gpu_addr[inst],
 					      NULL);
 
-			if (adev->enable_uni_mes || pipe == AMDGPU_MES_SCHED_PIPE) {
-				amdgpu_bo_free_kernel(&adev->mes.ring[inst].mqd_obj,
-						      &adev->mes.ring[inst].mqd_gpu_addr,
-						      &adev->mes.ring[inst].mqd_ptr);
-				amdgpu_ring_fini(&adev->mes.ring[inst]);
-			}
+			amdgpu_bo_free_kernel(&adev->mes.ring[inst].mqd_obj,
+					      &adev->mes.ring[inst].mqd_gpu_addr,
+					      &adev->mes.ring[inst].mqd_ptr);
+			amdgpu_ring_fini(&adev->mes.ring[inst]);
 		}
 	}
 
@@ -1680,13 +1587,6 @@ static int mes_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
 		amdgpu_ucode_release(&adev->mes.fw[pipe]);
 
 	for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) {
-		if (!adev->enable_uni_mes) {
-			amdgpu_bo_free_kernel(&adev->gfx.kiq[xcc_id].ring.mqd_obj,
-				      &adev->gfx.kiq[xcc_id].ring.mqd_gpu_addr,
-				      &adev->gfx.kiq[xcc_id].ring.mqd_ptr);
-			amdgpu_ring_fini(&adev->gfx.kiq[xcc_id].ring);
-		}
-
 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 			mes_v12_1_free_ucode_buffers(adev,
 				       AMDGPU_MES_KIQ_PIPE, xcc_id);
@@ -1699,14 +1599,12 @@ static int mes_v12_1_sw_fini(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static void mes_v12_1_kiq_dequeue_sched(struct amdgpu_device *adev,
-					  int xcc_id)
+static void mes_v12_1_kiq_dequeue(struct amdgpu_device *adev, int xcc_id)
 {
-	uint32_t data;
 	int i;
 
 	mutex_lock(&adev->srbm_mutex);
-	soc_v1_0_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0,
+	soc_v1_0_grbm_select(adev, 3, AMDGPU_MES_KIQ_PIPE, 0, 0,
 			     GET_INST(GC, xcc_id));
 
 	/* disable the queue if it's active */
@@ -1717,24 +1615,16 @@ static void mes_v12_1_kiq_dequeue_sched(struct amdgpu_device *adev,
 				break;
 			udelay(1);
 		}
+		WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 0);
 	}
-	data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL);
-	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
-				DOORBELL_EN, 0);
-	data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
-				DOORBELL_HIT, 1);
-	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, data);
 
 	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 0);
-
 	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO, 0);
 	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI, 0);
 	WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR, 0);
 
 	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
 	mutex_unlock(&adev->srbm_mutex);
-
-	adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready = false;
 }
 
 static void mes_v12_1_kiq_setting(struct amdgpu_ring *ring, int xcc_id)
@@ -1757,10 +1647,7 @@ static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
 	int r = 0;
 	struct amdgpu_ip_block *ip_block;
 
-	if (adev->enable_uni_mes)
-		mes_v12_1_kiq_setting(&adev->mes.ring[inst], xcc_id);
-	else
-		mes_v12_1_kiq_setting(&adev->gfx.kiq[xcc_id].ring, xcc_id);
+	mes_v12_1_kiq_setting(&adev->mes.ring[inst], xcc_id);
 
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
 
@@ -1795,19 +1682,17 @@ static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
 	if (r)
 		goto failure;
 
-	if (adev->enable_uni_mes) {
-		r = mes_v12_1_setup_coop_mode(adev, xcc_id);
-		if (r)
-			goto failure;
+	r = mes_v12_1_setup_coop_mode(adev, xcc_id);
+	if (r)
+		goto failure;
 
-		r = mes_v12_1_set_hw_resources(&adev->mes,
-						 AMDGPU_MES_KIQ_PIPE, xcc_id);
-		if (r)
-			goto failure;
+	r = mes_v12_1_set_hw_resources(&adev->mes,
+				       AMDGPU_MES_KIQ_PIPE, xcc_id);
+	if (r)
+		goto failure;
 
-		mes_v12_1_set_hw_resources_1(&adev->mes,
-					       AMDGPU_MES_KIQ_PIPE, xcc_id);
-	}
+	mes_v12_1_set_hw_resources_1(&adev->mes,
+				     AMDGPU_MES_KIQ_PIPE, xcc_id);
 
 	if (adev->mes.enable_legacy_queue_map) {
 		r = mes_v12_1_xcc_hw_init(ip_block, xcc_id);
@@ -1827,16 +1712,16 @@ static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id)
 	int inst = MES_PIPE_INST(xcc_id, AMDGPU_MES_SCHED_PIPE);
 
 	if (adev->mes.ring[inst].sched.ready) {
-		if (adev->enable_uni_mes)
-			amdgpu_mes_unmap_legacy_queue(adev,
-				      &adev->mes.ring[inst],
-				      RESET_QUEUES, 0, 0, xcc_id);
-		else
-			mes_v12_1_kiq_dequeue_sched(adev, xcc_id);
+		amdgpu_mes_unmap_legacy_queue(adev, &adev->mes.ring[inst],
+					      RESET_QUEUES, 0, 0, xcc_id);
 
 		adev->mes.ring[inst].sched.ready = false;
 	}
 
+	/* dequeue KIQ HQD on unbind to clear stale rptr/wptr on rebind */
+	if (!amdgpu_in_reset(adev) && !adev->in_suspend)
+		mes_v12_1_kiq_dequeue(adev, xcc_id);
+
 	mes_v12_1_enable(adev, false, xcc_id);
 
 	return 0;
@@ -1889,26 +1774,6 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id)
 	if (adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready)
 		goto out;
 
-	if (!adev->enable_mes_kiq) {
-		if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
-			r = mes_v12_1_load_microcode(adev,
-				       AMDGPU_MES_SCHED_PIPE, true, xcc_id);
-			if (r) {
-				DRM_ERROR("failed to MES fw, r=%d\n", r);
-				return r;
-			}
-
-			mes_v12_1_set_ucode_start_addr(adev, xcc_id);
-
-		} else if (adev->firmware.load_type ==
-			   AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
-
-			mes_v12_1_set_ucode_start_addr(adev, xcc_id);
-		}
-
-		mes_v12_1_enable(adev, true, xcc_id);
-	}
-
 	/* Enable the MES to handle doorbell ring on unmapped queue */
 	mes_v12_1_enable_unmapped_doorbell_handling(&adev->mes, true, xcc_id);
 
@@ -1921,10 +1786,7 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id)
 	if (r)
 		goto failure;
 
-	if (adev->enable_uni_mes) {
-		mes_v12_1_set_hw_resources_1(&adev->mes,
-					       AMDGPU_MES_SCHED_PIPE, xcc_id);
-	}
+	mes_v12_1_set_hw_resources_1(&adev->mes, AMDGPU_MES_SCHED_PIPE, xcc_id);
 	mes_v12_1_init_aggregated_doorbell(&adev->mes, xcc_id);
 
 	r = mes_v12_1_query_sched_status(&adev->mes,
@@ -1936,12 +1798,6 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id)
 
 	amdgpu_mes_validate_fw_version(adev);
 out:
-	/*
-	 * Disable KIQ ring usage from the driver once MES is enabled.
-	 * MES uses KIQ ring exclusively so driver cannot access KIQ ring
-	 * with MES enabled.
-	 */
-	adev->gfx.kiq[xcc_id].ring.sched.ready = false;
 	adev->mes.ring[MES_PIPE_INST(xcc_id, 0)].sched.ready = true;
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
index 5827c758b373d6..4495d52b4b07ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v4_2_0.c
@@ -29,10 +29,57 @@
 
 #include "soc15_common.h"
 #include "soc24_enum.h"
+#include "soc_v1_0.h"
 
 #define regMMVM_L2_CNTL3_DEFAULT				0x80100007
 #define regMMVM_L2_CNTL4_DEFAULT				0x000000c1
 #define regMMVM_L2_CNTL5_DEFAULT				0x00003fe0
+#define MMHUB_V4_2_0_MID1_EXT_ID				0x3
+
+static u64 mmhub_v4_2_0_get_reg_addr(int inst, u32 reg_offset)
+{
+	int ext_id = inst ? MMHUB_V4_2_0_MID1_EXT_ID : 0;
+
+	return (u64)reg_offset * 4 +
+	       soc_v1_0_encode_ext_smn_addressing(ext_id);
+}
+
+static u32 mmhub_v4_2_0_rreg(struct amdgpu_device *adev, int inst,
+			     u32 reg_offset)
+{
+	u64 reg_addr;
+
+	reg_addr = mmhub_v4_2_0_get_reg_addr(inst, reg_offset);
+
+	return RREG32_PCIE_EXT(reg_addr);
+}
+
+static void mmhub_v4_2_0_wreg(struct amdgpu_device *adev, int inst,
+			      u32 reg_offset, u32 value)
+{
+	u64 reg_addr;
+
+	reg_addr = mmhub_v4_2_0_get_reg_addr(inst, reg_offset);
+
+	WREG32_PCIE_EXT(reg_addr, value);
+}
+
+#define RREG32_MMHUB(inst, reg) \
+	mmhub_v4_2_0_rreg(adev, inst, \
+		(adev->reg_offset[MMHUB_HWIP][GET_INST(MMHUB, 0)] \
+				 [reg##_BASE_IDX] + (reg)))
+#define RREG32_MMHUB_OFFSET(inst, reg, offset) \
+	mmhub_v4_2_0_rreg(adev, inst, \
+		(adev->reg_offset[MMHUB_HWIP][GET_INST(MMHUB, 0)] \
+				 [reg##_BASE_IDX] + (reg) + (offset)))
+#define WREG32_MMHUB(inst, reg, value) \
+	mmhub_v4_2_0_wreg(adev, inst, \
+		(adev->reg_offset[MMHUB_HWIP][GET_INST(MMHUB, 0)] \
+				 [reg##_BASE_IDX] + (reg)), value)
+#define WREG32_MMHUB_OFFSET(inst, reg, offset, value) \
+	mmhub_v4_2_0_wreg(adev, inst, \
+		(adev->reg_offset[MMHUB_HWIP][GET_INST(MMHUB, 0)] \
+				 [reg##_BASE_IDX] + (reg) + (offset)), value)
 
 static const char *mmhub_client_ids_v4_2_0[][2] = {
 	[0][0] = "VMC",
@@ -137,22 +184,22 @@ static u64 mmhub_v4_2_0_get_mc_fb_offset(struct amdgpu_device *adev)
 static void mmhub_v4_2_0_mid_setup_vm_pt_regs(struct amdgpu_device *adev,
 					      uint32_t vmid,
 					      uint64_t page_table_base,
-					      uint32_t mid_mask)
+					      uint32_t inst_mask)
 {
 	struct amdgpu_vmhub *hub;
 	int i;
 
-	for_each_inst(i, mid_mask) {
+	for_each_inst(i, inst_mask) {
 		hub = &adev->vmhub[AMDGPU_MMHUB0(i)];
-		WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, i),
-				    regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
-				    hub->ctx_addr_distance * vmid,
-				    lower_32_bits(page_table_base));
-
-		WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, i),
-				    regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
-				    hub->ctx_addr_distance * vmid,
-				    upper_32_bits(page_table_base));
+		WREG32_MMHUB_OFFSET(i,
+				   regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+				   hub->ctx_addr_distance * vmid,
+				   lower_32_bits(page_table_base));
+
+		WREG32_MMHUB_OFFSET(i,
+				   regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+				   hub->ctx_addr_distance * vmid,
+				   upper_32_bits(page_table_base));
 	}
 }
 
@@ -160,16 +207,13 @@ static void mmhub_v4_2_0_setup_vm_pt_regs(struct amdgpu_device *adev,
 					  uint32_t vmid,
 					  uint64_t page_table_base)
 {
-	uint32_t mid_mask;
-
-	mid_mask = adev->aid_mask;
 	mmhub_v4_2_0_mid_setup_vm_pt_regs(adev, vmid,
 					  page_table_base,
-					  mid_mask);
+					  adev->mmhub.inst_mask);
 }
 
 static void mmhub_v4_2_0_mid_init_gart_aperture_regs(struct amdgpu_device *adev,
-						     uint32_t mid_mask)
+						     uint32_t inst_mask)
 {
 	uint64_t pt_base;
 	int i;
@@ -179,35 +223,35 @@ static void mmhub_v4_2_0_mid_init_gart_aperture_regs(struct amdgpu_device *adev,
 	else
 		pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
 
-	mmhub_v4_2_0_mid_setup_vm_pt_regs(adev, 0, pt_base, mid_mask);
+	mmhub_v4_2_0_mid_setup_vm_pt_regs(adev, 0, pt_base, inst_mask);
 
-	for_each_inst(i, mid_mask) {
+	for_each_inst(i, inst_mask) {
 		if (adev->gmc.pdb0_bo) {
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
 				     (u32)(adev->gmc.fb_start >> 12));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
 				     (u32)(adev->gmc.fb_start >> 44));
 
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
 				     (u32)(adev->gmc.gart_end >> 12));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
 				     (u32)(adev->gmc.gart_end >> 44));
 		} else {
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
 				     (u32)(adev->gmc.gart_start >> 12));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
 				     (u32)(adev->gmc.gart_start >> 44));
 
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
 				     (u32)(adev->gmc.gart_end >> 12));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
 				     (u32)(adev->gmc.gart_end >> 44));
 		}
@@ -215,7 +259,7 @@ static void mmhub_v4_2_0_mid_init_gart_aperture_regs(struct amdgpu_device *adev,
 }
 
 static void mmhub_v4_2_0_mid_init_system_aperture_regs(struct amdgpu_device *adev,
-						       uint32_t mid_mask)
+						       uint32_t inst_mask)
 {
 	uint64_t value;
 	uint32_t tmp;
@@ -229,71 +273,71 @@ static void mmhub_v4_2_0_mid_init_system_aperture_regs(struct amdgpu_device *ade
 	if (amdgpu_sriov_vf(adev))
 		return;
 
-	for_each_inst(i, mid_mask) {
+	for_each_inst(i, inst_mask) {
 		if (adev->gmc.pdb0_bo) {
 			/* Disable agp and system aperture
 			 * when vmid0 page table is enabled */
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_FB_LOCATION_TOP_LO32, 0);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_FB_LOCATION_TOP_HI32, 0);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_FB_LOCATION_BASE_LO32,
 				     0xFFFFFFFF);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_FB_LOCATION_BASE_HI32, 1);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_TOP_LO32, 0);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_TOP_HI32, 0);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_BOT_LO32,
 				     0xFFFFFFFF);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_BOT_HI32, 1);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
 				     0xFFFFFFFF);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
 				     0x7F);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0);
 		} else {
 			/* Program the AGP BAR */
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_BASE_LO32, 0);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_BASE_HI32, 0);
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_BOT_LO32,
 				     lower_32_bits(adev->gmc.agp_start >> 24));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_BOT_HI32,
 				     upper_32_bits(adev->gmc.agp_start >> 24));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_TOP_LO32,
 				     lower_32_bits(adev->gmc.agp_end >> 24));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_AGP_TOP_HI32,
 				     upper_32_bits(adev->gmc.agp_end >> 24));
 
 			/* Program the system aperture low logical page number. */
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32,
 				     lower_32_bits(min(adev->gmc.fb_start,
 						   adev->gmc.agp_start) >> 18));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32,
 				     upper_32_bits(min(adev->gmc.fb_start,
 						   adev->gmc.agp_start) >> 18));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32,
 				     lower_32_bits(max(adev->gmc.fb_end,
 						   adev->gmc.agp_end) >> 18));
-			WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+			WREG32_MMHUB(i,
 				     regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32,
 				     upper_32_bits(max(adev->gmc.fb_end,
 						   adev->gmc.agp_end) >> 18));
@@ -301,42 +345,40 @@ static void mmhub_v4_2_0_mid_init_system_aperture_regs(struct amdgpu_device *ade
 
 		/* Set default page address. */
 		value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
 			     (u32)(value >> 12));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
 			     (u32)(value >> 44));
 
 		/* Program "protection fault". */
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
 			     (u32)(adev->dummy_page_addr >> 12));
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 			     (u32)((u64)adev->dummy_page_addr >> 44));
 
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-				   regMMVM_L2_PROTECTION_FAULT_CNTL2);
+		tmp = RREG32_MMHUB(i, regMMVM_L2_PROTECTION_FAULT_CNTL2);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2,
 				    ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2,
 				    ENABLE_RETRY_FAULT_INTERRUPT, 0x1);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 	}
 }
 
 static void mmhub_v4_2_0_mid_init_tlb_regs(struct amdgpu_device *adev,
-					   uint32_t mid_mask)
+					   uint32_t inst_mask)
 {
 	uint32_t tmp;
 	int i;
 
-	for_each_inst(i, mid_mask) {
+	for_each_inst(i, inst_mask) {
 		/* Setup TLB control */
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-				   regMMMC_VM_MX_L1_TLB_CNTL);
+		tmp = RREG32_MMHUB(i, regMMMC_VM_MX_L1_TLB_CNTL);
 
 		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
 		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
@@ -344,17 +386,16 @@ static void mmhub_v4_2_0_mid_init_tlb_regs(struct amdgpu_device *adev,
 				    ENABLE_ADVANCED_DRIVER_MODEL, 1);
 		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
 				    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
-		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
 		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
 				    MTYPE, MTYPE_UC); /* UC, uncached */
 
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMMC_VM_MX_L1_TLB_CNTL, tmp);
 	}
 }
 
 static void mmhub_v4_2_0_mid_init_cache_regs(struct amdgpu_device *adev,
-					     uint32_t mid_mask)
+					     uint32_t inst_mask)
 {
 	uint32_t tmp;
 	int i;
@@ -365,11 +406,10 @@ static void mmhub_v4_2_0_mid_init_cache_regs(struct amdgpu_device *adev,
 	if (amdgpu_sriov_vf(adev))
 		return;
 
-	for_each_inst(i, mid_mask) {
+	for_each_inst(i, inst_mask) {
 		/* Setup L2 cache */
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL);
+		tmp = RREG32_MMHUB(i, regMMVM_L2_CNTL);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1);
-		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
 				    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
 		/* XXX for emulation, Refer to closed source code.*/
@@ -381,14 +421,7 @@ static void mmhub_v4_2_0_mid_init_cache_regs(struct amdgpu_device *adev,
 				    CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL,
 				    IDENTITY_MODE_FRAGMENT_SIZE, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL, tmp);
-
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL2);
-		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2,
-				    INVALIDATE_ALL_L1_TLBS, 1);
-		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2,
-				    INVALIDATE_L2_CACHE, 1);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL2, tmp);
+		WREG32_MMHUB(i, regMMVM_L2_CNTL, tmp);
 
 		tmp = regMMVM_L2_CNTL3_DEFAULT;
 		if (adev->gmc.translate_further) {
@@ -400,7 +433,7 @@ static void mmhub_v4_2_0_mid_init_cache_regs(struct amdgpu_device *adev,
 			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3,
 					    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
 		}
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL3, tmp);
+		WREG32_MMHUB(i, regMMVM_L2_CNTL3, tmp);
 
 		tmp = regMMVM_L2_CNTL4_DEFAULT;
 		/* For AMD APP APUs setup WC memory */
@@ -415,24 +448,23 @@ static void mmhub_v4_2_0_mid_init_cache_regs(struct amdgpu_device *adev,
 			tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4,
 					    VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
 		}
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL4, tmp);
+		WREG32_MMHUB(i, regMMVM_L2_CNTL4, tmp);
 
 		tmp = regMMVM_L2_CNTL5_DEFAULT;
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL5,
 				    L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i), regMMVM_L2_CNTL5, tmp);
+		WREG32_MMHUB(i, regMMVM_L2_CNTL5, tmp);
 	}
 }
 
 static void mmhub_v4_2_0_mid_enable_system_domain(struct amdgpu_device *adev,
-						  uint32_t mid_mask)
+						  uint32_t inst_mask)
 {
 	uint32_t tmp;
 	int i;
 
-	for_each_inst(i, mid_mask) {
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-				   regMMVM_CONTEXT0_CNTL);
+	for_each_inst(i, inst_mask) {
+		tmp = RREG32_MMHUB(i, regMMVM_CONTEXT0_CNTL);
 		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
 				    ENABLE_CONTEXT, 1);
 		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
@@ -442,13 +474,13 @@ static void mmhub_v4_2_0_mid_enable_system_domain(struct amdgpu_device *adev,
 				    adev->gmc.vmid0_page_table_block_size);
 		tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL,
 				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_CONTEXT0_CNTL, tmp);
 	}
 }
 
 static void mmhub_v4_2_0_mid_disable_identity_aperture(struct amdgpu_device *adev,
-						       uint32_t mid_mask)
+						       uint32_t inst_mask)
 {
 	int i;
 
@@ -458,46 +490,53 @@ static void mmhub_v4_2_0_mid_disable_identity_aperture(struct amdgpu_device *ade
 	if (amdgpu_sriov_vf(adev))
 		return;
 
-	for_each_inst(i, mid_mask) {
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+	for_each_inst(i, inst_mask) {
+		WREG32_MMHUB(i,
 			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
 			     0xFFFFFFFF);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
 			     0x00001FFF);
 
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
 			     0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
 			     0);
 
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
 			     0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
+		WREG32_MMHUB(i,
 			     regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
 			     0);
 	}
 }
 
 static void mmhub_v4_2_0_mid_setup_vmid_config(struct amdgpu_device *adev,
-					       uint32_t mid_mask)
+					       uint32_t inst_mask)
 {
 	struct amdgpu_vmhub *hub;
+	u32 num_level, block_size;
 	uint32_t tmp;
 	int i, j;
 
-	for_each_inst(j, mid_mask) {
+	num_level = adev->vm_manager.num_level;
+	block_size = adev->vm_manager.block_size;
+	if (adev->gmc.translate_further)
+		num_level -= 1;
+	else
+		block_size -= 9;
+
+	for_each_inst(j, inst_mask) {
 		hub = &adev->vmhub[AMDGPU_MMHUB0(j)];
 		for (i = 0; i <= 14; i++) {
-			tmp = RREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
-					          regMMVM_CONTEXT1_CNTL,
-						  i * hub->ctx_distance);
+			tmp = RREG32_MMHUB_OFFSET(j, regMMVM_CONTEXT1_CNTL,
+						 i * hub->ctx_distance);
 			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
 			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
-					    adev->vm_manager.num_level);
+					    num_level);
 			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
 					    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
@@ -514,24 +553,27 @@ static void mmhub_v4_2_0_mid_setup_vmid_config(struct amdgpu_device *adev,
 			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
 					    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
-					    PAGE_TABLE_BLOCK_SIZE,
-					    adev->vm_manager.block_size - 9);
+					    PAGE_TABLE_BLOCK_SIZE, block_size);
 			/* Send no-retry XNACK on fault to suppress VM fault storm. */
 			tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL,
 					    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
 					    !adev->gmc.noretry);
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_CNTL,
-					    i * hub->ctx_distance, tmp);
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
-					    i * hub->ctx_addr_distance, 0);
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
-					    i * hub->ctx_addr_distance, 0);
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
-					    i * hub->ctx_addr_distance,
-					    lower_32_bits(adev->vm_manager.max_pfn - 1));
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j), regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
-					    i * hub->ctx_addr_distance,
-					    upper_32_bits(adev->vm_manager.max_pfn - 1));
+			WREG32_MMHUB_OFFSET(j, regMMVM_CONTEXT1_CNTL,
+					   i * hub->ctx_distance, tmp);
+			WREG32_MMHUB_OFFSET(j,
+					   regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
+					   i * hub->ctx_addr_distance, 0);
+			WREG32_MMHUB_OFFSET(j,
+					   regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
+					   i * hub->ctx_addr_distance, 0);
+			WREG32_MMHUB_OFFSET(j,
+					   regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
+					   i * hub->ctx_addr_distance,
+					   lower_32_bits(adev->vm_manager.max_pfn - 1));
+			WREG32_MMHUB_OFFSET(j,
+					   regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
+					   i * hub->ctx_addr_distance,
+					   upper_32_bits(adev->vm_manager.max_pfn - 1));
 		}
 	}
 
@@ -539,93 +581,84 @@ static void mmhub_v4_2_0_mid_setup_vmid_config(struct amdgpu_device *adev,
 }
 
 static void mmhub_v4_2_0_mid_program_invalidation(struct amdgpu_device *adev,
-						  uint32_t mid_mask)
+						  uint32_t inst_mask)
 {
 	struct amdgpu_vmhub *hub;
 	unsigned int i, j;
 
-	for_each_inst(j, mid_mask) {
+	for_each_inst(j, inst_mask) {
 		hub = &adev->vmhub[AMDGPU_MMHUB0(j)];
 
 		for (i = 0; i < 18; ++i) {
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
-					    regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
-					    i * hub->eng_addr_distance, 0xffffffff);
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
-					    regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
-					    i * hub->eng_addr_distance, 0x3fff);
+			WREG32_MMHUB_OFFSET(j,
+					   regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+					   i * hub->eng_addr_distance, 0xffffffff);
+			WREG32_MMHUB_OFFSET(j,
+					   regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+					   i * hub->eng_addr_distance, 0x3fff);
 		}
 	}
 }
 
 static int mmhub_v4_2_0_mid_gart_enable(struct amdgpu_device *adev,
-					uint32_t mid_mask)
+					uint32_t inst_mask)
 {
 	/* GART Enable. */
-	mmhub_v4_2_0_mid_init_gart_aperture_regs(adev, mid_mask);
-	mmhub_v4_2_0_mid_init_system_aperture_regs(adev, mid_mask);
-	mmhub_v4_2_0_mid_init_tlb_regs(adev, mid_mask);
-	mmhub_v4_2_0_mid_init_cache_regs(adev, mid_mask);
+	mmhub_v4_2_0_mid_init_gart_aperture_regs(adev, inst_mask);
+	mmhub_v4_2_0_mid_init_system_aperture_regs(adev, inst_mask);
+	mmhub_v4_2_0_mid_init_tlb_regs(adev, inst_mask);
+	mmhub_v4_2_0_mid_init_cache_regs(adev, inst_mask);
 
-	mmhub_v4_2_0_mid_enable_system_domain(adev, mid_mask);
-	mmhub_v4_2_0_mid_disable_identity_aperture(adev, mid_mask);
-	mmhub_v4_2_0_mid_setup_vmid_config(adev, mid_mask);
-	mmhub_v4_2_0_mid_program_invalidation(adev, mid_mask);
+	mmhub_v4_2_0_mid_enable_system_domain(adev, inst_mask);
+	mmhub_v4_2_0_mid_disable_identity_aperture(adev, inst_mask);
+	mmhub_v4_2_0_mid_setup_vmid_config(adev, inst_mask);
+	mmhub_v4_2_0_mid_program_invalidation(adev, inst_mask);
 
 	return 0;
 }
 static int mmhub_v4_2_0_gart_enable(struct amdgpu_device *adev)
 {
-	uint32_t mid_mask;
-
-	mid_mask = adev->aid_mask;
-	return mmhub_v4_2_0_mid_gart_enable(adev, mid_mask);
+	return mmhub_v4_2_0_mid_gart_enable(adev, adev->mmhub.inst_mask);
 }
 
 static void mmhub_v4_2_0_mid_gart_disable(struct amdgpu_device *adev,
-					  uint32_t mid_mask)
+					  uint32_t inst_mask)
 {
 	struct amdgpu_vmhub *hub;
 	u32 tmp;
 	u32 i, j;
 
-	for_each_inst(j, mid_mask) {
+	for_each_inst(j, inst_mask) {
 		hub = &adev->vmhub[AMDGPU_MMHUB0(j)];
 		/* Disable all tables */
 		for (i = 0; i < 16; i++)
-			WREG32_SOC15_OFFSET(MMHUB, GET_INST(MMHUB, j),
-					    regMMVM_CONTEXT0_CNTL,
-					    i * hub->ctx_distance, 0);
+			WREG32_MMHUB_OFFSET(j, regMMVM_CONTEXT0_CNTL,
+					   i * hub->ctx_distance, 0);
 
 		/* Setup TLB control */
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, j),
-				   regMMMC_VM_MX_L1_TLB_CNTL);
+		tmp = RREG32_MMHUB(j, regMMMC_VM_MX_L1_TLB_CNTL);
 		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
 				    ENABLE_L1_TLB, 0);
 		tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL,
 				    ENABLE_ADVANCED_DRIVER_MODEL, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, j),
-			     regMMMC_VM_MX_L1_TLB_CNTL, tmp);
+		WREG32_MMHUB(j, regMMMC_VM_MX_L1_TLB_CNTL, tmp);
 
 		/* Setup L2 cache */
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, j), regMMVM_L2_CNTL);
+		tmp = RREG32_MMHUB(j, regMMVM_L2_CNTL);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 0);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, j), regMMVM_L2_CNTL, tmp);
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, j), regMMVM_L2_CNTL3, 0);
+		WREG32_MMHUB(j, regMMVM_L2_CNTL, tmp);
+		WREG32_MMHUB(j, regMMVM_L2_CNTL3, 0);
 	}
 }
 
 static void mmhub_v4_2_0_gart_disable(struct amdgpu_device *adev)
 {
-	uint32_t mid_mask;
-
-	mid_mask = adev->aid_mask;
-	mmhub_v4_2_0_mid_gart_disable(adev, mid_mask);
+	mmhub_v4_2_0_mid_gart_disable(adev, adev->mmhub.inst_mask);
 }
 
 static void
 mmhub_v4_2_0_mid_set_fault_enable_default(struct amdgpu_device *adev,
-					  bool value, uint32_t mid_mask)
+					  bool value, uint32_t inst_mask)
 {
 	u32 tmp;
 	int i;
@@ -636,9 +669,8 @@ mmhub_v4_2_0_mid_set_fault_enable_default(struct amdgpu_device *adev,
 	if (amdgpu_sriov_vf(adev))
 		return;
 
-	for_each_inst(i, mid_mask) {
-		tmp = RREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-				   regMMVM_L2_PROTECTION_FAULT_CNTL_LO32);
+	for_each_inst(i, inst_mask) {
+		tmp = RREG32_MMHUB(i, regMMVM_L2_PROTECTION_FAULT_CNTL_LO32);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
 				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 		tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
@@ -666,8 +698,7 @@ mmhub_v4_2_0_mid_set_fault_enable_default(struct amdgpu_device *adev,
 			tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL_LO32,
 					    CRASH_ON_NO_RETRY_FAULT, 1);
 		}
-		WREG32_SOC15(MMHUB, GET_INST(MMHUB, i),
-			     regMMVM_L2_PROTECTION_FAULT_CNTL_LO32, tmp);
+		WREG32_MMHUB(i, regMMVM_L2_PROTECTION_FAULT_CNTL_LO32, tmp);
 	}
 }
 
@@ -682,10 +713,8 @@ static void
 mmhub_v4_2_0_set_fault_enable_default(struct amdgpu_device *adev,
 				      bool value)
 {
-	uint32_t mid_mask;
-
-	mid_mask = adev->aid_mask;
-	mmhub_v4_2_0_mid_set_fault_enable_default(adev, value, mid_mask);
+	mmhub_v4_2_0_mid_set_fault_enable_default(adev, value,
+						  adev->mmhub.inst_mask);
 }
 
 static uint32_t mmhub_v4_2_0_get_invalidate_req(unsigned int vmid,
@@ -710,8 +739,18 @@ static uint32_t mmhub_v4_2_0_get_invalidate_req(unsigned int vmid,
 	return req;
 }
 
-/*TODO: l2 protection fault status is increased to 64bits.
- * some critical fields like FED are moved to STATUS_HI32 */
+static void
+mmhub_v4_2_0_print_l2_protection_fault_status_hi(struct amdgpu_device *adev,
+						 uint32_t status)
+{
+	dev_err(adev->dev,
+		"MMVM_L2_PROTECTION_FAULT_STATUS_HI32:0x%08X\n",
+		status);
+	dev_err(adev->dev, "\t FED: 0x%lx\n",
+		REG_GET_FIELD(status,
+			      MMVM_L2_PROTECTION_FAULT_STATUS_HI32, FED));
+}
+
 static void
 mmhub_v4_2_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 					      uint32_t status)
@@ -748,16 +787,18 @@ mmhub_v4_2_0_print_l2_protection_fault_status(struct amdgpu_device *adev,
 
 static const struct amdgpu_vmhub_funcs mmhub_v4_2_0_vmhub_funcs = {
 	.print_l2_protection_fault_status = mmhub_v4_2_0_print_l2_protection_fault_status,
+	.print_l2_protection_fault_status_hi =
+		mmhub_v4_2_0_print_l2_protection_fault_status_hi,
 	.get_invalidate_req = mmhub_v4_2_0_get_invalidate_req,
 };
 
 static void mmhub_v4_2_0_mid_init(struct amdgpu_device *adev,
-				  uint32_t mid_mask)
+				  uint32_t inst_mask)
 {
 	struct amdgpu_vmhub *hub;
 	int i;
 
-	for_each_inst(i, mid_mask) {
+	for_each_inst(i, inst_mask) {
 		hub = &adev->vmhub[AMDGPU_MMHUB0(i)];
 
 		hub->ctx0_ptb_addr_lo32 =
@@ -778,10 +819,12 @@ static void mmhub_v4_2_0_mid_init(struct amdgpu_device *adev,
 		hub->vm_context0_cntl =
 			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
 					 regMMVM_CONTEXT0_CNTL);
-		/* TODO: add a new member to accomandate additional fault status/cntl reg */
 		hub->vm_l2_pro_fault_status =
 			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
 					 regMMVM_L2_PROTECTION_FAULT_STATUS_LO32);
+		hub->vm_l2_pro_fault_status_hi =
+			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
+					 regMMVM_L2_PROTECTION_FAULT_STATUS_HI32);
 		hub->vm_l2_pro_fault_cntl =
 			SOC15_REG_OFFSET(MMHUB, GET_INST(MMHUB, i),
 					 regMMVM_L2_PROTECTION_FAULT_CNTL_LO32);
@@ -814,10 +857,7 @@ static void mmhub_v4_2_0_mid_init(struct amdgpu_device *adev,
 
 static void mmhub_v4_2_0_init(struct amdgpu_device *adev)
 {
-	uint32_t mid_mask;
-
-	mid_mask = adev->aid_mask;
-	mmhub_v4_2_0_mid_init(adev, mid_mask);
+	mmhub_v4_2_0_mid_init(adev, adev->mmhub.inst_mask);
 
 	amdgpu_mmhub_init_client_info(&adev->mmhub,
 				     mmhub_client_ids_v4_2_0,
diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
index 4cd325149b63e5..1dae7df9ee1708 100644
--- a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c
@@ -417,6 +417,10 @@ static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,
 		 */
 		wptr = le32_to_cpu(*ih->wptr_cpu);
 
+		if (ih == &adev->irq.ih_soft)
+			goto out;
+
+
 		if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
 			goto out;
 	}
@@ -625,12 +629,6 @@ static int navi10_ih_resume(struct amdgpu_ip_block *ip_block)
 	return navi10_ih_hw_init(ip_block);
 }
 
-static bool navi10_ih_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-	return true;
-}
-
 static int navi10_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* todo */
@@ -699,7 +697,6 @@ static const struct amd_ip_funcs navi10_ih_ip_funcs = {
 	.hw_fini = navi10_ih_hw_fini,
 	.suspend = navi10_ih_suspend,
 	.resume = navi10_ih_resume,
-	.is_idle = navi10_ih_is_idle,
 	.wait_for_idle = navi10_ih_wait_for_idle,
 	.soft_reset = navi10_ih_soft_reset,
 	.set_clockgating_state = navi10_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
index 61eb0513dc97db..a74b17a47fd28c 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
@@ -57,56 +57,12 @@
 #define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10                                    0x0021
 #define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10_BASE_IDX                           2
 
-#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_nbio_7_11_5                  0x8e13
-#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_nbio_7_11_5_BASE_IDX         5
-#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_nbio_7_11_5                   0x8e14
-#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_nbio_7_11_5_BASE_IDX          5
-#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_nbio_7_11_5                       0x8e15
-#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_nbio_7_11_5_BASE_IDX              5
-
-#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL_nbio_7_11_5                                 0x012d
-#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL_nbio_7_11_5_BASE_IDX                        2
-#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL_nbio_7_11_5                                 0x012e
-#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL_nbio_7_11_5_BASE_IDX                        2
-
-#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0_nbio_7_11_5                                  0x0021
-#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0_nbio_7_11_5_BASE_IDX                         2
-
-#define regBIF_BX1_BIF_FB_EN_nbio_7_11_5                                                0x0100
-#define regBIF_BX1_BIF_FB_EN_nbio_7_11_5_BASE_IDX                                       2
-
-#define regBIF_BX1_INTERRUPT_CNTL_nbio_7_11_5                                           0x00f1
-#define regBIF_BX1_INTERRUPT_CNTL_nbio_7_11_5_BASE_IDX                                  2
-#define regBIF_BX1_INTERRUPT_CNTL2_nbio_7_11_5                                          0x00f2
-#define regBIF_BX1_INTERRUPT_CNTL2_nbio_7_11_5_BASE_IDX                                 2
-
-#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ_nbio_7_11_5                                     0x0106
-#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ_nbio_7_11_5_BASE_IDX                            2
-#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE_nbio_7_11_5                                    0x0107
-#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE_nbio_7_11_5_BASE_IDX                           2
-
-#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL_nbio_7_11_5                          0x00f7
-#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL_nbio_7_11_5_BASE_IDX                 2
-
-//BIF_BX1_BIF_FB_EN
-#define BIF_BX1_BIF_FB_EN__FB_READ_EN__SHIFT_nbio_7_11_5                                0x0
-#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN__SHIFT_nbio_7_11_5                               0x1
-#define BIF_BX1_BIF_FB_EN__FB_READ_EN_MASK_nbio_7_11_5                                  0x00000001L
-#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN_MASK_nbio_7_11_5                                 0x00000002L
-
 static void nbif_v6_3_1_remap_hdp_registers(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) {
-		WREG32_SOC15(NBIO, 0, regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL_nbio_7_11_5,
-			adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
-		WREG32_SOC15(NBIO, 0, regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL_nbio_7_11_5,
-			adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
-	} else {
-		WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL,
-			adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
-		WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL,
-			adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
-	}
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL,
+		adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL,
+		adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
 }
 
 static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev)
@@ -115,8 +71,6 @@ static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev)
 
 	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
 		tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10);
-	else if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5))
-		tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0_nbio_7_11_5);
 	else
 		tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0);
 
@@ -128,21 +82,12 @@ static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev)
 
 static void nbif_v6_3_1_mc_access_enable(struct amdgpu_device *adev, bool enable)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5)) {
-		if (enable)
-			WREG32_SOC15(NBIO, 0, regBIF_BX1_BIF_FB_EN_nbio_7_11_5,
-				     BIF_BX1_BIF_FB_EN__FB_READ_EN_MASK_nbio_7_11_5 |
-				     BIF_BX1_BIF_FB_EN__FB_WRITE_EN_MASK_nbio_7_11_5);
-		else
-			WREG32_SOC15(NBIO, 0, regBIF_BX1_BIF_FB_EN_nbio_7_11_5, 0);
-	} else {
-		if (enable)
-			WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN,
-				     BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK |
-				     BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK);
-		else
-			WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0);
-	}
+	if (enable)
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN,
+			     BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK |
+			     BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK);
+	else
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0);
 }
 
 static u32 nbif_v6_3_1_get_memsize(struct amdgpu_device *adev)
@@ -155,14 +100,8 @@ static void nbif_v6_3_1_sdma_doorbell_range(struct amdgpu_device *adev,
 					    int doorbell_index,
 					    int doorbell_size)
 {
-	u32 doorbell_range;
 	if (instance == 0) {
-		if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4))
-			doorbell_range = RREG32_SOC15(NBIO, 0,
-					regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10);
-		else
-			doorbell_range = RREG32_SOC15(NBIO, 0,
-					regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL);
+		u32 doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL);
 
 		if (use_doorbell) {
 			doorbell_range = REG_SET_FIELD(doorbell_range,
@@ -191,10 +130,12 @@ static void nbif_v6_3_1_sdma_doorbell_range(struct amdgpu_device *adev,
 						       S2A_DOORBELL_PORT2_RANGE_SIZE,
 						       0);
 
-		if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4))
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10, doorbell_range);
+		if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
+			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_nbif_4_10,
+				     doorbell_range);
 		else
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range);
+			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+				     doorbell_range);
 	}
 }
 
@@ -239,15 +180,15 @@ static void nbif_v6_3_1_vcn_doorbell_range(struct amdgpu_device *adev,
 					       S2A_DOORBELL_PORT4_RANGE_SIZE,
 					       0);
 
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4))
-		WREG32_SOC15(NBIO, 0,
-				regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10,
-				doorbell_range);
-	else
+	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) {
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10,
+			     doorbell_range);
+	} else {
 		if (instance)
 			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
 		else
 			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL, doorbell_range);
+	}
 }
 
 static void nbif_v6_3_1_vpe_doorbell_range(struct amdgpu_device *adev,
@@ -299,7 +240,7 @@ static void nbif_v6_3_1_vpe_doorbell_range(struct amdgpu_device *adev,
 
 static void nbif_v6_3_1_gc_doorbell_init(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4)) {
+	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) {
 		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_nbif_4_10, 0x30000007);
 		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_nbif_4_10, 0x3000000d);
 	} else {
@@ -341,13 +282,7 @@ nbif_v6_3_1_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
 static void nbif_v6_3_1_ih_doorbell_range(struct amdgpu_device *adev,
 					  bool use_doorbell, int doorbell_index)
 {
-	u32 ih_doorbell_range;
-
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4))
-		ih_doorbell_range = RREG32_SOC15(NBIO, 0,
-				regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10);
-	else
-		ih_doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL);
+	u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL);
 
 	if (use_doorbell) {
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
@@ -365,7 +300,7 @@ static void nbif_v6_3_1_ih_doorbell_range(struct amdgpu_device *adev,
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
 						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
 						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  2);
+						  8);
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
 						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
 						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
@@ -376,11 +311,12 @@ static void nbif_v6_3_1_ih_doorbell_range(struct amdgpu_device *adev,
 						  S2A_DOORBELL_PORT1_RANGE_SIZE,
 						  0);
 
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4))
+	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
 		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_nbif_4_10,
-				ih_doorbell_range);
+			     ih_doorbell_range);
 	else
-		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range);
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+			     ih_doorbell_range);
 }
 
 static void nbif_v6_3_1_ih_control(struct amdgpu_device *adev)
@@ -388,13 +324,9 @@ static void nbif_v6_3_1_ih_control(struct amdgpu_device *adev)
 	u32 interrupt_cntl;
 
 	/* setup interrupt control */
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5))
-		WREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL2_nbio_7_11_5,
-				adev->dummy_page_addr >> 8);
-	else
-		WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
 
-	interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL_nbio_7_11_5);
+	interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL);
 	/*
 	 * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
 	 * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
@@ -406,10 +338,7 @@ static void nbif_v6_3_1_ih_control(struct amdgpu_device *adev)
 	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL,
 				       IH_REQ_NONSNOOP_EN, 0);
 
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5))
-		WREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL_nbio_7_11_5, interrupt_cntl);
-	else
-		WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl);
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl);
 }
 
 static void
@@ -432,23 +361,17 @@ nbif_v6_3_1_get_clockgating_state(struct amdgpu_device *adev,
 
 static u32 nbif_v6_3_1_get_hdp_flush_req_offset(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5))
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF1_GPU_HDP_FLUSH_REQ_nbio_7_11_5);
-	else
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ);
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_REQ);
 }
 
 static u32 nbif_v6_3_1_get_hdp_flush_done_offset(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5))
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF1_GPU_HDP_FLUSH_DONE_nbio_7_11_5);
-	else
-		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE);
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_GPU_HDP_FLUSH_DONE);
 }
 
 static u32 nbif_v6_3_1_get_pcie_index_offset(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4))
+	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
 		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX);
 	else
 		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_INDEX);
@@ -456,7 +379,7 @@ static u32 nbif_v6_3_1_get_pcie_index_offset(struct amdgpu_device *adev)
 
 static u32 nbif_v6_3_1_get_pcie_data_offset(struct amdgpu_device *adev)
 {
-	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) >= IP_VERSION(7, 11, 4))
+	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
 		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA);
 	else
 		return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF0_RSMU_DATA);
@@ -618,12 +541,8 @@ static void nbif_v6_3_1_set_reg_remap(struct amdgpu_device *adev)
 		adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
 		adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
 	} else {
-		if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 5))
-			adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
-				regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL_nbio_7_11_5) << 2;
-		else
-			adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
-				regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
+		adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
+			regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
 		adev->rmmio_remap.bus_addr = 0;
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
index f89e5f40e1a548..a66e3073aed83e 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
@@ -199,7 +199,7 @@ static void nbio_v4_3_ih_doorbell_range(struct amdgpu_device *adev,
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
 						  S2A_DOORBELL_ENTRY_1_CTRL,
 						  S2A_DOORBELL_PORT1_RANGE_SIZE,
-						  2);
+						  8);
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
 						  S2A_DOORBELL_ENTRY_1_CTRL,
 						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
index 5e8f466f23ad37..a40a13ca101137 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
@@ -83,6 +83,21 @@ static u32 nbio_v6_3_2_get_memsize(struct amdgpu_device *adev)
 	return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE);
 }
 
+static u32 nbio_v6_3_2_get_rom_offset(struct amdgpu_device *adev)
+{
+	u32 data, rom_offset;
+
+	data = RREG32_SOC15(NBIO, 0, regREGS_ROM_OFFSET_CTRL);
+	rom_offset = REG_GET_FIELD(data, REGS_ROM_OFFSET_CTRL, ROM_OFFSET);
+
+	/* Temp W/A for nbio v6.3.2. Retrieving rom_offset as 1 but vbios
+	 * image is actually located from offset 2
+	 */
+	rom_offset += 1;
+
+	return rom_offset;
+}
+
 static void nbio_v6_3_2_enable_doorbell_aperture(struct amdgpu_device *adev,
 						 bool enable)
 {
@@ -266,7 +281,7 @@ static void nbio_v6_3_2_vcn_doorbell_range(struct amdgpu_device *adev,
 		doorbell_range = REG_SET_FIELD(doorbell_range,
 					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
 					       S2A_DOORBELL_PORT2_RANGE_SIZE,
-					       8);
+					       11);
 		doorbell_range = REG_SET_FIELD(doorbell_range,
 					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
 					       S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
@@ -366,4 +381,5 @@ const struct amdgpu_nbio_funcs nbio_v6_3_2_funcs = {
 	.sdma_doorbell_range = nbio_v6_3_2_sdma_doorbell_range,
 	.vcn_doorbell_range = nbio_v6_3_2_vcn_doorbell_range,
 	.init_registers = nbio_v6_3_2_init_registers,
+	.get_rom_offset = nbio_v6_3_2_get_rom_offset,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c
new file mode 100644
index 00000000000000..0ea6e091a24ca5
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.c
@@ -0,0 +1,351 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "nbio_v7_11_5.h"
+
+#include "nbio/nbio_7_11_5_offset.h"
+#include "nbio/nbio_7_11_5_sh_mask.h"
+#include <uapi/linux/kfd_ioctl.h>
+
+static void nbio_v7_11_5_remap_hdp_registers(struct amdgpu_device *adev)
+{
+		WREG32_SOC15(NBIO, 0, regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL,
+			adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL);
+		WREG32_SOC15(NBIO, 0, regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL,
+			adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL);
+}
+
+static u32 nbio_v7_11_5_get_rev_id(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+		tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0);
+
+	tmp &= RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK;
+	tmp >>= RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT;
+
+	return tmp;
+}
+
+static void nbio_v7_11_5_mc_access_enable(struct amdgpu_device *adev, bool enable)
+{
+	if (enable)
+		WREG32_SOC15(NBIO, 0, regBIF_BX1_BIF_FB_EN,
+			BIF_BX1_BIF_FB_EN__FB_READ_EN_MASK |
+			BIF_BX1_BIF_FB_EN__FB_WRITE_EN_MASK);
+	else
+		WREG32_SOC15(NBIO, 0, regBIF_BX1_BIF_FB_EN, 0);
+}
+
+static u32 nbio_v7_11_5_get_memsize(struct amdgpu_device *adev)
+{
+	u32 memsize;
+
+	memsize = RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE);
+
+	return memsize;
+}
+
+static void nbio_v7_11_5_sdma_doorbell_range(struct amdgpu_device *adev,
+					    int instance, bool use_doorbell,
+					    int doorbell_index,
+					    int doorbell_size)
+{
+	u32 doorbell_range;
+
+	if (instance == 0) {
+		doorbell_range = RREG32_SOC15(NBIO, 0,
+					     regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL);
+
+		if (use_doorbell) {
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+						       S2A_DOORBELL_PORT2_ENABLE,
+						       0x1);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+						       S2A_DOORBELL_PORT2_AWID,
+						       0xe);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+						       S2A_DOORBELL_PORT2_RANGE_OFFSET,
+						       doorbell_index);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+						       S2A_DOORBELL_PORT2_RANGE_SIZE,
+						       doorbell_size);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+						       S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
+						       0x3);
+		} else {
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+						       S2A_DOORBELL_PORT2_RANGE_SIZE,
+						       0);
+		}
+
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range);
+	}
+}
+
+static void nbio_v7_11_5_vpe_doorbell_range(struct amdgpu_device *adev,
+							int instance, bool use_doorbell,
+							int doorbell_index,
+							int doorbell_size)
+{
+	u32 doorbell_range;
+
+	if (instance)
+		return;
+
+	doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL);
+
+	if (use_doorbell) {
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_ENABLE,
+					       0x1);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_AWID,
+					       0xf);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_RANGE_OFFSET,
+					       doorbell_index);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_RANGE_SIZE,
+					       doorbell_size);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE,
+					       0xf);
+	} else {
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_RANGE_SIZE,
+					       0);
+	}
+
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
+}
+
+static void nbio_v7_11_5_gc_doorbell_init(struct amdgpu_device *adev)
+{
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL, 0x30000007);
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL, 0x3000000d);
+}
+
+static void nbio_v7_11_5_enable_doorbell_aperture(struct amdgpu_device *adev,
+						bool enable)
+{
+	WREG32_FIELD15_PREREG(NBIO, 0, RCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN,
+			BIF_DOORBELL_APER_EN, enable ? 1 : 0);
+}
+
+static void nbio_v7_11_5_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
+							 bool enable)
+{
+	u32 tmp = 0;
+
+	if (enable) {
+		tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_EN, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_MODE, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_SIZE, 0);
+
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW,
+			     lower_32_bits(adev->doorbell.base));
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH,
+			     upper_32_bits(adev->doorbell.base));
+	}
+
+	WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, tmp);
+}
+
+static void nbio_v7_11_5_ih_doorbell_range(struct amdgpu_device *adev,
+					 bool use_doorbell, int doorbell_index)
+{
+	u32 ih_doorbell_range;
+
+	ih_doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL);
+
+	if (use_doorbell) {
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_ENABLE,
+						  0x1);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_AWID,
+						  0x0);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_RANGE_OFFSET,
+						  doorbell_index);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_RANGE_SIZE,
+						  2);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
+						  0x0);
+	} else {
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_RANGE_SIZE,
+						  0);
+	}
+
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range);
+}
+
+static void nbio_v7_11_5_ih_control(struct amdgpu_device *adev)
+{
+	u32 interrupt_cntl;
+
+	/* setup interrupt control */
+		WREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL2,
+				adev->dummy_page_addr >> 8);
+
+	interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL);
+	/*
+	 * BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
+	 * BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
+	 */
+	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX1_INTERRUPT_CNTL,
+				       IH_DUMMY_RD_OVERRIDE, 0);
+
+	/* BIF_BX1_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
+	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX1_INTERRUPT_CNTL,
+				       IH_REQ_NONSNOOP_EN, 0);
+
+		WREG32_SOC15(NBIO, 0, regBIF_BX1_INTERRUPT_CNTL, interrupt_cntl);
+}
+
+static void
+nbio_v7_11_5_get_clockgating_state(struct amdgpu_device *adev,
+				  u64 *flags)
+{
+}
+
+static u32 nbio_v7_11_5_get_hdp_flush_req_offset(struct amdgpu_device *adev)
+{
+	u32 offset;
+
+	offset = SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF1_GPU_HDP_FLUSH_REQ);
+
+	return offset;
+}
+
+static u32 nbio_v7_11_5_get_hdp_flush_done_offset(struct amdgpu_device *adev)
+{
+	u32 offset;
+
+	offset = SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF1_GPU_HDP_FLUSH_DONE);
+
+	return offset;
+}
+
+static u32 nbio_v7_11_5_get_pcie_index_offset(struct amdgpu_device *adev)
+{
+	u32 offset;
+
+	offset = SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF1_RSMU_INDEX);
+
+	return offset;
+}
+
+static u32 nbio_v7_11_5_get_pcie_data_offset(struct amdgpu_device *adev)
+{
+	u32 offset;
+
+	offset = SOC15_REG_OFFSET(NBIO, 0, regBIF_BX_PF1_RSMU_DATA);
+
+	return offset;
+}
+
+const struct nbio_hdp_flush_reg nbio_v7_11_5_hdp_flush_reg = {
+	.ref_and_mask_cp0 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP0_MASK,
+	.ref_and_mask_cp1 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP1_MASK,
+	.ref_and_mask_cp2 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP2_MASK,
+	.ref_and_mask_cp3 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP3_MASK,
+	.ref_and_mask_cp4 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP4_MASK,
+	.ref_and_mask_cp5 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP5_MASK,
+	.ref_and_mask_cp6 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP6_MASK,
+	.ref_and_mask_cp7 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP7_MASK,
+	.ref_and_mask_cp8 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP8_MASK,
+	.ref_and_mask_cp9 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP9_MASK,
+	.ref_and_mask_sdma0 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA0_MASK,
+	.ref_and_mask_sdma1 = BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA1_MASK,
+};
+
+static void nbio_v7_11_5_init_registers(struct amdgpu_device *adev)
+{
+	uint32_t data;
+
+	data = RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2);
+	data &= ~RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2_MASK;
+	WREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF2_STRAP2, data);
+}
+
+#define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE)
+
+static void nbio_v7_11_5_set_reg_remap(struct amdgpu_device *adev)
+{
+	if (!amdgpu_sriov_vf(adev) && (PAGE_SIZE <= 4096)) {
+		adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
+		adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
+	} else {
+		adev->rmmio_remap.reg_offset = SOC15_REG_OFFSET(NBIO, 0,
+			regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2;
+		adev->rmmio_remap.bus_addr = 0;
+	}
+}
+
+const struct amdgpu_nbio_funcs nbio_v7_11_5_funcs = {
+	.get_hdp_flush_req_offset = nbio_v7_11_5_get_hdp_flush_req_offset,
+	.get_hdp_flush_done_offset = nbio_v7_11_5_get_hdp_flush_done_offset,
+	.get_pcie_index_offset = nbio_v7_11_5_get_pcie_index_offset,
+	.get_pcie_data_offset = nbio_v7_11_5_get_pcie_data_offset,
+	.get_rev_id = nbio_v7_11_5_get_rev_id,
+	.mc_access_enable = nbio_v7_11_5_mc_access_enable,
+	.get_memsize = nbio_v7_11_5_get_memsize,
+	.sdma_doorbell_range = nbio_v7_11_5_sdma_doorbell_range,
+	.vpe_doorbell_range = nbio_v7_11_5_vpe_doorbell_range,
+	.gc_doorbell_init = nbio_v7_11_5_gc_doorbell_init,
+	.enable_doorbell_aperture = nbio_v7_11_5_enable_doorbell_aperture,
+	.enable_doorbell_selfring_aperture = nbio_v7_11_5_enable_doorbell_selfring_aperture,
+	.ih_doorbell_range = nbio_v7_11_5_ih_doorbell_range,
+	.get_clockgating_state = nbio_v7_11_5_get_clockgating_state,
+	.ih_control = nbio_v7_11_5_ih_control,
+	.init_registers = nbio_v7_11_5_init_registers,
+	.remap_hdp_registers = nbio_v7_11_5_remap_hdp_registers,
+	.set_reg_remap = nbio_v7_11_5_set_reg_remap,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.h b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.h
new file mode 100644
index 00000000000000..8003564f857e92
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_11_5.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NBIO_V7_11_5_H__
+#define __NBIO_V7_11_5_H__
+
+#include "soc15_common.h"
+
+extern const struct nbio_hdp_flush_reg nbio_v7_11_5_hdp_flush_reg;
+extern const struct amdgpu_nbio_funcs nbio_v7_11_5_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
index def02993b7cfd1..ad24441776d229 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_9.c
@@ -22,6 +22,7 @@
  */
 #include "amdgpu.h"
 #include "nbio_v7_9.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_ras.h"
 
 #include "nbio/nbio_7_9_0_offset.h"
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
index 96faaf8c5737eb..e9a1bd3f0d6d51 100644
--- a/drivers/gpu/drm/amd/amdgpu/nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -992,11 +992,6 @@ static int nv_common_resume(struct amdgpu_ip_block *ip_block)
 	return nv_common_hw_init(ip_block);
 }
 
-static bool nv_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int nv_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 					   enum amd_clockgating_state state)
 {
@@ -1058,7 +1053,6 @@ static const struct amd_ip_funcs nv_common_ip_funcs = {
 	.hw_fini = nv_common_hw_fini,
 	.suspend = nv_common_suspend,
 	.resume = nv_common_resume,
-	.is_idle = nv_common_is_idle,
 	.set_clockgating_state = nv_common_set_clockgating_state,
 	.set_powergating_state = nv_common_set_powergating_state,
 	.get_clockgating_state = nv_common_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index ad618b78685f65..0803cfc445dc77 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -53,6 +53,13 @@ enum psp_gfx_crtl_cmd_id
     GFX_CTRL_CMD_ID_MAX             = 0x000F0000,   /* max command ID */
 };
 
+typedef enum
+{
+    GFX_CTRL_ADDR_TYPE_AUTO_DETECT = 0,
+    GFX_CTRL_ADDR_TYPE_SYS_PHY_ADDR,
+    GFX_CTRL_ADDR_TYPE_GPU_PHY_ADDR,
+    GFX_CTRL_ADDR_TYPE_GPU_VIRT_ADDR
+}  GFX_CTRL_ADDR_TYPE;
 
 /*-----------------------------------------------------------------------------
     NOTE:   All physical addresses used in this interface are actually
@@ -110,6 +117,13 @@ enum psp_gfx_cmd_id
     GFX_CMD_ID_PERF_HW            = 0x0000004C,   /* performance monitor */
     GFX_CMD_ID_FB_FW_RESERV_ADDR  = 0x00000050,  /* Query FW reservation addr */
     GFX_CMD_ID_FB_FW_RESERV_EXT_ADDR = 0x00000051,  /* Query FW reservation extended addr */
+    GFX_CMD_ID_UAL_GET_INTERFACE_VER = 0x00000053,  /* Get UAL interface version */
+    GFX_CMD_ID_UAL_GET_CONFIG        = 0x00000054,  /* Get UAL full config */
+    GFX_CMD_ID_UAL_SET_PPOD_CONFIG   = 0x00000055,  /* Set UAL config */
+    GFX_CMD_ID_UAL_SET_VPOD_CONFIG   = 0x00000056,  /* Set UAL vPod config */
+    GFX_CMD_ID_UAL_SET_STATION_CONFIG= 0x00000057,  /* Set UAL Station config */
+    GFX_CMD_ID_UAL_SET_NPA_CONFIG    = 0x00000059,  /* Set UAL NPA config and VMID */
+    GFX_CMD_ID_UAL_SEND_COMPLETION   = 0x0000005A,  /* Ack from driver after UAL config completion */
     GFX_CMD_ID_SET_MMHUB_ECO_SEC_LEVEL = 0x0000005D,  /* Set MMHUB ECO sec lvls on VCN block */
 };
 
@@ -382,6 +396,136 @@ struct psp_gfx_cmd_req_perf_hw {
 	uint32_t pref_format2;
 };
 
+#define PSP_GFX_MAX_LOCAL_GPUS_UAL_V1   16       /* max gpu per local node */
+#define PSP_GFX_UAL_MAX_STATIONS_V1     64       /* max UALink stations */
+#define PSP_GFX_UAL_MAX_ACC_BIT_MASK    32       /* max accelerator bit mask 32*32 */
+
+/* Interrupt category identifiers (upper byte) - ASP interrupts to Driver */
+
+/* Commands from ASP */
+#define PSP_GFX_INT_CTXT_UAL_CAT_CMD 0x01000000
+/* Mask for Command bits */
+#define PSP_GFX_INT_CTXT_UAL_CAT_CMD_MASK 0x000000FF
+/* Notifications from ASP */
+#define PSP_GFX_INT_CTXT_UAL_CAT_NOTIFY 0x02000000
+/* Mask for category bits */
+#define PSP_GFX_INT_CTXT_UAL_CAT_MASK 0xFF000000
+
+#define PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID 0x00000001
+#define PSP_GFX_INT_CTXT_UAL_CMD_PAUSE_ID 0x00000002
+#define PSP_GFX_INT_CTXT_UAL_CMD_RESUME_ID 0x00000003
+
+/* Command interrupts from ASP (GIM must execute action) */
+#define PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE \
+	(PSP_GFX_INT_CTXT_UAL_CAT_CMD | PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE_ID)
+
+#define PSP_GFX_INT_CTXT_UAL_CMD_PAUSE \
+	(PSP_GFX_INT_CTXT_UAL_CAT_CMD | PSP_GFX_INT_CTXT_UAL_CMD_PAUSE_ID)
+
+#define PSP_GFX_INT_CTXT_UAL_CMD_RESUME \
+	(PSP_GFX_INT_CTXT_UAL_CAT_CMD | PSP_GFX_INT_CTXT_UAL_CMD_RESUME_ID)
+
+enum psp_gfx_ual_npa_address_mode {
+	PSP_GFX_UAL_NPA_ADDRESS_MODE_SOURCE_ALIASING = 0,
+	PSP_GFX_UAL_NPA_ADDRESS_MODE_SOURCE_IDENTIFICATION = 1,
+	PSP_GFX_UAL_NPA_ADDRESS_MODE_MAX
+};
+
+enum psp_gfx_ual_ports_per_station
+{
+    PSP_GFX_UAL_PPS_1 = 1,    // 1x 800Gbps
+    PSP_GFX_UAL_PPS_2 = 2,    // 2x 400Gbps
+    PSP_GFX_UAL_PPS_4 = 4     // 4x 200Gbps
+};
+
+enum psp_gfx_ual_link_type
+{
+    PSP_GFX_UALOE = 0,
+    PSP_GFX_UALLINK = 1,
+    PSP_GFX_UALMAX
+};
+
+enum psp_gfx_ual_config_state
+{
+	UAL_CFG_IDLE        = 0,
+	UAL_CFG_PPOD        = 1,
+	UAL_CFG_VPOD        = 2,
+	UAL_CFG_PAUSE       = 3,
+	UAL_CFG_STATION     = 4,
+	UAL_CFG_COMPLETE    = 5,
+	UAL_CFG_INVALID     = 0xFF
+};
+
+struct psp_gfx_cmd_set_ppod_config_ual_v1
+{
+    uint32_t accelerator_id;     // Accelerator ID
+    uint8_t  ppod_id[16];        // Physical Pod ID
+    uint32_t ppod_size;          // number of gpu in physical pod
+    uint32_t bandwidth;          // Bandwidth
+    uint32_t latency;            // Latency
+    uint32_t local_accelerators[PSP_GFX_MAX_LOCAL_GPUS_UAL_V1]; // local accelerator list, sorted in socket-id order
+};
+
+struct psp_gfx_cmd_set_vpod_config_ual_v1
+{
+    uint32_t vpod_id;                       // vPod ID
+    uint32_t vpod_size;                     // number of gpu in virtual pod
+    uint32_t vpod_active_accelerators[PSP_GFX_UAL_MAX_ACC_BIT_MASK];  // accelerator ID list in each vPOD
+    enum psp_gfx_ual_npa_address_mode addr_mode; // NPA Addressing Mode
+};
+
+struct psp_gfx_cmd_station_config_ual_v1
+{
+    /* Number of valid stations in this configuration
+     * Only lane_en_bitmap[0..n_stations-1] will be processed.
+     */
+    uint8_t num_stations;
+    /* Station configuration flags
+     *
+     * Bit [3:0]: PortPerStation (PPS) - 1, 2, or 4
+     * Bit [7:4]: Reserved
+     */
+    uint8_t station_flag;
+    uint8_t reserved[2]; /* Future use / alignment padding */
+    /* Bitmap of enabled lanes for each station
+     * in logical station order.
+     */
+    uint8_t lane_en_bitmap[PSP_GFX_UAL_MAX_STATIONS_V1];
+};
+
+struct psp_gfx_cmd_get_config_ual_v1
+{
+    uint32_t ual_cfg_addr_hi;               // UAL Config Buffer Address Hi
+    uint32_t ual_cfg_addr_lo;               // UAL Config Buffer Address Lo
+    uint32_t ual_cfg_size;                  // UAL Config Buffer Size
+    GFX_CTRL_ADDR_TYPE addr_type;    // Buffer Address Type
+};
+
+struct psp_gfx_get_config_ual_v1 {
+    enum psp_gfx_ual_link_type link_type;
+    uint32_t accelerator_id;
+    uint8_t ppod_id[16];
+    uint32_t ppod_size;
+    uint32_t bandwidth;
+    uint32_t latency;
+    uint32_t vpod_id;
+    uint32_t vpod_size;
+    uint32_t vpod_active_accelerators[PSP_GFX_UAL_MAX_ACC_BIT_MASK];
+    enum psp_gfx_ual_npa_address_mode addr_mode;
+    enum psp_gfx_ual_config_state config_state;
+};
+
+struct psp_gfx_cmd_send_completion_ual_v1
+{
+    uint32_t cmd_id;
+    uint32_t status;
+};
+
+struct psp_gfx_cmd_set_npa_config_ual_v1 {
+    uint32_t vmid;
+    uint32_t enable_npa_translation;
+};
+
 /* All GFX ring buffer commands. */
 union psp_gfx_commands
 {
@@ -399,6 +543,12 @@ union psp_gfx_commands
     struct psp_gfx_cmd_config_sq_perfmon config_sq_perfmon;
     struct psp_gfx_cmd_fb_memory_part cmd_memory_part;
     struct psp_gfx_cmd_req_perf_hw     cmd_req_perf_hw;
+    struct psp_gfx_cmd_get_config_ual_v1    cmd_get_config_ual;
+    struct psp_gfx_cmd_set_ppod_config_ual_v1 cmd_set_ppod_config_ual;
+    struct psp_gfx_cmd_set_vpod_config_ual_v1 cmd_set_vpod_config_ual;
+    struct psp_gfx_cmd_station_config_ual_v1  cmd_set_station_config_ual;
+    struct psp_gfx_cmd_send_completion_ual_v1 cmd_send_completion_ual;
+    struct psp_gfx_cmd_set_npa_config_ual_v1  cmd_set_npa_config_ual;
 };
 
 struct psp_gfx_uresp_reserved
@@ -432,6 +582,16 @@ struct psp_gfx_uresp_perf_hw {
 	uint32_t pref_format2;
 };
 
+struct psp_gfx_uresp_get_intf_ver_ual
+{
+    uint32_t intf_ver; /* [31:16] major version, [15:0] minor version */
+};
+
+struct psp_gfx_uresp_get_config_ual_v1
+{
+    uint32_t resp_size;
+};
+
 /* Union of command-specific responses for GPCOM ring. */
 union psp_gfx_uresp {
 	struct psp_gfx_uresp_reserved		reserved;
@@ -439,6 +599,8 @@ union psp_gfx_uresp {
 	struct psp_gfx_uresp_fwar_db_info	fwar_db_info;
 	struct psp_gfx_uresp_fw_reserve_info	fw_reserve_info;
 	struct psp_gfx_uresp_perf_hw		perf_hw_info;
+	struct psp_gfx_uresp_get_config_ual_v1  get_config_ual;
+	struct psp_gfx_uresp_get_intf_ver_ual   get_intf_ver_ual;
 };
 
 /* Structure of GFX Response buffer.
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
index ec20cd5eb75525..dcfc9be269b07b 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
@@ -33,6 +33,8 @@
 
 MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc.bin");
 MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc_1.bin");
+MODULE_FIRMWARE("amdgpu/psp_15_0_8_rl.bin");
+MODULE_FIRMWARE("amdgpu/psp_15_0_8_ta.bin");
 
 static int psp_v15_0_8_init_microcode(struct psp_context *psp)
 {
@@ -46,6 +48,14 @@ static int psp_v15_0_8_init_microcode(struct psp_context *psp)
 	if (err)
 		return err;
 
+	err = psp_init_rl_microcode(psp, ucode_prefix);
+	if (err)
+		return err;
+
+	err = psp_init_ta_microcode(psp, ucode_prefix);
+	if (err)
+		return err;
+
 	return 0;
 }
 
@@ -347,6 +357,141 @@ static int psp_v15_0_8_get_fw_type(struct amdgpu_firmware_info *ucode,
 	return 0;
 }
 
+/* TBD: These should be in psp_gfx_if.h */
+#define IH_INTERRUPT_ID_TO_DRIVER 0xFF
+
+static int psp_v15_0_8_irq_process(struct amdgpu_device *adev,
+				   struct amdgpu_irq_src *source,
+				   struct amdgpu_iv_entry *entry)
+{
+	struct psp_context *psp = &adev->psp;
+	uint32_t data;
+
+	/* Replayed from the PSP soft ring: run the registered handler. */
+	if (entry->ih == &adev->irq.ih_psp) {
+		amdgpu_psp_irq_mgr_dispatch(&psp->irq_mgr, entry);
+		return 1;
+	}
+
+	/* ACK ASP interrupt */
+	if (!amdgpu_sriov_vf(adev)) {
+		data = RREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT_CTRL);
+		data = REG_SET_FIELD(data, MPASP_SMN_IH_SW_INT_CTRL, INT_ACK, 1);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT_CTRL, data);
+	}
+
+	dev_dbg(adev->dev,
+		"psp_v15_0_8_irq_process: ctxid[0] = %d, ctxid[1] = %d\n",
+		entry->src_data[0], entry->src_data[1]);
+
+	/* Delegate to the PSP soft ring for process-context handling. */
+	amdgpu_irq_psp_delegate(adev, entry, 8);
+
+	return 1;
+}
+
+static int psp_v15_0_8_set_irq_state(struct amdgpu_device *adev,
+				     struct amdgpu_irq_src *source,
+				     unsigned int type,
+				     enum amdgpu_interrupt_state state)
+{
+	uint32_t val = 0;
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		val = RREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT_CTRL);
+		val = REG_SET_FIELD(val, MPASP_SMN_IH_SW_INT_CTRL, INT_MASK, 1);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT_CTRL, val);
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		val = RREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT);
+		val = REG_SET_FIELD(val, MPASP_SMN_IH_SW_INT, ID, 0xFE);
+		val = REG_SET_FIELD(val, MPASP_SMN_IH_SW_INT, VALID, 0);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT, val);
+
+		val = RREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT_CTRL);
+		val = REG_SET_FIELD(val, MPASP_SMN_IH_SW_INT_CTRL, INT_MASK, 0);
+		WREG32_SOC15(MP0, 0, regMPASP_SMN_IH_SW_INT_CTRL, val);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs psp_v15_0_8_irq_funcs = {
+	.set = psp_v15_0_8_set_irq_state,
+	.process = psp_v15_0_8_irq_process,
+};
+
+static void psp_v15_0_8_ual_cmd_cfg_update(struct amdgpu_psp_irq_mgr *mgr,
+					   u32 event_id,
+					   struct amdgpu_iv_entry *entry)
+{
+	struct amdgpu_device *adev = mgr->psp->adev;
+
+	amdgpu_ualink_config_update_handler(adev);
+}
+
+static void psp_v15_0_8_ual_cmd_pause(struct amdgpu_psp_irq_mgr *mgr,
+				      u32 event_id,
+				      struct amdgpu_iv_entry *entry)
+{
+	struct amdgpu_device *adev = mgr->psp->adev;
+
+	amdgpu_ualink_pause_handler(adev);
+}
+
+static void psp_v15_0_8_ual_cmd_resume(struct amdgpu_psp_irq_mgr *mgr,
+				       u32 event_id,
+				       struct amdgpu_iv_entry *entry)
+{
+	struct amdgpu_device *adev = mgr->psp->adev;
+
+	amdgpu_ualink_resume_handler(adev);
+}
+
+static const struct amdgpu_psp_irq_handler psp_v15_0_8_ual_irq_handlers[] = {
+	{
+		.event_id = PSP_GFX_INT_CTXT_UAL_CMD_CFG_UPDATE,
+		.callback = psp_v15_0_8_ual_cmd_cfg_update,
+	},
+	{
+		.event_id = PSP_GFX_INT_CTXT_UAL_CMD_PAUSE,
+		.callback = psp_v15_0_8_ual_cmd_pause,
+	},
+	{
+		.event_id = PSP_GFX_INT_CTXT_UAL_CMD_RESUME,
+		.callback = psp_v15_0_8_ual_cmd_resume,
+	},
+};
+
+static int psp_v15_0_8_register_irq_handler(struct amdgpu_psp_irq_mgr *mgr,
+					    struct amdgpu_irq_src *irq_src)
+{
+	struct psp_context *psp = mgr->psp;
+	struct amdgpu_device *adev = psp->adev;
+	int ret;
+
+	irq_src->num_types = 1;
+	irq_src->funcs = &psp_v15_0_8_irq_funcs;
+
+	ret = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_MP0,
+				IH_INTERRUPT_ID_TO_DRIVER, irq_src);
+	if (ret)
+		return ret;
+
+	ret = amdgpu_psp_irq_mgr_register(
+		mgr, psp_v15_0_8_ual_irq_handlers,
+		ARRAY_SIZE(psp_v15_0_8_ual_irq_handlers), NULL);
+
+	return ret;
+}
+
 static const struct psp_funcs psp_v15_0_8_funcs = {
 	.init_microcode = psp_v15_0_8_init_microcode,
 	.ring_create = psp_v15_0_8_ring_create,
@@ -356,6 +501,7 @@ static const struct psp_funcs psp_v15_0_8_funcs = {
 	.ring_set_wptr = psp_v15_0_8_ring_set_wptr,
 	.get_fw_type = psp_v15_0_8_get_fw_type,
 	.get_ras_capability = psp_v15_0_8_get_ras_capability,
+	.register_irq_handler = psp_v15_0_8_register_irq_handler,
 };
 
 void psp_v15_0_8_set_psp_funcs(struct psp_context *psp)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index c7253b908351c2..6b64cf83ee5c27 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 #include "vi.h"
 #include "vid.h"
@@ -223,14 +224,14 @@ static void sdma_v2_4_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -349,18 +350,6 @@ static void sdma_v2_4_gfx_stop(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * sdma_v2_4_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues (VI).
- */
-static void sdma_v2_4_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v2_4_enable - stop the async dma engines
  *
@@ -376,7 +365,6 @@ static void sdma_v2_4_enable(struct amdgpu_device *adev, bool enable)
 
 	if (!enable) {
 		sdma_v2_4_gfx_stop(adev);
-		sdma_v2_4_rlc_stop(adev);
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -477,20 +465,6 @@ static int sdma_v2_4_gfx_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-/**
- * sdma_v2_4_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them (VI).
- * Returns 0 for success, error for failure.
- */
-static int sdma_v2_4_rlc_resume(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-	return 0;
-}
-
 
 /**
  * sdma_v2_4_start - setup and start the async dma engines
@@ -509,9 +483,6 @@ static int sdma_v2_4_start(struct amdgpu_device *adev)
 
 	/* start the gfx rings and rlc compute queues */
 	r = sdma_v2_4_gfx_resume(adev);
-	if (r)
-		return r;
-	r = sdma_v2_4_rlc_resume(adev);
 	if (r)
 		return r;
 
@@ -583,7 +554,8 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
@@ -597,26 +569,31 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	tmp = 0xCAFEDEAD;
 	adev->wb.wb[index] = cpu_to_le32(tmp);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256,
-					AMDGPU_IB_POOL_DIRECT, &ib);
+
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r)
 		goto err0;
 
-	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -632,7 +609,6 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -733,12 +709,13 @@ static void sdma_v2_4_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,
 static void sdma_v2_4_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -860,13 +837,8 @@ static int sdma_v2_4_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = false;
-		sprintf(ring->name, "sdma%d", i);
-		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
-				     (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :
-				     AMDGPU_SDMA_IRQ_INSTANCE1,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d", i);
 		if (r)
 			return r;
 	}
@@ -919,18 +891,6 @@ static int sdma_v2_4_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v2_4_hw_init(ip_block);
 }
 
-static bool sdma_v2_4_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS2);
-
-	if (tmp & (SRBM_STATUS2__SDMA_BUSY_MASK |
-		   SRBM_STATUS2__SDMA1_BUSY_MASK))
-	    return false;
-
-	return true;
-}
-
 static int sdma_v2_4_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -1112,7 +1072,6 @@ static const struct amd_ip_funcs sdma_v2_4_ip_funcs = {
 	.hw_fini = sdma_v2_4_hw_fini,
 	.suspend = sdma_v2_4_suspend,
 	.resume = sdma_v2_4_resume,
-	.is_idle = sdma_v2_4_is_idle,
 	.wait_for_idle = sdma_v2_4_wait_for_idle,
 	.soft_reset = sdma_v2_4_soft_reset,
 	.set_clockgating_state = sdma_v2_4_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 05cce4bde73eb9..6f7243370acb8e 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 #include "vi.h"
 #include "vid.h"
@@ -400,14 +401,14 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -526,18 +527,6 @@ static void sdma_v3_0_gfx_stop(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * sdma_v3_0_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues (VI).
- */
-static void sdma_v3_0_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v3_0_ctx_switch_enable - stop the async dma engines context switch
  *
@@ -614,7 +603,6 @@ static void sdma_v3_0_enable(struct amdgpu_device *adev, bool enable)
 
 	if (!enable) {
 		sdma_v3_0_gfx_stop(adev);
-		sdma_v3_0_rlc_stop(adev);
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -752,20 +740,6 @@ static int sdma_v3_0_gfx_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-/**
- * sdma_v3_0_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them (VI).
- * Returns 0 for success, error for failure.
- */
-static int sdma_v3_0_rlc_resume(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-	return 0;
-}
-
 /**
  * sdma_v3_0_start - setup and start the async dma engines
  *
@@ -784,9 +758,6 @@ static int sdma_v3_0_start(struct amdgpu_device *adev)
 
 	/* start the gfx rings and rlc compute queues */
 	r = sdma_v3_0_gfx_resume(adev);
-	if (r)
-		return r;
-	r = sdma_v3_0_rlc_resume(adev);
 	if (r)
 		return r;
 
@@ -858,7 +829,8 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
@@ -872,26 +844,31 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	tmp = 0xCAFEDEAD;
 	adev->wb.wb[index] = cpu_to_le32(tmp);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256,
-					AMDGPU_IB_POOL_DIRECT, &ib);
+
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r)
 		goto err0;
 
-	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -906,7 +883,6 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	else
 		r = -EINVAL;
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1007,12 +983,13 @@ static void sdma_v3_0_vm_set_pte_pde(struct amdgpu_ib *ib, uint64_t pe,
 static void sdma_v3_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1141,7 +1118,6 @@ static int sdma_v3_0_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		if (!amdgpu_sriov_vf(adev)) {
 			ring->use_doorbell = true;
 			ring->doorbell_index = adev->doorbell_index.sdma_engine[i];
@@ -1149,11 +1125,7 @@ static int sdma_v3_0_sw_init(struct amdgpu_ip_block *ip_block)
 			ring->use_pollmem = true;
 		}
 
-		sprintf(ring->name, "sdma%d", i);
-		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
-				     (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :
-				     AMDGPU_SDMA_IRQ_INSTANCE1,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d", i);
 		if (r)
 			return r;
 	}
@@ -1209,18 +1181,6 @@ static int sdma_v3_0_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v3_0_hw_init(ip_block);
 }
 
-static bool sdma_v3_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS2);
-
-	if (tmp & (SRBM_STATUS2__SDMA_BUSY_MASK |
-		   SRBM_STATUS2__SDMA1_BUSY_MASK))
-	    return false;
-
-	return true;
-}
-
 static int sdma_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -1241,13 +1201,11 @@ static int sdma_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 static int sdma_v3_0_soft_reset(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
-	u32 srbm_soft_reset = 0;
+	u32 srbm_soft_reset;
 	u32 tmp;
 
-	if (!adev->sdma.srbm_soft_reset)
-		return 0;
-
-	srbm_soft_reset = adev->sdma.srbm_soft_reset;
+	srbm_soft_reset = SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK |
+			  SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;
 
 	if (srbm_soft_reset) {
 		tmp = RREG32(mmSRBM_SOFT_RESET);
@@ -1492,7 +1450,6 @@ static const struct amd_ip_funcs sdma_v3_0_ip_funcs = {
 	.hw_fini = sdma_v3_0_hw_fini,
 	.suspend = sdma_v3_0_suspend,
 	.resume = sdma_v3_0_resume,
-	.is_idle = sdma_v3_0_is_idle,
 	.wait_for_idle = sdma_v3_0_wait_for_idle,
 	.soft_reset = sdma_v3_0_soft_reset,
 	.set_clockgating_state = sdma_v3_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 20c8ebf0e1591b..902d3130032692 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 
 #include "sdma0/sdma0_4_2_offset.h"
@@ -783,14 +784,14 @@ static void sdma_v4_0_page_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -933,18 +934,6 @@ static void sdma_v4_0_gfx_enable(struct amdgpu_device *adev, bool enable)
 	}
 }
 
-/**
- * sdma_v4_0_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues (VEGA10).
- */
-static void sdma_v4_0_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v4_0_page_stop - stop the page async dma engines
  *
@@ -1047,7 +1036,6 @@ static void sdma_v4_0_enable(struct amdgpu_device *adev, bool enable)
 
 	if (!enable) {
 		sdma_v4_0_gfx_enable(adev, enable);
-		sdma_v4_0_rlc_stop(adev);
 		if (adev->sdma.has_page_queue)
 			sdma_v4_0_page_stop(adev);
 	}
@@ -1312,21 +1300,6 @@ static void sdma_v4_0_init_pg(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * sdma_v4_0_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them (VEGA10).
- * Returns 0 for success, error for failure.
- */
-static int sdma_v4_0_rlc_resume(struct amdgpu_device *adev)
-{
-	sdma_v4_0_init_pg(adev);
-
-	return 0;
-}
-
 /**
  * sdma_v4_0_load_microcode - load the sDMA ME ucode
  *
@@ -1426,9 +1399,7 @@ static int sdma_v4_0_start(struct amdgpu_device *adev)
 		sdma_v4_0_ctx_switch_enable(adev, true);
 		sdma_v4_0_enable(adev, true);
 	} else {
-		r = sdma_v4_0_rlc_resume(adev);
-		if (r)
-			return r;
+		sdma_v4_0_init_pg(adev);
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
@@ -1515,7 +1486,8 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	long r;
@@ -1529,26 +1501,31 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	tmp = 0xCAFEDEAD;
 	adev->wb.wb[index] = cpu_to_le32(tmp);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256,
-					AMDGPU_IB_POOL_DIRECT, &ib);
+
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r)
 		goto err0;
 
-	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1564,7 +1541,6 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1667,12 +1643,13 @@ static void sdma_v4_0_vm_set_pte_pde(struct amdgpu_ib *ib,
 static void sdma_v4_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1853,7 +1830,6 @@ static int sdma_v4_0_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 
 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
@@ -1873,16 +1849,12 @@ static int sdma_v4_0_sw_init(struct amdgpu_ip_block *ip_block)
 		else
 			ring->vm_hub = AMDGPU_MMHUB0(0);
 
-		sprintf(ring->name, "sdma%d", i);
-		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
-				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d", i);
 		if (r)
 			return r;
 
 		if (adev->sdma.has_page_queue) {
 			ring = &adev->sdma.instance[i].page;
-			ring->ring_obj = NULL;
 			ring->use_doorbell = true;
 
 			/* paging queue use same doorbell index/routing as gfx queue
@@ -1910,11 +1882,7 @@ static int sdma_v4_0_sw_init(struct amdgpu_ip_block *ip_block)
 			else
 				ring->vm_hub = AMDGPU_MMHUB0(0);
 
-			sprintf(ring->name, "page%d", i);
-			r = amdgpu_ring_init(adev, ring, 1024,
-					     &adev->sdma.trap_irq,
-					     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-					     AMDGPU_RING_PRIO_DEFAULT, NULL);
+			r = amdgpu_sdma_ring_init(adev, ring, i, "page%d", i);
 			if (r)
 				return r;
 		}
@@ -2027,21 +1995,6 @@ static int sdma_v4_0_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v4_0_hw_init(ip_block);
 }
 
-static bool sdma_v4_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32_SDMA(i, mmSDMA0_STATUS_REG);
-
-		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
-			return false;
-	}
-
-	return true;
-}
-
 static int sdma_v4_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i, j;
@@ -2061,13 +2014,6 @@ static int sdma_v4_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
-static int sdma_v4_0_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-
-	return 0;
-}
-
 static int sdma_v4_0_set_trap_irq_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *source,
 					unsigned type,
@@ -2412,9 +2358,7 @@ const struct amd_ip_funcs sdma_v4_0_ip_funcs = {
 	.hw_fini = sdma_v4_0_hw_fini,
 	.suspend = sdma_v4_0_suspend,
 	.resume = sdma_v4_0_resume,
-	.is_idle = sdma_v4_0_is_idle,
 	.wait_for_idle = sdma_v4_0_wait_for_idle,
-	.soft_reset = sdma_v4_0_soft_reset,
 	.set_clockgating_state = sdma_v4_0_set_clockgating_state,
 	.set_powergating_state = sdma_v4_0_set_powergating_state,
 	.get_clockgating_state = sdma_v4_0_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
index 6fe3d3d56c4062..df788361e86587 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -27,9 +27,11 @@
 #include <linux/pci.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_xcp.h"
 #include "amdgpu_ucode.h"
 #include "amdgpu_trace.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_reset.h"
 
 #include "sdma/sdma_4_4_2_offset.h"
@@ -348,14 +350,14 @@ static void sdma_v4_4_2_page_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v4_4_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -520,20 +522,6 @@ static void sdma_v4_4_2_inst_gfx_stop(struct amdgpu_device *adev,
 	}
 }
 
-/**
- * sdma_v4_4_2_inst_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- * @inst_mask: mask of dma engine instances to be disabled
- *
- * Stop the compute async dma queues.
- */
-static void sdma_v4_4_2_inst_rlc_stop(struct amdgpu_device *adev,
-				      uint32_t inst_mask)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v4_4_2_inst_page_stop - stop the page async dma engines
  *
@@ -632,7 +620,6 @@ static void sdma_v4_4_2_inst_enable(struct amdgpu_device *adev, bool enable,
 
 	if (!enable) {
 		sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
-		sdma_v4_4_2_inst_rlc_stop(adev, inst_mask);
 		if (adev->sdma.has_page_queue)
 			sdma_v4_4_2_inst_page_stop(adev, inst_mask);
 
@@ -675,12 +662,11 @@ static uint32_t sdma_v4_4_2_rb_cntl(struct amdgpu_ring *ring, uint32_t rb_cntl)
  *
  * @adev: amdgpu_device pointer
  * @i: instance to resume
- * @restore: used to restore wptr when restart
  *
  * Set up the gfx DMA ring buffers and enable them.
  * Returns 0 for success, error for failure.
  */
-static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
+static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i)
 {
 	struct amdgpu_ring *ring = &adev->sdma.instance[i].ring;
 	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
@@ -688,7 +674,6 @@ static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i, b
 	u32 doorbell;
 	u32 doorbell_offset;
 	u64 wptr_gpu_addr;
-	u64 rwptr;
 
 	wb_offset = (ring->rptr_offs * 4);
 
@@ -708,32 +693,16 @@ static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i, b
 	WREG32_SDMA(i, regSDMA_GFX_RB_BASE, ring->gpu_addr >> 8);
 	WREG32_SDMA(i, regSDMA_GFX_RB_BASE_HI, ring->gpu_addr >> 40);
 
-	if (!restore)
-		ring->wptr = 0;
+	ring->wptr = 0;
 
 	/* before programing wptr to a less value, need set minor_ptr_update first */
 	WREG32_SDMA(i, regSDMA_GFX_MINOR_PTR_UPDATE, 1);
 
-	/* For the guilty queue, set RPTR to the current wptr to skip bad commands,
-	 * It is not a guilty queue, restore cache_rptr and continue execution.
-	 */
-	if (adev->sdma.instance[i].gfx_guilty)
-		rwptr = ring->wptr;
-	else
-		rwptr = ring->cached_rptr;
-
 	/* Initialize the ring buffer's read and write pointers */
-	if (restore) {
-		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, lower_32_bits(rwptr << 2));
-		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, upper_32_bits(rwptr << 2));
-		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, lower_32_bits(rwptr << 2));
-		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, upper_32_bits(rwptr << 2));
-	} else {
-		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
-		WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
-		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
-		WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
-	}
+	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR, 0);
+	WREG32_SDMA(i, regSDMA_GFX_RB_RPTR_HI, 0);
+	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR, 0);
+	WREG32_SDMA(i, regSDMA_GFX_RB_WPTR_HI, 0);
 
 	doorbell = RREG32_SDMA(i, regSDMA_GFX_DOORBELL);
 	doorbell_offset = RREG32_SDMA(i, regSDMA_GFX_DOORBELL_OFFSET);
@@ -786,7 +755,7 @@ static void sdma_v4_4_2_gfx_resume(struct amdgpu_device *adev, unsigned int i, b
  * Set up the page DMA ring buffers and enable them.
  * Returns 0 for success, error for failure.
  */
-static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i, bool restore)
+static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i)
 {
 	struct amdgpu_ring *ring = &adev->sdma.instance[i].page;
 	u32 rb_cntl, ib_cntl, wptr_poll_cntl;
@@ -794,7 +763,6 @@ static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i,
 	u32 doorbell;
 	u32 doorbell_offset;
 	u64 wptr_gpu_addr;
-	u64 rwptr;
 
 	wb_offset = (ring->rptr_offs * 4);
 
@@ -802,26 +770,11 @@ static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i,
 	rb_cntl = sdma_v4_4_2_rb_cntl(ring, rb_cntl);
 	WREG32_SDMA(i, regSDMA_PAGE_RB_CNTL, rb_cntl);
 
-	/* For the guilty queue, set RPTR to the current wptr to skip bad commands,
-	 * It is not a guilty queue, restore cache_rptr and continue execution.
-	 */
-	if (adev->sdma.instance[i].page_guilty)
-		rwptr = ring->wptr;
-	else
-		rwptr = ring->cached_rptr;
-
 	/* Initialize the ring buffer's read and write pointers */
-	if (restore) {
-		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, lower_32_bits(rwptr << 2));
-		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, upper_32_bits(rwptr << 2));
-		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, lower_32_bits(rwptr << 2));
-		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, upper_32_bits(rwptr << 2));
-	} else {
-		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
-		WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
-		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
-		WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
-	}
+	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR, 0);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_HI, 0);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR, 0);
+	WREG32_SDMA(i, regSDMA_PAGE_RB_WPTR_HI, 0);
 
 	/* set the wb address whether it's enabled or not */
 	WREG32_SDMA(i, regSDMA_PAGE_RB_RPTR_ADDR_HI,
@@ -835,8 +788,7 @@ static void sdma_v4_4_2_page_resume(struct amdgpu_device *adev, unsigned int i,
 	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE, ring->gpu_addr >> 8);
 	WREG32_SDMA(i, regSDMA_PAGE_RB_BASE_HI, ring->gpu_addr >> 40);
 
-	if (!restore)
-		ring->wptr = 0;
+	ring->wptr = 0;
 
 	/* before programing wptr to a less value, need set minor_ptr_update first */
 	WREG32_SDMA(i, regSDMA_PAGE_MINOR_PTR_UPDATE, 1);
@@ -888,23 +840,6 @@ static void sdma_v4_4_2_init_pg(struct amdgpu_device *adev)
 
 }
 
-/**
- * sdma_v4_4_2_inst_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- * @inst_mask: mask of dma engine instances to be enabled
- *
- * Set up the compute DMA queues and enable them.
- * Returns 0 for success, error for failure.
- */
-static int sdma_v4_4_2_inst_rlc_resume(struct amdgpu_device *adev,
-				       uint32_t inst_mask)
-{
-	sdma_v4_4_2_init_pg(adev);
-
-	return 0;
-}
-
 /**
  * sdma_v4_4_2_inst_load_microcode - load the sDMA ME ucode
  *
@@ -991,9 +926,9 @@ static int sdma_v4_4_2_inst_start(struct amdgpu_device *adev,
 		uint32_t temp;
 
 		WREG32_SDMA(i, regSDMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
-		sdma_v4_4_2_gfx_resume(adev, i, restore);
+		sdma_v4_4_2_gfx_resume(adev, i);
 		if (adev->sdma.has_page_queue)
-			sdma_v4_4_2_page_resume(adev, i, restore);
+			sdma_v4_4_2_page_resume(adev, i);
 
 		/* set utc l1 enable flag always to 1 */
 		temp = RREG32_SDMA(i, regSDMA_CNTL);
@@ -1019,9 +954,7 @@ static int sdma_v4_4_2_inst_start(struct amdgpu_device *adev,
 		sdma_v4_4_2_inst_ctx_switch_enable(adev, true, inst_mask);
 		sdma_v4_4_2_inst_enable(adev, true, inst_mask);
 	} else {
-		r = sdma_v4_4_2_inst_rlc_resume(adev, inst_mask);
-		if (r)
-			return r;
+		sdma_v4_4_2_init_pg(adev);
 	}
 
 	tmp_mask = inst_mask;
@@ -1109,7 +1042,8 @@ static int sdma_v4_4_2_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	long r;
@@ -1123,26 +1057,31 @@ static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	tmp = 0xCAFEDEAD;
 	adev->wb.wb[index] = cpu_to_le32(tmp);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256,
-					AMDGPU_IB_POOL_DIRECT, &ib);
+
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r)
 		goto err0;
 
-	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1158,7 +1097,6 @@ static int sdma_v4_4_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1261,12 +1199,13 @@ static void sdma_v4_4_2_vm_set_pte_pde(struct amdgpu_ib *ib,
 static void sdma_v4_4_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1473,13 +1412,9 @@ static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		mutex_init(&adev->sdma.instance[i].engine_reset_mutex);
-		/* Initialize guilty flags for GFX and PAGE queues */
-		adev->sdma.instance[i].gfx_guilty = false;
-		adev->sdma.instance[i].page_guilty = false;
 		adev->sdma.instance[i].funcs = &sdma_v4_4_2_sdma_funcs;
 
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		aid_id = adev->sdma.instance[i].aid_id;
 
@@ -1490,18 +1425,13 @@ static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block)
 		ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1;
 		ring->vm_hub = AMDGPU_MMHUB0(aid_id);
 		ring->no_user_submission = adev->sdma.no_user_submission;
-
-		sprintf(ring->name, "sdma%d.%d", aid_id,
-				i % adev->sdma.num_inst_per_aid);
-		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
-				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d.%d", aid_id,
+					  i % adev->sdma.num_inst_per_aid);
 		if (r)
 			return r;
 
 		if (adev->sdma.has_page_queue) {
 			ring = &adev->sdma.instance[i].page;
-			ring->ring_obj = NULL;
 			ring->use_doorbell = true;
 
 			/* doorbell index of page queue is assigned right after
@@ -1510,13 +1440,9 @@ static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block)
 			ring->doorbell_index =
 				(adev->doorbell_index.sdma_engine[i] + 1) << 1;
 			ring->vm_hub = AMDGPU_MMHUB0(aid_id);
-
-			sprintf(ring->name, "page%d.%d", aid_id,
-					i % adev->sdma.num_inst_per_aid);
-			r = amdgpu_ring_init(adev, ring, 1024,
-					     &adev->sdma.trap_irq,
-					     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-					     AMDGPU_RING_PRIO_DEFAULT, NULL);
+			r = amdgpu_sdma_ring_init(adev, ring, i, "page%d.%d",
+						  aid_id,
+						  i % adev->sdma.num_inst_per_aid);
 			if (r)
 				return r;
 		}
@@ -1627,21 +1553,6 @@ static int sdma_v4_4_2_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v4_4_2_hw_init(ip_block);
 }
 
-static bool sdma_v4_4_2_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32_SDMA(i, regSDMA_STATUS_REG);
-
-		if (!(tmp & SDMA_STATUS_REG__IDLE_MASK))
-			return false;
-	}
-
-	return true;
-}
-
 static int sdma_v4_4_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i, j;
@@ -1661,65 +1572,14 @@ static int sdma_v4_4_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
-static int sdma_v4_4_2_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-
-	return 0;
-}
-
-static bool sdma_v4_4_2_is_queue_selected(struct amdgpu_device *adev, uint32_t instance_id, bool is_page_queue)
-{
-	uint32_t reg_offset = is_page_queue ? regSDMA_PAGE_CONTEXT_STATUS : regSDMA_GFX_CONTEXT_STATUS;
-	uint32_t context_status = RREG32(sdma_v4_4_2_get_reg_offset(adev, instance_id, reg_offset));
-
-	/* Check if the SELECTED bit is set */
-	return (context_status & SDMA_GFX_CONTEXT_STATUS__SELECTED_MASK) != 0;
-}
-
-static int sdma_v4_4_2_reset_queue(struct amdgpu_ring *ring,
-				   unsigned int vmid,
-				   struct amdgpu_fence *timedout_fence)
-{
-	struct amdgpu_device *adev = ring->adev;
-	u32 id = ring->me;
-	int r;
-
-	amdgpu_amdkfd_suspend(adev, true);
-	r = amdgpu_sdma_reset_engine(adev, id, false);
-	amdgpu_amdkfd_resume(adev, true);
-	return r;
-}
-
 static int sdma_v4_4_2_stop_queue(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	u32 instance_id = ring->me;
 	u32 inst_mask;
-	uint64_t rptr;
 
 	if (amdgpu_sriov_vf(adev))
 		return -EINVAL;
 
-	/* Check if this queue is the guilty one */
-	adev->sdma.instance[instance_id].gfx_guilty =
-		sdma_v4_4_2_is_queue_selected(adev, instance_id, false);
-	if (adev->sdma.has_page_queue)
-		adev->sdma.instance[instance_id].page_guilty =
-			sdma_v4_4_2_is_queue_selected(adev, instance_id, true);
-
-	/* Cache the rptr before reset, after the reset,
-	* all of the registers will be reset to 0
-	*/
-	rptr = amdgpu_ring_get_rptr(ring);
-	ring->cached_rptr = rptr;
-	/* Cache the rptr for the page queue if it exists */
-	if (adev->sdma.has_page_queue) {
-		struct amdgpu_ring *page_ring = &adev->sdma.instance[instance_id].page;
-		rptr = amdgpu_ring_get_rptr(page_ring);
-		page_ring->cached_rptr = rptr;
-	}
-
 	/* stop queue */
 	inst_mask = 1 << ring->me;
 	sdma_v4_4_2_inst_gfx_stop(adev, inst_mask);
@@ -2118,9 +1978,7 @@ const struct amd_ip_funcs sdma_v4_4_2_ip_funcs = {
 	.hw_fini = sdma_v4_4_2_hw_fini,
 	.suspend = sdma_v4_4_2_suspend,
 	.resume = sdma_v4_4_2_resume,
-	.is_idle = sdma_v4_4_2_is_idle,
 	.wait_for_idle = sdma_v4_4_2_wait_for_idle,
-	.soft_reset = sdma_v4_4_2_soft_reset,
 	.set_clockgating_state = sdma_v4_4_2_set_clockgating_state,
 	.set_powergating_state = sdma_v4_4_2_set_powergating_state,
 	.get_clockgating_state = sdma_v4_4_2_get_clockgating_state,
@@ -2157,7 +2015,7 @@ static const struct amdgpu_ring_funcs sdma_v4_4_2_ring_funcs = {
 	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
 	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
-	.reset = sdma_v4_4_2_reset_queue,
+	.reset = amdgpu_sdma_reset_queue_legacy,
 };
 
 static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
@@ -2189,7 +2047,7 @@ static const struct amdgpu_ring_funcs sdma_v4_4_2_page_ring_funcs = {
 	.emit_wreg = sdma_v4_4_2_ring_emit_wreg,
 	.emit_reg_wait = sdma_v4_4_2_ring_emit_reg_wait,
 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
-	.reset = sdma_v4_4_2_reset_queue,
+	.reset = amdgpu_sdma_reset_queue_legacy,
 };
 
 static void sdma_v4_4_2_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index 1ca0e7f65442ea..c6349f0e632f46 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 
 #include "gc/gc_10_1_0_offset.h"
@@ -406,14 +407,14 @@ static void sdma_v5_0_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v5_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -574,18 +575,6 @@ static void sdma_v5_0_gfx_stop(struct amdgpu_device *adev, uint32_t inst_mask)
 	}
 }
 
-/**
- * sdma_v5_0_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues (NAVI10).
- */
-static void sdma_v5_0_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v5_0_ctx_switch_enable - stop the async dma engines context switch
  *
@@ -661,7 +650,6 @@ static void sdma_v5_0_enable(struct amdgpu_device *adev, bool enable)
 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
 	if (!enable) {
 		sdma_v5_0_gfx_stop(adev, 1 << inst_mask);
-		sdma_v5_0_rlc_stop(adev);
 	}
 
 	if (amdgpu_sriov_vf(adev))
@@ -858,19 +846,6 @@ static int sdma_v5_0_gfx_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-/**
- * sdma_v5_0_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them (NAVI10).
- * Returns 0 for success, error for failure.
- */
-static int sdma_v5_0_rlc_resume(struct amdgpu_device *adev)
-{
-	return 0;
-}
-
 /**
  * sdma_v5_0_load_microcode - load the sDMA ME ucode
  *
@@ -951,7 +926,6 @@ static int sdma_v5_0_start(struct amdgpu_device *adev)
 	r = sdma_v5_0_gfx_resume(adev);
 	if (r)
 		return r;
-	r = sdma_v5_0_rlc_resume(adev);
 
 	return r;
 }
@@ -1073,7 +1047,8 @@ static int sdma_v5_0_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v5_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	long r;
@@ -1081,7 +1056,6 @@ static int sdma_v5_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 
 	tmp = 0xCAFEDEAD;
-	memset(&ib, 0, sizeof(ib));
 
 	r = amdgpu_wb_get(adev, &index);
 	if (r) {
@@ -1092,27 +1066,32 @@ static int sdma_v5_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(tmp);
 
-	r = amdgpu_ib_get(adev, NULL, 256,
-			  AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
 		goto err0;
 	}
 
-	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1132,7 +1111,6 @@ static int sdma_v5_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1236,12 +1214,13 @@ static void sdma_v5_0_vm_set_pte_pde(struct amdgpu_ib *ib,
 static void sdma_v5_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 0x7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1407,7 +1386,6 @@ static int sdma_v5_0_sw_init(struct amdgpu_ip_block *ip_block)
 		mutex_init(&adev->sdma.instance[i].engine_reset_mutex);
 		adev->sdma.instance[i].funcs = &sdma_v5_0_sdma_funcs;
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 
 		DRM_DEBUG("SDMA %d use_doorbell being set to: [%s]\n", i,
@@ -1418,11 +1396,7 @@ static int sdma_v5_0_sw_init(struct amdgpu_ip_block *ip_block)
 			: (adev->doorbell_index.sdma_engine[1] << 1); // get DWORD offset
 
 		ring->vm_hub = AMDGPU_GFXHUB(0);
-		sprintf(ring->name, "sdma%d", i);
-		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
-				     (i == 0) ? AMDGPU_SDMA_IRQ_INSTANCE0 :
-				     AMDGPU_SDMA_IRQ_INSTANCE1,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d", i);
 		if (r)
 			return r;
 	}
@@ -1501,21 +1475,6 @@ static int sdma_v5_0_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v5_0_hw_init(ip_block);
 }
 
-static bool sdma_v5_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32(sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_STATUS_REG));
-
-		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
-			return false;
-	}
-
-	return true;
-}
-
 static int sdma_v5_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -1533,36 +1492,6 @@ static int sdma_v5_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
-static int sdma_v5_0_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-
-	return 0;
-}
-
-static int sdma_v5_0_reset_queue(struct amdgpu_ring *ring,
-				 unsigned int vmid,
-				 struct amdgpu_fence *timedout_fence)
-{
-	struct amdgpu_device *adev = ring->adev;
-	int r;
-
-	if (ring->me >= adev->sdma.num_instances) {
-		dev_err(adev->dev, "sdma instance not found\n");
-		return -EINVAL;
-	}
-
-	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
-
-	amdgpu_amdkfd_suspend(adev, true);
-	r = amdgpu_sdma_reset_engine(adev, ring->me, true);
-	amdgpu_amdkfd_resume(adev, true);
-	if (r)
-		return r;
-
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
-}
-
 static int sdma_v5_0_stop_queue(struct amdgpu_ring *ring)
 {
 	u32 f32_cntl, freeze, cntl, stat1_reg;
@@ -1635,11 +1564,9 @@ static int sdma_v5_0_ring_preempt_ib(struct amdgpu_ring *ring)
 {
 	int i, r = 0;
 	struct amdgpu_device *adev = ring->adev;
-	u32 index = 0;
 	u64 sdma_gfx_preempt;
 
-	amdgpu_sdma_get_index_from_ring(ring, &index);
-	if (index == 0)
+	if (ring->me == 0)
 		sdma_gfx_preempt = mmSDMA0_GFX_PREEMPT;
 	else
 		sdma_gfx_preempt = mmSDMA1_GFX_PREEMPT;
@@ -1916,9 +1843,7 @@ static const struct amd_ip_funcs sdma_v5_0_ip_funcs = {
 	.hw_fini = sdma_v5_0_hw_fini,
 	.suspend = sdma_v5_0_suspend,
 	.resume = sdma_v5_0_resume,
-	.is_idle = sdma_v5_0_is_idle,
 	.wait_for_idle = sdma_v5_0_wait_for_idle,
-	.soft_reset = sdma_v5_0_soft_reset,
 	.set_clockgating_state = sdma_v5_0_set_clockgating_state,
 	.set_powergating_state = sdma_v5_0_set_powergating_state,
 	.get_clockgating_state = sdma_v5_0_get_clockgating_state,
@@ -1960,7 +1885,7 @@ static const struct amdgpu_ring_funcs sdma_v5_0_ring_funcs = {
 	.emit_reg_write_reg_wait = sdma_v5_0_ring_emit_reg_write_reg_wait,
 	.init_cond_exec = sdma_v5_0_ring_init_cond_exec,
 	.preempt_ib = sdma_v5_0_ring_preempt_ib,
-	.reset = sdma_v5_0_reset_queue,
+	.reset = amdgpu_sdma_reset_queue_legacy,
 };
 
 static void sdma_v5_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index 81f1e9882177bb..683f27c276651c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 
 #include "gc/gc_10_3_0_offset.h"
@@ -254,14 +255,14 @@ static void sdma_v5_2_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v5_2_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -424,18 +425,6 @@ static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev,  uint32_t inst_mask)
 	}
 }
 
-/**
- * sdma_v5_2_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues.
- */
-static void sdma_v5_2_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v5_2_ctx_switch_enable - stop the async dma engines context switch
  *
@@ -510,7 +499,6 @@ static void sdma_v5_2_enable(struct amdgpu_device *adev, bool enable)
 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
 	if (!enable) {
 		sdma_v5_2_gfx_stop(adev, inst_mask);
-		sdma_v5_2_rlc_stop(adev);
 	}
 
 	if (!amdgpu_sriov_vf(adev)) {
@@ -705,19 +693,6 @@ static int sdma_v5_2_gfx_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-/**
- * sdma_v5_2_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them.
- * Returns 0 for success, error for failure.
- */
-static int sdma_v5_2_rlc_resume(struct amdgpu_device *adev)
-{
-	return 0;
-}
-
 /**
  * sdma_v5_2_load_microcode - load the sDMA ME ucode
  *
@@ -851,7 +826,6 @@ static int sdma_v5_2_start(struct amdgpu_device *adev)
 	r = sdma_v5_2_gfx_resume(adev);
 	if (r)
 		return r;
-	r = sdma_v5_2_rlc_resume(adev);
 
 	return r;
 }
@@ -973,7 +947,8 @@ static int sdma_v5_2_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	long r;
@@ -981,7 +956,6 @@ static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 
 	tmp = 0xCAFEDEAD;
-	memset(&ib, 0, sizeof(ib));
 
 	r = amdgpu_wb_get(adev, &index);
 	if (r) {
@@ -992,26 +966,32 @@ static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(tmp);
 
-	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
 		goto err0;
 	}
 
-	ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1031,7 +1011,6 @@ static int sdma_v5_2_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1136,12 +1115,13 @@ static void sdma_v5_2_vm_set_pte_pde(struct amdgpu_ib *ib,
 static void sdma_v5_2_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 0x7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1326,7 +1306,6 @@ static int sdma_v5_2_sw_init(struct amdgpu_ip_block *ip_block)
 		mutex_init(&adev->sdma.instance[i].engine_reset_mutex);
 		adev->sdma.instance[i].funcs = &sdma_v5_2_sdma_funcs;
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
 
@@ -1337,10 +1316,7 @@ static int sdma_v5_2_sw_init(struct amdgpu_ip_block *ip_block)
 			(adev->doorbell_index.sdma_engine[i] << 1); //get DWORD offset
 
 		ring->vm_hub = AMDGPU_GFXHUB(0);
-		sprintf(ring->name, "sdma%d", i);
-		r = amdgpu_ring_init(adev, ring, 1024, &adev->sdma.trap_irq,
-				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d", i);
 		if (r)
 			return r;
 	}
@@ -1417,21 +1393,6 @@ static int sdma_v5_2_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v5_2_hw_init(ip_block);
 }
 
-static bool sdma_v5_2_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_STATUS_REG));
-
-		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
-			return false;
-	}
-
-	return true;
-}
-
 static int sdma_v5_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -1451,29 +1412,6 @@ static int sdma_v5_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
-static int sdma_v5_2_reset_queue(struct amdgpu_ring *ring,
-				 unsigned int vmid,
-				 struct amdgpu_fence *timedout_fence)
-{
-	struct amdgpu_device *adev = ring->adev;
-	int r;
-
-	if (ring->me >= adev->sdma.num_instances) {
-		dev_err(adev->dev, "sdma instance not found\n");
-		return -EINVAL;
-	}
-
-	amdgpu_ring_reset_helper_begin(ring, timedout_fence);
-
-	amdgpu_amdkfd_suspend(adev, true);
-	r = amdgpu_sdma_reset_engine(adev, ring->me, true);
-	amdgpu_amdkfd_resume(adev, true);
-	if (r)
-		return r;
-
-	return amdgpu_ring_reset_helper_end(ring, timedout_fence);
-}
-
 static int sdma_v5_2_stop_queue(struct amdgpu_ring *ring)
 {
 	u32 f32_cntl, freeze, cntl, stat1_reg;
@@ -1549,12 +1487,10 @@ static int sdma_v5_2_ring_preempt_ib(struct amdgpu_ring *ring)
 {
 	int i, r = 0;
 	struct amdgpu_device *adev = ring->adev;
-	u32 index = 0;
 	u64 sdma_gfx_preempt;
 
-	amdgpu_sdma_get_index_from_ring(ring, &index);
-	sdma_gfx_preempt =
-		sdma_v5_2_get_reg_offset(adev, index, mmSDMA0_GFX_PREEMPT);
+	sdma_gfx_preempt = sdma_v5_2_get_reg_offset(adev, ring->me,
+						    mmSDMA0_GFX_PREEMPT);
 
 	/* assert preemption condition */
 	amdgpu_ring_set_preempt_cond_exec(ring, false);
@@ -1921,7 +1857,6 @@ static const struct amd_ip_funcs sdma_v5_2_ip_funcs = {
 	.hw_fini = sdma_v5_2_hw_fini,
 	.suspend = sdma_v5_2_suspend,
 	.resume = sdma_v5_2_resume,
-	.is_idle = sdma_v5_2_is_idle,
 	.wait_for_idle = sdma_v5_2_wait_for_idle,
 	.soft_reset = sdma_v5_2_soft_reset,
 	.set_clockgating_state = sdma_v5_2_set_clockgating_state,
@@ -1967,7 +1902,7 @@ static const struct amdgpu_ring_funcs sdma_v5_2_ring_funcs = {
 	.emit_reg_write_reg_wait = sdma_v5_2_ring_emit_reg_write_reg_wait,
 	.init_cond_exec = sdma_v5_2_ring_init_cond_exec,
 	.preempt_ib = sdma_v5_2_ring_preempt_ib,
-	.reset = sdma_v5_2_reset_queue,
+	.reset = amdgpu_sdma_reset_queue_legacy,
 };
 
 static void sdma_v5_2_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index cf3d2997fff883..16649dba6329d2 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 
 #include "gc/gc_11_0_0_offset.h"
@@ -242,14 +243,14 @@ static void sdma_v6_0_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v6_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /*
@@ -407,18 +408,6 @@ static void sdma_v6_0_gfx_stop(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * sdma_v6_0_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues.
- */
-static void sdma_v6_0_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v6_0_ctxempty_int_enable - enable or disable context empty interrupts
  *
@@ -457,7 +446,6 @@ static void sdma_v6_0_enable(struct amdgpu_device *adev, bool enable)
 
 	if (!enable) {
 		sdma_v6_0_gfx_stop(adev);
-		sdma_v6_0_rlc_stop(adev);
 	}
 
 	if (amdgpu_sriov_vf(adev))
@@ -642,19 +630,6 @@ static int sdma_v6_0_gfx_resume(struct amdgpu_device *adev)
 	return 0;
 }
 
-/**
- * sdma_v6_0_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them.
- * Returns 0 for success, error for failure.
- */
-static int sdma_v6_0_rlc_resume(struct amdgpu_device *adev)
-{
-	return 0;
-}
-
 /**
  * sdma_v6_0_load_microcode - load the sDMA ME ucode
  *
@@ -832,7 +807,6 @@ static int sdma_v6_0_start(struct amdgpu_device *adev)
 	r = sdma_v6_0_gfx_resume(adev);
 	if (r)
 		return r;
-	r = sdma_v6_0_rlc_resume(adev);
 
 	return r;
 }
@@ -964,7 +938,8 @@ static int sdma_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	long r;
@@ -972,7 +947,6 @@ static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 
 	tmp = 0xCAFEDEAD;
-	memset(&ib, 0, sizeof(ib));
 
 	r = amdgpu_wb_get(adev, &index);
 	if (r) {
@@ -983,26 +957,32 @@ static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(tmp);
 
-	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
 		goto err0;
 	}
 
-	ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1022,7 +1002,6 @@ static int sdma_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1126,12 +1105,13 @@ static void sdma_v6_0_vm_set_pte_pde(struct amdgpu_ib *ib,
 static void sdma_v6_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 0x7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1330,7 +1310,6 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
 		ring->no_user_submission = adev->sdma.no_user_submission;
@@ -1342,11 +1321,8 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
 
 		ring->vm_hub = AMDGPU_GFXHUB(0);
-		sprintf(ring->name, "sdma%d", i);
-		r = amdgpu_ring_init(adev, ring, 1024,
-				     &adev->sdma.trap_irq,
-				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d", i);
 		if (r)
 			return r;
 	}
@@ -1489,21 +1465,6 @@ static int sdma_v6_0_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v6_0_hw_init(ip_block);
 }
 
-static bool sdma_v6_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32(sdma_v6_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
-
-		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
-			return false;
-	}
-
-	return true;
-}
-
 static int sdma_v6_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -1525,12 +1486,10 @@ static int sdma_v6_0_ring_preempt_ib(struct amdgpu_ring *ring)
 {
 	int i, r = 0;
 	struct amdgpu_device *adev = ring->adev;
-	u32 index = 0;
 	u64 sdma_gfx_preempt;
 
-	amdgpu_sdma_get_index_from_ring(ring, &index);
-	sdma_gfx_preempt =
-		sdma_v6_0_get_reg_offset(adev, index, regSDMA0_QUEUE0_PREEMPT);
+	sdma_gfx_preempt = sdma_v6_0_get_reg_offset(adev, ring->me,
+						    regSDMA0_QUEUE0_PREEMPT);
 
 	/* assert preemption condition */
 	amdgpu_ring_set_preempt_cond_exec(ring, false);
@@ -1566,6 +1525,34 @@ static int sdma_v6_0_ring_preempt_ib(struct amdgpu_ring *ring)
 	return r;
 }
 
+/* dword stride between adjacent per-queue register banks */
+#define SDMA_V6_0_QUEUE_REG_STRIDE \
+	(regSDMA0_QUEUE1_RB_CNTL - regSDMA0_QUEUE0_RB_CNTL)
+
+/* find the HW slot (instance, queue_id) whose doorbell matches doorbell_index */
+static bool sdma_v6_0_detect_hung_queue(struct amdgpu_device *adev,
+					u32 doorbell_index,
+					u32 *instance_id, u32 *queue_id)
+{
+	u32 i, q, reg, dboff;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		for (q = 0; q < 8; q++) {
+			reg = sdma_v6_0_get_reg_offset(adev, i,
+				regSDMA0_QUEUE0_DOORBELL_OFFSET +
+				q * SDMA_V6_0_QUEUE_REG_STRIDE);
+			dboff = (RREG32(reg) &
+				 SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET_MASK) >> 2;
+			if (dboff == doorbell_index) {
+				*instance_id = i;
+				*queue_id = q;
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
 static int sdma_v6_0_reset_queue(struct amdgpu_ring *ring,
 				 unsigned int vmid,
 				 struct amdgpu_fence *timedout_fence)
@@ -1727,7 +1714,6 @@ const struct amd_ip_funcs sdma_v6_0_ip_funcs = {
 	.hw_fini = sdma_v6_0_hw_fini,
 	.suspend = sdma_v6_0_suspend,
 	.resume = sdma_v6_0_resume,
-	.is_idle = sdma_v6_0_is_idle,
 	.wait_for_idle = sdma_v6_0_wait_for_idle,
 	.soft_reset = sdma_v6_0_soft_reset,
 	.set_clockgating_state = sdma_v6_0_set_clockgating_state,
@@ -1773,6 +1759,10 @@ static const struct amdgpu_ring_funcs sdma_v6_0_ring_funcs = {
 	.reset = sdma_v6_0_reset_queue,
 };
 
+static const struct amdgpu_sdma_funcs sdma_v6_0_sdma_funcs = {
+	.detect_hung_queue = sdma_v6_0_detect_hung_queue,
+};
+
 static void sdma_v6_0_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
@@ -1780,6 +1770,7 @@ static void sdma_v6_0_set_ring_funcs(struct amdgpu_device *adev)
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		adev->sdma.instance[i].ring.funcs = &sdma_v6_0_ring_funcs;
 		adev->sdma.instance[i].ring.me = i;
+		adev->sdma.instance[i].funcs = &sdma_v6_0_sdma_funcs;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 69cb89298a3e0b..edd9c437f7cb13 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 
 #include "gc/gc_12_0_0_offset.h"
@@ -244,14 +245,14 @@ static void sdma_v7_0_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -409,18 +410,6 @@ static void sdma_v7_0_gfx_stop(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * sdma_v7_0_rlc_stop - stop the compute async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Stop the compute async dma queues.
- */
-static void sdma_v7_0_rlc_stop(struct amdgpu_device *adev)
-{
-	/* XXX todo */
-}
-
 /**
  * sdma_v7_0_ctx_switch_enable - stop the async dma engines context switch
  *
@@ -448,7 +437,6 @@ static void sdma_v7_0_enable(struct amdgpu_device *adev, bool enable)
 
 	if (!enable) {
 		sdma_v7_0_gfx_stop(adev);
-		sdma_v7_0_rlc_stop(adev);
 	}
 
 	if (amdgpu_sriov_vf(adev))
@@ -643,19 +631,6 @@ static int sdma_v7_0_gfx_resume(struct amdgpu_device *adev)
 
 }
 
-/**
- * sdma_v7_0_rlc_resume - setup and start the async dma engines
- *
- * @adev: amdgpu_device pointer
- *
- * Set up the compute DMA queues and enable them.
- * Returns 0 for success, error for failure.
- */
-static int sdma_v7_0_rlc_resume(struct amdgpu_device *adev)
-{
-	return 0;
-}
-
 static void sdma_v12_0_free_ucode_buffer(struct amdgpu_device *adev)
 {
 	int i;
@@ -784,6 +759,34 @@ static int sdma_v7_0_soft_reset(struct amdgpu_ip_block *ip_block)
 	return sdma_v7_0_start(adev);
 }
 
+/* dword stride between adjacent per-queue register banks */
+#define SDMA_V7_0_QUEUE_REG_STRIDE \
+	(regSDMA0_QUEUE1_RB_CNTL - regSDMA0_QUEUE0_RB_CNTL)
+
+/* find the HW slot (instance, queue_id) whose doorbell matches doorbell_index */
+static bool sdma_v7_0_detect_hung_queue(struct amdgpu_device *adev,
+					u32 doorbell_index,
+					u32 *instance_id, u32 *queue_id)
+{
+	u32 i, q, reg, dboff;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		for (q = 0; q < 8; q++) {
+			reg = sdma_v7_0_get_reg_offset(adev, i,
+				regSDMA0_QUEUE0_DOORBELL_OFFSET +
+				q * SDMA_V7_0_QUEUE_REG_STRIDE);
+			dboff = (RREG32(reg) &
+				 SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET_MASK) >> 2;
+			if (dboff == doorbell_index) {
+				*instance_id = i;
+				*queue_id = q;
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
 static int sdma_v7_0_reset_queue(struct amdgpu_ring *ring,
 				 unsigned int vmid,
 				 struct amdgpu_fence *timedout_fence)
@@ -850,7 +853,6 @@ static int sdma_v7_0_start(struct amdgpu_device *adev)
 	r = sdma_v7_0_gfx_resume(adev);
 	if (r)
 		return r;
-	r = sdma_v7_0_rlc_resume(adev);
 
 	return r;
 }
@@ -979,7 +981,8 @@ static int sdma_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	long r;
@@ -987,7 +990,6 @@ static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 
 	tmp = 0xCAFEDEAD;
-	memset(&ib, 0, sizeof(ib));
 
 	r = amdgpu_wb_get(adev, &index);
 	if (r) {
@@ -998,26 +1000,32 @@ static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(tmp);
 
-	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r) {
 		drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r);
 		goto err0;
 	}
 
-	ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1037,7 +1045,6 @@ static int sdma_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1145,12 +1152,13 @@ static void sdma_v7_0_vm_set_pte_pde(struct amdgpu_ib *ib,
 static void sdma_v7_0_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 0x7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1314,7 +1322,6 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
 		ring->no_user_submission = adev->sdma.no_user_submission;
@@ -1326,11 +1333,7 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
 
 		ring->vm_hub = AMDGPU_GFXHUB(0);
-		sprintf(ring->name, "sdma%d", i);
-		r = amdgpu_ring_init(adev, ring, 1024,
-				     &adev->sdma.trap_irq,
-				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d", i);
 		if (r)
 			return r;
 	}
@@ -1443,21 +1446,6 @@ static int sdma_v7_0_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v7_0_hw_init(ip_block);
 }
 
-static bool sdma_v7_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32(sdma_v7_0_get_reg_offset(adev, i, regSDMA0_STATUS_REG));
-
-		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
-			return false;
-	}
-
-	return true;
-}
-
 static int sdma_v7_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -1479,12 +1467,10 @@ static int sdma_v7_0_ring_preempt_ib(struct amdgpu_ring *ring)
 {
 	int i, r = 0;
 	struct amdgpu_device *adev = ring->adev;
-	u32 index = 0;
 	u64 sdma_gfx_preempt;
 
-	amdgpu_sdma_get_index_from_ring(ring, &index);
-	sdma_gfx_preempt =
-		sdma_v7_0_get_reg_offset(adev, index, regSDMA0_QUEUE0_PREEMPT);
+	sdma_gfx_preempt = sdma_v7_0_get_reg_offset(adev, ring->me,
+						    regSDMA0_QUEUE0_PREEMPT);
 
 	/* assert preemption condition */
 	amdgpu_ring_set_preempt_cond_exec(ring, false);
@@ -1659,7 +1645,6 @@ const struct amd_ip_funcs sdma_v7_0_ip_funcs = {
 	.hw_fini = sdma_v7_0_hw_fini,
 	.suspend = sdma_v7_0_suspend,
 	.resume = sdma_v7_0_resume,
-	.is_idle = sdma_v7_0_is_idle,
 	.wait_for_idle = sdma_v7_0_wait_for_idle,
 	.soft_reset = sdma_v7_0_soft_reset,
 	.set_clockgating_state = sdma_v7_0_set_clockgating_state,
@@ -1705,6 +1690,10 @@ static const struct amdgpu_ring_funcs sdma_v7_0_ring_funcs = {
 	.reset = sdma_v7_0_reset_queue,
 };
 
+static const struct amdgpu_sdma_funcs sdma_v7_0_sdma_funcs = {
+	.detect_hung_queue = sdma_v7_0_detect_hung_queue,
+};
+
 static void sdma_v7_0_set_ring_funcs(struct amdgpu_device *adev)
 {
 	int i;
@@ -1712,6 +1701,7 @@ static void sdma_v7_0_set_ring_funcs(struct amdgpu_device *adev)
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		adev->sdma.instance[i].ring.funcs = &sdma_v7_0_ring_funcs;
 		adev->sdma.instance[i].ring.me = i;
+		adev->sdma.instance[i].funcs = &sdma_v7_0_sdma_funcs;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
index 18366e16ef3faa..321c67e014435c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
@@ -28,6 +28,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_trace.h"
 
 #include "gc/gc_12_1_0_offset.h"
@@ -238,14 +239,14 @@ static void sdma_v7_1_ring_set_wptr(struct amdgpu_ring *ring)
 static void sdma_v7_1_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
-	int i;
+	const u32 nop = ring->funcs->nop;
 
-	for (i = 0; i < count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
-			amdgpu_ring_write(ring, ring->funcs->nop |
-				SDMA_PKT_NOP_HEADER_COUNT(count - 1));
-		else
-			amdgpu_ring_write(ring, ring->funcs->nop);
+	if (count && sdma->burst_nop) {
+		--count;
+		amdgpu_ring_write(ring, nop | SDMA_PKT_NOP_HEADER_COUNT(count));
+	}
+
+	amdgpu_ring_fill(ring, nop, count);
 }
 
 /**
@@ -551,11 +552,16 @@ static int sdma_v7_1_gfx_resume_instance(struct amdgpu_device *adev, int i, bool
 
 	/* Set up sdma hang watchdog */
 	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL));
-	/* 100ms per unit */
-	temp = REG_SET_FIELD(temp, SDMA0_SDMA_WATCHDOG_CNTL, QUEUE_HANG_COUNT,
-			     max(adev->usec_timeout/100000, 1));
+	/* Disable sdma hang watchdog, need revist this when issue resolved.
+	Once issue resoved, the QUEUE_HANG_COUNT should be set to max(adev->usec_timeout/100000,1)
+	*/
+	temp = REG_SET_FIELD(temp, SDMA0_SDMA_WATCHDOG_CNTL, QUEUE_HANG_COUNT, 0);
+	temp = REG_SET_FIELD(temp, SDMA0_SDMA_WATCHDOG_CNTL, CMD_TIMEOUT_COUNT, 0);
 	WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL), temp);
 
+	dev_dbg(adev->dev, "Disable SDMA Hang WatchDog, regSDMA0_SDMA_WATCHDOG_CNTL = %d\n",
+		RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_WATCHDOG_CNTL)));
+
 	/* Set up RESP_MODE to non-copy addresses */
 	temp = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_UTCL1_CNTL));
 	temp = REG_SET_FIELD(temp, SDMA0_SDMA_UTCL1_CNTL, RESP_MODE, 3);
@@ -969,7 +975,8 @@ static int sdma_v7_1_ring_test_ring(struct amdgpu_ring *ring)
 static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	long r;
@@ -977,7 +984,6 @@ static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 
 	tmp = 0xCAFEDEAD;
-	memset(&ib, 0, sizeof(ib));
 
 	r = amdgpu_wb_get(adev, &index);
 	if (r) {
@@ -988,26 +994,32 @@ static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	adev->wb.wb[index] = cpu_to_le32(tmp);
 
-	r = amdgpu_ib_get(adev, NULL, 256, AMDGPU_IB_POOL_DIRECT, &ib);
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r) {
 		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
 		goto err0;
 	}
 
-	ib.ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
+	ib = &job->ibs[0];
+	ib->ptr[0] = SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_WRITE) |
 		SDMA_PKT_COPY_LINEAR_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr);
-	ib.ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
-	ib.ptr[4] = 0xDEADBEEF;
-	ib.ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
-	ib.length_dw = 8;
-
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr);
+	ib->ptr[3] = SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(0);
+	ib->ptr[4] = 0xDEADBEEF;
+	ib->ptr[5] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[6] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP);
+	ib->length_dw = 8;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -1027,7 +1039,6 @@ static int sdma_v7_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -1151,12 +1162,13 @@ static void sdma_v7_1_vm_set_pte_pde(struct amdgpu_ib *ib,
 static void sdma_v7_1_ring_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib)
 {
 	struct amdgpu_sdma_instance *sdma = amdgpu_sdma_get_instance_from_ring(ring);
+	const bool burst_nop = sdma->burst_nop;
 	u32 pad_count;
 	int i;
 
 	pad_count = (-ib->length_dw) & 0x7;
 	for (i = 0; i < pad_count; i++)
-		if (sdma && sdma->burst_nop && (i == 0))
+		if (i == 0 && burst_nop)
 			ib->ptr[ib->length_dw++] =
 				SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_NOP) |
 				SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1);
@@ -1295,7 +1307,6 @@ static int sdma_v7_1_sw_init(struct amdgpu_ip_block *ip_block)
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
-		ring->ring_obj = NULL;
 		ring->use_doorbell = true;
 		ring->me = i;
 		ring->no_user_submission = adev->sdma.no_user_submission;
@@ -1313,12 +1324,9 @@ static int sdma_v7_1_sw_init(struct amdgpu_ip_block *ip_block)
 			(adev->doorbell_index.sdma_engine[i] << 1); // get DWORD offset
 
 		ring->vm_hub = AMDGPU_GFXHUB(xcc_id);
-		sprintf(ring->name, "sdma%d.%d", xcc_id,
-				GET_INST(SDMA0, i) % adev->sdma.num_inst_per_xcc);
-		r = amdgpu_ring_init(adev, ring, 1024,
-				     &adev->sdma.trap_irq,
-				     AMDGPU_SDMA_IRQ_INSTANCE0 + i,
-				     AMDGPU_RING_PRIO_DEFAULT, NULL);
+		r = amdgpu_sdma_ring_init(adev, ring, i, "sdma%d.%d", xcc_id,
+					  GET_INST(SDMA0, i) %
+					  adev->sdma.num_inst_per_xcc);
 		if (r)
 			return r;
 	}
@@ -1365,6 +1373,22 @@ static int sdma_v7_1_sw_fini(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
+#define regSDMA0_SDMA_FE_CNTL0			0x10
+#define regSDMA0_SDMA_FE_CNTL0_BASE_IDX		0
+static void sdma_v7_1_rb_cmd_switch(struct amdgpu_device *adev,
+				    uint32_t inst_mask)
+{
+	int i, fe_cntl;
+	
+	if (adev->sdma.sdma_debug) {
+		for_each_inst(i, inst_mask) {
+			fe_cntl = RREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FE_CNTL0));
+			fe_cntl &= 0x7fffffff;
+			WREG32_SOC15_IP(GC, sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_FE_CNTL0), fe_cntl);
+		}
+	}
+}
+
 static int sdma_v7_1_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -1373,6 +1397,8 @@ static int sdma_v7_1_hw_init(struct amdgpu_ip_block *ip_block)
 
 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
 
+	sdma_v7_1_rb_cmd_switch(adev, inst_mask);
+
 	r = sdma_v7_1_inst_start(adev, inst_mask);
 	if (r)
 		return r;
@@ -1404,21 +1430,6 @@ static int sdma_v7_1_resume(struct amdgpu_ip_block *ip_block)
 	return sdma_v7_1_hw_init(ip_block);
 }
 
-static bool sdma_v7_1_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32(sdma_v7_1_get_reg_offset(adev, i, regSDMA0_SDMA_STATUS_REG));
-
-		if (!(tmp & SDMA0_SDMA_STATUS_REG__IDLE_MASK))
-			return false;
-	}
-
-	return true;
-}
-
 static int sdma_v7_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i, j;
@@ -1443,12 +1454,10 @@ static int sdma_v7_1_ring_preempt_ib(struct amdgpu_ring *ring)
 {
 	int i, r = 0;
 	struct amdgpu_device *adev = ring->adev;
-	u32 index = 0;
 	u64 sdma_gfx_preempt;
 
-	amdgpu_sdma_get_index_from_ring(ring, &index);
-	sdma_gfx_preempt =
-		sdma_v7_1_get_reg_offset(adev, index, regSDMA0_SDMA_QUEUE0_PREEMPT);
+	sdma_gfx_preempt = sdma_v7_1_get_reg_offset(adev, ring->me,
+						    regSDMA0_SDMA_QUEUE0_PREEMPT);
 
 	/* assert preemption condition */
 	amdgpu_ring_set_preempt_cond_exec(ring, false);
@@ -1624,7 +1633,6 @@ const struct amd_ip_funcs sdma_v7_1_ip_funcs = {
 	.hw_fini = sdma_v7_1_hw_fini,
 	.suspend = sdma_v7_1_suspend,
 	.resume = sdma_v7_1_resume,
-	.is_idle = sdma_v7_1_is_idle,
 	.wait_for_idle = sdma_v7_1_wait_for_idle,
 	.soft_reset = sdma_v7_1_soft_reset,
 	.set_clockgating_state = sdma_v7_1_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 7100f30ffeffac..0adfd4b101aaf8 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -2647,11 +2647,6 @@ static int si_common_resume(struct amdgpu_ip_block *ip_block)
 	return si_common_hw_init(ip_block);
 }
 
-static bool si_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int si_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 					    enum amd_clockgating_state state)
 {
@@ -2670,7 +2665,6 @@ static const struct amd_ip_funcs si_common_ip_funcs = {
 	.hw_init = si_common_hw_init,
 	.hw_fini = si_common_hw_fini,
 	.resume = si_common_resume,
-	.is_idle = si_common_is_idle,
 	.set_clockgating_state = si_common_set_clockgating_state,
 	.set_powergating_state = si_common_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 47f1d325bd1aff..31ab701d45d323 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -258,7 +258,8 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
 static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
 	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_ib ib;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
 	struct dma_fence *f = NULL;
 	unsigned index;
 	u32 tmp = 0;
@@ -272,20 +273,26 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	gpu_addr = adev->wb.gpu_addr + (index * 4);
 	tmp = 0xCAFEDEAD;
 	adev->wb.wb[index] = cpu_to_le32(tmp);
-	memset(&ib, 0, sizeof(ib));
-	r = amdgpu_ib_get(adev, NULL, 256,
-					AMDGPU_IB_POOL_DIRECT, &ib);
+
+	r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 256,
+				     AMDGPU_IB_POOL_DIRECT,
+				     AMDGPU_KERNEL_JOB_ID_SDMA_RING_TEST,
+				     &job);
 	if (r)
 		goto err0;
 
-	ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1);
-	ib.ptr[1] = lower_32_bits(gpu_addr);
-	ib.ptr[2] = upper_32_bits(gpu_addr) & 0xff;
-	ib.ptr[3] = 0xDEADBEEF;
-	ib.length_dw = 4;
-	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
-	if (r)
+	ib = &job->ibs[0];
+	ib->ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, 1);
+	ib->ptr[1] = lower_32_bits(gpu_addr);
+	ib->ptr[2] = upper_32_bits(gpu_addr) & 0xff;
+	ib->ptr[3] = 0xDEADBEEF;
+	ib->length_dw = 4;
+
+	r = amdgpu_job_submit_direct(job, ring, &f);
+	if (r) {
+		amdgpu_job_free(job);
 		goto err1;
+	}
 
 	r = dma_fence_wait_timeout(f, false, timeout);
 	if (r == 0) {
@@ -301,7 +308,6 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 		r = -EINVAL;
 
 err1:
-	amdgpu_ib_free(&ib, NULL);
 	dma_fence_put(f);
 err0:
 	amdgpu_wb_free(adev, index);
@@ -594,12 +600,6 @@ static int si_dma_wait_for_idle(struct amdgpu_ip_block *ip_block)
 	return -ETIMEDOUT;
 }
 
-static int si_dma_soft_reset(struct amdgpu_ip_block *ip_block)
-{
-	drm_info(adev_to_drm(ip_block->adev), "si_dma_soft_reset --- not implemented !!!!!!!\n");
-	return 0;
-}
-
 static int si_dma_set_trap_irq_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *src,
 					unsigned type,
@@ -725,9 +725,7 @@ static const struct amd_ip_funcs si_dma_ip_funcs = {
 	.hw_fini = si_dma_hw_fini,
 	.suspend = si_dma_suspend,
 	.resume = si_dma_resume,
-	.is_idle = si_dma_is_idle,
 	.wait_for_idle = si_dma_wait_for_idle,
-	.soft_reset = si_dma_soft_reset,
 	.set_clockgating_state = si_dma_set_clockgating_state,
 	.set_powergating_state = si_dma_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index 66f650f872435a..52118765dccf3c 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -297,7 +297,6 @@ static const struct amd_ip_funcs si_ih_ip_funcs = {
 	.hw_fini = si_ih_hw_fini,
 	.suspend = si_ih_suspend,
 	.resume = si_ih_resume,
-	.is_idle = si_ih_is_idle,
 	.wait_for_idle = si_ih_wait_for_idle,
 	.soft_reset = si_ih_soft_reset,
 	.set_clockgating_state = si_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 3d4573d9374220..b2024d05aa5fd1 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -1324,11 +1324,6 @@ static int soc15_common_resume(struct amdgpu_ip_block *ip_block)
 	return soc15_common_hw_init(ip_block);
 }
 
-static bool soc15_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static void soc15_update_drm_clock_gating(struct amdgpu_device *adev, bool enable)
 {
 	uint32_t def, data;
@@ -1480,7 +1475,6 @@ static const struct amd_ip_funcs soc15_common_ip_funcs = {
 	.hw_fini = soc15_common_hw_fini,
 	.suspend = soc15_common_suspend,
 	.resume = soc15_common_resume,
-	.is_idle = soc15_common_is_idle,
 	.set_clockgating_state = soc15_common_set_clockgating_state,
 	.set_powergating_state = soc15_common_set_powergating_state,
 	.get_clockgating_state= soc15_common_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index 47e0329b6f3f47..937d02ef71248f 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -26,11 +26,9 @@
 
 /* GET_INST returns the physical instance corresponding to a logical instance */
 #define GET_INST(ip, inst) \
-	(adev->ip_map.logical_to_dev_inst ? \
-	adev->ip_map.logical_to_dev_inst(adev, ip##_HWIP, inst) : inst)
+	amdgpu_ip_map_logical_to_dev_inst(&adev->ip_map, ip##_HWIP, inst)
 #define GET_MASK(ip, mask) \
-	(adev->ip_map.logical_to_dev_mask ? \
-	adev->ip_map.logical_to_dev_mask(adev, ip##_HWIP, mask) : mask)
+	amdgpu_ip_map_logical_to_dev_mask(&adev->ip_map, ip##_HWIP, mask)
 
 /* Register Access Macros */
 #define SOC15_REG_OFFSET(ip, inst, reg)	(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg)
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index 5a6c84c802f6af..599d47e1d38b6e 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -1042,11 +1042,6 @@ static int soc21_common_resume(struct amdgpu_ip_block *ip_block)
 	return soc21_common_hw_init(ip_block);
 }
 
-static bool soc21_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int soc21_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 					   enum amd_clockgating_state state)
 {
@@ -1111,7 +1106,6 @@ static const struct amd_ip_funcs soc21_common_ip_funcs = {
 	.hw_fini = soc21_common_hw_fini,
 	.suspend = soc21_common_suspend,
 	.resume = soc21_common_resume,
-	.is_idle = soc21_common_is_idle,
 	.set_clockgating_state = soc21_common_set_clockgating_state,
 	.set_powergating_state = soc21_common_set_powergating_state,
 	.get_clockgating_state = soc21_common_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c
index 9a658e4837f481..6c8b8ed4a44d4c 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc24.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc24.c
@@ -519,11 +519,6 @@ static int soc24_common_resume(struct amdgpu_ip_block *ip_block)
 	return soc24_common_hw_init(ip_block);
 }
 
-static bool soc24_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int soc24_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 					      enum amd_clockgating_state state)
 {
@@ -582,7 +577,6 @@ static const struct amd_ip_funcs soc24_common_ip_funcs = {
 	.hw_fini = soc24_common_hw_fini,
 	.suspend = soc24_common_suspend,
 	.resume = soc24_common_resume,
-	.is_idle = soc24_common_is_idle,
 	.set_clockgating_state = soc24_common_set_clockgating_state,
 	.set_powergating_state = soc24_common_set_powergating_state,
 	.get_clockgating_state = soc24_common_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
index 43e92e2a85e899..28eca48c659692 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
@@ -21,6 +21,7 @@
  *
  */
 #include "amdgpu.h"
+#include "amdgpu_discovery.h"
 #include "soc15.h"
 #include "soc15_common.h"
 #include "soc_v1_0.h"
@@ -30,6 +31,7 @@
 #include "sdma_v7_1.h"
 #include "gfx_v12_1.h"
 #include "amdgpu_video_codecs.h"
+#include "amdgpu_reset.h"
 
 #include "gc/gc_12_1_0_offset.h"
 #include "gc/gc_12_1_0_sh_mask.h"
@@ -47,6 +49,23 @@
 #define NORMALIZE_MID_REG_OFFSET(offset) \
 		(offset & 0x3FFFF)
 
+/*
+ * die_info[0].die_id from IP discovery encodes the silicon revision:
+ *   bit[15:12] Reserved
+ *   bit[11:8] MID revision
+ *   bit[7:4]  AID revision
+ *   bit[3:0]  XCD revision
+ * The value is copied into adev->rev_id[15:0] (rev_id[31:16] are 0) and mapped
+ * to adev->external_rev_id below.
+ */
+#define SOC_V1_0_DIE_REV_XCD__SHIFT 0
+#define SOC_V1_0_DIE_REV_AID__SHIFT 4
+#define SOC_V1_0_DIE_REV_MID__SHIFT 8
+#define SOC_V1_0_DIE_REV(mid, aid, xcd)           \
+	(((mid) << SOC_V1_0_DIE_REV_MID__SHIFT) | \
+	 ((aid) << SOC_V1_0_DIE_REV_AID__SHIFT) | \
+	 ((xcd) << SOC_V1_0_DIE_REV_XCD__SHIFT))
+
 static const struct amdgpu_video_codecs vcn_5_0_2_video_codecs_encode_vcn0 = {
 	.codec_count = 0,
 	.codec_array = NULL,
@@ -149,7 +168,11 @@ static u32 soc_v1_0_get_config_memsize(struct amdgpu_device *adev)
 
 static u32 soc_v1_0_get_xclk(struct amdgpu_device *adev)
 {
-	return adev->clock.spll.reference_freq;
+	/* 100MHz is the default */
+	if (!adev->bios)
+		return 10000;
+	else
+		return adev->clock.spll.reference_freq;
 }
 
 void soc_v1_0_grbm_select(struct amdgpu_device *adev,
@@ -258,6 +281,54 @@ static int soc_v1_0_asic_reset(struct amdgpu_device *adev)
 	return 0;
 }
 
+/*
+ * Function returns a pair of (size, offset_within_vram) to
+ * tell caller to skip the *reserve_size bytes starting at *offset inside VRAM.
+ */
+static void soc_v1_0_get_fw_reserved_info(struct amdgpu_device *adev,
+					  u64 *reserve_size,
+					  u64 *offset)
+{
+	struct mem_reserved_info umf_info, vram_info;
+	u64 vram_base;
+
+	if (amdgpu_discovery_get_mem_reserved_region_by_id(adev,
+							   MASTER_DIE_UMF_REGION_ID, &umf_info)) {
+		dev_warn(adev->dev,
+			 "MASTER_DIE_UMF region not found in discovery\n");
+		return;
+	}
+	/*
+	 * If SPECIFIC_PURPOSE_REGION exists and non-zero, use it as the VRAM base.
+	 * In multi-die layouts, VRAM may be placed at non-default bases.
+	 * Prefer to query SPECIFIC_PURPOSE_REGION to get the vram_base.
+	 */
+	if (!amdgpu_discovery_get_mem_reserved_region_by_id(adev,
+			SPECIFIC_PURPOSE_REGION_ID, &vram_info) &&
+	    vram_info.reserved_region_size)
+		vram_base = vram_info.reserved_region_start;
+	else
+		vram_base = adev->gmc.vram_start;
+	dev_dbg(adev->dev, "%s: vram_base=0x%llx\n", __func__, vram_base);
+
+	if (umf_info.reserved_region_size == 0 ||
+	    umf_info.reserved_region_start < vram_base ||
+	    umf_info.reserved_region_start + umf_info.reserved_region_size >
+			vram_base + adev->gmc.real_vram_size) {
+		dev_warn(adev->dev,
+			 "MASTER_DIE_UMF region out of VRAM: start=0x%llx size=0x%llx vram=[0x%llx,+0x%llx)\n",
+			 umf_info.reserved_region_start, umf_info.reserved_region_size,
+			 vram_base, adev->gmc.real_vram_size);
+		return;
+	}
+	/*
+	 * *offset is the distance from the start of VRAM to the start of the
+	 * UMF carveout.
+	 */
+	*reserve_size = umf_info.reserved_region_size;
+	*offset = umf_info.reserved_region_start - vram_base;
+}
+
 static const struct amdgpu_asic_funcs soc_v1_0_asic_funcs = {
 	.read_bios_from_rom = &amdgpu_soc15_read_bios_from_rom,
 	.read_register = &soc_v1_0_read_register,
@@ -269,11 +340,63 @@ static const struct amdgpu_asic_funcs soc_v1_0_asic_funcs = {
 	.reset = soc_v1_0_asic_reset,
 	.reset_method = &soc_v1_0_asic_reset_method,
 	.query_video_codecs = &soc_v1_0_query_video_codecs,
+	.get_fw_reserved_info = &soc_v1_0_get_fw_reserved_info,
 };
 
+enum soc_v1_0_external_rev_id {
+	SOC_V1_0_MID_A0_AID_A0_XCD_A0 = 0x1,
+	SOC_V1_0_MID_A0_AID_A0_XCD_B0 = 0x2,
+	SOC_V1_0_MID_A0_AID_A1_XCD_A0 = 0x3,
+	SOC_V1_0_MID_A0_AID_A1_XCD_B0 = 0x4,
+};
+
+static int soc_v1_0_set_rev_id(struct amdgpu_device *adev)
+{
+	u16 die_rev_id = 0, xcd_rev_id;
+	int r;
+
+	xcd_rev_id = amdgpu_device_get_rev_id(adev);
+	r = amdgpu_discovery_get_die_rev_id(adev, &die_rev_id);
+	if (r) {
+		die_rev_id = 0;
+		dev_warn(adev->dev,
+			"missing die rev id from ip discovery, assuming 0\n");
+	}
+
+	/*
+	 * FIXME: XCD revision is not yet populated in die_info[0].die_id by
+	 * ASP firmware. Use the PCI revision ID register as a temporary
+	 * fallback until firmware support is available.
+	 */
+	die_rev_id |= xcd_rev_id << SOC_V1_0_DIE_REV_XCD__SHIFT;
+	adev->rev_id = die_rev_id;
+
+	/* SOC_V1_0_DIE_REV(mid_rev, aid_rev, xcd_rev) */
+	switch (die_rev_id) {
+	case SOC_V1_0_DIE_REV(0, 0, 0):
+		adev->external_rev_id = SOC_V1_0_MID_A0_AID_A0_XCD_A0;
+		break;
+	case SOC_V1_0_DIE_REV(0, 0, 1):
+		adev->external_rev_id = SOC_V1_0_MID_A0_AID_A0_XCD_B0;
+		break;
+	case SOC_V1_0_DIE_REV(0, 1, 0):
+		adev->external_rev_id = SOC_V1_0_MID_A0_AID_A1_XCD_A0;
+		break;
+	case SOC_V1_0_DIE_REV(0, 1, 1):
+		adev->external_rev_id = SOC_V1_0_MID_A0_AID_A1_XCD_B0;
+		break;
+	default:
+		dev_warn(adev->dev, "unknown die rev id 0x%x\n", die_rev_id);
+		break;
+	}
+
+	return 0;
+}
+
 static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
+	int r;
 
 	adev->reg.pcie.rreg = &amdgpu_device_indirect_rreg;
 	adev->reg.pcie.wreg = &amdgpu_device_indirect_wreg;
@@ -288,15 +411,14 @@ static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block)
 
 	adev->asic_funcs = &soc_v1_0_asic_funcs;
 
-	adev->rev_id = amdgpu_device_get_rev_id(adev);
-	adev->external_rev_id = 0xff;
+	r = soc_v1_0_set_rev_id(adev);
+	if (r)
+		return r;
 
 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
 	case IP_VERSION(12, 1, 0):
-		adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG |
-			AMD_CG_SUPPORT_GFX_CGLS;
+		adev->cg_flags = 0;
 		adev->pg_flags = AMD_PG_SUPPORT_VCN_DPG;
-		adev->external_rev_id = adev->rev_id + 0x50;
 		break;
 	default:
 		/* FIXME: not supported yet */
@@ -320,11 +442,6 @@ static int soc_v1_0_common_late_init(struct amdgpu_ip_block *ip_block)
 	return 0;
 }
 
-static int soc_v1_0_common_sw_init(struct amdgpu_ip_block *ip_block)
-{
-	return 0;
-}
-
 static int soc_v1_0_common_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -355,11 +472,6 @@ static int soc_v1_0_common_resume(struct amdgpu_ip_block *ip_block)
 	return soc_v1_0_common_hw_init(ip_block);
 }
 
-static bool soc_v1_0_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int soc_v1_0_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 						 enum amd_clockgating_state state)
 {
@@ -382,12 +494,10 @@ static const struct amd_ip_funcs soc_v1_0_common_ip_funcs = {
 	.name = "soc_v1_0_common",
 	.early_init = soc_v1_0_common_early_init,
 	.late_init = soc_v1_0_common_late_init,
-	.sw_init = soc_v1_0_common_sw_init,
 	.hw_init = soc_v1_0_common_hw_init,
 	.hw_fini = soc_v1_0_common_hw_fini,
 	.suspend = soc_v1_0_common_suspend,
 	.resume = soc_v1_0_common_resume,
-	.is_idle = soc_v1_0_common_is_idle,
 	.set_clockgating_state = soc_v1_0_common_set_clockgating_state,
 	.set_powergating_state = soc_v1_0_common_set_powergating_state,
 	.get_clockgating_state = soc_v1_0_common_get_clockgating_state,
@@ -401,6 +511,288 @@ const struct amdgpu_ip_block_version soc_v1_0_common_ip_block = {
 	.funcs = &soc_v1_0_common_ip_funcs,
 };
 
+static struct amdgpu_reset_handler *
+soc_v1_0_get_reset_handler(struct amdgpu_reset_control *reset_ctl,
+			   struct amdgpu_reset_context *reset_context)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+	struct amdgpu_reset_handler *handler;
+	enum amd_reset_method method;
+	int i;
+
+	method = (reset_context->method == AMD_RESET_METHOD_NONE) ?
+		amdgpu_asic_reset_method(adev) :  reset_context->method;
+	for_each_handler(i, handler, reset_ctl) {
+		if (handler->reset_method == method)
+			return handler;
+	}
+
+	return NULL;
+}
+
+static inline u32 soc_v1_0_get_ip_block_mask(struct amdgpu_device *adev)
+{
+	u32 ip_block_mask = BIT(AMD_IP_BLOCK_TYPE_GFX) |
+				 BIT(AMD_IP_BLOCK_TYPE_MES) |
+				 BIT(AMD_IP_BLOCK_TYPE_SDMA) |
+				 BIT(AMD_IP_BLOCK_TYPE_IH);
+
+	return ip_block_mask;
+}
+
+static int soc_v1_0_mode2_suspend_ip(struct amdgpu_device *adev)
+{
+	u32 ip_block_mask = soc_v1_0_get_ip_block_mask(adev);
+	u32 ip_block;
+	int r, i;
+
+	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
+
+	/* SDMA suspend not required */
+	ip_block_mask &= ~BIT(AMD_IP_BLOCK_TYPE_SDMA);
+	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		ip_block = BIT(adev->ip_blocks[i].version->type);
+		if (!(ip_block_mask & ip_block))
+			continue;
+
+		r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int
+soc_v1_0_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,
+				 struct amdgpu_reset_context *reset_context)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+
+	return soc_v1_0_mode2_suspend_ip(adev);
+}
+
+static int soc_v1_0_mode2_reset(struct amdgpu_device *adev)
+{
+	adev->asic_reset_res = amdgpu_dpm_mode2_reset(adev);
+	return adev->asic_reset_res;
+}
+
+static void soc_v1_0_async_reset(struct work_struct *work)
+{
+	struct amdgpu_reset_handler *handler;
+	struct amdgpu_reset_control *reset_ctl =
+		container_of(work, struct amdgpu_reset_control, reset_work);
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+	int i;
+
+	for_each_handler(i, handler, reset_ctl) {
+		if (handler->reset_method == reset_ctl->active_reset) {
+			dev_dbg(adev->dev, "Resetting device\n");
+			handler->do_reset(adev);
+			break;
+		}
+	}
+}
+
+static int
+soc_v1_0_mode2_perform_reset(struct amdgpu_reset_control *reset_ctl,
+			     struct amdgpu_reset_context *reset_context)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+	struct list_head *reset_device_list = reset_context->reset_device_list;
+	struct amdgpu_device *tmp_adev = NULL;
+	int r = 0;
+
+	dev_dbg(adev->dev, "soc_v1_0 perform hw reset\n");
+
+	if (!reset_device_list)
+		return -EINVAL;
+
+	list_for_each_entry(tmp_adev, reset_device_list, reset_list) {
+		mutex_lock(&tmp_adev->reset_cntl->reset_lock);
+		tmp_adev->reset_cntl->active_reset = AMD_RESET_METHOD_MODE2;
+	}
+
+	list_for_each_entry(tmp_adev, reset_device_list, reset_list) {
+		r = soc_v1_0_mode2_reset(tmp_adev);
+		if (r) {
+			dev_err(tmp_adev->dev,
+				"ASIC reset failed with error, %d for drm dev, %s",
+				r, adev_to_drm(tmp_adev)->unique);
+			break;
+		}
+	}
+
+	list_for_each_entry(tmp_adev, reset_device_list, reset_list) {
+		mutex_unlock(&tmp_adev->reset_cntl->reset_lock);
+		tmp_adev->reset_cntl->active_reset = AMD_RESET_METHOD_NONE;
+	}
+
+	return r;
+}
+
+static int soc_v1_0_mode2_restore_ip(struct amdgpu_device *adev)
+{
+	u32 ip_block_mask = soc_v1_0_get_ip_block_mask(adev);
+	struct amdgpu_ip_block *ih_block;
+	u32 ip_block;
+	int i, r;
+
+	if (ip_block_mask & BIT(AMD_IP_BLOCK_TYPE_IH)) {
+		ih_block = amdgpu_device_ip_get_ip_block(adev,
+							 AMD_IP_BLOCK_TYPE_IH);
+		if (unlikely(!ih_block)) {
+			dev_err(adev->dev, "Failed to get IH handle\n");
+			return -EINVAL;
+		}
+		r = amdgpu_ip_block_resume(ih_block);
+		if (r)
+			return r;
+	}
+
+	/* IH Block resume completed can be removed from ip_block_mask */
+	ip_block_mask &= ~BIT(AMD_IP_BLOCK_TYPE_IH);
+
+	/* Reinit GFXHUB */
+	adev->gfxhub.funcs->init(adev);
+	r = adev->gfxhub.funcs->gart_enable(adev);
+	if (r) {
+		dev_err(adev->dev, "GFXHUB gart reenable failed after reset\n");
+		return r;
+	}
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		ip_block = BIT(adev->ip_blocks[i].version->type);
+		if (!(ip_block_mask & ip_block))
+			continue;
+		r = amdgpu_ip_block_resume(&adev->ip_blocks[i]);
+		if (r)
+			return r;
+	}
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!adev->ip_blocks[i].status.valid)
+			continue;
+		ip_block = BIT(adev->ip_blocks[i].version->type);
+		if (!(ip_block_mask & ip_block))
+			continue;
+
+		if (adev->ip_blocks[i].version->funcs->late_init) {
+			r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]);
+			if (r) {
+				dev_err(adev->dev,
+					"late_init of IP block <%s> failed %d after reset\n",
+					adev->ip_blocks[i].version->funcs->name,
+					r);
+				return r;
+			}
+		}
+		adev->ip_blocks[i].status.late_initialized = true;
+	}
+
+	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
+
+	return r;
+}
+
+static int
+soc_v1_0_mode2_restore_hwcontext(struct amdgpu_reset_control *reset_ctl,
+				 struct amdgpu_reset_context *reset_context)
+{
+	struct list_head *reset_device_list = reset_context->reset_device_list;
+	struct amdgpu_device *tmp_adev = NULL;
+	int r;
+
+	if (!reset_device_list)
+		return -EINVAL;
+
+	list_for_each_entry(tmp_adev, reset_device_list, reset_list) {
+		amdgpu_set_init_level(tmp_adev,
+				      AMDGPU_INIT_LEVEL_RESET_RECOVERY);
+		dev_info(tmp_adev->dev,
+			 "GPU reset succeeded, trying to resume\n");
+		amdgpu_ras_clear_err_state(tmp_adev);
+		r = soc_v1_0_mode2_restore_ip(tmp_adev);
+		if (r)
+			goto end;
+
+		/*
+		 * Add this ASIC as tracked as reset was already
+		 * complete successfully.
+		 */
+		amdgpu_register_gpu_instance(tmp_adev);
+
+		amdgpu_ras_resume(tmp_adev);
+
+		if (!r) {
+			amdgpu_set_init_level(tmp_adev,
+					      AMDGPU_INIT_LEVEL_DEFAULT);
+			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
+
+			r = amdgpu_ib_ring_tests(tmp_adev);
+			if (r) {
+				dev_err(tmp_adev->dev,
+					"ib ring test failed (%d).\n", r);
+				r = -EAGAIN;
+				tmp_adev->asic_reset_res = r;
+				goto end;
+			}
+		}
+	}
+
+end:
+	return r;
+}
+
+static struct amdgpu_reset_handler soc_v1_0_mode2_handler = {
+	.reset_method		= AMD_RESET_METHOD_MODE2,
+	.prepare_env		= NULL,
+	.prepare_hwcontext	= soc_v1_0_mode2_prepare_hwcontext,
+	.perform_reset		= soc_v1_0_mode2_perform_reset,
+	.restore_hwcontext	= soc_v1_0_mode2_restore_hwcontext,
+	.restore_env		= NULL,
+	.do_reset		= soc_v1_0_mode2_reset,
+};
+
+static struct amdgpu_reset_handler
+	*soc_v1_0_rst_handlers[AMDGPU_RESET_MAX_HANDLERS] = {
+		&soc_v1_0_mode2_handler,
+	};
+
+int soc_v1_0_reset_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_reset_control *reset_ctl;
+
+	reset_ctl = kzalloc_obj(*reset_ctl, GFP_KERNEL);
+	if (!reset_ctl)
+		return -ENOMEM;
+
+	reset_ctl->handle = adev;
+	reset_ctl->async_reset = soc_v1_0_async_reset;
+	reset_ctl->active_reset = AMD_RESET_METHOD_NONE;
+	reset_ctl->get_reset_handler = soc_v1_0_get_reset_handler;
+
+	INIT_WORK(&reset_ctl->reset_work, reset_ctl->async_reset);
+	/* Only mode2 is handled through reset control now */
+	reset_ctl->reset_handlers = &soc_v1_0_rst_handlers;
+
+	adev->reset_cntl = reset_ctl;
+
+	return 0;
+}
+
+int soc_v1_0_reset_fini(struct amdgpu_device *adev)
+{
+	kfree(adev->reset_cntl);
+	adev->reset_cntl = NULL;
+	return 0;
+}
+
 static enum amdgpu_gfx_partition __soc_v1_0_calc_xcp_mode(struct amdgpu_xcp_mgr *xcp_mgr)
 {
 	struct amdgpu_device *adev = xcp_mgr->adev;
@@ -565,7 +957,8 @@ static int soc_v1_0_get_xcp_res_info(struct amdgpu_xcp_mgr *xcp_mgr,
 		break;
 	case AMDGPU_DPX_PARTITION_MODE:
 		num_xcp = 2;
-		nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE);
+		nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) |
+			    BIT(AMDGPU_NPS2_PARTITION_MODE);
 		break;
 	case AMDGPU_TPX_PARTITION_MODE:
 		num_xcp = 3;
@@ -638,7 +1031,7 @@ static bool __soc_v1_0_is_valid_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 	case AMDGPU_SPX_PARTITION_MODE:
 		return adev->gmc.num_mem_partitions == 1 && num_xcc > 0;
 	case AMDGPU_DPX_PARTITION_MODE:
-		return adev->gmc.num_mem_partitions <= 2 && (num_xcc % 4) == 0;
+		return adev->gmc.num_mem_partitions <= 2 && (num_xcc % 2) == 0;
 	case AMDGPU_TPX_PARTITION_MODE:
 		return (adev->gmc.num_mem_partitions == 1 ||
 			adev->gmc.num_mem_partitions == 3) &&
@@ -653,8 +1046,10 @@ static bool __soc_v1_0_is_valid_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 		 * (num_xcc % adev->gmc.num_mem_partitions) == 0 because
 		 * num_compute_partitions can't be less than num_mem_partitions
 		 */
-		return ((num_xcc > 1) &&
-		       (num_xcc % adev->gmc.num_mem_partitions) == 0);
+		return (adev->gmc.num_mem_partitions == 1 ||
+			adev->gmc.num_mem_partitions == 4) &&
+		       (num_xcc > 1) &&
+		       (num_xcc % adev->gmc.num_mem_partitions) == 0;
 	default:
 		return false;
 	}
@@ -702,20 +1097,21 @@ static int soc_v1_0_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 	if (adev->kfd.init_complete && !amdgpu_in_reset(adev))
 		flags |= AMDGPU_XCP_OPS_KFD;
 
-	if (flags & AMDGPU_XCP_OPS_KFD) {
-		ret = amdgpu_amdkfd_check_and_lock_kfd(adev);
-		if (ret)
-			goto out;
-	}
-
 	ret = amdgpu_xcp_pre_partition_switch(xcp_mgr, flags);
 	if (ret)
-		goto unlock;
+		goto out;
 
 	num_xcc_per_xcp = __soc_v1_0_get_xcc_per_xcp(xcp_mgr, mode);
 	if (adev->gfx.imu.funcs &&
-	    adev->gfx.imu.funcs->switch_compute_partition)
-		adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode);
+	    adev->gfx.imu.funcs->switch_compute_partition) {
+		ret = adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode);
+		if (ret)
+			goto out;
+	}
+	if (adev->gfx.imu.funcs &&
+	    adev->gfx.imu.funcs->init_mcm_addr_lut &&
+	    amdgpu_emu_mode)
+		adev->gfx.imu.funcs->init_mcm_addr_lut(adev);
 
 	/* Init info about new xcps */
 	*num_xcps = num_xcc / num_xcc_per_xcp;
@@ -724,9 +1120,6 @@ static int soc_v1_0_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 	ret = amdgpu_xcp_post_partition_switch(xcp_mgr, flags);
 	if (!ret)
 		__soc_v1_0_update_available_partition_mode(xcp_mgr);
-unlock:
-	if (flags & AMDGPU_XCP_OPS_KFD)
-		amdgpu_amdkfd_unlock_kfd(adev);
 out:
 	return ret;
 }
@@ -828,19 +1221,27 @@ static int soc_v1_0_xcp_mgr_init(struct amdgpu_device *adev)
 	return ret;
 }
 
-int soc_v1_0_init_soc_config(struct amdgpu_device *adev)
+uint32_t soc_v1_0_get_aid_mask(uint16_t xcc_mask)
 {
-	int ret, i;
 	int xcc_inst_per_aid = 4;
-	uint16_t xcc_mask, sdma_mask = 0;
+	uint32_t aid_mask = 0;
+	int i;
 
-	xcc_mask = adev->gfx.xcc_mask;
-	adev->aid_mask = 0;
 	for (i = 0; xcc_mask; xcc_mask >>= xcc_inst_per_aid, i++) {
-		if (xcc_mask & ((1U << xcc_inst_per_aid) - 1))
-			adev->aid_mask |= (1 << i);
+		if (xcc_mask & GENMASK(xcc_inst_per_aid - 1, 0))
+			aid_mask |= BIT(i);
 	}
 
+	return aid_mask;
+}
+
+int soc_v1_0_init_soc_config(struct amdgpu_device *adev)
+{
+	int ret, i;
+	uint16_t sdma_mask = 0;
+
+	adev->aid_mask = soc_v1_0_get_aid_mask(adev->gfx.xcc_mask);
+
 	adev->sdma.num_inst_per_xcc = 2;
 	for_each_inst(i, adev->gfx.xcc_mask)
 		sdma_mask |=
@@ -849,6 +1250,14 @@ int soc_v1_0_init_soc_config(struct amdgpu_device *adev)
 	adev->sdma.sdma_mask = sdma_mask;
 	adev->sdma.num_instances = NUM_XCC(adev->sdma.sdma_mask);
 
+	adev->vcn.harvest_config = 0;
+	adev->vcn.num_inst_per_aid = 2;
+	adev->vcn.num_vcn_inst = hweight32(adev->vcn.inst_mask);
+
+	adev->jpeg.harvest_config = 0;
+	adev->jpeg.num_inst_per_aid = 2;
+	adev->jpeg.num_jpeg_inst = hweight32(adev->jpeg.inst_mask);
+
 	ret = soc_v1_0_xcp_mgr_init(adev);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
index 16c220fcc4e92a..82752964d2c1a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.h
@@ -30,10 +30,13 @@ void soc_v1_0_grbm_select(struct amdgpu_device *adev,
 			  u32 queue, u32 vmid,
 			  int xcc_id);
 int soc_v1_0_init_soc_config(struct amdgpu_device *adev);
+uint32_t soc_v1_0_get_aid_mask(uint16_t xcc_mask);
 bool soc_v1_0_normalize_xcc_reg_range(uint32_t reg);
 bool soc_v1_0_mid1_reg_range(uint32_t reg);
 uint32_t soc_v1_0_normalize_xcc_reg_offset(uint32_t reg);
 uint32_t soc_v1_0_normalize_reg_offset(uint32_t reg);
 u64 soc_v1_0_encode_ext_smn_addressing(int ext_id);
+int soc_v1_0_reset_init(struct amdgpu_device *adev);
+int soc_v1_0_reset_fini(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
index f6178da910d35e..3110140f739a9a 100644
--- a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
@@ -156,6 +156,7 @@ struct ta_ras_init_flags {
 	uint8_t nps_mode;
 	uint32_t active_umc_mask;
 	uint8_t vram_type;
+	uint32_t ext_umc_mask;
 };
 
 struct ta_ras_mca_addr {
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index a3e883f6f09922..d84c08698dff0c 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -363,17 +363,6 @@ static int tonga_ih_resume(struct amdgpu_ip_block *ip_block)
 	return tonga_ih_hw_init(ip_block);
 }
 
-static bool tonga_ih_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	u32 tmp = RREG32(mmSRBM_STATUS);
-
-	if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
-		return false;
-
-	return true;
-}
-
 static int tonga_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -442,7 +431,6 @@ static const struct amd_ip_funcs tonga_ih_ip_funcs = {
 	.hw_fini = tonga_ih_hw_fini,
 	.suspend = tonga_ih_suspend,
 	.resume = tonga_ih_resume,
-	.is_idle = tonga_ih_is_idle,
 	.wait_for_idle = tonga_ih_wait_for_idle,
 	.soft_reset = tonga_ih_soft_reset,
 	.set_clockgating_state = tonga_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/ualink_v1_0.c b/drivers/gpu/drm/amd/amdgpu/ualink_v1_0.c
new file mode 100644
index 00000000000000..d440ed6b573360
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/ualink_v1_0.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/delay.h>
+#include "amdgpu.h"
+#include "ivsrcid/mpnht/irqsrcs_mpnht_15_0.h"
+#include "amdgpu_ualink.h"
+#include "ualink_v1_0.h"
+
+/* nHT firmware mailbox registers via SMN, copy of MP1 */
+#define mmMPNHT_SMN_C2PMSG_CMD		0xAE10958
+#define mmMPNHT_SMN_C2PMSG_DATA		0xAE1095C
+#define mmMPNHT_SMN_C2PMSG_ADDR_LO	0xAE10960
+#define mmMPNHT_SMN_C2PMSG_ADDR_HI	0xAE10964
+#define mmMPNHT_SMN_C2PMSG_STATUS	0xAE10968
+
+/* nHT firmware commands */
+#define UALINK_V1_0_FW_CMD_LOAD_METADATA	0x1
+#define UALINK_V1_0_FW_CMD_HALT_OPERATION	0x2
+
+#define UALINK_V1_0_FW_POLL_COUNT	2000
+
+static u32 ualink_v1_0_check_status(struct amdgpu_device *adev)
+{
+	return RREG32_PCIE(mmMPNHT_SMN_C2PMSG_STATUS);
+}
+
+static int ualink_v1_0_send_metadata(struct amdgpu_device *adev,
+				     u64 metadata_gpu_addr, u32 accel_data)
+{
+	u32 status;
+	int i;
+
+	dev_dbg(adev->dev, "nht load metadata addr 0x%llx accel_data 0x%x\n",
+		metadata_gpu_addr, accel_data);
+
+	WREG32_PCIE(mmMPNHT_SMN_C2PMSG_ADDR_HI, upper_32_bits(metadata_gpu_addr));
+	WREG32_PCIE(mmMPNHT_SMN_C2PMSG_ADDR_LO, lower_32_bits(metadata_gpu_addr));
+	WREG32_PCIE(mmMPNHT_SMN_C2PMSG_DATA, accel_data);
+	WREG32_PCIE(mmMPNHT_SMN_C2PMSG_CMD, UALINK_V1_0_FW_CMD_LOAD_METADATA);
+
+	for (i = 0; i < UALINK_V1_0_FW_POLL_COUNT; i++) {
+		status = RREG32_PCIE(mmMPNHT_SMN_C2PMSG_STATUS);
+		if (status == AMDGPU_NHT_FW_ST_READY)
+			return 0;
+		mdelay(1);
+	}
+
+	dev_dbg(adev->dev, "f/w load metadata failed 0x%x\n", status);
+	return -ETIME;
+}
+
+static int ualink_v1_0_send_halt(struct amdgpu_device *adev)
+{
+	u32 status;
+	int i;
+
+	dev_dbg(adev->dev, "nht halt cmd 0x%x\n", UALINK_V1_0_FW_CMD_HALT_OPERATION);
+
+	WREG32_PCIE(mmMPNHT_SMN_C2PMSG_CMD, UALINK_V1_0_FW_CMD_HALT_OPERATION);
+
+	for (i = 0; i < UALINK_V1_0_FW_POLL_COUNT; i++) {
+		status = RREG32_PCIE(mmMPNHT_SMN_C2PMSG_STATUS);
+		if (status == AMDGPU_NHT_FW_ST_HALT)
+			return 0;
+		mdelay(1);
+	}
+
+	dev_warn(adev->dev, "f/w halt failed status 0x%x\n", status);
+	return -ETIME;
+}
+
+const struct amdgpu_ualink_msg_ctl ualink_v1_0_msg_ctl = {
+	.check_status = ualink_v1_0_check_status,
+	.send_metadata = ualink_v1_0_send_metadata,
+	.send_halt = ualink_v1_0_send_halt,
+};
+
+static int ualink_v1_0_early_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+
+	adev->ualink.msg_ctl = &ualink_v1_0_msg_ctl;
+
+	return 0;
+}
+
+static int ualink_v1_0_sw_init(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = ualink_ip_sw_init(ip_block);
+	if (r)
+		return r;
+
+	r = amdgpu_ualink_init_interrupt(adev, SOC_V1_0_IH_CLIENTID_nHT,
+					 MPNHT_15_0__SRCID__REMOTE_INTERRUPT);
+	if (r) {
+		dev_err(adev->dev, "Failed to add UALink irq: %d\n", r);
+		return r;
+	}
+
+	return 0;
+}
+
+static const struct amd_ip_funcs ualink_v1_0_ip_funcs = {
+	.name = "ualink",
+	.early_init = ualink_v1_0_early_init,
+	.late_init = ualink_ip_late_init,
+	.sw_init = ualink_v1_0_sw_init,
+	.sw_fini = ualink_ip_sw_fini,
+	.hw_init = ualink_ip_hw_init,
+	.hw_fini = ualink_ip_hw_fini,
+};
+
+const struct amdgpu_ip_block_version ualink_v1_0_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_UALINK,
+	.major = 1,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &ualink_v1_0_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/ualink_v1_0.h b/drivers/gpu/drm/amd/amdgpu/ualink_v1_0.h
new file mode 100644
index 00000000000000..d794b0e3e800eb
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/ualink_v1_0.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __UALINK_V1_0_H__
+#define __UALINK_V1_0_H__
+
+#include "amdgpu_ualink.h"
+
+extern const struct amdgpu_ualink_msg_ctl ualink_v1_0_msg_ctl;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
index efb3fde919ee3b..b72a0c34425a2a 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c
@@ -789,13 +789,6 @@ static int uvd_v3_1_resume(struct amdgpu_ip_block *ip_block)
 	return uvd_v3_1_hw_init(ip_block);
 }
 
-static bool uvd_v3_1_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	return !(RREG32(mmSRBM_STATUS) & SRBM_STATUS__UVD_BUSY_MASK);
-}
-
 static int uvd_v3_1_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -843,7 +836,6 @@ static const struct amd_ip_funcs uvd_v3_1_ip_funcs = {
 	.prepare_suspend = uvd_v3_1_prepare_suspend,
 	.suspend = uvd_v3_1_suspend,
 	.resume = uvd_v3_1_resume,
-	.is_idle = uvd_v3_1_is_idle,
 	.wait_for_idle = uvd_v3_1_wait_for_idle,
 	.soft_reset = uvd_v3_1_soft_reset,
 	.set_clockgating_state = uvd_v3_1_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 8a9ba2276275cf..cf304b489d8725 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -664,13 +664,6 @@ static void uvd_v4_2_set_dcm(struct amdgpu_device *adev,
 	WREG32_UVD_CTX(ixUVD_CGC_CTRL2, tmp2);
 }
 
-static bool uvd_v4_2_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	return !(RREG32(mmSRBM_STATUS) & SRBM_STATUS__UVD_BUSY_MASK);
-}
-
 static int uvd_v4_2_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -768,7 +761,6 @@ static const struct amd_ip_funcs uvd_v4_2_ip_funcs = {
 	.prepare_suspend = uvd_v4_2_prepare_suspend,
 	.suspend = uvd_v4_2_suspend,
 	.resume = uvd_v4_2_resume,
-	.is_idle = uvd_v4_2_is_idle,
 	.wait_for_idle = uvd_v4_2_wait_for_idle,
 	.soft_reset = uvd_v4_2_soft_reset,
 	.set_clockgating_state = uvd_v4_2_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index 454f109cbb2e29..564d8fb58905cf 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -580,13 +580,6 @@ static void uvd_v5_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
 	}
 }
 
-static bool uvd_v5_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-
-	return !(RREG32(mmSRBM_STATUS) & SRBM_STATUS__UVD_BUSY_MASK);
-}
-
 static int uvd_v5_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	unsigned i;
@@ -869,7 +862,6 @@ static const struct amd_ip_funcs uvd_v5_0_ip_funcs = {
 	.prepare_suspend = uvd_v5_0_prepare_suspend,
 	.suspend = uvd_v5_0_suspend,
 	.resume = uvd_v5_0_resume,
-	.is_idle = uvd_v5_0_is_idle,
 	.wait_for_idle = uvd_v5_0_wait_for_idle,
 	.soft_reset = uvd_v5_0_soft_reset,
 	.set_clockgating_state = uvd_v5_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index bbd0e05d39e510..dcb8487e22b179 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -1496,7 +1496,6 @@ static const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
 	.prepare_suspend = uvd_v6_0_prepare_suspend,
 	.suspend = uvd_v6_0_suspend,
 	.resume = uvd_v6_0_resume,
-	.is_idle = uvd_v6_0_is_idle,
 	.wait_for_idle = uvd_v6_0_wait_for_idle,
 	.soft_reset = uvd_v6_0_soft_reset,
 	.set_clockgating_state = uvd_v6_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
index 93253db5e2de4c..b1428f5bbc81e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
@@ -796,7 +796,6 @@ static const struct amd_ip_funcs vce_v1_0_ip_funcs = {
 	.hw_fini = vce_v1_0_hw_fini,
 	.suspend = vce_v1_0_suspend,
 	.resume = vce_v1_0_resume,
-	.is_idle = vce_v1_0_is_idle,
 	.wait_for_idle = vce_v1_0_wait_for_idle,
 	.set_clockgating_state = vce_v1_0_set_clockgating_state,
 	.set_powergating_state = vce_v1_0_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 3a6fc8604108c7..dd870ecac2a42f 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -633,7 +633,6 @@ static const struct amd_ip_funcs vce_v2_0_ip_funcs = {
 	.hw_fini = vce_v2_0_hw_fini,
 	.suspend = vce_v2_0_suspend,
 	.resume = vce_v2_0_resume,
-	.is_idle = vce_v2_0_is_idle,
 	.wait_for_idle = vce_v2_0_wait_for_idle,
 	.soft_reset = vce_v2_0_soft_reset,
 	.set_clockgating_state = vce_v2_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 4dbbeaf97ad19b..d5adccd3e6895f 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -857,7 +857,6 @@ static const struct amd_ip_funcs vce_v3_0_ip_funcs = {
 	.hw_fini = vce_v3_0_hw_fini,
 	.suspend = vce_v3_0_suspend,
 	.resume = vce_v3_0_resume,
-	.is_idle = vce_v3_0_is_idle,
 	.wait_for_idle = vce_v3_0_wait_for_idle,
 	.soft_reset = vce_v3_0_soft_reset,
 	.set_clockgating_state = vce_v3_0_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index e9d790914761c1..72fd3022b60624 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -2034,7 +2034,6 @@ static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
 	.hw_fini = vcn_v1_0_hw_fini,
 	.suspend = vcn_v1_0_suspend,
 	.resume = vcn_v1_0_resume,
-	.is_idle = vcn_v1_0_is_idle,
 	.wait_for_idle = vcn_v1_0_wait_for_idle,
 	.set_clockgating_state = vcn_v1_0_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
@@ -2065,7 +2064,7 @@ static int vcn_v1_0_validate_bo(struct amdgpu_cs_parser *parser,
 		return -EINVAL;
 	}
 
-	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr/AMDGPU_GPU_PAGE_SIZE);
+	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
 	if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
 		return -EINVAL;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
index 0442bfcfd384d3..7d2b9e4b913ac9 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
@@ -25,6 +25,7 @@
 #include <drm/drm_drv.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "soc15.h"
 #include "soc15d.h"
@@ -2102,7 +2103,6 @@ static const struct amd_ip_funcs vcn_v2_0_ip_funcs = {
 	.hw_fini = vcn_v2_0_hw_fini,
 	.suspend = vcn_v2_0_suspend,
 	.resume = vcn_v2_0_resume,
-	.is_idle = vcn_v2_0_is_idle,
 	.wait_for_idle = vcn_v2_0_wait_for_idle,
 	.set_clockgating_state = vcn_v2_0_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
index 0d8a3cea63ee20..10ef5616b2f5ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
@@ -25,6 +25,7 @@
 #include <drm/drm_drv.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "amdgpu_pm.h"
 #include "soc15.h"
@@ -2110,7 +2111,6 @@ static const struct amd_ip_funcs vcn_v2_5_ip_funcs = {
 	.hw_fini = vcn_v2_5_hw_fini,
 	.suspend = vcn_v2_5_suspend,
 	.resume = vcn_v2_5_resume,
-	.is_idle = vcn_v2_5_is_idle,
 	.wait_for_idle = vcn_v2_5_wait_for_idle,
 	.set_clockgating_state = vcn_v2_5_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
@@ -2127,7 +2127,6 @@ static const struct amd_ip_funcs vcn_v2_6_ip_funcs = {
         .hw_fini = vcn_v2_5_hw_fini,
         .suspend = vcn_v2_5_suspend,
         .resume = vcn_v2_5_resume,
-        .is_idle = vcn_v2_5_is_idle,
         .wait_for_idle = vcn_v2_5_wait_for_idle,
         .set_clockgating_state = vcn_v2_5_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index 00d8f35846f22c..0f4f4d66e2e4be 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -23,6 +23,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "amdgpu_pm.h"
 #include "amdgpu_cs.h"
@@ -2216,21 +2217,6 @@ static int vcn_v3_0_reset(struct amdgpu_vcn_inst *vinst)
 	return vcn_v3_0_start(vinst);
 }
 
-static bool vcn_v3_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int i, ret = 1;
-
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
-		if (adev->vcn.harvest_config & (1 << i))
-			continue;
-
-		ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
-	}
-
-	return ret;
-}
-
 static int vcn_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
@@ -2376,7 +2362,6 @@ static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
 	.hw_fini = vcn_v3_0_hw_fini,
 	.suspend = vcn_v3_0_suspend,
 	.resume = vcn_v3_0_resume,
-	.is_idle = vcn_v3_0_is_idle,
 	.wait_for_idle = vcn_v3_0_wait_for_idle,
 	.set_clockgating_state = vcn_v3_0_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index c2ddf3cb368fde..4b7f7bbe18da8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -23,6 +23,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "amdgpu_pm.h"
 #include "amdgpu_cs.h"
@@ -2091,28 +2092,6 @@ static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * vcn_v4_0_is_idle - check VCN block is idle
- *
- * @ip_block: Pointer to the amdgpu_ip_block structure
- *
- * Check whether VCN block is idle
- */
-static bool vcn_v4_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int i, ret = 1;
-
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
-		if (adev->vcn.harvest_config & (1 << i))
-			continue;
-
-		ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
-	}
-
-	return ret;
-}
-
 /**
  * vcn_v4_0_wait_for_idle - wait for VCN block idle
  *
@@ -2304,7 +2283,6 @@ static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
 	.hw_fini = vcn_v4_0_hw_fini,
 	.suspend = vcn_v4_0_suspend,
 	.resume = vcn_v4_0_resume,
-	.is_idle = vcn_v4_0_is_idle,
 	.wait_for_idle = vcn_v4_0_wait_for_idle,
 	.set_clockgating_state = vcn_v4_0_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
index 179b892fb410eb..3f39aa83c31927 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
@@ -25,6 +25,7 @@
 #include <drm/drm_drv.h>
 
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "amdgpu_pm.h"
 #include "soc15.h"
@@ -1689,7 +1690,8 @@ static int vcn_v4_0_3_reset_jpeg_pre_helper(struct amdgpu_device *adev, int inst
 		/* if Jobs are still pending after timeout,
 		 * We'll handle them in the bottom helper
 		 */
-		amdgpu_fence_wait_polling(ring, wait_seq, adev->video_timeout);
+		amdgpu_fence_wait_polling(ring, wait_seq,
+					  jiffies_to_usecs(adev->video_timeout));
        }
 
 	return 0;
@@ -1840,26 +1842,6 @@ static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * vcn_v4_0_3_is_idle - check VCN block is idle
- *
- * @ip_block: Pointer to the amdgpu_ip_block structure
- *
- * Check whether VCN block is idle
- */
-static bool vcn_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int i, ret = 1;
-
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
-		ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) ==
-			UVD_STATUS__IDLE);
-	}
-
-	return ret;
-}
-
 /**
  * vcn_v4_0_3_wait_for_idle - wait for VCN block idle
  *
@@ -2049,7 +2031,6 @@ static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = {
 	.hw_fini = vcn_v4_0_3_hw_fini,
 	.suspend = vcn_v4_0_3_suspend,
 	.resume = vcn_v4_0_3_resume,
-	.is_idle = vcn_v4_0_3_is_idle,
 	.wait_for_idle = vcn_v4_0_3_wait_for_idle,
 	.set_clockgating_state = vcn_v4_0_3_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
index c8879a6e5297de..f30f80c1b07fd9 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
@@ -23,6 +23,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "amdgpu_pm.h"
 #include "amdgpu_cs.h"
@@ -1531,28 +1532,6 @@ static void vcn_v4_0_5_set_unified_ring_funcs(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * vcn_v4_0_5_is_idle - check VCN block is idle
- *
- * @ip_block: Pointer to the amdgpu_ip_block structure
- *
- * Check whether VCN block is idle
- */
-static bool vcn_v4_0_5_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int i, ret = 1;
-
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
-		if (adev->vcn.harvest_config & (1 << i))
-			continue;
-
-		ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
-	}
-
-	return ret;
-}
-
 /**
  * vcn_v4_0_5_wait_for_idle - wait for VCN block idle
  *
@@ -1707,7 +1686,6 @@ static const struct amd_ip_funcs vcn_v4_0_5_ip_funcs = {
 	.hw_fini = vcn_v4_0_5_hw_fini,
 	.suspend = vcn_v4_0_5_suspend,
 	.resume = vcn_v4_0_5_resume,
-	.is_idle = vcn_v4_0_5_is_idle,
 	.wait_for_idle = vcn_v4_0_5_wait_for_idle,
 	.set_clockgating_state = vcn_v4_0_5_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
index 886e2c2074a9e4..7b7c3bc9f046a7 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
@@ -23,6 +23,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "amdgpu_pm.h"
 #include "soc15.h"
@@ -1317,28 +1318,6 @@ static void vcn_v5_0_0_set_unified_ring_funcs(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * vcn_v5_0_0_is_idle - check VCN block is idle
- *
- * @ip_block: Pointer to the amdgpu_ip_block structure
- *
- * Check whether VCN block is idle
- */
-static bool vcn_v5_0_0_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int i, ret = 1;
-
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
-		if (adev->vcn.harvest_config & (1 << i))
-			continue;
-
-		ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
-	}
-
-	return ret;
-}
-
 /**
  * vcn_v5_0_0_wait_for_idle - wait for VCN block idle
  *
@@ -1493,7 +1472,6 @@ static const struct amd_ip_funcs vcn_v5_0_0_ip_funcs = {
 	.hw_fini = vcn_v5_0_0_hw_fini,
 	.suspend = vcn_v5_0_0_suspend,
 	.resume = vcn_v5_0_0_resume,
-	.is_idle = vcn_v5_0_0_is_idle,
 	.wait_for_idle = vcn_v5_0_0_wait_for_idle,
 	.set_clockgating_state = vcn_v5_0_0_set_clockgating_state,
 	.set_powergating_state = vcn_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
index 1a07c3bf44253d..c9e990ab297d41 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
@@ -23,6 +23,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_vcn.h"
 #include "amdgpu_pm.h"
 #include "soc15.h"
@@ -1335,7 +1336,8 @@ static int vcn_v5_0_1_reset_jpeg_pre_helper(struct amdgpu_device *adev, int inst
 		/* if Jobs are still pending after timeout,
 		 * We'll handle them in the bottom helper
 		 */
-		amdgpu_fence_wait_polling(ring, wait_seq, adev->video_timeout);
+		amdgpu_fence_wait_polling(ring, wait_seq,
+					  jiffies_to_usecs(adev->video_timeout));
        }
 
 	return 0;
@@ -1483,24 +1485,6 @@ static void vcn_v5_0_1_set_unified_ring_funcs(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * vcn_v5_0_1_is_idle - check VCN block is idle
- *
- * @ip_block: Pointer to the amdgpu_ip_block structure
- *
- * Check whether VCN block is idle
- */
-static bool vcn_v5_0_1_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int i, ret = 1;
-
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
-		ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == UVD_STATUS__IDLE);
-
-	return ret;
-}
-
 /**
  * vcn_v5_0_1_wait_for_idle - wait for VCN block idle
  *
@@ -1672,7 +1656,6 @@ static const struct amd_ip_funcs vcn_v5_0_1_ip_funcs = {
 	.hw_fini = vcn_v5_0_1_hw_fini,
 	.suspend = vcn_v5_0_1_suspend,
 	.resume = vcn_v5_0_1_resume,
-	.is_idle = vcn_v5_0_1_is_idle,
 	.wait_for_idle = vcn_v5_0_1_wait_for_idle,
 	.soft_reset = NULL,
 	.set_clockgating_state = vcn_v5_0_1_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
index b9f6ae75ea727f..ed47ddd5b08ce3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
@@ -37,9 +37,11 @@
 #include "vcn_v5_0_0.h"
 #include "vcn_v5_0_1.h"
 #include "vcn_v5_0_2.h"
+#include "mmsch_v5_0.h"
 
 #include <drm/drm_drv.h>
 
+static int vcn_v5_0_2_start_sriov(struct amdgpu_device *adev);
 static void vcn_v5_0_2_set_unified_ring_funcs(struct amdgpu_device *adev);
 static void vcn_v5_0_2_set_irq_funcs(struct amdgpu_device *adev);
 static int vcn_v5_0_2_set_pg_state(struct amdgpu_vcn_inst *vinst,
@@ -112,6 +114,11 @@ static int vcn_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_VCN1,
+		VCN_5_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq);
+	if (r)
+		return r;
+
 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 		vcn_inst = GET_INST(VCN, i);
 
@@ -127,12 +134,16 @@ static int vcn_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
 
 		ring = &adev->vcn.inst[i].ring_enc[0];
 		ring->use_doorbell = true;
-
-		ring->doorbell_index =
-			(adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 32 * vcn_inst;
+		if (amdgpu_sriov_vf(adev)) {
+			ring->doorbell_index =
+				(adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 32 * vcn_inst;
+		} else {
+			ring->doorbell_index =
+				(adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 11 * vcn_inst;
+		}
 
 		ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id);
-		sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id);
+		sprintf(ring->name, "vcn_unified_%d", i);
 
 		r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
 					AMDGPU_RING_PRIO_DEFAULT, &adev->vcn.inst[i].sched_score);
@@ -142,9 +153,16 @@ static int vcn_v5_0_2_sw_init(struct amdgpu_ip_block *ip_block)
 		vcn_v5_0_2_fw_shared_init(adev, i);
 	}
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_virt_alloc_mm_table(adev);
+		if (r)
+			return r;
+	}
+
 	/* TODO: Add queue reset mask when FW fully supports it */
 	adev->vcn.supported_reset =
 		amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]);
+	/* SRIOV VF does not support per queue reset, please wrap with if(!amdgpu_sriov_vf()) check */
 
 	return amdgpu_vcn_sysfs_reset_mask_init(adev);
 }
@@ -173,6 +191,9 @@ static int vcn_v5_0_2_sw_fini(struct amdgpu_ip_block *ip_block)
 		drm_dev_exit(idx);
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_free_mm_table(adev);
+
 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 		r = amdgpu_vcn_suspend(adev, i);
 		if (r)
@@ -203,29 +224,41 @@ static int vcn_v5_0_2_hw_init(struct amdgpu_ip_block *ip_block)
 	int i, r, vcn_inst;
 	uint32_t tmp;
 
-	if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x200)
-		adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED);
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
-		vcn_inst = GET_INST(VCN, i);
-		ring = &adev->vcn.inst[i].ring_enc[0];
-
-		/* Remove Video Tiles antihang mechanism */
-		tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS);
-		tmp &= (~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
-		WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp);
-
-		if (ring->use_doorbell)
-			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
-				((adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
-				 11 * vcn_inst),
-				adev->vcn.inst[i].aid_id);
-
-		/* Re-init fw_shared, if required */
-		vcn_v5_0_2_fw_shared_init(adev, i);
-
-		r = amdgpu_ring_test_helper(ring);
+	if (amdgpu_sriov_vf(adev)) {
+		r = vcn_v5_0_2_start_sriov(adev);
 		if (r)
 			return r;
+
+		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+			ring = &adev->vcn.inst[i].ring_enc[0];
+			ring->wptr = 0;
+			ring->wptr_old = 0;
+			vcn_v5_0_2_unified_ring_set_wptr(ring);
+			ring->sched.ready = true;
+		}
+	} else {
+		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+			vcn_inst = GET_INST(VCN, i);
+			ring = &adev->vcn.inst[i].ring_enc[0];
+
+			/* Remove Video Tiles antihang mechanism */
+			tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS);
+			tmp &= (~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
+			WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp);
+
+			if (ring->use_doorbell)
+				adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
+					((adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
+					 11 * vcn_inst),
+					vcn_inst);
+
+			/* Re-init fw_shared, if required */
+			vcn_v5_0_2_fw_shared_init(adev, i);
+
+			r = amdgpu_ring_test_helper(ring);
+			if (r)
+				return r;
+		}
 	}
 
 	return 0;
@@ -587,7 +620,7 @@ static int vcn_v5_0_2_start_dpg_mode(struct amdgpu_vcn_inst *vinst,
 			(uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
 		/* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */
 		WREG32_SOC24_DPG_MODE(inst_idx, 0xDEADBEEF,
-				adev->vcn.inst[inst_idx].aid_id, 0, true);
+				vcn_inst, 0, true);
 	}
 
 	/* enable VCPU clock */
@@ -672,6 +705,198 @@ static int vcn_v5_0_2_start_dpg_mode(struct amdgpu_vcn_inst *vinst,
 	return 0;
 }
 
+
+/**
+ * vcn_v5_0_2_start_sriov - VCN start under SRIOV env
+ */
+
+static int vcn_v5_0_2_start_sriov(struct amdgpu_device *adev)
+{
+	int i, vcn_inst;
+	struct amdgpu_ring *ring_enc;
+	uint64_t cache_addr;
+	uint64_t rb_enc_addr;
+	uint64_t ctx_addr;
+	uint32_t param, resp, expected;
+	uint32_t offset, cache_size;
+	uint32_t tmp, timeout;
+
+	struct amdgpu_mm_table *table = &adev->virt.mm_table;
+	uint32_t *table_loc;
+	uint32_t table_size;
+	uint32_t size, size_dw;
+	uint32_t init_status;
+
+	struct mmsch_v5_0_cmd_direct_write
+		direct_wt = { {0} };
+	struct mmsch_v5_0_cmd_direct_read_modify_write
+		direct_rd_mod_wt = { {0} };
+	struct mmsch_v5_0_cmd_end end = { {0} };
+	struct mmsch_v5_0_init_header header;
+
+	struct amdgpu_vcn5_fw_shared *fw_shared;
+	struct amdgpu_fw_shared_rb_setup *rb_setup;
+
+	direct_wt.cmd_header.command_type =
+		MMSCH_COMMAND__DIRECT_REG_WRITE;
+	direct_rd_mod_wt.cmd_header.command_type =
+		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
+	end.cmd_header.command_type = MMSCH_COMMAND__END;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+		vcn_inst = GET_INST(VCN, i);
+
+		vcn_v5_0_2_fw_shared_init(adev, vcn_inst);
+
+		memset(&header, 0, sizeof(struct mmsch_v5_0_init_header));
+		header.version = MMSCH_VERSION;
+		header.total_size = sizeof(struct mmsch_v5_0_init_header) >> 2;
+
+		table_loc = (uint32_t *)table->cpu_addr;
+		table_loc += header.total_size;
+
+		table_size = 0;
+
+		MMSCH_V5_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS),
+			~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
+
+		cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4);
+
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+			MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
+
+			MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
+
+			offset = 0;
+			MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+				regUVD_VCPU_CACHE_OFFSET0), 0);
+		} else {
+			MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+				lower_32_bits(adev->vcn.inst[i].gpu_addr));
+			MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+				upper_32_bits(adev->vcn.inst[i].gpu_addr));
+		offset = cache_size;
+			MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+				regUVD_VCPU_CACHE_OFFSET0),
+				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+		}
+
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_VCPU_CACHE_SIZE0),
+			cache_size);
+
+		cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset;
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr));
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr));
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_VCPU_CACHE_OFFSET1), 0);
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE);
+
+		cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset +
+			AMDGPU_VCN_STACK_SIZE;
+
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr));
+
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+		regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr));
+
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_VCPU_CACHE_OFFSET2), 0);
+
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE);
+
+		fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr;
+		rb_setup = &fw_shared->rb_setup;
+
+		ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0];
+		ring_enc->wptr = 0;
+		rb_enc_addr = ring_enc->gpu_addr;
+
+		rb_setup->is_rb_enabled_flags |= RB_ENABLED;
+		rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
+		rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
+		rb_setup->rb_size = ring_enc->ring_size / 4;
+		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
+
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
+			lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr));
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
+			upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr));
+		MMSCH_V5_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0,
+			regUVD_VCPU_NONCACHE_SIZE0),
+			AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
+		MMSCH_V5_0_INSERT_END();
+
+		header.vcn0.init_status = 0;
+		header.vcn0.table_offset = header.total_size;
+		header.vcn0.table_size = table_size;
+		header.total_size += table_size;
+
+		/* Send init table to mmsch */
+		size = sizeof(struct mmsch_v5_0_init_header);
+		table_loc = (uint32_t *)table->cpu_addr;
+		memcpy((void *)table_loc, &header, size);
+
+		ctx_addr = table->gpu_addr;
+		WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
+		WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
+
+		tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID);
+		tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
+		tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
+		WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp);
+
+		size = header.total_size;
+		WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size);
+
+		WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0);
+
+		param = 0x00000001;
+		WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param);
+		tmp = 0;
+		timeout = amdgpu_emu_mode ? 1000000 : 1000;
+		resp = 0;
+		expected = MMSCH_VF_MAILBOX_RESP__OK;
+		while (resp != expected) {
+			resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP);
+			if (resp != 0)
+				break;
+
+			udelay(10);
+			tmp = tmp + 10;
+			if (tmp >= timeout) {
+				DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
+					" waiting for regMMSCH_VF_MAILBOX_RESP "\
+					"(expected=0x%08x, readback=0x%08x)\n",
+					tmp, expected, resp);
+				return -EBUSY;
+			}
+		}
+
+		init_status = ((struct mmsch_v5_0_init_header *)(table_loc))->vcn0.init_status;
+		if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
+					&& init_status != MMSCH_VF_ENGINE_STATUS__PASS) {
+			DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
+				"status for VCN%x: 0x%x\n", resp, vcn_inst, init_status);
+		}
+	}
+
+	return 0;
+}
+
 /**
  * vcn_v5_0_2_start - VCN start
  *
@@ -1040,24 +1265,6 @@ static void vcn_v5_0_2_set_unified_ring_funcs(struct amdgpu_device *adev)
 	}
 }
 
-/**
- * vcn_v5_0_2_is_idle - check VCN block is idle
- *
- * @ip_block: Pointer to the amdgpu_ip_block structure
- *
- * Check whether VCN block is idle
- */
-static bool vcn_v5_0_2_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	struct amdgpu_device *adev = ip_block->adev;
-	int i, ret = 1;
-
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i)
-		ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == UVD_STATUS__IDLE);
-
-	return ret;
-}
-
 /**
  * vcn_v5_0_2_wait_for_idle - wait for VCN block idle
  *
@@ -1113,8 +1320,18 @@ static int vcn_v5_0_2_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 static int vcn_v5_0_2_set_pg_state(struct amdgpu_vcn_inst *vinst,
 				   enum amd_powergating_state state)
 {
+	struct amdgpu_device *adev = vinst->adev;
 	int ret = 0;
 
+	/* for SRIOV, guest should not control VCN Power-gating
+	 * MMSCH FW should control Power-gating and clock-gating
+	 * guest should avoid touching CGC and PG
+	 */
+	if (amdgpu_sriov_vf(adev)) {
+		vinst->cur_state = AMD_PG_STATE_UNGATE;
+		return 0;
+	}
+
 	if (state == vinst->cur_state)
 		return 0;
 
@@ -1141,26 +1358,55 @@ static int vcn_v5_0_2_set_pg_state(struct amdgpu_vcn_inst *vinst,
 static int vcn_v5_0_2_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
 	struct amdgpu_iv_entry *entry)
 {
-	uint32_t i, inst;
-
-	i = node_id_to_phys_map[entry->node_id];
+	uint32_t mid, cid;
+	int i, inst;
 
 	DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n");
 
-	for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst)
-		if (adev->vcn.inst[inst].aid_id == i)
+	switch (entry->client_id) {
+	case SOC_V1_0_IH_CLIENTID_VCN:
+		cid = 0;
+		break;
+	case SOC_V1_0_IH_CLIENTID_VCN1:
+		cid = 1;
+		break;
+	default:
+		dev_WARN_ONCE(adev->dev, 1,
+				"Interrupt received for unknown VCN client_id %d",
+				entry->client_id);
+		return 0;
+	}
+
+	switch (entry->node_id) {
+	case 0:
+		mid = 0;
+		break;
+	case 8:
+		mid = 1;
+		break;
+	default:
+		dev_WARN_ONCE(adev->dev, 1,
+				"Interrupt received for unknown VCN node_id %d",
+				entry->node_id);
+		return 0;
+	}
+
+	inst = mid * adev->vcn.num_inst_per_aid + cid;
+
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++)
+		if (GET_INST(VCN, i) == inst)
 			break;
 
-	if (inst >= adev->vcn.num_vcn_inst) {
+	if (i >= adev->vcn.num_vcn_inst) {
 		dev_WARN_ONCE(adev->dev, 1,
 				"Interrupt received for unknown VCN instance %d",
-				entry->node_id);
+				inst);
 		return 0;
 	}
 
 	switch (entry->src_id) {
 	case VCN_5_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
-		amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]);
+		amdgpu_fence_process(&adev->vcn.inst[i].ring_enc[0]);
 		break;
 	default:
 		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
@@ -1201,7 +1447,6 @@ static const struct amd_ip_funcs vcn_v5_0_2_ip_funcs = {
 	.hw_fini = vcn_v5_0_2_hw_fini,
 	.suspend = vcn_v5_0_2_suspend,
 	.resume = vcn_v5_0_2_resume,
-	.is_idle = vcn_v5_0_2_is_idle,
 	.wait_for_idle = vcn_v5_0_2_wait_for_idle,
 	.soft_reset = NULL,
 	.set_clockgating_state = vcn_v5_0_2_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index eb16916c647383..4509e6130bc2af 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -555,12 +555,6 @@ static int vega10_ih_resume(struct amdgpu_ip_block *ip_block)
 	return vega10_ih_hw_init(ip_block);
 }
 
-static bool vega10_ih_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-	return true;
-}
-
 static int vega10_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* todo */
@@ -630,7 +624,6 @@ const struct amd_ip_funcs vega10_ih_ip_funcs = {
 	.hw_fini = vega10_ih_hw_fini,
 	.suspend = vega10_ih_suspend,
 	.resume = vega10_ih_resume,
-	.is_idle = vega10_ih_is_idle,
 	.wait_for_idle = vega10_ih_wait_for_idle,
 	.soft_reset = vega10_ih_soft_reset,
 	.set_clockgating_state = vega10_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c
index 85846fd08ce4de..460f4bf195e8ca 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c
@@ -655,12 +655,6 @@ static int vega20_ih_resume(struct amdgpu_ip_block *ip_block)
 	return vega20_ih_hw_init(ip_block);
 }
 
-static bool vega20_ih_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* todo */
-	return true;
-}
-
 static int vega20_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* todo */
@@ -727,7 +721,6 @@ const struct amd_ip_funcs vega20_ih_ip_funcs = {
 	.hw_fini = vega20_ih_hw_fini,
 	.suspend = vega20_ih_suspend,
 	.resume = vega20_ih_resume,
-	.is_idle = vega20_ih_is_idle,
 	.wait_for_idle = vega20_ih_wait_for_idle,
 	.soft_reset = vega20_ih_soft_reset,
 	.set_clockgating_state = vega20_ih_set_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 352c2710330546..2cd6d7d77b1cd7 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -1729,11 +1729,6 @@ static int vi_common_resume(struct amdgpu_ip_block *ip_block)
 	return vi_common_hw_init(ip_block);
 }
 
-static bool vi_common_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static void vi_update_bif_medium_grain_light_sleep(struct amdgpu_device *adev,
 						   bool enable)
 {
@@ -2025,7 +2020,6 @@ static const struct amd_ip_funcs vi_common_ip_funcs = {
 	.hw_fini = vi_common_hw_fini,
 	.suspend = vi_common_suspend,
 	.resume = vi_common_resume,
-	.is_idle = vi_common_is_idle,
 	.set_clockgating_state = vi_common_set_clockgating_state,
 	.set_powergating_state = vi_common_set_powergating_state,
 	.get_clockgating_state = vi_common_get_clockgating_state,
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
index 54fa76f374c903..5cf306de0d9681 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -3644,18 +3644,24 @@ static const uint32_t cwsr_trap_gfx9_4_3_hex[] = {
 };
 
 static const uint32_t cwsr_trap_gfx12_hex[] = {
-	0xbfa00001, 0xbfa00239,
-	0xb0804009, 0xb8eef81a,
-	0xbf880000, 0xb980081a,
-	0x00000000, 0xb8f8f804,
+	0xbfa00001, 0xbfa00255,
+	0xb0804009, 0xb8f8f804,
+	0xb8eef81a, 0xbf880000,
+	0xb980081a, 0x00000000,
 	0x9177ff77, 0x0c000000,
 	0x846e9a6e, 0x8c776e77,
 	0x9178ff78, 0x00008c00,
-	0xb8fbf811, 0x8b6eff78,
+	0xb8fbf811, 0xbf0c877b,
+	0xbfa20002, 0x916dff6d,
+	0xf0000000, 0xbf0d857b,
+	0xbfa10006, 0x917bff7b,
+	0x00000480, 0xb98001d1,
+	0x00000000, 0xb9800291,
+	0x00000000, 0x8b6eff78,
 	0x00004000, 0xbfa10008,
-	0x8b6eff7b, 0x00000080,
+	0x8b6eff7b, 0x00000480,
 	0xbfa20018, 0x8b6ea07b,
-	0xbfa2004a, 0xbf830010,
+	0xbfa2005a, 0xbf830010,
 	0xb8fbf811, 0xbfa0fffb,
 	0x8b6eff7b, 0x00000bd0,
 	0xbfa20010, 0xb8eef812,
@@ -3666,16 +3672,24 @@ static const uint32_t cwsr_trap_gfx12_hex[] = {
 	0xf0000000, 0xbfa20005,
 	0x8b6fff6f, 0x00000200,
 	0xbfa20002, 0x8b6ea07b,
-	0xbfa20034, 0xbefa4d82,
-	0xbf8a0000, 0x84fa887a,
-	0xbf0d8f7b, 0xbfa10002,
-	0x8c7bff7b, 0xffff0000,
-	0x8b6eff77, 0x0c000000,
-	0x916dff6d, 0x0c000000,
-	0x8c6d6e6d, 0xf4601bbd,
-	0xf8000010, 0xbf8a0000,
-	0x846e976e, 0x9177ff77,
-	0x00800000, 0x8c776e77,
+	0xbfa20044, 0xbeef007b,
+	0xbefa4d82, 0xbf8a0000,
+	0x84fa887a, 0xbf0d8f7b,
+	0xbfa10002, 0x8c7bff7b,
+	0xffff0000, 0x8b6eff77,
+	0x0c000000, 0x916dff6d,
+	0x0c000000, 0x8c6d6e6d,
+	0xf4601bbd, 0xf8000010,
+	0xbf8a0000, 0x846e976e,
+	0x9177ff77, 0x03800000,
+	0x8c776e77, 0x8b6eff6f,
+	0x00000480, 0xbfa1000c,
+	0xbf0d8e78, 0xbfa20002,
+	0xbf0d9877, 0xbfa20002,
+	0xb98001d1, 0x00000000,
+	0xbf0d8e78, 0xbfa20002,
+	0xbf0d9977, 0xbfa20002,
+	0xb9800291, 0x00000000,
 	0xf4603bbd, 0xf8000000,
 	0xbf8a0000, 0xf4603ebd,
 	0xf8000008, 0xbf8a0000,
@@ -4587,14 +4601,24 @@ static const uint32_t cwsr_trap_gfx9_5_0_hex[] = {
 };
 
 static const uint32_t cwsr_trap_gfx12_1_0_hex[] = {
-	0xbfa00001, 0xbfa003be,
+	0xbfa00001, 0xbfa003e2,
 	0xb0804009, 0xb8f8f804,
+	0xbefa007e, 0xbefe0081,
+	0xbeee0080, 0xbeef00c1,
+	0xee17406e, 0x00040000,
+	0x00000000, 0xbefe007a,
 	0x9178ff78, 0x00008c00,
-	0xb8fbf811, 0x8b6eff78,
+	0xb8fbf811, 0xbf0c877b,
+	0xbfa20002, 0x916dff6d,
+	0xf0000000, 0xbf0d857b,
+	0xbfa10006, 0x917bff7b,
+	0x00000480, 0xb98001d1,
+	0x00000000, 0xb9800291,
+	0x00000000, 0x8b6eff78,
 	0x00004000, 0xbfa10008,
-	0x8b6eff7b, 0x00000080,
+	0x8b6eff7b, 0x00000480,
 	0xbfa20018, 0x8b6ea07b,
-	0xbfa200da, 0xbf830010,
+	0xbfa200ea, 0xbf830010,
 	0xb8fbf811, 0xbfa0fffb,
 	0x8b6eff7b, 0x00000bd0,
 	0xbfa20010, 0xb8eef812,
@@ -4605,20 +4629,28 @@ static const uint32_t cwsr_trap_gfx12_1_0_hex[] = {
 	0xf0000000, 0xbfa20005,
 	0x8b6fff6f, 0x00000200,
 	0xbfa20002, 0x8b6ea07b,
-	0xbfa200c4, 0x9177ff77,
+	0xbfa200d4, 0x9177ff77,
 	0x007fc000, 0xb8fa04a1,
 	0x847a967a, 0x8c777a77,
 	0xb8fa0421, 0x847a957a,
 	0x8c777a77, 0xb8fa3021,
 	0x847a8e7a, 0x8c777a77,
 	0xb980f821, 0x00000000,
-	0xbefa4d82, 0xbf8a0000,
-	0x84fa887a, 0xbf0d987b,
-	0xbfa10002, 0x8c7bff7b,
-	0xfe000000, 0xf4601bbd,
-	0xf8000010, 0xbf8a0000,
-	0x846e976e, 0x9177ff77,
-	0x00800000, 0x8c776e77,
+	0xbeef007b, 0xbefa4d82,
+	0xbf8a0000, 0x84fa887a,
+	0xbf0d987b, 0xbfa10002,
+	0x8c7bff7b, 0xfe000000,
+	0xf4601bbd, 0xf8000010,
+	0xbf8a0000, 0x846e976e,
+	0x9177ff77, 0x03800000,
+	0x8c776e77, 0x8b6eff6f,
+	0x00000480, 0xbfa1000c,
+	0xbf0d8e78, 0xbfa20002,
+	0xbf0d9877, 0xbfa20002,
+	0xb98001d1, 0x00000000,
+	0xbf0d8e78, 0xbfa20002,
+	0xbf0d9977, 0xbfa20002,
+	0xb9800291, 0x00000000,
 	0xf4603bbd, 0xf8000000,
 	0xbf8a0000, 0xf4603ebd,
 	0xf8000008, 0xbf8a0000,
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
index d38ff404277b4e..b5af2c97a0916b 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm
@@ -41,13 +41,19 @@
 
 #define SINGLE_STEP_MISSED_WORKAROUND 1	//workaround for lost TRAP_AFTER_INST exception when SAVECTX raised
 #define HAVE_VALU_SGPR_HAZARD (ASIC_FAMILY == CHIP_GFX12)
+#define HAVE_HT_TRAP_ID_WA (ASIC_FAMILY >= CHIP_GFX12)
 #define WAVE32_ONLY (ASIC_FAMILY == CHIP_GC_12_0_3)
 #define SAVE_TTMPS_IN_SGPR_BLOCK (ASIC_FAMILY >= CHIP_GC_12_0_3)
+#define VMEM_ON_TRAP_ENTRY_WA (ASIC_FAMILY == CHIP_GC_12_0_3)
 
 #if HAVE_XNACK && !WAVE32_ONLY
 # error
 #endif
 
+#if VMEM_ON_TRAP_ENTRY_WA && RELAXED_SCHEDULING_IN_TRAP
+# error
+#endif
+
 #define ADDRESS_HI32_NUM_BITS ((HAVE_57BIT_ADDRESS ? 57 : 48) - 32)
 #define ADDRESS_HI32_MASK ((1 << ADDRESS_HI32_NUM_BITS) - 1)
 
@@ -60,6 +66,7 @@ var SQ_WAVE_STATE_PRIV_POISON_ERR_SHIFT		= 15
 var SQ_WAVE_STATUS_WAVE64_SHIFT			= 29
 var SQ_WAVE_STATUS_WAVE64_SIZE			= 1
 var SQ_WAVE_STATUS_NO_VGPRS_SHIFT		= 24
+var SQ_WAVE_STATUS_HALT_SHIFT			= 14
 var SQ_WAVE_STATUS_IN_WG_SHIFT			= 11
 var SQ_WAVE_STATE_PRIV_ALWAYS_CLEAR_MASK	= SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK|SQ_WAVE_STATE_PRIV_POISON_ERR_MASK
 var S_SAVE_PC_HI_TRAP_ID_MASK			= 0xF0000000
@@ -87,6 +94,8 @@ var SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_SHIFT	= 7
 var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_MASK	= 0x100
 var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT	= 8
 var SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_MASK	= 0x200
+var SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT_SHIFT	= 10
+var SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT_MASK	= 0x400
 var SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_MASK	= 0x800
 var SQ_WAVE_TRAP_CTRL_ADDR_WATCH_MASK		= 0x80
 var SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST_MASK	= 0x200
@@ -134,6 +143,10 @@ var SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SHIFT	= 12
 var SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SIZE	= 6
 #endif
 
+var TTMP11_STOCHASTIC_TRAP_ENABLED_SHIFT   	= 25
+var TTMP11_STOCHASTIC_TRAP_ENABLED_MASK    	= (1 << TTMP11_STOCHASTIC_TRAP_ENABLED_SHIFT)
+var TTMP11_HOST_TRAP_ENABLED_SHIFT		= 24
+var TTMP11_HOST_TRAP_ENABLED_MASK		= (1 << TTMP11_HOST_TRAP_ENABLED_SHIFT)
 var TTMP11_DEBUG_TRAP_ENABLED_SHIFT		= 23
 var TTMP11_DEBUG_TRAP_ENABLED_MASK		= 0x800000
 var TTMP11_FIRST_REPLAY_SHIFT			= 22
@@ -143,6 +156,9 @@ var TTMP11_REPLAY_W64H_MASK			= 0x200000
 var TTMP11_FXPTR_SHIFT				= 14
 var TTMP11_FXPTR_MASK				= 0x1FC000
 
+var TTMP11_FEATURES_ENABLED_FLAGS_SHIFT		= TTMP11_DEBUG_TRAP_ENABLED_SHIFT
+var TTMP11_FEATURES_ENABLED_FLAGS_MASK		= TTMP11_DEBUG_TRAP_ENABLED_MASK | TTMP11_HOST_TRAP_ENABLED_MASK | TTMP11_STOCHASTIC_TRAP_ENABLED_MASK
+
 // SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14]
 // when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
 var S_SAVE_BUF_RSRC_WORD1_STRIDE		= 0x00040000
@@ -227,6 +243,21 @@ L_JUMP_TO_RESTORE:
 	s_branch	L_RESTORE
 
 L_SKIP_RESTORE:
+	s_getreg_b32	s_save_state_priv, hwreg(HW_REG_WAVE_STATE_PRIV)	//save STATUS since we will change SCC
+
+#if VMEM_ON_TRAP_ENTRY_WA
+	s_mov_b32	ttmp14, exec_lo
+	s_mov_b32	exec_lo, 1
+
+	// No-op prefetch to random address 0xFFFFFFFFxxxxxxxx.
+	// Avoids user allocations which may be functionally affected by prefetch.
+	s_mov_b32	ttmp2, 0
+	s_mov_b32	ttmp3, 0xFFFFFFFF
+	global_prefetch_b8	v0, [ttmp2, ttmp3] scope:SCOPE_SE th:TH_LOAD_RT
+
+	s_mov_b32	exec_lo, ttmp14
+#endif
+
 #if RELAXED_SCHEDULING_IN_TRAP
 	// Assume most relaxed scheduling mode is set. Save and revert to normal mode.
 	s_getreg_b32	ttmp2, hwreg(HW_REG_WAVE_SCHED_MODE)
@@ -235,8 +266,6 @@ L_SKIP_RESTORE:
 		SQ_WAVE_SCHED_MODE_DEP_MODE_SHIFT, SQ_WAVE_SCHED_MODE_DEP_MODE_SIZE), 0
 #endif
 
-	s_getreg_b32	s_save_state_priv, hwreg(HW_REG_WAVE_STATE_PRIV)	//save STATUS since we will change SCC
-
 #if RELAXED_SCHEDULING_IN_TRAP
 	// Save SCHED_MODE[1:0] into ttmp11[27:26].
 	s_andn2_b32	ttmp11, ttmp11, TTMP11_SCHED_MODE_MASK
@@ -250,12 +279,54 @@ L_SKIP_RESTORE:
 
 	s_getreg_b32	s_save_excp_flag_priv, hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV)
 
+#if HAVE_HT_TRAP_ID_WA
+	// If we have both TRAP_ID (ttmp1[31:28]) != 0, and
+	// EXCP_FLAG_PRIV.HOST_TRAP == 1, we cannot reliably know if we entered
+	// the trap handler for an s_trap or a host trap.
+	//
+	// In both cases, if we see HOST_TRAP == 1, we can clear the TRAP_ID
+	// bits:
+	// - If this was a host trap, all good, those bits should have been 0
+	//   anyway, so they can be cleared.
+	// - If this was an s_trap, ttmp[0:1] still points to the s_trap
+	//   instruction, and it will be the first to be executed after s_rfe.
+	//   We do not change the behavior of the program, by not handling the
+	//   trap just yet, assume this was just a host trap and deal with the
+	//   s_trap on next entry.
+	s_bitcmp0_b32   s_save_excp_flag_priv, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_SHIFT
+	s_cbranch_scc1  L_NOT_HOST_TRAP_WA
+	s_andn2_b32     s_save_pc_hi, s_save_pc_hi, S_SAVE_PC_HI_TRAP_ID_MASK
+
+L_NOT_HOST_TRAP_WA:
+	// From now on in the trap handler, if we see TTMP1.TRAP_ID != 0, then it has
+	// to be because of a s_trap. If the second level trap handler sees
+	// both TRAP_ID != 0 and EXCP_FLAG_PRIV.HOST_TRAP == 1, it is because
+	// a HOST_TRAP was received while we were in the trap handler, future host
+	// trap can't influence ttmp1. So both the s_trap and host_trap can be handled
+	// in one go.
+#endif
+
+	// Ignore HOSTTRAP/PERF_SNAPSHOT if SAVECTX is also present
+	// Usually s_trap has a higher priority than context_save, HAVE_HT_TRAP_ID_WA
+	// will reverse these priority's order when s_trap, host_trap and context save
+	// are raised at the same time. We will end-up ignoring the s_trap (because of
+	// the ambiguity with the host trap), ignoring the host trap (to be sure high
+	// frequency sampling cannot prevent CWSR) and directly doing the context save.
+	// The s_trap will be re-executed after context save/restore.
+	s_bitcmp1_b32   s_save_excp_flag_priv, SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT
+	s_cbranch_scc0  L_NO_SAVE_AND_HT_ST
+	s_andn2_b32     s_save_excp_flag_priv, s_save_excp_flag_priv, \
+			SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK | SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT_MASK
+	s_setreg_imm32_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_SHIFT, 1), 0
+	s_setreg_imm32_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV, SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT_SHIFT, 1), 0
+
+L_NO_SAVE_AND_HT_ST:
 	s_and_b32       ttmp2, s_save_state_priv, SQ_WAVE_STATE_PRIV_HALT_MASK
 	s_cbranch_scc0	L_NOT_HALTED
 
-L_HALTED:
-	// Host trap may occur while wave is halted.
-	s_and_b32	ttmp2, s_save_excp_flag_priv, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK
+	// Host trap / stochastic may occur while wave is halted.
+	s_and_b32	ttmp2, s_save_excp_flag_priv, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK |\
+					SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT_MASK
 	s_cbranch_scc1	L_FETCH_2ND_TRAP
 
 L_CHECK_SAVE:
@@ -316,6 +387,7 @@ L_FETCH_2ND_TRAP:
 	// Read second-level TBA/TMA from first-level TMA and jump if available.
 	// ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data)
 	// ttmp12 holds SQ_WAVE_STATUS
+	s_mov_b32       ttmp3, s_save_excp_flag_priv
 	s_sendmsg_rtn_b64       [ttmp14, ttmp15], sendmsg(MSG_RTN_GET_TMA)
 	s_wait_idle
 	s_lshl_b64	[ttmp14, ttmp15], [ttmp14, ttmp15], 0x8
@@ -332,12 +404,46 @@ L_NO_SIGN_EXTEND_TMA:
 	s_or_b32	ttmp1, ttmp1, ttmp2
 #endif
 
-	s_load_dword    ttmp2, [ttmp14, ttmp15], 0x10 scope:SCOPE_SYS		// debug trap enabled flag
+	s_load_dword    ttmp2, [ttmp14, ttmp15], 0x10 scope:SCOPE_SYS           // trap enabled flag
 	s_wait_idle
-	s_lshl_b32      ttmp2, ttmp2, TTMP11_DEBUG_TRAP_ENABLED_SHIFT
-	s_andn2_b32     ttmp11, ttmp11, TTMP11_DEBUG_TRAP_ENABLED_MASK
+	s_lshl_b32      ttmp2, ttmp2, TTMP11_FEATURES_ENABLED_FLAGS_SHIFT
+	s_andn2_b32     ttmp11, ttmp11, TTMP11_FEATURES_ENABLED_FLAGS_MASK
 	s_or_b32        ttmp11, ttmp11, ttmp2
 
+	// If not a host / stochastic trap , then driver cannot mask this
+	s_and_b32	ttmp2, ttmp3, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_MASK |\
+					SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT_MASK
+	s_cbranch_scc0	L_NO_EXCEPTION_MASKING
+
+	s_bitcmp1_b32 s_save_state_priv, SQ_WAVE_STATUS_HALT_SHIFT
+	s_cbranch_scc1 L_MASK_HOST_TRAP
+
+	// Handle masking host traps if requested by the driver.
+	s_bitcmp1_b32   ttmp11, TTMP11_HOST_TRAP_ENABLED_SHIFT
+
+	// We got here because of either a non-maskable exception or a maskable
+	// one (HT).  The driver said the maskable HT should not be masked, so
+	// we can directly carry on to the 2nd level trap handler.
+	s_cbranch_scc1  L_CHECK_STOC
+
+L_MASK_HOST_TRAP:
+	// The driver asked to mask the HT, be sure to do so.
+	s_setreg_imm32_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV, SQ_WAVE_EXCP_FLAG_PRIV_HOST_TRAP_SHIFT, 1), 0x0
+
+L_CHECK_STOC:
+	s_bitcmp1_b32 s_save_state_priv, SQ_WAVE_STATUS_HALT_SHIFT
+	s_cbranch_scc1 L_MASKED_STOC
+
+	// If stochastic sampling trap was requested, should it be masked by the driver?
+	s_bitcmp1_b32    ttmp11, TTMP11_STOCHASTIC_TRAP_ENABLED_SHIFT
+	// If driver said stochastic traps are OK, allow the 2nd-level handler to take stochastic trap
+	s_cbranch_scc1   L_NO_EXCEPTION_MASKING
+
+L_MASKED_STOC:
+	// Otherwise, zero out TRAPSTS.stochastic_perf_snapshot bit, so possible 2nd level does not handle it
+	s_setreg_imm32_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV, SQ_WAVE_EXCP_FLAG_PRIV_PERF_SNAPSHOT_SHIFT, 1), 0x0
+
+L_NO_EXCEPTION_MASKING:
 	s_load_dwordx2	[ttmp2, ttmp3], [ttmp14, ttmp15], 0x0 scope:SCOPE_SYS	// second-level TBA
 	s_wait_idle
 	s_load_dwordx2	[ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 scope:SCOPE_SYS	// second-level TMA
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 504a286368ebd1..344da6c0e96ad5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -35,6 +35,7 @@
 #include <linux/time.h>
 #include <linux/mm.h>
 #include <linux/mman.h>
+#include <linux/pseudo_fs.h>
 #include <linux/ptrace.h>
 #include <linux/dma-buf.h>
 #include <linux/processor.h>
@@ -70,18 +71,54 @@ static const struct class kfd_class = {
 };
 
 /*
- * Cache the address space of the chardev on first open so that the reset
- * path can drop all userspace mappings of doorbell and MMIO ranges via
- * unmap_mapping_range().
+ * Private pseudo-filesystem for KFD, Provides a stable, module-owned
+ * inode whose address_space is the unmap target for all /dev/kfd
+ * openers during GPU reset.
  */
-static struct address_space *kfd_dev_mapping;
+static struct vfsmount *kfd_fs_mnt;
+static int kfd_fs_cnt;
 
-void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen)
+static int kfd_fs_init_fs_context(struct fs_context *fc)
+{
+	return init_pseudo(fc, 0x4b464400 /* "KFD" */) ? 0 : -ENOMEM;
+}
+
+static struct file_system_type kfd_fs_type = {
+	.name		= "kfd",
+	.init_fs_context = kfd_fs_init_fs_context,
+	.kill_sb	= kill_anon_super,
+};
+
+static struct inode *kfd_fs_inode_new(void)
 {
-	struct address_space *mapping = READ_ONCE(kfd_dev_mapping);
+	struct inode *inode;
+	int r;
+
+	r = simple_pin_fs(&kfd_fs_type, &kfd_fs_mnt, &kfd_fs_cnt);
+	if (r < 0)
+		return ERR_PTR(r);
+
+	inode = alloc_anon_inode(kfd_fs_mnt->mnt_sb);
+	if (IS_ERR(inode))
+		simple_release_fs(&kfd_fs_mnt, &kfd_fs_cnt);
 
-	if (mapping)
-		unmap_mapping_range(mapping, holebegin, holelen, 1);
+	return inode;
+}
+
+static void kfd_fs_inode_free(struct inode *inode)
+{
+	if (inode) {
+		iput(inode);
+		simple_release_fs(&kfd_fs_mnt, &kfd_fs_cnt);
+	}
+}
+
+static struct inode *kfd_anon_inode;
+
+void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen)
+{
+	if (kfd_anon_inode)
+		unmap_mapping_range(kfd_anon_inode->i_mapping, holebegin, holelen, 1);
 }
 
 static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
@@ -107,6 +144,13 @@ int kfd_chardev_init(void)
 {
 	int err = 0;
 
+	kfd_anon_inode = kfd_fs_inode_new();
+	if (IS_ERR(kfd_anon_inode)) {
+		err = PTR_ERR(kfd_anon_inode);
+		kfd_anon_inode = NULL;
+		return err;
+	}
+
 	kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
 	err = kfd_char_dev_major;
 	if (err < 0)
@@ -130,6 +174,8 @@ int kfd_chardev_init(void)
 err_class_create:
 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
 err_register_chrdev:
+	kfd_fs_inode_free(kfd_anon_inode);
+	kfd_anon_inode = NULL;
 	return err;
 }
 
@@ -138,6 +184,8 @@ void kfd_chardev_exit(void)
 	device_destroy(&kfd_class, MKDEV(kfd_char_dev_major, 0));
 	class_unregister(&kfd_class);
 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
+	kfd_fs_inode_free(kfd_anon_inode);
+	kfd_anon_inode = NULL;
 	kfd_device = NULL;
 }
 
@@ -150,12 +198,7 @@ static int kfd_open(struct inode *inode, struct file *filep)
 	if (iminor(inode) != 0)
 		return -ENODEV;
 
-	/*
-	 * /dev/kfd is a single chardev so all opens share one inode. Cache
-	 * its address_space on the first open for use by the reset path.
-	 */
-	if (!READ_ONCE(kfd_dev_mapping))
-		cmpxchg(&kfd_dev_mapping, NULL, inode->i_mapping);
+	filep->f_mapping = kfd_anon_inode->i_mapping;
 
 	is_32bit_user_mode = in_compat_syscall();
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 586e640f13dcec..23db8dcb076dd5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -34,6 +34,7 @@
 #include "kfd_svm.h"
 #include "kfd_migrate.h"
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_xcp.h"
 
 #define MQD_SIZE_ALIGNED 768
@@ -952,7 +953,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
 	svm_range_set_max_pages(kfd->adev);
 
-	kfd->init_complete = true;
+	/* Release pairs with the acquire in kfd_init_apertures(). */
+	smp_store_release(&kfd->init_complete, true);
 	dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,
 		 kfd->adev->pdev->device);
 
@@ -977,7 +979,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 	return kfd->init_complete;
 }
 
-void kgd2kfd_device_exit(struct kfd_dev *kfd)
+void kgd2kfd_device_fini(struct kfd_dev *kfd)
 {
 	if (kfd->init_complete) {
 		/* Cleanup KFD nodes */
@@ -991,6 +993,11 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
 	}
 
 	kfree(kfd);
+}
+
+void kgd2kfd_device_exit(struct kfd_dev *kfd)
+{
+	kgd2kfd_device_fini(kfd);
 
 	/* after remove a kfd device unlock kfd driver */
 	kgd2kfd_unlock_kfd(NULL);
@@ -1512,8 +1519,11 @@ void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)
  */
 unsigned int kfd_get_num_sdma_engines(struct kfd_node *node)
 {
-	/* If XGMI is not supported, all SDMA engines are PCIe */
-	if (!node->adev->gmc.xgmi.supported)
+	/* If XGMI is not supported, all SDMA engines are PCIe.
+	 * Also, on GC 12.1, all SDMA engines are the same.
+	 */
+	if (!node->adev->gmc.xgmi.supported ||
+	    KFD_GC_VERSION(node->kfd) == IP_VERSION(12, 1, 0))
 		return node->adev->sdma.num_instances/(int)node->kfd->num_nodes;
 
 	return min(node->adev->sdma.num_instances/(int)node->kfd->num_nodes, 2);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 2f78395a0c31cb..ace5609fbb4b0a 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -47,6 +47,9 @@
 /* See unmap_queues_cpsch() */
 #define USE_DEFAULT_GRACE_PERIOD 0xffffffff
 
+/* Interval for notifying MES of work on unmapped queues during oversubscription */
+#define DQM_MES_UNMAP_NOTIFY_DELAY_US 50
+
 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
 				  u32 pasid, unsigned int vmid);
 
@@ -276,8 +279,23 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 			q->properties.doorbell_off);
 		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 		kfd_hws_hang(dqm);
+		return r;
 	}
 
+	/*
+	 * GFX11: start notify timer only once when oversubscription begins.
+	 * Skip under SR-IOV: MES round-trips are far slower there, and this
+	 * notify call shares mes->mutex_hidden with add/remove_hw_queue, so
+	 * a slow notify can stall remapping queues. SR-IOV guests keep MES's
+	 * own firmware oversubscription timer instead.
+	 */
+	if (!amdgpu_sriov_vf(adev) &&
+	    KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
+	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
+	    dqm->active_cp_queue_count > get_cp_queues_num(dqm))
+		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
+				   usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US));
+
 	return r;
 }
 
@@ -354,7 +372,16 @@ static void set_perfcount(struct device_queue_manager *dqm, int enable)
 static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 			    struct qcm_process_device *qpd)
 {
-	return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+	int r = remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+
+	/* GFX11: stop notify timer when oversubscription clears */
+	if (!r &&
+	    KFD_GC_VERSION(dqm->dev) >= IP_VERSION(11, 0, 0) &&
+	    KFD_GC_VERSION(dqm->dev) < IP_VERSION(12, 0, 0) &&
+	    dqm->active_cp_queue_count <= get_cp_queues_num(dqm))
+		cancel_delayed_work(&dqm->notify_unmap_work);
+
+	return r;
 }
 
 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
@@ -1479,6 +1506,67 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
 	return retval;
 }
 
+static int clean_process_queues_cpsch(struct device_queue_manager *dqm,
+				      struct qcm_process_device *qpd)
+{
+	struct queue *q;
+	struct kfd_process_device *pdd;
+	int retval = 0;
+
+	dqm_lock(dqm);
+	if (qpd->evicted++ > 0) /* already evicted, do nothing */
+		goto out;
+
+	pdd = qpd_to_pdd(qpd);
+
+	/* The debugger creates processes that temporarily have not acquired
+	 * all VMs for all devices and has no VMs itself.
+	 * Skip queue eviction on process eviction.
+	 */
+	if (!pdd->drm_priv)
+		goto out;
+
+	pr_debug_ratelimited("Evicting process pid %d queues\n",
+			    pdd->process->lead_thread->pid);
+
+	/* Mark all queues as evicted. Deactivate all active queues on
+	 * the qpd.
+	 */
+	list_for_each_entry(q, &qpd->queues_list, list) {
+		q->properties.is_evicted = true;
+		if (!q->properties.is_active)
+			continue;
+
+		q->properties.is_active = false;
+		decrement_queue_count(dqm, qpd, q);
+
+		dqm_evict_mqd_bo(dqm, q);
+	}
+
+	pdd->last_evict_timestamp = get_jiffies_64();
+
+	if (!down_read_trylock(&dqm->dev->adev->reset_domain->sem)) {
+		retval =  -EIO;
+		goto out;
+	}
+
+	retval = unmap_queues_cpsch(dqm,
+			    KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
+			    0,
+			    USE_DEFAULT_GRACE_PERIOD,
+			    false);
+
+	if (!retval) {
+		dqm->dev->kfd2kgd->hqd_gfx_clean_fault(dqm->dev->adev);
+		retval = map_queues_cpsch(dqm);
+	}
+	up_read(&dqm->dev->adev->reset_domain->sem);
+
+out:
+	dqm_unlock(dqm);
+	return retval;
+}
+
 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
 					  struct qcm_process_device *qpd)
 {
@@ -2061,6 +2149,43 @@ static int unhalt_cpsch(struct device_queue_manager *dqm)
 	return ret;
 }
 
+/* Program CP_IQ_WAIT_TIME2 via the MES WRITE_REG op (MES has no HIQ). */
+static int init_dequeue_wait_counts_mes(struct device_queue_manager *dqm)
+{
+	struct kfd_node *dev = dqm->dev;
+	struct amdgpu_device *adev = dev->adev;
+	uint32_t sch_wave = 0, que_sleep = 1;
+	int inst, ret;
+
+	if (!dev->kfd2kgd->build_dequeue_wait_counts_packet_info)
+		return 0;
+
+	if (KFD_GC_VERSION(dev) != IP_VERSION(12, 1, 0))
+		return 0;
+
+	/* CP_IQ_WAIT_TIME2 is per-XCC; the offset must encode the target
+	 * XCC so MES routes the write to that XCC's local register.
+	 */
+	for_each_inst(inst, dev->xcc_mask) {
+		uint32_t reg_offset = 0, reg_data = 0;
+
+		dev->kfd2kgd->build_dequeue_wait_counts_packet_info(
+				adev, dqm->wait_times, sch_wave, que_sleep,
+				&reg_offset, &reg_data, inst);
+
+		ret = amdgpu_mes_wreg(adev, reg_offset, reg_data, inst);
+		if (ret) {
+			dev_err(adev->dev,
+				"Failed to set optimized dequeue wait via MES on xcc %d\n",
+				inst);
+			return ret;
+		}
+	}
+
+	update_dqm_wait_times(dqm);
+	return 0;
+}
+
 static int start_cpsch(struct device_queue_manager *dqm)
 {
 	struct device *dev = dqm->dev->adev->dev;
@@ -2102,6 +2227,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
 				KFD_DEQUEUE_WAIT_INIT, 0 /* unused */))
 			dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
 		execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, USE_DEFAULT_GRACE_PERIOD);
+	} else {
+		if (init_dequeue_wait_counts_mes(dqm))
+			dev_err(dev, "Setting optimized dequeue wait failed. Using default values\n");
 	}
 
 	/* setup per-queue reset detection buffer  */
@@ -2597,7 +2725,7 @@ static int reset_hung_queues_sdma(struct device_queue_manager *dqm)
 				continue;
 
 			/* Reset engine and check. */
-			if (amdgpu_sdma_reset_engine(dqm->dev->adev, i, false) ||
+			if (amdgpu_sdma_reset_engine(dqm->dev->adev, i) ||
 			    dqm->dev->kfd2kgd->hqd_sdma_get_doorbell(dqm->dev->adev, i, j) ||
 			    !set_sdma_queue_as_reset(dqm, doorbell_off)) {
 				r = -ENOTRECOVERABLE;
@@ -3215,6 +3343,20 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev,
 	amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem);
 }
 
+static void mes_notify_unmap_work_handler(struct work_struct *work)
+{
+	struct device_queue_manager *dqm =
+		container_of(work, struct device_queue_manager,
+			     notify_unmap_work.work);
+
+	amdgpu_mes_notify_unmap_queue((struct amdgpu_device *)dqm->dev->adev);
+
+	/* Re-arm if still oversubscribed */
+	if (READ_ONCE(dqm->active_cp_queue_count) > get_cp_queues_num(dqm))
+		queue_delayed_work(system_wq, &dqm->notify_unmap_work,
+				   usecs_to_jiffies(DQM_MES_UNMAP_NOTIFY_DELAY_US));
+}
+
 struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 {
 	struct device_queue_manager *dqm;
@@ -3269,6 +3411,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
 		dqm->ops.checkpoint_mqd = checkpoint_mqd;
 		dqm->ops.set_perfcount = set_perfcount;
+		dqm->ops.clean_process_queues_cpsch = clean_process_queues_cpsch;
 		break;
 	case KFD_SCHED_POLICY_NO_HWS:
 		/* initialize dqm for no cp scheduling */
@@ -3290,6 +3433,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
 		dqm->ops.checkpoint_mqd = checkpoint_mqd;
 		dqm->ops.set_perfcount = set_perfcount;
+		dqm->ops.clean_process_queues_cpsch = clean_process_queues_cpsch;
 		break;
 	default:
 		dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
@@ -3340,6 +3484,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 
 	if (!dqm->ops.initialize(dqm)) {
 		init_waitqueue_head(&dqm->destroy_wait);
+		INIT_DELAYED_WORK(&dqm->notify_unmap_work,
+				  mes_notify_unmap_work_handler);
 		return dqm;
 	}
 
@@ -3356,6 +3502,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 
 void device_queue_manager_uninit(struct device_queue_manager *dqm)
 {
+	cancel_delayed_work_sync(&dqm->notify_unmap_work);
 	dqm->ops.stop(dqm);
 	dqm->ops.uninitialize(dqm);
 	if (!dqm->dev->kfd->shared_resources.enable_mes)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index c9f9f7a8711175..f55f313d6cc287 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -201,6 +201,8 @@ struct device_queue_manager_ops {
 				  void *ctl_stack);
 	void	(*set_perfcount)(struct device_queue_manager *dqm,
 				  int enable);
+	int (*clean_process_queues_cpsch)(struct device_queue_manager *dqm,
+				    struct qcm_process_device *qpd);
 };
 
 struct device_queue_manager_asic_ops {
@@ -281,6 +283,7 @@ struct device_queue_manager {
 	uint32_t		wait_times;
 
 	wait_queue_head_t	destroy_wait;
+	struct delayed_work	notify_unmap_work;
 
 	/* for per-queue reset support */
 	struct dqm_detect_hang_info *detect_hang_info;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index 04c5e26f01ed91..da61d939b35c16 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -396,6 +396,12 @@ int kfd_init_apertures(struct kfd_process *process)
 			continue;
 		}
 
+		/* acquire pairs with the release in kgd2kfd_device_init() */
+		if (!smp_load_acquire(&dev->kfd->init_complete)) {
+			id++;
+			continue;
+		}
+
 		pdd = kfd_create_process_device_data(dev, process);
 		if (!pdd) {
 			dev_err(dev->adev->dev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
index 1688d8e595f2fb..9b48fe5985970b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c
@@ -359,6 +359,24 @@ static bool event_interrupt_isr_v9(struct kfd_node *dev,
 		!amdgpu_no_queue_eviction_on_vm_fault);
 }
 
+static void gfx942_clean_fault(uint16_t pasid)
+{
+	struct kfd_process *p;
+	struct kfd_process_device *pdd = NULL;
+	struct device_queue_manager *dqm;
+
+	p = kfd_lookup_process_by_pasid(pasid, &pdd);
+
+	if (!pdd)
+		return;
+
+	/* To Clean the GFX on the faulting device	 */
+	dqm = pdd->dev->dqm;
+	dqm->ops.clean_process_queues_cpsch(dqm, &pdd->qpd);
+
+	kfd_unref_process(p);
+}
+
 static void event_interrupt_wq_v9(struct kfd_node *dev,
 					const uint32_t *ih_ring_entry)
 {
@@ -547,6 +565,11 @@ static void event_interrupt_wq_v9(struct kfd_node *dev,
 			return;
 		}
 
+			/* GPR cleaner shader for MI2XX */
+			if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2))
+				gfx942_clean_fault(pasid);
+
+
 		info.vmid = vmid;
 		info.mc_id = client_id;
 		info.page_addr = ih_ring_entry[4] |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 253365a8257ef7..9535ebe5837aff 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -24,6 +24,9 @@
 #include <linux/dma-direction.h>
 #include <linux/dma-mapping.h>
 #include <linux/migrate.h>
+#include <linux/pagemap.h>
+#include "amdgpu.h"
+#include "amdgpu_sdma.h"
 #include "amdgpu_sync.h"
 #include "amdgpu_object.h"
 #include "amdgpu_vm.h"
@@ -384,6 +387,46 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange,
 	return r;
 }
 
+/* Check if a range of addresses is already resident in VRAM to avoid
+ * unnecessary migrations that would trigger MMU notifiers causing the range
+ * to be unmapped and faulted again unnecessarily.
+ *
+ * Called with prange->migrate_mutex held.
+ */
+static bool
+svm_range_window_resident(struct kfd_node *node, struct svm_range *prange,
+			  u64 start, u64 end)
+{
+	struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
+	unsigned long i, offset, npages;
+	dma_addr_t *addr;
+	uint32_t gpuid;
+	int32_t gpuidx;
+
+	if (kfd_process_gpuid_from_node(p, node, &gpuid, &gpuidx))
+		return false;
+	if (prange->actual_loc != gpuid)
+		return false;
+
+	addr = prange->dma_addr[gpuidx];
+	if (!addr)
+		return false;
+
+	start = max(start >> PAGE_SHIFT, prange->start);
+	end = min(end >> PAGE_SHIFT, prange->last + 1);
+	if (end <= start)
+		return true;
+
+	offset = start - prange->start;
+	npages = end - start;
+
+	for (i = offset; i < offset + npages; i++)
+		if (!(addr[i] & SVM_RANGE_VRAM_DOMAIN))
+			return false;
+
+	return true;
+}
+
 static long
 svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
 			struct vm_area_struct *vma, u64 start,
@@ -401,6 +444,12 @@ svm_migrate_vma_to_vram(struct kfd_node *node, struct svm_range *prange,
 	void *buf;
 	int r = -ENOMEM;
 
+	/* Avoid unnecessary migrations and MMU notifiers if all the data is
+	 * already resident in VRAM.
+	 */
+	if (svm_range_window_resident(node, prange, start, end))
+		return 0;
+
 	memset(&migrate, 0, sizeof(migrate));
 	migrate.vma = vma;
 	migrate.start = start;
@@ -729,7 +778,7 @@ svm_migrate_vma_to_ram(struct kfd_node *node, struct svm_range *prange,
 	migrate.start = start;
 	migrate.end = end;
 	migrate.pgmap_owner = SVM_ADEV_PGMAP_OWNER(adev);
-	if (adev->gmc.xgmi.connected_to_cpu)
+	if (adev->kfd.pgmap.type == MEMORY_DEVICE_COHERENT)
 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_COHERENT;
 	else
 		migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
@@ -1063,7 +1112,8 @@ int kgd2kfd_init_zone_device(struct amdgpu_device *adev)
 	 * should remove reserved size
 	 */
 	size = ALIGN(adev->gmc.real_vram_size, 2ULL << 20);
-	if (adev->gmc.xgmi.connected_to_cpu) {
+	if (adev->gmc.xgmi.connected_to_cpu &&
+	    amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(12, 1, 0)) {
 		pgmap->range.start = adev->gmc.aper_base;
 		pgmap->range.end = adev->gmc.aper_base + adev->gmc.aper_size - 1;
 		pgmap->type = MEMORY_DEVICE_COHERENT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index c709db0210ced8..d508ac5a18f85e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -31,6 +31,7 @@
 #include "gc/gc_12_1_0_sh_mask.h"
 #include "amdgpu_amdkfd.h"
 #include "kfd_device_queue_manager.h"
+#include "gmc_v12_1.h"
 
 static void update_mqd(struct mqd_manager *mm, void *mqd,
 		       struct queue_properties *q,
@@ -143,12 +144,31 @@ static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm,
 	u32 mqd_size = AMDGPU_MQD_SIZE_ALIGN(mm->mqd_size);
 	struct kfd_node *node = mm->dev;
 	struct kfd_mem_obj *mqd_mem_obj;
+	int retval;
 
 	if (q->type == KFD_QUEUE_TYPE_COMPUTE)
 		mqd_size *= NUM_XCC(node->xcc_mask);
 
-	if (kfd_gtt_sa_allocate(node, mqd_size, &mqd_mem_obj))
-		return NULL;
+	if (node->kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
+		mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
+		if (!mqd_mem_obj)
+			return NULL;
+		retval = amdgpu_amdkfd_alloc_kernel_mem(node->adev,
+			mqd_size,
+			AMDGPU_GEM_DOMAIN_GTT,
+			&(mqd_mem_obj->mem),
+			&(mqd_mem_obj->gpu_addr),
+			(void *)&(mqd_mem_obj->cpu_ptr), false);
+
+		if (retval) {
+			kfree(mqd_mem_obj);
+			return NULL;
+		}
+	} else {
+		retval = kfd_gtt_sa_allocate(node, mqd_size, &mqd_mem_obj);
+		if (retval)
+			return NULL;
+	}
 
 	return mqd_mem_obj;
 }
@@ -216,6 +236,12 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 	}
 
+	/*
+	 * coherent_aql_mtype (offset 509): program 0 when the driver maps local
+	 * or remote memory as MTYPE_NC, and 1 in all other cases.
+	 */
+	m->coherent_aql_mtype = gmc_v12_1_get_coherent_aql_mtype(mm->dev->adev);
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
@@ -248,6 +274,7 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 	m->cp_hqd_pq_control |=
 			ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
 	m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK;
+	m->cp_hqd_pq_control |= 3 << CP_HQD_PQ_CONTROL__SCOPE__SHIFT;
 	pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
 
 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
index 54868b81086e71..55d573fa3de380 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
@@ -306,7 +306,8 @@ static inline void pm_build_dequeue_wait_counts_packet_info(struct packet_manage
 		sch_value,
 		que_sleep,
 		reg_offset,
-		reg_data);
+		reg_data,
+		0);
 }
 
 /* pm_grace_period_0_supported - whether firmware tolerates a CWSR grace
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index d8631847f0eb93..e590fffab915bf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -45,7 +45,6 @@
 #include <drm/drm_device.h>
 #include <drm/drm_ioctl.h>
 #include <kgd_kfd_interface.h>
-#include <linux/swap.h>
 
 #include "amd_shared.h"
 #include "amdgpu.h"
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 0a7c1900da9593..226f52626e849d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -318,8 +318,10 @@ static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
 	 * XCCs in the partition.
 	 * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.
 	 */
+	amdgpu_gfx_off_ctrl(dev->adev, false);
 	dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,
 			&max_waves_per_cu, ffs(dev->xcc_mask) - 1);
+	amdgpu_gfx_off_ctrl(dev->adev, true);
 
 	for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {
 		if (cu_occupancy[i].wave_cnt != 0 &&
@@ -1408,23 +1410,16 @@ void kfd_process_notifier_release_internal(struct kfd_process *p)
 		kfd_unref_process(p);
 
 	/*
-	 * Dequeue and destroy user queues, it is not safe for GPU to access
-	 * system memory after mmu release notifier callback returns because
-	 * exit_mmap free process memory afterwards.
+	 * Disable debug BEFORE dequeuing/flushing the MES process context.
+	 *
+	 * kfd_dbg_trap_disable() -> kfd_dbg_trap_deactivate() issues
+	 * SET_SHADER_DEBUGGER (process_ctx_flush=0) packets that re-add the
+	 * process to the MES scheduler list. kfd_process_dequeue_from_all_devices()
+	 * sends the process_ctx_flush=1 that retires the context. If debug
+	 * teardown ran after the dequeue, those packets would put the process
+	 * back on the MES list and, once proc_ctx_bo is freed, MES would fault
+	 * on the freed process context. Retiring the context last avoids this.
 	 */
-	kfd_process_dequeue_from_all_devices(p);
-	pqm_uninit(&p->pqm);
-
-	for (i = 0; i < p->n_pdds; i++) {
-		struct kfd_process_device *pdd = p->pdds[i];
-
-		/* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
-		if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
-			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
-	}
-
-	/* Indicate to other users that MM is no longer valid */
-	p->mm = NULL;
 	kfd_dbg_trap_disable(p);
 
 	if (atomic_read(&p->debugged_process_count) > 0) {
@@ -1445,6 +1440,25 @@ void kfd_process_notifier_release_internal(struct kfd_process *p)
 		srcu_read_unlock(&kfd_processes_srcu, idx);
 	}
 
+	/*
+	 * Dequeue and destroy user queues, it is not safe for GPU to access
+	 * system memory after mmu release notifier callback returns because
+	 * exit_mmap free process memory afterwards.
+	 */
+	kfd_process_dequeue_from_all_devices(p);
+	pqm_uninit(&p->pqm);
+
+	for (i = 0; i < p->n_pdds; i++) {
+		struct kfd_process_device *pdd = p->pdds[i];
+
+		/* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
+		if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
+			amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
+	}
+
+	/* Indicate to other users that MM is no longer valid */
+	p->mm = NULL;
+
 	if (p->context_id == KFD_CONTEXT_ID_PRIMARY)
 		mmu_notifier_put(&p->mmu_notifier);
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 4fe40e9fcfc8b3..a15a826519ad67 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -339,12 +339,13 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 	unsigned int max_queues = 127; /* HWS limit */
 
 	/*
-	 * On GFX 9.4.3/9.5.0, increase the number of queues that
-	 * can be created to 255. No HWS limit on GFX 9.4.3/9.5.0.
+	 * On GFX 9.4.3/9.5.0/12.1.0, increase the number of queues that
+	 * can be created to 255. No HWS limit on GFX 9.4.3/9.5.0/12.1.0.
 	 */
 	if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) ||
 	    KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4) ||
-	    KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0))
+	    KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0) ||
+	    KFD_GC_VERSION(dev) == IP_VERSION(12, 1, 0))
 		max_queues = 255;
 
 	q = NULL;
@@ -663,7 +664,8 @@ int pqm_update_mqd(struct process_queue_manager *pqm,
 
 	/* ASICs that have WGPs must enforce pairwise enabled mask checks. */
 	if (minfo && minfo->cu_mask.ptr &&
-			KFD_GC_VERSION(pqn->q->device) >= IP_VERSION(10, 0, 0)) {
+			KFD_GC_VERSION(pqn->q->device) >= IP_VERSION(10, 0, 0) &&
+			KFD_GC_VERSION(pqn->q->device) < IP_VERSION(12, 1, 0)) {
 		int i;
 
 		for (i = 0; i < minfo->cu_mask.count; i += 2) {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
index 15eeaaebbbcd75..ae3e7c54014a86 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c
@@ -197,18 +197,18 @@ int kfd_queue_buffer_get(struct amdgpu_vm *vm, void __user *addr, struct amdgpu_
 			 u64 expected_size)
 {
 	struct amdgpu_bo_va_mapping *mapping;
-	u64 user_addr;
+	u64 user_pfn;
 	u64 size;
 
-	user_addr = (u64)addr >> AMDGPU_GPU_PAGE_SHIFT;
 	size = expected_size >> AMDGPU_GPU_PAGE_SHIFT;
 
-	mapping = amdgpu_vm_bo_lookup_mapping(vm, user_addr);
+	mapping = amdgpu_vm_bo_lookup_mapping(vm, (u64)(uintptr_t)addr);
 	if (!mapping)
 		goto out_err;
 
-	if (user_addr != mapping->start ||
-	    (size != 0 && user_addr + size - 1 != mapping->last)) {
+	user_pfn = (u64)(uintptr_t)addr >> AMDGPU_GPU_PAGE_SHIFT;
+	if (user_pfn != mapping->start ||
+	    (size != 0 && user_pfn + size - 1 != mapping->last)) {
 		pr_debug("expected size 0x%llx not equal to mapping addr 0x%llx size 0x%llx\n",
 			expected_size, mapping->start << AMDGPU_GPU_PAGE_SHIFT,
 			(mapping->last - mapping->start + 1) << AMDGPU_GPU_PAGE_SHIFT);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index fa4054d51f6066..c521ec07bf3c05 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1348,7 +1348,12 @@ svm_range_get_pte_flags(struct kfd_node *node, struct amdgpu_vm *vm,
 		mtype_local = amdgpu_mtype_local == 0 ? AMDGPU_VM_MTYPE_RW :
 				amdgpu_mtype_local == 1 ? AMDGPU_VM_MTYPE_NC :
 				is_aid_a1 ? AMDGPU_VM_MTYPE_RW : AMDGPU_VM_MTYPE_NC;
-		mtype_remote = is_aid_a1 ? AMDGPU_VM_MTYPE_NC : AMDGPU_VM_MTYPE_UC;
+		/* Remote memory defaults to MTYPE_UC on GFX 12.1 and can be
+		 * overridden through the amdgpu_mtype_remote module parameter
+		 * (0 = MTYPE_NC, 1 = MTYPE_UC).
+		 */
+		mtype_remote = amdgpu_mtype_remote == 0 ? AMDGPU_VM_MTYPE_NC :
+				AMDGPU_VM_MTYPE_UC;
 		snoop = true;
 
 		if (is_local) /* local HBM  */ {
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 1727d25646cc56..a319680c329b15 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -34,11 +34,20 @@ config DRM_AMD_DC_SI
 	bool "AMD DC support for Southern Islands ASICs"
 	depends on DRM_AMDGPU_SI
 	depends on DRM_AMD_DC
+	depends on DRM_AMD_DC_DCE
 	help
 	  Choose this option to enable new AMD DC support for SI asics
 	  by default. This includes Tahiti, Pitcairn, Cape Verde, Oland.
 	  Hainan is not supported by AMD DC and it has no physical DCE6.
 
+config DRM_AMD_DC_DCE
+	bool "AMD DC support for legacy DCE ASICs"
+	default y
+	depends on DRM_AMD_DC
+	help
+	  Enable support for the legacy DCE (DCE 8/10/11/12) display engines.
+	  Enabled by default so DC keeps support for these older ASICs.
+
 config DEBUG_KERNEL_DC
 	bool "Enable kgdb break in DC"
 	depends on DRM_AMD_DC
@@ -58,7 +67,8 @@ config DRM_AMD_SECURE_DISPLAY
 
 config DRM_AMD_DC_KUNIT_TEST
 	tristate "KUnit tests for the AMD DC display driver" if !KUNIT_ALL_TESTS
-	depends on DRM_AMD_DC && KUNIT && DEBUG_FS && DRM_KUNIT_TEST_HELPERS
+	depends on DRM_AMD_DC && KUNIT && DEBUG_FS
+	select DRM_KUNIT_TEST_HELPERS
 	default KUNIT_ALL_TESTS
 	help
 	  This option enables KUnit tests for the AMD Display Core driver.
diff --git a/drivers/gpu/drm/amd/display/Makefile b/drivers/gpu/drm/amd/display/Makefile
index 0084a8d5525457..96242bb2f97c94 100644
--- a/drivers/gpu/drm/amd/display/Makefile
+++ b/drivers/gpu/drm/amd/display/Makefile
@@ -42,6 +42,7 @@ subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dwb
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/hpo
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/mmhubbub
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/mpc
+subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/rmcm
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/opp
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/pg
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/soc_and_ip_translator
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 36d2f86f000a97..2b0686bcc7fd23 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -36,7 +36,6 @@
 #include "dc/inc/hw/dmcu.h"
 #include "dc/inc/hw/abm.h"
 #include "dc/dc_dmub_srv.h"
-#include "dc/dc_edid_parser.h"
 #include "dc/dc_stat.h"
 #include "dc/dc_state.h"
 #include "amdgpu_dm_trace.h"
@@ -141,12 +140,14 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev);
 /* removes and deallocates the drm structures, created by the above function */
 static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm);
 
-static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state);
+STATIC_IFN_KUNIT int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state);
 static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state);
 STATIC_IFN_KUNIT void dm_enable_per_frame_crtc_master_sync(struct dc_state *context);
 
+#if !IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
 static int amdgpu_dm_atomic_check(struct drm_device *dev,
 				  struct drm_atomic_commit *state);
+#endif
 
 static inline void amdgpu_dm_exit_ips_for_hw_access(struct dc *dc)
 {
@@ -224,13 +225,6 @@ STATIC_IFN_KUNIT int dm_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc
 }
 EXPORT_IF_KUNIT(dm_crtc_get_scanoutpos);
 
-STATIC_IFN_KUNIT bool dm_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* XXX todo */
-	return true;
-}
-EXPORT_IF_KUNIT(dm_is_idle);
-
 STATIC_IFN_KUNIT int dm_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* XXX todo */
@@ -260,6 +254,16 @@ STATIC_IFN_KUNIT int dm_plane_layer_index_cmp(const void *a, const void *b)
 }
 EXPORT_IF_KUNIT(dm_plane_layer_index_cmp);
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+static const struct amdgpu_dm_kunit_ops *amdgpu_dm_ops;
+
+#define dm_post_update_surfaces_to_stream	amdgpu_dm_ops->post_update_surfaces_to_stream
+#define dm_update_planes_and_stream		amdgpu_dm_ops->update_planes_and_stream
+#else
+#define dm_post_update_surfaces_to_stream	dc_post_update_surfaces_to_stream
+#define dm_update_planes_and_stream		dc_update_planes_and_stream
+#endif
+
 /**
  * update_planes_and_stream_adapter() - Send planes to be updated in DC
  *
@@ -270,19 +274,18 @@ EXPORT_IF_KUNIT(dm_plane_layer_index_cmp);
  * before passing control to DC.
  *
  * @dc: Display Core control structure
- * @update_type: specify whether it is FULL/MEDIUM/FAST update
  * @planes_count: planes count to update
  * @stream: stream state
  * @stream_update: stream update
  * @array_of_surface_update: dc surface update pointer
  *
  */
-static inline bool update_planes_and_stream_adapter(struct dc *dc,
-						    int update_type,
-						    int planes_count,
-						    struct dc_stream_state *stream,
-						    struct dc_stream_update *stream_update,
-						    struct dc_surface_update *array_of_surface_update)
+STATIC_IFN_KUNIT INLINE_IFN_KUNIT
+bool update_planes_and_stream_adapter(struct dc *dc,
+				      int planes_count,
+				      struct dc_stream_state *stream,
+				      struct dc_stream_update *stream_update,
+				      struct dc_surface_update *array_of_surface_update)
 {
 	sort(array_of_surface_update, planes_count,
 	     sizeof(*array_of_surface_update), dm_plane_layer_index_cmp, NULL);
@@ -290,14 +293,15 @@ static inline bool update_planes_and_stream_adapter(struct dc *dc,
 	/*
 	 * Previous frame finished and HW is ready for optimization.
 	 */
-	dc_post_update_surfaces_to_stream(dc);
+	dm_post_update_surfaces_to_stream(dc);
 
-	return dc_update_planes_and_stream(dc,
+	return dm_update_planes_and_stream(dc,
 					   array_of_surface_update,
 					   planes_count,
 					   stream,
 					   stream_update);
 }
+EXPORT_IF_KUNIT(update_planes_and_stream_adapter);
 
 STATIC_IFN_KUNIT int dm_set_clockgating_state(struct amdgpu_ip_block *ip_block,
 					      enum amd_clockgating_state state)
@@ -314,10 +318,34 @@ STATIC_IFN_KUNIT int dm_set_powergating_state(struct amdgpu_ip_block *ip_block,
 EXPORT_IF_KUNIT(dm_set_powergating_state);
 
 /* Prototypes of private functions */
-static int dm_early_init(struct amdgpu_ip_block *ip_block);
+STATIC_IFN_KUNIT int dm_early_init(struct amdgpu_ip_block *ip_block);
+
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+static const struct amdgpu_dm_kunit_ops amdgpu_dm_default_ops = {
+	.gmc_pd_addr = amdgpu_gmc_pd_addr,
+	.post_update_surfaces_to_stream = dc_post_update_surfaces_to_stream,
+	.update_planes_and_stream = dc_update_planes_and_stream,
+	.atomic_helper_suspend = drm_atomic_helper_suspend,
+};
+
+static const struct amdgpu_dm_kunit_ops *amdgpu_dm_ops = &amdgpu_dm_default_ops;
+
+void amdgpu_dm_kunit_set_ops(const struct amdgpu_dm_kunit_ops *ops)
+{
+	amdgpu_dm_ops = ops ? ops : &amdgpu_dm_default_ops;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_kunit_set_ops);
+
+#define dm_gmc_pd_addr			amdgpu_dm_ops->gmc_pd_addr
+#define dm_atomic_helper_suspend	amdgpu_dm_ops->atomic_helper_suspend
+#else
+#define dm_gmc_pd_addr			amdgpu_gmc_pd_addr
+#define dm_atomic_helper_suspend	drm_atomic_helper_suspend
+#endif
 
 /* Allocate memory for FBC compressed data  */
-static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_addr_space_config *pa_config)
+STATIC_IFN_KUNIT void mmhub_read_system_context(struct amdgpu_device *adev,
+						struct dc_phy_addr_space_config *pa_config)
 {
 	u64 pt_base;
 	u32 logical_addr_low;
@@ -362,7 +390,7 @@ static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_
 			logical_addr_high = max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18;
 	}
 
-	pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+	pt_base = dm_gmc_pd_addr(adev->gart.bo);
 
 	page_table_start.high_part = upper_32_bits(adev->gmc.gart_start >>
 						   AMDGPU_GPU_PAGE_SHIFT);
@@ -393,8 +421,9 @@ static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_
 	pa_config->is_hvm_enabled = adev->mode_info.gpu_vm_support;
 
 }
+EXPORT_IF_KUNIT(mmhub_read_system_context);
 
-static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm)
+STATIC_IFN_KUNIT int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm)
 {
 	struct mod_power_init_params init_data[MAX_NUM_EDP];
 
@@ -457,6 +486,7 @@ static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_init_power_module);
 
 static int amdgpu_dm_init(struct amdgpu_device *adev)
 {
@@ -678,6 +708,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 	/* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */
 	adev->dm.dc->debug.ignore_cable_id = true;
 
+	/* temporary enable immediate restore for FAMS2 */
+	adev->dm.dc->debug.fams2_imm_restore_drr = true;
+
 	if (adev->dm.dc->caps.dp_hdmi21_pcon_support)
 		drm_info(adev_to_drm(adev), "DP-HDMI FRL PCON supported\n");
 
@@ -837,7 +870,7 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 	return -EINVAL;
 }
 
-static int amdgpu_dm_early_fini(struct amdgpu_ip_block *ip_block)
+STATIC_IFN_KUNIT int amdgpu_dm_early_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
@@ -845,6 +878,7 @@ static int amdgpu_dm_early_fini(struct amdgpu_ip_block *ip_block)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_early_fini);
 
 static void amdgpu_dm_fini(struct amdgpu_device *adev)
 {
@@ -952,7 +986,7 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
 	mutex_destroy(&adev->dm.dpia_aux_lock);
 }
 
-static int load_dmcu_fw(struct amdgpu_device *adev)
+STATIC_IFN_KUNIT int load_dmcu_fw(struct amdgpu_device *adev)
 {
 	const char *fw_name_dmcu = NULL;
 	int r;
@@ -1062,8 +1096,9 @@ static int load_dmcu_fw(struct amdgpu_device *adev)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(load_dmcu_fw);
 
-static int dm_sw_init(struct amdgpu_ip_block *ip_block)
+STATIC_IFN_KUNIT int dm_sw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	int r;
@@ -1084,8 +1119,9 @@ static int dm_sw_init(struct amdgpu_ip_block *ip_block)
 
 	return load_dmcu_fw(adev);
 }
+EXPORT_IF_KUNIT(dm_sw_init);
 
-static int dm_sw_fini(struct amdgpu_ip_block *ip_block)
+STATIC_IFN_KUNIT int dm_sw_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	struct dal_allocation *da;
@@ -1115,6 +1151,7 @@ static int dm_sw_fini(struct amdgpu_ip_block *ip_block)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(dm_sw_fini);
 
 
 static void amdgpu_dm_boot_time_crc_init(struct amdgpu_device *adev)
@@ -1165,7 +1202,7 @@ static void amdgpu_dm_boot_time_crc_init(struct amdgpu_device *adev)
 	}
 }
 
-static int dm_late_init(struct amdgpu_ip_block *ip_block)
+STATIC_IFN_KUNIT int dm_late_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 
@@ -1215,8 +1252,9 @@ static int dm_late_init(struct amdgpu_ip_block *ip_block)
 
 	return amdgpu_dm_detect_mst_link_for_all_connectors(adev_to_drm(adev));
 }
+EXPORT_IF_KUNIT(dm_late_init);
 
-static void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr)
+STATIC_IFN_KUNIT void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr)
 {
 	u8 buf[UUID_SIZE];
 	guid_t guid;
@@ -1266,8 +1304,9 @@ static void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr)
 out_fail:
 	mutex_unlock(&mgr->lock);
 }
+EXPORT_IF_KUNIT(resume_mst_branch_status);
 
-static void s3_handle_mst(struct drm_device *dev, bool suspend)
+STATIC_IFN_KUNIT void s3_handle_mst(struct drm_device *dev, bool suspend)
 {
 	struct amdgpu_dm_connector *aconnector;
 	struct drm_connector *connector;
@@ -1308,8 +1347,9 @@ static void s3_handle_mst(struct drm_device *dev, bool suspend)
 	}
 	drm_connector_list_iter_end(&iter);
 }
+EXPORT_IF_KUNIT(s3_handle_mst);
 
-static int dm_oem_i2c_hw_init(struct amdgpu_device *adev)
+STATIC_IFN_KUNIT int dm_oem_i2c_hw_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_display_manager *dm = &adev->dm;
 	struct amdgpu_i2c_adapter *oem_i2c;
@@ -1335,6 +1375,7 @@ static int dm_oem_i2c_hw_init(struct amdgpu_device *adev)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(dm_oem_i2c_hw_init);
 
 /**
  * dm_hw_init() - Initialize DC device
@@ -1405,8 +1446,8 @@ static int dm_hw_fini(struct amdgpu_ip_block *ip_block)
 }
 
 
-static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev,
-				 struct dc_state *state, bool enable)
+STATIC_IFN_KUNIT void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev,
+						    struct dc_state *state, bool enable)
 {
 	enum dc_irq_source irq_source;
 	struct amdgpu_crtc *acrtc;
@@ -1457,10 +1498,11 @@ static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev,
 	}
 
 }
+EXPORT_IF_KUNIT(dm_gpureset_toggle_interrupts);
 
-DEFINE_FREE(state_release, struct dc_state *, if (_T) dc_state_release(_T))
+DEFINE_FREE(state_release, struct dc_state *, dc_state_release(_T))
 
-static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
+STATIC_IFN_KUNIT enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
 {
 	struct dc_state *context __free(state_release) = NULL;
 	int i;
@@ -1498,12 +1540,13 @@ static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc)
 
 	return dc_commit_streams(dc, &params);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_commit_zero_streams);
 
-static int dm_cache_state(struct amdgpu_device *adev)
+STATIC_IFN_KUNIT int dm_cache_state(struct amdgpu_device *adev)
 {
 	int r;
 
-	adev->dm.cached_state = drm_atomic_helper_suspend(adev_to_drm(adev));
+	adev->dm.cached_state = dm_atomic_helper_suspend(adev_to_drm(adev));
 	if (IS_ERR(adev->dm.cached_state)) {
 		r = PTR_ERR(adev->dm.cached_state);
 		adev->dm.cached_state = NULL;
@@ -1511,8 +1554,9 @@ static int dm_cache_state(struct amdgpu_device *adev)
 
 	return adev->dm.cached_state ? 0 : r;
 }
+EXPORT_IF_KUNIT(dm_cache_state);
 
-static void dm_destroy_cached_state(struct amdgpu_device *adev)
+STATIC_IFN_KUNIT void dm_destroy_cached_state(struct amdgpu_device *adev)
 {
 	struct amdgpu_display_manager *dm = &adev->dm;
 	struct drm_device *ddev = adev_to_drm(adev);
@@ -1562,6 +1606,7 @@ static void dm_destroy_cached_state(struct amdgpu_device *adev)
 
 	dm->cached_state = NULL;
 }
+EXPORT_IF_KUNIT(dm_destroy_cached_state);
 
 static int dm_suspend(struct amdgpu_ip_block *ip_block)
 {
@@ -1717,9 +1762,10 @@ void amdgpu_dm_emulated_link_detect(struct dc_link *link)
 		drm_err(dev, "Failed to read EDID\n");
 
 }
+EXPORT_IF_KUNIT(amdgpu_dm_emulated_link_detect);
 
-static void dm_gpureset_commit_state(struct dc_state *dc_state,
-				     struct amdgpu_display_manager *dm)
+STATIC_IFN_KUNIT void dm_gpureset_commit_state(struct dc_state *dc_state,
+					       struct amdgpu_display_manager *dm)
 {
 	struct {
 		struct dc_surface_update surface_updates[MAX_SURFACES];
@@ -1748,13 +1794,13 @@ static void dm_gpureset_commit_state(struct dc_state *dc_state,
 		}
 
 		update_planes_and_stream_adapter(dm->dc,
-					 UPDATE_TYPE_FULL,
 					 dc_state->stream_status[k].plane_count,
 					 dc_state->streams[k],
 					 &bundle->stream_update,
 					 bundle->surface_updates);
 	}
 }
+EXPORT_IF_KUNIT(dm_gpureset_commit_state);
 
 void amdgpu_dm_apply_delay_after_dpcd_poweroff(struct amdgpu_device *adev,
 											   struct dc_sink *sink)
@@ -1783,7 +1829,7 @@ EXPORT_IF_KUNIT(amdgpu_dm_apply_delay_after_dpcd_poweroff);
  * (MST) sinks. Should be called after connector detection is complete to see
  * the final state of all links.
  */
-static void amdgpu_dm_dump_links_and_sinks(struct amdgpu_device *adev)
+STATIC_IFN_KUNIT void amdgpu_dm_dump_links_and_sinks(struct amdgpu_device *adev)
 {
 	struct dc *dc = adev->dm.dc;
 	struct drm_device *dev = adev_to_drm(adev);
@@ -1835,6 +1881,7 @@ static void amdgpu_dm_dump_links_and_sinks(struct amdgpu_device *adev)
 		}
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_dump_links_and_sinks);
 
 static int dm_resume(struct amdgpu_ip_block *ip_block)
 {
@@ -2089,7 +2136,6 @@ static const struct amd_ip_funcs amdgpu_dm_funcs = {
 	.hw_fini = dm_hw_fini,
 	.suspend = dm_suspend,
 	.resume = dm_resume,
-	.is_idle = dm_is_idle,
 	.wait_for_idle = dm_wait_for_idle,
 	.soft_reset = dm_soft_reset,
 	.set_clockgating_state = dm_set_clockgating_state,
@@ -2148,6 +2194,7 @@ int dm_atomic_get_state(struct drm_atomic_commit *state,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(dm_atomic_get_state);
 
 STATIC_IFN_KUNIT struct dm_atomic_state *
 dm_atomic_get_new_state(struct drm_atomic_commit *state)
@@ -2168,7 +2215,7 @@ dm_atomic_get_new_state(struct drm_atomic_commit *state)
 }
 EXPORT_IF_KUNIT(dm_atomic_get_new_state);
 
-static struct drm_private_state *
+STATIC_IFN_KUNIT struct drm_private_state *
 dm_atomic_duplicate_state(struct drm_private_obj *obj)
 {
 	struct dm_atomic_state *old_state, *new_state;
@@ -2191,6 +2238,7 @@ dm_atomic_duplicate_state(struct drm_private_obj *obj)
 
 	return &new_state->base;
 }
+EXPORT_IF_KUNIT(dm_atomic_duplicate_state);
 
 STATIC_IFN_KUNIT void dm_atomic_destroy_state(struct drm_private_obj *obj,
 					      struct drm_private_state *state)
@@ -2233,7 +2281,7 @@ static struct drm_private_state_funcs dm_atomic_state_funcs = {
 	.atomic_destroy_state = dm_atomic_destroy_state,
 };
 
-static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
+STATIC_IFN_KUNIT int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
 {
 	int r;
 
@@ -2273,11 +2321,12 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_mode_config_init);
 
-static int initialize_plane(struct amdgpu_display_manager *dm,
-			    struct amdgpu_mode_info *mode_info, int plane_id,
-			    enum drm_plane_type plane_type,
-			    const struct dc_plane_cap *plane_cap)
+STATIC_IFN_KUNIT int initialize_plane(struct amdgpu_display_manager *dm,
+				      struct amdgpu_mode_info *mode_info, int plane_id,
+				      enum drm_plane_type plane_type,
+				      const struct dc_plane_cap *plane_cap)
 {
 	struct drm_plane *plane;
 	unsigned long possible_crtcs;
@@ -2313,6 +2362,7 @@ static int initialize_plane(struct amdgpu_display_manager *dm,
 
 	return ret;
 }
+EXPORT_IF_KUNIT(initialize_plane);
 
 
 /*
@@ -2729,7 +2779,7 @@ DEVICE_ATTR_WO(s3_debug);
 
 #endif
 
-static int dm_early_init(struct amdgpu_ip_block *ip_block)
+STATIC_IFN_KUNIT int dm_early_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	struct amdgpu_mode_info *mode_info = &adev->mode_info;
@@ -2879,6 +2929,7 @@ static int dm_early_init(struct amdgpu_ip_block *ip_block)
 
 	return dm_init_microcode(adev);
 }
+EXPORT_IF_KUNIT(dm_early_init);
 
 STATIC_IFN_KUNIT bool modereset_required(struct drm_crtc_state *crtc_state)
 {
@@ -2887,7 +2938,8 @@ STATIC_IFN_KUNIT bool modereset_required(struct drm_crtc_state *crtc_state)
 EXPORT_IF_KUNIT(modereset_required);
 
 STATIC_IFN_KUNIT int
-fill_plane_color_attributes(const struct drm_plane_state *plane_state,
+fill_plane_color_attributes(struct drm_atomic_commit *state,
+			    const struct drm_plane_state *plane_state,
 			    const enum surface_pixel_format format,
 			    enum dc_color_space *color_space)
 {
@@ -2896,7 +2948,7 @@ fill_plane_color_attributes(const struct drm_plane_state *plane_state,
 	*color_space = COLOR_SPACE_SRGB;
 
 	/* Ignore properties when DRM_CLIENT_CAP_PLANE_COLOR_PIPELINE is set */
-	if (plane_state->state && plane_state->state->plane_color_pipeline)
+	if (state && state->plane_color_pipeline)
 		return 0;
 
 	/* DRM color properties only affect non-RGB formats. */
@@ -2935,8 +2987,9 @@ fill_plane_color_attributes(const struct drm_plane_state *plane_state,
 }
 EXPORT_IF_KUNIT(fill_plane_color_attributes);
 
-static int
+STATIC_IFN_KUNIT int
 fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
+			    struct drm_atomic_commit *state,
 			    const struct drm_plane_state *plane_state,
 			    struct dc_plane_info *plane_info,
 			    struct dc_plane_address *address,
@@ -3029,7 +3082,7 @@ fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
 
 	plane_info->layer_index = plane_state->normalized_zpos;
 
-	ret = fill_plane_color_attributes(plane_state, plane_info->format,
+	ret = fill_plane_color_attributes(state, plane_state, plane_info->format,
 					  &plane_info->color_space);
 	if (ret)
 		return ret;
@@ -3049,11 +3102,12 @@ fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(fill_dc_plane_info_and_addr);
 
-static int fill_dc_plane_attributes(struct amdgpu_device *adev,
-				    struct dc_plane_state *dc_plane_state,
-				    struct drm_plane_state *plane_state,
-				    struct drm_crtc_state *crtc_state)
+STATIC_IFN_KUNIT int fill_dc_plane_attributes(struct amdgpu_device *adev,
+					      struct dc_plane_state *dc_plane_state,
+					      struct drm_plane_state *plane_state,
+					      struct drm_crtc_state *crtc_state)
 {
 	struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
 	struct amdgpu_framebuffer *afb = (struct amdgpu_framebuffer *)plane_state->fb;
@@ -3070,7 +3124,7 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev,
 	dc_plane_state->clip_rect = scaling_info.clip_rect;
 	dc_plane_state->scaling_quality = scaling_info.scaling_quality;
 
-	ret = fill_dc_plane_info_and_addr(adev, plane_state,
+	ret = fill_dc_plane_info_and_addr(adev, plane_state->state, plane_state,
 					  &plane_info,
 					  &dc_plane_state->address,
 					  afb->tmz_surface);
@@ -3106,6 +3160,7 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(fill_dc_plane_attributes);
 
 static inline void fill_dc_dirty_rect(struct drm_plane *plane,
 				      struct rect *dirty_rect, int32_t x,
@@ -3157,13 +3212,13 @@ static inline void fill_dc_dirty_rect(struct drm_plane *plane,
  * implicitly provide damage clips without any client support via the plane
  * bounds.
  */
-static void fill_dc_dirty_rects(struct drm_plane *plane,
-				struct drm_plane_state *old_plane_state,
-				struct drm_plane_state *new_plane_state,
-				struct drm_crtc_state *crtc_state,
-				struct dc_flip_addrs *flip_addrs,
-				bool is_psr_su,
-				bool *dirty_regions_changed)
+STATIC_IFN_KUNIT void fill_dc_dirty_rects(struct drm_plane *plane,
+					  struct drm_plane_state *old_plane_state,
+					  struct drm_plane_state *new_plane_state,
+					  struct drm_crtc_state *crtc_state,
+					  struct dc_flip_addrs *flip_addrs,
+					  bool is_psr_su,
+					  bool *dirty_regions_changed)
 {
 	struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state);
 	struct rect *dirty_rects = flip_addrs->dirty_rects;
@@ -3269,10 +3324,11 @@ static void fill_dc_dirty_rects(struct drm_plane *plane,
 			   dm_crtc_state->base.mode.crtc_vdisplay,
 			   &flip_addrs->dirty_rect_count, true);
 }
+EXPORT_IF_KUNIT(fill_dc_dirty_rects);
 
-static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_commit *state,
-					    struct dc_state *dc_state,
-					    struct dsc_mst_fairness_vars *vars)
+STATIC_IFN_KUNIT int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_commit *state,
+						      struct dc_state *dc_state,
+						      struct dsc_mst_fairness_vars *vars)
 {
 	struct dc_stream_state *stream = NULL;
 	struct drm_connector *connector;
@@ -3346,10 +3402,11 @@ static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_commit *state,
 	}
 	return 0;
 }
+EXPORT_IF_KUNIT(dm_update_mst_vcpi_slots_for_dsc);
 
-static void manage_dm_interrupts(struct amdgpu_device *adev,
-				 struct amdgpu_crtc *acrtc,
-				 struct dm_crtc_state *acrtc_state)
+STATIC_IFN_KUNIT void manage_dm_interrupts(struct amdgpu_device *adev,
+					   struct amdgpu_crtc *acrtc,
+					   struct dm_crtc_state *acrtc_state)
 {	/*
 	 * We cannot be sure that the frontend index maps to the same
 	 * backend index - some even map to more than one.
@@ -3428,9 +3485,10 @@ static void manage_dm_interrupts(struct amdgpu_device *adev,
 		drm_crtc_vblank_off(&acrtc->base);
 	}
 }
+EXPORT_IF_KUNIT(manage_dm_interrupts);
 
-static void dm_update_pflip_irq_state(struct amdgpu_device *adev,
-				      struct amdgpu_crtc *acrtc)
+STATIC_IFN_KUNIT void dm_update_pflip_irq_state(struct amdgpu_device *adev,
+						struct amdgpu_crtc *acrtc)
 {
 	int irq_type =
 		amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id);
@@ -3445,6 +3503,7 @@ static void dm_update_pflip_irq_state(struct amdgpu_device *adev,
 	 */
 	amdgpu_irq_update(adev, &adev->pageflip_irq, irq_type);
 }
+EXPORT_IF_KUNIT(dm_update_pflip_irq_state);
 
 STATIC_IFN_KUNIT bool
 is_scaling_state_different(const struct dm_connector_state *dm_state,
@@ -3597,7 +3656,7 @@ static void remove_stream(struct amdgpu_device *adev,
 	acrtc->enabled = false;
 }
 
-static void amdgpu_dm_commit_cursors(struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT void amdgpu_dm_commit_cursors(struct drm_atomic_commit *state)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *old_plane_state;
@@ -3611,6 +3670,7 @@ static void amdgpu_dm_commit_cursors(struct drm_atomic_commit *state)
 		if (plane->type == DRM_PLANE_TYPE_CURSOR)
 			amdgpu_dm_plane_handle_cursor_update(plane, old_plane_state);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_commit_cursors);
 
 static inline uint32_t get_mem_type(struct drm_framebuffer *fb)
 {
@@ -3619,9 +3679,9 @@ static inline uint32_t get_mem_type(struct drm_framebuffer *fb)
 	return abo->tbo.resource ? abo->tbo.resource->mem_type : 0;
 }
 
-static void amdgpu_dm_update_cursor(struct drm_plane *plane,
-				    struct drm_plane_state *old_plane_state,
-				    struct dc_stream_update *update)
+STATIC_IFN_KUNIT void amdgpu_dm_update_cursor(struct drm_plane *plane,
+					      struct drm_plane_state *old_plane_state,
+					      struct dc_stream_update *update)
 {
 	struct amdgpu_device *adev = drm_to_adev(plane->dev);
 	struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(plane->state->fb);
@@ -3690,11 +3750,12 @@ static void amdgpu_dm_update_cursor(struct drm_plane *plane,
 		update->cursor_position = &crtc_state->stream->cursor_position;
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_update_cursor);
 
-static void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm,
-					  struct amdgpu_crtc *acrtc_attach,
-					  const struct dm_crtc_state *acrtc_state,
-					  const u64 current_ts)
+STATIC_IFN_KUNIT void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm,
+						    struct amdgpu_crtc *acrtc_attach,
+						    const struct dm_crtc_state *acrtc_state,
+						    const u64 current_ts)
 {
 	struct psr_settings *psr = &acrtc_state->stream->link->psr_settings;
 	struct replay_settings *pr = &acrtc_state->stream->link->replay_settings;
@@ -3730,11 +3791,12 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm,
 		acrtc_attach->dm_irq_params.allow_sr_entry = false;
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_enable_self_refresh);
 
-static void dm_arm_vblank_event(struct amdgpu_crtc *acrtc,
-				struct dm_crtc_state *acrtc_state,
-				bool pflip_update,
-				bool cursor_update)
+STATIC_IFN_KUNIT void dm_arm_vblank_event(struct amdgpu_crtc *acrtc,
+					  struct dm_crtc_state *acrtc_state,
+					  bool pflip_update,
+					  bool cursor_update)
 {
 	assert_spin_locked(&acrtc->base.dev->event_lock);
 
@@ -3757,6 +3819,7 @@ static void dm_arm_vblank_event(struct amdgpu_crtc *acrtc,
 		acrtc->base.state->event = NULL;
 	}
 }
+EXPORT_IF_KUNIT(dm_arm_vblank_event);
 
 /**
  * dm_arm_vblank_event_pre_programming - Prepare for programming
@@ -3769,10 +3832,10 @@ static void dm_arm_vblank_event(struct amdgpu_crtc *acrtc,
  * be programmed. Do this before programming so the HW is not in any
  * idle-optimized state (such as PSR).
  */
-static void dm_arm_vblank_event_pre_programming(struct amdgpu_crtc *acrtc,
-						struct dm_crtc_state *acrtc_state,
-						bool pflip_update,
-						bool cursor_update)
+STATIC_IFN_KUNIT void dm_arm_vblank_event_pre_programming(struct amdgpu_crtc *acrtc,
+							  struct dm_crtc_state *acrtc_state,
+							  bool pflip_update,
+							  bool cursor_update)
 {
 	assert_spin_locked(&acrtc->base.dev->event_lock);
 
@@ -3782,6 +3845,7 @@ static void dm_arm_vblank_event_pre_programming(struct amdgpu_crtc *acrtc,
 	if (pflip_update || cursor_update)
 		drm_crtc_vblank_get(&acrtc->base);
 }
+EXPORT_IF_KUNIT(dm_arm_vblank_event_pre_programming);
 
 static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 				    struct drm_device *dev,
@@ -3809,6 +3873,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 	bool flip_latched_during_prog = false;
 	bool dirty_rects_changed = false;
 	bool updated_planes_and_streams = false;
+	bool stream_update_needed = false;
 	struct {
 		struct dc_surface_update surface_updates[MAX_SURFACES];
 		struct dc_plane_info plane_infos[MAX_SURFACES];
@@ -3842,6 +3907,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 
 		bundle->stream_update.cursor_position =
 				&acrtc_state->stream->cursor_position;
+		stream_update_needed = true;
 	}
 
 	if (acrtc_state->active_planes == 0 &&
@@ -3857,6 +3923,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 		bool plane_needs_flip;
 		struct dc_plane_state *dc_plane;
 		struct dm_plane_state *dm_new_plane_state = to_dm_plane_state(new_plane_state);
+		struct dm_plane_state *dm_old_plane_state = to_dm_plane_state(old_plane_state);
 
 		/* Cursor plane is handled after stream updates */
 		if (plane->type == DRM_PLANE_TYPE_CURSOR &&
@@ -3864,8 +3931,10 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 			if ((fb && crtc == pcrtc) ||
 			    (old_plane_state->fb && old_plane_state->crtc == pcrtc)) {
 				cursor_update = true;
-				if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0)
+				if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0) {
 					amdgpu_dm_update_cursor(plane, old_plane_state, &bundle->stream_update);
+					stream_update_needed = true;
+				}
 			}
 
 			continue;
@@ -3891,11 +3960,22 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 			bundle->surface_updates[planes_count].cm = &dc_plane->cm;
 		}
 
-		amdgpu_dm_plane_fill_dc_scaling_info(dm->adev, new_plane_state,
-				     &bundle->scaling_infos[planes_count]);
+		if (amdgpu_dm_plane_fill_dc_scaling_info(dm->adev, new_plane_state,
+							 &bundle->scaling_infos[planes_count])) {
+			planes_count += 1;
+			continue;
+		}
 
-		bundle->surface_updates[planes_count].scaling_info =
-			&bundle->scaling_infos[planes_count];
+		/* Cache the newly computed scaling_info in the plane state */
+		*dm_new_plane_state->scaling_info =
+			bundle->scaling_infos[planes_count];
+
+		/* Only send a scaling_info update if it changed vs the old state */
+		if (memcmp(dm_old_plane_state->scaling_info,
+			   dm_new_plane_state->scaling_info,
+			   sizeof(struct dc_scaling_info)))
+			bundle->surface_updates[planes_count].scaling_info =
+				&bundle->scaling_infos[planes_count];
 
 		plane_needs_flip = old_plane_state->fb && new_plane_state->fb;
 
@@ -3907,17 +3987,27 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 		}
 
 		fill_dc_plane_info_and_addr(
-			dm->adev, new_plane_state,
+			dm->adev, state, new_plane_state,
 			&bundle->plane_infos[planes_count],
 			&bundle->flip_addrs[planes_count].address,
 			afb->tmz_surface);
 
+		bundle->plane_infos[planes_count].color_space = dc_plane->color_space;
+
 		drm_dbg_state(state->dev, "plane: id=%d dcc_en=%d\n",
 				 new_plane_state->plane->index,
 				 bundle->plane_infos[planes_count].dcc.enable);
 
-		bundle->surface_updates[planes_count].plane_info =
-			&bundle->plane_infos[planes_count];
+		/* Cache the newly computed plane_info in the plane state */
+		*dm_new_plane_state->plane_info =
+			bundle->plane_infos[planes_count];
+
+		/* Only send a plane_info update if it changed vs the old state */
+		if (memcmp(dm_old_plane_state->plane_info,
+			   dm_new_plane_state->plane_info,
+			   sizeof(struct dc_plane_info)))
+			bundle->surface_updates[planes_count].plane_info =
+				&bundle->plane_infos[planes_count];
 
 		if (acrtc_state->stream->link->psr_settings.psr_feature_enabled ||
 		    acrtc_state->stream->link->replay_settings.replay_feature_enabled) {
@@ -3972,7 +4062,16 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 
 		timestamp_ns = ktime_get_ns();
 		bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000);
-		bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count];
+		/* Cache the newly computed flip_addr in the plane state */
+		*dm_new_plane_state->flip_addr =
+			bundle->flip_addrs[planes_count];
+
+		/* Only send a flip_addr update if it changed vs the old state */
+		if (memcmp(dm_old_plane_state->flip_addr,
+			   dm_new_plane_state->flip_addr,
+			   sizeof(struct dc_flip_addrs)))
+			bundle->surface_updates[planes_count].flip_addr =
+				&bundle->flip_addrs[planes_count];
 		bundle->surface_updates[planes_count].surface = dc_plane;
 
 		if (!bundle->surface_updates[planes_count].surface) {
@@ -4039,9 +4138,13 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 		}
 
 		if (acrtc_state->stream) {
-			if (acrtc_state->freesync_vrr_info_changed)
+			if (acrtc_state->freesync_vrr_info_changed) {
 				bundle->stream_update.vrr_infopacket =
 					&acrtc_state->stream->vrr_infopacket;
+				bundle->stream_update.vsp_infopacket =
+					&acrtc_state->stream->vsp_infopacket;
+				stream_update_needed = true;
+			}
 		}
 	}
 
@@ -4075,6 +4178,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 		if (new_pcrtc_state->mode_changed) {
 			bundle->stream_update.src = acrtc_state->stream->src;
 			bundle->stream_update.dst = acrtc_state->stream->dst;
+			stream_update_needed = true;
 		}
 
 		if (new_pcrtc_state->color_mgmt_changed) {
@@ -4092,11 +4196,14 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 				(struct dc_3dlut *) acrtc_state->stream->lut3d_func;
 			bundle->stream_update.func_shaper =
 				(struct dc_transfer_func *) acrtc_state->stream->func_shaper;
+			stream_update_needed = true;
 		}
 
 		acrtc_state->stream->abm_level = acrtc_state->abm_level;
-		if (acrtc_state->abm_level != dm_old_crtc_state->abm_level)
+		if (acrtc_state->abm_level != dm_old_crtc_state->abm_level) {
 			bundle->stream_update.abm_level = &acrtc_state->abm_level;
+			stream_update_needed = true;
+		}
 
 		/*
 		 * If FreeSync state on the stream has changed then we need to
@@ -4112,10 +4219,9 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 		}
 		mutex_lock(&dm->dc_lock);
 		update_planes_and_stream_adapter(dm->dc,
-					 acrtc_state->update_type,
 					 planes_count,
 					 acrtc_state->stream,
-					 &bundle->stream_update,
+					 stream_update_needed ? &bundle->stream_update : NULL,
 					 bundle->surface_updates);
 		updated_planes_and_streams = true;
 
@@ -4315,9 +4421,9 @@ bool amdgpu_dm_crtc_complete_writeback(struct amdgpu_crtc *acrtc)
 }
 EXPORT_IF_KUNIT(amdgpu_dm_crtc_complete_writeback);
 
-static void dm_clear_writeback(struct amdgpu_display_manager *dm,
-			      struct amdgpu_crtc *acrtc,
-			      struct dm_crtc_state *crtc_state)
+STATIC_IFN_KUNIT void dm_clear_writeback(struct amdgpu_display_manager *dm,
+					 struct amdgpu_crtc *acrtc,
+					 struct dm_crtc_state *crtc_state)
 {
 	dc_stream_remove_writeback(dm->dc, crtc_state->stream, 0);
 
@@ -4329,6 +4435,7 @@ static void dm_clear_writeback(struct amdgpu_display_manager *dm,
 	 */
 	amdgpu_dm_crtc_complete_writeback(acrtc);
 }
+EXPORT_IF_KUNIT(dm_clear_writeback);
 
 /**
  * amdgpu_dm_mod_power_update_streams - update mod_power stream state on modeset
@@ -4339,8 +4446,8 @@ static void dm_clear_writeback(struct amdgpu_display_manager *dm,
  * in preparation for hardware programming. See also
  * amdgpu_dm_mod_power_setup_streams() for post-modeset mod_power setup.
  */
-static void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
-					       struct amdgpu_display_manager *dm)
+STATIC_IFN_KUNIT void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
+							struct amdgpu_display_manager *dm)
 {
 	struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
@@ -4393,6 +4500,7 @@ static void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
 		}
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_mod_power_update_streams);
 
 /**
  * amdgpu_dm_mod_power_setup_streams - setup mod_power stream state post modeset
@@ -4402,8 +4510,8 @@ static void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
  * Notify mod_power of mode_change. This needs to be done after dc_stream
  * updates have been committed, and VRR parameters have been updated.
  */
-static void amdgpu_dm_mod_power_setup_streams(struct drm_atomic_commit *state,
-					      struct amdgpu_display_manager *dm)
+STATIC_IFN_KUNIT void amdgpu_dm_mod_power_setup_streams(struct drm_atomic_commit *state,
+						       struct amdgpu_display_manager *dm)
 {
 	struct dm_crtc_state *dm_new_crtc_state;
 	struct drm_crtc_state *new_crtc_state;
@@ -4440,6 +4548,7 @@ static void amdgpu_dm_mod_power_setup_streams(struct drm_atomic_commit *state,
 	}
 
 }
+EXPORT_IF_KUNIT(amdgpu_dm_mod_power_setup_streams);
 
 static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
 					struct dc_state *dc_state)
@@ -4496,6 +4605,13 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
 		    (!new_crtc_state->active ||
 		     drm_atomic_crtc_needs_modeset(new_crtc_state))) {
 			manage_dm_interrupts(adev, acrtc, NULL);
+			/*
+			 * ISM hysteresis lives on system_dfl_wq, not the
+			 * vblank workqueue. Wait it out so a timer armed while
+			 * the stream existed cannot allow idle after the
+			 * stream is released.
+			 */
+			amdgpu_dm_ism_flush(&acrtc->ism);
 			dc_stream_release(dm_old_crtc_state->stream);
 		}
 	}
@@ -4660,10 +4776,11 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
 	}
 }
 
-static void dm_set_writeback(struct amdgpu_display_manager *dm,
-			      struct dm_crtc_state *crtc_state,
-			      struct drm_connector *connector,
-			      struct drm_connector_state *new_con_state)
+STATIC_IFN_KUNIT void
+dm_set_writeback(struct amdgpu_display_manager *dm,
+		 struct dm_crtc_state *crtc_state,
+		 struct drm_connector *connector,
+		 struct drm_connector_state *new_con_state)
 {
 	struct drm_writeback_connector *wb_conn = drm_connector_to_writeback(connector);
 	struct amdgpu_device *adev = dm->adev;
@@ -4780,8 +4897,9 @@ static void dm_set_writeback(struct amdgpu_display_manager *dm,
 cleanup:
 	kfree(wb_info);
 }
+EXPORT_IF_KUNIT(dm_set_writeback);
 
-static void amdgpu_dm_update_hdcp(struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT void amdgpu_dm_update_hdcp(struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *old_con_state, *new_con_state;
 	struct drm_device *dev = state->dev;
@@ -4895,8 +5013,9 @@ static void amdgpu_dm_update_hdcp(struct drm_atomic_commit *state)
 		}
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_update_hdcp);
 
-static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
@@ -4928,6 +5047,7 @@ static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_atomic_setup_commit);
 
 STATIC_IFN_KUNIT void set_multisync_trigger_params(
 		struct dc_stream_state *stream)
@@ -5313,8 +5433,8 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
  * Grabs all modesetting locks to serialize against any blocking commits,
  * Waits for completion of all non blocking commits.
  */
-static int do_aquire_global_lock(struct drm_device *dev,
-				 struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT int do_aquire_global_lock(struct drm_device *dev,
+					   struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_commit *commit;
@@ -5359,14 +5479,15 @@ static int do_aquire_global_lock(struct drm_device *dev,
 
 	return ret < 0 ? ret : 0;
 }
-
-static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
-			 struct drm_atomic_commit *state,
-			 struct drm_crtc *crtc,
-			 struct drm_crtc_state *old_crtc_state,
-			 struct drm_crtc_state *new_crtc_state,
-			 bool enable,
-			 bool *lock_and_validation_needed)
+EXPORT_IF_KUNIT(do_aquire_global_lock);
+
+STATIC_IFN_KUNIT int dm_update_crtc_state(struct amdgpu_display_manager *dm,
+					  struct drm_atomic_commit *state,
+					  struct drm_crtc *crtc,
+					  struct drm_crtc_state *old_crtc_state,
+					  struct drm_crtc_state *new_crtc_state,
+					  bool enable,
+					  bool *needs_dc_state_realloc)
 {
 	struct dm_atomic_state *dm_state = NULL;
 	struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
@@ -5539,7 +5660,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 
 		amdgpu_dm_reset_freesync_config_for_crtc(dm_new_crtc_state);
 
-		*lock_and_validation_needed = true;
+		*needs_dc_state_realloc = true;
 
 	} else {/* Add stream for any updated/enabled CRTC */
 		/*
@@ -5577,14 +5698,16 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 				goto fail;
 			}
 
-			*lock_and_validation_needed = true;
+			*needs_dc_state_realloc = true;
 		}
 	}
 
 skip_modeset:
 	/* Release extra reference */
-	if (new_stream)
+	if (new_stream) {
 		dc_stream_release(new_stream);
+		new_stream = NULL;
+	}
 	new_stream = NULL;
 
 	/*
@@ -5635,11 +5758,12 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 		dc_stream_release(new_stream);
 	return ret;
 }
+EXPORT_IF_KUNIT(dm_update_crtc_state);
 
-static bool should_reset_plane(struct drm_atomic_commit *state,
-			       struct drm_plane *plane,
-			       struct drm_plane_state *old_plane_state,
-			       struct drm_plane_state *new_plane_state)
+STATIC_IFN_KUNIT bool should_reset_plane(struct drm_atomic_commit *state,
+					 struct drm_plane *plane,
+					 struct drm_plane_state *old_plane_state,
+					 struct drm_plane_state *new_plane_state)
 {
 	struct drm_plane *other;
 	struct drm_plane_state *old_other_state, *new_other_state;
@@ -5799,15 +5923,17 @@ static bool should_reset_plane(struct drm_atomic_commit *state,
 
 	return false;
 }
+EXPORT_IF_KUNIT(should_reset_plane);
 
-static int dm_update_plane_state(struct dc *dc,
-				 struct drm_atomic_commit *state,
-				 struct drm_plane *plane,
-				 struct drm_plane_state *old_plane_state,
-				 struct drm_plane_state *new_plane_state,
-				 bool enable,
-				 bool *lock_and_validation_needed,
-				 bool *is_top_most_overlay)
+STATIC_IFN_KUNIT int
+dm_update_plane_state(struct dc *dc,
+		      struct drm_atomic_commit *state,
+		      struct drm_plane *plane,
+		      struct drm_plane_state *old_plane_state,
+		      struct drm_plane_state *new_plane_state,
+		      bool enable,
+		      bool *needs_dc_state_realloc,
+		      bool *is_top_most_overlay)
 {
 
 	struct dm_atomic_state *dm_state = NULL;
@@ -5877,7 +6003,7 @@ static int dm_update_plane_state(struct dc *dc,
 
 		dm_new_plane_state->dc_state = NULL;
 
-		*lock_and_validation_needed = true;
+		*needs_dc_state_realloc = true;
 
 	} else { /* Add new planes */
 		struct dc_plane_state *dc_new_plane_state;
@@ -5901,6 +6027,19 @@ static int dm_update_plane_state(struct dc *dc,
 		if (ret)
 			goto out;
 
+		/*
+		 * Recreating the plane re-derives its DC color pipeline from the
+		 * colorop states in this commit. Pull the plane's colorops in so
+		 * the derivation sees the current pipeline; otherwise every stage
+		 * silently defaults to bypass (dropping YUV->RGB, shaper, 3D LUT
+		 * and regamma) when userspace didn't touch color in this commit.
+		 */
+		if (new_plane_state->color_pipeline) {
+			ret = drm_atomic_add_affected_colorops(state, plane);
+			if (ret)
+				goto out;
+		}
+
 		WARN_ON(dm_new_plane_state->dc_state);
 
 		dc_new_plane_state = dc_create_plane_state(dc);
@@ -5955,7 +6094,7 @@ static int dm_update_plane_state(struct dc *dc,
 		 */
 		dm_new_plane_state->dc_state->update_bits.full_update = 1;
 
-		*lock_and_validation_needed = true;
+		*needs_dc_state_realloc = true;
 	}
 
 out:
@@ -5971,6 +6110,7 @@ static int dm_update_plane_state(struct dc *dc,
 
 	return ret;
 }
+EXPORT_IF_KUNIT(dm_update_plane_state);
 
 /*
  * The normalized_zpos value cannot be used by this iterator directly. It's only
@@ -6017,7 +6157,8 @@ struct __drm_planes_state *amdgpu_dm_get_next_zpos(
 	return &state->planes[highest_i];
 }
 
-static int add_affected_mst_dsc_crtcs(struct drm_atomic_commit *state, struct drm_crtc *crtc)
+STATIC_IFN_KUNIT int add_affected_mst_dsc_crtcs(struct drm_atomic_commit *state,
+						struct drm_crtc *crtc)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *conn_state, *old_conn_state;
@@ -6046,10 +6187,11 @@ static int add_affected_mst_dsc_crtcs(struct drm_atomic_commit *state, struct dr
 
 	return drm_dp_mst_add_affected_dsc_crtcs(state, &aconnector->mst_root->mst_mgr);
 }
+EXPORT_IF_KUNIT(add_affected_mst_dsc_crtcs);
 
-static bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev,
-					    struct drm_atomic_commit *state,
-					    struct drm_crtc_state *crtc_state)
+STATIC_IFN_KUNIT bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev,
+						      struct drm_atomic_commit *state,
+						      struct drm_crtc_state *crtc_state)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *new_plane_state, *old_plane_state;
@@ -6068,6 +6210,7 @@ static bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev,
 
 	return false;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_mem_type_changed);
 
 /**
  * amdgpu_dm_atomic_check() - Atomic check implementation for AMDgpu DM.
@@ -6094,8 +6237,8 @@ static bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev,
  *
  * Return: -Error code if validation failed.
  */
-static int amdgpu_dm_atomic_check(struct drm_device *dev,
-				  struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT int amdgpu_dm_atomic_check(struct drm_device *dev,
+					    struct drm_atomic_commit *state)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct dm_atomic_state *dm_state = NULL;
@@ -6108,7 +6251,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	struct drm_plane_state *old_plane_state, *new_plane_state, *new_cursor_state;
 	enum dc_status status;
 	int ret, i;
-	bool lock_and_validation_needed = false;
+	bool needs_dc_state_realloc = false;
 	bool is_top_most_overlay = true;
 	struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
 	struct drm_dp_mst_topology_mgr *mgr;
@@ -6282,7 +6425,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					    old_plane_state,
 					    new_plane_state,
 					    false,
-					    &lock_and_validation_needed,
+					    &needs_dc_state_realloc,
 					    &is_top_most_overlay);
 		if (ret) {
 			drm_dbg_atomic(dev, "dm_update_plane_state() failed: %pe\n", ERR_PTR(ret));
@@ -6296,7 +6439,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					   old_crtc_state,
 					   new_crtc_state,
 					   false,
-					   &lock_and_validation_needed);
+					   &needs_dc_state_realloc);
 		if (ret) {
 			drm_dbg_atomic(dev, "DISABLE: dm_update_crtc_state() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
@@ -6309,7 +6452,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					   old_crtc_state,
 					   new_crtc_state,
 					   true,
-					   &lock_and_validation_needed);
+					   &needs_dc_state_realloc);
 		if (ret) {
 			drm_dbg_atomic(dev, "ENABLE: dm_update_crtc_state() failed: %pe\n", ERR_PTR(ret));
 			goto fail;
@@ -6322,7 +6465,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 					    old_plane_state,
 					    new_plane_state,
 					    true,
-					    &lock_and_validation_needed,
+					    &needs_dc_state_realloc,
 					    &is_top_most_overlay);
 		if (ret) {
 			drm_dbg_atomic(dev, "dm_update_plane_state() failed: %pe\n", ERR_PTR(ret));
@@ -6437,7 +6580,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		if (!is_scaling_state_different(dm_new_con_state, dm_old_con_state))
 			continue;
 
-		lock_and_validation_needed = true;
+		needs_dc_state_realloc = true;
 	}
 
 	/* set the slot info for each mst_state based on the link encoding format */
@@ -6473,7 +6616,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	 *
 	 * TODO: Remove this stall and drop DM state private objects.
 	 */
-	if (lock_and_validation_needed) {
+	if (needs_dc_state_realloc) {
 		ret = dm_atomic_get_state(state, &dm_state);
 		if (ret) {
 			drm_dbg_atomic(dev, "dm_atomic_get_state() failed: %pe\n", ERR_PTR(ret));
@@ -6574,7 +6717,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		 * the FB pitch, the DCC state, rotation, mem_type, etc.
 		 */
 		if (new_crtc_state->async_flip &&
-		    (lock_and_validation_needed ||
+		    (needs_dc_state_realloc ||
 		     amdgpu_dm_crtc_mem_type_changed(dev, state, new_crtc_state))) {
 			drm_dbg_atomic(crtc->dev,
 				       "[CRTC:%d:%s] async flips are only supported for fast updates\n",
@@ -6583,7 +6726,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 			goto fail;
 		}
 
-		dm_new_crtc_state->update_type = lock_and_validation_needed ?
+		dm_new_crtc_state->update_type = needs_dc_state_realloc ?
 			UPDATE_TYPE_FULL : UPDATE_TYPE_FAST;
 	}
 
@@ -6606,6 +6749,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 
 	return ret;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_atomic_check);
 
 void amdgpu_dm_trigger_timing_sync(struct drm_device *dev)
 {
@@ -6625,6 +6769,7 @@ void amdgpu_dm_trigger_timing_sync(struct drm_device *dev)
 	}
 	mutex_unlock(&adev->dm.dc_lock);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_trigger_timing_sync);
 
 void dm_write_reg_func(const struct dc_context *ctx, uint32_t address,
 		       u32 value, const char *func_name)
@@ -6669,3 +6814,4 @@ void dm_acpi_process_phy_transition_interlock(
 {
 	// Not yet implemented
 }
+EXPORT_IF_KUNIT(dm_acpi_process_phy_transition_interlock);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index 881c8d1c3cc0df..0143b42fa2bda3 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -86,6 +86,9 @@ struct dc;
 struct amdgpu_bo;
 struct dmub_srv;
 struct dc_plane_state;
+struct dc_stream_state;
+struct dc_stream_update;
+struct dc_surface_update;
 struct dmub_notification;
 struct dmub_cmd_fused_request;
 
@@ -330,6 +333,7 @@ struct hpd_rx_irq_offload_work {
  * @backlight_link: Link on which to control backlight
  * @backlight_caps: Capabilities of the backlight device
  * @freesync_module: Module handling freesync calculations
+ * @power_module: Module handling power calculations
  * @hdcp_workqueue: AMDGPU content protection queue
  * @fw_dmcu: Reference to DMCU firmware
  * @dmcu_fw_version: Version of the DMCU firmware
@@ -589,6 +593,11 @@ struct amdgpu_display_manager {
 	struct amdgpu_dm_backlight_caps backlight_caps[AMDGPU_DM_MAX_NUM_EDP];
 
 	struct mod_freesync *freesync_module;
+	/**
+	 * @power_module:
+	 *
+	 * Power management module.
+	 */
 	struct mod_power *power_module;
 	struct hdcp_workqueue *hdcp_workqueue;
 
@@ -728,16 +737,26 @@ struct amdgpu_display_manager {
 		char reply_data[0x40];  // Cannot include dmub_cmd here
 	} fused_io[8];
 	/**
-	 * @hdmi_frl_status_polling_work:
+	 * @hdmi_frl_status_polling_wq:
 	 *
-	 * workqueue for 200ms frl status polling
+	 * Workqueue for HDMI FRL status polling.
 	 */
 	struct workqueue_struct *hdmi_frl_status_polling_wq;
+	/**
+	 * @hdmi_frl_status_polling_work:
+	 *
+	 * Delayed work for 200ms HDMI FRL status polling.
+	 */
 	struct delayed_work hdmi_frl_status_polling_work;
+	/**
+	 * @hdmi_frl_status_polling_delay_ms:
+	 *
+	 * Delay, in milliseconds, between HDMI FRL status polls.
+	 */
 	unsigned int hdmi_frl_status_polling_delay_ms;
 
 	/**
-	 * @dm_boot_time_crc_info:
+	 * @boot_time_crc_info:
 	 *
 	 * Buffer info for the boot time crc.
 	 */
@@ -946,6 +965,10 @@ enum amdgpu_transfer_function {
 	AMDGPU_TRANSFER_FUNCTION_COUNT
 };
 
+struct dc_flip_addrs;
+struct dc_scaling_info;
+struct dc_plane_info;
+
 struct dm_plane_state {
 	struct drm_plane_state base;
 	struct dc_plane_state *dc_state;
@@ -1015,6 +1038,16 @@ struct dm_plane_state {
 	 * applying blend LUT.
 	 */
 	enum amdgpu_transfer_function blend_tf;
+
+	/* Cached per-plane surface descriptors kept in the DRM plane state.
+	 * The DRM atomic old/new state swap lets us compare the previous
+	 * commit's values (old) against the newly computed ones to detect a
+	 * real plane change (vs an address-only flip) so DC only gets a
+	 * scaling_info/plane_info surface update when it actually changed.
+	 */
+	struct dc_flip_addrs *flip_addr;
+	struct dc_scaling_info *scaling_info;
+	struct dc_plane_info *plane_info;
 };
 
 enum amdgpu_dm_cursor_mode {
@@ -1169,7 +1202,6 @@ struct __drm_planes_state *amdgpu_dm_get_next_zpos(struct drm_atomic_commit *sta
 
 #if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
 struct amdgpu_ip_block;
-bool dm_is_idle(struct amdgpu_ip_block *ip_block);
 int dm_wait_for_idle(struct amdgpu_ip_block *ip_block);
 int dm_soft_reset(struct amdgpu_ip_block *ip_block);
 int dm_set_clockgating_state(struct amdgpu_ip_block *ip_block,
@@ -1181,12 +1213,80 @@ u32 dm_vblank_get_counter(struct amdgpu_device *adev, int crtc);
 int dm_crtc_get_scanoutpos(struct amdgpu_device *adev, int crtc,
 			   u32 *vbl, u32 *position);
 struct dm_atomic_state *dm_atomic_get_new_state(struct drm_atomic_commit *state);
+struct drm_private_state *dm_atomic_duplicate_state(struct drm_private_obj *obj);
 void dm_atomic_destroy_state(struct drm_private_obj *obj,
 			     struct drm_private_state *state);
+struct dc_flip_addrs;
+void fill_dc_dirty_rects(struct drm_plane *plane,
+			 struct drm_plane_state *old_plane_state,
+			 struct drm_plane_state *new_plane_state,
+			 struct drm_crtc_state *crtc_state,
+			 struct dc_flip_addrs *flip_addrs,
+			 bool is_psr_su,
+			 bool *dirty_regions_changed);
+void dm_update_pflip_irq_state(struct amdgpu_device *adev,
+			       struct amdgpu_crtc *acrtc);
+void amdgpu_dm_commit_cursors(struct drm_atomic_commit *state);
+void amdgpu_dm_update_cursor(struct drm_plane *plane,
+			     struct drm_plane_state *old_plane_state,
+			     struct dc_stream_update *update);
+void dm_arm_vblank_event(struct amdgpu_crtc *acrtc,
+			 struct dm_crtc_state *acrtc_state,
+			 bool pflip_update, bool cursor_update);
+void dm_arm_vblank_event_pre_programming(struct amdgpu_crtc *acrtc,
+					 struct dm_crtc_state *acrtc_state,
+					 bool pflip_update, bool cursor_update);
+bool should_reset_plane(struct drm_atomic_commit *state,
+			struct drm_plane *plane,
+			struct drm_plane_state *old_plane_state,
+			struct drm_plane_state *new_plane_state);
+int add_affected_mst_dsc_crtcs(struct drm_atomic_commit *state,
+			       struct drm_crtc *crtc);
+bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev,
+				     struct drm_atomic_commit *state,
+				     struct drm_crtc_state *crtc_state);
+void amdgpu_dm_dump_links_and_sinks(struct amdgpu_device *adev);
+void amdgpu_dm_update_hdcp(struct drm_atomic_commit *state);
+int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state);
+int do_aquire_global_lock(struct drm_device *dev,
+			  struct drm_atomic_commit *state);
+int dm_update_crtc_state(struct amdgpu_display_manager *dm,
+			 struct drm_atomic_commit *state,
+			 struct drm_crtc *crtc,
+			 struct drm_crtc_state *old_crtc_state,
+			 struct drm_crtc_state *new_crtc_state,
+			 bool enable,
+			 bool *lock_and_validation_needed);
+int dm_update_plane_state(struct dc *dc,
+			  struct drm_atomic_commit *state,
+			  struct drm_plane *plane,
+			  struct drm_plane_state *old_plane_state,
+			  struct drm_plane_state *new_plane_state,
+			  bool enable,
+			  bool *lock_and_validation_needed,
+			  bool *is_top_most_overlay);
+int amdgpu_dm_atomic_check(struct drm_device *dev,
+			   struct drm_atomic_commit *state);
+void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
+					struct amdgpu_display_manager *dm);
+void amdgpu_dm_mod_power_setup_streams(struct drm_atomic_commit *state,
+				       struct amdgpu_display_manager *dm);
+int amdgpu_dm_early_fini(struct amdgpu_ip_block *ip_block);
+int dm_sw_fini(struct amdgpu_ip_block *ip_block);
+int dm_oem_i2c_hw_init(struct amdgpu_device *adev);
+void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr);
+void s3_handle_mst(struct drm_device *dev, bool suspend);
+void dm_gpureset_commit_state(struct dc_state *dc_state, struct amdgpu_display_manager *dm);
 int dm_plane_layer_index_cmp(const void *a, const void *b);
-int fill_plane_color_attributes(const struct drm_plane_state *plane_state,
-				const enum surface_pixel_format format,
-				enum dc_color_space *color_space);
+bool update_planes_and_stream_adapter(struct dc *dc,
+				      int planes_count,
+				      struct dc_stream_state *stream,
+				      struct dc_stream_update *stream_update,
+				      struct dc_surface_update *array_of_surface_update);
+int fill_plane_color_attributes(struct drm_atomic_commit *state,
+			    const struct drm_plane_state *plane_state,
+			    const enum surface_pixel_format format,
+			    enum dc_color_space *color_space);
 bool modereset_required(struct drm_crtc_state *crtc_state);
 bool is_scaling_state_different(const struct dm_connector_state *dm_state,
 				const struct dm_connector_state *old_dm_state);
@@ -1200,6 +1300,74 @@ bool is_content_protection_different(struct drm_crtc_state *new_crtc_state,
 				     struct drm_connector_state *old_conn_state,
 				     const struct drm_connector *connector,
 				     struct hdcp_workqueue *hdcp_w);
+
+struct amdgpu_dm_services_kunit_ops {
+	int (*bo_create_kernel)(struct amdgpu_device *adev, unsigned long size,
+				int align, u32 domain, struct amdgpu_bo **bo_ptr,
+				u64 *gpu_addr, void **cpu_addr);
+	void (*bo_free_kernel)(struct amdgpu_bo **bo, u64 *gpu_addr,
+			       void **cpu_addr);
+};
+
+void amdgpu_dm_services_kunit_set_ops(const struct amdgpu_dm_services_kunit_ops *ops);
+
+struct amdgpu_dm_kunit_ops {
+	uint64_t (*gmc_pd_addr)(struct amdgpu_bo *bo);
+	void (*post_update_surfaces_to_stream)(struct dc *dc);
+	bool (*update_planes_and_stream)(struct dc *dc,
+					 struct dc_surface_update *surface_updates,
+					 int surface_count,
+					 struct dc_stream_state *dc_stream,
+					 struct dc_stream_update *stream_update);
+	struct drm_atomic_commit *(*atomic_helper_suspend)(struct drm_device *dev);
+};
+
+void amdgpu_dm_kunit_set_ops(const struct amdgpu_dm_kunit_ops *ops);
+
+struct dc_phy_addr_space_config;
+struct dc_plane_address;
+struct dc_plane_cap;
+struct dc_plane_info;
+struct dsc_mst_fairness_vars;
+
+void mmhub_read_system_context(struct amdgpu_device *adev,
+			       struct dc_phy_addr_space_config *pa_config);
+int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm);
+void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, struct dc_state *state, bool enable);
+enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc);
+int dm_cache_state(struct amdgpu_device *adev);
+void dm_destroy_cached_state(struct amdgpu_device *adev);
+void dm_clear_writeback(struct amdgpu_display_manager *dm,
+			struct amdgpu_crtc *acrtc,
+			struct dm_crtc_state *crtc_state);
+void dm_set_writeback(struct amdgpu_display_manager *dm,
+		      struct dm_crtc_state *crtc_state,
+		      struct drm_connector *connector,
+		      struct drm_connector_state *new_con_state);
+int dm_early_init(struct amdgpu_ip_block *ip_block);
+int fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
+				struct drm_atomic_commit *state,
+				const struct drm_plane_state *plane_state,
+				struct dc_plane_info *plane_info,
+				struct dc_plane_address *address,
+				bool tmz_surface);
+int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_commit *state, struct dc_state *dc_state,
+				     struct dsc_mst_fairness_vars *vars);
+void manage_dm_interrupts(struct amdgpu_device *adev, struct amdgpu_crtc *acrtc,
+			  struct dm_crtc_state *acrtc_state);
+void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm,
+				   struct amdgpu_crtc *acrtc_attach,
+				   const struct dm_crtc_state *acrtc_state, const u64 current_ts);
+int load_dmcu_fw(struct amdgpu_device *adev);
+int dm_sw_init(struct amdgpu_ip_block *ip_block);
+int dm_late_init(struct amdgpu_ip_block *ip_block);
+int amdgpu_dm_mode_config_init(struct amdgpu_device *adev);
+int initialize_plane(struct amdgpu_display_manager *dm, struct amdgpu_mode_info *mode_info,
+		     int plane_id, enum drm_plane_type plane_type,
+		     const struct dc_plane_cap *plane_cap);
+int fill_dc_plane_attributes(struct amdgpu_device *adev, struct dc_plane_state *dc_plane_state,
+			     struct drm_plane_state *plane_state,
+			     struct drm_crtc_state *crtc_state);
 #endif
 
 #endif /* __AMDGPU_DM_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c
index b9e90ea449ca31..45e1732a5b041a 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c
@@ -49,6 +49,25 @@
 #include "amd_shared.h"
 #include "dm_helpers.h"
 
+#if defined(CONFIG_ACPI) || IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+STATIC_IFN_KUNIT
+void amdgpu_dm_validate_backlight_caps(struct amdgpu_display_manager *dm, int bl_idx)
+{
+	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[bl_idx];
+	int spread = caps->max_input_signal - caps->min_input_signal;
+
+	if (caps->max_input_signal > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
+	    caps->min_input_signal < 0 ||
+	    spread > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
+	    spread < AMDGPU_DM_MIN_SPREAD) {
+		drm_dbg_kms(adev_to_drm(dm->adev), "DM: Invalid backlight caps: min=%d, max=%d\n",
+			    caps->min_input_signal, caps->max_input_signal);
+		caps->caps_valid = false;
+	}
+}
+EXPORT_IF_KUNIT(amdgpu_dm_validate_backlight_caps);
+#endif
+
 void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
 				     int bl_idx)
 {
@@ -61,18 +80,8 @@ void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
 	amdgpu_acpi_get_backlight_caps(caps);
 
 	/* validate the firmware value is sane */
-	if (caps->caps_valid) {
-		int spread = caps->max_input_signal - caps->min_input_signal;
-
-		if (caps->max_input_signal > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
-		    caps->min_input_signal < 0 ||
-		    spread > AMDGPU_DM_DEFAULT_MAX_BACKLIGHT ||
-		    spread < AMDGPU_DM_MIN_SPREAD) {
-			drm_dbg_kms(adev_to_drm(dm->adev), "DM: Invalid backlight caps: min=%d, max=%d\n",
-				      caps->min_input_signal, caps->max_input_signal);
-			caps->caps_valid = false;
-		}
-	}
+	if (caps->caps_valid)
+		amdgpu_dm_validate_backlight_caps(dm, bl_idx);
 #else
 	if (caps->aux_support)
 		return;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h
index 07b75064847c7b..9ad888d3ae7b20 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.h
@@ -53,6 +53,7 @@ bool amdgpu_dm_should_create_sysfs(struct amdgpu_dm_connector *aconnector);
 extern const struct attribute_group amdgpu_group;
 
 #if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+void amdgpu_dm_validate_backlight_caps(struct amdgpu_display_manager *dm, int bl_idx);
 struct dc_stream_state *dm_find_stream_with_link(struct amdgpu_display_manager *dm,
 						 struct dc_link *link);
 int amdgpu_dm_backlight_update_status(struct backlight_device *bd);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index d55dc06167a8b7..1125ebac6ccbc8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -1551,6 +1551,8 @@ __set_colorop_in_tf_1d_curve(struct dc_plane_state *dc_plane_state,
 	struct dc_transfer_func *tf = &dc_plane_state->in_transfer_func;
 	struct drm_colorop *colorop = colorop_state->colorop;
 	struct drm_device *drm = colorop->dev;
+	struct dc_color_caps *color_caps = NULL;
+	bool is_subsampled_format;
 
 	if (colorop->type != DRM_COLOROP_1D_CURVE)
 		return -EINVAL;
@@ -1564,14 +1566,82 @@ __set_colorop_in_tf_1d_curve(struct dc_plane_state *dc_plane_state,
 	}
 
 	drm_dbg(drm, "Degamma colorop with ID: %d\n", colorop->base.id);
-
 	tf->type = TF_TYPE_PREDEFINED;
+
+	/* Check if format requires post-scale color processing (subsampled formats) */
+	is_subsampled_format = (dc_plane_state->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN &&
+				dc_plane_state->format < SURFACE_PIXEL_FORMAT_SUBSAMPLE_END);
+
 	tf->tf = amdgpu_colorop_tf_to_dc_tf(colorop_state->curve_1d_type);
 
+	if (dc_plane_state->ctx && dc_plane_state->ctx->dc)
+		color_caps = &dc_plane_state->ctx->dc->caps.color;
+
+	if (!mod_color_calculate_degamma_params(color_caps, tf, NULL,
+						is_subsampled_format)) {
+		drm_err(drm, "Failed to calculate degamma params\n");
+		return -EINVAL;
+	}
+
 	return 0;
 }
 EXPORT_IF_KUNIT(__set_colorop_in_tf_1d_curve);
 
+static int
+__set_dm_plane_colorop_fixed_matrix(struct drm_plane_state *plane_state,
+				    struct dc_plane_state *dc_plane_state,
+				    struct drm_colorop *colorop)
+{
+	struct drm_colorop *old_colorop;
+	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
+	struct drm_atomic_commit *state = plane_state->state;
+	int i = 0;
+
+	old_colorop = colorop;
+
+	for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
+		if (new_colorop_state->colorop == old_colorop) {
+			colorop_state = new_colorop_state;
+			break;
+		}
+	}
+
+	if (!colorop_state)
+		return -EINVAL;
+
+	if (colorop_state->bypass) {
+		dc_plane_state->color_space = COLOR_SPACE_SRGB;
+		return 0;
+	}
+
+	switch (colorop_state->fixed_matrix_type) {
+	case DRM_COLOROP_FM_YCBCR601_FULL_RGB:
+		dc_plane_state->color_space = COLOR_SPACE_YCBCR601;
+		break;
+	case DRM_COLOROP_FM_YCBCR601_LIMITED_RGB:
+		dc_plane_state->color_space = COLOR_SPACE_YCBCR601_LIMITED;
+		break;
+	case DRM_COLOROP_FM_YCBCR709_FULL_RGB:
+		dc_plane_state->color_space = COLOR_SPACE_YCBCR709;
+		break;
+	case DRM_COLOROP_FM_YCBCR709_LIMITED_RGB:
+		dc_plane_state->color_space = COLOR_SPACE_YCBCR709_LIMITED;
+		break;
+	case DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB:
+		dc_plane_state->color_space = COLOR_SPACE_2020_YCBCR_FULL;
+		break;
+	case DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB:
+		dc_plane_state->color_space = COLOR_SPACE_2020_YCBCR_LIMITED;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dc_plane_state->update_bits.full_update = 1;
+
+	return 0;
+}
+
 STATIC_IFN_KUNIT int
 __set_dm_plane_colorop_degamma(struct drm_plane_state *plane_state,
 			       struct dc_plane_state *dc_plane_state,
@@ -1976,10 +2046,21 @@ amdgpu_dm_plane_set_colorop_properties(struct drm_plane_state *plane_state,
 	bool has_3dlut = adev->dm.dc->caps.color.dpp.hw_3d_lut || adev->dm.dc->caps.color.mpc.preblend;
 	int ret;
 
-	/* 1D Curve - DEGAM TF */
+	/* Fixed Matrix (YUV to RGB) */
 	if (!colorop)
 		return -EINVAL;
 
+	ret = __set_dm_plane_colorop_fixed_matrix(plane_state, dc_plane_state, colorop);
+	if (ret)
+		return ret;
+
+	/* 1D Curve - DEGAM TF */
+	colorop = colorop->next;
+	if (!colorop) {
+		drm_dbg(dev, "no degamma colorop found\n");
+		return -EINVAL;
+	}
+
 	ret = __set_dm_plane_colorop_degamma(plane_state, dc_plane_state, colorop);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
index fae9007b9998ea..47d90b67120e5d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.c
@@ -55,6 +55,14 @@ const u64 amdgpu_dm_supported_blnd_tfs =
 	BIT(DRM_COLOROP_1D_CURVE_GAMMA22);
 EXPORT_IF_KUNIT(amdgpu_dm_supported_blnd_tfs);
 
+const u64 amdgpu_dm_supported_fm =
+	BIT(DRM_COLOROP_FM_YCBCR601_FULL_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR601_LIMITED_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR709_FULL_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR709_LIMITED_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB);
+
 #define MAX_COLOR_PIPELINE_OPS 10
 
 #define LUT3D_SIZE		17
@@ -63,6 +71,48 @@ static const struct drm_colorop_funcs dm_colorop_funcs = {
 	.destroy = drm_colorop_destroy,
 };
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+static struct drm_colorop *amdgpu_dm_colorop_kzalloc(void)
+{
+	return kzalloc_obj(struct drm_colorop);
+}
+
+static const struct amdgpu_dm_colorop_kunit_ops amdgpu_dm_colorop_default_ops = {
+	.colorop_kzalloc_obj = amdgpu_dm_colorop_kzalloc,
+	.curve_1d_init = drm_plane_colorop_curve_1d_init,
+	.curve_1d_lut_init = drm_plane_colorop_curve_1d_lut_init,
+	.ctm_3x4_init = drm_plane_colorop_ctm_3x4_init,
+	.mult_init = drm_plane_colorop_mult_init,
+	.lut3d_init = drm_plane_colorop_3dlut_init,
+};
+
+static const struct amdgpu_dm_colorop_kunit_ops *amdgpu_dm_colorop_ops =
+	&amdgpu_dm_colorop_default_ops;
+
+void amdgpu_dm_colorop_kunit_set_ops(const struct amdgpu_dm_colorop_kunit_ops *ops)
+{
+	amdgpu_dm_colorop_ops = ops ? ops : &amdgpu_dm_colorop_default_ops;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_colorop_kunit_set_ops);
+
+#define colorop_kzalloc_obj		amdgpu_dm_colorop_ops->colorop_kzalloc_obj
+#define colorop_curve_1d_init		amdgpu_dm_colorop_ops->curve_1d_init
+#define colorop_curve_1d_lut_init	amdgpu_dm_colorop_ops->curve_1d_lut_init
+#define colorop_ctm_3x4_init		amdgpu_dm_colorop_ops->ctm_3x4_init
+#define colorop_mult_init		amdgpu_dm_colorop_ops->mult_init
+#define colorop_3dlut_init		amdgpu_dm_colorop_ops->lut3d_init
+
+#else
+
+#define colorop_kzalloc_obj()		kzalloc_obj(struct drm_colorop)
+#define colorop_curve_1d_init		drm_plane_colorop_curve_1d_init
+#define colorop_curve_1d_lut_init	drm_plane_colorop_curve_1d_lut_init
+#define colorop_ctm_3x4_init		drm_plane_colorop_ctm_3x4_init
+#define colorop_mult_init		drm_plane_colorop_mult_init
+#define colorop_3dlut_init		drm_plane_colorop_3dlut_init
+
+#endif
+
 STATIC_IFN_KUNIT int
 amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane,
 				  bool hw_3d_lut, struct drm_prop_enum_list *list)
@@ -73,16 +123,16 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 
 	memset(ops, 0, sizeof(ops));
 
-	/* 1D curve - DEGAM TF */
+	/* Fixed Matrix (YUV to RGB) */
 	ops[i] = kzalloc_obj(*ops[0]);
 	if (!ops[i]) {
 		ret = -ENOMEM;
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
-					      amdgpu_dm_supported_degam_tfs,
-					      DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = drm_plane_colorop_fixed_matrix_init(dev, ops[i], plane, &dm_colorop_funcs,
+						  amdgpu_dm_supported_fm,
+						  DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -90,15 +140,32 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 
 	i++;
 
+	/* 1D curve - DEGAM TF */
+	ops[i] = colorop_kzalloc_obj();
+	if (!ops[i]) {
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	ret = colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
+				    amdgpu_dm_supported_degam_tfs,
+				    DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	if (ret)
+		goto cleanup;
+
+	drm_colorop_set_next_property(ops[i - 1], ops[i]);
+
+	i++;
+
 	/* Multiplier */
-	ops[i] = kzalloc_obj(struct drm_colorop);
+	ops[i] = colorop_kzalloc_obj();
 	if (!ops[i]) {
 		ret = -ENOMEM;
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_mult_init(dev, ops[i], plane, &dm_colorop_funcs,
-					  DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = colorop_mult_init(dev, ops[i], plane, &dm_colorop_funcs,
+				DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -107,15 +174,14 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 	i++;
 
 	/* 3x4 matrix */
-	ops[i] = kzalloc_obj(struct drm_colorop);
+	ops[i] = colorop_kzalloc_obj();
 	if (!ops[i]) {
 		ret = -ENOMEM;
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_ctm_3x4_init(dev, ops[i], plane,
-					     &dm_colorop_funcs,
-					     DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = colorop_ctm_3x4_init(dev, ops[i], plane, &dm_colorop_funcs,
+				   DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -125,15 +191,15 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 
 	if (hw_3d_lut) {
 		/* 1D curve - SHAPER TF */
-		ops[i] = kzalloc_obj(*ops[0]);
+		ops[i] = colorop_kzalloc_obj();
 		if (!ops[i]) {
 			ret = -ENOMEM;
 			goto cleanup;
 		}
 
-		ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
-						amdgpu_dm_supported_shaper_tfs,
-						DRM_COLOROP_FLAG_ALLOW_BYPASS);
+		ret = colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
+					    amdgpu_dm_supported_shaper_tfs,
+					    DRM_COLOROP_FLAG_ALLOW_BYPASS);
 		if (ret)
 			goto cleanup;
 
@@ -142,17 +208,16 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 		i++;
 
 		/* 1D LUT - SHAPER LUT */
-		ops[i] = kzalloc_obj(*ops[0]);
+		ops[i] = colorop_kzalloc_obj();
 		if (!ops[i]) {
 			ret = -ENOMEM;
 			goto cleanup;
 		}
 
-		ret = drm_plane_colorop_curve_1d_lut_init(dev, ops[i], plane,
-							&dm_colorop_funcs,
-							MAX_COLOR_LUT_ENTRIES,
-							DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
-							DRM_COLOROP_FLAG_ALLOW_BYPASS);
+		ret = colorop_curve_1d_lut_init(dev, ops[i], plane, &dm_colorop_funcs,
+						MAX_COLOR_LUT_ENTRIES,
+						DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
+						DRM_COLOROP_FLAG_ALLOW_BYPASS);
 		if (ret)
 			goto cleanup;
 
@@ -161,16 +226,16 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 		i++;
 
 		/* 3D LUT */
-		ops[i] = kzalloc_obj(*ops[0]);
+		ops[i] = colorop_kzalloc_obj();
 		if (!ops[i]) {
 			ret = -ENOMEM;
 			goto cleanup;
 		}
 
-		ret = drm_plane_colorop_3dlut_init(dev, ops[i], plane,
-					&dm_colorop_funcs, LUT3D_SIZE,
-					DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL,
-					DRM_COLOROP_FLAG_ALLOW_BYPASS);
+		ret = colorop_3dlut_init(dev, ops[i], plane, &dm_colorop_funcs,
+					 LUT3D_SIZE,
+					 DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL,
+					 DRM_COLOROP_FLAG_ALLOW_BYPASS);
 		if (ret)
 			goto cleanup;
 
@@ -180,15 +245,15 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 	}
 
 	/* 1D curve - BLND TF */
-	ops[i] = kzalloc_obj(*ops[0]);
+	ops[i] = colorop_kzalloc_obj();
 	if (!ops[i]) {
 		ret = -ENOMEM;
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
-					      amdgpu_dm_supported_blnd_tfs,
-					      DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = colorop_curve_1d_init(dev, ops[i], plane, &dm_colorop_funcs,
+				    amdgpu_dm_supported_blnd_tfs,
+				    DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
@@ -197,16 +262,16 @@ amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane
 	i++;
 
 	/* 1D LUT - BLND LUT */
-	ops[i] = kzalloc_obj(struct drm_colorop);
+	ops[i] = colorop_kzalloc_obj();
 	if (!ops[i]) {
 		ret = -ENOMEM;
 		goto cleanup;
 	}
 
-	ret = drm_plane_colorop_curve_1d_lut_init(dev, ops[i], plane, &dm_colorop_funcs,
-						  MAX_COLOR_LUT_ENTRIES,
-						  DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
-						  DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	ret = colorop_curve_1d_lut_init(dev, ops[i], plane, &dm_colorop_funcs,
+					MAX_COLOR_LUT_ENTRIES,
+					DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
+					DRM_COLOROP_FLAG_ALLOW_BYPASS);
 	if (ret)
 		goto cleanup;
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.h
index 77364d954d3bb8..edf7df7c19e9fe 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_colorop.h
@@ -30,12 +30,41 @@
 extern const u64 amdgpu_dm_supported_degam_tfs;
 extern const u64 amdgpu_dm_supported_shaper_tfs;
 extern const u64 amdgpu_dm_supported_blnd_tfs;
+extern const u64 amdgpu_dm_supported_fm;
 
 int amdgpu_dm_initialize_default_pipeline(struct drm_plane *plane, struct drm_prop_enum_list *list);
 
 #if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+#include <drm/drm_colorop.h>
+
+struct amdgpu_dm_colorop_kunit_ops {
+	struct drm_colorop *(*colorop_kzalloc_obj)(void);
+	int (*curve_1d_init)(struct drm_device *dev, struct drm_colorop *colorop,
+			     struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+			     u64 supported_tfs, uint32_t flags);
+	int (*curve_1d_lut_init)(struct drm_device *dev, struct drm_colorop *colorop,
+				 struct drm_plane *plane,
+				 const struct drm_colorop_funcs *funcs,
+				 uint32_t lut_size,
+				 enum drm_colorop_lut1d_interpolation_type interpolation,
+				 uint32_t flags);
+	int (*ctm_3x4_init)(struct drm_device *dev, struct drm_colorop *colorop,
+			    struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+			    uint32_t flags);
+	int (*mult_init)(struct drm_device *dev, struct drm_colorop *colorop,
+			 struct drm_plane *plane, const struct drm_colorop_funcs *funcs,
+			 uint32_t flags);
+	int (*lut3d_init)(struct drm_device *dev, struct drm_colorop *colorop,
+			  struct drm_plane *plane,
+			  const struct drm_colorop_funcs *funcs,
+			  uint32_t lut_size,
+			  enum drm_colorop_lut3d_interpolation_type interpolation,
+			  uint32_t flags);
+};
+
 int amdgpu_dm_build_default_pipeline(struct drm_device *dev, struct drm_plane *plane,
 				      bool hw_3d_lut, struct drm_prop_enum_list *list);
+void amdgpu_dm_colorop_kunit_set_ops(const struct amdgpu_dm_colorop_kunit_ops *ops);
 #endif
 
 #endif /* __AMDGPU_DM_COLOROP_H__*/
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
index 154e1f35dcb190..2a63a1c9b79d8f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c
@@ -27,7 +27,6 @@
 #include "dm_services_types.h"
 #include "dc.h"
 #include "dc/dc_dmub_srv.h"
-#include "dc/dc_edid_parser.h"
 #include "dc/dc_stat.h"
 #include "dc/dc_state.h"
 #include "dc/dc_stream.h"
@@ -171,6 +170,7 @@ const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs = {
 	.disable = dm_encoder_helper_disable,
 	.atomic_check = dm_encoder_helper_atomic_check
 };
+EXPORT_IF_KUNIT(amdgpu_dm_encoder_helper_funcs);
 
 int amdgpu_dm_get_encoder_crtc_mask(struct amdgpu_device *adev)
 {
@@ -215,6 +215,7 @@ int amdgpu_dm_encoder_init(struct drm_device *dev,
 
 	return res;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_encoder_init);
 
 STATIC_IFN_KUNIT enum drm_mode_subconnector get_subconnector_type(struct dc_link *link)
 {
@@ -254,7 +255,7 @@ STATIC_IFN_KUNIT void update_subconnector_property(struct amdgpu_dm_connector *a
 }
 EXPORT_IF_KUNIT(update_subconnector_property);
 
-static int amdgpu_dm_connector_get_modes(struct drm_connector *connector);
+STATIC_IFN_KUNIT int amdgpu_dm_connector_get_modes(struct drm_connector *connector);
 
 STATIC_IFN_KUNIT void amdgpu_dm_fbc_init(struct drm_connector *connector)
 {
@@ -1266,6 +1267,12 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
 	link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link,
 							dc_link_get_link_cap(aconnector->dc_link));
 
+	if (aconnector->dc_link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA &&
+	    aconnector->dc_link->dpia_bw_alloc_config.bw_alloc_enabled) {
+		dsc_options.max_target_bpp_limit_override_x16 = 8 * 16;
+		dsc_options.force_dsc_when_not_needed = true;
+	}
+
 	/* Set DSC policy according to dsc_clock_en */
 	dc_dsc_policy_set_enable_dsc_when_not_needed(
 		aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE);
@@ -2325,7 +2332,9 @@ amdgpu_dm_create_validate_stream_for_sink(struct drm_connector *connector,
 	 *    (drm_mode_is_420_also() clear), as required for HDMI compliance
 	 *    testing; dc_validate_stream() still rejects anything the link
 	 *    genuinely cannot carry. The YCbCr422/YCbCr444 forces stay gated on
-	 *    the sink's advertised caps.
+	 *    the sink's advertised caps. An RGB force pins to RGB alone and is
+	 *    always honoured, since RGB is the mandatory baseline every sink
+	 *    (DP or HDMI) supports.
 	 */
 	want_420 = (aconnector->force_yuv_pixel_format == PIXEL_ENCODING_YCBCR420) ||
 		(drm_state && drm_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_YCBCR420);
@@ -2345,7 +2354,8 @@ amdgpu_dm_create_validate_stream_for_sink(struct drm_connector *connector,
 		   (info->color_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444)) &&
 		   is_dp_or_hdmi) {
 		encoding_mask = BIT(PIXEL_ENCODING_YCBCR444);
-	} else if (drm_state && drm_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_RGB444) {
+	} else if ((aconnector->force_yuv_pixel_format == PIXEL_ENCODING_RGB) ||
+		   (drm_state && drm_state->color_format == DRM_CONNECTOR_COLOR_FORMAT_RGB444)) {
 		encoding_mask = BIT(PIXEL_ENCODING_RGB);
 	} else {
 		encoding_mask = BIT(PIXEL_ENCODING_RGB);
@@ -2575,7 +2585,7 @@ int amdgpu_dm_fill_hdr_info_packet(const struct drm_connector_state *state,
 }
 EXPORT_IF_KUNIT(amdgpu_dm_fill_hdr_info_packet);
 
-static int
+STATIC_IFN_KUNIT int
 amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
 				 struct drm_atomic_commit *state)
 {
@@ -2656,6 +2666,7 @@ amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_connector_atomic_check);
 
 static const struct drm_connector_helper_funcs
 amdgpu_dm_connector_helper_funcs = {
@@ -2892,6 +2903,7 @@ void amdgpu_set_panel_orientation(struct drm_connector *connector)
 						       native_mode->hdisplay,
 						       native_mode->vdisplay);
 }
+EXPORT_IF_KUNIT(amdgpu_set_panel_orientation);
 
 /*
  * The Apple Studio Display primary tile advertises both the full 5120x2880
@@ -2900,7 +2912,7 @@ void amdgpu_set_panel_orientation(struct drm_connector *connector)
  * per-tile timing from the primary connector so compositors only pick the full
  * 5K mode.
  */
-static void amdgpu_dm_prune_primary_tile_modes(struct drm_connector *connector)
+STATIC_IFN_KUNIT void amdgpu_dm_prune_primary_tile_modes(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
 	struct drm_display_mode *mode, *t;
@@ -2933,6 +2945,7 @@ static void amdgpu_dm_prune_primary_tile_modes(struct drm_connector *connector)
 		aconnector->num_modes--;
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_prune_primary_tile_modes);
 
 STATIC_IFN_KUNIT void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector,
 					      const struct drm_edid *drm_edid)
@@ -3086,7 +3099,7 @@ STATIC_IFN_KUNIT void amdgpu_dm_connector_add_freesync_modes(struct drm_connecto
 }
 EXPORT_IF_KUNIT(amdgpu_dm_connector_add_freesync_modes);
 
-static int amdgpu_dm_connector_get_modes(struct drm_connector *connector)
+STATIC_IFN_KUNIT int amdgpu_dm_connector_get_modes(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *amdgpu_dm_connector =
 			to_amdgpu_dm_connector(connector);
@@ -3125,6 +3138,7 @@ static int amdgpu_dm_connector_get_modes(struct drm_connector *connector)
 
 	return amdgpu_dm_connector->num_modes;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_connector_get_modes);
 
 static const u32 supported_colorspaces =
 	BIT(DRM_MODE_COLORIMETRY_BT709_YCC) |
@@ -3138,7 +3152,7 @@ static const u32 supported_colorformats =
 	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422) |
 	BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
 
-static void hdmi_frl_status_polling_work(struct work_struct *work)
+void hdmi_frl_status_polling_work(struct work_struct *work)
 {
 	struct amdgpu_display_manager *dm =
 		container_of(to_delayed_work(work), struct amdgpu_display_manager,
@@ -3172,6 +3186,7 @@ static void hdmi_frl_status_polling_work(struct work_struct *work)
 			   &dm->hdmi_frl_status_polling_work,
 			   msecs_to_jiffies(dm->hdmi_frl_status_polling_delay_ms));
 }
+EXPORT_IF_KUNIT(hdmi_frl_status_polling_work);
 
 void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 				     struct amdgpu_dm_connector *aconnector,
@@ -3316,6 +3331,7 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 		}
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_connector_init_helper);
 
 STATIC_IFN_KUNIT int amdgpu_dm_i2c_xfer(struct i2c_adapter *i2c_adap,
 			      struct i2c_msg *msgs, int num)
@@ -3397,6 +3413,7 @@ amdgpu_dm_create_i2c(struct ddc_service *ddc_service, bool oem)
 
 	return i2c;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_create_i2c);
 
 int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector)
 {
@@ -3419,6 +3436,7 @@ int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_initialize_hdmi_connector);
 
 /*
  * Note: this function assumes that dc_link_detect() was called for the
@@ -3496,8 +3514,9 @@ int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
 	}
 	return res;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_connector_init);
 
-static int dm_force_atomic_commit(struct drm_connector *connector)
+STATIC_IFN_KUNIT int dm_force_atomic_commit(struct drm_connector *connector)
 {
 	int ret = 0;
 	struct drm_device *ddev = connector->dev;
@@ -3557,6 +3576,7 @@ static int dm_force_atomic_commit(struct drm_connector *connector)
 
 	return ret;
 }
+EXPORT_IF_KUNIT(dm_force_atomic_commit);
 
 /*
  * This function handles all cases when set mode does not come upon hotplug.
@@ -3594,139 +3614,7 @@ void dm_restore_drm_connector_state(struct drm_device *dev,
 	if (acrtc_state->stream->sink != aconnector->dc_sink)
 		dm_force_atomic_commit(&aconnector->base);
 }
-
-static bool dm_edid_parser_send_cea(struct amdgpu_display_manager *dm,
-		unsigned int offset,
-		unsigned int total_length,
-		u8 *data,
-		unsigned int length,
-		struct amdgpu_hdmi_vsdb_info *vsdb)
-{
-	bool res;
-	union dmub_rb_cmd cmd;
-	struct dmub_cmd_send_edid_cea *input;
-	struct dmub_cmd_edid_cea_output *output;
-
-	if (length > DMUB_EDID_CEA_DATA_CHUNK_BYTES)
-		return false;
-
-	memset(&cmd, 0, sizeof(cmd));
-
-	input = &cmd.edid_cea.data.input;
-
-	cmd.edid_cea.header.type = DMUB_CMD__EDID_CEA;
-	cmd.edid_cea.header.sub_type = 0;
-	cmd.edid_cea.header.payload_bytes =
-		sizeof(cmd.edid_cea) - sizeof(cmd.edid_cea.header);
-	input->offset = offset;
-	input->length = length;
-	input->cea_total_length = total_length;
-	memcpy(input->payload, data, length);
-
-	res = dc_wake_and_execute_dmub_cmd(dm->dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY);
-	if (!res) {
-		drm_err(adev_to_drm(dm->adev), "EDID CEA parser failed\n");
-		return false;
-	}
-
-	output = &cmd.edid_cea.data.output;
-
-	if (output->type == DMUB_CMD__EDID_CEA_ACK) {
-		if (!output->ack.success) {
-			drm_err(adev_to_drm(dm->adev), "EDID CEA ack failed at offset %d\n",
-					output->ack.offset);
-		}
-	} else if (output->type == DMUB_CMD__EDID_CEA_AMD_VSDB) {
-		if (!output->amd_vsdb.vsdb_found)
-			return false;
-
-		vsdb->freesync_supported = output->amd_vsdb.freesync_supported;
-		vsdb->amd_vsdb_version = output->amd_vsdb.amd_vsdb_version;
-		vsdb->min_refresh_rate_hz = output->amd_vsdb.min_frame_rate;
-		vsdb->max_refresh_rate_hz = output->amd_vsdb.max_frame_rate;
-		vsdb->freesync_mccs_vcp_code = output->amd_vsdb.freesync_mccs_vcp_code;
-	} else {
-		drm_warn(adev_to_drm(dm->adev), "Unknown EDID CEA parser results\n");
-		return false;
-	}
-
-	return true;
-}
-
-static bool parse_edid_cea_dmcu(struct amdgpu_display_manager *dm,
-		u8 *edid_ext, int len,
-		struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	int i;
-
-	/* send extension block to DMCU for parsing */
-	for (i = 0; i < len; i += 8) {
-		bool res;
-		int offset;
-
-		/* send 8 bytes a time */
-		if (!dc_edid_parser_send_cea(dm->dc, i, len, &edid_ext[i], 8))
-			return false;
-
-		if (i+8 == len) {
-			/* EDID block sent completed, expect result */
-			int version, min_rate, max_rate;
-
-			res = dc_edid_parser_recv_amd_vsdb(dm->dc, &version, &min_rate, &max_rate);
-			if (res) {
-				/* amd vsdb found */
-				vsdb_info->freesync_supported = 1;
-				vsdb_info->amd_vsdb_version = version;
-				vsdb_info->min_refresh_rate_hz = min_rate;
-				vsdb_info->max_refresh_rate_hz = max_rate;
-				/* Not enabled on DMCU*/
-				vsdb_info->freesync_mccs_vcp_code = 0;
-				return true;
-			}
-			/* not amd vsdb */
-			return false;
-		}
-
-		/* check for ack*/
-		res = dc_edid_parser_recv_cea_ack(dm->dc, &offset);
-		if (!res)
-			return false;
-	}
-
-	return false;
-}
-
-static bool parse_edid_cea_dmub(struct amdgpu_display_manager *dm,
-		u8 *edid_ext, int len,
-		struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	int i;
-
-	/* send extension block to DMCU for parsing */
-	for (i = 0; i < len; i += 8) {
-		/* send 8 bytes a time */
-		if (!dm_edid_parser_send_cea(dm, i, len, &edid_ext[i], 8, vsdb_info))
-			return false;
-	}
-
-	return vsdb_info->freesync_supported;
-}
-
-static bool parse_edid_cea(struct amdgpu_dm_connector *aconnector,
-		u8 *edid_ext, int len,
-		struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	struct amdgpu_device *adev = drm_to_adev(aconnector->base.dev);
-	bool ret;
-
-	mutex_lock(&adev->dm.dc_lock);
-	if (adev->dm.dmub_srv)
-		ret = parse_edid_cea_dmub(&adev->dm, edid_ext, len, vsdb_info);
-	else
-		ret = parse_edid_cea_dmcu(&adev->dm, edid_ext, len, vsdb_info);
-	mutex_unlock(&adev->dm.dc_lock);
-	return ret;
-}
+EXPORT_IF_KUNIT(dm_restore_drm_connector_state);
 
 STATIC_IFN_KUNIT void parse_edid_displayid_vrr(struct drm_connector *connector,
 				     const struct edid *edid)
@@ -3779,44 +3667,15 @@ STATIC_IFN_KUNIT int get_amd_vsdb(struct amdgpu_dm_connector *aconnector,
 
 	vsdb_info->replay_mode = connector->display_info.amd_vsdb.replay_mode;
 	vsdb_info->amd_vsdb_version = connector->display_info.amd_vsdb.version;
+	vsdb_info->freesync_supported = connector->display_info.amd_vsdb.freesync_supported;
+	vsdb_info->min_refresh_rate_hz = connector->display_info.amd_vsdb.min_frame_rate;
+	vsdb_info->max_refresh_rate_hz = connector->display_info.amd_vsdb.max_frame_rate;
+	vsdb_info->freesync_mccs_vcp_code = connector->display_info.amd_vsdb.freesync_vcp_code;
 
 	return connector->display_info.amd_vsdb.version != 0;
 }
 EXPORT_IF_KUNIT(get_amd_vsdb);
 
-STATIC_IFN_KUNIT int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector,
-			       const struct edid *edid,
-			       struct amdgpu_hdmi_vsdb_info *vsdb_info)
-{
-	u8 *edid_ext = NULL;
-	int i;
-	bool valid_vsdb_found = false;
-
-	/*----- drm_find_cea_extension() -----*/
-	/* No EDID or EDID extensions */
-	if (edid == NULL || edid->extensions == 0)
-		return -ENODEV;
-
-	/* Find CEA extension */
-	for (i = 0; i < edid->extensions; i++) {
-		edid_ext = (uint8_t *)edid + EDID_LENGTH * (i + 1);
-		if (edid_ext[0] == CEA_EXT)
-			break;
-	}
-
-	if (i == edid->extensions)
-		return -ENODEV;
-
-	/*----- cea_db_offsets() -----*/
-	if (edid_ext[0] != CEA_EXT)
-		return -ENODEV;
-
-	valid_vsdb_found = parse_edid_cea(aconnector, edid_ext, EDID_LENGTH, vsdb_info);
-
-	return valid_vsdb_found ? i : -ENODEV;
-}
-EXPORT_IF_KUNIT(parse_hdmi_amd_vsdb);
-
 /**
  * amdgpu_dm_update_freesync_caps - Update Freesync capabilities
  *
@@ -3835,7 +3694,7 @@ EXPORT_IF_KUNIT(parse_hdmi_amd_vsdb);
 void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 				    const struct drm_edid *drm_edid, bool do_mccs)
 {
-	int i = 0;
+	bool has_vsdb = 0;
 	struct amdgpu_dm_connector *amdgpu_dm_connector =
 			to_amdgpu_dm_connector(connector);
 	struct dm_connector_state *dm_con_state = NULL;
@@ -3872,6 +3731,15 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 	if (!adev->dm.freesync_module || !dc_supports_vrr(sink->ctx->dce_version))
 		goto update;
 
+	drm_dbg_driver(adev_to_drm(adev),
+		       "VRR: enter signal=%d hdmi_vrr=%d mrange[%d-%d] hdmi.vrr_cap[sup=%d min=%d max=%d]\n",
+		       sink->sink_signal, connector->display_info.hdmi.vrr_cap.supported,
+		       connector->display_info.monitor_range.min_vfreq,
+		       connector->display_info.monitor_range.max_vfreq,
+		       connector->display_info.hdmi.vrr_cap.supported,
+		       connector->display_info.hdmi.vrr_cap.vrr_min,
+		       connector->display_info.hdmi.vrr_cap.vrr_max);
+
 	/* FIXME: Get rid of drm_edid_raw() */
 	edid = drm_edid_raw(drm_edid);
 
@@ -3880,66 +3748,139 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 	     connector->display_info.monitor_range.max_vfreq == 0))
 		parse_edid_displayid_vrr(connector, edid);
 
-	if (edid && (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT ||
-		     sink->sink_signal == SIGNAL_TYPE_EDP)) {
-		if (amdgpu_dm_connector->dc_link &&
-		    amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param) {
-			amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq;
-			amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq;
-			if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
-				freesync_capable = true;
+	has_vsdb = get_amd_vsdb(amdgpu_dm_connector, &vsdb_info) != 0;
+
+	if (has_vsdb) {
+		amdgpu_dm_connector->vsdb_info = vsdb_info;
+
+		/* copy refresh rate info as long as VSDB advertises FreeSync */
+		if (vsdb_info.freesync_supported) {
+			amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
+			amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
+
+			connector->display_info.monitor_range.min_vfreq =
+				vsdb_info.min_refresh_rate_hz;
+			connector->display_info.monitor_range.max_vfreq =
+				vsdb_info.max_refresh_rate_hz;
 		}
+	} else {
+		/* fall back to the base EDID range when there is no VSDB */
+		amdgpu_dm_connector->min_vfreq = connector->display_info.monitor_range.min_vfreq;
+		amdgpu_dm_connector->max_vfreq = connector->display_info.monitor_range.max_vfreq;
+	}
 
-		get_amd_vsdb(amdgpu_dm_connector, &vsdb_info);
+	if (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT ||
+	    sink->sink_signal == SIGNAL_TYPE_EDP) {
+		if (amdgpu_dm_connector->dc_link &&
+		    amdgpu_dm_connector->dc_link->dpcd_caps.allow_invalid_MSA_timing_param &&
+		    amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
+			freesync_capable = true;
 
 		if (vsdb_info.replay_mode) {
-			amdgpu_dm_connector->vsdb_info.replay_mode = vsdb_info.replay_mode;
-			amdgpu_dm_connector->vsdb_info.amd_vsdb_version = vsdb_info.amd_vsdb_version;
 			amdgpu_dm_connector->as_type = ADAPTIVE_SYNC_TYPE_EDP;
 		}
+	} else if (has_vsdb && (sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A ||
+				sink->sink_signal == SIGNAL_TYPE_HDMI_FRL)) {
+		amdgpu_dm_connector->vsdb_info = vsdb_info;
+		sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
+
+		if (vsdb_info.freesync_supported) {
+			amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
+			amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
+			if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
+				freesync_capable = true;
 
-	} else if (drm_edid && sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A) {
-		i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info);
-		if (i >= 0) {
-			amdgpu_dm_connector->vsdb_info = vsdb_info;
-			sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
+			connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz;
+			connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz;
+		}
+	}
 
-			if (vsdb_info.freesync_supported) {
-				amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
-				amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
-				if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
-					freesync_capable = true;
+	drm_dbg_driver(adev_to_drm(adev),
+		       "VRR: amd_vsdb fs_sup=%d min=%d max=%d fs_capable=%d\n",
+		       vsdb_info.freesync_supported,
+		       vsdb_info.min_refresh_rate_hz,
+		       vsdb_info.max_refresh_rate_hz, freesync_capable);
 
-				connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz;
-				connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz;
+	/*
+	 * If AMD VSDB didn't provide a valid FreeSync range, fall back to
+	 * the HDMI 2.1 VRR capability parsed from the HF-VSDB.
+	 */
+	if (!freesync_capable && connector->display_info.hdmi.vrr_cap.supported) {
+		struct drm_hdmi_vrr_cap *vrr_cap =
+			&connector->display_info.hdmi.vrr_cap;
+
+		drm_dbg_driver(adev_to_drm(adev),
+			       "VRR: HF-VSDB fallback: hdmi_vrr=1 vrr_cap[sup=%d min=%d max=%d] mrange_max=%d\n",
+			       vrr_cap->supported, vrr_cap->vrr_min, vrr_cap->vrr_max,
+			       connector->display_info.monitor_range.max_vfreq);
+
+		if (vrr_cap->supported && vrr_cap->vrr_min > 0) {
+			amdgpu_dm_connector->min_vfreq = vrr_cap->vrr_min;
+			amdgpu_dm_connector->max_vfreq = vrr_cap->vrr_max ?
+				vrr_cap->vrr_max :
+				connector->display_info.monitor_range.max_vfreq;
+
+			/*
+			 * VRRMAX = 0 in the HF-VSDB means "up to the Base
+			 * Refresh Rate". If the EDID also did not provide a
+			 * monitor range max, fall back to the Base Refresh
+			 * Rate (the highest refresh rate of the preferred
+			 * timing) so a valid VRR range is still reported to
+			 * userspace.
+			 */
+			if (!amdgpu_dm_connector->max_vfreq) {
+				struct drm_display_mode *brr_mode =
+					amdgpu_dm_get_highest_refresh_rate_mode(amdgpu_dm_connector, true);
+
+				if (brr_mode)
+					amdgpu_dm_connector->max_vfreq =
+						drm_mode_vrefresh(brr_mode);
 			}
+
+			if (amdgpu_dm_connector->max_vfreq -
+			    amdgpu_dm_connector->min_vfreq > 10)
+				freesync_capable = true;
+
+			connector->display_info.monitor_range.min_vfreq =
+				amdgpu_dm_connector->min_vfreq;
+			connector->display_info.monitor_range.max_vfreq =
+				amdgpu_dm_connector->max_vfreq;
 		}
 	}
 
 	if (amdgpu_dm_connector->dc_link)
 		as_type = dm_get_adaptive_sync_support_type(amdgpu_dm_connector->dc_link);
 
-	if (as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) {
-		i = parse_hdmi_amd_vsdb(amdgpu_dm_connector, edid, &vsdb_info);
-		if (i >= 0) {
-			amdgpu_dm_connector->vsdb_info = vsdb_info;
-			sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
-
-			if (vsdb_info.freesync_supported && vsdb_info.amd_vsdb_version > 0) {
-				amdgpu_dm_connector->pack_sdp_v1_3 = true;
-				amdgpu_dm_connector->as_type = as_type;
+	if (has_vsdb && as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) {
+		sink->edid_caps.freesync_vcp_code = vsdb_info.freesync_mccs_vcp_code;
 
-				amdgpu_dm_connector->min_vfreq = vsdb_info.min_refresh_rate_hz;
-				amdgpu_dm_connector->max_vfreq = vsdb_info.max_refresh_rate_hz;
-				if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
-					freesync_capable = true;
+		if (vsdb_info.freesync_supported && vsdb_info.amd_vsdb_version > 0) {
+			amdgpu_dm_connector->pack_sdp_v1_3 = true;
+			amdgpu_dm_connector->as_type = as_type;
 
-				connector->display_info.monitor_range.min_vfreq = vsdb_info.min_refresh_rate_hz;
-				connector->display_info.monitor_range.max_vfreq = vsdb_info.max_refresh_rate_hz;
-			}
+			if (amdgpu_dm_connector->max_vfreq - amdgpu_dm_connector->min_vfreq > 10)
+				freesync_capable = true;
 		}
 	}
 
+	/*
+	 * Apply per-monitor FreeSync range quirks. Some panels report a
+	 * VRR minimum that does not operate reliably; force it when the
+	 * monitor is quirked (see apply_edid_quirks()).
+	 */
+	if (sink->edid_caps.panel_patch.force_freesync_min_hz && freesync_capable) {
+		amdgpu_dm_connector->min_vfreq =
+			sink->edid_caps.panel_patch.force_freesync_min_hz;
+		connector->display_info.monitor_range.min_vfreq =
+			amdgpu_dm_connector->min_vfreq;
+	}
+
+	/* HDMI/PCON without an EDID FreeSync VCP code is not FreeSync-capable. */
+	if ((sink->sink_signal == SIGNAL_TYPE_HDMI_TYPE_A ||
+	     as_type == FREESYNC_TYPE_PCON_IN_WHITELIST) &&
+	    !sink->edid_caps.freesync_vcp_code)
+		freesync_capable = false;
+
 	/* Handle MCCS */
 	if (do_mccs) {
 		dm_helpers_read_mccs_caps(adev->dm.dc->ctx, amdgpu_dm_connector->dc_link, sink);
@@ -3955,6 +3896,11 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 	if (dm_con_state)
 		dm_con_state->freesync_capable = freesync_capable;
 
+	drm_dbg_driver(adev_to_drm(adev),
+		       "VRR: caps result: freesync_capable=%d min_vfreq=%d max_vfreq=%d\n",
+		       freesync_capable, amdgpu_dm_connector->min_vfreq,
+		       amdgpu_dm_connector->max_vfreq);
+
 	if (connector->state && amdgpu_dm_connector->dc_link && !freesync_capable &&
 	    amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported) {
 		amdgpu_dm_connector->dc_link->replay_settings.config.replay_supported = false;
@@ -3965,3 +3911,4 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 		drm_connector_set_vrr_capable_property(connector,
 						       freesync_capable);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_update_freesync_caps);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
index 4e9eb3ff2c904c..79698deacf616d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.h
@@ -71,6 +71,8 @@ void amdgpu_dm_connector_init_helper(struct amdgpu_display_manager *dm,
 				     struct dc_link *link,
 				     int link_index);
 
+void hdmi_frl_status_polling_work(struct work_struct *work);
+
 enum drm_mode_status amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
 						    const struct drm_display_mode *mode);
 
@@ -153,9 +155,6 @@ void parse_edid_displayid_vrr(struct drm_connector *connector,
 			      const struct edid *edid);
 int get_amd_vsdb(struct amdgpu_dm_connector *aconnector,
 		 struct amdgpu_hdmi_vsdb_info *vsdb_info);
-int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector,
-			const struct edid *edid,
-			struct amdgpu_hdmi_vsdb_info *vsdb_info);
 void amdgpu_dm_connector_funcs_force(struct drm_connector *connector);
 enum dc_status dm_validate_stream_and_context(struct dc *dc,
 					      struct dc_stream_state *stream);
@@ -168,6 +167,8 @@ void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder,
 					  struct drm_connector *connector);
 void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector,
 				       const struct drm_edid *drm_edid);
+int amdgpu_dm_connector_get_modes(struct drm_connector *connector);
+void amdgpu_dm_prune_primary_tile_modes(struct drm_connector *connector);
 uint add_fs_modes(struct amdgpu_dm_connector *aconnector);
 void amdgpu_dm_connector_add_freesync_modes(struct drm_connector *connector,
 					    const struct drm_edid *drm_edid);
@@ -222,5 +223,9 @@ void decide_crtc_timing_for_drm_display_mode(struct drm_display_mode *drm_mode,
 					     bool scale_enabled);
 void amdgpu_dm_set_panel_type(struct amdgpu_dm_connector *aconnector);
 void amdgpu_dm_update_cacp_caps(struct amdgpu_dm_connector *aconnector);
+struct drm_atomic_commit;
+int amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
+				     struct drm_atomic_commit *state);
+int dm_force_atomic_commit(struct drm_connector *connector);
 #endif
 #endif /* __AMDGPU_DM_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
index 1d941be73561c6..5036ac1c4fd540 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
@@ -250,6 +250,30 @@ STATIC_IFN_KUNIT void amdgpu_dm_crtc_vblank_control_worker(struct work_struct *w
 }
 EXPORT_IF_KUNIT(amdgpu_dm_crtc_vblank_control_worker);
 
+#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) || IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+STATIC_IFN_KUNIT int amdgpu_dm_crtc_set_vline0_irq(struct drm_crtc *crtc, int irq_type,
+						   bool enable)
+{
+	struct amdgpu_device *adev = drm_to_adev(crtc->dev);
+	int rc;
+
+	/* crtc vline0 interrupt, only available on DCN+ */
+	if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0)
+		return 0;
+
+	if (enable) {
+		rc = amdgpu_irq_get(adev, &adev->vline0_irq, irq_type);
+		drm_dbg_vbl(crtc->dev, "Get vline0_irq ret=%d\n", rc);
+	} else {
+		rc = amdgpu_irq_put(adev, &adev->vline0_irq, irq_type);
+		drm_dbg_vbl(crtc->dev, "Put vline0_irq ret=%d\n", rc);
+	}
+
+	return rc;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_set_vline0_irq);
+#endif
+
 static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable)
 {
 	struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
@@ -359,19 +383,9 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable)
 	}
 
 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
-	/* crtc vline0 interrupt, only available on DCN+ */
-	if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) {
-		if (enable) {
-			rc = amdgpu_irq_get(adev, &adev->vline0_irq, irq_type);
-			drm_dbg_vbl(crtc->dev, "Get vline0_irq ret=%d\n", rc);
-		} else {
-			rc = amdgpu_irq_put(adev, &adev->vline0_irq, irq_type);
-			drm_dbg_vbl(crtc->dev, "Put vline0_irq ret=%d\n", rc);
-		}
-
-		if (rc)
-			return rc;
-	}
+	rc = amdgpu_dm_crtc_set_vline0_irq(crtc, irq_type, enable);
+	if (rc)
+		return rc;
 #endif
 
 	if (amdgpu_in_reset(adev))
@@ -464,40 +478,41 @@ STATIC_IFN_KUNIT struct drm_crtc_state *amdgpu_dm_crtc_duplicate_state(struct dr
 }
 EXPORT_IF_KUNIT(amdgpu_dm_crtc_duplicate_state);
 
-static void amdgpu_dm_crtc_destroy(struct drm_crtc *crtc)
+STATIC_IFN_KUNIT void amdgpu_dm_crtc_destroy(struct drm_crtc *crtc)
 {
 	/*
-	 * amdgpu_dm_ism_fini() is intentionally called in amdgpu_dm_fini().
-	 * It must be called before dc_destroy() in amdgpu_dm_fini()
-	 * to avoid ISM accessing an invalid dc handle once dc is released.
+	 * ISM workers are intentionally quiesced by amdgpu_dm_ism_disable()
+	 * in amdgpu_dm_fini(). That must happen before dc_destroy() so ISM
+	 * cannot access an invalid dc handle once dc is released.
 	 */
 
 	drm_crtc_cleanup(crtc);
 	kfree(crtc);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_destroy);
 
-STATIC_IFN_KUNIT void amdgpu_dm_crtc_reset_state(struct drm_crtc *crtc)
+STATIC_IFN_KUNIT struct drm_crtc_state *amdgpu_dm_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct dm_crtc_state *state;
 
 	state = kzalloc_obj(*state);
 	if (!state)
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	if (crtc->state)
-		amdgpu_dm_crtc_destroy_state(crtc, crtc->state);
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
 
-	__drm_atomic_helper_crtc_reset(crtc, &state->base);
+	return &state->base;
 }
-EXPORT_IF_KUNIT(amdgpu_dm_crtc_reset_state);
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_create_state);
 
 #ifdef CONFIG_DEBUG_FS
-static int amdgpu_dm_crtc_late_register(struct drm_crtc *crtc)
+STATIC_IFN_KUNIT int amdgpu_dm_crtc_late_register(struct drm_crtc *crtc)
 {
 	crtc_debugfs_init(crtc);
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_late_register);
 #endif
 
 #ifdef AMD_PRIVATE_COLOR
@@ -566,7 +581,7 @@ amdgpu_dm_atomic_crtc_get_property(struct drm_crtc *crtc,
 
 /* Implemented only the options currently available for the driver */
 static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = {
-	.reset = amdgpu_dm_crtc_reset_state,
+	.atomic_create_state = amdgpu_dm_crtc_create_state,
 	.destroy = amdgpu_dm_crtc_destroy,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -649,7 +664,7 @@ STATIC_IFN_KUNIT bool amdgpu_dm_crtc_helper_mode_fixup(struct drm_crtc *crtc,
 }
 EXPORT_IF_KUNIT(amdgpu_dm_crtc_helper_mode_fixup);
 
-static int amdgpu_dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
+STATIC_IFN_KUNIT int amdgpu_dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
 					      struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
@@ -711,6 +726,7 @@ static int amdgpu_dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
 	DRM_DEBUG_ATOMIC("Failed DC stream validation\n");
 	return ret;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_helper_atomic_check);
 
 static const struct drm_crtc_helper_funcs amdgpu_dm_crtc_helper_funcs = {
 	.disable = amdgpu_dm_crtc_helper_disable,
@@ -782,9 +798,18 @@ int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm,
 
 	drm_crtc_helper_add(&acrtc->base, &amdgpu_dm_crtc_helper_funcs);
 
-	/* Create (reset) the plane state */
-	if (acrtc->base.funcs->reset)
-		acrtc->base.funcs->reset(&acrtc->base);
+	/* Create the plane state */
+	if (acrtc->base.funcs->atomic_create_state) {
+		struct drm_crtc_state *crtc_state;
+
+		crtc_state = acrtc->base.funcs->atomic_create_state(&acrtc->base);
+		if (IS_ERR(crtc_state)) {
+			res = PTR_ERR(crtc_state);
+			goto error_ism_fini;
+		}
+
+		acrtc->base.state = crtc_state;
+	}
 
 	acrtc->max_cursor_width = dm->adev->dm.dc->caps.max_cursor_size;
 	acrtc->max_cursor_height = dm->adev->dm.dc->caps.max_cursor_size;
@@ -816,9 +841,13 @@ int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm,
 #endif
 	return 0;
 
+error_ism_fini:
+	amdgpu_dm_ism_flush(&acrtc->ism);
+	drm_crtc_cleanup(&acrtc->base);
 fail:
 	kfree(acrtc);
 	kfree(cursor_plane);
 	return res;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_init);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
index 93c6d0d8d7fd30..223ef34c559120 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
@@ -49,12 +49,19 @@ bool amdgpu_dm_crtc_helper_mode_fixup(struct drm_crtc *crtc,
 void amdgpu_dm_crtc_destroy_state(struct drm_crtc *crtc,
 				  struct drm_crtc_state *state);
 struct drm_crtc_state *amdgpu_dm_crtc_duplicate_state(struct drm_crtc *crtc);
-void amdgpu_dm_crtc_reset_state(struct drm_crtc *crtc);
+struct drm_crtc_state *amdgpu_dm_crtc_create_state(struct drm_crtc *crtc);
+void amdgpu_dm_crtc_destroy(struct drm_crtc *crtc);
 int amdgpu_dm_crtc_count_crtc_active_planes(struct drm_crtc_state *new_crtc_state);
 void amdgpu_dm_crtc_update_crtc_active_planes(struct drm_crtc *crtc,
 					      struct drm_crtc_state *new_crtc_state);
 void amdgpu_dm_crtc_vblank_control_worker(struct work_struct *work);
 void amdgpu_dm_idle_worker(struct work_struct *work);
+int amdgpu_dm_crtc_set_vline0_irq(struct drm_crtc *crtc, int irq_type, bool enable);
+int amdgpu_dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
+				       struct drm_atomic_commit *state);
+#ifdef CONFIG_DEBUG_FS
+int amdgpu_dm_crtc_late_register(struct drm_crtc *crtc);
+#endif
 #endif
 
 bool amdgpu_dm_crtc_vrr_active(const struct dm_crtc_state *dm_state);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.c
index d565433ea06d86..8204ad4018e6b2 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.c
@@ -41,9 +41,9 @@
 #include "amdgpu_dm_cursor.h"
 #include "dm_helpers.h"
 
-static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc,
-			      struct drm_plane_state *new_plane_state,
-			      struct drm_framebuffer *fb)
+STATIC_IFN_KUNIT int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc,
+				       struct drm_plane_state *new_plane_state,
+				       struct drm_framebuffer *fb)
 {
 	struct amdgpu_device *adev = drm_to_adev(new_acrtc->base.dev);
 	struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(fb);
@@ -87,12 +87,8 @@ static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc,
 	 * check tiling flags when the FB doesn't have a modifier.
 	 */
 	if (!(fb->flags & DRM_MODE_FB_MODIFIERS)) {
-#if defined(CONFIG_DRM_AMD_DC_DCN6_0) || defined(CONFIG_DRM_AMD_DC_DCN5_0)
 		if (adev->family == AMDGPU_FAMILY_GC_12_0_0
 		    || adev->family == AMDGPU_FAMILY_GC_13_0_1) {
-#else
-		if (adev->family == AMDGPU_FAMILY_GC_12_0_0) {
-#endif
 			linear = AMDGPU_TILING_GET(afb->tiling_flags, GFX12_SWIZZLE_MODE) == 0;
 		} else if (adev->family >= AMDGPU_FAMILY_AI) {
 			linear = AMDGPU_TILING_GET(afb->tiling_flags, SWIZZLE_MODE) == 0;
@@ -109,6 +105,7 @@ static int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(dm_check_cursor_fb);
 
 /*
  * Helper function for checking the cursor in native mode
@@ -142,6 +139,7 @@ int amdgpu_dm_check_native_cursor_state(struct drm_crtc *new_plane_crtc,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_check_native_cursor_state);
 
 bool amdgpu_dm_should_update_native_cursor(struct drm_atomic_commit *state,
 					   struct drm_crtc *old_plane_crtc,
@@ -233,9 +231,9 @@ EXPORT_IF_KUNIT(dm_get_plane_scale);
  *
  * Return: true if the pipeline modifies pixels, false otherwise.
  */
-static bool dm_plane_color_pipeline_active(struct drm_atomic_commit *state,
-					   struct drm_plane *plane,
-					   bool use_old)
+STATIC_IFN_KUNIT bool dm_plane_color_pipeline_active(struct drm_atomic_commit *state,
+						     struct drm_plane *plane,
+						     bool use_old)
 {
 	struct drm_colorop *colorop;
 	struct drm_colorop_state *old_colorop_state, *new_colorop_state;
@@ -251,6 +249,7 @@ static bool dm_plane_color_pipeline_active(struct drm_atomic_commit *state,
 	}
 	return false;
 }
+EXPORT_IF_KUNIT(dm_plane_color_pipeline_active);
 
 /**
  * amdgpu_dm_crtc_get_cursor_mode() - Determine the required cursor mode on crtc
@@ -286,26 +285,34 @@ int amdgpu_dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 	int underlying_scale_w, underlying_scale_h;
 	int cursor_scale_w, cursor_scale_h;
 	int i;
+	bool skip_fmt_scale_restrictions = false;
 
-	/* Overlay cursor not supported on HW before DCN
-	 * DCN401/420 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions
-	 * as previous DCN generations, so enable native mode on DCN401/420
-	 *
+	/*
 	 * Always set native cursor mode when the CRTC is disabled,
 	 * to make sure it doesn't cause atomic commits to fail when
 	 * they are trying to disable the CRTC.
 	 */
+	if (!crtc_state->enable) {
+		*cursor_mode = DM_CURSOR_NATIVE_MODE;
+		return 0;
+	}
+
+	/* Overlay cursor not supported on HW before DCN
+	 * DCN401/420 does not have the cursor-on-scaled-plane or cursor-on-yuv-plane restrictions
+	 * as previous DCN generations, so enable native mode on DCN401/420
+	 */
 	if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 0, 1) ||
 	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 0) ||
-#if defined(CONFIG_DRM_AMD_DC_DCN6_0)
 	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 1) ||
-	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(6, 0, 0) ||
-#else
-	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 1) ||
-#endif
-	    !dm_crtc_state->base.enable) {
-		*cursor_mode = DM_CURSOR_NATIVE_MODE;
-		return 0;
+	    amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(6, 0, 0)) {
+		/*
+		 * Newer DCN has no cursor-on-scaled/yuv-plane restriction, so
+		 * skip those overlay triggers below. A plane that does not fill
+		 * the CRTC still needs overlay mode so the cursor renders over
+		 * the hole, so fall through to the coverage check instead of
+		 * unconditionally forcing native mode here.
+		 */
+		skip_fmt_scale_restrictions = true;
 	}
 
 	/* Init cursor_mode to be the same as current */
@@ -404,13 +411,15 @@ int amdgpu_dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 			continue;
 
 		/* Underlying plane is YUV format - use overlay cursor */
-		if (amdgpu_dm_plane_is_video_format(plane_state->fb->format->format)) {
+		if (!skip_fmt_scale_restrictions &&
+		    amdgpu_dm_plane_is_video_format(plane_state->fb->format->format)) {
 			*cursor_mode = DM_CURSOR_OVERLAY_MODE;
 			return 0;
 		}
 
 		/* Underlying plane has an active color pipeline - cursor would be transformed */
-		if (dm_plane_color_pipeline_active(state, plane, false)) {
+		if (!skip_fmt_scale_restrictions &&
+		    dm_plane_color_pipeline_active(state, plane, false)) {
 			*cursor_mode = DM_CURSOR_OVERLAY_MODE;
 			return 0;
 		}
@@ -421,7 +430,8 @@ int amdgpu_dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 				   &cursor_scale_w, &cursor_scale_h);
 
 		/* Underlying plane has different scale - use overlay cursor */
-		if (cursor_scale_w != underlying_scale_w &&
+		if (!skip_fmt_scale_restrictions &&
+		    cursor_scale_w != underlying_scale_w &&
 		    cursor_scale_h != underlying_scale_h) {
 			*cursor_mode = DM_CURSOR_OVERLAY_MODE;
 			return 0;
@@ -446,3 +456,4 @@ int amdgpu_dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_crtc_get_cursor_mode);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.h
index 5f619e4474aaf8..cda6d73e9bda01 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_cursor.h
@@ -43,6 +43,12 @@ int amdgpu_dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 				   enum amdgpu_dm_cursor_mode *cursor_mode);
 
 #if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+int dm_check_cursor_fb(struct amdgpu_crtc *new_acrtc,
+		       struct drm_plane_state *new_plane_state,
+		       struct drm_framebuffer *fb);
+bool dm_plane_color_pipeline_active(struct drm_atomic_commit *state,
+				    struct drm_plane *plane,
+				    bool use_old);
 void dm_get_oriented_plane_size(struct drm_plane_state *plane_state,
 				int *src_w, int *src_h);
 void dm_get_plane_scale(struct drm_plane_state *plane_state,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 2a6b48e24869a9..36f32ed4b87c4f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -1282,6 +1282,24 @@ static int internal_display_show(struct seq_file *m, void *data)
 	return 0;
 }
 
+/*
+ * Returns the maximum source viewport width (in pixels) this ASIC can
+ * downscale from. A value of 0 means there is no such limit.
+ * Example usage: cat /sys/kernel/debug/dri/0/eDP-1/max_downscale_src_width
+ */
+static int max_downscale_src_width_show(struct seq_file *m, void *unused)
+{
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(m->private);
+	struct dc_link *link = aconnector->dc_link;
+
+	if (!link)
+		return -ENODEV;
+
+	seq_printf(m, "%d\n", link->dc->debug.max_downscale_src_width);
+
+	return 0;
+}
+
 /*
  * Returns the number of segments used if ODM Combine mode is enabled.
  * Example usage: cat /sys/kernel/debug/dri/0/DP-1/odm_combine_segments
@@ -3041,6 +3059,7 @@ DEFINE_SHOW_ATTRIBUTE(dmub_tracebuffer);
 DEFINE_SHOW_ATTRIBUTE(dp_lttpr_status);
 DEFINE_SHOW_ATTRIBUTE(hdcp_sink_capability);
 DEFINE_SHOW_ATTRIBUTE(internal_display);
+DEFINE_SHOW_ATTRIBUTE(max_downscale_src_width);
 DEFINE_SHOW_ATTRIBUTE(odm_combine_segments);
 DEFINE_SHOW_ATTRIBUTE(replay_capability);
 DEFINE_SHOW_ATTRIBUTE(psr_capability);
@@ -3673,6 +3692,72 @@ DEFINE_DEBUGFS_ATTRIBUTE(ips_residency_cntl_fops, ips_residency_cntl_get,
 
 DEFINE_SHOW_ATTRIBUTE(current_backlight);
 DEFINE_SHOW_ATTRIBUTE(target_backlight);
+
+/*
+ * CACP ACE curve area. Read only.
+ * Returns the area under the ACE PWL curve computed by DMUB firmware on
+ * eDP panels that support CACP. The value is in DMU-internal units (PWL
+ * area); it is only meaningful for relative comparison between CACP
+ * levels on the same panel. Level 0 returns the linear baseline (largest).
+ *
+ * Example usage: cat /sys/kernel/debug/dri/0/eDP-1/cacp_ace_curve_area
+ */
+static int cacp_ace_curve_area_show(struct seq_file *m, void *unused)
+{
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(m->private);
+	struct dc_link *link = aconnector->dc_link;
+	struct amdgpu_device *adev = drm_to_adev(aconnector->base.dev);
+	struct dc *dc;
+	union dmub_rb_cmd cmd;
+	unsigned int panel_inst = 0;
+	bool reallow_idle = false;
+	int ret = 0;
+
+	if (!link || link->type == dc_connection_none)
+		return -ENODEV;
+
+	if (!(link->connector_signal & SIGNAL_TYPE_EDP))
+		return -ENODEV;
+
+	if (!link->panel_config.cacp.cacp_supported)
+		return -EOPNOTSUPP;
+
+	dc = link->ctx->dc;
+
+	if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst))
+		return -EINVAL;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cacp_get_ace_curve_area.header.type = DMUB_CMD__CACP;
+	cmd.cacp_get_ace_curve_area.header.sub_type = DMUB_CMD__CACP_GET_ACE_CURVE_AREA;
+	cmd.cacp_get_ace_curve_area.header.payload_bytes =
+		sizeof(cmd.cacp_get_ace_curve_area) -
+		sizeof(cmd.cacp_get_ace_curve_area.header);
+	cmd.cacp_get_ace_curve_area.data.in.panel_inst = panel_inst;
+
+	mutex_lock(&adev->dm.dc_lock);
+
+	if (dc->idle_optimizations_allowed) {
+		dc_allow_idle_optimizations(dc, false);
+		reallow_idle = true;
+	}
+
+	if (!dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd,
+					  DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY) ||
+	    cmd.cacp_get_ace_curve_area.header.ret_status)
+		ret = -EIO;
+	else
+		seq_printf(m, "%u\n", cmd.cacp_get_ace_curve_area.data.out.area);
+
+	if (reallow_idle)
+		dc_allow_idle_optimizations(dc, true);
+
+	mutex_unlock(&adev->dm.dc_lock);
+
+	return ret;
+}
+DEFINE_SHOW_ATTRIBUTE(cacp_ace_curve_area);
+
 DEFINE_SHOW_ATTRIBUTE(ips_status);
 DEFINE_SHOW_ATTRIBUTE(ips_residency);
 
@@ -3683,6 +3768,7 @@ static const struct {
 		{"force_yuv_pixel_format", &force_yuv_pixel_format_fops},
 		{"trigger_hotplug", &trigger_hotplug_debugfs_fops},
 		{"internal_display", &internal_display_fops},
+		{"max_downscale_src_width", &max_downscale_src_width_fops},
 		{"odm_combine_segments", &odm_combine_segments_fops}
 };
 
@@ -3873,6 +3959,8 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector)
 					&disallow_edp_enter_psr_fops);
 		debugfs_create_file("disallow_edp_enter_replay", 0644, dir, connector,
 					&disallow_edp_enter_replay_fops);
+		debugfs_create_file("cacp_ace_curve_area", 0444, dir, connector,
+				    &cacp_ace_curve_area_fops);
 	}
 
 	for (i = 0; i < ARRAY_SIZE(connector_debugfs_entries); i++) {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c
index 9ff25bc8ee5ae5..f4b0ced7d391c0 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.c
@@ -65,6 +65,31 @@ MODULE_FIRMWARE(FIRMWARE_DCN_42_DMUB);
 MODULE_FIRMWARE(FIRMWARE_DCN_42B_DMUB);
 MODULE_FIRMWARE(FIRMWARE_DCN_60_DMUB);
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+static const struct amdgpu_dm_dmub_kunit_ops amdgpu_dm_dmub_default_ops = {
+	.bo_create_kernel = amdgpu_bo_create_kernel,
+	.ucode_request = amdgpu_ucode_request,
+};
+
+static const struct amdgpu_dm_dmub_kunit_ops *amdgpu_dm_dmub_ops =
+	&amdgpu_dm_dmub_default_ops;
+
+void amdgpu_dm_dmub_kunit_set_ops(const struct amdgpu_dm_dmub_kunit_ops *ops)
+{
+	amdgpu_dm_dmub_ops = ops ? ops : &amdgpu_dm_dmub_default_ops;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_dmub_kunit_set_ops);
+
+#define dmub_bo_create_kernel	amdgpu_dm_dmub_ops->bo_create_kernel
+#define dmub_ucode_request	amdgpu_dm_dmub_ops->ucode_request
+
+#else
+
+#define dmub_bo_create_kernel	amdgpu_bo_create_kernel
+#define dmub_ucode_request	amdgpu_ucode_request
+
+#endif
+
 /**
  * dm_dmub_aux_setconfig_callback - Callback for AUX or SET_CONFIG command.
  * @adev: amdgpu_device pointer
@@ -468,20 +493,22 @@ enum dmub_ips_disable_type dm_get_default_ips_mode(
 }
 EXPORT_IF_KUNIT(dm_get_default_ips_mode);
 
-static uint32_t amdgpu_dm_dmub_reg_read(void *ctx, uint32_t address)
+STATIC_IFN_KUNIT uint32_t amdgpu_dm_dmub_reg_read(void *ctx, uint32_t address)
 {
 	struct amdgpu_device *adev = ctx;
 
 	return dm_read_reg(adev->dm.dc->ctx, address);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_dmub_reg_read);
 
-static void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address,
-				     uint32_t value)
+STATIC_IFN_KUNIT void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address,
+					       uint32_t value)
 {
 	struct amdgpu_device *adev = ctx;
 
 	return dm_write_reg(adev->dm.dc->ctx, address, value);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_dmub_reg_write);
 
 int dm_dmub_sw_init(struct amdgpu_device *adev)
 {
@@ -659,11 +686,11 @@ int dm_dmub_sw_init(struct amdgpu_device *adev)
 	 * Allocate a framebuffer based on the total size of all the regions.
 	 * TODO: Move this into GART.
 	 */
-	r = amdgpu_bo_create_kernel(adev, region_info.fb_size, PAGE_SIZE,
-				    mem_domain,
-				    &adev->dm.dmub_bo,
-				    &adev->dm.dmub_bo_gpu_addr,
-				    &adev->dm.dmub_bo_cpu_addr);
+	r = dmub_bo_create_kernel(adev, region_info.fb_size, PAGE_SIZE,
+				  mem_domain,
+				  &adev->dm.dmub_bo,
+				  &adev->dm.dmub_bo_gpu_addr,
+				  &adev->dm.dmub_bo_cpu_addr);
 	if (r)
 		return r;
 
@@ -766,8 +793,8 @@ int dm_init_microcode(struct amdgpu_device *adev)
 		/* ASIC doesn't support DMUB. */
 		return 0;
 	}
-	r = amdgpu_ucode_request(adev, &adev->dm.dmub_fw, AMDGPU_UCODE_REQUIRED,
-				 "%s", fw_name_dmub);
+	r = dmub_ucode_request(adev, &adev->dm.dmub_fw, AMDGPU_UCODE_REQUIRED,
+			       "%s", fw_name_dmub);
 	return r;
 }
 EXPORT_IF_KUNIT(dm_init_microcode);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h
index 31d5f8c265c0d4..df3e776c1ad14d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_dmub.h
@@ -73,6 +73,21 @@ struct dmub_cmd_fused_request;
 void *dm_dmub_get_vbios_bounding_box(struct amdgpu_device *adev);
 void abort_fused_io(struct dc_context *ctx,
 		    const struct dmub_cmd_fused_request *request);
+uint32_t amdgpu_dm_dmub_reg_read(void *ctx, uint32_t address);
+void amdgpu_dm_dmub_reg_write(void *ctx, uint32_t address, uint32_t value);
+
+/* Signatures are verbatim copies so an upstream change breaks the default ops. */
+struct amdgpu_dm_dmub_kunit_ops {
+	int (*bo_create_kernel)(struct amdgpu_device *adev,
+				unsigned long size, int align,
+				u32 domain, struct amdgpu_bo **bo_ptr,
+				u64 *gpu_addr, void **cpu_addr);
+	__printf(4, 5)
+	int (*ucode_request)(struct amdgpu_device *adev, const struct firmware **fw,
+			     enum amdgpu_ucode_required required, const char *fmt, ...);
+};
+
+void amdgpu_dm_dmub_kunit_set_ops(const struct amdgpu_dm_dmub_kunit_ops *ops);
 #endif
 
 #endif /* AMDGPU_DM_AMDGPU_DM_DMUB_H_ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c
index 7e484adb4120fd..e8ae238bbb6919 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_freesync.c
@@ -37,6 +37,7 @@
 #include "amdgpu_dm_freesync.h"
 #include "dm_helpers.h"
 #include "modules/inc/mod_freesync.h"
+#include "modules/inc/mod_info_packet.h"
 
 bool amdgpu_dm_is_dc_timing_adjust_needed(struct dm_crtc_state *old_state,
 					  struct dm_crtc_state *new_state)
@@ -151,6 +152,14 @@ void amdgpu_dm_get_freesync_config_for_crtc(
 	}
 out:
 	new_crtc_state->freesync_config = config;
+
+	drm_dbg_driver(new_con_state->base.connector->dev,
+		       "VRR: cfg vrr_enabled=%d vrr_supported=%d fs_capable=%d vrefresh=%d min=%d max=%d state=%d\n",
+		       new_crtc_state->base.vrr_enabled,
+		       new_crtc_state->vrr_supported,
+		       new_con_state->freesync_capable, vrefresh,
+		       aconnector->min_vfreq, aconnector->max_vfreq,
+		       config.state);
 }
 EXPORT_IF_KUNIT(amdgpu_dm_get_freesync_config_for_crtc);
 
@@ -229,6 +238,12 @@ void amdgpu_dm_update_freesync_state_on_stream(
 		&vrr_infopacket,
 		pack_sdp_v1_3);
 
+	/* Per HDMI 2.1, VTEM is valid on TMDS as well as FRL */
+	if (new_stream->signal == SIGNAL_TYPE_HDMI_FRL ||
+	    (new_stream->signal == SIGNAL_TYPE_HDMI_TYPE_A &&
+	     aconn && aconn->base.display_info.hdmi.vrr_cap.supported))
+		mod_build_infopacket_vtem(new_stream, &vrr_params, 0, &vrr_infopacket);
+
 	new_crtc_state->freesync_vrr_info_changed |=
 		(memcmp(&new_crtc_state->vrr_infopacket,
 			&vrr_infopacket,
@@ -240,6 +255,35 @@ void amdgpu_dm_update_freesync_state_on_stream(
 	new_stream->vrr_infopacket = vrr_infopacket;
 	new_stream->allow_freesync = mod_freesync_get_freesync_enabled(&vrr_params);
 
+	/*
+	 * HDMI ALLM: when Gaming-VRR is active (VRR_EN=1) and the sink
+	 * advertises ALLM in the SCDS, the Source shall transmit the HF-VSIF
+	 * with ALLM_Mode=1 (HDMI 2.1 Section 7.6.6).
+	 */
+	if (new_stream->signal == SIGNAL_TYPE_HDMI_TYPE_A ||
+	    new_stream->signal == SIGNAL_TYPE_HDMI_FRL) {
+		struct dc_info_packet vsp_infopacket = {0};
+		bool sink_allm = aconn && aconn->base.display_info.hdmi.allm;
+		bool allm = sink_allm &&
+			(vrr_params.state == VRR_STATE_ACTIVE_VARIABLE ||
+			 vrr_params.state == VRR_STATE_ACTIVE_FIXED);
+		bool allm_changed;
+
+		mod_build_hf_vsif_infopacket(new_stream, &vsp_infopacket, allm, allm);
+
+		allm_changed = memcmp(&new_stream->vsp_infopacket, &vsp_infopacket,
+				      sizeof(vsp_infopacket)) != 0;
+		new_crtc_state->freesync_vrr_info_changed |= allm_changed;
+		new_stream->vsp_infopacket = vsp_infopacket;
+
+		if (allm_changed)
+			drm_dbg_driver(adev_to_drm(adev),
+				       "ALLM: flip on crtc=%u: sink_allm=%d vrr_state=%d -> ALLM_Mode=%d\n",
+				    new_crtc_state->base.crtc->base.id,
+				    sink_allm,
+				    vrr_params.state, allm);
+	}
+
 	if (new_crtc_state->freesync_vrr_info_changed)
 		drm_dbg_kms(adev_to_drm(adev), "VRR packet update: crtc=%u enabled=%d state=%d",
 			      new_crtc_state->base.crtc->base.id,
@@ -248,6 +292,7 @@ void amdgpu_dm_update_freesync_state_on_stream(
 
 	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_update_freesync_state_on_stream);
 
 void amdgpu_dm_update_stream_irq_parameters(
 	struct amdgpu_display_manager *dm,
@@ -307,6 +352,7 @@ void amdgpu_dm_update_stream_irq_parameters(
 	acrtc->dm_irq_params.vrr_params = vrr_params;
 	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_update_stream_irq_parameters);
 
 void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm,
 				     struct dm_crtc_state *old_state,
@@ -360,3 +406,4 @@ void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm,
 		}
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_handle_vrr_transition);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index ced8b3d2d762e0..ec312eba80caba 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -131,6 +131,14 @@ STATIC_IFN_KUNIT void apply_edid_quirks(struct dc_link *link, struct edid *edid,
 		drm_dbg_driver(dev, "Disabling VSC on monitor with panel id %X\n", panel_id);
 		edid_caps->panel_patch.disable_colorimetry = true;
 		break;
+	case drm_edid_encode_panel_id('S', 'D', 'C', 0x4203):
+	case drm_edid_encode_panel_id('S', 'D', 'C', 0x4197):
+		if (dc_is_embedded_signal(link->connector_signal)) {
+			drm_dbg_driver(dev, "Force PHY power down/up level 2 on panel id %X\n",
+				       panel_id);
+			link->ctx->dc->debug.psr_phy_force_phy_power_down_up_level_2 = true;
+		}
+		break;
 	/* Workaround for monitors that get corrupted by the PHY SSC reduction */
 	case drm_edid_encode_panel_id('D', 'E', 'L', 0x4147):
 		drm_dbg_driver(dev, "Skip PHY SSC reduction on panel id %X\n", panel_id);
@@ -147,6 +155,24 @@ STATIC_IFN_KUNIT void apply_edid_quirks(struct dc_link *link, struct edid *edid,
 		drm_dbg_driver(dev, "Hiding secondary tile on panel id %X\n", panel_id);
 		edid_caps->panel_patch.disable_second_tile = true;
 		break;
+	/*
+	 * Workaround for the Acer Predator X34GS which reports a FreeSync
+	 * minimum refresh rate that does not operate reliably. Force the
+	 * FreeSync range minimum to 55 Hz.
+	 */
+	case drm_edid_encode_panel_id('A', 'C', 'R', 0x08AF):
+		drm_dbg_driver(dev, "Force FreeSync min to 55 Hz on panel id %X\n", panel_id);
+		edid_caps->panel_patch.force_freesync_min_hz = 55;
+		break;
+	/*
+	 * Workaround for the Lenovo G34w-30 which reports a FreeSync
+	 * minimum refresh rate (48 Hz) that fails to light up over DP.
+	 * Force the FreeSync range minimum to 60 Hz.
+	 */
+	case drm_edid_encode_panel_id('L', 'E', 'N', 0x66F1):
+		drm_dbg_driver(dev, "Force FreeSync min to 60 Hz on panel id %X\n", panel_id);
+		edid_caps->panel_patch.force_freesync_min_hz = 60;
+		break;
 	default:
 		return;
 	}
@@ -1334,6 +1360,8 @@ enum dc_edid_status dm_helpers_read_local_edid(
 
 	return edid_status;
 }
+EXPORT_IF_KUNIT(dm_helpers_read_local_edid);
+
 int dm_helper_dmub_aux_transfer_sync(
 		struct dc_context *ctx,
 		const struct dc_link *link,
@@ -1358,6 +1386,7 @@ int dm_helpers_dmub_set_config_sync(struct dc_context *ctx,
 	return amdgpu_dm_process_dmub_set_config_sync(ctx, link->link_index, payload,
 			operation_result);
 }
+EXPORT_IF_KUNIT(dm_helpers_dmub_set_config_sync);
 
 void dm_set_dcn_clocks(struct dc_context *ctx, struct dc_clocks *clks)
 {
@@ -1426,6 +1455,7 @@ void *dm_helpers_allocate_gpu_mem(
 
 	return dm_allocate_gpu_mem(adev, type, size, addr);
 }
+EXPORT_IF_KUNIT(dm_helpers_allocate_gpu_mem);
 
 void dm_helpers_free_gpu_mem(
 		struct dc_context *ctx,
@@ -1436,6 +1466,7 @@ void dm_helpers_free_gpu_mem(
 
 	dm_free_gpu_mem(adev, type, pvMem);
 }
+EXPORT_IF_KUNIT(dm_helpers_free_gpu_mem);
 
 bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enable)
 {
@@ -1650,6 +1681,7 @@ STATIC_IFN_KUNIT const uint32_t dm_freesync_pcon_whitelist[] = {
 	DP_BRANCH_DEVICE_ID_90CC24,
 	DP_BRANCH_DEVICE_ID_001CF8,
 	DP_BRANCH_DEVICE_ID_001FF2,
+	DP_BRANCH_DEVICE_ID_2B02F0,
 };
 EXPORT_IF_KUNIT(dm_freesync_pcon_whitelist);
 
@@ -1912,3 +1944,4 @@ bool dm_helpers_submit_i2c_over_aux(struct ddc_service *ddc, uint32_t address, u
 	//TODO: Implement this
 	return false;
 }
+EXPORT_IF_KUNIT(dm_helpers_submit_i2c_over_aux);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
index 74a8735168aaa5..c3ed6384320234 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
@@ -619,6 +619,9 @@ STATIC_IFN_KUNIT void amdgpu_dm_irq_schedule_work(struct amdgpu_device *adev,
 
 	DM_IRQ_TABLE_LOCK(adev, irq_table_flags);
 
+	if (!adev->dm.irq_wq)
+		goto out_unlock;
+
 	if (list_empty(handler_list))
 		goto out_unlock;
 
@@ -728,6 +731,11 @@ STATIC_IFN_KUNIT int amdgpu_dm_irq_handler(struct amdgpu_device *adev,
 			entry->src_id,
 			entry->src_data[0]);
 
+	if (!DAL_VALID_IRQ_SRC_NUM(src)) {
+		DRM_ERROR("DM_IRQ: invalid irq_source:%d!\n", src);
+		return 0;
+	}
+
 	amdgpu_dm_irq_ack(adev, src);
 
 	/* Call high irq work immediately */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h
index bc16ecc67329c4..6ba567df5f0abe 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h
@@ -44,6 +44,8 @@ enum dmub_notification_type;
  *
  * This function should be called exactly once - during DM initialization.
  *
+ * @adev: AMDGPU device.
+ *
  * Returns:
  *	0 - success
  *	non-zero - error
@@ -55,6 +57,8 @@ int amdgpu_dm_irq_init(struct amdgpu_device *adev);
  *
  * This function should be called exactly once - during DM destruction.
  *
+ * @adev: AMDGPU device.
+ *
  */
 void amdgpu_dm_irq_fini(struct amdgpu_device *adev);
 
@@ -82,6 +86,7 @@ void *amdgpu_dm_irq_register_interrupt(struct amdgpu_device *adev,
  *	by amdgpu_dm_irq_register_interrupt().
  *
  * @adev: AMD DRM device.
+ * @irq_source: IRQ source to unregister.
  * @ih_index: irq handler index which was returned by
  *	amdgpu_dm_irq_register_interrupt
  */
@@ -126,6 +131,8 @@ void amdgpu_dm_hpd_fini(struct amdgpu_device *adev);
 /**
  * amdgpu_dm_irq_suspend - disable ASIC interrupt during suspend.
  *
+ * @adev: AMDGPU device.
+ *
  */
 void amdgpu_dm_irq_suspend(struct amdgpu_device *adev);
 
@@ -133,6 +140,8 @@ void amdgpu_dm_irq_suspend(struct amdgpu_device *adev);
  * amdgpu_dm_irq_resume_early - enable HPDRX ASIC interrupts during resume.
  * amdgpu_dm_irq_resume - enable ASIC interrupt during resume.
  *
+ * @adev: AMDGPU device.
+ *
  */
 void amdgpu_dm_irq_resume_early(struct amdgpu_device *adev);
 void amdgpu_dm_irq_resume_late(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
index 4e57572e12b60f..127eaba4de60b5 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
@@ -647,9 +647,26 @@ void amdgpu_dm_ism_init(struct amdgpu_dm_ism *ism,
 EXPORT_IF_KUNIT(amdgpu_dm_ism_init);
 
 
-void amdgpu_dm_ism_fini(struct amdgpu_dm_ism *ism)
+/**
+ * amdgpu_dm_ism_flush - Cancel any pending, or wait out in-flight ISM work
+ *
+ * @ism: The CRTC's idle state manager
+ *
+ * Cancels the hysteresis and SSO timers and waits for a running worker to
+ * finish. Callers that are about to drop the CRTC's stream use this so that a
+ * timer armed while the stream was still around cannot allow idle afterwards.
+ *
+ * Must not be called with dc_lock held: the workers take dc_lock themselves,
+ * so waiting for them under it would deadlock.
+ */
+void amdgpu_dm_ism_flush(struct amdgpu_dm_ism *ism)
 {
+	struct amdgpu_crtc *acrtc = ism_to_amdgpu_crtc(ism);
+	struct amdgpu_device *adev = drm_to_adev(acrtc->base.dev);
+
+	lockdep_assert_not_held(&adev->dm.dc_lock);
+
 	cancel_delayed_work_sync(&ism->sso_delayed_work);
 	cancel_delayed_work_sync(&ism->delayed_work);
 }
-EXPORT_IF_KUNIT(amdgpu_dm_ism_fini);
+EXPORT_IF_KUNIT(amdgpu_dm_ism_flush);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h
index afce16f7085a98..893e062bb281ef 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h
@@ -142,7 +142,7 @@ struct amdgpu_dm_ism {
 
 void amdgpu_dm_ism_init(struct amdgpu_dm_ism *ism,
 			struct amdgpu_dm_ism_config *config);
-void amdgpu_dm_ism_fini(struct amdgpu_dm_ism *ism);
+void amdgpu_dm_ism_flush(struct amdgpu_dm_ism *ism);
 void amdgpu_dm_ism_commit_event(struct amdgpu_dm_ism *ism,
 				enum amdgpu_dm_ism_event event);
 void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index 045a7f88b75466..a5b15bef8aa0a8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -168,7 +168,7 @@ STATIC_IFN_KUNIT ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 }
 EXPORT_IF_KUNIT(dm_dp_aux_transfer);
 
-static void
+STATIC_IFN_KUNIT void
 dm_dp_mst_connector_destroy(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *aconnector =
@@ -186,8 +186,9 @@ dm_dp_mst_connector_destroy(struct drm_connector *connector)
 	drm_dp_mst_put_port_malloc(aconnector->mst_output_port);
 	kfree(aconnector);
 }
+EXPORT_IF_KUNIT(dm_dp_mst_connector_destroy);
 
-static int
+STATIC_IFN_KUNIT int
 amdgpu_dm_mst_connector_late_register(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *amdgpu_dm_connector =
@@ -205,6 +206,7 @@ amdgpu_dm_mst_connector_late_register(struct drm_connector *connector)
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_mst_connector_late_register);
 
 
 STATIC_IFN_KUNIT void
@@ -218,7 +220,7 @@ amdgpu_dm_mst_reset_mst_connector_setting(struct amdgpu_dm_connector *aconnector
 }
 EXPORT_IF_KUNIT(amdgpu_dm_mst_reset_mst_connector_setting);
 
-static void
+STATIC_IFN_KUNIT void
 amdgpu_dm_mst_connector_early_unregister(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *aconnector =
@@ -251,6 +253,7 @@ amdgpu_dm_mst_connector_early_unregister(struct drm_connector *connector)
 	aconnector->mst_status = MST_STATUS_DEFAULT;
 	drm_modeset_unlock(&root->mst_mgr.base.lock);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_mst_connector_early_unregister);
 
 static const struct drm_connector_funcs dm_dp_mst_connector_funcs = {
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -275,8 +278,8 @@ bool needs_dsc_aux_workaround(struct dc_link *link)
 }
 EXPORT_IF_KUNIT(needs_dsc_aux_workaround);
 
-#if defined(CONFIG_DRM_AMD_DC_FP)
-static bool is_synaptics_cascaded_panamera(struct dc_link *link, struct drm_dp_mst_port *port)
+#if defined(CONFIG_DRM_AMD_DC_FP) || IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+STATIC_IFN_KUNIT bool is_synaptics_cascaded_panamera(struct dc_link *link, struct drm_dp_mst_port *port)
 {
 	u8 branch_vendor_data[4] = { 0 }; // Vendor data 0x50C ~ 0x50F
 
@@ -290,8 +293,9 @@ static bool is_synaptics_cascaded_panamera(struct dc_link *link, struct drm_dp_m
 
 	return false;
 }
+EXPORT_IF_KUNIT(is_synaptics_cascaded_panamera);
 
-static bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnector)
+STATIC_IFN_KUNIT bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnector)
 {
 	struct dc_sink *dc_sink = aconnector->dc_sink;
 	struct drm_dp_mst_port *port = aconnector->mst_output_port;
@@ -335,6 +339,7 @@ static bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnecto
 
 	return true;
 }
+EXPORT_IF_KUNIT(validate_dsc_caps_on_connector);
 #endif
 
 STATIC_IFN_KUNIT bool retrieve_downstream_port_device(struct amdgpu_dm_connector *aconnector)
@@ -386,7 +391,7 @@ STATIC_IFN_KUNIT bool retrieve_branch_specific_data(struct amdgpu_dm_connector *
 }
 EXPORT_IF_KUNIT(retrieve_branch_specific_data);
 
-static int dm_dp_mst_get_modes(struct drm_connector *connector)
+STATIC_IFN_KUNIT int dm_dp_mst_get_modes(struct drm_connector *connector)
 {
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
 	int ret = 0;
@@ -501,8 +506,8 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 		amdgpu_dm_update_freesync_caps(
 				connector, aconnector->drm_edid, true);
 
-#if defined(CONFIG_DRM_AMD_DC_FP)
-		if (!validate_dsc_caps_on_connector(aconnector))
+#if defined(CONFIG_DRM_AMD_DC_FP) || IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+		if (IS_ENABLED(CONFIG_DRM_AMD_DC_FP) && !validate_dsc_caps_on_connector(aconnector))
 			memset(&aconnector->dc_sink->dsc_caps,
 			       0, sizeof(aconnector->dc_sink->dsc_caps));
 #endif
@@ -518,6 +523,7 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 
 	return ret;
 }
+EXPORT_IF_KUNIT(dm_dp_mst_get_modes);
 
 STATIC_IFN_KUNIT struct drm_encoder *
 dm_mst_atomic_best_encoder(struct drm_connector *connector,
@@ -656,7 +662,7 @@ dm_dp_create_fake_mst_encoders(struct amdgpu_device *adev)
 }
 EXPORT_IF_KUNIT(dm_dp_create_fake_mst_encoders);
 
-static struct drm_connector *
+STATIC_IFN_KUNIT struct drm_connector *
 dm_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 			struct drm_dp_mst_port *port,
 			const char *pathprop)
@@ -738,6 +744,7 @@ dm_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 
 	return connector;
 }
+EXPORT_IF_KUNIT(dm_dp_add_mst_connector);
 
 /*
  * Select the ESI[1] mask used to filter the MST sideband ready bits for a
@@ -933,7 +940,7 @@ struct dsc_mst_fairness_params {
 	struct amdgpu_dm_connector *aconnector;
 };
 
-#if defined(CONFIG_DRM_AMD_DC_FP)
+#if defined(CONFIG_DRM_AMD_DC_FP) || IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
 static uint16_t get_fec_overhead_multiplier(struct dc_link *dc_link)
 {
 	u8 link_coding_cap;
@@ -1224,8 +1231,8 @@ static int try_disable_dsc(struct drm_atomic_commit *state,
 	return 0;
 }
 
-static bool get_conv_frl_bw(struct amdgpu_dm_connector *aconnector,
-							uint32_t *bw_in_kbps, uint32_t *dsc_bw_in_kbps)
+STATIC_IFN_KUNIT bool get_conv_frl_bw(struct amdgpu_dm_connector *aconnector,
+				      uint32_t *bw_in_kbps, uint32_t *dsc_bw_in_kbps)
 {
 	unsigned int max_conv_bw_in_kbps = 0;
 	unsigned int max_sink_bw_in_kbps = 0;
@@ -1251,6 +1258,7 @@ static bool get_conv_frl_bw(struct amdgpu_dm_connector *aconnector,
 
 	return *bw_in_kbps > 0; // Frl endpoint is detected
 }
+EXPORT_IF_KUNIT(get_conv_frl_bw);
 
 static void build_frl_mst_dsc_params(struct amdgpu_dm_connector *aconnector,
 								struct dc_stream_state *stream,
@@ -1295,7 +1303,7 @@ static void build_frl_mst_dsc_params(struct amdgpu_dm_connector *aconnector,
 	}
 }
 
-static void log_dsc_params(int count, struct dsc_mst_fairness_vars *vars, int k)
+STATIC_IFN_KUNIT void log_dsc_params(int count, struct dsc_mst_fairness_vars *vars, int k)
 {
 	int i;
 
@@ -1303,6 +1311,7 @@ static void log_dsc_params(int count, struct dsc_mst_fairness_vars *vars, int k)
 		DRM_DEBUG_DRIVER("MST_DSC DSC params: stream #%d --- dsc_enabled = %d, bpp_x16 = %d, pbn = %d\n",
 				 i, vars[i + k].dsc_enabled, vars[i + k].bpp_x16, vars[i + k].pbn);
 }
+EXPORT_IF_KUNIT(log_dsc_params);
 
 static int compute_mst_dsc_configs_for_link(struct drm_atomic_commit *state,
 					    struct dc_state *dc_state,
@@ -1366,6 +1375,10 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_commit *state,
 		params[count].num_slices_v = aconnector->dsc_settings.dsc_num_slices_v;
 		params[count].bpp_overwrite = aconnector->dsc_settings.dsc_bits_per_pixel;
 		params[count].compression_possible = stream->sink->dsc_caps.dsc_dec_caps.is_dsc_supported;
+		if (params[count].compression_possible &&
+		    dc_link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA &&
+		    dc_link->dpia_bw_alloc_config.bw_alloc_enabled)
+			params[count].clock_force_enable = DSC_CLK_FORCE_ENABLE;
 		dc_dsc_get_policy_for_timing(params[count].timing, 0, &dsc_policy, dc_link_get_highest_encoding_format(stream->link));
 		is_frl_endpoint_present = get_conv_frl_bw(aconnector, &frl_conv_bw_in_kbps, &frl_conv_dsc_bw_in_kbps);
 		if (stream->sink->dsc_caps.dsc_dec_caps.is_dsc_supported &&
@@ -1377,7 +1390,10 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_commit *state,
 				stream->sink->ctx->dc->res_pool->dscs[0],
 				stream->sink->ctx->dc->debug.dsc_min_slice_height_override,
 				dsc_policy.min_target_bpp * 16,
-				dsc_policy.max_target_bpp * 16,
+				(dc_link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA &&
+				 dc_link->dpia_bw_alloc_config.bw_alloc_enabled) ?
+					dsc_policy.min_target_bpp * 16 :
+					dsc_policy.max_target_bpp * 16,
 				&stream->sink->dsc_caps.dsc_dec_caps,
 				&stream->timing,
 				dc_link_get_highest_encoding_format(dc_link),
@@ -1476,7 +1492,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_commit *state,
 	return 0;
 }
 
-static bool is_dsc_need_re_compute(
+STATIC_IFN_KUNIT bool is_dsc_need_re_compute(
 	struct drm_atomic_commit *state,
 	struct dc_state *dc_state,
 	struct dc_link *dc_link)
@@ -1607,6 +1623,7 @@ static bool is_dsc_need_re_compute(
 
 	return is_dsc_need_re_compute;
 }
+EXPORT_IF_KUNIT(is_dsc_need_re_compute);
 
 int compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
 				      struct dc_state *dc_state,
@@ -1677,10 +1694,11 @@ int compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
 
 	return ret;
 }
+EXPORT_IF_KUNIT(compute_mst_dsc_configs_for_state);
 
-static int pre_compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
-						 struct dc_state *dc_state,
-						 struct dsc_mst_fairness_vars *vars)
+STATIC_IFN_KUNIT int pre_compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
+							   struct dc_state *dc_state,
+							   struct dsc_mst_fairness_vars *vars)
 {
 	int i, j;
 	struct dc_stream_state *stream;
@@ -1730,9 +1748,10 @@ static int pre_compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state
 
 	return ret;
 }
+EXPORT_IF_KUNIT(pre_compute_mst_dsc_configs_for_state);
 
-static int find_crtc_index_in_state_by_stream(struct drm_atomic_commit *state,
-					      struct dc_stream_state *stream)
+STATIC_IFN_KUNIT int find_crtc_index_in_state_by_stream(struct drm_atomic_commit *state,
+							struct dc_stream_state *stream)
 {
 	int i;
 	struct drm_crtc *crtc;
@@ -1746,6 +1765,7 @@ static int find_crtc_index_in_state_by_stream(struct drm_atomic_commit *state,
 	}
 	return -1;
 }
+EXPORT_IF_KUNIT(find_crtc_index_in_state_by_stream);
 
 static bool is_link_to_dschub(struct dc_link *dc_link)
 {
@@ -1762,7 +1782,7 @@ static bool is_link_to_dschub(struct dc_link *dc_link)
 	return true;
 }
 
-static bool is_dsc_precompute_needed(struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT bool is_dsc_precompute_needed(struct drm_atomic_commit *state)
 {
 	int i;
 	struct drm_crtc *crtc;
@@ -1782,6 +1802,7 @@ static bool is_dsc_precompute_needed(struct drm_atomic_commit *state)
 	}
 	return ret;
 }
+EXPORT_IF_KUNIT(is_dsc_precompute_needed);
 
 int pre_validate_dsc(struct drm_atomic_commit *state,
 		     struct dm_atomic_state **dm_state_ptr,
@@ -1893,6 +1914,7 @@ int pre_validate_dsc(struct drm_atomic_commit *state,
 
 	return ret;
 }
+EXPORT_IF_KUNIT(pre_validate_dsc);
 
 static uint32_t kbps_from_pbn(unsigned int pbn)
 {
@@ -1924,8 +1946,8 @@ static bool is_dsc_common_config_possible(struct dc_stream_state *stream,
 }
 #endif
 
-#if defined(CONFIG_DRM_AMD_DC_FP)
-static bool dp_get_link_current_set_bw(struct drm_dp_aux *aux, uint32_t *cur_link_bw)
+#if defined(CONFIG_DRM_AMD_DC_FP) || IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+STATIC_IFN_KUNIT bool dp_get_link_current_set_bw(struct drm_dp_aux *aux, uint32_t *cur_link_bw)
 {
 	uint32_t total_data_bw_efficiency_x10000 = 0;
 	uint32_t link_rate_per_lane_kbps = 0;
@@ -1980,13 +2002,14 @@ static bool dp_get_link_current_set_bw(struct drm_dp_aux *aux, uint32_t *cur_lin
 	*cur_link_bw = link_rate_per_lane_kbps * lane_count.bits.LANE_COUNT_SET / 10000 * total_data_bw_efficiency_x10000;
 	return true;
 }
+EXPORT_IF_KUNIT(dp_get_link_current_set_bw);
 #endif
 
 enum dc_status dm_dp_mst_is_port_support_mode(
 	struct amdgpu_dm_connector *aconnector,
 	struct dc_stream_state *stream)
 {
-#if defined(CONFIG_DRM_AMD_DC_FP)
+#if defined(CONFIG_DRM_AMD_DC_FP) || IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
 	int branch_max_throughput_mps = 0;
 	struct dc_link_settings cur_link_settings;
 	uint32_t end_to_end_bw_in_kbps = 0;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
index 2ae9d4b88888ff..21fedc5f21bd81 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
@@ -37,7 +37,7 @@
 
 #define DP_BRANCH_VENDOR_SPECIFIC_START 0x50C
 
-/**
+/*
  * Panamera MST Hub detection
  * Offset DPCD 050Eh == 0x5A indicates cascaded MST hub case
  * Check from beginning of branch device vendor specific field (050Ch)
@@ -65,6 +65,8 @@ struct dc_state;
 struct dc_stream_state;
 struct dm_atomic_state;
 struct drm_atomic_commit;
+struct drm_dp_aux;
+struct drm_dp_mst_port;
 struct drm_dp_mst_topology_mgr;
 
 uint32_t dm_mst_get_pbn_divider(struct dc_link *link);
@@ -103,6 +105,20 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 
 #if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
 void amdgpu_dm_mst_reset_mst_connector_setting(struct amdgpu_dm_connector *aconnector);
+bool is_synaptics_cascaded_panamera(struct dc_link *link, struct drm_dp_mst_port *port);
+bool dp_get_link_current_set_bw(struct drm_dp_aux *aux, uint32_t *cur_link_bw);
+bool get_conv_frl_bw(struct amdgpu_dm_connector *aconnector,
+		     uint32_t *bw_in_kbps, uint32_t *dsc_bw_in_kbps);
+void log_dsc_params(int count, struct dsc_mst_fairness_vars *vars, int k);
+int find_crtc_index_in_state_by_stream(struct drm_atomic_commit *state,
+				       struct dc_stream_state *stream);
+bool is_dsc_precompute_needed(struct drm_atomic_commit *state);
+bool is_dsc_need_re_compute(struct drm_atomic_commit *state, struct dc_state *dc_state,
+			    struct dc_link *dc_link);
+int pre_compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
+					  struct dc_state *dc_state,
+					  struct dsc_mst_fairness_vars *vars);
+bool validate_dsc_caps_on_connector(struct amdgpu_dm_connector *aconnector);
 bool retrieve_downstream_port_device(struct amdgpu_dm_connector *aconnector);
 bool retrieve_branch_specific_data(struct amdgpu_dm_connector *aconnector);
 ssize_t dm_dp_aux_transfer_result(ssize_t result,
@@ -118,6 +134,13 @@ int dm_dp_mst_atomic_check(struct drm_connector *connector,
 				   struct drm_atomic_commit *state);
 int dm_dp_mst_detect(struct drm_connector *connector,
 			     struct drm_modeset_acquire_ctx *ctx, bool force);
+int dm_dp_mst_get_modes(struct drm_connector *connector);
+struct drm_connector *dm_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
+					      struct drm_dp_mst_port *port,
+					      const char *pathprop);
+int amdgpu_dm_mst_connector_late_register(struct drm_connector *connector);
+void amdgpu_dm_mst_connector_early_unregister(struct drm_connector *connector);
+void dm_dp_mst_connector_destroy(struct drm_connector *connector);
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
index e13b96358208c7..eb17d3d7d3c6ba 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
@@ -28,11 +28,13 @@
 #include <drm/drm_blend.h>
 #include "drm/drm_framebuffer.h"
 #include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_fourcc.h>
 
 #include "amdgpu.h"
+#include "dc.h"
 #include "dal_asic_id.h"
 #include "amdgpu_display.h"
 #include "amdgpu_dm_trace.h"
@@ -145,6 +147,7 @@ void amdgpu_dm_plane_fill_blending_from_plane_state(const struct drm_plane_state
 		 * 16-bit, so scale it down to the width the hardware expects.
 		 */
 		if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 0)
+		    || amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(4, 2, 1)
 		    || amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(6, 0, 0))
 			*global_alpha_value = plane_state->alpha >> 4;
 		else
@@ -504,9 +507,9 @@ static void amdgpu_dm_plane_add_gfx10_1_modifiers(const struct amdgpu_device *ad
  * present at specific indices.
  * See SiLib::HwlSetupTileInfo() and CiLib::HwlSetupTileInfo() in addrlib.
  */
-static u32 amdgpu_dm_plane_get_gfx6_tile_idx(const struct amdgpu_device *adev,
-					 const u32 bpp,
-					 const enum array_mode_values arr)
+STATIC_IFN_KUNIT u32 amdgpu_dm_plane_get_gfx6_tile_idx(const struct amdgpu_device *adev,
+						       const u32 bpp,
+						       const enum array_mode_values arr)
 {
 	/* Assume that the microtile mode is DISPLAY. */
 
@@ -527,6 +530,7 @@ static u32 amdgpu_dm_plane_get_gfx6_tile_idx(const struct amdgpu_device *adev,
 		return 12;
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_get_gfx6_tile_idx);
 
 /**
  * amdgpu_dm_plane_calc_gfx7_tile_split() - Calculate tile split on GFX7-8
@@ -542,9 +546,9 @@ static u32 amdgpu_dm_plane_get_gfx6_tile_idx(const struct amdgpu_device *adev,
  * can be calculated. The TILE_SPLIT field is only used for the depth micro tile mode.
  * See CiLib::HwlComputeMacroModeIndex() in addrlib.
  */
-static u32 amdgpu_dm_plane_calc_gfx7_tile_split(const struct amdgpu_device *adev,
-						const u32 bpp,
-						const u32 gb_tile_mode)
+STATIC_IFN_KUNIT u32 amdgpu_dm_plane_calc_gfx7_tile_split(const struct amdgpu_device *adev,
+							  const u32 bpp,
+							  const u32 gb_tile_mode)
 {
 	/* Assume 2D_TILED_THIN1 mode with non-DEPTH microtiles */
 	const u32 sample_split = (gb_tile_mode >> 25) & 0x3;
@@ -557,6 +561,7 @@ static u32 amdgpu_dm_plane_calc_gfx7_tile_split(const struct amdgpu_device *adev
 		     256,
 		     adev->gfx.config.mem_row_size_in_kb * 1024);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_calc_gfx7_tile_split);
 
 /**
  * amdgpu_dm_plane_get_gfx7_macro_tile_idx() - Get macro tile mode index on GFX7-8
@@ -570,7 +575,8 @@ static u32 amdgpu_dm_plane_calc_gfx7_tile_split(const struct amdgpu_device *adev
  * present at specific indices.
  * See CiLib::HwlComputeMacroModeIndex() in addrlib.
  */
-static u32 amdgpu_dm_plane_get_gfx7_macro_tile_idx(const u32 bpp, const u32 tile_split_bytes)
+STATIC_IFN_KUNIT u32 amdgpu_dm_plane_get_gfx7_macro_tile_idx(const u32 bpp,
+							     const u32 tile_split_bytes)
 {
 	const u32 thickness = 1;
 	const u32 tile_size_pixels = 8 * 8;
@@ -582,6 +588,7 @@ static u32 amdgpu_dm_plane_get_gfx7_macro_tile_idx(const u32 bpp, const u32 tile
 
 	return macro_tile_idx;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_get_gfx7_macro_tile_idx);
 
 /**
  * amdgpu_dm_plane_calc_gfx6_mod() - Calculate a DRM format modifier for GFX6-8
@@ -593,9 +600,9 @@ static u32 amdgpu_dm_plane_get_gfx7_macro_tile_idx(const u32 bpp, const u32 tile
  * Select suitable micro and macro tile modes for the given bits per pixel,
  * and calculate the corresponding DRM format modifier.
  */
-static u64 amdgpu_dm_plane_calc_gfx6_mod(const struct amdgpu_device *adev,
-					 const u32 bpp,
-					 const enum array_mode_values arr)
+STATIC_IFN_KUNIT u64 amdgpu_dm_plane_calc_gfx6_mod(const struct amdgpu_device *adev,
+						   const u32 bpp,
+						   const enum array_mode_values arr)
 {
 	u32 array_mode, micro_tile_mode, tile_split_bytes;
 	u32 gb_macrotile_mode, macrotile_idx;
@@ -645,6 +652,7 @@ static u64 amdgpu_dm_plane_calc_gfx6_mod(const struct amdgpu_device *adev,
 		AMD_FMT_MOD_SET(MACRO_TILE_ASPECT, (gb_macrotile_mode >> 4) & 0x3) |
 		AMD_FMT_MOD_SET(NUM_BANKS, (gb_macrotile_mode >> 6) & 0x3);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_calc_gfx6_mod);
 
 /**
  * amdgpu_dm_plane_gfx6_format_mod_supported() - Check if a modifier is supported on GFX6-8
@@ -656,9 +664,9 @@ static u64 amdgpu_dm_plane_calc_gfx6_mod(const struct amdgpu_device *adev,
  * On GFX6-8, not all DRM format modifier can be used with all image formats.
  * Check whether the specified modifier is supported with the given bits per pixel value.
  */
-static bool amdgpu_dm_plane_gfx6_format_mod_supported(const struct amdgpu_device *adev,
-						      const u32 bpp,
-						      const u64 modifier)
+STATIC_IFN_KUNIT bool amdgpu_dm_plane_gfx6_format_mod_supported(const struct amdgpu_device *adev,
+								const u32 bpp,
+								const u64 modifier)
 {
 	const u32 array_mode = AMD_FMT_MOD_GET(TILE, modifier);
 	const u32 micro_tile_mode = AMD_FMT_MOD_GET(MICROTILE, modifier);
@@ -697,6 +705,7 @@ static bool amdgpu_dm_plane_gfx6_format_mod_supported(const struct amdgpu_device
 	/* Verify that the modifier is the same that we'd expose for this bpp */
 	return amdgpu_dm_plane_calc_gfx6_mod(adev, bpp, array_mode) == modifier;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_gfx6_format_mod_supported);
 
 /**
  * amdgpu_dm_plane_add_gfx6_modifiers() - Expose modifiers for GFX6-8
@@ -1217,8 +1226,8 @@ int amdgpu_dm_plane_fill_plane_buffer_attributes(struct amdgpu_device *adev,
 }
 EXPORT_IF_KUNIT(amdgpu_dm_plane_fill_plane_buffer_attributes);
 
-static int amdgpu_dm_plane_helper_prepare_fb(struct drm_plane *plane,
-					     struct drm_plane_state *new_state)
+STATIC_IFN_KUNIT int amdgpu_dm_plane_helper_prepare_fb(struct drm_plane *plane,
+						       struct drm_plane_state *new_state)
 {
 	struct amdgpu_framebuffer *afb;
 	struct drm_gem_object *obj;
@@ -1315,9 +1324,10 @@ static int amdgpu_dm_plane_helper_prepare_fb(struct drm_plane *plane,
 	amdgpu_bo_unreserve(rbo);
 	return r;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_helper_prepare_fb);
 
-static void amdgpu_dm_plane_helper_cleanup_fb(struct drm_plane *plane,
-					      struct drm_plane_state *old_state)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_helper_cleanup_fb(struct drm_plane *plane,
+							struct drm_plane_state *old_state)
 {
 	struct amdgpu_bo *rbo;
 	int r;
@@ -1336,6 +1346,7 @@ static void amdgpu_dm_plane_helper_cleanup_fb(struct drm_plane *plane,
 	amdgpu_bo_unreserve(rbo);
 	amdgpu_bo_unref(&rbo);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_helper_cleanup_fb);
 
 STATIC_IFN_KUNIT void amdgpu_dm_plane_get_min_max_dc_plane_scaling(struct drm_device *dev,
 								   struct drm_framebuffer *fb,
@@ -1730,9 +1741,10 @@ void amdgpu_dm_plane_handle_cursor_update(struct drm_plane *plane,
 		mutex_unlock(&adev->dm.dc_lock);
 	}
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_handle_cursor_update);
 
-static void amdgpu_dm_plane_atomic_async_update(struct drm_plane *plane,
-						struct drm_atomic_commit *state)
+STATIC_IFN_KUNIT void amdgpu_dm_plane_atomic_async_update(struct drm_plane *plane,
+							  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -1754,6 +1766,7 @@ static void amdgpu_dm_plane_atomic_async_update(struct drm_plane *plane,
 
 	amdgpu_dm_plane_handle_cursor_update(plane, old_state);
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_atomic_async_update);
 
 STATIC_IFN_KUNIT void amdgpu_dm_plane_panic_flush(struct drm_plane *plane)
 {
@@ -1788,24 +1801,38 @@ static const struct drm_plane_helper_funcs dm_primary_plane_helper_funcs = {
 	.panic_flush = amdgpu_dm_plane_panic_flush,
 };
 
-STATIC_IFN_KUNIT void amdgpu_dm_plane_drm_plane_reset(struct drm_plane *plane)
+STATIC_IFN_KUNIT struct drm_plane_state *amdgpu_dm_plane_drm_plane_create_state(struct drm_plane *plane)
 {
 	struct dm_plane_state *amdgpu_state;
 
 	amdgpu_state = kzalloc_obj(*amdgpu_state);
 	if (!amdgpu_state)
-		return;
+		return ERR_PTR(-ENOMEM);
+
+	amdgpu_state->flip_addr = kzalloc_obj(*amdgpu_state->flip_addr);
+	amdgpu_state->scaling_info = kzalloc_obj(*amdgpu_state->scaling_info);
+	amdgpu_state->plane_info = kzalloc_obj(*amdgpu_state->plane_info);
+	if (!amdgpu_state->flip_addr || !amdgpu_state->scaling_info ||
+	    !amdgpu_state->plane_info) {
+		kfree(amdgpu_state->flip_addr);
+		kfree(amdgpu_state->scaling_info);
+		kfree(amdgpu_state->plane_info);
+		kfree(amdgpu_state);
+		return ERR_PTR(-ENOMEM);
+	}
 
 	if (plane->state)
 		plane->funcs->atomic_destroy_state(plane, plane->state);
 
-	__drm_atomic_helper_plane_reset(plane, &amdgpu_state->base);
+	__drm_atomic_helper_plane_state_init(&amdgpu_state->base, plane);
 	amdgpu_state->degamma_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT;
 	amdgpu_state->hdr_mult = AMDGPU_HDR_MULT_DEFAULT;
 	amdgpu_state->shaper_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT;
 	amdgpu_state->blend_tf = AMDGPU_TRANSFER_FUNCTION_DEFAULT;
+
+	return &amdgpu_state->base;
 }
-EXPORT_IF_KUNIT(amdgpu_dm_plane_drm_plane_reset);
+EXPORT_IF_KUNIT(amdgpu_dm_plane_drm_plane_create_state);
 
 STATIC_IFN_KUNIT struct drm_plane_state *
 amdgpu_dm_plane_drm_plane_duplicate_state(struct drm_plane *plane)
@@ -1817,6 +1844,21 @@ amdgpu_dm_plane_drm_plane_duplicate_state(struct drm_plane *plane)
 	if (!dm_plane_state)
 		return NULL;
 
+	dm_plane_state->flip_addr = kmemdup(old_dm_plane_state->flip_addr,
+		sizeof(*old_dm_plane_state->flip_addr), GFP_KERNEL);
+	dm_plane_state->scaling_info = kmemdup(old_dm_plane_state->scaling_info,
+		sizeof(*old_dm_plane_state->scaling_info), GFP_KERNEL);
+	dm_plane_state->plane_info = kmemdup(old_dm_plane_state->plane_info,
+		sizeof(*old_dm_plane_state->plane_info), GFP_KERNEL);
+	if (!dm_plane_state->flip_addr || !dm_plane_state->scaling_info ||
+	    !dm_plane_state->plane_info) {
+		kfree(dm_plane_state->flip_addr);
+		kfree(dm_plane_state->scaling_info);
+		kfree(dm_plane_state->plane_info);
+		kfree(dm_plane_state);
+		return NULL;
+	}
+
 	__drm_atomic_helper_plane_duplicate_state(plane, &dm_plane_state->base);
 
 	if (old_dm_plane_state->dc_state) {
@@ -1936,6 +1978,10 @@ STATIC_IFN_KUNIT void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *
 	if (dm_plane_state->blend_lut)
 		drm_property_blob_put(dm_plane_state->blend_lut);
 
+	kfree(dm_plane_state->flip_addr);
+	kfree(dm_plane_state->scaling_info);
+	kfree(dm_plane_state->plane_info);
+
 	if (dm_plane_state->dc_state)
 		dc_plane_state_release(dm_plane_state->dc_state);
 
@@ -2168,7 +2214,7 @@ static const struct drm_plane_funcs dm_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_helper_destroy,
-	.reset = amdgpu_dm_plane_drm_plane_reset,
+	.atomic_create_state = amdgpu_dm_plane_drm_plane_create_state,
 	.atomic_duplicate_state = amdgpu_dm_plane_drm_plane_duplicate_state,
 	.atomic_destroy_state = amdgpu_dm_plane_drm_plane_destroy_state,
 	.format_mod_supported = amdgpu_dm_plane_format_mod_supported,
@@ -2177,6 +2223,7 @@ static const struct drm_plane_funcs dm_plane_funcs = {
 	.atomic_set_property = dm_atomic_plane_set_property,
 	.atomic_get_property = dm_atomic_plane_get_property,
 #endif
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
@@ -2303,12 +2350,20 @@ int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
 		return res;
 #endif
 
-	/* Create (reset) the plane state */
-	if (plane->funcs->reset)
-		plane->funcs->reset(plane);
+	/* Create the plane state */
+	if (plane->funcs->atomic_create_state) {
+		struct drm_plane_state *plane_state;
+
+		plane_state = plane->funcs->atomic_create_state(plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+
+		plane->state = plane_state;
+	}
 
 	return 0;
 }
+EXPORT_IF_KUNIT(amdgpu_dm_plane_init);
 
 bool amdgpu_dm_plane_is_video_format(uint32_t format)
 {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
index 092ade738ce98d..3469a49d5f5636 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.h
@@ -117,7 +117,7 @@ int amdgpu_dm_plane_atomic_async_check(struct drm_plane *plane,
 int amdgpu_dm_plane_atomic_check(struct drm_plane *plane,
 				 struct drm_atomic_commit *state);
 void amdgpu_dm_plane_panic_flush(struct drm_plane *plane);
-void amdgpu_dm_plane_drm_plane_reset(struct drm_plane *plane);
+struct drm_plane_state *amdgpu_dm_plane_drm_plane_create_state(struct drm_plane *plane);
 struct drm_plane_state *amdgpu_dm_plane_drm_plane_duplicate_state(struct drm_plane *plane);
 void amdgpu_dm_plane_drm_plane_destroy_state(struct drm_plane *plane,
 					     struct drm_plane_state *state);
@@ -125,5 +125,17 @@ void amdgpu_dm_plane_add_modifier_dedup(uint64_t **mods, uint64_t *size,
 					uint64_t *cap, uint64_t mod);
 int amdgpu_dm_plane_fill_gfx6_tiling_info_from_modifier(struct dc_tiling_info *tiling_info,
 							uint64_t modifier);
+u32 amdgpu_dm_plane_get_gfx6_tile_idx(const struct amdgpu_device *adev, const u32 bpp,
+				      const enum array_mode_values arr);
+u32 amdgpu_dm_plane_calc_gfx7_tile_split(const struct amdgpu_device *adev, const u32 bpp,
+					 const u32 gb_tile_mode);
+u32 amdgpu_dm_plane_get_gfx7_macro_tile_idx(const u32 bpp, const u32 tile_split_bytes);
+u64 amdgpu_dm_plane_calc_gfx6_mod(const struct amdgpu_device *adev, const u32 bpp,
+				  const enum array_mode_values arr);
+bool amdgpu_dm_plane_gfx6_format_mod_supported(const struct amdgpu_device *adev, const u32 bpp,
+					       const u64 modifier);
+int amdgpu_dm_plane_helper_prepare_fb(struct drm_plane *plane, struct drm_plane_state *new_state);
+void amdgpu_dm_plane_helper_cleanup_fb(struct drm_plane *plane, struct drm_plane_state *old_state);
+void amdgpu_dm_plane_atomic_async_update(struct drm_plane *plane, struct drm_atomic_commit *state);
 #endif
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index 6dbc8d1b4ecd3d..23f1db81a15447 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -38,6 +38,31 @@
 #include "amdgpu_dm_trace.h"
 #include "dm_helpers.h"
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+static const struct amdgpu_dm_services_kunit_ops amdgpu_dm_services_default_ops = {
+	.bo_create_kernel = amdgpu_bo_create_kernel,
+	.bo_free_kernel = amdgpu_bo_free_kernel,
+};
+
+static const struct amdgpu_dm_services_kunit_ops *amdgpu_dm_services_ops =
+	&amdgpu_dm_services_default_ops;
+
+void amdgpu_dm_services_kunit_set_ops(const struct amdgpu_dm_services_kunit_ops *ops)
+{
+	amdgpu_dm_services_ops = ops ? ops : &amdgpu_dm_services_default_ops;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_services_kunit_set_ops);
+
+#define services_bo_create_kernel	amdgpu_dm_services_ops->bo_create_kernel
+#define services_bo_free_kernel		amdgpu_dm_services_ops->bo_free_kernel
+
+#else
+
+#define services_bo_create_kernel	amdgpu_bo_create_kernel
+#define services_bo_free_kernel		amdgpu_bo_free_kernel
+
+#endif
+
 	unsigned long long
 	dm_get_elapse_time_in_ns(struct dc_context *ctx,
 				 unsigned long long current_time_stamp,
@@ -113,9 +138,9 @@ dm_allocate_gpu_mem(
 	if (!da)
 		return NULL;
 
-	ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE,
-				      domain, &da->bo,
-				      &da->gpu_addr, &da->cpu_ptr);
+	ret = services_bo_create_kernel(adev, size, PAGE_SIZE,
+					domain, &da->bo,
+					&da->gpu_addr, &da->cpu_ptr);
 
 	*addr = da->gpu_addr;
 
@@ -129,6 +154,7 @@ dm_allocate_gpu_mem(
 
 	return da->cpu_ptr;
 }
+EXPORT_IF_KUNIT(dm_allocate_gpu_mem);
 
 void
 dm_free_gpu_mem(
@@ -141,7 +167,7 @@ dm_free_gpu_mem(
 	/* walk the da list in DM */
 	list_for_each_entry(da, &adev->dm.da_list, list) {
 		if (pvMem == da->cpu_ptr) {
-			amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr);
+			services_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr);
 			list_del(&da->list);
 			kfree(da);
 			break;
@@ -149,3 +175,4 @@ dm_free_gpu_mem(
 	}
 
 }
+EXPORT_IF_KUNIT(dm_free_gpu_mem);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c
index a7594012a0b2e4..6f91a992d43f8e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.c
@@ -41,6 +41,52 @@ static const u32 amdgpu_dm_wb_formats[] = {
 	DRM_FORMAT_XRGB2101010,
 };
 
+#if IS_ENABLED(CONFIG_DRM_AMD_DC_KUNIT_TEST)
+static const struct amdgpu_dm_wb_kunit_ops amdgpu_dm_wb_default_ops = {
+	.reserve = amdgpu_bo_reserve,
+	.reserve_fences = dma_resv_reserve_fences,
+	.pin = amdgpu_bo_pin,
+	.alloc_gart = amdgpu_ttm_alloc_gart,
+	.unreserve = amdgpu_bo_unreserve,
+	.gpu_offset = amdgpu_bo_gpu_offset,
+	.ref = amdgpu_bo_ref,
+	.unpin = amdgpu_bo_unpin,
+	.unref = amdgpu_bo_unref,
+};
+
+static const struct amdgpu_dm_wb_kunit_ops *amdgpu_dm_wb_ops =
+	&amdgpu_dm_wb_default_ops;
+
+void amdgpu_dm_wb_kunit_set_ops(const struct amdgpu_dm_wb_kunit_ops *ops)
+{
+	amdgpu_dm_wb_ops = ops ? ops : &amdgpu_dm_wb_default_ops;
+}
+EXPORT_IF_KUNIT(amdgpu_dm_wb_kunit_set_ops);
+
+#define wb_bo_reserve			amdgpu_dm_wb_ops->reserve
+#define wb_dma_resv_reserve_fences	amdgpu_dm_wb_ops->reserve_fences
+#define wb_bo_pin			amdgpu_dm_wb_ops->pin
+#define wb_ttm_alloc_gart		amdgpu_dm_wb_ops->alloc_gart
+#define wb_bo_unreserve			amdgpu_dm_wb_ops->unreserve
+#define wb_bo_gpu_offset		amdgpu_dm_wb_ops->gpu_offset
+#define wb_bo_ref			amdgpu_dm_wb_ops->ref
+#define wb_bo_unpin			amdgpu_dm_wb_ops->unpin
+#define wb_bo_unref			amdgpu_dm_wb_ops->unref
+
+#else
+
+#define wb_bo_reserve			amdgpu_bo_reserve
+#define wb_dma_resv_reserve_fences	dma_resv_reserve_fences
+#define wb_bo_pin			amdgpu_bo_pin
+#define wb_ttm_alloc_gart		amdgpu_ttm_alloc_gart
+#define wb_bo_unreserve			amdgpu_bo_unreserve
+#define wb_bo_gpu_offset		amdgpu_bo_gpu_offset
+#define wb_bo_ref			amdgpu_bo_ref
+#define wb_bo_unpin			amdgpu_bo_unpin
+#define wb_bo_unref			amdgpu_bo_unref
+
+#endif
+
 STATIC_IFN_KUNIT int amdgpu_dm_wb_encoder_atomic_check(struct drm_encoder *encoder,
 					struct drm_crtc_state *crtc_state,
 					struct drm_connector_state *conn_state)
@@ -105,13 +151,13 @@ STATIC_IFN_KUNIT int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb
 	rbo = gem_to_amdgpu_bo(obj);
 	adev = amdgpu_ttm_adev(rbo->tbo.bdev);
 
-	r = amdgpu_bo_reserve(rbo, true);
+	r = wb_bo_reserve(rbo, true);
 	if (r) {
 		drm_err(adev_to_drm(adev), "fail to reserve bo: %pe\n", ERR_PTR(r));
 		return r;
 	}
 
-	r = dma_resv_reserve_fences(rbo->tbo.base.resv, TTM_NUM_MOVE_FENCES);
+	r = wb_dma_resv_reserve_fences(rbo->tbo.base.resv, TTM_NUM_MOVE_FENCES);
 	if (r) {
 		drm_err(adev_to_drm(adev), "reserving fence slot failed: %pe\n", ERR_PTR(r));
 		goto error_unlock;
@@ -120,32 +166,32 @@ STATIC_IFN_KUNIT int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb
 	domain = amdgpu_display_supported_domains(adev, rbo->flags);
 
 	rbo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
-	r = amdgpu_bo_pin(rbo, domain);
+	r = wb_bo_pin(rbo, domain);
 	if (unlikely(r != 0)) {
 		if (r != -ERESTARTSYS)
 			DRM_ERROR("Failed to pin framebuffer: %pe\n", ERR_PTR(r));
 		goto error_unlock;
 	}
 
-	r = amdgpu_ttm_alloc_gart(&rbo->tbo);
+	r = wb_ttm_alloc_gart(&rbo->tbo);
 	if (unlikely(r != 0)) {
 		DRM_ERROR("%p bind failed: %pe\n", rbo, ERR_PTR(r));
 		goto error_unpin;
 	}
 
-	amdgpu_bo_unreserve(rbo);
+	wb_bo_unreserve(rbo);
 
-	afb->address = amdgpu_bo_gpu_offset(rbo);
+	afb->address = wb_bo_gpu_offset(rbo);
 
-	amdgpu_bo_ref(rbo);
+	wb_bo_ref(rbo);
 
 	return 0;
 
 error_unpin:
-	amdgpu_bo_unpin(rbo);
+	wb_bo_unpin(rbo);
 
 error_unlock:
-	amdgpu_bo_unreserve(rbo);
+	wb_bo_unreserve(rbo);
 	return r;
 }
 EXPORT_IF_KUNIT(amdgpu_dm_wb_prepare_job);
@@ -160,15 +206,15 @@ STATIC_IFN_KUNIT void amdgpu_dm_wb_cleanup_job(struct drm_writeback_connector *c
 		return;
 
 	rbo = gem_to_amdgpu_bo(job->fb->obj[0]);
-	r = amdgpu_bo_reserve(rbo, false);
+	r = wb_bo_reserve(rbo, false);
 	if (unlikely(r)) {
 		DRM_ERROR("failed to reserve rbo before unpin: %pe\n", ERR_PTR(r));
 		return;
 	}
 
-	amdgpu_bo_unpin(rbo);
-	amdgpu_bo_unreserve(rbo);
-	amdgpu_bo_unref(&rbo);
+	wb_bo_unpin(rbo);
+	wb_bo_unreserve(rbo);
+	wb_bo_unref(&rbo);
 }
 EXPORT_IF_KUNIT(amdgpu_dm_wb_cleanup_job);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h
index 5fd616bc43b5c0..8d39a37e80f0a3 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_wb.h
@@ -31,6 +31,9 @@
 
 struct amdgpu_display_manager;
 struct amdgpu_dm_wb_connector;
+struct amdgpu_bo;
+struct dma_resv;
+struct ttm_buffer_object;
 
 int amdgpu_dm_wb_connector_init(struct amdgpu_display_manager *dm,
 				struct amdgpu_dm_wb_connector *dm_wbcon,
@@ -40,6 +43,18 @@ int amdgpu_dm_wb_connector_init(struct amdgpu_display_manager *dm,
 #include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
 
+struct amdgpu_dm_wb_kunit_ops {
+	int (*reserve)(struct amdgpu_bo *bo, bool interruptible);
+	int (*reserve_fences)(struct dma_resv *resv, unsigned int num_fences);
+	int (*pin)(struct amdgpu_bo *bo, u32 domain);
+	int (*alloc_gart)(struct ttm_buffer_object *tbo);
+	void (*unreserve)(struct amdgpu_bo *bo);
+	u64 (*gpu_offset)(struct amdgpu_bo *bo);
+	struct amdgpu_bo *(*ref)(struct amdgpu_bo *bo);
+	void (*unpin)(struct amdgpu_bo *bo);
+	void (*unref)(struct amdgpu_bo **bo);
+};
+
 int amdgpu_dm_wb_encoder_atomic_check(struct drm_encoder *encoder,
 				      struct drm_crtc_state *crtc_state,
 				      struct drm_connector_state *conn_state);
@@ -48,6 +63,7 @@ int amdgpu_dm_wb_prepare_job(struct drm_writeback_connector *wb_connector,
 			     struct drm_writeback_job *job);
 void amdgpu_dm_wb_cleanup_job(struct drm_writeback_connector *connector,
 			      struct drm_writeback_job *job);
+void amdgpu_dm_wb_kunit_set_ops(const struct amdgpu_dm_wb_kunit_ops *ops);
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig
index c7c8527dbb1064..0c26c510f7a52d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/.kunitconfig
@@ -6,7 +6,6 @@ CONFIG_DRM_AMD_DC=y
 CONFIG_DEBUG_FS=y
 CONFIG_DRM_AMD_DC_KUNIT_TEST=y
 CONFIG_FW_LOADER=y
-CONFIG_DRM_KUNIT_TEST=y
 CONFIG_DRM_KUNIT_TEST_HELPERS=y
 CONFIG_DRM_KMS_HELPER=y
 CONFIG_DRM_TTM=y
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile
index 616881fcb9afae..e0549c04c62e92 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/Makefile
@@ -1,6 +1,8 @@
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-License-Identifier: GPL-2.0 OR MIT
 #
 # Makefile for amdgpu_dm KUnit tests.
+#
+# Copyright 2026 Advanced Micro Devices, Inc.
 
 ccflags-y += -I$(src)/..
 ccflags-y += -I$(src)/../..
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c
index 7ca17f803f9d78..5221050dec2f15 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_backlight_test.c
@@ -747,8 +747,6 @@ static void dm_test_backlight_caps_valid_short_circuit(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, caps->max_input_signal, 199);
 }
 
-#if !defined(CONFIG_ACPI)
-
 /**
  * dm_test_backlight_caps_aux_support_noop - Test Backlight caps aux support noop
  * @test: The KUnit test context
@@ -758,6 +756,9 @@ static void dm_test_backlight_caps_aux_support_noop(struct kunit *test)
 	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
 	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[0];
 
+	if (IS_ENABLED(CONFIG_ACPI))
+		kunit_skip(test, "caps are queried from ACPI firmware in this build");
+
 	caps->caps_valid = false;
 	caps->aux_support = true;
 	caps->min_input_signal = 11;
@@ -770,27 +771,47 @@ static void dm_test_backlight_caps_aux_support_noop(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, caps->max_input_signal, 222);
 }
 
+/* Tests for amdgpu_dm_validate_backlight_caps() */
+
+struct dm_validate_caps_param {
+	const char *name;
+	int min_input_signal;
+	int max_input_signal;
+	bool caps_valid;
+};
+
+static const struct dm_validate_caps_param dm_validate_caps_params[] = {
+	{ "sane", AMDGPU_DM_DEFAULT_MIN_BACKLIGHT, AMDGPU_DM_DEFAULT_MAX_BACKLIGHT, true },
+	{ "max_too_large", AMDGPU_DM_DEFAULT_MIN_BACKLIGHT,
+	  AMDGPU_DM_DEFAULT_MAX_BACKLIGHT + 1, false },
+	{ "negative_min", -1, AMDGPU_DM_DEFAULT_MAX_BACKLIGHT, false },
+	{ "narrow_spread", 100, 100 + AMDGPU_DM_MIN_SPREAD - 1, false },
+};
+
+KUNIT_ARRAY_PARAM_DESC(dm_validate_caps, dm_validate_caps_params, name);
+
 /**
- * dm_test_backlight_caps_non_aux_sets_defaults - Test Backlight caps non aux sets defaults
+ * dm_test_validate_backlight_caps - Test the firmware caps sanity check
  * @test: The KUnit test context
+ *
+ * Caps above the 8-bit range, with a negative min or with a spread narrower
+ * than AMDGPU_DM_MIN_SPREAD must be rejected.
  */
-static void dm_test_backlight_caps_non_aux_sets_defaults(struct kunit *test)
+static void dm_test_validate_backlight_caps(struct kunit *test)
 {
+	const struct dm_validate_caps_param *param = test->param_value;
 	struct amdgpu_display_manager *dm = dm_kunit_alloc_dm(test);
 	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[0];
 
-	caps->caps_valid = false;
-	caps->aux_support = false;
-	caps->min_input_signal = 0;
-	caps->max_input_signal = 0;
+	dm->adev = dm_kunit_alloc_adev(test);
+	caps->caps_valid = true;
+	caps->min_input_signal = param->min_input_signal;
+	caps->max_input_signal = param->max_input_signal;
 
-	amdgpu_dm_update_backlight_caps(dm, 0);
+	amdgpu_dm_validate_backlight_caps(dm, 0);
 
-	KUNIT_EXPECT_TRUE(test, caps->caps_valid);
-	KUNIT_EXPECT_EQ(test, caps->min_input_signal, AMDGPU_DM_DEFAULT_MIN_BACKLIGHT);
-	KUNIT_EXPECT_EQ(test, caps->max_input_signal, AMDGPU_DM_DEFAULT_MAX_BACKLIGHT);
+	KUNIT_EXPECT_EQ(test, caps->caps_valid, param->caps_valid);
 }
-#endif
 
 /* Tests for get_brightness_range() */
 
@@ -1923,11 +1944,11 @@ static struct kunit_case dm_backlight_test_cases[] = {
 	/* amdgpu_dm_backlight_get_device_index */
 	KUNIT_CASE(dm_test_backlight_device_index_matches_second),
 	KUNIT_CASE(dm_test_backlight_device_index_missing_fallback),
+	/* amdgpu_dm_update_backlight_caps */
 	KUNIT_CASE(dm_test_backlight_caps_valid_short_circuit),
-#if !defined(CONFIG_ACPI)
 	KUNIT_CASE(dm_test_backlight_caps_aux_support_noop),
-	KUNIT_CASE(dm_test_backlight_caps_non_aux_sets_defaults),
-#endif
+	/* amdgpu_dm_validate_backlight_caps */
+	KUNIT_CASE_PARAM(dm_test_validate_backlight_caps, dm_validate_caps_gen_params),
 	/* get_brightness_range */
 	KUNIT_CASE(dm_test_brightness_range_null_caps),
 	KUNIT_CASE(dm_test_brightness_range_pwm),
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c
index 2e557ff66818bc..90cc5592ef5de3 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_colorop_test.c
@@ -136,6 +136,30 @@ static void kunit_colorop_pipeline_destroy(void *drm)
 	drm_colorop_pipeline_destroy((struct drm_device *)drm);
 }
 
+/*
+ * A plain drm_device (not an amdgpu_device) is enough: building the pipeline
+ * only needs the DRM mode-config infrastructure. The pipeline is destroyed
+ * before the DRM device so the colorops are freed while the device is valid.
+ */
+static struct drm_plane *dm_colorop_alloc_plane(struct kunit *test, struct drm_device **drm)
+{
+	struct drm_plane *plane;
+	struct device *dev;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	*drm = __drm_kunit_helper_alloc_drm_device(test, dev, sizeof(**drm), 0, 0);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, *drm);
+
+	plane = drm_kunit_helper_create_primary_plane(test, *drm, NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane);
+
+	kunit_add_action(test, kunit_colorop_pipeline_destroy, *drm);
+
+	return plane;
+}
+
 static void dm_expect_colorop_pipeline(struct kunit *test, struct drm_device *drm,
 				       const struct drm_prop_enum_list *list,
 				       const enum drm_colorop_type *expected,
@@ -177,34 +201,12 @@ static void dm_test_initialize_default_pipeline(struct kunit *test)
 		DRM_COLOROP_1D_CURVE,	/* blnd TF */
 		DRM_COLOROP_1D_LUT,	/* blnd LUT */
 	};
-	struct device *dev;
 	struct drm_device *drm;
 	struct drm_plane *plane;
 	struct drm_prop_enum_list list = {};
 	int ret;
 
-	dev = drm_kunit_helper_alloc_device(test);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
-
-	/*
-	 * Allocate a plain drm_device (not an amdgpu_device) — sufficient
-	 * because amdgpu_dm_build_default_pipeline() only needs the DRM
-	 * mode-config infrastructure, not the amdgpu device wrapper.
-	 */
-	drm = __drm_kunit_helper_alloc_drm_device(test, dev,
-						   sizeof(*drm), 0, 0);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
-
-	plane = drm_kunit_helper_create_primary_plane(test, drm,
-						       NULL, NULL, NULL, 0, NULL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane);
-
-	/*
-	 * Destroy the pipeline before the DRM device is cleaned up so
-	 * that the colorop objects (kzalloc'd inside the function) are
-	 * freed while the device is still valid.
-	 */
-	kunit_add_action(test, kunit_colorop_pipeline_destroy, drm);
+	plane = dm_colorop_alloc_plane(test, &drm);
 
 	ret = amdgpu_dm_build_default_pipeline(drm, plane, true, &list);
 	KUNIT_ASSERT_EQ(test, ret, 0);
@@ -308,6 +310,187 @@ static void dm_test_initialize_default_pipeline_no_3d_lut(struct kunit *test)
 						 expected, ARRAY_SIZE(expected));
 }
 
+/* Failure paths of amdgpu_dm_build_default_pipeline() */
+
+/*
+ * The DRM colorop constructors cannot be made to fail through their arguments,
+ * so failures are injected with a fake allocator and with spies that delegate
+ * to the real constructors and override the return value of one call.
+ */
+struct dm_colorop_fault {
+	int alloc_fail_at;	/* allocation index returning NULL, -1 to disable */
+	int init_fail_at;	/* init call index returning an error, -1 to disable */
+	int allocs;
+	int inits;
+};
+
+static struct dm_colorop_fault dm_colorop_fault;
+
+static struct drm_colorop *dm_colorop_test_alloc(void)
+{
+	if (dm_colorop_fault.allocs++ == dm_colorop_fault.alloc_fail_at)
+		return NULL;
+
+	return kzalloc_obj(struct drm_colorop);
+}
+
+static int dm_colorop_test_init_ret(int ret)
+{
+	if (ret)
+		return ret;
+
+	if (dm_colorop_fault.inits++ == dm_colorop_fault.init_fail_at)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int dm_colorop_test_curve_1d_init(struct drm_device *dev, struct drm_colorop *colorop,
+					 struct drm_plane *plane,
+					 const struct drm_colorop_funcs *funcs,
+					 u64 supported_tfs, uint32_t flags)
+{
+	return dm_colorop_test_init_ret(drm_plane_colorop_curve_1d_init(dev, colorop, plane,
+									funcs, supported_tfs,
+									flags));
+}
+
+static int dm_colorop_test_1d_lut_init(struct drm_device *dev, struct drm_colorop *colorop,
+				       struct drm_plane *plane,
+				       const struct drm_colorop_funcs *funcs,
+				       uint32_t lut_size,
+				       enum drm_colorop_lut1d_interpolation_type interpolation,
+				       uint32_t flags)
+{
+	return dm_colorop_test_init_ret(drm_plane_colorop_curve_1d_lut_init(dev, colorop, plane,
+									    funcs, lut_size,
+									    interpolation,
+									    flags));
+}
+
+static int dm_colorop_test_ctm_3x4_init(struct drm_device *dev, struct drm_colorop *colorop,
+					struct drm_plane *plane,
+					const struct drm_colorop_funcs *funcs, uint32_t flags)
+{
+	return dm_colorop_test_init_ret(drm_plane_colorop_ctm_3x4_init(dev, colorop, plane,
+								       funcs, flags));
+}
+
+static int dm_colorop_test_mult_init(struct drm_device *dev, struct drm_colorop *colorop,
+				     struct drm_plane *plane,
+				     const struct drm_colorop_funcs *funcs, uint32_t flags)
+{
+	return dm_colorop_test_init_ret(drm_plane_colorop_mult_init(dev, colorop, plane,
+								    funcs, flags));
+}
+
+static int dm_colorop_test_3dlut_init(struct drm_device *dev, struct drm_colorop *colorop,
+				      struct drm_plane *plane,
+				      const struct drm_colorop_funcs *funcs, uint32_t lut_size,
+				      enum drm_colorop_lut3d_interpolation_type interpolation,
+				      uint32_t flags)
+{
+	return dm_colorop_test_init_ret(drm_plane_colorop_3dlut_init(dev, colorop, plane, funcs,
+								     lut_size, interpolation,
+								     flags));
+}
+
+static const struct amdgpu_dm_colorop_kunit_ops dm_colorop_test_ops = {
+	.colorop_kzalloc_obj = dm_colorop_test_alloc,
+	.curve_1d_init = dm_colorop_test_curve_1d_init,
+	.curve_1d_lut_init = dm_colorop_test_1d_lut_init,
+	.ctm_3x4_init = dm_colorop_test_ctm_3x4_init,
+	.mult_init = dm_colorop_test_mult_init,
+	.lut3d_init = dm_colorop_test_3dlut_init,
+};
+
+static void dm_colorop_test_reset_ops(void *unused)
+{
+	amdgpu_dm_colorop_kunit_set_ops(NULL);
+}
+
+/*
+ * Build the full (3D LUT) pipeline with one injected failure and return the
+ * error reported by amdgpu_dm_build_default_pipeline().
+ */
+static int dm_colorop_build_with_fault(struct kunit *test, struct drm_device **drm,
+				       int alloc_fail_at, int init_fail_at)
+{
+	struct drm_prop_enum_list list = {};
+	struct drm_plane *plane;
+	int ret;
+
+	plane = dm_colorop_alloc_plane(test, drm);
+
+	dm_colorop_fault = (struct dm_colorop_fault){
+		.alloc_fail_at = alloc_fail_at,
+		.init_fail_at = init_fail_at,
+	};
+
+	amdgpu_dm_colorop_kunit_set_ops(&dm_colorop_test_ops);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_colorop_test_reset_ops, NULL), 0);
+
+	ret = amdgpu_dm_build_default_pipeline(*drm, plane, true, &list);
+	kfree(list.name);
+
+	return ret;
+}
+
+struct dm_colorop_step_param {
+	const char *name;
+	int index;
+};
+
+/* Pipeline steps in creation order when the 3D LUT block is built. */
+static const struct dm_colorop_step_param dm_colorop_step_params[] = {
+	{ "degam_tf",	0 },
+	{ "multiplier",	1 },
+	{ "ctm_3x4",	2 },
+	{ "shaper_tf",	3 },
+	{ "shaper_lut",	4 },
+	{ "lut3d",	5 },
+	{ "blnd_tf",	6 },
+	{ "blnd_lut",	7 },
+};
+
+KUNIT_ARRAY_PARAM_DESC(dm_colorop_step, dm_colorop_step_params, name);
+
+/**
+ * dm_test_build_pipeline_alloc_fail() - Verify a failed alloc returns -ENOMEM and cleans up.
+ * @test: KUnit test context.
+ */
+static void dm_test_build_pipeline_alloc_fail(struct kunit *test)
+{
+	const struct dm_colorop_step_param *param = test->param_value;
+	struct drm_device *drm;
+	int ret;
+
+	ret = dm_colorop_build_with_fault(test, &drm, param->index, -1);
+
+	KUNIT_EXPECT_EQ(test, ret, -ENOMEM);
+	KUNIT_EXPECT_EQ(test, dm_colorop_fault.allocs, param->index + 1);
+	KUNIT_EXPECT_EQ(test, dm_colorop_fault.inits, param->index);
+	KUNIT_EXPECT_EQ(test, drm->mode_config.num_colorop, 0);
+}
+
+/**
+ * dm_test_build_pipeline_init_fail() - Verify a failed init propagates the error and cleans up.
+ * @test: KUnit test context.
+ */
+static void dm_test_build_pipeline_init_fail(struct kunit *test)
+{
+	const struct dm_colorop_step_param *param = test->param_value;
+	struct drm_device *drm;
+	int ret;
+
+	ret = dm_colorop_build_with_fault(test, &drm, -1, param->index);
+
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+	KUNIT_EXPECT_EQ(test, dm_colorop_fault.allocs, param->index + 1);
+	KUNIT_EXPECT_EQ(test, dm_colorop_fault.inits, param->index + 1);
+	KUNIT_EXPECT_EQ(test, drm->mode_config.num_colorop, 0);
+}
+
 static struct kunit_case dm_colorop_test_cases[] = {
 	/* degam TFs */
 	KUNIT_CASE(dm_test_supported_degam_tfs_has_srgb_eotf),
@@ -334,6 +517,9 @@ static struct kunit_case dm_colorop_test_cases[] = {
 	KUNIT_CASE(dm_test_initialize_default_pipeline_dpp_3d_lut),
 	KUNIT_CASE(dm_test_initialize_default_pipeline_mpc_preblend),
 	KUNIT_CASE(dm_test_initialize_default_pipeline_no_3d_lut),
+	/* amdgpu_dm_build_default_pipeline failure paths */
+	KUNIT_CASE_PARAM(dm_test_build_pipeline_alloc_fail, dm_colorop_step_gen_params),
+	KUNIT_CASE_PARAM(dm_test_build_pipeline_init_fail, dm_colorop_step_gen_params),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c
index 212a7536e65b7f..e049d1a32979b7 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c
@@ -16,6 +16,7 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_mode_object.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_property.h>
 #include <linux/hdmi.h>
 #include <linux/i2c.h>
@@ -27,8 +28,13 @@
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_connector.h"
 #include "amdgpu_dm_backlight.h"
+#include "amdgpu_dm_hdcp.h"
+#include "amdgpu_dm_psr.h"
 #include "include/grph_object_id.h"
 #include "amdgpu_dm_kunit_test_helpers.h"
+#include "inc/link_service.h"
+#include "inc/resource.h"
+#include "inc/hw/timing_generator.h"
 
 /* Tests for get_subconnector_type() */
 
@@ -1686,6 +1692,124 @@ static void dm_test_fill_hdr_zeroes_output(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)out.hb3, 0);
 }
 
+/*
+ * Build a minimal connector state carrying a valid static HDR metadata blob so
+ * the packing path can be exercised without a full atomic state. connector_type
+ * selects the sink-specific header layout.
+ */
+struct dm_test_hdr_ctx {
+	struct drm_connector conn;
+	struct drm_connector_state state;
+	struct drm_property_blob blob;
+	struct hdr_output_metadata meta;
+};
+
+static struct dm_test_hdr_ctx *
+dm_test_hdr_ctx_alloc(struct kunit *test, int connector_type)
+{
+	struct dm_test_hdr_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->conn.connector_type = connector_type;
+	ctx->state.connector = &ctx->conn;
+	ctx->blob.data = &ctx->meta;
+	ctx->blob.length = sizeof(ctx->meta);
+	ctx->state.hdr_output_metadata = &ctx->blob;
+
+	return ctx;
+}
+
+/**
+ * dm_test_fill_hdr_hdmi - Test the HDMI infopacket header layout
+ * @test: The KUnit test context
+ *
+ * A valid metadata blob on an HDMI connector packs a type 0x87 infoframe and
+ * marks the packet valid.
+ */
+static void dm_test_fill_hdr_hdmi(struct kunit *test)
+{
+	struct dm_test_hdr_ctx *ctx =
+		dm_test_hdr_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_info_packet out;
+
+	memset(&out, 0xAA, sizeof(out));
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_fill_hdr_info_packet(&ctx->state, &out), 0);
+	KUNIT_EXPECT_TRUE(test, out.valid);
+	KUNIT_EXPECT_EQ(test, (int)out.hb0, 0x87);
+	KUNIT_EXPECT_EQ(test, (int)out.hb1, 0x01);
+	KUNIT_EXPECT_EQ(test, (int)out.hb2, 0x1A);
+}
+
+/**
+ * dm_test_fill_hdr_dp - Test the DisplayPort SDP header layout
+ * @test: The KUnit test context
+ *
+ * A valid metadata blob on a DisplayPort connector packs the SDP header with
+ * the version/length subpacket bytes and marks the packet valid.
+ */
+static void dm_test_fill_hdr_dp(struct kunit *test)
+{
+	struct dm_test_hdr_ctx *ctx =
+		dm_test_hdr_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_info_packet out;
+
+	memset(&out, 0xAA, sizeof(out));
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_fill_hdr_info_packet(&ctx->state, &out), 0);
+	KUNIT_EXPECT_TRUE(test, out.valid);
+	KUNIT_EXPECT_EQ(test, (int)out.hb0, 0x00);
+	KUNIT_EXPECT_EQ(test, (int)out.hb1, 0x87);
+	KUNIT_EXPECT_EQ(test, (int)out.hb2, 0x1D);
+	KUNIT_EXPECT_EQ(test, (int)out.hb3, 0x13 << 2);
+	KUNIT_EXPECT_EQ(test, (int)out.sb[0], 0x01);
+	KUNIT_EXPECT_EQ(test, (int)out.sb[1], 0x1A);
+}
+
+/**
+ * dm_test_fill_hdr_unsupported_connector - Test unsupported sinks are rejected
+ * @test: The KUnit test context
+ *
+ * A connector type with no defined HDR infopacket layout returns -EINVAL and
+ * leaves the packet invalid.
+ */
+static void dm_test_fill_hdr_unsupported_connector(struct kunit *test)
+{
+	struct dm_test_hdr_ctx *ctx =
+		dm_test_hdr_ctx_alloc(test, DRM_MODE_CONNECTOR_VGA);
+	struct dc_info_packet out = {};
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_fill_hdr_info_packet(&ctx->state, &out),
+			-EINVAL);
+	KUNIT_EXPECT_FALSE(test, out.valid);
+}
+
+/**
+ * dm_test_fill_hdr_bad_metadata - Test malformed metadata is propagated
+ * @test: The KUnit test context
+ *
+ * A metadata blob with no payload makes drm_hdmi_infoframe_set_hdr_metadata()
+ * fail, and that error is returned unchanged.
+ */
+static void dm_test_fill_hdr_bad_metadata(struct kunit *test)
+{
+	struct dm_test_hdr_ctx *ctx =
+		dm_test_hdr_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_info_packet out = {};
+
+	ctx->blob.data = NULL;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_fill_hdr_info_packet(&ctx->state, &out),
+			-EINVAL);
+	KUNIT_EXPECT_FALSE(test, out.valid);
+}
+
 /* Tests for amdgpu_dm_connector_atomic_set_property() */
 
 /*
@@ -2611,7 +2735,7 @@ static void dm_test_update_subconnector_dp_with_sink(struct kunit *test)
 
 	update_subconnector_property(aconnector);
 
-	KUNIT_EXPECT_EQ(test, drm_object_property_get_value(&aconnector->base.base,
+	KUNIT_EXPECT_EQ(test, drm_object_property_get_default_value(&aconnector->base.base,
 				aconnector->base.dev->mode_config.dp_subconnector_property,
 				&val), 0);
 	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SUBCONNECTOR_VGA);
@@ -2654,7 +2778,7 @@ static void dm_test_update_subconnector_dp_no_sink(struct kunit *test)
 
 	update_subconnector_property(aconnector);
 
-	KUNIT_EXPECT_EQ(test, drm_object_property_get_value(&aconnector->base.base,
+	KUNIT_EXPECT_EQ(test, drm_object_property_get_default_value(&aconnector->base.base,
 				aconnector->base.dev->mode_config.dp_subconnector_property,
 				&val), 0);
 	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SUBCONNECTOR_Unknown);
@@ -2703,7 +2827,7 @@ static void dm_test_update_subconnector_non_dp_noop(struct kunit *test)
 	update_subconnector_property(aconnector);
 
 	/* Non-DP connector: value must remain what we seeded */
-	KUNIT_EXPECT_EQ(test, drm_object_property_get_value(&aconnector->base.base,
+	KUNIT_EXPECT_EQ(test, drm_object_property_get_default_value(&aconnector->base.base,
 				aconnector->base.dev->mode_config.dp_subconnector_property,
 				&val), 0);
 	KUNIT_EXPECT_EQ(test, (int)val, (int)DRM_MODE_SUBCONNECTOR_VGA);
@@ -2940,6 +3064,54 @@ static void dm_test_detect_mst_branch_without_aux(struct kunit *test)
 		amdgpu_dm_detect_mst_link_for_all_connectors(drm), 0);
 }
 
+/**
+ * dm_test_detect_mst_start_fail - Test the MST-start failure path
+ * @test: The KUnit test context
+ *
+ * An MST branch whose topology manager fails to start must downgrade the link
+ * to a single connection and attempt to stop the topology manager. The failure
+ * is forced by giving the manager a powered-down aux, so its initial DPCD read
+ * returns an error before any real transfer is attempted.
+ */
+static void dm_test_detect_mst_start_fail(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	struct drm_dp_aux *aux;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_DisplayPort);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+
+	link->type = dc_connection_mst_branch;
+	aconnector->dc_link = link;
+
+	/*
+	 * A powered-down aux makes the initial DPCD read return -EBUSY before
+	 * any transfer, so drm_dp_mst_topology_mgr_set_mst() fails fast.
+	 */
+	mutex_init(&aux->hw_mutex);
+	aux->name = "kunit-mst-aux";
+	aux->powered_down = true;
+	mutex_init(&aconnector->mst_mgr.lock);
+	aconnector->mst_mgr.dev = drm;
+	aconnector->mst_mgr.aux = aux;
+
+	/*
+	 * link->priv is left NULL so dm_helpers_dp_mst_stop_top_mgr() returns
+	 * early without recursing back into the topology manager.
+	 */
+	KUNIT_EXPECT_EQ(test,
+		amdgpu_dm_detect_mst_link_for_all_connectors(drm), 0);
+
+	/* The failure path downgrades the link to a single connection. */
+	KUNIT_EXPECT_EQ(test, link->type, dc_connection_single);
+}
+
 /* Tests for amdgpu_dm_find_first_crtc_matching_connector() */
 
 /*
@@ -3167,7 +3339,7 @@ static uint64_t dm_test_panel_prop_value(struct kunit *test,
 	uint64_t val = ~0ULL;
 
 	KUNIT_EXPECT_EQ(test,
-		drm_object_property_get_value(&ctx->aconnector->base.base,
+		drm_object_property_get_default_value(&ctx->aconnector->base.base,
 			ctx->drm->mode_config.panel_type_property, &val), 0);
 	return val;
 }
@@ -3761,6 +3933,82 @@ static void dm_test_fill_stream_non_hdmi_ep_keeps_depth(struct kunit *test)
 			(int)COLOR_DEPTH_101010);
 }
 
+/**
+ * dm_test_fill_stream_hdmi_infoframe - Test HDMI signal fills VIC from infoframe
+ * @test: The KUnit test context
+ *
+ * With an HDMI stream signal the AVI/vendor infoframe path runs and overwrites
+ * the VIC (seeded via an old stream) with the value derived from the mode.
+ */
+static void dm_test_fill_stream_hdmi_infoframe(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+	struct dc_stream_state *old_stream;
+	struct drm_display_mode *cea;
+
+	old_stream = kunit_kzalloc(test, sizeof(*old_stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_stream);
+	old_stream->timing.vic = 99;
+
+	/* A real CEA-16 (1080p60) mode so the AVI infoframe resolves VIC 16. */
+	cea = drm_display_mode_from_cea_vic(ctx->drm, 16);
+	KUNIT_ASSERT_NOT_NULL(test, cea);
+	*ctx->mode = *cea;
+	drm_mode_destroy(ctx->drm, cea);
+
+	ctx->stream->signal = SIGNAL_TYPE_HDMI_TYPE_A;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, old_stream, 8,
+		PIXEL_ENCODING_RGB, false);
+
+	/* The HDMI block overwrote the old stream's VIC of 99 with the AVI code. */
+	KUNIT_EXPECT_EQ(test, (int)timing->vic, 16);
+}
+
+/**
+ * dm_test_fill_stream_freesync_video - Test timing taken from the mode for FS video
+ * @test: The KUnit test context
+ *
+ * When the mode matches the connector's cached freesync base mode the timing is
+ * taken from the drm mode fields (not crtc_*), including a 10x pixel clock.
+ */
+static void dm_test_fill_stream_freesync_video(struct kunit *test)
+{
+	struct dm_test_fill_ctx *ctx = dm_test_fill_ctx_alloc(test);
+	struct dc_crtc_timing *timing = &ctx->stream->timing;
+
+	/* Cached high mode == the mode passed in => freesync video mode. */
+	ctx->aconnector->freesync_vid_base.clock = 148500;
+	ctx->aconnector->freesync_vid_base.hdisplay = 1920;
+	ctx->aconnector->freesync_vid_base.vdisplay = 1080;
+	ctx->aconnector->freesync_vid_base.hsync_start = 2008;
+	ctx->aconnector->freesync_vid_base.hsync_end = 2052;
+	ctx->aconnector->freesync_vid_base.htotal = 2200;
+	ctx->aconnector->freesync_vid_base.vsync_start = 1084;
+	ctx->aconnector->freesync_vid_base.vsync_end = 1089;
+	ctx->aconnector->freesync_vid_base.vtotal = 1125;
+
+	ctx->mode->clock = 148500;
+	ctx->mode->hdisplay = 1920;
+	ctx->mode->vdisplay = 1080;
+	ctx->mode->hsync_start = 2008;
+	ctx->mode->hsync_end = 2052;
+	ctx->mode->htotal = 2200;
+	ctx->mode->vsync_start = 1084;
+	ctx->mode->vsync_end = 1089;
+	ctx->mode->vtotal = 1125;
+
+	fill_stream_properties_from_drm_display_mode(ctx->stream, ctx->mode,
+		&ctx->aconnector->base, ctx->conn_state, NULL, 8,
+		PIXEL_ENCODING_RGB, false);
+
+	KUNIT_EXPECT_EQ(test, (int)timing->h_addressable, 1920);
+	KUNIT_EXPECT_EQ(test, (int)timing->v_addressable, 1080);
+	KUNIT_EXPECT_EQ(test, (int)timing->pix_clk_100hz, 1485000);
+}
+
 /* Tests for create_stream_for_sink() */
 
 /*
@@ -3936,121 +4184,393 @@ static void dm_test_create_stream_existing_sink(struct kunit *test)
 	dc_sink_release(sink);
 }
 
-/* Tests for amdgpu_dm_connector_detect() */
+/* Append a copy of the context mode to the connector's mode list. */
+static void dm_test_stream_add_mode(struct kunit *test, struct dm_test_stream_ctx *ctx,
+				    u32 type)
+{
+	struct drm_display_mode *mode = drm_mode_create(ctx->drm);
 
-/*
- * A non-DisplayPort connector keeps update_subconnector_property() a no-op and,
- * because the kunit thread is not the poll worker, the analog poll branch is
- * skipped. That leaves the forced-state and dc_sink presence branches as the
- * deterministic behaviour to exercise.
- */
-static struct amdgpu_dm_connector *dm_test_detect_connector(struct kunit *test)
+	KUNIT_ASSERT_NOT_NULL(test, mode);
+	drm_mode_copy(mode, ctx->mode);
+	mode->type = type;
+	list_add_tail(&mode->head, &ctx->aconnector->base.modes);
+}
+
+/* Give the context mode a full set of CRTC timings. */
+static void dm_test_stream_set_timings(struct dm_test_stream_ctx *ctx)
 {
-	struct drm_device *drm = dm_test_alloc_drm(test);
+	ctx->mode->hsync_start = 2008;
+	ctx->mode->hsync_end = 2052;
+	ctx->mode->htotal = 2200;
+	ctx->mode->vsync_start = 1084;
+	ctx->mode->vsync_end = 1089;
+	ctx->mode->vtotal = 1125;
+}
 
-	return dm_test_add_connector(test, drm, DRM_MODE_CONNECTOR_HDMIA);
+/* Create a real dc_sink of @signal owned by the caller. */
+static struct dc_sink *dm_test_stream_add_sink(struct kunit *test,
+					       struct dm_test_stream_ctx *ctx,
+					       enum signal_type signal)
+{
+	struct dc_sink_init_data sink_init = { 0 };
+	struct dc_sink *sink;
+
+	sink_init.link = ctx->link;
+	sink_init.sink_signal = signal;
+	sink = dc_sink_create(&sink_init);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink->sink_signal = signal;
+	ctx->aconnector->dc_sink = sink;
+
+	return sink;
 }
 
 /**
- * dm_test_detect_force_on - Test DRM_FORCE_ON reports connected
+ * dm_test_create_stream_preferred_mode - Test the preferred mode drives the timing
  * @test: The KUnit test context
+ *
+ * A preferred mode in the connector's mode list is used to recompute the CRTC
+ * timings of the requested mode before the stream properties are filled in.
  */
-static void dm_test_detect_force_on(struct kunit *test)
+static void dm_test_create_stream_preferred_mode(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+	struct dm_test_stream_ctx *ctx = dm_test_stream_ctx_alloc(test);
+	struct dc_stream_state *stream;
 
-	aconnector->base.force = DRM_FORCE_ON;
+	dm_test_stream_set_timings(ctx);
+	dm_test_stream_add_mode(test, ctx, DRM_MODE_TYPE_PREFERRED);
 
-	KUNIT_EXPECT_EQ(test,
-		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
-		(int)connector_status_connected);
+	stream = create_stream_for_sink(&ctx->aconnector->base, ctx->mode,
+					ctx->dm_state, NULL, 8,
+					PIXEL_ENCODING_RGB, false);
+
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_EXPECT_EQ(test, (int)stream->timing.h_addressable, 1920);
+	KUNIT_EXPECT_EQ(test, (int)stream->timing.v_addressable, 1080);
+	KUNIT_EXPECT_EQ(test, (int)stream->timing.h_total, 2200);
+	KUNIT_EXPECT_EQ(test, (int)stream->timing.v_total, 1125);
+	dc_stream_release(stream);
 }
 
 /**
- * dm_test_detect_force_on_digital - Test DRM_FORCE_ON_DIGITAL reports connected
+ * dm_test_create_stream_scaled_keeps_refresh - Test scaling reuses the old timing
  * @test: The KUnit test context
+ *
+ * With scaling enabled and an unchanged refresh rate the stream properties are
+ * filled from the old stream so the VIC and sync polarities are preserved.
  */
-static void dm_test_detect_force_on_digital(struct kunit *test)
+static void dm_test_create_stream_scaled_keeps_refresh(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+	struct dm_test_stream_ctx *ctx = dm_test_stream_ctx_alloc(test);
+	struct dc_stream_state *stream;
 
-	aconnector->base.force = DRM_FORCE_ON_DIGITAL;
+	dm_test_stream_set_timings(ctx);
+	dm_test_stream_add_mode(test, ctx, DRM_MODE_TYPE_PREFERRED);
+	ctx->dm_state->scaling = RMX_FULL;
 
-	KUNIT_EXPECT_EQ(test,
-		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
-		(int)connector_status_connected);
+	stream = create_stream_for_sink(&ctx->aconnector->base, ctx->mode,
+					ctx->dm_state, NULL, 8,
+					PIXEL_ENCODING_RGB, false);
+
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_EXPECT_EQ(test, (int)stream->timing.h_addressable, 1920);
+	dc_stream_release(stream);
 }
 
 /**
- * dm_test_detect_force_off - Test DRM_FORCE_OFF reports disconnected
+ * dm_test_create_stream_timing_override - Test the automated test timing override
  * @test: The KUnit test context
+ *
+ * A connector carrying a requested timing from an automated test replaces the
+ * timing derived from the mode.
  */
-static void dm_test_detect_force_off(struct kunit *test)
+static void dm_test_create_stream_timing_override(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+	struct dm_test_stream_ctx *ctx = dm_test_stream_ctx_alloc(test);
+	struct dc_crtc_timing *timing;
+	struct dc_stream_state *stream;
 
-	aconnector->base.force = DRM_FORCE_OFF;
+	timing = kunit_kzalloc(test, sizeof(*timing), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, timing);
+	timing->h_addressable = 1280;
+	timing->v_addressable = 720;
+	timing->display_color_depth = COLOR_DEPTH_101010;
 
-	KUNIT_EXPECT_EQ(test,
-		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
-		(int)connector_status_disconnected);
+	ctx->aconnector->timing_changed = true;
+	ctx->aconnector->timing_requested = timing;
+
+	stream = create_stream_for_sink(&ctx->aconnector->base, ctx->mode,
+					ctx->dm_state, NULL, 8,
+					PIXEL_ENCODING_RGB, false);
+
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_EXPECT_EQ(test, (int)stream->timing.h_addressable, 1280);
+	KUNIT_EXPECT_EQ(test, (int)stream->timing.display_color_depth,
+			(int)COLOR_DEPTH_101010);
+	dc_stream_release(stream);
 }
 
 /**
- * dm_test_detect_sink_present - Test a present dc_sink reports connected
+ * dm_test_create_stream_dp_vsc_colorimetry - Test the DP VSC SDP colorimetry decision
  * @test: The KUnit test context
+ *
+ * A DisplayPort sink on a DPCD 1.4 link advertising VSC SDP colorimetry, and
+ * without the disable-colorimetry quirk, enables VSC SDP colorimetry and
+ * arms the self refresh entry delay.
  */
-static void dm_test_detect_sink_present(struct kunit *test)
+static void dm_test_create_stream_dp_vsc_colorimetry(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+	struct dm_test_stream_ctx *ctx = dm_test_stream_ctx_alloc(test);
+	struct dc_stream_state *stream;
 	struct dc_sink *sink;
 
-	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink = dm_test_stream_add_sink(test, ctx, SIGNAL_TYPE_DISPLAY_PORT);
+	ctx->link->dpcd_caps.dpcd_rev.raw = 0x14;
+	ctx->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED = 1;
 
-	aconnector->base.force = DRM_FORCE_UNSPECIFIED;
-	aconnector->dc_sink = sink;
+	stream = create_stream_for_sink(&ctx->aconnector->base, ctx->mode,
+					ctx->dm_state, NULL, 8,
+					PIXEL_ENCODING_RGB, false);
 
-	KUNIT_EXPECT_EQ(test,
-		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
-		(int)connector_status_connected);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_EXPECT_EQ(test, (int)stream->signal, (int)SIGNAL_TYPE_DISPLAY_PORT);
+	KUNIT_EXPECT_TRUE(test, stream->use_vsc_sdp_for_colorimetry);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->sr_skip_count, AMDGPU_DM_PSR_ENTRY_DELAY);
+
+	dc_stream_release(stream);
+	dc_sink_release(sink);
 }
 
 /**
- * dm_test_detect_no_sink - Test a missing dc_sink reports disconnected
+ * dm_test_create_stream_dp_colorimetry_quirk - Test the disable colorimetry quirk
  * @test: The KUnit test context
  */
-static void dm_test_detect_no_sink(struct kunit *test)
+static void dm_test_create_stream_dp_colorimetry_quirk(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+	struct dm_test_stream_ctx *ctx = dm_test_stream_ctx_alloc(test);
+	struct dc_stream_state *stream;
+	struct dc_sink *sink;
 
-	aconnector->base.force = DRM_FORCE_UNSPECIFIED;
-	aconnector->dc_sink = NULL;
+	sink = dm_test_stream_add_sink(test, ctx, SIGNAL_TYPE_DISPLAY_PORT);
+	sink->edid_caps.panel_patch.disable_colorimetry = true;
+	ctx->link->dpcd_caps.dpcd_rev.raw = 0x14;
+	ctx->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED = 1;
 
-	KUNIT_EXPECT_EQ(test,
-		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
-		(int)connector_status_disconnected);
-}
+	stream = create_stream_for_sink(&ctx->aconnector->base, ctx->mode,
+					ctx->dm_state, NULL, 8,
+					PIXEL_ENCODING_RGB, false);
 
-/* Tests for amdgpu_dm_connector_poll() */
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_EXPECT_FALSE(test, stream->use_vsc_sdp_for_colorimetry);
+
+	dc_stream_release(stream);
+	dc_sink_release(sink);
+}
 
 /**
- * dm_test_poll_dac_load_returns_cached - Test the DAC load detection shortcut
+ * dm_test_create_stream_hdmi_vsif - Test the HDMI vendor specific infoframe
  * @test: The KUnit test context
  *
- * When the previous connection was established by analog DAC load detection and
- * polling is not forced, the connector is not re-detected and its cached status
- * is returned unchanged. The connector is embedded in an amdgpu_device so that
- * drm_to_adev() resolves.
+ * An HDMI sink takes the HDMI Forum vendor specific infoframe path. A plain
+ * 2D, non-HDMI-VIC timing carries no payload, so the packet stays invalid.
  */
-static void dm_test_poll_dac_load_returns_cached(struct kunit *test)
+static void dm_test_create_stream_hdmi_vsif(struct kunit *test)
 {
-	struct amdgpu_device *adev;
-	struct amdgpu_dm_connector *aconnector;
-	struct dc_link *link;
-	struct dc_sink *local_sink;
-	struct drm_device *drm;
-	struct device *dev;
+	struct dm_test_stream_ctx *ctx = dm_test_stream_ctx_alloc(test);
+	struct dc_stream_state *stream;
+	struct dc_sink *sink;
+
+	ctx->link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	sink = dm_test_stream_add_sink(test, ctx, SIGNAL_TYPE_HDMI_TYPE_A);
+
+	stream = create_stream_for_sink(&ctx->aconnector->base, ctx->mode,
+					ctx->dm_state, NULL, 8,
+					PIXEL_ENCODING_RGB, false);
+
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_EXPECT_EQ(test, (int)stream->signal, (int)SIGNAL_TYPE_HDMI_TYPE_A);
+	KUNIT_EXPECT_FALSE(test, stream->vsp_infopacket.valid);
+
+	dc_stream_release(stream);
+	dc_sink_release(sink);
+}
+
+/* Tests for amdgpu_dm_connector_detect() */
+
+/*
+ * A non-DisplayPort connector keeps update_subconnector_property() a no-op and,
+ * because the kunit thread is not the poll worker, the analog poll branch is
+ * skipped. That leaves the forced-state and dc_sink presence branches as the
+ * deterministic behaviour to exercise.
+ */
+static struct amdgpu_dm_connector *dm_test_detect_connector(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+
+	return dm_test_add_connector(test, drm, DRM_MODE_CONNECTOR_HDMIA);
+}
+
+/**
+ * dm_test_detect_force_on - Test DRM_FORCE_ON reports connected
+ * @test: The KUnit test context
+ */
+static void dm_test_detect_force_on(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+
+	aconnector->base.force = DRM_FORCE_ON;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
+		(int)connector_status_connected);
+}
+
+/**
+ * dm_test_detect_force_on_digital - Test DRM_FORCE_ON_DIGITAL reports connected
+ * @test: The KUnit test context
+ */
+static void dm_test_detect_force_on_digital(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+
+	aconnector->base.force = DRM_FORCE_ON_DIGITAL;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
+		(int)connector_status_connected);
+}
+
+/**
+ * dm_test_detect_force_off - Test DRM_FORCE_OFF reports disconnected
+ * @test: The KUnit test context
+ */
+static void dm_test_detect_force_off(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+
+	aconnector->base.force = DRM_FORCE_OFF;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
+		(int)connector_status_disconnected);
+}
+
+/**
+ * dm_test_detect_sink_present - Test a present dc_sink reports connected
+ * @test: The KUnit test context
+ */
+static void dm_test_detect_sink_present(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+	struct dc_sink *sink;
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	aconnector->base.force = DRM_FORCE_UNSPECIFIED;
+	aconnector->dc_sink = sink;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
+		(int)connector_status_connected);
+}
+
+/**
+ * dm_test_detect_no_sink - Test a missing dc_sink reports disconnected
+ * @test: The KUnit test context
+ */
+static void dm_test_detect_no_sink(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+
+	aconnector->base.force = DRM_FORCE_UNSPECIFIED;
+	aconnector->dc_sink = NULL;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
+		(int)connector_status_disconnected);
+}
+
+/*
+ * A 256-byte EDID whose DisplayID extension carries a tiled display
+ * topology block placing this connector at tile location (h=1, v=0), i.e.
+ * a secondary tile. drm_edid_connector_update() parses it to set has_tile
+ * and the non-origin tile location.
+ */
+static const u8 dm_test_detect_tile_edid[256] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x10, 0xac, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x80, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc4, 0x70, 0x12, 0x18, 0x00,
+	0x00, 0x12, 0x00, 0x15, 0x00, 0x11, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x8d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x90,
+};
+
+/**
+ * dm_test_detect_hides_secondary_tile - Test the secondary tile is hidden
+ * @test: The KUnit test context
+ *
+ * A sink whose panel patch requests disable_second_tile combined with a tiled
+ * EDID at a non-origin tile location makes the detect path hide the secondary
+ * display tile, reporting the connector as disconnected.
+ */
+static void dm_test_detect_hides_secondary_tile(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_detect_connector(test);
+	struct dc_sink *sink;
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink->edid_caps.panel_patch.disable_second_tile = true;
+
+	aconnector->base.force = DRM_FORCE_UNSPECIFIED;
+	aconnector->dc_sink = sink;
+	aconnector->drm_edid = drm_edid_alloc(dm_test_detect_tile_edid,
+					     sizeof(dm_test_detect_tile_edid));
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->drm_edid);
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_detect(&aconnector->base, false),
+		(int)connector_status_disconnected);
+
+	drm_edid_free(aconnector->drm_edid);
+}
+
+/* Tests for amdgpu_dm_connector_poll() */
+
+/**
+ * dm_test_poll_dac_load_returns_cached - Test the DAC load detection shortcut
+ * @test: The KUnit test context
+ *
+ * When the previous connection was established by analog DAC load detection and
+ * polling is not forced, the connector is not re-detected and its cached status
+ * is returned unchanged. The connector is embedded in an amdgpu_device so that
+ * drm_to_adev() resolves.
+ */
+static void dm_test_poll_dac_load_returns_cached(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	struct dc_sink *local_sink;
+	struct drm_device *drm;
+	struct device *dev;
 
 	dev = drm_kunit_helper_alloc_device(test);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
@@ -4085,6 +4605,168 @@ static void dm_test_poll_dac_load_returns_cached(struct kunit *test)
 		(int)connector_status_connected);
 }
 
+/*
+ * Fake link_service detection callbacks used by the connector poll tests.
+ * dc_link_detect_connection_type() and dc_link_detect() proxy through
+ * link->dc->link_srv, so faking these pointers exercises the poll paths
+ * without touching real hardware.
+ */
+static bool dm_test_poll_detect_type_connected(struct dc_link *link,
+						enum dc_connection_type *type)
+{
+	*type = dc_connection_single;
+	return true;
+}
+
+static bool dm_test_poll_detect_type_none(struct dc_link *link,
+					   enum dc_connection_type *type)
+{
+	*type = dc_connection_none;
+	return false;
+}
+
+static bool dm_test_poll_detect_link_connected(struct dc_link *link,
+					       enum dc_detect_reason reason)
+{
+	return true;
+}
+
+struct dm_test_poll_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	struct dc *dc;
+	struct dc_context *dc_ctx;
+	struct link_service *link_srv;
+	struct drm_device *drm;
+};
+
+/*
+ * Build an amdgpu_dm_connector embedded in an amdgpu_device (so drm_to_adev()
+ * resolves) with a faked dc_link->dc->link_srv. The hpd_lock and dc_lock
+ * mutexes are initialised because the non-shortcut poll path takes them.
+ */
+static struct dm_test_poll_ctx *dm_test_poll_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_poll_ctx *ctx;
+	struct device *dev;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(*ctx->adev),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+	ctx->adev = drm_to_adev(ctx->drm);
+	mutex_init(&ctx->adev->dm.dc_lock);
+
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+		drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+				    &dm_test_connector_funcs,
+				    DRM_MODE_CONNECTOR_VGA, NULL), 0);
+	mutex_init(&ctx->aconnector->hpd_lock);
+
+	ctx->dc = kunit_kzalloc(test, sizeof(*ctx->dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc);
+	ctx->link_srv = kunit_kzalloc(test, sizeof(*ctx->link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link_srv);
+	ctx->dc->link_srv = ctx->link_srv;
+
+	ctx->dc_ctx = kunit_kzalloc(test, sizeof(*ctx->dc_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc_ctx);
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->link->dc = ctx->dc;
+	ctx->link->ctx = ctx->dc_ctx;
+	ctx->aconnector->dc_link = ctx->link;
+
+	return ctx;
+}
+
+/**
+ * dm_test_poll_connected_cached_sink - Test the connected path reusing a sink
+ * @test: The KUnit test context
+ *
+ * detect_connection_type reports a connection and a local_sink already exists,
+ * so the short-circuit skips full detection and the status stays connected.
+ */
+static void dm_test_poll_connected_cached_sink(struct kunit *test)
+{
+	struct dm_test_poll_ctx *ctx = dm_test_poll_ctx_alloc(test);
+	struct dc_sink *local_sink;
+
+	ctx->link_srv->detect_connection_type = dm_test_poll_detect_type_connected;
+	local_sink = kunit_kzalloc(test, sizeof(*local_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, local_sink);
+	ctx->link->local_sink = local_sink;
+	ctx->link->type = dc_connection_single;
+	ctx->aconnector->base.status = connector_status_connected;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_poll(ctx->aconnector, false),
+		(int)connector_status_connected);
+}
+
+/**
+ * dm_test_poll_connected_new_sink - Test the connected path via full detection
+ * @test: The KUnit test context
+ *
+ * With no cached local_sink, dc_link_detect() is consulted and reports a
+ * connection, so the status becomes connected.
+ */
+static void dm_test_poll_connected_new_sink(struct kunit *test)
+{
+	struct dm_test_poll_ctx *ctx = dm_test_poll_ctx_alloc(test);
+
+	ctx->link_srv->detect_connection_type = dm_test_poll_detect_type_connected;
+	ctx->link_srv->detect_link = dm_test_poll_detect_link_connected;
+	ctx->link->local_sink = NULL;
+	ctx->aconnector->base.status = connector_status_connected;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_poll(ctx->aconnector, false),
+		(int)connector_status_connected);
+}
+
+/**
+ * dm_test_poll_disconnect_releases_sink - Test the disconnect teardown path
+ * @test: The KUnit test context
+ *
+ * detect_connection_type reports no connection while the cached status was
+ * connected, so the stale local_sink is released and cleared before the
+ * connector is re-evaluated.
+ */
+static void dm_test_poll_disconnect_releases_sink(struct kunit *test)
+{
+	struct dm_test_poll_ctx *ctx = dm_test_poll_ctx_alloc(test);
+	struct dc_sink_init_data sink_init = { 0 };
+	struct dc_sink *local_sink;
+
+	ctx->link_srv->detect_connection_type = dm_test_poll_detect_type_none;
+
+	sink_init.link = ctx->link;
+	sink_init.sink_signal = SIGNAL_TYPE_VIRTUAL;
+	local_sink = dc_sink_create(&sink_init);
+	KUNIT_ASSERT_NOT_NULL(test, local_sink);
+	ctx->link->local_sink = local_sink;
+
+	ctx->aconnector->base.status = connector_status_connected;
+	ctx->aconnector->dc_sink = NULL;
+
+	KUNIT_EXPECT_EQ(test,
+		(int)amdgpu_dm_connector_poll(ctx->aconnector, false),
+		(int)connector_status_disconnected);
+	KUNIT_EXPECT_NULL(test, ctx->link->local_sink);
+}
+
 /* Tests for amdgpu_dm_connector_late_register() and _unregister() */
 
 /*
@@ -4323,6 +5005,43 @@ static void dm_test_encoder_disable_noop(struct kunit *test)
  * connector and connector-state are stacked in their containers and wired
  * together through conn_state->connector.
  */
+/* Tests for amdgpu_dm_encoder_init() */
+
+/**
+ * dm_test_encoder_init_success - Test encoder init wires id, crtc mask and helpers
+ * @test: The KUnit test context
+ *
+ * On a DRM device embedded in an amdgpu_device, amdgpu_dm_encoder_init()
+ * registers a TMDS encoder, derives possible_crtcs from mode_info.num_crtc,
+ * records the link index as the encoder id and attaches the helper funcs.
+ */
+static void dm_test_encoder_init_success(struct kunit *test)
+{
+	struct device *dev;
+	struct drm_device *drm;
+	struct amdgpu_device *adev;
+	struct amdgpu_encoder *aencoder;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev, sizeof(*adev),
+						  offsetof(struct amdgpu_device, ddev),
+						  DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+	adev = drm_to_adev(drm);
+	adev->mode_info.num_crtc = 4;
+
+	/* Plain kzalloc: amdgpu_dm_encoder_destroy() kfree()s it on teardown. */
+	aencoder = kzalloc_obj(*aencoder);
+	KUNIT_ASSERT_NOT_NULL(test, aencoder);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_encoder_init(drm, aencoder, 2), 0);
+	KUNIT_EXPECT_EQ(test, aencoder->encoder_id, 2);
+	KUNIT_EXPECT_EQ(test, (int)aencoder->base.possible_crtcs, 0xf);
+	KUNIT_EXPECT_PTR_EQ(test, (const void *)aencoder->base.helper_private,
+			    (const void *)&amdgpu_dm_encoder_helper_funcs);
+}
+
 struct dm_test_atomic_check_ctx {
 	struct drm_device *drm;
 	struct amdgpu_encoder *aenc;
@@ -4429,6 +5148,252 @@ static void dm_test_atomic_check_non_mst_returns_zero(struct kunit *test)
 					       &ctx->dm_state->base), 0);
 }
 
+/**
+ * dm_test_atomic_check_mst_no_change_returns_zero - Test MST no-change short-circuit
+ * @test: The KUnit test context
+ *
+ * An MST connector (mst_output_port set) whose CRTC state reports neither a
+ * connectors nor a mode change hits the ``return 0`` before any topology state
+ * is fetched.
+ */
+static void dm_test_atomic_check_mst_no_change_returns_zero(struct kunit *test)
+{
+	struct dm_test_atomic_check_ctx *ctx =
+		dm_test_atomic_check_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct drm_dp_mst_port *mst_port;
+	struct amdgpu_dm_connector *mst_root;
+
+	mst_port = kunit_kzalloc(test, sizeof(*mst_port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mst_port);
+	mst_root = kunit_kzalloc(test, sizeof(*mst_root), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mst_root);
+
+	ctx->aconnector->mst_output_port = mst_port;
+	ctx->aconnector->mst_root = mst_root;
+	ctx->crtc_state->connectors_changed = false;
+	ctx->crtc_state->mode_changed = false;
+
+	KUNIT_EXPECT_EQ(test,
+		dm_encoder_helper_atomic_check(&ctx->aenc->base,
+					       ctx->crtc_state,
+					       &ctx->dm_state->base), 0);
+}
+
+
+static uint32_t dm_test_atomic_check_dp_link_bandwidth_kbps(
+	const struct dc_link *link,
+	const struct dc_link_settings *link_settings)
+{
+	return 4320000;
+}
+
+static const struct dc_link_settings *dm_test_atomic_check_dp_get_verified_link_cap(
+	const struct dc_link *link)
+{
+	return &link->verified_link_cap;
+}
+
+/* Fake fully-wired MST atomic state for the deep dm_encoder_helper_atomic_check() path. */
+struct dm_test_mst_scaffold {
+	struct drm_atomic_commit *state;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_dp_mst_atomic_payload *payload;
+	struct drm_dp_mst_port *mst_port;
+	struct amdgpu_dm_connector *mst_root;
+	struct __drm_private_objs_state *priv_objs;
+};
+
+/*
+ * dm_test_atomic_check_mst_scaffold - build a fake MST atomic state.
+ * @map_success: when true the topology manager maps to a valid topology state
+ *               and a fake DC link/link-service is wired so the PBN divider is
+ *               non-zero; when false the manager maps to an error pointer so
+ *               drm_atomic_get_mst_topology_state() reports failure.
+ */
+static void dm_test_atomic_check_mst_scaffold(struct kunit *test,
+		struct dm_test_atomic_check_ctx *ctx,
+		struct dm_test_mst_scaffold *s,
+		bool map_success)
+{
+	struct drm_modeset_acquire_ctx *acquire_ctx;
+	struct link_service *link_srv;
+	struct dc *dc;
+	struct dc_link *link;
+	struct drm_connector *port_conn;
+	struct drm_crtc *crtc;
+	struct drm_connector_state *port_conn_state;
+	struct __drm_connnectors_state *conns;
+
+	memset(s, 0, sizeof(*s));
+
+	s->mst_root = kunit_kzalloc(test, sizeof(*s->mst_root), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, s->mst_root);
+	s->mst_root->mst_mgr.dev = ctx->drm;
+
+	s->mst_port = kunit_kzalloc(test, sizeof(*s->mst_port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, s->mst_port);
+
+	ctx->aconnector->mst_root = s->mst_root;
+	ctx->aconnector->mst_output_port = s->mst_port;
+
+	acquire_ctx = kunit_kzalloc(test, sizeof(*acquire_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acquire_ctx);
+
+	s->state = kunit_kzalloc(test, sizeof(*s->state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, s->state);
+	s->state->dev = ctx->drm;
+	s->state->acquire_ctx = acquire_ctx;
+
+	s->priv_objs = kunit_kzalloc(test, sizeof(*s->priv_objs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, s->priv_objs);
+	s->priv_objs[0].ptr = &s->mst_root->mst_mgr.base;
+	s->state->num_private_objs = 1;
+	s->state->private_objs = s->priv_objs;
+
+	ctx->crtc_state->state = s->state;
+	ctx->crtc_state->connectors_changed = true;
+
+	if (!map_success) {
+		s->priv_objs[0].new_state = ERR_PTR(-ENOMEM);
+		return;
+	}
+
+	/* Fake DC link so dm_mst_get_pbn_divider() yields a non-zero divider. */
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	link_srv->dp_get_verified_link_cap =
+		dm_test_atomic_check_dp_get_verified_link_cap;
+	link_srv->dp_link_bandwidth_kbps =
+		dm_test_atomic_check_dp_link_bandwidth_kbps;
+	dc->link_srv = link_srv;
+	link->dc = dc;
+	s->mst_root->dc_link = link;
+
+	s->mst_state = kunit_kzalloc(test, sizeof(*s->mst_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, s->mst_state);
+	INIT_LIST_HEAD(&s->mst_state->payloads);
+	s->priv_objs[0].new_state = &s->mst_state->base;
+
+	/* Pre-seed a payload so find_time_slots() skips the port kref alloc path. */
+	s->payload = kunit_kzalloc(test, sizeof(*s->payload), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, s->payload);
+	s->payload->port = s->mst_port;
+	list_add(&s->payload->next, &s->mst_state->payloads);
+
+	/* Wire a connector+crtc so find_time_slots() can resolve the crtc mask. */
+	port_conn = kunit_kzalloc(test, sizeof(*port_conn), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	port_conn_state = kunit_kzalloc(test, sizeof(*port_conn_state), GFP_KERNEL);
+	conns = kunit_kzalloc(test, sizeof(*conns), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, port_conn);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, port_conn_state);
+	KUNIT_ASSERT_NOT_NULL(test, conns);
+	port_conn->index = 0;
+	crtc->index = 0;
+	port_conn_state->crtc = crtc;
+	s->mst_port->connector = port_conn;
+	conns[0].new_state = port_conn_state;
+	s->state->num_connector = 1;
+	s->state->connectors = conns;
+}
+
+/**
+ * dm_test_atomic_check_mst_finds_vcpi_slots - Test the MST time-slot allocation path
+ * @test: The KUnit test context
+ *
+ * With a fully wired MST atomic state and connectors_changed set, the check
+ * computes the color depth/PBN, allocates VCPI time slots and returns 0.
+ */
+static void dm_test_atomic_check_mst_finds_vcpi_slots(struct kunit *test)
+{
+	struct dm_test_atomic_check_ctx *ctx =
+		dm_test_atomic_check_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dm_test_mst_scaffold s;
+
+	dm_test_atomic_check_mst_scaffold(test, ctx, &s, true);
+	s.state->duplicated = false;
+	ctx->dm_state->base.max_requested_bpc = 8;
+
+	KUNIT_EXPECT_EQ(test,
+		dm_encoder_helper_atomic_check(&ctx->aenc->base,
+					       ctx->crtc_state,
+					       &ctx->dm_state->base), 0);
+	KUNIT_EXPECT_GE(test, ctx->dm_state->vcpi_slots, 0);
+}
+
+/**
+ * dm_test_atomic_check_mst_duplicated_skips_pbn - Test the duplicated-state fast path
+ * @test: The KUnit test context
+ *
+ * When the atomic state is duplicated the color depth/PBN recompute block is
+ * skipped, but time slots are still allocated and the check returns 0.
+ */
+static void dm_test_atomic_check_mst_duplicated_skips_pbn(struct kunit *test)
+{
+	struct dm_test_atomic_check_ctx *ctx =
+		dm_test_atomic_check_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dm_test_mst_scaffold s;
+
+	dm_test_atomic_check_mst_scaffold(test, ctx, &s, true);
+	s.state->duplicated = true;
+	ctx->dm_state->pbn = 0;
+
+	KUNIT_EXPECT_EQ(test,
+		dm_encoder_helper_atomic_check(&ctx->aenc->base,
+					       ctx->crtc_state,
+					       &ctx->dm_state->base), 0);
+}
+
+/**
+ * dm_test_atomic_check_mst_topology_err_propagates - Test topology-state error propagation
+ * @test: The KUnit test context
+ *
+ * When drm_atomic_get_mst_topology_state() returns an error pointer, the check
+ * propagates the error code back to the caller.
+ */
+static void dm_test_atomic_check_mst_topology_err_propagates(struct kunit *test)
+{
+	struct dm_test_atomic_check_ctx *ctx =
+		dm_test_atomic_check_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dm_test_mst_scaffold s;
+
+	dm_test_atomic_check_mst_scaffold(test, ctx, &s, false);
+
+	KUNIT_EXPECT_EQ(test,
+		dm_encoder_helper_atomic_check(&ctx->aenc->base,
+					       ctx->crtc_state,
+					       &ctx->dm_state->base), -ENOMEM);
+}
+
+/**
+ * dm_test_atomic_check_mst_vcpi_error_propagates - Test VCPI allocation error propagation
+ * @test: The KUnit test context
+ *
+ * A pre-existing payload marked for deletion makes drm_dp_atomic_find_time_slots()
+ * fail; the negative vcpi_slots result is propagated back to the caller.
+ */
+static void dm_test_atomic_check_mst_vcpi_error_propagates(struct kunit *test)
+{
+	struct dm_test_atomic_check_ctx *ctx =
+		dm_test_atomic_check_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dm_test_mst_scaffold s;
+
+	dm_test_atomic_check_mst_scaffold(test, ctx, &s, true);
+	s.state->duplicated = true;
+	ctx->dm_state->pbn = 100;
+	s.payload->delete = true;
+
+	KUNIT_EXPECT_EQ(test,
+		dm_encoder_helper_atomic_check(&ctx->aenc->base,
+					       ctx->crtc_state,
+					       &ctx->dm_state->base), -EINVAL);
+}
+
 /* Tests for hdmi_cec_unset_edid() */
 
 /**
@@ -4606,6 +5571,87 @@ static void dm_test_validate_stream_null_stream(struct kunit *test)
 			(int)DC_ERROR_UNEXPECTED);
 }
 
+static bool dm_test_vs_validate_timing_true(struct timing_generator *tg,
+					     const struct dc_crtc_timing *timing)
+{
+	return true;
+}
+
+static enum dc_status dm_test_vs_validate_mode_timing_ok(
+		const struct dc_stream_state *stream,
+		struct dc_link *link,
+		const struct dc_crtc_timing *timing)
+{
+	return DC_OK;
+}
+
+/**
+ * dm_test_validate_stream_dc_ok_no_pipe - Exercise the full validation body
+ * @test: The KUnit test context
+ *
+ * Build a minimal fake dc so stream and plane validation both pass, then
+ * let dc_state_add_stream() bail out early (timing_generator_count == 0) so
+ * the deep pipe-allocation paths are avoided. This drives plane population,
+ * dc_validate_stream(), dc_validate_plane() and cleanup.
+ */
+static void dm_test_validate_stream_dc_ok_no_pipe(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct dc *dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	struct dc_context *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	struct dal_logger *logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	struct resource_pool *pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL);
+	struct resource_funcs *rfuncs = kunit_kzalloc(test, sizeof(*rfuncs), GFP_KERNEL);
+	struct resource_caps *rcaps = kunit_kzalloc(test, sizeof(*rcaps), GFP_KERNEL);
+	struct timing_generator *tg = kunit_kzalloc(test, sizeof(*tg), GFP_KERNEL);
+	struct timing_generator_funcs *tgfuncs =
+		kunit_kzalloc(test, sizeof(*tgfuncs), GFP_KERNEL);
+	struct link_service *link_srv =
+		kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	struct dc_stream_state *stream =
+		kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_NULL(test, logger);
+	KUNIT_ASSERT_NOT_NULL(test, pool);
+	KUNIT_ASSERT_NOT_NULL(test, rfuncs);
+	KUNIT_ASSERT_NOT_NULL(test, rcaps);
+	KUNIT_ASSERT_NOT_NULL(test, tg);
+	KUNIT_ASSERT_NOT_NULL(test, tgfuncs);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	logger->dev = drm;
+	ctx->logger = logger;
+
+	tgfuncs->validate_timing = dm_test_vs_validate_timing_true;
+	tg->funcs = tgfuncs;
+
+	pool->funcs = rfuncs;
+	pool->res_cap = rcaps;
+	pool->timing_generators[0] = tg;
+	pool->timing_generator_count = 0;
+
+	link_srv->validate_mode_timing = dm_test_vs_validate_mode_timing_ok;
+
+	dc->ctx = ctx;
+	dc->res_pool = pool;
+	dc->link_srv = link_srv;
+
+	link->ep_type = DISPLAY_ENDPOINT_UNKNOWN;
+	stream->link = link;
+	stream->src.width = 1920;
+	stream->src.height = 1080;
+
+	KUNIT_EXPECT_EQ(test,
+			(int)dm_validate_stream_and_context(dc, stream),
+			(int)DC_ERROR_UNEXPECTED);
+}
+
 /**
  * dm_test_to_encoder_no_encoder - Test connector with no encoder returns NULL
  * @test: The KUnit test context
@@ -4687,9 +5733,11 @@ static void dm_test_native_mode_copies_preferred(struct kunit *test)
 	mode->clock = 148500;
 	mode->hdisplay = 1920;
 	mode->vdisplay = 1080;
+	mutex_lock(&ctx->drm->mode_config.mutex);
 	drm_mode_probed_add(&ctx->aconnector->base, mode);
 
 	amdgpu_dm_get_native_mode(&ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
 
 	KUNIT_EXPECT_EQ(test, ctx->aenc->native_mode.hdisplay, 1920);
 	KUNIT_EXPECT_EQ(test, ctx->aenc->native_mode.vdisplay, 1080);
@@ -4742,8 +5790,10 @@ static void dm_test_add_common_modes_non_edp_noop(struct kunit *test)
 	ctx->aenc->native_mode.hdisplay = 1920;
 	ctx->aenc->native_mode.vdisplay = 1200;
 
+	mutex_lock(&ctx->drm->mode_config.mutex);
 	amdgpu_dm_connector_add_common_modes(&ctx->aenc->base,
 					     &ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
 
 	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 0);
 }
@@ -4763,8 +5813,10 @@ static void dm_test_add_common_modes_edp_adds(struct kunit *test)
 	ctx->aenc->native_mode.hdisplay = 1920;
 	ctx->aenc->native_mode.vdisplay = 1200;
 
+	mutex_lock(&ctx->drm->mode_config.mutex);
 	amdgpu_dm_connector_add_common_modes(&ctx->aenc->base,
 					     &ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
 
 	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 10);
 }
@@ -4805,6 +5857,94 @@ static void dm_test_add_fs_modes_no_preferred_mode(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)add_fs_modes(aconnector), 0);
 }
 
+/*
+ * Build a DisplayPort connector whose highest-refresh mode is a fixed
+ * 1920x1080@60 timing. A non-zero freesync_vid_base.clock makes
+ * amdgpu_dm_get_highest_refresh_rate_mode() return that timing directly,
+ * giving the test full control of the reference mode.
+ */
+static struct amdgpu_dm_connector *dm_test_fs_setup(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector =
+		dm_test_add_connector(test, drm, DRM_MODE_CONNECTOR_DisplayPort);
+	struct drm_display_mode *m = &aconnector->freesync_vid_base;
+
+	m->clock = 148500;
+	m->hdisplay = 1920;
+	m->htotal = 2200;
+	m->vdisplay = 1080;
+	m->vsync_start = 1084;
+	m->vsync_end = 1089;
+	m->vtotal = 1125;
+
+	return aconnector;
+}
+
+/**
+ * dm_test_add_fs_modes_generates - Test FreeSync video modes are added
+ * @test: The KUnit test context
+ *
+ * With min/max vfreq spanning the standard rates, add_fs_modes() derives one
+ * mode per legal rate at or below the reference refresh. A second call finds
+ * every generated mode already present, so is_duplicate_mode() rejects them
+ * all and no new modes are added.
+ */
+static void dm_test_add_fs_modes_generates(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_fs_setup(test);
+	unsigned int count, duplicate_count;
+
+	aconnector->min_vfreq = 20;
+	aconnector->max_vfreq = 60;
+
+	mutex_lock(&aconnector->base.dev->mode_config.mutex);
+	count = add_fs_modes(aconnector);
+	duplicate_count = add_fs_modes(aconnector);
+	mutex_unlock(&aconnector->base.dev->mode_config.mutex);
+
+	KUNIT_EXPECT_EQ(test, count, 8);
+	KUNIT_EXPECT_EQ(test, duplicate_count, 0);
+}
+
+/**
+ * dm_test_add_fs_modes_out_of_range - Test no modes when rates fall outside range
+ * @test: The KUnit test context
+ *
+ * A vfreq window above the reference refresh leaves every standard rate either
+ * higher than the mode or outside [min_vfreq, max_vfreq], so add_fs_modes()
+ * skips them all and returns 0.
+ */
+static void dm_test_add_fs_modes_out_of_range(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_fs_setup(test);
+
+	aconnector->min_vfreq = 100;
+	aconnector->max_vfreq = 120;
+
+	KUNIT_EXPECT_EQ(test, (int)add_fs_modes(aconnector), 0);
+}
+
+/**
+ * dm_test_add_fs_modes_skips_illegal - Test illegal derived timings are dropped
+ * @test: The KUnit test context
+ *
+ * A reference vtotal of 1133 gives a true refresh of ~59.58 that rounds up to
+ * 60, so the 60000 rate clears the refresh check yet yields a negative vtotal
+ * delta. The resulting timing is illegal, so add_fs_modes() skips it and
+ * returns 0.
+ */
+static void dm_test_add_fs_modes_skips_illegal(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector = dm_test_fs_setup(test);
+
+	aconnector->freesync_vid_base.vtotal = 1133;
+	aconnector->min_vfreq = 59;
+	aconnector->max_vfreq = 60;
+
+	KUNIT_EXPECT_EQ(test, (int)add_fs_modes(aconnector), 0);
+}
+
 /**
  * dm_test_add_freesync_modes_null_edid_noop - Test NULL EDID adds no modes
  * @test: The KUnit test context
@@ -4867,115 +6007,399 @@ static void dm_test_i2c_xfer_no_ddc_pin(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_i2c_xfer(&i2c->base, NULL, 0), -EIO);
 }
 
-/**
- * dm_test_get_amd_vsdb_unsupported - Test a zero VSDB version reports no support
- * @test: The KUnit test context
+/* Fake i2c adapter chain used by the amdgpu_dm_i2c_xfer() tests. */
+struct dm_test_i2c_ctx {
+	struct amdgpu_i2c_adapter *i2c;
+	struct ddc_service *ddc;
+	struct dc_context *dc_ctx;
+	struct dc *dc;
+	struct dc_link *link;
+};
+
+/*
+ * Build an i2c adapter whose ddc_service has a ddc_pin (so the transfer is not
+ * rejected up front) and whose dc back-pointer resolves to a link at index 0.
+ * The link ddc has no ddc_pin, so the non-OEM dc_submit_i2c() path bails out
+ * and reports failure without touching real hardware.
  */
-static void dm_test_get_amd_vsdb_unsupported(struct kunit *test)
+static struct dm_test_i2c_ctx *dm_test_i2c_setup(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector;
-	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
-
-	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	struct dm_test_i2c_ctx *ctx;
+	struct ddc_service *link_ddc;
+	struct ddc *pin;
 
-	aconnector->base.display_info.amd_vsdb.version = 0;
-	aconnector->base.display_info.amd_vsdb.replay_mode = false;
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->i2c = kunit_kzalloc(test, sizeof(*ctx->i2c), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->i2c);
+	ctx->ddc = kunit_kzalloc(test, sizeof(*ctx->ddc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->ddc);
+	ctx->dc_ctx = kunit_kzalloc(test, sizeof(*ctx->dc_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc_ctx);
+	ctx->dc = kunit_kzalloc(test, sizeof(*ctx->dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc);
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	pin = kunit_kzalloc(test, sizeof(*pin), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, pin);
+	link_ddc = kunit_kzalloc(test, sizeof(*link_ddc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_ddc);
+
+	ctx->ddc->ddc_pin = pin;
+	ctx->ddc->ctx = ctx->dc_ctx;
+	ctx->ddc->link = ctx->link;
+	ctx->dc_ctx->dc = ctx->dc;
+	ctx->link->link_index = 0;
+	ctx->link->ddc = link_ddc;
+	ctx->dc->links[0] = ctx->link;
+	ctx->i2c->ddc_service = ctx->ddc;
+	i2c_set_adapdata(&ctx->i2c->base, ctx->i2c);
 
-	KUNIT_EXPECT_EQ(test, get_amd_vsdb(aconnector, &vsdb_info), 0);
-	KUNIT_EXPECT_EQ(test, vsdb_info.amd_vsdb_version, 0);
+	return ctx;
 }
 
 /**
- * dm_test_get_amd_vsdb_supported - Test a non-zero VSDB version is reported
+ * dm_test_i2c_xfer_hw_submit_fails - Test the non-OEM path reports a failed submit
  * @test: The KUnit test context
  *
- * The display info's VSDB version and replay mode are copied out and a
- * non-zero version reports support.
+ * With a ddc_pin present the transfer builds the payload list and dispatches
+ * through dc_submit_i2c(); the target link has no ddc_pin, so the submit fails
+ * and the transfer returns -EIO instead of the message count.
  */
-static void dm_test_get_amd_vsdb_supported(struct kunit *test)
+static void dm_test_i2c_xfer_hw_submit_fails(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector;
-	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
-
-	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, aconnector);
-
-	aconnector->base.display_info.amd_vsdb.version = 2;
-	aconnector->base.display_info.amd_vsdb.replay_mode = true;
+	struct dm_test_i2c_ctx *ctx = dm_test_i2c_setup(test);
+	u8 buf[2];
+	struct i2c_msg msgs[] = {
+		{ .addr = 0x50, .flags = I2C_M_RD, .len = sizeof(buf), .buf = buf },
+	};
 
-	KUNIT_EXPECT_EQ(test, get_amd_vsdb(aconnector, &vsdb_info), 1);
-	KUNIT_EXPECT_EQ(test, vsdb_info.amd_vsdb_version, 2);
-	KUNIT_EXPECT_TRUE(test, vsdb_info.replay_mode);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_i2c_xfer(&ctx->i2c->base, msgs, 1), -EIO);
 }
 
 /**
- * dm_test_parse_hdmi_amd_vsdb_null_edid - Test NULL EDID returns -ENODEV
+ * dm_test_i2c_xfer_oem_no_device - Test the OEM path with no OEM device fails
  * @test: The KUnit test context
+ *
+ * When the adapter is flagged as OEM the transfer routes through
+ * dc_submit_i2c_oem(); with no oem_device on the resource pool the submit
+ * fails and the transfer returns -EIO.
  */
-static void dm_test_parse_hdmi_amd_vsdb_null_edid(struct kunit *test)
+static void dm_test_i2c_xfer_oem_no_device(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector;
-	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
+	struct dm_test_i2c_ctx *ctx = dm_test_i2c_setup(test);
+	struct resource_pool *pool;
+	u8 buf[2] = { 0x12, 0x34 };
+	struct i2c_msg msgs[] = {
+		{ .addr = 0x50, .flags = 0, .len = sizeof(buf), .buf = buf },
+	};
 
-	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, pool);
+	ctx->dc->res_pool = pool;
+	ctx->i2c->oem = true;
 
-	KUNIT_EXPECT_EQ(test,
-			parse_hdmi_amd_vsdb(aconnector, NULL, &vsdb_info),
-			-ENODEV);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_i2c_xfer(&ctx->i2c->base, msgs, 1), -EIO);
 }
 
-/**
- * dm_test_parse_hdmi_amd_vsdb_no_extensions - Test EDID without extensions
- * @test: The KUnit test context
- *
- * An EDID that declares no extension blocks has no CEA block to parse.
+/* Fake ddc_service chain for amdgpu_dm_create_i2c() tests. */
+struct dm_test_create_i2c_ctx {
+	struct ddc_service *ddc;
+	struct amdgpu_device *adev;
+	struct pci_dev *pdev;
+	struct dc_link *link;
+};
+
+/*
+ * Build a ddc_service whose dc_context driver_context is an amdgpu_device with
+ * a pci_dev (for the parent device), plus a link used by the hardware-bus name.
  */
-static void dm_test_parse_hdmi_amd_vsdb_no_extensions(struct kunit *test)
+static struct dm_test_create_i2c_ctx *dm_test_create_i2c_setup(struct kunit *test)
 {
-	struct amdgpu_dm_connector *aconnector;
-	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
-	struct edid *edid;
+	struct dm_test_create_i2c_ctx *ctx;
+	struct dc_context *dc_ctx;
 
-	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->ddc = kunit_kzalloc(test, sizeof(*ctx->ddc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->ddc);
+	ctx->adev = kunit_kzalloc(test, sizeof(*ctx->adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->adev);
+	ctx->pdev = kunit_kzalloc(test, sizeof(*ctx->pdev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->pdev);
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	dc_ctx = kunit_kzalloc(test, sizeof(*dc_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_ctx);
+
+	ctx->adev->pdev = ctx->pdev;
+	dc_ctx->driver_context = ctx->adev;
+	ctx->ddc->ctx = dc_ctx;
+	ctx->ddc->link = ctx->link;
+
+	return ctx;
+}
+
+/**
+ * dm_test_create_i2c_oem - Test the OEM adapter name and wiring
+ * @test: The KUnit test context
+ *
+ * The OEM adapter is named for the OEM bus and is wired to the dm i2c
+ * algorithm, the backing ddc_service, and the pci parent device.
+ */
+static void dm_test_create_i2c_oem(struct kunit *test)
+{
+	struct dm_test_create_i2c_ctx *ctx = dm_test_create_i2c_setup(test);
+	struct amdgpu_i2c_adapter *i2c;
+
+	i2c = amdgpu_dm_create_i2c(ctx->ddc, true);
+	KUNIT_ASSERT_NOT_NULL(test, i2c);
+
+	KUNIT_EXPECT_TRUE(test, i2c->oem);
+	KUNIT_EXPECT_PTR_EQ(test, i2c->ddc_service, ctx->ddc);
+	KUNIT_EXPECT_PTR_EQ(test, i2c_get_adapdata(&i2c->base), i2c);
+	KUNIT_EXPECT_PTR_EQ(test, i2c->base.dev.parent, &ctx->adev->pdev->dev);
+	KUNIT_EXPECT_TRUE(test, i2c->base.algo->master_xfer == amdgpu_dm_i2c_xfer);
+	KUNIT_EXPECT_TRUE(test, i2c->base.algo->functionality == amdgpu_dm_i2c_func);
+	KUNIT_EXPECT_STREQ(test, i2c->base.name, "AMDGPU DM i2c OEM bus");
+
+	kfree(i2c);
+}
+
+/**
+ * dm_test_create_i2c_hw_bus - Test the hardware bus adapter name carries the index
+ * @test: The KUnit test context
+ *
+ * The non-OEM adapter is named for the hardware bus and embeds the link index,
+ * and the OEM flag is left clear.
+ */
+static void dm_test_create_i2c_hw_bus(struct kunit *test)
+{
+	struct dm_test_create_i2c_ctx *ctx = dm_test_create_i2c_setup(test);
+	struct amdgpu_i2c_adapter *i2c;
+
+	ctx->link->link_index = 7;
+
+	i2c = amdgpu_dm_create_i2c(ctx->ddc, false);
+	KUNIT_ASSERT_NOT_NULL(test, i2c);
+
+	KUNIT_EXPECT_FALSE(test, i2c->oem);
+	KUNIT_EXPECT_PTR_EQ(test, i2c->ddc_service, ctx->ddc);
+	KUNIT_EXPECT_STREQ(test, i2c->base.name, "AMDGPU DM i2c hw bus 7");
+
+	kfree(i2c);
+}
+
+/**
+ * dm_test_restore_state_writeback - Test writeback connectors are skipped
+ * @test: The KUnit test context
+ *
+ * A writeback connector short-circuits before dc_sink is ever read, so leaving
+ * it NULL must not crash and no connector state is created.
+ */
+static void dm_test_restore_state_writeback(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_WRITEBACK);
+
+	dm_restore_drm_connector_state(drm, &aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, aconnector->base.state);
+}
+
+/**
+ * dm_test_restore_state_no_dc_sink - Test a connector without a dc_sink is a no-op
+ * @test: The KUnit test context
+ *
+ * With no dc_sink there is nothing to restore, so the function returns before
+ * touching the connector state or encoder.
+ */
+static void dm_test_restore_state_no_dc_sink(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_HDMIA);
+	/* dc_sink left NULL by kzalloc. */
+
+	dm_restore_drm_connector_state(drm, &aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, aconnector->base.state);
+}
+
+/**
+ * dm_test_restore_state_no_connector_state - Test a NULL connector state bails out
+ * @test: The KUnit test context
+ *
+ * A dc_sink is present but the connector has no atomic state, so the function
+ * returns before dereferencing the encoder.
+ */
+static void dm_test_restore_state_no_connector_state(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_HDMIA);
+	aconnector->dc_sink = kunit_kzalloc(test, 1, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->dc_sink);
+	/* connector->state left NULL: the guard must catch it. */
+
+	dm_restore_drm_connector_state(drm, &aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, aconnector->base.encoder);
+}
+
+/**
+ * dm_test_restore_state_no_encoder - Test a NULL encoder bails out
+ * @test: The KUnit test context
+ *
+ * A dc_sink and connector state are present but the connector is not routed to
+ * any encoder, so the function returns before reading the encoder's crtc.
+ */
+static void dm_test_restore_state_no_encoder(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_HDMIA);
+	aconnector->dc_sink = kunit_kzalloc(test, 1, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->dc_sink);
+	aconnector->base.funcs->reset(&aconnector->base);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->base.state);
+	/* connector->encoder left NULL. */
+
+	dm_restore_drm_connector_state(drm, &aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, aconnector->base.encoder);
+}
+
+/**
+ * dm_test_restore_state_no_stream - Test a crtc without a stream bails out
+ * @test: The KUnit test context
+ *
+ * The connector is routed to an encoder and crtc, but the crtc state carries no
+ * dc stream, so the function returns before comparing sinks.
+ */
+static void dm_test_restore_state_no_stream(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_crtc_state *acrtc_state;
+	struct drm_encoder *enc;
+	struct drm_crtc *crtc;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_HDMIA);
+	aconnector->dc_sink = kunit_kzalloc(test, 1, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->dc_sink);
+	aconnector->base.funcs->reset(&aconnector->base);
+
+	enc = kunit_kzalloc(test, sizeof(*enc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, enc);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	acrtc_state = kunit_kzalloc(test, sizeof(*acrtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc_state);
+
+	crtc->state = &acrtc_state->base;
+	enc->crtc = crtc;
+	aconnector->base.encoder = enc;
+	/* acrtc_state->stream left NULL. */
+
+	dm_restore_drm_connector_state(drm, &aconnector->base);
+
+	KUNIT_EXPECT_NULL(test, acrtc_state->stream);
+}
+
+/**
+ * dm_test_restore_state_same_sink - Test an unchanged sink skips the commit
+ * @test: The KUnit test context
+ *
+ * When the streamed sink already matches the connector's dc_sink there is
+ * nothing to restore, so the forced atomic commit is not issued.
+ */
+static void dm_test_restore_state_same_sink(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_crtc_state *acrtc_state;
+	struct dc_stream_state *stream;
+	struct drm_encoder *enc;
+	struct drm_crtc *crtc;
+
+	aconnector = dm_test_add_connector(test, drm,
+					   DRM_MODE_CONNECTOR_HDMIA);
+	aconnector->dc_sink = kunit_kzalloc(test, 1, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector->dc_sink);
+	aconnector->base.funcs->reset(&aconnector->base);
+
+	enc = kunit_kzalloc(test, sizeof(*enc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, enc);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	acrtc_state = kunit_kzalloc(test, sizeof(*acrtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc_state);
+	stream = dm_kunit_alloc_stream(test, NULL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	/* Same sink as the connector: the final branch is not taken. */
+	stream->sink = aconnector->dc_sink;
+	acrtc_state->stream = stream;
+	crtc->state = &acrtc_state->base;
+	enc->crtc = crtc;
+	aconnector->base.encoder = enc;
+
+	dm_restore_drm_connector_state(drm, &aconnector->base);
+
+	KUNIT_EXPECT_PTR_EQ(test, stream->sink, aconnector->dc_sink);
+}
+
+/**
+ * dm_test_get_amd_vsdb_unsupported - Test a zero VSDB version reports no support
+ * @test: The KUnit test context
+ */
+static void dm_test_get_amd_vsdb_unsupported(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, aconnector);
-	edid = kunit_kzalloc(test, sizeof(*edid), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, edid);
 
-	edid->extensions = 0;
+	aconnector->base.display_info.amd_vsdb.version = 0;
+	aconnector->base.display_info.amd_vsdb.replay_mode = false;
 
-	KUNIT_EXPECT_EQ(test,
-			parse_hdmi_amd_vsdb(aconnector, edid, &vsdb_info),
-			-ENODEV);
+	KUNIT_EXPECT_EQ(test, get_amd_vsdb(aconnector, &vsdb_info), 0);
+	KUNIT_EXPECT_EQ(test, vsdb_info.amd_vsdb_version, 0);
 }
 
 /**
- * dm_test_parse_hdmi_amd_vsdb_no_cea_ext - Test EDID with no CEA extension
+ * dm_test_get_amd_vsdb_supported - Test a non-zero VSDB version is reported
  * @test: The KUnit test context
  *
- * An extension block that is not a CEA block leaves no VSDB to parse.
+ * The display info's VSDB version and replay mode are copied out and a
+ * non-zero version reports support.
  */
-static void dm_test_parse_hdmi_amd_vsdb_no_cea_ext(struct kunit *test)
+static void dm_test_get_amd_vsdb_supported(struct kunit *test)
 {
 	struct amdgpu_dm_connector *aconnector;
 	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
-	struct edid *edid;
-	u8 *raw;
 
 	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, aconnector);
 
-	/* Base block + one extension block that is NOT a CEA extension. */
-	raw = kunit_kzalloc(test, 2 * EDID_LENGTH, GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, raw);
-	edid = (struct edid *)raw;
-	edid->extensions = 1;
-	raw[EDID_LENGTH] = DM_TEST_DISPLAYID_EXT;
+	aconnector->base.display_info.amd_vsdb.version = 2;
+	aconnector->base.display_info.amd_vsdb.replay_mode = true;
 
-	KUNIT_EXPECT_EQ(test,
-			parse_hdmi_amd_vsdb(aconnector, edid, &vsdb_info),
-			-ENODEV);
+	KUNIT_EXPECT_EQ(test, get_amd_vsdb(aconnector, &vsdb_info), 1);
+	KUNIT_EXPECT_EQ(test, vsdb_info.amd_vsdb_version, 2);
+	KUNIT_EXPECT_TRUE(test, vsdb_info.replay_mode);
 }
 
 /**
@@ -5175,250 +6599,3186 @@ static void dm_test_create_validate_stream_null_dm_state(struct kunit *test)
 								    NULL, NULL, NULL));
 }
 
-/**
- * dm_test_update_after_detect_mst_noop - Test MST connectors are left to drm_mst
- * @test: The KUnit test context
- *
- * An MST connector is handled by the drm_mst framework, so the function
- * returns immediately and never dereferences the (NULL) dc_link.
- */
-static void dm_test_update_after_detect_mst_noop(struct kunit *test)
-{
+/* Tests for amdgpu_dm_create_validate_stream_for_sink() */
+
+/*
+ * Build a connector embedded in an amdgpu_device (so drm_to_adev() resolves)
+ * carrying a dc_link but a deliberately low atomic-requested bpc. Every
+ * candidate colour depth then exceeds that cap, so bpc_mask ends up empty and
+ * the enumeration returns NULL before create_stream_for_sink() and
+ * dc_validate_stream() (and thus the unpopulated dc handle) are ever reached.
+ * This lets the encoding/bpc mask-building branches be exercised on their own.
+ */
+struct dm_test_cvs_ctx {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
 	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	struct dm_connector_state *dm_state;
+	struct drm_display_mode *mode;
+};
 
-	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+static struct dm_test_cvs_ctx *
+dm_test_cvs_ctx_alloc(struct kunit *test, int connector_type)
+{
+	struct dm_test_cvs_ctx *ctx;
+	struct device *dev;
 
-	aconnector->mst_mgr.mst_state = true;
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
 
-	amdgpu_dm_update_connector_after_detect(aconnector);
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+			sizeof(*ctx->adev),
+			offsetof(struct amdgpu_device, ddev),
+			DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+	ctx->adev = drm_to_adev(ctx->drm);
+
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector),
+			GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+			&dm_test_connector_funcs, connector_type,
+			NULL), 0);
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->aconnector->dc_link = ctx->link;
+
+	ctx->dm_state = kunit_kzalloc(test, sizeof(*ctx->dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dm_state);
+	ctx->dm_state->base.max_requested_bpc = 4;
+
+	ctx->mode = kunit_kzalloc(test, sizeof(*ctx->mode), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->mode);
+	ctx->mode->hdisplay = 1920;
+	ctx->mode->vdisplay = 1080;
+
+	return ctx;
 }
 
 /**
- * dm_test_update_after_detect_sink_unchanged - Test the short-pulse no-op path
+ * dm_test_create_validate_stream_writeback - Test the writeback stream path
  * @test: The KUnit test context
  *
- * When the link reports no local sink and the connector already has no
- * dc_sink, the "sink didn't change" path returns without touching DC.
+ * A writeback connector has no sink EDID to enumerate, so the helper builds and
+ * returns a single RGB stream directly instead of running the validation loop.
  */
-static void dm_test_update_after_detect_sink_unchanged(struct kunit *test)
+static void dm_test_create_validate_stream_writeback(struct kunit *test)
 {
-	struct drm_device *drm = dm_test_alloc_drm(test);
-	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_dm_wb_connector *wbcon;
+	struct dm_connector_state *dm_state;
+	struct drm_display_mode *mode;
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct dc_context *dc_ctx;
+	struct drm_device *drm;
 	struct dc_link *link;
+	struct device *dev;
 
-	aconnector = dm_test_add_connector(test, drm, DRM_MODE_CONNECTOR_HDMIA);
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	drm = __drm_kunit_helper_alloc_drm_device(test, dev, sizeof(*adev),
+			offsetof(struct amdgpu_device, ddev),
+			DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, drm);
+	adev = drm_to_adev(drm);
+
+	wbcon = drmm_kzalloc(drm, sizeof(*wbcon), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, wbcon);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(drm, &wbcon->base.base,
+			&dm_test_connector_funcs,
+			DRM_MODE_CONNECTOR_WRITEBACK, NULL), 0);
+
+	dc_ctx = kunit_kzalloc(test, sizeof(*dc_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_ctx);
 	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, link);
+	link->ctx = dc_ctx;
+	link->connector_signal = SIGNAL_TYPE_VIRTUAL;
+	wbcon->link = link;
 
-	aconnector->dc_link = link;
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	dm_state->scaling = RMX_OFF;
 
-	/* link->local_sink and aconnector->dc_sink are both NULL. */
-	amdgpu_dm_update_connector_after_detect(aconnector);
+	mode = kunit_kzalloc(test, sizeof(*mode), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mode);
+	mode->hdisplay = 1920;
+	mode->vdisplay = 1080;
+	mode->clock = 148500;
 
-	KUNIT_EXPECT_NULL(test, aconnector->dc_sink);
+	stream = amdgpu_dm_create_validate_stream_for_sink(&wbcon->base.base,
+							   mode, dm_state, NULL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+	dc_stream_release(stream);
 }
 
-/* Tests for amdgpu_dm_update_stream_scaling_settings() */
-
 /**
- * dm_test_update_scaling_null_mode - Test NULL mode leaves the stream rects untouched
+ * dm_test_create_validate_stream_no_valid_bpc - Test the exhausted-mask path
  * @test: The KUnit test context
+ *
+ * On a plain DisplayPort sink using the default RGB encoding but a bpc cap below
+ * every candidate depth, no (encoding, bpc) pair is attempted and the helper
+ * returns NULL without touching the dc handle.
  */
-static void dm_test_update_scaling_null_mode(struct kunit *test)
+static void dm_test_create_validate_stream_no_valid_bpc(struct kunit *test)
 {
-	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
-	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dm_test_cvs_ctx *ctx =
+		dm_test_cvs_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_stream_state *stream;
 
-	stream->timing.h_addressable = 1920;
-	stream->timing.v_addressable = 1080;
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
 
-	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, NULL, NULL, stream);
+	stream = amdgpu_dm_create_validate_stream_for_sink(&ctx->aconnector->base,
+							   ctx->mode,
+							   ctx->dm_state, NULL);
+	KUNIT_EXPECT_NULL(test, stream);
+}
 
-	/* NULL mode: early return before touching src/dst */
-	KUNIT_EXPECT_EQ(test, stream->src.width, 0);
-	KUNIT_EXPECT_EQ(test, stream->dst.width, 0);
+/**
+ * dm_test_create_validate_stream_hdmi_ycbcr - Test the HDMI encoding mask
+ * @test: The KUnit test context
+ *
+ * A native HDMI sink advertising YCbCr 4:4:4 and 4:2:2 exercises HDMI endpoint
+ * detection and the YCbCr444/YCbCr422 mask branches; the low bpc cap still
+ * prunes every depth, so the helper returns NULL.
+ */
+static void dm_test_create_validate_stream_hdmi_ycbcr(struct kunit *test)
+{
+	struct dm_test_cvs_ctx *ctx =
+		dm_test_cvs_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_stream_state *stream;
+
+	ctx->link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	ctx->aconnector->base.display_info.color_formats =
+		BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) |
+		BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422);
+
+	stream = amdgpu_dm_create_validate_stream_for_sink(&ctx->aconnector->base,
+							   ctx->mode,
+							   ctx->dm_state, NULL);
+	KUNIT_EXPECT_NULL(test, stream);
 }
 
 /**
- * dm_test_update_scaling_fullscreen_default - Test full-screen default with no dm_state
+ * dm_test_create_validate_stream_force_ycbcr420 - Test the forced YCbCr420 mask
  * @test: The KUnit test context
+ *
+ * force_yuv_pixel_format pins the encoding mask to YCbCr420 even on an RGB sink;
+ * the low bpc cap prunes every depth, so the helper returns NULL.
  */
-static void dm_test_update_scaling_fullscreen_default(struct kunit *test)
+static void dm_test_create_validate_stream_force_ycbcr420(struct kunit *test)
 {
-	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
-	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
-	struct drm_display_mode mode = { 0 };
+	struct dm_test_cvs_ctx *ctx =
+		dm_test_cvs_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_stream_state *stream;
 
-	mode.hdisplay = 1920;
-	mode.vdisplay = 1080;
-	stream->timing.h_addressable = 2560;
-	stream->timing.v_addressable = 1440;
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	ctx->aconnector->force_yuv_pixel_format = PIXEL_ENCODING_YCBCR420;
 
-	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, NULL, stream);
+	stream = amdgpu_dm_create_validate_stream_for_sink(&ctx->aconnector->base,
+							   ctx->mode,
+							   ctx->dm_state, NULL);
+	KUNIT_EXPECT_NULL(test, stream);
+}
 
-	/* src = mode, dst = timing addressable, no centering without dm_state */
-	KUNIT_EXPECT_EQ(test, stream->src.width, 1920);
-	KUNIT_EXPECT_EQ(test, stream->src.height, 1080);
-	KUNIT_EXPECT_EQ(test, stream->dst.width, 2560);
-	KUNIT_EXPECT_EQ(test, stream->dst.height, 1440);
-	KUNIT_EXPECT_EQ(test, stream->dst.x, 0);
-	KUNIT_EXPECT_EQ(test, stream->dst.y, 0);
+/**
+ * dm_test_create_validate_stream_force_ycbcr422 - Test the forced YCbCr422 mask
+ * @test: The KUnit test context
+ *
+ * With the sink advertising YCbCr 4:2:2, a force_yuv override pins the encoding
+ * mask to YCbCr422; the low bpc cap prunes every depth, so the helper returns
+ * NULL.
+ */
+static void dm_test_create_validate_stream_force_ycbcr422(struct kunit *test)
+{
+	struct dm_test_cvs_ctx *ctx =
+		dm_test_cvs_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_stream_state *stream;
+
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	ctx->aconnector->force_yuv_pixel_format = PIXEL_ENCODING_YCBCR422;
+	ctx->aconnector->base.display_info.color_formats =
+		BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422);
+
+	stream = amdgpu_dm_create_validate_stream_for_sink(&ctx->aconnector->base,
+							   ctx->mode,
+							   ctx->dm_state, NULL);
+	KUNIT_EXPECT_NULL(test, stream);
 }
 
 /**
- * dm_test_update_scaling_rmx_full - Test RMX_FULL keeps a full-size, centered dst
+ * dm_test_create_validate_stream_force_ycbcr444 - Test the forced YCbCr444 mask
  * @test: The KUnit test context
+ *
+ * On a native HDMI sink advertising YCbCr 4:4:4, a force_yuv override pins the
+ * encoding mask to YCbCr444; the low bpc cap prunes every depth, so the helper
+ * returns NULL.
  */
-static void dm_test_update_scaling_rmx_full(struct kunit *test)
+static void dm_test_create_validate_stream_force_ycbcr444(struct kunit *test)
 {
-	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
-	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dm_test_cvs_ctx *ctx =
+		dm_test_cvs_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_stream_state *stream;
+
+	ctx->link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	ctx->aconnector->force_yuv_pixel_format = PIXEL_ENCODING_YCBCR444;
+	ctx->aconnector->base.display_info.color_formats =
+		BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444);
+
+	stream = amdgpu_dm_create_validate_stream_for_sink(&ctx->aconnector->base,
+							   ctx->mode,
+							   ctx->dm_state, NULL);
+	KUNIT_EXPECT_NULL(test, stream);
+}
+
+/*
+ * Drive the enumeration loop far enough to reach create_stream_for_sink() and
+ * dc_validate_stream() by providing a valid bpc plus a minimal fake dc. The
+ * connector carries a dc_link with a dc_context so a fake VIRTUAL sink and its
+ * stream can be built; adev->dm.dc is wired with just enough resource_pool /
+ * timing_generator / link_service state for dc_validate_stream() and
+ * dm_validate_stream_and_context() to run without a real pipe allocator.
+ */
+struct dm_test_cvs_dc {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
 	struct dm_connector_state *dm_state;
-	struct drm_display_mode mode = { 0 };
+	struct drm_display_mode *mode;
+	struct dc *dc;
+	struct timing_generator_funcs *tgfuncs;
+	struct link_service *link_srv;
+};
 
-	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+static bool dm_test_cvs_validate_timing_fail(struct timing_generator *tg,
+					     const struct dc_crtc_timing *timing)
+{
+	return false;
+}
 
-	mode.hdisplay = 1280;
-	mode.vdisplay = 720;
-	stream->timing.h_addressable = 1920;
-	stream->timing.v_addressable = 1080;
-	dm_state->scaling = RMX_FULL;
+static bool dm_test_cvs_validate_timing_ok(struct timing_generator *tg,
+					   const struct dc_crtc_timing *timing)
+{
+	return true;
+}
 
-	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+static enum dc_status dm_test_cvs_validate_mode_timing_ok(
+		const struct dc_stream_state *stream,
+		struct dc_link *link,
+		const struct dc_crtc_timing *timing)
+{
+	return DC_OK;
+}
 
-	/* RMX_FULL: dst stays full addressable, offset 0 */
-	KUNIT_EXPECT_EQ(test, stream->dst.width, 1920);
-	KUNIT_EXPECT_EQ(test, stream->dst.height, 1080);
-	KUNIT_EXPECT_EQ(test, stream->dst.x, 0);
-	KUNIT_EXPECT_EQ(test, stream->dst.y, 0);
+static struct dm_test_cvs_dc *dm_test_cvs_dc_alloc(struct kunit *test)
+{
+	struct timing_generator_funcs *tgfuncs;
+	struct resource_funcs *rfuncs;
+	struct resource_caps *rcaps;
+	struct resource_pool *pool;
+	struct timing_generator *tg;
+	struct dc_context *dcc;
+	struct dal_logger *logger;
+	struct dm_test_cvs_dc *c;
+	struct device *dev;
+
+	c = kunit_kzalloc(test, sizeof(*c), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, c);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	c->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+			sizeof(*c->adev),
+			offsetof(struct amdgpu_device, ddev),
+			DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, c->drm);
+	c->adev = drm_to_adev(c->drm);
+
+	c->aconnector = drmm_kzalloc(c->drm, sizeof(*c->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, c->aconnector);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(c->drm, &c->aconnector->base,
+			&dm_test_connector_funcs,
+			DRM_MODE_CONNECTOR_DisplayPort, NULL), 0);
+	c->aconnector->base.display_info.bpc = 8;
+
+	dcc = kunit_kzalloc(test, sizeof(*dcc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dcc);
+	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, logger);
+	logger->dev = c->drm;
+	dcc->logger = logger;
+
+	c->link = kunit_kzalloc(test, sizeof(*c->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, c->link);
+	c->link->ctx = dcc;
+	c->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	c->link->ep_type = DISPLAY_ENDPOINT_UNKNOWN;
+	c->aconnector->dc_link = c->link;
+
+	pool = kunit_kzalloc(test, sizeof(*pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, pool);
+	rfuncs = kunit_kzalloc(test, sizeof(*rfuncs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, rfuncs);
+	rcaps = kunit_kzalloc(test, sizeof(*rcaps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, rcaps);
+	tg = kunit_kzalloc(test, sizeof(*tg), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, tg);
+	tgfuncs = kunit_kzalloc(test, sizeof(*tgfuncs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, tgfuncs);
+	c->tgfuncs = tgfuncs;
+	c->link_srv = kunit_kzalloc(test, sizeof(*c->link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, c->link_srv);
+
+	tg->funcs = tgfuncs;
+	pool->funcs = rfuncs;
+	pool->res_cap = rcaps;
+	pool->timing_generators[0] = tg;
+	pool->timing_generator_count = 0;
+
+	c->dc = kunit_kzalloc(test, sizeof(*c->dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, c->dc);
+	c->dc->ctx = dcc;
+	c->dc->res_pool = pool;
+	c->dc->link_srv = c->link_srv;
+	c->adev->dm.dc = c->dc;
+
+	c->dm_state = kunit_kzalloc(test, sizeof(*c->dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, c->dm_state);
+	c->dm_state->base.max_requested_bpc = 8;
+	c->dm_state->scaling = RMX_OFF;
+
+	c->mode = kunit_kzalloc(test, sizeof(*c->mode), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, c->mode);
+	c->mode->hdisplay = 1920;
+	c->mode->vdisplay = 1080;
+	c->mode->clock = 148500;
+
+	return c;
+}
+
+/**
+ * dm_test_create_validate_stream_prune_timing - Test the dc_validate_stream prune
+ * @test: The KUnit test context
+ *
+ * A built stream fails dc_validate_stream() (timing_generator rejects the
+ * timing), so every candidate is pruned and released and the helper returns
+ * NULL once the enumeration is exhausted.
+ */
+static void dm_test_create_validate_stream_prune_timing(struct kunit *test)
+{
+	struct dm_test_cvs_dc *c = dm_test_cvs_dc_alloc(test);
+
+	c->tgfuncs->validate_timing = dm_test_cvs_validate_timing_fail;
+
+	KUNIT_EXPECT_NULL(test,
+			  amdgpu_dm_create_validate_stream_for_sink(&c->aconnector->base,
+								    c->mode,
+								    c->dm_state,
+								    NULL));
 }
 
 /**
- * dm_test_update_scaling_rmx_aspect_pillarbox - Test RMX_ASPECT preserves aspect ratio
+ * dm_test_create_validate_stream_prune_context - Test the context-validation prune
  * @test: The KUnit test context
+ *
+ * dc_validate_stream() succeeds but dm_validate_stream_and_context() fails (no
+ * pipe allocator), exercising the DC_OK sub-branches and MST check before the
+ * candidate is pruned; the exhausted enumeration returns NULL.
  */
-static void dm_test_update_scaling_rmx_aspect_pillarbox(struct kunit *test)
+static void dm_test_create_validate_stream_prune_context(struct kunit *test)
 {
-	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
-	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
-	struct dm_connector_state *dm_state;
-	struct drm_display_mode mode = { 0 };
+	struct dm_test_cvs_dc *c = dm_test_cvs_dc_alloc(test);
 
-	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	c->tgfuncs->validate_timing = dm_test_cvs_validate_timing_ok;
+	c->link_srv->validate_mode_timing = dm_test_cvs_validate_mode_timing_ok;
 
-	/* 4:3 source on a 16:9 panel -> pillarboxed */
-	mode.hdisplay = 1024;
-	mode.vdisplay = 768;
-	stream->timing.h_addressable = 1920;
-	stream->timing.v_addressable = 1080;
-	dm_state->scaling = RMX_ASPECT;
+	KUNIT_EXPECT_NULL(test,
+			  amdgpu_dm_create_validate_stream_for_sink(&c->aconnector->base,
+								    c->mode,
+								    c->dm_state,
+								    NULL));
+}
 
-	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+/* Further tests for amdgpu_dm_connector_mode_valid() */
 
-	/*
-	 * src.width*dst.height (1024*1080) < src.height*dst.width (768*1920):
-	 * width scaled to src.width*dst.height/src.height = 1440, height stays
-	 * 1080, centered horizontally at (1920-1440)/2 = 240.
+/**
+ * dm_test_mode_valid_no_dc_sink - Test the missing-sink rejection
+ * @test: The KUnit test context
+ *
+ * With no dc_sink and an unforced connector there is nothing to validate
+ * against, so mode_valid() logs and returns MODE_ERROR without reaching stream
+ * creation.
+ */
+static void dm_test_mode_valid_no_dc_sink(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector =
+		dm_test_add_connector(test, drm, DRM_MODE_CONNECTOR_HDMIA);
+	struct drm_display_mode *mode;
+
+	mode = kunit_kzalloc(test, sizeof(*mode), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mode);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_connector_mode_valid(&aconnector->base, mode),
+			MODE_ERROR);
+}
+
+/**
+ * dm_test_mode_valid_force_on_no_stream - Test the forced no-stream rejection
+ * @test: The KUnit test context
+ *
+ * A forced-on connector with dc_em_sink already set skips the EDID refresh and
+ * the missing-sink bail, reaching stream creation; with no validatable depth no
+ * stream is produced, so mode_valid() returns MODE_ERROR.
+ */
+static void dm_test_mode_valid_force_on_no_stream(struct kunit *test)
+{
+	struct dm_test_cvs_ctx *ctx =
+		dm_test_cvs_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_sink *em_sink;
+
+	em_sink = kunit_kzalloc(test, sizeof(*em_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	ctx->aconnector->dc_em_sink = em_sink;
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+
+	/* drm_connector_cleanup() kfree()s connector->state, so it must not
+	 * point at KUnit-managed memory.
 	 */
-	KUNIT_EXPECT_EQ(test, stream->dst.width, 1440);
-	KUNIT_EXPECT_EQ(test, stream->dst.height, 1080);
-	KUNIT_EXPECT_EQ(test, stream->dst.x, 240);
-	KUNIT_EXPECT_EQ(test, stream->dst.y, 0);
+	amdgpu_dm_connector_funcs_reset(&ctx->aconnector->base);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector->base.state);
+	ctx->aconnector->base.state->max_requested_bpc = 4;
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_connector_mode_valid(&ctx->aconnector->base,
+						       ctx->mode),
+			MODE_ERROR);
 }
 
 /**
- * dm_test_update_scaling_rmx_aspect_letterbox - Test RMX_ASPECT letterboxes wide sources
+ * dm_test_mode_valid_edid_mgmt_forced - Test the forced-connector EDID refresh
  * @test: The KUnit test context
+ *
+ * A forced connector with no emulated sink runs the one-time EDID mgmt refresh
+ * before validating; with no readable EDID no dc_sink appears and a non-on
+ * force still bails to MODE_ERROR.
  */
-static void dm_test_update_scaling_rmx_aspect_letterbox(struct kunit *test)
+static void dm_test_mode_valid_edid_mgmt_forced(struct kunit *test)
 {
-	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
-	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
-	struct dm_connector_state *dm_state;
-	struct drm_display_mode mode = { 0 };
+	struct dm_test_edid_ctx *ctx =
+		dm_test_edid_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct drm_display_mode *mode;
 
-	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	ctx->link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
 
-	/* 16:9 source on a 4:3 panel -> letterboxed */
-	mode.hdisplay = 1920;
-	mode.vdisplay = 1080;
-	stream->timing.h_addressable = 1024;
-	stream->timing.v_addressable = 768;
-	dm_state->scaling = RMX_ASPECT;
+	mode = kunit_kzalloc(test, sizeof(*mode), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mode);
 
-	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_connector_mode_valid(&ctx->aconnector->base, mode),
+			MODE_ERROR);
+}
 
-	KUNIT_EXPECT_EQ(test, stream->dst.width, 1024);
-	KUNIT_EXPECT_EQ(test, stream->dst.height, 576);
-	KUNIT_EXPECT_EQ(test, stream->dst.x, 0);
-	KUNIT_EXPECT_EQ(test, stream->dst.y, 96);
+/* Tests for amdgpu_dm_connector_atomic_check() */
+
+/*
+ * Build a single-connector atomic commit for amdgpu_dm_connector_atomic_check().
+ * The connector lives at index 0 so the hand-rolled connectors[] slot resolves
+ * both the old and new connector state; leaving crtc NULL keeps the check off
+ * the crtc-state machinery that a real atomic path would require.
+ */
+struct dm_test_conn_ac_ctx {
+	struct amdgpu_dm_connector *aconn;
+	struct drm_atomic_commit *state;
+	struct drm_connector_state *old_state;
+	struct drm_connector_state *new_state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+};
+
+static struct dm_test_conn_ac_ctx *
+dm_test_conn_ac_ctx_alloc(struct kunit *test, int connector_type)
+{
+	struct dm_test_conn_ac_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->aconn = kunit_kzalloc(test, sizeof(*ctx->aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconn);
+	ctx->old_state = kunit_kzalloc(test, sizeof(*ctx->old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->old_state);
+	ctx->new_state = kunit_kzalloc(test, sizeof(*ctx->new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->new_state);
+
+	ctx->state = dm_test_alloc_atomic_state(test, 1);
+
+	ctx->aconn->base.connector_type = connector_type;
+	ctx->new_state->connector = &ctx->aconn->base;
+	ctx->new_state->state = ctx->state;
+	ctx->old_state->connector = &ctx->aconn->base;
+
+	ctx->state->connectors[0].ptr = &ctx->aconn->base;
+	ctx->state->connectors[0].old_state = ctx->old_state;
+	ctx->state->connectors[0].new_state = ctx->new_state;
+
+	return ctx;
 }
 
 /**
- * dm_test_update_scaling_rmx_center - Test RMX_CENTER centers a 1:1 dst
+ * dm_test_conn_atomic_check_no_crtc - Test the unbound-connector short-circuit
  * @test: The KUnit test context
+ *
+ * A non-DisplayPort connector with no crtc has nothing to validate and returns
+ * 0 immediately.
  */
-static void dm_test_update_scaling_rmx_center(struct kunit *test)
+static void dm_test_conn_atomic_check_no_crtc(struct kunit *test)
 {
-	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
-	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
-	struct dm_connector_state *dm_state;
-	struct drm_display_mode mode = { 0 };
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	int ret;
 
-	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
 
-	mode.hdisplay = 1280;
-	mode.vdisplay = 720;
-	stream->timing.h_addressable = 1920;
-	stream->timing.v_addressable = 1080;
-	dm_state->scaling = RMX_CENTER;
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
 
-	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+/**
+ * dm_test_conn_atomic_check_dp_mst - Test the DisplayPort MST root check
+ * @test: The KUnit test context
+ *
+ * A DisplayPort connector runs the MST root atomic check; with no crtc bound on
+ * either state it finds nothing to reserve and returns 0.
+ */
+static void dm_test_conn_atomic_check_dp_mst(struct kunit *test)
+{
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	int ret;
 
-	/* RMX_CENTER: dst = src, centered on the addressable area */
-	KUNIT_EXPECT_EQ(test, stream->dst.width, 1280);
-	KUNIT_EXPECT_EQ(test, stream->dst.height, 720);
-	KUNIT_EXPECT_EQ(test, stream->dst.x, 320);
-	KUNIT_EXPECT_EQ(test, stream->dst.y, 180);
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
 }
 
 /**
- * dm_test_update_scaling_underscan - Test underscan borders shrink and offset dst
+ * dm_test_conn_atomic_check_no_change - Test the bound-but-unchanged path
  * @test: The KUnit test context
+ *
+ * With a crtc bound but no privacy/colorspace/content-type/HDR differences the
+ * check skips every modeset trigger and returns 0 without touching the crtc
+ * state.
  */
-static void dm_test_update_scaling_underscan(struct kunit *test)
+static void dm_test_conn_atomic_check_no_change(struct kunit *test)
 {
-	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
-	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
-	struct dm_connector_state *dm_state;
-	struct drm_display_mode mode = { 0 };
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct drm_crtc *crtc;
+	int ret;
 
-	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
 
-	mode.hdisplay = 1920;
-	mode.vdisplay = 1080;
-	stream->timing.h_addressable = 1920;
-	stream->timing.v_addressable = 1080;
-	dm_state->scaling = RMX_FULL;
-	dm_state->underscan_enable = true;
-	dm_state->underscan_hborder = 64;
-	dm_state->underscan_vborder = 32;
+	ctx->new_state->crtc = crtc;
 
-	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
 
-	/* Full dst, then underscan: x/y += border/2, width/height -= border */
-	KUNIT_EXPECT_EQ(test, stream->dst.x, 32);
-	KUNIT_EXPECT_EQ(test, stream->dst.y, 16);
-	KUNIT_EXPECT_EQ(test, stream->dst.width, 1856);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/*
+ * Bind a crtc to the commit with a pre-populated crtc slot so
+ * drm_atomic_get_crtc_state() returns @crtc_state via its new-state fast path
+ * rather than locking and duplicating through absent crtc funcs.
+ */
+static void dm_test_conn_ac_bind_crtc(struct kunit *test,
+				      struct dm_test_conn_ac_ctx *ctx)
+{
+	ctx->crtc = kunit_kzalloc(test, sizeof(*ctx->crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc);
+	ctx->crtc_state = kunit_kzalloc(test, sizeof(*ctx->crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc_state);
+
+	ctx->state->crtcs = kunit_kcalloc(test, 1, sizeof(*ctx->state->crtcs),
+					  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->crtcs);
+	ctx->state->crtcs[0].ptr = ctx->crtc;
+	ctx->state->crtcs[0].new_state = ctx->crtc_state;
+
+	/* Only needs to be non-NULL to satisfy the get_crtc_state() WARN. */
+	ctx->state->acquire_ctx = (void *)ctx;
+
+	ctx->new_state->crtc = ctx->crtc;
+}
+
+/**
+ * dm_test_conn_atomic_check_privacy_change - Test privacy toggle forces modeset
+ * @test: The KUnit test context
+ *
+ * A changed privacy-screen software state pulls in the crtc state and flags a
+ * modeset.
+ */
+static void dm_test_conn_atomic_check_privacy_change(struct kunit *test)
+{
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	int ret;
+
+	dm_test_conn_ac_bind_crtc(test, ctx);
+	ctx->new_state->privacy_screen_sw_state = PRIVACY_SCREEN_ENABLED;
+
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, ctx->crtc_state->mode_changed);
+}
+
+/**
+ * dm_test_conn_atomic_check_colorspace_change - Test colorspace change forces modeset
+ * @test: The KUnit test context
+ *
+ * A changed output colorspace pulls in the crtc state and flags a modeset.
+ */
+static void dm_test_conn_atomic_check_colorspace_change(struct kunit *test)
+{
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	int ret;
+
+	dm_test_conn_ac_bind_crtc(test, ctx);
+	ctx->new_state->colorspace = DRM_MODE_COLORIMETRY_BT2020_RGB;
+
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, ctx->crtc_state->mode_changed);
+}
+
+/**
+ * dm_test_conn_atomic_check_content_type_change - Test content-type change forces modeset
+ * @test: The KUnit test context
+ *
+ * A changed content type pulls in the crtc state and flags a modeset.
+ */
+static void dm_test_conn_atomic_check_content_type_change(struct kunit *test)
+{
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	int ret;
+
+	dm_test_conn_ac_bind_crtc(test, ctx);
+	ctx->new_state->content_type = DRM_MODE_CONTENT_TYPE_GRAPHICS;
+
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, ctx->crtc_state->mode_changed);
+}
+
+/**
+ * dm_test_conn_atomic_check_hdr_exit - Test exiting HDR forces a modeset
+ * @test: The KUnit test context
+ *
+ * Clearing previously set HDR metadata makes the metadata unequal; the fill
+ * succeeds for the empty new state and the enter/exit rule flags a modeset.
+ */
+static void dm_test_conn_atomic_check_hdr_exit(struct kunit *test)
+{
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct drm_property_blob *blob;
+	int ret;
+
+	blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, blob);
+
+	dm_test_conn_ac_bind_crtc(test, ctx);
+	ctx->old_state->hdr_output_metadata = blob;
+
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_TRUE(test, ctx->crtc_state->mode_changed);
+}
+
+/**
+ * dm_test_conn_atomic_check_hdr_fill_error - Test infopacket fill errors propagate
+ * @test: The KUnit test context
+ *
+ * A new HDR metadata blob with no payload makes the infopacket fill fail, and
+ * that error is returned before the crtc state is touched.
+ */
+static void dm_test_conn_atomic_check_hdr_fill_error(struct kunit *test)
+{
+	struct dm_test_conn_ac_ctx *ctx =
+		dm_test_conn_ac_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct drm_property_blob *blob;
+	int ret;
+
+	blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, blob);
+
+	dm_test_conn_ac_bind_crtc(test, ctx);
+	ctx->new_state->hdr_output_metadata = blob;
+
+	ret = amdgpu_dm_connector_atomic_check(&ctx->aconn->base, ctx->state);
+
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_update_after_detect_mst_noop - Test MST connectors are left to drm_mst
+ * @test: The KUnit test context
+ *
+ * An MST connector is handled by the drm_mst framework, so the function
+ * returns immediately and never dereferences the (NULL) dc_link.
+ */
+static void dm_test_update_after_detect_mst_noop(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	aconnector->mst_mgr.mst_state = true;
+
+	amdgpu_dm_update_connector_after_detect(aconnector);
+}
+
+/**
+ * dm_test_update_after_detect_sink_unchanged - Test the short-pulse no-op path
+ * @test: The KUnit test context
+ *
+ * When the link reports no local sink and the connector already has no
+ * dc_sink, the "sink didn't change" path returns without touching DC.
+ */
+static void dm_test_update_after_detect_sink_unchanged(struct kunit *test)
+{
+	struct drm_device *drm = dm_test_alloc_drm(test);
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	aconnector = dm_test_add_connector(test, drm, DRM_MODE_CONNECTOR_HDMIA);
+	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link);
+
+	aconnector->dc_link = link;
+
+	/* link->local_sink and aconnector->dc_sink are both NULL. */
+	amdgpu_dm_update_connector_after_detect(aconnector);
+
+	KUNIT_EXPECT_NULL(test, aconnector->dc_sink);
+}
+
+/* Tests for amdgpu_dm_update_connector_after_detect() */
+
+/*
+ * A minimal but structurally valid 128-byte EDID base block (correct header
+ * and checksum) so drm_edid_alloc()/drm_edid_connector_update() accept it when
+ * exercising the "sink carries EDID" branch.
+ */
+static const u8 dm_test_uad_edid[128] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x04, 0x21, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x80, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c,
+};
+
+/*
+ * Build an amdgpu_dm_connector registered against a real kunit drm_device that
+ * is embedded in an amdgpu_device, so drm_to_adev()/adev_to_drm() resolve for
+ * amdgpu_dm_update_connector_after_detect() and all of its helpers.
+ *
+ * bl_idx is forced to -1 and adev->dm.freesync_module is left NULL so the
+ * backlight, CEC and freesync helpers take their early-return paths and the
+ * test stays focused on the sink-adoption logic.
+ */
+struct dm_test_uad_ctx {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+};
+
+static struct dm_test_uad_ctx *
+dm_test_uad_ctx_alloc(struct kunit *test, int connector_type)
+{
+	struct dm_test_uad_ctx *ctx;
+	struct device *dev;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+			sizeof(*ctx->adev),
+			offsetof(struct amdgpu_device, ddev),
+			DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+	ctx->adev = drm_to_adev(ctx->drm);
+
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector),
+			GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+			&dm_test_connector_funcs, connector_type,
+			NULL), 0);
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->aconnector->dc_link = ctx->link;
+
+	/* Keep the backlight/CEC/freesync helpers on their early-return paths. */
+	ctx->aconnector->bl_idx = -1;
+
+	return ctx;
+}
+
+/* A real (non-kunit) sink the function is expected to free via dc_sink_release. */
+static struct dc_sink *dm_test_uad_owned_sink(void)
+{
+	struct dc_sink *sink = kzalloc_obj(*sink, GFP_KERNEL);
+
+	if (sink)
+		kref_init(&sink->refcount);
+	return sink;
+}
+
+/* A kunit-managed sink that stays referenced (never released to zero). */
+static struct dc_sink *dm_test_uad_kept_sink(struct kunit *test)
+{
+	struct dc_sink *sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+
+	if (sink)
+		kref_init(&sink->refcount);
+	return sink;
+}
+
+/**
+ * dm_test_update_after_detect_mst_sink - Test an MST sink is left to drm_mst
+ * @test: The KUnit test context
+ *
+ * A local sink reporting SIGNAL_TYPE_DISPLAY_PORT_MST is handled by the
+ * drm_mst framework, so the function returns before adopting it.
+ */
+static void dm_test_update_after_detect_mst_sink(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *sink = dm_test_uad_kept_sink(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink->sink_signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+	ctx->link->local_sink = sink;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_NULL(test, ctx->aconnector->dc_sink);
+}
+
+/**
+ * dm_test_update_after_detect_connect_edid - Test adopting a new sink with EDID
+ * @test: The KUnit test context
+ *
+ * A freshly detected DisplayPort sink carrying EDID is adopted: the connector
+ * takes the sink, allocates a drm_edid and a requested-timing structure.
+ */
+static void dm_test_update_after_detect_connect_edid(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *sink = dm_test_uad_kept_sink(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink->sink_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	memcpy(sink->dc_edid.raw_edid, dm_test_uad_edid, sizeof(dm_test_uad_edid));
+	sink->dc_edid.length = sizeof(dm_test_uad_edid);
+	ctx->link->local_sink = sink;
+	ctx->link->aux_mode = true;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, sink);
+	KUNIT_EXPECT_NOT_NULL(test, ctx->aconnector->timing_requested);
+
+	kfree(ctx->aconnector->timing_requested);
+	ctx->aconnector->timing_requested = NULL;
+}
+
+/**
+ * dm_test_update_after_detect_replace_no_edid - Test replacing a sink with no
+ * EDID and HDMI compression auto
+ * @test: The KUnit test context
+ *
+ * When a new EDID-less sink replaces an existing one, the old sink is released
+ * and, with hdmi_comp_auto set, an HDMI sink signal is promoted to FRL.
+ */
+static void dm_test_update_after_detect_replace_no_edid(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *old_sink = dm_test_uad_owned_sink();
+	struct dc_sink *new_sink = dm_test_uad_kept_sink(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, old_sink);
+	KUNIT_ASSERT_NOT_NULL(test, new_sink);
+
+	ctx->aconnector->dc_sink = old_sink;
+	new_sink->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	new_sink->dc_edid.length = 0;
+	ctx->link->local_sink = new_sink;
+	ctx->link->aux_mode = true;
+	ctx->aconnector->hdmi_comp_auto = true;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, new_sink);
+	KUNIT_EXPECT_EQ(test, (int)new_sink->sink_signal,
+			(int)SIGNAL_TYPE_HDMI_FRL);
+}
+
+/**
+ * dm_test_update_after_detect_disconnect - Test tearing down on unplug
+ * @test: The KUnit test context
+ *
+ * With no local sink but an existing dc_sink, the disconnect path releases the
+ * sink, clears modes/timing, downgrades content protection and notifies audio.
+ */
+static void dm_test_update_after_detect_disconnect(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *old_sink = dm_test_uad_owned_sink();
+	struct dc_crtc_timing *timing = kzalloc_obj(*timing, GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, old_sink);
+	KUNIT_ASSERT_NOT_NULL(test, timing);
+
+	mutex_init(&ctx->adev->dm.audio_lock);
+	ctx->aconnector->dc_sink = old_sink;
+	ctx->aconnector->timing_requested = timing;
+	ctx->aconnector->num_modes = 3;
+	ctx->aconnector->audio_inst = 5;
+	ctx->link->local_sink = NULL;
+
+	amdgpu_dm_connector_funcs_reset(&ctx->aconnector->base);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector->base.state);
+	ctx->aconnector->base.state->content_protection =
+		DRM_MODE_CONTENT_PROTECTION_ENABLED;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_NULL(test, ctx->aconnector->dc_sink);
+	KUNIT_EXPECT_NULL(test, ctx->aconnector->timing_requested);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 0);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->audio_inst, -1);
+	KUNIT_EXPECT_EQ(test,
+			(int)ctx->aconnector->base.state->content_protection,
+			(int)DRM_MODE_CONTENT_PROTECTION_DESIRED);
+}
+
+/**
+ * dm_test_update_after_detect_force_em_adopt - Test forced eml_sink adoption
+ * @test: The KUnit test context
+ *
+ * A forced connector with an emulated sink adopts a newly reported local sink
+ * under the mode_config lock.
+ */
+static void dm_test_update_after_detect_force_em_adopt(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *em_sink = dm_test_uad_kept_sink(test);
+	struct dc_sink *sink = dm_test_uad_kept_sink(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+	ctx->aconnector->dc_em_sink = em_sink;
+	ctx->link->local_sink = sink;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, sink);
+}
+
+/**
+ * dm_test_update_after_detect_force_em_replace - Test forced eml_sink replace
+ * @test: The KUnit test context
+ *
+ * A forced connector that already has a dc_sink releases it before adopting the
+ * newly reported local sink.
+ */
+static void dm_test_update_after_detect_force_em_replace(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *em_sink = dm_test_uad_kept_sink(test);
+	struct dc_sink *old_sink = dm_test_uad_owned_sink();
+	struct dc_sink *new_sink = dm_test_uad_kept_sink(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+	KUNIT_ASSERT_NOT_NULL(test, old_sink);
+	KUNIT_ASSERT_NOT_NULL(test, new_sink);
+
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+	ctx->aconnector->dc_em_sink = em_sink;
+	ctx->aconnector->dc_sink = old_sink;
+	ctx->link->local_sink = new_sink;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, new_sink);
+}
+
+/**
+ * dm_test_update_after_detect_force_em_fake - Test forced fallback to eml_sink
+ * @test: The KUnit test context
+ *
+ * A forced connector with no local sink and no dc_sink falls back to using the
+ * emulated sink so a headless stream can still be faked.
+ */
+static void dm_test_update_after_detect_force_em_fake(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *em_sink = dm_test_uad_kept_sink(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+	ctx->aconnector->dc_em_sink = em_sink;
+	ctx->link->local_sink = NULL;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, em_sink);
+}
+
+/**
+ * dm_test_update_after_detect_force_em_keep - Test forced no-sink keeps dc_sink
+ * @test: The KUnit test context
+ *
+ * A forced connector with no local sink but an existing dc_sink keeps that sink
+ * (the emulated-sink fallback is skipped).
+ */
+static void dm_test_update_after_detect_force_em_keep(struct kunit *test)
+{
+	struct dm_test_uad_ctx *ctx =
+		dm_test_uad_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct dc_sink *em_sink = dm_test_uad_kept_sink(test);
+	struct dc_sink *dc_sink = dm_test_uad_kept_sink(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+	KUNIT_ASSERT_NOT_NULL(test, dc_sink);
+
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+	ctx->aconnector->dc_em_sink = em_sink;
+	ctx->aconnector->dc_sink = dc_sink;
+	ctx->link->local_sink = NULL;
+
+	amdgpu_dm_update_connector_after_detect(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, dc_sink);
+}
+
+/* Tests for amdgpu_dm_connector_funcs_force() with a valid EDID */
+
+/*
+ * Fake i2c adapter that serves dm_test_uad_edid over the DDC read protocol
+ * drm_do_probe_ddc_edid() uses: a one-byte write to DDC_ADDR (0x50) sets the
+ * block offset and the following read returns bytes from that offset. This
+ * lets amdgpu_dm_connector_funcs_force() obtain a valid EDID without hardware.
+ */
+static int dm_test_force_edid_i2c_xfer(struct i2c_adapter *adap,
+				       struct i2c_msg *msgs, int num)
+{
+	u8 offset = 0;
+	int i;
+
+	for (i = 0; i < num; i++) {
+		if (msgs[i].flags & I2C_M_RD) {
+			size_t len = msgs[i].len;
+
+			if (offset >= sizeof(dm_test_uad_edid))
+				len = 0;
+			else if (len > sizeof(dm_test_uad_edid) - offset)
+				len = sizeof(dm_test_uad_edid) - offset;
+			memcpy(msgs[i].buf, dm_test_uad_edid + offset, len);
+		} else if (msgs[i].addr == 0x50 && msgs[i].len >= 1) {
+			offset = msgs[i].buf[0];
+		}
+	}
+
+	return num;
+}
+
+static u32 dm_test_force_edid_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C;
+}
+
+static const struct i2c_algorithm dm_test_force_edid_i2c_algo = {
+	.master_xfer = dm_test_force_edid_i2c_xfer,
+	.functionality = dm_test_force_edid_i2c_func,
+};
+
+static void dm_test_force_edid_lock_bus(struct i2c_adapter *adap, unsigned int flags) {}
+static int dm_test_force_edid_trylock_bus(struct i2c_adapter *adap, unsigned int flags)
+{
+	return 1;
+}
+static void dm_test_force_edid_unlock_bus(struct i2c_adapter *adap, unsigned int flags) {}
+
+static const struct i2c_lock_operations dm_test_force_edid_lock_ops = {
+	.lock_bus = dm_test_force_edid_lock_bus,
+	.trylock_bus = dm_test_force_edid_trylock_bus,
+	.unlock_bus = dm_test_force_edid_unlock_bus,
+};
+
+/**
+ * dm_test_funcs_force_reads_edid - Test force() caches EDID and updates em_sink
+ * @test: The KUnit test context
+ *
+ * A non-AUX link selects the connector's i2c adapter, which serves a valid
+ * EDID. force() must cache it in drm_edid and, because an emulated sink and a
+ * dc_link are present, copy the raw EDID into the sink and parse its caps.
+ */
+static void dm_test_funcs_force_reads_edid(struct kunit *test)
+{
+	struct dm_test_edid_ctx *ctx =
+		dm_test_edid_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct amdgpu_i2c_adapter *i2c;
+	struct dc_sink *em_sink;
+
+	i2c = kunit_kzalloc(test, sizeof(*i2c), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, i2c);
+	i2c->base.algo = &dm_test_force_edid_i2c_algo;
+	i2c->base.lock_ops = &dm_test_force_edid_lock_ops;
+	ctx->aconnector->i2c = i2c;
+
+	em_sink = kunit_kzalloc(test, sizeof(*em_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+	ctx->aconnector->dc_em_sink = em_sink;
+
+	/* Non-AUX link picks the i2c adapter; force-on skips the DDC probe. */
+	ctx->link->aux_mode = false;
+	ctx->link->priv = ctx->aconnector;
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+
+	amdgpu_dm_connector_funcs_force(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_NOT_NULL(test, ctx->aconnector->drm_edid);
+	KUNIT_EXPECT_MEMEQ(test, em_sink->dc_edid.raw_edid,
+			   dm_test_uad_edid, sizeof(dm_test_uad_edid));
+
+	drm_edid_free(ctx->aconnector->drm_edid);
+	ctx->aconnector->drm_edid = NULL;
+}
+
+/* Tests for create_eml_sink() with a valid EDID */
+
+/*
+ * create_eml_sink() reads the EDID off the connector DDC and, on success,
+ * installs an emulated sink from link_srv->add_remote_sink(). A non-AUX link
+ * selects the connector i2c adapter (serving dm_test_uad_edid) and a fake
+ * add_remote_sink() returns dm_test_ces_remote_sink, so the full EDID path
+ * runs without real hardware or DC.
+ */
+static struct dc_sink *dm_test_ces_remote_sink;
+
+static struct dc_sink *
+dm_test_ces_add_remote_sink(struct dc_link *link, const uint8_t *edid,
+			    unsigned int len, struct dc_sink_init_data *init_data)
+{
+	return dm_test_ces_remote_sink;
+}
+
+static void dm_test_ces_setup(struct kunit *test, struct dm_test_edid_ctx *ctx,
+			      struct dc_sink *em_sink)
+{
+	struct amdgpu_i2c_adapter *i2c;
+	struct link_service *link_srv;
+	struct dc *dc;
+
+	i2c = kunit_kzalloc(test, sizeof(*i2c), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, i2c);
+	i2c->base.algo = &dm_test_force_edid_i2c_algo;
+	i2c->base.lock_ops = &dm_test_force_edid_lock_ops;
+	ctx->aconnector->i2c = i2c;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	link_srv->add_remote_sink = dm_test_ces_add_remote_sink;
+	dc->link_srv = link_srv;
+
+	ctx->link->dc = dc;
+	ctx->link->aux_mode = false;
+	dm_test_ces_remote_sink = em_sink;
+}
+
+/**
+ * dm_test_create_eml_sink_reads_edid - Test the valid-EDID path builds a sink
+ * @test: The KUnit test context
+ *
+ * With a readable EDID and force unspecified, create_eml_sink() caches the EDID
+ * in drm_edid and installs the emulated sink while leaving dc_sink untouched.
+ */
+static void dm_test_create_eml_sink_reads_edid(struct kunit *test)
+{
+	struct dm_test_edid_ctx *ctx =
+	dm_test_edid_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_sink *em_sink;
+
+	em_sink = kunit_kzalloc(test, sizeof(*em_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+	dm_test_ces_setup(test, ctx, em_sink);
+	ctx->aconnector->base.force = DRM_FORCE_UNSPECIFIED;
+
+	create_eml_sink(ctx->aconnector);
+
+	KUNIT_EXPECT_NOT_NULL(test, ctx->aconnector->drm_edid);
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_em_sink, em_sink);
+	KUNIT_EXPECT_NULL(test, ctx->aconnector->dc_sink);
+
+	drm_edid_free(ctx->aconnector->drm_edid);
+	ctx->aconnector->drm_edid = NULL;
+}
+
+/**
+ * dm_test_create_eml_sink_force_on_em - Test force-on adopts the emulated sink
+ * @test: The KUnit test context
+ *
+ * With DRM_FORCE_ON and no local sink, create_eml_sink() adopts the emulated
+ * sink as dc_sink and retains it.
+ */
+static void dm_test_create_eml_sink_force_on_em(struct kunit *test)
+{
+	struct dm_test_edid_ctx *ctx =
+	dm_test_edid_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_sink *em_sink;
+
+	em_sink = kunit_kzalloc(test, sizeof(*em_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+	kref_init(&em_sink->refcount);
+	dm_test_ces_setup(test, ctx, em_sink);
+	ctx->link->local_sink = NULL;
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+
+	create_eml_sink(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, em_sink);
+	KUNIT_EXPECT_EQ(test, kref_read(&em_sink->refcount), 2);
+
+	drm_edid_free(ctx->aconnector->drm_edid);
+	ctx->aconnector->drm_edid = NULL;
+}
+
+/**
+ * dm_test_create_eml_sink_force_on_local - Test force-on prefers the local sink
+ * @test: The KUnit test context
+ *
+ * With DRM_FORCE_ON and a local sink present, create_eml_sink() adopts the
+ * local sink as dc_sink instead of the emulated one and retains it.
+ */
+static void dm_test_create_eml_sink_force_on_local(struct kunit *test)
+{
+	struct dm_test_edid_ctx *ctx =
+	dm_test_edid_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct dc_sink *em_sink;
+	struct dc_sink *local_sink;
+
+	em_sink = kunit_kzalloc(test, sizeof(*em_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, em_sink);
+	local_sink = kunit_kzalloc(test, sizeof(*local_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, local_sink);
+	kref_init(&local_sink->refcount);
+	dm_test_ces_setup(test, ctx, em_sink);
+	ctx->link->local_sink = local_sink;
+	ctx->aconnector->base.force = DRM_FORCE_ON;
+
+	create_eml_sink(ctx->aconnector);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_sink, local_sink);
+	KUNIT_EXPECT_EQ(test, kref_read(&local_sink->refcount), 2);
+
+	drm_edid_free(ctx->aconnector->drm_edid);
+	ctx->aconnector->drm_edid = NULL;
+}
+
+/* Tests for amdgpu_dm_connector_get_modes() */
+
+static enum dp_link_encoding dm_test_gm_enc_8b10b(const struct dc_link_settings *s)
+{
+	return DP_8b_10b_ENCODING;
+}
+
+static enum dp_link_encoding dm_test_gm_enc_128b(const struct dc_link_settings *s)
+{
+	return DP_128b_132b_ENCODING;
+}
+
+/*
+ * Build an amdgpu_dm_connector on an amdgpu_device-backed drm device (so
+ * drm_to_adev() resolves for amdgpu_dm_fbc_init()) with an attached encoder
+ * and a dc/dc_link whose link_srv reports a non-128b encoding by default. The
+ * fbc compressor is left NULL so amdgpu_dm_fbc_init() early-returns.
+ */
+struct dm_test_gm_ctx {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_encoder *aenc;
+	struct dc *dc;
+	struct link_service *link_srv;
+	struct dc_link *link;
+};
+
+static struct dm_test_gm_ctx *
+dm_test_gm_ctx_alloc(struct kunit *test, int connector_type)
+{
+	struct dm_test_gm_ctx *ctx;
+	struct device *dev;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(*ctx->adev),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+	ctx->adev = drm_to_adev(ctx->drm);
+
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector),
+				       GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	ret = drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+				  &dm_test_connector_funcs, connector_type,
+				  NULL);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ctx->aenc = drmm_kzalloc(ctx->drm, sizeof(*ctx->aenc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aenc);
+	ret = drmm_encoder_init(ctx->drm, &ctx->aenc->base, NULL,
+				DRM_MODE_ENCODER_TMDS, NULL);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	ret = drm_connector_attach_encoder(&ctx->aconnector->base,
+					   &ctx->aenc->base);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ctx->dc = kunit_kzalloc(test, sizeof(*ctx->dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc);
+	ctx->link_srv = kunit_kzalloc(test, sizeof(*ctx->link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link_srv);
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+
+	ctx->link_srv->dp_get_encoding_format = dm_test_gm_enc_8b10b;
+	ctx->dc->link_srv = ctx->link_srv;
+	ctx->adev->dm.dc = ctx->dc;
+	ctx->link->dc = ctx->dc;
+	ctx->aconnector->dc_link = ctx->link;
+
+	return ctx;
+}
+
+/**
+ * dm_test_get_modes_noedid_default - Test synthesized modes without an EDID
+ * @test: The KUnit test context
+ *
+ * With no cached EDID and a non-128b link, get_modes() synthesizes the default
+ * 640x480 fallback mode(s) and reports a non-zero count.
+ */
+static void dm_test_get_modes_noedid_default(struct kunit *test)
+{
+	struct dm_test_gm_ctx *ctx =
+		dm_test_gm_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	int count;
+
+	mutex_lock(&ctx->drm->mode_config.mutex);
+	count = amdgpu_dm_connector_get_modes(&ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
+
+	KUNIT_EXPECT_GT(test, count, 0);
+}
+
+/**
+ * dm_test_get_modes_noedid_128b_adds_more - Test 128b links add 1080p modes
+ * @test: The KUnit test context
+ *
+ * A 128b/132b link synthesizes the extra 1920x1080 fallback modes, so the mode
+ * count is strictly greater than for an 8b/10b link.
+ */
+static void dm_test_get_modes_noedid_128b_adds_more(struct kunit *test)
+{
+	struct dm_test_gm_ctx *ctx =
+		dm_test_gm_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	int n_8b, n_128b;
+
+	mutex_lock(&ctx->drm->mode_config.mutex);
+	n_8b = amdgpu_dm_connector_get_modes(&ctx->aconnector->base);
+
+	ctx->link_srv->dp_get_encoding_format = dm_test_gm_enc_128b;
+	n_128b = amdgpu_dm_connector_get_modes(&ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
+
+	KUNIT_EXPECT_GT(test, n_128b, n_8b);
+}
+
+/**
+ * dm_test_get_modes_noedid_analog_adds_common - Test analog sinks add common modes
+ * @test: The KUnit test context
+ *
+ * An analog VGA sink detected by load detection adds the common fallback modes
+ * on top of the default 640x480 mode(s).
+ */
+static void dm_test_get_modes_noedid_analog_adds_common(struct kunit *test)
+{
+	struct dm_test_gm_ctx *ctx =
+		dm_test_gm_ctx_alloc(test, DRM_MODE_CONNECTOR_VGA);
+	struct dc_sink *sink;
+	int n_base, n_analog;
+
+	mutex_lock(&ctx->drm->mode_config.mutex);
+	n_base = amdgpu_dm_connector_get_modes(&ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	sink->edid_caps.analog = true;
+	ctx->aconnector->dc_sink = sink;
+	ctx->link->link_id.id = CONNECTOR_ID_VGA;
+
+	mutex_lock(&ctx->drm->mode_config.mutex);
+	n_analog = amdgpu_dm_connector_get_modes(&ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
+
+	KUNIT_EXPECT_GT(test, n_analog, n_base);
+}
+
+/**
+ * dm_test_get_modes_with_edid - Test the cached-EDID path adds common modes
+ * @test: The KUnit test context
+ *
+ * With a cached EDID on an eDP connector, get_modes() takes the DDC path and
+ * adds the common downscaled modes derived from the encoder native mode.
+ */
+static void dm_test_get_modes_with_edid(struct kunit *test)
+{
+	struct dm_test_gm_ctx *ctx =
+		dm_test_gm_ctx_alloc(test, DRM_MODE_CONNECTOR_eDP);
+	const struct drm_edid *drm_edid;
+	int count;
+
+	drm_edid = drm_edid_alloc(dm_test_uad_edid, sizeof(dm_test_uad_edid));
+	KUNIT_ASSERT_NOT_NULL(test, drm_edid);
+	drm_edid_connector_update(&ctx->aconnector->base, drm_edid);
+	ctx->aconnector->drm_edid = drm_edid;
+
+	ctx->aenc->native_mode.hdisplay = 1920;
+	ctx->aenc->native_mode.vdisplay = 1200;
+
+	mutex_lock(&ctx->drm->mode_config.mutex);
+	count = amdgpu_dm_connector_get_modes(&ctx->aconnector->base);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
+
+	KUNIT_EXPECT_GT(test, count, 0);
+
+	drm_edid_free(drm_edid);
+	ctx->aconnector->drm_edid = NULL;
+}
+
+/* Tests for amdgpu_set_panel_orientation() */
+
+/**
+ * dm_test_panel_orientation_non_edp - Test non-eDP/LVDS connectors are skipped
+ * @test: The KUnit test context
+ *
+ * Only eDP and LVDS panels carry a fixed orientation, so a DisplayPort
+ * connector returns immediately with its panel_orientation left unknown.
+ */
+static void dm_test_panel_orientation_non_edp(struct kunit *test)
+{
+	struct dm_test_gm_ctx *ctx =
+	dm_test_gm_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+
+	amdgpu_set_panel_orientation(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test,
+			ctx->aconnector->base.display_info.panel_orientation,
+			DRM_MODE_PANEL_ORIENTATION_UNKNOWN);
+}
+
+/**
+ * dm_test_panel_orientation_no_native_mode - Test a missing native mode is skipped
+ * @test: The KUnit test context
+ *
+ * On an eDP connector with no cached EDID the encoder native mode stays 0x0, so
+ * the function returns before applying an orientation.
+ */
+static void dm_test_panel_orientation_no_native_mode(struct kunit *test)
+{
+	struct dm_test_gm_ctx *ctx =
+	dm_test_gm_ctx_alloc(test, DRM_MODE_CONNECTOR_eDP);
+
+	amdgpu_set_panel_orientation(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, ctx->aenc->native_mode.hdisplay, 0);
+	KUNIT_EXPECT_EQ(test,
+			ctx->aconnector->base.display_info.panel_orientation,
+			DRM_MODE_PANEL_ORIENTATION_UNKNOWN);
+}
+
+/**
+ * dm_test_panel_orientation_applies_quirk - Test a native mode reaches the quirk
+ * @test: The KUnit test context
+ *
+ * With a valid encoder native mode on an eDP connector the panel dimensions are
+ * forwarded to the orientation quirk lookup; absent a matching quirk the
+ * orientation stays unknown.
+ */
+static void dm_test_panel_orientation_applies_quirk(struct kunit *test)
+{
+	struct dm_test_gm_ctx *ctx =
+	dm_test_gm_ctx_alloc(test, DRM_MODE_CONNECTOR_eDP);
+
+	ctx->aenc->native_mode.hdisplay = 1920;
+	ctx->aenc->native_mode.vdisplay = 1200;
+
+	amdgpu_set_panel_orientation(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test,
+			ctx->aconnector->base.display_info.panel_orientation,
+			DRM_MODE_PANEL_ORIENTATION_UNKNOWN);
+}
+
+/* Tests for amdgpu_dm_prune_primary_tile_modes() */
+
+struct dm_test_prune_ctx {
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_sink *sink;
+};
+
+/*
+ * Build a connector configured as the primary tile of an Apple Studio Display:
+ * a dc_sink requesting the second-tile patch, has_tile set, tile location (0,0)
+ * and a per-tile timing of tile_h_size x tile_v_size. Individual tests relax a
+ * single precondition to exercise the early-return guards.
+ */
+static struct dm_test_prune_ctx *
+dm_test_prune_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_prune_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->drm = dm_test_alloc_drm(test);
+	ctx->aconnector = dm_test_add_connector(test, ctx->drm,
+						DRM_MODE_CONNECTOR_DisplayPort);
+
+	ctx->sink = kunit_kzalloc(test, sizeof(*ctx->sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->sink);
+	ctx->sink->edid_caps.panel_patch.disable_second_tile = true;
+	ctx->aconnector->dc_sink = ctx->sink;
+
+	ctx->aconnector->base.has_tile = true;
+	ctx->aconnector->base.tile_h_size = 2560;
+	ctx->aconnector->base.tile_v_size = 2880;
+
+	return ctx;
+}
+
+static struct drm_display_mode *
+dm_test_prune_add_mode(struct kunit *test, struct drm_connector *connector,
+		       int hdisplay, int vdisplay)
+{
+	struct drm_display_mode *mode = drm_mode_create(connector->dev);
+
+	KUNIT_ASSERT_NOT_NULL(test, mode);
+	mode->hdisplay = hdisplay;
+	mode->vdisplay = vdisplay;
+	list_add_tail(&mode->head, &connector->probed_modes);
+
+	return mode;
+}
+
+static int dm_test_prune_count(struct drm_connector *connector)
+{
+	struct drm_display_mode *mode;
+	int n = 0;
+
+	list_for_each_entry(mode, &connector->probed_modes, head)
+		n++;
+
+	return n;
+}
+
+/**
+ * dm_test_prune_no_sink - Test a connector without a sink is left untouched
+ * @test: The KUnit test context
+ */
+static void dm_test_prune_no_sink(struct kunit *test)
+{
+	struct dm_test_prune_ctx *ctx = dm_test_prune_ctx_alloc(test);
+
+	ctx->aconnector->dc_sink = NULL;
+	dm_test_prune_add_mode(test, &ctx->aconnector->base, 2560, 2880);
+	ctx->aconnector->num_modes = 1;
+
+	amdgpu_dm_prune_primary_tile_modes(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_test_prune_count(&ctx->aconnector->base), 1);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 1);
+}
+
+/**
+ * dm_test_prune_no_patch - Test the patch flag being unset skips pruning
+ * @test: The KUnit test context
+ */
+static void dm_test_prune_no_patch(struct kunit *test)
+{
+	struct dm_test_prune_ctx *ctx = dm_test_prune_ctx_alloc(test);
+
+	ctx->sink->edid_caps.panel_patch.disable_second_tile = false;
+	dm_test_prune_add_mode(test, &ctx->aconnector->base, 2560, 2880);
+	ctx->aconnector->num_modes = 1;
+
+	amdgpu_dm_prune_primary_tile_modes(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_test_prune_count(&ctx->aconnector->base), 1);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 1);
+}
+
+/**
+ * dm_test_prune_no_tile - Test a non-tiled connector skips pruning
+ * @test: The KUnit test context
+ */
+static void dm_test_prune_no_tile(struct kunit *test)
+{
+	struct dm_test_prune_ctx *ctx = dm_test_prune_ctx_alloc(test);
+
+	ctx->aconnector->base.has_tile = false;
+	dm_test_prune_add_mode(test, &ctx->aconnector->base, 2560, 2880);
+	ctx->aconnector->num_modes = 1;
+
+	amdgpu_dm_prune_primary_tile_modes(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_test_prune_count(&ctx->aconnector->base), 1);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 1);
+}
+
+/**
+ * dm_test_prune_secondary_tile - Test a secondary tile is left untouched
+ * @test: The KUnit test context
+ *
+ * Only the primary tile (location 0,0) is pruned; a secondary tile keeps its
+ * per-tile timing.
+ */
+static void dm_test_prune_secondary_tile(struct kunit *test)
+{
+	struct dm_test_prune_ctx *ctx = dm_test_prune_ctx_alloc(test);
+
+	ctx->aconnector->base.tile_h_loc = 1;
+	dm_test_prune_add_mode(test, &ctx->aconnector->base, 2560, 2880);
+	ctx->aconnector->num_modes = 1;
+
+	amdgpu_dm_prune_primary_tile_modes(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_test_prune_count(&ctx->aconnector->base), 1);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 1);
+}
+
+/**
+ * dm_test_prune_removes_per_tile - Test the per-tile timing is pruned
+ * @test: The KUnit test context
+ *
+ * On the primary tile the per-tile (tile_h_size x tile_v_size) timing is
+ * dropped while the full-resolution mode is kept and num_modes is decremented.
+ */
+static void dm_test_prune_removes_per_tile(struct kunit *test)
+{
+	struct dm_test_prune_ctx *ctx = dm_test_prune_ctx_alloc(test);
+	struct drm_display_mode *mode;
+	bool has_tile_mode = false;
+
+	dm_test_prune_add_mode(test, &ctx->aconnector->base, 2560, 2880);
+	dm_test_prune_add_mode(test, &ctx->aconnector->base, 5120, 2880);
+	ctx->aconnector->num_modes = 2;
+
+	amdgpu_dm_prune_primary_tile_modes(&ctx->aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_test_prune_count(&ctx->aconnector->base), 1);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->num_modes, 1);
+
+	list_for_each_entry(mode, &ctx->aconnector->base.probed_modes, head)
+		if (mode->hdisplay == 2560 && mode->vdisplay == 2880)
+			has_tile_mode = true;
+	KUNIT_EXPECT_FALSE(test, has_tile_mode);
+}
+
+/* Tests for amdgpu_dm_update_stream_scaling_settings() */
+
+/**
+ * dm_test_update_scaling_null_mode - Test NULL mode leaves the stream rects untouched
+ * @test: The KUnit test context
+ */
+static void dm_test_update_scaling_null_mode(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+
+	stream->timing.h_addressable = 1920;
+	stream->timing.v_addressable = 1080;
+
+	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, NULL, NULL, stream);
+
+	/* NULL mode: early return before touching src/dst */
+	KUNIT_EXPECT_EQ(test, stream->src.width, 0);
+	KUNIT_EXPECT_EQ(test, stream->dst.width, 0);
+}
+
+/**
+ * dm_test_update_scaling_fullscreen_default - Test full-screen default with no dm_state
+ * @test: The KUnit test context
+ */
+static void dm_test_update_scaling_fullscreen_default(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct drm_display_mode mode = { 0 };
+
+	mode.hdisplay = 1920;
+	mode.vdisplay = 1080;
+	stream->timing.h_addressable = 2560;
+	stream->timing.v_addressable = 1440;
+
+	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, NULL, stream);
+
+	/* src = mode, dst = timing addressable, no centering without dm_state */
+	KUNIT_EXPECT_EQ(test, stream->src.width, 1920);
+	KUNIT_EXPECT_EQ(test, stream->src.height, 1080);
+	KUNIT_EXPECT_EQ(test, stream->dst.width, 2560);
+	KUNIT_EXPECT_EQ(test, stream->dst.height, 1440);
+	KUNIT_EXPECT_EQ(test, stream->dst.x, 0);
+	KUNIT_EXPECT_EQ(test, stream->dst.y, 0);
+}
+
+/**
+ * dm_test_update_scaling_rmx_full - Test RMX_FULL keeps a full-size, centered dst
+ * @test: The KUnit test context
+ */
+static void dm_test_update_scaling_rmx_full(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dm_connector_state *dm_state;
+	struct drm_display_mode mode = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	mode.hdisplay = 1280;
+	mode.vdisplay = 720;
+	stream->timing.h_addressable = 1920;
+	stream->timing.v_addressable = 1080;
+	dm_state->scaling = RMX_FULL;
+
+	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+
+	/* RMX_FULL: dst stays full addressable, offset 0 */
+	KUNIT_EXPECT_EQ(test, stream->dst.width, 1920);
+	KUNIT_EXPECT_EQ(test, stream->dst.height, 1080);
+	KUNIT_EXPECT_EQ(test, stream->dst.x, 0);
+	KUNIT_EXPECT_EQ(test, stream->dst.y, 0);
+}
+
+/**
+ * dm_test_update_scaling_rmx_aspect_pillarbox - Test RMX_ASPECT preserves aspect ratio
+ * @test: The KUnit test context
+ */
+static void dm_test_update_scaling_rmx_aspect_pillarbox(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dm_connector_state *dm_state;
+	struct drm_display_mode mode = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	/* 4:3 source on a 16:9 panel -> pillarboxed */
+	mode.hdisplay = 1024;
+	mode.vdisplay = 768;
+	stream->timing.h_addressable = 1920;
+	stream->timing.v_addressable = 1080;
+	dm_state->scaling = RMX_ASPECT;
+
+	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+
+	/*
+	 * src.width*dst.height (1024*1080) < src.height*dst.width (768*1920):
+	 * width scaled to src.width*dst.height/src.height = 1440, height stays
+	 * 1080, centered horizontally at (1920-1440)/2 = 240.
+	 */
+	KUNIT_EXPECT_EQ(test, stream->dst.width, 1440);
+	KUNIT_EXPECT_EQ(test, stream->dst.height, 1080);
+	KUNIT_EXPECT_EQ(test, stream->dst.x, 240);
+	KUNIT_EXPECT_EQ(test, stream->dst.y, 0);
+}
+
+/**
+ * dm_test_update_scaling_rmx_aspect_letterbox - Test RMX_ASPECT letterboxes wide sources
+ * @test: The KUnit test context
+ */
+static void dm_test_update_scaling_rmx_aspect_letterbox(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dm_connector_state *dm_state;
+	struct drm_display_mode mode = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	/* 16:9 source on a 4:3 panel -> letterboxed */
+	mode.hdisplay = 1920;
+	mode.vdisplay = 1080;
+	stream->timing.h_addressable = 1024;
+	stream->timing.v_addressable = 768;
+	dm_state->scaling = RMX_ASPECT;
+
+	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+
+	KUNIT_EXPECT_EQ(test, stream->dst.width, 1024);
+	KUNIT_EXPECT_EQ(test, stream->dst.height, 576);
+	KUNIT_EXPECT_EQ(test, stream->dst.x, 0);
+	KUNIT_EXPECT_EQ(test, stream->dst.y, 96);
+}
+
+/**
+ * dm_test_update_scaling_rmx_center - Test RMX_CENTER centers a 1:1 dst
+ * @test: The KUnit test context
+ */
+static void dm_test_update_scaling_rmx_center(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dm_connector_state *dm_state;
+	struct drm_display_mode mode = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	mode.hdisplay = 1280;
+	mode.vdisplay = 720;
+	stream->timing.h_addressable = 1920;
+	stream->timing.v_addressable = 1080;
+	dm_state->scaling = RMX_CENTER;
+
+	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+
+	/* RMX_CENTER: dst = src, centered on the addressable area */
+	KUNIT_EXPECT_EQ(test, stream->dst.width, 1280);
+	KUNIT_EXPECT_EQ(test, stream->dst.height, 720);
+	KUNIT_EXPECT_EQ(test, stream->dst.x, 320);
+	KUNIT_EXPECT_EQ(test, stream->dst.y, 180);
+}
+
+/**
+ * dm_test_update_scaling_underscan - Test underscan borders shrink and offset dst
+ * @test: The KUnit test context
+ */
+static void dm_test_update_scaling_underscan(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dm_connector_state *dm_state;
+	struct drm_display_mode mode = { 0 };
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+
+	mode.hdisplay = 1920;
+	mode.vdisplay = 1080;
+	stream->timing.h_addressable = 1920;
+	stream->timing.v_addressable = 1080;
+	dm_state->scaling = RMX_FULL;
+	dm_state->underscan_enable = true;
+	dm_state->underscan_hborder = 64;
+	dm_state->underscan_vborder = 32;
+
+	amdgpu_dm_update_stream_scaling_settings(&adev->ddev, &mode, dm_state, stream);
+
+	/* Full dst, then underscan: x/y += border/2, width/height -= border */
+	KUNIT_EXPECT_EQ(test, stream->dst.x, 32);
+	KUNIT_EXPECT_EQ(test, stream->dst.y, 16);
+	KUNIT_EXPECT_EQ(test, stream->dst.width, 1856);
 	KUNIT_EXPECT_EQ(test, stream->dst.height, 1048);
 }
 
+/* Tests for hdmi_frl_status_polling_work() */
+
+static int dm_test_frl_poll_calls;
+static int dm_test_frl_detect_calls;
+
+static bool dm_test_frl_poll_true(struct dc_link *link)
+{
+	dm_test_frl_poll_calls++;
+	return true;
+}
+
+static bool dm_test_frl_poll_false(struct dc_link *link)
+{
+	dm_test_frl_poll_calls++;
+	return false;
+}
+
+static bool dm_test_frl_detect(struct dc_link *link, enum dc_detect_reason reason)
+{
+	dm_test_frl_detect_calls++;
+	return true;
+}
+
+struct dm_test_frl_ctx {
+	struct amdgpu_display_manager *dm;
+	struct dc *dc;
+	struct link_service *link_srv;
+};
+
+/*
+ * Build a display manager whose polling work re-arms on a real workqueue. The
+ * dc starts with an empty link array and link_srv hooks that count poll and
+ * detect calls, so a test can assert exactly how far a link progresses. A large
+ * re-arm delay keeps the requeued work dormant until the test cancels it.
+ */
+static struct dm_test_frl_ctx *dm_test_frl_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_frl_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->dm = kunit_kzalloc(test, sizeof(*ctx->dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dm);
+	ctx->dc = kunit_kzalloc(test, sizeof(*ctx->dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc);
+	ctx->link_srv = kunit_kzalloc(test, sizeof(*ctx->link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link_srv);
+
+	ctx->link_srv->hdmi_frl_poll_status_flag = dm_test_frl_poll_true;
+	ctx->link_srv->detect_link = dm_test_frl_detect;
+	ctx->dc->link_srv = ctx->link_srv;
+	ctx->dm->dc = ctx->dc;
+	mutex_init(&ctx->dm->dc_lock);
+	INIT_DELAYED_WORK(&ctx->dm->hdmi_frl_status_polling_work, hdmi_frl_status_polling_work);
+	ctx->dm->hdmi_frl_status_polling_delay_ms = 100000;
+	ctx->dm->hdmi_frl_status_polling_wq = system_wq;
+
+	dm_test_frl_poll_calls = 0;
+	dm_test_frl_detect_calls = 0;
+
+	return ctx;
+}
+
+/* Allocate a dc_link, wire its dc back-pointer and register it at index 0. */
+static struct dc_link *dm_test_frl_add_link(struct kunit *test, struct dm_test_frl_ctx *ctx)
+{
+	struct dc_link *link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, link);
+	link->dc = ctx->dc;
+	ctx->dc->links[0] = link;
+
+	return link;
+}
+
+/* Register a link that satisfies every guard up to the poll status check. */
+static struct dc_link *dm_test_frl_add_hdmi_link(struct kunit *test, struct dm_test_frl_ctx *ctx)
+{
+	struct dc_link *link = dm_test_frl_add_link(test, ctx);
+
+	link->local_sink = kunit_kzalloc(test, sizeof(*link->local_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link->local_sink);
+	link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	link->frl_link_settings.frl_link_rate = HDMI_FRL_LINK_RATE_3GBPS;
+
+	return link;
+}
+
+/* Run the work once directly, then cancel and tear down the re-armed work. */
+static void dm_test_frl_run(struct dm_test_frl_ctx *ctx)
+{
+	hdmi_frl_status_polling_work(&ctx->dm->hdmi_frl_status_polling_work.work);
+	cancel_delayed_work_sync(&ctx->dm->hdmi_frl_status_polling_work);
+}
+
+/**
+ * dm_test_frl_no_links - Test an empty link array is a no-op
+ * @test: The KUnit test context
+ *
+ * With no links registered the loop body never runs, so neither the poll nor
+ * the detect hook is invoked.
+ */
+static void dm_test_frl_no_links(struct kunit *test)
+{
+	struct dm_test_frl_ctx *ctx = dm_test_frl_ctx_alloc(test);
+
+	dm_test_frl_run(ctx);
+
+	KUNIT_EXPECT_EQ(test, dm_test_frl_poll_calls, 0);
+	KUNIT_EXPECT_EQ(test, dm_test_frl_detect_calls, 0);
+}
+
+/**
+ * dm_test_frl_skips_no_local_sink - Test a link without a local sink is skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_frl_skips_no_local_sink(struct kunit *test)
+{
+	struct dm_test_frl_ctx *ctx = dm_test_frl_ctx_alloc(test);
+
+	dm_test_frl_add_link(test, ctx);
+
+	dm_test_frl_run(ctx);
+
+	KUNIT_EXPECT_EQ(test, dm_test_frl_poll_calls, 0);
+}
+
+/**
+ * dm_test_frl_skips_non_hdmi - Test a non-HDMI link is skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_frl_skips_non_hdmi(struct kunit *test)
+{
+	struct dm_test_frl_ctx *ctx = dm_test_frl_ctx_alloc(test);
+	struct dc_link *link = dm_test_frl_add_link(test, ctx);
+
+	link->local_sink = kunit_kzalloc(test, sizeof(*link->local_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link->local_sink);
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	dm_test_frl_run(ctx);
+
+	KUNIT_EXPECT_EQ(test, dm_test_frl_poll_calls, 0);
+}
+
+/**
+ * dm_test_frl_skips_zero_rate - Test a link with no FRL rate is skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_frl_skips_zero_rate(struct kunit *test)
+{
+	struct dm_test_frl_ctx *ctx = dm_test_frl_ctx_alloc(test);
+	struct dc_link *link = dm_test_frl_add_hdmi_link(test, ctx);
+
+	link->frl_link_settings.frl_link_rate = 0;
+
+	dm_test_frl_run(ctx);
+
+	KUNIT_EXPECT_EQ(test, dm_test_frl_poll_calls, 0);
+}
+
+/**
+ * dm_test_frl_poll_no_update - Test a clear poll status skips retraining
+ * @test: The KUnit test context
+ *
+ * The poll hook is reached but reports no change, so dc_link_detect() is not
+ * called.
+ */
+static void dm_test_frl_poll_no_update(struct kunit *test)
+{
+	struct dm_test_frl_ctx *ctx = dm_test_frl_ctx_alloc(test);
+
+	ctx->link_srv->hdmi_frl_poll_status_flag = dm_test_frl_poll_false;
+	dm_test_frl_add_hdmi_link(test, ctx);
+
+	dm_test_frl_run(ctx);
+
+	KUNIT_EXPECT_EQ(test, dm_test_frl_poll_calls, 1);
+	KUNIT_EXPECT_EQ(test, dm_test_frl_detect_calls, 0);
+}
+
+/**
+ * dm_test_frl_poll_retrains - Test a set poll status triggers a retrain
+ * @test: The KUnit test context
+ *
+ * A poll status change drives dc_link_detect() with DETECT_REASON_RETRAIN.
+ */
+static void dm_test_frl_poll_retrains(struct kunit *test)
+{
+	struct dm_test_frl_ctx *ctx = dm_test_frl_ctx_alloc(test);
+
+	dm_test_frl_add_hdmi_link(test, ctx);
+
+	dm_test_frl_run(ctx);
+
+	KUNIT_EXPECT_EQ(test, dm_test_frl_poll_calls, 1);
+	KUNIT_EXPECT_EQ(test, dm_test_frl_detect_calls, 1);
+}
+
+/* Tests for amdgpu_dm_connector_init_helper() */
+
+/*
+ * Build an amdgpu_dm_connector on a kunit drm_device embedded in an
+ * amdgpu_device, with the amdgpu mode properties created exactly as
+ * amdgpu_dm_mode_config_init() does so the helper has real properties to
+ * attach. The dc_link carries a link encoder and leaves DIG mapping
+ * inflexible, so link_enc_cfg_get_link_enc() resolves without a dc instance.
+ */
+struct dm_test_init_helper_ctx {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_display_manager *dm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+};
+
+/*
+ * Stand in for amdgpu_display_modeset_create_props(), which the test module
+ * cannot link against. Only the properties the init helper attaches are
+ * created; underscan is a plain range because only attachment is under test.
+ */
+static void dm_test_create_mode_props(struct kunit *test, struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	struct drm_device *drm = adev_to_drm(adev);
+
+	KUNIT_ASSERT_EQ(test, drm_mode_create_scaling_mode_property(drm), 0);
+
+	mode_info->underscan_property =
+		drm_property_create_range(drm, 0, "underscan", 0, 1);
+	KUNIT_ASSERT_NOT_NULL(test, mode_info->underscan_property);
+	mode_info->underscan_hborder_property =
+		drm_property_create_range(drm, 0, "underscan hborder", 0, 128);
+	KUNIT_ASSERT_NOT_NULL(test, mode_info->underscan_hborder_property);
+	mode_info->underscan_vborder_property =
+		drm_property_create_range(drm, 0, "underscan vborder", 0, 128);
+	KUNIT_ASSERT_NOT_NULL(test, mode_info->underscan_vborder_property);
+}
+
+static struct dm_test_init_helper_ctx *
+dm_test_init_helper_ctx_alloc(struct kunit *test, int connector_type)
+{
+	struct dm_test_init_helper_ctx *ctx;
+	struct device *dev;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(struct amdgpu_device),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET | DRIVER_ATOMIC);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+
+	ctx->adev = drm_to_adev(ctx->drm);
+	ctx->adev->dev = dev;
+	dm_test_create_mode_props(test, ctx->adev);
+
+	ctx->dm = &ctx->adev->dm;
+	ctx->dm->adev = ctx->adev;
+	ctx->dm->ddev = ctx->drm;
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->link->link_enc = kunit_kzalloc(test, sizeof(*ctx->link->link_enc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link->link_enc);
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+					    &dm_test_connector_funcs, connector_type,
+					    NULL), 0);
+
+	return ctx;
+}
+
+/* True when @prop is attached to the connector's mode object. */
+static bool dm_test_has_prop(struct drm_connector *connector, struct drm_property *prop)
+{
+	struct drm_object_properties *props = connector->base.properties;
+	int i;
+
+	if (!prop)
+		return false;
+
+	for (i = 0; i < props->count; i++)
+		if (props->properties[i] == prop)
+			return true;
+
+	return false;
+}
+
+/**
+ * dm_test_init_helper_common_defaults - Test the connector defaults and properties
+ * @test: The KUnit test context
+ *
+ * The helper resets the connector state, records the link and its index, and
+ * attaches the scaling mode and the three underscan properties that every
+ * connector type receives.
+ */
+static void dm_test_init_helper_common_defaults(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_HDMIA, ctx->link, 3);
+
+	KUNIT_ASSERT_NOT_NULL(test, connector->state);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->connector_id, 3);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->bl_idx, -1);
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_link, ctx->link);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->audio_inst, -1);
+	KUNIT_EXPECT_FALSE(test, ctx->aconnector->pack_sdp_v1_3);
+	KUNIT_EXPECT_EQ(test, (int)ctx->aconnector->as_type, (int)ADAPTIVE_SYNC_TYPE_NONE);
+	KUNIT_EXPECT_FALSE(test, connector->interlace_allowed);
+	KUNIT_EXPECT_FALSE(test, connector->doublescan_allowed);
+	KUNIT_EXPECT_FALSE(test, connector->stereo_allowed);
+	KUNIT_EXPECT_EQ(test, connector->dpms, DRM_MODE_DPMS_OFF);
+	KUNIT_EXPECT_EQ(test, (int)ctx->aconnector->hpd.hpd, (int)AMDGPU_HPD_NONE);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->hdmi_hpd_debounce_delay_ms, 0);
+
+	/* The link encoder advertises no YCbCr 4:2:0 support. */
+	KUNIT_EXPECT_FALSE(test, connector->ycbcr_420_allowed);
+
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector, ctx->drm->mode_config.scaling_mode_property));
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector, ctx->adev->mode_info.underscan_property));
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector,
+					   ctx->adev->mode_info.underscan_hborder_property));
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector,
+					   ctx->adev->mode_info.underscan_vborder_property));
+}
+
+/**
+ * dm_test_init_helper_hdmi - Test the HDMI connector wiring
+ * @test: The KUnit test context
+ *
+ * HDMI polls on hotplug, takes YCbCr 4:2:0 support straight from the link
+ * encoder features and gets the max bpc, content type, colorspace, HDR
+ * metadata and VRR capable properties.
+ */
+static void dm_test_init_helper_hdmi(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	ctx->link->link_enc->features.hdmi_ycbcr420_supported = true;
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_HDMIA, ctx->link, 0);
+
+	KUNIT_EXPECT_EQ(test, (int)connector->polled, (int)DRM_CONNECTOR_POLL_HPD);
+	KUNIT_EXPECT_TRUE(test, connector->ycbcr_420_allowed);
+	KUNIT_ASSERT_NOT_NULL(test, connector->state);
+	KUNIT_EXPECT_EQ(test, connector->state->max_bpc, 16);
+	KUNIT_EXPECT_EQ(test, connector->state->max_requested_bpc, 16);
+	KUNIT_EXPECT_NOT_NULL(test, connector->max_bpc_property);
+	KUNIT_EXPECT_NOT_NULL(test, connector->broadcast_rgb_property);
+	KUNIT_EXPECT_NOT_NULL(test, connector->colorspace_property);
+	KUNIT_EXPECT_NOT_NULL(test, connector->vrr_capable_property);
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector, ctx->drm->mode_config.content_type_property));
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector,
+					   ctx->drm->mode_config.hdr_output_metadata_property));
+}
+
+/**
+ * dm_test_init_helper_dp - Test the DisplayPort connector wiring
+ * @test: The KUnit test context
+ *
+ * DisplayPort re-resolves the link encoder through the encoder configuration
+ * and takes YCbCr 4:2:0 support from its DP feature bit.
+ */
+static void dm_test_init_helper_dp(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	ctx->link->link_enc->features.dp_ycbcr420_supported = true;
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_DisplayPort, ctx->link, 1);
+
+	KUNIT_EXPECT_EQ(test, (int)connector->polled, (int)DRM_CONNECTOR_POLL_HPD);
+	KUNIT_EXPECT_TRUE(test, connector->ycbcr_420_allowed);
+	KUNIT_EXPECT_NOT_NULL(test, connector->broadcast_rgb_property);
+	KUNIT_EXPECT_NOT_NULL(test, connector->colorspace_property);
+	KUNIT_EXPECT_NOT_NULL(test, connector->vrr_capable_property);
+}
+
+/**
+ * dm_test_init_helper_dp_mst_root - Test an MST branch connector skips properties
+ * @test: The KUnit test context
+ *
+ * A connector below an MST root gets neither the broadcast RGB, max bpc,
+ * colorspace nor VRR capable property, because those live on the root.
+ */
+static void dm_test_init_helper_dp_mst_root(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_DisplayPort);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	ctx->aconnector->mst_root = ctx->aconnector;
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_DisplayPort, ctx->link, 0);
+
+	KUNIT_EXPECT_NULL(test, connector->broadcast_rgb_property);
+	KUNIT_EXPECT_NULL(test, connector->max_bpc_property);
+	KUNIT_EXPECT_NULL(test, connector->colorspace_property);
+	KUNIT_EXPECT_NULL(test, connector->vrr_capable_property);
+}
+
+/**
+ * dm_test_init_helper_dvid - Test DVI-D polls on hotplug
+ * @test: The KUnit test context
+ */
+static void dm_test_init_helper_dvid(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_DVID);
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_DVID, ctx->link, 0);
+
+	KUNIT_EXPECT_EQ(test, (int)ctx->aconnector->base.polled, (int)DRM_CONNECTOR_POLL_HPD);
+	KUNIT_EXPECT_FALSE(test, ctx->aconnector->base.ycbcr_420_allowed);
+}
+
+/**
+ * dm_test_init_helper_vga - Test VGA polls on connect and disconnect
+ * @test: The KUnit test context
+ */
+static void dm_test_init_helper_vga(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_VGA);
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_VGA, ctx->link, 0);
+
+	KUNIT_EXPECT_EQ(test, (int)ctx->aconnector->base.polled,
+			(int)(DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT));
+}
+
+/**
+ * dm_test_init_helper_unpolled_default - Test an unhandled type is left unpolled
+ * @test: The KUnit test context
+ */
+static void dm_test_init_helper_unpolled_default(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_Composite);
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_Composite, ctx->link, 0);
+
+	KUNIT_EXPECT_EQ(test, (int)ctx->aconnector->base.polled, 0);
+	KUNIT_EXPECT_NULL(test, ctx->aconnector->base.colorspace_property);
+	KUNIT_EXPECT_NULL(test, ctx->aconnector->base.vrr_capable_property);
+}
+
+/**
+ * dm_test_init_helper_edp - Test eDP gets the panel type property
+ * @test: The KUnit test context
+ *
+ * eDP shares the DisplayPort colorspace and HDR property set and additionally
+ * receives the panel type property. No privacy screen provider is registered
+ * in the test environment, so the lookup fails with -ENODEV and is ignored.
+ */
+static void dm_test_init_helper_edp(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_eDP);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_eDP, ctx->link, 0);
+
+	KUNIT_EXPECT_EQ(test, (int)connector->polled, 0);
+	KUNIT_EXPECT_NOT_NULL(test, connector->colorspace_property);
+	KUNIT_EXPECT_NOT_NULL(test, connector->vrr_capable_property);
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector, ctx->drm->mode_config.panel_type_property));
+	KUNIT_EXPECT_NULL(test, connector->privacy_screen);
+}
+
+/**
+ * dm_test_init_helper_hdcp_property - Test the content protection property
+ * @test: The KUnit test context
+ *
+ * The content protection property is only attached when a HDCP workqueue was
+ * created for the device.
+ */
+static void dm_test_init_helper_hdcp_property(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	ctx->adev->dm.hdcp_workqueue = kunit_kzalloc(test, sizeof(struct hdcp_workqueue),
+						     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->adev->dm.hdcp_workqueue);
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_HDMIA, ctx->link, 0);
+
+	KUNIT_EXPECT_TRUE(test,
+			  dm_test_has_prop(connector,
+					   ctx->drm->mode_config.content_protection_property));
+}
+
+/**
+ * dm_test_init_helper_hpd_debounce_disabled - Test the default HPD debounce delay
+ * @test: The KUnit test context
+ *
+ * With the hdmi_hpd_debounce_delay_ms module parameter left at its default the
+ * debounce work is not armed and the delay stays zero.
+ */
+static void dm_test_init_helper_hpd_debounce_disabled(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+
+	amdgpu_dm_connector_init_helper(ctx->dm, ctx->aconnector,
+					DRM_MODE_CONNECTOR_HDMIA, ctx->link, 0);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->hdmi_hpd_debounce_delay_ms, 0);
+}
+
+/* Tests for amdgpu_dm_initialize_hdmi_connector() */
+
+/**
+ * dm_test_initialize_hdmi_cec_registers - Test the CEC notifier is registered
+ * @test: The KUnit test context
+ *
+ * Without DC_DISABLE_HDMI_CEC in the debug mask a CEC notifier is created for
+ * the connector.
+ */
+static void dm_test_initialize_hdmi_cec_registers(struct kunit *test)
+{
+	struct dm_test_init_helper_ctx *ctx =
+		dm_test_init_helper_ctx_alloc(test, DRM_MODE_CONNECTOR_HDMIA);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_initialize_hdmi_connector(ctx->aconnector), 0);
+	KUNIT_EXPECT_NOT_NULL(test, ctx->aconnector->notifier);
+}
+
+/* Tests for amdgpu_dm_update_freesync_caps() */
+
+static void dm_test_fs_free_edid(void *data)
+{
+	drm_edid_free(data);
+}
+
+/*
+ * Build a two block EDID: the minimal base block with its extension count set
+ * to one, followed by @ext, with both checksums recomputed so drm_edid_raw()
+ * and the DRM EDID parsers accept it.
+ */
+static const struct drm_edid *dm_test_fs_edid_alloc(struct kunit *test, const u8 *ext)
+{
+	const struct drm_edid *drm_edid;
+	u8 *raw;
+	int i;
+
+	raw = kunit_kzalloc(test, 2 * EDID_LENGTH, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, raw);
+	memcpy(raw, dm_test_uad_edid, EDID_LENGTH);
+	memcpy(raw + EDID_LENGTH, ext, EDID_LENGTH);
+	raw[EDID_LENGTH - 2] = 1;
+
+	for (i = 0; i < 2; i++) {
+		u8 *block = raw + i * EDID_LENGTH;
+		u8 sum = 0;
+		int j;
+
+		for (j = 0; j < EDID_LENGTH - 1; j++)
+			sum += block[j];
+		block[EDID_LENGTH - 1] = -sum;
+	}
+
+	drm_edid = drm_edid_alloc(raw, 2 * EDID_LENGTH);
+	KUNIT_ASSERT_NOT_NULL(test, drm_edid);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_fs_free_edid,
+						  (void *)drm_edid), 0);
+
+	return drm_edid;
+}
+
+/*
+ * A DisplayID extension holding a dynamic video timing range descriptor, which
+ * is what parse_edid_displayid_vrr() scans for.
+ */
+static const struct drm_edid *
+dm_test_fs_edid_displayid(struct kunit *test, u8 min_vfreq, u8 max_vfreq)
+{
+	u8 ext[EDID_LENGTH] = {0};
+
+	ext[0] = DM_TEST_DISPLAYID_EXT;
+	ext[1] = 0x25;
+	ext[2] = 0x00;
+	ext[3] = 9;
+	ext[10] = min_vfreq;
+	ext[11] = max_vfreq;
+
+	return dm_test_fs_edid_alloc(test, ext);
+}
+
+/*
+ * A CTA-861 extension with no data blocks, enough for parse_hdmi_amd_vsdb() to
+ * accept the block and hand it to the (mocked) DMCU parser.
+ */
+static const struct drm_edid *dm_test_fs_edid_cea(struct kunit *test)
+{
+	u8 ext[EDID_LENGTH] = {0};
+
+	ext[0] = DM_TEST_CEA_EXT;
+	ext[1] = 3;
+	ext[2] = 4;
+
+	return dm_test_fs_edid_alloc(test, ext);
+}
+
+/*
+ * A CTA-861 extension carrying a 15-byte AMD VSDB v3 payload, which
+ * drm_parse_amd_vsdb() turns into connector->display_info.amd_vsdb and
+ * get_amd_vsdb() then reads back.
+ */
+static const struct drm_edid *
+dm_test_fs_edid_amd_vsdb(struct kunit *test, u8 feature_caps)
+{
+	u8 ext[EDID_LENGTH] = {0};
+
+	ext[0] = DM_TEST_CEA_EXT;
+	ext[1] = 3;
+	ext[2] = 4 + 1 + 15;
+	ext[4] = (3 << 5) | 15;		/* vendor data block, 15-byte payload */
+	ext[5] = 0x1a;			/* AMD IEEE OUI, LSB first */
+	ext[8] = 0x03;			/* AMD VSDB payload version */
+	ext[9] = feature_caps;
+
+	return dm_test_fs_edid_alloc(test, ext);
+}
+
+struct dm_test_fs_caps_ctx {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+	struct dc_sink *sink;
+	struct resource_pool *pool;
+};
+
+/*
+ * Build a connector that clears every guard at the top of
+ * amdgpu_dm_update_freesync_caps(): a reset connector state, a dc_sink whose
+ * dc_context reports a VRR capable DCE version, and a non-NULL freesync
+ * module. The resource pool carries no DMCU, so tests that need the AMD VSDB
+ * parser to succeed install one.
+ */
+static struct dm_test_fs_caps_ctx *
+dm_test_fs_caps_ctx_alloc(struct kunit *test, enum signal_type sink_signal)
+{
+	struct dm_test_fs_caps_ctx *ctx;
+	struct dc_context *dc_ctx;
+	struct device *dev;
+	struct dc *dc;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(struct amdgpu_device),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+	ctx->adev = drm_to_adev(ctx->drm);
+	ctx->adev->dm.adev = ctx->adev;
+	mutex_init(&ctx->adev->dm.dc_lock);
+
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+					    &dm_test_connector_funcs,
+					    DRM_MODE_CONNECTOR_DisplayPort, NULL), 0);
+	amdgpu_dm_connector_funcs_reset(&ctx->aconnector->base);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector->base.state);
+	KUNIT_ASSERT_EQ(test,
+			drm_connector_attach_vrr_capable_property(&ctx->aconnector->base), 0);
+
+	dc_ctx = kunit_kzalloc(test, sizeof(*dc_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_ctx);
+	dc_ctx->driver_context = ctx->adev;
+	dc_ctx->dce_version = DCE_VERSION_8_0;
+
+	ctx->pool = kunit_kzalloc(test, sizeof(*ctx->pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->pool);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	dc->ctx = dc_ctx;
+	dc->res_pool = ctx->pool;
+	ctx->adev->dm.dc = dc;
+
+	ctx->adev->dm.freesync_module =
+		kunit_kzalloc(test, sizeof(struct mod_freesync), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->adev->dm.freesync_module);
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->aconnector->dc_link = ctx->link;
+
+	ctx->sink = kunit_kzalloc(test, sizeof(*ctx->sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->sink);
+	ctx->sink->ctx = dc_ctx;
+	ctx->sink->sink_signal = sink_signal;
+	ctx->aconnector->dc_sink = ctx->sink;
+
+	return ctx;
+}
+
+static bool dm_test_fs_caps_capable(struct dm_test_fs_caps_ctx *ctx)
+{
+	return to_dm_connector_state(ctx->aconnector->base.state)->freesync_capable;
+}
+
+/**
+ * dm_test_fs_caps_dp_msa_range - Test a DP sink with an invalid-MSA range
+ * @test: The KUnit test context
+ *
+ * A DisplayPort sink allowing invalid MSA timing takes its refresh range from
+ * the monitor range that parse_edid_displayid_vrr() recovered, and a range
+ * wider than 10Hz marks the connector FreeSync capable.
+ */
+static void dm_test_fs_caps_dp_msa_range(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_DISPLAY_PORT);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_displayid(test, 40, 144);
+
+	ctx->link->dpcd_caps.allow_invalid_MSA_timing_param = true;
+
+	amdgpu_dm_update_freesync_caps(&ctx->aconnector->base, drm_edid, false);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->min_vfreq, 40);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->max_vfreq, 144);
+	KUNIT_EXPECT_TRUE(test, dm_test_fs_caps_capable(ctx));
+}
+
+/**
+ * dm_test_fs_caps_dp_narrow_range - Test a range of 10Hz or less is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_fs_caps_dp_narrow_range(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_DISPLAY_PORT);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_displayid(test, 60, 65);
+
+	ctx->link->dpcd_caps.allow_invalid_MSA_timing_param = true;
+
+	amdgpu_dm_update_freesync_caps(&ctx->aconnector->base, drm_edid, false);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->min_vfreq, 60);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->max_vfreq, 65);
+	KUNIT_EXPECT_FALSE(test, dm_test_fs_caps_capable(ctx));
+}
+
+/**
+ * dm_test_fs_caps_dp_msa_not_allowed - Test a DP sink requiring valid MSA
+ * @test: The KUnit test context
+ *
+ * Without allow_invalid_MSA_timing_param the reported range is ignored and the
+ * connector keeps a zero refresh range.
+ */
+static void dm_test_fs_caps_dp_msa_not_allowed(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_DISPLAY_PORT);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_displayid(test, 40, 144);
+
+	amdgpu_dm_update_freesync_caps(&ctx->aconnector->base, drm_edid, false);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->min_vfreq, 0);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->max_vfreq, 0);
+	KUNIT_EXPECT_FALSE(test, dm_test_fs_caps_capable(ctx));
+}
+
+/**
+ * dm_test_fs_caps_edp_replay_mode - Test an eDP panel advertising replay
+ * @test: The KUnit test context
+ *
+ * An AMD VSDB with the replay feature bit set records the replay mode and its
+ * VSDB version on the connector and selects the eDP adaptive sync type.
+ */
+static void dm_test_fs_caps_edp_replay_mode(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_EDP);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_amd_vsdb(test, 0x40);
+
+	amdgpu_dm_update_freesync_caps(&ctx->aconnector->base, drm_edid, false);
+
+	KUNIT_EXPECT_TRUE(test, ctx->aconnector->vsdb_info.replay_mode);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->vsdb_info.amd_vsdb_version, 3);
+	KUNIT_EXPECT_EQ(test, (int)ctx->aconnector->as_type, (int)ADAPTIVE_SYNC_TYPE_EDP);
+}
+
+/**
+ * dm_test_fs_caps_hdmi_no_vsdb - Test an HDMI sink with no AMD VSDB
+ * @test: The KUnit test context
+ *
+ * Without a DMCU the CEA parse fails, so no refresh range is recorded.
+ */
+static void dm_test_fs_caps_hdmi_no_vsdb(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_HDMI_TYPE_A);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_cea(test);
+
+	amdgpu_dm_update_freesync_caps(&ctx->aconnector->base, drm_edid, false);
+
+	KUNIT_EXPECT_FALSE(test, ctx->aconnector->vsdb_info.freesync_supported);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->max_vfreq, 0);
+	KUNIT_EXPECT_FALSE(test, dm_test_fs_caps_capable(ctx));
+}
+
+/**
+ * dm_test_fs_caps_force_min_hz_quirk - Test the forced FreeSync minimum quirk
+ * @test: The KUnit test context
+ *
+ * A quirked panel overrides the minimum refresh rate on both the connector and
+ * the published monitor range once FreeSync is otherwise supported.
+ */
+static void dm_test_fs_caps_force_min_hz_quirk(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_DISPLAY_PORT);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_displayid(test, 40, 144);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	ctx->link->dpcd_caps.allow_invalid_MSA_timing_param = true;
+	ctx->sink->edid_caps.panel_patch.force_freesync_min_hz = 50;
+
+	amdgpu_dm_update_freesync_caps(connector, drm_edid, false);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->min_vfreq, 50);
+	KUNIT_EXPECT_EQ(test, connector->display_info.monitor_range.min_vfreq, 50);
+}
+
+/**
+ * dm_test_fs_caps_mccs_clears_capability - Test MCCS withdraws FreeSync support
+ * @test: The KUnit test context
+ *
+ * When the sink advertises a FreeSync VCP code but MCCS reports no support,
+ * the connector loses its FreeSync capability.
+ */
+static void dm_test_fs_caps_mccs_clears_capability(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_DISPLAY_PORT);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_displayid(test, 40, 144);
+
+	ctx->link->dpcd_caps.allow_invalid_MSA_timing_param = true;
+	/* A non-DP, non-HDMI link keeps dm_helpers_read_mccs_caps() off the wire. */
+	ctx->link->connector_signal = SIGNAL_TYPE_VIRTUAL;
+	ctx->sink->edid_caps.freesync_vcp_code = 0x60;
+
+	amdgpu_dm_update_freesync_caps(&ctx->aconnector->base, drm_edid, true);
+
+	KUNIT_EXPECT_FALSE(test, ctx->sink->mccs_caps.freesync_supported);
+	KUNIT_EXPECT_FALSE(test, dm_test_fs_caps_capable(ctx));
+}
+
+/**
+ * dm_test_fs_caps_disables_replay - Test replay is dropped without FreeSync
+ * @test: The KUnit test context
+ *
+ * A link advertising replay support has it withdrawn when the connector ends
+ * up without FreeSync capability.
+ */
+static void dm_test_fs_caps_disables_replay(struct kunit *test)
+{
+	struct dm_test_fs_caps_ctx *ctx =
+		dm_test_fs_caps_ctx_alloc(test, SIGNAL_TYPE_DISPLAY_PORT);
+	const struct drm_edid *drm_edid = dm_test_fs_edid_displayid(test, 40, 144);
+
+	ctx->link->replay_settings.config.replay_supported = true;
+	ctx->link->replay_settings.replay_feature_enabled = true;
+
+	amdgpu_dm_update_freesync_caps(&ctx->aconnector->base, drm_edid, false);
+
+	KUNIT_EXPECT_FALSE(test, ctx->link->replay_settings.config.replay_supported);
+	KUNIT_EXPECT_FALSE(test, ctx->link->replay_settings.replay_feature_enabled);
+}
+
+/* Tests for amdgpu_dm_connector_init() and the get_modes() helper hook */
+
+/*
+ * amdgpu_dm_create_i2c() parents the adapter on adev->pdev->dev, so
+ * devm_i2c_add_adapter() needs a registered device there. Provide one by
+ * registering the device embedded in a KUnit allocated pci_dev.
+ */
+static void dm_test_conn_init_release_dev(struct device *dev)
+{
+	/* Backing storage is KUnit managed, so there is nothing to free. */
+}
+
+static void dm_test_conn_init_unregister_dev(void *data)
+{
+	device_unregister(data);
+}
+
+struct dm_test_conn_init_ctx {
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	struct amdgpu_display_manager *dm;
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_encoder *aencoder;
+	struct dc_link *link;
+};
+
+static struct dm_test_conn_init_ctx *
+dm_test_conn_init_ctx_alloc(struct kunit *test, enum signal_type signal)
+{
+	struct dm_test_conn_init_ctx *ctx;
+	struct link_service *link_srv;
+	struct dc_context *dc_ctx;
+	struct ddc_service *ddc;
+	struct pci_dev *pdev;
+	struct device *dev;
+	struct dc *dc;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(struct amdgpu_device),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+	ctx->adev = drm_to_adev(ctx->drm);
+	ctx->adev->dev = dev;
+	ctx->adev->mode_info.num_crtc = 1;
+	dm_test_create_mode_props(test, ctx->adev);
+
+	pdev = kunit_kzalloc(test, sizeof(*pdev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, pdev);
+	device_initialize(&pdev->dev);
+	pdev->dev.parent = dev;
+	pdev->dev.release = dm_test_conn_init_release_dev;
+	KUNIT_ASSERT_EQ(test, dev_set_name(&pdev->dev, "dm-test-i2c-parent"), 0);
+	KUNIT_ASSERT_EQ(test, device_add(&pdev->dev), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_conn_init_unregister_dev,
+						  &pdev->dev), 0);
+	ctx->adev->pdev = pdev;
+
+	ctx->dm = &ctx->adev->dm;
+	ctx->dm->adev = ctx->adev;
+	ctx->dm->ddev = ctx->drm;
+
+	dc_ctx = kunit_kzalloc(test, sizeof(*dc_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_ctx);
+	dc_ctx->driver_context = ctx->adev;
+
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	link_srv->dp_get_encoding_format = dm_test_gm_enc_8b10b;
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	dc->ctx = dc_ctx;
+	dc->link_srv = link_srv;
+
+	ctx->link = kunit_kzalloc(test, sizeof(*ctx->link), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link);
+	ctx->link->connector_signal = signal;
+	ctx->link->dc = dc;
+	ctx->link->link_enc = kunit_kzalloc(test, sizeof(*ctx->link->link_enc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link->link_enc);
+
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	ddc->ctx = dc_ctx;
+	ddc->link = ctx->link;
+	ctx->link->ddc = ddc;
+
+	dc->links[0] = ctx->link;
+	dc->link_count = 1;
+	ctx->dm->dc = dc;
+
+	/* amdgpu_dm_connector_destroy() and amdgpu_dm_encoder_destroy() kfree() these. */
+	ctx->aconnector = kzalloc_obj(*ctx->aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	ctx->aencoder = kzalloc_obj(*ctx->aencoder);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aencoder);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_encoder_init(ctx->drm, ctx->aencoder, 0), 0);
+
+	return ctx;
+}
+
+/**
+ * dm_test_conn_init_hdmi - Test a HDMI connector is fully brought up
+ * @test: The KUnit test context
+ *
+ * The DC link is bound to the connector, an i2c adapter is created and
+ * registered as the DDC bus, the DRM connector is initialized with the DM
+ * funcs and helpers, the encoder is attached and the CEC notifier registered.
+ */
+static void dm_test_conn_init_hdmi(struct kunit *test)
+{
+	struct dm_test_conn_init_ctx *ctx =
+		dm_test_conn_init_ctx_alloc(test, SIGNAL_TYPE_HDMI_TYPE_A);
+	struct drm_connector *connector = &ctx->aconnector->base;
+
+	KUNIT_ASSERT_EQ(test,
+			amdgpu_dm_connector_init(ctx->dm, ctx->aconnector, 0,
+						 ctx->aencoder), 0);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->link->priv, ctx->aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector->i2c);
+	KUNIT_EXPECT_PTR_EQ(test, connector->ddc, &ctx->aconnector->i2c->base);
+	KUNIT_EXPECT_EQ(test, connector->connector_type, DRM_MODE_CONNECTOR_HDMIA);
+	KUNIT_EXPECT_NOT_NULL(test, connector->helper_private);
+	KUNIT_EXPECT_PTR_EQ(test, ctx->aconnector->dc_link, ctx->link);
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->connector_id, 0);
+	KUNIT_EXPECT_EQ(test, connector->possible_encoders,
+			drm_encoder_mask(&ctx->aencoder->base));
+	KUNIT_EXPECT_NOT_NULL(test, ctx->aconnector->notifier);
+}
+
+/**
+ * dm_test_conn_init_dvi - Test a DVI link maps to a DVI-D connector
+ * @test: The KUnit test context
+ *
+ * A single link DVI signal is not HDMI, so no CEC notifier is registered.
+ */
+static void dm_test_conn_init_dvi(struct kunit *test)
+{
+	struct dm_test_conn_init_ctx *ctx =
+		dm_test_conn_init_ctx_alloc(test, SIGNAL_TYPE_DVI_SINGLE_LINK);
+
+	KUNIT_ASSERT_EQ(test,
+			amdgpu_dm_connector_init(ctx->dm, ctx->aconnector, 0,
+						 ctx->aencoder), 0);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconnector->base.connector_type,
+			DRM_MODE_CONNECTOR_DVID);
+	KUNIT_EXPECT_NULL(test, ctx->aconnector->notifier);
+}
+
+/**
+ * dm_test_conn_init_get_modes_hook - Test the installed get_modes helper
+ * @test: The KUnit test context
+ *
+ * The connector helper funcs that amdgpu_dm_connector_init() installs forward
+ * mode enumeration to amdgpu_dm_connector_get_modes(), which synthesizes the
+ * no-EDID fallback modes.
+ */
+static void dm_test_conn_init_get_modes_hook(struct kunit *test)
+{
+	struct dm_test_conn_init_ctx *ctx =
+		dm_test_conn_init_ctx_alloc(test, SIGNAL_TYPE_HDMI_TYPE_A);
+	const struct drm_connector_helper_funcs *helper;
+	struct drm_connector *connector = &ctx->aconnector->base;
+	int count;
+
+	KUNIT_ASSERT_EQ(test,
+			amdgpu_dm_connector_init(ctx->dm, ctx->aconnector, 0,
+						 ctx->aencoder), 0);
+
+	helper = connector->helper_private;
+	KUNIT_ASSERT_NOT_NULL(test, helper);
+	KUNIT_ASSERT_NOT_NULL(test, helper->get_modes);
+
+	mutex_lock(&ctx->drm->mode_config.mutex);
+	count = helper->get_modes(connector);
+	mutex_unlock(&ctx->drm->mode_config.mutex);
+
+	KUNIT_EXPECT_GT(test, count, 0);
+	KUNIT_EXPECT_GT(test, ctx->aconnector->num_modes, 0);
+}
+
+/* Tests for dm_force_atomic_commit() */
+
+/* Fails the very first state duplication so the error path is taken. */
+static struct drm_connector_state *
+dm_test_force_dup_state_fail(struct drm_connector *connector)
+{
+	return NULL;
+}
+
+static const struct drm_connector_funcs dm_test_force_fail_funcs = {
+	.reset = amdgpu_dm_connector_funcs_reset,
+	.atomic_duplicate_state = dm_test_force_dup_state_fail,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+/* The atomic modeset check dereferences the helper funcs unconditionally. */
+static int dm_test_force_get_modes(struct drm_connector *connector)
+{
+	return 0;
+}
+
+static const struct drm_connector_helper_funcs dm_test_force_conn_helper_funcs = {
+	.get_modes = dm_test_force_get_modes,
+};
+
+struct dm_test_force_ctx {
+	struct drm_device *drm;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_modeset_acquire_ctx acquire_ctx;
+};
+
+static void dm_test_force_drop_locks(void *data)
+{
+	struct dm_test_force_ctx *ctx = data;
+
+	ctx->drm->mode_config.acquire_ctx = NULL;
+	drm_modeset_drop_locks(&ctx->acquire_ctx);
+	drm_modeset_acquire_fini(&ctx->acquire_ctx);
+}
+
+/*
+ * Build a disabled but fully initialized pipe: a primary plane, a CRTC (whose
+ * drm_crtc is the first member of amdgpu_crtc, so to_amdgpu_crtc() resolves)
+ * and a connector wired to an encoder that still points at the CRTC, which is
+ * the state dm_force_atomic_commit() is asked to restore.
+ */
+static struct dm_test_force_ctx *
+dm_test_force_ctx_alloc(struct kunit *test, const struct drm_connector_funcs *funcs)
+{
+	struct dm_test_force_ctx *ctx;
+	struct drm_encoder *encoder;
+	struct drm_plane *primary;
+	struct drm_crtc *crtc;
+	struct device *dev;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(struct amdgpu_device),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET | DRIVER_ATOMIC);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+
+	primary = drm_kunit_helper_create_primary_plane(test, ctx->drm, NULL, NULL,
+							NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, primary);
+	crtc = drm_kunit_helper_create_crtc(test, ctx->drm, primary, NULL, NULL, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(ctx->drm, &ctx->aconnector->base, funcs,
+					    DRM_MODE_CONNECTOR_HDMIA, NULL), 0);
+	drm_connector_helper_add(&ctx->aconnector->base, &dm_test_force_conn_helper_funcs);
+
+	encoder = drmm_kzalloc(ctx->drm, sizeof(*encoder), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, encoder);
+	KUNIT_ASSERT_EQ(test,
+			drmm_encoder_init(ctx->drm, encoder, NULL,
+					  DRM_MODE_ENCODER_TMDS, NULL), 0);
+	encoder->crtc = crtc;
+	ctx->aconnector->base.encoder = encoder;
+
+	drm_mode_config_reset(ctx->drm);
+
+	drm_modeset_acquire_init(&ctx->acquire_ctx, 0);
+	ctx->drm->mode_config.acquire_ctx = &ctx->acquire_ctx;
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_force_drop_locks, ctx), 0);
+
+	return ctx;
+}
+
+/**
+ * dm_test_force_commit_succeeds - Test the forced restore commits the pipe
+ * @test: The KUnit test context
+ *
+ * The connector, its CRTC and the CRTC primary plane are pulled into a fresh
+ * atomic state, the CRTC is marked as needing a modeset and the commit
+ * succeeds.
+ */
+static void dm_test_force_commit_succeeds(struct kunit *test)
+{
+	struct dm_test_force_ctx *ctx =
+		dm_test_force_ctx_alloc(test, &dm_test_connector_funcs);
+
+	KUNIT_EXPECT_EQ(test, dm_force_atomic_commit(&ctx->aconnector->base), 0);
+}
+
+/**
+ * dm_test_force_commit_conn_state_error - Test a failed connector state duplication
+ * @test: The KUnit test context
+ *
+ * When the connector state cannot be duplicated the error is propagated and
+ * the partially built atomic state is released without committing.
+ */
+static void dm_test_force_commit_conn_state_error(struct kunit *test)
+{
+	struct dm_test_force_ctx *ctx =
+		dm_test_force_ctx_alloc(test, &dm_test_force_fail_funcs);
+
+	KUNIT_EXPECT_EQ(test, dm_force_atomic_commit(&ctx->aconnector->base), -ENOMEM);
+}
+
 static struct kunit_case amdgpu_dm_connector_tests[] = {
 	/* get_subconnector_type */
 	KUNIT_CASE(dm_test_subconnector_type_none),
@@ -5527,6 +9887,19 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	/* amdgpu_dm_fill_hdr_info_packet */
 	KUNIT_CASE(dm_test_fill_hdr_null_metadata),
 	KUNIT_CASE(dm_test_fill_hdr_zeroes_output),
+	KUNIT_CASE(dm_test_fill_hdr_hdmi),
+	KUNIT_CASE(dm_test_fill_hdr_dp),
+	KUNIT_CASE(dm_test_fill_hdr_unsupported_connector),
+	KUNIT_CASE(dm_test_fill_hdr_bad_metadata),
+	/* amdgpu_dm_connector_atomic_check */
+	KUNIT_CASE(dm_test_conn_atomic_check_no_crtc),
+	KUNIT_CASE(dm_test_conn_atomic_check_dp_mst),
+	KUNIT_CASE(dm_test_conn_atomic_check_no_change),
+	KUNIT_CASE(dm_test_conn_atomic_check_privacy_change),
+	KUNIT_CASE(dm_test_conn_atomic_check_colorspace_change),
+	KUNIT_CASE(dm_test_conn_atomic_check_content_type_change),
+	KUNIT_CASE(dm_test_conn_atomic_check_hdr_exit),
+	KUNIT_CASE(dm_test_conn_atomic_check_hdr_fill_error),
 	/* amdgpu_dm_connector_atomic_set_property */
 	KUNIT_CASE(dm_test_set_property_scaling_center),
 	KUNIT_CASE(dm_test_set_property_scaling_aspect),
@@ -5588,6 +9961,7 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	KUNIT_CASE(dm_test_detect_mst_skips_writeback),
 	KUNIT_CASE(dm_test_detect_mst_non_mst_link),
 	KUNIT_CASE(dm_test_detect_mst_branch_without_aux),
+	KUNIT_CASE(dm_test_detect_mst_start_fail),
 	/* amdgpu_dm_find_first_crtc_matching_connector */
 	KUNIT_CASE(dm_test_find_first_crtc_match),
 	KUNIT_CASE(dm_test_find_first_crtc_no_match),
@@ -5620,20 +9994,32 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	KUNIT_CASE(dm_test_fill_stream_encoding_from_caller_ycbcr444),
 	KUNIT_CASE(dm_test_fill_stream_hdmi_ep_clamps_depth),
 	KUNIT_CASE(dm_test_fill_stream_non_hdmi_ep_keeps_depth),
+	KUNIT_CASE(dm_test_fill_stream_hdmi_infoframe),
+	KUNIT_CASE(dm_test_fill_stream_freesync_video),
 	/* create_stream_for_sink */
 	KUNIT_CASE(dm_test_create_stream_fake_sink_success),
 	KUNIT_CASE(dm_test_create_stream_sets_dm_context),
 	KUNIT_CASE(dm_test_create_stream_virtual_signal),
 	KUNIT_CASE(dm_test_create_stream_scaling_src),
 	KUNIT_CASE(dm_test_create_stream_existing_sink),
+	KUNIT_CASE(dm_test_create_stream_preferred_mode),
+	KUNIT_CASE(dm_test_create_stream_scaled_keeps_refresh),
+	KUNIT_CASE(dm_test_create_stream_timing_override),
+	KUNIT_CASE(dm_test_create_stream_dp_vsc_colorimetry),
+	KUNIT_CASE(dm_test_create_stream_dp_colorimetry_quirk),
+	KUNIT_CASE(dm_test_create_stream_hdmi_vsif),
 	/* amdgpu_dm_connector_detect */
 	KUNIT_CASE(dm_test_detect_force_on),
 	KUNIT_CASE(dm_test_detect_force_on_digital),
 	KUNIT_CASE(dm_test_detect_force_off),
 	KUNIT_CASE(dm_test_detect_sink_present),
 	KUNIT_CASE(dm_test_detect_no_sink),
+	KUNIT_CASE(dm_test_detect_hides_secondary_tile),
 	/* amdgpu_dm_connector_poll */
 	KUNIT_CASE(dm_test_poll_dac_load_returns_cached),
+	KUNIT_CASE(dm_test_poll_connected_cached_sink),
+	KUNIT_CASE(dm_test_poll_connected_new_sink),
+	KUNIT_CASE(dm_test_poll_disconnect_releases_sink),
 	/* amdgpu_dm_connector_late_register */
 	KUNIT_CASE(dm_test_late_register_non_dp_succeeds),
 	/* amdgpu_dm_connector_unregister */
@@ -5648,17 +10034,43 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	KUNIT_CASE(dm_test_atomic_check_edp_native_keeps_scaling),
 	KUNIT_CASE(dm_test_atomic_check_lvds_non_native_enables_scaling),
 	KUNIT_CASE(dm_test_atomic_check_non_mst_returns_zero),
+	KUNIT_CASE(dm_test_atomic_check_mst_no_change_returns_zero),
+	KUNIT_CASE(dm_test_atomic_check_mst_finds_vcpi_slots),
+	KUNIT_CASE(dm_test_atomic_check_mst_duplicated_skips_pbn),
+	KUNIT_CASE(dm_test_atomic_check_mst_topology_err_propagates),
+	KUNIT_CASE(dm_test_atomic_check_mst_vcpi_error_propagates),
+	KUNIT_CASE(dm_test_encoder_init_success),
 	/* hdmi_cec_unset_edid */
 	KUNIT_CASE(dm_test_hdmi_cec_unset_edid_no_notifier),
 	/* create_eml_sink */
 	KUNIT_CASE(dm_test_create_eml_sink_no_edid),
+	KUNIT_CASE(dm_test_create_eml_sink_reads_edid),
+	KUNIT_CASE(dm_test_create_eml_sink_force_on_em),
+	KUNIT_CASE(dm_test_create_eml_sink_force_on_local),
 	/* handle_edid_mgmt */
 	KUNIT_CASE(dm_test_handle_edid_mgmt_dp_sets_link_caps),
 	KUNIT_CASE(dm_test_handle_edid_mgmt_non_dp_leaves_caps),
 	/* amdgpu_dm_connector_funcs_force */
 	KUNIT_CASE(dm_test_funcs_force_no_edid),
+	KUNIT_CASE(dm_test_funcs_force_reads_edid),
+	/* amdgpu_dm_connector_get_modes */
+	KUNIT_CASE(dm_test_get_modes_noedid_default),
+	KUNIT_CASE(dm_test_get_modes_noedid_128b_adds_more),
+	KUNIT_CASE(dm_test_get_modes_noedid_analog_adds_common),
+	KUNIT_CASE(dm_test_get_modes_with_edid),
+	/* amdgpu_set_panel_orientation */
+	KUNIT_CASE(dm_test_panel_orientation_non_edp),
+	KUNIT_CASE(dm_test_panel_orientation_no_native_mode),
+	KUNIT_CASE(dm_test_panel_orientation_applies_quirk),
+	/* amdgpu_dm_prune_primary_tile_modes */
+	KUNIT_CASE(dm_test_prune_no_sink),
+	KUNIT_CASE(dm_test_prune_no_patch),
+	KUNIT_CASE(dm_test_prune_no_tile),
+	KUNIT_CASE(dm_test_prune_secondary_tile),
+	KUNIT_CASE(dm_test_prune_removes_per_tile),
 	/* dm_validate_stream_and_context */
 	KUNIT_CASE(dm_test_validate_stream_null_stream),
+	KUNIT_CASE(dm_test_validate_stream_dc_ok_no_pipe),
 	/* amdgpu_dm_connector_to_encoder */
 	KUNIT_CASE(dm_test_to_encoder_no_encoder),
 	KUNIT_CASE(dm_test_to_encoder_returns_attached),
@@ -5675,19 +10087,30 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	KUNIT_CASE(dm_test_ddc_get_modes_null_edid),
 	/* add_fs_modes */
 	KUNIT_CASE(dm_test_add_fs_modes_no_preferred_mode),
+	KUNIT_CASE(dm_test_add_fs_modes_generates),
+	KUNIT_CASE(dm_test_add_fs_modes_out_of_range),
+	KUNIT_CASE(dm_test_add_fs_modes_skips_illegal),
 	/* amdgpu_dm_connector_add_freesync_modes */
 	KUNIT_CASE(dm_test_add_freesync_modes_null_edid_noop),
 	/* amdgpu_dm_i2c_func */
 	KUNIT_CASE(dm_test_i2c_func_returns_flags),
 	/* amdgpu_dm_i2c_xfer */
 	KUNIT_CASE(dm_test_i2c_xfer_no_ddc_pin),
+	KUNIT_CASE(dm_test_i2c_xfer_hw_submit_fails),
+	KUNIT_CASE(dm_test_i2c_xfer_oem_no_device),
+	/* amdgpu_dm_create_i2c */
+	KUNIT_CASE(dm_test_create_i2c_oem),
+	KUNIT_CASE(dm_test_create_i2c_hw_bus),
+	/* dm_restore_drm_connector_state */
+	KUNIT_CASE(dm_test_restore_state_writeback),
+	KUNIT_CASE(dm_test_restore_state_no_dc_sink),
+	KUNIT_CASE(dm_test_restore_state_no_connector_state),
+	KUNIT_CASE(dm_test_restore_state_no_encoder),
+	KUNIT_CASE(dm_test_restore_state_no_stream),
+	KUNIT_CASE(dm_test_restore_state_same_sink),
 	/* get_amd_vsdb */
 	KUNIT_CASE(dm_test_get_amd_vsdb_unsupported),
 	KUNIT_CASE(dm_test_get_amd_vsdb_supported),
-	/* parse_hdmi_amd_vsdb */
-	KUNIT_CASE(dm_test_parse_hdmi_amd_vsdb_null_edid),
-	KUNIT_CASE(dm_test_parse_hdmi_amd_vsdb_no_extensions),
-	KUNIT_CASE(dm_test_parse_hdmi_amd_vsdb_no_cea_ext),
 	/* parse_edid_displayid_vrr */
 	KUNIT_CASE(dm_test_parse_displayid_vrr_null_edid),
 	KUNIT_CASE(dm_test_parse_displayid_vrr_no_displayid),
@@ -5695,6 +10118,9 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	/* amdgpu_dm_connector_mode_valid */
 	KUNIT_CASE(dm_test_mode_valid_interlace_rejected),
 	KUNIT_CASE(dm_test_mode_valid_dblscan_rejected),
+	KUNIT_CASE(dm_test_mode_valid_no_dc_sink),
+	KUNIT_CASE(dm_test_mode_valid_force_on_no_stream),
+	KUNIT_CASE(dm_test_mode_valid_edid_mgmt_forced),
 	/* amdgpu_dm_hdmi_cec_set_edid */
 	KUNIT_CASE(dm_test_hdmi_cec_set_edid_no_notifier),
 	/* amdgpu_dm_s3_handle_hdmi_cec */
@@ -5702,9 +10128,25 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	KUNIT_CASE(dm_test_s3_handle_hdmi_cec_resume),
 	/* amdgpu_dm_create_validate_stream_for_sink */
 	KUNIT_CASE(dm_test_create_validate_stream_null_dm_state),
+	KUNIT_CASE(dm_test_create_validate_stream_writeback),
+	KUNIT_CASE(dm_test_create_validate_stream_no_valid_bpc),
+	KUNIT_CASE(dm_test_create_validate_stream_hdmi_ycbcr),
+	KUNIT_CASE(dm_test_create_validate_stream_force_ycbcr420),
+	KUNIT_CASE(dm_test_create_validate_stream_force_ycbcr422),
+	KUNIT_CASE(dm_test_create_validate_stream_force_ycbcr444),
+	KUNIT_CASE(dm_test_create_validate_stream_prune_timing),
+	KUNIT_CASE(dm_test_create_validate_stream_prune_context),
 	/* amdgpu_dm_update_connector_after_detect */
 	KUNIT_CASE(dm_test_update_after_detect_mst_noop),
 	KUNIT_CASE(dm_test_update_after_detect_sink_unchanged),
+	KUNIT_CASE(dm_test_update_after_detect_mst_sink),
+	KUNIT_CASE(dm_test_update_after_detect_connect_edid),
+	KUNIT_CASE(dm_test_update_after_detect_replace_no_edid),
+	KUNIT_CASE(dm_test_update_after_detect_disconnect),
+	KUNIT_CASE(dm_test_update_after_detect_force_em_adopt),
+	KUNIT_CASE(dm_test_update_after_detect_force_em_replace),
+	KUNIT_CASE(dm_test_update_after_detect_force_em_fake),
+	KUNIT_CASE(dm_test_update_after_detect_force_em_keep),
 	/* amdgpu_dm_update_stream_scaling_settings */
 	KUNIT_CASE(dm_test_update_scaling_null_mode),
 	KUNIT_CASE(dm_test_update_scaling_fullscreen_default),
@@ -5713,6 +10155,42 @@ static struct kunit_case amdgpu_dm_connector_tests[] = {
 	KUNIT_CASE(dm_test_update_scaling_rmx_aspect_letterbox),
 	KUNIT_CASE(dm_test_update_scaling_rmx_center),
 	KUNIT_CASE(dm_test_update_scaling_underscan),
+	/* hdmi_frl_status_polling_work */
+	KUNIT_CASE(dm_test_frl_no_links),
+	KUNIT_CASE(dm_test_frl_skips_no_local_sink),
+	KUNIT_CASE(dm_test_frl_skips_non_hdmi),
+	KUNIT_CASE(dm_test_frl_skips_zero_rate),
+	KUNIT_CASE(dm_test_frl_poll_no_update),
+	KUNIT_CASE(dm_test_frl_poll_retrains),
+	/* amdgpu_dm_connector_init_helper */
+	KUNIT_CASE(dm_test_init_helper_common_defaults),
+	KUNIT_CASE(dm_test_init_helper_hdmi),
+	KUNIT_CASE(dm_test_init_helper_dp),
+	KUNIT_CASE(dm_test_init_helper_dp_mst_root),
+	KUNIT_CASE(dm_test_init_helper_dvid),
+	KUNIT_CASE(dm_test_init_helper_vga),
+	KUNIT_CASE(dm_test_init_helper_unpolled_default),
+	KUNIT_CASE(dm_test_init_helper_edp),
+	KUNIT_CASE(dm_test_init_helper_hdcp_property),
+	KUNIT_CASE(dm_test_init_helper_hpd_debounce_disabled),
+	/* amdgpu_dm_initialize_hdmi_connector */
+	KUNIT_CASE(dm_test_initialize_hdmi_cec_registers),
+	/* amdgpu_dm_update_freesync_caps */
+	KUNIT_CASE(dm_test_fs_caps_dp_msa_range),
+	KUNIT_CASE(dm_test_fs_caps_dp_narrow_range),
+	KUNIT_CASE(dm_test_fs_caps_dp_msa_not_allowed),
+	KUNIT_CASE(dm_test_fs_caps_edp_replay_mode),
+	KUNIT_CASE(dm_test_fs_caps_hdmi_no_vsdb),
+	KUNIT_CASE(dm_test_fs_caps_force_min_hz_quirk),
+	KUNIT_CASE(dm_test_fs_caps_mccs_clears_capability),
+	KUNIT_CASE(dm_test_fs_caps_disables_replay),
+	/* amdgpu_dm_connector_init */
+	KUNIT_CASE(dm_test_conn_init_hdmi),
+	KUNIT_CASE(dm_test_conn_init_dvi),
+	KUNIT_CASE(dm_test_conn_init_get_modes_hook),
+	/* dm_force_atomic_commit */
+	KUNIT_CASE(dm_test_force_commit_succeeds),
+	KUNIT_CASE(dm_test_force_commit_conn_state_error),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c
index ae0f4da962521a..489dd56474b918 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_crtc_test.c
@@ -10,10 +10,13 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_kunit_helpers.h>
 #include <drm/drm_vblank.h>
+#include <linux/debugfs.h>
 
 #include "dc.h"
 #include "inc/core_types.h"
 #include "irq/irq_service.h"
+#include "inc/hw/timing_generator.h"
+#include "inc/link_service.h"
 #include "amdgpu.h"
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
@@ -39,10 +42,10 @@ static void dm_test_crtc_modeset_required_active_mode_changed(struct kunit *test
 }
 
 /**
- * dm_test_crtc_modeset_required_active_active_changed - Test Crtc modeset required active active changed
+ * dm_test_crtc_modeset_required_active_state_changed - Test modeset required when the active state changes
  * @test: The KUnit test context
  */
-static void dm_test_crtc_modeset_required_active_active_changed(struct kunit *test)
+static void dm_test_crtc_modeset_required_active_state_changed(struct kunit *test)
 {
 	struct drm_crtc_state state = {};
 
@@ -874,10 +877,39 @@ static bool dm_test_crtc_get_vblank_timestamp(struct drm_crtc *crtc,
 	return false;
 }
 
+static int dm_test_crtc_enable_vblank(struct drm_crtc *crtc)
+{
+	return 0;
+}
+
+static void dm_test_crtc_disable_vblank(struct drm_crtc *crtc)
+{
+}
+
+static u32 dm_test_crtc_get_vblank_counter(struct drm_crtc *crtc)
+{
+	return 0;
+}
+
 static const struct drm_crtc_funcs dm_test_crtc_funcs = {
+	.destroy = amdgpu_dm_crtc_destroy,
+	.atomic_duplicate_state = amdgpu_dm_crtc_duplicate_state,
+	.atomic_destroy_state = amdgpu_dm_crtc_destroy_state,
+	.enable_vblank = dm_test_crtc_enable_vblank,
+	.disable_vblank = dm_test_crtc_disable_vblank,
+	.get_vblank_counter = dm_test_crtc_get_vblank_counter,
 	.get_vblank_timestamp = dm_test_crtc_get_vblank_timestamp,
 };
 
+static void dm_test_crtc_cleanup(void *data)
+{
+	struct amdgpu_crtc *acrtc = data;
+
+	if (acrtc->base.dev && drm_dev_has_vblank(acrtc->base.dev))
+		drm_crtc_vblank_off(&acrtc->base);
+	list_del_init(&acrtc->base.head);
+}
+
 /*
  * dm_test_crtc_arm_irq_src - Prime an IRQ source so get()/put() short-circuit.
  * @test: The KUnit test context
@@ -944,8 +976,13 @@ dm_test_crtc_setup_enable(struct kunit *test, struct amdgpu_device **adev_out,
 	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
 	acrtc->base.dev = &adev->ddev;
+	acrtc->base.funcs = &dm_test_crtc_funcs;
 	acrtc->base.enabled = true;
 	acrtc->crtc_id = 0;
+	INIT_LIST_HEAD(&acrtc->base.head);
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_crtc_cleanup, acrtc), 0);
 
 	link = dm_kunit_alloc_link(test);
 	stream = dm_kunit_alloc_stream(test, link);
@@ -1227,6 +1264,103 @@ static void dm_test_crtc_enable_vblank_ips_restore_replay(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_crtc_enable_vblank(&acrtc->base), 0);
 }
 
+/* Tests for amdgpu_dm_crtc_set_vline0_irq() */
+
+/*
+ * dm_test_crtc_setup_vline0 - Build an adev/CRTC for the vline0 IRQ helper.
+ * @test: The KUnit test context
+ * @adev_out: Receives the allocated device
+ * @ip_version: DCE IP version stamped on the device (0 selects the DCE path)
+ *
+ * Returns a bare CRTC attached to the device. The IRQ subsystem is left
+ * uninstalled for callers to arm.
+ */
+static struct drm_crtc *dm_test_crtc_setup_vline0(struct kunit *test,
+						  struct amdgpu_device **adev_out,
+						  uint32_t ip_version)
+{
+	struct amdgpu_device *adev;
+	struct drm_crtc *crtc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+
+	adev->ip_versions[DCE_HWIP][0] = ip_version;
+
+	*adev_out = adev;
+	return crtc;
+}
+
+/**
+ * dm_test_crtc_set_vline0_irq_dce_noop - Test vline0 irq is skipped on DCE
+ * @test: The KUnit test context
+ *
+ * VLINE0 only exists on DCN+. With no DCE IP version stamped the helper must
+ * return 0 without touching the IRQ source, even though the IRQ subsystem is
+ * uninstalled (which would otherwise make amdgpu_irq_get() fail).
+ */
+static void dm_test_crtc_set_vline0_irq_dce_noop(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_crtc *crtc;
+
+	crtc = dm_test_crtc_setup_vline0(test, &adev, 0);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_crtc_set_vline0_irq(crtc, AMDGPU_CRTC_IRQ_VBLANK1, true), 0);
+}
+
+/**
+ * dm_test_crtc_set_vline0_irq_error - Test vline0 irq failure is propagated
+ * @test: The KUnit test context
+ *
+ * On DCN with the IRQ subsystem uninstalled, amdgpu_irq_get() returns -ENOENT
+ * and the helper must propagate it.
+ */
+static void dm_test_crtc_set_vline0_irq_error(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_crtc *crtc;
+
+	crtc = dm_test_crtc_setup_vline0(test, &adev, IP_VERSION(3, 5, 0));
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_crtc_set_vline0_irq(crtc, AMDGPU_CRTC_IRQ_VBLANK1, true),
+			-ENOENT);
+}
+
+/**
+ * dm_test_crtc_set_vline0_irq_enable_disable - Test vline0 irq refcounting
+ * @test: The KUnit test context
+ *
+ * On DCN with an armed IRQ source, enabling takes a vline0 reference and
+ * disabling drops it again.
+ */
+static void dm_test_crtc_set_vline0_irq_enable_disable(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_crtc *crtc;
+	atomic_t *refcount;
+
+	crtc = dm_test_crtc_setup_vline0(test, &adev, IP_VERSION(3, 5, 0));
+
+	adev->irq.installed = true;
+	dm_test_crtc_arm_irq_src(test, &adev->vline0_irq, 1);
+	refcount = &adev->vline0_irq.enabled_types[AMDGPU_CRTC_IRQ_VBLANK1];
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_crtc_set_vline0_irq(crtc, AMDGPU_CRTC_IRQ_VBLANK1, true), 0);
+	KUNIT_EXPECT_EQ(test, atomic_read(refcount), 2);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_crtc_set_vline0_irq(crtc, AMDGPU_CRTC_IRQ_VBLANK1, false), 0);
+	KUNIT_EXPECT_EQ(test, atomic_read(refcount), 1);
+}
+
 /* Tests for amdgpu_dm_crtc_update_crtc_active_planes() */
 
 /**
@@ -1408,18 +1542,118 @@ static void dm_test_crtc_duplicate_state_copies_fields(struct kunit *test)
 	amdgpu_dm_crtc_destroy_state(crtc, dup);
 }
 
-/* Tests for amdgpu_dm_crtc_reset_state() */
+/**
+ * dm_test_crtc_duplicate_state_retains_stream - Test duplicate retains the stream
+ * @test: The KUnit test context
+ *
+ * When the current CRTC state carries a DC stream, duplicating the state must
+ * copy the stream pointer and take an additional reference on it. Destroying
+ * the duplicate then drops that reference back to the KUnit-managed one.
+ */
+static void dm_test_crtc_duplicate_state_retains_stream(struct kunit *test)
+{
+	struct dc_stream_state *stream;
+	struct drm_crtc *crtc;
+	struct dm_crtc_state *cur;
+	struct drm_crtc_state *dup;
+	struct dm_crtc_state *dm_dup;
+	struct dc_link *link;
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	cur = kunit_kzalloc(test, sizeof(*cur), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cur);
+
+	link = dm_kunit_alloc_link(test);
+	stream = dm_kunit_alloc_stream(test, link);
+
+	cur->stream = stream;
+	crtc->state = &cur->base;
+
+	dup = amdgpu_dm_crtc_duplicate_state(crtc);
+	KUNIT_ASSERT_NOT_NULL(test, dup);
+
+	dm_dup = to_dm_crtc_state(dup);
+	KUNIT_EXPECT_PTR_EQ(test, dm_dup->stream, stream);
+	/* The duplicate took a second reference on top of the managed one. */
+	KUNIT_EXPECT_EQ(test, kref_read(&stream->refcount), 2);
+
+	/* Destroying the duplicate drops back to the KUnit-managed reference. */
+	amdgpu_dm_crtc_destroy_state(crtc, dup);
+	KUNIT_EXPECT_EQ(test, kref_read(&stream->refcount), 1);
+}
+
+/**
+ * dm_test_crtc_duplicate_state_null_state_returns_null - Test guard on missing state
+ * @test: The KUnit test context
+ *
+ * Duplicating a CRTC whose current state is NULL must trip the WARN_ON guard
+ * and return NULL without allocating a new state.
+ */
+static void dm_test_crtc_duplicate_state_null_state_returns_null(struct kunit *test)
+{
+	struct drm_crtc *crtc;
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->state = NULL;
+
+	KUNIT_EXPECT_NULL(test, amdgpu_dm_crtc_duplicate_state(crtc));
+}
+
+/* Tests for amdgpu_dm_crtc_destroy() */
 
 /**
- * dm_test_crtc_reset_state_allocates_state - Test reset installs a fresh state
+ * dm_test_crtc_destroy_cleans_up_and_frees - Test destroy tears down the CRTC
  * @test: The KUnit test context
  *
- * Resetting a CRTC with no existing state must allocate and install a new
- * drm_crtc_state.
+ * amdgpu_dm_crtc_destroy() is the drm_crtc .destroy callback: it must run
+ * drm_crtc_cleanup() and free the CRTC. Initialise a CRTC with a primary plane
+ * so it is registered on the device (num_crtc == 1), then destroy it and verify
+ * the CRTC was unregistered (num_crtc back to 0). The CRTC is a plain (unmanaged)
+ * allocation because amdgpu_dm_crtc_destroy() kfree()s it.
  */
-static void dm_test_crtc_reset_state_allocates_state(struct kunit *test)
+static void dm_test_crtc_destroy_cleans_up_and_frees(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct drm_plane *plane;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	plane = drm_kunit_helper_create_primary_plane(test, &adev->ddev,
+						      NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane);
+
+	/* amdgpu_dm_crtc_destroy() kfree()s the CRTC, so use a plain alloc. */
+	acrtc = kzalloc_obj(*acrtc, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	ret = drm_crtc_init_with_planes(&adev->ddev, &acrtc->base, plane,
+					NULL, &dm_test_crtc_funcs, NULL);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.num_crtc, 1);
+
+	amdgpu_dm_crtc_destroy(&acrtc->base);
+
+	/* drm_crtc_cleanup() ran: the CRTC was unregistered from the device. */
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.num_crtc, 0);
+}
+
+/* Tests for amdgpu_dm_crtc_create_state() */
+
+/**
+ * dm_test_crtc_create_state_allocates_state - Test create_state allocates a fresh state
+ * @test: The KUnit test context
+ *
+ * Creating state for a CRTC must allocate a new drm_crtc_state.
+ */
+static void dm_test_crtc_create_state_allocates_state(struct kunit *test)
 {
 	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_crtc_state *crtc_state;
 	struct drm_crtc *crtc;
 
 	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
@@ -1427,8 +1661,49 @@ static void dm_test_crtc_reset_state_allocates_state(struct kunit *test)
 	crtc->dev = &adev->ddev;
 	crtc->state = NULL;
 
-	amdgpu_dm_crtc_reset_state(crtc);
+	crtc_state = amdgpu_dm_crtc_create_state(crtc);
+	KUNIT_EXPECT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	if (!IS_ERR(crtc_state))
+		amdgpu_dm_crtc_destroy_state(crtc, crtc_state);
+}
+
+/**
+ * dm_test_crtc_reset_state_replaces_existing - Test reset frees the old state
+ * @test: The KUnit test context
+ *
+ * Resetting a CRTC that already carries a state must destroy the existing
+ * state before installing a fresh one. The old state holds a stream reference,
+ * so a successful reset drops that reference (via amdgpu_dm_crtc_destroy_state)
+ * and leaves the CRTC with a new, non-NULL state.
+ */
+static void dm_test_crtc_reset_state_replaces_existing(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream;
+	struct dm_crtc_state *old;
+	struct drm_crtc *crtc;
+	struct dc_link *link;
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+
+	link = dm_kunit_alloc_link(test);
+	stream = dm_kunit_alloc_stream(test, link);
+	/* Extra ref so destroying the old state drops back to the managed one. */
+	kref_get(&stream->refcount);
 
+	/* reset_state kfree()s the old state, so use a plain (unmanaged) alloc. */
+	old = kzalloc_obj(*old, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, old);
+	old->stream = stream;
+	crtc->state = &old->base;
+
+	amdgpu_dm_crtc_create_state(crtc);
+
+	/* Old state was destroyed (stream ref dropped) and a new one installed. */
+	KUNIT_EXPECT_EQ(test, kref_read(&stream->refcount), 1);
 	KUNIT_EXPECT_NOT_NULL(test, crtc->state);
 
 	if (crtc->state)
@@ -1541,6 +1816,11 @@ static void dm_test_crtc_handle_vblank_skips_when_flip_submitted(struct kunit *t
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, event);
 
 	acrtc->base.dev = &adev->ddev;
+	acrtc->base.funcs = &dm_test_crtc_funcs;
+	INIT_LIST_HEAD(&acrtc->base.head);
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_crtc_cleanup, acrtc), 0);
 	acrtc->event = event;
 	acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
 
@@ -1577,6 +1857,11 @@ static void dm_test_crtc_handle_vblank_completes_cursor_only(struct kunit *test)
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, event);
 
 	acrtc->base.dev = &adev->ddev;
+	acrtc->base.funcs = &dm_test_crtc_funcs;
+	INIT_LIST_HEAD(&acrtc->base.head);
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_crtc_cleanup, acrtc), 0);
 	acrtc->event = event;
 	acrtc->pflip_status = AMDGPU_FLIP_NONE;
 
@@ -1852,10 +2137,491 @@ static void dm_test_crtc_disable_vblank_queues_work(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, adev->dm.active_vblank_irq_count, 1);
 }
 
+#ifdef CONFIG_DEBUG_FS
+/**
+ * dm_test_crtc_late_register_inits_debugfs - Test late_register succeeds
+ * @test: The KUnit test context
+ *
+ * amdgpu_dm_crtc_late_register populates the CRTC's debugfs entries via
+ * crtc_debugfs_init and must return 0. Provide a debugfs parent so the
+ * helper attaches its files there, then tear it down.
+ */
+static void dm_test_crtc_late_register_inits_debugfs(struct kunit *test)
+{
+	struct drm_crtc *crtc;
+	int ret;
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+
+	crtc->debugfs_entry = debugfs_create_dir("dm_test_crtc", NULL);
+
+	ret = amdgpu_dm_crtc_late_register(crtc);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	debugfs_remove_recursive(crtc->debugfs_entry);
+}
+#endif
+
+/* Tests for amdgpu_dm_crtc_init() */
+
+/**
+ * dm_test_crtc_init_registers_crtc - Test amdgpu_dm_crtc_init success path
+ * @test: The KUnit test context
+ *
+ * amdgpu_dm_crtc_init allocates an amdgpu_crtc, wires it up with the primary
+ * and a new cursor plane, resets its state, and records it in mode_info. It
+ * must return 0, register the CRTC with the DRM device, and initialize the
+ * CRTC's identifying fields and cursor size limits.
+ */
+static void dm_test_crtc_init_registers_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct drm_plane *cursor;
+	struct drm_plane *plane;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	dc->caps.max_cursor_size = 256;
+
+	adev->dm.adev = adev;
+	adev->dm.ddev = &adev->ddev;
+	adev->dm.dc = dc;
+
+	plane = drm_kunit_helper_create_primary_plane(test, &adev->ddev,
+						      NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane);
+
+	ret = amdgpu_dm_crtc_init(&adev->dm, plane, 0);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.num_crtc, 1);
+
+	acrtc = adev->mode_info.crtcs[0];
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	KUNIT_EXPECT_EQ(test, acrtc->crtc_id, 0);
+	KUNIT_EXPECT_EQ(test, acrtc->otg_inst, -1);
+	KUNIT_EXPECT_FALSE(test, acrtc->base.enabled);
+	KUNIT_EXPECT_EQ(test, acrtc->max_cursor_width, 256);
+	KUNIT_EXPECT_EQ(test, acrtc->max_cursor_height, 256);
+
+	/*
+	 * Clean up the objects created inside amdgpu_dm_crtc_init(): free the
+	 * reset-installed CRTC state, destroy the CRTC (which kfree()s it), then
+	 * destroy the cursor plane it allocated.
+	 */
+	cursor = acrtc->base.cursor;
+	if (acrtc->base.state) {
+		amdgpu_dm_crtc_destroy_state(&acrtc->base, acrtc->base.state);
+		acrtc->base.state = NULL;
+	}
+	amdgpu_dm_crtc_destroy(&acrtc->base);
+	if (cursor)
+		cursor->funcs->destroy(cursor);
+}
+
+/**
+ * dm_test_crtc_init_enables_degamma - Test amdgpu_dm_crtc_init degamma path
+ * @test: The KUnit test context
+ *
+ * When the DPP reports a DCN architecture (and the ASIC is not DCN 4.01),
+ * amdgpu_dm_crtc_init must enable the DRM CRTC degamma LUT. Exercise that
+ * has_degamma == true branch and confirm the CRTC is still initialized and
+ * registered successfully.
+ */
+static void dm_test_crtc_init_enables_degamma(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct drm_plane *cursor;
+	struct drm_plane *plane;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	/* DCN DPP with a non-4.01 ASIC selects the degamma-enabled path. */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->ctx->dce_version = DCN_VERSION_3_0;
+
+	adev->dm.adev = adev;
+	adev->dm.ddev = &adev->ddev;
+	adev->dm.dc = dc;
+
+	plane = drm_kunit_helper_create_primary_plane(test, &adev->ddev,
+						      NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane);
+
+	ret = amdgpu_dm_crtc_init(&adev->dm, plane, 0);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.num_crtc, 1);
+
+	acrtc = adev->mode_info.crtcs[0];
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	KUNIT_EXPECT_EQ(test, acrtc->otg_inst, -1);
+
+	/* Free the reset state, destroy the CRTC, then the cursor plane. */
+	cursor = acrtc->base.cursor;
+	if (acrtc->base.state) {
+		amdgpu_dm_crtc_destroy_state(&acrtc->base, acrtc->base.state);
+		acrtc->base.state = NULL;
+	}
+	amdgpu_dm_crtc_destroy(&acrtc->base);
+	if (cursor)
+		cursor->funcs->destroy(cursor);
+}
+
+/* Tests for amdgpu_dm_crtc_helper_atomic_check() */
+
+/**
+ * dm_test_crtc_atomic_check_no_stream_passes - Test atomic_check with no stream
+ * @test: The KUnit test context
+ *
+ * A CRTC state without a stream (as during reset) and with no modeset
+ * requirement must pass atomic_check and return 0 before reaching DC stream
+ * validation.
+ */
+static void dm_test_crtc_atomic_check_no_stream_passes(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dm_crtc_state *dm_crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_crtc_state);
+	dm_crtc_state->base.crtc = crtc;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	state->dev = &adev->ddev;
+	state->crtcs = kunit_kzalloc(test, sizeof(*state->crtcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->crtcs);
+	state->crtcs[0].new_state = &dm_crtc_state->base;
+	dm_crtc_state->base.state = state;
+
+	ret = amdgpu_dm_crtc_helper_atomic_check(crtc, state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/* Stubs for the DC stream validation branches of atomic_check. */
+static bool dm_test_atomic_check_validate_timing_ok(struct timing_generator *tg,
+						    const struct dc_crtc_timing *timing)
+{
+	return true;
+}
+
+static bool dm_test_atomic_check_validate_timing_fail(struct timing_generator *tg,
+						      const struct dc_crtc_timing *timing)
+{
+	return false;
+}
+
+static enum dc_status dm_test_atomic_check_validate_mode_timing_ok(const struct dc_stream_state *stream,
+								  struct dc_link *link,
+								  const struct dc_crtc_timing *timing)
+{
+	return DC_OK;
+}
+
+/**
+ * dm_test_crtc_atomic_check_enable_without_primary_fails - Enabled CRTC needs primary
+ * @test: The KUnit test context
+ *
+ * An enabled CRTC whose primary plane is not part of the plane mask must be
+ * rejected with -EINVAL.
+ */
+static void dm_test_crtc_atomic_check_enable_without_primary_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dm_crtc_state *dm_crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_plane *primary;
+	struct drm_crtc *crtc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	primary = kunit_kzalloc(test, sizeof(*primary), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, primary);
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+	crtc->primary = primary;
+
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_crtc_state);
+	dm_crtc_state->base.crtc = crtc;
+	dm_crtc_state->base.enable = true;
+	dm_crtc_state->base.plane_mask = 0;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	state->dev = &adev->ddev;
+	state->crtcs = kunit_kzalloc(test, sizeof(*state->crtcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->crtcs);
+	state->crtcs[0].new_state = &dm_crtc_state->base;
+	dm_crtc_state->base.state = state;
+
+	ret = amdgpu_dm_crtc_helper_atomic_check(crtc, state);
+
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_crtc_atomic_check_async_flip_non_fast_fails - Async flip needs fast update
+ * @test: The KUnit test context
+ *
+ * An async flip is only permitted for fast updates; a non-fast update type must
+ * be rejected with -EINVAL.
+ */
+static void dm_test_crtc_atomic_check_async_flip_non_fast_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dm_crtc_state *dm_crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_crtc_state);
+	dm_crtc_state->base.crtc = crtc;
+	dm_crtc_state->base.async_flip = true;
+	dm_crtc_state->update_type = UPDATE_TYPE_FULL;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	state->dev = &adev->ddev;
+	state->crtcs = kunit_kzalloc(test, sizeof(*state->crtcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->crtcs);
+	state->crtcs[0].new_state = &dm_crtc_state->base;
+	dm_crtc_state->base.state = state;
+
+	ret = amdgpu_dm_crtc_helper_atomic_check(crtc, state);
+
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_crtc_atomic_check_vrr_pulls_primary - VRR path pulls in primary plane
+ * @test: The KUnit test context
+ *
+ * When VRR is active and the update is not a legacy cursor update, atomic_check
+ * pulls in the primary plane state. With an existing primary plane state already
+ * present and no stream attached, the check returns 0.
+ */
+static void dm_test_crtc_atomic_check_vrr_pulls_primary(struct kunit *test)
+{
+	struct drm_plane_state *primary_plane_state;
+	struct amdgpu_device *adev;
+	struct dm_crtc_state *dm_crtc_state;
+	struct drm_atomic_commit *state;
+	struct drm_plane *primary;
+	struct drm_crtc *crtc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	primary = kunit_kzalloc(test, sizeof(*primary), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, primary);
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+	crtc->primary = primary;
+
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_crtc_state);
+	dm_crtc_state->base.crtc = crtc;
+	dm_crtc_state->freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	state->dev = &adev->ddev;
+	state->crtcs = kunit_kzalloc(test, sizeof(*state->crtcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->crtcs);
+	state->crtcs[0].new_state = &dm_crtc_state->base;
+	dm_crtc_state->base.state = state;
+
+	/* acquire ctx must be set; existing plane state avoids taking a lock. */
+	state->acquire_ctx = kunit_kzalloc(test, sizeof(*state->acquire_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->acquire_ctx);
+	state->planes = kunit_kzalloc(test, sizeof(*state->planes), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->planes);
+	primary_plane_state = kunit_kzalloc(test, sizeof(*primary_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, primary_plane_state);
+	state->planes[0].new_state = primary_plane_state;
+
+	ret = amdgpu_dm_crtc_helper_atomic_check(crtc, state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/**
+ * dm_test_crtc_atomic_check_valid_stream_passes - Valid DC stream passes
+ * @test: The KUnit test context
+ *
+ * With a stream attached and DC stream validation returning DC_OK, atomic_check
+ * must return 0.
+ */
+static void dm_test_crtc_atomic_check_valid_stream_passes(struct kunit *test)
+{
+	struct timing_generator_funcs *tg_funcs;
+	struct timing_generator *tg;
+	struct resource_pool *res_pool;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dm_crtc_state *dm_crtc_state;
+	struct drm_atomic_commit *state;
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+	struct drm_crtc *crtc;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res_pool);
+	tg = kunit_kzalloc(test, sizeof(*tg), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tg);
+	tg_funcs = kunit_kzalloc(test, sizeof(*tg_funcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tg_funcs);
+	tg_funcs->validate_timing = dm_test_atomic_check_validate_timing_ok;
+	tg->funcs = tg_funcs;
+	res_pool->timing_generators[0] = tg;
+	dc->res_pool = res_pool;
+
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
+	link_srv->validate_mode_timing = dm_test_atomic_check_validate_mode_timing_ok;
+	dc->link_srv = link_srv;
+	adev->dm.dc = dc;
+
+	link = dm_kunit_alloc_link(test);
+	link->ep_type = DISPLAY_ENDPOINT_UNKNOWN;
+	stream = dm_kunit_alloc_stream(test, link);
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_crtc_state);
+	dm_crtc_state->base.crtc = crtc;
+	dm_crtc_state->stream = stream;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	state->dev = &adev->ddev;
+	state->crtcs = kunit_kzalloc(test, sizeof(*state->crtcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->crtcs);
+	state->crtcs[0].new_state = &dm_crtc_state->base;
+	dm_crtc_state->base.state = state;
+
+	ret = amdgpu_dm_crtc_helper_atomic_check(crtc, state);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/**
+ * dm_test_crtc_atomic_check_invalid_stream_fails - Invalid DC stream fails
+ * @test: The KUnit test context
+ *
+ * With a stream attached but DC stream validation failing (timing rejected),
+ * atomic_check must return -EINVAL.
+ */
+static void dm_test_crtc_atomic_check_invalid_stream_fails(struct kunit *test)
+{
+	struct timing_generator_funcs *tg_funcs;
+	struct timing_generator *tg;
+	struct resource_pool *res_pool;
+	struct amdgpu_device *adev;
+	struct dm_crtc_state *dm_crtc_state;
+	struct drm_atomic_commit *state;
+	struct dc_stream_state *stream;
+	struct dc_link *link;
+	struct drm_crtc *crtc;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, res_pool);
+	tg = kunit_kzalloc(test, sizeof(*tg), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tg);
+	tg_funcs = kunit_kzalloc(test, sizeof(*tg_funcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tg_funcs);
+	tg_funcs->validate_timing = dm_test_atomic_check_validate_timing_fail;
+	tg->funcs = tg_funcs;
+	res_pool->timing_generators[0] = tg;
+	dc->res_pool = res_pool;
+	adev->dm.dc = dc;
+
+	link = dm_kunit_alloc_link(test);
+	stream = dm_kunit_alloc_stream(test, link);
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+	crtc->dev = &adev->ddev;
+
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dm_crtc_state);
+	dm_crtc_state->base.crtc = crtc;
+	dm_crtc_state->stream = stream;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+	state->dev = &adev->ddev;
+	state->crtcs = kunit_kzalloc(test, sizeof(*state->crtcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state->crtcs);
+	state->crtcs[0].new_state = &dm_crtc_state->base;
+	dm_crtc_state->base.state = state;
+
+	ret = amdgpu_dm_crtc_helper_atomic_check(crtc, state);
+
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
 static struct kunit_case amdgpu_dm_crtc_tests[] = {
 	/* amdgpu_dm_crtc_modeset_required */
 	KUNIT_CASE(dm_test_crtc_modeset_required_active_mode_changed),
-	KUNIT_CASE(dm_test_crtc_modeset_required_active_active_changed),
+	KUNIT_CASE(dm_test_crtc_modeset_required_active_state_changed),
 	KUNIT_CASE(dm_test_crtc_modeset_required_active_connectors_changed),
 	KUNIT_CASE(dm_test_crtc_modeset_required_inactive),
 	KUNIT_CASE(dm_test_crtc_modeset_required_no_changes),
@@ -1904,6 +2670,10 @@ static struct kunit_case amdgpu_dm_crtc_tests[] = {
 	KUNIT_CASE(dm_test_crtc_enable_vblank_queues_work),
 	KUNIT_CASE(dm_test_crtc_enable_vblank_ips_restore),
 	KUNIT_CASE(dm_test_crtc_enable_vblank_ips_restore_replay),
+	/* amdgpu_dm_crtc_set_vline0_irq */
+	KUNIT_CASE(dm_test_crtc_set_vline0_irq_dce_noop),
+	KUNIT_CASE(dm_test_crtc_set_vline0_irq_error),
+	KUNIT_CASE(dm_test_crtc_set_vline0_irq_enable_disable),
 	/* amdgpu_dm_crtc_update_crtc_active_planes */
 	KUNIT_CASE(dm_test_crtc_update_active_planes_no_stream),
 	/* amdgpu_dm_crtc_count_crtc_active_planes */
@@ -1911,8 +2681,13 @@ static struct kunit_case amdgpu_dm_crtc_tests[] = {
 	KUNIT_CASE(dm_test_count_crtc_active_planes_mixed),
 	/* amdgpu_dm_crtc_duplicate_state */
 	KUNIT_CASE(dm_test_crtc_duplicate_state_copies_fields),
-	/* amdgpu_dm_crtc_reset_state */
-	KUNIT_CASE(dm_test_crtc_reset_state_allocates_state),
+	KUNIT_CASE(dm_test_crtc_duplicate_state_retains_stream),
+	KUNIT_CASE(dm_test_crtc_duplicate_state_null_state_returns_null),
+	/* amdgpu_dm_crtc_destroy */
+	KUNIT_CASE(dm_test_crtc_destroy_cleans_up_and_frees),
+	/* amdgpu_dm_crtc_create_state */
+	KUNIT_CASE(dm_test_crtc_create_state_allocates_state),
+	KUNIT_CASE(dm_test_crtc_reset_state_replaces_existing),
 	/* amdgpu_dm_crtc_destroy_state */
 	KUNIT_CASE(dm_test_crtc_destroy_state_no_stream),
 	KUNIT_CASE(dm_test_crtc_destroy_state_releases_stream),
@@ -1928,6 +2703,20 @@ static struct kunit_case amdgpu_dm_crtc_tests[] = {
 	KUNIT_CASE(dm_test_crtc_disable_vblank_no_irq_installed),
 	KUNIT_CASE(dm_test_crtc_disable_vblank_vrr),
 	KUNIT_CASE(dm_test_crtc_disable_vblank_queues_work),
+#ifdef CONFIG_DEBUG_FS
+	/* amdgpu_dm_crtc_late_register */
+	KUNIT_CASE(dm_test_crtc_late_register_inits_debugfs),
+#endif
+	/* amdgpu_dm_crtc_init */
+	KUNIT_CASE(dm_test_crtc_init_registers_crtc),
+	KUNIT_CASE(dm_test_crtc_init_enables_degamma),
+	/* amdgpu_dm_crtc_helper_atomic_check */
+	KUNIT_CASE(dm_test_crtc_atomic_check_no_stream_passes),
+	KUNIT_CASE(dm_test_crtc_atomic_check_enable_without_primary_fails),
+	KUNIT_CASE(dm_test_crtc_atomic_check_async_flip_non_fast_fails),
+	KUNIT_CASE(dm_test_crtc_atomic_check_vrr_pulls_primary),
+	KUNIT_CASE(dm_test_crtc_atomic_check_valid_stream_passes),
+	KUNIT_CASE(dm_test_crtc_atomic_check_invalid_stream_fails),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_cursor_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_cursor_test.c
index 3da76794ece968..2212451043fb1f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_cursor_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_cursor_test.c
@@ -8,7 +8,10 @@
 #include <kunit/test.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_blend.h>
+#include <drm/drm_colorop.h>
 #include <drm/drm_crtc.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
 #include <drm/drm_plane.h>
 
 #include "dc.h"
@@ -16,6 +19,708 @@
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_cursor.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
+
+struct dm_cursor_fb_fixture {
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct amdgpu_framebuffer *afb;
+	struct drm_plane_state *plane_state;
+};
+
+static struct dm_cursor_fb_fixture dm_test_alloc_cursor_fb_fixture(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = { 0 };
+
+	fixture.adev = dm_kunit_alloc_adev(test);
+	fixture.acrtc = kunit_kzalloc(test, sizeof(*fixture.acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.acrtc);
+	fixture.afb = kunit_kzalloc(test, sizeof(*fixture.afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.afb);
+	fixture.plane_state = kunit_kzalloc(test, sizeof(*fixture.plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.plane_state);
+
+	fixture.acrtc->base.dev = &fixture.adev->ddev;
+	fixture.acrtc->max_cursor_width = 256;
+	fixture.acrtc->max_cursor_height = 256;
+	fixture.afb->base.width = 64;
+	fixture.afb->base.height = 64;
+	fixture.afb->base.pitches[0] = 64 * 4;
+	fixture.afb->base.format = drm_format_info(DRM_FORMAT_ARGB8888);
+	fixture.plane_state->fb = &fixture.afb->base;
+	fixture.plane_state->src_w = 64 << 16;
+	fixture.plane_state->src_h = 64 << 16;
+
+	return fixture;
+}
+
+struct dm_cursor_mode_fixture {
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct dm_crtc_state *dm_crtc_state;
+	struct drm_crtc *crtc;
+	struct drm_plane *cursor;
+	struct drm_plane *primary;
+	struct drm_plane_state *old_cursor_state;
+	struct drm_plane_state *cursor_state;
+	struct drm_plane_state *old_primary_state;
+	struct drm_plane_state *primary_state;
+	struct drm_framebuffer *cursor_fb;
+	struct drm_framebuffer *primary_fb;
+};
+
+static struct dm_cursor_mode_fixture dm_test_alloc_cursor_mode_fixture(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = { 0 };
+
+	fixture.adev = dm_kunit_alloc_adev(test);
+	fixture.state = kunit_kzalloc(test, sizeof(*fixture.state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.state);
+	fixture.dm_crtc_state = kunit_kzalloc(test, sizeof(*fixture.dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.dm_crtc_state);
+	fixture.crtc = kunit_kzalloc(test, sizeof(*fixture.crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.crtc);
+	fixture.cursor = kunit_kzalloc(test, sizeof(*fixture.cursor), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.cursor);
+	fixture.primary = kunit_kzalloc(test, sizeof(*fixture.primary), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.primary);
+	fixture.old_cursor_state =
+		kunit_kzalloc(test, sizeof(*fixture.old_cursor_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.old_cursor_state);
+	fixture.cursor_state = kunit_kzalloc(test, sizeof(*fixture.cursor_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.cursor_state);
+	fixture.old_primary_state =
+		kunit_kzalloc(test, sizeof(*fixture.old_primary_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.old_primary_state);
+	fixture.primary_state = kunit_kzalloc(test, sizeof(*fixture.primary_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.primary_state);
+	fixture.cursor_fb = kunit_kzalloc(test, sizeof(*fixture.cursor_fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.cursor_fb);
+	fixture.primary_fb = kunit_kzalloc(test, sizeof(*fixture.primary_fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.primary_fb);
+	fixture.state->planes = kunit_kcalloc(test, 2, sizeof(*fixture.state->planes), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.state->planes);
+	fixture.state->acquire_ctx =
+		kunit_kzalloc(test, sizeof(*fixture.state->acquire_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.state->acquire_ctx);
+
+	fixture.adev->ddev.mode_config.num_total_plane = 2;
+	fixture.state->dev = &fixture.adev->ddev;
+	fixture.crtc->dev = &fixture.adev->ddev;
+	fixture.crtc->cursor = fixture.cursor;
+	fixture.cursor->dev = &fixture.adev->ddev;
+	fixture.cursor->index = 0;
+	fixture.cursor->type = DRM_PLANE_TYPE_CURSOR;
+	fixture.cursor->base.id = 2;
+	fixture.primary->dev = &fixture.adev->ddev;
+	fixture.primary->index = 1;
+	fixture.primary->type = DRM_PLANE_TYPE_PRIMARY;
+	fixture.primary->base.id = 1;
+
+	fixture.cursor_fb->format = drm_format_info(DRM_FORMAT_ARGB8888);
+	fixture.primary_fb->format = drm_format_info(DRM_FORMAT_XRGB8888);
+	fixture.old_cursor_state->plane = fixture.cursor;
+	fixture.old_cursor_state->crtc = fixture.crtc;
+	fixture.old_cursor_state->fb = fixture.cursor_fb;
+	fixture.old_cursor_state->src_w = 64 << 16;
+	fixture.old_cursor_state->src_h = 64 << 16;
+	fixture.old_cursor_state->crtc_w = 64;
+	fixture.old_cursor_state->crtc_h = 64;
+	fixture.old_cursor_state->zpos = 1;
+	*fixture.cursor_state = *fixture.old_cursor_state;
+	fixture.cursor_state->state = fixture.state;
+	fixture.old_primary_state->plane = fixture.primary;
+	fixture.old_primary_state->crtc = fixture.crtc;
+	fixture.old_primary_state->fb = fixture.primary_fb;
+	fixture.old_primary_state->src_w = 1920 << 16;
+	fixture.old_primary_state->src_h = 1080 << 16;
+	fixture.old_primary_state->crtc_w = 1920;
+	fixture.old_primary_state->crtc_h = 1080;
+	fixture.old_primary_state->zpos = 0;
+	*fixture.primary_state = *fixture.old_primary_state;
+	fixture.primary_state->state = fixture.state;
+
+	fixture.state->planes[0].ptr = fixture.cursor;
+	fixture.state->planes[0].old_state = fixture.old_cursor_state;
+	fixture.state->planes[0].new_state = fixture.cursor_state;
+	fixture.state->planes[1].ptr = fixture.primary;
+	fixture.state->planes[1].old_state = fixture.old_primary_state;
+	fixture.state->planes[1].new_state = fixture.primary_state;
+	fixture.dm_crtc_state->base.crtc = fixture.crtc;
+	fixture.dm_crtc_state->base.enable = true;
+	fixture.dm_crtc_state->base.plane_mask = drm_plane_mask(fixture.cursor) |
+						 drm_plane_mask(fixture.primary);
+	fixture.dm_crtc_state->base.zpos_changed = true;
+	fixture.dm_crtc_state->base.mode.hdisplay = 1920;
+	fixture.dm_crtc_state->base.mode.vdisplay = 1080;
+
+	return fixture;
+}
+
+static void dm_test_add_cursor_mode_colorop(struct kunit *test,
+					    struct dm_cursor_mode_fixture *fixture,
+					    bool old_bypass, bool new_bypass)
+{
+	struct drm_colorop_state *old_colorop_state;
+	struct drm_colorop_state *new_colorop_state;
+	struct drm_colorop *colorop;
+
+	colorop = kunit_kzalloc(test, sizeof(*colorop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, colorop);
+	old_colorop_state = kunit_kzalloc(test, sizeof(*old_colorop_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_colorop_state);
+	new_colorop_state = kunit_kzalloc(test, sizeof(*new_colorop_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_colorop_state);
+	fixture->state->colorops =
+		kunit_kzalloc(test, sizeof(*fixture->state->colorops), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture->state->colorops);
+
+	fixture->adev->ddev.mode_config.num_colorop = 1;
+	fixture->state->colorops[0].ptr = colorop;
+	fixture->state->colorops[0].old_state = old_colorop_state;
+	fixture->state->colorops[0].new_state = new_colorop_state;
+	colorop->plane = fixture->primary;
+	old_colorop_state->colorop = colorop;
+	old_colorop_state->bypass = old_bypass;
+	new_colorop_state->colorop = colorop;
+	new_colorop_state->bypass = new_bypass;
+}
+
+static int dm_test_get_cursor_mode(struct dm_cursor_mode_fixture *fixture,
+				   enum amdgpu_dm_cursor_mode *cursor_mode)
+{
+	return amdgpu_dm_crtc_get_cursor_mode(fixture->adev, fixture->state,
+					      fixture->dm_crtc_state, cursor_mode);
+}
+
+/* Tests for dm_check_cursor_fb() */
+
+/**
+ * dm_test_check_cursor_fb_valid_linear - Test a supported linear cursor framebuffer
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_valid_linear(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.adev->family = AMDGPU_FAMILY_AI;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			0);
+}
+
+/**
+ * dm_test_check_cursor_fb_rejects_size - Test an oversized cursor framebuffer
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_rejects_size(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.afb->base.width = fixture.acrtc->max_cursor_width + 1;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			-EINVAL);
+}
+
+/**
+ * dm_test_check_cursor_fb_rejects_cropping - Test cursor framebuffer cropping rejection
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_rejects_cropping(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.plane_state->src_w = 32 << 16;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			-EINVAL);
+}
+
+/**
+ * dm_test_check_cursor_fb_rejects_pitch - Test unsupported cursor framebuffer pitch
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_rejects_pitch(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.afb->base.pitches[0] = 96 * 4;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			-EINVAL);
+}
+
+/**
+ * dm_test_check_cursor_fb_rejects_unsupported_pitch - Test matching unsupported pitch
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_rejects_unsupported_pitch(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.afb->base.width = 32;
+	fixture.afb->base.pitches[0] = 32 * 4;
+	fixture.plane_state->src_w = 32 << 16;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			-EINVAL);
+}
+
+/**
+ * dm_test_check_cursor_fb_rejects_tiling - Test tiled cursor framebuffer rejection
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_rejects_tiling(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.adev->family = AMDGPU_FAMILY_AI;
+	fixture.afb->tiling_flags = AMDGPU_TILING_SET(SWIZZLE_MODE, 1);
+
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			-EINVAL);
+}
+
+/**
+ * dm_test_check_cursor_fb_gfx12_tiling - Test GFX12 cursor tiling decoding
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_gfx12_tiling(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.adev->family = AMDGPU_FAMILY_GC_12_0_0;
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			0);
+
+	fixture.afb->tiling_flags = AMDGPU_TILING_SET(GFX12_SWIZZLE_MODE, 1);
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			-EINVAL);
+}
+
+/**
+ * dm_test_check_cursor_fb_pre_ai_tiling - Test legacy cursor tiling decoding
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_pre_ai_tiling(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.adev->family = AMDGPU_FAMILY_CZ;
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			0);
+
+	fixture.afb->tiling_flags = AMDGPU_TILING_SET(ARRAY_MODE, DC_ARRAY_1D_TILED_THIN1);
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			-EINVAL);
+}
+
+/**
+ * dm_test_check_cursor_fb_modifier_skips_tiling - Test modifier validation stays in DRM core
+ * @test: The KUnit test context
+ */
+static void dm_test_check_cursor_fb_modifier_skips_tiling(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+
+	fixture.adev->family = AMDGPU_FAMILY_AI;
+	fixture.afb->base.flags = DRM_MODE_FB_MODIFIERS;
+	fixture.afb->tiling_flags = AMDGPU_TILING_SET(SWIZZLE_MODE, 1);
+
+	KUNIT_EXPECT_EQ(test,
+			dm_check_cursor_fb(fixture.acrtc, fixture.plane_state, &fixture.afb->base),
+			0);
+}
+
+/* Tests for amdgpu_dm_check_native_cursor_state() */
+
+/**
+ * dm_test_check_native_cursor_state_disabled - Test disabled cursor needs no validation
+ * @test: The KUnit test context
+ */
+static void dm_test_check_native_cursor_state_disabled(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_check_native_cursor_state(NULL, NULL, NULL, false),
+			0);
+}
+
+/**
+ * dm_test_check_native_cursor_state_rejects_offset - Test source offset rejection
+ * @test: The KUnit test context
+ */
+static void dm_test_check_native_cursor_state_rejects_offset(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+	struct drm_plane *plane;
+	struct drm_framebuffer *fb;
+	struct drm_plane_state *old_plane_state;
+	struct drm_plane_state *new_plane_state;
+	int ret;
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+	old_plane_state = kunit_kzalloc(test, sizeof(*old_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_plane_state);
+	new_plane_state = kunit_kzalloc(test, sizeof(*new_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_plane_state);
+
+	acrtc->base.dev = &adev->ddev;
+	old_plane_state->crtc = &acrtc->base;
+	new_plane_state->crtc = &acrtc->base;
+	new_plane_state->fb = fb;
+	new_plane_state->src_x = 1;
+	plane->state = old_plane_state;
+
+	ret = amdgpu_dm_check_native_cursor_state(&acrtc->base, plane,
+						  new_plane_state, true);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/**
+ * dm_test_check_native_cursor_state_checks_fb - Test framebuffer validation propagation
+ * @test: The KUnit test context
+ */
+static void dm_test_check_native_cursor_state_checks_fb(struct kunit *test)
+{
+	struct dm_cursor_fb_fixture fixture = dm_test_alloc_cursor_fb_fixture(test);
+	struct drm_plane *plane;
+	struct drm_plane_state *old_plane_state;
+	int ret;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	old_plane_state = kunit_kzalloc(test, sizeof(*old_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_plane_state);
+	old_plane_state->crtc = &fixture.acrtc->base;
+	fixture.plane_state->crtc = &fixture.acrtc->base;
+	plane->state = old_plane_state;
+
+	ret = amdgpu_dm_check_native_cursor_state(&fixture.acrtc->base, plane,
+						  fixture.plane_state, true);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	fixture.afb->base.width = fixture.acrtc->max_cursor_width + 1;
+	ret = amdgpu_dm_check_native_cursor_state(&fixture.acrtc->base, plane,
+						  fixture.plane_state, true);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+/* Tests for dm_plane_color_pipeline_active() */
+
+/**
+ * dm_test_plane_color_pipeline_active - Test old and new colorop activity
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_color_pipeline_active(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state;
+	struct drm_colorop *colorop;
+	struct drm_colorop_state *old_colorop_state;
+	struct drm_colorop_state *new_colorop_state;
+	struct drm_plane *plane;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	colorop = kunit_kzalloc(test, sizeof(*colorop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, colorop);
+	old_colorop_state = kunit_kzalloc(test, sizeof(*old_colorop_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_colorop_state);
+	new_colorop_state = kunit_kzalloc(test, sizeof(*new_colorop_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_colorop_state);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	state->colorops = kunit_kzalloc(test, sizeof(*state->colorops), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state->colorops);
+
+	adev->ddev.mode_config.num_colorop = 1;
+	state->dev = &adev->ddev;
+	state->colorops[0].ptr = colorop;
+	state->colorops[0].old_state = old_colorop_state;
+	state->colorops[0].new_state = new_colorop_state;
+	colorop->plane = plane;
+	old_colorop_state->colorop = colorop;
+	old_colorop_state->bypass = true;
+	new_colorop_state->colorop = colorop;
+	new_colorop_state->bypass = false;
+
+	KUNIT_EXPECT_FALSE(test, dm_plane_color_pipeline_active(state, plane, true));
+	KUNIT_EXPECT_TRUE(test, dm_plane_color_pipeline_active(state, plane, false));
+}
+
+/**
+ * dm_test_plane_color_pipeline_ignores_other_plane - Test unrelated colorops are ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_color_pipeline_ignores_other_plane(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state;
+	struct drm_colorop *colorop;
+	struct drm_colorop_state *colorop_state;
+	struct drm_plane *colorop_plane;
+	struct drm_plane *other_plane;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	colorop = kunit_kzalloc(test, sizeof(*colorop), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, colorop);
+	colorop_state = kunit_kzalloc(test, sizeof(*colorop_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, colorop_state);
+	colorop_plane = kunit_kzalloc(test, sizeof(*colorop_plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, colorop_plane);
+	other_plane = kunit_kzalloc(test, sizeof(*other_plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, other_plane);
+	state->colorops = kunit_kzalloc(test, sizeof(*state->colorops), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state->colorops);
+
+	adev->ddev.mode_config.num_colorop = 1;
+	state->dev = &adev->ddev;
+	state->colorops[0].ptr = colorop;
+	state->colorops[0].new_state = colorop_state;
+	colorop->plane = colorop_plane;
+	colorop_state->colorop = colorop;
+
+	KUNIT_EXPECT_FALSE(test, dm_plane_color_pipeline_active(state, other_plane, false));
+}
+
+/* Tests for amdgpu_dm_crtc_get_cursor_mode() */
+
+/**
+ * dm_test_crtc_get_cursor_mode_disabled_crtc - Test a disabled CRTC uses native cursor
+ * @test: The KUnit test context
+ *
+ * A disabled CRTC must report native mode regardless of what the planes look
+ * like, so that a commit disabling the CRTC is not rejected. The plane setup
+ * here would otherwise select overlay mode.
+ */
+static void dm_test_crtc_get_cursor_mode_disabled_crtc(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_OVERLAY_MODE;
+
+	fixture.dm_crtc_state->base.enable = false;
+	fixture.old_primary_state->crtc_w = 1280;
+	fixture.primary_state->crtc_w = 1280;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_NATIVE_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_new_hardware - Test dcn4x uses native cursor when the top plane fills the CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_new_hardware(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_OVERLAY_MODE;
+
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 2, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_NATIVE_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_new_hardware_hole - Test dcn4x falls back to
+ * overlay cursor when the top plane does not fill the CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_new_hardware_hole(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 2, 0);
+	fixture.old_primary_state->crtc_w = 1280;
+	fixture.primary_state->crtc_w = 1280;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_no_change - Test unchanged atomic state preserves cursor mode
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_no_change(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state;
+	struct dm_crtc_state *dm_crtc_state;
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+	int ret;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	dm_crtc_state = kunit_kzalloc(test, sizeof(*dm_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_crtc_state);
+	state->dev = &adev->ddev;
+	dm_crtc_state->base.enable = true;
+	dm_crtc_state->cursor_mode = DM_CURSOR_OVERLAY_MODE;
+
+	ret = amdgpu_dm_crtc_get_cursor_mode(adev, state, dm_crtc_state, &cursor_mode);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_disabled_cursor - Test disabled cursor preserves mode
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_disabled_cursor(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.cursor_state->crtc = NULL;
+	fixture.cursor_state->fb = NULL;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_NATIVE_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_yuv_plane - Test YUV plane requires overlay cursor
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_yuv_plane(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.primary_fb->format = drm_format_info(DRM_FORMAT_NV12);
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_scale_mismatch - Test different scaling requires overlay
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_scale_mismatch(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.old_primary_state->src_w = 960 << 16;
+	fixture.old_primary_state->src_h = 540 << 16;
+	fixture.primary_state->src_w = 960 << 16;
+	fixture.primary_state->src_h = 540 << 16;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_full_coverage - Test full RGB coverage uses native cursor
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_full_coverage(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_OVERLAY_MODE;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_NATIVE_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_hole - Test incomplete plane coverage uses overlay cursor
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_hole(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.old_primary_state->crtc_w = 1280;
+	fixture.primary_state->crtc_w = 1280;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_without_cursor - Test unrelated update avoids cursor state
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_without_cursor(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.state->planes[0].ptr = NULL;
+	fixture.dm_crtc_state->base.plane_mask = drm_plane_mask(fixture.primary);
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_NATIVE_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_scale_changed - Test scale change triggers reevaluation
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_scale_changed(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.primary_state->src_w = 960 << 16;
+	fixture.primary_state->src_h = 540 << 16;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_position_changed - Test destination move triggers reevaluation
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_position_changed(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	fixture.primary_state->crtc_x = 1;
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
+
+/**
+ * dm_test_crtc_get_cursor_mode_color_pipeline - Test active color pipeline requires overlay
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_cursor_mode_color_pipeline(struct kunit *test)
+{
+	struct dm_cursor_mode_fixture fixture = dm_test_alloc_cursor_mode_fixture(test);
+	enum amdgpu_dm_cursor_mode cursor_mode = DM_CURSOR_NATIVE_MODE;
+
+	dm_test_add_cursor_mode_colorop(test, &fixture, true, false);
+
+	KUNIT_EXPECT_EQ(test, dm_test_get_cursor_mode(&fixture, &cursor_mode), 0);
+	KUNIT_EXPECT_EQ(test, cursor_mode, DM_CURSOR_OVERLAY_MODE);
+}
 
 /* Tests for amdgpu_dm_should_update_native_cursor() */
 
@@ -240,6 +945,37 @@ static void dm_test_get_plane_scale_zero_src_width(struct kunit *test)
 }
 
 static struct kunit_case amdgpu_dm_cursor_tests[] = {
+	/* dm_check_cursor_fb */
+	KUNIT_CASE(dm_test_check_cursor_fb_valid_linear),
+	KUNIT_CASE(dm_test_check_cursor_fb_rejects_size),
+	KUNIT_CASE(dm_test_check_cursor_fb_rejects_cropping),
+	KUNIT_CASE(dm_test_check_cursor_fb_rejects_pitch),
+	KUNIT_CASE(dm_test_check_cursor_fb_rejects_unsupported_pitch),
+	KUNIT_CASE(dm_test_check_cursor_fb_rejects_tiling),
+	KUNIT_CASE(dm_test_check_cursor_fb_gfx12_tiling),
+	KUNIT_CASE(dm_test_check_cursor_fb_pre_ai_tiling),
+	KUNIT_CASE(dm_test_check_cursor_fb_modifier_skips_tiling),
+	/* amdgpu_dm_check_native_cursor_state */
+	KUNIT_CASE(dm_test_check_native_cursor_state_disabled),
+	KUNIT_CASE(dm_test_check_native_cursor_state_rejects_offset),
+	KUNIT_CASE(dm_test_check_native_cursor_state_checks_fb),
+	/* dm_plane_color_pipeline_active */
+	KUNIT_CASE(dm_test_plane_color_pipeline_active),
+	KUNIT_CASE(dm_test_plane_color_pipeline_ignores_other_plane),
+	/* amdgpu_dm_crtc_get_cursor_mode */
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_disabled_crtc),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_new_hardware),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_new_hardware_hole),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_no_change),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_disabled_cursor),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_yuv_plane),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_scale_mismatch),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_full_coverage),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_hole),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_without_cursor),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_scale_changed),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_position_changed),
+	KUNIT_CASE(dm_test_crtc_get_cursor_mode_color_pipeline),
 	/* amdgpu_dm_should_update_native_cursor */
 	KUNIT_CASE(dm_test_should_update_native_cursor_without_crtc),
 	KUNIT_CASE(dm_test_should_update_native_cursor_disable_native),
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c
index ead43eeb38c773..1e499acbcb9086 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_dmub_test.c
@@ -14,11 +14,18 @@
 #include "dc/inc/hw/abm.h"
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
+#include "dal_asic_id.h"
 #include "dm_services.h"
 #include "dmub/dmub_srv.h"
 #include "amdgpu_dm_dmub.h"
 
-#define DM_TEST_FW_SIZE	512
+/*
+ * The PSP footer probing in dmub_srv_get_fw_meta_info_from_raw_fw() walks back
+ * from the end of the instruction constant region twice, so the fake firmware
+ * needs an instruction constant region larger than two footers.
+ */
+#define DM_TEST_FW_INST_CONST_BYTES	(PSP_HEADER_BYTES_256 + 1024)
+#define DM_TEST_FW_SIZE			(DM_TEST_FW_INST_CONST_BYTES + 1024)
 
 /* Tests for dm_register_dmub_notify_callback() */
 
@@ -111,7 +118,7 @@ static const struct firmware *dm_test_alloc_dmub_fw(struct kunit *test)
 	hdr = (struct dmcub_firmware_header_v1_0 *)data;
 	hdr->header.ucode_array_offset_bytes = cpu_to_le32(0);
 	hdr->header.ucode_version = cpu_to_le32(DMUB_FW_VERSION(9, 9, 9));
-	hdr->inst_const_bytes = cpu_to_le32(PSP_HEADER_BYTES_256);
+	hdr->inst_const_bytes = cpu_to_le32(DM_TEST_FW_INST_CONST_BYTES);
 	hdr->bss_data_bytes = cpu_to_le32(0);
 
 	fw->size = DM_TEST_FW_SIZE;
@@ -1347,6 +1354,543 @@ static void dm_test_abort_fused_io_no_dmub_srv(struct kunit *test)
 	abort_fused_io(ctx, req);
 }
 
+/*
+ * Fake CGS device: the DMUB register callbacks reach the hardware through
+ * cgs_read_register()/cgs_write_register(), which dispatch via the ops table
+ * at the start of struct cgs_device.
+ */
+
+#define DM_TEST_GPINT_REG		(0x34c0 + 0x01f8)
+#define DM_TEST_GPINT_STATUS_MASK	0xF0000000U
+
+struct dm_test_cgs_ctx {
+	struct cgs_device dev;
+	u32 last_write;
+	unsigned int write_calls;
+	unsigned int read_calls;
+	unsigned int last_write_offset;
+	unsigned int last_read_offset;
+	/* 1-based index of the write that must never be acked; 0 acks all */
+	unsigned int fail_on_write;
+};
+
+static struct dm_test_cgs_ctx dm_test_cgs;
+
+static uint32_t dm_test_cgs_read_register(struct cgs_device *cgs_device,
+					  unsigned int offset)
+{
+	dm_test_cgs.read_calls++;
+	dm_test_cgs.last_read_offset = offset;
+
+	if (dm_test_cgs.fail_on_write == dm_test_cgs.write_calls)
+		return ~dm_test_cgs.last_write;
+
+	/* The firmware acks a GPINT by clearing the status nibble. */
+	return dm_test_cgs.last_write & ~DM_TEST_GPINT_STATUS_MASK;
+}
+
+static void dm_test_cgs_write_register(struct cgs_device *cgs_device,
+				       unsigned int offset, uint32_t value)
+{
+	dm_test_cgs.write_calls++;
+	dm_test_cgs.last_write_offset = offset;
+	dm_test_cgs.last_write = value;
+}
+
+static const struct cgs_ops dm_test_cgs_ops = {
+	.read_register = dm_test_cgs_read_register,
+	.write_register = dm_test_cgs_write_register,
+};
+
+/*
+ * Fake buffer object allocator: amdgpu_bo_create_kernel() and
+ * amdgpu_bo_free_kernel() need a live TTM device, so both the DMUB framebuffer
+ * allocation and dm_allocate_gpu_mem() are routed through this fake.
+ */
+
+#define DM_TEST_FAKE_GPU_ADDR	0x1234ABCD0000ULL
+
+struct dm_test_bo_ctx {
+	void *cpu_ptr;
+	int create_ret;
+	unsigned int create_calls;
+	unsigned int free_calls;
+	unsigned long create_size;
+	u32 create_domain;
+};
+
+static struct dm_test_bo_ctx dm_test_bo;
+
+static int dm_test_bo_create_kernel(struct amdgpu_device *adev, unsigned long size,
+				    int align, u32 domain, struct amdgpu_bo **bo_ptr,
+				    u64 *gpu_addr, void **cpu_addr)
+{
+	dm_test_bo.create_calls++;
+	dm_test_bo.create_size = size;
+	dm_test_bo.create_domain = domain;
+
+	if (dm_test_bo.create_ret)
+		return dm_test_bo.create_ret;
+
+	*gpu_addr = DM_TEST_FAKE_GPU_ADDR;
+	*cpu_addr = dm_test_bo.cpu_ptr;
+
+	return 0;
+}
+
+static void dm_test_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, void **cpu_addr)
+{
+	dm_test_bo.free_calls++;
+
+	*bo = NULL;
+	*gpu_addr = 0;
+	*cpu_addr = NULL;
+}
+
+static const struct amdgpu_dm_services_kunit_ops dm_test_services_ops = {
+	.bo_create_kernel = dm_test_bo_create_kernel,
+	.bo_free_kernel = dm_test_bo_free_kernel,
+};
+
+/* Fake firmware loader, records the firmware name the ASIC switch selected. */
+
+static char dm_test_ucode_name[64];
+static unsigned int dm_test_ucode_calls;
+static int dm_test_ucode_ret;
+
+static __printf(4, 5) int dm_test_ucode_request(struct amdgpu_device *adev,
+						const struct firmware **fw,
+						enum amdgpu_ucode_required required,
+						const char *fmt, ...)
+{
+	va_list args;
+
+	dm_test_ucode_calls++;
+
+	va_start(args, fmt);
+	vsnprintf(dm_test_ucode_name, sizeof(dm_test_ucode_name), fmt, args);
+	va_end(args);
+
+	return dm_test_ucode_ret;
+}
+
+static const struct amdgpu_dm_dmub_kunit_ops dm_test_dmub_ops = {
+	.bo_create_kernel = dm_test_bo_create_kernel,
+	.ucode_request = dm_test_ucode_request,
+};
+
+static int dm_test_dmub_hw_access_init(struct kunit *test)
+{
+	void *cpu_ptr;
+
+	cpu_ptr = kunit_kzalloc(test, PAGE_SIZE, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cpu_ptr);
+
+	dm_test_bo = (struct dm_test_bo_ctx) { .cpu_ptr = cpu_ptr };
+	dm_test_cgs = (struct dm_test_cgs_ctx) { .dev.ops = &dm_test_cgs_ops };
+	dm_test_ucode_name[0] = '\0';
+	dm_test_ucode_calls = 0;
+	dm_test_ucode_ret = 0;
+
+	amdgpu_dm_services_kunit_set_ops(&dm_test_services_ops);
+	amdgpu_dm_dmub_kunit_set_ops(&dm_test_dmub_ops);
+
+	return 0;
+}
+
+static void dm_test_dmub_hw_access_exit(struct kunit *test)
+{
+	amdgpu_dm_services_kunit_set_ops(NULL);
+	amdgpu_dm_dmub_kunit_set_ops(NULL);
+}
+
+static struct amdgpu_device *dm_test_alloc_adev_with_cgs(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->dm.cgs_device = &dm_test_cgs.dev;
+	INIT_LIST_HEAD(&adev->dm.da_list);
+
+	return adev;
+}
+
+/* Tests for amdgpu_dm_dmub_reg_read() and amdgpu_dm_dmub_reg_write() */
+
+/**
+ * dm_test_dmub_reg_write_then_read - Test the DMUB register access callbacks
+ * @test: The KUnit test context
+ *
+ * The callbacks the DMUB service is created with must forward to the DC
+ * context of the device passed as their opaque user context.
+ */
+static void dm_test_dmub_reg_write_then_read(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_dc(test);
+	struct dc_perf_trace *perf_trace;
+
+	perf_trace = kunit_kzalloc(test, sizeof(*perf_trace), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, perf_trace);
+
+	adev->dm.dc->ctx->cgs_device = &dm_test_cgs.dev;
+	adev->dm.dc->ctx->perf_trace = perf_trace;
+
+	amdgpu_dm_dmub_reg_write(adev, 0x1234, 0x0BADF00D);
+
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.write_calls, 1U);
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.last_write_offset, 0x1234U);
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.last_write, 0x0BADF00DU);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dmub_reg_read(adev, 0x1234), 0x0BADF00DU);
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.read_calls, 1U);
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.last_read_offset, 0x1234U);
+}
+
+/* Tests for dm_dmub_get_vbios_bounding_box() */
+
+/**
+ * dm_test_dmub_get_vbios_bounding_box_success - Test the full bounding box handshake
+ * @test: The KUnit test context
+ *
+ * Every ASIC with a bounding box allocates a non-zero GART buffer, then sends
+ * and gets an ack for the four address words and the copy request.
+ */
+static void dm_test_dmub_get_vbios_bounding_box_success(struct kunit *test)
+{
+	static const u32 ip_versions[] = {
+		IP_VERSION(4, 0, 1),
+		IP_VERSION(4, 2, 0),
+		IP_VERSION(4, 2, 1),
+		IP_VERSION(6, 0, 0),
+	};
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_cgs(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(ip_versions); i++) {
+		void *bb;
+
+		adev->ip_versions[DCE_HWIP][0] = ip_versions[i];
+
+		bb = dm_dmub_get_vbios_bounding_box(adev);
+
+		KUNIT_EXPECT_PTR_EQ_MSG(test, bb, dm_test_bo.cpu_ptr,
+					"IP version 0x%08x", ip_versions[i]);
+		KUNIT_EXPECT_GT_MSG(test, dm_test_bo.create_size, 0UL,
+				    "IP version 0x%08x", ip_versions[i]);
+		KUNIT_EXPECT_EQ(test, dm_test_bo.create_domain,
+				(u32)AMDGPU_GEM_DOMAIN_GTT);
+		KUNIT_EXPECT_EQ(test, dm_test_cgs.last_write_offset,
+				(unsigned int)DM_TEST_GPINT_REG);
+		/* Four address words plus the copy request per ASIC. */
+		KUNIT_EXPECT_EQ(test, dm_test_cgs.write_calls, 5 * (i + 1));
+
+		dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, bb);
+	}
+}
+
+/**
+ * dm_test_dmub_get_vbios_bounding_box_alloc_fails - Test the allocation failure path
+ * @test: The KUnit test context
+ *
+ * When the GPU memory allocation fails, no GPINT command is sent.
+ */
+static void dm_test_dmub_get_vbios_bounding_box_alloc_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_cgs(test);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(6, 0, 0);
+	dm_test_bo.create_ret = -ENOMEM;
+
+	KUNIT_EXPECT_NULL(test, dm_dmub_get_vbios_bounding_box(adev));
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.write_calls, 0U);
+}
+
+/**
+ * dm_test_dmub_get_vbios_bounding_box_addr_timeout - Test an unacked address word
+ * @test: The KUnit test context
+ *
+ * When the firmware never acks the first address word, the handshake stops
+ * there and the buffer is released.
+ */
+static void dm_test_dmub_get_vbios_bounding_box_addr_timeout(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_cgs(test);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(6, 0, 0);
+	dm_test_cgs.fail_on_write = 1;
+
+	KUNIT_EXPECT_NULL(test, dm_dmub_get_vbios_bounding_box(adev));
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.write_calls, 1U);
+	KUNIT_EXPECT_EQ(test, dm_test_bo.free_calls, 1U);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+}
+
+/**
+ * dm_test_dmub_get_vbios_bounding_box_copy_timeout - Test an unacked copy request
+ * @test: The KUnit test context
+ *
+ * When all four address words are acked but the copy request is not, the
+ * buffer is released and no bounding box is returned.
+ */
+static void dm_test_dmub_get_vbios_bounding_box_copy_timeout(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_cgs(test);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(6, 0, 0);
+	dm_test_cgs.fail_on_write = 5;
+
+	KUNIT_EXPECT_NULL(test, dm_dmub_get_vbios_bounding_box(adev));
+	KUNIT_EXPECT_EQ(test, dm_test_cgs.write_calls, 5U);
+	KUNIT_EXPECT_EQ(test, dm_test_bo.free_calls, 1U);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+}
+
+/* Tests for dm_init_microcode() */
+
+struct dm_test_ucode_case {
+	u32 dce_version;
+	u32 gc_version;
+	u32 external_rev_id;
+	const char *fw_name;
+};
+
+static const struct dm_test_ucode_case dm_test_ucode_cases[] = {
+	{ IP_VERSION(2, 1, 0), 0, 0, FIRMWARE_RENOIR_DMUB },
+	{ IP_VERSION(2, 1, 0), 0, GREEN_SARDINE_A0, FIRMWARE_GREEN_SARDINE_DMUB },
+	{ IP_VERSION(3, 0, 0), IP_VERSION(10, 3, 0), 0, FIRMWARE_SIENNA_CICHLID_DMUB },
+	{ IP_VERSION(3, 0, 0), IP_VERSION(10, 3, 2), 0, FIRMWARE_NAVY_FLOUNDER_DMUB },
+	{ IP_VERSION(3, 0, 1), 0, 0, FIRMWARE_VANGOGH_DMUB },
+	{ IP_VERSION(3, 0, 2), 0, 0, FIRMWARE_DIMGREY_CAVEFISH_DMUB },
+	{ IP_VERSION(3, 0, 3), 0, 0, FIRMWARE_BEIGE_GOBY_DMUB },
+	{ IP_VERSION(3, 1, 2), 0, 0, FIRMWARE_YELLOW_CARP_DMUB },
+	{ IP_VERSION(3, 1, 3), 0, 0, FIRMWARE_YELLOW_CARP_DMUB },
+	{ IP_VERSION(3, 1, 4), 0, 0, FIRMWARE_DCN_314_DMUB },
+	{ IP_VERSION(3, 1, 5), 0, 0, FIRMWARE_DCN_315_DMUB },
+	{ IP_VERSION(3, 1, 6), 0, 0, FIRMWARE_DCN316_DMUB },
+	{ IP_VERSION(3, 2, 0), 0, 0, FIRMWARE_DCN_V3_2_0_DMCUB },
+	{ IP_VERSION(3, 2, 1), 0, 0, FIRMWARE_DCN_V3_2_1_DMCUB },
+	{ IP_VERSION(3, 5, 0), 0, 0, FIRMWARE_DCN_35_DMUB },
+	{ IP_VERSION(3, 5, 1), 0, 0, FIRMWARE_DCN_351_DMUB },
+	{ IP_VERSION(3, 6, 0), 0, 0, FIRMWARE_DCN_36_DMUB },
+	{ IP_VERSION(4, 0, 1), 0, 0, FIRMWARE_DCN_401_DMUB },
+	{ IP_VERSION(4, 2, 0), 0, 0, FIRMWARE_DCN_42_DMUB },
+	{ IP_VERSION(4, 2, 1), 0, 0, FIRMWARE_DCN_42B_DMUB },
+	{ IP_VERSION(6, 0, 0), 0, 0, FIRMWARE_DCN_60_DMUB },
+};
+
+/**
+ * dm_test_init_microcode_fw_names - Test the ASIC to firmware name mapping
+ * @test: The KUnit test context
+ */
+static void dm_test_init_microcode_fw_names(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	unsigned int i;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	for (i = 0; i < ARRAY_SIZE(dm_test_ucode_cases); i++) {
+		const struct dm_test_ucode_case *c = &dm_test_ucode_cases[i];
+
+		adev->ip_versions[DCE_HWIP][0] = c->dce_version;
+		adev->ip_versions[GC_HWIP][0] = c->gc_version;
+		adev->external_rev_id = c->external_rev_id;
+		dm_test_ucode_name[0] = '\0';
+
+		KUNIT_EXPECT_EQ_MSG(test, dm_init_microcode(adev), 0,
+				    "IP version 0x%08x", c->dce_version);
+		KUNIT_EXPECT_STREQ_MSG(test, dm_test_ucode_name, c->fw_name,
+				       "IP version 0x%08x", c->dce_version);
+	}
+
+	KUNIT_EXPECT_EQ(test, dm_test_ucode_calls,
+			(unsigned int)ARRAY_SIZE(dm_test_ucode_cases));
+}
+
+/**
+ * dm_test_init_microcode_request_fails - Test a firmware request failure
+ * @test: The KUnit test context
+ */
+static void dm_test_init_microcode_request_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 5, 0);
+	dm_test_ucode_ret = -ENOENT;
+
+	KUNIT_EXPECT_EQ(test, dm_init_microcode(adev), -ENOENT);
+}
+
+/* Tests for dm_dmub_sw_init() */
+
+static struct amdgpu_device *dm_test_alloc_adev_for_sw_init(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_with_cgs(test);
+
+	adev->dm.dmub_fw = dm_test_alloc_dmub_fw(test);
+	adev->bios = kunit_kzalloc(test, 4, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev->bios);
+	adev->bios_size = 4;
+
+	return adev;
+}
+
+static void dm_test_free_sw_init(struct amdgpu_device *adev)
+{
+	kfree(adev->dm.dmub_srv);
+	kfree(adev->dm.dmub_fb_info);
+	adev->dm.dmub_srv = NULL;
+	adev->dm.dmub_fb_info = NULL;
+}
+
+/**
+ * dm_test_dmub_sw_init_asic_mapping - Test the ASIC to DMUB service mapping
+ * @test: The KUnit test context
+ *
+ * Every supported ASIC must get past DMUB service creation and region
+ * calculation. The framebuffer allocation is forced to fail so the tests stop
+ * before the memory layout is rebased onto a fake buffer.
+ */
+static void dm_test_dmub_sw_init_asic_mapping(struct kunit *test)
+{
+	static const u32 ip_versions[] = {
+		IP_VERSION(2, 1, 0), IP_VERSION(3, 0, 0), IP_VERSION(3, 0, 1),
+		IP_VERSION(3, 0, 2), IP_VERSION(3, 0, 3), IP_VERSION(3, 1, 2),
+		IP_VERSION(3, 1, 3), IP_VERSION(3, 1, 4), IP_VERSION(3, 1, 5),
+		IP_VERSION(3, 1, 6), IP_VERSION(3, 2, 0), IP_VERSION(3, 2, 1),
+		IP_VERSION(3, 5, 0), IP_VERSION(3, 5, 1), IP_VERSION(3, 6, 0),
+		IP_VERSION(4, 0, 1), IP_VERSION(4, 2, 0), IP_VERSION(4, 2, 1),
+		IP_VERSION(6, 0, 0),
+	};
+	struct amdgpu_device *adev = dm_test_alloc_adev_for_sw_init(test);
+	unsigned int i;
+
+	dm_test_bo.create_ret = -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(ip_versions); i++) {
+		adev->ip_versions[DCE_HWIP][0] = ip_versions[i];
+
+		KUNIT_EXPECT_EQ_MSG(test, dm_dmub_sw_init(adev), -ENOMEM,
+				    "IP version 0x%08x", ip_versions[i]);
+
+		dm_test_free_sw_init(adev);
+	}
+
+	KUNIT_EXPECT_EQ(test, dm_test_bo.create_calls, (unsigned int)ARRAY_SIZE(ip_versions));
+}
+
+/**
+ * dm_test_dmub_sw_init_gtt_only_asic - Test the GTT-only memory domain
+ * @test: The KUnit test context
+ *
+ * DCN32 and DCN321 keep the DMUB framebuffer in GTT; every other ASIC also
+ * allows VRAM.
+ */
+static void dm_test_dmub_sw_init_gtt_only_asic(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_for_sw_init(test);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 2, 0);
+	dm_test_bo.create_ret = -ENOMEM;
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_sw_init(adev), -ENOMEM);
+	KUNIT_EXPECT_EQ(test, dm_test_bo.create_domain, (u32)AMDGPU_GEM_DOMAIN_GTT);
+
+	dm_test_free_sw_init(adev);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 5, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_sw_init(adev), -ENOMEM);
+	KUNIT_EXPECT_EQ(test, dm_test_bo.create_domain,
+			(u32)(AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM));
+
+	dm_test_free_sw_init(adev);
+}
+
+/**
+ * dm_test_dmub_sw_init_success - Test a complete software init
+ * @test: The KUnit test context
+ *
+ * With a fake framebuffer allocator and a fake CGS device, software init
+ * should publish the framebuffer info, the bounding box and the instruction
+ * constant size.
+ */
+static void dm_test_dmub_sw_init_success(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_for_sw_init(test);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(6, 0, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_sw_init(adev), 0);
+	KUNIT_EXPECT_NOT_NULL(test, adev->dm.dmub_srv);
+	KUNIT_EXPECT_NOT_NULL(test, adev->dm.dmub_fb_info);
+	KUNIT_EXPECT_NOT_NULL(test, adev->dm.bb_from_dmub);
+	KUNIT_EXPECT_EQ(test, adev->dm.dmcub_fw_version, DMUB_FW_VERSION(9, 9, 9));
+	/* Meta info lookup fails on the fake firmware and trims a PSP footer. */
+	KUNIT_EXPECT_EQ(test, adev->dm.fw_inst_size,
+			(u32)(DM_TEST_FW_INST_CONST_BYTES - PSP_HEADER_BYTES_256 -
+			      PSP_FOOTER_BYTES_256));
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, adev->dm.bb_from_dmub);
+	dm_test_free_sw_init(adev);
+}
+
+/**
+ * dm_test_dmub_sw_init_bss_data - Test software init with a BSS data region
+ * @test: The KUnit test context
+ *
+ * A non-zero BSS data size makes software init point the firmware meta info
+ * lookup at the legacy metadata region instead of the instruction constants.
+ */
+static void dm_test_dmub_sw_init_bss_data(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_for_sw_init(test);
+	struct dmcub_firmware_header_v1_0 *hdr;
+
+	hdr = (struct dmcub_firmware_header_v1_0 *)adev->dm.dmub_fw->data;
+	hdr->bss_data_bytes = cpu_to_le32(512);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(6, 0, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_sw_init(adev), 0);
+	KUNIT_EXPECT_NOT_NULL(test, adev->dm.dmub_fb_info);
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, adev->dm.bb_from_dmub);
+	dm_test_free_sw_init(adev);
+}
+
+/**
+ * dm_test_dmub_sw_init_psp_load - Test the PSP firmware load registration
+ * @test: The KUnit test context
+ *
+ * When the firmware is loaded by the PSP, software init must register the
+ * DMCUB microcode with the AMDGPU firmware loader.
+ */
+static void dm_test_dmub_sw_init_psp_load(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev_for_sw_init(test);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(6, 0, 0);
+	adev->firmware.load_type = AMDGPU_FW_LOAD_PSP;
+
+	KUNIT_EXPECT_EQ(test, dm_dmub_sw_init(adev), 0);
+	KUNIT_EXPECT_EQ(test, adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].ucode_id,
+			AMDGPU_UCODE_ID_DMCUB);
+	KUNIT_EXPECT_PTR_EQ(test, adev->firmware.ucode[AMDGPU_UCODE_ID_DMCUB].fw,
+			    adev->dm.dmub_fw);
+	KUNIT_EXPECT_EQ(test, adev->firmware.fw_size,
+			(u32)ALIGN(DM_TEST_FW_INST_CONST_BYTES, PAGE_SIZE));
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, adev->dm.bb_from_dmub);
+	dm_test_free_sw_init(adev);
+}
+
 static struct kunit_case amdgpu_dm_dmub_tests[] = {
 	/* dm_register_dmub_notify_callback() */
 	KUNIT_CASE(dm_test_register_dmub_notify_callback_null_callback),
@@ -1412,7 +1956,35 @@ static struct kunit_suite amdgpu_dm_dmub_test_suite = {
 	.test_cases = amdgpu_dm_dmub_tests,
 };
 
-kunit_test_suite(amdgpu_dm_dmub_test_suite);
+static struct kunit_case amdgpu_dm_dmub_hw_access_tests[] = {
+	/* amdgpu_dm_dmub_reg_read() and amdgpu_dm_dmub_reg_write() */
+	KUNIT_CASE(dm_test_dmub_reg_write_then_read),
+	/* dm_dmub_get_vbios_bounding_box() */
+	KUNIT_CASE(dm_test_dmub_get_vbios_bounding_box_success),
+	KUNIT_CASE(dm_test_dmub_get_vbios_bounding_box_alloc_fails),
+	KUNIT_CASE(dm_test_dmub_get_vbios_bounding_box_addr_timeout),
+	KUNIT_CASE(dm_test_dmub_get_vbios_bounding_box_copy_timeout),
+	/* dm_init_microcode() */
+	KUNIT_CASE(dm_test_init_microcode_fw_names),
+	KUNIT_CASE(dm_test_init_microcode_request_fails),
+	/* dm_dmub_sw_init() */
+	KUNIT_CASE(dm_test_dmub_sw_init_asic_mapping),
+	KUNIT_CASE(dm_test_dmub_sw_init_gtt_only_asic),
+	KUNIT_CASE(dm_test_dmub_sw_init_success),
+	KUNIT_CASE(dm_test_dmub_sw_init_bss_data),
+	KUNIT_CASE(dm_test_dmub_sw_init_psp_load),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_dmub_hw_access_test_suite = {
+	.name = "amdgpu_dm_dmub_hw_access",
+	.init = dm_test_dmub_hw_access_init,
+	.exit = dm_test_dmub_hw_access_exit,
+	.test_cases = amdgpu_dm_dmub_hw_access_tests,
+};
+
+kunit_test_suites(&amdgpu_dm_dmub_test_suite,
+		  &amdgpu_dm_dmub_hw_access_test_suite);
 
 MODULE_AUTHOR("AMD");
 MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_dmub");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c
index 0ae636dcbbe4e4..5ad50bc3161375 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_freesync_test.c
@@ -10,6 +10,7 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_vblank.h>
 
 #include "dc.h"
 #include "inc/core_types.h"
@@ -20,6 +21,23 @@
 #include "amdgpu_dm_freesync.h"
 #include "amdgpu_dm_kunit_test_helpers.h"
 
+static u32 dm_test_freesync_vblank_counter(struct drm_crtc *crtc)
+{
+	return 0;
+}
+
+static const struct drm_crtc_funcs dm_test_freesync_crtc_funcs = {
+	.get_vblank_counter = dm_test_freesync_vblank_counter,
+};
+
+static void dm_test_freesync_crtc_cleanup(void *data)
+{
+	struct amdgpu_crtc *acrtc = data;
+
+	drm_crtc_vblank_off(&acrtc->base);
+	list_del_init(&acrtc->base.head);
+}
+
 /* Tests for amdgpu_dm_is_timing_unchanged_for_freesync() */
 
 /**
@@ -416,6 +434,397 @@ static void dm_test_reset_freesync_config(struct kunit *test)
 	KUNIT_EXPECT_FALSE(test, crtc_state->vrr_infopacket.valid);
 }
 
+/* Tests for amdgpu_dm_update_freesync_state_on_stream() */
+
+/**
+ * dm_test_update_freesync_state_no_stream - Test a NULL stream is ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_no_stream(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+
+	amdgpu_dm_update_freesync_state_on_stream(&adev->dm, &crtc_state, NULL, NULL, 0);
+
+	KUNIT_SUCCEED(test);
+}
+
+/**
+ * dm_test_update_freesync_state_invalid_timing - Test zero stream timing is ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_invalid_timing(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+
+	amdgpu_dm_update_freesync_state_on_stream(&adev->dm, &crtc_state, stream, NULL, 0);
+
+	KUNIT_SUCCEED(test);
+}
+
+/**
+ * dm_test_update_freesync_state_default_packet - Test default packet state update
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_default_packet(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+	stream->timing.h_total = 2200;
+	stream->timing.v_total = 1125;
+	adev->dm.adev = adev;
+
+	amdgpu_dm_update_freesync_state_on_stream(&adev->dm, &crtc_state, stream, NULL, 0);
+
+	KUNIT_EXPECT_FALSE(test, crtc_state.freesync_vrr_info_changed);
+	KUNIT_EXPECT_FALSE(test, crtc_state.vrr_infopacket.valid);
+	KUNIT_EXPECT_FALSE(test, stream->allow_freesync);
+}
+
+/**
+ * dm_test_update_freesync_state_packet_changed - Test packet changes are recorded
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_packet_changed(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dc_plane_state *surface;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	surface = kunit_kzalloc(test, sizeof(*surface), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, surface);
+	crtc_state.base.crtc = &acrtc->base;
+	crtc_state.vrr_infopacket.valid = true;
+	stream->timing.h_total = 2200;
+	stream->timing.v_total = 1125;
+	adev->family = AMDGPU_FAMILY_AI;
+	adev->dm.adev = adev;
+
+	amdgpu_dm_update_freesync_state_on_stream(&adev->dm, &crtc_state, stream, surface, 0);
+
+	KUNIT_EXPECT_TRUE(test, crtc_state.freesync_vrr_info_changed);
+	KUNIT_EXPECT_FALSE(test, crtc_state.vrr_infopacket.valid);
+}
+
+/**
+ * dm_test_update_freesync_state_pre_ai_active - Test pre-AI active VRR adjustment
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_pre_ai_active(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, NULL);
+	struct dc_plane_state *surface;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	surface = kunit_kzalloc(test, sizeof(*surface), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, surface);
+	crtc_state.base.crtc = &acrtc->base;
+	crtc_state.stream = stream;
+	crtc_state.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+	stream->timing.h_total = 2200;
+	stream->timing.v_total = 1125;
+	adev->dm.adev = adev;
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+	adev->dm.dc->current_state = dm_kunit_alloc_dc_state(test);
+
+	amdgpu_dm_update_freesync_state_on_stream(&adev->dm, &crtc_state, stream, surface, 0);
+
+	KUNIT_EXPECT_FALSE(test, stream->adjust.timing_adjust_pending);
+}
+
+static void dm_test_update_freesync_state_pcon_version(struct kunit *test, u8 version)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, link);
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	crtc_state.base.crtc = &acrtc->base;
+	stream->timing.h_total = 2200;
+	stream->timing.v_total = 1125;
+	stream->dm_stream_context = aconnector;
+	aconnector->as_type = FREESYNC_TYPE_PCON_IN_WHITELIST;
+	aconnector->vsdb_info.amd_vsdb_version = version;
+	adev->dm.adev = adev;
+
+	amdgpu_dm_update_freesync_state_on_stream(&adev->dm, &crtc_state, stream, NULL, 0);
+
+	KUNIT_EXPECT_TRUE(test, stream->adaptive_sync_infopacket.valid);
+	KUNIT_EXPECT_EQ(test, stream->adaptive_sync_infopacket.hb2, (u8)AS_SDP_VER_1);
+}
+
+/**
+ * dm_test_update_freesync_state_pcon_v1 - Test PCON AMD VSDB version 1
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_pcon_v1(struct kunit *test)
+{
+	dm_test_update_freesync_state_pcon_version(test, 1);
+}
+
+/**
+ * dm_test_update_freesync_state_pcon_v2 - Test PCON AMD VSDB version 2
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_pcon_v2(struct kunit *test)
+{
+	dm_test_update_freesync_state_pcon_version(test, 2);
+}
+
+/**
+ * dm_test_update_freesync_state_pcon_v3 - Test PCON AMD VSDB version 3
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_pcon_v3(struct kunit *test)
+{
+	dm_test_update_freesync_state_pcon_version(test, 3);
+}
+
+/**
+ * dm_test_update_freesync_state_pcon_default - Test unknown AMD VSDB version
+ * @test: The KUnit test context
+ */
+static void dm_test_update_freesync_state_pcon_default(struct kunit *test)
+{
+	dm_test_update_freesync_state_pcon_version(test, 0);
+}
+
+/* Tests for amdgpu_dm_update_stream_irq_parameters() */
+
+/**
+ * dm_test_update_stream_irq_parameters_no_stream - Test a NULL stream is ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_update_stream_irq_parameters_no_stream(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+
+	amdgpu_dm_update_stream_irq_parameters(&adev->dm, &crtc_state);
+
+	KUNIT_SUCCEED(test);
+}
+
+/**
+ * dm_test_update_stream_irq_parameters_invalid_timing - Test zero stream timing is ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_update_stream_irq_parameters_invalid_timing(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+	crtc_state.stream = dm_kunit_alloc_stream(test, NULL);
+
+	amdgpu_dm_update_stream_irq_parameters(&adev->dm, &crtc_state);
+
+	KUNIT_SUCCEED(test);
+}
+
+/**
+ * dm_test_update_stream_irq_parameters_unsupported - Test unsupported config update
+ * @test: The KUnit test context
+ */
+static void dm_test_update_stream_irq_parameters_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+	crtc_state.stream = dm_kunit_alloc_stream(test, NULL);
+	crtc_state.stream->timing.h_total = 2200;
+	crtc_state.stream->timing.v_total = 1125;
+	adev->dm.adev = adev;
+
+	amdgpu_dm_update_stream_irq_parameters(&adev->dm, &crtc_state);
+
+	KUNIT_EXPECT_EQ(test, (int)crtc_state.freesync_config.state,
+			(int)VRR_STATE_UNSUPPORTED);
+	KUNIT_EXPECT_EQ(test, (int)acrtc->dm_irq_params.freesync_config.state,
+			(int)VRR_STATE_UNSUPPORTED);
+}
+
+static void dm_test_update_stream_irq_parameters_supported(struct kunit *test, bool vrr_enabled)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+	crtc_state.base.vrr_enabled = vrr_enabled;
+	crtc_state.stream = dm_kunit_alloc_stream(test, NULL);
+	crtc_state.stream->timing.h_total = 2200;
+	crtc_state.stream->timing.v_total = 1125;
+	crtc_state.vrr_supported = true;
+	crtc_state.freesync_config.min_refresh_in_uhz = 48000000;
+	crtc_state.freesync_config.max_refresh_in_uhz = 120000000;
+	adev->dm.adev = adev;
+
+	amdgpu_dm_update_stream_irq_parameters(&adev->dm, &crtc_state);
+
+	KUNIT_EXPECT_EQ(test, (int)crtc_state.freesync_config.state,
+			vrr_enabled ? (int)VRR_STATE_ACTIVE_VARIABLE : (int)VRR_STATE_INACTIVE);
+}
+
+/**
+ * dm_test_update_stream_irq_parameters_variable - Test variable VRR configuration
+ * @test: The KUnit test context
+ */
+static void dm_test_update_stream_irq_parameters_variable(struct kunit *test)
+{
+	dm_test_update_stream_irq_parameters_supported(test, true);
+}
+
+/**
+ * dm_test_update_stream_irq_parameters_inactive - Test inactive VRR configuration
+ * @test: The KUnit test context
+ */
+static void dm_test_update_stream_irq_parameters_inactive(struct kunit *test)
+{
+	dm_test_update_stream_irq_parameters_supported(test, false);
+}
+
+/**
+ * dm_test_update_stream_irq_parameters_fixed - Test fixed VRR configuration
+ * @test: The KUnit test context
+ */
+static void dm_test_update_stream_irq_parameters_fixed(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state crtc_state = { 0 };
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	crtc_state.base.crtc = &acrtc->base;
+	crtc_state.base.mode_changed = true;
+	crtc_state.stream = dm_kunit_alloc_stream(test, NULL);
+	crtc_state.stream->timing.h_total = 2200;
+	crtc_state.stream->timing.v_total = 1125;
+	crtc_state.vrr_supported = true;
+	crtc_state.freesync_config.state = VRR_STATE_ACTIVE_FIXED;
+	crtc_state.freesync_config.min_refresh_in_uhz = 48000000;
+	crtc_state.freesync_config.max_refresh_in_uhz = 120000000;
+	crtc_state.freesync_config.fixed_refresh_in_uhz = 60000000;
+	adev->dm.adev = adev;
+
+	amdgpu_dm_update_stream_irq_parameters(&adev->dm, &crtc_state);
+
+	KUNIT_EXPECT_EQ(test, (int)acrtc->dm_irq_params.vrr_params.state,
+			(int)VRR_STATE_ACTIVE_FIXED);
+	KUNIT_EXPECT_EQ(test, acrtc->dm_irq_params.vrr_params.fixed_refresh_in_uhz,
+			60000000U);
+}
+
+/* Tests for amdgpu_dm_handle_vrr_transition() */
+
+/**
+ * dm_test_handle_vrr_transition_inactive - Test steady inactive state is a no-op
+ * @test: The KUnit test context
+ */
+static void dm_test_handle_vrr_transition_inactive(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dm_crtc_state old_state = { 0 };
+	struct dm_crtc_state new_state = { 0 };
+
+	adev->dm.adev = adev;
+	amdgpu_dm_handle_vrr_transition(&adev->dm, &old_state, &new_state);
+
+	KUNIT_SUCCEED(test);
+}
+
+/**
+ * dm_test_handle_vrr_transition_round_trip - Test VRR on and off transitions
+ * @test: The KUnit test context
+ */
+static void dm_test_handle_vrr_transition_round_trip(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, link);
+	struct drm_vblank_crtc *vblank;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state old_state = { 0 };
+	struct dm_crtc_state new_state = { 0 };
+
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	acrtc->base.dev = &adev->ddev;
+	acrtc->base.funcs = &dm_test_freesync_crtc_funcs;
+	INIT_LIST_HEAD(&acrtc->base.head);
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_freesync_crtc_cleanup, acrtc), 0);
+	acrtc->otg_inst = -1;
+	vblank = drm_crtc_vblank_crtc(&acrtc->base);
+	adev->ddev.vblank[0].enabled = true;
+	adev->dm.adev = adev;
+	adev->dm.dc = kunit_kzalloc(test, sizeof(*adev->dm.dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev->dm.dc);
+	mutex_init(&adev->dm.dc_lock);
+
+	old_state.base.crtc = &acrtc->base;
+	old_state.stream = stream;
+	new_state.base.crtc = &acrtc->base;
+	new_state.stream = stream;
+	new_state.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+
+	amdgpu_dm_handle_vrr_transition(&adev->dm, &old_state, &new_state);
+	KUNIT_EXPECT_EQ(test, atomic_read(&vblank->refcount), 1);
+
+	amdgpu_dm_handle_vrr_transition(&adev->dm, &new_state, &old_state);
+	KUNIT_EXPECT_EQ(test, atomic_read(&vblank->refcount), 0);
+}
+
 static struct kunit_case amdgpu_dm_freesync_tests[] = {
 	/* amdgpu_dm_is_timing_unchanged_for_freesync */
 	KUNIT_CASE(dm_test_timing_unchanged_null_args),
@@ -438,6 +847,26 @@ static struct kunit_case amdgpu_dm_freesync_tests[] = {
 	KUNIT_CASE(dm_test_freesync_config_active_fixed),
 	/* amdgpu_dm_reset_freesync_config_for_crtc */
 	KUNIT_CASE(dm_test_reset_freesync_config),
+	/* amdgpu_dm_update_freesync_state_on_stream */
+	KUNIT_CASE(dm_test_update_freesync_state_no_stream),
+	KUNIT_CASE(dm_test_update_freesync_state_invalid_timing),
+	KUNIT_CASE(dm_test_update_freesync_state_default_packet),
+	KUNIT_CASE(dm_test_update_freesync_state_packet_changed),
+	KUNIT_CASE(dm_test_update_freesync_state_pre_ai_active),
+	KUNIT_CASE(dm_test_update_freesync_state_pcon_v1),
+	KUNIT_CASE(dm_test_update_freesync_state_pcon_v2),
+	KUNIT_CASE(dm_test_update_freesync_state_pcon_v3),
+	KUNIT_CASE(dm_test_update_freesync_state_pcon_default),
+	/* amdgpu_dm_update_stream_irq_parameters */
+	KUNIT_CASE(dm_test_update_stream_irq_parameters_no_stream),
+	KUNIT_CASE(dm_test_update_stream_irq_parameters_invalid_timing),
+	KUNIT_CASE(dm_test_update_stream_irq_parameters_unsupported),
+	KUNIT_CASE(dm_test_update_stream_irq_parameters_variable),
+	KUNIT_CASE(dm_test_update_stream_irq_parameters_inactive),
+	KUNIT_CASE(dm_test_update_stream_irq_parameters_fixed),
+	/* amdgpu_dm_handle_vrr_transition */
+	KUNIT_CASE(dm_test_handle_vrr_transition_inactive),
+	KUNIT_CASE(dm_test_handle_vrr_transition_round_trip),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
index 639512bea27540..d37fe26eecea41 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_helpers_test.c
@@ -6,19 +6,25 @@
  */
 
 #include <kunit/test.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_kunit_helpers.h>
+#include <drm/drm_managed.h>
 #include <drm/display/drm_dp_mst_helper.h>
 
 #include "dc.h"
 #include "core_types.h"
+#include "clk_mgr.h"
+#include "link_service.h"
 #include "amdgpu.h"
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_mst_types.h"
 #include "dc_bios_types.h"
 #include "dm_helpers.h"
+#include "dpcd_defs.h"
 #include "ddc_service_types.h"
+#include "dmub/dmub_srv.h"
 #include "dmub_cmd.h"
 #include "amdgpu_dm_helpers.h"
 #include "amdgpu_dm_kunit_test_helpers.h"
@@ -170,6 +176,34 @@ static void dm_test_apply_edid_quirks_disable_colorimetry(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, edid_caps.panel_patch.disable_colorimetry);
 }
 
+/**
+ * dm_test_apply_edid_quirks_psr_phy_power_down - Test SDC PSR PHY power-down quirk
+ * @test: The KUnit test context
+ *
+ * The quirk is embedded-only, so an external signal leaves the debug option
+ * alone and an embedded one forces PHY power down/up level 2.
+ */
+static void dm_test_apply_edid_quirks_psr_phy_power_down(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+	struct edid *external = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('S', 'D', 'C', 0x4197));
+	struct edid *embedded = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('S', 'D', 'C', 0x4203));
+
+	link->ctx = dc->ctx;
+
+	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	apply_edid_quirks(link, external, &edid_caps);
+	KUNIT_EXPECT_FALSE(test, dc->debug.psr_phy_force_phy_power_down_up_level_2);
+
+	link->connector_signal = SIGNAL_TYPE_EDP;
+	apply_edid_quirks(link, embedded, &edid_caps);
+	KUNIT_EXPECT_TRUE(test, dc->debug.psr_phy_force_phy_power_down_up_level_2);
+}
+
 /**
  * dm_test_apply_edid_quirks_skip_phy_ssc - Test DEL 0x4147 PHY SSC quirk
  * @test: The KUnit test context
@@ -186,6 +220,22 @@ static void dm_test_apply_edid_quirks_skip_phy_ssc(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, link->wa_flags.skip_phy_ssc_reduction);
 }
 
+/**
+ * dm_test_apply_edid_quirks_force_freesync_min - Test ACR 0x08AF FreeSync min quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_force_freesync_min(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('A', 'C', 'R', 0x08AF));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_EQ(test, edid_caps.panel_patch.force_freesync_min_hz, 55U);
+}
+
 /**
  * dm_test_apply_edid_quirks_unknown_noop - Test unknown panel id is a no-op
  * @test: The KUnit test context
@@ -207,6 +257,38 @@ static void dm_test_apply_edid_quirks_unknown_noop(struct kunit *test)
 	KUNIT_EXPECT_FALSE(test, link->wa_flags.skip_phy_ssc_reduction);
 }
 
+/**
+ * dm_test_apply_edid_quirks_disable_second_tile - Test Apple Studio Display tile quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_disable_second_tile(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('A', 'P', 'P', 0xAE3A));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_TRUE(test, edid_caps.panel_patch.disable_second_tile);
+}
+
+/**
+ * dm_test_apply_edid_quirks_force_freesync_min_60 - Test Lenovo G34w-30 FreeSync min quirk
+ * @test: The KUnit test context
+ */
+static void dm_test_apply_edid_quirks_force_freesync_min_60(struct kunit *test)
+{
+	struct dc_edid_caps edid_caps = {0};
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct edid *edid = dm_test_edid_with_panel_id(test,
+			drm_edid_encode_panel_id('L', 'E', 'N', 0x66F1));
+
+	apply_edid_quirks(link, edid, &edid_caps);
+
+	KUNIT_EXPECT_EQ(test, edid_caps.panel_patch.force_freesync_min_hz, 60U);
+}
+
 /* Tests for dm_helpers_parse_edid_caps() */
 
 /*
@@ -409,6 +491,40 @@ static void dm_test_parse_edid_caps_hdmi_frl_dsc(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, edid_caps->frl_dsc_10bpc);
 }
 
+/**
+ * dm_test_parse_edid_caps_hdmi_comp_auto - Test the HDMI compliance-auto branch
+ * @test: The KUnit test context
+ *
+ * A connector flagged for automated HDMI compliance forces the FRL and FRL DSC
+ * debug options on and records the quirk in the panel patch.
+ */
+static void dm_test_parse_edid_caps_hdmi_comp_auto(struct kunit *test)
+{
+	struct dc_link *link = dm_test_quirk_link(test);
+	struct amdgpu_dm_connector *aconnector = link->priv;
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+	struct dc_edid_caps *edid_caps;
+	struct dc_edid *dc_edid;
+
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+	edid_caps = kunit_kzalloc(test, sizeof(*edid_caps), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, edid_caps);
+
+	link->dc = dc;
+	link->ctx = dc->ctx;
+
+	dm_test_fill_base_edid(dc_edid, true);
+
+	aconnector->base.display_info.is_hdmi = true;
+	aconnector->hdmi_comp_auto = true;
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_parse_edid_caps(link, dc_edid, edid_caps), EDID_OK);
+	KUNIT_EXPECT_TRUE(test, edid_caps->panel_patch.hdmi_comp_auto);
+	KUNIT_EXPECT_TRUE(test, dc->debug.force_frl_max);
+	KUNIT_EXPECT_TRUE(test, dc->debug.force_frl_dsc);
+}
+
 /*
  * Build a 2-block EDID: a valid base block plus a CTA-861 extension block
  * carrying one LPCM Short Audio Descriptor and, when @with_speaker is set, a
@@ -581,6 +697,25 @@ static void dm_test_read_acpi_edid_force_off(struct kunit *test)
 	KUNIT_EXPECT_NULL(test, dm_helpers_read_acpi_edid(aconnector));
 }
 
+/**
+ * dm_test_read_acpi_edid_panel_connector - Test the ACPI EDID read is attempted
+ * @test: The KUnit test context
+ *
+ * An eDP connector that is not forced off reaches drm_edid_read_custom(), whose
+ * probe callback finds no ACPI companion, so no EDID is returned.
+ */
+static void dm_test_read_acpi_edid_panel_connector(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_eDP;
+
+	KUNIT_EXPECT_NULL(test, dm_helpers_read_acpi_edid(aconnector));
+}
+
 struct dm_test_vbios_edid {
 	struct dc_bios bios;
 	enum bp_result result;
@@ -1331,6 +1466,8 @@ struct dm_test_synaptics_aux {
 	unsigned int downspread_reads;
 	unsigned int downspread_writes;
 	unsigned int dsc_enable_writes;
+	/* number of RC command reads that report the command still active */
+	unsigned int command_busy_reads;
 };
 
 static ssize_t dm_test_synaptics_aux_transfer(struct drm_dp_aux *aux,
@@ -1399,8 +1536,14 @@ static ssize_t dm_test_synaptics_aux_transfer(struct drm_dp_aux *aux,
 			fixture->downspread_reads++;
 			break;
 		case SYNAPTICS_RC_COMMAND:
-			if (msg->size)
-				buffer[0] = fixture->last_rc_command & 0x7f;
+			if (msg->size) {
+				if (fixture->command_busy_reads) {
+					fixture->command_busy_reads--;
+					buffer[0] = fixture->last_rc_command;
+				} else {
+					buffer[0] = fixture->last_rc_command & 0x7f;
+				}
+			}
 			fixture->rc_command_reads++;
 			break;
 		case SYNAPTICS_RC_RESULT:
@@ -1547,6 +1690,77 @@ static void dm_test_execute_synaptics_rc_command_write_fail(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, fixture->rc_command_count, 0U);
 }
 
+/**
+ * dm_test_execute_synaptics_rc_command_data_fail - Test RC data write failure
+ * @test: The KUnit test context
+ */
+static void dm_test_execute_synaptics_rc_command_data_fail(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 data = 0;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	fixture->fail_address = SYNAPTICS_RC_DATA;
+
+	KUNIT_EXPECT_FALSE(test, execute_synaptics_rc_command(&fixture->aux, true,
+							      0x01, sizeof(data), 0, &data));
+	KUNIT_EXPECT_EQ(test, fixture->last_rc_offset, 0U);
+}
+
+/**
+ * dm_test_execute_synaptics_rc_command_offset_fail - Test RC offset write failure
+ * @test: The KUnit test context
+ */
+static void dm_test_execute_synaptics_rc_command_offset_fail(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 data = 0;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	fixture->fail_address = SYNAPTICS_RC_OFFSET;
+
+	KUNIT_EXPECT_FALSE(test, execute_synaptics_rc_command(&fixture->aux, false,
+							      0x31, sizeof(data), 0, &data));
+	KUNIT_EXPECT_EQ(test, fixture->last_rc_length, 0U);
+}
+
+/**
+ * dm_test_execute_synaptics_rc_command_command_fail - Test RC command write failure
+ * @test: The KUnit test context
+ */
+static void dm_test_execute_synaptics_rc_command_command_fail(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 data = 0;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	fixture->fail_address = SYNAPTICS_RC_COMMAND;
+
+	KUNIT_EXPECT_FALSE(test, execute_synaptics_rc_command(&fixture->aux, false,
+							      0x31, sizeof(data), 0, &data));
+	KUNIT_EXPECT_EQ(test, fixture->rc_result_reads, 0U);
+}
+
+/**
+ * dm_test_execute_synaptics_rc_command_busy_poll - Test the RC command poll loop
+ * @test: The KUnit test context
+ *
+ * The command stays marked active for the first reads, so the helper sleeps and
+ * polls again before reading the result.
+ */
+static void dm_test_execute_synaptics_rc_command_busy_poll(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	u8 data = 0;
+
+	fixture = dm_test_alloc_synaptics_aux(test);
+	fixture->command_busy_reads = 2;
+
+	KUNIT_EXPECT_TRUE(test, execute_synaptics_rc_command(&fixture->aux, false,
+							     0x31, sizeof(data), 0, &data));
+	KUNIT_EXPECT_EQ(test, fixture->rc_command_reads, 3U);
+}
+
 /**
  * dm_test_apply_synaptics_fifo_reset_wa_full - Test full FIFO reset sequence
  * @test: The KUnit test context
@@ -2274,6 +2488,39 @@ static void dm_test_dmub_aux_transfer_sync_hpd_discon(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)result, (int)AUX_RET_ERROR_HPD_DISCON);
 }
 
+/**
+ * dm_test_dmub_aux_transfer_sync_hpd_connected - Test the transfer is forwarded to DMUB
+ * @test: The KUnit test context
+ *
+ * With HPD connected the helper forwards the payload to
+ * amdgpu_dm_process_dmub_aux_transfer_sync(). dc->link_count is zero, so the
+ * async transfer is rejected and the engine-acquire error is reported.
+ */
+static void dm_test_dmub_aux_transfer_sync_hpd_connected(struct kunit *test)
+{
+	struct aux_payload payload = {0};
+	enum aux_return_code_type result = AUX_RET_SUCCESS;
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	ctx = dm_kunit_alloc_dc_with_ctx(test)->ctx;
+	link = dm_kunit_alloc_link(test);
+
+	ctx->driver_context = adev;
+	mutex_init(&adev->dm.dpia_aux_lock);
+	init_completion(&adev->dm.dmub_aux_transfer_done);
+	link->hpd_status = true;
+
+	ret = dm_helper_dmub_aux_transfer_sync(ctx, link, &payload, &result);
+
+	KUNIT_EXPECT_EQ(test, ret, -1);
+	KUNIT_EXPECT_EQ(test, (int)result, (int)AUX_RET_ERROR_ENGINE_ACQUIRE);
+}
+
 /* Tests for empty stub functions (must not crash) */
 
 /**
@@ -2521,6 +2768,38 @@ static void dm_test_mst_enable_stream_features_writes_downspread(struct kunit *t
 	KUNIT_EXPECT_EQ(test, fixture->dpcd_write_value, (u8)BIT(7));
 }
 
+/**
+ * dm_test_mst_enable_stream_features_read_fail - Test early return on DPCD read failure
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_enable_stream_features_read_fail(struct kunit *test)
+{
+	struct dm_test_synaptics_aux *fixture;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+
+	adev = dm_kunit_alloc_adev(test);
+	fixture = dm_test_alloc_synaptics_aux_with_dev(test, &adev->ddev);
+	aconnector = dm_kunit_alloc_connector(test, adev, NULL);
+	link = dm_kunit_alloc_link(test);
+	stream = kunit_kzalloc(test, sizeof(*stream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream);
+
+	fixture->fail_address = DP_DOWNSPREAD_CTRL;
+	dm_test_current_aux_recorder = fixture;
+	aconnector->dm_dp_aux.aux.drm_dev = &adev->ddev;
+	aconnector->dm_dp_aux.aux.transfer = dm_test_current_aux_transfer;
+	drm_dp_aux_init(&aconnector->dm_dp_aux.aux);
+	link->priv = aconnector;
+	stream->link = link;
+
+	dm_helpers_mst_enable_stream_features(stream);
+
+	KUNIT_EXPECT_EQ(test, fixture->downspread_writes, 0U);
+}
+
 /* Tests for dm_helpers_enable_periodic_detection() */
 
 struct dm_test_idle_work {
@@ -2829,6 +3108,56 @@ static void dm_test_read_mccs_caps_i2c_failure(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, fake->reads, 0U);
 }
 
+/* Counts sized I2C-over-AUX writes; the bracketing bare-address ones have size 0. */
+static unsigned int dm_test_mccs_aux_writes;
+
+static ssize_t dm_test_mccs_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
+{
+	if ((msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_I2C_WRITE && msg->size)
+		dm_test_mccs_aux_writes++;
+
+	msg->reply = DP_AUX_I2C_REPLY_ACK;
+	return msg->size;
+}
+
+/*
+ * Route the fixture's link through an AUX DDC instead of the fake I2C adapter,
+ * so the MCCS helpers take their link->aux_mode branch.
+ */
+static void dm_test_mccs_use_aux(struct dm_test_mccs_fixture *fixture)
+{
+	struct drm_dp_aux *aux = &fixture->aconnector->dm_dp_aux.aux;
+
+	aux->drm_dev = &fixture->adev->ddev;
+	aux->transfer = dm_test_mccs_aux_transfer;
+	drm_dp_aux_init(aux);
+
+	fixture->aconnector->base.dev = &fixture->adev->ddev;
+	fixture->link->aux_mode = true;
+	dm_test_mccs_aux_writes = 0;
+}
+
+/**
+ * dm_test_read_mccs_caps_aux_mode - Test the MCCS VCP request over I2C-over-AUX
+ * @test: The KUnit test context
+ *
+ * The AUX DDC ACKs both transactions but answers with an empty VCP reply, so
+ * every attempt is retried and the request finally fails.
+ */
+static void dm_test_read_mccs_caps_aux_mode(struct kunit *test)
+{
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+
+	dm_test_mccs_use_aux(&fixture);
+	fixture.link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
+	fixture.sink->edid_caps.freesync_vcp_code = 0xe3;
+
+	dm_helpers_read_mccs_caps(fixture.ctx, fixture.link, fixture.sink);
+
+	KUNIT_EXPECT_FALSE(test, fixture.sink->mccs_caps.freesync_supported);
+	KUNIT_EXPECT_EQ(test, dm_test_mccs_aux_writes, 5U);
+}
+
 /* Tests for dm_helpers_mccs_vcp_set() */
 
 /**
@@ -2942,6 +3271,23 @@ static void dm_test_mccs_vcp_set_i2c_failure(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, fake->reads, 0U);
 }
 
+/**
+ * dm_test_mccs_vcp_set_aux_mode - Test the MCCS VCP set over I2C-over-AUX
+ * @test: The KUnit test context
+ */
+static void dm_test_mccs_vcp_set_aux_mode(struct kunit *test)
+{
+	struct dm_test_mccs_fixture fixture = dm_test_alloc_mccs_fixture(test);
+
+	dm_test_mccs_use_aux(&fixture);
+	fixture.sink->mccs_caps.freesync_supported = true;
+	fixture.sink->edid_caps.freesync_vcp_code = 0xe3;
+
+	dm_helpers_mccs_vcp_set(fixture.ctx, fixture.link, fixture.sink);
+
+	KUNIT_EXPECT_EQ(test, dm_test_mccs_aux_writes, 1U);
+}
+
 /* Tests for dm_helpers_construct_old_payload() */
 
 /**
@@ -3189,6 +3535,30 @@ static void dm_test_mst_start_top_mgr_set_mst_fail(struct kunit *test)
 	KUNIT_EXPECT_FALSE(test, dm_helpers_dp_mst_start_top_mgr(NULL, link, false));
 }
 
+/**
+ * dm_test_mst_start_top_mgr_already_started - Test MST start on an active manager
+ * @test: The KUnit test context
+ *
+ * With the topology manager already in MST mode,
+ * drm_dp_mst_topology_mgr_set_mst(true) is a no-op that returns success, so the
+ * helper logs the cached DPCD caps and returns true.
+ */
+static void dm_test_mst_start_top_mgr_already_started(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_link *link;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	link = dm_kunit_alloc_link(test);
+
+	mutex_init(&aconnector->mst_mgr.lock);
+	aconnector->mst_mgr.mst_state = true;
+	link->priv = aconnector;
+
+	KUNIT_EXPECT_TRUE(test, dm_helpers_dp_mst_start_top_mgr(NULL, link, false));
+}
+
 /**
  * dm_test_mst_stop_top_mgr_active - Test MST stop on an active topology manager
  * @test: The KUnit test context
@@ -3720,24 +4090,751 @@ static void dm_test_dp_handle_test_pattern_no_pipe(struct kunit *test)
 							       test_params));
 }
 
-static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
-	/* edid_extract_panel_id */
-	KUNIT_CASE(dm_test_edid_extract_panel_id_basic),
-	KUNIT_CASE(dm_test_edid_extract_panel_id_zeros),
-	/* apply_edid_quirks */
-	KUNIT_CASE(dm_test_apply_edid_quirks_dpcd_poweroff_delay),
-	KUNIT_CASE(dm_test_apply_edid_quirks_disable_fams),
-	KUNIT_CASE(dm_test_apply_edid_quirks_remove_sink_ext_caps),
-	KUNIT_CASE(dm_test_apply_edid_quirks_disable_colorimetry),
-	KUNIT_CASE(dm_test_apply_edid_quirks_skip_phy_ssc),
-	KUNIT_CASE(dm_test_apply_edid_quirks_unknown_noop),
-	/* dm_helpers_parse_edid_caps */
-	KUNIT_CASE(dm_test_parse_edid_caps_null_edid),
+/* Tests for dm_helpers_submit_i2c_over_aux() */
+
+/**
+ * dm_test_submit_i2c_over_aux_unimplemented - Test the unimplemented stub returns false
+ * @test: The KUnit test context
+ */
+static void dm_test_submit_i2c_over_aux_unimplemented(struct kunit *test)
+{
+	u8 buffer[1] = { 0 };
+
+	KUNIT_EXPECT_FALSE(test, dm_helpers_submit_i2c_over_aux(NULL, 0x50, 0, buffer,
+								sizeof(buffer), true));
+}
+
+/* Tests for dm_helpers_allocate_gpu_mem() and dm_helpers_free_gpu_mem() */
+
+/*
+ * Fake buffer object allocator: amdgpu_bo_create_kernel() and
+ * amdgpu_bo_free_kernel() need a live TTM device, so dm_allocate_gpu_mem() is
+ * routed through this fake.
+ */
+
+#define DM_TEST_FAKE_GPU_ADDR	0x1234ABCD0000ULL
+
+struct dm_test_bo_ctx {
+	void *cpu_ptr;
+	unsigned int create_calls;
+	unsigned int free_calls;
+	unsigned long create_size;
+	u32 create_domain;
+};
+
+static struct dm_test_bo_ctx dm_test_bo;
+
+static int dm_test_bo_create_kernel(struct amdgpu_device *adev, unsigned long size,
+				    int align, u32 domain, struct amdgpu_bo **bo_ptr,
+				    u64 *gpu_addr, void **cpu_addr)
+{
+	dm_test_bo.create_calls++;
+	dm_test_bo.create_size = size;
+	dm_test_bo.create_domain = domain;
+
+	*gpu_addr = DM_TEST_FAKE_GPU_ADDR;
+	*cpu_addr = dm_test_bo.cpu_ptr;
+
+	return 0;
+}
+
+static void dm_test_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, void **cpu_addr)
+{
+	dm_test_bo.free_calls++;
+
+	*bo = NULL;
+	*gpu_addr = 0;
+	*cpu_addr = NULL;
+}
+
+static const struct amdgpu_dm_services_kunit_ops dm_test_services_ops = {
+	.bo_create_kernel = dm_test_bo_create_kernel,
+	.bo_free_kernel = dm_test_bo_free_kernel,
+};
+
+/**
+ * dm_test_gpu_mem_alloc_and_free - Test the GPU memory helper wrappers
+ * @test: The KUnit test context
+ *
+ * Both helpers only unwrap the device from the DC context and forward to
+ * dm_allocate_gpu_mem()/dm_free_gpu_mem().
+ */
+static void dm_test_gpu_mem_alloc_and_free(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc_context *ctx;
+	long long addr = 0;
+	void *cpu_ptr;
+	void *mem;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	cpu_ptr = kunit_kzalloc(test, PAGE_SIZE, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, cpu_ptr);
+
+	ctx->driver_context = adev;
+	INIT_LIST_HEAD(&adev->dm.da_list);
+	dm_test_bo = (struct dm_test_bo_ctx) { .cpu_ptr = cpu_ptr };
+	amdgpu_dm_services_kunit_set_ops(&dm_test_services_ops);
+
+	mem = dm_helpers_allocate_gpu_mem(ctx, DC_MEM_ALLOC_TYPE_GART, 4096, &addr);
+
+	KUNIT_EXPECT_PTR_EQ(test, mem, cpu_ptr);
+	KUNIT_EXPECT_EQ(test, addr, (long long)DM_TEST_FAKE_GPU_ADDR);
+	KUNIT_EXPECT_EQ(test, dm_test_bo.create_calls, 1U);
+	KUNIT_EXPECT_EQ(test, dm_test_bo.create_size, 4096UL);
+	KUNIT_EXPECT_EQ(test, dm_test_bo.create_domain, (u32)AMDGPU_GEM_DOMAIN_GTT);
+
+	dm_helpers_free_gpu_mem(ctx, DC_MEM_ALLOC_TYPE_GART, mem);
+
+	KUNIT_EXPECT_EQ(test, dm_test_bo.free_calls, 1U);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+
+	amdgpu_dm_services_kunit_set_ops(NULL);
+}
+
+/* Tests for dm_helpers_dmub_set_config_sync() */
+
+/**
+ * dm_test_dmub_set_config_sync_unknown_error - Test SET_CONFIG without a DMUB service
+ * @test: The KUnit test context
+ *
+ * The helper only unwraps the link index before forwarding. With no DC DMUB
+ * service the command cannot reach the firmware and is reported as completed
+ * with SET_CONFIG_UNKNOWN_ERROR.
+ */
+static void dm_test_dmub_set_config_sync_unknown_error(struct kunit *test)
+{
+	struct set_config_cmd_payload payload = {0};
+	enum set_config_status result = SET_CONFIG_PENDING;
+	struct dmub_notification *notify;
+	struct amdgpu_device *adev;
+	struct dc_context *dc_ctx;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	dc_ctx = kunit_kzalloc(test, sizeof(*dc_ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_ctx);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	notify = kunit_kzalloc(test, sizeof(*notify), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, notify);
+	link = dm_kunit_alloc_link(test);
+
+	dc->ctx = dc_ctx;
+	dc->links[0] = link;
+	ctx->dc = dc;
+	ctx->driver_context = adev;
+	adev->dm.dmub_notify = notify;
+	mutex_init(&adev->dm.dpia_aux_lock);
+	init_completion(&adev->dm.dmub_aux_transfer_done);
+
+	KUNIT_EXPECT_EQ(test, dm_helpers_dmub_set_config_sync(ctx, link, &payload, &result), 0);
+	KUNIT_EXPECT_EQ(test, (int)result, (int)SET_CONFIG_UNKNOWN_ERROR);
+}
+
+/* Tests for dm_helpers_is_dp_sink_present() */
+
+static bool dm_test_dp_sink_present_true(struct dc_link *link)
+{
+	return true;
+}
+
+/**
+ * dm_test_is_dp_sink_present_queries_link_service - Test sink presence query
+ * @test: The KUnit test context
+ *
+ * With a connector present the helper takes the AUX hardware mutex and
+ * forwards to the link service.
+ */
+static void dm_test_is_dp_sink_present_queries_link_service(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	link = dm_kunit_alloc_link(test);
+	aconnector = dm_kunit_alloc_connector(test, adev, link);
+
+	aconnector->dm_dp_aux.aux.drm_dev = &adev->ddev;
+	aconnector->dm_dp_aux.aux.transfer = dm_test_dpcd_ack_transfer;
+	drm_dp_aux_init(&aconnector->dm_dp_aux.aux);
+
+	link_srv->dp_is_sink_present = dm_test_dp_sink_present_true;
+	dc->link_srv = link_srv;
+	link->dc = dc;
+	link->priv = aconnector;
+
+	KUNIT_EXPECT_TRUE(test, dm_helpers_is_dp_sink_present(link));
+}
+
+/* Tests for dm_helpers_read_local_edid() */
+
+/*
+ * dm_helpers_read_local_edid() feeds the EDID it reads into
+ * drm_edid_connector_update(), which needs a fully initialised connector so the
+ * EDID property blob is published. dm_kunit_alloc_connector() only fills in the
+ * device pointer, so build a real one here.
+ */
+static const struct drm_connector_funcs dm_test_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static struct amdgpu_dm_connector *dm_test_alloc_real_connector(struct kunit *test,
+								struct amdgpu_device *adev)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	KUNIT_ASSERT_EQ(test, drmm_mode_config_init(&adev->ddev), 0);
+
+	aconnector = drmm_kzalloc(&adev->ddev, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(&adev->ddev, &aconnector->base,
+					    &dm_test_connector_funcs,
+					    DRM_MODE_CONNECTOR_DisplayPort, NULL),
+			0);
+
+	return aconnector;
+}
+
+/*
+ * Fake AUX channel that serves an EDID over I2C-over-AUX and answers the
+ * native DPCD reads the compliance test-request path makes.
+ */
+struct dm_test_edid_aux {
+	u8 edid[EDID_LENGTH];
+	u8 offset;
+	u8 test_request;
+	u8 checksum_written;
+	u8 response_written;
+};
+
+static struct dm_test_edid_aux *dm_test_current_edid_aux;
+
+static ssize_t dm_test_edid_aux_transfer(struct drm_dp_aux *aux,
+					 struct drm_dp_aux_msg *msg)
+{
+	struct dm_test_edid_aux *fixture = dm_test_current_edid_aux;
+	u8 *buffer = msg->buffer;
+	size_t len;
+
+	switch (msg->request & ~DP_AUX_I2C_MOT) {
+	case DP_AUX_I2C_WRITE:
+		if (msg->size >= 1)
+			fixture->offset = buffer[0];
+		msg->reply = DP_AUX_I2C_REPLY_ACK;
+		return msg->size;
+	case DP_AUX_I2C_READ:
+		len = min_t(size_t, msg->size, EDID_LENGTH - fixture->offset);
+		memcpy(buffer, fixture->edid + fixture->offset, len);
+		fixture->offset += len;
+		msg->reply = DP_AUX_I2C_REPLY_ACK;
+		return len;
+	case DP_AUX_NATIVE_READ:
+		memset(buffer, 0, msg->size);
+		if (msg->address == DP_TEST_REQUEST && msg->size)
+			buffer[0] = fixture->test_request;
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+		return msg->size;
+	case DP_AUX_NATIVE_WRITE:
+		if (msg->size) {
+			if (msg->address == DP_TEST_EDID_CHECKSUM)
+				fixture->checksum_written = buffer[0];
+			else if (msg->address == DP_TEST_RESPONSE)
+				fixture->response_written = buffer[0];
+		}
+		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+		return msg->size;
+	}
+
+	msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+	return msg->size;
+}
+
+/* Fake I2C DDC adapter and prepare_ddc hook for the non-AUX EDID read path. */
+
+static int dm_test_ddc_no_sink_xfer(struct i2c_adapter *adapter,
+				    struct i2c_msg *msgs, int num)
+{
+	return -ENXIO;
+}
+
+static const struct i2c_algorithm dm_test_ddc_algorithm = {
+	.master_xfer = dm_test_ddc_no_sink_xfer,
+};
+
+static unsigned int dm_test_prepare_ddc_calls;
+
+static void dm_test_prepare_ddc(struct dc_link *link)
+{
+	dm_test_prepare_ddc_calls++;
+}
+
+struct dm_test_local_edid {
+	struct dm_test_edid_aux *aux;
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc_sink *sink;
+	struct dc *dc;
+};
+
+static struct dm_test_local_edid dm_test_setup_local_edid(struct kunit *test)
+{
+	struct dm_test_local_edid fixture = {0};
+	struct amdgpu_device *adev;
+	struct dc_edid *dc_edid;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	fixture.aconnector = dm_test_alloc_real_connector(test, adev);
+	fixture.aux = kunit_kzalloc(test, sizeof(*fixture.aux), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.aux);
+	fixture.ctx = kunit_kzalloc(test, sizeof(*fixture.ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.ctx);
+	fixture.sink = kunit_kzalloc(test, sizeof(*fixture.sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.sink);
+	fixture.dc = kunit_kzalloc(test, sizeof(*fixture.dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.dc);
+	fixture.link = dm_kunit_alloc_link(test);
+	dc_edid = kunit_kzalloc(test, sizeof(*dc_edid), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc_edid);
+
+	dm_test_fill_base_edid(dc_edid, true);
+	memcpy(fixture.aux->edid, dc_edid->raw_edid, EDID_LENGTH);
+	dm_test_current_edid_aux = fixture.aux;
+
+	fixture.aconnector->dm_dp_aux.aux.drm_dev = &adev->ddev;
+	fixture.aconnector->dm_dp_aux.aux.transfer = dm_test_edid_aux_transfer;
+	drm_dp_aux_init(&fixture.aconnector->dm_dp_aux.aux);
+
+	fixture.link->priv = fixture.aconnector;
+	fixture.link->dc = fixture.dc;
+	fixture.link->aux_mode = true;
+
+	return fixture;
+}
+
+/**
+ * dm_test_read_local_edid_aux_mode - Test reading EDID over the AUX DDC channel
+ * @test: The KUnit test context
+ *
+ * A DP link reads the EDID through the connector's AUX channel, copies it into
+ * the sink and parses the caps.
+ */
+static void dm_test_read_local_edid_aux_mode(struct kunit *test)
+{
+	struct dm_test_local_edid fixture = dm_test_setup_local_edid(test);
+
+	KUNIT_EXPECT_EQ(test,
+			dm_helpers_read_local_edid(fixture.ctx, fixture.link, fixture.sink),
+			EDID_OK);
+	KUNIT_EXPECT_EQ(test, fixture.sink->dc_edid.length, (uint32_t)EDID_LENGTH);
+	KUNIT_EXPECT_EQ(test, fixture.sink->edid_caps.manufacturer_id, 0xAC10);
+	KUNIT_EXPECT_EQ(test, fixture.sink->edid_caps.product_id, 0x1234);
+}
+
+/**
+ * dm_test_read_local_edid_test_request - Test the compliance EDID-read request
+ * @test: The KUnit test context
+ *
+ * When the sink raises the EDID_READ test request, the helper writes back the
+ * EDID checksum and the test response.
+ */
+static void dm_test_read_local_edid_test_request(struct kunit *test)
+{
+	struct dm_test_local_edid fixture = dm_test_setup_local_edid(test);
+	struct dm_test_edid_aux *aux = fixture.aux;
+	union test_response expected_response = {0};
+
+	aux->test_request = DP_TEST_LINK_EDID_READ;
+	expected_response.bits.EDID_CHECKSUM_WRITE = 1;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_helpers_read_local_edid(fixture.ctx, fixture.link, fixture.sink),
+			EDID_OK);
+	KUNIT_EXPECT_EQ(test, aux->checksum_written,
+			aux->edid[EDID_LENGTH - 1]);
+	KUNIT_EXPECT_EQ(test, aux->response_written, expected_response.raw);
+}
+
+/*
+ * Fake link service and clock manager for the test-pattern request path, which
+ * would otherwise reach the DSC config update, the clock update and the PHY
+ * test pattern programming.
+ */
+struct dm_test_pattern_state {
+	unsigned int dsc_config_calls;
+	unsigned int set_pattern_calls;
+	unsigned int update_clocks_calls;
+	enum dp_test_pattern last_pattern;
+	enum dp_test_pattern_color_space last_color_space;
+};
+
+static struct dm_test_pattern_state dm_test_pattern;
+
+static bool dm_test_link_update_dsc_config(struct pipe_ctx *pipe_ctx)
+{
+	dm_test_pattern.dsc_config_calls++;
+	return true;
+}
+
+static bool dm_test_link_dp_set_test_pattern(struct dc_link *link,
+					     enum dp_test_pattern test_pattern,
+					     enum dp_test_pattern_color_space color_space,
+					     const struct link_training_settings *settings,
+					     const unsigned char *custom_pattern,
+					     unsigned int custom_pattern_size)
+{
+	dm_test_pattern.set_pattern_calls++;
+	dm_test_pattern.last_pattern = test_pattern;
+	dm_test_pattern.last_color_space = color_space;
+	return true;
+}
+
+static void dm_test_update_clocks(struct clk_mgr *clk_mgr,
+				  struct dc_state *context, bool safe_to_lower)
+{
+	dm_test_pattern.update_clocks_calls++;
+}
+
+static struct clk_mgr_funcs dm_test_clk_mgr_funcs = {
+	.update_clocks = dm_test_update_clocks,
+};
+
+struct dm_test_pattern_fixture {
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_context *ctx;
+	struct dc_link *link;
+	struct dc_stream_state *stream;
+};
+
+static struct dm_test_pattern_fixture dm_test_setup_pattern(struct kunit *test)
+{
+	struct dm_test_pattern_fixture fixture = {0};
+	struct clk_bw_params *bw_params;
+	struct amdgpu_device *adev;
+	struct link_service *link_srv;
+	struct clk_mgr *clk_mgr;
+	struct dc_state *state;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	fixture.ctx = kunit_kzalloc(test, sizeof(*fixture.ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fixture.ctx);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	clk_mgr = kunit_kzalloc(test, sizeof(*clk_mgr), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, clk_mgr);
+	bw_params = kunit_kzalloc(test, sizeof(*bw_params), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, bw_params);
+	fixture.link = dm_kunit_alloc_link(test);
+	fixture.aconnector = dm_kunit_alloc_connector(test, adev, fixture.link);
+	fixture.stream = dm_kunit_alloc_stream(test, fixture.link);
+
+	link_srv->update_dsc_config = dm_test_link_update_dsc_config;
+	link_srv->dp_set_test_pattern = dm_test_link_dp_set_test_pattern;
+	clk_mgr->funcs = &dm_test_clk_mgr_funcs;
+	clk_mgr->bw_params = bw_params;
+
+	dc->link_srv = link_srv;
+	dc->clk_mgr = clk_mgr;
+	dc->current_state = state;
+	fixture.ctx->dc = dc;
+	fixture.link->dc = dc;
+	fixture.link->priv = fixture.aconnector;
+
+	/* the first pipe drives the link under test */
+	state->res_ctx.pipe_ctx[0].stream = fixture.stream;
+
+	dm_test_pattern = (struct dm_test_pattern_state) {0};
+
+	return fixture;
+}
+
+/**
+ * dm_test_dp_handle_test_pattern_patterns - Test the requested pattern mapping
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_handle_test_pattern_patterns(struct kunit *test)
+{
+	static const struct {
+		u8 pattern;
+		u8 dyn_range;
+		enum dp_test_pattern expected;
+	} cases[] = {
+		{ LINK_TEST_PATTERN_COLOR_RAMP, 0, DP_TEST_PATTERN_COLOR_RAMP },
+		{ LINK_TEST_PATTERN_VERTICAL_BARS, 0, DP_TEST_PATTERN_VERTICAL_BARS },
+		{ LINK_TEST_PATTERN_COLOR_SQUARES, TEST_DYN_RANGE_VESA, DP_TEST_PATTERN_COLOR_SQUARES },
+		{ LINK_TEST_PATTERN_COLOR_SQUARES, TEST_DYN_RANGE_CEA,  DP_TEST_PATTERN_COLOR_SQUARES_CEA },
+		/* PATTERN is 2 bits wide, so NONE is the only unhandled value */
+		{ LINK_TEST_PATTERN_NONE, 0, DP_TEST_PATTERN_VIDEO_MODE },
+	};
+	struct dm_test_pattern_fixture fixture = dm_test_setup_pattern(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		union link_test_pattern test_pattern = {0};
+		union test_misc test_params = {0};
+		bool ret;
+
+		dm_test_pattern = (struct dm_test_pattern_state) {0};
+		test_pattern.bits.PATTERN = cases[i].pattern;
+		test_params.bits.DYN_RANGE = cases[i].dyn_range;
+
+		ret = dm_helpers_dp_handle_test_pattern_request(fixture.ctx, fixture.link,
+								test_pattern, test_params);
+
+		KUNIT_EXPECT_FALSE(test, ret);
+		KUNIT_EXPECT_EQ_MSG(test, (int)dm_test_pattern.last_pattern,
+				    (int)cases[i].expected, "pattern %u", cases[i].pattern);
+		KUNIT_EXPECT_EQ(test, dm_test_pattern.set_pattern_calls, 1U);
+		KUNIT_EXPECT_EQ(test, dm_test_pattern.update_clocks_calls, 1U);
+	}
+}
+
+/**
+ * dm_test_dp_handle_test_pattern_color_spaces - Test the colour space mapping
+ * @test: The KUnit test context
+ */
+static void dm_test_dp_handle_test_pattern_color_spaces(struct kunit *test)
+{
+	static const struct {
+		u8 clr_format;
+		u8 ycbcr_coefs;
+		enum dp_test_pattern_color_space expected;
+	} cases[] = {
+		{ 0, 0, DP_TEST_PATTERN_COLOR_SPACE_RGB },
+		{ 1, 0, DP_TEST_PATTERN_COLOR_SPACE_YCBCR601 },
+		{ 2, 1, DP_TEST_PATTERN_COLOR_SPACE_YCBCR709 },
+	};
+	struct dm_test_pattern_fixture fixture = dm_test_setup_pattern(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		union link_test_pattern test_pattern = {0};
+		union test_misc test_params = {0};
+
+		dm_test_pattern = (struct dm_test_pattern_state) {0};
+		test_params.bits.CLR_FORMAT = cases[i].clr_format;
+		test_params.bits.YCBCR_COEFS = cases[i].ycbcr_coefs;
+
+		dm_helpers_dp_handle_test_pattern_request(fixture.ctx, fixture.link,
+							  test_pattern, test_params);
+
+		KUNIT_EXPECT_EQ_MSG(test, (int)dm_test_pattern.last_color_space,
+				    (int)cases[i].expected, "format %u", cases[i].clr_format);
+	}
+}
+
+/**
+ * dm_test_dp_handle_test_pattern_timing_change - Test the timing update branch
+ * @test: The KUnit test context
+ *
+ * A requested colour depth or pixel encoding that differs from the current
+ * stream timing reprograms the timing and updates the DSC config.
+ */
+static void dm_test_dp_handle_test_pattern_timing_change(struct kunit *test)
+{
+	static const struct {
+		u8 bpc;
+		u8 clr_format;
+		enum dc_color_depth depth;
+		enum dc_pixel_encoding encoding;
+	} cases[] = {
+		{ 2, 1, COLOR_DEPTH_101010, PIXEL_ENCODING_YCBCR422 },
+		{ 3, 2, COLOR_DEPTH_121212, PIXEL_ENCODING_YCBCR444 },
+	};
+	struct dm_test_pattern_fixture fixture = dm_test_setup_pattern(test);
+	struct dc_crtc_timing requested = {0};
+	unsigned int i;
+
+	fixture.aconnector->timing_requested = &requested;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		union link_test_pattern test_pattern = {0};
+		union test_misc test_params = {0};
+
+		dm_test_pattern = (struct dm_test_pattern_state) {0};
+		fixture.stream->timing.display_color_depth = COLOR_DEPTH_888;
+		fixture.stream->timing.pixel_encoding = PIXEL_ENCODING_RGB;
+		test_params.bits.BPC = cases[i].bpc;
+		test_params.bits.CLR_FORMAT = cases[i].clr_format;
+
+		dm_helpers_dp_handle_test_pattern_request(fixture.ctx, fixture.link,
+							  test_pattern, test_params);
+
+		KUNIT_EXPECT_EQ_MSG(test, (int)fixture.stream->timing.display_color_depth,
+				    (int)cases[i].depth, "bpc %u", cases[i].bpc);
+		KUNIT_EXPECT_EQ_MSG(test, (int)fixture.stream->timing.pixel_encoding,
+				    (int)cases[i].encoding, "format %u", cases[i].clr_format);
+		KUNIT_EXPECT_EQ(test, dm_test_pattern.dsc_config_calls, 1U);
+		KUNIT_EXPECT_TRUE(test, fixture.aconnector->timing_changed);
+		KUNIT_EXPECT_EQ(test, (int)requested.display_color_depth,
+				(int)cases[i].depth);
+	}
+}
+
+/**
+ * dm_test_dp_handle_test_pattern_no_timing_storage - Test the missing storage path
+ * @test: The KUnit test context
+ *
+ * Without a timing_requested buffer the helper still reprograms the timing but
+ * reports that it could not store it.
+ */
+static void dm_test_dp_handle_test_pattern_no_timing_storage(struct kunit *test)
+{
+	struct dm_test_pattern_fixture fixture = dm_test_setup_pattern(test);
+	union link_test_pattern test_pattern = {0};
+	union test_misc test_params = {0};
+
+	fixture.aconnector->timing_requested = NULL;
+	fixture.stream->timing.display_color_depth = COLOR_DEPTH_888;
+
+	/* 6 bpc differs from the current 8 bpc timing */
+	test_params.bits.BPC = 0;
+
+	dm_helpers_dp_handle_test_pattern_request(fixture.ctx, fixture.link,
+						  test_pattern, test_params);
+
+	KUNIT_EXPECT_EQ(test, (int)fixture.stream->timing.display_color_depth,
+			(int)COLOR_DEPTH_666);
+	KUNIT_EXPECT_TRUE(test, fixture.aconnector->timing_changed);
+}
+
+/**
+ * dm_test_read_local_edid_i2c_no_response - Test the I2C DDC path with no sink
+ * @test: The KUnit test context
+ *
+ * A non-AUX link reads the EDID over the connector's I2C adapter. With no sink
+ * responding, every retry fails and the helper reports EDID_NO_RESPONSE.
+ */
+static void dm_test_read_local_edid_i2c_no_response(struct kunit *test)
+{
+	struct dm_test_local_edid fixture = dm_test_setup_local_edid(test);
+	struct i2c_adapter *ddc;
+
+	ddc = kunit_kzalloc(test, sizeof(*ddc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ddc);
+	ddc->algo = &dm_test_ddc_algorithm;
+	ddc->lock_ops = &dm_test_i2c_lock_ops;
+	rt_mutex_init(&ddc->bus_lock);
+	rt_mutex_init(&ddc->mux_lock);
+
+	fixture.link->aux_mode = false;
+	fixture.link->ddc_hw_inst = 1;
+	fixture.aconnector->i2c = (struct amdgpu_i2c_adapter *)ddc;
+	fixture.dc->hwss.prepare_ddc = dm_test_prepare_ddc;
+	dm_test_prepare_ddc_calls = 0;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_helpers_read_local_edid(fixture.ctx, fixture.link, fixture.sink),
+			EDID_NO_RESPONSE);
+	KUNIT_EXPECT_EQ(test, dm_test_prepare_ddc_calls, 1U);
+}
+
+/**
+ * dm_test_read_local_edid_vbios_embedded - Test the VBIOS hardcoded EDID path
+ * @test: The KUnit test context
+ *
+ * An embedded panel with no DDC line has no I2C adapter to read from, so the
+ * helper falls back to the EDID hardcoded in the VBIOS embedded panel info.
+ */
+static void dm_test_read_local_edid_vbios_embedded(struct kunit *test)
+{
+	struct dm_test_local_edid fixture = dm_test_setup_local_edid(test);
+	struct dm_test_vbios_edid *vbios;
+
+	vbios = kunit_kzalloc(test, sizeof(*vbios), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, vbios);
+
+	vbios->bios.funcs = &dm_test_vbios_edid_funcs;
+	vbios->result = BP_RESULT_OK;
+	/* the fixture already holds a valid base EDID */
+	vbios->fake_edid = fixture.aux->edid;
+	vbios->fake_edid_size = EDID_LENGTH;
+
+	fixture.ctx->dc_bios = &vbios->bios;
+	fixture.link->ctx = fixture.ctx;
+	fixture.link->aux_mode = false;
+	fixture.link->ddc_hw_inst = GPIO_DDC_LINE_UNKNOWN;
+	fixture.link->connector_signal = SIGNAL_TYPE_EDP;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_helpers_read_local_edid(fixture.ctx, fixture.link, fixture.sink),
+			EDID_OK);
+	KUNIT_EXPECT_EQ(test, fixture.sink->dc_edid.length, (uint32_t)EDID_LENGTH);
+	KUNIT_EXPECT_EQ(test, fixture.sink->edid_caps.manufacturer_id, 0xAC10);
+}
+
+/**
+ * dm_test_read_local_edid_corrupt_checksum - Test the corrupt-EDID compliance path
+ * @test: The KUnit test context
+ *
+ * The sink serves an EDID with a bad checksum, so the DDC read fails and marks
+ * the connector's EDID corrupt. The helper writes the real checksum back over
+ * AUX, clears the corrupt flag and reports EDID_BAD_CHECKSUM.
+ */
+static void dm_test_read_local_edid_corrupt_checksum(struct kunit *test)
+{
+	struct dm_test_local_edid fixture = dm_test_setup_local_edid(test);
+
+	fixture.aux->edid[EDID_LENGTH - 1] ^= 0xFF;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_helpers_read_local_edid(fixture.ctx, fixture.link, fixture.sink),
+			EDID_BAD_CHECKSUM);
+	KUNIT_EXPECT_FALSE(test, fixture.aconnector->base.edid_corrupt);
+}
+
+static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
+	/* edid_extract_panel_id */
+	KUNIT_CASE(dm_test_edid_extract_panel_id_basic),
+	KUNIT_CASE(dm_test_edid_extract_panel_id_zeros),
+	/* apply_edid_quirks */
+	KUNIT_CASE(dm_test_apply_edid_quirks_dpcd_poweroff_delay),
+	KUNIT_CASE(dm_test_apply_edid_quirks_disable_fams),
+	KUNIT_CASE(dm_test_apply_edid_quirks_remove_sink_ext_caps),
+	KUNIT_CASE(dm_test_apply_edid_quirks_disable_colorimetry),
+	KUNIT_CASE(dm_test_apply_edid_quirks_psr_phy_power_down),
+	KUNIT_CASE(dm_test_apply_edid_quirks_skip_phy_ssc),
+	KUNIT_CASE(dm_test_apply_edid_quirks_force_freesync_min),
+	KUNIT_CASE(dm_test_apply_edid_quirks_unknown_noop),
+	KUNIT_CASE(dm_test_apply_edid_quirks_disable_second_tile),
+	KUNIT_CASE(dm_test_apply_edid_quirks_force_freesync_min_60),
+	/* dm_helpers_parse_edid_caps */
+	KUNIT_CASE(dm_test_parse_edid_caps_null_edid),
 	KUNIT_CASE(dm_test_parse_edid_caps_null_caps),
 	KUNIT_CASE(dm_test_parse_edid_caps_valid),
 	KUNIT_CASE(dm_test_parse_edid_caps_bad_checksum),
 	KUNIT_CASE(dm_test_parse_edid_caps_hdmi_frl),
 	KUNIT_CASE(dm_test_parse_edid_caps_hdmi_frl_dsc),
+	KUNIT_CASE(dm_test_parse_edid_caps_hdmi_comp_auto),
 	KUNIT_CASE(dm_test_parse_edid_caps_cea_audio),
 	KUNIT_CASE(dm_test_parse_edid_caps_cea_no_speaker),
 	/* ACPI / VBIOS / local EDID readers */
@@ -3745,6 +4842,7 @@ static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
 	KUNIT_CASE(dm_test_read_acpi_edid_debug_mask_disabled),
 	KUNIT_CASE(dm_test_read_acpi_edid_non_panel_connector),
 	KUNIT_CASE(dm_test_read_acpi_edid_force_off),
+	KUNIT_CASE(dm_test_read_acpi_edid_panel_connector),
 	KUNIT_CASE(dm_test_read_vbios_edid_non_embedded),
 	KUNIT_CASE(dm_test_read_vbios_edid_missing_callback),
 	KUNIT_CASE(dm_test_read_vbios_edid_callback_error),
@@ -3792,6 +4890,10 @@ static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
 	KUNIT_CASE(dm_test_execute_synaptics_rc_command_write_success),
 	KUNIT_CASE(dm_test_execute_synaptics_rc_command_read_success),
 	KUNIT_CASE(dm_test_execute_synaptics_rc_command_write_fail),
+	KUNIT_CASE(dm_test_execute_synaptics_rc_command_data_fail),
+	KUNIT_CASE(dm_test_execute_synaptics_rc_command_offset_fail),
+	KUNIT_CASE(dm_test_execute_synaptics_rc_command_command_fail),
+	KUNIT_CASE(dm_test_execute_synaptics_rc_command_busy_poll),
 	KUNIT_CASE(dm_test_apply_synaptics_fifo_reset_wa_full),
 	KUNIT_CASE(dm_test_apply_synaptics_fifo_reset_wa_first_fail),
 	KUNIT_CASE(dm_test_write_dsc_enable_synaptics_enable_inactive),
@@ -3805,6 +4907,7 @@ static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
 	KUNIT_CASE(dm_test_mst_stop_top_mgr_null_priv),
 	KUNIT_CASE(dm_test_mst_start_top_mgr_boot),
 	KUNIT_CASE(dm_test_mst_start_top_mgr_set_mst_fail),
+	KUNIT_CASE(dm_test_mst_start_top_mgr_already_started),
 	KUNIT_CASE(dm_test_mst_stop_top_mgr_active),
 	/* dm_helpers_dp_write_hblank_reduction */
 	KUNIT_CASE(dm_test_dp_write_hblank_reduction_false),
@@ -3833,6 +4936,7 @@ static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
 	KUNIT_CASE(dm_test_submit_i2c_partial_transfer),
 	/* dm_helper_dmub_aux_transfer_sync */
 	KUNIT_CASE(dm_test_dmub_aux_transfer_sync_hpd_discon),
+	KUNIT_CASE(dm_test_dmub_aux_transfer_sync_hpd_connected),
 	/* Empty stub functions */
 	KUNIT_CASE(dm_test_dp_update_branch_info_no_crash),
 	KUNIT_CASE(dm_test_mst_poll_pending_down_reply_no_crash),
@@ -3865,6 +4969,7 @@ static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
 	/* dm_helpers_mst_enable_stream_features */
 	KUNIT_CASE(dm_test_mst_enable_stream_features_aux_disabled),
 	KUNIT_CASE(dm_test_mst_enable_stream_features_writes_downspread),
+	KUNIT_CASE(dm_test_mst_enable_stream_features_read_fail),
 	/* dm_helpers_enable_periodic_detection */
 	KUNIT_CASE(dm_test_enable_periodic_detection_no_workqueue),
 	KUNIT_CASE(dm_test_enable_periodic_detection_updates_enable),
@@ -3877,12 +4982,14 @@ static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
 	KUNIT_CASE(dm_test_read_mccs_caps_hdmi_vcp_request),
 	KUNIT_CASE(dm_test_read_mccs_caps_legacy_pcon_vcp_request),
 	KUNIT_CASE(dm_test_read_mccs_caps_i2c_failure),
+	KUNIT_CASE(dm_test_read_mccs_caps_aux_mode),
 	/* dm_helpers_mccs_vcp_set */
 	KUNIT_CASE(dm_test_mccs_vcp_set_null_ctx),
 	KUNIT_CASE(dm_test_mccs_vcp_set_not_supported),
 	KUNIT_CASE(dm_test_mccs_vcp_set_null_link),
 	KUNIT_CASE(dm_test_mccs_vcp_set_i2c_packet),
 	KUNIT_CASE(dm_test_mccs_vcp_set_i2c_failure),
+	KUNIT_CASE(dm_test_mccs_vcp_set_aux_mode),
 	/* dm_helpers_construct_old_payload */
 	KUNIT_CASE(dm_test_construct_old_payload_empty_list),
 	KUNIT_CASE(dm_test_construct_old_payload_intervening),
@@ -3899,6 +5006,24 @@ static struct kunit_case amdgpu_dm_helpers_test_cases[] = {
 	KUNIT_CASE(dm_test_dp_write_dsc_enable_pcon_disable),
 	/* dm_helpers_dp_handle_test_pattern_request */
 	KUNIT_CASE(dm_test_dp_handle_test_pattern_no_pipe),
+	KUNIT_CASE(dm_test_dp_handle_test_pattern_patterns),
+	KUNIT_CASE(dm_test_dp_handle_test_pattern_color_spaces),
+	KUNIT_CASE(dm_test_dp_handle_test_pattern_timing_change),
+	KUNIT_CASE(dm_test_dp_handle_test_pattern_no_timing_storage),
+	/* dm_helpers_submit_i2c_over_aux */
+	KUNIT_CASE(dm_test_submit_i2c_over_aux_unimplemented),
+	/* dm_helpers_allocate_gpu_mem / dm_helpers_free_gpu_mem */
+	KUNIT_CASE(dm_test_gpu_mem_alloc_and_free),
+	/* dm_helpers_dmub_set_config_sync */
+	KUNIT_CASE(dm_test_dmub_set_config_sync_unknown_error),
+	/* dm_helpers_is_dp_sink_present */
+	KUNIT_CASE(dm_test_is_dp_sink_present_queries_link_service),
+	/* dm_helpers_read_local_edid */
+	KUNIT_CASE(dm_test_read_local_edid_aux_mode),
+	KUNIT_CASE(dm_test_read_local_edid_test_request),
+	KUNIT_CASE(dm_test_read_local_edid_i2c_no_response),
+	KUNIT_CASE(dm_test_read_local_edid_vbios_embedded),
+	KUNIT_CASE(dm_test_read_local_edid_corrupt_checksum),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
index 95322d8c861320..6e599657dac158 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_irq_test.c
@@ -6,16 +6,22 @@
  */
 
 #include <kunit/test.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_kunit_helpers.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_probe_helper.h>
+#include <drm/drm_vblank.h>
+#include <drm/drm_writeback.h>
+#include <drm/display/drm_dp_helper.h>
 
 #include "dc.h"
 #include "inc/core_types.h"
+#include "inc/hw/timing_generator.h"
 #include "irq/irq_service.h"
 #include "amdgpu.h"
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
+#include "amdgpu_dm_hdcp.h"
 #include "amdgpu_dm_irq.h"
 #include "amdgpu_dm_kunit_test_helpers.h"
 #include "dc_dmub_srv.h"
@@ -56,6 +62,23 @@ static bool dm_test_detect_link_false(struct dc_link *link,
 	return false;
 }
 
+static bool dm_test_detect_link_true(struct dc_link *link,
+				     enum dc_detect_reason reason)
+{
+	return true;
+}
+
+/* Spy on dc_link_detect() to prove the debounce early-return skipped it. */
+static int dm_test_detect_link_count;
+
+static bool dm_test_detect_link_false_count(struct dc_link *link,
+					    enum dc_detect_reason reason)
+{
+	dm_test_detect_link_count++;
+
+	return false;
+}
+
 static bool dm_test_detect_connection_single(struct dc_link *link,
 					     enum dc_connection_type *type)
 {
@@ -64,6 +87,12 @@ static bool dm_test_detect_connection_single(struct dc_link *link,
 	return true;
 }
 
+static bool dm_test_detect_connection_fail(struct dc_link *link,
+					   enum dc_connection_type *type)
+{
+	return false;
+}
+
 /* Recording stubs for the dm_handle_hpd_rx_offload_work() DP-IRQ branches. */
 static int dm_test_automated_test_count;
 static int dm_test_handle_link_loss_count;
@@ -95,6 +124,18 @@ static enum dc_status dm_test_dp_read_hpd_rx_irq_data_ok(struct dc_link *link,
 	return DC_OK;
 }
 
+static const struct dc_link_status *dm_test_get_link_status(const struct dc_link *link)
+{
+	return &link->link_status;
+}
+
+/* Fail every AUX transaction so the MST ESI read loop bails out immediately. */
+static ssize_t dm_test_aux_transfer_fail(struct drm_dp_aux *aux,
+					 struct drm_dp_aux_msg *msg)
+{
+	return -EIO;
+}
+
 /*
  * Allocate a refcounted dc_sink for tests without pulling in the DC-core
  * dc_sink_create()/dc_sink_release() symbols. Production code under test still
@@ -178,17 +219,70 @@ static bool dm_test_handle_hpd_rx_link_loss(struct dc_link *link,
 	return false;
 }
 
+static bool dm_test_handle_hpd_rx_result_true(struct dc_link *link,
+					      union hpd_irq_data *hpd_irq_data,
+					      bool *link_loss,
+					      bool defer_handling,
+					      bool *has_left_work)
+{
+	*link_loss = false;
+	*has_left_work = false;
+
+	return true;
+}
+
+static bool dm_test_handle_hpd_rx_cp_irq(struct dc_link *link,
+					 union hpd_irq_data *hpd_irq_data,
+					 bool *link_loss,
+					 bool defer_handling,
+					 bool *has_left_work)
+{
+	*link_loss = false;
+	*has_left_work = false;
+	hpd_irq_data->bytes.device_service_irq.bits.CP_IRQ = 1;
+
+	return false;
+}
+
 static bool dm_test_allow_hpd_rx_irq_true(const struct dc_link *link)
 {
 	return true;
 }
 
+/* Report a scanout position past the start of the front porch. */
+static void dm_test_tg_get_scanoutpos(struct timing_generator *tg,
+				      uint32_t *v_blank_start,
+				      uint32_t *v_blank_end,
+				      uint32_t *h_position,
+				      uint32_t *v_position)
+{
+	*v_blank_start = 100;
+	*v_blank_end = 110;
+	*h_position = 0;
+	*v_position = 200;
+}
+
+static const struct timing_generator_funcs dm_test_tg_funcs = {
+	.get_scanoutpos = dm_test_tg_get_scanoutpos,
+};
 
 static uint32_t dm_test_dmub_get_outbox0_wptr(struct dmub_srv *dmub)
 {
 	return 0;
 }
 
+/* One more than DMUB_TRACE_MAX_READ, so the drain loop reaches its cap. */
+#define DM_TEST_OUTBOX0_TRACE_ENTRIES	65
+#define DM_TEST_OUTBOX0_TRACE_BYTES \
+	(DM_TEST_OUTBOX0_TRACE_ENTRIES * sizeof(struct dmcub_trace_buf_entry))
+/* Keep the ring larger than the write pointer so rptr does not wrap. */
+#define DM_TEST_OUTBOX0_RB_SIZE		(2 * DM_TEST_OUTBOX0_TRACE_BYTES)
+
+static uint32_t dm_test_dmub_get_outbox0_wptr_full(struct dmub_srv *dmub)
+{
+	return DM_TEST_OUTBOX0_TRACE_BYTES;
+}
+
 static uint32_t dm_test_dmub_get_outbox1_wptr(struct dmub_srv *dmub)
 {
 	return 0;
@@ -233,6 +327,55 @@ static const struct irq_service_funcs dm_test_irq_service_funcs_dce110 = {
 	.to_dal_irq_source = dm_test_to_dal_irq_source_dce110
 };
 
+/* Maps nothing, so the first (VBLANK) register loop sees an invalid source. */
+static enum dc_irq_source dm_test_to_dal_irq_source_unmapped(
+		struct irq_service *irq_service,
+		uint32_t src_id,
+		uint32_t ext_id)
+{
+	return DC_IRQ_SOURCE_INVALID;
+}
+
+static const struct irq_service_funcs dm_test_irq_service_funcs_unmapped = {
+	.to_dal_irq_source = dm_test_to_dal_irq_source_unmapped
+};
+
+/* Maps VBLANK only, so the VUPDATE register loop sees an invalid source. */
+static enum dc_irq_source dm_test_to_dal_irq_source_vblank_only(
+		struct irq_service *irq_service,
+		uint32_t src_id,
+		uint32_t ext_id)
+{
+	if (src_id == VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0)
+		return DC_IRQ_SOURCE_VBLANK1;
+
+	return DC_IRQ_SOURCE_INVALID;
+}
+
+static const struct irq_service_funcs dm_test_irq_service_funcs_vblank_only = {
+	.to_dal_irq_source = dm_test_to_dal_irq_source_vblank_only
+};
+
+/* Maps everything but PFLIP, so the PFLIP register loop sees an invalid source. */
+static enum dc_irq_source dm_test_to_dal_irq_source_no_pflip(
+		struct irq_service *irq_service,
+		uint32_t src_id,
+		uint32_t ext_id)
+{
+	switch (src_id) {
+	case VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0:
+		return DC_IRQ_SOURCE_VBLANK1;
+	case VISLANDS30_IV_SRCID_D1_V_UPDATE_INT:
+		return DC_IRQ_SOURCE_VUPDATE1;
+	default:
+		return DC_IRQ_SOURCE_INVALID;
+	}
+}
+
+static const struct irq_service_funcs dm_test_irq_service_funcs_no_pflip = {
+	.to_dal_irq_source = dm_test_to_dal_irq_source_no_pflip
+};
+
 static enum dc_irq_source dm_test_to_dal_irq_source_dcn10(
 		struct irq_service *irq_service,
 		uint32_t src_id,
@@ -337,9 +480,44 @@ static void dm_test_crtc_list_del(void *data)
 {
 	struct amdgpu_crtc *acrtc = data;
 
+	if (acrtc->base.dev && drm_dev_has_vblank(acrtc->base.dev))
+		drm_crtc_vblank_off(&acrtc->base);
+
 	list_del_init(&acrtc->base.head);
 }
 
+/*
+ * The DRM vblank core looks CRTCs up by index on mode_config.crtc_list and
+ * dereferences crtc->funcs, so a listed CRTC must carry a funcs table even
+ * when it implements no callbacks.
+ */
+static const struct drm_crtc_funcs dm_test_crtc_funcs = {
+};
+
+/*
+ * Add an OTG instance 0 CRTC to the device so amdgpu_dm_get_crtc_by_otg_inst()
+ * finds it. The CRTC is removed from the list again on test teardown.
+ */
+static struct amdgpu_crtc *dm_test_add_crtc(struct kunit *test,
+					    struct amdgpu_device *adev)
+{
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+
+	INIT_LIST_HEAD(&acrtc->base.head);
+	acrtc->base.dev = &adev->ddev;
+	acrtc->base.funcs = &dm_test_crtc_funcs;
+	acrtc->otg_inst = 0;
+
+	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
+				dm_test_crtc_list_del, acrtc), 0);
+
+	return acrtc;
+}
+
 struct dm_test_hpd_rx_wq_ctx {
 	struct hpd_rx_irq_offload_work_queue *wq;
 	int count;
@@ -369,6 +547,55 @@ static void dm_test_connector_cleanup(void *data)
 	drm_connector_cleanup(data);
 }
 
+static void dm_test_drm_dev_unregister(void *data)
+{
+	drm_dev_unregister(data);
+}
+
+/*
+ * Registering the DRM device gives its primary minor a sysfs device, which the
+ * DRM hotplug uevent helpers dereference unconditionally.
+ */
+static void dm_test_register_drm_dev(struct kunit *test, struct amdgpu_device *adev)
+{
+	KUNIT_ASSERT_EQ(test, drm_dev_register(&adev->ddev, 0), 0);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_drm_dev_unregister,
+							&adev->ddev), 0);
+}
+
+static void dm_test_hdcp_dummy_work(struct work_struct *work)
+{
+}
+
+/*
+ * Allocate the minimal hdcp_workqueue that hdcp_reset_display() and
+ * hdcp_handle_cpirq() touch: the per-link mutex, the delayed works
+ * process_output() re-arms, and the CP IRQ work item.
+ */
+static struct hdcp_workqueue *dm_test_alloc_hdcp_workqueue(struct kunit *test)
+{
+	struct hdcp_workqueue *hdcp_work;
+
+	hdcp_work = kunit_kzalloc(test, sizeof(*hdcp_work), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hdcp_work);
+
+	mutex_init(&hdcp_work->mutex);
+	INIT_WORK(&hdcp_work->cpirq_work, dm_test_hdcp_dummy_work);
+	INIT_DELAYED_WORK(&hdcp_work->callback_dwork, dm_test_hdcp_dummy_work);
+	INIT_DELAYED_WORK(&hdcp_work->watchdog_timer_dwork, dm_test_hdcp_dummy_work);
+	INIT_DELAYED_WORK(&hdcp_work->property_validate_dwork, dm_test_hdcp_dummy_work);
+
+	return hdcp_work;
+}
+
+static void dm_test_flush_hdcp_workqueue(struct hdcp_workqueue *hdcp_work)
+{
+	flush_work(&hdcp_work->cpirq_work);
+	cancel_delayed_work_sync(&hdcp_work->callback_dwork);
+	cancel_delayed_work_sync(&hdcp_work->watchdog_timer_dwork);
+	cancel_delayed_work_sync(&hdcp_work->property_validate_dwork);
+}
+
 /* Tests for amdgpu_dm_hpd_to_dal_irq_source() */
 
 /**
@@ -441,27 +668,8 @@ static void dm_test_hpd_to_dal_irq_source_invalid(struct kunit *test)
 			(int)DC_IRQ_SOURCE_INVALID);
 }
 
-/**
- * dm_test_hpd_to_dal_irq_source_out_of_range - Test Hpd to dal irq source out of range
- * @test: The KUnit test context
- */
-static void dm_test_hpd_to_dal_irq_source_out_of_range(struct kunit *test)
-{
-	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_hpd_to_dal_irq_source(99),
-			(int)DC_IRQ_SOURCE_INVALID);
-}
-
 /* Tests for are_sinks_equal() */
 
-/**
- * dm_test_are_sinks_equal_both_null - Test Are sinks equal both null
- * @test: The KUnit test context
- */
-static void dm_test_are_sinks_equal_both_null(struct kunit *test)
-{
-	KUNIT_EXPECT_FALSE(test, are_sinks_equal(NULL, NULL));
-}
-
 /**
  * dm_test_are_sinks_equal_first_null - Test Are sinks equal first null
  * @test: The KUnit test context
@@ -553,29 +761,6 @@ static void dm_test_are_sinks_equal_different_edid_data(struct kunit *test)
 	KUNIT_EXPECT_FALSE(test, are_sinks_equal(sink1, sink2));
 }
 
-/**
- * dm_test_are_sinks_equal_identical - Test Are sinks equal identical
- * @test: The KUnit test context
- */
-static void dm_test_are_sinks_equal_identical(struct kunit *test)
-{
-	struct dc_sink *sink1, *sink2;
-
-	sink1 = kunit_kzalloc(test, sizeof(*sink1), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink1);
-	sink2 = kunit_kzalloc(test, sizeof(*sink2), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sink2);
-
-	sink1->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
-	sink2->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
-	sink1->dc_edid.length = 4;
-	sink2->dc_edid.length = 4;
-	memset(sink1->dc_edid.raw_edid, 0xAA, 4);
-	memset(sink2->dc_edid.raw_edid, 0xAA, 4);
-
-	KUNIT_EXPECT_TRUE(test, are_sinks_equal(sink1, sink2));
-}
-
 /**
  * dm_test_are_sinks_equal_zero_length - Test Are sinks equal zero length
  * @test: The KUnit test context
@@ -819,105 +1004,6 @@ static void dm_test_irq_register_rejects_invalid_source(struct kunit *test)
 						   dm_test_irq_handler, NULL));
 }
 
-/**
- * dm_test_irq_register_adds_low_context_handler - Test register adds low handler
- * @test: The KUnit test context
- */
-static void dm_test_irq_register_adds_low_context_handler(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc_interrupt_params int_params = { 0 };
-	void *handler;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
-	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
-
-	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
-	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
-
-	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
-						    dm_test_irq_handler, adev);
-
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
-	KUNIT_EXPECT_FALSE(test,
-			   list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
-	KUNIT_EXPECT_TRUE(test,
-			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD1]));
-
-	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
-						dm_test_irq_handler);
-	KUNIT_EXPECT_TRUE(test,
-			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1]));
-}
-
-/**
- * dm_test_irq_register_adds_high_context_handler - Test register adds high handler
- * @test: The KUnit test context
- */
-static void dm_test_irq_register_adds_high_context_handler(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc_interrupt_params int_params = { 0 };
-	void *handler;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
-	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
-
-	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-	int_params.irq_source = DC_IRQ_SOURCE_HPD2;
-
-	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
-						    dm_test_irq_handler, adev);
-
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
-	KUNIT_EXPECT_FALSE(test,
-			   list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD2]));
-	KUNIT_EXPECT_TRUE(test,
-			  list_empty(&adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD2]));
-
-	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD2,
-						dm_test_irq_handler);
-	KUNIT_EXPECT_TRUE(test,
-			  list_empty(&adev->dm.irq_handler_list_high_tab[DC_IRQ_SOURCE_HPD2]));
-}
-
-/**
- * dm_test_irq_register_multiple_handlers - Test register keeps multiple handlers
- * @test: The KUnit test context
- */
-static void dm_test_irq_register_multiple_handlers(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc_interrupt_params int_params = { 0 };
-	struct list_head *hnd_list;
-	void *handler1, *handler2;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
-	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
-
-	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
-	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
-
-	handler1 = amdgpu_dm_irq_register_interrupt(adev, &int_params,
-						    dm_test_irq_handler, adev);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler1);
-	handler2 = amdgpu_dm_irq_register_interrupt(adev, &int_params,
-						    dm_test_irq_handler_alt, adev);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler2);
-
-	hnd_list = &adev->dm.irq_handler_list_low_tab[DC_IRQ_SOURCE_HPD1];
-	KUNIT_EXPECT_EQ(test, list_count_nodes(hnd_list), 2);
-
-	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
-						dm_test_irq_handler);
-	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1,
-						dm_test_irq_handler_alt);
-	KUNIT_EXPECT_TRUE(test, list_empty(hnd_list));
-}
-
 /**
  * dm_test_irq_register_separate_contexts - Test register same source in two contexts
  * @test: The KUnit test context
@@ -1177,6 +1263,28 @@ static void dm_test_get_crtc_by_otg_inst_empty_list(struct kunit *test)
 	KUNIT_EXPECT_NULL(test, amdgpu_dm_get_crtc_by_otg_inst(adev, 0));
 }
 
+/**
+ * dm_test_get_crtc_by_otg_inst_unassigned_otg - Test CRTC lookup for a disabled OTG
+ * @test: The KUnit test context
+ *
+ * An IRQ that names an OTG instance of -1 cannot be matched, so the lookup
+ * trips its WARN_ON and falls back to the first CRTC rather than walking the
+ * list.
+ */
+static void dm_test_get_crtc_by_otg_inst_unassigned_otg(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = dm_kunit_alloc_adev(test);
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
+	adev->mode_info.crtcs[0] = acrtc;
+
+	KUNIT_EXPECT_PTR_EQ(test, amdgpu_dm_get_crtc_by_otg_inst(adev, -1), acrtc);
+}
+
 /* Tests for amdgpu_dm_set_irq_funcs() */
 
 /**
@@ -1624,49 +1732,7 @@ static void dm_test_set_pflip_irq_state_disable(struct kunit *test)
 }
 
 /**
- * dm_test_set_vline0_irq_state_enable - Test vline0 irq state reaches DC (enable)
- * @test: The KUnit test context
- */
-static void dm_test_set_vline0_irq_state_enable(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct amdgpu_crtc *acrtc;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
-	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
-
-	acrtc->otg_inst = 0;
-	adev->mode_info.crtcs[0] = acrtc;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_set_vline0_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
-}
-
-/**
- * dm_test_set_vupdate_irq_state_enable - Test vupdate irq state reaches DC (enable)
- * @test: The KUnit test context
- */
-static void dm_test_set_vupdate_irq_state_enable(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct amdgpu_crtc *acrtc;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
-	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
-
-	acrtc->otg_inst = 2;
-	adev->mode_info.crtcs[0] = acrtc;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_set_vupdate_irq_state(adev, NULL, 0, AMDGPU_IRQ_STATE_ENABLE), 0);
-}
-
-/**
- * dm_test_set_crtc_irq_state_allows_idle - Test the idle-optimization branch
+ * dm_test_set_crtc_irq_state_allows_idle - Test the idle-optimization branch
  * @test: The KUnit test context
  *
  * With a non-NULL DC that advertises IPS support and currently allows idle
@@ -1724,34 +1790,6 @@ static void dm_test_irq_immediate_work_empty(struct kunit *test)
 	amdgpu_dm_irq_immediate_work(adev, DC_IRQ_SOURCE_HPD1);
 }
 
-/**
- * dm_test_irq_immediate_work_invokes_handler - Test immediate work calls handler
- * @test: The KUnit test context
- */
-static void dm_test_irq_immediate_work_invokes_handler(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc_interrupt_params int_params = { 0 };
-	int count = 0;
-	void *handler;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
-	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
-
-	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
-	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
-	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
-						   dm_test_irq_handler_count, &count);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
-
-	/* High-context handlers are invoked synchronously, in-place. */
-	amdgpu_dm_irq_immediate_work(adev, DC_IRQ_SOURCE_HPD1);
-	KUNIT_EXPECT_EQ(test, count, 1);
-
-	amdgpu_dm_irq_unregister_interrupt(adev, DC_IRQ_SOURCE_HPD1, dm_test_irq_handler_count);
-}
-
 /**
  * dm_test_irq_immediate_work_invokes_all - Test immediate work calls all handlers
  * @test: The KUnit test context
@@ -1878,6 +1916,39 @@ static void dm_test_irq_schedule_work_requeue_fallback(struct kunit *test)
 	amdgpu_dm_irq_fini(adev);
 }
 
+/**
+ * dm_test_irq_schedule_work_no_workqueue - Test schedule work after wq teardown
+ * @test: The KUnit test context
+ *
+ * An interrupt that races DM teardown can reach the scheduler after the IRQ
+ * workqueue is gone, so a registered handler must simply not be queued.
+ */
+static void dm_test_irq_schedule_work_no_workqueue(struct kunit *test)
+{
+	struct dc_interrupt_params int_params = { 0 };
+	struct amdgpu_device *adev;
+	int count = 0;
+	void *handler;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_HPD1;
+	handler = amdgpu_dm_irq_register_interrupt(adev, &int_params,
+						   dm_test_irq_handler_count, &count);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, handler);
+
+	destroy_workqueue(adev->dm.irq_wq);
+	adev->dm.irq_wq = NULL;
+
+	amdgpu_dm_irq_schedule_work(adev, DC_IRQ_SOURCE_HPD1);
+	KUNIT_EXPECT_EQ(test, count, 0);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
 /* Tests for amdgpu_dm_set_hpd_irq_state() */
 
 /**
@@ -2114,69 +2185,52 @@ static void dm_test_hpd_init_fini_irq_ref(struct kunit *test)
 	amdgpu_dm_hpd_fini(adev);
 }
 
-/* Tests for dm_handle_hpd_rx_offload_work() */
-
 /**
- * dm_test_hpd_rx_offload_work_no_connector - Test missing connector early exit
+ * dm_test_hpd_init_skips_connector_without_link - Test HPD init skips linkless
  * @test: The KUnit test context
+ *
+ * A connector that does not carry a dc_link yet must be skipped by the init
+ * loop. Only init is exercised: fini has no such guard.
  */
-static void dm_test_hpd_rx_offload_work_no_connector(struct kunit *test)
+static void dm_test_hpd_init_skips_connector_without_link(struct kunit *test)
 {
-	struct hpd_rx_irq_offload_work_queue *offload_wq;
-	struct hpd_rx_irq_offload_work *offload_work;
+	struct amdgpu_dm_connector *aconn;
 	struct amdgpu_device *adev;
 
 	adev = dm_kunit_alloc_adev(test);
+	adev->mode_info.num_hpd = 0;
 
-	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
+	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
+						 &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_DisplayPort), 0);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_connector_cleanup,
+							&aconn->base), 0);
 
-	offload_work = kzalloc_obj(*offload_work);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
-	offload_work->offload_wq = offload_wq;
-	offload_work->adev = adev;
-	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
+	amdgpu_dm_hpd_init(adev);
 
-	dm_handle_hpd_rx_offload_work(&offload_work->work);
+	/* Skipped before the analog check, so polling was never requested. */
+	KUNIT_EXPECT_FALSE(test, adev->ddev.mode_config.poll_enabled);
 }
 
+/* Tests for dm_handle_hpd_rx_offload_work() */
+
 /**
- * dm_test_hpd_rx_offload_work_no_connection - Test no connection early exit
+ * dm_test_hpd_rx_offload_work_no_connector - Test missing connector early exit
  * @test: The KUnit test context
  */
-static void dm_test_hpd_rx_offload_work_no_connection(struct kunit *test)
+static void dm_test_hpd_rx_offload_work_no_connector(struct kunit *test)
 {
 	struct hpd_rx_irq_offload_work_queue *offload_wq;
 	struct hpd_rx_irq_offload_work *offload_work;
-	struct amdgpu_dm_connector *aconn;
-	struct link_service *link_srv;
 	struct amdgpu_device *adev;
-	struct dc_link *link;
-	struct dc *dc;
 
 	adev = dm_kunit_alloc_adev(test);
-	mutex_init(&adev->dm.dc_lock);
 
 	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
 
-	aconn = dm_kunit_alloc_connector(test, adev, NULL);
-	mutex_init(&aconn->hpd_lock);
-	offload_wq->aconnector = aconn;
-
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
-
-	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
-	link_srv->detect_connection_type = dm_test_detect_connection_none;
-	dc->link_srv = link_srv;
-
-	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
-	link->dc = dc;
-	aconn->dc_link = link;
-
 	offload_work = kzalloc_obj(*offload_work);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
 	offload_work->offload_wq = offload_wq;
@@ -2186,17 +2240,17 @@ static void dm_test_hpd_rx_offload_work_no_connection(struct kunit *test)
 	dm_handle_hpd_rx_offload_work(&offload_work->work);
 }
 
-/**
- * dm_test_hpd_rx_offload_work_automated_test - Test AUTOMATED_TEST branch
- * @test: The KUnit test context
+/*
+ * Build a queued hpd_rx_irq_offload_work for dm_handle_hpd_rx_offload_work():
+ * an adev that is not in reset, an offload queue owning a connector with an
+ * HPD lock, and a DisplayPort dc/link pair that reports a present sink.
+ * Caller sets the link_srv stubs and the service-IRQ bits its branch needs.
  *
- * With a present connection and the AUTOMATED_TEST service-IRQ bit set, the
- * worker runs dc_link_dp_handle_automated_test() (stubbed via link_srv) and
- * writes the test response with core_link_write_dpcd(). timing_changed is left
- * false so force_connector_state() (which needs a registered DRM device) is
- * skipped, and aux_access_disabled makes the DPCD write a safe no-op.
+ * The work item is kzalloc'd because the worker frees it.
  */
-static void dm_test_hpd_rx_offload_work_automated_test(struct kunit *test)
+static struct hpd_rx_irq_offload_work *
+dm_test_setup_hpd_rx_offload_work(struct kunit *test,
+				  struct link_service **link_srv_out)
 {
 	struct hpd_rx_irq_offload_work_queue *offload_wq;
 	struct hpd_rx_irq_offload_work *offload_work;
@@ -2207,8 +2261,6 @@ static void dm_test_hpd_rx_offload_work_automated_test(struct kunit *test)
 	struct dc_link *link;
 	struct dc *dc;
 
-	dm_test_automated_test_count = 0;
-
 	adev = dm_kunit_alloc_adev(test);
 	mutex_init(&adev->dm.dc_lock);
 	adev->reset_domain = kunit_kzalloc(test, sizeof(*adev->reset_domain),
@@ -2221,7 +2273,6 @@ static void dm_test_hpd_rx_offload_work_automated_test(struct kunit *test)
 
 	aconn = dm_kunit_alloc_connector(test, adev, NULL);
 	mutex_init(&aconn->hpd_lock);
-	aconn->timing_changed = false;
 	offload_wq->aconnector = aconn;
 
 	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
@@ -2234,159 +2285,209 @@ static void dm_test_hpd_rx_offload_work_automated_test(struct kunit *test)
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
 
 	link_srv->detect_connection_type = dm_test_detect_connection_single;
-	link_srv->dp_handle_automated_test = dm_test_dp_handle_automated_test;
 	dc->link_srv = link_srv;
 	dc->ctx = ctx;
 	ctx->dc = dc;
 	link->dc = dc;
 	link->ctx = ctx;
-	link->aux_access_disabled = true;
 	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
 	aconn->dc_link = link;
 
-	offload_work = kzalloc_obj(*offload_work);
+	offload_work = kzalloc_obj(*offload_work, GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
 	offload_work->offload_wq = offload_wq;
 	offload_work->adev = adev;
-	offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST = 1;
 	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
 
-	dm_handle_hpd_rx_offload_work(&offload_work->work);
+	*link_srv_out = link_srv;
 
-	KUNIT_EXPECT_EQ(test, dm_test_automated_test_count, 1);
+	return offload_work;
 }
 
 /**
- * dm_test_hpd_rx_offload_work_link_loss - Test link-loss branch
+ * dm_test_hpd_rx_offload_work_detect_type_fails - Test detect failure logging
  * @test: The KUnit test context
  *
- * With a present non-eDP connection and no service-IRQ bits set, the worker
- * takes the else-if link-loss path: dc_link_check_link_loss_status() and
- * dc_link_dp_allow_hpd_rx_irq() gate entry, then dc_link_dp_read_hpd_rx_irq_data()
- * returns DC_OK and a second link-loss check triggers dc_link_dp_handle_link_loss().
- * All four are stubbed via link_srv.
+ * When dc_link_detect_connection_type() fails the worker logs an error and
+ * carries on with the connection type left as none, so it still skips out
+ * before any DP IRQ handling.
  */
-static void dm_test_hpd_rx_offload_work_link_loss(struct kunit *test)
+static void dm_test_hpd_rx_offload_work_detect_type_fails(struct kunit *test)
 {
-	struct hpd_rx_irq_offload_work_queue *offload_wq;
 	struct hpd_rx_irq_offload_work *offload_work;
-	struct amdgpu_dm_connector *aconn;
 	struct link_service *link_srv;
-	struct amdgpu_device *adev;
-	struct dc_context *ctx;
-	struct dc_link *link;
-	struct dc *dc;
 
-	dm_test_handle_link_loss_count = 0;
+	dm_test_automated_test_count = 0;
 
-	adev = dm_kunit_alloc_adev(test);
-	mutex_init(&adev->dm.dc_lock);
-	adev->reset_domain = kunit_kzalloc(test, sizeof(*adev->reset_domain),
-					   GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev->reset_domain);
+	offload_work = dm_test_setup_hpd_rx_offload_work(test, &link_srv);
+	link_srv->detect_connection_type = dm_test_detect_connection_fail;
+	link_srv->dp_handle_automated_test = dm_test_dp_handle_automated_test;
+	offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST = 1;
 
-	offload_wq = kunit_kzalloc(test, sizeof(*offload_wq), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_wq);
-	spin_lock_init(&offload_wq->offload_lock);
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
 
-	aconn = dm_kunit_alloc_connector(test, adev, NULL);
-	mutex_init(&aconn->hpd_lock);
-	offload_wq->aconnector = aconn;
+	KUNIT_EXPECT_EQ(test, dm_test_automated_test_count, 0);
+}
 
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
-	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
-	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
-	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+/**
+ * dm_test_hpd_rx_offload_work_in_reset - Test the GPU-reset early exit
+ * @test: The KUnit test context
+ *
+ * A GPU reset owns the link, so a deferred HPD RX IRQ must be dropped before
+ * any DP IRQ handling runs.
+ */
+static void dm_test_hpd_rx_offload_work_in_reset(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct link_service *link_srv;
 
-	link_srv->detect_connection_type = dm_test_detect_connection_single;
-	link_srv->dp_parse_link_loss_status = dm_test_dp_parse_link_loss_true;
-	link_srv->dp_should_allow_hpd_rx_irq = dm_test_dp_should_allow_hpd_rx_irq_true;
-	link_srv->dp_read_hpd_rx_irq_data = dm_test_dp_read_hpd_rx_irq_data_ok;
-	link_srv->dp_handle_link_loss = dm_test_dp_handle_link_loss;
-	dc->link_srv = link_srv;
-	dc->ctx = ctx;
-	ctx->dc = dc;
-	link->dc = dc;
-	link->ctx = ctx;
-	link->connector_signal = SIGNAL_TYPE_DISPLAY_PORT;
-	aconn->dc_link = link;
+	dm_test_automated_test_count = 0;
 
-	offload_work = kzalloc_obj(*offload_work);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, offload_work);
-	offload_work->offload_wq = offload_wq;
-	offload_work->adev = adev;
-	INIT_WORK(&offload_work->work, dm_handle_hpd_rx_offload_work);
+	offload_work = dm_test_setup_hpd_rx_offload_work(test, &link_srv);
+	link_srv->dp_handle_automated_test = dm_test_dp_handle_automated_test;
+	offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST = 1;
+	atomic_set(&offload_work->adev->reset_domain->in_gpu_reset, 1);
 
 	dm_handle_hpd_rx_offload_work(&offload_work->work);
 
-	KUNIT_EXPECT_EQ(test, dm_test_handle_link_loss_count, 1);
+	KUNIT_EXPECT_EQ(test, dm_test_automated_test_count, 0);
 }
 
-/* Tests for amdgpu_dm_hdmi_hpd_debounce_work() */
-
 /**
- * dm_test_hdmi_hpd_debounce_detect_false - Test debounce false detect path
+ * dm_test_hpd_rx_offload_work_msg_rdy - Test the MST message-ready branch
  * @test: The KUnit test context
+ *
+ * A deferred UP_REQ_MSG_RDY drains the sideband messages and then re-opens the
+ * queue for the next MST message-ready event. The AUX transfers all fail, so
+ * the ESI read loop bails out on its first iteration.
  */
-static void dm_test_hdmi_hpd_debounce_detect_false(struct kunit *test)
+static void dm_test_hpd_rx_offload_work_msg_rdy(struct kunit *test)
 {
+	struct hpd_rx_irq_offload_work *offload_work;
 	struct amdgpu_dm_connector *aconn;
 	struct link_service *link_srv;
-	struct amdgpu_device *adev;
-	struct dc_context *ctx;
 	struct dc_link *link;
-	struct dc *dc;
 
-	adev = dm_kunit_alloc_adev(test);
-	mutex_init(&adev->dm.dc_lock);
+	offload_work = dm_test_setup_hpd_rx_offload_work(test, &link_srv);
+	aconn = offload_work->offload_wq->aconnector;
+	link = aconn->dc_link;
 
-	aconn = dm_kunit_alloc_connector(test, adev, NULL);
-	mutex_init(&aconn->hpd_lock);
-	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
-			  amdgpu_dm_hdmi_hpd_debounce_work);
+	link_srv->get_status = dm_test_get_link_status;
+	link->link_status.dpcd_caps = &link->dpcd_caps;
 
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
-	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
-	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
-	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+	mutex_init(&aconn->handle_mst_msg_ready);
+	aconn->dm_dp_aux.aux.name = "dm_irq_test_msg_rdy_aux";
+	aconn->dm_dp_aux.aux.transfer = dm_test_aux_transfer_fail;
+	drm_dp_aux_init(&aconn->dm_dp_aux.aux);
+	drm_dp_dpcd_set_probe(&aconn->dm_dp_aux.aux, false);
 
-	link_srv->detect_link = dm_test_detect_link_false;
-	dc->ctx = ctx;
-	dc->link_srv = link_srv;
-	ctx->dc = dc;
-	link->dc = dc;
-	link->ctx = ctx;
-	aconn->dc_link = link;
+	offload_work->offload_wq->is_handling_mst_msg_rdy_event = true;
+	offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY = 1;
 
-	amdgpu_dm_hdmi_hpd_debounce_work(&aconn->hdmi_hpd_debounce_work.work);
-	KUNIT_EXPECT_NULL(test, aconn->hdmi_prev_sink);
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+
+	KUNIT_EXPECT_FALSE(test, offload_work->offload_wq->is_handling_mst_msg_rdy_event);
 }
 
 /**
- * dm_test_hdmi_hpd_debounce_reallow_idle - Test debounce idle/sink-release tail
+ * dm_test_hpd_rx_offload_work_automated_test - Test AUTOMATED_TEST branch
  * @test: The KUnit test context
  *
- * With detection stubbed to return false, the if (ret) block is skipped, but
- * the function tail is still exercised: IPS support plus an idle-allowed dmub
- * makes reallow_idle true so dc_allow_idle_optimizations() runs on both entry
- * and exit (disable_idle_power_optimizations keeps those safe early returns),
- * and a cached hdmi_prev_sink is released and cleared.
+ * With a present connection and the AUTOMATED_TEST service-IRQ bit set, the
+ * worker runs dc_link_dp_handle_automated_test() (stubbed via link_srv) and
+ * writes the test response with core_link_write_dpcd(). timing_changed is left
+ * false so force_connector_state() (which needs a registered DRM device) is
+ * skipped, and aux_access_disabled makes the DPCD write a safe no-op.
  */
-static void dm_test_hdmi_hpd_debounce_reallow_idle(struct kunit *test)
+static void dm_test_hpd_rx_offload_work_automated_test(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct link_service *link_srv;
+
+	dm_test_automated_test_count = 0;
+
+	offload_work = dm_test_setup_hpd_rx_offload_work(test, &link_srv);
+	link_srv->dp_handle_automated_test = dm_test_dp_handle_automated_test;
+	offload_work->offload_wq->aconnector->dc_link->aux_access_disabled = true;
+	offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST = 1;
+
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+
+	KUNIT_EXPECT_EQ(test, dm_test_automated_test_count, 1);
+}
+
+/**
+ * dm_test_hpd_rx_offload_work_timing_changed - Test the forced reconnect path
+ * @test: The KUnit test context
+ *
+ * An automated test that changed the timing must force the connector off and
+ * back on again through force_connector_state(), which sends a hotplug uevent
+ * and therefore needs a registered DRM device.
+ */
+static void dm_test_hpd_rx_offload_work_timing_changed(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	offload_work = dm_test_setup_hpd_rx_offload_work(test, &link_srv);
+	aconn = offload_work->offload_wq->aconnector;
+	dm_test_register_drm_dev(test, offload_work->adev);
+
+	link_srv->dp_handle_automated_test = dm_test_dp_handle_automated_test;
+	aconn->timing_changed = true;
+	aconn->dc_link->aux_access_disabled = true;
+	offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST = 1;
+
+	/* The forced off/on cycle must leave the connector back at its default. */
+	aconn->base.force = DRM_FORCE_ON;
+
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+
+	KUNIT_EXPECT_EQ(test, aconn->base.force, DRM_FORCE_UNSPECIFIED);
+}
+
+/**
+ * dm_test_hpd_rx_offload_work_link_loss - Test link-loss branch
+ * @test: The KUnit test context
+ *
+ * With a present non-eDP connection and no service-IRQ bits set, the worker
+ * takes the else-if link-loss path: dc_link_check_link_loss_status() and
+ * dc_link_dp_allow_hpd_rx_irq() gate entry, then dc_link_dp_read_hpd_rx_irq_data()
+ * returns DC_OK and a second link-loss check triggers dc_link_dp_handle_link_loss().
+ * All four are stubbed via link_srv.
+ */
+static void dm_test_hpd_rx_offload_work_link_loss(struct kunit *test)
+{
+	struct hpd_rx_irq_offload_work *offload_work;
+	struct link_service *link_srv;
+
+	dm_test_handle_link_loss_count = 0;
+
+	offload_work = dm_test_setup_hpd_rx_offload_work(test, &link_srv);
+	link_srv->dp_parse_link_loss_status = dm_test_dp_parse_link_loss_true;
+	link_srv->dp_should_allow_hpd_rx_irq = dm_test_dp_should_allow_hpd_rx_irq_true;
+	link_srv->dp_read_hpd_rx_irq_data = dm_test_dp_read_hpd_rx_irq_data_ok;
+	link_srv->dp_handle_link_loss = dm_test_dp_handle_link_loss;
+
+	dm_handle_hpd_rx_offload_work(&offload_work->work);
+
+	KUNIT_EXPECT_EQ(test, dm_test_handle_link_loss_count, 1);
+}
+
+/* Tests for amdgpu_dm_hdmi_hpd_debounce_work() */
+
+/*
+ * Build an aconnector wired for amdgpu_dm_hdmi_hpd_debounce_work(): an HPD
+ * lock, the debounce work item and a dc/link pair whose re-detect fails.
+ * Caller sets the link_srv stubs and any sinks the branch under test needs.
+ */
+static struct amdgpu_dm_connector *dm_test_setup_hdmi_debounce(struct kunit *test,
+							       struct link_service **link_srv_out)
 {
 	struct amdgpu_dm_connector *aconn;
 	struct link_service *link_srv;
-	struct dc_dmub_srv *dmub_srv;
 	struct amdgpu_device *adev;
-	struct dal_logger *logger;
 	struct dc_context *ctx;
 	struct dc_link *link;
 	struct dc *dc;
@@ -2407,10 +2508,6 @@ static void dm_test_hdmi_hpd_debounce_reallow_idle(struct kunit *test)
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
 	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
-	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, logger);
-	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub_srv);
 
 	link_srv->detect_link = dm_test_detect_link_false;
 	dc->ctx = ctx;
@@ -2420,8 +2517,54 @@ static void dm_test_hdmi_hpd_debounce_reallow_idle(struct kunit *test)
 	link->ctx = ctx;
 	aconn->dc_link = link;
 
+	*link_srv_out = link_srv;
+
+	return aconn;
+}
+
+/**
+ * dm_test_hdmi_hpd_debounce_detect_false - Test debounce false detect path
+ * @test: The KUnit test context
+ */
+static void dm_test_hdmi_hpd_debounce_detect_false(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hdmi_debounce(test, &link_srv);
+
+	amdgpu_dm_hdmi_hpd_debounce_work(&aconn->hdmi_hpd_debounce_work.work);
+	KUNIT_EXPECT_NULL(test, aconn->hdmi_prev_sink);
+}
+
+/**
+ * dm_test_hdmi_hpd_debounce_reallow_idle - Test debounce idle/sink-release tail
+ * @test: The KUnit test context
+ *
+ * With detection stubbed to return false, the if (ret) block is skipped, but
+ * the function tail is still exercised: IPS support plus an idle-allowed dmub
+ * makes reallow_idle true so dc_allow_idle_optimizations() runs on both entry
+ * and exit (disable_idle_power_optimizations keeps those safe early returns),
+ * and a cached hdmi_prev_sink is released and cleared.
+ */
+static void dm_test_hdmi_hpd_debounce_reallow_idle(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct dc_dmub_srv *dmub_srv;
+	struct dal_logger *logger;
+	struct dc *dc;
+
+	aconn = dm_test_setup_hdmi_debounce(test, &link_srv);
+	dc = aconn->dc_link->ctx->dc;
+
+	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, logger);
+	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub_srv);
+
 	/* DC_LOG_* dereferences ctx->logger->dev, so wire a real drm device. */
-	logger->dev = &adev->ddev;
+	logger->dev = aconn->base.dev;
 	dc->ctx->logger = logger;
 
 	/* Make reallow_idle true and keep dc_allow_idle*() a safe early return. */
@@ -2431,10 +2574,71 @@ static void dm_test_hdmi_hpd_debounce_reallow_idle(struct kunit *test)
 	dc->debug.disable_idle_power_optimizations = true;
 
 	/* A cached previous sink must be released and cleared. */
+	aconn->hdmi_prev_sink = dm_test_sink_create(aconn->dc_link);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn->hdmi_prev_sink);
+
+	amdgpu_dm_hdmi_hpd_debounce_work(&aconn->hdmi_hpd_debounce_work.work);
+	KUNIT_EXPECT_NULL(test, aconn->hdmi_prev_sink);
+}
+
+/**
+ * dm_test_hdmi_hpd_debounce_sink_unchanged - Test the spontaneous-toggle path
+ * @test: The KUnit test context
+ *
+ * A re-detect that finds the same sink as before the HPD toggle is an internal
+ * re-enable, so the connector state is refreshed but no hotplug event is sent
+ * to userspace. An MST-state connector keeps
+ * amdgpu_dm_update_connector_after_detect() to its early return.
+ */
+static void dm_test_hdmi_hpd_debounce_sink_unchanged(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct dc_link *link;
+
+	aconn = dm_test_setup_hdmi_debounce(test, &link_srv);
+	link_srv->detect_link = dm_test_detect_link_true;
+	aconn->mst_mgr.mst_state = true;
+	link = aconn->dc_link;
+
+	/* Two identically zeroed sinks compare equal, i.e. a spurious toggle. */
+	link->local_sink = dm_test_sink_create(link);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link->local_sink);
 	aconn->hdmi_prev_sink = dm_test_sink_create(link);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn->hdmi_prev_sink);
 
 	amdgpu_dm_hdmi_hpd_debounce_work(&aconn->hdmi_hpd_debounce_work.work);
+
+	KUNIT_EXPECT_NULL(test, aconn->hdmi_prev_sink);
+
+	dm_test_sink_release(link->local_sink);
+}
+
+/**
+ * dm_test_hdmi_hpd_debounce_sink_changed - Test the real-hotplug path
+ * @test: The KUnit test context
+ *
+ * A re-detect that finds a different sink than the one cached before the HPD
+ * toggle is a real hotplug, so userspace must be notified. Sending the uevent
+ * needs a registered DRM device.
+ */
+static void dm_test_hdmi_hpd_debounce_sink_changed(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hdmi_debounce(test, &link_srv);
+	dm_test_register_drm_dev(test, drm_to_adev(aconn->base.dev));
+
+	link_srv->detect_link = dm_test_detect_link_true;
+	aconn->mst_mgr.mst_state = true;
+
+	/* The link lost its sink, so it cannot match the cached one. */
+	aconn->hdmi_prev_sink = dm_test_sink_create(aconn->dc_link);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn->hdmi_prev_sink);
+
+	amdgpu_dm_hdmi_hpd_debounce_work(&aconn->hdmi_hpd_debounce_work.work);
+
 	KUNIT_EXPECT_NULL(test, aconn->hdmi_prev_sink);
 }
 
@@ -2481,16 +2685,14 @@ static void dm_test_handle_hpd_irq_disabled(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, aconn->fake_enable);
 }
 
-/**
- * dm_test_handle_hpd_irq_helper_debounce_schedule - Test HDMI debounce branch
- * @test: The KUnit test context
- *
- * An HDMI link reporting a disconnect (connection type none) with a non-zero
- * debounce delay and a cached local_sink must take the debounce branch: it
- * caches local_sink in hdmi_prev_sink and schedules the delayed debounce work
- * instead of detecting immediately.
+/*
+ * Build an aconnector wired for handle_hpd_irq_helper(): an HPD lock, an armed
+ * debounce work item, an atomic state and a dc/link pair reporting no
+ * connection. Caller sets the link_srv stubs and any HDMI/debounce fields the
+ * branch under test needs.
  */
-static void dm_test_handle_hpd_irq_helper_debounce_schedule(struct kunit *test)
+static struct amdgpu_dm_connector *dm_test_setup_hpd_irq_helper(struct kunit *test,
+								struct link_service **link_srv_out)
 {
 	struct amdgpu_dm_connector *aconn;
 	struct dm_connector_state *state;
@@ -2501,6 +2703,7 @@ static void dm_test_handle_hpd_irq_helper_debounce_schedule(struct kunit *test)
 	struct dc *dc;
 
 	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
 
 	aconn = dm_kunit_alloc_connector(test, adev, NULL);
 	mutex_init(&aconn->hpd_lock);
@@ -2528,6 +2731,29 @@ static void dm_test_handle_hpd_irq_helper_debounce_schedule(struct kunit *test)
 	link->ctx = ctx;
 	aconn->dc_link = link;
 
+	*link_srv_out = link_srv;
+
+	return aconn;
+}
+
+/**
+ * dm_test_handle_hpd_irq_helper_debounce_schedule - Test HDMI debounce branch
+ * @test: The KUnit test context
+ *
+ * An HDMI link reporting a disconnect (connection type none) with a non-zero
+ * debounce delay and a cached local_sink must take the debounce branch: it
+ * caches local_sink in hdmi_prev_sink and schedules the delayed debounce work
+ * instead of detecting immediately.
+ */
+static void dm_test_handle_hpd_irq_helper_debounce_schedule(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+	struct dc_link *link;
+
+	aconn = dm_test_setup_hpd_irq_helper(test, &link_srv);
+	link = aconn->dc_link;
+
 	/* HDMI signal + debounce delay + cached sink -> debounce branch. */
 	aconn->hdmi_hpd_debounce_delay_ms = 100;
 	link->connector_signal = SIGNAL_TYPE_HDMI_TYPE_A;
@@ -2551,44 +2777,18 @@ static void dm_test_handle_hpd_irq_helper_debounce_schedule(struct kunit *test)
  * When the debounce branch is taken and a stale hdmi_prev_sink is already
  * cached from a previous HPD, it must be released before caching the current
  * local_sink. This exercises the dc_sink_release() of the previous sink.
+ *
+ * An already-armed debounce work also makes mod_delayed_work() re-schedule
+ * rather than queue.
  */
 static void dm_test_handle_hpd_irq_helper_debounce_release_prev(struct kunit *test)
 {
 	struct amdgpu_dm_connector *aconn;
-	struct dm_connector_state *state;
 	struct link_service *link_srv;
-	struct amdgpu_device *adev;
-	struct dc_context *ctx;
 	struct dc_link *link;
-	struct dc *dc;
-
-	adev = dm_kunit_alloc_adev(test);
-
-	aconn = dm_kunit_alloc_connector(test, adev, NULL);
-	mutex_init(&aconn->hpd_lock);
-	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
-			  amdgpu_dm_hdmi_hpd_debounce_work);
 
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
-	aconn->base.state = &state->base;
-
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
-	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
-	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
-	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
-
-	link_srv->detect_connection_type = dm_test_detect_connection_none;
-	dc->ctx = ctx;
-	dc->link_srv = link_srv;
-	ctx->dc = dc;
-	link->dc = dc;
-	link->ctx = ctx;
-	aconn->dc_link = link;
+	aconn = dm_test_setup_hpd_irq_helper(test, &link_srv);
+	link = aconn->dc_link;
 
 	/* HDMI signal + debounce delay + cached sink -> debounce branch. */
 	aconn->hdmi_hpd_debounce_delay_ms = 100;
@@ -2601,6 +2801,10 @@ static void dm_test_handle_hpd_irq_helper_debounce_release_prev(struct kunit *te
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn->hdmi_prev_sink);
 	KUNIT_ASSERT_PTR_NE(test, aconn->hdmi_prev_sink, link->local_sink);
 
+	KUNIT_ASSERT_TRUE(test,
+			  schedule_delayed_work(&aconn->hdmi_hpd_debounce_work,
+						10 * HZ));
+
 	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
 
 	/* Stale sink replaced by the current local_sink. */
@@ -2612,61 +2816,140 @@ static void dm_test_handle_hpd_irq_helper_debounce_release_prev(struct kunit *te
 }
 
 /**
- * dm_test_handle_hpd_irq_helper_detect_false - Test immediate detect branch
+ * dm_test_handle_hpd_irq_helper_detect_type_fails - Test detect failure logging
  * @test: The KUnit test context
  *
- * With no force, no debounce, and detection stubbed to report no connection,
- * the helper takes the else branch: dc_exit_ips_for_hw_access() is a safe
- * no-op (no IPS support) and dc_link_detect() returns false, so the connected
- * if (ret) block is skipped. fake_enable must still be cleared.
+ * When dc_link_detect_connection_type() fails the helper logs an error and
+ * carries on with the connection type left as none, so with no force and no
+ * debounce it still takes the immediate-detect branch.
  */
-static void dm_test_handle_hpd_irq_helper_detect_false(struct kunit *test)
+static void dm_test_handle_hpd_irq_helper_detect_type_fails(struct kunit *test)
 {
 	struct amdgpu_dm_connector *aconn;
-	struct dm_connector_state *state;
 	struct link_service *link_srv;
-	struct amdgpu_device *adev;
-	struct dc_context *ctx;
-	struct dc_link *link;
-	struct dc *dc;
 
-	adev = dm_kunit_alloc_adev(test);
-	mutex_init(&adev->dm.dc_lock);
+	aconn = dm_test_setup_hpd_irq_helper(test, &link_srv);
+	link_srv->detect_connection_type = dm_test_detect_connection_fail;
+	link_srv->detect_link = dm_test_detect_link_false;
+	aconn->fake_enable = true;
 
-	aconn = dm_kunit_alloc_connector(test, adev, NULL);
-	mutex_init(&aconn->hpd_lock);
-	INIT_DELAYED_WORK(&aconn->hdmi_hpd_debounce_work,
-			  amdgpu_dm_hdmi_hpd_debounce_work);
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
 
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
-	aconn->base.state = &state->base;
+	KUNIT_EXPECT_FALSE(test, aconn->fake_enable);
+}
 
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
-	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
-	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link_srv);
-	link = kunit_kzalloc(test, sizeof(*link), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, link);
+/**
+ * dm_test_handle_hpd_irq_helper_hdcp_reset - Test the HDCP reset branch
+ * @test: The KUnit test context
+ *
+ * An HPD invalidates the link's HDCP authentication, so a driver carrying an
+ * HDCP work queue must reset the display and flag the connector state for a
+ * property update.
+ */
+static void dm_test_handle_hpd_irq_helper_hdcp_reset(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct hdcp_workqueue *hdcp_work;
+	struct link_service *link_srv;
 
-	link_srv->detect_connection_type = dm_test_detect_connection_none;
+	aconn = dm_test_setup_hpd_irq_helper(test, &link_srv);
 	link_srv->detect_link = dm_test_detect_link_false;
-	dc->ctx = ctx;
-	dc->link_srv = link_srv;
-	ctx->dc = dc;
-	link->dc = dc;
-	link->ctx = ctx;
-	aconn->dc_link = link;
+
+	hdcp_work = dm_test_alloc_hdcp_workqueue(test);
+	drm_to_adev(aconn->base.dev)->dm.hdcp_workqueue = hdcp_work;
+
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
+
+	KUNIT_EXPECT_TRUE(test, to_dm_connector_state(aconn->base.state)->update_hdcp);
+
+	dm_test_flush_hdcp_workqueue(hdcp_work);
+}
+
+/**
+ * dm_test_handle_hpd_irq_helper_forced_detect - Test the forced-connector branch
+ * @test: The KUnit test context
+ *
+ * A connector forced on but reporting no connection is emulated rather than
+ * detected, and an HPD RX always notifies userspace. An unset connector signal
+ * keeps amdgpu_dm_emulated_link_detect() to its unsupported-signal early
+ * return.
+ */
+static void dm_test_handle_hpd_irq_helper_forced_detect(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	dm_test_detect_link_count = 0;
+
+	aconn = dm_test_setup_hpd_irq_helper(test, &link_srv);
+	dm_test_register_drm_dev(test, drm_to_adev(aconn->base.dev));
+
+	link_srv->detect_link = dm_test_detect_link_false_count;
+	aconn->base.force = DRM_FORCE_ON;
+	aconn->dc_link->ctx->driver_context = drm_to_adev(aconn->base.dev);
+
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPDRX);
+
+	/* The emulated path replaces detection entirely. */
+	KUNIT_EXPECT_EQ(test, dm_test_detect_link_count, 0);
+}
+
+/**
+ * dm_test_handle_hpd_irq_helper_detect_true - Test the connected detect branch
+ * @test: The KUnit test context
+ *
+ * A successful re-detect refreshes the connector state and notifies userspace.
+ * An MST-state connector keeps amdgpu_dm_update_connector_after_detect() to its
+ * early return.
+ */
+static void dm_test_handle_hpd_irq_helper_detect_true(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hpd_irq_helper(test, &link_srv);
+	dm_test_register_drm_dev(test, drm_to_adev(aconn->base.dev));
+
+	link_srv->detect_connection_type = dm_test_detect_connection_single;
+	link_srv->detect_link = dm_test_detect_link_true;
+	aconn->mst_mgr.mst_state = true;
 	aconn->fake_enable = true;
 
-	/* No debounce delay and no force -> immediate-detect else branch. */
 	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
 
 	KUNIT_EXPECT_FALSE(test, aconn->fake_enable);
 }
 
+/**
+ * dm_test_handle_hpd_irq_helper_debounce_pending - Test pending-debounce exit
+ * @test: The KUnit test context
+ *
+ * A debounce re-detect that is already scheduled owns the connector state, so
+ * an HPD that would otherwise detect immediately must return early and leave
+ * dc_link_detect() untouched.
+ */
+static void dm_test_handle_hpd_irq_helper_debounce_pending(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	dm_test_detect_link_count = 0;
+
+	aconn = dm_test_setup_hpd_irq_helper(test, &link_srv);
+	link_srv->detect_link = dm_test_detect_link_false_count;
+
+	/* Arm the debounce work far enough out that it cannot run here. */
+	KUNIT_ASSERT_TRUE(test,
+			  schedule_delayed_work(&aconn->hdmi_hpd_debounce_work,
+						10 * HZ));
+
+	handle_hpd_irq_helper(aconn, DETECT_REASON_HPD);
+
+	KUNIT_EXPECT_EQ(test, dm_test_detect_link_count, 0);
+
+	cancel_delayed_work_sync(&aconn->hdmi_hpd_debounce_work);
+}
+
 /* Tests for handle_hpd_rx_irq()/schedule_hpd_rx_offload_work() */
 
 /**
@@ -2799,24 +3082,6 @@ static struct amdgpu_dm_connector *dm_test_setup_hpd_rx_irq(struct kunit *test,
 	return aconn;
 }
 
-/**
- * dm_test_handle_hpd_rx_irq_no_left_work - Test HPDRX early out (no left work)
- * @test: The KUnit test context
- *
- * When dc_link_handle_hpd_rx_irq() reports no left-over work, the handler
- * jumps to out and returns without scheduling any offload work.
- */
-static void dm_test_handle_hpd_rx_irq_no_left_work(struct kunit *test)
-{
-	struct amdgpu_dm_connector *aconn;
-	struct link_service *link_srv;
-
-	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
-	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_no_work;
-
-	handle_hpd_rx_irq(aconn);
-}
-
 /**
  * dm_test_handle_hpd_rx_irq_automated_test - Test HPDRX automated-test path
  * @test: The KUnit test context
@@ -2881,6 +3146,123 @@ static void dm_test_handle_hpd_rx_irq_link_loss(struct kunit *test)
 				->dm.hpd_rx_offload_wq->is_handling_link_loss);
 }
 
+/**
+ * dm_test_handle_hpd_rx_irq_detect_type_fails - Test HPDRX detect failure logging
+ * @test: The KUnit test context
+ *
+ * A handled HPD RX IRQ on a connector that is not an MST root means a
+ * downstream port status change, so the handler must re-detect the link. The
+ * connection-type probe is stubbed to fail, so it logs an error and continues
+ * into the re-detect branch, and a non-MST-branch link also exercises the
+ * trailing CEC IRQ dispatch.
+ */
+static void dm_test_handle_hpd_rx_irq_detect_type_fails(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	dm_test_detect_link_count = 0;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_result_true;
+	link_srv->detect_connection_type = dm_test_detect_connection_fail;
+	link_srv->detect_link = dm_test_detect_link_false_count;
+
+	aconn->mst_mgr.mst_state = false;
+	aconn->dc_link->type = dc_connection_single;
+
+	handle_hpd_rx_irq(aconn);
+
+	KUNIT_EXPECT_EQ(test, dm_test_detect_link_count, 1);
+}
+
+/**
+ * dm_test_handle_hpd_rx_irq_forced_detect - Test HPDRX forced-connector branch
+ * @test: The KUnit test context
+ *
+ * A connector forced on but reporting no connection is emulated rather than
+ * detected, and userspace is always notified. An unset connector signal keeps
+ * amdgpu_dm_emulated_link_detect() to its unsupported-signal early return, and
+ * a sink-less link keeps amdgpu_dm_update_connector_after_detect() to its
+ * unchanged-sink early return.
+ */
+static void dm_test_handle_hpd_rx_irq_forced_detect(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	dm_test_detect_link_count = 0;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	dm_test_register_drm_dev(test, drm_to_adev(aconn->base.dev));
+
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_result_true;
+	link_srv->detect_connection_type = dm_test_detect_connection_none;
+	link_srv->detect_link = dm_test_detect_link_false_count;
+
+	aconn->mst_mgr.mst_state = false;
+	aconn->base.force = DRM_FORCE_ON;
+	aconn->dc_link->ctx->driver_context = drm_to_adev(aconn->base.dev);
+	aconn->fake_enable = true;
+
+	handle_hpd_rx_irq(aconn);
+
+	KUNIT_EXPECT_FALSE(test, aconn->fake_enable);
+	KUNIT_EXPECT_EQ(test, dm_test_detect_link_count, 0);
+}
+
+/**
+ * dm_test_handle_hpd_rx_irq_detect_true - Test HPDRX connected detect branch
+ * @test: The KUnit test context
+ *
+ * A downstream-port change that re-detects a display refreshes the connector
+ * and notifies userspace. A sink-less link keeps
+ * amdgpu_dm_update_connector_after_detect() to its unchanged-sink early return.
+ */
+static void dm_test_handle_hpd_rx_irq_detect_true(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	dm_test_register_drm_dev(test, drm_to_adev(aconn->base.dev));
+
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_result_true;
+	link_srv->detect_connection_type = dm_test_detect_connection_single;
+	link_srv->detect_link = dm_test_detect_link_true;
+
+	aconn->mst_mgr.mst_state = false;
+	aconn->fake_enable = true;
+
+	handle_hpd_rx_irq(aconn);
+
+	KUNIT_EXPECT_FALSE(test, aconn->fake_enable);
+}
+
+/**
+ * dm_test_handle_hpd_rx_irq_cp_irq - Test HPDRX content-protection dispatch
+ * @test: The KUnit test context
+ *
+ * A CP_IRQ in the device service field must be handed to the HDCP work queue
+ * for the connector that raised it.
+ */
+static void dm_test_handle_hpd_rx_irq_cp_irq(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconn;
+	struct hdcp_workqueue *hdcp_work;
+	struct link_service *link_srv;
+
+	aconn = dm_test_setup_hpd_rx_irq(test, &link_srv);
+	link_srv->dp_handle_hpd_rx_irq = dm_test_handle_hpd_rx_cp_irq;
+
+	hdcp_work = dm_test_alloc_hdcp_workqueue(test);
+	drm_to_adev(aconn->base.dev)->dm.hdcp_workqueue = hdcp_work;
+
+	handle_hpd_rx_irq(aconn);
+
+	dm_test_flush_hdcp_workqueue(hdcp_work);
+}
+
 /* Tests for dmub_hpd_callback()/dmub_hpd_sense_callback() */
 
 /**
@@ -2938,6 +3320,31 @@ static void dm_test_dmub_hpd_callback_empty_connectors(struct kunit *test)
 	dmub_hpd_sense_callback(adev, &notify);
 }
 
+/**
+ * dm_test_dmub_hpd_callback_suspend_skip - Test DMUB HPD skipped in suspend
+ * @test: The KUnit test context
+ *
+ * Plain HPD notifications are deferred while the device is suspended; they get
+ * re-probed on resume, so the callback must return before the connector walk.
+ */
+static void dm_test_dmub_hpd_callback_suspend_skip(struct kunit *test)
+{
+	struct dmub_notification notify = { 0 };
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
+	adev->dm.dc = dc;
+	adev->in_suspend = true;
+	dc->link_count = 1;
+	notify.type = DMUB_NOTIFICATION_HPD;
+
+	/* adev->dm.ddev is left NULL: reaching the connector walk would crash. */
+	dmub_hpd_callback(adev, &notify);
+}
+
 /**
  * dm_test_dmub_hpd_callback_unknown_type_match - Test DMUB callback matched
  *						   connector, unknown type
@@ -3145,10 +3552,15 @@ static void dm_test_register_hpd_handlers_empty(struct kunit *test)
 /**
  * dm_test_register_hpd_handlers_valid - Test HPD/HPDRX registration succeeds
  * @test: The KUnit test context
+ *
+ * A writeback connector must be skipped by the per-connector loop, while a
+ * DisplayPort connector carrying valid HPD and HPD RX sources registers a
+ * low-context handler for each.
  */
 static void dm_test_register_hpd_handlers_valid(struct kunit *test)
 {
 	struct amdgpu_dm_connector *aconn;
+	struct drm_connector *wbconn;
 	struct amdgpu_device *adev;
 	struct dc_link *link;
 	struct dc *dc;
@@ -3159,6 +3571,13 @@ static void dm_test_register_hpd_handlers_valid(struct kunit *test)
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc);
 	adev->dm.dc = dc;
 
+	wbconn = kunit_kzalloc(test, sizeof(*wbconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, wbconn);
+	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, wbconn, &dm_test_connector_funcs,
+						 DRM_MODE_CONNECTOR_WRITEBACK), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_connector_cleanup, wbconn), 0);
+
 	aconn = kunit_kzalloc(test, sizeof(*aconn), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, aconn);
 	KUNIT_ASSERT_EQ(test, drm_connector_init(&adev->ddev, &aconn->base,
@@ -3315,16 +3734,8 @@ static void dm_test_pflip_high_irq_not_submitted(struct kunit *test)
 	struct amdgpu_device *adev;
 
 	adev = dm_kunit_alloc_adev(test);
-	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
-
-	INIT_LIST_HEAD(&acrtc->base.head);
-	acrtc->base.dev = &adev->ddev;
-	acrtc->otg_inst = 0;
+	acrtc = dm_test_add_crtc(test, adev);
 	acrtc->pflip_status = AMDGPU_FLIP_NONE;
-	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
-	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
-				dm_test_crtc_list_del, acrtc), 0);
 
 	params.adev = adev;
 	params.irq_src = (enum dc_irq_source)IRQ_TYPE_PFLIP;
@@ -3333,6 +3744,163 @@ static void dm_test_pflip_high_irq_not_submitted(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
 }
 
+/**
+ * dm_test_pflip_high_irq_completes_flip - Test pflip high IRQ completes a flip
+ * @test: The KUnit test context
+ *
+ * With a submitted flip and an armed event on a non-VRR CRTC, the handler must
+ * send the pageflip completion event, drop the vblank reference taken when the
+ * flip was queued and return the CRTC to AMDGPU_FLIP_NONE.
+ */
+static void dm_test_pflip_high_irq_completes_flip(struct kunit *test)
+{
+	struct drm_pending_vblank_event *event;
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+
+	acrtc = dm_test_add_crtc(test, adev);
+	acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
+
+	/* drm_crtc_send_vblank_event() consumes (kfree()s) the event. */
+	event = kzalloc_obj(*event, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, event);
+	acrtc->event = event;
+
+	/* Balance the handler's drm_crtc_vblank_put(). */
+	adev->ddev.vblank[0].enabled = true;
+	KUNIT_ASSERT_EQ(test, drm_crtc_vblank_get(&acrtc->base), 0);
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_PFLIP;
+
+	dm_pflip_high_irq(&params);
+
+	KUNIT_EXPECT_NULL(test, acrtc->event);
+	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
+}
+
+/*
+ * Build a CRTC with a submitted flip on an active-VRR stream whose scanout
+ * position comes from the given fake timing generator (NULL for a stream that
+ * is not on any pipe, so the position is unavailable).
+ */
+static struct amdgpu_crtc *dm_test_setup_vrr_pflip_crtc(struct kunit *test,
+							struct timing_generator *tg)
+{
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	dc->current_state = kunit_kzalloc(test, sizeof(*dc->current_state),
+					  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc->current_state);
+	adev->dm.dc = dc;
+
+	link = dm_kunit_alloc_link(test);
+	stream = dm_kunit_alloc_stream(test, link);
+	stream->ctx = dc->ctx;
+
+	if (tg) {
+		dc->current_state->res_ctx.pipe_ctx[0].stream = stream;
+		dc->current_state->res_ctx.pipe_ctx[0].stream_res.tg = tg;
+	}
+
+	acrtc = dm_test_add_crtc(test, adev);
+	acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
+	acrtc->dm_irq_params.stream = stream;
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+
+	return acrtc;
+}
+
+/**
+ * dm_test_pflip_high_irq_vrr_no_scanoutpos - Test VRR flip with no scanout data
+ * @test: The KUnit test context
+ *
+ * When the stream is not on any pipe the scanout position is unavailable, so
+ * the handler cannot tell it is inside the front porch and completes the flip
+ * immediately.
+ */
+static void dm_test_pflip_high_irq_vrr_no_scanoutpos(struct kunit *test)
+{
+	struct drm_pending_vblank_event *event;
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = dm_test_setup_vrr_pflip_crtc(test, NULL);
+	adev = drm_to_adev(acrtc->base.dev);
+
+	/* drm_crtc_send_vblank_event() consumes (kfree()s) the event. */
+	event = kzalloc_obj(*event, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, event);
+	acrtc->event = event;
+
+	/* Balance the handler's drm_crtc_vblank_put(). */
+	adev->ddev.vblank[0].enabled = true;
+	KUNIT_ASSERT_EQ(test, drm_crtc_vblank_get(&acrtc->base), 0);
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_PFLIP;
+
+	dm_pflip_high_irq(&params);
+
+	KUNIT_EXPECT_NULL(test, acrtc->event);
+	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
+}
+
+/**
+ * dm_test_pflip_high_irq_vrr_front_porch - Test VRR flip inside the front porch
+ * @test: The KUnit test context
+ *
+ * A VRR flip that completes while scanout is still in the front porch has no
+ * valid vblank count yet, so the event is queued on the device's vblank event
+ * list for the late vblank handler to send out instead.
+ */
+static void dm_test_pflip_high_irq_vrr_front_porch(struct kunit *test)
+{
+	struct drm_pending_vblank_event *event;
+	struct common_irq_params params = { 0 };
+	struct timing_generator *tg;
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	tg = kunit_kzalloc(test, sizeof(*tg), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, tg);
+	tg->funcs = &dm_test_tg_funcs;
+
+	acrtc = dm_test_setup_vrr_pflip_crtc(test, tg);
+	adev = drm_to_adev(acrtc->base.dev);
+
+	event = kzalloc_obj(*event, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, event);
+	acrtc->event = event;
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_PFLIP;
+
+	dm_pflip_high_irq(&params);
+
+	KUNIT_EXPECT_NULL(test, acrtc->event);
+	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
+	KUNIT_ASSERT_PTR_EQ(test, adev->ddev.vblank_event_list.next,
+			    &event->base.link);
+
+	/* The event was queued rather than sent, so the test owns it. */
+	list_del(&event->base.link);
+	kfree(event);
+}
+
 /**
  * dm_test_vupdate_high_irq_no_crtc - Test vupdate high IRQ with no CRTC
  * @test: The KUnit test context
@@ -3349,6 +3917,158 @@ static void dm_test_vupdate_high_irq_no_crtc(struct kunit *test)
 	dm_vupdate_high_irq(&params);
 }
 
+/**
+ * dm_test_vupdate_high_irq_dcn_completes_flip - Test DCN vupdate flip delivery
+ * @test: The KUnit test context
+ *
+ * A non-zero DCE IP version routes vupdate to the shared CRTC handler. With a
+ * submitted flip, an armed event and no DC (so no flip is reported pending),
+ * the handler must deliver the event and clear the flip state. A non-zero last
+ * vblank timestamp also exercises the refresh-rate tracepoint.
+ */
+static void dm_test_vupdate_high_irq_dcn_completes_flip(struct kunit *test)
+{
+	struct drm_pending_vblank_event *event;
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 0, 0);
+	/* AI+ family so the handler does not return before the flip block. */
+	adev->family = AMDGPU_FAMILY_AI;
+
+	acrtc = dm_test_add_crtc(test, adev);
+	acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
+
+	event = kzalloc_obj(*event, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, event);
+	acrtc->event = event;
+
+	adev->ddev.vblank[0].time = ktime_get();
+	adev->ddev.vblank[0].enabled = true;
+	KUNIT_ASSERT_EQ(test, drm_crtc_vblank_get(&acrtc->base), 0);
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VUPDATE;
+
+	dm_vupdate_high_irq(&params);
+
+	KUNIT_EXPECT_NULL(test, acrtc->event);
+	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
+}
+
+/**
+ * dm_test_vupdate_high_irq_dcn_no_active_planes - Test DCN flip with no planes
+ * @test: The KUnit test context
+ *
+ * With no active planes the DCN HUBP may be clock-gated and the flip-pending
+ * status is undefined, so a submitted flip with no armed event must simply be
+ * retired rather than waiting on the OTG.
+ */
+static void dm_test_vupdate_high_irq_dcn_no_active_planes(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 0, 0);
+	adev->family = AMDGPU_FAMILY_AI;
+
+	acrtc = dm_test_add_crtc(test, adev);
+	acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED;
+	acrtc->dm_irq_params.active_planes = 0;
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VUPDATE;
+
+	dm_vupdate_high_irq(&params);
+
+	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
+}
+
+/*
+ * Build a CRTC with an active VRR stream on a device whose vmin/vmax updates
+ * are offloaded to adev->dm.vmin_vmax_wq, seeded with the given timing adjust.
+ * Caller sets adev->family to select the DCE or DCN path, then invokes the
+ * handler and flushes the queue.
+ */
+static struct amdgpu_crtc *dm_test_setup_vmin_vmax_crtc(struct kunit *test,
+							unsigned int v_total_min,
+							unsigned int v_total_max)
+{
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dc_link *link;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	mutex_init(&adev->dm.dc_lock);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	dc->current_state = kunit_kzalloc(test, sizeof(*dc->current_state),
+					  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc->current_state);
+	adev->dm.dc = dc;
+
+	link = dm_kunit_alloc_link(test);
+	stream = dm_kunit_alloc_stream(test, link);
+
+	acrtc = dm_test_add_crtc(test, adev);
+	/* VRR active so the !vrr_active vblank handler is skipped. */
+	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
+	acrtc->dm_irq_params.stream = stream;
+	acrtc->dm_irq_params.vrr_params.adjust.v_total_min = v_total_min;
+	acrtc->dm_irq_params.vrr_params.adjust.v_total_max = v_total_max;
+
+	return acrtc;
+}
+
+/**
+ * dm_test_vupdate_high_irq_dce_vrr_btr - Test DCE vupdate BTR handling
+ * @test: The KUnit test context
+ *
+ * On DCE (zero DCE IP version) with VRR active and a pre-DCE12 family, vupdate
+ * runs the below-the-range section: it handles the vblank and offloads the
+ * vmin/vmax update, which the worker then applies to the stream.
+ */
+static void dm_test_vupdate_high_irq_dce_vrr_btr(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = dm_test_setup_vmin_vmax_crtc(test, 2000, 2200);
+	adev = drm_to_adev(acrtc->base.dev);
+	stream = acrtc->dm_irq_params.stream;
+	/* Pre-AI family so the BTR block inside the DCE path runs. */
+	adev->family = AMDGPU_FAMILY_SI;
+
+	adev->ddev.vblank[0].time = ktime_get();
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VUPDATE;
+
+	dm_vupdate_high_irq(&params);
+
+	/* The offloaded worker applies the adjust and drops its stream ref. */
+	flush_workqueue(adev->dm.vmin_vmax_wq);
+	KUNIT_EXPECT_EQ(test, stream->adjust.v_total_min, 2000);
+	KUNIT_EXPECT_EQ(test, stream->adjust.v_total_max, 2200);
+	KUNIT_EXPECT_EQ(test, kref_read(&stream->refcount), 1);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
 /**
  * dm_test_crtc_high_irq_no_crtc - Test crtc high IRQ with no CRTC
  * @test: The KUnit test context
@@ -3380,17 +4100,9 @@ static void dm_test_crtc_high_irq_vrr_pre_ai(struct kunit *test)
 	struct amdgpu_device *adev;
 
 	adev = dm_kunit_alloc_adev(test);
-	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
-
-	INIT_LIST_HEAD(&acrtc->base.head);
-	acrtc->base.dev = &adev->ddev;
-	acrtc->otg_inst = 0;
+	acrtc = dm_test_add_crtc(test, adev);
 	/* VRR active so the !vrr_active vblank handler is skipped. */
 	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
-	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
-	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
-				dm_test_crtc_list_del, acrtc), 0);
 
 	/* Pre-AI family returns right after CRC handling. */
 	adev->family = AMDGPU_FAMILY_SI;
@@ -3417,18 +4129,10 @@ static void dm_test_crtc_high_irq_vrr_ai_no_stream(struct kunit *test)
 	struct amdgpu_device *adev;
 
 	adev = dm_kunit_alloc_adev(test);
-	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, acrtc);
-
-	INIT_LIST_HEAD(&acrtc->base.head);
-	acrtc->base.dev = &adev->ddev;
-	acrtc->otg_inst = 0;
+	acrtc = dm_test_add_crtc(test, adev);
 	acrtc->pflip_status = AMDGPU_FLIP_NONE;
 	/* VRR active so the !vrr_active vblank handler is skipped. */
 	acrtc->dm_irq_params.freesync_config.state = VRR_STATE_ACTIVE_VARIABLE;
-	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
-	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test,
-				dm_test_crtc_list_del, acrtc), 0);
 
 	/* AI+ family runs the freesync section; no stream skips it. */
 	adev->family = AMDGPU_FAMILY_AI;
@@ -3440,6 +4144,135 @@ static void dm_test_crtc_high_irq_vrr_ai_no_stream(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, acrtc->pflip_status, AMDGPU_FLIP_NONE);
 }
 
+/**
+ * dm_test_crtc_high_irq_writeback_disables - Test first writeback vblank
+ * @test: The KUnit test context
+ *
+ * The first vblank after a writeback is armed must disable frame capture so the
+ * hardware cannot overwrite the buffer, and defer signalling completion to the
+ * next vblank by marking wb_frame_done.
+ */
+static void dm_test_crtc_high_irq_writeback_disables(struct kunit *test)
+{
+	struct drm_writeback_connector *wb_conn;
+	struct common_irq_params params = { 0 };
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+
+	/* dc_stream_fc_disable_writeback() dereferences dc->res_pool. */
+	dc = dm_test_alloc_dc_with_irq_service(test,
+					       &dm_test_irq_service_funcs_dcn10);
+	adev->dm.dc = dc;
+	/* Pre-AI family returns right after CRC handling. */
+	adev->family = AMDGPU_FAMILY_SI;
+
+	acrtc = dm_test_add_crtc(test, adev);
+
+	wb_conn = kunit_kzalloc(test, sizeof(*wb_conn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, wb_conn);
+	acrtc->wb_conn = wb_conn;
+	acrtc->wb_pending = true;
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VBLANK;
+
+	dm_crtc_high_irq(&params);
+
+	KUNIT_EXPECT_TRUE(test, acrtc->wb_frame_done);
+	KUNIT_EXPECT_TRUE(test, acrtc->wb_pending);
+}
+
+/**
+ * dm_test_crtc_high_irq_writeback_completes - Test second writeback vblank
+ * @test: The KUnit test context
+ *
+ * Once wb_frame_done is set the DMA has had a full frame period to flush, so
+ * the next vblank must signal the writeback out fence and clear the pending
+ * state.
+ */
+static void dm_test_crtc_high_irq_writeback_completes(struct kunit *test)
+{
+	struct drm_writeback_connector *wb_conn;
+	struct common_irq_params params = { 0 };
+	struct drm_writeback_job *job;
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&adev->ddev, 1), 0);
+	adev->family = AMDGPU_FAMILY_SI;
+
+	acrtc = dm_test_add_crtc(test, adev);
+
+	wb_conn = kunit_kzalloc(test, sizeof(*wb_conn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, wb_conn);
+	spin_lock_init(&wb_conn->job_lock);
+	INIT_LIST_HEAD(&wb_conn->job_queue);
+
+	/*
+	 * The job carries no framebuffer or fence, so the deferred DRM cleanup
+	 * work only kfree()s it; it needs nothing from this test.
+	 */
+	job = kzalloc_obj(*job, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, job);
+	list_add_tail(&job->list_entry, &wb_conn->job_queue);
+
+	acrtc->wb_conn = wb_conn;
+	acrtc->wb_pending = true;
+	acrtc->wb_frame_done = true;
+
+	/* Balance the drm_crtc_vblank_put() done on completion. */
+	adev->ddev.vblank[0].enabled = true;
+	KUNIT_ASSERT_EQ(test, drm_crtc_vblank_get(&acrtc->base), 0);
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VBLANK;
+
+	dm_crtc_high_irq(&params);
+
+	KUNIT_EXPECT_FALSE(test, acrtc->wb_pending);
+	KUNIT_EXPECT_FALSE(test, acrtc->wb_frame_done);
+}
+
+/**
+ * dm_test_crtc_high_irq_schedules_vmin_vmax - Test the freesync BTR update
+ * @test: The KUnit test context
+ *
+ * On an AI+ family with an active VRR stream, the handler must run the freesync
+ * v-update and offload the vmin/vmax programming, which the worker then applies
+ * to the stream.
+ */
+static void dm_test_crtc_high_irq_schedules_vmin_vmax(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct dc_stream_state *stream;
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = dm_test_setup_vmin_vmax_crtc(test, 1000, 1100);
+	adev = drm_to_adev(acrtc->base.dev);
+	stream = acrtc->dm_irq_params.stream;
+	adev->family = AMDGPU_FAMILY_AI;
+	acrtc->dm_irq_params.vrr_params.supported = true;
+
+	params.adev = adev;
+	params.irq_src = (enum dc_irq_source)IRQ_TYPE_VBLANK;
+
+	dm_crtc_high_irq(&params);
+
+	/* The offloaded worker applies the adjust and drops its stream ref. */
+	flush_workqueue(adev->dm.vmin_vmax_wq);
+	KUNIT_EXPECT_EQ(test, stream->adjust.v_total_min, 1000);
+	KUNIT_EXPECT_EQ(test, stream->adjust.v_total_max, 1100);
+	KUNIT_EXPECT_EQ(test, kref_read(&stream->refcount), 1);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
 /* Tests for dm_handle_hpd_work() */
 
 /**
@@ -3464,6 +4297,28 @@ static void dm_test_handle_hpd_work_out_of_range(struct kunit *test)
 	dm_handle_hpd_work(&hpd_work->handle_hpd_work);
 }
 
+/**
+ * dm_test_handle_hpd_work_null_notify - Test HPD work with no notification
+ * @test: The KUnit test context
+ *
+ * A work item carrying no notification must be rejected before dispatch.
+ */
+static void dm_test_handle_hpd_work_null_notify(struct kunit *test)
+{
+	struct dmub_hpd_work *hpd_work;
+	struct amdgpu_device *adev;
+
+	adev = dm_kunit_alloc_adev(test);
+	hpd_work = kzalloc_obj(*hpd_work, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hpd_work);
+	hpd_work->adev = adev;
+	INIT_WORK(&hpd_work->handle_hpd_work, dm_handle_hpd_work);
+
+	/* The handler bails out before its own kfree(), so free it here. */
+	dm_handle_hpd_work(&hpd_work->handle_hpd_work);
+	kfree(hpd_work);
+}
+
 /* Tests for dm_dmub_outbox1_low_irq() */
 
 /**
@@ -3495,6 +4350,46 @@ static void dm_test_dmub_outbox1_low_irq_empty(struct kunit *test)
 	dm_dmub_outbox1_low_irq(&params);
 }
 
+/**
+ * dm_test_dmub_outbox1_low_irq_drains_trace - Test the trace drain loop
+ * @test: The KUnit test context
+ *
+ * A trace ring holding more entries than the handler reads in one pass must be
+ * drained up to the DMUB_TRACE_MAX_READ cap, leaving the read pointer parked
+ * after the last entry the handler consumed.
+ */
+static void dm_test_dmub_outbox1_low_irq_drains_trace(struct kunit *test)
+{
+	struct common_irq_params params = { 0 };
+	struct dc_dmub_srv *dc_dmub_srv;
+	struct amdgpu_device *adev;
+	struct dmub_srv *dmub;
+	struct dc *dc;
+	void *rb;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	dc_dmub_srv = kunit_kzalloc(test, sizeof(*dc_dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dc_dmub_srv);
+	dmub = kunit_kzalloc(test, sizeof(*dmub), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dmub);
+	rb = kunit_kzalloc(test, DM_TEST_OUTBOX0_RB_SIZE, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, rb);
+
+	dmub->outbox0_rb.base_address = rb;
+	dmub->outbox0_rb.capacity = DM_TEST_OUTBOX0_RB_SIZE;
+	dmub->hw_funcs.get_outbox0_wptr = dm_test_dmub_get_outbox0_wptr_full;
+	dc_dmub_srv->dmub = dmub;
+	dc->ctx->dmub_srv = dc_dmub_srv;
+	adev->dm.dc = dc;
+	params.adev = adev;
+	params.irq_src = DC_IRQ_SOURCE_DMCUB_OUTBOX;
+
+	dm_dmub_outbox1_low_irq(&params);
+
+	KUNIT_EXPECT_EQ(test, dmub->outbox0_rb.rptr, DM_TEST_OUTBOX0_TRACE_BYTES);
+}
+
 /*
  * dm_test_alloc_adev_outbox_notify - Build an adev wired for DMUB outbox
  * notification handling.
@@ -3625,14 +4520,14 @@ static void dm_test_dce110_register_irq_handlers_rejects_uninitialized_sources(s
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
 }
 
-/**
- * dm_test_dce110_register_irq_handlers_one_crtc - Test DCE110 with 1 CRTC
- * @test: The KUnit test context
- *
- * Exercises the VBLANK, VUPDATE, PFLIP and HPD for-loop bodies with a
- * fake IRQ service that maps source IDs to DC IRQ sources.
+/*
+ * Build an adev/dc pair with one CRTC, ready for either register_irq_handlers()
+ * entry point, using the given fake IRQ service to map source IDs. A service
+ * that only maps a subset lets a chosen register loop hit its invalid-source
+ * guard, and clearing an amdgpu_irq_src's funcs makes its add_id() fail.
  */
-static void dm_test_dce110_register_irq_handlers_one_crtc(struct kunit *test)
+static struct amdgpu_device *dm_test_setup_irq_regs(struct kunit *test,
+						   const struct irq_service_funcs *funcs)
 {
 	struct amdgpu_device *adev;
 	struct dc *dc;
@@ -3640,14 +4535,31 @@ static void dm_test_dce110_register_irq_handlers_one_crtc(struct kunit *test)
 	adev = dm_kunit_alloc_adev(test);
 	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_irq_sources,
 							 adev), 0);
-	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dce110);
+	dc = dm_test_alloc_dc_with_irq_service(test, funcs);
 	dc->ctx->dce_version = DCE_VERSION_11_0;
+	dc->caps.max_otg_num = 1;
 	adev->dm.dc = dc;
 	adev->mode_info.num_crtc = 1;
 	adev->mode_info.num_hpd = 1;
 	amdgpu_dm_set_irq_funcs(adev);
 	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
 
+	return adev;
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_one_crtc - Test DCE110 with 1 CRTC
+ * @test: The KUnit test context
+ *
+ * Exercises the VBLANK, VUPDATE, PFLIP and HPD for-loop bodies with a
+ * fake IRQ service that maps source IDs to DC IRQ sources.
+ */
+static void dm_test_dce110_register_irq_handlers_one_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_dce110);
+
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), 0);
 
 	/* Verify VBLANK params were populated */
@@ -3668,6 +4580,146 @@ static void dm_test_dce110_register_irq_handlers_one_crtc(struct kunit *test)
 	amdgpu_dm_irq_fini(adev);
 }
 
+/**
+ * dm_test_dce110_register_irq_handlers_soc15_client - Test DCE110 on AI+ ASICs
+ * @test: The KUnit test context
+ *
+ * From Vega on, the DCE interrupts are routed through the SOC15 DCE client
+ * rather than the legacy one, so the IRQ sources must be registered there.
+ */
+static void dm_test_dce110_register_irq_handlers_soc15_client(struct kunit *test)
+{
+	struct amdgpu_irq_src **soc15_sources;
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_dce110);
+	adev->family = AMDGPU_FAMILY_AI;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), 0);
+
+	KUNIT_EXPECT_NULL(test, adev->irq.client[AMDGPU_IRQ_CLIENTID_LEGACY].sources);
+	soc15_sources = adev->irq.client[SOC15_IH_CLIENTID_DCE].sources;
+	KUNIT_ASSERT_NOT_NULL(test, soc15_sources);
+	KUNIT_EXPECT_PTR_EQ(test,
+			    soc15_sources[VISLANDS30_IV_SRCID_D1_VERTICAL_INTERRUPT0],
+			    &adev->crtc_irq);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_crtc_add_id_fails - Test add-id failure
+ * @test: The KUnit test context
+ *
+ * amdgpu_irq_add_id() rejects a source with no funcs table, so registering the
+ * VBLANK interrupt before amdgpu_dm_set_irq_funcs() has run must fail.
+ */
+static void dm_test_dce110_register_irq_handlers_crtc_add_id_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dce110);
+	dc->ctx->dce_version = DCE_VERSION_11_0;
+	adev->dm.dc = dc;
+	adev->mode_info.num_crtc = 1;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_bad_vblank_source - Test VBLANK guard
+ * @test: The KUnit test context
+ *
+ * A source ID that DC does not map to a VBLANK IRQ source must be rejected
+ * rather than used to index the vblank parameter array.
+ */
+static void dm_test_dce110_register_irq_handlers_bad_vblank_source(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_unmapped);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_bad_vupdate_source - Test VUPDATE guard
+ * @test: The KUnit test context
+ *
+ * With VBLANK mapped but VUPDATE unmapped, registration must fail once the
+ * VUPDATE loop sees the invalid source.
+ */
+static void dm_test_dce110_register_irq_handlers_bad_vupdate_source(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_vblank_only);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_bad_pflip_source - Test PFLIP guard
+ * @test: The KUnit test context
+ *
+ * With VBLANK and VUPDATE mapped but PFLIP unmapped, registration must fail
+ * once the pageflip loop sees the invalid source.
+ */
+static void dm_test_dce110_register_irq_handlers_bad_pflip_source(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_no_pflip);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_vupdate_add_id_fails - Test add-id failure
+ * @test: The KUnit test context
+ *
+ * The VBLANK loop succeeds, then a VUPDATE source with no funcs table is
+ * rejected by amdgpu_irq_add_id().
+ */
+static void dm_test_dce110_register_irq_handlers_vupdate_add_id_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_dce110);
+	adev->vupdate_irq.funcs = NULL;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dce110_register_irq_handlers_hpd_add_id_fails - Test HPD add-id failure
+ * @test: The KUnit test context
+ *
+ * All three per-CRTC loops succeed, then the HPD source with no funcs table is
+ * rejected by amdgpu_irq_add_id().
+ */
+static void dm_test_dce110_register_irq_handlers_hpd_add_id_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_dce110);
+	adev->hpd_irq.funcs = NULL;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dce110_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
 /**
  * dm_test_dcn10_register_irq_handlers_zero_crtc - Test DCN10 zero-CRTC registration
  * @test: The KUnit test context
@@ -3702,18 +4754,8 @@ static void dm_test_dcn10_register_irq_handlers_zero_crtc(struct kunit *test)
 static void dm_test_dcn10_register_irq_handlers_one_crtc(struct kunit *test)
 {
 	struct amdgpu_device *adev;
-	struct dc *dc;
 
-	adev = dm_kunit_alloc_adev(test);
-	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_free_irq_sources,
-							 adev), 0);
-	dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_dcn10);
-	adev->dm.dc = dc;
-	adev->mode_info.num_crtc = 1;
-	adev->mode_info.num_hpd = 1;
-	dc->caps.max_otg_num = 1;
-	amdgpu_dm_set_irq_funcs(adev);
-	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_dcn10);
 
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_dcn10_register_irq_handlers(adev), 0);
 
@@ -3731,6 +4773,61 @@ static void dm_test_dcn10_register_irq_handlers_one_crtc(struct kunit *test)
 	amdgpu_dm_irq_fini(adev);
 }
 
+/**
+ * dm_test_dcn10_register_irq_handlers_vupdate_add_id_fails - Test add-id failure
+ * @test: The KUnit test context
+ *
+ * A VUPDATE source with no funcs table must be rejected by amdgpu_irq_add_id().
+ */
+static void dm_test_dcn10_register_irq_handlers_vupdate_add_id_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_dcn10);
+	adev->vupdate_irq.funcs = NULL;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dcn10_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dcn10_register_irq_handlers_bad_vupdate_source - Test VUPDATE guard
+ * @test: The KUnit test context
+ *
+ * A source ID that DC does not map to a VUPDATE IRQ source must be rejected
+ * rather than used to index the vupdate parameter array.
+ */
+static void dm_test_dcn10_register_irq_handlers_bad_vupdate_source(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_unmapped);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dcn10_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
+/**
+ * dm_test_dcn10_register_irq_handlers_hpd_add_id_fails - Test HPD add-id failure
+ * @test: The KUnit test context
+ *
+ * The VUPDATE loop succeeds, then the HPD source with no funcs table is
+ * rejected by amdgpu_irq_add_id().
+ */
+static void dm_test_dcn10_register_irq_handlers_hpd_add_id_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = dm_test_setup_irq_regs(test, &dm_test_irq_service_funcs_dcn10);
+	adev->hpd_irq.funcs = NULL;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_dcn10_register_irq_handlers(adev), -EINVAL);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
 /**
  * dm_test_register_outbox_irq_handlers_without_dmub - Test outbox registration without DMUB
  * @test: The KUnit test context
@@ -3755,6 +4852,25 @@ static void dm_test_register_outbox_irq_handlers_without_dmub(struct kunit *test
 		&adev->dmub_outbox_irq);
 }
 
+/**
+ * dm_test_register_outbox_irq_handlers_add_id_fails - Test outbox add-id failure
+ * @test: The KUnit test context
+ *
+ * Registering the outbox interrupt before amdgpu_dm_set_irq_funcs() has run
+ * leaves the source without a funcs table, so amdgpu_irq_add_id() rejects it.
+ */
+static void dm_test_register_outbox_irq_handlers_add_id_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct dc *dc;
+
+	adev = dm_kunit_alloc_adev(test);
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	adev->dm.dc = dc;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_register_outbox_irq_handlers(adev), -EINVAL);
+}
+
 /**
  * dm_test_register_outbox_irq_handlers_with_dmub - Test outbox registration with DMUB
  * @test: The KUnit test context
@@ -3859,6 +4975,50 @@ static void dm_test_irq_handler_dispatches_work(struct kunit *test)
 	amdgpu_dm_irq_fini(adev);
 }
 
+/**
+ * dm_test_irq_handler_invalid_source - Test the IRQ handler source guard
+ * @test: The KUnit test context
+ *
+ * When dc_interrupt_to_irq_source() cannot map the hardware entry it returns
+ * DC_IRQ_SOURCE_INVALID, which must not be used to index the DM handler
+ * tables. amdgpu_dm_irq_handler() should bail out early without dispatching
+ * to either the high- or low-context handlers.
+ */
+static void dm_test_irq_handler_invalid_source(struct kunit *test)
+{
+	struct dc_interrupt_params int_params = { 0 };
+	struct amdgpu_iv_entry entry = { 0 };
+	struct amdgpu_device *adev;
+	int high_count = 0;
+	int low_count = 0;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev);
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_irq_init(adev), 0);
+
+	/* This stub maps every src_id to DC_IRQ_SOURCE_INVALID. */
+	adev->dm.dc = dm_test_alloc_dc_with_irq_service(test, &dm_test_irq_service_funcs_unmapped);
+
+	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+	int_params.irq_source = DC_IRQ_SOURCE_VBLANK1;
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test,
+		amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_test_irq_handler_count,
+						 &high_count));
+
+	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test,
+		amdgpu_dm_irq_register_interrupt(adev, &int_params, dm_test_irq_handler_count,
+						 &low_count));
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_irq_handler(adev, NULL, &entry), 0);
+
+	flush_workqueue(adev->dm.irq_wq);
+	KUNIT_EXPECT_EQ(test, high_count, 0);
+	KUNIT_EXPECT_EQ(test, low_count, 0);
+
+	amdgpu_dm_irq_fini(adev);
+}
+
 /* Tests for dm_handle_vmin_vmax_update() */
 
 /**
@@ -3927,15 +5087,12 @@ static struct kunit_case amdgpu_dm_irq_tests[] = {
 	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd5),
 	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_hpd6),
 	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_invalid),
-	KUNIT_CASE(dm_test_hpd_to_dal_irq_source_out_of_range),
 	/* are_sinks_equal */
-	KUNIT_CASE(dm_test_are_sinks_equal_both_null),
 	KUNIT_CASE(dm_test_are_sinks_equal_first_null),
 	KUNIT_CASE(dm_test_are_sinks_equal_second_null),
 	KUNIT_CASE(dm_test_are_sinks_equal_different_signal),
 	KUNIT_CASE(dm_test_are_sinks_equal_different_edid_length),
 	KUNIT_CASE(dm_test_are_sinks_equal_different_edid_data),
-	KUNIT_CASE(dm_test_are_sinks_equal_identical),
 	KUNIT_CASE(dm_test_are_sinks_equal_zero_length),
 	KUNIT_CASE(dm_test_are_sinks_equal_full_edid_identical),
 	KUNIT_CASE(dm_test_are_sinks_equal_full_edid_last_byte_differs),
@@ -3955,9 +5112,6 @@ static struct kunit_case amdgpu_dm_irq_tests[] = {
 	KUNIT_CASE(dm_test_irq_register_rejects_null_params),
 	KUNIT_CASE(dm_test_irq_register_rejects_invalid_context),
 	KUNIT_CASE(dm_test_irq_register_rejects_invalid_source),
-	KUNIT_CASE(dm_test_irq_register_adds_low_context_handler),
-	KUNIT_CASE(dm_test_irq_register_adds_high_context_handler),
-	KUNIT_CASE(dm_test_irq_register_multiple_handlers),
 	KUNIT_CASE(dm_test_irq_register_separate_contexts),
 	/* amdgpu_dm_irq_unregister_interrupt */
 	KUNIT_CASE(dm_test_irq_unregister_rejects_invalid_source),
@@ -3970,6 +5124,7 @@ static struct kunit_case amdgpu_dm_irq_tests[] = {
 	KUNIT_CASE(dm_test_get_crtc_by_otg_inst_returns_match),
 	KUNIT_CASE(dm_test_get_crtc_by_otg_inst_returns_null),
 	KUNIT_CASE(dm_test_get_crtc_by_otg_inst_empty_list),
+	KUNIT_CASE(dm_test_get_crtc_by_otg_inst_unassigned_otg),
 	/* amdgpu_dm_set_irq_funcs */
 	KUNIT_CASE(dm_test_set_irq_funcs),
 	/* amdgpu_dm_irq_suspend/resume_early/resume_late */
@@ -3995,17 +5150,15 @@ static struct kunit_case amdgpu_dm_irq_tests[] = {
 	KUNIT_CASE(dm_test_set_crtc_irq_state_otg_disabled),
 	KUNIT_CASE(dm_test_set_crtc_irq_state_enable),
 	KUNIT_CASE(dm_test_set_pflip_irq_state_disable),
-	KUNIT_CASE(dm_test_set_vline0_irq_state_enable),
-	KUNIT_CASE(dm_test_set_vupdate_irq_state_enable),
 	KUNIT_CASE(dm_test_set_crtc_irq_state_allows_idle),
 	/* amdgpu_dm_irq_immediate_work */
 	KUNIT_CASE(dm_test_irq_immediate_work_empty),
-	KUNIT_CASE(dm_test_irq_immediate_work_invokes_handler),
 	KUNIT_CASE(dm_test_irq_immediate_work_invokes_all),
 	/* amdgpu_dm_irq_schedule_work */
 	KUNIT_CASE(dm_test_irq_schedule_work_empty),
 	KUNIT_CASE(dm_test_irq_schedule_work_queues_handler),
 	KUNIT_CASE(dm_test_irq_schedule_work_requeue_fallback),
+	KUNIT_CASE(dm_test_irq_schedule_work_no_workqueue),
 	/* amdgpu_dm_set_hpd_irq_state */
 	KUNIT_CASE(dm_test_set_hpd_irq_state_null_dc),
 	/* amdgpu_dm_set_dmub_outbox_irq_state */
@@ -4020,30 +5173,44 @@ static struct kunit_case amdgpu_dm_irq_tests[] = {
 	KUNIT_CASE(dm_test_hpd_init_fini_with_connectors),
 	KUNIT_CASE(dm_test_hpd_init_fini_analog_connector),
 	KUNIT_CASE(dm_test_hpd_init_fini_irq_ref),
+	KUNIT_CASE(dm_test_hpd_init_skips_connector_without_link),
 	/* dm_handle_hpd_rx_offload_work */
 	KUNIT_CASE(dm_test_hpd_rx_offload_work_no_connector),
-	KUNIT_CASE(dm_test_hpd_rx_offload_work_no_connection),
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_detect_type_fails),
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_in_reset),
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_msg_rdy),
 	KUNIT_CASE(dm_test_hpd_rx_offload_work_automated_test),
+	KUNIT_CASE(dm_test_hpd_rx_offload_work_timing_changed),
 	KUNIT_CASE(dm_test_hpd_rx_offload_work_link_loss),
 	/* amdgpu_dm_hdmi_hpd_debounce_work */
 	KUNIT_CASE(dm_test_hdmi_hpd_debounce_detect_false),
 	KUNIT_CASE(dm_test_hdmi_hpd_debounce_reallow_idle),
+	KUNIT_CASE(dm_test_hdmi_hpd_debounce_sink_unchanged),
+	KUNIT_CASE(dm_test_hdmi_hpd_debounce_sink_changed),
 	/* handle_hpd_irq/handle_hpd_irq_helper */
 	KUNIT_CASE(dm_test_handle_hpd_irq_disabled),
 	KUNIT_CASE(dm_test_handle_hpd_irq_helper_debounce_schedule),
 	KUNIT_CASE(dm_test_handle_hpd_irq_helper_debounce_release_prev),
-	KUNIT_CASE(dm_test_handle_hpd_irq_helper_detect_false),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_detect_type_fails),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_debounce_pending),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_hdcp_reset),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_forced_detect),
+	KUNIT_CASE(dm_test_handle_hpd_irq_helper_detect_true),
 	/* handle_hpd_rx_irq/schedule_hpd_rx_offload_work */
 	KUNIT_CASE(dm_test_handle_hpd_rx_irq_disabled),
-	KUNIT_CASE(dm_test_handle_hpd_rx_irq_no_left_work),
 	KUNIT_CASE(dm_test_handle_hpd_rx_irq_automated_test),
 	KUNIT_CASE(dm_test_handle_hpd_rx_irq_msg_rdy),
 	KUNIT_CASE(dm_test_handle_hpd_rx_irq_link_loss),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_detect_type_fails),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_forced_detect),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_detect_true),
+	KUNIT_CASE(dm_test_handle_hpd_rx_irq_cp_irq),
 	KUNIT_CASE(dm_test_schedule_hpd_rx_offload_work),
 	/* dmub_hpd_callback/dmub_hpd_sense_callback */
 	KUNIT_CASE(dm_test_dmub_hpd_callback_null_inputs),
 	KUNIT_CASE(dm_test_dmub_hpd_callback_invalid_index),
 	KUNIT_CASE(dm_test_dmub_hpd_callback_empty_connectors),
+	KUNIT_CASE(dm_test_dmub_hpd_callback_suspend_skip),
 	KUNIT_CASE(dm_test_dmub_hpd_callback_unknown_type_match),
 	KUNIT_CASE(dm_test_dmub_hpd_callback_hpd_type),
 	KUNIT_CASE(dm_test_dmub_hpd_callback_hpd_irq_type),
@@ -4056,26 +5223,49 @@ static struct kunit_case amdgpu_dm_irq_tests[] = {
 	/* pflip/vupdate/crtc high IRQ callbacks */
 	KUNIT_CASE(dm_test_pflip_high_irq_no_crtc),
 	KUNIT_CASE(dm_test_pflip_high_irq_not_submitted),
+	KUNIT_CASE(dm_test_pflip_high_irq_completes_flip),
+	KUNIT_CASE(dm_test_pflip_high_irq_vrr_no_scanoutpos),
+	KUNIT_CASE(dm_test_pflip_high_irq_vrr_front_porch),
 	KUNIT_CASE(dm_test_vupdate_high_irq_no_crtc),
+	KUNIT_CASE(dm_test_vupdate_high_irq_dcn_completes_flip),
+	KUNIT_CASE(dm_test_vupdate_high_irq_dcn_no_active_planes),
+	KUNIT_CASE(dm_test_vupdate_high_irq_dce_vrr_btr),
 	KUNIT_CASE(dm_test_crtc_high_irq_no_crtc),
 	KUNIT_CASE(dm_test_crtc_high_irq_vrr_pre_ai),
 	KUNIT_CASE(dm_test_crtc_high_irq_vrr_ai_no_stream),
+	KUNIT_CASE(dm_test_crtc_high_irq_writeback_disables),
+	KUNIT_CASE(dm_test_crtc_high_irq_writeback_completes),
+	KUNIT_CASE(dm_test_crtc_high_irq_schedules_vmin_vmax),
 	/* dm_handle_hpd_work */
 	KUNIT_CASE(dm_test_handle_hpd_work_out_of_range),
+	KUNIT_CASE(dm_test_handle_hpd_work_null_notify),
 	/* dm_dmub_outbox1_low_irq */
 	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_empty),
+	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_drains_trace),
 	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_no_handler),
 	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_direct_callback),
 	KUNIT_CASE(dm_test_dmub_outbox1_low_irq_offload),
 	/* IRQ handler registration helpers */
 	KUNIT_CASE(dm_test_dce110_register_irq_handlers_rejects_uninitialized_sources),
 	KUNIT_CASE(dm_test_dce110_register_irq_handlers_one_crtc),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_soc15_client),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_crtc_add_id_fails),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_bad_vblank_source),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_bad_vupdate_source),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_bad_pflip_source),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_vupdate_add_id_fails),
+	KUNIT_CASE(dm_test_dce110_register_irq_handlers_hpd_add_id_fails),
 	KUNIT_CASE(dm_test_dcn10_register_irq_handlers_zero_crtc),
 	KUNIT_CASE(dm_test_dcn10_register_irq_handlers_one_crtc),
+	KUNIT_CASE(dm_test_dcn10_register_irq_handlers_vupdate_add_id_fails),
+	KUNIT_CASE(dm_test_dcn10_register_irq_handlers_bad_vupdate_source),
+	KUNIT_CASE(dm_test_dcn10_register_irq_handlers_hpd_add_id_fails),
 	KUNIT_CASE(dm_test_register_outbox_irq_handlers_without_dmub),
+	KUNIT_CASE(dm_test_register_outbox_irq_handlers_add_id_fails),
 	KUNIT_CASE(dm_test_register_outbox_irq_handlers_with_dmub),
 	/* amdgpu_dm_irq_handler */
 	KUNIT_CASE(dm_test_irq_handler_dispatches_work),
+	KUNIT_CASE(dm_test_irq_handler_invalid_source),
 	/* dm_handle_vmin_vmax_update */
 	KUNIT_CASE(dm_test_handle_vmin_vmax_update),
 	{}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c
index 6394b967ac3ae6..a9c6485e2a9e45 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_ism_test.c
@@ -642,31 +642,6 @@ static void dm_test_ism_init_sets_initial_state(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, ism->config.sso_num_frames, config.sso_num_frames);
 }
 
-/* ===== Tests for amdgpu_dm_ism_fini ===== */
-
-/**
- * dm_test_ism_fini_after_init - fini cancels never-scheduled work without error
- * @test: KUnit test context
- */
-static void dm_test_ism_fini_after_init(struct kunit *test)
-{
-	struct amdgpu_dm_ism *ism = alloc_test_ism(test);
-	struct amdgpu_dm_ism_config config = {
-		.filter_num_frames = 5,
-		.filter_entry_count = 3,
-		.activation_num_delay_frames = 10,
-		.sso_num_frames = 2,
-	};
-
-	amdgpu_dm_ism_init(ism, &config);
-	/* Work was never scheduled; cancel_delayed_work_sync is a no-op. */
-	amdgpu_dm_ism_fini(ism);
-
-	/* FSM state is untouched by fini */
-	KUNIT_EXPECT_EQ(test, (int)ism->current_state,
-			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
-}
-
 /* ===== Tests for dm_ism_set_last_idle_ts ===== */
 
 /**
@@ -865,18 +840,14 @@ static struct amdgpu_crtc *alloc_test_acrtc(struct kunit *test,
 {
 	struct amdgpu_device *adev;
 	struct amdgpu_crtc *acrtc;
-	struct dc *dc;
 
 	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, adev);
 
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-
 	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, acrtc);
 
-	adev->dm.dc = dc;
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
 	adev->dm.ddev = &adev->ddev;
 	mutex_init(&adev->dm.dc_lock);
 
@@ -901,6 +872,32 @@ static void register_test_acrtc(struct amdgpu_device *adev,
 	list_add_tail(&acrtc->base.head, &adev->ddev.mode_config.crtc_list);
 }
 
+/* ===== Tests for amdgpu_dm_ism_flush ===== */
+
+/**
+ * dm_test_ism_flush_after_init - flush cancels never-scheduled work without error
+ * @test: KUnit test context
+ */
+static void dm_test_ism_flush_after_init(struct kunit *test)
+{
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, NULL);
+	struct amdgpu_dm_ism *ism = &acrtc->ism;
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 5,
+		.filter_entry_count = 3,
+		.activation_num_delay_frames = 10,
+		.sso_num_frames = 2,
+	};
+
+	amdgpu_dm_ism_init(ism, &config);
+	/* Work was never scheduled; cancel_delayed_work_sync is a no-op. */
+	amdgpu_dm_ism_flush(ism);
+
+	/* FSM state is untouched by flush */
+	KUNIT_EXPECT_EQ(test, (int)ism->current_state,
+			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+}
+
 /* ===== Tests for amdgpu_dm_ism_commit_event ===== */
 
 /**
@@ -926,7 +923,7 @@ static void dm_test_ism_commit_event_no_state(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
 			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
 
-	amdgpu_dm_ism_fini(&acrtc->ism);
+	amdgpu_dm_ism_flush(&acrtc->ism);
 }
 
 /**
@@ -962,7 +959,7 @@ static void dm_test_ism_commit_event_cursor_transition(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
 			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
 
-	amdgpu_dm_ism_fini(&acrtc->ism);
+	amdgpu_dm_ism_flush(&acrtc->ism);
 }
 
 /**
@@ -992,7 +989,7 @@ static void dm_test_ism_commit_event_invalid_event(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
 			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
 
-	amdgpu_dm_ism_fini(&acrtc->ism);
+	amdgpu_dm_ism_flush(&acrtc->ism);
 }
 
 /* ===== Tests for amdgpu_dm_ism_force_full_power ===== */
@@ -1024,7 +1021,7 @@ static void dm_test_ism_force_full_power(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
 			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
 
-	amdgpu_dm_ism_fini(&acrtc->ism);
+	amdgpu_dm_ism_flush(&acrtc->ism);
 }
 
 /* ===== Tests for amdgpu_dm_ism_disable / amdgpu_dm_ism_enable ===== */
@@ -1052,7 +1049,7 @@ static void dm_test_ism_disable_enable_cycle(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
 			(int)DM_ISM_STATE_FULL_POWER_RUNNING);
 
-	amdgpu_dm_ism_fini(&acrtc->ism);
+	amdgpu_dm_ism_flush(&acrtc->ism);
 }
 
 /* ===== Tests for dm_ism_dispatch_power_state (via commit_event) ===== */
@@ -1131,7 +1128,7 @@ static void dm_test_ism_dispatch_hysteresis_schedule_and_cancel(struct kunit *te
 				(int)DM_ISM_STATE_HYSTERESIS_BUSY);
 	}
 
-	amdgpu_dm_ism_fini(&acrtc->ism);
+	amdgpu_dm_ism_flush(&acrtc->ism);
 }
 
 /**
@@ -1187,7 +1184,358 @@ static void dm_test_ism_dispatch_optimized_idle_defers_sso(struct kunit *test)
 		cancel_delayed_work(&acrtc->ism.sso_delayed_work);
 	}
 
-	amdgpu_dm_ism_fini(&acrtc->ism);
+	amdgpu_dm_ism_flush(&acrtc->ism);
+}
+
+/*
+ * ===== Behavioural tests for dm_ism_commit_idle_optimization_state =====
+ *
+ * The commit path runs the real DC calls. dc_allow_idle_optimizations() only
+ * reaches hardware through dc->hwss.apply_idle_power_optimizations(), so the
+ * spy is installed there — DC's own seam — and the outcome is read back from
+ * dc->idle_optimizations_allowed. The callback fires only on an actual
+ * transition, so the call count is the number of idle state changes requested.
+ */
+
+static struct {
+	int calls;
+	bool last_allow;
+} dm_ism_test_idle;
+
+static bool dm_ism_test_apply_idle(struct dc *dc, bool allow)
+{
+	dm_ism_test_idle.calls++;
+	dm_ism_test_idle.last_allow = allow;
+
+	return true;
+}
+
+/*
+ * Initialise @acrtc's ISM against a stream with a non-NULL link (required by
+ * the static-screen arm) and make dc_allow_idle_optimizations() observable.
+ */
+static struct dc_stream_state *setup_commit_test(struct kunit *test,
+						 struct amdgpu_crtc *acrtc,
+						 struct amdgpu_dm_ism_config *config)
+{
+	struct amdgpu_device *adev = drm_to_adev(acrtc->base.dev);
+	struct dc_stream_state *stream;
+	struct dm_crtc_state *dm_state;
+	struct dal_logger *logger;
+	struct dc *dc = adev->dm.dc;
+
+	stream = dm_kunit_alloc_stream(test, dm_kunit_alloc_link(test));
+	stream->timing.v_total = 1125;
+	stream->timing.h_total = 2200;
+	stream->timing.pix_clk_100hz = 1485000;
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	dm_state->stream = stream;
+
+	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, logger);
+	logger->dev = &adev->ddev;
+	dc->ctx->logger = logger;
+	/* is_apu path avoids DC_LOG_DC()'s NULL-logger dereference. */
+	dc->caps.is_apu = true;
+	/* dc_allow() only calls apply() when clk_mgr is present. */
+	dc->clk_mgr = dm_kunit_alloc_clk_mgr(test);
+	dc->hwss.apply_idle_power_optimizations = dm_ism_test_apply_idle;
+	dm_ism_test_idle = (typeof(dm_ism_test_idle)){ };
+
+	amdgpu_dm_ism_init(&acrtc->ism, config);
+	acrtc->base.state = &dm_state->base;
+
+	return stream;
+}
+
+/**
+ * dm_test_ism_commit_allows_idle_on_optimized_idle - OPTIMIZED_IDLE commits idle-allow
+ * @test: KUnit test context
+ *
+ * With idle-allow filtering disabled (delay 0) ENTER_IDLE_REQUESTED cascades
+ * FULL_POWER_RUNNING -> HYSTERESIS_WAITING -> OPTIMIZED_IDLE. Because
+ * sso_num_frames >= filter_num_frames the OPTIMIZED_IDLE arm commits the idle
+ * state. With no active vblank and no SSO the commit must request allow-idle
+ * exactly once, without a preceding disallow.
+ */
+static void dm_test_ism_commit_allows_idle_on_optimized_idle(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 0,
+		.sso_num_frames = 2,
+	};
+
+	setup_commit_test(test, acrtc, &config);
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_OPTIMIZED_IDLE);
+
+		/* vblank disabled + no SSO -> a single allow-idle transition. */
+		KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 1);
+		KUNIT_EXPECT_TRUE(test, dm_ism_test_idle.last_allow);
+		KUNIT_EXPECT_TRUE(test, adev->dm.dc->idle_optimizations_allowed);
+
+		cancel_delayed_work(&acrtc->ism.sso_delayed_work);
+	}
+
+	amdgpu_dm_ism_flush(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_commit_enables_sso_on_optimized_idle_sso - SSO arm re-arms idle
+ * @test: KUnit test context
+ *
+ * With both idle-allow and SSO delays disabled the FSM cascades through
+ * OPTIMIZED_IDLE into OPTIMIZED_IDLE_SSO, whose arm commits with
+ * allow_panel_sso = true. Engaging SSO must first disallow idle and then
+ * re-allow it, so starting from an idle-allowed DC yields two transitions.
+ */
+static void dm_test_ism_commit_enables_sso_on_optimized_idle_sso(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 0,
+		.sso_num_frames = 0,
+	};
+
+	setup_commit_test(test, acrtc, &config);
+	adev->dm.dc->idle_optimizations_allowed = true;
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_OPTIMIZED_IDLE_SSO);
+
+		/* SSO engagement disallows idle, then re-allows it. */
+		KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 2);
+		KUNIT_EXPECT_TRUE(test, dm_ism_test_idle.last_allow);
+		KUNIT_EXPECT_TRUE(test, adev->dm.dc->idle_optimizations_allowed);
+	}
+
+	amdgpu_dm_ism_flush(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_commit_disallows_idle_on_timer_aborted - aborted timer disallows idle
+ * @test: KUnit test context
+ *
+ * A non-zero idle-allow delay parks the FSM in HYSTERESIS_WAITING.
+ * EXIT_IDLE_REQUESTED then drives HYSTERESIS_WAITING -> TIMER_ABORTED, whose
+ * arm commits with vblank_enabled = true: idle must be disallowed and, because
+ * vblank is active, never re-allowed. The FSM returns to FULL_POWER_RUNNING.
+ */
+static void dm_test_ism_commit_disallows_idle_on_timer_aborted(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct dc_stream_state *stream;
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 5,
+		.filter_entry_count = 3,
+		.activation_num_delay_frames = 10,
+		.filter_history_size = 8,
+		.filter_old_history_threshold = 0,
+		.sso_num_frames = 0,
+	};
+
+	stream = setup_commit_test(test, acrtc, &config);
+	setup_idle_delay_history(&acrtc->ism, stream);
+	adev->dm.dc->idle_optimizations_allowed = true;
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		/* Non-zero delay parks the FSM in HYSTERESIS_WAITING. */
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_HYSTERESIS_WAITING);
+
+		/* Abort: HYSTERESIS_WAITING -> TIMER_ABORTED -> FULL_POWER_RUNNING. */
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_EXIT_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+		/* TIMER_ABORTED commit runs with vblank_enabled=true. */
+		KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 1);
+		KUNIT_EXPECT_FALSE(test, dm_ism_test_idle.last_allow);
+		KUNIT_EXPECT_FALSE(test, adev->dm.dc->idle_optimizations_allowed);
+	}
+
+	amdgpu_dm_ism_flush(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_exit_from_optimized_idle_disallows_idle - leaving OPTIMIZED_IDLE
+ * @test: KUnit test context
+ *
+ * EXIT_IDLE_REQUESTED from OPTIMIZED_IDLE runs the previous-state arm of
+ * dm_ism_dispatch_power_state(): the pending SSO worker is cancelled, the idle
+ * period is recorded and the commit runs with vblank_enabled = true, which
+ * disallows idle again.
+ */
+static void dm_test_ism_exit_from_optimized_idle_disallows_idle(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 0,
+		.sso_num_frames = 2,
+	};
+
+	setup_commit_test(test, acrtc, &config);
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_OPTIMIZED_IDLE);
+
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_EXIT_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+		/* Allow on entry, disallow on exit. */
+		KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 2);
+		KUNIT_EXPECT_FALSE(test, dm_ism_test_idle.last_allow);
+		KUNIT_EXPECT_FALSE(test, adev->dm.dc->idle_optimizations_allowed);
+		/* The idle period was recorded on the way out. */
+		KUNIT_EXPECT_EQ(test, acrtc->ism.next_record_idx, 1);
+	}
+
+	amdgpu_dm_ism_flush(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_exit_from_sso_disallows_idle - leaving OPTIMIZED_IDLE_SSO
+ * @test: KUnit test context
+ *
+ * EXIT_IDLE_REQUESTED from OPTIMIZED_IDLE_SSO runs the SSO previous-state arm:
+ * the idle period is recorded and the commit disallows idle. Unlike the
+ * OPTIMIZED_IDLE arm there is no pending SSO worker left to cancel.
+ */
+static void dm_test_ism_exit_from_sso_disallows_idle(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 0,
+		.sso_num_frames = 0,
+	};
+
+	setup_commit_test(test, acrtc, &config);
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_OPTIMIZED_IDLE_SSO);
+
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_EXIT_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_FULL_POWER_RUNNING);
+
+		KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 2);
+		KUNIT_EXPECT_FALSE(test, dm_ism_test_idle.last_allow);
+		KUNIT_EXPECT_FALSE(test, adev->dm.dc->idle_optimizations_allowed);
+		KUNIT_EXPECT_EQ(test, acrtc->ism.next_record_idx, 1);
+	}
+
+	amdgpu_dm_ism_flush(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_delayed_work_runs_timer_elapsed - hysteresis worker fires
+ * @test: KUnit test context
+ *
+ * A non-zero idle-allow delay parks the FSM in HYSTERESIS_WAITING and arms the
+ * hysteresis worker. Flushing it runs dm_ism_delayed_work_func(), which takes
+ * dc_lock itself and posts TIMER_ELAPSED, cascading to OPTIMIZED_IDLE_SSO
+ * because the SSO delay is zero. The flush must happen without dc_lock held.
+ */
+static void dm_test_ism_delayed_work_runs_timer_elapsed(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct dc_stream_state *stream;
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 5,
+		.filter_entry_count = 3,
+		.activation_num_delay_frames = 10,
+		.filter_history_size = 8,
+		.filter_old_history_threshold = 0,
+		.sso_num_frames = 0,
+	};
+
+	stream = setup_commit_test(test, acrtc, &config);
+	setup_idle_delay_history(&acrtc->ism, stream);
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_HYSTERESIS_WAITING);
+		KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 0);
+	}
+
+	flush_delayed_work(&acrtc->ism.delayed_work);
+
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_OPTIMIZED_IDLE_SSO);
+	KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 1);
+	KUNIT_EXPECT_TRUE(test, adev->dm.dc->idle_optimizations_allowed);
+
+	amdgpu_dm_ism_flush(&acrtc->ism);
+}
+
+/**
+ * dm_test_ism_sso_delayed_work_runs_sso_elapsed - SSO worker fires
+ * @test: KUnit test context
+ *
+ * With sso_num_frames >= filter_num_frames the OPTIMIZED_IDLE arm allows idle
+ * and arms the SSO worker. Flushing it runs dm_ism_sso_delayed_work_func(),
+ * which posts SSO_TIMER_ELAPSED; engaging SSO disallows and re-allows idle, so
+ * two further transitions follow the initial allow.
+ */
+static void dm_test_ism_sso_delayed_work_runs_sso_elapsed(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc = alloc_test_acrtc(test, &adev);
+	struct amdgpu_dm_ism_config config = {
+		.filter_num_frames = 0,
+		.sso_num_frames = 2,
+	};
+
+	setup_commit_test(test, acrtc, &config);
+
+	scoped_guard(mutex, &adev->dm.dc_lock) {
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
+					   DM_ISM_EVENT_ENTER_IDLE_REQUESTED);
+		KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+				(int)DM_ISM_STATE_OPTIMIZED_IDLE);
+		KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 1);
+	}
+
+	flush_delayed_work(&acrtc->ism.sso_delayed_work);
+
+	KUNIT_EXPECT_EQ(test, (int)acrtc->ism.current_state,
+			(int)DM_ISM_STATE_OPTIMIZED_IDLE_SSO);
+	KUNIT_EXPECT_EQ(test, dm_ism_test_idle.calls, 3);
+	KUNIT_EXPECT_TRUE(test, adev->dm.dc->idle_optimizations_allowed);
+
+	amdgpu_dm_ism_flush(&acrtc->ism);
 }
 
 static struct kunit_case dm_ism_test_cases[] = {
@@ -1235,8 +1583,8 @@ static struct kunit_case dm_ism_test_cases[] = {
 	KUNIT_CASE(dm_test_ism_idle_delay_entry_count_exceeds_history_size),
 	/* amdgpu_dm_ism_init */
 	KUNIT_CASE(dm_test_ism_init_sets_initial_state),
-	/* amdgpu_dm_ism_fini */
-	KUNIT_CASE(dm_test_ism_fini_after_init),
+	/* amdgpu_dm_ism_flush */
+	KUNIT_CASE(dm_test_ism_flush_after_init),
 	/* dm_ism_set_last_idle_ts */
 	KUNIT_CASE(dm_test_ism_set_last_idle_ts_updates_timestamp),
 	/* dm_ism_insert_record */
@@ -1263,6 +1611,16 @@ static struct kunit_case dm_ism_test_cases[] = {
 	/* dm_ism_dispatch_power_state (via commit_event) */
 	KUNIT_CASE(dm_test_ism_dispatch_hysteresis_schedule_and_cancel),
 	KUNIT_CASE(dm_test_ism_dispatch_optimized_idle_defers_sso),
+	/* dm_ism_commit_idle_optimization_state (behavioural, via dc->hwss) */
+	KUNIT_CASE(dm_test_ism_commit_allows_idle_on_optimized_idle),
+	KUNIT_CASE(dm_test_ism_commit_enables_sso_on_optimized_idle_sso),
+	KUNIT_CASE(dm_test_ism_commit_disallows_idle_on_timer_aborted),
+	/* dm_ism_dispatch_power_state previous-state arms */
+	KUNIT_CASE(dm_test_ism_exit_from_optimized_idle_disallows_idle),
+	KUNIT_CASE(dm_test_ism_exit_from_sso_disallows_idle),
+	/* dm_ism_delayed_work_func / dm_ism_sso_delayed_work_func */
+	KUNIT_CASE(dm_test_ism_delayed_work_runs_timer_elapsed),
+	KUNIT_CASE(dm_test_ism_sso_delayed_work_runs_sso_elapsed),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c
index caedad4197bd9c..eafa3e6733a64b 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_mst_types_test.c
@@ -8,11 +8,16 @@
 #include <kunit/test.h>
 
 #include <drm/drm_drv.h>
+#include <drm/drm_atomic_uapi.h>
+#include <drm/drm_edid.h>
 #include <drm/drm_fixed.h>
 #include <drm/drm_kunit_helpers.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_connector.h>
 #include <drm/drm_mode_config.h>
+#include <drm/drm_modeset_lock.h>
+#include <drm/drm_property.h>
 #include <drm/display/drm_dp.h>
 #include <drm/display/drm_dp_helper.h>
 #include <drm/display/drm_dp_mst_helper.h>
@@ -23,8 +28,11 @@
 #include "amdgpu.h"
 #include "amdgpu_mode.h"
 #include "amdgpu_dm.h"
+#include "amdgpu_dm_hdcp.h"
 #include "amdgpu_dm_mst_types.h"
 #include "amdgpu_dm_kunit_test_helpers.h"
+#include "dsc/dsc.h"
+#include "inc/core_types.h"
 #include "inc/link_service.h"
 
 /*
@@ -38,6 +46,7 @@ static int dm_mst_test_aux_transfer_raw_result;
 static u8 dm_mst_test_aux_transfer_raw_reply;
 static enum aux_return_code_type dm_mst_test_aux_transfer_raw_operation_result;
 static ssize_t dm_mst_test_aux_transfer_override;
+static ssize_t dm_mst_test_aux_write_override;
 
 static int dm_mst_test_aux_transfer_raw(struct ddc_service *ddc,
 						struct aux_payload *payload,
@@ -109,6 +118,8 @@ static ssize_t dm_mst_test_aux_transfer(struct drm_dp_aux *aux,
 		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
 		return msg->size;
 	case DP_AUX_NATIVE_WRITE:
+		if (dm_mst_test_aux_write_override)
+			return dm_mst_test_aux_write_override;
 		msg->reply = DP_AUX_NATIVE_REPLY_ACK;
 		return msg->size;
 	default:
@@ -146,6 +157,7 @@ static struct amdgpu_dm_connector *dm_mst_test_alloc_sideband_connector(struct k
 
 	memset(dm_mst_test_dpcd, 0, sizeof(dm_mst_test_dpcd));
 	dm_mst_test_aux_transfer_override = 0;
+	dm_mst_test_aux_write_override = 0;
 
 	return aconnector;
 }
@@ -878,6 +890,26 @@ static void dm_mst_test_aux_transfer_non_ack_reply(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, dm_mst_test_last_payload.address, 4U);
 }
 
+/**
+ * dm_mst_test_aux_transfer_oversized - oversized AUX message is rejected.
+ * @test: KUnit test context.
+ *
+ * The payload is copied into a 16-byte stack buffer, so a larger message must
+ * be rejected with -E2BIG before the AUX handle is touched. The guard warns,
+ * which is the expected behaviour for this caller bug.
+ */
+static void dm_mst_test_aux_transfer_oversized(struct kunit *test)
+{
+	struct drm_dp_aux_msg msg = { 0 };
+	u8 buffer[17] = { 0 };
+
+	msg.request = DP_AUX_NATIVE_WRITE;
+	msg.buffer = buffer;
+	msg.size = sizeof(buffer);
+
+	KUNIT_EXPECT_EQ(test, dm_dp_aux_transfer(NULL, &msg), (ssize_t)-E2BIG);
+}
+
 /**
  * dm_mst_test_fill_payload_flags_native_write - native write request decode.
  * @test: KUnit test context.
@@ -1270,6 +1302,212 @@ static void dm_mst_test_create_fake_mst_encoders(struct kunit *test)
 	}
 }
 
+/* Tests for dm_dp_add_mst_connector */
+
+struct dm_mst_test_add_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *master;
+	struct drm_dp_mst_port *port;
+	struct dc_link *link;
+};
+
+static void dm_mst_test_cleanup_add_ctx(void *data)
+{
+	struct amdgpu_dm_connector *master = data;
+
+	drm_connector_cleanup(&master->base);
+}
+
+/*
+ * Stand in for amdgpu_display_modeset_create_props(), which the test module
+ * cannot link against. Only the properties amdgpu_dm_connector_init_helper()
+ * attaches are created.
+ */
+static void dm_mst_test_create_mode_props(struct kunit *test, struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	struct drm_device *drm = adev_to_drm(adev);
+
+	KUNIT_ASSERT_EQ(test, drm_mode_create_scaling_mode_property(drm), 0);
+
+	mode_info->underscan_property =
+		drm_property_create_range(drm, 0, "underscan", 0, 1);
+	mode_info->underscan_hborder_property =
+		drm_property_create_range(drm, 0, "underscan hborder", 0, 128);
+	mode_info->underscan_vborder_property =
+		drm_property_create_range(drm, 0, "underscan vborder", 0, 128);
+	KUNIT_ASSERT_NOT_NULL(test, mode_info->underscan_property);
+	KUNIT_ASSERT_NOT_NULL(test, mode_info->underscan_hborder_property);
+	KUNIT_ASSERT_NOT_NULL(test, mode_info->underscan_vborder_property);
+}
+
+/*
+ * Build the MST root connector and the topology port the new downstream
+ * connector hangs off. The port has no parent branch so the callback takes the
+ * "no branch descriptor" path without needing a sideband AUX backend.
+ */
+static void dm_mst_test_init_add_ctx(struct kunit *test, struct dm_mst_test_add_ctx *ctx)
+{
+	struct amdgpu_device *adev;
+	struct drm_device *drm;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	drm = &adev->ddev;
+	ret = drmm_mode_config_init(drm);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	dm_mst_test_create_mode_props(test, adev);
+
+	adev->dm.adev = adev;
+	adev->dm.ddev = drm;
+	adev->dm.display_indexes_num = 2;
+	adev->mode_info.num_crtc = 2;
+	dm_dp_create_fake_mst_encoders(adev);
+
+	ctx->adev = adev;
+	ctx->link = dm_kunit_alloc_link(test);
+	/* Inflexible DIG mapping lets link_enc_cfg_get_link_enc() skip the dc lookup. */
+	ctx->link->link_enc = kunit_kzalloc(test, sizeof(*ctx->link->link_enc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->link->link_enc);
+
+	ctx->master = dm_kunit_alloc_connector(test, adev, ctx->link);
+	ctx->master->connector_id = 5;
+	ret = drm_connector_init(drm, &ctx->master->base, &dm_mst_test_connector_funcs,
+				 DRM_MODE_CONNECTOR_DisplayPort);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_mst_test_cleanup_add_ctx, ctx->master), 0);
+
+	ctx->port = kunit_kzalloc(test, sizeof(*ctx->port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->port);
+	kref_init(&ctx->port->malloc_kref);
+	ctx->port->mgr = &ctx->master->mst_mgr;
+}
+
+/* True when @prop is attached to the connector's mode object. */
+static bool dm_mst_test_has_prop(struct drm_connector *connector, struct drm_property *prop)
+{
+	struct drm_object_properties *props = connector->base.properties;
+	int i;
+
+	if (!prop)
+		return false;
+
+	for (i = 0; i < props->count; i++)
+		if (props->properties[i] == prop)
+			return true;
+
+	return false;
+}
+
+/**
+ * dm_mst_test_add_mst_connector_creates - Test downstream connector creation
+ * @test: KUnit test context
+ *
+ * The topology callback must allocate a DisplayPort connector bound to the port
+ * and the root connector, inherit the root's dc_link and connector id, attach
+ * every fake MST encoder, publish the path property and take a malloc
+ * reference on the port.
+ */
+static void dm_mst_test_add_mst_connector_creates(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_mst_test_add_ctx ctx;
+	struct drm_connector *connector;
+
+	dm_mst_test_init_add_ctx(test, &ctx);
+
+	connector = dm_dp_add_mst_connector(&ctx.master->mst_mgr, ctx.port, "1-3");
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	aconnector = to_amdgpu_dm_connector(connector);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->mst_output_port, ctx.port);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->mst_root, ctx.master);
+	KUNIT_EXPECT_PTR_EQ(test, aconnector->dc_link, ctx.link);
+	KUNIT_EXPECT_EQ(test, aconnector->connector_id, 5);
+	KUNIT_EXPECT_TRUE(test, aconnector->mst_status & MST_PROBE);
+	KUNIT_EXPECT_EQ(test, aconnector->branch_ieee_oui, 0U);
+	KUNIT_EXPECT_EQ(test, connector->connector_type, DRM_MODE_CONNECTOR_DisplayPort);
+	KUNIT_EXPECT_EQ(test, connector->possible_encoders, 0x3U);
+	KUNIT_EXPECT_NOT_NULL(test, connector->state);
+	KUNIT_EXPECT_NOT_NULL(test, connector->path_blob_ptr);
+	KUNIT_EXPECT_EQ(test, kref_read(&ctx.port->malloc_kref), 2U);
+	KUNIT_EXPECT_TRUE(test,
+			  dm_mst_test_has_prop(connector,
+					       ctx.adev->ddev.mode_config.path_property));
+	KUNIT_EXPECT_TRUE(test,
+			  dm_mst_test_has_prop(connector,
+					       ctx.adev->ddev.mode_config.tile_property));
+
+	dm_dp_mst_connector_destroy(connector);
+}
+
+/**
+ * dm_mst_test_add_mst_connector_inherits_props - Test optional property sharing
+ * @test: KUnit test context
+ *
+ * Max bpc, VRR capable and colorspace properties are only attached to the new
+ * connector when the root connector already owns them, so the downstream
+ * connector shares the root's property objects.
+ */
+static void dm_mst_test_add_mst_connector_inherits_props(struct kunit *test)
+{
+	struct dm_mst_test_add_ctx ctx;
+	struct drm_connector *connector;
+	struct drm_device *drm;
+
+	dm_mst_test_init_add_ctx(test, &ctx);
+	drm = &ctx.adev->ddev;
+
+	ctx.master->base.max_bpc_property =
+		drm_property_create_range(drm, 0, "max bpc", 8, 16);
+	ctx.master->base.vrr_capable_property =
+		drm_property_create_bool(drm, DRM_MODE_PROP_IMMUTABLE, "vrr_capable");
+	KUNIT_ASSERT_NOT_NULL(test, ctx.master->base.max_bpc_property);
+	KUNIT_ASSERT_NOT_NULL(test, ctx.master->base.vrr_capable_property);
+	KUNIT_ASSERT_EQ(test, drm_mode_create_dp_colorspace_property(&ctx.master->base, 0), 0);
+
+	connector = dm_dp_add_mst_connector(&ctx.master->mst_mgr, ctx.port, "1-4");
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+
+	KUNIT_EXPECT_PTR_EQ(test, connector->max_bpc_property, ctx.master->base.max_bpc_property);
+	KUNIT_EXPECT_PTR_EQ(test, connector->vrr_capable_property,
+			    ctx.master->base.vrr_capable_property);
+	KUNIT_EXPECT_PTR_EQ(test, connector->colorspace_property,
+			    ctx.master->base.colorspace_property);
+	KUNIT_EXPECT_TRUE(test, dm_mst_test_has_prop(connector, connector->max_bpc_property));
+	KUNIT_EXPECT_TRUE(test, dm_mst_test_has_prop(connector, connector->colorspace_property));
+
+	dm_dp_mst_connector_destroy(connector);
+}
+
+/**
+ * dm_mst_test_add_mst_connector_init_fails - Test the connector init failure path
+ * @test: KUnit test context
+ *
+ * Exhausting the device connector index space makes drm_connector_dynamic_init()
+ * fail, so the callback must free the connector it allocated, leave the port
+ * malloc reference untouched and report no connector to the MST helpers.
+ */
+static void dm_mst_test_add_mst_connector_init_fails(struct kunit *test)
+{
+	struct dm_mst_test_add_ctx ctx;
+	struct drm_device *drm;
+	int id;
+
+	dm_mst_test_init_add_ctx(test, &ctx);
+	drm = &ctx.adev->ddev;
+
+	/* drm_connector_init_only() takes an index out of the same [0, 31] space. */
+	do {
+		id = ida_alloc_max(&drm->mode_config.connector_ida, 31, GFP_KERNEL);
+	} while (id >= 0);
+	KUNIT_ASSERT_EQ(test, id, -ENOSPC);
+
+	KUNIT_EXPECT_NULL(test, dm_dp_add_mst_connector(&ctx.master->mst_mgr, ctx.port, "1-5"));
+	KUNIT_EXPECT_EQ(test, kref_read(&ctx.port->malloc_kref), 1U);
+}
+
 /**
  * dm_mst_test_atomic_check_no_old_crtc - Test atomic check no-op path
  * @test: KUnit test context
@@ -1339,20 +1577,2464 @@ static void dm_mst_test_detect_unregistered(struct kunit *test)
 			(int)connector_status_disconnected);
 }
 
-#if !defined(CONFIG_DRM_AMD_DC_FP)
+/*
+ * Fake DC remote-sink backend. dc_link_add_remote_sink() and
+ * dc_link_remove_remote_sink() are thin wrappers over these link_service
+ * callbacks, so overriding them is enough to run the MST connector paths
+ * without a DC core.
+ */
+static struct dc_sink *dm_mst_test_next_remote_sink;
+static struct dc_sink *dm_mst_test_removed_sink;
+static unsigned int dm_mst_test_add_remote_sink_calls;
+static unsigned int dm_mst_test_remove_remote_sink_calls;
+
+/* Signature copied verbatim from struct link_service::add_remote_sink. */
+static struct dc_sink *dm_mst_test_add_remote_sink(
+		struct dc_link *link,
+		const uint8_t *edid,
+		unsigned int len,
+		struct dc_sink_init_data *init_data)
+{
+	struct dc_sink *sink = dm_mst_test_next_remote_sink;
+
+	dm_mst_test_add_remote_sink_calls++;
+
+	if (sink) {
+		sink->link = link;
+		sink->sink_signal = init_data->sink_signal;
+		link->sink_count++;
+	}
+
+	return sink;
+}
+
+/* Signature copied verbatim from struct link_service::remove_remote_sink. */
+static void dm_mst_test_remove_remote_sink(struct dc_link *link, struct dc_sink *sink)
+{
+	dm_mst_test_remove_remote_sink_calls++;
+	dm_mst_test_removed_sink = sink;
+
+	if (link->sink_count)
+		link->sink_count--;
+}
+
+/*
+ * dc_sink_release() frees the sink with kfree(), so these must not come from
+ * the KUnit managed allocator.
+ */
+static struct dc_sink *dm_mst_test_alloc_sink(struct kunit *test)
+{
+	struct dc_sink *sink = kzalloc_obj(*sink);
+
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	kref_init(&sink->refcount);
+
+	return sink;
+}
+
+struct dm_mst_test_child {
+	struct amdgpu_device *adev;
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_dm_connector *root;
+	struct drm_dp_mst_port *port;
+	struct dc_link *link;
+};
+
+/*
+ * Build an MST downstream connector hanging off a root connector. The topology
+ * manager is left empty (no mst_primary) so the DRM MST helpers take their
+ * "port is gone" paths instead of requiring a real branch device.
+ */
+static void dm_mst_test_init_child(struct kunit *test, struct dm_mst_test_child *child)
+{
+	struct link_service *link_srv;
+	struct amdgpu_device *adev;
+	struct dc *dc;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	ret = drmm_mode_config_init(&adev->ddev);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	child->port = kunit_kzalloc(test, sizeof(*child->port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, child->port);
+
+	link_srv->add_remote_sink = dm_mst_test_add_remote_sink;
+	link_srv->remove_remote_sink = dm_mst_test_remove_remote_sink;
+	dc->link_srv = link_srv;
+
+	child->adev = adev;
+	child->link = dm_kunit_alloc_link(test);
+	child->link->dc = dc;
+
+	child->root = dm_kunit_alloc_connector(test, adev, child->link);
+	child->root->mst_mgr.dev = &adev->ddev;
+	mutex_init(&child->root->mst_mgr.lock);
+	drm_modeset_lock_init(&child->root->mst_mgr.base.lock);
+
+	child->port->mgr = &child->root->mst_mgr;
+	child->port->aux.name = "dm_mst_test_port_aux";
+	child->port->aux.transfer = dm_mst_test_aux_transfer;
+	drm_dp_aux_init(&child->port->aux);
+	drm_dp_dpcd_set_probe(&child->port->aux, false);
+
+	child->aconnector = dm_kunit_alloc_connector(test, adev, child->link);
+	child->aconnector->mst_root = child->root;
+	child->aconnector->mst_output_port = child->port;
+
+	ret = drm_connector_init(&adev->ddev, &child->aconnector->base,
+				 &dm_mst_test_connector_funcs,
+				 DRM_MODE_CONNECTOR_DisplayPort);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	/* The MST (un)register helpers log through the connector's kernel device. */
+	child->aconnector->base.kdev = adev->ddev.dev;
+
+	memset(dm_mst_test_dpcd, 0, sizeof(dm_mst_test_dpcd));
+	dm_mst_test_aux_transfer_override = 0;
+	dm_mst_test_aux_write_override = 0;
+	dm_mst_test_next_remote_sink = NULL;
+	dm_mst_test_removed_sink = NULL;
+	dm_mst_test_add_remote_sink_calls = 0;
+	dm_mst_test_remove_remote_sink_calls = 0;
+}
+
+static void dm_mst_test_fini_child(struct dm_mst_test_child *child)
+{
+	drm_edid_free(child->aconnector->drm_edid);
+	drm_edid_free(child->port->cached_edid);
+	drm_connector_cleanup(&child->aconnector->base);
+	drm_modeset_lock_fini(&child->root->mst_mgr.base.lock);
+	mutex_destroy(&child->root->mst_mgr.lock);
+}
+
+/* Tests for dm_dp_mst_get_modes */
+
+/**
+ * dm_mst_test_get_modes_no_edid_adds_default_sink - Test default remote sink
+ * @test: KUnit test context
+ *
+ * When the remote EDID cannot be read, dm_dp_mst_get_modes() must clear the
+ * MST_REMOTE_EDID status and register a default remote sink for the connector.
+ */
+static void dm_mst_test_get_modes_no_edid_adds_default_sink(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+	struct dc_sink *sink;
+
+	dm_mst_test_init_child(test, &child);
+	sink = dm_mst_test_alloc_sink(test);
+	dm_mst_test_next_remote_sink = sink;
+	child.aconnector->mst_status = MST_REMOTE_EDID;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_get_modes(&child.aconnector->base), 0);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_add_remote_sink_calls, 1U);
+	KUNIT_EXPECT_PTR_EQ(test, child.aconnector->dc_sink, sink);
+	KUNIT_EXPECT_PTR_EQ(test, sink->priv, (void *)child.aconnector);
+	KUNIT_EXPECT_EQ(test, child.aconnector->mst_status & MST_REMOTE_EDID, 0);
+
+	dc_sink_release(sink);
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_get_modes_no_edid_sink_alloc_fails - Test remote sink failure
+ * @test: KUnit test context
+ *
+ * If DC cannot add the default remote sink, dm_dp_mst_get_modes() must bail out
+ * with zero modes and leave the connector without a sink.
+ */
+static void dm_mst_test_get_modes_no_edid_sink_alloc_fails(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_get_modes(&child.aconnector->base), 0);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_add_remote_sink_calls, 1U);
+	KUNIT_EXPECT_NULL(test, child.aconnector->dc_sink);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_get_modes_no_edid_keeps_existing_sink - Test cached sink reuse
+ * @test: KUnit test context
+ *
+ * With no readable EDID but an existing remote sink, dm_dp_mst_get_modes() must
+ * keep that sink and must not ask DC for another one.
+ */
+static void dm_mst_test_get_modes_no_edid_keeps_existing_sink(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+	struct dc_sink *sink;
+
+	dm_mst_test_init_child(test, &child);
+	sink = dm_mst_test_alloc_sink(test);
+	child.aconnector->dc_sink = sink;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_get_modes(&child.aconnector->base), 0);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_add_remote_sink_calls, 0U);
+	KUNIT_EXPECT_PTR_EQ(test, child.aconnector->dc_sink, sink);
+
+	dc_sink_release(sink);
+	dm_mst_test_fini_child(&child);
+}
+
+/* Minimal EDID base block: valid header, no extensions, correct checksum. */
+static const u8 dm_mst_test_edid[EDID_LENGTH] = {
+	0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00,
+	[EDID_LENGTH - 1] = 0x06,
+};
+
+/**
+ * dm_mst_test_get_modes_cached_edid_replaces_virtual_sink - Test EDID sink path
+ * @test: KUnit test context
+ *
+ * With a cached EDID and a placeholder virtual sink, dm_dp_mst_get_modes() must
+ * release the virtual sink, add a real remote sink built from the EDID, and
+ * update the connector's mode list.
+ */
+static void dm_mst_test_get_modes_cached_edid_replaces_virtual_sink(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+	struct dc_sink *virtual_sink;
+	struct dc_sink *sink;
+
+	dm_mst_test_init_child(test, &child);
+
+	child.aconnector->drm_edid = drm_edid_alloc(dm_mst_test_edid, sizeof(dm_mst_test_edid));
+	KUNIT_ASSERT_NOT_NULL(test, child.aconnector->drm_edid);
+
+	virtual_sink = dm_mst_test_alloc_sink(test);
+	virtual_sink->sink_signal = SIGNAL_TYPE_VIRTUAL;
+	child.aconnector->dc_sink = virtual_sink;
+
+	sink = dm_mst_test_alloc_sink(test);
+	dm_mst_test_next_remote_sink = sink;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_get_modes(&child.aconnector->base), 0);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_add_remote_sink_calls, 1U);
+	KUNIT_EXPECT_PTR_EQ(test, child.aconnector->dc_sink, sink);
+	KUNIT_EXPECT_EQ(test, sink->sink_signal, SIGNAL_TYPE_DISPLAY_PORT_MST);
+	KUNIT_EXPECT_PTR_EQ(test, sink->priv, (void *)child.aconnector);
+
+	dc_sink_release(sink);
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_get_modes_cached_edid_sink_alloc_fails - Test EDID sink failure
+ * @test: KUnit test context
+ *
+ * If DC cannot add the remote sink built from the cached EDID,
+ * dm_dp_mst_get_modes() must bail out with zero modes.
+ */
+static void dm_mst_test_get_modes_cached_edid_sink_alloc_fails(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+
+	child.aconnector->drm_edid = drm_edid_alloc(dm_mst_test_edid, sizeof(dm_mst_test_edid));
+	KUNIT_ASSERT_NOT_NULL(test, child.aconnector->drm_edid);
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_get_modes(&child.aconnector->base), 0);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_add_remote_sink_calls, 1U);
+	KUNIT_EXPECT_NULL(test, child.aconnector->dc_sink);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_get_modes_restores_hdcp_properties - Test HDCP property restore
+ * @test: KUnit test context
+ *
+ * A connector re-plugged at the same display index must have its content
+ * protection state restored from the HDCP workqueue when the remote sink is
+ * re-created.
+ */
+static void dm_mst_test_get_modes_restores_hdcp_properties(struct kunit *test)
+{
+	struct dm_connector_state *conn_state;
+	struct dm_mst_test_child child;
+	struct hdcp_workqueue *hdcp;
+	struct dc_sink *sink;
+	unsigned int index;
+
+	dm_mst_test_init_child(test, &child);
+
+	child.aconnector->drm_edid = drm_edid_alloc(dm_mst_test_edid, sizeof(dm_mst_test_edid));
+	KUNIT_ASSERT_NOT_NULL(test, child.aconnector->drm_edid);
+
+	/* drm_connector_cleanup() hands the state back to the DRM helper's kfree(). */
+	conn_state = kzalloc_obj(*conn_state);
+	hdcp = kunit_kzalloc(test, sizeof(*hdcp), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+	KUNIT_ASSERT_NOT_NULL(test, hdcp);
+
+	index = child.aconnector->base.index;
+	hdcp->hdcp_content_type[index] = DRM_MODE_HDCP_CONTENT_TYPE1;
+	hdcp->content_protection[index] = DRM_MODE_CONTENT_PROTECTION_ENABLED;
+	child.adev->dm.hdcp_workqueue = hdcp;
+	child.aconnector->base.state = &conn_state->base;
+
+	sink = dm_mst_test_alloc_sink(test);
+	dm_mst_test_next_remote_sink = sink;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_get_modes(&child.aconnector->base), 0);
+
+	KUNIT_EXPECT_PTR_EQ(test, child.aconnector->dc_sink, sink);
+	KUNIT_EXPECT_EQ(test, child.aconnector->base.state->hdcp_content_type,
+			(unsigned int)DRM_MODE_HDCP_CONTENT_TYPE1);
+	KUNIT_EXPECT_EQ(test, child.aconnector->base.state->content_protection,
+			(unsigned int)DRM_MODE_CONTENT_PROTECTION_ENABLED);
+
+	dc_sink_release(sink);
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_get_modes_reads_remote_edid - Test remote EDID caching
+ * @test: KUnit test context
+ *
+ * When the MST port is still in the topology, dm_dp_mst_get_modes() must read
+ * its EDID, cache it on the connector and flag MST_REMOTE_EDID.
+ */
+static void dm_mst_test_get_modes_reads_remote_edid(struct kunit *test)
+{
+	struct drm_dp_mst_branch *mstb;
+	struct dm_mst_test_child child;
+	struct dc_sink *sink;
+
+	dm_mst_test_init_child(test, &child);
+
+	/* Minimal topology so drm_dp_mst_edid_read() can validate the port. */
+	mstb = kunit_kzalloc(test, sizeof(*mstb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mstb);
+	mstb->mgr = &child.root->mst_mgr;
+	INIT_LIST_HEAD(&mstb->ports);
+	list_add(&child.port->next, &mstb->ports);
+	kref_init(&child.port->topology_kref);
+	child.root->mst_mgr.mst_primary = mstb;
+
+	child.port->cached_edid = drm_edid_alloc(dm_mst_test_edid, sizeof(dm_mst_test_edid));
+	KUNIT_ASSERT_NOT_NULL(test, child.port->cached_edid);
+
+	sink = dm_mst_test_alloc_sink(test);
+	dm_mst_test_next_remote_sink = sink;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_get_modes(&child.aconnector->base), 0);
+
+	KUNIT_EXPECT_NOT_NULL(test, child.aconnector->drm_edid);
+	KUNIT_EXPECT_EQ(test, child.aconnector->mst_status & MST_REMOTE_EDID, (int)MST_REMOTE_EDID);
+	KUNIT_EXPECT_PTR_EQ(test, child.aconnector->dc_sink, sink);
+
+	dc_sink_release(sink);
+	dm_mst_test_fini_child(&child);
+}
+
+/* Tests for dm_dp_mst_detect */
+
+/**
+ * dm_mst_test_detect_reads_port_dpcd_rev - Test DPCD revision probing
+ * @test: KUnit test context
+ *
+ * For a peer device with an unknown DPCD revision, dm_dp_mst_detect() must probe
+ * DP_DP13_DPCD_REV over the port AUX and cache the returned revision.
+ */
+static void dm_mst_test_detect_reads_port_dpcd_rev(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+	child.port->pdt = DP_PEER_DEVICE_SST_SINK;
+	/* Both DP_DP13_DPCD_REV and DP_DPCD_REV alias to offset 0 in the fake. */
+	dm_mst_test_dpcd[0] = 0x13;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_detect(&child.aconnector->base, NULL, false),
+			(int)connector_status_disconnected);
+
+	KUNIT_EXPECT_EQ(test, (int)child.port->dpcd_rev, 0x13);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_detect_unknown_dpcd_rev - Test unreadable DPCD revision
+ * @test: KUnit test context
+ *
+ * When both DPCD revision registers read back as zero, dm_dp_mst_detect() must
+ * leave the cached revision cleared instead of reporting a bogus value.
+ */
+static void dm_mst_test_detect_unknown_dpcd_rev(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+	child.port->pdt = DP_PEER_DEVICE_SST_SINK;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_detect(&child.aconnector->base, NULL, false),
+			(int)connector_status_disconnected);
+
+	KUNIT_EXPECT_EQ(test, (int)child.port->dpcd_rev, 0);
+
+	dm_mst_test_fini_child(&child);
+}
+
 /**
- * dm_mst_test_fp_guarded_public_stubs - Test FP-off public fallbacks
+ * dm_mst_test_detect_dpcd_read_error - Test unreachable port DPCD
  * @test: KUnit test context
  *
- * When CONFIG_DRM_AMD_DC_FP is disabled, the public DSC validation helper
- * has no FP body and must return DC_OK without touching its arguments.
+ * When the remote DPCD read is NAKed, dm_dp_mst_detect() must leave the cached
+ * revision untouched instead of storing a garbage value.
  */
-static void dm_mst_test_fp_guarded_public_stubs(struct kunit *test)
+static void dm_mst_test_detect_dpcd_read_error(struct kunit *test)
 {
-	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(NULL, NULL),
-			(enum dc_status)DC_OK);
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+	child.port->pdt = DP_PEER_DEVICE_SST_SINK;
+	dm_mst_test_aux_transfer_override = -EIO;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_detect(&child.aconnector->base, NULL, false),
+			(int)connector_status_disconnected);
+
+	KUNIT_EXPECT_EQ(test, (int)child.port->dpcd_rev, 0);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_detect_disconnect_releases_sink - Test unplug sink release
+ * @test: KUnit test context
+ *
+ * A port with no peer device must have its cached DPCD revision cleared, and the
+ * resulting disconnected status must release the remote sink and reset the MST
+ * connector state.
+ */
+static void dm_mst_test_detect_disconnect_releases_sink(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+	struct dc_sink *sink;
+
+	dm_mst_test_init_child(test, &child);
+	child.port->pdt = DP_PEER_DEVICE_NONE;
+	child.port->dpcd_rev = 0x14;
+
+	sink = dm_mst_test_alloc_sink(test);
+	child.aconnector->dc_sink = sink;
+	child.aconnector->dsc_aux = &child.port->aux;
+	child.aconnector->mst_local_bw = 1234;
+	child.link->sink_count = 1;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_detect(&child.aconnector->base, NULL, false),
+			(int)connector_status_disconnected);
+
+	KUNIT_EXPECT_EQ(test, (int)child.port->dpcd_rev, 0);
+	KUNIT_EXPECT_EQ(test, dm_mst_test_remove_remote_sink_calls, 1U);
+	KUNIT_EXPECT_PTR_EQ(test, dm_mst_test_removed_sink, sink);
+	KUNIT_EXPECT_NULL(test, child.aconnector->dc_sink);
+	KUNIT_EXPECT_NULL(test, child.aconnector->dsc_aux);
+	KUNIT_EXPECT_EQ(test, child.aconnector->mst_local_bw, 0U);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/* Tests for amdgpu_dm_mst_connector_late_register */
+
+/**
+ * dm_mst_test_connector_late_register - Test MST connector late registration
+ * @test: KUnit test context
+ *
+ * amdgpu_dm_mst_connector_late_register() must register the port's remote AUX
+ * bus and report success.
+ */
+static void dm_mst_test_connector_late_register(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_mst_connector_late_register(&child.aconnector->base), 0);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/* Tests for amdgpu_dm_mst_connector_early_unregister */
+
+/**
+ * dm_mst_test_connector_early_unregister_no_sink - Test unregister without sink
+ * @test: KUnit test context
+ *
+ * With no remote sink attached, amdgpu_dm_mst_connector_early_unregister() must
+ * only reset the MST status.
+ */
+static void dm_mst_test_connector_early_unregister_no_sink(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+	child.aconnector->mst_status = MST_REMOTE_EDID;
+
+	amdgpu_dm_mst_connector_early_unregister(&child.aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_remove_remote_sink_calls, 0U);
+	KUNIT_EXPECT_EQ(test, (int)child.aconnector->mst_status, (int)MST_STATUS_DEFAULT);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_connector_early_unregister_releases_sink - Test sink release
+ * @test: KUnit test context
+ *
+ * When the port leaves the topology, amdgpu_dm_mst_connector_early_unregister()
+ * must remove the remote sink from the link and reset the MST connector state.
+ */
+static void dm_mst_test_connector_early_unregister_releases_sink(struct kunit *test)
+{
+	struct dm_mst_test_child child;
+	struct dc_sink *sink;
+
+	dm_mst_test_init_child(test, &child);
+
+	sink = dm_mst_test_alloc_sink(test);
+	child.aconnector->dc_sink = sink;
+	child.aconnector->vc_full_pbn = 42;
+	child.link->sink_count = 1;
+
+	amdgpu_dm_mst_connector_early_unregister(&child.aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_remove_remote_sink_calls, 1U);
+	KUNIT_EXPECT_PTR_EQ(test, dm_mst_test_removed_sink, sink);
+	KUNIT_EXPECT_NULL(test, child.aconnector->dc_sink);
+	KUNIT_EXPECT_EQ(test, child.aconnector->vc_full_pbn, 0U);
+	KUNIT_EXPECT_EQ(test, (int)child.aconnector->mst_status, (int)MST_STATUS_DEFAULT);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/* Tests for dm_dp_mst_connector_destroy */
+
+/*
+ * dm_dp_mst_connector_destroy() frees both the connector and the MST port, so
+ * they are allocated outside the KUnit managed allocator. The parent branch
+ * device takes an extra malloc reference so it survives the port teardown.
+ */
+static struct amdgpu_dm_connector *
+dm_mst_test_alloc_destroyable_connector(struct kunit *test, struct dm_mst_test_child *child)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_branch *mstb;
+	struct drm_dp_mst_port *port;
+	int ret;
+
+	aconnector = kzalloc_obj(*aconnector);
+	port = kzalloc_obj(*port);
+	mstb = kunit_kzalloc(test, sizeof(*mstb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	KUNIT_ASSERT_NOT_NULL(test, mstb);
+
+	mstb->mgr = &child->root->mst_mgr;
+	kref_init(&mstb->malloc_kref);
+	kref_get(&mstb->malloc_kref);
+
+	port->mgr = &child->root->mst_mgr;
+	port->parent = mstb;
+	kref_init(&port->malloc_kref);
+
+	aconnector->dc_link = child->link;
+	aconnector->mst_root = child->root;
+	aconnector->mst_output_port = port;
+
+	ret = drm_connector_init(&child->adev->ddev, &aconnector->base,
+				 &dm_mst_test_connector_funcs,
+				 DRM_MODE_CONNECTOR_DisplayPort);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	return aconnector;
+}
+
+/**
+ * dm_mst_test_connector_destroy_no_sink - Test destroy without a remote sink
+ * @test: KUnit test context
+ *
+ * dm_dp_mst_connector_destroy() must clean up the DRM connector and drop the MST
+ * port reference even when no remote sink was ever attached.
+ */
+static void dm_mst_test_connector_destroy_no_sink(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_mst_test_child child;
+
+	dm_mst_test_init_child(test, &child);
+	aconnector = dm_mst_test_alloc_destroyable_connector(test, &child);
+
+	dm_dp_mst_connector_destroy(&aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_remove_remote_sink_calls, 0U);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/**
+ * dm_mst_test_connector_destroy_releases_sink - Test destroy releases the sink
+ * @test: KUnit test context
+ *
+ * dm_dp_mst_connector_destroy() must remove the remote sink from the DC link
+ * before tearing down the connector.
+ */
+static void dm_mst_test_connector_destroy_releases_sink(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_mst_test_child child;
+	struct dc_sink *sink;
+
+	dm_mst_test_init_child(test, &child);
+	aconnector = dm_mst_test_alloc_destroyable_connector(test, &child);
+
+	sink = dm_mst_test_alloc_sink(test);
+	aconnector->dc_sink = sink;
+	child.link->sink_count = 1;
+
+	dm_dp_mst_connector_destroy(&aconnector->base);
+
+	KUNIT_EXPECT_EQ(test, dm_mst_test_remove_remote_sink_calls, 1U);
+	KUNIT_EXPECT_PTR_EQ(test, dm_mst_test_removed_sink, sink);
+
+	dm_mst_test_fini_child(&child);
+}
+
+/*
+ * Sideband connector with a live topology manager and the DOWN_REP ready bit
+ * armed, so dm_handle_mst_sideband_msg_ready_event() reaches its ack path.
+ */
+static struct amdgpu_dm_connector *
+dm_mst_test_alloc_armed_sideband_connector(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_device *adev;
+	int ret;
+
+	adev = dm_kunit_alloc_adev(test);
+	ret = drmm_mode_config_init(&adev->ddev);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	aconnector = dm_mst_test_alloc_sideband_connector(test);
+
+	ret = drm_dp_mst_topology_mgr_init(&aconnector->mst_mgr, &adev->ddev,
+					   &aconnector->dm_dp_aux.aux, 16, 4, 0);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	aconnector->mst_mgr.mst_state = true;
+	dm_mst_test_dpcd[(DP_SINK_COUNT_ESI + 1) & 0xf] = DP_DOWN_REP_MSG_RDY;
+
+	return aconnector;
+}
+
+static void dm_mst_test_free_armed_sideband_connector(struct amdgpu_dm_connector *aconnector)
+{
+	aconnector->mst_mgr.mst_state = false;
+	drm_dp_mst_topology_mgr_destroy(&aconnector->mst_mgr);
+}
+
+/**
+ * dm_mst_test_sideband_msg_ready_acks_down_rep - Test DOWN_REP ack handling
+ * @test: KUnit test context
+ *
+ * With an active topology manager and the DOWN_REP ready bit set, the sideband
+ * handler must acknowledge the event at DPCD and keep polling until the
+ * iteration limit is reached, since the fake sideband message never completes.
+ */
+static void dm_mst_test_sideband_msg_ready_acks_down_rep(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_mst_test_alloc_armed_sideband_connector(test);
+
+	dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_REP_MSG_RDY_EVENT);
+
+	KUNIT_EXPECT_EQ(test, aconnector->mst_mgr.sink_count, 0);
+
+	dm_mst_test_free_armed_sideband_connector(aconnector);
+}
+
+/**
+ * dm_mst_test_sideband_msg_ready_ack_write_fails - Test failed DPCD ack
+ * @test: KUnit test context
+ *
+ * When the DPCD acknowledge write keeps failing, the sideband handler must give
+ * up after the third retry instead of looping forever.
+ */
+static void dm_mst_test_sideband_msg_ready_ack_write_fails(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+
+	aconnector = dm_mst_test_alloc_armed_sideband_connector(test);
+	dm_mst_test_aux_write_override = -EIO;
+
+	dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_REP_MSG_RDY_EVENT);
+
+	KUNIT_EXPECT_EQ(test, aconnector->mst_mgr.sink_count, 0);
+
+	dm_mst_test_free_armed_sideband_connector(aconnector);
+}
+
+/*
+ * Fake DPCD backing store for the DSC helpers. Unlike the sideband fake above
+ * it is addressed absolutely, because these helpers read offsets spread across
+ * the whole map (0x60 DSC caps, 0x100 link settings, 0x500 branch descriptor).
+ */
+#define DM_MST_TEST_DSC_DPCD_SIZE 0x600
+
+static u8 dm_mst_test_dsc_dpcd[DM_MST_TEST_DSC_DPCD_SIZE];
+static struct drm_dp_aux *dm_mst_test_dsc_aux_fail;
+
+static ssize_t dm_mst_test_dsc_aux_transfer(struct drm_dp_aux *aux,
+					    struct drm_dp_aux_msg *msg)
+{
+	size_t i;
+
+	if (aux == dm_mst_test_dsc_aux_fail)
+		return -EIO;
+
+	if (msg->address + msg->size > DM_MST_TEST_DSC_DPCD_SIZE)
+		return -EINVAL;
+
+	msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+
+	if ((msg->request & ~DP_AUX_I2C_MOT) == DP_AUX_NATIVE_WRITE)
+		return msg->size;
+
+	for (i = 0; i < msg->size; i++)
+		((u8 *)msg->buffer)[i] = dm_mst_test_dsc_dpcd[msg->address + i];
+
+	return msg->size;
+}
+
+/* Clears the shared store; call once per test before initialising any AUX. */
+static void dm_mst_test_reset_dsc_dpcd(void)
+{
+	memset(dm_mst_test_dsc_dpcd, 0, sizeof(dm_mst_test_dsc_dpcd));
+	dm_mst_test_dsc_aux_fail = NULL;
+}
+
+static void dm_mst_test_init_dsc_aux(struct drm_dp_aux *aux, const char *name)
+{
+	aux->name = name;
+	aux->transfer = dm_mst_test_dsc_aux_transfer;
+	drm_dp_aux_init(aux);
+	drm_dp_dpcd_set_probe(aux, false);
+}
+
+static struct drm_dp_aux *dm_mst_test_alloc_dsc_aux(struct kunit *test, const char *name)
+{
+	struct drm_dp_aux *aux = kunit_kzalloc(test, sizeof(*aux), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, aux);
+	dm_mst_test_init_dsc_aux(aux, name);
+
+	return aux;
+}
+
+/* Tests for dp_get_link_current_set_bw */
+
+/*
+ * Program DPCD 0x100..0x10f, the 16-byte window the helper reads in one go.
+ * @coding is the raw DP_MAIN_LINK_CHANNEL_CODING_SET byte at 0x108.
+ */
+static void dm_mst_test_set_link_settings(u8 link_bw_set, u8 lane_count, u8 coding)
+{
+	dm_mst_test_dsc_dpcd[DP_LINK_BW_SET] = link_bw_set;
+	dm_mst_test_dsc_dpcd[DP_LANE_COUNT_SET] = lane_count;
+	dm_mst_test_dsc_dpcd[DP_MAIN_LINK_CHANNEL_CODING_SET] = coding;
+}
+
+struct dm_mst_link_bw_param {
+	const char *name;
+	u8 link_bw_set;
+	u8 lane_count;
+	u8 coding;
+	bool aux_fails;
+	bool supported;
+	uint32_t cur_link_bw;
+};
+
+static const struct dm_mst_link_bw_param dm_mst_link_bw_params[] = {
+	{ "hbr2_8b_10b", DP_LINK_BW_5_4, 4, DP_8b_10b_ENCODING, false, true, 16761600 },
+	{ "uhbr10", DP_LINK_BW_10, 4, DP_128b_132b_ENCODING, false, true, 38564000 },
+	{ "uhbr13_5", DP_LINK_BW_13_5, 4, DP_128b_132b_ENCODING, false, true, 52061400 },
+	{ "uhbr20", DP_LINK_BW_20, 4, DP_128b_132b_ENCODING, false, true, 77128000 },
+	{ "unlisted_uhbr_rate", 0x1e, 4, DP_128b_132b_ENCODING, false, false, 0 },
+	{ "unknown_encoding", DP_LINK_BW_5_4, 4, DP_UNKNOWN_ENCODING, false, false, 0 },
+	{ "dpcd_read_error", DP_LINK_BW_5_4, 4, DP_8b_10b_ENCODING, true, false, 0 },
+};
+
+KUNIT_ARRAY_PARAM_DESC(dm_mst_link_bw, dm_mst_link_bw_params, name);
+
+/**
+ * dm_mst_test_link_current_set_bw - the current link settings are priced
+ * @test: KUnit test context
+ *
+ * For 8b/10b the raw DP_LINK_BW_SET byte is the link rate in 27MHz units, so
+ * HBR2 (0x14) yields 20 * 27000 * 10 kbps per lane scaled by the 80% data
+ * efficiency and the 97% FEC efficiency. For 128b/132b the byte instead
+ * selects a UHBR rate scaled by its own efficiency. An unlisted UHBR rate, an
+ * unknown channel coding and a short DPCD read all report no bandwidth.
+ */
+static void dm_mst_test_link_current_set_bw(struct kunit *test)
+{
+	const struct dm_mst_link_bw_param *param = test->param_value;
+	uint32_t cur_link_bw = 0xdeadbeef;
+	struct drm_dp_aux *aux;
+
+	dm_mst_test_reset_dsc_dpcd();
+	aux = dm_mst_test_alloc_dsc_aux(test, "dm_mst_test_link_bw_aux");
+	dm_mst_test_set_link_settings(param->link_bw_set, param->lane_count, param->coding);
+	if (param->aux_fails)
+		dm_mst_test_dsc_aux_fail = aux;
+
+	KUNIT_EXPECT_EQ(test, dp_get_link_current_set_bw(aux, &cur_link_bw), param->supported);
+	KUNIT_EXPECT_EQ(test, cur_link_bw, param->cur_link_bw);
+}
+
+/* Tests for is_synaptics_cascaded_panamera */
+
+/*
+ * A Panamera hub is identified by its branch device ID plus the high nibble of
+ * the fifth branch device name byte; the cascaded variant additionally reports
+ * SYNAPTICS_CASCADED_HUB_ID at DPCD 0x50e.
+ */
+static void dm_mst_test_set_panamera_ids(struct dc_link *link, u32 branch_dev_id,
+					 u8 dev_name_4, u8 cascaded_id)
+{
+	link->dpcd_caps.branch_dev_id = branch_dev_id;
+	link->dpcd_caps.branch_dev_name[4] = dev_name_4;
+	dm_mst_test_dsc_dpcd[DP_BRANCH_VENDOR_SPECIFIC_START + 2] = cascaded_id;
+}
+
+static struct drm_dp_mst_port *dm_mst_test_alloc_mgr_port(struct kunit *test)
+{
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_port *port;
+
+	mgr = kunit_kzalloc(test, sizeof(*mgr), GFP_KERNEL);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mgr);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+
+	mutex_init(&mgr->lock);
+	mgr->aux = dm_mst_test_alloc_dsc_aux(test, "dm_mst_test_mgr_aux");
+	port->mgr = mgr;
+
+	return port;
+}
+
+struct dm_mst_panamera_param {
+	const char *name;
+	u32 branch_dev_id;
+	u8 dev_name_4;
+	u8 cascaded_id;
+	bool aux_fails;
+	bool cascaded;
+};
+
+static const struct dm_mst_panamera_param dm_mst_panamera_params[] = {
+	{ "cascaded_hub", DP_BRANCH_DEVICE_ID_90CC24, 0x50,
+	  SYNAPTICS_CASCADED_HUB_ID, false, true },
+	{ "wrong_dev_id", 0x001122, 0x50, SYNAPTICS_CASCADED_HUB_ID, false, false },
+	{ "not_panamera", DP_BRANCH_DEVICE_ID_90CC24, 0x40,
+	  SYNAPTICS_CASCADED_HUB_ID, false, false },
+	{ "not_cascaded", DP_BRANCH_DEVICE_ID_90CC24, 0x50, 0, false, false },
+	{ "dpcd_read_error", DP_BRANCH_DEVICE_ID_90CC24, 0x50,
+	  SYNAPTICS_CASCADED_HUB_ID, true, false },
+};
+
+KUNIT_ARRAY_PARAM_DESC(dm_mst_panamera, dm_mst_panamera_params, name);
+
+/**
+ * dm_mst_test_synaptics_cascaded - only a cascaded Panamera hub is detected
+ * @test: KUnit test context
+ *
+ * All three conditions must hold: the Synaptics branch device ID, a Panamera
+ * device name and the cascaded hub ID in the branch vendor data. A failed
+ * vendor read leaves the hub undetected.
+ */
+static void dm_mst_test_synaptics_cascaded(struct kunit *test)
+{
+	const struct dm_mst_panamera_param *param = test->param_value;
+	struct drm_dp_mst_port *port;
+	struct dc_link *link;
+
+	dm_mst_test_reset_dsc_dpcd();
+	port = dm_mst_test_alloc_mgr_port(test);
+	link = dm_kunit_alloc_link(test);
+	dm_mst_test_set_panamera_ids(link, param->branch_dev_id, param->dev_name_4,
+				     param->cascaded_id);
+	if (param->aux_fails)
+		dm_mst_test_dsc_aux_fail = port->mgr->aux;
+
+	KUNIT_EXPECT_EQ(test, is_synaptics_cascaded_panamera(link, port), param->cascaded);
+}
+
+/* Tests for validate_dsc_caps_on_connector */
+
+struct dm_mst_test_dsc_ctx {
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_dm_connector *root;
+	struct drm_dp_mst_port *port;
+	struct dc_sink *sink;
+	struct dc_link *link;
+};
+
+/*
+ * Build an MST endpoint whose port hangs directly off the primary branch. With
+ * an all-zero DPCD, drm_dp_mst_dsc_aux_for_port() finds no DSC-capable AUX, so
+ * each test can steer the driver's own fallbacks in isolation.
+ */
+static void dm_mst_test_init_dsc_ctx(struct kunit *test, struct dm_mst_test_dsc_ctx *ctx)
+{
+	struct drm_dp_mst_branch *branch;
+	struct dc *dc;
+
+	dm_mst_test_reset_dsc_dpcd();
+
+	ctx->aconnector = kunit_kzalloc(test, sizeof(*ctx->aconnector), GFP_KERNEL);
+	ctx->root = kunit_kzalloc(test, sizeof(*ctx->root), GFP_KERNEL);
+	ctx->sink = kunit_kzalloc(test, sizeof(*ctx->sink), GFP_KERNEL);
+	branch = kunit_kzalloc(test, sizeof(*branch), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->root);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->sink);
+	KUNIT_ASSERT_NOT_NULL(test, branch);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	ctx->link = dm_kunit_alloc_link(test);
+	ctx->link->ctx = dc->ctx;
+
+	ctx->port = dm_mst_test_alloc_mgr_port(test);
+	ctx->port->parent = branch;
+	dm_mst_test_init_dsc_aux(&ctx->port->aux, "dm_mst_test_port_dsc_aux");
+	dm_mst_test_init_dsc_aux(&ctx->root->dm_dp_aux.aux, "dm_mst_test_root_dsc_aux");
+
+	ctx->aconnector->dc_link = ctx->link;
+	ctx->aconnector->dc_sink = ctx->sink;
+	ctx->aconnector->mst_output_port = ctx->port;
+	ctx->aconnector->mst_root = ctx->root;
+}
+
+/* Make needs_dsc_aux_workaround() accept the link. */
+static void dm_mst_test_arm_dsc_aux_workaround(struct dc_link *link)
+{
+	link->dpcd_caps.branch_dev_id = DP_BRANCH_DEVICE_ID_90CC24;
+	link->dpcd_caps.dpcd_rev.raw = DPCD_REV_14;
+	link->dpcd_caps.sink_count.bits.SINK_COUNT = 2;
+}
+
+/**
+ * dm_mst_test_validate_dsc_caps_no_aux - no DSC AUX means no DSC capabilities
+ * @test: KUnit test context
+ *
+ * When neither the DRM helper, the Synaptics workaround nor the cascaded hub
+ * quirk yields a DSC AUX channel, validation must fail without touching DPCD.
+ */
+static void dm_mst_test_validate_dsc_caps_no_aux(struct kunit *test)
+{
+	struct dm_mst_test_dsc_ctx ctx;
+
+	dm_mst_test_init_dsc_ctx(test, &ctx);
+
+	KUNIT_EXPECT_FALSE(test, validate_dsc_caps_on_connector(ctx.aconnector));
+	KUNIT_EXPECT_NULL(test, ctx.aconnector->dsc_aux);
+}
+
+/**
+ * dm_mst_test_validate_dsc_caps_aux_workaround - MST dock AUX fallback is used
+ * @test: KUnit test context
+ *
+ * A DSC capable dock in front of a non-DSC display leaves the DRM helper
+ * without an AUX, so the driver must fall back to the MST root's AUX and parse
+ * the DSC capabilities read from it.
+ */
+static void dm_mst_test_validate_dsc_caps_aux_workaround(struct kunit *test)
+{
+	struct dm_mst_test_dsc_ctx ctx;
+
+	dm_mst_test_init_dsc_ctx(test, &ctx);
+	dm_mst_test_arm_dsc_aux_workaround(ctx.link);
+	dm_mst_test_dsc_dpcd[DP_DSC_SUPPORT] = DP_DSC_DECOMPRESSION_IS_SUPPORTED;
+
+	KUNIT_EXPECT_TRUE(test, validate_dsc_caps_on_connector(ctx.aconnector));
+	KUNIT_EXPECT_PTR_EQ(test, ctx.aconnector->dsc_aux, &ctx.root->dm_dp_aux.aux);
+	KUNIT_EXPECT_TRUE(test, ctx.sink->dsc_caps.dsc_dec_caps.is_dsc_supported);
+}
+
+/**
+ * dm_mst_test_validate_dsc_caps_cascaded_hub - cascaded hub uses the mgr AUX
+ * @test: KUnit test context
+ *
+ * On a cascaded Synaptics Panamera hub the DSC decoder lives on the primary
+ * branch, so the topology manager's AUX must win over every other fallback.
+ */
+static void dm_mst_test_validate_dsc_caps_cascaded_hub(struct kunit *test)
+{
+	struct dm_mst_test_dsc_ctx ctx;
+
+	dm_mst_test_init_dsc_ctx(test, &ctx);
+	dm_mst_test_arm_dsc_aux_workaround(ctx.link);
+	dm_mst_test_set_panamera_ids(ctx.link, DP_BRANCH_DEVICE_ID_90CC24, 0x50,
+				     SYNAPTICS_CASCADED_HUB_ID);
+	dm_mst_test_dsc_dpcd[DP_DSC_SUPPORT] = DP_DSC_DECOMPRESSION_IS_SUPPORTED;
+
+	KUNIT_EXPECT_TRUE(test, validate_dsc_caps_on_connector(ctx.aconnector));
+	KUNIT_EXPECT_PTR_EQ(test, ctx.aconnector->dsc_aux, ctx.port->mgr->aux);
+}
+
+/**
+ * dm_mst_test_validate_dsc_caps_read_error - a failed DSC caps read is fatal
+ * @test: KUnit test context
+ */
+static void dm_mst_test_validate_dsc_caps_read_error(struct kunit *test)
+{
+	struct dm_mst_test_dsc_ctx ctx;
+
+	dm_mst_test_init_dsc_ctx(test, &ctx);
+	dm_mst_test_arm_dsc_aux_workaround(ctx.link);
+	dm_mst_test_dsc_aux_fail = &ctx.root->dm_dp_aux.aux;
+
+	KUNIT_EXPECT_FALSE(test, validate_dsc_caps_on_connector(ctx.aconnector));
+	KUNIT_EXPECT_PTR_EQ(test, ctx.aconnector->dsc_aux, &ctx.root->dm_dp_aux.aux);
+}
+
+/**
+ * dm_mst_test_validate_dsc_caps_unsupported - a sink without DSC is rejected
+ * @test: KUnit test context
+ *
+ * The AUX resolves but the sink reports no decompression support, so parsing
+ * the raw capabilities fails and the connector must not be marked DSC capable.
+ */
+static void dm_mst_test_validate_dsc_caps_unsupported(struct kunit *test)
+{
+	struct dm_mst_test_dsc_ctx ctx;
+
+	dm_mst_test_init_dsc_ctx(test, &ctx);
+	dm_mst_test_arm_dsc_aux_workaround(ctx.link);
+
+	KUNIT_EXPECT_FALSE(test, validate_dsc_caps_on_connector(ctx.aconnector));
+	KUNIT_EXPECT_FALSE(test, ctx.sink->dsc_caps.dsc_dec_caps.is_dsc_supported);
+}
+
+/* Tests for dm_dp_mst_is_port_support_mode */
+
+static uint32_t dm_mst_test_root_link_bw_kbps;
+
+static uint32_t dm_mst_test_root_bandwidth_kbps(const struct dc_link *link,
+						const struct dc_link_settings *link_settings)
+{
+	return dm_mst_test_root_link_bw_kbps;
+}
+
+struct dm_mst_test_port_mode_ctx {
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_stream_state *stream;
+	struct drm_dp_mst_port *port;
+	struct dc_sink *sink;
+	struct dc_link *link;
+	struct dc *dc;
+};
+
+/*
+ * Build the minimum needed to price a mode: a link whose root bandwidth is
+ * faked, a virtual channel sized by the port's full_pbn, and a 24bpp timing
+ * that needs pix_clk_100hz / 10 * 24 kbps uncompressed.
+ */
+static void dm_mst_test_init_port_mode_ctx(struct kunit *test,
+					   struct dm_mst_test_port_mode_ctx *ctx,
+					   uint32_t root_link_bw_kbps, u32 full_pbn,
+					   uint32_t pix_clk_100hz)
+{
+	struct link_service *link_srv;
+	struct resource_pool *res_pool;
+
+	dm_mst_test_reset_dsc_dpcd();
+
+	ctx->aconnector = kunit_kzalloc(test, sizeof(*ctx->aconnector), GFP_KERNEL);
+	ctx->sink = kunit_kzalloc(test, sizeof(*ctx->sink), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->sink);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, res_pool);
+
+	dm_mst_test_root_link_bw_kbps = root_link_bw_kbps;
+	link_srv->dp_link_bandwidth_kbps = dm_mst_test_root_bandwidth_kbps;
+
+	ctx->dc = dm_kunit_alloc_dc_with_ctx(test);
+	ctx->dc->link_srv = link_srv;
+	ctx->dc->res_pool = res_pool;
+
+	ctx->link = dm_kunit_alloc_link(test);
+	ctx->link->dc = ctx->dc;
+	ctx->link->ctx = ctx->dc->ctx;
+
+	ctx->sink->ctx = ctx->dc->ctx;
+	ctx->port = dm_mst_test_alloc_mgr_port(test);
+	ctx->port->full_pbn = full_pbn;
+
+	ctx->stream = dm_kunit_alloc_stream(test, ctx->link);
+	ctx->stream->sink = ctx->sink;
+	ctx->stream->timing.display_color_depth = COLOR_DEPTH_888;
+	ctx->stream->timing.pix_clk_100hz = pix_clk_100hz;
+
+	ctx->aconnector->dc_link = ctx->link;
+	ctx->aconnector->dc_sink = ctx->sink;
+	ctx->aconnector->mst_output_port = ctx->port;
+}
+
+/**
+ * dm_mst_test_port_mode_fits_without_dsc - sufficient bandwidth needs no DSC
+ * @test: KUnit test context
+ *
+ * When the uncompressed stream fits into the smaller of the root link and the
+ * virtual channel bandwidth, the mode is accepted before any DSC evaluation.
+ */
+static void dm_mst_test_port_mode_fits_without_dsc(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 1000000, 1000, 100000);
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream), DC_OK);
+	KUNIT_EXPECT_EQ(test, (u32)ctx.stream->timing.flags.DSC, 0U);
+}
+
+/**
+ * dm_mst_test_port_mode_no_dsc_aux - DSC is required but unavailable
+ * @test: KUnit test context
+ *
+ * A mode that exceeds the end-to-end bandwidth on a connector without a DSC
+ * AUX channel cannot be compressed, so validation must fail.
+ */
+static void dm_mst_test_port_mode_no_dsc_aux(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 100000, 10, 100000);
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream),
+			DC_FAIL_BANDWIDTH_VALIDATE);
+}
+
+/**
+ * dm_mst_test_port_mode_no_common_dsc_config - no shared DSC config is fatal
+ * @test: KUnit test context
+ *
+ * With a DSC AUX but no DSC encoder in the resource pool, source and sink share
+ * no usable DSC configuration and the mode must be rejected.
+ */
+static void dm_mst_test_port_mode_no_common_dsc_config(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 100000, 10, 100000);
+	ctx.aconnector->dsc_aux = ctx.port->mgr->aux;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream),
+			DC_FAIL_BANDWIDTH_VALIDATE);
+}
+
+/*
+ * 1920x1080@60 RGB 8bpc needs 3564000 kbps uncompressed, and roughly 1188000
+ * kbps at the 8bpp DSC policy minimum.
+ */
+#define DM_MST_TEST_DSC_PIX_CLK_100HZ	1485000
+
+/* Enough of dsc2_get_enc_caps() for the DSC policy and bandwidth maths. */
+static void dm_mst_test_dsc_get_enc_caps(struct dsc_enc_caps *dsc_enc_caps, int pixel_clock_100Hz)
+{
+	dsc_enc_caps->dsc_version = 0x21;
+	dsc_enc_caps->slice_caps.bits.NUM_SLICES_1 = 1;
+	dsc_enc_caps->slice_caps.bits.NUM_SLICES_2 = 1;
+	dsc_enc_caps->slice_caps.bits.NUM_SLICES_4 = 1;
+	dsc_enc_caps->lb_bit_depth = 13;
+	dsc_enc_caps->is_block_pred_supported = true;
+	dsc_enc_caps->color_formats.bits.RGB = 1;
+	dsc_enc_caps->color_depth.bits.COLOR_DEPTH_8_BPC = 1;
+	dsc_enc_caps->max_total_throughput_mps = 4800;
+	dsc_enc_caps->max_slice_width = 5184;
+	dsc_enc_caps->bpp_increment_div = 16;
+}
+
+static const struct dsc_funcs dm_mst_test_dsc_funcs = {
+	.dsc_get_enc_caps = dm_mst_test_dsc_get_enc_caps,
+};
+
+/* Install the DSC encoder fake and matching sink decoder capabilities. */
+static void dm_mst_test_setup_dsc_caps(struct kunit *test, struct dc *dc, struct dc_sink *sink)
+{
+	struct dsc_dec_dpcd_caps *dec = &sink->dsc_caps.dsc_dec_caps;
+	struct display_stream_compressor *dsc;
+
+	dsc = kunit_kzalloc(test, sizeof(*dsc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dsc);
+
+	dsc->funcs = &dm_mst_test_dsc_funcs;
+	dsc->ctx = dc->ctx;
+	dc->res_pool->dscs[0] = dsc;
+
+	dec->is_dsc_supported = true;
+	dec->dsc_version = 0x21;
+	dec->rc_buffer_size = 16 * 1024;
+	dec->slice_caps1.bits.NUM_SLICES_1 = 1;
+	dec->slice_caps1.bits.NUM_SLICES_2 = 1;
+	dec->slice_caps1.bits.NUM_SLICES_4 = 1;
+	dec->lb_bit_depth = 13;
+	dec->is_block_pred_supported = true;
+	dec->color_formats.bits.RGB = 1;
+	dec->color_depth.bits.COLOR_DEPTH_8_BPC = 1;
+	dec->throughput_mode_0_mps = 1000;
+	dec->throughput_mode_1_mps = 1000;
+	dec->max_slice_width = 5120;
+	dec->bpp_increment_div = 16;
+}
+
+static void dm_mst_test_set_dsc_timing(struct dc_crtc_timing *timing)
+{
+	timing->display_color_depth = COLOR_DEPTH_888;
+	timing->pixel_encoding = PIXEL_ENCODING_RGB;
+	timing->h_addressable = 1920;
+	timing->v_addressable = 1080;
+	timing->pix_clk_100hz = DM_MST_TEST_DSC_PIX_CLK_100HZ;
+}
+
+/*
+ * Give the source a DSC encoder and the sink matching decoder capabilities so
+ * is_dsc_common_config_possible() succeeds, and switch to a timing that does
+ * not fit uncompressed.
+ */
+static void dm_mst_test_enable_dsc(struct kunit *test, struct dm_mst_test_port_mode_ctx *ctx)
+{
+	struct drm_dp_mst_branch *branch;
+
+	branch = kunit_kzalloc(test, sizeof(*branch), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, branch);
+
+	dm_mst_test_setup_dsc_caps(test, ctx->dc, ctx->sink);
+	dm_mst_test_set_dsc_timing(&ctx->stream->timing);
+
+	ctx->port->parent = branch;
+	dm_mst_test_init_dsc_aux(&ctx->port->aux, "dm_mst_test_port_mode_aux");
+	ctx->aconnector->dsc_aux = ctx->port->mgr->aux;
+}
+
+/**
+ * dm_mst_test_port_mode_dsc_passthrough_fits - DSC passthrough accepts the mode
+ * @test: KUnit test context
+ *
+ * With DSC passthrough the compressed stream travels the whole path, so a
+ * minimum compression that fits the end-to-end bandwidth enables DSC.
+ */
+static void dm_mst_test_port_mode_dsc_passthrough_fits(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 2500000, 1000, 100000);
+	dm_mst_test_enable_dsc(test, &ctx);
+	ctx.port->passthrough_aux = ctx.port->mgr->aux;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream), DC_OK);
+	KUNIT_EXPECT_EQ(test, (u32)ctx.stream->timing.flags.DSC, 1U);
+}
+
+/**
+ * dm_mst_test_port_mode_dsc_passthrough_too_narrow - max compression still too big
+ * @test: KUnit test context
+ *
+ * When even the smallest DSC bitstream exceeds the end-to-end bandwidth the
+ * mode must be rejected instead of enabling DSC.
+ */
+static void dm_mst_test_port_mode_dsc_passthrough_too_narrow(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 1000000, 1000, 100000);
+	dm_mst_test_enable_dsc(test, &ctx);
+	ctx.port->passthrough_aux = ctx.port->mgr->aux;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream),
+			DC_FAIL_BANDWIDTH_VALIDATE);
+	KUNIT_EXPECT_EQ(test, (u32)ctx.stream->timing.flags.DSC, 0U);
+}
+
+/**
+ * dm_mst_test_port_mode_last_link_too_slow - the uncompressed last link is checked
+ * @test: KUnit test context
+ *
+ * DSC is decoded at the endpoint, so the last DP link still carries the
+ * uncompressed stream. Its current link settings are read from DPCD and cached
+ * on the connector, and a mode that does not fit is rejected.
+ */
+static void dm_mst_test_port_mode_last_link_too_slow(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 2500000, 1000, 100000);
+	dm_mst_test_enable_dsc(test, &ctx);
+	ctx.port->pdt = DP_PEER_DEVICE_SST_SINK;
+	/* RBR x1: 1257120 kbps, far below the 3564000 kbps the mode needs. */
+	dm_mst_test_set_link_settings(DP_LINK_BW_1_62, 1, DP_8b_10b_ENCODING);
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream),
+			DC_FAIL_BANDWIDTH_VALIDATE);
+	KUNIT_EXPECT_EQ(test, ctx.aconnector->mst_local_bw, 1257120U);
+	KUNIT_EXPECT_EQ(test, ctx.aconnector->vc_full_pbn, ctx.port->full_pbn);
+}
+
+/**
+ * dm_mst_test_port_mode_last_link_cached_bw - a cached last link bandwidth is reused
+ * @test: KUnit test context
+ *
+ * While the virtual channel allocation is unchanged the previously read link
+ * bandwidth is reused instead of going out to DPCD again.
+ */
+static void dm_mst_test_port_mode_last_link_cached_bw(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 2500000, 1000, 100000);
+	dm_mst_test_enable_dsc(test, &ctx);
+	ctx.port->pdt = DP_PEER_DEVICE_SST_SINK;
+	ctx.aconnector->vc_full_pbn = ctx.port->full_pbn;
+	ctx.aconnector->mst_local_bw = 100000;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream),
+			DC_FAIL_BANDWIDTH_VALIDATE);
+	KUNIT_EXPECT_EQ(test, ctx.aconnector->mst_local_bw, 100000U);
+}
+
+/**
+ * dm_mst_test_port_mode_last_link_synaptics_quirk - Synaptics hubs skip the check
+ * @test: KUnit test context
+ *
+ * Synaptics branch devices misreport their last link settings, so the last
+ * link bandwidth check is skipped for them and the mode is accepted.
+ */
+static void dm_mst_test_port_mode_last_link_synaptics_quirk(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 2500000, 1000, 100000);
+	dm_mst_test_enable_dsc(test, &ctx);
+	ctx.port->pdt = DP_PEER_DEVICE_SST_SINK;
+	ctx.aconnector->branch_ieee_oui = DP_BRANCH_DEVICE_ID_90CC24;
+	dm_mst_test_set_link_settings(DP_LINK_BW_1_62, 1, DP_8b_10b_ENCODING);
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream), DC_OK);
+	KUNIT_EXPECT_EQ(test, (u32)ctx.stream->timing.flags.DSC, 1U);
+}
+
+/**
+ * dm_mst_test_port_mode_upstream_vc_too_small - the upstream hop is the bottleneck
+ * @test: KUnit test context
+ *
+ * For a nested topology the virtual channel of the link before the last one
+ * caps the compressed bandwidth, and a mode that exceeds it is rejected.
+ */
+static void dm_mst_test_port_mode_upstream_vc_too_small(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+	struct drm_dp_mst_port *upstream;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 2500000, 1000, 100000);
+	dm_mst_test_enable_dsc(test, &ctx);
+	ctx.port->pdt = DP_PEER_DEVICE_NONE;
+
+	upstream = kunit_kzalloc(test, sizeof(*upstream), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, upstream);
+	/* 100 PBN is about 670950 kbps, below the DSC minimum for this mode. */
+	upstream->full_pbn = 100;
+	ctx.port->parent->port_parent = upstream;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream),
+			DC_FAIL_BANDWIDTH_VALIDATE);
+}
+
+/**
+ * dm_mst_test_port_mode_branch_throughput_exceeded - branch decoder limit applies
+ * @test: KUnit test context
+ *
+ * Even with a valid DSC configuration the branch decoder's overall throughput
+ * for RGB must be able to carry the pixel rate.
+ */
+static void dm_mst_test_port_mode_branch_throughput_exceeded(struct kunit *test)
+{
+	struct dm_mst_test_port_mode_ctx ctx;
+
+	dm_mst_test_init_port_mode_ctx(test, &ctx, 2500000, 1000, 100000);
+	dm_mst_test_enable_dsc(test, &ctx);
+	ctx.port->passthrough_aux = ctx.port->mgr->aux;
+	/* 100 Mpix/s cannot carry the 148.5 Mpix/s this mode needs. */
+	ctx.sink->dsc_caps.dsc_dec_caps.branch_overall_throughput_0_mps = 100;
+
+	KUNIT_EXPECT_EQ(test, dm_dp_mst_is_port_support_mode(ctx.aconnector, ctx.stream),
+			DC_FAIL_BANDWIDTH_VALIDATE);
+}
+
+/* Tests for get_conv_frl_bw */
+
+/* Deterministic stand-in for the DC raw-FRL-rate lookup table. */
+static uint32_t dm_mst_test_bw_kbps_from_raw_frl(uint8_t bw)
+{
+	return bw * 3000;
+}
+
+/*
+ * Wire up a DP-to-HDMI2.1 protocol converter: the PCON capability lives on the
+ * DC caps, the converter's own limit in the downstream port caps, and the sink
+ * limits in the EDID caps.
+ */
+static struct amdgpu_dm_connector *dm_mst_test_alloc_frl_connector(struct kunit *test,
+								   bool pcon_support,
+								   u8 dwn_strm_port_type,
+								   u8 max_encoded_link_bw,
+								   u8 max_frl_rate,
+								   u8 frl_dsc_max_frl_rate)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct link_service *link_srv;
+	struct dc_sink *sink;
+	struct dc_link *link;
+	struct dc *dc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+
+	link_srv->bw_kbps_from_raw_frl_link_rate_data = dm_mst_test_bw_kbps_from_raw_frl;
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	dc->link_srv = link_srv;
+	dc->caps.dp_hdmi21_pcon_support = pcon_support;
+
+	link = dm_kunit_alloc_link(test);
+	link->dc = dc;
+
+	sink->edid_caps.max_frl_rate = max_frl_rate;
+	sink->edid_caps.frl_dsc_max_frl_rate = frl_dsc_max_frl_rate;
+
+	aconnector->dc_link = link;
+	aconnector->dc_sink = sink;
+	aconnector->mst_downstream_port_caps.bytes.byte0.bits.DWN_STRM_PORTX_TYPE =
+		dwn_strm_port_type;
+	aconnector->mst_downstream_port_caps.bytes.byte2.bits.MAX_ENCODED_LINK_BW_SUPPORT =
+		max_encoded_link_bw;
+
+	return aconnector;
+}
+
+/**
+ * dm_mst_test_conv_frl_bw_no_pcon_support - DC without PCON support finds no FRL
+ * @test: KUnit test context
+ */
+static void dm_mst_test_conv_frl_bw_no_pcon_support(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	uint32_t bw_in_kbps = 0;
+	uint32_t dsc_bw_in_kbps = 0;
+
+	aconnector = dm_mst_test_alloc_frl_connector(test, false, DOWN_STREAM_DETAILED_HDMI,
+						     3, 5, 2);
+
+	KUNIT_EXPECT_FALSE(test, get_conv_frl_bw(aconnector, &bw_in_kbps, &dsc_bw_in_kbps));
+	KUNIT_EXPECT_EQ(test, bw_in_kbps, 0U);
+	KUNIT_EXPECT_EQ(test, dsc_bw_in_kbps, 0U);
+}
+
+/**
+ * dm_mst_test_conv_frl_bw_not_hdmi_port - a non-HDMI downstream port has no FRL
+ * @test: KUnit test context
+ */
+static void dm_mst_test_conv_frl_bw_not_hdmi_port(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	uint32_t bw_in_kbps = 0;
+	uint32_t dsc_bw_in_kbps = 0;
+
+	aconnector = dm_mst_test_alloc_frl_connector(test, true, DOWN_STREAM_DETAILED_DP, 3, 5, 2);
+
+	KUNIT_EXPECT_FALSE(test, get_conv_frl_bw(aconnector, &bw_in_kbps, &dsc_bw_in_kbps));
+	KUNIT_EXPECT_EQ(test, bw_in_kbps, 0U);
+}
+
+/**
+ * dm_mst_test_conv_frl_bw_sink_without_frl - a sink not reporting FRL is skipped
+ * @test: KUnit test context
+ *
+ * Without a sink FRL rate in the EDID there is no endpoint to negotiate with,
+ * so no bandwidth is reported even though the converter advertises one.
+ */
+static void dm_mst_test_conv_frl_bw_sink_without_frl(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	uint32_t bw_in_kbps = 0;
+	uint32_t dsc_bw_in_kbps = 0;
+
+	aconnector = dm_mst_test_alloc_frl_connector(test, true, DOWN_STREAM_DETAILED_HDMI,
+						     3, 0, 2);
+
+	KUNIT_EXPECT_FALSE(test, get_conv_frl_bw(aconnector, &bw_in_kbps, &dsc_bw_in_kbps));
+	KUNIT_EXPECT_EQ(test, bw_in_kbps, 0U);
+}
+
+/**
+ * dm_mst_test_conv_frl_bw_bottleneck - the converter and sink limits are combined
+ * @test: KUnit test context
+ *
+ * The reported bandwidth is the smaller of the converter and sink FRL rates,
+ * and the DSC bandwidth is further capped by the sink's DSC FRL rate.
+ */
+static void dm_mst_test_conv_frl_bw_bottleneck(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	uint32_t bw_in_kbps = 0;
+	uint32_t dsc_bw_in_kbps = 0;
+
+	aconnector = dm_mst_test_alloc_frl_connector(test, true, DOWN_STREAM_DETAILED_HDMI,
+						     3, 5, 2);
+
+	KUNIT_EXPECT_TRUE(test, get_conv_frl_bw(aconnector, &bw_in_kbps, &dsc_bw_in_kbps));
+	KUNIT_EXPECT_EQ(test, bw_in_kbps, 9000U);
+	KUNIT_EXPECT_EQ(test, dsc_bw_in_kbps, 6000U);
+}
+
+/* Tests for log_dsc_params */
+
+/**
+ * dm_mst_test_log_dsc_params - logging the fairness vars leaves them untouched
+ * @test: KUnit test context
+ *
+ * log_dsc_params() only traces, so the only observable contract is that it
+ * walks @count entries starting at @k without modifying them.
+ */
+static void dm_mst_test_log_dsc_params(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[3] = {
+		{ .pbn = 100, .dsc_enabled = false, .bpp_x16 = 160 },
+		{ .pbn = 200, .dsc_enabled = true, .bpp_x16 = 192 },
+		{ .pbn = 300, .dsc_enabled = true, .bpp_x16 = 256 },
+	};
+
+	log_dsc_params(2, vars, 1);
+
+	KUNIT_EXPECT_EQ(test, vars[1].pbn, 200);
+	KUNIT_EXPECT_EQ(test, vars[2].bpp_x16, 256);
+}
+
+/* Tests for find_crtc_index_in_state_by_stream and is_dsc_precompute_needed */
+
+struct dm_mst_test_crtc_state_ctx {
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtcs;
+	struct dm_crtc_state *crtc_states;
+	struct drm_connector *connector;
+	struct drm_connector_state *conn_state;
+};
+
+/*
+ * Hand-build an atomic state with @num_crtc CRTCs, each carrying a DM CRTC
+ * state. drm_atomic_state_alloc() would need a fully registered mode config,
+ * and the helpers under test only walk the crtcs/connectors arrays.
+ */
+static void dm_mst_test_init_crtc_state_ctx(struct kunit *test,
+					    struct dm_mst_test_crtc_state_ctx *ctx,
+					    unsigned int num_crtc)
+{
+	struct amdgpu_device *adev;
+	unsigned int i;
+
+	adev = dm_kunit_alloc_adev(test);
+	adev->ddev.mode_config.num_crtc = num_crtc;
+
+	ctx->state = kunit_kzalloc(test, sizeof(*ctx->state), GFP_KERNEL);
+	ctx->crtcs = kunit_kcalloc(test, num_crtc, sizeof(*ctx->crtcs), GFP_KERNEL);
+	ctx->crtc_states = kunit_kcalloc(test, num_crtc, sizeof(*ctx->crtc_states), GFP_KERNEL);
+	ctx->connector = kunit_kzalloc(test, sizeof(*ctx->connector), GFP_KERNEL);
+	ctx->conn_state = kunit_kzalloc(test, sizeof(*ctx->conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtcs);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc_states);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->connector);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->conn_state);
+
+	ctx->state->dev = &adev->ddev;
+	ctx->state->crtcs = kunit_kcalloc(test, num_crtc, sizeof(*ctx->state->crtcs), GFP_KERNEL);
+	ctx->state->connectors = kunit_kzalloc(test, sizeof(*ctx->state->connectors), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->crtcs);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->connectors);
+
+	for (i = 0; i < num_crtc; i++) {
+		ctx->crtcs[i].dev = &adev->ddev;
+		ctx->crtc_states[i].base.crtc = &ctx->crtcs[i];
+		ctx->state->crtcs[i].ptr = &ctx->crtcs[i];
+		ctx->state->crtcs[i].new_state = &ctx->crtc_states[i].base;
+		ctx->state->crtcs[i].old_state = &ctx->crtc_states[i].base;
+	}
+}
+
+/* Attach the context's connector to @crtc so the CRTC looks driven. */
+static void dm_mst_test_attach_connector(struct dm_mst_test_crtc_state_ctx *ctx,
+					 struct drm_crtc *crtc)
+{
+	ctx->conn_state->crtc = crtc;
+	ctx->state->connectors[0].ptr = ctx->connector;
+	ctx->state->connectors[0].new_state = ctx->conn_state;
+	ctx->state->num_connector = 1;
+}
+
+/**
+ * dm_mst_test_find_crtc_index_matches - the CRTC driving a stream is found
+ * @test: KUnit test context
+ */
+static void dm_mst_test_find_crtc_index_matches(struct kunit *test)
+{
+	struct dm_mst_test_crtc_state_ctx ctx;
+	struct dc_stream_state *stream;
+
+	dm_mst_test_init_crtc_state_ctx(test, &ctx, 3);
+	stream = dm_kunit_alloc_stream(test, NULL);
+	ctx.crtc_states[2].stream = stream;
+
+	KUNIT_EXPECT_EQ(test, find_crtc_index_in_state_by_stream(ctx.state, stream), 2);
+}
+
+/**
+ * dm_mst_test_find_crtc_index_no_match - an unknown stream yields -1
+ * @test: KUnit test context
+ */
+static void dm_mst_test_find_crtc_index_no_match(struct kunit *test)
+{
+	struct dm_mst_test_crtc_state_ctx ctx;
+	struct dc_stream_state *stream;
+
+	dm_mst_test_init_crtc_state_ctx(test, &ctx, 2);
+	stream = dm_kunit_alloc_stream(test, NULL);
+
+	KUNIT_EXPECT_EQ(test, find_crtc_index_in_state_by_stream(ctx.state, stream), -1);
+}
+
+struct dm_mst_precompute_param {
+	const char *name;
+	bool connector_attached;
+	bool has_stream;
+	enum dc_connection_type link_type;
+	bool dsc_support;
+	bool dsc_passthrough;
+	bool needed;
+};
+
+static const struct dm_mst_precompute_param dm_mst_precompute_params[] = {
+	{ "no_connector", false, true, dc_connection_mst_branch, true, false, false },
+	{ "dsc_hub", true, true, dc_connection_mst_branch, true, false, true },
+	{ "dsc_passthrough_hub", true, true, dc_connection_mst_branch, false, true, true },
+	{ "sst_link", true, true, dc_connection_single, true, false, false },
+	{ "mst_without_dsc", true, true, dc_connection_mst_branch, false, false, false },
+	{ "no_stream", true, false, dc_connection_mst_branch, true, false, false },
+};
+
+KUNIT_ARRAY_PARAM_DESC(dm_mst_precompute, dm_mst_precompute_params, name);
+
+/**
+ * dm_mst_test_dsc_precompute_needed - precompute needs a driven DSC MST hub
+ * @test: KUnit test context
+ *
+ * A DSC or DSC passthrough capable MST branch that a connector in the state
+ * drives needs its bandwidth precomputed. A CRTC no connector drives aborts
+ * the scan, and SST links or MST branches without DSC never need it.
+ */
+static void dm_mst_test_dsc_precompute_needed(struct kunit *test)
+{
+	const struct dm_mst_precompute_param *param = test->param_value;
+	struct dm_mst_test_crtc_state_ctx ctx;
+	struct dc_link *link;
+
+	dm_mst_test_init_crtc_state_ctx(test, &ctx, 1);
+	if (param->connector_attached)
+		dm_mst_test_attach_connector(&ctx, &ctx.crtcs[0]);
+
+	if (param->has_stream) {
+		link = dm_kunit_alloc_link(test);
+		link->type = param->link_type;
+		link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT =
+			param->dsc_support;
+		link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_PASSTHROUGH_SUPPORT =
+			param->dsc_passthrough;
+		ctx.crtc_states[0].stream = dm_kunit_alloc_stream(test, link);
+	}
+
+	KUNIT_EXPECT_EQ(test, is_dsc_precompute_needed(ctx.state), param->needed);
+}
+
+/* Tests for is_dsc_need_re_compute */
+
+struct dm_mst_test_recompute_ctx {
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector_state *conn_state;
+	struct drm_crtc_state *crtc_state;
+	struct dc_state *dc_state;
+	struct dc_link *link;
+	struct dc *dc;
+};
+
+/*
+ * One MST connector on one CRTC, with an empty new dc_state and an empty
+ * current dc_state. Tests populate the streams they need and adjust the CRTC
+ * state flags to select the branch under test.
+ */
+static void dm_mst_test_init_recompute_ctx(struct kunit *test,
+					   struct dm_mst_test_recompute_ctx *ctx)
+{
+	struct drm_crtc *crtc;
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->adev->ddev.mode_config.num_crtc = 1;
+
+	ctx->dc = dm_kunit_alloc_dc_with_ctx(test);
+	ctx->dc->current_state = dm_kunit_alloc_dc_state(test);
+	ctx->dc_state = dm_kunit_alloc_dc_state(test);
+
+	ctx->link = dm_kunit_alloc_link(test);
+	ctx->link->dc = ctx->dc;
+	ctx->link->type = dc_connection_mst_branch;
+
+	ctx->aconnector = dm_kunit_alloc_connector(test, ctx->adev, ctx->link);
+	ctx->conn_state = kunit_kzalloc(test, sizeof(*ctx->conn_state), GFP_KERNEL);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	ctx->crtc_state = kunit_kzalloc(test, sizeof(*ctx->crtc_state), GFP_KERNEL);
+	ctx->state = kunit_kzalloc(test, sizeof(*ctx->state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->conn_state);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state);
+
+	ctx->state->dev = &ctx->adev->ddev;
+	ctx->state->crtcs = kunit_kzalloc(test, sizeof(*ctx->state->crtcs), GFP_KERNEL);
+	ctx->state->connectors = kunit_kzalloc(test, sizeof(*ctx->state->connectors), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->crtcs);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->connectors);
+
+	crtc->dev = &ctx->adev->ddev;
+	ctx->crtc_state->crtc = crtc;
+	ctx->state->crtcs[0].ptr = crtc;
+	ctx->state->crtcs[0].new_state = ctx->crtc_state;
+	ctx->state->crtcs[0].old_state = ctx->crtc_state;
+
+	ctx->conn_state->connector = &ctx->aconnector->base;
+	ctx->conn_state->crtc = crtc;
+	ctx->state->connectors[0].ptr = &ctx->aconnector->base;
+	ctx->state->connectors[0].new_state = ctx->conn_state;
+	ctx->state->num_connector = 1;
+}
+
+static struct dc_stream_state *dm_mst_test_add_link_stream(struct kunit *test,
+							   struct dc_state *dc_state,
+							   struct dc_link *link,
+							   struct amdgpu_dm_connector *aconnector)
+{
+	struct dc_stream_state *stream = dm_kunit_alloc_stream(test, link);
+
+	KUNIT_ASSERT_LT(test, dc_state->stream_count, MAX_PIPES);
+	stream->dm_stream_context = aconnector;
+	dc_state->streams[dc_state->stream_count++] = stream;
+
+	return stream;
+}
+
+/**
+ * dm_mst_test_recompute_not_mst_branch - only MST branches are recomputed
+ * @test: KUnit test context
+ */
+static void dm_mst_test_recompute_not_mst_branch(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	ctx.link->type = dc_connection_single;
+
+	KUNIT_EXPECT_FALSE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/**
+ * dm_mst_test_recompute_legacy_hub_without_dsc - old hubs without DSC are skipped
+ * @test: KUnit test context
+ *
+ * A hub matching the no-virtual-DPCD workaround but reporting neither DSC nor
+ * DSC passthrough support cannot use MST DSC at all.
+ */
+static void dm_mst_test_recompute_legacy_hub_without_dsc(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	dm_mst_test_arm_dsc_aux_workaround(ctx.link);
+
+	KUNIT_EXPECT_FALSE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/**
+ * dm_mst_test_recompute_no_stream_on_link - no stream on the link, nothing to do
+ * @test: KUnit test context
+ *
+ * The new state only drives another link, so this hub keeps its current DSC
+ * configuration.
+ */
+static void dm_mst_test_recompute_no_stream_on_link(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dc_link *other_link;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	other_link = dm_kunit_alloc_link(test);
+	dm_mst_test_add_link_stream(test, ctx.dc_state, other_link, ctx.aconnector);
+
+	KUNIT_EXPECT_FALSE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/**
+ * dm_mst_test_recompute_on_mode_change - a modeset on the link forces a recompute
+ * @test: KUnit test context
+ */
+static void dm_mst_test_recompute_on_mode_change(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	ctx.crtc_state->enable = true;
+	ctx.crtc_state->active = true;
+	ctx.crtc_state->mode_changed = true;
+
+	KUNIT_EXPECT_TRUE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/**
+ * dm_mst_test_recompute_unchanged_stream - an untouched stream needs no recompute
+ * @test: KUnit test context
+ *
+ * The same stream is present in both the new and the current state and its
+ * CRTC reports no change, so the existing DSC configuration still applies.
+ */
+static void dm_mst_test_recompute_unchanged_stream(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	dm_mst_test_add_link_stream(test, ctx.dc->current_state, ctx.link, ctx.aconnector);
+	ctx.crtc_state->enable = true;
+	ctx.crtc_state->active = true;
+
+	KUNIT_EXPECT_FALSE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/**
+ * dm_mst_test_recompute_stream_removed - dropping a stream forces a recompute
+ * @test: KUnit test context
+ *
+ * A stream that is on the link in the current state but absent from the new
+ * state frees up bandwidth, so the remaining streams must be recomputed.
+ */
+static void dm_mst_test_recompute_stream_removed(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct amdgpu_dm_connector *gone;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	gone = dm_kunit_alloc_connector(test, ctx.adev, ctx.link);
+	dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	dm_mst_test_add_link_stream(test, ctx.dc->current_state, ctx.link, gone);
+	ctx.crtc_state->enable = true;
+	ctx.crtc_state->active = true;
+
+	KUNIT_EXPECT_TRUE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/**
+ * dm_mst_test_recompute_stream_without_connector - streams need a DM connector
+ * @test: KUnit test context
+ *
+ * A stream on the link whose DM context is not set cannot be attributed to a
+ * connector, so it does not count towards the streams on the link.
+ */
+static void dm_mst_test_recompute_stream_without_connector(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, NULL);
+
+	KUNIT_EXPECT_FALSE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/**
+ * dm_mst_test_recompute_connector_without_crtc - a disabled connector is skipped
+ * @test: KUnit test context
+ */
+static void dm_mst_test_recompute_connector_without_crtc(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	dm_mst_test_add_link_stream(test, ctx.dc->current_state, ctx.link, ctx.aconnector);
+	ctx.conn_state->crtc = NULL;
+
+	KUNIT_EXPECT_FALSE(test, is_dsc_need_re_compute(ctx.state, ctx.dc_state, ctx.link));
+}
+
+/* Tests for compute_mst_dsc_configs_for_state and pre_validate_dsc */
+
+typedef enum dc_status (*dm_mst_test_remove_stream_fn)(struct dc *dc, struct dc_state *new_ctx,
+						       struct dc_stream_state *stream);
+
+static enum dc_status dm_mst_test_remove_stream_fails(struct dc *dc, struct dc_state *new_ctx,
+						      struct dc_stream_state *stream)
+{
+	return DC_ERROR_UNEXPECTED;
+}
+
+/*
+ * Add a stream to @ctx that walks as far into the DSC config helpers as the
+ * caller allows: an MST signal, a DM connector, an MST output port and a DSC
+ * capable sink. The helpers dereference res_pool->funcs unconditionally, so
+ * the pool is always given one, carrying @remove_stream.
+ */
+static struct dc_stream_state *
+dm_mst_test_add_mst_dsc_stream(struct kunit *test, struct dm_mst_test_recompute_ctx *ctx,
+			       dm_mst_test_remove_stream_fn remove_stream)
+{
+	struct dc_stream_state *stream;
+	struct resource_pool *res_pool;
+	struct drm_dp_mst_port *port;
+	struct resource_funcs *funcs;
+	struct dc_sink *sink;
+
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	funcs = kunit_kzalloc(test, sizeof(*funcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	KUNIT_ASSERT_NOT_NULL(test, res_pool);
+	KUNIT_ASSERT_NOT_NULL(test, funcs);
+
+	funcs->remove_stream_from_ctx = remove_stream;
+	res_pool->funcs = funcs;
+	ctx->dc->res_pool = res_pool;
+	sink->dsc_caps.dsc_dec_caps.is_dsc_supported = true;
+	ctx->aconnector->dc_sink = sink;
+	ctx->aconnector->mst_output_port = port;
+
+	stream = dm_mst_test_add_link_stream(test, ctx->dc_state, ctx->link, ctx->aconnector);
+	stream->ctx = ctx->dc->ctx;
+	stream->signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+
+	return stream;
+}
+
+/**
+ * dm_mst_test_compute_configs_skips_sst - non-MST streams are not considered
+ * @test: KUnit test context
+ */
+static void dm_mst_test_compute_configs_skips_sst(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dc_stream_state *stream;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	stream = dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	stream->ctx = ctx.dc->ctx;
+	stream->signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	KUNIT_EXPECT_EQ(test, compute_mst_dsc_configs_for_state(ctx.state, ctx.dc_state, vars), 0);
+}
+
+/**
+ * dm_mst_test_compute_configs_skips_incomplete - streams without a sink are skipped
+ * @test: KUnit test context
+ *
+ * An MST stream whose connector has neither a sink nor an output port yet is
+ * not ready for DSC bandwidth sharing.
+ */
+static void dm_mst_test_compute_configs_skips_incomplete(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dc_stream_state *stream;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	stream = dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	stream->ctx = ctx.dc->ctx;
+	stream->signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+
+	KUNIT_EXPECT_EQ(test, compute_mst_dsc_configs_for_state(ctx.state, ctx.dc_state, vars), 0);
+}
+
+/**
+ * dm_mst_test_compute_configs_remove_stream_fails - a DC resource failure aborts
+ * @test: KUnit test context
+ */
+static void dm_mst_test_compute_configs_remove_stream_fails(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	dm_mst_test_add_mst_dsc_stream(test, &ctx, dm_mst_test_remove_stream_fails);
+
+	KUNIT_EXPECT_EQ(test, compute_mst_dsc_configs_for_state(ctx.state, ctx.dc_state, vars),
+			-EINVAL);
+}
+
+/**
+ * dm_mst_test_compute_configs_no_recompute - an unchanged topology is left alone
+ * @test: KUnit test context
+ *
+ * The stream is DSC capable but is_dsc_need_re_compute() reports no change, so
+ * the existing configuration is kept and no DSC resource is requested.
+ */
+static void dm_mst_test_compute_configs_no_recompute(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	ctx.link->type = dc_connection_single;
+	dm_mst_test_add_mst_dsc_stream(test, &ctx, NULL);
+
+	KUNIT_EXPECT_EQ(test, compute_mst_dsc_configs_for_state(ctx.state, ctx.dc_state, vars), 0);
+}
+
+/**
+ * dm_mst_test_pre_validate_dsc_not_needed - precompute is skipped when unneeded
+ * @test: KUnit test context
+ *
+ * Without a DSC capable MST hub in the state there is nothing to precompute,
+ * so pre_validate_dsc() must succeed without touching the DM atomic state.
+ */
+static void dm_mst_test_pre_validate_dsc_not_needed(struct kunit *test)
+{
+	struct dm_mst_test_crtc_state_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_atomic_state *dm_state = NULL;
+
+	dm_mst_test_init_crtc_state_ctx(test, &ctx, 1);
+
+	KUNIT_EXPECT_EQ(test, pre_validate_dsc(ctx.state, &dm_state, vars), 0);
+	KUNIT_EXPECT_NULL(test, dm_state);
+}
+
+/* Tests for pre_compute_mst_dsc_configs_for_state */
+
+/**
+ * dm_mst_test_pre_compute_configs_skips_sst - non-MST streams are not considered
+ * @test: KUnit test context
+ */
+static void dm_mst_test_pre_compute_configs_skips_sst(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dc_stream_state *stream;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	stream = dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	stream->ctx = ctx.dc->ctx;
+	stream->signal = SIGNAL_TYPE_DISPLAY_PORT;
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx.state, ctx.dc_state, vars), 0);
+}
+
+/**
+ * dm_mst_test_pre_compute_configs_skips_incomplete - streams without a sink are skipped
+ * @test: KUnit test context
+ */
+static void dm_mst_test_pre_compute_configs_skips_incomplete(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dc_stream_state *stream;
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	stream = dm_mst_test_add_link_stream(test, ctx.dc_state, ctx.link, ctx.aconnector);
+	stream->ctx = ctx.dc->ctx;
+	stream->signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx.state, ctx.dc_state, vars), 0);
+}
+
+/**
+ * dm_mst_test_pre_compute_configs_no_recompute - an unchanged topology is left alone
+ * @test: KUnit test context
+ *
+ * The stream is DSC capable but is_dsc_need_re_compute() reports no change, so
+ * the precompute leaves the existing configuration in place.
+ */
+static void dm_mst_test_pre_compute_configs_no_recompute(struct kunit *test)
+{
+	struct dm_mst_test_recompute_ctx ctx;
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+
+	dm_mst_test_init_recompute_ctx(test, &ctx);
+	ctx.link->type = dc_connection_single;
+	dm_mst_test_add_mst_dsc_stream(test, &ctx, NULL);
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx.state, ctx.dc_state, vars), 0);
+}
+
+/* Tests for compute_mst_dsc_configs_for_link */
+
+static const struct drm_connector_funcs dm_mst_test_dsc_link_conn_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static enum dp_link_encoding dm_mst_test_mst_encoding_format(const struct dc_link *link)
+{
+	return DP_8b_10b_ENCODING;
+}
+
+struct dm_mst_test_dsc_link_ctx {
+	struct drm_modeset_acquire_ctx acquire_ctx;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_atomic_commit *state;
+	struct dc_stream_state *stream;
+	struct drm_device *drm;
+	struct dc_state *dc_state;
+};
+
+static void dm_mst_test_dsc_link_drop_locks(void *data)
+{
+	struct dm_mst_test_dsc_link_ctx *ctx = data;
+
+	ctx->drm->mode_config.acquire_ctx = NULL;
+	drm_modeset_drop_locks(&ctx->acquire_ctx);
+	drm_modeset_acquire_fini(&ctx->acquire_ctx);
+}
+
+static void dm_mst_test_destroy_mst_mgr(void *data)
+{
+	struct drm_dp_mst_topology_mgr *mgr = data;
+
+	mgr->mst_state = false;
+	drm_dp_mst_topology_mgr_destroy(mgr);
+}
+
+/*
+ * Full fixture for the DSC bandwidth sharing loop: a real DRM pipe and atomic
+ * state (the DRM MST helpers need private object locking), a real topology
+ * manager, and one MST stream whose 1920x1080 timing does not fit
+ * uncompressed. @total_avail_slots decides which compression pass succeeds.
+ *
+ * The context is KUnit allocated because the modeset locks are dropped from a
+ * deferred action, long after the test body has returned.
+ */
+static struct dm_mst_test_dsc_link_ctx *
+dm_mst_test_alloc_dsc_link_ctx(struct kunit *test, int total_avail_slots)
+{
+	struct dm_mst_test_dsc_link_ctx *ctx;
+	struct drm_connector_state *conn_state;
+	struct drm_crtc_state *crtc_state;
+	struct link_service *link_srv;
+	struct resource_pool *res_pool;
+	struct drm_dp_mst_port *port;
+	struct drm_plane *primary;
+	struct drm_crtc *crtc;
+	struct dc_sink *sink;
+	struct dc_link *link;
+	struct device *dev;
+	struct dc *dc;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	ctx->drm = __drm_kunit_helper_alloc_drm_device(test, dev,
+						       sizeof(struct amdgpu_device),
+						       offsetof(struct amdgpu_device, ddev),
+						       DRIVER_MODESET | DRIVER_ATOMIC);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->drm);
+
+	primary = drm_kunit_helper_create_primary_plane(test, ctx->drm, NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, primary);
+	crtc = drm_kunit_helper_create_crtc(test, ctx->drm, primary, NULL, NULL, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+
+	ctx->aconnector = drmm_kzalloc(ctx->drm, sizeof(*ctx->aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconnector);
+	KUNIT_ASSERT_EQ(test,
+			drmm_connector_init(ctx->drm, &ctx->aconnector->base,
+					    &dm_mst_test_dsc_link_conn_funcs,
+					    DRM_MODE_CONNECTOR_DisplayPort, NULL), 0);
+	drm_mode_config_reset(ctx->drm);
+
+	KUNIT_ASSERT_EQ(test,
+			drm_dp_mst_topology_mgr_init(&ctx->aconnector->mst_mgr, ctx->drm,
+						     &ctx->aconnector->dm_dp_aux.aux, 16, 4,
+						     ctx->aconnector->base.base.id), 0);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_mst_test_destroy_mst_mgr,
+						  &ctx->aconnector->mst_mgr), 0);
+
+	drm_modeset_acquire_init(&ctx->acquire_ctx, 0);
+	ctx->drm->mode_config.acquire_ctx = &ctx->acquire_ctx;
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_mst_test_dsc_link_drop_locks, ctx), 0);
+
+	ctx->state = drm_kunit_helper_atomic_state_alloc(test, ctx->drm, &ctx->acquire_ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->state);
+
+	crtc_state = drm_atomic_get_crtc_state(ctx->state, crtc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+	crtc_state->enable = true;
+	crtc_state->active = true;
+	crtc_state->mode_changed = true;
+
+	conn_state = drm_atomic_get_connector_state(ctx->state, &ctx->aconnector->base);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state);
+	/* Takes the connector reference that the state teardown drops again. */
+	KUNIT_ASSERT_EQ(test, drm_atomic_set_crtc_for_connector(conn_state, crtc), 0);
+
+	ctx->mst_state = drm_atomic_get_mst_topology_state(ctx->state, &ctx->aconnector->mst_mgr);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->mst_state);
+	/* One time slot per 10 PBN keeps the slot counts small and exact. */
+	ctx->mst_state->pbn_div.full = dfixed_const(10);
+	ctx->mst_state->total_avail_slots = total_avail_slots;
+	/* Needed for the payload limit check; mst_primary stays NULL. */
+	ctx->aconnector->mst_mgr.mst_state = true;
+
+	link_srv = kunit_kzalloc(test, sizeof(*link_srv), GFP_KERNEL);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, link_srv);
+	KUNIT_ASSERT_NOT_NULL(test, res_pool);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+
+	link_srv->mst_decide_link_encoding_format = dm_mst_test_mst_encoding_format;
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	dc->link_srv = link_srv;
+	dc->res_pool = res_pool;
+	dc->current_state = dm_kunit_alloc_dc_state(test);
+
+	link = dm_kunit_alloc_link(test);
+	link->dc = dc;
+	link->ctx = dc->ctx;
+	link->type = dc_connection_mst_branch;
+
+	sink->ctx = dc->ctx;
+	dm_mst_test_setup_dsc_caps(test, dc, sink);
+
+	/* The payload allocation takes a reference, so start above zero. */
+	kref_init(&port->malloc_kref);
+	port->mgr = &ctx->aconnector->mst_mgr;
+	port->connector = &ctx->aconnector->base;
+	port->full_pbn = 2000;
+
+	ctx->aconnector->dc_link = link;
+	ctx->aconnector->dc_sink = sink;
+	ctx->aconnector->mst_output_port = port;
+
+	ctx->dc_state = dm_kunit_alloc_dc_state(test);
+	ctx->stream = dm_kunit_alloc_stream(test, link);
+	ctx->stream->ctx = dc->ctx;
+	ctx->stream->sink = sink;
+	ctx->stream->dm_stream_context = ctx->aconnector;
+	ctx->stream->signal = SIGNAL_TYPE_DISPLAY_PORT_MST;
+	dm_mst_test_set_dsc_timing(&ctx->stream->timing);
+	ctx->dc_state->streams[0] = ctx->stream;
+	ctx->dc_state->stream_count = 1;
+
+	return ctx;
+}
+
+/**
+ * dm_mst_test_dsc_link_fits_uncompressed - plenty of time slots means no DSC
+ * @test: KUnit test context
+ *
+ * The first pass allocates the uncompressed peak PBN and the MST check
+ * succeeds, so DSC is left disabled for the stream.
+ */
+static void dm_mst_test_dsc_link_fits_uncompressed(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 64);
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), 0);
+	KUNIT_EXPECT_FALSE(test, vars[0].dsc_enabled);
+	KUNIT_EXPECT_EQ(test, (u32)ctx->stream->timing.flags.DSC, 0U);
+	KUNIT_EXPECT_PTR_EQ(test, vars[0].aconnector, ctx->aconnector);
+}
+
+/**
+ * dm_mst_test_dsc_link_enables_compression - a tight link is compressed
+ * @test: KUnit test context
+ *
+ * The uncompressed allocation does not fit, so the driver falls back to
+ * maximum compression and then optimises the bits per pixel back up. DSC ends
+ * up enabled with a bpp above the policy minimum.
+ */
+static void dm_mst_test_dsc_link_enables_compression(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 30);
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), 0);
+	KUNIT_EXPECT_TRUE(test, vars[0].dsc_enabled);
+	KUNIT_EXPECT_GT(test, vars[0].bpp_x16, 0);
+	KUNIT_EXPECT_EQ(test, (u32)ctx->stream->timing.flags.DSC, 1U);
+}
+
+/**
+ * dm_mst_test_dsc_link_out_of_slots - even max compression can be too big
+ * @test: KUnit test context
+ *
+ * When the fully compressed stream still exceeds the available time slots the
+ * -ENOSPC from the MST check is propagated to the caller.
+ */
+static void dm_mst_test_dsc_link_out_of_slots(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 10);
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), -ENOSPC);
+}
+
+/**
+ * dm_mst_test_dsc_link_forced_dsc - forced DSC discards the uncompressed pass
+ * @test: KUnit test context
+ *
+ * With DSC forced on from debugfs the uncompressed allocation is not applied
+ * even though it fits. The MST check returned 0 rather than -ENOSPC, so the
+ * helper returns that success without programming a DSC config.
+ */
+static void dm_mst_test_dsc_link_forced_dsc(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 64);
+	ctx->aconnector->dsc_settings.dsc_force_enable = DSC_CLK_FORCE_ENABLE;
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), 0);
+	KUNIT_EXPECT_FALSE(test, vars[0].dsc_enabled);
+	KUNIT_EXPECT_EQ(test, (u32)ctx->stream->timing.flags.DSC, 0U);
+}
+
+/**
+ * dm_mst_test_dsc_link_forced_dsc_stays_on - forced DSC is never disabled again
+ * @test: KUnit test context
+ *
+ * Once compression is needed, try_disable_dsc() only reconsiders streams left
+ * at the default DSC clock setting, so a forced stream keeps DSC enabled.
+ */
+static void dm_mst_test_dsc_link_forced_dsc_stays_on(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 30);
+	ctx->aconnector->dsc_settings.dsc_force_enable = DSC_CLK_FORCE_ENABLE;
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), 0);
+	KUNIT_EXPECT_TRUE(test, vars[0].dsc_enabled);
+}
+
+/**
+ * dm_mst_test_dsc_link_bpp_overwrite - a forced bpp overrides the computed one
+ * @test: KUnit test context
+ */
+static void dm_mst_test_dsc_link_bpp_overwrite(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 30);
+	ctx->aconnector->dsc_settings.dsc_bits_per_pixel = 10 * 16;
+	ctx->aconnector->dsc_settings.dsc_num_slices_h = 2;
+	ctx->aconnector->dsc_settings.dsc_num_slices_v = 4;
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), 0);
+	KUNIT_EXPECT_EQ(test, ctx->stream->timing.dsc_cfg.bits_per_pixel, 10 * 16);
+	KUNIT_EXPECT_EQ(test, (u32)ctx->stream->timing.dsc_cfg.num_slices_h, 2U);
+	KUNIT_EXPECT_EQ(test, (u32)ctx->stream->timing.dsc_cfg.num_slices_v, 4U);
+}
+
+/**
+ * dm_mst_test_dsc_link_compression_disabled - forcing DSC off keeps it off
+ * @test: KUnit test context
+ *
+ * A stream whose DSC clock is force disabled is allocated its uncompressed
+ * bandwidth in the max compression pass as well.
+ */
+static void dm_mst_test_dsc_link_compression_disabled(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 30);
+	ctx->aconnector->dsc_settings.dsc_force_enable = DSC_CLK_FORCE_DISABLE;
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), -ENOSPC);
+}
+
+/**
+ * dm_mst_test_dsc_link_bpp_between_limits - spare slots raise the bits per pixel
+ * @test: KUnit test context
+ *
+ * With only part of the slack available the optimisation loop hands out a fair
+ * share of the free time slots and derives the resulting bits per pixel from
+ * the new PBN, landing between the policy minimum and maximum.
+ */
+static void dm_mst_test_dsc_link_bpp_between_limits(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {};
+	struct dm_mst_test_dsc_link_ctx *ctx;
+
+	ctx = dm_mst_test_alloc_dsc_link_ctx(test, 63);
+	/* Smaller slots leave the fair share below the full slack. */
+	ctx->mst_state->pbn_div.full = dfixed_const(4);
+
+	KUNIT_EXPECT_EQ(test, pre_compute_mst_dsc_configs_for_state(ctx->state, ctx->dc_state, vars), 0);
+	KUNIT_EXPECT_TRUE(test, vars[0].dsc_enabled);
+	KUNIT_EXPECT_GT(test, vars[0].bpp_x16, 8 * 16);
+	KUNIT_EXPECT_LT(test, vars[0].bpp_x16, 16 * 16);
 }
-#endif
 
 static struct kunit_case dm_mst_types_test_cases[] = {
 	/* needs_dsc_aux_workaround tests */
@@ -1385,6 +4067,7 @@ static struct kunit_case dm_mst_types_test_cases[] = {
 	KUNIT_CASE(dm_mst_test_aux_transfer_error_result),
 	KUNIT_CASE(dm_mst_test_aux_transfer_hpd_discon_quirk),
 	KUNIT_CASE(dm_mst_test_aux_transfer_non_ack_reply),
+	KUNIT_CASE(dm_mst_test_aux_transfer_oversized),
 	/* dm_dp_aux_fill_payload_flags tests */
 	KUNIT_CASE(dm_mst_test_fill_payload_flags_native_write),
 	KUNIT_CASE(dm_mst_test_fill_payload_flags_native_read),
@@ -1399,6 +4082,8 @@ static struct kunit_case dm_mst_types_test_cases[] = {
 	KUNIT_CASE(dm_mst_test_sideband_msg_ready_no_ready_bits),
 	KUNIT_CASE(dm_mst_test_sideband_msg_ready_read_error),
 	KUNIT_CASE(dm_mst_test_sideband_msg_ready_without_mst_state),
+	KUNIT_CASE(dm_mst_test_sideband_msg_ready_acks_down_rep),
+	KUNIT_CASE(dm_mst_test_sideband_msg_ready_ack_write_fails),
 	KUNIT_CASE(dm_mst_test_down_rep_msg_ready_wrapper),
 	/* amdgpu_dm_initialize_dp_connector tests */
 	KUNIT_CASE(dm_mst_test_initialize_dp_connector_edp),
@@ -1407,14 +4092,96 @@ static struct kunit_case dm_mst_types_test_cases[] = {
 	KUNIT_CASE(dm_mst_test_atomic_best_encoder),
 	/* dm_dp_create_fake_mst_encoders tests */
 	KUNIT_CASE(dm_mst_test_create_fake_mst_encoders),
+	/* dm_dp_add_mst_connector tests */
+	KUNIT_CASE(dm_mst_test_add_mst_connector_creates),
+	KUNIT_CASE(dm_mst_test_add_mst_connector_inherits_props),
+	KUNIT_CASE(dm_mst_test_add_mst_connector_init_fails),
 	/* dm_dp_mst_atomic_check tests */
 	KUNIT_CASE(dm_mst_test_atomic_check_no_old_crtc),
 	/* dm_dp_mst_detect tests */
 	KUNIT_CASE(dm_mst_test_detect_unregistered),
-	/* CONFIG_DRM_AMD_DC_FP disabled public paths */
-#if !defined(CONFIG_DRM_AMD_DC_FP)
-	KUNIT_CASE(dm_mst_test_fp_guarded_public_stubs),
-#endif
+	KUNIT_CASE(dm_mst_test_detect_reads_port_dpcd_rev),
+	KUNIT_CASE(dm_mst_test_detect_unknown_dpcd_rev),
+	KUNIT_CASE(dm_mst_test_detect_dpcd_read_error),
+	KUNIT_CASE(dm_mst_test_detect_disconnect_releases_sink),
+	/* dm_dp_mst_get_modes tests */
+	KUNIT_CASE(dm_mst_test_get_modes_no_edid_adds_default_sink),
+	KUNIT_CASE(dm_mst_test_get_modes_no_edid_sink_alloc_fails),
+	KUNIT_CASE(dm_mst_test_get_modes_no_edid_keeps_existing_sink),
+	KUNIT_CASE(dm_mst_test_get_modes_cached_edid_replaces_virtual_sink),
+	KUNIT_CASE(dm_mst_test_get_modes_cached_edid_sink_alloc_fails),
+	KUNIT_CASE(dm_mst_test_get_modes_restores_hdcp_properties),
+	KUNIT_CASE(dm_mst_test_get_modes_reads_remote_edid),
+	/* amdgpu_dm_mst_connector_late_register tests */
+	KUNIT_CASE(dm_mst_test_connector_late_register),
+	/* amdgpu_dm_mst_connector_early_unregister tests */
+	KUNIT_CASE(dm_mst_test_connector_early_unregister_no_sink),
+	KUNIT_CASE(dm_mst_test_connector_early_unregister_releases_sink),
+	/* dm_dp_mst_connector_destroy tests */
+	KUNIT_CASE(dm_mst_test_connector_destroy_no_sink),
+	KUNIT_CASE(dm_mst_test_connector_destroy_releases_sink),
+	/* dp_get_link_current_set_bw tests */
+	KUNIT_CASE_PARAM(dm_mst_test_link_current_set_bw, dm_mst_link_bw_gen_params),
+	/* is_synaptics_cascaded_panamera tests */
+	KUNIT_CASE_PARAM(dm_mst_test_synaptics_cascaded, dm_mst_panamera_gen_params),
+	/* validate_dsc_caps_on_connector tests */
+	KUNIT_CASE(dm_mst_test_validate_dsc_caps_no_aux),
+	KUNIT_CASE(dm_mst_test_validate_dsc_caps_aux_workaround),
+	KUNIT_CASE(dm_mst_test_validate_dsc_caps_cascaded_hub),
+	KUNIT_CASE(dm_mst_test_validate_dsc_caps_read_error),
+	KUNIT_CASE(dm_mst_test_validate_dsc_caps_unsupported),
+	/* dm_dp_mst_is_port_support_mode tests */
+	KUNIT_CASE(dm_mst_test_port_mode_fits_without_dsc),
+	KUNIT_CASE(dm_mst_test_port_mode_no_dsc_aux),
+	KUNIT_CASE(dm_mst_test_port_mode_no_common_dsc_config),
+	KUNIT_CASE(dm_mst_test_port_mode_dsc_passthrough_fits),
+	KUNIT_CASE(dm_mst_test_port_mode_dsc_passthrough_too_narrow),
+	KUNIT_CASE(dm_mst_test_port_mode_last_link_too_slow),
+	KUNIT_CASE(dm_mst_test_port_mode_last_link_cached_bw),
+	KUNIT_CASE(dm_mst_test_port_mode_last_link_synaptics_quirk),
+	KUNIT_CASE(dm_mst_test_port_mode_upstream_vc_too_small),
+	KUNIT_CASE(dm_mst_test_port_mode_branch_throughput_exceeded),
+	/* get_conv_frl_bw tests */
+	KUNIT_CASE(dm_mst_test_conv_frl_bw_no_pcon_support),
+	KUNIT_CASE(dm_mst_test_conv_frl_bw_not_hdmi_port),
+	KUNIT_CASE(dm_mst_test_conv_frl_bw_sink_without_frl),
+	KUNIT_CASE(dm_mst_test_conv_frl_bw_bottleneck),
+	/* log_dsc_params tests */
+	KUNIT_CASE(dm_mst_test_log_dsc_params),
+	/* find_crtc_index_in_state_by_stream tests */
+	KUNIT_CASE(dm_mst_test_find_crtc_index_matches),
+	KUNIT_CASE(dm_mst_test_find_crtc_index_no_match),
+	/* is_dsc_precompute_needed tests */
+	KUNIT_CASE_PARAM(dm_mst_test_dsc_precompute_needed, dm_mst_precompute_gen_params),
+	/* is_dsc_need_re_compute tests */
+	KUNIT_CASE(dm_mst_test_recompute_not_mst_branch),
+	KUNIT_CASE(dm_mst_test_recompute_legacy_hub_without_dsc),
+	KUNIT_CASE(dm_mst_test_recompute_no_stream_on_link),
+	KUNIT_CASE(dm_mst_test_recompute_on_mode_change),
+	KUNIT_CASE(dm_mst_test_recompute_unchanged_stream),
+	KUNIT_CASE(dm_mst_test_recompute_stream_removed),
+	KUNIT_CASE(dm_mst_test_recompute_stream_without_connector),
+	KUNIT_CASE(dm_mst_test_recompute_connector_without_crtc),
+	/* compute_mst_dsc_configs_for_state tests */
+	KUNIT_CASE(dm_mst_test_compute_configs_skips_sst),
+	KUNIT_CASE(dm_mst_test_compute_configs_skips_incomplete),
+	KUNIT_CASE(dm_mst_test_compute_configs_remove_stream_fails),
+	KUNIT_CASE(dm_mst_test_compute_configs_no_recompute),
+	/* pre_validate_dsc tests */
+	KUNIT_CASE(dm_mst_test_pre_validate_dsc_not_needed),
+	/* pre_compute_mst_dsc_configs_for_state tests */
+	KUNIT_CASE(dm_mst_test_pre_compute_configs_skips_sst),
+	KUNIT_CASE(dm_mst_test_pre_compute_configs_skips_incomplete),
+	KUNIT_CASE(dm_mst_test_pre_compute_configs_no_recompute),
+	/* compute_mst_dsc_configs_for_link tests */
+	KUNIT_CASE(dm_mst_test_dsc_link_fits_uncompressed),
+	KUNIT_CASE(dm_mst_test_dsc_link_enables_compression),
+	KUNIT_CASE(dm_mst_test_dsc_link_out_of_slots),
+	KUNIT_CASE(dm_mst_test_dsc_link_forced_dsc),
+	KUNIT_CASE(dm_mst_test_dsc_link_forced_dsc_stays_on),
+	KUNIT_CASE(dm_mst_test_dsc_link_bpp_overwrite),
+	KUNIT_CASE(dm_mst_test_dsc_link_compression_disabled),
+	KUNIT_CASE(dm_mst_test_dsc_link_bpp_between_limits),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c
index 23ac8ae41c680b..7422e8ba95642e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_plane_test.c
@@ -10,11 +10,14 @@
 #include <drm/drm_blend.h>
 #include "link_enc_cfg.h"
 #include "amdgpu_dm_plane.h"
+#include "amdgpu_dm_kunit_test_helpers.h"
 #include "amdgpu_rlc.h"
 #include "gc/gc_11_0_0_offset.h"
 #include "gc/gc_11_0_0_sh_mask.h"
 #include <drm/amdgpu_drm.h>
+#include <drm/drm_mode_config.h>
 #include <drm/drm_plane.h>
+#include <drm/drm_property.h>
 
 struct dm_test_dcc_cap_ctx {
 	bool callback_ret;
@@ -114,142 +117,111 @@ static void dm_test_plane_is_video_format_known_video(struct kunit *test)
 }
 
 /**
- * dm_test_fill_blending_defaults() - Verify default blending output values.
+ * dm_test_fill_blending_pixel_blend_modes() - Verify per-pixel blending modes.
  * @test: KUnit test context.
  *
- * Verify if default blending output values are used for opaque alpha and no
- * per-pixel blending.
+ * Verify if opaque alpha keeps the default outputs, and if the premultiplied
+ * and coverage modes enable per-pixel alpha for an alpha-capable format.
  */
-static void dm_test_fill_blending_defaults(struct kunit *test)
+static void dm_test_fill_blending_pixel_blend_modes(struct kunit *test)
 {
-	struct drm_plane_state state = { 0 };
-	bool per_pixel_alpha;
-	bool pre_multiplied_alpha;
-	bool global_alpha;
-	int global_alpha_value;
-
-	state.pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
-	state.alpha = 0xffff;
-
-	amdgpu_dm_plane_fill_blending_from_plane_state(&state,
-						       &per_pixel_alpha,
-						       &pre_multiplied_alpha,
-						       &global_alpha,
-						       &global_alpha_value);
-
-	KUNIT_EXPECT_FALSE(test, per_pixel_alpha);
-	KUNIT_EXPECT_TRUE(test, pre_multiplied_alpha);
-	KUNIT_EXPECT_FALSE(test, global_alpha);
-	KUNIT_EXPECT_EQ(test, global_alpha_value, 0xff);
-}
-
-/**
- * dm_test_fill_blending_premulti_alpha_format() - Verify premultiplied alpha path.
- * @test: KUnit test context.
- *
- * Verify if premultiplied mode enables per-pixel alpha for ARGB8888.
- */
-static void dm_test_fill_blending_premulti_alpha_format(struct kunit *test)
-{
-	struct drm_plane_state state = { 0 };
-	struct drm_framebuffer fb = { 0 };
-	bool per_pixel_alpha;
-	bool pre_multiplied_alpha;
-	bool global_alpha;
-	int global_alpha_value;
-
-	fb.format = drm_format_info(DRM_FORMAT_ARGB8888);
-	KUNIT_ASSERT_NOT_NULL(test, fb.format);
-
-	state.fb = &fb;
-	state.pixel_blend_mode = DRM_MODE_BLEND_PREMULTI;
-	state.alpha = 0xffff;
-
-	amdgpu_dm_plane_fill_blending_from_plane_state(&state,
-						       &per_pixel_alpha,
-						       &pre_multiplied_alpha,
-						       &global_alpha,
-						       &global_alpha_value);
-
-	KUNIT_EXPECT_TRUE(test, per_pixel_alpha);
-	KUNIT_EXPECT_TRUE(test, pre_multiplied_alpha);
-	KUNIT_EXPECT_FALSE(test, global_alpha);
-	KUNIT_EXPECT_EQ(test, global_alpha_value, 0xff);
-}
-
-/**
- * dm_test_fill_blending_coverage_alpha_format() - Verify coverage mode behavior.
- * @test: KUnit test context.
- *
- * Verify if coverage mode sets per-pixel alpha and disables
- * pre_multiplied_alpha for ARGB8888.
- */
-static void dm_test_fill_blending_coverage_alpha_format(struct kunit *test)
-{
-	struct drm_plane_state state = { 0 };
+	static const struct {
+		const char *name;
+		unsigned int blend_mode;
+		bool with_fb;
+		bool per_pixel_alpha;
+		bool pre_multiplied_alpha;
+	} cases[] = {
+		{ "defaults", DRM_MODE_BLEND_PIXEL_NONE, false, false, true },
+		{ "premultiplied", DRM_MODE_BLEND_PREMULTI, true, true, true },
+		{ "coverage", DRM_MODE_BLEND_COVERAGE, true, true, false },
+	};
 	struct drm_framebuffer fb = { 0 };
-	bool per_pixel_alpha;
-	bool pre_multiplied_alpha;
-	bool global_alpha;
-	int global_alpha_value;
+	unsigned int i;
 
 	fb.format = drm_format_info(DRM_FORMAT_ARGB8888);
 	KUNIT_ASSERT_NOT_NULL(test, fb.format);
 
-	state.fb = &fb;
-	state.pixel_blend_mode = DRM_MODE_BLEND_COVERAGE;
-	state.alpha = 0xffff;
-
-	amdgpu_dm_plane_fill_blending_from_plane_state(&state,
-						       &per_pixel_alpha,
-						       &pre_multiplied_alpha,
-						       &global_alpha,
-						       &global_alpha_value);
-
-	KUNIT_EXPECT_TRUE(test, per_pixel_alpha);
-	KUNIT_EXPECT_FALSE(test, pre_multiplied_alpha);
-	KUNIT_EXPECT_FALSE(test, global_alpha);
-	KUNIT_EXPECT_EQ(test, global_alpha_value, 0xff);
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct drm_plane_state state = { 0 };
+		bool per_pixel_alpha;
+		bool pre_multiplied_alpha;
+		bool global_alpha;
+		int global_alpha_value;
+
+		if (cases[i].with_fb)
+			state.fb = &fb;
+		state.pixel_blend_mode = cases[i].blend_mode;
+		state.alpha = 0xffff;
+
+		amdgpu_dm_plane_fill_blending_from_plane_state(&state,
+							       &per_pixel_alpha,
+							       &pre_multiplied_alpha,
+							       &global_alpha,
+							       &global_alpha_value);
+
+		KUNIT_EXPECT_EQ_MSG(test, per_pixel_alpha, cases[i].per_pixel_alpha,
+				    "%s", cases[i].name);
+		KUNIT_EXPECT_EQ_MSG(test, pre_multiplied_alpha, cases[i].pre_multiplied_alpha,
+				    "%s", cases[i].name);
+		KUNIT_EXPECT_FALSE_MSG(test, global_alpha, "%s", cases[i].name);
+		KUNIT_EXPECT_EQ_MSG(test, global_alpha_value, 0xff, "%s", cases[i].name);
+	}
 }
 
 /**
- * dm_test_fill_blending_global_alpha() - Verify global alpha conversion to 8 bits.
+ * dm_test_fill_blending_global_alpha() - Verify global alpha down-scaling.
  * @test: KUnit test context.
  *
- * Verify if global alpha is enabled and converted from 16-bit to 8-bit.
+ * Verify if global alpha is enabled and the 16-bit DRM alpha is scaled down to
+ * the 8-bit hardware field, or to the 12-bit field used by DCN 4.2.
  */
 static void dm_test_fill_blending_global_alpha(struct kunit *test)
 {
-	struct amdgpu_device *adev;
-	struct drm_plane *plane;
-	struct drm_plane_state *state;
-	bool per_pixel_alpha;
-	bool pre_multiplied_alpha;
-	bool global_alpha;
-	int global_alpha_value;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, state);
-
-	plane->dev = &adev->ddev;
-	state->plane = plane;
-	state->pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
-	state->alpha = 0x8000;
-
-	amdgpu_dm_plane_fill_blending_from_plane_state(state,
-						       &per_pixel_alpha,
-						       &pre_multiplied_alpha,
-						       &global_alpha,
-						       &global_alpha_value);
-
-	KUNIT_EXPECT_FALSE(test, per_pixel_alpha);
-	KUNIT_EXPECT_TRUE(test, pre_multiplied_alpha);
-	KUNIT_EXPECT_TRUE(test, global_alpha);
-	KUNIT_EXPECT_EQ(test, global_alpha_value, 0x80);
+	static const struct {
+		const char *name;
+		u32 dce_version;
+		int expected;
+	} cases[] = {
+		{ "8-bit field", IP_VERSION(4, 0, 0), 0x80 },
+		{ "DCN 4.2 12-bit field", IP_VERSION(4, 2, 0), 0x800 },
+	};
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct amdgpu_device *adev;
+		struct drm_plane *plane;
+		struct drm_plane_state *state;
+		bool per_pixel_alpha;
+		bool pre_multiplied_alpha;
+		bool global_alpha;
+		int global_alpha_value;
+
+		adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+		plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+		state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+		KUNIT_ASSERT_NOT_NULL(test, adev);
+		KUNIT_ASSERT_NOT_NULL(test, plane);
+		KUNIT_ASSERT_NOT_NULL(test, state);
+
+		adev->ip_versions[DCE_HWIP][0] = cases[i].dce_version;
+		plane->dev = &adev->ddev;
+		state->plane = plane;
+		state->pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
+		state->alpha = 0x8000;
+
+		amdgpu_dm_plane_fill_blending_from_plane_state(state,
+							       &per_pixel_alpha,
+							       &pre_multiplied_alpha,
+							       &global_alpha,
+							       &global_alpha_value);
+
+		KUNIT_EXPECT_FALSE_MSG(test, per_pixel_alpha, "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, pre_multiplied_alpha, "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, global_alpha, "%s", cases[i].name);
+		KUNIT_EXPECT_EQ_MSG(test, global_alpha_value, cases[i].expected,
+				    "%s", cases[i].name);
+	}
 }
 
 /**
@@ -286,18 +258,23 @@ static void dm_test_modifier_gfx9_swizzle_mode(struct kunit *test)
  * dm_test_get_plane_formats() - Verify plane format counts for key plane types.
  * @test: KUnit test context.
  *
- * Verify if returned format counts match primary, overlay, and cursor planes.
+ * Verify if returned format counts match primary, overlay, and cursor planes,
+ * and if an overlay with a DCN universal plane cap reports the RGB list
+ * instead of the overlay-only list.
  */
 static void dm_test_get_plane_formats(struct kunit *test)
 {
 	struct drm_plane *plane;
 	struct dc_plane_cap *cap;
+	struct dc_plane_cap *universal_cap;
 	uint32_t formats[32] = {0};
 
 	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
 	cap = kunit_kzalloc(test, sizeof(*cap), GFP_KERNEL);
+	universal_cap = kunit_kzalloc(test, sizeof(*universal_cap), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, plane);
 	KUNIT_ASSERT_NOT_NULL(test, cap);
+	KUNIT_ASSERT_NOT_NULL(test, universal_cap);
 
 	plane->type = DRM_PLANE_TYPE_PRIMARY;
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_plane_formats(plane, NULL, formats, 32), 14);
@@ -310,6 +287,11 @@ static void dm_test_get_plane_formats(struct kunit *test)
 	plane->type = DRM_PLANE_TYPE_OVERLAY;
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_plane_formats(plane, NULL, formats, 32), 9);
 
+	universal_cap->type = DC_PLANE_TYPE_DCN_UNIVERSAL;
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_plane_formats(plane, universal_cap, formats, 32),
+			14);
+
 	plane->type = DRM_PLANE_TYPE_CURSOR;
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_plane_formats(plane, NULL, formats, 32), 1);
 }
@@ -370,7 +352,8 @@ static void dm_test_fill_dc_scaling_info(struct kunit *test)
  * dm_test_get_min_max_dc_plane_scaling() - Verify format-specific cap selection and 1->1000 conversion.
  * @test: KUnit test context.
  *
- * Verify if min/max scaling values are correct for NV12 and XRGB8888 formats.
+ * Verify if min/max scaling values are correct for the NV12, XRGB8888 and fp16
+ * caps.
  */
 static void dm_test_get_min_max_dc_plane_scaling(struct kunit *test)
 {
@@ -392,6 +375,8 @@ static void dm_test_get_min_max_dc_plane_scaling(struct kunit *test)
 	dc->caps.planes[0].max_downscale_factor.nv12 = 1;
 	dc->caps.planes[0].max_upscale_factor.argb8888 = 1600;
 	dc->caps.planes[0].max_downscale_factor.argb8888 = 250;
+	dc->caps.planes[0].max_upscale_factor.fp16 = 2000;
+	dc->caps.planes[0].max_downscale_factor.fp16 = 500;
 
 	fb->format = drm_format_info(DRM_FORMAT_NV12);
 	KUNIT_ASSERT_NOT_NULL(test, fb->format);
@@ -404,13 +389,20 @@ static void dm_test_get_min_max_dc_plane_scaling(struct kunit *test)
 	amdgpu_dm_plane_get_min_max_dc_plane_scaling(&adev->ddev, fb, &min_downscale, &max_upscale);
 	KUNIT_EXPECT_EQ(test, min_downscale, 250);
 	KUNIT_EXPECT_EQ(test, max_upscale, 1600);
+
+	fb->format = drm_format_info(DRM_FORMAT_ARGB16161616F);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+	amdgpu_dm_plane_get_min_max_dc_plane_scaling(&adev->ddev, fb, &min_downscale, &max_upscale);
+	KUNIT_EXPECT_EQ(test, min_downscale, 500);
+	KUNIT_EXPECT_EQ(test, max_upscale, 2000);
 }
 
 /**
  * dm_test_get_cursor_position() - Verify cursor clipping and off-screen handling.
  * @test: KUnit test context.
  *
- * Verify if cursor clipping, hotspot adjustment, and off-screen disable behavior work.
+ * Verify if cursor clipping, hotspot adjustment, off-screen disable behavior
+ * and the oversized-cursor rejection work.
  */
 static void dm_test_get_cursor_position(struct kunit *test)
 {
@@ -461,6 +453,11 @@ static void dm_test_get_cursor_position(struct kunit *test)
 			amdgpu_dm_plane_get_cursor_position(plane, &amdgpu_crtc->base, &position),
 			0);
 	KUNIT_EXPECT_FALSE(test, position.enable);
+
+	state->crtc_w = 128;
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_cursor_position(plane, &amdgpu_crtc->base, &position),
+			-EINVAL);
 }
 
 /**
@@ -503,65 +500,11 @@ static void dm_test_format_mod_supported(struct kunit *test)
 
 	KUNIT_EXPECT_FALSE(test,
 			   amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_NV12, listed_mod));
-}
 
-/**
- * dm_test_fill_gfx12_plane_attributes_from_modifiers() - Verify GFX12 DCC mapping path.
- * @test: KUnit test context.
- *
- * Verify if GFX12 modifier parsing enables DCC and sets expected DCC block mode.
- */
-static void dm_test_fill_gfx12_plane_attributes_from_modifiers(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {
-		.callback_ret = true,
-		.capable = true,
-		.output_independent_64b_blks = false,
-	};
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, afb);
-
-	adev->family = AMDGPU_FAMILY_GC_12_0_0;
-	adev->dm.dc = dc;
-	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
-	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
-	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 256;
-	adev->gfx.config.gb_addr_config_fields.num_se = 1;
-	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 2;
-	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1;
-	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
-	dm_test_dcc_ctx = &ctx;
-
-	afb->base.modifier = AMD_FMT_MOD |
-			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
-			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
-			     AMD_FMT_MOD_SET(DCC, 1) |
-			     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, 1);
-	plane_size.surface_size.width = 1920;
-	plane_size.surface_size.height = 1080;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers(
-			adev, afb, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
-			ROTATION_ANGLE_0, &plane_size, &tiling_info, &dcc, &address),
-			0);
-	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxAddr3);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_128b);
-
-	dm_test_dcc_ctx = NULL;
+	/* 4 bytes per pixel, but its extra plane collides with the DCC metadata plane. */
+	KUNIT_EXPECT_FALSE(test,
+			   amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_XRGB8888_A8,
+								listed_mod));
 }
 
 /**
@@ -607,191 +550,155 @@ static void dm_test_fill_gfx9_plane_attributes_from_modifiers(struct kunit *test
 }
 
 /**
- * dm_test_helper_check_state_viewport_reject() - Verify viewport outside screen rejects state.
- * @test: KUnit test context.
- *
- * Verify if plane state is rejected when the viewport is outside display bounds.
- */
-static void dm_test_helper_check_state_viewport_reject(struct kunit *test)
-{
-	struct drm_plane *plane;
-	struct drm_plane_state *state;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_framebuffer *fb;
-
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
-	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
-	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, state);
-	KUNIT_ASSERT_NOT_NULL(test, crtc);
-	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
-	KUNIT_ASSERT_NOT_NULL(test, fb);
-
-	plane->type = DRM_PLANE_TYPE_OVERLAY;
-	state->plane = plane;
-	state->fb = fb;
-	state->crtc = crtc;
-	state->crtc_x = 200;
-	state->crtc_y = 0;
-	state->crtc_w = 100;
-	state->crtc_h = 100;
-	new_crtc_state->mode.crtc_hdisplay = 100;
-	new_crtc_state->mode.crtc_vdisplay = 100;
-
-	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_helper_check_state(state, new_crtc_state), -EINVAL);
-}
-
-/**
- * dm_test_validate_dcc_disabled_returns_success() - Verify disabled DCC is accepted.
- * @test: KUnit test context.
- *
- * Verify if DCC validation succeeds when DCC is disabled.
- */
-static void dm_test_validate_dcc_disabled_returns_success(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct plane_size plane_size = {0};
-
-	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
-					 &plane_size, test);
-	dcc.enable = 0;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
-			ROTATION_ANGLE_0, &tiling_info, &dcc,
-			&address, &plane_size),
-			0);
-}
-
-/**
- * dm_test_validate_dcc_video_non_gfx12_fails() - Verify video format restriction on pre-GFX12.
- * @test: KUnit test context.
- *
- * Verify if video format DCC validation fails on non-GFX12 devices.
- */
-static void dm_test_validate_dcc_video_non_gfx12_fails(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct plane_size plane_size = {0};
-
-	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
-					 &plane_size, test);
-	adev->family = AMDGPU_FAMILY_NV;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
-			ROTATION_ANGLE_0, &tiling_info, &dcc,
-			&address, &plane_size),
-			-EINVAL);
-}
-
-/**
- * dm_test_validate_dcc_missing_cap_func_fails() - Verify missing capability callback fails.
+ * dm_test_helper_check_state_viewport_rejects() - Verify viewport rejections.
  * @test: KUnit test context.
  *
- * Verify if validation fails when DCC capability callback is not provided.
+ * Verify if a viewport outside the display bounds, or one clipped by an edge
+ * to below the minimum pipe-split width or height, is rejected.
  */
-static void dm_test_validate_dcc_missing_cap_func_fails(struct kunit *test)
+static void dm_test_helper_check_state_viewport_rejects(struct kunit *test)
 {
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct plane_size plane_size = {0};
-
-	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
-					 &plane_size, test);
-	dc->cap_funcs.get_dcc_compression_cap = NULL;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
-			ROTATION_ANGLE_0, &tiling_info, &dcc,
-			&address, &plane_size),
-			-EINVAL);
+	static const struct {
+		const char *name;
+		int crtc_x;
+		int crtc_y;
+		int crtc_w;
+		int crtc_h;
+		int hdisplay;
+		int vdisplay;
+	} cases[] = {
+		{ "outside the display", 200, 0, 100, 100, 100, 100 },
+		{ "width below minimum", 0, 0, 10, 100, 1920, 1080 },
+		{ "top-clipped height", -2, -95, 100, 100, 1920, 1080 },
+		{ "bottom-clipped height", 0, 95, 100, 100, 1920, 100 },
+	};
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct drm_plane *plane;
+		struct drm_plane_state *state;
+		struct drm_crtc *crtc;
+		struct drm_crtc_state *new_crtc_state;
+		struct drm_framebuffer *fb;
+		int ret;
+
+		plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+		state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+		crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+		new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
+		fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+		KUNIT_ASSERT_NOT_NULL(test, plane);
+		KUNIT_ASSERT_NOT_NULL(test, state);
+		KUNIT_ASSERT_NOT_NULL(test, crtc);
+		KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
+		KUNIT_ASSERT_NOT_NULL(test, fb);
+
+		plane->type = DRM_PLANE_TYPE_OVERLAY;
+		state->plane = plane;
+		state->fb = fb;
+		state->crtc = crtc;
+		state->crtc_x = cases[i].crtc_x;
+		state->crtc_y = cases[i].crtc_y;
+		state->crtc_w = cases[i].crtc_w;
+		state->crtc_h = cases[i].crtc_h;
+		new_crtc_state->mode.crtc_hdisplay = cases[i].hdisplay;
+		new_crtc_state->mode.crtc_vdisplay = cases[i].vdisplay;
+
+		ret = amdgpu_dm_plane_helper_check_state(state, new_crtc_state);
+		KUNIT_EXPECT_EQ_MSG(test, ret, -EINVAL, "%s", cases[i].name);
+	}
 }
 
 /**
- * dm_test_validate_dcc_cap_callback_fails() - Verify callback failure path.
+ * dm_test_validate_dcc_early_exits() - Verify the checks before the DC query.
  * @test: KUnit test context.
  *
- * Verify if validation fails when the DCC capability callback returns false.
+ * Verify if disabled DCC is accepted without consulting DC, and if a video
+ * format on a pre-GFX12 device and a missing capability callback are rejected.
  */
-static void dm_test_validate_dcc_cap_callback_fails(struct kunit *test)
+static void dm_test_validate_dcc_early_exits(struct kunit *test)
 {
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct plane_size plane_size = {0};
-	enum surface_pixel_format format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
-	enum dc_rotation_angle rotation = ROTATION_ANGLE_0;
-	struct dm_test_dcc_cap_ctx ctx = {
-		.callback_ret = false,
-		.capable = true,
+	static const struct {
+		const char *name;
+		bool dcc_enable;
+		enum surface_pixel_format format;
+		int expected;
+	} cases[] = {
+		{ "dcc disabled", false, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888, 0 },
+		{ "video before gfx12", true, SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr, -EINVAL },
+		{ "no capability callback", true, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888, -EINVAL },
 	};
-	int ret;
-
-	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
-					 &plane_size, test);
-	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
-	dm_test_dcc_ctx = &ctx;
-
-	ret = amdgpu_dm_plane_validate_dcc(adev, format, rotation, &tiling_info,
-					   &dcc, &address, &plane_size);
-	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
-	KUNIT_EXPECT_TRUE(test, ctx.called);
-
-	dm_test_dcc_ctx = NULL;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct amdgpu_device *adev;
+		struct dc *dc;
+		struct dc_tiling_info tiling_info = {0};
+		struct dc_plane_dcc_param dcc = {0};
+		struct dc_plane_address address = {0};
+		struct plane_size plane_size = {0};
+		int ret;
+
+		dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+						 &plane_size, test);
+		dcc.enable = cases[i].dcc_enable;
+
+		ret = amdgpu_dm_plane_validate_dcc(adev, cases[i].format, ROTATION_ANGLE_0,
+						   &tiling_info, &dcc, &address, &plane_size);
+		KUNIT_EXPECT_EQ_MSG(test, ret, cases[i].expected, "%s", cases[i].name);
+	}
 }
 
 /**
- * dm_test_validate_dcc_not_capable_fails() - Verify not-capable callback output.
+ * dm_test_validate_dcc_rejects_from_cap() - Verify rejections from the DC query.
  * @test: KUnit test context.
  *
- * Verify if validation fails when the DCC capability callback reports that the
- * surface is not DCC capable.
+ * Verify if a failing capability callback, a surface reported as not DCC
+ * capable, and an independent_64b_blks mismatch are all rejected.
  */
-static void dm_test_validate_dcc_not_capable_fails(struct kunit *test)
+static void dm_test_validate_dcc_rejects_from_cap(struct kunit *test)
 {
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct plane_size plane_size = {0};
-	enum surface_pixel_format format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
-	enum dc_rotation_angle rotation = ROTATION_ANGLE_0;
-	struct dm_test_dcc_cap_ctx ctx = {
-		.callback_ret = true,
-		.capable = false,
+	static const struct {
+		const char *name;
+		bool callback_ret;
+		bool capable;
+		bool output_independent_64b_blks;
+		u32 independent_64b_blks;
+	} cases[] = {
+		{ "callback fails", false, true, false, 1 },
+		{ "not capable", true, false, false, 1 },
+		{ "independent 64B mismatch", true, true, true, 0 },
 	};
-	int ret;
-
-	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
-					 &plane_size, test);
-	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
-	dm_test_dcc_ctx = &ctx;
-
-	ret = amdgpu_dm_plane_validate_dcc(adev, format, rotation, &tiling_info,
-					   &dcc, &address, &plane_size);
-	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
-	KUNIT_EXPECT_TRUE(test, ctx.called);
-
-	dm_test_dcc_ctx = NULL;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct amdgpu_device *adev;
+		struct dc *dc;
+		struct dc_tiling_info tiling_info = {0};
+		struct dc_plane_dcc_param dcc = {0};
+		struct dc_plane_address address = {0};
+		struct plane_size plane_size = {0};
+		struct dm_test_dcc_cap_ctx ctx = {
+			.callback_ret = cases[i].callback_ret,
+			.capable = cases[i].capable,
+			.output_independent_64b_blks = cases[i].output_independent_64b_blks,
+		};
+		int ret;
+
+		dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
+						 &plane_size, test);
+		dcc.independent_64b_blks = cases[i].independent_64b_blks;
+		dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
+		dm_test_dcc_ctx = &ctx;
+
+		ret = amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
+						   ROTATION_ANGLE_0, &tiling_info, &dcc,
+						   &address, &plane_size);
+		dm_test_dcc_ctx = NULL;
+
+		KUNIT_EXPECT_EQ_MSG(test, ret, -EINVAL, "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, ctx.called, "%s", cases[i].name);
+	}
 }
 
 /**
@@ -833,47 +740,13 @@ static void dm_test_validate_dcc_success_and_scan_mapping(struct kunit *test)
 }
 
 /**
- * dm_test_validate_dcc_independent_64b_mismatch_fails() - Verify 64B compatibility check.
+ * dm_test_add_modifier_appends_and_grows() - Verify append and capacity growth.
  * @test: KUnit test context.
  *
- * Verify if validation fails when independent_64b_blks values do not match.
+ * Verify if a modifier is appended within the existing capacity, and if a full
+ * list is grown while preserving the modifiers already stored.
  */
-static void dm_test_validate_dcc_independent_64b_mismatch_fails(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct plane_size plane_size = {0};
-	struct dm_test_dcc_cap_ctx ctx = {
-		.callback_ret = true,
-		.capable = true,
-		.output_independent_64b_blks = true,
-	};
-
-	dm_test_init_validate_dcc_inputs(&adev, &dc, &tiling_info, &dcc, &address,
-					 &plane_size, test);
-	dcc.independent_64b_blks = 0;
-	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
-	dm_test_dcc_ctx = &ctx;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_validate_dcc(adev, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
-			ROTATION_ANGLE_0, &tiling_info, &dcc,
-			&address, &plane_size),
-			-EINVAL);
-
-	dm_test_dcc_ctx = NULL;
-}
-
-/**
- * dm_test_add_modifier_appends_value() - Verify one modifier append.
- * @test: KUnit test context.
- *
- * Verify if a modifier is appended and size is updated.
- */
-static void dm_test_add_modifier_appends_value(struct kunit *test)
+static void dm_test_add_modifier_appends_and_grows(struct kunit *test)
 {
 	uint64_t size = 0;
 	uint64_t cap = 2;
@@ -889,20 +762,10 @@ static void dm_test_add_modifier_appends_value(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, mods[0], 0x1234ULL);
 
 	kfree(mods);
-}
-
-/**
- * dm_test_add_modifier_grows_capacity() - Verify add triggers growth and preserves old data.
- * @test: KUnit test context.
- *
- * Verify if modifier array growth keeps old data and appends new data.
- */
-static void dm_test_add_modifier_grows_capacity(struct kunit *test)
-{
-	uint64_t size = 1;
-	uint64_t cap = 1;
-	uint64_t *mods = kmalloc_array(cap, sizeof(*mods), GFP_KERNEL);
 
+	size = 1;
+	cap = 1;
+	mods = kmalloc_array(cap, sizeof(*mods), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, mods);
 	mods[0] = 0xAAULL;
 
@@ -918,12 +781,14 @@ static void dm_test_add_modifier_grows_capacity(struct kunit *test)
 }
 
 /**
- * dm_test_add_modifier_noop_when_mods_null() - Verify helper is a no-op on NULL mods list.
+ * dm_test_add_modifier_failure_paths() - Verify the two failure paths.
  * @test: KUnit test context.
  *
- * Verify if add_modifier does nothing when the modifier list is NULL.
+ * Verify if the append is a no-op once the modifier list is NULL, and if a
+ * failed capacity growth releases and clears the list so the caller can detect
+ * the allocation failure.
  */
-static void dm_test_add_modifier_noop_when_mods_null(struct kunit *test)
+static void dm_test_add_modifier_failure_paths(struct kunit *test)
 {
 	uint64_t size = 3;
 	uint64_t cap = 7;
@@ -934,15 +799,27 @@ static void dm_test_add_modifier_noop_when_mods_null(struct kunit *test)
 	KUNIT_EXPECT_PTR_EQ(test, mods, NULL);
 	KUNIT_EXPECT_EQ(test, size, 3ULL);
 	KUNIT_EXPECT_EQ(test, cap, 7ULL);
+
+	/* Doubling this capacity overflows the kmalloc_array() size product. */
+	cap = 1ULL << 62;
+	size = cap;
+	mods = kmalloc_array(1, sizeof(*mods), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+
+	amdgpu_dm_plane_add_modifier(&mods, &size, &cap, 0x1234ULL);
+
+	KUNIT_EXPECT_PTR_EQ(test, mods, NULL);
+	KUNIT_EXPECT_EQ(test, size, 1ULL << 62);
 }
 
 /**
- * dm_test_fill_gfx9_tiling_info_from_device_pre_10_3() - Verify GFX9 field copy before 10.3.
+ * dm_test_fill_gfx9_tiling_info_from_device() - Verify GFX9 device field copy.
  * @test: KUnit test context.
  *
- * Verify if pre-10.3 device fields are copied and existing num_pkrs is kept.
+ * Verify if the device tiling fields are copied, and if num_pkrs is left
+ * untouched before GFX 10.3 but taken from the device from GFX 10.3 onwards.
  */
-static void dm_test_fill_gfx9_tiling_info_from_device_pre_10_3(struct kunit *test)
+static void dm_test_fill_gfx9_tiling_info_from_device(struct kunit *test)
 {
 	struct amdgpu_device *adev;
 	struct dc_tiling_info tiling_info = {0};
@@ -971,21 +848,6 @@ static void dm_test_fill_gfx9_tiling_info_from_device_pre_10_3(struct kunit *tes
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_rb_per_se, 2U);
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 0x5aU);
-}
-
-/**
- * dm_test_fill_gfx9_tiling_info_from_device_10_3_plus() - Verify num_pkrs update on 10.3+.
- * @test: KUnit test context.
- *
- * Verify if 10.3+ device fields are copied and num_pkrs is updated.
- */
-static void dm_test_fill_gfx9_tiling_info_from_device_10_3_plus(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc_tiling_info tiling_info = {0};
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
 
 	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
 	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
@@ -996,6 +858,8 @@ static void dm_test_fill_gfx9_tiling_info_from_device_10_3_plus(struct kunit *te
 	adev->gfx.config.gb_addr_config_fields.num_pkrs = 6;
 	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
 
+	memset(&tiling_info, 0, sizeof(tiling_info));
+
 	amdgpu_dm_plane_fill_gfx9_tiling_info_from_device(adev, &tiling_info);
 
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pipes, 2U);
@@ -1009,15 +873,18 @@ static void dm_test_fill_gfx9_tiling_info_from_device_10_3_plus(struct kunit *te
 }
 
 /**
- * dm_test_fill_gfx9_tiling_info_from_modifier_linear() - Verify non-AMD modifier keeps device values.
+ * dm_test_fill_gfx9_tiling_info_from_modifier() - Verify GFX9 modifier decoding.
  * @test: KUnit test context.
  *
- * Verify if linear modifier path keeps values from device configuration.
+ * Verify if a non-AMD modifier keeps the device values, if an AMD modifier
+ * updates the bank count on pre-NV families, and if it updates the packer
+ * count on NV and later.
  */
-static void dm_test_fill_gfx9_tiling_info_from_modifier_linear(struct kunit *test)
+static void dm_test_fill_gfx9_tiling_info_from_modifier(struct kunit *test)
 {
 	struct amdgpu_device *adev;
 	struct dc_tiling_info tiling_info = {0};
+	uint64_t modifier;
 
 	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, adev);
@@ -1043,33 +910,13 @@ static void dm_test_fill_gfx9_tiling_info_from_modifier_linear(struct kunit *tes
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_rb_per_se, 2U);
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 3U);
-}
-
-/**
- * dm_test_fill_gfx9_tiling_info_from_modifier_pre_nv() - Verify AMD modifier updates banks on pre-NV.
- * @test: KUnit test context.
- *
- * Verify if AMD modifier updates pre-NV pipe, engine, and bank fields.
- */
-static void dm_test_fill_gfx9_tiling_info_from_modifier_pre_nv(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc_tiling_info tiling_info = {0};
-	uint64_t modifier;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
 
 	adev->family = AMDGPU_FAMILY_RV;
-	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
 	adev->gfx.config.gb_addr_config_fields.num_banks = 16;
-	adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 256;
-	adev->gfx.config.gb_addr_config_fields.num_se = 2;
-	adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1;
-	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
 	adev->gfx.config.gb_addr_config_fields.num_pkrs = 7;
 	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 2, 9);
 
+	memset(&tiling_info, 0, sizeof(tiling_info));
 	tiling_info.gfx9.num_pkrs = 0x5a;
 
 	modifier = AMD_FMT_MOD |
@@ -1085,22 +932,6 @@ static void dm_test_fill_gfx9_tiling_info_from_modifier_pre_nv(struct kunit *tes
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_banks, 8U);
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.num_pkrs, 0x5aU);
 	KUNIT_EXPECT_EQ(test, tiling_info.gfx9.shaderEnable, 1U);
-}
-
-/**
- * dm_test_fill_gfx9_tiling_info_from_modifier_nv() - Verify AMD modifier updates packers on NV+.
- * @test: KUnit test context.
- *
- * Verify if AMD modifier updates NV+ pipe, engine, and packer fields.
- */
-static void dm_test_fill_gfx9_tiling_info_from_modifier_nv(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc_tiling_info tiling_info = {0};
-	uint64_t modifier;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
 
 	adev->family = AMDGPU_FAMILY_NV;
 	adev->gfx.config.gb_addr_config_fields.num_pipes = 2;
@@ -1112,6 +943,8 @@ static void dm_test_fill_gfx9_tiling_info_from_modifier_nv(struct kunit *test)
 	adev->gfx.config.gb_addr_config_fields.num_pkrs = 2;
 	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
 
+	memset(&tiling_info, 0, sizeof(tiling_info));
+
 	modifier = AMD_FMT_MOD |
 		    AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
 		    AMD_FMT_MOD_SET(PIPE_XOR_BITS, 6) |
@@ -1149,68 +982,39 @@ static void dm_test_get_format_info(struct kunit *test)
 	KUNIT_EXPECT_NOT_NULL(test, format_info);
 }
 
-/**
- * dm_test_fill_blending_global_alpha_dcn42() - Verify DCN 4.2 alpha scaling.
- * @test: KUnit test context.
- *
- * Verify if DCN 4.2 scales the 16-bit DRM alpha down by 4 bits instead of 8.
- */
-static void dm_test_fill_blending_global_alpha_dcn42(struct kunit *test)
+/* Modifier lists are terminated by DRM_FORMAT_MOD_INVALID, which is not counted. */
+static int dm_test_mods_count(const u64 *mods)
 {
-	struct amdgpu_device *adev;
-	struct drm_plane *plane;
-	struct drm_plane_state *state;
-	bool per_pixel_alpha;
-	bool pre_multiplied_alpha;
-	bool global_alpha;
-	int global_alpha_value;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, state);
+	int i = 0;
 
-	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(4, 2, 0);
-	plane->dev = &adev->ddev;
-	state->plane = plane;
-	state->pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
-	state->alpha = 0x8000;
+	while (mods[i] != DRM_FORMAT_MOD_INVALID)
+		i++;
 
-	amdgpu_dm_plane_fill_blending_from_plane_state(state,
-						       &per_pixel_alpha,
-						       &pre_multiplied_alpha,
-						       &global_alpha,
-						       &global_alpha_value);
-
-	KUNIT_EXPECT_TRUE(test, global_alpha);
-	KUNIT_EXPECT_EQ(test, global_alpha_value, 0x800);
+	return i;
 }
 
 static void dm_test_expect_mods_terminated(struct kunit *test, struct amdgpu_device *adev)
 {
 	u64 *mods = NULL;
 	int ret;
-	int i;
+	int count;
 
 	ret = amdgpu_dm_plane_get_plane_modifiers(adev, DRM_PLANE_TYPE_PRIMARY, &mods);
 	KUNIT_ASSERT_EQ(test, ret, 0);
 	KUNIT_ASSERT_NOT_NULL(test, mods);
 
-	for (i = 0; mods[i] != DRM_FORMAT_MOD_INVALID; i++)
-		;
-
-	KUNIT_EXPECT_GT(test, i, 0);
-	KUNIT_EXPECT_EQ(test, mods[i - 1], DRM_FORMAT_MOD_LINEAR);
+	count = dm_test_mods_count(mods);
+	KUNIT_EXPECT_GT(test, count, 0);
+	KUNIT_EXPECT_EQ(test, mods[count - 1], DRM_FORMAT_MOD_LINEAR);
 	kfree(mods);
 }
 
 static bool dm_test_mods_contain(const u64 *mods, u64 expected)
 {
+	int count = dm_test_mods_count(mods);
 	int i;
 
-	for (i = 0; mods[i] != DRM_FORMAT_MOD_INVALID; i++) {
+	for (i = 0; i < count; i++) {
 		if (mods[i] == expected)
 			return true;
 	}
@@ -1359,32 +1163,6 @@ static u64 dm_test_gfx11_dcc_4k_modifier(u32 pipe_xor_bits, u32 pkrs, u32 tile)
 	       AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_64B);
 }
 
-/**
- * dm_test_get_plane_formats_overlay_universal_cap() - Verify universal overlay.
- * @test: KUnit test context.
- *
- * Verify if an overlay plane with a DCN universal plane cap reports the RGB
- * format list instead of the overlay-only list.
- */
-static void dm_test_get_plane_formats_overlay_universal_cap(struct kunit *test)
-{
-	struct drm_plane *plane;
-	struct dc_plane_cap *cap;
-	u32 formats[32] = {0};
-
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	cap = kunit_kzalloc(test, sizeof(*cap), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, cap);
-
-	plane->type = DRM_PLANE_TYPE_OVERLAY;
-	cap->type = DC_PLANE_TYPE_DCN_UNIVERSAL;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_get_plane_formats(plane, cap, formats, 32),
-			14);
-}
-
 /**
  * dm_test_get_plane_modifiers_gfx9() - Verify GFX9 modifier list generation.
  * @test: KUnit test context.
@@ -1409,152 +1187,117 @@ static void dm_test_get_plane_modifiers_gfx9(struct kunit *test)
 }
 
 /**
- * dm_test_get_plane_modifiers_rv() - Verify RV modifier list generation.
+ * dm_test_get_plane_modifiers_gfx10_and_gfx12() - Verify newer family dispatch.
  * @test: KUnit test context.
  *
- * Verify if pre-Raven2 RV devices add RV-specific S-swizzle modifiers and
- * non-constant-encode DCC modifiers.
+ * Verify if the GFX10.1, GFX10.3 and GFX12 configurations each dispatch to
+ * their modifier builder and produce a terminated list.
  */
-static void dm_test_get_plane_modifiers_rv(struct kunit *test)
+static void dm_test_get_plane_modifiers_gfx10_and_gfx12(struct kunit *test)
 {
-	struct amdgpu_device *adev;
-	u64 *mods;
-	u64 dcc_mod;
-	u64 s_x_mod;
-	u64 s_mod;
-	int pipes = 2;
-	int pipe_xor_bits = 3;
-	int bank_xor_bits = 2;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
+	static const struct {
+		u32 family;
+		u32 gc_ip_version;
+		u32 num_pipes;
+		u32 num_pkrs;
+	} cases[] = {
+		{ AMDGPU_FAMILY_NV, IP_VERSION(10, 1, 0), 4, 0 },
+		{ AMDGPU_FAMILY_NV, IP_VERSION(10, 3, 0), 4, 2 },
+		{ AMDGPU_FAMILY_GC_12_0_0, 0, 0, 0 },
+	};
+	unsigned int i;
 
-	adev->family = AMDGPU_FAMILY_RV;
-	adev->asic_type = CHIP_RAVEN;
-	adev->external_rev_id = 0x80;
-	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
-	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
-	adev->gfx.config.gb_addr_config_fields.num_se = 2;
-	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct amdgpu_device *adev;
 
-	mods = dm_test_get_primary_mods(test, adev);
-	dcc_mod = AMD_FMT_MOD |
-		  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
-		  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
-		  AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
-		  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits) |
-		  AMD_FMT_MOD_SET(DCC, 1) |
-		  AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, 1) |
-		  AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_64B) |
-		  AMD_FMT_MOD_SET(DCC_CONSTANT_ENCODE, 0);
-	s_x_mod = AMD_FMT_MOD |
-		  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
-		  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
-		  AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
-		  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits);
-	s_mod = AMD_FMT_MOD |
-		AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S) |
-		AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9);
+		adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+		KUNIT_ASSERT_NOT_NULL(test, adev);
 
-	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod));
-	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod |
-							     AMD_FMT_MOD_SET(DCC_RETILE, 1) |
-							     AMD_FMT_MOD_SET(RB, 2) |
-							     AMD_FMT_MOD_SET(PIPE, pipes)));
-	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, s_x_mod));
-	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, s_mod));
+		adev->family = cases[i].family;
+		adev->ip_versions[GC_HWIP][0] = cases[i].gc_ip_version;
+		adev->gfx.config.gb_addr_config_fields.num_pipes = cases[i].num_pipes;
+		adev->gfx.config.gb_addr_config_fields.num_pkrs = cases[i].num_pkrs;
 
-	kfree(mods);
+		dm_test_expect_mods_terminated(test, adev);
+	}
 }
 
 /**
- * dm_test_get_plane_modifiers_rv_constant_encode() - Verify Raven2+ modifiers.
+ * dm_test_get_plane_modifiers_rv() - Verify RV modifier list generation.
  * @test: KUnit test context.
  *
- * Verify if Raven2 and later RV devices add the constant-encode modifier
- * variants.
+ * Verify if RV devices add the RV-specific S-swizzle modifiers, and if the
+ * constant-encode DCC variants only appear from Raven2 onwards.
  */
-static void dm_test_get_plane_modifiers_rv_constant_encode(struct kunit *test)
+static void dm_test_get_plane_modifiers_rv(struct kunit *test)
 {
-	struct amdgpu_device *adev;
-	u64 *mods;
-	u64 dcc_mod;
+	static const struct {
+		const char *name;
+		u32 external_rev_id;
+		u32 constant_encode;
+	} cases[] = {
+		{ "raven", 0x80, 0 },
+		{ "raven2", 0x81, 1 },
+	};
 	int pipes = 2;
 	int pipe_xor_bits = 3;
 	int bank_xor_bits = 2;
+	u64 s_x_mod;
+	u64 s_mod;
+	unsigned int i;
 
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-
-	adev->family = AMDGPU_FAMILY_RV;
-	adev->asic_type = CHIP_RAVEN;
-	adev->external_rev_id = 0x81;
-	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
-	adev->gfx.config.gb_addr_config_fields.num_banks = 4;
-	adev->gfx.config.gb_addr_config_fields.num_se = 2;
-	adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
-
-	mods = dm_test_get_primary_mods(test, adev);
-	dcc_mod = AMD_FMT_MOD |
+	s_x_mod = AMD_FMT_MOD |
 		  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
 		  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
 		  AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
-		  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits) |
-		  AMD_FMT_MOD_SET(DCC, 1) |
-		  AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, 1) |
-		  AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_64B) |
-		  AMD_FMT_MOD_SET(DCC_CONSTANT_ENCODE, 1);
-
-	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod));
-	KUNIT_EXPECT_TRUE(test, dm_test_mods_contain(mods, dcc_mod |
-							     AMD_FMT_MOD_SET(DCC_RETILE, 1) |
-							     AMD_FMT_MOD_SET(RB, 2) |
-							     AMD_FMT_MOD_SET(PIPE, pipes)));
-
-	kfree(mods);
-}
-
-/**
- * dm_test_get_plane_modifiers_gfx10_1() - Verify GFX10.1 modifier list generation.
- * @test: KUnit test context.
- *
- * Verify if a pre-10.3 NV family device dispatches to the GFX10.1 modifier
- * builder and produces a terminated list.
- */
-static void dm_test_get_plane_modifiers_gfx10_1(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-
-	adev->family = AMDGPU_FAMILY_NV;
-	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
-	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 1, 0);
-
-	dm_test_expect_mods_terminated(test, adev);
-}
-
-/**
- * dm_test_get_plane_modifiers_gfx10_3() - Verify GFX10.3 modifier list generation.
- * @test: KUnit test context.
- *
- * Verify if a 10.3+ NV family device dispatches to the GFX10.3 modifier
- * builder and produces a terminated list.
- */
-static void dm_test_get_plane_modifiers_gfx10_3(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-
-	adev->family = AMDGPU_FAMILY_NV;
-	adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
-	adev->gfx.config.gb_addr_config_fields.num_pkrs = 2;
-	adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+		  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits);
+	s_mod = AMD_FMT_MOD |
+		AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S) |
+		AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9);
 
-	dm_test_expect_mods_terminated(test, adev);
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct amdgpu_device *adev;
+		u64 *mods;
+		u64 dcc_mod;
+		u64 retile_mod;
+
+		adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+		KUNIT_ASSERT_NOT_NULL(test, adev);
+
+		adev->family = AMDGPU_FAMILY_RV;
+		adev->asic_type = CHIP_RAVEN;
+		adev->external_rev_id = cases[i].external_rev_id;
+		adev->gfx.config.gb_addr_config_fields.num_pipes = 4;
+		adev->gfx.config.gb_addr_config_fields.num_banks = 4;
+		adev->gfx.config.gb_addr_config_fields.num_se = 2;
+		adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 2;
+
+		mods = dm_test_get_primary_mods(test, adev);
+		dcc_mod = AMD_FMT_MOD |
+			  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX9_64K_S_X) |
+			  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9) |
+			  AMD_FMT_MOD_SET(PIPE_XOR_BITS, pipe_xor_bits) |
+			  AMD_FMT_MOD_SET(BANK_XOR_BITS, bank_xor_bits) |
+			  AMD_FMT_MOD_SET(DCC, 1) |
+			  AMD_FMT_MOD_SET(DCC_INDEPENDENT_64B, 1) |
+			  AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, AMD_FMT_MOD_DCC_BLOCK_64B) |
+			  AMD_FMT_MOD_SET(DCC_CONSTANT_ENCODE, cases[i].constant_encode);
+		retile_mod = dcc_mod |
+			     AMD_FMT_MOD_SET(DCC_RETILE, 1) |
+			     AMD_FMT_MOD_SET(RB, 2) |
+			     AMD_FMT_MOD_SET(PIPE, pipes);
+
+		KUNIT_EXPECT_TRUE_MSG(test, dm_test_mods_contain(mods, dcc_mod),
+				      "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, dm_test_mods_contain(mods, retile_mod),
+				      "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, dm_test_mods_contain(mods, s_x_mod),
+				      "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, dm_test_mods_contain(mods, s_mod),
+				      "%s", cases[i].name);
+
+		kfree(mods);
+	}
 }
 
 /**
@@ -1639,25 +1382,6 @@ static void dm_test_get_plane_modifiers_gfx11_256k_first(struct kunit *test)
 	kfree(mods);
 }
 
-/**
- * dm_test_get_plane_modifiers_gfx12() - Verify GFX12 modifier list generation.
- * @test: KUnit test context.
- *
- * Verify if the GFX12 family dispatches to the GFX12 modifier builder and
- * produces a terminated list.
- */
-static void dm_test_get_plane_modifiers_gfx12(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-
-	adev->family = AMDGPU_FAMILY_GC_12_0_0;
-
-	dm_test_expect_mods_terminated(test, adev);
-}
-
 /**
  * dm_test_fill_gfx9_plane_attributes_dcc() - Verify GFX9 DCC modifier path.
  * @test: KUnit test context.
@@ -1755,24 +1479,36 @@ static void dm_test_fill_gfx9_plane_attributes_validate_fails(struct kunit *test
 }
 
 /**
- * dm_test_fill_gfx9_plane_attributes_dcc_rbplus_64b_no_128bcl() - Verify block mode.
+ * dm_test_fill_gfx9_plane_attributes_dcc_ind_blk() - Verify DCC block modes.
  * @test: KUnit test context.
  *
- * Verify if a GFX10-RBPLUS modifier with both 64B and 128B independent block
- * bits selects the 64B-no-128BCL block mode.
+ * Verify if the independent 64B and 128B block bits of a GFX9 or GFX10-RBPLUS
+ * DCC modifier select the matching HUBP independent block mode.
  */
-static void dm_test_fill_gfx9_plane_attributes_dcc_rbplus_64b_no_128bcl(struct kunit *test)
+static void dm_test_fill_gfx9_plane_attributes_dcc_ind_blk(struct kunit *test)
 {
+	static const struct {
+		const char *name;
+		u64 tile_version;
+		bool independent_64b_blks;
+		bool independent_128b_blks;
+		int expected_ind_blk;
+	} cases[] = {
+		{ "rbplus 64B no 128BCL", AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS, true, true,
+		  hubp_ind_block_64b_no_128bcl },
+		{ "rbplus 128B", AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS, false, true,
+		  hubp_ind_block_128b },
+		{ "rbplus unconstrained", AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS, false, false,
+		  hubp_ind_block_unconstrained },
+		{ "gfx9 64B", AMD_FMT_MOD_TILE_VER_GFX9, true, false, hubp_ind_block_64b },
+		{ "gfx9 unconstrained", AMD_FMT_MOD_TILE_VER_GFX9, false, false,
+		  hubp_ind_block_unconstrained },
+	};
+	struct plane_size plane_size = {0};
 	struct amdgpu_device *adev;
-	struct dc *dc;
 	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {0};
-	u64 tile_version = AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS;
-	int ret;
+	struct dc *dc;
+	unsigned int i;
 
 	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
 	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
@@ -1781,262 +1517,62 @@ static void dm_test_fill_gfx9_plane_attributes_dcc_rbplus_64b_no_128bcl(struct k
 	KUNIT_ASSERT_NOT_NULL(test, dc);
 	KUNIT_ASSERT_NOT_NULL(test, afb);
 
-	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, true);
-	afb->base.modifier = dm_test_gfx9_dcc_modifier(tile_version, true, true);
 	plane_size.surface_size.width = 1920;
 	plane_size.surface_size.height = 1080;
 
-	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
-				 &address);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
-			(int)hubp_ind_block_64b_no_128bcl);
-
-	dm_test_dcc_ctx = NULL;
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct dc_tiling_info tiling_info = {0};
+		struct dc_plane_dcc_param dcc = {0};
+		struct dc_plane_address address = {0};
+		struct dm_test_dcc_cap_ctx ctx = {0};
+		int ret;
+
+		dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, cases[i].independent_64b_blks);
+		afb->base.modifier = dm_test_gfx9_dcc_modifier(cases[i].tile_version,
+							       cases[i].independent_64b_blks,
+							       cases[i].independent_128b_blks);
+
+		ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+					 &address);
+		dm_test_dcc_ctx = NULL;
+
+		KUNIT_EXPECT_EQ_MSG(test, ret, 0, "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, dcc.enable, "%s", cases[i].name);
+		KUNIT_EXPECT_EQ_MSG(test, (int)dcc.dcc_ind_blk, cases[i].expected_ind_blk,
+				    "%s", cases[i].name);
+	}
 }
 
 /**
- * dm_test_fill_gfx9_plane_attributes_dcc_rbplus_128b() - Verify 128B block mode.
+ * dm_test_fill_gfx12_plane_attributes_dcc_blocks() - Verify GFX12 DCC blocks.
  * @test: KUnit test context.
  *
- * Verify if a GFX10-RBPLUS modifier with only the 128B independent block bit
- * selects the 128B block mode.
+ * Verify if the max-compressed-block field of a GFX12 DCC modifier selects the
+ * matching HUBP independent block mode and independent 64B flag, and if the
+ * GFX addr3 tiling version is reported.
  */
-static void dm_test_fill_gfx9_plane_attributes_dcc_rbplus_128b(struct kunit *test)
+static void dm_test_fill_gfx12_plane_attributes_dcc_blocks(struct kunit *test)
 {
+	static const struct {
+		const char *name;
+		u64 max_compressed_block;
+		bool independent_64b_blks;
+		int expected_ind_blk;
+	} cases[] = {
+		{ "64B", 0, true, hubp_ind_block_64b },
+		{ "128B", 1, false, hubp_ind_block_128b },
+		{ "unconstrained", 2, false, hubp_ind_block_unconstrained },
+	};
+	struct dm_test_dcc_cap_ctx ctx = {
+		.callback_ret = true,
+		.capable = true,
+		.output_independent_64b_blks = false,
+	};
+	struct plane_size plane_size = {0};
 	struct amdgpu_device *adev;
-	struct dc *dc;
 	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {0};
-	u64 tile_version = AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS;
-	int ret;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, afb);
-
-	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, false);
-	afb->base.modifier = dm_test_gfx9_dcc_modifier(tile_version, false, true);
-	plane_size.surface_size.width = 1920;
-	plane_size.surface_size.height = 1080;
-
-	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
-				 &address);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_128b);
-
-	dm_test_dcc_ctx = NULL;
-}
-
-/**
- * dm_test_fill_gfx9_plane_attributes_dcc_rbplus_unconstrained() - Verify block mode.
- * @test: KUnit test context.
- *
- * Verify if a GFX10-RBPLUS modifier without independent block bits selects the
- * unconstrained block mode.
- */
-static void dm_test_fill_gfx9_plane_attributes_dcc_rbplus_unconstrained(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {0};
-	u64 tile_version = AMD_FMT_MOD_TILE_VER_GFX10_RBPLUS;
-	int ret;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, afb);
-
-	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, false);
-	afb->base.modifier = dm_test_gfx9_dcc_modifier(tile_version, false, false);
-	plane_size.surface_size.width = 1920;
-	plane_size.surface_size.height = 1080;
-
-	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
-				 &address);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
-			(int)hubp_ind_block_unconstrained);
-
-	dm_test_dcc_ctx = NULL;
-}
-
-/**
- * dm_test_fill_gfx9_plane_attributes_dcc_gfx9_64b() - Verify legacy 64B mode.
- * @test: KUnit test context.
- *
- * Verify if a pre-RBPLUS GFX9 modifier with the 64B independent block bit
- * selects the 64B block mode.
- */
-static void dm_test_fill_gfx9_plane_attributes_dcc_gfx9_64b(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {0};
-	int ret;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, afb);
-
-	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, true);
-	afb->base.modifier = dm_test_gfx9_dcc_modifier(AMD_FMT_MOD_TILE_VER_GFX9,
-						       true, false);
-	plane_size.surface_size.width = 1920;
-	plane_size.surface_size.height = 1080;
-
-	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
-				 &address);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_64b);
-
-	dm_test_dcc_ctx = NULL;
-}
-
-/**
- * dm_test_fill_gfx9_plane_attributes_dcc_gfx9_unconstrained() - Verify legacy mode.
- * @test: KUnit test context.
- *
- * Verify if a pre-RBPLUS GFX9 modifier without the 64B independent block bit
- * selects the unconstrained block mode.
- */
-static void dm_test_fill_gfx9_plane_attributes_dcc_gfx9_unconstrained(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {0};
-	int ret;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, afb);
-
-	dm_test_setup_gfx9_dcc_device(adev, dc, &ctx, false);
-	afb->base.modifier = dm_test_gfx9_dcc_modifier(AMD_FMT_MOD_TILE_VER_GFX9,
-						       false, false);
-	plane_size.surface_size.width = 1920;
-	plane_size.surface_size.height = 1080;
-
-	ret = dm_test_gfx9_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
-				 &address);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
-			(int)hubp_ind_block_unconstrained);
-
-	dm_test_dcc_ctx = NULL;
-}
-
-/**
- * dm_test_fill_gfx12_plane_attributes_block0() - Verify GFX12 64B max-compressed-block path.
- * @test: KUnit test context.
- *
- * Verify if a zero max-compressed-block modifier selects the 64B independent
- * block mode on GFX12.
- */
-static void dm_test_fill_gfx12_plane_attributes_block0(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {
-		.callback_ret = true,
-		.capable = true,
-		.output_independent_64b_blks = false,
-	};
-	int ret;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, afb);
-
-	adev->family = AMDGPU_FAMILY_GC_12_0_0;
-	adev->dm.dc = dc;
-	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
-	dm_test_dcc_ctx = &ctx;
-
-	afb->base.modifier = AMD_FMT_MOD |
-			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
-			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
-			     AMD_FMT_MOD_SET(DCC, 1) |
-			     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, 0);
-	plane_size.surface_size.width = 1920;
-	plane_size.surface_size.height = 1080;
-
-	ret = dm_test_gfx12_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
-				  &address);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_TRUE(test, dcc.independent_64b_blks);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk, (int)hubp_ind_block_64b);
-
-	dm_test_dcc_ctx = NULL;
-}
-
-/**
- * dm_test_fill_gfx12_plane_attributes_block_unconstrained() - Verify block path.
- * @test: KUnit test context.
- *
- * Verify if a max-compressed-block value above one selects the unconstrained
- * independent block mode on GFX12.
- */
-static void dm_test_fill_gfx12_plane_attributes_block_unconstrained(struct kunit *test)
-{
-	struct amdgpu_device *adev;
 	struct dc *dc;
-	struct amdgpu_framebuffer *afb;
-	struct plane_size plane_size = {0};
-	struct dc_tiling_info tiling_info = {0};
-	struct dc_plane_dcc_param dcc = {0};
-	struct dc_plane_address address = {0};
-	struct dm_test_dcc_cap_ctx ctx = {
-		.callback_ret = true,
-		.capable = true,
-		.output_independent_64b_blks = false,
-	};
-	int ret;
+	unsigned int i;
 
 	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
 	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
@@ -2048,25 +1584,36 @@ static void dm_test_fill_gfx12_plane_attributes_block_unconstrained(struct kunit
 	adev->family = AMDGPU_FAMILY_GC_12_0_0;
 	adev->dm.dc = dc;
 	dc->cap_funcs.get_dcc_compression_cap = dm_test_get_dcc_compression_cap;
-	dm_test_dcc_ctx = &ctx;
-
-	afb->base.modifier = AMD_FMT_MOD |
-			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
-			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
-			     AMD_FMT_MOD_SET(DCC, 1) |
-			     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK, 2);
 	plane_size.surface_size.width = 1920;
 	plane_size.surface_size.height = 1080;
 
-	ret = dm_test_gfx12_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
-				  &address);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-	KUNIT_EXPECT_TRUE(test, dcc.enable);
-	KUNIT_EXPECT_FALSE(test, dcc.independent_64b_blks);
-	KUNIT_EXPECT_EQ(test, (int)dcc.dcc_ind_blk,
-			(int)hubp_ind_block_unconstrained);
-
-	dm_test_dcc_ctx = NULL;
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct dc_tiling_info tiling_info = {0};
+		struct dc_plane_dcc_param dcc = {0};
+		struct dc_plane_address address = {0};
+		int ret;
+
+		afb->base.modifier = AMD_FMT_MOD |
+				     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX12_64K_2D) |
+				     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX12) |
+				     AMD_FMT_MOD_SET(DCC, 1) |
+				     AMD_FMT_MOD_SET(DCC_MAX_COMPRESSED_BLOCK,
+						     cases[i].max_compressed_block);
+		dm_test_dcc_ctx = &ctx;
+
+		ret = dm_test_gfx12_attrs(adev, afb, &plane_size, &tiling_info, &dcc,
+					  &address);
+		dm_test_dcc_ctx = NULL;
+
+		KUNIT_EXPECT_EQ_MSG(test, ret, 0, "%s", cases[i].name);
+		KUNIT_EXPECT_EQ_MSG(test, (int)tiling_info.gfxversion, (int)DcGfxAddr3,
+				    "%s", cases[i].name);
+		KUNIT_EXPECT_TRUE_MSG(test, dcc.enable, "%s", cases[i].name);
+		KUNIT_EXPECT_EQ_MSG(test, (bool)dcc.independent_64b_blks,
+				    cases[i].independent_64b_blks, "%s", cases[i].name);
+		KUNIT_EXPECT_EQ_MSG(test, (int)dcc.dcc_ind_blk, cases[i].expected_ind_blk,
+				    "%s", cases[i].name);
+	}
 }
 
 /**
@@ -2208,230 +1755,73 @@ static void dm_test_fill_plane_buffer_attributes_gfx12(struct kunit *test)
 }
 
 /**
- * dm_test_get_min_max_dc_plane_scaling_fp16() - Verify fp16 cap selection.
+ * dm_test_helper_check_state_scaling_caps() - Verify DC scaling caps are applied.
  * @test: KUnit test context.
  *
- * Verify if 64bpp fp16 formats use the fp16 scaling caps.
+ * Verify if helper_check_state converts DC plane scaling caps to DRM scale
+ * limits and rejects scaling outside those limits.
  */
-static void dm_test_get_min_max_dc_plane_scaling_fp16(struct kunit *test)
+static void dm_test_helper_check_state_scaling_caps(struct kunit *test)
 {
 	struct amdgpu_device *adev;
 	struct dc *dc;
-	struct drm_framebuffer *fb;
-	int min_downscale = 0;
-	int max_upscale = 0;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, fb);
-
-	adev->dm.dc = dc;
-	dc->caps.planes[0].max_upscale_factor.fp16 = 2000;
-	dc->caps.planes[0].max_downscale_factor.fp16 = 500;
-
-	fb->format = drm_format_info(DRM_FORMAT_ARGB16161616F);
-	KUNIT_ASSERT_NOT_NULL(test, fb->format);
-	amdgpu_dm_plane_get_min_max_dc_plane_scaling(&adev->ddev, fb,
-						     &min_downscale, &max_upscale);
-	KUNIT_EXPECT_EQ(test, min_downscale, 500);
-	KUNIT_EXPECT_EQ(test, max_upscale, 2000);
-}
-
-/**
- * dm_test_helper_check_state_small_viewport_width() - Verify width rejection.
- * @test: KUnit test context.
- *
- * Verify if a viewport width below the minimum pipe-split width is rejected.
- */
-static void dm_test_helper_check_state_small_viewport_width(struct kunit *test)
-{
 	struct drm_plane *plane;
 	struct drm_plane_state *state;
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_framebuffer *fb;
 
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
 	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
 	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
 	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
 	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
 	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, dc);
 	KUNIT_ASSERT_NOT_NULL(test, plane);
 	KUNIT_ASSERT_NOT_NULL(test, state);
 	KUNIT_ASSERT_NOT_NULL(test, crtc);
 	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
 	KUNIT_ASSERT_NOT_NULL(test, fb);
 
+	adev->dm.dc = dc;
+	dc->caps.planes[0].max_upscale_factor.argb8888 = 1;
+	dc->caps.planes[0].max_downscale_factor.argb8888 = 1;
+
 	plane->type = DRM_PLANE_TYPE_OVERLAY;
+	plane->dev = &adev->ddev;
+	crtc->dev = &adev->ddev;
+	fb->width = 100;
+	fb->height = 100;
+	fb->format = drm_format_info(DRM_FORMAT_XRGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, fb->format);
+
 	state->plane = plane;
 	state->fb = fb;
 	state->crtc = crtc;
-	state->crtc_x = 0;
-	state->crtc_y = 0;
-	state->crtc_w = 10;
-	state->crtc_h = 100;
+	state->src_w = 100 << 16;
+	state->src_h = 100 << 16;
+	state->crtc_w = 200;
+	state->crtc_h = 200;
+	new_crtc_state->crtc = crtc;
 	new_crtc_state->mode.crtc_hdisplay = 1920;
 	new_crtc_state->mode.crtc_vdisplay = 1080;
 
 	KUNIT_EXPECT_EQ(test,
 			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
-			-EINVAL);
+			-ERANGE);
 }
 
 /**
- * dm_test_helper_check_state_small_viewport_height() - Verify height rejection.
+ * dm_test_fill_dc_scaling_info_nv12_dcn1x() - Verify NV12 DCN1x rejection.
  * @test: KUnit test context.
  *
- * Verify if a negative-offset viewport with a too-small height is rejected.
+ * Verify if a non-zero NV12 source origin is rejected on DCN 1.0 to avoid the
+ * known DCN1x hang.
  */
-static void dm_test_helper_check_state_small_viewport_height(struct kunit *test)
-{
-	struct drm_plane *plane;
-	struct drm_plane_state *state;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_framebuffer *fb;
-
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
-	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
-	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, state);
-	KUNIT_ASSERT_NOT_NULL(test, crtc);
-	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
-	KUNIT_ASSERT_NOT_NULL(test, fb);
-
-	plane->type = DRM_PLANE_TYPE_OVERLAY;
-	state->plane = plane;
-	state->fb = fb;
-	state->crtc = crtc;
-	state->crtc_x = -2;
-	state->crtc_y = -95;
-	state->crtc_w = 100;
-	state->crtc_h = 100;
-	new_crtc_state->mode.crtc_hdisplay = 1920;
-	new_crtc_state->mode.crtc_vdisplay = 1080;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
-			-EINVAL);
-}
-
-/**
- * dm_test_helper_check_state_bottom_clipped_height() - Verify bottom clipping.
- * @test: KUnit test context.
- *
- * Verify if a viewport clipped by the bottom edge to below the minimum height
- * is rejected.
- */
-static void dm_test_helper_check_state_bottom_clipped_height(struct kunit *test)
-{
-	struct drm_plane *plane;
-	struct drm_plane_state *state;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_framebuffer *fb;
-
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
-	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
-	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, state);
-	KUNIT_ASSERT_NOT_NULL(test, crtc);
-	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
-	KUNIT_ASSERT_NOT_NULL(test, fb);
-
-	plane->type = DRM_PLANE_TYPE_OVERLAY;
-	state->plane = plane;
-	state->fb = fb;
-	state->crtc = crtc;
-	state->crtc_x = 0;
-	state->crtc_y = 95;
-	state->crtc_w = 100;
-	state->crtc_h = 100;
-	new_crtc_state->mode.crtc_hdisplay = 1920;
-	new_crtc_state->mode.crtc_vdisplay = 100;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
-			-EINVAL);
-}
-
-/**
- * dm_test_helper_check_state_scaling_caps() - Verify DC scaling caps are applied.
- * @test: KUnit test context.
- *
- * Verify if helper_check_state converts DC plane scaling caps to DRM scale
- * limits and rejects scaling outside those limits.
- */
-static void dm_test_helper_check_state_scaling_caps(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct dc *dc;
-	struct drm_plane *plane;
-	struct drm_plane_state *state;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_framebuffer *fb;
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	dc = kunit_kzalloc(test, sizeof(*dc), GFP_KERNEL);
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
-	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
-	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, dc);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, state);
-	KUNIT_ASSERT_NOT_NULL(test, crtc);
-	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
-	KUNIT_ASSERT_NOT_NULL(test, fb);
-
-	adev->dm.dc = dc;
-	dc->caps.planes[0].max_upscale_factor.argb8888 = 1;
-	dc->caps.planes[0].max_downscale_factor.argb8888 = 1;
-
-	plane->type = DRM_PLANE_TYPE_OVERLAY;
-	plane->dev = &adev->ddev;
-	crtc->dev = &adev->ddev;
-	fb->width = 100;
-	fb->height = 100;
-	fb->format = drm_format_info(DRM_FORMAT_XRGB8888);
-	KUNIT_ASSERT_NOT_NULL(test, fb->format);
-
-	state->plane = plane;
-	state->fb = fb;
-	state->crtc = crtc;
-	state->src_w = 100 << 16;
-	state->src_h = 100 << 16;
-	state->crtc_w = 200;
-	state->crtc_h = 200;
-	new_crtc_state->crtc = crtc;
-	new_crtc_state->mode.crtc_hdisplay = 1920;
-	new_crtc_state->mode.crtc_vdisplay = 1080;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_helper_check_state(state, new_crtc_state),
-			-ERANGE);
-}
-
-/**
- * dm_test_fill_dc_scaling_info_nv12_dcn1x() - Verify NV12 DCN1x rejection.
- * @test: KUnit test context.
- *
- * Verify if a non-zero NV12 source origin is rejected on DCN 1.0 to avoid the
- * known DCN1x hang.
- */
-static void dm_test_fill_dc_scaling_info_nv12_dcn1x(struct kunit *test)
+static void dm_test_fill_dc_scaling_info_nv12_dcn1x(struct kunit *test)
 {
 	struct amdgpu_device *adev;
 	struct drm_plane_state state = {0};
@@ -2513,45 +1903,6 @@ static void dm_test_fill_dc_scaling_info_plane_caps(struct kunit *test)
 			0);
 }
 
-/**
- * dm_test_get_cursor_position_bad_size() - Verify oversized cursor rejection.
- * @test: KUnit test context.
- *
- * Verify if a cursor larger than the CRTC maximum is rejected.
- */
-static void dm_test_get_cursor_position_bad_size(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct amdgpu_crtc *amdgpu_crtc;
-	struct drm_plane *plane;
-	struct drm_plane_state *state;
-	struct drm_framebuffer *fb;
-	struct dc_cursor_position position = {0};
-
-	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
-	amdgpu_crtc = kunit_kzalloc(test, sizeof(*amdgpu_crtc), GFP_KERNEL);
-	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
-	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
-	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, adev);
-	KUNIT_ASSERT_NOT_NULL(test, amdgpu_crtc);
-	KUNIT_ASSERT_NOT_NULL(test, plane);
-	KUNIT_ASSERT_NOT_NULL(test, state);
-	KUNIT_ASSERT_NOT_NULL(test, fb);
-
-	amdgpu_crtc->max_cursor_width = 64;
-	amdgpu_crtc->max_cursor_height = 64;
-	plane->dev = &adev->ddev;
-	plane->state = state;
-	state->fb = fb;
-	state->crtc_w = 128;
-	state->crtc_h = 32;
-
-	KUNIT_EXPECT_EQ(test,
-			amdgpu_dm_plane_get_cursor_position(plane, &amdgpu_crtc->base, &position),
-			-EINVAL);
-}
-
 /**
  * dm_test_format_mod_supported_d_swizzle_reject() - Verify D swizzle rejection.
  * @test: KUnit test context.
@@ -2820,43 +2171,29 @@ static void dm_test_atomic_check_missing_crtc_state(struct kunit *test)
 }
 
 /**
- * dm_test_atomic_check_helper_failure() - Verify helper-check failures return.
+ * dm_test_atomic_check_rejects() - Verify the checks before DC validation.
  * @test: KUnit test context.
  *
  * Verify if atomic_check returns before DC validation when the DRM helper state
- * validation rejects the plane.
+ * validation rejects the plane, when both the plane COLOR_PIPELINE and the CRTC
+ * DEGAMMA_LUT are in use, and when filling the scaling info fails.
  */
-static void dm_test_atomic_check_helper_failure(struct kunit *test)
+static void dm_test_atomic_check_rejects(struct kunit *test)
 {
 	struct drm_atomic_commit *state;
 	struct drm_plane *plane;
 	struct dm_plane_state *dm_plane_state;
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_framebuffer *fb;
+	struct amdgpu_device *adev;
+	void *color_pipeline;
+	void *degamma_lut;
 
 	dm_test_init_atomic_check_state(test, &state, &plane, &dm_plane_state,
 					&new_crtc_state, &fb);
 	dm_plane_state->base.crtc_w = 10;
 
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), -EINVAL);
-}
-
-/**
- * dm_test_atomic_check_color_pipeline_conflict() - Verify color conflict rejection.
- * @test: KUnit test context.
- *
- * Verify if atomic_check rejects use of both plane COLOR_PIPELINE and CRTC
- * DEGAMMA_LUT before DC validation.
- */
-static void dm_test_atomic_check_color_pipeline_conflict(struct kunit *test)
-{
-	struct drm_atomic_commit *state;
-	struct drm_plane *plane;
-	struct dm_plane_state *dm_plane_state;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_framebuffer *fb;
-	void *color_pipeline;
-	void *degamma_lut;
 
 	dm_test_init_atomic_check_state(test, &state, &plane, &dm_plane_state,
 					&new_crtc_state, &fb);
@@ -2869,22 +2206,6 @@ static void dm_test_atomic_check_color_pipeline_conflict(struct kunit *test)
 	new_crtc_state->degamma_lut = degamma_lut;
 
 	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), -EINVAL);
-}
-
-/**
- * dm_test_atomic_check_scaling_failure() - Verify scaling-info failures return.
- * @test: KUnit test context.
- *
- * Verify if atomic_check returns the scaling-info error before DC validation.
- */
-static void dm_test_atomic_check_scaling_failure(struct kunit *test)
-{
-	struct amdgpu_device *adev;
-	struct drm_atomic_commit *state;
-	struct drm_plane *plane;
-	struct dm_plane_state *dm_plane_state;
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_framebuffer *fb;
 
 	adev = dm_test_init_atomic_check_state(test, &state, &plane, &dm_plane_state,
 					       &new_crtc_state, &fb);
@@ -2924,42 +2245,34 @@ static const struct drm_plane_funcs dm_test_plane_reset_funcs = {
 };
 
 /**
- * dm_test_plane_reset_initializes_state() - Verify reset installs default state.
+ * dm_test_plane_create_state_initializes_state() - Verify create_state allocates default state.
  * @test: KUnit test context.
  *
- * Verify amdgpu_dm_plane_drm_plane_reset() destroys the existing plane state,
- * allocates a fresh dm_plane_state, and initializes the AMD-specific transfer
- * function and HDR multiplier defaults.
+ * Verify amdgpu_dm_plane_drm_plane_create_state() allocates a fresh
+ * dm_plane_state, and initializes the AMD-specific transfer function and HDR
+ * multiplier defaults.
  */
-static void dm_test_plane_reset_initializes_state(struct kunit *test)
+static void dm_test_plane_create_state_initializes_state(struct kunit *test)
 {
-	struct dm_plane_state *old_state;
+	struct drm_plane_state *plane_state;
 	struct dm_plane_state *new_state;
 	struct drm_plane *plane;
 
 	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_NULL(test, plane);
 
-	/*
-	 * Provide an existing state plus a funcs table so reset exercises the
-	 * destroy-existing-state path. The destroy hook frees this state, so it
-	 * must be a plain (non-KUnit-managed) allocation.
-	 */
-	old_state = kzalloc_obj(*old_state);
-	KUNIT_ASSERT_NOT_NULL(test, old_state);
 	plane->funcs = &dm_test_plane_reset_funcs;
-	plane->state = &old_state->base;
 
-	amdgpu_dm_plane_drm_plane_reset(plane);
+	plane_state = amdgpu_dm_plane_drm_plane_create_state(plane);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_state);
 
-	KUNIT_ASSERT_NOT_NULL(test, plane->state);
-	new_state = to_dm_plane_state(plane->state);
+	new_state = to_dm_plane_state(plane_state);
 	KUNIT_EXPECT_EQ(test, new_state->degamma_tf, AMDGPU_TRANSFER_FUNCTION_DEFAULT);
 	KUNIT_EXPECT_EQ(test, new_state->hdr_mult, AMDGPU_HDR_MULT_DEFAULT);
 	KUNIT_EXPECT_EQ(test, new_state->shaper_tf, AMDGPU_TRANSFER_FUNCTION_DEFAULT);
 	KUNIT_EXPECT_EQ(test, new_state->blend_tf, AMDGPU_TRANSFER_FUNCTION_DEFAULT);
 
-	kfree(new_state);
+	amdgpu_dm_plane_drm_plane_destroy_state(plane, &new_state->base);
 }
 
 /**
@@ -2982,6 +2295,14 @@ static void dm_test_plane_duplicate_state_copies_fields(struct kunit *test)
 	KUNIT_ASSERT_NOT_NULL(test, plane);
 	KUNIT_ASSERT_NOT_NULL(test, old_state);
 
+	/* duplicate_state kmemdup()s these, so the source state must own them. */
+	old_state->flip_addr = kunit_kzalloc(test, sizeof(*old_state->flip_addr), GFP_KERNEL);
+	old_state->scaling_info = kunit_kzalloc(test, sizeof(*old_state->scaling_info), GFP_KERNEL);
+	old_state->plane_info = kunit_kzalloc(test, sizeof(*old_state->plane_info), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_state->flip_addr);
+	KUNIT_ASSERT_NOT_NULL(test, old_state->scaling_info);
+	KUNIT_ASSERT_NOT_NULL(test, old_state->plane_info);
+
 	old_state->degamma_tf = AMDGPU_TRANSFER_FUNCTION_PQ_EOTF;
 	old_state->hdr_mult = 0x123456789ULL;
 	old_state->shaper_tf = AMDGPU_TRANSFER_FUNCTION_IDENTITY;
@@ -2998,7 +2319,88 @@ static void dm_test_plane_duplicate_state_copies_fields(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, dup_state->blend_tf, AMDGPU_TRANSFER_FUNCTION_SRGB_EOTF);
 	KUNIT_EXPECT_NULL(test, dup_state->dc_state);
 
-	kfree(dup_state);
+	amdgpu_dm_plane_drm_plane_destroy_state(plane, dup_base);
+}
+
+/*
+ * Attach a blob to every color property of @state. Each blob starts with a
+ * single reference that the tested function is expected to drop or share.
+ */
+static void dm_test_attach_color_blobs(struct kunit *test, struct drm_device *dev,
+				       struct dm_plane_state *state)
+{
+	struct drm_property_blob **blobs[] = {
+		&state->degamma_lut, &state->ctm, &state->lut3d,
+		&state->shaper_lut, &state->blend_lut,
+	};
+	u32 blob_data = 0;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(blobs); i++) {
+		*blobs[i] = drm_property_create_blob(dev, sizeof(blob_data), &blob_data);
+		KUNIT_ASSERT_NOT_ERR_OR_NULL(test, *blobs[i]);
+	}
+}
+
+/**
+ * dm_test_plane_duplicate_state_copies_resources() - Verify blob and DC state sharing.
+ * @test: KUnit test context.
+ *
+ * Verify amdgpu_dm_plane_drm_plane_duplicate_state() shares the DC plane state
+ * and every color blob with the duplicate, taking an extra reference on each.
+ */
+static void dm_test_plane_duplicate_state_copies_resources(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dm_plane_state *old_state;
+	struct drm_plane_state *dup_base;
+	struct dm_plane_state *dup_state;
+	struct dc_plane_state *dc_plane_state;
+	struct drm_plane *plane;
+
+	KUNIT_ASSERT_EQ(test, drmm_mode_config_init(&adev->ddev), 0);
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	dc_plane_state = kunit_kzalloc(test, sizeof(*dc_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, old_state);
+	KUNIT_ASSERT_NOT_NULL(test, dc_plane_state);
+
+	kref_init(&dc_plane_state->refcount);
+	old_state->dc_state = dc_plane_state;
+	dm_test_attach_color_blobs(test, &adev->ddev, old_state);
+
+	/* duplicate_state kmemdup()s these, so the source state must own them. */
+	old_state->flip_addr = kunit_kzalloc(test, sizeof(*old_state->flip_addr), GFP_KERNEL);
+	old_state->scaling_info = kunit_kzalloc(test, sizeof(*old_state->scaling_info), GFP_KERNEL);
+	old_state->plane_info = kunit_kzalloc(test, sizeof(*old_state->plane_info), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_state->flip_addr);
+	KUNIT_ASSERT_NOT_NULL(test, old_state->scaling_info);
+	KUNIT_ASSERT_NOT_NULL(test, old_state->plane_info);
+
+	plane->state = &old_state->base;
+
+	dup_base = amdgpu_dm_plane_drm_plane_duplicate_state(plane);
+	KUNIT_ASSERT_NOT_NULL(test, dup_base);
+
+	dup_state = to_dm_plane_state(dup_base);
+	KUNIT_EXPECT_PTR_EQ(test, dup_state->dc_state, dc_plane_state);
+	KUNIT_EXPECT_EQ(test, kref_read(&dc_plane_state->refcount), 2U);
+	KUNIT_EXPECT_PTR_EQ(test, dup_state->degamma_lut, old_state->degamma_lut);
+	KUNIT_EXPECT_PTR_EQ(test, dup_state->ctm, old_state->ctm);
+	KUNIT_EXPECT_PTR_EQ(test, dup_state->shaper_lut, old_state->shaper_lut);
+	KUNIT_EXPECT_PTR_EQ(test, dup_state->lut3d, old_state->lut3d);
+	KUNIT_EXPECT_PTR_EQ(test, dup_state->blend_lut, old_state->blend_lut);
+
+	/* Drops the duplicate's references; the originals are released below. */
+	amdgpu_dm_plane_drm_plane_destroy_state(plane, dup_base);
+
+	drm_property_blob_put(old_state->degamma_lut);
+	drm_property_blob_put(old_state->ctm);
+	drm_property_blob_put(old_state->lut3d);
+	drm_property_blob_put(old_state->shaper_lut);
+	drm_property_blob_put(old_state->blend_lut);
 }
 
 /**
@@ -3024,105 +2426,923 @@ static void dm_test_plane_destroy_state_minimal(struct kunit *test)
 	amdgpu_dm_plane_drm_plane_destroy_state(plane, &dm_plane_state->base);
 }
 
+/**
+ * dm_test_plane_destroy_state_releases_resources() - Verify blob and DC state release.
+ * @test: KUnit test context.
+ *
+ * Verify amdgpu_dm_plane_drm_plane_destroy_state() drops a reference on every
+ * attached color blob and on the DC plane state.
+ */
+static void dm_test_plane_destroy_state_releases_resources(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dm_plane_state *dm_plane_state;
+	struct dc_plane_state *dc_plane_state;
+	struct drm_plane *plane;
+
+	KUNIT_ASSERT_EQ(test, drmm_mode_config_init(&adev->ddev), 0);
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	dc_plane_state = kunit_kzalloc(test, sizeof(*dc_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, dc_plane_state);
+
+	/* destroy_state frees the state itself, so use a plain allocation. */
+	dm_plane_state = kzalloc_obj(*dm_plane_state, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_plane_state);
+
+	/* Hold an extra reference so the release does not free KUnit memory. */
+	kref_init(&dc_plane_state->refcount);
+	kref_get(&dc_plane_state->refcount);
+	dm_plane_state->dc_state = dc_plane_state;
+	dm_test_attach_color_blobs(test, &adev->ddev, dm_plane_state);
+
+	amdgpu_dm_plane_drm_plane_destroy_state(plane, &dm_plane_state->base);
+
+	KUNIT_EXPECT_EQ(test, kref_read(&dc_plane_state->refcount), 1U);
+}
+
+/**
+ * dm_test_add_modifier_dedup() - Verify the de-duplicating append.
+ * @test: KUnit test context.
+ *
+ * Verify if a modifier already present in the list is not appended a second
+ * time, and if the append is a no-op once the list is NULL after an earlier
+ * allocation failure.
+ */
+static void dm_test_add_modifier_dedup(struct kunit *test)
+{
+	u64 size = 2;
+	u64 cap = 4;
+	u64 *mods = kmalloc_array(cap, sizeof(*mods), GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_NULL(test, mods);
+	mods[0] = 0xAAULL;
+	mods[1] = 0xBBULL;
+
+	amdgpu_dm_plane_add_modifier_dedup(&mods, &size, &cap, 0xAAULL);
+	KUNIT_EXPECT_EQ(test, size, 2ULL);
+
+	amdgpu_dm_plane_add_modifier_dedup(&mods, &size, &cap, 0xBBULL);
+	KUNIT_EXPECT_EQ(test, size, 2ULL);
+
+	amdgpu_dm_plane_add_modifier_dedup(&mods, &size, &cap, 0xCCULL);
+	KUNIT_ASSERT_EQ(test, size, 3ULL);
+	KUNIT_EXPECT_EQ(test, mods[2], 0xCCULL);
+
+	kfree(mods);
+
+	size = 3;
+	cap = 7;
+	mods = NULL;
+
+	amdgpu_dm_plane_add_modifier_dedup(&mods, &size, &cap, 0x55ULL);
+
+	KUNIT_EXPECT_PTR_EQ(test, mods, NULL);
+	KUNIT_EXPECT_EQ(test, size, 3ULL);
+	KUNIT_EXPECT_EQ(test, cap, 7ULL);
+}
+
+static int dm_test_gfx6_tiling(struct dc_tiling_info *tiling_info, u64 modifier)
+{
+	return amdgpu_dm_plane_fill_gfx6_tiling_info_from_modifier(tiling_info, modifier);
+}
+
+/**
+ * dm_test_fill_gfx6_tiling_info_from_modifier() - Verify GFX6-8 modifier decoding.
+ * @test: KUnit test context.
+ *
+ * Verify if a linear modifier maps to the generic linear array mode, if non-AMD
+ * modifiers and newer tile versions are rejected, if a 1D tiled modifier stops
+ * before the macro tiling fields, and if a 2D tiled modifier decodes them all.
+ */
+static void dm_test_fill_gfx6_tiling_info_from_modifier(struct kunit *test)
+{
+	struct dc_tiling_info tiling_info = {0};
+	u64 not_amd_mod = DRM_FORMAT_MOD_VENDOR_AMD;
+	u64 gfx9_mod = AMD_FMT_MOD | AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9);
+	u64 mod_1d = AMD_FMT_MOD |
+		     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX6) |
+		     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX6_1D_TILED_THIN1) |
+		     AMD_FMT_MOD_SET(MICROTILE, AMD_FMT_MOD_MICROTILE_DISPLAY) |
+		     AMD_FMT_MOD_SET(NUM_BANKS, 3);
+	u64 mod_2d = AMD_FMT_MOD |
+		     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX6) |
+		     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1) |
+		     AMD_FMT_MOD_SET(MICROTILE, AMD_FMT_MOD_MICROTILE_DISPLAY) |
+		     AMD_FMT_MOD_SET(PIPE_CONFIG, 5) |
+		     AMD_FMT_MOD_SET(TILE_SPLIT, 4) |
+		     AMD_FMT_MOD_SET(BANK_WIDTH, 1) |
+		     AMD_FMT_MOD_SET(BANK_HEIGHT, 2) |
+		     AMD_FMT_MOD_SET(MACRO_TILE_ASPECT, 3) |
+		     AMD_FMT_MOD_SET(NUM_BANKS, 2);
+
+	KUNIT_EXPECT_EQ(test, dm_test_gfx6_tiling(&tiling_info, DRM_FORMAT_MOD_LINEAR), 0);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfx8.array_mode, (int)DC_ARRAY_LINEAR_GENERAL);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxVersion8);
+
+	KUNIT_EXPECT_EQ(test, dm_test_gfx6_tiling(&tiling_info, not_amd_mod), -EINVAL);
+	KUNIT_EXPECT_EQ(test, dm_test_gfx6_tiling(&tiling_info, gfx9_mod), -EINVAL);
+
+	memset(&tiling_info, 0, sizeof(tiling_info));
+	KUNIT_EXPECT_EQ(test, dm_test_gfx6_tiling(&tiling_info, mod_1d), 0);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfx8.array_mode,
+			(int)AMD_FMT_MOD_TILE_GFX6_1D_TILED_THIN1);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxVersion8);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.num_banks, 0U);
+
+	memset(&tiling_info, 0, sizeof(tiling_info));
+	KUNIT_EXPECT_EQ(test, dm_test_gfx6_tiling(&tiling_info, mod_2d), 0);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfx8.array_mode,
+			(int)AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfx8.tile_mode,
+			(int)AMD_FMT_MOD_MICROTILE_DISPLAY);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.pipe_config, 5U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.tile_split, 4U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.bank_width, 1U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.bank_height, 2U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.tile_aspect, 3U);
+	KUNIT_EXPECT_EQ(test, tiling_info.gfx8.num_banks, 2U);
+}
+
+static u64 dm_test_gfx6_mod(u32 tile, u32 pipe_config, u32 tile_split, u32 bank_width,
+			    u32 bank_height, u32 aspect, u32 num_banks)
+{
+	return AMD_FMT_MOD |
+	       AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX6) |
+	       AMD_FMT_MOD_SET(TILE, tile) |
+	       AMD_FMT_MOD_SET(MICROTILE, AMD_FMT_MOD_MICROTILE_DISPLAY) |
+	       AMD_FMT_MOD_SET(PIPE_CONFIG, pipe_config) |
+	       AMD_FMT_MOD_SET(TILE_SPLIT, tile_split) |
+	       AMD_FMT_MOD_SET(BANK_WIDTH, bank_width) |
+	       AMD_FMT_MOD_SET(BANK_HEIGHT, bank_height) |
+	       AMD_FMT_MOD_SET(MACRO_TILE_ASPECT, aspect) |
+	       AMD_FMT_MOD_SET(NUM_BANKS, num_banks);
+}
+
+static struct amdgpu_device *dm_test_alloc_gfx7_device(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_VI;
+	adev->gfx.config.mem_row_size_in_kb = 4;
+	/* 2D_TILED_THIN1 array mode (bits 5:2) and pipe config 2 (bits 10:6). */
+	adev->gfx.config.tile_mode_array[10] = (4u << 2) | (2u << 6);
+	/*
+	 * Bank width (bits 1:0), bank height (bits 3:2), macro tile aspect
+	 * (bits 5:4) and num banks (bits 7:6). Entries 1 and 2 are identical so
+	 * the 16 and 32 bpp modifiers collide and the dedup path is taken.
+	 */
+	adev->gfx.config.macrotile_mode_array[1] = 0xb9u;
+	adev->gfx.config.macrotile_mode_array[2] = 0xb9u;
+	adev->gfx.config.macrotile_mode_array[3] = 0x2au;
+
+	return adev;
+}
+
+/**
+ * dm_test_get_gfx6_tile_idx() - Verify GFX6-8 tile mode index selection.
+ * @test: KUnit test context.
+ *
+ * Verify if the micro tiled mode maps to a fixed index, GFX7+ uses a single
+ * macro tiled index, and GFX6 selects the index from the bits per pixel.
+ */
+static void dm_test_get_gfx6_tile_idx(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->family = AMDGPU_FAMILY_VI;
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_gfx6_tile_idx(adev, 32, DC_ARRAY_1D_TILED_THIN1),
+			9U);
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_gfx6_tile_idx(adev, 32, DC_ARRAY_2D_TILED_THIN1),
+			10U);
+
+	adev->family = AMDGPU_FAMILY_SI;
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_gfx6_tile_idx(adev, 8, DC_ARRAY_2D_TILED_THIN1),
+			10U);
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_gfx6_tile_idx(adev, 16, DC_ARRAY_2D_TILED_THIN1),
+			11U);
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_get_gfx6_tile_idx(adev, 32, DC_ARRAY_2D_TILED_THIN1),
+			12U);
+}
+
+/**
+ * dm_test_calc_gfx7_tile_split() - Verify GFX7-8 tile split calculation.
+ * @test: KUnit test context.
+ *
+ * Verify if the tile split is derived from the sample split factor and clamped
+ * to the 256 byte minimum and the memory row size maximum.
+ */
+static void dm_test_calc_gfx7_tile_split(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+
+	adev->gfx.config.mem_row_size_in_kb = 4;
+
+	/* 8 bpp gives 64 bytes, clamped up to the 256 byte minimum. */
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_calc_gfx7_tile_split(adev, 8, 0u << 25), 256U);
+	/* 32 bpp with a sample split factor of 2. */
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_calc_gfx7_tile_split(adev, 32, 1u << 25), 512U);
+
+	/* 64 bpp with a sample split factor of 8, clamped to the memory row size. */
+	adev->gfx.config.mem_row_size_in_kb = 1;
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_calc_gfx7_tile_split(adev, 64, 3u << 25), 1024U);
+}
+
+/**
+ * dm_test_get_gfx7_macro_tile_idx() - Verify GFX7-8 macro tile index selection.
+ * @test: KUnit test context.
+ *
+ * Verify if the macro tile index is the log2 of the tile size in 64 byte units,
+ * clamped to the tile split.
+ */
+static void dm_test_get_gfx7_macro_tile_idx(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_gfx7_macro_tile_idx(8, 1024), 0U);
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_gfx7_macro_tile_idx(32, 1024), 2U);
+	/* 64 bpp needs 512 bytes but the tile split caps it at 256. */
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_get_gfx7_macro_tile_idx(64, 256), 2U);
+}
+
+/**
+ * dm_test_calc_gfx6_mod() - Verify the GFX6-8 modifier calculation.
+ * @test: KUnit test context.
+ *
+ * Verify if a micro tiled array mode returns early with only the base modifier
+ * fields set, if GFX7-8 combines the tile mode pipe config and computed tile
+ * split with the bank fields from the macro tile mode table, and if GFX6 takes
+ * the tile split and bank fields directly from the tile mode register.
+ */
+static void dm_test_calc_gfx6_mod(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_gfx7_device(test);
+	struct amdgpu_device *si_adev;
+	u64 expected_1d = AMD_FMT_MOD |
+			  AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX6) |
+			  AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX6_1D_TILED_THIN1) |
+			  AMD_FMT_MOD_SET(MICROTILE, AMD_FMT_MOD_MICROTILE_DISPLAY);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_calc_gfx6_mod(adev, 0, DC_ARRAY_1D_TILED_THIN1),
+			expected_1d);
+
+	/* 32 bpp: 256 byte tile split (log2 of 4 units), macro tile mode 2. */
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_calc_gfx6_mod(adev, 32, DC_ARRAY_2D_TILED_THIN1),
+			dm_test_gfx6_mod(AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1, 2, 2, 1, 2, 3, 2));
+
+	/* 64 bpp: 512 byte tile split (log2 of 8 units), macro tile mode 3. */
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_calc_gfx6_mod(adev, 64, DC_ARRAY_2D_TILED_THIN1),
+			dm_test_gfx6_mod(AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1, 2, 3, 2, 2, 2, 0));
+
+	si_adev = kunit_kzalloc(test, sizeof(*si_adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, si_adev);
+
+	si_adev->family = AMDGPU_FAMILY_SI;
+	/* 2D_TILED_THIN1, pipe config 3, 256 byte tile split, banks 1/2/3/1. */
+	si_adev->gfx.config.tile_mode_array[12] = (4u << 2) | (3u << 6) | (2u << 11) |
+						  (1u << 14) | (2u << 16) | (3u << 18) |
+						  (1u << 20);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_plane_calc_gfx6_mod(si_adev, 32, DC_ARRAY_2D_TILED_THIN1),
+			dm_test_gfx6_mod(AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1, 3, 2, 1, 2, 3, 1));
+}
+
+/**
+ * dm_test_gfx6_format_mod_supported() - Verify GFX6-8 modifier support checks.
+ * @test: KUnit test context.
+ *
+ * Verify if only the canonical modifier for a given bits per pixel is accepted,
+ * and that non-AMD modifiers, newer tile versions, GFX9+ devices, DCC, and
+ * non-displayable tile modes are rejected.
+ */
+static void dm_test_gfx6_format_mod_supported(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_gfx7_device(test);
+	u64 modifier = amdgpu_dm_plane_calc_gfx6_mod(adev, 32, DC_ARRAY_2D_TILED_THIN1);
+	u64 gfx9_mod = AMD_FMT_MOD | AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9);
+	/* Neither 1D nor 2D THIN array mode. */
+	u64 bad_tile_mod = AMD_FMT_MOD | AMD_FMT_MOD_SET(TILE, 3);
+	/* Non-displayable micro tile mode. */
+	u64 bad_micro_mod = AMD_FMT_MOD |
+			    AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1) |
+			    AMD_FMT_MOD_SET(MICROTILE, AMD_FMT_MOD_MICROTILE_THIN);
+	u64 not_amd_mod = DRM_FORMAT_MOD_VENDOR_AMD;
+	u64 dcc_mod = modifier | AMD_FMT_MOD_SET(DCC, 1);
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 32, modifier));
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 64, modifier));
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 32, not_amd_mod));
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 32, gfx9_mod));
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 32, dcc_mod));
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 32, bad_tile_mod));
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 32, bad_micro_mod));
+
+	adev->family = AMDGPU_FAMILY_AI;
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_plane_gfx6_format_mod_supported(adev, 32, modifier));
+}
+
+/**
+ * dm_test_get_plane_modifiers_gfx6() - Verify GFX6-8 modifier list generation.
+ * @test: KUnit test context.
+ *
+ * Verify if a GFX6-8 family device exposes one macro tiled modifier per
+ * distinct bits per pixel plus a micro tiled one, and that the duplicate 32 bpp
+ * modifier is de-duplicated.
+ */
+static void dm_test_get_plane_modifiers_gfx6(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_gfx7_device(test);
+	u64 *mods = dm_test_get_primary_mods(test, adev);
+	u64 mod_1d = AMD_FMT_MOD |
+		     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX6) |
+		     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX6_1D_TILED_THIN1) |
+		     AMD_FMT_MOD_SET(MICROTILE, AMD_FMT_MOD_MICROTILE_DISPLAY);
+
+	/* 16 bpp, 64 bpp, micro tiled and linear; the 32 bpp duplicate is dropped. */
+	KUNIT_EXPECT_EQ(test, dm_test_mods_count(mods), 4);
+	KUNIT_EXPECT_EQ(test, mods[0],
+			dm_test_gfx6_mod(AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1, 2, 2, 1, 2, 3, 2));
+	KUNIT_EXPECT_EQ(test, mods[1],
+			dm_test_gfx6_mod(AMD_FMT_MOD_TILE_GFX6_2D_TILED_THIN1, 2, 3, 2, 2, 2, 0));
+	KUNIT_EXPECT_EQ(test, mods[2], mod_1d);
+	KUNIT_EXPECT_EQ(test, mods[3], DRM_FORMAT_MOD_LINEAR);
+
+	kfree(mods);
+}
+
+/**
+ * dm_test_format_mod_supported_gfx6() - Verify dispatch to the GFX6-8 check.
+ * @test: KUnit test context.
+ *
+ * Verify if a listed GFX6 tile version modifier is validated through the
+ * GFX6-8 specific helper.
+ */
+static void dm_test_format_mod_supported_gfx6(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_gfx7_device(test);
+	struct drm_plane *plane;
+	u64 listed_mod = amdgpu_dm_plane_calc_gfx6_mod(adev, 32, DC_ARRAY_2D_TILED_THIN1);
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	plane->dev = &adev->ddev;
+	plane->modifiers = &listed_mod;
+	plane->modifier_count = 1;
+
+	KUNIT_EXPECT_TRUE(test,
+			  amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_XRGB8888,
+							       listed_mod));
+	KUNIT_EXPECT_FALSE(test,
+			   amdgpu_dm_plane_format_mod_supported(plane, DRM_FORMAT_XRGB16161616,
+								listed_mod));
+}
+
+/**
+ * dm_test_helper_prepare_fb_no_fb() - Verify prepare_fb without a framebuffer.
+ * @test: KUnit test context.
+ *
+ * Verify if prepare_fb succeeds without touching any buffer object when the new
+ * plane state has no framebuffer bound.
+ */
+static void dm_test_helper_prepare_fb_no_fb(struct kunit *test)
+{
+	struct drm_plane_state state = {0};
+	struct drm_plane *plane;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_helper_prepare_fb(plane, &state), 0);
+}
+
+/**
+ * dm_test_helper_cleanup_fb_no_fb() - Verify cleanup_fb without a framebuffer.
+ * @test: KUnit test context.
+ *
+ * Verify if cleanup_fb returns without unpinning anything when the old plane
+ * state has no framebuffer bound.
+ */
+static void dm_test_helper_cleanup_fb_no_fb(struct kunit *test)
+{
+	struct drm_plane_state state = {0};
+	struct drm_plane *plane;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	amdgpu_dm_plane_helper_cleanup_fb(plane, &state);
+}
+
+/**
+ * dm_test_handle_cursor_update_no_fb() - Verify cursor update without framebuffers.
+ * @test: KUnit test context.
+ *
+ * Verify if the cursor update returns early, without touching the CRTC, when
+ * neither the new nor the old plane state has a framebuffer bound.
+ */
+static void dm_test_handle_cursor_update_no_fb(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane_state old_state = {0};
+	struct drm_plane_state state = {0};
+	struct drm_plane *plane;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+
+	plane->dev = &adev->ddev;
+	plane->state = &state;
+
+	amdgpu_dm_plane_handle_cursor_update(plane, &old_state);
+}
+
+/**
+ * dm_test_atomic_async_update_copies_state() - Verify async cursor state copy.
+ * @test: KUnit test context.
+ *
+ * Verify if the async update swaps the framebuffer and copies the source and
+ * CRTC rectangles from the new state into the current plane state.
+ */
+static void dm_test_atomic_async_update_copies_state(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct __drm_planes_state *planes;
+	struct drm_plane_state *cur_state;
+	struct drm_plane_state *new_state;
+	struct drm_plane_state *old_state;
+	struct amdgpu_framebuffer *afb;
+	struct drm_plane *plane;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	planes = kunit_kzalloc(test, sizeof(*planes), GFP_KERNEL);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	cur_state = kunit_kzalloc(test, sizeof(*cur_state), GFP_KERNEL);
+	new_state = kunit_kzalloc(test, sizeof(*new_state), GFP_KERNEL);
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	KUNIT_ASSERT_NOT_NULL(test, planes);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, cur_state);
+	KUNIT_ASSERT_NOT_NULL(test, new_state);
+	KUNIT_ASSERT_NOT_NULL(test, old_state);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	plane->dev = &adev->ddev;
+	plane->index = 0;
+	plane->state = cur_state;
+	cur_state->fb = &afb->base;
+	new_state->src_x = 1 << 16;
+	new_state->src_y = 2 << 16;
+	new_state->src_w = 64 << 16;
+	new_state->src_h = 64 << 16;
+	new_state->crtc_x = 10;
+	new_state->crtc_y = 20;
+	new_state->crtc_w = 64;
+	new_state->crtc_h = 64;
+	state->planes = planes;
+	state->planes[0].new_state = new_state;
+	state->planes[0].old_state = old_state;
+
+	amdgpu_dm_plane_atomic_async_update(plane, state);
+
+	KUNIT_EXPECT_PTR_EQ(test, cur_state->fb, NULL);
+	KUNIT_EXPECT_PTR_EQ(test, new_state->fb, &afb->base);
+	KUNIT_EXPECT_EQ(test, cur_state->src_x, 1U << 16);
+	KUNIT_EXPECT_EQ(test, cur_state->src_h, 64U << 16);
+	KUNIT_EXPECT_EQ(test, cur_state->crtc_x, 10);
+	KUNIT_EXPECT_EQ(test, cur_state->crtc_h, 64U);
+}
+
+/**
+ * dm_test_fill_plane_buffer_attributes_gfx6() - Verify the pre-GFX9 dispatch.
+ * @test: KUnit test context.
+ *
+ * Verify if a pre-Vega family decodes tiling from the GFX6 modifier instead of
+ * taking the GFX9 or GFX12 modifier paths, and propagates the decode error for
+ * a modifier the GFX6 decoder does not understand.
+ */
+static void dm_test_fill_plane_buffer_attributes_gfx6(struct kunit *test)
+{
+	struct dc_tiling_info tiling_info = {0};
+	struct dc_plane_dcc_param dcc = {0};
+	struct dc_plane_address address = {0};
+	struct plane_size plane_size = {0};
+	struct amdgpu_framebuffer *afb;
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	adev->family = AMDGPU_FAMILY_CZ;
+	afb->address = 0x80000000ULL;
+	afb->base.width = 1920;
+	afb->base.height = 1080;
+	afb->base.pitches[0] = 7680;
+	afb->base.format = drm_format_info(DRM_FORMAT_XRGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, afb->base.format);
+	afb->base.modifier = AMD_FMT_MOD |
+			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX6) |
+			     AMD_FMT_MOD_SET(TILE, AMD_FMT_MOD_TILE_GFX6_1D_TILED_THIN1) |
+			     AMD_FMT_MOD_SET(MICROTILE, AMD_FMT_MOD_MICROTILE_DISPLAY);
+
+	KUNIT_EXPECT_EQ(test, dm_test_graphics_attrs(adev, afb, &tiling_info,
+						     &plane_size, &dcc, &address), 0);
+	KUNIT_EXPECT_EQ(test, address.type, (int)PLN_ADDR_TYPE_GRAPHICS);
+	KUNIT_EXPECT_EQ(test, (int)tiling_info.gfxversion, (int)DcGfxVersion8);
+
+	afb->base.modifier = AMD_FMT_MOD |
+			     AMD_FMT_MOD_SET(TILE_VERSION, AMD_FMT_MOD_TILE_VER_GFX9);
+
+	KUNIT_EXPECT_EQ(test, dm_test_graphics_attrs(adev, afb, &tiling_info,
+						     &plane_size, &dcc, &address), -EINVAL);
+}
+
+/**
+ * dm_test_atomic_check_success() - Verify a fully valid plane state is accepted.
+ * @test: KUnit test context.
+ *
+ * Verify if atomic_check reports success once DC validation accepts the plane.
+ * The resource pool exposes no validate_plane callback, so DC falls back to
+ * checking the source and destination rectangles only.
+ */
+static void dm_test_atomic_check_success(struct kunit *test)
+{
+	struct dm_plane_state *dm_plane_state;
+	struct drm_crtc_state *new_crtc_state;
+	struct drm_atomic_commit *state;
+	struct resource_pool *res_pool;
+	struct amdgpu_device *adev;
+	struct drm_framebuffer *fb;
+	struct drm_plane *plane;
+
+	adev = dm_test_init_atomic_check_state(test, &state, &plane, &dm_plane_state,
+					       &new_crtc_state, &fb);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, res_pool);
+	res_pool->funcs = kunit_kzalloc(test, sizeof(*res_pool->funcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, res_pool->funcs);
+
+	adev->dm.dc->res_pool = res_pool;
+
+	dm_plane_state->dc_state->src_rect.width = 100;
+	dm_plane_state->dc_state->src_rect.height = 100;
+	dm_plane_state->dc_state->dst_rect.width = 100;
+	dm_plane_state->dc_state->dst_rect.height = 100;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_plane_atomic_check(plane, state), 0);
+}
+
+struct dm_test_cursor_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state *crtc_state;
+	struct drm_plane_state *state;
+	struct drm_plane_state old_state;
+	struct drm_plane *plane;
+};
+
+/*
+ * Build a 64x64 cursor plane bound to an amdgpu_crtc whose DM CRTC state
+ * carries no DC stream, so the cursor update stops before programming DC.
+ */
+static struct dm_test_cursor_ctx *dm_test_alloc_cursor_ctx(struct kunit *test)
+{
+	struct amdgpu_framebuffer *afb;
+	struct dm_test_cursor_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->acrtc = kunit_kzalloc(test, sizeof(*ctx->acrtc), GFP_KERNEL);
+	ctx->crtc_state = kunit_kzalloc(test, sizeof(*ctx->crtc_state), GFP_KERNEL);
+	ctx->state = kunit_kzalloc(test, sizeof(*ctx->state), GFP_KERNEL);
+	ctx->plane = kunit_kzalloc(test, sizeof(*ctx->plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc_state);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->plane);
+
+	ctx->adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	ctx->acrtc->base.dev = &ctx->adev->ddev;
+	ctx->acrtc->base.state = &ctx->crtc_state->base;
+	ctx->acrtc->max_cursor_width = 64;
+	ctx->acrtc->max_cursor_height = 64;
+
+	afb->address = 0x80000000ULL;
+	afb->base.pitches[0] = 256;
+	afb->base.format = drm_format_info(DRM_FORMAT_ARGB8888);
+	KUNIT_ASSERT_NOT_NULL(test, afb->base.format);
+
+	ctx->state->fb = &afb->base;
+	ctx->state->crtc = &ctx->acrtc->base;
+	ctx->state->crtc_w = 64;
+	ctx->state->crtc_h = 64;
+
+	ctx->plane->dev = &ctx->adev->ddev;
+	ctx->plane->state = ctx->state;
+
+	return ctx;
+}
+
+/**
+ * dm_test_handle_cursor_update_disabled() - Verify the cursor-off path.
+ * @test: KUnit test context.
+ *
+ * Verify if a cursor positioned fully off the left edge is treated as disabled
+ * and returns before the cursor geometry is latched on the CRTC.
+ */
+static void dm_test_handle_cursor_update_disabled(struct kunit *test)
+{
+	struct dm_test_cursor_ctx *ctx = dm_test_alloc_cursor_ctx(test);
+
+	ctx->state->crtc_x = -64;
+
+	amdgpu_dm_plane_handle_cursor_update(ctx->plane, &ctx->old_state);
+
+	KUNIT_EXPECT_EQ(test, ctx->acrtc->cursor_width, 0);
+	KUNIT_EXPECT_EQ(test, ctx->acrtc->cursor_height, 0);
+}
+
+/**
+ * dm_test_handle_cursor_update_no_stream() - Verify cursor attribute assembly.
+ * @test: KUnit test context.
+ *
+ * Verify if an enabled cursor latches its geometry on the CRTC and builds the
+ * DC cursor attributes, including the degamma ROM bit, before stopping at the
+ * missing DC stream.
+ */
+static void dm_test_handle_cursor_update_no_stream(struct kunit *test)
+{
+	struct dm_test_cursor_ctx *ctx = dm_test_alloc_cursor_ctx(test);
+
+	ctx->crtc_state->cm_is_degamma_srgb = true;
+	ctx->adev->dm.dc->caps.color.dpp.gamma_corr = true;
+
+	amdgpu_dm_plane_handle_cursor_update(ctx->plane, &ctx->old_state);
+
+	KUNIT_EXPECT_EQ(test, ctx->acrtc->cursor_width, 64);
+	KUNIT_EXPECT_EQ(test, ctx->acrtc->cursor_height, 64);
+}
+
+static int dm_test_clear_dcc_tiling_count;
+
+static void dm_test_clear_surface_dcc_and_tiling(struct pipe_ctx *pipe_ctx,
+						 struct dc_plane_state *plane_state,
+						 bool clear_tiling)
+{
+	if (clear_tiling)
+		dm_test_clear_dcc_tiling_count++;
+}
+
+/**
+ * dm_test_panic_flush_disables_dcc() - Verify panic flush disables DCC.
+ * @test: KUnit test context.
+ *
+ * Verify if panic_flush asks DC to clear DCC on the active pipe, and requests
+ * the tiling teardown as well when the framebuffer is not linear.
+ */
+static void dm_test_panic_flush_disables_dcc(struct kunit *test)
+{
+	struct dc_plane_state *dc_plane_state;
+	struct dm_plane_state *dm_plane_state;
+	struct resource_pool *res_pool;
+	struct drm_framebuffer *fb;
+	struct drm_plane *plane;
+	struct dc *dc;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	dm_plane_state = kunit_kzalloc(test, sizeof(*dm_plane_state), GFP_KERNEL);
+	dc_plane_state = kunit_kzalloc(test, sizeof(*dc_plane_state), GFP_KERNEL);
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	res_pool = kunit_kzalloc(test, sizeof(*res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	KUNIT_ASSERT_NOT_NULL(test, dm_plane_state);
+	KUNIT_ASSERT_NOT_NULL(test, dc_plane_state);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+	KUNIT_ASSERT_NOT_NULL(test, res_pool);
+
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	dc->current_state = dm_kunit_alloc_dc_state(test);
+	dc->hwss.clear_surface_dcc_and_tiling = dm_test_clear_surface_dcc_and_tiling;
+	res_pool->pipe_count = 1;
+	dc->res_pool = res_pool;
+	dc_plane_state->ctx = dc->ctx;
+
+	fb->modifier = AMD_FMT_MOD;
+	dm_plane_state->base.fb = fb;
+	dm_plane_state->dc_state = dc_plane_state;
+	plane->state = &dm_plane_state->base;
+
+	dm_test_clear_dcc_tiling_count = 0;
+
+	amdgpu_dm_plane_panic_flush(plane);
+
+	KUNIT_EXPECT_EQ(test, dm_test_clear_dcc_tiling_count, 1);
+}
+
+/*
+ * Build a DM ready to initialize planes on a real DRM device. The plane itself
+ * is plain kzalloc'd because DRM frees it through drm_plane_helper_destroy()
+ * when the mode config is torn down.
+ */
+static struct drm_plane *dm_test_alloc_init_plane(struct kunit *test,
+						  struct amdgpu_device **adev_out)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+
+	adev = dm_kunit_alloc_adev(test);
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->dm.adev = adev;
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	plane = kzalloc_obj(*plane, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	plane->type = DRM_PLANE_TYPE_OVERLAY;
+
+	*adev_out = adev;
+
+	return plane;
+}
+
+/**
+ * dm_test_plane_init_creates_optional_properties() - Verify optional properties.
+ * @test: KUnit test context.
+ *
+ * Verify if a Bonaire-or-newer non-cursor plane on a DCE version above 3.0.1
+ * gains the rotation property and framebuffer damage clips.
+ */
+static void dm_test_plane_init_creates_optional_properties(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+
+	plane = dm_test_alloc_init_plane(test, &adev);
+	adev->asic_type = CHIP_BONAIRE;
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 0);
+
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_plane_init(&adev->dm, plane, 1, NULL), 0);
+
+	KUNIT_EXPECT_NOT_NULL(test, plane->rotation_property);
+	KUNIT_EXPECT_NOT_NULL(test, adev->ddev.mode_config.prop_fb_damage_clips);
+}
+
+/**
+ * dm_test_plane_init_creates_color_pipeline() - Verify color pipeline creation.
+ * @test: KUnit test context.
+ *
+ * Verify if a non-cursor plane on DCN 3.0 or newer gets a default color
+ * pipeline exposed through the COLOR_PIPELINE property.
+ */
+static void dm_test_plane_init_creates_color_pipeline(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+
+	plane = dm_test_alloc_init_plane(test, &adev);
+	adev->dm.dc->ctx->dce_version = DCN_VERSION_3_0;
+
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_plane_init(&adev->dm, plane, 1, NULL), 0);
+
+	KUNIT_EXPECT_NOT_NULL(test, plane->color_pipeline_property);
+}
+
 static struct kunit_case amdgpu_dm_plane_test_cases[] = {
 	/* amdgpu_dm_plane_is_video_format() */
 	KUNIT_CASE(dm_test_plane_is_video_format_known_video),
 	/* amdgpu_dm_plane_get_format_info() */
 	KUNIT_CASE(dm_test_get_format_info),
 	/* amdgpu_dm_plane_fill_blending_from_plane_state() */
-	KUNIT_CASE(dm_test_fill_blending_defaults),
-	KUNIT_CASE(dm_test_fill_blending_premulti_alpha_format),
-	KUNIT_CASE(dm_test_fill_blending_coverage_alpha_format),
+	KUNIT_CASE(dm_test_fill_blending_pixel_blend_modes),
 	KUNIT_CASE(dm_test_fill_blending_global_alpha),
-	KUNIT_CASE(dm_test_fill_blending_global_alpha_dcn42),
 	/* amdgpu_dm_plane_modifier_* helpers() */
 	KUNIT_CASE(dm_test_modifier_has_dcc),
 	KUNIT_CASE(dm_test_modifier_gfx9_swizzle_mode),
 	/* amdgpu_dm_plane_get_plane_formats() */
 	KUNIT_CASE(dm_test_get_plane_formats),
-	KUNIT_CASE(dm_test_get_plane_formats_overlay_universal_cap),
 	/* amdgpu_dm_plane_get_plane_modifiers() */
 	KUNIT_CASE(dm_test_get_plane_modifiers),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx6),
 	KUNIT_CASE(dm_test_get_plane_modifiers_gfx9),
 	KUNIT_CASE(dm_test_get_plane_modifiers_rv),
-	KUNIT_CASE(dm_test_get_plane_modifiers_rv_constant_encode),
-	KUNIT_CASE(dm_test_get_plane_modifiers_gfx10_1),
-	KUNIT_CASE(dm_test_get_plane_modifiers_gfx10_3),
+	KUNIT_CASE(dm_test_get_plane_modifiers_gfx10_and_gfx12),
 	KUNIT_CASE(dm_test_get_plane_modifiers_gfx11_64k_first),
 	KUNIT_CASE(dm_test_get_plane_modifiers_gfx11_256k_first),
-	KUNIT_CASE(dm_test_get_plane_modifiers_gfx12),
 	/* amdgpu_dm_plane_fill_dc_scaling_info() */
 	KUNIT_CASE(dm_test_fill_dc_scaling_info),
 	KUNIT_CASE(dm_test_fill_dc_scaling_info_nv12_dcn1x),
 	KUNIT_CASE(dm_test_fill_dc_scaling_info_plane_caps),
 	/* amdgpu_dm_plane_get_min_max_dc_plane_scaling() */
 	KUNIT_CASE(dm_test_get_min_max_dc_plane_scaling),
-	KUNIT_CASE(dm_test_get_min_max_dc_plane_scaling_fp16),
 	/* amdgpu_dm_plane_fill_plane_buffer_attributes() */
 	KUNIT_CASE(dm_test_fill_plane_buffer_attributes_video),
 	KUNIT_CASE(dm_test_fill_plane_buffer_attributes_gfx12),
+	KUNIT_CASE(dm_test_fill_plane_buffer_attributes_gfx6),
 	/* amdgpu_dm_plane_get_cursor_position() */
 	KUNIT_CASE(dm_test_get_cursor_position),
-	KUNIT_CASE(dm_test_get_cursor_position_bad_size),
 	/* amdgpu_dm_plane_format_mod_supported() */
 	KUNIT_CASE(dm_test_format_mod_supported),
 	KUNIT_CASE(dm_test_format_mod_supported_d_swizzle_reject),
+	KUNIT_CASE(dm_test_format_mod_supported_gfx6),
 	/* amdgpu_dm_plane_fill_gfx12_attrs_from_modifiers() */
-	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_from_modifiers),
-	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_block0),
-	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_block_unconstrained),
+	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_dcc_blocks),
 	KUNIT_CASE(dm_test_fill_gfx12_plane_attributes_validate_fails),
 	/* amdgpu_dm_plane_fill_gfx9_attrs_from_modifiers() */
 	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_from_modifiers),
 	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc),
 	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_validate_fails),
-	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_rbplus_64b_no_128bcl),
-	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_rbplus_128b),
-	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_rbplus_unconstrained),
-	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_gfx9_64b),
-	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_gfx9_unconstrained),
+	KUNIT_CASE(dm_test_fill_gfx9_plane_attributes_dcc_ind_blk),
 	/* amdgpu_dm_plane_helper_check_state() */
-	KUNIT_CASE(dm_test_helper_check_state_viewport_reject),
-	KUNIT_CASE(dm_test_helper_check_state_small_viewport_width),
-	KUNIT_CASE(dm_test_helper_check_state_small_viewport_height),
-	KUNIT_CASE(dm_test_helper_check_state_bottom_clipped_height),
+	KUNIT_CASE(dm_test_helper_check_state_viewport_rejects),
 	KUNIT_CASE(dm_test_helper_check_state_scaling_caps),
+	/* amdgpu_dm_plane_helper_prepare_fb() */
+	KUNIT_CASE(dm_test_helper_prepare_fb_no_fb),
+	/* amdgpu_dm_plane_helper_cleanup_fb() */
+	KUNIT_CASE(dm_test_helper_cleanup_fb_no_fb),
+	/* amdgpu_dm_plane_handle_cursor_update() */
+	KUNIT_CASE(dm_test_handle_cursor_update_no_fb),
+	KUNIT_CASE(dm_test_handle_cursor_update_disabled),
+	KUNIT_CASE(dm_test_handle_cursor_update_no_stream),
+	/* amdgpu_dm_plane_atomic_async_update() */
+	KUNIT_CASE(dm_test_atomic_async_update_copies_state),
 	/* amdgpu_dm_plane_atomic_async_check() */
 	KUNIT_CASE(dm_test_atomic_async_check_rejects),
 	KUNIT_CASE(dm_test_atomic_async_check_overlay_cursor),
 	/* amdgpu_dm_plane_atomic_check() */
 	KUNIT_CASE(dm_test_atomic_check_no_dc_state),
 	KUNIT_CASE(dm_test_atomic_check_missing_crtc_state),
-	KUNIT_CASE(dm_test_atomic_check_helper_failure),
-	KUNIT_CASE(dm_test_atomic_check_color_pipeline_conflict),
-	KUNIT_CASE(dm_test_atomic_check_scaling_failure),
+	KUNIT_CASE(dm_test_atomic_check_rejects),
+	KUNIT_CASE(dm_test_atomic_check_success),
 	/* amdgpu_dm_plane_panic_flush() */
 	KUNIT_CASE(dm_test_panic_flush_no_dc_state),
-	/* amdgpu_dm_plane_drm_plane_reset() */
-	KUNIT_CASE(dm_test_plane_reset_initializes_state),
+	KUNIT_CASE(dm_test_panic_flush_disables_dcc),
+	/* amdgpu_dm_plane_drm_plane_create_state() */
+	KUNIT_CASE(dm_test_plane_create_state_initializes_state),
 	/* amdgpu_dm_plane_drm_plane_duplicate_state() */
 	KUNIT_CASE(dm_test_plane_duplicate_state_copies_fields),
+	KUNIT_CASE(dm_test_plane_duplicate_state_copies_resources),
 	/* amdgpu_dm_plane_drm_plane_destroy_state() */
 	KUNIT_CASE(dm_test_plane_destroy_state_minimal),
+	KUNIT_CASE(dm_test_plane_destroy_state_releases_resources),
 	/* amdgpu_dm_plane_add_modifier() */
-	KUNIT_CASE(dm_test_add_modifier_appends_value),
-	KUNIT_CASE(dm_test_add_modifier_grows_capacity),
-	KUNIT_CASE(dm_test_add_modifier_noop_when_mods_null),
+	KUNIT_CASE(dm_test_add_modifier_appends_and_grows),
+	KUNIT_CASE(dm_test_add_modifier_failure_paths),
+	/* amdgpu_dm_plane_add_modifier_dedup() */
+	KUNIT_CASE(dm_test_add_modifier_dedup),
+	/* amdgpu_dm_plane_fill_gfx6_tiling_info_from_modifier() */
+	KUNIT_CASE(dm_test_fill_gfx6_tiling_info_from_modifier),
+	/* amdgpu_dm_plane_get_gfx6_tile_idx() */
+	KUNIT_CASE(dm_test_get_gfx6_tile_idx),
+	/* amdgpu_dm_plane_calc_gfx7_tile_split() */
+	KUNIT_CASE(dm_test_calc_gfx7_tile_split),
+	/* amdgpu_dm_plane_get_gfx7_macro_tile_idx() */
+	KUNIT_CASE(dm_test_get_gfx7_macro_tile_idx),
+	/* amdgpu_dm_plane_calc_gfx6_mod() */
+	KUNIT_CASE(dm_test_calc_gfx6_mod),
+	/* amdgpu_dm_plane_gfx6_format_mod_supported() */
+	KUNIT_CASE(dm_test_gfx6_format_mod_supported),
 	/* amdgpu_dm_plane_fill_gfx9_tiling_info_from_device() */
-	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_device_pre_10_3),
-	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_device_10_3_plus),
+	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_device),
 	/* amdgpu_dm_plane_fill_gfx9_tiling_info_from_modifier() */
-	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_modifier_linear),
-	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_modifier_pre_nv),
-	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_modifier_nv),
+	KUNIT_CASE(dm_test_fill_gfx9_tiling_info_from_modifier),
 	/* amdgpu_dm_plane_validate_dcc() */
-	KUNIT_CASE(dm_test_validate_dcc_disabled_returns_success),
-	KUNIT_CASE(dm_test_validate_dcc_video_non_gfx12_fails),
-	KUNIT_CASE(dm_test_validate_dcc_missing_cap_func_fails),
-	KUNIT_CASE(dm_test_validate_dcc_cap_callback_fails),
-	KUNIT_CASE(dm_test_validate_dcc_not_capable_fails),
+	KUNIT_CASE(dm_test_validate_dcc_early_exits),
+	KUNIT_CASE(dm_test_validate_dcc_rejects_from_cap),
 	KUNIT_CASE(dm_test_validate_dcc_success_and_scan_mapping),
-	KUNIT_CASE(dm_test_validate_dcc_independent_64b_mismatch_fails),
+	/* amdgpu_dm_plane_init() */
+	KUNIT_CASE(dm_test_plane_init_creates_optional_properties),
+	KUNIT_CASE(dm_test_plane_init_creates_color_pipeline),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c
index e48bac7fb024ce..bdbceb83a8a8fd 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_services_test.c
@@ -277,6 +277,212 @@ static void dm_test_query_brightness_caps_non_lcd1_uses_second_slot(struct kunit
 	KUNIT_EXPECT_EQ(test, caps.data_points[0].signal_level, 0);
 }
 
+/* Tests for dm_allocate_gpu_mem() and dm_free_gpu_mem() */
+
+#define DM_TEST_FAKE_GPU_ADDR 0xF00DBEEFULL
+
+struct dm_test_bo_ops_ctx {
+	void *cpu_ptr;
+	u64 gpu_addr;
+	int create_ret;
+	unsigned long create_size;
+	int create_align;
+	u32 create_domain;
+	unsigned int create_calls;
+	unsigned int free_calls;
+};
+
+static struct dm_test_bo_ops_ctx dm_test_bo_ctx;
+
+static int dm_test_bo_create_kernel(struct amdgpu_device *adev, unsigned long size,
+				    int align, u32 domain, struct amdgpu_bo **bo_ptr,
+				    u64 *gpu_addr, void **cpu_addr)
+{
+	dm_test_bo_ctx.create_calls++;
+	dm_test_bo_ctx.create_size = size;
+	dm_test_bo_ctx.create_align = align;
+	dm_test_bo_ctx.create_domain = domain;
+
+	if (dm_test_bo_ctx.create_ret)
+		return dm_test_bo_ctx.create_ret;
+
+	*gpu_addr = dm_test_bo_ctx.gpu_addr;
+	*cpu_addr = dm_test_bo_ctx.cpu_ptr;
+
+	return 0;
+}
+
+static void dm_test_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, void **cpu_addr)
+{
+	dm_test_bo_ctx.free_calls++;
+
+	*bo = NULL;
+	*gpu_addr = 0;
+	*cpu_addr = NULL;
+}
+
+static const struct amdgpu_dm_services_kunit_ops dm_test_bo_ops = {
+	.bo_create_kernel = dm_test_bo_create_kernel,
+	.bo_free_kernel = dm_test_bo_free_kernel,
+};
+
+/**
+ * dm_test_gpu_mem_init - Install the fake buffer object ops
+ * @test: The KUnit test context
+ *
+ * amdgpu_bo_create_kernel() and amdgpu_bo_free_kernel() need a live TTM
+ * device, so route them through a fake table for the duration of the test.
+ */
+static int dm_test_gpu_mem_init(struct kunit *test)
+{
+	void *cpu_ptr;
+
+	cpu_ptr = kunit_kzalloc(test, sizeof(cpu_ptr), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, cpu_ptr);
+
+	dm_test_bo_ctx = (struct dm_test_bo_ops_ctx) {
+		.cpu_ptr = cpu_ptr,
+		.gpu_addr = DM_TEST_FAKE_GPU_ADDR,
+	};
+
+	amdgpu_dm_services_kunit_set_ops(&dm_test_bo_ops);
+
+	return 0;
+}
+
+static void dm_test_gpu_mem_exit(struct kunit *test)
+{
+	amdgpu_dm_services_kunit_set_ops(NULL);
+}
+
+static struct amdgpu_device *dm_test_alloc_adev(struct kunit *test)
+{
+	struct amdgpu_device *adev;
+
+	adev = kunit_kzalloc(test, sizeof(*adev), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev);
+	INIT_LIST_HEAD(&adev->dm.da_list);
+
+	return adev;
+}
+
+/**
+ * dm_test_allocate_gpu_mem_gart - Test Allocate gpu mem gart
+ * @test: The KUnit test context
+ */
+static void dm_test_allocate_gpu_mem_gart(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev(test);
+	long long addr = 0;
+	void *mem;
+
+	mem = dm_allocate_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, 4096, &addr);
+
+	KUNIT_EXPECT_PTR_EQ(test, mem, dm_test_bo_ctx.cpu_ptr);
+	KUNIT_EXPECT_EQ(test, addr, (long long)DM_TEST_FAKE_GPU_ADDR);
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.create_calls, 1U);
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.create_size, 4096UL);
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.create_align, (int)PAGE_SIZE);
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.create_domain, (u32)AMDGPU_GEM_DOMAIN_GTT);
+	KUNIT_EXPECT_FALSE(test, list_empty(&adev->dm.da_list));
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, mem);
+}
+
+/**
+ * dm_test_allocate_gpu_mem_frame_buffer - Test Allocate gpu mem frame buffer
+ * @test: The KUnit test context
+ *
+ * Any non-GART allocation type must land in the VRAM domain.
+ */
+static void dm_test_allocate_gpu_mem_frame_buffer(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev(test);
+	long long addr = 0;
+	void *mem;
+
+	mem = dm_allocate_gpu_mem(adev, DC_MEM_ALLOC_TYPE_FRAME_BUFFER, 8192, &addr);
+
+	KUNIT_EXPECT_PTR_EQ(test, mem, dm_test_bo_ctx.cpu_ptr);
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.create_domain, (u32)AMDGPU_GEM_DOMAIN_VRAM);
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.create_size, 8192UL);
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_FRAME_BUFFER, mem);
+}
+
+/**
+ * dm_test_allocate_gpu_mem_create_fails - Test Allocate gpu mem create fails
+ * @test: The KUnit test context
+ */
+static void dm_test_allocate_gpu_mem_create_fails(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev(test);
+	long long addr = 0;
+	void *mem;
+
+	dm_test_bo_ctx.create_ret = -ENOMEM;
+
+	mem = dm_allocate_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, 4096, &addr);
+
+	KUNIT_EXPECT_NULL(test, mem);
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.create_calls, 1U);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+}
+
+/**
+ * dm_test_free_gpu_mem_matching - Test Free gpu mem matching
+ * @test: The KUnit test context
+ */
+static void dm_test_free_gpu_mem_matching(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev(test);
+	long long addr = 0;
+	void *mem;
+
+	mem = dm_allocate_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, 4096, &addr);
+	KUNIT_ASSERT_NOT_NULL(test, mem);
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, mem);
+
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.free_calls, 1U);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+}
+
+/**
+ * dm_test_free_gpu_mem_no_match - Test Free gpu mem no match
+ * @test: The KUnit test context
+ */
+static void dm_test_free_gpu_mem_no_match(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev(test);
+	long long addr = 0;
+	void *mem;
+
+	mem = dm_allocate_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, 4096, &addr);
+	KUNIT_ASSERT_NOT_NULL(test, mem);
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, (char *)mem + 1);
+
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.free_calls, 0U);
+	KUNIT_EXPECT_FALSE(test, list_empty(&adev->dm.da_list));
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, mem);
+}
+
+/**
+ * dm_test_free_gpu_mem_empty_list - Test Free gpu mem empty list
+ * @test: The KUnit test context
+ */
+static void dm_test_free_gpu_mem_empty_list(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_alloc_adev(test);
+
+	dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, dm_test_bo_ctx.cpu_ptr);
+
+	KUNIT_EXPECT_EQ(test, dm_test_bo_ctx.free_calls, 0U);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+}
+
 static struct kunit_case amdgpu_dm_services_test_cases[] = {
 	/* dm_get_elapse_time_in_ns */
 	KUNIT_CASE(dm_test_get_elapse_time_zero_delta),
@@ -307,7 +513,27 @@ static struct kunit_suite amdgpu_dm_services_test_suite = {
 	.test_cases = amdgpu_dm_services_test_cases,
 };
 
-kunit_test_suite(amdgpu_dm_services_test_suite);
+static struct kunit_case amdgpu_dm_services_gpu_mem_test_cases[] = {
+	/* dm_allocate_gpu_mem */
+	KUNIT_CASE(dm_test_allocate_gpu_mem_gart),
+	KUNIT_CASE(dm_test_allocate_gpu_mem_frame_buffer),
+	KUNIT_CASE(dm_test_allocate_gpu_mem_create_fails),
+	/* dm_free_gpu_mem */
+	KUNIT_CASE(dm_test_free_gpu_mem_matching),
+	KUNIT_CASE(dm_test_free_gpu_mem_no_match),
+	KUNIT_CASE(dm_test_free_gpu_mem_empty_list),
+	{}
+};
+
+static struct kunit_suite amdgpu_dm_services_gpu_mem_test_suite = {
+	.name = "amdgpu_dm_services_gpu_mem",
+	.init = dm_test_gpu_mem_init,
+	.exit = dm_test_gpu_mem_exit,
+	.test_cases = amdgpu_dm_services_gpu_mem_test_cases,
+};
+
+kunit_test_suites(&amdgpu_dm_services_test_suite,
+		  &amdgpu_dm_services_gpu_mem_test_suite);
 
 MODULE_DESCRIPTION("KUnit tests for amdgpu_dm_services");
 MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c
index 5645866610fcc9..4c6e9e5d4d7225 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_test.c
@@ -8,30 +8,38 @@
 #include <kunit/test.h>
 #include <linux/pci.h>
 #include <drm/drm_atomic.h>
+#include <drm/drm_blend.h>
 #include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_kunit_helpers.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_modeset_lock.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_property.h>
+#include <drm/drm_vblank.h>
 #include <drm/drm_writeback.h>
 
 #include "dc.h"
+#include "dc/dc_dmub_srv.h"
+#include "dal_asic_id.h"
+#include "dm_services_types.h"
+#include "dmub/dmub_srv.h"
 #include "inc/core_types.h"
+#include "logger_types.h"
 #include "amd_shared.h"
 #include "amdgpu.h"
 #include "amdgpu_mode.h"
+#include "atom.h"
 #include "amdgpu_dm.h"
+#include "amdgpu_dm_audio.h"
+#include "amdgpu_dm_hdcp.h"
+#include "amdgpu_dm_mst_types.h"
 #include "amdgpu_dm_kunit_test_helpers.h"
 
 /* Tests for simple DM callbacks */
 
-/**
- * dm_test_is_idle - Test placeholder idle callback returns true
- * @test: The KUnit test context
- */
-static void dm_test_is_idle(struct kunit *test)
-{
-	KUNIT_EXPECT_TRUE(test, dm_is_idle(NULL));
-}
-
 /**
  * dm_test_wait_for_idle - Test placeholder wait-for-idle callback returns success
  * @test: The KUnit test context
@@ -319,6 +327,119 @@ static void dm_test_plane_layer_index_cmp_ascending(struct kunit *test)
 	KUNIT_EXPECT_LT(test, dm_plane_layer_index_cmp(&sa, &sb), 0);
 }
 
+struct dm_test_plane_update_ops_ctx {
+	struct dc *dc;
+	struct dc_surface_update *surface_updates;
+	struct dc_stream_state *stream;
+	struct dc_stream_update *stream_update;
+	int surface_count;
+	unsigned int call_seq;
+	unsigned int post_update_seq;
+	unsigned int update_seq;
+	bool update_ret;
+};
+
+static struct dm_test_plane_update_ops_ctx dm_test_plane_update_ctx;
+
+static void dm_test_post_update_surfaces_to_stream(struct dc *dc)
+{
+	dm_test_plane_update_ctx.dc = dc;
+	dm_test_plane_update_ctx.post_update_seq = ++dm_test_plane_update_ctx.call_seq;
+}
+
+static bool dm_test_update_planes_and_stream(struct dc *dc,
+					     struct dc_surface_update *surface_updates,
+					     int surface_count,
+					     struct dc_stream_state *dc_stream,
+					     struct dc_stream_update *stream_update)
+{
+	dm_test_plane_update_ctx.dc = dc;
+	dm_test_plane_update_ctx.surface_updates = surface_updates;
+	dm_test_plane_update_ctx.surface_count = surface_count;
+	dm_test_plane_update_ctx.stream = dc_stream;
+	dm_test_plane_update_ctx.stream_update = stream_update;
+	dm_test_plane_update_ctx.update_seq = ++dm_test_plane_update_ctx.call_seq;
+
+	return dm_test_plane_update_ctx.update_ret;
+}
+
+static const struct amdgpu_dm_kunit_ops dm_test_plane_update_ops = {
+	.post_update_surfaces_to_stream = dm_test_post_update_surfaces_to_stream,
+	.update_planes_and_stream = dm_test_update_planes_and_stream,
+};
+
+static void dm_test_restore_dm_ops(void *ctx)
+{
+	amdgpu_dm_kunit_set_ops(NULL);
+}
+
+static void dm_test_install_dm_ops(struct kunit *test,
+				   const struct amdgpu_dm_kunit_ops *ops)
+{
+	amdgpu_dm_kunit_set_ops(ops);
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_restore_dm_ops, NULL), 0);
+}
+
+/**
+ * dm_test_update_planes_adapter_sorts_and_forwards - Test sorting and call order
+ * @test: The KUnit test context
+ */
+static void dm_test_update_planes_adapter_sorts_and_forwards(struct kunit *test)
+{
+	struct dc_surface_update *updates;
+	struct dc_plane_state *planes;
+	struct dc_stream_update *stream_update;
+	struct dc_stream_state *stream;
+	struct dc *dc;
+
+	updates = kunit_kcalloc(test, 3, sizeof(*updates), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, updates);
+	planes = kunit_kcalloc(test, 3, sizeof(*planes), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, planes);
+	stream_update = kunit_kzalloc(test, sizeof(*stream_update), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, stream_update);
+	stream = dm_kunit_alloc_stream(test, NULL);
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	planes[0].layer_index = 1;
+	planes[1].layer_index = 5;
+	planes[2].layer_index = 3;
+	updates[0].surface = &planes[0];
+	updates[1].surface = &planes[1];
+	updates[2].surface = &planes[2];
+	dm_test_plane_update_ctx = (struct dm_test_plane_update_ops_ctx) {
+		.update_ret = true,
+	};
+	dm_test_install_dm_ops(test, &dm_test_plane_update_ops);
+
+	KUNIT_EXPECT_TRUE(test, update_planes_and_stream_adapter(dc, 3,
+								 stream, stream_update, updates));
+	KUNIT_EXPECT_EQ(test, updates[0].surface->layer_index, 5);
+	KUNIT_EXPECT_EQ(test, updates[1].surface->layer_index, 3);
+	KUNIT_EXPECT_EQ(test, updates[2].surface->layer_index, 1);
+	KUNIT_EXPECT_PTR_EQ(test, dm_test_plane_update_ctx.dc, dc);
+	KUNIT_EXPECT_PTR_EQ(test, dm_test_plane_update_ctx.surface_updates, &updates[0]);
+	KUNIT_EXPECT_EQ(test, dm_test_plane_update_ctx.surface_count, 3);
+	KUNIT_EXPECT_PTR_EQ(test, dm_test_plane_update_ctx.stream, stream);
+	KUNIT_EXPECT_PTR_EQ(test, dm_test_plane_update_ctx.stream_update, stream_update);
+	KUNIT_EXPECT_LT(test, dm_test_plane_update_ctx.post_update_seq,
+			dm_test_plane_update_ctx.update_seq);
+}
+
+/**
+ * dm_test_update_planes_adapter_propagates_failure - Test DC failure is returned
+ * @test: The KUnit test context
+ */
+static void dm_test_update_planes_adapter_propagates_failure(struct kunit *test)
+{
+	dm_test_plane_update_ctx = (struct dm_test_plane_update_ops_ctx) { 0 };
+	dm_test_install_dm_ops(test, &dm_test_plane_update_ops);
+
+	KUNIT_EXPECT_FALSE(test, update_planes_and_stream_adapter(NULL, 0,
+								  NULL, NULL, NULL));
+}
+
 /* Tests for fill_plane_color_attributes() */
 
 /**
@@ -335,7 +456,7 @@ static void dm_test_fill_color_attr_rgb_format(struct kunit *test)
 	plane_state.color_encoding = DRM_COLOR_YCBCR_BT709;
 	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_GRPH_ARGB8888,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, 0);
@@ -355,7 +476,7 @@ static void dm_test_fill_color_attr_bt601_full(struct kunit *test)
 	plane_state.color_encoding = DRM_COLOR_YCBCR_BT601;
 	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, 0);
@@ -375,7 +496,7 @@ static void dm_test_fill_color_attr_bt601_limited(struct kunit *test)
 	plane_state.color_encoding = DRM_COLOR_YCBCR_BT601;
 	plane_state.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, 0);
@@ -396,7 +517,7 @@ static void dm_test_fill_color_attr_bt709_full(struct kunit *test)
 	plane_state.color_encoding = DRM_COLOR_YCBCR_BT709;
 	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, 0);
@@ -416,7 +537,7 @@ static void dm_test_fill_color_attr_bt709_limited(struct kunit *test)
 	plane_state.color_encoding = DRM_COLOR_YCBCR_BT709;
 	plane_state.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, 0);
@@ -437,7 +558,7 @@ static void dm_test_fill_color_attr_bt2020_full(struct kunit *test)
 	plane_state.color_encoding = DRM_COLOR_YCBCR_BT2020;
 	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, 0);
@@ -458,7 +579,7 @@ static void dm_test_fill_color_attr_bt2020_limited(struct kunit *test)
 	plane_state.color_encoding = DRM_COLOR_YCBCR_BT2020;
 	plane_state.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, 0);
@@ -479,7 +600,7 @@ static void dm_test_fill_color_attr_invalid_encoding(struct kunit *test)
 	plane_state.color_encoding = 99;
 	plane_state.color_range = DRM_COLOR_YCBCR_FULL_RANGE;
 
-	ret = fill_plane_color_attributes(&plane_state,
+	ret = fill_plane_color_attributes(NULL, &plane_state,
 					  SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr,
 					  &color_space);
 	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
@@ -1150,9 +1271,3951 @@ static void dm_test_apply_delay_nonzero_wait(struct kunit *test)
 	amdgpu_dm_apply_delay_after_dpcd_poweroff(adev, sink);
 }
 
+/*
+ * Attach a DC stream to CRTC 0 of @adev so the scanout helpers walk the DC
+ * resource context instead of bailing out early.
+ */
+static void dm_test_crtc_with_stream(struct kunit *test,
+				     struct amdgpu_device *adev, struct dc *dc)
+{
+	struct dc_stream_state *stream;
+	struct amdgpu_crtc *acrtc;
+	struct dal_logger *logger;
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+
+	logger = kunit_kzalloc(test, sizeof(*logger), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, logger);
+	logger->dev = &adev->ddev;
+	dc->ctx->logger = logger;
+
+	dc->current_state = dm_kunit_alloc_dc_state(test);
+	KUNIT_ASSERT_NOT_NULL(test, dc->current_state);
+
+	stream = dm_kunit_alloc_stream(test, NULL);
+	stream->ctx = dc->ctx;
+
+	adev->dm.dc = dc;
+	adev->mode_info.num_crtc = 1;
+	adev->mode_info.crtcs[0] = acrtc;
+	acrtc->dm_irq_params.stream = stream;
+}
+
+/**
+ * dm_test_vblank_get_counter_unmapped_stream - Test a stream without a pipe returns zero
+ * @test: The KUnit test context
+ */
+static void dm_test_vblank_get_counter_unmapped_stream(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	dm_test_crtc_with_stream(test, adev, dc);
+
+	KUNIT_EXPECT_EQ(test, dm_vblank_get_counter(adev, 0), 0U);
+}
+
+/**
+ * dm_test_crtc_get_scanoutpos_unmapped_stream - Test scanout position for an unmapped stream
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_scanoutpos_unmapped_stream(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+	u32 vbl = 0xdeadbeef;
+	u32 position = 0xdeadbeef;
+
+	dm_test_crtc_with_stream(test, adev, dc);
+
+	KUNIT_EXPECT_EQ(test, dm_crtc_get_scanoutpos(adev, 0, &vbl, &position), 0);
+	KUNIT_EXPECT_EQ(test, vbl, 0U);
+	KUNIT_EXPECT_EQ(test, position, 0U);
+}
+
+/**
+ * dm_test_crtc_get_scanoutpos_exits_idle - Test idle optimizations are disabled first
+ * @test: The KUnit test context
+ */
+static void dm_test_crtc_get_scanoutpos_exits_idle(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+	u32 vbl = 0;
+	u32 position = 0;
+
+	dm_test_crtc_with_stream(test, adev, dc);
+	dc->caps.ips_support = true;
+	dc->idle_optimizations_allowed = true;
+
+	KUNIT_EXPECT_EQ(test, dm_crtc_get_scanoutpos(adev, 0, &vbl, &position), 0);
+}
+
+static struct drm_atomic_commit *dm_test_alloc_commit(struct kunit *test,
+						     struct amdgpu_device *adev)
+{
+	struct drm_atomic_commit *state;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	state->dev = &adev->ddev;
+
+	return state;
+}
+
+/**
+ * dm_test_atomic_get_state_already_acquired - Test an acquired DM state is returned as is
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_get_state_already_acquired(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state = dm_test_alloc_commit(test, adev);
+	struct dm_atomic_state *dm_state;
+	struct dm_atomic_state *acquired;
+
+	dm_state = kunit_kzalloc(test, sizeof(*dm_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	acquired = dm_state;
+
+	KUNIT_EXPECT_EQ(test, dm_atomic_get_state(state, &acquired), 0);
+	KUNIT_EXPECT_PTR_EQ(test, acquired, dm_state);
+}
+
+/**
+ * dm_test_atomic_duplicate_state_no_context - Test duplication fails without a DC context
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_duplicate_state_no_context(struct kunit *test)
+{
+	struct dm_atomic_state *old_state;
+	struct drm_private_obj *obj;
+
+	obj = kunit_kzalloc(test, sizeof(*obj), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, obj);
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_state);
+
+	obj->state = &old_state->base;
+
+	KUNIT_EXPECT_NULL(test, dm_atomic_duplicate_state(obj));
+}
+
+/* Tests for add_affected_mst_dsc_crtcs() */
+
+/**
+ * dm_test_add_affected_mst_dsc_crtcs_no_connector - Test an empty state adds nothing
+ * @test: The KUnit test context
+ */
+static void dm_test_add_affected_mst_dsc_crtcs_no_connector(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state = dm_test_alloc_commit(test, adev);
+	struct drm_crtc *crtc;
+
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	KUNIT_EXPECT_EQ(test, add_affected_mst_dsc_crtcs(state, crtc), 0);
+}
+
+/*
+ * Build an atomic state holding a single connector bound to @crtc.
+ */
+static struct drm_atomic_commit *
+dm_test_state_with_connector(struct kunit *test, struct drm_connector *connector,
+			     struct drm_crtc *crtc)
+{
+	struct drm_connector_state *conn_state;
+	struct drm_atomic_commit *state;
+
+	state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	conn_state = kunit_kzalloc(test, sizeof(*conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+
+	state->connectors = kunit_kzalloc(test, sizeof(*state->connectors),
+					  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state->connectors);
+
+	conn_state->crtc = crtc;
+	state->num_connector = 1;
+	state->connectors[0].ptr = connector;
+	state->connectors[0].old_state = conn_state;
+	state->connectors[0].new_state = conn_state;
+
+	return state;
+}
+
+/**
+ * dm_test_add_affected_mst_dsc_crtcs_writeback - Test writeback connectors are ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_add_affected_mst_dsc_crtcs_writeback(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_WRITEBACK;
+	state = dm_test_state_with_connector(test, &aconnector->base, crtc);
+
+	KUNIT_EXPECT_EQ(test, add_affected_mst_dsc_crtcs(state, crtc), 0);
+}
+
+/**
+ * dm_test_add_affected_mst_dsc_crtcs_not_mst - Test a non-MST connector adds nothing
+ * @test: The KUnit test context
+ */
+static void dm_test_add_affected_mst_dsc_crtcs_not_mst(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_DisplayPort;
+	state = dm_test_state_with_connector(test, &aconnector->base, crtc);
+
+	KUNIT_EXPECT_EQ(test, add_affected_mst_dsc_crtcs(state, crtc), 0);
+}
+
+/**
+ * dm_test_add_affected_mst_dsc_crtcs_other_crtc - Test connectors on other CRTCs are skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_add_affected_mst_dsc_crtcs_other_crtc(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc, *other_crtc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	other_crtc = kunit_kzalloc(test, sizeof(*other_crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, other_crtc);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_DisplayPort;
+	state = dm_test_state_with_connector(test, &aconnector->base, other_crtc);
+
+	KUNIT_EXPECT_EQ(test, add_affected_mst_dsc_crtcs(state, crtc), 0);
+}
+
+/**
+ * dm_test_add_affected_mst_dsc_crtcs_disabled - Test a disabled connector uses its old state
+ * @test: The KUnit test context
+ */
+static void dm_test_add_affected_mst_dsc_crtcs_disabled(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_connector_state *new_conn_state;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	new_conn_state = kunit_kzalloc(test, sizeof(*new_conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_conn_state);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_DisplayPort;
+	state = dm_test_state_with_connector(test, &aconnector->base, crtc);
+	state->connectors[0].new_state = new_conn_state;
+
+	KUNIT_EXPECT_EQ(test, add_affected_mst_dsc_crtcs(state, crtc), 0);
+}
+
+/* Tests for amdgpu_dm_commit_cursors() */
+
+/**
+ * dm_test_commit_cursors_no_planes - Test an empty plane set is a no-op
+ * @test: The KUnit test context
+ */
+static void dm_test_commit_cursors_no_planes(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_commit_cursors(dm_test_alloc_commit(test, adev));
+}
+
+/*
+ * Build a commit holding a single plane of @type with empty old and new plane
+ * states.
+ */
+static struct drm_atomic_commit *
+dm_test_commit_with_plane(struct kunit *test, struct amdgpu_device *adev,
+			  enum drm_plane_type type)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_commit(test, adev);
+	struct drm_plane_state *old_state, *new_state;
+	struct drm_plane *plane;
+
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	old_state = kunit_kzalloc(test, sizeof(*old_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_state);
+	new_state = kunit_kzalloc(test, sizeof(*new_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_state);
+	state->planes = kunit_kzalloc(test, sizeof(*state->planes), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state->planes);
+
+	plane->type = type;
+	plane->dev = &adev->ddev;
+	plane->state = new_state;
+	adev->ddev.mode_config.num_total_plane = 1;
+	state->planes[0].ptr = plane;
+	state->planes[0].old_state = old_state;
+
+	return state;
+}
+
+/**
+ * dm_test_commit_cursors_skips_non_cursor - Test non-cursor planes are skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_commit_cursors_skips_non_cursor(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_commit_cursors(dm_test_commit_with_plane(test, adev,
+							   DRM_PLANE_TYPE_PRIMARY));
+}
+
+/**
+ * dm_test_commit_cursors_updates_cursor - Test cursor planes reach the cursor update
+ * @test: The KUnit test context
+ */
+static void dm_test_commit_cursors_updates_cursor(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_commit_cursors(dm_test_commit_with_plane(test, adev,
+							   DRM_PLANE_TYPE_CURSOR));
+}
+
+/**
+ * dm_test_update_cursor_no_framebuffer - Test missing framebuffers leave the update untouched
+ * @test: The KUnit test context
+ */
+static void dm_test_update_cursor_no_framebuffer(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state;
+	struct dc_stream_update *update;
+	struct drm_plane *plane;
+
+	state = dm_test_commit_with_plane(test, adev, DRM_PLANE_TYPE_CURSOR);
+	plane = state->planes[0].ptr;
+	update = kunit_kzalloc(test, sizeof(*update), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, update);
+
+	update->cursor_attributes = (void *)1;
+	update->cursor_position = (void *)1;
+
+	amdgpu_dm_update_cursor(plane, state->planes[0].old_state, update);
+
+	KUNIT_EXPECT_PTR_EQ(test, update->cursor_attributes, (void *)1);
+	KUNIT_EXPECT_PTR_EQ(test, update->cursor_position, (void *)1);
+}
+
+/**
+ * dm_test_update_cursor_disables_stream - Test removing the framebuffer disables the cursor
+ * @test: The KUnit test context
+ */
+static void dm_test_update_cursor_disables_stream(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state;
+	struct dc_stream_update *update;
+	struct drm_plane_state *old_plane_state;
+	struct dm_crtc_state *crtc_state;
+	struct amdgpu_framebuffer *afb;
+	struct amdgpu_crtc *acrtc;
+	struct drm_plane *plane;
+
+	state = dm_test_commit_with_plane(test, adev, DRM_PLANE_TYPE_CURSOR);
+	plane = state->planes[0].ptr;
+	old_plane_state = state->planes[0].old_state;
+	crtc_state = kunit_kzalloc(test, sizeof(*crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc_state);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	update = kunit_kzalloc(test, sizeof(*update), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, update);
+
+	old_plane_state->fb = &afb->base;
+	old_plane_state->crtc = &acrtc->base;
+	acrtc->base.state = &crtc_state->base;
+	crtc_state->stream = dm_kunit_alloc_stream(test, NULL);
+	crtc_state->stream->cursor_position.enable = true;
+
+	amdgpu_dm_update_cursor(plane, old_plane_state, update);
+
+	KUNIT_EXPECT_FALSE(test, crtc_state->stream->cursor_position.enable);
+	KUNIT_EXPECT_PTR_EQ(test, update->cursor_position,
+			    &crtc_state->stream->cursor_position);
+	KUNIT_EXPECT_NULL(test, update->cursor_attributes);
+}
+
+/* Tests for dm_arm_vblank_event() */
+
+struct dm_test_vblank_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state *acrtc_state;
+	struct drm_pending_vblank_event *event;
+};
+
+static int dm_test_vblank_enable(struct drm_crtc *crtc)
+{
+	return 0;
+}
+
+static void dm_test_vblank_disable(struct drm_crtc *crtc)
+{
+}
+
+static u32 dm_test_vblank_counter(struct drm_crtc *crtc)
+{
+	return 0;
+}
+
+static const struct drm_crtc_funcs dm_test_vblank_crtc_funcs = {
+	.enable_vblank = dm_test_vblank_enable,
+	.disable_vblank = dm_test_vblank_disable,
+	.get_vblank_counter = dm_test_vblank_counter,
+};
+
+static void dm_test_vblank_ctx_cleanup(void *data)
+{
+	struct amdgpu_crtc *acrtc = data;
+
+	if (drm_dev_has_vblank(acrtc->base.dev))
+		drm_crtc_vblank_off(&acrtc->base);
+	list_del_init(&acrtc->base.head);
+}
+
+/*
+ * A CRTC with one active plane and a pending vblank event. There is no
+ * initialised vblank, so drm_crtc_vblank_get() fails, which the function under
+ * test ignores.
+ */
+static struct dm_test_vblank_ctx *dm_test_vblank_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->acrtc = kunit_kzalloc(test, sizeof(*ctx->acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc);
+	ctx->acrtc_state = kunit_kzalloc(test, sizeof(*ctx->acrtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc_state);
+	ctx->event = kunit_kzalloc(test, sizeof(*ctx->event), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->event);
+
+	ctx->acrtc->base.dev = &ctx->adev->ddev;
+	ctx->acrtc->base.funcs = &dm_test_vblank_crtc_funcs;
+	INIT_LIST_HEAD(&ctx->acrtc->base.head);
+	list_add_tail(&ctx->acrtc->base.head,
+		      &ctx->adev->ddev.mode_config.crtc_list);
+	ctx->acrtc->base.state = &ctx->acrtc_state->base;
+	ctx->acrtc_state->base.event = ctx->event;
+	ctx->acrtc_state->active_planes = 1;
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_vblank_ctx_cleanup, ctx->acrtc), 0);
+
+	return ctx;
+}
+
+static void dm_test_arm_vblank(struct dm_test_vblank_ctx *ctx, bool pflip_update,
+			       bool cursor_update)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&ctx->adev->ddev.event_lock, flags);
+	dm_arm_vblank_event(ctx->acrtc, ctx->acrtc_state, pflip_update,
+			    cursor_update);
+	spin_unlock_irqrestore(&ctx->adev->ddev.event_lock, flags);
+}
+
+static void dm_test_arm_vblank_pre_programming(struct dm_test_vblank_ctx *ctx,
+					       bool pflip_update, bool cursor_update)
+{
+	struct dm_crtc_state *state = ctx->acrtc_state;
+	struct amdgpu_crtc *acrtc = ctx->acrtc;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ctx->adev->ddev.event_lock, flags);
+	dm_arm_vblank_event_pre_programming(acrtc, state, pflip_update, cursor_update);
+	spin_unlock_irqrestore(&ctx->adev->ddev.event_lock, flags);
+}
+
+/**
+ * dm_test_arm_vblank_pre_programming_no_event - Test missing event takes no reference
+ * @test: The KUnit test context
+ */
+static void dm_test_arm_vblank_pre_programming_no_event(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx = dm_test_vblank_ctx_alloc(test);
+	struct drm_vblank_crtc *vblank;
+
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&ctx->adev->ddev, 1), 0);
+	vblank = drm_crtc_vblank_crtc(&ctx->acrtc->base);
+	ctx->acrtc_state->base.event = NULL;
+
+	dm_test_arm_vblank_pre_programming(ctx, true, false);
+
+	KUNIT_EXPECT_EQ(test, atomic_read(&vblank->refcount), 0);
+}
+
+/**
+ * dm_test_arm_vblank_pre_programming_no_planes - Test inactive CRTC takes no reference
+ * @test: The KUnit test context
+ */
+static void dm_test_arm_vblank_pre_programming_no_planes(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx = dm_test_vblank_ctx_alloc(test);
+	struct drm_vblank_crtc *vblank;
+
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&ctx->adev->ddev, 1), 0);
+	vblank = drm_crtc_vblank_crtc(&ctx->acrtc->base);
+	ctx->acrtc_state->active_planes = 0;
+
+	dm_test_arm_vblank_pre_programming(ctx, false, true);
+
+	KUNIT_EXPECT_EQ(test, atomic_read(&vblank->refcount), 0);
+}
+
+/**
+ * dm_test_arm_vblank_pre_programming_update - Test an update takes a vblank reference
+ * @test: The KUnit test context
+ */
+static void dm_test_arm_vblank_pre_programming_update(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx = dm_test_vblank_ctx_alloc(test);
+	struct drm_vblank_crtc *vblank;
+
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&ctx->adev->ddev, 1), 0);
+	vblank = drm_crtc_vblank_crtc(&ctx->acrtc->base);
+
+	dm_test_arm_vblank_pre_programming(ctx, true, false);
+
+	KUNIT_EXPECT_EQ(test, atomic_read(&vblank->refcount), 1);
+	drm_crtc_vblank_put(&ctx->acrtc->base);
+}
+
+/**
+ * dm_test_arm_vblank_event_no_event - Test a commit without an event is a no-op
+ * @test: The KUnit test context
+ */
+static void dm_test_arm_vblank_event_no_event(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx = dm_test_vblank_ctx_alloc(test);
+
+	ctx->acrtc_state->base.event = NULL;
+
+	dm_test_arm_vblank(ctx, true, false);
+
+	KUNIT_EXPECT_NULL(test, ctx->acrtc->event);
+}
+
+/**
+ * dm_test_arm_vblank_event_no_active_planes - Test an event is left armed without planes
+ * @test: The KUnit test context
+ */
+static void dm_test_arm_vblank_event_no_active_planes(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx = dm_test_vblank_ctx_alloc(test);
+
+	ctx->acrtc_state->active_planes = 0;
+
+	dm_test_arm_vblank(ctx, false, true);
+
+	KUNIT_EXPECT_NULL(test, ctx->acrtc->event);
+	KUNIT_EXPECT_PTR_EQ(test, ctx->acrtc_state->base.event, ctx->event);
+}
+
+/**
+ * dm_test_arm_vblank_event_pflip - Test a page flip arms the flip ISR
+ * @test: The KUnit test context
+ */
+static void dm_test_arm_vblank_event_pflip(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx = dm_test_vblank_ctx_alloc(test);
+
+	dm_test_arm_vblank(ctx, true, false);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->acrtc->event, ctx->event);
+	KUNIT_EXPECT_NULL(test, ctx->acrtc_state->base.event);
+	KUNIT_EXPECT_EQ(test, (int)ctx->acrtc->pflip_status,
+			(int)AMDGPU_FLIP_SUBMITTED);
+}
+
+/**
+ * dm_test_arm_vblank_event_cursor - Test a cursor update consumes the event
+ * @test: The KUnit test context
+ */
+static void dm_test_arm_vblank_event_cursor(struct kunit *test)
+{
+	struct dm_test_vblank_ctx *ctx = dm_test_vblank_ctx_alloc(test);
+
+	dm_test_arm_vblank(ctx, false, true);
+
+	KUNIT_EXPECT_PTR_EQ(test, ctx->acrtc->event, ctx->event);
+	KUNIT_EXPECT_NULL(test, ctx->acrtc_state->base.event);
+	KUNIT_EXPECT_EQ(test, (int)ctx->acrtc->pflip_status,
+			(int)AMDGPU_FLIP_NONE);
+}
+
+/**
+ * dm_test_update_pflip_irq_state_dcn - Test DCN skips the GRPH_PFLIP reapply
+ * @test: The KUnit test context
+ */
+static void dm_test_update_pflip_irq_state_dcn(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+
+	adev->mode_info.num_crtc = 1;
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 2, 0);
+
+	dm_update_pflip_irq_state(adev, acrtc);
+}
+
+/*
+ * Spy for the pageflip IRQ source: amdgpu_irq_update() always dispatches
+ * through src->funcs->set(), which needs a registered IH ring on real
+ * hardware. Recording the requested state instead keeps the DCE reapply path
+ * reachable and observable.
+ */
+struct dm_test_irq_spy {
+	unsigned int set_count;
+	unsigned int last_type;
+	enum amdgpu_interrupt_state last_state;
+};
+
+static struct dm_test_irq_spy dm_test_irq_spy_data;
+
+static int dm_test_irq_set(struct amdgpu_device *adev,
+			   struct amdgpu_irq_src *src, unsigned int type,
+			   enum amdgpu_interrupt_state state)
+{
+	dm_test_irq_spy_data.set_count++;
+	dm_test_irq_spy_data.last_type = type;
+	dm_test_irq_spy_data.last_state = state;
+
+	return 0;
+}
+
+static const struct amdgpu_irq_src_funcs dm_test_irq_funcs = {
+	.set = dm_test_irq_set,
+};
+
+/**
+ * dm_test_update_pflip_irq_state_dce - Test DCE reapplies the GRPH_PFLIP state
+ * @test: The KUnit test context
+ */
+static void dm_test_update_pflip_irq_state_dce(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_crtc *acrtc;
+
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+
+	memset(&dm_test_irq_spy_data, 0, sizeof(dm_test_irq_spy_data));
+	adev->mode_info.num_crtc = 2;
+	adev->ip_versions[DCE_HWIP][0] = 0;
+	adev->pageflip_irq.funcs = &dm_test_irq_funcs;
+	acrtc->crtc_id = 1;
+	spin_lock_init(&adev->irq.lock);
+
+	dm_update_pflip_irq_state(adev, acrtc);
+
+	KUNIT_EXPECT_EQ(test, dm_test_irq_spy_data.set_count, 1U);
+	KUNIT_EXPECT_EQ(test, dm_test_irq_spy_data.last_type,
+			(unsigned int)AMDGPU_CRTC_IRQ_VBLANK2);
+	KUNIT_EXPECT_EQ(test, (int)dm_test_irq_spy_data.last_state,
+			(int)AMDGPU_IRQ_STATE_DISABLE);
+}
+
+/* Tests for amdgpu_dm_crtc_mem_type_changed() */
+
+struct dm_test_mem_type_ctx {
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct drm_crtc_state *crtc_state;
+	struct drm_plane *plane;
+	struct drm_plane_state *old_plane_state;
+	struct drm_plane_state *new_plane_state;
+};
+
+/*
+ * Register a single plane on the CRTC's plane mask. get_mem_type() walks
+ * fb->obj[0] back to an amdgpu_bo, so the framebuffers are backed by fake
+ * buffer objects with a TTM resource instead of a live TTM device.
+ */
+static struct drm_framebuffer *dm_test_alloc_fb(struct kunit *test,
+						u32 mem_type)
+{
+	struct drm_framebuffer *fb;
+	struct ttm_resource *res;
+	struct amdgpu_bo *abo;
+
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+	abo = kunit_kzalloc(test, sizeof(*abo), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, abo);
+	res = kunit_kzalloc(test, sizeof(*res), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, res);
+
+	res->mem_type = mem_type;
+	abo->tbo.resource = res;
+	fb->obj[0] = &abo->tbo.base;
+
+	return fb;
+}
+
+static struct dm_test_mem_type_ctx *dm_test_mem_type_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_mem_type_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->state = dm_test_alloc_commit(test, ctx->adev);
+	ctx->crtc_state = kunit_kzalloc(test, sizeof(*ctx->crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc_state);
+	ctx->plane = drm_kunit_helper_create_primary_plane(test, &ctx->adev->ddev,
+							   NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->plane);
+	ctx->old_plane_state = kunit_kzalloc(test, sizeof(*ctx->old_plane_state),
+					     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->old_plane_state);
+	ctx->new_plane_state = kunit_kzalloc(test, sizeof(*ctx->new_plane_state),
+					     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->new_plane_state);
+	ctx->state->planes = kunit_kcalloc(test, ctx->plane->index + 1,
+					   sizeof(*ctx->state->planes), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->planes);
+
+	ctx->crtc_state->plane_mask = drm_plane_mask(ctx->plane);
+	ctx->state->planes[ctx->plane->index].ptr = ctx->plane;
+
+	return ctx;
+}
+
+/**
+ * dm_test_mem_type_changed_no_planes - Test an empty plane mask reports no change
+ * @test: The KUnit test context
+ */
+static void dm_test_mem_type_changed_no_planes(struct kunit *test)
+{
+	struct dm_test_mem_type_ctx *ctx = dm_test_mem_type_ctx_alloc(test);
+
+	ctx->crtc_state->plane_mask = 0;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_mem_type_changed(&ctx->adev->ddev,
+								ctx->state,
+								ctx->crtc_state));
+}
+
+/**
+ * dm_test_mem_type_changed_missing_state - Test a plane without both states is skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_mem_type_changed_missing_state(struct kunit *test)
+{
+	struct dm_test_mem_type_ctx *ctx = dm_test_mem_type_ctx_alloc(test);
+
+	ctx->state->planes[ctx->plane->index].new_state = ctx->new_plane_state;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_mem_type_changed(&ctx->adev->ddev,
+								ctx->state,
+								ctx->crtc_state));
+}
+
+/**
+ * dm_test_mem_type_changed_same_domain - Test identical memory domains report no change
+ * @test: The KUnit test context
+ */
+static void dm_test_mem_type_changed_same_domain(struct kunit *test)
+{
+	struct dm_test_mem_type_ctx *ctx = dm_test_mem_type_ctx_alloc(test);
+
+	ctx->old_plane_state->fb = dm_test_alloc_fb(test, TTM_PL_VRAM);
+	ctx->new_plane_state->fb = dm_test_alloc_fb(test, TTM_PL_VRAM);
+	ctx->state->planes[ctx->plane->index].old_state = ctx->old_plane_state;
+	ctx->state->planes[ctx->plane->index].new_state = ctx->new_plane_state;
+
+	KUNIT_EXPECT_FALSE(test, amdgpu_dm_crtc_mem_type_changed(&ctx->adev->ddev,
+								ctx->state,
+								ctx->crtc_state));
+}
+
+/**
+ * dm_test_mem_type_changed_different_domain - Test a domain migration is detected
+ * @test: The KUnit test context
+ */
+static void dm_test_mem_type_changed_different_domain(struct kunit *test)
+{
+	struct dm_test_mem_type_ctx *ctx = dm_test_mem_type_ctx_alloc(test);
+
+	ctx->old_plane_state->fb = dm_test_alloc_fb(test, TTM_PL_TT);
+	ctx->new_plane_state->fb = dm_test_alloc_fb(test, TTM_PL_VRAM);
+	ctx->state->planes[ctx->plane->index].old_state = ctx->old_plane_state;
+	ctx->state->planes[ctx->plane->index].new_state = ctx->new_plane_state;
+
+	KUNIT_EXPECT_TRUE(test, amdgpu_dm_crtc_mem_type_changed(&ctx->adev->ddev,
+							       ctx->state,
+							       ctx->crtc_state));
+}
+
+/* Tests for fill_dc_dirty_rects() */
+
+struct dm_test_dirty_ctx {
+	struct amdgpu_device *adev;
+	struct drm_plane *plane;
+	struct drm_plane_state *old_plane_state;
+	struct drm_plane_state *new_plane_state;
+	struct dm_crtc_state *crtc_state;
+	struct dc_flip_addrs *flip_addrs;
+	bool dirty_regions_changed;
+};
+
+static struct dm_test_dirty_ctx *dm_test_dirty_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->plane = drm_kunit_helper_create_primary_plane(test, &ctx->adev->ddev,
+							   NULL, NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx->plane);
+	drm_plane_enable_fb_damage_clips(ctx->plane);
+	ctx->old_plane_state = kunit_kzalloc(test, sizeof(*ctx->old_plane_state),
+					     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->old_plane_state);
+	ctx->new_plane_state = kunit_kzalloc(test, sizeof(*ctx->new_plane_state),
+					     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->new_plane_state);
+	ctx->crtc_state = kunit_kzalloc(test, sizeof(*ctx->crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc_state);
+	ctx->flip_addrs = kunit_kzalloc(test, sizeof(*ctx->flip_addrs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->flip_addrs);
+
+	ctx->plane->type = DRM_PLANE_TYPE_PRIMARY;
+	ctx->new_plane_state->plane = ctx->plane;
+	ctx->new_plane_state->rotation = DRM_MODE_ROTATE_0;
+	ctx->old_plane_state->plane = ctx->plane;
+	ctx->crtc_state->base.mode.crtc_hdisplay = 1920;
+	ctx->crtc_state->base.mode.crtc_vdisplay = 1080;
+
+	return ctx;
+}
+
+static void dm_test_fill_dirty_rects(struct dm_test_dirty_ctx *ctx, bool is_psr_su)
+{
+	fill_dc_dirty_rects(ctx->plane, ctx->old_plane_state,
+			    ctx->new_plane_state, &ctx->crtc_state->base,
+			    ctx->flip_addrs, is_psr_su,
+			    &ctx->dirty_regions_changed);
+}
+
+/*
+ * Attach @count damage clips to the new plane state. The blob is only ever
+ * read through drm_plane_get_damage_clips(), so a bare blob is enough.
+ */
+static struct drm_mode_rect *dm_test_add_damage_clips(struct kunit *test,
+						      struct drm_plane_state *state,
+						      unsigned int count)
+{
+	struct drm_property_blob *blob;
+	struct drm_mode_rect *clips;
+
+	blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, blob);
+	clips = kunit_kcalloc(test, count, sizeof(*clips), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, clips);
+
+	blob->length = count * sizeof(*clips);
+	blob->data = clips;
+	state->fb_damage_clips = blob;
+
+	return clips;
+}
+
+/**
+ * dm_test_dirty_rects_cursor_plane - Test cursor planes are left to their own path
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_cursor_plane(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+
+	ctx->plane->type = DRM_PLANE_TYPE_CURSOR;
+
+	dm_test_fill_dirty_rects(ctx, true);
+
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 0U);
+}
+
+/**
+ * dm_test_dirty_rects_rotation_ffu - Test a rotated plane falls back to full frame update
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_rotation_ffu(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+
+	ctx->new_plane_state->rotation = DRM_MODE_ROTATE_90;
+
+	dm_test_fill_dirty_rects(ctx, true);
+
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 1U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].x, 0);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].y, 0);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 1920);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].height, 1080);
+}
+
+/**
+ * dm_test_dirty_rects_no_clips_ffu - Test a damage-unaware client gets a full frame update
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_no_clips_ffu(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+
+	dm_test_fill_dirty_rects(ctx, true);
+
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 1U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 1920);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].height, 1080);
+}
+
+/**
+ * dm_test_dirty_rects_ignored_damage_clips - Test ignored damage clips force a full update
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_ignored_damage_clips(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+
+	dm_test_add_damage_clips(test, ctx->new_plane_state, 1);
+	ctx->new_plane_state->ignore_damage_clips = true;
+
+	dm_test_fill_dirty_rects(ctx, true);
+
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 1U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 1920);
+}
+
+/**
+ * dm_test_dirty_rects_too_many_clips_ffu - Test exceeding DC_MAX_DIRTY_RECTS falls back
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_too_many_clips_ffu(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+
+	dm_test_add_damage_clips(test, ctx->new_plane_state,
+				 DC_MAX_DIRTY_RECTS + 1);
+
+	dm_test_fill_dirty_rects(ctx, false);
+
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 1U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 1920);
+}
+
+/**
+ * dm_test_dirty_rects_damage_clips - Test damage clips are copied verbatim
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_damage_clips(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+	struct drm_mode_rect *clips;
+
+	clips = dm_test_add_damage_clips(test, ctx->new_plane_state, 2);
+	clips[0].x1 = 10;
+	clips[0].y1 = 20;
+	clips[0].x2 = 40;
+	clips[0].y2 = 60;
+	clips[1].x1 = 100;
+	clips[1].y1 = 200;
+	clips[1].x2 = 150;
+	clips[1].y2 = 260;
+
+	dm_test_fill_dirty_rects(ctx, false);
+
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 2U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].x, 10);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].y, 20);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 30);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].height, 40);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[1].x, 100);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[1].width, 50);
+	KUNIT_EXPECT_FALSE(test, ctx->dirty_regions_changed);
+}
+
+/**
+ * dm_test_dirty_rects_mpo_bb_changed - Test MPO adds both plane bounding boxes
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_mpo_bb_changed(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+	struct drm_framebuffer *fb;
+
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	ctx->crtc_state->mpo_requested = true;
+	ctx->old_plane_state->fb = fb;
+	ctx->new_plane_state->fb = fb;
+	ctx->old_plane_state->crtc_x = 0;
+	ctx->old_plane_state->crtc_y = 0;
+	ctx->old_plane_state->crtc_w = 640;
+	ctx->old_plane_state->crtc_h = 480;
+	ctx->new_plane_state->crtc_x = 100;
+	ctx->new_plane_state->crtc_y = 50;
+	ctx->new_plane_state->crtc_w = 800;
+	ctx->new_plane_state->crtc_h = 600;
+
+	dm_test_fill_dirty_rects(ctx, true);
+
+	KUNIT_EXPECT_TRUE(test, ctx->dirty_regions_changed);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 2U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].x, 100);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 800);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[1].x, 0);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[1].width, 640);
+}
+
+/**
+ * dm_test_dirty_rects_mpo_fb_changed - Test MPO flips add the new plane bounding box
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_mpo_fb_changed(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+	struct drm_framebuffer *old_fb;
+	struct drm_framebuffer *new_fb;
+
+	old_fb = kunit_kzalloc(test, sizeof(*old_fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_fb);
+	new_fb = kunit_kzalloc(test, sizeof(*new_fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_fb);
+
+	old_fb->base.id = 1;
+	new_fb->base.id = 2;
+	ctx->crtc_state->mpo_requested = true;
+	ctx->old_plane_state->fb = old_fb;
+	ctx->new_plane_state->fb = new_fb;
+	ctx->new_plane_state->crtc_w = 800;
+	ctx->new_plane_state->crtc_h = 600;
+	ctx->old_plane_state->crtc_w = 800;
+	ctx->old_plane_state->crtc_h = 600;
+
+	dm_test_fill_dirty_rects(ctx, true);
+
+	KUNIT_EXPECT_FALSE(test, ctx->dirty_regions_changed);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 1U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 800);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].height, 600);
+}
+
+/**
+ * dm_test_dirty_rects_psr_su_ffu - Test PSR SU ignores clips in auto damage mode
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_psr_su_ffu(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+
+	dm_test_add_damage_clips(test, ctx->new_plane_state, 1);
+
+	dm_test_fill_dirty_rects(ctx, true);
+
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 1U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 1920);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].height, 1080);
+}
+
+/**
+ * dm_test_dirty_rects_mpo_clips - Test MPO copies damage clips when the box is stable
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_mpo_clips(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+	struct drm_mode_rect *clips;
+	struct drm_framebuffer *fb;
+
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	clips = dm_test_add_damage_clips(test, ctx->new_plane_state, 2);
+	clips[0].x1 = 5;
+	clips[0].y1 = 6;
+	clips[0].x2 = 25;
+	clips[0].y2 = 36;
+	clips[1].x1 = 50;
+	clips[1].y1 = 60;
+	clips[1].x2 = 90;
+	clips[1].y2 = 110;
+
+	ctx->crtc_state->mpo_requested = true;
+	ctx->old_plane_state->fb = fb;
+	ctx->new_plane_state->fb = fb;
+
+	dm_test_fill_dirty_rects(ctx, false);
+
+	KUNIT_EXPECT_FALSE(test, ctx->dirty_regions_changed);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 2U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].x, 5);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 20);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[1].x, 50);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[1].height, 50);
+}
+
+/**
+ * dm_test_dirty_rects_mpo_overflow_ffu - Test MPO falls back when clips plus boxes overflow
+ * @test: The KUnit test context
+ */
+static void dm_test_dirty_rects_mpo_overflow_ffu(struct kunit *test)
+{
+	struct dm_test_dirty_ctx *ctx = dm_test_dirty_ctx_alloc(test);
+	struct drm_framebuffer *fb;
+
+	fb = kunit_kzalloc(test, sizeof(*fb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	dm_test_add_damage_clips(test, ctx->new_plane_state, 2);
+
+	ctx->crtc_state->mpo_requested = true;
+	ctx->old_plane_state->fb = fb;
+	ctx->new_plane_state->fb = fb;
+	ctx->new_plane_state->crtc_x = 100;
+
+	dm_test_fill_dirty_rects(ctx, false);
+
+	KUNIT_EXPECT_TRUE(test, ctx->dirty_regions_changed);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rect_count, 1U);
+	KUNIT_EXPECT_EQ(test, ctx->flip_addrs->dirty_rects[0].width, 1920);
+}
+
+/* Tests for should_reset_plane() */
+
+struct dm_test_reset_plane_ctx {
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+	struct drm_plane *plane;
+	struct drm_plane_state *old_plane_state;
+	struct drm_plane_state *new_plane_state;
+	struct dm_crtc_state *old_crtc_state;
+	struct dm_crtc_state *new_crtc_state;
+	struct drm_plane *other;
+	struct dm_plane_state *other_old;
+	struct dm_plane_state *other_new;
+};
+
+/*
+ * Build a fast-update baseline: a DCN 3.2 device with one plane bound to an
+ * unchanged CRTC, so should_reset_plane() runs to the end and returns false.
+ */
+static struct dm_test_reset_plane_ctx *
+dm_test_reset_plane_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 2, 0);
+	ctx->adev->reset_domain = kunit_kzalloc(test,
+						sizeof(*ctx->adev->reset_domain),
+						GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->adev->reset_domain);
+
+	ctx->state = dm_test_alloc_commit(test, ctx->adev);
+	ctx->crtc = kunit_kzalloc(test, sizeof(*ctx->crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc);
+	ctx->plane = kunit_kzalloc(test, sizeof(*ctx->plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->plane);
+	ctx->old_plane_state = kunit_kzalloc(test, sizeof(*ctx->old_plane_state),
+					     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->old_plane_state);
+	ctx->new_plane_state = kunit_kzalloc(test, sizeof(*ctx->new_plane_state),
+					     GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->new_plane_state);
+	ctx->old_crtc_state = kunit_kzalloc(test, sizeof(*ctx->old_crtc_state),
+					    GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->old_crtc_state);
+	ctx->new_crtc_state = kunit_kzalloc(test, sizeof(*ctx->new_crtc_state),
+					    GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->new_crtc_state);
+	ctx->state->crtcs = kunit_kzalloc(test, sizeof(*ctx->state->crtcs),
+					  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->crtcs);
+
+	ctx->plane->dev = &ctx->adev->ddev;
+	ctx->plane->type = DRM_PLANE_TYPE_PRIMARY;
+	ctx->old_plane_state->crtc = ctx->crtc;
+	ctx->new_plane_state->crtc = ctx->crtc;
+	ctx->state->crtcs[0].ptr = ctx->crtc;
+	ctx->state->crtcs[0].old_state = &ctx->old_crtc_state->base;
+	ctx->state->crtcs[0].new_state = &ctx->new_crtc_state->base;
+
+	return ctx;
+}
+
+static bool dm_test_should_reset_plane(struct dm_test_reset_plane_ctx *ctx)
+{
+	return should_reset_plane(ctx->state, ctx->plane, ctx->old_plane_state,
+				  ctx->new_plane_state);
+}
+
+/**
+ * dm_test_reset_plane_pre_dcn32_modeset - Test pre-DCN3.2 modesets always reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_pre_dcn32_modeset(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 2);
+	ctx->state->allow_modeset = true;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/*
+ * Add a connector of @type to the commit so should_reset_plane() walks its
+ * writeback check.
+ */
+static struct drm_connector_state *
+dm_test_reset_plane_add_connector(struct kunit *test,
+				  struct dm_test_reset_plane_ctx *ctx, int type)
+{
+	struct drm_connector_state *conn_state;
+	struct drm_connector *connector;
+
+	connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, connector);
+	conn_state = kunit_kzalloc(test, sizeof(*conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+	ctx->state->connectors = kunit_kzalloc(test,
+					       sizeof(*ctx->state->connectors),
+					       GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->connectors);
+
+	connector->connector_type = type;
+	ctx->state->num_connector = 1;
+	ctx->state->connectors[0].ptr = connector;
+	ctx->state->connectors[0].new_state = conn_state;
+
+	return conn_state;
+}
+
+/**
+ * dm_test_reset_plane_writeback_job - Test a writeback commit resets planes
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_writeback_job(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+	struct drm_connector_state *conn_state;
+
+	conn_state = dm_test_reset_plane_add_connector(test, ctx,
+						       DRM_MODE_CONNECTOR_WRITEBACK);
+	conn_state->writeback_job = kunit_kzalloc(test,
+						  sizeof(*conn_state->writeback_job),
+						  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state->writeback_job);
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_crtc_changed - Test moving a plane between CRTCs resets it
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_crtc_changed(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->old_plane_state->crtc = NULL;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_not_in_context - Test a plane outside the context is not reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_not_in_context(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->old_plane_state->crtc = NULL;
+	ctx->new_plane_state->crtc = NULL;
+
+	KUNIT_EXPECT_FALSE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_no_new_crtc_state - Test a missing new CRTC state resets the plane
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_no_new_crtc_state(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->state->crtcs[0].new_state = NULL;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_cursor_mode_changed - Test a cursor mode switch resets the plane
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_cursor_mode_changed(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->plane->type = DRM_PLANE_TYPE_CURSOR;
+	ctx->old_crtc_state->cursor_mode = DM_CURSOR_NATIVE_MODE;
+	ctx->new_crtc_state->cursor_mode = DM_CURSOR_OVERLAY_MODE;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_color_mgmt_changed - Test a degamma change resets the plane
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_color_mgmt_changed(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.color_mgmt_changed = true;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_zpos_changed - Test a z-order change resets the plane
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_zpos_changed(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->old_plane_state->normalized_zpos = 0;
+	ctx->new_plane_state->normalized_zpos = 1;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_modeset - Test a CRTC modeset resets the plane
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_modeset(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.mode_changed = true;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_fast_update - Test an unchanged plane stays on the fast path
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_fast_update(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	KUNIT_EXPECT_FALSE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_skips_non_writeback - Test non-writeback connectors are skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_skips_non_writeback(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_connector(test, ctx,
+					  DRM_MODE_CONNECTOR_DisplayPort);
+
+	KUNIT_EXPECT_FALSE(test, dm_test_should_reset_plane(ctx));
+}
+
+/*
+ * Add a second plane to the commit so should_reset_plane() walks its
+ * cross-plane loop. Both states start identical and bound to the same CRTC, so
+ * each test only has to perturb the one field it cares about.
+ */
+static void dm_test_reset_plane_add_other(struct kunit *test,
+					  struct dm_test_reset_plane_ctx *ctx)
+{
+	ctx->other = kunit_kzalloc(test, sizeof(*ctx->other), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->other);
+	ctx->other_old = kunit_kzalloc(test, sizeof(*ctx->other_old), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->other_old);
+	ctx->other_new = kunit_kzalloc(test, sizeof(*ctx->other_new), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->other_new);
+	ctx->state->planes = kunit_kzalloc(test, sizeof(*ctx->state->planes),
+					   GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->planes);
+
+	ctx->other->type = DRM_PLANE_TYPE_PRIMARY;
+	ctx->other_old->base.crtc = ctx->crtc;
+	ctx->other_new->base.crtc = ctx->crtc;
+
+	ctx->adev->ddev.mode_config.num_total_plane = 1;
+	ctx->state->planes[0].ptr = ctx->other;
+	ctx->state->planes[0].old_state = &ctx->other_old->base;
+	ctx->state->planes[0].new_state = &ctx->other_new->base;
+}
+
+/**
+ * dm_test_reset_plane_other_cursor_skipped - Test other cursor planes are ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_cursor_skipped(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other->type = DRM_PLANE_TYPE_CURSOR;
+	ctx->other_new->base.rotation = DRM_MODE_ROTATE_90;
+
+	KUNIT_EXPECT_FALSE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_crtc_skipped - Test planes on other CRTCs are ignored
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_crtc_skipped(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_old->base.crtc = NULL;
+	ctx->other_new->base.crtc = NULL;
+	ctx->other_new->base.rotation = DRM_MODE_ROTATE_90;
+
+	KUNIT_EXPECT_FALSE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_crtc_moved - Test moving another plane forces a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_crtc_moved(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_old->base.crtc = NULL;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_scaling - Test src/dst size changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_scaling(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_new->base.src_w = 1 << 16;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_rotation - Test rotation changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_rotation(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_new->base.rotation = DRM_MODE_ROTATE_180;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_blending - Test blend mode changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_blending(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_new->base.pixel_blend_mode = DRM_MODE_BLEND_COVERAGE;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_alpha - Test alpha changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_alpha(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_new->base.alpha = DRM_BLEND_ALPHA_OPAQUE;
+	ctx->other_old->base.alpha = DRM_BLEND_ALPHA_OPAQUE / 2;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_colorspace - Test colorspace changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_colorspace(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_new->base.color_encoding = DRM_COLOR_YCBCR_BT2020;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_hdr_mult - Test transfer function changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_hdr_mult(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_new->hdr_mult = 1;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_no_fb - Test planes without framebuffers are skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_no_fb(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	ctx->other_new->base.fb = kunit_kzalloc(test,
+						sizeof(*ctx->other_new->base.fb),
+						GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->other_new->base.fb);
+
+	KUNIT_EXPECT_FALSE(test, dm_test_should_reset_plane(ctx));
+}
+
+/*
+ * Give the second plane a matching pair of framebuffers. should_reset_plane()
+ * casts them to amdgpu_framebuffer, so both are allocated as such.
+ */
+static void dm_test_reset_plane_add_other_fbs(struct kunit *test,
+					      struct dm_test_reset_plane_ctx *ctx)
+{
+	struct amdgpu_framebuffer *old_afb, *new_afb;
+	struct drm_format_info *format;
+
+	format = kunit_kzalloc(test, sizeof(*format), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, format);
+	old_afb = kunit_kzalloc(test, sizeof(*old_afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_afb);
+	new_afb = kunit_kzalloc(test, sizeof(*new_afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_afb);
+
+	old_afb->base.format = format;
+	new_afb->base.format = format;
+	ctx->other_old->base.fb = &old_afb->base;
+	ctx->other_new->base.fb = &new_afb->base;
+}
+
+/**
+ * dm_test_reset_plane_other_format - Test pixel format changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_format(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+	struct drm_format_info *new_format;
+
+	dm_test_reset_plane_add_other(test, ctx);
+	dm_test_reset_plane_add_other_fbs(test, ctx);
+	new_format = kunit_kzalloc(test, sizeof(*new_format), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_format);
+	ctx->other_new->base.fb->format = new_format;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/**
+ * dm_test_reset_plane_other_modifier - Test tiling/DCC changes force a reset
+ * @test: The KUnit test context
+ */
+static void dm_test_reset_plane_other_modifier(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+
+	dm_test_reset_plane_add_other(test, ctx);
+	dm_test_reset_plane_add_other_fbs(test, ctx);
+	ctx->other_new->base.fb->modifier = 1;
+
+	KUNIT_EXPECT_TRUE(test, dm_test_should_reset_plane(ctx));
+}
+
+/* Tests for amdgpu_dm_dump_links_and_sinks() */
+
+/**
+ * dm_test_dump_links_no_dc - Test a device without DC dumps nothing
+ * @test: The KUnit test context
+ */
+static void dm_test_dump_links_no_dc(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_dump_links_and_sinks(adev);
+}
+
+/**
+ * dm_test_dump_links_no_links - Test a DC without links dumps nothing
+ * @test: The KUnit test context
+ */
+static void dm_test_dump_links_no_links(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	amdgpu_dm_dump_links_and_sinks(adev);
+}
+
+/*
+ * Attach a single DC link to @adev so amdgpu_dm_dump_links_and_sinks() walks it.
+ */
+static struct dc_link *dm_test_dc_with_link(struct kunit *test,
+					   struct amdgpu_device *adev)
+{
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+	struct dc_link *link = dm_kunit_alloc_link(test);
+
+	dc->links[0] = link;
+	dc->link_count = 1;
+	adev->dm.dc = dc;
+
+	return link;
+}
+
+/**
+ * dm_test_dump_links_with_sinks - Test local and remote sinks are walked
+ * @test: The KUnit test context
+ */
+static void dm_test_dump_links_with_sinks(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_link *link = dm_test_dc_with_link(test, adev);
+	struct dc_sink *local_sink, *remote_sink;
+
+	local_sink = kunit_kzalloc(test, sizeof(*local_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, local_sink);
+	remote_sink = kunit_kzalloc(test, sizeof(*remote_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, remote_sink);
+
+	strscpy(local_sink->edid_caps.display_name, "local");
+	strscpy(remote_sink->edid_caps.display_name, "remote");
+	link->local_sink = local_sink;
+	link->sink_count = 1;
+	link->remote_sinks[0] = remote_sink;
+	adev->dm.dc->link_count = 2;
+
+	amdgpu_dm_dump_links_and_sinks(adev);
+}
+
+/**
+ * dm_test_dump_links_unnamed_sinks - Test links without a sink or EDID name
+ * @test: The KUnit test context
+ */
+static void dm_test_dump_links_unnamed_sinks(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_link *link = dm_test_dc_with_link(test, adev);
+	struct dc_sink *remote_sink;
+
+	remote_sink = kunit_kzalloc(test, sizeof(*remote_sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, remote_sink);
+
+	link->sink_count = 2;
+	link->remote_sinks[0] = NULL;
+	link->remote_sinks[1] = remote_sink;
+
+	amdgpu_dm_dump_links_and_sinks(adev);
+}
+
+/**
+ * dm_test_update_hdcp_no_workqueue - Test HDCP update is skipped without a workqueue
+ * @test: The KUnit test context
+ */
+static void dm_test_update_hdcp_no_workqueue(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_update_hdcp(dm_test_alloc_commit(test, adev));
+}
+
+/**
+ * dm_test_atomic_setup_commit_empty - Test an empty commit needs no color setup
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_setup_commit_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	KUNIT_EXPECT_EQ(test,
+			amdgpu_dm_atomic_setup_commit(dm_test_alloc_commit(test, adev)),
+			0);
+}
+
+/**
+ * dm_test_atomic_check_empty - Test an empty atomic commit needs no DC validation
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_check_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_atomic_commit *state = dm_test_alloc_commit(test, adev);
+
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_atomic_check(&adev->ddev, state), 0);
+	KUNIT_EXPECT_EQ(test, state->num_private_objs, 0U);
+}
+
+/*
+ * A commit with one connector of @type bound to a CRTC that keeps its stream.
+ * The content protection state is unchanged, so amdgpu_dm_update_hdcp() walks
+ * the connector without reaching hdcp_update_display() or hdcp_reset_display(),
+ * which are the only users of the HDCP workqueue contents.
+ */
+static struct drm_atomic_commit *dm_test_hdcp_commit(struct kunit *test,
+						     struct amdgpu_device *adev,
+						     int type)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_commit(test, adev);
+	struct dm_crtc_state *old_crtc_state, *new_crtc_state;
+	struct dm_connector_state *old_dm, *new_dm;
+	struct amdgpu_dm_connector *aconnector;
+	struct amdgpu_crtc *acrtc;
+	struct dc_sink *sink;
+
+	adev->dm.hdcp_workqueue = kunit_kzalloc(test,
+						sizeof(*adev->dm.hdcp_workqueue),
+						GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev->dm.hdcp_workqueue);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	old_dm = kunit_kzalloc(test, sizeof(*old_dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_dm);
+	new_dm = kunit_kzalloc(test, sizeof(*new_dm), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_dm);
+	old_crtc_state = kunit_kzalloc(test, sizeof(*old_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_crtc_state);
+	new_crtc_state = kunit_kzalloc(test, sizeof(*new_crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_crtc_state);
+	sink = kunit_kzalloc(test, sizeof(*sink), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, sink);
+	state->connectors = kunit_kzalloc(test, sizeof(*state->connectors), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state->connectors);
+	state->crtcs = kunit_kzalloc(test, sizeof(*state->crtcs), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, state->crtcs);
+
+	strscpy(sink->edid_caps.display_name, "panel");
+	sink->sink_signal = SIGNAL_TYPE_DISPLAY_PORT;
+	aconnector->dc_sink = sink;
+	aconnector->base.connector_type = type;
+	aconnector->base.dpms = DRM_MODE_DPMS_ON;
+	aconnector->base.state = &new_dm->base;
+	new_dm->base.crtc = &acrtc->base;
+	new_crtc_state->stream = dm_kunit_alloc_stream(test, NULL);
+
+	state->num_connector = 1;
+	state->connectors[0].ptr = &aconnector->base;
+	state->connectors[0].old_state = &old_dm->base;
+	state->connectors[0].new_state = &new_dm->base;
+	state->crtcs[0].ptr = &acrtc->base;
+	state->crtcs[0].old_state = &old_crtc_state->base;
+	state->crtcs[0].new_state = &new_crtc_state->base;
+
+	return state;
+}
+
+/**
+ * dm_test_update_hdcp_writeback_skipped - Test writeback connectors are skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_update_hdcp_writeback_skipped(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_update_hdcp(dm_test_hdcp_commit(test, adev,
+						  DRM_MODE_CONNECTOR_WRITEBACK));
+}
+
+/**
+ * dm_test_update_hdcp_unchanged - Test an unchanged connector needs no HDCP update
+ * @test: The KUnit test context
+ */
+static void dm_test_update_hdcp_unchanged(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_update_hdcp(dm_test_hdcp_commit(test, adev,
+						  DRM_MODE_CONNECTOR_DisplayPort));
+}
+
+/*
+ * Run do_aquire_global_lock() with a fresh acquire context, releasing the locks
+ * it leaves held on @state->acquire_ctx.
+ */
+static int dm_test_run_global_lock(struct kunit *test,
+				   struct amdgpu_device *adev)
+{
+	struct drm_atomic_commit *state = dm_test_alloc_commit(test, adev);
+	struct drm_modeset_acquire_ctx acquire_ctx;
+	int ret;
+
+	drm_modeset_acquire_init(&acquire_ctx, 0);
+	state->acquire_ctx = &acquire_ctx;
+
+	ret = do_aquire_global_lock(&adev->ddev, state);
+
+	drm_modeset_drop_locks(&acquire_ctx);
+	drm_modeset_acquire_fini(&acquire_ctx);
+
+	return ret;
+}
+
+/**
+ * dm_test_aquire_global_lock_no_crtc - Test the global lock is taken without CRTCs
+ * @test: The KUnit test context
+ */
+static void dm_test_aquire_global_lock_no_crtc(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	KUNIT_EXPECT_EQ(test, dm_test_run_global_lock(test, adev), 0);
+}
+
+/*
+ * A CRTC registered on @adev, needed to walk the per-CRTC commit loop.
+ */
+static struct drm_crtc *dm_test_alloc_crtc(struct kunit *test,
+					   struct amdgpu_device *adev)
+{
+	struct drm_plane *primary;
+	struct drm_crtc *crtc;
+
+	primary = drm_kunit_helper_create_primary_plane(test, &adev->ddev, NULL,
+							NULL, NULL, 0, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, primary);
+	crtc = drm_kunit_helper_create_crtc(test, &adev->ddev, primary, NULL,
+					    NULL, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc);
+
+	return crtc;
+}
+
+/**
+ * dm_test_aquire_global_lock_no_commit - Test a CRTC without a pending commit
+ * @test: The KUnit test context
+ */
+static void dm_test_aquire_global_lock_no_commit(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	dm_test_alloc_crtc(test, adev);
+
+	KUNIT_EXPECT_EQ(test, dm_test_run_global_lock(test, adev), 0);
+}
+
+/**
+ * dm_test_aquire_global_lock_waits_commit - Test a completed commit is waited on
+ * @test: The KUnit test context
+ */
+static void dm_test_aquire_global_lock_waits_commit(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_crtc_commit *commit;
+	struct drm_crtc *crtc;
+	int ret;
+
+	crtc = dm_test_alloc_crtc(test, adev);
+	commit = kunit_kzalloc(test, sizeof(*commit), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, commit);
+
+	/*
+	 * The extra reference taken by the loop is dropped again, so the commit
+	 * never reaches zero and stays KUnit-managed.
+	 */
+	kref_init(&commit->ref);
+	init_completion(&commit->hw_done);
+	init_completion(&commit->flip_done);
+	complete_all(&commit->hw_done);
+	complete_all(&commit->flip_done);
+	list_add_tail(&commit->commit_entry, &crtc->commit_list);
+
+	ret = dm_test_run_global_lock(test, adev);
+
+	list_del(&commit->commit_entry);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+}
+
+/**
+ * dm_test_update_crtc_state_unchanged - Test unchanged state needs no validation
+ * @test: The KUnit test context
+ */
+static void dm_test_update_crtc_state_unchanged(struct kunit *test)
+{
+	struct dm_test_reset_plane_ctx *ctx = dm_test_reset_plane_ctx_alloc(test);
+	bool lock_and_validation_needed = false;
+
+	ctx->adev->dm.adev = ctx->adev;
+	KUNIT_EXPECT_EQ(test,
+			dm_update_crtc_state(&ctx->adev->dm, ctx->state, ctx->crtc,
+					     &ctx->old_crtc_state->base,
+					     &ctx->new_crtc_state->base, true,
+					     &lock_and_validation_needed),
+			0);
+	KUNIT_EXPECT_FALSE(test, lock_and_validation_needed);
+	KUNIT_EXPECT_NULL(test, ctx->new_crtc_state->stream);
+}
+
+/**
+ * dm_test_update_plane_state_detached - Test a detached plane needs no DC update
+ * @test: The KUnit test context
+ */
+static void dm_test_update_plane_state_detached(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dm_plane_state *old_plane_state;
+	struct dm_plane_state *new_plane_state;
+	struct drm_plane *plane;
+	bool lock_and_validation_needed = false;
+	bool is_top_most_overlay = true;
+
+	old_plane_state = kunit_kzalloc(test, sizeof(*old_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, old_plane_state);
+	new_plane_state = kunit_kzalloc(test, sizeof(*new_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, new_plane_state);
+	plane = kunit_kzalloc(test, sizeof(*plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane);
+	adev->reset_domain = kunit_kzalloc(test, sizeof(*adev->reset_domain), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, adev->reset_domain);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 2, 0);
+	plane->dev = &adev->ddev;
+	plane->type = DRM_PLANE_TYPE_PRIMARY;
+
+	KUNIT_EXPECT_EQ(test,
+			dm_update_plane_state(NULL, dm_test_alloc_commit(test, adev), plane,
+					      &old_plane_state->base,
+					      &new_plane_state->base, false,
+					      &lock_and_validation_needed,
+					      &is_top_most_overlay),
+			0);
+	KUNIT_EXPECT_FALSE(test, lock_and_validation_needed);
+	KUNIT_EXPECT_TRUE(test, is_top_most_overlay);
+	KUNIT_EXPECT_NULL(test, new_plane_state->dc_state);
+}
+
+/**
+ * dm_test_mod_power_update_streams_empty - Test an empty commit updates no streams
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_update_streams_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_mod_power_update_streams(dm_test_alloc_commit(test, adev),
+					   &adev->dm);
+}
+
+/**
+ * dm_test_mod_power_setup_streams_empty - Test an empty commit sets up no streams
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_setup_streams_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	amdgpu_dm_mod_power_setup_streams(dm_test_alloc_commit(test, adev),
+					  &adev->dm);
+}
+
+/* Tests for amdgpu_dm_trigger_timing_sync() */
+
+/**
+ * dm_test_trigger_timing_sync_no_state - Test no DC state leaves the sync untouched
+ * @test: The KUnit test context
+ */
+static void dm_test_trigger_timing_sync_no_state(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+	mutex_init(&adev->dm.dc_lock);
+
+	amdgpu_dm_trigger_timing_sync(&adev->ddev);
+}
+
+/**
+ * dm_test_trigger_timing_sync_streams - Test the force flag reaches every stream
+ * @test: The KUnit test context
+ */
+static void dm_test_trigger_timing_sync_streams(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+	struct dc_stream_state *stream;
+	struct dc_state *context;
+
+	context = dm_kunit_alloc_dc_state(test);
+	KUNIT_ASSERT_NOT_NULL(test, context);
+	stream = dm_kunit_alloc_stream(test, NULL);
+
+	context->streams[0] = stream;
+	context->stream_count = 1;
+	dc->current_state = context;
+	adev->dm.dc = dc;
+	adev->dm.force_timing_sync = true;
+	mutex_init(&adev->dm.dc_lock);
+
+	amdgpu_dm_trigger_timing_sync(&adev->ddev);
+
+	KUNIT_EXPECT_TRUE(test, stream->triggered_crtc_reset.enabled);
+}
+
+/**
+ * dm_test_acpi_phy_transition_interlock - Test the PHY transition interlock stub
+ * @test: The KUnit test context
+ */
+static void dm_test_acpi_phy_transition_interlock(struct kunit *test)
+{
+	struct dm_process_phy_transition_init_params params = { 0 };
+
+	dm_acpi_process_phy_transition_interlock(NULL, params);
+}
+
+/**
+ * dm_test_early_fini_audio_disabled - Test early fini with audio never enabled
+ * @test: The KUnit test context
+ */
+static void dm_test_early_fini_audio_disabled(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_ip_block ip_block = { .adev = adev };
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_early_fini(&ip_block), 0);
+}
+
+/**
+ * dm_test_sw_fini_releases_state - Test sw fini drops the DMUB software state
+ * @test: The KUnit test context
+ */
+static void dm_test_sw_fini_releases_state(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_ip_block ip_block = { .adev = adev };
+
+	INIT_LIST_HEAD(&adev->dm.da_list);
+	adev->dm.dmub_fb_info = kzalloc_obj(*adev->dm.dmub_fb_info);
+	KUNIT_ASSERT_NOT_NULL(test, adev->dm.dmub_fb_info);
+
+	KUNIT_EXPECT_EQ(test, dm_sw_fini(&ip_block), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.dmub_fb_info);
+}
+
+/**
+ * dm_test_oem_i2c_hw_init_no_device - Test no OEM I2C device leaves the bus unset
+ * @test: The KUnit test context
+ */
+static void dm_test_oem_i2c_hw_init_no_device(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	dc->res_pool = kunit_kzalloc(test, sizeof(*dc->res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc->res_pool);
+	adev->dm.dc = dc;
+
+	KUNIT_EXPECT_EQ(test, dm_oem_i2c_hw_init(adev), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.oem_i2c);
+}
+
+/**
+ * dm_test_resume_mst_no_primary - Test a missing primary branch releases the topology lock
+ * @test: The KUnit test context
+ */
+static void dm_test_resume_mst_no_primary(struct kunit *test)
+{
+	struct drm_dp_mst_topology_mgr *mgr;
+
+	mgr = kunit_kzalloc(test, sizeof(*mgr), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, mgr);
+	mutex_init(&mgr->lock);
+
+	resume_mst_branch_status(mgr);
+
+	KUNIT_ASSERT_TRUE(test, mutex_trylock(&mgr->lock));
+	mutex_unlock(&mgr->lock);
+}
+
+/**
+ * dm_test_s3_handle_mst_empty - Test empty connector lists need no MST action
+ * @test: The KUnit test context
+ */
+static void dm_test_s3_handle_mst_empty(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	s3_handle_mst(&adev->ddev, true);
+
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->ddev.mode_config.connector_list));
+}
+
+/**
+ * dm_test_gpureset_commit_state_no_streams - Test an empty DC state programs nothing
+ * @test: The KUnit test context
+ */
+static void dm_test_gpureset_commit_state_no_streams(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_state *context = dm_kunit_alloc_dc_state(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, context);
+	adev->dm.ddev = &adev->ddev;
+
+	dm_gpureset_commit_state(context, &adev->dm);
+}
+
+/**
+ * dm_test_emulated_link_detect_bad_signal - Test an unknown connector signal is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_emulated_link_detect_bad_signal(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_link *link = dm_kunit_alloc_link_with_ctx(test);
+
+	link->ctx->driver_context = adev;
+	link->type = dc_connection_single;
+	link->connector_signal = SIGNAL_TYPE_VIRTUAL;
+
+	amdgpu_dm_emulated_link_detect(link);
+
+	KUNIT_EXPECT_EQ(test, (int)link->type, (int)dc_connection_none);
+	KUNIT_EXPECT_NULL(test, link->local_sink);
+}
+
+/* Tests for the mod_power modeset helpers */
+
+struct dm_test_modeset_ctx {
+	struct amdgpu_device *adev;
+	struct drm_atomic_commit *state;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state *old_crtc_state;
+	struct dm_crtc_state *new_crtc_state;
+};
+
+/*
+ * A modeset commit on one CRTC with an old and a new stream. dm->power_module
+ * stays NULL, which every mod_power entry point treats as a no-op, so the DM
+ * side of the modeset can be walked without a live power module.
+ */
+static struct dm_test_modeset_ctx *dm_test_modeset_ctx_alloc(struct kunit *test)
+{
+	struct amdgpu_dm_connector *aconnector;
+	struct dm_test_modeset_ctx *ctx;
+	struct dc_link *link;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->state = dm_test_alloc_commit(test, ctx->adev);
+	ctx->acrtc = kunit_kzalloc(test, sizeof(*ctx->acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc);
+	ctx->old_crtc_state = kunit_kzalloc(test, sizeof(*ctx->old_crtc_state),
+					    GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->old_crtc_state);
+	ctx->new_crtc_state = kunit_kzalloc(test, sizeof(*ctx->new_crtc_state),
+					    GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->new_crtc_state);
+	ctx->state->crtcs = kunit_kzalloc(test, sizeof(*ctx->state->crtcs),
+					  GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->state->crtcs);
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+
+	link = dm_kunit_alloc_link(test);
+	ctx->old_crtc_state->stream = dm_kunit_alloc_stream(test, link);
+	ctx->new_crtc_state->stream = dm_kunit_alloc_stream(test, link);
+	ctx->new_crtc_state->stream->dm_stream_context = aconnector;
+
+	ctx->adev->ddev.mode_config.num_crtc = 1;
+	ctx->adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+	mutex_init(&ctx->adev->dm.dc_lock);
+	ctx->new_crtc_state->base.mode_changed = true;
+	ctx->new_crtc_state->base.state = ctx->state;
+	ctx->state->crtcs[0].ptr = &ctx->acrtc->base;
+	ctx->state->crtcs[0].old_state = &ctx->old_crtc_state->base;
+	ctx->state->crtcs[0].new_state = &ctx->new_crtc_state->base;
+
+	return ctx;
+}
+
+/**
+ * dm_test_mod_power_update_streams_no_modeset - Test fast updates are skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_update_streams_no_modeset(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.mode_changed = false;
+	ctx->new_crtc_state->base.active = true;
+
+	amdgpu_dm_mod_power_update_streams(ctx->state, &ctx->adev->dm);
+}
+
+/**
+ * dm_test_mod_power_update_streams_enable - Test enabling a CRTC adds the stream
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_update_streams_enable(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.active = true;
+
+	amdgpu_dm_mod_power_update_streams(ctx->state, &ctx->adev->dm);
+}
+
+/**
+ * dm_test_mod_power_update_streams_replace - Test a re-modeset replaces the stream
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_update_streams_replace(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->old_crtc_state->base.active = true;
+	ctx->new_crtc_state->base.active = true;
+
+	amdgpu_dm_mod_power_update_streams(ctx->state, &ctx->adev->dm);
+}
+
+/**
+ * dm_test_mod_power_update_streams_disable - Test disabling a CRTC removes the stream
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_update_streams_disable(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->old_crtc_state->base.active = true;
+
+	amdgpu_dm_mod_power_update_streams(ctx->state, &ctx->adev->dm);
+}
+
+/**
+ * dm_test_mod_power_setup_streams_modeset - Test a modeset sets up the new stream
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_setup_streams_modeset(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.active = true;
+
+	amdgpu_dm_mod_power_setup_streams(ctx->state, &ctx->adev->dm);
+}
+
+/**
+ * dm_test_mod_power_setup_streams_no_modeset - Test fast updates set up no streams
+ * @test: The KUnit test context
+ */
+static void dm_test_mod_power_setup_streams_no_modeset(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.mode_changed = false;
+	ctx->new_crtc_state->base.active = true;
+
+	amdgpu_dm_mod_power_setup_streams(ctx->state, &ctx->adev->dm);
+}
+
+/**
+ * dm_test_atomic_setup_commit_color_mgmt - Test color management is reprogrammed
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_setup_commit_color_mgmt(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.active = true;
+	ctx->new_crtc_state->base.color_mgmt_changed = true;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_atomic_setup_commit(ctx->state), 0);
+}
+
+/**
+ * dm_test_atomic_setup_commit_modeset - Test a modeset alone reprograms color state
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_setup_commit_modeset(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+
+	ctx->new_crtc_state->base.active = true;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_atomic_setup_commit(ctx->state), 0);
+}
+
+/**
+ * dm_test_atomic_setup_commit_bad_lut - Test an invalid degamma LUT is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_atomic_setup_commit_bad_lut(struct kunit *test)
+{
+	struct dm_test_modeset_ctx *ctx = dm_test_modeset_ctx_alloc(test);
+	struct drm_property_blob *blob;
+	struct drm_color_lut *lut;
+
+	blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, blob);
+	lut = kunit_kzalloc(test, sizeof(*lut), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, lut);
+
+	blob->length = sizeof(*lut);
+	blob->data = lut;
+	ctx->new_crtc_state->base.active = true;
+	ctx->new_crtc_state->base.degamma_lut = blob;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_atomic_setup_commit(ctx->state), -EINVAL);
+}
+
+/* Tests for mmhub_read_system_context() */
+
+#define DM_TEST_PD_ADDR		0x1234000ULL
+
+/*
+ * Stub for the page directory address read: amdgpu_gmc_pd_addr() walks a live
+ * TTM buffer object back to its device, so return a fixed address instead.
+ */
+static uint64_t dm_test_gmc_pd_addr(struct amdgpu_bo *bo)
+{
+	return DM_TEST_PD_ADDR;
+}
+
+static const struct amdgpu_dm_kunit_ops dm_test_dm_ops = {
+	.gmc_pd_addr = dm_test_gmc_pd_addr,
+};
+
+/*
+ * A device whose AGP aperture is disabled (bot above top), so the frame buffer
+ * alone decides the logical address range.
+ */
+static struct amdgpu_device *dm_test_mmhub_adev(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	dm_test_install_dm_ops(test, &dm_test_dm_ops);
+
+	adev->gmc.agp_start = 0x2000000;
+	adev->gmc.agp_end = 0x1000000;
+	adev->gmc.fb_start = 0x40000000;
+	adev->gmc.fb_end = 0x7fffffff;
+	adev->gmc.gart_start = 0x100000000ULL;
+	adev->gmc.gart_end = 0x1ffffffffULL;
+	adev->vm_manager.vram_base_offset = 0x800000;
+
+	return adev;
+}
+
+/**
+ * dm_test_mmhub_agp_disabled - Test a disabled AGP aperture uses the frame buffer
+ * @test: The KUnit test context
+ */
+static void dm_test_mmhub_agp_disabled(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_mmhub_adev(test);
+	struct dc_phy_addr_space_config pa_config;
+
+	mmhub_read_system_context(adev, &pa_config);
+
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.start_addr, 0x40000000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.end_addr, 0x7ffc0000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.agp_base, 0ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.agp_bot, 0x2000000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.agp_top, 0x1000000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.fb_base, 0x40000000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.fb_offset, 0x800000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.fb_top, 0x7fffffffULL);
+	KUNIT_EXPECT_EQ(test, pa_config.gart_config.page_table_start_addr, 0x100000000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.gart_config.page_table_end_addr, 0x1fffff000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.gart_config.page_table_base_addr, DM_TEST_PD_ADDR);
+	KUNIT_EXPECT_FALSE(test, pa_config.is_hvm_enabled);
+}
+
+/**
+ * dm_test_mmhub_agp_disabled_raven2 - Test the Raven2 aperture workaround
+ * @test: The KUnit test context
+ */
+static void dm_test_mmhub_agp_disabled_raven2(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_mmhub_adev(test);
+	struct dc_phy_addr_space_config pa_config;
+
+	adev->apu_flags = AMD_APU_IS_RAVEN2;
+
+	mmhub_read_system_context(adev, &pa_config);
+
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.end_addr, 0x80000000ULL);
+}
+
+/**
+ * dm_test_mmhub_agp_enabled - Test an enabled AGP aperture widens the range
+ * @test: The KUnit test context
+ */
+static void dm_test_mmhub_agp_enabled(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_mmhub_adev(test);
+	struct dc_phy_addr_space_config pa_config;
+
+	adev->gmc.agp_start = 0x1000000;
+	adev->gmc.agp_end = 0x2000000;
+	adev->mode_info.gpu_vm_support = true;
+
+	mmhub_read_system_context(adev, &pa_config);
+
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.start_addr, 0x1000000ULL);
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.end_addr, 0x7ffc0000ULL);
+	KUNIT_EXPECT_TRUE(test, pa_config.is_hvm_enabled);
+}
+
+/**
+ * dm_test_mmhub_agp_enabled_renoir - Test the Renoir aperture workaround
+ * @test: The KUnit test context
+ */
+static void dm_test_mmhub_agp_enabled_renoir(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_mmhub_adev(test);
+	struct dc_phy_addr_space_config pa_config;
+
+	adev->gmc.agp_start = 0x1000000;
+	adev->gmc.agp_end = 0x2000000;
+	adev->apu_flags = AMD_APU_IS_RENOIR;
+
+	mmhub_read_system_context(adev, &pa_config);
+
+	KUNIT_EXPECT_EQ(test, pa_config.system_aperture.end_addr, 0x80000000ULL);
+}
+
+/* Tests for amdgpu_dm_init_power_module() */
+
+/**
+ * dm_test_init_power_module_no_edp - Test no eDP skips the power module
+ * @test: The KUnit test context
+ */
+static void dm_test_init_power_module_no_edp(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->dm.ddev = &adev->ddev;
+	adev->dm.num_of_edps = 0;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_init_power_module(&adev->dm), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.power_module);
+}
+
+/**
+ * dm_test_init_power_module_alloc_failure - Test a failed power module create
+ * @test: The KUnit test context
+ *
+ * mod_power_create() rejects a NULL DC, which walks the full parameter setup
+ * loop and then reports the allocation failure.
+ */
+static void dm_test_init_power_module_alloc_failure(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->dm.ddev = &adev->ddev;
+	adev->dm.num_of_edps = 1;
+	adev->dm.backlight_caps[0].min_input_signal = 0x10;
+	adev->dm.backlight_caps[0].max_input_signal = 0xff;
+
+	KUNIT_EXPECT_EQ(test, amdgpu_dm_init_power_module(&adev->dm), -ENOMEM);
+	KUNIT_EXPECT_NULL(test, adev->dm.power_module);
+}
+
+/* Tests for fill_dc_plane_info_and_addr() */
+
+struct dm_test_plane_info_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_framebuffer *afb;
+	struct drm_plane *plane;
+	struct dm_plane_state *dm_plane_state;
+	struct drm_plane_state *plane_state;
+	struct dc_plane_info plane_info;
+	struct dc_plane_address address;
+};
+
+/*
+ * A linear GFX9 framebuffer of @drm_format bound to an unrotated plane, which
+ * is the simplest input that lets the buffer attribute helper succeed. @adev is
+ * passed in because a test may only allocate one mock DRM device.
+ */
+static struct dm_test_plane_info_ctx *
+dm_test_plane_info_ctx_alloc(struct kunit *test, struct amdgpu_device *adev,
+			     u32 drm_format)
+{
+	struct dm_test_plane_info_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = adev;
+	ctx->afb = kunit_kzalloc(test, sizeof(*ctx->afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->afb);
+	ctx->plane = kunit_kzalloc(test, sizeof(*ctx->plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->plane);
+	ctx->dm_plane_state = kunit_kzalloc(test, sizeof(*ctx->dm_plane_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dm_plane_state);
+	ctx->plane_state = &ctx->dm_plane_state->base;
+
+	ctx->adev->family = AMDGPU_FAMILY_NV;
+	ctx->adev->ip_versions[GC_HWIP][0] = IP_VERSION(10, 3, 0);
+
+	ctx->afb->address = 0x80000000ULL;
+	ctx->afb->base.width = 1920;
+	ctx->afb->base.height = 1080;
+	ctx->afb->base.offsets[1] = 0x200000;
+	ctx->afb->base.pitches[0] = 1920 * 4;
+	ctx->afb->base.pitches[1] = 1920 * 4;
+	ctx->afb->base.modifier = DRM_FORMAT_MOD_LINEAR;
+	ctx->afb->base.format = drm_format_info(drm_format);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->afb->base.format);
+
+	ctx->plane->dev = &ctx->adev->ddev;
+	ctx->plane_state->plane = ctx->plane;
+	ctx->plane_state->fb = &ctx->afb->base;
+	ctx->plane_state->rotation = DRM_MODE_ROTATE_0;
+	ctx->plane_state->alpha = DRM_BLEND_ALPHA_OPAQUE;
+	ctx->plane_state->src_w = 1920 << 16;
+	ctx->plane_state->src_h = 1080 << 16;
+	ctx->plane_state->crtc_w = 1920;
+	ctx->plane_state->crtc_h = 1080;
+
+	return ctx;
+}
+
+static int dm_test_fill_plane_info(struct dm_test_plane_info_ctx *ctx)
+{
+	return fill_dc_plane_info_and_addr(ctx->adev, ctx->plane_state->state,
+					   ctx->plane_state,
+					   &ctx->plane_info, &ctx->address, false);
+}
+
+static const struct {
+	u32 drm_format;
+	enum surface_pixel_format dc_format;
+} dm_test_plane_graphics_formats[] = {
+	{ DRM_FORMAT_C8, SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS },
+	{ DRM_FORMAT_RGB565, SURFACE_PIXEL_FORMAT_GRPH_RGB565 },
+	{ DRM_FORMAT_XRGB8888, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888 },
+	{ DRM_FORMAT_ARGB8888, SURFACE_PIXEL_FORMAT_GRPH_ARGB8888 },
+	{ DRM_FORMAT_XRGB2101010, SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010 },
+	{ DRM_FORMAT_ARGB2101010, SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010 },
+	{ DRM_FORMAT_XBGR2101010, SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010 },
+	{ DRM_FORMAT_ABGR2101010, SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010 },
+	{ DRM_FORMAT_XBGR8888, SURFACE_PIXEL_FORMAT_GRPH_ABGR8888 },
+	{ DRM_FORMAT_ABGR8888, SURFACE_PIXEL_FORMAT_GRPH_ABGR8888 },
+	{ DRM_FORMAT_XRGB16161616F, SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F },
+	{ DRM_FORMAT_ARGB16161616F, SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F },
+	{ DRM_FORMAT_XBGR16161616F, SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F },
+	{ DRM_FORMAT_ABGR16161616F, SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F },
+	{ DRM_FORMAT_XRGB16161616, SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 },
+	{ DRM_FORMAT_ARGB16161616, SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 },
+	{ DRM_FORMAT_XBGR16161616, SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616 },
+	{ DRM_FORMAT_ABGR16161616, SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616 },
+};
+
+/**
+ * dm_test_plane_info_graphics_formats - Test every graphics format mapping
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_info_graphics_formats(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dm_test_plane_graphics_formats); i++) {
+		u32 fmt = dm_test_plane_graphics_formats[i].drm_format;
+		struct dm_test_plane_info_ctx *ctx;
+
+		ctx = dm_test_plane_info_ctx_alloc(test, adev, fmt);
+
+		KUNIT_EXPECT_EQ_MSG(test, dm_test_fill_plane_info(ctx), 0,
+				    "format %p4cc", &fmt);
+		KUNIT_EXPECT_EQ_MSG(test, (int)ctx->plane_info.format,
+				    (int)dm_test_plane_graphics_formats[i].dc_format,
+				    "format %p4cc", &fmt);
+		/* Graphics formats ignore the DRM colour properties. */
+		KUNIT_EXPECT_EQ(test, (int)ctx->plane_info.color_space,
+				(int)COLOR_SPACE_SRGB);
+	}
+}
+
+static const struct {
+	u32 drm_format;
+	enum surface_pixel_format dc_format;
+} dm_test_plane_video_formats[] = {
+	{ DRM_FORMAT_NV21, SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr },
+	{ DRM_FORMAT_NV12, SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb },
+	{ DRM_FORMAT_P010, SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb },
+};
+
+/**
+ * dm_test_plane_info_video_formats - Test every video format mapping
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_info_video_formats(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dm_test_plane_video_formats); i++) {
+		u32 fmt = dm_test_plane_video_formats[i].drm_format;
+		struct dm_test_plane_info_ctx *ctx;
+
+		ctx = dm_test_plane_info_ctx_alloc(test, adev, fmt);
+		ctx->plane_state->color_encoding = DRM_COLOR_YCBCR_BT709;
+		ctx->plane_state->color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+
+		KUNIT_EXPECT_EQ_MSG(test, dm_test_fill_plane_info(ctx), 0,
+				    "format %p4cc", &fmt);
+		KUNIT_EXPECT_EQ_MSG(test, (int)ctx->plane_info.format,
+				    (int)dm_test_plane_video_formats[i].dc_format,
+				    "format %p4cc", &fmt);
+		KUNIT_EXPECT_EQ(test, (int)ctx->plane_info.color_space,
+				(int)COLOR_SPACE_YCBCR709_LIMITED);
+	}
+}
+
+/**
+ * dm_test_plane_info_unsupported_format - Test an unsupported format is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_info_unsupported_format(struct kunit *test)
+{
+	struct dm_test_plane_info_ctx *ctx;
+
+	ctx = dm_test_plane_info_ctx_alloc(test, dm_kunit_alloc_adev(test),
+					   DRM_FORMAT_YUYV);
+
+	KUNIT_EXPECT_EQ(test, dm_test_fill_plane_info(ctx), -EINVAL);
+}
+
+/**
+ * dm_test_plane_info_bad_color_encoding - Test an invalid colour encoding is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_info_bad_color_encoding(struct kunit *test)
+{
+	struct dm_test_plane_info_ctx *ctx;
+
+	ctx = dm_test_plane_info_ctx_alloc(test, dm_kunit_alloc_adev(test),
+					   DRM_FORMAT_NV12);
+	ctx->plane_state->color_encoding = DRM_COLOR_ENCODING_MAX;
+
+	KUNIT_EXPECT_EQ(test, dm_test_fill_plane_info(ctx), -EINVAL);
+}
+
+/**
+ * dm_test_plane_info_rotations - Test every DRM rotation maps to a DC angle
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_info_rotations(struct kunit *test)
+{
+	static const struct {
+		unsigned int drm_rotation;
+		enum dc_rotation_angle dc_rotation;
+	} cases[] = {
+		{ DRM_MODE_ROTATE_0, ROTATION_ANGLE_0 },
+		{ DRM_MODE_ROTATE_90, ROTATION_ANGLE_90 },
+		{ DRM_MODE_ROTATE_180, ROTATION_ANGLE_180 },
+		{ DRM_MODE_ROTATE_270, ROTATION_ANGLE_270 },
+		/* No rotation bit set falls back to the unrotated angle. */
+		{ 0, ROTATION_ANGLE_0 },
+	};
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		struct dm_test_plane_info_ctx *ctx;
+
+		ctx = dm_test_plane_info_ctx_alloc(test, adev, DRM_FORMAT_ARGB8888);
+		ctx->plane_state->rotation = cases[i].drm_rotation;
+
+		KUNIT_EXPECT_EQ(test, dm_test_fill_plane_info(ctx), 0);
+		KUNIT_EXPECT_EQ(test, (int)ctx->plane_info.rotation,
+				(int)cases[i].dc_rotation);
+	}
+}
+
+/**
+ * dm_test_plane_info_layer_and_blending - Test z-order and blending are forwarded
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_info_layer_and_blending(struct kunit *test)
+{
+	struct dm_test_plane_info_ctx *ctx;
+
+	ctx = dm_test_plane_info_ctx_alloc(test, dm_kunit_alloc_adev(test),
+					   DRM_FORMAT_ARGB8888);
+	ctx->plane_state->normalized_zpos = 3;
+	ctx->plane_state->pixel_blend_mode = DRM_MODE_BLEND_PREMULTI;
+	ctx->plane_state->alpha = DRM_BLEND_ALPHA_OPAQUE / 2;
+
+	KUNIT_EXPECT_EQ(test, dm_test_fill_plane_info(ctx), 0);
+	KUNIT_EXPECT_TRUE(test, ctx->plane_info.visible);
+	KUNIT_EXPECT_EQ(test, ctx->plane_info.layer_index, 3);
+	KUNIT_EXPECT_EQ(test, (int)ctx->plane_info.stereo_format,
+			(int)PLANE_STEREO_FORMAT_NONE);
+	KUNIT_EXPECT_TRUE(test, ctx->plane_info.per_pixel_alpha);
+	KUNIT_EXPECT_TRUE(test, ctx->plane_info.global_alpha);
+	KUNIT_EXPECT_EQ(test, ctx->plane_info.global_alpha_value, 0x7f);
+}
+
+/* Tests for amdgpu_dm_enable_self_refresh() */
+
+struct dm_test_sr_ctx {
+	struct amdgpu_display_manager *dm;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state *acrtc_state;
+	struct amdgpu_dm_connector *aconn;
+	struct dc_link *link;
+};
+
+/*
+ * A fast-update CRTC whose stream has a self-refresh capable link. The power
+ * module stays NULL, which every mod_power entry point treats as a no-op.
+ */
+static struct dm_test_sr_ctx *dm_test_sr_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_sr_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->dm = dm_kunit_alloc_dm(test);
+	ctx->acrtc = kunit_kzalloc(test, sizeof(*ctx->acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc);
+	ctx->acrtc_state = kunit_kzalloc(test, sizeof(*ctx->acrtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc_state);
+	ctx->aconn = kunit_kzalloc(test, sizeof(*ctx->aconn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->aconn);
+
+	ctx->link = dm_kunit_alloc_link(test);
+	ctx->acrtc_state->stream = dm_kunit_alloc_stream(test, ctx->link);
+	ctx->acrtc_state->stream->dm_stream_context = ctx->aconn;
+	ctx->acrtc_state->update_type = UPDATE_TYPE_FAST;
+
+	return ctx;
+}
+
+static void dm_test_enable_sr(struct dm_test_sr_ctx *ctx, u64 current_ts)
+{
+	amdgpu_dm_enable_self_refresh(ctx->dm, ctx->acrtc, ctx->acrtc_state,
+				      current_ts);
+}
+
+/**
+ * dm_test_self_refresh_full_update - Test a full update blocks self refresh
+ * @test: The KUnit test context
+ */
+static void dm_test_self_refresh_full_update(struct kunit *test)
+{
+	struct dm_test_sr_ctx *ctx = dm_test_sr_ctx_alloc(test);
+
+	ctx->acrtc_state->update_type = UPDATE_TYPE_FULL;
+	ctx->link->psr_settings.psr_feature_enabled = true;
+	ctx->acrtc->dm_irq_params.allow_sr_entry = true;
+
+	dm_test_enable_sr(ctx, 0);
+
+	KUNIT_EXPECT_FALSE(test, ctx->acrtc->dm_irq_params.allow_sr_entry);
+}
+
+/**
+ * dm_test_self_refresh_unsupported_link - Test a link without PSR or Replay
+ * @test: The KUnit test context
+ */
+static void dm_test_self_refresh_unsupported_link(struct kunit *test)
+{
+	struct dm_test_sr_ctx *ctx = dm_test_sr_ctx_alloc(test);
+
+	ctx->acrtc->dm_irq_params.allow_sr_entry = true;
+
+	dm_test_enable_sr(ctx, 0);
+
+	KUNIT_EXPECT_FALSE(test, ctx->acrtc->dm_irq_params.allow_sr_entry);
+}
+
+/**
+ * dm_test_self_refresh_decrements_skip_count - Test the skip count gates entry
+ * @test: The KUnit test context
+ */
+static void dm_test_self_refresh_decrements_skip_count(struct kunit *test)
+{
+	struct dm_test_sr_ctx *ctx = dm_test_sr_ctx_alloc(test);
+
+	ctx->link->psr_settings.psr_feature_enabled = true;
+	ctx->aconn->sr_skip_count = 2;
+
+	dm_test_enable_sr(ctx, 0);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconn->sr_skip_count, 1);
+	KUNIT_EXPECT_FALSE(test, ctx->acrtc->dm_irq_params.allow_sr_entry);
+}
+
+/**
+ * dm_test_self_refresh_allows_entry - Test a drained skip count allows entry
+ * @test: The KUnit test context
+ */
+static void dm_test_self_refresh_allows_entry(struct kunit *test)
+{
+	struct dm_test_sr_ctx *ctx = dm_test_sr_ctx_alloc(test);
+
+	ctx->link->psr_settings.psr_feature_enabled = true;
+	ctx->aconn->sr_skip_count = 1;
+
+	/* Well past the 500ms settle window, so the events are cleared. */
+	dm_test_enable_sr(ctx, 2ULL * NSEC_PER_SEC);
+
+	KUNIT_EXPECT_EQ(test, ctx->aconn->sr_skip_count, 0);
+	KUNIT_EXPECT_TRUE(test, ctx->acrtc->dm_irq_params.allow_sr_entry);
+}
+
+/**
+ * dm_test_self_refresh_within_settle_window - Test a recent damage change holds off
+ * @test: The KUnit test context
+ */
+static void dm_test_self_refresh_within_settle_window(struct kunit *test)
+{
+	struct dm_test_sr_ctx *ctx = dm_test_sr_ctx_alloc(test);
+
+	ctx->link->psr_settings.psr_feature_enabled = true;
+	ctx->link->psr_settings.psr_dirty_rects_change_timestamp_ns = 1;
+
+	dm_test_enable_sr(ctx, 2);
+
+	KUNIT_EXPECT_TRUE(test, ctx->acrtc->dm_irq_params.allow_sr_entry);
+}
+
+/**
+ * dm_test_self_refresh_replay_link - Test a Replay capable link takes the same path
+ * @test: The KUnit test context
+ */
+static void dm_test_self_refresh_replay_link(struct kunit *test)
+{
+	struct dm_test_sr_ctx *ctx = dm_test_sr_ctx_alloc(test);
+
+	ctx->link->replay_settings.replay_feature_enabled = true;
+
+	dm_test_enable_sr(ctx, 2ULL * NSEC_PER_SEC);
+
+	KUNIT_EXPECT_TRUE(test, ctx->acrtc->dm_irq_params.allow_sr_entry);
+}
+
+/* Tests for manage_dm_interrupts() */
+
+struct dm_test_irq_mgmt_ctx {
+	struct amdgpu_device *adev;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state *acrtc_state;
+};
+
+/*
+ * A CRTC with a single initialised vblank and a 1080p60 stream timing, which
+ * is what the vblank off-delay estimate is derived from.
+ */
+static struct dm_test_irq_mgmt_ctx *dm_test_irq_mgmt_ctx_alloc(struct kunit *test)
+{
+	struct dm_test_irq_mgmt_ctx *ctx;
+	struct dc_crtc_timing *timing;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_EQ(test, drm_vblank_init(&ctx->adev->ddev, 1), 0);
+
+	ctx->acrtc = kunit_kzalloc(test, sizeof(*ctx->acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc);
+	ctx->acrtc_state = kunit_kzalloc(test, sizeof(*ctx->acrtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->acrtc_state);
+
+	ctx->adev->mode_info.num_crtc = 1;
+	ctx->acrtc->base.dev = &ctx->adev->ddev;
+	ctx->acrtc->base.funcs = &dm_test_vblank_crtc_funcs;
+	INIT_LIST_HEAD(&ctx->acrtc->base.head);
+	list_add_tail(&ctx->acrtc->base.head,
+		      &ctx->adev->ddev.mode_config.crtc_list);
+	ctx->acrtc->crtc_id = 0;
+	KUNIT_ASSERT_EQ(test,
+			kunit_add_action_or_reset(test, dm_test_vblank_ctx_cleanup, ctx->acrtc), 0);
+
+	ctx->acrtc_state->stream = dm_kunit_alloc_stream(test, NULL);
+	timing = &ctx->acrtc_state->stream->timing;
+	timing->h_total = 2200;
+	timing->v_total = 1125;
+	timing->pix_clk_100hz = 1485000;
+
+	return ctx;
+}
+
+/**
+ * dm_test_manage_interrupts_offdelay - Test the off delay is derived from timing
+ * @test: The KUnit test context
+ *
+ * DCN3.0 also takes the extra vupdate reference. The IRQ subsystem is not
+ * installed, so amdgpu_irq_get() only reports the missing source.
+ */
+static void dm_test_manage_interrupts_offdelay(struct kunit *test)
+{
+	struct dm_test_irq_mgmt_ctx *ctx = dm_test_irq_mgmt_ctx_alloc(test);
+
+	/* Pre-DCN3.5 keeps the two frame off delay. */
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 0, 0);
+
+	manage_dm_interrupts(ctx->adev, ctx->acrtc, ctx->acrtc_state);
+
+	KUNIT_EXPECT_EQ(test, ctx->adev->ddev.vblank[0].config.offdelay_ms, 34);
+	KUNIT_EXPECT_FALSE(test, ctx->adev->ddev.vblank[0].config.disable_immediate);
+}
+
+/**
+ * dm_test_manage_interrupts_offdelay_fallback - Test a zero delay falls back to 30ms
+ * @test: The KUnit test context
+ */
+static void dm_test_manage_interrupts_offdelay_fallback(struct kunit *test)
+{
+	struct dm_test_irq_mgmt_ctx *ctx = dm_test_irq_mgmt_ctx_alloc(test);
+
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 1, 2);
+	/* A zero line count makes the computed delay round down to zero. */
+	ctx->acrtc_state->stream->timing.v_total = 0;
+
+	manage_dm_interrupts(ctx->adev, ctx->acrtc, ctx->acrtc_state);
+
+	KUNIT_EXPECT_EQ(test, ctx->adev->ddev.vblank[0].config.offdelay_ms, 30);
+}
+
+/**
+ * dm_test_manage_interrupts_apu_instant_off - Test DCN3.5 APUs use instant off
+ * @test: The KUnit test context
+ */
+static void dm_test_manage_interrupts_apu_instant_off(struct kunit *test)
+{
+	struct dm_test_irq_mgmt_ctx *ctx = dm_test_irq_mgmt_ctx_alloc(test);
+
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 5, 0);
+	ctx->adev->flags |= AMD_IS_APU;
+
+	manage_dm_interrupts(ctx->adev, ctx->acrtc, ctx->acrtc_state);
+
+	KUNIT_EXPECT_EQ(test, ctx->adev->ddev.vblank[0].config.offdelay_ms, 1);
+	KUNIT_EXPECT_TRUE(test, ctx->adev->ddev.vblank[0].config.disable_immediate);
+}
+
+/**
+ * dm_test_manage_interrupts_disable - Test a NULL CRTC state turns vblank off
+ * @test: The KUnit test context
+ */
+static void dm_test_manage_interrupts_disable(struct kunit *test)
+{
+	struct dm_test_irq_mgmt_ctx *ctx = dm_test_irq_mgmt_ctx_alloc(test);
+
+	ctx->adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 0, 0);
+	manage_dm_interrupts(ctx->adev, ctx->acrtc, ctx->acrtc_state);
+
+	manage_dm_interrupts(ctx->adev, ctx->acrtc, NULL);
+
+	KUNIT_EXPECT_FALSE(test, ctx->adev->ddev.vblank[0].enabled);
+}
+
+/* Tests for dm_early_init() */
+
+#define DM_TEST_ATOM_BIOS_SIZE	512
+
+/*
+ * A fake ATOM BIOS image. Every byte is non-zero, so the master data table
+ * reports a present entry for whichever index the object header lives at,
+ * which is all amdgpu_atom_parse_data_header() checks here.
+ */
+static struct atom_context *dm_test_alloc_atom_context(struct kunit *test,
+						       bool object_header)
+{
+	struct atom_context *ctx;
+	void *bios;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	bios = kunit_kzalloc(test, DM_TEST_ATOM_BIOS_SIZE, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, bios);
+
+	if (object_header)
+		memset(bios, 0x01, DM_TEST_ATOM_BIOS_SIZE);
+
+	ctx->bios = bios;
+	ctx->bios_size = DM_TEST_ATOM_BIOS_SIZE;
+	ctx->data_table = 0;
+
+	return ctx;
+}
+
+static struct amdgpu_device *dm_test_early_init_adev(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->mode_info.atom_context = dm_test_alloc_atom_context(test, true);
+
+	return adev;
+}
+
+static int dm_test_run_early_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ip_block ip_block = { .adev = adev };
+
+	return dm_early_init(&ip_block);
+}
+
+/**
+ * dm_test_early_init_no_object_header - Test a BIOS without an object header
+ * @test: The KUnit test context
+ */
+static void dm_test_early_init_no_object_header(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->mode_info.atom_context = dm_test_alloc_atom_context(test, false);
+
+	KUNIT_EXPECT_EQ(test, dm_test_run_early_init(adev), -ENOENT);
+	KUNIT_EXPECT_TRUE(test, adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK);
+	KUNIT_EXPECT_FALSE(test, adev->dc_enabled);
+}
+
+/**
+ * dm_test_early_init_legacy_asics - Test the display counts of legacy ASICs
+ * @test: The KUnit test context
+ */
+static void dm_test_early_init_legacy_asics(struct kunit *test)
+{
+	static const struct {
+		enum amd_asic_type asic_type;
+		u32 num_crtc;
+		u32 num_hpd;
+		u32 num_dig;
+	} cases[] = {
+		{ CHIP_BONAIRE, 6, 6, 6 },
+		{ CHIP_HAWAII, 6, 6, 6 },
+		{ CHIP_KAVERI, 4, 6, 7 },
+		{ CHIP_KABINI, 2, 6, 6 },
+		{ CHIP_MULLINS, 2, 6, 6 },
+		{ CHIP_FIJI, 6, 6, 7 },
+		{ CHIP_TONGA, 6, 6, 7 },
+		{ CHIP_CARRIZO, 3, 6, 9 },
+		{ CHIP_STONEY, 2, 6, 9 },
+		{ CHIP_POLARIS11, 5, 5, 5 },
+		{ CHIP_POLARIS12, 5, 5, 5 },
+		{ CHIP_POLARIS10, 6, 6, 6 },
+		{ CHIP_VEGAM, 6, 6, 6 },
+		{ CHIP_VEGA10, 6, 6, 6 },
+		{ CHIP_VEGA12, 6, 6, 6 },
+		{ CHIP_VEGA20, 6, 6, 6 },
+	};
+	struct amdgpu_device *adev = dm_test_early_init_adev(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		adev->asic_type = cases[i].asic_type;
+
+		KUNIT_EXPECT_EQ_MSG(test, dm_test_run_early_init(adev), 0,
+				    "asic_type %d", cases[i].asic_type);
+		KUNIT_EXPECT_EQ_MSG(test, adev->mode_info.num_crtc, cases[i].num_crtc,
+				    "asic_type %d", cases[i].asic_type);
+		KUNIT_EXPECT_EQ_MSG(test, adev->mode_info.num_hpd, cases[i].num_hpd,
+				    "asic_type %d", cases[i].asic_type);
+		KUNIT_EXPECT_EQ_MSG(test, adev->mode_info.num_dig, cases[i].num_dig,
+				    "asic_type %d", cases[i].asic_type);
+	}
+
+	KUNIT_EXPECT_NOT_NULL(test, adev->mode_info.funcs);
+	KUNIT_EXPECT_TRUE(test, adev->dc_enabled);
+}
+
+/**
+ * dm_test_early_init_dcn_versions - Test the display counts of DCN IP versions
+ * @test: The KUnit test context
+ *
+ * Only IP versions without DMUB firmware are used, so dm_init_microcode() does
+ * not reach a firmware request.
+ */
+static void dm_test_early_init_dcn_versions(struct kunit *test)
+{
+	static const struct {
+		u32 ip_version;
+		u32 num_crtc;
+	} cases[] = {
+		{ IP_VERSION(2, 0, 2), 6 },
+		{ IP_VERSION(2, 0, 0), 5 },
+		{ IP_VERSION(2, 0, 3), 2 },
+		{ IP_VERSION(1, 0, 0), 4 },
+		{ IP_VERSION(1, 0, 1), 4 },
+	};
+	struct amdgpu_device *adev = dm_test_early_init_adev(test);
+	unsigned int i;
+
+	adev->asic_type = CHIP_IP_DISCOVERY;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		adev->ip_versions[DCE_HWIP][0] = cases[i].ip_version;
+
+		KUNIT_EXPECT_EQ_MSG(test, dm_test_run_early_init(adev), 0,
+				    "ip_version 0x%x", cases[i].ip_version);
+		KUNIT_EXPECT_EQ_MSG(test, adev->mode_info.num_crtc, cases[i].num_crtc,
+				    "ip_version 0x%x", cases[i].ip_version);
+		/* Every DCN entry keeps hpd and dig in step with the CRTC count. */
+		KUNIT_EXPECT_EQ(test, adev->mode_info.num_hpd, cases[i].num_crtc);
+		KUNIT_EXPECT_EQ(test, adev->mode_info.num_dig, cases[i].num_crtc);
+	}
+}
+
+/**
+ * dm_test_early_init_unsupported_version - Test an unknown IP version is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_early_init_unsupported_version(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_early_init_adev(test);
+
+	adev->asic_type = CHIP_IP_DISCOVERY;
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(9, 9, 9);
+
+	KUNIT_EXPECT_EQ(test, dm_test_run_early_init(adev), -EINVAL);
+	KUNIT_EXPECT_FALSE(test, adev->dc_enabled);
+}
+
+/* Tests for the remaining suspend and resume helpers */
+
+/**
+ * dm_test_commit_zero_streams_empty - Test committing an already empty DC state
+ * @test: The KUnit test context
+ */
+static void dm_test_commit_zero_streams_empty(struct kunit *test)
+{
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	dc->current_state = dm_kunit_alloc_dc_state(test);
+	KUNIT_ASSERT_NOT_NULL(test, dc->current_state);
+
+	KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_commit_zero_streams(dc), (int)DC_OK);
+}
+
+/**
+ * dm_test_cache_state_empty_device - Test caching an empty DRM atomic state
+ * @test: The KUnit test context
+ */
+static void dm_test_cache_state_empty_device(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	KUNIT_ASSERT_EQ(test, dm_cache_state(adev), 0);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adev->dm.cached_state);
+
+	dm_destroy_cached_state(adev);
+	KUNIT_EXPECT_NULL(test, adev->dm.cached_state);
+}
+
+static struct drm_atomic_commit *dm_test_atomic_helper_suspend_error(struct drm_device *dev)
+{
+	return ERR_PTR(-EIO);
+}
+
+static const struct amdgpu_dm_kunit_ops dm_test_cache_state_ops = {
+	.atomic_helper_suspend = dm_test_atomic_helper_suspend_error,
+};
+
+/**
+ * dm_test_cache_state_error - Test atomic suspend errors are returned and cleared
+ * @test: The KUnit test context
+ */
+static void dm_test_cache_state_error(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	dm_test_install_dm_ops(test, &dm_test_cache_state_ops);
+
+	KUNIT_EXPECT_EQ(test, dm_cache_state(adev), -EIO);
+	KUNIT_EXPECT_NULL(test, adev->dm.cached_state);
+}
+
+/**
+ * dm_test_destroy_cached_state_none - Test no cached state is a no-op
+ * @test: The KUnit test context
+ */
+static void dm_test_destroy_cached_state_none(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	dm_destroy_cached_state(adev);
+
+	KUNIT_EXPECT_NULL(test, adev->dm.cached_state);
+}
+
+static bool dm_test_update_bandwidth(struct dc *dc, struct dc_state *context)
+{
+	return true;
+}
+
+/**
+ * dm_test_clear_writeback_removes_stream - Test teardown removes the stream writeback
+ * @test: The KUnit test context
+ */
+static void dm_test_clear_writeback_removes_stream(struct kunit *test)
+{
+	struct amdgpu_display_manager *dm;
+	struct dm_crtc_state *crtc_state;
+	struct amdgpu_crtc *acrtc;
+	struct dc_stream_state *stream;
+
+	dm = dm_kunit_alloc_dm(test);
+	crtc_state = kunit_kzalloc(test, sizeof(*crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc_state);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	stream = dm_kunit_alloc_stream(test, NULL);
+	dm->dc->hwss.update_bandwidth = dm_test_update_bandwidth;
+	crtc_state->stream = stream;
+	stream->num_wb_info = 1;
+	stream->writeback_info[0].dwb_pipe_inst = 0;
+
+	dm_clear_writeback(dm, acrtc, crtc_state);
+
+	KUNIT_EXPECT_EQ(test, stream->num_wb_info, 0U);
+}
+
+/**
+ * dm_test_set_writeback_no_pipe - Test a stream without a DC pipe is not armed
+ * @test: The KUnit test context
+ */
+static void dm_test_set_writeback_no_pipe(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct drm_writeback_connector *wb_conn;
+	struct drm_connector_state *conn_state;
+	struct drm_writeback_job *job;
+	struct amdgpu_framebuffer *afb;
+	struct dm_crtc_state *crtc_state;
+	struct amdgpu_crtc *acrtc;
+	struct dc *dc;
+
+	wb_conn = kunit_kzalloc(test, sizeof(*wb_conn), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, wb_conn);
+	conn_state = kunit_kzalloc(test, sizeof(*conn_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, conn_state);
+	job = kunit_kzalloc(test, sizeof(*job), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, job);
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+	crtc_state = kunit_kzalloc(test, sizeof(*crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc_state);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+	dc = dm_kunit_alloc_dc_with_ctx(test);
+	dc->current_state = dm_kunit_alloc_dc_state(test);
+
+	adev->dm.adev = adev;
+	adev->dm.dc = dc;
+	wb_conn->encoder.crtc = &acrtc->base;
+	job->fb = &afb->base;
+	conn_state->writeback_job = job;
+	crtc_state->stream = dm_kunit_alloc_stream(test, NULL);
+
+	dm_set_writeback(&adev->dm, crtc_state, &wb_conn->base, conn_state);
+
+	KUNIT_EXPECT_NULL(test, acrtc->wb_conn);
+	KUNIT_EXPECT_FALSE(test, acrtc->wb_pending);
+}
+
+/* Tests for dm_update_mst_vcpi_slots_for_dsc() */
+
+/**
+ * dm_test_mst_vcpi_slots_no_connector - Test an empty commit allocates no slots
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_vcpi_slots_no_connector(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {0};
+	struct dc_state *dc_state = dm_kunit_alloc_dc_state(test);
+
+	KUNIT_ASSERT_NOT_NULL(test, dc_state);
+
+	KUNIT_EXPECT_EQ(test,
+			dm_update_mst_vcpi_slots_for_dsc(dm_test_alloc_commit(test, adev),
+							 dc_state, vars),
+			0);
+}
+
+/**
+ * dm_test_mst_vcpi_slots_skips_writeback - Test writeback connectors are skipped
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_vcpi_slots_skips_writeback(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {0};
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_state *dc_state;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	dc_state = dm_kunit_alloc_dc_state(test);
+	KUNIT_ASSERT_NOT_NULL(test, dc_state);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_WRITEBACK;
+	state = dm_test_state_with_connector(test, &aconnector->base, crtc);
+
+	KUNIT_EXPECT_EQ(test,
+			dm_update_mst_vcpi_slots_for_dsc(state, dc_state, vars), 0);
+}
+
+/**
+ * dm_test_mst_vcpi_slots_skips_non_mst - Test a connector without an MST port
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_vcpi_slots_skips_non_mst(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {0};
+	struct amdgpu_dm_connector *aconnector;
+	struct dc_state *dc_state;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	dc_state = dm_kunit_alloc_dc_state(test);
+	KUNIT_ASSERT_NOT_NULL(test, dc_state);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_DisplayPort;
+	state = dm_test_state_with_connector(test, &aconnector->base, crtc);
+
+	KUNIT_EXPECT_EQ(test,
+			dm_update_mst_vcpi_slots_for_dsc(state, dc_state, vars), 0);
+}
+
+/**
+ * dm_test_mst_vcpi_slots_no_matching_stream - Test a connector with no DC stream
+ * @test: The KUnit test context
+ */
+static void dm_test_mst_vcpi_slots_no_matching_stream(struct kunit *test)
+{
+	struct dsc_mst_fairness_vars vars[MAX_PIPES] = {0};
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_port *port;
+	struct dc_state *dc_state;
+	struct drm_atomic_commit *state;
+	struct drm_crtc *crtc;
+
+	aconnector = kunit_kzalloc(test, sizeof(*aconnector), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, aconnector);
+	port = kunit_kzalloc(test, sizeof(*port), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, port);
+	crtc = kunit_kzalloc(test, sizeof(*crtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc);
+	dc_state = dm_kunit_alloc_dc_state(test);
+	KUNIT_ASSERT_NOT_NULL(test, dc_state);
+
+	aconnector->base.connector_type = DRM_MODE_CONNECTOR_DisplayPort;
+	aconnector->mst_output_port = port;
+	state = dm_test_state_with_connector(test, &aconnector->base, crtc);
+
+	/* No DC stream references the connector, so the slot update is skipped. */
+	KUNIT_EXPECT_EQ(test,
+			dm_update_mst_vcpi_slots_for_dsc(state, dc_state, vars), 0);
+}
+
+/* Tests for fill_dc_plane_attributes() */
+
+struct dm_test_plane_attr_ctx {
+	struct amdgpu_device *adev;
+	struct dm_crtc_state *crtc_state;
+	struct dm_test_plane_info_ctx *plane;
+	struct dc_plane_state *dc_plane;
+};
+
+/*
+ * A DC plane, plus a CRTC state complete enough for the colour management
+ * update at the end of fill_dc_plane_attributes(). A scaling factor of 1 in the
+ * plane caps means "no scaling", which is what the 1:1 geometry of the plane
+ * info context needs.
+ */
+static struct dm_test_plane_attr_ctx *
+dm_test_plane_attr_ctx_alloc(struct kunit *test, u32 drm_format)
+{
+	struct dm_test_plane_attr_ctx *ctx;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+	ctx->crtc_state = kunit_kzalloc(test, sizeof(*ctx->crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->crtc_state);
+	ctx->dc_plane = kunit_kzalloc(test, sizeof(*ctx->dc_plane), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, ctx->dc_plane);
+
+	ctx->adev = dm_kunit_alloc_adev(test);
+	ctx->adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+	ctx->adev->dm.dc->caps.planes[0].max_upscale_factor.argb8888 = 1;
+	ctx->adev->dm.dc->caps.planes[0].max_downscale_factor.argb8888 = 1;
+
+	ctx->crtc_state->stream = dm_kunit_alloc_stream(test, NULL);
+	/* Colour management resolves the device through the commit backpointer. */
+	ctx->crtc_state->base.state = dm_test_alloc_commit(test, ctx->adev);
+	ctx->plane = dm_test_plane_info_ctx_alloc(test, ctx->adev, drm_format);
+
+	return ctx;
+}
+
+static int dm_test_fill_plane_attr(struct dm_test_plane_attr_ctx *ctx)
+{
+	return fill_dc_plane_attributes(ctx->adev, ctx->dc_plane,
+					ctx->plane->plane_state, &ctx->crtc_state->base);
+}
+
+/**
+ * dm_test_plane_attributes_success - Test plane info is copied into the DC plane
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_attributes_success(struct kunit *test)
+{
+	struct dm_test_plane_attr_ctx *ctx;
+
+	ctx = dm_test_plane_attr_ctx_alloc(test, DRM_FORMAT_ARGB8888);
+	ctx->plane->plane_state->normalized_zpos = 2;
+
+	KUNIT_EXPECT_EQ(test, dm_test_fill_plane_attr(ctx), 0);
+	KUNIT_EXPECT_EQ(test, (int)ctx->dc_plane->format,
+			(int)SURFACE_PIXEL_FORMAT_GRPH_ARGB8888);
+	KUNIT_EXPECT_EQ(test, (int)ctx->dc_plane->color_space, (int)COLOR_SPACE_SRGB);
+	KUNIT_EXPECT_EQ(test, (int)ctx->dc_plane->rotation, (int)ROTATION_ANGLE_0);
+	KUNIT_EXPECT_EQ(test, ctx->dc_plane->src_rect.width, 1920U);
+	KUNIT_EXPECT_EQ(test, ctx->dc_plane->dst_rect.width, 1920U);
+	KUNIT_EXPECT_EQ(test, ctx->dc_plane->layer_index, 2);
+	KUNIT_EXPECT_TRUE(test, ctx->dc_plane->visible);
+	KUNIT_EXPECT_TRUE(test, ctx->dc_plane->flip_int_enabled);
+}
+
+/**
+ * dm_test_plane_attributes_bad_scaling - Test an empty source rectangle is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_attributes_bad_scaling(struct kunit *test)
+{
+	struct dm_test_plane_attr_ctx *ctx;
+
+	ctx = dm_test_plane_attr_ctx_alloc(test, DRM_FORMAT_ARGB8888);
+	ctx->plane->plane_state->src_w = 0;
+
+	KUNIT_EXPECT_EQ(test, dm_test_fill_plane_attr(ctx), -EINVAL);
+}
+
+/**
+ * dm_test_plane_attributes_bad_format - Test an unsupported format is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_attributes_bad_format(struct kunit *test)
+{
+	struct dm_test_plane_attr_ctx *ctx;
+
+	ctx = dm_test_plane_attr_ctx_alloc(test, DRM_FORMAT_YUYV);
+
+	KUNIT_EXPECT_EQ(test, dm_test_fill_plane_attr(ctx), -EINVAL);
+}
+
+/**
+ * dm_test_plane_attributes_bad_color_mgmt - Test a bad 3D LUT is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_plane_attributes_bad_color_mgmt(struct kunit *test)
+{
+	struct dm_test_plane_attr_ctx *ctx;
+	struct drm_property_blob *blob;
+	struct drm_color_lut *lut;
+
+	ctx = dm_test_plane_attr_ctx_alloc(test, DRM_FORMAT_ARGB8888);
+	blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, blob);
+	lut = kunit_kcalloc(test, 16, sizeof(*lut), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, lut);
+
+	/* A 3D LUT that is not a perfect cube fails validation. */
+	blob->data = lut;
+	blob->length = 16 * sizeof(*lut);
+	ctx->adev->dm.dc->caps.color.dpp.hw_3d_lut = true;
+	ctx->plane->dm_plane_state->lut3d = blob;
+
+	KUNIT_EXPECT_EQ(test, dm_test_fill_plane_attr(ctx), -EINVAL);
+}
+
+/* Tests for load_dmcu_fw() */
+
+/**
+ * dm_test_load_dmcu_fw_no_dmcu - Test ASICs and IP versions without a DMCU
+ * @test: The KUnit test context
+ */
+static void dm_test_load_dmcu_fw_no_dmcu(struct kunit *test)
+{
+	static const enum amd_asic_type cases[] = {
+		CHIP_BONAIRE, CHIP_HAWAII, CHIP_KAVERI, CHIP_KABINI, CHIP_MULLINS,
+		CHIP_TONGA, CHIP_FIJI, CHIP_CARRIZO, CHIP_STONEY, CHIP_POLARIS11,
+		CHIP_POLARIS10, CHIP_POLARIS12, CHIP_VEGAM, CHIP_VEGA10,
+		CHIP_VEGA12, CHIP_VEGA20,
+	};
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		adev->asic_type = cases[i];
+
+		KUNIT_EXPECT_EQ_MSG(test, load_dmcu_fw(adev), 0, "asic_type %d",
+				    cases[i]);
+		KUNIT_EXPECT_NULL(test, adev->dm.fw_dmcu);
+	}
+}
+
+/**
+ * dm_test_load_dmcu_fw_dcn - Test DCN IP versions report no DMCU firmware
+ * @test: The KUnit test context
+ */
+static void dm_test_load_dmcu_fw_dcn(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->asic_type = CHIP_IP_DISCOVERY;
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 5, 0);
+
+	KUNIT_EXPECT_EQ(test, load_dmcu_fw(adev), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.fw_dmcu);
+}
+
+/**
+ * dm_test_load_dmcu_fw_unsupported - Test an unknown IP version is rejected
+ * @test: The KUnit test context
+ */
+static void dm_test_load_dmcu_fw_unsupported(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->asic_type = CHIP_IP_DISCOVERY;
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(9, 9, 9);
+
+	KUNIT_EXPECT_EQ(test, load_dmcu_fw(adev), -EINVAL);
+}
+
+/**
+ * dm_test_load_dmcu_fw_raven - Test the Raven revision selects a DMCU
+ * @test: The KUnit test context
+ */
+static void dm_test_load_dmcu_fw_raven(struct kunit *test)
+{
+	static const u32 cases[] = { PICASSO_A0, RAVEN2_A0, 0 };
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	unsigned int i;
+
+	adev->asic_type = CHIP_RAVEN;
+	adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
+
+	/* Picasso and Raven2 name a DMCU; a bare Raven has none. */
+	for (i = 0; i < ARRAY_SIZE(cases); i++) {
+		adev->external_rev_id = cases[i];
+
+		KUNIT_EXPECT_EQ_MSG(test, load_dmcu_fw(adev), 0, "rev 0x%x",
+				    cases[i]);
+	}
+}
+
+/**
+ * dm_test_load_dmcu_fw_missing_firmware - Test a missing DMCU image is not fatal
+ * @test: The KUnit test context
+ */
+static void dm_test_load_dmcu_fw_missing_firmware(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+
+	adev->dev = adev->ddev.dev;
+	adev->asic_type = CHIP_NAVI12;
+	adev->firmware.load_type = AMDGPU_FW_LOAD_PSP;
+
+	/* No firmware is installed in the test environment. */
+	KUNIT_EXPECT_EQ(test, load_dmcu_fw(adev), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.fw_dmcu);
+}
+
+/* Tests for dm_sw_init() and dm_sw_fini() */
+
+/**
+ * dm_test_sw_init_no_dmub - Test software init on an ASIC without a DMUB
+ * @test: The KUnit test context
+ */
+static void dm_test_sw_init_no_dmub(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_ip_block ip_block = { .adev = adev };
+
+	adev->asic_type = CHIP_BONAIRE;
+
+	KUNIT_EXPECT_EQ(test, dm_sw_init(&ip_block), 0);
+	KUNIT_ASSERT_NOT_NULL(test, adev->dm.cgs_device);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+
+	/* amdgpu_cgs_destroy_device() is not exported and is a plain kfree(). */
+	kfree(adev->dm.cgs_device);
+}
+
+/**
+ * dm_test_sw_fini_releases_bounding_box - Test the DMUB bounding box is released
+ * @test: The KUnit test context
+ */
+static void dm_test_sw_fini_releases_bounding_box(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_ip_block ip_block = { .adev = adev };
+	struct dal_allocation *da;
+	void *bb;
+
+	bb = kunit_kzalloc(test, sizeof(*da), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, bb);
+	/* Freed by dm_sw_fini(), so it must not be KUnit-managed. */
+	da = kzalloc_obj(*da);
+	KUNIT_ASSERT_NOT_NULL(test, da);
+
+	INIT_LIST_HEAD(&adev->dm.da_list);
+	da->cpu_ptr = bb;
+	list_add(&da->list, &adev->dm.da_list);
+	adev->dm.bb_from_dmub = bb;
+
+	KUNIT_EXPECT_EQ(test, dm_sw_fini(&ip_block), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.bb_from_dmub);
+	KUNIT_EXPECT_TRUE(test, list_empty(&adev->dm.da_list));
+}
+
+/* Tests for dm_late_init() */
+
+/*
+ * A DC without a DMCU and with an empty link list, so the ABM configuration is
+ * skipped and only the MST detection sweep runs.
+ */
+static struct amdgpu_device *dm_test_late_init_adev(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc *dc = dm_kunit_alloc_dc_with_ctx(test);
+
+	dc->res_pool = kunit_kzalloc(test, sizeof(*dc->res_pool), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dc->res_pool);
+	adev->dm.dc = dc;
+
+	return adev;
+}
+
+static int dm_test_run_late_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ip_block ip_block = { .adev = adev };
+
+	return dm_late_init(&ip_block);
+}
+
+/**
+ * dm_test_late_init_no_dmcu - Test a DC without a DMCU or DMUB
+ * @test: The KUnit test context
+ */
+static void dm_test_late_init_no_dmcu(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_late_init_adev(test);
+
+	KUNIT_EXPECT_EQ(test, dm_test_run_late_init(adev), 0);
+}
+
+/**
+ * dm_test_late_init_boot_crc_no_dmub - Test boot time CRC needs a DMUB service
+ * @test: The KUnit test context
+ */
+static void dm_test_late_init_boot_crc_no_dmub(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_late_init_adev(test);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 6, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_test_run_late_init(adev), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.boot_time_crc_info.bo_ptr);
+}
+
+static union dmub_fw_boot_options dm_test_fw_boot_options;
+
+static union dmub_fw_boot_options dm_test_get_fw_boot_option(struct dmub_srv *dmub)
+{
+	return dm_test_fw_boot_options;
+}
+
+/**
+ * dm_test_late_init_boot_crc_disabled - Test a disabled boot time CRC allocates nothing
+ * @test: The KUnit test context
+ */
+static void dm_test_late_init_boot_crc_disabled(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_late_init_adev(test);
+	struct dc_dmub_srv *dmub_srv;
+	struct dmub_srv *dmub;
+
+	dmub_srv = kunit_kzalloc(test, sizeof(*dmub_srv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dmub_srv);
+	dmub = kunit_kzalloc(test, sizeof(*dmub), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, dmub);
+
+	dm_test_fw_boot_options.bits.bootcrc_en_at_S0i3 = 0;
+	dmub->hw_funcs.get_fw_boot_option = dm_test_get_fw_boot_option;
+	dmub_srv->dmub = dmub;
+	adev->dm.dc->ctx->dmub_srv = dmub_srv;
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 6, 0);
+
+	KUNIT_EXPECT_EQ(test, dm_test_run_late_init(adev), 0);
+	KUNIT_EXPECT_NULL(test, adev->dm.boot_time_crc_info.bo_ptr);
+}
+
+/* Tests for amdgpu_dm_mode_config_init() */
+
+static void dm_test_fini_atomic_obj(void *ctx)
+{
+	drm_atomic_private_obj_fini(ctx);
+}
+
+static void dm_test_restore_audio_param(void *ctx)
+{
+	amdgpu_dm_audio_set_param((long)ctx);
+}
+
+/*
+ * A device ready for mode config init: DM creates its private object state
+ * from the current DC state, and audio is disabled so no audio component is
+ * left registered on the mock device.
+ */
+static struct amdgpu_device *dm_test_mode_config_adev(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	long saved_audio = amdgpu_dm_audio_get_param();
+
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+	adev->dm.dc->current_state = dm_kunit_alloc_dc_state(test);
+	KUNIT_ASSERT_NOT_NULL(test, adev->dm.dc->current_state);
+
+	amdgpu_dm_audio_set_param(0);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_restore_audio_param,
+							(void *)saved_audio), 0);
+
+	return adev;
+}
+
+/**
+ * dm_test_mode_config_init - Test the mode config and DM private object are set up
+ * @test: The KUnit test context
+ */
+static void dm_test_mode_config_init(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_mode_config_adev(test);
+	struct dm_atomic_state *dm_state;
+
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_mode_config_init(adev), 0);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_fini_atomic_obj,
+							&adev->dm.atomic_obj), 0);
+
+	KUNIT_EXPECT_TRUE(test, adev->mode_info.mode_config_initialized);
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.max_width, 16384);
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.max_height, 16384);
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.preferred_depth, 24);
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.prefer_shadow, 1);
+	KUNIT_EXPECT_TRUE(test, adev->ddev.mode_config.async_page_flip);
+
+	/* drm_atomic_private_obj_init() creates the state through DM. */
+	dm_state = to_dm_atomic_state(adev->dm.atomic_obj.state);
+	KUNIT_ASSERT_NOT_NULL(test, dm_state);
+	KUNIT_EXPECT_NOT_NULL(test, dm_state->context);
+}
+
+/**
+ * dm_test_mode_config_init_hawaii - Test Hawaii disables the preferred shadow
+ * @test: The KUnit test context
+ */
+static void dm_test_mode_config_init_hawaii(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_test_mode_config_adev(test);
+
+	adev->asic_type = CHIP_HAWAII;
+
+	KUNIT_ASSERT_EQ(test, amdgpu_dm_mode_config_init(adev), 0);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_test_fini_atomic_obj,
+							&adev->dm.atomic_obj), 0);
+
+	KUNIT_EXPECT_EQ(test, adev->ddev.mode_config.prefer_shadow, 0);
+}
+
+/* Tests for initialize_plane() */
+
+/**
+ * dm_test_initialize_plane_primary - Test a primary plane is stored in mode info
+ * @test: The KUnit test context
+ */
+static void dm_test_initialize_plane_primary(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->dm.adev = adev;
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+	adev->dm.dc->caps.max_streams = 1;
+
+	KUNIT_ASSERT_EQ(test, initialize_plane(&adev->dm, mode_info, 0,
+					       DRM_PLANE_TYPE_PRIMARY, NULL), 0);
+
+	KUNIT_ASSERT_NOT_NULL(test, mode_info->planes[0]);
+	KUNIT_EXPECT_EQ(test, (int)mode_info->planes[0]->type,
+			(int)DRM_PLANE_TYPE_PRIMARY);
+	KUNIT_EXPECT_EQ(test, mode_info->planes[0]->possible_crtcs, 1U);
+}
+
+/**
+ * dm_test_initialize_plane_overlay - Test an overlay plane can target any CRTC
+ * @test: The KUnit test context
+ */
+static void dm_test_initialize_plane_overlay(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_plane_cap *plane_cap;
+
+	plane_cap = kunit_kzalloc(test, sizeof(*plane_cap), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, plane_cap);
+
+	adev->family = AMDGPU_FAMILY_NV;
+	adev->dm.adev = adev;
+	adev->dm.dc = dm_kunit_alloc_dc_with_ctx(test);
+	adev->dm.dc->caps.max_streams = 1;
+	plane_cap->per_pixel_alpha = true;
+	plane_cap->pixel_format_support.nv12 = true;
+
+	/* Plane id at or above max_streams is never a primary, so any CRTC works. */
+	KUNIT_ASSERT_EQ(test, initialize_plane(&adev->dm, NULL, 1,
+					       DRM_PLANE_TYPE_OVERLAY, plane_cap), 0);
+}
+
+/* Tests for dm_gpureset_toggle_interrupts() */
+
+/**
+ * dm_test_gpureset_toggle_interrupts_dcn - Test DCN only toggles the vupdate IRQ
+ * @test: The KUnit test context
+ */
+static void dm_test_gpureset_toggle_interrupts_dcn(struct kunit *test)
+{
+	struct amdgpu_device *adev = dm_kunit_alloc_adev(test);
+	struct dc_state *state = dm_kunit_alloc_dc_state(test);
+	struct amdgpu_crtc *acrtc;
+
+	KUNIT_ASSERT_NOT_NULL(test, state);
+	acrtc = kunit_kzalloc(test, sizeof(*acrtc), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acrtc);
+
+	adev->ip_versions[DCE_HWIP][0] = IP_VERSION(3, 2, 0);
+	adev->mode_info.num_crtc = 1;
+	adev->mode_info.crtcs[0] = acrtc;
+	acrtc->base.dev = &adev->ddev;
+	acrtc->otg_inst = 0;
+	state->stream_count = 1;
+	state->stream_status[0].primary_otg_inst = 0;
+	state->stream_status[0].plane_count = 1;
+
+	/* GRPH_PFLIP is unused on DCN, so only the vupdate IRQ is touched. */
+	dm_gpureset_toggle_interrupts(adev, state, true);
+}
+
 static struct kunit_case amdgpu_dm_tests[] = {
 	/* Simple DM callbacks */
-	KUNIT_CASE(dm_test_is_idle),
 	KUNIT_CASE(dm_test_wait_for_idle),
 	KUNIT_CASE(dm_test_soft_reset),
 	KUNIT_CASE(dm_test_set_clockgating_state),
@@ -1171,6 +5234,8 @@ static struct kunit_case amdgpu_dm_tests[] = {
 	KUNIT_CASE(dm_test_plane_layer_index_cmp_equal),
 	KUNIT_CASE(dm_test_plane_layer_index_cmp_descending),
 	KUNIT_CASE(dm_test_plane_layer_index_cmp_ascending),
+	KUNIT_CASE(dm_test_update_planes_adapter_sorts_and_forwards),
+	KUNIT_CASE(dm_test_update_planes_adapter_propagates_failure),
 	/* fill_plane_color_attributes */
 	KUNIT_CASE(dm_test_fill_color_attr_rgb_format),
 	KUNIT_CASE(dm_test_fill_color_attr_bt601_full),
@@ -1217,6 +5282,187 @@ static struct kunit_case amdgpu_dm_tests[] = {
 	KUNIT_CASE(dm_test_apply_delay_null_sink),
 	KUNIT_CASE(dm_test_apply_delay_zero_wait),
 	KUNIT_CASE(dm_test_apply_delay_nonzero_wait),
+	/* dm_vblank_get_counter / dm_crtc_get_scanoutpos with a stream */
+	KUNIT_CASE(dm_test_vblank_get_counter_unmapped_stream),
+	KUNIT_CASE(dm_test_crtc_get_scanoutpos_unmapped_stream),
+	KUNIT_CASE(dm_test_crtc_get_scanoutpos_exits_idle),
+	/* dm_atomic_get_state / dm_atomic_duplicate_state */
+	KUNIT_CASE(dm_test_atomic_get_state_already_acquired),
+	KUNIT_CASE(dm_test_atomic_duplicate_state_no_context),
+	/* add_affected_mst_dsc_crtcs */
+	KUNIT_CASE(dm_test_add_affected_mst_dsc_crtcs_no_connector),
+	KUNIT_CASE(dm_test_add_affected_mst_dsc_crtcs_writeback),
+	KUNIT_CASE(dm_test_add_affected_mst_dsc_crtcs_not_mst),
+	KUNIT_CASE(dm_test_add_affected_mst_dsc_crtcs_other_crtc),
+	KUNIT_CASE(dm_test_add_affected_mst_dsc_crtcs_disabled),
+	/* amdgpu_dm_commit_cursors */
+	KUNIT_CASE(dm_test_commit_cursors_no_planes),
+	KUNIT_CASE(dm_test_commit_cursors_skips_non_cursor),
+	KUNIT_CASE(dm_test_commit_cursors_updates_cursor),
+	KUNIT_CASE(dm_test_update_cursor_no_framebuffer),
+	KUNIT_CASE(dm_test_update_cursor_disables_stream),
+	/* dm_arm_vblank_event */
+	KUNIT_CASE(dm_test_arm_vblank_event_no_event),
+	KUNIT_CASE(dm_test_arm_vblank_event_no_active_planes),
+	KUNIT_CASE(dm_test_arm_vblank_event_pflip),
+	KUNIT_CASE(dm_test_arm_vblank_event_cursor),
+	KUNIT_CASE(dm_test_arm_vblank_pre_programming_no_event),
+	KUNIT_CASE(dm_test_arm_vblank_pre_programming_no_planes),
+	KUNIT_CASE(dm_test_arm_vblank_pre_programming_update),
+	/* dm_update_pflip_irq_state */
+	KUNIT_CASE(dm_test_update_pflip_irq_state_dcn),
+	KUNIT_CASE(dm_test_update_pflip_irq_state_dce),
+	/* amdgpu_dm_crtc_mem_type_changed */
+	KUNIT_CASE(dm_test_mem_type_changed_no_planes),
+	KUNIT_CASE(dm_test_mem_type_changed_missing_state),
+	KUNIT_CASE(dm_test_mem_type_changed_same_domain),
+	KUNIT_CASE(dm_test_mem_type_changed_different_domain),
+	/* fill_dc_dirty_rects */
+	KUNIT_CASE(dm_test_dirty_rects_cursor_plane),
+	KUNIT_CASE(dm_test_dirty_rects_rotation_ffu),
+	KUNIT_CASE(dm_test_dirty_rects_no_clips_ffu),
+	KUNIT_CASE(dm_test_dirty_rects_ignored_damage_clips),
+	KUNIT_CASE(dm_test_dirty_rects_too_many_clips_ffu),
+	KUNIT_CASE(dm_test_dirty_rects_damage_clips),
+	KUNIT_CASE(dm_test_dirty_rects_mpo_bb_changed),
+	KUNIT_CASE(dm_test_dirty_rects_mpo_fb_changed),
+	KUNIT_CASE(dm_test_dirty_rects_psr_su_ffu),
+	KUNIT_CASE(dm_test_dirty_rects_mpo_clips),
+	KUNIT_CASE(dm_test_dirty_rects_mpo_overflow_ffu),
+	/* should_reset_plane */
+	KUNIT_CASE(dm_test_reset_plane_pre_dcn32_modeset),
+	KUNIT_CASE(dm_test_reset_plane_writeback_job),
+	KUNIT_CASE(dm_test_reset_plane_crtc_changed),
+	KUNIT_CASE(dm_test_reset_plane_not_in_context),
+	KUNIT_CASE(dm_test_reset_plane_no_new_crtc_state),
+	KUNIT_CASE(dm_test_reset_plane_cursor_mode_changed),
+	KUNIT_CASE(dm_test_reset_plane_color_mgmt_changed),
+	KUNIT_CASE(dm_test_reset_plane_zpos_changed),
+	KUNIT_CASE(dm_test_reset_plane_modeset),
+	KUNIT_CASE(dm_test_reset_plane_fast_update),
+	KUNIT_CASE(dm_test_reset_plane_skips_non_writeback),
+	KUNIT_CASE(dm_test_reset_plane_other_cursor_skipped),
+	KUNIT_CASE(dm_test_reset_plane_other_crtc_skipped),
+	KUNIT_CASE(dm_test_reset_plane_other_crtc_moved),
+	KUNIT_CASE(dm_test_reset_plane_other_scaling),
+	KUNIT_CASE(dm_test_reset_plane_other_rotation),
+	KUNIT_CASE(dm_test_reset_plane_other_blending),
+	KUNIT_CASE(dm_test_reset_plane_other_alpha),
+	KUNIT_CASE(dm_test_reset_plane_other_colorspace),
+	KUNIT_CASE(dm_test_reset_plane_other_hdr_mult),
+	KUNIT_CASE(dm_test_reset_plane_other_no_fb),
+	KUNIT_CASE(dm_test_reset_plane_other_format),
+	KUNIT_CASE(dm_test_reset_plane_other_modifier),
+	/* amdgpu_dm_dump_links_and_sinks */
+	KUNIT_CASE(dm_test_dump_links_no_dc),
+	KUNIT_CASE(dm_test_dump_links_no_links),
+	KUNIT_CASE(dm_test_dump_links_with_sinks),
+	KUNIT_CASE(dm_test_dump_links_unnamed_sinks),
+	/* commit-tail helpers */
+	KUNIT_CASE(dm_test_update_hdcp_no_workqueue),
+	KUNIT_CASE(dm_test_update_hdcp_writeback_skipped),
+	KUNIT_CASE(dm_test_update_hdcp_unchanged),
+	KUNIT_CASE(dm_test_atomic_setup_commit_empty),
+	KUNIT_CASE(dm_test_atomic_setup_commit_color_mgmt),
+	KUNIT_CASE(dm_test_atomic_setup_commit_modeset),
+	KUNIT_CASE(dm_test_atomic_setup_commit_bad_lut),
+	KUNIT_CASE(dm_test_atomic_check_empty),
+	KUNIT_CASE(dm_test_aquire_global_lock_no_crtc),
+	KUNIT_CASE(dm_test_aquire_global_lock_no_commit),
+	KUNIT_CASE(dm_test_aquire_global_lock_waits_commit),
+	KUNIT_CASE(dm_test_update_crtc_state_unchanged),
+	KUNIT_CASE(dm_test_update_plane_state_detached),
+	KUNIT_CASE(dm_test_mod_power_update_streams_empty),
+	KUNIT_CASE(dm_test_mod_power_update_streams_no_modeset),
+	KUNIT_CASE(dm_test_mod_power_update_streams_enable),
+	KUNIT_CASE(dm_test_mod_power_update_streams_replace),
+	KUNIT_CASE(dm_test_mod_power_update_streams_disable),
+	KUNIT_CASE(dm_test_mod_power_setup_streams_empty),
+	KUNIT_CASE(dm_test_mod_power_setup_streams_modeset),
+	KUNIT_CASE(dm_test_mod_power_setup_streams_no_modeset),
+	/* amdgpu_dm_trigger_timing_sync */
+	KUNIT_CASE(dm_test_trigger_timing_sync_no_state),
+	KUNIT_CASE(dm_test_trigger_timing_sync_streams),
+	/* dm_acpi_process_phy_transition_interlock */
+	KUNIT_CASE(dm_test_acpi_phy_transition_interlock),
+	/* IP block lifecycle helpers */
+	KUNIT_CASE(dm_test_early_fini_audio_disabled),
+	KUNIT_CASE(dm_test_sw_fini_releases_state),
+	KUNIT_CASE(dm_test_oem_i2c_hw_init_no_device),
+	KUNIT_CASE(dm_test_resume_mst_no_primary),
+	KUNIT_CASE(dm_test_s3_handle_mst_empty),
+	KUNIT_CASE(dm_test_gpureset_commit_state_no_streams),
+	KUNIT_CASE(dm_test_emulated_link_detect_bad_signal),
+	/* mmhub_read_system_context */
+	KUNIT_CASE(dm_test_mmhub_agp_disabled),
+	KUNIT_CASE(dm_test_mmhub_agp_disabled_raven2),
+	KUNIT_CASE(dm_test_mmhub_agp_enabled),
+	KUNIT_CASE(dm_test_mmhub_agp_enabled_renoir),
+	/* amdgpu_dm_init_power_module */
+	KUNIT_CASE(dm_test_init_power_module_no_edp),
+	KUNIT_CASE(dm_test_init_power_module_alloc_failure),
+	/* fill_dc_plane_info_and_addr */
+	KUNIT_CASE(dm_test_plane_info_graphics_formats),
+	KUNIT_CASE(dm_test_plane_info_video_formats),
+	KUNIT_CASE(dm_test_plane_info_unsupported_format),
+	KUNIT_CASE(dm_test_plane_info_bad_color_encoding),
+	KUNIT_CASE(dm_test_plane_info_rotations),
+	KUNIT_CASE(dm_test_plane_info_layer_and_blending),
+	/* amdgpu_dm_enable_self_refresh */
+	KUNIT_CASE(dm_test_self_refresh_full_update),
+	KUNIT_CASE(dm_test_self_refresh_unsupported_link),
+	KUNIT_CASE(dm_test_self_refresh_decrements_skip_count),
+	KUNIT_CASE(dm_test_self_refresh_allows_entry),
+	KUNIT_CASE(dm_test_self_refresh_within_settle_window),
+	KUNIT_CASE(dm_test_self_refresh_replay_link),
+	/* manage_dm_interrupts */
+	KUNIT_CASE(dm_test_manage_interrupts_offdelay),
+	KUNIT_CASE(dm_test_manage_interrupts_offdelay_fallback),
+	KUNIT_CASE(dm_test_manage_interrupts_apu_instant_off),
+	KUNIT_CASE(dm_test_manage_interrupts_disable),
+	/* dm_early_init */
+	KUNIT_CASE(dm_test_early_init_no_object_header),
+	KUNIT_CASE(dm_test_early_init_legacy_asics),
+	KUNIT_CASE(dm_test_early_init_dcn_versions),
+	KUNIT_CASE(dm_test_early_init_unsupported_version),
+	/* suspend and resume helpers */
+	KUNIT_CASE(dm_test_commit_zero_streams_empty),
+	KUNIT_CASE(dm_test_cache_state_empty_device),
+	KUNIT_CASE(dm_test_cache_state_error),
+	KUNIT_CASE(dm_test_destroy_cached_state_none),
+	KUNIT_CASE(dm_test_clear_writeback_removes_stream),
+	KUNIT_CASE(dm_test_set_writeback_no_pipe),
+	/* dm_update_mst_vcpi_slots_for_dsc */
+	KUNIT_CASE(dm_test_mst_vcpi_slots_no_connector),
+	KUNIT_CASE(dm_test_mst_vcpi_slots_skips_writeback),
+	KUNIT_CASE(dm_test_mst_vcpi_slots_skips_non_mst),
+	KUNIT_CASE(dm_test_mst_vcpi_slots_no_matching_stream),
+	/* fill_dc_plane_attributes */
+	KUNIT_CASE(dm_test_plane_attributes_success),
+	KUNIT_CASE(dm_test_plane_attributes_bad_scaling),
+	KUNIT_CASE(dm_test_plane_attributes_bad_format),
+	KUNIT_CASE(dm_test_plane_attributes_bad_color_mgmt),
+	/* load_dmcu_fw */
+	KUNIT_CASE(dm_test_load_dmcu_fw_no_dmcu),
+	KUNIT_CASE(dm_test_load_dmcu_fw_dcn),
+	KUNIT_CASE(dm_test_load_dmcu_fw_unsupported),
+	KUNIT_CASE(dm_test_load_dmcu_fw_raven),
+	KUNIT_CASE(dm_test_load_dmcu_fw_missing_firmware),
+	/* dm_sw_init / dm_sw_fini */
+	KUNIT_CASE(dm_test_sw_init_no_dmub),
+	KUNIT_CASE(dm_test_sw_fini_releases_bounding_box),
+	/* dm_late_init */
+	KUNIT_CASE(dm_test_late_init_no_dmcu),
+	KUNIT_CASE(dm_test_late_init_boot_crc_no_dmub),
+	KUNIT_CASE(dm_test_late_init_boot_crc_disabled),
+	/* amdgpu_dm_mode_config_init */
+	KUNIT_CASE(dm_test_mode_config_init),
+	KUNIT_CASE(dm_test_mode_config_init_hawaii),
+	/* initialize_plane */
+	KUNIT_CASE(dm_test_initialize_plane_primary),
+	KUNIT_CASE(dm_test_initialize_plane_overlay),
+	/* dm_gpureset_toggle_interrupts */
+	KUNIT_CASE(dm_test_gpureset_toggle_interrupts_dcn),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c
index 3454cf37141481..4bc033fdfa97a8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_wb_test.c
@@ -23,6 +23,151 @@
 #include "amdgpu_dm_wb.h"
 #include "amdgpu_dm_kunit_test_helpers.h"
 
+struct dm_wb_test_bo {
+	struct amdgpu_bo bo;
+	struct amdgpu_device *adev;
+	int reserve_ret;
+	int reserve_fences_ret;
+	int pin_ret;
+	int alloc_gart_ret;
+	u64 gpu_offset;
+	unsigned int reserve_count;
+	unsigned int reserve_fences_count;
+	unsigned int pin_count;
+	unsigned int alloc_gart_count;
+	unsigned int unreserve_count;
+	unsigned int ref_count;
+	unsigned int unpin_count;
+	unsigned int unref_count;
+	unsigned int call_seq;
+	unsigned int reserve_seq;
+	unsigned int unreserve_seq;
+	unsigned int unpin_seq;
+	unsigned int unref_seq;
+};
+
+static struct dm_wb_test_bo *to_dm_wb_test_bo(struct amdgpu_bo *bo)
+{
+	return container_of(bo, struct dm_wb_test_bo, bo);
+}
+
+static int dm_wb_test_reserve(struct amdgpu_bo *bo, bool interruptible)
+{
+	struct dm_wb_test_bo *test_bo = to_dm_wb_test_bo(bo);
+
+	test_bo->reserve_count++;
+	test_bo->reserve_seq = ++test_bo->call_seq;
+	return test_bo->reserve_ret;
+}
+
+static int dm_wb_test_reserve_fences(struct dma_resv *resv,
+				     unsigned int num_fences)
+{
+	struct dm_wb_test_bo *test_bo;
+
+	test_bo = container_of(resv, struct dm_wb_test_bo, bo.tbo.base._resv);
+	test_bo->reserve_fences_count++;
+	return test_bo->reserve_fences_ret;
+}
+
+static int dm_wb_test_pin(struct amdgpu_bo *bo, u32 domain)
+{
+	struct dm_wb_test_bo *test_bo = to_dm_wb_test_bo(bo);
+
+	test_bo->pin_count++;
+	return test_bo->pin_ret;
+}
+
+static int dm_wb_test_alloc_gart(struct ttm_buffer_object *tbo)
+{
+	struct dm_wb_test_bo *test_bo = to_dm_wb_test_bo(ttm_to_amdgpu_bo(tbo));
+
+	test_bo->alloc_gart_count++;
+	return test_bo->alloc_gart_ret;
+}
+
+static void dm_wb_test_unreserve(struct amdgpu_bo *bo)
+{
+	struct dm_wb_test_bo *test_bo = to_dm_wb_test_bo(bo);
+
+	test_bo->unreserve_count++;
+	test_bo->unreserve_seq = ++test_bo->call_seq;
+}
+
+static u64 dm_wb_test_gpu_offset(struct amdgpu_bo *bo)
+{
+	return to_dm_wb_test_bo(bo)->gpu_offset;
+}
+
+static struct amdgpu_bo *dm_wb_test_ref(struct amdgpu_bo *bo)
+{
+	to_dm_wb_test_bo(bo)->ref_count++;
+	return bo;
+}
+
+static void dm_wb_test_unpin(struct amdgpu_bo *bo)
+{
+	struct dm_wb_test_bo *test_bo = to_dm_wb_test_bo(bo);
+
+	test_bo->unpin_count++;
+	test_bo->unpin_seq = ++test_bo->call_seq;
+}
+
+static void dm_wb_test_unref(struct amdgpu_bo **bo)
+{
+	struct dm_wb_test_bo *test_bo = to_dm_wb_test_bo(*bo);
+
+	test_bo->unref_count++;
+	test_bo->unref_seq = ++test_bo->call_seq;
+	*bo = NULL;
+}
+
+static const struct amdgpu_dm_wb_kunit_ops dm_wb_test_ops = {
+	.reserve = dm_wb_test_reserve,
+	.reserve_fences = dm_wb_test_reserve_fences,
+	.pin = dm_wb_test_pin,
+	.alloc_gart = dm_wb_test_alloc_gart,
+	.unreserve = dm_wb_test_unreserve,
+	.gpu_offset = dm_wb_test_gpu_offset,
+	.ref = dm_wb_test_ref,
+	.unpin = dm_wb_test_unpin,
+	.unref = dm_wb_test_unref,
+};
+
+static void dm_wb_test_reset_ops(void *unused)
+{
+	amdgpu_dm_wb_kunit_set_ops(NULL);
+}
+
+static struct drm_writeback_job *dm_wb_test_alloc_job(struct kunit *test,
+						      struct dm_wb_test_bo **test_bo)
+{
+	struct amdgpu_framebuffer *afb;
+	struct drm_writeback_job *job;
+
+	*test_bo = kunit_kzalloc(test, sizeof(**test_bo), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, *test_bo);
+	(*test_bo)->adev = dm_kunit_alloc_adev(test);
+	KUNIT_ASSERT_NOT_NULL(test, (*test_bo)->adev);
+
+	/* Let the driver's real BO and device lookups resolve to the fake BO. */
+	(*test_bo)->bo.tbo.bdev = &(*test_bo)->adev->mman.bdev;
+	(*test_bo)->bo.tbo.base.resv = &(*test_bo)->bo.tbo.base._resv;
+
+	afb = kunit_kzalloc(test, sizeof(*afb), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, afb);
+	afb->base.obj[0] = &(*test_bo)->bo.tbo.base;
+
+	job = kunit_kzalloc(test, sizeof(*job), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, job);
+	job->fb = &afb->base;
+
+	amdgpu_dm_wb_kunit_set_ops(&dm_wb_test_ops);
+	KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dm_wb_test_reset_ops, NULL), 0);
+
+	return job;
+}
+
 
 /* Helper functions */
 
@@ -273,7 +418,9 @@ static void dm_test_wb_get_modes_returns_modes(struct kunit *test)
 	drmm_connector_init(drm, connector, &dm_wb_test_connector_funcs,
 			    DRM_MODE_CONNECTOR_VIRTUAL, NULL);
 
+	mutex_lock(&drm->mode_config.mutex);
 	count = amdgpu_dm_wb_connector_get_modes(connector);
+	mutex_unlock(&drm->mode_config.mutex);
 
 	/* drm_add_modes_noedid should return at least one mode */
 	KUNIT_EXPECT_GT(test, count, 0);
@@ -308,7 +455,9 @@ static void dm_test_wb_get_modes_bounded_by_max(struct kunit *test)
 	drmm_connector_init(drm, connector, &dm_wb_test_connector_funcs,
 			    DRM_MODE_CONNECTOR_VIRTUAL, NULL);
 
+	mutex_lock(&drm->mode_config.mutex);
 	amdgpu_dm_wb_connector_get_modes(connector);
+	mutex_unlock(&drm->mode_config.mutex);
 
 	/* All modes must fit within 3840x2160 */
 	list_for_each_entry(mode, &connector->probed_modes, head) {
@@ -387,6 +536,134 @@ static void dm_test_wb_prepare_job_no_fb(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, ret, 0);
 }
 
+/**
+ * dm_test_wb_prepare_job_success - Verify successful BO preparation
+ * @test: KUnit test context
+ *
+ * The writeback BO should be reserved, pinned, mapped into GART, referenced,
+ * and assigned its GPU address.
+ */
+static void dm_test_wb_prepare_job_success(struct kunit *test)
+{
+	struct dm_wb_test_bo *test_bo;
+	struct drm_writeback_job *job;
+	struct amdgpu_framebuffer *afb;
+	int ret;
+
+	job = dm_wb_test_alloc_job(test, &test_bo);
+	afb = to_amdgpu_framebuffer(job->fb);
+	test_bo->gpu_offset = 0x12340000;
+
+	ret = amdgpu_dm_wb_prepare_job(NULL, job);
+
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, test_bo->reserve_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->reserve_fences_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->pin_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->alloc_gart_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->unreserve_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->ref_count, 1);
+	KUNIT_EXPECT_EQ(test, afb->address, test_bo->gpu_offset);
+	KUNIT_EXPECT_TRUE(test, test_bo->bo.flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS);
+}
+
+/**
+ * dm_test_wb_prepare_job_reserve_failure - Verify reserve errors are returned
+ * @test: KUnit test context
+ *
+ * A BO reserve failure should stop preparation without attempting cleanup on
+ * a BO that was never reserved.
+ */
+static void dm_test_wb_prepare_job_reserve_failure(struct kunit *test)
+{
+	struct dm_wb_test_bo *test_bo;
+	struct drm_writeback_job *job;
+	int ret;
+
+	job = dm_wb_test_alloc_job(test, &test_bo);
+	test_bo->reserve_ret = -EBUSY;
+
+	ret = amdgpu_dm_wb_prepare_job(NULL, job);
+
+	KUNIT_EXPECT_EQ(test, ret, -EBUSY);
+	KUNIT_EXPECT_EQ(test, test_bo->reserve_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->reserve_fences_count, 0);
+	KUNIT_EXPECT_EQ(test, test_bo->unreserve_count, 0);
+}
+
+/**
+ * dm_test_wb_prepare_job_fence_failure - Verify fence reservation cleanup
+ * @test: KUnit test context
+ *
+ * Failure to reserve fence slots should release the BO reservation without
+ * attempting to pin the BO.
+ */
+static void dm_test_wb_prepare_job_fence_failure(struct kunit *test)
+{
+	struct dm_wb_test_bo *test_bo;
+	struct drm_writeback_job *job;
+	int ret;
+
+	job = dm_wb_test_alloc_job(test, &test_bo);
+	test_bo->reserve_fences_ret = -ENOMEM;
+
+	ret = amdgpu_dm_wb_prepare_job(NULL, job);
+
+	KUNIT_EXPECT_EQ(test, ret, -ENOMEM);
+	KUNIT_EXPECT_EQ(test, test_bo->reserve_fences_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->pin_count, 0);
+	KUNIT_EXPECT_EQ(test, test_bo->unreserve_count, 1);
+}
+
+/**
+ * dm_test_wb_prepare_job_pin_failure - Verify pin failure cleanup
+ * @test: KUnit test context
+ *
+ * A pin failure should release the BO reservation without trying to unpin a
+ * BO that was not successfully pinned.
+ */
+static void dm_test_wb_prepare_job_pin_failure(struct kunit *test)
+{
+	struct dm_wb_test_bo *test_bo;
+	struct drm_writeback_job *job;
+	int ret;
+
+	job = dm_wb_test_alloc_job(test, &test_bo);
+	test_bo->pin_ret = -EINVAL;
+
+	ret = amdgpu_dm_wb_prepare_job(NULL, job);
+
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+	KUNIT_EXPECT_EQ(test, test_bo->pin_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->alloc_gart_count, 0);
+	KUNIT_EXPECT_EQ(test, test_bo->unpin_count, 0);
+	KUNIT_EXPECT_EQ(test, test_bo->unreserve_count, 1);
+}
+
+/**
+ * dm_test_wb_prepare_job_gart_failure - Verify GART allocation cleanup
+ * @test: KUnit test context
+ *
+ * A GART allocation failure should unpin and unreserve the BO.
+ */
+static void dm_test_wb_prepare_job_gart_failure(struct kunit *test)
+{
+	struct dm_wb_test_bo *test_bo;
+	struct drm_writeback_job *job;
+	int ret;
+
+	job = dm_wb_test_alloc_job(test, &test_bo);
+	test_bo->alloc_gart_ret = -ENOMEM;
+
+	ret = amdgpu_dm_wb_prepare_job(NULL, job);
+
+	KUNIT_EXPECT_EQ(test, ret, -ENOMEM);
+	KUNIT_EXPECT_EQ(test, test_bo->alloc_gart_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->unpin_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->unreserve_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->ref_count, 0);
+}
+
 /**
  * dm_test_wb_cleanup_job_no_fb - Verify cleanup_job early return without a framebuffer
  * @test: KUnit test context
@@ -406,6 +683,56 @@ static void dm_test_wb_cleanup_job_no_fb(struct kunit *test)
 	amdgpu_dm_wb_cleanup_job(NULL, job);
 }
 
+/**
+ * dm_test_wb_cleanup_job_success - Verify successful BO cleanup
+ * @test: KUnit test context
+ *
+ * Cleanup should reserve, unpin, unreserve, and drop the writeback BO
+ * reference in order.
+ */
+static void dm_test_wb_cleanup_job_success(struct kunit *test)
+{
+	struct dm_wb_test_bo *test_bo;
+	struct drm_writeback_job *job;
+
+	job = dm_wb_test_alloc_job(test, &test_bo);
+
+	amdgpu_dm_wb_cleanup_job(NULL, job);
+
+	KUNIT_EXPECT_EQ(test, test_bo->reserve_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->unpin_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->unreserve_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->unref_count, 1);
+
+	/* The BO stays reserved across the unpin, and is released last. */
+	KUNIT_EXPECT_LT(test, test_bo->reserve_seq, test_bo->unpin_seq);
+	KUNIT_EXPECT_LT(test, test_bo->unpin_seq, test_bo->unreserve_seq);
+	KUNIT_EXPECT_LT(test, test_bo->unreserve_seq, test_bo->unref_seq);
+}
+
+/**
+ * dm_test_wb_cleanup_job_reserve_failure - Verify cleanup reserve failure
+ * @test: KUnit test context
+ *
+ * If cleanup cannot reserve the BO, it should leave the pin and reference
+ * untouched for a later cleanup attempt.
+ */
+static void dm_test_wb_cleanup_job_reserve_failure(struct kunit *test)
+{
+	struct dm_wb_test_bo *test_bo;
+	struct drm_writeback_job *job;
+
+	job = dm_wb_test_alloc_job(test, &test_bo);
+	test_bo->reserve_ret = -EBUSY;
+
+	amdgpu_dm_wb_cleanup_job(NULL, job);
+
+	KUNIT_EXPECT_EQ(test, test_bo->reserve_count, 1);
+	KUNIT_EXPECT_EQ(test, test_bo->unpin_count, 0);
+	KUNIT_EXPECT_EQ(test, test_bo->unreserve_count, 0);
+	KUNIT_EXPECT_EQ(test, test_bo->unref_count, 0);
+}
+
 static struct kunit_case dm_wb_test_cases[] = {
 	/* amdgpu_dm_wb_encoder_atomic_check */
 	KUNIT_CASE(dm_test_wb_atomic_check_no_job),
@@ -422,7 +749,14 @@ static struct kunit_case dm_wb_test_cases[] = {
 	KUNIT_CASE(dm_test_wb_connector_init_success),
 	/* amdgpu_dm_wb_prepare_job / amdgpu_dm_wb_cleanup_job */
 	KUNIT_CASE(dm_test_wb_prepare_job_no_fb),
+	KUNIT_CASE(dm_test_wb_prepare_job_success),
+	KUNIT_CASE(dm_test_wb_prepare_job_reserve_failure),
+	KUNIT_CASE(dm_test_wb_prepare_job_fence_failure),
+	KUNIT_CASE(dm_test_wb_prepare_job_pin_failure),
+	KUNIT_CASE(dm_test_wb_prepare_job_gart_failure),
 	KUNIT_CASE(dm_test_wb_cleanup_job_no_fb),
+	KUNIT_CASE(dm_test_wb_cleanup_job_success),
+	KUNIT_CASE(dm_test_wb_cleanup_job_reserve_failure),
 	{}
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile
index 93d02956c5eb59..6528bfd8e76b2e 100644
--- a/drivers/gpu/drm/amd/display/dc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/Makefile
@@ -22,7 +22,7 @@
 #
 # Makefile for Display Core (dc) component.
 
-DC_LIBS = basics bios dml clk_mgr dce gpio hwss irq link dsc resource optc dpp hubbub dccg hubp dio dwb hpo mmhubbub mpc opp pg
+DC_LIBS = basics bios dml clk_mgr dce gpio hwss irq link dsc resource optc dpp hubbub dccg hubp dio dwb hpo mmhubbub mpc opp pg rmcm
 
 ifdef CONFIG_DRM_AMD_DC_FP
 
@@ -37,6 +37,7 @@ DC_LIBS += dcn301
 DC_LIBS += dcn31
 DC_LIBS += dml
 DC_LIBS += dml2_0
+DC_LIBS += dml2_wrapper
 DC_LIBS += soc_and_ip_translator
 endif
 
@@ -63,9 +64,9 @@ include $(AMD_DC)
 
 FILES =
 FILES += dc_dmub_srv.o
-FILES += dc_edid_parser.o
 FILES += dc_fused_io.o
 FILES += dc_helper.o
+FILES += dc_memory_pool.o
 FILES += core/dc.o
 FILES += core/dc_debug.o
 FILES += core/dc_hw_sequencer.o
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c
index 91bc8a06e2cff4..ab67771ddab49c 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c
@@ -35,7 +35,10 @@ bool dal_bios_parser_init_cmd_tbl_helper(
 	const struct command_table_helper **h,
 	enum dce_version dce)
 {
-	switch (dce) {
+  if (!h)
+    return false;
+
+  switch (dce) {
 #if defined(CONFIG_DRM_AMD_DC_SI)
 	case DCE_VERSION_6_0:
 	case DCE_VERSION_6_1:
@@ -44,6 +47,7 @@ bool dal_bios_parser_init_cmd_tbl_helper(
 		return true;
 #endif
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 	case DCE_VERSION_8_0:
 	case DCE_VERSION_8_1:
 	case DCE_VERSION_8_3:
@@ -62,6 +66,7 @@ bool dal_bios_parser_init_cmd_tbl_helper(
 	case DCE_VERSION_11_22:
 		*h = dal_cmd_tbl_helper_dce112_get_table();
 		return true;
+#endif
 
 	default:
 		/* Unsupported DCE */
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
index 9ae5cf42754a11..ea0a270e9de305 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c
@@ -45,6 +45,7 @@ bool dal_bios_parser_init_cmd_tbl_helper2(
 		return true;
 #endif
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 	case DCE_VERSION_8_0:
 	case DCE_VERSION_8_1:
 	case DCE_VERSION_8_3:
@@ -58,6 +59,7 @@ bool dal_bios_parser_init_cmd_tbl_helper2(
 	case DCE_VERSION_11_0:
 		*h = dal_cmd_tbl_helper_dce110_get_table();
 		return true;
+#endif
 
 	case DCE_VERSION_11_2:
 	case DCE_VERSION_11_22:
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
index 8cfd7bf57fbe16..c803a2bdf74940 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/Makefile
@@ -29,6 +29,7 @@ AMD_DAL_CLK_MGR = $(addprefix $(AMDDALPATH)/dc/clk_mgr/,$(CLK_MGR))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR)
 
+ifdef CONFIG_DRM_AMD_DC_DCE
 ###############################################################################
 # DCE 100 and DCE8x
 ###############################################################################
@@ -62,6 +63,13 @@ CLK_MGR_DCE120 = dce120_clk_mgr.o
 AMD_DAL_CLK_MGR_DCE120 = $(addprefix $(AMDDALPATH)/dc/clk_mgr/dce120/,$(CLK_MGR_DCE120))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_CLK_MGR_DCE120)
+endif # CONFIG_DRM_AMD_DC_DCE
+
+# Shared clock helpers used by both the DCE and DCN clock managers. Built
+# unconditionally (not gated on CONFIG_DRM_AMD_DC_FP) so non-FP configs that
+# still build the DCE clock managers can resolve them.
+AMD_DISPLAY_FILES += $(AMDDALPATH)/dc/clk_mgr/dcn10/dcn10_clk_mgr.o
+
 ifdef CONFIG_DRM_AMD_DC_FP
 ###############################################################################
 # DCN10
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
index bf80eaf23e9aeb..7f24433a42c1cb 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c
@@ -30,10 +30,12 @@
 #include "dc_state_priv.h"
 #include "link_service.h"
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 #include "dce100/dce_clk_mgr.h"
 #include "dce110/dce110_clk_mgr.h"
 #include "dce112/dce112_clk_mgr.h"
 #include "dce120/dce120_clk_mgr.h"
+#endif
 #include "dcn10/rv1_clk_mgr.h"
 #include "dcn10/rv2_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
@@ -152,6 +154,7 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 	struct hw_asic_id asic_id = ctx->asic_id;
 
 	switch (asic_id.chip_family) {
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 	case FAMILY_SI:
 	case FAMILY_CI:
 	case FAMILY_KV: {
@@ -211,6 +214,7 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p
 			dce120_clk_mgr_construct(ctx, clk_mgr);
 		return &clk_mgr->base;
 	}
+#endif
 #if defined(CONFIG_DRM_AMD_DC_FP)
 	case FAMILY_RV: {
 		struct clk_mgr_internal *clk_mgr = kzalloc_obj(*clk_mgr);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c
index 08ed6f88025fab..3da0c220d17f94 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce112/dce112_clk_mgr.c
@@ -107,7 +107,7 @@ int dce112_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_clk_khz)
 	struct dc_bios *bp = clk_mgr->base.ctx->dc_bios;
 	struct dc *dc = clk_mgr->base.ctx->dc;
 	struct dmcu *dmcu = dc->res_pool->dmcu;
-	int actual_clock = requested_clk_khz;
+	int actual_clock;
 	/* Prepare to program display clock*/
 	memset(&dce_clk_params, 0, sizeof(dce_clk_params));
 
@@ -123,11 +123,11 @@ int dce112_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_clk_khz)
 	bp->funcs->set_dce_clock(bp, &dce_clk_params);
 	actual_clock = dce_clk_params.target_clock_frequency;
 
-	if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
-		if (clk_mgr->dfs_bypass_disp_clk != actual_clock)
-			dmcu->funcs->set_psr_wait_loop(dmcu,
-					actual_clock / 1000 / 7);
-	}
+		if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
+			if (clk_mgr->dfs_bypass_disp_clk != actual_clock)
+				dmcu->funcs->set_psr_wait_loop(dmcu,
+						actual_clock / 1000 / 7);
+		}
 
 	clk_mgr->dfs_bypass_disp_clk = actual_clock;
 	return actual_clock;
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.c
new file mode 100644
index 00000000000000..172bba20a156b6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "clk_mgr_internal.h"
+#include "core_types.h"
+#include "dal_asic_id.h"
+#include "dcn10_clk_mgr.h"
+
+unsigned int dcn10_dentist_get_divider_from_did(unsigned int did)
+{
+	if (did < DENTIST_BASE_DID_1)
+		did = DENTIST_BASE_DID_1;
+	if (did > DENTIST_MAX_DID)
+		did = DENTIST_MAX_DID;
+
+	if (did < DENTIST_BASE_DID_2) {
+		return DENTIST_DIVIDER_RANGE_1_START + DENTIST_DIVIDER_RANGE_1_STEP
+							* (did - DENTIST_BASE_DID_1);
+	} else if (did < DENTIST_BASE_DID_3) {
+		return DENTIST_DIVIDER_RANGE_2_START + DENTIST_DIVIDER_RANGE_2_STEP
+							* (did - DENTIST_BASE_DID_2);
+	} else if (did < DENTIST_BASE_DID_4) {
+		return DENTIST_DIVIDER_RANGE_3_START + DENTIST_DIVIDER_RANGE_3_STEP
+							* (did - DENTIST_BASE_DID_3);
+	} else {
+		return DENTIST_DIVIDER_RANGE_4_START + DENTIST_DIVIDER_RANGE_4_STEP
+							* (did - DENTIST_BASE_DID_4);
+	}
+}
+
+/* SW will adjust DP REF Clock average value for all purposes
+ * (DP DTO / DP Audio DTO and DP GTC)
+ if clock is spread for all cases:
+ -if SS enabled on DP Ref clock and HW de-spreading enabled with SW
+ calculations for DS_INCR/DS_MODULO (this is planned to be default case)
+ -if SS enabled on DP Ref clock and HW de-spreading enabled with HW
+ calculations (not planned to be used, but average clock should still
+ be valid)
+ -if SS enabled on DP Ref clock and HW de-spreading disabled
+ (should not be case with CIK) then SW should program all rates
+ generated according to average value (case as with previous ASICs)
+  */
+int dcn10_adjust_dp_ref_freq_for_ss(struct clk_mgr_internal *clk_mgr_dce, int dp_ref_clk_khz)
+{
+	if (clk_mgr_dce->ss_on_dprefclk && clk_mgr_dce->dprefclk_ss_divider != 0) {
+		struct fixed31_32 ss_percentage = dc_fixpt_div_int(
+				dc_fixpt_from_fraction(clk_mgr_dce->dprefclk_ss_percentage,
+							clk_mgr_dce->dprefclk_ss_divider), 200);
+		struct fixed31_32 adj_dp_ref_clk_khz;
+
+		ss_percentage = dc_fixpt_sub(dc_fixpt_one, ss_percentage);
+		adj_dp_ref_clk_khz = dc_fixpt_mul_int(ss_percentage, dp_ref_clk_khz);
+		dp_ref_clk_khz = dc_fixpt_floor(adj_dp_ref_clk_khz);
+	}
+	return dp_ref_clk_khz;
+}
+
+int dcn10_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr_dce = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+
+	return dcn10_adjust_dp_ref_freq_for_ss(clk_mgr_dce, clk_mgr_base->dprefclk_khz);
+}
+
+void dcn10_clock_read_ss_info(struct clk_mgr_internal *clk_mgr_dce)
+{
+	struct dc_bios *bp = clk_mgr_dce->base.ctx->dc_bios;
+	int ss_info_num = bp->funcs->get_ss_entry_number(
+			bp, AS_SIGNAL_TYPE_GPU_PLL);
+
+	if (ss_info_num) {
+		struct spread_spectrum_info info = { { 0 } };
+		enum bp_result result = bp->funcs->get_spread_spectrum_info(
+				bp, AS_SIGNAL_TYPE_GPU_PLL, 0, &info);
+
+		/* Based on VBIOS, VBIOS will keep entry for GPU PLL SS
+		 * even if SS not enabled and in that case
+		 * SSInfo.spreadSpectrumPercentage !=0 would be sign
+		 * that SS is enabled
+		 */
+		if (result == BP_RESULT_OK &&
+				info.spread_spectrum_percentage != 0) {
+			clk_mgr_dce->ss_on_dprefclk = true;
+			clk_mgr_dce->dprefclk_ss_divider = info.spread_percentage_divider;
+
+			if (info.type.CENTER_MODE == 0) {
+				/* TODO: Currently for DP Reference clock we
+				 * need only SS percentage for
+				 * downspread */
+				clk_mgr_dce->dprefclk_ss_percentage =
+						info.spread_spectrum_percentage;
+			}
+
+			return;
+		}
+
+		result = bp->funcs->get_spread_spectrum_info(
+				bp, AS_SIGNAL_TYPE_DISPLAY_PORT, 0, &info);
+
+		/* Based on VBIOS, VBIOS will keep entry for DPREFCLK SS
+		 * even if SS not enabled and in that case
+		 * SSInfo.spreadSpectrumPercentage !=0 would be sign
+		 * that SS is enabled
+		 */
+		if (result == BP_RESULT_OK &&
+				info.spread_spectrum_percentage != 0) {
+			clk_mgr_dce->ss_on_dprefclk = true;
+			clk_mgr_dce->dprefclk_ss_divider = info.spread_percentage_divider;
+
+			if (info.type.CENTER_MODE == 0) {
+				/* Currently for DP Reference clock we
+				 * need only SS percentage for
+				 * downspread */
+				clk_mgr_dce->dprefclk_ss_percentage =
+						info.spread_spectrum_percentage;
+			}
+			if (clk_mgr_dce->base.ctx->dc->config.ignore_dpref_ss)
+				clk_mgr_dce->dprefclk_ss_percentage = 0;
+		}
+	}
+}
+
+int dcn10_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_clk_khz)
+{
+	struct bp_set_dce_clock_parameters dce_clk_params;
+	struct dc_bios *bp = clk_mgr->base.ctx->dc_bios;
+	struct dc *dc = clk_mgr->base.ctx->dc;
+	struct dmcu *dmcu = dc->res_pool->dmcu;
+	int actual_clock;
+	/* Prepare to program display clock*/
+	memset(&dce_clk_params, 0, sizeof(dce_clk_params));
+
+	/* Make sure requested clock isn't lower than minimum threshold*/
+	if (requested_clk_khz > 0)
+		requested_clk_khz = max(requested_clk_khz,
+				clk_mgr->base.dentist_vco_freq_khz / 62);
+
+	dce_clk_params.target_clock_frequency = requested_clk_khz;
+	dce_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
+	dce_clk_params.clock_type = DCECLOCK_TYPE_DISPLAY_CLOCK;
+
+	bp->funcs->set_dce_clock(bp, &dce_clk_params);
+	actual_clock = dce_clk_params.target_clock_frequency;
+
+		if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) {
+			if (clk_mgr->dfs_bypass_disp_clk != actual_clock)
+				dmcu->funcs->set_psr_wait_loop(dmcu,
+						actual_clock / 1000 / 7);
+		}
+
+	clk_mgr->dfs_bypass_disp_clk = actual_clock;
+	return actual_clock;
+
+}
+
+int dcn10_set_dprefclk(struct clk_mgr_internal *clk_mgr)
+{
+	struct bp_set_dce_clock_parameters dce_clk_params;
+	struct dc_bios *bp = clk_mgr->base.ctx->dc_bios;
+
+	memset(&dce_clk_params, 0, sizeof(dce_clk_params));
+
+	/*Program DP ref Clock*/
+	/*VBIOS will determine DPREFCLK frequency, so we don't set it*/
+	dce_clk_params.target_clock_frequency = 0;
+	dce_clk_params.pll_id = CLOCK_SOURCE_ID_DFS;
+	dce_clk_params.clock_type = DCECLOCK_TYPE_DPREFCLK;
+	if (!((clk_mgr->base.ctx->asic_id.chip_family == FAMILY_AI) &&
+	       ASICREV_IS_VEGA20_P(clk_mgr->base.ctx->asic_id.hw_internal_rev)))
+		dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK =
+			(dce_clk_params.pll_id ==
+					CLOCK_SOURCE_COMBO_DISPLAY_PLL0);
+	else
+		dce_clk_params.flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK = false;
+
+	bp->funcs->set_dce_clock(bp, &dce_clk_params);
+
+	/* Returns the dp_refclk that was set */
+	return dce_clk_params.target_clock_frequency;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.h
new file mode 100644
index 00000000000000..99beafbf009274
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/dcn10_clk_mgr.h
@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
+
+#ifndef __DCN10_CLK_MGR_H__
+#define __DCN10_CLK_MGR_H__
+
+#include "dc.h"
+
+struct clk_mgr_internal;
+
+/*
+ * Shared clock helpers referenced by both the DCE and DCN clock managers.
+ */
+int dcn10_adjust_dp_ref_freq_for_ss(struct clk_mgr_internal *clk_mgr_dce, int dp_ref_clk_khz);
+
+void dcn10_clock_read_ss_info(struct clk_mgr_internal *clk_mgr_dce);
+
+int dcn10_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base);
+
+unsigned int dcn10_dentist_get_divider_from_did(unsigned int did);
+
+int dcn10_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_clk_khz);
+
+int dcn10_set_dprefclk(struct clk_mgr_internal *clk_mgr);
+
+#endif /* __DCN10_CLK_MGR_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
index 06a51f47aed781..9450b1733f60fd 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
@@ -26,8 +26,7 @@
 #include "core_types.h"
 #include "clk_mgr_internal.h"
 #include "rv1_clk_mgr.h"
-#include "dce100/dce_clk_mgr.h"
-#include "dce112/dce112_clk_mgr.h"
+#include "dcn10/dcn10_clk_mgr.h"
 #include "rv1_clk_mgr_vbios_smu.h"
 #include "rv1_clk_mgr_clk.h"
 
@@ -303,14 +302,14 @@ static void rv1_enable_pme_wa(struct clk_mgr *clk_mgr_base)
 
 static struct clk_mgr_funcs rv1_clk_funcs = {
 	.init_clocks = rv1_init_clocks,
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = rv1_update_clocks,
 	.enable_pme_wa = rv1_enable_pme_wa,
 };
 
 static struct clk_mgr_internal_funcs rv1_clk_internal_funcs = {
 	.set_dispclk = rv1_vbios_smu_set_dispclk,
-	.set_dprefclk = dce112_set_dprefclk
+	.set_dprefclk = dcn10_set_dprefclk
 };
 
 void rv1_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr, struct pp_smu_funcs *pp_smu)
@@ -342,7 +341,7 @@ void rv1_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_
 		if (bp->integrated_info->gpu_cap_info & DFS_BYPASS_ENABLE)
 			clk_mgr->dfs_bypass_enabled = true;
 
-	dce_clock_read_ss_info(clk_mgr);
+	dcn10_clock_read_ss_info(clk_mgr);
 }
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c
index b9ba6dbc2b46c8..3a9849b4df749a 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv2_clk_mgr.c
@@ -27,11 +27,11 @@
 #include "clk_mgr_internal.h"
 #include "rv1_clk_mgr.h"
 #include "rv2_clk_mgr.h"
-#include "dce112/dce112_clk_mgr.h"
+#include "dcn10/dcn10_clk_mgr.h"
 
 static struct clk_mgr_internal_funcs rv2_clk_internal_funcs = {
-	.set_dispclk = dce112_set_dispclk,
-	.set_dprefclk = dce112_set_dprefclk
+	.set_dispclk = dcn10_set_dispclk,
+	.set_dprefclk = dcn10_set_dprefclk
 };
 
 void rv2_clk_mgr_construct(struct dc_context *ctx, struct clk_mgr_internal *clk_mgr, struct pp_smu_funcs *pp_smu)
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
index cbd989b6a3df0c..86f3f32768e3f4 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
@@ -26,7 +26,7 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-#include "dce100/dce_clk_mgr.h"
+#include "dcn10/dcn10_clk_mgr.h"
 #include "dcn20_clk_mgr.h"
 #include "reg_helper.h"
 #include "core_types.h"
@@ -436,8 +436,8 @@ void dcn2_read_clocks_from_hw_dentist(struct clk_mgr *clk_mgr_base)
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_WDIVIDER, &dppclk_wdivider);
 
-	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
-	dpp_divider = dentist_get_divider_from_did(dppclk_wdivider);
+	disp_divider = dcn10_dentist_get_divider_from_did(dispclk_wdivider);
+	dpp_divider = dcn10_dentist_get_divider_from_did(dppclk_wdivider);
 
 	if (disp_divider && dpp_divider) {
 		/* Calculate the current DFS clock, in kHz.*/
@@ -518,7 +518,7 @@ static void dcn2_notify_link_rate_change(struct clk_mgr *clk_mgr_base, struct dc
 }
 
 static struct clk_mgr_funcs dcn2_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = dcn2_update_clocks,
 	.init_clocks = dcn2_init_clocks,
 	.enable_pme_wa = dcn2_enable_pme_wa,
@@ -558,7 +558,7 @@ void dcn20_clk_mgr_construct(
 	/* DFS Slice 2 should be used for DPREFCLK */
 	dprefclk_did = REG_READ(CLK3_CLK2_DFS_CNTL);
 	/* Convert DPREFCLK DFS Slice DID to actual divider */
-	target_div = dentist_get_divider_from_did(dprefclk_did);
+	target_div = dcn10_dentist_get_divider_from_did(dprefclk_did);
 	/* get FbMult value */
 	pll_req_reg = REG_READ(CLK3_CLK_PLL_REQ);
 
@@ -588,5 +588,5 @@ void dcn20_clk_mgr_construct(
 	//Also there is no plan for now that DFS BYPASS will be used on NV10/12/14.
 	clk_mgr->dfs_bypass_enabled = false;
 
-	dce_clock_read_ss_info(clk_mgr);
+	dcn10_clock_read_ss_info(clk_mgr);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
index 76c612ecfe3cdb..7ab548e41b406e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn201/dcn201_clk_mgr.c
@@ -28,8 +28,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 #include "dcn201_clk_mgr.h"
+#include "dcn10/dcn10_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
-#include "dce100/dce_clk_mgr.h"
 #include "dm_helpers.h"
 #include "dm_services.h"
 
@@ -170,7 +170,7 @@ static void dcn201_update_clocks(struct clk_mgr *clk_mgr_base,
 }
 
 static struct clk_mgr_funcs dcn201_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = dcn201_update_clocks,
 	.init_clocks = dcn201_init_clocks,
 	.get_clock = dcn2_get_clock,
@@ -213,5 +213,5 @@ void dcn201_clk_mgr_construct(struct dc_context *ctx,
 		if (bp->integrated_info->gpu_cap_info & DFS_BYPASS_ENABLE)
 			clk_mgr->dfs_bypass_enabled = true;
 
-	dce_clock_read_ss_info(clk_mgr);
+	dcn10_clock_read_ss_info(clk_mgr);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
index b5378344d2bc09..488ae92d24fa99 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
@@ -26,6 +26,7 @@
 #include "dccg.h"
 #include "rn_clk_mgr.h"
 
+#include "dcn10/dcn10_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
 #include "dml/dcn20/dcn20_fpu.h"
 
@@ -565,7 +566,7 @@ static void rn_notify_link_rate_change(struct clk_mgr *clk_mgr_base, struct dc_l
 }
 
 static struct clk_mgr_funcs dcn21_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = rn_update_clocks,
 	.init_clocks = rn_init_clocks,
 	.enable_pme_wa = rn_enable_pme_wa,
@@ -770,7 +771,7 @@ void rn_clk_mgr_construct(
 	rn_dump_clk_registers(&clk_mgr->base.boot_snapshot, &clk_mgr->base, &log_info);
 
 	clk_mgr->base.dprefclk_khz = 600000;
-	dce_clock_read_ss_info(clk_mgr);
+	dcn10_clock_read_ss_info(clk_mgr);
 
 
 	clk_mgr->base.bw_params = &rn_bw_params;
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
index 5641f9089b0b28..80bdaf762a2368 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
@@ -26,8 +26,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 #include "dcn30_clk_mgr_smu_msg.h"
+#include "dcn10/dcn10_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
-#include "dce100/dce_clk_mgr.h"
 #include "dcn30/dcn30_clk_mgr.h"
 #include "dml/dcn30/dcn30_fpu.h"
 #include "dcn30/dcn30m_clk_mgr.h"
@@ -493,7 +493,7 @@ static void dcn30_notify_link_rate_change(struct clk_mgr *clk_mgr_base, struct d
 }
 
 static struct clk_mgr_funcs dcn3_funcs = {
-		.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+		.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 		.update_clocks = dcn3_update_clocks,
 		.init_clocks = dcn3_init_clocks,
 		.notify_wm_ranges = dcn3_notify_wm_ranges,
@@ -517,7 +517,7 @@ static void dcn3_init_clocks_fpga(struct clk_mgr *clk_mgr)
 }
 
 struct clk_mgr_funcs dcn3_fpga_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = dcn2_update_clocks_fpga,
 	.init_clocks = dcn3_init_clocks_fpga,
 };
@@ -567,7 +567,7 @@ void dcn3_clk_mgr_construct(
 
 	clk_mgr->smu_present = false;
 
-	dce_clock_read_ss_info(clk_mgr);
+	dcn10_clock_read_ss_info(clk_mgr);
 
 	clk_mgr->base.bw_params = kzalloc_obj(*clk_mgr->base.bw_params);
 	if (!clk_mgr->base.bw_params) {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
index e40bbc495fc439..ace3ceb228c673 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
@@ -26,8 +26,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
@@ -482,7 +482,7 @@ static bool vg_are_clock_states_equal(struct dc_clocks *a,
 
 
 static struct clk_mgr_funcs vg_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = vg_update_clocks,
 	.init_clocks = vg_init_clocks,
 	.enable_pme_wa = vg_enable_pme_wa,
@@ -748,7 +748,7 @@ void vg_clk_mgr_construct(
 	vg_dump_clk_registers(&clk_mgr->base.base.boot_snapshot, &clk_mgr->base.base, &log_info);
 
 	clk_mgr->base.base.dprefclk_khz = 600000;
-	dce_clock_read_ss_info(&clk_mgr->base);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
 
 	clk_mgr->base.base.bw_params = &vg_bw_params;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
index ff47af3854b650..938bc2bbe6d71a 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
@@ -28,8 +28,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
@@ -666,7 +666,7 @@ int dcn31_get_dtb_ref_freq_khz(struct clk_mgr *clk_mgr_base)
 }
 
 static struct clk_mgr_funcs dcn31_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 	.update_clocks = dcn31_update_clocks,
 	.init_clocks = dcn31_init_clocks,
@@ -744,9 +744,9 @@ void dcn31_clk_mgr_construct(
 
 	clk_mgr->base.base.dprefclk_khz = 600000;
 	clk_mgr->base.base.clks.ref_dtbclk_khz = 600000;
-	dce_clock_read_ss_info(&clk_mgr->base);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
 	/*if bios enabled SS, driver needs to adjust dtb clock, only enable with correct bios*/
-	//clk_mgr->base.dccg->ref_dtbclk_khz = dce_adjust_dp_ref_freq_for_ss(clk_mgr_internal, clk_mgr->base.base.dprefclk_khz);
+	//clk_mgr->base.dccg->ref_dtbclk_khz = dcn10_adjust_dp_ref_freq_for_ss(clk_mgr_internal, clk_mgr->base.base.dprefclk_khz);
 
 	clk_mgr->base.base.bw_params = &dcn31_bw_params;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
index 24f6304011aece..2aa3e8e66cb762 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
@@ -31,8 +31,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
@@ -200,7 +200,7 @@ void dcn314_init_clocks(struct clk_mgr *clk_mgr)
 	// to adjust dp_dto reference clock if ssc is enable otherwise to apply dprefclk
 	if (dcn314_is_spll_ssc_enabled(clk_mgr))
 		clk_mgr->dp_dto_source_clock_in_khz =
-			dce_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr->dprefclk_khz);
+			dcn10_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr->dprefclk_khz);
 	else
 		clk_mgr->dp_dto_source_clock_in_khz = clk_mgr->dprefclk_khz;
 }
@@ -766,7 +766,7 @@ static void dcn314_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *cl
 }
 
 static struct clk_mgr_funcs dcn314_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 	.update_clocks = dcn314_update_clocks,
 	.init_clocks = dcn314_init_clocks,
@@ -861,7 +861,7 @@ void dcn314_clk_mgr_construct(
 
 	clk_mgr->base.base.dprefclk_khz = 600000;
 	clk_mgr->base.base.clks.ref_dtbclk_khz = 600000;
-	dce_clock_read_ss_info(&clk_mgr->base);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
 	dcn314_read_ss_info_from_lut(&clk_mgr->base);
 	/*if bios enabled SS, driver needs to adjust dtb clock, only enable with correct bios*/
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
index 75d39cb26dbabf..8ca367b7232530 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
@@ -28,8 +28,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
 #include "dcn31/dcn31_clk_mgr.h"
@@ -600,7 +600,7 @@ static void dcn315_enable_pme_wa(struct clk_mgr *clk_mgr_base)
 }
 
 static struct clk_mgr_funcs dcn315_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 	.update_clocks = dcn315_update_clocks,
 	.init_clocks = dcn31_init_clocks,
@@ -673,16 +673,16 @@ void dcn315_clk_mgr_construct(
 				  &clk_mgr->base.base, &log_info);
 
 	clk_mgr->base.base.dprefclk_khz = 600000;
-	clk_mgr->base.base.dprefclk_khz = dcn315_smu_get_dpref_clk(&clk_mgr->base);
 	clk_mgr->base.base.clks.ref_dtbclk_khz = clk_mgr->base.base.dprefclk_khz;
-	dce_clock_read_ss_info(&clk_mgr->base);
-	clk_mgr->base.base.clks.ref_dtbclk_khz = dce_adjust_dp_ref_freq_for_ss(&clk_mgr->base, clk_mgr->base.base.dprefclk_khz);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
+	clk_mgr->base.base.clks.ref_dtbclk_khz = dcn10_adjust_dp_ref_freq_for_ss(&clk_mgr->base, clk_mgr->base.base.dprefclk_khz);
 
 	clk_mgr->base.base.bw_params = &dcn315_bw_params;
 
-	if (clk_mgr->base.base.ctx->dc->debug.pstate_enabled) {
+	if (clk_mgr->base.base.ctx->dc->debug.pstate_enabled && clk_mgr->base.smu_present) {
 		int i;
 
+		clk_mgr->base.base.dprefclk_khz = dcn315_smu_get_dpref_clk(&clk_mgr->base);
 		dcn315_get_dpm_table_from_smu(&clk_mgr->base, &smu_dpm_clks);
 		DC_LOG_SMU("NumDcfClkLevelsEnabled: %d\n"
 				   "NumDispClkLevelsEnabled: %d\n"
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
index c7fecbdfda2c28..96bd653a9b71e9 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
@@ -28,8 +28,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
 #include "dcn31/dcn31_clk_mgr.h"
@@ -576,7 +576,7 @@ static void dcn316_clk_mgr_helper_populate_bw_params(
 
 static struct clk_mgr_funcs dcn316_funcs = {
 	.enable_pme_wa = dcn316_enable_pme_wa,
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 	.update_clocks = dcn316_update_clocks,
 	.init_clocks = dcn31_init_clocks,
@@ -659,9 +659,9 @@ void dcn316_clk_mgr_construct(
 	clk_mgr->base.base.dprefclk_khz = 600000;
 	clk_mgr->base.base.dprefclk_khz = dcn316_smu_get_dpref_clk(&clk_mgr->base);
 	clk_mgr->base.base.clks.ref_dtbclk_khz = clk_mgr->base.base.dprefclk_khz;
-	dce_clock_read_ss_info(&clk_mgr->base);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
 	/*clk_mgr->base.dccg->ref_dtbclk_khz =
-	dce_adjust_dp_ref_freq_for_ss(&clk_mgr->base, clk_mgr->base.base.dprefclk_khz);*/
+	dcn10_adjust_dp_ref_freq_for_ss(&clk_mgr->base, clk_mgr->base.base.dprefclk_khz);*/
 
 	clk_mgr->base.base.bw_params = &dcn316_bw_params;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
index f9886a0eca92d3..22dfca6efba667 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
@@ -26,8 +26,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 #include "dcn32/dcn32_clk_mgr_smu_msg.h"
+#include "dcn10/dcn10_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
-#include "dce100/dce_clk_mgr.h"
 #include "dcn31/dcn31_clk_mgr.h"
 #include "dcn32/dcn32_clk_mgr.h"
 #include "reg_helper.h"
@@ -409,7 +409,7 @@ static void dcn32_update_clocks_update_dentist(
 		}
 	} else if (new_dispclk_wdivider == 127 && old_dispclk_wdivider != 127) {
 		/* request clock with 126 divider first */
-		uint32_t temp_disp_divider = dentist_get_divider_from_did(126);
+		uint32_t temp_disp_divider = dcn10_dentist_get_divider_from_did(126);
 		uint32_t temp_dispclk_khz = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR * clk_mgr->base.dentist_vco_freq_khz) / temp_disp_divider;
 
 		if (clk_mgr->smu_present)
@@ -490,7 +490,7 @@ static int dcn32_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
 	unsigned int disp_divider;
 
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
-	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
+	disp_divider = dcn10_dentist_get_divider_from_did(dispclk_wdivider);
 
 	/* Return DISPCLK freq in Khz */
 	if (disp_divider)
@@ -922,31 +922,31 @@ static void dcn32_dump_clk_registers(struct clk_state_registers_and_bypass *regs
 	}
 
 	/* Convert DISPCLK DFS Slice DID to divider*/
-	target_div = dentist_get_divider_from_did(dispclk_did);
+	target_div = dcn10_dentist_get_divider_from_did(dispclk_did);
 	//Get dispclk in khz
 	regs_and_bypass->dispclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 			* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 	/* Convert DISPCLK DFS Slice DID to divider*/
-	target_div = dentist_get_divider_from_did(dppclk_did);
+	target_div = dcn10_dentist_get_divider_from_did(dppclk_did);
 	//Get dppclk in khz
 	regs_and_bypass->dppclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 			* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 	/* Convert DPREFCLK DFS Slice DID to divider*/
-	target_div = dentist_get_divider_from_did(dprefclk_did);
+	target_div = dcn10_dentist_get_divider_from_did(dprefclk_did);
 	//Get dprefclk in khz
 	regs_and_bypass->dprefclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 			* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 	/* Convert DCFCLK DFS Slice DID to divider*/
-	target_div = dentist_get_divider_from_did(dcfclk_did);
+	target_div = dcn10_dentist_get_divider_from_did(dcfclk_did);
 	//Get dcfclk in khz
 	regs_and_bypass->dcfclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 			* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 	/* Convert DTBCLK DFS Slice DID to divider*/
-	target_div = dentist_get_divider_from_did(dtbclk_did);
+	target_div = dcn10_dentist_get_divider_from_did(dtbclk_did);
 	//Get dtbclk in khz
 	regs_and_bypass->dtbclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 			* clk_mgr->base.dentist_vco_freq_khz) / target_div;
@@ -1139,7 +1139,7 @@ static void dcn32_set_min_memclk(struct clk_mgr *clk_mgr_base, unsigned int memc
 }
 
 static struct clk_mgr_funcs dcn32_funcs = {
-		.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+		.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 		.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 		.update_clocks = dcn32_update_clocks,
 		.dump_clk_registers = dcn32_dump_clk_registers,
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
index a69824e1eb26e1..990c362623d8fa 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
@@ -29,8 +29,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
@@ -758,7 +758,7 @@ void dcn35_init_clocks(struct clk_mgr *clk_mgr)
 	// to adjust dp_dto reference clock if ssc is enable otherwise to apply dprefclk
 	if (dcn35_is_spll_ssc_enabled(clk_mgr))
 		clk_mgr->dp_dto_source_clock_in_khz =
-			dce_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr->dprefclk_khz);
+			dcn10_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr->dprefclk_khz);
 	else
 		clk_mgr->dp_dto_source_clock_in_khz = clk_mgr->dprefclk_khz;
 
@@ -1346,7 +1346,7 @@ static unsigned int dcn35_get_max_clock_khz(struct clk_mgr *clk_mgr_base, enum c
 }
 
 static struct clk_mgr_funcs dcn35_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 	.update_clocks = dcn35_update_clocks,
 	.init_clocks = dcn35_init_clocks,
@@ -1360,7 +1360,7 @@ static struct clk_mgr_funcs dcn35_funcs = {
 };
 
 struct clk_mgr_funcs dcn35_fpga_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = dcn35_update_clocks_fpga,
 	.init_clocks = dcn35_init_clocks_fpga,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
@@ -1499,7 +1499,7 @@ void dcn35_clk_mgr_construct(
 	clk_mgr->base.base.dprefclk_khz = dcn35_smu_get_dprefclk(&clk_mgr->base);
 	clk_mgr->base.base.clks.ref_dtbclk_khz = 600000;
 
-	dce_clock_read_ss_info(&clk_mgr->base);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
 	/*when clk src is from FCH, it could have ss, same clock src as DPREF clk*/
 
 	dcn35_read_ss_info_from_lut(&clk_mgr->base);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
index 6ee3da89d05825..120f9dd691f34d 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
@@ -5,8 +5,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 #include "dcn401/dcn401_clk_mgr_smu_msg.h"
+#include "dcn10/dcn10_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
-#include "dce100/dce_clk_mgr.h"
 #include "dcn31/dcn31_clk_mgr.h"
 #include "dcn32/dcn32_clk_mgr.h"
 #include "dcn401/dcn401_clk_mgr.h"
@@ -358,37 +358,37 @@ static void dcn401_dump_clk_registers(struct clk_state_registers_and_bypass *reg
 		fclk_did = REG_READ(CLK2_CLK2_DFS_CNTL);
 
 		/* Convert DISPCLK DFS Slice DID to divider*/
-		target_div = dentist_get_divider_from_did(dispclk_did);
+		target_div = dcn10_dentist_get_divider_from_did(dispclk_did);
 		//Get dispclk in khz
 		regs_and_bypass->dispclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 				* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 		/* Convert DISPCLK DFS Slice DID to divider*/
-		target_div = dentist_get_divider_from_did(dppclk_did);
+		target_div = dcn10_dentist_get_divider_from_did(dppclk_did);
 		//Get dppclk in khz
 		regs_and_bypass->dppclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 				* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 		/* Convert DPREFCLK DFS Slice DID to divider*/
-		target_div = dentist_get_divider_from_did(dprefclk_did);
+		target_div = dcn10_dentist_get_divider_from_did(dprefclk_did);
 		//Get dprefclk in khz
 		regs_and_bypass->dprefclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 				* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 		/* Convert DCFCLK DFS Slice DID to divider*/
-		target_div = dentist_get_divider_from_did(dcfclk_did);
+		target_div = dcn10_dentist_get_divider_from_did(dcfclk_did);
 		//Get dcfclk in khz
 		regs_and_bypass->dcfclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 				* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 		/* Convert DTBCLK DFS Slice DID to divider*/
-		target_div = dentist_get_divider_from_did(dtbclk_did);
+		target_div = dcn10_dentist_get_divider_from_did(dtbclk_did);
 		//Get dtbclk in khz
 		regs_and_bypass->dtbclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 				* clk_mgr->base.dentist_vco_freq_khz) / target_div;
 
 		/* Convert DTBCLK DFS Slice DID to divider*/
-		target_div = dentist_get_divider_from_did(fclk_did);
+		target_div = dcn10_dentist_get_divider_from_did(fclk_did);
 		//Get fclk in khz
 		regs_and_bypass->fclk = (DENTIST_DIVIDER_RANGE_SCALE_FACTOR
 				* clk_mgr->base.dentist_vco_freq_khz) / target_div;
@@ -1552,7 +1552,7 @@ static int dcn401_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
 	unsigned int disp_divider;
 
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
-	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
+	disp_divider = dcn10_dentist_get_divider_from_did(dispclk_wdivider);
 
 	/* Return DISPCLK freq in Khz */
 	if (disp_divider)
@@ -1599,7 +1599,7 @@ static void dcn401_execute_clk_mgr_block_sequence_bls(struct clk_mgr *clk_mgr_ba
 }
 
 static struct clk_mgr_funcs dcn401_funcs = {
-		.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+		.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 		.get_dtb_ref_clk_frequency = dcn401_get_dtb_ref_freq_khz,
 		.update_clocks = dcn401_update_clocks,
 		.dump_clk_registers = dcn401_dump_clk_registers,
@@ -1632,6 +1632,7 @@ struct clk_mgr_internal *dcn401_clk_mgr_construct(
 
 	clk_mgr = &clk_mgr401->base;
 	clk_mgr->base.ctx = ctx;
+	clk_mgr->base.inst = 0;
 	clk_mgr->base.funcs = &dcn401_funcs;
 	clk_mgr->regs = &clk_mgr_regs_dcn401;
 	clk_mgr->clk_mgr_shift = &clk_mgr_shift_dcn401;
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
index 19ef0b92e8e604..405145889ba27b 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
@@ -7,8 +7,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
@@ -214,6 +214,7 @@ void dcn42_update_clocks(struct clk_mgr *clk_mgr_base,
 {
 	union dmub_rb_cmd cmd;
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	struct clk_mgr_dcn42 *clk_mgr_dcn42 = TO_CLK_MGR_DCN42(clk_mgr);
 	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
 	struct dc *dc = clk_mgr_base->ctx->dc;
 	bool update_dppclk = false;
@@ -364,6 +365,16 @@ void dcn42_update_clocks(struct clk_mgr *clk_mgr_base,
 		clk_mgr_base->clks.dcfclk_deep_sleep_khz;
 	cmd.notify_clocks.clocks.dispclk_khz = clk_mgr_base->clks.dispclk_khz;
 	cmd.notify_clocks.clocks.dppclk_khz = clk_mgr_base->clks.dppclk_khz;
+	/* Communicate the minimum (DPM0) clocks and bypass ceiling so DMUB does not
+	 * assume hardcoded per-ASIC values (e.g. for Z8-Retention clock lowering).
+	 * DPM0 is the lowest populated DPM level (entries[0]); 0 when the SMU DPM
+	 * table is unavailable, in which case DMUB falls back to its own defaults.
+	 */
+	cmd.notify_clocks.clocks.dpm0_dispclk_khz =
+		clk_mgr_base->bw_params->clk_table.entries[0].dispclk_mhz * 1000;
+	cmd.notify_clocks.clocks.dpm0_dppclk_khz =
+		clk_mgr_base->bw_params->clk_table.entries[0].dppclk_mhz * 1000;
+	cmd.notify_clocks.clocks.max_bypass_clk_khz = clk_mgr_dcn42->max_bypass_clk_khz;
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 }
@@ -381,7 +392,7 @@ void dcn42_notify_cstate_disable(struct clk_mgr *clk_mgr_base, bool disable)
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	bool target_allow = !disable;
 
-	DC_LOGGER_INIT(clk_mgr_base->ctx->logger);
+DC_LOGGER_INIT(clk_mgr_base->ctx->logger);
 
 	/* Idempotent: only send when the cached vote actually changes. */
 	if (clk_mgr_base->clks.cstate_allow == target_allow)
@@ -608,7 +619,7 @@ void dcn42_init_clocks(struct clk_mgr *clk_mgr_base)
 	struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	struct clk_mgr_dcn42 *clk_mgr = TO_CLK_MGR_DCN42(clk_mgr_int);
 
-	DC_LOGGER_INIT(clk_mgr_base->ctx->logger);
+DC_LOGGER_INIT(clk_mgr_base->ctx->logger);
 	(void)dc_logger;
 
 	init_clk_states(clk_mgr_base);
@@ -625,7 +636,7 @@ void dcn42_init_clocks(struct clk_mgr *clk_mgr_base)
 	// to adjust dp_dto reference clock if ssc is enable otherwise to apply dprefclk
 	if (dcn42_is_spll_ssc_enabled(clk_mgr_base))
 		clk_mgr_base->dp_dto_source_clock_in_khz =
-			dce_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr_base->dprefclk_khz);
+			dcn10_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr_base->dprefclk_khz);
 	else
 		clk_mgr_base->dp_dto_source_clock_in_khz = clk_mgr_base->dprefclk_khz;
 
@@ -803,7 +814,7 @@ void dcn42_set_low_power_state(struct clk_mgr *clk_mgr_base)
 
 void dcn42_exit_low_power_state(struct clk_mgr *clk_mgr_base)
 {
-	(void)clk_mgr_base;
+(void)clk_mgr_base;
 
 }
 
@@ -912,7 +923,7 @@ int dcn42_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
 	unsigned int disp_divider;
 
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
-	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
+	disp_divider = dcn10_dentist_get_divider_from_did(dispclk_wdivider);
 
 	/* Return DISPCLK freq in Khz */
 	if (disp_divider)
@@ -1074,7 +1085,7 @@ void dcn42_request_dtbclk(struct clk_mgr *clk_mgr_base, bool enable)
 	}
 }
 static struct clk_mgr_funcs dcn42_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
 	.update_clocks = dcn42_update_clocks,
 	.init_clocks = dcn42_init_clocks,
@@ -1091,7 +1102,7 @@ static struct clk_mgr_funcs dcn42_funcs = {
 };
 
 struct clk_mgr_funcs dcn42_fpga_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.update_clocks = dcn42_update_clocks_fpga,
 	.init_clocks = dcn42_init_clocks_fpga,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
@@ -1104,6 +1115,7 @@ void dcn42_clk_mgr_construct(
 		struct dccg *dccg)
 {
 	clk_mgr->base.base.ctx = ctx;
+	clk_mgr->base.base.inst = 0;
 	clk_mgr->base.base.funcs = &dcn42_funcs;
 	clk_mgr->base.regs = &clk_mgr_regs_dcn42;
 	clk_mgr->base.clk_mgr_shift = &clk_mgr_shift_dcn42;
@@ -1113,6 +1125,7 @@ void dcn42_clk_mgr_construct(
 
 	clk_mgr->base.dccg = dccg;
 	clk_mgr->base.dfs_bypass_disp_clk = 0;
+	clk_mgr->max_bypass_clk_khz = 0;
 
 	clk_mgr->base.dprefclk_ss_percentage = 0;
 	clk_mgr->base.dprefclk_ss_divider = 1000;
@@ -1153,7 +1166,7 @@ void dcn42_clk_mgr_construct(
 		/* Saved clocks configured at boot for debug purposes */
 		dcn42_dump_clk_registers(&clk_mgr->base.base.boot_snapshot, clk_mgr);
 
-	dce_clock_read_ss_info(&clk_mgr->base);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
 	/*when clk src is from FCH, it could have ss, same clock src as DPREF clk*/
 
 	dcn42_read_ss_info_from_lut(&clk_mgr->base);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h
index f90c6e2e85c2fc..f64c1ccee8c3c5 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.h
@@ -44,6 +44,11 @@ struct dcn42_ss_info_table {
 struct clk_mgr_dcn42 {
 	struct clk_mgr_internal base;
 	struct dcn42_smu_watermark_set smu_wm_set;
+	/* Max dispclk/dppclk (kHz) that can run on a bypass source with the PLL
+	 * powered down. Communicated to DMUB for Z8-Retention clock lowering.
+	 * SMU does not expose this, so it is a fixed per-ASIC value set at construct.
+	 */
+	uint32_t max_bypass_clk_khz;
 };
 
 bool dcn42_are_clock_states_equal(struct dc_clocks *a,
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c
index 2c2ba09221e4d8..d500a3210eed78 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42b/dcn42b_clk_mgr.c
@@ -9,8 +9,8 @@
 #include "dccg.h"
 #include "clk_mgr_internal.h"
 
-// For dce12_get_dp_ref_freq_khz
-#include "dce100/dce_clk_mgr.h"
+// For dcn10_get_dp_ref_freq_khz
+#include "dcn10/dcn10_clk_mgr.h"
 
 // For dcn20_update_clocks_update_dpp_dto
 #include "dcn20/dcn20_clk_mgr.h"
@@ -35,6 +35,8 @@
 #include "dcn/dcn_4_2_1_offset.h"
 #include "dcn/dcn_4_2_1_sh_mask.h"
 
+#include <linux/power_supply.h>
+
 #define DCN_BASE__INST0_SEG0                       0x00000012
 #define DCN_BASE__INST0_SEG1                       0x000000C0
 
@@ -226,7 +228,7 @@ void dcn42b_init_clocks(struct clk_mgr *clk_mgr_base)
 	struct clk_mgr_internal *clk_mgr_int = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	struct clk_mgr_dcn42 *clk_mgr = TO_CLK_MGR_DCN42B(clk_mgr_int);
 
-	DC_LOGGER_INIT(clk_mgr_base->ctx->logger);
+DC_LOGGER_INIT(clk_mgr_base->ctx->logger);
 	(void)dc_logger;
 
 	init_clk_states(clk_mgr_base);
@@ -234,7 +236,7 @@ void dcn42b_init_clocks(struct clk_mgr *clk_mgr_base)
 	// to adjust dp_dto reference clock if ssc is enable otherwise to apply dprefclk
 	if (dcn42_is_spll_ssc_enabled(clk_mgr_base))
 		clk_mgr_base->dp_dto_source_clock_in_khz =
-			dce_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr_base->dprefclk_khz);
+			dcn10_adjust_dp_ref_freq_for_ss(clk_mgr_int, clk_mgr_base->dprefclk_khz);
 	else
 		clk_mgr_base->dp_dto_source_clock_in_khz = clk_mgr_base->dprefclk_khz;
 
@@ -389,14 +391,46 @@ uint32_t dcn42b_get_clock_freq_from_clkip(struct clk_mgr *clk_mgr_base, enum clo
 	return (uint32_t)clock_freq_mhz;
 }
 
-/* dcn42b_get_dispclk_from_dentist removed: reuse dcn42_get_dispclk_from_dentist.
- * DENTIST_DISPCLK_CNTL is a DCN register with the same offset on both dcn42 and dcn42b.
+#define DCN42B_MIN_DCFCLK_MHZ 200
+#define DCN42B_MAX_DCFCLK_MHZ 600
+#define DCN42B_AC_MIN_DCFCLK_MHZ 300
+
+/*
+ * dcn42b_update_clocks - DCN42B wrapper around dcn42_update_clocks.
+ *
+ * Establishes the DCN42B force-min-DCFCLK floor consumed by the shared
+ * dcn42_update_clocks(), which applies it to new_clocks->dcfclk_khz:
+ *   - a user override (dc->debug.force_min_dcfclk_mhz)
+ *     wins, sanitized to the [200, 600] MHz supported range;
+ *   - otherwise DCFCLK is floored at 300 MHz on wall power, and left
+ *     unconstrained on battery.
+ * dc->debug is restored afterwards so the override survives AC/DC transitions.
  */
+static void dcn42b_update_clocks(struct clk_mgr *clk_mgr_base,
+			struct dc_state *context,
+			bool safe_to_lower)
+{
+	struct dc *dc = clk_mgr_base->ctx->dc;
+	unsigned int user_force_min_dcfclk_mhz = dc->debug.force_min_dcfclk_mhz;
+
+	if (dc->debug.force_min_dcfclk_mhz > 0) {
+		if (dc->debug.force_min_dcfclk_mhz < DCN42B_MIN_DCFCLK_MHZ)
+			dc->debug.force_min_dcfclk_mhz = DCN42B_MIN_DCFCLK_MHZ;
+		else if (dc->debug.force_min_dcfclk_mhz > DCN42B_MAX_DCFCLK_MHZ)
+			dc->debug.force_min_dcfclk_mhz = DCN42B_MAX_DCFCLK_MHZ;
+	} else if (power_supply_is_system_supplied()) {
+		dc->debug.force_min_dcfclk_mhz = DCN42B_AC_MIN_DCFCLK_MHZ;
+	}
+
+	dcn42_update_clocks(clk_mgr_base, context, safe_to_lower);
+
+	dc->debug.force_min_dcfclk_mhz = user_force_min_dcfclk_mhz;
+}
 
 static struct clk_mgr_funcs dcn42b_funcs = {
-	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+	.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 	.get_dtb_ref_clk_frequency = dcn31_get_dtb_ref_freq_khz,
-	.update_clocks = dcn42_update_clocks,
+	.update_clocks = dcn42b_update_clocks,
 	.init_clocks = dcn42b_init_clocks,
 	.enable_pme_wa = dcn42_enable_pme_wa,
 	.are_clock_states_equal = dcn42_are_clock_states_equal,
@@ -585,6 +619,7 @@ void dcn42b_clk_mgr_construct(
 		struct dccg *dccg)
 {
 	clk_mgr->base.base.ctx = ctx;
+	clk_mgr->base.base.inst = 0;
 	clk_mgr->base.base.funcs = &dcn42b_funcs;
 	clk_mgr->base.regs = &clk_mgr_regs_dcn42b;
 	clk_mgr->base.clk_mgr_shift = &clk_mgr_shift_dcn42b;
@@ -594,6 +629,7 @@ void dcn42b_clk_mgr_construct(
 
 	clk_mgr->base.dccg = dccg;
 	clk_mgr->base.dfs_bypass_disp_clk = 0;
+	clk_mgr->max_bypass_clk_khz = 0;
 
 	clk_mgr->base.dprefclk_ss_percentage = 0;
 	clk_mgr->base.dprefclk_ss_divider = 1000;
@@ -633,7 +669,7 @@ void dcn42b_clk_mgr_construct(
 		/* Saved clocks configured at boot for debug purposes */
 		dcn42b_dump_clk_registers(&clk_mgr->base.base.boot_snapshot, clk_mgr);
 
-	dce_clock_read_ss_info(&clk_mgr->base);
+	dcn10_clock_read_ss_info(&clk_mgr->base);
 	/*when clk src is from FCH, it could have ss, same clock src as DPREF clk*/
 
 	dcn42b_read_ss_info_from_lut(&clk_mgr->base);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dalsmc.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dalsmc.h
index e10621b5eeff57..234777bdd1831a 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dalsmc.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dalsmc.h
@@ -6,287 +6,30 @@
 #define DALSMC_H
 
 /**
- * @file dalsmc.h
- *
- * @brief VBIOS and DAL to PMFW Interface
- *
- * Clients:  VBIOS and DAL
- * Protocols: dalsmc
- *
- * @date 2016 - 2026
- */
+* @file dalsmc.h
+*
+* @brief VBIOS and DAL to PMFW Interface
+*
+* Clients:  VBIOS and DAL
+* Protocols:
+*
+* @date 2016 - 2026
+*/
 
 /**
  * @mainpage PMFW-DAL Message Interface
+ * This documentation contains the subsections:
  *
- * The protocol uses six registers:
- *
- * - MSG_REG   — write the message ID (DALSMC_MSG_*) to trigger the transaction
- * - ARG_REG_0 — input argument Reg0; also carries response data on completion
- * - ARG_REG_1 — input argument Reg1
- * - ARG_REG_2 — input argument Reg2
- * - ARG_REG_3 — input argument Reg3
- * - RESP_REG  — poll until non-zero; value is a DALSMC_Result_* response code
- *
- * Programming sequence:
- * 1. Clear RESP_REG to 0
- * 2. Write input arguments to ARG_REG_0..3
- * 3. Write message ID to MSG_REG (triggers PMFW interrupt)
- * 4. Poll RESP_REG until non-zero — value is the result code
- * 5. Read response data from ARG_REG_0 (message-specific)
- *
- * For payloads too large for the four argument registers, the protocol supports
- * DRAM table transfers where DAL allocates a DRAM buffer and exchanges bulk data
- * with PMFW through system memory.
- *
- * This documentation contains the subsections:\n\n
- * @ref ResponseCodes\n
- * @ref Messages\n
- * @ref DramTables\n
- */
-
-#define DALSMC_VERSION  0x1
-
-/** @defgroup ResponseCodes PMFW Response Codes
- *  @{
- */
-// SMU Response Codes:
-#define DALSMC_Result_OK                    0x01
-#define DALSMC_Result_Failed                0xFF
-#define DALSMC_Result_UnknownCmd            0xFE
-#define DALSMC_Result_CmdRejectedPrereq     0xFD
-#define DALSMC_Result_CmdRejectedBusy       0xFC
-/** @} */
-
-/** @defgroup Messages Message definitions
- *  @{
+ * @ref ResponseCodes
+ * @ref definitions
  */
 
-/** Generic register overlay — four 32-bit C2PMSG argument registers. */
-typedef struct {
-	uint32_t Reg0;
-	uint32_t Reg1;
-	uint32_t Reg2;
-	uint32_t Reg3;
-} DALSMC_args_t;
+/** @def DAL_PMFW_IF_VER
+* VBIOS and DAL to PMFW IF version 1.0
+*/
+#define DAL_PMFW_IF_VER "1.0" ///< Major.Minor
 
-/**
- * DALSMC_MSG_TestMessage - Test interface connectivity.
- *
- * Echos back the argument value incremented by 1. Use to verify the mailbox
- * is functional before sending real messages.
- *
- * Request:  TestValue — arbitrary test integer
- * Response: Reg0      — TestValue + 1
- */
-#define DALSMC_MSG_TestMessage                  0x01
-typedef union {
-	struct {
-		uint32_t TestValue;
-		uint32_t Reserved[3];
-	};
-	DALSMC_args_t Args;
-} DALSMC_TestMessage_arg_t;
-
-/**
- * DALSMC_MSG_GetMsgHeaderVersion - Query the DALSMC header version running on PMFW.
- *
- * DAL uses the returned version to determine which messages are supported in
- * environments that require backwards compatibility.
- *
- * Request:  (none)
- * Response: Reg0 — DALSMC_VERSION value compiled into PMFW
- */
-#define DALSMC_MSG_GetMsgHeaderVersion          0x02
-
-/**
- * DALSMC_MSG_TransferTableSmu2Dram - Transfer a PMFW table into DRAM.
- * DALSMC_MSG_TransferTableDram2Smu - Transfer a DRAM buffer into PMFW.
- *
- * Both directions use the same argument layout. The DRAM address must be set
- * beforehand (AddrLow / AddrHigh are the GPU MC address bits [31:0] / [63:32]).
- *
- * Smu2Dram supported tables: TABLE_DAL_INIT (DPM clocks + UTM QoS + memory config)
- * Dram2Smu supported tables: TABLE_SOC_UTM  (debug override of UTM QoS parameters)
- *
- * Request:  TableId  — table identifier (TABLE_* defines below)
- *           AddrLow  — GPU MC address bits [31:0]  of destination/source buffer
- *           AddrHigh — GPU MC address bits [63:32] of destination/source buffer
- * Response: (none beyond result code)
- */
-#define DALSMC_MSG_TransferTableSmu2Dram        0x03
-#define DALSMC_MSG_TransferTableDram2Smu        0x04
-typedef union {
-	struct {
-		uint32_t TableId;
-		uint32_t AddrLow;
-		uint32_t AddrHigh;
-		uint32_t Reserved;
-	};
-	DALSMC_args_t Args;
-} DALSMC_TransferTable_arg_t;
-
-/**
- * DALSMC_MSG_SetHardMinByFreq - Set a lower bound frequency constraint on a PPCLK.
- *
- * Response does not indicate that the effective clock has already been raised to
- * meet the minimum requirement; poll DALSMC_MSG_ReturnHardMinStatus for the status
- * of the request.
- *
- * Supported clocks: SOCCLK, DISPCLK, DPPCLK, DCFCLK, DTBCLK.
- *
- * Request:  FreqKhz[23:0] — target minimum frequency in kHz (0 to ~16.7 GHz)
- *           Ppclk[31:24]  — PPCLK_e clock identifier
- * Response: (none beyond result code)
- */
-#define DALSMC_MSG_SetHardMinByFreq             0x05
-typedef union {
-	struct {
-		uint32_t FreqKhz : 24;
-		uint32_t Ppclk   : 8;
-		uint32_t Reserved[3];
-	};
-	DALSMC_args_t Args;
-} DALSMC_SetHardMinByFreq_arg_t;
-
-/**
- * DALSMC_MSG_SetMinDeepSleepDcfclk - Set the minimum DCFCLK frequency in deep sleep.
- *
- * Request:  MinDcfclkMhz — minimum DCFCLK frequency in MHz during deep sleep
- * Response: (none beyond result code)
- */
-#define DALSMC_MSG_SetMinDeepSleepDcfclk        0x06
-typedef union {
-	struct {
-		uint32_t MinDcfclkMhz;
-		uint32_t Reserved[3];
-	};
-	DALSMC_args_t Args;
-} DALSMC_SetMinDeepSleepDcfclk_arg_t;
-
-/**
- * DALSMC_MSG_BacoAudioD3PME - Wake the audio block from D3/BACO.
- *
- * Triggers PMFW to bring the AZ (audio) block out of its D3 power state.
- * No arguments or response data; result code indicates success.
- */
-#define DALSMC_MSG_BacoAudioD3PME               0x07
-
-/**
- * DALSMC_MSG_ReturnHardMinStatus - Query outstanding hard-min request status.
- *
- * Returns a bitmask reporting which PPCLK hard-min requests have been satisfied
- * by the arbiter. Each bit position corresponds to the matching PPCLK_e value.
- * A set bit means the arbiter has reached or exceeded the requested minimum.
- *
- * Request:  (none)
- * Response: Reg0 — bitmask of satisfied PPCLKs (bit N set ↔ PPCLK_e N is satisfied)
- */
-#define DALSMC_MSG_ReturnHardMinStatus          0x08
-
-/**
- * DALSMC_MSG_IndicatePstateStatus - Indicate to PMFW various DMU behaviors required
- * to support UCLK P-state, for example whether or not DMU needs to modulate refresh
- * rate to perform UCLK switches.
- *
- * Request:  WaitResp[0]   — DAL requires a synchronous response before proceeding
- *           DrrEnable[1]  — DRR (dynamic refresh rate modulation) is active
- *           AltCh[2]      — alternate-channel mode is active
- *           AllowUclk[16] — DCN can tolerate UCLK P-state switches
- *           AllowFclk[17] — DCN can tolerate FCLK P-state switches
- * Response: (none beyond result code)
- */
-#define DALSMC_MSG_IndicatePstateStatus         0x09
-typedef union {
-	struct {
-		uint32_t WaitResp  : 1;
-		uint32_t DrrEnable : 1;
-		uint32_t AltCh     : 1;
-		uint32_t Reserved  : 13;
-		uint32_t AllowUclk : 1;
-		uint32_t AllowFclk : 1;
-		uint32_t Reserved1 : 14;
-		uint32_t Reserved2[3];
-	};
-	DALSMC_args_t Args;
-} DALSMC_IndicatePstateStatus_arg_t;
-
-/**
- * DALSMC_MSG_UpdateUTMQoSRequest - Update the active UTM QoS bandwidth/latency request.
- *
- * Passes the current display bandwidth and latency requirements to PMFW so it
- * can select the appropriate SoC operating point (UCLK/FCLK level) from the
- * UTM table. Called whenever the display configuration changes.
- *
- * The QoS requirement must take effect before PMFW sends its response.
- *
- * Request:  LatencySopIndex      — index into the UTM SOP table that satisfies latency
- *           NominalBandwidthKBps — required nominal (average) bandwidth in KB/s
- *           UrgentBandwidthKBps  — required urgent bandwidth in KB/s
- *           LsdmaBandwidthKBps   — required LSDMA bandwidth in KB/s
- * Response: (none beyond result code)
- */
-#define DALSMC_MSG_UpdateUTMQoSRequest          0x0A
-typedef union {
-	struct {
-		uint32_t LatencySopIndex;
-		uint32_t NominalBandwidthKBps;
-		uint32_t UrgentBandwidthKBps;
-		uint32_t LsdmaBandwidthKBps;
-	};
-	DALSMC_args_t Args;
-} DALSMC_UpdateUTMQoSRequest_arg_t;
-
-/**
- * DALSMC_MSG_SetDisplayIdleOptimizations - Notify PMFW of DCN idle-state conditions.
- *
- * Indicates which display-side power optimizations are currently safe to apply.
- * PMFW uses these flags to gate deeper SoC power states such as S0i2.
- *
- * Request:  DfRequestDisabled[0] — DF (data fabric) requests from DCN are disabled
- *           PhyRefClkOff[1]      — PHY reference clock has been gated off
- *           S0i2Rdy[2]           — DCN is ready for the system to enter S0i2
- * Response: (none beyond result code)
- */
-#define DALSMC_MSG_SetDisplayIdleOptimizations  0x0B
-typedef union {
-	struct {
-		uint32_t DfRequestDisabled : 1;
-		uint32_t PhyRefClkOff      : 1;
-		uint32_t S0i2Rdy           : 1;
-		uint32_t Reserved          : 29;
-		uint32_t Reserved1[3];
-	};
-	DALSMC_args_t Args;
-} DALSMC_SetDisplayIdleOptimizations_arg_t;
-
-/**
- * DALSMC_MSG_SetStutterEfficiency - Report DCN stutter efficiency to PMFW.
- *
- * Informs PMFW of the current stutter utilisation for base and low-power stutter
- * modes so PMFW can adjust memory power policy accordingly.
- *
- * Base mode    — lower enter+exit latency (PHY LP1, no UCIE LP).
- * Low-power mode — higher enter+exit latency (PHY LP2, UCIE LP1).
- *
- * Request:  BaseEfficiencyPct[7:0]     — stutter efficiency % in base mode
- *           LowPowerEfficiencyPct[15:8] — stutter efficiency % in low-power mode
- * Response: (none beyond result code)
- */
-#define DALSMC_MSG_SetStutterEfficiency         0x0C
-typedef union {
-	struct {
-		uint32_t BaseEfficiencyPct     : 8;
-		uint32_t LowPowerEfficiencyPct : 8;
-		uint32_t Reserved              : 16;
-		uint32_t Reserved1[3];
-	};
-	DALSMC_args_t Args;
-} DALSMC_SetStutterEfficiency_arg_t;
-
-#define DALSMC_Message_Count                    0x0D ///< Total number of messages
-
-/** @} */
+#define DALSMC_VERSION 0x1
 
 /** @defgroup DramTables DRAM Tables
  *  @brief Bulk data structures exchanged between DAL and PMFW via system DRAM.
@@ -362,14 +105,14 @@ typedef struct {
 	+ (SopIndex)) * sizeof(SocUtmSopEntry_t))
 typedef struct {
 	SocUtmTableHeader_t Header;
-	SocUtmSopEntry_t Sops[MAX_UTM_LOAD_LEVEL_COUNT][DALSMC_MAX_UTM_SOP_COUNT];
+	SocUtmSopEntry_t Sops[MAX_UTM_LOAD_LEVEL_COUNT][DALSMC_MAX_UTM_SOP_COUNT];// 4 and 3 for dGPU PMFW
 } SocUtmTable_t;
 
 /**
  * TABLE_DAL_INIT - Full TABLE_DAL_INIT payload transferred from SMU to DRAM.
  */
 #define TABLE_DAL_INIT    0xD
-#define MAX_PPCLK_COUNT   12
+#define MAX_PPCLK_COUNT   22
 #define DPM_CLOCK_OFFSET(Ppclk) \
 	(sizeof(DalInitHeader_t) + (Ppclk) * sizeof(DpmClock_t))
 #define UTM_TABLE_OFFSET \
@@ -384,6 +127,52 @@ typedef struct {
 	MemoryConfig_t    MemoryConfig;
 } DalInitTable_t;
 
-/** @} */
 
-#endif /* DALSMC_H */
+/** @defgroup ResponseCodes PMFW Response Codes
+*  @{
+*/
+// SMU Response Codes:
+#define DALSMC_Result_OK                    0x01 ///< Message Response OK
+#define DALSMC_Result_Failed                0xFF ///< Message Response Failed
+#define DALSMC_Result_UnknownCmd            0xFE ///< Message Response Unknown Command
+#define DALSMC_Result_CmdRejectedPrereq     0xFD ///< Message Response Command Failed Prerequisite
+#define DALSMC_Result_CmdRejectedBusy       0xFC ///< Message Response Command Rejected due to PMFW is busy. Sender should retry sending this message
+/** @}*/
+
+// Message Definitions:
+/** @defgroup definitions Message definitions
+*  @{
+*/
+#define DALSMC_MSG_TestMessage                  0x01 ///< To check if PMFW is alive and responding. Requirement specified by PMFW team
+#define DALSMC_MSG_GetPmfwVersion               0x02 ///< Get version
+#define DALSMC_MSG_SetDispclkFreq               0x03 ///< Set display clock frequency in MHZ
+#define DALSMC_MSG_SetDppclkFreq                0x04 ///< Set DPP clock frequency in MHZ
+#define DALSMC_MSG_SetHardMinDcfclkByFreq       0x05 ///< Set DCF clock frequency hard min in MHZ
+#define DALSMC_MSG_SetMinDeepSleepDcfclk        0x06 ///< Set DCF clock minimum frequency in deep sleep in MHZ
+#define DALSMC_MSG_GetFclkFrequency             0x07 ///< Get FCLK frequency, return frequemcy in MHZ
+#define DALSMC_MSG_UpdatePmeRestore             0x08 ///< To ask PMFW to write into Azalia for PME wake up event
+#define DALSMC_MSG_SetDramAddrHigh              0x09 ///< Set DRAM address high 32 bits for WM table transfer
+#define DALSMC_MSG_SetDramAddrLow               0x0A ///< Set DRAM address low 32 bits for WM table transfer
+#define DALSMC_MSG_TransferTableSmu2Dram        0x0B ///< Transfer table from PMFW SRAM to system DRAM
+#define DALSMC_MSG_TransferTableDram2Smu        0x0C ///< Transfer table from system DRAM to PMFW
+#define DALSMC_MSG_SetDisplayIdleOptimizations  0x0D ///< Set Idle state optimization for display off
+#define DALSMC_MSG_GetDprefclkFreq              0x0E ///< Get DPREF clock frequency. Return in MHZ
+#define DALSMC_MSG_GetDtbclkFreq                0x0F ///< Get DTB clock frequency. Return in MHZ
+#define DALSMC_MSG_AllowZstatesEntry_Spare0     0x10 ///< Inform PMFW of display allowing Zstate entry
+#define DALSMC_MSG_SetDtbClk                    0x11 ///< Set DTB clock frequency in MHZ
+#define DALSMC_MSG_DispIPS2Exit_Spare1          0x12 ///< Display IPS2 exit
+#define DALSMC_MSG_QueryIPS2Support             0x13 ///< Return 1: support; else not supported
+#define DALSMC_MSG_BandWidthRequest             0x14 ///< Arg 1-15: BwUrg; Arg 16-31: BwNonUrg
+#define DALSMC_MSG_UpdateUtmQosParams           0x15
+#define DALSMC_MSG_StutterEfficiency            0x16
+#define DALSMC_MSG_GetMsgHeaderVersion          0x17 ///< Get message header version
+#define DALSMC_MSG_SacoAudioD3PME               0x18 ///< Saco audio D3 PME
+#define DALSMC_MSG_ReturnHardMinStatus          0x19 ///< Return hard min status
+#define DALSMC_MSG_GetDispclkFreq               0x1A ///< Get display clock frequency. Return in MHZ
+#define DALSMC_MSG_GetDppclkFreq                0x1B ///< Get DPP clock frequency. Return in MHZ
+#define DALSMC_MSG_IndicatePstateStatus         0x1C ///< Set UCLK/FCLK related actions
+#define DALSMC_Message_Count                    0x1D ///< Total number of VBIS and DAL messages
+
+/** @}*/
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.c
index 7dd88d90eba2fb..7f33180f5e755e 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.c
@@ -6,13 +6,14 @@
 #include "clk_mgr_internal.h"
 #include "dcn60/dcn60_clk_mgr_smu_msg.h"
 #include "hw_sequencer.h"
-#include "dce100/dce_clk_mgr.h"
+#include "dcn10/dcn10_clk_mgr.h"
 #include "dcn20/dcn20_clk_mgr.h"
 #include "dcn401/dcn401_clk_mgr.h"
 #include "dcn60/dcn60_clk_mgr.h"
 #include "soc_and_ip_translator.h"
 #include "bounding_boxes/utm_qos_model_types.h"
 #include "bounding_boxes/utm_qos_model_dchub_v3.h"
+#include "bounding_boxes/dcn6_soc_bb.h"
 #include "reg_helper.h"
 #include "core_types.h"
 #include "dm_helpers.h"
@@ -213,13 +214,13 @@ static void dcn60_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
 	}
 }
 
-static int dcn60_set_hard_min_by_freq_optimized(struct clk_mgr_internal *clk_mgr, PPCLK_e clk, int requested_clk_khz)
+static int dcn60_set_clock_freq_optimized(struct clk_mgr_internal *clk_mgr, PPCLK_e clk, int requested_clk_khz)
 {
 	if (!clk_mgr->smu_present || !dcn60_is_ppclk_dpm_enabled(clk_mgr, clk))
 		return 0;
 
 	/*
-	 * SMU set hard min interface takes requested clock in mhz and return
+	 * SMU set clock interface takes requested clock in mhz and returns
 	 * actual clock configured in khz. If we floor requested clk to mhz,
 	 * there is a chance that the actual clock configured in khz is less
 	 * than requested. If we ceil it to mhz, there is a chance that it
@@ -228,11 +229,11 @@ static int dcn60_set_hard_min_by_freq_optimized(struct clk_mgr_internal *clk_mgr
 	 * clock returned is less than requested, then we will ceil the
 	 * requested value to mhz and call it again.
 	 */
-	int actual_clk_khz = dcn60_smu_set_hard_min_by_freq(clk_mgr, clk,
+	int actual_clk_khz = dcn60_smu_set_clock_freq(clk_mgr, clk,
 		(uint16_t)khz_to_mhz_floor(requested_clk_khz));
 
 	if (actual_clk_khz < requested_clk_khz)
-		actual_clk_khz = dcn60_smu_set_hard_min_by_freq(clk_mgr, clk,
+		actual_clk_khz = dcn60_smu_set_clock_freq(clk_mgr, clk,
 			(uint16_t)khz_to_mhz_ceil(requested_clk_khz));
 
 	return actual_clk_khz;
@@ -324,8 +325,8 @@ static unsigned int dcn60_build_update_display_clocks_sequence(
 		should_set_clock(safe_to_lower, new_clocks->ref_dtbclk_khz / 1000, clk_mgr_base->clks.ref_dtbclk_khz / 1000) && //TODO these should be ceiled
 		dcn60_is_ppclk_dpm_enabled(clk_mgr_internal, PPCLK_DTBCLK)) {
 		/* DCCG requires KHz precision for DTBCLK */
-		block_sequence[num_steps].params.update_hardmin_params.ppclk = PPCLK_DTBCLK;
-		block_sequence[num_steps].params.update_hardmin_params.freq_mhz =
+		block_sequence[num_steps].params.set_ppclk_params.ppclk = PPCLK_DTBCLK;
+		block_sequence[num_steps].params.set_ppclk_params.freq_mhz =
 			(uint16_t)khz_to_mhz_ceil(new_clocks->ref_dtbclk_khz);
 		for (i = 0; i < context->stream_count; i++) {
 			otg_master = resource_get_otg_master_for_stream(
@@ -338,11 +339,11 @@ static unsigned int dcn60_build_update_display_clocks_sequence(
 			}
 		}
 		if (frl_present)
-			block_sequence[num_steps].params.update_hardmin_params.freq_mhz =
+			block_sequence[num_steps].params.set_ppclk_params.freq_mhz =
 				(uint16_t)clk_mgr_base->bw_params->clk_table.entries[
 					clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dtbclk_levels - 1].dtbclk_mhz;
-		block_sequence[num_steps].params.update_hardmin_params.response = &clk_mgr_base->clks.ref_dtbclk_khz;
-		block_sequence[num_steps].func = CLK_MGR60_UPDATE_HARDMIN_PPCLK;
+		block_sequence[num_steps].params.set_ppclk_params.response = &clk_mgr_base->clks.ref_dtbclk_khz;
+		block_sequence[num_steps].func = CLK_MGR60_SET_PPCLK;
 		num_steps++;
 
 		/* Update DTO in DCCG */
@@ -366,10 +367,10 @@ static unsigned int dcn60_build_update_display_clocks_sequence(
 		clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz;
 
 		if (dcn60_is_ppclk_dpm_enabled(clk_mgr_internal, PPCLK_DISPCLK)) {
-			block_sequence[num_steps].params.update_hardmin_optimized_params.ppclk = PPCLK_DISPCLK;
-			block_sequence[num_steps].params.update_hardmin_optimized_params.freq_khz = clk_mgr_base->clks.dispclk_khz;
-			block_sequence[num_steps].params.update_hardmin_optimized_params.response = &clk_mgr_base->clks.actual_dispclk_khz;
-			block_sequence[num_steps].func = CLK_MGR60_UPDATE_HARDMIN_PPCLK_OPTIMIZED;
+			block_sequence[num_steps].params.set_ppclk_optimized_params.ppclk = PPCLK_DISPCLK;
+			block_sequence[num_steps].params.set_ppclk_optimized_params.freq_khz = clk_mgr_base->clks.dispclk_khz;
+			block_sequence[num_steps].params.set_ppclk_optimized_params.response = &clk_mgr_base->clks.actual_dispclk_khz;
+			block_sequence[num_steps].func = CLK_MGR60_SET_PPCLK_OPTIMIZED;
 			num_steps++;
 		}
 
@@ -386,10 +387,10 @@ static unsigned int dcn60_build_update_display_clocks_sequence(
 			num_steps++;
 
 			if (dcn60_is_ppclk_dpm_enabled(clk_mgr_internal, PPCLK_DPPCLK)) {
-				block_sequence[num_steps].params.update_hardmin_optimized_params.ppclk = PPCLK_DPPCLK;
-				block_sequence[num_steps].params.update_hardmin_optimized_params.freq_khz = clk_mgr_base->clks.dppclk_khz;
-				block_sequence[num_steps].params.update_hardmin_optimized_params.response = &clk_mgr_base->clks.actual_dppclk_khz;
-				block_sequence[num_steps].func = CLK_MGR60_UPDATE_HARDMIN_PPCLK_OPTIMIZED;
+				block_sequence[num_steps].params.set_ppclk_optimized_params.ppclk = PPCLK_DPPCLK;
+				block_sequence[num_steps].params.set_ppclk_optimized_params.freq_khz = clk_mgr_base->clks.dppclk_khz;
+				block_sequence[num_steps].params.set_ppclk_optimized_params.response = &clk_mgr_base->clks.actual_dppclk_khz;
+				block_sequence[num_steps].func = CLK_MGR60_SET_PPCLK_OPTIMIZED;
 				num_steps++;
 
 				block_sequence[num_steps].params.update_dppclk_dto_params.context = context;
@@ -401,10 +402,10 @@ static unsigned int dcn60_build_update_display_clocks_sequence(
 		} else {
 			/* if clock is being raised, increase refclk before lowering DTO */
 			if (update_dppclk && dcn60_is_ppclk_dpm_enabled(clk_mgr_internal, PPCLK_DPPCLK)) {
-				block_sequence[num_steps].params.update_hardmin_optimized_params.ppclk = PPCLK_DPPCLK;
-				block_sequence[num_steps].params.update_hardmin_optimized_params.freq_khz = clk_mgr_base->clks.dppclk_khz;
-				block_sequence[num_steps].params.update_hardmin_optimized_params.response = &clk_mgr_base->clks.actual_dppclk_khz;
-				block_sequence[num_steps].func = CLK_MGR60_UPDATE_HARDMIN_PPCLK_OPTIMIZED;
+				block_sequence[num_steps].params.set_ppclk_optimized_params.ppclk = PPCLK_DPPCLK;
+				block_sequence[num_steps].params.set_ppclk_optimized_params.freq_khz = clk_mgr_base->clks.dppclk_khz;
+				block_sequence[num_steps].params.set_ppclk_optimized_params.response = &clk_mgr_base->clks.actual_dppclk_khz;
+				block_sequence[num_steps].func = CLK_MGR60_SET_PPCLK_OPTIMIZED;
 				num_steps++;
 			}
 
@@ -478,6 +479,39 @@ static int dcn60_get_dtb_ref_freq_khz(struct clk_mgr *clk_mgr_base)
 	return dtb_ref_clk_khz;
 }
 
+/**
+ * dcn60_override_dc_mode_limit - Override DC mode limits from the clock table.
+ * @dc_limit: output DC mode limit to populate
+ * @clk_table: clock table already populated (and possibly overridden)
+ *
+ * Sets the DC mode max frequency for each clock to the highest populated DPM
+ * level in the clock table. Deriving the limit from the clock table (rather
+ * than the raw DAL init table) ensures any overrides applied to the clock
+ * levels are respected.
+ */
+static void dcn60_override_dc_mode_limit(
+		struct clk_limit_table_entry *dc_limit,
+		const struct clk_limit_table *clk_table)
+{
+	const struct clk_limit_table_entry *entries = clk_table->entries;
+	const struct clk_limit_num_entries *num_entries = &clk_table->num_entries_per_clk;
+
+	dc_limit->dcfclk_mhz  = num_entries->num_dcfclk_levels ?
+			entries[num_entries->num_dcfclk_levels - 1].dcfclk_mhz : 0;
+	dc_limit->socclk_mhz  = num_entries->num_socclk_levels ?
+			entries[num_entries->num_socclk_levels - 1].socclk_mhz : 0;
+	dc_limit->dtbclk_mhz  = num_entries->num_dtbclk_levels ?
+			entries[num_entries->num_dtbclk_levels - 1].dtbclk_mhz : 0;
+	dc_limit->dispclk_mhz = num_entries->num_dispclk_levels ?
+			entries[num_entries->num_dispclk_levels - 1].dispclk_mhz : 0;
+	dc_limit->dppclk_mhz  = num_entries->num_dppclk_levels ?
+			entries[num_entries->num_dppclk_levels - 1].dppclk_mhz : 0;
+	dc_limit->memclk_mhz  = num_entries->num_memclk_levels ?
+			entries[num_entries->num_memclk_levels - 1].memclk_mhz : 0;
+	dc_limit->fclk_mhz    = num_entries->num_fclk_levels ?
+			entries[num_entries->num_fclk_levels - 1].fclk_mhz : 0;
+}
+
 static unsigned int dcn60_get_dc_mode_limit_mhz(const DpmClock_t *dpm_clk)
 {
 	if (dpm_clk->NumClocks
@@ -494,8 +528,10 @@ static unsigned int dcn60_get_dc_mode_limit_mhz(const DpmClock_t *dpm_clk)
  *
  * Sets the DC mode max frequency for each clock. If DcMaxClock equals the
  * highest DPM level, the limit is set to 0 (no DC-specific cap).
+ *
+ * Temporarily unused.
  */
-static void dcn60_populate_dc_mode_limit(
+static void __maybe_unused dcn60_populate_dc_mode_limit(
 		struct clk_limit_table_entry *dc_limit,
 		const DalInitTable_t *init_table)
 {
@@ -528,37 +564,37 @@ static void dcn60_populate_clk_table(struct clk_mgr_internal *clk_mgr,
 	dpm_clk = &init_table->PPClocks[PPCLK_DCFCLK];
 	num_entries->num_dcfclk_levels = dpm_clk->NumClocks;
 	for (i = 0; i < dpm_clk->NumClocks && i < NUM_CLOCK_LEVELS; i++)
-		clk_table->entries[i].dcfclk_mhz = dpm_clk->Clocks[i];
+		clk_table->entries[i].dcfclk_mhz = khz_to_mhz_ceil(dpm_clk->Clocks[i]);
 
 	/* SOCCLK */
 	dpm_clk = &init_table->PPClocks[PPCLK_SOCCLK];
 	num_entries->num_socclk_levels = dpm_clk->NumClocks;
 	for (i = 0; i < dpm_clk->NumClocks && i < NUM_CLOCK_LEVELS; i++)
-		clk_table->entries[i].socclk_mhz = dpm_clk->Clocks[i];
+		clk_table->entries[i].socclk_mhz = khz_to_mhz_ceil(dpm_clk->Clocks[i]);
 
 	/* DTBCLK */
 	dpm_clk = &init_table->PPClocks[PPCLK_DTBCLK];
 	num_entries->num_dtbclk_levels = dpm_clk->NumClocks;
 	for (i = 0; i < dpm_clk->NumClocks && i < NUM_CLOCK_LEVELS; i++)
-		clk_table->entries[i].dtbclk_mhz = dpm_clk->Clocks[i];
+		clk_table->entries[i].dtbclk_mhz = khz_to_mhz_ceil(dpm_clk->Clocks[i]);
 
 	/* DISPCLK */
 	dpm_clk = &init_table->PPClocks[PPCLK_DISPCLK];
 	num_entries->num_dispclk_levels = dpm_clk->NumClocks;
 	for (i = 0; i < dpm_clk->NumClocks && i < NUM_CLOCK_LEVELS; i++)
-		clk_table->entries[i].dispclk_mhz = dpm_clk->Clocks[i];
+		clk_table->entries[i].dispclk_mhz = khz_to_mhz_ceil(dpm_clk->Clocks[i]);
 
 	/* DPPCLK */
 	dpm_clk = &init_table->PPClocks[PPCLK_DPPCLK];
 	num_entries->num_dppclk_levels = dpm_clk->NumClocks;
 	for (i = 0; i < dpm_clk->NumClocks && i < NUM_CLOCK_LEVELS; i++)
-		clk_table->entries[i].dppclk_mhz = dpm_clk->Clocks[i];
+		clk_table->entries[i].dppclk_mhz = khz_to_mhz_ceil(dpm_clk->Clocks[i]);
 
 	/* UCLK */
 	dpm_clk = &init_table->PPClocks[PPCLK_UCLK];
 	num_entries->num_memclk_levels = dpm_clk->NumClocks;
 	for (i = 0; i < dpm_clk->NumClocks && i < NUM_CLOCK_LEVELS; i++)
-		clk_table->entries[i].memclk_mhz = dpm_clk->Clocks[i];
+		clk_table->entries[i].memclk_mhz = khz_to_mhz_ceil(dpm_clk->Clocks[i]);
 	if (num_entries->num_memclk_levels)
 		clk_mgr->base.bw_params->max_memclk_mhz =
 				clk_table->entries[num_entries->num_memclk_levels - 1].memclk_mhz;
@@ -567,7 +603,7 @@ static void dcn60_populate_clk_table(struct clk_mgr_internal *clk_mgr,
 	dpm_clk = &init_table->PPClocks[PPCLK_FCLK];
 	num_entries->num_fclk_levels = dpm_clk->NumClocks;
 	for (i = 0; i < dpm_clk->NumClocks && i < NUM_CLOCK_LEVELS; i++)
-		clk_table->entries[i].fclk_mhz = dpm_clk->Clocks[i];
+		clk_table->entries[i].fclk_mhz = khz_to_mhz_ceil(dpm_clk->Clocks[i]);
 
 	if (num_entries->num_memclk_levels >= num_entries->num_fclk_levels)
 		clk_table->num_entries = num_entries->num_memclk_levels;
@@ -577,6 +613,20 @@ static void dcn60_populate_clk_table(struct clk_mgr_internal *clk_mgr,
 		clk_table->num_entries = 1;
 }
 
+/**
+ * dcn60_override_clk_table - Override the clock table with hardcoded values.
+ * @clk_table: clock table to override
+ *
+ * Temporary debug/bring-up override that replaces the DPM clock levels
+ * populated from the DAL init table (see dcn60_populate_clk_table) with a
+ * fixed set of hardcoded values. Implement any override as needed.
+ */
+static void dcn60_override_clk_table(struct clk_limit_table *clk_table)
+{
+	/* Override as needed */
+	(void)clk_table;
+}
+
 static void dcn60_override_bw_params(struct clk_mgr_internal *clk_mgr,
 		struct clk_bw_params *bw_params)
 {
@@ -603,6 +653,18 @@ static void dcn60_override_bw_params(struct clk_mgr_internal *clk_mgr,
 		bw_params->dc_mode_limit.dtbclk_mhz = 0;
 
 	bw_params->dc_mode_softmax_memclk = bw_params->dc_mode_limit.memclk_mhz;
+
+	/* Override as needed - temporary for debug only. */
+	if (bw_params->utm_qos_model && bw_params->utm_qos_model->dchub_v3) {
+		dcn6_test_initialize_utm_qos_model_v3(
+				(struct utm_qos_model *)bw_params->utm_qos_model,
+				(struct utm_qos_model_dchub_v3 *)bw_params->utm_qos_model->dchub_v3);
+
+		// Override for lsdma here is redundant with the above call, but this may need to outlive
+		// the test_initialize call for debug purposes so keep it here for now.
+		dcn6_test_override_lsdma_bandwidth_v3(
+				(struct utm_qos_model_dchub_v3 *)bw_params->utm_qos_model->dchub_v3);
+	}
 }
 
 /**
@@ -625,9 +687,19 @@ static void dcn60_populate_utm_qos_model(
 	const SocUtmTable_t *utm_table = &init_table->UtmTable;
 	unsigned int ll, sop;
 
+	if (utm_table->Header.LoadLevelCount == 0 || utm_table->Header.SopCount == 0) {
+		*qos_model = NULL;
+		return;
+	}
+
 	memset(dchub, 0, sizeof(*dchub));
 	dchub->load_level_count = (uint8_t)utm_table->Header.LoadLevelCount;
 	dchub->sop_count = (uint8_t)utm_table->Header.SopCount;
+	/* WORKAROUND: limit sop_count to 5 for now instead of using
+	 * utm_table->Header.SopCount.
+	 */
+	if (utm_table->Header.SopCount > 5)
+		dchub->sop_count = 5;
 
 	for (ll = 0; ll < dchub->load_level_count
 			&& ll < UTM_QOS_MODEL_V3_MAX_LOAD_LEVEL_COUNT; ll++) {
@@ -667,7 +739,9 @@ static bool dcn60_fetch_dal_init_table(struct clk_mgr_internal *clk_mgr)
 	clk_mgr->smu_ver = init_table->Header.SmuVersion;
 
 	dcn60_populate_clk_table(clk_mgr, &bw_params->clk_table, init_table);
-	dcn60_populate_dc_mode_limit(&bw_params->dc_mode_limit, init_table);
+	// Comment out for now - DC mode limit is not yet used in DCN6 and the current
+	// population from init table will cause undefined behaviors.
+	//dcn60_populate_dc_mode_limit(&bw_params->dc_mode_limit, init_table);
 
 	bw_params->num_channels = init_table->MemoryConfig.NumUmcChannels;
 	bw_params->dram_channel_width_bytes =
@@ -676,34 +750,12 @@ static bool dcn60_fetch_dal_init_table(struct clk_mgr_internal *clk_mgr)
 	dcn60_populate_utm_qos_model(clk_mgr, &bw_params->utm_qos_model, init_table);
 
 	dcn60_override_bw_params(clk_mgr, bw_params);
+	dcn60_override_clk_table(&bw_params->clk_table);
+	dcn60_override_dc_mode_limit(&bw_params->dc_mode_limit, &bw_params->clk_table);
 
 	return true;
 }
 
-void dcn60_init_clocks(struct clk_mgr *clk_mgr_base)
-{
-	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-	uint32_t smu_header_ver = 0;
-
-	memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks));
-	clk_mgr_base->clks.p_state_change_support = true;
-	clk_mgr_base->clks.fclk_p_state_change_support = false;
-	clk_mgr_base->force_smu_not_present = true; // temporary until SMU ready
-	clk_mgr->smu_present = !clk_mgr_base->force_smu_not_present /* not force-disabled */
-			&& dcn60_smu_get_msg_header_version(clk_mgr, &smu_header_ver)
-			&& smu_header_ver != 0;
-
-	clk_mgr->dpm_present = clk_mgr->smu_present
-			&& dcn60_fetch_dal_init_table(clk_mgr)
-			&& clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dcfclk_levels
-			&& clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dtbclk_levels
-			&& clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dispclk_levels;
-
-	if (clk_mgr->dpm_present)
-		clk_mgr_base->ctx->dc->res_pool->funcs->update_bw_bounding_box(
-				clk_mgr_base->ctx->dc, clk_mgr_base->bw_params);
-}
-
 static inline uint32_t count_to_khz(uint32_t count, uint32_t timer_ths, uint32_t refclk_khz)
 {
 	if (timer_ths == 0)
@@ -824,6 +876,80 @@ static void dcn60_dump_clk_registers(struct clk_state_registers_and_bypass *regs
 	}
 }
 
+static void dcn60_dump_and_assign_boot_clocks(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_log_info log_info = {0};
+
+	dcn60_dump_clk_registers(&clk_mgr_base->boot_snapshot, clk_mgr_base, &log_info);
+
+	if (clk_mgr_base->ctx->dc->debug.disable_dtb_ref_clk_switch &&
+			clk_mgr_base->clks.ref_dtbclk_khz != clk_mgr_base->boot_snapshot.dtbclk) {
+		clk_mgr_base->clks.ref_dtbclk_khz = clk_mgr_base->boot_snapshot.dtbclk;
+	}
+
+	if (clk_mgr_base->boot_snapshot.dprefclk != 0)
+		clk_mgr_base->dprefclk_khz = clk_mgr_base->boot_snapshot.dprefclk;
+}
+
+static void dcn60_clock_read_ss_info(struct clk_mgr_internal *clk_mgr)
+{
+	struct dc_bios *bp = clk_mgr->base.ctx->dc_bios;
+	int ss_info_num = bp->funcs->get_ss_entry_number(
+			bp, AS_SIGNAL_TYPE_GPU_PLL);
+
+	if (ss_info_num) {
+		struct spread_spectrum_info info = { { 0 } };
+		enum bp_result result = bp->funcs->get_spread_spectrum_info(
+				bp, AS_SIGNAL_TYPE_GPU_PLL, 0, &info);
+
+		/* SSInfo.spreadSpectrumPercentage !=0 would be sign
+		 * that SS is enabled
+		 */
+		if (result == BP_RESULT_OK &&
+				info.spread_spectrum_percentage != 0) {
+			clk_mgr->ss_on_dprefclk = true;
+			clk_mgr->dprefclk_ss_divider = info.spread_percentage_divider;
+
+			if (info.type.CENTER_MODE == 0) {
+				/* Currently for DP Reference clock we
+				 * need only SS percentage for
+				 * downspread
+				 */
+				clk_mgr->dprefclk_ss_percentage =
+						info.spread_spectrum_percentage;
+			}
+		}
+	}
+}
+
+void dcn60_init_clocks(struct clk_mgr *clk_mgr_base)
+{
+	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
+	uint32_t smu_header_ver = 0;
+
+	memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks));
+	clk_mgr_base->clks.p_state_change_support = true;
+	clk_mgr_base->clks.fclk_p_state_change_support = false;
+
+	dcn60_dump_and_assign_boot_clocks(clk_mgr_base);
+	dcn60_clock_read_ss_info(clk_mgr);
+
+	clk_mgr->smu_present = !clk_mgr_base->force_smu_not_present /* not force-disabled */
+			&& dcn60_smu_get_msg_header_version(clk_mgr, &smu_header_ver)
+			&& smu_header_ver != 0;
+
+	clk_mgr->dpm_present = clk_mgr->smu_present
+			&& dcn60_fetch_dal_init_table(clk_mgr)
+			&& clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dcfclk_levels
+			&& clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dtbclk_levels
+			&& clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_dispclk_levels;
+
+	if (clk_mgr->dpm_present)
+		clk_mgr_base->ctx->dc->res_pool->funcs->update_bw_bounding_box(
+				clk_mgr_base->ctx->dc, clk_mgr_base->bw_params);
+}
+
+
 static void dcn60_auto_dpm_test_log(
 		struct dc_clocks *new_clocks,
 		struct clk_mgr_internal *clk_mgr,
@@ -951,28 +1077,38 @@ static void dcn60_execute_block_sequence(struct clk_mgr *clk_mgr_base, unsigned
 		params = &clk_mgr60->block_sequence[i].params;
 
 		switch (clk_mgr60->block_sequence[i].func) {
-		case CLK_MGR60_UPDATE_HARDMIN_PPCLK:
-			if (params->update_hardmin_params.response)
-				*params->update_hardmin_params.response = dcn60_smu_set_hard_min_by_freq(
+		case CLK_MGR60_SET_PPCLK:
+			if (params->set_ppclk_params.response)
+				*params->set_ppclk_params.response = dcn60_smu_set_clock_freq(
 						clk_mgr_internal,
-						params->update_hardmin_params.ppclk,
-						params->update_hardmin_params.freq_mhz);
+						params->set_ppclk_params.ppclk,
+						params->set_ppclk_params.freq_mhz);
 			else
-				dcn60_smu_set_hard_min_by_freq(clk_mgr_internal,
-						params->update_hardmin_params.ppclk,
-						params->update_hardmin_params.freq_mhz);
+				dcn60_smu_set_clock_freq(clk_mgr_internal,
+						params->set_ppclk_params.ppclk,
+						params->set_ppclk_params.freq_mhz);
 			break;
-		case CLK_MGR60_UPDATE_HARDMIN_PPCLK_OPTIMIZED:
-			if (params->update_hardmin_optimized_params.response)
-				*params->update_hardmin_optimized_params.response =
-					dcn60_set_hard_min_by_freq_optimized(
+		case CLK_MGR60_SET_PPCLK_OPTIMIZED:
+			if (params->set_ppclk_optimized_params.response)
+				*params->set_ppclk_optimized_params.response =
+					dcn60_set_clock_freq_optimized(
 						clk_mgr_internal,
-						params->update_hardmin_optimized_params.ppclk,
-						params->update_hardmin_optimized_params.freq_khz);
+						params->set_ppclk_optimized_params.ppclk,
+						params->set_ppclk_optimized_params.freq_khz);
 			else
-				dcn60_set_hard_min_by_freq_optimized(clk_mgr_internal,
-						params->update_hardmin_optimized_params.ppclk,
-						params->update_hardmin_optimized_params.freq_khz);
+				dcn60_set_clock_freq_optimized(clk_mgr_internal,
+						params->set_ppclk_optimized_params.ppclk,
+						params->set_ppclk_optimized_params.freq_khz);
+			break;
+		case CLK_MGR60_UPDATE_HARDMIN_DCFCLK:
+			if (params->update_hardmin_dcfclk_params.response)
+				*params->update_hardmin_dcfclk_params.response =
+					dcn60_smu_set_hard_min_dcfclk_by_freq(
+						clk_mgr_internal,
+						params->update_hardmin_dcfclk_params.freq_mhz);
+			else
+				dcn60_smu_set_hard_min_dcfclk_by_freq(clk_mgr_internal,
+						params->update_hardmin_dcfclk_params.freq_mhz);
 			break;
 		case CLK_MGR60_UPDATE_DEEP_SLEEP_DCFCLK:
 			dcn60_smu_set_min_deep_sleep_dcfclk(
@@ -1161,7 +1297,8 @@ static unsigned int dcn60_build_bandwidth_clocks_block_sequence_with_action(
 				action->fams.enable
 				|| clk_mgr_base->clks.fw_based_mclk_switching;
 		block_sequence[num_steps].params.indicate_pstate_status_params.wait_resp =
-				action->fams.enable;
+				action->fams.enable
+				|| clk_mgr_base->clks.fw_based_mclk_switching;
 		block_sequence[num_steps].params.indicate_pstate_status_params.alt_ch_enable =
 				action->alt_ch.enable
 				|| clk_mgr_base->clks.alt_ch_pstate_switch;
@@ -1169,13 +1306,12 @@ static unsigned int dcn60_build_bandwidth_clocks_block_sequence_with_action(
 		num_steps++;
 	}
 
-	/* CLK_MGR60_UPDATE_HARDMIN_PPCLK — DCFCLK */
+	/* CLK_MGR60_UPDATE_HARDMIN_DCFCLK — DCFCLK */
 	if (action->dcfclk.send_message) {
-		block_sequence[num_steps].params.update_hardmin_params.ppclk = PPCLK_DCFCLK;
-		block_sequence[num_steps].params.update_hardmin_params.freq_mhz =
+		block_sequence[num_steps].params.update_hardmin_dcfclk_params.freq_mhz =
 			(uint16_t)khz_to_mhz_ceil(new_clocks->dcfclk_khz);
-		block_sequence[num_steps].params.update_hardmin_params.response = NULL;
-		block_sequence[num_steps].func = CLK_MGR60_UPDATE_HARDMIN_PPCLK;
+		block_sequence[num_steps].params.update_hardmin_dcfclk_params.response = NULL;
+		block_sequence[num_steps].func = CLK_MGR60_UPDATE_HARDMIN_DCFCLK;
 		num_steps++;
 	}
 
@@ -1361,37 +1497,6 @@ static void dcn60_update_clocks(struct clk_mgr *clk_mgr_base,
 
 }
 
-static void dcn60_clock_read_ss_info(struct clk_mgr_internal *clk_mgr)
-{
-	struct dc_bios *bp = clk_mgr->base.ctx->dc_bios;
-	int ss_info_num = bp->funcs->get_ss_entry_number(
-			bp, AS_SIGNAL_TYPE_GPU_PLL);
-
-	if (ss_info_num) {
-		struct spread_spectrum_info info = { { 0 } };
-		enum bp_result result = bp->funcs->get_spread_spectrum_info(
-				bp, AS_SIGNAL_TYPE_GPU_PLL, 0, &info);
-
-		/* SSInfo.spreadSpectrumPercentage !=0 would be sign
-		 * that SS is enabled
-		 */
-		if (result == BP_RESULT_OK &&
-				info.spread_spectrum_percentage != 0) {
-			clk_mgr->ss_on_dprefclk = true;
-			clk_mgr->dprefclk_ss_divider = info.spread_percentage_divider;
-
-			if (info.type.CENTER_MODE == 0) {
-				/* Currently for DP Reference clock we
-				 * need only SS percentage for
-				 * downspread
-				 */
-				clk_mgr->dprefclk_ss_percentage =
-						info.spread_spectrum_percentage;
-			}
-		}
-	}
-}
-
 /* Set min memclk to minimum, either constrained by the current mode or DPM0 */
 static void dcn60_set_hard_min_memclk(struct clk_mgr *clk_mgr_base, bool current_mode)
 {
@@ -1521,6 +1626,9 @@ static void dcn60_set_idle_power_optimizations(struct clk_mgr *clk_mgr_base, boo
 	if (!clk_mgr->smu_present)
 		return;
 
+	if (!ASICREV_IS_DCN6_VARIANT_LITE3(clk_mgr_base->ctx->asic_id.hw_internal_rev))
+		return;
+
 	dcn60_smu_set_display_idle_optimization(clk_mgr, enable);
 }
 
@@ -1571,7 +1679,7 @@ static void dcn60_execute_clk_mgr_block_sequence_bls(struct clk_mgr *clk_mgr_bas
 }
 
 static struct clk_mgr_funcs dcn60_funcs = {
-		.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
+		.get_dp_ref_clk_frequency = dcn10_get_dp_ref_freq_khz,
 		.get_dtb_ref_clk_frequency = dcn60_get_dtb_ref_freq_khz,
 		.update_clocks = dcn60_update_clocks,
 		.dump_clk_registers = dcn60_dump_clk_registers,
@@ -1594,7 +1702,6 @@ struct clk_mgr_internal *dcn60_clk_mgr_construct(
 		struct dc_context *ctx,
 		struct dccg *dccg)
 {
-	struct clk_log_info log_info = {0};
 	struct dcn60_clk_mgr *clk_mgr60 = kzalloc_obj(struct dcn60_clk_mgr);
 	struct clk_mgr_internal *clk_mgr;
 
@@ -1603,6 +1710,7 @@ struct clk_mgr_internal *dcn60_clk_mgr_construct(
 
 	clk_mgr = &clk_mgr60->base;
 	clk_mgr->base.ctx = ctx;
+	clk_mgr->base.inst = 0;
 	clk_mgr->base.funcs = &dcn60_funcs;
 	clk_mgr->regs = &clk_mgr_regs_dcn60;
 	clk_mgr->clk_mgr_shift = &clk_mgr_shift_dcn60;
@@ -1629,17 +1737,6 @@ struct clk_mgr_internal *dcn60_clk_mgr_construct(
 		if (clk_mgr->base.dentist_vco_freq_khz == 0)
 			clk_mgr->base.dentist_vco_freq_khz = 4500000;
 
-		dcn60_dump_clk_registers(&clk_mgr->base.boot_snapshot, &clk_mgr->base, &log_info);
-
-		if (ctx->dc->debug.disable_dtb_ref_clk_switch &&
-				clk_mgr->base.clks.ref_dtbclk_khz != clk_mgr->base.boot_snapshot.dtbclk) {
-			clk_mgr->base.clks.ref_dtbclk_khz = clk_mgr->base.boot_snapshot.dtbclk;
-		}
-
-		if (clk_mgr->base.boot_snapshot.dprefclk != 0)
-			clk_mgr->base.dprefclk_khz = clk_mgr->base.boot_snapshot.dprefclk;
-		dcn60_clock_read_ss_info(clk_mgr);
-
 	clk_mgr->dfs_bypass_enabled = false;
 
 	clk_mgr->smu_present = false;
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.h
index 1c9cd4e95aac1d..0625f060e8c6bf 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr.h
@@ -17,14 +17,20 @@ union dcn60_clk_mgr_block_sequence_params {
 		uint16_t freq_mhz;
 		/* outputs */
 		int *response;
-	} update_hardmin_params;
+	} set_ppclk_params;
 	struct {
 		/* inputs */
 		uint32_t ppclk;
 		int freq_khz;
 		/* outputs */
 		int *response;
-	} update_hardmin_optimized_params;
+	} set_ppclk_optimized_params;
+	struct {
+		/* inputs */
+		uint16_t freq_mhz;
+		/* outputs */
+		int *response;
+	} update_hardmin_dcfclk_params;
 	struct {
 		/* inputs */
 		uint16_t freq_mhz;
@@ -72,8 +78,7 @@ union dcn60_clk_mgr_block_sequence_params {
 
 enum dcn60_clk_mgr_block_sequence_func {
 	CLK_MGR60_READ_CLOCKS_FROM_DENTIST,
-	CLK_MGR60_UPDATE_HARDMIN_PPCLK,
-	CLK_MGR60_UPDATE_HARDMIN_PPCLK_OPTIMIZED,
+	CLK_MGR60_UPDATE_HARDMIN_DCFCLK,
 	CLK_MGR60_UPDATE_DEEP_SLEEP_DCFCLK,
 	CLK_MGR60_INDICATE_PSTATE_STATUS,
 	CLK_MGR60_UPDATE_DPPCLK_DTO,
@@ -81,7 +86,9 @@ enum dcn60_clk_mgr_block_sequence_func {
 	CLK_MGR60_UPDATE_DENTIST,
 	CLK_MGR60_UPDATE_PSR_WAIT_LOOP,
 	CLK_MGR60_UPDATE_STUTTER_EFFICIENCY,
-	CLK_MGR60_UPDATE_UTM_QOS_REQUEST
+	CLK_MGR60_UPDATE_UTM_QOS_REQUEST,
+	CLK_MGR60_SET_PPCLK,
+	CLK_MGR60_SET_PPCLK_OPTIMIZED,
 };
 
 struct dcn60_clk_mgr_block_sequence {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.c
index 697ed8f747d4ce..5c1c95160892c5 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.c
@@ -8,9 +8,9 @@
 #include "reg_helper.h"
 
 #include "dalsmc.h"
-#include "dcn401/dcn401_smu14_driver_if.h"
+#include "dcn60/dcn60_smu_driver_if.h"
 
-/* MMIO = MP1_BASE__INST0_SEG1 (0x16200 from at2_offset.h)
+/* MMIO = MP1_BASE__INST0_SEG1 (0x16200 from <dgpu>_offset.h)
  *      + regMP1_SMN_C2PMSG_N  (sequential from 0xA2)
  * MSG=C2PMSG_98, RESP=C2PMSG_99, ARG0..3=C2PMSG_100..103
  */
@@ -159,11 +159,9 @@ static unsigned int dcn60_smu_get_hard_min_status(struct clk_mgr_internal *clk_m
 }
 
 
-static bool dcn60_smu_wait_hard_min_status(struct clk_mgr_internal *clk_mgr, uint32_t ppclk)
+static bool dcn60_smu_wait_hard_min_status(struct clk_mgr_internal *clk_mgr)
 {
 	const unsigned int max_delay_us = 1000000;
-
-	unsigned int hardmin_status_mask = (1 << ppclk);
 	unsigned int total_delay_us = 0;
 	bool hardmin_done = false;
 
@@ -182,37 +180,183 @@ static bool dcn60_smu_wait_hard_min_status(struct clk_mgr_internal *clk_mgr, uin
 
 		hardmin_status = dcn60_smu_get_hard_min_status(clk_mgr, &no_timeout, &read_total_delay_us);
 		total_delay_us += read_total_delay_us;
-		hardmin_done = hardmin_status & hardmin_status_mask;
+		hardmin_done = hardmin_status;
 	}
 
 	return hardmin_done;
 }
 
-/* TODO: update callers to pass kHz directly for finer granularity
- * now that the DALSMC interface supports 24-bit kHz encoding.
+/*
+ * DCFCLK still uses a hard-min arbiter request. Send the DCFCLK-specific
+ * hard-min message and poll ReturnHardMinStatus until the arbiter reports the
+ * minimum has been satisfied. PMFW returns the actual configured freq (kHz).
  */
-unsigned int dcn60_smu_set_hard_min_by_freq(struct clk_mgr_internal *clk_mgr, uint32_t clk, uint16_t freq_mhz)
+unsigned int dcn60_smu_set_hard_min_dcfclk_by_freq(struct clk_mgr_internal *clk_mgr, uint16_t freq_mhz)
 {
-	DALSMC_SetHardMinByFreq_arg_t arg = {};
+	DALSMC_SetClockFreq_arg_t arg = {};
 	uint32_t response = 0;
 	bool hard_min_done = false;
 
-	smu_print("SMU Set hard min by freq: clk = %d, freq_mhz = %d MHz\n", clk, freq_mhz);
+	smu_print("SMU Set DCFCLK hard min by freq: freq_mhz = %d MHz\n", freq_mhz);
 
-	/* New interface encodes frequency in kHz (24-bit) and PPCLK in bits [31:24] */
-	arg.FreqKhz = (uint32_t)freq_mhz * 1000;
-	arg.Ppclk   = clk;
+	arg.FreqMhz = freq_mhz;
 
 	dcn60_smu_send_msg_with_args(clk_mgr,
-			DALSMC_MSG_SetHardMinByFreq, arg.Args, &response, NULL);
+			DALSMC_MSG_SetHardMinDcfclkByFreq, arg.Args, &response, NULL);
 
 	/* wait until hardmin acknowledged */
-	hard_min_done = dcn60_smu_wait_hard_min_status(clk_mgr, clk);
-	smu_print("SMU Frequency set = %d KHz hard_min_done %d\n", response, hard_min_done);
+	hard_min_done = dcn60_smu_wait_hard_min_status(clk_mgr);
+	smu_print("SMU DCFCLK set = %d KHz hard_min_done %d\n", response, hard_min_done);
+
+	return response;
+}
+
+/*
+ * DISPCLK / DPPCLK / DTBCLK are programmed directly (not via a hard-min arbiter
+ * request). PMFW returns the actual configured frequency in kHz.
+ */
+static unsigned int dcn60_smu_set_dispclk_freq(struct clk_mgr_internal *clk_mgr, uint16_t freq_mhz)
+{
+	DALSMC_SetClockFreq_arg_t arg = {};
+	uint32_t response = 0;
+
+	arg.FreqMhz = freq_mhz;
+	dcn60_smu_send_msg_with_args(clk_mgr,
+			DALSMC_MSG_SetDispclkFreq, arg.Args, &response, NULL);
 
 	return response;
 }
 
+static unsigned int dcn60_smu_set_dppclk_freq(struct clk_mgr_internal *clk_mgr, uint16_t freq_mhz)
+{
+	DALSMC_SetClockFreq_arg_t arg = {};
+	uint32_t response = 0;
+
+	arg.FreqMhz = freq_mhz;
+	dcn60_smu_send_msg_with_args(clk_mgr,
+			DALSMC_MSG_SetDppclkFreq, arg.Args, &response, NULL);
+
+	return response;
+}
+
+static unsigned int dcn60_smu_set_dtb_clk(struct clk_mgr_internal *clk_mgr, uint16_t freq_mhz)
+{
+	DALSMC_SetClockFreq_arg_t arg = {};
+	uint32_t response = 0;
+
+	arg.FreqMhz = freq_mhz;
+	dcn60_smu_send_msg_with_args(clk_mgr,
+			DALSMC_MSG_SetDtbClk, arg.Args, &response, NULL);
+
+	return response;
+}
+
+static unsigned int dcn60_smu_get_dispclk_freq_khz(struct clk_mgr_internal *clk_mgr)
+{
+	DALSMC_args_t args = {};
+	uint32_t response = 0;
+
+	dcn60_smu_send_msg_with_args(clk_mgr,
+			DALSMC_MSG_GetDispclkFreq, args, &response, NULL);
+
+	return response;
+}
+
+static unsigned int dcn60_smu_get_dppclk_freq_khz(struct clk_mgr_internal *clk_mgr)
+{
+	DALSMC_args_t args = {};
+	uint32_t response = 0;
+
+	dcn60_smu_send_msg_with_args(clk_mgr,
+			DALSMC_MSG_GetDppclkFreq, args, &response, NULL);
+
+	return response;
+}
+
+static unsigned int dcn60_smu_get_dtbclk_freq_khz(struct clk_mgr_internal *clk_mgr)
+{
+	DALSMC_args_t args = {};
+	uint32_t response = 0;
+
+	dcn60_smu_send_msg_with_args(clk_mgr,
+			DALSMC_MSG_GetDtbclkFreq, args, &response, NULL);
+
+	return response;
+}
+
+/* Return the current frequency (KHz) for a display-domain clock. */
+static unsigned int dcn60_smu_get_clock_freq_khz(struct clk_mgr_internal *clk_mgr, uint32_t clk)
+{
+	switch (clk) {
+	case PPCLK_DISPCLK:
+		return dcn60_smu_get_dispclk_freq_khz(clk_mgr);
+	case PPCLK_DPPCLK:
+		return dcn60_smu_get_dppclk_freq_khz(clk_mgr);
+	case PPCLK_DTBCLK:
+		return dcn60_smu_get_dtbclk_freq_khz(clk_mgr);
+	default:
+		return 0;
+	}
+}
+
+/*
+ * Poll the matching Get*Freq message until the reported frequency reaches the
+ * requested target. Replaces the ReturnHardMinStatus poll used for hard-mins.
+ */
+static bool dcn60_smu_wait_clock_freq(struct clk_mgr_internal *clk_mgr,
+		uint32_t clk, uint16_t target_mhz)
+{
+	const unsigned int max_delay_us = 1000000;
+	unsigned int total_delay_us = 0;
+	bool done = false;
+
+	while (!done && total_delay_us < max_delay_us) {
+		done = khz_to_mhz_ceil(dcn60_smu_get_clock_freq_khz(clk_mgr, clk)) >= target_mhz;
+		if (!done) {
+			udelay(500);
+			total_delay_us += 500;
+		}
+	}
+
+	return done;
+}
+
+/*
+ * Set a display-domain clock (DISPCLK, DPPCLK or DTBCLK) frequency and poll the
+ * matching Get*Freq message until the clock reaches the requested target.
+ * Only these three clocks are handled here.
+ */
+unsigned int dcn60_smu_set_clock_freq(struct clk_mgr_internal *clk_mgr,
+		uint32_t clk, uint16_t freq_mhz)
+{
+	unsigned int actual_clk_khz = 0;
+
+	smu_print("SMU Set clock freq: clk = %d, freq_mhz = %d MHz\n", clk, freq_mhz);
+
+	switch (clk) {
+	case PPCLK_DISPCLK:
+		actual_clk_khz = dcn60_smu_set_dispclk_freq(clk_mgr, freq_mhz);
+		break;
+	case PPCLK_DPPCLK:
+		actual_clk_khz = dcn60_smu_set_dppclk_freq(clk_mgr, freq_mhz);
+		break;
+	case PPCLK_DTBCLK:
+		actual_clk_khz = dcn60_smu_set_dtb_clk(clk_mgr, freq_mhz);
+		break;
+	default:
+		/* set_clock_freq only handles DISPCLK, DPPCLK and DTBCLK */
+		smu_print("SMU Set clock freq: unsupported clk %d\n", clk);
+		return 0;
+	}
+
+	/* confirm the clock reached the requested target */
+	dcn60_smu_wait_clock_freq(clk_mgr, clk, freq_mhz);
+
+	smu_print("SMU Clock set = %d KHz\n", actual_clk_khz);
+
+	return actual_clk_khz;
+}
+
 
 void dcn60_smu_set_stutter_efficiency(struct clk_mgr_internal *clk_mgr, uint8_t base_efficiency, uint8_t low_power_efficiency)
 {
@@ -225,7 +369,7 @@ void dcn60_smu_set_stutter_efficiency(struct clk_mgr_internal *clk_mgr, uint8_t
 	arg.LowPowerEfficiencyPct = low_power_efficiency;
 
 	dcn60_smu_send_msg_with_args(clk_mgr,
-			DALSMC_MSG_SetStutterEfficiency, arg.Args, NULL, NULL);
+			DALSMC_MSG_StutterEfficiency, arg.Args, NULL, NULL);
 }
 
 void dcn60_smu_set_min_deep_sleep_dcfclk(struct clk_mgr_internal *clk_mgr, uint32_t freq_mhz)
@@ -247,7 +391,7 @@ void dcn60_smu_set_pme_workaround(struct clk_mgr_internal *clk_mgr)
 	smu_print("SMU Set PME workaround (BacoAudioD3PME)\n");
 
 	dcn60_smu_send_msg_with_args(clk_mgr,
-			DALSMC_MSG_BacoAudioD3PME, args, NULL, NULL);
+			DALSMC_MSG_SacoAudioD3PME, args, NULL, NULL);
 }
 
 void dcn60_smu_indicate_pstate_status(struct clk_mgr_internal *clk_mgr,
@@ -341,7 +485,7 @@ bool dcn60_smu_update_utm_qos_request(struct clk_mgr_internal *clk_mgr,
 	arg.LsdmaBandwidthKBps   = lsdma_bandwidth_KBps;
 
 	return dcn60_smu_send_msg_with_args(clk_mgr,
-			DALSMC_MSG_UpdateUTMQoSRequest, arg.Args, NULL, NULL);
+			DALSMC_MSG_UpdateUtmQosParams, arg.Args, NULL, NULL);
 }
 
 bool dcn60_smu_get_msg_header_version(struct clk_mgr_internal *clk_mgr,
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.h
index b1a212246c54e3..656213ca4d9ee3 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_clk_mgr_smu_msg.h
@@ -11,7 +11,172 @@
 
 struct clk_mgr_internal;
 
-unsigned int dcn60_smu_set_hard_min_by_freq(struct clk_mgr_internal *clk_mgr, uint32_t clk, uint16_t freq_mhz);
+/** Generic register overlay — four 32-bit C2PMSG argument registers. */
+typedef struct {
+	uint32_t Reg0;
+	uint32_t Reg1;
+	uint32_t Reg2;
+	uint32_t Reg3;
+} DALSMC_args_t;
+
+
+/**
+ * DALSMC_MSG_SetDispclkFreq - Set the DISPCLK frequency (MHz).
+ * DALSMC_MSG_SetDppclkFreq  - Set the DPPCLK frequency (MHz).
+ * DALSMC_MSG_SetDtbClk      - Set the DTBCLK frequency (MHz).
+ *
+ * Directly programs the target display-domain clock instead of requesting an
+ * arbiter hard-min. PMFW returns the actual configured frequency (kHz) in Reg0.
+ * Poll the matching DALSMC_MSG_Get*Freq message to confirm the clock has
+ * reached the requested target.
+ *
+ * Request:  FreqMhz — target frequency in MHz
+ * Response: Reg0    — actual configured frequency in kHz
+ */
+typedef union {
+	struct {
+		uint32_t FreqMhz;
+		uint32_t Reserved[3];
+	};
+	DALSMC_args_t Args;
+} DALSMC_SetClockFreq_arg_t;
+
+/**
+ * DALSMC_MSG_StutterEfficiency - Report DCN stutter efficiency to PMFW.
+ *
+ * Informs PMFW of the current stutter utilisation for base and low-power stutter
+ * modes so PMFW can adjust memory power policy accordingly.
+ *
+ * Base mode    — lower enter+exit latency (PHY LP1, no UCIE LP).
+ * Low-power mode — higher enter+exit latency (PHY LP2, UCIE LP1).
+ *
+ * Request:  BaseEfficiencyPct[7:0]     — stutter efficiency % in base mode
+ *           LowPowerEfficiencyPct[15:8] — stutter efficiency % in low-power mode
+ * Response: (none beyond result code)
+ */
+typedef union {
+	struct {
+		uint32_t BaseEfficiencyPct     : 8;
+		uint32_t LowPowerEfficiencyPct : 8;
+		uint32_t Reserved              : 16;
+		uint32_t Reserved1[3];
+	};
+	DALSMC_args_t Args;
+} DALSMC_SetStutterEfficiency_arg_t;
+
+/**
+ * DALSMC_MSG_SetMinDeepSleepDcfclk - Set the minimum DCFCLK frequency in deep sleep.
+ *
+ * Request:  MinDcfclkMhz — minimum DCFCLK frequency in MHz during deep sleep
+ * Response: (none beyond result code)
+ */
+typedef union {
+	struct {
+		uint32_t MinDcfclkMhz;
+		uint32_t Reserved[3];
+	};
+	DALSMC_args_t Args;
+} DALSMC_SetMinDeepSleepDcfclk_arg_t;
+
+/**
+ * DALSMC_MSG_IndicatePstateStatus - Indicate to PMFW various DMU behaviors required
+ * to support UCLK P-state, for example whether or not DMU needs to modulate refresh
+ * rate to perform UCLK switches.
+ *
+ * Request:  WaitResp[0]   — DAL requires a synchronous response before proceeding
+ *           DrrEnable[1]  — DRR (dynamic refresh rate modulation) is active
+ *           AltCh[2]      — alternate-channel mode is active
+ *           AllowUclk[16] — DCN can tolerate UCLK P-state switches
+ *           AllowFclk[17] — DCN can tolerate FCLK P-state switches
+ * Response: (none beyond result code)
+ */
+typedef union {
+	struct {
+		uint32_t WaitResp  : 1;
+		uint32_t DrrEnable : 1;
+		uint32_t AltCh     : 1;
+		uint32_t Reserved  : 13;
+		uint32_t AllowUclk : 1;
+		uint32_t AllowFclk : 1;
+		uint32_t Reserved1 : 14;
+		uint32_t Reserved2[3];
+	};
+	DALSMC_args_t Args;
+} DALSMC_IndicatePstateStatus_arg_t;
+
+/**
+ * DALSMC_MSG_TransferTableSmu2Dram - Transfer a PMFW table into DRAM.
+ * DALSMC_MSG_TransferTableDram2Smu - Transfer a DRAM buffer into PMFW.
+ *
+ * Both directions use the same argument layout. The DRAM address must be set
+ * beforehand (AddrLow / AddrHigh are the GPU MC address bits [31:0] / [63:32]).
+ *
+ * Smu2Dram supported tables: TABLE_DAL_INIT (DPM clocks + UTM QoS + memory config)
+ * Dram2Smu supported tables: TABLE_SOC_UTM  (debug override of UTM QoS parameters)
+ *
+ * Request:  TableId  — table identifier (TABLE_* defines below)
+ *           AddrLow  — GPU MC address bits [31:0]  of destination/source buffer
+ *           AddrHigh — GPU MC address bits [63:32] of destination/source buffer
+ * Response: (none beyond result code)
+ */
+typedef union {
+	struct {
+		uint32_t TableId;
+		uint32_t AddrLow;
+		uint32_t AddrHigh;
+		uint32_t Reserved;
+	};
+	DALSMC_args_t Args;
+} DALSMC_TransferTable_arg_t;
+
+/**
+ * DALSMC_MSG_UpdateUTMQoSRequest - Update the active UTM QoS bandwidth/latency request.
+ *
+ * Passes the current display bandwidth and latency requirements to PMFW so it
+ * can select the appropriate SoC operating point (UCLK/FCLK level) from the
+ * UTM table. Called whenever the display configuration changes.
+ *
+ * The QoS requirement must take effect before PMFW sends its response.
+ *
+ * Request:  LatencySopIndex      — index into the UTM SOP table that satisfies latency
+ *           NominalBandwidthKBps — required nominal (average) bandwidth in KB/s
+ *           UrgentBandwidthKBps  — required urgent bandwidth in KB/s
+ *           LsdmaBandwidthKBps   — required LSDMA bandwidth in KB/s
+ * Response: (none beyond result code)
+ */
+typedef union {
+	struct {
+		uint32_t LatencySopIndex;
+		uint32_t NominalBandwidthKBps;
+		uint32_t UrgentBandwidthKBps;
+		uint32_t LsdmaBandwidthKBps;
+	};
+	DALSMC_args_t Args;
+} DALSMC_UpdateUTMQoSRequest_arg_t;
+
+/**
+ * DALSMC_MSG_SetDisplayIdleOptimizations - Notify PMFW of DCN idle-state conditions.
+ *
+ * Indicates which display-side power optimizations are currently safe to apply.
+ * PMFW uses these flags to gate deeper SoC power states such as S0i2.
+ *
+ * Request:  DfRequestDisabled[0] — DF (data fabric) requests from DCN are disabled
+ *           PhyRefClkOff[1]      — PHY reference clock has been gated off
+ *           S0i2Rdy[2]           — DCN is ready for the system to enter S0i2
+ * Response: (none beyond result code)
+ */
+typedef union {
+	struct {
+		uint32_t DfRequestDisabled : 1;
+		uint32_t PhyRefClkOff      : 1;
+		uint32_t S0i2Rdy           : 1;
+		uint32_t Reserved          : 29;
+		uint32_t Reserved1[3];
+	};
+	DALSMC_args_t Args;
+} DALSMC_SetDisplayIdleOptimizations_arg_t;
+unsigned int dcn60_smu_set_hard_min_dcfclk_by_freq(struct clk_mgr_internal *clk_mgr, uint16_t freq_mhz);
+unsigned int dcn60_smu_set_clock_freq(struct clk_mgr_internal *clk_mgr, uint32_t clk, uint16_t freq_mhz);
 void dcn60_smu_set_stutter_efficiency(struct clk_mgr_internal *clk_mgr,
 		uint8_t base_efficiency, uint8_t low_power_efficiency);
 void dcn60_smu_set_min_deep_sleep_dcfclk(struct clk_mgr_internal *clk_mgr, uint32_t freq_mhz);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_smu_driver_if.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_smu_driver_if.h
new file mode 100644
index 00000000000000..ca23518d88b943
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn60/dcn60_smu_driver_if.h
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+//
+// This is a stripped-down version of the smu13_driver_if.h file for the relevant DAL interfaces.
+
+#define DCN6_DRIVER_IF_VERSION  0x1
+
+//Only Clks that have DPM descriptors are listed here
+// This should be the same order as DPM_e
+typedef enum {
+	PPCLK_GFXCLK = 0,
+	PPCLK_NVECLK1,
+	PPCLK_NVECLK2,
+	PPCLK_NVECLK3,
+	PPCLK_LCLK,
+	PPCLK_UCLK,
+	PPCLK_FCLK,
+	PPCLK_G7FCLK,
+	PPCLK_SOCCLK,
+	PPCLK_ACLK,
+	PPCLK_DCFCLK,
+	PPCLK_VPECLK,
+	PPCLK_DISPCLK,
+	PPCLK_DPPCLK,
+	PPCLK_DPREFCLK,
+	PPCLK_CFPUCLK,
+	PPCLK_LOGANCLK,
+	PPCLK_MSPHCLK,
+	PPCLK_MSPLCLK,
+	PPCLK_MSPEDECLK,
+	PPCLK_MOVADCLK,
+	PPCLK_DTBCLK,
+	PPCLK_COUNT,
+} PPCLK_e;
+
+typedef struct {
+  uint8_t  WmSetting;
+  uint8_t  Flags;
+  uint8_t  Padding[2];
+
+} WatermarkRowGeneric_t;
+
+#define NUM_WM_RANGES 4
+
+typedef enum {
+  WATERMARKS_CLOCK_RANGE = 0,
+  WATERMARKS_DUMMY_PSTATE,
+  WATERMARKS_MALL,
+  WATERMARKS_COUNT,
+} WATERMARKS_FLAGS_e;
+
+typedef struct {
+  // Watermarks
+  WatermarkRowGeneric_t WatermarkRow[NUM_WM_RANGES];
+} Watermarks_t;
+
+typedef struct {
+	Watermarks_t Watermarks;
+  uint32_t  Spare[16];
+
+  uint32_t     MmHubPadding[8]; // SMU internal use
+} WatermarksExternal_t;
+
+// Table types
+#define TABLE_PMFW_PPTABLE            0
+#define TABLE_COMBO_PPTABLE           1
+#define TABLE_WATERMARKS              2
+#define TABLE_AVFS_PSM_DEBUG          3
+#define TABLE_PMSTATUSLOG             4
+#define TABLE_SMU_METRICS             5
+#define TABLE_DRIVER_SMU_CONFIG       6
+#define TABLE_ACTIVITY_MONITOR_COEFF  7
+#define TABLE_OVERDRIVE               8
+#define TABLE_I2C_COMMANDS            9
+#define TABLE_DRIVER_INFO             10
+#define TABLE_ECCINFO                 11
+#define TABLE_COUNT                   12
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 519ac878ada1be..c7e34502a81f1b 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -85,7 +85,7 @@
 #include "hw_sequencer_private.h"
 
 #if defined(CONFIG_DRM_AMD_DC_FP)
-#include "dml2_0/dml2_internal_types.h"
+#include "dml2_wrapper/dml2_internal_types.h"
 #include "soc_and_ip_translator.h"
 #endif
 
@@ -992,8 +992,6 @@ static bool dc_construct_ctx(struct dc *dc,
 	if (!dc_ctx)
 		return false;
 
-	dc_stream_init_rmcm_3dlut(dc);
-
 	dc_ctx->cgs_device = init_params->cgs_device;
 	dc_ctx->driver_context = init_params->driver;
 	dc_ctx->dc = dc;
@@ -1220,7 +1218,7 @@ static void apply_ctx_interdependent_lock(struct dc *dc,
 			if (stream == pipe_ctx->stream) {
 				if (resource_is_pipe_type(pipe_ctx, OPP_HEAD) &&
 					(pipe_ctx->plane_state || old_pipe_ctx->plane_state))
-					dc->hwss.pipe_control_lock(dc, pipe_ctx, lock);
+					hwss_pipe_control_lock(dc, pipe_ctx, lock);
 			}
 		}
 	}
@@ -1228,7 +1226,9 @@ static void apply_ctx_interdependent_lock(struct dc *dc,
 
 static void dc_update_visual_confirm_color(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe_ctx)
 {
-	if (dc->debug.visual_confirm & VISUAL_CONFIRM_EXPLICIT) {
+	/* EXPLICIT and DM passthrough both apply the per-plane color set on the plane state */
+	if ((dc->debug.visual_confirm & VISUAL_CONFIRM_EXPLICIT) ||
+		dc->debug.visual_confirm == VISUAL_CONFIRM_DM_PASSTHROUGH) {
 		memcpy(&pipe_ctx->visual_confirm_color, &pipe_ctx->plane_state->visual_confirm_color,
 		sizeof(pipe_ctx->visual_confirm_color));
 		return;
@@ -1869,6 +1869,12 @@ bool dc_validate_boot_timing(const struct dc *dc,
 		return false;
 	}
 
+	if (sink->edid_caps.panel_patch.disable_fec &&
+		(link->fec_state == dc_link_fec_enabled)) {
+		DC_LOG_DEBUG("boot timing validation failed due to FEC WA & FEC HW is active\n");
+		return false;
+	}
+
 	/* Check for enabled DIG to identify enabled display */
 	if (!link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
 		DC_LOG_DEBUG("boot timing validation failed due to disabled DIG\n");
@@ -2272,6 +2278,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	struct dc_stream_state *dc_streams[MAX_STREAMS] = {0};
 	struct dc_state *old_state;
 	bool subvp_prev_use = false;
+	bool dmub_locked = false;
 
 	dc_z10_restore(dc);
 	dc_allow_idle_optimizations(dc, false);
@@ -2323,7 +2330,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	if (dc->hwss.subvp_pipe_control_lock)
 		dc->hwss.subvp_pipe_control_lock(dc, context, true, true, NULL, subvp_prev_use);
 	if (dc->hwss.dmub_hw_control_lock)
-		dc->hwss.dmub_hw_control_lock(dc, context, true);
+		dmub_locked = dc->hwss.dmub_hw_control_lock(dc, context, true);
 
 	if (dc->hwss.update_dsc_pg)
 		dc->hwss.update_dsc_pg(dc, context, false);
@@ -2399,7 +2406,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 		dc->hwss.commit_subvp_config(dc, context);
 	if (dc->hwss.subvp_pipe_control_lock)
 		dc->hwss.subvp_pipe_control_lock(dc, context, false, true, NULL, subvp_prev_use);
-	if (dc->hwss.dmub_hw_control_lock)
+	if (dc->hwss.dmub_hw_control_lock && dmub_locked)
 		dc->hwss.dmub_hw_control_lock(dc, context, false);
 
 	for (i = 0; i < context->stream_count; i++) {
@@ -3177,8 +3184,10 @@ static struct dc_update_descriptor check_update_surfaces_for_stream(
 				stream_update->vrr_active_variable || stream_update->vrr_active_fixed))
 			su_flags->bits.fams_changed = 1;
 
-		if (stream_update->scaler_sharpener_update)
+		if (stream_update->scaler_sharpener_update) {
 			su_flags->bits.scaler_sharpener = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM);
+		}
 
 		if (stream_update->sharpening_required)
 			su_flags->bits.sharpening_required = 1;
@@ -3244,6 +3253,16 @@ static struct dc_update_descriptor check_update_surfaces_for_stream(
 			su_flags->bits.cursor_pos = 1;
 			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 		}
+
+		if (stream_update->func_shaper) {
+			su_flags->bits.func_shaper = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->lut3d_func) {
+			su_flags->bits.lut3d_func = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM);
+		}
 	}
 
 	for (int i = 0 ; i < surface_count; i++) {
@@ -3352,6 +3371,10 @@ static void copy_surface_update_to_plane(
 			surface->time.index = 0;
 
 		surface->triplebuffer_flips = srf_update->flip_addr->triplebuffer_flips;
+
+		/* DM passthrough mode: stash the per-flip color on the plane for dc_update_visual_confirm_color */
+		if (surface->ctx->dc->debug.visual_confirm == VISUAL_CONFIRM_DM_PASSTHROUGH)
+			surface->visual_confirm_color = srf_update->flip_addr->visual_confirm_color;
 	}
 
 	if (srf_update->scaling_info) {
@@ -3950,9 +3973,8 @@ static void program_cursor_attributes_sequence(
 
 		hwss_add_set_cursor_attribute(seq_state, dc, tmp_pipe);
 		if (dc->ctx->dmub_srv)
-			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe, k);
-		if (dc->hwss.set_cursor_sdr_white_level)
-			hwss_add_set_cursor_sdr_white_level(seq_state, dc, tmp_pipe);
+			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe);
+		hwss_add_set_cursor_sdr_white_level(seq_state, tmp_pipe);
 		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe)
 			hwss_add_update_cursor_offload_pipe(seq_state, dc, tmp_pipe);
 	}
@@ -4002,7 +4024,7 @@ static void program_cursor_position_sequence(
 			hwss_add_update_cursor_offload_pipe(seq_state, dc, tmp_pipe);
 
 		if (dc->ctx->dmub_srv)
-			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe, k);
+			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe);
 	}
 
 	if (pipe_to_program) {
@@ -4368,8 +4390,8 @@ static void commit_planes_do_stream_update(struct dc *dc,
 
 		if (resource_is_pipe_type(pipe_ctx, OTG_MASTER) && pipe_ctx->stream == stream) {
 
-			if (stream_update->periodic_interrupt && dc->hwss.setup_periodic_interrupt)
-				dc->hwss.setup_periodic_interrupt(dc, pipe_ctx);
+			if (stream_update->periodic_interrupt)
+				hwss_setup_periodic_interrupt(dc, pipe_ctx);
 
 			if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) ||
 					stream_update->output_color_space ||
@@ -4874,6 +4896,7 @@ static void commit_planes_for_stream(struct dc *dc,
 	bool should_lock_all_pipes = (update_type != UPDATE_TYPE_FAST);
 	bool subvp_prev_use = false;
 	bool subvp_curr_use = false;
+	bool dmub_locked = false;
 	uint8_t current_stream_mask = 0;
 
 	if (should_lock_all_pipes)
@@ -4969,9 +4992,7 @@ static void commit_planes_for_stream(struct dc *dc,
 						top_pipe_to_program->stream_res.tg);
 		}
 
-	if (dc->hwss.wait_for_dcc_meta_propagation) {
-		dc->hwss.wait_for_dcc_meta_propagation(dc, top_pipe_to_program);
-	}
+	hwss_hubp_wait_for_dcc_meta_prop(dc, top_pipe_to_program);
 
 	if (dc->hwseq->funcs.wait_for_pipe_update_if_needed)
 		dc->hwseq->funcs.wait_for_pipe_update_if_needed(dc, top_pipe_to_program, update_type < UPDATE_TYPE_FULL);
@@ -4981,7 +5002,7 @@ static void commit_planes_for_stream(struct dc *dc,
 			dc->hwss.subvp_pipe_control_lock(dc, context, true, should_lock_all_pipes, NULL, subvp_prev_use);
 
 		if (dc->hwss.dmub_hw_control_lock)
-			dc->hwss.dmub_hw_control_lock(dc, context, true);
+			dmub_locked = dc->hwss.dmub_hw_control_lock(dc, context, true);
 
 		dc->hwss.interdependent_update_lock(dc, context, true);
 	} else {
@@ -4989,13 +5010,13 @@ static void commit_planes_for_stream(struct dc *dc,
 			dc->hwss.subvp_pipe_control_lock(dc, context, true, should_lock_all_pipes, top_pipe_to_program, subvp_prev_use);
 
 		if (dc->hwss.dmub_hw_control_lock)
-			dc->hwss.dmub_hw_control_lock(dc, context, true);
+			dmub_locked = dc->hwss.dmub_hw_control_lock(dc, context, true);
 
 		/* Lock the top pipe while updating plane addrs, since freesync requires
 		 *  plane addr update event triggers to be synchronized.
 		 *  top_pipe_to_program is expected to never be NULL
 		 */
-		dc->hwss.pipe_control_lock(dc, top_pipe_to_program, true);
+		hwss_pipe_control_lock(dc, top_pipe_to_program, true);
 	}
 
 	dc_dmub_update_dirty_rect(dc, surface_count, stream, srf_updates, context);
@@ -5017,7 +5038,7 @@ static void commit_planes_for_stream(struct dc *dc,
 		if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
 			dc->hwss.interdependent_update_lock(dc, context, false);
 		} else {
-			dc->hwss.pipe_control_lock(dc, top_pipe_to_program, false);
+			hwss_pipe_control_lock(dc, top_pipe_to_program, false);
 		}
 		dc->hwss.post_unlock_program_front_end(dc, context);
 
@@ -5032,7 +5053,7 @@ static void commit_planes_for_stream(struct dc *dc,
 			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes,
 							 NULL, subvp_prev_use);
 
-		if (dc->hwss.dmub_hw_control_lock)
+		if (dc->hwss.dmub_hw_control_lock && dmub_locked)
 			dc->hwss.dmub_hw_control_lock(dc, context, false);
 		return;
 	}
@@ -5118,7 +5139,7 @@ static void commit_planes_for_stream(struct dc *dc,
 		if (dc->hwss.program_triplebuffer != NULL && dc->debug.enable_tri_buf) {
 			/*turn off triple buffer for full update*/
 			dc->hwss.program_triplebuffer(
-				dc, pipe_ctx, pipe_ctx->plane_state->triplebuffer_flips);
+				pipe_ctx->plane_res.hubp, pipe_ctx->plane_state->triplebuffer_flips);
 		}
 	}
 
@@ -5193,7 +5214,7 @@ static void commit_planes_for_stream(struct dc *dc,
 				if (dc->hwss.program_triplebuffer != NULL && dc->debug.enable_tri_buf) {
 					/*only enable triplebuffer for fast_update*/
 					dc->hwss.program_triplebuffer(
-						dc, pipe_ctx, pipe_ctx->plane_state->triplebuffer_flips);
+						pipe_ctx->plane_res.hubp, pipe_ctx->plane_state->triplebuffer_flips);
 				}
 				if (pipe_ctx->plane_state->update_bits.addr_update)
 					dc->hwss.update_plane_addr(dc, pipe_ctx);
@@ -5204,7 +5225,7 @@ static void commit_planes_for_stream(struct dc *dc,
 	if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
 		dc->hwss.interdependent_update_lock(dc, context, false);
 	} else {
-		dc->hwss.pipe_control_lock(dc, top_pipe_to_program, false);
+		hwss_pipe_control_lock(dc, top_pipe_to_program, false);
 	}
 
 	if ((update_type != UPDATE_TYPE_FAST) && stream->update_flags.bits.dsc_changed)
@@ -5268,12 +5289,12 @@ static void commit_planes_for_stream(struct dc *dc,
 	if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
 		if (dc->hwss.subvp_pipe_control_lock)
 			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
-		if (dc->hwss.dmub_hw_control_lock)
+		if (dc->hwss.dmub_hw_control_lock && dmub_locked)
 			dc->hwss.dmub_hw_control_lock(dc, context, false);
 	} else {
 		if (dc->hwss.subvp_pipe_control_lock)
 			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, top_pipe_to_program, subvp_prev_use);
-		if (dc->hwss.dmub_hw_control_lock)
+		if (dc->hwss.dmub_hw_control_lock && dmub_locked)
 			dc->hwss.dmub_hw_control_lock(dc, context, false);
 	}
 
@@ -5732,127 +5753,6 @@ static bool commit_minimal_transition_state(struct dc *dc,
 	return true;
 }
 
-void populate_fast_updates(struct dc_fast_update *fast_update,
-		struct dc_surface_update *srf_updates,
-		int surface_count,
-		struct dc_stream_update *stream_update)
-{
-	int i = 0;
-
-	if (stream_update) {
-		fast_update[0].out_transfer_func = stream_update->out_transfer_func;
-		fast_update[0].output_csc_transform = stream_update->output_csc_transform;
-		fast_update[0].cursor_attributes = stream_update->cursor_attributes;
-		fast_update[0].cursor_position = stream_update->cursor_position;
-		fast_update[0].periodic_interrupt = stream_update->periodic_interrupt;
-		fast_update[0].dither_option = stream_update->dither_option;
-		fast_update[0].gamut_remap = stream_update->gamut_remap;
-		fast_update[0].vrr_infopacket = stream_update->vrr_infopacket;
-		fast_update[0].vsc_infopacket = stream_update->vsc_infopacket;
-		fast_update[0].vsp_infopacket = stream_update->vsp_infopacket;
-		fast_update[0].hfvsif_infopacket = stream_update->hfvsif_infopacket;
-		fast_update[0].vtem_infopacket = stream_update->vtem_infopacket;
-		fast_update[0].adaptive_sync_infopacket = stream_update->adaptive_sync_infopacket;
-		fast_update[0].avi_infopacket = stream_update->avi_infopacket;
-		fast_update[0].hdr_static_metadata = stream_update->hdr_static_metadata;
-	} else {
-		fast_update[0].out_transfer_func = NULL;
-		fast_update[0].output_csc_transform = NULL;
-		fast_update[0].cursor_attributes = NULL;
-		fast_update[0].cursor_position = NULL;
-		fast_update[0].periodic_interrupt = NULL;
-		fast_update[0].dither_option = NULL;
-		fast_update[0].gamut_remap = NULL;
-		fast_update[0].vrr_infopacket = NULL;
-		fast_update[0].vsc_infopacket = NULL;
-		fast_update[0].vsp_infopacket = NULL;
-		fast_update[0].hfvsif_infopacket = NULL;
-		fast_update[0].vtem_infopacket = NULL;
-		fast_update[0].adaptive_sync_infopacket = NULL;
-		fast_update[0].avi_infopacket = NULL;
-		fast_update[0].hdr_static_metadata = NULL;
-	}
-
-	for (i = 0; i < surface_count; i++) {
-		fast_update[i].flip_addr = srf_updates[i].flip_addr;
-		fast_update[i].gamma = srf_updates[i].gamma;
-		fast_update[i].gamut_remap_matrix = srf_updates[i].gamut_remap_matrix;
-		fast_update[i].input_csc_color_matrix = srf_updates[i].input_csc_color_matrix;
-		fast_update[i].coeff_reduction_factor = srf_updates[i].coeff_reduction_factor;
-		fast_update[i].cursor_csc_color_matrix = srf_updates[i].cursor_csc_color_matrix;
-		fast_update[i].cm_hist_control = srf_updates[i].cm_hist_control;
-	}
-}
-
-static bool fast_updates_exist(const struct dc_fast_update *fast_update, int surface_count)
-{
-	int i;
-
-	if (fast_update[0].out_transfer_func ||
-		fast_update[0].output_csc_transform ||
-		fast_update[0].cursor_attributes ||
-		fast_update[0].cursor_position ||
-		fast_update[0].periodic_interrupt ||
-		fast_update[0].dither_option ||
-		fast_update[0].gamut_remap ||
-		fast_update[0].vrr_infopacket ||
-		fast_update[0].vsc_infopacket ||
-		fast_update[0].vsp_infopacket ||
-		fast_update[0].hfvsif_infopacket ||
-		fast_update[0].vtem_infopacket ||
-		fast_update[0].adaptive_sync_infopacket ||
-		fast_update[0].avi_infopacket ||
-		fast_update[0].hdr_static_metadata)
-		return true;
-
-	for (i = 0; i < surface_count; i++) {
-		if (fast_update[i].flip_addr ||
-				fast_update[i].gamma ||
-				fast_update[i].gamut_remap_matrix ||
-				fast_update[i].input_csc_color_matrix ||
-				fast_update[i].cursor_csc_color_matrix ||
-				fast_update[i].cm_hist_control ||
-				fast_update[i].coeff_reduction_factor)
-			return true;
-	}
-
-	return false;
-}
-
-bool fast_nonaddr_updates_exist(struct dc_fast_update *fast_update, int surface_count)
-{
-	int i;
-
-	if (fast_update[0].out_transfer_func ||
-		fast_update[0].output_csc_transform ||
-		fast_update[0].gamut_remap ||
-		fast_update[0].cursor_attributes ||
-		fast_update[0].cursor_position ||
-		fast_update[0].periodic_interrupt ||
-		fast_update[0].dither_option ||
-		fast_update[0].vrr_infopacket ||
-		fast_update[0].vsc_infopacket ||
-		fast_update[0].vsp_infopacket ||
-		fast_update[0].hfvsif_infopacket ||
-		fast_update[0].vtem_infopacket ||
-		fast_update[0].adaptive_sync_infopacket ||
-		fast_update[0].avi_infopacket ||
-		fast_update[0].hdr_static_metadata)
-		return true;
-
-	for (i = 0; i < surface_count; i++) {
-		if (fast_update[i].input_csc_color_matrix ||
-				fast_update[i].gamma ||
-				fast_update[i].gamut_remap_matrix ||
-				fast_update[i].coeff_reduction_factor ||
-				fast_update[i].cm_hist_control ||
-				fast_update[i].cursor_csc_color_matrix)
-			return true;
-	}
-
-	return false;
-}
-
 static bool full_update_required_weak(
 		const struct dc *dc,
 		const struct dc_surface_update *srf_updates,
@@ -5881,67 +5781,6 @@ static bool full_update_required_weak(
 	return false;
 }
 
-static bool full_update_required(
-		const struct dc *dc,
-		const struct dc_surface_update *srf_updates,
-		int surface_count,
-		const struct dc_stream_update *stream_update,
-		const struct dc_stream_state *stream)
-{
-	if (full_update_required_weak(dc, srf_updates, surface_count, stream_update, stream))
-		return true;
-
-	for (int i = 0; i < surface_count; i++) {
-		if (srf_updates &&
-				(srf_updates[i].plane_info ||
-				srf_updates[i].scaling_info ||
-				(srf_updates[i].hdr_mult.value &&
-				srf_updates[i].hdr_mult.value != srf_updates->surface->hdr_mult.value) ||
-				(srf_updates[i].sdr_white_level_nits &&
-				srf_updates[i].sdr_white_level_nits != srf_updates->surface->sdr_white_level_nits) ||
-				srf_updates[i].in_transfer_func ||
-				srf_updates[i].surface->force_full_update ||
-				(srf_updates[i].flip_addr &&
-				srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface)))
-			return true;
-	}
-
-	if (stream_update &&
-			(((stream_update->src.height != 0 && stream_update->src.width != 0) ||
-			(stream_update->dst.height != 0 && stream_update->dst.width != 0) ||
-			stream_update->integer_scaling_update) ||
-			stream_update->abm_level ||
-			stream_update->dpms_off ||
-			stream_update->allow_freesync ||
-			stream_update->vrr_active_variable ||
-			stream_update->vrr_active_fixed ||
-			stream_update->output_color_space ||
-			stream_update->wb_update ||
-			stream_update->dsc_config ||
-			stream_update->mst_bw_update ||
-			stream_update->func_shaper ||
-			stream_update->lut3d_func ||
-			stream_update->pending_test_pattern ||
-			stream_update->crtc_timing_adjust ||
-			stream_update->scaler_sharpener_update ||
-			stream_update->hw_cursor_req))
-		return true;
-
-	return false;
-}
-
-static bool fast_update_only(
-		const struct dc *dc,
-		const struct dc_fast_update *fast_update,
-		const struct dc_surface_update *srf_updates,
-		int surface_count,
-		const struct dc_stream_update *stream_update,
-		const struct dc_stream_state *stream)
-{
-	return fast_updates_exist(fast_update, surface_count)
-			&& !full_update_required(dc, srf_updates, surface_count, stream_update, stream);
-}
-
 static bool update_planes_and_stream_v2(struct dc *dc,
 		struct dc_surface_update *srf_updates, int surface_count,
 		struct dc_stream_state *stream,
@@ -5949,7 +5788,6 @@ static bool update_planes_and_stream_v2(struct dc *dc,
 {
 	struct dc_state *context;
 	enum dc_update_type update_type;
-	struct dc_fast_update fast_update[MAX_SURFACES] = {0};
 
 	/* In cases where MPO and split or ODM are used transitions can
 	 * cause underflow. Apply stream configuration with minimal pipe
@@ -5957,11 +5795,7 @@ static bool update_planes_and_stream_v2(struct dc *dc,
 	 */
 	bool force_minimal_pipe_splitting = 0;
 	bool is_plane_addition = 0;
-	bool is_fast_update_only;
 
-	populate_fast_updates(fast_update, srf_updates, surface_count, stream_update);
-	is_fast_update_only = fast_update_only(dc, fast_update, srf_updates,
-			surface_count, stream_update, stream);
 	force_minimal_pipe_splitting = could_mpcc_tree_change_for_active_pipes(
 			dc,
 			stream,
@@ -5999,7 +5833,7 @@ static bool update_planes_and_stream_v2(struct dc *dc,
 		commit_minimal_transition_state_in_dc_update(dc, context, stream,
 				srf_updates, surface_count);
 
-	if (is_fast_update_only && !dc->check_config.enable_legacy_fast_update) {
+	if (update_type == UPDATE_TYPE_FAST && !dc->check_config.enable_legacy_fast_update) {
 		commit_planes_for_stream_fast(dc,
 				srf_updates,
 				surface_count,
@@ -6035,13 +5869,8 @@ static void commit_planes_and_stream_update_on_current_context(struct dc *dc,
 		struct dc_stream_update *stream_update,
 		enum dc_update_type update_type)
 {
-	struct dc_fast_update fast_update[MAX_SURFACES] = {0};
-
 	ASSERT(update_type < UPDATE_TYPE_FULL);
-	populate_fast_updates(fast_update, srf_updates, surface_count,
-			stream_update);
-	if (fast_update_only(dc, fast_update, srf_updates, surface_count,
-			stream_update, stream) &&
+	if (update_type == UPDATE_TYPE_FAST &&
 			!dc->check_config.enable_legacy_fast_update)
 		commit_planes_for_stream_fast(dc,
 				srf_updates,
@@ -8237,23 +8066,6 @@ static bool update_planes_and_stream_prepare_v3(
 	ASSERT(scratch->flow == UPDATE_V3_FLOW_INVALID);
 	dc_exit_ips_for_hw_access(scratch->dc);
 
-	/* HWSS path determination needs to be done prior to updating the surface and stream states. */
-	struct dc_fast_update fast_update[MAX_SURFACES] = { 0 };
-
-	populate_fast_updates(fast_update,
-			      scratch->surface_updates,
-			      scratch->surface_count,
-			      scratch->stream_update);
-
-	const bool is_hwss_fast_path_only =
-		fast_update_only(scratch->dc,
-				 fast_update,
-				 scratch->surface_updates,
-				 scratch->surface_count,
-				 scratch->stream_update,
-				 scratch->stream) &&
-		!scratch->dc->check_config.enable_legacy_fast_update;
-
 	if (!update_planes_and_stream_state(
 			scratch->dc,
 			scratch->surface_updates,
@@ -8269,7 +8081,8 @@ static bool update_planes_and_stream_prepare_v3(
 	if (scratch->new_context == scratch->dc->current_state) {
 		ASSERT(scratch->update_type < UPDATE_TYPE_FULL);
 
-		scratch->flow = is_hwss_fast_path_only
+		scratch->flow = (scratch->update_type == UPDATE_TYPE_FAST &&
+				!scratch->dc->check_config.enable_legacy_fast_update)
 				? UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST
 				: UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FULL;
 		return true;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
index 2933b660e3d72c..f3348e0cb1d68f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
@@ -274,6 +274,8 @@ char *dc_status_to_str(enum dc_status status)
 		return "HW Cursor not supported";
 	case DC_FAIL_DP_TUNNEL_BW_VALIDATE:
 		return "Fail DP Tunnel BW validation";
+	case DC_NO_DRAM_BUFFER_RESOURCE:
+		return "No DRAM buffer resource";
 	case DC_ERROR_UNEXPECTED:
 		return "Unexpected error";
 	default:
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index ef4987d449cb2c..28897addb553f2 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -754,6 +754,61 @@ static bool get_update_dchubp_dpp_flags_status(struct pipe_ctx *pipe)
 	return false;
 }
 
+static void calc_vline_position(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		uint32_t *start_line,
+		uint32_t *end_line)
+{
+	if (!dc->hwss.get_vupdate_offset_from_vsync)
+		return;
+
+	const struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
+	int vline_pos = pipe_ctx->stream->periodic_interrupt.lines_offset;
+
+	if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_UPDATE) {
+		if (vline_pos > 0)
+			vline_pos--;
+		else if (vline_pos < 0)
+			vline_pos++;
+
+		vline_pos += dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx);
+		if (vline_pos >= 0)
+			*start_line = vline_pos - ((vline_pos / timing->v_total) * timing->v_total);
+		else
+			*start_line = vline_pos + ((-vline_pos / timing->v_total) + 1) * timing->v_total - 1;
+		*end_line = (*start_line + 2) % timing->v_total;
+	} else if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_SYNC) {
+		// vsync is line 0 so start_line is just the requested line offset
+		*start_line = vline_pos;
+		*end_line = (*start_line + 2) % timing->v_total;
+	} else
+		ASSERT(0);
+}
+
+struct dpp_cursor_attributes calc_sdr_cursor_attributes(struct pipe_ctx *pipe_ctx)
+{
+	uint32_t sdr_white_level = pipe_ctx->stream->cursor_attributes.sdr_white_level;
+	struct fixed31_32 multiplier;
+	struct dpp_cursor_attributes opt_attr = { 0 };
+	uint32_t hw_scale = 0x3c00; // 1.0 default multiplier
+	struct custom_float_format fmt;
+
+	fmt.exponenta_bits = 5;
+	fmt.mantissa_bits = 10;
+	fmt.sign = true;
+
+	if (sdr_white_level > 80) {
+		multiplier = dc_fixpt_from_fraction(sdr_white_level, 80);
+		convert_to_custom_float_format(multiplier, &fmt, &hw_scale);
+	}
+
+	opt_attr.scale = hw_scale;
+	opt_attr.bias = 0;
+
+	return opt_attr;
+}
+
 // Function to check if any update flags are set
 static bool get_pipe_update_bits_status(struct pipe_ctx *pipe, struct dc_plane_state *plane, struct dc_stream_state *stream)
 {
@@ -800,7 +855,9 @@ void hwss_build_full_sequence(struct dc *dc,
 
 			if (pipe->plane_state) {
 				/* Turn off triple buffer for full update */
-				hwss_add_hubp_program_triplebuffer(&seq_state, dc, pipe, pipe->plane_state->triplebuffer_flips);
+				hwss_add_hubp_program_triplebuffer(&seq_state,
+					pipe->plane_res.hubp,
+					pipe->plane_state->triplebuffer_flips);
 			}
 		}
 	}
@@ -968,11 +1025,9 @@ void hwss_build_full_sequence(struct dc *dc,
 					if (current_pipe->plane_res.hubp->funcs->hubp_program_mcache_id_and_split_coordinate)
 						hwss_add_hubp_program_mcache_id(&seq_state, current_pipe->plane_res.hubp, &current_pipe->mcache_regs);
 
-					if (current_pipe->plane_res.dpp->funcs->dpp_program_upsp) {
-						block_sequence[*num_steps].params.program_upsp_params.pipe_ctx = current_pipe;
-						block_sequence[*num_steps].func = DPP_PROGRAM_UPSP;
-						(*num_steps)++;
-					}
+					if (current_pipe->plane_res.dpp->funcs->dpp_program_upsp)
+						hwss_add_dpp_program_upsp(&seq_state, current_pipe->plane_res.dpp,
+							&current_pipe->plane_res.scl_data.dscl_prog_data);
 				}
 			}
 			current_pipe = current_pipe->bottom_pipe;
@@ -1019,7 +1074,7 @@ void hwss_build_post_unlock_full_sequence(struct dc *dc,
 
 			unsigned int polling_interval_us;
 				polling_interval_us = 1;
-			hwss_add_hubp_wait_flip_pending(&seq_state, pipe->plane_res.hubp, 100000, polling_interval_us);
+			hwss_add_hubp_wait_flip_pending(&seq_state, pipe->plane_res.hubp, polling_interval_us);
 		}
 	}
 
@@ -1049,11 +1104,13 @@ void hwss_build_post_unlock_full_sequence(struct dc *dc,
 		if (pipe->plane_state && !pipe->top_pipe) {
 			while (pipe) {
 				if (pipe->stream && dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_PHANTOM) {
-					/* Apply update flags for phantom pipes using existing HWS functions */
+					/* Phantom SW state must be updated before the pipe sequence is built,
+					 * since the build reads the update flags and scaling params.
+					 */
 					if (dc->hwss.apply_update_flags_for_phantom)
-						hwss_add_hws_apply_update_flags_for_phantom(&seq_state, pipe);
+						dc->hwss.apply_update_flags_for_phantom(pipe);
 					if (dc->hwss.update_phantom_vp_position)
-						hwss_add_hws_update_phantom_vp_position(&seq_state, dc, context, pipe);
+						dc->hwss.update_phantom_vp_position(dc, context, pipe);
 
 					/* Program the phantom pipe - use program_pipe_sequence if available */
 					if (dc->hwseq && dc->hwseq->funcs.program_pipe_sequence)
@@ -1119,6 +1176,22 @@ void hwss_build_post_unlock_full_sequence(struct dc *dc,
 	}
 }
 
+static uint32_t get_dcc_meta_propagation_delay(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	/* check if any surfaces are updating address while using flip immediate and dcc */
+	while (pipe_ctx != NULL) {
+		if (pipe_ctx->plane_state &&
+				pipe_ctx->plane_state->dcc.enable &&
+				pipe_ctx->plane_state->flip_immediate &&
+				pipe_ctx->plane_state->update_bits.addr_update) {
+			return dc->debug.dcc_meta_propagation_delay_us;
+		}
+		/* check next pipe */
+		pipe_ctx = pipe_ctx->bottom_pipe;
+	}
+	return 0;
+}
+
 void hwss_build_fast_sequence(struct dc *dc,
 		struct dc_dmub_cmd *dc_dmub_cmd,
 		unsigned int dmub_cmd_count,
@@ -1135,6 +1208,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 	struct pipe_ctx *current_mpc_pipe = NULL;
 	bool is_dmub_lock_required = false;
 	unsigned int i = 0;
+	struct block_sequence_state seq_state = { .steps = block_sequence, .num_steps = num_steps };
 
 	*num_steps = 0; // Initialize to 0
 
@@ -1142,8 +1216,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 		return;
 
 	if (dc->hwss.wait_for_dcc_meta_propagation) {
-		block_sequence[*num_steps].params.wait_for_dcc_meta_propagation_params.dc = dc;
-		block_sequence[*num_steps].params.wait_for_dcc_meta_propagation_params.top_pipe_to_program = pipe_ctx;
+		block_sequence[*num_steps].params.wait_for_dcc_meta_propagation_params.delay = get_dcc_meta_propagation_delay(dc, pipe_ctx);
 		block_sequence[*num_steps].func = HUBP_WAIT_FOR_DCC_META_PROP;
 		(*num_steps)++;
 	}
@@ -1165,13 +1238,9 @@ void hwss_build_fast_sequence(struct dc *dc,
 		block_sequence[*num_steps].func = DMUB_HW_CONTROL_LOCK_FAST;
 		(*num_steps)++;
 	}
-	if (dc->hwss.pipe_control_lock) {
-		block_sequence[*num_steps].params.pipe_control_lock_params.dc = dc;
-		block_sequence[*num_steps].params.pipe_control_lock_params.lock = true;
-		block_sequence[*num_steps].params.pipe_control_lock_params.pipe_ctx = pipe_ctx;
-		block_sequence[*num_steps].func = OPTC_PIPE_CONTROL_LOCK;
-		(*num_steps)++;
-	}
+	hwss_add_optc_pipe_control_lock(
+			&(struct block_sequence_state){ block_sequence, num_steps },
+			dc, pipe_ctx, true);
 
 	for (i = 0; i < dmub_cmd_count; i++) {
 		block_sequence[*num_steps].params.send_dmcub_cmd_params.ctx = dc->ctx;
@@ -1182,8 +1251,16 @@ void hwss_build_fast_sequence(struct dc *dc,
 	}
 
 	if (dc->hwss.setup_periodic_interrupt && stream->update_flags.bits.periodic_interrupt) {
-		block_sequence[*num_steps].params.setup_periodic_interrupt_params.dc = dc;
-		block_sequence[*num_steps].params.setup_periodic_interrupt_params.pipe_ctx = pipe_ctx;
+		uint32_t start_line = 0;
+		uint32_t end_line = 0;
+
+		calc_vline_position(dc, pipe_ctx, &start_line, &end_line);
+		block_sequence[*num_steps].params.setup_periodic_interrupt_params.tg =
+				pipe_ctx->stream_res.tg;
+		block_sequence[*num_steps].params.setup_periodic_interrupt_params.start_line =
+				start_line;
+		block_sequence[*num_steps].params.setup_periodic_interrupt_params.end_line =
+				end_line;
 		block_sequence[*num_steps].func = HWSS_SETUP_PERIODIC_INTERRUPT;
 		(*num_steps)++;
 	}
@@ -1224,11 +1301,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 	}
 
 	/* Track cursor lock state - separate locks for attribute and position updates */
-	bool enable_cursor_offload = false;
-
-	if ((dc->hwss.set_cursor_attribute && stream->update_flags.bits.cursor_attr) ||
-		(dc->hwss.set_cursor_position && stream->update_flags.bits.cursor_pos))
-		enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
 
 	/* Cursor attribute updates - separate lock/iterate/unlock */
 	if (dc->hwss.set_cursor_attribute && stream->update_flags.bits.cursor_attr) {
@@ -1244,11 +1317,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 				cursor_pipe_to_program = current_pipe;
 
 				if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
-					block_sequence[*num_steps].params.begin_cursor_offload_update_params.dc = dc;
-					block_sequence[*num_steps].params.begin_cursor_offload_update_params.pipe_ctx =
-						current_pipe;
-					block_sequence[*num_steps].func = HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE;
-					(*num_steps)++;
+					hwss_add_begin_cursor_offload_update(&seq_state, dc, current_pipe);
 				} else {
 					block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
 					block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = current_pipe;
@@ -1281,38 +1350,22 @@ void hwss_build_fast_sequence(struct dc *dc,
 			block_sequence[*num_steps].func = DPP_SET_CURSOR_ATTRIBUTES;
 			(*num_steps)++;
 
-			if (dc->ctx->dmub_srv) {
-				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_ctx =
-					current_pipe;
-				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_idx =
-					current_pipe->pipe_idx;
-				block_sequence[*num_steps].func = DC_SEND_CURSOR_INFO_TO_DMU;
-				(*num_steps)++;
-			}
+			if (dc->ctx->dmub_srv)
+				hwss_add_send_update_cursor_info_to_dmu(
+					&(struct block_sequence_state){ block_sequence, num_steps },
+					current_pipe);
 
-			block_sequence[*num_steps].params.set_cursor_sdr_white_level_params.dc = dc;
-			block_sequence[*num_steps].params.set_cursor_sdr_white_level_params.pipe_ctx =
-				current_pipe;
-			block_sequence[*num_steps].func = SET_CURSOR_SDR_WHITE_LEVEL;
-			(*num_steps)++;
+			hwss_add_set_cursor_sdr_white_level(&seq_state, current_pipe);
 
 			if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe) {
-				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.dc = dc;
-				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.pipe_ctx =
-					current_pipe;
-				block_sequence[*num_steps].func = HWSS_UPDATE_CURSOR_OFFLOAD_PIPE;
-				(*num_steps)++;
+				hwss_add_update_cursor_offload_pipe(&seq_state, dc, current_pipe);
 			}
 		}
 
 		/* Unlock cursor attributes after all pipes have been programmed */
 		if (cursor_pipe_to_program) {
 			if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
-				block_sequence[*num_steps].params.commit_cursor_offload_update_params.dc = dc;
-				block_sequence[*num_steps].params.commit_cursor_offload_update_params.pipe_ctx =
-					cursor_pipe_to_program;
-				block_sequence[*num_steps].func = HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE;
-				(*num_steps)++;
+				hwss_add_commit_cursor_offload_update(&seq_state, dc, cursor_pipe_to_program);
 			} else {
 				block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
 				block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = cursor_pipe_to_program;
@@ -1333,10 +1386,14 @@ void hwss_build_fast_sequence(struct dc *dc,
 	}
 
 	/* Cursor position updates */
-	if (dc->hwss.set_cursor_position && stream->update_flags.bits.cursor_pos) {
+	if (dc->hwseq && dc->hwseq->funcs.build_cursor_pos_update_params &&
+			dc->hwss.set_cursor_position && stream->update_flags.bits.cursor_pos) {
 		struct pipe_ctx *cursor_pipe_to_program = NULL;
 
 		for (i = 0; i < MAX_PIPES; i++) {
+			struct dc_cursor_position pos;
+			struct dc_cursor_mi_param param;
+
 			current_pipe = &context->res_ctx.pipe_ctx[i];
 
 			if (current_pipe->stream != stream ||
@@ -1350,11 +1407,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 				cursor_pipe_to_program = current_pipe;
 
 				if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
-					block_sequence[*num_steps].params.begin_cursor_offload_update_params.dc = dc;
-					block_sequence[*num_steps].params.begin_cursor_offload_update_params.pipe_ctx =
-						current_pipe;
-					block_sequence[*num_steps].func = HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE;
-					(*num_steps)++;
+					hwss_add_begin_cursor_offload_update(&seq_state, dc, current_pipe);
 				} else {
 					block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
 					block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = current_pipe;
@@ -1364,37 +1417,30 @@ void hwss_build_fast_sequence(struct dc *dc,
 				}
 			}
 
+			dc->hwseq->funcs.build_cursor_pos_update_params(current_pipe, &pos, &param);
+
 			block_sequence[*num_steps].params.set_cursor_position_params.dc = dc;
-			block_sequence[*num_steps].params.set_cursor_position_params.pipe_ctx = current_pipe;
+			block_sequence[*num_steps].params.set_cursor_position_params.hubp = current_pipe->plane_res.hubp;
+			block_sequence[*num_steps].params.set_cursor_position_params.dpp = current_pipe->plane_res.dpp;
+			block_sequence[*num_steps].params.set_cursor_position_params.pos = pos;
+			block_sequence[*num_steps].params.set_cursor_position_params.param = param;
 			block_sequence[*num_steps].func = SET_CURSOR_POSITION;
 			(*num_steps)++;
 
 			if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe) {
-				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.dc = dc;
-				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.pipe_ctx =
-					current_pipe;
-				block_sequence[*num_steps].func = HWSS_UPDATE_CURSOR_OFFLOAD_PIPE;
-				(*num_steps)++;
+				hwss_add_update_cursor_offload_pipe(&seq_state, dc, current_pipe);
 			}
 
-			if (dc->ctx->dmub_srv) {
-				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_ctx =
-					current_pipe;
-				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_idx =
-					current_pipe->pipe_idx;
-				block_sequence[*num_steps].func = DC_SEND_CURSOR_INFO_TO_DMU;
-				(*num_steps)++;
-			}
+			if (dc->ctx->dmub_srv)
+				hwss_add_send_update_cursor_info_to_dmu(
+					&(struct block_sequence_state){ block_sequence, num_steps },
+					current_pipe);
 		}
 
 		/* Unlock cursor position after all pipes have been programmed */
 		if (cursor_pipe_to_program) {
 			if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
-				block_sequence[*num_steps].params.commit_cursor_offload_update_params.dc = dc;
-				block_sequence[*num_steps].params.commit_cursor_offload_update_params.pipe_ctx =
-					cursor_pipe_to_program;
-				block_sequence[*num_steps].func = HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE;
-				(*num_steps)++;
+				hwss_add_commit_cursor_offload_update(&seq_state, dc, cursor_pipe_to_program);
 			} else {
 				block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
 				block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = cursor_pipe_to_program;
@@ -1426,13 +1472,11 @@ void hwss_build_fast_sequence(struct dc *dc,
 					(*num_steps)++;
 				}
 				if (dc->hwss.program_triplebuffer && dc->debug.enable_tri_buf && dc_pipe_update_bits_is_any_set(&current_mpc_pipe->plane_state->update_bits)) {
-					block_sequence[*num_steps].params.program_triplebuffer_params.dc = dc;
-					block_sequence[*num_steps].params.program_triplebuffer_params.pipe_ctx = current_mpc_pipe;
-					block_sequence[*num_steps].params.program_triplebuffer_params.enableTripleBuffer = current_mpc_pipe->plane_state->triplebuffer_flips;
-					block_sequence[*num_steps].func = HUBP_PROGRAM_TRIPLEBUFFER;
-					(*num_steps)++;
+					hwss_add_hubp_program_triplebuffer(&seq_state,
+							current_mpc_pipe->plane_res.hubp,
+							current_mpc_pipe->plane_state->triplebuffer_flips);
 				}
-				if (dc->hwss.update_plane_addr && current_mpc_pipe->plane_state->update_bits.addr_update) {
+				if (dc->hwss.prepare_plane_addr_update && current_mpc_pipe->plane_state->update_bits.addr_update) {
 					if (resource_is_pipe_type(current_mpc_pipe, OTG_MASTER) &&
 							stream_status->mall_stream_config.type == SUBVP_MAIN) {
 						block_sequence[*num_steps].params.subvp_save_surf_addr.dc_dmub_srv = dc->ctx->dmub_srv;
@@ -1442,15 +1486,14 @@ void hwss_build_fast_sequence(struct dc *dc,
 						(*num_steps)++;
 					}
 
-					block_sequence[*num_steps].params.update_plane_addr_params.dc = dc;
-					block_sequence[*num_steps].params.update_plane_addr_params.pipe_ctx = current_mpc_pipe;
-					block_sequence[*num_steps].func = HUBP_UPDATE_PLANE_ADDR;
-					(*num_steps)++;
+					hwss_add_hubp_update_plane_addr(&seq_state, dc, current_mpc_pipe);
 				}
 
 				if (current_mpc_pipe->plane_state->update_bits.lut_3d &&
 						current_mpc_pipe->plane_state->cm.flags.bits.lut3d_dma_enable &&
-						current_mpc_pipe->plane_state->cm.flags.bits.shaper_enable &&
+						/* RMCM drives the 3DLUT without a shaper */
+						(current_mpc_pipe->plane_state->cm.flags.bits.shaper_enable ||
+							current_mpc_pipe->plane_state->cm.flags.bits.rmcm_enable) &&
 						current_mpc_pipe->plane_state->cm.flags.bits.lut3d_enable &&
 						current_mpc_pipe->plane_res.hubp->funcs->hubp_enable_3dlut_fl) {
 					block_sequence[*num_steps].params.hubp_enable_3dlut_fl_params.hubp =
@@ -1459,9 +1502,23 @@ void hwss_build_fast_sequence(struct dc *dc,
 					(*num_steps)++;
 				}
 				if (hws->funcs.set_input_transfer_func && current_mpc_pipe->plane_state->update_bits.gamma_change) {
-					block_sequence[*num_steps].params.set_input_transfer_func_params.dc = dc;
-					block_sequence[*num_steps].params.set_input_transfer_func_params.pipe_ctx = current_mpc_pipe;
-					block_sequence[*num_steps].params.set_input_transfer_func_params.plane_state = current_mpc_pipe->plane_state;
+					struct pipe_ctx *primary_dpp_pipe =
+						resource_get_primary_dpp_pipe(current_mpc_pipe);
+
+					block_sequence[*num_steps].params.set_input_transfer_func_params =
+						(struct set_input_transfer_func_params) {
+						.dc = dc,
+						.dpp = current_mpc_pipe->plane_res.dpp,
+						.hubp = current_mpc_pipe->plane_res.hubp,
+						.primary_hubp = primary_dpp_pipe ?
+							primary_dpp_pipe->plane_res.hubp : current_mpc_pipe->plane_res.hubp,
+						.ipp = current_mpc_pipe->plane_res.ipp,
+						.mpc = dc->res_pool->mpc,
+						.rmcm = current_mpc_pipe->plane_res.rmcm,
+						.mpcc_id = current_mpc_pipe->plane_res.mpcc_inst,
+						.stream = current_mpc_pipe->stream,
+						.plane_state = current_mpc_pipe->plane_state,
+					};
 					block_sequence[*num_steps].func = DPP_SET_INPUT_TRANSFER_FUNC;
 					(*num_steps)++;
 				}
@@ -1491,7 +1548,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 					params->dpp = current_mpc_pipe->plane_res.dpp;
 					params->mpc = dc->res_pool->mpc;
 					params->xfm = current_mpc_pipe->plane_res.xfm;
-					params->mpcc_id = current_mpc_pipe->plane_res.hubp->inst;
+					params->mpcc_id = current_mpc_pipe->plane_res.mpcc_inst;
 					params->plane = current_mpc_pipe->plane_state;
 					params->stream = current_mpc_pipe->stream;
 					params->is_top_pipe = current_mpc_pipe->top_pipe == NULL;
@@ -1499,14 +1556,14 @@ void hwss_build_fast_sequence(struct dc *dc,
 					(*num_steps)++;
 				}
 				if (current_mpc_pipe->plane_state->update_bits.input_csc_change) {
-					block_sequence[*num_steps].params.setup_dpp_params.pipe_ctx = current_mpc_pipe;
-					block_sequence[*num_steps].func = DPP_SETUP_DPP;
-					(*num_steps)++;
+					hwss_add_dpp_setup_dpp(&seq_state,
+						current_mpc_pipe->plane_res.dpp,
+						current_mpc_pipe->plane_state);
 				}
 				if (current_mpc_pipe->plane_state->update_bits.coeff_reduction_change) {
-					block_sequence[*num_steps].params.program_bias_and_scale_params.pipe_ctx = current_mpc_pipe;
-					block_sequence[*num_steps].func = DPP_PROGRAM_BIAS_AND_SCALE;
-					(*num_steps)++;
+					hwss_add_dpp_program_bias_and_scale(&seq_state,
+						current_mpc_pipe->plane_res.dpp,
+						current_mpc_pipe->plane_state);
 				}
 				if (current_mpc_pipe->plane_state->update_bits.cm_hist_change) {
 					block_sequence[*num_steps].params.control_cm_hist_params.dpp
@@ -1536,7 +1593,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 				otf_params->dpp = current_mpc_pipe->plane_res.dpp;
 				otf_params->xfm = current_mpc_pipe->plane_res.xfm;
 				otf_params->mpc = dc->res_pool->mpc;
-				otf_params->mpcc_id = current_mpc_pipe->plane_res.hubp->inst;
+				otf_params->mpcc_id = current_mpc_pipe->plane_res.mpcc_inst;
 				otf_params->is_top_pipe = resource_is_pipe_type(current_mpc_pipe, OPP_HEAD);
 				otf_params->stream = current_mpc_pipe->stream;
 				block_sequence[*num_steps].func = DPP_SET_OUTPUT_TRANSFER_FUNC;
@@ -1544,15 +1601,15 @@ void hwss_build_fast_sequence(struct dc *dc,
 			}
 			if (dc->debug.visual_confirm != VISUAL_CONFIRM_DISABLE &&
 				dc->hwss.update_visual_confirm_color) {
-				block_sequence[*num_steps].params.update_visual_confirm_params.dc = dc;
-				block_sequence[*num_steps].params.update_visual_confirm_params.pipe_ctx = current_mpc_pipe;
-				block_sequence[*num_steps].params.update_visual_confirm_params.mpcc_id = current_mpc_pipe->plane_res.hubp->inst;
+				block_sequence[*num_steps].params.update_visual_confirm_params.mpc = dc->res_pool->mpc;
+				block_sequence[*num_steps].params.update_visual_confirm_params.color = &current_mpc_pipe->visual_confirm_color;
+				block_sequence[*num_steps].params.update_visual_confirm_params.mpcc_id = current_mpc_pipe->plane_res.mpcc_inst;
 				block_sequence[*num_steps].func = MPC_UPDATE_VISUAL_CONFIRM;
 				(*num_steps)++;
 			}
 			if (current_mpc_pipe->stream->update_flags.bits.out_csc) {
 				block_sequence[*num_steps].params.power_on_mpc_mem_pwr_params.mpc = dc->res_pool->mpc;
-				block_sequence[*num_steps].params.power_on_mpc_mem_pwr_params.mpcc_id = current_mpc_pipe->plane_res.hubp->inst;
+				block_sequence[*num_steps].params.power_on_mpc_mem_pwr_params.mpcc_id = current_mpc_pipe->plane_res.mpcc_inst;
 				block_sequence[*num_steps].params.power_on_mpc_mem_pwr_params.power_on = true;
 				block_sequence[*num_steps].func = MPC_POWER_ON_MPC_MEM_PWR;
 				(*num_steps)++;
@@ -1578,13 +1635,9 @@ void hwss_build_fast_sequence(struct dc *dc,
 		current_pipe = current_pipe->next_odm_pipe;
 	}
 
-	if (dc->hwss.pipe_control_lock) {
-		block_sequence[*num_steps].params.pipe_control_lock_params.dc = dc;
-		block_sequence[*num_steps].params.pipe_control_lock_params.lock = false;
-		block_sequence[*num_steps].params.pipe_control_lock_params.pipe_ctx = pipe_ctx;
-		block_sequence[*num_steps].func = OPTC_PIPE_CONTROL_LOCK;
-		(*num_steps)++;
-	}
+	hwss_add_optc_pipe_control_lock(
+			&(struct block_sequence_state){ block_sequence, num_steps },
+			dc, pipe_ctx, false);
 	if (dc->hwss.subvp_pipe_control_lock_fast) {
 		block_sequence[*num_steps].params.subvp_pipe_control_lock_fast_params.dc = dc;
 		block_sequence[*num_steps].params.subvp_pipe_control_lock_fast_params.lock = false;
@@ -1617,9 +1670,7 @@ void hwss_build_fast_sequence(struct dc *dc,
 					(*num_steps)++;
 				}
 
-				block_sequence[*num_steps].params.program_manual_trigger_params.pipe_ctx = current_mpc_pipe;
-				block_sequence[*num_steps].func = OPTC_PROGRAM_MANUAL_TRIGGER;
-				(*num_steps)++;
+				hwss_add_optc_program_manual_trigger(&seq_state, current_mpc_pipe->stream_res.tg);
 			}
 			current_mpc_pipe = current_mpc_pipe->bottom_pipe;
 		}
@@ -1642,10 +1693,14 @@ void hwss_execute_sequence(struct dc *dc,
 		case DMUB_SUBVP_PIPE_CONTROL_LOCK_FAST:
 			dc->hwss.subvp_pipe_control_lock_fast(params);
 			break;
-		case OPTC_PIPE_CONTROL_LOCK:
-			dc->hwss.pipe_control_lock(params->pipe_control_lock_params.dc,
-					params->pipe_control_lock_params.pipe_ctx,
-					params->pipe_control_lock_params.lock);
+		case TG_LOCK:
+			dc->hwss.tg_lock(&params->tg_lock_params);
+			break;
+		case TG_3DLUT_WA_UNLOCK:
+			if (dc->hwseq->funcs.perform_3dlut_wa_unlock)
+				dc->hwseq->funcs.perform_3dlut_wa_unlock(
+						params->tg_3dlut_wa_unlock_params.tg,
+						params->tg_3dlut_wa_unlock_params.hubp);
 			break;
 		case HUBP_SET_FLIP_CONTROL_GSL:
 			params->set_flip_control_gsl_params.hubp->funcs->hubp_set_flip_control_surface_gsl(
@@ -1653,18 +1708,17 @@ void hwss_execute_sequence(struct dc *dc,
 				params->set_flip_control_gsl_params.flip_immediate);
 			break;
 		case HUBP_PROGRAM_TRIPLEBUFFER:
-			dc->hwss.program_triplebuffer(params->program_triplebuffer_params.dc,
-					params->program_triplebuffer_params.pipe_ctx,
-					params->program_triplebuffer_params.enableTripleBuffer);
+			hwss_program_triplebuffer(params);
 			break;
 		case HUBP_UPDATE_PLANE_ADDR:
-			dc->hwss.update_plane_addr(params->update_plane_addr_params.dc,
-					params->update_plane_addr_params.pipe_ctx);
+			params->update_plane_addr_params.hubp->funcs->hubp_program_surface_flip_and_addr(
+					params->update_plane_addr_params.hubp,
+					&params->update_plane_addr_params.address,
+					params->update_plane_addr_params.flip_immediate,
+					params->update_plane_addr_params.dcc);
 			break;
 		case DPP_SET_INPUT_TRANSFER_FUNC:
-			hws->funcs.set_input_transfer_func(params->set_input_transfer_func_params.dc,
-					params->set_input_transfer_func_params.pipe_ctx,
-					params->set_input_transfer_func_params.plane_state);
+			hws->funcs.set_input_transfer_func(&params->set_input_transfer_func_params);
 			break;
 		case DPP_PROGRAM_GAMUT_REMAP:
 			if (dc->hwss.program_gamut_remap)
@@ -1677,7 +1731,11 @@ void hwss_execute_sequence(struct dc *dc,
 			hwss_tg_setup_vertical_interrupt0(params);
 			break;
 		case HWSS_SETUP_PERIODIC_INTERRUPT:
-			hwss_setup_periodic_interrupt(dc, params);
+			if (dc->hwss.setup_periodic_interrupt)
+				dc->hwss.setup_periodic_interrupt(
+						params->setup_periodic_interrupt_params.tg,
+						params->setup_periodic_interrupt_params.start_line,
+						params->setup_periodic_interrupt_params.end_line);
 			break;
 		case HWSS_UPDATE_INFO_FRAME:
 			hwss_update_info_frame(dc, params);
@@ -1701,9 +1759,11 @@ void hwss_execute_sequence(struct dc *dc,
 			hwss_program_upsp(params);
 			break;
 		case MPC_UPDATE_VISUAL_CONFIRM:
-			dc->hwss.update_visual_confirm_color(params->update_visual_confirm_params.dc,
-					params->update_visual_confirm_params.pipe_ctx,
-					params->update_visual_confirm_params.mpcc_id);
+			if (params->update_visual_confirm_params.mpc->funcs->set_bg_color)
+				params->update_visual_confirm_params.mpc->funcs->set_bg_color(
+						params->update_visual_confirm_params.mpc,
+						params->update_visual_confirm_params.color,
+						params->update_visual_confirm_params.mpcc_id);
 			break;
 		case MPC_POWER_ON_MPC_MEM_PWR:
 			hwss_power_on_mpc_mem_pwr(params);
@@ -1717,13 +1777,14 @@ void hwss_execute_sequence(struct dc *dc,
 		case DMUB_SEND_DMCUB_CMD:
 			hwss_send_dmcub_cmd(params);
 			break;
+		case LSDMA_SEND_PIO_COPY:
+			hwss_lsdma_send_pio_copy(params);
+			break;
 		case DMUB_SUBVP_SAVE_SURF_ADDR:
 			hwss_subvp_save_surf_addr(params);
 			break;
 		case HUBP_WAIT_FOR_DCC_META_PROP:
-			dc->hwss.wait_for_dcc_meta_propagation(
-					params->wait_for_dcc_meta_propagation_params.dc,
-					params->wait_for_dcc_meta_propagation_params.top_pipe_to_program);
+			dc->hwss.wait_for_dcc_meta_propagation(params->wait_for_dcc_meta_propagation_params.delay);
 			break;
 		case DMUB_HW_CONTROL_LOCK_FAST:
 			dc->hwss.dmub_hw_control_lock_fast(params);
@@ -1746,14 +1807,6 @@ void hwss_execute_sequence(struct dc *dc,
 			params->hubp_wait_pipe_read_start_params.hubp->funcs->hubp_wait_pipe_read_start(
 				params->hubp_wait_pipe_read_start_params.hubp);
 			break;
-		case HWS_APPLY_UPDATE_FLAGS_FOR_PHANTOM:
-			dc->hwss.apply_update_flags_for_phantom(params->apply_update_flags_for_phantom_params.pipe_ctx);
-			break;
-		case HWS_UPDATE_PHANTOM_VP_POSITION:
-			dc->hwss.update_phantom_vp_position(params->update_phantom_vp_position_params.dc,
-				params->update_phantom_vp_position_params.context,
-				params->update_phantom_vp_position_params.pipe_ctx);
-			break;
 		case OPTC_SET_ODM_COMBINE:
 			hwss_set_odm_combine(params);
 			break;
@@ -1857,7 +1910,8 @@ void hwss_execute_sequence(struct dc *dc,
 			hwss_tg_set_gsl_source_select(params);
 			break;
 		case HUBP_WAIT_FLIP_PENDING:
-			hwss_hubp_wait_flip_pending(params);
+			hwss_hubp_wait_flip_pending(params->hubp_wait_flip_pending_params.hubp,
+				params->hubp_wait_flip_pending_params.polling_interval_us);
 			break;
 		case TG_WAIT_DOUBLE_BUFFER_PENDING:
 			hwss_tg_wait_double_buffer_pending(params);
@@ -2045,20 +2099,20 @@ void hwss_execute_sequence(struct dc *dc,
 		case HUBP_MEM_PROGRAM_VIEWPORT:
 			hwss_hubp_mem_program_viewport(params);
 			break;
-		case ABORT_CURSOR_OFFLOAD_UPDATE:
-			hwss_abort_cursor_offload_update(params);
-			break;
 		case HWSS_CURSOR_LOCK:
 			hwss_cursor_lock(params);
 			break;
 		case HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE:
-			hwss_begin_cursor_offload_update(params);
+			hwss_begin_cursor_offload_update(dc, params);
 			break;
 		case HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE:
-			hwss_commit_cursor_offload_update(params);
+			hwss_commit_cursor_offload_update(dc, params);
 			break;
 		case HWSS_UPDATE_CURSOR_OFFLOAD_PIPE:
-			hwss_update_cursor_offload_pipe(params);
+			hwss_update_cursor_offload_pipe(dc, params);
+			break;
+		case ABORT_CURSOR_OFFLOAD_UPDATE:
+			hwss_abort_cursor_offload_update(dc, params);
 			break;
 		case DC_SEND_CURSOR_INFO_TO_DMU:
 			hwss_send_cursor_info_to_dmu(params);
@@ -2166,21 +2220,83 @@ void hwss_execute_sequence(struct dc *dc,
 	}
 }
 
-/*
- * Helper function to add OPTC pipe control lock to block sequence
- */
 void hwss_add_optc_pipe_control_lock(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
 		bool lock)
 {
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.pipe_control_lock_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.pipe_control_lock_params.pipe_ctx = pipe_ctx;
-		seq_state->steps[*seq_state->num_steps].params.pipe_control_lock_params.lock = lock;
-		seq_state->steps[*seq_state->num_steps].func = OPTC_PIPE_CONTROL_LOCK;
-		(*seq_state->num_steps)++;
+	struct pipe_control_lock_params params = { 0 };
+	unsigned int hubp_idx;
+	unsigned int polling_interval_us = 1;
+
+	if (!dc->hwss.build_pipe_control_lock_sequence ||
+			!dc->hwss.build_pipe_control_lock_sequence(dc, pipe_ctx, lock, &params))
+		return;
+
+	for (hubp_idx = 0; hubp_idx < MAX_PIPES; hubp_idx++)
+		if (params.hubps_to_wait_for_flip[hubp_idx])
+			hwss_add_hubp_wait_flip_pending(seq_state, params.hubps_to_wait_for_flip[hubp_idx],
+				polling_interval_us);
+
+	if (params.gsl_lock) {
+		hwss_add_tg_set_gsl(seq_state, params.gsl.tg, params.gsl.gsl);
+		hwss_add_tg_set_gsl_source_select(seq_state, params.gsl_source_select.tg,
+			params.gsl_source_select.group_idx, params.gsl_source_select.gsl_ready_signal);
+	}
+
+	if (params.tg_3dlut_wa_unlock) {
+		if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+			seq_state->steps[*seq_state->num_steps].params.tg_3dlut_wa_unlock_params =
+					params.tg_3dlut_wa_unlock_params;
+			seq_state->steps[*seq_state->num_steps].func = TG_3DLUT_WA_UNLOCK;
+			(*seq_state->num_steps)++;
+		}
+	} else {
+		if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+			seq_state->steps[*seq_state->num_steps].params.tg_lock_params = params.tg_lock;
+			seq_state->steps[*seq_state->num_steps].func = TG_LOCK;
+			(*seq_state->num_steps)++;
+		}
+	}
+}
+
+void hwss_pipe_control_lock(struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		bool lock)
+{
+	struct pipe_control_lock_params params = { 0 };
+	unsigned int hubp_idx;
+	unsigned int polling_interval_us = 1;
+
+	if (!dc->hwss.build_pipe_control_lock_sequence ||
+			!dc->hwss.build_pipe_control_lock_sequence(dc, pipe_ctx, lock, &params))
+		return;
+
+	for (hubp_idx = 0; hubp_idx < MAX_PIPES; hubp_idx++)
+		if (params.hubps_to_wait_for_flip[hubp_idx])
+			hwss_hubp_wait_flip_pending(
+					params.hubps_to_wait_for_flip[hubp_idx],
+					polling_interval_us);
+
+	if (params.gsl_lock) {
+		if (params.gsl.tg->funcs->set_gsl)
+			params.gsl.tg->funcs->set_gsl(params.gsl.tg, &params.gsl.gsl);
+		if (params.gsl_source_select.tg->funcs->set_gsl_source_select)
+			params.gsl_source_select.tg->funcs->set_gsl_source_select(
+					params.gsl_source_select.tg,
+					params.gsl_source_select.group_idx,
+					params.gsl_source_select.gsl_ready_signal);
+	}
+
+	if (params.tg_3dlut_wa_unlock) {
+		dc->hwseq->funcs.perform_3dlut_wa_unlock(
+				params.tg_3dlut_wa_unlock_params.tg,
+				params.tg_3dlut_wa_unlock_params.hubp);
+		return;
 	}
+
+	if (dc->hwss.tg_lock)
+		dc->hwss.tg_lock(&params.tg_lock);
 }
 
 /*
@@ -2202,13 +2318,14 @@ void hwss_add_hubp_set_flip_control_gsl(struct block_sequence_state *seq_state,
  * Helper function to add HUBP program triplebuffer to block sequence
  */
 void hwss_add_hubp_program_triplebuffer(struct block_sequence_state *seq_state,
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx,
+		struct hubp *hubp,
 		bool enableTripleBuffer)
 {
+	if (!hubp || !hubp->funcs->hubp_enable_tripleBuffer)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.program_triplebuffer_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.program_triplebuffer_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.program_triplebuffer_params.hubp = hubp;
 		seq_state->steps[*seq_state->num_steps].params.program_triplebuffer_params.enableTripleBuffer = enableTripleBuffer;
 		seq_state->steps[*seq_state->num_steps].func = HUBP_PROGRAM_TRIPLEBUFFER;
 		(*seq_state->num_steps)++;
@@ -2222,9 +2339,19 @@ void hwss_add_hubp_update_plane_addr(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
 {
+	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+
+	if (!dc->hwss.prepare_plane_addr_update || !hubp || !hubp->funcs->hubp_program_surface_flip_and_addr)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.update_plane_addr_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.update_plane_addr_params.pipe_ctx = pipe_ctx;
+		struct update_plane_addr_params *params =
+			&seq_state->steps[*seq_state->num_steps].params.update_plane_addr_params;
+
+		dc->hwss.prepare_plane_addr_update(dc, pipe_ctx,
+				&params->address, &params->flip_immediate);
+		params->hubp = hubp;
+		params->dcc = pipe_ctx->plane_state ? pipe_ctx->plane_state->dcc.enable : false;
 		seq_state->steps[*seq_state->num_steps].func = HUBP_UPDATE_PLANE_ADDR;
 		(*seq_state->num_steps)++;
 	}
@@ -2233,15 +2360,26 @@ void hwss_add_hubp_update_plane_addr(struct block_sequence_state *seq_state,
 /*
  * Helper function to add DPP set input transfer function to block sequence
  */
-void hwss_add_dpp_set_input_transfer_func(struct block_sequence_state *seq_state,
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx,
-		struct dc_plane_state *plane_state)
+void hwss_add_dpp_set_input_transfer_func(struct block_sequence_state *seq_state, struct dc *dc,
+	struct pipe_ctx *pipe_ctx)
 {
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.set_input_transfer_func_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.set_input_transfer_func_params.pipe_ctx = pipe_ctx;
-		seq_state->steps[*seq_state->num_steps].params.set_input_transfer_func_params.plane_state = plane_state;
+		struct pipe_ctx *primary_dpp_pipe = resource_get_primary_dpp_pipe(pipe_ctx);
+
+		seq_state->steps[*seq_state->num_steps].params.set_input_transfer_func_params =
+			(struct set_input_transfer_func_params) {
+			.dc = dc,
+			.dpp = pipe_ctx->plane_res.dpp,
+			.hubp = pipe_ctx->plane_res.hubp,
+			.primary_hubp = primary_dpp_pipe ?
+				primary_dpp_pipe->plane_res.hubp : pipe_ctx->plane_res.hubp,
+			.ipp = pipe_ctx->plane_res.ipp,
+			.mpc = dc->res_pool->mpc,
+			.rmcm = pipe_ctx->plane_res.rmcm,
+			.mpcc_id = pipe_ctx->plane_res.mpcc_inst,
+			.plane_state = pipe_ctx->plane_state,
+			.stream = pipe_ctx->stream,
+		};
 		seq_state->steps[*seq_state->num_steps].func = DPP_SET_INPUT_TRANSFER_FUNC;
 		(*seq_state->num_steps)++;
 	}
@@ -2258,7 +2396,7 @@ void hwss_add_dpp_program_gamut_remap(struct block_sequence_state *seq_state,
 		params->xfm = pipe_ctx->plane_res.xfm;
 		params->dpp = pipe_ctx->plane_res.dpp;
 		params->mpc = pipe_ctx->stream->ctx->dc->res_pool->mpc;
-		params->mpcc_id = pipe_ctx->plane_res.hubp->inst;
+		params->mpcc_id = pipe_ctx->plane_res.mpcc_inst;
 		params->plane = pipe_ctx->plane_state;
 		params->stream = pipe_ctx->stream;
 		params->is_top_pipe = pipe_ctx->top_pipe == NULL;
@@ -2270,11 +2408,18 @@ void hwss_add_dpp_program_gamut_remap(struct block_sequence_state *seq_state,
 /*
  * Helper function to add DPP program bias and scale to block sequence
  */
-void hwss_add_dpp_program_bias_and_scale(struct block_sequence_state *seq_state, struct pipe_ctx *pipe_ctx)
+void hwss_add_dpp_program_bias_and_scale(struct block_sequence_state *seq_state,
+		struct dpp *dpp,
+		struct dc_plane_state *plane_state)
 {
+	if (!plane_state)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.program_bias_and_scale_params.pipe_ctx = pipe_ctx;
 		seq_state->steps[*seq_state->num_steps].func = DPP_PROGRAM_BIAS_AND_SCALE;
+		seq_state->steps[*seq_state->num_steps].params.program_bias_and_scale_params.dpp = dpp;
+		seq_state->steps[*seq_state->num_steps].params.program_bias_and_scale_params.bias_and_scale =
+			plane_state->bias_and_scale;
 		(*seq_state->num_steps)++;
 	}
 }
@@ -2283,10 +2428,13 @@ void hwss_add_dpp_program_bias_and_scale(struct block_sequence_state *seq_state,
  * Helper function to add OPTC program manual trigger to block sequence
  */
 void hwss_add_optc_program_manual_trigger(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx)
+		struct timing_generator *tg)
 {
+	if (!tg)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.program_manual_trigger_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.program_manual_trigger_params.tg = tg;
 		seq_state->steps[*seq_state->num_steps].func = OPTC_PROGRAM_MANUAL_TRIGGER;
 		(*seq_state->num_steps)++;
 	}
@@ -2304,7 +2452,7 @@ void hwss_add_dpp_set_output_transfer_func(struct block_sequence_state *seq_stat
 			.xfm = pipe_ctx->plane_res.xfm,
 			.dpp = pipe_ctx->plane_res.dpp,
 			.mpc = dc->res_pool->mpc,
-			.mpcc_id = pipe_ctx->plane_res.hubp->inst,
+			.mpcc_id = pipe_ctx->plane_res.mpcc_inst,
 			.is_top_pipe = resource_is_pipe_type(pipe_ctx, OPP_HEAD),
 			.stream = pipe_ctx->stream,
 		};
@@ -2321,8 +2469,7 @@ void hwss_set_output_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx)
 				.xfm = pipe_ctx->plane_res.xfm,
 				.dpp = pipe_ctx->plane_res.dpp,
 				.mpc = dc->res_pool->mpc,
-				.mpcc_id = pipe_ctx->plane_res.hubp ?
-						pipe_ctx->plane_res.hubp->inst : 0,
+				.mpcc_id = pipe_ctx->plane_res.mpcc_inst,
 				.is_top_pipe = resource_is_pipe_type(pipe_ctx, OPP_HEAD),
 				.stream = pipe_ctx->stream,
 			}
@@ -2339,8 +2486,8 @@ void hwss_add_mpc_update_visual_confirm(struct block_sequence_state *seq_state,
 		int mpcc_id)
 {
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.update_visual_confirm_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.update_visual_confirm_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.update_visual_confirm_params.mpc = dc->res_pool->mpc;
+		seq_state->steps[*seq_state->num_steps].params.update_visual_confirm_params.color = &pipe_ctx->visual_confirm_color;
 		seq_state->steps[*seq_state->num_steps].params.update_visual_confirm_params.mpcc_id = mpcc_id;
 		seq_state->steps[*seq_state->num_steps].func = MPC_UPDATE_VISUAL_CONFIRM;
 		(*seq_state->num_steps)++;
@@ -2436,21 +2583,6 @@ void hwss_add_dmub_subvp_save_surf_addr(struct block_sequence_state *seq_state,
 	}
 }
 
-/*
- * Helper function to add HUBP wait for DCC meta propagation to block sequence
- */
-void hwss_add_hubp_wait_for_dcc_meta_prop(struct block_sequence_state *seq_state,
-		struct dc *dc,
-		struct pipe_ctx *top_pipe_to_program)
-{
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.wait_for_dcc_meta_propagation_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.wait_for_dcc_meta_propagation_params.top_pipe_to_program = top_pipe_to_program;
-		seq_state->steps[*seq_state->num_steps].func = HUBP_WAIT_FOR_DCC_META_PROP;
-		(*seq_state->num_steps)++;
-	}
-}
-
 /*
  * Helper function to add HUBP wait pipe read start to block sequence
  */
@@ -2464,36 +2596,6 @@ void hwss_add_hubp_wait_pipe_read_start(struct block_sequence_state *seq_state,
 	}
 }
 
-/*
- * Helper function to add HWS apply update flags for phantom to block sequence
- */
-void hwss_add_hws_apply_update_flags_for_phantom(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx)
-{
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.apply_update_flags_for_phantom_params.pipe_ctx = pipe_ctx;
-		seq_state->steps[*seq_state->num_steps].func = HWS_APPLY_UPDATE_FLAGS_FOR_PHANTOM;
-		(*seq_state->num_steps)++;
-	}
-}
-
-/*
- * Helper function to add HWS update phantom VP position to block sequence
- */
-void hwss_add_hws_update_phantom_vp_position(struct block_sequence_state *seq_state,
-		struct dc *dc,
-		struct dc_state *context,
-		struct pipe_ctx *pipe_ctx)
-{
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].params.update_phantom_vp_position_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.update_phantom_vp_position_params.context = context;
-		seq_state->steps[*seq_state->num_steps].params.update_phantom_vp_position_params.pipe_ctx = pipe_ctx;
-		seq_state->steps[*seq_state->num_steps].func = HWS_UPDATE_PHANTOM_VP_POSITION;
-		(*seq_state->num_steps)++;
-	}
-}
-
 /*
  * Helper function to add OPTC set ODM combine to block sequence
  */
@@ -2535,6 +2637,17 @@ void hwss_send_dmcub_cmd(union block_sequence_params *params)
 	dc_wake_and_execute_dmub_cmd(ctx, cmd, wait_type);
 }
 
+void hwss_lsdma_send_pio_copy(union block_sequence_params *params)
+{
+	struct dc_dmub_srv *dc_dmub_srv = params->lsdma_send_pio_copy_params.dc_dmub_srv;
+	uint64_t src_addr = params->lsdma_send_pio_copy_params.src_addr;
+	uint64_t dst_addr = params->lsdma_send_pio_copy_params.dst_addr;
+	uint32_t byte_count = params->lsdma_send_pio_copy_params.byte_count;
+	uint32_t overlap_disable = params->lsdma_send_pio_copy_params.overlap_disable;
+
+	dmub_lsdma_send_pio_copy_command(dc_dmub_srv, src_addr, dst_addr, byte_count, overlap_disable);
+}
+
 /*
  * Helper function to add TG program global sync to block sequence
  */
@@ -2791,12 +2904,10 @@ void hwss_add_tg_enable_crtc(struct block_sequence_state *seq_state,
  */
 void hwss_add_hubp_wait_flip_pending(struct block_sequence_state *seq_state,
 		struct hubp *hubp,
-		unsigned int timeout_us,
 		unsigned int polling_interval_us)
 {
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
 		seq_state->steps[*seq_state->num_steps].params.hubp_wait_flip_pending_params.hubp = hubp;
-		seq_state->steps[*seq_state->num_steps].params.hubp_wait_flip_pending_params.timeout_us = timeout_us;
 		seq_state->steps[*seq_state->num_steps].params.hubp_wait_flip_pending_params.polling_interval_us = polling_interval_us;
 		seq_state->steps[*seq_state->num_steps].func = HUBP_WAIT_FLIP_PENDING;
 		(*seq_state->num_steps)++;
@@ -2830,60 +2941,53 @@ void hwss_add_hubp_enable_3dlut_fl(struct block_sequence_state *seq_state,
 	}
 }
 
+void hwss_program_triplebuffer(union block_sequence_params *params)
+{
+	struct hubp *hubp = params->program_triplebuffer_params.hubp;
+
+	if (hubp && hubp->funcs->hubp_enable_tripleBuffer)
+		hubp->funcs->hubp_enable_tripleBuffer(hubp,
+				params->program_triplebuffer_params.enableTripleBuffer);
+}
+
 void hwss_program_manual_trigger(union block_sequence_params *params)
 {
-	struct pipe_ctx *pipe_ctx = params->program_manual_trigger_params.pipe_ctx;
+	struct timing_generator *tg = params->program_manual_trigger_params.tg;
 
-	if (pipe_ctx->stream_res.tg->funcs->program_manual_trigger)
-		pipe_ctx->stream_res.tg->funcs->program_manual_trigger(pipe_ctx->stream_res.tg);
+	if (tg && tg->funcs->program_manual_trigger)
+		tg->funcs->program_manual_trigger(tg);
 }
 
 void hwss_setup_dpp(union block_sequence_params *params)
 {
-	struct pipe_ctx *pipe_ctx = params->setup_dpp_params.pipe_ctx;
-	struct dpp *dpp = pipe_ctx->plane_res.dpp;
-	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
+	struct setup_dpp_params *p = &params->setup_dpp_params;
 
-	if (!plane_state)
-		return;
-
-	if (dpp && dpp->funcs->dpp_setup) {
-		// program the input csc
-		dpp->funcs->dpp_setup(dpp,
-				plane_state->format,
+	if (p->dpp && p->dpp->funcs->dpp_setup) {
+		p->dpp->funcs->dpp_setup(p->dpp,
+				p->format,
 				EXPANSION_MODE_ZERO,
-				plane_state->input_csc_color_matrix,
-				plane_state->color_space,
+				p->input_csc_color_matrix,
+				p->color_space,
 				NULL);
 	}
 }
 
 void hwss_program_bias_and_scale(union block_sequence_params *params)
 {
-	struct pipe_ctx *pipe_ctx = params->program_bias_and_scale_params.pipe_ctx;
-	struct dpp *dpp = pipe_ctx->plane_res.dpp;
-	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
-	struct dc_bias_and_scale bns_params = plane_state->bias_and_scale;
+	struct program_bias_and_scale_params *p = &params->program_bias_and_scale_params;
 
 	//TODO :for CNVC set scale and bias registers if necessary
-	if (dpp->funcs->dpp_program_bias_and_scale)
-		dpp->funcs->dpp_program_bias_and_scale(dpp, &bns_params);
+	if (p->dpp && p->dpp->funcs->dpp_program_bias_and_scale)
+		p->dpp->funcs->dpp_program_bias_and_scale(p->dpp, &p->bias_and_scale);
 }
 
 void hwss_program_upsp(union block_sequence_params *params)
 {
-	struct pipe_ctx *pipe_ctx = params->program_upsp_params.pipe_ctx;
-	struct dpp *dpp = pipe_ctx->plane_res.dpp;
-	struct dscl_prog_data *dscl_prog_data = &pipe_ctx->plane_res.scl_data.dscl_prog_data;
-
-	if (!dpp || !dscl_prog_data)
-		return;
+	struct dpp *dpp = params->program_upsp_params.dpp;
+	const struct dscl_prog_data *dscl_prog_data = params->program_upsp_params.dscl_prog_data;
 
-	if (dpp && dpp->funcs->dpp_program_upsp) {
-		// program upsampler
-		dpp->funcs->dpp_program_upsp(dpp,
-				dscl_prog_data);
-	}
+	if (dpp && dscl_prog_data && dpp->funcs->dpp_program_upsp)
+		dpp->funcs->dpp_program_upsp(dpp, dscl_prog_data);
 }
 
 void hwss_power_on_mpc_mem_pwr(union block_sequence_params *params)
@@ -3274,11 +3378,10 @@ void hwss_dsc_calculate_and_set_config(union block_sequence_params *params)
 
 void hwss_dsc_enable_with_opp(union block_sequence_params *params)
 {
-	struct pipe_ctx *pipe_ctx = params->dsc_enable_with_opp_params.pipe_ctx;
-	struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+	struct display_stream_compressor *dsc = params->dsc_enable_with_opp_params.dsc;
 
 	if (dsc && dsc->funcs->dsc_enable)
-		dsc->funcs->dsc_enable(dsc, pipe_ctx->stream_res.opp->inst);
+		dsc->funcs->dsc_enable(dsc, params->dsc_enable_with_opp_params.opp_inst);
 }
 
 void hwss_tg_program_global_sync(union block_sequence_params *params)
@@ -3331,12 +3434,18 @@ void hwss_update_info_frame(struct dc *dc, union block_sequence_params *params)
 		dc->hwss.update_info_frame(pipe_ctx);
 }
 
-void hwss_setup_periodic_interrupt(struct dc *dc, union block_sequence_params *params)
+void hwss_setup_periodic_interrupt(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
-	struct pipe_ctx *pipe_ctx = params->setup_periodic_interrupt_params.pipe_ctx;
+	uint32_t start_line = 0;
+	uint32_t end_line = 0;
 
-	if (dc->hwss.setup_periodic_interrupt)
-		dc->hwss.setup_periodic_interrupt(dc, pipe_ctx);
+	if (dc->hwss.setup_periodic_interrupt) {
+		calc_vline_position(dc, pipe_ctx, &start_line, &end_line);
+		dc->hwss.setup_periodic_interrupt(
+				pipe_ctx->stream_res.tg,
+				start_line,
+				end_line);
+	}
 }
 
 void hwss_tg_setup_vertical_interrupt0(union block_sequence_params *params)
@@ -3411,17 +3520,10 @@ void hwss_opp_program_fmt(union block_sequence_params *params)
 void hwss_opp_program_bit_depth_reduction(union block_sequence_params *params)
 {
 	struct output_pixel_processor *opp = params->opp_program_bit_depth_reduction_params.opp;
-	bool use_default_params = params->opp_program_bit_depth_reduction_params.use_default_params;
-	struct pipe_ctx *pipe_ctx = params->opp_program_bit_depth_reduction_params.pipe_ctx;
-	struct bit_depth_reduction_params bit_depth_params;
-
-	if (use_default_params)
-		memset(&bit_depth_params, 0, sizeof(bit_depth_params));
-	else
-		resource_build_bit_depth_reduction_params(pipe_ctx->stream, &bit_depth_params);
 
-	if (opp->funcs->opp_program_bit_depth_reduction)
-		opp->funcs->opp_program_bit_depth_reduction(opp, &bit_depth_params);
+	if (opp && opp->funcs->opp_program_bit_depth_reduction)
+		opp->funcs->opp_program_bit_depth_reduction(opp,
+			&params->opp_program_bit_depth_reduction_params.bit_depth_params);
 }
 
 void hwss_opp_set_disp_pattern_generator(union block_sequence_params *params)
@@ -3543,11 +3645,9 @@ void hwss_tg_set_gsl_source_select(union block_sequence_params *params)
 		tg->funcs->set_gsl_source_select(tg, group_idx, gsl_ready_signal);
 }
 
-void hwss_hubp_wait_flip_pending(union block_sequence_params *params)
+void hwss_hubp_wait_flip_pending(struct hubp *hubp, unsigned int polling_interval_us)
 {
-	struct hubp *hubp = params->hubp_wait_flip_pending_params.hubp;
-	unsigned int timeout_us = params->hubp_wait_flip_pending_params.timeout_us;
-	unsigned int polling_interval_us = params->hubp_wait_flip_pending_params.polling_interval_us;
+	const unsigned int timeout_us = 100000U;
 	unsigned int j = 0;
 
 	for (j = 0; j < timeout_us / polling_interval_us
@@ -4170,15 +4270,6 @@ void hwss_hubp_mem_program_viewport(union block_sequence_params *params)
 		hubp->funcs->mem_program_viewport(hubp, viewport, viewport_c);
 }
 
-void hwss_abort_cursor_offload_update(union block_sequence_params *params)
-{
-	struct dc *dc = params->abort_cursor_offload_update_params.dc;
-	struct pipe_ctx *pipe_ctx = params->abort_cursor_offload_update_params.pipe_ctx;
-
-	if (dc && dc->hwss.abort_cursor_offload_update)
-		dc->hwss.abort_cursor_offload_update(dc, pipe_ctx);
-}
-
 void hwss_cursor_lock(union block_sequence_params *params)
 {
 	struct dc *dc = params->cursor_lock_params.dc;
@@ -4189,39 +4280,56 @@ void hwss_cursor_lock(union block_sequence_params *params)
 		dc->hwss.cursor_lock(dc, pipe_ctx, lock);
 }
 
-void hwss_begin_cursor_offload_update(union block_sequence_params *params)
+void hwss_begin_cursor_offload_update(struct dc *dc, union block_sequence_params *params)
 {
-	struct dc *dc = params->begin_cursor_offload_update_params.dc;
-	struct pipe_ctx *pipe_ctx = params->begin_cursor_offload_update_params.pipe_ctx;
-
-	if (dc && dc->hwss.begin_cursor_offload_update)
-		dc->hwss.begin_cursor_offload_update(dc, pipe_ctx);
+	if (dc->hwss.begin_cursor_offload_update)
+		dc->hwss.begin_cursor_offload_update(
+			params->begin_cursor_offload_update_params.dmub,
+			params->begin_cursor_offload_update_params.dpp,
+			params->begin_cursor_offload_update_params.hubp,
+			params->begin_cursor_offload_update_params.stream_idx);
 }
 
-void hwss_commit_cursor_offload_update(union block_sequence_params *params)
+void hwss_commit_cursor_offload_update(struct dc *dc, union block_sequence_params *params)
 {
-	struct dc *dc = params->commit_cursor_offload_update_params.dc;
-	struct pipe_ctx *pipe_ctx = params->commit_cursor_offload_update_params.pipe_ctx;
-
-	if (dc && dc->hwss.commit_cursor_offload_update)
-		dc->hwss.commit_cursor_offload_update(dc, pipe_ctx);
+	if (dc->hwss.commit_cursor_offload_update)
+		dc->hwss.commit_cursor_offload_update(
+			params->commit_cursor_offload_update_params.dmub,
+			params->commit_cursor_offload_update_params.dpp,
+			params->commit_cursor_offload_update_params.hubp,
+			params->commit_cursor_offload_update_params.stream_idx);
 }
 
-void hwss_update_cursor_offload_pipe(union block_sequence_params *params)
+void hwss_update_cursor_offload_pipe(struct dc *dc, union block_sequence_params *params)
 {
-	struct dc *dc = params->update_cursor_offload_pipe_params.dc;
-	struct pipe_ctx *pipe_ctx = params->update_cursor_offload_pipe_params.pipe_ctx;
+	if (dc->hwss.update_cursor_offload_pipe)
+		dc->hwss.update_cursor_offload_pipe(
+			params->update_cursor_offload_pipe_params.dmub,
+			params->update_cursor_offload_pipe_params.stream_idx,
+			params->update_cursor_offload_pipe_params.pipe_idx,
+			params->update_cursor_offload_pipe_params.dpp,
+			params->update_cursor_offload_pipe_params.hubp);
+}
 
-	if (dc && dc->hwss.update_cursor_offload_pipe)
-		dc->hwss.update_cursor_offload_pipe(dc, pipe_ctx);
+void hwss_abort_cursor_offload_update(struct dc *dc, union block_sequence_params *params)
+{
+	if (dc->hwss.abort_cursor_offload_update)
+		dc->hwss.abort_cursor_offload_update(
+			params->abort_cursor_offload_update_params.dmub,
+			params->abort_cursor_offload_update_params.dpp,
+			params->abort_cursor_offload_update_params.hubp,
+			params->abort_cursor_offload_update_params.stream_idx);
 }
 
 void hwss_send_cursor_info_to_dmu(union block_sequence_params *params)
 {
-	struct pipe_ctx *pipe_ctx = params->send_cursor_info_to_dmu_params.pipe_ctx;
-	int pipe_idx = params->send_cursor_info_to_dmu_params.pipe_idx;
-
-	dc_send_update_cursor_info_to_dmu(pipe_ctx, (uint8_t)pipe_idx);
+	dc_send_update_cursor_info_to_dmu(
+		params->send_cursor_info_to_dmu_params.ctx,
+		params->send_cursor_info_to_dmu_params.pipe_idx,
+		params->send_cursor_info_to_dmu_params.hubp,
+		params->send_cursor_info_to_dmu_params.dpp,
+		params->send_cursor_info_to_dmu_params.otg_inst,
+		params->send_cursor_info_to_dmu_params.panel_inst);
 }
 
 void hwss_set_cursor_attribute(union block_sequence_params *params)
@@ -4254,19 +4362,36 @@ void hwss_dpp_set_cursor_attributes(union block_sequence_params *params)
 void hwss_set_cursor_position(union block_sequence_params *params)
 {
 	struct dc *dc = params->set_cursor_position_params.dc;
-	struct pipe_ctx *pipe_ctx = params->set_cursor_position_params.pipe_ctx;
+	struct hubp *hubp = params->set_cursor_position_params.hubp;
+	struct dpp *dpp = params->set_cursor_position_params.dpp;
+	const struct dc_cursor_position *pos = &params->set_cursor_position_params.pos;
+	const struct dc_cursor_mi_param *param = &params->set_cursor_position_params.param;
 
+	/* DCN path: SW logic ran in the builder; just invoke the BLCs */
 	if (dc && dc->hwss.set_cursor_position)
-		dc->hwss.set_cursor_position(pipe_ctx);
+		dc->hwss.set_cursor_position(hubp, dpp, pos, param);
+}
+
+void hwss_program_cursor_position(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	if (dc->hwseq && dc->hwseq->funcs.build_cursor_pos_update_params && dc->hwss.set_cursor_position) {
+		struct dc_cursor_position pos;
+		struct dc_cursor_mi_param param;
+
+		dc->hwseq->funcs.build_cursor_pos_update_params(pipe_ctx, &pos, &param);
+		dc->hwss.set_cursor_position(pipe_ctx->plane_res.hubp,
+				pipe_ctx->plane_res.dpp, &pos, &param);
+	} else if (dc->hwss.set_cursor_position_legacy) {
+		dc->hwss.set_cursor_position_legacy(pipe_ctx);
+	}
 }
 
 void hwss_set_cursor_sdr_white_level(union block_sequence_params *params)
 {
-	struct dc *dc = params->set_cursor_sdr_white_level_params.dc;
-	struct pipe_ctx *pipe_ctx = params->set_cursor_sdr_white_level_params.pipe_ctx;
+	struct dpp *dpp = params->set_cursor_sdr_white_level_params.dpp;
 
-	if (dc && dc->hwss.set_cursor_sdr_white_level)
-		dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
+	if (dpp->funcs->set_optional_cursor_attributes)
+		dpp->funcs->set_optional_cursor_attributes(dpp, &params->set_cursor_sdr_white_level_params.attr);
 }
 
 void hwss_program_gamut_remap(struct pipe_ctx *pipe_ctx)
@@ -4278,7 +4403,7 @@ void hwss_program_gamut_remap(struct pipe_ctx *pipe_ctx)
 			.xfm = pipe_ctx->plane_res.xfm,
 			.dpp = pipe_ctx->plane_res.dpp,
 			.mpc = dc->res_pool->mpc,
-			.mpcc_id = pipe_ctx->plane_res.hubp->inst,
+			.mpcc_id = pipe_ctx->plane_res.mpcc_inst,
 			.stream = pipe_ctx->stream,
 			.plane = pipe_ctx->plane_state,
 			.is_top_pipe = pipe_ctx->top_pipe == NULL,
@@ -4497,6 +4622,27 @@ void hwss_disable_audio_stream(struct dc *dc, union block_sequence_params *param
 			params->disable_audio_stream_params.pipe_ctx);
 }
 
+void hwss_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	if (dc->hwseq->funcs.set_input_transfer_func) {
+		struct pipe_ctx *primary_dpp_pipe = resource_get_primary_dpp_pipe(pipe_ctx);
+
+		dc->hwseq->funcs.set_input_transfer_func(&(struct set_input_transfer_func_params) {
+			.dc = dc,
+			.dpp = pipe_ctx->plane_res.dpp,
+			.hubp = pipe_ctx->plane_res.hubp,
+			.primary_hubp = primary_dpp_pipe ?
+				primary_dpp_pipe->plane_res.hubp : pipe_ctx->plane_res.hubp,
+			.ipp = pipe_ctx->plane_res.ipp,
+			.mpc = dc->res_pool->mpc,
+			.rmcm = pipe_ctx->plane_res.rmcm,
+			.mpcc_id = pipe_ctx->plane_res.mpcc_inst,
+			.plane_state = pipe_ctx->plane_state,
+			.stream = pipe_ctx->stream,
+		});
+	}
+}
+
 void hwss_prepare_bandwidth(struct dc *dc, union block_sequence_params *params)
 {
 	if (dc && dc->hwss.prepare_bandwidth)
@@ -4650,9 +4796,15 @@ void hwss_add_hubp_disconnect(struct block_sequence_state *seq_state,
 void hwss_add_dsc_enable_with_opp(struct block_sequence_state *seq_state,
 		struct pipe_ctx *pipe_ctx)
 {
+	if (!pipe_ctx || !pipe_ctx->stream_res.dsc || !pipe_ctx->stream_res.opp)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
 		seq_state->steps[*seq_state->num_steps].func = DSC_ENABLE_WITH_OPP;
-		seq_state->steps[*seq_state->num_steps].params.dsc_enable_with_opp_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.dsc_enable_with_opp_params.dsc =
+			pipe_ctx->stream_res.dsc;
+		seq_state->steps[*seq_state->num_steps].params.dsc_enable_with_opp_params.opp_inst =
+			pipe_ctx->stream_res.opp->inst;
 		(*seq_state->num_steps)++;
 	}
 }
@@ -4804,13 +4956,23 @@ void hwss_add_opp_program_bit_depth_reduction(struct block_sequence_state *seq_s
 		bool use_default_params,
 		struct pipe_ctx *pipe_ctx)
 {
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].func = OPP_PROGRAM_BIT_DEPTH_REDUCTION;
-		seq_state->steps[*seq_state->num_steps].params.opp_program_bit_depth_reduction_params.opp = opp;
-		seq_state->steps[*seq_state->num_steps].params.opp_program_bit_depth_reduction_params.use_default_params = use_default_params;
-		seq_state->steps[*seq_state->num_steps].params.opp_program_bit_depth_reduction_params.pipe_ctx = pipe_ctx;
-		(*seq_state->num_steps)++;
-	}
+	struct opp_program_bit_depth_reduction_params *bdr_params;
+
+	if (!opp)
+		return;
+
+	if (*seq_state->num_steps >= MAX_HWSS_BLOCK_SEQUENCE_SIZE)
+		return;
+
+	bdr_params = &seq_state->steps[*seq_state->num_steps].params.opp_program_bit_depth_reduction_params;
+	memset(bdr_params, 0, sizeof(*bdr_params));
+	bdr_params->opp = opp;
+
+	if (!use_default_params && pipe_ctx)
+		resource_build_bit_depth_reduction_params(pipe_ctx->stream, &bdr_params->bit_depth_params);
+
+	seq_state->steps[*seq_state->num_steps].func = OPP_PROGRAM_BIT_DEPTH_REDUCTION;
+	(*seq_state->num_steps)++;
 }
 
 void hwss_add_dpp_program_cm_hist(struct block_sequence_state *seq_state,
@@ -5055,6 +5217,21 @@ void hwss_add_hubbub_perfmon_arm_out_of_order_bw(struct block_sequence_state *se
 	}
 }
 
+void hwss_add_lsdma_send_pio_copy(struct block_sequence_state *seq_state,
+		struct dc_dmub_srv *dc_dmub_srv, uint64_t src_addr, uint64_t dst_addr,
+		uint32_t byte_count, uint32_t overlap_disable)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = LSDMA_SEND_PIO_COPY;
+		seq_state->steps[*seq_state->num_steps].params.lsdma_send_pio_copy_params.dc_dmub_srv = dc_dmub_srv;
+		seq_state->steps[*seq_state->num_steps].params.lsdma_send_pio_copy_params.src_addr = src_addr;
+		seq_state->steps[*seq_state->num_steps].params.lsdma_send_pio_copy_params.dst_addr = dst_addr;
+		seq_state->steps[*seq_state->num_steps].params.lsdma_send_pio_copy_params.byte_count = byte_count;
+		seq_state->steps[*seq_state->num_steps].params.lsdma_send_pio_copy_params.overlap_disable = overlap_disable;
+		(*seq_state->num_steps)++;
+	}
+}
+
 void hwss_add_hubbub_perfmon_start_out_of_order_bw(struct block_sequence_state *seq_state,
 		struct hubbub *hubbub)
 {
@@ -5518,11 +5695,21 @@ void hwss_add_hubp_program_surface_config(struct block_sequence_state *seq_state
 }
 
 void hwss_add_dpp_setup_dpp(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx)
+		struct dpp *dpp,
+		struct dc_plane_state *plane_state)
 {
+	if (!plane_state)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		struct setup_dpp_params *p =
+			&seq_state->steps[*seq_state->num_steps].params.setup_dpp_params;
+
 		seq_state->steps[*seq_state->num_steps].func = DPP_SETUP_DPP;
-		seq_state->steps[*seq_state->num_steps].params.setup_dpp_params.pipe_ctx = pipe_ctx;
+		p->dpp = dpp;
+		p->format = plane_state->format;
+		p->input_csc_color_matrix = plane_state->input_csc_color_matrix;
+		p->color_space = plane_state->color_space;
 		(*seq_state->num_steps)++;
 	}
 }
@@ -5553,6 +5740,18 @@ void hwss_add_dpp_set_scaler(struct block_sequence_state *seq_state,
 	}
 }
 
+void hwss_add_dpp_program_upsp(struct block_sequence_state *seq_state,
+		struct dpp *dpp,
+		const struct dscl_prog_data *dscl_prog_data)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DPP_PROGRAM_UPSP;
+		seq_state->steps[*seq_state->num_steps].params.program_upsp_params.dpp = dpp;
+		seq_state->steps[*seq_state->num_steps].params.program_upsp_params.dscl_prog_data = dscl_prog_data;
+		(*seq_state->num_steps)++;
+	}
+}
+
 void hwss_add_hubp_mem_program_viewport(struct block_sequence_state *seq_state,
 		struct hubp *hubp,
 		const struct rect *viewport,
@@ -5567,18 +5766,6 @@ void hwss_add_hubp_mem_program_viewport(struct block_sequence_state *seq_state,
 	}
 }
 
-void hwss_add_abort_cursor_offload_update(struct block_sequence_state *seq_state,
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx)
-{
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].func = ABORT_CURSOR_OFFLOAD_UPDATE;
-		seq_state->steps[*seq_state->num_steps].params.abort_cursor_offload_update_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.abort_cursor_offload_update_params.pipe_ctx = pipe_ctx;
-		(*seq_state->num_steps)++;
-	}
-}
-
 void hwss_add_set_cursor_attribute(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
@@ -5619,24 +5806,38 @@ void hwss_add_set_cursor_position(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
 {
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE &&
+			dc->hwseq && dc->hwseq->funcs.build_cursor_pos_update_params &&
+			dc->hwss.set_cursor_position) {
+		struct dc_cursor_position pos;
+		struct dc_cursor_mi_param param;
+
+		dc->hwseq->funcs.build_cursor_pos_update_params(pipe_ctx, &pos, &param);
+
 		seq_state->steps[*seq_state->num_steps].func = SET_CURSOR_POSITION;
 		seq_state->steps[*seq_state->num_steps].params.set_cursor_position_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.set_cursor_position_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.set_cursor_position_params.hubp = pipe_ctx->plane_res.hubp;
+		seq_state->steps[*seq_state->num_steps].params.set_cursor_position_params.dpp = pipe_ctx->plane_res.dpp;
+		seq_state->steps[*seq_state->num_steps].params.set_cursor_position_params.pos = pos;
+		seq_state->steps[*seq_state->num_steps].params.set_cursor_position_params.param = param;
 		(*seq_state->num_steps)++;
 	}
 }
 
 void hwss_add_set_cursor_sdr_white_level(struct block_sequence_state *seq_state,
-		struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
 {
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].func = SET_CURSOR_SDR_WHITE_LEVEL;
-		seq_state->steps[*seq_state->num_steps].params.set_cursor_sdr_white_level_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.set_cursor_sdr_white_level_params.pipe_ctx = pipe_ctx;
-		(*seq_state->num_steps)++;
-	}
+	struct dpp *dpp = pipe_ctx->plane_res.dpp;
+	struct dpp_cursor_attributes attr;
+
+	if (dpp && dpp->funcs->set_optional_cursor_attributes)
+		if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+			attr = calc_sdr_cursor_attributes(pipe_ctx);
+			seq_state->steps[*seq_state->num_steps].func = SET_CURSOR_SDR_WHITE_LEVEL;
+			seq_state->steps[*seq_state->num_steps].params.set_cursor_sdr_white_level_params.dpp = dpp;
+			seq_state->steps[*seq_state->num_steps].params.set_cursor_sdr_white_level_params.attr = attr;
+			(*seq_state->num_steps)++;
+		}
 }
 
 void hwss_add_program_output_csc(struct block_sequence_state *seq_state,
@@ -5715,18 +5916,6 @@ void hwss_add_tg_get_frame_count(struct block_sequence_state *seq_state,
 	}
 }
 
-void hwss_add_begin_cursor_offload_update(struct block_sequence_state *seq_state,
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx)
-{
-	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
-		seq_state->steps[*seq_state->num_steps].func = HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE;
-		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.pipe_ctx = pipe_ctx;
-		(*seq_state->num_steps)++;
-	}
-}
-
 void hwss_add_cursor_lock(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
@@ -5746,13 +5935,27 @@ void hwss_add_cursor_lock(struct block_sequence_state *seq_state,
 }
 
 void hwss_add_send_update_cursor_info_to_dmu(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx,
-		int index)
+		struct pipe_ctx *pipe_ctx)
 {
+	unsigned int panel_inst = 0;
+
+	if (!dc_dmub_should_update_cursor_data(pipe_ctx))
+		return;
+
+	dc_get_edp_link_panel_inst(pipe_ctx->stream->ctx->dc,
+			pipe_ctx->stream->link, &panel_inst);
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		struct send_cursor_info_to_dmu_params *p =
+			&seq_state->steps[*seq_state->num_steps].params.send_cursor_info_to_dmu_params;
+
 		seq_state->steps[*seq_state->num_steps].func = DC_SEND_CURSOR_INFO_TO_DMU;
-		seq_state->steps[*seq_state->num_steps].params.send_cursor_info_to_dmu_params.pipe_ctx = pipe_ctx;
-		seq_state->steps[*seq_state->num_steps].params.send_cursor_info_to_dmu_params.pipe_idx = index;
+		p->ctx = pipe_ctx->stream->ctx;
+		p->pipe_idx = pipe_ctx->pipe_idx;
+		p->hubp = pipe_ctx->plane_res.hubp;
+		p->dpp = pipe_ctx->plane_res.dpp;
+		p->otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
+		p->panel_inst = (uint8_t)panel_inst;
 		(*seq_state->num_steps)++;
 	}
 }
@@ -5761,10 +5964,27 @@ void hwss_add_update_cursor_offload_pipe(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
 {
+	const struct pipe_ctx *top_pipe;
+
+	if (!dc->hwss.update_cursor_offload_pipe)
+		return;
+
+	top_pipe = resource_get_otg_master(pipe_ctx);
+	if (!top_pipe)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
 		seq_state->steps[*seq_state->num_steps].func = HWSS_UPDATE_CURSOR_OFFLOAD_PIPE;
-		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.dmub =
+			dc->ctx->dmub_srv->dmub;
+		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.stream_idx =
+			top_pipe->pipe_idx;
+		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.pipe_idx =
+			pipe_ctx->pipe_idx;
+		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.dpp =
+			pipe_ctx->plane_res.dpp;
+		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.hubp =
+			pipe_ctx->plane_res.hubp;
 		(*seq_state->num_steps)++;
 	}
 }
@@ -5773,10 +5993,67 @@ void hwss_add_commit_cursor_offload_update(struct block_sequence_state *seq_stat
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
 {
+	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+	if (!top_pipe)
+		return;
+
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
 		seq_state->steps[*seq_state->num_steps].func = HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE;
-		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.dmub =
+			dc->ctx->dmub_srv->dmub;
+		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.stream_idx =
+			top_pipe->pipe_idx;
+		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.dpp =
+			 pipe_ctx->plane_res.dpp;
+		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.hubp =
+			pipe_ctx->plane_res.hubp;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_begin_cursor_offload_update(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx)
+{
+	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+	if (!top_pipe)
+		return;
+
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE;
+		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.dmub =
+			dc->ctx->dmub_srv->dmub;
+		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.stream_idx =
+			top_pipe->pipe_idx;
+		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.dpp =
+			 pipe_ctx->plane_res.dpp;
+		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.hubp =
+			pipe_ctx->plane_res.hubp;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_abort_cursor_offload_update(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx)
+{
+	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+	if (!top_pipe)
+		return;
+
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = ABORT_CURSOR_OFFLOAD_UPDATE;
+		seq_state->steps[*seq_state->num_steps].params.abort_cursor_offload_update_params.dmub =
+			dc->ctx->dmub_srv->dmub;
+		seq_state->steps[*seq_state->num_steps].params.abort_cursor_offload_update_params.stream_idx =
+			top_pipe->pipe_idx;
+		seq_state->steps[*seq_state->num_steps].params.abort_cursor_offload_update_params.dpp =
+			 pipe_ctx->plane_res.dpp;
+		seq_state->steps[*seq_state->num_steps].params.abort_cursor_offload_update_params.hubp =
+			pipe_ctx->plane_res.hubp;
 		(*seq_state->num_steps)++;
 	}
 }
@@ -5943,9 +6220,17 @@ void hwss_add_setup_periodic_interrupt(struct block_sequence_state *seq_state,
 		struct pipe_ctx *pipe_ctx)
 {
 	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		uint32_t start_line = 0;
+		uint32_t end_line = 0;
+
+		calc_vline_position(dc, pipe_ctx, &start_line, &end_line);
 		seq_state->steps[*seq_state->num_steps].func = HWSS_SETUP_PERIODIC_INTERRUPT;
-		seq_state->steps[*seq_state->num_steps].params.setup_periodic_interrupt_params.dc = dc;
-		seq_state->steps[*seq_state->num_steps].params.setup_periodic_interrupt_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.setup_periodic_interrupt_params.tg =
+				pipe_ctx->stream_res.tg;
+		seq_state->steps[*seq_state->num_steps].params.setup_periodic_interrupt_params.start_line =
+				start_line;
+		seq_state->steps[*seq_state->num_steps].params.setup_periodic_interrupt_params.end_line =
+				end_line;
 		(*seq_state->num_steps)++;
 	}
 }
@@ -6140,3 +6425,11 @@ void get_refresh_rate_confirm_color(struct pipe_ctx *pipe_ctx, struct tg_color *
 		pipe_ctx->visual_confirm_color.color_b_cb = (uint16_t)color_value;
 	}
 }
+
+void hwss_hubp_wait_for_dcc_meta_prop(struct dc *dc, struct pipe_ctx *top_pipe_to_program)
+{
+	if (dc->hwss.wait_for_dcc_meta_propagation) {
+		uint32_t delay = get_dcc_meta_propagation_delay(dc, top_pipe_to_program);
+		dc->hwss.wait_for_dcc_meta_propagation(delay);
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
index 14fc4ef482f16f..fef857c7336b1a 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
@@ -368,8 +368,7 @@ enum dc_link_encoding_format dc_link_get_highest_encoding_format(const struct dc
 
 		if (frl_link_settings->frl_link_rate == HDMI_FRL_LINK_RATE_DISABLE)
 			return DC_LINK_ENCODING_HDMI_TMDS;
-		else if (frl_link_settings->frl_link_rate >= HDMI_FRL_LINK_RATE_3GBPS &&
-				frl_link_settings->frl_link_rate <= HDMI_FRL_LINK_RATE_12GBPS)
+		else if (frl_link_settings->frl_link_rate >= HDMI_FRL_LINK_RATE_3GBPS)
 			return DC_LINK_ENCODING_HDMI_FRL;
 	}
 
@@ -601,3 +600,25 @@ void dc_link_get_alpm_support(struct dc_link *link,
 {
 	link->dc->link_srv->edp_get_alpm_support(link, auxless_support, auxwake_support);
 }
+
+void dc_link_set_panel_polarity_enable(struct dc_link *link, bool enable)
+{
+	if (link->dc->link_srv->edp_set_panel_polarity_enabled)
+		link->dc->link_srv->edp_set_panel_polarity_enabled(link, enable);
+}
+
+void dc_link_panel_polarity_reset(struct dc_link *link)
+{
+	if (link->dc->link_srv->edp_panel_polarity_reset)
+		link->dc->link_srv->edp_panel_polarity_reset(link);
+}
+
+bool dc_link_get_panel_polarity(struct dc_link *link, int32_t *polarity)
+{
+	bool ret = false;
+
+	if (link->dc->link_srv->edp_get_panel_polarity)
+		ret = link->dc->link_srv->edp_get_panel_polarity(link, polarity);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index fc9080f0c0935f..549f9885b762b0 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -55,11 +55,13 @@
 #if defined(CONFIG_DRM_AMD_DC_SI)
 #include "dce60/dce60_resource.h"
 #endif
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 #include "dce80/dce80_resource.h"
 #include "dce100/dce100_resource.h"
 #include "dce110/dce110_resource.h"
 #include "dce112/dce112_resource.h"
 #include "dce120/dce120_resource.h"
+#endif
 #include "dcn10/dcn10_resource.h"
 #include "dcn20/dcn20_resource.h"
 #include "dcn21/dcn21_resource.h"
@@ -100,7 +102,7 @@
 #define DC_LOGGER_INIT(logger)
 #include "link/hwss/link_hwss_hpo_frl.h"
 #include "dml/dml1_frl_cap_chk.h"
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #define UNABLE_TO_SPLIT -1
 
@@ -293,6 +295,7 @@ struct resource_pool *dc_create_resource_pool(struct dc  *dc,
 			init_data->num_virtual_links, dc);
 		break;
 #endif
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 	case DCE_VERSION_8_0:
 		res_pool = dce80_create_resource_pool(
 				(uint8_t)init_data->num_virtual_links, dc);
@@ -324,6 +327,7 @@ struct resource_pool *dc_create_resource_pool(struct dc  *dc,
 		res_pool = dce120_create_resource_pool(
 				(uint8_t)init_data->num_virtual_links, dc);
 		break;
+#endif
 
 #if defined(CONFIG_DRM_AMD_DC_FP)
 	case DCN_VERSION_1_0:
@@ -1045,7 +1049,7 @@ static struct rect calculate_mpc_slice_in_timing_active(
 	return mpc_rec;
 }
 
-static void calculate_adjust_recout_for_visual_confirm(struct pipe_ctx *pipe_ctx,
+static void calculate_adjust_recout_for_visual_confirm(const struct pipe_ctx *pipe_ctx,
 	unsigned int *base_offset, unsigned int *dpp_offset)
 {
 	struct dc *dc = pipe_ctx->stream->ctx->dc;
@@ -1066,7 +1070,7 @@ static void calculate_adjust_recout_for_visual_confirm(struct pipe_ctx *pipe_ctx
 }
 
 static void reverse_adjust_recout_for_visual_confirm(struct rect *recout,
-		struct pipe_ctx *pipe_ctx)
+		const struct pipe_ctx *pipe_ctx)
 {
 	unsigned int dpp_offset, base_offset;
 
@@ -1553,11 +1557,21 @@ void resource_build_test_pattern_params(struct resource_context *res_ctx,
 	}
 }
 
-enum upsp_mode resource_is_upsp_required(enum surface_pixel_format format)
+enum upsp_mode resource_is_upsp_required(enum surface_pixel_format format,
+		enum dc_scaling_linearity scaling_linearity)
 {
-	if (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN && format <= SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb) //420 Formats
+	bool scaling_in_linear = (scaling_linearity == DC_SCALING_LINEARITY_LINEAR);
+	bool is_420_format = (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN &&
+			format <= SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb);
+	bool is_422_format = (format > SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb &&
+			format < SURFACE_PIXEL_FORMAT_SUBSAMPLE_END);
+
+	/* UPSP (chroma upsampling) is only needed when subsampled YUV is scaled in
+	 * linear space.
+	 */
+	if (scaling_in_linear && is_420_format)
 		return UPSP_HORIZONTAL_VERTICAL_UPSAMPLING;
-	if (format > SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb && format < SURFACE_PIXEL_FORMAT_SUBSAMPLE_END) //422 Formats
+	if (scaling_in_linear && is_422_format)
 		return UPSP_HORIZONTAL_UPSAMPLING_ONLY;
 	return UPSP_BYPASS;
 }
@@ -1607,7 +1621,7 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->plane_res.scl_data.lb_params.depth = LB_PIXEL_DEPTH_30BPP;
 
 		pipe_ctx->plane_res.scl_data.lb_params.alpha_en = plane_state->per_pixel_alpha;
-		pipe_ctx->plane_res.scl_data.upsp = resource_is_upsp_required(plane_state->format);
+		pipe_ctx->plane_res.scl_data.upsp = resource_is_upsp_required(plane_state->format, plane_state->scaling_linearity);
 
 		// Convert pipe_ctx to respective input params for SPL
 		translate_SPL_in_params_from_pipe_ctx(pipe_ctx, spl_in);
@@ -1761,7 +1775,7 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx)
 	return res;
 }
 
-bool resource_can_pipe_disable_cursor(struct pipe_ctx *pipe_ctx)
+bool resource_can_pipe_disable_cursor(const struct pipe_ctx *pipe_ctx)
 {
 	struct pipe_ctx *test_pipe, *split_pipe;
 	struct rect r1 = pipe_ctx->plane_res.scl_data.recout;
@@ -3003,6 +3017,11 @@ static inline int find_acquired_dio_link_enc_for_link(
 
 static inline int find_fixed_dio_link_enc(const struct dc_link *link)
 {
+	/* virtual links own their link encoder directly and are never
+	 * registered into pool->link_encoders[]. */
+	if (link->connector_signal == SIGNAL_TYPE_VIRTUAL)
+		return -1;
+
 	/* the 8b10b dp phy can only use fixed link encoder */
 	return link->eng_id;
 }
@@ -3148,6 +3167,8 @@ static bool add_dio_link_enc_to_ctx(const struct dc *dc,
 
 	if (enc_index >= 0)
 		pipe_ctx->link_res.dio_link_enc = pool->link_encoders[enc_index];
+	else
+		pipe_ctx->link_res.dio_link_enc = stream->link->link_enc;
 
 	return pipe_ctx->link_res.dio_link_enc != NULL;
 }
@@ -3368,6 +3389,95 @@ static bool acquire_secondary_dpp_pipes_and_add_plane(
 	return true;
 }
 
+/*
+ * resource_assign_rmcm() - Assign the RMCM instances for a whole context
+ *
+ * RMCM is pre-blend, so every pipe asking for one needs its own instance. Ownership is
+ * tracked per instance in res_ctx->rmcm_in_use[]; the owning pipe is the one holding the
+ * matching plane_res.rmcm pointer.
+ *
+ * A pipe that already drives an RMCM keeps it, so a plane that is only now asking never
+ * evicts one that is already using an instance - it simply gets none when the pool is
+ * exhausted. This has to see the finished context, hence one pass over all pipes rather
+ * than one per appended plane.
+ */
+void resource_assign_rmcm(
+		struct dc_state *new_ctx,
+		const struct dc_state *cur_ctx,
+		const struct resource_pool *pool)
+{
+	struct rmcm *held[MAX_PIPES] = { NULL };
+	unsigned int p;
+	int i;
+
+	/* new_ctx and cur_ctx can be the same object, so snapshot before clearing */
+	for (p = 0; p < pool->pipe_count && p < MAX_PIPES; p++)
+		held[p] = cur_ctx ? cur_ctx->res_ctx.pipe_ctx[p].plane_res.rmcm : NULL;
+
+	for (p = 0; p < pool->pipe_count && p < MAX_PIPES; p++)
+		new_ctx->res_ctx.pipe_ctx[p].plane_res.rmcm = NULL;
+
+	for (i = 0; i < MAX_RMCM_INST; i++)
+		new_ctx->res_ctx.rmcm_in_use[i] = false;
+
+	/* incumbents first, so a later pipe cannot take an instance out from under them */
+	for (p = 0; p < pool->pipe_count && p < MAX_PIPES; p++) {
+		struct pipe_ctx *pipe = &new_ctx->res_ctx.pipe_ctx[p];
+
+		if (!pipe->plane_state || !pipe->plane_state->cm.flags.bits.rmcm_enable)
+			continue;
+
+		if (!held[p] || new_ctx->res_ctx.rmcm_in_use[held[p]->inst])
+			continue;
+
+		pipe->plane_res.rmcm = held[p];
+		new_ctx->res_ctx.rmcm_in_use[held[p]->inst] = true;
+	}
+
+	for (p = 0; p < pool->pipe_count && p < MAX_PIPES; p++) {
+		struct pipe_ctx *pipe = &new_ctx->res_ctx.pipe_ctx[p];
+
+		if (!pipe->plane_state || !pipe->plane_state->cm.flags.bits.rmcm_enable)
+			continue;
+
+		if (pipe->plane_res.rmcm)
+			continue;
+
+		for (i = 0; i < pool->res_cap->num_rmcm && i < MAX_RMCM_INST; i++) {
+			if (!pool->rmcm[i] || new_ctx->res_ctx.rmcm_in_use[i])
+				continue;
+
+			pipe->plane_res.rmcm = pool->rmcm[i];
+			new_ctx->res_ctx.rmcm_in_use[i] = true;
+			break;
+		}
+	}
+}
+
+/*
+ * resource_release_rmcm() - Release an RMCM instance for a pipe.
+ *
+ * Marks the instance free and NULLs the caller's pointer.
+ */
+void resource_release_rmcm(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct rmcm **rmcm)
+{
+	int i;
+
+	if (!*rmcm)
+		return;
+
+	for (i = 0; i < pool->res_cap->num_rmcm && i < MAX_RMCM_INST; i++) {
+		if (pool->rmcm[i] == *rmcm) {
+			res_ctx->rmcm_in_use[i] = false;
+			*rmcm = NULL;
+			return;
+		}
+	}
+}
+
 bool resource_append_dpp_pipes_for_plane_composition(
 		struct dc_state *new_ctx,
 		struct dc_state *cur_ctx,
@@ -3393,6 +3503,12 @@ bool resource_append_dpp_pipes_for_plane_composition(
 					pool, plane_state);
 	}
 
+	/* Idempotent, so the last append of a commit leaves the final assignment behind.
+	 * Releasing is driven by the MPC_RMCM_CNTL mirror (see dcn42_release_rmcm_from_mpcc).
+	 */
+	if (success)
+		resource_assign_rmcm(new_ctx, cur_ctx, pool);
+
 	return success;
 }
 
@@ -3407,6 +3523,12 @@ void resource_remove_dpp_pipes_for_plane_composition(
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
 		if (pipe_ctx->plane_state == plane_state) {
+			/* Release RMCM instance for this pipe */
+			if (pipe_ctx->plane_res.rmcm)
+				resource_release_rmcm(&context->res_ctx,
+						pool,
+						&pipe_ctx->plane_res.rmcm);
+
 			if (pipe_ctx->top_pipe)
 				pipe_ctx->top_pipe->bottom_pipe = pipe_ctx->bottom_pipe;
 
@@ -4459,11 +4581,35 @@ enum dc_status resource_validate_probe_set(struct dc *dc,
 		return DC_NOT_SUPPORTED;
 
 	for (i = 0; i < probe_count; i++) {
-		if (probes[i].target_state == DC_PROBE_MEASURING)
+		switch (probes[i].type) {
+		/* These global probes support a one-frame measurement and reset. */
+		case DC_PROBE_PEAK_MEM_BW:
+		case DC_PROBE_PEAK_MEM_BW_STRESSED:
+		case DC_PROBE_AVG_MEM_BW:
+		case DC_PROBE_MEM_LATENCY:
+		case DC_PROBE_PREFETCH_DATA_SIZE:
+			if (probes[i].scope.type != DC_PROBE_SCOPE_GLOBAL)
+				return DC_NOT_SUPPORTED;
+			if (probes[i].target_state != DC_PROBE_NOT_MEASURING &&
+					probes[i].target_state != DC_PROBE_MEASURED)
+				return DC_NOT_SUPPORTED;
+			break;
+		/* The global urgent assertion counter additionally supports polling. */
+		case DC_PROBE_URGENT_ASSERTION_COUNT:
+			if (probes[i].scope.type != DC_PROBE_SCOPE_GLOBAL)
+				return DC_NOT_SUPPORTED;
+			if (probes[i].target_state != DC_PROBE_NOT_MEASURING &&
+					probes[i].target_state != DC_PROBE_MEASURING &&
+					probes[i].target_state != DC_PROBE_MEASURED)
+				return DC_NOT_SUPPORTED;
+			break;
+		default:
 			return DC_NOT_SUPPORTED;
+		}
 
-		if (probes[i].scope.type != DC_PROBE_SCOPE_GLOBAL)
-			return DC_NOT_SUPPORTED;
+		if (probes[i].type == DC_PROBE_PEAK_MEM_BW_STRESSED &&
+				!dc->res_pool->lsdma_scratch.buffer)
+			return DC_NO_DRAM_BUFFER_RESOURCE;
 	}
 
 	return DC_OK;
@@ -5306,6 +5452,81 @@ void resource_build_info_frame(struct pipe_ctx *pipe_ctx)
 	patch_gamut_packet_checksum(&info->gamut);
 }
 
+/*
+ * Physical-clock resource mapping used by the DCE and DCN resource pools.
+ * Unit tests provide their own stub (see dc_resource_ut_helpers.c).
+ */
+enum {
+	CLK_SRC_PLL0 = 0,
+	CLK_SRC_PLL1,
+	CLK_SRC_PLL2,
+	CLK_SRC_PLL3,
+	CLK_SRC_PLL4,
+	CLK_SRC_PLL5,
+};
+
+static struct clock_source *find_matching_pll(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		const struct dc_stream_state *const stream)
+{
+	(void)res_ctx;
+	switch (stream->link->link_enc->transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return pool->clock_sources[CLK_SRC_PLL0];
+	case TRANSMITTER_UNIPHY_B:
+		return pool->clock_sources[CLK_SRC_PLL1];
+	case TRANSMITTER_UNIPHY_C:
+		return pool->clock_sources[CLK_SRC_PLL2];
+	case TRANSMITTER_UNIPHY_D:
+		return pool->clock_sources[CLK_SRC_PLL3];
+	case TRANSMITTER_UNIPHY_E:
+		return pool->clock_sources[CLK_SRC_PLL4];
+	case TRANSMITTER_UNIPHY_F:
+		return pool->clock_sources[CLK_SRC_PLL5];
+	default:
+		return NULL;
+	}
+}
+
+enum dc_status resource_map_phy_clock_resources(
+		const struct dc *dc,
+		struct dc_state *context,
+		struct dc_stream_state *stream)
+{
+
+	/* acquire new resources */
+	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(
+			&context->res_ctx, stream);
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+
+	if (dc_is_dp_signal(pipe_ctx->stream->signal)
+		|| dc_is_virtual_signal(pipe_ctx->stream->signal))
+		pipe_ctx->clock_source =
+				dc->res_pool->dp_clock_source;
+	else if (pipe_ctx->stream->signal == SIGNAL_TYPE_HDMI_FRL)
+			pipe_ctx->clock_source =
+				dc->res_pool->dp_clock_source;
+	else {
+		if (stream && stream->link && stream->link->link_enc)
+			pipe_ctx->clock_source = find_matching_pll(
+				&context->res_ctx, dc->res_pool,
+				stream);
+	}
+
+	if (pipe_ctx->clock_source == NULL)
+		return DC_NO_CLOCK_SOURCE_RESOURCE;
+
+	resource_reference_clock_source(
+		&context->res_ctx,
+		dc->res_pool,
+		pipe_ctx->clock_source);
+
+	return DC_OK;
+}
+
 enum dc_status resource_map_clock_resources(
 		const struct dc  *dc,
 		struct dc_state *context,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_state.c b/drivers/gpu/drm/amd/display/dc/core/dc_state.c
index 666212cac105d3..edb50c23ce2431 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_state.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_state.c
@@ -36,8 +36,8 @@
 #include "link_enc_cfg.h"
 
 #if defined(CONFIG_DRM_AMD_DC_FP)
-#include "dml2_0/dml2_wrapper.h"
-#include "dml2_0/dml2_internal_types.h"
+#include "dml2_wrapper/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_internal_types.h"
 #endif
 
 #define DC_LOGGER \
@@ -467,8 +467,6 @@ enum dc_status dc_state_remove_stream(
 		return DC_ERROR_UNEXPECTED;
 	}
 
-	dc_stream_release_3dlut_for_stream(dc, stream);
-
 	dc_stream_release(state->streams[i]);
 	state->stream_count--;
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 9f3068e899e1ed..b446c311e6623f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -34,6 +34,7 @@
 #include "dc_dmub_srv.h"
 #include "dc_state_priv.h"
 #include "dc_stream_priv.h"
+#include "dce/dmub_hw_lock_mgr.h"
 
 #define DC_LOGGER dc->ctx->logger
 #ifndef MIN
@@ -284,6 +285,21 @@ struct dc_link *dc_stream_get_link(
 	return stream->link;
 }
 
+static void update_cursor_info_to_dmu(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	unsigned int panel_inst;
+
+	if (!dc->ctx->dmub_srv || !dc_dmub_should_update_cursor_data(pipe_ctx))
+		return;
+
+	if (!dc_get_edp_link_panel_inst(dc, pipe_ctx->stream->link, &panel_inst))
+		panel_inst = 0;
+
+	dc_send_update_cursor_info_to_dmu(pipe_ctx->stream->ctx, pipe_ctx->pipe_idx,
+			pipe_ctx->plane_res.hubp, pipe_ctx->plane_res.dpp,
+			(uint8_t)pipe_ctx->stream_res.tg->inst, (uint8_t)panel_inst);
+}
+
 void program_cursor_attributes(
 	struct dc *dc,
 	struct dc_stream_state *stream)
@@ -292,6 +308,7 @@ void program_cursor_attributes(
 	struct resource_context *res_ctx;
 	struct pipe_ctx *pipe_to_program = NULL;
 	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+	bool unlock_dmub = false;
 
 	if (!stream)
 		return;
@@ -308,8 +325,19 @@ void program_cursor_attributes(
 			pipe_to_program = pipe_ctx;
 
 			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
-				dc->hwss.begin_cursor_offload_update(dc, pipe_ctx);
+				struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+				if (top_pipe)
+					dc->hwss.begin_cursor_offload_update(dc->ctx->dmub_srv->dmub,
+						pipe_ctx->plane_res.dpp, pipe_ctx->plane_res.hubp,
+						top_pipe->pipe_idx);
 			} else {
+				if (dc->hwss.dmub_hw_control_lock && pipe_ctx->stream &&
+				    should_use_dmub_inbox0_lock_for_link(dc, pipe_ctx->stream->link)) {
+					dc->hwss.dmub_hw_control_lock(dc, dc->current_state, true);
+					unlock_dmub = true;
+				}
+
 				dc->hwss.cursor_lock(dc, pipe_to_program, true);
 				if (pipe_to_program->next_odm_pipe)
 					dc->hwss.cursor_lock(dc, pipe_to_program->next_odm_pipe, true);
@@ -317,21 +345,34 @@ void program_cursor_attributes(
 		}
 
 		dc->hwss.set_cursor_attribute(pipe_ctx);
-		if (dc->ctx->dmub_srv)
-			dc_send_update_cursor_info_to_dmu(pipe_ctx, i);
+		update_cursor_info_to_dmu(dc, pipe_ctx);
 		if (dc->hwss.set_cursor_sdr_white_level)
 			dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
-		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe)
-			dc->hwss.update_cursor_offload_pipe(dc, pipe_ctx);
+		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe) {
+			struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+			if (top_pipe)
+				dc->hwss.update_cursor_offload_pipe(dc->ctx->dmub_srv->dmub,
+					top_pipe->pipe_idx, pipe_ctx->pipe_idx,
+					pipe_ctx->plane_res.dpp, pipe_ctx->plane_res.hubp);
+		}
 	}
 
 	if (pipe_to_program) {
 		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
-			dc->hwss.commit_cursor_offload_update(dc, pipe_to_program);
+			struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_to_program);
+
+			if (top_pipe)
+				dc->hwss.commit_cursor_offload_update(dc->ctx->dmub_srv->dmub,
+					pipe_to_program->plane_res.dpp, pipe_to_program->plane_res.hubp,
+					top_pipe->pipe_idx);
 		} else {
 			dc->hwss.cursor_lock(dc, pipe_to_program, false);
 			if (pipe_to_program->next_odm_pipe)
 				dc->hwss.cursor_lock(dc, pipe_to_program->next_odm_pipe, false);
+
+			if (unlock_dmub)
+				dc->hwss.dmub_hw_control_lock(dc, dc->current_state, false);
 		}
 	}
 }
@@ -451,6 +492,7 @@ void program_cursor_position(
 	struct resource_context *res_ctx;
 	struct pipe_ctx *pipe_to_program = NULL;
 	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+	bool unlock_dmub = false;
 
 	if (!stream)
 		return;
@@ -470,25 +512,51 @@ void program_cursor_position(
 		if (!pipe_to_program) {
 			pipe_to_program = pipe_ctx;
 
-			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update)
-				dc->hwss.begin_cursor_offload_update(dc, pipe_ctx);
-			else
+			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
+				struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+				if (top_pipe)
+					dc->hwss.begin_cursor_offload_update(dc->ctx->dmub_srv->dmub,
+						pipe_ctx->plane_res.dpp, pipe_ctx->plane_res.hubp,
+						top_pipe->pipe_idx);
+			} else {
+				if (dc->hwss.dmub_hw_control_lock && pipe_ctx->stream &&
+				    should_use_dmub_inbox0_lock_for_link(dc, pipe_ctx->stream->link)) {
+					dc->hwss.dmub_hw_control_lock(dc, dc->current_state, true);
+					unlock_dmub = true;
+				}
+
 				dc->hwss.cursor_lock(dc, pipe_to_program, true);
+			}
 		}
 
-		dc->hwss.set_cursor_position(pipe_ctx);
-		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe)
-			dc->hwss.update_cursor_offload_pipe(dc, pipe_ctx);
+		hwss_program_cursor_position(dc, pipe_ctx);
+		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe) {
+			struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
 
-		if (dc->ctx->dmub_srv)
-			dc_send_update_cursor_info_to_dmu(pipe_ctx, i);
+			if (top_pipe)
+				dc->hwss.update_cursor_offload_pipe(dc->ctx->dmub_srv->dmub,
+					top_pipe->pipe_idx, pipe_ctx->pipe_idx,
+					pipe_ctx->plane_res.dpp, pipe_ctx->plane_res.hubp);
+		}
+
+		update_cursor_info_to_dmu(dc, pipe_ctx);
 	}
 
 	if (pipe_to_program) {
-		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update)
-			dc->hwss.commit_cursor_offload_update(dc, pipe_to_program);
-		else
+		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
+			struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_to_program);
+
+			if (top_pipe)
+				dc->hwss.commit_cursor_offload_update(dc->ctx->dmub_srv->dmub,
+					pipe_to_program->plane_res.dpp, pipe_to_program->plane_res.hubp,
+					top_pipe->pipe_idx);
+		} else {
 			dc->hwss.cursor_lock(dc, pipe_to_program, false);
+
+			if (unlock_dmub)
+				dc->hwss.dmub_hw_control_lock(dc, dc->current_state, false);
+		}
 	}
 }
 
@@ -991,72 +1059,6 @@ void dc_stream_log(const struct dc *dc, const struct dc_stream_state *stream)
 	}
 }
 
-/* TODO - move to per plane ownership? */
-/*
-*	dc_stream_get_3dlut()
-*	Requirements:
-*	1. Is stream already owns an RMCM instance, return it.
-*	2. If it doesn't and we don't need to allocate, return NULL.
-*	3. If there's a free RMCM instance, assign to stream and return it.
-*	4. If no free RMCM instances, return NULL.
-*/
-
-struct dc_rmcm_3dlut *dc_stream_get_3dlut_for_stream(
-	const struct dc *dc,
-	const struct dc_stream_state *stream,
-	bool allocate_one)
-{
-	unsigned int num_rmcm = dc->caps.color.mpc.num_rmcm_3dluts;
-
-	// see if one is allocated for this stream
-	for (unsigned int i = 0; i < num_rmcm; i++) {
-		if (dc->res_pool->rmcm_3dlut[i].isInUse &&
-			dc->res_pool->rmcm_3dlut[i].stream == stream)
-			return &dc->res_pool->rmcm_3dlut[i];
-	}
-
-	//case: not found one, and dont need to allocate
-	if (!allocate_one)
-		return NULL;
-
-	//see if there is an unused 3dlut, allocate
-	for (unsigned int i = 0; i < num_rmcm; i++) {
-		if (!dc->res_pool->rmcm_3dlut[i].isInUse) {
-			dc->res_pool->rmcm_3dlut[i].isInUse = true;
-			dc->res_pool->rmcm_3dlut[i].stream = stream;
-			return &dc->res_pool->rmcm_3dlut[i];
-		}
-	}
-
-	//dont have a 3dlut
-	return NULL;
-}
-
-
-void dc_stream_release_3dlut_for_stream(
-	const struct dc *dc,
-	const struct dc_stream_state *stream)
-{
-	struct dc_rmcm_3dlut *rmcm_3dlut =
-		dc_stream_get_3dlut_for_stream(dc, stream, false);
-
-	if (rmcm_3dlut) {
-		rmcm_3dlut->isInUse = false;
-		rmcm_3dlut->stream  = NULL;
-	}
-}
-
-
-void dc_stream_init_rmcm_3dlut(struct dc *dc)
-{
-	unsigned int num_rmcm = dc->caps.color.mpc.num_rmcm_3dluts;
-
-	for (unsigned int i = 0; i < num_rmcm; i++) {
-		dc->res_pool->rmcm_3dlut[i].isInUse = false;
-		dc->res_pool->rmcm_3dlut[i].stream = NULL;
-	}
-}
-
 /*
  * Finds the greatest index in refresh_rate_hz that contains a value <= refresh
  */
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 705c500ad71143..c8e6e949ce7954 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -37,13 +37,14 @@
 #include "link_service_types.h"
 #include "grph_object_ctrl_defs.h"
 #include <inc/hw/opp.h>
+#include <inc/hw/rmcm.h>
 
 #include "hwss/hw_sequencer.h"
 #include "inc/compressor.h"
 #include "inc/hw/dmcu.h"
 #include "dml/display_mode_lib.h"
 
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #include "dmub/inc/dmub_cmd.h"
 
@@ -66,7 +67,7 @@ struct dcn_dsc_reg_state;
 struct dcn_optc_reg_state;
 struct dcn_dccg_reg_state;
 
-#define DC_VER "3.2.392"
+#define DC_VER "3.2.399"
 
 /**
  * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC
@@ -111,10 +112,10 @@ struct frl_cap_chk_params_fixed31_32 {
 	int      lanes;
 	struct fixed31_32   f_pixel_clock_nominal;   /* Pixel Clock rate (Hz)  */
 	struct fixed31_32   r_bit_nominal;           /* FRL bitrate (bps) */
-	int      audio_packet_type;
+	unsigned int audio_packet_type;
 	struct fixed31_32   f_audio;                 /* Audio rate (Hz) */
-	int      h_active;                /* Active pixels per line */
-	int      h_blank;                 /* Blanking pixels per line */
+	uint32_t h_active;                /* Active pixels per line */
+	uint32_t h_blank;                 /* Blanking pixels per line */
 	int      bpc;                     /* Bits per component */
 	int      vic;                     /* Video Identification Code */
 
@@ -432,6 +433,7 @@ struct dc_caps {
 	uint8_t num_of_dpias_per_host_router;
 	/* limit of the ODM only, could be limited by other factors (like pipe count)*/
 	uint8_t max_odm_combine_factor;
+	bool utm_support;
 };
 
 struct dc_bug_wa {
@@ -583,6 +585,7 @@ struct dc_config {
 	bool forced_clocks;
 	union allow_lttpr_non_transparent_mode allow_lttpr_non_transparent_mode;
 	bool multi_mon_pp_mclk_switch;
+	bool lsdma_peak_bw_contention_support;
 	bool disable_dmcu;
 	bool allow_4to1MPC;
 	bool enable_windowed_mpo_odm;
@@ -652,6 +655,7 @@ enum visual_confirm {
 	VISUAL_CONFIRM_VABC = 21,
 	VISUAL_CONFIRM_DCC = 22,
 	VISUAL_CONFIRM_BOOSTED_REFRESH_RATE = 23,
+	VISUAL_CONFIRM_DM_PASSTHROUGH = 26,
 	VISUAL_CONFIRM_EXPLICIT = 0x80000000,
 };
 
@@ -1284,8 +1288,10 @@ struct dc_debug_options {
 	bool dml21_force_pstate_method;
 	uint32_t dml21_force_pstate_method_values[MAX_PIPES];
 	uint32_t dml21_disable_pstate_method_mask;
+	bool force_optional_uclk_pstate_support;
 	union fw_assisted_mclk_switch_version fams_version;
 	union dmub_fams2_global_feature_config fams2_config;
+	bool fams2_imm_restore_drr;
 	unsigned int force_cositing;
 	unsigned int disable_spl;
 	unsigned int force_easf;
@@ -1319,8 +1325,11 @@ struct dc_debug_options {
 	bool enable_replay_esd_recovery;
 	uint8_t iommu_mismatch_temp_wka;
 	bool disable_dynamic_expansion_for_test_pattern;
+	bool psr_phy_force_phy_power_down_up_level_2;
 	uint32_t dml21_custom_derate_num_dpms;
 	uint32_t dml21_custom_derate_at_dpm[DML2_MAX_NUM_DPM_LVL];
+	bool override_utm_client_qc_profile;
+	uint8_t utm_client_qc_profiles[4];
 };
 
 
@@ -1507,11 +1516,6 @@ struct lut_mem_mapping {
 	uint16_t size;
 };
 
-struct dc_rmcm_3dlut {
-	bool isInUse;
-	const struct dc_stream_state *stream;
-};
-
 struct dc_3dlut {
 	struct kref refcount;
 	struct tetrahedral_params lut_3d;
@@ -2087,32 +2091,6 @@ struct dc_scaling_info {
 	struct scaling_taps scaling_quality;
 };
 
-struct dc_fast_update {
-	const struct dc_flip_addrs *flip_addr;
-	const struct dc_gamma *gamma;
-	const struct colorspace_transform *gamut_remap_matrix;
-	const struct dc_csc_transform *input_csc_color_matrix;
-	const struct fixed31_32 *coeff_reduction_factor;
-	struct dc_transfer_func *out_transfer_func;
-	struct dc_csc_transform *output_csc_transform;
-	const struct dc_csc_transform *cursor_csc_color_matrix;
-	struct cm_hist_control *cm_hist_control;
-	/* stream-level fast updates */
-	const struct colorspace_transform *gamut_remap;
-	const struct dc_cursor_attributes *cursor_attributes;
-	const struct dc_cursor_position *cursor_position;
-	const struct periodic_interrupt_config *periodic_interrupt;
-	const enum dc_dither_option *dither_option;
-	struct dc_info_packet *vrr_infopacket;
-	struct dc_info_packet *vsc_infopacket;
-	struct dc_info_packet *vsp_infopacket;
-	struct dc_info_packet *hfvsif_infopacket;
-	struct dc_info_packet *vtem_infopacket;
-	struct dc_info_packet *adaptive_sync_infopacket;
-	struct dc_info_packet *avi_infopacket;
-	struct dc_info_packet *hdr_static_metadata;
-};
-
 struct dc_surface_update {
 	struct dc_plane_state *surface;
 
@@ -2220,6 +2198,7 @@ struct dc_probe_latencies {
 /**
  * struct dc_probe_status - results for a probe.
  * @valid: true if a measurement was latched.
+ * @measuring: true if the hardware counter is currently running.
  * @type: type of the probe that produced this result.
  * @u.bandwidth_mbps:         peak BW in Mbps (DC_PROBE_PEAK_MEM_BW).
  * @u.latency:                min/max/avg memory latency in ns (DC_PROBE_MEM_LATENCY),
@@ -2229,6 +2208,7 @@ struct dc_probe_latencies {
  */
 struct dc_probe_status {
 	bool                       valid;
+	bool                       measuring;
 	enum dc_probe_type         type;
 	union {
 		uint32_t bandwidth_mbps;
@@ -2416,11 +2396,6 @@ bool dc_resource_is_dsc_encoding_supported(const struct dc *dc);
 void get_audio_check(struct audio_info *aud_modes,
 	struct audio_check *aud_chk);
 
-bool fast_nonaddr_updates_exist(struct dc_fast_update *fast_update, int surface_count);
-void populate_fast_updates(struct dc_fast_update *fast_update,
-		struct dc_surface_update *srf_updates,
-		int surface_count,
-		struct dc_stream_update *stream_update);
 /*
  * Set up streams and links associated to drive sinks
  * The streams parameter is an absolute set of all active streams.
@@ -3895,4 +3870,11 @@ unsigned int dc_override_memory_bandwidth_request(
 		struct dc *dc,
 		unsigned int bw_mbps);
 
+/**
+ * Panel Polarity Control
+ */
+void dc_link_set_panel_polarity_enable(struct dc_link *link, bool enable);
+void dc_link_panel_polarity_reset(struct dc_link *link);
+bool dc_link_get_panel_polarity(struct dc_link *link, int32_t *polarity);
+
 #endif /* DC_INTERFACE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 47fa3d4265e080..d9af53f720f76d 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -41,8 +41,6 @@
 #define DC_LOGGER CTX->logger
 #define GPINT_RETRY_NUM 20
 
-#define MAX_WAIT_US 100000
-
 static void dc_dmub_srv_construct(struct dc_dmub_srv *dc_srv, struct dc *dc,
 				  struct dmub_srv *dmub)
 {
@@ -93,7 +91,7 @@ bool dc_dmub_srv_wait_for_pending(struct dc_dmub_srv *dc_dmub_srv)
 	dmub = dc_dmub_srv->dmub;
 
 	do {
-		status = dmub_srv_wait_for_pending(dmub, MAX_WAIT_US);
+		status = dmub_srv_wait_for_pending(dmub, DMUB_CMD_DEFAULT_MAX_WAIT_US);
 	} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 	if (status != DMUB_STATUS_OK) {
@@ -123,7 +121,7 @@ void dc_dmub_srv_wait_for_inbox0_ack(struct dc_dmub_srv *dc_dmub_srv)
 	struct dc_context *dc_ctx = dc_dmub_srv->ctx;
 	enum dmub_status status = DMUB_STATUS_OK;
 
-	status = dmub_srv_wait_for_inbox0_ack(dmub, MAX_WAIT_US);
+	status = dmub_srv_wait_for_inbox0_ack(dmub, DMUB_CMD_DEFAULT_MAX_WAIT_US);
 	if (status != DMUB_STATUS_OK) {
 		DC_ERROR("Error waiting for INBOX0 HW Lock Ack\n");
 		dc_dmub_srv_handle_failure(dc_dmub_srv);
@@ -146,7 +144,8 @@ void dc_dmub_srv_send_inbox0_cmd(struct dc_dmub_srv *dc_dmub_srv,
 
 static bool dc_dmub_srv_reg_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_srv,
 		unsigned int count,
-		union dmub_rb_cmd *cmd_list)
+		const union dmub_rb_cmd *cmd_list,
+		unsigned int timeout_us)
 {
 	struct dc_context *dc_ctx;
 	struct dmub_srv *dmub;
@@ -162,7 +161,7 @@ static bool dc_dmub_srv_reg_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_s
 	for (i = 0 ; i < count; i++) {
 		/* confirm no messages pending */
 		do {
-			status = dmub_srv_wait_for_idle(dmub, MAX_WAIT_US);
+			status = dmub_srv_wait_for_idle(dmub, timeout_us);
 		} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 		/* queue command */
@@ -188,7 +187,8 @@ static bool dc_dmub_srv_reg_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_s
 
 static bool dc_dmub_srv_fb_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_srv,
 		unsigned int count,
-		union dmub_rb_cmd *cmd_list)
+		const union dmub_rb_cmd *cmd_list,
+		unsigned int timeout_us)
 {
 	struct dc_context *dc_ctx;
 	struct dmub_srv *dmub;
@@ -217,7 +217,7 @@ static bool dc_dmub_srv_fb_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_sr
 				return false;
 
 			do {
-					status = dmub_srv_wait_for_inbox_free(dmub, MAX_WAIT_US, count - i);
+					status = dmub_srv_wait_for_inbox_free(dmub, timeout_us, count - i);
 			} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 			/* Requeue the command. */
@@ -245,17 +245,18 @@ static bool dc_dmub_srv_fb_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_sr
 	return true;
 }
 
-bool dc_dmub_srv_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_srv,
+bool dc_dmub_srv_cmd_list_queue_execute_timeout(struct dc_dmub_srv *dc_dmub_srv,
 		unsigned int count,
-		union dmub_rb_cmd *cmd_list)
+		const union dmub_rb_cmd *cmd_list,
+		unsigned int timeout_us)
 {
 	bool res = false;
 
 	if (dc_dmub_srv && dc_dmub_srv->dmub) {
 		if (dc_dmub_srv->dmub->inbox_type == DMUB_CMD_INTERFACE_REG) {
-			res = dc_dmub_srv_reg_cmd_list_queue_execute(dc_dmub_srv, count, cmd_list);
+			res = dc_dmub_srv_reg_cmd_list_queue_execute(dc_dmub_srv, count, cmd_list, timeout_us);
 		} else {
-			res = dc_dmub_srv_fb_cmd_list_queue_execute(dc_dmub_srv, count, cmd_list);
+			res = dc_dmub_srv_fb_cmd_list_queue_execute(dc_dmub_srv, count, cmd_list, timeout_us);
 		}
 
 		if (res)
@@ -265,9 +266,60 @@ bool dc_dmub_srv_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_srv,
 	return res;
 }
 
+bool dc_dmub_srv_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_srv,
+		unsigned int count,
+		const union dmub_rb_cmd *cmd_list)
+{
+	return dc_dmub_srv_cmd_list_queue_execute_timeout(dc_dmub_srv, count, cmd_list, DMUB_CMD_DEFAULT_MAX_WAIT_US);
+}
+
+static void dc_dmub_srv_log_timeout(const struct dc_dmub_srv *dc_dmub_srv,
+		struct dmub_timeout_info *timeout_info,
+		union dmub_rb_cmd *cmd)
+{
+	struct dmub_srv *dmub;
+
+	if (!dc_dmub_srv || !dc_dmub_srv->dmub)
+		return;
+
+	dmub = dc_dmub_srv->dmub;
+
+	// timeout already ocurred, do not overwrite
+	if (timeout_info->timeout_occured)
+		return;
+
+	timeout_info->timeout_occured = true;
+	if (cmd)
+		timeout_info->timeout_cmd = *cmd;
+	timeout_info->timestamp = dm_get_timestamp(dc_dmub_srv->ctx);
+
+	// capture last trace entries from the DMCUB trace buffer
+	dmub_srv_get_trace_snapshot(dmub,
+			&timeout_info->trace_snapshot);
+}
+
+static void dc_dmub_srv_log_timeouts(struct dc_dmub_srv *dc_dmub_srv,
+		union dmub_rb_cmd *cmd)
+{
+	struct dmub_srv *dmub;
+
+	if (!dc_dmub_srv || !dc_dmub_srv->dmub)
+		return;
+
+	dmub = dc_dmub_srv->dmub;
+
+	// log if this is the first timeout
+	dc_dmub_srv_log_timeout(dc_dmub_srv, &dmub->debug.first_timeout_info, cmd);
+
+	// clear valid flag and log latest timeout
+	dmub->debug.latest_timeout_info.timeout_occured = false;
+	dc_dmub_srv_log_timeout(dc_dmub_srv, &dmub->debug.latest_timeout_info, cmd);
+}
+
 bool dc_dmub_srv_wait_for_idle(struct dc_dmub_srv *dc_dmub_srv,
 		enum dm_dmub_wait_type wait_type,
-		union dmub_rb_cmd *cmd_list)
+		union dmub_rb_cmd *cmd_list,
+		unsigned int timeout_us)
 {
 	struct dmub_srv *dmub;
 	enum dmub_status status;
@@ -280,17 +332,13 @@ bool dc_dmub_srv_wait_for_idle(struct dc_dmub_srv *dc_dmub_srv,
 	// Wait for DMUB to process command
 	if (wait_type != DM_DMUB_WAIT_TYPE_NO_WAIT) {
 		do {
-			status = dmub_srv_wait_for_idle(dmub, MAX_WAIT_US);
+			status = dmub_srv_wait_for_idle(dmub, timeout_us);
 		} while (dc_dmub_srv->ctx->dc->debug.disable_timeout && status != DMUB_STATUS_OK);
 
 		if (status != DMUB_STATUS_OK) {
 			DC_LOG_DEBUG("No reply for DMUB command: status=%d\n", status);
-			if (!dmub->debug.timeout_info.timeout_occured) {
-				dmub->debug.timeout_info.timeout_occured = true;
-				if (cmd_list)
-					dmub->debug.timeout_info.timeout_cmd = *cmd_list;
-				dmub->debug.timeout_info.timestamp = dm_get_timestamp(dc_dmub_srv->ctx);
-			}
+
+			dc_dmub_srv_log_timeouts(dc_dmub_srv, cmd_list);
 			dc_dmub_srv_handle_failure(dc_dmub_srv);
 			return false;
 		}
@@ -314,7 +362,7 @@ bool dc_dmub_srv_cmd_run_list(struct dc_dmub_srv *dc_dmub_srv, unsigned int coun
 	if (!dc_dmub_srv_cmd_list_queue_execute(dc_dmub_srv, count, cmd_list))
 		return false;
 
-	return dc_dmub_srv_wait_for_idle(dc_dmub_srv, wait_type, cmd_list);
+	return dc_dmub_srv_wait_for_idle(dc_dmub_srv, wait_type, cmd_list, DMUB_CMD_DEFAULT_MAX_WAIT_US);
 }
 
 bool dc_dmub_srv_optimized_init_done(struct dc_dmub_srv *dc_dmub_srv)
@@ -976,47 +1024,47 @@ void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv)
 	}
 
 	DC_LOG_DEBUG("DMCUB STATE:");
-	DC_LOG_DEBUG("    dmcub_version      : %08x", dc_dmub_srv->dmub->debug.dmcub_version);
-	DC_LOG_DEBUG("    scratch  [0]       : %08x", dc_dmub_srv->dmub->debug.scratch[0]);
-	DC_LOG_DEBUG("    scratch  [1]       : %08x", dc_dmub_srv->dmub->debug.scratch[1]);
-	DC_LOG_DEBUG("    scratch  [2]       : %08x", dc_dmub_srv->dmub->debug.scratch[2]);
-	DC_LOG_DEBUG("    scratch  [3]       : %08x", dc_dmub_srv->dmub->debug.scratch[3]);
-	DC_LOG_DEBUG("    scratch  [4]       : %08x", dc_dmub_srv->dmub->debug.scratch[4]);
-	DC_LOG_DEBUG("    scratch  [5]       : %08x", dc_dmub_srv->dmub->debug.scratch[5]);
-	DC_LOG_DEBUG("    scratch  [6]       : %08x", dc_dmub_srv->dmub->debug.scratch[6]);
-	DC_LOG_DEBUG("    scratch  [7]       : %08x", dc_dmub_srv->dmub->debug.scratch[7]);
-	DC_LOG_DEBUG("    scratch  [8]       : %08x", dc_dmub_srv->dmub->debug.scratch[8]);
-	DC_LOG_DEBUG("    scratch  [9]       : %08x", dc_dmub_srv->dmub->debug.scratch[9]);
-	DC_LOG_DEBUG("    scratch [10]       : %08x", dc_dmub_srv->dmub->debug.scratch[10]);
-	DC_LOG_DEBUG("    scratch [11]       : %08x", dc_dmub_srv->dmub->debug.scratch[11]);
-	DC_LOG_DEBUG("    scratch [12]       : %08x", dc_dmub_srv->dmub->debug.scratch[12]);
-	DC_LOG_DEBUG("    scratch [13]       : %08x", dc_dmub_srv->dmub->debug.scratch[13]);
-	DC_LOG_DEBUG("    scratch [14]       : %08x", dc_dmub_srv->dmub->debug.scratch[14]);
-	DC_LOG_DEBUG("    scratch [15]       : %08x", dc_dmub_srv->dmub->debug.scratch[15]);
+	DC_LOG_DEBUG("    dmcub_version      : %08x", dc_dmub_srv->dmub->debug.hw.dmcub_version);
+	DC_LOG_DEBUG("    scratch  [0]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[0]);
+	DC_LOG_DEBUG("    scratch  [1]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[1]);
+	DC_LOG_DEBUG("    scratch  [2]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[2]);
+	DC_LOG_DEBUG("    scratch  [3]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[3]);
+	DC_LOG_DEBUG("    scratch  [4]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[4]);
+	DC_LOG_DEBUG("    scratch  [5]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[5]);
+	DC_LOG_DEBUG("    scratch  [6]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[6]);
+	DC_LOG_DEBUG("    scratch  [7]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[7]);
+	DC_LOG_DEBUG("    scratch  [8]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[8]);
+	DC_LOG_DEBUG("    scratch  [9]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[9]);
+	DC_LOG_DEBUG("    scratch [10]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[10]);
+	DC_LOG_DEBUG("    scratch [11]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[11]);
+	DC_LOG_DEBUG("    scratch [12]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[12]);
+	DC_LOG_DEBUG("    scratch [13]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[13]);
+	DC_LOG_DEBUG("    scratch [14]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[14]);
+	DC_LOG_DEBUG("    scratch [15]       : %08x", dc_dmub_srv->dmub->debug.hw.scratch[15]);
 	for (i = 0; i < DMUB_PC_SNAPSHOT_COUNT; i++)
-		DC_LOG_DEBUG("    pc[%d]             : %08x", i, dc_dmub_srv->dmub->debug.pc[i]);
-	DC_LOG_DEBUG("    unk_fault_addr     : %08x", dc_dmub_srv->dmub->debug.undefined_address_fault_addr);
-	DC_LOG_DEBUG("    inst_fault_addr    : %08x", dc_dmub_srv->dmub->debug.inst_fetch_fault_addr);
-	DC_LOG_DEBUG("    data_fault_addr    : %08x", dc_dmub_srv->dmub->debug.data_write_fault_addr);
-	DC_LOG_DEBUG("    inbox1_rptr        : %08x", dc_dmub_srv->dmub->debug.inbox1_rptr);
-	DC_LOG_DEBUG("    inbox1_wptr        : %08x", dc_dmub_srv->dmub->debug.inbox1_wptr);
-	DC_LOG_DEBUG("    inbox1_size        : %08x", dc_dmub_srv->dmub->debug.inbox1_size);
-	DC_LOG_DEBUG("    inbox0_rptr        : %08x", dc_dmub_srv->dmub->debug.inbox0_rptr);
-	DC_LOG_DEBUG("    inbox0_wptr        : %08x", dc_dmub_srv->dmub->debug.inbox0_wptr);
-	DC_LOG_DEBUG("    inbox0_size        : %08x", dc_dmub_srv->dmub->debug.inbox0_size);
-	DC_LOG_DEBUG("    outbox1_rptr       : %08x", dc_dmub_srv->dmub->debug.outbox1_rptr);
-	DC_LOG_DEBUG("    outbox1_wptr       : %08x", dc_dmub_srv->dmub->debug.outbox1_wptr);
-	DC_LOG_DEBUG("    outbox1_size       : %08x", dc_dmub_srv->dmub->debug.outbox1_size);
-	DC_LOG_DEBUG("    is_enabled         : %d", dc_dmub_srv->dmub->debug.is_dmcub_enabled);
-	DC_LOG_DEBUG("    is_soft_reset      : %d", dc_dmub_srv->dmub->debug.is_dmcub_soft_reset);
-	DC_LOG_DEBUG("    is_secure_reset    : %d", dc_dmub_srv->dmub->debug.is_dmcub_secure_reset);
-	DC_LOG_DEBUG("    is_traceport_en    : %d", dc_dmub_srv->dmub->debug.is_traceport_en);
-	DC_LOG_DEBUG("    is_cw0_en          : %d", dc_dmub_srv->dmub->debug.is_cw0_enabled);
-	DC_LOG_DEBUG("    is_cw6_en          : %d", dc_dmub_srv->dmub->debug.is_cw6_enabled);
-	DC_LOG_DEBUG("    is_pwait           : %d", dc_dmub_srv->dmub->debug.is_pwait);
-}
-
-static bool dc_dmub_should_update_cursor_data(struct pipe_ctx *pipe_ctx)
+		DC_LOG_DEBUG("    pc[%d]             : %08x", i, dc_dmub_srv->dmub->debug.hw.pc[i]);
+	DC_LOG_DEBUG("    unk_fault_addr     : %08x", dc_dmub_srv->dmub->debug.hw.undefined_address_fault_addr);
+	DC_LOG_DEBUG("    inst_fault_addr    : %08x", dc_dmub_srv->dmub->debug.hw.inst_fetch_fault_addr);
+	DC_LOG_DEBUG("    data_fault_addr    : %08x", dc_dmub_srv->dmub->debug.hw.data_write_fault_addr);
+	DC_LOG_DEBUG("    inbox1_rptr        : %08x", dc_dmub_srv->dmub->debug.hw.inbox1_rptr);
+	DC_LOG_DEBUG("    inbox1_wptr        : %08x", dc_dmub_srv->dmub->debug.hw.inbox1_wptr);
+	DC_LOG_DEBUG("    inbox1_size        : %08x", dc_dmub_srv->dmub->debug.hw.inbox1_size);
+	DC_LOG_DEBUG("    inbox0_rptr        : %08x", dc_dmub_srv->dmub->debug.hw.inbox0_rptr);
+	DC_LOG_DEBUG("    inbox0_wptr        : %08x", dc_dmub_srv->dmub->debug.hw.inbox0_wptr);
+	DC_LOG_DEBUG("    inbox0_size        : %08x", dc_dmub_srv->dmub->debug.hw.inbox0_size);
+	DC_LOG_DEBUG("    outbox1_rptr       : %08x", dc_dmub_srv->dmub->debug.hw.outbox1_rptr);
+	DC_LOG_DEBUG("    outbox1_wptr       : %08x", dc_dmub_srv->dmub->debug.hw.outbox1_wptr);
+	DC_LOG_DEBUG("    outbox1_size       : %08x", dc_dmub_srv->dmub->debug.hw.outbox1_size);
+	DC_LOG_DEBUG("    is_enabled         : %d", dc_dmub_srv->dmub->debug.hw.is_dmcub_enabled);
+	DC_LOG_DEBUG("    is_soft_reset      : %d", dc_dmub_srv->dmub->debug.hw.is_dmcub_soft_reset);
+	DC_LOG_DEBUG("    is_secure_reset    : %d", dc_dmub_srv->dmub->debug.hw.is_dmcub_secure_reset);
+	DC_LOG_DEBUG("    is_traceport_en    : %d", dc_dmub_srv->dmub->debug.hw.is_traceport_en);
+	DC_LOG_DEBUG("    is_cw0_en          : %d", dc_dmub_srv->dmub->debug.hw.is_cw0_enabled);
+	DC_LOG_DEBUG("    is_cw6_en          : %d", dc_dmub_srv->dmub->debug.hw.is_cw6_enabled);
+	DC_LOG_DEBUG("    is_pwait           : %d", dc_dmub_srv->dmub->debug.hw.is_pwait);
+}
+
+bool dc_dmub_should_update_cursor_data(struct pipe_ctx *pipe_ctx)
 {
 	if (pipe_ctx->plane_state != NULL) {
 		if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE ||
@@ -1036,20 +1084,16 @@ static bool dc_dmub_should_update_cursor_data(struct pipe_ctx *pipe_ctx)
 }
 
 static void dc_build_cursor_update_payload0(
-		struct pipe_ctx *pipe_ctx, uint8_t p_idx,
+		const struct dc_context *ctx, uint8_t p_idx,
+		struct hubp *hubp, uint8_t otg_inst, uint8_t panel_inst,
 		struct dmub_cmd_update_cursor_payload0 *payload)
 {
-	struct dc *dc = pipe_ctx->stream->ctx->dc;
-	struct hubp *hubp = pipe_ctx->plane_res.hubp;
-	unsigned int panel_inst = 0;
+	struct dc *dc = ctx->dc;
 
 	if (dc->config.frame_update_cmd_version2 == true) {
 		/* Don't need panel_inst for command version2 */
 		payload->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_2;
 	} else {
-		if (!dc_get_edp_link_panel_inst(hubp->ctx->dc,
-			pipe_ctx->stream->link, &panel_inst))
-			return;
 		payload->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_1;
 	}
 
@@ -1064,8 +1108,8 @@ static void dc_build_cursor_update_payload0(
 
 	payload->enable      = (uint8_t)hubp->pos.cur_ctl.bits.cur_enable;
 	payload->pipe_idx    = p_idx;
-	payload->panel_inst  = (uint8_t)panel_inst;
-	payload->otg_inst    = (uint8_t)pipe_ctx->stream_res.tg->inst;
+	payload->panel_inst  = panel_inst;
+	payload->otg_inst    = otg_inst;
 }
 
 static void dc_build_cursor_position_update_payload0(
@@ -1102,14 +1146,21 @@ static void dc_build_cursor_attribute_update_payload1(
 /**
  * dc_send_update_cursor_info_to_dmu - Populate the DMCUB Cursor update info command
  *
- * @pCtx: [in] pipe context
+ * @ctx: [in] dc context
  * @pipe_idx: [in] pipe index
+ * @hubp: [in] hubp resource providing cursor position/attribute caches
+ * @dpp: [in] dpp resource providing cursor position/attribute caches
+ * @otg_inst: [in] OTG instance driving the pipe
+ * @panel_inst: [in] eDP panel instance (command version 1 only)
  *
  * This function would store the cursor related information and pass it into
- * dmub
+ * dmub. The caller is responsible for gating with
+ * dc_dmub_should_update_cursor_data().
  */
 void dc_send_update_cursor_info_to_dmu(
-		struct pipe_ctx *pCtx, uint8_t pipe_idx)
+		const struct dc_context *ctx, uint8_t pipe_idx,
+		struct hubp *hubp, struct dpp *dpp,
+		uint8_t otg_inst, uint8_t panel_inst)
 {
 	union dmub_rb_cmd cmd[2];
 	union dmub_cmd_update_cursor_info_data *update_cursor_info_0 =
@@ -1117,8 +1168,6 @@ void dc_send_update_cursor_info_to_dmu(
 
 	memset(cmd, 0, sizeof(cmd));
 
-	if (!dc_dmub_should_update_cursor_data(pCtx))
-		return;
 	/*
 	 * Since we use multi_cmd_pending for dmub command, the 2nd command is
 	 * only assigned to store cursor attributes info.
@@ -1138,11 +1187,12 @@ void dc_send_update_cursor_info_to_dmu(
 		cmd[0].update_cursor_info.header.multi_cmd_pending = 1; //To combine multi dmu cmd, 1st cmd
 
 		/* Prepare Payload */
-		dc_build_cursor_update_payload0(pCtx, pipe_idx, &update_cursor_info_0->payload0);
+		dc_build_cursor_update_payload0(ctx, pipe_idx, hubp, otg_inst, panel_inst,
+				&update_cursor_info_0->payload0);
 
 		dc_build_cursor_position_update_payload0(&update_cursor_info_0->payload0, pipe_idx,
-				pCtx->plane_res.hubp, pCtx->plane_res.dpp);
-		}
+				hubp, dpp);
+	}
 	{
 		/* Build Payload#1 Header */
 		cmd[1].update_cursor_info.header.type = DMUB_CMD__UPDATE_CURSOR_INFO;
@@ -1151,10 +1201,10 @@ void dc_send_update_cursor_info_to_dmu(
 
 		dc_build_cursor_attribute_update_payload1(
 				&cmd[1].update_cursor_info.update_cursor_info_data.payload1.attribute_cfg,
-				pipe_idx, pCtx->plane_res.hubp, pCtx->plane_res.dpp);
+				pipe_idx, hubp, dpp);
 
 		/* Combine 2nd cmds update_curosr_info to DMU */
-		dc_wake_and_execute_dmub_cmd_list(pCtx->stream->ctx, 2, cmd, DM_DMUB_WAIT_TYPE_WAIT);
+		dc_wake_and_execute_dmub_cmd_list(ctx, 2, cmd, DM_DMUB_WAIT_TYPE_WAIT);
 	}
 }
 
@@ -1368,7 +1418,7 @@ static void dc_dmub_srv_notify_idle(const struct dc *dc, bool allow_idle)
 			ips_fw->signals.bits.ips1_commit,
 			ips_fw->signals.bits.ips2_commit);
 
-		dc_dmub_srv_wait_for_idle(dc->ctx->dmub_srv, DM_DMUB_WAIT_TYPE_WAIT, NULL);
+		dc_dmub_srv_wait_for_idle(dc->ctx->dmub_srv, DM_DMUB_WAIT_TYPE_WAIT, NULL, DMUB_CMD_DEFAULT_MAX_WAIT_US);
 
 		memset(&new_signals, 0, sizeof(new_signals));
 
@@ -1845,6 +1895,9 @@ static void dc_dmub_srv_rb_based_fams2_update_config(struct dc *dc,
 			memcpy(&stream_sub_state_cmd->config,
 					&context->bw_ctx.bw.dcn.fams2_stream_sub_params[i],
 					sizeof(union dmub_cmd_fams2_config));
+
+			/* temp override */
+			stream_base_cmd->config.stream_v1.base.config.bits.imm_restore_drr = dc->debug.fams2_imm_restore_drr;
 		}
 	}
 
@@ -2475,3 +2528,73 @@ void dc_dmub_srv_get_fams2_debug_meta(struct dc_dmub_srv *dc_dmub_srv)
 	dm_execute_dmub_cmd_list(dc_dmub_srv->ctx, 1, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 
 }
+
+void dc_dmub_srv_panel_polarity_set_enable(struct dc_dmub_srv *dc_dmub_srv, uint8_t panel_inst, bool enable)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *ctx = dc_dmub_srv->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	cmd.panel_polarity_enable.header.type = DMUB_CMD__PANEL_POLARITY;
+	cmd.panel_polarity_enable.header.sub_type = DMUB_CMD__PANEL_POLARITY_ENABLE;
+	cmd.panel_polarity_enable.header.payload_bytes = sizeof(cmd.panel_polarity_enable.data);
+	cmd.panel_polarity_enable.data.enable = enable ? 1 : 0;
+	cmd.panel_polarity_enable.data.otg_inst = panel_inst;
+
+	dc_wake_and_execute_dmub_cmd(ctx, &cmd, DM_DMUB_WAIT_TYPE_NO_WAIT);
+}
+
+void dc_dmub_srv_panel_polarity_reset(struct dc_dmub_srv *dc_dmub_srv, uint8_t panel_inst)
+{
+	union dmub_rb_cmd cmd;
+	struct dc_context *ctx = dc_dmub_srv->ctx;
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	cmd.panel_polarity_enable.header.type = DMUB_CMD__PANEL_POLARITY;
+	cmd.panel_polarity_enable.header.sub_type = DMUB_CMD__PANEL_POLARITY_RESET;
+	cmd.panel_polarity_enable.header.payload_bytes = sizeof(cmd.panel_polarity_enable.data);
+	cmd.panel_polarity_enable.data.otg_inst = panel_inst;
+
+	dc_wake_and_execute_dmub_cmd(ctx, &cmd, DM_DMUB_WAIT_TYPE_NO_WAIT);
+}
+
+bool dc_dmub_srv_panel_polarity_get_polarity(struct dc_dmub_srv *dc_dmub_srv, uint8_t panel_inst, int32_t *polarity)
+{
+	bool ret = false;
+	uint32_t raw_polarity = 0;
+	uint32_t retry_count = 0;
+	struct dc_context *ctx = dc_dmub_srv->ctx;
+
+	*polarity = 0;
+
+	do {
+		/* Send gpint command and wait for ack */
+		if (dc_wake_and_execute_gpint(ctx, DMUB_GPINT__PANEL_POLARITY_GET_BIAS, panel_inst, &raw_polarity,
+			DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY)) {
+			*polarity = (int32_t)raw_polarity;
+			ret = true;
+		}
+	} while (++retry_count <= 1000 && ret == false);
+
+	return ret;
+}
+
+void dc_dmub_srv_hubbub_set_riommu_pctrl(const struct dc_context *ctx, uint32_t value)
+{
+	union dmub_rb_cmd cmd;
+
+	if (!(ctx->dce_version == DCN_VERSION_4_2 || ctx->dce_version == DCN_VERSION_4_2B))
+		return;
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	cmd.dc_bls_dchvm_init.header.type = DMUB_CMD__DC_BLS;
+	cmd.dc_bls_dchvm_init.header.sub_type = DMUB_CMD__DC_BLS_DCHVM_INIT;
+	cmd.dc_bls_dchvm_init.header.payload_bytes = sizeof(struct dmub_cmd_dc_bls_dchvm_init_data);
+
+	cmd.dc_bls_dchvm_init.data.riommu_pctrl_val = value;
+
+	dc_wake_and_execute_dmub_cmd(ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
index 6d15ed77c17de7..8056824d90e08a 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
@@ -29,9 +29,14 @@
 #include "dm_services_types.h"
 #include "dmub/dmub_srv.h"
 
+#define DMUB_CMD_DEFAULT_MAX_WAIT_US 500000
+
 struct dmub_srv;
 struct dc;
+struct dc_context;
 struct pipe_ctx;
+struct hubp;
+struct dpp;
 struct dc_crtc_timing_adjust;
 struct dc_crtc_timing;
 struct dc_state;
@@ -56,11 +61,17 @@ bool dc_dmub_srv_optimized_init_done(struct dc_dmub_srv *dc_dmub_srv);
 
 bool dc_dmub_srv_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_srv,
 		unsigned int count,
-		union dmub_rb_cmd *cmd_list);
+		const union dmub_rb_cmd *cmd_list);
+
+bool dc_dmub_srv_cmd_list_queue_execute_timeout(struct dc_dmub_srv *dc_dmub_srv,
+		unsigned int count,
+		const union dmub_rb_cmd *cmd_list,
+		unsigned int timeout_us);
 
 bool dc_dmub_srv_wait_for_idle(struct dc_dmub_srv *dc_dmub_srv,
 		enum dm_dmub_wait_type wait_type,
-		union dmub_rb_cmd *cmd_list);
+		union dmub_rb_cmd *cmd_list,
+		unsigned int timeout_us);
 
 bool dc_dmub_srv_cmd_run(struct dc_dmub_srv *dc_dmub_srv, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type);
 
@@ -91,7 +102,9 @@ bool dc_dmub_srv_get_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv);
 void dc_dmub_setup_subvp_dmub_command(struct dc *dc, struct dc_state *context, bool enable);
 void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv);
 
-void dc_send_update_cursor_info_to_dmu(struct pipe_ctx *pCtx, uint8_t pipe_idx);
+bool dc_dmub_should_update_cursor_data(struct pipe_ctx *pipe_ctx);
+void dc_send_update_cursor_info_to_dmu(const struct dc_context *ctx, uint8_t pipe_idx,
+		struct hubp *hubp, struct dpp *dpp, uint8_t otg_inst, uint8_t panel_inst);
 bool dc_dmub_check_min_version(struct dmub_srv *srv);
 
 void dc_dmub_srv_enable_dpia_trace(const struct dc *dc);
@@ -424,4 +437,43 @@ bool dc_dmub_srv_ihc_set_dig_hdcp_interrupt_dest(
  * @dc_dmub_srv - pointer to DMUB service object
  */
 void dc_dmub_srv_get_fams2_debug_meta(struct dc_dmub_srv *dc_dmub_srv);
+
+/**
+ * dc_dmub_srv_panel_polarity_set_enable() - Enables or disables panel polarity.
+ *
+ * @dc_dmub_srv: DMUB service handle
+ * @panel_inst: Panel instance
+ * @enable: true to enable, false to disable
+ */
+void dc_dmub_srv_panel_polarity_set_enable(struct dc_dmub_srv *dc_dmub_srv, uint8_t panel_inst, bool enable);
+
+/**
+ * dc_dmub_srv_panel_polarity_reset() - Resets panel polarity.
+ *
+ * @dc_dmub_srv: DMUB service handle
+ * @panel_inst: Panel instance
+ */
+void dc_dmub_srv_panel_polarity_reset(struct dc_dmub_srv *dc_dmub_srv, uint8_t panel_inst);
+
+/**
+ * dc_dmub_srv_panel_polarity_get_polarity() - Gets the current panel polarity.
+ *
+ * @dc_dmub_srv: DMUB service handle
+ * @panel_inst: Panel instance
+ * @polarity: Pointer to store the current polarity
+ */
+bool dc_dmub_srv_panel_polarity_get_polarity(struct dc_dmub_srv *dc_dmub_srv, uint8_t panel_inst, int32_t *polarity);
+
+/**
+ * dc_dmub_srv_hubbub_set_riommu_pctrl() - Program the RIOMMU PCTRL register via DMCUB.
+ *
+ * The RIOMMU PCTRL register (SMN address 0x16B0A100) controls the SDP port
+ * disconnection hysteresis between rIOMMU & DCHVM. It is not accessible from
+ * x86, so the write is offloaded to DMCUB.
+ *
+ * @ctx: dc context
+ * @value: value to program into the register (e.g. 0x20)
+ */
+void dc_dmub_srv_hubbub_set_riommu_pctrl(const struct dc_context *ctx, uint32_t value);
+
 #endif /* _DMUB_DC_SRV_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c
deleted file mode 100644
index 0db5b49e9d5e90..00000000000000
--- a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dce/dce_dmcu.h"
-#include "dc_edid_parser.h"
-
-bool dc_edid_parser_send_cea(struct dc *dc,
-		int offset,
-		int total_length,
-		uint8_t *data,
-		int length)
-{
-	struct dmcu *dmcu = dc->res_pool->dmcu;
-
-	if (dmcu &&
-	    dmcu->funcs->is_dmcu_initialized(dmcu) &&
-	    dmcu->funcs->send_edid_cea) {
-		return dmcu->funcs->send_edid_cea(dmcu,
-				offset,
-				total_length,
-				data,
-				length);
-	}
-
-	return false;
-}
-
-bool dc_edid_parser_recv_cea_ack(struct dc *dc, int *offset)
-{
-	struct dmcu *dmcu = dc->res_pool->dmcu;
-
-	if (dmcu &&
-	    dmcu->funcs->is_dmcu_initialized(dmcu) &&
-	    dmcu->funcs->recv_edid_cea_ack) {
-		return dmcu->funcs->recv_edid_cea_ack(dmcu, offset);
-	}
-
-	return false;
-}
-
-bool dc_edid_parser_recv_amd_vsdb(struct dc *dc,
-		int *version,
-		int *min_frame_rate,
-		int *max_frame_rate)
-{
-	struct dmcu *dmcu = dc->res_pool->dmcu;
-
-	if (dmcu &&
-	    dmcu->funcs->is_dmcu_initialized(dmcu) &&
-	    dmcu->funcs->recv_amd_vsdb) {
-		return dmcu->funcs->recv_amd_vsdb(dmcu,
-				version,
-				min_frame_rate,
-				max_frame_rate);
-	}
-
-	return false;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h
deleted file mode 100644
index da67ec06f0a2f3..00000000000000
--- a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DC_EDID_PARSER_H_
-#define _DC_EDID_PARSER_H_
-
-#include "core_types.h"
-
-bool dc_edid_parser_send_cea(struct dc *dc,
-		int offset,
-		int total_length,
-		uint8_t *data,
-		int length);
-
-bool dc_edid_parser_recv_cea_ack(struct dc *dc, int *offset);
-
-bool dc_edid_parser_recv_amd_vsdb(struct dc *dc,
-		int *version,
-		int *min_frame_rate,
-		int *max_frame_rate);
-
-#endif /* _DC_EDID_PARSER_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index aeac5a45e13b78..40cd4e5d134000 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -269,6 +269,14 @@ enum dc_pixel_format {
 	PIXEL_FORMAT_UNKNOWN
 };
 
+#define MAX_TG_COLOR_VALUE 0x3FF
+struct tg_color {
+	/* Maximum 10 bits color value */
+	uint16_t color_r_cr;
+	uint16_t color_g_y;
+	uint16_t color_b_cb;
+};
+
 /*
  * This structure holds a surface address.  There could be multiple addresses
  * in cases such as Stereo 3D, Planar YUV, etc.  Other per-flip attributes such
@@ -283,6 +291,7 @@ struct dc_flip_addrs {
 	bool triplebuffer_flips;
 	unsigned int dirty_rect_count;
 	struct rect dirty_rects[DC_MAX_DIRTY_RECTS];
+	struct tg_color visual_confirm_color; /* DM passthrough visual confirm color, applied in VISUAL_CONFIRM_DM_PASSTHROUGH mode */
 };
 
 enum tile_split_values {
@@ -553,6 +562,9 @@ struct dc_cursor_mi_param {
 	enum dc_rotation_angle rotation;
 	bool mirror;
 	struct dc_stream_state *stream;
+	int dst_x_offset;
+	int cur_rect_x;
+	int cur_rect_y;
 };
 
 /* IPP related types */
@@ -1187,15 +1199,6 @@ struct mcif_buf_params {
 	uint8_t				tmz_id;
 };
 
-
-#define MAX_TG_COLOR_VALUE 0x3FF
-struct tg_color {
-	/* Maximum 10 bits color value */
-	uint16_t color_r_cr;
-	uint16_t color_g_y;
-	uint16_t color_b_cb;
-};
-
 struct fva_adj {
 	unsigned int pixel_clock_100hz;
 	unsigned int max_pixel_clock_100hz;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_memory_pool.c b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.c
new file mode 100644
index 00000000000000..087f04d314592b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.c
@@ -0,0 +1,275 @@
+/**
+ * Copyright (C) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * You may not use this software and documentation (if any) (collectively, the
+ * "Materials") except in compliance with the terms and conditions of the
+ * Software License Agreement included with the Materials or otherwise as set
+ * forth in writing and signed by you and an authorized signatory of AMD.
+ *
+ * If you do not have a copy of the Software License Agreement, contact your AMD
+ * representative for a copy. You agree that you will not reverse engineer or
+ * decompile the Materials, in whole or in part, except as allowed by applicable
+ * law.
+ *
+ * THE MATERIALS ARE DISTRIBUTED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+ */
+#include "dc_memory_pool.h"
+
+#include <linux/atomic.h>
+
+enum {
+	DC_MEMORY_POOL_SENTINEL = -1,
+	DC_MEMORY_POOL_ACQUIRED = -2,
+	DC_MEMORY_POOL_RELEASED = -3,
+
+	// Correct autoincrement for negative numbers
+	DC_MEMORY_POOL_ENUM_SIZE_IMPL,
+	DC_MEMORY_POOL_ENUM_SIZE = 1 - DC_MEMORY_POOL_ENUM_SIZE_IMPL,
+};
+
+// Align everything to size of page to avoid false sharing
+struct dc_memory_pool_page {
+	__aligned(PAGE_SIZE) char _dummy[PAGE_SIZE];
+};
+
+// Generation counter protects against ABA-problem
+union index_t {
+	struct {
+		int32_t value;
+		uint32_t generation;
+	} s;
+
+	int64_t raw;
+};
+
+struct dc_memory_pool {
+	// Values and pointers constant after initialization, can share page
+	__aligned(PAGE_SIZE) size_t size;
+	size_t capacity;
+
+	void *unaligned_pool;
+	void *unaligned_memory;
+	struct dc_memory_pool_page *memory;
+	atomic_t *free_list;
+
+	// Updated every operation, use separate page to avoid false sharing
+	__aligned(PAGE_SIZE) atomic64_t free_head; // index_t
+	char _reserved2[PAGE_SIZE - sizeof(atomic64_t)];
+};
+
+static_assert(sizeof(struct dc_memory_pool) == 2 * PAGE_SIZE);
+static_assert(offsetof(struct dc_memory_pool, free_head) == PAGE_SIZE);
+
+static size_t divide_ceiling(size_t x, size_t d)
+{
+	return (x + d - 1) / d;
+}
+
+static size_t round_up_to_multiple(size_t x, size_t m)
+{
+	return divide_ceiling(x, m) * m;
+}
+
+static size_t size_in_pages(size_t x)
+{
+	return divide_ceiling(x, PAGE_SIZE);
+}
+
+static void *page_align_up(void *p)
+{
+	intptr_t i = (intptr_t)p;
+
+	i = (intptr_t)round_up_to_multiple((size_t)i, PAGE_SIZE);
+	return (void *)i;
+}
+
+__must_check struct dc_memory_pool *dc_memory_pool_create(size_t size,
+							  size_t capacity)
+{
+	if (!size || !capacity)
+		return NULL;
+
+	// Limited by split between size and generation in index_t
+	if (capacity >= (uint32_t)-DC_MEMORY_POOL_ENUM_SIZE)
+		return NULL;
+
+	// uint32_t because that's what alloc functions take
+	const uint32_t block_pages = (uint32_t)size_in_pages(size);
+	const uint32_t lines = (uint32_t)capacity * block_pages;
+	const uint32_t padded_struct_size =
+		sizeof(struct dc_memory_pool) + PAGE_SIZE;
+
+	void *unaligned_pool = kzalloc(padded_struct_size, GFP_KERNEL);
+
+	if (!unaligned_pool)
+		return NULL;
+
+	struct dc_memory_pool *pool = page_align_up(unaligned_pool);
+
+	// Compound literals create temporary in debug driver, exceeding stack size
+	pool->size = size;
+	pool->capacity = capacity;
+	pool->unaligned_pool = unaligned_pool;
+	pool->unaligned_memory = kcalloc(lines + 1, PAGE_SIZE, GFP_KERNEL);
+	pool->memory = page_align_up(pool->unaligned_memory);
+	pool->free_list = kcalloc((uint32_t)capacity, sizeof(atomic_t), GFP_KERNEL);
+	pool->free_head = (atomic64_t)ATOMIC_INIT(0);
+
+	if (!pool->unaligned_memory || !pool->free_list) {
+		dc_memory_pool_destroy(pool);
+		return NULL;
+	}
+
+	for (int32_t i = 0; i < (int32_t)capacity; i++)
+		atomic_set_release(&pool->free_list[i], i + 1);
+
+	atomic_set_release(&pool->free_list[capacity - 1],
+			   DC_MEMORY_POOL_SENTINEL);
+
+	return pool;
+}
+
+void dc_memory_pool_destroy(struct dc_memory_pool *pool)
+{
+	if (!pool)
+		return;
+
+	kfree(pool->free_list);
+	kfree(pool->unaligned_memory);
+	kfree(pool->unaligned_pool);
+}
+
+static int32_t dc_memory_pool_get_index(const struct dc_memory_pool *pool,
+					const void *memory)
+{
+	if (!pool || !memory) {
+		ASSERT(false);
+		return DC_MEMORY_POOL_SENTINEL;
+	}
+
+	// Direct pointer arithmetic would be UB if called by false `owns()`
+	if ((uintptr_t)memory < (uintptr_t)pool->memory)
+		return DC_MEMORY_POOL_SENTINEL;
+
+	const uintptr_t distance = (uintptr_t)memory - (uintptr_t)pool->memory;
+	const size_t block_size = size_in_pages(pool->size) * PAGE_SIZE;
+	const size_t i = (size_t)distance / block_size;
+
+	if (distance % (uintptr_t)block_size != 0)
+		return DC_MEMORY_POOL_SENTINEL;
+
+	if (i >= pool->capacity)
+		return DC_MEMORY_POOL_SENTINEL;
+
+	return (int32_t)i;
+}
+
+static void *dc_memory_pool_get_page(const struct dc_memory_pool *pool,
+				     int32_t index)
+{
+	if (!pool) {
+		ASSERT(false);
+		return NULL;
+	}
+
+	if (index < 0 || index >= (int32_t)pool->capacity) {
+		ASSERT(false);
+		return NULL;
+	}
+
+	return &pool->memory[(size_t)index * size_in_pages(pool->size)];
+}
+
+__must_check void *dc_memory_pool_acquire(struct dc_memory_pool *pool)
+{
+	if (!pool) {
+		ASSERT(false);
+		return NULL;
+	}
+
+	// CAS-atomic `out = head; head = head->next;`
+	union index_t old_head = {
+		.raw = atomic64_read_acquire(&pool->free_head),
+	};
+	union index_t new_head = {
+		.raw = 0,
+	};
+	int32_t i = 0;
+
+	do {
+		i = old_head.s.value;
+		if (i == DC_MEMORY_POOL_SENTINEL)
+			return NULL;
+
+		new_head = (union index_t){
+			.s.value = atomic_read_acquire(&pool->free_list[i]),
+			.s.generation = old_head.s.generation + 1,
+		};
+	} while (!atomic64_try_cmpxchg(&pool->free_head, &old_head.raw,
+				       new_head.raw));
+
+	atomic_set_release(&pool->free_list[i], DC_MEMORY_POOL_ACQUIRED);
+	return dc_memory_pool_get_page(pool, i);
+}
+
+void dc_memory_pool_release(struct dc_memory_pool *pool, void *memory)
+{
+	if (!dc_memory_pool_owns(pool, memory)) {
+		// Likely acquired in different pool or (racing?) double free
+		ASSERT(false);
+		return;
+	}
+
+	const int32_t i = dc_memory_pool_get_index(pool, memory);
+
+	if (atomic_xchg(&pool->free_list[i], DC_MEMORY_POOL_RELEASED) !=
+	    DC_MEMORY_POOL_ACQUIRED) {
+		// Double free from two racing threads, use krefs to sync
+		ASSERT(false);
+		return;
+	}
+
+	// CAS-atomic `in->next = head; head = in;`
+	union index_t old_head = {
+		.raw = atomic64_read_acquire(&pool->free_head),
+	};
+	union index_t new_head = {
+		.raw = 0,
+	};
+
+	do {
+		atomic_set_release(&pool->free_list[i], old_head.s.value);
+		new_head = (union index_t){
+			.s.value = i,
+			.s.generation = old_head.s.generation + 1,
+		};
+	} while (!atomic64_try_cmpxchg(&pool->free_head, &old_head.raw,
+				       new_head.raw));
+}
+
+__must_check bool dc_memory_pool_owns(const struct dc_memory_pool *pool,
+				      const void *memory)
+{
+	const int32_t i = dc_memory_pool_get_index(pool, memory);
+
+	if (i < 0)
+		return false;
+
+	if (atomic_read_acquire(&pool->free_list[i]) != DC_MEMORY_POOL_ACQUIRED)
+		return false;
+
+	return true;
+}
+
+__must_check size_t dc_memory_pool_size(const struct dc_memory_pool *pool)
+{
+	ASSERT(pool);
+	return pool->size;
+}
+
+__must_check size_t dc_memory_pool_capacity(const struct dc_memory_pool *pool)
+{
+	ASSERT(pool);
+	return pool->capacity;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dc_memory_pool.h b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.h
new file mode 100644
index 00000000000000..0ec305813d66b6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dc_memory_pool.h
@@ -0,0 +1,107 @@
+/**
+ * Copyright (C) Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * You may not use this software and documentation (if any) (collectively, the
+ * "Materials") except in compliance with the terms and conditions of the
+ * Software License Agreement included with the Materials or otherwise as set
+ * forth in writing and signed by you and an authorized signatory of AMD.
+ *
+ * If you do not have a copy of the Software License Agreement, contact your AMD
+ * representative for a copy. You agree that you will not reverse engineer or
+ * decompile the Materials, in whole or in part, except as allowed by applicable
+ * law.
+ *
+ * THE MATERIALS ARE DISTRIBUTED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+ */
+#ifndef DC_MEMORY_POOL_H
+#define DC_MEMORY_POOL_H
+
+#include "os_types.h"
+
+/**
+ * Non-resizable lock-free memory pool with fixed capacity.
+ *
+ * Custom implementation is used over Linux llist, as llist requires external
+ * locking if more than one thread pops from the list.
+ *
+ * Emplace and erase operations do not use any mutexes or spinlocks, which
+ * guarantees forward progress even if another thread
+ * has been suspended in the middle of the call. High contention might result
+ * in multiple internal retries, but will eventually either succeed once other
+ * threads stop actively modifying the pool, or fail if the pool is empty.
+ *
+ * Implemented as Treiber stack using indexed free list and head with generation
+ * counter to solve ABA problem, as each modification of the head increments
+ * the generation, preventing issue of `push(A)` being indistinguishable from
+ * `push(A); { push(B); pop(B); }` to another thread, corrupting data structure.
+ */
+struct dc_memory_pool;
+
+/**
+ * Create dc_memory_pool.
+ *
+ * @param size Non-zero pool block size, all acquires will be of this size.
+ * @param capacity Non-zero number of blocks that can be acquired from the pool.
+ * @return Pointer to the pool if succeeded, null if failed.
+ */
+__must_check struct dc_memory_pool *dc_memory_pool_create(size_t size,
+							  size_t capacity);
+
+/**
+ * Destroy given pool and free all allocated memory, invalidating any pointers.
+ *
+ * This operation is not synchronized, calling any other operation on the pool
+ * while it is being destroyed results in Undefined Behavior.
+ *
+ * @param pool Can be null to support common destruction patterns.
+ */
+void dc_memory_pool_destroy(struct dc_memory_pool *pool);
+
+/**
+ * Acquire single block from pool.
+ *
+ * @param pool Cannot be null.
+ * @return Pointer to block if succeeded, null if empty.
+ */
+__must_check void *dc_memory_pool_acquire(struct dc_memory_pool *pool);
+
+/**
+ * Release previously acquired block, freeing it for others to acquire.
+ *
+ * @param pool Cannot be null.
+ * @param memory Cannot be null, has to be owned by the pool.
+ */
+void dc_memory_pool_release(struct dc_memory_pool *pool, void *memory);
+
+/**
+ * Check if given memory is owned by the pool to facilitate arenas.
+ *
+ * @warning if (owns(pool, p)) release(pool, p);` pattern is not safe
+ * if used with same `p` from multiple unsynchronized threads.
+ * If multiple owning threads are desired, use krefs to assure single release.
+ *
+ * @param pool Cannot be null.
+ * @param memory Cannot be null.
+ * @return True if the memory was previously acquired from the pool.
+ */
+__must_check bool dc_memory_pool_owns(const struct dc_memory_pool *pool,
+				      const void *memory);
+
+/**
+ * Get pool block size.
+ *
+ * @param pool Cannot be null.
+ * @return Pool block size as given to dc_memory_pool_create().
+ */
+__must_check size_t dc_memory_pool_size(const struct dc_memory_pool *pool);
+
+/**
+ * Get pool block count.
+ *
+ * @param pool Cannot be null.
+ * @return Pool block count as given to dc_memory_pool_create().
+ */
+__must_check size_t dc_memory_pool_capacity(const struct dc_memory_pool *pool);
+
+#endif // Header guard
diff --git a/drivers/gpu/drm/amd/display/dc/dc_probe.h b/drivers/gpu/drm/amd/display/dc/dc_probe.h
index ebf33b162b6349..52bffbebf7ad71 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_probe.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_probe.h
@@ -20,6 +20,7 @@ enum dc_probe_type {
 	DC_PROBE_URGENT_RAMP_LATENCY,
 	DC_PROBE_URGENT_ASSERTION_COUNT,
 	DC_PROBE_PREFETCH_DATA_SIZE,
+	DC_PROBE_PEAK_MEM_BW_STRESSED, /* peak BW under synthetic memory contention */
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index 934ae381e73062..fec5e7a84bb958 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -123,6 +123,9 @@ union stream_update_flags {
 		uint32_t info_frame : 1;
 		uint32_t dmdata : 1;
 		uint32_t dither : 1;
+		uint32_t func_shaper : 1;
+		uint32_t lut3d_func : 1;
+		uint32_t reserved : 11; /* adjust when adding new flags */
 	} bits;
 
 	uint32_t raw;
@@ -611,17 +614,6 @@ bool dc_stream_set_gamut_remap(struct dc *dc,
 bool dc_stream_program_csc_matrix(struct dc *dc,
 				  struct dc_stream_state *stream);
 
-struct dc_rmcm_3dlut *dc_stream_get_3dlut_for_stream(
-	const struct dc *dc,
-	const struct dc_stream_state *stream,
-	bool allocate_one);
-
-void dc_stream_release_3dlut_for_stream(
-	const struct dc *dc,
-	const struct dc_stream_state *stream);
-
-void dc_stream_init_rmcm_3dlut(struct dc *dc);
-
 struct pipe_ctx *dc_stream_get_pipe_ctx(struct dc_stream_state *stream);
 
 void dc_dmub_update_dirty_rect(struct dc *dc,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index 7da673111ed911..7ff2152ff2334c 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -194,6 +194,7 @@ struct dc_panel_patch {
 	bool oled_optimize_display_on;
 	unsigned int force_mst_blocked_discovery;
 	unsigned int wait_after_dpcd_poweroff_ms;
+	unsigned int force_freesync_min_hz;
 };
 
 /**
@@ -688,7 +689,8 @@ union dmcu_psr_level {
 		unsigned int SKIP_SINGLE_OTG_DISABLE:1;
 		unsigned int DISABLE_ALPM:1;
 		unsigned int ALPM_DEFAULT_PD_MODE:1;
-		unsigned int RESERVED:20;
+		unsigned int FORCE_P2CPM:1;
+		unsigned int RESERVED:19;
 	} bits;
 	unsigned int u32all;
 };
@@ -1307,6 +1309,8 @@ struct replay_settings {
 	uint16_t frame_skip_number;
 	/* Current Panel Replay events */
 	uint32_t replay_events;
+	/* Last ABM periodic keep-alive FFU allowed bit sent to DMUB (VESA PR) */
+	bool last_abm_periodic_ffu_allowed;
 };
 
 /* To split out "global" and "per-panel" config settings.
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.c
index 13ba7f5ce13ece..71ebdd54f8f086 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.c
@@ -212,6 +212,7 @@ struct dccg *dccg2_create(
 	base->ctx = ctx;
 	base->funcs = &dccg2_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h
index cecfb0a6803097..f47733db96f78c 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn20/dcn20_dccg.h
@@ -256,6 +256,10 @@
 	type DSCCLK1_EN;\
 	type DSCCLK2_EN;\
 	type DSCCLK3_EN;\
+	type DSCCLK0_DTO_DB_EN;\
+	type DSCCLK1_DTO_DB_EN;\
+	type DSCCLK2_DTO_DB_EN;\
+	type DSCCLK3_DTO_DB_EN;\
 	type DISPCLK_DCCG_GATE_DISABLE;\
 	type DCCG_GLOBAL_FGCG_REP_DIS; \
 	type PHYASYMCLK_EN;\
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn201/dcn201_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn201/dcn201_dccg.c
index 5b9ba9a811ea5d..ee27d7519d2dec 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn201/dcn201_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn201/dcn201_dccg.c
@@ -82,6 +82,7 @@ struct dccg *dccg201_create(
 	base->ctx = ctx;
 	base->funcs = &dccg201_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c
index 1f23dfccf07aa6..d0735022f198fe 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn21/dcn21_dccg.c
@@ -158,6 +158,7 @@ struct dccg *dccg21_create(
 	base->ctx = ctx;
 	base->funcs = &dccg21_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn30/dcn30_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn30/dcn30_dccg.c
index 9df06df6c7f1be..005ca01d7ecc87 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn30/dcn30_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn30/dcn30_dccg.c
@@ -114,6 +114,7 @@ struct dccg *dccg3_create(
 	base->ctx = ctx;
 	base->funcs = &dccg3_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
@@ -139,6 +140,7 @@ struct dccg *dccg30_create(
 	base->ctx = ctx;
 	base->funcs = &dccg3_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn301/dcn301_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn301/dcn301_dccg.c
index 837f4e3d1a60a8..022e8d3dfce245 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn301/dcn301_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn301/dcn301_dccg.c
@@ -73,6 +73,7 @@ struct dccg *dccg301_create(
 	base->ctx = ctx;
 	base->funcs = &dccg301_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c
index ce90cba853f787..341509ad6f3557 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c
@@ -965,6 +965,7 @@ struct dccg *dccg31_create(
 	base->ctx = ctx;
 	base->funcs = &dccg31_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c
index 1a4d4cd58ad884..a71851779b3760 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn314/dcn314_dccg.c
@@ -429,6 +429,7 @@ struct dccg *dccg314_create(
 	base->ctx = ctx;
 	base->funcs = &dccg314_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn32/dcn32_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn32/dcn32_dccg.c
index f75c5aa2d23edc..52161c745a4873 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn32/dcn32_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn32/dcn32_dccg.c
@@ -441,6 +441,7 @@ struct dccg *dccg32_create(
 	base->ctx = ctx;
 	base->funcs = &dccg32_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
index 71903a6c63a0d5..ba624f4429153c 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn35/dcn35_dccg.c
@@ -1661,6 +1661,7 @@ void dccg35_set_hdmistreamclk(
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
 
+	dccg35_set_hdmistreamclk_rcg(dccg, 0, false);
 	/* set the dtbclk_p source */
 	dccg35_set_dtbclk_p_src(dccg, src, otg_inst);
 
@@ -2803,6 +2804,7 @@ struct dccg *dccg35_create(
 	base->ctx = ctx;
 	base->funcs = &dccg35_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
index 7bff1b9da6f912..a4ece209d600ea 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
@@ -999,6 +999,7 @@ struct dccg *dccg401_create(
 	base->ctx = ctx;
 	base->funcs = &dccg401_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
index c2fa7fd56acf7f..bdf2bd388be762 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
@@ -308,13 +308,25 @@ static void dccg42_init(struct dccg *dccg)
 		}
 	}
 }
+void dccg42_set_hdmistreamclk_root_clock_gating(struct dccg *dccg, bool enable)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+
+	if (!dccg->ctx->dc->debug.root_clock_optimization.bits.hdmistream && !enable) {
+		DC_LOG_DEBUG("%s: HDMISTREAMCLK0_ROOT_GATE DISABLE = 0 bypassed", __func__);
+		return;
+	}
+	REG_UPDATE(DCCG_GATE_DISABLE_CNTL6, HDMISTREAMCLK0_ROOT_GATE_DISABLE, enable ? 1 : 0);
+	REG_UPDATE(DCCG_GATE_DISABLE_CNTL3, HDMISTREAMCLK0_GATE_DISABLE, enable ? 1 : 0);
 
+	DC_LOG_DEBUG("%s: HDMISTREAMCLK0_ROOT_GATE_DISABLE = %d\n", __func__, enable ? 1 : 0);
+}
 
 static const struct dccg_funcs dccg42_funcs = {
 	.enable_hdmicharclk = dccg401_enable_hdmicharclk,
 	.disable_hdmicharclk = dccg42_disable_hdmicharclk,
 	.set_hdmistreamclk = dccg35_set_hdmistreamclk,
-	.set_hdmistreamclk_root_clock_gating = dccg35_set_hdmistreamclk_root_clock_gating,
+	.set_hdmistreamclk_root_clock_gating = dccg42_set_hdmistreamclk_root_clock_gating,
 	.update_dpp_dto = dccg35_update_dpp_dto,
 	.dpp_root_clock_control = dccg35_dpp_root_clock_control,
 	.get_dccg_ref_freq = dccg401_get_dccg_ref_freq,
@@ -370,6 +382,7 @@ struct dccg *dccg42_create(
 	base->ctx = ctx;
 	base->funcs = &dccg42_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
index ebd3cec1a97795..198dc7a6c71ebc 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.h
@@ -262,7 +262,7 @@ void dccg42_set_pixel_rate_div(
 		enum pixel_rate_div unused);
 
 void dccg42_trigger_dio_fifo_resync(struct dccg *dccg);
-
+void dccg42_set_hdmistreamclk_root_clock_gating(struct dccg *dccg, bool enable);
 struct dccg *dccg42_create(
 	struct dc_context *ctx,
 	const struct dccg_registers *regs,
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.c
index 1d6d193b23c71b..9f15e6adc715d7 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.c
@@ -76,6 +76,7 @@ static void dccg60_set_dto_dscclk(struct dccg *dccg, uint32_t inst,
 
 	switch (inst) {
 	case 0:
+		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK0_DTO_DB_EN, 1);
 		REG_UPDATE_2(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_PHASE, 1,
 				DSCCLK0_DTO_MODULO, 1);
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK0_EN, 1);
@@ -92,6 +93,7 @@ static void dccg60_set_dto_dscclk(struct dccg *dccg, uint32_t inst,
 
 		break;
 	case 1:
+		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK1_DTO_DB_EN, 1);
 		REG_UPDATE_2(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_PHASE, 1,
 				DSCCLK1_DTO_MODULO, 1);
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK1_EN, 1);
@@ -103,6 +105,7 @@ static void dccg60_set_dto_dscclk(struct dccg *dccg, uint32_t inst,
 
 		break;
 	case 2:
+		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK2_DTO_DB_EN, 1);
 		REG_UPDATE_2(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_PHASE, 1,
 				DSCCLK2_DTO_MODULO, 1);
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK2_EN, 1);
@@ -114,6 +117,7 @@ static void dccg60_set_dto_dscclk(struct dccg *dccg, uint32_t inst,
 
 		break;
 	case 3:
+		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK3_DTO_DB_EN, 1);
 		REG_UPDATE_2(DSCCLK3_DTO_PARAM, DSCCLK3_DTO_PHASE, 1,
 				DSCCLK3_DTO_MODULO, 1);
 		REG_UPDATE(DSCCLK_DTO_CTRL, DSCCLK3_EN, 1);
@@ -179,6 +183,7 @@ struct dccg *dccg60_create(
 	base->ctx = ctx;
 	base->funcs = &dccg60_funcs;
 
+	base->inst = 0;
 	dccg_dcn->regs = regs;
 	dccg_dcn->dccg_shift = dccg_shift;
 	dccg_dcn->dccg_mask = dccg_mask;
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.h b/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.h
index 3f068ccfcdd54b..3a6b65dee154cc 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn60/dcn60_dccg.h
@@ -96,6 +96,10 @@
 	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK1_EN, mask_sh),\
 	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK2_EN, mask_sh),\
 	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK3_EN, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK0_DTO_DB_EN, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK1_DTO_DB_EN, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK2_DTO_DB_EN, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK3_DTO_DB_EN, mask_sh),\
 	DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_PHASE, mask_sh),\
 	DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_MODULO, mask_sh),\
 	DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_PHASE, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
index 469b4b8f88a30e..bd9e0173ff9d40 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
@@ -220,6 +220,7 @@ static bool dce_abm_set_level(struct abm *abm, uint32_t level)
 
 static bool dce_abm_immediate_disable(struct abm *abm, uint32_t panel_inst)
 {
+
 	if (abm->dmcu_is_running == false)
 		return true;
 
@@ -272,6 +273,7 @@ static void dce_abm_construct(
 	base->ctx = ctx;
 	base->funcs = &dce_funcs;
 	base->dmcu_is_running = false;
+	base->inst = 0;
 
 	abm_dce->regs = regs;
 	abm_dce->abm_shift = abm_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
index 6cb5e8152cf10f..4abf51c8ab52c9 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -432,14 +432,24 @@ static bool acquire_aux_engine_without_ddc_pin(
 	struct dce_aux *engine,
 	struct ddc *ddc)
 {
-	(void)ddc;
 	if ((engine == NULL) || !is_engine_available(engine))
 		return false;
 
+	/* Open the DDC in AUX mode so the I3C PAD set_config hook runs and
+	 * enables the PAD RX (disabled by default); otherwise DPCD reads time
+	 * out even though the connector has no native I2C/DDC pin. */
+	if (ddc != NULL &&
+		dal_ddc_open(ddc, GPIO_MODE_HARDWARE,
+			GPIO_DDC_CONFIG_TYPE_MODE_AUX) != GPIO_RESULT_OK)
+		return false;
+
 	if (!acquire_engine(engine)) {
+		engine->ddc = ddc;
 		release_engine(engine);
 		return false;
 	}
+
+	engine->ddc = ddc;
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index 20ccecf23677d2..145c993342bdb9 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2012-15 Advanced Micro Devices, Inc.
  *
@@ -1073,7 +1074,6 @@ static bool dcn401_program_pix_clk(
 		enum dp_link_encoding encoding,
 		struct pll_settings *pll_settings)
 {
-	(void)encoding;
 	struct dce110_clk_src *clk_src = TO_DCE110_CLK_SRC(clock_source);
 	unsigned int inst = pix_clk_params->controller_id - CONTROLLER_ID_D0;
 	const struct pixel_rate_range_table_entry *e =
@@ -1081,6 +1081,7 @@ static bool dcn401_program_pix_clk(
 	struct bp_pixel_clock_parameters bp_pc_params = {0};
 	enum transmitter_color_depth bp_pc_colour_depth = TRANSMITTER_COLOR_DEPTH_24;
 	struct dp_dto_params dto_params = { 0 };
+	(void)encoding;
 
 	dto_params.otg_inst = inst;
 	dto_params.signal = pix_clk_params->signal_type;
@@ -1232,9 +1233,9 @@ static bool get_dp_dto_frequency_100hz(
 			 */
 			modulo_hz = REG_READ(MODULO[inst]);
 			if (modulo_hz) {
-				temp = clock_hz * dp_dto_ref_khz * 10;
+				temp = (unsigned long long)clock_hz * dp_dto_ref_khz * 10;
 				ASSERT(temp <= UINT_MAX * modulo_hz * 100ULL);
-				*pixel_clk_100hz = div_u64(temp, modulo_hz * 100);
+				*pixel_clk_100hz = (unsigned int)div_u64(temp, modulo_hz * 100);
 			} else
 				*pixel_clk_100hz = 0;
 		} else {
@@ -1293,7 +1294,7 @@ static bool dcn401_get_dp_dto_frequency_100hz(const struct clock_source *clock_s
 			BREAK_TO_DEBUGGER();
 			*pixel_clk_100hz = 0;
 		} else {
-			*pixel_clk_100hz = div_u64(temp, 100);
+			*pixel_clk_100hz = (unsigned int)div_u64(temp, 100);
 		}
 
 		return true;
@@ -1756,6 +1757,7 @@ bool dce110_clk_src_construct(
 	clk_src->base.ctx = ctx;
 	clk_src->bios = bios;
 	clk_src->base.id = id;
+	clk_src->base.inst = (unsigned int)id;
 	clk_src->base.funcs = &dce110_clk_src_funcs;
 
 	clk_src->regs = regs;
@@ -1854,6 +1856,7 @@ bool dce112_clk_src_construct(
 	clk_src->base.ctx = ctx;
 	clk_src->bios = bios;
 	clk_src->base.id = id;
+	clk_src->base.inst = (unsigned int)id;
 	clk_src->base.funcs = &dce112_clk_src_funcs;
 
 	clk_src->regs = regs;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
index 25ebd8a52ae419..23de1c793ec399 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
@@ -56,9 +56,6 @@
 #define MCP_BL_SET_PWM_FRAC 0x6A  /* Enable or disable Fractional PWM */
 #define CRC_WIN_NOTIFY 0x92
 #define CRC_STOP_UPDATE 0x93
-#define MCP_SEND_EDID_CEA 0xA0
-#define EDID_CEA_CMD_ACK 1
-#define EDID_CEA_CMD_NACK 2
 #define MASTER_COMM_CNTL_REG__MASTER_COMM_INTERRUPT_MASK   0x00000001L
 
 // PSP FW version
@@ -812,121 +809,6 @@ static bool dcn20_unlock_phy(struct dmcu *dmcu)
 	return true;
 }
 
-static bool dcn10_send_edid_cea(struct dmcu *dmcu,
-		int offset,
-		int total_length,
-		uint8_t *data,
-		int length)
-{
-	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
-	uint32_t header, data1, data2;
-
-	/* If microcontroller is not running, do nothing */
-	if (dmcu->dmcu_state != DMCU_RUNNING)
-		return false;
-
-	if (length > 8 || length <= 0)
-		return false;
-
-	header = ((uint32_t)offset & 0xFFFF) << 16 | (total_length & 0xFFFF);
-	data1 = (((uint32_t)data[0]) << 24) | (((uint32_t)data[1]) << 16) |
-		(((uint32_t)data[2]) << 8) | ((uint32_t)data[3]);
-	data2 = (((uint32_t)data[4]) << 24) | (((uint32_t)data[5]) << 16) |
-		(((uint32_t)data[6]) << 8) | ((uint32_t)data[7]);
-
-	/* waitDMCUReadyForCmd */
-	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
-
-	/* setDMCUParam_Cmd */
-	REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0, MCP_SEND_EDID_CEA);
-
-	REG_WRITE(MASTER_COMM_DATA_REG1, header);
-	REG_WRITE(MASTER_COMM_DATA_REG2, data1);
-	REG_WRITE(MASTER_COMM_DATA_REG3, data2);
-
-	/* notifyDMCUMsg */
-	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
-
-	/* waitDMCUReadyForCmd */
-	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
-
-	return true;
-}
-
-static bool dcn10_get_scp_results(struct dmcu *dmcu,
-		uint32_t *cmd,
-		uint32_t *data1,
-		uint32_t *data2,
-		uint32_t *data3)
-{
-	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
-
-	/* If microcontroller is not running, do nothing */
-	if (dmcu->dmcu_state != DMCU_RUNNING)
-		return false;
-
-	*cmd = REG_READ(SLAVE_COMM_CMD_REG);
-	*data1 =  REG_READ(SLAVE_COMM_DATA_REG1);
-	*data2 =  REG_READ(SLAVE_COMM_DATA_REG2);
-	*data3 =  REG_READ(SLAVE_COMM_DATA_REG3);
-
-	/* clear SCP interrupt */
-	REG_UPDATE(SLAVE_COMM_CNTL_REG, SLAVE_COMM_INTERRUPT, 0);
-
-	return true;
-}
-
-static bool dcn10_recv_amd_vsdb(struct dmcu *dmcu,
-		int *version,
-		int *min_frame_rate,
-		int *max_frame_rate)
-{
-	uint32_t data[4];
-	int cmd, ack, len;
-
-	if (!dcn10_get_scp_results(dmcu, &data[0], &data[1], &data[2], &data[3]))
-		return false;
-
-	cmd = data[0] & 0x3FF;
-	len = (data[0] >> 10) & 0x3F;
-	ack = data[1];
-
-	if (cmd != MCP_SEND_EDID_CEA || ack != EDID_CEA_CMD_ACK || len != 12)
-		return false;
-
-	if ((data[2] & 0xFF)) {
-		*version = (data[2] >> 8) & 0xFF;
-		*min_frame_rate = (data[3] >> 16) & 0xFFFF;
-		*max_frame_rate = data[3] & 0xFFFF;
-		return true;
-	}
-
-	return false;
-}
-
-static bool dcn10_recv_edid_cea_ack(struct dmcu *dmcu, int *offset)
-{
-	uint32_t data[4];
-	int cmd, ack;
-
-	if (!dcn10_get_scp_results(dmcu,
-				&data[0], &data[1], &data[2], &data[3]))
-		return false;
-
-	cmd = data[0] & 0x3FF;
-	ack = data[1];
-
-	if (cmd != MCP_SEND_EDID_CEA)
-		return false;
-
-	if (ack == EDID_CEA_CMD_ACK)
-		return true;
-
-	*offset = data[2]; /* nack */
-	return false;
-}
-
-
 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
 static void dcn10_forward_crc_window(struct dmcu *dmcu,
 					struct rect *rect,
@@ -1030,9 +912,6 @@ static const struct dmcu_funcs dcn10_funcs = {
 	.get_psr_state = dcn10_get_dmcu_psr_state,
 	.set_psr_wait_loop = dcn10_psr_wait_loop,
 	.get_psr_wait_loop = dcn10_get_psr_wait_loop,
-	.send_edid_cea = dcn10_send_edid_cea,
-	.recv_amd_vsdb = dcn10_recv_amd_vsdb,
-	.recv_edid_cea_ack = dcn10_recv_edid_cea_ack,
 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
 	.forward_crc_window = dcn10_forward_crc_window,
 	.stop_crc_win_update = dcn10_stop_crc_win_update,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
index 2df6e82b794fa6..8cca8599b9171f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c
@@ -280,6 +280,7 @@ static void dmub_abm_construct(
 	base->ctx = ctx;
 	base->funcs = &abm_funcs;
 	base->dmcu_is_running = false;
+	base->inst = 0;
 
 	abm_dce->regs = regs;
 	abm_dce->abm_shift = abm_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c
index 52673e2f504c01..2ac1485a34381c 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c
@@ -36,6 +36,14 @@ static bool dmub_hw_lock_has_inbox0_lock(const struct dc *dc)
 	       dc->ctx->dmub_srv->dmub->meta_info.feature_bits.bits.inbox0_lock_support;
 }
 
+static bool dmub_hw_lock_has_inbox0_lock_split(const struct dc *dc)
+{
+	return dc->ctx && dc->ctx->dmub_srv &&
+		dc->hwss.dmub_hw_control_lock &&
+		dc->hwss.dmub_hw_control_lock_fast &&
+		dc->ctx->dmub_srv->dmub->meta_info.feature_bits.bits.inbox0_lock_split;
+}
+
 void dmub_hw_lock_mgr_cmd(struct dc_dmub_srv *dmub_srv,
 				bool lock,
 				union dmub_hw_lock_flags *hw_locks,
@@ -121,5 +129,5 @@ bool should_use_dmub_inbox1_lock(const struct dc *dc, const struct dc_link *link
 
 bool should_use_dmub_inbox0_lock_for_link(const struct dc *dc, const struct dc_link *link)
 {
-	return dmub_hw_lock_has_inbox0_lock(dc) && dmub_hw_lock_mgr_does_link_require_lock(dc, link);
+	return dmub_hw_lock_has_inbox0_lock_split(dc) && dmub_hw_lock_mgr_does_link_require_lock(dc, link);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
index 45630c3effe1a7..66d8ba73d38b90 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
@@ -481,6 +481,7 @@ static void dmub_psr_construct(struct dmub_psr *psr, struct dc_context *ctx)
 {
 	psr->ctx = ctx;
 	psr->funcs = &psr_funcs;
+	psr->inst = 0;
 }
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h
index a6e282d950c356..45ba9902ebbb51 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.h
@@ -34,6 +34,7 @@ struct dmub_psr_funcs;
 struct dmub_psr {
 	struct dc_context *ctx;
 	const struct dmub_psr_funcs *funcs;
+	uint32_t inst;
 };
 
 struct dmub_psr_funcs {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
index eda3a2f1402c60..e6adca240ccf24 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
@@ -217,13 +217,13 @@ static void dmub_replay_set_coasting_vtotal(struct dmub_replay *dmub,
 		uint8_t panel_inst,
 		uint16_t frame_skip_number)
 {
-	(void)panel_inst;
 	union dmub_rb_cmd cmd;
 	struct dc_context *dc = dmub->ctx;
 	struct dmub_rb_cmd_replay_set_coasting_vtotal *pCmd = NULL;
 
 	pCmd = &(cmd.replay_set_coasting_vtotal);
 
+	(void)panel_inst;
 	memset(&cmd, 0, sizeof(cmd));
 	pCmd->header.type = DMUB_CMD__REPLAY;
 	pCmd->header.sub_type = DMUB_CMD__REPLAY_SET_COASTING_VTOTAL;
@@ -415,6 +415,30 @@ static void dmub_replay_send_cmd(struct dmub_replay *dmub,
 	dc_wake_and_execute_dmub_cmd(ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 }
 
+static bool dmub_replay_get_cumulative_residency(struct dmub_replay *dmub,
+	uint8_t panel_inst, uint32_t *residency_milli_pct)
+{
+	uint32_t milli_pct = 0;
+	uint16_t param = (uint16_t)panel_inst;
+
+	if (!residency_milli_pct)
+		return false;
+
+	/* Zero on failure so the caller never reads a stale value, and report
+	 * false so a failed query is still distinguishable from a genuine 0%.
+	 */
+	*residency_milli_pct = 0;
+
+	if (!dc_wake_and_execute_gpint(dmub->ctx,
+			DMUB_GPINT__REPLAY_SNAPSHOT_CUMULATIVE_RESIDENCY,
+			param, &milli_pct, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY))
+		return false;
+
+	*residency_milli_pct = milli_pct;
+
+	return true;
+}
+
 static const struct dmub_replay_funcs replay_funcs = {
 	.replay_copy_settings				= dmub_replay_copy_settings,
 	.replay_enable					= dmub_replay_enable,
@@ -424,6 +448,7 @@ static const struct dmub_replay_funcs replay_funcs = {
 	.replay_residency				= dmub_replay_residency,
 	.replay_set_power_opt_and_coasting_vtotal	= dmub_replay_set_power_opt_and_coasting_vtotal,
 	.replay_send_cmd				= dmub_replay_send_cmd,
+	.replay_get_cumulative_residency		= dmub_replay_get_cumulative_residency,
 };
 
 /*
@@ -433,6 +458,7 @@ static void dmub_replay_construct(struct dmub_replay *replay, struct dc_context
 {
 	replay->ctx = ctx;
 	replay->funcs = &replay_funcs;
+	replay->inst = 0;
 }
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.h b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.h
index 07c79739a98092..fa17cd395e5cd3 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.h
@@ -13,6 +13,7 @@ struct dmub_replay_funcs;
 struct dmub_replay {
 	struct dc_context *ctx;
 	const struct dmub_replay_funcs *funcs;
+	uint32_t inst;
 };
 
 struct dmub_replay_funcs {
@@ -33,6 +34,8 @@ struct dmub_replay_funcs {
 	void (*replay_set_power_opt_and_coasting_vtotal)(struct dmub_replay *dmub,
 		unsigned int power_opt, uint8_t panel_inst, uint32_t coasting_vtotal,
 		uint16_t frame_skip_number);
+	bool (*replay_get_cumulative_residency)(struct dmub_replay *dmub,
+		uint8_t panel_inst, uint32_t *residency_milli_pct);
 };
 
 struct dmub_replay *dmub_replay_create(struct dc_context *ctx);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c
index 5bc3bc60a2ac04..ebbe7900205e14 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_vmid.c
@@ -46,7 +46,7 @@ static void dcn20_wait_for_vmid_ready(struct dcn20_vmid *vmid)
 	/* According the hardware spec, we need to poll for the lowest
 	 * bit of PAGE_TABLE_BASE_ADDR_LO32 = 1 any time a GPUVM
 	 * context is updated. We can't use REG_WAIT here since we
-	 * don't have a seperate field to wait on.
+	 * don't have a separate field to wait on.
 	 *
 	 * TODO: Confirm timeout / poll interval with hardware team
 	 */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
index 62ca235cd649d1..eb299f7f9c41af 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
@@ -356,20 +356,20 @@ bool cm3_helper_translate_curve_to_degamma_hw_format(
 	region_end = 0;
 
 	for (i = region_end - region_start; i < MAX_REGIONS_NUMBER ; i++)
-		seg_distr[i] = -1;
+		seg_distr[i] = (uint32_t)-1;
 
 	for (k = 0; k < MAX_REGIONS_NUMBER; k++) {
-		if (seg_distr[k] != -1)
+		if (seg_distr[k] != (uint32_t)-1)
 			hw_points += (1 << seg_distr[k]);
 	}
 
 	j = 0;
-	for (k = 0; k < (region_end - region_start); k++) {
+	for (k = 0; k < (uint32_t)(region_end - region_start); k++) {
 		increment = (NUMBER_SW_SEGMENTS << 4) / (1 << seg_distr[k]);
 		start_index = (region_start + k + MAX_LOW_POINT) *
 				NUMBER_SW_SEGMENTS;
 		for (i = (start_index << 4);
-		     i < (start_index << 4) + (NUMBER_SW_SEGMENTS << 4);
+		     i < (int32_t)(start_index << 4) + (NUMBER_SW_SEGMENTS << 4);
 		     i += increment) {
 			if (j == hw_points - 1)
 				break;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_panel_cntl.c
index 9e96a3ace2077c..1e11328050e5e6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_panel_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_panel_cntl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2020 Advanced Micro Devices, Inc.
  *
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c
index f9961a6446f3ef..d7c3f26777e3e8 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_panel_cntl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2021 Advanced Micro Devices, Inc.
  *
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.c b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.c
index edcf4e67483bb9..458ce1053c967a 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.c
@@ -40,6 +40,7 @@ void dcn10_dio_construct(
 {
 	dio10->base.ctx = ctx;
 	dio10->base.funcs = &dcn10_dio_funcs;
+	dio10->base.inst = 0;
 
 	dio10->regs = regs;
 	dio10->shifts = shifts;
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
index 00e82d627d9183..32e12d894e31f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
@@ -781,6 +781,8 @@ void dcn10_link_encoder_construct(
 	if (enc10->base.ctx->dc->debug.hdmi20_disable) {
 		enc10->base.features.flags.bits.HDMI_6GB_EN = 0;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
 
 bool dcn10_link_encoder_validate_output_with_stream(
@@ -1166,6 +1168,10 @@ void dcn10_link_encoder_dp_set_phy_pattern(
 {
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
 
+	if (!param) {
+		ASSERT_CRITICAL(false);
+		return;
+	}
 	switch (param->dp_phy_pattern) {
 	case DP_TEST_PATTERN_TRAINING_PATTERN1:
 		dcn10_link_encoder_set_dp_phy_pattern_training_pattern(enc, 0);
@@ -1189,8 +1195,7 @@ void dcn10_link_encoder_dp_set_phy_pattern(
 		set_dp_phy_pattern_prbs7(enc10);
 		break;
 	case DP_TEST_PATTERN_80BIT_CUSTOM:
-		if (param)
-			set_dp_phy_pattern_80bit_custom(
+		set_dp_phy_pattern_80bit_custom(
 				enc10, param->custom_pattern);
 		break;
 	case DP_TEST_PATTERN_CP2520_1:
@@ -1203,8 +1208,7 @@ void dcn10_link_encoder_dp_set_phy_pattern(
 		set_dp_phy_pattern_hbr2_compliance_cp2520_2(enc10, 3);
 		break;
 	case DP_TEST_PATTERN_VIDEO_MODE: {
-		if (param)
-			set_dp_phy_pattern_passthrough_mode(
+		set_dp_phy_pattern_passthrough_mode(
 				enc10, param->dp_panel_mode);
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.c
index d913f065ecca1b..c937788d0763b0 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_stream_encoder.c
@@ -1625,5 +1625,7 @@ void dcn10_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = eng_id - ENGINE_ID_DIGA;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c
index 3bd35f3392dcc4..dbd4c534c5c3bf 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_link_encoder.c
@@ -502,4 +502,6 @@ void dcn20_link_encoder_construct(
 	if (enc10->base.ctx->dc->debug.hdmi20_disable) {
 		enc10->base.features.flags.bits.HDMI_6GB_EN = 0;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_stream_encoder.c
index bec0b4aaeb2b98..ac6ca7771d2f89 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn20/dcn20_stream_encoder.c
@@ -657,5 +657,7 @@ void dcn20_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = eng_id - ENGINE_ID_DIGA;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c
index 8c4010c5b2e7c0..5fd5158482ea04 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_link_encoder.c
@@ -456,6 +456,8 @@ void dcn30_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_10G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
 
 #define AUX_REG(reg)\
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c
index c14fb51c261165..9757d3bcdf320c 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c
@@ -899,5 +899,7 @@ void dcn30_dio_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = vpg->inst;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c
index 02fb844670f9d1..7106669eab5b15 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn301/dcn301_dio_link_encoder.c
@@ -209,4 +209,6 @@ void dcn301_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_10G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
index f57f3ba68a02e5..1b9ba0e9f068bf 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn31/dcn31_dio_link_encoder.c
@@ -509,6 +509,8 @@ void dcn31_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_10G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
 
 void dcn31_link_encoder_construct_minimal(
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn314/dcn314_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn314/dcn314_dio_stream_encoder.c
index d1fd5462dca5b6..482c52c1d7d6b1 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn314/dcn314_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn314/dcn314_dio_stream_encoder.c
@@ -504,5 +504,7 @@ void dcn314_dio_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = vpg->inst;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c
index 68e0dc93ae7d6c..63990c47ac14eb 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_link_encoder.c
@@ -551,4 +551,6 @@ void dcn32_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_10G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_stream_encoder.c
index edafa380845578..3dd8fe423232a5 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn32/dcn32_dio_stream_encoder.c
@@ -491,5 +491,7 @@ void dcn32_dio_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = vpg->inst;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c
index 4108fd0f43802f..67d6050d51633a 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn321/dcn321_dio_link_encoder.c
@@ -210,4 +210,6 @@ void dcn321_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_10G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c
index 8fa9b8c500f203..9375c9b939bedf 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_link_encoder.c
@@ -290,6 +290,8 @@ void dcn35_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
 
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
 
 /* DPIA equivalent of link_transmitter_control. */
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_stream_encoder.c
index de24dcd27e6cea..dabcb45840a7e0 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn35/dcn35_dio_stream_encoder.c
@@ -517,5 +517,7 @@ void dcn35_dio_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = vpg->inst;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c
index dad4ef565168ec..6594fbb3fa43e0 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_link_encoder.c
@@ -557,4 +557,6 @@ void dcn401_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_10G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c
index 79d0c2896d2954..4616b6f76aeb0f 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c
@@ -793,6 +793,7 @@ void dcn401_dio_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = vpg->inst;
+	enc1->base.inst = enc1->base.stream_enc_inst;
 }
 
 void enc401_set_dynamic_metadata(struct stream_encoder *enc,
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c
index fae49b3b170e79..3b11be9991c3f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.c
@@ -58,8 +58,8 @@ bool dcn42_program_hpd_filter(struct link_encoder *enc, int delay_on_connect_in_
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
 
 	HPD_REG_SET_2(DC_HPD_TOGGLE_FILT_CNTL, 0,
-			DC_HPD_CONNECT_INT_DELAY, delay_on_connect_in_ms,
-			DC_HPD_DISCONNECT_INT_DELAY, delay_on_disconnect_in_ms);
+			DC_HPD_CONNECT_INT_DELAY, delay_on_connect_in_ms / 10,
+			DC_HPD_DISCONNECT_INT_DELAY, delay_on_disconnect_in_ms / 10);
 
 	return true;
 }
@@ -223,4 +223,6 @@ void dcn42_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_10G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_12G_CAPABLE = 1;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
index 35845e6dd5aa47..b7671d41ea243d 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
@@ -581,4 +581,6 @@ void dcn42_dio_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = vpg->inst;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_link_encoder.c
index 6fe15324ea50ed..f65326a0f6d240 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_link_encoder.c
@@ -370,4 +370,6 @@ void dcn60_link_encoder_construct(
 		enc10->base.features.flags.bits.IS_FRL_20G_CAPABLE = 1;
 		enc10->base.features.flags.bits.IS_FRL_24G_CAPABLE = 0;
 	}
+
+	enc10->base.inst = (unsigned int)enc10->base.preferred_engine;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_stream_encoder.c
index b76cc891857d9f..b645914c1c7188 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn60/dcn60_dio_stream_encoder.c
@@ -535,4 +535,6 @@ void dcn60_dio_stream_encoder_construct(
 	enc1->se_shift = se_shift;
 	enc1->se_mask = se_mask;
 	enc1->base.stream_enc_inst = vpg->inst;
+	enc1->base.inst = enc1->base.stream_enc_inst;
+
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_link_encoder.c
index 5b2bba0eff0ec0..f847fef069c98c 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_link_encoder.c
@@ -27,6 +27,7 @@
 #include "dm_services_types.h"
 
 #include "virtual_link_encoder.h"
+#include "stream_encoder.h"
 
 static bool virtual_link_encoder_validate_output_with_stream(
 	struct link_encoder *enc,
@@ -167,5 +168,7 @@ bool virtual_link_encoder_construct(
 
 	enc->preferred_engine = ENGINE_ID_VIRTUAL;
 
+	enc->inst = 0;
+
 	return true;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_stream_encoder.c
index 27448f2b24675a..f6b1e2a42baacb 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/virtual/virtual_stream_encoder.c
@@ -190,6 +190,7 @@ bool virtual_stream_encoder_construct(
 	struct dc_context *ctx,
 	struct dc_bios *bp)
 {
+	enc->inst = 0;
 	if (!enc)
 		return false;
 	if (!bp)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile
index 10d4ace04d4f19..91465ac05c9717 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile
@@ -29,14 +29,14 @@ dml_ccflags := $(CC_FLAGS_FPU)
 dml_rcflags := $(CC_FLAGS_NO_FPU)
 
 ifneq ($(CONFIG_FRAME_WARN),0)
-    ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y)
+    ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)$(CONFIG_UBSAN)),y)
         ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_COMPILE_TEST),yy)
             frame_warn_limit := 4096
         else
             frame_warn_limit := 3072
         endif
     else
-        frame_warn_limit := 2048
+	frame_warn_limit := 3072
     endif
 
     ifeq ($(call test-lt, $(CONFIG_FRAME_WARN), $(frame_warn_limit)),y)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c
index 5679b79d6f5352..ae6b2d47f3795d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c
@@ -28,7 +28,6 @@
 
 #include "dcn10_fpu.h"
 #include "resource.h"
-#include "amdgpu_dm/dc_fpu.h"
 
 /**
  * DOC: DCN10 FPU manipulation Overview
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
index 6f5a11d3789920..b8e5985f930779 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
@@ -774,12 +774,19 @@ void hpo_fpu_enc3_validate_hdmi_frl_output_link(struct hpo_frl_stream_encoder *e
 		frl_params->r_bit_nominal = 8.0e9;
 		break;
 	case HDMI_FRL_LINK_RATE_10GBPS:
-	default:
 		frl_params->r_bit_nominal = 10.0e9;
 		break;
 	case HDMI_FRL_LINK_RATE_12GBPS:
 		frl_params->r_bit_nominal = 12.0e9;
 		break;
+	case HDMI_FRL_LINK_RATE_16GBPS:
+		frl_params->r_bit_nominal = 16.0e9;
+		break;
+	case HDMI_FRL_LINK_RATE_20GBPS:
+		frl_params->r_bit_nominal = 20.0e9;
+		break;
+	default:
+		break;
 	}
 
 	if (timing->flags.DSC &&
@@ -802,12 +809,19 @@ void hpo_fpu_enc3_validate_hdmi_frl_output_link(struct hpo_frl_stream_encoder *e
 			frl_params->r_bit_nominal = 8.0e9;
 			break;
 		case HDMI_FRL_LINK_RATE_10GBPS:
-		default:
 			frl_params->r_bit_nominal = 10.0e9;
 			break;
 		case HDMI_FRL_LINK_RATE_12GBPS:
 			frl_params->r_bit_nominal = 12.0e9;
 			break;
+		case HDMI_FRL_LINK_RATE_16GBPS:
+			frl_params->r_bit_nominal = 16.0e9;
+			break;
+		case HDMI_FRL_LINK_RATE_20GBPS:
+			frl_params->r_bit_nominal = 20.0e9;
+			break;
+		default:
+			break;
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
index 33b5136f7ce39c..75bdad22bf6925 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
@@ -1805,7 +1805,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 	v->DISPCLKWithRamping = 0;
 	v->DISPCLKWithoutRamping = 0;
 	v->GlobalDPPCLK = 0.0;
-	/* DAL custom code: need to update ReturnBW in case min dcfclk is overriden */
+	/* DAL custom code: need to update ReturnBW in case min dcfclk is overridden */
 	v->IdealSDPPortBandwidthPerState[v->VoltageLevel][v->maxMpcComb] = dml_min3(
 			v->ReturnBusWidth * v->DCFCLK,
 			v->DRAMSpeedPerState[v->VoltageLevel] * v->NumberOfChannels * v->DRAMChannelWidth,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
index 8064c4b3fd259f..3f62ea49378738 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
@@ -2045,7 +2045,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 	v->DISPCLKWithRamping = 0;
 	v->DISPCLKWithoutRamping = 0;
 	v->GlobalDPPCLK = 0.0;
-	/* DAL custom code: need to update ReturnBW in case min dcfclk is overriden */
+	/* DAL custom code: need to update ReturnBW in case min dcfclk is overridden */
 	{
 	double IdealFabricAndSDPPortBandwidthPerState = dml_min(
 			v->ReturnBusWidth * v->DCFCLKState[v->VoltageLevel][v->maxMpcComb],
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml1_frl_cap_chk.c b/drivers/gpu/drm/amd/display/dc/dml/dml1_frl_cap_chk.c
index da2c92e5c02f52..bfbc64163d0158 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dml1_frl_cap_chk.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dml1_frl_cap_chk.c
@@ -680,12 +680,18 @@ static double calculate_compressed_active_time(uint32_t h_active,
 		r_bit_nominal = 8.0e9;
 		break;
 	case FRL_LINK_RATE_10GBPS:
-	default:
 		r_bit_nominal = 10.0e9;
 		break;
 	case FRL_LINK_RATE_12GBPS:
 		r_bit_nominal = 12.0e9;
 		break;
+	case FRL_LINK_RATE_16GBPS:
+		r_bit_nominal = 16.0e9;
+		break;
+	case FRL_LINK_RATE_20GBPS:
+	default:
+		r_bit_nominal = 20.0e9;
+		break;
 	}
 
 	f_tb_average = pix_clk / (h_active + h_blank)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
index b8fc27a5361e85..71eedb3301e49a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
@@ -26,7 +26,6 @@
 #include "rc_calc_fpu.h"
 
 #include "qp_tables.h"
-#include "amdgpu_dm/dc_fpu.h"
 
 #define table_hash(mode, bpc, max_min) ((mode << 16) | (bpc << 8) | max_min)
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
index 5388bf094fbcd8..e8bd061be05c10 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile
@@ -28,14 +28,14 @@ dml2_ccflags := $(CC_FLAGS_FPU)
 dml2_rcflags := $(CC_FLAGS_NO_FPU)
 
 ifneq ($(CONFIG_FRAME_WARN),0)
-    ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)),y)
+    ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)$(CONFIG_UBSAN)),y)
         ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_COMPILE_TEST),yy)
             frame_warn_limit := 4096
         else
             frame_warn_limit := 3072
         endif
     else
-        frame_warn_limit := 2056
+        frame_warn_limit := 3072
     endif
 
     ifeq ($(call test-lt, $(CONFIG_FRAME_WARN), $(frame_warn_limit)),y)
@@ -62,7 +62,8 @@ subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dml2_0/dml21/src/dml2_cga/
 THIS_MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 DML2_ABS_PATH := $(patsubst %/,%,$(dir $(THIS_MAKEFILE_PATH)))
 
-DML2_C_FILES := $(shell find $(DML2_ABS_PATH) -name '*.c')
+rwildcard = $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$(2)) $(filter $(subst *,%,$(2)),$d))
+DML2_C_FILES := $(call rwildcard,$(DML2_ABS_PATH)/,*.c)
 
 # FPU flags step 2: Convert to .o and make paths relative to $(AMDDALPATH)/dc/dml2_0/
 DML2_RELATIVE_O_FILES := $(patsubst $(DML2_ABS_PATH)/%,dc/dml2_0/%,$(patsubst %.c,%.o,$(DML2_C_FILES)))
@@ -79,8 +80,7 @@ CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_pro
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_support.o := $(dml2_ccflags) $(frame_warn_flag)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.o := $(dml2_ccflags) $(frame_warn_flag)
 CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.o := $(dml2_ccflags) $(frame_warn_flag)
-CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml2_wrapper.o := $(dml2_rcflags)
-CFLAGS_$(AMDDALPATH)/dc/dml2_0/dml21/dml21_wrapper.o := $(dml2_rcflags)
+
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/display_mode_core.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_utils.o := $(dml2_rcflags)
@@ -88,11 +88,9 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_m
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_support.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.o := $(dml2_rcflags)
 CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.o := $(dml2_rcflags)
-CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml2_wrapper.o := $(dml2_ccflags)
-CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_0/dml21/dml21_wrapper.o := $(dml2_ccflags)
 
-DML2 = display_mode_core.o display_mode_util.o dml2_wrapper_fpu.o dml2_wrapper.o \
-		dml2_utils.o dml2_policy.o dml2_translation_helper.o dml2_dc_resource_mgmt.o dml2_mall_phantom.o \
+# DML2 library core files
+DML2 = display_mode_core.o display_mode_util.o \
 		dml_display_rq_dlg_calc.o
 DML2 += dml_frl_cap_chk.o
 
@@ -130,6 +128,7 @@ DML21 += src/dml2_utm_soc_bb/dml2_utm_soc_bb_dcn5.o
 DML21 += src/dml2_utm_soc_bb/dml2_utm_soc_bb_factory.o
 DML21 += src/dml2_cga/dml2_cga_dcn6.o
 DML21 += src/dml2_cga/dml2_cga_factory.o
+DML21 += src/dml2_core/dml2_core_dcn6_calcs.o
 DML21 += src/dml2_core/dml2_core_dcn6_calcs_dchub.o
 DML21 += src/dml2_core/dml2_core_dcn6_funcs_initialize.o
 DML21 += src/dml2_core/dml2_core_dcn6_funcs_mode_programming.o
@@ -140,10 +139,6 @@ DML21 += src/dml2_pmo/dml2_pmo_dcn6.o
 DML21 += src/dml2_pmo/dml2_pmo_dcn6_stage_optimizers.o
 DML21 += src/dml2_standalone_libraries/alternate_pstate_shared_lib.o
 DML21 += src/dml2_utm_soc_bb/dml2_utm_soc_bb_dcn6.o
-DML21 += dml21_translation_helper.o
-DML21 += dml21_wrapper.o
-DML21 += dml21_wrapper_fpu.o
-DML21 += dml21_utils.o
 
 AMD_DAL_DML21 = $(addprefix $(AMDDALPATH)/dc/dml2_0/dml21/,$(DML21))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h
deleted file mode 100644
index c4813c51251b19..00000000000000
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.h
+++ /dev/null
@@ -1,106 +0,0 @@
-// SPDX-License-Identifier: MIT
-//
-// Copyright 2024 Advanced Micro Devices, Inc.
-
-
-#ifndef _DML21_WRAPPER_H_
-#define _DML21_WRAPPER_H_
-
-#include "os_types.h"
-#include "dml_top_soc_parameter_types.h"
-#include "dml_top_display_cfg_types.h"
-
-struct dc;
-struct dc_state;
-struct dml2_configuration_options;
-struct dml2_context;
-enum dc_validate_mode;
-
-/**
- * dml21_create - Creates dml21_context.
- * @in_dc: dc.
- * @dml_ctx: Created dml21 context.
- * @config: dml21 configuration options.
- *
- * Create of DML21 is done as part of dc_state creation.
- * DML21 IP, SOC and STATES are initialized at
- * creation time.
- *
- * Return: True if dml2 is successfully created, false otherwise.
- */
-bool dml21_create(const struct dc *in_dc, struct dml2_context **dml_ctx, const struct dml2_configuration_options *config);
-void dml21_destroy(struct dml2_context *dml2);
-void dml21_copy(struct dml2_context *dst_dml_ctx,
-	struct dml2_context *src_dml_ctx);
-bool dml21_create_copy(struct dml2_context **dst_dml_ctx,
-	struct dml2_context *src_dml_ctx);
-
-/* Structure for inputting external SOCBB and DCNIP values for tool based debugging. */
-struct socbb_ip_params_external {
-	struct dml2_ip_capabilities ip_params;
-	struct dml2_soc_bb soc_bb;
-};
-
-/*mcache parameters decided by dml*/
-struct dc_mcache_params {
-	bool valid;
-	/*
-	* For iMALL, dedicated mall mcaches are required (sharing of last
-	* slice possible), for legacy phantom or phantom without return
-	* the only mall mcaches need to be valid.
-	*/
-	bool requires_dedicated_mall_mcache;
-	unsigned int num_mcaches_plane0;
-	unsigned int num_mcaches_plane1;
-	/*
-	* Generally, plane0/1 slices must use a disjoint set of caches
-	* but in some cases the final segement of the two planes can
-	* use the same cache. If plane0_plane1 is set, then this is
-	* allowed.
-	*
-	* Similarly, the caches allocated to MALL prefetcher are generally
-	* disjoint, but if mall_prefetch is set, then the final segment
-	* between the main and the mall pixel requestor can use the same
-	* cache.
-	*
-	* Note that both bits may be set at the same time.
-	*/
-	struct {
-		bool mall_comb_mcache_p0;
-		bool mall_comb_mcache_p1;
-		bool plane0_plane1;
-	} last_slice_sharing;
-	/*
-	* A plane is divided into vertical slices of mcaches,
-	* which wrap on the surface width.
-	*
-	* For example, if the surface width is 7680, and split into
-	* three slices of equal width, the boundary array would contain
-	* [2560, 5120, 7680]
-	*
-	* The assignments are
-	* 0 = [0 .. 2559]
-	* 1 = [2560 .. 5119]
-	* 2 = [5120 .. 7679]
-	* 0 = [7680 .. INF]
-	* The final element implicitly is the same as the first, and
-	* at first seems invalid since it is never referenced (since)
-	* it is outside the surface. However, its useful when shifting
-	* (see below).
-	*
-	* For any given valid mcache assignment, a shifted version, wrapped
-	* on the surface width boundary is also assumed to be valid.
-	*
-	* For example, shifting [2560, 5120, 7680] by -50 results in
-	* [2510, 5170, 7630].
-	*
-	* The assignments are now:
-	* 0 = [0 .. 2509]
-	* 1 = [2510 .. 5169]
-	* 2 = [5170 .. 7629]
-	* 0 = [7630 .. INF]
-	*/
-	int mcache_x_offsets_plane0[DML2_MAX_MCACHES + 1];
-	int mcache_x_offsets_plane1[DML2_MAX_MCACHES + 1];
-};
-#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
index 51cae3efe310ec..04c5f09499acaa 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42_soc_bb.h
@@ -11,18 +11,11 @@ static const struct dml2_soc_qos_parameters dml_dcn42_variant_a_soc_qos_params =
 	.derate_table = {
 		.system_active_urgent = {
 			.dram_derate_percent_pixel = 65,
-			.dram_derate_percent_vm = 30,
+			.dram_derate_percent_vm = 50,
 			.dram_derate_percent_pixel_and_vm = 60,
 			.fclk_derate_percent = 80,
 			.dcfclk_derate_percent = 80,
 		},
-		.system_active_average = {
-			.dram_derate_percent_pixel = 30,
-			.dram_derate_percent_vm = 30,
-			.dram_derate_percent_pixel_and_vm = 30,
-			.fclk_derate_percent = 60,
-			.dcfclk_derate_percent = 60,
-		},
 		.dcn_mall_prefetch_urgent = {
 			.dram_derate_percent_pixel = 65,
 			.dram_derate_percent_vm = 30,
@@ -31,11 +24,11 @@ static const struct dml2_soc_qos_parameters dml_dcn42_variant_a_soc_qos_params =
 			.dcfclk_derate_percent = 80,
 		},
 		.dcn_mall_prefetch_average = {
-			.dram_derate_percent_pixel = 30,
+			.dram_derate_percent_pixel = 33,
 			.dram_derate_percent_vm = 30,
 			.dram_derate_percent_pixel_and_vm = 30,
-			.fclk_derate_percent = 60,
-			.dcfclk_derate_percent = 60,
+			.fclk_derate_percent = 66,
+			.dcfclk_derate_percent = 66,
 		},
 		.system_idle_average = {
 			.dram_derate_percent_pixel = 30,
@@ -45,6 +38,21 @@ static const struct dml2_soc_qos_parameters dml_dcn42_variant_a_soc_qos_params =
 			.dcfclk_derate_percent = 60,
 		},
 	},
+	.derate_table_per_dpm = {
+		// DPM0 can have slightly higher derate values. Rest of the DPMs should match the global values.
+		.dram_per_dpm_derate_pixel = {
+			{.derate_percent = 33, .clk_upperbound_threshold_khz = 400000},
+			{.derate_percent = 30, .clk_upperbound_threshold_khz = 0},
+		},
+		.fclk_per_dpm_derate = {
+			{.derate_percent = 66, .clk_upperbound_threshold_khz = 400000},
+			{.derate_percent = 60, .clk_upperbound_threshold_khz = 0},
+		},
+		.dcfclk_per_dpm_derate = {
+			{.derate_percent = 66, .clk_upperbound_threshold_khz = 200000},
+			{.derate_percent = 60, .clk_upperbound_threshold_khz = 0},
+		},
+	},
 	.writeback = {
 		.base_latency_us = 12,
 		.scaling_factor_us = 0,
@@ -134,13 +142,6 @@ static const struct dml2_soc_bb dml2_socbb_dcn42 = {
 				.fclk_derate_percent = 80,
 				.dcfclk_derate_percent = 80,
 			},
-			.system_active_average = {
-				.dram_derate_percent_pixel = 30,
-				.dram_derate_percent_vm = 30,
-				.dram_derate_percent_pixel_and_vm = 30,
-				.fclk_derate_percent = 60,
-				.dcfclk_derate_percent = 60,
-			},
 			.dcn_mall_prefetch_urgent = {
 				.dram_derate_percent_pixel = 65,
 				.dram_derate_percent_vm = 30,
@@ -219,7 +220,7 @@ static const struct dml2_soc_bb dml2_socbb_dcn42 = {
 };
 
 /* DCN42 params for DDR5 */
-struct dml2_soc_power_management_parameters dcn42_ddr5_power_management_parameters = {
+static struct dml2_soc_power_management_parameters dcn42_ddr5_power_management_parameters = {
 	.dram_clk_change_blackout_us = 36,
 	.fclk_change_blackout_us = 0,
 	.g7_ppt_blackout_us = 0,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h
index 5d844ff4b67155..e83a9ac018db2c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn42b_soc_bb.h
@@ -17,13 +17,6 @@ static const struct dml2_soc_qos_parameters dml_dcn42b_variant_a_soc_qos_params
 			.fclk_derate_percent = 80,
 			.dcfclk_derate_percent = 80,
 		},
-		.system_active_average = {
-			.dram_derate_percent_pixel = 30,
-			.dram_derate_percent_vm = 30,
-			.dram_derate_percent_pixel_and_vm = 30,
-			.fclk_derate_percent = 60,
-			.dcfclk_derate_percent = 60,
-		},
 		.dcn_mall_prefetch_urgent = {
 			.dram_derate_percent_pixel = 65,
 			.dram_derate_percent_vm = 30,
@@ -46,6 +39,17 @@ static const struct dml2_soc_qos_parameters dml_dcn42b_variant_a_soc_qos_params
 			.dcfclk_derate_percent = 60,
 		},
 	},
+	.derate_table_per_dpm = {
+		.dram_per_dpm_derate_pixel = {
+			{.derate_percent = 30, .clk_upperbound_threshold_khz = 0},
+		},
+		.fclk_per_dpm_derate = {
+			{.derate_percent = 60, .clk_upperbound_threshold_khz = 0},
+		},
+		.dcfclk_per_dpm_derate = {
+			{.derate_percent = 60, .clk_upperbound_threshold_khz = 0},
+		},
+	},
 	.writeback = {
 		.base_latency_us = 12,
 		.scaling_factor_us = 0,
@@ -53,7 +57,6 @@ static const struct dml2_soc_qos_parameters dml_dcn42b_variant_a_soc_qos_params
 	},
 	.qos_params = {
 		.dcn32x = {
-			.loaded_round_trip_latency_fclk_cycles = 106,
 			.urgent_latency_us = {
 				.base_latency_us = 4,
 				.base_latency_pixel_vm_us = 4,
@@ -61,8 +64,9 @@ static const struct dml2_soc_qos_parameters dml_dcn42b_variant_a_soc_qos_params
 				.scaling_factor_fclk_us = 0,
 				.scaling_factor_mhz = 0,
 			},
-			.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 4096,
+			.loaded_round_trip_latency_fclk_cycles = 106,
 			.urgent_out_of_order_return_per_channel_pixel_only_bytes = 4096,
+			.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 4096,
 			.urgent_out_of_order_return_per_channel_vm_only_bytes = 4096,
 		},
 	},
@@ -134,13 +138,6 @@ static const struct dml2_soc_bb dml2_socbb_dcn42b = {
 				.fclk_derate_percent = 80,
 				.dcfclk_derate_percent = 80,
 			},
-			.system_active_average = {
-				.dram_derate_percent_pixel = 30,
-				.dram_derate_percent_vm = 30,
-				.dram_derate_percent_pixel_and_vm = 30,
-				.fclk_derate_percent = 60,
-				.dcfclk_derate_percent = 60,
-			},
 			.dcn_mall_prefetch_urgent = {
 				.dram_derate_percent_pixel = 65,
 				.dram_derate_percent_vm = 30,
@@ -170,7 +167,6 @@ static const struct dml2_soc_bb dml2_socbb_dcn42b = {
 		},
 		.qos_params = {
 			.dcn32x = {
-				.loaded_round_trip_latency_fclk_cycles = 106,
 				.urgent_latency_us = {
 					.base_latency_us = 4,
 					.base_latency_pixel_vm_us = 4,
@@ -178,8 +174,9 @@ static const struct dml2_soc_bb dml2_socbb_dcn42b = {
 					.scaling_factor_fclk_us = 0,
 					.scaling_factor_mhz = 0,
 				},
-				.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 4096,
+				.loaded_round_trip_latency_fclk_cycles = 106,
 				.urgent_out_of_order_return_per_channel_pixel_only_bytes = 4096,
+				.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 4096,
 				.urgent_out_of_order_return_per_channel_vm_only_bytes = 4096,
 			},
 		},
@@ -205,14 +202,14 @@ static const struct dml2_soc_bb dml2_socbb_dcn42b = {
 	.xtalclk_mhz = 24,
 	.pcie_refclk_mhz = 100,
 	.dchub_refclk_mhz = 50,
-	.mall_allocated_for_dcn_mbytes = 64,
+	.mall_allocated_for_dcn_mbytes = 0,
 	.max_outstanding_reqs = 256,
 	.fabric_datapath_to_dcn_data_return_bytes = 32,
 	.return_bus_width_bytes = 64,
 	.hostvm_min_page_size_kbytes = 4,
 	.gpuvm_min_page_size_kbytes = 4,
-	.gpuvm_max_page_table_levels = 1,
 	.hostvm_max_non_cached_page_table_levels = 2,
+	.gpuvm_max_page_table_levels = 1,
 	.phy_downspread_percent = 0.38,
 	.dcn_downspread_percent = 0.38,
 	.dispclk_dppclk_vco_speed_mhz = 3000,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn4_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn4_soc_bb.h
index 16a4f97bca4eb0..ed82ef55b651ff 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn4_soc_bb.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn4_soc_bb.h
@@ -16,13 +16,6 @@ static const struct dml2_soc_qos_parameters dml_dcn4_variant_a_soc_qos_params =
 			.fclk_derate_percent = 76,
 			.dcfclk_derate_percent = 100,
 		},
-		.system_active_average = {
-			.dram_derate_percent_pixel = 17,
-			.dram_derate_percent_vm = 0,
-			.dram_derate_percent_pixel_and_vm = 0,
-			.fclk_derate_percent = 57,
-			.dcfclk_derate_percent = 75,
-		},
 		.dcn_mall_prefetch_urgent = {
 			.dram_derate_percent_pixel = 40,
 			.dram_derate_percent_vm = 0,
@@ -45,6 +38,17 @@ static const struct dml2_soc_qos_parameters dml_dcn4_variant_a_soc_qos_params =
 			.dcfclk_derate_percent = 100,
 		},
 	},
+	.derate_table_per_dpm = {
+		.dram_per_dpm_derate_pixel = {
+			{.derate_percent = 17, .clk_upperbound_threshold_khz = 0},
+		},
+		.fclk_per_dpm_derate = {
+			{.derate_percent = 57, .clk_upperbound_threshold_khz = 0},
+		},
+		.dcfclk_per_dpm_derate = {
+			{.derate_percent = 75, .clk_upperbound_threshold_khz = 0},
+		},
+	},
 	.writeback = {
 		.base_latency_us = 12,
 		.scaling_factor_us = 0,
@@ -142,13 +146,6 @@ static const struct dml2_soc_bb dml2_socbb_dcn401 = {
 				.fclk_derate_percent = 76,
 				.dcfclk_derate_percent = 100,
 			},
-			.system_active_average = {
-				.dram_derate_percent_pixel = 15,
-				.dram_derate_percent_vm = 0,
-				.dram_derate_percent_pixel_and_vm = 0,
-				.fclk_derate_percent = 57,
-				.dcfclk_derate_percent = 75,
-			},
 			.dcn_mall_prefetch_urgent = {
 				.dram_derate_percent_pixel = 40,
 				.dram_derate_percent_vm = 0,
@@ -345,7 +342,7 @@ static const struct dml2_ip_capabilities dml2_dcn401_max_ip_caps = {
 	.compressed_buffer_segment_size_in_kbytes = 64,
 	.cursor_buffer_size = 24,
 	.max_flip_time_us = 80,
-	.max_flip_time_lines = 32,
+	.max_flip_time_lines = 34,
 	.hostvm_mode = 0,
 	.subvp_drr_scheduling_margin_us = 100,
 	.subvp_prefetch_end_to_mall_start_us = 15,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn6_soc_bb.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn6_soc_bb.h
index 19d43e5b7cc275..04292e77698b12 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn6_soc_bb.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/bounding_boxes/dcn6_soc_bb.h
@@ -206,6 +206,18 @@ static inline void dcn6_test_initialize_v3_sop_latencies_all_levels(
 				df_response_time_ps);
 }
 
+/**
+ * dcn6_test_override_lsdma_bandwidth_v3 - Debug / test interface that
+ * can be used to override the LSDMA bandwidth as needed for any LL and SOP.
+ * @dchub: DCN6 flat UTM QoS v3 table to modify
+ */
+static inline void dcn6_test_override_lsdma_bandwidth_v3(
+		struct utm_qos_model_dchub_v3 *dchub)
+{
+	/* Implement override as needed */
+	(void)dchub;
+}
+
 static inline void dcn6_test_initialize_utm_qos_model_v3(
 		struct utm_qos_model *qos_model,
 		struct utm_qos_model_dchub_v3 *dchub)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_dchub_registers.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_dchub_registers.h
index 5a5f7b92c324af..cfa014ddfbebc0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_dchub_registers.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_dchub_registers.h
@@ -87,6 +87,7 @@ struct dml2_display_arb_regs {
 	uint32_t hvm_max_qos_commit_threshold;
 	uint32_t hvm_min_req_outstand_commit_threshold;
 	uint32_t compbuf_reserved_space_kbytes;
+	uint32_t compbuf_reserved_space_zs;
 	uint32_t compbuf_size;
 	uint32_t sdpif_request_rate_limit;
 	uint32_t allow_sdpif_rate_limit_when_cstate_req;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
index f402bb933f7dc0..27ffe7ed11c7d0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_display_cfg_types.h
@@ -487,6 +487,7 @@ struct dml2_display_cfg {
 			bool mode_support_check_disable;
 			bool mcache_admissibility_check_disable;
 			bool surface_viewport_size_check_disable;
+			bool outstanding_requests_check_disable;
 			double dlg_ref_clk_mhz;
 			double dispclk_mhz;
 			double dcfclk_mhz;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
index b830e95b27c873..100d162811f26b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_soc_parameter_types.h
@@ -20,7 +20,6 @@ struct dml2_soc_derate_values {
 
 struct dml2_soc_derates {
 	struct dml2_soc_derate_values system_active_urgent;
-	struct dml2_soc_derate_values system_active_average;
 	struct dml2_soc_derate_values dcn_mall_prefetch_urgent;
 	struct dml2_soc_derate_values dcn_mall_prefetch_average;
 	struct dml2_soc_derate_values system_idle_average;
@@ -71,21 +70,30 @@ enum dml2_qos_param_type {
 	dml2_qos_param_type_dcn4x
 };
 
-//Indicies mapped to DPM level
-// Unpopulated indicies should fallback to the global derate value.
+// Per-DPM derate structure: each element pairs a derate percentage with its clock threshold
+// Requirements:
+// - Index 0 must always be populated (the base/default derate)
+// - Last populated entry must have clk_upperbound_threshold_khz = 0
+// - All clock arrays within a power state should have the same number of entries
+// - If a non-zero DPM index has derate_percent = 0, fall back to index 0
+// Unpopulated derates should fallback to the global derate value.
+// Each element pairs a derate percentage with its threshold
 struct dml2_soc_derate_values_per_dpm {
-	unsigned int dram_derate_percent_pixel[DML_MAX_CLK_TABLE_SIZE];
-	unsigned int fclk_derate_percent[DML_MAX_CLK_TABLE_SIZE];
-	unsigned int dcfclk_derate_percent[DML_MAX_CLK_TABLE_SIZE];
+	unsigned int derate_percent;
+	unsigned int clk_upperbound_threshold_khz;
 };
 
+// Per-DPM derate arrays for system_active_average (the only power state that varies by DPM)
+// Urgent derates are constant across DPMs and come from derate_table.system_active_urgent
 struct dml2_soc_derates_per_dpm {
-	struct dml2_soc_derate_values_per_dpm system_active_derates_per_dpm;
+	struct dml2_soc_derate_values_per_dpm dram_per_dpm_derate_pixel[DML_MAX_CLK_TABLE_SIZE];
+	struct dml2_soc_derate_values_per_dpm fclk_per_dpm_derate[DML_MAX_CLK_TABLE_SIZE];
+	struct dml2_soc_derate_values_per_dpm dcfclk_per_dpm_derate[DML_MAX_CLK_TABLE_SIZE];
 };
 
 struct dml2_soc_qos_parameters {
-	struct dml2_soc_derates derate_table;
-	struct dml2_soc_derates_per_dpm derate_table_per_dpm;
+	struct dml2_soc_derates derate_table; // Single-value derates (source of truth for urgent, idle, mall_prefetch)
+	struct dml2_soc_derates_per_dpm derate_table_per_dpm; // Per-DPM derates for system_active_average only
 	struct {
 		unsigned int base_latency_us;
 		unsigned int scaling_factor_us;
@@ -164,6 +172,8 @@ struct dml2_soc_bb {
 	double lower_bound_bandwidth_dchub;
 	double fraction_of_urgent_bandwidth_nominal_target;
 	double fraction_of_urgent_bandwidth_flip_target;
+	double hostvm_inefficiency_fraction;
+	unsigned int max_lsdma_bandwidth_kbps;
 	unsigned int dprefclk_mhz;
 	unsigned int xtalclk_mhz;
 	unsigned int pcie_refclk_mhz;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
index bf71f4e1a45197..5a8550f592caeb 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
@@ -380,6 +380,10 @@ struct dml2_mode_support_info {
 	bool temp_read_or_ppt_support;
 	bool qos_bandwidth_support;
 	bool dcfclk_support;
+	bool alternate_channel_size_support;
+	bool EnoughUrgentLatencyHidingSupport;
+	bool DSCSlicesODMModeSupported;
+	bool ODMSupport;
 }; // dml2_mode_support_info
 
 struct dml2_memory_path_latency {
@@ -402,6 +406,7 @@ struct dml2_memory_path_bandwidth {
 		struct {
 			double urgent_bandwidth_kbps; // kbytes per sec
 			double non_urgent_bandwidth_kbps; // kbytes per sec
+			double lsdma_bandwidth_kbps; // kbytes per sec
 		} dcn5;
 	};
 };
@@ -480,6 +485,7 @@ struct dml2_display_cfg_programming {
 	struct {
 		bool meets_eco; // Stutter cycles will meet Z8 ECO criteria
 		bool supported_in_blank; // Changing to configurations where this is false requires Z8 to be disabled during the transition
+		bool global_support; // Z8 supported in vblank or vactive
 	} z8_stutter;
 
 	struct dml2_dchub_global_register_set global_regs;
@@ -542,6 +548,14 @@ struct dml2_display_cfg_programming {
 			double max_active_fclk_change_latency_supported;
 			unsigned int min_return_latency_in_dcfclk;
 
+			struct {
+				struct {
+					double sdp_bw_mbps;
+					double dram_bw_mbps;
+					double dram_vm_only_bw_mbps;
+				} sys_active;
+			} non_urg_bw_available;
+
 			struct {
 				struct {
 					double sdp_bw_mbps;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_cga/dml2_cga_dcn6.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_cga/dml2_cga_dcn6.c
index 8c3dd030e77511..43ee8a0b21b3cb 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_cga/dml2_cga_dcn6.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_cga/dml2_cga_dcn6.c
@@ -56,7 +56,9 @@ static double cga_dcn6_adjust_to_dfs_clock_value_mhz(const struct dml2_clock_gra
 	double vco_divider;
 	double adjusted_clock_mhz;
 
-	DML_ASSERT_MSG(adjuster->dispclk_dppclk_vco_speed_mhz > 1, "invalid dispclk_dppclk_vco_speed_mhz value!\n");
+	if (adjuster->dispclk_dppclk_vco_speed_mhz == 0.0)
+		return clk_mhz;
+
 	if (clk_mhz == 0)
 		/* There are cases when a clock is not needed */
 		return 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c
index 6ea36aedcf203c..a040c3f4733717 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4.c
@@ -701,6 +701,10 @@ bool core_dcn4_mode_programming(struct dml2_core_mode_programming_in_out *in_out
 	result = dml2_core_calcs_mode_programming_ex(&l->mode_programming_ex_params);
 
 	if (result) {
+		// Populate the global Z8 stutter support (common to all packing paths)
+		dml2_core_calcs_get_z8_stutter_support(&core->clean_me_up.mode_lib,
+			&in_out->programming->z8_stutter.global_support);
+
 		// If the input display configuration contains implict SVP, we need to use a special packer
 		if (in_out->display_cfg->display_config.overrides.enable_subvp_implicit_pmo) {
 			pack_mode_programming_params_with_implicit_subvp(core, in_out->display_cfg, &l->svp_expanded_display_cfg, in_out->programming, &core->scratch);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
index 72acf12e92516c..2d43a35c4c427c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.c
@@ -357,6 +357,8 @@ dml_get_var_func(svp_prefetch_urg_bw_available_sdp, double, mode_lib->mp.urg_ban
 dml_get_var_func(svp_prefetch_urg_bw_available_dram, double, mode_lib->mp.urg_bandwidth_available[dml2_core_internal_soc_state_svp_prefetch][dml2_core_internal_bw_dram]);
 dml_get_var_func(svp_prefetch_urg_bw_available_dram_vm_only, double, mode_lib->mp.urg_bandwidth_available_vm_only[dml2_core_internal_soc_state_svp_prefetch]);
 
+dml_get_var_func(sys_active_non_urg_bw_available_sdp, double, mode_lib->mp.non_urg_bandwidth_available[dml2_core_internal_soc_state_sys_active][dml2_core_internal_bw_sdp]);
+
 dml_get_var_func(urgent_latency, double, mode_lib->mp.UrgentLatency);
 dml_get_var_func(max_urgent_latency_us, double, mode_lib->ms.support.max_urgent_latency_us);
 dml_get_var_func(max_non_urgent_latency_us, double, mode_lib->ms.support.max_non_urgent_latency_us);
@@ -1999,6 +2001,10 @@ static void CalculateMALLUseForStaticScreen(
 #endif
 	}
 
+	/*  If no MALL is allocated for DCN, there is no need to calculate usage for static screen. */
+	if (MALLAllocatedForDCN == 0)
+		return;
+
 	SurfaceToAddToMALL = 0;
 	CanAddAnotherSurfaceToMALL = true;
 	while (CanAddAnotherSurfaceToMALL) {
@@ -2739,16 +2745,15 @@ static double dml_get_return_bandwidth_available(
 			derate_fabric_factor = soc->qos_parameters.derate_table.dcn_mall_prefetch_average.fclk_derate_percent / 100.0;
 			derate_dram_factor = soc->qos_parameters.derate_table.dcn_mall_prefetch_average.dram_derate_percent_pixel / 100.0;
 		} else { // just assume sys_active
-			// use per dpm derates if the values are populated. Otherwise use global derates
-			derate_sdp_factor = soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dcfclk_derate_percent[uclk_dpm_level] != 0 ?
-				soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dcfclk_derate_percent[uclk_dpm_level] / 100.0 :
-				soc->qos_parameters.derate_table.system_active_average.dcfclk_derate_percent / 100.0;
-			derate_fabric_factor = soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.fclk_derate_percent[uclk_dpm_level] != 0 ?
-				soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.fclk_derate_percent[uclk_dpm_level] / 100.0 :
-				soc->qos_parameters.derate_table.system_active_average.fclk_derate_percent / 100.0;
-			derate_dram_factor = soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dram_derate_percent_pixel[uclk_dpm_level] != 0 ?
-				soc->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dram_derate_percent_pixel[uclk_dpm_level] / 100.0 :
-				soc->qos_parameters.derate_table.system_active_average.dram_derate_percent_pixel / 100.0;
+			derate_sdp_factor = (soc->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[uclk_dpm_level].derate_percent != 0 ?
+				soc->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[uclk_dpm_level].derate_percent :
+				soc->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[0].derate_percent) / 100.0;
+			derate_fabric_factor = (soc->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[uclk_dpm_level].derate_percent != 0 ?
+				soc->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[uclk_dpm_level].derate_percent :
+				soc->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[0].derate_percent) / 100.0;
+			derate_dram_factor = (soc->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[uclk_dpm_level].derate_percent != 0 ?
+				soc->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[uclk_dpm_level].derate_percent :
+				soc->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[0].derate_percent) / 100.0;
 		}
 	} else { // urgent bw
 		if (state_type == dml2_core_internal_soc_state_svp_prefetch) {
@@ -4117,9 +4122,7 @@ static bool ValidateODMMode(enum dml2_odm_mode ODMMode,
 	bool UseDSC,
 	unsigned int NumberOfDSCSlices,
 	unsigned int TotalNumberOfActiveDPP,
-	unsigned int TotalNumberOfActiveOPP,
 	unsigned int MaxNumDPP,
-	unsigned int MaxNumOPP,
 	double DISPCLKRequired,
 	unsigned int NumberOfDPPRequired,
 	unsigned int MaxHActiveForDSC,
@@ -4135,7 +4138,7 @@ static bool ValidateODMMode(enum dml2_odm_mode ODMMode,
 
 	if (DISPCLKRequired > MaxDispclk)
 		return false;
-	if ((TotalNumberOfActiveDPP + NumberOfDPPRequired) > MaxNumDPP || (TotalNumberOfActiveOPP + NumberOfDPPRequired) > MaxNumOPP)
+	if ((TotalNumberOfActiveDPP + NumberOfDPPRequired) > MaxNumDPP)
 		return false;
 	if (are_odm_segments_symmetrical) {
 		if (HActive % (NumberOfDPPRequired * pixels_per_clock_cycle))
@@ -4181,9 +4184,7 @@ static noinline_for_stack void CalculateODMMode(
 	double MaxDispclk,
 	bool DSCEnable,
 	unsigned int TotalNumberOfActiveDPP,
-	unsigned int TotalNumberOfActiveOPP,
 	unsigned int MaxNumDPP,
-	unsigned int MaxNumOPP,
 	double PixelClock,
 	unsigned int NumberOfDSCSlices,
 
@@ -4253,9 +4254,7 @@ static noinline_for_stack void CalculateODMMode(
 			UseDSC,
 			NumberOfDSCSlices,
 			TotalNumberOfActiveDPP,
-			TotalNumberOfActiveOPP,
 			MaxNumDPP,
-			MaxNumOPP,
 			DISPCLKRequired,
 			NumberOfDPPRequired,
 			MaxHActiveForDSC,
@@ -6556,7 +6555,6 @@ static void CalculateFlipSchedule(
 	bool GPUVMEnable,
 	double vm_bytes, // vm_bytes
 	double DPTEBytesPerRow, // dpte_row_bytes
-	double BandwidthAvailableForImmediateFlip,
 	unsigned int TotImmediateFlipBytes,
 	enum dml2_source_format_class SourcePixelFormat,
 	double LineTime,
@@ -6568,7 +6566,6 @@ static void CalculateFlipSchedule(
 	bool use_one_row_for_frame_flip,
 	unsigned int max_flip_time_us,
 	unsigned int max_flip_time_lines,
-	unsigned int per_pipe_flip_bytes,
 	unsigned int meta_row_bytes,
 	unsigned int meta_row_height,
 	unsigned int meta_row_height_chroma,
@@ -6590,7 +6587,6 @@ static void CalculateFlipSchedule(
 	DML_LOG_VERBOSE("DML::%s: GPUVMEnable = %u\n", __func__, GPUVMEnable);
 	DML_LOG_VERBOSE("DML::%s: ip.max_flip_time_us = %d\n", __func__, max_flip_time_us);
 	DML_LOG_VERBOSE("DML::%s: ip.max_flip_time_lines = %d\n", __func__, max_flip_time_lines);
-	DML_LOG_VERBOSE("DML::%s: BandwidthAvailableForImmediateFlip = %f\n", __func__, BandwidthAvailableForImmediateFlip);
 	DML_LOG_VERBOSE("DML::%s: TotImmediateFlipBytes = %u\n", __func__, TotImmediateFlipBytes);
 	DML_LOG_VERBOSE("DML::%s: use_lb_flip_bw = %u\n", __func__, use_lb_flip_bw);
 	DML_LOG_VERBOSE("DML::%s: iflip_enable = %u\n", __func__, iflip_enable);
@@ -6638,104 +6634,71 @@ static void CalculateFlipSchedule(
 #endif
 		DML_ASSERT(l->min_row_time > 0);
 
-		if (use_lb_flip_bw) {
-			// For mode check, calculation the flip bw requirement with worst case flip time
-			l->max_flip_time = math_min2(math_min2(l->min_row_time, (double)max_flip_time_lines * LineTime / VRatio),
-				math_max2(Tvm_trips_flip_rounded + 2 * Tr0_trips_flip_rounded, (double)max_flip_time_us));
-
-			//The lower bound on flip bandwidth
-			// Note: The get_urgent_bandwidth_required already consider dpte_row_bw and meta_row_bw in bandwidth calculation, so leave final_flip_bw = 0 if iflip not required
-			l->lb_flip_bw = 0;
-
-			if (iflip_enable) {
-				l->hvm_scaled_vm_bytes = vm_bytes * HostVMInefficiencyFactor;
-				l->num_rows = 2;
-				l->hvm_scaled_row_bytes = (l->num_rows * l->dpte_row_bytes * HostVMInefficiencyFactor + l->num_rows * meta_row_bytes);
-				l->hvm_scaled_vm_row_bytes = l->hvm_scaled_vm_bytes + l->hvm_scaled_row_bytes;
-				l->lb_flip_bw = math_max3(
-					l->hvm_scaled_vm_row_bytes / (l->max_flip_time - Tno_bw_flip),
-					l->hvm_scaled_vm_bytes / (l->max_flip_time - Tno_bw_flip - 2 * Tr0_trips_flip_rounded),
-					l->hvm_scaled_row_bytes / (l->max_flip_time - Tvm_trips_flip_rounded));
-#ifdef __DML_VBA_DEBUG__
-				DML_LOG_VERBOSE("DML::%s: max_flip_time = %f\n", __func__, l->max_flip_time);
-				DML_LOG_VERBOSE("DML::%s: total vm bytes (hvm ineff scaled) = %f\n", __func__, l->hvm_scaled_vm_bytes);
-				DML_LOG_VERBOSE("DML::%s: total row bytes (%f row, hvm ineff scaled) = %f\n", __func__, l->num_rows, l->hvm_scaled_row_bytes);
-				DML_LOG_VERBOSE("DML::%s: total vm+row bytes (hvm ineff scaled) = %f\n", __func__, l->hvm_scaled_vm_row_bytes);
-				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm and row = %f\n", __func__, l->hvm_scaled_vm_row_bytes / (l->max_flip_time - Tno_bw_flip));
-				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm = %f\n", __func__, l->hvm_scaled_vm_bytes / (l->max_flip_time - Tno_bw_flip - 2 * Tr0_trips_flip_rounded));
-				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for row = %f\n", __func__, l->hvm_scaled_row_bytes / (l->max_flip_time - Tvm_trips_flip_rounded));
-
-				if (l->lb_flip_bw > 0) {
-					DML_LOG_VERBOSE("DML::%s: mode_support est Tvm_flip = %f (bw-based)\n", __func__, Tno_bw_flip + l->hvm_scaled_vm_bytes / l->lb_flip_bw);
-					DML_LOG_VERBOSE("DML::%s: mode_support est Tr0_flip = %f (bw-based)\n", __func__, l->hvm_scaled_row_bytes / l->lb_flip_bw / l->num_rows);
-					DML_LOG_VERBOSE("DML::%s: mode_support est dst_y_per_vm_flip = %f (bw-based)\n", __func__, Tno_bw_flip + l->hvm_scaled_vm_bytes / l->lb_flip_bw / LineTime);
-					DML_LOG_VERBOSE("DML::%s: mode_support est dst_y_per_row_flip = %f (bw-based)\n", __func__, l->hvm_scaled_row_bytes / l->lb_flip_bw / LineTime / l->num_rows);
-					DML_LOG_VERBOSE("DML::%s: Tvm_trips_flip_rounded + 2*Tr0_trips_flip_rounded = %f\n", __func__, (Tvm_trips_flip_rounded + 2 * Tr0_trips_flip_rounded));
-				}
-#endif
-				l->lb_flip_bw = math_max3(l->lb_flip_bw,
-						l->hvm_scaled_vm_bytes / (31 * LineTime) - Tno_bw_flip,
-						(l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / (15 * LineTime));
-
-#ifdef __DML_VBA_DEBUG__
-				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm reg limit = %f\n", __func__, l->hvm_scaled_vm_bytes / (31 * LineTime) - Tno_bw_flip);
-				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for row reg limit = %f\n", __func__, (l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / (15 * LineTime));
-#endif
-			}
-
-			*final_flip_bw = l->lb_flip_bw;
-
-			*dst_y_per_vm_flip = 1; // not used
-			*dst_y_per_row_flip = 1; // not used
-			*ImmediateFlipSupportedForPipe = l->min_row_time >= (Tvm_trips_flip_rounded + 2 * Tr0_trips_flip_rounded);
-		} else {
-			if (iflip_enable) {
-				l->ImmediateFlipBW = (double)per_pipe_flip_bytes * BandwidthAvailableForImmediateFlip / (double)TotImmediateFlipBytes; // flip_bw(i)
-
-#ifdef __DML_VBA_DEBUG__
-				DML_LOG_VERBOSE("DML::%s: per_pipe_flip_bytes = %d\n", __func__, per_pipe_flip_bytes);
-				DML_LOG_VERBOSE("DML::%s: BandwidthAvailableForImmediateFlip = %f\n", __func__, BandwidthAvailableForImmediateFlip);
-				DML_LOG_VERBOSE("DML::%s: ImmediateFlipBW = %f\n", __func__, l->ImmediateFlipBW);
-				DML_LOG_VERBOSE("DML::%s: portion of flip bw = %f\n", __func__, (double)per_pipe_flip_bytes / (double)TotImmediateFlipBytes);
-#endif
-				if (l->ImmediateFlipBW == 0) {
-					l->Tvm_flip = 0;
-					l->Tr0_flip = 0;
-				} else {
-					l->Tvm_flip = math_max3(Tvm_trips_flip,
-						Tno_bw_flip + vm_bytes * HostVMInefficiencyFactor / l->ImmediateFlipBW,
-						LineTime / 4.0);
-
-					l->Tr0_flip = math_max3(Tr0_trips_flip,
-						(l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / l->ImmediateFlipBW,
-						LineTime / 4.0);
-				}
-#ifdef __DML_VBA_DEBUG__
-				DML_LOG_VERBOSE("DML::%s: total vm bytes (hvm ineff scaled) = %f\n", __func__, vm_bytes * HostVMInefficiencyFactor);
-				DML_LOG_VERBOSE("DML::%s: total row bytes (hvm ineff scaled, one row) = %f\n", __func__, (l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes));
-
-				DML_LOG_VERBOSE("DML::%s: Tvm_flip = %f (bw-based), Tvm_trips_flip = %f (latency-based)\n", __func__, Tno_bw_flip + vm_bytes * HostVMInefficiencyFactor / l->ImmediateFlipBW, Tvm_trips_flip);
-				DML_LOG_VERBOSE("DML::%s: Tr0_flip = %f (bw-based), Tr0_trips_flip = %f (latency-based)\n", __func__, (l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / l->ImmediateFlipBW, Tr0_trips_flip);
-#endif
-				*dst_y_per_vm_flip = math_ceil2(4.0 * (l->Tvm_flip / LineTime), 1.0) / 4.0;
-				*dst_y_per_row_flip = math_ceil2(4.0 * (l->Tr0_flip / LineTime), 1.0) / 4.0;
-
-				*final_flip_bw = math_max2(vm_bytes * HostVMInefficiencyFactor / (*dst_y_per_vm_flip * LineTime),
-					(l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / (*dst_y_per_row_flip * LineTime));
-
-				if (*dst_y_per_vm_flip >= 32 || *dst_y_per_row_flip >= 16 || l->Tvm_flip + 2 * l->Tr0_flip > l->min_row_time) {
-					*ImmediateFlipSupportedForPipe = false;
-				} else {
-					*ImmediateFlipSupportedForPipe = iflip_enable;
-				}
+		// For mode check, calculation the flip bw requirement with worst case flip time
+		l->max_flip_time = math_min2(math_min2(l->min_row_time, (double)max_flip_time_lines * LineTime / VRatio),
+			math_max2(Tvm_trips_flip_rounded + 2 * Tr0_trips_flip_rounded, (double)max_flip_time_us));
+
+		//The lower bound on flip bandwidth
+		// Note: The get_urgent_bandwidth_required already consider dpte_row_bw and meta_row_bw in bandwidth calculation, so leave final_flip_bw = 0 if iflip not required
+		l->lb_flip_bw = 0;
+
+		l->vm_and_row_time_budget = l->max_flip_time - Tno_bw_flip;
+		l->vm_time_budget = l->max_flip_time - Tno_bw_flip - 2 * Tr0_trips_flip_rounded;
+		l->row_time_budget = l->max_flip_time - Tvm_trips_flip_rounded;
+
+		if (iflip_enable) {
+			l->hvm_scaled_vm_bytes = vm_bytes * HostVMInefficiencyFactor;
+			l->num_rows = 2;
+			l->hvm_scaled_row_bytes = (l->num_rows * l->dpte_row_bytes * HostVMInefficiencyFactor + l->num_rows * meta_row_bytes);
+			l->hvm_scaled_vm_row_bytes = l->hvm_scaled_vm_bytes + l->hvm_scaled_row_bytes;
+			l->lb_flip_bw = math_max3(
+				l->hvm_scaled_vm_row_bytes / l->vm_and_row_time_budget,
+				l->hvm_scaled_vm_bytes / math_min2(l->vm_time_budget, 31.75 * LineTime - Tno_bw_flip),
+				(l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / math_min2(l->row_time_budget / 2, 15.75 * LineTime)); //use bandwidth for a single row
+#ifdef __DML_VBA_DEBUG__
+			DML_LOG_VERBOSE("DML::%s: max_flip_time = %f\n", __func__, l->max_flip_time);
+			DML_LOG_VERBOSE("DML::%s: total vm bytes (hvm ineff scaled) = %f\n", __func__, l->hvm_scaled_vm_bytes);
+			DML_LOG_VERBOSE("DML::%s: total row bytes (%f row, hvm ineff scaled) = %f\n", __func__, l->num_rows, l->hvm_scaled_row_bytes);
+			DML_LOG_VERBOSE("DML::%s: total vm+row bytes (hvm ineff scaled) = %f\n", __func__, l->hvm_scaled_vm_row_bytes);
+			DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm and row = %f\n", __func__, l->hvm_scaled_vm_row_bytes / l->vm_and_row_time_budget);
+			DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm = %f\n", __func__, l->hvm_scaled_vm_bytes / l->vm_time_budget);
+			DML_LOG_VERBOSE("DML::%s: lb_flip_bw for row = %f\n", __func__, l->hvm_scaled_row_bytes / l->row_time_budget);
+			DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm reg limit = %f\n", __func__, l->hvm_scaled_vm_bytes / (31.75 * LineTime - Tno_bw_flip));
+			DML_LOG_VERBOSE("DML::%s: lb_flip_bw for row reg limit = %f\n", __func__, (l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / (15.75 * LineTime));
+#endif
+		}
+
+		*final_flip_bw = l->lb_flip_bw;
+
+		if (l->lb_flip_bw > 0) {
+			DML_LOG_VERBOSE("DML::%s: mode_support est Tvm_flip = %f (bw-based)\n", __func__, Tno_bw_flip + l->hvm_scaled_vm_bytes / l->lb_flip_bw);
+			DML_LOG_VERBOSE("DML::%s: mode_support est Tr0_flip = %f (bw-based)\n", __func__, l->hvm_scaled_row_bytes / l->lb_flip_bw / l->num_rows);
+			DML_LOG_VERBOSE("DML::%s: mode_support est dst_y_per_vm_flip = %f (bw-based)\n", __func__, Tno_bw_flip + l->hvm_scaled_vm_bytes / l->lb_flip_bw / LineTime);
+			DML_LOG_VERBOSE("DML::%s: mode_support est dst_y_per_row_flip = %f (bw-based)\n", __func__, l->hvm_scaled_row_bytes / l->lb_flip_bw / LineTime / l->num_rows);
+			DML_LOG_VERBOSE("DML::%s: Tvm_trips_flip_rounded + 2*Tr0_trips_flip_rounded = %f\n", __func__, (Tvm_trips_flip_rounded + 2 * Tr0_trips_flip_rounded));
+
+			l->Tvm_flip = math_max3(Tvm_trips_flip,
+					Tno_bw_flip + vm_bytes * HostVMInefficiencyFactor / l->lb_flip_bw,
+					LineTime / 4.0);
+			l->Tr0_flip = math_max3(Tr0_trips_flip,
+					(l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / l->lb_flip_bw,
+					LineTime / 4.0);
+
+			*dst_y_per_vm_flip = math_ceil2(4.0 * l->Tvm_flip / LineTime, 1.0) / 4.0;
+			*dst_y_per_row_flip = math_ceil2(4.0 * l->Tr0_flip / LineTime, 1.0) / 4.0;
+
+			if (*dst_y_per_vm_flip >= 32 || *dst_y_per_row_flip >= 16 || l->Tvm_flip + 2 * l->Tr0_flip > l->min_row_time) {
+				*ImmediateFlipSupportedForPipe = false;
 			} else {
-				l->Tvm_flip = 0;
-				l->Tr0_flip = 0;
-				*dst_y_per_vm_flip = 0;
-				*dst_y_per_row_flip = 0;
-				*final_flip_bw = 0;
 				*ImmediateFlipSupportedForPipe = iflip_enable;
 			}
+		} else {
+			l->Tvm_flip = 0;
+			l->Tr0_flip = 0;
+			*dst_y_per_vm_flip = 0;
+			*dst_y_per_row_flip = 0;
+			*final_flip_bw = 0;
+			*ImmediateFlipSupportedForPipe = iflip_enable;
 		}
 	} else {
 		l->Tvm_flip = 0;
@@ -6876,6 +6839,8 @@ static void CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 
 	*p->global_fclk_change_supported = true;
 	*p->global_dram_clock_change_supported = true;
+	if (p->global_z8_stutter_supported)
+		*p->global_z8_stutter_supported = true;
 
 	for (unsigned int k = 0; k < p->NumberOfActiveSurfaces; ++k) {
 		double h_total = (double)p->display_cfg->stream_descriptors[p->display_cfg->plane_descriptors[k].stream_index].timing.h_total;
@@ -6990,6 +6955,12 @@ static void CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 		if (p->DRAMClockChangeSupport[k] == dml2_pstate_change_unsupported)
 			*p->global_dram_clock_change_supported = false;
 
+		if (p->global_z8_stutter_supported &&
+		    !dml_is_phantom_pipe(&p->display_cfg->plane_descriptors[k]) &&
+		    !(reserved_vblank_time_us > p->mmSOCParameters.SREnterPlusExitZ8Time) &&
+		    !((s->ActiveClockChangeLatencyHiding - p->Watermark->Z8StutterEnterPlusExitWatermark) > 0))
+			*p->global_z8_stutter_supported = false;
+
 		s->dst_y_pstate = (unsigned int)(math_ceil2((p->mmSOCParameters.DRAMClockChangeLatency + p->mmSOCParameters.UrgentLatency) / (h_total / pixel_clock_mhz), 1));
 		s->src_y_pstate_l = (unsigned int)(math_ceil2(s->dst_y_pstate * v_ratio, p->SwathHeightY[k]));
 		s->src_y_ahead_l = (unsigned int)(math_floor2(p->DETBufferSizeY[k] / p->BytePerPixelDETY[k] / p->SwathWidthY[k], p->SwathHeightY[k]) + s->LBLatencyHidingSourceLinesY[k]);
@@ -7884,7 +7855,6 @@ static noinline_for_stack void dml_core_ms_prefetch_check(struct dml2_core_inter
 				display_cfg->gpuvm_enable,
 				mode_lib->ms.vm_bytes[k],
 				mode_lib->ms.DPTEBytesPerRow[k],
-				mode_lib->ms.BandwidthAvailableForImmediateFlip,
 				mode_lib->ms.TotImmediateFlipBytes,
 				display_cfg->plane_descriptors[k].pixel_format,
 				(display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].timing.h_total / ((double)display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].timing.pixel_clock_khz / 1000)),
@@ -7896,7 +7866,6 @@ static noinline_for_stack void dml_core_ms_prefetch_check(struct dml2_core_inter
 				mode_lib->ms.use_one_row_for_frame_flip[k],
 				mode_lib->ip.max_flip_time_us,
 				mode_lib->ip.max_flip_time_lines,
-				s->per_pipe_flip_bytes[k],
 				mode_lib->ms.meta_row_bytes[k],
 				s->meta_row_height_luma[k],
 				s->meta_row_height_chroma[k],
@@ -8027,6 +7996,7 @@ static noinline_for_stack void dml_core_ms_prefetch_check(struct dml2_core_inter
 	CalculateWatermarks_params->MaxActiveFCLKChangeLatencySupported = &s->dummy_single[0]; // double *MaxActiveFCLKChangeLatencySupported
 	CalculateWatermarks_params->USRRetrainingSupport = &mode_lib->ms.support.USRRetrainingSupport;
 	CalculateWatermarks_params->g6_temp_read_support = &mode_lib->ms.support.g6_temp_read_support;
+	CalculateWatermarks_params->global_z8_stutter_supported = NULL; // only consumed by mode programming
 	CalculateWatermarks_params->VActiveLatencyHidingMargin = mode_lib->ms.VActiveLatencyHidingMargin;
 	CalculateWatermarks_params->VActiveLatencyHidingUs = mode_lib->ms.VActiveLatencyHidingUs;
 
@@ -8483,7 +8453,6 @@ static bool dml_core_mode_support(struct dml2_core_calcs_mode_support_ex *in_out
 	CalculateSwathAndDETConfiguration(&mode_lib->scratch, CalculateSwathAndDETConfiguration_params);
 
 	mode_lib->ms.TotalNumberOfActiveDPP = 0;
-	mode_lib->ms.TotalNumberOfActiveOPP = 0;
 	mode_lib->ms.support.TotalAvailablePipesSupport = true;
 
 	for (k = 0; k < mode_lib->ms.num_active_planes; ++k) {
@@ -8519,9 +8488,7 @@ static bool dml_core_mode_support(struct dml2_core_calcs_mode_support_ex *in_out
 			mode_lib->ms.max_dispclk_freq_mhz,
 			false, // DSCEnable
 			mode_lib->ms.TotalNumberOfActiveDPP,
-			mode_lib->ms.TotalNumberOfActiveOPP,
 			mode_lib->ip.max_num_dpp,
-			mode_lib->ip.max_num_opp,
 			((double)display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].timing.pixel_clock_khz / 1000),
 			mode_lib->ms.support.NumberOfDSCSlices[k],
 
@@ -8540,9 +8507,7 @@ static bool dml_core_mode_support(struct dml2_core_calcs_mode_support_ex *in_out
 			mode_lib->ms.max_dispclk_freq_mhz,
 			true, // DSCEnable
 			mode_lib->ms.TotalNumberOfActiveDPP,
-			mode_lib->ms.TotalNumberOfActiveOPP,
 			mode_lib->ip.max_num_dpp,
-			mode_lib->ip.max_num_opp,
 			((double)display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].timing.pixel_clock_khz / 1000),
 			mode_lib->ms.support.NumberOfDSCSlices[k],
 
@@ -8680,12 +8645,25 @@ static bool dml_core_mode_support(struct dml2_core_calcs_mode_support_ex *in_out
 #endif
 	}
 
+	// TotalNumberOfActiveDPP is the sum of all planes
 	mode_lib->ms.TotalNumberOfActiveDPP = 0;
-	mode_lib->ms.TotalNumberOfActiveOPP = 0;
 	for (k = 0; k < mode_lib->ms.num_active_planes; ++k) {
 		mode_lib->ms.TotalNumberOfActiveDPP += mode_lib->ms.NoOfDPP[k];
-		mode_lib->ms.TotalNumberOfActiveOPP += mode_lib->ms.NoOfOPP[k];
 	}
+
+	// TotalNumberOfActiveOPP is the sum of the per stream max of all planes
+	mode_lib->ms.TotalNumberOfActiveOPP = 0;
+	for (k = 0; k < display_cfg->num_streams; ++k) {
+		unsigned int NoOfOppPerStream = 0;
+		for (m = 0; m < display_cfg->num_planes; ++m) {
+			if (display_cfg->plane_descriptors[m].stream_index == k) {
+				NoOfOppPerStream = NoOfOppPerStream < mode_lib->ms.NoOfOPP[m] ? mode_lib->ms.NoOfOPP[m] : NoOfOppPerStream;
+			}
+		}
+
+		mode_lib->ms.TotalNumberOfActiveOPP += NoOfOppPerStream;
+	}
+
 	if (mode_lib->ms.TotalNumberOfActiveDPP > (unsigned int)mode_lib->ip.max_num_dpp)
 		mode_lib->ms.support.TotalAvailablePipesSupport = false;
 	if (mode_lib->ms.TotalNumberOfActiveOPP > (unsigned int)mode_lib->ip.max_num_opp)
@@ -11676,7 +11654,6 @@ static bool dml_core_mode_programming(struct dml2_core_calcs_mode_programming_ex
 					display_cfg->gpuvm_enable,
 					mode_lib->mp.vm_bytes[k],
 					mode_lib->mp.PixelPTEBytesPerRow[k],
-					mode_lib->mp.BandwidthAvailableForImmediateFlip,
 					mode_lib->mp.TotImmediateFlipBytes,
 					display_cfg->plane_descriptors[k].pixel_format,
 					display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].timing.h_total / ((double)display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].timing.pixel_clock_khz / 1000),
@@ -11688,7 +11665,6 @@ static bool dml_core_mode_programming(struct dml2_core_calcs_mode_programming_ex
 					mode_lib->mp.use_one_row_for_frame_flip[k],
 					mode_lib->ip.max_flip_time_us,
 					mode_lib->ip.max_flip_time_lines,
-					s->per_pipe_flip_bytes[k],
 					mode_lib->mp.meta_row_bytes[k],
 					mode_lib->mp.meta_row_height[k],
 					mode_lib->mp.meta_row_height_chroma[k],
@@ -11883,6 +11859,7 @@ static bool dml_core_mode_programming(struct dml2_core_calcs_mode_programming_ex
 		CalculateWatermarks_params->MaxActiveFCLKChangeLatencySupported = &mode_lib->mp.MaxActiveFCLKChangeLatencySupported;
 		CalculateWatermarks_params->USRRetrainingSupport = &mode_lib->mp.USRRetrainingSupport;
 		CalculateWatermarks_params->g6_temp_read_support = &mode_lib->mp.g6_temp_read_support;
+		CalculateWatermarks_params->global_z8_stutter_supported = &mode_lib->mp.global_z8_stutter_supported;
 		CalculateWatermarks_params->VActiveLatencyHidingMargin = 0;
 		CalculateWatermarks_params->VActiveLatencyHidingUs = 0;
 
@@ -12909,6 +12886,11 @@ void dml2_core_calcs_get_mcif_arb_params(const struct dml2_core_internal_display
 	out->wm_regs[0].temp_read_or_ppt = (unsigned int)(mode_lib->mp.Watermark.writeback_temp_read_or_ppt_watermark_us * 1000.0);
 }
 
+void dml2_core_calcs_get_z8_stutter_support(const struct dml2_core_internal_display_mode_lib *mode_lib, bool *out)
+{
+	*out = mode_lib->mp.global_z8_stutter_supported;
+}
+
 void dml2_core_calcs_get_pipe_regs(const struct dml2_display_cfg *display_cfg,
 	struct dml2_core_internal_display_mode_lib *mode_lib,
 	struct dml2_dchub_per_pipe_register_set *out, int pipe_index)
@@ -13230,6 +13212,10 @@ void dml2_core_calcs_get_informative(const struct dml2_core_internal_display_mod
 	out->informative.mode_support_info.InvalidCombinationOfMALLUseForPState = mode_lib->ms.support.InvalidCombinationOfMALLUseForPState;
 	out->informative.mode_support_info.ExceededMALLSize = mode_lib->ms.support.ExceededMALLSize;
 	out->informative.mode_support_info.EnoughWritebackUnits = mode_lib->ms.support.EnoughWritebackUnits;
+	out->informative.mode_support_info.EnoughUrgentLatencyHidingSupport = mode_lib->ms.support.EnoughUrgentLatencyHidingSupport;
+	out->informative.mode_support_info.alternate_channel_size_support = mode_lib->ms.support.alternate_channel_size_support;
+	out->informative.mode_support_info.DSCSlicesODMModeSupported = mode_lib->ms.support.DSCSlicesODMModeSupported;
+	out->informative.mode_support_info.ODMSupport = mode_lib->ms.support.ODMSupport;
 	out->informative.mode_support_info.temp_read_or_ppt_support = mode_lib->ms.support.global_temp_read_or_ppt_supported;
 	out->informative.mode_support_info.g6_temp_read_support = mode_lib->ms.support.g6_temp_read_support;
 
@@ -13374,6 +13360,8 @@ void dml2_core_calcs_get_informative(const struct dml2_core_internal_display_mod
 	out->informative.qos.urg_bw_available.svp_prefetch.dram_bw_mbps = dml_get_svp_prefetch_urg_bw_available_dram(mode_lib);
 	out->informative.qos.urg_bw_available.svp_prefetch.dram_vm_only_bw_mbps = dml_get_svp_prefetch_urg_bw_available_dram_vm_only(mode_lib);
 
+	out->informative.qos.non_urg_bw_available.sys_active.sdp_bw_mbps = dml_get_sys_active_non_urg_bw_available_sdp(mode_lib);
+
 	out->informative.qos.urg_bw_required.sys_active.sdp_bw_mbps = dml_get_sys_active_urg_bw_required_sdp(mode_lib);
 	out->informative.qos.urg_bw_required.sys_active.dram_bw_mbps = dml_get_sys_active_urg_bw_required_dram(mode_lib);
 	out->informative.qos.urg_bw_required.svp_prefetch.sdp_bw_mbps = dml_get_svp_prefetch_urg_bw_required_sdp(mode_lib);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.h
index 3249f6bcf7bc3e..e0e397b14bb4f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn4_calcs.h
@@ -20,6 +20,7 @@ unsigned int dml2_core_calcs_mode_support_ex(struct dml2_core_calcs_mode_support
 bool dml2_core_calcs_mode_programming_ex(struct dml2_core_calcs_mode_programming_ex *in_out_params);
 void dml2_core_calcs_get_watermarks(const struct dml2_display_cfg *display_cfg, const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_dchub_watermark_regs *out);
 void dml2_core_calcs_get_mcif_arb_params(const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_mcif_global_register_set *out);
+void dml2_core_calcs_get_z8_stutter_support(const struct dml2_core_internal_display_mode_lib *mode_lib, bool *out);
 void dml2_core_calcs_get_arb_params(const struct dml2_display_cfg *display_cfg, const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_display_arb_regs *out);
 void dml2_core_calcs_get_pipe_regs(const struct dml2_display_cfg *dml2_display_cfg, struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_dchub_per_pipe_register_set *out, int pipe_index);
 void dml2_core_calcs_get_stream_programming(const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_per_stream_programming *out, int pipe_index);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c
index 38ccf9dab31f67..f407afefb37fe0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.c
@@ -1811,6 +1811,8 @@ void dcn5_calculate_urgent_burst_factor(
 		double VRatioC,
 		double BytePerPixelInDETY,
 		double BytePerPixelInDETC,
+		bool UnboundedRequestEnabled,
+		unsigned int CompressedBufferSizeInkByte,
 		unsigned int DETBufferSizeY,
 		unsigned int DETBufferSizeC,
 		// Output
@@ -1837,7 +1839,9 @@ void dcn5_calculate_urgent_burst_factor(
 	DML_LOG_VERBOSE("DML::%s: LineTime = %f\n", __func__, LineTime);
 	DML_ASSERT(VRatio > 0);
 
-	LinesInDETLuma = DETBufferSizeY / BytePerPixelInDETY / swath_width_luma_ub;
+	unsigned int EffectiveBufferY = DETBufferSizeY + (UnboundedRequestEnabled ? CompressedBufferSizeInkByte * 1024 : 0);
+
+	LinesInDETLuma = EffectiveBufferY / BytePerPixelInDETY / swath_width_luma_ub;
 
 	DETBufferSizeInTimeLuma = math_floor2(LinesInDETLuma, SwathHeightY) * LineTime / VRatio;
 	if (DETBufferSizeInTimeLuma - UrgentLatency <= 0) {
@@ -2419,7 +2423,7 @@ void dcn5_calculate_tdlut_setting(
 	else
 		tdlut_footprint_bytes = tdlut_pitch_bytes;
 
-	if (!p->gpuvm_enable) {
+	if (!p->gpuvm_enable || vmpg_bytes == 0) {
 		tdlut_vmpg_per_frame = 0;
 		tdlut_pte_req_per_frame = 0;
 	} else {
@@ -2632,7 +2636,7 @@ double dcn5_calculate_t_wait(
 	return TWait;
 }
 
-static void dcn5_calculate_v_update_and_dynamic_metadata_parameters(
+void dcn5_calculate_v_update_and_dynamic_metadata_parameters(
 		unsigned int MaxInterDCNTileRepeaters,
 		double Dppclk,
 		double Dispclk,
@@ -4102,10 +4106,10 @@ void dcn5_calculate_flip_schedule(
 					DML_LOG_VERBOSE("DML::%s: Tvm_trips_flip_rounded + 2*Tr0_trips_flip_rounded = %f\n", __func__, (Tvm_trips_flip_rounded + 2 * Tr0_trips_flip_rounded));
 				}
 				l->lb_flip_bw = math_max3(l->lb_flip_bw,
-						l->hvm_scaled_vm_bytes / (31 * LineTime) - Tno_bw_flip,
+						l->hvm_scaled_vm_bytes / ((31 * LineTime) - Tno_bw_flip),
 						(l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / (15 * LineTime));
 
-				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm reg limit = %f\n", __func__, l->hvm_scaled_vm_bytes / (31 * LineTime) - Tno_bw_flip);
+				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm reg limit = %f\n", __func__, l->hvm_scaled_vm_bytes / ((31 * LineTime) - Tno_bw_flip));
 				DML_LOG_VERBOSE("DML::%s: lb_flip_bw for row reg limit = %f\n", __func__, (l->dpte_row_bytes * HostVMInefficiencyFactor + meta_row_bytes) / (15 * LineTime));
 			}
 
@@ -4225,9 +4229,9 @@ static double dcn5_calculate_writeback_latency_hiding_us(
 		unsigned int dwb_index)
 {
 	double byte_per_pixel_luma_in_buffer = 1.0;
-	double buffer_for_luma_bytes = (double)writeback_buffer_size_bytes * 1024.0;
+	double buffer_for_luma_bytes = (double)writeback_buffer_size_bytes;
 	double line_time_us = (double)display_cfg->stream_descriptors[stream_index].timing.h_total /
-			(double)display_cfg->stream_descriptors[stream_index].timing.pixel_clock_khz / 1000.0;
+			((double)display_cfg->stream_descriptors[stream_index].timing.pixel_clock_khz / 1000.0);
 
 	if (display_cfg->stream_descriptors[stream_index].writeback.writeback_stream[dwb_index].pixel_format == dml2_444_64) {
 		byte_per_pixel_luma_in_buffer = 8.0;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.h
index d4869571db9df5..18a2268a255f28 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_dchub.h
@@ -103,6 +103,8 @@ void dcn5_calculate_urgent_burst_factor(
 		double VRatioC,
 		double BytePerPixelInDETY,
 		double BytePerPixelInDETC,
+		bool UnboundedRequestEnabled,
+		unsigned int CompressedBufferSizeInkByte,
 		unsigned int DETBufferSizeY,
 		unsigned int DETBufferSizeC,
 		// Output
@@ -423,4 +425,25 @@ void dcn5_get_per_dwb_params(const struct dml2_display_cfg *display_cfg,
 		int stream_index,
 		int dwb_index);
 
+void dcn5_calculate_v_update_and_dynamic_metadata_parameters(
+		unsigned int MaxInterDCNTileRepeaters,
+		double Dppclk,
+		double Dispclk,
+		double DCFClkDeepSleep,
+		double PixelClock,
+		unsigned int HTotal,
+		unsigned int VBlank,
+		unsigned int DynamicMetadataTransmittedBytes,
+		unsigned int DynamicMetadataLinesBeforeActiveRequired,
+		unsigned int InterlaceEnable,
+		bool ProgressiveToInterlaceUnitInOPP,
+
+		// Output
+		double *TSetup,
+		double *Tdmbf,
+		double *Tdmec,
+		double *Tdmsks,
+		unsigned int *VUpdateOffsetPix,
+		unsigned int *VUpdateWidthPix,
+		unsigned int *VReadyOffsetPix);
 #endif /* __DML2_CORE_DCN5_CALCS_DCHUB_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_display_pipe.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_display_pipe.c
index 83098c1fa50b75..d5ceb5d803f35c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_display_pipe.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_calcs_display_pipe.c
@@ -648,7 +648,7 @@ void dcn5_calculate_odm_mode(
 	*ODMMode = DecidedODMMode;
 	*TotalAvailablePipesSupport = success;
 	*NumberOfDPP = NumberOfDPPRequired;
-	*RequiredDISPCLKPerSurface = success ? DISPCLKRequired : 0;
+	*RequiredDISPCLKPerSurface = DISPCLKRequired;
 #ifdef __DML_VBA_DEBUG__
 	DML_LOG_VERBOSE("DML::%s: ODMMode = %d\n", __func__, *ODMMode);
 	DML_LOG_VERBOSE("DML::%s: NumberOfDPP = %d\n", __func__, *NumberOfDPP);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_initialize.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_initialize.c
index 31681c7f655e96..5907cc155505a7 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_initialize.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_initialize.c
@@ -33,7 +33,7 @@ static void patch_ip_caps_with_explicit_ip_params(struct dml2_ip_capabilities *i
 static void patch_ip_params_with_ip_caps(struct dml2_core_ip_params *ip_params, const struct dml2_ip_capabilities *ip_caps)
 {
 	ip_params->max_num_dpp = ip_caps->pipe_count;
-	ip_params->max_num_opp = ip_caps->pipe_count;
+	ip_params->max_num_opp = ip_caps->otg_count;
 	ip_params->max_num_otg = ip_caps->otg_count;
 	ip_params->num_dsc = ip_caps->num_dsc;
 	ip_params->max_num_dp2p0_streams = ip_caps->max_num_dp2p0_streams;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_programming.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_programming.c
index 297e21e7c68dad..d7a48ca44b68c2 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_programming.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_programming.c
@@ -592,6 +592,8 @@ static bool dcn5_mode_programming(struct dml2_core_calcs_mode_programming_ex *in
 			display_cfg->plane_descriptors[k].composition.scaler_info.plane1.v_ratio,
 			mode_lib->mp.BytePerPixelInDETY[k],
 			mode_lib->mp.BytePerPixelInDETC[k],
+			mode_lib->mp.UnboundedRequestEnabled,
+			mode_lib->mp.CompressedBufferSizeInkByte,
 			mode_lib->mp.DETBufferSizeY[k],
 			mode_lib->mp.DETBufferSizeC[k],
 
@@ -801,6 +803,8 @@ static bool dcn5_mode_programming(struct dml2_core_calcs_mode_programming_ex *in
 					mode_lib->mp.VRatioPrefetchC[k],
 					mode_lib->mp.BytePerPixelInDETY[k],
 					mode_lib->mp.BytePerPixelInDETC[k],
+					mode_lib->mp.UnboundedRequestEnabled,
+					mode_lib->mp.CompressedBufferSizeInkByte,
 					mode_lib->mp.DETBufferSizeY[k],
 					mode_lib->mp.DETBufferSizeC[k],
 					/* Output */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_support.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_support.c
index 9d68c9dd0a1de6..09519f0f225128 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_support.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn5_funcs_mode_support.c
@@ -795,22 +795,26 @@ static void dcn5_ms_calculate_num_of_dpp_required(
 		const struct dml2_display_cfg *display_cfg,
 		struct dml2_core_internal_display_mode_lib *mode_lib)
 {
-	unsigned int k;
+	unsigned int k, m;
 	const struct dml2_plane_parameters *plane;
 
 	for (k = 0; k < mode_lib->ms.num_active_planes; ++k) {
 		plane = &display_cfg->plane_descriptors[k];
 		mode_lib->ms.MPCCombine[k] = false;
 		mode_lib->ms.NoOfDPP[k] = 1;
+		mode_lib->ms.NoOfOPP[k] = 1;
 		if (mode_lib->ms.ODMMode[k] == dml2_odm_mode_combine_4to1) {
 			mode_lib->ms.MPCCombine[k] = false;
 			mode_lib->ms.NoOfDPP[k] = 4;
+			mode_lib->ms.NoOfOPP[k] = 4;
 		} else if (mode_lib->ms.ODMMode[k] == dml2_odm_mode_combine_3to1) {
 			mode_lib->ms.MPCCombine[k] = false;
 			mode_lib->ms.NoOfDPP[k] = 3;
+			mode_lib->ms.NoOfOPP[k] = 3;
 		} else if (mode_lib->ms.ODMMode[k] == dml2_odm_mode_combine_2to1) {
 			mode_lib->ms.MPCCombine[k] = false;
 			mode_lib->ms.NoOfDPP[k] = 2;
+			mode_lib->ms.NoOfOPP[k] = 2;
 		} else if (plane->overrides.mpcc_combine_factor == 2) {
 			mode_lib->ms.MPCCombine[k] = true;
 			mode_lib->ms.NoOfDPP[k] = 2;
@@ -832,12 +836,27 @@ static void dcn5_ms_calculate_num_of_dpp_required(
 		DML_LOG_VERBOSE("DML::%s: k=%d, NoOfDPP = %d\n", __func__, k,
 				mode_lib->ms.NoOfDPP[k]);
 	}
+
+
+	// TotalNumberOfActiveOPP is the sum of the per stream max NoOfOPP of all planes driving that stream
+	mode_lib->ms.TotalNumberOfActiveOPP = 0;
+	for (k = 0; k < display_cfg->num_streams; ++k) {
+		unsigned int NoOfOppPerStream = 0;
+
+		for (m = 0; m < mode_lib->ms.num_active_planes; ++m) {
+			if (display_cfg->plane_descriptors[m].stream_index == k)
+				NoOfOppPerStream = NoOfOppPerStream < mode_lib->ms.NoOfOPP[m] ? mode_lib->ms.NoOfOPP[m] : NoOfOppPerStream;
+		}
+
+		mode_lib->ms.TotalNumberOfActiveOPP += NoOfOppPerStream;
+	}
 }
 
 static bool dcn5_ms_check_total_available_pipes_support(
 		struct dml2_core_internal_display_mode_lib *mode_lib)
 {
-	return mode_lib->ms.TotalNumberOfActiveDPP <= (unsigned int) mode_lib->ip.max_num_dpp;
+	return mode_lib->ms.TotalNumberOfActiveDPP <= (unsigned int) mode_lib->ip.max_num_dpp
+			&& mode_lib->ms.TotalNumberOfActiveOPP <= (unsigned int) mode_lib->ip.max_num_opp;
 }
 
 static void dcn5_ms_calculate_total_num_of_single_dpp_surfaces(
@@ -973,11 +992,6 @@ static bool dcn5_ms_check_writeback_count_support(
 	for (stream_idx = 0; stream_idx < display_cfg->num_streams; stream_idx++) {
 		totalNumberOfActiveWriteback +=
 				display_cfg->stream_descriptors[stream_idx].writeback.active_writebacks_per_stream;
-
-		/* >1 writeback per stream is currently not supported */
-		if (display_cfg->stream_descriptors[stream_idx].writeback.active_writebacks_per_stream > 1) {
-			writeback_per_stream_supported = false;
-		}
 	}
 
 	return writeback_per_stream_supported &&
@@ -1533,6 +1547,8 @@ static void dcn5_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urge
 				plane->composition.scaler_info.plane1.v_ratio,
 				mode_lib->ms.BytePerPixelInDETY[k],
 				mode_lib->ms.BytePerPixelInDETC[k],
+				mode_lib->ms.UnboundedRequestEnabled,
+				mode_lib->ms.CompressedBufferSizeInkByte,
 				mode_lib->ms.DETBufferSizeY[k],
 				mode_lib->ms.DETBufferSizeC[k],
 
@@ -2081,6 +2097,8 @@ static void dcn5_ms_calculate_urgent_burst_factor_for_prefetch(
 				mode_lib->ms.VRatioPreC[k],
 				mode_lib->ms.BytePerPixelInDETY[k],
 				mode_lib->ms.BytePerPixelInDETC[k],
+				mode_lib->ms.UnboundedRequestEnabled,
+				mode_lib->ms.CompressedBufferSizeInkByte,
 				mode_lib->ms.DETBufferSizeY[k],
 				mode_lib->ms.DETBufferSizeC[k],
 				/* Output */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.c
index c86e9c5feec40e..a20eaaf9f1e3d4 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.c
@@ -3,9 +3,64 @@
 // Copyright 2026 Advanced Micro Devices, Inc.
 
 #include "dml2_core_dcn6_calcs.h"
+#include "dml2_core_dcn6_calcs_dchub.h"
+#include "dml2_core_dcn5_calcs_dchub.h"
+#include "dml2_core_dcn5_calcs_display_pipe.h"
 
-/*
- * Placeholder for future DCN6 calcs table implementation.
- * Remove this typedef once real declarations are added here.
+static const struct dml2_core_dcn6_calcs dcn6_calcs_funcs = {
+	.calculate_max_vstartup = dcn6_calculate_max_vstartup,
+	.calculate_alternate_params = dcn6_calculate_alternate_params,
+	.calculate_alternate_svp_lines = dcn6_calculate_alternate_svp_lines,
+	.calculate_flip_schedule = dcn6_calculate_flip_schedule,
+	.get_pipe_regs = dcn6_get_pipe_regs,
+	.calculate_watermarks_and_dram_speed_change_support = dcn6_calculate_watermarks_and_dram_speed_change_support,
+	.calculate_stutter_efficiency = dcn6_calculate_stutter_efficiency,
+	.get_watermarks = dcn6_get_watermarks,
+	.calculate_excess_vactive_bandwidth_required = dcn6_calculate_excess_vactive_bandwidth_required,
+	.calculate_pstate_schedule_windows = dcn6_calculate_pstate_schedule_windows,
+	.calculate_pstate_schedule_admissibility = dcn6_calculate_pstate_schedule_admissibility,
+
+	.calculate_max_det_and_min_compressed_buffer_size = dcn5_calculate_max_det_and_min_compressed_buffer_size,
+	.adjust_pixel_clock_for_progressive_to_interlace_unit = dcn5_adjust_pixel_clock_for_progressive_to_interlace_unit,
+	.calculate_byte_per_pixel_and_block_sizes = dcn5_calculate_byte_per_pixel_and_block_sizes,
+	.calculate_single_pipe_dppclk_and_scl_throughput = dcn5_calculate_single_pipe_dppclk_and_scl_throughput,
+	.calculate_swath_and_det_configuration = dcn5_calculate_swath_and_det_configuration,
+	.calculate_output_link = dcn5_calculate_output_link,
+	.calculate_odm_mode = dcn5_calculate_odm_mode,
+	.calculate_write_back_dispclk = dcn5_calculate_write_back_dispclk,
+	.calculate_required_dtbclk = dcn5_calculate_required_dtbclk,
+	.calculate_dsc_delay_requirement = dcn5_calculate_dsc_delay_requirement,
+	.calculate_vm_row_and_swath = dcn5_calculate_vm_row_and_swath,
+	.calculate_bytes_to_fetch_required_to_hide_latency = dcn5_calculate_bytes_to_fetch_required_to_hide_latency,
+	.calculate_cursor_req_attributes = dcn5_calculate_cursor_req_attributes,
+	.calculate_cursor_urgent_burst_factor = dcn5_calculate_cursor_urgent_burst_factor,
+	.calculate_urgent_burst_factor = dcn5_calculate_urgent_burst_factor,
+	.calculate_dcfclk_deep_sleep = dcn5_calculate_dcfclk_deep_sleep,
+	.calculate_write_back_delay = dcn5_calculate_write_back_delay,
+	.calculate_mcache_setting = dcn5_calculate_mcache_setting,
+	.calculate_avg_bandwidth_required = dcn5_calculate_avg_bandwidth_required,
+	.calculate_hostvm_inefficiency_factor = dcn5_calculate_hostvm_inefficiency_factor,
+	.calculate_tdlut_setting = dcn5_calculate_tdlut_setting,
+	.calculate_extra_latency = dcn5_calculate_extra_latency,
+	.calculate_t_wait = dcn5_calculate_t_wait,
+	.calculate_prefetch_schedule = dcn5_calculate_prefetch_schedule,
+	.calculate_peak_bandwidth_required = dcn5_calculate_peak_bandwidth_required,
+	.calculate_vactive_det_fill_latency = dcn5_calculate_vactive_det_fill_latency,
+	.calculate_dcc_configuration = dcn5_calculate_dcc_configuration,
+	.calculate_pixel_delivery_times = dcn5_calculate_pixel_delivery_times,
+	.calculate_meta_and_pte_times = dcn5_calculate_meta_and_pte_times,
+	.calculate_vm_group_and_request_times = dcn5_calculate_vm_group_and_request_times,
+	.calculate_pstate_keepout_dst_lines = dcn5_calculate_pstate_keepout_dst_lines,
+	.get_arb_params = dcn5_get_arb_params,
+};
+
+/**
+ * dml2_core_dcn6_calcs_init() - Register the DCN6 calcs function pointer table.
+ * @calcs: calcs union to populate.
+ *
+ * Return: void
  */
-typedef int dml2_core_dcn6_calcs_placeholder;
+void dml2_core_dcn6_calcs_init(union dml2_core_calcs *calcs)
+{
+	calcs->dcn6 = &dcn6_calcs_funcs;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.h
index 35f7d8a050a790..87252b658c1629 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs.h
@@ -4,5 +4,473 @@
 
 #ifndef __DML2_CORE_DCN6_CALCS_H__
 #define __DML2_CORE_DCN6_CALCS_H__
+#include "dml2_internal_shared_types.h"
+
+/*
+ * Flat table of every calcs function reachable from the DCN6 funcs layer
+ * (both DCN6-specific calcs and DCN5 calcs reused unmodified by DCN6).
+ * Slot names carry no generation prefix, only the assigned symbols do.
+ */
+struct dml2_core_dcn6_calcs {
+	unsigned int (*calculate_max_vstartup)(
+			bool ptoi_supported,
+			unsigned int vblank_nom_default_us,
+			const struct dml2_timing_cfg *timing,
+			enum dml2_uclk_pstate_change_strategy pstate_strategy,
+			double write_back_delay_us,
+			unsigned int svp_lines);
+	void (*calculate_alternate_params)(struct dml2_core_calcs_calculate_alternate_params *p);
+	void (*calculate_alternate_svp_lines)(struct dml2_core_calcs_calculate_alternate_svp_lines *p);
+	void (*calculate_flip_schedule)(
+			struct dml2_core_internal_scratch *s,
+			bool iflip_enable,
+			bool ihostvm_enable,
+			bool iffbm_enable,
+			double HostVMInefficiencyFactor,
+			double Tvm_trips_flip,
+			double Tr0_trips_flip,
+			double Tvm_trips_flip_rounded,
+			double Tr0_trips_flip_rounded,
+			bool GPUVMEnable,
+			double vm_bytes,
+			double DPTEBytesPerRow,
+			enum dml2_source_format_class SourcePixelFormat,
+			double LineTime,
+			double VRatio,
+			double VRatioChroma,
+			double Tno_bw_flip,
+			unsigned int dpte_row_height,
+			unsigned int dpte_row_height_chroma,
+			unsigned int max_flip_time_us,
+			unsigned int max_flip_time_lines,
+			unsigned int meta_row_height,
+			unsigned int meta_row_height_chroma,
+
+			// Output
+			double *dst_y_per_vm_flip,
+			double *dst_y_per_row_flip,
+			double *final_flip_bw,
+			bool *ImmediateFlipSupportedForPipe);
+	void (*get_pipe_regs)(const struct dml2_display_cfg *display_cfg,
+			const struct dml2_core_internal_display_mode_lib *mode_lib,
+			struct dml2_dchub_per_pipe_register_set *out, int pipe_index, const struct dml2_utm_soc_bb *utm_soc_bb,
+			struct dml2_core_internal_scratch *s);
+	void (*calculate_watermarks_and_dram_speed_change_support)(
+			struct dml2_core_internal_scratch *scratch,
+			struct dml2_core_calcs_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport_params *p);
+	void (*calculate_stutter_efficiency)(struct dml2_core_internal_scratch *scratch,
+			struct dml2_core_calcs_CalculateStutterEfficiency_params *p);
+	void (*get_watermarks)(const struct dml2_display_cfg *display_cfg, const struct dml2_core_internal_display_mode_lib *mode_lib,
+			const struct dml2_utm_soc_bb *utm_soc_bb, struct dml2_dchub_watermark_regs *out);
+	void (*calculate_excess_vactive_bandwidth_required)(
+			const struct dml2_display_cfg *display_cfg,
+			unsigned int bytes_required_l[dml2_pstate_type_count][DML2_MAX_PLANES],
+			unsigned int bytes_required_c[dml2_pstate_type_count][DML2_MAX_PLANES],
+			/* outputs */
+			double excess_vactive_fill_bw_l[],
+			double excess_vactive_fill_bw_c[]);
+	void (*calculate_pstate_schedule_windows)(
+			int num_active_planes,
+			const unsigned int v_blank_start[DML2_MAX_PLANES],
+			const unsigned int v_blank_end[DML2_MAX_PLANES],
+			const double otg_vline_time_us[DML2_MAX_PLANES],
+			const double det_fill_delay_us[DML2_MAX_PLANES],
+			const double reserved_vblank_us[DML2_MAX_PLANES],
+			const double blackout_us,
+			// Outputs
+			double allow_start_us[DML2_MAX_PLANES],
+			double allow_end_us[DML2_MAX_PLANES]);
+	void (*calculate_pstate_schedule_admissibility)(
+			uint32_t num_active_planes,
+			double max_allow_delay_us,
+			double min_allow_width_us,
+			const uint32_t timing_group_id[DML2_MAX_PLANES],
+			uint32_t timing_group_count,
+			const double frame_time_us[DML2_MAX_PLANES],
+			const double allow_start_us[DML2_MAX_PLANES],
+			const double allow_end_us[DML2_MAX_PLANES],
+			const enum dml2_pstate_method pstate_method[DML2_MAX_PLANES],
+			const bool drr_enabled[DML2_MAX_DCN_PIPES],
+			// Output
+			double allow_window_us[DML2_MAX_DCN_PIPES],
+			double disallow_window_us[DML2_MAX_DCN_PIPES],
+			bool *pstate_admissible);
+
+	/* DCN5 calcs reused unmodified by DCN6 */
+	void (*calculate_max_det_and_min_compressed_buffer_size)(
+			unsigned int ConfigReturnBufferSizeInKByte,
+			unsigned int ConfigReturnBufferSegmentSizeInKByte,
+			unsigned int ROBBufferSizeInKByte,
+			unsigned int MaxNumDPP,
+			unsigned int nomDETInKByteOverrideEnable,
+			unsigned int nomDETInKByteOverrideValue,
+			bool is_mrq_present,
+
+			// Output
+			unsigned int *MaxTotalDETInKByte,
+			unsigned int *nomDETInKByte,
+			unsigned int *MinCompressedBufferSizeInKByte);
+	void (*adjust_pixel_clock_for_progressive_to_interlace_unit)(const struct dml2_display_cfg *display_cfg,
+			bool ptoi_supported, double *PixelClockBackEnd);
+	void (*calculate_byte_per_pixel_and_block_sizes)(
+			enum dml2_source_format_class SourcePixelFormat,
+			enum dml2_swizzle_mode SurfaceTiling,
+			unsigned int pitch_y,
+			unsigned int pitch_c,
+
+			// Output
+			unsigned int *BytePerPixelY,
+			unsigned int *BytePerPixelC,
+			double *BytePerPixelDETY,
+			double *BytePerPixelDETC,
+			unsigned int *BlockHeight256BytesY,
+			unsigned int *BlockHeight256BytesC,
+			unsigned int *BlockWidth256BytesY,
+			unsigned int *BlockWidth256BytesC,
+			unsigned int *MacroTileHeightY,
+			unsigned int *MacroTileHeightC,
+			unsigned int *MacroTileWidthY,
+			unsigned int *MacroTileWidthC,
+			bool *surf_linear128_l,
+			bool *surf_linear128_c);
+	void (*calculate_single_pipe_dppclk_and_scl_throughput)(
+			double HRatio,
+			double HRatioChroma,
+			double VRatio,
+			double VRatioChroma,
+			double MaxDCHUBToPSCLThroughput,
+			double MaxPSCLToLBThroughput,
+			double PixelClock,
+			enum dml2_source_format_class SourcePixelFormat,
+			unsigned int HTaps,
+			unsigned int HTapsChroma,
+			unsigned int VTaps,
+			unsigned int VTapsChroma,
+
+			// Output
+			double *PSCL_THROUGHPUT,
+			double *PSCL_THROUGHPUT_CHROMA,
+			double *DPPCLKUsingSingleDPP);
+	void (*calculate_swath_and_det_configuration)(struct dml2_core_internal_scratch *scratch,
+			struct dml2_core_calcs_CalculateSwathAndDETConfiguration_params *p);
+	void (*calculate_output_link)(
+			struct dml2_core_internal_scratch *s,
+			double PHYCLK,
+			double PHYCLKD18,
+			double PHYCLKD32,
+			double Downspreading,
+			enum dml2_output_encoder_class Output,
+			enum dml2_output_format_class OutputFormat,
+			unsigned int HTotal,
+			unsigned int HActive,
+			double PixelClockBackEnd,
+			double ForcedOutputLinkBPP,
+			unsigned int DSCInputBitPerComponent,
+			unsigned int NumberOfDSCSlices,
+			double AudioSampleRate,
+			unsigned int AudioSampleLayout,
+			enum dml2_odm_mode ODMModeNoDSC,
+			enum dml2_odm_mode ODMModeDSC,
+			enum dml2_dsc_enable_option DSCEnable,
+			unsigned int OutputLinkDPLanes,
+			enum dml2_output_link_dp_rate OutputLinkDPRate,
+
+			// Output
+			bool *RequiresDSC,
+			bool *RequiresFEC,
+			double *OutBpp,
+			enum dml2_core_internal_output_type *OutputType,
+			enum dml2_core_internal_output_type_rate *OutputRate,
+			unsigned int *RequiredSlots);
+	void (*calculate_odm_mode)(
+			unsigned int MaximumPixelsPerLinePerDSCUnit,
+			unsigned int HActive,
+			enum dml2_output_format_class OutFormat,
+			enum dml2_output_encoder_class Output,
+			enum dml2_odm_mode ODMUse,
+			double MaxDispclk,
+			bool DSCEnable,
+			unsigned int TotalNumberOfActiveDPP,
+			unsigned int MaxNumDPP,
+			double PixelClock,
+			unsigned int MaximumSlicesPerDSCUnit,
+			unsigned int NumberOfDSCSlices,
+			unsigned int odm_combine_support_mask,
+
+			// Output
+			bool *TotalAvailablePipesSupport,
+			unsigned int *NumberOfDPP,
+			enum dml2_odm_mode *ODMMode,
+			double *RequiredDISPCLKPerSurface);
+	double (*calculate_write_back_dispclk)(
+			enum dml2_source_format_class WritebackPixelFormat,
+			double PixelClock,
+			enum dml2_odm_mode ODMMode,
+			double WritebackHRatio,
+			double WritebackVRatio,
+			unsigned int WritebackHTaps,
+			unsigned int WritebackVTaps,
+			unsigned int WritebackHTapsChroma,
+			unsigned int WritebackVTapsChroma,
+			unsigned int WritebackSourceWidth,
+			unsigned int WritebackDestinationWidth,
+			unsigned int HTotal,
+			unsigned int WritebackLineBufferSize);
+	double (*calculate_required_dtbclk)(
+			bool DSCEnable,
+			double PixelClock,
+			enum dml2_output_format_class OutputFormat,
+			double OutputBpp,
+			unsigned int DSCSlices,
+			unsigned int HTotal,
+			unsigned int HActive,
+			unsigned int AudioRate,
+			unsigned int AudioLayout);
+	unsigned int (*calculate_dsc_delay_requirement)(
+			bool DSCEnabled,
+			enum dml2_odm_mode ODMMode,
+			unsigned int DSCInputBitPerComponent,
+			double OutputBpp,
+			unsigned int HActive,
+			unsigned int HTotal,
+			unsigned int NumberOfDSCSlices,
+			enum dml2_output_format_class OutputFormat,
+			enum dml2_output_encoder_class Output,
+			double PixelClock,
+			double PixelClockBackEnd,
+			bool use_legacy_dsc_delay_formula);
+	void (*calculate_vm_row_and_swath)(struct dml2_core_internal_scratch *scratch,
+			struct dml2_core_calcs_CalculateVMRowAndSwath_params *p);
+	void (*calculate_bytes_to_fetch_required_to_hide_latency)(
+			struct dml2_core_calcs_calculate_bytes_to_fetch_required_to_hide_latency_params *p);
+	void (*calculate_cursor_req_attributes)(
+			unsigned int cursor_width,
+			unsigned int cursor_bpp,
+
+			// output
+			unsigned int *cursor_lines_per_chunk,
+			unsigned int *cursor_bytes_per_line,
+			unsigned int *cursor_bytes_per_chunk,
+			unsigned int *cursor_bytes);
+	void (*calculate_cursor_urgent_burst_factor)(
+			unsigned int CursorBufferSize,
+			unsigned int CursorWidth,
+			unsigned int cursor_bytes_per_chunk,
+			unsigned int cursor_lines_per_chunk,
+			double LineTime,
+			double UrgentLatency,
+
+			double *UrgentBurstFactorCursor,
+			bool *NotEnoughUrgentLatencyHiding);
+	void (*calculate_urgent_burst_factor)(
+			const struct dml2_plane_parameters *plane_cfg,
+			unsigned int swath_width_luma_ub,
+			unsigned int swath_width_chroma_ub,
+			unsigned int SwathHeightY,
+			unsigned int SwathHeightC,
+			double LineTime,
+			double UrgentLatency,
+			double VRatio,
+			double VRatioC,
+			double BytePerPixelInDETY,
+			double BytePerPixelInDETC,
+			bool UnboundedRequestEnabled,
+			unsigned int CompressedBufferSizeInkByte,
+			unsigned int DETBufferSizeY,
+			unsigned int DETBufferSizeC,
+			// Output
+			double *UrgentBurstFactorLuma,
+			double *UrgentBurstFactorChroma,
+			bool *NotEnoughUrgentLatencyHiding);
+	void (*calculate_dcfclk_deep_sleep)(
+			const struct dml2_display_cfg *display_cfg,
+			unsigned int NumberOfActiveSurfaces,
+			unsigned int BytePerPixelY[],
+			unsigned int BytePerPixelC[],
+			unsigned int SwathWidthY[],
+			unsigned int SwathWidthC[],
+			unsigned int DPPPerSurface[],
+			double PSCL_THROUGHPUT[],
+			double PSCL_THROUGHPUT_CHROMA[],
+			double Dppclk[],
+			double ReadBandwidthLuma[],
+			double ReadBandwidthChroma[],
+			unsigned int ReturnBusWidth,
+
+			// Output
+			double *DCFClkDeepSleep);
+	double (*calculate_write_back_delay)(
+			enum dml2_source_format_class WritebackPixelFormat,
+			double WritebackHRatio,
+			double WritebackVRatio,
+			unsigned int WritebackVTaps,
+			unsigned int WritebackVTapsChroma,
+			unsigned int WritebackDestinationWidth,
+			unsigned int WritebackDestinationHeight,
+			unsigned int WritebackSourceWidth,
+			unsigned int WritebackSourceHeight,
+			unsigned int HTotal);
+	void (*calculate_mcache_setting)(
+			struct dml2_core_internal_scratch *scratch,
+			struct dml2_core_calcs_calculate_mcache_setting_params *p);
+	void (*calculate_avg_bandwidth_required)(
+			double *avg_bandwidth_required,
+
+			// input
+			unsigned int num_active_planes,
+			double ReadBandwidthLuma[],
+			double ReadBandwidthChroma[],
+			double cursor_bw[],
+			double dcc_dram_bw_nom_overhead_factor_p0[],
+			double dcc_dram_bw_nom_overhead_factor_p1[]);
+	void (*calculate_hostvm_inefficiency_factor)(
+			double *HostVMInefficiencyFactor,
+			double *HostVMInefficiencyFactorPrefetch,
+
+			bool gpuvm_enable,
+			bool hostvm_enable,
+			unsigned int remote_iommu_outstanding_translations,
+			unsigned int max_outstanding_reqs,
+			double urg_bandwidth_avail_active_pixel_and_vm,
+			double urg_bandwidth_avail_active_vm_only);
+	void (*calculate_tdlut_setting)(
+			struct dml2_core_internal_scratch *scratch,
+			struct dml2_core_calcs_calculate_tdlut_setting_params *p);
+	void (*calculate_extra_latency)(
+			const struct dml2_display_cfg *display_cfg,
+			unsigned int ROBBufferSizeInKByte,
+			unsigned int RoundTripPingLatencyCycles,
+			unsigned int ReorderingBytes,
+			double DCFCLK,
+			double FabricClock,
+			unsigned int PixelChunkSizeInKByte,
+			double ReturnBW,
+			unsigned int NumberOfActiveSurfaces,
+			unsigned int NumberOfDPP[],
+			unsigned int dpte_group_bytes[],
+			unsigned int tdlut_bytes_per_group[],
+			double HostVMInefficiencyFactor,
+			double HostVMInefficiencyFactorPrefetch,
+			enum dml2_qos_param_type qos_type,
+			bool max_outstanding_when_urgent_expected,
+			unsigned int max_outstanding_requests,
+			unsigned int request_size_bytes_luma[],
+			unsigned int request_size_bytes_chroma[],
+			unsigned int MetaChunkSize,
+			unsigned int dchub_arb_to_ret_delay,
+			double Ttrip,
+			unsigned int hostvm_mode,
+
+			// output
+			double *ExtraLatency,
+			double *ExtraLatency_sr,
+			double *ExtraLatencyPrefetch);
+	double (*calculate_t_wait)(
+			long reserved_vblank_time_ns,
+			double UrgentLatency,
+			double Ttrip,
+			double temp_read_or_ppt_blackout_us,
+			bool drr_enabled);
+	bool (*calculate_prefetch_schedule)(struct dml2_core_internal_scratch *scratch, struct dml2_core_calcs_CalculatePrefetchSchedule_params *p);
+	void (*calculate_peak_bandwidth_required)(
+			struct dml2_core_internal_scratch *s,
+			struct dml2_core_calcs_calculate_peak_bandwidth_required_params *p);
+	void (*calculate_vactive_det_fill_latency)(
+			const struct dml2_display_cfg *display_cfg,
+			unsigned int num_active_planes,
+			unsigned int bytes_required_l[],
+			unsigned int bytes_required_c[],
+			double dcc_dram_bw_nom_overhead_factor_p0[],
+			double dcc_dram_bw_nom_overhead_factor_p1[],
+			double surface_read_bw_l[],
+			double surface_read_bw_c[],
+			double surface_avg_vactive_required_bw[],
+			double surface_peak_required_bw[],
+			/* output */
+			double vactive_det_fill_delay_us[]);
+	void (*calculate_dcc_configuration)(
+			bool DCCEnabled,
+			bool DCCProgrammingAssumesScanDirectionUnknown,
+			enum dml2_source_format_class SourcePixelFormat,
+			unsigned int SurfaceWidthLuma,
+			unsigned int SurfaceWidthChroma,
+			unsigned int SurfaceHeightLuma,
+			unsigned int SurfaceHeightChroma,
+			unsigned int nomDETInKByte,
+			unsigned int RequestHeight256ByteLuma,
+			unsigned int RequestHeight256ByteChroma,
+			enum dml2_swizzle_mode TilingFormat,
+			unsigned int BytePerPixelY,
+			unsigned int BytePerPixelC,
+			double BytePerPixelDETY,
+			double BytePerPixelDETC,
+			enum dml2_rotation_angle RotationAngle,
+
+			// Output
+			enum dml2_core_internal_request_type *RequestLuma,
+			enum dml2_core_internal_request_type *RequestChroma,
+			unsigned int *MaxUncompressedBlockLuma,
+			unsigned int *MaxUncompressedBlockChroma,
+			unsigned int *MaxCompressedBlockLuma,
+			unsigned int *MaxCompressedBlockChroma,
+			unsigned int *IndependentBlockLuma,
+			unsigned int *IndependentBlockChroma);
+	void (*calculate_pixel_delivery_times)(
+			const struct dml2_display_cfg *display_cfg,
+			unsigned int NoOfDPP[DML2_MAX_PLANES],
+			unsigned int NumberOfActiveSurfaces,
+			double VRatioPrefetchY[],
+			double VRatioPrefetchC[],
+			unsigned int swath_width_luma_ub[],
+			unsigned int swath_width_chroma_ub[],
+			double PSCL_THROUGHPUT[],
+			double PSCL_THROUGHPUT_CHROMA[],
+			double Dppclk[],
+			double DCFCLKDeepSleep,
+			unsigned int BytePerPixelY[],
+			unsigned int BytePerPixelC[],
+			unsigned int req_per_swath_ub_l[],
+			unsigned int req_per_swath_ub_c[],
+
+			// Output
+			double DisplayPipeLineDeliveryTimeLuma[],
+			double DisplayPipeLineDeliveryTimeChroma[],
+			double DisplayPipeLineDeliveryTimeLumaPrefetch[],
+			double DisplayPipeLineDeliveryTimeChromaPrefetch[],
+			double DisplayPipeRequestDeliveryTimeLuma[],
+			double DisplayPipeRequestDeliveryTimeChroma[],
+			double DisplayPipeRequestDeliveryTimeLumaPrefetch[],
+			double DisplayPipeRequestDeliveryTimeChromaPrefetch[]);
+	void (*calculate_meta_and_pte_times)(struct dml2_core_shared_CalculateMetaAndPTETimes_params *p);
+	void (*calculate_vm_group_and_request_times)(
+			const struct dml2_display_cfg *display_cfg,
+			unsigned int NumberOfActiveSurfaces,
+			unsigned int BytePerPixelC[],
+			double dst_y_per_vm_vblank[],
+			double dst_y_per_vm_flip[],
+			unsigned int dpte_row_width_luma_ub[],
+			unsigned int dpte_row_width_chroma_ub[],
+			unsigned int vm_group_bytes[],
+			unsigned int dpde0_bytes_per_frame_ub_l[],
+			unsigned int dpde0_bytes_per_frame_ub_c[],
+			unsigned int tdlut_pte_bytes_per_frame[],
+			unsigned int meta_pte_bytes_per_frame_ub_l[],
+			unsigned int meta_pte_bytes_per_frame_ub_c[],
+			bool mrq_present,
+
+			// Output
+			double TimePerVMGroupVBlank[],
+			double TimePerVMGroupFlip[],
+			double TimePerVMRequestVBlank[],
+			double TimePerVMRequestFlip[]);
+	void (*calculate_pstate_keepout_dst_lines)(
+			const struct dml2_display_cfg *display_cfg,
+			const struct dml2_core_internal_watermarks *watermarks,
+			unsigned int pstate_keepout_dst_lines[]);
+	void (*get_arb_params)(const struct dml2_display_cfg *display_cfg, const struct dml2_core_internal_display_mode_lib *mode_lib,
+			const struct dml2_utm_soc_bb *utm_soc_bb, struct dml2_display_arb_regs *out);
+};
+
+void dml2_core_dcn6_calcs_init(union dml2_core_calcs *calcs);
 
 #endif /* __DML2_CORE_DCN6_CALCS_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c
index a4e2f8604650fb..78bdf11d0d46c5 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_calcs_dchub.c
@@ -203,14 +203,32 @@ static unsigned int calculate_copy_swaths(double pre_first_hdl,
 	return num_swaths;
 }
 
+/**
+  * *******************************************************************************************************************************************************
+  * calculate_tile_aligned_size: Applies tiled-surface padding / alignment rules to an unaligned copy region
+  *
+  * Given the unaligned copy region dimensions (copy_width in pixels, copy_height in lines) and the plane's macro tile
+  * (swizzle block) dimensions, this returns the number of bytes reserved for the region following addrlib sizing rules:
+  * - The pitch is padded up to the macro tile width and the height is padded up to the macro tile height.
+  *   size = alignedPitch * alignedHeight * bpp, which is inherently macro-tile (block) aligned
+  * - The final ceil2 to 256 is a harmless floor for degenerate tile sizes.
+  *
+  * *******************************************************************************************************************************************************
+  */
+static unsigned int calculate_tile_aligned_size(unsigned int copy_width, unsigned int copy_height, unsigned int tile_width, unsigned int tile_height, unsigned int byte_per_pixel)
+{
+	unsigned int mem_width = (unsigned int)math_ceil2(copy_width, tile_width);
+	unsigned int mem_height = (unsigned int)math_ceil2(copy_height, tile_height);
+
+	return (unsigned int)math_ceil2(mem_width * mem_height * byte_per_pixel, 256);
+}
+
  /**
   * *******************************************************************************************************************************************************
   * calculate_max_mem_size_per_plane_per_dpp: Calculate (loose) upper bound for total number of bytes reserved in memory for the copy given number of swaths
   *
   * - The copy width / height is the min of the vp_width/height and maximum number of pixels that can fit across an ODM slice
   *     - This is to account for recout positions that cross the ODM seam but are "mostly" within the same ODM slice
-  * - For mem width assume an extra block width and tile width is required (the memory reserved must take into account pitch which must be tiled aligned)
-  * - For mem height assumes two extra block heights are required
   *
   * *******************************************************************************************************************************************************
   */
@@ -222,12 +240,11 @@ static unsigned int calculate_max_mem_size_per_plane_per_dpp(const struct dml2_c
 	unsigned int copy_src_lines = copy_swaths * (chroma ? p->SwathHeightC[plane_idx] : p->SwathHeightY[plane_idx]);
 	unsigned int vp_width = chroma ? p->display_cfg->plane_descriptors[plane_idx].composition.viewport.plane1.width : p->display_cfg->plane_descriptors[plane_idx].composition.viewport.plane0.width;
 	unsigned int vp_height = chroma ? p->display_cfg->plane_descriptors[plane_idx].composition.viewport.plane1.height : p->display_cfg->plane_descriptors[plane_idx].composition.viewport.plane0.height;
-	unsigned int block256_width = chroma ? p->Read256BlockWidthC[plane_idx] : p->Read256BlockWidthY[plane_idx];
-	unsigned int block256_height = chroma ? p->Read256BlockHeightC[plane_idx] : p->Read256BlockHeightY[plane_idx];
 	unsigned int tile_width = chroma ? p->MacroTileWidthC[plane_idx] : p->MacroTileWidthY[plane_idx];
+	unsigned int tile_height = chroma ? p->MacroTileHeightC[plane_idx] : p->MacroTileHeightY[plane_idx];
 	unsigned int byte_per_pixel = chroma ? p->BytePerPixelC[plane_idx] : p->BytePerPixelY[plane_idx];
-	unsigned int mem_width;
-	unsigned int mem_height;
+	unsigned int copy_width;
+	unsigned int copy_height;
 	unsigned int odm_combine_factor;
 	double odm_slice_pixels;
 
@@ -241,12 +258,14 @@ static unsigned int calculate_max_mem_size_per_plane_per_dpp(const struct dml2_c
 		odm_combine_factor = 1;
 
 	odm_slice_pixels = (double)h_active / odm_combine_factor * h_ratio + (odm_combine_factor == 3 ? 2 : 0);
-	mem_width = (vertical_access ? copy_src_lines : (unsigned int)math_ceil(math_min2(odm_slice_pixels, vp_width))) + block256_width + tile_width;
-	mem_height = (vertical_access ? (unsigned int)math_ceil(math_min2(odm_slice_pixels, vp_height)) : copy_src_lines) + 2 * block256_height;
 
-	return (unsigned int)math_ceil2(mem_width * mem_height * byte_per_pixel, 256);
-}
+	/* Unaligned copy region dimensions (in pixels / lines) prior to tile alignment.
+	 * Also divide vp_widht/height by NoOfDPP for MPC combine scenarios (i.e., MPC split within an ODM slice). */
+	copy_width = vertical_access ? copy_src_lines : (unsigned int)math_ceil(math_min2(odm_slice_pixels, vp_width / p->NoOfDPP[plane_idx]));
+	copy_height = vertical_access ? (unsigned int)math_ceil(math_min2(odm_slice_pixels, vp_height / p->NoOfDPP[plane_idx])) : copy_src_lines;
 
+	return calculate_tile_aligned_size(copy_width, copy_height, tile_width, tile_height, byte_per_pixel);
+}
  /**
   * ****************************************************************************************************************************************
   * calculate_ub_copy_size_per_plane_per_dpp: Calculate tight upper bound for total number of bytes required for the copy given number of swaths.
@@ -422,9 +441,9 @@ static void calculate_swath_params(const struct dml2_core_calcs_calculate_altern
 	out->prefetch_hdl_delta = (double)swath_height / vratio_pre;
 }
 
-static unsigned int calc_svp_size_64kb_aligned(unsigned int total_size_bytes)
+static unsigned int calc_svp_size_256kb_aligned(unsigned int total_size_bytes)
 {
-	return ((total_size_bytes + 0xFFFF) >> 16) << 16; // Round up to nearest 64KB boundary
+	return ((total_size_bytes + 0x3FFFFu) >> 18) << 18; // Round up to nearest 256KB boundary
 }
 
 void dcn6_calculate_alternate_params(struct dml2_core_calcs_calculate_alternate_params *p)
@@ -439,6 +458,37 @@ void dcn6_calculate_alternate_params(struct dml2_core_calcs_calculate_alternate_
 	unsigned int fw_delay;
 	struct swath_params swath_params;
 
+	/* the alternate channel outputs are only meaningful when the alternate channel is in use;
+	 * otherwise zero all outputs and return early */
+	if (!p->alt_chan_in_use) {
+		*p->svp0_max_bytes = 0;
+		*p->svp1_max_bytes = 0;
+		*p->lsdma_bw_req_for_alt_kbps = 0;
+
+		for (i = 0; i < p->display_cfg->num_planes; i++) {
+			p->svp0_max_bytes_per_dpp[i] = 0;
+			p->svp0_max_bytes_per_dpp_c[i] = 0;
+			p->svp1_max_bytes_per_dpp[i] = 0;
+			p->svp1_max_bytes_per_dpp_c[i] = 0;
+			p->total_swaths[i] = 0;
+			p->total_swaths_c[i] = 0;
+			p->prefetch_swaths[i] = 0;
+			p->prefetch_swaths_c[i] = 0;
+			p->prefetch_hdl_delta[i] = 0;
+			p->recout_hdl_delta[i] = 0;
+			p->prefetch_hdl_delta_c[i] = 0;
+			p->recout_hdl_delta_c[i] = 0;
+		}
+
+		for (i = 0; i < p->display_cfg->num_streams; i++) {
+			p->nom_req_limit_alt[i] = 0;
+			p->min_lead_dst_lines[i] = 0;
+			p->max_prefetch_in_lines[i] = 0;
+		}
+
+		return;
+	}
+
 	*p->svp0_max_bytes = 0;
 	*p->svp1_max_bytes = 0;
 	svp_max_bytes[0] = 0;
@@ -471,7 +521,7 @@ void dcn6_calculate_alternate_params(struct dml2_core_calcs_calculate_alternate_
 				p->prefetch_hdl_delta[j] = swath_params.prefetch_hdl_delta;
 				for (k = 0; k < 2; k++) {
 					svp_max_bytes_per_dpp[k] = calculate_ub_copy_size_per_plane_per_dpp_per_svp(p, svp_dst_lines[k], j, false);
-					svp_max_bytes[k] += calc_svp_size_64kb_aligned(svp_max_bytes_per_dpp[k]) * p->NoOfDPP[j];
+					svp_max_bytes[k] += calc_svp_size_256kb_aligned(svp_max_bytes_per_dpp[k]) * p->NoOfDPP[j];
 				}
 				p->svp0_max_bytes_per_dpp[j] = svp_max_bytes_per_dpp[0];
 				p->svp1_max_bytes_per_dpp[j] = svp_max_bytes_per_dpp[1];
@@ -485,7 +535,7 @@ void dcn6_calculate_alternate_params(struct dml2_core_calcs_calculate_alternate_
 
 					for (k = 0; k < 2; k++) {
 						svp_max_bytes_per_dpp[k] = calculate_ub_copy_size_per_plane_per_dpp_per_svp(p, svp_dst_lines[k], j, true);
-						svp_max_bytes[k] += calc_svp_size_64kb_aligned(svp_max_bytes_per_dpp[k]) * p->NoOfDPP[j];
+						svp_max_bytes[k] += calc_svp_size_256kb_aligned(svp_max_bytes_per_dpp[k]) * p->NoOfDPP[j];
 					}
 					p->svp0_max_bytes_per_dpp_c[j] = svp_max_bytes_per_dpp[0];
 					p->svp1_max_bytes_per_dpp_c[j] = svp_max_bytes_per_dpp[1];
@@ -499,7 +549,7 @@ void dcn6_calculate_alternate_params(struct dml2_core_calcs_calculate_alternate_
 
 	*p->svp0_max_bytes = svp_max_bytes[0];
 	*p->svp1_max_bytes = svp_max_bytes[1];
-	*p->lsdma_bw_req_for_alt_kbps = p->dcn_non_urgent_bandwidth_kbps;
+	*p->lsdma_bw_req_for_alt_kbps = math_min2(p->dcn_non_urgent_bandwidth_kbps, p->max_lsdma_bandwidth_kbps);
 	copy_time_us = p->display_cfg->overrides.hw.force_alt_chan_copy_time.enable ? p->display_cfg->overrides.hw.force_alt_chan_copy_time.copy_time_us : calculate_alt_copy_time_us(p);
 	fw_delay =  p->display_cfg->overrides.hw.force_alt_chan_fw_delay.enable ? p->display_cfg->overrides.hw.force_alt_chan_fw_delay.fw_delay_us : p->alt_chan_fw_delay_us;
 	for (i = 0; i < p->display_cfg->num_streams; i++) {
@@ -542,6 +592,8 @@ void dcn6_calculate_flip_schedule(
 	double *final_flip_bw,
 	bool *ImmediateFlipSupportedForPipe)
 {
+	(void)meta_row_height;
+	(void)meta_row_height_chroma;
 	struct dml2_core_shared_CalculateFlipSchedule_locals *l = &s->CalculateFlipSchedule_locals;
 
 	l->dual_plane = dml2_core_utils_is_420(SourcePixelFormat) || dml2_core_utils_is_422_planar(SourcePixelFormat) || SourcePixelFormat == dml2_rgbe_alpha;
@@ -612,10 +664,10 @@ void dcn6_calculate_flip_schedule(
 				DML_LOG_VERBOSE("DML::%s: Tvm_trips_flip_rounded + 2*Tr0_trips_flip_rounded = %f\n", __func__, (Tvm_trips_flip_rounded + 2 * Tr0_trips_flip_rounded));
 			}
 			l->lb_flip_bw = math_max3(l->lb_flip_bw,
-				l->hvm_scaled_vm_bytes / (31 * LineTime) - Tno_bw_flip,
+				l->hvm_scaled_vm_bytes / ((31 * LineTime) - Tno_bw_flip),
 				l->dpte_row_bytes * HostVMInefficiencyFactor / (15 * LineTime));
 
-			DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm reg limit = %f\n", __func__, l->hvm_scaled_vm_bytes / (31 * LineTime) - Tno_bw_flip);
+			DML_LOG_VERBOSE("DML::%s: lb_flip_bw for vm reg limit = %f\n", __func__, l->hvm_scaled_vm_bytes / ((31 * LineTime) - Tno_bw_flip));
 		}
 
 		*final_flip_bw = l->lb_flip_bw;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_initialize.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_initialize.c
index 23fd6edeb0cf28..f20cee3c46fa38 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_initialize.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_initialize.c
@@ -3,6 +3,7 @@
 // Copyright 2024 Advanced Micro Devices, Inc.
 
 #include "dml2_core_dcn6_funcs_initialize.h"
+#include "dml2_core_dcn6_calcs.h"
 #include "dml2_debug.h"
 
 struct dml2_core_ip_params core_dcn6_ip_caps_base = {
@@ -24,6 +25,7 @@ struct dml2_core_ip_params core_dcn6_ip_caps_base = {
 	.writeback_interface_buffer_size_kbytes = 90,
 	//Number of pipes after DCN Pipe harvesting
 	.max_num_dpp = 4,
+	.max_num_opp = 4,
 	.max_num_otg = 4,
 	.max_num_wb = 1,
 	.zero_size_buffer_entries = 512,
@@ -111,6 +113,7 @@ static void patch_ip_caps_with_explicit_ip_params(struct dml2_ip_capabilities *i
 static void patch_ip_params_with_ip_caps(struct dml2_core_ip_params *ip_params, const struct dml2_ip_capabilities *ip_caps)
 {
 	ip_params->max_num_dpp = ip_caps->pipe_count;
+	ip_params->max_num_opp = ip_caps->otg_count;
 	ip_params->max_num_otg = ip_caps->otg_count;
 	ip_params->TDLUT_33cube_count = ip_caps->TDLUT_33cube_count;
 	ip_params->num_dsc = ip_caps->num_dsc;
@@ -158,6 +161,7 @@ bool dml2_core_dcn6_funcs_initialize(struct dml2_core_initialize_in_out *in_out)
 	memcpy(&core->clean_me_up.mode_lib.ip_caps, in_out->ip_caps, sizeof(struct dml2_ip_capabilities));
 	core->utm_soc_bb = in_out->utm_soc_bb;
 	core->clock_adjuster = in_out->clock_adjuster;
+	dml2_core_dcn6_calcs_init(&core->calcs);
 
 	DML_LOG_DEBUG("%s exit with true\n", __func__);
 	DML_LOG_COMP_IF_EXIT();
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c
index 39f0cd14ca3e13..aeddd113b7a1fc 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.c
@@ -2,11 +2,14 @@
 //
 // Copyright 2025 Advanced Micro Devices, Inc.
 #include "dml2_core_dcn6_funcs_mode_programming.h"
-#include "dml2_core_dcn5_calcs_dchub.h"
-#include "dml2_core_dcn6_calcs_dchub.h"
-#include "dml2_core_dcn5_calcs_display_pipe.h"
+#include "dml2_core_dcn6_calcs.h"
 #include "dml2_core_utils.h"
 
+static const struct dml2_core_dcn6_calcs *get_calcs(const struct dml2_core_calculate_mp_context *ctx)
+{
+	return ctx->calcs->dcn6;
+}
+
 static void dcn6_mp_populate_odm_mode(const struct dml2_display_solution *solution,
 		struct dml2_core_internal_mode_program *outputs)
 {
@@ -58,7 +61,7 @@ static void dcn6_mp_calculate_dcc_configurations(struct dml2_core_calculate_mp_c
 
 	for (k = 0; k < display_cfg->num_planes; ++k) {
 		DML_LOG_VERBOSE("DML::%s: Calculate DCC configuration for surface k=%u\n", __func__, k);
-		dcn5_calculate_dcc_configuration(
+		get_calcs(ctx)->calculate_dcc_configuration(
 				display_cfg->plane_descriptors[k].surface.dcc.enable,
 				display_cfg->overrides.dcc_programming_assumes_scan_direction_unknown,
 				display_cfg->plane_descriptors[k].pixel_format,
@@ -96,7 +99,7 @@ static void dcn6_mp_calculate_pixel_delivery_times(struct dml2_core_calculate_mp
 	struct dml2_core_internal_mode_program *outputs = states;
 
 	//Display Pipeline Delivery Time in Prefetch, Groups
-	dcn5_calculate_pixel_delivery_times(
+	get_calcs(ctx)->calculate_pixel_delivery_times(
 			display_cfg,
 			inputs->NoOfDPP,
 			display_cfg->num_planes,
@@ -185,7 +188,7 @@ static void dcn6_mp_calculate_meta_and_pte_times(struct dml2_core_calculate_mp_c
 	p->TimePerMetaChunkFlip = outputs->TimePerMetaChunkFlip;
 	p->TimePerChromaMetaChunkFlip = outputs->TimePerChromaMetaChunkFlip;
 
-	dcn5_calculate_meta_and_pte_times(p);
+	get_calcs(ctx)->calculate_meta_and_pte_times(p);
 }
 
 static void dcn6_mp_calculate_vm_group_and_request_times(struct dml2_core_calculate_mp_context *ctx,
@@ -196,7 +199,7 @@ static void dcn6_mp_calculate_vm_group_and_request_times(struct dml2_core_calcul
 	struct dml2_core_internal_mode_program *inputs = states;
 	struct dml2_core_internal_mode_program *outputs = states;
 
-	dcn5_calculate_vm_group_and_request_times(
+	get_calcs(ctx)->calculate_vm_group_and_request_times(
 		display_cfg,
 		display_cfg->num_planes,
 		inputs->BytePerPixelC,
@@ -431,7 +434,7 @@ static void dcn6_mp_calculate_stutter_efficiency(struct dml2_core_calculate_mp_c
 	p->DCHUBBUB_ARB_CSTATE_MAX_CAP_MODE = &outputs->DCHUBBUB_ARB_CSTATE_MAX_CAP_MODE;
 
 	// Stutter Efficiency
-	dcn6_calculate_stutter_efficiency(func_params, p);
+	get_calcs(ctx)->calculate_stutter_efficiency(func_params, p);
 
 #ifdef __DML_VBA_ALLOW_DELTA__
 	// Calculate z8 stutter eff assuming 0 reserved space
@@ -444,7 +447,7 @@ static void dcn6_mp_calculate_stutter_efficiency(struct dml2_core_calculate_mp_c
 	p->StutterPeriod = &outputs->StutterPeriodBestCase;
 
 	// Stutter Efficiency
-	dcn6_calculate_stutter_efficiency(func_params, p);
+	get_calcs(ctx)->calculate_stutter_efficiency(func_params, p);
 #else
 	outputs->Z8StutterEfficiencyNotIncludingVBlankBestCase = outputs->Z8StutterEfficiencyNotIncludingVBlank;
 	outputs->Z8StutterEfficiencyBestCase = outputs->Z8StutterEfficiency;
@@ -506,7 +509,7 @@ static void dcn6_mp_calculate_pstate_keepout_dst_lines(struct dml2_core_calculat
 	struct dml2_core_internal_mode_program *inputs = states;
 	struct dml2_core_internal_mode_program *outputs = states;
 
-	dcn5_calculate_pstate_keepout_dst_lines(display_cfg, &inputs->Watermark,
+	get_calcs(ctx)->calculate_pstate_keepout_dst_lines(display_cfg, &inputs->Watermark,
 			outputs->pstate_keepout_dst_lines);
 }
 
@@ -536,7 +539,7 @@ static void dcn6_mp_calculate_dcfclk_deep_sleep_hysteresis(struct dml2_core_calc
 					- min_return_latency_in_DCFCLK_cycles);
 }
 
-static void dcn6_calculate_mode_programming(struct dml2_core_calculate_mp_context *ctx,
+void dcn6_calculate_mode_programming(struct dml2_core_calculate_mp_context *ctx,
 		struct dml2_core_internal_mode_program *states)
 {
 	dcn6_mp_calculate_fraction_of_urgent_bandwidth(ctx, states);
@@ -564,7 +567,7 @@ static void dcn6_calculate_mode_programming(struct dml2_core_calculate_mp_contex
 	DML_LOG_VERBOSE("DML::%s: --- END --- \n", __func__);
 }
 
-static void dcn6_get_global_sync_programming(const struct dml2_core_internal_display_mode_lib *mode_lib, union dml2_global_sync_programming *out, int pipe_index)
+void dcn6_get_global_sync_programming(const struct dml2_core_internal_display_mode_lib *mode_lib, union dml2_global_sync_programming *out, int pipe_index)
 {
 	out->dcn4x.vready_offset_pixels = mode_lib->mp.VReadyOffsetPix[mode_lib->mp.pipe_plane[pipe_index]];
 	out->dcn4x.vstartup_lines = mode_lib->mp.VStartup[mode_lib->mp.pipe_plane[pipe_index]];
@@ -573,7 +576,7 @@ static void dcn6_get_global_sync_programming(const struct dml2_core_internal_dis
 	out->dcn4x.pstate_keepout_start_lines = mode_lib->mp.pstate_keepout_dst_lines[mode_lib->mp.pipe_plane[pipe_index]];
 }
 
-static void dcn6_get_stream_programming(const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_per_stream_programming *out, int pipe_index)
+void dcn6_get_stream_programming(const struct dml2_core_internal_display_mode_lib *mode_lib, struct dml2_per_stream_programming *out, int pipe_index)
 {
 	dcn6_get_global_sync_programming(mode_lib, &out->global_sync, pipe_index);
 }
@@ -958,7 +961,8 @@ static void dcn6_populate_qos_bound(struct dml2_display_cfg_programming *program
 	programming->qos_bound.lsdma_bandwidth_lb_kbps = solution->validation_result.mode_support.global.lsdma_bw_req_for_alt_kbps;
 }
 
-static void dcn6_populate_mode_programming(struct dml2_display_cfg_programming *programming,
+void dcn6_populate_mode_programming(struct dml2_core_calculate_mp_context *ctx,
+		struct dml2_display_cfg_programming *programming,
 		struct dml2_core_internal_scratch *s,
 		const struct dml2_core_internal_display_mode_lib *mode_lib,
 		const struct dml2_display_solution *solution,
@@ -976,9 +980,10 @@ static void dcn6_populate_mode_programming(struct dml2_display_cfg_programming *
 
 	memcpy(&programming->display_config, &solution->dispcfg, sizeof(struct dml2_display_cfg));
 	dcn6_populate_min_clocks(programming, solution, utm_soc_bb);
-	dcn5_get_arb_params(&programming->display_config, mode_lib, utm_soc_bb, &programming->global_regs.arb_regs);
+	get_calcs(ctx)->get_arb_params(&programming->display_config, mode_lib, utm_soc_bb, &programming->global_regs.arb_regs);
 	programming->global_regs.num_watermark_sets = 1;
-	dcn6_get_watermarks(&programming->display_config, mode_lib, utm_soc_bb, &programming->global_regs.wm_regs[0]);
+	get_calcs(ctx)->get_watermarks(&programming->display_config, mode_lib, utm_soc_bb,
+			&programming->global_regs.wm_regs[0]);
 	dcn6_populate_stutter_support(programming, mode_lib, solution, utm_soc_bb);
 	dcn6_populate_mcache_allocation(programming, solution);
 	dcn6_populate_qos_bound(programming, solution);
@@ -1023,7 +1028,9 @@ static void dcn6_populate_mode_programming(struct dml2_display_cfg_programming *
 			total_pipe_regs_copied++;
 
 			// Populate
-			dcn6_get_pipe_regs(&programming->display_config, mode_lib, programming->plane_programming[plane_index].pipe_regs[pipe_offset], dml_internal_pipe_index, utm_soc_bb, s);
+			get_calcs(ctx)->get_pipe_regs(&programming->display_config, mode_lib,
+					programming->plane_programming[plane_index].pipe_regs[pipe_offset],
+					dml_internal_pipe_index, utm_soc_bb, s);
 
 			main_stream_index = programming->display_config.plane_descriptors[plane_index].stream_index;
 
@@ -1059,7 +1066,7 @@ static void dcn6_populate_mode_programming(struct dml2_display_cfg_programming *
  * To move mode support result to mode programming, the data needs to be saved in cfg_support_info in the end of
  * mode support, so we can access cfg_support_info and populate mode programming data in this function based on it.
  */
-static void dcn6_mp_initialize_from_solution(struct dml2_core_internal_mode_program *outputs,
+void dcn6_mp_initialize_from_solution(struct dml2_core_internal_mode_program *outputs,
 		const struct dml2_display_solution *solution,
 		const struct dml2_utm_soc_bb *utm_soc_bb)
 {
@@ -1110,6 +1117,8 @@ static void dcn6_mp_initialize_from_solution(struct dml2_core_internal_mode_prog
 	outputs->min_available_urgent_bandwidth_MBps = solution->validation_result.mode_support.bandwidth_upper_bound.dcn5.urgent_bandwidth_kbps / 1000.0;
 	**outputs->urg_bandwidth_available = math_min2(solution->sop_constraint.dcn5.min_available_urgent_bandwidth_KBps / 1000.0,
 		outputs->Dcfclk * utm_soc_bb->urgent_sdp_derate_percent / 100.0 * utm_soc_bb->return_bus_width_bytes);
+	**outputs->non_urg_bandwidth_available = math_min2(solution->sop_constraint.dcn5.min_available_non_urgent_bandwidth_KBps / 1000.0,
+		outputs->Dcfclk * utm_soc_bb->nominal_sdp_derate_percent / 100.0 * utm_soc_bb->return_bus_width_bytes);
 
 	outputs->UrgentLatency = solution->sop_constraint.dcn5.latency.dcn5.urgent_ramp;
 	outputs->TripToMemory = math_max2(solution->sop_constraint.dcn5.latency.dcn5.urgent_ramp,
@@ -1124,7 +1133,7 @@ static void dcn6_mp_initialize_from_solution(struct dml2_core_internal_mode_prog
 			sizeof(solution->uclk_pstate_params.pstate_switch_modes));
 }
 
-static void dcn6_mp_build_calculate_mp_context(struct dml2_core_calculate_mp_context *ctx,
+void dcn6_mp_build_calculate_mp_context(struct dml2_core_calculate_mp_context *ctx,
 		struct dml2_core_instance *core,
 		const struct dml2_display_solution *solution)
 {
@@ -1136,9 +1145,10 @@ static void dcn6_mp_build_calculate_mp_context(struct dml2_core_calculate_mp_con
 	ctx->ms = &mode_lib->ms;
 	ctx->dummies = &mode_lib->scratch.dml_core_mode_programming_locals;
 	ctx->func_params = &mode_lib->scratch;
+	ctx->calcs = &core->calcs;
 }
 
-static void dcn6_mp_initialize_from_ms(struct dml2_core_internal_mode_program *outputs,
+void dcn6_mp_initialize_from_ms(struct dml2_core_internal_mode_program *outputs,
 		const struct dml2_core_internal_mode_support *ms)
 {
 	outputs->DCFCLKDeepSleep = ms->dcfclk_deepsleep;
@@ -1381,7 +1391,7 @@ enum dml2_status dml2_core_dcn6_funcs_populate_programming(struct dml2_core_inst
 	dcn6_mp_initialize_from_solution(&mode_lib->mp, solution, core->utm_soc_bb);
 	dcn6_mp_build_calculate_mp_context(calc_mp_ctx, core, solution);
 	dcn6_calculate_mode_programming(calc_mp_ctx, &mode_lib->mp);
-	dcn6_populate_mode_programming(programming, &mode_lib->scratch, mode_lib, solution, core->utm_soc_bb);
+	dcn6_populate_mode_programming(calc_mp_ctx, programming, &mode_lib->scratch, mode_lib, solution, core->utm_soc_bb);
 
 	DML_LOG_DEBUG("%s exit\n", __func__);
 	DML_LOG_COMP_IF_EXIT();
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.h
index f182c27aea7b71..542d96c648ddc0 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_programming.h
@@ -5,7 +5,38 @@
 #ifndef __DML2_CORE_DCN6_FUNCS_MODE_PROGRAMMING_H__
 #define __DML2_CORE_DCN6_FUNCS_MODE_PROGRAMMING_H__
 #include "dml2_internal_shared_types.h"
+
 enum dml2_status dml2_core_dcn6_funcs_populate_programming(struct dml2_core_instance *core,
 		const struct dml2_display_solution *solution,
 		struct dml2_display_cfg_programming *programming);
+
+void dcn6_mode_programming(struct dml2_core_calculate_mp_context *ctx,
+		struct dml2_core_internal_mode_program *states);
+
+void dcn6_get_global_sync_programming(const struct dml2_core_internal_display_mode_lib *mode_lib,
+		union dml2_global_sync_programming *out, int pipe_index);
+
+void dcn6_get_stream_programming(const struct dml2_core_internal_display_mode_lib *mode_lib,
+		struct dml2_per_stream_programming *out, int pipe_index);
+
+void dcn6_populate_mode_programming(struct dml2_core_calculate_mp_context *ctx,
+		struct dml2_display_cfg_programming *programming,
+		struct dml2_core_internal_scratch *s,
+		const struct dml2_core_internal_display_mode_lib *mode_lib,
+		const struct dml2_display_solution *solution,
+		const struct dml2_utm_soc_bb *utm_soc_bb);
+
+void dcn6_mp_initialize_from_ms(struct dml2_core_internal_mode_program *outputs,
+		const struct dml2_core_internal_mode_support *ms);
+
+void dcn6_mp_initialize_from_solution(struct dml2_core_internal_mode_program *outputs,
+		const struct dml2_display_solution *solution,
+		const struct dml2_utm_soc_bb *utm_soc_bb);
+
+void dcn6_mp_build_calculate_mp_context(struct dml2_core_calculate_mp_context *ctx,
+		struct dml2_core_instance *core,
+		const struct dml2_display_solution *solution);
+
+void dcn6_calculate_mode_programming(struct dml2_core_calculate_mp_context *ctx,
+		struct dml2_core_internal_mode_program *states);
 #endif /* __DML2_CORE_DCN6_FUNCS_MODE_PROGRAMMING_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c
index d83c7eea462b7c..71d14fb8c8305a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.c
@@ -3,12 +3,15 @@
 // Copyright 2025 Advanced Micro Devices, Inc.
 #include "dml2_core_dcn6_funcs_mode_support.h"
 #include "dml2_core_utils.h"
-#include "dml2_core_dcn5_calcs_dchub.h"
-#include "dml2_core_dcn5_calcs_display_pipe.h"
-#include "dml2_core_dcn6_calcs_dchub.h"
+#include "dml2_core_dcn6_calcs.h"
 #include "dml_top_types.h"
 
-static void dcn6_ms_check_input_sanity(
+static const struct dml2_core_dcn6_calcs *get_calcs(const struct dml2_core_calculate_ms_context *ctx)
+{
+	return ctx->calcs->dcn6;
+}
+
+void dcn6_ms_check_input_sanity(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -43,7 +46,7 @@ static void dcn6_ms_check_input_sanity(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_desired_output_bpp(
+void dcn6_ms_calculate_desired_output_bpp(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -57,7 +60,7 @@ static void dcn6_ms_calculate_desired_output_bpp(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_max_det_and_min_compressed_buffer_size(
+void dcn6_ms_calculate_max_det_and_min_compressed_buffer_size(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -66,7 +69,7 @@ static void dcn6_ms_calculate_max_det_and_min_compressed_buffer_size(
 	struct dml2_core_internal_mode_support *outputs = states;
 
 	DML_LOG_FUNC_ENTER();
-	dcn5_calculate_max_det_and_min_compressed_buffer_size(
+	get_calcs(ctx)->calculate_max_det_and_min_compressed_buffer_size(
 			ip->config_return_buffer_size_in_kbytes,
 			ip->config_return_buffer_segment_size_in_kbytes,
 			ip->rob_buffer_size_kbytes,
@@ -86,7 +89,7 @@ static void dcn6_ms_calculate_max_det_and_min_compressed_buffer_size(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_effective_pixel_clock(
+void dcn6_ms_calculate_effective_pixel_clock(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -99,13 +102,14 @@ static void dcn6_ms_calculate_effective_pixel_clock(
 	 * This function should probably be removed since ptoi is never true, so the function is a noop; not really
 	 * obvious if the comment means DML2.1 doesn't support interlace today.
 	 */
-	dcn5_adjust_pixel_clock_for_progressive_to_interlace_unit(display_cfg, ip->ptoi_supported, outputs->PixelClockBackEnd);
+	get_calcs(ctx)->adjust_pixel_clock_for_progressive_to_interlace_unit(display_cfg, ip->ptoi_supported,
+			outputs->PixelClockBackEnd);
 
 	DML_LOG_DEBUG_ARRAY_DOUBLE(outputs->PixelClockBackEnd, display_cfg->num_planes);
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_scaler_support(
+bool dcn6_ms_check_scaler_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -159,7 +163,7 @@ static bool dcn6_ms_check_scaler_support(
 	return outputs->support.ScaleRatioAndTapsSupport;
 }
 
-static bool dcn6_ms_check_source_format_and_scan_direction(
+bool dcn6_ms_check_source_format_and_scan_direction(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -185,7 +189,7 @@ static bool dcn6_ms_check_source_format_and_scan_direction(
 	return outputs->support.SourceFormatPixelAndScanSupport;
 }
 
-static void dcn6_ms_calculate_byte_per_pixel_and_block_sizes(
+void dcn6_ms_calculate_byte_per_pixel_and_block_sizes(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -197,7 +201,7 @@ static void dcn6_ms_calculate_byte_per_pixel_and_block_sizes(
 	DML_LOG_FUNC_ENTER();
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		plane = &display_cfg->plane_descriptors[k];
-		dcn5_calculate_byte_per_pixel_and_block_sizes(
+		get_calcs(ctx)->calculate_byte_per_pixel_and_block_sizes(
 				plane->pixel_format,
 				plane->surface.tiling,
 				plane->surface.plane0.pitch,
@@ -236,7 +240,7 @@ static void dcn6_ms_calculate_byte_per_pixel_and_block_sizes(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_read_bandwidth(
+void dcn6_ms_calculate_read_bandwidth(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -288,7 +292,7 @@ static void dcn6_ms_calculate_read_bandwidth(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_writeback_bandwidth(
+void dcn6_ms_calculate_writeback_bandwidth(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -328,7 +332,7 @@ static void dcn6_ms_calculate_writeback_bandwidth(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_writeback_bandwidth_latency_support(
+bool dcn6_ms_check_writeback_bandwidth_latency_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -368,7 +372,7 @@ static bool dcn6_ms_check_writeback_bandwidth_latency_support(
 	return outputs->support.WritebackLatencySupport;
 }
 
-static bool dcn6_ms_check_writeback_scale_ratio_and_taps_support(
+bool dcn6_ms_check_writeback_scale_ratio_and_taps_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -445,7 +449,7 @@ static bool dcn6_ms_check_writeback_scale_ratio_and_taps_support(
 	return outputs->support.WritebackScaleRatioAndTapsSupport;
 }
 
-static void dcn6_ms_calculate_single_pipe_dppclk_and_pscl_factor(
+void dcn6_ms_calculate_single_pipe_dppclk_and_pscl_factor(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -460,7 +464,7 @@ static void dcn6_ms_calculate_single_pipe_dppclk_and_pscl_factor(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		plane = &display_cfg->plane_descriptors[k];
 		stream = &display_cfg->stream_descriptors[plane->stream_index];
-		dcn5_calculate_single_pipe_dppclk_and_scl_throughput(
+		get_calcs(ctx)->calculate_single_pipe_dppclk_and_scl_throughput(
 				plane->composition.scaler_info.plane0.h_ratio,
 				plane->composition.scaler_info.plane1.h_ratio,
 				plane->composition.scaler_info.plane0.v_ratio,
@@ -486,7 +490,7 @@ static void dcn6_ms_calculate_single_pipe_dppclk_and_pscl_factor(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_max_swath_widths(
+void dcn6_ms_calculate_max_swath_widths(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -570,7 +574,7 @@ static void dcn6_ms_calculate_max_swath_widths(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_cursor_support(
+bool dcn6_ms_check_cursor_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -599,7 +603,7 @@ static bool dcn6_ms_check_cursor_support(
 	return outputs->support.CursorSupport;
 }
 
-static bool dcn6_ms_check_surface_alginment_requirements(
+bool dcn6_ms_check_surface_alginment_requirements(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -747,14 +751,14 @@ static void dcn6_ms_calculate_swath_and_det_configuration_for_single_dpp(
 	p->ViewportSizeSupportPerSurface = outputs->SingleDPPViewportSizeSupportPerSurface;
 	p->ViewportSizeSupport = &dummies->dummy_boolean[1];
 	// This calls is just to find out if there is enough DET space to support full vp in 1 pipe.
-	dcn5_calculate_swath_and_det_configuration(ctx->func_params, p);
+	get_calcs(ctx)->calculate_swath_and_det_configuration(ctx->func_params, p);
 
 	DML_LOG_DEBUG_ARRAY_BOOL(outputs->SingleDPPViewportSizeSupportPerSurface, display_cfg->num_planes);
 	DML_LOG_FUNC_EXIT();
 
 }
 
-static void dcn6_ms_calculate_estimated_num_of_dsc_slices(
+void dcn6_ms_calculate_estimated_num_of_dsc_slices(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -794,7 +798,7 @@ static void dcn6_ms_calculate_estimated_num_of_dsc_slices(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_output_link(
+void dcn6_ms_calculate_output_link(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -809,7 +813,7 @@ static void dcn6_ms_calculate_output_link(
 	DML_LOG_FUNC_ENTER();
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		stream = &display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index];
-		dcn5_calculate_output_link(
+		get_calcs(ctx)->calculate_output_link(
 				ctx->func_params,
 				((double) soc_bb->max_phyclk_khz / 1000),
 				((double) soc_bb->max_phyclk_d18_khz / 1000),
@@ -849,7 +853,7 @@ static void dcn6_ms_calculate_output_link(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_odm_mode(
+void dcn6_ms_calculate_odm_mode(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -869,7 +873,7 @@ static void dcn6_ms_calculate_odm_mode(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		stream = &display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index];
 
-		dcn5_calculate_odm_mode(
+		get_calcs(ctx)->calculate_odm_mode(
 				ip->maximum_pixels_per_line_per_dsc_unit,
 				stream->timing.h_active,
 				stream->output.output_format,
@@ -902,7 +906,7 @@ static void dcn6_ms_calculate_odm_mode(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_num_of_dsc_slices(
+void dcn6_ms_calculate_num_of_dsc_slices(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -935,7 +939,7 @@ static void dcn6_ms_calculate_num_of_dsc_slices(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_num_of_dsc_slices_support(
+bool dcn6_ms_check_num_of_dsc_slices_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -988,12 +992,16 @@ static void dcn6_ms_calculate_num_of_dpp_required(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		plane = &display_cfg->plane_descriptors[k];
 		outputs->NoOfDPP[k] = 1;
+		outputs->NoOfOPP[k] = 1;
 		if (inputs->ODMMode[k] == dml2_odm_mode_combine_4to1) {
 			outputs->NoOfDPP[k] = 4;
+			outputs->NoOfOPP[k] = 4;
 		} else if (inputs->ODMMode[k] == dml2_odm_mode_combine_3to1) {
 			outputs->NoOfDPP[k] = 3;
+			outputs->NoOfOPP[k] = 3;
 		} else if (inputs->ODMMode[k] == dml2_odm_mode_combine_2to1) {
 			outputs->NoOfDPP[k] = 2;
+			outputs->NoOfOPP[k] = 2;
 		} else if (plane->overrides.mpcc_combine_factor == 2) {
 			outputs->MPCCombine[k] = true;
 			outputs->NoOfDPP[k] = 2;
@@ -1018,7 +1026,7 @@ static void dcn6_ms_calculate_num_of_dpp_required(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_total_available_pipes_support(
+bool dcn6_ms_check_total_available_pipes_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1027,12 +1035,28 @@ static bool dcn6_ms_check_total_available_pipes_support(
 	struct dml2_core_internal_mode_support *inputs = states;
 	struct dml2_core_internal_mode_support *outputs = states;
 	unsigned int totalNumOfActiveDPP = 0;
-	unsigned int k;
+	unsigned int k, m;
 
 	DML_LOG_FUNC_ENTER();
 	for (k = 0; k < display_cfg->num_planes; k++)
 		totalNumOfActiveDPP += inputs->NoOfDPP[k];
-	outputs->support.TotalAvailablePipesSupport = totalNumOfActiveDPP <= (unsigned int)ip->max_num_dpp;
+
+
+	// TotalNumberOfActiveOPP is the sum of the per stream max NoOfOPP of all planes driving that stream
+	outputs->TotalNumberOfActiveOPP = 0;
+	for (k = 0; k < display_cfg->num_streams; k++) {
+		unsigned int NoOfOppPerStream = 0;
+
+		for (m = 0; m < display_cfg->num_planes; m++) {
+			if (display_cfg->plane_descriptors[m].stream_index == k)
+				NoOfOppPerStream = NoOfOppPerStream < inputs->NoOfOPP[m] ? inputs->NoOfOPP[m] : NoOfOppPerStream;
+		}
+
+		outputs->TotalNumberOfActiveOPP += NoOfOppPerStream;
+	}
+
+	outputs->support.TotalAvailablePipesSupport = totalNumOfActiveDPP <= (unsigned int)ip->max_num_dpp
+			&& outputs->TotalNumberOfActiveOPP <= (unsigned int)ip->max_num_opp;
 
 	DML_LOG_DEBUG_BOOL(outputs->support.TotalAvailablePipesSupport);
 	DML_LOG_FUNC_EXIT();
@@ -1040,7 +1064,7 @@ static bool dcn6_ms_check_total_available_pipes_support(
 	return outputs->support.TotalAvailablePipesSupport;
 }
 
-static bool dcn6_ms_check_total_available_TDLUT_33cube_support(
+bool dcn6_ms_check_total_available_TDLUT_33cube_support(
 	const struct dml2_core_calculate_ms_context *ctx,
 	struct dml2_core_internal_mode_support *states)
 {
@@ -1063,7 +1087,7 @@ static bool dcn6_ms_check_total_available_TDLUT_33cube_support(
 	return outputs->support.NumberOfTDLUT33cubeSupport;
 }
 
-static void dcn6_ms_calculate_total_num_of_single_dpp_surfaces(
+void dcn6_ms_calculate_total_num_of_single_dpp_surfaces(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1082,7 +1106,7 @@ static void dcn6_ms_calculate_total_num_of_single_dpp_surfaces(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_dispclk_and_dppclk_required(
+void dcn6_ms_calculate_dispclk_and_dppclk_required(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1101,7 +1125,7 @@ static void dcn6_ms_calculate_dispclk_and_dppclk_required(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		stream = &display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index];
 		for (j = 0; j < stream->writeback.active_writebacks_per_stream; ++j) {
-			writeback_required_dispclk = dcn5_calculate_write_back_dispclk(
+			writeback_required_dispclk = get_calcs(ctx)->calculate_write_back_dispclk(
 				stream->writeback.writeback_stream[j].pixel_format,
 				((double)stream->timing.pixel_clock_khz / 1000),
 				inputs->ODMMode[k],
@@ -1141,7 +1165,7 @@ static void dcn6_ms_calculate_dispclk_and_dppclk_required(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_dispclk_and_dppclk_support(
+bool dcn6_ms_check_dispclk_and_dppclk_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1159,7 +1183,7 @@ static bool dcn6_ms_check_dispclk_and_dppclk_support(
 	return outputs->support.DISPCLK_DPPCLK_Support;
 }
 
-static bool dcn6_ms_check_otg_count_support(
+bool dcn6_ms_check_otg_count_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1187,7 +1211,7 @@ static bool dcn6_ms_check_otg_count_support(
 	return outputs->support.NumberOfOTGSupport;
 }
 
-static bool dcn6_ms_check_hpo_frl_encoder_count_support(
+bool dcn6_ms_check_hpo_frl_encoder_count_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1216,7 +1240,7 @@ static bool dcn6_ms_check_hpo_frl_encoder_count_support(
 	return outputs->support.NumberOfHDMIFRLSupport;
 }
 
-static bool dcn6_ms_check_hpo_dp_encoder_count_support(
+bool dcn6_ms_check_hpo_dp_encoder_count_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1251,7 +1275,7 @@ static bool dcn6_ms_check_hpo_dp_encoder_count_support(
 	return outputs->support.NumberOfDP2p0Support;
 }
 
-static bool dcn6_ms_check_writeback_count_support(
+bool dcn6_ms_check_writeback_count_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1272,10 +1296,6 @@ static bool dcn6_ms_check_writeback_count_support(
 
 		totalNumberOfActiveWriteback +=
 				display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].writeback.active_writebacks_per_stream;
-
-		/* >1 writeback per stream is currently not supported */
-		if (display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index].writeback.active_writebacks_per_stream > 1)
-			writeback_per_stream_supported = false;
 	}
 	outputs->support.EnoughWritebackUnits = writeback_per_stream_supported &&
 			totalNumberOfActiveWriteback <= ip->max_num_wb;
@@ -1286,7 +1306,7 @@ static bool dcn6_ms_check_writeback_count_support(
 	return outputs->support.EnoughWritebackUnits;
 }
 
-static bool dcn6_ms_check_link_bandwidth_support(
+bool dcn6_ms_check_link_bandwidth_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1314,7 +1334,7 @@ static bool dcn6_ms_check_link_bandwidth_support(
 	return outputs->support.LinkCapacitySupport;
 }
 
-static void dcn6_ms_check_misc_link_supports(
+void dcn6_ms_check_misc_link_supports(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1370,7 +1390,7 @@ static void dcn6_ms_check_misc_link_supports(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_dtbclk_required(
+void dcn6_ms_calculate_dtbclk_required(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1385,7 +1405,7 @@ static void dcn6_ms_calculate_dtbclk_required(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		stream = &display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index];
 		if (stream->output.output_encoder == dml2_hdmifrl) {
-			outputs->RequiredDTBCLK[k] = dcn5_calculate_required_dtbclk(
+			outputs->RequiredDTBCLK[k] = get_calcs(ctx)->calculate_required_dtbclk(
 				inputs->RequiresDSC[k],
 				inputs->PixelClockBackEnd[k],
 				stream->output.output_format,
@@ -1406,7 +1426,7 @@ static void dcn6_ms_calculate_dtbclk_required(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_dtbclk_support(
+bool dcn6_ms_check_dtbclk_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1430,7 +1450,7 @@ static bool dcn6_ms_check_dtbclk_support(
 	return support;
 }
 
-static void dcn6_ms_calculate_dscclk_required(
+void dcn6_ms_calculate_dscclk_required(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1484,7 +1504,7 @@ static void dcn6_ms_calculate_dscclk_required(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_dscclk_support(
+bool dcn6_ms_check_dscclk_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1509,7 +1529,7 @@ static bool dcn6_ms_check_dscclk_support(
 	return !outputs->support.DSCCLKRequiredMoreThanSupported;
 }
 
-static void dcn6_ms_check_dsc_engine_supports(
+void dcn6_ms_check_dsc_engine_supports(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1567,7 +1587,7 @@ static void dcn6_ms_check_dsc_engine_supports(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_dsc_delay(
+void dcn6_ms_calculate_dsc_delay(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1582,7 +1602,7 @@ static void dcn6_ms_calculate_dsc_delay(
 	/*DSC Delay per state*/
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		stream = &display_cfg->stream_descriptors[display_cfg->plane_descriptors[k].stream_index];
-		outputs->DSCDelay[k] = dcn5_calculate_dsc_delay_requirement(
+		outputs->DSCDelay[k] = get_calcs(ctx)->calculate_dsc_delay_requirement(
 				inputs->RequiresDSC[k],
 				inputs->ODMMode[k],
 				ip->maximum_dsc_bits_per_component,
@@ -1669,7 +1689,7 @@ static void dcn6_ms_calculate_swath_and_det_configuration(
 	p->CompressedBufferSizeInkByte = &outputs->CompressedBufferSizeInkByte;
 	p->ViewportSizeSupportPerSurface = dummies->dummy_boolean_array[0];
 	p->ViewportSizeSupport = &outputs->support.ViewportSizeSupport;
-	dcn5_calculate_swath_and_det_configuration(ctx->func_params, p);
+	get_calcs(ctx)->calculate_swath_and_det_configuration(ctx->func_params, p);
 
 	DML_LOG_DEBUG_ARRAY_UINT(outputs->SwathWidthY, display_cfg->num_planes);
 	DML_LOG_DEBUG_UINT(outputs->CompressedBufferSizeInkByte);
@@ -1692,7 +1712,7 @@ static void dcn6_ms_calculate_swath_and_det_configuration(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_total_num_of_dcc_active_dpp(
+void dcn6_ms_calculate_total_num_of_dcc_active_dpp(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1716,7 +1736,7 @@ static void dcn6_ms_calculate_total_num_of_dcc_active_dpp(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_vm_row_and_swath_and_calculate_dcc_meta_cache_requirements(
+void dcn6_ms_calculate_vm_row_and_swath_and_calculate_dcc_meta_cache_requirements(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1844,7 +1864,7 @@ static void dcn6_ms_calculate_vm_row_and_swath_and_calculate_dcc_meta_cache_requ
 	p->meta_row_width_chroma = outputs->meta_row_width_chroma;
 	p->meta_row_height_chroma = outputs->meta_row_height_chroma;
 	p->meta_pte_bytes_per_frame_ub_c = outputs->meta_pte_bytes_per_frame_ub_c;
-	dcn5_calculate_vm_row_and_swath(ctx->func_params, p);
+	get_calcs(ctx)->calculate_vm_row_and_swath(ctx->func_params, p);
 
 	DML_LOG_DEBUG_ARRAY_BOOL(outputs->PTEBufferSizeNotExceeded, display_cfg->num_planes);
 	DML_LOG_DEBUG_ARRAY_UINT(outputs->dpte_row_width_luma_ub, display_cfg->num_planes);
@@ -1901,7 +1921,7 @@ static void dcn6_ms_calculate_vm_row_and_swath_and_calculate_dcc_meta_cache_requ
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_pte_buffer_size_support(
+bool dcn6_ms_check_pte_buffer_size_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1926,7 +1946,7 @@ static bool dcn6_ms_check_pte_buffer_size_support(
 	return outputs->support.PTEBufferSizeNotExceeded;
 }
 
-static bool dcn6_ms_check_dcc_meta_cache_support(
+bool dcn6_ms_check_dcc_meta_cache_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -1981,7 +2001,7 @@ static double dcn6_ms_pstate_type_to_blackout_us(
 	return blackout_us;
 }
 
-static void dcn6_ms_calculate_vactive_pstate_requirements(
+void dcn6_ms_calculate_vactive_pstate_requirements(
 	const struct dml2_core_calculate_ms_context *ctx,
 	struct dml2_core_internal_mode_support *states)
 {
@@ -2040,11 +2060,11 @@ static void dcn6_ms_calculate_vactive_pstate_requirements(
 		p->bytes_required_l = outputs->pstate_bytes_required_l[pstate_type];
 		p->bytes_required_c = outputs->pstate_bytes_required_c[pstate_type];
 
-		dcn5_calculate_bytes_to_fetch_required_to_hide_latency(p);
+		get_calcs(ctx)->calculate_bytes_to_fetch_required_to_hide_latency(p);
 	}
 
 	/* Excess VActive bandwidth required to fill DET */
-	dcn6_calculate_excess_vactive_bandwidth_required(display_cfg,
+	get_calcs(ctx)->calculate_excess_vactive_bandwidth_required(display_cfg,
 			outputs->pstate_bytes_required_l,
 			outputs->pstate_bytes_required_c,
 
@@ -2066,7 +2086,7 @@ static void dcn6_ms_calculate_vactive_pstate_requirements(
 }
 
 /* FIXME - break it down according the function name */
-static void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urgent_latency_hiding(
+void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urgent_latency_hiding(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2088,7 +2108,7 @@ static void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urge
 		stream = &display_cfg->stream_descriptors[plane->stream_index];
 		line_time_us = stream->timing.h_total / ((double) stream->timing.pixel_clock_khz / 1000);
 		cursor_not_enough_urgent_latency_hiding = 0;
-		dcn5_calculate_cursor_req_attributes(
+		get_calcs(ctx)->calculate_cursor_req_attributes(
 				plane->cursor.cursor_width,
 				plane->cursor.cursor_bpp,
 				// output
@@ -2096,7 +2116,7 @@ static void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urge
 				&outputs->cursor_bytes_per_line[k],
 				&outputs->cursor_bytes_per_chunk[k],
 				&cursor_bytes);
-		dcn5_calculate_cursor_urgent_burst_factor(
+		get_calcs(ctx)->calculate_cursor_urgent_burst_factor(
 				ip->cursor_buffer_size,
 				plane->cursor.cursor_width,
 				outputs->cursor_bytes_per_chunk[k],
@@ -2112,7 +2132,7 @@ static void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urge
 				plane->composition.scaler_info.plane0.v_ratio);
 		DML_LOG_VERBOSE("DML::%s: k=%d, VRatioChroma=%f\n", __func__, k,
 				plane->composition.scaler_info.plane1.v_ratio);
-		dcn5_calculate_urgent_burst_factor(
+		get_calcs(ctx)->calculate_urgent_burst_factor(
 				&display_cfg->plane_descriptors[k],
 				inputs->swath_width_luma_ub[k],
 				inputs->swath_width_chroma_ub[k],
@@ -2124,6 +2144,8 @@ static void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urge
 				plane->composition.scaler_info.plane1.v_ratio,
 				inputs->BytePerPixelInDETY[k],
 				inputs->BytePerPixelInDETC[k],
+				inputs->UnboundedRequestEnabled,
+				inputs->CompressedBufferSizeInkByte,
 				inputs->DETBufferSizeY[k],
 				inputs->DETBufferSizeC[k],
 
@@ -2146,7 +2168,7 @@ static void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urge
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_min_dcfclk_deepsleep_clock(
+void dcn6_ms_calculate_min_dcfclk_deepsleep_clock(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2158,7 +2180,7 @@ static void dcn6_ms_calculate_min_dcfclk_deepsleep_clock(
 	double raw_dcfclk_deepsleep_mhz = 0;
 
 	DML_LOG_FUNC_ENTER();
-	dcn5_calculate_dcfclk_deep_sleep(
+	get_calcs(ctx)->calculate_dcfclk_deep_sleep(
 			display_cfg,
 			display_cfg->num_planes,
 			inputs->BytePerPixelY,
@@ -2180,7 +2202,7 @@ static void dcn6_ms_calculate_min_dcfclk_deepsleep_clock(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_writeback_delay(
+void dcn6_ms_calculate_writeback_delay(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2199,7 +2221,7 @@ static void dcn6_ms_calculate_writeback_delay(
 		for (j = 0; j < stream->writeback.active_writebacks_per_stream; j++) {
 			outputs->WritebackDelayTime[k] = math_max2(outputs->WritebackDelayTime[k],
 					soc_bb->writeback_base_latency_us
-					+ dcn5_calculate_write_back_delay(
+					+ get_calcs(ctx)->calculate_write_back_delay(
 							stream->writeback.writeback_stream[j].pixel_format,
 							stream->writeback.writeback_stream[j].h_ratio,
 							stream->writeback.writeback_stream[j].v_ratio,
@@ -2218,7 +2240,7 @@ static void dcn6_ms_calculate_writeback_delay(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_alternate_params(const struct dml2_core_calculate_ms_context *ctx,
+void dcn6_ms_calculate_alternate_params(const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
 	struct dml2_core_internal_scratch *func_params = ctx->func_params;
@@ -2226,8 +2248,19 @@ static void dcn6_ms_calculate_alternate_params(const struct dml2_core_calculate_
 	const struct dml2_utm_soc_bb *soc_bb = ctx->soc_bb;
 	struct dml2_core_internal_mode_support *inputs = states;
 	struct dml2_core_internal_mode_support *outputs = states;
+	unsigned int i;
+	bool is_alternate_pstate_required = false;
 
 	DML_LOG_FUNC_ENTER();
+
+	for (i = 0; i < ctx->display_cfg->num_planes; i++) {
+		if (ctx->display_cfg->plane_descriptors[i].overrides.uclk_pstate_change_strategy ==
+				dml2_uclk_pstate_change_strategy_force_alternate) {
+			is_alternate_pstate_required = true;
+			break;
+		}
+	}
+
 	p->display_cfg = ctx->display_cfg;
 	p->dst_y_prefetch = inputs->dst_y_prefetch;
 	p->SwathHeightY = inputs->SwathHeightY;
@@ -2246,6 +2279,8 @@ static void dcn6_ms_calculate_alternate_params(const struct dml2_core_calculate_
 	p->Read256BlockHeightC = inputs->Read256BlockHeightC;
 	p->MacroTileWidthY = inputs->MacroTileWidthY;
 	p->MacroTileWidthC = inputs->MacroTileWidthC;
+	p->MacroTileHeightY = inputs->MacroTileHeightY;
+	p->MacroTileHeightC = inputs->MacroTileHeightC;
 	p->VInitPrefillY = inputs->PrefillY;
 	p->VInitPrefillC = inputs->PrefillC;
 	p->VRatioPrefetchY = inputs->VRatioPreY;
@@ -2253,16 +2288,17 @@ static void dcn6_ms_calculate_alternate_params(const struct dml2_core_calculate_
 	p->NoOfDPP = inputs->NoOfDPP;
 	p->max_num_dpp = ctx->ip->max_num_dpp;
 	p->dram_blackout_us = soc_bb->power_management_parameters.dram_clk_change_blackout_us;
-	p->VActiveLatencyHidingUs = inputs->VActiveLatencyHidingUs;
 	p->svp0_dst_lines = inputs->svp0_dst_lines;
 	p->svp1_dst_lines = inputs->svp1_dst_lines;
 	p->svp_req_limit = inputs->svp_req_limit;
-	p->dcn_non_urgent_bandwidth_kbps = inputs->support.bandwidth_upper_bound.dcn5.non_urgent_bandwidth_kbps;
+	p->dcn_non_urgent_bandwidth_kbps = **inputs->support.non_urg_bandwidth_required_flip * 1000;
+	p->max_lsdma_bandwidth_kbps = soc_bb->max_lsdma_bandwidth_kbps;
 	p->alt_chan_fw_delay_us = ctx->ip->alt_chan_fw_delay_us;
 	p->dst_y_per_vm_vblank = inputs->LinesForVM;
 	p->dst_y_per_row_vblank = inputs->LinesForDPTERow;
 	p->DSTYAfterScaler = inputs->DSTYAfterScaler;
 	p->ODMMode = inputs->ODMMode;
+	p->alt_chan_in_use = is_alternate_pstate_required;
 
 	p->svp0_max_bytes = &outputs->svp0_max_bytes;
 	p->svp1_max_bytes = &outputs->svp1_max_bytes;
@@ -2283,7 +2319,7 @@ static void dcn6_ms_calculate_alternate_params(const struct dml2_core_calculate_
 	p->max_prefetch_in_lines = outputs->max_prefetch_in_lines;
 	p->lsdma_bw_req_for_alt_kbps = &outputs->lsdma_bw_req_for_alt_kbps;
 
-	dcn6_calculate_alternate_params(p);
+	get_calcs(ctx)->calculate_alternate_params(p);
 
 	DML_LOG_DEBUG_UINT(outputs->svp0_max_bytes);
 	DML_LOG_DEBUG_UINT(outputs->svp1_max_bytes);
@@ -2306,7 +2342,7 @@ static void dcn6_ms_calculate_alternate_params(const struct dml2_core_calculate_
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_alternate_svp_lines(const struct dml2_core_calculate_ms_context *ctx,
+void dcn6_ms_calculate_alternate_svp_lines(const struct dml2_core_calculate_ms_context *ctx,
 	struct dml2_core_internal_mode_support *states)
 {
 	struct dml2_core_internal_scratch *func_params = ctx->func_params;
@@ -2316,6 +2352,7 @@ static void dcn6_ms_calculate_alternate_svp_lines(const struct dml2_core_calcula
 	struct dml2_core_internal_mode_support *outputs = states;
 
 	DML_LOG_FUNC_ENTER();
+
 	p->display_cfg = ctx->display_cfg;
 	p->SwathHeightY = inputs->SwathHeightY;
 	p->SwathHeightC = inputs->SwathHeightC;
@@ -2326,7 +2363,7 @@ static void dcn6_ms_calculate_alternate_svp_lines(const struct dml2_core_calcula
 	p->svp1_dst_lines = outputs->svp1_dst_lines;
 	p->svp_req_limit = outputs->svp_req_limit;
 
-	dcn6_calculate_alternate_svp_lines(p);
+	get_calcs(ctx)->calculate_alternate_svp_lines(p);
 
 	DML_LOG_DEBUG_ARRAY_UINT(outputs->svp0_dst_lines, ctx->display_cfg->num_streams);
 	DML_LOG_DEBUG_ARRAY_UINT(outputs->svp1_dst_lines, ctx->display_cfg->num_streams);
@@ -2334,7 +2371,7 @@ static void dcn6_ms_calculate_alternate_svp_lines(const struct dml2_core_calcula
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_max_vstartup(
+void dcn6_ms_calculate_max_vstartup(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2350,7 +2387,7 @@ static void dcn6_ms_calculate_max_vstartup(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		stream_index = display_cfg->plane_descriptors[k].stream_index;
 		stream = &display_cfg->stream_descriptors[stream_index];
-		outputs->MaximumVStartup[k] = dcn6_calculate_max_vstartup(
+		outputs->MaximumVStartup[k] = get_calcs(ctx)->calculate_max_vstartup(
 				ip->ptoi_supported,
 				ip->vblank_nom_default_us,
 				&stream->timing,
@@ -2366,7 +2403,7 @@ static void dcn6_ms_calculate_max_vstartup(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_check_average_latency_supports(
+void dcn6_ms_check_average_latency_supports(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2380,6 +2417,13 @@ static void dcn6_ms_check_average_latency_supports(
 	DML_LOG_FUNC_ENTER();
 	outputs->support.OutstandingRequestsSupport = true;
 	outputs->support.OutstandingRequestsUrgencyAvoidance = true;
+	/* An SOP being capable of high bandwidth drives DCFCLK up, which shrinks the outstanding-request
+	 * buffer window below the request latency and (perversely) fails this check. For analysis of future
+	 * SoCs the ROB size / request limit is not yet fixed, so allow the check to be opted out. */
+	if (display_cfg->overrides.hw.outstanding_requests_check_disable) {
+		DML_LOG_FUNC_EXIT();
+		return;
+	}
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		outstanding_latency_us = soc_bb->max_outstanding_reqs
 				* inputs->support.request_size_bytes_luma[k]
@@ -2415,7 +2459,7 @@ static void dcn6_ms_check_average_latency_supports(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_mcache_setting(
+void dcn6_ms_calculate_mcache_setting(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2491,7 +2535,7 @@ static void dcn6_ms_calculate_mcache_setting(
 			p->mall_comb_mcache_l = &outputs->mall_comb_mcache_l[k];
 			p->mall_comb_mcache_c = &outputs->mall_comb_mcache_c[k];
 			p->lc_comb_mcache = &outputs->lc_comb_mcache[k];
-			dcn5_calculate_mcache_setting(ctx->func_params, p);
+			get_calcs(ctx)->calculate_mcache_setting(ctx->func_params, p);
 		}
 	}
 
@@ -2517,7 +2561,7 @@ static void dcn6_ms_calculate_mcache_setting(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_avg_bandwidth_and_dcfclk_lb_required(
+void dcn6_ms_calculate_avg_bandwidth_and_dcfclk_lb_required(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2527,7 +2571,7 @@ static void dcn6_ms_calculate_avg_bandwidth_and_dcfclk_lb_required(
 
 	DML_LOG_FUNC_ENTER();
 	// Average BW support check
-	dcn5_calculate_avg_bandwidth_required(
+	get_calcs(ctx)->calculate_avg_bandwidth_required(
 			*outputs->support.avg_bandwidth_required,
 			// input
 			display_cfg->num_planes,
@@ -2544,7 +2588,7 @@ static void dcn6_ms_calculate_avg_bandwidth_and_dcfclk_lb_required(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_urgent_latency_hiding_support(
+bool dcn6_ms_check_urgent_latency_hiding_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2569,7 +2613,7 @@ static bool dcn6_ms_check_urgent_latency_hiding_support(
 	return outputs->support.EnoughUrgentLatencyHidingSupport;
 }
 
-static void dcn6_ms_calculate_t_calc(
+void dcn6_ms_calculate_t_calc(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2584,7 +2628,7 @@ static void dcn6_ms_calculate_t_calc(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_hostvm_inefficiency_factor(
+void dcn6_ms_calculate_hostvm_inefficiency_factor(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2594,7 +2638,7 @@ static void dcn6_ms_calculate_hostvm_inefficiency_factor(
 	struct dml2_core_internal_mode_support *outputs = states;
 
 	DML_LOG_FUNC_ENTER();
-	dcn5_calculate_hostvm_inefficiency_factor(
+	get_calcs(ctx)->calculate_hostvm_inefficiency_factor(
 			&outputs->HostVMInefficiencyFactor,
 			&outputs->HostVMInefficiencyFactorPrefetch,
 			display_cfg->gpuvm_enable,
@@ -2611,7 +2655,7 @@ static void dcn6_ms_calculate_hostvm_inefficiency_factor(
 
 // Using approximate ratio for VM bandwidth
 
-static void dcn6_ms_calculate_3dlut_settings(
+void dcn6_ms_calculate_3dlut_settings(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2647,7 +2691,7 @@ static void dcn6_ms_calculate_3dlut_settings(
 		p->tdlut_opt_time = &outputs->tdlut_opt_time[k];
 		p->tdlut_drain_time = &outputs->tdlut_drain_time[k];
 		p->tdlut_bytes_per_group = &outputs->tdlut_bytes_per_group[k];
-		dcn5_calculate_tdlut_setting(ctx->func_params, p);
+		get_calcs(ctx)->calculate_tdlut_setting(ctx->func_params, p);
 	}
 
 	DML_LOG_DEBUG_ARRAY_UINT(outputs->tdlut_bytes_per_group, display_cfg->num_planes);
@@ -2659,7 +2703,7 @@ static void dcn6_ms_calculate_3dlut_settings(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_urgent_latency(
+void dcn6_ms_calculate_urgent_latency(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2670,7 +2714,7 @@ static void dcn6_ms_calculate_urgent_latency(
 	struct dml2_core_internal_mode_support *outputs = states;
 
 	DML_LOG_FUNC_ENTER();
-	dcn5_calculate_extra_latency(
+	get_calcs(ctx)->calculate_extra_latency(
 			display_cfg,
 			ip->rob_buffer_size_kbytes,
 			0,
@@ -2730,7 +2774,7 @@ static void dcn6_ms_calculate_prefetch_schedule(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		plane = &display_cfg->plane_descriptors[k];
 		stream = &display_cfg->stream_descriptors[plane->stream_index];
-		outputs->TWait[k] = dcn5_calculate_t_wait(
+		outputs->TWait[k] = get_calcs(ctx)->calculate_t_wait(
 				plane->overrides.reserved_vblank_time_ns,
 				inputs->UrgLatency,
 				inputs->TripToMemory,
@@ -2843,7 +2887,7 @@ static void dcn6_ms_calculate_prefetch_schedule(
 		p->Tpre_rounded = &Tpre_rounded;
 		p->Tpre_oto = &Tpre_oto;
 
-		outputs->NoTimeForPrefetch[k] = dcn5_calculate_prefetch_schedule(ctx->func_params, p);
+		outputs->NoTimeForPrefetch[k] = get_calcs(ctx)->calculate_prefetch_schedule(ctx->func_params, p);
 		DML_LOG_VERBOSE("DML::%s: k=%d, dst_y_per_vm_vblank = %f\n", __func__, k, *p->dst_y_per_vm_vblank);
 		DML_LOG_VERBOSE("DML::%s: k=%d, dst_y_per_row_vblank = %f\n", __func__, k, *p->dst_y_per_row_vblank);
 		outputs->VStartupMin[k] = inputs->MaxVStartupLines[k];
@@ -2879,7 +2923,7 @@ static void dcn6_ms_calculate_prefetch_schedule(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_prefetch_support(
+bool dcn6_ms_check_prefetch_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2912,7 +2956,7 @@ static bool dcn6_ms_check_prefetch_support(
 	return outputs->support.PrefetchScheduleSupported;
 }
 
-static bool dcn6_ms_check_dynamic_metadata_support(
+bool dcn6_ms_check_dynamic_metadata_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2935,7 +2979,7 @@ static bool dcn6_ms_check_dynamic_metadata_support(
 	return outputs->support.DynamicMetadataSupported;
 }
 
-static bool dcn6_ms_check_v_ratio_in_prefetch_support(
+bool dcn6_ms_check_v_ratio_in_prefetch_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2962,7 +3006,7 @@ static bool dcn6_ms_check_v_ratio_in_prefetch_support(
 	return outputs->support.VRatioInPrefetchSupported;
 }
 
-static void dcn6_ms_calculate_urgent_burst_factor_for_prefetch(
+void dcn6_ms_calculate_urgent_burst_factor_for_prefetch(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -2980,7 +3024,7 @@ static void dcn6_ms_calculate_urgent_burst_factor_for_prefetch(
 		DML_LOG_VERBOSE("DML::%s: k=%d, Calling CalculateUrgentBurstFactor (for prefetch)\n", __func__, k);
 		DML_LOG_VERBOSE("DML::%s: k=%d, VRatioPreY=%f\n", __func__, k, inputs->VRatioPreY[k]);
 		DML_LOG_VERBOSE("DML::%s: k=%d, VRatioPreC=%f\n", __func__, k, inputs->VRatioPreC[k]);
-		dcn5_calculate_urgent_burst_factor(
+		get_calcs(ctx)->calculate_urgent_burst_factor(
 				&display_cfg->plane_descriptors[k],
 				inputs->swath_width_luma_ub[k],
 				inputs->swath_width_chroma_ub[k],
@@ -2992,6 +3036,8 @@ static void dcn6_ms_calculate_urgent_burst_factor_for_prefetch(
 				inputs->VRatioPreC[k],
 				inputs->BytePerPixelInDETY[k],
 				inputs->BytePerPixelInDETC[k],
+				inputs->UnboundedRequestEnabled,
+				inputs->CompressedBufferSizeInkByte,
 				inputs->DETBufferSizeY[k],
 				inputs->DETBufferSizeC[k],
 				/* Output */
@@ -3050,7 +3096,7 @@ static void dcn6_ms_calculate_peak_bandwidth_required(
 	p->urgent_burst_factor_prefetch_l = inputs->UrgentBurstFactorLumaPre;
 	p->urgent_burst_factor_prefetch_c = inputs->UrgentBurstFactorChromaPre;
 	p->urgent_burst_factor_prefetch_cursor = inputs->UrgentBurstFactorCursorPre;
-	dcn5_calculate_peak_bandwidth_required(ctx->func_params, p);
+	get_calcs(ctx)->calculate_peak_bandwidth_required(ctx->func_params, p);
 
 	p->urg_vactive_bandwidth_required = dummies->dummy_bw;
 	p->urg_bandwidth_required = outputs->support.urg_bandwidth_required_flip;
@@ -3058,7 +3104,7 @@ static void dcn6_ms_calculate_peak_bandwidth_required(
 	p->surface_avg_vactive_required_bw = dummies->surface_dummy_bw;
 	p->surface_peak_required_bw = outputs->surface_peak_required_bw;
 	p->inc_flip_bw = 1;
-	dcn5_calculate_peak_bandwidth_required(ctx->func_params, p);
+	get_calcs(ctx)->calculate_peak_bandwidth_required(ctx->func_params, p);
 
 	DML_LOG_DEBUG_2D_ARRAY_DOUBLE(outputs->support.urg_bandwidth_required,
 				dml2_core_internal_soc_state_max, dml2_core_internal_bw_max);
@@ -3073,7 +3119,7 @@ static void dcn6_ms_calculate_peak_bandwidth_required(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_final_prefetch_support(
+bool dcn6_ms_check_final_prefetch_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3098,7 +3144,7 @@ static bool dcn6_ms_check_final_prefetch_support(
 	return outputs->support.PrefetchSupported;
 }
 
-static void dcn6_ms_calculate_flip_schedule(
+void dcn6_ms_calculate_flip_schedule(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3115,7 +3161,7 @@ static void dcn6_ms_calculate_flip_schedule(
 	for (k = 0; k < display_cfg->num_planes; k++) {
 		plane = &display_cfg->plane_descriptors[k];
 		stream = &display_cfg->stream_descriptors[plane->stream_index];
-		dcn6_calculate_flip_schedule(
+		get_calcs(ctx)->calculate_flip_schedule(
 				ctx->func_params,
 				display_cfg->plane_descriptors[k].immediate_flip,
 				display_cfg->hostvm_enable,
@@ -3155,7 +3201,7 @@ static void dcn6_ms_calculate_flip_schedule(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_immediate_flip_support(const struct dml2_core_calculate_ms_context *ctx,
+bool dcn6_ms_check_immediate_flip_support(const struct dml2_core_calculate_ms_context *ctx,
 	struct dml2_core_internal_mode_support *states)
 {
 	const struct dml2_display_cfg *display_cfg = ctx->display_cfg;
@@ -3178,7 +3224,7 @@ static bool dcn6_ms_check_immediate_flip_support(const struct dml2_core_calculat
 	return outputs->support.ImmediateFlipSupport;
 }
 
-static void dcn6_ms_calculate_bandwidth_upper_bound(const struct dml2_core_calculate_ms_context *ctx,
+void dcn6_ms_calculate_bandwidth_upper_bound(const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
 	struct dml2_core_internal_mode_support *inputs = states;
@@ -3191,12 +3237,15 @@ static void dcn6_ms_calculate_bandwidth_upper_bound(const struct dml2_core_calcu
 		math_max3(**inputs->support.urg_bandwidth_required_flip,
 			**inputs->support.non_urg_bandwidth_required / ctx->soc_bb->fraction_of_urgent_bandwidth_nominal_target,
 			**inputs->support.non_urg_bandwidth_required_flip / ctx->soc_bb->fraction_of_urgent_bandwidth_flip_target) * 1000;
+	outputs->support.bandwidth_upper_bound.dcn5.lsdma_bandwidth_kbps = inputs->lsdma_bw_req_for_alt_kbps;
+
 	DML_LOG_DEBUG_DOUBLE(outputs->support.bandwidth_upper_bound.dcn5.non_urgent_bandwidth_kbps);
 	DML_LOG_DEBUG_DOUBLE(outputs->support.bandwidth_upper_bound.dcn5.urgent_bandwidth_kbps);
+	DML_LOG_DEBUG_DOUBLE(outputs->support.bandwidth_upper_bound.dcn5.lsdma_bandwidth_kbps);
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_qos_bandwidth_support(
+bool dcn6_ms_check_qos_bandwidth_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3214,7 +3263,7 @@ static bool dcn6_ms_check_qos_bandwidth_support(
 	return outputs->support.qos_bandwidth_support;
 }
 
-static bool dcn6_ms_check_reordering_support(
+bool dcn6_ms_check_reordering_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3242,7 +3291,7 @@ static bool dcn6_ms_check_reordering_support(
 	return outputs->support.ROBSupport;
 }
 
-static void dcn6_ms_calculate_vactive_det_fill_latency(
+void dcn6_ms_calculate_vactive_det_fill_latency(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3254,7 +3303,7 @@ static void dcn6_ms_calculate_vactive_det_fill_latency(
 	DML_LOG_FUNC_ENTER();
 	/* VActive fill time calculations (informative) */
 	for (pstate_type = 0; pstate_type < dml2_pstate_type_count; pstate_type++) {
-		dcn5_calculate_vactive_det_fill_latency(
+		get_calcs(ctx)->calculate_vactive_det_fill_latency(
 				display_cfg,
 				display_cfg->num_planes,
 				inputs->pstate_bytes_required_l[pstate_type],
@@ -3276,7 +3325,7 @@ static void dcn6_ms_calculate_vactive_det_fill_latency(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_mode_support(
+bool dcn6_ms_check_mode_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states,
 		enum dml2_status status)
@@ -3393,7 +3442,7 @@ static bool dcn6_ms_check_mode_support(
 	return outputs->support.ModeSupport;
 }
 
-static void dcn6_ms_populate_informative(
+void dcn6_ms_populate_informative(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3429,7 +3478,7 @@ static void dcn6_ms_populate_informative(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_get_plane_support_info(
+void dcn6_ms_get_plane_support_info(
 		const struct dml2_core_calculate_ms_context *ctx,
 		const struct dml2_core_internal_mode_support *states,
 		struct core_plane_support_info *plane_support,
@@ -3462,7 +3511,7 @@ static void dcn6_ms_get_plane_support_info(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_get_stream_support_info(
+void dcn6_ms_get_stream_support_info(
 		const struct dml2_core_calculate_ms_context *ctx,
 		const struct dml2_core_internal_mode_support *states,
 		struct core_stream_support_info *stream_support,
@@ -3476,7 +3525,7 @@ static void dcn6_ms_get_stream_support_info(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_populate_mode_support_result(
+void dcn6_ms_populate_mode_support_result(
 		const struct dml2_core_calculate_ms_context *ctx,
 		const struct dml2_core_internal_mode_support *states,
 		struct dml2_core_mode_support_result *result)
@@ -3590,7 +3639,7 @@ static void dcn6_ms_populate_mode_support_result(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_dcfclk_support(
+bool dcn6_ms_check_dcfclk_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3620,12 +3669,10 @@ static bool dcn6_ms_check_dcfclk_support(
 	return outputs->support.dcfclk_support;
 }
 
-static void dcn6_ms_check_alternate_channel_size_support(
+void dcn6_ms_check_alternate_channel_size_support(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
-	unsigned int i;
-	bool alt_chan_in_use = false;
 	const struct dml2_utm_soc_bb *soc_bb = ctx->soc_bb;
 	struct dml2_core_internal_mode_support *inputs = states;
 	struct dml2_core_internal_mode_support *outputs = states;
@@ -3635,16 +3682,8 @@ static void dcn6_ms_check_alternate_channel_size_support(
 	DML_LOG_FUNC_ENTER();
 	outputs->support.alternate_channel_size_support = true;
 
-	//Alternate Channel Size Support Check - only fail if alternate channels are used AND exceed carveout limit
-	for (i = 0; i < ctx->display_cfg->num_planes; i++) {
-		if (ctx->display_cfg->plane_descriptors[i].overrides.uclk_pstate_change_strategy == dml2_uclk_pstate_change_strategy_force_alternate) {
-			alt_chan_in_use = true;
-			break;
-		}
-	}
-
-	if (alt_chan_in_use && (inputs->svp0_max_bytes > alternate_carveout_size_bytes ||
-					inputs->svp1_max_bytes > alternate_carveout_size_bytes)) {
+	if (inputs->svp0_max_bytes > alternate_carveout_size_bytes ||
+			inputs->svp1_max_bytes > alternate_carveout_size_bytes) {
 		outputs->support.alternate_channel_size_support = false;
 	}
 
@@ -3656,7 +3695,7 @@ static void dcn6_ms_check_alternate_channel_size_support(
 	return;
 }
 
-static void dcn6_ms_calculate_watermarks(
+void dcn6_ms_calculate_watermarks(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3740,7 +3779,7 @@ static void dcn6_ms_calculate_watermarks(
 	p->temp_read_or_ppt_support = outputs->support.temp_read_or_ppt_support;
 	p->VActiveLatencyHidingMargin = outputs->VActiveLatencyHidingMargin;
 	p->VActiveLatencyHidingUs = outputs->VActiveLatencyHidingUs;
-	dcn6_calculate_watermarks_and_dram_speed_change_support(ctx->func_params, p);
+	get_calcs(ctx)->calculate_watermarks_and_dram_speed_change_support(ctx->func_params, p);
 
 	DML_LOG_DEBUG_BOOL(outputs->support.global_dram_clock_change_supported);
 	DML_LOG_DEBUG_BOOL(outputs->support.global_fclk_change_supported);
@@ -3769,7 +3808,7 @@ static void dcn6_ms_calculate_watermarks(
 	DML_LOG_FUNC_EXIT();
 }
 
-static void dcn6_ms_calculate_pstate_schedule_windows(
+void dcn6_ms_calculate_pstate_schedule_windows(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3798,7 +3837,7 @@ static void dcn6_ms_calculate_pstate_schedule_windows(
 
 	/* fclk pstate */
 	if (inputs->support.global_fclk_change_supported)
-		dcn6_calculate_pstate_schedule_windows(
+		get_calcs(ctx)->calculate_pstate_schedule_windows(
 			display_cfg->num_planes,
 			v_blank_start,
 			v_blank_end,
@@ -3813,7 +3852,7 @@ static void dcn6_ms_calculate_pstate_schedule_windows(
 
 	/* ppt pstate */
 	if (inputs->support.global_temp_read_or_ppt_supported)
-		dcn6_calculate_pstate_schedule_windows(
+		get_calcs(ctx)->calculate_pstate_schedule_windows(
 			display_cfg->num_planes,
 			v_blank_start,
 			v_blank_end,
@@ -3830,7 +3869,7 @@ static void dcn6_ms_calculate_pstate_schedule_windows(
 
 	/* temp read pstate */
 	if (inputs->support.global_temp_read_or_ppt_supported)
-		dcn6_calculate_pstate_schedule_windows(
+		get_calcs(ctx)->calculate_pstate_schedule_windows(
 			display_cfg->num_planes,
 			v_blank_start,
 			v_blank_end,
@@ -3852,7 +3891,7 @@ static void dcn6_ms_calculate_pstate_schedule_windows(
 	DML_LOG_FUNC_EXIT();
 }
 
-static bool dcn6_ms_check_pstate_schedule_admissibility(
+bool dcn6_ms_check_pstate_schedule_admissibility(
 		const struct dml2_core_calculate_ms_context *ctx,
 		struct dml2_core_internal_mode_support *states)
 {
@@ -3885,7 +3924,7 @@ static bool dcn6_ms_check_pstate_schedule_admissibility(
 	/* fclk pstate */
 	outputs->support.fclk_pstate_schedule_admissible = false;
 	if (inputs->support.global_fclk_change_supported)
-		dcn6_calculate_pstate_schedule_admissibility(
+		get_calcs(ctx)->calculate_pstate_schedule_admissibility(
 			display_cfg->num_planes,
 			ip->fams2_max_allow_delay_us,
 			ip->fams2_min_allow_width_us,
@@ -3905,7 +3944,7 @@ static bool dcn6_ms_check_pstate_schedule_admissibility(
 	/* ppt pstate */
 	outputs->support.ppt_pstate_schedule_admissible = false;
 	if (inputs->support.global_temp_read_or_ppt_supported)
-		dcn6_calculate_pstate_schedule_admissibility(
+		get_calcs(ctx)->calculate_pstate_schedule_admissibility(
 			display_cfg->num_planes,
 			ip->ppt_max_allow_delay_us,
 			ip->fams2_min_allow_width_us,
@@ -3925,7 +3964,7 @@ static bool dcn6_ms_check_pstate_schedule_admissibility(
 	/* temp read pstate */
 	outputs->support.temp_read_pstate_schedule_admissible = false;
 	if (inputs->support.global_temp_read_or_ppt_supported)
-		dcn6_calculate_pstate_schedule_admissibility(
+		get_calcs(ctx)->calculate_pstate_schedule_admissibility(
 			display_cfg->num_planes,
 			ip->temp_read_max_allow_delay_us,
 			ip->fams2_min_allow_width_us,
@@ -3950,7 +3989,7 @@ static bool dcn6_ms_check_pstate_schedule_admissibility(
 			&& (!inputs->temp_read_pstate_required || outputs->support.temp_read_pstate_schedule_admissible);
 }
 
-static void dcn6_ms_initialize_from_solution(struct dml2_core_internal_mode_support *outputs,
+void dcn6_ms_initialize_from_solution(struct dml2_core_internal_mode_support *outputs,
 		const struct dml2_display_solution *solution,
 		const struct dml2_utm_soc_bb *utm_soc_bb)
 {
@@ -4055,6 +4094,8 @@ static enum dml2_status dcn6_ms_validate_prefetch(
 
 		dcn6_ms_calculate_peak_bandwidth_required(ctx, states);
 
+		dcn6_ms_calculate_alternate_params(ctx, states);
+
 		dcn6_ms_calculate_bandwidth_upper_bound(ctx, states);
 
 		dcn6_ms_check_qos_bandwidth_support(ctx, states);
@@ -4069,8 +4110,6 @@ static enum dml2_status dcn6_ms_validate_prefetch(
 
 		dcn6_ms_calculate_vactive_det_fill_latency(ctx, states);
 
-		dcn6_ms_calculate_alternate_params(ctx, states);
-
 		dcn6_ms_check_alternate_channel_size_support(ctx, states);
 
 		dcn6_ms_calculate_pstate_schedule_windows(ctx, states);
@@ -4194,6 +4233,21 @@ static enum dml2_status dcn6_mode_support(
 	return status;
 }
 
+void dcn6_ms_build_calculate_ms_context(struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_instance *core,
+		const struct dml2_display_solution *solution)
+{
+	struct dml2_core_internal_display_mode_lib *mode_lib = &core->clean_me_up.mode_lib;
+
+	ctx->display_cfg = &solution->dispcfg;
+	ctx->ip = &mode_lib->ip;
+	ctx->soc_bb = core->utm_soc_bb;
+	ctx->clock_adjuster = core->clock_adjuster;
+	ctx->dummies = &mode_lib->scratch.dml_core_mode_support_locals;
+	ctx->func_params = &mode_lib->scratch;
+	ctx->calcs = &core->calcs;
+}
+
 enum dml2_status dml2_core_dcn6_funcs_validate_solution(struct dml2_core_instance *core,
 		const struct dml2_display_solution *solution,
 		struct dml2_validation_result *result)
@@ -4206,6 +4260,8 @@ enum dml2_status dml2_core_dcn6_funcs_validate_solution(struct dml2_core_instanc
 	unsigned int i;
 
 	DML_LOG_COMP_IF_ENTER();
+	dcn6_ms_build_calculate_ms_context(calc_ms_ctx, core, solution);
+
 	if (solution->unvalidated_change.bits.mpc_combine_overrides
 			|| solution->unvalidated_change.bits.odm_combine_overrides
 			|| solution->unvalidated_change.bits.reserved_vblank_time
@@ -4218,12 +4274,6 @@ enum dml2_status dml2_core_dcn6_funcs_validate_solution(struct dml2_core_instanc
 		result->is_prefetch_valid = false;
 
 	if (!result->is_mode_support_valid) {
-		calc_ms_ctx->display_cfg = &solution->dispcfg;
-		calc_ms_ctx->ip = &core->clean_me_up.mode_lib.ip;
-		calc_ms_ctx->soc_bb = core->utm_soc_bb;
-		calc_ms_ctx->clock_adjuster = core->clock_adjuster;
-		calc_ms_ctx->dummies = &mode_lib->scratch.dml_core_mode_support_locals;
-		calc_ms_ctx->func_params = &mode_lib->scratch;
 		memset(calc_ms_ctx->func_params, 0, sizeof(struct dml2_core_internal_scratch));
 		memset(&mode_lib->ms, 0, sizeof(struct dml2_core_internal_mode_support));
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.h
index c035cd099cc604..bfa5f9f583ca48 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_dcn6_funcs_mode_support.h
@@ -8,4 +8,328 @@
 enum dml2_status dml2_core_dcn6_funcs_validate_solution(struct dml2_core_instance *core,
 		const struct dml2_display_solution *solution,
 		struct dml2_validation_result *result);
+
+void dcn6_ms_initialize_from_solution(struct dml2_core_internal_mode_support *outputs,
+		const struct dml2_display_solution *solution,
+		const struct dml2_utm_soc_bb *utm_soc_bb);
+
+// Exposed for Higher DCN versions
+void dcn6_ms_populate_mode_support_result(
+	const struct dml2_core_calculate_ms_context *ctx,
+	const struct dml2_core_internal_mode_support *states,
+	struct dml2_core_mode_support_result *result);
+
+void dcn6_ms_build_calculate_ms_context(struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_instance *core,
+		const struct dml2_display_solution *solution);
+
+void dcn6_ms_check_input_sanity(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_scaler_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_source_format_and_scan_direction(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_byte_per_pixel_and_block_sizes(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_read_bandwidth(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_writeback_bandwidth(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_writeback_bandwidth_latency_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_writeback_scale_ratio_and_taps_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_single_pipe_dppclk_and_pscl_factor(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_max_swath_widths(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_cursor_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_surface_alginment_requirements(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_effective_pixel_clock(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_estimated_num_of_dsc_slices(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_desired_output_bpp(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_output_link(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_odm_mode(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_num_of_dsc_slices(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_num_of_dsc_slices_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_max_det_and_min_compressed_buffer_size(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_total_available_pipes_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_total_available_TDLUT_33cube_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_total_num_of_single_dpp_surfaces(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_dispclk_and_dppclk_required(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_dispclk_and_dppclk_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_dtbclk_required(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_dtbclk_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_otg_count_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_hpo_frl_encoder_count_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_hpo_dp_encoder_count_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_writeback_count_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_link_bandwidth_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_check_misc_link_supports(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_dscclk_required(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_dscclk_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_check_dsc_engine_supports(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_dsc_delay(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_total_num_of_dcc_active_dpp(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_det_buffer_time_value_urgent_burst_factor_and_urgent_latency_hiding(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_min_dcfclk_deepsleep_clock(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_writeback_delay(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_alternate_svp_lines(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_max_vstartup(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_mcache_setting(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_avg_bandwidth_and_dcfclk_lb_required(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_check_average_latency_supports(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_t_calc(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_hostvm_inefficiency_factor(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_3dlut_settings(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_urgent_latency(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_urgent_latency_hiding_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_dynamic_metadata_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_prefetch_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_v_ratio_in_prefetch_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_urgent_burst_factor_for_prefetch(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_final_prefetch_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_flip_schedule(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_immediate_flip_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_bandwidth_upper_bound(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_qos_bandwidth_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_dcfclk_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_watermarks(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_reordering_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_vactive_det_fill_latency(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_alternate_params(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_check_alternate_channel_size_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_pstate_schedule_windows(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_pstate_schedule_admissibility(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_mode_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states,
+		enum dml2_status status);
+
+void dcn6_ms_populate_informative(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_vm_row_and_swath_and_calculate_dcc_meta_cache_requirements(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_pte_buffer_size_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+bool dcn6_ms_check_dcc_meta_cache_support(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_calculate_vactive_pstate_requirements(
+		const struct dml2_core_calculate_ms_context *ctx,
+		struct dml2_core_internal_mode_support *states);
+
+void dcn6_ms_get_plane_support_info(
+		const struct dml2_core_calculate_ms_context *ctx,
+		const struct dml2_core_internal_mode_support *states,
+		struct core_plane_support_info *plane_support,
+		unsigned int plane_idx);
+
+void dcn6_ms_get_stream_support_info(
+		const struct dml2_core_calculate_ms_context *ctx,
+		const struct dml2_core_internal_mode_support *states,
+		struct core_stream_support_info *stream_support,
+		unsigned int plane_index);
+
 #endif /* __DML2_CORE_DCN6_FUNCS_MODE_SUPPORT_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
index 28f4a53d061743..9eabee1fc0015a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_shared_types.h
@@ -1039,6 +1039,7 @@ struct dml2_core_internal_mode_program {
 	double MaxActiveFCLKChangeLatencySupported;
 	bool USRRetrainingSupport;
 	bool g6_temp_read_support;
+	bool global_z8_stutter_supported;
 	enum dml2_pstate_change_support FCLKChangeSupport[DML2_MAX_PLANES];
 	enum dml2_pstate_change_support DRAMClockChangeSupport[DML2_MAX_PLANES];
 	enum dml2_pstate_change_support temp_read_or_ppt_support[DML2_MAX_PLANES];
@@ -1069,6 +1070,7 @@ struct dml2_core_internal_mode_program {
 	double urg_bandwidth_available[dml2_core_internal_soc_state_max][dml2_core_internal_bw_max];
 	double urg_bandwidth_available_vm_only[dml2_core_internal_soc_state_max]; // the min of sdp bw and dram_vm_only bw, sdp has no different derate for vm/non-vm traffic etc.
 	double urg_bandwidth_available_pixel_and_vm[dml2_core_internal_soc_state_max]; // the min of sdp bw and dram_pixel_and_vm bw, sdp has no different derate for vm/non-vm etc.
+	double non_urg_bandwidth_available[dml2_core_internal_soc_state_max][dml2_core_internal_bw_max];
 
 	double dcc_dram_bw_nom_overhead_factor_p0[DML2_MAX_PLANES];
 	double dcc_dram_bw_nom_overhead_factor_p1[DML2_MAX_PLANES];
@@ -1657,6 +1659,9 @@ struct dml2_core_shared_CalculateFlipSchedule_locals {
 	double num_rows;
 	double hvm_scaled_row_bytes;
 	double hvm_scaled_vm_row_bytes;
+	double vm_time_budget;
+	double row_time_budget;
+	double vm_and_row_time_budget;
 	bool dual_plane;
 };
 
@@ -1841,6 +1846,7 @@ struct dml2_core_calcs_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport_param
 	bool *g6_temp_read_support;
 	enum dml2_pstate_change_support *temp_read_or_ppt_support;
 	bool *global_temp_read_or_ppt_supported;
+	bool *global_z8_stutter_supported;
 };
 
 struct dml2_core_calcs_CalculateSwathAndDETConfiguration_params {
@@ -2295,6 +2301,8 @@ struct dml2_core_calcs_calculate_alternate_params {
 	unsigned int *Read256BlockHeightC;
 	unsigned int *MacroTileWidthY;
 	unsigned int *MacroTileWidthC;
+	unsigned int *MacroTileHeightY;
+	unsigned int *MacroTileHeightC;
 	unsigned int *VInitPrefillY;
 	unsigned int *VInitPrefillC;
 	double *VRatioPrefetchY;
@@ -2302,16 +2310,17 @@ struct dml2_core_calcs_calculate_alternate_params {
 	unsigned int *NoOfDPP;
 	unsigned int max_num_dpp;
 	double dram_blackout_us;
-	double *VActiveLatencyHidingUs;
 	unsigned int *svp0_dst_lines;
 	unsigned int *svp1_dst_lines;
 	unsigned int *svp_req_limit;
 	double dcn_non_urgent_bandwidth_kbps;
+	double max_lsdma_bandwidth_kbps;
 	unsigned int alt_chan_fw_delay_us;
 	double *dst_y_per_vm_vblank;
 	double *dst_y_per_row_vblank;
 	unsigned int *DSTYAfterScaler;
 	enum dml2_odm_mode *ODMMode;
+	bool alt_chan_in_use;
 
 	/* output params */
 	unsigned int *svp0_max_bytes;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
index 22c757cdac07ea..54a1f9c13dbf09 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_dcn4.c
@@ -6,6 +6,8 @@
 #include "dml2_internal_shared_types.h"
 #include "dml_top_types.h"
 #include "lib_float_math.h"
+#include "dml2_debug.h"
+
 
 static double dram_bw_kbps_to_uclk_khz(unsigned long long bandwidth_kbps, const struct dml2_dram_params *dram_config, struct dml2_mcg_dram_bw_to_min_clk_table *dram_bw_table)
 {
@@ -55,16 +57,101 @@ static unsigned long dml_round_up(double a)
 	return (unsigned long)a;
 }
 
+/* Calculate DPM index to use for per-DPM derates
+ * Iterates through DPM levels - if calculated clocks exceed thresholds increment to next DPM level
+ */
+static void calculate_derate_index(struct dml2_dpmm_map_mode_to_soc_dpm_params_in_out *in_out)
+{
+	const struct dml2_core_mode_support_result *mode_support_result = &in_out->display_cfg->mode_support_result;
+	double required_bw_dram_kbps, required_bw_sdp_kbps;
+	int i;
+
+	/* Default to DPM 0 (lowest power state) */
+	in_out->derate_dpm_index = 0;
+
+	/* Get required bandwidths from mode support */
+	required_bw_dram_kbps = mode_support_result->global.active.average_bw_dram_kbps;
+	required_bw_sdp_kbps = mode_support_result->global.active.average_bw_sdp_kbps;
+
+	/* Iterate through DPM levels from lowest to highest to find the first level
+	 * whose available bandwidth can satisfy the requirements.
+	 */
+	for (i = 0; i < DML_MAX_CLK_TABLE_SIZE; i++) {
+		unsigned int dram_threshold = in_out->soc_bb->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[i].clk_upperbound_threshold_khz;
+		unsigned int fclk_threshold = in_out->soc_bb->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[i].clk_upperbound_threshold_khz;
+		unsigned int dcfclk_threshold = in_out->soc_bb->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[i].clk_upperbound_threshold_khz;
+		unsigned int uclk_derate = in_out->soc_bb->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[i].derate_percent;
+		unsigned int fclk_derate = in_out->soc_bb->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[i].derate_percent;
+		unsigned int dcfclk_derate = in_out->soc_bb->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[i].derate_percent;
+		double max_dram_bw_for_dpm = 0, max_fclk_bw_for_dpm = 0, max_dcfclk_bw_for_dpm = 0;
+
+		/* Skip entries with 0 derate (use fallback logic) */
+		if (uclk_derate == 0 || fclk_derate == 0 || dcfclk_derate == 0) {
+			DML_ASSERT(0); // derate should be populated (at least 1 entry in the per_dpm table that can be applied for all DPMs
+		}
+
+		/* Threshold of 0 indicates last entry - this DPM level handles all higher bandwidths */
+		if (dram_threshold == 0 && fclk_threshold == 0 && dcfclk_threshold == 0) {
+			in_out->derate_dpm_index = i;
+			break;
+		}
+
+		/* Calculate maximum available bandwidth at this DPM level
+			* Available BW = max_clock_at_threshold * derate * conversion_factor */
+
+			/* For DRAM: convert threshold clock to bandwidth */
+		if (!in_out->soc_bb->clk_table.dram_config.alt_clock_bw_conversion) {
+			unsigned long uclk_bytes_per_tick = in_out->soc_bb->clk_table.dram_config.channel_count
+				* in_out->soc_bb->clk_table.dram_config.channel_width_bytes
+				* in_out->soc_bb->clk_table.dram_config.transactions_per_clock;
+			max_dram_bw_for_dpm = (double)dram_threshold * uclk_bytes_per_tick * ((double)uclk_derate / 100.0);
+		} else {
+			/* use pre-computed table - find bandwidth at threshold clock */
+			for (unsigned int j = 0; j < in_out->min_clk_table->dram_bw_table.num_entries; j++) {
+				if (in_out->min_clk_table->dram_bw_table.entries[j].min_uclk_khz >= dram_threshold) {
+					max_dram_bw_for_dpm = (double)in_out->min_clk_table->dram_bw_table.entries[j].pre_derate_dram_bw_kbps;
+					break;
+				}
+			}
+		}
+
+		max_fclk_bw_for_dpm = (double)fclk_threshold
+			* in_out->soc_bb->fabric_datapath_to_dcn_data_return_bytes
+			* ((double)fclk_derate / 100.0);
+
+		max_dcfclk_bw_for_dpm = (double)dcfclk_threshold
+			* in_out->soc_bb->return_bus_width_bytes
+			* ((double)dcfclk_derate / 100.0);
+
+		/* Check if this DPM level's available bandwidth satisfies all requirements
+			* All bandwidth paths must be satisfied to use this DPM level */
+		if (((dram_threshold == 0) || (required_bw_dram_kbps <= max_dram_bw_for_dpm)) &&
+			((fclk_threshold == 0) || (required_bw_sdp_kbps <= max_fclk_bw_for_dpm)) &&
+			((dcfclk_threshold == 0) || (required_bw_sdp_kbps <= max_dcfclk_bw_for_dpm))) {
+			in_out->derate_dpm_index = i;
+			break;
+		}
+	}
+}
+
 static void calculate_system_active_minimums(struct dml2_dpmm_map_mode_to_soc_dpm_params_in_out *in_out)
 {
 	double min_uclk_avg, min_uclk_urgent, min_uclk_bw;
 	double min_fclk_avg, min_fclk_urgent, min_fclk_bw;
 	double min_dcfclk_avg, min_dcfclk_urgent, min_dcfclk_bw;
 	double min_uclk_latency, min_fclk_latency, min_dcfclk_latency;
+	int dpm_index, uclk_avg_derate_percent, fclk_avg_derate_percent, dcfclk_avg_derate_percent;
 	const struct dml2_core_mode_support_result *mode_support_result = &in_out->display_cfg->mode_support_result;
 
+	dpm_index = in_out->derate_dpm_index;
+
+	/* Get average derates from system_active_average per-DPM table */
+	uclk_avg_derate_percent = (in_out->soc_bb->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[dpm_index].derate_percent != 0 || dpm_index == 0) ?
+		in_out->soc_bb->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[dpm_index].derate_percent :
+		in_out->soc_bb->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[0].derate_percent;
+
 	min_uclk_avg = dram_bw_kbps_to_uclk_khz((unsigned long long)(mode_support_result->global.active.average_bw_dram_kbps
-											/ ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_average.dram_derate_percent_pixel / 100)),
+											/ ((double)uclk_avg_derate_percent / 100)),
 							&in_out->soc_bb->clk_table.dram_config, &in_out->min_clk_table->dram_bw_table);
 
 	if (in_out->display_cfg->display_config.hostvm_enable)
@@ -78,16 +165,22 @@ static void calculate_system_active_minimums(struct dml2_dpmm_map_mode_to_soc_dp
 
 	min_uclk_bw = min_uclk_urgent > min_uclk_avg ? min_uclk_urgent : min_uclk_avg;
 
+	fclk_avg_derate_percent = (in_out->soc_bb->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[dpm_index].derate_percent != 0 || dpm_index == 0) ?
+		in_out->soc_bb->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[dpm_index].derate_percent :
+		in_out->soc_bb->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[0].derate_percent;
 	min_fclk_avg = (double)mode_support_result->global.active.average_bw_sdp_kbps / in_out->soc_bb->fabric_datapath_to_dcn_data_return_bytes;
-	min_fclk_avg = (double)min_fclk_avg / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_average.fclk_derate_percent / 100);
+	min_fclk_avg = (double)min_fclk_avg / ((double)fclk_avg_derate_percent / 100);
 
 	min_fclk_urgent = (double)mode_support_result->global.active.urgent_bw_sdp_kbps / in_out->soc_bb->fabric_datapath_to_dcn_data_return_bytes;
 	min_fclk_urgent = (double)min_fclk_urgent / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_urgent.fclk_derate_percent / 100);
 
 	min_fclk_bw = min_fclk_urgent > min_fclk_avg ? min_fclk_urgent : min_fclk_avg;
 
+	dcfclk_avg_derate_percent = (in_out->soc_bb->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[dpm_index].derate_percent != 0 || dpm_index == 0) ?
+		in_out->soc_bb->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[dpm_index].derate_percent :
+		in_out->soc_bb->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[0].derate_percent;
 	min_dcfclk_avg = (double)mode_support_result->global.active.average_bw_sdp_kbps / in_out->soc_bb->return_bus_width_bytes;
-	min_dcfclk_avg = (double)min_dcfclk_avg / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_average.dcfclk_derate_percent / 100);
+	min_dcfclk_avg = (double)min_dcfclk_avg / ((double)dcfclk_avg_derate_percent / 100);
 
 	min_dcfclk_urgent = (double)mode_support_result->global.active.urgent_bw_sdp_kbps / in_out->soc_bb->return_bus_width_bytes;
 	min_dcfclk_urgent = (double)min_dcfclk_urgent / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_urgent.dcfclk_derate_percent / 100);
@@ -145,7 +238,7 @@ static void calculate_svp_prefetch_minimums(struct dml2_dpmm_map_mode_to_soc_dpm
 
 	/* assumes DF throttling is disabled */
 	min_uclk_avg = dram_bw_kbps_to_uclk_khz((unsigned long long)(mode_support_result->global.svp_prefetch.average_bw_dram_kbps
-										/ ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_average.dram_derate_percent_pixel / 100)),
+										/ ((double)in_out->soc_bb->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[in_out->derate_dpm_index].derate_percent / 100)),
 								&in_out->soc_bb->clk_table.dram_config, &in_out->min_clk_table->dram_bw_table);
 
 	min_uclk_urgent = dram_bw_kbps_to_uclk_khz((unsigned long long)(mode_support_result->global.svp_prefetch.urgent_bw_dram_kbps
@@ -155,7 +248,7 @@ static void calculate_svp_prefetch_minimums(struct dml2_dpmm_map_mode_to_soc_dpm
 	min_uclk_bw = min_uclk_urgent > min_uclk_avg ? min_uclk_urgent : min_uclk_avg;
 
 	min_fclk_avg = (double)mode_support_result->global.svp_prefetch.average_bw_sdp_kbps / in_out->soc_bb->fabric_datapath_to_dcn_data_return_bytes;
-	min_fclk_avg = (double)min_fclk_avg / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_average.fclk_derate_percent / 100);
+	min_fclk_avg = (double)min_fclk_avg / ((double)in_out->soc_bb->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[in_out->derate_dpm_index].derate_percent / 100);
 
 	min_fclk_urgent = (double)mode_support_result->global.svp_prefetch.urgent_bw_sdp_kbps / in_out->soc_bb->fabric_datapath_to_dcn_data_return_bytes;
 	min_fclk_urgent = (double)min_fclk_urgent / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_urgent.fclk_derate_percent / 100);
@@ -163,7 +256,7 @@ static void calculate_svp_prefetch_minimums(struct dml2_dpmm_map_mode_to_soc_dpm
 	min_fclk_bw = min_fclk_urgent > min_fclk_avg ? min_fclk_urgent : min_fclk_avg;
 
 	min_dcfclk_avg = (double)mode_support_result->global.svp_prefetch.average_bw_sdp_kbps / in_out->soc_bb->return_bus_width_bytes;
-	min_dcfclk_avg = (double)min_dcfclk_avg / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_average.dcfclk_derate_percent / 100);
+	min_dcfclk_avg = (double)min_dcfclk_avg / ((double)in_out->soc_bb->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[in_out->derate_dpm_index].derate_percent / 100);
 
 	min_dcfclk_urgent = (double)mode_support_result->global.svp_prefetch.urgent_bw_sdp_kbps / in_out->soc_bb->return_bus_width_bytes;
 	min_dcfclk_urgent = (double)min_dcfclk_urgent / ((double)in_out->soc_bb->qos_parameters.derate_table.system_active_urgent.dcfclk_derate_percent / 100);
@@ -633,6 +726,7 @@ static bool map_mode_to_soc_dpm(struct dml2_dpmm_map_mode_to_soc_dpm_params_in_o
 	double dispclk_khz;
 	const struct dml2_core_mode_support_result *mode_support_result = &in_out->display_cfg->mode_support_result;
 
+	calculate_derate_index(in_out);
 	calculate_system_active_minimums(in_out);
 	calculate_svp_prefetch_minimums(in_out);
 	calculate_idle_minimums(in_out);
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c
index 7ae02eccd598b7..de078ccd91a2af 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_dcn4_fams2.c
@@ -887,6 +887,7 @@ bool pmo_dcn4_fams2_optimize_for_vmin(struct dml2_pmo_optimize_for_vmin_in_out *
 	const struct dml2_core_mode_support_result *mode_support_result =
 			&in_out->base_display_config->mode_support_result;
 	unsigned int odms_used;
+	int odm_combine_limit;
 	struct dml2_stream_parameters *stream_descriptor;
 	bool optimizable = false;
 
@@ -901,8 +902,16 @@ bool pmo_dcn4_fams2_optimize_for_vmin(struct dml2_pmo_optimize_for_vmin_in_out *
 			in_out->base_display_config->stage4.unoptimizable_streams[stream_index])
 		return false;
 
+	/*
+	 * Enforce a max cap on ODM combine factor of 2:1 for eDP stream
+	 */
+	odm_combine_limit = in_out->instance->odm_combine_limit;
+	if (display_config->stream_descriptors[stream_index].output.output_encoder == dml2_edp &&
+			odm_combine_limit > 2)
+		odm_combine_limit = 2;
+
 	odms_used = mode_support_result->cfg_support_info.stream_support_info[stream_index].odms_used;
-	if ((int)odms_used >= in_out->instance->odm_combine_limit)
+	if ((int)odms_used >= odm_combine_limit)
 		return false;
 
 	memcpy(in_out->optimized_display_config,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c
index 53ead9f687ac92..f082f058001e1f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/alternate_pstate_shared_lib.c
@@ -884,27 +884,7 @@ static void populate_lsdma_start_end_lines(
 
 static uint8_t element_size_to_bytes_per_pixel(uint8_t element_size)
 {
-	uint8_t bytes_per_pixel = 0;
-
-	switch (element_size) { // 0: 8bpp, 1: 16bpp, 2: 32bpp, 3: 64bpp, 4: 128bpp - Values supported by LSDMA Controller
-	case 0:
-		bytes_per_pixel = 1;
-		break;
-	case 1:
-		bytes_per_pixel = 2;
-		break;
-	case 2:
-		bytes_per_pixel = 4;
-		break;
-	case 3:
-		bytes_per_pixel = 8;
-		break;
-	case 4:
-		bytes_per_pixel = 16;
-		break;
-	}
-
-	return bytes_per_pixel;
+	return element_size < 8 ? (uint8_t)(1U << element_size) : 1;
 }
 
 static void populate_lsdma(
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_utm.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_utm.c
index ab9cadaf5a603b..77c9308bb9e6ac 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_utm.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_utm.c
@@ -123,7 +123,7 @@ static enum dml2_status dml2_top_perform_stage_optimization(struct dml2_instance
 	DML_ASSERT_MSG(worksheet->validation_result.is_mode_support_valid
 			&& worksheet->validation_result.is_mcache_allocation_valid
 			&& worksheet->validation_result.is_prefetch_valid,
-			"worksheet must be valid on exit independent from optmization resul!\n");
+			"worksheet must be valid on exit independent from optimization result!\n");
 //	DML_ASSERT_MSG(iteration <= MAX_OPTIMIZATION_ITERATIONS,
 //			"exceeds max optimization iterations!\n"
 //			"\t is_permissible_found = %s\n"
@@ -290,7 +290,7 @@ static bool dml2_top_utm_check_mode_supported(struct dml2_check_mode_supported_i
 	DML_LOG_INFO("%s exit with %s\n", __func__, dml2_status_str(status));
 	DML_LOG_TOP_IF_EXIT();
 
-	return true;
+	return status == DML2_STATUS_OK;
 }
 
 static bool dml2_top_utm_build_mode_programming(struct dml2_build_mode_programming_in_out *in_out)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_utm_soc_bb/dml2_utm_soc_bb_dcn6.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_utm_soc_bb/dml2_utm_soc_bb_dcn6.c
index 9c6892edca8ad2..9fce6b472aa6cc 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_utm_soc_bb/dml2_utm_soc_bb_dcn6.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_utm_soc_bb/dml2_utm_soc_bb_dcn6.c
@@ -32,6 +32,8 @@ static void dcn6_sop_table_get_sop_constraint_at_index(const struct dml2_sop_tab
 	constraint->dcn5.latency.dcn5.avg_req_latency_non_urg = dchub->latencies[index].avg_req_latency_non_urg_ps / 1000000.0;
 	constraint->dcn5.latency.dcn5.df_response_time_us = dchub->latencies[index].df_response_time_ps / 1000000.0;
 	constraint->dcn5.min_available_urgent_bandwidth_KBps = table->sop_min_available_urgent_bandwidths_KBps[index];
+	constraint->dcn5.min_available_non_urgent_bandwidth_KBps = table->sop_min_available_non_urgent_bandwidths_KBps[index];
+
 	constraint->dcn5.min_sop_index = index;
 }
 
@@ -134,6 +136,9 @@ static void dml2_utm_soc_bb_dcn6_build_sop_table(struct dml2_sop_table *table,
 		table->sop_min_available_urgent_bandwidths_KBps[i] = (uint32_t) math_floor(
 				total_available_bandwidth.urgent_bandwidth_KBps
 				* (utm_soc_bb->qos_model.dchub_v2->min_urgent_utm_budget_percent / 100.0));
+		table->sop_min_available_non_urgent_bandwidths_KBps[i] = (uint32_t)math_floor(
+			total_available_bandwidth.nominal_bandwidth_KBps
+			* (utm_soc_bb->qos_model.dchub_v2->min_nominal_utm_budget_percent / 100.0));
 	}
 
 	DML_ASSERT_MSG(table->model->sop_count > 0, "qos_model must contain at least 1 sop\n");
@@ -181,6 +186,8 @@ static bool dcn6_v3_sop_table_is_bandwidth_supported_at_index(
 			&dchub->sops[UTM_QOS_MODEL_V3_LOAD_LEVEL_IDLE][index];
 	const struct utm_qos_model_dchub_v3_sop_entry *active_entry =
 			&dchub->sops[UTM_QOS_MODEL_V3_LOAD_LEVEL_ACTIVE_ALTERNATE_PSTATE][highest_sop_index];
+	const struct utm_qos_model_dchub_v3_sop_entry *pstate_entry =
+			&dchub->sops[UTM_QOS_MODEL_V3_LOAD_LEVEL_ACTIVE_ALTERNATE_PSTATE][index];
 
 	if (bw->dcn5.non_urgent_bandwidth_kbps > idle_entry->nominal_bandwidth_KBps
 			|| bw->dcn5.urgent_bandwidth_kbps > idle_entry->urgent_bandwidth_KBps)
@@ -190,6 +197,10 @@ static bool dcn6_v3_sop_table_is_bandwidth_supported_at_index(
 			|| bw->dcn5.urgent_bandwidth_kbps > active_entry->urgent_bandwidth_KBps)
 		return false;
 
+	/* check if the requested lsdma bandwidth fits within the current sop's alt-pstate lsdma budget */
+	if (bw->dcn5.lsdma_bandwidth_kbps > pstate_entry->lsdma_bandwidth_KBps)
+		return false;
+
 	return true;
 }
 
@@ -277,8 +288,6 @@ static void dcn6_copy_utm_qos_model(struct utm_qos_model *dest, struct utm_qos_m
 static void dcn6_initialize_from_soc_bb(struct dml2_utm_soc_bb *utm_soc_bb,
 		const struct dml2_soc_bb *soc_bb)
 {
-	DML_ASSERT_MSG(soc_bb->clk_table.dcfclk.num_clk_values == 2, "soc_bb must provide min and max dcfclk values!\n");
-
 	/* initialize based on soc bb */
 	utm_soc_bb->max_dispclk_khz = soc_bb->clk_table.dispclk.clk_values_khz[soc_bb->clk_table.dispclk.num_clk_values - 1];
 	utm_soc_bb->max_dppclk_khz = soc_bb->clk_table.dppclk.clk_values_khz[soc_bb->clk_table.dppclk.num_clk_values - 1];
@@ -321,6 +330,8 @@ static void dcn6_initialize_from_soc_bb(struct dml2_utm_soc_bb *utm_soc_bb,
 	utm_soc_bb->lower_bound_bandwidth_dchub = soc_bb->lower_bound_bandwidth_dchub;
 	utm_soc_bb->fraction_of_urgent_bandwidth_nominal_target = soc_bb->fraction_of_urgent_bandwidth_nominal_target;
 	utm_soc_bb->fraction_of_urgent_bandwidth_flip_target = soc_bb->fraction_of_urgent_bandwidth_flip_target;
+	utm_soc_bb->hostvm_inefficiency_fraction = soc_bb->hostvm_inefficiency_fraction;
+	utm_soc_bb->max_lsdma_bandwidth_kbps = soc_bb->max_lsdma_bandwidth_kbps;
 }
 
 static void dcn6_initialize_from_qos_model(struct dml2_utm_soc_bb *utm_soc_bb,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
index a18f067fe53fd7..dc9d4be9d7f73e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/inc/dml2_internal_shared_types.h
@@ -87,6 +87,7 @@ struct dml2_sop_constraint {
 			struct dml2_memory_path_latency latency;
 			struct dml2_soc_operating_point clocks;
 			double min_available_urgent_bandwidth_KBps; // minimum guaranteed urgent bandwidth at active
+			double min_available_non_urgent_bandwidth_KBps; // minimum guaranteed nominal bandwidth at active
 		} dcn5;
 	};
 };
@@ -95,6 +96,7 @@ struct dml2_sop_table {
 	bool is_initialized;
 	const struct utm_qos_model *model;
 	uint32_t sop_min_available_urgent_bandwidths_KBps[MAX_UTM_SOP_COUNT];
+	uint32_t sop_min_available_non_urgent_bandwidths_KBps[MAX_UTM_SOP_COUNT];
 	uint32_t sop_optimal_dcfclks_khz[MAX_UTM_SOP_COUNT];
 	unsigned int (*get_highest_sop_index)(const struct dml2_sop_table *sop_table);
 	void (*get_sop_constraint_at_index)(const struct dml2_sop_table *sop_table, unsigned int index, struct dml2_sop_constraint *constraint);
@@ -117,6 +119,8 @@ struct dml2_utm_soc_bb {
 	double lower_bound_bandwidth_dchub;
 	double fraction_of_urgent_bandwidth_nominal_target;
 	double fraction_of_urgent_bandwidth_flip_target;
+	double hostvm_inefficiency_fraction;
+	unsigned int max_lsdma_bandwidth_kbps;
 	unsigned int dchub_refclk_mhz;
 	unsigned int max_outstanding_reqs;
 	unsigned long return_bus_width_bytes;
@@ -246,6 +250,7 @@ struct dml2_dpmm_map_mode_to_soc_dpm_params_in_out {
 	* Output
 	*/
 	struct dml2_display_cfg_programming *programming;
+	int derate_dpm_index;
 };
 
 struct dml2_dpmm_map_watermarks_params_in_out {
@@ -829,6 +834,17 @@ struct dml2_core_internal_state_inputs {
 struct dml2_core_internal_state_intermediates {
 	unsigned int dummy;
 };
+/*
+ * Per-generation table of DML2 core "calcs" (leaf computation) functions.
+ * Each generation registers exactly one table pointer in this union.  The funcs
+ * layer reaches the active table through get_calcs(ctx), keeping it free of
+ * direct references to generation-specific calcs symbols.
+ */
+struct dml2_core_dcn6_calcs;
+
+union dml2_core_calcs {
+	const struct dml2_core_dcn6_calcs *dcn6;
+};
 
 struct dml2_core_calculate_mp_context {
 	const struct dml2_display_cfg *display_cfg;
@@ -837,6 +853,7 @@ struct dml2_core_calculate_mp_context {
 	const struct dml2_core_internal_mode_support *ms;
 	struct dml2_core_calcs_mode_programming_locals *dummies;
 	struct dml2_core_internal_scratch *func_params;
+	union dml2_core_calcs *calcs;
 };
 struct dml2_core_calculate_ms_context {
 	const struct dml2_display_cfg *display_cfg;
@@ -845,6 +862,7 @@ struct dml2_core_calculate_ms_context {
 	const struct dml2_clock_granularity_adjuster *clock_adjuster;
 	struct dml2_core_calcs_mode_support_locals *dummies;
 	struct dml2_core_internal_scratch *func_params;
+	union dml2_core_calcs *calcs;
 };
 
 struct dml2_core_mode_support_locals {
@@ -899,6 +917,7 @@ struct dml2_core_instance {
 	struct {
 		struct dml2_core_internal_display_mode_lib mode_lib;
 	} clean_me_up;
+	union dml2_core_calcs calcs;
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_wrapper/Makefile b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/Makefile
new file mode 100644
index 00000000000000..f8a17d622c32ca
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/Makefile
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: MIT
+#
+# Copyright 2023 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# Authors: AMD
+#
+# Makefile for dml2_wrapper.
+
+dml2_wrapper_ccflags := $(CC_FLAGS_FPU)
+dml2_wrapper_rcflags := $(CC_FLAGS_NO_FPU)
+
+subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dml2_wrapper
+subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/dml2_wrapper/dml21_wrapper
+
+# Add FPU flags to all dml2_wrapper files by default, remove NO_FPU flags.
+# FPU flags step 1: Find all .c files in dal/dc/dml2_wrapper and it's subfolders
+DML2_WRAPPER_ABS_PATH := $(FULL_AMD_DISPLAY_PATH)/dc/dml2_wrapper
+DML2_WRAPPER_C_FILES := $(shell find $(DML2_WRAPPER_ABS_PATH) -name '*.c' -type f)
+
+# FPU flags step 2: Convert to .o and make paths relative to $(AMDDALPATH)/dc/dml2_wrapper/
+DML2_WRAPPER_RELATIVE_O_FILES := $(patsubst $(DML2_WRAPPER_ABS_PATH)/%,dc/dml2_wrapper/%,$(patsubst %.c,%.o,$(DML2_WRAPPER_C_FILES)))
+
+# FPU flags step 3: Apply FPU flags to all .o files from dal/dc/dml2_wrapper and it's subfolders
+$(foreach obj,$(DML2_WRAPPER_RELATIVE_O_FILES),$(eval CFLAGS_$(AMDDALPATH)/$(obj) := $(dml2_wrapper_ccflags)))
+$(foreach obj,$(DML2_WRAPPER_RELATIVE_O_FILES),$(eval CFLAGS_REMOVE_$(AMDDALPATH)/$(obj) := $(dml2_wrapper_rcflags)))
+
+# FPU flags step 4: Replace CFLAGS per file for files with additional flags beyond dml2_ccflags and dml2_rcflags
+CFLAGS_$(AMDDALPATH)/dc/dml2_wrapper/dml2_wrapper.o := $(dml2_wrapper_rcflags)
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_wrapper/dml2_wrapper.o := $(dml2_wrapper_ccflags)
+CFLAGS_$(AMDDALPATH)/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.o := $(dml2_wrapper_rcflags)
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.o := $(dml2_wrapper_ccflags)
+
+DML2_WRAPPER = dml2_wrapper_fpu.o dml2_wrapper.o \
+		dml2_utils.o dml2_policy.o dml2_translation_helper.o dml2_dc_resource_mgmt.o dml2_mall_phantom.o
+
+DML21_WRAPPER = dml21_wrapper/dml21_wrapper_fpu.o dml21_wrapper/dml21_wrapper.o \
+		dml21_wrapper/dml21_utils.o dml21_wrapper/dml21_translation_helper.o
+
+AMD_DAL_DML2_WRAPPER = $(addprefix $(AMDDALPATH)/dc/dml2_wrapper/,$(DML2_WRAPPER))
+AMD_DAL_DML21_WRAPPER = $(addprefix $(AMDDALPATH)/dc/dml2_wrapper/,$(DML21_WRAPPER))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_DML2_WRAPPER)
+AMD_DISPLAY_FILES += $(AMD_DAL_DML21_WRAPPER)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_translation_helper.c
similarity index 96%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_translation_helper.c
index 1b0d55e2d7b013..366767f4413e73 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_translation_helper.c
@@ -31,6 +31,7 @@ static void dml21_populate_pmo_options(struct dml2_pmo_options *pmo_options,
 			disable_fams2;
 	pmo_options->disable_drr_var = ((in_dc->debug.dml21_disable_pstate_method_mask >> 4) & 1) ||
 			disable_fams2;
+	pmo_options->disable_alternate_memory_training = ((in_dc->debug.dml21_disable_pstate_method_mask >> 5) & 1);
 	pmo_options->disable_fams2 = disable_fams2;
 
 	pmo_options->disable_drr_var_when_var_active = in_dc->debug.disable_fams_gaming == INGAME_FAMS_DISABLE ||
@@ -38,11 +39,13 @@ static void dml21_populate_pmo_options(struct dml2_pmo_options *pmo_options,
 	pmo_options->disable_drr_clamped_when_var_active = in_dc->debug.disable_fams_gaming == INGAME_FAMS_DISABLE;
 
 	pmo_options->force_mandatory_uclk_pstate_support = config->pmo.force_mandatory_uclk_pstate_support;
+	pmo_options->force_optional_uclk_pstate_support = in_dc->debug.force_optional_uclk_pstate_support;
 }
 
 static enum dml2_project_id dml21_dcn_revision_to_dml2_project_id(const struct dc *in_dc)
 {
 	enum dml2_project_id project_id;
+
 	switch (in_dc->ctx->dce_version) {
 	case DCN_VERSION_4_01:
 		project_id = dml2_project_dcn4x_stage2_auto_drr_svp;
@@ -53,6 +56,8 @@ static enum dml2_project_id dml21_dcn_revision_to_dml2_project_id(const struct d
 		break;
 	case DCN_VERSION_6_0:
 		project_id = dml2_project_dcn6x_soc_var_a;
+		if (ASICREV_IS_DCN6_VARIANT_LITE3(in_dc->ctx->asic_id.hw_internal_rev))
+			project_id = dml2_project_dcn6x_soc_var_b;
 		break;
 	default:
 		project_id = dml2_project_invalid;
@@ -94,6 +99,32 @@ static unsigned int calc_max_hardware_v_total(const struct dc_stream_state *stre
 	return max_hw_v_total;
 }
 
+static unsigned int calc_vblank_nom_lines(const struct dc_stream_state *stream, const unsigned int default_vblank_nom_us)
+{
+	unsigned int vblank_avail = stream->timing.v_total - stream->timing.v_addressable;
+	unsigned int vblank_nom = (unsigned int)div64_u64((uint64_t)default_vblank_nom_us * 1000ULL, // vblank_nom_ns
+			div64_u64((uint64_t)stream->timing.h_total * 10000000ULL, (uint64_t)stream->timing.pix_clk_100hz)); // line_time_ns
+
+	/*
+	 * HW requirement for SDP to be in FP ahead of vsync (line 0).
+	 * TODO: Move this out of the translation layer into DC core as per-ASIC policy.
+	 */
+	if (stream->adaptive_sync_infopacket.valid ||
+			(stream->link && stream->link->replay_settings.config.replay_supported)) {
+		const unsigned int v_active = stream->timing.v_border_top + stream->timing.v_addressable +
+					      stream->timing.v_border_bottom;
+		const unsigned int blank_lines = stream->timing.v_total - v_active;
+		const unsigned int bp_lines = blank_lines - stream->timing.v_front_porch;
+		const unsigned int min_vblank_nom = bp_lines + 2;
+
+		vblank_nom = max(vblank_nom, min_vblank_nom);
+	}
+
+	vblank_nom = min(vblank_nom, vblank_avail);
+
+	return vblank_nom;
+}
+
 static void populate_dml21_timing_config_from_stream_state(struct dml2_timing_cfg *timing,
 		struct dc_stream_state *stream,
 		struct pipe_ctx *otg_master_pipe,
@@ -152,6 +183,7 @@ static void populate_dml21_timing_config_from_stream_state(struct dml2_timing_cf
 
 	{
 		uint64_t min_refresh = max((uint64_t)stream->timing.min_refresh_in_uhz, min_hardware_refresh_in_uhz);
+
 		ASSERT(min_refresh <= ULONG_MAX);
 		timing->drr_config.min_refresh_uhz = (unsigned long)min_refresh;
 	}
@@ -199,7 +231,7 @@ static void populate_dml21_timing_config_from_stream_state(struct dml2_timing_cf
 		break;
 	}
 
-	timing->vblank_nom = timing->v_total - timing->v_active;
+	timing->vblank_nom = calc_vblank_nom_lines(stream, dml_ctx->v21.dml_init.ip_caps.vblank_nom_default_us);
 }
 
 static void populate_dml21_output_config_from_stream_state(struct dml2_link_output_cfg *output,
@@ -280,7 +312,7 @@ static void populate_dml21_writeback_config_from_stream_state(struct dml2_writeb
 		writeback->active_writebacks_per_stream = stream->num_wb_info <= DML2_MAX_WRITEBACK ?
 				stream->num_wb_info : DML2_MAX_WRITEBACK;
 
-		ASSERT(stream->num_wb_info <= DML2_MAX_WRITEBACK);
+		ASSERT(stream->num_wb_info <= MAX_DWB_PIPES);
 
 		for (unsigned int wb_index = 0; wb_index < stream->num_wb_info; wb_index++) {
 			const struct dc_writeback_info *dc_wb_info = &stream->writeback_info[wb_index];
@@ -678,7 +710,8 @@ static void populate_dml21_plane_config_from_plane_state(struct dml2_context *dm
 	plane->composition.scaler_info.upsp_enabled = (scaler_data->upsp != UPSP_BYPASS);
 
 	/* always_scale is only used for debug purposes not used in production but has to be
-	 * maintained for certain complainces. */
+	 * maintained for certain complainces.
+	 */
 	if (plane_state->ctx->dc->debug.always_scale == true) {
 		plane->composition.scaler_info.enabled = true;
 	}
@@ -794,7 +827,7 @@ static void populate_dml21_plane_config_from_plane_state(struct dml2_context *dm
 
 	plane->composition.rect_out_height_spans_vactive =
 		plane_state->dst_rect.height >= stream->src.height &&
-		stream->dst.height >= stream->timing.v_addressable;
+		stream->dst.height >= (int)stream->timing.v_addressable;
 }
 
 //TODO : Could be possibly moved to a common helper layer.
@@ -919,6 +952,7 @@ bool dml21_map_dc_state_into_dml_display_cfg(const struct dc *in_dc, struct dc_s
 		ASSERT(otg_master_pipe);
 
 		ASSERT(disp_cfg_stream_location >= 0 && disp_cfg_stream_location < __DML2_WRAPPER_MAX_STREAMS_PLANES__);
+
 		populate_dml21_timing_config_from_stream_state(&dml_dispcfg->stream_descriptors[disp_cfg_stream_location].timing, context->streams[stream_index], otg_master_pipe, dml_ctx);
 		populate_dml21_output_config_from_stream_state(&dml_dispcfg->stream_descriptors[disp_cfg_stream_location].output, context->streams[stream_index], otg_master_pipe);
 		populate_dml21_writeback_config_from_stream_state(&dml_dispcfg->stream_descriptors[disp_cfg_stream_location].writeback, context->streams[stream_index]);
@@ -977,7 +1011,7 @@ bool dml21_map_dc_state_into_dml_display_cfg(const struct dc *in_dc, struct dc_s
 
 void dml21_copy_clocks_to_dc_state(struct dml2_context *in_ctx, struct dc_state *context)
 {
-	/* TODO these should be the max of active, svp prefetch and idle should be tracked seperately */
+	/* TODO these should be the max of active, svp prefetch and idle should be tracked separately */
 	context->bw_ctx.bw.dcn.clk.dispclk_khz = in_ctx->v21.mode_programming.programming->min_clocks.dcn4x.dispclk_khz;
 	context->bw_ctx.bw.dcn.clk.dcfclk_khz = in_ctx->v21.mode_programming.programming->min_clocks.dcn4x.active.dcfclk_khz;
 	context->bw_ctx.bw.dcn.clk.dramclk_khz = in_ctx->v21.mode_programming.programming->min_clocks.dcn4x.active.uclk_khz;
@@ -999,7 +1033,7 @@ void dml21_copy_clocks_to_dc_state(struct dml2_context *in_ctx, struct dc_state
 	context->bw_ctx.bw.dcn.clk.stutter_efficiency.low_power_efficiency = (uint8_t)in_ctx->v21.mode_programming.programming->stutter.low_power_percent_efficiency;
 	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_efficiency = (uint8_t)in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_efficiency;
 	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_period = (int)in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_period;
-	context->bw_ctx.bw.dcn.clk.zstate_support = in_ctx->v21.mode_programming.programming->z8_stutter.supported_in_blank; /*ignore meets_eco since it is not used*/
+	context->bw_ctx.bw.dcn.clk.zstate_support = in_ctx->v21.mode_programming.programming->z8_stutter.global_support;
 }
 
 static struct dml2_dchub_watermark_regs *wm_set_index_to_dc_wm_set(union dcn_watermark_set *watermarks, const enum dml2_dchub_watermark_reg_set_index wm_index)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_translation_helper.h
similarity index 94%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_translation_helper.h
index f51d3d8a52c3f8..ceb50a94f7a05e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_translation_helper.h
@@ -1,6 +1,7 @@
-// SPDX-License-Identifier: MIT
-//
-// Copyright 2024 Advanced Micro Devices, Inc.
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ */
 
 
 #ifndef _DML21_TRANSLATION_HELPER_H_
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_utils.c
similarity index 97%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_utils.c
index 1ade48b8b621a9..0aecfa8b909f50 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_utils.c
@@ -14,6 +14,7 @@
 int dml21_helper_find_dml_pipe_idx_by_stream_id(struct dml2_context *ctx, unsigned int stream_id)
 {
 	int i;
+
 	for (i = 0; i < __DML2_WRAPPER_MAX_STREAMS_PLANES__; i++) {
 		if (ctx->v21.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id_valid[i] && ctx->v21.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id[i] == stream_id)
 			return  i;
@@ -25,6 +26,7 @@ int dml21_helper_find_dml_pipe_idx_by_stream_id(struct dml2_context *ctx, unsign
 int dml21_find_dml_pipe_idx_by_plane_id(struct dml2_context *ctx, unsigned int plane_id)
 {
 	int i;
+
 	for (i = 0; i < __DML2_WRAPPER_MAX_STREAMS_PLANES__; i++) {
 		if (ctx->v21.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id_valid[i] && ctx->v21.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id[i] == plane_id)
 			return  i;
@@ -120,6 +122,7 @@ int dml21_find_dc_pipes_for_plane(const struct dc *in_dc,
 	} else {
 		/* stream was configured with dummy plane, so get pipes from opp head */
 		struct pipe_ctx *otg_master_pipe = dml_ctx->config.callbacks.get_otg_master_for_stream(&context->res_ctx, dc_main_stream);
+
 		if (otg_master_pipe != NULL)
 			num_pipes = dml_ctx->config.callbacks.get_opp_heads_for_otg_master(otg_master_pipe, &context->res_ctx, dc_main_pipes);
 	}
@@ -196,7 +199,7 @@ bool check_dp2p0_output_encoder(const struct pipe_ctx *pipe_ctx)
 
 static bool is_sub_vp_enabled(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
@@ -375,11 +378,13 @@ void dml21_handle_phantom_streams_planes(const struct dc *dc, struct dc_state *c
 		dml2_map_dc_pipes(dml_ctx, context, NULL, &dml_ctx->v21.dml_to_dc_pipe_mapping, dc->current_state);
 }
 
-static uint32_t calc_svp_size_64kb(uint32_t total_size_bytes)
+static uint32_t calc_svp_size_256kb(uint64_t addr_bytes)
 {
-	return (total_size_bytes + 0xFFFF) >> 16;
+	/* ceil to 256KB, then return bits [47:16] */
+	return (uint32_t)(((addr_bytes + 0x3FFFFull) >> 18) << 2);
 }
 
+
 static unsigned int dml21_build_fams2_stream_programming_v3(const struct dc *dc,
 		struct dc_state *context,
 		struct dml2_context *dml_ctx)
@@ -387,7 +392,7 @@ static unsigned int dml21_build_fams2_stream_programming_v3(const struct dc *dc,
 	int dml_stream_idx, dc_stream_idx, dc_plane_idx, svp_idx;
 	unsigned int dc_pipe_idx;
 	unsigned int num_fams2_streams = 0;
-	unsigned int svp_size_64kb[2] = {0};
+	unsigned int svp_size_256kb[2] = {0};
 	struct pipe_ctx *pipe;
 
 	for (dc_stream_idx = 0; dc_stream_idx < context->stream_count; dc_stream_idx++) {
@@ -477,12 +482,12 @@ static unsigned int dml21_build_fams2_stream_programming_v3(const struct dc *dc,
 					}
 					for (svp_idx = 0; svp_idx < 2; svp_idx++) {
 						alternate_static_state->pipe_copy_addr_47_16[svp_idx][dc_pipe_idx] =
-								dml_ctx->config.alt_ch_cfg.region_base_addr_47_16[svp_idx] + svp_size_64kb[svp_idx];
-						svp_size_64kb[svp_idx] += calc_svp_size_64kb(alternate_static_state->pipe_copy_max_size[svp_idx][dc_plane_idx]);
+								dml_ctx->config.alt_ch_cfg.region_base_addr_47_16[svp_idx] + svp_size_256kb[svp_idx];
+						svp_size_256kb[svp_idx] += calc_svp_size_256kb(alternate_static_state->pipe_copy_max_size[svp_idx][dc_plane_idx]);
 						if (alternate_static_state->config[dc_plane_idx].bits.is_multi_planar) {
 							alternate_static_state->pipe_copy_addr_47_16_c[svp_idx][dc_pipe_idx] =
-									dml_ctx->config.alt_ch_cfg.region_base_addr_47_16[svp_idx] + svp_size_64kb[svp_idx];
-							svp_size_64kb[svp_idx] += calc_svp_size_64kb(alternate_static_state->pipe_copy_max_size_c[svp_idx][dc_plane_idx]);
+									dml_ctx->config.alt_ch_cfg.region_base_addr_47_16[svp_idx] + svp_size_256kb[svp_idx];
+							svp_size_256kb[svp_idx] += calc_svp_size_256kb(alternate_static_state->pipe_copy_max_size_c[svp_idx][dc_plane_idx]);
 						}
 					}
 				}
@@ -504,7 +509,8 @@ static unsigned int dml21_build_fams2_stream_programming_v2(const struct dc *dc,
 		struct dc_state *context,
 		struct dml2_context *dml_ctx)
 {
-	int dc_stream_idx, dc_plane_idx, dc_pipe_idx;
+	int dc_stream_idx, dc_plane_idx;
+	unsigned int dc_pipe_idx;
 	unsigned int num_fams2_streams = 0;
 
 	for (dc_stream_idx = 0; dc_stream_idx < context->stream_count; dc_stream_idx++) {
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_utils.h
similarity index 96%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_utils.h
index 7312c33143c694..c30a5b2a99a695 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_utils.h
@@ -1,6 +1,7 @@
-// SPDX-License-Identifier: MIT
-//
-// Copyright 2024 Advanced Micro Devices, Inc.
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ */
 
 
 #ifndef _DML21_UTILS_H_
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.c
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.c
index 8bed59e976d163..fb7c1759dc0929 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.c
@@ -13,6 +13,7 @@
 #include "dml2_wrapper_fpu.h"
 #include "dml21_wrapper.h"
 #include "dml21_wrapper_fpu.h"
+
 #include "dc_fpu.h"
 
 #if !defined(DC_RUN_WITH_PREEMPTION_ENABLED)
@@ -38,7 +39,6 @@ static bool dml21_allocate_memory(struct dml2_context **dml_ctx)
 	(*dml_ctx)->v21.mode_programming.display_config = (*dml_ctx)->v21.mode_support.display_config;
 
 	DC_RUN_WITH_PREEMPTION_ENABLED((*dml_ctx)->v21.mode_programming.programming = vzalloc(sizeof(struct dml2_display_cfg_programming)));
-
 	if (!((*dml_ctx)->v21.mode_programming.programming))
 		return false;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.h
new file mode 100644
index 00000000000000..46ea67c1246dd8
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper.h
@@ -0,0 +1,107 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2024 Advanced Micro Devices, Inc.
+ */
+
+
+#ifndef _DML21_WRAPPER_H_
+#define _DML21_WRAPPER_H_
+
+#include "os_types.h"
+#include "dml_top_soc_parameter_types.h"
+#include "dml_top_display_cfg_types.h"
+
+struct dc;
+struct dc_state;
+struct dml2_configuration_options;
+struct dml2_context;
+enum dc_validate_mode;
+
+/**
+ * dml21_create - Creates dml21_context.
+ * @in_dc: dc.
+ * @dml_ctx: Created dml21 context.
+ * @config: dml21 configuration options.
+ *
+ * Create of DML21 is done as part of dc_state creation.
+ * DML21 IP, SOC and STATES are initialized at
+ * creation time.
+ *
+ * Return: True if dml2 is successfully created, false otherwise.
+ */
+bool dml21_create(const struct dc *in_dc, struct dml2_context **dml_ctx, const struct dml2_configuration_options *config);
+void dml21_destroy(struct dml2_context *dml2);
+void dml21_copy(struct dml2_context *dst_dml_ctx,
+	struct dml2_context *src_dml_ctx);
+bool dml21_create_copy(struct dml2_context **dst_dml_ctx,
+	struct dml2_context *src_dml_ctx);
+
+/* Structure for inputting external SOCBB and DCNIP values for tool based debugging. */
+struct socbb_ip_params_external {
+	struct dml2_ip_capabilities ip_params;
+	struct dml2_soc_bb soc_bb;
+};
+
+/*mcache parameters decided by dml*/
+struct dc_mcache_params {
+	bool valid;
+	/*
+	 * For iMALL, dedicated mall mcaches are required (sharing of last
+	 * slice possible), for legacy phantom or phantom without return
+	 * the only mall mcaches need to be valid.
+	 */
+	bool requires_dedicated_mall_mcache;
+	unsigned int num_mcaches_plane0;
+	unsigned int num_mcaches_plane1;
+	/*
+	 * Generally, plane0/1 slices must use a disjoint set of caches
+	 * but in some cases the final segment of the two planes can
+	 * use the same cache. If plane0_plane1 is set, then this is
+	 * allowed.
+	 *
+	 * Similarly, the caches allocated to MALL prefetcher are generally
+	 * disjoint, but if mall_prefetch is set, then the final segment
+	 * between the main and the mall pixel requestor can use the same
+	 * cache.
+	 *
+	 * Note that both bits may be set at the same time.
+	 */
+	struct {
+		bool mall_comb_mcache_p0;
+		bool mall_comb_mcache_p1;
+		bool plane0_plane1;
+	} last_slice_sharing;
+	/*
+	 * A plane is divided into vertical slices of mcaches,
+	 * which wrap on the surface width.
+	 *
+	 * For example, if the surface width is 7680, and split into
+	 * three slices of equal width, the boundary array would contain
+	 * [2560, 5120, 7680]
+	 *
+	 * The assignments are
+	 * 0 = [0 .. 2559]
+	 * 1 = [2560 .. 5119]
+	 * 2 = [5120 .. 7679]
+	 * 0 = [7680 .. INF]
+	 * The final element implicitly is the same as the first, and
+	 * at first seems invalid since it is never referenced (since)
+	 * it is outside the surface. However, its useful when shifting
+	 * (see below).
+	 *
+	 * For any given valid mcache assignment, a shifted version, wrapped
+	 * on the surface width boundary is also assumed to be valid.
+	 *
+	 * For example, shifting [2560, 5120, 7680] by -50 results in
+	 * [2510, 5170, 7630].
+	 *
+	 * The assignments are now:
+	 * 0 = [0 .. 2509]
+	 * 1 = [2510 .. 5169]
+	 * 2 = [5170 .. 7629]
+	 * 0 = [7630 .. INF]
+	 */
+	int mcache_x_offsets_plane0[DML2_MAX_MCACHES + 1];
+	int mcache_x_offsets_plane1[DML2_MAX_MCACHES + 1];
+};
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper_fpu.c
similarity index 96%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper_fpu.c
index aba7dd65c539e2..a1525f7e0e61b2 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper_fpu.c
@@ -69,6 +69,7 @@ static void dml21_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_sta
 	(void)out_new_hw_state;
 	(void)pipe_cnt;
 	unsigned int dml_prog_idx = 0, dc_pipe_index = 0, num_dpps_required = 0;
+	unsigned int pipe_count = 0;
 	struct dml2_per_plane_programming *pln_prog = NULL;
 	struct dml2_per_stream_programming *stream_prog = NULL;
 	struct pipe_ctx *dc_main_pipes[__DML2_WRAPPER_MAX_STREAMS_PLANES__];
@@ -109,8 +110,10 @@ static void dml21_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_sta
 		if (num_pipes <= 0)
 			continue;
 
+		pipe_count = (unsigned int)num_pipes;
+
 		/* program each pipe */
-		for (dc_pipe_index = 0; dc_pipe_index < num_pipes; dc_pipe_index++) {
+		for (dc_pipe_index = 0; dc_pipe_index < pipe_count; dc_pipe_index++) {
 			dml21_program_dc_pipe(in_ctx, context, dc_main_pipes[dc_pipe_index], pln_prog, stream_prog);
 
 			if (pln_prog->phantom_plane.valid && dc_phantom_pipes[dc_pipe_index]) {
@@ -302,7 +305,7 @@ bool dml21_validate(const struct dc *in_dc, struct dc_state *context, struct dml
 
 void dml21_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context, struct dml2_context *dml_ctx)
 {
-	unsigned int dml_prog_idx, dml_phantom_prog_idx, dc_pipe_index;
+	unsigned int dml_prog_idx, dml_phantom_prog_idx, dc_pipe_index, pipe_count;
 	int num_pipes;
 	struct pipe_ctx *dc_main_pipes[__DML2_WRAPPER_MAX_STREAMS_PLANES__];
 	struct pipe_ctx *dc_phantom_pipes[__DML2_WRAPPER_MAX_STREAMS_PLANES__] = {0};
@@ -342,8 +345,10 @@ void dml21_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context
 		    dc_main_pipes[0]->plane_state == NULL)
 			continue;
 
+		pipe_count = (unsigned int)num_pipes;
+
 		/* get config for each pipe */
-		for (dc_pipe_index = 0; dc_pipe_index < num_pipes; dc_pipe_index++) {
+		for (dc_pipe_index = 0; dc_pipe_index < pipe_count; dc_pipe_index++) {
 			ASSERT(dc_main_pipes[dc_pipe_index]);
 			dml21_get_pipe_mcache_config(context, dc_main_pipes[dc_pipe_index], pln_prog, &mcache_config->pipe_configurations[dc_pipe_index]);
 		}
@@ -361,7 +366,7 @@ void dml21_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context
 			mcache_config->num_pipes = (char)pln_prog->num_dpps_required;
 			l->build_mcache_programming_params.num_configurations++;
 
-			for (dc_pipe_index = 0; dc_pipe_index < num_pipes; dc_pipe_index++) {
+			for (dc_pipe_index = 0; dc_pipe_index < pipe_count; dc_pipe_index++) {
 				ASSERT(dc_phantom_pipes[dc_pipe_index]);
 				dml21_get_pipe_mcache_config(context, dc_phantom_pipes[dc_pipe_index], pln_prog, &mcache_config->pipe_configurations[dc_pipe_index]);
 			}
@@ -383,8 +388,10 @@ void dml21_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context
 		    dc_main_pipes[0]->plane_state == NULL)
 			continue;
 
+		pipe_count = (unsigned int)num_pipes;
+
 		/* get config for each pipe */
-		for (dc_pipe_index = 0; dc_pipe_index < num_pipes; dc_pipe_index++) {
+		for (dc_pipe_index = 0; dc_pipe_index < pipe_count; dc_pipe_index++) {
 			ASSERT(dc_main_pipes[dc_pipe_index]);
 			if (l->build_mcache_programming_params.per_plane_pipe_mcache_regs[dml_prog_idx][dc_pipe_index]) {
 				memcpy(&dc_main_pipes[dc_pipe_index]->mcache_regs,
@@ -398,7 +405,7 @@ void dml21_prepare_mcache_programming(struct dc *in_dc, struct dc_state *context
 				dc_phantom_pipes[0] &&
 				dc_main_pipes[0]->stream &&
 				dc_phantom_pipes[0]->plane_state) {
-			for (dc_pipe_index = 0; dc_pipe_index < num_pipes; dc_pipe_index++) {
+			for (dc_pipe_index = 0; dc_pipe_index < pipe_count; dc_pipe_index++) {
 				ASSERT(dc_phantom_pipes[dc_pipe_index]);
 				if (l->build_mcache_programming_params.per_plane_pipe_mcache_regs[dml_phantom_prog_idx][dc_pipe_index]) {
 					memcpy(&dc_phantom_pipes[dc_pipe_index]->mcache_regs,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper_fpu.h
similarity index 95%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper_fpu.h
index e5d9a456645fd8..570129229ae27e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_wrapper_fpu.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml21_wrapper/dml21_wrapper_fpu.h
@@ -1,6 +1,7 @@
-// SPDX-License-Identifier: MIT
-//
-// Copyright 2026 Advanced Micro Devices, Inc.
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ */
 
 #ifndef _DML21_WRAPPER_FPU_H_
 #define _DML21_WRAPPER_FPU_H_
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_dc_resource_mgmt.c
similarity index 96%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_dc_resource_mgmt.c
index 6b78334c255420..4050b0a47caf9c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_dc_resource_mgmt.c
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: MIT */
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2023 Advanced Micro Devices, Inc.
  *
@@ -332,7 +332,7 @@ static bool is_pipe_in_candidate_array(const unsigned int pipe_idx,
 	const unsigned int *candidate_array,
 	const unsigned int candidate_array_size)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < candidate_array_size; i++) {
 		if (candidate_array[i] == pipe_idx)
@@ -355,6 +355,7 @@ static bool find_more_pipes_for_stream(struct dml2_context *ctx,
 	unsigned int last_resort_pipe_candidates[MAX_PIPES] = {0};
 	unsigned int num_preferred_candidates = 0;
 	unsigned int num_last_resort_candidates = 0;
+	unsigned int idx;
 	int i;
 
 	if (existing_state) {
@@ -366,13 +367,13 @@ static bool find_more_pipes_for_stream(struct dml2_context *ctx,
 	}
 
 	// First see if any of the preferred are unmapped, and choose those instead
-	for (i = 0; pipes_needed > 0 && i < num_preferred_candidates; i++) {
-		pipe = &state->res_ctx.pipe_ctx[preferred_pipe_candidates[i]];
+	for (idx = 0; pipes_needed > 0 && idx < num_preferred_candidates; idx++) {
+		pipe = &state->res_ctx.pipe_ctx[preferred_pipe_candidates[idx]];
 		if (!is_plane_using_pipe(pipe)) {
 			pipes_needed--;
 			// TODO: This doens't make sense really, pipe_idx should always be valid
-			ASSERT(preferred_pipe_candidates[i] <= 0xFF);
-			pipe->pipe_idx = (uint8_t)preferred_pipe_candidates[i];
+			ASSERT(preferred_pipe_candidates[idx] <= 0xFF);
+			pipe->pipe_idx = (uint8_t)preferred_pipe_candidates[idx];
 			assigned_pipes[(*assigned_pipe_count)++] = pipe->pipe_idx;
 		}
 	}
@@ -395,13 +396,13 @@ static bool find_more_pipes_for_stream(struct dml2_context *ctx,
 	}
 
 	// Only use the last resort pipe candidates as a last resort
-	for (i = 0; pipes_needed > 0 && i < num_last_resort_candidates; i++) {
-		pipe = &state->res_ctx.pipe_ctx[last_resort_pipe_candidates[i]];
+	for (idx = 0; pipes_needed > 0 && idx < num_last_resort_candidates; idx++) {
+		pipe = &state->res_ctx.pipe_ctx[last_resort_pipe_candidates[idx]];
 		if (!is_plane_using_pipe(pipe)) {
 			pipes_needed--;
 			// TODO: This doens't make sense really, pipe_idx should always be valid
-			ASSERT(last_resort_pipe_candidates[i] <= 0xFF);
-			pipe->pipe_idx = (uint8_t)last_resort_pipe_candidates[i];
+			ASSERT(last_resort_pipe_candidates[idx] <= 0xFF);
+			pipe->pipe_idx = (uint8_t)last_resort_pipe_candidates[idx];
 			assigned_pipes[(*assigned_pipe_count)++] = pipe->pipe_idx;
 		}
 	}
@@ -424,6 +425,7 @@ static bool find_more_free_pipes(struct dml2_context *ctx,
 	unsigned int last_resort_pipe_candidates[MAX_PIPES] = {0};
 	unsigned int num_preferred_candidates = 0;
 	unsigned int num_last_resort_candidates = 0;
+	unsigned int idx;
 	int i;
 
 	if (existing_state) {
@@ -435,13 +437,13 @@ static bool find_more_free_pipes(struct dml2_context *ctx,
 	}
 
 	// First see if any of the preferred are unmapped, and choose those instead
-	for (i = 0; pipes_needed > 0 && i < num_preferred_candidates; i++) {
-		pipe = &state->res_ctx.pipe_ctx[preferred_pipe_candidates[i]];
+	for (idx = 0; pipes_needed > 0 && idx < num_preferred_candidates; idx++) {
+		pipe = &state->res_ctx.pipe_ctx[preferred_pipe_candidates[idx]];
 		if (is_pipe_free(pipe)) {
 			pipes_needed--;
 			// TODO: This doens't make sense really, pipe_idx should always be valid
-			ASSERT(preferred_pipe_candidates[i] <= 0xFF);
-			pipe->pipe_idx = (uint8_t)preferred_pipe_candidates[i];
+			ASSERT(preferred_pipe_candidates[idx] <= 0xFF);
+			pipe->pipe_idx = (uint8_t)preferred_pipe_candidates[idx];
 			assigned_pipes[(*assigned_pipe_count)++] = pipe->pipe_idx;
 		}
 	}
@@ -457,20 +459,20 @@ static bool find_more_free_pipes(struct dml2_context *ctx,
 		if (is_pipe_free(pipe)) {
 			pipes_needed--;
 			// TODO: This doens't make sense really, pipe_idx should always be valid
-			ASSERT(i >= 0 && i <= 0xFF);
+			ASSERT(i >= 0 && i < MAX_PIPES);
 			pipe->pipe_idx = (uint8_t)i;
 			assigned_pipes[(*assigned_pipe_count)++] = pipe->pipe_idx;
 		}
 	}
 
 	// Only use the last resort pipe candidates as a last resort
-	for (i = 0; pipes_needed > 0 && i < num_last_resort_candidates; i++) {
-		pipe = &state->res_ctx.pipe_ctx[last_resort_pipe_candidates[i]];
+	for (idx = 0; pipes_needed > 0 && idx < num_last_resort_candidates; idx++) {
+		pipe = &state->res_ctx.pipe_ctx[last_resort_pipe_candidates[idx]];
 		if (is_pipe_free(pipe)) {
 			pipes_needed--;
 			// TODO: This doens't make sense really, pipe_idx should always be valid
-			ASSERT(last_resort_pipe_candidates[i] <= 0xFF);
-			pipe->pipe_idx = (uint8_t)last_resort_pipe_candidates[i];
+			ASSERT(last_resort_pipe_candidates[idx] <= 0xFF);
+			pipe->pipe_idx = (uint8_t)last_resort_pipe_candidates[idx];
 			assigned_pipes[(*assigned_pipe_count)++] = pipe->pipe_idx;
 		}
 	}
@@ -526,7 +528,7 @@ static void sort_pipes_for_splitting(struct dc_plane_pipe_pool *pipes)
 static void calculate_odm_slices(const struct dc_stream_state *stream, unsigned int odm_factor, unsigned int *odm_slice_end_x)
 {
 	unsigned int slice_size = 0;
-	int i;
+	unsigned int i;
 
 	if (odm_factor < 1 || odm_factor > 4) {
 		ASSERT(false);
@@ -749,7 +751,7 @@ static void remove_pipes_from_blend_trees(struct dml2_context *ctx, struct dc_st
 static void map_pipes_for_stream(struct dml2_context *ctx, struct dc_state *state, const struct dc_stream_state *stream,
 		struct dc_pipe_mapping_scratch *scratch, const struct dc_state *existing_state)
 {
-	int odm_slice_index;
+	unsigned int odm_slice_index;
 	struct pipe_ctx *master_pipe = NULL;
 
 
@@ -769,7 +771,7 @@ static void map_pipes_for_stream(struct dml2_context *ctx, struct dc_state *stat
 static void map_pipes_for_plane(struct dml2_context *ctx, struct dc_state *state, const struct dc_stream_state *stream, const struct dc_plane_state *plane,
 		int plane_index, struct dc_pipe_mapping_scratch *scratch, const struct dc_state *existing_state)
 {
-	int odm_slice_index;
+	unsigned int odm_slice_index;
 	unsigned int plane_id;
 	struct pipe_ctx *master_pipe = NULL;
 	int i;
@@ -819,6 +821,10 @@ static unsigned int get_target_mpc_factor(struct dml2_context *ctx,
 		get_plane_id(ctx, state, status->plane_states[plane_idx],
 				stream->stream_id, plane_idx, &plane_id);
 		cfg_idx = find_disp_cfg_idx_by_plane_id(mapping, plane_id);
+		if (!disp_cfg) {
+			ASSERT(0); // disp_cfg pointer is expected on DML2.0
+			return 1;
+		}
 		mpc_factor = (unsigned int)disp_cfg->hw.DPPPerSurface[cfg_idx];
 	} else if (ctx->architecture == dml2_architecture_21) {
 		if (ctx->config.svp_pstate.callbacks.get_stream_subvp_type(state, stream) == SUBVP_PHANTOM) {
@@ -872,6 +878,10 @@ static unsigned int get_target_odm_factor(
 	if (ctx->architecture == dml2_architecture_20) {
 		cfg_idx = find_disp_cfg_idx_by_stream_id(
 				mapping, stream->stream_id);
+		if (!disp_cfg) {
+			ASSERT(0); // disp_cfg pointer is expected on DML2.0
+			return 1;
+		}
 		switch (disp_cfg->hw.ODMMode[cfg_idx]) {
 		case dml_odm_mode_bypass:
 			return 1;
@@ -1082,6 +1092,10 @@ bool dml2_map_dc_pipes(struct dml2_context *ctx, struct dc_state *state, const s
 		DPPPerSurface = (const unsigned int *)dpp_per_surface_array;
 		disp_cfg_index_max = __DML2_WRAPPER_MAX_STREAMS_PLANES__;
 	} else {
+		if (disp_cfg == NULL) {
+			ASSERT(0); // disp_cfg pointer is expected on DML2.0
+			return false;
+		}
 		ODMMode = (unsigned int *)disp_cfg->hw.ODMMode;
 		DPPPerSurface = disp_cfg->hw.DPPPerSurface;
 		disp_cfg_index_max = __DML_NUM_PLANES__;
@@ -1110,7 +1124,7 @@ bool dml2_map_dc_pipes(struct dml2_context *ctx, struct dc_state *state, const s
 		/* After DML2.1 update, ODM interpretation needs to change and is no longer same as for DML2.0.
 		 * This is not an issue with new resource management logic. This block ensure backcompat
 		 * with legacy pipe management with updated DML.
-		 * */
+		 */
 			if (ODMMode[stream_disp_cfg_index] == 1) {
 				scratch.odm_info.odm_factor = 1;
 			} else if (ODMMode[stream_disp_cfg_index] == 2) {
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_dc_resource_mgmt.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_resource_mgmt.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_dc_resource_mgmt.h
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_types.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_dc_types.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_dc_types.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_dc_types.h
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_internal_types.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_internal_types.h
similarity index 99%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_internal_types.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_internal_types.h
index 55b3e3ca54f793..cb301c116615e4 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_internal_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_internal_types.h
@@ -33,7 +33,7 @@
 #include "dml2_policy.h"
 
 #include "dml_top.h"
-#include "dml21_wrapper.h"
+#include "dml21_wrapper/dml21_wrapper.h"
 
 struct dml2_wrapper_optimize_configuration_params {
 	struct display_mode_lib_st *dml_core_ctx;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_mall_phantom.c
similarity index 97%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_mall_phantom.c
index 4543a60a068396..edc4dcdde9529a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_mall_phantom.c
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: MIT */
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2023 Advanced Micro Devices, Inc.
  *
@@ -40,7 +40,7 @@ unsigned int dml2_helper_calculate_num_ways_for_subvp(struct dml2_context *ctx,
 	uint32_t bytes_in_mall = 0;
 	uint32_t num_mblks = 0;
 	uint32_t cache_lines_per_plane = 0;
-	uint32_t i = 0;
+	int i = 0;
 	uint32_t mblk_width = 0;
 	uint32_t mblk_height = 0;
 	uint32_t full_vp_width_blk_aligned = 0;
@@ -185,7 +185,7 @@ static bool mpo_in_use(const struct dc_state *context)
  */
 static unsigned int get_num_free_pipes(struct dml2_context *ctx, struct dc_state *state)
 {
-	unsigned int i;
+	int i;
 	unsigned int free_pipes = 0;
 	unsigned int num_pipes = 0;
 
@@ -228,7 +228,8 @@ static unsigned int get_num_free_pipes(struct dml2_context *ctx, struct dc_state
  */
 static bool assign_subvp_pipe(struct dml2_context *ctx, struct dc_state *context, unsigned int *index)
 {
-	unsigned int i, pipe_idx;
+	int i;
+	unsigned int pipe_idx;
 	unsigned int max_frame_time = 0;
 	bool valid_assignment_found = false;
 	unsigned int free_pipes = 2; //dcn32_get_num_free_pipes(dc, context);
@@ -272,8 +273,8 @@ static bool assign_subvp_pipe(struct dml2_context *ctx, struct dc_state *context
 					valid_assignment_found = true;
 					current_assignment_freesync = false;
 				/* For the 2-Freesync display case, still choose the one with the
-			     * longest frame time
-			     */
+				 * longest frame time
+				 */
 				} else if (stream->ignore_msa_timing_param && (!valid_assignment_found ||
 						(current_assignment_freesync && frame_us > max_frame_time))) {
 					*index = i;
@@ -309,8 +310,10 @@ static bool assign_subvp_pipe(struct dml2_context *ctx, struct dc_state *context
  */
 static bool enough_pipes_for_subvp(struct dml2_context *ctx, struct dc_state *state)
 {
-	unsigned int i, split_cnt, free_pipes;
-	unsigned int min_pipe_split = ctx->config.dcn_pipe_count + 1; // init as max number of pipes + 1
+	int i;
+	int split_cnt;
+	int free_pipes;
+	int min_pipe_split = ctx->config.dcn_pipe_count + 1; // init as max number of pipes + 1
 	bool subvp_possible = false;
 
 	for (i = 0; i < ctx->config.dcn_pipe_count; i++) {
@@ -361,13 +364,13 @@ static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *c
 	struct dc_stream_state *phantom = NULL;
 	uint32_t microschedule_lines = 0;
 	uint32_t index = 0;
-	uint32_t i;
-	uint32_t max_microschedule_us = 0;
+	int i;
+	int32_t max_microschedule_us = 0;
 	int32_t vactive1_us, vactive2_us, vblank1_us, vblank2_us;
 
 	for (i = 0; i < ctx->config.dcn_pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		uint32_t time_us = 0;
+		int32_t time_us = 0;
 
 		if (pipe == NULL || pipe->stream == NULL)
 			continue;
@@ -378,6 +381,10 @@ static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *c
 		if (pipe->stream && pipe->plane_state && !pipe->top_pipe &&
 				ctx->config.svp_pstate.callbacks.get_pipe_subvp_type(context, pipe) == SUBVP_MAIN) {
 			phantom = ctx->config.svp_pstate.callbacks.get_paired_subvp_stream(context, pipe->stream);
+			if (phantom == NULL) {
+				ASSERT(0); // Not expected to happen
+				return false;
+			}
 			microschedule_lines = (phantom->timing.v_total - phantom->timing.v_front_porch) +
 					phantom->timing.v_addressable;
 
@@ -440,7 +447,7 @@ static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *c
 bool dml2_svp_drr_schedulable(struct dml2_context *ctx, struct dc_state *context, struct dc_crtc_timing *drr_timing)
 {
 	bool schedulable = false;
-	uint32_t i;
+	int i;
 	struct pipe_ctx *pipe = NULL;
 	struct dc_crtc_timing *main_timing = NULL;
 	struct dc_crtc_timing *phantom_timing = NULL;
@@ -473,6 +480,11 @@ bool dml2_svp_drr_schedulable(struct dml2_context *ctx, struct dc_state *context
 		return false;
 	}
 	phantom_stream = ctx->config.svp_pstate.callbacks.get_paired_subvp_stream(context, pipe->stream);
+	if (phantom_stream == NULL) {
+		// Defensive: should never happen, try to catch in debug
+		ASSERT(0);
+		return false;
+	}
 	main_timing = &pipe->stream->timing;
 	phantom_timing = &phantom_stream->timing;
 	prefetch_us = (int32_t)((phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
@@ -524,7 +536,7 @@ static bool subvp_vblank_schedulable(struct dml2_context *ctx, struct dc_state *
 	struct pipe_ctx *subvp_pipe = NULL;
 	bool found = false;
 	bool schedulable = false;
-	uint32_t i = 0;
+	int i = 0;
 	uint8_t vblank_index = 0;
 	uint32_t prefetch_us = 0;
 	uint32_t mall_region_us = 0;
@@ -544,7 +556,7 @@ static bool subvp_vblank_schedulable(struct dml2_context *ctx, struct dc_state *
 	 * displays which are synchronized (in which case they have identical
 	 * timings).
 	 */
-	for (i = 0; i < ctx->config.dcn_pipe_count; i++) {
+	for (i = 0; i < ctx->config.dcn_pipe_count && i < MAX_PIPES ; i++) {
 		pipe = &context->res_ctx.pipe_ctx[i];
 		pipe_mall_type = ctx->config.svp_pstate.callbacks.get_pipe_subvp_type(context, pipe);
 
@@ -594,14 +606,11 @@ static bool subvp_vblank_schedulable(struct dml2_context *ctx, struct dc_state *
 		subvp_active_us = (uint32_t)(main_timing->v_addressable * main_timing->h_total /
 				(double)(main_timing->pix_clk_100hz * 100) * 1000000);
 		max_vblank_mallregion = vblank_blank_us > mall_region_us ? vblank_blank_us : mall_region_us;
-		const uint64_t required_us = (uint64_t)prefetch_us +
-					     (uint64_t)vblank_frame_us +
-					     (uint64_t)max_vblank_mallregion;
 
 		// Schedulable if VACTIVE region of the SubVP pipe can fit the MALL prefetch, VBLANK frame time,
 		// and the max of (VBLANK blanking time, MALL region)
 		// TODO: Possibly add some margin (i.e. the below conditions should be [...] > X instead of [...] > 0)
-		if ((uint64_t)subvp_active_us > required_us)
+		if (subvp_active_us - prefetch_us - vblank_frame_us - max_vblank_mallregion > 0)
 			schedulable = true;
 	}
 	return schedulable;
@@ -627,7 +636,8 @@ bool dml2_svp_validate_static_schedulability(struct dml2_context *ctx, struct dc
 {
 	bool schedulable = true;	// true by default for single display case
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
-	uint32_t i, pipe_idx;
+	int i;
+	uint32_t pipe_idx;
 	uint8_t subvp_count = 0;
 	uint8_t vactive_count = 0;
 
@@ -679,7 +689,7 @@ static void set_phantom_stream_timing(struct dml2_context *ctx, struct dc_state
 				     unsigned int svp_height,
 				     unsigned int svp_vstartup)
 {
-	unsigned int i;
+	int i;
 	double line_time, fp_and_sync_width_time;
 	struct pipe_ctx *pipe;
 	uint32_t phantom_vactive, phantom_bp, pstate_width_fw_delay_lines;
@@ -743,6 +753,10 @@ static struct dc_stream_state *enable_phantom_stream(struct dml2_context *ctx, s
 			state,
 			ref_pipe->stream);
 
+	if (!phantom_stream) {
+		ASSERT(0); //Phantom stream is expected to be created.
+		return NULL;
+	}
 	/* stream has limited viewport and small timing */
 	memcpy(&phantom_stream->timing, &ref_pipe->stream->timing, sizeof(phantom_stream->timing));
 	memcpy(&phantom_stream->src, &ref_pipe->stream->src, sizeof(phantom_stream->src));
@@ -813,12 +827,16 @@ static void enable_phantom_plane(struct dml2_context *ctx,
 static void add_phantom_pipes_for_main_pipe(struct dml2_context *ctx, struct dc_state *state, unsigned int main_pipe_idx, unsigned int svp_height, unsigned int vstartup)
 {
 	struct dc_stream_state *phantom_stream = NULL;
-	unsigned int i;
+	int i;
 
-	// The index of the DC pipe passed into this function is guarenteed to
+	// The index of the DC pipe passed into this function is guaranteed to
 	// be a valid candidate for SubVP (i.e. has a plane, stream, doesn't
 	// already have phantom pipe assigned, etc.) by previous checks.
 	phantom_stream = enable_phantom_stream(ctx, state, main_pipe_idx, svp_height, vstartup);
+	if (phantom_stream == NULL) {
+		ASSERT(0); // Phantom stream is expected to be created.
+		return;
+	}
 	enable_phantom_plane(ctx, state, phantom_stream, main_pipe_idx);
 
 	for (i = 0; i < ctx->config.dcn_pipe_count; i++) {
@@ -845,10 +863,10 @@ static bool remove_all_phantom_planes_for_stream(struct dml2_context *ctx, struc
 	struct dc_plane_state *del_planes[MAX_SURFACES] = { 0 };
 
 	for (i = 0; i < context->stream_count; i++)
-			if (context->streams[i] == stream) {
-				stream_status = &context->stream_status[i];
-				break;
-			}
+		if (context->streams[i] == stream) {
+			stream_status = &context->stream_status[i];
+			break;
+		}
 
 	if (stream_status == NULL) {
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_mall_phantom.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_mall_phantom.h
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_policy.c
similarity index 99%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_policy.c
index ab2964811c5bcb..7b043ef3f2ce87 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_policy.c
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: MIT */
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2023 Advanced Micro Devices, Inc.
  *
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_policy.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_policy.h
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_translation_helper.c
similarity index 99%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_translation_helper.c
index c82886323a5182..a0130bbfc12a53 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_translation_helper.c
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: MIT */
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2023 Advanced Micro Devices, Inc.
  *
@@ -720,6 +720,7 @@ void dml2_translate_socbb_params(const struct dc *in, struct soc_bounding_box_st
 void dml2_translate_soc_states(const struct dc *dc, struct soc_states_st *out, int num_states)
 {
 	unsigned int i = 0;
+
 	out->num_states = num_states;
 
 	for (i = 0; i < out->num_states; i++) {
@@ -1096,7 +1097,7 @@ static void populate_dml_plane_cfg_from_plane_state(struct dml_plane_cfg_st *out
 
 	/* Current driver code base uses LBBitPerPixel as 57. There is a discrepancy
 	 * from the HW/DML teams about this value. Initialize LBBitPerPixel with the
-	 * value current used in Navi3x .
+	 * value current used in DCN32.
 	 */
 
 	out->LBBitPerPixel[location] = 57;
@@ -1292,12 +1293,12 @@ static void dml2_map_hpo_stream_encoder_to_hpo_link_encoder_index(struct dml2_co
 	for (i = 0; i < MAX_PIPES; i++) {
 		current_pipe_context = &context->res_ctx.pipe_ctx[i];
 		if (current_pipe_context->stream &&
-			current_pipe_context->stream_res.hpo_dp_stream_enc &&
-			current_pipe_context->link_res.hpo_dp_link_enc &&
-			dc_is_dp_signal(current_pipe_context->stream->signal)) {
-				dml2->v20.scratch.hpo_stream_to_link_encoder_mapping[current_pipe_context->stream_res.hpo_dp_stream_enc->inst] =
-					current_pipe_context->link_res.hpo_dp_link_enc->inst;
-			}
+		    current_pipe_context->stream_res.hpo_dp_stream_enc &&
+		    current_pipe_context->link_res.hpo_dp_link_enc &&
+		    dc_is_dp_signal(current_pipe_context->stream->signal)) {
+			dml2->v20.scratch.hpo_stream_to_link_encoder_mapping[current_pipe_context->stream_res.hpo_dp_stream_enc->inst] =
+				current_pipe_context->link_res.hpo_dp_link_enc->inst;
+		}
 	}
 }
 
@@ -1330,7 +1331,8 @@ void map_dc_state_into_dml_display_cfg(struct dml2_context *dml2, struct dc_stat
 		current_pipe_context = NULL;
 		for (k = 0; k < MAX_PIPES; k++) {
 			/* find one pipe allocated to this stream for the purpose of getting
-			info about the link later */
+			 * info about the link later
+			 */
 			if (context->streams[i] == context->res_ctx.pipe_ctx[k].stream) {
 				current_pipe_context = &context->res_ctx.pipe_ctx[k];
 				break;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_translation_helper.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_translation_helper.h
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_utils.c
similarity index 98%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_utils.c
index a3ce011612f78d..8688792aa95667 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_utils.c
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: MIT */
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2023 Advanced Micro Devices, Inc.
  *
@@ -168,7 +168,7 @@ bool is_dp2p0_output_encoder(const struct pipe_ctx *pipe_ctx)
 
 bool is_dtbclk_required(const struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		if (!context->res_ctx.pipe_ctx[i].stream)
@@ -196,6 +196,7 @@ void dml2_copy_clocks_to_dc_state(struct dml2_dcn_clocks *out_clks, struct dc_st
 int dml2_helper_find_dml_pipe_idx_by_stream_id(struct dml2_context *ctx, unsigned int stream_id)
 {
 	int i;
+
 	for (i = 0; i < __DML2_WRAPPER_MAX_STREAMS_PLANES__; i++) {
 		if (ctx->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id_valid[i] && ctx->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_stream_id[i] == stream_id)
 			return  i;
@@ -207,6 +208,7 @@ int dml2_helper_find_dml_pipe_idx_by_stream_id(struct dml2_context *ctx, unsigne
 static int find_dml_pipe_idx_by_plane_id(struct dml2_context *ctx, unsigned int plane_id)
 {
 	int i;
+
 	for (i = 0; i < __DML2_WRAPPER_MAX_STREAMS_PLANES__; i++) {
 		if (ctx->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id_valid[i] && ctx->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_id[i] == plane_id)
 			return  i;
@@ -218,7 +220,8 @@ static int find_dml_pipe_idx_by_plane_id(struct dml2_context *ctx, unsigned int
 static bool get_plane_id(struct dml2_context *dml2, const struct dc_state *state, const struct dc_plane_state *plane,
 	unsigned int stream_id, unsigned int plane_index, unsigned int *plane_id)
 {
-	unsigned int i, j;
+	unsigned int i;
+	int j;
 	bool is_plane_duplicate = dml2->v20.scratch.plane_duplicate_exists;
 
 	if (!plane_id)
@@ -228,8 +231,8 @@ static bool get_plane_id(struct dml2_context *dml2, const struct dc_state *state
 		if (state->streams[i]->stream_id == stream_id) {
 			for (j = 0; j < state->stream_status[i].plane_count; j++) {
 				if (state->stream_status[i].plane_states[j] == plane &&
-					(!is_plane_duplicate || (j == plane_index))) {
-					*plane_id = (i << 16) | j;
+					(!is_plane_duplicate || ((unsigned int)j == plane_index))) {
+					*plane_id = ((unsigned int)i << 16) | (unsigned int)j;
 					return true;
 				}
 			}
@@ -293,8 +296,8 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 	else
 		context->bw_ctx.bw.dcn.clk.fclk_p_state_change_support = true;
 
-	if (context->bw_ctx.bw.dcn.clk.dispclk_khz < dc->debug.min_disp_clk_khz)
-		context->bw_ctx.bw.dcn.clk.dispclk_khz = dc->debug.min_disp_clk_khz;
+	if (context->bw_ctx.bw.dcn.clk.dispclk_khz < (int)dc->debug.min_disp_clk_khz)
+		context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)dc->debug.min_disp_clk_khz;
 
 	context->bw_ctx.bw.dcn.compbuf_size_kb = in_ctx->v20.dml_core_ctx.ip.config_return_buffer_size_in_kbytes;
 
@@ -304,7 +307,8 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 		/* The DML2 and the DC logic of determining pipe indices are different from each other so
 		 * there is a need to know which DML pipe index maps to which DC pipe. The code below
 		 * finds a dml_pipe_index from the plane id if a plane is valid. If a plane is not valid then
-		 * it finds a dml_pipe_index from the stream id. */
+		 * it finds a dml_pipe_index from the stream id.
+		 */
 		if (get_plane_id(in_ctx, context, context->res_ctx.pipe_ctx[dc_pipe_ctx_index].plane_state,
 			context->res_ctx.pipe_ctx[dc_pipe_ctx_index].stream->stream_id,
 			in_ctx->v20.scratch.dml_to_dc_pipe_mapping.dml_pipe_idx_to_plane_index[context->res_ctx.pipe_ctx[dc_pipe_ctx_index].pipe_idx], &plane_id)) {
@@ -371,7 +375,7 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 
 	if (dc->config.forced_clocks || dc->debug.max_disp_clk) {
 		context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz;
-		context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz ;
+		context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz;
 	}
 }
 
@@ -449,7 +453,7 @@ void dml2_extract_writeback_wm(struct dc_state *context, struct display_mode_lib
 		bw_writeback->mcif_wb_arb[i].arbitration_slice = 2;
 		bw_writeback->mcif_wb_arb[i].max_scaled_time =
 			dml2_calc_max_scaled_time(wb_arb_params->time_per_pixel,
-					wbif_mode, 	wb_arb_params->cli_watermark[0]);
+					wbif_mode, wb_arb_params->cli_watermark[0]);
 		/*not required any more*/
 		bw_writeback->mcif_wb_arb[i].dram_speed_change_duration =
 			(unsigned int)(dml_get_wm_writeback_dram_clock_change(dml_core_ctx) * 1000);
@@ -500,8 +504,8 @@ void dml2_apply_det_buffer_allocation_policy(struct dml2_context *in_ctx, struct
 			dml_dispcfg->plane.DETSizeOverride[plane_index] = ((max_det_size / num_of_streams) / num_of_planes_per_stream[stream_index] / in_ctx->det_helper_scratch.dpps_per_surface[plane_index]);
 
 			/* If the override size is not divisible by det_segment_size then round off to nearest number divisible by det_segment_size as
-				* this is a requirement.
-				*/
+			 * this is a requirement.
+			 */
 			if (dml_dispcfg->plane.DETSizeOverride[plane_index] % in_ctx->config.det_segment_size != 0) {
 				dml_dispcfg->plane.DETSizeOverride[plane_index] = dml_dispcfg->plane.DETSizeOverride[plane_index] & ~0x3F;
 			}
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_utils.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_utils.h
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper.c
similarity index 97%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper.c
index 1772e74349c7ba..2fe3c4643f5861 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper.c
@@ -8,8 +8,8 @@
 #include "dml2_internal_types.h"
 #include "dml2_wrapper.h"
 #include "dml2_wrapper_fpu.h"
-#include "dml21_wrapper.h"
-#include "dml21_wrapper_fpu.h"
+#include "dml21_wrapper/dml21_wrapper.h"
+#include "dml21_wrapper/dml21_wrapper_fpu.h"
 
 #include "dc_fpu.h"
 
@@ -24,6 +24,7 @@ struct dml2_context *dml2_allocate_memory(void)
 	DC_RUN_WITH_PREEMPTION_ENABLED(dml2 = vzalloc(sizeof(struct dml2_context)));
 	return dml2;
 }
+
 bool dml2_validate(const struct dc *in_dc, struct dc_state *context, struct dml2_context *dml2,
 	enum dc_validate_mode validate_mode)
 {
@@ -87,7 +88,6 @@ static void dml2_init(const struct dc *in_dc, const struct dml2_configuration_op
 	initialize_dml2_soc_bbox(*dml2, in_dc, &(*dml2)->v20.dml_core_ctx.soc);
 
 	initialize_dml2_soc_states(*dml2, in_dc, &(*dml2)->v20.dml_core_ctx.soc, &(*dml2)->v20.dml_core_ctx.states);
-
 }
 
 bool dml2_create(const struct dc *in_dc, const struct dml2_configuration_options *config, struct dml2_context **dml2)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper.h
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper_fpu.c
similarity index 93%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper_fpu.c
index a14e3004a7b722..8a31986bc2990a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper_fpu.c
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: MIT */
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2023 Advanced Micro Devices, Inc.
  *
@@ -33,8 +33,8 @@
 #include "dml2_dc_resource_mgmt.h"
 #include "dml2_wrapper.h"
 #include "dml2_wrapper_fpu.h"
-#include "dml21_wrapper.h"
-#include "dml21_wrapper_fpu.h"
+#include "dml21_wrapper/dml21_wrapper.h"
+#include "dml21_wrapper/dml21_wrapper_fpu.h"
 
 void initialize_dml2_ip_params(struct dml2_context *dml2, const struct dc *in_dc, struct ip_params_st *out)
 {
@@ -65,7 +65,8 @@ static void map_hw_resources(struct dml2_context *dml2,
 		struct dml_display_cfg_st *in_out_display_cfg, struct dml_mode_support_info_st *mode_support_info)
 {
 	unsigned int num_pipes = 0;
-	int i, j;
+	int i;
+	unsigned int j;
 
 	for (i = 0; i < __DML_NUM_PLANES__; i++) {
 		in_out_display_cfg->hw.ODMMode[i] = mode_support_info->ODMMode[i];
@@ -120,7 +121,7 @@ static bool optimize_configuration(struct dml2_context *dml2, struct dml2_wrappe
 	int unused_dpps = p->ip_params->max_num_dpp;
 	int i;
 	int odms_needed;
-	int largest_blend_and_timing = 0;
+	unsigned int largest_blend_and_timing = 0;
 	bool optimization_done = false;
 
 	for (i = 0; i < (int) p->cur_display_config->num_timings; i++) {
@@ -164,7 +165,8 @@ static int calculate_lowest_supported_state_for_temp_read(struct dml2_context *d
 	struct dml2_wrapper_scratch *s_global = &dml2->v20.scratch;
 
 	unsigned int dml_result = 0;
-	int result = -1, i, j;
+	unsigned int state_idx;
+	int result = -1, i;
 
 	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
 
@@ -191,13 +193,13 @@ static int calculate_lowest_supported_state_for_temp_read(struct dml2_context *d
 
 	map_dc_state_into_dml_display_cfg(dml2, display_state, &s->cur_display_config);
 
-	for (i = 0; i < dml2->v20.dml_core_ctx.states.num_states; i++) {
-		s->uclk_change_latencies[i] = dml2->v20.dml_core_ctx.states.state_array[i].dram_clock_change_latency_us;
+	for (state_idx = 0; state_idx < (unsigned int)dml2->v20.dml_core_ctx.states.num_states; state_idx++) {
+		s->uclk_change_latencies[state_idx] = dml2->v20.dml_core_ctx.states.state_array[state_idx].dram_clock_change_latency_us;
 	}
 
 	for (i = 0; i < 4; i++) {
-		for (j = 0; j < dml2->v20.dml_core_ctx.states.num_states; j++) {
-			dml2->v20.dml_core_ctx.states.state_array[j].dram_clock_change_latency_us = s_global->dummy_pstate_table[i].dummy_pstate_latency_us;
+		for (state_idx = 0; state_idx < (unsigned int)dml2->v20.dml_core_ctx.states.num_states; state_idx++) {
+			dml2->v20.dml_core_ctx.states.state_array[state_idx].dram_clock_change_latency_us = s_global->dummy_pstate_table[i].dummy_pstate_latency_us;
 		}
 
 		dml_result = pack_and_call_dml_mode_support_ex(dml2, &s->cur_display_config, &s->evaluation_info,
@@ -221,8 +223,8 @@ static int calculate_lowest_supported_state_for_temp_read(struct dml2_context *d
 		}
 	}
 
-	for (i = 0; i < dml2->v20.dml_core_ctx.states.num_states; i++) {
-		dml2->v20.dml_core_ctx.states.state_array[i].dram_clock_change_latency_us = s->uclk_change_latencies[i];
+	for (state_idx = 0; state_idx < (unsigned int)dml2->v20.dml_core_ctx.states.num_states; state_idx++) {
+		dml2->v20.dml_core_ctx.states.state_array[state_idx].dram_clock_change_latency_us = s->uclk_change_latencies[state_idx];
 	}
 
 	return result;
@@ -230,7 +232,7 @@ static int calculate_lowest_supported_state_for_temp_read(struct dml2_context *d
 
 static void copy_dummy_pstate_table(struct dummy_pstate_entry *dest, struct dummy_pstate_entry *src, unsigned int num_entries)
 {
-	for (int i = 0; i < num_entries; i++) {
+	for (unsigned int i = 0; i < num_entries; i++) {
 		dest[i] = src[i];
 	}
 }
@@ -239,7 +241,7 @@ static bool are_timings_requiring_odm_doing_blending(const struct dml_display_cf
 		const struct dml_mode_support_info_st *evaluation_info)
 {
 	unsigned int planes_per_timing[__DML_NUM_PLANES__] = {0};
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < display_cfg->num_surfaces; i++)
 		planes_per_timing[display_cfg->plane.BlendingAndTiming[i]]++;
@@ -270,7 +272,8 @@ static bool dml_mode_support_wrapper(struct dml2_context *dml2,
 		enum dc_validate_mode validate_mode)
 {
 	struct dml2_wrapper_scratch *s = &dml2->v20.scratch;
-	unsigned int result = 0, i;
+	unsigned int result = 0;
+	int i;
 	unsigned int optimized_result = true;
 
 	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
@@ -383,7 +386,8 @@ static bool call_dml_mode_support_and_programming(struct dc_state *context, enum
 	 */
 	if (!context->streams[0]->sink->link->dc->caps.is_apu) {
 		if (min_state_for_g6_temp_read >= 0)
-			min_state = min_state_for_g6_temp_read > s->mode_support_params.out_lowest_state_idx ? min_state_for_g6_temp_read : s->mode_support_params.out_lowest_state_idx;
+			min_state = (unsigned int)min_state_for_g6_temp_read > s->mode_support_params.out_lowest_state_idx ?
+				(unsigned int)min_state_for_g6_temp_read : s->mode_support_params.out_lowest_state_idx;
 		else
 			min_state = s->mode_support_params.out_lowest_state_idx;
 	}
@@ -463,6 +467,7 @@ bool dml2_validate_and_build_resource(const struct dc *in_dc, struct dc_state *c
 
 	if (result) {
 		unsigned int lowest_state_idx = s->mode_support_params.out_lowest_state_idx;
+
 		out_clks.dispclk_khz = (unsigned int)dml2->v20.dml_core_ctx.mp.Dispclk_calculated * 1000;
 		out_clks.p_state_supported = s->mode_support_info.DRAMClockChangeSupport[0] != dml_dram_clock_change_unsupported;
 		if (in_dc->config.use_default_clock_table &&
@@ -499,8 +504,8 @@ bool dml2_validate_and_build_resource(const struct dc *in_dc, struct dc_state *c
 		cstate_enter_plus_exit_z8_ns = context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns;
 
 		if (context->bw_ctx.dml.vba.StutterPeriod < in_dc->debug.minimum_z8_residency_time &&
-				cstate_enter_plus_exit_z8_ns < in_dc->debug.minimum_z8_residency_time * 1000)
-			cstate_enter_plus_exit_z8_ns = in_dc->debug.minimum_z8_residency_time * 1000;
+				cstate_enter_plus_exit_z8_ns < (uint32_t)in_dc->debug.minimum_z8_residency_time * 1000)
+			cstate_enter_plus_exit_z8_ns = (uint32_t)in_dc->debug.minimum_z8_residency_time * 1000;
 
 		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns = cstate_enter_plus_exit_z8_ns;
 	}
@@ -527,8 +532,8 @@ bool dml2_validate_only(struct dc_state *context, enum dc_validate_mode validate
 	build_unoptimized_policy_settings(dml2->v20.dml_core_ctx.project, &dml2->v20.dml_core_ctx.policy);
 
 	map_dc_state_into_dml_display_cfg(dml2, context, &dml2->v20.scratch.cur_display_config);
-	 if (!dml2->config.skip_hw_state_mapping)
-		 dml2_apply_det_buffer_allocation_policy(dml2, &dml2->v20.scratch.cur_display_config);
+	if (!dml2->config.skip_hw_state_mapping)
+		dml2_apply_det_buffer_allocation_policy(dml2, &dml2->v20.scratch.cur_display_config);
 
 	result = pack_and_call_dml_mode_support_ex(dml2,
 		&dml2->v20.scratch.cur_display_config,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.h b/drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper_fpu.h
similarity index 100%
rename from drivers/gpu/drm/amd/display/dc/dml2_0/dml2_wrapper_fpu.h
rename to drivers/gpu/drm/amd/display/dc/dml2_wrapper/dml2_wrapper_fpu.h
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
index 8fe3474411b78d..083ab6acb6870d 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
@@ -1,6 +1,8 @@
 /*
  * Copyright 2020 Advanced Micro Devices, Inc.
  *
+ * SPDX-License-Identifier: MIT
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
index a57eb1cd77a6dd..e34d291de37b94 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.h
@@ -725,6 +725,8 @@ void dpp401_set_cursor_position(
 	uint32_t width,
 	uint32_t height);
 
+void dpp401_cursor_refresh_state(struct dpp *dpp_base);
+
 void dpp401_set_optional_cursor_attributes(
 	struct dpp *dpp_base,
 	struct dpp_cursor_attributes *attr);
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c
index 340111e3deacbf..c6de8518108e46 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c
@@ -144,6 +144,25 @@ void dpp401_set_cursor_position(
 	dpp_base->att.cur0_ctl.bits.cur0_enable = cur_en;
 }
 
+/* Resync the SW cursor cache from real hardware state. */
+void dpp401_cursor_refresh_state(struct dpp *dpp_base)
+{
+	struct dcn401_dpp *dpp = TO_DCN401_DPP(dpp_base);
+	uint32_t cur0_enable = 0, mode = 0, expansion_mode = 0, rom_en = 0;
+
+	REG_GET_4(CURSOR0_CONTROL,
+		CUR0_ENABLE, &cur0_enable,
+		CUR0_MODE, &mode,
+		CUR0_EXPANSION_MODE, &expansion_mode,
+		CUR0_ROM_EN, &rom_en);
+
+	dpp_base->pos.cur0_ctl.bits.cur0_enable = cur0_enable;
+	dpp_base->att.cur0_ctl.bits.cur0_enable = cur0_enable;
+	dpp_base->att.cur0_ctl.bits.mode = mode;
+	dpp_base->att.cur0_ctl.bits.expansion_mode = expansion_mode;
+	dpp_base->att.cur0_ctl.bits.cur0_rom_en = rom_en;
+}
+
 void dpp401_set_optional_cursor_attributes(
 	struct dpp *dpp_base,
 	struct dpp_cursor_attributes *attr)
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c
index 04cab7d3a5c27e..38739ed048fae8 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.c
@@ -25,17 +25,27 @@
 void dpp50_set_pregam_state(
 	struct dpp *dpp_base,
 	enum dc_transfer_func_predefined tr,
-	enum dc_scaling_linearity scaling)
+	enum dc_scaling_linearity scaling,
+	bool source_is_linear)
 {
 	struct dcn50_dpp *dpp = TO_DCN50_DPP(dpp_base);
 	enum pregam_mode pre_degam_en = PREGAM_DEGAM;
 	enum degam_lut degamma_lut_selection = 0;
 
 	if (scaling == DC_SCALING_LINEARITY_SOURCE) {
-		//If scaling in non-linear, apply regamma
-		REG_SET_2(PRE_GAM, 0,
-			PRE_GAM_MODE, PREGAM_REGAM,
-			PRE_REGAM_SELECT, REGAM_20);
+		if (source_is_linear) {
+			//Linear source data (e.g. FP16) must be de-linearized
+			//(apply regamma) before scaling in source/non-linear space.
+			REG_SET_2(PRE_GAM, 0,
+				PRE_GAM_MODE, PREGAM_REGAM,
+				PRE_REGAM_SELECT, REGAM_20);
+		} else {
+			//Non-linear source data (e.g. YUV) is already in source
+			//space, so scale directly with no pregam conversion.
+			REG_SET_2(PRE_GAM, 0,
+				PRE_GAM_MODE, PREGAM_BYPASS,
+				PRE_DEGAM_SELECT, 0);
+		}
 	} else {
 		//If scaling in linear, apply degamma based on TF
 		switch (tr) {
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h
index efa76579dc9ba2..c81e910ae095a4 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn50/dcn50_dpp.h
@@ -11,28 +11,29 @@
 #include "dcn30/dcn30_dpp.h"
 #include "dcn32/dcn32_dpp.h"
 #include "dcn401/dcn401_dpp.h"
+#include "dcn42/dcn42_dpp.h"
 
 
 #define TO_DCN50_DPP(dpp)\
 	container_of(dpp, struct dcn50_dpp, base)
 
 #define DPP_REG_LIST_SH_MASK_DCN50_COMMON(mask_sh)\
-	DPP_REG_LIST_SH_MASK_DCN401_COMMON(mask_sh), \
+	DPP_REG_LIST_SH_MASK_DCN42_COMMON(mask_sh), \
 	TF_SF(CNVC_CFG0_PRE_GAM, PRE_GAM_MODE, mask_sh), \
 	TF_SF(CNVC_CFG0_PRE_GAM, PRE_DEGAM_SELECT, mask_sh), \
 	TF_SF(CNVC_CFG0_PRE_GAM, PRE_REGAM_SELECT, mask_sh)
 
 #define DPP_REG_FIELD_LIST_DCN50(type) \
-	DPP_REG_FIELD_LIST_DCN401(type); \
+	DPP_REG_FIELD_LIST_DCN42(type); \
 	type PRE_GAM_MODE; \
 	type PRE_REGAM_SELECT
 
 #define DPP_REG_VARIABLE_LIST_DCN50 \
-	DPP_REG_VARIABLE_LIST_DCN401; \
-	uint32_t PRE_GAM;
+	DPP_REG_VARIABLE_LIST_DCN42; \
+	uint32_t PRE_GAM
 
 struct dcn50_dpp_registers {
-	DPP_REG_VARIABLE_LIST_DCN50
+	DPP_REG_VARIABLE_LIST_DCN50;
 };
 
 struct dcn50_dpp_shift {
@@ -81,6 +82,7 @@ void dpp50_dpp_setup(
 void dpp50_set_pregam_state(
 	struct dpp *dpp_base,
 	enum dc_transfer_func_predefined tr,
-	enum dc_scaling_linearity scaling);
+	enum dc_scaling_linearity scaling,
+	bool source_is_linear);
 
 #endif /* __DCN50_DPP_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c
index 24a686e9417167..f02aebcedd701f 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.c
@@ -308,6 +308,7 @@ static struct dpp_funcs dcn60_dpp_funcs = {
 	.dpp_cnv_set_alpha_keyer	= dpp2_cnv_set_alpha_keyer,
 	.set_cursor_attributes		= dpp401_set_cursor_attributes,
 	.set_cursor_position		= dpp401_set_cursor_position,
+	.refresh_cursor_state		= dpp401_cursor_refresh_state,
 	.set_optional_cursor_attributes	= dpp401_set_optional_cursor_attributes,
 	.dpp_dppclk_control			= dpp1_dppclk_control,
 	.dpp_set_hdr_multiplier		= dpp3_set_hdr_multiplier,
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h
index 96ff26a5ed65b7..a0cc57e79ff780 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn60/dcn60_dpp.h
@@ -414,9 +414,7 @@
 	TF_SF(CNVC_CFG0_CNVC_UPSP_CLAMP, UPSP_CLAMP_MIN, mask_sh)
 
 #define DPP_REG_FIELD_LIST_DCN60(type) \
-	DPP_REG_FIELD_LIST_DCN42(type); \
-	type PRE_GAM_MODE; \
-	type PRE_REGAM_SELECT; \
+	DPP_REG_FIELD_LIST_DCN50(type); \
 	type AUTOCAL_FRAC_MODE; \
 	type SCL_BLACK_COLOR_RGB_Y; \
 	type SCL_BLACK_COLOR_CBCR; \
@@ -448,8 +446,7 @@
 	type UPSP_CLAMP_MIN
 
 #define DPP_REG_VARIABLE_LIST_DCN60 \
-	DPP_REG_VARIABLE_LIST_DCN42; \
-	uint32_t PRE_GAM; \
+	DPP_REG_VARIABLE_LIST_DCN50; \
 	uint32_t SCL_BLACK_COLOR; \
 	uint32_t UPSP_MODE; \
 	uint32_t UPSP_V_COEF_P0; \
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
index ba4988118c689f..febb6af43827a3 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
@@ -1290,6 +1290,14 @@ static bool convert_bandwidth_to_frl_params(
 		*num_lanes = 4;
 		*frl_rate = 12000;
 		break;
+	case 64000000:
+		*num_lanes = 4;
+		*frl_rate = 16000;
+		break;
+	case 80000000:
+		*num_lanes = 4;
+		*frl_rate = 20000;
+		break;
 	default:
 		return false;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dcn60/dcn60_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dcn60/dcn60_dsc.c
index 0cfedd4268c676..5bb4ed1d1ba73b 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dcn60/dcn60_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dcn60/dcn60_dsc.c
@@ -60,7 +60,7 @@ void dsc60_construct(struct dcn60_dsc *dsc,
 	dsc->dsc_shift = dsc_shift;
 	dsc->dsc_mask = dsc_mask;
 
-	dsc->max_image_width = 5184;
+	dsc->max_image_width = 5760;
 }
 
 static void dsc60_init_reg_values(struct dsc60_reg_values *reg_vals)
@@ -181,7 +181,7 @@ static bool dsc60_prepare_config(const struct dsc_config *dsc_cfg, struct dsc60_
 
 	// TODO: in addition to validating slice height (pic height must be divisible by slice height),
 	// see what happens when the same condition doesn't apply for slice_width/pic_width.
-	dsc_reg_vals->pps.slice_width = (u16)(dsc_cfg->pic_width / dsc_cfg->dc_dsc_cfg.num_slices_h);
+	dsc_reg_vals->pps.slice_width = (u16)((dsc_cfg->pic_width + dsc_cfg->dsc_padding + dsc_cfg->dc_dsc_cfg.num_slices_h - 1) / dsc_cfg->dc_dsc_cfg.num_slices_h);
 	dsc_reg_vals->pps.slice_height = (u16)(dsc_cfg->pic_height / dsc_cfg->dc_dsc_cfg.num_slices_v);
 
 	ASSERT(dsc_reg_vals->pps.slice_height * dsc_cfg->dc_dsc_cfg.num_slices_v == dsc_cfg->pic_height);
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
index cf9c7a2089a184..c7c98a03ba7f23 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
@@ -469,10 +469,11 @@ static enum gpio_result dal_hw_ddc_set_config_i3cpad(
 	switch (config_data->config.ddc.type) {
 	/* For ASICs with i3cpad module there is no dual pad mode for i3cpads */
 	case GPIO_DDC_CONFIG_TYPE_MODE_I2C:
-		/* Enable the RX for the PAD (it is disabled by default). */
-		REG_UPDATE(dc_i3cpad_control1, DC_I3CPAD_RXSEL, 0);
-		return GPIO_RESULT_OK;
 	case GPIO_DDC_CONFIG_TYPE_MODE_AUX:
+		/* Enable the RX for the PAD (it is disabled by default). Required for
+		 * both I2C (EDID) and AUX (DPCD); without it the sink reply is never
+		 * received and the transaction times out. */
+		REG_UPDATE(dc_i3cpad_control1, DC_I3CPAD_RXSEL, 0);
 		return GPIO_RESULT_OK;
 
 	case GPIO_DDC_CONFIG_TYPE_POLL_FOR_CONNECT:
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
index 3300dbc67f3202..368ebcbf3fed4b 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c
@@ -43,9 +43,11 @@
 #if defined(CONFIG_DRM_AMD_DC_SI)
 #include "dce60/hw_factory_dce60.h"
 #endif
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 #include "dce80/hw_factory_dce80.h"
 #include "dce110/hw_factory_dce110.h"
 #include "dce120/hw_factory_dce120.h"
+#endif
 #include "dcn10/hw_factory_dcn10.h"
 #include "dcn20/hw_factory_dcn20.h"
 #include "dcn21/hw_factory_dcn21.h"
@@ -71,6 +73,7 @@ bool dal_hw_factory_init(
 		dal_hw_factory_dce60_init(factory);
 		return true;
 #endif
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 	case DCE_VERSION_8_0:
 	case DCE_VERSION_8_1:
 	case DCE_VERSION_8_3:
@@ -89,6 +92,7 @@ bool dal_hw_factory_init(
 	case DCE_VERSION_12_1:
 		dal_hw_factory_dce120_init(factory);
 		return true;
+#endif
 	case DCN_VERSION_1_0:
 	case DCN_VERSION_1_01:
 		dal_hw_factory_dcn10_init(factory);
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
index 5a231f97aee310..46cdc8b1069aab 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_translate.c
@@ -43,9 +43,11 @@
 #if defined(CONFIG_DRM_AMD_DC_SI)
 #include "dce60/hw_translate_dce60.h"
 #endif
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 #include "dce80/hw_translate_dce80.h"
 #include "dce110/hw_translate_dce110.h"
 #include "dce120/hw_translate_dce120.h"
+#endif
 #include "dcn10/hw_translate_dcn10.h"
 #include "dcn20/hw_translate_dcn20.h"
 #include "dcn21/hw_translate_dcn21.h"
@@ -75,6 +77,7 @@ bool dal_hw_translate_init(
 		dal_hw_translate_dce60_init(translate);
 		return true;
 #endif
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 	case DCE_VERSION_8_0:
 	case DCE_VERSION_8_1:
 	case DCE_VERSION_8_3:
@@ -90,6 +93,7 @@ bool dal_hw_translate_init(
 	case DCE_VERSION_12_1:
 		dal_hw_translate_dce120_init(translate);
 		return true;
+#endif
 	case DCN_VERSION_1_0:
 	case DCN_VERSION_1_01:
 		dal_hw_translate_dcn10_init(translate);
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn30/dcn30_hpo_frl_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/hpo/dcn30/dcn30_hpo_frl_stream_encoder.c
index cff5c95a771c40..f0128c5bf9936e 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/dcn30/dcn30_hpo_frl_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn30/dcn30_hpo_frl_stream_encoder.c
@@ -921,9 +921,9 @@ void hpo_enc3_read_state(
 	struct hpo_frl_stream_encoder *enc,
 	struct hpo_frl_stream_encoder_state *state)
 {
-	int pixel_encoding;
-	int color_depth;
-	int odm_combine;
+	uint32_t pixel_encoding;
+	uint32_t color_depth;
+	uint32_t odm_combine;
 	struct dcn30_hpo_frl_stream_encoder *enc3 = DCN30_HPO_FRL_STRENC_FROM_HPO_FRL_STRENC(enc);
 
 	ASSERT(state);
@@ -1040,4 +1040,6 @@ void dcn30_hpo_frl_stream_encoder_construct(
 	enc3->hpo_se_shift = hpo_se_shift;
 	enc3->hpo_se_mask = hpo_se_mask;
 	enc3->base.stream_enc_inst = vpg->inst;
+	enc3->base.inst = enc3->base.stream_enc_inst;
+
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn401/dcn401_hpo_frl_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/hpo/dcn401/dcn401_hpo_frl_stream_encoder.c
index 85b7a44c0a1170..4b14a3bfe56d6d 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/dcn401/dcn401_hpo_frl_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn401/dcn401_hpo_frl_stream_encoder.c
@@ -904,4 +904,6 @@ void dcn401_hpo_frl_stream_encoder_construct(
 	enc401->hpo_se_shift = hpo_se_shift;
 	enc401->hpo_se_mask = hpo_se_mask;
 	enc401->base.stream_enc_inst = vpg->inst;
+	enc401->base.inst = enc401->base.stream_enc_inst;
+
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_frl_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_frl_stream_encoder.c
index d4f66e62c729d5..1aed194b58bd06 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_frl_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn42/dcn42_hpo_frl_stream_encoder.c
@@ -159,11 +159,16 @@ void hpo_enc42_audio_mute_control(
 	struct hpo_frl_stream_encoder *enc,
 	bool mute)
 {
+	struct dcn401_hpo_frl_stream_encoder *enc401 = DCN401_HPO_FRL_STRENC_FROM_HPO_FRL_STRENC(enc);
 	ASSERT (enc->apg);
-	if (mute)
+
+	if (mute) {
 		enc->apg->funcs->disable_apg(enc->apg);
-	else
+		REG_UPDATE(HDMI_STREAM_ENC_AUDIO_CONTROL, HDMI_STREAM_ENC_APG_CLOCK_EN, 0);
+	} else {
+		REG_UPDATE(HDMI_STREAM_ENC_AUDIO_CONTROL, HDMI_STREAM_ENC_APG_CLOCK_EN, 1);
 		enc->apg->funcs->enable_apg(enc->apg);
+	}
 }
 
 static const struct hpo_frl_stream_encoder_funcs dcn42_str_enc_funcs = {
@@ -204,4 +209,6 @@ void dcn42_hpo_frl_stream_encoder_construct(
 	enc42->hpo_se_shift = hpo_se_shift;
 	enc42->hpo_se_mask = hpo_se_mask;
 	enc42->base.stream_enc_inst = vpg->inst;
+	enc42->base.inst = enc42->base.stream_enc_inst;
+
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn60/dcn60_hpo_frl_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/hpo/dcn60/dcn60_hpo_frl_stream_encoder.c
index 3f3704b68021ab..de42f8c5f7f0f1 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/dcn60/dcn60_hpo_frl_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn60/dcn60_hpo_frl_stream_encoder.c
@@ -32,144 +32,20 @@
 
 #define VBI_LINE_0 0
 
-/* setup stream encoder in hdmi mode */
-/* Precondition: link is trained */
-static void hpo_enc60_set_hdmi_stream_attribute(
-	struct hpo_frl_stream_encoder *enc,
-	struct dc_crtc_timing *crtc_timing,
-	struct frl_borrow_params *borrow_params,
-	int odm_combine_num_segments)
-{
-	(void)odm_combine_num_segments;
-	uint32_t h_active;
-	uint32_t h_blank;
-	struct dcn401_hpo_frl_stream_encoder *enc401 = DCN401_HPO_FRL_STRENC_FROM_HPO_FRL_STRENC(enc);
-
-	DC_LOG_DEBUG("Entering [%s]\n", __func__);
-
-	/* Configure pixel encoding */
-	switch (crtc_timing->pixel_encoding) {
-	case PIXEL_ENCODING_YCBCR422:
-		REG_UPDATE(HDMI_TB_ENC_PIXEL_FORMAT,
-				HDMI_PIXEL_ENCODING, 1);
-		REG_UPDATE_2(HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0,
-				FIFO_PIXEL_ENCODING_TYPE, 0,
-				FIFO_UNCOMPRESSED_PIXEL_FORMAT, 0);
-		break;
-	case PIXEL_ENCODING_YCBCR420:
-		REG_UPDATE(HDMI_TB_ENC_PIXEL_FORMAT,
-				HDMI_PIXEL_ENCODING, 2);
-		REG_UPDATE_2(HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0,
-				FIFO_PIXEL_ENCODING_TYPE, 0,
-				FIFO_UNCOMPRESSED_PIXEL_FORMAT, 1);
-		break;
-	default:
-		REG_UPDATE(HDMI_TB_ENC_PIXEL_FORMAT,
-				HDMI_PIXEL_ENCODING, 0);
-		REG_UPDATE_2(HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0,
-				FIFO_PIXEL_ENCODING_TYPE, 0,
-				FIFO_UNCOMPRESSED_PIXEL_FORMAT, 0);
-		break;
-	}
-
-	/* Configure color depth */
-	switch (crtc_timing->display_color_depth) {
-	case COLOR_DEPTH_888:
-		REG_UPDATE_2(HDMI_TB_ENC_PIXEL_FORMAT,
-				HDMI_DEEP_COLOR_DEPTH, 0,
-				HDMI_DEEP_COLOR_ENABLE, 0);
-		break;
-	case COLOR_DEPTH_101010:
-		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
-			REG_UPDATE_2(HDMI_TB_ENC_PIXEL_FORMAT,
-					HDMI_DEEP_COLOR_DEPTH, 1,
-					HDMI_DEEP_COLOR_ENABLE, 0);
-		} else {
-			REG_UPDATE_2(HDMI_TB_ENC_PIXEL_FORMAT,
-					HDMI_DEEP_COLOR_DEPTH, 1,
-					HDMI_DEEP_COLOR_ENABLE, 1);
-		}
-		break;
-	case COLOR_DEPTH_121212:
-		if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
-			REG_UPDATE_2(HDMI_TB_ENC_PIXEL_FORMAT,
-					HDMI_DEEP_COLOR_DEPTH, 2,
-					HDMI_DEEP_COLOR_ENABLE, 0);
-		} else {
-			REG_UPDATE_2(HDMI_TB_ENC_PIXEL_FORMAT,
-					HDMI_DEEP_COLOR_DEPTH, 2,
-					HDMI_DEEP_COLOR_ENABLE, 1);
-		}
-		break;
-	default:
-		break;
-	}
-
-	/* When compression active, CD/PP/Phase field shall be zero in GCP */
-	if (crtc_timing->flags.DSC) {
-		REG_UPDATE_2(HDMI_TB_ENC_PIXEL_FORMAT,
-				HDMI_DEEP_COLOR_DEPTH, 0,
-				HDMI_DEEP_COLOR_ENABLE, 0);
-	}
-
-	/* Configure horizontal active and blank size */
-	h_active = crtc_timing->h_addressable + crtc_timing->h_border_left + crtc_timing->h_border_right;
-	h_blank = crtc_timing->h_total - h_active;
-
-	if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR420 ||
-			crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422) {
-		h_active /= 2;
-		h_blank /= 2;
-	}
-
-	REG_SET_2(HDMI_TB_ENC_H_ACTIVE_BLANK, 0,
-			HDMI_H_ACTIVE, h_active,
-			HDMI_H_BLANK, h_blank);
-
-	/* Configure borrow parameters */
-	REG_UPDATE(HDMI_TB_ENC_MODE,
-			HDMI_BORROW_MODE, borrow_params->borrow_mode);
-	REG_UPDATE(HDMI_TB_ENC_PACKET_CONTROL,
-			HDMI_MAX_PACKETS_PER_LINE, borrow_params->audio_packets_line);
-	REG_SET_2(HDMI_TB_ENC_HC_ACTIVE_BLANK, 0,
-			HDMI_HC_ACTIVE, borrow_params->hc_active_target,
-			HDMI_HC_BLANK, borrow_params->hc_blank_target);
-
-	/* Enable transmission of General Control packet on every frame */
-	REG_UPDATE_2(HDMI_TB_ENC_VBI_PACKET_CONTROL1,
-		HDMI_GC_CONT, 1,
-		HDMI_GC_SEND, 1);
-
-	/* Disable Audio Content Protection packet transmission */
-	/* TODO: review if this needs to be here */
-	REG_UPDATE(HDMI_TB_ENC_VBI_PACKET_CONTROL1, HDMI_ACP_SEND, 0);
-
-	/* Enable Audio InfoFrame packet transmission. */
-	REG_UPDATE(HDMI_TB_ENC_VBI_PACKET_CONTROL1, HDMI_AUDIO_INFO_SEND, 1);
-
-	/* update double-buffered AUDIO_INFO registers immediately */
-//	ASSERT(enc->afmt);
-//	enc->afmt->funcs->audio_info_immediate_update(enc->afmt);
-
-	/* Select line number on which to send Audio InfoFrame packets */
-	REG_UPDATE(HDMI_TB_ENC_VBI_PACKET_CONTROL1, HDMI_AUDIO_INFO_LINE,
-				VBI_LINE_0 + 2);
-
-	/* set HDMI GC AVMUTE */
-	REG_UPDATE(HDMI_TB_ENC_GC_CONTROL, HDMI_GC_AVMUTE, 0);
-
-	DC_LOG_DEBUG("Exiting [%s]\n", __func__);
-}
-
 static void hpo_enc60_audio_mute_control(
 	struct hpo_frl_stream_encoder *enc,
 	bool mute)
 {
+	struct dcn401_hpo_frl_stream_encoder *enc401 = DCN401_HPO_FRL_STRENC_FROM_HPO_FRL_STRENC(enc);
 	ASSERT (enc->apg);
-	if (mute)
+
+	if (mute) {
 		enc->apg->funcs->disable_apg(enc->apg);
-	else
+		REG_UPDATE(HDMI_STREAM_ENC_AUDIO_CONTROL, HDMI_STREAM_ENC_APG_CLOCK_EN, 0);
+	} else {
+		REG_UPDATE(HDMI_STREAM_ENC_AUDIO_CONTROL, HDMI_STREAM_ENC_APG_CLOCK_EN, 1);
 		enc->apg->funcs->enable_apg(enc->apg);
+	}
 }
 
 //Covered both, rounding up or rounding down from FRL Link Rate /18.
@@ -321,7 +197,7 @@ static const struct hpo_frl_stream_encoder_funcs dcn401_str_enc_funcs = {
 	.hdmi_frl_enable		= hpo_enc401_enable,
 	.hdmi_frl_unblank		= hpo_enc401_unblank,
 	.hdmi_frl_blank			= hpo_enc401_blank,
-	.hdmi_frl_set_stream_attribute	= hpo_enc60_set_hdmi_stream_attribute,
+	.hdmi_frl_set_stream_attribute	= hpo_enc401_set_hdmi_stream_attribute,
 	.validate_hdmi_frl_output	= hpo_enc3_validate_hdmi_frl_output,
 	.update_hdmi_info_packets	= hpo_enc401_update_hdmi_info_packets,
 	.stop_hdmi_info_packets		= hpo_enc401_stop_hdmi_info_packets,
@@ -355,4 +231,6 @@ void dcn60_hpo_frl_stream_encoder_construct(
 	enc401->hpo_se_shift = hpo_se_shift;
 	enc401->hpo_se_mask = hpo_se_mask;
 	enc401->base.stream_enc_inst = vpg->inst;
+	enc401->base.inst = enc401->base.stream_enc_inst;
+
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c
index d683d0740c1336..4e33648f74bbef 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c
@@ -933,6 +933,8 @@ void hubbub1_construct(struct hubbub *hubbub,
 
 	hubbub1->base.ctx = ctx;
 
+	hubbub1->base.inst = 0;
+
 	hubbub1->base.funcs = &hubbub1_funcs;
 
 	hubbub1->regs = hubbub_regs;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
index accb79c6dde57d..53fac324748d38 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.h
@@ -143,6 +143,7 @@ struct dcn_hubbub_registers {
 	uint32_t DCHVM_CLK_CTRL;
 	uint32_t DCHVM_RIOMMU_CTRL0;
 	uint32_t DCHVM_RIOMMU_STAT0;
+	uint32_t DCHVM_PREFETCH_VFID;
 	uint32_t DCHUBBUB_DET0_CTRL;
 	uint32_t DCHUBBUB_DET1_CTRL;
 	uint32_t DCHUBBUB_DET2_CTRL;
@@ -215,6 +216,7 @@ struct dcn_hubbub_registers {
 	uint32_t DC_PERFMON5_PERFMON_HI;
 	uint32_t DC_PERFMON5_PERFMON_LOW;
 	uint32_t FMON_CTRL;
+	uint32_t UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0;
 };
 
 #define HUBBUB_REG_FIELD_LIST_DCN32(type) \
@@ -397,7 +399,8 @@ struct dcn_hubbub_registers {
 		type HOSTVM_PREFETCH_REQ; \
 		type HOSTVM_POWERSTATUS; \
 		type RIOMMU_ACTIVE; \
-		type HOSTVM_PREFETCH_DONE
+		type HOSTVM_PREFETCH_DONE; \
+		type HOSTVM_PREFETCH_VFID
 
 #define HUBBUB_RET_REG_FIELD_LIST(type) \
 		type DET_DEPTH;\
@@ -545,7 +548,11 @@ struct dcn_hubbub_registers {
 
 #define HUBBUB_REG_FIELD_LIST_DCN60(type) \
 		type DCHUBBUB_ARB_BUFFER_FULLNESS_WATERMARK_A;\
-		type DCHUBBUB_ARB_BUFFER_FULLNESS_WATERMARK_B
+		type DCHUBBUB_ARB_BUFFER_FULLNESS_WATERMARK_B;\
+		type SYSTEM_PROFILE_QC0;\
+		type SYSTEM_PROFILE_QC1;\
+		type SYSTEM_PROFILE_QC2;\
+		type SYSTEM_PROFILE_QC3
 
 struct dcn_hubbub_shift {
 	DCN_HUBBUB_REG_FIELD_LIST(uint8_t);
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c
index 053a08b6d3a3da..e5b55fe502ca6b 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c
@@ -574,9 +574,8 @@ void hubbub2_get_dchub_ref_freq(struct hubbub *hubbub,
 		else
 			*dchub_ref_freq_inKhz = dccg_ref_freq_inKhz;
 
-		// DC hub reference frequency must be around 50Mhz, otherwise there may be
-		// overflow/underflow issues when doing HUBBUB programming
-		if (*dchub_ref_freq_inKhz < 40000 || *dchub_ref_freq_inKhz > 60000)
+		// DC Hub reference frequency valid range is 20 to 50 MHz
+		if (*dchub_ref_freq_inKhz < 20000 || *dchub_ref_freq_inKhz > 50000)
 			ASSERT_CRITICAL(false);
 
 		return;
@@ -681,6 +680,8 @@ void hubbub2_construct(struct dcn20_hubbub *hubbub,
 {
 	hubbub->base.ctx = ctx;
 
+	hubbub->base.inst = 0;
+
 	hubbub->base.funcs = &hubbub2_funcs;
 
 	hubbub->regs = hubbub_regs;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn201/dcn201_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn201/dcn201_hubbub.c
index 63798132ed95b3..e6b527cd2047e9 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn201/dcn201_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn201/dcn201_hubbub.c
@@ -96,6 +96,8 @@ void hubbub201_construct(struct dcn20_hubbub *hubbub,
 {
 	hubbub->base.ctx = ctx;
 
+	hubbub->base.inst = 0;
+
 	hubbub->base.funcs = &hubbub201_funcs;
 
 	hubbub->regs = hubbub_regs;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c
index d790d6ee359a21..881e91e9385d86 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c
@@ -712,6 +712,8 @@ void hubbub21_construct(struct dcn20_hubbub *hubbub,
 {
 	hubbub->base.ctx = ctx;
 
+	hubbub->base.inst = 0;
+
 	hubbub->base.funcs = &hubbub21_funcs;
 
 	hubbub->regs = hubbub_regs;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c
index 6a7c1bee574720..582d647da81296 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c
@@ -478,6 +478,8 @@ void hubbub3_construct(struct dcn20_hubbub *hubbub3,
 	const struct dcn_hubbub_mask *hubbub_mask)
 {
 	hubbub3->base.ctx = ctx;
+
+	hubbub3->base.inst = 0;
 	hubbub3->base.funcs = &hubbub30_funcs;
 	hubbub3->regs = hubbub_regs;
 	hubbub3->shifts = hubbub_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn301/dcn301_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn301/dcn301_hubbub.c
index c1959672df50a2..f432874b32ddf2 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn301/dcn301_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn301/dcn301_hubbub.c
@@ -74,6 +74,8 @@ void hubbub301_construct(struct dcn20_hubbub *hubbub3,
 	const struct dcn_hubbub_mask *hubbub_mask)
 {
 	hubbub3->base.ctx = ctx;
+
+	hubbub3->base.inst = 0;
 	hubbub3->base.funcs = &hubbub301_funcs;
 	hubbub3->regs = hubbub_regs;
 	hubbub3->shifts = hubbub_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
index cbcd2278901398..8dd4b7b9ba36ff 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
@@ -1048,11 +1048,6 @@ void hubbub31_init(struct hubbub *hubbub)
 				DISPCLK_R_DCHUBBUB_GATE_DIS, 1,
 				DCFCLK_R_DCHUBBUB_GATE_DIS, 1);
 	}
-
-	/*
-	only the DCN will determine when to connect the SDP port
-	*/
-	REG_UPDATE(DCHUBBUB_SDPIF_CFG0,	SDPIF_PORT_CONTROL, 1);
 }
 static const struct hubbub_funcs hubbub31_funcs = {
 	.update_dchub = hubbub2_update_dchub,
@@ -1085,6 +1080,7 @@ void hubbub31_construct(struct dcn20_hubbub *hubbub31,
 	int pixel_chunk_size_kb,
 	int config_return_buffer_size_kb)
 {
+	hubbub31->base.inst = 0;
 
 	hubbub3_construct(hubbub31, ctx, hubbub_regs, hubbub_shift, hubbub_mask);
 	hubbub31->base.funcs = &hubbub31_funcs;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn32/dcn32_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn32/dcn32_hubbub.c
index 237331b3537868..dd9bc1f08b566f 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn32/dcn32_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn32/dcn32_hubbub.c
@@ -1050,6 +1050,8 @@ void hubbub32_construct(struct dcn20_hubbub *hubbub2,
 	int config_return_buffer_size_kb)
 {
 	hubbub2->base.ctx = ctx;
+
+	hubbub2->base.inst = 0;
 	hubbub2->base.funcs = &hubbub32_funcs;
 	hubbub2->regs = hubbub_regs;
 	hubbub2->shifts = hubbub_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c
index 5e5a7a74346d61..ad333cd3241bdd 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn35/dcn35_hubbub.c
@@ -31,6 +31,7 @@
 #include "dcn35_hubbub.h"
 #include "dm_services.h"
 #include "reg_helper.h"
+#include "dc_dmub_srv.h"
 
 
 #define CTX \
@@ -530,12 +531,7 @@ void hubbub35_init(struct hubbub *hubbub)
 	hubbub35_set_fgcg(hubbub2,
 			  hubbub->ctx->dc->debug.enable_fine_grain_clock_gating
 				  .bits.dchubbub);
-	/*
-	ignore the "df_pre_cstate_req" from the SDP port control.
-	only the DCN will determine when to connect the SDP port
-	*/
-	REG_UPDATE(DCHUBBUB_SDPIF_CFG0,
-			SDPIF_PORT_CONTROL, 1);
+
 	/*Set SDP's max outstanding request
 	When set to 1: Max outstanding is 512
 	When set to 0: Max outstanding is 256
@@ -580,7 +576,13 @@ void dcn35_dchvm_init(struct hubbub *hubbub)
 
 		//Reflect the power status of DCHUBBUB
 		REG_UPDATE(DCHVM_RIOMMU_CTRL0, HOSTVM_POWERSTATUS, 1);
+		udelay(5);
+	}
+
+	// generically re-allow the DCHVM<->rIOMMU SDP port to disconnect after powerstatus=1
+	dc_dmub_srv_hubbub_set_riommu_pctrl(hubbub->ctx, 0x20);
 
+	if (riommu_active) {
 		//Start rIOMMU prefetching
 		REG_UPDATE(DCHVM_RIOMMU_CTRL0, HOSTVM_PREFETCH_REQ, 1);
 
@@ -653,6 +655,8 @@ void hubbub35_construct(struct dcn20_hubbub *hubbub2,
 	int config_return_buffer_size_kb)
 {
 	hubbub2->base.ctx = ctx;
+
+	hubbub2->base.inst = 0;
 	hubbub2->base.funcs = &hubbub35_funcs;
 	hubbub2->regs = hubbub_regs;
 	hubbub2->shifts = hubbub_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
index e9ff20c02b1eb6..d948525bc09eef 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
@@ -1264,6 +1264,8 @@ void hubbub401_construct(struct dcn20_hubbub *hubbub2,
 	int config_return_buffer_size_kb)
 {
 	hubbub2->base.ctx = ctx;
+
+	hubbub2->base.inst = 0;
 	hubbub2->base.funcs = &hubbub4_01_funcs;
 	hubbub2->regs = hubbub_regs;
 	hubbub2->shifts = hubbub_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
index ddfc60e845a49d..b8657de84d3175 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
@@ -431,13 +431,6 @@ static void hubbub42_allow_self_refresh_control(struct hubbub *hubbub, bool allo
 			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE, 0,
 			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE, !allow);
 }
-static void hubbub42_set_sdp_control(struct hubbub *hubbub, bool dc_control)
-{
-	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
-
-	REG_UPDATE(DCHUBBUB_SDPIF_CFG0,
-			SDPIF_PORT_CONTROL, dc_control);
-}
 
 static bool hubbub42_program_watermarks(
 		struct hubbub *hubbub,
@@ -449,9 +442,8 @@ static bool hubbub42_program_watermarks(
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
 
 	if (!safe_to_lower && hubbub->ctx->dc->debug.disable_stutter_for_wm_program) {
-		/* before raising watermarks, SDP control give to DF, stutter must be disabled */
+		/* before raising watermarks, stutter must be disabled */
 		wm_pending = true;
-		hubbub42_set_sdp_control(hubbub, false);
 		hubbub42_allow_self_refresh_control(hubbub, false);
 	}
 	if (hubbub42_program_urgent_watermarks(hubbub, watermarks, safe_to_lower))
@@ -478,9 +470,6 @@ static bool hubbub42_program_watermarks(
 
 	if (safe_to_lower || hubbub->ctx->dc->debug.disable_stutter)
 		hubbub42_allow_self_refresh_control(hubbub, !hubbub->ctx->dc->debug.disable_stutter);
-	if (safe_to_lower && hubbub->ctx->dc->debug.disable_stutter_for_wm_program) {
-		hubbub42_set_sdp_control(hubbub, true);
-	}
 	hubbub32_force_usr_retraining_allow(hubbub, hubbub->ctx->dc->debug.force_usr_allow);
 
 	return wm_pending;
@@ -566,6 +555,8 @@ void hubbub42_construct(struct dcn20_hubbub *hubbub2,
 	int config_return_buffer_size_kb)
 {
 	hubbub2->base.ctx = ctx;
+
+	hubbub2->base.inst = 0;
 	hubbub2->base.funcs = &hubbub42_funcs;
 	hubbub2->regs = hubbub_regs;
 	hubbub2->shifts = hubbub_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c
index 3994e73a299200..639c71cf112301 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.c
@@ -18,6 +18,92 @@
 #define FN(reg_name, field_name) \
 	hubbub2->shifts->field_name, hubbub2->masks->field_name
 
+static void dcn60_control_dchvm_gating(const struct hubbub *hubbub, uint32_t disable_gating)
+{
+	const struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+
+	if (disable_gating != 0) {
+		REG_UPDATE(DCHVM_MEM_CTRL, HVM_GPUVMRET_FORCE_REQ, 0);
+	}
+
+	REG_UPDATE(DCHVM_MEM_CTRL, HVM_GPUVMRET_PWR_REQ_DIS, disable_gating);
+
+	REG_UPDATE_4(DCHVM_CLK_CTRL,
+		     HVM_DISPCLK_R_GATE_DIS, disable_gating,
+		     HVM_DISPCLK_G_GATE_DIS, disable_gating,
+		     HVM_DCFCLK_R_GATE_DIS, disable_gating,
+		     HVM_DCFCLK_G_GATE_DIS, disable_gating);
+
+	udelay(1);
+}
+
+static void dcn60_dchvm_init(struct hubbub *hubbub)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+	uint32_t riommu_active = 0;
+	uint32_t prefetch_done = 0;
+	unsigned int i;
+
+	/* If already prefetched return early. */
+	REG_GET(DCHVM_RIOMMU_STAT0, HOSTVM_PREFETCH_DONE, &prefetch_done);
+
+	if (prefetch_done) {
+		hubbub->riommu_active = true;
+		return;
+	}
+
+	/* Init DCHVM block. */
+	REG_UPDATE(DCHVM_CTRL0, HOSTVM_INIT_REQ, 1);
+
+	/* Poll until the rIOMMU and main IOMMU are active. */
+	for (i = 0; i < 100u; i++) {
+		REG_GET(DCHVM_RIOMMU_STAT0, RIOMMU_ACTIVE, &riommu_active);
+		if (riommu_active)
+			break;
+
+		udelay(5);
+	}
+
+	if (!riommu_active) {
+		DC_LOG_ERROR("Timed out waiting for DCHVM RIOMMU_ACTIVE\n");
+		return;
+	}
+
+	/* Disable gating and memory power requests. */
+	dcn60_control_dchvm_gating(hubbub, 1);
+
+	/* Select the physical function's host table for the initial prefetch. */
+	REG_UPDATE(DCHVM_PREFETCH_VFID, HOSTVM_PREFETCH_VFID, 1U << 31);
+
+	/* Reflect the power status of DCHUBBUB. */
+	REG_UPDATE(DCHVM_RIOMMU_CTRL0, HOSTVM_POWERSTATUS, 1);
+
+	/* Wait for power status to propagate to rIOMMU before prefetch req.*/
+	udelay(5);
+
+	/* Start rIOMMU prefetching. */
+	REG_UPDATE(DCHVM_RIOMMU_CTRL0, HOSTVM_PREFETCH_REQ, 1);
+
+	/* Poll for prefetch to be done. */
+	REG_WAIT(DCHVM_RIOMMU_STAT0, HOSTVM_PREFETCH_DONE, 1, 5, 100);
+
+	/* Re-enable gating and memory power requests. */
+	dcn60_control_dchvm_gating(hubbub, 0);
+
+	hubbub->riommu_active = true;
+}
+
+static int hubbub60_init_dchub_sys_ctx(struct hubbub *hubbub,
+		struct dcn_hubbub_phys_addr_config *pa_config)
+{
+	int num_vmids = hubbub3_init_dchub_sys_ctx(hubbub, pa_config);
+
+	if (hubbub->funcs->dchvm_init)
+		hubbub->funcs->dchvm_init(hubbub);
+
+	return num_vmids;
+}
+
 static void dcn60_init_crb(struct hubbub *hubbub)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
@@ -656,9 +742,19 @@ static void hubbub60_wm_read_state(struct hubbub *hubbub,
 static void hubbub60_perfmon_reset(struct hubbub *hubbub)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+	unsigned int counter;
 
 	REG_WRITE(DC_PERFMON5_PERFMON_CNTL, 0);
 	REG_WRITE(DC_PERFMON5_PERFMON_CNTL2, 0);
+
+	for (counter = 0; counter < 8; counter++) {
+		REG_SET_2(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
+				PERFCOUNTER_CNTL_SEL, counter,
+				PERFCOUNTER_OFF_MASK, 0x1);
+		REG_SET(DC_PERFMON5_PERFCOUNTER_CNTL2, 0,
+				PERFCOUNTER_CNTL2_SEL, counter);
+	}
+
 	REG_WRITE(DC_PERFMON5_PERFCOUNTER_STATE, 0);
 	REG_WRITE(DC_PERFMON5_PERFMON_CVALUE_INT_MISC, 0xFF00);
 	REG_WRITE(DC_PERFMON5_PERFMON_CVALUE_LOW, 0);
@@ -1288,21 +1384,33 @@ static uint32_t hubbub60_perfmon_get_urgent_ramp_latency_ns(
  * @hubbub: pointer to the hubbub hardware instance
  *
  * Configures the performance monitoring counters to measure out-of-order
- * (peak prefetch) bandwidth using hardware counters 0, 1, and 4:
+ * (peak prefetch) bandwidth using hardware counters 0, 1, 2, and 4:
  *
  * Counter 0: count-off counter — counts response-valid events and gates the
  *            measurement window once it reaches its target value.
  * Counter 1: data counter — tracks total data received during the measurement
- *            period; generates an interrupt when measurement completes.
+ *            period.
+ * Counter 2: target-reached generator — duplicates counter 1's event/target
+ *            and generates the counter-off event that stops counters 1 and 4.
  * Counter 4: duration timer — measures elapsed time in refclk cycles.
  *
  * UTM_FILTER_SEL is set to 0 so that isolation comes from OTG-vblank gating
- * rather than the silicon-broken HW filter.  The first 200 prefetch requests
- * are skipped (ramp-up), and the subsequent 200 are the measurement window.
+ * rather than the silicon-broken HW filter.
  *
  * This function configures counters only; call
  * hubbub60_perfmon_start_measuring_out_of_order_bandwidth() to enable them.
  */
+
+/* Counter0's ramp-up target and counter1's measurement-window target,
+ * packed into PERFMON_CVALUE_LOW's low/high words respectively. Tunable via
+ * windbg (ed dcn60_debug_peak_bw_ramp_size / ed dcn60_debug_peak_bw_window_size).
+ */
+static unsigned int dcn60_debug_peak_bw_ramp_size = 0x100;
+static unsigned int dcn60_debug_peak_bw_window_size = 5000;
+
+/* Minimum count4 (refclk ticks) for a peak-BW sample to be trusted. */
+#define DCN60_OUT_OF_ORDER_BW_MIN_DURATION_TICKS 20
+
 static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 		struct hubbub *hubbub)
 {
@@ -1321,7 +1429,7 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			UTM_FILTER_SEL, 0x0);
 
 	/* Program counter 0 as the count off counter */
-	REG_SET_8(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
+	REG_SET_9(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
 			PERFCOUNTER_CNTL_SEL, 0x0, // select counter 0
 			PERFCOUNTER_EVENT_SEL, 259, // response vld
 			PERFCOUNTER_CVALUE_SEL, 0x1, // use cvalue bits 15-0
@@ -1329,15 +1437,17 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			PERFCOUNTER_HW_CNTL_SEL, 0x0, // simutaneous mode
 			PERFCOUNTER_RUNEN_MODE, 0x0, // counter runs as long as run_enable is high
 			PERFCOUNTER_RESTART_EN, 0x0, // stop after counting is done
-			PERFCOUNTER_ACTIVE, 1);
+			PERFCOUNTER_OFF_MASK, 0x1, // do not generate pmon_counter_off_out
+			PERFCOUNTER_ACTIVE, 0x1);
 	REG_SET_4(DC_PERFMON5_PERFCOUNTER_CNTL2, 0,
 			PERFCOUNTER_CNTL2_SEL, 0x0, // select counter 0
 			PERFCOUNTER_COUNTED_VALUE_TYPE, 0x0, // count the accumulated value
 			PERFCOUNTER_HW_STOP1_SEL, 0x1, // the stop trigger is that perfcounter meet the target CVALUE
 			PERFCOUNTER_HW_STOP2_SEL, 0x0); // ignored
 
-	/* Program counter 1 to count total data received */
-	REG_SET_9(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
+	/* Program counter 1 as a passive data accumulator, slaved to the same
+	 * external stop signal as counter 4 (generated by counter 2 below). */
+	REG_SET_10(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
 			PERFCOUNTER_CNTL_SEL, 0x1, // select counter 1
 			PERFCOUNTER_EVENT_SEL, 259, // response vld
 			PERFCOUNTER_CVALUE_SEL, 0x2, // use cvalue bits 31-16
@@ -1345,17 +1455,39 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			PERFCOUNTER_HW_CNTL_SEL, 0x1, // independent mode
 			PERFCOUNTER_RUNEN_MODE, 0x0, // counter runs as long as run_enable is high
 			PERFCOUNTER_RESTART_EN, 0x0, // stop after counting is done
-			PERFCOUNTER_INT_EN, 1, // signal when the measurement is complete
+			PERFCOUNTER_INT_EN, 0, // counter 2 is the completion source
+			PERFCOUNTER_OFF_MASK, 0x1, // do not generate pmon_counter_off_out
 			PERFCOUNTER_ACTIVE, 0x1);
 	REG_SET_5(DC_PERFMON5_PERFCOUNTER_CNTL2, 0,
 			PERFCOUNTER_CNTL2_SEL, 0x1,
 			PERFCOUNTER_CNTOFF_SEL, 0, // start when count0 stops
 			PERFCOUNTER_COUNTED_VALUE_TYPE, 0x0, // count the accumulated value
 			PERFCOUNTER_HW_STOP1_SEL, 0x0, // ignored
+			PERFCOUNTER_HW_STOP2_SEL, 0x1); // stop via the same external signal as counter 4
+
+	/* Counter 2 is a dedicated "target reached" generator: it duplicates
+	 * counter 1's event/target and is the sole source of the counter-off
+	 * event driving PERFMON_RUN_ENABLE_STOP_SEL. */
+	REG_SET_10(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
+			PERFCOUNTER_CNTL_SEL, 0x2, // select counter 2
+			PERFCOUNTER_EVENT_SEL, 259, // response vld
+			PERFCOUNTER_CVALUE_SEL, 0x2, // use cvalue bits 31-16 (same window target as counter 1)
+			PERFCOUNTER_INC_MODE, 0x2, // count LSB level
+			PERFCOUNTER_HW_CNTL_SEL, 0x1, // independent mode
+			PERFCOUNTER_RUNEN_MODE, 0x0, // counter runs as long as run_enable is high
+			PERFCOUNTER_RESTART_EN, 0x0, // stop after counting is done
+			PERFCOUNTER_INT_EN, 0, // counter-off is the completion source
+			PERFCOUNTER_OFF_MASK, 0x0, // generate pmon_counter_off_out at CVALUE
+			PERFCOUNTER_ACTIVE, 0x1);
+	REG_SET_5(DC_PERFMON5_PERFCOUNTER_CNTL2, 0,
+			PERFCOUNTER_CNTL2_SEL, 0x2,
+			PERFCOUNTER_CNTOFF_SEL, 0, // start when count0 stops
+			PERFCOUNTER_COUNTED_VALUE_TYPE, 0x0, // count the accumulated value
+			PERFCOUNTER_HW_STOP1_SEL, 0x0, // ignored
 			PERFCOUNTER_HW_STOP2_SEL, 0x0); // the stop trigger is that perfcounter meet the target CVALUE
 
 	/* Program counter 4 to count the measuring time */
-	REG_SET_8(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
+	REG_SET_9(DC_PERFMON5_PERFCOUNTER_CNTL, 0,
 			PERFCOUNTER_CNTL_SEL, 0x4, // select counter 4
 			PERFCOUNTER_EVENT_SEL, 19, // always 1 event
 			PERFCOUNTER_CVALUE_SEL, 0x0, // ignored
@@ -1363,6 +1495,7 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			PERFCOUNTER_HW_CNTL_SEL, 0x1, // independent mode
 			PERFCOUNTER_RUNEN_MODE, 0x0, // counter runs as long as run_enable is high
 			PERFCOUNTER_RESTART_EN, 0x0, // stop after counting is done
+			PERFCOUNTER_OFF_MASK, 0x1, // do not generate pmon_counter_off_out
 			PERFCOUNTER_ACTIVE, 1);
 	REG_SET_5(DC_PERFMON5_PERFCOUNTER_CNTL2, 0,
 			PERFCOUNTER_CNTL2_SEL, 0x4, // select counter 4
@@ -1372,25 +1505,25 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
 			PERFCOUNTER_HW_STOP2_SEL, 0x1); // the stop trigger is from the external 64 pairs of start/stop events
 
 	/* Program perfcounter states */
-	REG_SET_6(DC_PERFMON5_PERFCOUNTER_STATE, 0,
+	REG_SET_8(DC_PERFMON5_PERFCOUNTER_STATE, 0,
 			PERFCOUNTER_STATE_SEL0, 0x1, // independent state mode
 			PERFCOUNTER_CNT0_STATE, 0x3, // hw mode
 			PERFCOUNTER_STATE_SEL1, 0x1, // independent state mode
 			PERFCOUNTER_CNT1_STATE, 0x3, // hw mode
+			PERFCOUNTER_STATE_SEL2, 0x1, // independent state mode
+			PERFCOUNTER_CNT2_STATE, 0x3, // hw mode
 			PERFCOUNTER_STATE_SEL4, 0x1, // independent state mode
 			PERFCOUNTER_CNT4_STATE, 0x3); // hw mode
 
-	/*
-	 * The cvalue is derived based on experimental results at the lowest
-	 * clock state. Out-of-order bandwidth requires ~200 prefetch requests
-	 * to ramp up to full speed; the subsequent 200 requests form the
-	 * measurement window.
-	 */
-	REG_SET(DC_PERFMON5_PERFMON_CVALUE_LOW, 0, PERFMON_CVALUE_LOW, 200 | 200 << 16);
+	/* Ramp/window sizes are derived from experimental results at the
+	 * lowest clock state. */
+	REG_SET(DC_PERFMON5_PERFMON_CVALUE_LOW, 0, PERFMON_CVALUE_LOW,
+			(dcn60_debug_peak_bw_ramp_size & 0xFFFF)
+					| (dcn60_debug_peak_bw_window_size & 0xFFFF) << 16);
 
 	REG_SET_2(DC_PERFMON5_PERFMON_CNTL2, 0,
 			PERFMON_RUN_ENABLE_START_SEL, 0x0,
-			PERFMON_RUN_ENABLE_STOP_SEL, 11); // perfmon counter off event
+			PERFMON_RUN_ENABLE_STOP_SEL, 10); // stop on counter-off event
 }
 
 /**
@@ -1398,9 +1531,9 @@ static void hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(
  * @hubbub: pointer to the hubbub hardware instance
  *
  * Enables the performance monitor counters previously configured by
- * hubbub60_perfmon_arm_measuring_out_of_order_bandwidth().  The counter
- * self-stops once the count-off counter reaches its target; there is no
- * explicit stop step for the peak-BW path.
+ * hubbub60_perfmon_arm_measuring_out_of_order_bandwidth(). Counter 2 stops
+ * the measurement once it reaches its target; there is no explicit stop
+ * step for the peak-BW path.
  */
 static void hubbub60_perfmon_start_measuring_out_of_order_bandwidth(
 		struct hubbub *hubbub)
@@ -1418,28 +1551,49 @@ static void hubbub60_perfmon_start_measuring_out_of_order_bandwidth(
  * @refclk_mhz: reference clock frequency in MHz, used for duration conversion
  * @duration_ns: output parameter; receives the measured duration in nanoseconds
  *
- * Reads hardware counters 1 (data) and 4 (duration) and converts the raw
- * refclk-cycle count into bandwidth in Mbps.
+ * Reads hardware counters 1 (data), 2 (target-reached generator), and 4
+ * (duration), validates the sample, and converts the raw refclk-cycle count
+ * into bandwidth in Mbps.
  *
- * Return: out-of-order bandwidth in Mbps
+ * Return: out-of-order bandwidth in Mbps, or 0 if the sample is invalid
  */
 static uint32_t hubbub60_perfmon_get_out_of_order_bandwidth_mbps(
 		struct hubbub *hubbub, uint32_t refclk_mhz, uint32_t *duration_ns)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
-	uint32_t count0 = 0, count1 = 0, count4 = 0,
+	uint32_t count1 = 0, count2 = 0, count4 = 0,
 			out_of_order_bandwidth_mbps = 0, measuring_duration_ns = 0;
 	struct fixed31_32 temp;
 
-	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x0);
-	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count0);
-
 	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x1);
 	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count1);
 
+	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x2);
+	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count2);
+
 	REG_SET(DC_PERFMON5_PERFMON_HI, 0, PERFMON_READ_SEL, 0x4);
 	REG_GET(DC_PERFMON5_PERFMON_LOW, PERFMON_LOW, &count4);
 
+	/* count1 should never exceed count2's target, doing so means it
+	 * missed the shared stop edge and the sample is corrupted. */
+	if (count1 > count2 + (count2 / 20)) {
+		DC_LOG_WARNING("hubbub60_perfmon_get_out_of_order_bandwidth_mbps: "
+				"count1=%u exceeds count2=%u target, discarding corrupted sample\n",
+				count1, count2);
+		if (duration_ns)
+			*duration_ns = 0;
+		return 0;
+	}
+
+	if (count4 < DCN60_OUT_OF_ORDER_BW_MIN_DURATION_TICKS) {
+		DC_LOG_WARNING("hubbub60_perfmon_get_out_of_order_bandwidth_mbps: "
+				"count4=%u below minimum %u ticks, discarding sample\n",
+				count4, DCN60_OUT_OF_ORDER_BW_MIN_DURATION_TICKS);
+		if (duration_ns)
+			*duration_ns = 0;
+		return 0;
+	}
+
 	if (refclk_mhz == 0)
 		return 0;
 
@@ -1573,7 +1727,8 @@ static uint32_t hubbub60_perfmon_get_in_order_bandwidth_mbps(
 		return 0;
 
 	measuring_duration_ns = count4 * 1000 / refclk_mhz;
-	*duration_ns = measuring_duration_ns;
+	if (duration_ns)
+		*duration_ns = measuring_duration_ns;
 	if (min_duration_ns > measuring_duration_ns)
 		return 0;
 
@@ -1730,9 +1885,29 @@ static void hubbub60_reset_display_qos_profile(struct hubbub *hubbub)
 	REG_UPDATE(DCHUBBUB_ARB_QOS_FORCE, DCHUBBUB_ARB_UTM_FORCE_URGENT, 0);
 }
 
+static void hubbub60_override_utm_client_qc_profile(struct hubbub *hubbub, uint8_t qc_profile, int index)
+{
+	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
+
+	switch (index) {
+	case 0:
+		REG_UPDATE(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0, SYSTEM_PROFILE_QC0, qc_profile);
+		break;
+	case 1:
+		REG_UPDATE(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0, SYSTEM_PROFILE_QC1, qc_profile);
+		break;
+	case 2:
+		REG_UPDATE(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0, SYSTEM_PROFILE_QC2, qc_profile);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+}
+
 static const struct hubbub_funcs hubbub60_funcs = {
 	.update_dchub = hubbub2_update_dchub,
-	.init_dchub_sys_ctx = hubbub3_init_dchub_sys_ctx,
+	.init_dchub_sys_ctx = hubbub60_init_dchub_sys_ctx,
 	.init_vm_ctx = hubbub2_init_vm_ctx,
 	.dcc_support_swizzle_addr3 = hubbub401_dcc_support_swizzle,
 	.dcc_support_pixel_format_plane0_plane1 =
@@ -1748,6 +1923,7 @@ static const struct hubbub_funcs hubbub60_funcs = {
 	.force_pstate_change_control = hubbub3_force_pstate_change_control,
 	.init_watermarks = hubbub60_init_watermarks,
 	.init_crb = dcn60_init_crb,
+	.dchvm_init = dcn60_dchvm_init,
 	.hubbub_read_state = hubbub2_read_state,
 	.force_usr_retraining_allow = NULL,
 	.set_request_limit = hubbub32_set_request_limit,
@@ -1755,6 +1931,7 @@ static const struct hubbub_funcs hubbub60_funcs = {
 	.program_compbuf_segments = dcn60_program_compbuf_segments,
 	.wait_for_det_update = dcn60_wait_for_det_update,
 	.program_arbiter = dcn401_program_arbiter,
+	.override_utm_client_qc_profile = hubbub60_override_utm_client_qc_profile,
 	.hubbub_read_reg_state = hubbub3_read_reg_state,
 	.perfmon = {
 		.reset = hubbub60_perfmon_reset,
@@ -1805,6 +1982,8 @@ void hubbub60_construct(struct dcn20_hubbub *hubbub2,
 	int config_return_buffer_size_kb)
 {
 	hubbub2->base.ctx = ctx;
+
+	hubbub2->base.inst = 0;
 	hubbub2->base.funcs = &hubbub60_funcs;
 	hubbub2->regs = hubbub_regs;
 	hubbub2->shifts = hubbub_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.h b/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.h
index 204f422c338336..ed47c462e8fe37 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn60/dcn60_hubbub.h
@@ -16,6 +16,19 @@
 #define DCN6_0_CRB_SEGMENT_SIZE_KB 64
 
  #define HUBBUB_MASK_SH_LIST_DCN6_0(mask_sh)\
+	HUBBUB_SF(DCHVM_CTRL0, HOSTVM_INIT_REQ, mask_sh), \
+	HUBBUB_SF(DCHVM_MEM_CTRL, HVM_GPUVMRET_PWR_REQ_DIS, mask_sh), \
+	HUBBUB_SF(DCHVM_MEM_CTRL, HVM_GPUVMRET_FORCE_REQ, mask_sh), \
+	HUBBUB_SF(DCHVM_MEM_CTRL, HVM_GPUVMRET_POWER_STATUS, mask_sh), \
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DISPCLK_R_GATE_DIS, mask_sh), \
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DISPCLK_G_GATE_DIS, mask_sh), \
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DCFCLK_R_GATE_DIS, mask_sh), \
+	HUBBUB_SF(DCHVM_CLK_CTRL, HVM_DCFCLK_G_GATE_DIS, mask_sh), \
+	HUBBUB_SF(DCHVM_RIOMMU_CTRL0, HOSTVM_PREFETCH_REQ, mask_sh), \
+	HUBBUB_SF(DCHVM_RIOMMU_CTRL0, HOSTVM_POWERSTATUS, mask_sh), \
+	HUBBUB_SF(DCHVM_RIOMMU_STAT0, RIOMMU_ACTIVE, mask_sh), \
+	HUBBUB_SF(DCHVM_RIOMMU_STAT0, HOSTVM_PREFETCH_DONE, mask_sh), \
+	HUBBUB_SF(DCHVM_PREFETCH_VFID, HOSTVM_PREFETCH_VFID, mask_sh), \
 	HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, mask_sh), \
 	HUBBUB_SF(DCHUBBUB_SOFT_RESET, DCHUBBUB_GLOBAL_SOFT_RESET, mask_sh), \
 	HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, mask_sh), \
@@ -193,7 +206,11 @@
 	HUBBUB_SF(DCHUBBUB_ARB_QOS_FORCE, DCHUBBUB_ARB_UTM_FORCE_URGENT, mask_sh), \
 	HUBBUB_SF(DCHUBBUB_ARB_QOS_FORCE, DCHUBBUB_ARB_UTM_FORCE_BANDWIDTH_OVERHEAD, mask_sh), \
 	HUBBUB_SF(DCHUBBUB_ARB_QOS_FORCE, DCHUBBUB_ARB_UTM_FORCE_ENABLE, mask_sh), \
-	HUBBUB_SF(DCHUBBUB_ARB_QOS_FORCE, DCHUBBUB_ARB_DO_NOT_FORCE_URGENCY_DURING_PSTATE_CHANGE_REQUEST, mask_sh)
+	HUBBUB_SF(DCHUBBUB_ARB_QOS_FORCE, DCHUBBUB_ARB_DO_NOT_FORCE_URGENCY_DURING_PSTATE_CHANGE_REQUEST, mask_sh), \
+	HUBBUB_SF(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0, SYSTEM_PROFILE_QC0, mask_sh), \
+	HUBBUB_SF(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0, SYSTEM_PROFILE_QC1, mask_sh), \
+	HUBBUB_SF(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0, SYSTEM_PROFILE_QC2, mask_sh), \
+	HUBBUB_SF(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0, SYSTEM_PROFILE_QC3, mask_sh)
 
 void hubbub60_construct(struct dcn20_hubbub *hubbub2,
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c
index d8eb5996b57745..2a4be0c7a724b5 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c
@@ -350,10 +350,12 @@ void hubp1_program_pixel_format(
 bool hubp1_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate)
+	bool flip_immediate,
+	bool dcc)
 {
 	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
 
+	(void)dcc;
 
 	//program flip type
 	REG_UPDATE(DCSURF_FLIP_CONTROL,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.h
index f2571076fc501d..0c1d5e518f970e 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.h
@@ -905,7 +905,8 @@ void hubp_reset(struct hubp *hubp);
 bool hubp1_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate);
+	bool flip_immediate,
+	bool dcc);
 
 bool hubp1_is_flip_pending(struct hubp *hubp);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c
index cf2c9f80056eb8..79a42981406179 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c
@@ -730,10 +730,13 @@ bool hubp2_dmdata_status_done(struct hubp *hubp)
 bool hubp2_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate)
+	bool flip_immediate,
+	bool dcc)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
+	(void)dcc;
+
 	//program flip type
 	REG_UPDATE(DCSURF_FLIP_CONTROL,
 			SURFACE_FLIP_TYPE, flip_immediate);
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
index ea90dee05d5f31..3ae06bf89f7951 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.h
@@ -399,7 +399,8 @@ void hubp2_program_deadline(
 bool hubp2_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate);
+	bool flip_immediate,
+	bool dcc);
 
 void hubp2_dcc_control(struct hubp *hubp, bool enable,
 		enum hubp_ind_block_size independent_64b_blks);
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c
index 67828505939a3e..c667db319baf82 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c
@@ -74,7 +74,7 @@
  * then update the regs with the new values.
  *
  */
-void apply_DEDCN21_142_wa_for_hostvm_deadline(
+void apply_hubp21_wa_for_hostvm_deadline(
 		struct hubp *hubp,
 		struct _vcs_dpi_display_dlg_regs_st *dlg_attr)
 {
@@ -131,7 +131,7 @@ void hubp21_program_deadline(
 {
 	hubp2_program_deadline(hubp, dlg_attr, ttu_attr);
 
-	apply_DEDCN21_142_wa_for_hostvm_deadline(hubp, dlg_attr);
+	apply_hubp21_wa_for_hostvm_deadline(hubp, dlg_attr);
 }
 
 void hubp21_program_requestor(
@@ -696,10 +696,13 @@ static void dmcub_PLAT_54186_wa(struct hubp *hubp,
 static bool hubp21_program_surface_flip_and_addr(
 		struct hubp *hubp,
 		const struct dc_plane_address *address,
-		bool flip_immediate)
+		bool flip_immediate,
+		bool dcc)
 {
 	struct surface_flip_registers flip_regs = { 0 };
 
+	(void)dcc;
+
 	flip_regs.vmid = address->vmid;
 
 	switch (address->type) {
@@ -803,7 +806,7 @@ static bool hubp21_program_surface_flip_and_addr(
 
 static void hubp21_init(struct hubp *hubp)
 {
-	// DEDCN21-133: Inconsistent row starting line for flip between DPTE and Meta
+	// Inconsistent row starting line for flip between DPTE and Meta
 	// This is a chicken bit to enable the ECO fix.
 
 	struct dcn21_hubp *hubp21 = TO_DCN21_HUBP(hubp);
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.h
index 9873b6cbc5bab5..418a981b8c9b85 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.h
@@ -119,7 +119,7 @@ bool hubp21_construct(
 	const struct dcn_hubp2_shift *hubp_shift,
 	const struct dcn_hubp2_mask *hubp_mask);
 
-void apply_DEDCN21_142_wa_for_hostvm_deadline(
+void apply_hubp21_wa_for_hostvm_deadline(
 		struct hubp *hubp,
 		struct _vcs_dpi_display_dlg_regs_st *dlg_attr);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c
index 3e5ae0eae39f47..72698b5cf5f79d 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c
@@ -68,10 +68,13 @@ void hubp3_set_vm_system_aperture_settings(struct hubp *hubp,
 bool hubp3_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate)
+	bool flip_immediate,
+	bool dcc)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
+	(void)dcc;
+
 	//program flip type
 	REG_UPDATE(DCSURF_FLIP_CONTROL,
 			SURFACE_FLIP_TYPE, flip_immediate);
@@ -615,7 +618,7 @@ void hubp3_setup(
 
 void hubp3_init(struct hubp *hubp)
 {
-	// DEDCN21-133: Inconsistent row starting line for flip between DPTE and Meta
+	// Inconsistent row starting line for flip between DPTE and Meta
 	// This is a chicken bit to enable the ECO fix.
 
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.h
index c767e9f4f9b312..faa7e0379f4e1f 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.h
@@ -260,7 +260,8 @@ void hubp3_set_vm_system_aperture_settings(struct hubp *hubp,
 bool hubp3_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate);
+	bool flip_immediate,
+	bool dcc);
 
 void hubp3_program_surface_config(
 	struct hubp *hubp,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn31/dcn31_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn31/dcn31_hubp.c
index 189045f8503997..9cc5766235f06d 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn31/dcn31_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn31/dcn31_hubp.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2012-20 Advanced Micro Devices, Inc.
  *
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
index a2e0bafb711a2a..798d0dad9153e2 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
@@ -126,6 +126,10 @@ static enum hubp_3dlut_fl_width hubp401_get_3dlut_fl_width(
 	enum hubp_3dlut_fl_width width = 0;
 
 	switch (size) {
+	case CM_LUT_SIZE_999:
+		ASSERT(swizzle != CM_LUT_1D_PACKED_LINEAR);
+		width = hubp_3dlut_fl_width_9;
+		break;
 	case CM_LUT_SIZE_333333:
 		ASSERT(swizzle != CM_LUT_1D_PACKED_LINEAR);
 		width = hubp_3dlut_fl_width_33;
@@ -482,10 +486,13 @@ void hubp401_setup_interdependent(
 bool hubp401_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate)
+	bool flip_immediate,
+	bool dcc)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
+	(void)dcc;
+
 	//program flip type
 	REG_UPDATE(DCSURF_FLIP_CONTROL,
 			SURFACE_FLIP_TYPE, flip_immediate);
@@ -809,24 +816,11 @@ void hubp401_cursor_set_position(
 	const struct dc_cursor_mi_param *param)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
-	int rec_x_offset = pos->x - pos->x_hotspot;
-	int rec_y_offset = pos->y - pos->y_hotspot;
-	int dst_x_offset;
-	int x_pos_viewport = 0;
-	int x_hot_viewport = 0;
+	int dst_x_offset = param->dst_x_offset;
 	uint32_t cur_en = pos->enable ? 1 : 0;
-	uint32_t x_hotspot_clamped = pos->x_hotspot;
+	/* CURSOR_HOT_SPOT_X is 8 bits wide; clamp only the register write */
+	uint32_t x_hotspot_clamped = pos->x_hotspot > 0xFF ? 0xFF : pos->x_hotspot;
 	hubp->curs_pos = *pos;
-	/* Recout is zero for pipes if the entire dst_rect is contained
-	 * within preceeding ODM slices.
-	 */
-	if (param->recout.width) {
-		x_pos_viewport = pos->x * param->viewport.width / param->recout.width;
-		x_hot_viewport = pos->x_hotspot * param->viewport.width / param->recout.width;
-	} else {
-		ASSERT(!cur_en || pos->x == 0);
-		ASSERT(!cur_en || pos->x_hotspot == 0);
-	}
 
 	/*
 	 * Guard aganst cursor_set_position() from being called with invalid
@@ -835,25 +829,6 @@ void hubp401_cursor_set_position(
 	if (hubp->curs_attr.address.quad_part == 0)
 		return;
 
-	/* Translate the x position of the cursor from rect
-	 * space into viewport space. CURSOR_DST_X_OFFSET
-	 * is the offset relative to viewport start position.
-	 */
-	dst_x_offset = x_pos_viewport - x_hot_viewport *
-			(1 + hubp->curs_attr.attribute_flags.bits.ENABLE_MAGNIFICATION);
-	dst_x_offset = (dst_x_offset >= 0) ? dst_x_offset : 0;
-	dst_x_offset *= param->ref_clk_khz;
-	dst_x_offset /= param->pixel_clk_khz;
-
-	ASSERT(param->h_scale_ratio.value);
-
-	if (x_hotspot_clamped > 0xFF)
-		x_hotspot_clamped = 0xFF;
-	if (param->h_scale_ratio.value)
-		dst_x_offset = dc_fixpt_floor(dc_fixpt_div(
-			dc_fixpt_from_int(dst_x_offset),
-			param->h_scale_ratio));
-
 	if (hubp->pos.cur_ctl.bits.cur_enable != cur_en) {
 		if (cur_en && REG_READ(CURSOR_SURFACE_ADDRESS) == 0)
 			hubp->funcs->set_cursor_attributes(hubp, &hubp->curs_attr);
@@ -882,18 +857,63 @@ void hubp401_cursor_set_position(
 	hubp->pos.hot_spot.bits.x_hot = pos->x_hotspot;
 	hubp->pos.hot_spot.bits.y_hot = pos->y_hotspot;
 	hubp->pos.dst_offset.bits.dst_x_offset = dst_x_offset;
-	/* Cursor Rectangle Cache
-	 * Cursor bitmaps have different hotspot values
-	 * There's a possibility that the above logic returns a negative value,
-	 * so we clamp them to 0
-	 */
-	if (rec_x_offset < 0)
-		rec_x_offset = 0;
-	if (rec_y_offset < 0)
-		rec_y_offset = 0;
 	/* Save necessary cursor info x, y position. w, h is saved in attribute func. */
-	hubp->cur_rect.x = rec_x_offset + param->recout.x;
-	hubp->cur_rect.y = rec_y_offset + param->recout.y;
+	hubp->cur_rect.x = param->cur_rect_x;
+	hubp->cur_rect.y = param->cur_rect_y;
+}
+
+void hubp401_cursor_refresh_state(struct hubp *hubp)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+	uint32_t enable = 0, x_pos = 0, y_pos = 0, x_hot = 0, y_hot = 0, dst_x_offset = 0;
+	uint32_t addr_high = 0, addr_low = 0, width = 0, height = 0;
+	uint32_t mode = 0, magnify = 0, pitch = 0, lpc = 0;
+	uint32_t dst_y_offset = 0, chunk_hdl_adjust = 0;
+
+	/* Resync the SW cursor cache from real hardware state. */
+
+	/* Position */
+	REG_GET(CURSOR_CONTROL, CURSOR_ENABLE, &enable);
+	REG_GET_2(CURSOR_POSITION,
+		CURSOR_X_POSITION, &x_pos,
+		CURSOR_Y_POSITION, &y_pos);
+	REG_GET_2(CURSOR_HOT_SPOT,
+		CURSOR_HOT_SPOT_X, &x_hot,
+		CURSOR_HOT_SPOT_Y, &y_hot);
+	REG_GET(CURSOR_DST_OFFSET, CURSOR_DST_X_OFFSET, &dst_x_offset);
+
+	hubp->pos.cur_ctl.bits.cur_enable = enable;
+	hubp->pos.position.bits.x_pos = x_pos;
+	hubp->pos.position.bits.y_pos = y_pos;
+	hubp->pos.hot_spot.bits.x_hot = x_hot;
+	hubp->pos.hot_spot.bits.y_hot = y_hot;
+	hubp->pos.dst_offset.bits.dst_x_offset = dst_x_offset;
+
+	/* Attributes */
+	REG_GET(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR_SURFACE_ADDRESS_HIGH, &addr_high);
+	REG_GET(CURSOR_SURFACE_ADDRESS, CURSOR_SURFACE_ADDRESS, &addr_low);
+	REG_GET_2(CURSOR_SIZE,
+		CURSOR_WIDTH, &width,
+		CURSOR_HEIGHT, &height);
+	REG_GET_4(CURSOR_CONTROL,
+		CURSOR_MODE, &mode,
+		CURSOR_2X_MAGNIFY, &magnify,
+		CURSOR_PITCH, &pitch,
+		CURSOR_LINES_PER_CHUNK, &lpc);
+	REG_GET_2(CURSOR_SETTINGS,
+		CURSOR0_DST_Y_OFFSET, &dst_y_offset,
+		CURSOR0_CHUNK_HDL_ADJUST, &chunk_hdl_adjust);
+
+	hubp->att.SURFACE_ADDR_HIGH = addr_high;
+	hubp->att.SURFACE_ADDR = addr_low;
+	hubp->att.size.bits.width = width;
+	hubp->att.size.bits.height = height;
+	hubp->att.cur_ctl.bits.mode = mode;
+	hubp->att.cur_ctl.bits.cur_2x_magnify = magnify;
+	hubp->att.cur_ctl.bits.pitch = pitch;
+	hubp->att.cur_ctl.bits.line_per_chunk = lpc;
+	hubp->att.settings.bits.dst_y_offset = dst_y_offset;
+	hubp->att.settings.bits.chunk_hdl_adjust = chunk_hdl_adjust;
 }
 
 void hubp401_read_state(struct hubp *hubp)
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
index adf34383765284..05078a21a12318 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
@@ -271,7 +271,8 @@ void hubp401_setup_interdependent(
 bool hubp401_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate);
+	bool flip_immediate,
+	bool dcc);
 
 void hubp401_dcc_control(struct hubp *hubp,
 		struct dc_plane_dcc_param *dcc);
@@ -312,6 +313,8 @@ void hubp401_cursor_set_position(
 	const struct dc_cursor_position *pos,
 	const struct dc_cursor_mi_param *param);
 
+void hubp401_cursor_refresh_state(struct hubp *hubp);
+
 void hubp401_read_state(struct hubp *hubp);
 
 bool hubp401_construct(
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
index 86e49eb84a545c..0c8a23cef1e0bd 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
@@ -398,10 +398,13 @@ void hubp42_program_3dlut_fl_config(struct hubp *hubp,
 static bool hubp42_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate)
+	bool flip_immediate,
+	bool dcc)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
+	(void)dcc;
+
 	//program flip type
 	REG_UPDATE(DCSURF_FLIP_CONTROL,
 			SURFACE_FLIP_TYPE, flip_immediate);
@@ -732,7 +735,6 @@ struct hubp_funcs dcn42_hubp_funcs = {
 	.hubp_program_3dlut_fl_crossbar = hubp42_program_3dlut_fl_crossbar,
 	.hubp_get_3dlut_fl_done = hubp401_get_3dlut_fl_done,
 	.hubp_clear_tiling = hubp3_clear_tiling,
-	.hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config,
 	.hubp_read_reg_state = hubp3_read_reg_state
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.c
index 0230f03590cf76..ba86e30b118040 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.c
@@ -22,7 +22,8 @@
 bool hubp50_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate)
+	bool flip_immediate,
+	bool dcc)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
@@ -55,8 +56,10 @@ bool hubp50_program_surface_flip_and_addr(
 		if (address->grph.addr.quad_part == 0)
 			break;
 
-		REG_UPDATE(DCSURF_SURFACE_CONTROL,
-				PRIMARY_SURFACE_TMZ, address->tmz_surface);
+		REG_UPDATE_3(DCSURF_SURFACE_CONTROL,
+				PRIMARY_SURFACE_TMZ, address->tmz_surface,
+				PRIMARY_SURFACE_DCC_EN, dcc,
+				SECONDARY_SURFACE_DCC_EN, dcc);
 
 		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, 0,
 				PRIMARY_SURFACE_ADDRESS_HIGH,
@@ -71,9 +74,11 @@ bool hubp50_program_surface_flip_and_addr(
 				|| address->video_progressive.chroma_addr.quad_part == 0)
 			break;
 
-		REG_UPDATE_2(DCSURF_SURFACE_CONTROL,
+		REG_UPDATE_4(DCSURF_SURFACE_CONTROL,
 				PRIMARY_SURFACE_TMZ, address->tmz_surface,
-				PRIMARY_SURFACE_TMZ_C, address->tmz_surface);
+				PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
+				PRIMARY_SURFACE_DCC_EN, dcc,
+				SECONDARY_SURFACE_DCC_EN, dcc);
 
 		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, 0,
 				PRIMARY_SURFACE_ADDRESS_HIGH_C,
@@ -96,11 +101,13 @@ bool hubp50_program_surface_flip_and_addr(
 			|| address->grph_stereo.right_addr.quad_part == 0)
 			break;
 
-		REG_UPDATE_4(DCSURF_SURFACE_CONTROL,
+		REG_UPDATE_6(DCSURF_SURFACE_CONTROL,
 				PRIMARY_SURFACE_TMZ, address->tmz_surface,
 				PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
 				SECONDARY_SURFACE_TMZ, address->tmz_surface,
-				SECONDARY_SURFACE_TMZ_C, address->tmz_surface);
+				SECONDARY_SURFACE_TMZ_C, address->tmz_surface,
+				PRIMARY_SURFACE_DCC_EN, dcc,
+				SECONDARY_SURFACE_DCC_EN, dcc);
 
 		REG_SET(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C, 0,
 				SECONDARY_SURFACE_ADDRESS_HIGH_C,
@@ -139,9 +146,11 @@ bool hubp50_program_surface_flip_and_addr(
 				|| address->rgbea.alpha_addr.quad_part == 0)
 			break;
 
-		REG_UPDATE_2(DCSURF_SURFACE_CONTROL,
+		REG_UPDATE_4(DCSURF_SURFACE_CONTROL,
 				PRIMARY_SURFACE_TMZ, address->tmz_surface,
-				PRIMARY_SURFACE_TMZ_C, address->tmz_surface);
+				PRIMARY_SURFACE_TMZ_C, address->tmz_surface,
+				PRIMARY_SURFACE_DCC_EN, dcc,
+				SECONDARY_SURFACE_DCC_EN, dcc);
 
 		REG_SET(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, 0,
 				PRIMARY_SURFACE_ADDRESS_HIGH_C,
@@ -457,7 +466,6 @@ void hubp50_program_surface_config(
 	(void)compat_level;
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	hubp401_dcc_control(hubp, dcc);
 	hubp50_program_tiling(hubp2, tiling_info, format);
 	hubp401_program_size(hubp, format, plane_size, dcc);
 	hubp2_program_rotation(hubp, rotation, horizontal_mirror);
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.h
index e7d9bb3623b980..a497a90947b694 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn50/dcn50_hubp.h
@@ -21,7 +21,8 @@
 bool hubp50_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
-	bool flip_immediate);
+	bool flip_immediate,
+	bool dcc);
 
 void hubp50_program_surface_config(
 	struct hubp *hubp,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn60/dcn60_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn60/dcn60_hubp.c
index d22fe26aae6238..4af60e473bd9c2 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn60/dcn60_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn60/dcn60_hubp.c
@@ -25,6 +25,9 @@ static void hubp60_program_deadline(
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
+	/* Make sure DLG test mode is disabled */
+	REG_WRITE(HUBPREQ_DEBUG_DB, 0);
+
 	/* DLG - Per hubp */
 	REG_SET_2(BLANK_OFFSET_0, 0,
 		REFCYC_H_BLANK_END, dlg_attr->refcyc_h_blank_end,
@@ -424,6 +427,7 @@ static struct hubp_funcs dcn60_hubp_funcs = {
 	.mem_program_viewport = hubp401_set_viewport,
 	.set_cursor_attributes	= hubp60_cursor_set_attributes,
 	.set_cursor_position	= hubp401_cursor_set_position,
+	.refresh_cursor_state	= hubp401_cursor_refresh_state,
 	.hubp_clk_cntl = hubp2_clk_cntl,
 	.hubp_vtg_sel = hubp2_vtg_sel,
 	.dmdata_set_attributes = hubp3_dmdata_set_attributes,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/Makefile b/drivers/gpu/drm/amd/display/dc/hwss/Makefile
index 9769ec9a32dfaf..7c2afb773685f4 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/hwss/Makefile
@@ -36,11 +36,13 @@ endif
 
 ###############################################################################
 
+ifdef CONFIG_DRM_AMD_DC_DCE
 HWSS_DCE80 = dce80_hwseq.o
 
 AMD_DAL_HWSS_DCE80 = $(addprefix $(AMDDALPATH)/dc/hwss/dce80/,$(HWSS_DCE80))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCE80)
+endif
 
 ###############################################################################
 
@@ -52,11 +54,13 @@ AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCE)
 
 ###############################################################################
 
+ifdef CONFIG_DRM_AMD_DC_DCE
 HWSS_DCE100 = dce100_hwseq.o
 
 AMD_DAL_HWSS_DCE100 = $(addprefix $(AMDDALPATH)/dc/hwss/dce100/,$(HWSS_DCE100))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCE100)
+endif
 
 ###############################################################################
 
@@ -68,6 +72,7 @@ AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCE110)
 
 ###############################################################################
 
+ifdef CONFIG_DRM_AMD_DC_DCE
 HWSS_DCE112 = dce112_hwseq.o
 
 AMD_DAL_HWSS_DCE112 = $(addprefix $(AMDDALPATH)/dc/hwss/dce112/,$(HWSS_DCE112))
@@ -81,6 +86,7 @@ HWSS_DCE120 = dce120_hwseq.o
 AMD_DAL_HWSS_DCE120 = $(addprefix $(AMDDALPATH)/dc/hwss/dce120/,$(HWSS_DCE120))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_HWSS_DCE120)
+endif
 
 ifdef CONFIG_DRM_AMD_DC_FP
 ###############################################################################
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.c
index 4202fadb2c0e9a..0f44577572fbef 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.c
@@ -44,20 +44,36 @@ void dce_enable_fe_clock(struct dce_hwseq *hws,
 			DCFE_CLOCK_ENABLE, enable);
 }
 
-void dce_pipe_control_lock(struct dc *dc,
+bool dce_build_pipe_control_lock_sequence(struct dc *dc,
 		struct pipe_ctx *pipe,
-		bool lock)
+		bool lock,
+		struct pipe_control_lock_params *params)
 {
-	uint32_t lock_val = lock ? 1 : 0;
+	if (!pipe || pipe->top_pipe)
+		return false;
+
+	params->lock = lock;
+	params->tg_lock.dc = dc;
+	params->tg_lock.tg = pipe->stream_res.tg;
+	params->tg_lock.lock = lock;
+	params->tg_lock.triplebuffer_flips = pipe->plane_state &&
+			pipe->plane_state->triplebuffer_flips;
+	return true;
+}
+
+void dce_tg_lock(struct tg_lock_params *params)
+{
+	uint32_t lock_val = params->lock ? 1 : 0;
 	uint32_t dcp_grph, scl, blnd, update_lock_mode, val;
-	struct dce_hwseq *hws = dc->hwseq;
+	struct dce_hwseq *hws = params->dc->hwseq;
+	unsigned int tg_inst = params->tg->inst;
 
 	/* Not lock pipe when blank */
-	if (lock && pipe->stream_res.tg->funcs->is_blanked &&
-	    pipe->stream_res.tg->funcs->is_blanked(pipe->stream_res.tg))
+	if (params->lock && params->tg->funcs->is_blanked &&
+			params->tg->funcs->is_blanked(params->tg))
 		return;
 
-	val = REG_GET_4(BLND_V_UPDATE_LOCK[pipe->stream_res.tg->inst],
+	val = REG_GET_4(BLND_V_UPDATE_LOCK[tg_inst],
 			BLND_DCP_GRPH_V_UPDATE_LOCK, &dcp_grph,
 			BLND_SCL_V_UPDATE_LOCK, &scl,
 			BLND_BLND_V_UPDATE_LOCK, &blnd,
@@ -68,28 +84,28 @@ void dce_pipe_control_lock(struct dc *dc,
 	blnd = lock_val;
 	update_lock_mode = lock_val;
 
-	REG_SET_2(BLND_V_UPDATE_LOCK[pipe->stream_res.tg->inst], val,
+	REG_SET_2(BLND_V_UPDATE_LOCK[tg_inst], val,
 			BLND_DCP_GRPH_V_UPDATE_LOCK, dcp_grph,
 			BLND_SCL_V_UPDATE_LOCK, scl);
 
 	if (hws->masks->BLND_BLND_V_UPDATE_LOCK != 0)
-		REG_SET_2(BLND_V_UPDATE_LOCK[pipe->stream_res.tg->inst], val,
+		REG_SET_2(BLND_V_UPDATE_LOCK[tg_inst], val,
 				BLND_BLND_V_UPDATE_LOCK, blnd,
 				BLND_V_UPDATE_LOCK_MODE, update_lock_mode);
 
 	if (hws->wa.blnd_crtc_trigger) {
-		if (!lock) {
-			uint32_t value = REG_READ(CRTC_H_BLANK_START_END[pipe->stream_res.tg->inst]);
-			REG_WRITE(CRTC_H_BLANK_START_END[pipe->stream_res.tg->inst], value);
+		if (!params->lock) {
+			uint32_t value = REG_READ(CRTC_H_BLANK_START_END[tg_inst]);
+
+			REG_WRITE(CRTC_H_BLANK_START_END[tg_inst], value);
 		}
 	}
 }
 
 #if defined(CONFIG_DRM_AMD_DC_SI)
-void dce60_pipe_control_lock(struct dc *dc,
-		struct pipe_ctx *pipe,
-		bool lock)
+void dce60_tg_lock(struct tg_lock_params *params)
 {
+	(void)params;
 	/* DCE6 has no BLND_V_UPDATE_LOCK register */
 }
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
index 2cc4e7516a5d76..0b9b4186261106 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
@@ -26,6 +26,7 @@
 #define __DCE_HWSEQ_H__
 
 #include "dc_types.h"
+#include "hw_sequencer_private.h"
 
 #define HWSEQ_DCEF_REG_LIST_DCE8() \
 	.DCFE_CLOCK_CONTROL[0] = mmCRTC0_CRTC_DCFE_CLOCK_CONTROL, \
@@ -1299,17 +1300,16 @@ struct clock_source;
 void dce_enable_fe_clock(struct dce_hwseq *hwss,
 		unsigned int inst, bool enable);
 
-void dce_pipe_control_lock(struct dc *dc,
-		struct pipe_ctx *pipe,
-		bool lock);
+bool dce_build_pipe_control_lock_sequence(struct dc *dc,
+		struct pipe_ctx *pipe, bool lock,
+		struct pipe_control_lock_params *params);
+void dce_tg_lock(struct tg_lock_params *params);
 
 void dce_set_blender_mode(struct dce_hwseq *hws,
 	unsigned int blnd_inst, enum blnd_mode mode);
 
 #if defined(CONFIG_DRM_AMD_DC_SI)
-void dce60_pipe_control_lock(struct dc *dc,
-		struct pipe_ctx *pipe,
-		bool lock);
+void dce60_tg_lock(struct tg_lock_params *params);
 #endif
 
 void dce_clock_gating_power_up(struct dce_hwseq *hws,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index 328321961fba26..a7e53563924512 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -283,11 +283,10 @@ static void dce110_prescale_params(struct ipp_prescale_params *prescale_params,
 }
 
 static bool
-dce110_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
-			       const struct dc_plane_state *plane_state)
+dce110_set_input_transfer_func(struct set_input_transfer_func_params *params)
 {
-	(void)dc;
-	struct input_pixel_processor *ipp = pipe_ctx->plane_res.ipp;
+	struct input_pixel_processor *ipp = params->ipp;
+	struct dc_plane_state *plane_state = params->plane_state;
 	const struct dc_transfer_func *tf = NULL;
 	struct ipp_prescale_params prescale_params = { 0 };
 	bool result = true;
@@ -3140,7 +3139,6 @@ static void dce110_program_front_end_for_pipe(
 	struct xfm_grph_csc_adjustment adjust;
 	struct out_csc_color_matrix tbl_entry;
 	unsigned int i;
-	struct dce_hwseq *hws = dc->hwseq;
 
 	memset(&tbl_entry, 0, sizeof(tbl_entry));
 
@@ -3199,7 +3197,7 @@ static void dce110_program_front_end_for_pipe(
 	if (pipe_ctx->plane_state->update_bits.full_update ||
 			pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
 			pipe_ctx->plane_state->update_bits.gamma_change)
-		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
+		hwss_set_input_transfer_func(dc, pipe_ctx);
 
 	if (pipe_ctx->plane_state->update_bits.full_update)
 		hwss_set_output_transfer_func(dc, pipe_ctx);
@@ -3489,6 +3487,10 @@ void dce110_enable_tmds_link_output(struct dc_link *link,
 		uint32_t pixel_clock)
 {
 	(void)link_res;
+
+	if (!link->link_enc)
+		return;
+
 	link->link_enc->funcs->enable_tmds_output(
 			link->link_enc,
 			clock_source,
@@ -3499,12 +3501,12 @@ void dce110_enable_tmds_link_output(struct dc_link *link,
 
 	// For dongle Type 2 with no I2C support on board, setup sw mode of Ri/Pj check with proper aux instance
 	if (link->force_to_use_aux) {
-		if (link->link_enc && link->link_enc->funcs->setup_ri_pj_check_in_sw_or_hw_mode)
+		if (link->link_enc->funcs->setup_ri_pj_check_in_sw_or_hw_mode)
 			link->link_enc->funcs->setup_ri_pj_check_in_sw_or_hw_mode(link->link_enc,
 				link->aux_hw_inst, true);
 	} else
 		// For HDMI setup hw mode of Ri/Pj check with proper ddc pin instance
-		if (link->link_enc && link->link_enc->funcs->setup_ri_pj_check_in_sw_or_hw_mode)
+		if (link->link_enc->funcs->setup_ri_pj_check_in_sw_or_hw_mode)
 			link->link_enc->funcs->setup_ri_pj_check_in_sw_or_hw_mode(link->link_enc,
 				link->ddc_hw_inst, false);
 }
@@ -3605,8 +3607,11 @@ void dce110_disable_link_output(struct dc_link *link,
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->lock_phy(dmcu);
 
-	link_hwss->disable_link_output(link, link_res, signal);
-	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+	if (!(signal == SIGNAL_TYPE_EDP &&
+			link->forced_psr_active)) {
+		link_hwss->disable_link_output(link, link_res, signal);
+		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+	}
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
 	 * from enable/disable link output and only call edp panel control
@@ -3617,6 +3622,7 @@ void dce110_disable_link_output(struct dc_link *link,
 	dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
 }
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 static const struct hw_sequencer_funcs dce110_funcs = {
 	.program_gamut_remap = program_gamut_remap,
 	.program_output_csc = program_output_csc,
@@ -3638,9 +3644,10 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dce110_power_down_fe,
-	.pipe_control_lock = dce_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dce_build_pipe_control_lock_sequence,
+	.tg_lock = dce_tg_lock,
 	.interdependent_update_lock = NULL,
-	.cursor_lock = dce_pipe_control_lock,
+	.cursor_lock = hwss_pipe_control_lock,
 	.prepare_bandwidth = dce110_prepare_bandwidth,
 	.optimize_bandwidth = dce110_optimize_bandwidth,
 	.set_drr = set_drr,
@@ -3652,7 +3659,7 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.edp_backlight_control = dce110_edp_backlight_control,
 	.edp_power_control = dce110_edp_power_control,
 	.edp_wait_for_hpd_ready = dce110_edp_wait_for_hpd_ready,
-	.set_cursor_position = dce110_set_cursor_position,
+	.set_cursor_position_legacy = dce110_set_cursor_position,
 	.set_cursor_attribute = dce110_set_cursor_attribute,
 	.set_backlight_level = dce110_set_backlight_level,
 	.set_abm_immediate_disable = dce110_set_abm_immediate_disable,
@@ -3686,3 +3693,5 @@ void dce110_hw_sequencer_construct(struct dc *dc)
 	dc->hwseq->funcs = dce110_private_funcs;
 }
 
+#endif /* CONFIG_DRM_AMD_DC_DCE */
+
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c
index 221996b348abf0..b73a10b0acbd7b 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce60/dce60_hwseq.c
@@ -275,7 +275,6 @@ dce60_program_front_end_for_pipe(
 	struct xfm_grph_csc_adjustment adjust;
 	struct out_csc_color_matrix tbl_entry;
 	unsigned int i;
-	struct dce_hwseq *hws = dc->hwseq;
 
 	DC_LOGGER_INIT();
 	memset(&tbl_entry, 0, sizeof(tbl_entry));
@@ -335,7 +334,7 @@ dce60_program_front_end_for_pipe(
 	if (pipe_ctx->plane_state->update_bits.full_update ||
 			pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
 			pipe_ctx->plane_state->update_bits.gamma_change)
-		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
+		hwss_set_input_transfer_func(dc, pipe_ctx);
 
 	if (pipe_ctx->plane_state->update_bits.full_update)
 		hwss_set_output_transfer_func(dc, pipe_ctx);
@@ -424,8 +423,9 @@ void dce60_hw_sequencer_construct(struct dc *dc)
 
 	dc->hwseq->funcs.enable_display_power_gating = dce100_enable_display_power_gating;
 	dc->hwss.apply_ctx_for_surface = dce60_apply_ctx_for_surface;
-	dc->hwss.cursor_lock = dce60_pipe_control_lock;
-	dc->hwss.pipe_control_lock = dce60_pipe_control_lock;
+	dc->hwss.cursor_lock = hwss_pipe_control_lock;
+	dc->hwss.build_pipe_control_lock_sequence = dce_build_pipe_control_lock_sequence;
+	dc->hwss.tg_lock = dce60_tg_lock;
 	dc->hwss.prepare_bandwidth = dce100_prepare_bandwidth;
 	dc->hwss.optimize_bandwidth = dce100_optimize_bandwidth;
 	dc->hwss.clear_surface_dcc_and_tiling = dce100_reset_surface_dcc_and_tiling;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce80/dce80_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce80/dce80_hwseq.c
index 76fd45550c5e7b..f44f41b13007d2 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce80/dce80_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce80/dce80_hwseq.c
@@ -47,7 +47,8 @@ void dce80_hw_sequencer_construct(struct dc *dc)
 	dce110_hw_sequencer_construct(dc);
 
 	dc->hwseq->funcs.enable_display_power_gating = dce100_enable_display_power_gating;
-	dc->hwss.pipe_control_lock = dce_pipe_control_lock;
+	dc->hwss.build_pipe_control_lock_sequence = dce_build_pipe_control_lock_sequence;
+	dc->hwss.tg_lock = dce_tg_lock;
 	dc->hwss.prepare_bandwidth = dce100_prepare_bandwidth;
 	dc->hwss.optimize_bandwidth = dce100_optimize_bandwidth;
 	dc->hwss.clear_surface_dcc_and_tiling = dce100_reset_surface_dcc_and_tiling;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
index 097e4ac2df0583..884319b22cc128 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
@@ -41,6 +41,7 @@
 #include "opp.h"
 #include "ipp.h"
 #include "mpc.h"
+#include "rmcm.h"
 #include "reg_helper.h"
 #include "dcn10/dcn10_hubp.h"
 #include "dcn10/dcn10_hubbub.h"
@@ -246,9 +247,9 @@ void dcn10_lock_all_pipes(struct dc *dc,
 			continue;
 
 		if (lock)
-			dc->hwss.pipe_control_lock(dc, pipe_ctx, true);
+			hwss_pipe_control_lock(dc, pipe_ctx, true);
 		else
-			dc->hwss.pipe_control_lock(dc, pipe_ctx, false);
+			hwss_pipe_control_lock(dc, pipe_ctx, false);
 	}
 }
 
@@ -565,31 +566,32 @@ static void dcn10_log_color_state(struct dc *dc,
 		"OUT_OFFSET", "OUT_SCALE", "FL_DONE", "SOFT_UNDERFLOW", "HARD_UNDERFLOW",
 		"MEM_PWR_ST", "FORCE", "DIS", "MODE"};
 
-	for (i = 0; i < pool->mpcc_count; i++) {
-		struct mpcc_state s = {0};
+	/* Read RMCM state separately - RMCM is a shared resource (only 2 instances) */
+	for (i = 0; i < MAX_RMCM_INST && i < pool->mpcc_count; i++) {
+		struct rmcm_state r = {0};
 
-		pool->mpc->funcs->read_mpcc_state(pool->mpc, i, &s);
-		if (s.opp_id != 0xf) {
+		if (pool->rmcm[i] && pool->rmcm[i]->funcs->read_rmcm_state) {
+			pool->rmcm[i]->funcs->read_rmcm_state(pool->rmcm[i], i, &r);
 			uint32_t values[] = {
 				i,
-				s.rmcm_regs.rmcm_3dlut_size,
-				s.rmcm_regs.rmcm_3dlut_mode,
-				s.rmcm_regs.rmcm_3dlut_mode_cur,
-				s.rmcm_regs.rmcm_3dlut_read_sel,
-				s.rmcm_regs.rmcm_3dlut_30bit_en,
-				s.rmcm_regs.rmcm_3dlut_wr_en_mask,
-				s.rmcm_regs.rmcm_3dlut_ram_sel,
-				s.rmcm_regs.rmcm_3dlut_out_norm_factor,
-				s.rmcm_regs.rmcm_3dlut_fl_sel,
-				s.rmcm_regs.rmcm_3dlut_out_offset_r,
-				s.rmcm_regs.rmcm_3dlut_out_scale_r,
-				s.rmcm_regs.rmcm_3dlut_fl_done,
-				s.rmcm_regs.rmcm_3dlut_fl_soft_underflow,
-				s.rmcm_regs.rmcm_3dlut_fl_hard_underflow,
-				s.rmcm_regs.rmcm_3dlut_mem_pwr_state,
-				s.rmcm_regs.rmcm_3dlut_mem_pwr_force,
-				s.rmcm_regs.rmcm_3dlut_mem_pwr_dis,
-				s.rmcm_regs.rmcm_3dlut_mem_pwr_mode};
+				r.regs.rmcm_3dlut_size,
+				r.regs.rmcm_3dlut_mode,
+				r.regs.rmcm_3dlut_mode_cur,
+				r.regs.rmcm_3dlut_read_sel,
+				r.regs.rmcm_3dlut_30bit_en,
+				r.regs.rmcm_3dlut_wr_en_mask,
+				r.regs.rmcm_3dlut_ram_sel,
+				r.regs.rmcm_3dlut_out_norm_factor,
+				r.regs.rmcm_3dlut_fl_sel,
+				r.regs.rmcm_3dlut_out_offset_r,
+				r.regs.rmcm_3dlut_out_scale_r,
+				r.regs.rmcm_3dlut_fl_done,
+				r.regs.rmcm_3dlut_fl_soft_underflow,
+				r.regs.rmcm_3dlut_fl_hard_underflow,
+				r.regs.rmcm_3dlut_mem_pwr_state,
+				r.regs.rmcm_3dlut_mem_pwr_force,
+				r.regs.rmcm_3dlut_mem_pwr_dis,
+				r.regs.rmcm_3dlut_mem_pwr_mode};
 
 			int num_elements = 19;
 
@@ -600,17 +602,18 @@ static void dcn10_log_color_state(struct dc *dc,
 	DTN_INFO("\n");
 	DTN_INFO("===== MPC RMCM Shaper =====\n");
 	DTN_INFO("MPCC:  CNTL  LUT_MODE  MODE_CUR  WR_EN_MASK  WR_SEL  OFFSET  SCALE  START_B	START_SEG_B	END_B	END_BASE_B	MEM_PWR_ST	FORCE	DIS	MODE\n");
-	for (i = 0; i < pool->mpcc_count; i++) {
-		struct mpcc_state s = {0};
+	for (i = 0; i < MAX_RMCM_INST && i < pool->mpcc_count; i++) {
+		struct rmcm_state r = {0};
 
-		pool->mpc->funcs->read_mpcc_state(pool->mpc, i, &s);
-		if (s.opp_id != 0xf)
+		if (pool->rmcm[i] && pool->rmcm[i]->funcs->read_rmcm_state) {
+			pool->rmcm[i]->funcs->read_rmcm_state(pool->rmcm[i], i, &r);
 			DTN_INFO("[%2d]:  %4xh  %4xh  %6xh  %4x  %4x  %4x  %4x  %4x %4xh  %4xh  %6xh  %4x  %4x  %4x  %4x\n",
-				i, s.rmcm_regs.rmcm_cntl, s.rmcm_regs.rmcm_shaper_lut_mode, s.rmcm_regs.rmcm_shaper_mode_cur,
-				s.rmcm_regs.rmcm_shaper_lut_write_en_mask, s.rmcm_regs.rmcm_shaper_lut_write_sel, s.rmcm_regs.rmcm_shaper_offset_b,
-				s.rmcm_regs.rmcm_shaper_scale_b, s.rmcm_regs.rmcm_shaper_rama_exp_region_start_b, s.rmcm_regs.rmcm_shaper_rama_exp_region_start_seg_b,
-				s.rmcm_regs.rmcm_shaper_rama_exp_region_end_b, s.rmcm_regs.rmcm_shaper_rama_exp_region_end_base_b, s.rmcm_regs.rmcm_shaper_mem_pwr_state,
-				s.rmcm_regs.rmcm_shaper_mem_pwr_force, s.rmcm_regs.rmcm_shaper_mem_pwr_dis, s.rmcm_regs.rmcm_shaper_mem_pwr_mode);
+				i, r.regs.rmcm_cntl, r.regs.rmcm_shaper_lut_mode, r.regs.rmcm_shaper_mode_cur,
+				r.regs.rmcm_shaper_lut_write_en_mask, r.regs.rmcm_shaper_lut_write_sel, r.regs.rmcm_shaper_offset_b,
+				r.regs.rmcm_shaper_scale_b, r.regs.rmcm_shaper_rama_exp_region_start_b, r.regs.rmcm_shaper_rama_exp_region_start_seg_b,
+				r.regs.rmcm_shaper_rama_exp_region_end_b, r.regs.rmcm_shaper_rama_exp_region_end_base_b, r.regs.rmcm_shaper_mem_pwr_state,
+				r.regs.rmcm_shaper_mem_pwr_force, r.regs.rmcm_shaper_mem_pwr_dis, r.regs.rmcm_shaper_mem_pwr_mode);
+		}
 	}
 }
 
@@ -1346,7 +1349,12 @@ static void dcn10_reset_back_end_for_pipe(
 	 * screen only, the dpms_off would be true but
 	 * VBIOS lit up eDP, so check link status too.
 	 */
-	if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
+	if (link->connector_signal == SIGNAL_TYPE_EDP &&
+	    link->forced_psr_active) {
+		/* forced psr is active for seamless switch; skip dpms-off. */
+		if (pipe_ctx->stream_res.audio)
+			dc->hwss.disable_audio_stream(pipe_ctx);
+	} else if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
 		dc->link_srv->set_dpms_off(pipe_ctx);
 	else if (pipe_ctx->stream_res.audio)
 		dc->hwss.disable_audio_stream(pipe_ctx);
@@ -1370,13 +1378,15 @@ static void dcn10_reset_back_end_for_pipe(
 	 * parent pipe.
 	 */
 	if (pipe_ctx->top_pipe == NULL) {
+		if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
+			link->forced_psr_active)) {
+			if (pipe_ctx->stream_res.abm)
+				dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-		if (pipe_ctx->stream_res.abm)
-			dc->hwss.set_abm_immediate_disable(pipe_ctx);
-
-		pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
+			pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
 
-		pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+			pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+		}
 		set_drr_and_clear_adjust_pending(pipe_ctx, pipe_ctx->stream, NULL);
 		if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal))
 			pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0;
@@ -1740,7 +1750,9 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 
 		if (tg->funcs->is_tg_enabled(tg)) {
 			if (tg->funcs->init_odm)
-				tg->funcs->init_odm(tg);
+				if (!tg->funcs->is_odm_enabled ||
+					!tg->funcs->is_odm_enabled(tg))
+						tg->funcs->init_odm(tg);
 		}
 
 		tg->funcs->tg_init(tg);
@@ -2050,7 +2062,8 @@ static bool patch_address_for_sbs_tb_stereo(
 	return false;
 }
 
-void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
+void dcn10_prepare_plane_addr_update(const struct dc *dc, struct pipe_ctx *pipe_ctx,
+		struct dc_plane_address *addr_to_program, bool *flip_immediate)
 {
 	(void)dc;
 	bool addr_patched = false;
@@ -2062,10 +2075,8 @@ void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 
 	addr_patched = patch_address_for_sbs_tb_stereo(pipe_ctx, &addr);
 
-	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
-			pipe_ctx->plane_res.hubp,
-			&plane_state->address,
-			plane_state->flip_immediate);
+	*addr_to_program = plane_state->address;
+	*flip_immediate = plane_state->flip_immediate;
 
 	plane_state->status.requested_address = plane_state->address;
 
@@ -2076,39 +2087,55 @@ void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 		pipe_ctx->plane_state->address.grph_stereo.left_addr = addr;
 }
 
-bool dcn10_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
-			const struct dc_plane_state *plane_state)
+void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
-	(void)dc;
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+	struct dc_plane_address address;
+	bool flip_immediate;
+
+	if (pipe_ctx->plane_state == NULL)
+		return;
+
+	dcn10_prepare_plane_addr_update(dc, pipe_ctx, &address, &flip_immediate);
+
+	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
+			pipe_ctx->plane_res.hubp,
+			&address,
+			flip_immediate,
+			pipe_ctx->plane_state->dcc.enable);
+}
+
+bool dcn10_set_input_transfer_func(struct set_input_transfer_func_params *params)
+{
+	struct dpp *dpp = params->dpp;
+	struct dc_plane_state *plane_state = params->plane_state;
 	const struct dc_transfer_func *tf = NULL;
 	bool result = true;
 
-	if (dpp_base == NULL)
+	if (dpp == NULL)
 		return false;
 
 	tf = &plane_state->in_transfer_func;
 
-	if (!dpp_base->ctx->dc->debug.always_use_regamma
+	if (!dpp->ctx->dc->debug.always_use_regamma
 		&& !plane_state->gamma_correction.is_identity
 			&& dce_use_lut(plane_state->format))
-		dpp_base->funcs->dpp_program_input_lut(dpp_base, &plane_state->gamma_correction);
+		dpp->funcs->dpp_program_input_lut(dpp, &plane_state->gamma_correction);
 
 	if (tf->type == TF_TYPE_PREDEFINED) {
 		switch (tf->tf) {
 		case TRANSFER_FUNCTION_SRGB:
-			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_HW_sRGB);
+			dpp->funcs->dpp_set_degamma(dpp, IPP_DEGAMMA_MODE_HW_sRGB);
 			break;
 		case TRANSFER_FUNCTION_BT709:
-			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_HW_xvYCC);
+			dpp->funcs->dpp_set_degamma(dpp, IPP_DEGAMMA_MODE_HW_xvYCC);
 			break;
 		case TRANSFER_FUNCTION_LINEAR:
-			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
+			dpp->funcs->dpp_set_degamma(dpp, IPP_DEGAMMA_MODE_BYPASS);
 			break;
 		case TRANSFER_FUNCTION_PQ:
-			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_USER_PWL);
-			cm_helper_translate_curve_to_degamma_hw_format(tf, &dpp_base->degamma_params);
-			dpp_base->funcs->dpp_program_degamma_pwl(dpp_base, &dpp_base->degamma_params);
+			dpp->funcs->dpp_set_degamma(dpp, IPP_DEGAMMA_MODE_USER_PWL);
+			cm_helper_translate_curve_to_degamma_hw_format(tf, &dpp->degamma_params);
+			dpp->funcs->dpp_program_degamma_pwl(dpp, &dpp->degamma_params);
 			result = true;
 			break;
 		default:
@@ -2116,12 +2143,12 @@ bool dcn10_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
 			break;
 		}
 	} else if (tf->type == TF_TYPE_BYPASS) {
-		dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
+		dpp->funcs->dpp_set_degamma(dpp, IPP_DEGAMMA_MODE_BYPASS);
 	} else {
 		cm_helper_translate_curve_to_degamma_hw_format(tf,
-					&dpp_base->degamma_params);
-		dpp_base->funcs->dpp_program_degamma_pwl(dpp_base,
-				&dpp_base->degamma_params);
+					&dpp->degamma_params);
+		dpp->funcs->dpp_program_degamma_pwl(dpp,
+				&dpp->degamma_params);
 		result = true;
 	}
 
@@ -2196,29 +2223,20 @@ bool dcn10_set_output_transfer_func(struct set_output_transfer_func_params *para
 	return true;
 }
 
-void dcn10_pipe_control_lock(
-	struct dc *dc,
-	struct pipe_ctx *pipe,
-	bool lock)
+void dcn10_tg_lock(struct tg_lock_params *params)
 {
-	struct dce_hwseq *hws = dc->hwseq;
+	struct dce_hwseq *hws = params->dc->hwseq;
 
-	/* use TG master update lock to lock everything on the TG
-	 * therefore only top pipe need to lock
-	 */
-	if (!pipe || pipe->top_pipe)
-		return;
+	if (params->dc->debug.sanity_checks)
+		hws->funcs.verify_allow_pstate_change_high(params->dc);
 
-	if (dc->debug.sanity_checks)
-		hws->funcs.verify_allow_pstate_change_high(dc);
-
-	if (lock)
-		pipe->stream_res.tg->funcs->lock(pipe->stream_res.tg);
+	if (params->lock)
+		params->tg->funcs->lock(params->tg);
 	else
-		pipe->stream_res.tg->funcs->unlock(pipe->stream_res.tg);
+		params->tg->funcs->unlock(params->tg);
 
-	if (dc->debug.sanity_checks)
-		hws->funcs.verify_allow_pstate_change_high(dc);
+	if (params->dc->debug.sanity_checks)
+		hws->funcs.verify_allow_pstate_change_high(params->dc);
 }
 
 /**
@@ -3148,11 +3166,17 @@ static void dcn10_update_dchubp_dpp(
 	}
 
 	if (pipe_ctx->stream->cursor_attributes.address.quad_part != 0) {
-		if (dc->hwss.abort_cursor_offload_update)
-			dc->hwss.abort_cursor_offload_update(dc, pipe_ctx);
+		if (dc_dmub_srv_is_cursor_offload_enabled(dc) && dc->hwss.abort_cursor_offload_update) {
+			struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+			if (top_pipe)
+				dc->hwss.abort_cursor_offload_update(dc->ctx->dmub_srv->dmub,
+					pipe_ctx->plane_res.dpp, pipe_ctx->plane_res.hubp,
+					top_pipe->pipe_idx);
+		}
 
 		dc->hwss.set_cursor_attribute(pipe_ctx);
-		dc->hwss.set_cursor_position(pipe_ctx);
+		hwss_program_cursor_position(dc, pipe_ctx);
 
 		if (dc->hwss.set_cursor_sdr_white_level)
 			dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
@@ -3296,7 +3320,7 @@ void dcn10_program_pipe(
 	if (pipe_ctx->plane_state->update_bits.full_update ||
 			pipe_ctx->plane_state->update_bits.in_transfer_func_change ||
 			pipe_ctx->plane_state->update_bits.gamma_change)
-		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
+		hwss_set_input_transfer_func(dc, pipe_ctx);
 
 	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
 	 * only do gamma programming for full update.
@@ -3705,11 +3729,11 @@ void dcn10_update_dchub(struct dce_hwseq *hws, struct dchub_init_data *dh_data)
 	hubbub->funcs->update_dchub(hubbub, dh_data);
 }
 
-void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
+void dcn10_build_cursor_pos_update_params(const struct pipe_ctx *pipe_ctx,
+		struct dc_cursor_position *pos_out,
+		struct dc_cursor_mi_param *param_out)
 {
 	struct dc_cursor_position pos_cpy = pipe_ctx->stream->cursor_position;
-	struct hubp *hubp = pipe_ctx->plane_res.hubp;
-	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 	struct dc_cursor_mi_param param = {
 		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10,
 		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz,
@@ -3925,8 +3949,16 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->plane_res.scl_data.viewport.height - pos_cpy.y;
 	}
 
-	hubp->funcs->set_cursor_position(hubp, &pos_cpy, &param);
-	dpp->funcs->set_cursor_position(dpp, &pos_cpy, &param, hubp->curs_attr.width, hubp->curs_attr.height);
+	*pos_out = pos_cpy;
+	*param_out = param;
+}
+
+void dcn10_set_cursor_position(struct hubp *hubp, struct dpp *dpp,
+		const struct dc_cursor_position *pos,
+		const struct dc_cursor_mi_param *param)
+{
+	hubp->funcs->set_cursor_position(hubp, pos, param);
+	dpp->funcs->set_cursor_position(dpp, pos, param, hubp->curs_attr.width, hubp->curs_attr.height);
 }
 
 void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
@@ -3941,29 +3973,13 @@ void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
 
 void dcn10_set_cursor_sdr_white_level(struct pipe_ctx *pipe_ctx)
 {
-	uint32_t sdr_white_level = pipe_ctx->stream->cursor_attributes.sdr_white_level;
-	struct fixed31_32 multiplier;
-	struct dpp_cursor_attributes opt_attr = { 0 };
-	uint32_t hw_scale = 0x3c00; // 1.0 default multiplier
-	struct custom_float_format fmt;
-
-	if (!pipe_ctx->plane_res.dpp->funcs->set_optional_cursor_attributes)
-		return;
-
-	fmt.exponenta_bits = 5;
-	fmt.mantissa_bits = 10;
-	fmt.sign = true;
+	struct dpp *dpp = pipe_ctx->plane_res.dpp;
+	struct dpp_cursor_attributes attr;
 
-	if (sdr_white_level > 80) {
-		multiplier = dc_fixpt_from_fraction(sdr_white_level, 80);
-		convert_to_custom_float_format(multiplier, &fmt, &hw_scale);
+	if (dpp && dpp->funcs->set_optional_cursor_attributes) {
+		attr = calc_sdr_cursor_attributes(pipe_ctx);
+		dpp->funcs->set_optional_cursor_attributes(dpp, &attr);
 	}
-
-	opt_attr.scale = hw_scale;
-	opt_attr.bias = 0;
-
-	pipe_ctx->plane_res.dpp->funcs->set_optional_cursor_attributes(
-			pipe_ctx->plane_res.dpp, &opt_attr);
 }
 
 /*
@@ -4026,45 +4042,12 @@ void dcn10_calc_vupdate_position(
 	*end_line = (*start_line + 2) % timing->v_total;
 }
 
-static void dcn10_cal_vline_position(
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx,
-		uint32_t *start_line,
-		uint32_t *end_line)
-{
-	const struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
-	int vline_pos = pipe_ctx->stream->periodic_interrupt.lines_offset;
-
-	if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_UPDATE) {
-		if (vline_pos > 0)
-			vline_pos--;
-		else if (vline_pos < 0)
-			vline_pos++;
-
-		vline_pos += dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx);
-		if (vline_pos >= 0)
-			*start_line = vline_pos - ((vline_pos / timing->v_total) * timing->v_total);
-		else
-			*start_line = vline_pos + ((-vline_pos / timing->v_total) + 1) * timing->v_total - 1;
-		*end_line = (*start_line + 2) % timing->v_total;
-	} else if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_SYNC) {
-		// vsync is line 0 so start_line is just the requested line offset
-		*start_line = vline_pos;
-		*end_line = (*start_line + 2) % timing->v_total;
-	} else
-		ASSERT(0);
-}
-
 void dcn10_setup_periodic_interrupt(
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx)
+		struct timing_generator *tg,
+		uint32_t start_line,
+		uint32_t end_line
+	)
 {
-	struct timing_generator *tg = pipe_ctx->stream_res.tg;
-	uint32_t start_line = 0;
-	uint32_t end_line = 0;
-
-	dcn10_cal_vline_position(dc, pipe_ctx, &start_line, &end_line);
-
 	tg->funcs->setup_vertical_interrupt0(tg, start_line, end_line);
 }
 
@@ -4210,5 +4193,6 @@ void dcn10_reset_surface_dcc_and_tiling(struct pipe_ctx *pipe_ctx,
 	/* force page flip to see the new content of the framebuffer */
 	hubp->funcs->hubp_program_surface_flip_and_addr(hubp,
 							&plane_state->address,
-							true);
+							true,
+							plane_state->dcc.enable);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.h
index 2cb674ba54e164..3839985315030c 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.h
@@ -57,10 +57,7 @@ void dcn10_wait_for_pipe_update_if_needed(
 void dcn10_set_wait_for_update_needed_for_pipe(
 	struct dc *dc,
 	struct pipe_ctx *pipe_ctx);
-void dcn10_pipe_control_lock(
-	struct dc *dc,
-	struct pipe_ctx *pipe,
-	bool lock);
+void dcn10_tg_lock(struct tg_lock_params *params);
 void dcn10_cursor_lock(struct dc *dc, struct pipe_ctx *pipe, bool lock);
 void dcn10_blank_pixel_data(
 		struct dc *dc,
@@ -74,9 +71,10 @@ void dcn10_program_output_csc(struct dc *dc,
 		uint16_t *matrix,
 		int opp_id);
 bool dcn10_set_output_transfer_func(struct set_output_transfer_func_params *params);
-bool dcn10_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
-			const struct dc_plane_state *plane_state);
+bool dcn10_set_input_transfer_func(struct set_input_transfer_func_params *params);
 void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx);
+void dcn10_prepare_plane_addr_update(const struct dc *dc, struct pipe_ctx *pipe_ctx,
+		struct dc_plane_address *addr_to_program, bool *flip_immediate);
 void dcn10_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx);
 void dcn10_reset_hw_ctx_wrap(
 		struct dc *dc,
@@ -179,12 +177,18 @@ void dce110_edp_power_control(
 void dce110_edp_wait_for_hpd_ready(
 		struct dc_link *link,
 		bool power_up);
-void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx);
+void dcn10_build_cursor_pos_update_params(const struct pipe_ctx *pipe_ctx,
+		struct dc_cursor_position *pos,
+		struct dc_cursor_mi_param *param);
+void dcn10_set_cursor_position(struct hubp *hubp, struct dpp *dpp,
+		const struct dc_cursor_position *pos,
+		const struct dc_cursor_mi_param *param);
 void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx);
 void dcn10_set_cursor_sdr_white_level(struct pipe_ctx *pipe_ctx);
 void dcn10_setup_periodic_interrupt(
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx);
+		struct timing_generator *tg,
+		uint32_t start_line,
+		uint32_t end_line);
 enum dc_status dcn10_set_clock(struct dc *dc,
 		enum dc_clock_type clock_type,
 		uint32_t clk_khz,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c
index b5e82e19012474..bc3ac96de688a9 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c
@@ -27,6 +27,7 @@
 #include "dce110/dce110_hwseq.h"
 #include "dcn10/dcn10_hwseq.h"
 #include "dcn20/dcn20_hwseq.h"
+#include "dc/hwss/dce/dce_hwseq.h"
 
 static const struct hw_sequencer_funcs dcn10_funcs = {
 	.program_gamut_remap = dcn10_program_gamut_remap,
@@ -38,6 +39,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.post_unlock_program_front_end = dcn10_post_unlock_program_front_end,
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.update_plane_addr = dcn10_update_plane_addr,
+	.prepare_plane_addr_update = dcn10_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.clear_surface_dcc_and_tiling = dcn10_reset_surface_dcc_and_tiling,
@@ -54,7 +56,8 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn10_disable_plane,
-	.pipe_control_lock = dcn10_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dce_build_pipe_control_lock_sequence,
+	.tg_lock = dcn10_tg_lock,
 	.cursor_lock = dcn10_cursor_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.prepare_bandwidth = dcn10_prepare_bandwidth,
@@ -91,6 +94,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn10_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.program_pipe = dcn10_program_pipe,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
index 16bb6f678567ce..45a4a770750745 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
@@ -217,15 +217,18 @@ static int find_free_gsl_group(const struct dc *dc)
  * gsl_0 <=> pipe_ctx->stream_res.gsl_group == 1
  * Using a magic value like -1 would require tracking all inits/resets
  */
-void dcn20_setup_gsl_group_as_lock(
+void dcn20_build_gsl_group_as_lock(
 		const struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
-		bool enable)
+		bool enable,
+		struct tg_set_gsl_params *gsl_params,
+		struct tg_set_gsl_source_select_params *gsl_source_select_params)
 {
-	struct gsl_params gsl;
 	int group_idx;
 
-	memset(&gsl, 0, sizeof(struct gsl_params));
+	memset(&gsl_params->gsl, 0, sizeof(gsl_params->gsl));
+	gsl_params->tg = pipe_ctx->stream_res.tg;
+	gsl_source_select_params->tg = pipe_ctx->stream_res.tg;
 
 	if (enable) {
 		/* return if group already assigned since GSL was set up
@@ -241,22 +244,22 @@ void dcn20_setup_gsl_group_as_lock(
 		/* set gsl group reg field and mark resource used */
 		switch (group_idx) {
 		case 1:
-			gsl.gsl0_en = 1;
+			gsl_params->gsl.gsl0_en = 1;
 			dc->res_pool->gsl_groups.gsl_0 = 1;
 			break;
 		case 2:
-			gsl.gsl1_en = 1;
+			gsl_params->gsl.gsl1_en = 1;
 			dc->res_pool->gsl_groups.gsl_1 = 1;
 			break;
 		case 3:
-			gsl.gsl2_en = 1;
+			gsl_params->gsl.gsl2_en = 1;
 			dc->res_pool->gsl_groups.gsl_2 = 1;
 			break;
 		default:
 			BREAK_TO_DEBUGGER();
 			return; // invalid case
 		}
-		gsl.gsl_master_en = 1;
+		gsl_params->gsl.gsl_master_en = 1;
 	} else {
 		group_idx = pipe_ctx->stream_res.gsl_group;
 		if (group_idx == 0)
@@ -267,32 +270,46 @@ void dcn20_setup_gsl_group_as_lock(
 		/* unset gsl group reg field and mark resource free */
 		switch (group_idx) {
 		case 1:
-			gsl.gsl0_en = 0;
 			dc->res_pool->gsl_groups.gsl_0 = 0;
 			break;
 		case 2:
-			gsl.gsl1_en = 0;
 			dc->res_pool->gsl_groups.gsl_1 = 0;
 			break;
 		case 3:
-			gsl.gsl2_en = 0;
 			dc->res_pool->gsl_groups.gsl_2 = 0;
 			break;
 		default:
 			BREAK_TO_DEBUGGER();
 			return;
 		}
-		gsl.gsl_master_en = 0;
 	}
 
+	gsl_source_select_params->group_idx = group_idx;
+	gsl_source_select_params->gsl_ready_signal = enable ? 4 : 0;
+}
+
+void dcn20_setup_gsl_group_as_lock(
+		const struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		bool enable)
+{
+	struct tg_set_gsl_params gsl_params = { 0 };
+	struct tg_set_gsl_source_select_params gsl_source_select_params = { 0 };
+
+	dcn20_build_gsl_group_as_lock(dc, pipe_ctx, enable,
+			&gsl_params, &gsl_source_select_params);
+	if (!gsl_source_select_params.group_idx)
+		return;
+
 	/* at this point we want to program whether it's to enable or disable */
 	if (pipe_ctx->stream_res.tg->funcs->set_gsl != NULL) {
 		pipe_ctx->stream_res.tg->funcs->set_gsl(
 			pipe_ctx->stream_res.tg,
-			&gsl);
+			&gsl_params.gsl);
 		if (pipe_ctx->stream_res.tg->funcs->set_gsl_source_select != NULL)
 			pipe_ctx->stream_res.tg->funcs->set_gsl_source_select(
-				pipe_ctx->stream_res.tg, group_idx, enable ? 4 : 0);
+				pipe_ctx->stream_res.tg, gsl_source_select_params.group_idx,
+				gsl_source_select_params.gsl_ready_signal);
 	} else
 		BREAK_TO_DEBUGGER();
 }
@@ -377,14 +394,12 @@ void dcn20_disable_vga(
 }
 
 void dcn20_program_triple_buffer(
-	const struct dc *dc,
-	struct pipe_ctx *pipe_ctx,
+	struct hubp *hubp,
 	bool enable_triple_buffer)
 {
-	(void)dc;
-	if (pipe_ctx->plane_res.hubp && pipe_ctx->plane_res.hubp->funcs) {
-		pipe_ctx->plane_res.hubp->funcs->hubp_enable_tripleBuffer(
-			pipe_ctx->plane_res.hubp,
+	if (hubp && hubp->funcs) {
+		hubp->funcs->hubp_enable_tripleBuffer(
+			hubp,
 			enable_triple_buffer);
 	}
 }
@@ -702,6 +717,10 @@ void dcn20_plane_atomic_disable(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
 	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 
+	/* Clearing 3DLUT fast load writes to HUBP, so it must precede hubp_clk_cntl() below */
+	if (hws->funcs.disable_rmcm_luts)
+		hws->funcs.disable_rmcm_luts(dc, pipe_ctx->plane_res.rmcm, hubp, pipe_ctx->plane_res.mpcc_inst);
+
 	dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe_ctx);
 
 	/* In flip immediate with pipe splitting case GSL is used for
@@ -1065,9 +1084,8 @@ bool dcn20_set_output_transfer_func(struct set_output_transfer_func_params *otf_
 }
 
 bool dcn20_set_blend_lut(
-	struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state)
+	struct dpp *dpp, struct dc_plane_state *plane_state)
 {
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	bool result = true;
 	const struct pwl_params *blend_lut = NULL;
 
@@ -1076,18 +1094,17 @@ bool dcn20_set_blend_lut(
 	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		cm_helper_translate_curve_to_hw_format(plane_state->ctx,
 				&plane_state->cm.blend_func,
-				&dpp_base->regamma_params, false);
-		blend_lut = &dpp_base->regamma_params;
+				&dpp->regamma_params, false);
+		blend_lut = &dpp->regamma_params;
 	}
-	result = dpp_base->funcs->dpp_program_blnd_lut(dpp_base, blend_lut);
+	result = dpp->funcs->dpp_program_blnd_lut(dpp, blend_lut);
 
 	return result;
 }
 
 bool dcn20_set_shaper_3dlut(
-	struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state)
+	struct dpp *dpp, struct dc_plane_state *plane_state)
 {
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	bool result = true;
 	const struct pwl_params *shaper_lut = NULL;
 
@@ -1096,35 +1113,34 @@ bool dcn20_set_shaper_3dlut(
 	else if (plane_state->cm.shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		cm_helper_translate_curve_to_hw_format(plane_state->ctx,
 				&plane_state->cm.shaper_func,
-				&dpp_base->shaper_params, true);
-		shaper_lut = &dpp_base->shaper_params;
+				&dpp->shaper_params, true);
+		shaper_lut = &dpp->shaper_params;
 	}
 
-	result = dpp_base->funcs->dpp_program_shaper_lut(dpp_base, shaper_lut);
+	dpp->funcs->dpp_program_shaper_lut(dpp, shaper_lut);
 	if (plane_state->cm.lut3d_func.state.bits.initialized == 1)
-		result = dpp_base->funcs->dpp_program_3dlut(dpp_base,
+		result = dpp->funcs->dpp_program_3dlut(dpp,
 								&plane_state->cm.lut3d_func.lut_3d);
 	else
-		result = dpp_base->funcs->dpp_program_3dlut(dpp_base, NULL);
+		result = dpp->funcs->dpp_program_3dlut(dpp, NULL);
 
 	return result;
 }
 
-bool dcn20_set_input_transfer_func(struct dc *dc,
-				struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
+bool dcn20_set_input_transfer_func(struct set_input_transfer_func_params *params)
 {
-	struct dce_hwseq *hws = dc->hwseq;
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+	struct dpp *dpp = params->dpp;
+	struct dc_plane_state *plane_state = params->plane_state;
+	struct dce_hwseq *hws = params->dc->hwseq;
 	const struct dc_transfer_func *tf = NULL;
 	bool result = true;
 	bool use_degamma_ram = false;
 
-	if (dpp_base == NULL || plane_state == NULL)
+	if (dpp == NULL || plane_state == NULL)
 		return false;
 
-	hws->funcs.set_shaper_3dlut(pipe_ctx, plane_state);
-	hws->funcs.set_blend_lut(pipe_ctx, plane_state);
+	hws->funcs.set_shaper_3dlut(dpp, plane_state);
+	hws->funcs.set_blend_lut(dpp, plane_state);
 
 	tf = &plane_state->in_transfer_func;
 
@@ -1133,13 +1149,13 @@ bool dcn20_set_input_transfer_func(struct dc *dc,
 
 	if (use_degamma_ram == true) {
 		if (tf->type == TF_TYPE_HWPWL)
-			dpp_base->funcs->dpp_program_degamma_pwl(dpp_base,
+			dpp->funcs->dpp_program_degamma_pwl(dpp,
 					&tf->pwl);
 		else if (tf->type == TF_TYPE_DISTRIBUTED_POINTS) {
 			cm_helper_translate_curve_to_degamma_hw_format(tf,
-					&dpp_base->degamma_params);
-			dpp_base->funcs->dpp_program_degamma_pwl(dpp_base,
-				&dpp_base->degamma_params);
+					&dpp->degamma_params);
+			dpp->funcs->dpp_program_degamma_pwl(dpp,
+				&dpp->degamma_params);
 		}
 		return true;
 	}
@@ -1149,21 +1165,21 @@ bool dcn20_set_input_transfer_func(struct dc *dc,
 	if (tf->type == TF_TYPE_PREDEFINED) {
 		switch (tf->tf) {
 		case TRANSFER_FUNCTION_SRGB:
-			dpp_base->funcs->dpp_set_degamma(dpp_base,
+			dpp->funcs->dpp_set_degamma(dpp,
 					IPP_DEGAMMA_MODE_HW_sRGB);
 			break;
 		case TRANSFER_FUNCTION_BT709:
-			dpp_base->funcs->dpp_set_degamma(dpp_base,
+			dpp->funcs->dpp_set_degamma(dpp,
 					IPP_DEGAMMA_MODE_HW_xvYCC);
 			break;
 		case TRANSFER_FUNCTION_LINEAR:
-			dpp_base->funcs->dpp_set_degamma(dpp_base,
+			dpp->funcs->dpp_set_degamma(dpp,
 					IPP_DEGAMMA_MODE_BYPASS);
 			break;
 		case TRANSFER_FUNCTION_PQ:
-			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_USER_PWL);
-			cm_helper_translate_curve_to_degamma_hw_format(tf, &dpp_base->degamma_params);
-			dpp_base->funcs->dpp_program_degamma_pwl(dpp_base, &dpp_base->degamma_params);
+			dpp->funcs->dpp_set_degamma(dpp, IPP_DEGAMMA_MODE_USER_PWL);
+			cm_helper_translate_curve_to_degamma_hw_format(tf, &dpp->degamma_params);
+			dpp->funcs->dpp_program_degamma_pwl(dpp, &dpp->degamma_params);
 			result = true;
 			break;
 		default:
@@ -1171,7 +1187,7 @@ bool dcn20_set_input_transfer_func(struct dc *dc,
 			break;
 		}
 	} else if (tf->type == TF_TYPE_BYPASS)
-		dpp_base->funcs->dpp_set_degamma(dpp_base,
+		dpp->funcs->dpp_set_degamma(dpp,
 				IPP_DEGAMMA_MODE_BYPASS);
 	else {
 		/*
@@ -1179,7 +1195,7 @@ bool dcn20_set_input_transfer_func(struct dc *dc,
 		 * fix is required for this use case
 		 */
 		BREAK_TO_DEBUGGER();
-		dpp_base->funcs->dpp_set_degamma(dpp_base,
+		dpp->funcs->dpp_set_degamma(dpp,
 				IPP_DEGAMMA_MODE_BYPASS);
 	}
 
@@ -1389,19 +1405,30 @@ void dcn20_enable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx,
 //	}
 }
 
-void dcn20_pipe_control_lock(
-	struct dc *dc,
-	struct pipe_ctx *pipe,
-	bool lock)
+bool dcn20_build_pipe_control_lock_sequence(
+		struct dc *dc,
+		struct pipe_ctx *pipe,
+		bool lock,
+		struct pipe_control_lock_params *params)
 {
 	struct pipe_ctx *temp_pipe;
 	bool flip_immediate = false;
+	unsigned int hubp_count = 0;
 
 	/* use TG master update lock to lock everything on the TG
 	 * therefore only top pipe need to lock
 	 */
 	if (!pipe || pipe->top_pipe)
-		return;
+		return false;
+
+	params->lock = lock;
+	params->tg_lock.dc = dc;
+	params->tg_lock.lock = lock;
+	params->tg_lock.tg = pipe->stream_res.tg;
+	params->tg_lock.use_dmub_inbox1 = pipe->stream &&
+			should_use_dmub_inbox1_lock(dc, pipe->stream->link);
+	params->tg_lock.triplebuffer_flips = pipe->plane_state &&
+			pipe->plane_state->triplebuffer_flips;
 
 	if (pipe->plane_state != NULL)
 		flip_immediate = pipe->plane_state->flip_immediate;
@@ -1416,22 +1443,10 @@ void dcn20_pipe_control_lock(
 	}
 
 	if (flip_immediate && lock) {
-		const unsigned int TIMEOUT_FOR_FLIP_PENDING_US = 100000U;
-		unsigned int polling_interval_us = 1;
-		unsigned int i;
-
 		temp_pipe = pipe;
 		while (temp_pipe) {
-			if (temp_pipe->plane_state && temp_pipe->plane_state->flip_immediate) {
-				for (i = 0; i < TIMEOUT_FOR_FLIP_PENDING_US / polling_interval_us; ++i) {
-					if (!temp_pipe->plane_res.hubp->funcs->hubp_is_flip_pending(temp_pipe->plane_res.hubp))
-						break;
-					udelay(polling_interval_us);
-				}
-
-				/* no reason it should take this long for immediate flips */
-				ASSERT(i != TIMEOUT_FOR_FLIP_PENDING_US);
-			}
+			if (temp_pipe->plane_state && temp_pipe->plane_state->flip_immediate)
+				params->hubps_to_wait_for_flip[hubp_count++] = temp_pipe->plane_res.hubp;
 			temp_pipe = temp_pipe->bottom_pipe;
 		}
 	}
@@ -1441,8 +1456,11 @@ void dcn20_pipe_control_lock(
 	 */
 	if (lock && (pipe->bottom_pipe != NULL || !flip_immediate))
 		if ((flip_immediate && pipe->stream_res.gsl_group == 0) ||
-		    (!flip_immediate && pipe->stream_res.gsl_group > 0))
-			dcn20_setup_gsl_group_as_lock(dc, pipe, flip_immediate);
+		    (!flip_immediate && pipe->stream_res.gsl_group > 0)) {
+			params->gsl_lock = true;
+			dcn20_build_gsl_group_as_lock(dc, pipe, flip_immediate,
+					&params->gsl, &params->gsl_source_select);
+		}
 
 	if (pipe->plane_state != NULL)
 		flip_immediate = pipe->plane_state->flip_immediate;
@@ -1455,37 +1473,53 @@ void dcn20_pipe_control_lock(
 	}
 
 	if (!lock && pipe->stream_res.gsl_group > 0 && pipe->plane_state &&
-		!flip_immediate)
-	    dcn20_setup_gsl_group_as_lock(dc, pipe, false);
+			!flip_immediate) {
+		params->gsl_lock = true;
+		dcn20_build_gsl_group_as_lock(dc, pipe, false,
+				&params->gsl, &params->gsl_source_select);
+	}
+
+	if (!lock && !params->tg_lock.use_dmub_inbox1 &&
+			!params->tg_lock.triplebuffer_flips &&
+			dc->hwseq->funcs.perform_3dlut_wa_unlock) {
+		const struct pipe_ctx *otg_master_pipe = resource_get_otg_master(pipe);
+		const struct pipe_ctx *primary_dpp_pipe = resource_is_pipe_type(pipe, DPP_PIPE) ?
+				resource_get_primary_dpp_pipe(pipe) : pipe;
+
+		if (otg_master_pipe && otg_master_pipe->stream_res.tg &&
+			 primary_dpp_pipe && primary_dpp_pipe->plane_state &&
+				primary_dpp_pipe->plane_state->cm.flags.bits.lut3d_enable &&
+				primary_dpp_pipe->plane_state->cm.flags.bits.lut3d_dma_enable) {
+			params->tg_3dlut_wa_unlock = true;
+			params->tg_3dlut_wa_unlock_params.tg = otg_master_pipe->stream_res.tg;
+			params->tg_3dlut_wa_unlock_params.hubp = primary_dpp_pipe->plane_res.hubp;
+		}
+	}
 
-	if (pipe->stream && should_use_dmub_inbox1_lock(dc, pipe->stream->link)) {
+	return true;
+}
+
+void dcn20_tg_lock(struct tg_lock_params *params)
+{
+	if (params->use_dmub_inbox1) {
 		union dmub_hw_lock_flags hw_locks = { 0 };
 		struct dmub_hw_lock_inst_flags inst_flags = { 0 };
 
 		hw_locks.bits.lock_pipe = 1;
-		inst_flags.otg_inst = (uint8_t)pipe->stream_res.tg->inst;
-
-		if (pipe->plane_state != NULL)
-			hw_locks.bits.triple_buffer_lock = pipe->plane_state->triplebuffer_flips;
-
-		dmub_hw_lock_mgr_cmd(dc->ctx->dmub_srv,
-					lock,
-					&hw_locks,
-					&inst_flags);
-	} else if (pipe->plane_state != NULL && pipe->plane_state->triplebuffer_flips) {
-		if (lock)
-			pipe->stream_res.tg->funcs->triplebuffer_lock(pipe->stream_res.tg);
+		inst_flags.otg_inst = (uint8_t)params->tg->inst;
+		hw_locks.bits.triple_buffer_lock = params->triplebuffer_flips;
+		dmub_hw_lock_mgr_cmd(params->dc->ctx->dmub_srv, params->lock,
+				&hw_locks, &inst_flags);
+	} else if (params->triplebuffer_flips) {
+		if (params->lock)
+			params->tg->funcs->triplebuffer_lock(params->tg);
 		else
-			pipe->stream_res.tg->funcs->triplebuffer_unlock(pipe->stream_res.tg);
+			params->tg->funcs->triplebuffer_unlock(params->tg);
 	} else {
-		if (lock)
-			pipe->stream_res.tg->funcs->lock(pipe->stream_res.tg);
-		else {
-			if (dc->hwseq->funcs.perform_3dlut_wa_unlock)
-				dc->hwseq->funcs.perform_3dlut_wa_unlock(pipe);
-			else
-				pipe->stream_res.tg->funcs->unlock(pipe->stream_res.tg);
-		}
+		if (params->lock)
+			params->tg->funcs->lock(params->tg);
+		else
+			params->tg->funcs->unlock(params->tg);
 	}
 }
 
@@ -1802,11 +1836,17 @@ void dcn20_update_dchubp_dpp(
 	if ((pipe_ctx->update_flags.bits.enable || pipe_ctx->update_flags.bits.opp_changed ||
 			pipe_ctx->update_flags.bits.scaler || viewport_changed == true) &&
 			pipe_ctx->stream->cursor_attributes.address.quad_part != 0) {
-		if (dc->hwss.abort_cursor_offload_update)
-			dc->hwss.abort_cursor_offload_update(dc, pipe_ctx);
+		if (dc_dmub_srv_is_cursor_offload_enabled(dc) && dc->hwss.abort_cursor_offload_update) {
+			struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+			if (top_pipe)
+				dc->hwss.abort_cursor_offload_update(dc->ctx->dmub_srv->dmub,
+					pipe_ctx->plane_res.dpp, pipe_ctx->plane_res.hubp,
+					top_pipe->pipe_idx);
+		}
 
 		dc->hwss.set_cursor_attribute(pipe_ctx);
-		dc->hwss.set_cursor_position(pipe_ctx);
+		hwss_program_cursor_position(dc, pipe_ctx);
 
 		if (dc->hwss.set_cursor_sdr_white_level)
 			dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
@@ -1987,7 +2027,7 @@ static void dcn20_program_pipe(
 			pipe_ctx->plane_state->update_bits.gamma_change ||
 			pipe_ctx->plane_state->update_bits.lut_3d ||
 			pipe_ctx->update_flags.bits.enable))
-		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
+		hwss_set_input_transfer_func(dc, pipe_ctx);
 
 	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
 	 * only do gamma programming for powering on, internal memcmp to avoid
@@ -2068,7 +2108,7 @@ void dcn20_program_front_end_for_ctx(
 				ASSERT(!pipe->plane_state->triplebuffer_flips);
 				/*turn off triple buffer for full update*/
 				dc->hwss.program_triplebuffer(
-					dc, pipe, pipe->plane_state->triplebuffer_flips);
+					pipe->plane_res.hubp, pipe->plane_state->triplebuffer_flips);
 			}
 		}
 	}
@@ -2735,7 +2775,8 @@ static bool patch_address_for_sbs_tb_stereo(
 	return false;
 }
 
-void dcn20_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
+void dcn20_prepare_plane_addr_update(const struct dc *dc, struct pipe_ctx *pipe_ctx,
+		struct dc_plane_address *addr_to_program, bool *flip_immediate)
 {
 	bool addr_patched = false;
 	PHYSICAL_ADDRESS_LOC addr;
@@ -2750,10 +2791,8 @@ void dcn20_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 	vm_helper_mark_vmid_used(dc->vm_helper, plane_state->address.vmid,
 			(uint8_t)pipe_ctx->plane_res.hubp->inst);
 
-	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
-			pipe_ctx->plane_res.hubp,
-			&plane_state->address,
-			plane_state->flip_immediate);
+	*addr_to_program = plane_state->address;
+	*flip_immediate = plane_state->flip_immediate;
 
 	plane_state->status.requested_address = plane_state->address;
 
@@ -2764,6 +2803,23 @@ void dcn20_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 		pipe_ctx->plane_state->address.grph_stereo.left_addr = addr;
 }
 
+void dcn20_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	struct dc_plane_address address;
+	bool flip_immediate;
+
+	if (pipe_ctx->plane_state == NULL)
+		return;
+
+	dcn20_prepare_plane_addr_update(dc, pipe_ctx, &address, &flip_immediate);
+
+	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
+			pipe_ctx->plane_res.hubp,
+			&address,
+			flip_immediate,
+			pipe_ctx->plane_state->dcc.enable);
+}
+
 void dcn20_unblank_stream(struct pipe_ctx *pipe_ctx,
 		struct dc_link_settings *link_settings)
 {
@@ -2849,7 +2905,12 @@ void dcn20_reset_back_end_for_pipe(
 	 * screen only, the dpms_off would be true but
 	 * VBIOS lit up eDP, so check link status too.
 	 */
-	if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
+	if (link->connector_signal == SIGNAL_TYPE_EDP &&
+	    link->forced_psr_active) {
+		/* forced psr is active for seamless switch; skip dpms-off. */
+		if (pipe_ctx->stream_res.audio)
+			dc->hwss.disable_audio_stream(pipe_ctx);
+	} else if (!pipe_ctx->stream->dpms_off || link->link_status.link_active)
 		dc->link_srv->set_dpms_off(pipe_ctx);
 	else if (pipe_ctx->stream_res.audio)
 		dc->hwss.disable_audio_stream(pipe_ctx);
@@ -2874,12 +2935,14 @@ void dcn20_reset_back_end_for_pipe(
 	 * parent pipe.
 	 */
 	if (pipe_ctx->top_pipe == NULL) {
+		if (!(link->connector_signal == SIGNAL_TYPE_EDP &&
+				link->forced_psr_active)) {
+			dc->hwss.set_abm_immediate_disable(pipe_ctx);
 
-		dc->hwss.set_abm_immediate_disable(pipe_ctx);
+			pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
 
-		pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
-
-		pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+			pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
+		}
 		if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
 			pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
 					pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
@@ -3169,10 +3232,6 @@ void dcn20_fpga_init_hw(struct dc *dc)
 	if (hws->funcs.enable_power_gating_plane)
 		hws->funcs.enable_power_gating_plane(hws, true);
 
-	// Specific to FPGA dccg and registers
-	REG_WRITE(RBBMIF_TIMEOUT_DIS, 0xFFFFFFFF);
-	REG_WRITE(RBBMIF_TIMEOUT_DIS_2, 0xFFFFFFFF);
-
 	dcn10_hubbub_global_timer_enable(dc->res_pool->hubbub, true, 2);
 
 	if (hws->funcs.dccg_init)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.h
index 749348d3c7938c..9a5addaa9a03ea 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.h
@@ -31,9 +31,9 @@
 void dcn20_log_color_state(struct dc *dc,
 			   struct dc_log_buffer_ctx *log_ctx);
 bool dcn20_set_blend_lut(
-	struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state);
+	struct dpp *dpp, struct dc_plane_state *plane_state);
 bool dcn20_set_shaper_3dlut(
-	struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state);
+	struct dpp *dpp, struct dc_plane_state *plane_state);
 void dcn20_program_front_end_for_ctx(
 		struct dc *dc,
 		struct dc_state *context);
@@ -41,9 +41,10 @@ void dcn20_post_unlock_program_front_end(
 		struct dc *dc,
 		struct dc_state *context);
 void dcn20_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx);
+void dcn20_prepare_plane_addr_update(const struct dc *dc, struct pipe_ctx *pipe_ctx,
+		struct dc_plane_address *addr_to_program, bool *flip_immediate);
 void dcn20_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx);
-bool dcn20_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx,
-			const struct dc_plane_state *plane_state);
+bool dcn20_set_input_transfer_func(struct set_input_transfer_func_params *params);
 bool dcn20_set_output_transfer_func(struct set_output_transfer_func_params *params);
 void dcn20_program_output_csc(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
@@ -62,10 +63,16 @@ void dcn20_blank_pixel_data(
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
 		bool blank);
-void dcn20_pipe_control_lock(
-	struct dc *dc,
-	struct pipe_ctx *pipe,
-	bool lock);
+bool dcn20_build_pipe_control_lock_sequence(struct dc *dc,
+		struct pipe_ctx *pipe, bool lock,
+		struct pipe_control_lock_params *params);
+void dcn20_build_gsl_group_as_lock(
+		const struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		bool enable,
+		struct tg_set_gsl_params *gsl_params,
+		struct tg_set_gsl_source_select_params *gsl_source_select_params);
+void dcn20_tg_lock(struct tg_lock_params *params);
 void dcn20_prepare_bandwidth(
 		struct dc *dc,
 		struct dc_state *context);
@@ -107,8 +114,7 @@ void dcn20_hubp_pg_control(
 		unsigned int hubp_inst,
 		bool power_on);
 void dcn20_program_triple_buffer(
-	const struct dc *dc,
-	struct pipe_ctx *pipe_ctx,
+	struct hubp *hubp,
 	bool enable_triple_buffer);
 void dcn20_enable_writeback(
 		struct dc *dc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
index 1797a91b0186eb..789bde43919268 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
@@ -40,6 +40,7 @@ static const struct hw_sequencer_funcs dcn20_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -56,7 +57,8 @@ static const struct hw_sequencer_funcs dcn20_funcs = {
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn20_prepare_bandwidth,
@@ -105,6 +107,7 @@ static const struct hw_sequencer_funcs dcn20_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn20_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
index 7b820bdae55b4d..c22587188c98a7 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c
@@ -133,7 +133,8 @@ static void plane_address_in_gpu_space_to_uma(struct dce_hwseq *hwseq,
 	}
 }
 
-void dcn201_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
+void dcn201_prepare_plane_addr_update(const struct dc *dc, struct pipe_ctx *pipe_ctx,
+		struct dc_plane_address *addr_to_program, bool *flip_immediate)
 {
 	bool addr_patched = false;
 	PHYSICAL_ADDRESS_LOC addr;
@@ -149,10 +150,9 @@ void dcn201_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 
 	plane_address_in_gpu_space_to_uma(hws, &uma);
 
-	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
-			pipe_ctx->plane_res.hubp,
-			&uma,
-			plane_state->flip_immediate);
+	/* snapshot the UMA-converted address before the stereo patch is restored */
+	*addr_to_program = uma;
+	*flip_immediate = plane_state->flip_immediate;
 
 	plane_state->status.requested_address = plane_state->address;
 
@@ -163,6 +163,23 @@ void dcn201_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 		pipe_ctx->plane_state->address.grph_stereo.left_addr = addr;
 }
 
+void dcn201_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	struct dc_plane_address address;
+	bool flip_immediate;
+
+	if (pipe_ctx->plane_state == NULL)
+		return;
+
+	dcn201_prepare_plane_addr_update(dc, pipe_ctx, &address, &flip_immediate);
+
+	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
+			pipe_ctx->plane_res.hubp,
+			&address,
+			flip_immediate,
+			pipe_ctx->plane_state->dcc.enable);
+}
+
 /* Blank pixel data during initialization */
 void dcn201_init_blank(
 		struct dc *dc,
@@ -525,35 +542,27 @@ void dcn201_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	hubp->mpcc_id = mpcc_id;
 }
 
-void dcn201_pipe_control_lock(
-	struct dc *dc,
-	struct pipe_ctx *pipe,
-	bool lock)
+void dcn201_tg_lock(struct tg_lock_params *params)
 {
-	struct dce_hwseq *hws = dc->hwseq;
-	/* use TG master update lock to lock everything on the TG
-	 * therefore only top pipe need to lock
-	 */
-	if (pipe->top_pipe)
-		return;
+	struct dce_hwseq *hws = params->dc->hwseq;
 
-	if (dc->debug.sanity_checks)
-		hws->funcs.verify_allow_pstate_change_high(dc);
+	if (params->dc->debug.sanity_checks)
+		hws->funcs.verify_allow_pstate_change_high(params->dc);
 
-	if (pipe->plane_state != NULL && pipe->plane_state->triplebuffer_flips) {
-		if (lock)
-			pipe->stream_res.tg->funcs->triplebuffer_lock(pipe->stream_res.tg);
+	if (params->triplebuffer_flips) {
+		if (params->lock)
+			params->tg->funcs->triplebuffer_lock(params->tg);
 		else
-			pipe->stream_res.tg->funcs->triplebuffer_unlock(pipe->stream_res.tg);
+			params->tg->funcs->triplebuffer_unlock(params->tg);
 	} else {
-		if (lock)
-			pipe->stream_res.tg->funcs->lock(pipe->stream_res.tg);
+		if (params->lock)
+			params->tg->funcs->lock(params->tg);
 		else
-			pipe->stream_res.tg->funcs->unlock(pipe->stream_res.tg);
+			params->tg->funcs->unlock(params->tg);
 	}
 
-	if (dc->debug.sanity_checks)
-		hws->funcs.verify_allow_pstate_change_high(dc);
+	if (params->dc->debug.sanity_checks)
+		hws->funcs.verify_allow_pstate_change_high(params->dc);
 }
 
 void dcn201_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.h
index 6a50a9894be6ae..dbcaa0a3a02cc6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.h
@@ -33,13 +33,12 @@ void dcn201_init_hw(struct dc *dc);
 void dcn201_unblank_stream(struct pipe_ctx *pipe_ctx,
 		struct dc_link_settings *link_settings);
 void dcn201_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx);
+void dcn201_prepare_plane_addr_update(const struct dc *dc, struct pipe_ctx *pipe_ctx,
+		struct dc_plane_address *addr_to_program, bool *flip_immediate);
 void dcn201_plane_atomic_disconnect(struct dc *dc, struct dc_state *state, struct pipe_ctx *pipe_ctx);
 void dcn201_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx);
 void dcn201_set_cursor_attribute(struct pipe_ctx *pipe_ctx);
-void dcn201_pipe_control_lock(
-	struct dc *dc,
-	struct pipe_ctx *pipe,
-	bool lock);
+void dcn201_tg_lock(struct tg_lock_params *params);
 void dcn201_init_blank(
 		struct dc *dc,
 		struct timing_generator *tg);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_init.c
index dec57fb4c05c97..33c22d9bbd2e2e 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_init.c
@@ -23,6 +23,7 @@
  *
  */
 
+#include "dce/dce_hwseq.h"
 #include "dce110/dce110_hwseq.h"
 #include "dcn10/dcn10_hwseq.h"
 #include "dcn20/dcn20_hwseq.h"
@@ -40,6 +41,7 @@ static const struct hw_sequencer_funcs dcn201_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn10_post_unlock_program_front_end,
 	.update_plane_addr = dcn201_update_plane_addr,
+	.prepare_plane_addr_update = dcn201_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -55,7 +57,8 @@ static const struct hw_sequencer_funcs dcn201_funcs = {
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn10_disable_plane,
-	.pipe_control_lock = dcn201_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dce_build_pipe_control_lock_sequence,
+	.tg_lock = dcn201_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn20_prepare_bandwidth,
@@ -96,6 +99,7 @@ static const struct hw_sequencer_funcs dcn201_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn201_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = NULL,
 	.plane_atomic_disconnect = dcn201_plane_atomic_disconnect,
 	.program_pipe = dcn10_program_pipe,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_init.c
index c7701a8b574aa3..39bf0e51ec7cf4 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_init.c
@@ -41,6 +41,7 @@ static const struct hw_sequencer_funcs dcn21_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -56,7 +57,8 @@ static const struct hw_sequencer_funcs dcn21_funcs = {
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn20_prepare_bandwidth,
@@ -108,6 +110,7 @@ static const struct hw_sequencer_funcs dcn21_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn21_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
index d669b47af12032..79f40b483caf99 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
@@ -234,25 +234,23 @@ void dcn30_log_color_state(struct dc *dc,
 }
 
 bool dcn30_set_blend_lut(
-	struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state)
+	struct dpp *dpp, struct dc_plane_state *plane_state)
 {
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	bool result = true;
 	const struct pwl_params *blend_lut = NULL;
 
 	if (plane_state->cm.blend_func.type == TF_TYPE_HWPWL)
 		blend_lut = &plane_state->cm.blend_func.pwl;
 	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
-		result = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
+		result = cm3_helper_translate_curve_to_degamma_hw_format(
 				&plane_state->cm.blend_func,
-				&dpp_base->regamma_params,
-				false);
+				&dpp->regamma_params);
 		if (!result)
 			return result;
 
-		blend_lut = &dpp_base->regamma_params;
+		blend_lut = &dpp->regamma_params;
 	}
-	result = dpp_base->funcs->dpp_program_blnd_lut(dpp_base, blend_lut);
+	result = dpp->funcs->dpp_program_blnd_lut(dpp, blend_lut);
 
 	return result;
 }
@@ -314,17 +312,16 @@ static bool dcn30_set_mpc_shaper_3dlut(struct dpp *dpp, struct mpc *mpc,
 	return result;
 }
 
-bool dcn30_set_input_transfer_func(struct dc *dc,
-				struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
+bool dcn30_set_input_transfer_func(struct set_input_transfer_func_params *params)
 {
-	struct dce_hwseq *hws = dc->hwseq;
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+	struct dpp *dpp = params->dpp;
+	struct dce_hwseq *hws = params->dc->hwseq;
+	struct dc_plane_state *plane_state = params->plane_state;
 	enum dc_transfer_func_predefined tf;
 	bool result = true;
-	const struct pwl_params *params = NULL;
+	const struct pwl_params *pwl_params = NULL;
 
-	if (dpp_base == NULL || plane_state == NULL)
+	if (dpp == NULL || plane_state == NULL)
 		return false;
 
 	tf = TRANSFER_FUNCTION_UNITY;
@@ -332,25 +329,23 @@ bool dcn30_set_input_transfer_func(struct dc *dc,
 	if (plane_state->in_transfer_func.type == TF_TYPE_PREDEFINED)
 		tf = plane_state->in_transfer_func.tf;
 
-	dpp_base->funcs->dpp_set_pre_degam(dpp_base, tf);
+	dpp->funcs->dpp_set_pre_degam(dpp, tf);
 
 	if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL)
-		params = &plane_state->in_transfer_func.pwl;
+		pwl_params = &plane_state->in_transfer_func.pwl;
 	else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS &&
-		cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-							&plane_state->in_transfer_func,
-							&dpp_base->degamma_params, false))
-		params = &dpp_base->degamma_params;
-
-	result = dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params);
-
-	if (pipe_ctx->stream_res.opp && pipe_ctx->stream_res.opp->ctx) {
-		if (dpp_base->funcs->dpp_program_blnd_lut)
-			hws->funcs.set_blend_lut(pipe_ctx, plane_state);
-		if (dpp_base->funcs->dpp_program_shaper_lut &&
-				dpp_base->funcs->dpp_program_3dlut)
-			hws->funcs.set_shaper_3dlut(pipe_ctx, plane_state);
-	}
+		cm3_helper_translate_curve_to_degamma_hw_format(&plane_state->in_transfer_func,
+								&dpp->degamma_params))
+		pwl_params = &dpp->degamma_params;
+
+	result = dpp->funcs->dpp_program_gamcor_lut(dpp, pwl_params);
+
+	if (dpp->funcs->dpp_program_blnd_lut)
+		hws->funcs.set_blend_lut(dpp, plane_state);
+	if (dpp->funcs->dpp_program_shaper_lut &&
+			dpp->funcs->dpp_program_3dlut)
+		hws->funcs.set_shaper_3dlut(dpp, plane_state);
+
 
 	return result;
 }
@@ -966,6 +961,12 @@ enum dc_status dcn30_setup_hdmi_frl_link(
 		frl_phy_clock_source_id,
 		link->frl_link_settings.frl_link_rate);
 	link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+
+	/* Enable HPO link encoder */
+	link->hpo_frl_link_enc->funcs->setup_link_encoder(
+			link->hpo_frl_link_enc,
+			link->frl_link_settings.frl_num_lanes);
+
 	return status;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h
index 0399642076eb50..65014d42195cae 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.h
@@ -56,12 +56,10 @@ bool dcn30_mmhubbub_warmup(
 void dcn30_log_color_state(struct dc *dc,
 			   struct dc_log_buffer_ctx *log_ctx);
 
-bool dcn30_set_blend_lut(struct pipe_ctx *pipe_ctx,
-		const struct dc_plane_state *plane_state);
+bool dcn30_set_blend_lut(struct dpp *dpp,
+		struct dc_plane_state *plane_state);
 
-bool dcn30_set_input_transfer_func(struct dc *dc,
-				struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state);
+bool dcn30_set_input_transfer_func(struct set_input_transfer_func_params *params);
 
 void dcn30_program_gamut_remap(struct program_gamut_remap_params *params);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c
index d5aa58462855cf..ce42b8216f3b40 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c
@@ -41,6 +41,7 @@ static const struct hw_sequencer_funcs dcn30_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -57,7 +58,8 @@ static const struct hw_sequencer_funcs dcn30_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
 	.disable_pixel_data = dcn20_disable_pixel_data,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn30_prepare_bandwidth,
@@ -115,6 +117,7 @@ static const struct hw_sequencer_funcs dcn30_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn30_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c
index a570333aeac186..5ed4305e46639a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c
@@ -43,6 +43,7 @@ static const struct hw_sequencer_funcs dcn301_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -58,7 +59,8 @@ static const struct hw_sequencer_funcs dcn301_funcs = {
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn20_prepare_bandwidth,
@@ -111,6 +113,7 @@ static const struct hw_sequencer_funcs dcn301_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn301_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c
index 7197414e5bd693..c7055312cc3d2e 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c
@@ -44,6 +44,7 @@ static const struct hw_sequencer_funcs dcn31_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -60,7 +61,8 @@ static const struct hw_sequencer_funcs dcn31_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
 	.disable_pixel_data = dcn20_disable_pixel_data,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn20_prepare_bandwidth,
@@ -117,6 +119,7 @@ static const struct hw_sequencer_funcs dcn31_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn31_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
index 6ddc678bacf961..56d41c6fa3eec6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
@@ -517,9 +517,11 @@ void dcn314_disable_link_output(struct dc_link *link,
 		link->dc->hwss.edp_backlight_control(link, false);
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->lock_phy(dmcu);
-
-	link_hwss->disable_link_output(link, link_res, signal);
-	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+	if (!(signal == SIGNAL_TYPE_EDP &&
+		link->forced_psr_active)) {
+		link_hwss->disable_link_output(link, link_res, signal);
+		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+	}
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
 	 * from enable/disable link output and only call edp panel control
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
index 4966c044a86440..2b04598b09355b 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
@@ -46,6 +46,7 @@ static const struct hw_sequencer_funcs dcn314_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -62,7 +63,8 @@ static const struct hw_sequencer_funcs dcn314_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
 	.disable_pixel_data = dcn20_disable_pixel_data,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn20_prepare_bandwidth,
@@ -120,6 +122,7 @@ static const struct hw_sequencer_funcs dcn314_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn314_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
index 51fd2b3fe106fb..5de38b86cb547a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
@@ -477,26 +477,30 @@ bool dcn32_set_mpc_shaper_3dlut(struct dpp *dpp, struct mpc *mpc,
 	return result;
 }
 
-bool dcn32_set_mcm_luts(
-	struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state)
+bool dcn32_set_mcm_luts(struct dc *dc, struct dpp *dpp, struct hubp *hubp,
+	struct hubp *primary_hubp, struct mpc *mpc, int mpcc_id,
+	struct dc_stream_state *stream,
+	struct dc_plane_state *plane_state)
 {
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
-	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
-	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
 	bool rval, result;
 	const struct pwl_params *lut_params = NULL;
 
+	(void)dc;
+	(void)hubp;
+	(void)primary_hubp;
+	(void)stream;
+
 	// 1D LUT
 	if (plane_state->cm.blend_func.type == TF_TYPE_HWPWL)
 		lut_params = &plane_state->cm.blend_func.pwl;
 	else if (plane_state->cm.blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 		result = cm3_helper_translate_curve_to_degamma_hw_format(
 			&plane_state->cm.blend_func,
-			&dpp_base->regamma_params);
+			&dpp->regamma_params);
 		if (!result)
 			return result;
 
-		lut_params = &dpp_base->regamma_params;
+		lut_params = &dpp->regamma_params;
 	}
 	mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id);
 	lut_params = NULL;
@@ -505,12 +509,12 @@ bool dcn32_set_mcm_luts(
 	if (plane_state->cm.shaper_func.type == TF_TYPE_HWPWL)
 		lut_params = &plane_state->cm.shaper_func.pwl;
 	else if (plane_state->cm.shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
-		// TODO: dpp_base replace
+		// TODO: dpp replace
 		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
 			&plane_state->cm.shaper_func,
-			&dpp_base->shaper_params,
+			&dpp->shaper_params,
 			true);
-		lut_params = rval ? &dpp_base->shaper_params : NULL;
+		lut_params = rval ? &dpp->shaper_params : NULL;
 	}
 
 	mpc->funcs->program_shaper(mpc, lut_params, mpcc_id);
@@ -524,19 +528,35 @@ bool dcn32_set_mcm_luts(
 	return result;
 }
 
-bool dcn32_set_input_transfer_func(struct dc *dc,
-				struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
+/*
+ * FP16 / 64bpp 16161616 surfaces store pixel data in linear light. These are
+ * the only source formats that must be de-linearized before scaling in
+ * source/non-linear space; every other format is already non-linear.
+ */
+static bool is_source_pixel_format_linear(enum surface_pixel_format format)
 {
-	struct dce_hwseq *hws = dc->hwseq;
-	struct mpc *mpc = dc->res_pool->mpc;
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+	switch (format) {
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616:
+	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
+	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool dcn32_set_input_transfer_func(struct set_input_transfer_func_params *params)
+{
+	struct dce_hwseq *hws = params->dc->hwseq;
+	struct dpp *dpp = params->dpp;
+	struct dc_plane_state *plane_state = params->plane_state;
 
 	enum dc_transfer_func_predefined tf;
 	bool result = true;
-	const struct pwl_params *params = NULL;
+	const struct pwl_params *pwl_params = NULL;
 
-	if (mpc == NULL || plane_state == NULL)
+	if (params->mpc == NULL || plane_state == NULL)
 		return false;
 
 	tf = TRANSFER_FUNCTION_UNITY;
@@ -544,24 +564,39 @@ bool dcn32_set_input_transfer_func(struct dc *dc,
 	if (plane_state->in_transfer_func.type == TF_TYPE_PREDEFINED)
 		tf = plane_state->in_transfer_func.tf;
 
-	if (dpp_base->funcs->dpp_set_pregam_state)
-		dpp_base->funcs->dpp_set_pregam_state(dpp_base, tf, plane_state->scaling_linearity);
+	if (dpp->funcs->dpp_set_pregam_state)
+		dpp->funcs->dpp_set_pregam_state(dpp, tf, plane_state->scaling_linearity,
+				is_source_pixel_format_linear(plane_state->format));
 	else
-		dpp_base->funcs->dpp_set_pre_degam(dpp_base, tf);
+		dpp->funcs->dpp_set_pre_degam(dpp, tf);
 
 	if (plane_state->in_transfer_func.type == TF_TYPE_HWPWL)
-		params = &plane_state->in_transfer_func.pwl;
+		pwl_params = &plane_state->in_transfer_func.pwl;
 	else if (plane_state->in_transfer_func.type == TF_TYPE_DISTRIBUTED_POINTS &&
 		cm3_helper_translate_curve_to_degamma_hw_format(&plane_state->in_transfer_func,
-								&dpp_base->degamma_params))
-		params = &dpp_base->degamma_params;
+								&dpp->degamma_params))
+		pwl_params = &dpp->degamma_params;
+
+	dpp->funcs->dpp_program_gamcor_lut(dpp, pwl_params);
 
-	dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params);
+	/* MCM and RMCM are mutually exclusive - tear the inactive one out before programming
+	 * the active one, which then owns the shared HUBP 3DLUT fast load.
+	 */
+	if (plane_state->cm.flags.bits.rmcm_enable && hws->funcs.set_rmcm_luts) {
+		if (hws->funcs.disable_mcm_luts)
+			hws->funcs.disable_mcm_luts(params->mpc, params->mpcc_id);
 
-	if (pipe_ctx->stream_res.opp &&
-			pipe_ctx->stream_res.opp->ctx &&
-			hws->funcs.set_mcm_luts)
-		result = hws->funcs.set_mcm_luts(pipe_ctx, plane_state);
+		result = hws->funcs.set_rmcm_luts(params);
+	} else {
+		if (hws->funcs.disable_rmcm_luts)
+			hws->funcs.disable_rmcm_luts(params->dc, params->rmcm,
+					params->hubp, params->mpcc_id);
+
+		if (hws->funcs.set_mcm_luts)
+			result = hws->funcs.set_mcm_luts(params->dc, dpp, params->hubp,
+				params->primary_hubp, params->mpc, params->mpcc_id,
+				params->stream, plane_state);
+	}
 
 	return result;
 }
@@ -1432,8 +1467,11 @@ void dcn32_disable_link_output(struct dc_link *link,
 	else if (dmcu != NULL && dmcu->funcs->lock_phy)
 		dmcu->funcs->lock_phy(dmcu);
 
-	link_hwss->disable_link_output(link, link_res, signal);
-	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+	if (!(signal == SIGNAL_TYPE_EDP &&
+		link->forced_psr_active)) {
+		link_hwss->disable_link_output(link, link_res, signal);
+		link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+	}
 	/*
 	 * Add the logic to extract BOTH power up and power down sequences
 	 * from enable/disable link output and only call edp panel control
@@ -1862,9 +1900,9 @@ void dcn32_interdependent_update_lock(struct dc *dc,
 			continue;
 
 		if (lock)
-			dc->hwss.pipe_control_lock(dc, pipe, true);
+			hwss_pipe_control_lock(dc, pipe, true);
 		else
-			dc->hwss.pipe_control_lock(dc, pipe, false);
+			hwss_pipe_control_lock(dc, pipe, false);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.h
index 090d94d38343b3..c85ff14596afb0 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.h
@@ -47,12 +47,12 @@ void dcn32_cab_for_ss_control(struct dc *dc, bool enable);
 
 void dcn32_commit_subvp_config(struct dc *dc, struct dc_state *context);
 
-bool dcn32_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state);
+bool dcn32_set_mcm_luts(struct dc *dc, struct dpp *dpp, struct hubp *hubp,
+				struct hubp *primary_hubp, struct mpc *mpc, int mpcc_id,
+				struct dc_stream_state *stream,
+				struct dc_plane_state *plane_state);
 
-bool dcn32_set_input_transfer_func(struct dc *dc,
-				struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state);
+bool dcn32_set_input_transfer_func(struct set_input_transfer_func_params *params);
 
 bool dcn32_set_mpc_shaper_3dlut(struct dpp *dpp_base, struct mpc *mpc,
 	int mpcc_id, const struct dc_stream_state *stream);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c
index 364b4108f5d6f5..9d3c7b40b21ffb 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c
@@ -44,6 +44,7 @@ static const struct hw_sequencer_funcs dcn32_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -60,7 +61,8 @@ static const struct hw_sequencer_funcs dcn32_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
 	.disable_pixel_data = dcn20_disable_pixel_data,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn32_interdependent_update_lock,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn32_prepare_bandwidth,
@@ -128,6 +130,7 @@ static const struct hw_sequencer_funcs dcn32_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn32_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index 6e1079377f4890..a959d8c37b6a96 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -872,9 +872,13 @@ void dcn35_enable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx,
  */
 void dcn35_plane_atomic_disable(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
+	struct dce_hwseq *hws = dc->hwseq;
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
 	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 
+	/* Clearing 3DLUT fast load writes to HUBP, so it must precede hubp_clk_cntl() below */
+	if (hws->funcs.disable_rmcm_luts)
+		hws->funcs.disable_rmcm_luts(dc, pipe_ctx->plane_res.rmcm, hubp, pipe_ctx->plane_res.mpcc_inst);
 
 	dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe_ctx);
 
@@ -1642,10 +1646,9 @@ void dcn35_hardware_release(struct dc *dc)
 		dc->hwss.hw_block_power_up(dc, &pg_update_state);
 }
 
-void dcn35_abort_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pipe)
+void dcn35_abort_cursor_offload_update(struct dmub_srv *dmub, struct dpp *dpp, struct hubp *hubp, uint32_t stream_idx)
 {
-	if (!dc_dmub_srv_is_cursor_offload_enabled(dc))
-		return;
+	struct dc *dc = dpp->ctx->dc;
 
 	/*
 	 * Insert a blank update to modify the write index and set pipe_mask to 0.
@@ -1664,56 +1667,57 @@ void dcn35_abort_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pip
 	 */
 
 	if (dc->hwss.begin_cursor_offload_update)
-		dc->hwss.begin_cursor_offload_update(dc, pipe);
+		dc->hwss.begin_cursor_offload_update(dmub, dpp, hubp, stream_idx);
 
 	if (dc->hwss.commit_cursor_offload_update)
-		dc->hwss.commit_cursor_offload_update(dc, pipe);
+		dc->hwss.commit_cursor_offload_update(dmub, dpp, hubp, stream_idx);
+
+	/*
+	 * The aborted payload is dropped by firmware, so resync the SW cursor
+	 * cache from real hardware state. This lets the next direct
+	 * set_cursor_position re-program CURSOR_ENABLE instead of skipping it
+	 * because of a stale cache.
+	 */
+	if (dpp->funcs->refresh_cursor_state)
+		dpp->funcs->refresh_cursor_state(dpp);
+	if (hubp && hubp->funcs->refresh_cursor_state)
+		hubp->funcs->refresh_cursor_state(hubp);
 }
 
-void dcn35_begin_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pipe)
+void dcn35_begin_cursor_offload_update(struct dmub_srv *dmub, struct dpp *dpp, struct hubp *hubp, uint32_t stream_idx)
 {
-	volatile struct dmub_cursor_offload_v1 *cs = dc->ctx->dmub_srv->dmub->cursor_offload_v1;
-	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe);
-	uint32_t stream_idx, write_idx, payload_idx;
-
-	if (!top_pipe)
-		return;
+	volatile struct dmub_cursor_offload_v1 *cs = dmub->cursor_offload_v1;
+	uint32_t write_idx, payload_idx;
 
-	stream_idx = top_pipe->pipe_idx;
 	write_idx = cs->offload_streams[stream_idx].write_idx + 1; /*  new payload (+1) */
 	payload_idx = write_idx % ARRAY_SIZE(cs->offload_streams[stream_idx].payloads);
 
 	cs->offload_streams[stream_idx].payloads[payload_idx].write_idx_start = write_idx;
 	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask = 0;
 
-	if (pipe->plane_res.hubp)
-		pipe->plane_res.hubp->cursor_offload = true;
+	if (hubp)
+		hubp->cursor_offload = true;
 
-	if (pipe->plane_res.dpp)
-		pipe->plane_res.dpp->cursor_offload = true;
+	if (dpp)
+		dpp->cursor_offload = true;
 }
 
-void dcn35_commit_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pipe)
+void dcn35_commit_cursor_offload_update(struct dmub_srv *dmub, struct dpp *dpp, struct hubp *hubp, uint32_t stream_idx)
 {
-	volatile struct dmub_cursor_offload_v1 *cs = dc->ctx->dmub_srv->dmub->cursor_offload_v1;
+	volatile struct dmub_cursor_offload_v1 *cs = dmub->cursor_offload_v1;
 	volatile struct dmub_shared_state_cursor_offload_stream_v1 *shared_stream;
-	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe);
-	uint32_t stream_idx, write_idx, payload_idx;
-
-	if (pipe->plane_res.hubp)
-		pipe->plane_res.hubp->cursor_offload = false;
+	uint32_t write_idx, payload_idx;
 
-	if (pipe->plane_res.dpp)
-		pipe->plane_res.dpp->cursor_offload = false;
+	if (hubp)
+		hubp->cursor_offload = false;
 
-	if (!top_pipe)
-		return;
+	if (dpp)
+		dpp->cursor_offload = false;
 
-	stream_idx = top_pipe->pipe_idx;
 	write_idx = cs->offload_streams[stream_idx].write_idx + 1; /*  new payload (+1) */
 	payload_idx = write_idx % ARRAY_SIZE(cs->offload_streams[stream_idx].payloads);
 
-	shared_stream = &dc->ctx->dmub_srv->dmub->shared_state[DMUB_SHARED_STATE_FEATURE__CURSOR_OFFLOAD_V1]
+	shared_stream = &dmub->shared_state[DMUB_SHARED_STATE_FEATURE__CURSOR_OFFLOAD_V1]
 				 .data.cursor_offload_v1.offload_streams[stream_idx];
 
 	shared_stream->last_write_idx = write_idx;
@@ -1722,23 +1726,17 @@ void dcn35_commit_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pi
 	cs->offload_streams[stream_idx].payloads[payload_idx].write_idx_finish = write_idx;
 }
 
-void dcn35_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe)
+void dcn35_update_cursor_offload_pipe(struct dmub_srv *dmub, uint32_t stream_idx,
+		uint8_t pipe_idx, const struct dpp *dpp, const struct hubp *hubp)
 {
-	volatile struct dmub_cursor_offload_v1 *cs = dc->ctx->dmub_srv->dmub->cursor_offload_v1;
-	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe);
-	const struct hubp *hubp = pipe->plane_res.hubp;
-	const struct dpp *dpp = pipe->plane_res.dpp;
+	volatile struct dmub_cursor_offload_v1 *cs = dmub->cursor_offload_v1;
 	volatile struct dmub_cursor_offload_pipe_data_dcn30_v1 *p;
-	uint32_t stream_idx, write_idx, payload_idx;
-
-	if (!top_pipe || !hubp || !dpp)
-		return;
+	uint32_t write_idx, payload_idx;
 
-	stream_idx = top_pipe->pipe_idx;
 	write_idx = cs->offload_streams[stream_idx].write_idx + 1; /*  new payload (+1) */
 	payload_idx = write_idx % ARRAY_SIZE(cs->offload_streams[stream_idx].payloads);
 
-	p = &cs->offload_streams[stream_idx].payloads[payload_idx].pipe_data[pipe->pipe_idx].dcn30;
+	p = &cs->offload_streams[stream_idx].payloads[payload_idx].pipe_data[pipe_idx].dcn30;
 
 	p->CURSOR0_0_CURSOR_SURFACE_ADDRESS = hubp->att.SURFACE_ADDR;
 	p->CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH = hubp->att.SURFACE_ADDR_HIGH;
@@ -1767,7 +1765,7 @@ void dcn35_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe
 	p->HUBPREQ0_CURSOR_SETTINGS__CURSOR0_DST_Y_OFFSET = hubp->att.settings.bits.dst_y_offset;
 	p->HUBPREQ0_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST = hubp->att.settings.bits.chunk_hdl_adjust;
 
-	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask |= (1u << pipe->pipe_idx);
+	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask |= (1u << pipe_idx);
 }
 
 void dcn35_notify_cursor_offload_drr_update(struct dc *dc, struct dc_state *context,
@@ -1835,3 +1833,49 @@ void dcn35_disable_link_output(struct dc_link *link,
 
 	dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
 }
+
+bool dcn35_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock)
+{
+	union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 };
+
+	if (!dc->ctx || !dc->ctx->dmub_srv)
+		return false;
+
+	/* if not support inbox0 lock, would not use inbox0 lock mechanism  */
+	if (!dc->ctx->dmub_srv->dmub->meta_info.feature_bits.bits.inbox0_lock_support)
+		return false;
+
+	if (lock) {
+		if (!dc_dmub_srv_is_cursor_offload_enabled(dc) &&
+			!dmub_hw_lock_mgr_does_context_require_lock(dc, context))
+			return false;
+	}
+
+	hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK;
+	hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER;
+	hw_lock_cmd.bits.lock = lock;
+	hw_lock_cmd.bits.should_release = !lock;
+	dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd);
+
+	return true;
+}
+
+void dcn35_dmub_hw_control_lock_fast(union block_sequence_params *params)
+{
+	struct dc *dc = params->dmub_hw_control_lock_fast_params.dc;
+	bool lock = params->dmub_hw_control_lock_fast_params.lock;
+
+	/* if not support inbox0 lock, would not use inbox0 lock mechanism  */
+	if (!dc->ctx->dmub_srv->dmub->meta_info.feature_bits.bits.inbox0_lock_support)
+		return;
+
+	if (params->dmub_hw_control_lock_fast_params.is_required) {
+		union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 };
+
+		hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK;
+		hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER;
+		hw_lock_cmd.bits.lock = lock;
+		hw_lock_cmd.bits.should_release = !lock;
+		dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd);
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
index 235ebf00bd1f21..9f779e408ec63b 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.h
@@ -27,6 +27,8 @@
 #ifndef __DC_HWSS_DCN35_H__
 #define __DC_HWSS_DCN35_H__
 
+#include "inc/core_types.h"
+#include "dc.h"
 #include "hw_sequencer_private.h"
 
 struct dc;
@@ -103,15 +105,19 @@ bool dcn35_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx);
 
 void dcn35_hardware_release(struct dc *dc);
 
-void dcn35_abort_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pipe);
-void dcn35_begin_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pipe);
-void dcn35_commit_cursor_offload_update(struct dc *dc, const struct pipe_ctx *pipe);
-void dcn35_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe);
+void dcn35_abort_cursor_offload_update(struct dmub_srv *dmub, struct dpp *dpp, struct hubp *hubp, uint32_t stream_idx);
+void dcn35_begin_cursor_offload_update(struct dmub_srv *dmub, struct dpp *dpp, struct hubp *hubp, uint32_t stream_idx);
+void dcn35_commit_cursor_offload_update(struct dmub_srv *dmub, struct dpp *dpp,
+	struct hubp *hubp, uint32_t stream_idx);
+void dcn35_update_cursor_offload_pipe(struct dmub_srv *dmub, uint32_t stream_idx,
+		uint8_t pipe_idx, const struct dpp *dpp, const struct hubp *hubp);
 void dcn35_notify_cursor_offload_drr_update(struct dc *dc, struct dc_state *context,
 					    const struct dc_stream_state *stream);
 void dcn35_program_cursor_offload_now(struct dc *dc, const struct pipe_ctx *pipe);
 void dcn35_disable_link_output(struct dc_link *link,
 		const struct link_resource *link_res,
 		enum signal_type signal);
+bool dcn35_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock);
+void dcn35_dmub_hw_control_lock_fast(union block_sequence_params *params);
 
 #endif /* __DC_HWSS_DCN35_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
index fc18d2207711f8..b1933a80fc4772 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_init.c
@@ -48,6 +48,7 @@ static const struct hw_sequencer_funcs dcn35_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -64,7 +65,8 @@ static const struct hw_sequencer_funcs dcn35_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn35_disable_plane,
 	.disable_pixel_data = dcn20_disable_pixel_data,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn35_prepare_bandwidth,
@@ -138,6 +140,7 @@ static const struct hw_sequencer_funcs dcn35_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn35_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn35_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
index 19ec5b4edfdc95..dbd8480b9e058d 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_init.c
@@ -47,6 +47,7 @@ static const struct hw_sequencer_funcs dcn351_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn20_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -63,7 +64,8 @@ static const struct hw_sequencer_funcs dcn351_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn35_disable_plane,
 	.disable_pixel_data = dcn20_disable_pixel_data,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn10_lock_all_pipes,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn35_prepare_bandwidth,
@@ -127,6 +129,7 @@ static const struct hw_sequencer_funcs dcn351_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn351_private_funcs = {
+	.build_cursor_pos_update_params = dcn10_build_cursor_pos_update_params,
 	.init_pipes = dcn35_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn20_update_mpcc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
index 51596f445a09f8..7780f37c615cc6 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
@@ -32,6 +32,7 @@
 #include "dcn10/dcn10_cm_common.h"
 #include "dcn10/dcn10_hubbub.h"
 #include "dcn20/dcn20_optc.h"
+#include "dcn20/dcn20_hwseq.h"
 #include "dcn30/dcn30_cm_common.h"
 #include "dcn32/dcn32_hwseq.h"
 #include "dcn401_hwseq.h"
@@ -76,7 +77,8 @@ void dcn401_initialize_min_clocks(struct dc *dc)
 		if (dc->clk_mgr->funcs->get_dispclk_from_dentist) {
 			clocks->dispclk_khz = dc->clk_mgr->funcs->get_dispclk_from_dentist(dc->clk_mgr);
 		} else {
-			clocks->dispclk_khz = dc->clk_mgr->boot_snapshot.dispclk * 1000;
+			// Boot snapshot is already in Khz
+			clocks->dispclk_khz = dc->clk_mgr->boot_snapshot.dispclk;
 		}
 	}
 	clocks->ref_dtbclk_khz = dc->clk_mgr->bw_params->clk_table.entries[0].dtbclk_mhz * 1000;
@@ -420,18 +422,12 @@ void dcn401_trigger_3dlut_dma_load(struct pipe_ctx *pipe_ctx)
 	}
 }
 
-bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
+bool dcn401_set_mcm_luts(struct dc *dc, struct dpp *dpp, struct hubp *hubp,
+				struct hubp *primary_hubp, struct mpc *mpc, int mpcc_id,
+				struct dc_stream_state *stream,
+				struct dc_plane_state *plane_state)
 {
-	struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc;
-	const struct pipe_ctx *primary_dpp_pipe_ctx = resource_get_primary_dpp_pipe(pipe_ctx);
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
-	struct hubp *hubp = pipe_ctx->plane_res.hubp;
-	struct hubp *primary_hubp = primary_dpp_pipe_ctx ?
-			primary_dpp_pipe_ctx->plane_res.hubp : hubp;   /* fall back to current pipe */
 	const struct dc_plane_cm *cm = &plane_state->cm;
-	int mpcc_id = hubp->inst;
-	struct mpc *mpc = dc->res_pool->mpc;
 	union mcm_lut_params m_lut_params;
 	struct dc_3dlut_dma lut3d_dma;
 	bool lut_enable;
@@ -439,6 +435,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 	bool rval;
 	bool result = true;
 
+	(void)dc;
+	(void)stream;
+
 	/* decide LUT bank based on current in use */
 	mpc->funcs->get_lut_mode(mpc, MCM_LUT_1DLUT, mpcc_id, &lut_enable, &lut_bank_a);
 	if (!lut_enable) {
@@ -458,6 +457,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 
 	/* 1D LUT */
 	lut_enable = cm->flags.bits.blend_enable != 0u;
+	if (cm->flags.bits.rmcm_enable)
+		lut_enable = false;
+
 	memset(&m_lut_params, 0, sizeof(m_lut_params));
 	if (lut_enable) {
 		if (cm->blend_func.type == TF_TYPE_HWPWL)
@@ -465,9 +467,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 		else if (cm->blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
 					&cm->blend_func,
-					&dpp_base->regamma_params,
+					&dpp->regamma_params,
 					false);
-			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
+			m_lut_params.pwl = rval ? &dpp->regamma_params : NULL;
 		}
 
 		if (!m_lut_params.pwl) {
@@ -484,6 +486,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 
 	/* Shaper */
 	lut_enable = cm->flags.bits.shaper_enable != 0u;
+	if (cm->flags.bits.rmcm_enable)
+		lut_enable = false;
+
 	if (lut_enable) {
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
 		if (cm->shaper_func.type == TF_TYPE_HWPWL)
@@ -492,9 +497,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 			ASSERT(false);
 			rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
 					&cm->shaper_func,
-					&dpp_base->shaper_params,
+					&dpp->shaper_params,
 					true);
-			m_lut_params.pwl = rval ? &dpp_base->shaper_params : NULL;
+			m_lut_params.pwl = rval ? &dpp->shaper_params : NULL;
 		}
 		if (!m_lut_params.pwl) {
 			lut_enable = false;
@@ -515,6 +520,9 @@ bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 
 	/* 3DLUT */
 	lut_enable = cm->flags.bits.lut3d_enable != 0u;
+	if (cm->flags.bits.rmcm_enable)
+		lut_enable = false;
+
 	if (lut_enable && cm->flags.bits.lut3d_dma_enable) {
 		/* Fast (DMA) Load Mode */
 		/* MPC */
@@ -1037,11 +1045,12 @@ void dcn401_disable_link_output(struct dc_link *link,
 	dc->link_srv->dp_trace_source_sequence(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
 }
 
-void dcn401_set_cursor_position(struct pipe_ctx *pipe_ctx)
+void dcn401_build_cursor_pos_update_params(const struct pipe_ctx *pipe_ctx,
+		struct dc_cursor_position *pos_out,
+		struct dc_cursor_mi_param *param_out)
 {
 	struct dc_cursor_position pos_cpy = pipe_ctx->stream->cursor_position;
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
-	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 	struct dc_cursor_mi_param param = {
 		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10,
 		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz,
@@ -1065,6 +1074,11 @@ void dcn401_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	int y_pos = pos_cpy.y;
 	int recout_x_pos = 0;
 	int recout_y_pos = 0;
+	int x_pos_viewport = 0;
+	int x_hot_viewport = 0;
+	int dst_x_offset = 0;
+	int rec_x_offset = 0;
+	int rec_y_offset = 0;
 
 	if ((pipe_ctx->top_pipe != NULL) || (pipe_ctx->bottom_pipe != NULL)) {
 		if ((pipe_ctx->plane_state->src_rect.width != pipe_ctx->plane_res.scl_data.viewport.width) ||
@@ -1207,8 +1221,59 @@ void dcn401_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	pos_cpy.x = x_pos;
 	pos_cpy.y = y_pos;
 
-	hubp->funcs->set_cursor_position(hubp, &pos_cpy, &param);
-	dpp->funcs->set_cursor_position(dpp, &pos_cpy, &param, hubp->curs_attr.width, hubp->curs_attr.height);
+	/* Precompute the HW destination offset so the executor only programs
+	 * registers. Translate cursor x position from rect space into viewport
+	 * space; CURSOR_DST_X_OFFSET is relative to the viewport start position.
+	 */
+	if (param.recout.width) {
+		x_pos_viewport = pos_cpy.x * param.viewport.width / param.recout.width;
+		x_hot_viewport = pos_cpy.x_hotspot * param.viewport.width / param.recout.width;
+	} else {
+		ASSERT(!pos_cpy.enable || pos_cpy.x == 0);
+		ASSERT(!pos_cpy.enable || pos_cpy.x_hotspot == 0);
+	}
+
+	/* pixel_clk_khz is 0 when no timing is programmed on the stream */
+	if (param.pixel_clk_khz) {
+		dst_x_offset = x_pos_viewport - x_hot_viewport *
+				(1 + hubp->curs_attr.attribute_flags.bits.ENABLE_MAGNIFICATION);
+		dst_x_offset = (dst_x_offset >= 0) ? dst_x_offset : 0;
+		dst_x_offset *= param.ref_clk_khz;
+		dst_x_offset /= param.pixel_clk_khz;
+
+		ASSERT(param.h_scale_ratio.value);
+
+		if (param.h_scale_ratio.value)
+			dst_x_offset = dc_fixpt_floor(dc_fixpt_div(
+				dc_fixpt_from_int(dst_x_offset),
+				param.h_scale_ratio));
+
+		param.dst_x_offset = dst_x_offset;
+	}
+
+	/* Cursor rectangle cache origin: derived from the final cursor position
+	 * minus hotspot, clamped to 0, offset by the recout origin. Precompute
+	 * here so the executor only stores the value.
+	 */
+	rec_x_offset = pos_cpy.x - pos_cpy.x_hotspot;
+	rec_y_offset = pos_cpy.y - pos_cpy.y_hotspot;
+	if (rec_x_offset < 0)
+		rec_x_offset = 0;
+	if (rec_y_offset < 0)
+		rec_y_offset = 0;
+	param.cur_rect_x = rec_x_offset + param.recout.x;
+	param.cur_rect_y = rec_y_offset + param.recout.y;
+
+	*pos_out = pos_cpy;
+	*param_out = param;
+}
+
+void dcn401_set_cursor_position(struct hubp *hubp, struct dpp *dpp,
+		const struct dc_cursor_position *pos,
+		const struct dc_cursor_mi_param *param)
+{
+	hubp->funcs->set_cursor_position(hubp, pos, param);
+	dpp->funcs->set_cursor_position(dpp, pos, param, hubp->curs_attr.width, hubp->curs_attr.height);
 }
 
 static bool dcn401_check_no_memory_request_for_cab(struct dc *dc)
@@ -1332,28 +1397,10 @@ bool dcn401_apply_idle_power_optimizations(struct dc *dc, bool enable)
 	return true;
 }
 
-void dcn401_wait_for_dcc_meta_propagation(const struct dc *dc,
-		const struct pipe_ctx *top_pipe)
+void dcn401_wait_for_dcc_meta_propagation(uint32_t delay)
 {
-	bool is_wait_needed = false;
-	const struct pipe_ctx *pipe_ctx = top_pipe;
-
-	/* check if any surfaces are updating address while using flip immediate and dcc */
-	while (pipe_ctx != NULL) {
-		if (pipe_ctx->plane_state &&
-				pipe_ctx->plane_state->dcc.enable &&
-				pipe_ctx->plane_state->flip_immediate &&
-				pipe_ctx->plane_state->update_bits.addr_update) {
-			is_wait_needed = true;
-			break;
-		}
-
-		/* check next pipe */
-		pipe_ctx = pipe_ctx->bottom_pipe;
-	}
-
-	if (is_wait_needed && dc->debug.dcc_meta_propagation_delay_us > 0) {
-		udelay(dc->debug.dcc_meta_propagation_delay_us);
+	if (delay > 0) {
+		udelay(delay);
 	}
 }
 
@@ -1566,25 +1613,27 @@ void dcn401_optimize_bandwidth_sequence(struct dc *dc,
 				dc->clk_mgr, context, true, seq_state);
 }
 
-void dcn401_dmub_hw_control_lock(struct dc *dc,
+bool dcn401_dmub_hw_control_lock(struct dc *dc,
 		struct dc_state *context,
 		bool lock)
 {
 	(void)context;
-	/* use always for now */
 	union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 };
 
 	if (!dc->ctx || !dc->ctx->dmub_srv)
-		return;
+		return false;
 
-	if (!dc->debug.fams2_config.bits.enable && !dc_dmub_srv_is_cursor_offload_enabled(dc))
-		return;
+	if (lock) {
+		if (!dc->debug.fams2_config.bits.enable && !dc_dmub_srv_is_cursor_offload_enabled(dc))
+			return false;
+	}
 
 	hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK;
 	hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER;
 	hw_lock_cmd.bits.lock = lock;
 	hw_lock_cmd.bits.should_release = !lock;
 	dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd);
+	return true;
 }
 
 void dcn401_dmub_hw_control_lock_fast(union block_sequence_params *params)
@@ -1864,7 +1913,10 @@ void dcn401_unblank_stream(struct pipe_ctx *pipe_ctx,
 	}
 
 	if (dc_is_hdmi_frl_signal(pipe_ctx->stream->signal)) {
-		if (link->link_status.link_active && link->frl_link_settings.frl_link_rate != 0)
+		bool unblank_required =
+			link->link_status.link_active && link->frl_link_settings.frl_link_rate != 0;
+
+		if (unblank_required)
 			pipe_ctx->stream_res.hpo_frl_stream_enc->funcs->hdmi_frl_unblank(
 					pipe_ctx->stream_res.hpo_frl_stream_enc,
 					pipe_ctx->stream_res.tg->inst);
@@ -1952,7 +2004,7 @@ void dcn401_interdependent_update_lock(struct dc *dc,
 					!tg->funcs->is_tg_enabled(tg) ||
 					dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_PHANTOM)
 				continue;
-			dc->hwss.pipe_control_lock(dc, pipe, true);
+			hwss_pipe_control_lock(dc, pipe, true);
 		}
 	} else {
 		/* Need to free DET being used first and have pipe update, then unlock the remaining pipes*/
@@ -1968,7 +2020,7 @@ void dcn401_interdependent_update_lock(struct dc *dc,
 
 			if (dc->scratch.pipes_to_unlock_first[i]) {
 				struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
-				dc->hwss.pipe_control_lock(dc, pipe, false);
+				hwss_pipe_control_lock(dc, pipe, false);
 				/* Assumes pipe of the same index in current_state is also an OTG_MASTER pipe*/
 				dcn401_wait_for_det_buffer_update_under_otg_master(dc, dc->current_state, old_pipe);
 			}
@@ -1987,12 +2039,13 @@ void dcn401_interdependent_update_lock(struct dc *dc,
 				continue;
 			}
 
-			dc->hwss.pipe_control_lock(dc, pipe, false);
+			hwss_pipe_control_lock(dc, pipe, false);
 		}
 	}
 }
 
-void dcn401_perform_3dlut_wa_unlock(struct pipe_ctx *pipe_ctx)
+void dcn401_perform_3dlut_wa_unlock(struct timing_generator *tg,
+		struct hubp *primary_hubp)
 {
 	/* If 3DLUT FL is enabled and 3DLUT is in use, follow the workaround sequence for pipe unlock to make sure that
 	 * HUBP will properly fetch 3DLUT contents after unlock.
@@ -2000,45 +2053,24 @@ void dcn401_perform_3dlut_wa_unlock(struct pipe_ctx *pipe_ctx)
 	 * This is meant to work around a known HW issue where VREADY will cancel the pending 3DLUT_ENABLE signal regardless
 	 * of whether OTG lock is currently being held or not.
 	 */
-	if (!pipe_ctx)
+	if (!tg)
 		return;
 
-	const struct pipe_ctx *otg_master_pipe_ctx = resource_get_otg_master(pipe_ctx);
-	struct timing_generator *tg = otg_master_pipe_ctx ?
-			otg_master_pipe_ctx->stream_res.tg : NULL;
-	const struct pipe_ctx *primary_dpp_pipe_ctx = resource_is_pipe_type(pipe_ctx, DPP_PIPE) ?
-			resource_get_primary_dpp_pipe(pipe_ctx) : pipe_ctx;
-	struct hubp *primary_hubp = primary_dpp_pipe_ctx ?
-			primary_dpp_pipe_ctx->plane_res.hubp : NULL;
-
-	if (!otg_master_pipe_ctx || !tg) {
-		return;
-	}
-
-	if (primary_dpp_pipe_ctx &&
-			primary_dpp_pipe_ctx->plane_state &&
-			primary_dpp_pipe_ctx->plane_state->cm.flags.bits.lut3d_enable &&
-			primary_dpp_pipe_ctx->plane_state->cm.flags.bits.lut3d_dma_enable) {
-		if (tg->funcs->set_vupdate_keepout)
-			tg->funcs->set_vupdate_keepout(tg, true);
+	if (tg->funcs->set_vupdate_keepout)
+		tg->funcs->set_vupdate_keepout(tg, true);
 
-		if (primary_hubp && primary_hubp->funcs->hubp_enable_3dlut_fl) {
-			primary_hubp->funcs->hubp_enable_3dlut_fl(primary_hubp, true);
-		}
+	if (primary_hubp && primary_hubp->funcs->hubp_enable_3dlut_fl)
+		primary_hubp->funcs->hubp_enable_3dlut_fl(primary_hubp, true);
 
-		tg->funcs->unlock(tg);
-		if (tg->funcs->wait_update_lock_status)
-			tg->funcs->wait_update_lock_status(tg, false);
+	tg->funcs->unlock(tg);
+	if (tg->funcs->wait_update_lock_status)
+		tg->funcs->wait_update_lock_status(tg, false);
 
-		if (primary_hubp && primary_hubp->funcs->hubp_enable_3dlut_fl) {
-			primary_hubp->funcs->hubp_enable_3dlut_fl(primary_hubp, true);
-		}
+	if (primary_hubp && primary_hubp->funcs->hubp_enable_3dlut_fl)
+		primary_hubp->funcs->hubp_enable_3dlut_fl(primary_hubp, true);
 
-		if (tg->funcs->set_vupdate_keepout)
-			tg->funcs->set_vupdate_keepout(tg, false);
-	} else {
-		tg->funcs->unlock(tg);
-	}
+	if (tg->funcs->set_vupdate_keepout)
+		tg->funcs->set_vupdate_keepout(tg, false);
 }
 
 void dcn401_program_outstanding_updates(struct dc *dc,
@@ -2305,7 +2337,7 @@ void dcn401_program_pipe(
 			pipe_ctx->plane_state->update_bits.gamma_change ||
 			pipe_ctx->plane_state->update_bits.lut_3d ||
 			pipe_ctx->update_flags.bits.enable))
-		hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
+		hwss_set_input_transfer_func(dc, pipe_ctx);
 
 	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
 	 * only do gamma programming for powering on, internal memcmp to avoid
@@ -2462,7 +2494,7 @@ void dcn401_program_pipe_sequence(
 			pipe_ctx->plane_state->update_bits.lut_3d ||
 			pipe_ctx->update_flags.bits.enable)) {
 
-		hwss_add_dpp_set_input_transfer_func(seq_state, dc, pipe_ctx, pipe_ctx->plane_state);
+		hwss_add_dpp_set_input_transfer_func(seq_state, dc, pipe_ctx);
 	}
 
 	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
@@ -2558,7 +2590,7 @@ void dcn401_program_front_end_for_ctx(
 
 				/*turn off triple buffer for full update*/
 				dc->hwss.program_triplebuffer(
-					dc, pipe, pipe->plane_state->triplebuffer_flips);
+					pipe->plane_res.hubp, pipe->plane_state->triplebuffer_flips);
 			}
 		}
 	}
@@ -3125,23 +3157,17 @@ void dcn401_plane_atomic_power_down(struct dc *dc,
 		hws->funcs.dpp_root_clock_control(hws, dpp->inst, false);
 }
 
-void dcn401_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe)
+void dcn401_update_cursor_offload_pipe(struct dmub_srv *dmub, uint32_t stream_idx,
+		uint8_t pipe_idx, const struct dpp *dpp, const struct hubp *hubp)
 {
-	volatile struct dmub_cursor_offload_v1 *cs = dc->ctx->dmub_srv->dmub->cursor_offload_v1;
-	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe);
-	const struct hubp *hubp = pipe->plane_res.hubp;
-	const struct dpp *dpp = pipe->plane_res.dpp;
+	volatile struct dmub_cursor_offload_v1 *cs = dmub->cursor_offload_v1;
 	volatile struct dmub_cursor_offload_pipe_data_dcn401_v1 *p;
-	uint32_t stream_idx, write_idx, payload_idx;
-
-	if (!top_pipe || !hubp || !dpp)
-		return;
+	uint32_t write_idx, payload_idx;
 
-	stream_idx = top_pipe->pipe_idx;
 	write_idx = cs->offload_streams[stream_idx].write_idx + 1; /*  new payload (+1) */
 	payload_idx = write_idx % ARRAY_SIZE(cs->offload_streams[stream_idx].payloads);
 
-	p = &cs->offload_streams[stream_idx].payloads[payload_idx].pipe_data[pipe->pipe_idx].dcn401;
+	p = &cs->offload_streams[stream_idx].payloads[payload_idx].pipe_data[pipe_idx].dcn401;
 
 	p->CURSOR0_0_CURSOR_SURFACE_ADDRESS = hubp->att.SURFACE_ADDR;
 	p->CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH = hubp->att.SURFACE_ADDR_HIGH;
@@ -3178,7 +3204,7 @@ void dcn401_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pip
 	p->HUBPREQ0_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST = hubp->att.settings.bits.chunk_hdl_adjust;
 	p->HUBP0_DCHUBP_MALL_CONFIG__USE_MALL_FOR_CURSOR = hubp->use_mall_for_cursor;
 
-	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask |= (1u << pipe->pipe_idx);
+	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask |= (1u << pipe_idx);
 }
 
 void dcn401_plane_atomic_power_down_sequence(struct dc *dc,
@@ -3479,89 +3505,24 @@ void dcn401_update_writeback_sequence(
 	hwss_add_mcif_wb_config_buf(seq_state, mcif_wb, &wb_info->mcif_buf_params, wb_info->dwb_params.dest_height);
 }
 
-static int find_free_gsl_group(const struct dc *dc)
-{
-	if (dc->res_pool->gsl_groups.gsl_0 == 0)
-		return 1;
-	if (dc->res_pool->gsl_groups.gsl_1 == 0)
-		return 2;
-	if (dc->res_pool->gsl_groups.gsl_2 == 0)
-		return 3;
-
-	return 0;
-}
-
 void dcn401_setup_gsl_group_as_lock_sequence(
 		const struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
 		bool enable,
 		struct block_sequence_state *seq_state)
 {
-	struct gsl_params gsl;
-	int group_idx;
-
-	memset(&gsl, 0, sizeof(struct gsl_params));
-
-	if (enable) {
-		/* return if group already assigned since GSL was set up
-		 * for vsync flip, we would unassign so it can't be "left over"
-		 */
-		if (pipe_ctx->stream_res.gsl_group > 0)
-			return;
-
-		group_idx = find_free_gsl_group(dc);
-		ASSERT(group_idx != 0);
-		pipe_ctx->stream_res.gsl_group = (uint8_t)group_idx;
-
-		/* set gsl group reg field and mark resource used */
-		switch (group_idx) {
-		case 1:
-			gsl.gsl0_en = 1;
-			dc->res_pool->gsl_groups.gsl_0 = 1;
-			break;
-		case 2:
-			gsl.gsl1_en = 1;
-			dc->res_pool->gsl_groups.gsl_1 = 1;
-			break;
-		case 3:
-			gsl.gsl2_en = 1;
-			dc->res_pool->gsl_groups.gsl_2 = 1;
-			break;
-		default:
-			BREAK_TO_DEBUGGER();
-			return; // invalid case
-		}
-		gsl.gsl_master_en = 1;
-	} else {
-		group_idx = pipe_ctx->stream_res.gsl_group;
-		if (group_idx == 0)
-			return; // if not in use, just return
-
-		pipe_ctx->stream_res.gsl_group = 0;
+	struct tg_set_gsl_params gsl_params = { 0 };
+	struct tg_set_gsl_source_select_params gsl_source_select_params = { 0 };
 
-		/* unset gsl group reg field and mark resource free */
-		switch (group_idx) {
-		case 1:
-			gsl.gsl0_en = 0;
-			dc->res_pool->gsl_groups.gsl_0 = 0;
-			break;
-		case 2:
-			gsl.gsl1_en = 0;
-			dc->res_pool->gsl_groups.gsl_1 = 0;
-			break;
-		case 3:
-			gsl.gsl2_en = 0;
-			dc->res_pool->gsl_groups.gsl_2 = 0;
-			break;
-		default:
-			BREAK_TO_DEBUGGER();
-			return;
-		}
-		gsl.gsl_master_en = 0;
-	}
+	dcn20_build_gsl_group_as_lock(dc, pipe_ctx, enable,
+			&gsl_params, &gsl_source_select_params);
+	if (!gsl_source_select_params.group_idx)
+		return;
 
-	hwss_add_tg_set_gsl(seq_state, pipe_ctx->stream_res.tg, gsl);
-	hwss_add_tg_set_gsl_source_select(seq_state, pipe_ctx->stream_res.tg, group_idx, enable ? 4 : 0);
+	hwss_add_tg_set_gsl(seq_state, gsl_params.tg, gsl_params.gsl);
+	hwss_add_tg_set_gsl_source_select(seq_state, gsl_source_select_params.tg,
+			gsl_source_select_params.group_idx,
+			gsl_source_select_params.gsl_ready_signal);
 }
 
 void dcn401_disable_plane_sequence(
@@ -3782,7 +3743,7 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 			plane_state->update_bits.input_csc_change ||
 			plane_state->update_bits.color_space_change ||
 			plane_state->update_bits.coeff_reduction_change) {
-		hwss_add_dpp_setup_dpp(seq_state, pipe_ctx);
+		hwss_add_dpp_setup_dpp(seq_state, dpp, plane_state);
 
 		/* Step 8: DPP cursor matrix setup */
 		if (dpp->funcs->set_cursor_matrix) {
@@ -3792,7 +3753,7 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 
 		/* Step 9: DPP program bias and scale */
 		if (dpp->funcs->dpp_program_bias_and_scale)
-			hwss_add_dpp_program_bias_and_scale(seq_state, pipe_ctx);
+			hwss_add_dpp_program_bias_and_scale(seq_state, dpp, plane_state);
 	}
 
 	/* Step 10: MPCC updates */
@@ -3844,8 +3805,7 @@ void dcn401_update_dchubp_dpp_sequence(struct dc *dc,
 		hwss_add_set_cursor_position(seq_state, dc, pipe_ctx);
 
 		/* Step 16: Cursor SDR white level */
-		if (dc->hwss.set_cursor_sdr_white_level)
-			hwss_add_set_cursor_sdr_white_level(seq_state, dc, pipe_ctx);
+		hwss_add_set_cursor_sdr_white_level(seq_state, pipe_ctx);
 	}
 
 	/* Step 17: Gamut remap and output CSC */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
index 69c7dc73a29a1c..66a0fc8de8baeb 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.h
@@ -36,8 +36,10 @@ void dcn401_program_gamut_remap(struct program_gamut_remap_params *params);
 
 void dcn401_init_hw(struct dc *dc);
 
-bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state);
+bool dcn401_set_mcm_luts(struct dc *dc, struct dpp *dpp, struct hubp *hubp,
+				struct hubp *primary_hubp, struct mpc *mpc, int mpcc_id,
+				struct dc_stream_state *stream,
+				struct dc_plane_state *plane_state);
 bool dcn401_set_output_transfer_func(struct set_output_transfer_func_params *params);
 void dcn401_trigger_3dlut_dma_load(struct pipe_ctx *pipe_ctx);
 void dcn401_calculate_dccg_tmds_div_value(struct pipe_ctx *pipe_ctx,
@@ -53,12 +55,16 @@ void dcn401_disable_link_output(struct dc_link *link,
 		const struct link_resource *link_res,
 		enum signal_type signal);
 
-void dcn401_set_cursor_position(struct pipe_ctx *pipe_ctx);
+void dcn401_build_cursor_pos_update_params(const struct pipe_ctx *pipe_ctx,
+		struct dc_cursor_position *pos,
+		struct dc_cursor_mi_param *param);
+void dcn401_set_cursor_position(struct hubp *hubp, struct dpp *dpp,
+		const struct dc_cursor_position *pos,
+		const struct dc_cursor_mi_param *param);
 
 bool dcn401_apply_idle_power_optimizations(struct dc *dc, bool enable);
 
-void dcn401_wait_for_dcc_meta_propagation(const struct dc *dc,
-		const struct pipe_ctx *top_pipe_to_program);
+void dcn401_wait_for_dcc_meta_propagation(uint32_t delay);
 
 void dcn401_prepare_bandwidth(struct dc *dc,
 		struct dc_state *context);
@@ -77,7 +83,7 @@ void dcn401_optimize_bandwidth_sequence(struct dc *dc,
 		struct dc_state *context,
 		struct block_sequence_state *seq_state);
 
-void dcn401_dmub_hw_control_lock(struct dc *dc,
+bool dcn401_dmub_hw_control_lock(struct dc *dc,
 		struct dc_state *context,
 		bool lock);
 void dcn401_fams2_update_config(struct dc *dc, struct dc_state *context, bool enable);
@@ -108,7 +114,8 @@ void dcn401_program_pipe_sequence(
 	struct pipe_ctx *pipe_ctx,
 	struct dc_state *context,
 	struct block_sequence_state *seq_state);
-void dcn401_perform_3dlut_wa_unlock(struct pipe_ctx *pipe_ctx);
+void dcn401_perform_3dlut_wa_unlock(struct timing_generator *tg,
+		struct hubp *primary_hubp);
 void dcn401_program_front_end_for_ctx(struct dc *dc, struct dc_state *context);
 void dcn401_post_unlock_program_front_end(struct dc *dc, struct dc_state *context);
 bool dcn401_update_bandwidth(struct dc *dc, struct dc_state *context);
@@ -134,7 +141,8 @@ void dcn401_blank_pixel_data_sequence(
 	bool blank,
 	struct block_sequence_state *seq_state);
 void dcn401_initialize_min_clocks(struct dc *dc);
-void dcn401_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe);
+void dcn401_update_cursor_offload_pipe(struct dmub_srv *dmub, uint32_t stream_idx,
+		uint8_t pipe_idx, const struct dpp *dpp, const struct hubp *hubp);
 
 void dcn401_program_all_writeback_pipes_in_tree_sequence(
 		struct dc *dc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
index 7039f40ea880da..2d3c46c4e00c2d 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
@@ -23,6 +23,7 @@ static const struct hw_sequencer_funcs dcn401_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn401_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
@@ -40,7 +41,8 @@ static const struct hw_sequencer_funcs dcn401_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
 	.disable_plane_sequence = dcn401_disable_plane_sequence,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn401_interdependent_update_lock,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn401_prepare_bandwidth,
@@ -123,6 +125,7 @@ static const struct hw_sequencer_funcs dcn401_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn401_private_funcs = {
+	.build_cursor_pos_update_params = dcn401_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.plane_atomic_disconnect_sequence = dcn401_plane_atomic_disconnect_sequence,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
index 13580d9f9303e8..21c3d108abe9ac 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
@@ -402,141 +402,238 @@ void dcn42_program_cm_hist(
 			plane_state->cm_hist_control, plane_state->color_space);
 }
 
-static bool dc_is_rmcm_3dlut_supported(struct hubp *hubp, struct mpc *mpc)
+/* RMCM is per-pipe and reloads through its own HUBP, unlike MCM which is shared by
+ * the plane's pipes and always reloads through the primary one.
+ */
+void dcn42_trigger_3dlut_dma_load(struct pipe_ctx *pipe_ctx)
 {
-	if (mpc->funcs->rmcm.power_on_shaper_3dlut &&
-		mpc->funcs->rmcm.fl_3dlut_configure &&
-		hubp->funcs->hubp_program_3dlut_fl_config)
-		return true;
+	struct hubp *hubp;
+	bool use_rmcm = pipe_ctx->plane_state && pipe_ctx->plane_state->cm.flags.bits.rmcm_enable;
+
+	if (use_rmcm) {
+		hubp = pipe_ctx->plane_res.hubp;
+	} else {
+		const struct pipe_ctx *primary_dpp_pipe_ctx = resource_get_primary_dpp_pipe(pipe_ctx);
+
+		hubp = primary_dpp_pipe_ctx ? primary_dpp_pipe_ctx->plane_res.hubp : NULL;
+	}
 
-	return false;
+	if (hubp && hubp->funcs->hubp_enable_3dlut_fl)
+		hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
 }
 
-bool dcn42_program_rmcm_luts(
-	struct hubp *hubp,
-	struct pipe_ctx *pipe_ctx,
-	const struct dc_plane_cm *cm,
-	struct mpc *mpc,
-	int mpcc_id)
+/* Takes the MCM out of the pipe. The caller programs the HUBP fast load afterwards. */
+void dcn42_disable_mcm_luts(struct mpc *mpc, int mpcc_id)
 {
-	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
-	union mcm_lut_params m_lut_params = {0};
+	if (!mpc || !mpc->funcs->program_lut_mode)
+		return;
 
-	struct dc *dc = hubp->ctx->dc;
-	struct mpc_fl_3dlut_config mpc_fl_config;
+	mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, false, false, CM_LUT_SIZE_NONE, mpcc_id);
+	mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, false, false, CM_LUT_SIZE_NONE, mpcc_id);
+	mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, false, false, CM_LUT_SIZE_NONE, mpcc_id);
+}
 
-	struct dc_stream_state *stream = pipe_ctx->stream;
-	bool bypass_rmcm_shaper = false;
-	// true->false when it can be allocated at DI time
-	struct dc_rmcm_3dlut *rmcm_3dlut = dc_stream_get_3dlut_for_stream(dc, stream, false);
+static void dcn42_detach_rmcm(struct rmcm *rmcm)
+{
+	if (rmcm->funcs->update_3dlut_fast_load_select)
+		rmcm->funcs->update_3dlut_fast_load_select(rmcm, rmcm->inst, RMCM_FL_HUBP_IDX_NONE);
 
-	bool lut_enable = false;
-	bool lut_bank_a = true;
+	if (rmcm->funcs->connect_mpcc)
+		rmcm->funcs->connect_mpcc(rmcm, rmcm->inst, RMCM_MPCC_ID_NONE);
+}
 
-	//check to see current pipe is part of a stream with allocated rmcm 3dlut
-	if (!rmcm_3dlut)
-		return false;
+/* MPC makes an MPCC bypass RMCM once more than one instance is bound to it, so drop every
+ * instance that could still point at this MPCC. One the driver has not programmed yet counts,
+ * as its MPC_RMCM_CNTL may still hold a boot value.
+ */
+void dcn42_release_rmcm_from_mpcc(struct dc *dc, int mpcc_id, const struct rmcm *keep)
+{
+	int i;
 
-	/* Determine the LUT bank currently active in HW and switch to the other
-	 * bank to preserve double-buffering (mirrors dcn401_set_mcm_luts).
-	 */
-	if (mpc->funcs->rmcm.get_3dlut_mode)
-		mpc->funcs->rmcm.get_3dlut_mode(mpc, mpcc_id, &lut_enable, &lut_bank_a);
+	for (i = 0; i < dc->res_pool->res_cap->num_rmcm && i < MAX_RMCM_INST; i++) {
+		struct rmcm *rmcm = dc->res_pool->rmcm[i];
+
+		if (!rmcm || !rmcm->funcs)
+			continue;
+
+		if (rmcm == keep)
+			continue;
 
-	if (lut_enable)
+		/* An instance the driver has not programmed yet counts as bound, because its
+		 * MPC_RMCM_CNTL may still hold a boot value.
+		 */
+		if (rmcm->mpcc_id == mpcc_id || rmcm->mpcc_id == RMCM_MPCC_ID_UNKNOWN)
+			dcn42_detach_rmcm(rmcm);
+	}
+}
+
+/* Leaves mpcc_id with no RMCM in its path. A NULL hubp keeps the shared 3DLUT fast load
+ * untouched, for when another block already owns that HUBP.
+ */
+void dcn42_disable_rmcm_luts(struct dc *dc, struct rmcm *rmcm, struct hubp *hubp, int mpcc_id)
+{
+	if (rmcm && rmcm->funcs) {
+		/* HW blocks 3DLUT writes until FL mode is cleared from HUBP on VUpdate, so
+		 * tear fast load down while the HUBP is still clocked.
+		 */
+		if (hubp && hubp->funcs->hubp_enable_3dlut_fl) {
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
+		}
+
+		dcn42_detach_rmcm(rmcm);
+	}
+
+	dcn42_release_rmcm_from_mpcc(dc, mpcc_id, NULL);
+}
+
+bool dcn42_set_rmcm_luts(struct set_input_transfer_func_params *params)
+{
+	struct dpp *dpp_base = params->dpp;
+	struct hubp *hubp = params->hubp;
+	const struct dc_plane_cm *cm = &params->plane_state->cm;
+	struct rmcm *rmcm = params->rmcm;
+	int rmcm_inst;
+	union rmcm_lut_params m_lut_params = {0};
+	struct dc_3dlut_dma lut3d_dma;
+	bool lut_enable;
+	bool lut_bank_a;
+	bool rval;
+	bool result = true;
+
+	/* No RMCM on this plane, so there is nothing to program */
+	if (!rmcm || !rmcm->funcs) {
+		return true;
+	}
+
+	rmcm_inst = rmcm->inst;
+
+	/* decide LUT bank based on current in use */
+	if (rmcm->funcs->get_lut_mode) {
+		rmcm->funcs->get_lut_mode(rmcm, MCM_LUT_SHAPER, rmcm_inst, &lut_enable, &lut_bank_a);
+		if (!lut_enable) {
+			rmcm->funcs->get_lut_mode(rmcm, MCM_LUT_3DLUT, rmcm_inst, &lut_enable, &lut_bank_a);
+		}
+	} else {
+		lut_enable = false;
+		lut_bank_a = true;
+	}
+
+	/* switch to the next bank */
+	if (lut_enable) {
 		lut_bank_a = !lut_bank_a;
+	}
 
 	/* Shaper */
-	if (cm->flags.bits.shaper_enable) {
+	lut_enable = cm->flags.bits.shaper_enable != 0;
+	if (lut_enable) {
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-
-		if (cm->shaper_func.type == TF_TYPE_HWPWL) {
+		if (cm->shaper_func.type == TF_TYPE_HWPWL)
 			m_lut_params.pwl = &cm->shaper_func.pwl;
-		} else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
 			ASSERT(false);
-			cm_helper_translate_curve_to_hw_format(
-					dc->ctx,
+			rval = cm_helper_translate_curve_to_hw_format(params->plane_state->ctx,
 					&cm->shaper_func,
-					&dpp_base->shaper_params, true);
-			m_lut_params.pwl = &dpp_base->shaper_params;
+					&dpp_base->shaper_params,
+					true);
+			m_lut_params.pwl = rval ? &dpp_base->shaper_params : NULL;
 		}
-		if (m_lut_params.pwl) {
-			if (mpc->funcs->rmcm.populate_lut)
-				mpc->funcs->rmcm.populate_lut(mpc, m_lut_params, lut_bank_a, mpcc_id);
-			if (mpc->funcs->rmcm.program_lut_mode)
-				mpc->funcs->rmcm.program_lut_mode(mpc, !bypass_rmcm_shaper, lut_bank_a, mpcc_id);
-		} else {
-			//RMCM 3dlut won't work without its shaper
-			return false;
+		if (!m_lut_params.pwl) {
+			lut_enable = false;
 		}
+	} else {
+		lut_enable = false;
 	}
 
-	/* 3DLUT */
-	if (!cm->flags.bits.lut3d_dma_enable) {
-		/* RMCM host (non-DMA) 3DLUT load is not implemented; fail fast
-		 * instead of silently reporting success
-		 */
-		BREAK_TO_DEBUGGER();
-		return false;
-	} else {
-		if (!dc_is_rmcm_3dlut_supported(hubp, mpc))
-			return false;
+	if (rmcm->funcs->program_lut_mode)
+		rmcm->funcs->program_lut_mode(rmcm, MCM_LUT_SHAPER, lut_enable, lut_bank_a,
+			CM_LUT_SIZE_NONE, 0, 0, rmcm_inst);
+	if (lut_enable && rmcm->funcs->populate_lut)
+		rmcm->funcs->populate_lut(rmcm, MCM_LUT_SHAPER, m_lut_params, lut_bank_a, rmcm_inst);
 
-		//seems to be only for the MCM
-		mpc_fl_config.enabled			= cm->flags.bits.lut3d_enable != 0u;
-		mpc_fl_config.size	            = cm->lut3d_dma.size;
-		mpc_fl_config.select_lut_bank_a = lut_bank_a;
-		mpc_fl_config.bit_depth		    = 0;
-		mpc_fl_config.hubp_index		= hubp->inst;
-		mpc_fl_config.bias	= cm->lut3d_dma.bias;
-		mpc_fl_config.scale	= cm->lut3d_dma.scale;
+	/* NOTE: Toggling from DMA->Host is not supported atomically as hardware
+	 * blocks writes until 3DLUT FL mode is cleared from HUBP on VUpdate.
+	 * Expectation is either option is used consistently.
+	*/
 
-		//1. power down the block
-		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, false);
+	/* 3DLUT */
+	lut_enable = cm->flags.bits.lut3d_enable != 0;
+	if (lut_enable && cm->flags.bits.lut3d_dma_enable) {
+		/* Fast (DMA) Load Mode */
+		if (rmcm->funcs->program_lut_mode)
+			rmcm->funcs->program_lut_mode(rmcm, MCM_LUT_3DLUT, lut_enable, lut_bank_a,
+				cm->lut3d_dma.size,
+				cm->lut3d_dma.bias, cm->lut3d_dma.scale,
+				rmcm_inst);
 
-		//2. program RMCM - 3dlut reg programming
-		mpc->funcs->rmcm.fl_3dlut_configure(mpc, &mpc_fl_config, mpcc_id);
+		if (rmcm->funcs->program_lut_read_write_control)
+			rmcm->funcs->program_lut_read_write_control(rmcm, MCM_LUT_3DLUT, lut_bank_a, true, rmcm_inst);
+
+		if (rmcm->funcs->update_3dlut_fast_load_select)
+			rmcm->funcs->update_3dlut_fast_load_select(rmcm, rmcm_inst, hubp->inst);
 
 		/* HUBP */
 		if (hubp->funcs->hubp_program_3dlut_fl_config)
 			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma);
 
+		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
+			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp, cm->lut3d_dma.format);
+
 		if (hubp->funcs->hubp_program_3dlut_fl_addr)
 			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr);
 
-		//3. power on the block
-		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, true);
+		if (hubp->funcs->hubp_enable_3dlut_fl) {
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
+		} else {
+			/* GPU memory only supports fast load path */
+			BREAK_TO_DEBUGGER();
+			lut_enable = false;
+			result = false;
+		}
+	} else {
+		/* RMCM has no host load path - its 3DLUT can only be fetched by HUBP fast
+		 * load, so asking for one without DMA is a programming error.
+		 */
+		ASSERT(!lut_enable);
+		result = !lut_enable;
+		lut_enable = false;
+
+		if (rmcm->funcs->program_lut_mode)
+			rmcm->funcs->program_lut_mode(rmcm, MCM_LUT_3DLUT, lut_enable, lut_bank_a,
+				CM_LUT_SIZE_NONE, 0, 0, rmcm_inst);
+
+		if (rmcm->funcs->update_3dlut_fast_load_select)
+			rmcm->funcs->update_3dlut_fast_load_select(rmcm, rmcm_inst, RMCM_FL_HUBP_IDX_NONE);
+
+		/* HUBP */
+		memset(&lut3d_dma, 0, sizeof(lut3d_dma));
+		if (hubp->funcs->hubp_program_3dlut_fl_config)
+			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &lut3d_dma);
+
+		if (hubp->funcs->hubp_enable_3dlut_fl)
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
 	}
 
-	return true;
+	/* Connect to the MPCC only once the LUTs are loaded. Teardown is done by
+	 * disable_rmcm_luts, at plane teardown or when the plane switches back to MCM.
+	 */
+	dcn42_release_rmcm_from_mpcc(params->dc, params->mpcc_id, rmcm);
+
+	if (rmcm->funcs->connect_mpcc)
+		rmcm->funcs->connect_mpcc(rmcm, rmcm_inst, params->mpcc_id);
+
+	return result;
 }
 
-bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
+bool dcn42_set_mcm_luts(struct dc *dc, struct dpp *dpp, struct hubp *hubp,
+				struct hubp *primary_hubp, struct mpc *mpc, int mpcc_id,
+				struct dc_stream_state *stream,
+				struct dc_plane_state *plane_state)
 {
 	bool result;
 
 	/* MCM */
-	result = dcn401_set_mcm_luts(pipe_ctx, plane_state);
-
-	/* RMCM */
-	{
-		struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc;
-		struct hubp *hubp = pipe_ctx->plane_res.hubp;
-		const struct dc_plane_cm *cm = &plane_state->cm;
-		struct mpc *mpc = dc->res_pool->mpc;
-		int mpcc_id = hubp->inst;
-
-		if (cm->flags.bits.rmcm_enable && cm->flags.bits.lut3d_dma_enable) {
-			/* TODO - move RMCM to its own block */
-			dcn42_program_rmcm_luts(
-				hubp,
-				pipe_ctx,
-				cm,
-				mpc,
-				mpcc_id);
-		}
-	}
+	result = dcn401_set_mcm_luts(dc, dpp, hubp, primary_hubp, mpc, mpcc_id,
+			stream, plane_state);
 
 	return result;
 }
@@ -686,7 +783,7 @@ void dcn42_prepare_bandwidth(
 	/* valid C-state watermarks have now been committed to HW, so it
 	 * is safe to vote "allow" to PMFW.
 	 */
-	if (dc->clk_mgr && dc->clk_mgr->funcs && dc->clk_mgr->funcs->notify_cstate_disable)
+	if (dc->clk_mgr->funcs->notify_cstate_disable)
 		dc->clk_mgr->funcs->notify_cstate_disable(dc->clk_mgr, false);
 }
 
@@ -1091,25 +1188,28 @@ void dcn42_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc)
 
 	return;
 }
-void dcn42_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock)
+bool dcn42_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock)
 {
 
 	union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 };
 
 	if (!dc->ctx || !dc->ctx->dmub_srv)
-		return;
+		return false;
 
 	/* Use helper to check PSR/Replay for all streams in context */
 
-	if (!dc->debug.fams2_config.bits.enable && !dc_dmub_srv_is_cursor_offload_enabled(dc)
-		&& !dmub_hw_lock_mgr_does_context_require_lock(dc, context))
-		return;
+	if (lock) {
+		if (!dc->debug.fams2_config.bits.enable && !dc_dmub_srv_is_cursor_offload_enabled(dc)
+			&& !dmub_hw_lock_mgr_does_context_require_lock(dc, context))
+			return false;
+	}
 
 	hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK;
 	hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER;
 	hw_lock_cmd.bits.lock = lock;
 	hw_lock_cmd.bits.should_release = !lock;
 	dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd);
+	return true;
 }
 
 void dcn42_dmub_hw_control_lock_fast(union block_sequence_params *params)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
index c4cfeed45b1905..31b92274a2f5d0 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
@@ -15,15 +15,18 @@ void dcn42_program_cm_hist(
 	struct pipe_ctx *pipe_ctx,
 	const struct dc_plane_state *plane_state);
 
-bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-	const struct dc_plane_state *plane_state);
+bool dcn42_set_mcm_luts(struct dc *dc, struct dpp *dpp, struct hubp *hubp,
+	struct hubp *primary_hubp, struct mpc *mpc, int mpcc_id,
+	struct dc_stream_state *stream,
+	struct dc_plane_state *plane_state);
+
+void dcn42_disable_rmcm_luts(struct dc *dc, struct rmcm *rmcm, struct hubp *hubp, int mpcc_id);
+void dcn42_release_rmcm_from_mpcc(struct dc *dc, int mpcc_id, const struct rmcm *keep);
+void dcn42_disable_mcm_luts(struct mpc *mpc, int mpcc_id);
+void dcn42_trigger_3dlut_dma_load(struct pipe_ctx *pipe_ctx);
+
+bool dcn42_set_rmcm_luts(struct set_input_transfer_func_params *params);
 
-bool dcn42_program_rmcm_luts(
-	struct hubp *hubp,
-	struct pipe_ctx *pipe_ctx,
-	const struct dc_plane_cm *cm,
-	struct mpc *mpc,
-	int mpcc_id);
 void dcn42_hardware_release(struct dc *dc);
 
 void dcn42_prepare_bandwidth(
@@ -40,7 +43,7 @@ void dcn42_hw_block_power_up(struct dc *dc,
 		struct pg_block_update *update_state);
 void dcn42_root_clock_control(struct dc *dc,
 		struct pg_block_update *update_state, bool power_on);
-void dcn42_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock);
+bool dcn42_dmub_hw_control_lock(struct dc *dc, struct dc_state *context, bool lock);
 void dcn42_dmub_hw_control_lock_fast(union block_sequence_params *params);
 void dcn42_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc);
 void dcn42_power_down_on_boot(struct dc *dc);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
index ba12a97a220fe6..a2814941182108 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_init.c
@@ -26,10 +26,11 @@ static const struct hw_sequencer_funcs dcn42_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn401_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
-	.trigger_3dlut_dma_load = dcn401_trigger_3dlut_dma_load,
+	.trigger_3dlut_dma_load = dcn42_trigger_3dlut_dma_load,
 	.enable_accelerated_mode = dce110_enable_accelerated_mode,
 	.enable_timing_synchronization = dcn10_enable_timing_synchronization,
 	.enable_per_frame_crtc_position_reset = dcn10_enable_per_frame_crtc_position_reset,
@@ -42,7 +43,8 @@ static const struct hw_sequencer_funcs dcn42_funcs = {
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn35_disable_plane,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn401_interdependent_update_lock,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn42_prepare_bandwidth,
@@ -127,6 +129,7 @@ static const struct hw_sequencer_funcs dcn42_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn42_private_funcs = {
+	.build_cursor_pos_update_params = dcn401_build_cursor_pos_update_params,
 	.init_pipes = dcn35_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.update_mpcc = dcn42_update_mpcc,
@@ -150,6 +153,9 @@ static const struct hwseq_private_funcs dcn42_private_funcs = {
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
 	.set_mcm_luts = dcn42_set_mcm_luts,
+	.set_rmcm_luts = dcn42_set_rmcm_luts,
+	.disable_mcm_luts = dcn42_disable_mcm_luts,
+	.disable_rmcm_luts = dcn42_disable_rmcm_luts,
 	.program_mall_pipe_config = dcn32_program_mall_pipe_config,
 	.update_mall_sel = dcn32_update_mall_sel,
 	.calculate_dccg_k1_k2_values = NULL,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c
index e549556b967967..0361c520b7d983 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn50/dcn50_hwseq.c
@@ -202,11 +202,17 @@ void dcn50_update_dchubp_dpp(
 	if ((pipe_ctx->update_flags.bits.enable || pipe_ctx->update_flags.bits.opp_changed ||
 		pipe_ctx->update_flags.bits.scaler || viewport_changed == true) &&
 		pipe_ctx->stream->cursor_attributes.address.quad_part != 0) {
-		if (dc->hwss.abort_cursor_offload_update)
-			dc->hwss.abort_cursor_offload_update(dc, pipe_ctx);
+		if (dc_dmub_srv_is_cursor_offload_enabled(dc) && dc->hwss.abort_cursor_offload_update) {
+			struct pipe_ctx *top_pipe = resource_get_otg_master(pipe_ctx);
+
+			if (top_pipe)
+				dc->hwss.abort_cursor_offload_update(dc->ctx->dmub_srv->dmub,
+					pipe_ctx->plane_res.dpp, pipe_ctx->plane_res.hubp,
+					top_pipe->pipe_idx);
+		}
 
 		dc->hwss.set_cursor_attribute(pipe_ctx);
-		dc->hwss.set_cursor_position(pipe_ctx);
+		hwss_program_cursor_position(dc, pipe_ctx);
 
 		if (dc->hwss.set_cursor_sdr_white_level)
 			dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
@@ -309,7 +315,7 @@ void dcn50_update_dchubp_dpp_sequence(struct dc *dc,
 			plane_state->update_bits.input_csc_change ||
 			plane_state->update_bits.color_space_change ||
 			plane_state->update_bits.coeff_reduction_change) {
-		hwss_add_dpp_setup_dpp(seq_state, pipe_ctx);
+		hwss_add_dpp_setup_dpp(seq_state, dpp, plane_state);
 
 		/* Step 8: DPP cursor matrix setup */
 		if (dpp->funcs->set_cursor_matrix) {
@@ -319,7 +325,7 @@ void dcn50_update_dchubp_dpp_sequence(struct dc *dc,
 
 		/* Step 9: DPP program bias and scale */
 		if (dpp->funcs->dpp_program_bias_and_scale)
-			hwss_add_dpp_program_bias_and_scale(seq_state, pipe_ctx);
+			hwss_add_dpp_program_bias_and_scale(seq_state, dpp, plane_state);
 	}
 
 	/* Step 10: MPCC updates */
@@ -367,8 +373,7 @@ void dcn50_update_dchubp_dpp_sequence(struct dc *dc,
 		hwss_add_set_cursor_position(seq_state, dc, pipe_ctx);
 
 		/* Step 15: Cursor SDR white level */
-		if (dc->hwss.set_cursor_sdr_white_level)
-			hwss_add_set_cursor_sdr_white_level(seq_state, dc, pipe_ctx);
+		hwss_add_set_cursor_sdr_white_level(seq_state, pipe_ctx);
 	}
 
 	/* Step 16: Gamut remap and output CSC */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
index 61ad6efa7a7456..ef1c10bc9bfaa2 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.c
@@ -31,6 +31,7 @@
 #include "dce110/dce110_hwseq.h"
 #include "dcn32/dcn32_hwseq.h"
 #include "dcn401/dcn401_hwseq.h"
+#include "dcn42/dcn42_hwseq.h"
 #include "dcn50/dcn50_hwseq.h"
 #include "dcn60_hwseq.h"
 #include "dcn401/dcn401_resource.h"
@@ -52,6 +53,127 @@
 #define FN(reg_name, field_name) \
 	hws->shifts->field_name, hws->masks->field_name
 
+bool dcn60_set_rmcm_luts(struct set_input_transfer_func_params *params)
+{
+	struct dpp *dpp_base = params->dpp;
+	struct hubp *hubp = params->hubp;
+	const struct dc_plane_cm *cm = &params->plane_state->cm;
+	struct rmcm *rmcm = params->rmcm;
+	int rmcm_inst;
+	union rmcm_lut_params m_lut_params = {0};
+	struct dc_3dlut_dma lut3d_dma;
+	bool lut_enable;
+	/* DCN60 has a single LUT SRAM per RMCM - always bank A. */
+	bool rval;
+	bool result = true;
+
+	/* No RMCM on this plane, so there is nothing to program */
+	if (!rmcm || !rmcm->funcs) {
+		return true;
+	}
+
+	rmcm_inst = rmcm->inst;
+
+	/* Shaper */
+	lut_enable = cm->flags.bits.shaper_enable != 0;
+	if (lut_enable) {
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+		if (cm->shaper_func.type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &cm->shaper_func.pwl;
+		else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+			ASSERT(false);
+			rval = cm_helper_translate_curve_to_hw_format(params->plane_state->ctx,
+					&cm->shaper_func,
+					&dpp_base->shaper_params,
+					true);
+			m_lut_params.pwl = rval ? &dpp_base->shaper_params : NULL;
+		}
+		if (!m_lut_params.pwl) {
+			lut_enable = false;
+		}
+	} else {
+		lut_enable = false;
+	}
+
+	if (rmcm->funcs->program_lut_mode)
+		rmcm->funcs->program_lut_mode(rmcm, MCM_LUT_SHAPER, lut_enable, true,
+			CM_LUT_SIZE_NONE, 0, 0, rmcm_inst);
+	if (lut_enable && rmcm->funcs->populate_lut)
+		rmcm->funcs->populate_lut(rmcm, MCM_LUT_SHAPER, m_lut_params, true, rmcm_inst);
+
+	/* NOTE: Toggling from DMA->Host is not supported atomically as hardware
+	 * blocks writes until 3DLUT FL mode is cleared from HUBP on VUpdate.
+	 * Expectation is either option is used consistently.
+	*/
+
+	/* 3DLUT */
+	lut_enable = cm->flags.bits.lut3d_enable != 0;
+	if (lut_enable && cm->flags.bits.lut3d_dma_enable) {
+		/* Fast (DMA) Load Mode */
+		if (rmcm->funcs->program_lut_mode)
+			rmcm->funcs->program_lut_mode(rmcm, MCM_LUT_3DLUT, lut_enable, true,
+				cm->lut3d_dma.size,
+				cm->lut3d_dma.bias, cm->lut3d_dma.scale,
+				rmcm_inst);
+
+		if (rmcm->funcs->program_lut_read_write_control)
+			rmcm->funcs->program_lut_read_write_control(rmcm, MCM_LUT_3DLUT, true, true, rmcm_inst);
+
+		if (rmcm->funcs->update_3dlut_fast_load_select)
+			rmcm->funcs->update_3dlut_fast_load_select(rmcm, rmcm_inst, hubp->inst);
+
+		/* HUBP */
+		if (hubp->funcs->hubp_program_3dlut_fl_config)
+			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma);
+
+		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
+			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp, cm->lut3d_dma.format);
+
+		if (hubp->funcs->hubp_program_3dlut_fl_addr)
+			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr);
+
+		if (hubp->funcs->hubp_enable_3dlut_fl) {
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
+		} else {
+			/* GPU memory only supports fast load path */
+			BREAK_TO_DEBUGGER();
+			result = false;
+		}
+	} else {
+		/* RMCM 3DLUT is fetched only by HUBP fast load, so a host load request is a
+		 * programming error.
+		 */
+		ASSERT(!lut_enable);
+		result = !lut_enable;
+		lut_enable = false;
+
+		if (rmcm->funcs->program_lut_mode)
+			rmcm->funcs->program_lut_mode(rmcm, MCM_LUT_3DLUT, lut_enable, true,
+				CM_LUT_SIZE_NONE, 0, 0, rmcm_inst);
+
+		if (rmcm->funcs->update_3dlut_fast_load_select)
+			rmcm->funcs->update_3dlut_fast_load_select(rmcm, rmcm_inst, RMCM_FL_HUBP_IDX_NONE);
+
+		/* HUBP */
+		memset(&lut3d_dma, 0, sizeof(lut3d_dma));
+		if (hubp->funcs->hubp_program_3dlut_fl_config)
+			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &lut3d_dma);
+
+		if (hubp->funcs->hubp_enable_3dlut_fl)
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
+	}
+
+	/* Connect to the MPCC only once the LUTs are loaded. Teardown is done by
+	 * disable_rmcm_luts, at plane teardown or when the plane switches back to MCM.
+	 */
+	dcn42_release_rmcm_from_mpcc(params->dc, params->mpcc_id, rmcm);
+
+	if (rmcm->funcs->connect_mpcc)
+		rmcm->funcs->connect_mpcc(rmcm, rmcm_inst, params->mpcc_id);
+
+	return result;
+}
+
 static void dcn60_build_audio_output(
 	struct dc_state *state,
 	const struct pipe_ctx *pipe_ctx,
@@ -291,7 +413,9 @@ enum dc_status dcn60_apply_single_controller_ctx_to_hw(
 			dc_is_virtual_signal(pipe_ctx->stream->signal)))
 			dc->link_srv->set_dsc_enable(pipe_ctx, true);
 	}
-	if (!stream->dpms_off)
+	if (!stream->dpms_off &&
+	    !(link->connector_signal == SIGNAL_TYPE_EDP &&
+	      link->forced_psr_active))
 		dc->link_srv->set_dpms_on(context, pipe_ctx);
 
 	/* DCN3.1 FPGA Workaround
@@ -310,7 +434,9 @@ enum dc_status dcn60_apply_single_controller_ctx_to_hw(
 	 * is constructed with the same sink). Make sure not to override
 	 * and link programming on the main.
 	 */
-	if (dc_state_get_pipe_subvp_type(context, pipe_ctx) != SUBVP_PHANTOM) {
+	if (dc_state_get_pipe_subvp_type(context, pipe_ctx) != SUBVP_PHANTOM &&
+	    !(link->connector_signal == SIGNAL_TYPE_EDP &&
+	      link->forced_psr_active)) {
 		pipe_ctx->stream->link->psr_settings.psr_feature_enabled = false;
 		pipe_ctx->stream->link->replay_settings.replay_feature_enabled = false;
 	}
@@ -754,6 +880,12 @@ void dcn60_init_hw(struct dc *dc)
 	if (dc->res_pool->hubbub->funcs->set_request_limit && dc->config.sdpif_request_limit_words_per_umc > 0)
 		dc->res_pool->hubbub->funcs->set_request_limit(dc->res_pool->hubbub, dc->ctx->dc_bios->vram_info.num_chans, dc->config.sdpif_request_limit_words_per_umc);
 
+	if (dc->res_pool->hubbub->funcs->override_utm_client_qc_profile && dc->debug.override_utm_client_qc_profile) {
+		dc->res_pool->hubbub->funcs->override_utm_client_qc_profile(dc->res_pool->hubbub, dc->debug.utm_client_qc_profiles[0], 0);
+		dc->res_pool->hubbub->funcs->override_utm_client_qc_profile(dc->res_pool->hubbub, dc->debug.utm_client_qc_profiles[1], 1);
+		dc->res_pool->hubbub->funcs->override_utm_client_qc_profile(dc->res_pool->hubbub, dc->debug.utm_client_qc_profiles[2], 2);
+	}
+
 	// Get DMCUB capabilities
 	if (dc->ctx->dmub_srv) {
 		dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv);
@@ -781,23 +913,17 @@ void dcn60_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
 		pipe_ctx->plane_res.dpp, attributes);
 }
 
-void dcn60_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe)
+void dcn60_update_cursor_offload_pipe(struct dmub_srv *dmub, uint32_t stream_idx,
+		uint8_t pipe_idx, const struct dpp *dpp, const struct hubp *hubp)
 {
-	volatile struct dmub_cursor_offload_v1 *cs = dc->ctx->dmub_srv->dmub->cursor_offload_v1;
-	const struct pipe_ctx *top_pipe = resource_get_otg_master(pipe);
-	const struct hubp *hubp = pipe->plane_res.hubp;
-	const struct dpp *dpp = pipe->plane_res.dpp;
+	volatile struct dmub_cursor_offload_v1 *cs = dmub->cursor_offload_v1;
 	volatile struct dmub_cursor_offload_pipe_data_dcn60_v1 *p;
-	uint32_t stream_idx, write_idx, payload_idx;
-
-	if (!top_pipe || !hubp || !dpp)
-		return;
+	uint32_t write_idx, payload_idx;
 
-	stream_idx = top_pipe->pipe_idx;
 	write_idx = cs->offload_streams[stream_idx].write_idx + 1; /*  new payload (+1) */
 	payload_idx = write_idx % ARRAY_SIZE(cs->offload_streams[stream_idx].payloads);
 
-	p = &cs->offload_streams[stream_idx].payloads[payload_idx].pipe_data[pipe->pipe_idx].dcn60;
+	p = &cs->offload_streams[stream_idx].payloads[payload_idx].pipe_data[pipe_idx].dcn60;
 
 	p->CURSOR0_0_CURSOR_SURFACE_ADDRESS = hubp->att.SURFACE_ADDR;
 	p->CURSOR0_0_CURSOR_SURFACE_ADDRESS_HIGH = hubp->att.SURFACE_ADDR_HIGH;
@@ -834,7 +960,7 @@ void dcn60_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe
 	p->HUBPREQ0_CURSOR_SETTINGS__CURSOR0_CHUNK_HDL_ADJUST = hubp->att.settings.bits.chunk_hdl_adjust;
 	p->HUBPREQ0_CURSOR_SETTINGS__FORCE_CURSOR_TO_DISP_PREF = hubp->att.settings.bits.force_cursor_to_disp_pref;
 
-	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask |= (1u << pipe->pipe_idx);
+	cs->offload_streams[stream_idx].payloads[payload_idx].pipe_mask |= (1u << pipe_idx);
 }
 
 /**
@@ -865,7 +991,7 @@ static struct timing_generator *dcn60_get_ref_tg_for_hubbub_probe(
  * @dc:             DC structure
  * @context:        Committed dc state to resolve streams from
  * @probe:          Probe state to build sequence for
- * @status:         Perfmon status to update with probe results
+ * @status:         Prior perfmon status, then the result sink for the sequence
  * @block_sequence: Block sequence to append steps to
  * @num_steps:      Number of steps in the block sequence
  *
@@ -884,33 +1010,38 @@ static void dcn60_build_hubbub_perfmon_sequence(
 	uint32_t refclk_mhz = dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000;
 	struct timing_generator *ref_tg = dcn60_get_ref_tg_for_hubbub_probe(context);
 	struct block_sequence_state seq_state = { .steps = block_sequence, .num_steps = num_steps };
-	uint32_t duration_ns = 0;
+	bool was_measuring = status->measuring;
 
 	if (!hubbub || !hubbub->funcs || !hubbub->funcs->perfmon.reset)
 		return;
 
-	status->type = probe->type;
-
 	if (probe->target_state == DC_PROBE_NOT_MEASURING) {
 		hwss_add_hubbub_perfmon_reset(&seq_state, hubbub);
 		return;
 	}
 
-	if (probe->target_state != DC_PROBE_MEASURED || !ref_tg)
-		return;
+	if (probe->target_state == DC_PROBE_MEASURING) {
+		if (probe->type != DC_PROBE_URGENT_ASSERTION_COUNT ||
+				!hubbub->funcs->perfmon.start_measuring_urgent_assertion_count ||
+				!hubbub->funcs->perfmon.get_urgent_assertion_count)
+			return;
 
-	/* Peak BW needs a single timing group. The out-of-order counter spans one
-	 * prefetch window, which is meaningless when streams in separate timing
-	 * groups have non-overlapping prefetch windows. */
-	if (probe->type == DC_PROBE_PEAK_MEM_BW) {
-		int group_size = context->stream_status[0].timing_sync_info.group_size;
+		if (!was_measuring) {
+			hwss_add_hubbub_perfmon_reset(&seq_state, hubbub);
+			hwss_add_hubbub_perfmon_start_urgent_assertion_count(&seq_state, hubbub);
+		}
 
-		if (group_size != context->stream_count)
-			return;
+		hwss_add_hubbub_perfmon_get_urgent_assertion_count(&seq_state, hubbub,
+				refclk_mhz, &status->u.urgent_assertion_count);
+		return;
 	}
 
+	if (probe->target_state != DC_PROBE_MEASURED || !ref_tg)
+		return;
+
 	switch (probe->type) {
 	case DC_PROBE_PEAK_MEM_BW:
+	case DC_PROBE_PEAK_MEM_BW_STRESSED:
 		/* Start at the vblank edge and stop at the next vactive so the counter
 		 * spans exactly one prefetch window, capturing prefetch traffic only. */
 		if (!hubbub->funcs->perfmon.arm_measuring_out_of_order_bandwidth ||
@@ -920,13 +1051,31 @@ static void dcn60_build_hubbub_perfmon_sequence(
 
 		hwss_add_hubbub_perfmon_reset(&seq_state, hubbub);
 		hwss_add_hubbub_perfmon_arm_out_of_order_bw(&seq_state, hubbub);
+
+		if (probe->type == DC_PROBE_PEAK_MEM_BW_STRESSED) {
+			struct dc_plane_state *plane = (context->stream_status[0].plane_count > 0) ?
+					context->stream_status[0].plane_states[0] : NULL;
+
+			if (dc->res_pool->lsdma_scratch.buffer && plane) {
+				unsigned int surface_bytes = plane->plane_size.surface_pitch *
+						plane->plane_size.surface_size.height;
+				unsigned int copy_bytes = (surface_bytes < dc->res_pool->lsdma_scratch.size) ?
+						surface_bytes : dc->res_pool->lsdma_scratch.size;
+
+				hwss_add_lsdma_send_pio_copy(&seq_state, dc->ctx->dmub_srv,
+						(uint64_t)plane->address.grph.addr.quad_part,
+						(uint64_t)dc->res_pool->lsdma_scratch.pa,
+						copy_bytes, /* overlap_disable */ 1);
+			}
+		}
+
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VACTIVE);
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VBLANK);
 		hwss_add_hubbub_perfmon_start_out_of_order_bw(&seq_state, hubbub);
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VACTIVE);
 		hwss_add_hubbub_perfmon_get_out_of_order_bw(&seq_state, hubbub,
-				refclk_mhz, &status->u.bandwidth_mbps, &duration_ns);
-		break;
+				refclk_mhz, &status->u.bandwidth_mbps, NULL);
+		return;
 
 	case DC_PROBE_AVG_MEM_BW:
 		/* In-order counter accumulates over a full frame, so no timing group
@@ -942,8 +1091,8 @@ static void dcn60_build_hubbub_perfmon_sequence(
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VACTIVE);
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VBLANK);
 		hwss_add_hubbub_perfmon_get_in_order_bw(&seq_state, hubbub,
-				refclk_mhz, 0, &status->u.bandwidth_mbps, &duration_ns);
-		break;
+				refclk_mhz, 0, &status->u.bandwidth_mbps, NULL);
+		return;
 
 	case DC_PROBE_MEM_LATENCY:
 		if (!hubbub->funcs->perfmon.start_measuring_memory_latencies ||
@@ -958,7 +1107,7 @@ static void dcn60_build_hubbub_perfmon_sequence(
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VBLANK);
 		hwss_add_hubbub_perfmon_get_memory_latencies(&seq_state, hubbub,
 				refclk_mhz, &status->u.latency);
-		break;
+		return;
 
 	case DC_PROBE_URGENT_ASSERTION_COUNT:
 		if (!hubbub->funcs->perfmon.start_measuring_urgent_assertion_count ||
@@ -973,7 +1122,7 @@ static void dcn60_build_hubbub_perfmon_sequence(
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VBLANK);
 		hwss_add_hubbub_perfmon_get_urgent_assertion_count(&seq_state, hubbub,
 				refclk_mhz, &status->u.urgent_assertion_count);
-		break;
+		return;
 
 	case DC_PROBE_PREFETCH_DATA_SIZE:
 		if (!hubbub->funcs->perfmon.start_measuring_prefetch_data_size ||
@@ -988,7 +1137,7 @@ static void dcn60_build_hubbub_perfmon_sequence(
 		hwss_add_tg_wait_for_state(&seq_state, ref_tg, CRTC_STATE_VBLANK);
 		hwss_add_hubbub_perfmon_get_prefetch_data_size(&seq_state, hubbub,
 				&status->u.prefetch_data_size);
-		break;
+		return;
 
 	case DC_PROBE_URGENT_RAMP_LATENCY:
 		/* Requires caller-supplied window params not available in probe model. */
@@ -1004,10 +1153,24 @@ static void dcn60_build_hubbub_perfmon_sequence(
  * @status: result sink whose u was written by the GET BLS step during execute
  * @probe: current probe state used to determine measurement type and validity
  */
-static void dcn60_update_probe_status(struct dc_probe_status *status)
+static void dcn60_update_probe_status(
+		struct dc_probe_status *status,
+		const struct dc_probe_state *probe)
 {
-	switch (status->type) {
+	struct dc_probe_status result = *status;
+
+	memset(status, 0, sizeof(*status));
+
+	if (probe->target_state == DC_PROBE_NOT_MEASURING)
+		return;
+
+	status->type = probe->type;
+	status->measuring = probe->target_state == DC_PROBE_MEASURING;
+	status->u = result.u;
+
+	switch (probe->type) {
 	case DC_PROBE_PEAK_MEM_BW:
+	case DC_PROBE_PEAK_MEM_BW_STRESSED:
 	case DC_PROBE_AVG_MEM_BW:
 		/* Zero bandwidth means the counter did not fire — treat as invalid. */
 		status->valid = (status->u.bandwidth_mbps != 0);
@@ -1032,6 +1195,7 @@ static bool is_probe_measurement_type_for_hubbub(enum dc_probe_type type)
 {
 	switch (type) {
 	case DC_PROBE_PEAK_MEM_BW:
+	case DC_PROBE_PEAK_MEM_BW_STRESSED:
 	case DC_PROBE_AVG_MEM_BW:
 	case DC_PROBE_MEM_LATENCY:
 	case DC_PROBE_URGENT_ASSERTION_COUNT:
@@ -1061,11 +1225,11 @@ void dcn60_program_perfmon(struct dc *dc, struct dc_state *context)
 		return;
 
 	context->block_sequence_steps = 0;
-	memset(context->probe_status, 0, sizeof(context->probe_status));
 
 	for (i = 0; i < context->probe_count; i++) {
 		if (is_probe_measurement_type_for_hubbub(context->probes[i].type))
-			dcn60_build_hubbub_perfmon_sequence(dc, context, &context->probes[i],
+			dcn60_build_hubbub_perfmon_sequence(dc, context,
+					&context->probes[i],
 					&context->probe_status[i],
 					context->block_sequence,
 					&context->block_sequence_steps);
@@ -1074,7 +1238,7 @@ void dcn60_program_perfmon(struct dc *dc, struct dc_state *context)
 	hwss_execute_sequence(dc, context->block_sequence, context->block_sequence_steps);
 
 	for (i = 0; i < context->probe_count; i++)
-		dcn60_update_probe_status(&context->probe_status[i]);
+		dcn60_update_probe_status(&context->probe_status[i], &context->probes[i]);
 }
 
 static bool dcn60_has_active_memory_request(const struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h
index b57d22d310b08f..b880988ff1fc61 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_hwseq.h
@@ -24,9 +24,12 @@ enum dc_status dcn60_apply_single_controller_ctx_to_hw(
 
 void dcn60_init_hw(struct dc *dc);
 void dcn60_set_cursor_attribute(struct pipe_ctx *pipe_ctx);
-void dcn60_update_cursor_offload_pipe(struct dc *dc, const struct pipe_ctx *pipe);
+void dcn60_update_cursor_offload_pipe(struct dmub_srv *dmub, uint32_t stream_idx,
+	uint8_t pipe_idx, const struct dpp *dpp, const struct hubp *hubp);
 
 void dcn60_program_perfmon(struct dc *dc, struct dc_state *context);
 bool dcn60_apply_idle_power_optimizations(struct dc *dc, bool enable);
 
+bool dcn60_set_rmcm_luts(struct set_input_transfer_func_params *params);
+
 #endif /* __DC_HWSS_DCN60_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c
index 1c908964153b7a..c89051a542cb7e 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn60/dcn60_init.c
@@ -25,10 +25,11 @@ static const struct hw_sequencer_funcs dcn60_funcs = {
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.post_unlock_program_front_end = dcn50_post_unlock_program_front_end,
 	.update_plane_addr = dcn20_update_plane_addr,
+	.prepare_plane_addr_update = dcn20_prepare_plane_addr_update,
 	.update_dchub = dcn10_update_dchub,
 	.update_pending_status = dcn10_update_pending_status,
 	.program_output_csc = dcn20_program_output_csc,
-	.trigger_3dlut_dma_load = dcn401_trigger_3dlut_dma_load,
+	.trigger_3dlut_dma_load = dcn42_trigger_3dlut_dma_load,
 	.enable_accelerated_mode = dce110_enable_accelerated_mode,
 	.enable_timing_synchronization = dcn10_enable_timing_synchronization,
 	.enable_per_frame_crtc_position_reset = dcn10_enable_per_frame_crtc_position_reset,
@@ -42,7 +43,8 @@ static const struct hw_sequencer_funcs dcn60_funcs = {
 	.disable_audio_stream = dce110_disable_audio_stream,
 	.disable_plane = dcn20_disable_plane,
 	.disable_plane_sequence = dcn401_disable_plane_sequence,
-	.pipe_control_lock = dcn20_pipe_control_lock,
+	.build_pipe_control_lock_sequence = dcn20_build_pipe_control_lock_sequence,
+	.tg_lock = dcn20_tg_lock,
 	.interdependent_update_lock = dcn401_interdependent_update_lock,
 	.cursor_lock = dcn10_cursor_lock,
 	.prepare_bandwidth = dcn401_prepare_bandwidth,
@@ -126,6 +128,7 @@ static const struct hw_sequencer_funcs dcn60_funcs = {
 };
 
 static const struct hwseq_private_funcs dcn60_private_funcs = {
+	.build_cursor_pos_update_params = dcn401_build_cursor_pos_update_params,
 	.init_pipes = dcn10_init_pipes,
 	.plane_atomic_disconnect = dcn10_plane_atomic_disconnect,
 	.plane_atomic_disconnect_sequence = dcn401_plane_atomic_disconnect_sequence,
@@ -146,7 +149,7 @@ static const struct hwseq_private_funcs dcn60_private_funcs = {
 	.init_blank = dcn32_init_blank,
 	.disable_vga = NULL,
 	.bios_golden_init = dcn10_bios_golden_init,
-	.plane_atomic_disable = dcn20_plane_atomic_disable,
+	.plane_atomic_disable = dcn35_plane_atomic_disable,
 	.plane_atomic_power_down = dcn401_plane_atomic_power_down,
 	.plane_atomic_power_down_sequence = dcn401_plane_atomic_power_down_sequence,
 	.enable_power_gating_plane = NULL,
@@ -161,6 +164,9 @@ static const struct hwseq_private_funcs dcn60_private_funcs = {
 	.set_hdr_multiplier_sequence = dcn401_set_hdr_multiplier_sequence,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
 	.set_mcm_luts = dcn42_set_mcm_luts,
+	.set_rmcm_luts = dcn60_set_rmcm_luts,
+	.disable_mcm_luts = dcn42_disable_mcm_luts,
+	.disable_rmcm_luts = dcn42_disable_rmcm_luts,
 	.program_mall_pipe_config = NULL,
 	.update_mall_sel = NULL,
 	.calculate_dccg_k1_k2_values = NULL,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
index cdf2562a9a4021..7c5f88bf44c723 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
@@ -46,6 +46,7 @@ struct resource_pool;
 struct dc_phy_addr_space_config;
 struct dc_virtual_addr_space_config;
 struct dpp;
+struct rmcm;
 struct dce_hwseq;
 struct link_resource;
 struct dc_dmub_cmd;
@@ -53,6 +54,7 @@ struct pg_block_update;
 struct drr_params;
 struct dc_underflow_debug_data;
 struct dsc_optc_config;
+struct dscl_prog_data;
 struct vm_system_aperture_param;
 struct stream_encoder;
 struct hpo_dp_stream_encoder;
@@ -66,10 +68,17 @@ struct subvp_pipe_control_lock_fast_params {
 	bool subvp_immediate_flip;
 };
 
-struct pipe_control_lock_params {
+struct tg_lock_params {
 	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct timing_generator *tg;
 	bool lock;
+	bool use_dmub_inbox1;
+	bool triplebuffer_flips;
+};
+
+struct tg_3dlut_wa_unlock_params {
+	struct timing_generator *tg;
+	struct hubp *hubp;
 };
 
 struct set_flip_control_gsl_params {
@@ -78,19 +87,27 @@ struct set_flip_control_gsl_params {
 };
 
 struct program_triplebuffer_params {
-	const struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct hubp *hubp;
 	bool enableTripleBuffer;
 };
 
 struct update_plane_addr_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct hubp *hubp;
+	struct dc_plane_address address;
+	bool flip_immediate;
+	bool dcc;
 };
 
 struct set_input_transfer_func_params {
 	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct dpp *dpp;
+	struct hubp *hubp;
+	struct hubp *primary_hubp;
+	struct mpc *mpc;
+	struct rmcm *rmcm;
+	int mpcc_id;
+	struct dc_stream_state *stream;
+	struct input_pixel_processor *ipp;
 	struct dc_plane_state *plane_state;
 };
 
@@ -120,7 +137,7 @@ struct update_info_frame_params {
 };
 
 struct program_manual_trigger_params {
-	struct pipe_ctx *pipe_ctx;
+	struct timing_generator *tg;
 };
 
 struct send_dmcub_cmd_params {
@@ -129,12 +146,24 @@ struct send_dmcub_cmd_params {
 	enum dm_dmub_wait_type wait_type;
 };
 
+struct lsdma_send_pio_copy_params {
+	struct dc_dmub_srv *dc_dmub_srv;
+	uint64_t src_addr;
+	uint64_t dst_addr;
+	uint32_t byte_count;
+	uint32_t overlap_disable;
+};
+
 struct setup_dpp_params {
-	struct pipe_ctx *pipe_ctx;
+	struct dpp *dpp;
+	enum surface_pixel_format format;
+	struct dc_csc_transform input_csc_color_matrix;
+	enum dc_color_space color_space;
 };
 
 struct program_bias_and_scale_params {
-	struct pipe_ctx *pipe_ctx;
+	struct dpp *dpp;
+	struct dc_bias_and_scale bias_and_scale;
 };
 
 struct set_output_transfer_func_params {
@@ -146,12 +175,13 @@ struct set_output_transfer_func_params {
 	const struct dc_stream_state *stream;
 };
 struct program_upsp_params {
-	struct pipe_ctx *pipe_ctx;
+	struct dpp *dpp;
+	const struct dscl_prog_data *dscl_prog_data;
 };
 
 struct update_visual_confirm_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct mpc *mpc;
+	struct tg_color *color;
 	int mpcc_id;
 };
 
@@ -182,8 +212,7 @@ struct subvp_save_surf_addr {
 };
 
 struct wait_for_dcc_meta_propagation_params {
-	const struct dc *dc;
-	const struct pipe_ctx *top_pipe_to_program;
+	uint32_t delay;
 };
 
 struct dmub_hw_control_lock_fast_params {
@@ -223,16 +252,6 @@ struct hubp_wait_pipe_read_start_params {
 	struct hubp *hubp;
 };
 
-struct apply_update_flags_for_phantom_params {
-	struct pipe_ctx *pipe_ctx;
-};
-
-struct update_phantom_vp_position_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
-	struct dc_state *context;
-};
-
 struct set_odm_combine_params {
 	struct timing_generator *tg;
 	int opp_inst[MAX_PIPES];
@@ -297,7 +316,8 @@ struct dsc_calculate_and_set_config_params {
 };
 
 struct dsc_enable_with_opp_params {
-	struct pipe_ctx *pipe_ctx;
+	struct display_stream_compressor *dsc;
+	int opp_inst;
 };
 
 struct program_tg_params {
@@ -337,6 +357,17 @@ struct tg_set_gsl_source_select_params {
 	uint32_t gsl_ready_signal;
 };
 
+struct pipe_control_lock_params {
+	bool lock;
+	struct hubp *hubps_to_wait_for_flip[MAX_PIPES];
+	bool gsl_lock;
+	struct tg_set_gsl_params gsl;
+	struct tg_set_gsl_source_select_params gsl_source_select;
+	struct tg_lock_params tg_lock;
+	bool tg_3dlut_wa_unlock;
+	struct tg_3dlut_wa_unlock_params tg_3dlut_wa_unlock_params;
+};
+
 struct setup_vupdate_interrupt_params {
 	struct dc *dc;
 	struct pipe_ctx *pipe_ctx;
@@ -385,8 +416,7 @@ struct opp_program_fmt_params {
 
 struct opp_program_bit_depth_reduction_params {
 	struct output_pixel_processor *opp;
-	bool use_default_params;
-	struct pipe_ctx *pipe_ctx;
+	struct bit_depth_reduction_params bit_depth_params;
 };
 
 struct opp_set_disp_pattern_generator_params {
@@ -482,7 +512,6 @@ struct tg_enable_crtc_params {
 
 struct hubp_wait_flip_pending_params {
 	struct hubp *hubp;
-	unsigned int timeout_us;
 	unsigned int polling_interval_us;
 };
 
@@ -807,11 +836,6 @@ struct hubp_program_mcache_id_and_split_coordinate_params {
 	struct mcache_regs_struct *mcache_regs;
 };
 
-struct abort_cursor_offload_update_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
-};
-
 struct cursor_lock_params {
 	struct dc *dc;
 	struct pipe_ctx *pipe_ctx;
@@ -819,13 +843,18 @@ struct cursor_lock_params {
 };
 
 struct setup_periodic_interrupt_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct timing_generator *tg;
+	uint32_t start_line;
+	uint32_t end_line;
 };
 
 struct send_cursor_info_to_dmu_params {
-	struct pipe_ctx *pipe_ctx;
-	int pipe_idx;
+	const struct dc_context *ctx;
+	uint8_t pipe_idx;
+	struct hubp *hubp;
+	struct dpp *dpp;
+	uint8_t otg_inst;
+	uint8_t panel_inst;
 };
 
 struct set_cursor_attribute_params {
@@ -845,12 +874,15 @@ struct dpp_set_cursor_attributes_params {
 
 struct set_cursor_position_params {
 	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct hubp *hubp;
+	struct dpp *dpp;
+	struct dc_cursor_position pos;
+	struct dc_cursor_mi_param param;
 };
 
 struct set_cursor_sdr_white_level_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct dpp *dpp;
+	struct dpp_cursor_attributes attr;
 };
 
 struct program_output_csc_params {
@@ -871,18 +903,32 @@ struct phantom_hubp_post_enable_params {
 };
 
 struct begin_cursor_offload_update_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct dmub_srv *dmub;
+	struct dpp *dpp;
+	struct hubp *hubp;
+	uint32_t stream_idx;
 };
 
 struct update_cursor_offload_pipe_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct dmub_srv *dmub;
+	const struct dpp *dpp;
+	const struct hubp *hubp;
+	uint32_t stream_idx;
+	uint8_t pipe_idx;
 };
 
 struct commit_cursor_offload_update_params {
-	struct dc *dc;
-	struct pipe_ctx *pipe_ctx;
+	struct dmub_srv *dmub;
+	struct dpp *dpp;
+	struct hubp *hubp;
+	uint32_t stream_idx;
+};
+
+struct abort_cursor_offload_update_params {
+	struct dmub_srv *dmub;
+	struct dpp *dpp;
+	struct hubp *hubp;
+	uint32_t stream_idx;
 };
 
 struct stream_enc_update_hdmi_info_packets_params {
@@ -1020,7 +1066,8 @@ struct link_set_dpms_on_params {
 union block_sequence_params {
 	struct update_plane_addr_params update_plane_addr_params;
 	struct subvp_pipe_control_lock_fast_params subvp_pipe_control_lock_fast_params;
-	struct pipe_control_lock_params pipe_control_lock_params;
+	struct tg_lock_params tg_lock_params;
+	struct tg_3dlut_wa_unlock_params tg_3dlut_wa_unlock_params;
 	struct set_flip_control_gsl_params set_flip_control_gsl_params;
 	struct program_triplebuffer_params program_triplebuffer_params;
 	struct set_input_transfer_func_params set_input_transfer_func_params;
@@ -1030,6 +1077,7 @@ union block_sequence_params {
 	struct update_info_frame_params update_info_frame_params;
 	struct program_manual_trigger_params program_manual_trigger_params;
 	struct send_dmcub_cmd_params send_dmcub_cmd_params;
+	struct lsdma_send_pio_copy_params lsdma_send_pio_copy_params;
 	struct setup_dpp_params setup_dpp_params;
 	struct program_bias_and_scale_params program_bias_and_scale_params;
 	struct set_output_transfer_func_params set_output_transfer_func_params;
@@ -1046,8 +1094,6 @@ union block_sequence_params {
 	struct control_cm_hist_params control_cm_hist_params;
 	struct program_cursor_update_now_params program_cursor_update_now_params;
 	struct hubp_wait_pipe_read_start_params hubp_wait_pipe_read_start_params;
-	struct apply_update_flags_for_phantom_params apply_update_flags_for_phantom_params;
-	struct update_phantom_vp_position_params update_phantom_vp_position_params;
 	struct set_odm_combine_params set_odm_combine_params;
 	struct set_odm_bypass_params set_odm_bypass_params;
 	struct opp_pipe_clock_control_params opp_pipe_clock_control_params;
@@ -1194,7 +1240,8 @@ union block_sequence_params {
 
 enum block_sequence_func {
 	DMUB_SUBVP_PIPE_CONTROL_LOCK_FAST = 0,
-	OPTC_PIPE_CONTROL_LOCK,
+	TG_LOCK,
+	TG_3DLUT_WA_UNLOCK,
 	HUBP_SET_FLIP_CONTROL_GSL,
 	HUBP_PROGRAM_TRIPLEBUFFER,
 	HUBP_UPDATE_PLANE_ADDR,
@@ -1207,6 +1254,7 @@ enum block_sequence_func {
 	HUBP_SET_DMDATA_ATTRIBUTES,
 	OPTC_PROGRAM_MANUAL_TRIGGER,
 	DMUB_SEND_DMCUB_CMD,
+	LSDMA_SEND_PIO_COPY,
 	DPP_SETUP_DPP,
 	DPP_PROGRAM_BIAS_AND_SCALE,
 	DPP_SET_OUTPUT_TRANSFER_FUNC,
@@ -1224,8 +1272,6 @@ enum block_sequence_func {
 	DPP_PROGRAM_CM_HIST,
 	PROGRAM_CURSOR_UPDATE_NOW,
 	HUBP_WAIT_PIPE_READ_START,
-	HWS_APPLY_UPDATE_FLAGS_FOR_PHANTOM,
-	HWS_UPDATE_PHANTOM_VP_POSITION,
 	OPTC_SET_ODM_COMBINE,
 	OPTC_SET_ODM_BYPASS,
 	OPP_PIPE_CLOCK_CONTROL,
@@ -1407,6 +1453,10 @@ struct hw_sequencer_funcs {
 			struct dc_state *context);
 	void (*update_plane_addr)(const struct dc *dc,
 			struct pipe_ctx *pipe_ctx);
+	void (*prepare_plane_addr_update)(const struct dc *dc,
+			struct pipe_ctx *pipe_ctx,
+			struct dc_plane_address *addr_to_program,
+			bool *flip_immediate);
 	void (*update_dchub)(struct dce_hwseq *hws,
 			struct dchub_init_data *dh_data);
 	void (*wait_for_mpcc_disconnect)(struct dc *dc,
@@ -1419,15 +1469,16 @@ struct hw_sequencer_funcs {
 	void (*edp_backlight_control)(
 			struct dc_link *link,
 			bool enable);
-	void (*program_triplebuffer)(const struct dc *dc,
-		struct pipe_ctx *pipe_ctx, bool enableTripleBuffer);
+	void (*program_triplebuffer)(struct hubp *hubp, bool enableTripleBuffer);
 	void (*update_pending_status)(struct pipe_ctx *pipe_ctx);
 	void (*update_dsc_pg)(struct dc *dc, struct dc_state *context, bool safe_to_disable);
 	void (*clear_surface_dcc_and_tiling)(struct pipe_ctx *pipe_ctx, struct dc_plane_state *plane_state, bool clear_tiling);
 
 	/* Pipe Lock Related */
-	void (*pipe_control_lock)(struct dc *dc,
-			struct pipe_ctx *pipe, bool lock);
+	bool (*build_pipe_control_lock_sequence)(struct dc *dc,
+			struct pipe_ctx *pipe, bool lock,
+			struct pipe_control_lock_params *params);
+	void (*tg_lock)(struct tg_lock_params *params);
 	void (*interdependent_update_lock)(struct dc *dc,
 			struct dc_state *context, bool lock);
 	void (*set_flip_control_gsl)(struct pipe_ctx *pipe_ctx,
@@ -1452,8 +1503,9 @@ struct hw_sequencer_funcs {
 	void (*enable_vblanks_synchronization)(struct dc *dc,
 			int group_index, int group_size,
 			struct pipe_ctx *grouped_pipes[]);
-	void (*setup_periodic_interrupt)(struct dc *dc,
-			struct pipe_ctx *pipe_ctx);
+	void (*setup_periodic_interrupt)(struct timing_generator *tg,
+			uint32_t start_line,
+			uint32_t end_line);
 	void (*set_drr)(struct pipe_ctx **pipe_ctx, int num_pipes,
 			struct dc_crtc_timing_adjust adjust);
 	void (*set_static_screen_control)(struct pipe_ctx **pipe_ctx,
@@ -1490,13 +1542,24 @@ struct hw_sequencer_funcs {
 	bool (*dmdata_status_done)(struct pipe_ctx *pipe_ctx);
 
 	/* Cursor Related */
-	void (*set_cursor_position)(struct pipe_ctx *pipe);
+	void (*set_cursor_position)(struct hubp *hubp, struct dpp *dpp,
+			const struct dc_cursor_position *pos,
+			const struct dc_cursor_mi_param *param);
+	/* Fallback for DCE */
+	void (*set_cursor_position_legacy)(struct pipe_ctx *pipe);
 	void (*set_cursor_attribute)(struct pipe_ctx *pipe);
 	void (*set_cursor_sdr_white_level)(struct pipe_ctx *pipe);
-	void (*abort_cursor_offload_update)(struct dc *dc, const struct pipe_ctx *pipe);
-	void (*begin_cursor_offload_update)(struct dc *dc, const struct pipe_ctx *pipe);
-	void (*commit_cursor_offload_update)(struct dc *dc, const struct pipe_ctx *pipe);
-	void (*update_cursor_offload_pipe)(struct dc *dc, const struct pipe_ctx *pipe);
+	void (*abort_cursor_offload_update)(struct dmub_srv *dmub, struct dpp *dpp,
+			struct hubp *hubp, uint32_t stream_idx);
+	void (*begin_cursor_offload_update)(struct dmub_srv *dmub, struct dpp *dpp,
+			struct hubp *hubp, uint32_t stream_idx);
+	void (*commit_cursor_offload_update)(struct dmub_srv *dmub, struct dpp *dpp,
+			struct hubp *hubp, uint32_t stream_idx);
+	void (*update_cursor_offload_pipe)(struct dmub_srv *dmub,
+			uint32_t stream_idx,
+			uint8_t pipe_idx,
+			const struct dpp *dpp,
+			const struct hubp *hubp);
 	void (*notify_cursor_offload_drr_update)(struct dc *dc, struct dc_state *context,
 						 const struct dc_stream_state *stream);
 	void (*program_cursor_offload_now)(struct dc *dc, const struct pipe_ctx *pipe);
@@ -1652,9 +1715,8 @@ struct hw_sequencer_funcs {
 	bool (*is_pipe_topology_transition_seamless)(struct dc *dc,
 			const struct dc_state *cur_ctx,
 			const struct dc_state *new_ctx);
-	void (*wait_for_dcc_meta_propagation)(const struct dc *dc,
-		const struct pipe_ctx *top_pipe_to_program);
-	void (*dmub_hw_control_lock)(struct dc *dc,
+	void (*wait_for_dcc_meta_propagation)(uint32_t delay);
+	bool (*dmub_hw_control_lock)(struct dc *dc,
 			struct dc_state *context,
 			bool lock);
 	void (*fams2_update_config)(struct dc *dc,
@@ -1775,6 +1837,8 @@ void set_drr_and_clear_adjust_pending(
 		struct dc_stream_state *stream,
 		struct drr_params *params);
 
+struct dpp_cursor_attributes calc_sdr_cursor_attributes(struct pipe_ctx *pipe_ctx);
+
 void hwss_execute_sequence(struct dc *dc,
 		struct block_sequence block_sequence[MAX_HWSS_BLOCK_SEQUENCE_SIZE],
 		int num_steps);
@@ -1814,6 +1878,9 @@ void hwss_process_outstanding_hw_updates(struct dc *dc,
 		struct dc_state *dc_context);
 
 void hwss_send_dmcub_cmd(union block_sequence_params *params);
+void hwss_lsdma_send_pio_copy(union block_sequence_params *params);
+
+void hwss_program_triplebuffer(union block_sequence_params *params);
 
 void hwss_program_manual_trigger(union block_sequence_params *params);
 
@@ -1953,7 +2020,8 @@ void hwss_tg_set_gsl(union block_sequence_params *params);
 
 void hwss_tg_set_gsl_source_select(union block_sequence_params *params);
 
-void hwss_hubp_wait_flip_pending(union block_sequence_params *params);
+void hwss_hubp_wait_flip_pending(struct hubp *hubp,
+		unsigned int polling_interval_us);
 
 void hwss_tg_wait_double_buffer_pending(union block_sequence_params *params);
 
@@ -2081,8 +2149,6 @@ void hwss_dpp_set_scaler(union block_sequence_params *params);
 
 void hwss_hubp_mem_program_viewport(union block_sequence_params *params);
 
-void hwss_abort_cursor_offload_update(union block_sequence_params *params);
-
 void hwss_send_cursor_info_to_dmu(union block_sequence_params *params);
 
 void hwss_set_cursor_attribute(union block_sequence_params *params);
@@ -2093,6 +2159,8 @@ void hwss_dpp_set_cursor_attributes(union block_sequence_params *params);
 
 void hwss_set_cursor_position(union block_sequence_params *params);
 
+void hwss_program_cursor_position(struct dc *dc, struct pipe_ctx *pipe_ctx);
+
 void hwss_set_cursor_sdr_white_level(union block_sequence_params *params);
 
 void hwss_program_gamut_remap(struct pipe_ctx *pipe_ctx);
@@ -2107,39 +2175,49 @@ void hwss_phantom_hubp_post_enable(union block_sequence_params *params);
 
 void hwss_cursor_lock(union block_sequence_params *params);
 
-void hwss_begin_cursor_offload_update(union block_sequence_params *params);
+void hwss_begin_cursor_offload_update(struct dc *dc, union block_sequence_params *params);
 
-void hwss_commit_cursor_offload_update(union block_sequence_params *params);
+void hwss_commit_cursor_offload_update(struct dc *dc, union block_sequence_params *params);
 
-void hwss_update_cursor_offload_pipe(union block_sequence_params *params);
+void hwss_update_cursor_offload_pipe(struct dc *dc, union block_sequence_params *params);
 
-void hwss_setup_periodic_interrupt(struct dc *dc, union block_sequence_params *params);
+void hwss_abort_cursor_offload_update(struct dc *dc, union block_sequence_params *params);
+
+void hwss_setup_periodic_interrupt(struct dc *dc, struct pipe_ctx *pipe_ctx);
 
 void hwss_disable_audio_stream(struct dc *dc, union block_sequence_params *params);
 
+void hwss_hubp_wait_for_dcc_meta_prop(struct dc *dc, struct pipe_ctx *top_pipe_to_program);
+
+void hwss_set_input_transfer_func(struct dc *dc, struct pipe_ctx *pipe_ctx);
+
 void hwss_add_optc_pipe_control_lock(struct block_sequence_state *seq_state,
 		struct dc *dc, struct pipe_ctx *pipe_ctx, bool lock);
 
+void hwss_pipe_control_lock(struct dc *dc,
+		struct pipe_ctx *pipe_ctx, bool lock);
+
 void hwss_add_hubp_set_flip_control_gsl(struct block_sequence_state *seq_state,
 		struct hubp *hubp, bool flip_immediate);
 
 void hwss_add_hubp_program_triplebuffer(struct block_sequence_state *seq_state,
-		struct dc *dc, struct pipe_ctx *pipe_ctx, bool enableTripleBuffer);
+		struct hubp *hubp, bool enableTripleBuffer);
 
 void hwss_add_hubp_update_plane_addr(struct block_sequence_state *seq_state,
 		struct dc *dc, struct pipe_ctx *pipe_ctx);
 
 void hwss_add_dpp_set_input_transfer_func(struct block_sequence_state *seq_state,
-		struct dc *dc, struct pipe_ctx *pipe_ctx, struct dc_plane_state *plane_state);
+		struct dc *dc, struct pipe_ctx *pipe_ctx);
 
 void hwss_add_dpp_program_gamut_remap(struct block_sequence_state *seq_state,
 		struct pipe_ctx *pipe_ctx);
 
 void hwss_add_dpp_program_bias_and_scale(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx);
+		struct dpp *dpp,
+		struct dc_plane_state *plane_state);
 
 void hwss_add_optc_program_manual_trigger(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx);
+		struct timing_generator *tg);
 
 void hwss_add_dpp_set_output_transfer_func(struct block_sequence_state *seq_state,
 		struct dc *dc, struct pipe_ctx *pipe_ctx);
@@ -2164,18 +2242,12 @@ void hwss_add_dmub_send_dmcub_cmd(struct block_sequence_state *seq_state,
 void hwss_add_dmub_subvp_save_surf_addr(struct block_sequence_state *seq_state,
 		struct dc_dmub_srv *dc_dmub_srv, struct dc_plane_address *addr, uint8_t subvp_index);
 
-void hwss_add_hubp_wait_for_dcc_meta_prop(struct block_sequence_state *seq_state,
-		struct dc *dc, struct pipe_ctx *top_pipe_to_program);
-
+void hwss_add_lsdma_send_pio_copy(struct block_sequence_state *seq_state,
+		struct dc_dmub_srv *dc_dmub_srv, uint64_t src_addr, uint64_t dst_addr,
+		uint32_t byte_count, uint32_t overlap_disable);
 void hwss_add_hubp_wait_pipe_read_start(struct block_sequence_state *seq_state,
 		struct hubp *hubp);
 
-void hwss_add_hws_apply_update_flags_for_phantom(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx);
-
-void hwss_add_hws_update_phantom_vp_position(struct block_sequence_state *seq_state,
-		struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe_ctx);
-
 void hwss_add_optc_set_odm_combine(struct block_sequence_state *seq_state,
 		struct timing_generator *tg, int opp_inst[MAX_PIPES], int opp_head_count,
 		int odm_slice_width, int last_odm_slice_width);
@@ -2236,7 +2308,7 @@ void hwss_add_tg_enable_crtc(struct block_sequence_state *seq_state,
 		struct timing_generator *tg);
 
 void hwss_add_hubp_wait_flip_pending(struct block_sequence_state *seq_state,
-		struct hubp *hubp, unsigned int timeout_us, unsigned int polling_interval_us);
+		struct hubp *hubp, unsigned int polling_interval_us);
 
 void hwss_add_tg_wait_double_buffer_pending(struct block_sequence_state *seq_state,
 		struct timing_generator *tg, unsigned int timeout_us, unsigned int polling_interval_us);
@@ -2523,7 +2595,8 @@ void hwss_add_hubp_program_surface_config(struct block_sequence_state *seq_state
 		int compat_level);
 
 void hwss_add_dpp_setup_dpp(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx);
+		struct dpp *dpp,
+		struct dc_plane_state *plane_state);
 
 void hwss_add_dpp_set_cursor_matrix(struct block_sequence_state *seq_state,
 		struct dpp *dpp,
@@ -2552,6 +2625,10 @@ void hwss_add_dpp_set_scaler(struct block_sequence_state *seq_state,
 		struct dpp *dpp,
 		const struct scaler_data *scl_data);
 
+void hwss_add_dpp_program_upsp(struct block_sequence_state *seq_state,
+		struct dpp *dpp,
+		const struct dscl_prog_data *dscl_prog_data);
+
 void hwss_add_hubp_mem_program_viewport(struct block_sequence_state *seq_state,
 		struct hubp *hubp,
 		const struct rect *viewport,
@@ -2578,7 +2655,6 @@ void hwss_add_set_cursor_position(struct block_sequence_state *seq_state,
 		struct pipe_ctx *pipe_ctx);
 
 void hwss_add_set_cursor_sdr_white_level(struct block_sequence_state *seq_state,
-		struct dc *dc,
 		struct pipe_ctx *pipe_ctx);
 
 void hwss_add_program_output_csc(struct block_sequence_state *seq_state,
@@ -2630,8 +2706,7 @@ void hwss_add_cursor_lock(struct block_sequence_state *seq_state,
 		bool lock);
 
 void hwss_add_send_update_cursor_info_to_dmu(struct block_sequence_state *seq_state,
-		struct pipe_ctx *pipe_ctx,
-		int index);
+		struct pipe_ctx *pipe_ctx);
 
 void hwss_add_update_cursor_offload_pipe(struct block_sequence_state *seq_state,
 		struct dc *dc,
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
index 0f7881a2146206..9586029dcec103 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer_private.h
@@ -73,6 +73,7 @@ struct timing_generator;
 struct tg_color;
 struct output_pixel_processor;
 struct mpcc_blnd_cfg;
+struct rmcm;
 
 struct hwseq_private_funcs {
 
@@ -90,9 +91,7 @@ struct hwseq_private_funcs {
 	void (*update_mpcc)(struct dc *dc, struct pipe_ctx *pipe_ctx);
 	void (*update_mpcc_sequence)(struct dc *dc, struct pipe_ctx *pipe_ctx,
 			struct block_sequence_state *seq_state);
-	bool (*set_input_transfer_func)(struct dc *dc,
-				struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state);
+	bool (*set_input_transfer_func)(struct set_input_transfer_func_params *params);
 	bool (*set_output_transfer_func)(struct set_output_transfer_func_params *params);
 	void (*power_down)(struct dc *dc);
 	void (*enable_display_pipe_clock_gating)(struct dc_context *ctx,
@@ -187,12 +186,17 @@ struct hwseq_private_funcs {
 		struct block_sequence_state *seq_state);
 	bool (*wait_for_blank_complete)(struct output_pixel_processor *opp);
 	void (*dccg_init)(struct dce_hwseq *hws);
-	bool (*set_blend_lut)(struct pipe_ctx *pipe_ctx,
-			const struct dc_plane_state *plane_state);
-	bool (*set_shaper_3dlut)(struct pipe_ctx *pipe_ctx,
-			const struct dc_plane_state *plane_state);
-	bool (*set_mcm_luts)(struct pipe_ctx *pipe_ctx,
-			const struct dc_plane_state *plane_state);
+	bool (*set_blend_lut)(struct dpp *dpp,
+			struct dc_plane_state *plane_state);
+	bool (*set_shaper_3dlut)(struct dpp *dpp,
+			struct dc_plane_state *plane_state);
+	bool (*set_mcm_luts)(struct dc *dc, struct dpp *dpp, struct hubp *hubp,
+			struct hubp *primary_hubp, struct mpc *mpc, int mpcc_id,
+			struct dc_stream_state *stream,
+			struct dc_plane_state *plane_state);
+	bool (*set_rmcm_luts)(struct set_input_transfer_func_params *params);
+	void (*disable_mcm_luts)(struct mpc *mpc, int mpcc_id);
+	void (*disable_rmcm_luts)(struct dc *dc, struct rmcm *rmcm, struct hubp *hubp, int mpcc_id);
 	void (*PLAT_58856_wa)(struct dc_state *context,
 			struct pipe_ctx *pipe_ctx);
 	void (*setup_hpo_hw_control)(const struct dce_hwseq *hws, bool enable);
@@ -217,13 +221,17 @@ struct hwseq_private_funcs {
 	void (*reset_back_end_for_pipe)(struct dc *dc,
 			struct pipe_ctx *pipe_ctx,
 			struct dc_state *context);
-	void (*perform_3dlut_wa_unlock)(struct pipe_ctx *pipe_ctx);
+	void (*perform_3dlut_wa_unlock)(struct timing_generator *tg,
+			struct hubp *primary_hubp);
 	void (*wait_for_pipe_update_if_needed)(struct dc *dc, struct pipe_ctx *pipe_ctx, bool is_surface_update_only);
 	void (*set_wait_for_update_needed_for_pipe)(struct dc *dc, struct pipe_ctx *pipe_ctx);
 	void (*dc_ip_request_cntl)(struct dc *dc, bool enable);
 	void (*program_cm_hist)(struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
 		const struct dc_plane_state *plane_state);
+	void (*build_cursor_pos_update_params)(const struct pipe_ctx *pipe_ctx,
+		struct dc_cursor_position *pos,
+		struct dc_cursor_mi_param *param);
 };
 
 struct dce_hwseq {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/clock_source.h b/drivers/gpu/drm/amd/display/dc/inc/clock_source.h
index 6bbbb8ea7dadfc..0bf5419d9d9fa0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/clock_source.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/clock_source.h
@@ -1,4 +1,5 @@
-/*
+/* SPDX-License-Identifier: MIT
+ *
  * Copyright 2012-15 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
@@ -185,6 +186,7 @@ struct clock_source {
 	const struct clock_source_funcs *funcs;
 	struct dc_context *ctx;
 	enum clock_source_id id;
+	unsigned int inst;
 	bool dp_clk_src;
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_status.h b/drivers/gpu/drm/amd/display/dc/inc/core_status.h
index 1a17e727ed0449..69215d0e8f4623 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_status.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_status.h
@@ -72,6 +72,8 @@ enum dc_status {
 	/// Handshake failed, programming aborted, DCN may be in inconsistent state.
 	DC_DPMS_FAILED_INCOMPLETE = 33,
 
+	DC_NO_DRAM_BUFFER_RESOURCE = 34,
+
 	DC_ERROR_UNEXPECTED = -1
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index b18d607fb222ab..6563eaaa0c0cfb 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -247,6 +247,8 @@ struct audio_support{
 #define NO_UNDERLAY_PIPE -1
 
 struct resource_pool {
+	struct dc_context *ctx;
+
 	struct mem_input *mis[MAX_PIPES];
 	struct hubp *hubps[MAX_PIPES];
 	struct input_pixel_processor *ipps[MAX_PIPES];
@@ -264,6 +266,13 @@ struct resource_pool {
 	struct dce_i2c_sw *sw_i2cs[MAX_PIPES];
 	bool i2c_hw_buffer_in_use;
 
+	/* LSDMA scratch memory. buffer is NULL when not allocated. */
+	struct {
+		void *buffer;
+		long long pa;
+		unsigned int size;
+	} lsdma_scratch;
+
 	struct dwbc *dwbc[MAX_DWB_PIPES];
 	struct mcif_wb *mcif_wb[MAX_DWB_PIPES];
 	struct {
@@ -299,7 +308,7 @@ struct resource_pool {
 	struct hpo_dp_link_encoder *hpo_dp_link_enc[MAX_HPO_DP2_LINK_ENCODERS];
 	struct dc_3dlut *mpc_lut[MAX_PIPES];
 	struct dc_transfer_func *mpc_shaper[MAX_PIPES];
-	struct dc_rmcm_3dlut rmcm_3dlut[MAX_RMCM_INST];
+	struct rmcm *rmcm[MAX_RMCM_INST];
 
 	struct {
 		unsigned int xtalin_clock_inKhz;
@@ -392,6 +401,7 @@ struct plane_resource {
 	struct transform *xfm;
 	struct dpp *dpp;
 	uint8_t mpcc_inst;
+	struct rmcm *rmcm;
 
 	struct dcn_fe_bandwidth bw;
 };
@@ -549,6 +559,7 @@ struct resource_context {
 	unsigned int hpo_dp_link_enc_to_link_idx[MAX_HPO_DP2_LINK_ENCODERS];
 	int hpo_dp_link_enc_ref_cnts[MAX_HPO_DP2_LINK_ENCODERS];
 	bool is_mpc_3dlut_acquired[MAX_PIPES];
+	bool rmcm_in_use[MAX_RMCM_INST];
 	/* used to build scalar data in dml2 and for edp backlight programming */
 	struct pipe_ctx temp_pipe;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/inc/custom_float.h b/drivers/gpu/drm/amd/display/dc/inc/custom_float.h
index f57239672216d2..897759a041e4f4 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/custom_float.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/custom_float.h
@@ -26,10 +26,25 @@
 #ifndef CUSTOM_FLOAT_H_
 #define CUSTOM_FLOAT_H_
 
-#include "bw_fixed.h"
-#include "hw_shared.h"
-#include "opp.h"
+#include "os_types.h"    /* uint32_t, bool */
+#include "fixed31_32.h"  /* struct fixed31_32 (used by value) */
 
+/*
+ * Moved here from opp.h so consumers of custom-float do not have to pull in the
+ * heavy opp.h
+ */
+struct custom_float_format {
+	uint32_t mantissa_bits;
+	uint32_t exponenta_bits;
+	bool sign;
+};
+
+struct custom_float_value {
+	uint32_t mantissa;
+	uint32_t exponenta;
+	uint32_t value;
+	bool negative;
+};
 
 bool convert_to_custom_float_format(
 	struct fixed31_32 value,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
index 6ae7cf10956512..e3fa987304da61 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
@@ -33,6 +33,7 @@ struct abm {
 	struct dc_context *ctx;
 	const struct abm_funcs *funcs;
 	bool dmcu_is_running;
+	unsigned int inst;
 };
 
 struct abm_funcs {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index 614beb5924fd68..5b06659423d569 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -433,6 +433,7 @@ struct clk_mgr_funcs {
 struct clk_mgr {
 	struct dc_context *ctx;
 	struct clk_mgr_funcs *funcs;
+	int inst;
 	struct dc_clocks clks;
 	bool psr_allow_active_cache;
 	bool force_smu_not_present;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
index e756719308abc7..4b7c55eeb0c639 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
@@ -193,6 +193,7 @@ struct dcn_dccg_reg_state {
 struct dccg {
 	struct dc_context *ctx;
 	const struct dccg_funcs *funcs;
+	int inst;
 	int pipe_dppclk_khz[MAX_PIPES];
 	int ref_dppclk;
 	bool dpp_clock_gated[MAX_PIPES];
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
index 02bc0010e565ef..8d1024a875a71c 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
@@ -253,6 +253,7 @@ struct hubbub_funcs {
 	void (*wait_for_det_update)(struct hubbub *hubbub, int hubp_inst);
 	bool (*program_arbiter)(struct hubbub *hubbub, struct dml2_display_arb_regs *arb_regs, bool safe_to_lower);
 	void (*dchvm_init)(struct hubbub *hubbub);
+	void (*override_utm_client_qc_profile)(struct hubbub *hubbub, uint8_t qc_profile, int index);
 
 	/* Performance monitoring related functions */
 	struct hubbub_perfmon_funcs {
@@ -300,6 +301,7 @@ struct hubbub_funcs {
 struct hubbub {
 	const struct hubbub_funcs *funcs;
 	struct dc_context *ctx;
+	int inst;
 	bool riommu_active;
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dio.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dio.h
index 532bf54cf2c489..916ac15b6b5897 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dio.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dio.h
@@ -17,6 +17,7 @@ struct dio_funcs {
 struct dio {
 	const struct dio_funcs *funcs;
 	struct dc_context *ctx;
+	unsigned int inst;
 };
 
 #endif /* __DC_DIO_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
index de3113ecbc7750..eebfd177463d12 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
@@ -74,16 +74,6 @@ struct dmcu_funcs {
 	bool (*is_dmcu_initialized)(struct dmcu *dmcu);
 	bool (*lock_phy)(struct dmcu *dmcu);
 	bool (*unlock_phy)(struct dmcu *dmcu);
-	bool (*send_edid_cea)(struct dmcu *dmcu,
-			int offset,
-			int total_length,
-			uint8_t *data,
-			int length);
-	bool (*recv_amd_vsdb)(struct dmcu *dmcu,
-			int *version,
-			int *min_frame_rate,
-			int *max_frame_rate);
-	bool (*recv_edid_cea_ack)(struct dmcu *dmcu, int *offset);
 #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
 	void (*forward_crc_window)(struct dmcu *dmcu,
 			struct rect *rect,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index b31b144d2e8fc7..71244d3b6c908a 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -330,6 +330,8 @@ struct dpp_funcs {
 			uint32_t height
 			);
 
+	void (*refresh_cursor_state)(struct dpp *dpp_base);
+
 	void (*dpp_set_hdr_multiplier)(
 			struct dpp *dpp_base,
 			uint32_t multiplier);
@@ -378,7 +380,8 @@ struct dpp_funcs {
 	void (*dpp_set_pregam_state)(
 			struct dpp *dpp_base,
 			enum dc_transfer_func_predefined tr,
-			enum dc_scaling_linearity scaling);
+			enum dc_scaling_linearity scaling,
+			bool source_is_linear);
 	void (*dpp_program_upsp)(
 			struct dpp *dpp_base,
 			const struct dscl_prog_data *dscl_prog_data);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 9dc9317c28a44b..d1cbcb87d27128 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -87,6 +87,7 @@ enum hubp_3dlut_fl_addressing_mode {
 };
 
 enum hubp_3dlut_fl_width {
+	hubp_3dlut_fl_width_9 = 9,
 	hubp_3dlut_fl_width_17 = 17,
 	hubp_3dlut_fl_width_33 = 33,
 	hubp_3dlut_fl_width_17_transformed    = 4916, //mpc default
@@ -175,7 +176,8 @@ struct hubp_funcs {
 	bool (*hubp_program_surface_flip_and_addr)(
 		struct hubp *hubp,
 		const struct dc_plane_address *address,
-		bool flip_immediate);
+		bool flip_immediate,
+		bool dcc);
 
 	void (*hubp_program_pte_vm)(
 		struct hubp *hubp,
@@ -217,6 +219,8 @@ struct hubp_funcs {
 			const struct dc_cursor_position *pos,
 			const struct dc_cursor_mi_param *param);
 
+	void (*refresh_cursor_state)(struct hubp *hubp);
+
 	void (*hubp_disconnect)(struct hubp *hubp);
 
 	void (*hubp_clk_cntl)(struct hubp *hubp, bool enable);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index 176d048a64e704..6109d22e109389 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -153,13 +153,6 @@ struct dc_rgb {
 	uint32_t blue;
 };
 
-struct tetrahedral_33x33x33 {
-	struct dc_rgb lut0[8985];
-	struct dc_rgb lut1[8984];
-	struct dc_rgb lut2[8984];
-	struct dc_rgb lut3[8984];
-};
-
 struct tetrahedral_17x17x17 {
 	struct dc_rgb lut0[1229];
 	struct dc_rgb lut1[1228];
@@ -182,14 +175,11 @@ enum lut_dimension {
 
 struct tetrahedral_params {
 	union {
-//TODO: Uncomment when in use.
-//		struct tetrahedral_33x33x33 tetrahedral_33;
 		struct tetrahedral_17x17x17 tetrahedral_17;
 		struct tetrahedral_9x9x9 tetrahedral_9;
 	};
 	bool use_tetrahedral_9;
 	bool use_12bits;
-	enum lut_dimension lut_dim;
 };
 
 /* arr_curve_points - regamma regions/segments specification
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
index 70545df981ef39..f3ef5bbfad81f0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
@@ -100,6 +100,7 @@ struct link_encoder {
 	struct graphics_object_id connector;
 	uint32_t output_signals;
 	enum engine_id preferred_engine;
+	unsigned int inst;
 	enum engine_id analog_engine;
 	struct encoder_feature_support features;
 	enum transmitter transmitter;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
index d863f4bdccadc3..75ebfab87efcd5 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
@@ -193,42 +193,6 @@ struct mpc_grph_gamut_adjustment {
 	enum mpcc_gamut_remap_id mpcc_gamut_remap_block_id;
 };
 
-struct mpc_rmcm_regs {
-	uint32_t rmcm_3dlut_mem_pwr_state;
-	uint32_t rmcm_3dlut_mem_pwr_force;
-	uint32_t rmcm_3dlut_mem_pwr_dis;
-	uint32_t rmcm_3dlut_mem_pwr_mode;
-	uint32_t rmcm_3dlut_size;
-	uint32_t rmcm_3dlut_mode;
-	uint32_t rmcm_3dlut_mode_cur;
-	uint32_t rmcm_3dlut_read_sel;
-	uint32_t rmcm_3dlut_30bit_en;
-	uint32_t rmcm_3dlut_wr_en_mask;
-	uint32_t rmcm_3dlut_ram_sel;
-	uint32_t rmcm_3dlut_out_norm_factor;
-	uint32_t rmcm_3dlut_fl_sel;
-	uint32_t rmcm_3dlut_out_offset_r;
-	uint32_t rmcm_3dlut_out_scale_r;
-	uint32_t rmcm_3dlut_fl_done;
-	uint32_t rmcm_3dlut_fl_soft_underflow;
-	uint32_t rmcm_3dlut_fl_hard_underflow;
-	uint32_t rmcm_cntl;
-	uint32_t rmcm_shaper_mem_pwr_state;
-	uint32_t rmcm_shaper_mem_pwr_force;
-	uint32_t rmcm_shaper_mem_pwr_dis;
-	uint32_t rmcm_shaper_mem_pwr_mode;
-	uint32_t rmcm_shaper_lut_mode;
-	uint32_t rmcm_shaper_mode_cur;
-	uint32_t rmcm_shaper_lut_write_en_mask;
-	uint32_t rmcm_shaper_lut_write_sel;
-	uint32_t rmcm_shaper_offset_b;
-	uint32_t rmcm_shaper_scale_b;
-	uint32_t rmcm_shaper_rama_exp_region_start_b;
-	uint32_t rmcm_shaper_rama_exp_region_start_seg_b;
-	uint32_t rmcm_shaper_rama_exp_region_end_b;
-	uint32_t rmcm_shaper_rama_exp_region_end_base_b;
-};
-
 struct mpcc_sm_cfg {
 	bool enable;
 	/* 0-single plane,2-row subsampling,4-column subsampling,6-checkboard subsampling */
@@ -317,6 +281,7 @@ struct mpc_tree {
 struct mpc {
 	const struct mpc_funcs *funcs;
 	struct dc_context *ctx;
+	int inst;
 
 	struct mpcc mpcc_array[MAX_MPCC];
 	struct pwl_params blender_params;
@@ -340,7 +305,6 @@ struct mpcc_state {
 	uint32_t rgam_mode;
 	uint32_t rgam_lut;
 	struct mpc_grph_gamut_adjustment gamut_remap;
-	struct mpc_rmcm_regs rmcm_regs;
 };
 
 struct dcn_mpc_reg_state {
@@ -1143,33 +1107,6 @@ struct mpc_funcs {
 			const int mpcc_id,
 			bool *enable,
 			bool *lut_bank_a);
-
-	/**
-	 * @rmcm:
-	 *
-	 * MPC RMCM new HW sequential programming functions
-	 */
-	struct {
-		void (*fl_3dlut_configure)(struct mpc *mpc, struct mpc_fl_3dlut_config *cfg, int mpcc_id);
-		void (*enable_3dlut_fl)(struct mpc *mpc, bool enable, int mpcc_id);
-		void (*update_3dlut_fast_load_select)(struct mpc *mpc, int mpcc_id, int hubp_idx);
-		void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id,
-			bool lut_bank_a, bool enabled, int mpcc_id);
-		void (*program_lut_mode)(struct mpc *mpc,
-			bool enable,
-			bool lut_bank_a,
-			int mpcc_id);
-		void (*program_3dlut_size)(struct mpc *mpc, const enum dc_cm_lut_size size, int mpcc_id);
-		void (*program_bias_scale)(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id);
-		void (*program_bit_depth)(struct mpc *mpc, uint16_t bit_depth, int mpcc_id);
-		bool (*is_config_supported)(uint32_t width);
-
-		void (*power_on_shaper_3dlut)(struct mpc *mpc, uint32_t mpcc_id, bool power_on);
-		void (*populate_lut)(struct mpc *mpc, const union mcm_lut_params params,
-			bool lut_bank_a, int mpcc_id);
-		void (*get_3dlut_mode)(struct mpc *mpc, int mpcc_id,
-			bool *enable, bool *lut_bank_a);
-	} rmcm;
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
index 73cc34ea772637..f2d16d88b0bc3f 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
@@ -43,6 +43,7 @@
 #define __DAL_OPP_H__
 
 #include "hw_shared.h"
+#include "../custom_float.h"
 #include "dc_hw_types.h"
 #include "transform.h"
 #include "mpc.h"
@@ -142,19 +143,6 @@ enum channel_name {
 	CHANNEL_NAME_BLUE
 };
 
-struct custom_float_format {
-	uint32_t mantissa_bits;
-	uint32_t exponenta_bits;
-	bool sign;
-};
-
-struct custom_float_value {
-	uint32_t mantissa;
-	uint32_t exponenta;
-	uint32_t value;
-	bool negative;
-};
-
 struct hw_x_point {
 	uint32_t custom_float_x;
 	struct fixed31_32 x;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h b/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h
index 227e3f8d7e5f56..3e514099393560 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h
@@ -34,6 +34,7 @@ struct pg_cntl {
 	const struct pg_cntl_funcs *funcs;
 	bool pg_pipe_res_enable[PG_HW_PIPE_RESOURCES_NUM_ELEMENT][MAX_PIPES];
 	bool pg_res_enable[PG_HW_RESOURCES_NUM_ELEMENT];
+	unsigned int inst;
 };
 
 struct pg_cntl_funcs {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/rmcm.h b/drivers/gpu/drm/amd/display/dc/inc/hw/rmcm.h
new file mode 100644
index 00000000000000..636cd097043804
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/rmcm.h
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RMCM_H__
+#define __DC_RMCM_H__
+
+#include "dc_types.h"
+
+#define MAX_RMCM 2
+
+/* Detach values for the RMCM binding registers: no MPCC connected (MPC_RMCM_CNTL)
+ * and no HUBP driving 3DLUT fast load (MPC_RMCM_3DLUT_FL_SEL).
+ */
+#define RMCM_MPCC_ID_NONE 0xF
+#define RMCM_FL_HUBP_IDX_NONE 0xF
+
+/* MPC_RMCM_CNTL has not been written by the driver yet, so it may still hold
+ * whatever the boot state left there.
+ */
+#define RMCM_MPCC_ID_UNKNOWN (-1)
+
+/* MCM_LUT_ID is defined in mpc.h */
+enum MCM_LUT_ID;
+
+union rmcm_lut_params {
+	const struct pwl_params *pwl;
+	const struct tetrahedral_params *lut3d;
+};
+
+struct rmcm_regs {
+	uint32_t rmcm_3dlut_mem_pwr_state;
+	uint32_t rmcm_3dlut_mem_pwr_force;
+	uint32_t rmcm_3dlut_mem_pwr_dis;
+	uint32_t rmcm_3dlut_mem_pwr_mode;
+	uint32_t rmcm_3dlut_size;
+	uint32_t rmcm_3dlut_mode;
+	uint32_t rmcm_3dlut_mode_cur;
+	uint32_t rmcm_3dlut_read_sel;
+	uint32_t rmcm_3dlut_30bit_en;
+	uint32_t rmcm_3dlut_wr_en_mask;
+	uint32_t rmcm_3dlut_ram_sel;
+	uint32_t rmcm_3dlut_out_norm_factor;
+	uint32_t rmcm_3dlut_fl_sel;
+	uint32_t rmcm_3dlut_out_offset_r;
+	uint32_t rmcm_3dlut_out_scale_r;
+	uint32_t rmcm_3dlut_fl_done;
+	uint32_t rmcm_3dlut_fl_soft_underflow;
+	uint32_t rmcm_3dlut_fl_hard_underflow;
+	uint32_t rmcm_cntl;
+	uint32_t rmcm_shaper_mem_pwr_state;
+	uint32_t rmcm_shaper_mem_pwr_force;
+	uint32_t rmcm_shaper_mem_pwr_dis;
+	uint32_t rmcm_shaper_mem_pwr_mode;
+	uint32_t rmcm_shaper_lut_mode;
+	uint32_t rmcm_shaper_mode_cur;
+	uint32_t rmcm_shaper_lut_write_en_mask;
+	uint32_t rmcm_shaper_lut_write_sel;
+	uint32_t rmcm_shaper_offset_b;
+	uint32_t rmcm_shaper_scale_b;
+	uint32_t rmcm_shaper_rama_exp_region_start_b;
+	uint32_t rmcm_shaper_rama_exp_region_start_seg_b;
+	uint32_t rmcm_shaper_rama_exp_region_end_b;
+	uint32_t rmcm_shaper_rama_exp_region_end_base_b;
+};
+
+/**
+ * struct rmcm_state - RMCM hardware state
+ *
+ * Captures the current hardware state of a Relocatable MCM instance.
+ * Used for debugging, state readback, and hardware validation.
+ */
+struct rmcm_state {
+	/**
+	 * @regs: Register state snapshot
+	 */
+	struct rmcm_regs regs;
+};
+
+/**
+ * struct rmcm - RMCM hardware instance
+ *
+ * Represents a single RMCM (Relocatable MCM) hardware instance.
+ * DCN 4.2+ has 2 RMCM instances that are shared across streams.
+ */
+struct rmcm {
+	struct dc_context *ctx;
+	int inst;
+	/* MPCC bound through MPC_RMCM_CNTL; MPC bypasses RMCM entirely when two
+	 * instances are bound to the same MPCC.
+	 */
+	int mpcc_id;
+	const struct rmcm_funcs *funcs;
+};
+
+/**
+ * struct rmcm_funcs - RMCM hardware abstraction function pointers
+ *
+ * Function pointers for RMCM (Relocatable MCM) hardware operations.
+ * These provide the interface for programming the shared cross-stream
+ * color manager instances.
+ */
+struct rmcm_funcs {
+	void (*get_lut_mode)(struct rmcm *rmcm,	const enum MCM_LUT_ID id, int rmcm_id, bool *enable, bool *lut_bank_a);
+	void (*program_lut_mode)(struct rmcm *rmcm, const enum MCM_LUT_ID id,
+		bool enable, bool lut_bank_a, const enum dc_cm_lut_size size,
+		uint16_t bias, uint16_t scale, int rmcm_id);
+	void (*populate_lut)(struct rmcm *rmcm, const enum MCM_LUT_ID id,
+		const union rmcm_lut_params params, bool lut_bank_a, int rmcm_id);
+	void (*program_lut_read_write_control)(struct rmcm *rmcm, const enum MCM_LUT_ID id,
+		bool lut_bank_a, bool enabled, int rmcm_id);
+	void (*update_3dlut_fast_load_select)(struct rmcm *rmcm, int rmcm_id, int hubp_idx);
+	void (*get_3dlut_fast_load_status)(struct rmcm *rmcm, int rmcm_id,
+		bool *done, bool *soft_underflow, bool *hard_underflow);
+	void (*read_rmcm_state)(struct rmcm *rmcm, int rmcm_id, struct rmcm_state *s);
+	void (*connect_mpcc)(struct rmcm *rmcm, int rmcm_id, int mpcc_id);
+};
+
+#endif // __DC_RMCM_H__
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
index 2b95cb11eede29..5b8688b3aaefcd 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
@@ -126,6 +126,7 @@ struct stream_encoder {
 	struct dc_bios *bp;
 	enum engine_id id;
 	uint32_t stream_enc_inst;
+	unsigned int inst;
 	struct vpg *vpg;
 	struct afmt *afmt;
 	struct apg *apg;
@@ -319,6 +320,7 @@ struct hpo_frl_stream_encoder {
 	 * @stream_enc_inst: Instance ID reference.
 	 */
 	uint32_t stream_enc_inst;
+	unsigned int inst;
 
 	/**
 	 * @ctx: DC context.
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index 2eaba65894ab07..52d4d57358cd14 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -512,6 +512,7 @@ struct timing_generator_funcs {
 			uint32_t vtotal_change_limit);
 
 	void (*init_odm)(struct timing_generator *tg);
+	bool (*is_odm_enabled)(struct timing_generator *tg);
 	void (*wait_drr_doublebuffer_pending_clear)(struct timing_generator *tg);
 	void (*set_long_vtotal)(struct timing_generator *optc, const struct long_vtotal_params *params);
 	void (*wait_odm_doublebuffer_pending_clear)(struct timing_generator *tg);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_service.h b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
index 026d28046eea03..b2f86c51d470ba 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/link_service.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
@@ -297,6 +297,8 @@ struct link_service {
 	bool (*edp_replay_residency)(const struct dc_link *link,
 			unsigned int *residency, const bool is_start,
 			const enum pr_residency_mode mode);
+	bool (*edp_replay_get_cumulative_residency)(const struct dc_link *link,
+			uint32_t *residency_milli_pct);
 	bool (*edp_set_replay_power_opt_and_coasting_vtotal)(struct dc_link *link,
 			const unsigned int *power_opts, uint32_t coasting_vtotal, uint16_t frame_skip_number);
 
@@ -309,13 +311,15 @@ struct link_service {
 	bool (*edp_receiver_ready_T7)(struct dc_link *link);
 	bool (*edp_power_alpm_dpcd_enable)(struct dc_link *link, bool enable);
 	bool (*dp_setup_replay)(struct dc_link *link, const struct dc_stream_state *stream);
-	bool (*dp_pr_get_panel_inst)(const struct dc *dc, const struct dc_link *link, unsigned int *inst_out);
+	bool (*dp_pr_get_pr_panel_inst)(const struct dc *dc, const struct dc_link *link, unsigned int *inst_out);
 	bool (*dp_pr_enable)(struct dc_link *link, bool enable);
 	bool (*dp_pr_update_state)(struct dc_link *link, struct dmub_cmd_pr_update_state_data *update_state_data);
 	bool (*dp_pr_set_general_cmd)(struct dc_link *link, struct dmub_cmd_pr_general_cmd_data *general_cmd_data);
 	bool (*dp_pr_get_state)(const struct dc_link *link, uint64_t *state);
 	void (*edp_set_panel_power)(struct dc_link *link, bool powerOn);
-
+	void (*edp_set_panel_polarity_enabled)(const struct dc_link *link, bool enable);
+	void (*edp_panel_polarity_reset)(struct dc_link *link);
+	bool (*edp_get_panel_polarity)(struct dc_link *link, int32_t *polarity);
 
 	/*************************** HDMI FRL *********************************/
 	bool (*hdmi_frl_poll_status_flag)(struct dc_link *link);
@@ -363,5 +367,6 @@ struct link_service {
 	uint64_t (*dp_trace_get_edp_poweroff_timestamp)(struct dc_link *link);
 	void (*dp_trace_source_sequence)(
 			struct dc_link *link, uint8_t dp_test_mode);
+
 };
 #endif /* __DC_LINK_HPD_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 8e21aaac06b3d6..0d9c9771ea7c33 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -123,7 +123,8 @@ void resource_build_test_pattern_params(
 		struct resource_context *res_ctx,
 		struct pipe_ctx *pipe_ctx);
 
-enum upsp_mode resource_is_upsp_required(enum surface_pixel_format format);
+enum upsp_mode resource_is_upsp_required(enum surface_pixel_format format,
+		enum dc_scaling_linearity scaling_linearity);
 
 bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx);
 
@@ -171,7 +172,7 @@ bool resource_attach_surfaces_to_context(
 		struct dc_state *context,
 		const struct resource_pool *pool);
 
-bool resource_can_pipe_disable_cursor(struct pipe_ctx *pipe_ctx);
+bool resource_can_pipe_disable_cursor(const struct pipe_ctx *pipe_ctx);
 
 #define FREE_PIPE_INDEX_NOT_FOUND -1
 
@@ -335,6 +336,21 @@ void resource_remove_dpp_pipes_for_plane_composition(
 		const struct resource_pool *pool,
 		const struct dc_plane_state *plane_state);
 
+/*
+ * RMCM resource management. Instances are handed out for a whole context at once so that a
+ * plane that is only now asking cannot evict one that is already using an instance.
+ * Tracking is via res_ctx->rmcm_in_use[].
+ */
+void resource_assign_rmcm(
+		struct dc_state *new_ctx,
+		const struct dc_state *cur_ctx,
+		const struct resource_pool *pool);
+
+void resource_release_rmcm(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct rmcm **rmcm);
+
 /*
  * Update ODM slice count by acquiring or releasing pipes. If new slices need
  * to be added, it is going to add them to the last ODM index. If existing
diff --git a/drivers/gpu/drm/amd/display/dc/irq/Makefile b/drivers/gpu/drm/amd/display/dc/irq/Makefile
index bbec53719c5d78..497794a1f1d132 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/irq/Makefile
@@ -41,6 +41,7 @@ AMD_DAL_IRQ_DCE60 = $(addprefix $(AMDDALPATH)/dc/irq/dce60/,$(IRQ_DCE60))
 AMD_DISPLAY_FILES += $(AMD_DAL_IRQ_DCE60)
 endif
 
+ifdef CONFIG_DRM_AMD_DC_DCE
 ###############################################################################
 # DCE 8x
 ###############################################################################
@@ -67,6 +68,7 @@ IRQ_DCE12 = irq_service_dce120.o
 AMD_DAL_IRQ_DCE12 = $(addprefix $(AMDDALPATH)/dc/irq/dce120/,$(IRQ_DCE12))
 
 AMD_DISPLAY_FILES += $(AMD_DAL_IRQ_DCE12)
+endif # CONFIG_DRM_AMD_DC_DCE
 
 ###############################################################################
 # DCN 1x
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
index 8958d286dde1fc..7f3084c6dd49f7 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
@@ -179,27 +179,6 @@ static struct irq_source_info_funcs vupdate_irq_info_funcs = {
 #define dc_underflow_int_entry(reg_num) \
 	[DC_IRQ_SOURCE_DC ## reg_num ## UNDERFLOW] = dummy_irq_entry()
 
-bool dal_irq_service_dummy_set(struct irq_service *irq_service,
-			       const struct irq_source_info *info,
-			       bool enable)
-{
-	(void)enable;
-	DC_LOG_ERROR("%s: called for non-implemented irq source, src_id=%u, ext_id=%u\n",
-		     __func__, info->src_id, info->ext_id);
-
-	return false;
-}
-
-bool dal_irq_service_dummy_ack(struct irq_service *irq_service,
-			       const struct irq_source_info *info)
-{
-	DC_LOG_ERROR("%s: called for non-implemented irq source, src_id=%u, ext_id=%u\n",
-		     __func__, info->src_id, info->ext_id);
-
-	return false;
-}
-
-
 bool dce110_vblank_set(struct irq_service *irq_service,
 		       const struct irq_source_info *info,
 		       bool enable)
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.h b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.h
index 9237646c095952..b641e6a9b1358b 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.h
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.h
@@ -36,15 +36,6 @@ enum dc_irq_source to_dal_irq_source_dce110(
 		uint32_t src_id,
 		uint32_t ext_id);
 
-bool dal_irq_service_dummy_set(
-	struct irq_service *irq_service,
-	const struct irq_source_info *info,
-	bool enable);
-
-bool dal_irq_service_dummy_ack(
-	struct irq_service *irq_service,
-	const struct irq_source_info *info);
-
 bool dce110_vblank_set(
 	struct irq_service *irq_service,
 	const struct irq_source_info *info,
diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
index b595a11c5eafa7..f8c8ca266ec198 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c
@@ -28,14 +28,18 @@
 #include "include/irq_service_interface.h"
 #include "include/logger_interface.h"
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 #include "dce110/irq_service_dce110.h"
+#endif
 
 #if defined(CONFIG_DRM_AMD_DC_SI)
 #include "dce60/irq_service_dce60.h"
 #endif
 
+#if defined(CONFIG_DRM_AMD_DC_DCE)
 #include "dce80/irq_service_dce80.h"
 #include "dce120/irq_service_dce120.h"
+#endif
 #include "dcn10/irq_service_dcn10.h"
 
 #include "reg_helper.h"
@@ -105,6 +109,30 @@ void dal_irq_service_set_generic(
 	dm_write_reg(irq_service->ctx, addr, value);
 }
 
+/*
+ * Base "non-implemented source" irq handlers referenced by irq_service.c and
+ * every DCN irq vtable.
+ */
+bool dal_irq_service_dummy_set(struct irq_service *irq_service,
+			       const struct irq_source_info *info,
+			       bool enable)
+{
+	(void)enable;
+	DC_LOG_ERROR("%s: called for non-implemented irq source, src_id=%u, ext_id=%u\n",
+		     __func__, info->src_id, info->ext_id);
+
+	return false;
+}
+
+bool dal_irq_service_dummy_ack(struct irq_service *irq_service,
+			       const struct irq_source_info *info)
+{
+	DC_LOG_ERROR("%s: called for non-implemented irq source, src_id=%u, ext_id=%u\n",
+		     __func__, info->src_id, info->ext_id);
+
+	return false;
+}
+
 bool dal_irq_service_set(
 	struct irq_service *irq_service,
 	enum dc_irq_source source,
diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.h b/drivers/gpu/drm/amd/display/dc/irq/irq_service.h
index bbcef3d2fe3340..c13986754a4a49 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.h
+++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.h
@@ -90,4 +90,13 @@ bool hpd1_ack(
 	struct irq_service *irq_service,
 	const struct irq_source_info *info);
 
+bool dal_irq_service_dummy_set(
+	struct irq_service *irq_service,
+	const struct irq_source_info *info,
+	bool enable);
+
+bool dal_irq_service_dummy_ack(
+	struct irq_service *irq_service,
+	const struct irq_source_info *info);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
index 04df75114dd5b4..81104812340125 100644
--- a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
@@ -117,16 +117,18 @@ void enable_hpo_dp_link_output(struct dc_link *link,
 		return;
 	}
 
-	if (link->dc->res_pool->dccg->funcs->set_symclk32_le_root_clock_gating)
-		link->dc->res_pool->dccg->funcs->set_symclk32_le_root_clock_gating(
-				link->dc->res_pool->dccg,
-				link_res->hpo_dp_link_enc->inst,
-				true);
-	link_res->hpo_dp_link_enc->funcs->enable_link_phy(
-			link_res->hpo_dp_link_enc,
-			link_settings,
-			link->link_enc->transmitter,
-			link->link_enc->hpd_source);
+	{
+		if (link->dc->res_pool->dccg->funcs->set_symclk32_le_root_clock_gating)
+			link->dc->res_pool->dccg->funcs->set_symclk32_le_root_clock_gating(
+					link->dc->res_pool->dccg,
+					link_res->hpo_dp_link_enc->inst,
+					true);
+		link_res->hpo_dp_link_enc->funcs->enable_link_phy(
+				link_res->hpo_dp_link_enc,
+				link_settings,
+				link->link_enc->transmitter,
+				link->link_enc->hpd_source);
+	}
 }
 
 void disable_hpo_dp_link_output(struct dc_link *link,
@@ -231,4 +233,3 @@ const struct link_hwss *get_hpo_dp_link_hwss(void)
 {
 	return &hpo_dp_link_hwss;
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
index 22ad6a7430386a..12fd4fd24a908f 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
@@ -934,7 +934,7 @@ static void verify_link_capability_non_destructive(struct dc_link *link)
 		else
 			link->verified_link_cap = dp_get_max_link_cap(link);
 	} else if (dc_is_hdmi_signal(link->local_sink->sink_signal)) {
-		link->verified_link_cap = link->reported_link_cap;
+		link->frl_verified_link_cap = link->frl_reported_link_cap;
 
 		if (is_hdmi_frl_in_use(link)) {
 			link->local_sink->sink_signal = SIGNAL_TYPE_HDMI_TYPE_A;
@@ -995,6 +995,8 @@ static bool should_verify_link_capability_destructively(struct dc_link *link,
 					}
 				}
 			}
+		} else if (link->dc->debug.skip_detection_link_training) {
+			destrictive = false;
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
index 7146f8356233be..3b192c539f8b03 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
@@ -223,6 +223,7 @@ static void construct_link_service_edp_panel_control(struct link_service *link_s
 	link_srv->edp_send_replay_cmd = edp_send_replay_cmd;
 	link_srv->edp_set_coasting_vtotal = edp_set_coasting_vtotal;
 	link_srv->edp_replay_residency = edp_replay_residency;
+	link_srv->edp_replay_get_cumulative_residency = edp_replay_get_cumulative_residency;
 	link_srv->edp_set_replay_power_opt_and_coasting_vtotal = edp_set_replay_power_opt_and_coasting_vtotal;
 
 	link_srv->edp_wait_for_t12 = edp_wait_for_t12;
@@ -234,6 +235,10 @@ static void construct_link_service_edp_panel_control(struct link_service *link_s
 	link_srv->edp_receiver_ready_T7 = edp_receiver_ready_T7;
 	link_srv->edp_power_alpm_dpcd_enable = edp_power_alpm_dpcd_enable;
 	link_srv->edp_set_panel_power = edp_set_panel_power;
+
+	link_srv->edp_set_panel_polarity_enabled = edp_set_panel_polarity_enabled;
+	link_srv->edp_panel_polarity_reset = edp_panel_polarity_reset;
+	link_srv->edp_get_panel_polarity = edp_get_panel_polarity;
 }
 
 /* link dp panel replay implements DP panel replay functionality.
@@ -241,7 +246,7 @@ static void construct_link_service_edp_panel_control(struct link_service *link_s
 static void construct_link_service_dp_panel_replay(struct link_service *link_srv)
 {
 	link_srv->dp_setup_replay = dp_setup_replay;
-	link_srv->dp_pr_get_panel_inst = dp_pr_get_panel_inst;
+	link_srv->dp_pr_get_pr_panel_inst = dp_pr_get_pr_panel_inst;
 	link_srv->dp_pr_enable = dp_pr_enable;
 	link_srv->dp_pr_update_state = dp_pr_update_state;
 	link_srv->dp_pr_set_general_cmd = dp_pr_set_general_cmd;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
index 24e3bbec15ffba..ad1a499b0d9d23 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
@@ -997,7 +997,8 @@ bool link_decide_link_settings(struct dc_stream_state *stream,
 		 * TODO: add MST specific link training routine
 		 */
 		decide_mst_link_settings(link, link_setting);
-	} else if (stream->signal == SIGNAL_TYPE_VIRTUAL) {
+	} else if (dc_is_virtual_signal(stream->signal)) {
+		/* virtual signals skip link training; use a valid dummy setting. */
 		link_setting->lane_count = LANE_COUNT_FOUR;
 		link_setting->link_rate = LINK_RATE_HIGH3;
 	} else if (link->connector_signal == SIGNAL_TYPE_EDP) {
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
index 0d4f88ff844d47..65a791ccf6b2da 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
@@ -82,7 +82,7 @@ static bool dp_setup_panel_replay(struct dc_link *link, const struct dc_stream_s
 	struct dc *dc;
 	struct dmub_replay *replay;
 	int i;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 	struct replay_context replay_context = { 0 };
 	unsigned int lineTimeInNs = 0;
 
@@ -117,7 +117,7 @@ static bool dp_setup_panel_replay(struct dc_link *link, const struct dc_stream_s
 	if (!replay)
 		return false;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	replay_context.aux_inst = (enum channel_id) link_get_ddc_aux_inst(link);
@@ -202,7 +202,7 @@ static bool dp_setup_panel_replay(struct dc_link *link, const struct dc_stream_s
 }
 
 
-bool dp_pr_get_panel_inst(const struct dc *dc,
+bool dp_pr_get_pr_panel_inst(const struct dc *dc,
 		const struct dc_link *link,
 		unsigned int *inst_out)
 {
@@ -246,10 +246,10 @@ bool dp_setup_replay(struct dc_link *link, const struct dc_stream_state *stream)
 bool dp_pr_enable(struct dc_link *link, bool enable)
 {
 	struct dc *dc = link->ctx->dc;
-	unsigned int panel_inst = 0;
+	unsigned int pr_panel_inst = 0;
 	union dmub_rb_cmd cmd;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	if (link->replay_settings.replay_allow_active == enable)
@@ -264,7 +264,7 @@ bool dp_pr_enable(struct dc_link *link, bool enable)
 	cmd.pr_enable.header.type = DMUB_CMD__PR;
 	cmd.pr_enable.header.sub_type = DMUB_CMD__PR_ENABLE;
 	cmd.pr_enable.header.payload_bytes = sizeof(struct dmub_cmd_pr_enable_data);
-	cmd.pr_enable.data.panel_inst = (uint8_t)panel_inst;
+	cmd.pr_enable.data.panel_inst = (uint8_t)pr_panel_inst;
 	cmd.pr_enable.data.enable = enable ? 1 : 0;
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 
@@ -276,11 +276,11 @@ bool dp_pr_enable(struct dc_link *link, bool enable)
 bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_context)
 {
 	struct dc *dc = link->ctx->dc;
-	unsigned int panel_inst = 0;
+	unsigned int pr_panel_inst = 0;
 	union dmub_rb_cmd cmd;
 	struct pipe_ctx *pipe_ctx = NULL;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	for (unsigned int i = 0; i < MAX_PIPES; i++) {
@@ -301,7 +301,7 @@ bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_con
 	cmd.pr_copy_settings.header.type = DMUB_CMD__PR;
 	cmd.pr_copy_settings.header.sub_type = DMUB_CMD__PR_COPY_SETTINGS;
 	cmd.pr_copy_settings.header.payload_bytes = sizeof(struct dmub_cmd_pr_copy_settings_data);
-	cmd.pr_copy_settings.data.panel_inst = (uint8_t)panel_inst;
+	cmd.pr_copy_settings.data.panel_inst = (uint8_t)pr_panel_inst;
 	// HW inst
 	cmd.pr_copy_settings.data.aux_inst = replay_context->aux_inst;
 	cmd.pr_copy_settings.data.digbe_inst = replay_context->digbe_inst;
@@ -359,10 +359,10 @@ bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_con
 bool dp_pr_update_state(struct dc_link *link, struct dmub_cmd_pr_update_state_data *update_state_data)
 {
 	struct dc *dc = link->ctx->dc;
-	unsigned int panel_inst = 0;
+	unsigned int pr_panel_inst = 0;
 	union dmub_rb_cmd cmd;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	memset(&cmd, 0, sizeof(cmd));
@@ -371,7 +371,7 @@ bool dp_pr_update_state(struct dc_link *link, struct dmub_cmd_pr_update_state_da
 	cmd.pr_update_state.header.type = DMUB_CMD__PR;
 	cmd.pr_update_state.header.sub_type = DMUB_CMD__PR_UPDATE_STATE;
 	cmd.pr_update_state.header.payload_bytes = sizeof(struct dmub_cmd_pr_update_state_data);
-	cmd.pr_update_state.data.panel_inst = (uint8_t)panel_inst;
+	cmd.pr_update_state.data.panel_inst = (uint8_t)pr_panel_inst;
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 	return true;
@@ -380,10 +380,10 @@ bool dp_pr_update_state(struct dc_link *link, struct dmub_cmd_pr_update_state_da
 bool dp_pr_set_general_cmd(struct dc_link *link, struct dmub_cmd_pr_general_cmd_data *general_cmd_data)
 {
 	struct dc *dc = link->ctx->dc;
-	unsigned int panel_inst = 0;
+	unsigned int pr_panel_inst = 0;
 	union dmub_rb_cmd cmd;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	memset(&cmd, 0, sizeof(cmd));
@@ -392,7 +392,7 @@ bool dp_pr_set_general_cmd(struct dc_link *link, struct dmub_cmd_pr_general_cmd_
 	cmd.pr_general_cmd.header.type = DMUB_CMD__PR;
 	cmd.pr_general_cmd.header.sub_type = DMUB_CMD__PR_GENERAL_CMD;
 	cmd.pr_general_cmd.header.payload_bytes = sizeof(struct dmub_cmd_pr_general_cmd_data);
-	cmd.pr_general_cmd.data.panel_inst = (uint8_t)panel_inst;
+	cmd.pr_general_cmd.data.panel_inst = (uint8_t)pr_panel_inst;
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 	return true;
@@ -401,16 +401,16 @@ bool dp_pr_set_general_cmd(struct dc_link *link, struct dmub_cmd_pr_general_cmd_
 bool dp_pr_get_state(const struct dc_link *link, uint64_t *state)
 {
 	const struct dc *dc = link->ctx->dc;
-	unsigned int panel_inst = 0;
+	unsigned int pr_panel_inst = 0;
 	uint32_t retry_count = 0;
 	uint32_t replay_state = PR_STATE_INVALID;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	do {
 		// Send gpint command and wait for ack
-		if (!dc_wake_and_execute_gpint(dc->ctx, DMUB_GPINT__GET_REPLAY_STATE, (uint16_t)panel_inst,
+		if (!dc_wake_and_execute_gpint(dc->ctx, DMUB_GPINT__GET_REPLAY_STATE, (uint16_t)pr_panel_inst,
 					       &replay_state, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY)) {
 			// Return invalid state when GPINT times out
 			replay_state = PR_STATE_INVALID;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.h
index 5522d5911fd130..e669053e1a8fca 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.h
@@ -28,7 +28,7 @@
 #include "link_service.h"
 
 bool dp_setup_replay(struct dc_link *link, const struct dc_stream_state *stream);
-bool dp_pr_get_panel_inst(const struct dc *dc, const struct dc_link *link, unsigned int *inst_out);
+bool dp_pr_get_pr_panel_inst(const struct dc *dc, const struct dc_link *link, unsigned int *inst_out);
 bool dp_pr_enable(struct dc_link *link, bool enable);
 bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_context);
 bool dp_pr_update_state(struct dc_link *link, struct dmub_cmd_pr_update_state_data *update_state_data);
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
index 16951a9550f25d..e904c7dd320e02 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
@@ -108,7 +108,7 @@ enum dp_panel_mode dp_get_panel_mode(struct dc_link *link)
 			/* alternate scrambler reset is required for Travis
 			 * for the case when external chip does not
 			 * provide sink device id, alternate scrambler
-			 * scheme will  be overriden later by querying
+			 * scheme will  be overridden later by querying
 			 * Encoder features
 			 */
 			if (strncmp(
@@ -124,7 +124,7 @@ enum dp_panel_mode dp_get_panel_mode(struct dc_link *link)
 			/* alternate scrambler reset is required for NUTMEG
 			 * for the case when external chip does not provide
 			 * sink device id, alternate scrambler scheme will
-			 * be overriden later by querying Encoder feature
+			 * be overridden later by querying Encoder feature
 			 */
 			if (strncmp(link->dpcd_caps.branch_dev_name,
 				DP_VGA_LVDS_CONVERTER_ID_3,
@@ -883,6 +883,9 @@ bool edp_setup_psr(struct dc_link *link,
 
 	psr_context->os_request_force_ffu = psr_config->os_request_force_ffu;
 
+	if (dc->debug.psr_phy_force_phy_power_down_up_level_2)
+		psr_context->psr_level.bits.FORCE_P2CPM = 1;
+
 	if (psr) {
 		link->psr_settings.psr_feature_enabled = psr->funcs->psr_copy_settings(psr,
 			link, psr_context, (uint8_t)panel_inst);
@@ -935,19 +938,19 @@ bool edp_set_replay_allow_active(struct dc_link *link, const bool *allow_active,
 {
 	struct dc  *dc = link->ctx->dc;
 	struct dmub_replay *replay = dc->res_pool->replay;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 
 	if (replay == NULL && force_static)
 		return false;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	/* Set power optimization flag */
 	if (power_opts && link->replay_settings.replay_power_opt_active != *power_opts) {
 		if (replay != NULL && link->replay_settings.replay_feature_enabled &&
 			replay->funcs->replay_set_power_opt) {
-			replay->funcs->replay_set_power_opt(replay, *power_opts, (uint8_t)panel_inst);
+			replay->funcs->replay_set_power_opt(replay, *power_opts, (uint8_t)pr_panel_inst);
 			link->replay_settings.replay_power_opt_active = *power_opts;
 		}
 	}
@@ -957,7 +960,7 @@ bool edp_set_replay_allow_active(struct dc_link *link, const bool *allow_active,
 		// TODO: Handle mux change case if force_static is set
 		// If force_static is set, just change the replay_allow_active state directly
 		if (replay != NULL && link->replay_settings.replay_feature_enabled)
-			replay->funcs->replay_enable(replay, *allow_active, wait, (uint8_t)panel_inst);
+			replay->funcs->replay_enable(replay, *allow_active, wait, (uint8_t)pr_panel_inst);
 		link->replay_settings.replay_allow_active = *allow_active;
 	}
 
@@ -968,14 +971,14 @@ bool edp_get_replay_state(const struct dc_link *link, uint64_t *state)
 {
 	struct dc  *dc = link->ctx->dc;
 	struct dmub_replay *replay = dc->res_pool->replay;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 	enum replay_state pr_state = REPLAY_STATE_0;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	if (replay != NULL && link->replay_settings.replay_feature_enabled)
-		replay->funcs->replay_get_state(replay, &pr_state, (uint8_t)panel_inst);
+		replay->funcs->replay_get_state(replay, &pr_state, (uint8_t)pr_panel_inst);
 	*state = pr_state;
 
 	return true;
@@ -988,7 +991,7 @@ bool edp_setup_freesync_replay(struct dc_link *link, const struct dc_stream_stat
 	struct dc *dc;
 	struct dmub_replay *replay;
 	int i;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 	struct replay_context replay_context = { 0 };
 	unsigned int lineTimeInNs = 0;
 
@@ -1018,7 +1021,7 @@ bool edp_setup_freesync_replay(struct dc_link *link, const struct dc_stream_stat
 	if (!replay)
 		return false;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	replay_context.aux_inst = link_get_ddc_aux_inst(link);
@@ -1046,7 +1049,7 @@ bool edp_setup_freesync_replay(struct dc_link *link, const struct dc_stream_stat
 	replay_context.os_request_force_ffu = link->replay_settings.config.os_request_force_ffu;
 
 	link->replay_settings.replay_feature_enabled =
-			replay->funcs->replay_copy_settings(replay, link, &replay_context, (uint8_t)panel_inst);
+			replay->funcs->replay_copy_settings(replay, link, &replay_context, (uint8_t)pr_panel_inst);
 	if (link->replay_settings.replay_feature_enabled) {
 
 		replay_config.bits.FREESYNC_PANEL_REPLAY_MODE = 1;
@@ -1091,13 +1094,13 @@ bool edp_send_replay_cmd(struct dc_link *link,
 {
 	struct dc *dc = link->ctx->dc;
 	struct dmub_replay *replay = dc->res_pool->replay;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 
 	if (!replay)
 		return false;
 
-	if (dp_pr_get_panel_inst(dc, link, &panel_inst))
-		cmd_data->panel_inst = (uint8_t)panel_inst;
+	if (dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
+		cmd_data->panel_inst = (uint8_t)pr_panel_inst;
 	else {
 		DC_LOG_DC("%s(): get edp panel inst fail ", __func__);
 		return false;
@@ -1112,17 +1115,17 @@ bool edp_set_coasting_vtotal(struct dc_link *link, uint32_t coasting_vtotal, uin
 {
 	struct dc *dc = link->ctx->dc;
 	struct dmub_replay *replay = dc->res_pool->replay;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 
 	if (!replay)
 		return false;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	if (coasting_vtotal && (link->replay_settings.coasting_vtotal != coasting_vtotal ||
 		link->replay_settings.frame_skip_number != frame_skip_number)) {
-		replay->funcs->replay_set_coasting_vtotal(replay, coasting_vtotal, (uint8_t)panel_inst, frame_skip_number);
+		replay->funcs->replay_set_coasting_vtotal(replay, coasting_vtotal, (uint8_t)pr_panel_inst, frame_skip_number);
 		link->replay_settings.coasting_vtotal = coasting_vtotal;
 		link->replay_settings.frame_skip_number = frame_skip_number;
 	}
@@ -1135,30 +1138,52 @@ bool edp_replay_residency(const struct dc_link *link,
 {
 	struct dc  *dc = link->ctx->dc;
 	struct dmub_replay *replay = dc->res_pool->replay;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	if (!residency)
 		return false;
 
 	if (replay != NULL && link->replay_settings.replay_feature_enabled)
-		replay->funcs->replay_residency(replay, (uint8_t)panel_inst, residency, is_start, mode);
+		replay->funcs->replay_residency(replay, (uint8_t)pr_panel_inst, residency, is_start, mode);
 	else
 		*residency = 0;
 
 	return true;
 }
 
+bool edp_replay_get_cumulative_residency(const struct dc_link *link,
+	uint32_t *residency_milli_pct)
+{
+	struct dc *dc = link->ctx->dc;
+	struct dmub_replay *replay = dc->res_pool->replay;
+	unsigned int panel_inst;
+
+	if (!dp_pr_get_pr_panel_inst(dc, link, &panel_inst))
+		return false;
+
+	if (replay && link->replay_settings.replay_feature_enabled &&
+	    replay->funcs->replay_get_cumulative_residency)
+		return replay->funcs->replay_get_cumulative_residency(replay,
+			(uint8_t)panel_inst, residency_milli_pct);
+
+	if (residency_milli_pct)
+		*residency_milli_pct = 0;
+
+	return true;
+}
+
+
 bool edp_set_replay_power_opt_and_coasting_vtotal(struct dc_link *link,
 	const unsigned int *power_opts, uint32_t coasting_vtotal, uint16_t frame_skip_number)
 {
 	struct dc  *dc = link->ctx->dc;
 	struct dmub_replay *replay = dc->res_pool->replay;
-	unsigned int panel_inst;
+	unsigned int pr_panel_inst;
 
-	if (!dp_pr_get_panel_inst(dc, link, &panel_inst))
+	if (!dp_pr_get_pr_panel_inst(dc, link, &pr_panel_inst))
 		return false;
 
 	/* Only both power and coasting vtotal changed, this func could return true */
@@ -1169,7 +1194,7 @@ bool edp_set_replay_power_opt_and_coasting_vtotal(struct dc_link *link,
 		if (link->replay_settings.replay_feature_enabled &&
 			replay->funcs->replay_set_power_opt_and_coasting_vtotal) {
 			replay->funcs->replay_set_power_opt_and_coasting_vtotal(replay,
-				*power_opts, (uint8_t)panel_inst, coasting_vtotal, frame_skip_number);
+				*power_opts, (uint8_t)pr_panel_inst, coasting_vtotal, frame_skip_number);
 			link->replay_settings.replay_power_opt_active = *power_opts;
 			link->replay_settings.coasting_vtotal = coasting_vtotal;
 			link->replay_settings.frame_skip_number = frame_skip_number;
@@ -1301,3 +1326,33 @@ void edp_set_panel_assr(struct dc_link *link, struct pipe_ctx *pipe_ctx,
 			*panel_mode = DP_PANEL_MODE_DEFAULT;
 	}
 }
+
+void edp_set_panel_polarity_enabled(const struct dc_link *link, bool enable)
+{
+	struct dc *dc = link->ctx->dc;
+	struct dc_dmub_srv *dc_dmub_srv = dc->ctx->dmub_srv;
+
+	if (dc_dmub_srv)
+		dc_dmub_srv_panel_polarity_set_enable(dc_dmub_srv, 0, enable);
+}
+
+void edp_panel_polarity_reset(struct dc_link *link)
+{
+	struct dc *dc = link->ctx->dc;
+	struct dc_dmub_srv *dc_dmub_srv = dc->ctx->dmub_srv;
+
+	if (dc_dmub_srv)
+		dc_dmub_srv_panel_polarity_reset(dc_dmub_srv, 0);
+}
+
+bool edp_get_panel_polarity(struct dc_link *link, int32_t *polarity)
+{
+	struct dc *dc = link->ctx->dc;
+	struct dc_dmub_srv *dc_dmub_srv = dc->ctx->dmub_srv;
+	bool ret = false;
+
+	if (dc_dmub_srv)
+		ret = dc_dmub_srv_panel_polarity_get_polarity(dc_dmub_srv, 0, polarity);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
index 8fdb76d9953ec5..0082f5e91878dd 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.h
@@ -60,6 +60,8 @@ bool edp_send_replay_cmd(struct dc_link *link,
 bool edp_set_coasting_vtotal(struct dc_link *link, uint32_t coasting_vtotal, uint16_t frame_skip_number);
 bool edp_replay_residency(const struct dc_link *link,
 	unsigned int *residency, const bool is_start, const enum pr_residency_mode mode);
+bool edp_replay_get_cumulative_residency(const struct dc_link *link,
+	uint32_t *residency_milli_pct);
 bool edp_get_replay_state(const struct dc_link *link, uint64_t *state);
 bool edp_set_replay_power_opt_and_coasting_vtotal(struct dc_link *link,
 	const unsigned int *power_opts, uint32_t coasting_vtotal, uint16_t frame_skip_number);
@@ -77,4 +79,8 @@ bool edp_setup_freesync_replay(struct dc_link *link, const struct dc_stream_stat
 void edp_set_panel_power(struct dc_link *link, bool powerOn);
 void edp_set_panel_assr(struct dc_link *link, struct pipe_ctx *pipe_ctx,
 		enum dp_panel_mode *panel_mode, bool enable);
+
+void edp_set_panel_polarity_enabled(const struct dc_link *link, bool enable);
+void edp_panel_polarity_reset(struct dc_link *link);
+bool edp_get_panel_polarity(struct dc_link *link, int32_t *polarity);
 #endif /* __DC_LINK_EDP_POWER_CONTROL_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c
index 714168e1ece6d9..7f930097da1809 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_hdmi_frl.c
@@ -528,10 +528,10 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 
 		FRL_INFO("FRL LINK TRAINING:  Poll for FLT_UPDATE.\n");
 		/*LTS:3: Start Link Training*/
-		/*Start FLT Timer = 200 ms*/
+		/*Start FLT Timer = 200 ms, or 300ms if link rate >= 16Gbps*/
 		num_polls = 0;
-		if (flt_no_timeout)
-			max_polls = 500;
+		if (link_settings->frl_link_rate >= HDMI_FRL_LINK_RATE_16GBPS)
+			max_polls = 155;
 
 		while (num_polls < max_polls) {
 			flt_poll_cur_time = dm_get_timestamp(ddc_service->ctx);
@@ -572,7 +572,7 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 				FRL_INFO("FRL LINK TRAINING:  TxFFE = %d.\n", current_FFE);
 				override_FFE = false;
 			}
-			if (scdc_update.fields.FLT_UPDATE) {
+			if (scdc_update.fields.FLT_UPDATE || flt_no_timeout) {
 				offset = HDMI_SCDC_LTP_REQ;
 				link_query_ddc_data(ddc_service, slave_address,
 								&offset, sizeof(offset), ltp_req.byte,
@@ -680,21 +680,19 @@ static enum link_result hdmi_frl_perform_link_training(struct ddc_service *ddc_s
 				/* Workaround for DEDCN3AG-111
 				 * HDMI-FRL Incorrect Serialization Order for LTP4
 				 */
+				if (flt_no_timeout) {
+					return LINK_RESULT_SUCCESS;
+				}
 			}
-
-		}
-		if (flt_no_timeout) {
-			return LINK_RESULT_SUCCESS;
-		} else {
-			FRL_INFO("FRL LINK TRAINING:  FAILED - Timeout waiting for FLT_UPDATE to be set by sink.\n");
-			write_buffer[0] = HDMI_SCDC_CONFIG_1;
-			/*FRL_RATE*/
-			write_buffer[1] = HDMI_FRL_LINK_RATE_DISABLE | (0 << 4);
-			link_query_ddc_data(ddc_service, slave_address,
-					write_buffer, sizeof(write_buffer), NULL, 0);
-			hdmi_frl_LTS_clear_Link_Setting(ddc_service);
-			result = LINK_RESULT_TIMEOUT;
 		}
+		FRL_INFO("FRL LINK TRAINING:  FAILED - Timeout waiting for FLT_UPDATE to be set by sink.\n");
+		write_buffer[0] = HDMI_SCDC_CONFIG_1;
+		/*FRL_RATE*/
+		write_buffer[1] = HDMI_FRL_LINK_RATE_DISABLE | (0 << 4);
+		link_query_ddc_data(ddc_service, slave_address,
+				write_buffer, sizeof(write_buffer), NULL, 0);
+		hdmi_frl_LTS_clear_Link_Setting(ddc_service);
+		result = LINK_RESULT_TIMEOUT;
 	} else {
 		FRL_INFO("FRL LINK TRAINING:  FAILED - FLT_READY not set by sink.\n");
 		result = LINK_RESULT_TIMEOUT;
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
index 0f2294bedcc33e..f9bd1d6e16423a 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
@@ -503,6 +503,7 @@ void dcn10_mpc_construct(struct dcn10_mpc *mpc10,
 	int i;
 
 	mpc10->base.ctx = ctx;
+	mpc10->base.inst = 0;
 
 	mpc10->base.funcs = &dcn10_mpc_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c
index 0e09d073ab2924..905d4f23f92a38 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn20/dcn20_mpc.c
@@ -588,6 +588,7 @@ void dcn20_mpc_construct(struct dcn20_mpc *mpc20,
 	int i;
 
 	mpc20->base.ctx = ctx;
+	mpc20->base.inst = 0;
 
 	mpc20->base.funcs = &dcn20_mpc_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c
index 9cc6fd38da19f5..f3213ecb174500 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c
@@ -1589,6 +1589,7 @@ void dcn30_mpc_construct(struct dcn30_mpc *mpc30,
 	int i;
 
 	mpc30->base.ctx = ctx;
+	mpc30->base.inst = 0;
 
 	mpc30->base.funcs = &dcn30_mpc_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c
index 1f15ada109b6af..61a1e3f0657555 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn32/dcn32_mpc.c
@@ -1036,6 +1036,7 @@ void dcn32_mpc_construct(struct dcn30_mpc *mpc30,
 	int i;
 
 	mpc30->base.ctx = ctx;
+	mpc30->base.inst = 0;
 
 	mpc30->base.funcs = &dcn32_mpc_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
index df92d8008fe604..e594540906914e 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
@@ -680,6 +680,7 @@ void dcn401_mpc_construct(struct dcn401_mpc *mpc401,
 	int i;
 
 	mpc401->base.ctx = ctx;
+	mpc401->base.inst = 0;
 
 	mpc401->base.funcs = &dcn401_mpc_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
index 5a888e1e1cc56c..37d024cfb3fb82 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
@@ -63,680 +63,14 @@ void mpc42_update_blending(
 	mpcc->blnd_cfg = *blnd_cfg;
 }
 
-/* RMCM Shaper functions */
-void mpc42_power_on_rmcm_shaper_3dlut(
-	struct mpc *mpc,
-	uint32_t mpcc_id,
-	bool power_on)
-{
-	uint32_t power_status_shaper = 2;
-	uint32_t power_status_3dlut  = 2;
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-	int max_retries = 10;
-
-	REG_SET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], 0,
-		MPC_RMCM_3DLUT_MEM_PWR_DIS, power_on == true ? 0 : 1);
-	REG_SET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], 0,
-		MPC_RMCM_SHAPER_MEM_PWR_DIS, power_on == true ? 0 : 1);
-	/* wait for memory to fully power up */
-	if (power_on && mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) {
-		REG_WAIT(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_SHAPER_MEM_PWR_STATE, 0, 1, max_retries);
-		REG_WAIT(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_3DLUT_MEM_PWR_STATE, 0, 1, max_retries);
-	}
-
-	/*read status is not mandatory, it is just for debugging*/
-	REG_GET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_SHAPER_MEM_PWR_STATE, &power_status_shaper);
-	REG_GET(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_3DLUT_MEM_PWR_STATE, &power_status_3dlut);
-
-	if (power_status_shaper != 0 && power_on == true)
-		BREAK_TO_DEBUGGER();
-
-	if (power_status_3dlut != 0 && power_on == true)
-		BREAK_TO_DEBUGGER();
-}
-
-void mpc42_configure_rmcm_shaper_lut(
-	struct mpc *mpc,
-	bool is_ram_a,
-	uint32_t mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_G_B[mpcc_id],
-		MPC_RMCM_SHAPER_SCALE_B, 0x7000);
-	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_G_B[mpcc_id],
-		MPC_RMCM_SHAPER_SCALE_G, 0x7000);
-	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_R[mpcc_id],
-		MPC_RMCM_SHAPER_SCALE_R, 0x7000);
-	REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
-			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
-	REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
-			MPC_RMCM_SHAPER_LUT_WRITE_SEL, is_ram_a == true ? 0:1);
-	REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_INDEX, 0);
-}
-
-void mpc42_program_rmcm_shaper_luta_settings(
-	struct mpc *mpc,
-	const struct pwl_params *params,
-	uint32_t mpcc_id)
-{
-	const struct gamma_curve *curve;
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_B[mpcc_id], 0,
-		MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
-		MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_G[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].green.custom_float_x,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_R[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].red.custom_float_x,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
-
-	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_B[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_G[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].green.custom_float_x,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].green.custom_float_y);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_R[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].red.custom_float_x,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].red.custom_float_y);
-
-	curve = params->arr_curve_points;
-	if (curve) {
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_0_1[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_2_3[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_4_5[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_6_7[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_8_9[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_10_11[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_12_13[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_14_15[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_16_17[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_18_19[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_20_21[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_22_23[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_24_25[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_26_27[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_28_29[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_30_31[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_32_33[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-	}
-}
-
-
-void mpc42_program_rmcm_shaper_lutb_settings(
-	struct mpc *mpc,
-	const struct pwl_params *params,
-	uint32_t mpcc_id)
-{
-	const struct gamma_curve *curve;
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_B[mpcc_id], 0,
-		MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
-		MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_G[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].green.custom_float_x,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_R[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].red.custom_float_x,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
-
-	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_B[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_G[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].green.custom_float_x,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].green.custom_float_y);
-	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_R[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].red.custom_float_x,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].red.custom_float_y);
-
-	curve = params->arr_curve_points;
-	if (curve) {
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_0_1[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_2_3[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_4_5[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_6_7[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_8_9[mpcc_id], 0,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_10_11[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_12_13[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_14_15[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_16_17[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_18_19[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_20_21[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_22_23[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_24_25[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_26_27[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_28_29[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_30_31[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-		curve += 2;
-		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_32_33[mpcc_id], 0,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-	}
-}
-
-void mpc42_program_rmcm_shaper_lut(
-	struct mpc *mpc,
-	const struct pwl_result_data *rgb,
-	uint32_t num,
-	uint32_t mpcc_id)
-{
-	uint32_t i, red, green, blue;
-	uint32_t  red_delta, green_delta, blue_delta;
-	uint32_t  red_value, green_value, blue_value;
-
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	for (i = 0; i < num; i++) {
-
-		red   = rgb[i].red_reg;
-		green = rgb[i].green_reg;
-		blue  = rgb[i].blue_reg;
-
-		red_delta   = rgb[i].delta_red_reg;
-		green_delta = rgb[i].delta_green_reg;
-		blue_delta  = rgb[i].delta_blue_reg;
-
-		red_value   = ((red_delta   & 0x3ff) << 14) | (red   & 0x3fff);
-		green_value = ((green_delta & 0x3ff) << 14) | (green & 0x3fff);
-		blue_value  = ((blue_delta  & 0x3ff) << 14) | (blue  & 0x3fff);
-
-		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_DATA, red_value);
-		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_DATA, green_value);
-		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[mpcc_id], 0, MPC_RMCM_SHAPER_LUT_DATA, blue_value);
-	}
-}
-
-void mpc42_enable_3dlut_fl(struct mpc *mpc, bool enable, int mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	//if enabled cho0se mpc 0, else: off (default value)
-	REG_UPDATE(MPC_RMCM_CNTL[mpcc_id], MPC_RMCM_CNTL, enable ? 0 : 0xF); //0xF is not connected
-
-	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPC_RMCM_3DLUT_WRITE_EN_MASK, 0);
-
-	REG_UPDATE(MPC_RMCM_MEM_PWR_CTRL[mpcc_id], MPC_RMCM_3DLUT_MEM_PWR_DIS, enable ? 0 : 3);
-}
-
-void mpc42_update_3dlut_fast_load_select(struct mpc *mpc, int mpcc_id, int hubp_idx)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	REG_SET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[mpcc_id], 0,
-		MPC_RMCM_3DLUT_FL_SEL,
-		hubp_idx);
-}
-
-void mpc42_populate_rmcm_lut(struct mpc *mpc, const union mcm_lut_params params,
-	bool lut_bank_a, int mpcc_id)
-{
-	const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B;
-	const struct pwl_params *lut_shaper = params.pwl;
-
-	if (lut_shaper == NULL)
-		return;
-	if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
-		mpc42_power_on_rmcm_shaper_3dlut(mpc, mpcc_id, true);
-
-	mpc42_configure_rmcm_shaper_lut(mpc, next_mode == LUT_RAM_A, mpcc_id);
-
-	if (next_mode == LUT_RAM_A)
-		mpc42_program_rmcm_shaper_luta_settings(mpc, lut_shaper, mpcc_id);
-	else
-		mpc42_program_rmcm_shaper_lutb_settings(mpc, lut_shaper, mpcc_id);
-
-	mpc42_program_rmcm_shaper_lut(
-			mpc, lut_shaper->rgb_resulted, lut_shaper->hw_points_num, mpcc_id);
-
-	mpc42_power_on_rmcm_shaper_3dlut(mpc, mpcc_id, false);
-}
-
-void mpc42_program_rmcm_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id,
-	bool lut_bank_a, bool enabled, int mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	switch (id) {
-	case MCM_LUT_3DLUT:
-		REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_MODE,
-			(!enabled) ? 0 :
-			(lut_bank_a) ? 1 : 2);
-
-		REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
-			MPC_RMCM_3DLUT_RAM_SEL,
-			(lut_bank_a) ? 0 : 1);
-		break;
-	case MCM_LUT_SHAPER:
-		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
-			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
-
-		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
-			MPC_RMCM_SHAPER_LUT_WRITE_SEL,
-			lut_bank_a == true ? 0:1);
-
-		REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[mpcc_id], 0,
-			MPC_RMCM_SHAPER_LUT_INDEX, 0);
-		break;
-	default:
-		break;
-	}
-}
-
-void mpc42_program_lut_mode(struct mpc *mpc,
-	bool enable,
-	bool lut_bank_a,
-	int mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	if (enable) {
-		REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
-	} else {
-		REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, 0);
-	}
-}
-
-static uint32_t mpc42_get_rmcm_3dlut_width(
-		const enum dc_cm_lut_size size)
-{
-	uint32_t width = 0;
-
-	switch (size) {
-	case CM_LUT_SIZE_333333:
-		width = 2;
-		break;
-	case CM_LUT_SIZE_171717:
-	default:
-		width = 0;
-		break;
-	}
-
-	return width;
-}
-
-void mpc42_program_rmcm_3dlut_size(struct mpc *mpc,
-		const enum dc_cm_lut_size size,
-		int mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-	uint32_t width = mpc42_get_rmcm_3dlut_width(size);
-
-	REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id],
-			MPC_RMCM_3DLUT_SIZE, width);
-}
 
-void mpc42_program_rmcm_3dlut_fast_load_bias_scale(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_R[mpcc_id],
-		MPC_RMCM_3DLUT_OUT_OFFSET_R, bias,
-		MPC_RMCM_3DLUT_OUT_SCALE_R, scale);
-
-	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_G[mpcc_id],
-		MPC_RMCM_3DLUT_OUT_OFFSET_G, bias,
-		MPC_RMCM_3DLUT_OUT_SCALE_G, scale);
-
-	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_B[mpcc_id],
-		MPC_RMCM_3DLUT_OUT_OFFSET_B, bias,
-		MPC_RMCM_3DLUT_OUT_SCALE_B, scale);
-}
-
-void mpc42_program_rmcm_bit_depth(struct mpc *mpc, uint16_t bit_depth, int mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPC_RMCM_3DLUT_WRITE_EN_MASK, 0xF);
-
-	//program bit_depth
-	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
-		MPC_RMCM_3DLUT_30BIT_EN,
-		(bit_depth == 10) ? 1 : 0);
-}
-
-void mpc42_set_fl_config(
-	struct mpc *mpc,
-	struct mpc_fl_3dlut_config *cfg,
-	int mpcc_id)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
-	uint32_t width = mpc42_get_rmcm_3dlut_width(cfg->size);
-	/*
-	From: Jie Zhou
-
-		To program any of the memories content.  The following sequence is used.
-	Set the MPCC_OGAM/SHAPER/3DLUT/1DLUT_PWR_DIS to 1 (Only need to set the one
-	that is being programmed) Set DISPCLK_G_PIPE<i>_GATE_DISABLE to 1 for the
-	MPCC pipe that’s being used, so the memory’s clock is ungated. Program the
-	target memory. Set the MPCC_OGAM/SHAPER/3DLUT/1DLUT_PWR_DIS back to 0.
-	Set DISPCLK_G_PIPE<i>_GATE_DISABLE back to 0
-	*/
-
-	//disconnect fl from mpc
-	REG_SET(MPCC_MCM_3DLUT_FAST_LOAD_SELECT[mpcc_id], 0,
-		MPCC_MCM_3DLUT_FL_SEL, 0xF);
-
-	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
-		MPC_RMCM_3DLUT_WRITE_EN_MASK, 0xF);
-
-	//program bit_depth
-	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
-		MPC_RMCM_3DLUT_30BIT_EN, (cfg->bit_depth == 10) ? 1 : 0);
-
-	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
-		MPC_RMCM_3DLUT_RAM_SEL, (cfg->select_lut_bank_a) ? 0 : 1);
-
-	//bias and scale
-	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_R[mpcc_id],
-		MPC_RMCM_3DLUT_OUT_OFFSET_R, cfg->bias,
-		MPC_RMCM_3DLUT_OUT_SCALE_R, cfg->scale);
-
-	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_G[mpcc_id],
-		MPC_RMCM_3DLUT_OUT_OFFSET_G, cfg->bias,
-		MPC_RMCM_3DLUT_OUT_SCALE_G, cfg->scale);
-
-	REG_UPDATE_2(MPC_RMCM_3DLUT_OUT_OFFSET_B[mpcc_id],
-		MPC_RMCM_3DLUT_OUT_OFFSET_B, cfg->bias,
-		MPC_RMCM_3DLUT_OUT_SCALE_B, cfg->scale);
-
-	//width
-	REG_UPDATE_2(MPC_RMCM_3DLUT_MODE[mpcc_id],
-		MPC_RMCM_3DLUT_SIZE, width,
-		MPC_RMCM_3DLUT_MODE, (!cfg->enabled) ? 0 : (cfg->select_lut_bank_a) ? 1 : 2);
-
-	//connect to hubp
-	REG_SET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[mpcc_id], 0,
-		MPC_RMCM_3DLUT_FL_SEL, cfg->hubp_index);
-
-	//ENABLE
-	//if enabled pick mpc 0, else: off (0xF)
-	//in future we'll select specific MPC
-	REG_UPDATE(MPC_RMCM_CNTL[mpcc_id], MPC_RMCM_CNTL, cfg->enabled ? 0 : 0xF);
-}
-
-void mpc42_get_rmcm_3dlut_mode(
-	struct mpc *mpc,
-	int mpcc_id,
-	bool *enable,
-	bool *lut_bank_a)
-{
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-	uint32_t mode_current   = 0;
-
-	REG_GET(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_MODE_CURRENT, &mode_current);
-
-	/* MPC_RMCM_3DLUT_MODE encoding:
-	 *   0 -> disabled, 1 -> bank A, 2 -> bank B
-	 */
-	*enable     = mode_current != 0;
-	*lut_bank_a = mode_current != 2;
-}
 
 void mpc42_read_mpcc_state(
 		struct mpc *mpc,
 		int mpcc_inst,
 		struct mpcc_state *s)
 {
-	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-
 	mpc1_read_mpcc_state(mpc, mpcc_inst, s);
-
-	if (mpcc_inst < 2) {
-		/* RMCM 3DLUT Status */
-		REG_GET_4(MPC_RMCM_MEM_PWR_CTRL[mpcc_inst], MPC_RMCM_3DLUT_MEM_PWR_FORCE, &s->rmcm_regs.rmcm_3dlut_mem_pwr_force,
-				MPC_RMCM_3DLUT_MEM_PWR_DIS, &s->rmcm_regs.rmcm_3dlut_mem_pwr_dis,
-				MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE, &s->rmcm_regs.rmcm_3dlut_mem_pwr_mode,
-				MPC_RMCM_3DLUT_MEM_PWR_STATE, &s->rmcm_regs.rmcm_3dlut_mem_pwr_state);
-
-		REG_GET_3(MPC_RMCM_3DLUT_MODE[mpcc_inst], MPC_RMCM_3DLUT_SIZE, &s->rmcm_regs.rmcm_3dlut_size,
-				MPC_RMCM_3DLUT_MODE, &s->rmcm_regs.rmcm_3dlut_mode,
-				MPC_RMCM_3DLUT_MODE_CURRENT, &s->rmcm_regs.rmcm_3dlut_mode_cur);
-
-		REG_GET_4(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[mpcc_inst], MPC_RMCM_3DLUT_READ_SEL, &s->rmcm_regs.rmcm_3dlut_read_sel,
-				MPC_RMCM_3DLUT_30BIT_EN, &s->rmcm_regs.rmcm_3dlut_30bit_en,
-				MPC_RMCM_3DLUT_WRITE_EN_MASK, &s->rmcm_regs.rmcm_3dlut_wr_en_mask,
-				MPC_RMCM_3DLUT_RAM_SEL, &s->rmcm_regs.rmcm_3dlut_ram_sel);
-
-		REG_GET(MPC_RMCM_3DLUT_OUT_NORM_FACTOR[mpcc_inst], MPC_RMCM_3DLUT_OUT_NORM_FACTOR, &s->rmcm_regs.rmcm_3dlut_out_norm_factor);
-
-		REG_GET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[mpcc_inst], MPC_RMCM_3DLUT_FL_SEL, &s->rmcm_regs.rmcm_3dlut_fl_sel);
-
-		REG_GET_2(MPC_RMCM_3DLUT_OUT_OFFSET_R[mpcc_inst], MPC_RMCM_3DLUT_OUT_OFFSET_R, &s->rmcm_regs.rmcm_3dlut_out_offset_r,
-				MPC_RMCM_3DLUT_OUT_SCALE_R, &s->rmcm_regs.rmcm_3dlut_out_scale_r);
-
-		REG_GET_3(MPC_RMCM_3DLUT_FAST_LOAD_STATUS[mpcc_inst], MPC_RMCM_3DLUT_FL_DONE, &s->rmcm_regs.rmcm_3dlut_fl_done,
-				MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, &s->rmcm_regs.rmcm_3dlut_fl_soft_underflow,
-				MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, &s->rmcm_regs.rmcm_3dlut_fl_hard_underflow);
-
-		/* RMCM Shaper Status */
-		REG_GET_4(MPC_RMCM_MEM_PWR_CTRL[mpcc_inst], MPC_RMCM_SHAPER_MEM_PWR_FORCE, &s->rmcm_regs.rmcm_shaper_mem_pwr_force,
-				MPC_RMCM_SHAPER_MEM_PWR_DIS, &s->rmcm_regs.rmcm_shaper_mem_pwr_dis,
-				MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE, &s->rmcm_regs.rmcm_shaper_mem_pwr_mode,
-				MPC_RMCM_SHAPER_MEM_PWR_STATE, &s->rmcm_regs.rmcm_shaper_mem_pwr_state);
-
-		REG_GET_2(MPC_RMCM_SHAPER_CONTROL[mpcc_inst], MPC_RMCM_SHAPER_LUT_MODE, &s->rmcm_regs.rmcm_shaper_lut_mode,
-				MPC_RMCM_SHAPER_MODE_CURRENT, &s->rmcm_regs.rmcm_shaper_mode_cur);
-
-		REG_GET_2(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_inst], MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, &s->rmcm_regs.rmcm_shaper_lut_write_en_mask,
-				MPC_RMCM_SHAPER_LUT_WRITE_SEL, &s->rmcm_regs.rmcm_shaper_lut_write_sel);
-
-		REG_GET(MPC_RMCM_SHAPER_OFFSET_B[mpcc_inst], MPC_RMCM_SHAPER_OFFSET_B, &s->rmcm_regs.rmcm_shaper_offset_b);
-
-		REG_GET(MPC_RMCM_SHAPER_SCALE_G_B[mpcc_inst], MPC_RMCM_SHAPER_SCALE_B, &s->rmcm_regs.rmcm_shaper_scale_b);
-
-		REG_GET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_B[mpcc_inst], MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_start_b,
-				MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_start_seg_b);
-
-		REG_GET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_B[mpcc_inst], MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_end_b,
-				MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, &s->rmcm_regs.rmcm_shaper_rama_exp_region_end_base_b);
-
-		REG_GET(MPC_RMCM_CNTL[mpcc_inst], MPC_RMCM_CNTL, &s->rmcm_regs.rmcm_cntl);
-	}
 }
 
 static const struct mpc_funcs dcn42_mpc_funcs = {
@@ -774,19 +108,6 @@ static const struct mpc_funcs dcn42_mpc_funcs = {
 	.program_lut_read_write_control = mpc401_program_lut_read_write_control,
 	.program_lut_mode = mpc401_program_lut_mode,
 	.get_lut_mode = mpc401_get_lut_mode,
-	.rmcm = {
-		.enable_3dlut_fl = mpc42_enable_3dlut_fl,
-		.update_3dlut_fast_load_select = mpc42_update_3dlut_fast_load_select,
-		.program_lut_read_write_control = mpc42_program_rmcm_lut_read_write_control,
-		.program_lut_mode = mpc42_program_lut_mode,
-		.program_3dlut_size = mpc42_program_rmcm_3dlut_size,
-		.program_bias_scale = mpc42_program_rmcm_3dlut_fast_load_bias_scale,
-		.program_bit_depth = mpc42_program_rmcm_bit_depth,
-		.power_on_shaper_3dlut = mpc42_power_on_rmcm_shaper_3dlut,
-		.populate_lut = mpc42_populate_rmcm_lut,
-		.fl_3dlut_configure = mpc42_set_fl_config,
-		.get_3dlut_mode = mpc42_get_rmcm_3dlut_mode,
-	},
 };
 
 void dcn42_mpc_construct(struct dcn42_mpc *mpc42,
@@ -800,6 +121,7 @@ void dcn42_mpc_construct(struct dcn42_mpc *mpc42,
 	int i;
 
 	mpc42->base.ctx = ctx;
+	mpc42->base.inst = 0;
 
 	mpc42->base.funcs = &dcn42_mpc_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
index af3b90af5396ff..21338083208781 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
@@ -196,327 +196,7 @@
 	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM_3DLUT_OUT_OFFSET_G, mask_sh),\
 	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM_3DLUT_OUT_SCALE_G, mask_sh),\
 	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM_3DLUT_OUT_OFFSET_B, mask_sh),\
-	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM_3DLUT_OUT_SCALE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_LUT_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_MODE_CURRENT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R, MPC_RMCM_SHAPER_OFFSET_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G, MPC_RMCM_SHAPER_OFFSET_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B, MPC_RMCM_SHAPER_OFFSET_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R, MPC_RMCM_SHAPER_SCALE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX, MPC_RMCM_SHAPER_LUT_INDEX, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA, MPC_RMCM_SHAPER_LUT_DATA, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_SEL, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_SIZE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE_CURRENT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_INDEX, MPC_RMCM_3DLUT_INDEX, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA0, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA1, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT, MPC_RMCM_3DLUT_DATA_30BIT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_WRITE_EN_MASK, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_RAM_SEL, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_30BIT_EN, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_READ_SEL, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR, MPC_RMCM_3DLUT_OUT_NORM_FACTOR, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_OFFSET_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_SCALE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_OFFSET_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_SCALE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_OFFSET_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_SCALE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE_CURRENT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C11_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C12_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C13_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C14_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C21_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C22_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C23_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C24_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C31_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C32_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C33_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C34_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C11_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C12_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C13_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C14_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C21_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C22_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C23_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C24_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C31_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C32_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C33_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C34_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_FORCE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_DIS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_FORCE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_DIS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_STATE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_STATE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT, MPC_RMCM_3DLUT_FL_SEL, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_DONE, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_CNTL, MPC_RMCM_CNTL, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_INDEX, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_WRITE_EN, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA, MPC_RMCM_TEST_DEBUG_DATA, mask_sh)
-
-#define MPC_RMCM_REG_LIST_DCN42(inst)\
-	SRII(MPC_RMCM_SHAPER_CONTROL, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_OFFSET_R, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_OFFSET_G, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_OFFSET_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_SCALE_R, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_LUT_INDEX, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_LUT_DATA, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_MODE, MPC_RMCM, inst), /*TODO: may need to add other 3DLUT regs*/\
-	SRII(MPC_RMCM_3DLUT_INDEX, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_DATA, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_DATA_30BIT, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_OUT_NORM_FACTOR, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_FAST_LOAD_SELECT, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_CNTL, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM, inst),\
-	SRII(MPC_RMCM_TEST_DEBUG_DATA, MPC_RMCM, inst)
-
+	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM_3DLUT_OUT_SCALE_B, mask_sh)
 
 #define MPC_REG_LIST_DCN42(inst) \
 	MPC_REG_LIST_DCN4_01_RI(inst),\
@@ -529,328 +209,11 @@
 	type MPCC_MCM_3DLUT_OUT_OFFSET_G;\
 	type MPCC_MCM_3DLUT_OUT_SCALE_G;\
 	type MPCC_MCM_3DLUT_OUT_OFFSET_B;\
-	type MPCC_MCM_3DLUT_OUT_SCALE_B;\
-	type MPC_RMCM_SHAPER_LUT_MODE;\
-	type MPC_RMCM_SHAPER_MODE_CURRENT;\
-	type MPC_RMCM_SHAPER_OFFSET_R;\
-	type MPC_RMCM_SHAPER_OFFSET_G;\
-	type MPC_RMCM_SHAPER_OFFSET_B;\
-	type MPC_RMCM_SHAPER_SCALE_R;\
-	type MPC_RMCM_SHAPER_SCALE_G;\
-	type MPC_RMCM_SHAPER_SCALE_B;\
-	type MPC_RMCM_SHAPER_LUT_INDEX;\
-	type MPC_RMCM_SHAPER_LUT_DATA;\
-	type MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK;\
-	type MPC_RMCM_SHAPER_LUT_WRITE_SEL; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS; \
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET;\
-	type MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS;\
-	type MPC_RMCM_3DLUT_MODE;\
-	type MPC_RMCM_3DLUT_SIZE;\
-	type MPC_RMCM_3DLUT_MODE_CURRENT;\
-	type MPC_RMCM_3DLUT_INDEX; \
-	type MPC_RMCM_3DLUT_DATA0; \
-	type MPC_RMCM_3DLUT_DATA1; \
-	type MPC_RMCM_3DLUT_DATA_30BIT; \
-	type MPC_RMCM_3DLUT_WRITE_EN_MASK;\
-	type MPC_RMCM_3DLUT_RAM_SEL;\
-	type MPC_RMCM_3DLUT_30BIT_EN;\
-	type MPC_RMCM_3DLUT_READ_SEL;\
-	type MPC_RMCM_3DLUT_OUT_NORM_FACTOR;\
-	type MPC_RMCM_3DLUT_OUT_OFFSET_R;\
-	type MPC_RMCM_3DLUT_OUT_SCALE_R; \
-	type MPC_RMCM_3DLUT_OUT_OFFSET_G; \
-	type MPC_RMCM_3DLUT_OUT_SCALE_G; \
-	type MPC_RMCM_3DLUT_OUT_OFFSET_B; \
-	type MPC_RMCM_3DLUT_OUT_SCALE_B;\
-	type MPC_RMCM_GAMUT_REMAP_COEF_FORMAT;\
-	type MPC_RMCM_GAMUT_REMAP_MODE;\
-	type MPC_RMCM_GAMUT_REMAP_MODE_CURRENT;\
-	type MPC_RMCM_GAMUT_REMAP_C11_A;\
-	type MPC_RMCM_GAMUT_REMAP_C12_A;\
-	type MPC_RMCM_GAMUT_REMAP_C13_A; \
-	type MPC_RMCM_GAMUT_REMAP_C14_A; \
-	type MPC_RMCM_GAMUT_REMAP_C21_A; \
-	type MPC_RMCM_GAMUT_REMAP_C22_A; \
-	type MPC_RMCM_GAMUT_REMAP_C23_A;\
-	type MPC_RMCM_GAMUT_REMAP_C24_A;\
-	type MPC_RMCM_GAMUT_REMAP_C31_A;\
-	type MPC_RMCM_GAMUT_REMAP_C32_A;\
-	type MPC_RMCM_GAMUT_REMAP_C33_A;\
-	type MPC_RMCM_GAMUT_REMAP_C34_A;\
-	type MPC_RMCM_GAMUT_REMAP_C11_B;\
-	type MPC_RMCM_GAMUT_REMAP_C12_B;\
-	type MPC_RMCM_GAMUT_REMAP_C13_B; \
-	type MPC_RMCM_GAMUT_REMAP_C14_B; \
-	type MPC_RMCM_GAMUT_REMAP_C21_B; \
-	type MPC_RMCM_GAMUT_REMAP_C22_B; \
-	type MPC_RMCM_GAMUT_REMAP_C23_B;\
-	type MPC_RMCM_GAMUT_REMAP_C24_B;\
-	type MPC_RMCM_GAMUT_REMAP_C31_B;\
-	type MPC_RMCM_GAMUT_REMAP_C32_B;\
-	type MPC_RMCM_GAMUT_REMAP_C33_B;\
-	type MPC_RMCM_GAMUT_REMAP_C34_B;\
-	type MPC_RMCM_SHAPER_MEM_PWR_FORCE; \
-	type MPC_RMCM_SHAPER_MEM_PWR_DIS; \
-	type MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE; \
-	type MPC_RMCM_3DLUT_MEM_PWR_FORCE;\
-	type MPC_RMCM_3DLUT_MEM_PWR_DIS;\
-	type MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE;\
-	type MPC_RMCM_SHAPER_MEM_PWR_STATE;\
-	type MPC_RMCM_3DLUT_MEM_PWR_STATE;\
-	type MPC_RMCM_3DLUT_FL_SEL;\
-	type MPC_RMCM_3DLUT_FL_DONE; \
-	type MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW; \
-	type MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW; \
-	type MPC_RMCM_CNTL; \
-	type MPC_RMCM_TEST_DEBUG_INDEX;\
-	type MPC_RMCM_TEST_DEBUG_WRITE_EN;\
-	type MPC_RMCM_TEST_DEBUG_DATA
+	type MPCC_MCM_3DLUT_OUT_SCALE_B
 
 #define MPC_REG_VARIABLE_LIST_DCN42 \
 	MPC_REG_VARIABLE_LIST_DCN4_01 \
-	uint32_t MPCC_CONTROL2[MAX_MPCC];\
-	uint32_t MPC_RMCM_SHAPER_CONTROL[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_OFFSET_R[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_OFFSET_G[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_OFFSET_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_SCALE_R[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_SCALE_G_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_LUT_INDEX[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_LUT_DATA[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_G[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_R[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_G[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_R[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_0_1[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_2_3[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_4_5[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_6_7[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_8_9[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_10_11[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_12_13[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_14_15[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_16_17[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_18_19[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_20_21[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_22_23[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_24_25[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_26_27[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_28_29[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_30_31[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_32_33[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_G[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_R[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_G[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_R[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_0_1[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_2_3[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_4_5[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_6_7[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_8_9[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_10_11[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_12_13[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_14_15[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_16_17[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_18_19[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_20_21[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_22_23[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_24_25[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_26_27[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_28_29[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_30_31[MAX_MPCC]; \
-	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_32_33[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_MODE[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_INDEX[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_DATA[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_DATA_30BIT[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_READ_WRITE_CONTROL[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_OUT_NORM_FACTOR[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_R[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_G[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_COEF_FORMAT[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_MODE[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C11_C12_A[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C13_C14_A[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C21_C22_A[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C23_C24_A[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C31_C32_A[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C33_C34_A[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C11_C12_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C13_C14_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C21_C22_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C23_C24_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C31_C32_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_GAMUT_REMAP_C33_C34_B[MAX_MPCC]; \
-	uint32_t MPC_RMCM_MEM_PWR_CTRL[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_FAST_LOAD_SELECT[MAX_MPCC]; \
-	uint32_t MPC_RMCM_3DLUT_FAST_LOAD_STATUS[MAX_MPCC]; \
-	uint32_t MPC_RMCM_CNTL[MAX_MPCC]; \
-	uint32_t MPC_RMCM_TEST_DEBUG_INDEX[MAX_MPCC]; \
-	uint32_t MPC_RMCM_TEST_DEBUG_DATA[MAX_MPCC]
+	uint32_t MPCC_CONTROL2[MAX_MPCC]
 
 struct dcn42_mpc_shift {
 	MPC_REG_FIELD_LIST_DCN42(uint8_t);
@@ -884,77 +247,6 @@ void dcn42_mpc_construct(struct dcn42_mpc *mpc401,
 
 void mpc42_init_mpcc(struct mpcc *mpcc, int mpcc_inst);
 
-/* RMCM */
-void mpc42_program_rmcm_shaper_lut(
-		struct mpc *mpc,
-		const struct pwl_result_data *rgb,
-		uint32_t num,
-		uint32_t mpcc_id);
-void mpc42_program_rmcm_shaper_lutb_settings(
-		struct mpc *mpc,
-		const struct pwl_params *params,
-		uint32_t mpcc_id);
-void mpc42_program_rmcm_shaper_luta_settings(
-		struct mpc *mpc,
-		const struct pwl_params *params,
-		uint32_t mpcc_id);
-void mpc42_configure_rmcm_shaper_lut(
-		struct mpc *mpc,
-		bool is_ram_a,
-		uint32_t mpcc_id);
-void mpc42_power_on_rmcm_shaper_3dlut(
-		struct mpc *mpc,
-		uint32_t mpcc_id,
-		bool power_on);
-void mpc42_enable_3dlut_fl(
-		struct mpc *mpc,
-		bool enable,
-		int mpcc_id);
-void mpc42_update_3dlut_fast_load_select(
-		struct mpc *mpc,
-		int mpcc_id,
-		int hubp_idx);
-void mpc42_populate_rmcm_lut(
-		struct mpc *mpc,
-		const union mcm_lut_params params,
-		bool lut_bank_a,
-		int mpcc_id);
-void mpc42_program_rmcm_lut_read_write_control(
-		struct mpc *mpc,
-		const enum MCM_LUT_ID id,
-		bool lut_bank_a,
-		bool enabled,
-		int mpcc_id);
-void mpc42_program_lut_mode(
-		struct mpc *mpc,
-		bool enable,
-		bool lut_bank_a,
-		int mpcc_id);
-void mpc42_program_rmcm_3dlut_size(
-		struct mpc *mpc,
-		const enum dc_cm_lut_size size,
-		int mpcc_id);
-void mpc42_program_rmcm_3dlut_fast_load_bias_scale(
-		struct mpc *mpc,
-		uint16_t bias,
-		uint16_t scale,
-		int mpcc_id);
-void mpc42_program_rmcm_bit_depth(
-		struct mpc *mpc,
-		uint16_t bit_depth,
-		int mpcc_id);
-
-void mpc42_set_fl_config(
-	struct mpc *mpc,
-	struct mpc_fl_3dlut_config *cfg,
-	int mpcc_id);
-
-void mpc42_get_rmcm_3dlut_mode(
-	struct mpc *mpc,
-	int mpcc_id,
-	bool *enable,
-	bool *lut_bank_a);
-
 void mpc42_read_mpcc_state(
 		struct mpc *mpc,
 		int mpcc_inst,
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.c
index 2c0b92854a9c28..e38582f0a319ae 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.c
@@ -111,34 +111,6 @@ static struct mpcc *mpc60_insert_plane(
 	return new_mpcc;
 }
 
-void mpc60_program_rmcm_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id,
-	bool lut_bank_a, bool enabled, int mpcc_id)
-{
-	struct dcn60_mpc *mpc60 = TO_DCN60_MPC(mpc);
-
-	switch (id) {
-	case MCM_LUT_3DLUT:
-		REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_MODE,
-			(!enabled) ? 0 :
-			(lut_bank_a) ? 1 : 2);
-		break;
-
-	case MCM_LUT_SHAPER:
-		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
-			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
-
-		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
-			MPC_RMCM_SHAPER_LUT_WRITE_SEL,
-			lut_bank_a == true ? 0 : 1);
-
-		REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[mpcc_id], 0,
-			MPC_RMCM_SHAPER_LUT_INDEX, 0);
-		break;
-	default:
-		break;
-	}
-}
-
 static void mpc60_program_lut_read_write_control(struct mpc *mpc,
 		const enum MCM_LUT_ID id,
 		const bool lut_bank_a,
@@ -288,18 +260,6 @@ static const struct mpc_funcs dcn60_mpc_funcs = {
 	.program_lut_read_write_control = mpc60_program_lut_read_write_control,
 	.program_lut_mode = mpc60_program_lut_mode,
 	.get_lut_mode = mpc401_get_lut_mode,
-	.rmcm = {
-		.enable_3dlut_fl = mpc42_enable_3dlut_fl,
-		.update_3dlut_fast_load_select = mpc42_update_3dlut_fast_load_select,
-		.program_lut_read_write_control = mpc60_program_rmcm_lut_read_write_control,
-		.program_lut_mode = mpc42_program_lut_mode,
-		.program_3dlut_size = mpc42_program_rmcm_3dlut_size,
-		.program_bias_scale = mpc42_program_rmcm_3dlut_fast_load_bias_scale,
-		.program_bit_depth = mpc42_program_rmcm_bit_depth,
-		.power_on_shaper_3dlut = mpc42_power_on_rmcm_shaper_3dlut,
-		.populate_lut = mpc42_populate_rmcm_lut,
-		.get_3dlut_mode = mpc42_get_rmcm_3dlut_mode,
-	},
 };
 
 void dcn60_mpc_construct(struct dcn60_mpc *mpc60,
@@ -313,6 +273,7 @@ void dcn60_mpc_construct(struct dcn60_mpc *mpc60,
 	int i;
 
 	mpc60->base.ctx = ctx;
+	mpc60->base.inst = 0;
 
 	mpc60->base.funcs = &dcn60_mpc_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.h
index 640a322d15c16d..44ee84d7410469 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn60/dcn60_mpc.h
@@ -180,238 +180,6 @@
 	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM_3DLUT_OUT_SCALE_G, mask_sh),\
 	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM_3DLUT_OUT_OFFSET_B, mask_sh),\
 	SF(MPCC_MCM0_MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM_3DLUT_OUT_SCALE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_LUT_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_MODE_CURRENT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R, MPC_RMCM_SHAPER_OFFSET_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G, MPC_RMCM_SHAPER_OFFSET_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B, MPC_RMCM_SHAPER_OFFSET_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R, MPC_RMCM_SHAPER_SCALE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX, MPC_RMCM_SHAPER_LUT_INDEX, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA, MPC_RMCM_SHAPER_LUT_DATA, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_SEL, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_SIZE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE_CURRENT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_INDEX, MPC_RMCM_3DLUT_INDEX, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA0, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA1, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT, MPC_RMCM_3DLUT_DATA_30BIT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_WRITE_EN_MASK, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_30BIT_EN, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_READ_SEL, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR, MPC_RMCM_3DLUT_OUT_NORM_FACTOR, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_OFFSET_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_SCALE_R, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_OFFSET_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_SCALE_G, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_OFFSET_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_SCALE_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE_CURRENT, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C11_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C12_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C13_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C14_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C21_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C22_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C23_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C24_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C31_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C32_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C33_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C34_A, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C11_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C12_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C13_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C14_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C21_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C22_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C23_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C24_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C31_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C32_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C33_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C34_B, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_FORCE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_DIS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_FORCE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_DIS, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_STATE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_STATE, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT, MPC_RMCM_3DLUT_FL_SEL, mask_sh),\
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_DONE, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_CNTL, MPC_RMCM_CNTL, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_INDEX, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_WRITE_EN, mask_sh), \
-	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA, MPC_RMCM_TEST_DEBUG_DATA, mask_sh), \
 	SF(MPC_OUT0_CSC_MODE, MPC_OCSC_MODE, mask_sh),\
 	SF(MPC_OUT0_CSC_C11_C12_A, MPC_OCSC_C11_A, mask_sh),\
 	SF(MPC_OUT0_CSC_C11_C12_A, MPC_OCSC_C12_A, mask_sh),\
@@ -463,10 +231,4 @@ void dcn60_mpc_construct(struct dcn60_mpc *mpc401,
 	const struct dcn60_mpc_mask *mpc_mask,
 	int num_mpcc,
 	int num_rmu);
-
-void mpc60_program_rmcm_lut_read_write_control(struct mpc *mpc,
-	const enum MCM_LUT_ID id,
-	bool lut_bank_a,
-	bool enabled,
-	int mpcc_id);
 #endif /* DCN60_MPC_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
index 881b8da656b27c..8486175a3b5674 100644
--- a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
@@ -455,6 +455,8 @@ void dcn20_opp_construct(struct dcn20_opp *oppn20,
 	oppn20->base.ctx = ctx;
 	oppn20->base.inst = inst;
 	oppn20->base.funcs = &dcn20_opp_funcs;
+	oppn20->base.mpc_tree_params.opp_id = inst;
+	oppn20->base.mpc_tree_params.opp_list = NULL;
 
 	oppn20->regs = regs;
 	oppn20->opp_shift = opp_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
index 3ace83e1b50f0d..ce9cd32b807aca 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.c
@@ -221,6 +221,15 @@ void optc31_set_drr(
 	}
 }
 
+bool optc3_is_odm_enabled(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t num_segments = 0;
+
+	REG_GET(OPTC_DATA_SOURCE_SELECT, OPTC_NUM_OF_INPUT_SEGMENT, &num_segments);
+	return num_segments > 0;
+}
+
 void optc3_init_odm(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
@@ -503,6 +512,7 @@ static const struct timing_generator_funcs dcn31_tg_funcs = {
 		.setup_manual_trigger = optc2_setup_manual_trigger,
 		.get_hw_timing = optc1_get_hw_timing,
 		.init_odm = optc3_init_odm,
+		.is_odm_enabled = optc3_is_odm_enabled,
 		.is_two_pixels_per_container = optc1_is_two_pixels_per_container,
 		.read_otg_state = optc31_read_otg_state,
 		.optc_read_reg_state = optc31_read_reg_state,
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.h b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.h
index 119a954fe3e521..19d90c65a22bf2 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn31/dcn31_optc.h
@@ -274,6 +274,7 @@ bool optc31_immediate_disable_crtc(struct timing_generator *optc);
 void optc31_set_drr(struct timing_generator *optc, const struct drr_params *params);
 
 void optc3_init_odm(struct timing_generator *optc);
+bool optc3_is_odm_enabled(struct timing_generator *optc);
 
 void optc31_read_otg_state(struct timing_generator *optc,
 		struct dcn_otg_state *s);
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c
index 7250478a50925a..f5faa49680ef8d 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn314/dcn314_optc.c
@@ -247,6 +247,7 @@ static const struct timing_generator_funcs dcn314_tg_funcs = {
 		.setup_manual_trigger = optc2_setup_manual_trigger,
 		.get_hw_timing = optc1_get_hw_timing,
 		.init_odm = optc3_init_odm,
+		.is_odm_enabled = optc3_is_odm_enabled,
 		.set_odm_bypass = optc314_set_odm_bypass,
 		.set_odm_combine = optc314_set_odm_combine,
 		.set_h_timing_div_manual_mode = optc314_set_h_timing_div_manual_mode,
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
index 897f857db44817..406446e5eae548 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
@@ -614,6 +614,7 @@ static const struct timing_generator_funcs dcn35_tg_funcs = {
 		.setup_manual_trigger = optc2_setup_manual_trigger,
 		.get_hw_timing = optc1_get_hw_timing,
 		.init_odm = optc3_init_odm,
+		.is_odm_enabled = optc3_is_odm_enabled,
 		.set_long_vtotal = optc35_set_long_vtotal,
 		.is_two_pixels_per_container = optc1_is_two_pixels_per_container,
 		.read_otg_state = optc31_read_otg_state,
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
index fd9c2b6b8b3c68..1e4f2051d7327b 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
@@ -261,6 +261,7 @@ static struct timing_generator_funcs dcn42_tg_funcs = {
 		.setup_manual_trigger = optc2_setup_manual_trigger,
 		.get_hw_timing = optc1_get_hw_timing,
 		.init_odm = optc3_init_odm,
+		.is_odm_enabled = optc3_is_odm_enabled,
 		.set_long_vtotal = optc35_set_long_vtotal,
 		.is_two_pixels_per_container = optc1_is_two_pixels_per_container,
 		.get_optc_double_buffer_pending = optc3_get_optc_double_buffer_pending,
diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h
index 538d0010573882..27d4a96b958fe3 100644
--- a/drivers/gpu/drm/amd/display/dc/os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/os_types.h
@@ -27,6 +27,7 @@
 #ifndef _OS_TYPES_H_
 #define _OS_TYPES_H_
 
+#include <linux/compiler_attributes.h>
 #include <linux/slab.h>
 #include <linux/kgdb.h>
 #include <linux/delay.h>
@@ -78,6 +79,23 @@
 #define dc_breakpoint()		do {} while (0)
 #endif
 
+#ifdef __COVERITY__
+/*
+ * Static analysers see the WARN_ON() test as proof that the asserted condition
+ * can fail, then keep walking the failing path because nothing stops it. Model
+ * the assert as a path terminator so only genuinely reachable defects survive.
+ * __COVERITY__ is defined during analysis capture only, so the code emitted by
+ * a real build is unchanged.
+ */
+void __coverity_panic__(void);
+
+#define ASSERT_CRITICAL(expr) do {		\
+		if (!(expr))			\
+			__coverity_panic__();	\
+	} while (0)
+
+#define ASSERT(expr) ASSERT_CRITICAL(expr)
+#else
 #define ASSERT_CRITICAL(expr) do {		\
 		if (WARN_ON(!(expr)))		\
 			dc_breakpoint();	\
@@ -87,6 +105,7 @@
 		if (WARN_ON_ONCE(!(expr)))	\
 			dc_breakpoint();	\
 	} while (0)
+#endif
 
 #define BREAK_TO_DEBUGGER() \
 	do { \
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c
index 1d0181534a67fb..6e5ea1bbca023c 100644
--- a/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c
@@ -553,6 +553,7 @@ struct pg_cntl *pg_cntl35_create(
 	base = &pg_cntl_dcn->base;
 	base->ctx = ctx;
 	base->funcs = &pg_cntl35_funcs;
+	base->inst = 0;
 
 	pg_cntl_dcn->regs = regs;
 	pg_cntl_dcn->pg_cntl_shift = pg_cntl_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
index 2c87e33e909ee3..e9125d2467fcf6 100644
--- a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
@@ -6,6 +6,7 @@
 #include "core_types.h"
 #include "dcn42_pg_cntl.h"
 #include "dccg.h"
+#include "resource.h"
 
 #define TO_DCN_PG_CNTL(pg_cntl)\
 	container_of(pg_cntl, struct dcn_pg_cntl, base)
@@ -582,7 +583,10 @@ void pg_cntl42_init_pg_status(struct pg_cntl *pg_cntl)
 		block_enabled = pg_cntl42_hubp_dpp_pg_status(pg_cntl, i);
 		pg_cntl->pg_pipe_res_enable[PG_HUBP][i] = block_enabled;
 		pg_cntl->pg_pipe_res_enable[PG_DPP][i] = block_enabled;
+	}
 
+	/* DSC count can be lower than pipe count (e.g. DCN42B has 4 pipes, 3 DSCs) */
+	for (i = 0; i < (unsigned int)pg_cntl->ctx->dc->res_pool->res_cap->num_dsc; i++) {
 		block_enabled = pg_cntl42_dsc_pg_status(pg_cntl, i);
 		pg_cntl->pg_pipe_res_enable[PG_DSC][i] = block_enabled;
 	}
@@ -626,6 +630,7 @@ struct pg_cntl *pg_cntl42_create(
 	base = &pg_cntl_dcn->base;
 	base->ctx = ctx;
 	base->funcs = &pg_cntl42_funcs;
+	base->inst = 0;
 
 	pg_cntl_dcn->regs = regs;
 	pg_cntl_dcn->pg_cntl_shift = pg_cntl_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/Makefile b/drivers/gpu/drm/amd/display/dc/resource/Makefile
index b29986266888c1..bf127b9882f737 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/resource/Makefile
@@ -37,6 +37,8 @@ endif
 
 ###############################################################################
 
+ifdef CONFIG_DRM_AMD_DC_DCE
+
 RESOURCE_DCE80 = dce80_resource.o
 
 AMD_DAL_RESOURCE_DCE80 = $(addprefix $(AMDDALPATH)/dc/resource/dce80/,$(RESOURCE_DCE80))
@@ -75,6 +77,8 @@ AMD_DAL_RESOURCE_DCE120 = $(addprefix $(AMDDALPATH)/dc/resource/dce120/,$(RESOUR
 
 AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCE120)
 
+endif # CONFIG_DRM_AMD_DC_DCE
+
 ifdef CONFIG_DRM_AMD_DC_FP
 ###############################################################################
 # DCN
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
index 8c4494a8ec8f4c..2da5d44e52038e 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
@@ -848,30 +848,6 @@ static void dce112_resource_destruct(struct dce110_resource_pool *pool)
 	}
 }
 
-static struct clock_source *find_matching_pll(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		const struct dc_stream_state *const stream)
-{
-	(void)res_ctx;
-	switch (stream->link->link_enc->transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL0];
-	case TRANSMITTER_UNIPHY_B:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL1];
-	case TRANSMITTER_UNIPHY_C:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL2];
-	case TRANSMITTER_UNIPHY_D:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL3];
-	case TRANSMITTER_UNIPHY_E:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL4];
-	case TRANSMITTER_UNIPHY_F:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL5];
-	default:
-		return NULL;
-	}
-}
-
 static enum dc_status build_mapped_resource(
 		const struct dc *dc,
 		struct dc_state *context,
@@ -963,44 +939,6 @@ enum dc_status dce112_validate_bandwidth(
 	return result ? DC_OK : DC_FAIL_BANDWIDTH_VALIDATE;
 }
 
-enum dc_status resource_map_phy_clock_resources(
-		const struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-
-	/* acquire new resources */
-	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(
-			&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-	if (dc_is_dp_signal(pipe_ctx->stream->signal)
-		|| dc_is_virtual_signal(pipe_ctx->stream->signal))
-		pipe_ctx->clock_source =
-				dc->res_pool->dp_clock_source;
-	else if (pipe_ctx->stream->signal == SIGNAL_TYPE_HDMI_FRL)
-			pipe_ctx->clock_source =
-				dc->res_pool->dp_clock_source;
-	else {
-		if (stream && stream->link && stream->link->link_enc)
-			pipe_ctx->clock_source = find_matching_pll(
-				&context->res_ctx, dc->res_pool,
-				stream);
-	}
-
-	if (pipe_ctx->clock_source == NULL)
-		return DC_NO_CLOCK_SOURCE_RESOURCE;
-
-	resource_reference_clock_source(
-		&context->res_ctx,
-		dc->res_pool,
-		pipe_ctx->clock_source);
-
-	return DC_OK;
-}
-
 static bool dce112_validate_surface_sets(
 		struct dc_state *context)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
index a2e66761b8ec03..3a6a489388435c 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
@@ -97,7 +97,7 @@
 #include "dc_state_priv.h"
 #include "dc_fpu.h"
 
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #if !defined(DC_RUN_WITH_PREEMPTION_ENABLED)
 #define DC_RUN_WITH_PREEMPTION_ENABLED(code) code
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
index fd7a22fcca5906..940dd00e97fa73 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
@@ -34,7 +34,7 @@
 #include "include/irq_service_interface.h"
 #include "basics/conversion.h"
 #include "dcn35_resource.h"
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #include "dcn20/dcn20_resource.h"
 #include "dcn30/dcn30_resource.h"
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
index 83248ee01d1123..df2aaa46e70c5c 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
@@ -87,7 +87,7 @@
 #include "vm_helper.h"
 #include "dcn20/dcn20_vmid.h"
 
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #include "link_enc_cfg.h"
 #define DC_LOGGER_INIT(logger)
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
index 8d1baa76c347f1..e8580f844af9f2 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
@@ -12,7 +12,7 @@
 #include "include/irq_service_interface.h"
 #include "basics/conversion.h"
 #include "dcn36_resource.h"
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #include "dcn20/dcn20_resource.h"
 #include "dcn30/dcn30_resource.h"
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
index 0cfdca82bef0fd..b78b0541d1134d 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
@@ -78,7 +78,7 @@
 
 #include "dc_state_priv.h"
 
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #define DC_LOGGER_INIT(logger)
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
index 28192d6dda7a1b..e5e94e955157a9 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
@@ -27,6 +27,7 @@
 #include "dcn42/dcn42_hubbub.h"
 #include "dcn401/dcn401_mpc.h"
 #include "dcn42/dcn42_mpc.h"
+#include "dcn42/dcn42_rmcm.h"
 #include "dcn35/dcn35_hubp.h"
 #include "dcn42/dcn42_hubp.h"
 #include "irq/dcn42/irq_service_dcn42.h"
@@ -87,7 +88,7 @@
 #include "dc_state_priv.h"
 #include "link_enc_cfg.h"
 
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #define regBIF_BX0_BIOS_SCRATCH_3                            0x003b
 #define regBIF_BX0_BIOS_SCRATCH_3_BASE_IDX                   1
@@ -436,9 +437,7 @@ static struct dcn42_mpc_registers mpc_regs;
 		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1), \
 		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2), \
 		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3), \
-		MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0), \
-		MPC_RMCM_REG_LIST_DCN42(0),		   \
-		MPC_RMCM_REG_LIST_DCN42(1)
+		MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
 
 static const struct dcn42_mpc_shift mpc_shift = {
 	MPC_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
@@ -446,6 +445,18 @@ static const struct dcn42_mpc_shift mpc_shift = {
 static const struct dcn42_mpc_mask mpc_mask = {
 	MPC_COMMON_MASK_SH_LIST_DCN42(_MASK)};
 
+static struct dcn42_rmcm_registers rmcm_regs;
+
+#define dcn_rmcm_regs_init()               \
+	MPC_RMCM_REG_LIST_DCN42(0),            \
+		MPC_RMCM_REG_LIST_DCN42(1)
+
+static const struct dcn42_rmcm_shift rmcm_shift = {
+	MPC_RMCM_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn42_rmcm_mask rmcm_mask = {
+	MPC_RMCM_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
 #define optc_regs_init(id) \
 	OPTC_COMMON_REG_LIST_DCN42_RI(id)
 
@@ -1069,6 +1080,29 @@ static struct mpc *dcn42_mpc_create(
 	return &mpc401->base;
 }
 
+static struct rmcm *dcn42_rmcm_create(
+	struct dc_context *ctx,
+	int inst)
+{
+	struct dcn42_rmcm *rmcm42 = kzalloc(sizeof(struct dcn42_rmcm), GFP_KERNEL);
+
+	if (!rmcm42)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT rmcm_regs
+	dcn_rmcm_regs_init();
+
+	dcn42_rmcm_construct(rmcm42,
+		ctx,
+		&rmcm_regs,
+		&rmcm_shift,
+		&rmcm_mask,
+		inst);
+
+	return &rmcm42->base;
+}
+
 static struct output_pixel_processor *dcn42_opp_create(
 	struct dc_context *ctx, uint32_t inst)
 {
@@ -1548,6 +1582,12 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
 		kfree(TO_DCN20_MPC(pool->base.mpc));
 		pool->base.mpc = NULL;
 	}
+	for (i = 0; i < MAX_RMCM_INST; i++) {
+		if (pool->base.rmcm[i] != NULL) {
+			kfree(TO_DCN42_RMCM(pool->base.rmcm[i]));
+			pool->base.rmcm[i] = NULL;
+		}
+	}
 	if (pool->base.hubbub != NULL) {
 		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
 		pool->base.hubbub = NULL;
@@ -2330,6 +2370,16 @@ static bool dcn42_resource_construct(
 		goto create_fail;
 	}
 
+	/* RMCMs */
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_rmcm && i < (unsigned int)MAX_RMCM_INST; i++) {
+		pool->base.rmcm[i] = dcn42_rmcm_create(ctx, i);
+		if (pool->base.rmcm[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create rmcm%d!\n", i);
+			goto create_fail;
+		}
+	}
+
 	/* DSCs */
 	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		pool->base.dscs[i] = dcn42_dsc_create(ctx, i);
@@ -2403,9 +2453,7 @@ static bool dcn42_resource_construct(
 	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
 	 /*this will use real soc clock table*/
 	dc->dml2_options.use_native_soc_bb_construction = true;
-	dc->dml2_options.minimize_dispclk_using_odm = false;
-	if (dc->config.EnableMinDispClkODM)
-		dc->dml2_options.minimize_dispclk_using_odm = true;
+	dc->dml2_options.minimize_dispclk_using_odm = dc->config.EnableMinDispClkODM;
 	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
 	dc->dml2_options.map_dc_pipes_with_callbacks = true;
 	dc->dml2_options.force_tdlut_enable = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
index f36d0f828166f8..de3bd23f473656 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42b/dcn42b_resource.c
@@ -31,6 +31,7 @@
 #include "dcn42/dcn42_hubbub.h"
 #include "dcn401/dcn401_mpc.h"
 #include "dcn42/dcn42_mpc.h"
+#include "dcn42/dcn42_rmcm.h"
 #include "dcn35/dcn35_hubp.h"
 #include "dcn42/dcn42_hubp.h"
 #include "irq/dcn42/irq_service_dcn42.h"
@@ -88,7 +89,7 @@
 #include "dc_state_priv.h"
 #include "link_enc_cfg.h"
 
-#include "dml2_0/dml2_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
 
 #define DC_LOGGER_INIT(logger)
 
@@ -486,9 +487,7 @@ static struct dcn42_mpc_registers mpc_regs;
 		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1), \
 		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2), \
 		MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3), \
-		MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0), \
-		MPC_RMCM_REG_LIST_DCN42(0),		   \
-		MPC_RMCM_REG_LIST_DCN42(1)
+		MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
 
 static const struct dcn42_mpc_shift mpc_shift = {
 	MPC_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
@@ -496,6 +495,18 @@ static const struct dcn42_mpc_shift mpc_shift = {
 static const struct dcn42_mpc_mask mpc_mask = {
 	MPC_COMMON_MASK_SH_LIST_DCN42(_MASK)};
 
+static struct dcn42_rmcm_registers rmcm_regs;
+
+#define dcn_rmcm_regs_init()               \
+	MPC_RMCM_REG_LIST_DCN42(0),            \
+		MPC_RMCM_REG_LIST_DCN42(1)
+
+static const struct dcn42_rmcm_shift rmcm_shift = {
+	MPC_RMCM_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn42_rmcm_mask rmcm_mask = {
+	MPC_RMCM_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
 #define optc_regs_init(id) \
 	OPTC_COMMON_REG_LIST_DCN42B_RI(id)
 
@@ -758,6 +769,7 @@ static const struct resource_caps res_cap_dcn42b = {
 	.num_ddc = 0,
 	.num_vmid = 16,
 	.num_mpc_3dlut = 2,
+	.num_rmcm = 2,
 	.num_dsc = 3,
 	.num_rmcm = 2,
 	.num_mpc = 4,
@@ -790,9 +802,13 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.force_abm_enable = false,
 	.clock_trace = true,
 	.disable_pplib_clock_request = false,
-	.disable_dpp_power_gate = true,
-	.disable_hubp_power_gate = true,
+	.ignore_pg = false,
+	.disable_dpp_power_gate = false,
+	.disable_hubp_power_gate = false,
 	.disable_optc_power_gate = true,
+	.disable_dsc_power_gate = false,
+	.disable_dio_power_gate = true,
+	.disable_hpo_power_gate = true,
 	.pipe_split_policy = MPC_SPLIT_AVOID,
 	.force_single_disp_pipe_split = false,
 	.disable_dcc = DCC_ENABLE,
@@ -818,14 +834,14 @@ static const struct dc_debug_options debug_defaults_drv = {
 		}},
 	.root_clock_optimization = {
 		.bits = {
-			.dpp = false,
-			.dsc = false,/*dscclk and dsc pg*/
+			.dpp = true,
+			.dsc = true,/*dscclk and dsc pg*/
 			.hdmistream = false,
 			.hdmichar = false,
-			.dpstream = false,
-			.symclk32_se = false,
-			.symclk32_le = false,
-			.symclk_fe = false,
+			.dpstream = true,
+			.symclk32_se = true,
+			.symclk32_le = true,
+			.symclk_fe = true,
 			.physymclk = false,
 			.dpiasymclk = false,
 		}
@@ -835,7 +851,8 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.minimum_z8_residency_time = 1, /* Always allow when other conditions are met */
 	.support_eDP1_5 = true,
 	.use_max_lb = true,
-	.force_disable_subvp = false,
+	/* SubVP phantom surfaces require MALL, which DCN42B does not have */
+	.force_disable_subvp = true,
 	.exit_idle_opt_for_cursor_updates = true,
 	.using_dml2 = true,
 	.using_dml21 = true,
@@ -854,7 +871,6 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.min_disp_clk_khz = 50000,
 	.static_screen_wait_frames = 2,
 	.disable_z10 = false,
-	.ignore_pg = true,
 	.disable_stutter_for_wm_program = true,
 	.min_deep_sleep_dcfclk_khz = 8000,
 	.replay_skip_crtc_disabled = true,
@@ -1128,6 +1144,29 @@ static struct mpc *dcn42b_mpc_create(
 	return &mpc42b->base;
 }
 
+static struct rmcm *dcn42b_rmcm_create(
+	struct dc_context *ctx,
+	int inst)
+{
+	struct dcn42_rmcm *rmcm42 = kzalloc(sizeof(struct dcn42_rmcm), GFP_KERNEL);
+
+	if (!rmcm42)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT rmcm_regs
+	dcn_rmcm_regs_init();
+
+	dcn42_rmcm_construct(rmcm42,
+		ctx,
+		&rmcm_regs,
+		&rmcm_shift,
+		&rmcm_mask,
+		inst);
+
+	return &rmcm42->base;
+}
+
 static struct output_pixel_processor *dcn42b_opp_create(
 	struct dc_context *ctx, uint32_t inst)
 {
@@ -1536,6 +1575,12 @@ static void dcn42b_resource_destruct(struct dcn42b_resource_pool *pool)
 		kfree(TO_DCN20_MPC(pool->base.mpc));
 		pool->base.mpc = NULL;
 	}
+	for (i = 0; i < MAX_RMCM_INST; i++) {
+		if (pool->base.rmcm[i] != NULL) {
+			kfree(TO_DCN42_RMCM(pool->base.rmcm[i]));
+			pool->base.rmcm[i] = NULL;
+		}
+	}
 	if (pool->base.hubbub != NULL) {
 		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
 		pool->base.hubbub = NULL;
@@ -1926,8 +1971,6 @@ static struct resource_funcs dcn42b_res_pool_funcs = {
 	.get_panel_config_defaults = dcn42b_get_panel_config_defaults,
 	//.get_preferred_eng_id_dpia = dcn42b_get_preferred_eng_id_dpia,
 	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-	.add_phantom_pipes = dcn32_add_phantom_pipes,
-	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
 	.prepare_mcache_programming = dcn42b_prepare_mcache_programming,
 	.build_pipe_pix_clk_params = dcn42b_build_pipe_pix_clk_params,
 	.get_power_profile = dcn401_get_power_profile,
@@ -1993,27 +2036,7 @@ static bool dcn42b_resource_construct(
 	dc->caps.cursor_not_scaled = true;
 	dc->caps.min_horizontal_blanking_period = 80;
 	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.mall_size_per_mem_channel = 4;
-	/* total size = mall per channel * num channels * 1024 * 1024 */
-	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel *
-		dc->ctx->dc_bios->vram_info.num_chans * 1048576;
 	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-	dc->caps.cache_line_size = 64;
-	dc->caps.cache_num_ways = 16;
-
-	/* Calculate the available MALL space */
-	dc->caps.max_cab_allocation_bytes =
-		dcn32_calc_num_avail_chans_for_mall(dc, dc->ctx->dc_bios->vram_info.num_chans) *
-				dc->caps.mall_size_per_mem_channel * 1024 * 1024;
-	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
-
-	dc->caps.subvp_fw_processing_delay_us = 15;
-	dc->caps.subvp_drr_max_vblank_margin_us = 40;
-	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
-	dc->caps.subvp_swath_height_margin_lines = 16;
-	dc->caps.subvp_pstate_allow_width_us = 20;
-	dc->caps.subvp_vertical_int_margin_us = 30;
-	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
 
 	dc->caps.max_slave_planes = 2;
 	dc->caps.max_slave_yuv_planes = 2;
@@ -2306,6 +2329,16 @@ static bool dcn42b_resource_construct(
 		goto create_fail;
 	}
 
+	/* RMCMs */
+	for (i = 0; i < pool->base.res_cap->num_rmcm && i < MAX_RMCM_INST; i++) {
+		pool->base.rmcm[i] = dcn42b_rmcm_create(ctx, i);
+		if (pool->base.rmcm[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create rmcm%d!\n", i);
+			goto create_fail;
+		}
+	}
+
 	/* DSCs */
 	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
 		pool->base.dscs[i] = dcn42b_dsc_create(ctx, i);
@@ -2383,9 +2416,7 @@ static bool dcn42b_resource_construct(
 	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
 	 /*this will use real soc clock table*/
 	dc->dml2_options.use_native_soc_bb_construction = true;
-	dc->dml2_options.minimize_dispclk_using_odm = false;
-	if (dc->config.EnableMinDispClkODM)
-		dc->dml2_options.minimize_dispclk_using_odm = true;
+	dc->dml2_options.minimize_dispclk_using_odm = dc->config.EnableMinDispClkODM;
 	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
 	dc->dml2_options.map_dc_pipes_with_callbacks = true;
 	dc->dml2_options.force_tdlut_enable = true;
@@ -2393,26 +2424,6 @@ static bool dcn42b_resource_construct(
 	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
 	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch =
 			&dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
-	dc->dml2_options.svp_pstate.callbacks.calculate_mall_ways_from_bytes =
-		pool->base.funcs->calculate_mall_ways_from_bytes;
-
-	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
-	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
-	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
-	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
-
-	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
-	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
-
-	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
-	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
-	dc->dml2_options.mall_cfg.max_cab_allocation_bytes =
-				dc->caps.max_cab_allocation_bytes;
-	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
-	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
-	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
-	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
 
 	dc->dml2_options.max_segments_per_hubp = 24;
 	dc->dml2_options.det_segment_size = DCN42_CRB_SEGMENT_SIZE_KB;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.c
index d091ea55cb5d36..ad841431b70c45 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.c
@@ -3,6 +3,7 @@
 // Copyright 2024 Advanced Micro Devices, Inc.
 
 #include "dm_services.h"
+#include "dm_helpers.h"
 #include "dc.h"
 
 #include "dcn32/dcn32_init.h"
@@ -18,6 +19,7 @@
 #include "dcn321/dcn321_resource.h"
 #include "dcn401/dcn401_resource.h"
 #include "dcn42/dcn42_resource.h"
+#include "dcn60/dcn60_rmcm.h"
 #include "dcn60_resource.h"
 
 #include "dcn10/dcn10_ipp.h"
@@ -81,8 +83,10 @@
 
 #include "dc_state_priv.h"
 
-#include "dml2_0/dml2_wrapper.h"
-#include "dml2_0/dml21/dml21_wrapper.h"
+#include "dml2_wrapper/dml2_wrapper.h"
+#include "dml2_wrapper/dml21_wrapper/dml21_wrapper.h"
+
+#define LSDMA_CONTENTION_BUFFER_SIZE (64 * 1024 * 1024)
 
 #define DC_LOGGER_INIT(logger)
 
@@ -286,7 +290,7 @@ static const struct dcn31_vpg_mask vpg_mask = {
 #define apg_regs_init(id)\
 	APG_DCN31_REG_LIST_RI(id)
 
-static struct dcn31_apg_registers apg_regs[4];
+static struct dcn31_apg_registers apg_regs[5];
 
 static const struct dcn31_apg_shift apg_shift = {
 	DCN31_APG_MASK_SH_LIST(__SHIFT)
@@ -372,11 +376,11 @@ static const struct dcn30_hpo_frl_link_encoder_mask hpo_le_mask = {
 static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
 
 static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+	DCN4_2_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
 };
 
 static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+	DCN4_2_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
 };
 
 #define hpo_dp_link_encoder_reg_init(id)\
@@ -461,9 +465,7 @@ static struct dcn60_mpc_registers mpc_regs;
 	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
 	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
 	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
-	MPC_RMCM_REG_LIST_DCN42(0),\
-	MPC_RMCM_REG_LIST_DCN42(1)
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3)
 
 static const struct dcn60_mpc_shift mpc_shift = {
 	MPC_COMMON_MASK_SH_LIST_DCN6_0(__SHIFT)
@@ -668,6 +670,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 			.enable_ppt_check = true,
 			.enable_offload_flip = true,
 			.enable_stall_recovery = true,
+			.alternate_channel_workaround = true,
 		}
 	},
 	.force_cositing = CHROMA_COSITING_NONE + 1,
@@ -1386,7 +1389,8 @@ static struct apg *dcn60_apg_create(
 	apg_regs_init(0),
 	apg_regs_init(1),
 	apg_regs_init(2),
-	apg_regs_init(3);
+	apg_regs_init(3),
+	apg_regs_init(4);
 
 	apg31_construct(apg60, ctx, inst,
 			&apg_regs[inst],
@@ -1622,6 +1626,12 @@ static void dcn60_resource_destruct(struct dcn60_resource_pool *pool)
 {
 	unsigned int i;
 
+	if (pool->base.lsdma_scratch.buffer) {
+		dm_helpers_free_gpu_mem(pool->base.ctx,
+				DC_MEM_ALLOC_TYPE_GART, pool->base.lsdma_scratch.buffer);
+		pool->base.lsdma_scratch.buffer = NULL;
+	}
+
 	for (i = 0; i < pool->base.stream_enc_count; i++) {
 		if (pool->base.stream_enc[i] != NULL) {
 			if (pool->base.stream_enc[i]->vpg != NULL) {
@@ -1683,6 +1693,8 @@ static void dcn60_resource_destruct(struct dcn60_resource_pool *pool)
 		kfree(TO_DCN20_MPC(pool->base.mpc));
 		pool->base.mpc = NULL;
 	}
+	for (i = 0; i < MAX_RMCM_INST; i++)
+		dcn60_rmcm_destroy(&pool->base.rmcm[i]);
 	if (pool->base.hubbub != NULL) {
 		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
 		pool->base.hubbub = NULL;
@@ -1960,6 +1972,8 @@ static bool dcn60_resource_construct(
 	struct ddc_service_init_data ddc_init_data = {0};
 	uint32_t pipe_fuses = 0;
 	uint32_t num_pipes  = 4;
+	bool is_lite3 =
+		ASICREV_IS_DCN6_VARIANT_LITE3(ctx->asic_id.hw_internal_rev);
 
 #undef REG_STRUCT
 #define REG_STRUCT bios_regs
@@ -2022,9 +2036,9 @@ static bool dcn60_resource_construct(
 	dc->caps.cache_line_size = 64;
 	dc->caps.cache_num_ways = 16;
 
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.max_slave_planes = 3;
+	dc->caps.max_slave_yuv_planes = 3;
+	dc->caps.max_slave_rgb_planes = 3;
 	dc->caps.post_blend_color_processing = true;
 	dc->caps.force_dp_tps4_for_cp2520 = true;
 	dc->caps.hdmi_hpo = true;
@@ -2033,8 +2047,19 @@ static bool dcn60_resource_construct(
 	dc->caps.edp_dsc_support = true;
 	dc->caps.extended_aux_timeout_support = true;
 	dc->caps.dmcub_support = true;
+	dc->caps.utm_support = true;
 	dc->caps.max_v_total = (1 << 15) - 1;
 
+	pool->base.ctx = ctx;
+
+	if (dc->config.lsdma_peak_bw_contention_support) {
+		pool->base.lsdma_scratch.buffer = dm_helpers_allocate_gpu_mem(ctx,
+				DC_MEM_ALLOC_TYPE_GART, LSDMA_CONTENTION_BUFFER_SIZE,
+				&pool->base.lsdma_scratch.pa);
+		if (pool->base.lsdma_scratch.buffer)
+			pool->base.lsdma_scratch.size = LSDMA_CONTENTION_BUFFER_SIZE;
+	}
+
 	if (ASICREV_IS_GC_12_0_1_A0(dc->ctx->asic_id.hw_internal_rev))
 		dc->caps.dcc_plane_width_limit = 7680;
 
@@ -2103,6 +2128,7 @@ static bool dcn60_resource_construct(
 	dc->config.enable_windowed_mpo_odm = true;
 	dc->config.set_pipe_unlock_order = true; /* Need to ensure DET gets freed before allocating */
 	dc->config.dp_connector_no_native_i2c = true;
+	dc->caps.fused_io_supported = true;
 	/* read VBIOS LTTPR caps */
 	{
 		if (ctx->dc_bios->funcs->get_lttpr_caps) {
@@ -2262,6 +2288,16 @@ static bool dcn60_resource_construct(
 		goto create_fail;
 	}
 
+	/* RMCMs */
+	for (i = 0; i < pool->base.res_cap->num_rmcm && i < MAX_RMCM_INST; i++) {
+		pool->base.rmcm[i] = dcn60_rmcm_create(ctx, i);
+		if (pool->base.rmcm[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create rmcm%d!\n", i);
+			goto create_fail;
+		}
+	}
+
 	/* DSCs */
 	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
 		pool->base.dscs[i] = dcn60_dsc_create(ctx, i);
@@ -2339,6 +2375,10 @@ static bool dcn60_resource_construct(
 
 	dc->dml2_options.max_segments_per_hubp = 20;
 	dc->dml2_options.det_segment_size = DCN6_0_CRB_SEGMENT_SIZE_KB;
+	if (is_lite3) {
+		dc->dml2_options.gpuvm_enable = true;
+		dc->dml2_options.hostvm_enable = true;
+	}
 
 	/* SPL */
 	dc->caps.scl_caps.sharpener_support = true;
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h
index c4b6df4838c63e..b1a802f9f4b47d 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn60/dcn60_resource.h
@@ -59,6 +59,12 @@ bool dcn50_program_mcache_pipe_config(struct dc_state *context,
 
 /* HUBBUB */
 #define HUBBUB_REG_LIST_DCN60_RI(id)                                         \
+	SR(DCHVM_CTRL0),                                                         \
+	SR(DCHVM_MEM_CTRL),                                                      \
+	SR(DCHVM_CLK_CTRL),                                                      \
+	SR(DCHVM_RIOMMU_CTRL0),                                                  \
+	SR(DCHVM_RIOMMU_STAT0),                                                  \
+	SR(DCHVM_PREFETCH_VFID),                                                 \
 	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),                               \
 	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),                               \
 	SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),                                  \
@@ -140,7 +146,8 @@ bool dcn50_program_mcache_pipe_config(struct dc_state *context,
 	SR(DC_PERFMON5_PERFMON_HI),                                              \
 	SR(DC_PERFMON5_PERFMON_LOW),                                             \
 	SR(FMON_CTRL),                                                           \
-	SR(DCHUBBUB_ARB_QOS_FORCE)
+	SR(DCHUBBUB_ARB_QOS_FORCE),                                              \
+	SR(UTM_CLIENT_TO_SYSTEM_PROFILE_MAPPING_0)
 
 /* HUBP */
 #define HUBP_REG_LIST_DCN60_RI(id)                                               \
diff --git a/drivers/gpu/drm/amd/display/dc/rmcm/Makefile b/drivers/gpu/drm/amd/display/dc/rmcm/Makefile
new file mode 100644
index 00000000000000..9dfc86a850dac7
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/rmcm/Makefile
@@ -0,0 +1,44 @@
+#
+# Copyright 2026 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# Authors: AMD
+#
+#
+
+ifdef CONFIG_DRM_AMD_DC_FP
+###############################################################################
+# DCN42
+###############################################################################
+RMCM_DCN42 = dcn42_rmcm.o
+
+AMD_DAL_RMCM_DCN42 = $(addprefix $(AMDDALPATH)/dc/rmcm/dcn42/,$(RMCM_DCN42))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RMCM_DCN42)
+
+###############################################################################
+# DCN60
+###############################################################################
+RMCM_DCN60 = dcn60_rmcm.o
+
+AMD_DAL_RMCM_DCN60 = $(addprefix $(AMDDALPATH)/dc/rmcm/dcn60/,$(RMCM_DCN60))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RMCM_DCN60)
+endif
diff --git a/drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.c b/drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.c
new file mode 100644
index 00000000000000..952f0ade24a515
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.c
@@ -0,0 +1,867 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "reg_helper.h"
+#include "dc.h"
+#include "dcn42_rmcm.h"
+#include "dcn10/dcn10_cm_common.h"
+#include "basics/conversion.h"
+#include "hw_shared.h"
+#include "mpc.h"
+
+#define REG(reg)\
+	rmcm42->rmcm_regs->reg
+
+#define CTX \
+	rmcm42->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+	rmcm42->rmcm_shift->field_name, rmcm42->rmcm_mask->field_name
+
+/* RMCM Shaper functions */
+
+void rmcm42_power_on_shaper_3dlut(
+	struct rmcm *rmcm,
+	uint32_t rmcm_id,
+	bool power_on)
+{
+	uint32_t power_status_shaper = 2;
+	uint32_t power_status_3dlut  = 2;
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+	int max_retries = 10;
+
+	REG_SET(MPC_RMCM_MEM_PWR_CTRL[rmcm_id], 0,
+		MPC_RMCM_3DLUT_MEM_PWR_DIS, power_on == true ? 0 : 1);
+	REG_SET(MPC_RMCM_MEM_PWR_CTRL[rmcm_id], 0,
+		MPC_RMCM_SHAPER_MEM_PWR_DIS, power_on == true ? 0 : 1);
+	/* wait for memory to fully power up */
+	if (power_on && rmcm->ctx->dc->debug.enable_mem_low_power.bits.mpc) {
+		REG_WAIT(MPC_RMCM_MEM_PWR_CTRL[rmcm_id], MPC_RMCM_SHAPER_MEM_PWR_STATE, 0, 1, max_retries);
+		REG_WAIT(MPC_RMCM_MEM_PWR_CTRL[rmcm_id], MPC_RMCM_3DLUT_MEM_PWR_STATE, 0, 1, max_retries);
+	}
+
+	/*read status is not mandatory, it is just for debugging*/
+	REG_GET(MPC_RMCM_MEM_PWR_CTRL[rmcm_id], MPC_RMCM_SHAPER_MEM_PWR_STATE, &power_status_shaper);
+	REG_GET(MPC_RMCM_MEM_PWR_CTRL[rmcm_id], MPC_RMCM_3DLUT_MEM_PWR_STATE, &power_status_3dlut);
+
+	if (power_status_shaper != 0 && power_on == true)
+		BREAK_TO_DEBUGGER();
+
+	if (power_status_3dlut != 0 && power_on == true)
+		BREAK_TO_DEBUGGER();
+}
+
+static void rmcm42_configure_shaper_lut(
+	struct rmcm *rmcm,
+	bool is_ram_a,
+	uint32_t rmcm_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_G_B[rmcm_id],
+		MPC_RMCM_SHAPER_SCALE_B, 0x7000);
+	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_G_B[rmcm_id],
+		MPC_RMCM_SHAPER_SCALE_G, 0x7000);
+	REG_UPDATE(MPC_RMCM_SHAPER_SCALE_R[rmcm_id],
+		MPC_RMCM_SHAPER_SCALE_R, 0x7000);
+	REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[rmcm_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
+	REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[rmcm_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_SEL, is_ram_a == true ? 0:1);
+	REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[rmcm_id], 0, MPC_RMCM_SHAPER_LUT_INDEX, 0);
+}
+
+static void rmcm42_program_shaper_luta_settings(
+	struct rmcm *rmcm,
+	const struct pwl_params *params,
+	uint32_t rmcm_id)
+{
+	const struct gamma_curve *curve;
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_B[rmcm_id], 0,
+		MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
+		MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_G[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_R[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, params->corner_points[0].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, 0);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_B[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_G[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].green.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_R[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, params->corner_points[1].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, params->corner_points[1].red.custom_float_y);
+
+	curve = params->arr_curve_points;
+	if (curve) {
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_0_1[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_2_3[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_4_5[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_6_7[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_8_9[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_10_11[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_12_13[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_14_15[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_16_17[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_18_19[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_20_21[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_22_23[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_24_25[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_26_27[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_28_29[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_30_31[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMA_REGION_32_33[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+	}
+}
+
+
+static void rmcm42_program_shaper_lutb_settings(
+	struct rmcm *rmcm,
+	const struct pwl_params *params,
+	uint32_t rmcm_id)
+{
+	const struct gamma_curve *curve;
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_B[rmcm_id], 0,
+		MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].blue.custom_float_x,
+		MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_G[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_START_CNTL_R[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, params->corner_points[0].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, 0);
+
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_B[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].blue.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].blue.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_G[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].green.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].green.custom_float_y);
+	REG_SET_2(MPC_RMCM_SHAPER_RAMB_END_CNTL_R[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, params->corner_points[1].red.custom_float_x,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, params->corner_points[1].red.custom_float_y);
+
+	curve = params->arr_curve_points;
+	if (curve) {
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_0_1[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_2_3[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_4_5[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_6_7[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_8_9[rmcm_id], 0,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+			MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_10_11[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_12_13[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_14_15[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_16_17[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_18_19[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_20_21[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_22_23[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_24_25[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_26_27[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_28_29[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_30_31[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+
+		curve += 2;
+		REG_SET_4(MPC_RMCM_SHAPER_RAMB_REGION_32_33[rmcm_id], 0,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+				MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+	}
+}
+
+static void rmcm42_program_shaper_lut(
+	struct rmcm *rmcm,
+	const struct pwl_result_data *rgb,
+	uint32_t num,
+	uint32_t rmcm_id)
+{
+	uint32_t i, red, green, blue;
+	uint32_t  red_delta, green_delta, blue_delta;
+	uint32_t  red_value, green_value, blue_value;
+
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	for (i = 0; i < num; i++) {
+
+		red   = rgb[i].red_reg;
+		green = rgb[i].green_reg;
+		blue  = rgb[i].blue_reg;
+
+		red_delta   = rgb[i].delta_red_reg;
+		green_delta = rgb[i].delta_green_reg;
+		blue_delta  = rgb[i].delta_blue_reg;
+
+		red_value   = ((red_delta   & 0x3ff) << 14) | (red   & 0x3fff);
+		green_value = ((green_delta & 0x3ff) << 14) | (green & 0x3fff);
+		blue_value  = ((blue_delta  & 0x3ff) << 14) | (blue  & 0x3fff);
+
+		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[rmcm_id], 0, MPC_RMCM_SHAPER_LUT_DATA, red_value);
+		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[rmcm_id], 0, MPC_RMCM_SHAPER_LUT_DATA, green_value);
+		REG_SET(MPC_RMCM_SHAPER_LUT_DATA[rmcm_id], 0, MPC_RMCM_SHAPER_LUT_DATA, blue_value);
+	}
+}
+
+void rmcm42_update_3dlut_fast_load_select(struct rmcm *rmcm, int rmcm_id, int hubp_idx)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	REG_SET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[rmcm_id], 0,
+		MPC_RMCM_3DLUT_FL_SEL,
+		hubp_idx);
+}
+
+static void rmcm42_select_3dlut_ram_mask(
+		struct rmcm *rmcm,
+		uint32_t ram_selection_mask,
+		int rmcm_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+			MPC_RMCM_3DLUT_WRITE_EN_MASK, ram_selection_mask);
+	REG_SET(MPC_RMCM_3DLUT_INDEX[rmcm_id], 0, MPC_RMCM_3DLUT_INDEX, 0);
+}
+
+static void rmcm42_set3dlut_ram12(
+		struct rmcm *rmcm,
+		const struct dc_rgb *lut,
+		uint32_t entries,
+		int rmcm_id)
+{
+	uint32_t i, red, green, blue, red1, green1, blue1;
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	for (i = 0; i < entries; i += 2) {
+		red   = lut[i].red << 4;
+		green = lut[i].green << 4;
+		blue  = lut[i].blue << 4;
+		red1   = lut[i + 1].red << 4;
+		green1 = lut[i + 1].green << 4;
+		blue1  = lut[i + 1].blue << 4;
+
+		REG_SET_2(MPC_RMCM_3DLUT_DATA[rmcm_id], 0,
+				MPC_RMCM_3DLUT_DATA0, red,
+				MPC_RMCM_3DLUT_DATA1, red1);
+
+		REG_SET_2(MPC_RMCM_3DLUT_DATA[rmcm_id], 0,
+				MPC_RMCM_3DLUT_DATA0, green,
+				MPC_RMCM_3DLUT_DATA1, green1);
+
+		REG_SET_2(MPC_RMCM_3DLUT_DATA[rmcm_id], 0,
+				MPC_RMCM_3DLUT_DATA0, blue,
+				MPC_RMCM_3DLUT_DATA1, blue1);
+	}
+}
+
+static void rmcm42_set3dlut_ram10(
+		struct rmcm *rmcm,
+		const struct dc_rgb *lut,
+		uint32_t entries,
+		int rmcm_id)
+{
+	uint32_t i, red, green, blue, value;
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	for (i = 0; i < entries; i++) {
+		red   = lut[i].red;
+		green = lut[i].green;
+		blue  = lut[i].blue;
+		value = (red << 20) | (green << 10) | blue;
+
+		REG_SET(MPC_RMCM_3DLUT_DATA_30BIT[rmcm_id], 0,
+				MPC_RMCM_3DLUT_DATA_30BIT, value);
+	}
+}
+
+void rmcm42_populate_lut(struct rmcm *rmcm, const enum MCM_LUT_ID id,
+	const union rmcm_lut_params params, bool lut_bank_a, int rmcm_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+	const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B;
+
+	switch (id) {
+	case MCM_LUT_SHAPER: {
+		const struct pwl_params *lut_shaper = params.pwl;
+
+		if (lut_shaper == NULL)
+			return;
+		if (rmcm->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+			rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, true);
+
+		rmcm42_configure_shaper_lut(rmcm, next_mode == LUT_RAM_A, rmcm_id);
+
+		if (next_mode == LUT_RAM_A)
+			rmcm42_program_shaper_luta_settings(rmcm, lut_shaper, rmcm_id);
+		else
+			rmcm42_program_shaper_lutb_settings(rmcm, lut_shaper, rmcm_id);
+
+		rmcm42_program_shaper_lut(
+				rmcm, lut_shaper->rgb_resulted, lut_shaper->hw_points_num, rmcm_id);
+
+		rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, false);
+		break;
+	}
+	case MCM_LUT_3DLUT: {
+		const struct tetrahedral_params *lut3d = params.lut3d;
+		bool is_17x17x17;
+		bool is_12bits_color_channel;
+		const struct dc_rgb *lut0;
+		const struct dc_rgb *lut1;
+		const struct dc_rgb *lut2;
+		const struct dc_rgb *lut3;
+		int lut_size0;
+		int lut_size;
+
+		if (lut3d == NULL)
+			return;
+
+		rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, true);
+
+		/* Program bit depth before loading data */
+		is_12bits_color_channel = lut3d->use_12bits;
+		REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+			MPC_RMCM_3DLUT_WRITE_EN_MASK, 0xF);
+		REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+			MPC_RMCM_3DLUT_30BIT_EN,
+			is_12bits_color_channel ? 0 : 1);
+
+		is_17x17x17 = !lut3d->use_tetrahedral_9;
+		is_12bits_color_channel = lut3d->use_12bits;
+		if (is_17x17x17) {
+			lut0 = lut3d->tetrahedral_17.lut0;
+			lut1 = lut3d->tetrahedral_17.lut1;
+			lut2 = lut3d->tetrahedral_17.lut2;
+			lut3 = lut3d->tetrahedral_17.lut3;
+			lut_size0 = sizeof(lut3d->tetrahedral_17.lut0) /
+						sizeof(lut3d->tetrahedral_17.lut0[0]);
+			lut_size  = sizeof(lut3d->tetrahedral_17.lut1) /
+						sizeof(lut3d->tetrahedral_17.lut1[0]);
+		} else {
+			lut0 = lut3d->tetrahedral_9.lut0;
+			lut1 = lut3d->tetrahedral_9.lut1;
+			lut2 = lut3d->tetrahedral_9.lut2;
+			lut3 = lut3d->tetrahedral_9.lut3;
+			lut_size0 = sizeof(lut3d->tetrahedral_9.lut0) /
+					sizeof(lut3d->tetrahedral_9.lut0[0]);
+			lut_size  = sizeof(lut3d->tetrahedral_9.lut1) /
+					sizeof(lut3d->tetrahedral_9.lut1[0]);
+		}
+
+		rmcm42_select_3dlut_ram_mask(rmcm, 0x1, rmcm_id);
+		if (is_12bits_color_channel)
+			rmcm42_set3dlut_ram12(rmcm, lut0, lut_size0, rmcm_id);
+		else
+			rmcm42_set3dlut_ram10(rmcm, lut0, lut_size0, rmcm_id);
+
+		rmcm42_select_3dlut_ram_mask(rmcm, 0x2, rmcm_id);
+		if (is_12bits_color_channel)
+			rmcm42_set3dlut_ram12(rmcm, lut1, lut_size, rmcm_id);
+		else
+			rmcm42_set3dlut_ram10(rmcm, lut1, lut_size, rmcm_id);
+
+		rmcm42_select_3dlut_ram_mask(rmcm, 0x4, rmcm_id);
+		if (is_12bits_color_channel)
+			rmcm42_set3dlut_ram12(rmcm, lut2, lut_size, rmcm_id);
+		else
+			rmcm42_set3dlut_ram10(rmcm, lut2, lut_size, rmcm_id);
+
+		rmcm42_select_3dlut_ram_mask(rmcm, 0x8, rmcm_id);
+		if (is_12bits_color_channel)
+			rmcm42_set3dlut_ram12(rmcm, lut3, lut_size, rmcm_id);
+		else
+			rmcm42_set3dlut_ram10(rmcm, lut3, lut_size, rmcm_id);
+
+		if (rmcm->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+			rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, false);
+
+		break;
+	}
+	case MCM_LUT_1DLUT:
+		/* RMCM doesn't support 1DLUT */
+		break;
+	}
+}
+
+static void rmcm42_program_lut_read_write_control(struct rmcm *rmcm, const enum MCM_LUT_ID id,
+	bool lut_bank_a, bool enabled, int rmcm_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	switch (id) {
+	case MCM_LUT_3DLUT:
+		REG_UPDATE(MPC_RMCM_3DLUT_MODE[rmcm_id], MPC_RMCM_3DLUT_MODE,
+			(!enabled) ? 0 :
+			(lut_bank_a) ? 1 : 2);
+
+		REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+			MPC_RMCM_3DLUT_RAM_SEL,
+			(lut_bank_a) ? 0 : 1);
+		break;
+	case MCM_LUT_SHAPER:
+		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[rmcm_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
+
+		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[rmcm_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_SEL,
+			lut_bank_a == true ? 0:1);
+
+		REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[rmcm_id], 0,
+			MPC_RMCM_SHAPER_LUT_INDEX, 0);
+		break;
+	default:
+		break;
+	}
+}
+
+uint32_t rmcm42_get_3dlut_width(
+		const enum dc_cm_lut_size size)
+{
+	uint32_t width = 0;
+
+	switch (size) {
+	case CM_LUT_SIZE_999:
+		width = 1;
+		break;
+	case CM_LUT_SIZE_333333:
+		width = 2;
+		break;
+	case CM_LUT_SIZE_171717:
+	default:
+		width = 0;
+		break;
+	}
+
+	return width;
+}
+
+static void rmcm42_program_lut_mode(struct rmcm *rmcm,
+	const enum MCM_LUT_ID id,
+	bool enable,
+	bool lut_bank_a,
+	const enum dc_cm_lut_size size,
+	uint16_t bias,
+	uint16_t scale,
+	int rmcm_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	(void)bias;
+	(void)scale;
+
+	switch (id) {
+	case MCM_LUT_3DLUT:
+		if (enable) {
+			uint32_t lut_size = rmcm42_get_3dlut_width(size);
+
+			REG_UPDATE_2(MPC_RMCM_3DLUT_MODE[rmcm_id],
+					MPC_RMCM_3DLUT_MODE, lut_bank_a ? 1 : 2,
+					MPC_RMCM_3DLUT_SIZE, lut_size);
+
+			/* Default to 12-bit (30BIT_EN=0) for DMA path;
+			 * populate_lut overrides for host-load path.
+			 */
+			REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+				MPC_RMCM_3DLUT_WRITE_EN_MASK, 0xF);
+			REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+				MPC_RMCM_3DLUT_30BIT_EN, 0);
+		} else {
+			if (rmcm->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+				rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, false);
+			REG_UPDATE(MPC_RMCM_3DLUT_MODE[rmcm_id], MPC_RMCM_3DLUT_MODE, 0);
+		}
+		break;
+	case MCM_LUT_SHAPER:
+		if (enable) {
+			REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[rmcm_id],
+					MPC_RMCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
+		} else {
+			if (rmcm->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+				rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, false);
+			REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[rmcm_id],
+					MPC_RMCM_SHAPER_LUT_MODE, 0);
+		}
+		break;
+	case MCM_LUT_1DLUT:
+		/* RMCM doesn't support 1DLUT */
+		break;
+	}
+}
+
+static void rmcm42_get_lut_mode(struct rmcm *rmcm,
+		const enum MCM_LUT_ID id,
+		int rmcm_id,
+		bool *enable,
+		bool *lut_bank_a)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+	uint32_t lut_mode = 0;
+
+	*enable = false;
+	*lut_bank_a = true;
+
+	switch (id) {
+	case MCM_LUT_SHAPER:
+		REG_GET(MPC_RMCM_SHAPER_CONTROL[rmcm_id],
+				MPC_RMCM_SHAPER_MODE_CURRENT, &lut_mode);
+		*enable = lut_mode != 0;
+		*lut_bank_a = lut_mode != 2;
+		break;
+	case MCM_LUT_1DLUT:
+		/* RMCM doesn't support 1DLUT, return disabled */
+		*enable = false;
+		*lut_bank_a = true;
+		break;
+	case MCM_LUT_3DLUT:
+	default:
+		REG_GET(MPC_RMCM_3DLUT_MODE[rmcm_id],
+				MPC_RMCM_3DLUT_MODE_CURRENT, &lut_mode);
+		*enable = lut_mode != 0;
+		*lut_bank_a = lut_mode != 2;
+		break;
+	}
+}
+
+void rmcm42_get_3dlut_fast_load_status(struct rmcm *rmcm, int rmcm_id,
+	bool *done, bool *soft_underflow, bool *hard_underflow)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+	uint32_t fl_done = 0, fl_soft = 0, fl_hard = 0;
+
+	REG_GET_3(MPC_RMCM_3DLUT_FAST_LOAD_STATUS[rmcm_id],
+		MPC_RMCM_3DLUT_FL_DONE, &fl_done,
+		MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, &fl_soft,
+		MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, &fl_hard);
+
+	*done = (fl_done != 0);
+	*soft_underflow = (fl_soft != 0);
+	*hard_underflow = (fl_hard != 0);
+}
+
+void rmcm42_read_state(struct rmcm *rmcm, int rmcm_inst, struct rmcm_state *s)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	/* RMCM 3DLUT Status */
+	REG_GET_4(MPC_RMCM_MEM_PWR_CTRL[rmcm_inst], MPC_RMCM_3DLUT_MEM_PWR_FORCE, &s->regs.rmcm_3dlut_mem_pwr_force,
+			MPC_RMCM_3DLUT_MEM_PWR_DIS, &s->regs.rmcm_3dlut_mem_pwr_dis,
+			MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE, &s->regs.rmcm_3dlut_mem_pwr_mode,
+			MPC_RMCM_3DLUT_MEM_PWR_STATE, &s->regs.rmcm_3dlut_mem_pwr_state);
+
+	REG_GET_3(MPC_RMCM_3DLUT_MODE[rmcm_inst], MPC_RMCM_3DLUT_SIZE, &s->regs.rmcm_3dlut_size,
+			MPC_RMCM_3DLUT_MODE, &s->regs.rmcm_3dlut_mode,
+			MPC_RMCM_3DLUT_MODE_CURRENT, &s->regs.rmcm_3dlut_mode_cur);
+
+	REG_GET_4(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_inst], MPC_RMCM_3DLUT_READ_SEL, &s->regs.rmcm_3dlut_read_sel,
+			MPC_RMCM_3DLUT_30BIT_EN, &s->regs.rmcm_3dlut_30bit_en,
+			MPC_RMCM_3DLUT_WRITE_EN_MASK, &s->regs.rmcm_3dlut_wr_en_mask,
+			MPC_RMCM_3DLUT_RAM_SEL, &s->regs.rmcm_3dlut_ram_sel);
+
+	REG_GET(MPC_RMCM_3DLUT_OUT_NORM_FACTOR[rmcm_inst], MPC_RMCM_3DLUT_OUT_NORM_FACTOR, &s->regs.rmcm_3dlut_out_norm_factor);
+
+	REG_GET(MPC_RMCM_3DLUT_FAST_LOAD_SELECT[rmcm_inst], MPC_RMCM_3DLUT_FL_SEL, &s->regs.rmcm_3dlut_fl_sel);
+
+	REG_GET_2(MPC_RMCM_3DLUT_OUT_OFFSET_R[rmcm_inst], MPC_RMCM_3DLUT_OUT_OFFSET_R, &s->regs.rmcm_3dlut_out_offset_r,
+			MPC_RMCM_3DLUT_OUT_SCALE_R, &s->regs.rmcm_3dlut_out_scale_r);
+
+	REG_GET_3(MPC_RMCM_3DLUT_FAST_LOAD_STATUS[rmcm_inst], MPC_RMCM_3DLUT_FL_DONE, &s->regs.rmcm_3dlut_fl_done,
+			MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, &s->regs.rmcm_3dlut_fl_soft_underflow,
+			MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, &s->regs.rmcm_3dlut_fl_hard_underflow);
+
+	/* RMCM Shaper Status */
+	REG_GET_4(MPC_RMCM_MEM_PWR_CTRL[rmcm_inst], MPC_RMCM_SHAPER_MEM_PWR_FORCE, &s->regs.rmcm_shaper_mem_pwr_force,
+			MPC_RMCM_SHAPER_MEM_PWR_DIS, &s->regs.rmcm_shaper_mem_pwr_dis,
+			MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE, &s->regs.rmcm_shaper_mem_pwr_mode,
+			MPC_RMCM_SHAPER_MEM_PWR_STATE, &s->regs.rmcm_shaper_mem_pwr_state);
+
+	REG_GET_2(MPC_RMCM_SHAPER_CONTROL[rmcm_inst], MPC_RMCM_SHAPER_LUT_MODE, &s->regs.rmcm_shaper_lut_mode,
+			MPC_RMCM_SHAPER_MODE_CURRENT, &s->regs.rmcm_shaper_mode_cur);
+
+	REG_GET_2(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[rmcm_inst], MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, &s->regs.rmcm_shaper_lut_write_en_mask,
+			MPC_RMCM_SHAPER_LUT_WRITE_SEL, &s->regs.rmcm_shaper_lut_write_sel);
+
+	REG_GET(MPC_RMCM_SHAPER_OFFSET_B[rmcm_inst], MPC_RMCM_SHAPER_OFFSET_B, &s->regs.rmcm_shaper_offset_b);
+
+	REG_GET(MPC_RMCM_SHAPER_SCALE_G_B[rmcm_inst], MPC_RMCM_SHAPER_SCALE_B, &s->regs.rmcm_shaper_scale_b);
+
+	REG_GET_2(MPC_RMCM_SHAPER_RAMA_START_CNTL_B[rmcm_inst], MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, &s->regs.rmcm_shaper_rama_exp_region_start_b,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, &s->regs.rmcm_shaper_rama_exp_region_start_seg_b);
+
+	REG_GET_2(MPC_RMCM_SHAPER_RAMA_END_CNTL_B[rmcm_inst], MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, &s->regs.rmcm_shaper_rama_exp_region_end_b,
+			MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, &s->regs.rmcm_shaper_rama_exp_region_end_base_b);
+
+	REG_GET(MPC_RMCM_CNTL[rmcm_inst], MPC_RMCM_CNTL, &s->regs.rmcm_cntl);
+}
+
+/*
+ * rmcm42_connect_mpcc() - Program RMCM to MPCC routing mux
+ * @rmcm: RMCM instance
+ * @rmcm_id: RMCM hardware index
+ * @mpcc_id: MPCC to connect (0-3), or RMCM_MPCC_ID_NONE to disconnect
+ *
+ * Programs MPC_RMCM_CNTL to route the specified MPCC to this RMCM instance,
+ * allowing any MPCC[x] to RMCM[y] mapping.
+ */
+void rmcm42_connect_mpcc(struct rmcm *rmcm, int rmcm_id, int mpcc_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	REG_UPDATE(MPC_RMCM_CNTL[rmcm_id], MPC_RMCM_CNTL, mpcc_id);
+
+	rmcm->mpcc_id = mpcc_id;
+}
+
+const struct rmcm_funcs dcn42_rmcm_funcs = {
+	.get_lut_mode = rmcm42_get_lut_mode,
+	.program_lut_mode = rmcm42_program_lut_mode,
+	.populate_lut = rmcm42_populate_lut,
+	.program_lut_read_write_control = rmcm42_program_lut_read_write_control,
+	.update_3dlut_fast_load_select = rmcm42_update_3dlut_fast_load_select,
+	.get_3dlut_fast_load_status = rmcm42_get_3dlut_fast_load_status,
+	.read_rmcm_state = rmcm42_read_state,
+	.connect_mpcc = rmcm42_connect_mpcc,
+};
+
+void dcn42_rmcm_construct(
+	struct dcn42_rmcm *rmcm42,
+	struct dc_context *ctx,
+	const struct dcn42_rmcm_registers *regs,
+	const struct dcn42_rmcm_shift *shift,
+	const struct dcn42_rmcm_mask *mask,
+	int inst)
+{
+	rmcm42->base.ctx = ctx;
+	rmcm42->base.inst = inst;
+	rmcm42->base.mpcc_id = RMCM_MPCC_ID_UNKNOWN;
+	rmcm42->base.funcs = &dcn42_rmcm_funcs;
+	rmcm42->rmcm_regs = regs;
+	rmcm42->rmcm_shift = shift;
+	rmcm42->rmcm_mask = mask;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.h b/drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.h
new file mode 100644
index 00000000000000..caffc14fab057a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/rmcm/dcn42/dcn42_rmcm.h
@@ -0,0 +1,717 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RMCM_DCN42_H__
+#define __DC_RMCM_DCN42_H__
+
+#include "rmcm.h"
+
+#define TO_DCN42_RMCM(rmcm_base) \
+	container_of(rmcm_base, struct dcn42_rmcm, base)
+
+#define MPC_RMCM_COMMON_MASK_SH_LIST_DCN42(mask_sh) \
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_LUT_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_CONTROL, MPC_RMCM_SHAPER_MODE_CURRENT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_R, MPC_RMCM_SHAPER_OFFSET_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_G, MPC_RMCM_SHAPER_OFFSET_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_OFFSET_B, MPC_RMCM_SHAPER_OFFSET_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_R, MPC_RMCM_SHAPER_SCALE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM_SHAPER_SCALE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_INDEX, MPC_RMCM_SHAPER_LUT_INDEX, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_DATA, MPC_RMCM_SHAPER_LUT_DATA, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM_SHAPER_LUT_WRITE_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_SIZE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_MODE, MPC_RMCM_3DLUT_MODE_CURRENT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_INDEX, MPC_RMCM_3DLUT_INDEX, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA0, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA, MPC_RMCM_3DLUT_DATA1, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_DATA_30BIT, MPC_RMCM_3DLUT_DATA_30BIT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_WRITE_EN_MASK, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_RAM_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_30BIT_EN, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM_3DLUT_READ_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_NORM_FACTOR, MPC_RMCM_3DLUT_OUT_NORM_FACTOR, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_OFFSET_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM_3DLUT_OUT_SCALE_R, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_OFFSET_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM_3DLUT_OUT_SCALE_G, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_OFFSET_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM_3DLUT_OUT_SCALE_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM_GAMUT_REMAP_MODE_CURRENT, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C11_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM_GAMUT_REMAP_C12_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C13_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM_GAMUT_REMAP_C14_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C21_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM_GAMUT_REMAP_C22_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C23_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM_GAMUT_REMAP_C24_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C31_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM_GAMUT_REMAP_C32_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C33_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM_GAMUT_REMAP_C34_A, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C11_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM_GAMUT_REMAP_C12_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C13_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM_GAMUT_REMAP_C14_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C21_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM_GAMUT_REMAP_C22_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C23_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM_GAMUT_REMAP_C24_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C31_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM_GAMUT_REMAP_C32_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C33_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM_GAMUT_REMAP_C34_B, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_FORCE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_DIS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_FORCE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_DIS, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_SHAPER_MEM_PWR_STATE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM_3DLUT_MEM_PWR_STATE, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_SELECT, MPC_RMCM_3DLUT_FL_SEL, mask_sh),\
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_DONE, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_CNTL, MPC_RMCM_CNTL, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_INDEX, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM_TEST_DEBUG_WRITE_EN, mask_sh), \
+	SF(MPC_RMCM0_MPC_RMCM_TEST_DEBUG_DATA, MPC_RMCM_TEST_DEBUG_DATA, mask_sh)
+#define MPC_RMCM_REG_LIST_DCN42(inst)\
+	SRII(MPC_RMCM_SHAPER_CONTROL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_OFFSET_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_OFFSET_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_OFFSET_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_SCALE_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_SCALE_G_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_LUT_INDEX, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_LUT_DATA, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_START_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_END_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_0_1, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_2_3, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_4_5, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_6_7, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_8_9, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_10_11, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_12_13, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_14_15, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_16_17, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_18_19, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_20_21, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_22_23, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_24_25, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_26_27, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_28_29, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_30_31, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMA_REGION_32_33, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_START_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_END_CNTL_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_0_1, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_2_3, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_4_5, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_6_7, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_8_9, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_10_11, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_12_13, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_14_15, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_16_17, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_18_19, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_20_21, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_22_23, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_24_25, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_26_27, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_28_29, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_30_31, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_SHAPER_RAMB_REGION_32_33, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_MODE, MPC_RMCM, inst), /*TODO: may need to add other 3DLUT regs*/\
+	SRII(MPC_RMCM_3DLUT_INDEX, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_DATA, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_DATA_30BIT, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_READ_WRITE_CONTROL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_NORM_FACTOR, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_R, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_G, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_OUT_OFFSET_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_COEF_FORMAT, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_MODE, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C11_C12_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C13_C14_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C21_C22_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C23_C24_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C31_C32_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C33_C34_A, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C11_C12_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C13_C14_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C21_C22_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C23_C24_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C31_C32_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_GAMUT_REMAP_C33_C34_B, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_MEM_PWR_CTRL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_FAST_LOAD_SELECT, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_3DLUT_FAST_LOAD_STATUS, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_CNTL, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_TEST_DEBUG_INDEX, MPC_RMCM, inst),\
+	SRII(MPC_RMCM_TEST_DEBUG_DATA, MPC_RMCM, inst)
+
+#define MPC_RMCM_REG_FIELD_LIST_DCN42(type) \
+	type MPC_RMCM_SHAPER_LUT_MODE;\
+	type MPC_RMCM_SHAPER_MODE_CURRENT;\
+	type MPC_RMCM_SHAPER_OFFSET_R;\
+	type MPC_RMCM_SHAPER_OFFSET_G;\
+	type MPC_RMCM_SHAPER_OFFSET_B;\
+	type MPC_RMCM_SHAPER_SCALE_R;\
+	type MPC_RMCM_SHAPER_SCALE_G;\
+	type MPC_RMCM_SHAPER_SCALE_B;\
+	type MPC_RMCM_SHAPER_LUT_INDEX;\
+	type MPC_RMCM_SHAPER_LUT_DATA;\
+	type MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK;\
+	type MPC_RMCM_SHAPER_LUT_WRITE_SEL; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_B; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_B; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_G; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_G;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_START_SEGMENT_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_B;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_B;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_G;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_G;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION_END_BASE_R;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION0_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION0_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION1_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION1_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION2_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION2_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION3_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION3_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION4_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION4_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION5_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION5_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION6_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION6_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION7_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION7_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION8_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION8_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION9_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION9_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION10_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION10_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION11_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION11_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION12_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION12_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION13_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION13_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION14_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION14_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION15_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION15_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION16_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION16_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION17_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION17_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION18_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION18_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION19_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION19_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION20_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION20_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION21_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION21_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION22_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION22_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION23_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION23_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION24_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION24_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION25_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION25_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION26_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION26_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION27_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION27_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION28_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION28_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION29_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION29_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION30_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION30_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION31_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION31_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION32_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION32_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_B; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_B; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_G; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_G;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_START_SEGMENT_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_B;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_B;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_G;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_G;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION_END_BASE_R;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION0_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION0_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION1_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION1_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION2_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION2_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION3_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION3_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION4_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION4_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION5_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION5_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION6_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION6_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION7_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION7_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION8_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION8_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION9_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION9_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION10_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION10_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION11_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION11_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION12_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION12_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION13_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION13_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION14_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION14_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION15_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION15_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION16_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION16_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION17_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION17_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION18_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION18_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION19_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION19_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION20_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION20_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION21_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION21_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION22_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION22_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION23_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION23_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION24_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION24_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION25_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION25_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION26_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION26_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION27_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION27_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION28_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION28_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION29_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION29_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION30_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION30_NUM_SEGMENTS;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION31_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION31_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION32_LUT_OFFSET; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION32_NUM_SEGMENTS; \
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION33_LUT_OFFSET;\
+	type MPC_RMCM_SHAPER_RAMB_EXP_REGION33_NUM_SEGMENTS;\
+	type MPC_RMCM_3DLUT_MODE;\
+	type MPC_RMCM_3DLUT_SIZE;\
+	type MPC_RMCM_3DLUT_MODE_CURRENT;\
+	type MPC_RMCM_3DLUT_INDEX; \
+	type MPC_RMCM_3DLUT_DATA0; \
+	type MPC_RMCM_3DLUT_DATA1; \
+	type MPC_RMCM_3DLUT_DATA_30BIT; \
+	type MPC_RMCM_3DLUT_WRITE_EN_MASK;\
+	type MPC_RMCM_3DLUT_RAM_SEL;\
+	type MPC_RMCM_3DLUT_30BIT_EN;\
+	type MPC_RMCM_3DLUT_READ_SEL;\
+	type MPC_RMCM_3DLUT_OUT_NORM_FACTOR;\
+	type MPC_RMCM_3DLUT_OUT_OFFSET_R;\
+	type MPC_RMCM_3DLUT_OUT_SCALE_R; \
+	type MPC_RMCM_3DLUT_OUT_OFFSET_G; \
+	type MPC_RMCM_3DLUT_OUT_SCALE_G; \
+	type MPC_RMCM_3DLUT_OUT_OFFSET_B; \
+	type MPC_RMCM_3DLUT_OUT_SCALE_B;\
+	type MPC_RMCM_GAMUT_REMAP_COEF_FORMAT;\
+	type MPC_RMCM_GAMUT_REMAP_MODE;\
+	type MPC_RMCM_GAMUT_REMAP_MODE_CURRENT;\
+	type MPC_RMCM_GAMUT_REMAP_C11_A;\
+	type MPC_RMCM_GAMUT_REMAP_C12_A;\
+	type MPC_RMCM_GAMUT_REMAP_C13_A; \
+	type MPC_RMCM_GAMUT_REMAP_C14_A; \
+	type MPC_RMCM_GAMUT_REMAP_C21_A; \
+	type MPC_RMCM_GAMUT_REMAP_C22_A; \
+	type MPC_RMCM_GAMUT_REMAP_C23_A;\
+	type MPC_RMCM_GAMUT_REMAP_C24_A;\
+	type MPC_RMCM_GAMUT_REMAP_C31_A;\
+	type MPC_RMCM_GAMUT_REMAP_C32_A;\
+	type MPC_RMCM_GAMUT_REMAP_C33_A;\
+	type MPC_RMCM_GAMUT_REMAP_C34_A;\
+	type MPC_RMCM_GAMUT_REMAP_C11_B;\
+	type MPC_RMCM_GAMUT_REMAP_C12_B;\
+	type MPC_RMCM_GAMUT_REMAP_C13_B; \
+	type MPC_RMCM_GAMUT_REMAP_C14_B; \
+	type MPC_RMCM_GAMUT_REMAP_C21_B; \
+	type MPC_RMCM_GAMUT_REMAP_C22_B; \
+	type MPC_RMCM_GAMUT_REMAP_C23_B;\
+	type MPC_RMCM_GAMUT_REMAP_C24_B;\
+	type MPC_RMCM_GAMUT_REMAP_C31_B;\
+	type MPC_RMCM_GAMUT_REMAP_C32_B;\
+	type MPC_RMCM_GAMUT_REMAP_C33_B;\
+	type MPC_RMCM_GAMUT_REMAP_C34_B;\
+	type MPC_RMCM_SHAPER_MEM_PWR_FORCE; \
+	type MPC_RMCM_SHAPER_MEM_PWR_DIS; \
+	type MPC_RMCM_SHAPER_MEM_LOW_PWR_MODE; \
+	type MPC_RMCM_3DLUT_MEM_PWR_FORCE;\
+	type MPC_RMCM_3DLUT_MEM_PWR_DIS;\
+	type MPC_RMCM_3DLUT_MEM_LOW_PWR_MODE;\
+	type MPC_RMCM_SHAPER_MEM_PWR_STATE;\
+	type MPC_RMCM_3DLUT_MEM_PWR_STATE;\
+	type MPC_RMCM_3DLUT_FL_SEL;\
+	type MPC_RMCM_3DLUT_FL_DONE; \
+	type MPC_RMCM_3DLUT_FL_SOFT_UNDERFLOW; \
+	type MPC_RMCM_3DLUT_FL_HARD_UNDERFLOW; \
+	type MPC_RMCM_CNTL; \
+	type MPC_RMCM_TEST_DEBUG_INDEX;\
+	type MPC_RMCM_TEST_DEBUG_WRITE_EN;\
+	type MPC_RMCM_TEST_DEBUG_DATA
+
+#define MPC_RMCM_REG_VARIABLE_LIST_DCN42 \
+	uint32_t MPC_RMCM_SHAPER_CONTROL[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_OFFSET_R[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_OFFSET_G[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_OFFSET_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_SCALE_R[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_SCALE_G_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_LUT_INDEX[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_LUT_DATA[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_G[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_START_CNTL_R[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_G[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_END_CNTL_R[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_0_1[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_2_3[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_4_5[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_6_7[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_8_9[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_10_11[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_12_13[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_14_15[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_16_17[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_18_19[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_20_21[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_22_23[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_24_25[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_26_27[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_28_29[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_30_31[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMA_REGION_32_33[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_G[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_START_CNTL_R[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_G[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_END_CNTL_R[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_0_1[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_2_3[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_4_5[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_6_7[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_8_9[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_10_11[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_12_13[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_14_15[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_16_17[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_18_19[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_20_21[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_22_23[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_24_25[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_26_27[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_28_29[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_30_31[MAX_RMCM]; \
+	uint32_t MPC_RMCM_SHAPER_RAMB_REGION_32_33[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_MODE[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_INDEX[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_DATA[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_DATA_30BIT[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_READ_WRITE_CONTROL[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_NORM_FACTOR[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_R[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_G[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_OUT_OFFSET_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_COEF_FORMAT[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_MODE[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C11_C12_A[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C13_C14_A[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C21_C22_A[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C23_C24_A[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C31_C32_A[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C33_C34_A[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C11_C12_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C13_C14_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C21_C22_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C23_C24_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C31_C32_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_GAMUT_REMAP_C33_C34_B[MAX_RMCM]; \
+	uint32_t MPC_RMCM_MEM_PWR_CTRL[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_FAST_LOAD_SELECT[MAX_RMCM]; \
+	uint32_t MPC_RMCM_3DLUT_FAST_LOAD_STATUS[MAX_RMCM]; \
+	uint32_t MPC_RMCM_CNTL[MAX_RMCM]; \
+	uint32_t MPC_RMCM_TEST_DEBUG_INDEX[MAX_RMCM]; \
+	uint32_t MPC_RMCM_TEST_DEBUG_DATA[MAX_RMCM]
+
+struct dcn42_rmcm_shift {
+	MPC_RMCM_REG_FIELD_LIST_DCN42(uint8_t);
+};
+
+struct dcn42_rmcm_mask {
+	MPC_RMCM_REG_FIELD_LIST_DCN42(uint32_t);
+};
+
+struct dcn42_rmcm_registers {
+	MPC_RMCM_REG_VARIABLE_LIST_DCN42;
+};
+
+struct dcn42_rmcm {
+	struct rmcm base;
+	const struct dcn42_rmcm_registers *rmcm_regs;
+	const struct dcn42_rmcm_shift *rmcm_shift;
+	const struct dcn42_rmcm_mask *rmcm_mask;
+};
+
+void dcn42_rmcm_construct(
+	struct dcn42_rmcm *rmcm42,
+	struct dc_context *ctx,
+	const struct dcn42_rmcm_registers *regs,
+	const struct dcn42_rmcm_shift *shift,
+	const struct dcn42_rmcm_mask *mask,
+	int inst);
+
+/*
+ * Register-programming helpers shared with the DCN60 RMCM component. They are bank-count
+ * agnostic (LUT data, power, fast load, MPCC routing), so the single-SRAM DCN60 variant
+ * only overrides bank selection.
+ */
+void rmcm42_power_on_shaper_3dlut(struct rmcm *rmcm, uint32_t rmcm_id, bool power_on);
+uint32_t rmcm42_get_3dlut_width(const enum dc_cm_lut_size size);
+void rmcm42_populate_lut(struct rmcm *rmcm, const enum MCM_LUT_ID id,
+	const union rmcm_lut_params params, bool lut_bank_a, int rmcm_id);
+void rmcm42_update_3dlut_fast_load_select(struct rmcm *rmcm, int rmcm_id, int hubp_idx);
+void rmcm42_get_3dlut_fast_load_status(struct rmcm *rmcm, int rmcm_id,
+	bool *done, bool *soft_underflow, bool *hard_underflow);
+void rmcm42_read_state(struct rmcm *rmcm, int rmcm_inst, struct rmcm_state *s);
+void rmcm42_connect_mpcc(struct rmcm *rmcm, int rmcm_id, int mpcc_id);
+
+#endif // __DC_RMCM_DCN42_H__
diff --git a/drivers/gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.c b/drivers/gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.c
new file mode 100644
index 00000000000000..df4d6476d70a23
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.c
@@ -0,0 +1,251 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+#include "reg_helper.h"
+
+#include "dcn/dcn_6_0_0_offset.h"
+#include "dcn/dcn_6_0_0_sh_mask.h"
+
+#include "dcn60_rmcm.h"
+
+#define SF(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+/* DCN60 renamed MPC_RMCM_3DLUT_RAM_SEL to MPC_RMCM_3DLUT_READ_SEL */
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_RAM_SEL__SHIFT \
+	MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_READ_SEL__SHIFT
+#define MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_RAM_SEL_MASK \
+	MPC_RMCM0_MPC_RMCM_3DLUT_READ_WRITE_CONTROL__MPC_RMCM_3DLUT_READ_SEL_MASK
+
+/* Resource-level macros needed by MPC_RMCM_REG_LIST_DCN42 */
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SRII(reg_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+static struct dcn42_rmcm_registers rmcm_regs;
+
+#define dcn_rmcm_regs_init()               \
+	MPC_RMCM_REG_LIST_DCN42(0),            \
+		MPC_RMCM_REG_LIST_DCN42(1)
+
+static const struct dcn42_rmcm_shift rmcm_shift = {
+	MPC_RMCM_COMMON_MASK_SH_LIST_DCN42(__SHIFT)};
+
+static const struct dcn42_rmcm_mask rmcm_mask = {
+	MPC_RMCM_COMMON_MASK_SH_LIST_DCN42(_MASK)};
+
+#define REG(reg)\
+	rmcm42->rmcm_regs->reg
+
+#define CTX \
+	rmcm42->base.ctx
+
+#undef FN
+#define FN(reg_name, field_name) \
+	rmcm42->rmcm_shift->field_name, rmcm42->rmcm_mask->field_name
+
+/*
+ * DCN60 RMCM LUT-bank programming (single-SRAM variant).
+ *
+ * DCN60 has one LUT SRAM per RMCM instance instead of the DCN42 A/B pair, and the 3DLUT
+ * read/write-control RAM select was renamed RAM_SEL -> READ_SEL. These overrides always
+ * target bank A and ignore lut_bank_a, so nothing can select a bank that does not exist.
+ * Everything bank-agnostic (LUT upload, fast load, power, MPCC routing) is reused from the
+ * DCN42 rmcm42_* helpers.
+ */
+static void rmcm60_get_lut_mode(struct rmcm *rmcm,
+		const enum MCM_LUT_ID id,
+		int rmcm_id,
+		bool *enable,
+		bool *lut_bank_a)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+	uint32_t lut_mode = 0;
+
+	*enable = false;
+	*lut_bank_a = true; /* single SRAM: always bank A */
+
+	switch (id) {
+	case MCM_LUT_SHAPER:
+		REG_GET(MPC_RMCM_SHAPER_CONTROL[rmcm_id],
+				MPC_RMCM_SHAPER_MODE_CURRENT, &lut_mode);
+		*enable = lut_mode != 0;
+		break;
+	case MCM_LUT_1DLUT:
+		/* RMCM doesn't support 1DLUT, return disabled */
+		*enable = false;
+		break;
+	case MCM_LUT_3DLUT:
+	default:
+		REG_GET(MPC_RMCM_3DLUT_MODE[rmcm_id],
+				MPC_RMCM_3DLUT_MODE_CURRENT, &lut_mode);
+		*enable = lut_mode != 0;
+		break;
+	}
+}
+
+static void rmcm60_program_lut_mode(struct rmcm *rmcm,
+	const enum MCM_LUT_ID id,
+	bool enable,
+	bool lut_bank_a,
+	const enum dc_cm_lut_size size,
+	uint16_t bias,
+	uint16_t scale,
+	int rmcm_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	(void)bias;
+	(void)scale;
+	(void)lut_bank_a; /* single SRAM: always bank A (mode 1) */
+
+	switch (id) {
+	case MCM_LUT_3DLUT:
+		if (enable) {
+			uint32_t lut_size = rmcm42_get_3dlut_width(size);
+
+			REG_UPDATE_2(MPC_RMCM_3DLUT_MODE[rmcm_id],
+					MPC_RMCM_3DLUT_MODE, 1,
+					MPC_RMCM_3DLUT_SIZE, lut_size);
+
+			/* Default to 12-bit (30BIT_EN=0) for DMA path;
+			 * populate_lut overrides for host-load path.
+			 */
+			REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+				MPC_RMCM_3DLUT_WRITE_EN_MASK, 0xF);
+			REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+				MPC_RMCM_3DLUT_30BIT_EN, 0);
+		} else {
+			if (rmcm->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+				rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, false);
+			REG_UPDATE(MPC_RMCM_3DLUT_MODE[rmcm_id], MPC_RMCM_3DLUT_MODE, 0);
+		}
+		break;
+	case MCM_LUT_SHAPER:
+		if (enable) {
+			REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[rmcm_id],
+					MPC_RMCM_SHAPER_LUT_MODE, 1);
+		} else {
+			if (rmcm->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+				rmcm42_power_on_shaper_3dlut(rmcm, rmcm_id, false);
+			REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[rmcm_id],
+					MPC_RMCM_SHAPER_LUT_MODE, 0);
+		}
+		break;
+	case MCM_LUT_1DLUT:
+		/* RMCM doesn't support 1DLUT */
+		break;
+	}
+}
+
+static void rmcm60_program_lut_read_write_control(struct rmcm *rmcm, const enum MCM_LUT_ID id,
+	bool lut_bank_a, bool enabled, int rmcm_id)
+{
+	struct dcn42_rmcm *rmcm42 = TO_DCN42_RMCM(rmcm);
+
+	(void)lut_bank_a; /* single SRAM: always bank A */
+
+	switch (id) {
+	case MCM_LUT_3DLUT:
+		REG_UPDATE(MPC_RMCM_3DLUT_MODE[rmcm_id], MPC_RMCM_3DLUT_MODE,
+			enabled ? 1 : 0);
+
+		/* DCN60 renamed RAM_SEL -> READ_SEL; single bank -> always 0 */
+		REG_UPDATE(MPC_RMCM_3DLUT_READ_WRITE_CONTROL[rmcm_id],
+			MPC_RMCM_3DLUT_RAM_SEL, 0);
+		break;
+	case MCM_LUT_SHAPER:
+		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[rmcm_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK, 7);
+
+		REG_UPDATE(MPC_RMCM_SHAPER_LUT_WRITE_EN_MASK[rmcm_id],
+			MPC_RMCM_SHAPER_LUT_WRITE_SEL, 0);
+
+		REG_SET(MPC_RMCM_SHAPER_LUT_INDEX[rmcm_id], 0,
+			MPC_RMCM_SHAPER_LUT_INDEX, 0);
+		break;
+	default:
+		break;
+	}
+}
+
+/* DCN60 has no RAM_SEL - it became READ_SEL, which the reused DCN42 read_state maps onto
+ * via the compat macro. Clear the duplicate so the reported state stays unambiguous.
+ */
+static void rmcm60_read_state(struct rmcm *rmcm, int rmcm_inst,
+		struct rmcm_state *s)
+{
+	rmcm42_read_state(rmcm, rmcm_inst, s);
+	s->regs.rmcm_3dlut_ram_sel = 0;
+}
+
+static const struct rmcm_funcs dcn60_rmcm_funcs = {
+	.get_lut_mode = rmcm60_get_lut_mode,
+	.program_lut_mode = rmcm60_program_lut_mode,
+	.populate_lut = rmcm42_populate_lut,
+	.program_lut_read_write_control = rmcm60_program_lut_read_write_control,
+	.update_3dlut_fast_load_select = rmcm42_update_3dlut_fast_load_select,
+	.get_3dlut_fast_load_status = rmcm42_get_3dlut_fast_load_status,
+	.read_rmcm_state = rmcm60_read_state,
+	.connect_mpcc = rmcm42_connect_mpcc,
+};
+
+struct rmcm *dcn60_rmcm_create(struct dc_context *ctx, int inst)
+{
+	struct dcn42_rmcm *rmcm42 = kzalloc(sizeof(struct dcn42_rmcm), GFP_KERNEL);
+
+	if (!rmcm42)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT rmcm_regs
+	dcn_rmcm_regs_init();
+
+	dcn42_rmcm_construct(rmcm42,
+		ctx,
+		&rmcm_regs,
+		&rmcm_shift,
+		&rmcm_mask,
+		inst);
+
+	/* Override the DCN42 defaults with the DCN60 single-SRAM LUT-bank ops. */
+	rmcm42->base.funcs = &dcn60_rmcm_funcs;
+
+	return &rmcm42->base;
+}
+
+void dcn60_rmcm_destroy(struct rmcm **rmcm)
+{
+	if (*rmcm) {
+		kfree(TO_DCN42_RMCM(*rmcm));
+		*rmcm = NULL;
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.h b/drivers/gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.h
new file mode 100644
index 00000000000000..8a8b2db2762a72
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/rmcm/dcn60/dcn60_rmcm.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RMCM_DCN60_H__
+#define __DC_RMCM_DCN60_H__
+
+#include "dcn42/dcn42_rmcm.h"
+
+struct rmcm *dcn60_rmcm_create(struct dc_context *ctx, int inst);
+
+void dcn60_rmcm_destroy(struct rmcm **rmcm);
+
+#endif /* __DC_RMCM_DCN60_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
index 0c8e652c35321e..c3dfcf2a73e345 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
@@ -278,11 +278,11 @@ void dcn401_update_soc_bb_with_values_from_software_policy(struct dml2_soc_bb *s
 	 *  bits 16-23: dcfclk_derate_percent
 	 */
 	for (unsigned int i = 0; i < dc->debug.dml21_custom_derate_num_dpms; i++) {
-		soc_bb->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dram_derate_percent_pixel[i]
+		soc_bb->qos_parameters.derate_table_per_dpm.dram_per_dpm_derate_pixel[i].derate_percent
 			= dc->debug.dml21_custom_derate_at_dpm[i] & 0xFF;
-		soc_bb->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.fclk_derate_percent[i]
+		soc_bb->qos_parameters.derate_table_per_dpm.fclk_per_dpm_derate[i].derate_percent
 			= (dc->debug.dml21_custom_derate_at_dpm[i] >> 8) & 0xFF;
-		soc_bb->qos_parameters.derate_table_per_dpm.system_active_derates_per_dpm.dcfclk_derate_percent[i]
+		soc_bb->qos_parameters.derate_table_per_dpm.dcfclk_per_dpm_derate[i].derate_percent
 			= (dc->debug.dml21_custom_derate_at_dpm[i] >> 16) & 0xFF;
 	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn60/dcn60_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn60/dcn60_soc_and_ip_translator.c
index c26ad090397521..f1ee2225e9b8f1 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn60/dcn60_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn60/dcn60_soc_and_ip_translator.c
@@ -5,6 +5,7 @@
 #include "dcn60_soc_and_ip_translator.h"
 #include "soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.h"
 #include "bounding_boxes/dcn6_soc_bb.h"
+#include "dal_asic_id.h"
 
 /* soc_and_ip_translator component used to get up-to-date values for bounding box.
  * Bounding box values are stored in several locations and locations can vary with DCN revision.
@@ -94,7 +95,10 @@ static void apply_soc_bb_updates(struct dml2_soc_bb *soc_bb, const struct dc *dc
 static void dcn60_get_soc_bb(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config)
 {
 	//get default soc_bb with static values
-	dcn6_test_initialize_soc_bb(soc_bb);
+	if (ASICREV_IS_DCN6_VARIANT_LITE3(dc->ctx->asic_id.hw_internal_rev))
+		dcn6b_test_initialize_soc_bb(soc_bb);
+	else
+		dcn6_test_initialize_soc_bb(soc_bb);
 	//get default soc_bb with static values
 	apply_soc_bb_updates(soc_bb, dc, config);
 
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/Makefile b/drivers/gpu/drm/amd/display/dc/sspl/Makefile
index 5e3e4aa1382031..fdf37a2408be9f 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/sspl/Makefile
@@ -23,7 +23,10 @@
 # Makefile for the 'spl' sub-component of DAL.
 # It provides the scaling library interface.
 
-SPL = dc_spl.o dc_spl_scl_filters.o dc_spl_scl_easf_filters.o dc_spl_isharp_filters.o dc_spl_filters.o spl_fixpt31_32.o spl_custom_float.o
+# spl_fixpt31_32 and spl_custom_float are now provided by the host driver
+# (dc_fixpt_* / convert_to_custom_float_format) via the shim headers
+# spl_fixpt31_32.h / spl_custom_float.h.
+SPL = dc_spl.o dc_spl_scl_filters.o dc_spl_scl_easf_filters.o dc_spl_isharp_filters.o dc_spl_filters.o
 
 AMD_DAL_SPL = $(addprefix $(AMDDALPATH)/dc/sspl/,$(SPL))
 
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
index 1f1318769d5b36..f14e3069acfd56 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl.c
@@ -7,8 +7,22 @@
 #include "dc_spl_isharp_filters.h"
 #include "spl_debug.h"
 
-#define IDENTITY_RATIO(ratio) (SPL_NAMESPACE(spl_fixpt_u3d19(ratio)) == (1 << 19))
+/*
+ * Custom float is provided by the parent repo's custom_float.h. It is included
+ * here (after dc_spl.h -> dc_spl_types.h, which fully defines struct
+ * dscl_prog_data) so the opp.h -> transform.h -> dc_spl_types.h chain that
+ * custom_float.h drags in does not re-enter dc_spl_types.h before
+ * dscl_prog_data is complete.
+ */
+#define spl_div64_s64 div64_s64
+#define spl_swap      swap
+#include "custom_float.h"
+#define spl_custom_float_format custom_float_format
+#define spl_convert_to_custom_float_format convert_to_custom_float_format
+
+#define IDENTITY_RATIO(ratio) (spl_fixpt_u3d19(ratio) == (1 << 19))
 #define MIN_VIEWPORT_SIZE 12
+
 static bool spl_is_yuv420(enum spl_pixel_format format)
 {
 	if ((format >= SPL_PIXEL_FORMAT_420BPP8) &&
@@ -169,24 +183,24 @@ static struct spl_rect calculate_plane_rec_in_timing_active(
 	struct spl_fixed31_32 temp;
 
 
-	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
+	temp = spl_fixpt_from_fraction(
 			rec_in->x * (long long)stream_dst->width,
-			stream_src->width));
+			stream_src->width);
 	rec_out.x = stream_dst->x + spl_fixpt_round(temp);
 
-	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
+	temp = spl_fixpt_from_fraction(
 			(rec_in->x + rec_in->width) * (long long)stream_dst->width,
-			stream_src->width));
+			stream_src->width);
 	rec_out.width = stream_dst->x + spl_fixpt_round(temp) - rec_out.x;
 
-	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
+	temp = spl_fixpt_from_fraction(
 			rec_in->y * (long long)stream_dst->height,
-			stream_src->height));
+			stream_src->height);
 	rec_out.y = stream_dst->y + spl_fixpt_round(temp);
 
-	temp = SPL_NAMESPACE(spl_fixpt_from_fraction(
+	temp = spl_fixpt_from_fraction(
 			(rec_in->y + rec_in->height) * (long long)stream_dst->height,
-			stream_src->height));
+			stream_src->height);
 	rec_out.height = stream_dst->y + spl_fixpt_round(temp) - rec_out.y;
 
 	return rec_out;
@@ -453,12 +467,12 @@ static void spl_calculate_scaling_ratios(struct spl_in *spl_in,
 		spl_in->basic_in.rotation == SPL_ROTATION_ANGLE_270)
 		spl_swap(surf_src.height, surf_src.width);
 
-	spl_scratch->scl_data.ratios.horz = SPL_NAMESPACE(spl_fixpt_from_fraction(
+	spl_scratch->scl_data.ratios.horz = spl_fixpt_from_fraction(
 					surf_src.width,
-					spl_in->basic_in.dst_rect.width));
-	spl_scratch->scl_data.ratios.vert = SPL_NAMESPACE(spl_fixpt_from_fraction(
+					spl_in->basic_in.dst_rect.width);
+	spl_scratch->scl_data.ratios.vert = spl_fixpt_from_fraction(
 					surf_src.height,
-					spl_in->basic_in.dst_rect.height));
+					spl_in->basic_in.dst_rect.height);
 
 	if (spl_in->basic_out.view_format == SPL_VIEW_3D_SIDE_BY_SIDE)
 		spl_scratch->scl_data.ratios.horz.value *= 2;
@@ -497,14 +511,14 @@ static void spl_calculate_scaling_ratios(struct spl_in *spl_in,
 	 * that is output/input.  Currently we calculate input/output
 	 * Store 1/ratio in recip_ratio for those lookups
 	 */
-	spl_scratch->scl_data.recip_ratios.horz = SPL_NAMESPACE(spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.horz));
-	spl_scratch->scl_data.recip_ratios.vert = SPL_NAMESPACE(spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.vert));
-	spl_scratch->scl_data.recip_ratios.horz_c = SPL_NAMESPACE(spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.horz_c));
-	spl_scratch->scl_data.recip_ratios.vert_c = SPL_NAMESPACE(spl_fixpt_recip(
-			spl_scratch->scl_data.ratios.vert_c));
+	spl_scratch->scl_data.recip_ratios.horz = spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.horz);
+	spl_scratch->scl_data.recip_ratios.vert = spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.vert);
+	spl_scratch->scl_data.recip_ratios.horz_c = spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.horz_c);
+	spl_scratch->scl_data.recip_ratios.vert_c = spl_fixpt_recip(
+			spl_scratch->scl_data.ratios.vert_c);
 }
 
 /* Calculate Viewport size */
@@ -664,11 +678,11 @@ static void spl_calculate_inits_and_viewports(struct spl_in *spl_in,
 
 		switch (spl_in->basic_in.cositing) {
 		case CHROMA_COSITING_TOPLEFT:
-			init_adj_h = SPL_NAMESPACE(spl_fixpt_from_fraction(h_sign, 4));
-			init_adj_v = SPL_NAMESPACE(spl_fixpt_from_fraction(v_sign, 4));
+			init_adj_h = spl_fixpt_from_fraction(h_sign, 4);
+			init_adj_v = spl_fixpt_from_fraction(v_sign, 4);
 			break;
 		case CHROMA_COSITING_LEFT:
-			init_adj_h = SPL_NAMESPACE(spl_fixpt_from_fraction(h_sign, 4));
+			init_adj_h = spl_fixpt_from_fraction(h_sign, 4);
 			init_adj_v = spl_fixpt_zero;
 			break;
 		case CHROMA_COSITING_NONE:
@@ -961,16 +975,16 @@ static void spl_get_taps_non_adaptive_scaler(
 	 * Max downscale supported is 6.0x.  Add ASSERT to catch if go beyond that
 	 */
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.horz,
-		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
+		spl_fixpt_from_fraction(6, 1));
 	SPL_ASSERT(check_max_downscale);
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.vert,
-		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
+		spl_fixpt_from_fraction(6, 1));
 	SPL_ASSERT(check_max_downscale);
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.horz_c,
-		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
+		spl_fixpt_from_fraction(6, 1));
 	SPL_ASSERT(check_max_downscale);
 	check_max_downscale = spl_fixpt_le(spl_scratch->scl_data.ratios.vert_c,
-		SPL_NAMESPACE(spl_fixpt_from_fraction(6, 1)));
+		spl_fixpt_from_fraction(6, 1));
 	SPL_ASSERT(check_max_downscale);
 
 
@@ -1216,39 +1230,39 @@ static void spl_set_manual_ratio_init_data(struct dscl_prog_data *dscl_prog_data
 {
 	struct spl_fixed31_32 bot;
 
-	dscl_prog_data->ratios.h_scale_ratio = SPL_NAMESPACE(spl_fixpt_u3d19(
-			scl_data->ratios.horz)) << 5;
-	dscl_prog_data->ratios.v_scale_ratio = SPL_NAMESPACE(spl_fixpt_u3d19(
-			scl_data->ratios.vert)) << 5;
-	dscl_prog_data->ratios.h_scale_ratio_c = SPL_NAMESPACE(spl_fixpt_u3d19(
-			scl_data->ratios.horz_c)) << 5;
-	dscl_prog_data->ratios.v_scale_ratio_c = SPL_NAMESPACE(spl_fixpt_u3d19(
-			scl_data->ratios.vert_c)) << 5;
+	dscl_prog_data->ratios.h_scale_ratio = spl_fixpt_u3d19(
+			scl_data->ratios.horz) << 5;
+	dscl_prog_data->ratios.v_scale_ratio = spl_fixpt_u3d19(
+			scl_data->ratios.vert) << 5;
+	dscl_prog_data->ratios.h_scale_ratio_c = spl_fixpt_u3d19(
+			scl_data->ratios.horz_c) << 5;
+	dscl_prog_data->ratios.v_scale_ratio_c = spl_fixpt_u3d19(
+			scl_data->ratios.vert_c) << 5;
 	/*
 	 * 0.24 format for fraction, first five bits zeroed
 	 */
 	dscl_prog_data->init.h_filter_init_frac =
-			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.h)) << 5;
+			spl_fixpt_u0d19(scl_data->inits.h) << 5;
 	dscl_prog_data->init.h_filter_init_int =
 			spl_fixpt_floor(scl_data->inits.h);
 	dscl_prog_data->init.h_filter_init_frac_c =
-			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.h_c)) << 5;
+			spl_fixpt_u0d19(scl_data->inits.h_c) << 5;
 	dscl_prog_data->init.h_filter_init_int_c =
 			spl_fixpt_floor(scl_data->inits.h_c);
 	dscl_prog_data->init.v_filter_init_frac =
-			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.v)) << 5;
+			spl_fixpt_u0d19(scl_data->inits.v) << 5;
 	dscl_prog_data->init.v_filter_init_int =
 			spl_fixpt_floor(scl_data->inits.v);
 	dscl_prog_data->init.v_filter_init_frac_c =
-			SPL_NAMESPACE(spl_fixpt_u0d19(scl_data->inits.v_c)) << 5;
+			spl_fixpt_u0d19(scl_data->inits.v_c) << 5;
 	dscl_prog_data->init.v_filter_init_int_c =
 			spl_fixpt_floor(scl_data->inits.v_c);
 
 	bot = spl_fixpt_add(scl_data->inits.v, scl_data->ratios.vert);
-	dscl_prog_data->init.v_filter_init_bot_frac = SPL_NAMESPACE(spl_fixpt_u0d19(bot)) << 5;
+	dscl_prog_data->init.v_filter_init_bot_frac = spl_fixpt_u0d19(bot) << 5;
 	dscl_prog_data->init.v_filter_init_bot_int = spl_fixpt_floor(bot);
 	bot = spl_fixpt_add(scl_data->inits.v_c, scl_data->ratios.vert_c);
-	dscl_prog_data->init.v_filter_init_bot_frac_c = SPL_NAMESPACE(spl_fixpt_u0d19(bot)) << 5;
+	dscl_prog_data->init.v_filter_init_bot_frac_c = spl_fixpt_u0d19(bot) << 5;
 	dscl_prog_data->init.v_filter_init_bot_int_c = spl_fixpt_floor(bot);
 }
 
@@ -1312,31 +1326,31 @@ static void spl_calculate_c0_c3_hdr(struct dscl_prog_data *dscl_prog_data, uint3
 	else
 		hdr_multx100_int = 100; /* default for 80 nits otherwise */
 
-	hdr_mult = SPL_NAMESPACE(spl_fixpt_from_fraction((long long)hdr_multx100_int, 100LL));
-	c0_mult = SPL_NAMESPACE(spl_fixpt_from_fraction(2126LL, 10000LL));
-	c1_mult = SPL_NAMESPACE(spl_fixpt_from_fraction(7152LL, 10000LL));
-	c2_mult = SPL_NAMESPACE(spl_fixpt_from_fraction(722LL, 10000LL));
+	hdr_mult = spl_fixpt_from_fraction((long long)hdr_multx100_int, 100LL);
+	c0_mult = spl_fixpt_from_fraction(2126LL, 10000LL);
+	c1_mult = spl_fixpt_from_fraction(7152LL, 10000LL);
+	c2_mult = spl_fixpt_from_fraction(722LL, 10000LL);
 
-	c0_calc = SPL_NAMESPACE(spl_fixpt_mul(hdr_mult, SPL_NAMESPACE(spl_fixpt_mul(c0_mult,
-		SPL_NAMESPACE(spl_fixpt_from_fraction(16384LL, 125LL))))));
-	c1_calc = SPL_NAMESPACE(spl_fixpt_mul(hdr_mult, SPL_NAMESPACE(spl_fixpt_mul(c1_mult,
-		SPL_NAMESPACE(spl_fixpt_from_fraction(16384LL, 125LL))))));
-	c2_calc = SPL_NAMESPACE(spl_fixpt_mul(hdr_mult, SPL_NAMESPACE(spl_fixpt_mul(c2_mult,
-		SPL_NAMESPACE(spl_fixpt_from_fraction(16384LL, 125LL))))));
+	c0_calc = spl_fixpt_mul(hdr_mult, spl_fixpt_mul(c0_mult,
+		spl_fixpt_from_fraction(16384LL, 125LL)));
+	c1_calc = spl_fixpt_mul(hdr_mult, spl_fixpt_mul(c1_mult,
+		spl_fixpt_from_fraction(16384LL, 125LL)));
+	c2_calc = spl_fixpt_mul(hdr_mult, spl_fixpt_mul(c2_mult,
+		spl_fixpt_from_fraction(16384LL, 125LL)));
 
 	fmt.exponenta_bits = 5;
 	fmt.mantissa_bits = 10;
 	fmt.sign = true;
 
 	// fp1.5.10, C0 coefficient (LN_rec709:  HDR_MULT * 0.212600 * 2^14/125)
-	SPL_NAMESPACE(spl_convert_to_custom_float_format(c0_calc, &fmt,
-		&dscl_prog_data->easf_matrix_c0));
+	spl_convert_to_custom_float_format(c0_calc, &fmt,
+		&dscl_prog_data->easf_matrix_c0);
 	// fp1.5.10, C1 coefficient (LN_rec709:  HDR_MULT * 0.715200 * 2^14/125)
-	SPL_NAMESPACE(spl_convert_to_custom_float_format(c1_calc, &fmt,
-		&dscl_prog_data->easf_matrix_c1));
+	spl_convert_to_custom_float_format(c1_calc, &fmt,
+		&dscl_prog_data->easf_matrix_c1);
 	// fp1.5.10, C2 coefficient (LN_rec709:  HDR_MULT * 0.072200 * 2^14/125)
-	SPL_NAMESPACE(spl_convert_to_custom_float_format(c2_calc, &fmt,
-		&dscl_prog_data->easf_matrix_c2));
+	spl_convert_to_custom_float_format(c2_calc, &fmt,
+		&dscl_prog_data->easf_matrix_c2);
 	dscl_prog_data->easf_matrix_c3 = 0x0; // fp1.5.10, C3 coefficient
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c
index a75ab23b0726a0..66f0e9d971be99 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_isharp_filters.c
@@ -367,8 +367,8 @@ static unsigned int spl_calculate_sharpness_level_adj(struct spl_fixed31_32 rati
 	sharpness_level_down_adj = 0;
 	lookup_ptr = sharpness_level_adj;
 	while (j < NUM_SHARPNESS_ADJ_LEVELS) {
-		ratio_level = SPL_NAMESPACE(spl_fixpt_from_fraction(lookup_ptr->ratio_numer,
-			lookup_ptr->ratio_denom));
+		ratio_level = spl_fixpt_from_fraction(lookup_ptr->ratio_numer,
+			lookup_ptr->ratio_denom);
 		if (ratio.value >= ratio_level.value) {
 			sharpness_level_down_adj = lookup_ptr->level_down_adj;
 			break;
@@ -462,7 +462,7 @@ void SPL_NAMESPACE(spl_build_isharp_1dlut_from_reference_curve(
 	unsigned int sharpnessX1000 = spl_calculate_sharpness_level(ratio,
 			sharpness.sharpness_level, setup,
 			sharpness.sharpness_range, scale_to_sharpness_policy);
-	sharp_level = SPL_NAMESPACE(spl_fixpt_from_fraction(sharpnessX1000, 1000));
+	sharp_level = spl_fixpt_from_fraction(sharpnessX1000, 1000);
 
 	/*
 	 * Check if pregen 1dlut table is already precalculated
@@ -487,11 +487,11 @@ void SPL_NAMESPACE(spl_build_isharp_1dlut_from_reference_curve(
 	memset(byte_ptr_1dlut_dst, 0, size_1dlut);
 	for (j = 0; j < size_1dlut; j++) {
 		sharp_base = spl_fixpt_from_int((int)*byte_ptr_1dlut_src);
-		sharp_calc = SPL_NAMESPACE(spl_fixpt_mul(sharp_base, sharp_level));
+		sharp_calc = spl_fixpt_mul(sharp_base, sharp_level);
 		sharp_calc = spl_fixpt_div(sharp_calc, spl_fixpt_from_int(3));
 		sharp_calc = spl_fixpt_min(spl_fixpt_from_int(255), sharp_calc);
 		sharp_calc = spl_fixpt_add(sharp_calc,
-			SPL_NAMESPACE(spl_fixpt_from_fraction(1, 2)));
+			spl_fixpt_from_fraction(1, 2));
 		sharp_calc_int = spl_fixpt_floor(sharp_calc);
 		/* Clamp it at 0x7F so it doesn't wrap */
 		if (sharp_calc_int > 127)
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c
index de16ee58607389..ffc35041d90104 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_easf_filters.c
@@ -2194,19 +2194,19 @@ static struct scale_ratio_to_reg_value_lookup easf_3tap_uptilt2_offset_lookup[]
 
 static const uint16_t *spl_get_easf_filter_3tap_64p(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
+	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
 		return easf_filter_3tap_64p_ratio_0_30_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
 		return easf_filter_3tap_64p_ratio_0_40_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
 		return easf_filter_3tap_64p_ratio_0_50_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
 		return easf_filter_3tap_64p_ratio_0_60_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
 		return easf_filter_3tap_64p_ratio_0_70_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
 		return easf_filter_3tap_64p_ratio_0_80_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
 		return easf_filter_3tap_64p_ratio_0_90_s1_12;
 	else
 		return easf_filter_3tap_64p_ratio_1_00_s1_12;
@@ -2214,19 +2214,19 @@ static const uint16_t *spl_get_easf_filter_3tap_64p(struct spl_fixed31_32 ratio)
 
 static const uint16_t *spl_get_easf_filter_4tap_64p(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
+	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
 		return easf_filter_4tap_64p_ratio_0_30_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
 		return easf_filter_4tap_64p_ratio_0_40_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
 		return easf_filter_4tap_64p_ratio_0_50_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
 		return easf_filter_4tap_64p_ratio_0_60_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
 		return easf_filter_4tap_64p_ratio_0_70_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
 		return easf_filter_4tap_64p_ratio_0_80_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
 		return easf_filter_4tap_64p_ratio_0_90_s1_12;
 	else
 		return easf_filter_4tap_64p_ratio_1_00_s1_12;
@@ -2234,19 +2234,19 @@ static const uint16_t *spl_get_easf_filter_4tap_64p(struct spl_fixed31_32 ratio)
 
 static const uint16_t *spl_get_easf_filter_6tap_64p(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
+	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
 		return easf_filter_6tap_64p_ratio_0_30_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
 		return easf_filter_6tap_64p_ratio_0_40_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
 		return easf_filter_6tap_64p_ratio_0_50_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
 		return easf_filter_6tap_64p_ratio_0_60_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
 		return easf_filter_6tap_64p_ratio_0_70_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
 		return easf_filter_6tap_64p_ratio_0_80_s1_12;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
 		return easf_filter_6tap_64p_ratio_0_90_s1_12;
 	else
 		return easf_filter_6tap_64p_ratio_1_00_s1_12;
@@ -2270,19 +2270,19 @@ const uint16_t *SPL_NAMESPACE(spl_dscl_get_easf_filter_coeffs_64p(
 
 static const uint16_t *spl_get_easf_filter_3tap_64p_s1_10(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
+	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
 		return easf_filter_3tap_64p_ratio_0_30;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
 		return easf_filter_3tap_64p_ratio_0_40;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
 		return easf_filter_3tap_64p_ratio_0_50;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
 		return easf_filter_3tap_64p_ratio_0_60;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
 		return easf_filter_3tap_64p_ratio_0_70;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
 		return easf_filter_3tap_64p_ratio_0_80;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
 		return easf_filter_3tap_64p_ratio_0_90;
 	else
 		return easf_filter_3tap_64p_ratio_1_00;
@@ -2290,19 +2290,19 @@ static const uint16_t *spl_get_easf_filter_3tap_64p_s1_10(struct spl_fixed31_32
 
 static const uint16_t *spl_get_easf_filter_4tap_64p_s1_10(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
+	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
 		return easf_filter_4tap_64p_ratio_0_30;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
 		return easf_filter_4tap_64p_ratio_0_40;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
 		return easf_filter_4tap_64p_ratio_0_50;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
 		return easf_filter_4tap_64p_ratio_0_60;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
 		return easf_filter_4tap_64p_ratio_0_70;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
 		return easf_filter_4tap_64p_ratio_0_80;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
 		return easf_filter_4tap_64p_ratio_0_90;
 	else
 		return easf_filter_4tap_64p_ratio_1_00;
@@ -2310,19 +2310,19 @@ static const uint16_t *spl_get_easf_filter_4tap_64p_s1_10(struct spl_fixed31_32
 
 static const uint16_t *spl_get_easf_filter_6tap_64p_s1_10(struct spl_fixed31_32 ratio)
 {
-	if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(3, 10)).value)
+	if (ratio.value < spl_fixpt_from_fraction(3, 10).value)
 		return easf_filter_6tap_64p_ratio_0_30;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 10).value)
 		return easf_filter_6tap_64p_ratio_0_40;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 10).value)
 		return easf_filter_6tap_64p_ratio_0_50;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(6, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(6, 10).value)
 		return easf_filter_6tap_64p_ratio_0_60;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(7, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(7, 10).value)
 		return easf_filter_6tap_64p_ratio_0_70;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(8, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(8, 10).value)
 		return easf_filter_6tap_64p_ratio_0_80;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(9, 10)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(9, 10).value)
 		return easf_filter_6tap_64p_ratio_0_90;
 	else
 		return easf_filter_6tap_64p_ratio_1_00;
@@ -2397,9 +2397,9 @@ static uint32_t spl_easf_get_scale_ratio_to_reg_value(struct spl_fixed31_32 rati
 		if (lookup_table_index_ptr->numer < 0)
 			break;
 
-		if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(
+		if (ratio.value < spl_fixpt_from_fraction(
 			lookup_table_index_ptr->numer,
-			lookup_table_index_ptr->denom)).value) {
+			lookup_table_index_ptr->denom).value) {
 			value = lookup_table_index_ptr->reg_value;
 			break;
 		}
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c
index 2d73d0dce5ffd6..b6306c33378a44 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_scl_filters.c
@@ -1134,9 +1134,9 @@ static const uint16_t *spl_get_filter_3tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_3tap_64p_upscale;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
 		return filter_3tap_64p_116;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
 		return filter_3tap_64p_149;
 	else
 		return filter_3tap_64p_183;
@@ -1146,9 +1146,9 @@ static const uint16_t *spl_get_filter_4tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_4tap_64p_upscale;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
 		return filter_4tap_64p_116;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
 		return filter_4tap_64p_149;
 	else
 		return filter_4tap_64p_183;
@@ -1158,9 +1158,9 @@ static const uint16_t *spl_get_filter_5tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_5tap_64p_upscale;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
 		return filter_5tap_64p_116;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
 		return filter_5tap_64p_149;
 	else
 		return filter_5tap_64p_183;
@@ -1170,9 +1170,9 @@ static const uint16_t *spl_get_filter_6tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_6tap_64p_upscale;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
 		return filter_6tap_64p_116;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
 		return filter_6tap_64p_149;
 	else
 		return filter_6tap_64p_183;
@@ -1182,9 +1182,9 @@ static const uint16_t *spl_get_filter_7tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_7tap_64p_upscale;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
 		return filter_7tap_64p_116;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
 		return filter_7tap_64p_149;
 	else
 		return filter_7tap_64p_183;
@@ -1194,9 +1194,9 @@ static const uint16_t *spl_get_filter_8tap_64p(struct spl_fixed31_32 ratio)
 {
 	if (ratio.value < spl_fixpt_one.value)
 		return filter_8tap_64p_upscale;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(4, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(4, 3).value)
 		return filter_8tap_64p_116;
-	else if (ratio.value < SPL_NAMESPACE(spl_fixpt_from_fraction(5, 3)).value)
+	else if (ratio.value < spl_fixpt_from_fraction(5, 3).value)
 		return filter_8tap_64p_149;
 	else
 		return filter_8tap_64p_183;
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_types.h b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_types.h
index b24f1ef3fe17bb..1f14f1b6e084de 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_types.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/dc_spl_types.h
@@ -7,8 +7,64 @@
 
 #include "spl_debug.h"
 #include "spl_os_types.h"   // swap
-#include "spl_fixpt31_32.h"	// fixed31_32 and related functions
-#include "spl_custom_float.h" // custom float and related functions
+
+/*
+ * Fixed-point (spl_fixpt_*) and custom-float (spl_custom_float_*) are provided
+ * by parent repo.  The spl_* API is aliased onto the parent repo's own
+ * fixed-point and custom-float implementation; both share the same
+ * "struct fixed31_32" / "struct custom_float_format".
+ */
+
+#include "os_types.h"   /* ASSERT, div64_u64_rem, div64_s64, swap */
+
+#include "fixed31_32.h"
+
+#define spl_fixed31_32          fixed31_32
+
+#define spl_fixpt_zero          dc_fixpt_zero
+#define spl_fixpt_epsilon       dc_fixpt_epsilon
+#define spl_fixpt_half          dc_fixpt_half
+#define spl_fixpt_one           dc_fixpt_one
+
+#define spl_fixpt_from_fraction dc_fixpt_from_fraction
+#define spl_fixpt_from_int      dc_fixpt_from_int
+#define spl_fixpt_neg           dc_fixpt_neg
+#define spl_fixpt_abs           dc_fixpt_abs
+#define spl_fixpt_lt            dc_fixpt_lt
+#define spl_fixpt_le            dc_fixpt_le
+#define spl_fixpt_eq            dc_fixpt_eq
+#define spl_fixpt_min           dc_fixpt_min
+#define spl_fixpt_max           dc_fixpt_max
+#define spl_fixpt_clamp         dc_fixpt_clamp
+#define spl_fixpt_shl           dc_fixpt_shl
+#define spl_fixpt_shr           dc_fixpt_shr
+#define spl_fixpt_add           dc_fixpt_add
+#define spl_fixpt_add_int       dc_fixpt_add_int
+#define spl_fixpt_sub           dc_fixpt_sub
+#define spl_fixpt_sub_int       dc_fixpt_sub_int
+#define spl_fixpt_mul           dc_fixpt_mul
+#define spl_fixpt_mul_int       dc_fixpt_mul_int
+#define spl_fixpt_sqr           dc_fixpt_sqr
+#define spl_fixpt_div_int       dc_fixpt_div_int
+#define spl_fixpt_div           dc_fixpt_div
+#define spl_fixpt_recip         dc_fixpt_recip
+#define spl_fixpt_sinc          dc_fixpt_sinc
+#define spl_fixpt_sin           dc_fixpt_sin
+#define spl_fixpt_cos           dc_fixpt_cos
+#define spl_fixpt_exp           dc_fixpt_exp
+#define spl_fixpt_log           dc_fixpt_log
+#define spl_fixpt_pow           dc_fixpt_pow
+#define spl_fixpt_floor         dc_fixpt_floor
+#define spl_fixpt_round         dc_fixpt_round
+#define spl_fixpt_ceil          dc_fixpt_ceil
+#define spl_fixpt_u4d19         dc_fixpt_u4d19
+#define spl_fixpt_u3d19         dc_fixpt_u3d19
+#define spl_fixpt_u2d19         dc_fixpt_u2d19
+#define spl_fixpt_u0d19         dc_fixpt_u0d19
+#define spl_fixpt_clamp_u0d14   dc_fixpt_clamp_u0d14
+#define spl_fixpt_clamp_u0d10   dc_fixpt_clamp_u0d10
+#define spl_fixpt_s4d19         dc_fixpt_s4d19
+#define spl_fixpt_truncate      dc_fixpt_truncate
 
 struct spl_size {
 	uint32_t width;
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c b/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c
deleted file mode 100644
index 0700b3dbbda76e..00000000000000
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.c
+++ /dev/null
@@ -1,152 +0,0 @@
-// SPDX-License-Identifier: MIT
-//
-// Copyright 2024 Advanced Micro Devices, Inc.
-
-#include "spl_debug.h"
-#include "spl_custom_float.h"
-
-static bool spl_build_custom_float(struct spl_fixed31_32 value,
-			       const struct spl_custom_float_format *format,
-			       bool *negative,
-			       uint32_t *mantissa,
-			       uint32_t *exponenta)
-{
-	uint32_t exp_offset = (1 << (format->exponenta_bits - 1)) - 1;
-
-	const struct spl_fixed31_32 mantissa_constant_plus_max_fraction =
-		SPL_NAMESPACE(spl_fixpt_from_fraction((1LL << (format->mantissa_bits + 1)) - 1,
-				       1LL << format->mantissa_bits));
-
-	struct spl_fixed31_32 mantiss;
-
-	if (spl_fixpt_eq(value, spl_fixpt_zero)) {
-		*negative = false;
-		*mantissa = 0;
-		*exponenta = 0;
-		return true;
-	}
-
-	if (spl_fixpt_lt(value, spl_fixpt_zero)) {
-		*negative = format->sign;
-		value = spl_fixpt_neg(value);
-	} else {
-		*negative = false;
-	}
-
-	if (spl_fixpt_lt(value, spl_fixpt_one)) {
-		uint32_t i = 1;
-
-		do {
-			value = spl_fixpt_shl(value, 1);
-			++i;
-		} while (spl_fixpt_lt(value, spl_fixpt_one));
-
-		--i;
-
-		if (exp_offset <= i) {
-			*mantissa = 0;
-			*exponenta = 0;
-			return true;
-		}
-
-		*exponenta = exp_offset - i;
-	} else if (spl_fixpt_le(mantissa_constant_plus_max_fraction, value)) {
-		uint32_t i = 1;
-
-		do {
-			value = spl_fixpt_shr(value, 1);
-			++i;
-		} while (spl_fixpt_lt(mantissa_constant_plus_max_fraction, value));
-
-		*exponenta = exp_offset + i - 1;
-	} else {
-		*exponenta = exp_offset;
-	}
-
-	mantiss = spl_fixpt_sub(value, spl_fixpt_one);
-
-	if (spl_fixpt_lt(mantiss, spl_fixpt_zero) ||
-	    spl_fixpt_lt(spl_fixpt_one, mantiss))
-		mantiss = spl_fixpt_zero;
-	else
-		mantiss = spl_fixpt_shl(mantiss, format->mantissa_bits);
-
-	*mantissa = spl_fixpt_floor(mantiss);
-
-	return true;
-}
-
-static bool spl_setup_custom_float(const struct spl_custom_float_format *format,
-			       bool negative,
-			       uint32_t mantissa,
-			       uint32_t exponenta,
-			       uint32_t *result)
-{
-	uint32_t i = 0;
-	uint32_t j = 0;
-	uint32_t value = 0;
-
-	/* verification code:
-	 * once calculation is ok we can remove it
-	 */
-
-	const uint32_t mantissa_mask =
-		(1 << (format->mantissa_bits + 1)) - 1;
-
-	const uint32_t exponenta_mask =
-		(1 << (format->exponenta_bits + 1)) - 1;
-
-	if (mantissa & ~mantissa_mask) {
-		SPL_BREAK_TO_DEBUGGER();
-		mantissa = mantissa_mask;
-	}
-
-	if (exponenta & ~exponenta_mask) {
-		SPL_BREAK_TO_DEBUGGER();
-		exponenta = exponenta_mask;
-	}
-
-	/* end of verification code */
-
-	while (i < format->mantissa_bits) {
-		uint32_t mask = 1 << i;
-
-		if (mantissa & mask)
-			value |= mask;
-
-		++i;
-	}
-
-	while (j < format->exponenta_bits) {
-		uint32_t mask = 1 << j;
-
-		if (exponenta & mask)
-			value |= mask << i;
-
-		++j;
-	}
-
-	if (negative && format->sign)
-		value |= 1 << (i + j);
-
-	*result = value;
-
-	return true;
-}
-
-bool SPL_NAMESPACE(spl_convert_to_custom_float_format(
-	struct spl_fixed31_32 value,
-	const struct spl_custom_float_format *format,
-	uint32_t *result))
-{
-	uint32_t mantissa;
-	uint32_t exponenta;
-	bool negative;
-
-	return spl_build_custom_float(value, format, &negative, &mantissa, &exponenta) &&
-				  spl_setup_custom_float(format,
-						     negative,
-						     mantissa,
-						     exponenta,
-						     result);
-}
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h
deleted file mode 100644
index f3fd8d30e6380b..00000000000000
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_custom_float.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-
-/* Copyright 2024 Advanced Micro Devices, Inc. */
-
-#ifndef SPL_CUSTOM_FLOAT_H_
-#define SPL_CUSTOM_FLOAT_H_
-
-#include "spl_os_types.h"
-#include "spl_fixpt31_32.h"
-
-struct spl_custom_float_format {
-	uint32_t mantissa_bits;
-	uint32_t exponenta_bits;
-	bool sign;
-};
-
-struct spl_custom_float_value {
-	uint32_t mantissa;
-	uint32_t exponenta;
-	uint32_t value;
-	bool negative;
-};
-
-bool SPL_NAMESPACE(spl_convert_to_custom_float_format(
-	struct spl_fixed31_32 value,
-	const struct spl_custom_float_format *format,
-	uint32_t *result));
-
-#endif //SPL_CUSTOM_FLOAT_H_
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c b/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c
deleted file mode 100644
index ff0bdc3c33c807..00000000000000
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.c
+++ /dev/null
@@ -1,495 +0,0 @@
-// SPDX-License-Identifier: MIT
-//
-// Copyright 2024 Advanced Micro Devices, Inc.
-
-#include "spl_fixpt31_32.h"
-
-static const struct spl_fixed31_32 spl_fixpt_two_pi = { 26986075409LL };
-static const struct spl_fixed31_32 spl_fixpt_ln2 = { 2977044471LL };
-static const struct spl_fixed31_32 spl_fixpt_ln2_div_2 = { 1488522236LL };
-
-static inline unsigned long long abs_i64(
-	long long arg)
-{
-	if (arg > 0)
-		return (unsigned long long)arg;
-	else
-		return (unsigned long long)(-arg);
-}
-
-/*
- * @brief
- * result = dividend / divisor
- * *remainder = dividend % divisor
- */
-static inline unsigned long long spl_complete_integer_division_u64(
-	unsigned long long dividend,
-	unsigned long long divisor,
-	unsigned long long *remainder)
-{
-	unsigned long long result;
-
-	result = spl_div64_u64_rem(dividend, divisor, remainder);
-
-	return result;
-}
-
-
-#define FRACTIONAL_PART_MASK \
-	((1ULL << FIXED31_32_BITS_PER_FRACTIONAL_PART) - 1)
-
-#define GET_INTEGER_PART(x) \
-	((x) >> FIXED31_32_BITS_PER_FRACTIONAL_PART)
-
-#define GET_FRACTIONAL_PART(x) \
-	(FRACTIONAL_PART_MASK & (x))
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_fraction(
-	long long numerator, long long denominator))
-{
-	struct spl_fixed31_32 res;
-
-	bool arg1_negative = numerator < 0;
-	bool arg2_negative = denominator < 0;
-
-	unsigned long long arg1_value = arg1_negative ? -numerator : numerator;
-	unsigned long long arg2_value = arg2_negative ? -denominator : denominator;
-
-	unsigned long long remainder;
-
-	/* determine integer part */
-
-	unsigned long long res_value = spl_complete_integer_division_u64(
-		arg1_value, arg2_value, &remainder);
-
-	SPL_ASSERT(res_value <= (unsigned long long)LONG_MAX);
-
-	/* determine fractional part */
-	{
-		unsigned int i = FIXED31_32_BITS_PER_FRACTIONAL_PART;
-
-		do {
-			remainder <<= 1;
-
-			res_value <<= 1;
-
-			if (remainder >= arg2_value) {
-				res_value |= 1;
-				remainder -= arg2_value;
-			}
-		} while (--i != 0);
-	}
-
-	/* round up LSB */
-	{
-		unsigned long long summand = (remainder << 1) >= arg2_value;
-
-		SPL_ASSERT(res_value <= (unsigned long long)LLONG_MAX - summand);
-
-		res_value += summand;
-	}
-
-	res.value = (long long)res_value;
-
-	if (arg1_negative ^ arg2_negative)
-		res.value = -res.value;
-
-	return res;
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_mul(
-	struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2))
-{
-	struct spl_fixed31_32 res;
-
-	bool arg1_negative = arg1.value < 0;
-	bool arg2_negative = arg2.value < 0;
-
-	unsigned long long arg1_value = arg1_negative ? -arg1.value : arg1.value;
-	unsigned long long arg2_value = arg2_negative ? -arg2.value : arg2.value;
-
-	unsigned long long arg1_int = GET_INTEGER_PART(arg1_value);
-	unsigned long long arg2_int = GET_INTEGER_PART(arg2_value);
-
-	unsigned long long arg1_fra = GET_FRACTIONAL_PART(arg1_value);
-	unsigned long long arg2_fra = GET_FRACTIONAL_PART(arg2_value);
-
-	unsigned long long tmp;
-
-	res.value = arg1_int * arg2_int;
-
-	SPL_ASSERT(res.value <= (long long)LONG_MAX);
-
-	res.value <<= FIXED31_32_BITS_PER_FRACTIONAL_PART;
-
-	tmp = arg1_int * arg2_fra;
-
-	SPL_ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
-
-	res.value += tmp;
-
-	tmp = arg2_int * arg1_fra;
-
-	SPL_ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
-
-	res.value += tmp;
-
-	tmp = arg1_fra * arg2_fra;
-
-	tmp = (tmp >> FIXED31_32_BITS_PER_FRACTIONAL_PART) +
-		(tmp >= (unsigned long long)spl_fixpt_half.value);
-
-	SPL_ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
-
-	res.value += tmp;
-
-	if (arg1_negative ^ arg2_negative)
-		res.value = -res.value;
-
-	return res;
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sqr(struct spl_fixed31_32 arg))
-{
-	struct spl_fixed31_32 res;
-
-	unsigned long long arg_value = abs_i64(arg.value);
-
-	unsigned long long arg_int = GET_INTEGER_PART(arg_value);
-
-	unsigned long long arg_fra = GET_FRACTIONAL_PART(arg_value);
-
-	unsigned long long tmp;
-
-	res.value = arg_int * arg_int;
-
-	SPL_ASSERT(res.value <= (long long)LONG_MAX);
-
-	res.value <<= FIXED31_32_BITS_PER_FRACTIONAL_PART;
-
-	tmp = arg_int * arg_fra;
-
-	SPL_ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
-
-	res.value += tmp;
-
-	SPL_ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
-
-	res.value += tmp;
-
-	tmp = arg_fra * arg_fra;
-
-	tmp = (tmp >> FIXED31_32_BITS_PER_FRACTIONAL_PART) +
-		(tmp >= (unsigned long long)spl_fixpt_half.value);
-
-	SPL_ASSERT(tmp <= (unsigned long long)(LLONG_MAX - res.value));
-
-	res.value += tmp;
-
-	return res;
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_recip(struct spl_fixed31_32 arg))
-{
-	/*
-	 * @note
-	 * Good idea to use Newton's method
-	 */
-
-	return SPL_NAMESPACE(spl_fixpt_from_fraction(
-		spl_fixpt_one.value,
-		arg.value));
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sinc(struct spl_fixed31_32 arg))
-{
-	struct spl_fixed31_32 square;
-
-	struct spl_fixed31_32 res = spl_fixpt_one;
-
-	int n = 27;
-
-	struct spl_fixed31_32 arg_norm = arg;
-
-	if (spl_fixpt_le(
-		spl_fixpt_two_pi,
-		spl_fixpt_abs(arg))) {
-		arg_norm = spl_fixpt_sub(
-			arg_norm,
-			spl_fixpt_mul_int(
-				spl_fixpt_two_pi,
-				(int)spl_div64_s64(
-					arg_norm.value,
-					spl_fixpt_two_pi.value)));
-	}
-
-	square = SPL_NAMESPACE(spl_fixpt_sqr(arg_norm));
-
-	do {
-		res = spl_fixpt_sub(
-			spl_fixpt_one,
-			spl_fixpt_div_int(
-				SPL_NAMESPACE(spl_fixpt_mul(
-					square,
-					res)),
-				n * (n - 1)));
-
-		n -= 2;
-	} while (n > 2);
-
-	if (arg.value != arg_norm.value)
-		res = spl_fixpt_div(
-			SPL_NAMESPACE(spl_fixpt_mul(res, arg_norm)),
-			arg);
-
-	return res;
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sin(struct spl_fixed31_32 arg))
-{
-	return SPL_NAMESPACE(spl_fixpt_mul(
-		arg,
-		SPL_NAMESPACE(spl_fixpt_sinc(arg))));
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_cos(struct spl_fixed31_32 arg))
-{
-	/* TODO implement argument normalization */
-
-	const struct spl_fixed31_32 square = SPL_NAMESPACE(spl_fixpt_sqr(arg));
-
-	struct spl_fixed31_32 res = spl_fixpt_one;
-
-	int n = 26;
-
-	do {
-		res = spl_fixpt_sub(
-			spl_fixpt_one,
-			spl_fixpt_div_int(
-				SPL_NAMESPACE(spl_fixpt_mul(
-					square,
-					res)),
-				n * (n - 1)));
-
-		n -= 2;
-	} while (n != 0);
-
-	return res;
-}
-
-/*
- * @brief
- * result = exp(arg),
- * where abs(arg) < 1
- *
- * Calculated as Taylor series.
- */
-static struct spl_fixed31_32 spl_fixed31_32_exp_from_taylor_series(struct spl_fixed31_32 arg)
-{
-	unsigned int n = 9;
-
-	struct spl_fixed31_32 res = SPL_NAMESPACE(spl_fixpt_from_fraction(
-		n + 2,
-		n + 1));
-	/* TODO find correct res */
-
-	SPL_ASSERT(spl_fixpt_lt(arg, spl_fixpt_one));
-
-	do
-		res = spl_fixpt_add(
-			spl_fixpt_one,
-			spl_fixpt_div_int(
-				SPL_NAMESPACE(spl_fixpt_mul(
-					arg,
-					res)),
-				n));
-	while (--n != 1);
-
-	return spl_fixpt_add(
-		spl_fixpt_one,
-		SPL_NAMESPACE(spl_fixpt_mul(
-			arg,
-			res)));
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_exp(struct spl_fixed31_32 arg))
-{
-	/*
-	 * @brief
-	 * Main equation is:
-	 * exp(x) = exp(r + m * ln(2)) = (1 << m) * exp(r),
-	 * where m = round(x / ln(2)), r = x - m * ln(2)
-	 */
-
-	if (spl_fixpt_le(
-		spl_fixpt_ln2_div_2,
-		spl_fixpt_abs(arg))) {
-		int m = spl_fixpt_round(
-			spl_fixpt_div(
-				arg,
-				spl_fixpt_ln2));
-
-		struct spl_fixed31_32 r = spl_fixpt_sub(
-			arg,
-			spl_fixpt_mul_int(
-				spl_fixpt_ln2,
-				m));
-
-		SPL_ASSERT(m != 0);
-
-		SPL_ASSERT(spl_fixpt_lt(
-			spl_fixpt_abs(r),
-			spl_fixpt_one));
-
-		if (m > 0)
-			return spl_fixpt_shl(
-				spl_fixed31_32_exp_from_taylor_series(r),
-				(unsigned int)m);
-		else
-			return spl_fixpt_div_int(
-				spl_fixed31_32_exp_from_taylor_series(r),
-				1LL << -m);
-	} else if (arg.value != 0)
-		return spl_fixed31_32_exp_from_taylor_series(arg);
-	else
-		return spl_fixpt_one;
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_log(struct spl_fixed31_32 arg))
-{
-	struct spl_fixed31_32 res = spl_fixpt_neg(spl_fixpt_one);
-	/* TODO improve 1st estimation */
-
-	struct spl_fixed31_32 error;
-
-	SPL_ASSERT(arg.value > 0);
-	/* TODO if arg is negative, return NaN */
-	/* TODO if arg is zero, return -INF */
-
-	do {
-		struct spl_fixed31_32 res1 = spl_fixpt_add(
-			spl_fixpt_sub(
-				res,
-				spl_fixpt_one),
-			spl_fixpt_div(
-				arg,
-				SPL_NAMESPACE(spl_fixpt_exp(res))));
-
-		error = spl_fixpt_sub(
-			res,
-			res1);
-
-		res = res1;
-		/* TODO determine max_allowed_error based on quality of exp() */
-	} while (abs_i64(error.value) > 100ULL);
-
-	return res;
-}
-
-
-/* this function is a generic helper to translate fixed point value to
- * specified integer format that will consist of integer_bits integer part and
- * fractional_bits fractional part. For example it is used in
- * spl_fixpt_u2d19 to receive 2 bits integer part and 19 bits fractional
- * part in 32 bits. It is used in hw programming (scaler)
- */
-
-static inline unsigned int spl_ux_dy(
-	long long value,
-	unsigned int integer_bits,
-	unsigned int fractional_bits)
-{
-	/* 1. create mask of integer part */
-	unsigned int result = (1 << integer_bits) - 1;
-	/* 2. mask out fractional part */
-	unsigned int fractional_part = FRACTIONAL_PART_MASK & value;
-	/* 3. shrink fixed point integer part to be of integer_bits width*/
-	result &= GET_INTEGER_PART(value);
-	/* 4. make space for fractional part to be filled in after integer */
-	result <<= fractional_bits;
-	/* 5. shrink fixed point fractional part to of fractional_bits width*/
-	fractional_part >>= FIXED31_32_BITS_PER_FRACTIONAL_PART - fractional_bits;
-	/* 6. merge the result */
-	return result | fractional_part;
-}
-
-static inline unsigned int spl_clamp_ux_dy(
-	long long value,
-	unsigned int integer_bits,
-	unsigned int fractional_bits,
-	unsigned int min_clamp)
-{
-	unsigned int truncated_val = spl_ux_dy(value, integer_bits, fractional_bits);
-
-	if (value >= (1LL << (integer_bits + FIXED31_32_BITS_PER_FRACTIONAL_PART)))
-		return (1 << (integer_bits + fractional_bits)) - 1;
-	else if (truncated_val > min_clamp)
-		return truncated_val;
-	else
-		return min_clamp;
-}
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u4d19(struct spl_fixed31_32 arg))
-{
-	return spl_ux_dy(arg.value, 4, 19);
-}
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u3d19(struct spl_fixed31_32 arg))
-{
-	return spl_ux_dy(arg.value, 3, 19);
-}
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u2d19(struct spl_fixed31_32 arg))
-{
-	return spl_ux_dy(arg.value, 2, 19);
-}
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u0d19(struct spl_fixed31_32 arg))
-{
-	return spl_ux_dy(arg.value, 0, 19);
-}
-
-unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d14(struct spl_fixed31_32 arg))
-{
-	return spl_clamp_ux_dy(arg.value, 0, 14, 1);
-}
-
-unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d10(struct spl_fixed31_32 arg))
-{
-	return spl_clamp_ux_dy(arg.value, 0, 10, 1);
-}
-
-int SPL_NAMESPACE(spl_fixpt_s4d19(struct spl_fixed31_32 arg))
-{
-	if (arg.value < 0)
-		return -(int)spl_ux_dy(spl_fixpt_abs(arg).value, 4, 19);
-	else
-		return spl_ux_dy(arg.value, 4, 19);
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_ux_dy(unsigned int value,
-	unsigned int integer_bits,
-	unsigned int fractional_bits))
-{
-	struct spl_fixed31_32 fixpt_value = spl_fixpt_zero;
-	struct spl_fixed31_32 fixpt_int_value = spl_fixpt_zero;
-	long long frac_mask = ((long long)1 << (long long)integer_bits) - 1;
-
-	fixpt_value.value = (long long)value << (FIXED31_32_BITS_PER_FRACTIONAL_PART - fractional_bits);
-	frac_mask = frac_mask << fractional_bits;
-	fixpt_int_value.value = value & frac_mask;
-	fixpt_int_value.value <<= (FIXED31_32_BITS_PER_FRACTIONAL_PART - fractional_bits);
-	fixpt_value.value |= fixpt_int_value.value;
-	return fixpt_value;
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_int_dy(unsigned int int_value,
-	unsigned int frac_value,
-	unsigned int integer_bits,
-	unsigned int fractional_bits))
-{
-	struct spl_fixed31_32 fixpt_value = spl_fixpt_from_int(int_value);
-
-	fixpt_value.value |= (long long)frac_value << (FIXED31_32_BITS_PER_FRACTIONAL_PART - fractional_bits);
-	return fixpt_value;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h
deleted file mode 100644
index b0e639d6e97de7..00000000000000
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_fixpt31_32.h
+++ /dev/null
@@ -1,526 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-
-/* Copyright 2024 Advanced Micro Devices, Inc. */
-
-#ifndef __SPL_FIXED31_32_H__
-#define __SPL_FIXED31_32_H__
-
-#include "spl_debug.h"
-#include "spl_os_types.h"   // swap
-
-#ifndef LLONG_MAX
-#define LLONG_MAX 9223372036854775807ll
-#endif
-#ifndef LLONG_MIN
-#define LLONG_MIN (-LLONG_MAX - 1ll)
-#endif
-
-#define FIXED31_32_BITS_PER_FRACTIONAL_PART 32
-#ifndef LLONG_MIN
-#define LLONG_MIN (1LL<<63)
-#endif
-#ifndef LLONG_MAX
-#define LLONG_MAX (-1LL>>1)
-#endif
-
-/*
- * @brief
- * Arithmetic operations on real numbers
- * represented as fixed-point numbers.
- * There are: 1 bit for sign,
- * 31 bit for integer part,
- * 32 bits for fractional part.
- *
- * @note
- * Currently, overflows and underflows are asserted;
- * no special result returned.
- */
-
-struct spl_fixed31_32 {
-	long long value;
-};
-
-
-/*
- * @brief
- * Useful constants
- */
-
-static const struct spl_fixed31_32 spl_fixpt_zero = { 0 };
-static const struct spl_fixed31_32 spl_fixpt_epsilon = { 1LL };
-static const struct spl_fixed31_32 spl_fixpt_half = { 0x80000000LL };
-static const struct spl_fixed31_32 spl_fixpt_one = { 0x100000000LL };
-
-/*
- * @brief
- * Initialization routines
- */
-
-/*
- * @brief
- * result = numerator / denominator
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_fraction(
-	long long numerator, long long denominator));
-
-/*
- * @brief
- * result = arg
- */
-static inline struct spl_fixed31_32 spl_fixpt_from_int(int arg)
-{
-	struct spl_fixed31_32 res;
-
-	res.value = (long long) arg << FIXED31_32_BITS_PER_FRACTIONAL_PART;
-
-	return res;
-}
-
-/*
- * @brief
- * Unary operators
- */
-
-/*
- * @brief
- * result = -arg
- */
-static inline struct spl_fixed31_32 spl_fixpt_neg(struct spl_fixed31_32 arg)
-{
-	struct spl_fixed31_32 res;
-
-	res.value = -arg.value;
-
-	return res;
-}
-
-/*
- * @brief
- * result = abs(arg) := (arg >= 0) ? arg : -arg
- */
-static inline struct spl_fixed31_32 spl_fixpt_abs(struct spl_fixed31_32 arg)
-{
-	if (arg.value < 0)
-		return spl_fixpt_neg(arg);
-	else
-		return arg;
-}
-
-/*
- * @brief
- * Binary relational operators
- */
-
-/*
- * @brief
- * result = arg1 < arg2
- */
-static inline bool spl_fixpt_lt(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	return arg1.value < arg2.value;
-}
-
-/*
- * @brief
- * result = arg1 <= arg2
- */
-static inline bool spl_fixpt_le(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	return arg1.value <= arg2.value;
-}
-
-/*
- * @brief
- * result = arg1 == arg2
- */
-static inline bool spl_fixpt_eq(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	return arg1.value == arg2.value;
-}
-
-/*
- * @brief
- * result = min(arg1, arg2) := (arg1 <= arg2) ? arg1 : arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_min(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	if (arg1.value <= arg2.value)
-		return arg1;
-	else
-		return arg2;
-}
-
-/*
- * @brief
- * result = max(arg1, arg2) := (arg1 <= arg2) ? arg2 : arg1
- */
-static inline struct spl_fixed31_32 spl_fixpt_max(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	if (arg1.value <= arg2.value)
-		return arg2;
-	else
-		return arg1;
-}
-
-/*
- * @brief
- *          | min_value, when arg <= min_value
- * result = | arg, when min_value < arg < max_value
- *          | max_value, when arg >= max_value
- */
-static inline struct spl_fixed31_32 spl_fixpt_clamp(
-	struct spl_fixed31_32 arg,
-	struct spl_fixed31_32 min_value,
-	struct spl_fixed31_32 max_value)
-{
-	if (spl_fixpt_le(arg, min_value))
-		return min_value;
-	else if (spl_fixpt_le(max_value, arg))
-		return max_value;
-	else
-		return arg;
-}
-
-/*
- * @brief
- * Binary shift operators
- */
-
-/*
- * @brief
- * result = arg << shift
- */
-static inline struct spl_fixed31_32 spl_fixpt_shl(struct spl_fixed31_32 arg, unsigned int shift)
-{
-	SPL_ASSERT(((arg.value >= 0) && (arg.value <= LLONG_MAX >> shift)) ||
-		((arg.value < 0) && (arg.value >= ~(LLONG_MAX >> shift))));
-
-	arg.value = arg.value << shift;
-
-	return arg;
-}
-
-/*
- * @brief
- * result = arg >> shift
- */
-static inline struct spl_fixed31_32 spl_fixpt_shr(struct spl_fixed31_32 arg, unsigned int shift)
-{
-	bool negative = arg.value < 0;
-
-	if (negative)
-		arg.value = -arg.value;
-	arg.value = arg.value >> shift;
-	if (negative)
-		arg.value = -arg.value;
-	return arg;
-}
-
-/*
- * @brief
- * Binary additive operators
- */
-
-/*
- * @brief
- * result = arg1 + arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_add(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	struct spl_fixed31_32 res;
-
-	SPL_ASSERT(((arg1.value >= 0) && (LLONG_MAX - arg1.value >= arg2.value)) ||
-		((arg1.value < 0) && (LLONG_MIN - arg1.value <= arg2.value)));
-
-	res.value = arg1.value + arg2.value;
-
-	return res;
-}
-
-/*
- * @brief
- * result = arg1 + arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_add_int(struct spl_fixed31_32 arg1, int arg2)
-{
-	return spl_fixpt_add(arg1, spl_fixpt_from_int(arg2));
-}
-
-/*
- * @brief
- * result = arg1 - arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_sub(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	struct spl_fixed31_32 res;
-
-	SPL_ASSERT(((arg2.value >= 0) && (LLONG_MIN + arg2.value <= arg1.value)) ||
-		((arg2.value < 0) && (LLONG_MAX + arg2.value >= arg1.value)));
-
-	res.value = arg1.value - arg2.value;
-
-	return res;
-}
-
-/*
- * @brief
- * result = arg1 - arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_sub_int(struct spl_fixed31_32 arg1, int arg2)
-{
-	return spl_fixpt_sub(arg1, spl_fixpt_from_int(arg2));
-}
-
-
-/*
- * @brief
- * Binary multiplicative operators
- */
-
-/*
- * @brief
- * result = arg1 * arg2
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_mul(
-	struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2));
-
-
-/*
- * @brief
- * result = arg1 * arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_mul_int(struct spl_fixed31_32 arg1, int arg2)
-{
-	return SPL_NAMESPACE(spl_fixpt_mul(arg1, spl_fixpt_from_int(arg2)));
-}
-
-/*
- * @brief
- * result = square(arg) := arg * arg
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sqr(struct spl_fixed31_32 arg));
-
-/*
- * @brief
- * result = arg1 / arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_div_int(struct spl_fixed31_32 arg1, long long arg2)
-{
-	return SPL_NAMESPACE(spl_fixpt_from_fraction(arg1.value,
-		spl_fixpt_from_int((int)arg2).value));
-}
-
-/*
- * @brief
- * result = arg1 / arg2
- */
-static inline struct spl_fixed31_32 spl_fixpt_div(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	return SPL_NAMESPACE(spl_fixpt_from_fraction(arg1.value, arg2.value));
-}
-
-/*
- * @brief
- * Reciprocal function
- */
-
-/*
- * @brief
- * result = reciprocal(arg) := 1 / arg
- *
- * @note
- * No special actions taken in case argument is zero.
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_recip(struct spl_fixed31_32 arg));
-
-/*
- * @brief
- * Trigonometric functions
- */
-
-/*
- * @brief
- * result = sinc(arg) := sin(arg) / arg
- *
- * @note
- * Argument specified in radians,
- * internally it's normalized to [-2pi...2pi] range.
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sinc(struct spl_fixed31_32 arg));
-
-/*
- * @brief
- * result = sin(arg)
- *
- * @note
- * Argument specified in radians,
- * internally it's normalized to [-2pi...2pi] range.
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_sin(struct spl_fixed31_32 arg));
-
-/*
- * @brief
- * result = cos(arg)
- *
- * @note
- * Argument specified in radians
- * and should be in [-2pi...2pi] range -
- * passing arguments outside that range
- * will cause incorrect result!
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_cos(struct spl_fixed31_32 arg));
-
-/*
- * @brief
- * Transcendent functions
- */
-
-/*
- * @brief
- * result = exp(arg)
- *
- * @note
- * Currently, function is verified for abs(arg) <= 1.
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_exp(struct spl_fixed31_32 arg));
-
-/*
- * @brief
- * result = log(arg)
- *
- * @note
- * Currently, abs(arg) should be less than 1.
- * No normalization is done.
- * Currently, no special actions taken
- * in case of invalid argument(s). Take care!
- */
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_log(struct spl_fixed31_32 arg));
-
-/*
- * @brief
- * Power function
- */
-
-/*
- * @brief
- * result = pow(arg1, arg2)
- *
- * @note
- * Currently, abs(arg1) should be less than 1. Take care!
- */
-static inline struct spl_fixed31_32 spl_fixpt_pow(struct spl_fixed31_32 arg1, struct spl_fixed31_32 arg2)
-{
-	if (arg1.value == 0)
-		return arg2.value == 0 ? spl_fixpt_one : spl_fixpt_zero;
-
-	return SPL_NAMESPACE(spl_fixpt_exp(
-		SPL_NAMESPACE(spl_fixpt_mul(
-			SPL_NAMESPACE(spl_fixpt_log(arg1)),
-			arg2))));
-}
-
-/*
- * @brief
- * Rounding functions
- */
-
-/*
- * @brief
- * result = floor(arg) := greatest integer lower than or equal to arg
- */
-static inline int spl_fixpt_floor(struct spl_fixed31_32 arg)
-{
-	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
-
-	if (arg.value >= 0)
-		return (int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
-	else
-		return -(int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
-}
-
-/*
- * @brief
- * result = round(arg) := integer nearest to arg
- */
-static inline int spl_fixpt_round(struct spl_fixed31_32 arg)
-{
-	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
-
-	const long long summand = spl_fixpt_half.value;
-
-	SPL_ASSERT(LLONG_MAX - (long long)arg_value >= summand);
-
-	arg_value += summand;
-
-	if (arg.value >= 0)
-		return (int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
-	else
-		return -(int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
-}
-
-/*
- * @brief
- * result = ceil(arg) := lowest integer greater than or equal to arg
- */
-static inline int spl_fixpt_ceil(struct spl_fixed31_32 arg)
-{
-	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
-
-	const long long summand = spl_fixpt_one.value -
-		spl_fixpt_epsilon.value;
-
-	SPL_ASSERT(LLONG_MAX - (long long)arg_value >= summand);
-
-	arg_value += summand;
-
-	if (arg.value >= 0)
-		return (int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
-	else
-		return -(int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
-}
-
-/* the following two function are used in scaler hw programming to convert fixed
- * point value to format 2 bits from integer part and 19 bits from fractional
- * part. The same applies for u0d19, 0 bits from integer part and 19 bits from
- * fractional
- */
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u4d19(struct spl_fixed31_32 arg));
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u3d19(struct spl_fixed31_32 arg));
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u2d19(struct spl_fixed31_32 arg));
-
-unsigned int SPL_NAMESPACE(spl_fixpt_u0d19(struct spl_fixed31_32 arg));
-
-unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d14(struct spl_fixed31_32 arg));
-
-unsigned int SPL_NAMESPACE(spl_fixpt_clamp_u0d10(struct spl_fixed31_32 arg));
-
-int SPL_NAMESPACE(spl_fixpt_s4d19(struct spl_fixed31_32 arg));
-
-static inline struct spl_fixed31_32 spl_fixpt_truncate(struct spl_fixed31_32 arg, unsigned int frac_bits)
-{
-	bool negative = arg.value < 0;
-
-	if (frac_bits >= FIXED31_32_BITS_PER_FRACTIONAL_PART) {
-		SPL_ASSERT(frac_bits == FIXED31_32_BITS_PER_FRACTIONAL_PART);
-		return arg;
-	}
-
-	if (negative)
-		arg.value = -arg.value;
-	arg.value &= (~0ULL) << (FIXED31_32_BITS_PER_FRACTIONAL_PART - frac_bits);
-	if (negative)
-		arg.value = -arg.value;
-	return arg;
-}
-
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_ux_dy(unsigned int value,
-		unsigned int integer_bits, unsigned int fractional_bits));
-struct spl_fixed31_32 SPL_NAMESPACE(spl_fixpt_from_int_dy(unsigned int int_value,
-		unsigned int frac_value,
-		unsigned int integer_bits,
-		unsigned int fractional_bits));
-
-#endif
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_namespace.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_namespace.h
new file mode 100644
index 00000000000000..5bea362f960eea
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_namespace.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: MIT */
+
+/* Copyright 2024 Advanced Micro Devices, Inc. */
+
+#ifndef _SPL_NAMESPACE_H_
+#define _SPL_NAMESPACE_H_
+
+/* SPL namespace macros */
+#ifndef SPL_PFX_
+#define SPL_PFX_
+#endif
+
+#define SPL_EXPAND2(a, b)         a##b
+#define SPL_EXPAND(a, b)          SPL_EXPAND2(a, b)
+#define SPL_NAMESPACE(symbol)     SPL_EXPAND(SPL_PFX_, symbol)
+
+#endif /* _SPL_NAMESPACE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h b/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h
index ae2d24c856cf53..26553cd6890f29 100644
--- a/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/sspl/spl_os_types.h
@@ -15,51 +15,10 @@
 #include <linux/delay.h>
 #include <linux/mm.h>
 
-/*
- *
- * general debug capabilities
- *
- */
-
-static inline uint64_t spl_div_u64_rem(uint64_t dividend, uint32_t divisor, uint32_t *remainder)
-{
-	return div_u64_rem(dividend, divisor, remainder);
-}
-
-static inline uint64_t spl_div_u64(uint64_t dividend, uint32_t divisor)
-{
-	return div_u64(dividend, divisor);
-}
-
-static inline uint64_t spl_div64_u64(uint64_t dividend, uint64_t divisor)
-{
-	return div64_u64(dividend, divisor);
-}
-
-static inline uint64_t spl_div64_u64_rem(uint64_t dividend, uint64_t divisor, uint64_t *remainder)
-{
-	return div64_u64_rem(dividend, divisor, remainder);
-}
-
-static inline int64_t spl_div64_s64(int64_t dividend, int64_t divisor)
-{
-	return div64_s64(dividend, divisor);
-}
-
-#define spl_swap(a, b) \
-	do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
-
 #ifndef spl_min
 #define spl_min(a, b)    (((a) < (b)) ? (a):(b))
 #endif
 
-/* SPL namespace macros */
-#ifndef SPL_PFX_
-#define SPL_PFX_
-#endif
-
-#define SPL_EXPAND2(a, b)         a##b
-#define SPL_EXPAND(a, b)          SPL_EXPAND2(a, b)
-#define SPL_NAMESPACE(symbol)     SPL_EXPAND(SPL_PFX_, symbol)
+#include "spl_namespace.h"
 
 #endif /* _SPL_OS_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
index d8a18cd97811c7..17579ff2e1bf7e 100644
--- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
@@ -69,6 +69,7 @@
 #include "dc/dc_types.h"
 
 #define DMUB_PC_SNAPSHOT_COUNT 10
+#define DMUB_TRACE_SNAPSHOT_COUNT 8
 
 /* Default tracebuffer size if meta is absent. */
 #define DMUB_TRACE_BUFFER_SIZE (64 * 1024)
@@ -81,6 +82,7 @@ struct dmub_srv;
 struct dmub_srv_common_regs;
 struct dmub_srv_dcn31_regs;
 
+struct dmcub_trace_buf;
 struct dmcub_trace_buf_entry;
 
 /* enum dmub_window_memory_type - memory location type specification for windows */
@@ -358,6 +360,26 @@ struct dmub_srv_hw_params {
 	bool disable_dpia_bw_allocation;
 };
 
+#define DMUB_TIMEOUT_TRACE_COUNT 8
+
+/**
+ * struct dmub_trace_buf_header - Header for the DMUB trace buffer
+ * @entry_count: Number of valid entries in the trace buffer
+ * @reserved: Reserved for future use
+ */
+struct dmub_trace_buf_header {
+	uint32_t entry_count;
+	uint32_t reserved[3];
+};
+
+/**
+ * struct dmub_trace_snapshot - Snapshot of the DMUB trace buffer
+ * @traces: Array of trace buffer entries
+ */
+struct dmub_trace_snapshot {
+	struct dmcub_trace_buf_entry traces[DMUB_TRACE_SNAPSHOT_COUNT];
+};
+
 /**
  * struct dmub_srv_debug - Debug info for dmub_srv
  * @timeout_occured: Indicates a timeout occured on any message from driver to dmub
@@ -367,13 +389,13 @@ struct dmub_timeout_info {
 	bool timeout_occured;
 	union dmub_rb_cmd timeout_cmd;
 	unsigned long long timestamp;
+	struct dmub_trace_snapshot trace_snapshot;
 };
 
 /**
- * struct dmub_diagnostic_data - Diagnostic data retrieved from DMCUB for
- * debugging purposes, including logging, crash analysis, etc.
+ * struct dmub_hw_diagnostic_data - Hardware diagnostic data for the DMUB
  */
-struct dmub_diagnostic_data {
+struct dmub_hw_diagnostic_data {
 	uint32_t dmcub_version;
 	uint32_t scratch[17];
 	uint32_t pc[DMUB_PC_SNAPSHOT_COUNT];
@@ -390,7 +412,6 @@ struct dmub_diagnostic_data {
 	uint32_t outbox1_wptr;
 	uint32_t outbox1_size;
 	uint32_t gpint_datain0;
-	struct dmub_timeout_info timeout_info;
 	uint8_t is_dmcub_enabled : 1;
 	uint8_t is_dmcub_soft_reset : 1;
 	uint8_t is_dmcub_secure_reset : 1;
@@ -400,6 +421,17 @@ struct dmub_diagnostic_data {
 	uint8_t is_pwait : 1;
 };
 
+/**
+ * struct dmub_diagnostic_data - Diagnostic data retrieved from DMCUB for
+ * debugging purposes, including logging, crash analysis, etc.
+ */
+struct dmub_diagnostic_data {
+	struct dmub_hw_diagnostic_data hw;
+	struct dmub_timeout_info first_timeout_info;
+	struct dmub_timeout_info latest_timeout_info;
+	struct dmub_trace_snapshot trace_snapshot;
+};
+
 /**
  * struct dmub_preos_info - preos fw info before loading post os fw.
  */
@@ -537,7 +569,7 @@ struct dmub_srv_hw_funcs {
 	void (*send_inbox0_cmd)(struct dmub_srv *dmub, union dmub_inbox0_data_register data);
 	uint32_t (*get_current_time)(struct dmub_srv *dmub);
 
-	void (*get_diagnostic_data)(struct dmub_srv *dmub);
+	void (*get_diagnostic_data)(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 	bool (*get_preos_fw_info)(struct dmub_srv *dmub);
 
 	bool (*should_detect)(struct dmub_srv *dmub);
@@ -992,6 +1024,14 @@ enum dmub_status dmub_srv_set_skip_panel_power_sequence(struct dmub_srv *dmub,
 
 bool dmub_srv_get_outbox0_msg(struct dmub_srv *dmub, struct dmcub_trace_buf_entry *entry);
 
+/**
+ * dmub_srv_get_diagnostic_data() - Get diagnostic data from the DMUB.
+ * @dmub: the dmub service
+ *
+ * Returns:
+ *   true - if the diagnostic data was successfully retrieved
+ *   false - otherwise
+ */
 bool dmub_srv_get_diagnostic_data(struct dmub_srv *dmub);
 
 bool dmub_srv_should_detect(struct dmub_srv *dmub);
@@ -1073,7 +1113,7 @@ void dmub_srv_set_power_state(struct dmub_srv *dmub, enum dmub_srv_power_state_t
  *   DMUB_STATUS_OK - success
  *   DMUB_STATUS_INVALID - unspecified error
  */
-enum dmub_status dmub_srv_reg_cmd_execute(struct dmub_srv *dmub, union dmub_rb_cmd *cmd);
+enum dmub_status dmub_srv_reg_cmd_execute(struct dmub_srv *dmub, const union dmub_rb_cmd *cmd);
 
 
 /**
@@ -1151,4 +1191,12 @@ bool dmub_srv_get_preos_info(struct dmub_srv *dmub);
 enum dmub_status dmub_srv_get_fw_meta_info_from_raw_fw(struct dmub_srv_fw_meta_info_params *params,
 						       struct dmub_fw_meta_info *fw_info_out);
 
+/**
+ * dmub_srv_get_trace_snapshot() - Fetch the latest trace entries from the DMCUB trace buffer
+ * @dmub: the dmub service
+ * @trace_snapshot: output buffer for trace entries
+ *
+ */
+void dmub_srv_get_trace_snapshot(struct dmub_srv *dmub, struct dmub_trace_snapshot *trace_snapshot);
+
 #endif /* _DMUB_SRV_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index f0231f355fa730..525e679b81d578 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -582,7 +582,14 @@ union replay_debug_flags {
 		 */
 		uint32_t enable_sub_feature_visual_confirm : 1;
 
-		uint32_t reserved : 16;
+		/**
+		 * 0x10000 (bit 16)
+		 * @enable_oem_visual_confirm: Enable OEM visual confirm
+		 * Black = state 0, Blue = state non-zero, Green = frame skipping active
+		 */
+		uint32_t enable_oem_visual_confirm : 1;
+
+		uint32_t reserved : 15;
 	} bitfields;
 
 	uint32_t u32All;
@@ -743,7 +750,14 @@ union pr_debug_flags {
 		 */
 		uint32_t force_hubp_on : 1;
 
-		uint32_t reserved : 24;
+		/**
+		 * 0x100 (bit 8)
+		 * @enable_oem_visual_confirm: Enable OEM visual confirm
+		 * Black = state 0, Blue = state non-zero
+		 */
+		uint32_t enable_oem_visual_confirm : 1;
+
+		uint32_t reserved : 23;
 	} bitfields;
 
 	uint32_t u32All;
@@ -1004,7 +1018,8 @@ union dmub_fw_meta_feature_bits {
 		uint32_t shared_state_link_detection : 1; /**< 1 supports link detection via shared state */
 		uint32_t cursor_offload_v1_support: 1; /**< 1 supports cursor offload */
 		uint32_t inbox0_lock_support: 1; /**< 1 supports inbox0 lock mechanism */
-		uint32_t reserved : 29;
+		uint32_t inbox0_lock_split: 1; /**< 1 supports inbox0 lock acquire/release split mechanism */
+		uint32_t reserved : 28;
 	} bits; /**< status bits */
 	uint32_t all; /**< 32-bit access to status bits */
 };
@@ -1169,6 +1184,16 @@ enum dmub_ips_comand_type {
 	DMUB_CMD__IPS_QUERY_RESIDENCY_INFO = 1,
 };
 
+/**
+ * enum dmub_dc_bls_command_type - DC_BLS subcommands.
+ */
+enum dmub_dc_bls_command_type {
+	/**
+	 * Placeholder for block-level initialization of the DCHVM
+	 */
+	DMUB_CMD__DC_BLS_DCHVM_INIT = 0,
+};
+
 /**
  * enum dmub_cursor_offload_comand_type - Cursor offload subcommands.
  */
@@ -1749,6 +1774,30 @@ enum dmub_gpint_command {
 	 *       1 - Enable panel polarity
 	 */
 	DMUB_GPINT__PANEL_POLARITY_DEBUG_ENABLE = 140,
+	/**
+	 * DESC: Reset the boot-time cumulative PHY-off residency counter.
+	 * PARAMS: [7:0] panel_inst
+	 * RETURN: 0 (ACK)
+	 */
+	DMUB_GPINT__REPLAY_RESET_CUMULATIVE_RESIDENCY = 141,
+	/**
+	 * DESC: Query boot-time cumulative PHY-off residency, low 32 bits (microseconds).
+	 * PARAMS: [7:0] panel_inst
+	 * RETURN: Lower 32 bits of accumulated PHY-off time in microseconds.
+	 */
+	DMUB_GPINT__REPLAY_GET_CUMULATIVE_RESIDENCY_US_LO = 142,
+	/**
+	 * DESC: Query boot-time cumulative PHY-off residency, high 32 bits (microseconds).
+	 * PARAMS: [7:0] panel_inst
+	 * RETURN: Upper 32 bits of accumulated PHY-off time in microseconds.
+	 */
+	DMUB_GPINT__REPLAY_GET_CUMULATIVE_RESIDENCY_US_HI = 143,
+	/**
+	 * DESC: Compute + log cumulative PHY-off residency snapshot; return milli-percent.
+	 * PARAMS: [7:0] panel_inst
+	 * RETURN: Residency in milli-percent (0-100000). Also emits DMUB trace log.
+	 */
+	DMUB_GPINT__REPLAY_SNAPSHOT_CUMULATIVE_RESIDENCY = 144,
 };
 
 /**
@@ -1923,10 +1972,6 @@ enum dmub_cmd_type {
 	 * Command type used for interfacing with DPIA.
 	 */
 	DMUB_CMD__DPIA = 77,
-	/**
-	 * Command type used for EDID CEA parsing
-	 */
-	DMUB_CMD__EDID_CEA = 79,
 	/**
 	 * Command type used for getting usbc cable ID
 	 */
@@ -2008,6 +2053,16 @@ enum dmub_cmd_type {
 	 */
 	DMUB_CMD__PANEL_POLARITY = 97,
 
+	/**
+	 * Command type used for all DC_BLS commands.
+	 */
+	DMUB_CMD__DC_BLS = 98,
+
+	/**
+	 * Command type used to notify nbif az pme restore.
+	 */
+	DMUB_CMD__NBIF_AZ_PME_RESTORE = 99,
+
 	/**
 	 * Command type use for VBIOS shared commands.
 	 */
@@ -2762,6 +2817,7 @@ struct dmub_fams2_stream_static_state {
 			uint8_t is_drr : 1; // stream is DRR enabled
 			uint8_t clamp_vtotal_min : 1; // clamp vtotal to min instead of nominal
 			uint8_t min_ttu_vblank_usable : 1; // if min ttu vblank is above wm, no force pstate is needed in blank
+			uint8_t imm_restore_drr : 1; // does not wait to latch DRR vtotal on restore
 		} bits;
 		uint8_t all;
 	} config;
@@ -2794,6 +2850,7 @@ struct dmub_fams2_cmd_stream_static_base_state {
 			uint8_t is_drr : 1; // stream is DRR enabled
 			uint8_t clamp_vtotal_min : 1; // clamp vtotal to min instead of nominal
 			uint8_t min_ttu_vblank_usable : 1; // if min ttu vblank is above wm, no force pstate is needed in blank
+			uint8_t imm_restore_drr : 1; // does not wait to latch DRR vtotal on restore
 		} bits;
 		uint8_t all;
 	} config;
@@ -2975,6 +3032,9 @@ struct dmub_clocks {
 	uint32_t dppclk_khz; /**< dppclk kHz */
 	uint32_t dcfclk_khz; /**< dcfclk kHz */
 	uint32_t dcfclk_deep_sleep_khz; /**< dcfclk deep sleep kHz */
+	uint32_t dpm0_dispclk_khz; /**< DPM0 (minimum) dispclk kHz from SMU DPM table, 0 if unknown */
+	uint32_t dpm0_dppclk_khz; /**< DPM0 (minimum) dppclk kHz from SMU DPM table, 0 if unknown */
+	uint32_t max_bypass_clk_khz; /**< max dispclk/dppclk achievable on a bypass source (no PLL) kHz, 0 if unknown */
 };
 
 /**
@@ -3592,6 +3652,28 @@ struct dmub_rb_cmd_query_hpd_state {
 	struct dmub_cmd_hpd_state_query_data data;
 };
 
+/**
+ * Data passed from driver to FW in a DMUB_CMD__NBIF_AZ_PME_RESTORE command.
+ */
+struct dmub_cmd_nbif_az_pme_restore_data {
+	uint8_t az_inst; /**< Azalia codec endpoint instance */
+	uint8_t pad[3]; /**< Alignment */
+};
+
+/**
+ * Definition of a DMUB_CMD__NBIF_AZ_PME_RESTORE command.
+ */
+struct dmub_rb_cmd_nbif_az_pme_restore {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+	/**
+	 * Data passed from driver to FW in a DMUB_CMD__NBIF_AZ_PME_RESTORE command.
+	 */
+	struct dmub_cmd_nbif_az_pme_restore_data data;
+};
+
 /**
  * struct dmub_rb_cmd_hpd_sense_notify - HPD sense notification data.
  */
@@ -6886,69 +6968,6 @@ struct dmub_rb_cmd_transmitter_set_phy_fsm {
 	struct dmub_rb_cmd_transmitter_set_phy_fsm_data data; /**< payload */
 };
 
-/**
- * Maximum number of bytes a chunk sent to DMUB for parsing
- */
-#define DMUB_EDID_CEA_DATA_CHUNK_BYTES 8
-
-/**
- *  Represent a chunk of CEA blocks sent to DMUB for parsing
- */
-struct dmub_cmd_send_edid_cea {
-	uint16_t offset;	/**< offset into the CEA block */
-	uint8_t length;	/**< number of bytes in payload to copy as part of CEA block */
-	uint16_t cea_total_length;  /**< total length of the CEA block */
-	uint8_t payload[DMUB_EDID_CEA_DATA_CHUNK_BYTES]; /**< data chunk of the CEA block */
-	uint8_t pad[3]; /**< padding and for future expansion */
-};
-
-/**
- * Result of VSDB parsing from CEA block
- */
-struct dmub_cmd_edid_cea_amd_vsdb {
-	uint8_t vsdb_found;		/**< 1 if parsing has found valid AMD VSDB */
-	uint8_t freesync_supported;	/**< 1 if Freesync is supported */
-	uint16_t amd_vsdb_version;	/**< AMD VSDB version */
-	uint16_t min_frame_rate;	/**< Maximum frame rate */
-	uint16_t max_frame_rate;	/**< Minimum frame rate */
-	uint8_t freesync_mccs_vcp_code; /**< Freesync MCCS VCP code */
-};
-
-/**
- * Result of sending a CEA chunk
- */
-struct dmub_cmd_edid_cea_ack {
-	uint16_t offset;	/**< offset of the chunk into the CEA block */
-	uint8_t success;	/**< 1 if this sending of chunk succeeded */
-	uint8_t pad;		/**< padding and for future expansion */
-};
-
-/**
- * Specify whether the result is an ACK/NACK or the parsing has finished
- */
-enum dmub_cmd_edid_cea_reply_type {
-	DMUB_CMD__EDID_CEA_AMD_VSDB	= 1, /**< VSDB parsing has finished */
-	DMUB_CMD__EDID_CEA_ACK		= 2, /**< acknowledges the CEA sending is OK or failing */
-};
-
-/**
- * Definition of a DMUB_CMD__EDID_CEA command.
- */
-struct dmub_rb_cmd_edid_cea {
-	struct dmub_cmd_header header;	/**< Command header */
-	union dmub_cmd_edid_cea_data {
-		struct dmub_cmd_send_edid_cea input; /**< input to send CEA chunks */
-		struct dmub_cmd_edid_cea_output { /**< output with results */
-			uint8_t type;	/**< dmub_cmd_edid_cea_reply_type */
-			union {
-				struct dmub_cmd_edid_cea_amd_vsdb amd_vsdb;
-				struct dmub_cmd_edid_cea_ack ack;
-			};
-		} output;	/**< output to retrieve ACK/NACK or VSDB parsing results */
-	} data;	/**< Command data */
-
-};
-
 /**
  * struct dmub_cmd_cable_id_input - Defines the input of DMUB_CMD_GET_USBC_CABLE_ID command.
  */
@@ -7188,6 +7207,25 @@ struct dmub_rb_cmd_ips_residency_cntl {
 	struct dmub_cmd_ips_residency_cntl_data cntl_data;
 };
 
+/**
+ * Data passed from driver to FW in a DMUB_CMD__DC_BLS_DCHVM_INIT command.
+ */
+struct dmub_cmd_dc_bls_dchvm_init_data {
+	/**
+	 * The value to program to rIOMMU PCTRL register. x86 cannot access this SMN
+	 * register, so the write is performed by DMCUB on the driver's behalf.
+	 */
+	uint32_t riommu_pctrl_val;
+};
+
+/**
+ * Definition of a DMUB_CMD__DC_BLS_DCHVM_INIT command.
+ */
+struct dmub_rb_cmd_dc_bls_dchvm_init {
+	struct dmub_cmd_header header;
+	struct dmub_cmd_dc_bls_dchvm_init_data data;
+};
+
 /**
  * Data passed from FW to driver in a DMUB_CMD__IPS_QUERY_RESIDENCY_INFO command.
  */
@@ -7801,10 +7839,6 @@ union dmub_rb_cmd {
 	 * Definition of a DMUB_CMD__DPIA_SET_TPS_NOTIFICATION command.
 	 */
 	struct dmub_rb_cmd_set_tps_notification set_tps_notification;
-	/**
-	 * Definition of a DMUB_CMD__EDID_CEA command.
-	 */
-	struct dmub_rb_cmd_edid_cea edid_cea;
 	/**
 	 * Definition of a DMUB_CMD_GET_USBC_CABLE_ID command.
 	 */
@@ -7814,6 +7848,11 @@ union dmub_rb_cmd {
 	 * Definition of a DMUB_CMD__QUERY_HPD_STATE command.
 	 */
 	struct dmub_rb_cmd_query_hpd_state query_hpd;
+
+	/**
+	 * Definition of a DMUB_CMD__NBIF_AZ_PME_RESTORE command.
+	 */
+	struct dmub_rb_cmd_nbif_az_pme_restore nbif_az_pme_restore;
 	/**
 	 * Definition of a DMUB_CMD__SECURE_DISPLAY command.
 	 */
@@ -7988,6 +8027,8 @@ union dmub_rb_cmd {
 	struct dmub_rb_cmd_panel_polarity_enable panel_polarity_enable;
 	struct dmub_rb_cmd_panel_polarity_get_bias panel_polarity_get_bias;
 	struct dmub_rb_cmd_panel_polarity_reset panel_polarity_reset;
+
+	struct dmub_rb_cmd_dc_bls_dchvm_init dc_bls_dchvm_init;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c
index 0b152926f75b3c..08c23ca47094ae 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c
@@ -416,72 +416,68 @@ uint32_t dmub_dcn20_get_current_time(struct dmub_srv *dmub)
 	return REG_READ(DMCUB_TIMER_CURRENT);
 }
 
-void dmub_dcn20_get_diagnostic_data(struct dmub_srv *dmub)
+void dmub_dcn20_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data)
 {
 	uint32_t is_dmub_enabled, is_soft_reset, is_sec_reset;
 	uint32_t is_traceport_enabled, is_cw0_enabled, is_cw6_enabled;
-	struct dmub_timeout_info timeout = {0};
 
-	if (!dmub)
+	if (!dmub || !hw_data)
 		return;
 
-	/* timeout data filled externally, cache before resetting memory */
-	timeout = dmub->debug.timeout_info;
-	memset(&dmub->debug, 0, sizeof(dmub->debug));
-	dmub->debug.timeout_info = timeout;
-
-	dmub->debug.dmcub_version = dmub->fw_version;
-
-	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
-	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
-	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
-	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
-	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
-	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
-	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
-	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
-	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
-	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
-	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
-	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
-	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
-	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
-	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
-	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
-
-	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
-	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
-	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
-
-	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
-	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
-	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
-
-	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
-	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
-	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+	memset(hw_data, 0, sizeof(*hw_data));
+
+	hw_data->dmcub_version = dmub->fw_version;
+
+	hw_data->scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	hw_data->scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	hw_data->scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	hw_data->scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	hw_data->scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	hw_data->scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	hw_data->scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	hw_data->scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	hw_data->scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	hw_data->scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	hw_data->scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	hw_data->scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	hw_data->scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	hw_data->scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	hw_data->scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	hw_data->scratch[15] = REG_READ(DMCUB_SCRATCH15);
+
+	hw_data->undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	hw_data->inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	hw_data->data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	hw_data->inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	hw_data->inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	hw_data->inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	hw_data->inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	hw_data->inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	hw_data->inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
 
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
 	ASSERT(is_dmub_enabled <= 0xFF);
-	dmub->debug.is_dmcub_enabled = (uint8_t)is_dmub_enabled;
+	hw_data->is_dmcub_enabled = (uint8_t)is_dmub_enabled;
 
 	REG_GET(DMCUB_CNTL, DMCUB_SOFT_RESET, &is_soft_reset);
 	ASSERT(is_soft_reset <= 0xFF);
-	dmub->debug.is_dmcub_soft_reset = (uint8_t)is_soft_reset;
+	hw_data->is_dmcub_soft_reset = (uint8_t)is_soft_reset;
 
 	REG_GET(DMCUB_SEC_CNTL, DMCUB_SEC_RESET_STATUS, &is_sec_reset);
 	ASSERT(is_sec_reset <= 0xFF);
-	dmub->debug.is_dmcub_secure_reset = (uint8_t)is_sec_reset;
+	hw_data->is_dmcub_secure_reset = (uint8_t)is_sec_reset;
 
 	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
 	ASSERT(is_traceport_enabled <= 0xFF);
-	dmub->debug.is_traceport_en  = (uint8_t)is_traceport_enabled;
+	hw_data->is_traceport_en  = (uint8_t)is_traceport_enabled;
 
 	REG_GET(DMCUB_REGION3_CW0_TOP_ADDRESS, DMCUB_REGION3_CW0_ENABLE, &is_cw0_enabled);
 	ASSERT(is_cw0_enabled <= 0xFF);
-	dmub->debug.is_cw0_enabled = (uint8_t)is_cw0_enabled;
+	hw_data->is_cw0_enabled = (uint8_t)is_cw0_enabled;
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	ASSERT(is_cw6_enabled <= 0xFF);
-	dmub->debug.is_cw6_enabled = (uint8_t)is_cw6_enabled;
+	hw_data->is_cw6_enabled = (uint8_t)is_cw6_enabled;
 }
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h
index 42c1fb4bc73f66..fd89d341573bf0 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h
@@ -247,6 +247,6 @@ bool dmub_dcn20_use_cached_trace_buffer(struct dmub_srv *dmub);
 
 uint32_t dmub_dcn20_get_current_time(struct dmub_srv *dmub);
 
-void dmub_dcn20_get_diagnostic_data(struct dmub_srv *dmub);
+void dmub_dcn20_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 
 #endif /* _DMUB_DCN20_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c
index 478d79a6e2461c..cd0ba8d413576c 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c
@@ -418,82 +418,78 @@ uint32_t dmub_dcn31_get_current_time(struct dmub_srv *dmub)
 	return REG_READ(DMCUB_TIMER_CURRENT);
 }
 
-void dmub_dcn31_get_diagnostic_data(struct dmub_srv *dmub)
+void dmub_dcn31_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data)
 {
 	uint32_t is_dmub_enabled, is_soft_reset, is_sec_reset, is_pwait;
 	uint32_t is_traceport_enabled, is_cw0_enabled, is_cw6_enabled;
-	struct dmub_timeout_info timeout = {0};
 
-	if (!dmub)
+	if (!dmub || !hw_data)
 		return;
 
-	/* timeout data filled externally, cache before resetting memory */
-	timeout = dmub->debug.timeout_info;
-	memset(&dmub->debug, 0, sizeof(dmub->debug));
-	dmub->debug.timeout_info = timeout;
-
-	dmub->debug.dmcub_version = dmub->fw_version;
-
-	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
-	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
-	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
-	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
-	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
-	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
-	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
-	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
-	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
-	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
-	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
-	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
-	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
-	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
-	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
-	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
-
-	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
-	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
-	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
-
-	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
-	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
-	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
-
-	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
-	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
-	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
-
-	dmub->debug.outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
-	dmub->debug.outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
-	dmub->debug.outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
+	memset(hw_data, 0, sizeof(*hw_data));
+
+	hw_data->dmcub_version = dmub->fw_version;
+
+	hw_data->scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	hw_data->scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	hw_data->scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	hw_data->scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	hw_data->scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	hw_data->scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	hw_data->scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	hw_data->scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	hw_data->scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	hw_data->scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	hw_data->scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	hw_data->scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	hw_data->scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	hw_data->scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	hw_data->scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	hw_data->scratch[15] = REG_READ(DMCUB_SCRATCH15);
+
+	hw_data->undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	hw_data->inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	hw_data->data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	hw_data->inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	hw_data->inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	hw_data->inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	hw_data->inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	hw_data->inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	hw_data->inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+
+	hw_data->outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
+	hw_data->outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
+	hw_data->outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
 
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
 	ASSERT(is_dmub_enabled <= 0xFF);
-	dmub->debug.is_dmcub_enabled = (uint8_t)is_dmub_enabled;
+	hw_data->is_dmcub_enabled = (uint8_t)is_dmub_enabled;
 
 	REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &is_pwait);
 	ASSERT(is_pwait <= 0xFF);
-	dmub->debug.is_pwait = (uint8_t)is_pwait;
+	hw_data->is_pwait = (uint8_t)is_pwait;
 
 	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &is_soft_reset);
 	ASSERT(is_soft_reset <= 0xFF);
-	dmub->debug.is_dmcub_soft_reset = (uint8_t)is_soft_reset;
+	hw_data->is_dmcub_soft_reset = (uint8_t)is_soft_reset;
 
 	REG_GET(DMCUB_SEC_CNTL, DMCUB_SEC_RESET_STATUS, &is_sec_reset);
 	ASSERT(is_sec_reset <= 0xFF);
-	dmub->debug.is_dmcub_secure_reset = (uint8_t)is_sec_reset;
+	hw_data->is_dmcub_secure_reset = (uint8_t)is_sec_reset;
 
 	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
 	ASSERT(is_traceport_enabled <= 0xFF);
-	dmub->debug.is_traceport_en  = (uint8_t)is_traceport_enabled;
+	hw_data->is_traceport_en  = (uint8_t)is_traceport_enabled;
 
 	REG_GET(DMCUB_REGION3_CW0_TOP_ADDRESS, DMCUB_REGION3_CW0_ENABLE, &is_cw0_enabled);
 	ASSERT(is_cw0_enabled <= 0xFF);
-	dmub->debug.is_cw0_enabled = (uint8_t)is_cw0_enabled;
+	hw_data->is_cw0_enabled = (uint8_t)is_cw0_enabled;
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	ASSERT(is_cw6_enabled <= 0xFF);
-	dmub->debug.is_cw6_enabled = (uint8_t)is_cw6_enabled;
+	hw_data->is_cw6_enabled = (uint8_t)is_cw6_enabled;
 }
 
 bool dmub_dcn31_should_detect(struct dmub_srv *dmub)
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.h
index 1c43ef2bca6686..ec878c58d09b67 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.h
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.h
@@ -251,7 +251,7 @@ void dmub_dcn31_set_outbox0_rptr(struct dmub_srv *dmub, uint32_t rptr_offset);
 
 uint32_t dmub_dcn31_get_current_time(struct dmub_srv *dmub);
 
-void dmub_dcn31_get_diagnostic_data(struct dmub_srv *dmub);
+void dmub_dcn31_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 
 bool dmub_dcn31_should_detect(struct dmub_srv *dmub);
 
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c
index 3f9fb9e05b7999..0d585dc63364f9 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.c
@@ -437,77 +437,73 @@ uint32_t dmub_dcn32_get_current_time(struct dmub_srv *dmub)
 	return REG_READ(DMCUB_TIMER_CURRENT);
 }
 
-void dmub_dcn32_get_diagnostic_data(struct dmub_srv *dmub)
+void dmub_dcn32_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data)
 {
 	uint32_t is_dmub_enabled, is_soft_reset, is_pwait;
 	uint32_t is_traceport_enabled, is_cw6_enabled;
-	struct dmub_timeout_info timeout = {0};
 
-	if (!dmub)
+	if (!dmub || !hw_data)
 		return;
 
-	/* timeout data filled externally, cache before resetting memory */
-	timeout = dmub->debug.timeout_info;
-	memset(&dmub->debug, 0, sizeof(dmub->debug));
-	dmub->debug.timeout_info = timeout;
-
-	dmub->debug.dmcub_version = dmub->fw_version;
-
-	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
-	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
-	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
-	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
-	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
-	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
-	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
-	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
-	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
-	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
-	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
-	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
-	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
-	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
-	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
-	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
-	dmub->debug.scratch[16] = REG_READ(DMCUB_SCRATCH16);
-
-	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
-	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
-	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
-
-	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
-	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
-	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
-
-	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
-	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
-	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
-
-	dmub->debug.outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
-	dmub->debug.outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
-	dmub->debug.outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
+	memset(hw_data, 0, sizeof(*hw_data));
+
+	hw_data->dmcub_version = dmub->fw_version;
+
+	hw_data->scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	hw_data->scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	hw_data->scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	hw_data->scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	hw_data->scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	hw_data->scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	hw_data->scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	hw_data->scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	hw_data->scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	hw_data->scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	hw_data->scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	hw_data->scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	hw_data->scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	hw_data->scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	hw_data->scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	hw_data->scratch[15] = REG_READ(DMCUB_SCRATCH15);
+	hw_data->scratch[16] = REG_READ(DMCUB_SCRATCH16);
+
+	hw_data->undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	hw_data->inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	hw_data->data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	hw_data->inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	hw_data->inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	hw_data->inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	hw_data->inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	hw_data->inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	hw_data->inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+
+	hw_data->outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
+	hw_data->outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
+	hw_data->outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
 
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
 	ASSERT(is_dmub_enabled <= 0xFF);
-	dmub->debug.is_dmcub_enabled = (uint8_t)is_dmub_enabled;
+	hw_data->is_dmcub_enabled = (uint8_t)is_dmub_enabled;
 
 	REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &is_pwait);
 	ASSERT(is_pwait <= 0xFF);
-	dmub->debug.is_pwait = (uint8_t)is_pwait;
+	hw_data->is_pwait = (uint8_t)is_pwait;
 
 	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &is_soft_reset);
 	ASSERT(is_soft_reset <= 0xFF);
-	dmub->debug.is_dmcub_soft_reset = (uint8_t)is_soft_reset;
+	hw_data->is_dmcub_soft_reset = (uint8_t)is_soft_reset;
 
 	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
 	ASSERT(is_traceport_enabled <= 0xFF);
-	dmub->debug.is_traceport_en  = (uint8_t)is_traceport_enabled;
+	hw_data->is_traceport_en  = (uint8_t)is_traceport_enabled;
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	ASSERT(is_cw6_enabled <= 0xFF);
-	dmub->debug.is_cw6_enabled = (uint8_t)is_cw6_enabled;
+	hw_data->is_cw6_enabled = (uint8_t)is_cw6_enabled;
 
-	dmub->debug.gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+	hw_data->gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
 }
 void dmub_dcn32_configure_dmub_in_system_memory(struct dmub_srv *dmub)
 {
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.h
index daf81027d66317..ccf849e07b38f1 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.h
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn32.h
@@ -260,7 +260,7 @@ void dmub_dcn32_set_outbox0_rptr(struct dmub_srv *dmub, uint32_t rptr_offset);
 
 uint32_t dmub_dcn32_get_current_time(struct dmub_srv *dmub);
 
-void dmub_dcn32_get_diagnostic_data(struct dmub_srv *dmub);
+void dmub_dcn32_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 
 void dmub_dcn32_configure_dmub_in_system_memory(struct dmub_srv *dmub);
 void dmub_dcn32_send_inbox0_cmd(struct dmub_srv *dmub, union dmub_inbox0_data_register data);
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c
index 69fb6084232eab..bd7637276aa6fe 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.c
@@ -458,77 +458,73 @@ uint32_t dmub_dcn35_get_current_time(struct dmub_srv *dmub)
 	return REG_READ(DMCUB_TIMER_CURRENT);
 }
 
-void dmub_dcn35_get_diagnostic_data(struct dmub_srv *dmub)
+void dmub_dcn35_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data)
 {
 	uint32_t is_dmub_enabled, is_soft_reset, is_pwait;
 	uint32_t is_traceport_enabled, is_cw6_enabled;
-	struct dmub_timeout_info timeout = {0};
 
-	if (!dmub)
+	if (!dmub || !hw_data)
 		return;
 
-	/* timeout data filled externally, cache before resetting memory */
-	timeout = dmub->debug.timeout_info;
-	memset(&dmub->debug, 0, sizeof(dmub->debug));
-	dmub->debug.timeout_info = timeout;
-
-	dmub->debug.dmcub_version = dmub->fw_version;
-
-	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
-	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
-	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
-	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
-	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
-	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
-	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
-	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
-	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
-	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
-	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
-	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
-	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
-	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
-	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
-	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
-	dmub->debug.scratch[16] = REG_READ(DMCUB_SCRATCH16);
-
-	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
-	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
-	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
-
-	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
-	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
-	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
-
-	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
-	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
-	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
-
-	dmub->debug.outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
-	dmub->debug.outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
-	dmub->debug.outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
+	memset(hw_data, 0, sizeof(*hw_data));
+
+	hw_data->dmcub_version = dmub->fw_version;
+
+	hw_data->scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	hw_data->scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	hw_data->scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	hw_data->scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	hw_data->scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	hw_data->scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	hw_data->scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	hw_data->scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	hw_data->scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	hw_data->scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	hw_data->scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	hw_data->scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	hw_data->scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	hw_data->scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	hw_data->scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	hw_data->scratch[15] = REG_READ(DMCUB_SCRATCH15);
+	hw_data->scratch[16] = REG_READ(DMCUB_SCRATCH16);
+
+	hw_data->undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	hw_data->inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	hw_data->data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	hw_data->inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	hw_data->inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	hw_data->inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	hw_data->inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	hw_data->inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	hw_data->inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+
+	hw_data->outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
+	hw_data->outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
+	hw_data->outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
 
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
 	ASSERT(is_dmub_enabled <= 0xFF);
-	dmub->debug.is_dmcub_enabled = (uint8_t)is_dmub_enabled;
+	hw_data->is_dmcub_enabled = (uint8_t)is_dmub_enabled;
 
 	REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &is_pwait);
 	ASSERT(is_pwait <= 0xFF);
-	dmub->debug.is_pwait = (uint8_t)is_pwait;
+	hw_data->is_pwait = (uint8_t)is_pwait;
 
 	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &is_soft_reset);
 	ASSERT(is_soft_reset <= 0xFF);
-	dmub->debug.is_dmcub_soft_reset = (uint8_t)is_soft_reset;
+	hw_data->is_dmcub_soft_reset = (uint8_t)is_soft_reset;
 
 	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
 	ASSERT(is_traceport_enabled <= 0xFF);
-	dmub->debug.is_traceport_en  = (uint8_t)is_traceport_enabled;
+	hw_data->is_traceport_en  = (uint8_t)is_traceport_enabled;
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	ASSERT(is_cw6_enabled <= 0xFF);
-	dmub->debug.is_cw6_enabled = (uint8_t)is_cw6_enabled;
+	hw_data->is_cw6_enabled = (uint8_t)is_cw6_enabled;
 
-	dmub->debug.gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+	hw_data->gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
 }
 
 bool dmub_dcn35_get_preos_fw_info(struct dmub_srv *dmub)
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.h
index 92e6695a2c9bad..8690952849938e 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.h
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn35.h
@@ -269,7 +269,7 @@ void dmub_dcn35_set_outbox0_rptr(struct dmub_srv *dmub, uint32_t rptr_offset);
 
 uint32_t dmub_dcn35_get_current_time(struct dmub_srv *dmub);
 
-void dmub_dcn35_get_diagnostic_data(struct dmub_srv *dmub);
+void dmub_dcn35_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 
 void dmub_dcn35_configure_dmub_in_system_memory(struct dmub_srv *dmub);
 
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c
index 3444eb7d7d812d..02ab26516584b8 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c
@@ -423,85 +423,81 @@ uint32_t dmub_dcn401_get_current_time(struct dmub_srv *dmub)
 	return REG_READ(DMCUB_TIMER_CURRENT);
 }
 
-void dmub_dcn401_get_diagnostic_data(struct dmub_srv *dmub)
+void dmub_dcn401_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data)
 {
 	uint32_t is_dmub_enabled, is_soft_reset, is_sec_reset, is_pwait;
 	uint32_t is_traceport_enabled, is_cw0_enabled, is_cw6_enabled;
-	struct dmub_timeout_info timeout = {0};
 
-	if (!dmub)
+	if (!dmub || !hw_data)
 		return;
 
-	/* timeout data filled externally, cache before resetting memory */
-	timeout = dmub->debug.timeout_info;
-	memset(&dmub->debug, 0, sizeof(dmub->debug));
-	dmub->debug.timeout_info = timeout;
-
-	dmub->debug.dmcub_version = dmub->fw_version;
-
-	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
-	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
-	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
-	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
-	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
-	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
-	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
-	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
-	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
-	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
-	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
-	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
-	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
-	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
-	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
-	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
-	dmub->debug.scratch[16] = REG_READ(DMCUB_SCRATCH16);
-
-	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
-	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
-	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
-
-	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
-	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
-	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
-
-	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
-	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
-	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
-
-	dmub->debug.outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
-	dmub->debug.outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
-	dmub->debug.outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
+	memset(hw_data, 0, sizeof(*hw_data));
+
+	hw_data->dmcub_version = dmub->fw_version;
+
+	hw_data->scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	hw_data->scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	hw_data->scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	hw_data->scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	hw_data->scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	hw_data->scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	hw_data->scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	hw_data->scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	hw_data->scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	hw_data->scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	hw_data->scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	hw_data->scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	hw_data->scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	hw_data->scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	hw_data->scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	hw_data->scratch[15] = REG_READ(DMCUB_SCRATCH15);
+	hw_data->scratch[16] = REG_READ(DMCUB_SCRATCH16);
+
+	hw_data->undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	hw_data->inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	hw_data->data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	hw_data->inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	hw_data->inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	hw_data->inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	hw_data->inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	hw_data->inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	hw_data->inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+
+	hw_data->outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
+	hw_data->outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
+	hw_data->outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
 
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
 	ASSERT(is_dmub_enabled <= 0xFF);
-	dmub->debug.is_dmcub_enabled = (uint8_t)is_dmub_enabled;
+	hw_data->is_dmcub_enabled = (uint8_t)is_dmub_enabled;
 
 	REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &is_pwait);
 	ASSERT(is_pwait <= 0xFF);
-	dmub->debug.is_pwait = (uint8_t)is_pwait;
+	hw_data->is_pwait = (uint8_t)is_pwait;
 
 	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &is_soft_reset);
 	ASSERT(is_soft_reset <= 0xFF);
-	dmub->debug.is_dmcub_soft_reset = (uint8_t)is_soft_reset;
+	hw_data->is_dmcub_soft_reset = (uint8_t)is_soft_reset;
 
 	REG_GET(DMCUB_SEC_CNTL, DMCUB_SEC_RESET_STATUS, &is_sec_reset);
 	ASSERT(is_sec_reset <= 0xFF);
-	dmub->debug.is_dmcub_secure_reset = (uint8_t)is_sec_reset;
+	hw_data->is_dmcub_secure_reset = (uint8_t)is_sec_reset;
 
 	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
 	ASSERT(is_traceport_enabled <= 0xFF);
-	dmub->debug.is_traceport_en  = (uint8_t)is_traceport_enabled;
+	hw_data->is_traceport_en  = (uint8_t)is_traceport_enabled;
 
 	REG_GET(DMCUB_REGION3_CW0_TOP_ADDRESS, DMCUB_REGION3_CW0_ENABLE, &is_cw0_enabled);
 	ASSERT(is_cw0_enabled <= 0xFF);
-	dmub->debug.is_cw0_enabled = (uint8_t)is_cw0_enabled;
+	hw_data->is_cw0_enabled = (uint8_t)is_cw0_enabled;
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	ASSERT(is_cw6_enabled <= 0xFF);
-	dmub->debug.is_cw6_enabled = (uint8_t)is_cw6_enabled;
+	hw_data->is_cw6_enabled = (uint8_t)is_cw6_enabled;
 
-	dmub->debug.gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+	hw_data->gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
 }
 void dmub_dcn401_configure_dmub_in_system_memory(struct dmub_srv *dmub)
 {
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.h
index 88c3a44d67d9a0..de7840d9359214 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.h
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.h
@@ -264,7 +264,7 @@ void dmub_dcn401_set_outbox0_rptr(struct dmub_srv *dmub, uint32_t rptr_offset);
 
 uint32_t dmub_dcn401_get_current_time(struct dmub_srv *dmub);
 
-void dmub_dcn401_get_diagnostic_data(struct dmub_srv *dmub);
+void dmub_dcn401_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 
 void dmub_dcn401_configure_dmub_in_system_memory(struct dmub_srv *dmub);
 void dmub_dcn401_send_inbox0_cmd(struct dmub_srv *dmub, union dmub_inbox0_data_register data);
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
index e86fcfc7498566..53e70c682a7772 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
@@ -626,85 +626,81 @@ uint32_t dmub_dcn42_get_current_time(struct dmub_srv *dmub)
 	return REG_READ(DMCUB_TIMER_CURRENT);
 }
 
-void dmub_dcn42_get_diagnostic_data(struct dmub_srv *dmub)
+void dmub_dcn42_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data)
 {
 	uint32_t is_dmub_enabled, is_soft_reset, is_sec_reset, is_pwait;
 	uint32_t is_traceport_enabled, is_cw0_enabled, is_cw6_enabled;
-	struct dmub_timeout_info timeout = {0};
 
-	if (!dmub)
+	if (!dmub || !hw_data)
 		return;
 
-	/* timeout data filled externally, cache before resetting memory */
-	timeout = dmub->debug.timeout_info;
-	memset(&dmub->debug, 0, sizeof(dmub->debug));
-	dmub->debug.timeout_info = timeout;
-
-	dmub->debug.dmcub_version = dmub->fw_version;
-
-	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
-	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
-	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
-	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
-	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
-	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
-	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
-	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
-	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
-	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
-	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
-	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
-	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
-	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
-	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
-	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
-	dmub->debug.scratch[16] = REG_READ(DMCUB_SCRATCH16);
-
-	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
-	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
-	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
-
-	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
-	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
-	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
-
-	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
-	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
-	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
-
-	dmub->debug.outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
-	dmub->debug.outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
-	dmub->debug.outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
+	memset(hw_data, 0, sizeof(*hw_data));
+
+	hw_data->dmcub_version = dmub->fw_version;
+
+	hw_data->scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	hw_data->scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	hw_data->scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	hw_data->scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	hw_data->scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	hw_data->scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	hw_data->scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	hw_data->scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	hw_data->scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	hw_data->scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	hw_data->scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	hw_data->scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	hw_data->scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	hw_data->scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	hw_data->scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	hw_data->scratch[15] = REG_READ(DMCUB_SCRATCH15);
+	hw_data->scratch[16] = REG_READ(DMCUB_SCRATCH16);
+
+	hw_data->undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	hw_data->inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	hw_data->data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	hw_data->inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	hw_data->inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	hw_data->inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	hw_data->inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	hw_data->inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	hw_data->inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+
+	hw_data->outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
+	hw_data->outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
+	hw_data->outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
 
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
 	ASSERT(is_dmub_enabled <= 0xFF);
-	dmub->debug.is_dmcub_enabled = (uint8_t)is_dmub_enabled;
+	hw_data->is_dmcub_enabled = (uint8_t)is_dmub_enabled;
 
 	REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &is_pwait);
 	ASSERT(is_pwait <= 0xFF);
-	dmub->debug.is_pwait = (uint8_t)is_pwait;
+	hw_data->is_pwait = (uint8_t)is_pwait;
 
 	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &is_soft_reset);
 	ASSERT(is_soft_reset <= 0xFF);
-	dmub->debug.is_dmcub_soft_reset = (uint8_t)is_soft_reset;
+	hw_data->is_dmcub_soft_reset = (uint8_t)is_soft_reset;
 
 	REG_GET(DMCUB_SEC_CNTL, DMCUB_SEC_RESET_STATUS, &is_sec_reset);
 	ASSERT(is_sec_reset <= 0xFF);
-	dmub->debug.is_dmcub_secure_reset = (uint8_t)is_sec_reset;
+	hw_data->is_dmcub_secure_reset = (uint8_t)is_sec_reset;
 
 	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
 	ASSERT(is_traceport_enabled <= 0xFF);
-	dmub->debug.is_traceport_en  = (uint8_t)is_traceport_enabled;
+	hw_data->is_traceport_en  = (uint8_t)is_traceport_enabled;
 
 	REG_GET(DMCUB_REGION3_CW0_TOP_ADDRESS, DMCUB_REGION3_CW0_ENABLE, &is_cw0_enabled);
 	ASSERT(is_cw0_enabled <= 0xFF);
-	dmub->debug.is_cw0_enabled = (uint8_t)is_cw0_enabled;
+	hw_data->is_cw0_enabled = (uint8_t)is_cw0_enabled;
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	ASSERT(is_cw6_enabled <= 0xFF);
-	dmub->debug.is_cw6_enabled = (uint8_t)is_cw6_enabled;
+	hw_data->is_cw6_enabled = (uint8_t)is_cw6_enabled;
 
-	dmub->debug.gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+	hw_data->gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
 }
 
 bool dmub_dcn42_get_preos_fw_info(struct dmub_srv *dmub)
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h
index c53f7691d1a886..c879e94272d7af 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.h
@@ -297,7 +297,7 @@ union dmub_fw_boot_options dmub_dcn42_get_fw_boot_option(struct dmub_srv *dmub);
 
 /* Timing and diagnostics */
 uint32_t dmub_dcn42_get_current_time(struct dmub_srv *dmub);
-void dmub_dcn42_get_diagnostic_data(struct dmub_srv *dmub);
+void dmub_dcn42_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 bool dmub_dcn42_get_preos_fw_info(struct dmub_srv *dmub);
 
 #endif /* _DMUB_DCN42_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.c
index 5c524f8d3fdf99..8c3cc51c07a3b3 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.c
@@ -469,85 +469,81 @@ uint32_t dmub_dcn60_get_current_time(struct dmub_srv *dmub)
 	return REG_READ(DMCUB_TIMER_CURRENT);
 }
 
-void dmub_dcn60_get_diagnostic_data(struct dmub_srv *dmub)
+void dmub_dcn60_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data)
 {
 	uint32_t is_dmub_enabled, is_soft_reset, is_sec_reset, is_pwait;
 	uint32_t is_traceport_enabled, is_cw0_enabled, is_cw6_enabled;
-	struct dmub_timeout_info timeout = {0};
 
-	if (!dmub)
+	if (!dmub || !hw_data)
 		return;
 
-	/* timeout data filled externally, cache before resetting memory */
-	timeout = dmub->debug.timeout_info;
-	memset(&dmub->debug, 0, sizeof(dmub->debug));
-	dmub->debug.timeout_info = timeout;
-
-	dmub->debug.dmcub_version = dmub->fw_version;
-
-	dmub->debug.scratch[0] = REG_READ(DMCUB_SCRATCH0);
-	dmub->debug.scratch[1] = REG_READ(DMCUB_SCRATCH1);
-	dmub->debug.scratch[2] = REG_READ(DMCUB_SCRATCH2);
-	dmub->debug.scratch[3] = REG_READ(DMCUB_SCRATCH3);
-	dmub->debug.scratch[4] = REG_READ(DMCUB_SCRATCH4);
-	dmub->debug.scratch[5] = REG_READ(DMCUB_SCRATCH5);
-	dmub->debug.scratch[6] = REG_READ(DMCUB_SCRATCH6);
-	dmub->debug.scratch[7] = REG_READ(DMCUB_SCRATCH7);
-	dmub->debug.scratch[8] = REG_READ(DMCUB_SCRATCH8);
-	dmub->debug.scratch[9] = REG_READ(DMCUB_SCRATCH9);
-	dmub->debug.scratch[10] = REG_READ(DMCUB_SCRATCH10);
-	dmub->debug.scratch[11] = REG_READ(DMCUB_SCRATCH11);
-	dmub->debug.scratch[12] = REG_READ(DMCUB_SCRATCH12);
-	dmub->debug.scratch[13] = REG_READ(DMCUB_SCRATCH13);
-	dmub->debug.scratch[14] = REG_READ(DMCUB_SCRATCH14);
-	dmub->debug.scratch[15] = REG_READ(DMCUB_SCRATCH15);
-	dmub->debug.scratch[16] = REG_READ(DMCUB_SCRATCH16);
-
-	dmub->debug.undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
-	dmub->debug.inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
-	dmub->debug.data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
-
-	dmub->debug.inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
-	dmub->debug.inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
-	dmub->debug.inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
-
-	dmub->debug.inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
-	dmub->debug.inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
-	dmub->debug.inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
-
-	dmub->debug.outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
-	dmub->debug.outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
-	dmub->debug.outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
+	memset(hw_data, 0, sizeof(*hw_data));
+
+	hw_data->dmcub_version = dmub->fw_version;
+
+	hw_data->scratch[0] = REG_READ(DMCUB_SCRATCH0);
+	hw_data->scratch[1] = REG_READ(DMCUB_SCRATCH1);
+	hw_data->scratch[2] = REG_READ(DMCUB_SCRATCH2);
+	hw_data->scratch[3] = REG_READ(DMCUB_SCRATCH3);
+	hw_data->scratch[4] = REG_READ(DMCUB_SCRATCH4);
+	hw_data->scratch[5] = REG_READ(DMCUB_SCRATCH5);
+	hw_data->scratch[6] = REG_READ(DMCUB_SCRATCH6);
+	hw_data->scratch[7] = REG_READ(DMCUB_SCRATCH7);
+	hw_data->scratch[8] = REG_READ(DMCUB_SCRATCH8);
+	hw_data->scratch[9] = REG_READ(DMCUB_SCRATCH9);
+	hw_data->scratch[10] = REG_READ(DMCUB_SCRATCH10);
+	hw_data->scratch[11] = REG_READ(DMCUB_SCRATCH11);
+	hw_data->scratch[12] = REG_READ(DMCUB_SCRATCH12);
+	hw_data->scratch[13] = REG_READ(DMCUB_SCRATCH13);
+	hw_data->scratch[14] = REG_READ(DMCUB_SCRATCH14);
+	hw_data->scratch[15] = REG_READ(DMCUB_SCRATCH15);
+	hw_data->scratch[16] = REG_READ(DMCUB_SCRATCH16);
+
+	hw_data->undefined_address_fault_addr = REG_READ(DMCUB_UNDEFINED_ADDRESS_FAULT_ADDR);
+	hw_data->inst_fetch_fault_addr = REG_READ(DMCUB_INST_FETCH_FAULT_ADDR);
+	hw_data->data_write_fault_addr = REG_READ(DMCUB_DATA_WRITE_FAULT_ADDR);
+
+	hw_data->inbox1_rptr = REG_READ(DMCUB_INBOX1_RPTR);
+	hw_data->inbox1_wptr = REG_READ(DMCUB_INBOX1_WPTR);
+	hw_data->inbox1_size = REG_READ(DMCUB_INBOX1_SIZE);
+
+	hw_data->inbox0_rptr = REG_READ(DMCUB_INBOX0_RPTR);
+	hw_data->inbox0_wptr = REG_READ(DMCUB_INBOX0_WPTR);
+	hw_data->inbox0_size = REG_READ(DMCUB_INBOX0_SIZE);
+
+	hw_data->outbox1_rptr = REG_READ(DMCUB_OUTBOX1_RPTR);
+	hw_data->outbox1_wptr = REG_READ(DMCUB_OUTBOX1_WPTR);
+	hw_data->outbox1_size = REG_READ(DMCUB_OUTBOX1_SIZE);
 
 	REG_GET(DMCUB_CNTL, DMCUB_ENABLE, &is_dmub_enabled);
 	ASSERT(is_dmub_enabled <= 0xFF);
-	dmub->debug.is_dmcub_enabled = (uint8_t)is_dmub_enabled;
+	hw_data->is_dmcub_enabled = (uint8_t)is_dmub_enabled;
 
 	REG_GET(DMCUB_CNTL, DMCUB_PWAIT_MODE_STATUS, &is_pwait);
 	ASSERT(is_pwait <= 0xFF);
-	dmub->debug.is_pwait = (uint8_t)is_pwait;
+	hw_data->is_pwait = (uint8_t)is_pwait;
 
 	REG_GET(DMCUB_CNTL2, DMCUB_SOFT_RESET, &is_soft_reset);
 	ASSERT(is_soft_reset <= 0xFF);
-	dmub->debug.is_dmcub_soft_reset = (uint8_t)is_soft_reset;
+	hw_data->is_dmcub_soft_reset = (uint8_t)is_soft_reset;
 
 	REG_GET(DMCUB_SEC_CNTL, DMCUB_SEC_RESET_STATUS, &is_sec_reset);
 	ASSERT(is_sec_reset <= 0xFF);
-	dmub->debug.is_dmcub_secure_reset = (uint8_t)is_sec_reset;
+	hw_data->is_dmcub_secure_reset = (uint8_t)is_sec_reset;
 
 	REG_GET(DMCUB_CNTL, DMCUB_TRACEPORT_EN, &is_traceport_enabled);
 	ASSERT(is_traceport_enabled <= 0xFF);
-	dmub->debug.is_traceport_en  = (uint8_t)is_traceport_enabled;
+	hw_data->is_traceport_en  = (uint8_t)is_traceport_enabled;
 
 	REG_GET(DMCUB_REGION3_CW0_TOP_ADDRESS, DMCUB_REGION3_CW0_ENABLE, &is_cw0_enabled);
 	ASSERT(is_cw0_enabled <= 0xFF);
-	dmub->debug.is_cw0_enabled = (uint8_t)is_cw0_enabled;
+	hw_data->is_cw0_enabled = (uint8_t)is_cw0_enabled;
 
 	REG_GET(DMCUB_REGION3_CW6_TOP_ADDRESS, DMCUB_REGION3_CW6_ENABLE, &is_cw6_enabled);
 	ASSERT(is_cw6_enabled <= 0xFF);
-	dmub->debug.is_cw6_enabled = (uint8_t)is_cw6_enabled;
+	hw_data->is_cw6_enabled = (uint8_t)is_cw6_enabled;
 
-	dmub->debug.gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
+	hw_data->gpint_datain0 = REG_READ(DMCUB_GPINT_DATAIN0);
 }
 void dmub_dcn60_configure_dmub_in_system_memory(struct dmub_srv *dmub)
 {
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.h
index 5007ed1c19d730..238cba46d366a8 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.h
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn60.h
@@ -277,7 +277,7 @@ void dmub_dcn60_set_outbox0_rptr(struct dmub_srv *dmub, uint32_t rptr_offset);
 
 uint32_t dmub_dcn60_get_current_time(struct dmub_srv *dmub);
 
-void dmub_dcn60_get_diagnostic_data(struct dmub_srv *dmub);
+void dmub_dcn60_get_diagnostic_data(struct dmub_srv *dmub, struct dmub_hw_diagnostic_data *hw_data);
 
 void dmub_dcn60_configure_dmub_in_system_memory(struct dmub_srv *dmub);
 void dmub_dcn60_send_inbox0_cmd(struct dmub_srv *dmub, union dmub_inbox0_data_register data);
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
index 204375f1eff867..41b14b137643d5 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c
@@ -584,7 +584,7 @@ static bool dmub_srv_hw_setup(struct dmub_srv *dmub, enum dmub_asic asic)
 		return false;
 	}
 
-	/* set default inbox type if not overriden */
+	/* set default inbox type if not overridden */
 	if (dmub->inbox_type == DMUB_CMD_INTERFACE_DEFAULT) {
 		if (default_inbox_type != DMUB_CMD_INTERFACE_DEFAULT) {
 			/* use default inbox type as specified by DCN rev */
@@ -1315,11 +1315,63 @@ bool dmub_srv_get_outbox0_msg(struct dmub_srv *dmub, struct dmcub_trace_buf_entr
 	return dmub_rb_out_trace_buffer_front(&dmub->outbox0_rb, (void *)entry);
 }
 
+void dmub_srv_get_trace_snapshot(struct dmub_srv *dmub, struct dmub_trace_snapshot *trace_snapshot)
+{
+	const unsigned int snapshot_max_entry_count =
+			sizeof(trace_snapshot->traces) / sizeof(struct dmcub_trace_buf_entry);
+
+	struct dmub_trace_buf_header *trace_buf_header;
+	struct dmcub_trace_buf_entry *trace_buf;
+	unsigned int trace_buf_max_entry_count;
+	unsigned int entry_count, start_idx, first_chunk_entry_count;
+
+	if (!dmub || !dmub->fb_info || !trace_snapshot)
+		return;
+
+	// capture last trace entries from the DMCUB trace buffer
+	trace_buf_header = (struct dmub_trace_buf_header *)dmub->fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].cpu_addr;
+	trace_buf_max_entry_count = dmub->fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].size == 0 ? 0 :
+			(dmub->fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].size -
+			sizeof(struct dmub_trace_buf_header)) /
+			sizeof(struct dmcub_trace_buf_entry);
+	trace_buf_max_entry_count = trace_buf_max_entry_count > DMUB_TRACE_BUFFER_SIZE ?
+			DMUB_TRACE_BUFFER_SIZE : trace_buf_max_entry_count;
+
+	if (trace_buf_header && trace_buf_max_entry_count > 0) {
+		entry_count = trace_buf_header->entry_count;
+		start_idx = 0;
+		if (entry_count >= snapshot_max_entry_count) {
+			start_idx = (entry_count - snapshot_max_entry_count) % trace_buf_max_entry_count;
+		}
+
+		trace_buf = (struct dmcub_trace_buf_entry *)(trace_buf_header + 1);
+
+		if (start_idx + snapshot_max_entry_count <= trace_buf_max_entry_count) {
+			// contiguous
+			memcpy(trace_snapshot->traces,
+					&trace_buf[start_idx],
+					sizeof(struct dmcub_trace_buf_entry) * snapshot_max_entry_count);
+		} else {
+			// wraparound case
+			first_chunk_entry_count = trace_buf_max_entry_count - start_idx;
+
+			memcpy(trace_snapshot->traces,
+					&trace_buf[start_idx],
+					sizeof(struct dmcub_trace_buf_entry) * first_chunk_entry_count);
+			memcpy(&trace_snapshot->traces[first_chunk_entry_count],
+					&trace_buf[0],
+					sizeof(struct dmcub_trace_buf_entry) * (snapshot_max_entry_count - first_chunk_entry_count));
+		}
+	}
+}
+
 bool dmub_srv_get_diagnostic_data(struct dmub_srv *dmub)
 {
 	if (!dmub || !dmub->hw_funcs.get_diagnostic_data)
 		return false;
-	dmub->hw_funcs.get_diagnostic_data(dmub);
+
+	dmub->hw_funcs.get_diagnostic_data(dmub, &dmub->debug.hw);
+	dmub_srv_get_trace_snapshot(dmub, &dmub->debug.trace_snapshot);
 
 	return true;
 }
@@ -1385,7 +1437,7 @@ void dmub_srv_set_power_state(struct dmub_srv *dmub, enum dmub_srv_power_state_t
 	dmub->power_state = dmub_srv_power_state;
 }
 
-enum dmub_status dmub_srv_reg_cmd_execute(struct dmub_srv *dmub, union dmub_rb_cmd *cmd)
+enum dmub_status dmub_srv_reg_cmd_execute(struct dmub_srv *dmub, const union dmub_rb_cmd *cmd)
 {
 	uint64_t num_pending = 0;
 
@@ -1411,7 +1463,10 @@ enum dmub_status dmub_srv_reg_cmd_execute(struct dmub_srv *dmub, union dmub_rb_c
 
 	/* clear last rsp ack and send message */
 	dmub->hw_funcs.clear_reg_inbox0_rsp_int_ack(dmub);
-	dmub->hw_funcs.send_reg_inbox0_cmd_msg(dmub, cmd);
+	/* hw_funcs signature is not const as some ASIC variants may reuse the
+	 * buffer; send_reg_inbox0_cmd_msg implementations only read from cmd.
+	 */
+	dmub->hw_funcs.send_reg_inbox0_cmd_msg(dmub, (union dmub_rb_cmd *)cmd);
 
 	dmub->reg_inbox0.num_submitted++;
 	dmub->reg_inbox0.is_pending = true;
diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
index 7ed785be8473bd..f4c6ddb6378e6e 100644
--- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h
+++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
@@ -275,6 +275,8 @@ enum {
 	DCN6_VARIANT2_B0        = 0x11,
 	DCN6_VARIANT3_A0        = 0x20,
 	DCN6_VARIANT3_UPPER     = 0x2F,
+	DCN6_VARIANT_LITE3_A0   = 0x50,
+	DCN6_VARIANT_LITE3_B0   = 0x51,
 	DCN6_UNKNOWN		= 0xFF
 };
 
@@ -284,6 +286,9 @@ enum {
 #define ASICREV_IS_DCN6_VARIANT2_A0(eChipRev)     (eChipRev >= DCN6_VARIANT2_A0 && eChipRev < DCN6_VARIANT2_B0)
 #define ASICREV_IS_DCN6_VARIANT2_B0(eChipRev)     (eChipRev >= DCN6_VARIANT2_B0 && eChipRev < DCN6_VARIANT3_A0)
 #define ASICREV_IS_DCN6_VARIANT3(eChipRev)        (eChipRev >= DCN6_VARIANT3_A0 && eChipRev <= DCN6_VARIANT3_UPPER)
+#define ASICREV_IS_DCN6_VARIANT_LITE3(eChipRev)    (eChipRev >= DCN6_VARIANT_LITE3_A0 && eChipRev < DCN6_UNKNOWN)
+#define ASICREV_IS_DCN6_VARIANT_LITE3_A0(eChipRev) (eChipRev >= DCN6_VARIANT_LITE3_A0 && eChipRev < DCN6_VARIANT_LITE3_B0)
+#define ASICREV_IS_DCN6_VARIANT_LITE3_B0(eChipRev) (eChipRev >= DCN6_VARIANT_LITE3_B0 && eChipRev < DCN6_UNKNOWN)
 
 enum {
 	GC_12_0_0_A0 = 0x50,
diff --git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
index 827e9bd7c5cff3..d2a8e712d4a712 100644
--- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
@@ -37,6 +37,7 @@
 #define DP_BRANCH_DEVICE_ID_001CF8 0x001CF8
 #define DP_BRANCH_DEVICE_ID_0060AD 0x0060AD
 #define DP_BRANCH_DEVICE_ID_001FF2 0x001FF2
+#define DP_BRANCH_DEVICE_ID_2B02F0 0x2B02F0 /* Chrontel CH7218 */
 #define DP_BRANCH_HW_REV_10 0x10
 #define DP_BRANCH_HW_REV_20 0x20
 
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index 03d88e78165d5a..5786ef5787ac6d 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -1696,7 +1696,8 @@ bool mod_color_calculate_degamma_params(struct dc_color_caps *dc_caps,
 			return true;
 
 		if (dc_caps != NULL &&
-			dc_caps->dpp.dcn_arch == 1) {
+			dc_caps->dpp.dcn_arch == 1 &&
+			!map_user_ramp) {
 
 			if (input_tf->tf == TRANSFER_FUNCTION_PQ &&
 					dc_caps->dpp.dgam_rom_caps.pq == 1)
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
index eee8206bc531a8..5181d889fe7fed 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
@@ -67,6 +67,10 @@ struct AS_Df_params {
 	struct frame_duration_op decrease;
 };
 
+void mod_build_infopacket_vtem(const struct dc_stream_state *stream,
+		const struct mod_vrr_params *vrr, int fva_factor,
+		struct dc_info_packet *infopacket);
+
 void mod_build_adaptive_sync_infopacket(const struct dc_stream_state *stream,
 		enum adaptive_sync_type asType, const struct AS_Df_params *param,
 		struct dc_info_packet *info_packet);
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_power.h b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
index 02bee3b1956de9..25a0c96ac75493 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
@@ -174,6 +174,7 @@ enum replay_event {
 	replay_event_os_request_force_ffu = 0x2000000,
 	replay_event_os_override_hold = 0x4000000,
 	replay_event_crc_window_active = 0x8000000,
+	replay_event_waiting_stream_regen = 0x10000000,
 };
 
 enum replay_enable_option {
@@ -374,6 +375,9 @@ bool mod_power_replay_set_coasting_vtotal(struct mod_power *mod_power,
 void mod_power_replay_residency(const struct dc_stream_state *stream,
 	unsigned int *residency, const bool is_start, const bool is_alpm);
 
+bool mod_power_replay_get_cumulative_residency(const struct dc_stream_state *stream,
+	uint32_t *residency_milli_pct);
+
 bool mod_power_replay_set_power_opt_and_coasting_vtotal(struct mod_power *mod_power,
 	const struct dc_stream_state *stream, unsigned int active_replay_events, uint32_t coasting_vtotal,
 	bool is_ultra_sleep_mode, uint16_t frame_skip_number);
diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
index f5ac4bf32a784c..32b697f46788b5 100644
--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
@@ -291,6 +291,21 @@ void set_vsc_packet_colorimetry_data(
 	info_packet->sb[18] = 0;
 }
 
+static void set_field_with_mask(unsigned char *dest, unsigned int mask, unsigned int value)
+{
+	unsigned int shift = 0;
+
+	if (!mask || !dest)
+		return;
+
+	while (!((mask >> shift) & 1))
+		shift++;
+
+	*dest = *dest & ~mask;
+	value = value & (mask >> shift);
+	*dest = *dest | (value << shift);
+}
+
 void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
 		struct dc_info_packet *info_packet,
 		enum dc_color_space cs,
@@ -644,6 +659,100 @@ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream,
 		info_packet->valid = true;
 }
 
+static void build_vtem_infopacket_data(const struct dc_stream_state *stream,
+		const struct mod_vrr_params *vrr, int fva_factor,
+		struct dc_info_packet *infopacket)
+{
+	unsigned int field_rate_in_hz;
+
+	/* FVA Factor setting */
+	set_field_with_mask(&infopacket->sb[VTEM_MD0], MASK_VTEM_MD0__FVA_FACTOR_M1,
+			(fva_factor > 0) ? (fva_factor - 1) : 0);
+	/* VRR Parameters */
+	if (vrr->state == VRR_STATE_ACTIVE_VARIABLE ||
+	    vrr->state == VRR_STATE_ACTIVE_FIXED) {
+		set_field_with_mask(&infopacket->sb[VTEM_MD0], MASK_VTEM_MD0__VRR_EN, 1);
+	} else {
+		set_field_with_mask(&infopacket->sb[VTEM_MD0], MASK_VTEM_MD0__VRR_EN, 0);
+	}
+
+	if (vrr->state == VRR_STATE_ACTIVE_FIXED)
+		set_field_with_mask(&infopacket->sb[VTEM_MD0], MASK_VTEM_MD0__M_CONST, vrr->m_const);
+
+	if (!stream->timing.vic) {
+		set_field_with_mask(&infopacket->sb[VTEM_MD1], MASK_VTEM_MD1__BASE_VFRONT,
+				stream->timing.v_front_porch);
+
+
+		/* TODO: In dal2, we check mode flags for a reduced blanking timing.
+		 * Need a way to relay that information to this function.
+		 * if("ReducedBlanking")
+		 * {
+		 *   set_field_with_mask(&infopacket->sb[VRR_VTEM_MD2], MASK__VRR_VTEM_MD2__RB, 1;
+		 * }
+		 */
+
+		field_rate_in_hz = stream->timing.pix_clk_100hz * 100;
+		field_rate_in_hz /= stream->timing.h_total;
+		field_rate_in_hz = (field_rate_in_hz + stream->timing.v_total / 2)
+						/ stream->timing.v_total;
+
+		set_field_with_mask(&infopacket->sb[VTEM_MD2],  MASK_VTEM_MD2__BASE_REFRESH_RATE_98,
+				field_rate_in_hz >> 8);
+		set_field_with_mask(&infopacket->sb[VTEM_MD3], MASK_VTEM_MD3__BASE_REFRESH_RATE_07,
+				field_rate_in_hz);
+
+	}
+
+	/*
+	 * When no VTEM feature is enabled (neither VRR nor FVA), signal a
+	 * zero-length data set (MLDS) by clearing Data_Set_Length. HDMI 2.1
+	 * 10.10.2.4 requires the Source to either stop transmitting the VTEM
+	 * or set Data_Set_Length = 0 when no feature is enabled; keeping the
+	 * VTEM with Data_Set_Length = 0 preserves the every-MTW cadence while
+	 * staying compliant (e.g. HDMI GCTS HF1-58 step 6.2).
+	 */
+	if (vrr->state != VRR_STATE_ACTIVE_VARIABLE &&
+	    vrr->state != VRR_STATE_ACTIVE_FIXED && fva_factor == 0)
+		set_field_with_mask(&infopacket->sb[VTEM_PB6],
+				 MASK_VTEM_PB6__DATA_SET_LENGTH_LSB, 0);
+
+	infopacket->valid = true;
+}
+
+static void build_infopacket_header_vtem(enum signal_type signal,
+		struct dc_info_packet *infopacket)
+{
+	/* HEADER */
+
+	/* HB0, HB1, HB2 indicates PacketType VTEMPacket */
+	infopacket->hb0 = 0x7F;
+	infopacket->hb1 = 0xC0;
+	infopacket->hb2 = 0x00; /* sequence_index */
+
+	set_field_with_mask(&infopacket->sb[VTEM_PB0], MASK_VTEM_PB0__VFR, 1);
+	set_field_with_mask(&infopacket->sb[VTEM_PB2], MASK_VTEM_PB2__ORGANIZATION_ID, 1);
+	set_field_with_mask(&infopacket->sb[VTEM_PB3], MASK_VTEM_PB3__DATA_SET_TAG_MSB, 0);
+	set_field_with_mask(&infopacket->sb[VTEM_PB4], MASK_VTEM_PB4__DATA_SET_TAG_LSB, 1);
+	set_field_with_mask(&infopacket->sb[VTEM_PB5], MASK_VTEM_PB5__DATA_SET_LENGTH_MSB, 0);
+	set_field_with_mask(&infopacket->sb[VTEM_PB6], MASK_VTEM_PB6__DATA_SET_LENGTH_LSB, 4);
+}
+
+void mod_build_infopacket_vtem(const struct dc_stream_state *stream,
+		const struct mod_vrr_params *vrr, int fva_factor,
+		struct dc_info_packet *infopacket)
+{
+	/* VTEM info packet for HdmiVrr */
+
+	memset(infopacket, 0, sizeof(struct dc_info_packet));
+
+	/* VTEM Packet is structured differently */
+	build_infopacket_header_vtem(stream->signal, infopacket);
+	build_vtem_infopacket_data(stream, vrr, fva_factor, infopacket);
+
+	infopacket->valid = true;
+}
+
 void mod_build_adaptive_sync_infopacket(const struct dc_stream_state *stream,
 		enum adaptive_sync_type asType,
 		const struct AS_Df_params *param,
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_replay.c b/drivers/gpu/drm/amd/display/modules/power/power_replay.c
index 31d9cb3a12bacd..7861e46c612d4f 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_replay.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power_replay.c
@@ -761,6 +761,19 @@ void mod_power_replay_residency(const struct dc_stream_state *stream,
 		link->dc->link_srv->edp_replay_residency(link, residency, is_start, mode);
 }
 
+bool mod_power_replay_get_cumulative_residency(const struct dc_stream_state *stream,
+	uint32_t *residency_milli_pct)
+{
+	const struct dc_link *link = dc_stream_get_link(stream);
+
+	if (!link || !link->dc || !link->dc->link_srv)
+		return false;
+
+	return link->dc->link_srv->edp_replay_get_cumulative_residency(
+		link, residency_milli_pct);
+}
+
+
 bool mod_power_replay_set_power_opt_and_coasting_vtotal(struct mod_power *mod_power,
 	const struct dc_stream_state *stream, unsigned int active_replay_events, uint32_t coasting_vtotal,
 	bool is_ultra_sleep_mode, uint16_t frame_skip_number)
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 6d0c74ebdb58fa..e81ce9d7900978 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -90,6 +90,7 @@ enum amd_apu_flags {
 * @AMD_IP_BLOCK_TYPE_UMSCH_MM: User Mode Scheduler for Multimedia
 * @AMD_IP_BLOCK_TYPE_ISP: Image Signal Processor
 * @AMD_IP_BLOCK_TYPE_RAS: Reliability, Availability, Serviceability
+* @AMD_IP_BLOCK_TYPE_UALINK: Ultra Accelerator Link
 * @AMD_IP_BLOCK_TYPE_NUM: Total number of IP block types
 */
 enum amd_ip_block_type {
@@ -111,6 +112,7 @@ enum amd_ip_block_type {
 	AMD_IP_BLOCK_TYPE_UMSCH_MM,
 	AMD_IP_BLOCK_TYPE_ISP,
 	AMD_IP_BLOCK_TYPE_RAS,
+	AMD_IP_BLOCK_TYPE_UALINK,
 	AMD_IP_BLOCK_TYPE_NUM,
 };
 
@@ -287,7 +289,7 @@ enum DC_FEATURE_MASK {
 	 */
 	DC_REPLAY_MASK = (1 << 9),
 	/**
-	 * @DC_FRL_MASK: (0x400) disabled by default
+	 * @DC_FRL_MASK: (0x400) enabled by default
 	 */
 	DC_FRL_MASK = (1 << 10),
 };
@@ -437,7 +439,6 @@ enum amd_dpm_forced_level;
  * @suspend: handles IP specific hw/sw changes for suspend
  * @resume: handles IP specific hw/sw changes for resume
  * @complete: handles IP specific changes after resume
- * @is_idle: returns current IP block idle status
  * @wait_for_idle: poll for idle
  * @soft_reset: soft reset the IP block
  * @set_clockgating_state: enable/disable cg for the IP block
@@ -466,7 +467,6 @@ struct amd_ip_funcs {
 	int (*suspend)(struct amdgpu_ip_block *ip_block);
 	int (*resume)(struct amdgpu_ip_block *ip_block);
 	void (*complete)(struct amdgpu_ip_block *ip_block);
-	bool (*is_idle)(struct amdgpu_ip_block *ip_block);
 	int (*wait_for_idle)(struct amdgpu_ip_block *ip_block);
 	int (*soft_reset)(struct amdgpu_ip_block *ip_block);
 	int (*set_clockgating_state)(struct amdgpu_ip_block *ip_block,
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
index 025e5490e6f12d..dbd497d9fac15e 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
@@ -7369,6 +7369,8 @@
 #define regGFX_IMU_TIMER1_VALUE                                                                         0x40d1
 #define regGFX_IMU_TIMER1_VALUE_BASE_IDX                                                                1
 #define regGFX_IMU_FUSE_CTRL                                                                            0x40e0
+#define regGFX_IMU_PARTITION_SWITCH_SHADOW                                                              0x40f8
+#define regGFX_IMU_PARTITION_SWITCH_SHADOW_BASE_IDX                                                     1
 #define regGFX_IMU_FUSE_CTRL_BASE_IDX                                                                   1
 #define regGFX_IMU_D_RAM_ADDR                                                                           0x40fc
 #define regGFX_IMU_D_RAM_ADDR_BASE_IDX                                                                  1
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
index 1cf890de35cbdf..68dfafd8d027f8 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
@@ -23556,7 +23556,9 @@
 //GFX_IMU_I_RAM_DATA
 #define GFX_IMU_I_RAM_DATA__DATA__SHIFT                                                                       0x0
 #define GFX_IMU_I_RAM_DATA__DATA_MASK                                                                         0xFFFFFFFFL
-
+//GFX_IMU_PARTITION_SWITCH_SHADOW
+#define GFX_IMU_PARTITION_SWITCH_SHADOW__TOTAL_XCCS_IN_XCP__SHIFT                                             0x2
+#define GFX_IMU_PARTITION_SWITCH_SHADOW__TOTAL_XCCS_IN_XCP_MASK                                               0x0000003CL
 
 // addressBlock: CHIP_XCD_gfxip_xcc_gfx_se_gfx_se_grbmhdec
 //GRBMH_CNTL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_offset.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_offset.h
new file mode 100644
index 00000000000000..759ab11d7923ed
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_offset.h
@@ -0,0 +1,11074 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _nbio_7_11_5_OFFSET_HEADER
+#define _nbio_7_11_5_OFFSET_HEADER
+
+
+// addressBlock: nbif0_bif_cfg_dev0_rc_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_RC_VENDOR_ID                                                                    0x0000
+#define cfgBIF_CFG_DEV0_RC_DEVICE_ID                                                                    0x0002
+#define cfgBIF_CFG_DEV0_RC_COMMAND                                                                      0x0004
+#define cfgBIF_CFG_DEV0_RC_STATUS                                                                       0x0006
+#define cfgBIF_CFG_DEV0_RC_REVISION_ID                                                                  0x0008
+#define cfgBIF_CFG_DEV0_RC_PROG_INTERFACE                                                               0x0009
+#define cfgBIF_CFG_DEV0_RC_SUB_CLASS                                                                    0x000a
+#define cfgBIF_CFG_DEV0_RC_BASE_CLASS                                                                   0x000b
+#define cfgBIF_CFG_DEV0_RC_CACHE_LINE                                                                   0x000c
+#define cfgBIF_CFG_DEV0_RC_LATENCY                                                                      0x000d
+#define cfgBIF_CFG_DEV0_RC_HEADER                                                                       0x000e
+#define cfgBIF_CFG_DEV0_RC_BIST                                                                         0x000f
+#define cfgBIF_CFG_DEV0_RC_BASE_ADDR_1                                                                  0x0010
+#define cfgBIF_CFG_DEV0_RC_BASE_ADDR_2                                                                  0x0014
+#define cfgBIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY                                                       0x0018
+#define cfgBIF_CFG_DEV0_RC_IO_BASE_LIMIT                                                                0x001c
+#define cfgBIF_CFG_DEV0_RC_SECONDARY_STATUS                                                             0x001e
+#define cfgBIF_CFG_DEV0_RC_MEM_BASE_LIMIT                                                               0x0020
+#define cfgBIF_CFG_DEV0_RC_PREF_BASE_LIMIT                                                              0x0024
+#define cfgBIF_CFG_DEV0_RC_PREF_BASE_UPPER                                                              0x0028
+#define cfgBIF_CFG_DEV0_RC_PREF_LIMIT_UPPER                                                             0x002c
+#define cfgBIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI                                                             0x0030
+#define cfgBIF_CFG_DEV0_RC_CAP_PTR                                                                      0x0034
+#define cfgBIF_CFG_DEV0_RC_ROM_BASE_ADDR                                                                0x0038
+#define cfgBIF_CFG_DEV0_RC_INTERRUPT_LINE                                                               0x003c
+#define cfgBIF_CFG_DEV0_RC_INTERRUPT_PIN                                                                0x003d
+#define cfgBIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL                                                              0x003e
+#define cfgBIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL                                                              0x0040
+#define cfgBIF_CFG_DEV0_RC_PMI_CAP_LIST                                                                 0x0050
+#define cfgBIF_CFG_DEV0_RC_PMI_CAP                                                                      0x0052
+#define cfgBIF_CFG_DEV0_RC_PMI_STATUS_CNTL                                                              0x0054
+#define cfgBIF_CFG_DEV0_RC_PCIE_CAP_LIST                                                                0x0058
+#define cfgBIF_CFG_DEV0_RC_PCIE_CAP                                                                     0x005a
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CAP                                                                   0x005c
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CNTL                                                                  0x0060
+#define cfgBIF_CFG_DEV0_RC_DEVICE_STATUS                                                                0x0062
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP                                                                     0x0064
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL                                                                    0x0068
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS                                                                  0x006a
+#define cfgBIF_CFG_DEV0_RC_SLOT_CAP                                                                     0x006c
+#define cfgBIF_CFG_DEV0_RC_SLOT_CNTL                                                                    0x0070
+#define cfgBIF_CFG_DEV0_RC_SLOT_STATUS                                                                  0x0072
+#define cfgBIF_CFG_DEV0_RC_ROOT_CNTL                                                                    0x0074
+#define cfgBIF_CFG_DEV0_RC_ROOT_CAP                                                                     0x0076
+#define cfgBIF_CFG_DEV0_RC_ROOT_STATUS                                                                  0x0078
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CAP2                                                                  0x007c
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CNTL2                                                                 0x0080
+#define cfgBIF_CFG_DEV0_RC_DEVICE_STATUS2                                                               0x0082
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP2                                                                    0x0084
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL2                                                                   0x0088
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS2                                                                 0x008a
+#define cfgBIF_CFG_DEV0_RC_SLOT_CAP2                                                                    0x008c
+#define cfgBIF_CFG_DEV0_RC_SLOT_CNTL2                                                                   0x0090
+#define cfgBIF_CFG_DEV0_RC_SLOT_STATUS2                                                                 0x0092
+#define cfgBIF_CFG_DEV0_RC_MSI_CAP_LIST                                                                 0x00a0
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_CNTL                                                                 0x00a2
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO                                                              0x00a4
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI                                                              0x00a8
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_DATA                                                                 0x00a8
+#define cfgBIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA                                                             0x00aa
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_DATA_64                                                              0x00ac
+#define cfgBIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64                                                          0x00ae
+#define cfgBIF_CFG_DEV0_RC_SSID_CAP_LIST                                                                0x00c0
+#define cfgBIF_CFG_DEV0_RC_SSID_CAP                                                                     0x00c4
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                            0x0100
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR                                                     0x0104
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1                                                        0x0108
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2                                                        0x010c
+#define cfgBIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST                                                  0x0110
+#define cfgBIF_CFG_DEV0_RC_PCIE_LINK_CNTL3                                                              0x0114
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS                                                       0x0118
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL                                                0x011c
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL                                                0x011e
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL                                                0x0120
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL                                                0x0122
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL                                                0x0124
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL                                                0x0126
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL                                                0x0128
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL                                                0x012a
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL                                                0x012c
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL                                                0x012e
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL                                               0x0130
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL                                               0x0132
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL                                               0x0134
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL                                               0x0136
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL                                               0x0138
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL                                               0x013a
+#define cfgBIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST                                                   0x01b8
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP_16GT                                                                0x01bc
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL_16GT                                                               0x01c0
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS_16GT                                                             0x01c4
+#define cfgBIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT                                            0x01c8
+#define cfgBIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT                                             0x01cc
+#define cfgBIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT                                             0x01d0
+#define cfgBIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT                                                0x01d8
+#define cfgBIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT                                                0x01d9
+#define cfgBIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT                                                0x01da
+#define cfgBIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT                                                0x01db
+#define cfgBIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT                                                0x01dc
+#define cfgBIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT                                                0x01dd
+#define cfgBIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT                                                0x01de
+#define cfgBIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT                                                0x01df
+#define cfgBIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT                                                0x01e0
+#define cfgBIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT                                                0x01e1
+#define cfgBIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT                                               0x01e2
+#define cfgBIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT                                               0x01e3
+#define cfgBIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT                                               0x01e4
+#define cfgBIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT                                               0x01e5
+#define cfgBIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT                                               0x01e6
+#define cfgBIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT                                               0x01e7
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP_32GT                                                                0x01fc
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL_32GT                                                               0x0200
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS_32GT                                                             0x0204
+#define cfgBIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST                                                  0x0250
+#define cfgBIF_CFG_DEV0_RC_MARGINING_PORT_CAP                                                           0x0254
+#define cfgBIF_CFG_DEV0_RC_MARGINING_PORT_STATUS                                                        0x0256
+#define cfgBIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL                                                   0x0258
+#define cfgBIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS                                                 0x025a
+#define cfgBIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL                                                   0x025c
+#define cfgBIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS                                                 0x025e
+#define cfgBIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL                                                   0x0260
+#define cfgBIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS                                                 0x0262
+#define cfgBIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL                                                   0x0264
+#define cfgBIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS                                                 0x0266
+#define cfgBIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL                                                   0x0268
+#define cfgBIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS                                                 0x026a
+#define cfgBIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL                                                   0x026c
+#define cfgBIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS                                                 0x026e
+#define cfgBIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL                                                   0x0270
+#define cfgBIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS                                                 0x0272
+#define cfgBIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL                                                   0x0274
+#define cfgBIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS                                                 0x0276
+#define cfgBIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL                                                   0x0278
+#define cfgBIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS                                                 0x027a
+#define cfgBIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL                                                   0x027c
+#define cfgBIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS                                                 0x027e
+#define cfgBIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL                                                  0x0280
+#define cfgBIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS                                                0x0282
+#define cfgBIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL                                                  0x0284
+#define cfgBIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS                                                0x0286
+#define cfgBIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL                                                  0x0288
+#define cfgBIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS                                                0x028a
+#define cfgBIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL                                                  0x028c
+#define cfgBIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS                                                0x028e
+#define cfgBIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL                                                  0x0290
+#define cfgBIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS                                                0x0292
+#define cfgBIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL                                                  0x0294
+#define cfgBIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS                                                0x0296
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST                                                         0x0404
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1                                                        0x0408
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2                                                        0x040c
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL                                                            0x0410
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS                                                          0x0412
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP                                                        0x0414
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL                                                       0x0418
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS                                                     0x041e
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP                                                        0x0420
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL                                                       0x0424
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS                                                     0x042a
+#define cfgBIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST                                                        0x0430
+#define cfgBIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP                                                        0x0434
+#define cfgBIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS                                                     0x0438
+#define cfgBIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                                0x0604
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS                                                       0x0608
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK                                                         0x060c
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY                                                     0x0610
+#define cfgBIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS                                                         0x0614
+#define cfgBIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK                                                           0x0618
+#define cfgBIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL                                                        0x061c
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG0                                                                0x0620
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG1                                                                0x0624
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG2                                                                0x0628
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG3                                                                0x062c
+#define cfgBIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD                                                            0x0630
+#define cfgBIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS                                                         0x0634
+#define cfgBIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID                                                              0x0638
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0                                                         0x063c
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1                                                         0x0640
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2                                                         0x0644
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3                                                         0x0648
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                             0x0664
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1                                                      0x0668
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2                                                      0x066c
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST                                                        0x06dc
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_CAP                                                                 0x06e0
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_CNTL                                                                0x06e2
+
+
+// addressBlock: nbif0_bif_cfg_dev1_rc_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV1_RC_VENDOR_ID                                                                    0x0000
+#define cfgBIF_CFG_DEV1_RC_DEVICE_ID                                                                    0x0002
+#define cfgBIF_CFG_DEV1_RC_COMMAND                                                                      0x0004
+#define cfgBIF_CFG_DEV1_RC_STATUS                                                                       0x0006
+#define cfgBIF_CFG_DEV1_RC_REVISION_ID                                                                  0x0008
+#define cfgBIF_CFG_DEV1_RC_PROG_INTERFACE                                                               0x0009
+#define cfgBIF_CFG_DEV1_RC_SUB_CLASS                                                                    0x000a
+#define cfgBIF_CFG_DEV1_RC_BASE_CLASS                                                                   0x000b
+#define cfgBIF_CFG_DEV1_RC_CACHE_LINE                                                                   0x000c
+#define cfgBIF_CFG_DEV1_RC_LATENCY                                                                      0x000d
+#define cfgBIF_CFG_DEV1_RC_HEADER                                                                       0x000e
+#define cfgBIF_CFG_DEV1_RC_BIST                                                                         0x000f
+#define cfgBIF_CFG_DEV1_RC_BASE_ADDR_1                                                                  0x0010
+#define cfgBIF_CFG_DEV1_RC_BASE_ADDR_2                                                                  0x0014
+#define cfgBIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY                                                       0x0018
+#define cfgBIF_CFG_DEV1_RC_IO_BASE_LIMIT                                                                0x001c
+#define cfgBIF_CFG_DEV1_RC_SECONDARY_STATUS                                                             0x001e
+#define cfgBIF_CFG_DEV1_RC_MEM_BASE_LIMIT                                                               0x0020
+#define cfgBIF_CFG_DEV1_RC_PREF_BASE_LIMIT                                                              0x0024
+#define cfgBIF_CFG_DEV1_RC_PREF_BASE_UPPER                                                              0x0028
+#define cfgBIF_CFG_DEV1_RC_PREF_LIMIT_UPPER                                                             0x002c
+#define cfgBIF_CFG_DEV1_RC_IO_BASE_LIMIT_HI                                                             0x0030
+#define cfgBIF_CFG_DEV1_RC_CAP_PTR                                                                      0x0034
+#define cfgBIF_CFG_DEV1_RC_ROM_BASE_ADDR                                                                0x0038
+#define cfgBIF_CFG_DEV1_RC_INTERRUPT_LINE                                                               0x003c
+#define cfgBIF_CFG_DEV1_RC_INTERRUPT_PIN                                                                0x003d
+#define cfgBIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL                                                              0x003e
+#define cfgBIF_CFG_DEV1_RC_EXT_BRIDGE_CNTL                                                              0x0040
+#define cfgBIF_CFG_DEV1_RC_PMI_CAP_LIST                                                                 0x0050
+#define cfgBIF_CFG_DEV1_RC_PMI_CAP                                                                      0x0052
+#define cfgBIF_CFG_DEV1_RC_PMI_STATUS_CNTL                                                              0x0054
+#define cfgBIF_CFG_DEV1_RC_PCIE_CAP_LIST                                                                0x0058
+#define cfgBIF_CFG_DEV1_RC_PCIE_CAP                                                                     0x005a
+#define cfgBIF_CFG_DEV1_RC_DEVICE_CAP                                                                   0x005c
+#define cfgBIF_CFG_DEV1_RC_DEVICE_CNTL                                                                  0x0060
+#define cfgBIF_CFG_DEV1_RC_DEVICE_STATUS                                                                0x0062
+#define cfgBIF_CFG_DEV1_RC_LINK_CAP                                                                     0x0064
+#define cfgBIF_CFG_DEV1_RC_LINK_CNTL                                                                    0x0068
+#define cfgBIF_CFG_DEV1_RC_LINK_STATUS                                                                  0x006a
+#define cfgBIF_CFG_DEV1_RC_SLOT_CAP                                                                     0x006c
+#define cfgBIF_CFG_DEV1_RC_SLOT_CNTL                                                                    0x0070
+#define cfgBIF_CFG_DEV1_RC_SLOT_STATUS                                                                  0x0072
+#define cfgBIF_CFG_DEV1_RC_ROOT_CNTL                                                                    0x0074
+#define cfgBIF_CFG_DEV1_RC_ROOT_CAP                                                                     0x0076
+#define cfgBIF_CFG_DEV1_RC_ROOT_STATUS                                                                  0x0078
+#define cfgBIF_CFG_DEV1_RC_DEVICE_CAP2                                                                  0x007c
+#define cfgBIF_CFG_DEV1_RC_DEVICE_CNTL2                                                                 0x0080
+#define cfgBIF_CFG_DEV1_RC_DEVICE_STATUS2                                                               0x0082
+#define cfgBIF_CFG_DEV1_RC_LINK_CAP2                                                                    0x0084
+#define cfgBIF_CFG_DEV1_RC_LINK_CNTL2                                                                   0x0088
+#define cfgBIF_CFG_DEV1_RC_LINK_STATUS2                                                                 0x008a
+#define cfgBIF_CFG_DEV1_RC_SLOT_CAP2                                                                    0x008c
+#define cfgBIF_CFG_DEV1_RC_SLOT_CNTL2                                                                   0x0090
+#define cfgBIF_CFG_DEV1_RC_SLOT_STATUS2                                                                 0x0092
+#define cfgBIF_CFG_DEV1_RC_MSI_CAP_LIST                                                                 0x00a0
+#define cfgBIF_CFG_DEV1_RC_MSI_MSG_CNTL                                                                 0x00a2
+#define cfgBIF_CFG_DEV1_RC_MSI_MSG_ADDR_LO                                                              0x00a4
+#define cfgBIF_CFG_DEV1_RC_MSI_MSG_ADDR_HI                                                              0x00a8
+#define cfgBIF_CFG_DEV1_RC_MSI_MSG_DATA                                                                 0x00a8
+#define cfgBIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA                                                             0x00aa
+#define cfgBIF_CFG_DEV1_RC_MSI_MSG_DATA_64                                                              0x00ac
+#define cfgBIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA_64                                                          0x00ae
+#define cfgBIF_CFG_DEV1_RC_SSID_CAP_LIST                                                                0x00c0
+#define cfgBIF_CFG_DEV1_RC_SSID_CAP                                                                     0x00c4
+#define cfgBIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                            0x0100
+#define cfgBIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR                                                     0x0104
+#define cfgBIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC1                                                        0x0108
+#define cfgBIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC2                                                        0x010c
+#define cfgBIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST                                                  0x0110
+#define cfgBIF_CFG_DEV1_RC_PCIE_LINK_CNTL3                                                              0x0114
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_ERROR_STATUS                                                       0x0118
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL                                                0x011c
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL                                                0x011e
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL                                                0x0120
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL                                                0x0122
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL                                                0x0124
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL                                                0x0126
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL                                                0x0128
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL                                                0x012a
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL                                                0x012c
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL                                                0x012e
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL                                               0x0130
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL                                               0x0132
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL                                               0x0134
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL                                               0x0136
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL                                               0x0138
+#define cfgBIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL                                               0x013a
+#define cfgBIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST                                                   0x01b8
+#define cfgBIF_CFG_DEV1_RC_LINK_CAP_16GT                                                                0x01bc
+#define cfgBIF_CFG_DEV1_RC_LINK_CNTL_16GT                                                               0x01c0
+#define cfgBIF_CFG_DEV1_RC_LINK_STATUS_16GT                                                             0x01c4
+#define cfgBIF_CFG_DEV1_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT                                            0x01c8
+#define cfgBIF_CFG_DEV1_RC_RTM1_PARITY_MISMATCH_STATUS_16GT                                             0x01cc
+#define cfgBIF_CFG_DEV1_RC_RTM2_PARITY_MISMATCH_STATUS_16GT                                             0x01d0
+#define cfgBIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_16GT                                                0x01d8
+#define cfgBIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_16GT                                                0x01d9
+#define cfgBIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_16GT                                                0x01da
+#define cfgBIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_16GT                                                0x01db
+#define cfgBIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_16GT                                                0x01dc
+#define cfgBIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_16GT                                                0x01dd
+#define cfgBIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_16GT                                                0x01de
+#define cfgBIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_16GT                                                0x01df
+#define cfgBIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_16GT                                                0x01e0
+#define cfgBIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_16GT                                                0x01e1
+#define cfgBIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_16GT                                               0x01e2
+#define cfgBIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_16GT                                               0x01e3
+#define cfgBIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_16GT                                               0x01e4
+#define cfgBIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_16GT                                               0x01e5
+#define cfgBIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_16GT                                               0x01e6
+#define cfgBIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_16GT                                               0x01e7
+#define cfgBIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST                                                  0x0250
+#define cfgBIF_CFG_DEV1_RC_MARGINING_PORT_CAP                                                           0x0254
+#define cfgBIF_CFG_DEV1_RC_MARGINING_PORT_STATUS                                                        0x0256
+#define cfgBIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL                                                   0x0258
+#define cfgBIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS                                                 0x025a
+#define cfgBIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL                                                   0x025c
+#define cfgBIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS                                                 0x025e
+#define cfgBIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL                                                   0x0260
+#define cfgBIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS                                                 0x0262
+#define cfgBIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL                                                   0x0264
+#define cfgBIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS                                                 0x0266
+#define cfgBIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL                                                   0x0268
+#define cfgBIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS                                                 0x026a
+#define cfgBIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL                                                   0x026c
+#define cfgBIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS                                                 0x026e
+#define cfgBIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL                                                   0x0270
+#define cfgBIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS                                                 0x0272
+#define cfgBIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL                                                   0x0274
+#define cfgBIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS                                                 0x0276
+#define cfgBIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL                                                   0x0278
+#define cfgBIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS                                                 0x027a
+#define cfgBIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL                                                   0x027c
+#define cfgBIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS                                                 0x027e
+#define cfgBIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL                                                  0x0280
+#define cfgBIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS                                                0x0282
+#define cfgBIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL                                                  0x0284
+#define cfgBIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS                                                0x0286
+#define cfgBIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL                                                  0x0288
+#define cfgBIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS                                                0x028a
+#define cfgBIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL                                                  0x028c
+#define cfgBIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS                                                0x028e
+#define cfgBIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL                                                  0x0290
+#define cfgBIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS                                                0x0292
+#define cfgBIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL                                                  0x0294
+#define cfgBIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS                                                0x0296
+#define cfgBIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST                                                         0x0404
+#define cfgBIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1                                                        0x0408
+#define cfgBIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG2                                                        0x040c
+#define cfgBIF_CFG_DEV1_RC_PCIE_PORT_VC_CNTL                                                            0x0410
+#define cfgBIF_CFG_DEV1_RC_PCIE_PORT_VC_STATUS                                                          0x0412
+#define cfgBIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP                                                        0x0414
+#define cfgBIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL                                                       0x0418
+#define cfgBIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_STATUS                                                     0x041e
+#define cfgBIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP                                                        0x0420
+#define cfgBIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL                                                       0x0424
+#define cfgBIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_STATUS                                                     0x042a
+#define cfgBIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST                                                        0x0430
+#define cfgBIF_CFG_DEV1_RC_DATA_LINK_FEATURE_CAP                                                        0x0434
+#define cfgBIF_CFG_DEV1_RC_DATA_LINK_FEATURE_STATUS                                                     0x0438
+#define cfgBIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                                0x0604
+#define cfgBIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS                                                       0x0608
+#define cfgBIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK                                                         0x060c
+#define cfgBIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY                                                     0x0610
+#define cfgBIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS                                                         0x0614
+#define cfgBIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK                                                           0x0618
+#define cfgBIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL                                                        0x061c
+#define cfgBIF_CFG_DEV1_RC_PCIE_HDR_LOG0                                                                0x0620
+#define cfgBIF_CFG_DEV1_RC_PCIE_HDR_LOG1                                                                0x0624
+#define cfgBIF_CFG_DEV1_RC_PCIE_HDR_LOG2                                                                0x0628
+#define cfgBIF_CFG_DEV1_RC_PCIE_HDR_LOG3                                                                0x062c
+#define cfgBIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD                                                            0x0630
+#define cfgBIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS                                                         0x0634
+#define cfgBIF_CFG_DEV1_RC_PCIE_ERR_SRC_ID                                                              0x0638
+#define cfgBIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG0                                                         0x063c
+#define cfgBIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG1                                                         0x0640
+#define cfgBIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG2                                                         0x0644
+#define cfgBIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG3                                                         0x0648
+#define cfgBIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                             0x0664
+#define cfgBIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW1                                                      0x0668
+#define cfgBIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW2                                                      0x066c
+#define cfgBIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST                                                        0x06dc
+#define cfgBIF_CFG_DEV1_RC_PCIE_ACS_CAP                                                                 0x06e0
+#define cfgBIF_CFG_DEV1_RC_PCIE_ACS_CNTL                                                                0x06e2
+
+
+// addressBlock: nbif0_bif_cfg_dev2_rc_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY                                                       0x0018
+#define cfgBIF_CFG_DEV2_RC_IO_BASE_LIMIT                                                                0x001c
+#define cfgBIF_CFG_DEV2_RC_SECONDARY_STATUS                                                             0x001e
+#define cfgBIF_CFG_DEV2_RC_MEM_BASE_LIMIT                                                               0x0020
+#define cfgBIF_CFG_DEV2_RC_PREF_BASE_LIMIT                                                              0x0024
+#define cfgBIF_CFG_DEV2_RC_PREF_BASE_UPPER                                                              0x0028
+#define cfgBIF_CFG_DEV2_RC_PREF_LIMIT_UPPER                                                             0x002c
+#define cfgBIF_CFG_DEV2_RC_IO_BASE_LIMIT_HI                                                             0x0030
+#define cfgBIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL                                                              0x003e
+#define cfgBIF_CFG_DEV2_RC_SLOT_CAP                                                                     0x006c
+#define cfgBIF_CFG_DEV2_RC_SLOT_CNTL                                                                    0x0070
+#define cfgBIF_CFG_DEV2_RC_SLOT_STATUS                                                                  0x0072
+#define cfgBIF_CFG_DEV2_RC_SLOT_CAP2                                                                    0x008c
+#define cfgBIF_CFG_DEV2_RC_SLOT_CNTL2                                                                   0x0090
+#define cfgBIF_CFG_DEV2_RC_SLOT_STATUS2                                                                 0x0092
+#define cfgBIF_CFG_DEV2_RC_SSID_CAP_LIST                                                                0x00c0
+#define cfgBIF_CFG_DEV2_RC_SSID_CAP                                                                     0x00c4
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF0_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF0_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF0_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF0_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF0_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF0_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF0_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF0_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF0_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF0_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF0_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF0_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF0_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF0_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF0_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF0_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF0_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF0_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF0_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF0_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF0_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0604
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS                                                     0x0608
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK                                                       0x060c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY                                                   0x0610
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS                                                       0x0614
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK                                                         0x0618
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL                                                      0x061c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0                                                              0x0620
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1                                                              0x0624
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2                                                              0x0628
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3                                                              0x062c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0                                                       0x063c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1                                                       0x0640
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2                                                       0x0644
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3                                                       0x0648
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                           0x0664
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1                                                    0x0668
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2                                                    0x066c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST                                                      0x06dc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_CAP                                                               0x06e0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL                                                              0x06e2
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0728
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT                                                0x072c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA                                                       0x0730
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP                                                        0x0734
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST                                                      0x0738
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_CAP                                                               0x073c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR                                                 0x0740
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS                                                            0x0744
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL                                                              0x0746
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0748
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0749
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x074a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x074b
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x074c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x074d
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x074e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x074f
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST                                                      0x0768
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP                                                               0x076c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST                                                      0x0770
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_CAP                                                               0x0774
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL                                                              0x0776
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST                                                      0x0778
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP                                                              0x077c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL                                                             0x0780
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP                                                              0x0784
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL                                                             0x0788
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP                                                              0x078c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL                                                             0x0790
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP                                                              0x0794
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL                                                             0x0798
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP                                                              0x079c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL                                                             0x07a0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP                                                              0x07a4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL                                                             0x07a8
+#define cfgPCIE_ATS_ENH_CAP_LIST                                                                        0x07ac
+#define cfgPCIE_ATS_CAP                                                                                 0x07b0
+#define cfgPCIE_ATS_CNTL                                                                                0x07b2
+#define cfgPCIE_PAGE_REQ_ENH_CAP_LIST                                                                   0x07b4
+#define cfgPCIE_PAGE_REQ_CNTL                                                                           0x07b8
+#define cfgPCIE_PAGE_REQ_STATUS                                                                         0x07ba
+#define cfgPCIE_OUTSTAND_PAGE_REQ_CAPACITY                                                              0x07bc
+#define cfgPCIE_OUTSTAND_PAGE_REQ_ALLOC                                                                 0x07c0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST                                                    0x07c4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PASID_CAP                                                             0x07c8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL                                                            0x07ca
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST                                                    0x07cc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP                                                             0x07d0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL                                                         0x07d4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_STATUS                                                          0x07d6
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_INITIAL_VFS                                                     0x07d8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_TOTAL_VFS                                                       0x07da
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_NUM_VFS                                                         0x07dc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_FUNC_DEP_LINK                                                   0x07de
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_FIRST_VF_OFFSET                                                 0x07e0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_STRIDE                                                       0x07e2
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_DEVICE_ID                                                    0x07e6
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE                                             0x07e8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_SYSTEM_PAGE_SIZE                                                0x07ec
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_0                                                  0x07f0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_1                                                  0x07f4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_2                                                  0x07f8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_3                                                  0x07fc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_4                                                  0x0800
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_5                                                  0x0804
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET                                 0x0808
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                            0x08e0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CAP                                                    0x08e4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL                                                   0x08e8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CAP                                                    0x08ec
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL                                                   0x08f0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CAP                                                    0x08f4
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL                                                   0x08f8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CAP                                                    0x08fc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL                                                   0x0900
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CAP                                                    0x0904
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL                                                   0x0908
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CAP                                                    0x090c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL                                                   0x0910
+#define cfgPCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV                                                     0x09ac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV                                                              0x09b0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE                                                  0x09b8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS                                                  0x09bc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL                                                0x09c0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0                                                0x09c4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1                                                0x09c8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2                                                0x09cc
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST                                                0x0110
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3                                                            0x0114
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS                                                     0x0118
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL                                              0x011c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL                                              0x011e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL                                              0x0120
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL                                              0x0122
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL                                              0x0124
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL                                              0x0126
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL                                              0x0128
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL                                              0x012a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL                                              0x012c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL                                              0x012e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL                                             0x0130
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL                                             0x0132
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL                                             0x0134
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL                                             0x0136
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL                                             0x0138
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL                                             0x013a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST                                                 0x01b8
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP_16GT                                                              0x01bc
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL_16GT                                                             0x01c0
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS_16GT                                                           0x01c4
+#define cfgBIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                          0x01c8
+#define cfgBIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT                                           0x01cc
+#define cfgBIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT                                           0x01d0
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT                                              0x01d8
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT                                              0x01d9
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT                                              0x01da
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT                                              0x01db
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT                                              0x01dc
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT                                              0x01dd
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT                                              0x01de
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT                                              0x01df
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT                                              0x01e0
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT                                              0x01e1
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT                                             0x01e2
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT                                             0x01e3
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT                                             0x01e4
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT                                             0x01e5
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT                                             0x01e6
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT                                             0x01e7
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP_32GT                                                              0x01fc
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL_32GT                                                             0x0200
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS_32GT                                                           0x0204
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST                                                0x0250
+#define cfgBIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP                                                         0x0254
+#define cfgBIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS                                                      0x0256
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL                                                 0x0258
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS                                               0x025a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL                                                 0x025c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS                                               0x025e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL                                                 0x0260
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS                                               0x0262
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL                                                 0x0264
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS                                               0x0266
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL                                                 0x0268
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS                                               0x026a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL                                                 0x026c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS                                               0x026e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL                                                 0x0270
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS                                               0x0272
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL                                                 0x0274
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS                                               0x0276
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL                                                 0x0278
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS                                               0x027a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL                                                 0x027c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS                                               0x027e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL                                                0x0280
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS                                              0x0282
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL                                                0x0284
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS                                              0x0286
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL                                                0x0288
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS                                              0x028a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL                                                0x028c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS                                              0x028e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL                                                0x0290
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS                                              0x0292
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL                                                0x0294
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS                                              0x0296
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST                                                       0x0404
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1                                                      0x0408
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2                                                      0x040c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL                                                          0x0410
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS                                                        0x0412
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP                                                      0x0414
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL                                                     0x0418
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS                                                   0x041e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP                                                      0x0420
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL                                                     0x0424
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS                                                   0x042a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST                                                      0x0430
+#define cfgBIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP                                                      0x0434
+#define cfgBIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS                                                   0x0438
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF1_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF1_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF1_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF1_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF1_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF1_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF1_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF1_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF1_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF1_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF1_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF1_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF1_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF1_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF1_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF1_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF1_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF1_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF1_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF1_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF1_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF1_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF1_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF1_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF1_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF1_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF1_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF1_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF1_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF1_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF1_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0604
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS                                                     0x0608
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK                                                       0x060c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY                                                   0x0610
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS                                                       0x0614
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK                                                         0x0618
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL                                                      0x061c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0                                                              0x0620
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1                                                              0x0624
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2                                                              0x0628
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3                                                              0x062c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0                                                       0x063c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1                                                       0x0640
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2                                                       0x0644
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3                                                       0x0648
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                           0x0664
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW1                                                    0x0668
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW2                                                    0x066c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST                                                      0x06dc
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ACS_CAP                                                               0x06e0
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL                                                              0x06e2
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0728
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT                                                0x072c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA                                                       0x0730
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP                                                        0x0734
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST                                                      0x0738
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_CAP                                                               0x073c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR                                                 0x0740
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS                                                            0x0744
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL                                                              0x0746
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0748
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0749
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x074a
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x074b
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x074c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x074d
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x074e
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x074f
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST                                                      0x0770
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ARI_CAP                                                               0x0774
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL                                                              0x0776
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST                                                      0x0778
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP                                                              0x077c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL                                                             0x0780
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP                                                              0x0784
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL                                                             0x0788
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP                                                              0x078c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL                                                             0x0790
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP                                                              0x0794
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL                                                             0x0798
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP                                                              0x079c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL                                                             0x07a0
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP                                                              0x07a4
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL                                                             0x07a8
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST                                                    0x07c4
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PASID_CAP                                                             0x07c8
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL                                                            0x07ca
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                            0x08e0
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP                                                    0x08e4
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL                                                   0x08e8
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP                                                    0x08ec
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL                                                   0x08f0
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP                                                    0x08f4
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL                                                   0x08f8
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP                                                    0x08fc
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL                                                   0x0900
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP                                                    0x0904
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL                                                   0x0908
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP                                                    0x090c
+#define cfgBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL                                                   0x0910
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf2_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF2_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF2_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF2_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF2_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF2_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF2_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF2_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF2_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF2_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF2_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF2_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF2_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF2_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF2_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF2_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF2_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF2_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF2_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF2_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF2_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF2_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF2_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF2_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF2_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF2_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF2_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF2_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF2_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF2_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF2_SBRN                                                                       0x0060
+#define cfgBIF_CFG_DEV0_EPF2_FLADJ                                                                      0x0061
+#define cfgBIF_CFG_DEV0_EPF2_DBESL_DBESLD                                                               0x0062
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF2_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF2_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF2_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF2_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF2_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF2_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF2_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF2_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF2_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF2_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF2_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF2_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF2_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF2_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF2_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF2_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF2_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF2_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF2_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF2_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF2_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF2_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF2_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0604
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS                                                     0x0608
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK                                                       0x060c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY                                                   0x0610
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS                                                       0x0614
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK                                                         0x0618
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL                                                      0x061c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_HDR_LOG0                                                              0x0620
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_HDR_LOG1                                                              0x0624
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_HDR_LOG2                                                              0x0628
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_HDR_LOG3                                                              0x062c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG0                                                       0x063c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG1                                                       0x0640
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG2                                                       0x0644
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG3                                                       0x0648
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST                                                      0x06dc
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ACS_CAP                                                               0x06e0
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL                                                              0x06e2
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0728
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA_SELECT                                                0x072c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA                                                       0x0730
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_CAP                                                        0x0734
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST                                                      0x0738
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_CAP                                                               0x073c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_LATENCY_INDICATOR                                                 0x0740
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_STATUS                                                            0x0744
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_CNTL                                                              0x0746
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0748
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0749
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x074a
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x074b
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x074c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x074d
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x074e
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x074f
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST                                                      0x0770
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ARI_CAP                                                               0x0774
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL                                                              0x0776
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST                                                      0x0778
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR1_CAP                                                              0x077c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL                                                             0x0780
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR2_CAP                                                              0x0784
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL                                                             0x0788
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR3_CAP                                                              0x078c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL                                                             0x0790
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR4_CAP                                                              0x0794
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL                                                             0x0798
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR5_CAP                                                              0x079c
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL                                                             0x07a0
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR6_CAP                                                              0x07a4
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL                                                             0x07a8
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST                                                    0x07c4
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_PASID_CAP                                                             0x07c8
+#define cfgBIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL                                                            0x07ca
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf3_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF3_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF3_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF3_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF3_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF3_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF3_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF3_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF3_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF3_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF3_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF3_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF3_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF3_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF3_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF3_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF3_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF3_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF3_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF3_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF3_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF3_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF3_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF3_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF3_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF3_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF3_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF3_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF3_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF3_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF3_SBRN                                                                       0x0060
+#define cfgBIF_CFG_DEV0_EPF3_FLADJ                                                                      0x0061
+#define cfgBIF_CFG_DEV0_EPF3_DBESL_DBESLD                                                               0x0062
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF3_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF3_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF3_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF3_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF3_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF3_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF3_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF3_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF3_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF3_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF3_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF3_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF3_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF3_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF3_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF3_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF3_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF3_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF3_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF3_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF3_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF3_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF3_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0604
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS                                                     0x0608
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK                                                       0x060c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY                                                   0x0610
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS                                                       0x0614
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK                                                         0x0618
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL                                                      0x061c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_HDR_LOG0                                                              0x0620
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_HDR_LOG1                                                              0x0624
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_HDR_LOG2                                                              0x0628
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_HDR_LOG3                                                              0x062c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG0                                                       0x063c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG1                                                       0x0640
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG2                                                       0x0644
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG3                                                       0x0648
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST                                                      0x06dc
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ACS_CAP                                                               0x06e0
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL                                                              0x06e2
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0728
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA_SELECT                                                0x072c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA                                                       0x0730
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_CAP                                                        0x0734
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST                                                      0x0738
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_CAP                                                               0x073c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_LATENCY_INDICATOR                                                 0x0740
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_STATUS                                                            0x0744
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_CNTL                                                              0x0746
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0748
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0749
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x074a
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x074b
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x074c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x074d
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x074e
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x074f
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST                                                      0x0770
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ARI_CAP                                                               0x0774
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL                                                              0x0776
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST                                                      0x0778
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR1_CAP                                                              0x077c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL                                                             0x0780
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR2_CAP                                                              0x0784
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL                                                             0x0788
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR3_CAP                                                              0x078c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL                                                             0x0790
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR4_CAP                                                              0x0794
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL                                                             0x0798
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR5_CAP                                                              0x079c
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL                                                             0x07a0
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR6_CAP                                                              0x07a4
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL                                                             0x07a8
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST                                                    0x07c4
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_PASID_CAP                                                             0x07c8
+#define cfgBIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL                                                            0x07ca
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf5_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF5_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF5_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF5_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF5_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF5_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF5_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF5_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF5_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF5_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF5_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF5_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF5_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF5_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF5_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF5_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF5_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF5_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF5_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF5_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF5_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF5_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF5_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF5_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF5_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF5_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF5_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF5_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF5_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF5_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF5_SBRN                                                                       0x0060
+#define cfgBIF_CFG_DEV0_EPF5_FLADJ                                                                      0x0061
+#define cfgBIF_CFG_DEV0_EPF5_DBESL_DBESLD                                                               0x0062
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF5_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF5_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF5_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF5_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF5_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF5_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF5_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF5_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF5_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF5_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF5_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF5_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF5_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF5_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF5_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF5_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF5_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF5_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF5_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF5_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF5_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF5_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF5_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0604
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS                                                     0x0608
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK                                                       0x060c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY                                                   0x0610
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS                                                       0x0614
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK                                                         0x0618
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL                                                      0x061c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_HDR_LOG0                                                              0x0620
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_HDR_LOG1                                                              0x0624
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_HDR_LOG2                                                              0x0628
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_HDR_LOG3                                                              0x062c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG0                                                       0x063c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG1                                                       0x0640
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG2                                                       0x0644
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG3                                                       0x0648
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST                                                      0x06dc
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ACS_CAP                                                               0x06e0
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL                                                              0x06e2
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0728
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA_SELECT                                                0x072c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA                                                       0x0730
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_CAP                                                        0x0734
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST                                                      0x0738
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_CAP                                                               0x073c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_LATENCY_INDICATOR                                                 0x0740
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_STATUS                                                            0x0744
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_CNTL                                                              0x0746
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0748
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0749
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x074a
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x074b
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x074c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x074d
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x074e
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x074f
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST                                                      0x0770
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ARI_CAP                                                               0x0774
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL                                                              0x0776
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST                                                      0x0778
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR1_CAP                                                              0x077c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL                                                             0x0780
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR2_CAP                                                              0x0784
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL                                                             0x0788
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR3_CAP                                                              0x078c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL                                                             0x0790
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR4_CAP                                                              0x0794
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL                                                             0x0798
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR5_CAP                                                              0x079c
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL                                                             0x07a0
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR6_CAP                                                              0x07a4
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL                                                             0x07a8
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST                                                    0x07c4
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_PASID_CAP                                                             0x07c8
+#define cfgBIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL                                                            0x07ca
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf6_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF6_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF6_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF6_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF6_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF6_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF6_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF6_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF6_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF6_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF6_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF6_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF6_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF6_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF6_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF6_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF6_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF6_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF6_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF6_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF6_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF6_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF6_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF6_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF6_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF6_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF6_MAX_LATENCY                                                                0x003f
+#define cfgBIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST                                                            0x0048
+#define cfgBIF_CFG_DEV0_EPF6_ADAPTER_ID_W                                                               0x004c
+#define cfgBIF_CFG_DEV0_EPF6_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF6_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF6_SBRN                                                                       0x0060
+#define cfgBIF_CFG_DEV0_EPF6_FLADJ                                                                      0x0061
+#define cfgBIF_CFG_DEV0_EPF6_DBESL_DBESLD                                                               0x0062
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF6_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF6_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF6_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF6_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF6_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF6_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF6_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF6_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF6_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF6_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF6_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF6_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF6_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF6_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF6_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF6_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF6_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF6_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF6_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF6_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF6_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF6_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF6_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0604
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS                                                     0x0608
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK                                                       0x060c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY                                                   0x0610
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS                                                       0x0614
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK                                                         0x0618
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL                                                      0x061c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_HDR_LOG0                                                              0x0620
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_HDR_LOG1                                                              0x0624
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_HDR_LOG2                                                              0x0628
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_HDR_LOG3                                                              0x062c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG0                                                       0x063c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG1                                                       0x0640
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG2                                                       0x0644
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG3                                                       0x0648
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST                                                      0x06dc
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ACS_CAP                                                               0x06e0
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL                                                              0x06e2
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x0728
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA_SELECT                                                0x072c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA                                                       0x0730
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_CAP                                                        0x0734
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST                                                      0x0738
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_CAP                                                               0x073c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_LATENCY_INDICATOR                                                 0x0740
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_STATUS                                                            0x0744
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_CNTL                                                              0x0746
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x0748
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x0749
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x074a
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x074b
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x074c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x074d
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x074e
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x074f
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST                                                      0x0770
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ARI_CAP                                                               0x0774
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL                                                              0x0776
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST                                                      0x0778
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR1_CAP                                                              0x077c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL                                                             0x0780
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR2_CAP                                                              0x0784
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL                                                             0x0788
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR3_CAP                                                              0x078c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL                                                             0x0790
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR4_CAP                                                              0x0794
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL                                                             0x0798
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR5_CAP                                                              0x079c
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL                                                             0x07a0
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR6_CAP                                                              0x07a4
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL                                                             0x07a8
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST                                                    0x07c4
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_PASID_CAP                                                             0x07c8
+#define cfgBIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL                                                            0x07ca
+
+
+// addressBlock: nbif0_bif_cfg_dev0_rc_bifcfgdecp
+// base address: 0x10100000
+#define regBIF_CFG_DEV0_RC0_VENDOR_ID                                                                   0x0000
+#define regBIF_CFG_DEV0_RC0_VENDOR_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_DEVICE_ID                                                                   0x0000
+#define regBIF_CFG_DEV0_RC0_DEVICE_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_COMMAND                                                                     0x0001
+#define regBIF_CFG_DEV0_RC0_COMMAND_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_STATUS                                                                      0x0001
+#define regBIF_CFG_DEV0_RC0_STATUS_BASE_IDX                                                             5
+#define regBIF_CFG_DEV0_RC0_REVISION_ID                                                                 0x0002
+#define regBIF_CFG_DEV0_RC0_REVISION_ID_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_PROG_INTERFACE                                                              0x0002
+#define regBIF_CFG_DEV0_RC0_PROG_INTERFACE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_SUB_CLASS                                                                   0x0002
+#define regBIF_CFG_DEV0_RC0_SUB_CLASS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_BASE_CLASS                                                                  0x0002
+#define regBIF_CFG_DEV0_RC0_BASE_CLASS_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_CACHE_LINE                                                                  0x0003
+#define regBIF_CFG_DEV0_RC0_CACHE_LINE_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_LATENCY                                                                     0x0003
+#define regBIF_CFG_DEV0_RC0_LATENCY_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_HEADER                                                                      0x0003
+#define regBIF_CFG_DEV0_RC0_HEADER_BASE_IDX                                                             5
+#define regBIF_CFG_DEV0_RC0_BIST                                                                        0x0003
+#define regBIF_CFG_DEV0_RC0_BIST_BASE_IDX                                                               5
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_1                                                                 0x0004
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_1_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_2                                                                 0x0005
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY                                                      0x0006
+#define regBIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT                                                               0x0007
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_SECONDARY_STATUS                                                            0x0007
+#define regBIF_CFG_DEV0_RC0_SECONDARY_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_MEM_BASE_LIMIT                                                              0x0008
+#define regBIF_CFG_DEV0_RC0_MEM_BASE_LIMIT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_LIMIT                                                             0x0009
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_LIMIT_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_UPPER                                                             0x000a
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_UPPER_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER                                                            0x000b
+#define regBIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI                                                            0x000c
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_CAP_PTR                                                                     0x000d
+#define regBIF_CFG_DEV0_RC0_CAP_PTR_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_ROM_BASE_ADDR                                                               0x000e
+#define regBIF_CFG_DEV0_RC0_ROM_BASE_ADDR_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_LINE                                                              0x000f
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_LINE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_PIN                                                               0x000f
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_PIN_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL                                                             0x000f
+#define regBIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL                                                             0x0010
+#define regBIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_LIST                                                                0x0014
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_PMI_CAP                                                                     0x0014
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_PMI_STATUS_CNTL                                                             0x0015
+#define regBIF_CFG_DEV0_RC0_PMI_STATUS_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_LIST                                                               0x0016
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP                                                                    0x0016
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP                                                                  0x0017
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL                                                                 0x0018
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS                                                               0x0018
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP                                                                    0x0019
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL                                                                   0x001a
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS                                                                 0x001a
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP                                                                    0x001b
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL                                                                   0x001c
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS                                                                 0x001c
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_ROOT_CNTL                                                                   0x001d
+#define regBIF_CFG_DEV0_RC0_ROOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_ROOT_CAP                                                                    0x001d
+#define regBIF_CFG_DEV0_RC0_ROOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_ROOT_STATUS                                                                 0x001e
+#define regBIF_CFG_DEV0_RC0_ROOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP2                                                                 0x001f
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL2                                                                0x0020
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS2                                                              0x0020
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP2                                                                   0x0021
+#define regBIF_CFG_DEV0_RC0_LINK_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL2                                                                  0x0022
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS2                                                                0x0022
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP2                                                                   0x0023
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL2                                                                  0x0024
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS2                                                                0x0024
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_CAP_LIST                                                                0x0028
+#define regBIF_CFG_DEV0_RC0_MSI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_CNTL                                                                0x0028
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO                                                             0x0029
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI                                                             0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA                                                                0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA                                                            0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_64                                                             0x002b
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_64_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64                                                         0x002b
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_LIST                                                               0x0030
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_SSID_CAP                                                                    0x0031
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                           0x0040
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR                                                    0x0041
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1                                                       0x0042
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2                                                       0x0043
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST                                                 0x0044
+#define regBIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3                                                             0x0045
+#define regBIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS                                                      0x0046
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL                                               0x0047
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL                                               0x0047
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL                                               0x0048
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL                                               0x0048
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL                                               0x0049
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL                                               0x0049
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL                                               0x004a
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL                                               0x004a
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL                                               0x004b
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL                                               0x004b
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL                                              0x004c
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL                                              0x004c
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL                                              0x004d
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL                                              0x004d
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL                                              0x004e
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL                                              0x004e
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST                                                  0x006e
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_16GT                                                               0x006f
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_16GT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_16GT                                                              0x0070
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_16GT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_16GT                                                            0x0071
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                           0x0072
+#define regBIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT                                            0x0073
+#define regBIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT                                            0x0074
+#define regBIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT                                               0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT                                               0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT                                              0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT                                              0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_32GT                                                               0x007f
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_32GT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_32GT                                                              0x0080
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_32GT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_32GT                                                            0x0081
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_32GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST                                                 0x0094
+#define regBIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_CAP                                                          0x0095
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_CAP_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS                                                       0x0095
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL                                                  0x0096
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS                                                0x0096
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL                                                  0x0097
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS                                                0x0097
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL                                                  0x0098
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS                                                0x0098
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL                                                  0x0099
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS                                                0x0099
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL                                                  0x009a
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS                                                0x009a
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL                                                  0x009b
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS                                                0x009b
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL                                                  0x009c
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS                                                0x009c
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL                                                  0x009d
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS                                                0x009d
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL                                                  0x009e
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS                                                0x009e
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL                                                  0x009f
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS                                                0x009f
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL                                                 0x00a0
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS                                               0x00a0
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL                                                 0x00a1
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS                                               0x00a1
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL                                                 0x00a2
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS                                               0x00a2
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL                                                 0x00a3
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS                                               0x00a3
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL                                                 0x00a4
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS                                               0x00a4
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL                                                 0x00a5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS                                               0x00a5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST                                                        0x0101
+#define regBIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1                                                       0x0102
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2                                                       0x0103
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL                                                           0x0104
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS                                                         0x0104
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP                                                       0x0105
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL                                                      0x0106
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS                                                    0x0107
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP                                                       0x0108
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL                                                      0x0109
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS                                                    0x010a
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST                                                       0x010c
+#define regBIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP                                                       0x010d
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS                                                    0x010e
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                               0x0181
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS                                                      0x0182
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK                                                        0x0183
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY                                                    0x0184
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS                                                        0x0185
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK                                                          0x0186
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL                                                       0x0187
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG0                                                               0x0188
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG0_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG1                                                               0x0189
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG1_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG2                                                               0x018a
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG3                                                               0x018b
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG3_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD                                                           0x018c
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS                                                        0x018d
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID                                                             0x018e
+#define regBIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0                                                        0x018f
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1                                                        0x0190
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2                                                        0x0191
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3                                                        0x0192
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                            0x0199
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1                                                     0x019a
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2                                                     0x019b
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST                                                       0x01b7
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CAP                                                                0x01b8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CAP_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CNTL                                                               0x01b8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CNTL_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST                                                       0x01c7
+#define regBIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_RTR_DATA1                                                                   0x01c8
+#define regBIF_CFG_DEV0_RC0_RTR_DATA1_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_RTR_DATA2                                                                   0x01c9
+#define regBIF_CFG_DEV0_RC0_RTR_DATA2_BASE_IDX                                                          5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_ID                                                                0x10000
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_ID                                                                0x10000
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_COMMAND                                                                  0x10001
+#define regBIF_CFG_DEV0_EPF0_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_STATUS                                                                   0x10001
+#define regBIF_CFG_DEV0_EPF0_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_0_REVISION_ID                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PROG_INTERFACE                                                           0x10002
+#define regBIF_CFG_DEV0_EPF0_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_SUB_CLASS                                                                0x10002
+#define regBIF_CFG_DEV0_EPF0_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_CLASS                                                               0x10002
+#define regBIF_CFG_DEV0_EPF0_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_CACHE_LINE                                                               0x10003
+#define regBIF_CFG_DEV0_EPF0_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LATENCY                                                                  0x10003
+#define regBIF_CFG_DEV0_EPF0_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_HEADER                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_0_BIST                                                                     0x10003
+#define regBIF_CFG_DEV0_EPF0_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_1                                                              0x10004
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_2                                                              0x10005
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_3                                                              0x10006
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_4                                                              0x10007
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_5                                                              0x10008
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_6                                                              0x10009
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR                                                          0x1000a
+#define regBIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID                                                               0x1000b
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR                                                            0x1000c
+#define regBIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_CAP_PTR                                                                  0x1000d
+#define regBIF_CFG_DEV0_EPF0_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE                                                           0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_MIN_GRANT                                                                0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_MAX_LATENCY                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST                                                          0x10012
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W                                                             0x10013
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST                                                             0x10014
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP                                                                  0x10014
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL                                                          0x10015
+#define regBIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST                                                            0x10019
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP                                                                 0x10019
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP                                                               0x1001a
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL                                                              0x1001b
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP2                                                              0x10022
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2                                                             0x10023
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST                                                             0x10028
+#define regBIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL                                                             0x10028
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO                                                          0x10029
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI                                                          0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA                                                             0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA                                                         0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK                                                                 0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64                                                          0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64                                                      0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_64                                                              0x1002c
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING                                                              0x1002c
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_64                                                           0x1002d
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST                                                            0x10030
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL                                                            0x10030
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_TABLE                                                               0x10031
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_PBA                                                                 0x10032
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x10181
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS                                                   0x10182
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK                                                     0x10183
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x10184
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS                                                     0x10185
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK                                                       0x10186
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x10187
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0                                                            0x10188
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1                                                            0x10189
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2                                                            0x1018a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3                                                            0x1018b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0                                                     0x1018f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1                                                     0x10190
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2                                                     0x10191
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3                                                     0x10192
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                         0x10199
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX                                5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1                                                  0x1019a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2                                                  0x1019b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST                                                    0x101b7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP                                                             0x101b8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL                                                            0x101b8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST                                                    0x101c7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA1                                                                0x101c8
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA2                                                                0x101c9
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x101ca
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x101cb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA                                                     0x101cc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP                                                      0x101cd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST                                                    0x101ce
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP                                                             0x101cf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR                                               0x101d0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS                                                          0x101d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL                                                            0x101d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST                                                    0x101da
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP                                                             0x101db
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST                                                    0x101dc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP                                                             0x101dd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL                                                            0x101dd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST                                                    0x101de
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP                                                            0x101df
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL                                                           0x101e0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP                                                            0x101e1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL                                                           0x101e2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP                                                            0x101e3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL                                                           0x101e4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP                                                            0x101e5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL                                                           0x101e6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP                                                            0x101e7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL                                                           0x101e8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP                                                            0x101e9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL                                                           0x101ea
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST                                                    0x101eb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP                                                             0x101ec
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL                                                            0x101ec
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST                                               0x101ed
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL                                                       0x101ee
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS                                                     0x101ee
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY                                          0x101ef
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC                                             0x101f0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST                                                  0x101f1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP                                                           0x101f2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL                                                          0x101f2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST                                                  0x101f3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP                                                           0x101f4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL                                                       0x101f5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS                                                        0x101f5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS                                                   0x101f6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS                                                     0x101f6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS                                                       0x101f7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK                                                 0x101f7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET                                               0x101f8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE                                                     0x101f8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID                                                  0x101f9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE                                           0x101fa
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE                                              0x101fb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0                                                0x101fc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1                                                0x101fd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2                                                0x101fe
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3                                                0x101ff
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4                                                0x10200
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5                                                0x10201
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET                               0x10202
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_BASE_IDX                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                          0x10238
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP                                                  0x10239
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL                                                 0x1023a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP                                                  0x1023b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL                                                 0x1023c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP                                                  0x1023d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL                                                 0x1023e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP                                                  0x1023f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL                                                 0x10240
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP                                                  0x10241
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL                                                 0x10242
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP                                                  0x10243
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL                                                 0x10244
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV                                 0x1026b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV                                          0x1026c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE                              0x1026e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE_BASE_IDX                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS                              0x1026f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS_BASE_IDX                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL                            0x10270
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL_BASE_IDX                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0                            0x10271
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0_BASE_IDX                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1                            0x10272
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1_BASE_IDX                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2                            0x10273
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2_BASE_IDX                   5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP                                                                 0x1001c
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL                                                                0x1001d
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP2                                                                0x10024
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL2                                                               0x10025
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x10040
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x10041
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1                                                    0x10042
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2                                                    0x10043
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST                                              0x10044
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3                                                          0x10045
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS                                                   0x10046
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL                                            0x10047
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL                                            0x10047
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL                                            0x10048
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL                                            0x10048
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL                                            0x10049
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL                                            0x10049
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL                                            0x1004a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL                                            0x1004a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL                                            0x1004b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL                                            0x1004b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL                                           0x1004c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL                                           0x1004c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL                                           0x1004d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL                                           0x1004d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL                                           0x1004e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL                                           0x1004e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST                                               0x1006e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT                                                            0x1006f
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT                                                           0x10070
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT                                                         0x10071
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                        0x10072
+#define regBIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT                                         0x10073
+#define regBIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT                                         0x10074
+#define regBIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT                                            0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT                                            0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT                                           0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT                                           0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT                                                            0x1007f
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT                                                           0x10080
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT                                                         0x10081
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST                                              0x10094
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP                                                       0x10095
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS                                                    0x10095
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL                                               0x10096
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS                                             0x10096
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL                                               0x10097
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS                                             0x10097
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL                                               0x10098
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS                                             0x10098
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL                                               0x10099
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS                                             0x10099
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL                                               0x1009a
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS                                             0x1009a
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL                                               0x1009b
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS                                             0x1009b
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL                                               0x1009c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS                                             0x1009c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL                                               0x1009d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS                                             0x1009d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL                                               0x1009e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS                                             0x1009e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL                                               0x1009f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS                                             0x1009f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL                                              0x100a0
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS                                            0x100a0
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL                                              0x100a1
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS                                            0x100a1
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL                                              0x100a2
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS                                            0x100a2
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL                                              0x100a3
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS                                            0x100a3
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL                                              0x100a4
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS                                            0x100a4
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL                                              0x100a5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS                                            0x100a5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST                                                     0x10101
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1                                                    0x10102
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2                                                    0x10103
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL                                                        0x10104
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS                                                      0x10104
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP                                                    0x10105
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL                                                   0x10106
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS                                                 0x10107
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP                                                    0x10108
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL                                                   0x10109
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS                                                 0x1010a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST                                                    0x1010c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP                                                    0x1010d
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS                                                 0x1010e
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                        5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+// base address: 0x10141000
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_ID                                                                0x10400
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_ID                                                                0x10400
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_COMMAND                                                                  0x10401
+#define regBIF_CFG_DEV0_EPF1_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_0_STATUS                                                                   0x10401
+#define regBIF_CFG_DEV0_EPF1_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF1_0_REVISION_ID                                                              0x10402
+#define regBIF_CFG_DEV0_EPF1_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_PROG_INTERFACE                                                           0x10402
+#define regBIF_CFG_DEV0_EPF1_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_SUB_CLASS                                                                0x10402
+#define regBIF_CFG_DEV0_EPF1_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_BASE_CLASS                                                               0x10402
+#define regBIF_CFG_DEV0_EPF1_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_0_CACHE_LINE                                                               0x10403
+#define regBIF_CFG_DEV0_EPF1_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_0_LATENCY                                                                  0x10403
+#define regBIF_CFG_DEV0_EPF1_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_0_HEADER                                                                   0x10403
+#define regBIF_CFG_DEV0_EPF1_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF1_0_BIST                                                                     0x10403
+#define regBIF_CFG_DEV0_EPF1_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_1                                                              0x10404
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_2                                                              0x10405
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_3                                                              0x10406
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_4                                                              0x10407
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_5                                                              0x10408
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_6                                                              0x10409
+#define regBIF_CFG_DEV0_EPF1_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR                                                          0x1040a
+#define regBIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID                                                               0x1040b
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR                                                            0x1040c
+#define regBIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_CAP_PTR                                                                  0x1040d
+#define regBIF_CFG_DEV0_EPF1_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE                                                           0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN                                                            0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_MIN_GRANT                                                                0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_MAX_LATENCY                                                              0x1040f
+#define regBIF_CFG_DEV0_EPF1_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST                                                          0x10412
+#define regBIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W                                                             0x10413
+#define regBIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST                                                             0x10414
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP                                                                  0x10414
+#define regBIF_CFG_DEV0_EPF1_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL                                                          0x10415
+#define regBIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST                                                            0x10419
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP                                                                 0x10419
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP                                                               0x1041a
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL                                                              0x1041b
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS                                                            0x1041b
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP                                                                 0x1041c
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL                                                                0x1041d
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS                                                              0x1041d
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP2                                                              0x10422
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2                                                             0x10423
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2                                                           0x10423
+#define regBIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP2                                                                0x10424
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL2                                                               0x10425
+#define regBIF_CFG_DEV0_EPF1_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS2                                                             0x10425
+#define regBIF_CFG_DEV0_EPF1_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST                                                             0x10428
+#define regBIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL                                                             0x10428
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO                                                          0x10429
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI                                                          0x1042a
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA                                                             0x1042a
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA                                                         0x1042a
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK                                                                 0x1042b
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64                                                          0x1042b
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64                                                      0x1042b
+#define regBIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK_64                                                              0x1042c
+#define regBIF_CFG_DEV0_EPF1_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING                                                              0x1042c
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING_64                                                           0x1042d
+#define regBIF_CFG_DEV0_EPF1_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST                                                            0x10430
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL                                                            0x10430
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_TABLE                                                               0x10431
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_PBA                                                                 0x10432
+#define regBIF_CFG_DEV0_EPF1_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x10440
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x10441
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1                                                    0x10442
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2                                                    0x10443
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x10581
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS                                                   0x10582
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK                                                     0x10583
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x10584
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS                                                     0x10585
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK                                                       0x10586
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x10587
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0                                                            0x10588
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1                                                            0x10589
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2                                                            0x1058a
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3                                                            0x1058b
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0                                                     0x1058f
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1                                                     0x10590
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2                                                     0x10591
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3                                                     0x10592
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                         0x10599
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX                                5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW1                                                  0x1059a
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW2                                                  0x1059b
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST                                                    0x105b7
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP                                                             0x105b8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL                                                            0x105b8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST                                                    0x105c7
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_RTR_DATA1                                                                0x105c8
+#define regBIF_CFG_DEV0_EPF1_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_RTR_DATA2                                                                0x105c9
+#define regBIF_CFG_DEV0_EPF1_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x105ca
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x105cb
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA                                                     0x105cc
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP                                                      0x105cd
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST                                                    0x105ce
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP                                                             0x105cf
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR                                               0x105d0
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS                                                          0x105d1
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL                                                            0x105d1
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x105d2
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x105d2
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x105d2
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x105d2
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x105d3
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x105d3
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x105d3
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x105d3
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST                                                    0x105dc
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP                                                             0x105dd
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL                                                            0x105dd
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST                                                    0x105de
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP                                                            0x105df
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL                                                           0x105e0
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP                                                            0x105e1
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL                                                           0x105e2
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP                                                            0x105e3
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL                                                           0x105e4
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP                                                            0x105e5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL                                                           0x105e6
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP                                                            0x105e7
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL                                                           0x105e8
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP                                                            0x105e9
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL                                                           0x105ea
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST                                                  0x105f1
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP                                                           0x105f2
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL                                                          0x105f2
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                          0x10638
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP                                                  0x10639
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL                                                 0x1063a
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP                                                  0x1063b
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL                                                 0x1063c
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP                                                  0x1063d
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL                                                 0x1063e
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP                                                  0x1063f
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL                                                 0x10640
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP                                                  0x10641
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL                                                 0x10642
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP                                                  0x10643
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL                                                 0x10644
+#define regBIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL_BASE_IDX                                        5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf2_bifcfgdecp
+// base address: 0x10142000
+#define regBIF_CFG_DEV0_EPF2_0_VENDOR_ID                                                                0x10800
+#define regBIF_CFG_DEV0_EPF2_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_ID                                                                0x10800
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_COMMAND                                                                  0x10801
+#define regBIF_CFG_DEV0_EPF2_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF2_0_STATUS                                                                   0x10801
+#define regBIF_CFG_DEV0_EPF2_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF2_0_REVISION_ID                                                              0x10802
+#define regBIF_CFG_DEV0_EPF2_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_PROG_INTERFACE                                                           0x10802
+#define regBIF_CFG_DEV0_EPF2_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_SUB_CLASS                                                                0x10802
+#define regBIF_CFG_DEV0_EPF2_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_BASE_CLASS                                                               0x10802
+#define regBIF_CFG_DEV0_EPF2_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF2_0_CACHE_LINE                                                               0x10803
+#define regBIF_CFG_DEV0_EPF2_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF2_0_LATENCY                                                                  0x10803
+#define regBIF_CFG_DEV0_EPF2_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF2_0_HEADER                                                                   0x10803
+#define regBIF_CFG_DEV0_EPF2_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF2_0_BIST                                                                     0x10803
+#define regBIF_CFG_DEV0_EPF2_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_1                                                              0x10804
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_2                                                              0x10805
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_3                                                              0x10806
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_4                                                              0x10807
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_5                                                              0x10808
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_6                                                              0x10809
+#define regBIF_CFG_DEV0_EPF2_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_CARDBUS_CIS_PTR                                                          0x1080a
+#define regBIF_CFG_DEV0_EPF2_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF2_0_ADAPTER_ID                                                               0x1080b
+#define regBIF_CFG_DEV0_EPF2_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR                                                            0x1080c
+#define regBIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_CAP_PTR                                                                  0x1080d
+#define regBIF_CFG_DEV0_EPF2_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF2_0_INTERRUPT_LINE                                                           0x1080f
+#define regBIF_CFG_DEV0_EPF2_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_INTERRUPT_PIN                                                            0x1080f
+#define regBIF_CFG_DEV0_EPF2_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_MIN_GRANT                                                                0x1080f
+#define regBIF_CFG_DEV0_EPF2_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_MAX_LATENCY                                                              0x1080f
+#define regBIF_CFG_DEV0_EPF2_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST                                                          0x10812
+#define regBIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF2_0_ADAPTER_ID_W                                                             0x10813
+#define regBIF_CFG_DEV0_EPF2_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_PMI_CAP_LIST                                                             0x10814
+#define regBIF_CFG_DEV0_EPF2_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_PMI_CAP                                                                  0x10814
+#define regBIF_CFG_DEV0_EPF2_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL                                                          0x10815
+#define regBIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF2_0_SBRN                                                                     0x10818
+#define regBIF_CFG_DEV0_EPF2_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF2_0_FLADJ                                                                    0x10818
+#define regBIF_CFG_DEV0_EPF2_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF2_0_DBESL_DBESLD                                                             0x10818
+#define regBIF_CFG_DEV0_EPF2_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CAP_LIST                                                            0x10819
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CAP                                                                 0x10819
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CAP                                                               0x1081a
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CNTL                                                              0x1081b
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_STATUS                                                            0x1081b
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CAP                                                                 0x1081c
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CNTL                                                                0x1081d
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_LINK_STATUS                                                              0x1081d
+#define regBIF_CFG_DEV0_EPF2_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CAP2                                                              0x10822
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2                                                             0x10823
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_STATUS2                                                           0x10823
+#define regBIF_CFG_DEV0_EPF2_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CAP2                                                                0x10824
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CNTL2                                                               0x10825
+#define regBIF_CFG_DEV0_EPF2_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF2_0_LINK_STATUS2                                                             0x10825
+#define regBIF_CFG_DEV0_EPF2_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_CAP_LIST                                                             0x10828
+#define regBIF_CFG_DEV0_EPF2_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL                                                             0x10828
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_LO                                                          0x10829
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_HI                                                          0x1082a
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA                                                             0x1082a
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA                                                         0x1082a
+#define regBIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MASK                                                                 0x1082b
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA_64                                                          0x1082b
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA_64                                                      0x1082b
+#define regBIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MASK_64                                                              0x1082c
+#define regBIF_CFG_DEV0_EPF2_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_PENDING                                                              0x1082c
+#define regBIF_CFG_DEV0_EPF2_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF2_0_MSI_PENDING_64                                                           0x1082d
+#define regBIF_CFG_DEV0_EPF2_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_CAP_LIST                                                            0x10830
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL                                                            0x10830
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_TABLE                                                               0x10831
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_PBA                                                                 0x10832
+#define regBIF_CFG_DEV0_EPF2_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x10840
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x10841
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC1                                                    0x10842
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC2                                                    0x10843
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x10981
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS                                                   0x10982
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK                                                     0x10983
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x10984
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS                                                     0x10985
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK                                                       0x10986
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x10987
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG0                                                            0x10988
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG1                                                            0x10989
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG2                                                            0x1098a
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG3                                                            0x1098b
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG0                                                     0x1098f
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG1                                                     0x10990
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG2                                                     0x10991
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG3                                                     0x10992
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST                                                    0x109b7
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP                                                             0x109b8
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL                                                            0x109b8
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST                                                    0x109c7
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_RTR_DATA1                                                                0x109c8
+#define regBIF_CFG_DEV0_EPF2_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_RTR_DATA2                                                                0x109c9
+#define regBIF_CFG_DEV0_EPF2_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x109ca
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x109cb
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA                                                     0x109cc
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_CAP                                                      0x109cd
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST                                                    0x109ce
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP                                                             0x109cf
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_LATENCY_INDICATOR                                               0x109d0
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_STATUS                                                          0x109d1
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_CNTL                                                            0x109d1
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x109d2
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x109d2
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x109d2
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x109d2
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x109d3
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x109d3
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x109d3
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x109d3
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST                                                    0x109dc
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP                                                             0x109dd
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL                                                            0x109dd
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST                                                    0x109de
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CAP                                                            0x109df
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL                                                           0x109e0
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CAP                                                            0x109e1
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL                                                           0x109e2
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CAP                                                            0x109e3
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL                                                           0x109e4
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CAP                                                            0x109e5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL                                                           0x109e6
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CAP                                                            0x109e7
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL                                                           0x109e8
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CAP                                                            0x109e9
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL                                                           0x109ea
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST                                                  0x109f1
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP                                                           0x109f2
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL                                                          0x109f2
+#define regBIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf3_bifcfgdecp
+// base address: 0x10143000
+#define regBIF_CFG_DEV0_EPF3_0_VENDOR_ID                                                                0x10c00
+#define regBIF_CFG_DEV0_EPF3_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_ID                                                                0x10c00
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_COMMAND                                                                  0x10c01
+#define regBIF_CFG_DEV0_EPF3_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF3_0_STATUS                                                                   0x10c01
+#define regBIF_CFG_DEV0_EPF3_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF3_0_REVISION_ID                                                              0x10c02
+#define regBIF_CFG_DEV0_EPF3_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_PROG_INTERFACE                                                           0x10c02
+#define regBIF_CFG_DEV0_EPF3_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_SUB_CLASS                                                                0x10c02
+#define regBIF_CFG_DEV0_EPF3_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_BASE_CLASS                                                               0x10c02
+#define regBIF_CFG_DEV0_EPF3_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF3_0_CACHE_LINE                                                               0x10c03
+#define regBIF_CFG_DEV0_EPF3_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF3_0_LATENCY                                                                  0x10c03
+#define regBIF_CFG_DEV0_EPF3_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF3_0_HEADER                                                                   0x10c03
+#define regBIF_CFG_DEV0_EPF3_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF3_0_BIST                                                                     0x10c03
+#define regBIF_CFG_DEV0_EPF3_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_1                                                              0x10c04
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_2                                                              0x10c05
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_3                                                              0x10c06
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_4                                                              0x10c07
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_5                                                              0x10c08
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_6                                                              0x10c09
+#define regBIF_CFG_DEV0_EPF3_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_CARDBUS_CIS_PTR                                                          0x10c0a
+#define regBIF_CFG_DEV0_EPF3_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF3_0_ADAPTER_ID                                                               0x10c0b
+#define regBIF_CFG_DEV0_EPF3_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR                                                            0x10c0c
+#define regBIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_CAP_PTR                                                                  0x10c0d
+#define regBIF_CFG_DEV0_EPF3_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF3_0_INTERRUPT_LINE                                                           0x10c0f
+#define regBIF_CFG_DEV0_EPF3_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_INTERRUPT_PIN                                                            0x10c0f
+#define regBIF_CFG_DEV0_EPF3_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_MIN_GRANT                                                                0x10c0f
+#define regBIF_CFG_DEV0_EPF3_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_MAX_LATENCY                                                              0x10c0f
+#define regBIF_CFG_DEV0_EPF3_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST                                                          0x10c12
+#define regBIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF3_0_ADAPTER_ID_W                                                             0x10c13
+#define regBIF_CFG_DEV0_EPF3_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_PMI_CAP_LIST                                                             0x10c14
+#define regBIF_CFG_DEV0_EPF3_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_PMI_CAP                                                                  0x10c14
+#define regBIF_CFG_DEV0_EPF3_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL                                                          0x10c15
+#define regBIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF3_0_SBRN                                                                     0x10c18
+#define regBIF_CFG_DEV0_EPF3_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF3_0_FLADJ                                                                    0x10c18
+#define regBIF_CFG_DEV0_EPF3_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF3_0_DBESL_DBESLD                                                             0x10c18
+#define regBIF_CFG_DEV0_EPF3_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CAP_LIST                                                            0x10c19
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CAP                                                                 0x10c19
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CAP                                                               0x10c1a
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CNTL                                                              0x10c1b
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_STATUS                                                            0x10c1b
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CAP                                                                 0x10c1c
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CNTL                                                                0x10c1d
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_LINK_STATUS                                                              0x10c1d
+#define regBIF_CFG_DEV0_EPF3_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CAP2                                                              0x10c22
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2                                                             0x10c23
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_STATUS2                                                           0x10c23
+#define regBIF_CFG_DEV0_EPF3_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CAP2                                                                0x10c24
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CNTL2                                                               0x10c25
+#define regBIF_CFG_DEV0_EPF3_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF3_0_LINK_STATUS2                                                             0x10c25
+#define regBIF_CFG_DEV0_EPF3_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_CAP_LIST                                                             0x10c28
+#define regBIF_CFG_DEV0_EPF3_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL                                                             0x10c28
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_LO                                                          0x10c29
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_HI                                                          0x10c2a
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA                                                             0x10c2a
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA                                                         0x10c2a
+#define regBIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MASK                                                                 0x10c2b
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA_64                                                          0x10c2b
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA_64                                                      0x10c2b
+#define regBIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MASK_64                                                              0x10c2c
+#define regBIF_CFG_DEV0_EPF3_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_PENDING                                                              0x10c2c
+#define regBIF_CFG_DEV0_EPF3_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF3_0_MSI_PENDING_64                                                           0x10c2d
+#define regBIF_CFG_DEV0_EPF3_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_CAP_LIST                                                            0x10c30
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL                                                            0x10c30
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_TABLE                                                               0x10c31
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_PBA                                                                 0x10c32
+#define regBIF_CFG_DEV0_EPF3_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x10c40
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x10c41
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC1                                                    0x10c42
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC2                                                    0x10c43
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x10d81
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS                                                   0x10d82
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK                                                     0x10d83
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x10d84
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS                                                     0x10d85
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK                                                       0x10d86
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x10d87
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG0                                                            0x10d88
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG1                                                            0x10d89
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG2                                                            0x10d8a
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG3                                                            0x10d8b
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG0                                                     0x10d8f
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG1                                                     0x10d90
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG2                                                     0x10d91
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG3                                                     0x10d92
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST                                                    0x10db7
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP                                                             0x10db8
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL                                                            0x10db8
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST                                                    0x10dc7
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_RTR_DATA1                                                                0x10dc8
+#define regBIF_CFG_DEV0_EPF3_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_RTR_DATA2                                                                0x10dc9
+#define regBIF_CFG_DEV0_EPF3_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x10dca
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x10dcb
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA                                                     0x10dcc
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_CAP                                                      0x10dcd
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST                                                    0x10dce
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP                                                             0x10dcf
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_LATENCY_INDICATOR                                               0x10dd0
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_STATUS                                                          0x10dd1
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_CNTL                                                            0x10dd1
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x10dd2
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x10dd2
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x10dd2
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x10dd2
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x10dd3
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x10dd3
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x10dd3
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x10dd3
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST                                                    0x10ddc
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP                                                             0x10ddd
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL                                                            0x10ddd
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST                                                    0x10dde
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CAP                                                            0x10ddf
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL                                                           0x10de0
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CAP                                                            0x10de1
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL                                                           0x10de2
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CAP                                                            0x10de3
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL                                                           0x10de4
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CAP                                                            0x10de5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL                                                           0x10de6
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CAP                                                            0x10de7
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL                                                           0x10de8
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CAP                                                            0x10de9
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL                                                           0x10dea
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST                                                  0x10df1
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP                                                           0x10df2
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL                                                          0x10df2
+#define regBIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf4_bifcfgdecp
+// base address: 0x10144000
+#define regBIF_CFG_DEV0_EPF4_0_VENDOR_ID                                                                0x11000
+#define regBIF_CFG_DEV0_EPF4_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_ID                                                                0x11000
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_COMMAND                                                                  0x11001
+#define regBIF_CFG_DEV0_EPF4_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF4_0_STATUS                                                                   0x11001
+#define regBIF_CFG_DEV0_EPF4_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF4_0_REVISION_ID                                                              0x11002
+#define regBIF_CFG_DEV0_EPF4_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_PROG_INTERFACE                                                           0x11002
+#define regBIF_CFG_DEV0_EPF4_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_SUB_CLASS                                                                0x11002
+#define regBIF_CFG_DEV0_EPF4_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_BASE_CLASS                                                               0x11002
+#define regBIF_CFG_DEV0_EPF4_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF4_0_CACHE_LINE                                                               0x11003
+#define regBIF_CFG_DEV0_EPF4_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF4_0_LATENCY                                                                  0x11003
+#define regBIF_CFG_DEV0_EPF4_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF4_0_HEADER                                                                   0x11003
+#define regBIF_CFG_DEV0_EPF4_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF4_0_BIST                                                                     0x11003
+#define regBIF_CFG_DEV0_EPF4_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_1                                                              0x11004
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_2                                                              0x11005
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_3                                                              0x11006
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_4                                                              0x11007
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_5                                                              0x11008
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_6                                                              0x11009
+#define regBIF_CFG_DEV0_EPF4_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_CARDBUS_CIS_PTR                                                          0x1100a
+#define regBIF_CFG_DEV0_EPF4_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF4_0_ADAPTER_ID                                                               0x1100b
+#define regBIF_CFG_DEV0_EPF4_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR                                                            0x1100c
+#define regBIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_CAP_PTR                                                                  0x1100d
+#define regBIF_CFG_DEV0_EPF4_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF4_0_INTERRUPT_LINE                                                           0x1100f
+#define regBIF_CFG_DEV0_EPF4_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_INTERRUPT_PIN                                                            0x1100f
+#define regBIF_CFG_DEV0_EPF4_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_MIN_GRANT                                                                0x1100f
+#define regBIF_CFG_DEV0_EPF4_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_MAX_LATENCY                                                              0x1100f
+#define regBIF_CFG_DEV0_EPF4_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST                                                          0x11012
+#define regBIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF4_0_ADAPTER_ID_W                                                             0x11013
+#define regBIF_CFG_DEV0_EPF4_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_PMI_CAP_LIST                                                             0x11014
+#define regBIF_CFG_DEV0_EPF4_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_PMI_CAP                                                                  0x11014
+#define regBIF_CFG_DEV0_EPF4_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL                                                          0x11015
+#define regBIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF4_0_SBRN                                                                     0x11018
+#define regBIF_CFG_DEV0_EPF4_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF4_0_FLADJ                                                                    0x11018
+#define regBIF_CFG_DEV0_EPF4_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF4_0_DBESL_DBESLD                                                             0x11018
+#define regBIF_CFG_DEV0_EPF4_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CAP_LIST                                                            0x11019
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CAP                                                                 0x11019
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CAP                                                               0x1101a
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CNTL                                                              0x1101b
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_STATUS                                                            0x1101b
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CAP                                                                 0x1101c
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CNTL                                                                0x1101d
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_LINK_STATUS                                                              0x1101d
+#define regBIF_CFG_DEV0_EPF4_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CAP2                                                              0x11022
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2                                                             0x11023
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_STATUS2                                                           0x11023
+#define regBIF_CFG_DEV0_EPF4_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CAP2                                                                0x11024
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CNTL2                                                               0x11025
+#define regBIF_CFG_DEV0_EPF4_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF4_0_LINK_STATUS2                                                             0x11025
+#define regBIF_CFG_DEV0_EPF4_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_CAP_LIST                                                             0x11028
+#define regBIF_CFG_DEV0_EPF4_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL                                                             0x11028
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_LO                                                          0x11029
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_HI                                                          0x1102a
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA                                                             0x1102a
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA                                                         0x1102a
+#define regBIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MASK                                                                 0x1102b
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA_64                                                          0x1102b
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA_64                                                      0x1102b
+#define regBIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MASK_64                                                              0x1102c
+#define regBIF_CFG_DEV0_EPF4_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_PENDING                                                              0x1102c
+#define regBIF_CFG_DEV0_EPF4_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF4_0_MSI_PENDING_64                                                           0x1102d
+#define regBIF_CFG_DEV0_EPF4_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_CAP_LIST                                                            0x11030
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL                                                            0x11030
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_TABLE                                                               0x11031
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_PBA                                                                 0x11032
+#define regBIF_CFG_DEV0_EPF4_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x11040
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x11041
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC1                                                    0x11042
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC2                                                    0x11043
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x11181
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS                                                   0x11182
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK                                                     0x11183
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x11184
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS                                                     0x11185
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK                                                       0x11186
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x11187
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG0                                                            0x11188
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG1                                                            0x11189
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG2                                                            0x1118a
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG3                                                            0x1118b
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG0                                                     0x1118f
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG1                                                     0x11190
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG2                                                     0x11191
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG3                                                     0x11192
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST                                                    0x111b7
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP                                                             0x111b8
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL                                                            0x111b8
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST                                                    0x111c7
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_RTR_DATA1                                                                0x111c8
+#define regBIF_CFG_DEV0_EPF4_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_RTR_DATA2                                                                0x111c9
+#define regBIF_CFG_DEV0_EPF4_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x111ca
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x111cb
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA                                                     0x111cc
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_CAP                                                      0x111cd
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST                                                    0x111ce
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP                                                             0x111cf
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_LATENCY_INDICATOR                                               0x111d0
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_STATUS                                                          0x111d1
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_CNTL                                                            0x111d1
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x111d2
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x111d2
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x111d2
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x111d2
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x111d3
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x111d3
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x111d3
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x111d3
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST                                                    0x111dc
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP                                                             0x111dd
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL                                                            0x111dd
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST                                                    0x111de
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CAP                                                            0x111df
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL                                                           0x111e0
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CAP                                                            0x111e1
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL                                                           0x111e2
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CAP                                                            0x111e3
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL                                                           0x111e4
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CAP                                                            0x111e5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL                                                           0x111e6
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CAP                                                            0x111e7
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL                                                           0x111e8
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CAP                                                            0x111e9
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL                                                           0x111ea
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST                                                  0x111f1
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP                                                           0x111f2
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL                                                          0x111f2
+#define regBIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf5_bifcfgdecp
+// base address: 0x10145000
+#define regBIF_CFG_DEV0_EPF5_0_VENDOR_ID                                                                0x11400
+#define regBIF_CFG_DEV0_EPF5_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_ID                                                                0x11400
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_COMMAND                                                                  0x11401
+#define regBIF_CFG_DEV0_EPF5_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF5_0_STATUS                                                                   0x11401
+#define regBIF_CFG_DEV0_EPF5_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF5_0_REVISION_ID                                                              0x11402
+#define regBIF_CFG_DEV0_EPF5_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_PROG_INTERFACE                                                           0x11402
+#define regBIF_CFG_DEV0_EPF5_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_SUB_CLASS                                                                0x11402
+#define regBIF_CFG_DEV0_EPF5_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_BASE_CLASS                                                               0x11402
+#define regBIF_CFG_DEV0_EPF5_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF5_0_CACHE_LINE                                                               0x11403
+#define regBIF_CFG_DEV0_EPF5_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF5_0_LATENCY                                                                  0x11403
+#define regBIF_CFG_DEV0_EPF5_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF5_0_HEADER                                                                   0x11403
+#define regBIF_CFG_DEV0_EPF5_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF5_0_BIST                                                                     0x11403
+#define regBIF_CFG_DEV0_EPF5_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_1                                                              0x11404
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_2                                                              0x11405
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_3                                                              0x11406
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_4                                                              0x11407
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_5                                                              0x11408
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_6                                                              0x11409
+#define regBIF_CFG_DEV0_EPF5_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_CARDBUS_CIS_PTR                                                          0x1140a
+#define regBIF_CFG_DEV0_EPF5_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF5_0_ADAPTER_ID                                                               0x1140b
+#define regBIF_CFG_DEV0_EPF5_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR                                                            0x1140c
+#define regBIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_CAP_PTR                                                                  0x1140d
+#define regBIF_CFG_DEV0_EPF5_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF5_0_INTERRUPT_LINE                                                           0x1140f
+#define regBIF_CFG_DEV0_EPF5_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_INTERRUPT_PIN                                                            0x1140f
+#define regBIF_CFG_DEV0_EPF5_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_MIN_GRANT                                                                0x1140f
+#define regBIF_CFG_DEV0_EPF5_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_MAX_LATENCY                                                              0x1140f
+#define regBIF_CFG_DEV0_EPF5_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST                                                          0x11412
+#define regBIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF5_0_ADAPTER_ID_W                                                             0x11413
+#define regBIF_CFG_DEV0_EPF5_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_PMI_CAP_LIST                                                             0x11414
+#define regBIF_CFG_DEV0_EPF5_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_PMI_CAP                                                                  0x11414
+#define regBIF_CFG_DEV0_EPF5_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL                                                          0x11415
+#define regBIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF5_0_SBRN                                                                     0x11418
+#define regBIF_CFG_DEV0_EPF5_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF5_0_FLADJ                                                                    0x11418
+#define regBIF_CFG_DEV0_EPF5_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF5_0_DBESL_DBESLD                                                             0x11418
+#define regBIF_CFG_DEV0_EPF5_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CAP_LIST                                                            0x11419
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CAP                                                                 0x11419
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CAP                                                               0x1141a
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CNTL                                                              0x1141b
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_STATUS                                                            0x1141b
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CAP                                                                 0x1141c
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CNTL                                                                0x1141d
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_LINK_STATUS                                                              0x1141d
+#define regBIF_CFG_DEV0_EPF5_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CAP2                                                              0x11422
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2                                                             0x11423
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_STATUS2                                                           0x11423
+#define regBIF_CFG_DEV0_EPF5_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CAP2                                                                0x11424
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CNTL2                                                               0x11425
+#define regBIF_CFG_DEV0_EPF5_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF5_0_LINK_STATUS2                                                             0x11425
+#define regBIF_CFG_DEV0_EPF5_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_CAP_LIST                                                             0x11428
+#define regBIF_CFG_DEV0_EPF5_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL                                                             0x11428
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_LO                                                          0x11429
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_HI                                                          0x1142a
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA                                                             0x1142a
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA                                                         0x1142a
+#define regBIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MASK                                                                 0x1142b
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA_64                                                          0x1142b
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA_64                                                      0x1142b
+#define regBIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MASK_64                                                              0x1142c
+#define regBIF_CFG_DEV0_EPF5_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_PENDING                                                              0x1142c
+#define regBIF_CFG_DEV0_EPF5_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF5_0_MSI_PENDING_64                                                           0x1142d
+#define regBIF_CFG_DEV0_EPF5_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_CAP_LIST                                                            0x11430
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL                                                            0x11430
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_TABLE                                                               0x11431
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_PBA                                                                 0x11432
+#define regBIF_CFG_DEV0_EPF5_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x11440
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x11441
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC1                                                    0x11442
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC2                                                    0x11443
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x11581
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS                                                   0x11582
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK                                                     0x11583
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x11584
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS                                                     0x11585
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK                                                       0x11586
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x11587
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG0                                                            0x11588
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG1                                                            0x11589
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG2                                                            0x1158a
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG3                                                            0x1158b
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG0                                                     0x1158f
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG1                                                     0x11590
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG2                                                     0x11591
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG3                                                     0x11592
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST                                                    0x115b7
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP                                                             0x115b8
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL                                                            0x115b8
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST                                                    0x115c7
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_RTR_DATA1                                                                0x115c8
+#define regBIF_CFG_DEV0_EPF5_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_RTR_DATA2                                                                0x115c9
+#define regBIF_CFG_DEV0_EPF5_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x115ca
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x115cb
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA                                                     0x115cc
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_CAP                                                      0x115cd
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST                                                    0x115ce
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP                                                             0x115cf
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_LATENCY_INDICATOR                                               0x115d0
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_STATUS                                                          0x115d1
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_CNTL                                                            0x115d1
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x115d2
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x115d2
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x115d2
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x115d2
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x115d3
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x115d3
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x115d3
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x115d3
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST                                                    0x115dc
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP                                                             0x115dd
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL                                                            0x115dd
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST                                                    0x115de
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CAP                                                            0x115df
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL                                                           0x115e0
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CAP                                                            0x115e1
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL                                                           0x115e2
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CAP                                                            0x115e3
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL                                                           0x115e4
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CAP                                                            0x115e5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL                                                           0x115e6
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CAP                                                            0x115e7
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL                                                           0x115e8
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CAP                                                            0x115e9
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL                                                           0x115ea
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST                                                  0x115f1
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP                                                           0x115f2
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL                                                          0x115f2
+#define regBIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf6_bifcfgdecp
+// base address: 0x10146000
+#define regBIF_CFG_DEV0_EPF6_0_VENDOR_ID                                                                0x11800
+#define regBIF_CFG_DEV0_EPF6_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_ID                                                                0x11800
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_COMMAND                                                                  0x11801
+#define regBIF_CFG_DEV0_EPF6_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF6_0_STATUS                                                                   0x11801
+#define regBIF_CFG_DEV0_EPF6_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF6_0_REVISION_ID                                                              0x11802
+#define regBIF_CFG_DEV0_EPF6_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_PROG_INTERFACE                                                           0x11802
+#define regBIF_CFG_DEV0_EPF6_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_SUB_CLASS                                                                0x11802
+#define regBIF_CFG_DEV0_EPF6_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_BASE_CLASS                                                               0x11802
+#define regBIF_CFG_DEV0_EPF6_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF6_0_CACHE_LINE                                                               0x11803
+#define regBIF_CFG_DEV0_EPF6_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF6_0_LATENCY                                                                  0x11803
+#define regBIF_CFG_DEV0_EPF6_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF6_0_HEADER                                                                   0x11803
+#define regBIF_CFG_DEV0_EPF6_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF6_0_BIST                                                                     0x11803
+#define regBIF_CFG_DEV0_EPF6_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_1                                                              0x11804
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_2                                                              0x11805
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_3                                                              0x11806
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_4                                                              0x11807
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_5                                                              0x11808
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_6                                                              0x11809
+#define regBIF_CFG_DEV0_EPF6_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_CARDBUS_CIS_PTR                                                          0x1180a
+#define regBIF_CFG_DEV0_EPF6_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF6_0_ADAPTER_ID                                                               0x1180b
+#define regBIF_CFG_DEV0_EPF6_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR                                                            0x1180c
+#define regBIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_CAP_PTR                                                                  0x1180d
+#define regBIF_CFG_DEV0_EPF6_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF6_0_INTERRUPT_LINE                                                           0x1180f
+#define regBIF_CFG_DEV0_EPF6_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_INTERRUPT_PIN                                                            0x1180f
+#define regBIF_CFG_DEV0_EPF6_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_MIN_GRANT                                                                0x1180f
+#define regBIF_CFG_DEV0_EPF6_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_MAX_LATENCY                                                              0x1180f
+#define regBIF_CFG_DEV0_EPF6_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST                                                          0x11812
+#define regBIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF6_0_ADAPTER_ID_W                                                             0x11813
+#define regBIF_CFG_DEV0_EPF6_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_PMI_CAP_LIST                                                             0x11814
+#define regBIF_CFG_DEV0_EPF6_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_PMI_CAP                                                                  0x11814
+#define regBIF_CFG_DEV0_EPF6_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL                                                          0x11815
+#define regBIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF6_0_SBRN                                                                     0x11818
+#define regBIF_CFG_DEV0_EPF6_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF6_0_FLADJ                                                                    0x11818
+#define regBIF_CFG_DEV0_EPF6_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF6_0_DBESL_DBESLD                                                             0x11818
+#define regBIF_CFG_DEV0_EPF6_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CAP_LIST                                                            0x11819
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CAP                                                                 0x11819
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CAP                                                               0x1181a
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CNTL                                                              0x1181b
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_STATUS                                                            0x1181b
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CAP                                                                 0x1181c
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CNTL                                                                0x1181d
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_LINK_STATUS                                                              0x1181d
+#define regBIF_CFG_DEV0_EPF6_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CAP2                                                              0x11822
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2                                                             0x11823
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_STATUS2                                                           0x11823
+#define regBIF_CFG_DEV0_EPF6_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CAP2                                                                0x11824
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CNTL2                                                               0x11825
+#define regBIF_CFG_DEV0_EPF6_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF6_0_LINK_STATUS2                                                             0x11825
+#define regBIF_CFG_DEV0_EPF6_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_CAP_LIST                                                             0x11828
+#define regBIF_CFG_DEV0_EPF6_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL                                                             0x11828
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_LO                                                          0x11829
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_HI                                                          0x1182a
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA                                                             0x1182a
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA                                                         0x1182a
+#define regBIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MASK                                                                 0x1182b
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA_64                                                          0x1182b
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA_64                                                      0x1182b
+#define regBIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MASK_64                                                              0x1182c
+#define regBIF_CFG_DEV0_EPF6_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_PENDING                                                              0x1182c
+#define regBIF_CFG_DEV0_EPF6_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF6_0_MSI_PENDING_64                                                           0x1182d
+#define regBIF_CFG_DEV0_EPF6_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_CAP_LIST                                                            0x11830
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL                                                            0x11830
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_TABLE                                                               0x11831
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_PBA                                                                 0x11832
+#define regBIF_CFG_DEV0_EPF6_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x11840
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x11841
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC1                                                    0x11842
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC2                                                    0x11843
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x11981
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS                                                   0x11982
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK                                                     0x11983
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x11984
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS                                                     0x11985
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK                                                       0x11986
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x11987
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG0                                                            0x11988
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG1                                                            0x11989
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG2                                                            0x1198a
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG3                                                            0x1198b
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG0                                                     0x1198f
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG1                                                     0x11990
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG2                                                     0x11991
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG3                                                     0x11992
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST                                                    0x119b7
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP                                                             0x119b8
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL                                                            0x119b8
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST                                                    0x119c7
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_RTR_DATA1                                                                0x119c8
+#define regBIF_CFG_DEV0_EPF6_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_RTR_DATA2                                                                0x119c9
+#define regBIF_CFG_DEV0_EPF6_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x119ca
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x119cb
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA                                                     0x119cc
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_CAP                                                      0x119cd
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST                                                    0x119ce
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP                                                             0x119cf
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_LATENCY_INDICATOR                                               0x119d0
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_STATUS                                                          0x119d1
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_CNTL                                                            0x119d1
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x119d2
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x119d2
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x119d2
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x119d2
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x119d3
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x119d3
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x119d3
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x119d3
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST                                                    0x119dc
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP                                                             0x119dd
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL                                                            0x119dd
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST                                                    0x119de
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CAP                                                            0x119df
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL                                                           0x119e0
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CAP                                                            0x119e1
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL                                                           0x119e2
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CAP                                                            0x119e3
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL                                                           0x119e4
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CAP                                                            0x119e5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL                                                           0x119e6
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CAP                                                            0x119e7
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL                                                           0x119e8
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CAP                                                            0x119e9
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL                                                           0x119ea
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST                                                  0x119f1
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP                                                           0x119f2
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL                                                          0x119f2
+#define regBIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf7_bifcfgdecp
+// base address: 0x10147000
+#define regBIF_CFG_DEV0_EPF7_0_VENDOR_ID                                                                0x11c00
+#define regBIF_CFG_DEV0_EPF7_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_ID                                                                0x11c00
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_COMMAND                                                                  0x11c01
+#define regBIF_CFG_DEV0_EPF7_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF7_0_STATUS                                                                   0x11c01
+#define regBIF_CFG_DEV0_EPF7_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF7_0_REVISION_ID                                                              0x11c02
+#define regBIF_CFG_DEV0_EPF7_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_PROG_INTERFACE                                                           0x11c02
+#define regBIF_CFG_DEV0_EPF7_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_SUB_CLASS                                                                0x11c02
+#define regBIF_CFG_DEV0_EPF7_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_BASE_CLASS                                                               0x11c02
+#define regBIF_CFG_DEV0_EPF7_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF7_0_CACHE_LINE                                                               0x11c03
+#define regBIF_CFG_DEV0_EPF7_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF7_0_LATENCY                                                                  0x11c03
+#define regBIF_CFG_DEV0_EPF7_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF7_0_HEADER                                                                   0x11c03
+#define regBIF_CFG_DEV0_EPF7_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF7_0_BIST                                                                     0x11c03
+#define regBIF_CFG_DEV0_EPF7_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_1                                                              0x11c04
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_2                                                              0x11c05
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_3                                                              0x11c06
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_4                                                              0x11c07
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_5                                                              0x11c08
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_6                                                              0x11c09
+#define regBIF_CFG_DEV0_EPF7_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_CARDBUS_CIS_PTR                                                          0x11c0a
+#define regBIF_CFG_DEV0_EPF7_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF7_0_ADAPTER_ID                                                               0x11c0b
+#define regBIF_CFG_DEV0_EPF7_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR                                                            0x11c0c
+#define regBIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_CAP_PTR                                                                  0x11c0d
+#define regBIF_CFG_DEV0_EPF7_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF7_0_INTERRUPT_LINE                                                           0x11c0f
+#define regBIF_CFG_DEV0_EPF7_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_INTERRUPT_PIN                                                            0x11c0f
+#define regBIF_CFG_DEV0_EPF7_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_MIN_GRANT                                                                0x11c0f
+#define regBIF_CFG_DEV0_EPF7_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_MAX_LATENCY                                                              0x11c0f
+#define regBIF_CFG_DEV0_EPF7_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST                                                          0x11c12
+#define regBIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF7_0_ADAPTER_ID_W                                                             0x11c13
+#define regBIF_CFG_DEV0_EPF7_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_PMI_CAP_LIST                                                             0x11c14
+#define regBIF_CFG_DEV0_EPF7_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_PMI_CAP                                                                  0x11c14
+#define regBIF_CFG_DEV0_EPF7_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL                                                          0x11c15
+#define regBIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF7_0_SBRN                                                                     0x11c18
+#define regBIF_CFG_DEV0_EPF7_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF7_0_FLADJ                                                                    0x11c18
+#define regBIF_CFG_DEV0_EPF7_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF7_0_DBESL_DBESLD                                                             0x11c18
+#define regBIF_CFG_DEV0_EPF7_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CAP_LIST                                                            0x11c19
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CAP                                                                 0x11c19
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CAP                                                               0x11c1a
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CNTL                                                              0x11c1b
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_STATUS                                                            0x11c1b
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CAP                                                                 0x11c1c
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CNTL                                                                0x11c1d
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_LINK_STATUS                                                              0x11c1d
+#define regBIF_CFG_DEV0_EPF7_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CAP2                                                              0x11c22
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2                                                             0x11c23
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_STATUS2                                                           0x11c23
+#define regBIF_CFG_DEV0_EPF7_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CAP2                                                                0x11c24
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CNTL2                                                               0x11c25
+#define regBIF_CFG_DEV0_EPF7_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF7_0_LINK_STATUS2                                                             0x11c25
+#define regBIF_CFG_DEV0_EPF7_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_CAP_LIST                                                             0x11c28
+#define regBIF_CFG_DEV0_EPF7_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL                                                             0x11c28
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_LO                                                          0x11c29
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_HI                                                          0x11c2a
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA                                                             0x11c2a
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA                                                         0x11c2a
+#define regBIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MASK                                                                 0x11c2b
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA_64                                                          0x11c2b
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA_64                                                      0x11c2b
+#define regBIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MASK_64                                                              0x11c2c
+#define regBIF_CFG_DEV0_EPF7_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_PENDING                                                              0x11c2c
+#define regBIF_CFG_DEV0_EPF7_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF7_0_MSI_PENDING_64                                                           0x11c2d
+#define regBIF_CFG_DEV0_EPF7_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_CAP_LIST                                                            0x11c30
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL                                                            0x11c30
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_TABLE                                                               0x11c31
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_PBA                                                                 0x11c32
+#define regBIF_CFG_DEV0_EPF7_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x11c40
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x11c41
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC1                                                    0x11c42
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC2                                                    0x11c43
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x11d81
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS                                                   0x11d82
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK                                                     0x11d83
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x11d84
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS                                                     0x11d85
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK                                                       0x11d86
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x11d87
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG0                                                            0x11d88
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG1                                                            0x11d89
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG2                                                            0x11d8a
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG3                                                            0x11d8b
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG0                                                     0x11d8f
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG1                                                     0x11d90
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG2                                                     0x11d91
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG3                                                     0x11d92
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST                                                    0x11db7
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP                                                             0x11db8
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL                                                            0x11db8
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST                                                    0x11dc7
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_RTR_DATA1                                                                0x11dc8
+#define regBIF_CFG_DEV0_EPF7_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_RTR_DATA2                                                                0x11dc9
+#define regBIF_CFG_DEV0_EPF7_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x11dca
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x11dcb
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA                                                     0x11dcc
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_CAP                                                      0x11dcd
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST                                                    0x11dce
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP                                                             0x11dcf
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_LATENCY_INDICATOR                                               0x11dd0
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_STATUS                                                          0x11dd1
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_CNTL                                                            0x11dd1
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x11dd2
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x11dd2
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x11dd2
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x11dd2
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x11dd3
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x11dd3
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x11dd3
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x11dd3
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST                                                    0x11ddc
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP                                                             0x11ddd
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL                                                            0x11ddd
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST                                                    0x11dde
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CAP                                                            0x11ddf
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL                                                           0x11de0
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CAP                                                            0x11de1
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL                                                           0x11de2
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CAP                                                            0x11de3
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL                                                           0x11de4
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CAP                                                            0x11de5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL                                                           0x11de6
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CAP                                                            0x11de7
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL                                                           0x11de8
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CAP                                                            0x11de9
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL                                                           0x11dea
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST                                                  0x11df1
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP                                                           0x11df2
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL                                                          0x11df2
+#define regBIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev1_rc_bifcfgdecp
+// base address: 0x10101000
+#define regBIF_CFG_DEV1_RC0_VENDOR_ID                                                                   0x0400
+#define regBIF_CFG_DEV1_RC0_VENDOR_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_DEVICE_ID                                                                   0x0400
+#define regBIF_CFG_DEV1_RC0_DEVICE_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_COMMAND                                                                     0x0401
+#define regBIF_CFG_DEV1_RC0_COMMAND_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_RC0_STATUS                                                                      0x0401
+#define regBIF_CFG_DEV1_RC0_STATUS_BASE_IDX                                                             5
+#define regBIF_CFG_DEV1_RC0_REVISION_ID                                                                 0x0402
+#define regBIF_CFG_DEV1_RC0_REVISION_ID_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_PROG_INTERFACE                                                              0x0402
+#define regBIF_CFG_DEV1_RC0_PROG_INTERFACE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_RC0_SUB_CLASS                                                                   0x0402
+#define regBIF_CFG_DEV1_RC0_SUB_CLASS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_BASE_CLASS                                                                  0x0402
+#define regBIF_CFG_DEV1_RC0_BASE_CLASS_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_RC0_CACHE_LINE                                                                  0x0403
+#define regBIF_CFG_DEV1_RC0_CACHE_LINE_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_RC0_LATENCY                                                                     0x0403
+#define regBIF_CFG_DEV1_RC0_LATENCY_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_RC0_HEADER                                                                      0x0403
+#define regBIF_CFG_DEV1_RC0_HEADER_BASE_IDX                                                             5
+#define regBIF_CFG_DEV1_RC0_BIST                                                                        0x0403
+#define regBIF_CFG_DEV1_RC0_BIST_BASE_IDX                                                               5
+#define regBIF_CFG_DEV1_RC0_BASE_ADDR_1                                                                 0x0404
+#define regBIF_CFG_DEV1_RC0_BASE_ADDR_1_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_BASE_ADDR_2                                                                 0x0405
+#define regBIF_CFG_DEV1_RC0_BASE_ADDR_2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY                                                      0x0406
+#define regBIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_RC0_IO_BASE_LIMIT                                                               0x0407
+#define regBIF_CFG_DEV1_RC0_IO_BASE_LIMIT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_SECONDARY_STATUS                                                            0x0407
+#define regBIF_CFG_DEV1_RC0_SECONDARY_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_RC0_MEM_BASE_LIMIT                                                              0x0408
+#define regBIF_CFG_DEV1_RC0_MEM_BASE_LIMIT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_RC0_PREF_BASE_LIMIT                                                             0x0409
+#define regBIF_CFG_DEV1_RC0_PREF_BASE_LIMIT_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_PREF_BASE_UPPER                                                             0x040a
+#define regBIF_CFG_DEV1_RC0_PREF_BASE_UPPER_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_PREF_LIMIT_UPPER                                                            0x040b
+#define regBIF_CFG_DEV1_RC0_PREF_LIMIT_UPPER_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_RC0_IO_BASE_LIMIT_HI                                                            0x040c
+#define regBIF_CFG_DEV1_RC0_IO_BASE_LIMIT_HI_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_RC0_CAP_PTR                                                                     0x040d
+#define regBIF_CFG_DEV1_RC0_CAP_PTR_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_RC0_ROM_BASE_ADDR                                                               0x040e
+#define regBIF_CFG_DEV1_RC0_ROM_BASE_ADDR_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_INTERRUPT_LINE                                                              0x040f
+#define regBIF_CFG_DEV1_RC0_INTERRUPT_LINE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_RC0_INTERRUPT_PIN                                                               0x040f
+#define regBIF_CFG_DEV1_RC0_INTERRUPT_PIN_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL                                                             0x040f
+#define regBIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_EXT_BRIDGE_CNTL                                                             0x0410
+#define regBIF_CFG_DEV1_RC0_EXT_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_PMI_CAP_LIST                                                                0x0414
+#define regBIF_CFG_DEV1_RC0_PMI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_PMI_CAP                                                                     0x0414
+#define regBIF_CFG_DEV1_RC0_PMI_CAP_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_RC0_PMI_STATUS_CNTL                                                             0x0415
+#define regBIF_CFG_DEV1_RC0_PMI_STATUS_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_PCIE_CAP_LIST                                                               0x0416
+#define regBIF_CFG_DEV1_RC0_PCIE_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_CAP                                                                    0x0416
+#define regBIF_CFG_DEV1_RC0_PCIE_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV1_RC0_DEVICE_CAP                                                                  0x0417
+#define regBIF_CFG_DEV1_RC0_DEVICE_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_RC0_DEVICE_CNTL                                                                 0x0418
+#define regBIF_CFG_DEV1_RC0_DEVICE_CNTL_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_DEVICE_STATUS                                                               0x0418
+#define regBIF_CFG_DEV1_RC0_DEVICE_STATUS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_LINK_CAP                                                                    0x0419
+#define regBIF_CFG_DEV1_RC0_LINK_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV1_RC0_LINK_CNTL                                                                   0x041a
+#define regBIF_CFG_DEV1_RC0_LINK_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_LINK_STATUS                                                                 0x041a
+#define regBIF_CFG_DEV1_RC0_LINK_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_SLOT_CAP                                                                    0x041b
+#define regBIF_CFG_DEV1_RC0_SLOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV1_RC0_SLOT_CNTL                                                                   0x041c
+#define regBIF_CFG_DEV1_RC0_SLOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_SLOT_STATUS                                                                 0x041c
+#define regBIF_CFG_DEV1_RC0_SLOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_ROOT_CNTL                                                                   0x041d
+#define regBIF_CFG_DEV1_RC0_ROOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_ROOT_CAP                                                                    0x041d
+#define regBIF_CFG_DEV1_RC0_ROOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV1_RC0_ROOT_STATUS                                                                 0x041e
+#define regBIF_CFG_DEV1_RC0_ROOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_DEVICE_CAP2                                                                 0x041f
+#define regBIF_CFG_DEV1_RC0_DEVICE_CAP2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_RC0_DEVICE_CNTL2                                                                0x0420
+#define regBIF_CFG_DEV1_RC0_DEVICE_CNTL2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_DEVICE_STATUS2                                                              0x0420
+#define regBIF_CFG_DEV1_RC0_DEVICE_STATUS2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_RC0_LINK_CAP2                                                                   0x0421
+#define regBIF_CFG_DEV1_RC0_LINK_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_LINK_CNTL2                                                                  0x0422
+#define regBIF_CFG_DEV1_RC0_LINK_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_RC0_LINK_STATUS2                                                                0x0422
+#define regBIF_CFG_DEV1_RC0_LINK_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_SLOT_CAP2                                                                   0x0423
+#define regBIF_CFG_DEV1_RC0_SLOT_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_SLOT_CNTL2                                                                  0x0424
+#define regBIF_CFG_DEV1_RC0_SLOT_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_RC0_SLOT_STATUS2                                                                0x0424
+#define regBIF_CFG_DEV1_RC0_SLOT_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_MSI_CAP_LIST                                                                0x0428
+#define regBIF_CFG_DEV1_RC0_MSI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_CNTL                                                                0x0428
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_ADDR_LO                                                             0x0429
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_ADDR_LO_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_ADDR_HI                                                             0x042a
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_ADDR_HI_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_DATA                                                                0x042a
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_DATA_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA                                                            0x042a
+#define regBIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_DATA_64                                                             0x042b
+#define regBIF_CFG_DEV1_RC0_MSI_MSG_DATA_64_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA_64                                                         0x042b
+#define regBIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV1_RC0_SSID_CAP_LIST                                                               0x0430
+#define regBIF_CFG_DEV1_RC0_SSID_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_SSID_CAP                                                                    0x0431
+#define regBIF_CFG_DEV1_RC0_SSID_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                           0x0440
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR                                                    0x0441
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC1                                                       0x0442
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC2                                                       0x0443
+#define regBIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST                                                 0x0444
+#define regBIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3                                                             0x0445
+#define regBIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_ERROR_STATUS                                                      0x0446
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL                                               0x0447
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL                                               0x0447
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL                                               0x0448
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL                                               0x0448
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL                                               0x0449
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL                                               0x0449
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL                                               0x044a
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL                                               0x044a
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL                                               0x044b
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL                                               0x044b
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL                                              0x044c
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL                                              0x044c
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL                                              0x044d
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL                                              0x044d
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL                                              0x044e
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL                                              0x044e
+#define regBIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST                                                  0x046e
+#define regBIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LINK_CAP_16GT                                                               0x046f
+#define regBIF_CFG_DEV1_RC0_LINK_CAP_16GT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_LINK_CNTL_16GT                                                              0x0470
+#define regBIF_CFG_DEV1_RC0_LINK_CNTL_16GT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_RC0_LINK_STATUS_16GT                                                            0x0471
+#define regBIF_CFG_DEV1_RC0_LINK_STATUS_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                           0x0472
+#define regBIF_CFG_DEV1_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT                                            0x0473
+#define regBIF_CFG_DEV1_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT                                            0x0474
+#define regBIF_CFG_DEV1_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_RC0_LANE_0_EQUALIZATION_CNTL_16GT                                               0x0476
+#define regBIF_CFG_DEV1_RC0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_1_EQUALIZATION_CNTL_16GT                                               0x0476
+#define regBIF_CFG_DEV1_RC0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_2_EQUALIZATION_CNTL_16GT                                               0x0476
+#define regBIF_CFG_DEV1_RC0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_3_EQUALIZATION_CNTL_16GT                                               0x0476
+#define regBIF_CFG_DEV1_RC0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_4_EQUALIZATION_CNTL_16GT                                               0x0477
+#define regBIF_CFG_DEV1_RC0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_5_EQUALIZATION_CNTL_16GT                                               0x0477
+#define regBIF_CFG_DEV1_RC0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_6_EQUALIZATION_CNTL_16GT                                               0x0477
+#define regBIF_CFG_DEV1_RC0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_7_EQUALIZATION_CNTL_16GT                                               0x0477
+#define regBIF_CFG_DEV1_RC0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_8_EQUALIZATION_CNTL_16GT                                               0x0478
+#define regBIF_CFG_DEV1_RC0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_9_EQUALIZATION_CNTL_16GT                                               0x0478
+#define regBIF_CFG_DEV1_RC0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_10_EQUALIZATION_CNTL_16GT                                              0x0478
+#define regBIF_CFG_DEV1_RC0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_LANE_11_EQUALIZATION_CNTL_16GT                                              0x0478
+#define regBIF_CFG_DEV1_RC0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_LANE_12_EQUALIZATION_CNTL_16GT                                              0x0479
+#define regBIF_CFG_DEV1_RC0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_LANE_13_EQUALIZATION_CNTL_16GT                                              0x0479
+#define regBIF_CFG_DEV1_RC0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_LANE_14_EQUALIZATION_CNTL_16GT                                              0x0479
+#define regBIF_CFG_DEV1_RC0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_LANE_15_EQUALIZATION_CNTL_16GT                                              0x0479
+#define regBIF_CFG_DEV1_RC0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST                                                 0x0494
+#define regBIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_MARGINING_PORT_CAP                                                          0x0495
+#define regBIF_CFG_DEV1_RC0_MARGINING_PORT_CAP_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_RC0_MARGINING_PORT_STATUS                                                       0x0495
+#define regBIF_CFG_DEV1_RC0_MARGINING_PORT_STATUS_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL                                                  0x0496
+#define regBIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS                                                0x0496
+#define regBIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL                                                  0x0497
+#define regBIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS                                                0x0497
+#define regBIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL                                                  0x0498
+#define regBIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS                                                0x0498
+#define regBIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL                                                  0x0499
+#define regBIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS                                                0x0499
+#define regBIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL                                                  0x049a
+#define regBIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS                                                0x049a
+#define regBIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL                                                  0x049b
+#define regBIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS                                                0x049b
+#define regBIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL                                                  0x049c
+#define regBIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS                                                0x049c
+#define regBIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL                                                  0x049d
+#define regBIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS                                                0x049d
+#define regBIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL                                                  0x049e
+#define regBIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS                                                0x049e
+#define regBIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL                                                  0x049f
+#define regBIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS                                                0x049f
+#define regBIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL                                                 0x04a0
+#define regBIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS                                               0x04a0
+#define regBIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL                                                 0x04a1
+#define regBIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS                                               0x04a1
+#define regBIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL                                                 0x04a2
+#define regBIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS                                               0x04a2
+#define regBIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL                                                 0x04a3
+#define regBIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS                                               0x04a3
+#define regBIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL                                                 0x04a4
+#define regBIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS                                               0x04a4
+#define regBIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL                                                 0x04a5
+#define regBIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS                                               0x04a5
+#define regBIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST                                                        0x0501
+#define regBIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1                                                       0x0502
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG2                                                       0x0503
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_CNTL                                                           0x0504
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_STATUS                                                         0x0504
+#define regBIF_CFG_DEV1_RC0_PCIE_PORT_VC_STATUS_BASE_IDX                                                5
+#define regBIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP                                                       0x0505
+#define regBIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL                                                      0x0506
+#define regBIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_STATUS                                                    0x0507
+#define regBIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP                                                       0x0508
+#define regBIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL                                                      0x0509
+#define regBIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_STATUS                                                    0x050a
+#define regBIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST                                                       0x050c
+#define regBIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_CAP                                                       0x050d
+#define regBIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_STATUS                                                    0x050e
+#define regBIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                               0x0581
+#define regBIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS                                                      0x0582
+#define regBIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK                                                        0x0583
+#define regBIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY                                                    0x0584
+#define regBIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS                                                        0x0585
+#define regBIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK                                                          0x0586
+#define regBIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL                                                       0x0587
+#define regBIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG0                                                               0x0588
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG0_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG1                                                               0x0589
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG1_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG2                                                               0x058a
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG3                                                               0x058b
+#define regBIF_CFG_DEV1_RC0_PCIE_HDR_LOG3_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD                                                           0x058c
+#define regBIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS                                                        0x058d
+#define regBIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_ERR_SRC_ID                                                             0x058e
+#define regBIF_CFG_DEV1_RC0_PCIE_ERR_SRC_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG0                                                        0x058f
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG1                                                        0x0590
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG2                                                        0x0591
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG3                                                        0x0592
+#define regBIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                            0x0599
+#define regBIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW1                                                     0x059a
+#define regBIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW2                                                     0x059b
+#define regBIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST                                                       0x05b7
+#define regBIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_PCIE_ACS_CAP                                                                0x05b8
+#define regBIF_CFG_DEV1_RC0_PCIE_ACS_CAP_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_RC0_PCIE_ACS_CNTL                                                               0x05b8
+#define regBIF_CFG_DEV1_RC0_PCIE_ACS_CNTL_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST                                                       0x05c7
+#define regBIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_RC0_RTR_DATA1                                                                   0x05c8
+#define regBIF_CFG_DEV1_RC0_RTR_DATA1_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_RC0_RTR_DATA2                                                                   0x05c9
+#define regBIF_CFG_DEV1_RC0_RTR_DATA2_BASE_IDX                                                          5
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf0_bifcfgdecp
+// base address: 0x10148000
+#define regBIF_CFG_DEV1_EPF0_0_VENDOR_ID                                                                0x12000
+#define regBIF_CFG_DEV1_EPF0_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_ID                                                                0x12000
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_COMMAND                                                                  0x12001
+#define regBIF_CFG_DEV1_EPF0_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF0_0_STATUS                                                                   0x12001
+#define regBIF_CFG_DEV1_EPF0_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_EPF0_0_REVISION_ID                                                              0x12002
+#define regBIF_CFG_DEV1_EPF0_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_PROG_INTERFACE                                                           0x12002
+#define regBIF_CFG_DEV1_EPF0_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_SUB_CLASS                                                                0x12002
+#define regBIF_CFG_DEV1_EPF0_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_BASE_CLASS                                                               0x12002
+#define regBIF_CFG_DEV1_EPF0_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF0_0_CACHE_LINE                                                               0x12003
+#define regBIF_CFG_DEV1_EPF0_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LATENCY                                                                  0x12003
+#define regBIF_CFG_DEV1_EPF0_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF0_0_HEADER                                                                   0x12003
+#define regBIF_CFG_DEV1_EPF0_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_EPF0_0_BIST                                                                     0x12003
+#define regBIF_CFG_DEV1_EPF0_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_1                                                              0x12004
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_2                                                              0x12005
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_3                                                              0x12006
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_4                                                              0x12007
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_5                                                              0x12008
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_6                                                              0x12009
+#define regBIF_CFG_DEV1_EPF0_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_CARDBUS_CIS_PTR                                                          0x1200a
+#define regBIF_CFG_DEV1_EPF0_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_ADAPTER_ID                                                               0x1200b
+#define regBIF_CFG_DEV1_EPF0_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR                                                            0x1200c
+#define regBIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_CAP_PTR                                                                  0x1200d
+#define regBIF_CFG_DEV1_EPF0_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF0_0_INTERRUPT_LINE                                                           0x1200f
+#define regBIF_CFG_DEV1_EPF0_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_INTERRUPT_PIN                                                            0x1200f
+#define regBIF_CFG_DEV1_EPF0_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_MIN_GRANT                                                                0x1200f
+#define regBIF_CFG_DEV1_EPF0_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_MAX_LATENCY                                                              0x1200f
+#define regBIF_CFG_DEV1_EPF0_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST                                                          0x12012
+#define regBIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_ADAPTER_ID_W                                                             0x12013
+#define regBIF_CFG_DEV1_EPF0_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PMI_CAP_LIST                                                             0x12014
+#define regBIF_CFG_DEV1_EPF0_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PMI_CAP                                                                  0x12014
+#define regBIF_CFG_DEV1_EPF0_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL                                                          0x12015
+#define regBIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_SBRN                                                                     0x12018
+#define regBIF_CFG_DEV1_EPF0_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_EPF0_0_FLADJ                                                                    0x12018
+#define regBIF_CFG_DEV1_EPF0_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV1_EPF0_0_DBESL_DBESLD                                                             0x12018
+#define regBIF_CFG_DEV1_EPF0_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CAP_LIST                                                            0x12019
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CAP                                                                 0x12019
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CAP                                                               0x1201a
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CNTL                                                              0x1201b
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_STATUS                                                            0x1201b
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CAP2                                                              0x12022
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2                                                             0x12023
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_STATUS2                                                           0x12023
+#define regBIF_CFG_DEV1_EPF0_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_CAP_LIST                                                             0x12028
+#define regBIF_CFG_DEV1_EPF0_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL                                                             0x12028
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_LO                                                          0x12029
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_HI                                                          0x1202a
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA                                                             0x1202a
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA                                                         0x1202a
+#define regBIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MASK                                                                 0x1202b
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA_64                                                          0x1202b
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA_64                                                      0x1202b
+#define regBIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MASK_64                                                              0x1202c
+#define regBIF_CFG_DEV1_EPF0_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_PENDING                                                              0x1202c
+#define regBIF_CFG_DEV1_EPF0_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_MSI_PENDING_64                                                           0x1202d
+#define regBIF_CFG_DEV1_EPF0_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_CAP_LIST                                                            0x12030
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL                                                            0x12030
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_TABLE                                                               0x12031
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_PBA                                                                 0x12032
+#define regBIF_CFG_DEV1_EPF0_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x12181
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS                                                   0x12182
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK                                                     0x12183
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x12184
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS                                                     0x12185
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK                                                       0x12186
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x12187
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG0                                                            0x12188
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG1                                                            0x12189
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG2                                                            0x1218a
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG3                                                            0x1218b
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG0                                                     0x1218f
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG1                                                     0x12190
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG2                                                     0x12191
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG3                                                     0x12192
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST                                                    0x121b7
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP                                                             0x121b8
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL                                                            0x121b8
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST                                                    0x121c7
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_RTR_DATA1                                                                0x121c8
+#define regBIF_CFG_DEV1_EPF0_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_RTR_DATA2                                                                0x121c9
+#define regBIF_CFG_DEV1_EPF0_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x121ca
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x121cb
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA                                                     0x121cc
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_CAP                                                      0x121cd
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST                                                    0x121ce
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP                                                             0x121cf
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_LATENCY_INDICATOR                                               0x121d0
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_STATUS                                                          0x121d1
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_CNTL                                                            0x121d1
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x121d2
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x121d2
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x121d2
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x121d2
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x121d3
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x121d3
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x121d3
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x121d3
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST                                                    0x121da
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP                                                             0x121db
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST                                                    0x121dc
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP                                                             0x121dd
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL                                                            0x121dd
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST                                                    0x121de
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CAP                                                            0x121df
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL                                                           0x121e0
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CAP                                                            0x121e1
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL                                                           0x121e2
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CAP                                                            0x121e3
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL                                                           0x121e4
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CAP                                                            0x121e5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL                                                           0x121e6
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CAP                                                            0x121e7
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL                                                           0x121e8
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CAP                                                            0x121e9
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL                                                           0x121ea
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST                                                  0x121f1
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP                                                           0x121f2
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL                                                          0x121f2
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf0_bifcfgpciedecp
+// base address: 0x10148000
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CAP                                                                 0x1201c
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CNTL                                                                0x1201d
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_STATUS                                                              0x1201d
+#define regBIF_CFG_DEV1_EPF0_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CAP2                                                                0x12024
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CNTL2                                                               0x12025
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_STATUS2                                                             0x12025
+#define regBIF_CFG_DEV1_EPF0_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x12040
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x12041
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC1                                                    0x12042
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC2                                                    0x12043
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST                                              0x12044
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3                                                          0x12045
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_ERROR_STATUS                                                   0x12046
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL                                            0x12047
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL                                            0x12047
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL                                            0x12048
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL                                            0x12048
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL                                            0x12049
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL                                            0x12049
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL                                            0x1204a
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL                                            0x1204a
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL                                            0x1204b
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL                                            0x1204b
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL                                           0x1204c
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL                                           0x1204c
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL                                           0x1204d
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL                                           0x1204d
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL                                           0x1204e
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL                                           0x1204e
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST                                               0x1206e
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CAP_16GT                                                            0x1206f
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CAP_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CNTL_16GT                                                           0x12070
+#define regBIF_CFG_DEV1_EPF0_0_LINK_CNTL_16GT_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT                                                         0x12071
+#define regBIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT_BASE_IDX                                                5
+#define regBIF_CFG_DEV1_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                        0x12072
+#define regBIF_CFG_DEV1_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                               5
+#define regBIF_CFG_DEV1_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT                                         0x12073
+#define regBIF_CFG_DEV1_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV1_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT                                         0x12074
+#define regBIF_CFG_DEV1_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT                                            0x12076
+#define regBIF_CFG_DEV1_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT                                            0x12076
+#define regBIF_CFG_DEV1_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT                                            0x12076
+#define regBIF_CFG_DEV1_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT                                            0x12076
+#define regBIF_CFG_DEV1_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT                                            0x12077
+#define regBIF_CFG_DEV1_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT                                            0x12077
+#define regBIF_CFG_DEV1_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT                                            0x12077
+#define regBIF_CFG_DEV1_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT                                            0x12077
+#define regBIF_CFG_DEV1_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT                                            0x12078
+#define regBIF_CFG_DEV1_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT                                            0x12078
+#define regBIF_CFG_DEV1_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT                                           0x12078
+#define regBIF_CFG_DEV1_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT                                           0x12078
+#define regBIF_CFG_DEV1_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT                                           0x12079
+#define regBIF_CFG_DEV1_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT                                           0x12079
+#define regBIF_CFG_DEV1_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT                                           0x12079
+#define regBIF_CFG_DEV1_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT                                           0x12079
+#define regBIF_CFG_DEV1_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST                                              0x12094
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_MARGINING_PORT_CAP                                                       0x12095
+#define regBIF_CFG_DEV1_EPF0_0_MARGINING_PORT_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_EPF0_0_MARGINING_PORT_STATUS                                                    0x12095
+#define regBIF_CFG_DEV1_EPF0_0_MARGINING_PORT_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL                                               0x12096
+#define regBIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS                                             0x12096
+#define regBIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL                                               0x12097
+#define regBIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS                                             0x12097
+#define regBIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL                                               0x12098
+#define regBIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS                                             0x12098
+#define regBIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL                                               0x12099
+#define regBIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS                                             0x12099
+#define regBIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL                                               0x1209a
+#define regBIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS                                             0x1209a
+#define regBIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL                                               0x1209b
+#define regBIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS                                             0x1209b
+#define regBIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL                                               0x1209c
+#define regBIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS                                             0x1209c
+#define regBIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL                                               0x1209d
+#define regBIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS                                             0x1209d
+#define regBIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL                                               0x1209e
+#define regBIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS                                             0x1209e
+#define regBIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL                                               0x1209f
+#define regBIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS                                             0x1209f
+#define regBIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL                                              0x120a0
+#define regBIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS                                            0x120a0
+#define regBIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL                                              0x120a1
+#define regBIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS                                            0x120a1
+#define regBIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL                                              0x120a2
+#define regBIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS                                            0x120a2
+#define regBIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL                                              0x120a3
+#define regBIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS                                            0x120a3
+#define regBIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL                                              0x120a4
+#define regBIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS                                            0x120a4
+#define regBIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL                                              0x120a5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS                                            0x120a5
+#define regBIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST                                                     0x12101
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1                                                    0x12102
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG2                                                    0x12103
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CNTL                                                        0x12104
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CNTL_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_STATUS                                                      0x12104
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP                                                    0x12105
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL                                                   0x12106
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_STATUS                                                 0x12107
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP                                                    0x12108
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL                                                   0x12109
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_STATUS                                                 0x1210a
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST                                                    0x1210c
+#define regBIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_CAP                                                    0x1210d
+#define regBIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_STATUS                                                 0x1210e
+#define regBIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                        5
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf1_bifcfgdecp
+// base address: 0x10149000
+#define regBIF_CFG_DEV1_EPF1_0_VENDOR_ID                                                                0x12400
+#define regBIF_CFG_DEV1_EPF1_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_ID                                                                0x12400
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_COMMAND                                                                  0x12401
+#define regBIF_CFG_DEV1_EPF1_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF1_0_STATUS                                                                   0x12401
+#define regBIF_CFG_DEV1_EPF1_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_EPF1_0_REVISION_ID                                                              0x12402
+#define regBIF_CFG_DEV1_EPF1_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_PROG_INTERFACE                                                           0x12402
+#define regBIF_CFG_DEV1_EPF1_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_SUB_CLASS                                                                0x12402
+#define regBIF_CFG_DEV1_EPF1_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_BASE_CLASS                                                               0x12402
+#define regBIF_CFG_DEV1_EPF1_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF1_0_CACHE_LINE                                                               0x12403
+#define regBIF_CFG_DEV1_EPF1_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF1_0_LATENCY                                                                  0x12403
+#define regBIF_CFG_DEV1_EPF1_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF1_0_HEADER                                                                   0x12403
+#define regBIF_CFG_DEV1_EPF1_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV1_EPF1_0_BIST                                                                     0x12403
+#define regBIF_CFG_DEV1_EPF1_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_1                                                              0x12404
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_2                                                              0x12405
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_3                                                              0x12406
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_4                                                              0x12407
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_5                                                              0x12408
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_6                                                              0x12409
+#define regBIF_CFG_DEV1_EPF1_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_CARDBUS_CIS_PTR                                                          0x1240a
+#define regBIF_CFG_DEV1_EPF1_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_ADAPTER_ID                                                               0x1240b
+#define regBIF_CFG_DEV1_EPF1_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR                                                            0x1240c
+#define regBIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_CAP_PTR                                                                  0x1240d
+#define regBIF_CFG_DEV1_EPF1_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF1_0_INTERRUPT_LINE                                                           0x1240f
+#define regBIF_CFG_DEV1_EPF1_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_INTERRUPT_PIN                                                            0x1240f
+#define regBIF_CFG_DEV1_EPF1_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_MIN_GRANT                                                                0x1240f
+#define regBIF_CFG_DEV1_EPF1_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_MAX_LATENCY                                                              0x1240f
+#define regBIF_CFG_DEV1_EPF1_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST                                                          0x12412
+#define regBIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_ADAPTER_ID_W                                                             0x12413
+#define regBIF_CFG_DEV1_EPF1_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_PMI_CAP_LIST                                                             0x12414
+#define regBIF_CFG_DEV1_EPF1_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_PMI_CAP                                                                  0x12414
+#define regBIF_CFG_DEV1_EPF1_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL                                                          0x12415
+#define regBIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_SBRN                                                                     0x12418
+#define regBIF_CFG_DEV1_EPF1_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV1_EPF1_0_FLADJ                                                                    0x12418
+#define regBIF_CFG_DEV1_EPF1_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV1_EPF1_0_DBESL_DBESLD                                                             0x12418
+#define regBIF_CFG_DEV1_EPF1_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CAP_LIST                                                            0x12419
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CAP                                                                 0x12419
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CAP                                                               0x1241a
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CNTL                                                              0x1241b
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_STATUS                                                            0x1241b
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CAP                                                                 0x1241c
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CNTL                                                                0x1241d
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_LINK_STATUS                                                              0x1241d
+#define regBIF_CFG_DEV1_EPF1_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CAP2                                                              0x12422
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2                                                             0x12423
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_STATUS2                                                           0x12423
+#define regBIF_CFG_DEV1_EPF1_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CAP2                                                                0x12424
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CNTL2                                                               0x12425
+#define regBIF_CFG_DEV1_EPF1_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF1_0_LINK_STATUS2                                                             0x12425
+#define regBIF_CFG_DEV1_EPF1_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_CAP_LIST                                                             0x12428
+#define regBIF_CFG_DEV1_EPF1_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL                                                             0x12428
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_LO                                                          0x12429
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_HI                                                          0x1242a
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA                                                             0x1242a
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA                                                         0x1242a
+#define regBIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MASK                                                                 0x1242b
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA_64                                                          0x1242b
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA_64                                                      0x1242b
+#define regBIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MASK_64                                                              0x1242c
+#define regBIF_CFG_DEV1_EPF1_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_PENDING                                                              0x1242c
+#define regBIF_CFG_DEV1_EPF1_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV1_EPF1_0_MSI_PENDING_64                                                           0x1242d
+#define regBIF_CFG_DEV1_EPF1_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_CAP_LIST                                                            0x12430
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL                                                            0x12430
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_TABLE                                                               0x12431
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_PBA                                                                 0x12432
+#define regBIF_CFG_DEV1_EPF1_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x12440
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x12441
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC1                                                    0x12442
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC2                                                    0x12443
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x12581
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS                                                   0x12582
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK                                                     0x12583
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x12584
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS                                                     0x12585
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK                                                       0x12586
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x12587
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG0                                                            0x12588
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG1                                                            0x12589
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG2                                                            0x1258a
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG3                                                            0x1258b
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG0                                                     0x1258f
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG1                                                     0x12590
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG2                                                     0x12591
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG3                                                     0x12592
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST                                                    0x125b7
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP                                                             0x125b8
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL                                                            0x125b8
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST                                                    0x125c7
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_RTR_DATA1                                                                0x125c8
+#define regBIF_CFG_DEV1_EPF1_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_RTR_DATA2                                                                0x125c9
+#define regBIF_CFG_DEV1_EPF1_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x125ca
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x125cb
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA                                                     0x125cc
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_CAP                                                      0x125cd
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST                                                    0x125ce
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP                                                             0x125cf
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_LATENCY_INDICATOR                                               0x125d0
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_STATUS                                                          0x125d1
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_CNTL                                                            0x125d1
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x125d2
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x125d2
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x125d2
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x125d2
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x125d3
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x125d3
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x125d3
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x125d3
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST                                                    0x125dc
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP                                                             0x125dd
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL                                                            0x125dd
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST                                                    0x125de
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CAP                                                            0x125df
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL                                                           0x125e0
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CAP                                                            0x125e1
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL                                                           0x125e2
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CAP                                                            0x125e3
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL                                                           0x125e4
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CAP                                                            0x125e5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL                                                           0x125e6
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CAP                                                            0x125e7
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL                                                           0x125e8
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CAP                                                            0x125e9
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL                                                           0x125ea
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST                                                  0x125f1
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP                                                           0x125f2
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL                                                          0x125f2
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST                                                  0x125f3
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP                                                           0x125f4
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL                                                       0x125f5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_STATUS                                                        0x125f5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_INITIAL_VFS                                                   0x125f6
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_INITIAL_VFS_BASE_IDX                                          5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_TOTAL_VFS                                                     0x125f6
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_TOTAL_VFS_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_NUM_VFS                                                       0x125f7
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_NUM_VFS_BASE_IDX                                              5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FUNC_DEP_LINK                                                 0x125f7
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FUNC_DEP_LINK_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FIRST_VF_OFFSET                                               0x125f8
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FIRST_VF_OFFSET_BASE_IDX                                      5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_STRIDE                                                     0x125f8
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_STRIDE_BASE_IDX                                            5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_DEVICE_ID                                                  0x125f9
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_DEVICE_ID_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE                                           0x125fa
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE_BASE_IDX                                  5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE                                              0x125fb
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE_BASE_IDX                                     5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_0                                                0x125fc
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_0_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_1                                                0x125fd
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_1_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_2                                                0x125fe
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_2_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_3                                                0x125ff
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_3_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_4                                                0x12600
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_4_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_5                                                0x12601
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_5_BASE_IDX                                       5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                          0x12638
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP                                                  0x12639
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL                                                 0x1263a
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP                                                  0x1263b
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL                                                 0x1263c
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP                                                  0x1263d
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL                                                 0x1263e
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP                                                  0x1263f
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL                                                 0x12640
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP                                                  0x12641
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL                                                 0x12642
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP                                                  0x12643
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL                                                 0x12644
+#define regBIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL_BASE_IDX                                        5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_rc_bifcfgdecp
+// base address: 0x10102000
+#define regBIF_CFG_DEV2_RC0_VENDOR_ID                                                                   0x0800
+#define regBIF_CFG_DEV2_RC0_VENDOR_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_DEVICE_ID                                                                   0x0800
+#define regBIF_CFG_DEV2_RC0_DEVICE_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_COMMAND                                                                     0x0801
+#define regBIF_CFG_DEV2_RC0_COMMAND_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_RC0_STATUS                                                                      0x0801
+#define regBIF_CFG_DEV2_RC0_STATUS_BASE_IDX                                                             5
+#define regBIF_CFG_DEV2_RC0_REVISION_ID                                                                 0x0802
+#define regBIF_CFG_DEV2_RC0_REVISION_ID_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_PROG_INTERFACE                                                              0x0802
+#define regBIF_CFG_DEV2_RC0_PROG_INTERFACE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_RC0_SUB_CLASS                                                                   0x0802
+#define regBIF_CFG_DEV2_RC0_SUB_CLASS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_BASE_CLASS                                                                  0x0802
+#define regBIF_CFG_DEV2_RC0_BASE_CLASS_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_RC0_CACHE_LINE                                                                  0x0803
+#define regBIF_CFG_DEV2_RC0_CACHE_LINE_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_RC0_LATENCY                                                                     0x0803
+#define regBIF_CFG_DEV2_RC0_LATENCY_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_RC0_HEADER                                                                      0x0803
+#define regBIF_CFG_DEV2_RC0_HEADER_BASE_IDX                                                             5
+#define regBIF_CFG_DEV2_RC0_BIST                                                                        0x0803
+#define regBIF_CFG_DEV2_RC0_BIST_BASE_IDX                                                               5
+#define regBIF_CFG_DEV2_RC0_BASE_ADDR_1                                                                 0x0804
+#define regBIF_CFG_DEV2_RC0_BASE_ADDR_1_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_BASE_ADDR_2                                                                 0x0805
+#define regBIF_CFG_DEV2_RC0_BASE_ADDR_2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY                                                      0x0806
+#define regBIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_RC0_IO_BASE_LIMIT                                                               0x0807
+#define regBIF_CFG_DEV2_RC0_IO_BASE_LIMIT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_SECONDARY_STATUS                                                            0x0807
+#define regBIF_CFG_DEV2_RC0_SECONDARY_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_RC0_MEM_BASE_LIMIT                                                              0x0808
+#define regBIF_CFG_DEV2_RC0_MEM_BASE_LIMIT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_RC0_PREF_BASE_LIMIT                                                             0x0809
+#define regBIF_CFG_DEV2_RC0_PREF_BASE_LIMIT_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_PREF_BASE_UPPER                                                             0x080a
+#define regBIF_CFG_DEV2_RC0_PREF_BASE_UPPER_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_PREF_LIMIT_UPPER                                                            0x080b
+#define regBIF_CFG_DEV2_RC0_PREF_LIMIT_UPPER_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_RC0_IO_BASE_LIMIT_HI                                                            0x080c
+#define regBIF_CFG_DEV2_RC0_IO_BASE_LIMIT_HI_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_RC0_CAP_PTR                                                                     0x080d
+#define regBIF_CFG_DEV2_RC0_CAP_PTR_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_RC0_ROM_BASE_ADDR                                                               0x080e
+#define regBIF_CFG_DEV2_RC0_ROM_BASE_ADDR_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_INTERRUPT_LINE                                                              0x080f
+#define regBIF_CFG_DEV2_RC0_INTERRUPT_LINE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_RC0_INTERRUPT_PIN                                                               0x080f
+#define regBIF_CFG_DEV2_RC0_INTERRUPT_PIN_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL                                                             0x080f
+#define regBIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_EXT_BRIDGE_CNTL                                                             0x0810
+#define regBIF_CFG_DEV2_RC0_EXT_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_PMI_CAP_LIST                                                                0x0814
+#define regBIF_CFG_DEV2_RC0_PMI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_PMI_CAP                                                                     0x0814
+#define regBIF_CFG_DEV2_RC0_PMI_CAP_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_RC0_PMI_STATUS_CNTL                                                             0x0815
+#define regBIF_CFG_DEV2_RC0_PMI_STATUS_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_PCIE_CAP_LIST                                                               0x0816
+#define regBIF_CFG_DEV2_RC0_PCIE_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_CAP                                                                    0x0816
+#define regBIF_CFG_DEV2_RC0_PCIE_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_RC0_DEVICE_CAP                                                                  0x0817
+#define regBIF_CFG_DEV2_RC0_DEVICE_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_RC0_DEVICE_CNTL                                                                 0x0818
+#define regBIF_CFG_DEV2_RC0_DEVICE_CNTL_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_DEVICE_STATUS                                                               0x0818
+#define regBIF_CFG_DEV2_RC0_DEVICE_STATUS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_LINK_CAP                                                                    0x0819
+#define regBIF_CFG_DEV2_RC0_LINK_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_RC0_LINK_CNTL                                                                   0x081a
+#define regBIF_CFG_DEV2_RC0_LINK_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_LINK_STATUS                                                                 0x081a
+#define regBIF_CFG_DEV2_RC0_LINK_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_SLOT_CAP                                                                    0x081b
+#define regBIF_CFG_DEV2_RC0_SLOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_RC0_SLOT_CNTL                                                                   0x081c
+#define regBIF_CFG_DEV2_RC0_SLOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_SLOT_STATUS                                                                 0x081c
+#define regBIF_CFG_DEV2_RC0_SLOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_ROOT_CNTL                                                                   0x081d
+#define regBIF_CFG_DEV2_RC0_ROOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_ROOT_CAP                                                                    0x081d
+#define regBIF_CFG_DEV2_RC0_ROOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_RC0_ROOT_STATUS                                                                 0x081e
+#define regBIF_CFG_DEV2_RC0_ROOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_DEVICE_CAP2                                                                 0x081f
+#define regBIF_CFG_DEV2_RC0_DEVICE_CAP2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_RC0_DEVICE_CNTL2                                                                0x0820
+#define regBIF_CFG_DEV2_RC0_DEVICE_CNTL2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_DEVICE_STATUS2                                                              0x0820
+#define regBIF_CFG_DEV2_RC0_DEVICE_STATUS2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_RC0_LINK_CAP2                                                                   0x0821
+#define regBIF_CFG_DEV2_RC0_LINK_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_LINK_CNTL2                                                                  0x0822
+#define regBIF_CFG_DEV2_RC0_LINK_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_RC0_LINK_STATUS2                                                                0x0822
+#define regBIF_CFG_DEV2_RC0_LINK_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_SLOT_CAP2                                                                   0x0823
+#define regBIF_CFG_DEV2_RC0_SLOT_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_SLOT_CNTL2                                                                  0x0824
+#define regBIF_CFG_DEV2_RC0_SLOT_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_RC0_SLOT_STATUS2                                                                0x0824
+#define regBIF_CFG_DEV2_RC0_SLOT_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_MSI_CAP_LIST                                                                0x0828
+#define regBIF_CFG_DEV2_RC0_MSI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_CNTL                                                                0x0828
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_ADDR_LO                                                             0x0829
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_ADDR_LO_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_ADDR_HI                                                             0x082a
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_ADDR_HI_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_DATA                                                                0x082a
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_DATA_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA                                                            0x082a
+#define regBIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_DATA_64                                                             0x082b
+#define regBIF_CFG_DEV2_RC0_MSI_MSG_DATA_64_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA_64                                                         0x082b
+#define regBIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_RC0_SSID_CAP_LIST                                                               0x0830
+#define regBIF_CFG_DEV2_RC0_SSID_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_SSID_CAP                                                                    0x0831
+#define regBIF_CFG_DEV2_RC0_SSID_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                           0x0840
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR                                                    0x0841
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC1                                                       0x0842
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC2                                                       0x0843
+#define regBIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST                                                 0x0844
+#define regBIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3                                                             0x0845
+#define regBIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_ERROR_STATUS                                                      0x0846
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL                                               0x0847
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL                                               0x0847
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL                                               0x0848
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL                                               0x0848
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL                                               0x0849
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL                                               0x0849
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL                                               0x084a
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL                                               0x084a
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL                                               0x084b
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL                                               0x084b
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL                                              0x084c
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL                                              0x084c
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL                                              0x084d
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL                                              0x084d
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL                                              0x084e
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL                                              0x084e
+#define regBIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST                                                  0x086e
+#define regBIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LINK_CAP_16GT                                                               0x086f
+#define regBIF_CFG_DEV2_RC0_LINK_CAP_16GT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_LINK_CNTL_16GT                                                              0x0870
+#define regBIF_CFG_DEV2_RC0_LINK_CNTL_16GT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_RC0_LINK_STATUS_16GT                                                            0x0871
+#define regBIF_CFG_DEV2_RC0_LINK_STATUS_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                           0x0872
+#define regBIF_CFG_DEV2_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT                                            0x0873
+#define regBIF_CFG_DEV2_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT                                            0x0874
+#define regBIF_CFG_DEV2_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_RC0_LANE_0_EQUALIZATION_CNTL_16GT                                               0x0876
+#define regBIF_CFG_DEV2_RC0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_1_EQUALIZATION_CNTL_16GT                                               0x0876
+#define regBIF_CFG_DEV2_RC0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_2_EQUALIZATION_CNTL_16GT                                               0x0876
+#define regBIF_CFG_DEV2_RC0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_3_EQUALIZATION_CNTL_16GT                                               0x0876
+#define regBIF_CFG_DEV2_RC0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_4_EQUALIZATION_CNTL_16GT                                               0x0877
+#define regBIF_CFG_DEV2_RC0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_5_EQUALIZATION_CNTL_16GT                                               0x0877
+#define regBIF_CFG_DEV2_RC0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_6_EQUALIZATION_CNTL_16GT                                               0x0877
+#define regBIF_CFG_DEV2_RC0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_7_EQUALIZATION_CNTL_16GT                                               0x0877
+#define regBIF_CFG_DEV2_RC0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_8_EQUALIZATION_CNTL_16GT                                               0x0878
+#define regBIF_CFG_DEV2_RC0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_9_EQUALIZATION_CNTL_16GT                                               0x0878
+#define regBIF_CFG_DEV2_RC0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_10_EQUALIZATION_CNTL_16GT                                              0x0878
+#define regBIF_CFG_DEV2_RC0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_LANE_11_EQUALIZATION_CNTL_16GT                                              0x0878
+#define regBIF_CFG_DEV2_RC0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_LANE_12_EQUALIZATION_CNTL_16GT                                              0x0879
+#define regBIF_CFG_DEV2_RC0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_LANE_13_EQUALIZATION_CNTL_16GT                                              0x0879
+#define regBIF_CFG_DEV2_RC0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_LANE_14_EQUALIZATION_CNTL_16GT                                              0x0879
+#define regBIF_CFG_DEV2_RC0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_LANE_15_EQUALIZATION_CNTL_16GT                                              0x0879
+#define regBIF_CFG_DEV2_RC0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST                                                 0x0894
+#define regBIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_MARGINING_PORT_CAP                                                          0x0895
+#define regBIF_CFG_DEV2_RC0_MARGINING_PORT_CAP_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_RC0_MARGINING_PORT_STATUS                                                       0x0895
+#define regBIF_CFG_DEV2_RC0_MARGINING_PORT_STATUS_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL                                                  0x0896
+#define regBIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS                                                0x0896
+#define regBIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL                                                  0x0897
+#define regBIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS                                                0x0897
+#define regBIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL                                                  0x0898
+#define regBIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS                                                0x0898
+#define regBIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL                                                  0x0899
+#define regBIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS                                                0x0899
+#define regBIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL                                                  0x089a
+#define regBIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS                                                0x089a
+#define regBIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL                                                  0x089b
+#define regBIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS                                                0x089b
+#define regBIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL                                                  0x089c
+#define regBIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS                                                0x089c
+#define regBIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL                                                  0x089d
+#define regBIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS                                                0x089d
+#define regBIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL                                                  0x089e
+#define regBIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS                                                0x089e
+#define regBIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL                                                  0x089f
+#define regBIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS                                                0x089f
+#define regBIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL                                                 0x08a0
+#define regBIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS                                               0x08a0
+#define regBIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL                                                 0x08a1
+#define regBIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS                                               0x08a1
+#define regBIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL                                                 0x08a2
+#define regBIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS                                               0x08a2
+#define regBIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL                                                 0x08a3
+#define regBIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS                                               0x08a3
+#define regBIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL                                                 0x08a4
+#define regBIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS                                               0x08a4
+#define regBIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL                                                 0x08a5
+#define regBIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS                                               0x08a5
+#define regBIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST                                                        0x0901
+#define regBIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1                                                       0x0902
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG2                                                       0x0903
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_CNTL                                                           0x0904
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_STATUS                                                         0x0904
+#define regBIF_CFG_DEV2_RC0_PCIE_PORT_VC_STATUS_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP                                                       0x0905
+#define regBIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL                                                      0x0906
+#define regBIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_STATUS                                                    0x0907
+#define regBIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP                                                       0x0908
+#define regBIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL                                                      0x0909
+#define regBIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_STATUS                                                    0x090a
+#define regBIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST                                                       0x090c
+#define regBIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_CAP                                                       0x090d
+#define regBIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_STATUS                                                    0x090e
+#define regBIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                               0x0981
+#define regBIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS                                                      0x0982
+#define regBIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK                                                        0x0983
+#define regBIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY                                                    0x0984
+#define regBIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS                                                        0x0985
+#define regBIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK                                                          0x0986
+#define regBIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL                                                       0x0987
+#define regBIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG0                                                               0x0988
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG0_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG1                                                               0x0989
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG1_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG2                                                               0x098a
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG3                                                               0x098b
+#define regBIF_CFG_DEV2_RC0_PCIE_HDR_LOG3_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD                                                           0x098c
+#define regBIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS                                                        0x098d
+#define regBIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_ERR_SRC_ID                                                             0x098e
+#define regBIF_CFG_DEV2_RC0_PCIE_ERR_SRC_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG0                                                        0x098f
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG1                                                        0x0990
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG2                                                        0x0991
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG3                                                        0x0992
+#define regBIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                            0x0999
+#define regBIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW1                                                     0x099a
+#define regBIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW2                                                     0x099b
+#define regBIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST                                                       0x09b7
+#define regBIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_PCIE_ACS_CAP                                                                0x09b8
+#define regBIF_CFG_DEV2_RC0_PCIE_ACS_CAP_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_RC0_PCIE_ACS_CNTL                                                               0x09b8
+#define regBIF_CFG_DEV2_RC0_PCIE_ACS_CNTL_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST                                                       0x09c7
+#define regBIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_RC0_RTR_DATA1                                                                   0x09c8
+#define regBIF_CFG_DEV2_RC0_RTR_DATA1_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_RC0_RTR_DATA2                                                                   0x09c9
+#define regBIF_CFG_DEV2_RC0_RTR_DATA2_BASE_IDX                                                          5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf0_bifcfgdecp
+// base address: 0x10150000
+#define regBIF_CFG_DEV2_EPF0_0_VENDOR_ID                                                                0x14000
+#define regBIF_CFG_DEV2_EPF0_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_ID                                                                0x14000
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_COMMAND                                                                  0x14001
+#define regBIF_CFG_DEV2_EPF0_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF0_0_STATUS                                                                   0x14001
+#define regBIF_CFG_DEV2_EPF0_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF0_0_REVISION_ID                                                              0x14002
+#define regBIF_CFG_DEV2_EPF0_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_PROG_INTERFACE                                                           0x14002
+#define regBIF_CFG_DEV2_EPF0_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_SUB_CLASS                                                                0x14002
+#define regBIF_CFG_DEV2_EPF0_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_BASE_CLASS                                                               0x14002
+#define regBIF_CFG_DEV2_EPF0_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF0_0_CACHE_LINE                                                               0x14003
+#define regBIF_CFG_DEV2_EPF0_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LATENCY                                                                  0x14003
+#define regBIF_CFG_DEV2_EPF0_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF0_0_HEADER                                                                   0x14003
+#define regBIF_CFG_DEV2_EPF0_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF0_0_BIST                                                                     0x14003
+#define regBIF_CFG_DEV2_EPF0_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_1                                                              0x14004
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_2                                                              0x14005
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_3                                                              0x14006
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_4                                                              0x14007
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_5                                                              0x14008
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_6                                                              0x14009
+#define regBIF_CFG_DEV2_EPF0_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_CARDBUS_CIS_PTR                                                          0x1400a
+#define regBIF_CFG_DEV2_EPF0_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_ADAPTER_ID                                                               0x1400b
+#define regBIF_CFG_DEV2_EPF0_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR                                                            0x1400c
+#define regBIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_CAP_PTR                                                                  0x1400d
+#define regBIF_CFG_DEV2_EPF0_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF0_0_INTERRUPT_LINE                                                           0x1400f
+#define regBIF_CFG_DEV2_EPF0_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_INTERRUPT_PIN                                                            0x1400f
+#define regBIF_CFG_DEV2_EPF0_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_MIN_GRANT                                                                0x1400f
+#define regBIF_CFG_DEV2_EPF0_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_MAX_LATENCY                                                              0x1400f
+#define regBIF_CFG_DEV2_EPF0_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST                                                          0x14012
+#define regBIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_ADAPTER_ID_W                                                             0x14013
+#define regBIF_CFG_DEV2_EPF0_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PMI_CAP_LIST                                                             0x14014
+#define regBIF_CFG_DEV2_EPF0_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PMI_CAP                                                                  0x14014
+#define regBIF_CFG_DEV2_EPF0_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL                                                          0x14015
+#define regBIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_SBRN                                                                     0x14018
+#define regBIF_CFG_DEV2_EPF0_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF0_0_FLADJ                                                                    0x14018
+#define regBIF_CFG_DEV2_EPF0_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_EPF0_0_DBESL_DBESLD                                                             0x14018
+#define regBIF_CFG_DEV2_EPF0_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CAP_LIST                                                            0x14019
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CAP                                                                 0x14019
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CAP                                                               0x1401a
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CNTL                                                              0x1401b
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_STATUS                                                            0x1401b
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CAP2                                                              0x14022
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2                                                             0x14023
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_STATUS2                                                           0x14023
+#define regBIF_CFG_DEV2_EPF0_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_CAP_LIST                                                             0x14028
+#define regBIF_CFG_DEV2_EPF0_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL                                                             0x14028
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_LO                                                          0x14029
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_HI                                                          0x1402a
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA                                                             0x1402a
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA                                                         0x1402a
+#define regBIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MASK                                                                 0x1402b
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA_64                                                          0x1402b
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA_64                                                      0x1402b
+#define regBIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MASK_64                                                              0x1402c
+#define regBIF_CFG_DEV2_EPF0_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_PENDING                                                              0x1402c
+#define regBIF_CFG_DEV2_EPF0_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_MSI_PENDING_64                                                           0x1402d
+#define regBIF_CFG_DEV2_EPF0_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_CAP_LIST                                                            0x14030
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL                                                            0x14030
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_TABLE                                                               0x14031
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_PBA                                                                 0x14032
+#define regBIF_CFG_DEV2_EPF0_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x14181
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS                                                   0x14182
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK                                                     0x14183
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x14184
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS                                                     0x14185
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK                                                       0x14186
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x14187
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG0                                                            0x14188
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG1                                                            0x14189
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG2                                                            0x1418a
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG3                                                            0x1418b
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG0                                                     0x1418f
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG1                                                     0x14190
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG2                                                     0x14191
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG3                                                     0x14192
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST                                                    0x141b7
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP                                                             0x141b8
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL                                                            0x141b8
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST                                                    0x141c7
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_RTR_DATA1                                                                0x141c8
+#define regBIF_CFG_DEV2_EPF0_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_RTR_DATA2                                                                0x141c9
+#define regBIF_CFG_DEV2_EPF0_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x141ca
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x141cb
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA                                                     0x141cc
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_CAP                                                      0x141cd
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST                                                    0x141ce
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP                                                             0x141cf
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_LATENCY_INDICATOR                                               0x141d0
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_STATUS                                                          0x141d1
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_CNTL                                                            0x141d1
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x141d2
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x141d2
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x141d2
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x141d2
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x141d3
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x141d3
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x141d3
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x141d3
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST                                                    0x141da
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP                                                             0x141db
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST                                                    0x141dc
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP                                                             0x141dd
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL                                                            0x141dd
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST                                                    0x141de
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CAP                                                            0x141df
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL                                                           0x141e0
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CAP                                                            0x141e1
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL                                                           0x141e2
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CAP                                                            0x141e3
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL                                                           0x141e4
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CAP                                                            0x141e5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL                                                           0x141e6
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CAP                                                            0x141e7
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL                                                           0x141e8
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CAP                                                            0x141e9
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL                                                           0x141ea
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST                                                  0x141f1
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP                                                           0x141f2
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL                                                          0x141f2
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf0_bifcfgpciedecp
+// base address: 0x10150000
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CAP                                                                 0x1401c
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CNTL                                                                0x1401d
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_STATUS                                                              0x1401d
+#define regBIF_CFG_DEV2_EPF0_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CAP2                                                                0x14024
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CNTL2                                                               0x14025
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_STATUS2                                                             0x14025
+#define regBIF_CFG_DEV2_EPF0_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x14040
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x14041
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC1                                                    0x14042
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC2                                                    0x14043
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST                                              0x14044
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3                                                          0x14045
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_ERROR_STATUS                                                   0x14046
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL                                            0x14047
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL                                            0x14047
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL                                            0x14048
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL                                            0x14048
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL                                            0x14049
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL                                            0x14049
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL                                            0x1404a
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL                                            0x1404a
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL                                            0x1404b
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL                                            0x1404b
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL                                           0x1404c
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL                                           0x1404c
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL                                           0x1404d
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL                                           0x1404d
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL                                           0x1404e
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL                                           0x1404e
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST                                               0x1406e
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CAP_16GT                                                            0x1406f
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CAP_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CNTL_16GT                                                           0x14070
+#define regBIF_CFG_DEV2_EPF0_0_LINK_CNTL_16GT_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT                                                         0x14071
+#define regBIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                        0x14072
+#define regBIF_CFG_DEV2_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT                                         0x14073
+#define regBIF_CFG_DEV2_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV2_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT                                         0x14074
+#define regBIF_CFG_DEV2_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT                                            0x14076
+#define regBIF_CFG_DEV2_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT                                            0x14076
+#define regBIF_CFG_DEV2_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT                                            0x14076
+#define regBIF_CFG_DEV2_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT                                            0x14076
+#define regBIF_CFG_DEV2_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT                                            0x14077
+#define regBIF_CFG_DEV2_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT                                            0x14077
+#define regBIF_CFG_DEV2_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT                                            0x14077
+#define regBIF_CFG_DEV2_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT                                            0x14077
+#define regBIF_CFG_DEV2_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT                                            0x14078
+#define regBIF_CFG_DEV2_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT                                            0x14078
+#define regBIF_CFG_DEV2_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT                                           0x14078
+#define regBIF_CFG_DEV2_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT                                           0x14078
+#define regBIF_CFG_DEV2_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT                                           0x14079
+#define regBIF_CFG_DEV2_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT                                           0x14079
+#define regBIF_CFG_DEV2_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT                                           0x14079
+#define regBIF_CFG_DEV2_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT                                           0x14079
+#define regBIF_CFG_DEV2_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST                                              0x14094
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_MARGINING_PORT_CAP                                                       0x14095
+#define regBIF_CFG_DEV2_EPF0_0_MARGINING_PORT_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF0_0_MARGINING_PORT_STATUS                                                    0x14095
+#define regBIF_CFG_DEV2_EPF0_0_MARGINING_PORT_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL                                               0x14096
+#define regBIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS                                             0x14096
+#define regBIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL                                               0x14097
+#define regBIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS                                             0x14097
+#define regBIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL                                               0x14098
+#define regBIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS                                             0x14098
+#define regBIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL                                               0x14099
+#define regBIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS                                             0x14099
+#define regBIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL                                               0x1409a
+#define regBIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS                                             0x1409a
+#define regBIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL                                               0x1409b
+#define regBIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS                                             0x1409b
+#define regBIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL                                               0x1409c
+#define regBIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS                                             0x1409c
+#define regBIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL                                               0x1409d
+#define regBIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS                                             0x1409d
+#define regBIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL                                               0x1409e
+#define regBIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS                                             0x1409e
+#define regBIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL                                               0x1409f
+#define regBIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS                                             0x1409f
+#define regBIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL                                              0x140a0
+#define regBIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS                                            0x140a0
+#define regBIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL                                              0x140a1
+#define regBIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS                                            0x140a1
+#define regBIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL                                              0x140a2
+#define regBIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS                                            0x140a2
+#define regBIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL                                              0x140a3
+#define regBIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS                                            0x140a3
+#define regBIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL                                              0x140a4
+#define regBIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS                                            0x140a4
+#define regBIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL                                              0x140a5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS                                            0x140a5
+#define regBIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST                                                     0x14101
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1                                                    0x14102
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG2                                                    0x14103
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CNTL                                                        0x14104
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CNTL_BASE_IDX                                               5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_STATUS                                                      0x14104
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP                                                    0x14105
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL                                                   0x14106
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_STATUS                                                 0x14107
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP                                                    0x14108
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL                                                   0x14109
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_STATUS                                                 0x1410a
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST                                                    0x1410c
+#define regBIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_CAP                                                    0x1410d
+#define regBIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_STATUS                                                 0x1410e
+#define regBIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                        5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf1_bifcfgdecp
+// base address: 0x10151000
+#define regBIF_CFG_DEV2_EPF1_0_VENDOR_ID                                                                0x14400
+#define regBIF_CFG_DEV2_EPF1_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_ID                                                                0x14400
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_COMMAND                                                                  0x14401
+#define regBIF_CFG_DEV2_EPF1_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF1_0_STATUS                                                                   0x14401
+#define regBIF_CFG_DEV2_EPF1_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF1_0_REVISION_ID                                                              0x14402
+#define regBIF_CFG_DEV2_EPF1_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_PROG_INTERFACE                                                           0x14402
+#define regBIF_CFG_DEV2_EPF1_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_SUB_CLASS                                                                0x14402
+#define regBIF_CFG_DEV2_EPF1_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_BASE_CLASS                                                               0x14402
+#define regBIF_CFG_DEV2_EPF1_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF1_0_CACHE_LINE                                                               0x14403
+#define regBIF_CFG_DEV2_EPF1_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF1_0_LATENCY                                                                  0x14403
+#define regBIF_CFG_DEV2_EPF1_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF1_0_HEADER                                                                   0x14403
+#define regBIF_CFG_DEV2_EPF1_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF1_0_BIST                                                                     0x14403
+#define regBIF_CFG_DEV2_EPF1_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_1                                                              0x14404
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_2                                                              0x14405
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_3                                                              0x14406
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_4                                                              0x14407
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_5                                                              0x14408
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_6                                                              0x14409
+#define regBIF_CFG_DEV2_EPF1_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_CARDBUS_CIS_PTR                                                          0x1440a
+#define regBIF_CFG_DEV2_EPF1_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF1_0_ADAPTER_ID                                                               0x1440b
+#define regBIF_CFG_DEV2_EPF1_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR                                                            0x1440c
+#define regBIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_CAP_PTR                                                                  0x1440d
+#define regBIF_CFG_DEV2_EPF1_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF1_0_INTERRUPT_LINE                                                           0x1440f
+#define regBIF_CFG_DEV2_EPF1_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_INTERRUPT_PIN                                                            0x1440f
+#define regBIF_CFG_DEV2_EPF1_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_MIN_GRANT                                                                0x1440f
+#define regBIF_CFG_DEV2_EPF1_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_MAX_LATENCY                                                              0x1440f
+#define regBIF_CFG_DEV2_EPF1_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST                                                          0x14412
+#define regBIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF1_0_ADAPTER_ID_W                                                             0x14413
+#define regBIF_CFG_DEV2_EPF1_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_PMI_CAP_LIST                                                             0x14414
+#define regBIF_CFG_DEV2_EPF1_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_PMI_CAP                                                                  0x14414
+#define regBIF_CFG_DEV2_EPF1_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL                                                          0x14415
+#define regBIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF1_0_SBRN                                                                     0x14418
+#define regBIF_CFG_DEV2_EPF1_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF1_0_FLADJ                                                                    0x14418
+#define regBIF_CFG_DEV2_EPF1_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_EPF1_0_DBESL_DBESLD                                                             0x14418
+#define regBIF_CFG_DEV2_EPF1_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CAP_LIST                                                            0x14419
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CAP                                                                 0x14419
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CAP                                                               0x1441a
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CNTL                                                              0x1441b
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_STATUS                                                            0x1441b
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CAP                                                                 0x1441c
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CNTL                                                                0x1441d
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_LINK_STATUS                                                              0x1441d
+#define regBIF_CFG_DEV2_EPF1_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CAP2                                                              0x14422
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2                                                             0x14423
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_STATUS2                                                           0x14423
+#define regBIF_CFG_DEV2_EPF1_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CAP2                                                                0x14424
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CNTL2                                                               0x14425
+#define regBIF_CFG_DEV2_EPF1_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF1_0_LINK_STATUS2                                                             0x14425
+#define regBIF_CFG_DEV2_EPF1_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_CAP_LIST                                                             0x14428
+#define regBIF_CFG_DEV2_EPF1_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL                                                             0x14428
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_LO                                                          0x14429
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_HI                                                          0x1442a
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA                                                             0x1442a
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA                                                         0x1442a
+#define regBIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MASK                                                                 0x1442b
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA_64                                                          0x1442b
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA_64                                                      0x1442b
+#define regBIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MASK_64                                                              0x1442c
+#define regBIF_CFG_DEV2_EPF1_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_PENDING                                                              0x1442c
+#define regBIF_CFG_DEV2_EPF1_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF1_0_MSI_PENDING_64                                                           0x1442d
+#define regBIF_CFG_DEV2_EPF1_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_CAP_LIST                                                            0x14430
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL                                                            0x14430
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_TABLE                                                               0x14431
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_PBA                                                                 0x14432
+#define regBIF_CFG_DEV2_EPF1_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x14440
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x14441
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC1                                                    0x14442
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC2                                                    0x14443
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x14581
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS                                                   0x14582
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK                                                     0x14583
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x14584
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS                                                     0x14585
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK                                                       0x14586
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x14587
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG0                                                            0x14588
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG1                                                            0x14589
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG2                                                            0x1458a
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG3                                                            0x1458b
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG0                                                     0x1458f
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG1                                                     0x14590
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG2                                                     0x14591
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG3                                                     0x14592
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST                                                    0x145b7
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP                                                             0x145b8
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL                                                            0x145b8
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST                                                    0x145c7
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_RTR_DATA1                                                                0x145c8
+#define regBIF_CFG_DEV2_EPF1_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_RTR_DATA2                                                                0x145c9
+#define regBIF_CFG_DEV2_EPF1_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x145ca
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x145cb
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA                                                     0x145cc
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_CAP                                                      0x145cd
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST                                                    0x145ce
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP                                                             0x145cf
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_LATENCY_INDICATOR                                               0x145d0
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_STATUS                                                          0x145d1
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_CNTL                                                            0x145d1
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x145d2
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x145d2
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x145d2
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x145d2
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x145d3
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x145d3
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x145d3
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x145d3
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST                                                    0x145dc
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP                                                             0x145dd
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL                                                            0x145dd
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST                                                    0x145de
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CAP                                                            0x145df
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL                                                           0x145e0
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CAP                                                            0x145e1
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL                                                           0x145e2
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CAP                                                            0x145e3
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL                                                           0x145e4
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CAP                                                            0x145e5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL                                                           0x145e6
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CAP                                                            0x145e7
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL                                                           0x145e8
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CAP                                                            0x145e9
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL                                                           0x145ea
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST                                                  0x145f1
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP                                                           0x145f2
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL                                                          0x145f2
+#define regBIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf2_bifcfgdecp
+// base address: 0x10152000
+#define regBIF_CFG_DEV2_EPF2_0_VENDOR_ID                                                                0x14800
+#define regBIF_CFG_DEV2_EPF2_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_ID                                                                0x14800
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_COMMAND                                                                  0x14801
+#define regBIF_CFG_DEV2_EPF2_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF2_0_STATUS                                                                   0x14801
+#define regBIF_CFG_DEV2_EPF2_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF2_0_REVISION_ID                                                              0x14802
+#define regBIF_CFG_DEV2_EPF2_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_PROG_INTERFACE                                                           0x14802
+#define regBIF_CFG_DEV2_EPF2_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_SUB_CLASS                                                                0x14802
+#define regBIF_CFG_DEV2_EPF2_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_BASE_CLASS                                                               0x14802
+#define regBIF_CFG_DEV2_EPF2_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF2_0_CACHE_LINE                                                               0x14803
+#define regBIF_CFG_DEV2_EPF2_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF2_0_LATENCY                                                                  0x14803
+#define regBIF_CFG_DEV2_EPF2_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF2_0_HEADER                                                                   0x14803
+#define regBIF_CFG_DEV2_EPF2_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF2_0_BIST                                                                     0x14803
+#define regBIF_CFG_DEV2_EPF2_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_1                                                              0x14804
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_2                                                              0x14805
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_3                                                              0x14806
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_4                                                              0x14807
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_5                                                              0x14808
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_6                                                              0x14809
+#define regBIF_CFG_DEV2_EPF2_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_CARDBUS_CIS_PTR                                                          0x1480a
+#define regBIF_CFG_DEV2_EPF2_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF2_0_ADAPTER_ID                                                               0x1480b
+#define regBIF_CFG_DEV2_EPF2_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR                                                            0x1480c
+#define regBIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_CAP_PTR                                                                  0x1480d
+#define regBIF_CFG_DEV2_EPF2_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF2_0_INTERRUPT_LINE                                                           0x1480f
+#define regBIF_CFG_DEV2_EPF2_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_INTERRUPT_PIN                                                            0x1480f
+#define regBIF_CFG_DEV2_EPF2_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_MIN_GRANT                                                                0x1480f
+#define regBIF_CFG_DEV2_EPF2_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_MAX_LATENCY                                                              0x1480f
+#define regBIF_CFG_DEV2_EPF2_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST                                                          0x14812
+#define regBIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF2_0_ADAPTER_ID_W                                                             0x14813
+#define regBIF_CFG_DEV2_EPF2_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_PMI_CAP_LIST                                                             0x14814
+#define regBIF_CFG_DEV2_EPF2_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_PMI_CAP                                                                  0x14814
+#define regBIF_CFG_DEV2_EPF2_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL                                                          0x14815
+#define regBIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CAP_LIST                                                            0x14819
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CAP                                                                 0x14819
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CAP                                                               0x1481a
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CNTL                                                              0x1481b
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_STATUS                                                            0x1481b
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CAP                                                                 0x1481c
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CNTL                                                                0x1481d
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_LINK_STATUS                                                              0x1481d
+#define regBIF_CFG_DEV2_EPF2_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CAP2                                                              0x14822
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2                                                             0x14823
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_STATUS2                                                           0x14823
+#define regBIF_CFG_DEV2_EPF2_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CAP2                                                                0x14824
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CNTL2                                                               0x14825
+#define regBIF_CFG_DEV2_EPF2_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF2_0_LINK_STATUS2                                                             0x14825
+#define regBIF_CFG_DEV2_EPF2_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_CAP_LIST                                                             0x14828
+#define regBIF_CFG_DEV2_EPF2_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL                                                             0x14828
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_LO                                                          0x14829
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_HI                                                          0x1482a
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA                                                             0x1482a
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA                                                         0x1482a
+#define regBIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MASK                                                                 0x1482b
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA_64                                                          0x1482b
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA_64                                                      0x1482b
+#define regBIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MASK_64                                                              0x1482c
+#define regBIF_CFG_DEV2_EPF2_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_PENDING                                                              0x1482c
+#define regBIF_CFG_DEV2_EPF2_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF2_0_MSI_PENDING_64                                                           0x1482d
+#define regBIF_CFG_DEV2_EPF2_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_CAP_LIST                                                            0x14830
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL                                                            0x14830
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_TABLE                                                               0x14831
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_PBA                                                                 0x14832
+#define regBIF_CFG_DEV2_EPF2_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x14840
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x14841
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC1                                                    0x14842
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC2                                                    0x14843
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x14981
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS                                                   0x14982
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK                                                     0x14983
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x14984
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS                                                     0x14985
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK                                                       0x14986
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x14987
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG0                                                            0x14988
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG1                                                            0x14989
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG2                                                            0x1498a
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG3                                                            0x1498b
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG0                                                     0x1498f
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG1                                                     0x14990
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG2                                                     0x14991
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG3                                                     0x14992
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST                                                    0x149b7
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP                                                             0x149b8
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL                                                            0x149b8
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST                                                    0x149c7
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_RTR_DATA1                                                                0x149c8
+#define regBIF_CFG_DEV2_EPF2_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_RTR_DATA2                                                                0x149c9
+#define regBIF_CFG_DEV2_EPF2_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x149ca
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x149cb
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA                                                     0x149cc
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_CAP                                                      0x149cd
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST                                                    0x149ce
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP                                                             0x149cf
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_LATENCY_INDICATOR                                               0x149d0
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_STATUS                                                          0x149d1
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_CNTL                                                            0x149d1
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x149d2
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x149d2
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x149d2
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x149d2
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x149d3
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x149d3
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x149d3
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x149d3
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST                                                    0x149dc
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP                                                             0x149dd
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL                                                            0x149dd
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST                                                    0x149de
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CAP                                                            0x149df
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL                                                           0x149e0
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CAP                                                            0x149e1
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL                                                           0x149e2
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CAP                                                            0x149e3
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL                                                           0x149e4
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CAP                                                            0x149e5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL                                                           0x149e6
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CAP                                                            0x149e7
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL                                                           0x149e8
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CAP                                                            0x149e9
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL                                                           0x149ea
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST                                                  0x149f1
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP                                                           0x149f2
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL                                                          0x149f2
+#define regBIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf3_bifcfgdecp
+// base address: 0x10153000
+#define regBIF_CFG_DEV2_EPF3_0_VENDOR_ID                                                                0x14c00
+#define regBIF_CFG_DEV2_EPF3_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_ID                                                                0x14c00
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_COMMAND                                                                  0x14c01
+#define regBIF_CFG_DEV2_EPF3_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF3_0_STATUS                                                                   0x14c01
+#define regBIF_CFG_DEV2_EPF3_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF3_0_REVISION_ID                                                              0x14c02
+#define regBIF_CFG_DEV2_EPF3_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_PROG_INTERFACE                                                           0x14c02
+#define regBIF_CFG_DEV2_EPF3_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_SUB_CLASS                                                                0x14c02
+#define regBIF_CFG_DEV2_EPF3_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_BASE_CLASS                                                               0x14c02
+#define regBIF_CFG_DEV2_EPF3_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF3_0_CACHE_LINE                                                               0x14c03
+#define regBIF_CFG_DEV2_EPF3_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF3_0_LATENCY                                                                  0x14c03
+#define regBIF_CFG_DEV2_EPF3_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF3_0_HEADER                                                                   0x14c03
+#define regBIF_CFG_DEV2_EPF3_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF3_0_BIST                                                                     0x14c03
+#define regBIF_CFG_DEV2_EPF3_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_1                                                              0x14c04
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_2                                                              0x14c05
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_3                                                              0x14c06
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_4                                                              0x14c07
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_5                                                              0x14c08
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_6                                                              0x14c09
+#define regBIF_CFG_DEV2_EPF3_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_ADAPTER_ID                                                               0x14c0b
+#define regBIF_CFG_DEV2_EPF3_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR                                                            0x14c0c
+#define regBIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_CAP_PTR                                                                  0x14c0d
+#define regBIF_CFG_DEV2_EPF3_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF3_0_INTERRUPT_LINE                                                           0x14c0f
+#define regBIF_CFG_DEV2_EPF3_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_INTERRUPT_PIN                                                            0x14c0f
+#define regBIF_CFG_DEV2_EPF3_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_MIN_GRANT                                                                0x14c0f
+#define regBIF_CFG_DEV2_EPF3_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_MAX_LATENCY                                                              0x14c0f
+#define regBIF_CFG_DEV2_EPF3_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST                                                          0x14c12
+#define regBIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF3_0_ADAPTER_ID_W                                                             0x14c13
+#define regBIF_CFG_DEV2_EPF3_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_PMI_CAP_LIST                                                             0x14c14
+#define regBIF_CFG_DEV2_EPF3_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_PMI_CAP                                                                  0x14c14
+#define regBIF_CFG_DEV2_EPF3_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL                                                          0x14c15
+#define regBIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF3_0_SBRN                                                                     0x14c18
+#define regBIF_CFG_DEV2_EPF3_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF3_0_FLADJ                                                                    0x14c18
+#define regBIF_CFG_DEV2_EPF3_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_EPF3_0_DBESL_DBESLD                                                             0x14c18
+#define regBIF_CFG_DEV2_EPF3_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CAP_LIST                                                            0x14c19
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CAP                                                                 0x14c19
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CAP                                                               0x14c1a
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CNTL                                                              0x14c1b
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_STATUS                                                            0x14c1b
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CAP                                                                 0x14c1c
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CNTL                                                                0x14c1d
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_LINK_STATUS                                                              0x14c1d
+#define regBIF_CFG_DEV2_EPF3_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CAP2                                                              0x14c22
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2                                                             0x14c23
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_STATUS2                                                           0x14c23
+#define regBIF_CFG_DEV2_EPF3_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CAP2                                                                0x14c24
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CNTL2                                                               0x14c25
+#define regBIF_CFG_DEV2_EPF3_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF3_0_LINK_STATUS2                                                             0x14c25
+#define regBIF_CFG_DEV2_EPF3_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_CAP_LIST                                                             0x14c28
+#define regBIF_CFG_DEV2_EPF3_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL                                                             0x14c28
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_LO                                                          0x14c29
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_HI                                                          0x14c2a
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA                                                             0x14c2a
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA                                                         0x14c2a
+#define regBIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MASK                                                                 0x14c2b
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA_64                                                          0x14c2b
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA_64                                                      0x14c2b
+#define regBIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MASK_64                                                              0x14c2c
+#define regBIF_CFG_DEV2_EPF3_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_PENDING                                                              0x14c2c
+#define regBIF_CFG_DEV2_EPF3_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF3_0_MSI_PENDING_64                                                           0x14c2d
+#define regBIF_CFG_DEV2_EPF3_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_CAP_LIST                                                            0x14c30
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL                                                            0x14c30
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_TABLE                                                               0x14c31
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_PBA                                                                 0x14c32
+#define regBIF_CFG_DEV2_EPF3_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x14c40
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x14c41
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC1                                                    0x14c42
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC2                                                    0x14c43
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x14d81
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS                                                   0x14d82
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK                                                     0x14d83
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x14d84
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS                                                     0x14d85
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK                                                       0x14d86
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x14d87
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG0                                                            0x14d88
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG1                                                            0x14d89
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG2                                                            0x14d8a
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG3                                                            0x14d8b
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG0                                                     0x14d8f
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG1                                                     0x14d90
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG2                                                     0x14d91
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG3                                                     0x14d92
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST                                                    0x14db7
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP                                                             0x14db8
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL                                                            0x14db8
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST                                                    0x14dc7
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_RTR_DATA1                                                                0x14dc8
+#define regBIF_CFG_DEV2_EPF3_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_RTR_DATA2                                                                0x14dc9
+#define regBIF_CFG_DEV2_EPF3_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x14dca
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x14dcb
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA                                                     0x14dcc
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_CAP                                                      0x14dcd
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST                                                    0x14dce
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP                                                             0x14dcf
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_LATENCY_INDICATOR                                               0x14dd0
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_STATUS                                                          0x14dd1
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_CNTL                                                            0x14dd1
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x14dd2
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x14dd2
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x14dd2
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x14dd2
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x14dd3
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x14dd3
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x14dd3
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x14dd3
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST                                                    0x14ddc
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP                                                             0x14ddd
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL                                                            0x14ddd
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST                                                    0x14dde
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CAP                                                            0x14ddf
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL                                                           0x14de0
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CAP                                                            0x14de1
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL                                                           0x14de2
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CAP                                                            0x14de3
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL                                                           0x14de4
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CAP                                                            0x14de5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL                                                           0x14de6
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CAP                                                            0x14de7
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL                                                           0x14de8
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CAP                                                            0x14de9
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL                                                           0x14dea
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST                                                  0x14df1
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP                                                           0x14df2
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL                                                          0x14df2
+#define regBIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf4_bifcfgdecp
+// base address: 0x10154000
+#define regBIF_CFG_DEV2_EPF4_0_VENDOR_ID                                                                0x15000
+#define regBIF_CFG_DEV2_EPF4_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_ID                                                                0x15000
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_COMMAND                                                                  0x15001
+#define regBIF_CFG_DEV2_EPF4_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF4_0_STATUS                                                                   0x15001
+#define regBIF_CFG_DEV2_EPF4_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF4_0_REVISION_ID                                                              0x15002
+#define regBIF_CFG_DEV2_EPF4_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_PROG_INTERFACE                                                           0x15002
+#define regBIF_CFG_DEV2_EPF4_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_SUB_CLASS                                                                0x15002
+#define regBIF_CFG_DEV2_EPF4_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_BASE_CLASS                                                               0x15002
+#define regBIF_CFG_DEV2_EPF4_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF4_0_CACHE_LINE                                                               0x15003
+#define regBIF_CFG_DEV2_EPF4_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF4_0_LATENCY                                                                  0x15003
+#define regBIF_CFG_DEV2_EPF4_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF4_0_HEADER                                                                   0x15003
+#define regBIF_CFG_DEV2_EPF4_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF4_0_BIST                                                                     0x15003
+#define regBIF_CFG_DEV2_EPF4_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_1                                                              0x15004
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_2                                                              0x15005
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_3                                                              0x15006
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_4                                                              0x15007
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_5                                                              0x15008
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_6                                                              0x15009
+#define regBIF_CFG_DEV2_EPF4_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_ADAPTER_ID                                                               0x1500b
+#define regBIF_CFG_DEV2_EPF4_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR                                                            0x1500c
+#define regBIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_CAP_PTR                                                                  0x1500d
+#define regBIF_CFG_DEV2_EPF4_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF4_0_INTERRUPT_LINE                                                           0x1500f
+#define regBIF_CFG_DEV2_EPF4_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_INTERRUPT_PIN                                                            0x1500f
+#define regBIF_CFG_DEV2_EPF4_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_MIN_GRANT                                                                0x1500f
+#define regBIF_CFG_DEV2_EPF4_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_MAX_LATENCY                                                              0x1500f
+#define regBIF_CFG_DEV2_EPF4_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST                                                          0x15012
+#define regBIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF4_0_ADAPTER_ID_W                                                             0x15013
+#define regBIF_CFG_DEV2_EPF4_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_PMI_CAP_LIST                                                             0x15014
+#define regBIF_CFG_DEV2_EPF4_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_PMI_CAP                                                                  0x15014
+#define regBIF_CFG_DEV2_EPF4_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL                                                          0x15015
+#define regBIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF4_0_SBRN                                                                     0x15018
+#define regBIF_CFG_DEV2_EPF4_0_SBRN_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF4_0_FLADJ                                                                    0x15018
+#define regBIF_CFG_DEV2_EPF4_0_FLADJ_BASE_IDX                                                           5
+#define regBIF_CFG_DEV2_EPF4_0_DBESL_DBESLD                                                             0x15018
+#define regBIF_CFG_DEV2_EPF4_0_DBESL_DBESLD_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CAP_LIST                                                            0x15019
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CAP                                                                 0x15019
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CAP                                                               0x1501a
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CNTL                                                              0x1501b
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_STATUS                                                            0x1501b
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CAP                                                                 0x1501c
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CNTL                                                                0x1501d
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_LINK_STATUS                                                              0x1501d
+#define regBIF_CFG_DEV2_EPF4_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CAP2                                                              0x15022
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2                                                             0x15023
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_STATUS2                                                           0x15023
+#define regBIF_CFG_DEV2_EPF4_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CAP2                                                                0x15024
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CNTL2                                                               0x15025
+#define regBIF_CFG_DEV2_EPF4_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF4_0_LINK_STATUS2                                                             0x15025
+#define regBIF_CFG_DEV2_EPF4_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_CAP_LIST                                                             0x15028
+#define regBIF_CFG_DEV2_EPF4_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL                                                             0x15028
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_LO                                                          0x15029
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_HI                                                          0x1502a
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA                                                             0x1502a
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA                                                         0x1502a
+#define regBIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MASK                                                                 0x1502b
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA_64                                                          0x1502b
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA_64                                                      0x1502b
+#define regBIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MASK_64                                                              0x1502c
+#define regBIF_CFG_DEV2_EPF4_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_PENDING                                                              0x1502c
+#define regBIF_CFG_DEV2_EPF4_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF4_0_MSI_PENDING_64                                                           0x1502d
+#define regBIF_CFG_DEV2_EPF4_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_CAP_LIST                                                            0x15030
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL                                                            0x15030
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_TABLE                                                               0x15031
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_PBA                                                                 0x15032
+#define regBIF_CFG_DEV2_EPF4_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x15040
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x15041
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC1                                                    0x15042
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC2                                                    0x15043
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x15181
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS                                                   0x15182
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK                                                     0x15183
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x15184
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS                                                     0x15185
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK                                                       0x15186
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x15187
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG0                                                            0x15188
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG1                                                            0x15189
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG2                                                            0x1518a
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG3                                                            0x1518b
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG0                                                     0x1518f
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG1                                                     0x15190
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG2                                                     0x15191
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG3                                                     0x15192
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST                                                    0x151b7
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP                                                             0x151b8
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL                                                            0x151b8
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST                                                    0x151c7
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_RTR_DATA1                                                                0x151c8
+#define regBIF_CFG_DEV2_EPF4_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_RTR_DATA2                                                                0x151c9
+#define regBIF_CFG_DEV2_EPF4_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x151ca
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x151cb
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA                                                     0x151cc
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_CAP                                                      0x151cd
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST                                                    0x151ce
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP                                                             0x151cf
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_LATENCY_INDICATOR                                               0x151d0
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_STATUS                                                          0x151d1
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_CNTL                                                            0x151d1
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x151d2
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x151d2
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x151d2
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x151d2
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x151d3
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x151d3
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x151d3
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x151d3
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST                                                    0x151dc
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP                                                             0x151dd
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL                                                            0x151dd
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST                                                    0x151de
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CAP                                                            0x151df
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL                                                           0x151e0
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CAP                                                            0x151e1
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL                                                           0x151e2
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CAP                                                            0x151e3
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL                                                           0x151e4
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CAP                                                            0x151e5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL                                                           0x151e6
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CAP                                                            0x151e7
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL                                                           0x151e8
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CAP                                                            0x151e9
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL                                                           0x151ea
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST                                                  0x151f1
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP                                                           0x151f2
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL                                                          0x151f2
+#define regBIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf5_bifcfgdecp
+// base address: 0x10155000
+#define regBIF_CFG_DEV2_EPF5_0_VENDOR_ID                                                                0x15400
+#define regBIF_CFG_DEV2_EPF5_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_ID                                                                0x15400
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_COMMAND                                                                  0x15401
+#define regBIF_CFG_DEV2_EPF5_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF5_0_STATUS                                                                   0x15401
+#define regBIF_CFG_DEV2_EPF5_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF5_0_REVISION_ID                                                              0x15402
+#define regBIF_CFG_DEV2_EPF5_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_PROG_INTERFACE                                                           0x15402
+#define regBIF_CFG_DEV2_EPF5_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_SUB_CLASS                                                                0x15402
+#define regBIF_CFG_DEV2_EPF5_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_BASE_CLASS                                                               0x15402
+#define regBIF_CFG_DEV2_EPF5_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF5_0_CACHE_LINE                                                               0x15403
+#define regBIF_CFG_DEV2_EPF5_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF5_0_LATENCY                                                                  0x15403
+#define regBIF_CFG_DEV2_EPF5_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF5_0_HEADER                                                                   0x15403
+#define regBIF_CFG_DEV2_EPF5_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF5_0_BIST                                                                     0x15403
+#define regBIF_CFG_DEV2_EPF5_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_1                                                              0x15404
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_2                                                              0x15405
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_3                                                              0x15406
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_4                                                              0x15407
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_5                                                              0x15408
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_6                                                              0x15409
+#define regBIF_CFG_DEV2_EPF5_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_ADAPTER_ID                                                               0x1540b
+#define regBIF_CFG_DEV2_EPF5_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR                                                            0x1540c
+#define regBIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_CAP_PTR                                                                  0x1540d
+#define regBIF_CFG_DEV2_EPF5_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF5_0_INTERRUPT_LINE                                                           0x1540f
+#define regBIF_CFG_DEV2_EPF5_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_INTERRUPT_PIN                                                            0x1540f
+#define regBIF_CFG_DEV2_EPF5_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_MIN_GRANT                                                                0x1540f
+#define regBIF_CFG_DEV2_EPF5_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_MAX_LATENCY                                                              0x1540f
+#define regBIF_CFG_DEV2_EPF5_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST                                                          0x15412
+#define regBIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF5_0_ADAPTER_ID_W                                                             0x15413
+#define regBIF_CFG_DEV2_EPF5_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_PMI_CAP_LIST                                                             0x15414
+#define regBIF_CFG_DEV2_EPF5_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_PMI_CAP                                                                  0x15414
+#define regBIF_CFG_DEV2_EPF5_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL                                                          0x15415
+#define regBIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CAP_LIST                                                            0x15419
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CAP                                                                 0x15419
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CAP                                                               0x1541a
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CNTL                                                              0x1541b
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_STATUS                                                            0x1541b
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CAP                                                                 0x1541c
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CNTL                                                                0x1541d
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_LINK_STATUS                                                              0x1541d
+#define regBIF_CFG_DEV2_EPF5_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CAP2                                                              0x15422
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2                                                             0x15423
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_STATUS2                                                           0x15423
+#define regBIF_CFG_DEV2_EPF5_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CAP2                                                                0x15424
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CNTL2                                                               0x15425
+#define regBIF_CFG_DEV2_EPF5_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF5_0_LINK_STATUS2                                                             0x15425
+#define regBIF_CFG_DEV2_EPF5_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_CAP_LIST                                                             0x15428
+#define regBIF_CFG_DEV2_EPF5_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL                                                             0x15428
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_LO                                                          0x15429
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_HI                                                          0x1542a
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA                                                             0x1542a
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA                                                         0x1542a
+#define regBIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MASK                                                                 0x1542b
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA_64                                                          0x1542b
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA_64                                                      0x1542b
+#define regBIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MASK_64                                                              0x1542c
+#define regBIF_CFG_DEV2_EPF5_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_PENDING                                                              0x1542c
+#define regBIF_CFG_DEV2_EPF5_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF5_0_MSI_PENDING_64                                                           0x1542d
+#define regBIF_CFG_DEV2_EPF5_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_CAP_LIST                                                            0x15430
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL                                                            0x15430
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_TABLE                                                               0x15431
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_PBA                                                                 0x15432
+#define regBIF_CFG_DEV2_EPF5_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x15440
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x15441
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC1                                                    0x15442
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC2                                                    0x15443
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x15581
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS                                                   0x15582
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK                                                     0x15583
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x15584
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS                                                     0x15585
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK                                                       0x15586
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x15587
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG0                                                            0x15588
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG1                                                            0x15589
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG2                                                            0x1558a
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG3                                                            0x1558b
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG0                                                     0x1558f
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG1                                                     0x15590
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG2                                                     0x15591
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG3                                                     0x15592
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST                                                    0x155b7
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP                                                             0x155b8
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL                                                            0x155b8
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST                                                    0x155c7
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_RTR_DATA1                                                                0x155c8
+#define regBIF_CFG_DEV2_EPF5_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_RTR_DATA2                                                                0x155c9
+#define regBIF_CFG_DEV2_EPF5_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x155ca
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x155cb
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA                                                     0x155cc
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_CAP                                                      0x155cd
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST                                                    0x155ce
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP                                                             0x155cf
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_LATENCY_INDICATOR                                               0x155d0
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_STATUS                                                          0x155d1
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_CNTL                                                            0x155d1
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x155d2
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x155d2
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x155d2
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x155d2
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x155d3
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x155d3
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x155d3
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x155d3
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST                                                    0x155dc
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP                                                             0x155dd
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL                                                            0x155dd
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST                                                    0x155de
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CAP                                                            0x155df
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL                                                           0x155e0
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CAP                                                            0x155e1
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL                                                           0x155e2
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CAP                                                            0x155e3
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL                                                           0x155e4
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CAP                                                            0x155e5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL                                                           0x155e6
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CAP                                                            0x155e7
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL                                                           0x155e8
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CAP                                                            0x155e9
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL                                                           0x155ea
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST                                                  0x155f1
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP                                                           0x155f2
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL                                                          0x155f2
+#define regBIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf6_bifcfgdecp
+// base address: 0x10156000
+#define regBIF_CFG_DEV2_EPF6_0_VENDOR_ID                                                                0x15800
+#define regBIF_CFG_DEV2_EPF6_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_ID                                                                0x15800
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_COMMAND                                                                  0x15801
+#define regBIF_CFG_DEV2_EPF6_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF6_0_STATUS                                                                   0x15801
+#define regBIF_CFG_DEV2_EPF6_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF6_0_REVISION_ID                                                              0x15802
+#define regBIF_CFG_DEV2_EPF6_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_PROG_INTERFACE                                                           0x15802
+#define regBIF_CFG_DEV2_EPF6_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_SUB_CLASS                                                                0x15802
+#define regBIF_CFG_DEV2_EPF6_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_BASE_CLASS                                                               0x15802
+#define regBIF_CFG_DEV2_EPF6_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF6_0_CACHE_LINE                                                               0x15803
+#define regBIF_CFG_DEV2_EPF6_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF6_0_LATENCY                                                                  0x15803
+#define regBIF_CFG_DEV2_EPF6_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF6_0_HEADER                                                                   0x15803
+#define regBIF_CFG_DEV2_EPF6_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV2_EPF6_0_BIST                                                                     0x15803
+#define regBIF_CFG_DEV2_EPF6_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_1                                                              0x15804
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_2                                                              0x15805
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_3                                                              0x15806
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_4                                                              0x15807
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_5                                                              0x15808
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_6                                                              0x15809
+#define regBIF_CFG_DEV2_EPF6_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_ADAPTER_ID                                                               0x1580b
+#define regBIF_CFG_DEV2_EPF6_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR                                                            0x1580c
+#define regBIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_CAP_PTR                                                                  0x1580d
+#define regBIF_CFG_DEV2_EPF6_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF6_0_INTERRUPT_LINE                                                           0x1580f
+#define regBIF_CFG_DEV2_EPF6_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_INTERRUPT_PIN                                                            0x1580f
+#define regBIF_CFG_DEV2_EPF6_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_MIN_GRANT                                                                0x1580f
+#define regBIF_CFG_DEV2_EPF6_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_MAX_LATENCY                                                              0x1580f
+#define regBIF_CFG_DEV2_EPF6_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST                                                          0x15812
+#define regBIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF6_0_ADAPTER_ID_W                                                             0x15813
+#define regBIF_CFG_DEV2_EPF6_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_PMI_CAP_LIST                                                             0x15814
+#define regBIF_CFG_DEV2_EPF6_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_PMI_CAP                                                                  0x15814
+#define regBIF_CFG_DEV2_EPF6_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL                                                          0x15815
+#define regBIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CAP_LIST                                                            0x15819
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CAP                                                                 0x15819
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CAP                                                               0x1581a
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CNTL                                                              0x1581b
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_STATUS                                                            0x1581b
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CAP                                                                 0x1581c
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CNTL                                                                0x1581d
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_LINK_STATUS                                                              0x1581d
+#define regBIF_CFG_DEV2_EPF6_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CAP2                                                              0x15822
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2                                                             0x15823
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_STATUS2                                                           0x15823
+#define regBIF_CFG_DEV2_EPF6_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CAP2                                                                0x15824
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CNTL2                                                               0x15825
+#define regBIF_CFG_DEV2_EPF6_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF6_0_LINK_STATUS2                                                             0x15825
+#define regBIF_CFG_DEV2_EPF6_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_CAP_LIST                                                             0x15828
+#define regBIF_CFG_DEV2_EPF6_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL                                                             0x15828
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_LO                                                          0x15829
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_HI                                                          0x1582a
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA                                                             0x1582a
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA                                                         0x1582a
+#define regBIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MASK                                                                 0x1582b
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA_64                                                          0x1582b
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA_64                                                      0x1582b
+#define regBIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MASK_64                                                              0x1582c
+#define regBIF_CFG_DEV2_EPF6_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_PENDING                                                              0x1582c
+#define regBIF_CFG_DEV2_EPF6_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV2_EPF6_0_MSI_PENDING_64                                                           0x1582d
+#define regBIF_CFG_DEV2_EPF6_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_CAP_LIST                                                            0x15830
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL                                                            0x15830
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_TABLE                                                               0x15831
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_PBA                                                                 0x15832
+#define regBIF_CFG_DEV2_EPF6_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x15840
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x15841
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC1                                                    0x15842
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC2                                                    0x15843
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x15981
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS                                                   0x15982
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK                                                     0x15983
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x15984
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS                                                     0x15985
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK                                                       0x15986
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x15987
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG0                                                            0x15988
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG1                                                            0x15989
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG2                                                            0x1598a
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG3                                                            0x1598b
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG0                                                     0x1598f
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG1                                                     0x15990
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG2                                                     0x15991
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG3                                                     0x15992
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST                                                    0x159b7
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP                                                             0x159b8
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL                                                            0x159b8
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST                                                    0x159c7
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_RTR_DATA1                                                                0x159c8
+#define regBIF_CFG_DEV2_EPF6_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_RTR_DATA2                                                                0x159c9
+#define regBIF_CFG_DEV2_EPF6_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x159ca
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x159cb
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA                                                     0x159cc
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_CAP                                                      0x159cd
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST                                                    0x159ce
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP                                                             0x159cf
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_LATENCY_INDICATOR                                               0x159d0
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_STATUS                                                          0x159d1
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_CNTL                                                            0x159d1
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x159d2
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x159d2
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x159d2
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x159d2
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x159d3
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x159d3
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x159d3
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x159d3
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST                                                    0x159dc
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP                                                             0x159dd
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL                                                            0x159dd
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST                                                    0x159de
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CAP                                                            0x159df
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL                                                           0x159e0
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CAP                                                            0x159e1
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL                                                           0x159e2
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CAP                                                            0x159e3
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL                                                           0x159e4
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CAP                                                            0x159e5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL                                                           0x159e6
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CAP                                                            0x159e7
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL                                                           0x159e8
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CAP                                                            0x159e9
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL                                                           0x159ea
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST                                                  0x159f1
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP                                                           0x159f2
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL                                                          0x159f2
+#define regBIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_rcc_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DEV0_0_RCC_VDM_SUPPORT                                                                   0xc440
+#define regRCC_DEV0_0_RCC_VDM_SUPPORT_BASE_IDX                                                          5
+#define regRCC_DEV0_0_RCC_BUS_CNTL                                                                      0xc441
+#define regRCC_DEV0_0_RCC_BUS_CNTL_BASE_IDX                                                             5
+#define regRCC_DEV0_0_RCC_FEATURES_CONTROL_MISC                                                         0xc442
+#define regRCC_DEV0_0_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                5
+#define regRCC_DEV0_0_RCC_DEV0_LINK_CNTL                                                                0xc443
+#define regRCC_DEV0_0_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       5
+#define regRCC_DEV0_0_RCC_CMN_LINK_CNTL                                                                 0xc444
+#define regRCC_DEV0_0_RCC_CMN_LINK_CNTL_BASE_IDX                                                        5
+#define regRCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE                                                        0xc445
+#define regRCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               5
+#define regRCC_DEV0_0_RCC_LTR_LSWITCH_CNTL                                                              0xc446
+#define regRCC_DEV0_0_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     5
+#define regRCC_DEV0_0_RCC_MH_ARB_CNTL                                                                   0xc447
+#define regRCC_DEV0_0_RCC_MH_ARB_CNTL_BASE_IDX                                                          5
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0                                                            0xc448
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1                                                            0xc449
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   5
+
+
+// addressBlock: nbif0_rcc_ep_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_EP_DEV0_0_EP_PCIE_SCRATCH                                                                0xc44c
+#define regRCC_EP_DEV0_0_EP_PCIE_SCRATCH_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_0_EP_PCIE_CNTL                                                                   0xc44e
+#define regRCC_EP_DEV0_0_EP_PCIE_CNTL_BASE_IDX                                                          5
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_CNTL                                                               0xc44f
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_STATUS                                                             0xc450
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_STATUS_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL2                                                               0xc451
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL2_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_0_EP_PCIE_BUS_CNTL                                                               0xc452
+#define regRCC_EP_DEV0_0_EP_PCIE_BUS_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_0_EP_PCIE_CFG_CNTL                                                               0xc453
+#define regRCC_EP_DEV0_0_EP_PCIE_CFG_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL                                                            0xc454
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC                                                             0xc455
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2                                                            0xc456
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP                                                             0xc457
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0xc458
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL                                                            0xc458
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0xc458
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0xc459
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0xc459
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0xc459
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0xc459
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0xc45a
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0xc45a
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0xc45a
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_EP_PCIE_PME_CONTROL                                                            0xc45c
+#define regRCC_EP_DEV0_0_EP_PCIE_PME_CONTROL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_0_EP_PCIEP_RESERVED                                                              0xc45d
+#define regRCC_EP_DEV0_0_EP_PCIEP_RESERVED_BASE_IDX                                                     5
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_CNTL                                                                0xc45f
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID                                                        0xc460
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               5
+#define regRCC_EP_DEV0_0_EP_PCIE_ERR_CNTL                                                               0xc461
+#define regRCC_EP_DEV0_0_EP_PCIE_ERR_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL                                                                0xc462
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL                                                          0xc463
+#define regRCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DWN_DEV0_0_DN_PCIE_RESERVED                                                              0xc468
+#define regRCC_DWN_DEV0_0_DN_PCIE_RESERVED_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_0_DN_PCIE_SCRATCH                                                               0xc469
+#define regRCC_DWN_DEV0_0_DN_PCIE_SCRATCH_BASE_IDX                                                      5
+#define regRCC_DWN_DEV0_0_DN_PCIE_CNTL                                                                  0xc46b
+#define regRCC_DWN_DEV0_0_DN_PCIE_CNTL_BASE_IDX                                                         5
+#define regRCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL                                                           0xc46c
+#define regRCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  5
+#define regRCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2                                                              0xc46d
+#define regRCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL                                                              0xc46e
+#define regRCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL                                                              0xc46f
+#define regRCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_F0                                                              0xc470
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_F0_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC                                                            0xc471
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC_BASE_IDX                                                   5
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2                                                           0xc472
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DWNP_DEV0_0_PCIE_ERR_CNTL                                                                0xc475
+#define regRCC_DWNP_DEV0_0_PCIE_ERR_CNTL_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_0_PCIE_RX_CNTL                                                                 0xc476
+#define regRCC_DWNP_DEV0_0_PCIE_RX_CNTL_BASE_IDX                                                        5
+#define regRCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL                                                           0xc477
+#define regRCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  5
+#define regRCC_DWNP_DEV0_0_PCIE_LC_CNTL2                                                                0xc478
+#define regRCC_DWNP_DEV0_0_PCIE_LC_CNTL2_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_0_PCIEP_STRAP_MISC                                                             0xc479
+#define regRCC_DWNP_DEV0_0_PCIEP_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP                                                         0xc47a
+#define regRCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                5
+
+
+// addressBlock: nbif0_rcc_pfc_amdgfx_RCCPFCDEC
+// base address: 0x10134000
+#define regRCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL                                                              0xd040
+#define regRCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL_BASE_IDX                                                     5
+#define regRCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE                                                           0xd041
+#define regRCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE_BASE_IDX                                                  5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0                                                      0xd042
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1                                                      0xd043
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2                                                      0xd044
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3                                                      0xd045
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4                                                      0xd046
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5                                                      0xd047
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL                                                           0xd048
+#define regRCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_pfc_amdgfxaz_RCCPFCDEC
+// base address: 0x10134200
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL                                                            0xd0c0
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL_BASE_IDX                                                   5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE                                                         0xd0c1
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE_BASE_IDX                                                5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0                                                    0xd0c2
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                           5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_1                                                    0xd0c3
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                           5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_2                                                    0xd0c4
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                           5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_3                                                    0xd0c5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                           5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_4                                                    0xd0c6
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                           5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_5                                                    0xd0c7
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                           5
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_AUXPWR_CNTL                                                         0xd0c8
+#define regRCC_PFC_AMDGFXAZ_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                5
+
+
+// addressBlock: nbif0_rcc_pfc_psp_RCCPFCDEC
+// base address: 0x10134400
+#define regRCC_PFC_PSP_RCC_PFC_LTR_CNTL                                                                 0xd140
+#define regRCC_PFC_PSP_RCC_PFC_LTR_CNTL_BASE_IDX                                                        5
+#define regRCC_PFC_PSP_RCC_PFC_PME_RESTORE                                                              0xd141
+#define regRCC_PFC_PSP_RCC_PFC_PME_RESTORE_BASE_IDX                                                     5
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0                                                         0xd142
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                                5
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_1                                                         0xd143
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                                5
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_2                                                         0xd144
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                                5
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_3                                                         0xd145
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                                5
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_4                                                         0xd146
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                                5
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_5                                                         0xd147
+#define regRCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                                5
+#define regRCC_PFC_PSP_RCC_PFC_AUXPWR_CNTL                                                              0xd148
+#define regRCC_PFC_PSP_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_rcc_pfc_usb3_1_RCCPFCDEC
+// base address: 0x10134800
+#define regRCC_PFC_USB3_1_RCC_PFC_LTR_CNTL                                                              0xd240
+#define regRCC_PFC_USB3_1_RCC_PFC_LTR_CNTL_BASE_IDX                                                     5
+#define regRCC_PFC_USB3_1_RCC_PFC_PME_RESTORE                                                           0xd241
+#define regRCC_PFC_USB3_1_RCC_PFC_PME_RESTORE_BASE_IDX                                                  5
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0                                                      0xd242
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                             5
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_1                                                      0xd243
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                             5
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_2                                                      0xd244
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                             5
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_3                                                      0xd245
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                             5
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_4                                                      0xd246
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                             5
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_5                                                      0xd247
+#define regRCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                             5
+#define regRCC_PFC_USB3_1_RCC_PFC_AUXPWR_CNTL                                                           0xd248
+#define regRCC_PFC_USB3_1_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_pfc_acp_RCCPFCDEC
+// base address: 0x10134a00
+#define regRCC_PFC_ACP_RCC_PFC_LTR_CNTL                                                                 0xd2c0
+#define regRCC_PFC_ACP_RCC_PFC_LTR_CNTL_BASE_IDX                                                        5
+#define regRCC_PFC_ACP_RCC_PFC_PME_RESTORE                                                              0xd2c1
+#define regRCC_PFC_ACP_RCC_PFC_PME_RESTORE_BASE_IDX                                                     5
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0                                                         0xd2c2
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                                5
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_1                                                         0xd2c3
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                                5
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_2                                                         0xd2c4
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                                5
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_3                                                         0xd2c5
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                                5
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_4                                                         0xd2c6
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                                5
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_5                                                         0xd2c7
+#define regRCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                                5
+#define regRCC_PFC_ACP_RCC_PFC_AUXPWR_CNTL                                                              0xd2c8
+#define regRCC_PFC_ACP_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_rcc_pfc_az_RCCPFCDEC
+// base address: 0x10134c00
+#define regRCC_PFC_AZ_RCC_PFC_LTR_CNTL                                                                  0xd340
+#define regRCC_PFC_AZ_RCC_PFC_LTR_CNTL_BASE_IDX                                                         5
+#define regRCC_PFC_AZ_RCC_PFC_PME_RESTORE                                                               0xd341
+#define regRCC_PFC_AZ_RCC_PFC_PME_RESTORE_BASE_IDX                                                      5
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0                                                          0xd342
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                                 5
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_1                                                          0xd343
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                                 5
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_2                                                          0xd344
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                                 5
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_3                                                          0xd345
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                                 5
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_4                                                          0xd346
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                                 5
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_5                                                          0xd347
+#define regRCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                                 5
+#define regRCC_PFC_AZ_RCC_PFC_AUXPWR_CNTL                                                               0xd348
+#define regRCC_PFC_AZ_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                      5
+
+
+// addressBlock: nbif0_rcc_pfc_mp2_RCCPFCDEC
+// base address: 0x10134e00
+#define regRCC_PFC_MP2_RCC_PFC_LTR_CNTL                                                                 0xd3c0
+#define regRCC_PFC_MP2_RCC_PFC_LTR_CNTL_BASE_IDX                                                        5
+#define regRCC_PFC_MP2_RCC_PFC_PME_RESTORE                                                              0xd3c1
+#define regRCC_PFC_MP2_RCC_PFC_PME_RESTORE_BASE_IDX                                                     5
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0                                                         0xd3c2
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                                5
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_1                                                         0xd3c3
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                                5
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_2                                                         0xd3c4
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                                5
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_3                                                         0xd3c5
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                                5
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_4                                                         0xd3c6
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                                5
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_5                                                         0xd3c7
+#define regRCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                                5
+#define regRCC_PFC_MP2_RCC_PFC_AUXPWR_CNTL                                                              0xd3c8
+#define regRCC_PFC_MP2_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_rcc_dev1_RCCPORTDEC
+// base address: 0x10131200
+#define regRCC_DEV1_RCC_VDM_SUPPORT                                                                     0xc4c0
+#define regRCC_DEV1_RCC_VDM_SUPPORT_BASE_IDX                                                            5
+#define regRCC_DEV1_RCC_BUS_CNTL                                                                        0xc4c1
+#define regRCC_DEV1_RCC_BUS_CNTL_BASE_IDX                                                               5
+#define regRCC_DEV1_RCC_FEATURES_CONTROL_MISC                                                           0xc4c2
+#define regRCC_DEV1_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                  5
+#define regRCC_DEV1_RCC_DEV0_LINK_CNTL                                                                  0xc4c3
+#define regRCC_DEV1_RCC_DEV0_LINK_CNTL_BASE_IDX                                                         5
+#define regRCC_DEV1_RCC_CMN_LINK_CNTL                                                                   0xc4c4
+#define regRCC_DEV1_RCC_CMN_LINK_CNTL_BASE_IDX                                                          5
+#define regRCC_DEV1_RCC_EP_REQUESTERID_RESTORE                                                          0xc4c5
+#define regRCC_DEV1_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                                 5
+#define regRCC_DEV1_RCC_LTR_LSWITCH_CNTL                                                                0xc4c6
+#define regRCC_DEV1_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                       5
+#define regRCC_DEV1_RCC_MH_ARB_CNTL                                                                     0xc4c7
+#define regRCC_DEV1_RCC_MH_ARB_CNTL_BASE_IDX                                                            5
+#define regRCC_DEV1_RCC_MARGIN_PARAM_CNTL0                                                              0xc4c8
+#define regRCC_DEV1_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                     5
+#define regRCC_DEV1_RCC_MARGIN_PARAM_CNTL1                                                              0xc4c9
+#define regRCC_DEV1_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_rcc_ep_dev1_RCCPORTDEC
+// base address: 0x10131200
+#define regRCC_EP_DEV1_EP_PCIE_SCRATCH                                                                  0xc4cc
+#define regRCC_EP_DEV1_EP_PCIE_SCRATCH_BASE_IDX                                                         5
+#define regRCC_EP_DEV1_EP_PCIE_CNTL                                                                     0xc4ce
+#define regRCC_EP_DEV1_EP_PCIE_CNTL_BASE_IDX                                                            5
+#define regRCC_EP_DEV1_EP_PCIE_INT_CNTL                                                                 0xc4cf
+#define regRCC_EP_DEV1_EP_PCIE_INT_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV1_EP_PCIE_INT_STATUS                                                               0xc4d0
+#define regRCC_EP_DEV1_EP_PCIE_INT_STATUS_BASE_IDX                                                      5
+#define regRCC_EP_DEV1_EP_PCIE_RX_CNTL2                                                                 0xc4d1
+#define regRCC_EP_DEV1_EP_PCIE_RX_CNTL2_BASE_IDX                                                        5
+#define regRCC_EP_DEV1_EP_PCIE_BUS_CNTL                                                                 0xc4d2
+#define regRCC_EP_DEV1_EP_PCIE_BUS_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV1_EP_PCIE_CFG_CNTL                                                                 0xc4d3
+#define regRCC_EP_DEV1_EP_PCIE_CFG_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL                                                              0xc4d4
+#define regRCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                     5
+#define regRCC_EP_DEV1_EP_PCIE_STRAP_MISC                                                               0xc4d5
+#define regRCC_EP_DEV1_EP_PCIE_STRAP_MISC_BASE_IDX                                                      5
+#define regRCC_EP_DEV1_EP_PCIE_STRAP_MISC2                                                              0xc4d6
+#define regRCC_EP_DEV1_EP_PCIE_STRAP_MISC2_BASE_IDX                                                     5
+#define regRCC_EP_DEV1_EP_PCIE_F0_DPA_CAP                                                               0xc4d7
+#define regRCC_EP_DEV1_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                      5
+#define regRCC_EP_DEV1_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                                 0xc4d8
+#define regRCC_EP_DEV1_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                        5
+#define regRCC_EP_DEV1_EP_PCIE_F0_DPA_CNTL                                                              0xc4d8
+#define regRCC_EP_DEV1_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                     5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                                 0xc4d8
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                        5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                                 0xc4d9
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                        5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                                 0xc4d9
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                        5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                                 0xc4d9
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                        5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                                 0xc4d9
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                        5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                                 0xc4da
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                        5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                                 0xc4da
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                        5
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                                 0xc4da
+#define regRCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                        5
+#define regRCC_EP_DEV1_EP_PCIE_PME_CONTROL                                                              0xc4dc
+#define regRCC_EP_DEV1_EP_PCIE_PME_CONTROL_BASE_IDX                                                     5
+#define regRCC_EP_DEV1_EP_PCIEP_RESERVED                                                                0xc4dd
+#define regRCC_EP_DEV1_EP_PCIEP_RESERVED_BASE_IDX                                                       5
+#define regRCC_EP_DEV1_EP_PCIE_TX_CNTL                                                                  0xc4df
+#define regRCC_EP_DEV1_EP_PCIE_TX_CNTL_BASE_IDX                                                         5
+#define regRCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID                                                          0xc4e0
+#define regRCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                                 5
+#define regRCC_EP_DEV1_EP_PCIE_ERR_CNTL                                                                 0xc4e1
+#define regRCC_EP_DEV1_EP_PCIE_ERR_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV1_EP_PCIE_RX_CNTL                                                                  0xc4e2
+#define regRCC_EP_DEV1_EP_PCIE_RX_CNTL_BASE_IDX                                                         5
+#define regRCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL                                                            0xc4e3
+#define regRCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                   5
+
+
+// addressBlock: nbif0_rcc_dwn_dev1_RCCPORTDEC
+// base address: 0x10131200
+#define regRCC_DWN_DEV1_DN_PCIE_RESERVED                                                                0xc4e8
+#define regRCC_DWN_DEV1_DN_PCIE_RESERVED_BASE_IDX                                                       5
+#define regRCC_DWN_DEV1_DN_PCIE_SCRATCH                                                                 0xc4e9
+#define regRCC_DWN_DEV1_DN_PCIE_SCRATCH_BASE_IDX                                                        5
+#define regRCC_DWN_DEV1_DN_PCIE_CNTL                                                                    0xc4eb
+#define regRCC_DWN_DEV1_DN_PCIE_CNTL_BASE_IDX                                                           5
+#define regRCC_DWN_DEV1_DN_PCIE_CONFIG_CNTL                                                             0xc4ec
+#define regRCC_DWN_DEV1_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                    5
+#define regRCC_DWN_DEV1_DN_PCIE_RX_CNTL2                                                                0xc4ed
+#define regRCC_DWN_DEV1_DN_PCIE_RX_CNTL2_BASE_IDX                                                       5
+#define regRCC_DWN_DEV1_DN_PCIE_BUS_CNTL                                                                0xc4ee
+#define regRCC_DWN_DEV1_DN_PCIE_BUS_CNTL_BASE_IDX                                                       5
+#define regRCC_DWN_DEV1_DN_PCIE_CFG_CNTL                                                                0xc4ef
+#define regRCC_DWN_DEV1_DN_PCIE_CFG_CNTL_BASE_IDX                                                       5
+#define regRCC_DWN_DEV1_DN_PCIE_STRAP_F0                                                                0xc4f0
+#define regRCC_DWN_DEV1_DN_PCIE_STRAP_F0_BASE_IDX                                                       5
+#define regRCC_DWN_DEV1_DN_PCIE_STRAP_MISC                                                              0xc4f1
+#define regRCC_DWN_DEV1_DN_PCIE_STRAP_MISC_BASE_IDX                                                     5
+#define regRCC_DWN_DEV1_DN_PCIE_STRAP_MISC2                                                             0xc4f2
+#define regRCC_DWN_DEV1_DN_PCIE_STRAP_MISC2_BASE_IDX                                                    5
+
+
+// addressBlock: nbif0_rcc_dwnp_dev1_RCCPORTDEC
+// base address: 0x10131200
+#define regRCC_DWNP_DEV1_PCIE_ERR_CNTL                                                                  0xc4f5
+#define regRCC_DWNP_DEV1_PCIE_ERR_CNTL_BASE_IDX                                                         5
+#define regRCC_DWNP_DEV1_PCIE_RX_CNTL                                                                   0xc4f6
+#define regRCC_DWNP_DEV1_PCIE_RX_CNTL_BASE_IDX                                                          5
+#define regRCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL                                                             0xc4f7
+#define regRCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL_BASE_IDX                                                    5
+#define regRCC_DWNP_DEV1_PCIE_LC_CNTL2                                                                  0xc4f8
+#define regRCC_DWNP_DEV1_PCIE_LC_CNTL2_BASE_IDX                                                         5
+#define regRCC_DWNP_DEV1_PCIEP_STRAP_MISC                                                               0xc4f9
+#define regRCC_DWNP_DEV1_PCIEP_STRAP_MISC_BASE_IDX                                                      5
+#define regRCC_DWNP_DEV1_LTR_MSG_INFO_FROM_EP                                                           0xc4fa
+#define regRCC_DWNP_DEV1_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_dev2_RCCPORTDEC
+// base address: 0x10131400
+#define regRCC_DEV2_RCC_VDM_SUPPORT                                                                     0xc540
+#define regRCC_DEV2_RCC_VDM_SUPPORT_BASE_IDX                                                            5
+#define regRCC_DEV2_RCC_BUS_CNTL                                                                        0xc541
+#define regRCC_DEV2_RCC_BUS_CNTL_BASE_IDX                                                               5
+#define regRCC_DEV2_RCC_FEATURES_CONTROL_MISC                                                           0xc542
+#define regRCC_DEV2_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                  5
+#define regRCC_DEV2_RCC_DEV0_LINK_CNTL                                                                  0xc543
+#define regRCC_DEV2_RCC_DEV0_LINK_CNTL_BASE_IDX                                                         5
+#define regRCC_DEV2_RCC_CMN_LINK_CNTL                                                                   0xc544
+#define regRCC_DEV2_RCC_CMN_LINK_CNTL_BASE_IDX                                                          5
+#define regRCC_DEV2_RCC_EP_REQUESTERID_RESTORE                                                          0xc545
+#define regRCC_DEV2_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                                 5
+#define regRCC_DEV2_RCC_LTR_LSWITCH_CNTL                                                                0xc546
+#define regRCC_DEV2_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                       5
+#define regRCC_DEV2_RCC_MH_ARB_CNTL                                                                     0xc547
+#define regRCC_DEV2_RCC_MH_ARB_CNTL_BASE_IDX                                                            5
+#define regRCC_DEV2_RCC_MARGIN_PARAM_CNTL0                                                              0xc548
+#define regRCC_DEV2_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                     5
+#define regRCC_DEV2_RCC_MARGIN_PARAM_CNTL1                                                              0xc549
+#define regRCC_DEV2_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_rcc_ep_dev2_RCCPORTDEC
+// base address: 0x10131400
+#define regRCC_EP_DEV2_EP_PCIE_SCRATCH                                                                  0xc54c
+#define regRCC_EP_DEV2_EP_PCIE_SCRATCH_BASE_IDX                                                         5
+#define regRCC_EP_DEV2_EP_PCIE_CNTL                                                                     0xc54e
+#define regRCC_EP_DEV2_EP_PCIE_CNTL_BASE_IDX                                                            5
+#define regRCC_EP_DEV2_EP_PCIE_INT_CNTL                                                                 0xc54f
+#define regRCC_EP_DEV2_EP_PCIE_INT_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV2_EP_PCIE_INT_STATUS                                                               0xc550
+#define regRCC_EP_DEV2_EP_PCIE_INT_STATUS_BASE_IDX                                                      5
+#define regRCC_EP_DEV2_EP_PCIE_RX_CNTL2                                                                 0xc551
+#define regRCC_EP_DEV2_EP_PCIE_RX_CNTL2_BASE_IDX                                                        5
+#define regRCC_EP_DEV2_EP_PCIE_BUS_CNTL                                                                 0xc552
+#define regRCC_EP_DEV2_EP_PCIE_BUS_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV2_EP_PCIE_CFG_CNTL                                                                 0xc553
+#define regRCC_EP_DEV2_EP_PCIE_CFG_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL                                                              0xc554
+#define regRCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                     5
+#define regRCC_EP_DEV2_EP_PCIE_STRAP_MISC                                                               0xc555
+#define regRCC_EP_DEV2_EP_PCIE_STRAP_MISC_BASE_IDX                                                      5
+#define regRCC_EP_DEV2_EP_PCIE_STRAP_MISC2                                                              0xc556
+#define regRCC_EP_DEV2_EP_PCIE_STRAP_MISC2_BASE_IDX                                                     5
+#define regRCC_EP_DEV2_EP_PCIE_F0_DPA_CAP                                                               0xc557
+#define regRCC_EP_DEV2_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                      5
+#define regRCC_EP_DEV2_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                                 0xc558
+#define regRCC_EP_DEV2_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                        5
+#define regRCC_EP_DEV2_EP_PCIE_F0_DPA_CNTL                                                              0xc558
+#define regRCC_EP_DEV2_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                     5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                                 0xc558
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                        5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                                 0xc559
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                        5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                                 0xc559
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                        5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                                 0xc559
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                        5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                                 0xc559
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                        5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                                 0xc55a
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                        5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                                 0xc55a
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                        5
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                                 0xc55a
+#define regRCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                        5
+#define regRCC_EP_DEV2_EP_PCIE_PME_CONTROL                                                              0xc55c
+#define regRCC_EP_DEV2_EP_PCIE_PME_CONTROL_BASE_IDX                                                     5
+#define regRCC_EP_DEV2_EP_PCIEP_RESERVED                                                                0xc55d
+#define regRCC_EP_DEV2_EP_PCIEP_RESERVED_BASE_IDX                                                       5
+#define regRCC_EP_DEV2_EP_PCIE_TX_CNTL                                                                  0xc55f
+#define regRCC_EP_DEV2_EP_PCIE_TX_CNTL_BASE_IDX                                                         5
+#define regRCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID                                                          0xc560
+#define regRCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                                 5
+#define regRCC_EP_DEV2_EP_PCIE_ERR_CNTL                                                                 0xc561
+#define regRCC_EP_DEV2_EP_PCIE_ERR_CNTL_BASE_IDX                                                        5
+#define regRCC_EP_DEV2_EP_PCIE_RX_CNTL                                                                  0xc562
+#define regRCC_EP_DEV2_EP_PCIE_RX_CNTL_BASE_IDX                                                         5
+#define regRCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL                                                            0xc563
+#define regRCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                   5
+
+
+// addressBlock: nbif0_rcc_dwn_dev2_RCCPORTDEC
+// base address: 0x10131400
+#define regRCC_DWN_DEV2_DN_PCIE_RESERVED                                                                0xc568
+#define regRCC_DWN_DEV2_DN_PCIE_RESERVED_BASE_IDX                                                       5
+#define regRCC_DWN_DEV2_DN_PCIE_SCRATCH                                                                 0xc569
+#define regRCC_DWN_DEV2_DN_PCIE_SCRATCH_BASE_IDX                                                        5
+#define regRCC_DWN_DEV2_DN_PCIE_CNTL                                                                    0xc56b
+#define regRCC_DWN_DEV2_DN_PCIE_CNTL_BASE_IDX                                                           5
+#define regRCC_DWN_DEV2_DN_PCIE_CONFIG_CNTL                                                             0xc56c
+#define regRCC_DWN_DEV2_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                    5
+#define regRCC_DWN_DEV2_DN_PCIE_RX_CNTL2                                                                0xc56d
+#define regRCC_DWN_DEV2_DN_PCIE_RX_CNTL2_BASE_IDX                                                       5
+#define regRCC_DWN_DEV2_DN_PCIE_BUS_CNTL                                                                0xc56e
+#define regRCC_DWN_DEV2_DN_PCIE_BUS_CNTL_BASE_IDX                                                       5
+#define regRCC_DWN_DEV2_DN_PCIE_CFG_CNTL                                                                0xc56f
+#define regRCC_DWN_DEV2_DN_PCIE_CFG_CNTL_BASE_IDX                                                       5
+#define regRCC_DWN_DEV2_DN_PCIE_STRAP_F0                                                                0xc570
+#define regRCC_DWN_DEV2_DN_PCIE_STRAP_F0_BASE_IDX                                                       5
+#define regRCC_DWN_DEV2_DN_PCIE_STRAP_MISC                                                              0xc571
+#define regRCC_DWN_DEV2_DN_PCIE_STRAP_MISC_BASE_IDX                                                     5
+#define regRCC_DWN_DEV2_DN_PCIE_STRAP_MISC2                                                             0xc572
+#define regRCC_DWN_DEV2_DN_PCIE_STRAP_MISC2_BASE_IDX                                                    5
+
+
+// addressBlock: nbif0_rcc_dwnp_dev2_RCCPORTDEC
+// base address: 0x10131400
+#define regRCC_DWNP_DEV2_PCIE_ERR_CNTL                                                                  0xc575
+#define regRCC_DWNP_DEV2_PCIE_ERR_CNTL_BASE_IDX                                                         5
+#define regRCC_DWNP_DEV2_PCIE_RX_CNTL                                                                   0xc576
+#define regRCC_DWNP_DEV2_PCIE_RX_CNTL_BASE_IDX                                                          5
+#define regRCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL                                                             0xc577
+#define regRCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL_BASE_IDX                                                    5
+#define regRCC_DWNP_DEV2_PCIE_LC_CNTL2                                                                  0xc578
+#define regRCC_DWNP_DEV2_PCIE_LC_CNTL2_BASE_IDX                                                         5
+#define regRCC_DWNP_DEV2_PCIEP_STRAP_MISC                                                               0xc579
+#define regRCC_DWNP_DEV2_PCIEP_STRAP_MISC_BASE_IDX                                                      5
+#define regRCC_DWNP_DEV2_LTR_MSG_INFO_FROM_EP                                                           0xc57a
+#define regRCC_DWNP_DEV2_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_pfc_usb3_0_RCCPFCDEC
+// base address: 0x10136000
+#define regRCC_PFC_USB3_0_RCC_PFC_LTR_CNTL                                                              0xd840
+#define regRCC_PFC_USB3_0_RCC_PFC_LTR_CNTL_BASE_IDX                                                     5
+#define regRCC_PFC_USB3_0_RCC_PFC_PME_RESTORE                                                           0xd841
+#define regRCC_PFC_USB3_0_RCC_PFC_PME_RESTORE_BASE_IDX                                                  5
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0                                                      0xd842
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                             5
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_1                                                      0xd843
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                             5
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_2                                                      0xd844
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                             5
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_3                                                      0xd845
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                             5
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_4                                                      0xd846
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                             5
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_5                                                      0xd847
+#define regRCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                             5
+#define regRCC_PFC_USB3_0_RCC_PFC_AUXPWR_CNTL                                                           0xd848
+#define regRCC_PFC_USB3_0_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_strap_rcc_strap_internal
+// base address: 0x10100000
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP0                                                              0xc400
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP1                                                              0xc401
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP2                                                              0xc402
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP3                                                              0xc403
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP4                                                              0xc404
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP5                                                              0xc405
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP6                                                              0xc406
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP7                                                              0xc407
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP8                                                              0xc408
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP9                                                              0xc409
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP10                                                             0xc40a
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP11                                                             0xc40b
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP12                                                             0xc40c
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP13                                                             0xc40d
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP14                                                             0xc40e
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP14_BASE_IDX                                                    5
+#define regRCC_DEV1_PORT_STRAP0                                                                         0xc480
+#define regRCC_DEV1_PORT_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP1                                                                         0xc481
+#define regRCC_DEV1_PORT_STRAP1_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP2                                                                         0xc482
+#define regRCC_DEV1_PORT_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP3                                                                         0xc483
+#define regRCC_DEV1_PORT_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP4                                                                         0xc484
+#define regRCC_DEV1_PORT_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP5                                                                         0xc485
+#define regRCC_DEV1_PORT_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP6                                                                         0xc486
+#define regRCC_DEV1_PORT_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP7                                                                         0xc487
+#define regRCC_DEV1_PORT_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP8                                                                         0xc488
+#define regRCC_DEV1_PORT_STRAP8_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP9                                                                         0xc489
+#define regRCC_DEV1_PORT_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP10                                                                        0xc48a
+#define regRCC_DEV1_PORT_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP11                                                                        0xc48b
+#define regRCC_DEV1_PORT_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP12                                                                        0xc48c
+#define regRCC_DEV1_PORT_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP13                                                                        0xc48d
+#define regRCC_DEV1_PORT_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP14                                                                        0xc48e
+#define regRCC_DEV1_PORT_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP0                                                                         0xc500
+#define regRCC_DEV2_PORT_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP1                                                                         0xc501
+#define regRCC_DEV2_PORT_STRAP1_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP2                                                                         0xc502
+#define regRCC_DEV2_PORT_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP3                                                                         0xc503
+#define regRCC_DEV2_PORT_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP4                                                                         0xc504
+#define regRCC_DEV2_PORT_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP5                                                                         0xc505
+#define regRCC_DEV2_PORT_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP6                                                                         0xc506
+#define regRCC_DEV2_PORT_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP7                                                                         0xc507
+#define regRCC_DEV2_PORT_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP8                                                                         0xc508
+#define regRCC_DEV2_PORT_STRAP8_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP9                                                                         0xc509
+#define regRCC_DEV2_PORT_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP10                                                                        0xc50a
+#define regRCC_DEV2_PORT_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP11                                                                        0xc50b
+#define regRCC_DEV2_PORT_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP12                                                                        0xc50c
+#define regRCC_DEV2_PORT_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP13                                                                        0xc50d
+#define regRCC_DEV2_PORT_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP14                                                                        0xc50e
+#define regRCC_DEV2_PORT_STRAP14_BASE_IDX                                                               5
+#define regRCC_STRAP0_RCC_BIF_STRAP0                                                                    0xc600
+#define regRCC_STRAP0_RCC_BIF_STRAP0_BASE_IDX                                                           5
+#define regRCC_STRAP0_RCC_BIF_STRAP1                                                                    0xc601
+#define regRCC_STRAP0_RCC_BIF_STRAP1_BASE_IDX                                                           5
+#define regRCC_STRAP0_RCC_BIF_STRAP2                                                                    0xc602
+#define regRCC_STRAP0_RCC_BIF_STRAP2_BASE_IDX                                                           5
+#define regRCC_STRAP0_RCC_BIF_STRAP3                                                                    0xc603
+#define regRCC_STRAP0_RCC_BIF_STRAP3_BASE_IDX                                                           5
+#define regRCC_STRAP0_RCC_BIF_STRAP4                                                                    0xc604
+#define regRCC_STRAP0_RCC_BIF_STRAP4_BASE_IDX                                                           5
+#define regRCC_STRAP0_RCC_BIF_STRAP5                                                                    0xc605
+#define regRCC_STRAP0_RCC_BIF_STRAP5_BASE_IDX                                                           5
+#define regRCC_STRAP0_RCC_BIF_STRAP6                                                                    0xc606
+#define regRCC_STRAP0_RCC_BIF_STRAP6_BASE_IDX                                                           5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0                                                              0xd000
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP1                                                              0xd001
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP2                                                              0xd002
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP3                                                              0xd003
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP4                                                              0xd004
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP5                                                              0xd005
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP8                                                              0xd008
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP9                                                              0xd009
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP13                                                             0xd00d
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP14                                                             0xd00e
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP15                                                             0xd00f
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP16                                                             0xd010
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP17                                                             0xd011
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP18                                                             0xd012
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP0                                                              0xd080
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP2                                                              0xd082
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP3                                                              0xd083
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP4                                                              0xd084
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP5                                                              0xd085
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP6                                                              0xd086
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP7                                                              0xd087
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP20                                                             0xd094
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP20_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP21                                                             0xd095
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP21_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP23                                                             0xd097
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP23_BASE_IDX                                                    5
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP24                                                             0xd098
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP24_BASE_IDX                                                    5
+#define regRCC_DEV0_EPF2_STRAP0                                                                         0xd100
+#define regRCC_DEV0_EPF2_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP2                                                                         0xd102
+#define regRCC_DEV0_EPF2_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP3                                                                         0xd103
+#define regRCC_DEV0_EPF2_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP4                                                                         0xd104
+#define regRCC_DEV0_EPF2_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP5                                                                         0xd105
+#define regRCC_DEV0_EPF2_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP6                                                                         0xd106
+#define regRCC_DEV0_EPF2_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP7                                                                         0xd107
+#define regRCC_DEV0_EPF2_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP10                                                                        0xd10a
+#define regRCC_DEV0_EPF2_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP11                                                                        0xd10b
+#define regRCC_DEV0_EPF2_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP12                                                                        0xd10c
+#define regRCC_DEV0_EPF2_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP13                                                                        0xd10d
+#define regRCC_DEV0_EPF2_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP14                                                                        0xd10e
+#define regRCC_DEV0_EPF2_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP20                                                                        0xd114
+#define regRCC_DEV0_EPF2_STRAP20_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP0                                                                         0xd180
+#define regRCC_DEV0_EPF3_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP2                                                                         0xd182
+#define regRCC_DEV0_EPF3_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP3                                                                         0xd183
+#define regRCC_DEV0_EPF3_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP4                                                                         0xd184
+#define regRCC_DEV0_EPF3_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP5                                                                         0xd185
+#define regRCC_DEV0_EPF3_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP6                                                                         0xd186
+#define regRCC_DEV0_EPF3_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP7                                                                         0xd187
+#define regRCC_DEV0_EPF3_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP10                                                                        0xd18a
+#define regRCC_DEV0_EPF3_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP11                                                                        0xd18b
+#define regRCC_DEV0_EPF3_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP12                                                                        0xd18c
+#define regRCC_DEV0_EPF3_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP13                                                                        0xd18d
+#define regRCC_DEV0_EPF3_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP14                                                                        0xd18e
+#define regRCC_DEV0_EPF3_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP20                                                                        0xd194
+#define regRCC_DEV0_EPF3_STRAP20_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF4_STRAP0                                                                         0xd200
+#define regRCC_DEV0_EPF4_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP2                                                                         0xd202
+#define regRCC_DEV0_EPF4_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP3                                                                         0xd203
+#define regRCC_DEV0_EPF4_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP4                                                                         0xd204
+#define regRCC_DEV0_EPF4_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP5                                                                         0xd205
+#define regRCC_DEV0_EPF4_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP6                                                                         0xd206
+#define regRCC_DEV0_EPF4_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP7                                                                         0xd207
+#define regRCC_DEV0_EPF4_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP13                                                                        0xd20d
+#define regRCC_DEV0_EPF4_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF4_STRAP14                                                                        0xd20e
+#define regRCC_DEV0_EPF4_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF5_STRAP0                                                                         0xd280
+#define regRCC_DEV0_EPF5_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP2                                                                         0xd282
+#define regRCC_DEV0_EPF5_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP3                                                                         0xd283
+#define regRCC_DEV0_EPF5_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP4                                                                         0xd284
+#define regRCC_DEV0_EPF5_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP5                                                                         0xd285
+#define regRCC_DEV0_EPF5_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP6                                                                         0xd286
+#define regRCC_DEV0_EPF5_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP7                                                                         0xd287
+#define regRCC_DEV0_EPF5_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP13                                                                        0xd28d
+#define regRCC_DEV0_EPF5_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF5_STRAP14                                                                        0xd28e
+#define regRCC_DEV0_EPF5_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF6_STRAP0                                                                         0xd300
+#define regRCC_DEV0_EPF6_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP2                                                                         0xd302
+#define regRCC_DEV0_EPF6_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP3                                                                         0xd303
+#define regRCC_DEV0_EPF6_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP4                                                                         0xd304
+#define regRCC_DEV0_EPF6_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP5                                                                         0xd305
+#define regRCC_DEV0_EPF6_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP6                                                                         0xd306
+#define regRCC_DEV0_EPF6_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP13                                                                        0xd30d
+#define regRCC_DEV0_EPF6_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF6_STRAP14                                                                        0xd30e
+#define regRCC_DEV0_EPF6_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF7_STRAP0                                                                         0xd380
+#define regRCC_DEV0_EPF7_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP2                                                                         0xd382
+#define regRCC_DEV0_EPF7_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP3                                                                         0xd383
+#define regRCC_DEV0_EPF7_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP4                                                                         0xd384
+#define regRCC_DEV0_EPF7_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP5                                                                         0xd385
+#define regRCC_DEV0_EPF7_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP6                                                                         0xd386
+#define regRCC_DEV0_EPF7_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP13                                                                        0xd38d
+#define regRCC_DEV0_EPF7_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF7_STRAP14                                                                        0xd38e
+#define regRCC_DEV0_EPF7_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF0_STRAP0                                                                         0xd400
+#define regRCC_DEV1_EPF0_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP2                                                                         0xd402
+#define regRCC_DEV1_EPF0_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP3                                                                         0xd403
+#define regRCC_DEV1_EPF0_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP4                                                                         0xd404
+#define regRCC_DEV1_EPF0_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP5                                                                         0xd405
+#define regRCC_DEV1_EPF0_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP6                                                                         0xd406
+#define regRCC_DEV1_EPF0_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP13                                                                        0xd40d
+#define regRCC_DEV1_EPF0_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF0_STRAP14                                                                        0xd40e
+#define regRCC_DEV1_EPF0_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP0                                                                         0xd480
+#define regRCC_DEV1_EPF1_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP2                                                                         0xd482
+#define regRCC_DEV1_EPF1_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP3                                                                         0xd483
+#define regRCC_DEV1_EPF1_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP4                                                                         0xd484
+#define regRCC_DEV1_EPF1_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP5                                                                         0xd485
+#define regRCC_DEV1_EPF1_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP6                                                                         0xd486
+#define regRCC_DEV1_EPF1_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP7                                                                         0xd487
+#define regRCC_DEV1_EPF1_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP0                                                                         0xd500
+#define regRCC_DEV1_EPF2_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP2                                                                         0xd502
+#define regRCC_DEV1_EPF2_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP3                                                                         0xd503
+#define regRCC_DEV1_EPF2_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP4                                                                         0xd504
+#define regRCC_DEV1_EPF2_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP5                                                                         0xd505
+#define regRCC_DEV1_EPF2_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP6                                                                         0xd506
+#define regRCC_DEV1_EPF2_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP0                                                                         0xd580
+#define regRCC_DEV1_EPF3_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP2                                                                         0xd582
+#define regRCC_DEV1_EPF3_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP3                                                                         0xd583
+#define regRCC_DEV1_EPF3_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP4                                                                         0xd584
+#define regRCC_DEV1_EPF3_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP5                                                                         0xd585
+#define regRCC_DEV1_EPF3_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP6                                                                         0xd586
+#define regRCC_DEV1_EPF3_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF4_STRAP0                                                                         0xd600
+#define regRCC_DEV1_EPF4_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF4_STRAP2                                                                         0xd602
+#define regRCC_DEV1_EPF4_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF4_STRAP3                                                                         0xd603
+#define regRCC_DEV1_EPF4_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF4_STRAP4                                                                         0xd604
+#define regRCC_DEV1_EPF4_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF4_STRAP5                                                                         0xd605
+#define regRCC_DEV1_EPF4_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF4_STRAP6                                                                         0xd606
+#define regRCC_DEV1_EPF4_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF5_STRAP0                                                                         0xd680
+#define regRCC_DEV1_EPF5_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF5_STRAP2                                                                         0xd682
+#define regRCC_DEV1_EPF5_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF5_STRAP3                                                                         0xd683
+#define regRCC_DEV1_EPF5_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF5_STRAP4                                                                         0xd684
+#define regRCC_DEV1_EPF5_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF5_STRAP5                                                                         0xd685
+#define regRCC_DEV1_EPF5_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF5_STRAP6                                                                         0xd686
+#define regRCC_DEV1_EPF5_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP0                                                                         0xd800
+#define regRCC_DEV2_EPF0_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP2                                                                         0xd802
+#define regRCC_DEV2_EPF0_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP3                                                                         0xd803
+#define regRCC_DEV2_EPF0_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP4                                                                         0xd804
+#define regRCC_DEV2_EPF0_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP5                                                                         0xd805
+#define regRCC_DEV2_EPF0_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP6                                                                         0xd806
+#define regRCC_DEV2_EPF0_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP7                                                                         0xd807
+#define regRCC_DEV2_EPF0_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP13                                                                        0xd80d
+#define regRCC_DEV2_EPF0_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF0_STRAP14                                                                        0xd80e
+#define regRCC_DEV2_EPF0_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF1_STRAP0                                                                         0xd880
+#define regRCC_DEV2_EPF1_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP2                                                                         0xd882
+#define regRCC_DEV2_EPF1_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP3                                                                         0xd883
+#define regRCC_DEV2_EPF1_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP4                                                                         0xd884
+#define regRCC_DEV2_EPF1_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP5                                                                         0xd885
+#define regRCC_DEV2_EPF1_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP6                                                                         0xd886
+#define regRCC_DEV2_EPF1_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP13                                                                        0xd88d
+#define regRCC_DEV2_EPF1_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF1_STRAP14                                                                        0xd88e
+#define regRCC_DEV2_EPF1_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF2_STRAP0                                                                         0xd900
+#define regRCC_DEV2_EPF2_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP2                                                                         0xd902
+#define regRCC_DEV2_EPF2_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP3                                                                         0xd903
+#define regRCC_DEV2_EPF2_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP4                                                                         0xd904
+#define regRCC_DEV2_EPF2_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP5                                                                         0xd905
+#define regRCC_DEV2_EPF2_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP6                                                                         0xd906
+#define regRCC_DEV2_EPF2_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP13                                                                        0xd90d
+#define regRCC_DEV2_EPF2_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF2_STRAP14                                                                        0xd90e
+#define regRCC_DEV2_EPF2_STRAP14_BASE_IDX                                                               5
+
+
+// addressBlock: nbif0_bif_rst_bif_rst_regblk
+// base address: 0x10100000
+#define regHARD_RST_CTRL                                                                                0xe000
+#define regHARD_RST_CTRL_BASE_IDX                                                                       5
+#define regRSMU_SOFT_RST_CTRL                                                                           0xe001
+#define regRSMU_SOFT_RST_CTRL_BASE_IDX                                                                  5
+#define regSELF_SOFT_RST                                                                                0xe002
+#define regSELF_SOFT_RST_BASE_IDX                                                                       5
+#define regBIF_GFX_DRV_VPU_RST                                                                          0xe003
+#define regBIF_GFX_DRV_VPU_RST_BASE_IDX                                                                 5
+#define regBIF_RST_MISC_CTRL                                                                            0xe004
+#define regBIF_RST_MISC_CTRL_BASE_IDX                                                                   5
+#define regBIF_RST_MISC_CTRL2                                                                           0xe005
+#define regBIF_RST_MISC_CTRL2_BASE_IDX                                                                  5
+#define regBIF_RST_MISC_CTRL3                                                                           0xe006
+#define regBIF_RST_MISC_CTRL3_BASE_IDX                                                                  5
+#define regDEV0_PF0_FLR_RST_CTRL                                                                        0xe008
+#define regDEV0_PF0_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF1_FLR_RST_CTRL                                                                        0xe009
+#define regDEV0_PF1_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF2_FLR_RST_CTRL                                                                        0xe00a
+#define regDEV0_PF2_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF3_FLR_RST_CTRL                                                                        0xe00b
+#define regDEV0_PF3_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF4_FLR_RST_CTRL                                                                        0xe00c
+#define regDEV0_PF4_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF5_FLR_RST_CTRL                                                                        0xe00d
+#define regDEV0_PF5_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF6_FLR_RST_CTRL                                                                        0xe00e
+#define regDEV0_PF6_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF7_FLR_RST_CTRL                                                                        0xe00f
+#define regDEV0_PF7_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regBIF_INST_RESET_INTR_STS                                                                      0xe010
+#define regBIF_INST_RESET_INTR_STS_BASE_IDX                                                             5
+#define regBIF_PF_FLR_INTR_STS                                                                          0xe011
+#define regBIF_PF_FLR_INTR_STS_BASE_IDX                                                                 5
+#define regBIF_D3HOTD0_INTR_STS                                                                         0xe012
+#define regBIF_D3HOTD0_INTR_STS_BASE_IDX                                                                5
+#define regBIF_POWER_INTR_STS                                                                           0xe014
+#define regBIF_POWER_INTR_STS_BASE_IDX                                                                  5
+#define regBIF_PF_DSTATE_INTR_STS                                                                       0xe015
+#define regBIF_PF_DSTATE_INTR_STS_BASE_IDX                                                              5
+#define regSELF_SOFT_RST_2                                                                              0xe016
+#define regSELF_SOFT_RST_2_BASE_IDX                                                                     5
+#define regBIF_INST_RESET_INTR_MASK                                                                     0xe020
+#define regBIF_INST_RESET_INTR_MASK_BASE_IDX                                                            5
+#define regBIF_PF_FLR_INTR_MASK                                                                         0xe021
+#define regBIF_PF_FLR_INTR_MASK_BASE_IDX                                                                5
+#define regBIF_D3HOTD0_INTR_MASK                                                                        0xe022
+#define regBIF_D3HOTD0_INTR_MASK_BASE_IDX                                                               5
+#define regBIF_POWER_INTR_MASK                                                                          0xe024
+#define regBIF_POWER_INTR_MASK_BASE_IDX                                                                 5
+#define regBIF_PF_DSTATE_INTR_MASK                                                                      0xe025
+#define regBIF_PF_DSTATE_INTR_MASK_BASE_IDX                                                             5
+#define regBIF_PF_FLR_RST                                                                               0xe040
+#define regBIF_PF_FLR_RST_BASE_IDX                                                                      5
+#define regBIF_DEV0_PF0_DSTATE_VALUE                                                                    0xe050
+#define regBIF_DEV0_PF0_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF1_DSTATE_VALUE                                                                    0xe051
+#define regBIF_DEV0_PF1_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF2_DSTATE_VALUE                                                                    0xe052
+#define regBIF_DEV0_PF2_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF3_DSTATE_VALUE                                                                    0xe053
+#define regBIF_DEV0_PF3_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF4_DSTATE_VALUE                                                                    0xe054
+#define regBIF_DEV0_PF4_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF5_DSTATE_VALUE                                                                    0xe055
+#define regBIF_DEV0_PF5_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF6_DSTATE_VALUE                                                                    0xe056
+#define regBIF_DEV0_PF6_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF7_DSTATE_VALUE                                                                    0xe057
+#define regBIF_DEV0_PF7_DSTATE_VALUE_BASE_IDX                                                           5
+#define regDEV0_PF0_D3HOTD0_RST_CTRL                                                                    0xe07e
+#define regDEV0_PF0_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF1_D3HOTD0_RST_CTRL                                                                    0xe07f
+#define regDEV0_PF1_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF2_D3HOTD0_RST_CTRL                                                                    0xe080
+#define regDEV0_PF2_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF3_D3HOTD0_RST_CTRL                                                                    0xe081
+#define regDEV0_PF3_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF4_D3HOTD0_RST_CTRL                                                                    0xe082
+#define regDEV0_PF4_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF5_D3HOTD0_RST_CTRL                                                                    0xe083
+#define regDEV0_PF5_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF6_D3HOTD0_RST_CTRL                                                                    0xe084
+#define regDEV0_PF6_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF7_D3HOTD0_RST_CTRL                                                                    0xe085
+#define regDEV0_PF7_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV1_PF0_FLR_RST_CTRL                                                                        0xe200
+#define regDEV1_PF0_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV1_PF1_FLR_RST_CTRL                                                                        0xe201
+#define regDEV1_PF1_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV1_PF2_FLR_RST_CTRL                                                                        0xe202
+#define regDEV1_PF2_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV1_PF3_FLR_RST_CTRL                                                                        0xe203
+#define regDEV1_PF3_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV1_PF4_FLR_RST_CTRL                                                                        0xe204
+#define regDEV1_PF4_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regBIF_DEV1_PF0_DSTATE_VALUE                                                                    0xe208
+#define regBIF_DEV1_PF0_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV1_PF1_DSTATE_VALUE                                                                    0xe209
+#define regBIF_DEV1_PF1_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV1_PF2_DSTATE_VALUE                                                                    0xe20a
+#define regBIF_DEV1_PF2_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV1_PF3_DSTATE_VALUE                                                                    0xe20b
+#define regBIF_DEV1_PF3_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV1_PF4_DSTATE_VALUE                                                                    0xe20c
+#define regBIF_DEV1_PF4_DSTATE_VALUE_BASE_IDX                                                           5
+#define regDEV1_PF0_D3HOTD0_RST_CTRL                                                                    0xe210
+#define regDEV1_PF0_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV1_PF1_D3HOTD0_RST_CTRL                                                                    0xe211
+#define regDEV1_PF1_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV1_PF2_D3HOTD0_RST_CTRL                                                                    0xe212
+#define regDEV1_PF2_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV1_PF3_D3HOTD0_RST_CTRL                                                                    0xe213
+#define regDEV1_PF3_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV1_PF4_D3HOTD0_RST_CTRL                                                                    0xe214
+#define regDEV1_PF4_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV2_PF0_FLR_RST_CTRL                                                                        0xe218
+#define regDEV2_PF0_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV2_PF1_FLR_RST_CTRL                                                                        0xe219
+#define regDEV2_PF1_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV2_PF2_FLR_RST_CTRL                                                                        0xe21a
+#define regDEV2_PF2_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV2_PF3_FLR_RST_CTRL                                                                        0xe21b
+#define regDEV2_PF3_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV2_PF4_FLR_RST_CTRL                                                                        0xe21c
+#define regDEV2_PF4_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV2_PF5_FLR_RST_CTRL                                                                        0xe21d
+#define regDEV2_PF5_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV2_PF6_FLR_RST_CTRL                                                                        0xe21e
+#define regDEV2_PF6_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regBIF_DEV2_PF0_DSTATE_VALUE                                                                    0xe220
+#define regBIF_DEV2_PF0_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV2_PF1_DSTATE_VALUE                                                                    0xe221
+#define regBIF_DEV2_PF1_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV2_PF2_DSTATE_VALUE                                                                    0xe222
+#define regBIF_DEV2_PF2_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV2_PF3_DSTATE_VALUE                                                                    0xe223
+#define regBIF_DEV2_PF3_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV2_PF4_DSTATE_VALUE                                                                    0xe224
+#define regBIF_DEV2_PF4_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV2_PF5_DSTATE_VALUE                                                                    0xe225
+#define regBIF_DEV2_PF5_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV2_PF6_DSTATE_VALUE                                                                    0xe226
+#define regBIF_DEV2_PF6_DSTATE_VALUE_BASE_IDX                                                           5
+#define regDEV2_PF0_D3HOTD0_RST_CTRL                                                                    0xe228
+#define regDEV2_PF0_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV2_PF1_D3HOTD0_RST_CTRL                                                                    0xe229
+#define regDEV2_PF1_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV2_PF2_D3HOTD0_RST_CTRL                                                                    0xe22a
+#define regDEV2_PF2_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV2_PF3_D3HOTD0_RST_CTRL                                                                    0xe22b
+#define regDEV2_PF3_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV2_PF4_D3HOTD0_RST_CTRL                                                                    0xe22c
+#define regDEV2_PF4_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV2_PF5_D3HOTD0_RST_CTRL                                                                    0xe22d
+#define regDEV2_PF5_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV2_PF6_D3HOTD0_RST_CTRL                                                                    0xe22e
+#define regDEV2_PF6_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regBIF_PORT0_DSTATE_VALUE                                                                       0xe230
+#define regBIF_PORT0_DSTATE_VALUE_BASE_IDX                                                              5
+#define regBIF_PORT1_DSTATE_VALUE                                                                       0xe231
+#define regBIF_PORT1_DSTATE_VALUE_BASE_IDX                                                              5
+#define regBIF_PORT2_DSTATE_VALUE                                                                       0xe232
+#define regBIF_PORT2_DSTATE_VALUE_BASE_IDX                                                              5
+
+
+// addressBlock: nbif0_bif_misc_bif_misc_regblk
+// base address: 0x10100000
+#define regREGS_ROM_OFFSET_CTRL                                                                         0xcc23
+#define regREGS_ROM_OFFSET_CTRL_BASE_IDX                                                                5
+#define regNBIF_STRAP_BIOS_CNTL                                                                         0xcc89
+#define regNBIF_STRAP_BIOS_CNTL_BASE_IDX                                                                5
+#define regMISC_SCRATCH                                                                                 0xe800
+#define regMISC_SCRATCH_BASE_IDX                                                                        5
+#define regINTR_LINE_POLARITY                                                                           0xe801
+#define regINTR_LINE_POLARITY_BASE_IDX                                                                  5
+#define regINTR_LINE_ENABLE                                                                             0xe802
+#define regINTR_LINE_ENABLE_BASE_IDX                                                                    5
+#define regOUTSTANDING_VC_ALLOC                                                                         0xe803
+#define regOUTSTANDING_VC_ALLOC_BASE_IDX                                                                5
+#define regBIFC_MISC_CTRL0                                                                              0xe804
+#define regBIFC_MISC_CTRL0_BASE_IDX                                                                     5
+#define regBIFC_MISC_CTRL1                                                                              0xe805
+#define regBIFC_MISC_CTRL1_BASE_IDX                                                                     5
+#define regBIFC_BME_ERR_LOG_LB                                                                          0xe806
+#define regBIFC_BME_ERR_LOG_LB_BASE_IDX                                                                 5
+#define regBIFC_LC_TIMER_CTRL                                                                           0xe807
+#define regBIFC_LC_TIMER_CTRL_BASE_IDX                                                                  5
+#define regBIFC_RCCBIH_BME_ERR_LOG0                                                                     0xe808
+#define regBIFC_RCCBIH_BME_ERR_LOG0_BASE_IDX                                                            5
+#define regBIFC_RCCBIH_BME_ERR_LOG1                                                                     0xe809
+#define regBIFC_RCCBIH_BME_ERR_LOG1_BASE_IDX                                                            5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1                                                            0xe80a
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3                                                            0xe80b
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5                                                            0xe80c
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7                                                            0xe80d
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1                                                            0xe80e
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3                                                            0xe80f
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5                                                            0xe810
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7                                                            0xe811
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1                                                            0xe812
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3                                                            0xe813
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5                                                            0xe814
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7                                                            0xe815
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_CNTL2_DEV0                                                                     0xe81a
+#define regBIFC_DMA_ATTR_CNTL2_DEV0_BASE_IDX                                                            5
+#define regBIFC_DMA_ATTR_CNTL2_DEV1                                                                     0xe81b
+#define regBIFC_DMA_ATTR_CNTL2_DEV1_BASE_IDX                                                            5
+#define regBIFC_DMA_ATTR_CNTL2_DEV2                                                                     0xe81c
+#define regBIFC_DMA_ATTR_CNTL2_DEV2_BASE_IDX                                                            5
+#define regBIFC_MISC_CTRL2                                                                              0xe822
+#define regBIFC_MISC_CTRL2_BASE_IDX                                                                     5
+#define regBME_DUMMY_CNTL_0                                                                             0xe825
+#define regBME_DUMMY_CNTL_0_BASE_IDX                                                                    5
+#define regBME_DUMMY_CNTL_1                                                                             0xe826
+#define regBME_DUMMY_CNTL_1_BASE_IDX                                                                    5
+#define regBIFC_THT_CNTL                                                                                0xe827
+#define regBIFC_THT_CNTL_BASE_IDX                                                                       5
+#define regBIFC_HSTARB_CNTL                                                                             0xe828
+#define regBIFC_HSTARB_CNTL_BASE_IDX                                                                    5
+#define regBIFC_GSI_CNTL                                                                                0xe829
+#define regBIFC_GSI_CNTL_BASE_IDX                                                                       5
+#define regBIFC_PCIEFUNC_CNTL                                                                           0xe82a
+#define regBIFC_PCIEFUNC_CNTL_BASE_IDX                                                                  5
+#define regBIFC_PASID_CHECK_DIS                                                                         0xe82b
+#define regBIFC_PASID_CHECK_DIS_BASE_IDX                                                                5
+#define regBIFC_SDP_CNTL_0                                                                              0xe82c
+#define regBIFC_SDP_CNTL_0_BASE_IDX                                                                     5
+#define regBIFC_SDP_CNTL_1                                                                              0xe82d
+#define regBIFC_SDP_CNTL_1_BASE_IDX                                                                     5
+#define regBIFC_PASID_STS                                                                               0xe82e
+#define regBIFC_PASID_STS_BASE_IDX                                                                      5
+#define regBIFC_ATHUB_ACT_CNTL                                                                          0xe82f
+#define regBIFC_ATHUB_ACT_CNTL_BASE_IDX                                                                 5
+#define regBIFC_PERF_CNTL_0                                                                             0xe830
+#define regBIFC_PERF_CNTL_0_BASE_IDX                                                                    5
+#define regBIFC_PERF_CNTL_1                                                                             0xe831
+#define regBIFC_PERF_CNTL_1_BASE_IDX                                                                    5
+#define regBIFC_PERF_CNT_MMIO_RD_L32BIT                                                                 0xe832
+#define regBIFC_PERF_CNT_MMIO_RD_L32BIT_BASE_IDX                                                        5
+#define regBIFC_PERF_CNT_MMIO_WR_L32BIT                                                                 0xe833
+#define regBIFC_PERF_CNT_MMIO_WR_L32BIT_BASE_IDX                                                        5
+#define regBIFC_PERF_CNT_DMA_RD_L32BIT                                                                  0xe834
+#define regBIFC_PERF_CNT_DMA_RD_L32BIT_BASE_IDX                                                         5
+#define regBIFC_PERF_CNT_DMA_WR_L32BIT                                                                  0xe835
+#define regBIFC_PERF_CNT_DMA_WR_L32BIT_BASE_IDX                                                         5
+#define regNBIF_REGIF_ERRSET_CTRL                                                                       0xe836
+#define regNBIF_REGIF_ERRSET_CTRL_BASE_IDX                                                              5
+#define regBIFC_SDP_CNTL_2                                                                              0xe837
+#define regBIFC_SDP_CNTL_2_BASE_IDX                                                                     5
+#define regNBIF_PGMST_CTRL                                                                              0xe838
+#define regNBIF_PGMST_CTRL_BASE_IDX                                                                     5
+#define regNBIF_PGSLV_CTRL                                                                              0xe839
+#define regNBIF_PGSLV_CTRL_BASE_IDX                                                                     5
+#define regNBIF_PG_MISC_CTRL                                                                            0xe83a
+#define regNBIF_PG_MISC_CTRL_BASE_IDX                                                                   5
+#define regNBIF_HST_MISC_CTRL                                                                           0xe83b
+#define regNBIF_HST_MISC_CTRL_BASE_IDX                                                                  5
+#define regSMN_MST_EP_CNTL3                                                                             0xe83c
+#define regSMN_MST_EP_CNTL3_BASE_IDX                                                                    5
+#define regSMN_MST_EP_CNTL4                                                                             0xe83d
+#define regSMN_MST_EP_CNTL4_BASE_IDX                                                                    5
+#define regSMN_MST_CNTL1                                                                                0xe83e
+#define regSMN_MST_CNTL1_BASE_IDX                                                                       5
+#define regSMN_MST_EP_CNTL5                                                                             0xe83f
+#define regSMN_MST_EP_CNTL5_BASE_IDX                                                                    5
+#define regBIF_SELFRING_BUFFER_VID                                                                      0xe840
+#define regBIF_SELFRING_BUFFER_VID_BASE_IDX                                                             5
+#define regBIF_SELFRING_VECTOR_CNTL                                                                     0xe841
+#define regBIF_SELFRING_VECTOR_CNTL_BASE_IDX                                                            5
+#define regNBIF_STRAP_WRITE_CTRL                                                                        0xe845
+#define regNBIF_STRAP_WRITE_CTRL_BASE_IDX                                                               5
+#define regNBIF_INTX_DSTATE_MISC_CNTL                                                                   0xe846
+#define regNBIF_INTX_DSTATE_MISC_CNTL_BASE_IDX                                                          5
+#define regNBIF_PENDING_MISC_CNTL                                                                       0xe847
+#define regNBIF_PENDING_MISC_CNTL_BASE_IDX                                                              5
+#define regBIF_GMI_WRR_WEIGHT                                                                           0xe848
+#define regBIF_GMI_WRR_WEIGHT_BASE_IDX                                                                  5
+#define regBIF_GMI_WRR_WEIGHT2                                                                          0xe849
+#define regBIF_GMI_WRR_WEIGHT2_BASE_IDX                                                                 5
+#define regBIF_GMI_WRR_WEIGHT3                                                                          0xe84a
+#define regBIF_GMI_WRR_WEIGHT3_BASE_IDX                                                                 5
+#define regNBIF_PWRBRK_REQUEST                                                                          0xe84c
+#define regNBIF_PWRBRK_REQUEST_BASE_IDX                                                                 5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F0                                                                   0xe850
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F0_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F1                                                                   0xe851
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F1_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F2                                                                   0xe852
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F2_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F3                                                                   0xe853
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F3_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F4                                                                   0xe854
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F4_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F5                                                                   0xe855
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F5_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F6                                                                   0xe856
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F6_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F7                                                                   0xe857
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F7_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV1_F0                                                                   0xe858
+#define regBIF_ATOMIC_ERR_LOG_DEV1_F0_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV1_F1                                                                   0xe859
+#define regBIF_ATOMIC_ERR_LOG_DEV1_F1_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F0                                                                   0xe85d
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F0_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F1                                                                   0xe85e
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F1_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F2                                                                   0xe85f
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F2_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F3                                                                   0xe860
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F3_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F4                                                                   0xe861
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F4_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F5                                                                   0xe862
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F5_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F6                                                                   0xe863
+#define regBIF_ATOMIC_ERR_LOG_DEV2_F6_BASE_IDX                                                          5
+#define regBIF_DMA_MP4_ERR_LOG                                                                          0xe870
+#define regBIF_DMA_MP4_ERR_LOG_BASE_IDX                                                                 5
+#define regBIF_PASID_ERR_LOG                                                                            0xe871
+#define regBIF_PASID_ERR_LOG_BASE_IDX                                                                   5
+#define regBIF_PASID_ERR_CLR                                                                            0xe872
+#define regBIF_PASID_ERR_CLR_BASE_IDX                                                                   5
+#define regOBFF_EMU_CFG                                                                                 0xe874
+#define regOBFF_EMU_CFG_BASE_IDX                                                                        5
+#define regEP0_INTR_URGENT_CAP                                                                          0xe875
+#define regEP0_INTR_URGENT_CAP_BASE_IDX                                                                 5
+#define regEP1_INTR_URGENT_CAP                                                                          0xe876
+#define regEP1_INTR_URGENT_CAP_BASE_IDX                                                                 5
+#define regEP2_INTR_URGENT_CAP                                                                          0xe877
+#define regEP2_INTR_URGENT_CAP_BASE_IDX                                                                 5
+#define regEP_PEND_BLOCK_MSK                                                                            0xe87c
+#define regEP_PEND_BLOCK_MSK_BASE_IDX                                                                   5
+#define regNBIF_VWIRE_CTRL                                                                              0xe880
+#define regNBIF_VWIRE_CTRL_BASE_IDX                                                                     5
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL                                                                   0xe881
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_BASE_IDX                                                          5
+#define regNBIF_SMN_VWR_VCHG_RST_CTRL0                                                                  0xe882
+#define regNBIF_SMN_VWR_VCHG_RST_CTRL0_BASE_IDX                                                         5
+#define regNBIF_SMN_VWR_VCHG_TRIG                                                                       0xe884
+#define regNBIF_SMN_VWR_VCHG_TRIG_BASE_IDX                                                              5
+#define regNBIF_SMN_VWR_WTRIG_CNTL                                                                      0xe885
+#define regNBIF_SMN_VWR_WTRIG_CNTL_BASE_IDX                                                             5
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_1                                                                 0xe886
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_1_BASE_IDX                                                        5
+#define regNBIF_MGCG_CTRL_LCLK                                                                          0xe887
+#define regNBIF_MGCG_CTRL_LCLK_BASE_IDX                                                                 5
+#define regNBIF_DS_CTRL_LCLK                                                                            0xe888
+#define regNBIF_DS_CTRL_LCLK_BASE_IDX                                                                   5
+#define regSMN_MST_CNTL0                                                                                0xe889
+#define regSMN_MST_CNTL0_BASE_IDX                                                                       5
+#define regSMN_MST_EP_CNTL1                                                                             0xe88a
+#define regSMN_MST_EP_CNTL1_BASE_IDX                                                                    5
+#define regSMN_MST_EP_CNTL2                                                                             0xe88b
+#define regSMN_MST_EP_CNTL2_BASE_IDX                                                                    5
+#define regNBIF_SDP_VWR_VCHG_DIS_CTRL                                                                   0xe88c
+#define regNBIF_SDP_VWR_VCHG_DIS_CTRL_BASE_IDX                                                          5
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL0                                                                  0xe88d
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL0_BASE_IDX                                                         5
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL1                                                                  0xe88e
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL1_BASE_IDX                                                         5
+#define regNBIF_SDP_VWR_VCHG_TRIG                                                                       0xe88f
+#define regNBIF_SDP_VWR_VCHG_TRIG_BASE_IDX                                                              5
+#define regNBIF_SHUB_TODET_CTRL                                                                         0xe898
+#define regNBIF_SHUB_TODET_CTRL_BASE_IDX                                                                5
+#define regNBIF_SHUB_TODET_CLIENT_CTRL                                                                  0xe899
+#define regNBIF_SHUB_TODET_CLIENT_CTRL_BASE_IDX                                                         5
+#define regNBIF_SHUB_TODET_CLIENT_STATUS                                                                0xe89a
+#define regNBIF_SHUB_TODET_CLIENT_STATUS_BASE_IDX                                                       5
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL                                                               0xe89b
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL_BASE_IDX                                                      5
+#define regNBIF_SHUB_TODET_CLIENT_CTRL2                                                                 0xe89c
+#define regNBIF_SHUB_TODET_CLIENT_CTRL2_BASE_IDX                                                        5
+#define regNBIF_SHUB_TODET_CLIENT_STATUS2                                                               0xe89d
+#define regNBIF_SHUB_TODET_CLIENT_STATUS2_BASE_IDX                                                      5
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL2                                                              0xe89e
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL2_BASE_IDX                                                     5
+#define regBIFC_BME_ERR_LOG_HB                                                                          0xe8ab
+#define regBIFC_BME_ERR_LOG_HB_BASE_IDX                                                                 5
+#define regNBIF_SMN_VWR_VCHG_TRIG_HI                                                                    0xe8ac
+#define regNBIF_SMN_VWR_VCHG_TRIG_HI_BASE_IDX                                                           5
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_HI                                                                0xe8b4
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_HI_BASE_IDX                                                       5
+#define regNBIF_SMN_VWR_VCHG_RST_CTRL0_HI                                                               0xe8b5
+#define regNBIF_SMN_VWR_VCHG_RST_CTRL0_HI_BASE_IDX                                                      5
+#define regNBIF_SMN_VWR_WTRIG_CNTL_HI                                                                   0xe8b6
+#define regNBIF_SMN_VWR_WTRIG_CNTL_HI_BASE_IDX                                                          5
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI                                                              0xe8b7
+#define regNBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI_BASE_IDX                                                     5
+#define regBIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC                                                            0xe8c0
+#define regBIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regBIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC                                                            0xe8c1
+#define regBIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regBIFC_GMI_SDP_REQ_POOLCRED_ALLOC                                                              0xe8c2
+#define regBIFC_GMI_SDP_REQ_POOLCRED_ALLOC_BASE_IDX                                                     5
+#define regBIFC_GMI_SDP_DAT_POOLCRED_ALLOC                                                              0xe8c3
+#define regBIFC_GMI_SDP_DAT_POOLCRED_ALLOC_BASE_IDX                                                     5
+#define regBIFC_GMI_SST_RDRSP_POOLCRED_ALLOC                                                            0xe8c4
+#define regBIFC_GMI_SST_RDRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regBIFC_GMI_SST_WRRSP_POOLCRED_ALLOC                                                            0xe8c5
+#define regBIFC_GMI_SST_WRRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regDISCON_HYSTERESIS_HEAD_CTRL                                                                  0xe8c6
+#define regDISCON_HYSTERESIS_HEAD_CTRL_BASE_IDX                                                         5
+#define regBIFC_Z10_CTRL0                                                                               0xe8c9
+#define regBIFC_Z10_CTRL0_BASE_IDX                                                                      5
+#define regBIFC_Z10_CTRL1                                                                               0xe8ca
+#define regBIFC_Z10_CTRL1_BASE_IDX                                                                      5
+#define regBIFC_Z10_STATUS                                                                              0xe8cc
+#define regBIFC_Z10_STATUS_BASE_IDX                                                                     5
+#define regBIFC_EARLY_WAKEUP_CNTL                                                                       0xe8d2
+#define regBIFC_EARLY_WAKEUP_CNTL_BASE_IDX                                                              5
+#define regBIFC_PERF_CNT_MMIO_RD_H16BIT                                                                 0xe8f0
+#define regBIFC_PERF_CNT_MMIO_RD_H16BIT_BASE_IDX                                                        5
+#define regBIFC_PERF_CNT_MMIO_WR_H16BIT                                                                 0xe8f1
+#define regBIFC_PERF_CNT_MMIO_WR_H16BIT_BASE_IDX                                                        5
+#define regBIFC_PERF_CNT_DMA_RD_H16BIT                                                                  0xe8f2
+#define regBIFC_PERF_CNT_DMA_RD_H16BIT_BASE_IDX                                                         5
+#define regBIFC_PERF_CNT_DMA_WR_H16BIT                                                                  0xe8f3
+#define regBIFC_PERF_CNT_DMA_WR_H16BIT_BASE_IDX                                                         5
+#define regNBIF_PERF_COM_COUNT_ENABLE                                                                   0xe8f4
+#define regNBIF_PERF_COM_COUNT_ENABLE_BASE_IDX                                                          5
+#define regNBIF_BX_PERF_CNT_FSM                                                                         0xe8ff
+#define regNBIF_BX_PERF_CNT_FSM_BASE_IDX                                                                5
+#define regNBIF_COM_COUNT_VALUE                                                                         0xe908
+#define regNBIF_COM_COUNT_VALUE_BASE_IDX                                                                5
+
+
+// addressBlock: nbif0_bif_ras_bif_ras_regblk
+// base address: 0x10100000
+#define regBIFL_RAS_CENTRAL_CNTL                                                                        0xe400
+#define regBIFL_RAS_CENTRAL_CNTL_BASE_IDX                                                               5
+#define regBIFL_RAS_CENTRAL_STATUS                                                                      0xe404
+#define regBIFL_RAS_CENTRAL_STATUS_BASE_IDX                                                             5
+#define regBIFL_RAS_LEAF0_CTRL                                                                          0xe408
+#define regBIFL_RAS_LEAF0_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF1_CTRL                                                                          0xe409
+#define regBIFL_RAS_LEAF1_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF2_CTRL                                                                          0xe40a
+#define regBIFL_RAS_LEAF2_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF3_CTRL                                                                          0xe40b
+#define regBIFL_RAS_LEAF3_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF0_STATUS                                                                        0xe434
+#define regBIFL_RAS_LEAF0_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF1_STATUS                                                                        0xe435
+#define regBIFL_RAS_LEAF1_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF2_STATUS                                                                        0xe436
+#define regBIFL_RAS_LEAF2_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF3_STATUS                                                                        0xe437
+#define regBIFL_RAS_LEAF3_STATUS_BASE_IDX                                                               5
+#define regBIFL_IOHUB_RAS_IH_CNTL                                                                       0xe7fe
+#define regBIFL_IOHUB_RAS_IH_CNTL_BASE_IDX                                                              5
+#define regBIFL_RAS_VWR_FROM_IOHUB                                                                      0xe7ff
+#define regBIFL_RAS_VWR_FROM_IOHUB_BASE_IDX                                                             5
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DWN_DEV0_1_DN_PCIE_RESERVED                                                              0x8d88
+#define regRCC_DWN_DEV0_1_DN_PCIE_RESERVED_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_SCRATCH                                                               0x8d89
+#define regRCC_DWN_DEV0_1_DN_PCIE_SCRATCH_BASE_IDX                                                      5
+#define regRCC_DWN_DEV0_1_DN_PCIE_CNTL                                                                  0x8d8b
+#define regRCC_DWN_DEV0_1_DN_PCIE_CNTL_BASE_IDX                                                         5
+#define regRCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL                                                           0x8d8c
+#define regRCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  5
+#define regRCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2                                                              0x8d8d
+#define regRCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL                                                              0x8d8e
+#define regRCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL                                                              0x8d8f
+#define regRCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_F0                                                              0x8d90
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_F0_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC                                                            0x8d91
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC_BASE_IDX                                                   5
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2                                                           0x8d92
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DWNP_DEV0_1_PCIE_ERR_CNTL                                                                0x8d94
+#define regRCC_DWNP_DEV0_1_PCIE_ERR_CNTL_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_1_PCIE_RX_CNTL                                                                 0x8d95
+#define regRCC_DWNP_DEV0_1_PCIE_RX_CNTL_BASE_IDX                                                        5
+#define regRCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL                                                           0x8d96
+#define regRCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  5
+#define regRCC_DWNP_DEV0_1_PCIE_LC_CNTL2                                                                0x8d97
+#define regRCC_DWNP_DEV0_1_PCIE_LC_CNTL2_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_1_PCIEP_STRAP_MISC                                                             0x8d98
+#define regRCC_DWNP_DEV0_1_PCIEP_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP                                                         0x8d99
+#define regRCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                5
+
+
+// addressBlock: nbif0_rcc_ep_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_EP_DEV0_1_EP_PCIE_SCRATCH                                                                0x8d6b
+#define regRCC_EP_DEV0_1_EP_PCIE_SCRATCH_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_1_EP_PCIE_CNTL                                                                   0x8d6d
+#define regRCC_EP_DEV0_1_EP_PCIE_CNTL_BASE_IDX                                                          5
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_CNTL                                                               0x8d6e
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_STATUS                                                             0x8d6f
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_STATUS_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL2                                                               0x8d70
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL2_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_BUS_CNTL                                                               0x8d71
+#define regRCC_EP_DEV0_1_EP_PCIE_BUS_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_CFG_CNTL                                                               0x8d72
+#define regRCC_EP_DEV0_1_EP_PCIE_CFG_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL                                                            0x8d74
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0                                               0x8d75
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1                                               0x8d75
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2                                               0x8d75
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3                                               0x8d75
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4                                               0x8d76
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5                                               0x8d76
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6                                               0x8d76
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7                                               0x8d76
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC                                                             0x8d77
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2                                                            0x8d78
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP                                                             0x8d7a
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0x8d7b
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL                                                            0x8d7b
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0x8d7b
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0x8d7c
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0x8d7c
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0x8d7c
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0x8d7c
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0x8d7d
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0x8d7d
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0x8d7d
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_PME_CONTROL                                                            0x8d7d
+#define regRCC_EP_DEV0_1_EP_PCIE_PME_CONTROL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_EP_PCIEP_RESERVED                                                              0x8d7e
+#define regRCC_EP_DEV0_1_EP_PCIEP_RESERVED_BASE_IDX                                                     5
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_CNTL                                                                0x8d80
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID                                                        0x8d81
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               5
+#define regRCC_EP_DEV0_1_EP_PCIE_ERR_CNTL                                                               0x8d82
+#define regRCC_EP_DEV0_1_EP_PCIE_ERR_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL                                                                0x8d83
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL                                                          0x8d84
+#define regRCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_rcc_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DEV0_0_RCC_ERR_INT_CNTL                                                                  0x8da6
+#define regRCC_DEV0_0_RCC_ERR_INT_CNTL_BASE_IDX                                                         5
+#define regRCC_DEV0_0_RCC_BACO_CNTL_MISC                                                                0x8da7
+#define regRCC_DEV0_0_RCC_BACO_CNTL_MISC_BASE_IDX                                                       5
+#define regRCC_DEV0_0_RCC_RESET_EN                                                                      0x8da8
+#define regRCC_DEV0_0_RCC_RESET_EN_BASE_IDX                                                             5
+#define regRCC_DEV0_1_RCC_VDM_SUPPORT                                                                   0x8da9
+#define regRCC_DEV0_1_RCC_VDM_SUPPORT_BASE_IDX                                                          5
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0                                                            0x8daa
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1                                                            0x8dab
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_GPUIOV_REGION                                                                 0x8dac
+#define regRCC_DEV0_0_RCC_GPUIOV_REGION_BASE_IDX                                                        5
+#define regRCC_DEV0_0_RCC_GPU_HOSTVM_EN                                                                 0x8dad
+#define regRCC_DEV0_0_RCC_GPU_HOSTVM_EN_BASE_IDX                                                        5
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL                                                         0x8dae
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL_BASE_IDX                                                5
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET                                                   0x8daf
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET_BASE_IDX                                          5
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE                                                         0x8daf
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE_BASE_IDX                                                5
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE0                                                               0x8dde
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE0_BASE_IDX                                                      5
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE1                                                               0x8ddf
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE1_BASE_IDX                                                      5
+#define regRCC_DEV0_1_RCC_BUS_CNTL                                                                      0x8de1
+#define regRCC_DEV0_1_RCC_BUS_CNTL_BASE_IDX                                                             5
+#define regRCC_DEV0_0_RCC_CONFIG_CNTL                                                                   0x8de2
+#define regRCC_DEV0_0_RCC_CONFIG_CNTL_BASE_IDX                                                          5
+#define regRCC_DEV0_0_RCC_CONFIG_F0_BASE                                                                0x8de6
+#define regRCC_DEV0_0_RCC_CONFIG_F0_BASE_BASE_IDX                                                       5
+#define regRCC_DEV0_0_RCC_CONFIG_APER_SIZE                                                              0x8de7
+#define regRCC_DEV0_0_RCC_CONFIG_APER_SIZE_BASE_IDX                                                     5
+#define regRCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE                                                          0x8de8
+#define regRCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE_BASE_IDX                                                 5
+#define regRCC_DEV0_0_RCC_XDMA_LO                                                                       0x8de9
+#define regRCC_DEV0_0_RCC_XDMA_LO_BASE_IDX                                                              5
+#define regRCC_DEV0_0_RCC_XDMA_HI                                                                       0x8dea
+#define regRCC_DEV0_0_RCC_XDMA_HI_BASE_IDX                                                              5
+#define regRCC_DEV0_1_RCC_FEATURES_CONTROL_MISC                                                         0x8deb
+#define regRCC_DEV0_1_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                5
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL1                                                                  0x8dec
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL1_BASE_IDX                                                         5
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST0                                                                  0x8ded
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST0_BASE_IDX                                                         5
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST1                                                                  0x8dee
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST1_BASE_IDX                                                         5
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL2                                                                  0x8def
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL2_BASE_IDX                                                         5
+#define regRCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM                                                           0x8df0
+#define regRCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM_BASE_IDX                                                  5
+#define regRCC_DEV0_0_RCC_HOST_BUSNUM                                                                   0x8df1
+#define regRCC_DEV0_0_RCC_HOST_BUSNUM_BASE_IDX                                                          5
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI                                                            0x8df2
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO                                                            0x8df3
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI                                                            0x8df4
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO                                                            0x8df5
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI                                                            0x8df6
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO                                                            0x8df7
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI                                                            0x8df8
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO                                                            0x8df9
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST0                                                              0x8dfa
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST0_BASE_IDX                                                     5
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST1                                                              0x8dfb
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST1_BASE_IDX                                                     5
+#define regRCC_DEV0_1_RCC_DEV0_LINK_CNTL                                                                0x8dfd
+#define regRCC_DEV0_1_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       5
+#define regRCC_DEV0_1_RCC_CMN_LINK_CNTL                                                                 0x8dfe
+#define regRCC_DEV0_1_RCC_CMN_LINK_CNTL_BASE_IDX                                                        5
+#define regRCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE                                                        0x8dff
+#define regRCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               5
+#define regRCC_DEV0_1_RCC_LTR_LSWITCH_CNTL                                                              0x8e00
+#define regRCC_DEV0_1_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     5
+#define regRCC_DEV0_1_RCC_MH_ARB_CNTL                                                                   0x8e01
+#define regRCC_DEV0_1_RCC_MH_ARB_CNTL_BASE_IDX                                                          5
+
+
+// addressBlock: nbif0_bif_bx_SYSDEC
+// base address: 0x10120000
+#define regBIF_BX0_PCIE_INDEX                                                                           0x800c
+#define regBIF_BX0_PCIE_INDEX_BASE_IDX                                                                  5
+#define regBIF_BX0_PCIE_DATA                                                                            0x800d
+#define regBIF_BX0_PCIE_DATA_BASE_IDX                                                                   5
+#define regBIF_BX0_PCIE_INDEX2                                                                          0x800e
+#define regBIF_BX0_PCIE_INDEX2_BASE_IDX                                                                 5
+#define regBIF_BX0_PCIE_DATA2                                                                           0x800f
+#define regBIF_BX0_PCIE_DATA2_BASE_IDX                                                                  5
+#define regBIF_BX0_PCIE_INDEX_HI                                                                        0x8010
+#define regBIF_BX0_PCIE_INDEX_HI_BASE_IDX                                                               5
+#define regBIF_BX0_PCIE_INDEX2_HI                                                                       0x8011
+#define regBIF_BX0_PCIE_INDEX2_HI_BASE_IDX                                                              5
+#define regBIF_BX0_SBIOS_SCRATCH_0                                                                      0x8048
+#define regBIF_BX0_SBIOS_SCRATCH_0_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_1                                                                      0x8049
+#define regBIF_BX0_SBIOS_SCRATCH_1_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_2                                                                      0x804a
+#define regBIF_BX0_SBIOS_SCRATCH_2_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_3                                                                      0x804b
+#define regBIF_BX0_SBIOS_SCRATCH_3_BASE_IDX                                                             5
+#define regBIF_BX0_BIOS_SCRATCH_0                                                                       0x804c
+#define regBIF_BX0_BIOS_SCRATCH_0_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_1                                                                       0x804d
+#define regBIF_BX0_BIOS_SCRATCH_1_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_2                                                                       0x804e
+#define regBIF_BX0_BIOS_SCRATCH_2_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_3                                                                       0x804f
+#define regBIF_BX0_BIOS_SCRATCH_3_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_4                                                                       0x8050
+#define regBIF_BX0_BIOS_SCRATCH_4_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_5                                                                       0x8051
+#define regBIF_BX0_BIOS_SCRATCH_5_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_6                                                                       0x8052
+#define regBIF_BX0_BIOS_SCRATCH_6_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_7                                                                       0x8053
+#define regBIF_BX0_BIOS_SCRATCH_7_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_8                                                                       0x8054
+#define regBIF_BX0_BIOS_SCRATCH_8_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_9                                                                       0x8055
+#define regBIF_BX0_BIOS_SCRATCH_9_BASE_IDX                                                              5
+#define regBIF_BX0_BIOS_SCRATCH_10                                                                      0x8056
+#define regBIF_BX0_BIOS_SCRATCH_10_BASE_IDX                                                             5
+#define regBIF_BX0_BIOS_SCRATCH_11                                                                      0x8057
+#define regBIF_BX0_BIOS_SCRATCH_11_BASE_IDX                                                             5
+#define regBIF_BX0_BIOS_SCRATCH_12                                                                      0x8058
+#define regBIF_BX0_BIOS_SCRATCH_12_BASE_IDX                                                             5
+#define regBIF_BX0_BIOS_SCRATCH_13                                                                      0x8059
+#define regBIF_BX0_BIOS_SCRATCH_13_BASE_IDX                                                             5
+#define regBIF_BX0_BIOS_SCRATCH_14                                                                      0x805a
+#define regBIF_BX0_BIOS_SCRATCH_14_BASE_IDX                                                             5
+#define regBIF_BX0_BIOS_SCRATCH_15                                                                      0x805b
+#define regBIF_BX0_BIOS_SCRATCH_15_BASE_IDX                                                             5
+#define regBIF_BX0_BIF_RLC_INTR_CNTL                                                                    0x8060
+#define regBIF_BX0_BIF_RLC_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX0_BIF_VCE_INTR_CNTL                                                                    0x8061
+#define regBIF_BX0_BIF_VCE_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX0_BIF_UVD_INTR_CNTL                                                                    0x8062
+#define regBIF_BX0_BIF_UVD_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR0                                                                0x8080
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR0_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0                                                          0x8081
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR1                                                                0x8082
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR1_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1                                                          0x8083
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR2                                                                0x8084
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR2_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2                                                          0x8085
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR3                                                                0x8086
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR3_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3                                                          0x8087
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR4                                                                0x8088
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR4_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4                                                          0x8089
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR5                                                                0x808a
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR5_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5                                                          0x808b
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR6                                                                0x808c
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR6_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6                                                          0x808d
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR7                                                                0x808e
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR7_BASE_IDX                                                       5
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7                                                          0x808f
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7_BASE_IDX                                                 5
+#define regBIF_BX0_GFX_MMIOREG_CAM_CNTL                                                                 0x8090
+#define regBIF_BX0_GFX_MMIOREG_CAM_CNTL_BASE_IDX                                                        5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL                                                             0x8091
+#define regBIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL_BASE_IDX                                                    5
+#define regBIF_BX0_GFX_MMIOREG_CAM_ONE_CPL                                                              0x8092
+#define regBIF_BX0_GFX_MMIOREG_CAM_ONE_CPL_BASE_IDX                                                     5
+#define regBIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL                                                     0x8093
+#define regBIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL_BASE_IDX                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_0                                                                     0x8094
+#define regBIF_BX0_DRIVER_SCRATCH_0_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_1                                                                     0x8095
+#define regBIF_BX0_DRIVER_SCRATCH_1_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_2                                                                     0x8096
+#define regBIF_BX0_DRIVER_SCRATCH_2_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_3                                                                     0x8097
+#define regBIF_BX0_DRIVER_SCRATCH_3_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_4                                                                     0x8098
+#define regBIF_BX0_DRIVER_SCRATCH_4_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_5                                                                     0x8099
+#define regBIF_BX0_DRIVER_SCRATCH_5_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_6                                                                     0x809a
+#define regBIF_BX0_DRIVER_SCRATCH_6_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_7                                                                     0x809b
+#define regBIF_BX0_DRIVER_SCRATCH_7_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_8                                                                     0x809c
+#define regBIF_BX0_DRIVER_SCRATCH_8_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_9                                                                     0x809d
+#define regBIF_BX0_DRIVER_SCRATCH_9_BASE_IDX                                                            5
+#define regBIF_BX0_DRIVER_SCRATCH_10                                                                    0x809e
+#define regBIF_BX0_DRIVER_SCRATCH_10_BASE_IDX                                                           5
+#define regBIF_BX0_DRIVER_SCRATCH_11                                                                    0x809f
+#define regBIF_BX0_DRIVER_SCRATCH_11_BASE_IDX                                                           5
+#define regBIF_BX0_DRIVER_SCRATCH_12                                                                    0x80a0
+#define regBIF_BX0_DRIVER_SCRATCH_12_BASE_IDX                                                           5
+#define regBIF_BX0_DRIVER_SCRATCH_13                                                                    0x80a1
+#define regBIF_BX0_DRIVER_SCRATCH_13_BASE_IDX                                                           5
+#define regBIF_BX0_DRIVER_SCRATCH_14                                                                    0x80a2
+#define regBIF_BX0_DRIVER_SCRATCH_14_BASE_IDX                                                           5
+#define regBIF_BX0_DRIVER_SCRATCH_15                                                                    0x80a3
+#define regBIF_BX0_DRIVER_SCRATCH_15_BASE_IDX                                                           5
+#define regBIF_BX0_FW_SCRATCH_0                                                                         0x80a4
+#define regBIF_BX0_FW_SCRATCH_0_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_1                                                                         0x80a5
+#define regBIF_BX0_FW_SCRATCH_1_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_2                                                                         0x80a6
+#define regBIF_BX0_FW_SCRATCH_2_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_3                                                                         0x80a7
+#define regBIF_BX0_FW_SCRATCH_3_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_4                                                                         0x80a8
+#define regBIF_BX0_FW_SCRATCH_4_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_5                                                                         0x80a9
+#define regBIF_BX0_FW_SCRATCH_5_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_6                                                                         0x80aa
+#define regBIF_BX0_FW_SCRATCH_6_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_7                                                                         0x80ab
+#define regBIF_BX0_FW_SCRATCH_7_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_8                                                                         0x80ac
+#define regBIF_BX0_FW_SCRATCH_8_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_9                                                                         0x80ad
+#define regBIF_BX0_FW_SCRATCH_9_BASE_IDX                                                                5
+#define regBIF_BX0_FW_SCRATCH_10                                                                        0x80ae
+#define regBIF_BX0_FW_SCRATCH_10_BASE_IDX                                                               5
+#define regBIF_BX0_FW_SCRATCH_11                                                                        0x80af
+#define regBIF_BX0_FW_SCRATCH_11_BASE_IDX                                                               5
+#define regBIF_BX0_FW_SCRATCH_12                                                                        0x80b0
+#define regBIF_BX0_FW_SCRATCH_12_BASE_IDX                                                               5
+#define regBIF_BX0_FW_SCRATCH_13                                                                        0x80b1
+#define regBIF_BX0_FW_SCRATCH_13_BASE_IDX                                                               5
+#define regBIF_BX0_FW_SCRATCH_14                                                                        0x80b2
+#define regBIF_BX0_FW_SCRATCH_14_BASE_IDX                                                               5
+#define regBIF_BX0_FW_SCRATCH_15                                                                        0x80b3
+#define regBIF_BX0_FW_SCRATCH_15_BASE_IDX                                                               5
+#define regBIF_BX0_SBIOS_SCRATCH_4                                                                      0x80b4
+#define regBIF_BX0_SBIOS_SCRATCH_4_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_5                                                                      0x80b5
+#define regBIF_BX0_SBIOS_SCRATCH_5_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_6                                                                      0x80b6
+#define regBIF_BX0_SBIOS_SCRATCH_6_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_7                                                                      0x80b7
+#define regBIF_BX0_SBIOS_SCRATCH_7_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_8                                                                      0x80b8
+#define regBIF_BX0_SBIOS_SCRATCH_8_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_9                                                                      0x80b9
+#define regBIF_BX0_SBIOS_SCRATCH_9_BASE_IDX                                                             5
+#define regBIF_BX0_SBIOS_SCRATCH_10                                                                     0x80ba
+#define regBIF_BX0_SBIOS_SCRATCH_10_BASE_IDX                                                            5
+#define regBIF_BX0_SBIOS_SCRATCH_11                                                                     0x80bb
+#define regBIF_BX0_SBIOS_SCRATCH_11_BASE_IDX                                                            5
+#define regBIF_BX0_SBIOS_SCRATCH_12                                                                     0x80bc
+#define regBIF_BX0_SBIOS_SCRATCH_12_BASE_IDX                                                            5
+#define regBIF_BX0_SBIOS_SCRATCH_13                                                                     0x80bd
+#define regBIF_BX0_SBIOS_SCRATCH_13_BASE_IDX                                                            5
+#define regBIF_BX0_SBIOS_SCRATCH_14                                                                     0x80be
+#define regBIF_BX0_SBIOS_SCRATCH_14_BASE_IDX                                                            5
+#define regBIF_BX0_SBIOS_SCRATCH_15                                                                     0x80bf
+#define regBIF_BX0_SBIOS_SCRATCH_15_BASE_IDX                                                            5
+
+
+// addressBlock: nbif0_bif_bx_pf_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_PF0_MM_INDEX                                                                          0x8000
+#define regBIF_BX_PF0_MM_INDEX_BASE_IDX                                                                 5
+#define regBIF_BX_PF0_MM_DATA                                                                           0x8001
+#define regBIF_BX_PF0_MM_DATA_BASE_IDX                                                                  5
+#define regBIF_BX_PF0_MM_INDEX_HI                                                                       0x8006
+#define regBIF_BX_PF0_MM_INDEX_HI_BASE_IDX                                                              5
+#define regBIF_BX_PF0_RSMU_INDEX                                                                        0x8014
+#define regBIF_BX_PF0_RSMU_INDEX_BASE_IDX                                                               5
+#define regBIF_BX_PF0_RSMU_DATA                                                                         0x8015
+#define regBIF_BX_PF0_RSMU_DATA_BASE_IDX                                                                5
+#define regBIF_BX_PF0_RSMU_INDEX_HI                                                                     0x8016
+#define regBIF_BX_PF0_RSMU_INDEX_HI_BASE_IDX                                                            5
+
+
+// addressBlock: nbif0_bif_bx_BIFDEC1
+// base address: 0x10120000
+#define regBIF_BX0_BIF_MM_INDACCESS_CNTL                                                                0x8e06
+#define regBIF_BX0_BIF_MM_INDACCESS_CNTL_BASE_IDX                                                       5
+#define regBIF_BX0_BUS_CNTL                                                                             0x8e07
+#define regBIF_BX0_BUS_CNTL_BASE_IDX                                                                    5
+#define regBIF_BX0_BIF_SCRATCH0                                                                         0x8e08
+#define regBIF_BX0_BIF_SCRATCH0_BASE_IDX                                                                5
+#define regBIF_BX0_BIF_SCRATCH1                                                                         0x8e09
+#define regBIF_BX0_BIF_SCRATCH1_BASE_IDX                                                                5
+#define regBIF_BX0_BX_RESET_EN                                                                          0x8e0d
+#define regBIF_BX0_BX_RESET_EN_BASE_IDX                                                                 5
+#define regBIF_BX0_MM_CFGREGS_CNTL                                                                      0x8e0e
+#define regBIF_BX0_MM_CFGREGS_CNTL_BASE_IDX                                                             5
+#define regBIF_BX0_BX_RESET_CNTL                                                                        0x8e10
+#define regBIF_BX0_BX_RESET_CNTL_BASE_IDX                                                               5
+#define regBIF_BX0_INTERRUPT_CNTL                                                                       0x8e11
+#define regBIF_BX0_INTERRUPT_CNTL_BASE_IDX                                                              5
+#define regBIF_BX0_INTERRUPT_CNTL2                                                                      0x8e12
+#define regBIF_BX0_INTERRUPT_CNTL2_BASE_IDX                                                             5
+#define regBIF_BX0_CLKREQB_PAD_CNTL                                                                     0x8e18
+#define regBIF_BX0_CLKREQB_PAD_CNTL_BASE_IDX                                                            5
+#define regBIF_BX0_BIF_FEATURES_CONTROL_MISC                                                            0x8e1b
+#define regBIF_BX0_BIF_FEATURES_CONTROL_MISC_BASE_IDX                                                   5
+#define regBIF_BX0_HDP_ATOMIC_CONTROL_MISC                                                              0x8e1c
+#define regBIF_BX0_HDP_ATOMIC_CONTROL_MISC_BASE_IDX                                                     5
+#define regBIF_BX0_BIF_DOORBELL_CNTL                                                                    0x8e1d
+#define regBIF_BX0_BIF_DOORBELL_CNTL_BASE_IDX                                                           5
+#define regBIF_BX0_BIF_FB_EN                                                                            0x8e20
+#define regBIF_BX0_BIF_FB_EN_BASE_IDX                                                                   5
+#define regBIF_BX0_BIF_MST_TRANS_PENDING_VF                                                             0x8e29
+#define regBIF_BX0_BIF_MST_TRANS_PENDING_VF_BASE_IDX                                                    5
+#define regBIF_BX0_BIF_SLV_TRANS_PENDING_VF                                                             0x8e2a
+#define regBIF_BX0_BIF_SLV_TRANS_PENDING_VF_BASE_IDX                                                    5
+#define regBIF_BX0_BACO_CNTL                                                                            0x8e2b
+#define regBIF_BX0_BACO_CNTL_BASE_IDX                                                                   5
+#define regBIF_BX0_BIF_BACO_EXIT_TIME0                                                                  0x8e2c
+#define regBIF_BX0_BIF_BACO_EXIT_TIME0_BASE_IDX                                                         5
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER1                                                                 0x8e2d
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER1_BASE_IDX                                                        5
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER2                                                                 0x8e2e
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER2_BASE_IDX                                                        5
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER3                                                                 0x8e2f
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER3_BASE_IDX                                                        5
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER4                                                                 0x8e30
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER4_BASE_IDX                                                        5
+#define regBIF_BX0_MEM_TYPE_CNTL                                                                        0x8e31
+#define regBIF_BX0_MEM_TYPE_CNTL_BASE_IDX                                                               5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_CNTL                                                               0x8e33
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_CNTL_BASE_IDX                                                      5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_0                                                                  0x8e34
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_0_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_1                                                                  0x8e35
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_1_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_2                                                                  0x8e36
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_2_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_3                                                                  0x8e37
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_3_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_4                                                                  0x8e38
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_4_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_5                                                                  0x8e39
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_5_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_6                                                                  0x8e3a
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_6_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_7                                                                  0x8e3b
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_7_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_8                                                                  0x8e3c
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_8_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_9                                                                  0x8e3d
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_9_BASE_IDX                                                         5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_10                                                                 0x8e3e
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_10_BASE_IDX                                                        5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_11                                                                 0x8e3f
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_11_BASE_IDX                                                        5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_12                                                                 0x8e40
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_12_BASE_IDX                                                        5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_13                                                                 0x8e41
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_13_BASE_IDX                                                        5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_14                                                                 0x8e42
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_14_BASE_IDX                                                        5
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_15                                                                 0x8e43
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_15_BASE_IDX                                                        5
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x8e4d
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX                                                    5
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL                                                             0x8e4e
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX                                                    5
+#define regBIF_BX0_MAILBOX_INDEX                                                                        0x8e55
+#define regBIF_BX0_MAILBOX_INDEX_BASE_IDX                                                               5
+#define regBIF_BX0_BIF_PERSTB_PAD_CNTL                                                                  0x8e65
+#define regBIF_BX0_BIF_PERSTB_PAD_CNTL_BASE_IDX                                                         5
+#define regBIF_BX0_BIF_PX_EN_PAD_CNTL                                                                   0x8e66
+#define regBIF_BX0_BIF_PX_EN_PAD_CNTL_BASE_IDX                                                          5
+#define regBIF_BX0_BIF_REFPADKIN_PAD_CNTL                                                               0x8e67
+#define regBIF_BX0_BIF_REFPADKIN_PAD_CNTL_BASE_IDX                                                      5
+#define regBIF_BX0_BIF_CLKREQB_PAD_CNTL                                                                 0x8e69
+#define regBIF_BX0_BIF_CLKREQB_PAD_CNTL_BASE_IDX                                                        5
+#define regBIF_BX0_BIF_PWRBRK_PAD_CNTL                                                                  0x8e6a
+#define regBIF_BX0_BIF_PWRBRK_PAD_CNTL_BASE_IDX                                                         5
+
+
+// addressBlock: nbif0_bif_bx_pf_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_PF0_BIF_BME_STATUS                                                                    0x8e0b
+#define regBIF_BX_PF0_BIF_BME_STATUS_BASE_IDX                                                           5
+#define regBIF_BX_PF0_BIF_ATOMIC_ERR_LOG                                                                0x8e0c
+#define regBIF_BX_PF0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                                       5
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x8e13
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                                     5
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x8e14
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                                      5
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x8e15
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                          5
+#define regBIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL                                                      0x8e16
+#define regBIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                             5
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x8e17
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                             5
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                                 0x8e19
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                                        5
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                            0x8e1a
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                                   5
+#define regBIF_BX_PF0_BIF_DOORBELL_INT_CNTL                                                             0x8e1e
+#define regBIF_BX_PF0_BIF_DOORBELL_INT_CNTL_BASE_IDX                                                    5
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ                                                                 0x8e26
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ_BASE_IDX                                                        5
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE                                                                0x8e27
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE_BASE_IDX                                                       5
+#define regBIF_BX_PF0_BIF_TRANS_PENDING                                                                 0x8e28
+#define regBIF_BX_PF0_BIF_TRANS_PENDING_BASE_IDX                                                        5
+#define regBIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS                                                          0x8e32
+#define regBIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                                 5
+#define regBIF_BX_PF0_BIF_RB_CNTL                                                                       0x8e4f
+#define regBIF_BX_PF0_BIF_RB_CNTL_BASE_IDX                                                              5
+#define regBIF_BX_PF0_BIF_RB_BASE                                                                       0x8e50
+#define regBIF_BX_PF0_BIF_RB_BASE_BASE_IDX                                                              5
+#define regBIF_BX_PF0_BIF_RB_RPTR                                                                       0x8e51
+#define regBIF_BX_PF0_BIF_RB_RPTR_BASE_IDX                                                              5
+#define regBIF_BX_PF0_BIF_RB_WPTR                                                                       0x8e52
+#define regBIF_BX_PF0_BIF_RB_WPTR_BASE_IDX                                                              5
+#define regBIF_BX_PF0_BIF_RB_WPTR_ADDR_HI                                                               0x8e53
+#define regBIF_BX_PF0_BIF_RB_WPTR_ADDR_HI_BASE_IDX                                                      5
+#define regBIF_BX_PF0_BIF_RB_WPTR_ADDR_LO                                                               0x8e54
+#define regBIF_BX_PF0_BIF_RB_WPTR_ADDR_LO_BASE_IDX                                                      5
+
+
+// addressBlock: nbif0_bif_bx_SYSDEC:1
+// base address: 0x0
+#define regBIF_BX1_PCIE_INDEX                                                                           0x000c
+#define regBIF_BX1_PCIE_INDEX_BASE_IDX                                                                  0
+#define regBIF_BX1_PCIE_DATA                                                                            0x000d
+#define regBIF_BX1_PCIE_DATA_BASE_IDX                                                                   0
+#define regBIF_BX1_PCIE_INDEX2                                                                          0x000e
+#define regBIF_BX1_PCIE_INDEX2_BASE_IDX                                                                 0
+#define regBIF_BX1_PCIE_DATA2                                                                           0x000f
+#define regBIF_BX1_PCIE_DATA2_BASE_IDX                                                                  0
+#define regBIF_BX1_PCIE_INDEX_HI                                                                        0x0010
+#define regBIF_BX1_PCIE_INDEX_HI_BASE_IDX                                                               0
+#define regBIF_BX1_PCIE_INDEX2_HI                                                                       0x0011
+#define regBIF_BX1_PCIE_INDEX2_HI_BASE_IDX                                                              0
+#define regBIF_BX1_SBIOS_SCRATCH_0                                                                      0x0034
+#define regBIF_BX1_SBIOS_SCRATCH_0_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_1                                                                      0x0035
+#define regBIF_BX1_SBIOS_SCRATCH_1_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_2                                                                      0x0036
+#define regBIF_BX1_SBIOS_SCRATCH_2_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_3                                                                      0x0037
+#define regBIF_BX1_SBIOS_SCRATCH_3_BASE_IDX                                                             1
+#define regBIF_BX1_BIOS_SCRATCH_0                                                                       0x0038
+#define regBIF_BX1_BIOS_SCRATCH_0_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_1                                                                       0x0039
+#define regBIF_BX1_BIOS_SCRATCH_1_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_2                                                                       0x003a
+#define regBIF_BX1_BIOS_SCRATCH_2_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_3                                                                       0x003b
+#define regBIF_BX1_BIOS_SCRATCH_3_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_4                                                                       0x003c
+#define regBIF_BX1_BIOS_SCRATCH_4_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_5                                                                       0x003d
+#define regBIF_BX1_BIOS_SCRATCH_5_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_6                                                                       0x003e
+#define regBIF_BX1_BIOS_SCRATCH_6_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_7                                                                       0x003f
+#define regBIF_BX1_BIOS_SCRATCH_7_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_8                                                                       0x0040
+#define regBIF_BX1_BIOS_SCRATCH_8_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_9                                                                       0x0041
+#define regBIF_BX1_BIOS_SCRATCH_9_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_10                                                                      0x0042
+#define regBIF_BX1_BIOS_SCRATCH_10_BASE_IDX                                                             1
+#define regBIF_BX1_BIOS_SCRATCH_11                                                                      0x0043
+#define regBIF_BX1_BIOS_SCRATCH_11_BASE_IDX                                                             1
+#define regBIF_BX1_BIOS_SCRATCH_12                                                                      0x0044
+#define regBIF_BX1_BIOS_SCRATCH_12_BASE_IDX                                                             1
+#define regBIF_BX1_BIOS_SCRATCH_13                                                                      0x0045
+#define regBIF_BX1_BIOS_SCRATCH_13_BASE_IDX                                                             1
+#define regBIF_BX1_BIOS_SCRATCH_14                                                                      0x0046
+#define regBIF_BX1_BIOS_SCRATCH_14_BASE_IDX                                                             1
+#define regBIF_BX1_BIOS_SCRATCH_15                                                                      0x0047
+#define regBIF_BX1_BIOS_SCRATCH_15_BASE_IDX                                                             1
+#define regBIF_BX1_BIF_RLC_INTR_CNTL                                                                    0x004c
+#define regBIF_BX1_BIF_RLC_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX1_BIF_VCE_INTR_CNTL                                                                    0x004d
+#define regBIF_BX1_BIF_VCE_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX1_BIF_UVD_INTR_CNTL                                                                    0x004e
+#define regBIF_BX1_BIF_UVD_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR0                                                                0x006c
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR0_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0                                                          0x006d
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR1                                                                0x006e
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR1_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1                                                          0x006f
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR2                                                                0x0070
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR2_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2                                                          0x0071
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR3                                                                0x0072
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR3_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3                                                          0x0073
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR4                                                                0x0074
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR4_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4                                                          0x0075
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR5                                                                0x0076
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR5_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5                                                          0x0077
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR6                                                                0x0078
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR6_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6                                                          0x0079
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR7                                                                0x007a
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR7_BASE_IDX                                                       1
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7                                                          0x007b
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7_BASE_IDX                                                 1
+#define regBIF_BX1_GFX_MMIOREG_CAM_CNTL                                                                 0x007c
+#define regBIF_BX1_GFX_MMIOREG_CAM_CNTL_BASE_IDX                                                        1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL                                                             0x007d
+#define regBIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL_BASE_IDX                                                    1
+#define regBIF_BX1_GFX_MMIOREG_CAM_ONE_CPL                                                              0x007e
+#define regBIF_BX1_GFX_MMIOREG_CAM_ONE_CPL_BASE_IDX                                                     1
+#define regBIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL                                                     0x007f
+#define regBIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL_BASE_IDX                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_0                                                                     0x0080
+#define regBIF_BX1_DRIVER_SCRATCH_0_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_1                                                                     0x0081
+#define regBIF_BX1_DRIVER_SCRATCH_1_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_2                                                                     0x0082
+#define regBIF_BX1_DRIVER_SCRATCH_2_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_3                                                                     0x0083
+#define regBIF_BX1_DRIVER_SCRATCH_3_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_4                                                                     0x0084
+#define regBIF_BX1_DRIVER_SCRATCH_4_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_5                                                                     0x0085
+#define regBIF_BX1_DRIVER_SCRATCH_5_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_6                                                                     0x0086
+#define regBIF_BX1_DRIVER_SCRATCH_6_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_7                                                                     0x0087
+#define regBIF_BX1_DRIVER_SCRATCH_7_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_8                                                                     0x0088
+#define regBIF_BX1_DRIVER_SCRATCH_8_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_9                                                                     0x0089
+#define regBIF_BX1_DRIVER_SCRATCH_9_BASE_IDX                                                            1
+#define regBIF_BX1_DRIVER_SCRATCH_10                                                                    0x008a
+#define regBIF_BX1_DRIVER_SCRATCH_10_BASE_IDX                                                           1
+#define regBIF_BX1_DRIVER_SCRATCH_11                                                                    0x008b
+#define regBIF_BX1_DRIVER_SCRATCH_11_BASE_IDX                                                           1
+#define regBIF_BX1_DRIVER_SCRATCH_12                                                                    0x008c
+#define regBIF_BX1_DRIVER_SCRATCH_12_BASE_IDX                                                           1
+#define regBIF_BX1_DRIVER_SCRATCH_13                                                                    0x008d
+#define regBIF_BX1_DRIVER_SCRATCH_13_BASE_IDX                                                           1
+#define regBIF_BX1_DRIVER_SCRATCH_14                                                                    0x008e
+#define regBIF_BX1_DRIVER_SCRATCH_14_BASE_IDX                                                           1
+#define regBIF_BX1_DRIVER_SCRATCH_15                                                                    0x008f
+#define regBIF_BX1_DRIVER_SCRATCH_15_BASE_IDX                                                           1
+#define regBIF_BX1_FW_SCRATCH_0                                                                         0x0090
+#define regBIF_BX1_FW_SCRATCH_0_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_1                                                                         0x0091
+#define regBIF_BX1_FW_SCRATCH_1_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_2                                                                         0x0092
+#define regBIF_BX1_FW_SCRATCH_2_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_3                                                                         0x0093
+#define regBIF_BX1_FW_SCRATCH_3_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_4                                                                         0x0094
+#define regBIF_BX1_FW_SCRATCH_4_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_5                                                                         0x0095
+#define regBIF_BX1_FW_SCRATCH_5_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_6                                                                         0x0096
+#define regBIF_BX1_FW_SCRATCH_6_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_7                                                                         0x0097
+#define regBIF_BX1_FW_SCRATCH_7_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_8                                                                         0x0098
+#define regBIF_BX1_FW_SCRATCH_8_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_9                                                                         0x0099
+#define regBIF_BX1_FW_SCRATCH_9_BASE_IDX                                                                1
+#define regBIF_BX1_FW_SCRATCH_10                                                                        0x009a
+#define regBIF_BX1_FW_SCRATCH_10_BASE_IDX                                                               1
+#define regBIF_BX1_FW_SCRATCH_11                                                                        0x009b
+#define regBIF_BX1_FW_SCRATCH_11_BASE_IDX                                                               1
+#define regBIF_BX1_FW_SCRATCH_12                                                                        0x009c
+#define regBIF_BX1_FW_SCRATCH_12_BASE_IDX                                                               1
+#define regBIF_BX1_FW_SCRATCH_13                                                                        0x009d
+#define regBIF_BX1_FW_SCRATCH_13_BASE_IDX                                                               1
+#define regBIF_BX1_FW_SCRATCH_14                                                                        0x009e
+#define regBIF_BX1_FW_SCRATCH_14_BASE_IDX                                                               1
+#define regBIF_BX1_FW_SCRATCH_15                                                                        0x009f
+#define regBIF_BX1_FW_SCRATCH_15_BASE_IDX                                                               1
+#define regBIF_BX1_SBIOS_SCRATCH_4                                                                      0x00a0
+#define regBIF_BX1_SBIOS_SCRATCH_4_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_5                                                                      0x00a1
+#define regBIF_BX1_SBIOS_SCRATCH_5_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_6                                                                      0x00a2
+#define regBIF_BX1_SBIOS_SCRATCH_6_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_7                                                                      0x00a3
+#define regBIF_BX1_SBIOS_SCRATCH_7_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_8                                                                      0x00a4
+#define regBIF_BX1_SBIOS_SCRATCH_8_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_9                                                                      0x00a5
+#define regBIF_BX1_SBIOS_SCRATCH_9_BASE_IDX                                                             1
+#define regBIF_BX1_SBIOS_SCRATCH_10                                                                     0x00a6
+#define regBIF_BX1_SBIOS_SCRATCH_10_BASE_IDX                                                            1
+#define regBIF_BX1_SBIOS_SCRATCH_11                                                                     0x00a7
+#define regBIF_BX1_SBIOS_SCRATCH_11_BASE_IDX                                                            1
+#define regBIF_BX1_SBIOS_SCRATCH_12                                                                     0x00a8
+#define regBIF_BX1_SBIOS_SCRATCH_12_BASE_IDX                                                            1
+#define regBIF_BX1_SBIOS_SCRATCH_13                                                                     0x00a9
+#define regBIF_BX1_SBIOS_SCRATCH_13_BASE_IDX                                                            1
+#define regBIF_BX1_SBIOS_SCRATCH_14                                                                     0x00aa
+#define regBIF_BX1_SBIOS_SCRATCH_14_BASE_IDX                                                            1
+#define regBIF_BX1_SBIOS_SCRATCH_15                                                                     0x00ab
+#define regBIF_BX1_SBIOS_SCRATCH_15_BASE_IDX                                                            1
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_BIFDEC1:1
+// base address: 0x0
+#define regRCC_DWN_DEV0_2_DN_PCIE_RESERVED                                                              0x0068
+#define regRCC_DWN_DEV0_2_DN_PCIE_RESERVED_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_2_DN_PCIE_SCRATCH                                                               0x0069
+#define regRCC_DWN_DEV0_2_DN_PCIE_SCRATCH_BASE_IDX                                                      2
+#define regRCC_DWN_DEV0_2_DN_PCIE_CNTL                                                                  0x006b
+#define regRCC_DWN_DEV0_2_DN_PCIE_CNTL_BASE_IDX                                                         2
+#define regRCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL                                                           0x006c
+#define regRCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  2
+#define regRCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2                                                              0x006d
+#define regRCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL                                                              0x006e
+#define regRCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL                                                              0x006f
+#define regRCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_F0                                                              0x0070
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_F0_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC                                                            0x0071
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC_BASE_IDX                                                   2
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2                                                           0x0072
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  2
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_BIFDEC1:1
+// base address: 0x0
+#define regRCC_DWNP_DEV0_2_PCIE_ERR_CNTL                                                                0x0074
+#define regRCC_DWNP_DEV0_2_PCIE_ERR_CNTL_BASE_IDX                                                       2
+#define regRCC_DWNP_DEV0_2_PCIE_RX_CNTL                                                                 0x0075
+#define regRCC_DWNP_DEV0_2_PCIE_RX_CNTL_BASE_IDX                                                        2
+#define regRCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL                                                           0x0076
+#define regRCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  2
+#define regRCC_DWNP_DEV0_2_PCIE_LC_CNTL2                                                                0x0077
+#define regRCC_DWNP_DEV0_2_PCIE_LC_CNTL2_BASE_IDX                                                       2
+#define regRCC_DWNP_DEV0_2_PCIEP_STRAP_MISC                                                             0x0078
+#define regRCC_DWNP_DEV0_2_PCIEP_STRAP_MISC_BASE_IDX                                                    2
+#define regRCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP                                                         0x0079
+#define regRCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                2
+
+
+// addressBlock: nbif0_rcc_ep_dev0_BIFDEC1:1
+// base address: 0x0
+#define regRCC_EP_DEV0_2_EP_PCIE_SCRATCH                                                                0x004b
+#define regRCC_EP_DEV0_2_EP_PCIE_SCRATCH_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_2_EP_PCIE_CNTL                                                                   0x004d
+#define regRCC_EP_DEV0_2_EP_PCIE_CNTL_BASE_IDX                                                          2
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_CNTL                                                               0x004e
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_STATUS                                                             0x004f
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_STATUS_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL2                                                               0x0050
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL2_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_BUS_CNTL                                                               0x0051
+#define regRCC_EP_DEV0_2_EP_PCIE_BUS_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_CFG_CNTL                                                               0x0052
+#define regRCC_EP_DEV0_2_EP_PCIE_CFG_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL                                                            0x0054
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0                                               0x0055
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1                                               0x0055
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2                                               0x0055
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3                                               0x0055
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4                                               0x0056
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5                                               0x0056
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6                                               0x0056
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      2
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7                                               0x0056
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC                                                             0x0057
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2                                                            0x0058
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP                                                             0x005a
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0x005b
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL                                                            0x005b
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0x005b
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0x005c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0x005c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0x005c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0x005c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0x005d
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0x005d
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0x005d
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_PME_CONTROL                                                            0x005d
+#define regRCC_EP_DEV0_2_EP_PCIE_PME_CONTROL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_2_EP_PCIEP_RESERVED                                                              0x005e
+#define regRCC_EP_DEV0_2_EP_PCIEP_RESERVED_BASE_IDX                                                     2
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_CNTL                                                                0x0060
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_CNTL_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID                                                        0x0061
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               2
+#define regRCC_EP_DEV0_2_EP_PCIE_ERR_CNTL                                                               0x0062
+#define regRCC_EP_DEV0_2_EP_PCIE_ERR_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL                                                                0x0063
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL                                                          0x0064
+#define regRCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 2
+
+
+// addressBlock: nbif0_bif_bx_pf_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_PF1_MM_INDEX                                                                          0x0000
+#define regBIF_BX_PF1_MM_INDEX_BASE_IDX                                                                 0
+#define regBIF_BX_PF1_MM_DATA                                                                           0x0001
+#define regBIF_BX_PF1_MM_DATA_BASE_IDX                                                                  0
+#define regBIF_BX_PF1_MM_INDEX_HI                                                                       0x0006
+#define regBIF_BX_PF1_MM_INDEX_HI_BASE_IDX                                                              0
+#define regBIF_BX_PF1_RSMU_INDEX                                                                        0x0000
+#define regBIF_BX_PF1_RSMU_INDEX_BASE_IDX                                                               1
+#define regBIF_BX_PF1_RSMU_DATA                                                                         0x0001
+#define regBIF_BX_PF1_RSMU_DATA_BASE_IDX                                                                1
+
+
+// addressBlock: nbif0_bif_bx_BIFDEC1:1
+// base address: 0x0
+#define regBIF_BX1_BIF_MM_INDACCESS_CNTL                                                                0x00e6
+#define regBIF_BX1_BIF_MM_INDACCESS_CNTL_BASE_IDX                                                       2
+#define regBIF_BX1_BUS_CNTL                                                                             0x00e7
+#define regBIF_BX1_BUS_CNTL_BASE_IDX                                                                    2
+#define regBIF_BX1_BIF_SCRATCH0                                                                         0x00e8
+#define regBIF_BX1_BIF_SCRATCH0_BASE_IDX                                                                2
+#define regBIF_BX1_BIF_SCRATCH1                                                                         0x00e9
+#define regBIF_BX1_BIF_SCRATCH1_BASE_IDX                                                                2
+#define regBIF_BX1_BX_RESET_EN                                                                          0x00ed
+#define regBIF_BX1_BX_RESET_EN_BASE_IDX                                                                 2
+#define regBIF_BX1_MM_CFGREGS_CNTL                                                                      0x00ee
+#define regBIF_BX1_MM_CFGREGS_CNTL_BASE_IDX                                                             2
+#define regBIF_BX1_BX_RESET_CNTL                                                                        0x00f0
+#define regBIF_BX1_BX_RESET_CNTL_BASE_IDX                                                               2
+#define regBIF_BX1_INTERRUPT_CNTL                                                                       0x00f1
+#define regBIF_BX1_INTERRUPT_CNTL_BASE_IDX                                                              2
+#define regBIF_BX1_INTERRUPT_CNTL2                                                                      0x00f2
+#define regBIF_BX1_INTERRUPT_CNTL2_BASE_IDX                                                             2
+#define regBIF_BX1_CLKREQB_PAD_CNTL                                                                     0x00f8
+#define regBIF_BX1_CLKREQB_PAD_CNTL_BASE_IDX                                                            2
+#define regBIF_BX1_BIF_FEATURES_CONTROL_MISC                                                            0x00fb
+#define regBIF_BX1_BIF_FEATURES_CONTROL_MISC_BASE_IDX                                                   2
+#define regBIF_BX1_HDP_ATOMIC_CONTROL_MISC                                                              0x00fc
+#define regBIF_BX1_HDP_ATOMIC_CONTROL_MISC_BASE_IDX                                                     2
+#define regBIF_BX1_BIF_DOORBELL_CNTL                                                                    0x00fd
+#define regBIF_BX1_BIF_DOORBELL_CNTL_BASE_IDX                                                           2
+#define regBIF_BX1_BIF_FB_EN                                                                            0x0100
+#define regBIF_BX1_BIF_FB_EN_BASE_IDX                                                                   2
+#define regBIF_BX1_BIF_MST_TRANS_PENDING_VF                                                             0x0109
+#define regBIF_BX1_BIF_MST_TRANS_PENDING_VF_BASE_IDX                                                    2
+#define regBIF_BX1_BIF_SLV_TRANS_PENDING_VF                                                             0x010a
+#define regBIF_BX1_BIF_SLV_TRANS_PENDING_VF_BASE_IDX                                                    2
+#define regBIF_BX1_BACO_CNTL                                                                            0x010b
+#define regBIF_BX1_BACO_CNTL_BASE_IDX                                                                   2
+#define regBIF_BX1_BIF_BACO_EXIT_TIME0                                                                  0x010c
+#define regBIF_BX1_BIF_BACO_EXIT_TIME0_BASE_IDX                                                         2
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER1                                                                 0x010d
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER1_BASE_IDX                                                        2
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER2                                                                 0x010e
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER2_BASE_IDX                                                        2
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER3                                                                 0x010f
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER3_BASE_IDX                                                        2
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER4                                                                 0x0110
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER4_BASE_IDX                                                        2
+#define regBIF_BX1_MEM_TYPE_CNTL                                                                        0x0111
+#define regBIF_BX1_MEM_TYPE_CNTL_BASE_IDX                                                               2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_CNTL                                                               0x0113
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_CNTL_BASE_IDX                                                      2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_0                                                                  0x0114
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_0_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_1                                                                  0x0115
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_1_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_2                                                                  0x0116
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_2_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_3                                                                  0x0117
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_3_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_4                                                                  0x0118
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_4_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_5                                                                  0x0119
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_5_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_6                                                                  0x011a
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_6_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_7                                                                  0x011b
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_7_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_8                                                                  0x011c
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_8_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_9                                                                  0x011d
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_9_BASE_IDX                                                         2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_10                                                                 0x011e
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_10_BASE_IDX                                                        2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_11                                                                 0x011f
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_11_BASE_IDX                                                        2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_12                                                                 0x0120
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_12_BASE_IDX                                                        2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_13                                                                 0x0121
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_13_BASE_IDX                                                        2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_14                                                                 0x0122
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_14_BASE_IDX                                                        2
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_15                                                                 0x0123
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_15_BASE_IDX                                                        2
+#define regBIF_BX1_VF_REGWR_EN                                                                          0x0124
+#define regBIF_BX1_VF_REGWR_EN_BASE_IDX                                                                 2
+#define regBIF_BX1_VF_DOORBELL_EN                                                                       0x0125
+#define regBIF_BX1_VF_DOORBELL_EN_BASE_IDX                                                              2
+#define regBIF_BX1_VF_FB_EN                                                                             0x0126
+#define regBIF_BX1_VF_FB_EN_BASE_IDX                                                                    2
+#define regBIF_BX1_VF_REGWR_STATUS                                                                      0x0127
+#define regBIF_BX1_VF_REGWR_STATUS_BASE_IDX                                                             2
+#define regBIF_BX1_VF_DOORBELL_STATUS                                                                   0x0128
+#define regBIF_BX1_VF_DOORBELL_STATUS_BASE_IDX                                                          2
+#define regBIF_BX1_VF_FB_STATUS                                                                         0x0129
+#define regBIF_BX1_VF_FB_STATUS_BASE_IDX                                                                2
+#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x012d
+#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL                                                             0x012e
+#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX1_MAILBOX_INDEX                                                                        0x0135
+#define regBIF_BX1_MAILBOX_INDEX_BASE_IDX                                                               2
+#define regBIF_BX1_BIF_PERSTB_PAD_CNTL                                                                  0x0145
+#define regBIF_BX1_BIF_PERSTB_PAD_CNTL_BASE_IDX                                                         2
+#define regBIF_BX1_BIF_PX_EN_PAD_CNTL                                                                   0x0146
+#define regBIF_BX1_BIF_PX_EN_PAD_CNTL_BASE_IDX                                                          2
+#define regBIF_BX1_BIF_REFPADKIN_PAD_CNTL                                                               0x0147
+#define regBIF_BX1_BIF_REFPADKIN_PAD_CNTL_BASE_IDX                                                      2
+#define regBIF_BX1_BIF_CLKREQB_PAD_CNTL                                                                 0x0149
+#define regBIF_BX1_BIF_CLKREQB_PAD_CNTL_BASE_IDX                                                        2
+#define regBIF_BX1_BIF_PWRBRK_PAD_CNTL                                                                  0x014a
+#define regBIF_BX1_BIF_PWRBRK_PAD_CNTL_BASE_IDX                                                         2
+
+
+// addressBlock: nbif0_rcc_dev0_BIFDEC1:1
+// base address: 0x0
+#define regRCC_DEV0_1_RCC_ERR_INT_CNTL                                                                  0x0086
+#define regRCC_DEV0_1_RCC_ERR_INT_CNTL_BASE_IDX                                                         2
+#define regRCC_DEV0_1_RCC_BACO_CNTL_MISC                                                                0x0087
+#define regRCC_DEV0_1_RCC_BACO_CNTL_MISC_BASE_IDX                                                       2
+#define regRCC_DEV0_1_RCC_RESET_EN                                                                      0x0088
+#define regRCC_DEV0_1_RCC_RESET_EN_BASE_IDX                                                             2
+#define regRCC_DEV0_2_RCC_VDM_SUPPORT                                                                   0x0089
+#define regRCC_DEV0_2_RCC_VDM_SUPPORT_BASE_IDX                                                          2
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0                                                            0x008a
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   2
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1                                                            0x008b
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_GPUIOV_REGION                                                                 0x008c
+#define regRCC_DEV0_1_RCC_GPUIOV_REGION_BASE_IDX                                                        2
+#define regRCC_DEV0_1_RCC_GPU_HOSTVM_EN                                                                 0x008d
+#define regRCC_DEV0_1_RCC_GPU_HOSTVM_EN_BASE_IDX                                                        2
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL                                                         0x008e
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL_BASE_IDX                                                2
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET                                                   0x008f
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET_BASE_IDX                                          2
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE                                                         0x008f
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE_BASE_IDX                                                2
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE0                                                               0x00be
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE0_BASE_IDX                                                      2
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE1                                                               0x00bf
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE1_BASE_IDX                                                      2
+#define regRCC_DEV0_2_RCC_BUS_CNTL                                                                      0x00c1
+#define regRCC_DEV0_2_RCC_BUS_CNTL_BASE_IDX                                                             2
+#define regRCC_DEV0_1_RCC_CONFIG_CNTL                                                                   0x00c2
+#define regRCC_DEV0_1_RCC_CONFIG_CNTL_BASE_IDX                                                          2
+#define regRCC_DEV0_1_RCC_CONFIG_F0_BASE                                                                0x00c6
+#define regRCC_DEV0_1_RCC_CONFIG_F0_BASE_BASE_IDX                                                       2
+#define regRCC_DEV0_1_RCC_CONFIG_APER_SIZE                                                              0x00c7
+#define regRCC_DEV0_1_RCC_CONFIG_APER_SIZE_BASE_IDX                                                     2
+#define regRCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE                                                          0x00c8
+#define regRCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE_BASE_IDX                                                 2
+#define regRCC_DEV0_1_RCC_XDMA_LO                                                                       0x00c9
+#define regRCC_DEV0_1_RCC_XDMA_LO_BASE_IDX                                                              2
+#define regRCC_DEV0_1_RCC_XDMA_HI                                                                       0x00ca
+#define regRCC_DEV0_1_RCC_XDMA_HI_BASE_IDX                                                              2
+#define regRCC_DEV0_2_RCC_FEATURES_CONTROL_MISC                                                         0x00cb
+#define regRCC_DEV0_2_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                2
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL1                                                                  0x00cc
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL1_BASE_IDX                                                         2
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST0                                                                  0x00cd
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST0_BASE_IDX                                                         2
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST1                                                                  0x00ce
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST1_BASE_IDX                                                         2
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL2                                                                  0x00cf
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL2_BASE_IDX                                                         2
+#define regRCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM                                                           0x00d0
+#define regRCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM_BASE_IDX                                                  2
+#define regRCC_DEV0_1_RCC_HOST_BUSNUM                                                                   0x00d1
+#define regRCC_DEV0_1_RCC_HOST_BUSNUM_BASE_IDX                                                          2
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI                                                            0x00d2
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO                                                            0x00d3
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI                                                            0x00d4
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO                                                            0x00d5
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI                                                            0x00d6
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO                                                            0x00d7
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI                                                            0x00d8
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO                                                            0x00d9
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST0                                                              0x00da
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST0_BASE_IDX                                                     2
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST1                                                              0x00db
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST1_BASE_IDX                                                     2
+#define regRCC_DEV0_2_RCC_DEV0_LINK_CNTL                                                                0x00dd
+#define regRCC_DEV0_2_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       2
+#define regRCC_DEV0_2_RCC_CMN_LINK_CNTL                                                                 0x00de
+#define regRCC_DEV0_2_RCC_CMN_LINK_CNTL_BASE_IDX                                                        2
+#define regRCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE                                                        0x00df
+#define regRCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               2
+#define regRCC_DEV0_2_RCC_LTR_LSWITCH_CNTL                                                              0x00e0
+#define regRCC_DEV0_2_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     2
+#define regRCC_DEV0_2_RCC_MH_ARB_CNTL                                                                   0x00e1
+#define regRCC_DEV0_2_RCC_MH_ARB_CNTL_BASE_IDX                                                          2
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_LO                                                        0x0400
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_HI                                                        0x0401
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_MSG_DATA                                                       0x0402
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                              3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_CONTROL                                                        0x0403
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT0_CONTROL_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_LO                                                        0x0404
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_HI                                                        0x0405
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_MSG_DATA                                                       0x0406
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                              3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_CONTROL                                                        0x0407
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT1_CONTROL_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_LO                                                        0x0408
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_HI                                                        0x0409
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_MSG_DATA                                                       0x040a
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                              3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_CONTROL                                                        0x040b
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT2_CONTROL_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_LO                                                        0x040c
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_HI                                                        0x040d
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_MSG_DATA                                                       0x040e
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                              3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_CONTROL                                                        0x040f
+#define regRCC_DEV0_EPF0_0_GFXMSIX_VECT3_CONTROL_BASE_IDX                                               3
+#define regRCC_DEV0_EPF0_0_GFXMSIX_PBA                                                                  0x0800
+#define regRCC_DEV0_EPF0_0_GFXMSIX_PBA_BASE_IDX                                                         3
+
+
+// addressBlock: nbif0_rcc_strap_BIFDEC1
+// base address: 0x0
+#define regRCC_STRAP1_RCC_BIF_STRAP0                                                                    0x0000
+#define regRCC_STRAP1_RCC_BIF_STRAP0_BASE_IDX                                                           2
+#define regRCC_STRAP1_RCC_BIF_STRAP1                                                                    0x0001
+#define regRCC_STRAP1_RCC_BIF_STRAP1_BASE_IDX                                                           2
+#define regRCC_STRAP1_RCC_BIF_STRAP2                                                                    0x0006
+#define regRCC_STRAP1_RCC_BIF_STRAP2_BASE_IDX                                                           2
+#define regRCC_STRAP1_RCC_BIF_STRAP3                                                                    0x0007
+#define regRCC_STRAP1_RCC_BIF_STRAP3_BASE_IDX                                                           2
+#define regRCC_STRAP1_RCC_BIF_STRAP4                                                                    0x0008
+#define regRCC_STRAP1_RCC_BIF_STRAP4_BASE_IDX                                                           2
+#define regRCC_STRAP1_RCC_BIF_STRAP5                                                                    0x0009
+#define regRCC_STRAP1_RCC_BIF_STRAP5_BASE_IDX                                                           2
+#define regRCC_STRAP1_RCC_BIF_STRAP6                                                                    0x000a
+#define regRCC_STRAP1_RCC_BIF_STRAP6_BASE_IDX                                                           2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP0                                                              0x000e
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP1                                                              0x000f
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP10                                                             0x0010
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP11                                                             0x0011
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP12                                                             0x0012
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP13                                                             0x0013
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP14                                                             0x0014
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP14_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP2                                                              0x0019
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP3                                                              0x001a
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP4                                                              0x001b
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP5                                                              0x001c
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP6                                                              0x001d
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP7                                                              0x001e
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP8                                                              0x001f
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP9                                                              0x0020
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0                                                              0x0021
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP1                                                              0x0022
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP13                                                             0x0023
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP14                                                             0x0024
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP15                                                             0x0025
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP16                                                             0x0026
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP17                                                             0x0027
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP18                                                             0x0028
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP2                                                              0x002a
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP3                                                              0x002e
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP4                                                              0x002f
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP5                                                              0x0030
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP8                                                              0x0031
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP9                                                              0x0032
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP0                                                              0x0033
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP2                                                              0x003f
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP20                                                             0x0040
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP20_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP21                                                             0x0041
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP21_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP23                                                             0x0042
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP23_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP24                                                             0x0043
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP24_BASE_IDX                                                    2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP3                                                              0x0044
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP4                                                              0x0045
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP5                                                              0x0046
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP6                                                              0x0047
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     2
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP7                                                              0x0048
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     2
+
+
+// addressBlock: nbif0_bif_bx_pf_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_PF1_BIF_BME_STATUS                                                                    0x00eb
+#define regBIF_BX_PF1_BIF_BME_STATUS_BASE_IDX                                                           2
+#define regBIF_BX_PF1_BIF_ATOMIC_ERR_LOG                                                                0x00ec
+#define regBIF_BX_PF1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                                       2
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x00f3
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                                     2
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x00f4
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                                      2
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x00f5
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                          2
+#define regBIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL                                                      0x00f6
+#define regBIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                             2
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x00f7
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                             2
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                                 0x00f9
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                                        2
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                            0x00fa
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                                   2
+#define regBIF_BX_PF1_BIF_DOORBELL_INT_CNTL                                                             0x00fe
+#define regBIF_BX_PF1_BIF_DOORBELL_INT_CNTL_BASE_IDX                                                    2
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ                                                                 0x0106
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ_BASE_IDX                                                        2
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE                                                                0x0107
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE_BASE_IDX                                                       2
+#define regBIF_BX_PF1_BIF_TRANS_PENDING                                                                 0x0108
+#define regBIF_BX_PF1_BIF_TRANS_PENDING_BASE_IDX                                                        2
+#define regBIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS                                                          0x0112
+#define regBIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                                 2
+#define regBIF_BX_PF1_BIF_RB_CNTL                                                                       0x012f
+#define regBIF_BX_PF1_BIF_RB_CNTL_BASE_IDX                                                              2
+#define regBIF_BX_PF1_BIF_RB_BASE                                                                       0x0130
+#define regBIF_BX_PF1_BIF_RB_BASE_BASE_IDX                                                              2
+#define regBIF_BX_PF1_BIF_RB_RPTR                                                                       0x0131
+#define regBIF_BX_PF1_BIF_RB_RPTR_BASE_IDX                                                              2
+#define regBIF_BX_PF1_BIF_RB_WPTR                                                                       0x0132
+#define regBIF_BX_PF1_BIF_RB_WPTR_BASE_IDX                                                              2
+#define regBIF_BX_PF1_BIF_RB_WPTR_ADDR_HI                                                               0x0133
+#define regBIF_BX_PF1_BIF_RB_WPTR_ADDR_HI_BASE_IDX                                                      2
+#define regBIF_BX_PF1_BIF_RB_WPTR_ADDR_LO                                                               0x0134
+#define regBIF_BX_PF1_BIF_RB_WPTR_ADDR_LO_BASE_IDX                                                      2
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_0_RCC_ERR_LOG                                                                  0x0085
+#define regRCC_DEV0_EPF0_0_RCC_ERR_LOG_BASE_IDX                                                         2
+#define regRCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN                                                         0x00c0
+#define regRCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE                                                           0x00c3
+#define regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE_BASE_IDX                                                  2
+#define regRCC_DEV0_EPF0_0_RCC_CONFIG_RESERVED                                                          0x00c4
+#define regRCC_DEV0_EPF0_0_RCC_CONFIG_RESERVED_BASE_IDX                                                 2
+#define regRCC_DEV0_EPF0_0_RCC_IOV_FUNC_IDENTIFIER                                                      0x00c5
+#define regRCC_DEV0_EPF0_0_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                             2
+
+
+// addressBlock: nbif0_rcc_strap_BIFDEC1:1
+// base address: 0x10120000
+#define regRCC_STRAP2_RCC_BIF_STRAP0                                                                    0x8d20
+#define regRCC_STRAP2_RCC_BIF_STRAP0_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP1                                                                    0x8d21
+#define regRCC_STRAP2_RCC_BIF_STRAP1_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP2                                                                    0x8d26
+#define regRCC_STRAP2_RCC_BIF_STRAP2_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP3                                                                    0x8d27
+#define regRCC_STRAP2_RCC_BIF_STRAP3_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP4                                                                    0x8d28
+#define regRCC_STRAP2_RCC_BIF_STRAP4_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP5                                                                    0x8d29
+#define regRCC_STRAP2_RCC_BIF_STRAP5_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP6                                                                    0x8d2a
+#define regRCC_STRAP2_RCC_BIF_STRAP6_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP0                                                              0x8d2e
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP1                                                              0x8d2f
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP10                                                             0x8d30
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP11                                                             0x8d31
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP12                                                             0x8d32
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP13                                                             0x8d33
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP14                                                             0x8d34
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP2                                                              0x8d39
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP3                                                              0x8d3a
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP4                                                              0x8d3b
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP5                                                              0x8d3c
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP6                                                              0x8d3d
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP7                                                              0x8d3e
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP8                                                              0x8d3f
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP9                                                              0x8d40
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP0                                                              0x8d41
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP1                                                              0x8d42
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP13                                                             0x8d43
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP14                                                             0x8d44
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP15                                                             0x8d45
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP16                                                             0x8d46
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP17                                                             0x8d47
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP18                                                             0x8d48
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP2                                                              0x8d4a
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP3                                                              0x8d4e
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP4                                                              0x8d4f
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP5                                                              0x8d50
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP8                                                              0x8d51
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP9                                                              0x8d52
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP0                                                              0x8d53
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP2                                                              0x8d5f
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP20                                                             0x8d60
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP20_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP21                                                             0x8d61
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP21_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP23                                                             0x8d62
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP23_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP24                                                             0x8d63
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP24_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP3                                                              0x8d64
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP4                                                              0x8d65
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP5                                                              0x8d66
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP6                                                              0x8d67
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP7                                                              0x8d68
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_gdc_GDCDEC
+// base address: 0x1400000
+#define regGDC0_NGDC_SDP_PORT_CTRL                                                                      0x4f0aa0
+#define regGDC0_NGDC_SDP_PORT_CTRL_BASE_IDX                                                             3
+#define regGDC0_SHUB_REGS_IF_CTL                                                                        0x4f0aa1
+#define regGDC0_SHUB_REGS_IF_CTL_BASE_IDX                                                               3
+#define regGDC0_NGDC_MGCG_CTRL                                                                          0x4f0aa7
+#define regGDC0_NGDC_MGCG_CTRL_BASE_IDX                                                                 3
+#define regGDC0_S2A_MISC_CNTL                                                                           0x4f0aa8
+#define regGDC0_S2A_MISC_CNTL_BASE_IDX                                                                  3
+#define regGDC0_NGDC_PG_MISC_CTRL                                                                       0x4f0ab0
+#define regGDC0_NGDC_PG_MISC_CTRL_BASE_IDX                                                              3
+#define regGDC0_NGDC_PGMST_CTRL                                                                         0x4f0ab1
+#define regGDC0_NGDC_PGMST_CTRL_BASE_IDX                                                                3
+#define regGDC0_NGDC_PGSLV_CTRL                                                                         0x4f0ab2
+#define regGDC0_NGDC_PGSLV_CTRL_BASE_IDX                                                                3
+#define regGDC0_SHUBCLK_DPM_CTRL                                                                        0x4f0ab3
+#define regGDC0_SHUBCLK_DPM_CTRL_BASE_IDX                                                               3
+#define regGDC0_SHUBCLK_DPM_WR_WEIGHT                                                                   0x4f0ab4
+#define regGDC0_SHUBCLK_DPM_WR_WEIGHT_BASE_IDX                                                          3
+#define regGDC0_SHUBCLK_DPM_RD_WEIGHT                                                                   0x4f0ab5
+#define regGDC0_SHUBCLK_DPM_RD_WEIGHT_BASE_IDX                                                          3
+#define regGDC0_SHUBCLK_DPM_WR_CNT                                                                      0x4f0ab6
+#define regGDC0_SHUBCLK_DPM_WR_CNT_BASE_IDX                                                             3
+#define regGDC0_SHUBCLK_DPM_RD_CNT                                                                      0x4f0ab7
+#define regGDC0_SHUBCLK_DPM_RD_CNT_BASE_IDX                                                             3
+#define regGDC0_ATDMA_MISC_CNTL                                                                         0x4f0b13
+#define regGDC0_ATDMA_MISC_CNTL_BASE_IDX                                                                3
+
+
+// addressBlock: nbif0_gdc_s2a_GDCS2A_DEC
+// base address: 0x1400000
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL                                                           0x4f0aeb
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL                                                           0x4f0aed
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL                                                           0x4f0aef
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL                                                           0x4f0af1
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL                                                           0x4f0af3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL                                                           0x4f0af5
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL                                                           0x4f0af7
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL                                                           0x4f0af9
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL                                                           0x4f0afb
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL                                                           0x4f0afd
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL                                                          0x4f0aff
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL                                                          0x4f0b01
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL                                                          0x4f0b03
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL                                                          0x4f0b05
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL                                                          0x4f0b07
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL                                                          0x4f0b09
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG                                                        0x4f0b0b
+#define regGDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG_BASE_IDX                                               3
+#define regGDC_S2A0_NBIF_GFX_DOORBELL_STATUS                                                            0x4f0b0c
+#define regGDC_S2A0_NBIF_GFX_DOORBELL_STATUS_BASE_IDX                                                   3
+
+
+// addressBlock: nbif0_gdc_GDC_LINEAR_REGION
+// base address: 0x1400000
+#define regGDC1_NGDC_SDP_PORT_CTRL                                                                      0x4f25ce
+#define regGDC1_NGDC_SDP_PORT_CTRL_BASE_IDX                                                             3
+#define regGDC1_SHUB_REGS_IF_CTL                                                                        0x4f25cf
+#define regGDC1_SHUB_REGS_IF_CTL_BASE_IDX                                                               3
+#define regGDC1_NGDC_MGCG_CTRL                                                                          0x4f25d1
+#define regGDC1_NGDC_MGCG_CTRL_BASE_IDX                                                                 3
+#define regGDC1_S2A_MISC_CNTL                                                                           0x4f25d2
+#define regGDC1_S2A_MISC_CNTL_BASE_IDX                                                                  3
+#define regGDC1_NGDC_EARLY_WAKEUP_CTRL                                                                  0x4f25d5
+#define regGDC1_NGDC_EARLY_WAKEUP_CTRL_BASE_IDX                                                         3
+#define regGDC1_SHUBCLK_DPM_CTRL                                                                        0x4f25d8
+#define regGDC1_SHUBCLK_DPM_CTRL_BASE_IDX                                                               3
+#define regGDC1_SHUBCLK_DPM_WR_WEIGHT                                                                   0x4f25d9
+#define regGDC1_SHUBCLK_DPM_WR_WEIGHT_BASE_IDX                                                          3
+#define regGDC1_SHUBCLK_DPM_RD_WEIGHT                                                                   0x4f25da
+#define regGDC1_SHUBCLK_DPM_RD_WEIGHT_BASE_IDX                                                          3
+#define regGDC1_SHUBCLK_DPM_WR_CNT                                                                      0x4f25db
+#define regGDC1_SHUBCLK_DPM_WR_CNT_BASE_IDX                                                             3
+#define regGDC1_SHUBCLK_DPM_RD_CNT                                                                      0x4f25dc
+#define regGDC1_SHUBCLK_DPM_RD_CNT_BASE_IDX                                                             3
+#define regGDC1_ATDMA_MISC_CNTL                                                                         0x4f25e4
+#define regGDC1_ATDMA_MISC_CNTL_BASE_IDX                                                                3
+
+
+// addressBlock: nbif0_gdc_s2a_GDC_LINEAR_REGION
+// base address: 0x1400000
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL                                                           0x4f25ab
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL                                                           0x4f25ad
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL                                                           0x4f25af
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL                                                           0x4f25b1
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL                                                           0x4f25b3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL                                                           0x4f25b5
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL                                                           0x4f25b7
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL                                                           0x4f25b9
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL                                                           0x4f25bb
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL                                                           0x4f25bd
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL                                                          0x4f25bf
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL                                                          0x4f25c1
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL                                                          0x4f25c3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL                                                          0x4f25c5
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL                                                          0x4f25c7
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL                                                          0x4f25c9
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG                                                        0x4f25cb
+#define regGDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG_BASE_IDX                                               3
+#define regGDC_S2A1_NBIF_GFX_DOORBELL_STATUS                                                            0x4f25cc
+#define regGDC_S2A1_NBIF_GFX_DOORBELL_STATUS_BASE_IDX                                                   3
+
+
+// addressBlock: nbif0_syshub_mmreg_syshubdec
+// base address: 0x0
+#define regSYSHUB_INDEX                                                                                 0x0008
+#define regSYSHUB_INDEX_BASE_IDX                                                                        0
+#define regSYSHUB_DATA                                                                                  0x0009
+#define regSYSHUB_DATA_BASE_IDX                                                                         0
+
+
+// addressBlock: nbif0_bif_bx_SYSDEC
+// base address: 0xd0000000
+#define regBIF_BX2_PCIE_INDEX                                                                           0x2ffc000c
+#define regBIF_BX2_PCIE_INDEX_BASE_IDX                                                                  5
+#define regBIF_BX2_PCIE_DATA                                                                            0x2ffc000d
+#define regBIF_BX2_PCIE_DATA_BASE_IDX                                                                   5
+#define regBIF_BX2_PCIE_INDEX2                                                                          0x2ffc000e
+#define regBIF_BX2_PCIE_INDEX2_BASE_IDX                                                                 5
+#define regBIF_BX2_PCIE_DATA2                                                                           0x2ffc000f
+#define regBIF_BX2_PCIE_DATA2_BASE_IDX                                                                  5
+#define regBIF_BX2_SBIOS_SCRATCH_0                                                                      0x2ffc0048
+#define regBIF_BX2_SBIOS_SCRATCH_0_BASE_IDX                                                             5
+#define regBIF_BX2_SBIOS_SCRATCH_1                                                                      0x2ffc0049
+#define regBIF_BX2_SBIOS_SCRATCH_1_BASE_IDX                                                             5
+#define regBIF_BX2_SBIOS_SCRATCH_2                                                                      0x2ffc004a
+#define regBIF_BX2_SBIOS_SCRATCH_2_BASE_IDX                                                             5
+#define regBIF_BX2_SBIOS_SCRATCH_3                                                                      0x2ffc004b
+#define regBIF_BX2_SBIOS_SCRATCH_3_BASE_IDX                                                             5
+#define regBIF_BX2_BIOS_SCRATCH_0                                                                       0x2ffc004c
+#define regBIF_BX2_BIOS_SCRATCH_0_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_1                                                                       0x2ffc004d
+#define regBIF_BX2_BIOS_SCRATCH_1_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_2                                                                       0x2ffc004e
+#define regBIF_BX2_BIOS_SCRATCH_2_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_3                                                                       0x2ffc004f
+#define regBIF_BX2_BIOS_SCRATCH_3_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_4                                                                       0x2ffc0050
+#define regBIF_BX2_BIOS_SCRATCH_4_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_5                                                                       0x2ffc0051
+#define regBIF_BX2_BIOS_SCRATCH_5_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_6                                                                       0x2ffc0052
+#define regBIF_BX2_BIOS_SCRATCH_6_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_7                                                                       0x2ffc0053
+#define regBIF_BX2_BIOS_SCRATCH_7_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_8                                                                       0x2ffc0054
+#define regBIF_BX2_BIOS_SCRATCH_8_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_9                                                                       0x2ffc0055
+#define regBIF_BX2_BIOS_SCRATCH_9_BASE_IDX                                                              5
+#define regBIF_BX2_BIOS_SCRATCH_10                                                                      0x2ffc0056
+#define regBIF_BX2_BIOS_SCRATCH_10_BASE_IDX                                                             5
+#define regBIF_BX2_BIOS_SCRATCH_11                                                                      0x2ffc0057
+#define regBIF_BX2_BIOS_SCRATCH_11_BASE_IDX                                                             5
+#define regBIF_BX2_BIOS_SCRATCH_12                                                                      0x2ffc0058
+#define regBIF_BX2_BIOS_SCRATCH_12_BASE_IDX                                                             5
+#define regBIF_BX2_BIOS_SCRATCH_13                                                                      0x2ffc0059
+#define regBIF_BX2_BIOS_SCRATCH_13_BASE_IDX                                                             5
+#define regBIF_BX2_BIOS_SCRATCH_14                                                                      0x2ffc005a
+#define regBIF_BX2_BIOS_SCRATCH_14_BASE_IDX                                                             5
+#define regBIF_BX2_BIOS_SCRATCH_15                                                                      0x2ffc005b
+#define regBIF_BX2_BIOS_SCRATCH_15_BASE_IDX                                                             5
+#define regBIF_BX2_BIF_RLC_INTR_CNTL                                                                    0x2ffc0060
+#define regBIF_BX2_BIF_RLC_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX2_BIF_VCE_INTR_CNTL                                                                    0x2ffc0061
+#define regBIF_BX2_BIF_VCE_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX2_BIF_UVD_INTR_CNTL                                                                    0x2ffc0062
+#define regBIF_BX2_BIF_UVD_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR0                                                                0x2ffc0080
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR0_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR0                                                          0x2ffc0081
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR0_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR1                                                                0x2ffc0082
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR1_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR1                                                          0x2ffc0083
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR1_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR2                                                                0x2ffc0084
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR2_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR2                                                          0x2ffc0085
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR2_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR3                                                                0x2ffc0086
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR3_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR3                                                          0x2ffc0087
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR3_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR4                                                                0x2ffc0088
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR4_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR4                                                          0x2ffc0089
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR4_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR5                                                                0x2ffc008a
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR5_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR5                                                          0x2ffc008b
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR5_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR6                                                                0x2ffc008c
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR6_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR6                                                          0x2ffc008d
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR6_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR7                                                                0x2ffc008e
+#define regBIF_BX2_GFX_MMIOREG_CAM_ADDR7_BASE_IDX                                                       5
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR7                                                          0x2ffc008f
+#define regBIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR7_BASE_IDX                                                 5
+#define regBIF_BX2_GFX_MMIOREG_CAM_CNTL                                                                 0x2ffc0090
+#define regBIF_BX2_GFX_MMIOREG_CAM_CNTL_BASE_IDX                                                        5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ZERO_CPL                                                             0x2ffc0091
+#define regBIF_BX2_GFX_MMIOREG_CAM_ZERO_CPL_BASE_IDX                                                    5
+#define regBIF_BX2_GFX_MMIOREG_CAM_ONE_CPL                                                              0x2ffc0092
+#define regBIF_BX2_GFX_MMIOREG_CAM_ONE_CPL_BASE_IDX                                                     5
+#define regBIF_BX2_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL                                                     0x2ffc0093
+#define regBIF_BX2_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL_BASE_IDX                                            5
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_BIFDEC1
+// base address: 0xd0000000
+#define regRCC_DWN_DEV0_3_DN_PCIE_RESERVED                                                              0x2ffc0d88
+#define regRCC_DWN_DEV0_3_DN_PCIE_RESERVED_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_3_DN_PCIE_SCRATCH                                                               0x2ffc0d89
+#define regRCC_DWN_DEV0_3_DN_PCIE_SCRATCH_BASE_IDX                                                      5
+#define regRCC_DWN_DEV0_3_DN_PCIE_CNTL                                                                  0x2ffc0d8b
+#define regRCC_DWN_DEV0_3_DN_PCIE_CNTL_BASE_IDX                                                         5
+#define regRCC_DWN_DEV0_3_DN_PCIE_CONFIG_CNTL                                                           0x2ffc0d8c
+#define regRCC_DWN_DEV0_3_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  5
+#define regRCC_DWN_DEV0_3_DN_PCIE_RX_CNTL2                                                              0x2ffc0d8d
+#define regRCC_DWN_DEV0_3_DN_PCIE_RX_CNTL2_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_3_DN_PCIE_BUS_CNTL                                                              0x2ffc0d8e
+#define regRCC_DWN_DEV0_3_DN_PCIE_BUS_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL                                                              0x2ffc0d8f
+#define regRCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_3_DN_PCIE_STRAP_F0                                                              0x2ffc0d90
+#define regRCC_DWN_DEV0_3_DN_PCIE_STRAP_F0_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC                                                            0x2ffc0d91
+#define regRCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC_BASE_IDX                                                   5
+#define regRCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC2                                                           0x2ffc0d92
+#define regRCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_BIFDEC1
+// base address: 0xd0000000
+#define regRCC_DWNP_DEV0_3_PCIE_ERR_CNTL                                                                0x2ffc0d94
+#define regRCC_DWNP_DEV0_3_PCIE_ERR_CNTL_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_3_PCIE_RX_CNTL                                                                 0x2ffc0d95
+#define regRCC_DWNP_DEV0_3_PCIE_RX_CNTL_BASE_IDX                                                        5
+#define regRCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL                                                           0x2ffc0d96
+#define regRCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  5
+#define regRCC_DWNP_DEV0_3_PCIE_LC_CNTL2                                                                0x2ffc0d97
+#define regRCC_DWNP_DEV0_3_PCIE_LC_CNTL2_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_3_PCIEP_STRAP_MISC                                                             0x2ffc0d98
+#define regRCC_DWNP_DEV0_3_PCIEP_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_DWNP_DEV0_3_LTR_MSG_INFO_FROM_EP                                                         0x2ffc0d99
+#define regRCC_DWNP_DEV0_3_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                5
+
+
+// addressBlock: nbif0_rcc_ep_dev0_BIFDEC1
+// base address: 0xd0000000
+#define regRCC_EP_DEV0_3_EP_PCIE_SCRATCH                                                                0x2ffc0d6b
+#define regRCC_EP_DEV0_3_EP_PCIE_SCRATCH_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_3_EP_PCIE_CNTL                                                                   0x2ffc0d6d
+#define regRCC_EP_DEV0_3_EP_PCIE_CNTL_BASE_IDX                                                          5
+#define regRCC_EP_DEV0_3_EP_PCIE_INT_CNTL                                                               0x2ffc0d6e
+#define regRCC_EP_DEV0_3_EP_PCIE_INT_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_INT_STATUS                                                             0x2ffc0d6f
+#define regRCC_EP_DEV0_3_EP_PCIE_INT_STATUS_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_3_EP_PCIE_RX_CNTL2                                                               0x2ffc0d70
+#define regRCC_EP_DEV0_3_EP_PCIE_RX_CNTL2_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_BUS_CNTL                                                               0x2ffc0d71
+#define regRCC_EP_DEV0_3_EP_PCIE_BUS_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_CFG_CNTL                                                               0x2ffc0d72
+#define regRCC_EP_DEV0_3_EP_PCIE_CFG_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL                                                            0x2ffc0d74
+#define regRCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0                                               0x2ffc0d75
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1                                               0x2ffc0d75
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2                                               0x2ffc0d75
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3                                               0x2ffc0d75
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4                                               0x2ffc0d76
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5                                               0x2ffc0d76
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6                                               0x2ffc0d76
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7                                               0x2ffc0d76
+#define regRCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_STRAP_MISC                                                             0x2ffc0d77
+#define regRCC_EP_DEV0_3_EP_PCIE_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_3_EP_PCIE_STRAP_MISC2                                                            0x2ffc0d78
+#define regRCC_EP_DEV0_3_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP                                                             0x2ffc0d7a
+#define regRCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_3_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0x2ffc0d7b
+#define regRCC_EP_DEV0_3_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_F0_DPA_CNTL                                                            0x2ffc0d7b
+#define regRCC_EP_DEV0_3_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0x2ffc0d7b
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0x2ffc0d7c
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0x2ffc0d7c
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0x2ffc0d7c
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0x2ffc0d7c
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0x2ffc0d7d
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0x2ffc0d7d
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0x2ffc0d7d
+#define regRCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_PME_CONTROL                                                            0x2ffc0d7d
+#define regRCC_EP_DEV0_3_EP_PCIE_PME_CONTROL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_3_EP_PCIEP_RESERVED                                                              0x2ffc0d7e
+#define regRCC_EP_DEV0_3_EP_PCIEP_RESERVED_BASE_IDX                                                     5
+#define regRCC_EP_DEV0_3_EP_PCIE_TX_CNTL                                                                0x2ffc0d80
+#define regRCC_EP_DEV0_3_EP_PCIE_TX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID                                                        0x2ffc0d81
+#define regRCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               5
+#define regRCC_EP_DEV0_3_EP_PCIE_ERR_CNTL                                                               0x2ffc0d82
+#define regRCC_EP_DEV0_3_EP_PCIE_ERR_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_3_EP_PCIE_RX_CNTL                                                                0x2ffc0d83
+#define regRCC_EP_DEV0_3_EP_PCIE_RX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL                                                          0x2ffc0d84
+#define regRCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_bif_bx_pf_SYSPFVFDEC
+// base address: 0xd0000000
+#define regBIF_BX_PF2_MM_INDEX                                                                          0x2ffc0000
+#define regBIF_BX_PF2_MM_INDEX_BASE_IDX                                                                 5
+#define regBIF_BX_PF2_MM_DATA                                                                           0x2ffc0001
+#define regBIF_BX_PF2_MM_DATA_BASE_IDX                                                                  5
+#define regBIF_BX_PF2_MM_INDEX_HI                                                                       0x2ffc0006
+#define regBIF_BX_PF2_MM_INDEX_HI_BASE_IDX                                                              5
+
+
+// addressBlock: nbif0_bif_bx_BIFDEC1
+// base address: 0xd0000000
+#define regBIF_BX2_BIF_MM_INDACCESS_CNTL                                                                0x2ffc0e06
+#define regBIF_BX2_BIF_MM_INDACCESS_CNTL_BASE_IDX                                                       5
+#define regBIF_BX2_BUS_CNTL                                                                             0x2ffc0e07
+#define regBIF_BX2_BUS_CNTL_BASE_IDX                                                                    5
+#define regBIF_BX2_BIF_SCRATCH0                                                                         0x2ffc0e08
+#define regBIF_BX2_BIF_SCRATCH0_BASE_IDX                                                                5
+#define regBIF_BX2_BIF_SCRATCH1                                                                         0x2ffc0e09
+#define regBIF_BX2_BIF_SCRATCH1_BASE_IDX                                                                5
+#define regBIF_BX2_BX_RESET_EN                                                                          0x2ffc0e0d
+#define regBIF_BX2_BX_RESET_EN_BASE_IDX                                                                 5
+#define regBIF_BX2_MM_CFGREGS_CNTL                                                                      0x2ffc0e0e
+#define regBIF_BX2_MM_CFGREGS_CNTL_BASE_IDX                                                             5
+#define regBIF_BX2_BX_RESET_CNTL                                                                        0x2ffc0e10
+#define regBIF_BX2_BX_RESET_CNTL_BASE_IDX                                                               5
+#define regBIF_BX2_INTERRUPT_CNTL                                                                       0x2ffc0e11
+#define regBIF_BX2_INTERRUPT_CNTL_BASE_IDX                                                              5
+#define regBIF_BX2_INTERRUPT_CNTL2                                                                      0x2ffc0e12
+#define regBIF_BX2_INTERRUPT_CNTL2_BASE_IDX                                                             5
+#define regBIF_BX2_CLKREQB_PAD_CNTL                                                                     0x2ffc0e18
+#define regBIF_BX2_CLKREQB_PAD_CNTL_BASE_IDX                                                            5
+#define regBIF_BX2_BIF_FEATURES_CONTROL_MISC                                                            0x2ffc0e1b
+#define regBIF_BX2_BIF_FEATURES_CONTROL_MISC_BASE_IDX                                                   5
+#define regBIF_BX2_HDP_ATOMIC_CONTROL_MISC                                                              0x2ffc0e1c
+#define regBIF_BX2_HDP_ATOMIC_CONTROL_MISC_BASE_IDX                                                     5
+#define regBIF_BX2_BIF_DOORBELL_CNTL                                                                    0x2ffc0e1d
+#define regBIF_BX2_BIF_DOORBELL_CNTL_BASE_IDX                                                           5
+#define regBIF_BX2_BIF_FB_EN                                                                            0x2ffc0e20
+#define regBIF_BX2_BIF_FB_EN_BASE_IDX                                                                   5
+#define regBIF_BX2_BIF_MST_TRANS_PENDING_VF                                                             0x2ffc0e29
+#define regBIF_BX2_BIF_MST_TRANS_PENDING_VF_BASE_IDX                                                    5
+#define regBIF_BX2_BIF_SLV_TRANS_PENDING_VF                                                             0x2ffc0e2a
+#define regBIF_BX2_BIF_SLV_TRANS_PENDING_VF_BASE_IDX                                                    5
+#define regBIF_BX2_BACO_CNTL                                                                            0x2ffc0e2b
+#define regBIF_BX2_BACO_CNTL_BASE_IDX                                                                   5
+#define regBIF_BX2_BIF_BACO_EXIT_TIME0                                                                  0x2ffc0e2c
+#define regBIF_BX2_BIF_BACO_EXIT_TIME0_BASE_IDX                                                         5
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER1                                                                 0x2ffc0e2d
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER1_BASE_IDX                                                        5
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER2                                                                 0x2ffc0e2e
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER2_BASE_IDX                                                        5
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER3                                                                 0x2ffc0e2f
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER3_BASE_IDX                                                        5
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER4                                                                 0x2ffc0e30
+#define regBIF_BX2_BIF_BACO_EXIT_TIMER4_BASE_IDX                                                        5
+#define regBIF_BX2_MEM_TYPE_CNTL                                                                        0x2ffc0e31
+#define regBIF_BX2_MEM_TYPE_CNTL_BASE_IDX                                                               5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_CNTL                                                               0x2ffc0e33
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_CNTL_BASE_IDX                                                      5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_0                                                                  0x2ffc0e34
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_0_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_1                                                                  0x2ffc0e35
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_1_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_2                                                                  0x2ffc0e36
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_2_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_3                                                                  0x2ffc0e37
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_3_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_4                                                                  0x2ffc0e38
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_4_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_5                                                                  0x2ffc0e39
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_5_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_6                                                                  0x2ffc0e3a
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_6_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_7                                                                  0x2ffc0e3b
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_7_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_8                                                                  0x2ffc0e3c
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_8_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_9                                                                  0x2ffc0e3d
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_9_BASE_IDX                                                         5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_10                                                                 0x2ffc0e3e
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_10_BASE_IDX                                                        5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_11                                                                 0x2ffc0e3f
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_11_BASE_IDX                                                        5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_12                                                                 0x2ffc0e40
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_12_BASE_IDX                                                        5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_13                                                                 0x2ffc0e41
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_13_BASE_IDX                                                        5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_14                                                                 0x2ffc0e42
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_14_BASE_IDX                                                        5
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_15                                                                 0x2ffc0e43
+#define regBIF_BX2_NBIF_GFX_ADDR_LUT_15_BASE_IDX                                                        5
+#define regBIF_BX2_VF_REGWR_EN                                                                          0x2ffc0e44
+#define regBIF_BX2_VF_REGWR_EN_BASE_IDX                                                                 5
+#define regBIF_BX2_VF_DOORBELL_EN                                                                       0x2ffc0e45
+#define regBIF_BX2_VF_DOORBELL_EN_BASE_IDX                                                              5
+#define regBIF_BX2_VF_FB_EN                                                                             0x2ffc0e46
+#define regBIF_BX2_VF_FB_EN_BASE_IDX                                                                    5
+#define regBIF_BX2_VF_REGWR_STATUS                                                                      0x2ffc0e47
+#define regBIF_BX2_VF_REGWR_STATUS_BASE_IDX                                                             5
+#define regBIF_BX2_VF_DOORBELL_STATUS                                                                   0x2ffc0e48
+#define regBIF_BX2_VF_DOORBELL_STATUS_BASE_IDX                                                          5
+#define regBIF_BX2_VF_FB_STATUS                                                                         0x2ffc0e49
+#define regBIF_BX2_VF_FB_STATUS_BASE_IDX                                                                5
+#define regBIF_BX2_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x2ffc0e4d
+#define regBIF_BX2_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX                                                    5
+#define regBIF_BX2_REMAP_HDP_REG_FLUSH_CNTL                                                             0x2ffc0e4e
+#define regBIF_BX2_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX                                                    5
+#define regBIF_BX2_MAILBOX_INDEX                                                                        0x2ffc0e55
+#define regBIF_BX2_MAILBOX_INDEX_BASE_IDX                                                               5
+#define regBIF_BX2_BIF_PERSTB_PAD_CNTL                                                                  0x2ffc0e65
+#define regBIF_BX2_BIF_PERSTB_PAD_CNTL_BASE_IDX                                                         5
+#define regBIF_BX2_BIF_PX_EN_PAD_CNTL                                                                   0x2ffc0e66
+#define regBIF_BX2_BIF_PX_EN_PAD_CNTL_BASE_IDX                                                          5
+#define regBIF_BX2_BIF_REFPADKIN_PAD_CNTL                                                               0x2ffc0e67
+#define regBIF_BX2_BIF_REFPADKIN_PAD_CNTL_BASE_IDX                                                      5
+#define regBIF_BX2_BIF_CLKREQB_PAD_CNTL                                                                 0x2ffc0e69
+#define regBIF_BX2_BIF_CLKREQB_PAD_CNTL_BASE_IDX                                                        5
+#define regBIF_BX2_BIF_PWRBRK_PAD_CNTL                                                                  0x2ffc0e6a
+#define regBIF_BX2_BIF_PWRBRK_PAD_CNTL_BASE_IDX                                                         5
+
+
+// addressBlock: nbif0_rcc_dev0_BIFDEC1
+// base address: 0xd0000000
+#define regRCC_DEV0_2_RCC_ERR_INT_CNTL                                                                  0x2ffc0da6
+#define regRCC_DEV0_2_RCC_ERR_INT_CNTL_BASE_IDX                                                         5
+#define regRCC_DEV0_2_RCC_BACO_CNTL_MISC                                                                0x2ffc0da7
+#define regRCC_DEV0_2_RCC_BACO_CNTL_MISC_BASE_IDX                                                       5
+#define regRCC_DEV0_2_RCC_RESET_EN                                                                      0x2ffc0da8
+#define regRCC_DEV0_2_RCC_RESET_EN_BASE_IDX                                                             5
+#define regRCC_DEV0_3_RCC_VDM_SUPPORT                                                                   0x2ffc0da9
+#define regRCC_DEV0_3_RCC_VDM_SUPPORT_BASE_IDX                                                          5
+#define regRCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0                                                            0x2ffc0daa
+#define regRCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   5
+#define regRCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1                                                            0x2ffc0dab
+#define regRCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_GPUIOV_REGION                                                                 0x2ffc0dac
+#define regRCC_DEV0_2_RCC_GPUIOV_REGION_BASE_IDX                                                        5
+#define regRCC_DEV0_2_RCC_GPU_HOSTVM_EN                                                                 0x2ffc0dad
+#define regRCC_DEV0_2_RCC_GPU_HOSTVM_EN_BASE_IDX                                                        5
+#define regRCC_DEV0_2_RCC_CONSOLE_IOV_MODE_CNTL                                                         0x2ffc0dae
+#define regRCC_DEV0_2_RCC_CONSOLE_IOV_MODE_CNTL_BASE_IDX                                                5
+#define regRCC_DEV0_2_RCC_CONSOLE_IOV_FIRST_VF_OFFSET                                                   0x2ffc0daf
+#define regRCC_DEV0_2_RCC_CONSOLE_IOV_FIRST_VF_OFFSET_BASE_IDX                                          5
+#define regRCC_DEV0_2_RCC_CONSOLE_IOV_VF_STRIDE                                                         0x2ffc0daf
+#define regRCC_DEV0_2_RCC_CONSOLE_IOV_VF_STRIDE_BASE_IDX                                                5
+#define regRCC_DEV0_2_RCC_PEER_REG_RANGE0                                                               0x2ffc0dde
+#define regRCC_DEV0_2_RCC_PEER_REG_RANGE0_BASE_IDX                                                      5
+#define regRCC_DEV0_2_RCC_PEER_REG_RANGE1                                                               0x2ffc0ddf
+#define regRCC_DEV0_2_RCC_PEER_REG_RANGE1_BASE_IDX                                                      5
+#define regRCC_DEV0_3_RCC_BUS_CNTL                                                                      0x2ffc0de1
+#define regRCC_DEV0_3_RCC_BUS_CNTL_BASE_IDX                                                             5
+#define regRCC_DEV0_2_RCC_CONFIG_CNTL                                                                   0x2ffc0de2
+#define regRCC_DEV0_2_RCC_CONFIG_CNTL_BASE_IDX                                                          5
+#define regRCC_DEV0_2_RCC_CONFIG_F0_BASE                                                                0x2ffc0de6
+#define regRCC_DEV0_2_RCC_CONFIG_F0_BASE_BASE_IDX                                                       5
+#define regRCC_DEV0_2_RCC_CONFIG_APER_SIZE                                                              0x2ffc0de7
+#define regRCC_DEV0_2_RCC_CONFIG_APER_SIZE_BASE_IDX                                                     5
+#define regRCC_DEV0_2_RCC_CONFIG_REG_APER_SIZE                                                          0x2ffc0de8
+#define regRCC_DEV0_2_RCC_CONFIG_REG_APER_SIZE_BASE_IDX                                                 5
+#define regRCC_DEV0_2_RCC_XDMA_LO                                                                       0x2ffc0de9
+#define regRCC_DEV0_2_RCC_XDMA_LO_BASE_IDX                                                              5
+#define regRCC_DEV0_2_RCC_XDMA_HI                                                                       0x2ffc0dea
+#define regRCC_DEV0_2_RCC_XDMA_HI_BASE_IDX                                                              5
+#define regRCC_DEV0_3_RCC_FEATURES_CONTROL_MISC                                                         0x2ffc0deb
+#define regRCC_DEV0_3_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                5
+#define regRCC_DEV0_2_RCC_BUSNUM_CNTL1                                                                  0x2ffc0dec
+#define regRCC_DEV0_2_RCC_BUSNUM_CNTL1_BASE_IDX                                                         5
+#define regRCC_DEV0_2_RCC_BUSNUM_LIST0                                                                  0x2ffc0ded
+#define regRCC_DEV0_2_RCC_BUSNUM_LIST0_BASE_IDX                                                         5
+#define regRCC_DEV0_2_RCC_BUSNUM_LIST1                                                                  0x2ffc0dee
+#define regRCC_DEV0_2_RCC_BUSNUM_LIST1_BASE_IDX                                                         5
+#define regRCC_DEV0_2_RCC_BUSNUM_CNTL2                                                                  0x2ffc0def
+#define regRCC_DEV0_2_RCC_BUSNUM_CNTL2_BASE_IDX                                                         5
+#define regRCC_DEV0_2_RCC_CAPTURE_HOST_BUSNUM                                                           0x2ffc0df0
+#define regRCC_DEV0_2_RCC_CAPTURE_HOST_BUSNUM_BASE_IDX                                                  5
+#define regRCC_DEV0_2_RCC_HOST_BUSNUM                                                                   0x2ffc0df1
+#define regRCC_DEV0_2_RCC_HOST_BUSNUM_BASE_IDX                                                          5
+#define regRCC_DEV0_2_RCC_PEER0_FB_OFFSET_HI                                                            0x2ffc0df2
+#define regRCC_DEV0_2_RCC_PEER0_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_PEER0_FB_OFFSET_LO                                                            0x2ffc0df3
+#define regRCC_DEV0_2_RCC_PEER0_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_PEER1_FB_OFFSET_HI                                                            0x2ffc0df4
+#define regRCC_DEV0_2_RCC_PEER1_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_PEER1_FB_OFFSET_LO                                                            0x2ffc0df5
+#define regRCC_DEV0_2_RCC_PEER1_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_PEER2_FB_OFFSET_HI                                                            0x2ffc0df6
+#define regRCC_DEV0_2_RCC_PEER2_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_PEER2_FB_OFFSET_LO                                                            0x2ffc0df7
+#define regRCC_DEV0_2_RCC_PEER2_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_PEER3_FB_OFFSET_HI                                                            0x2ffc0df8
+#define regRCC_DEV0_2_RCC_PEER3_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_PEER3_FB_OFFSET_LO                                                            0x2ffc0df9
+#define regRCC_DEV0_2_RCC_PEER3_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_DEVFUNCNUM_LIST0                                                              0x2ffc0dfa
+#define regRCC_DEV0_2_RCC_DEVFUNCNUM_LIST0_BASE_IDX                                                     5
+#define regRCC_DEV0_2_RCC_DEVFUNCNUM_LIST1                                                              0x2ffc0dfb
+#define regRCC_DEV0_2_RCC_DEVFUNCNUM_LIST1_BASE_IDX                                                     5
+#define regRCC_DEV0_3_RCC_DEV0_LINK_CNTL                                                                0x2ffc0dfd
+#define regRCC_DEV0_3_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       5
+#define regRCC_DEV0_3_RCC_CMN_LINK_CNTL                                                                 0x2ffc0dfe
+#define regRCC_DEV0_3_RCC_CMN_LINK_CNTL_BASE_IDX                                                        5
+#define regRCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE                                                        0x2ffc0dff
+#define regRCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               5
+#define regRCC_DEV0_3_RCC_LTR_LSWITCH_CNTL                                                              0x2ffc0e00
+#define regRCC_DEV0_3_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     5
+#define regRCC_DEV0_3_RCC_MH_ARB_CNTL                                                                   0x2ffc0e01
+#define regRCC_DEV0_3_RCC_MH_ARB_CNTL_BASE_IDX                                                          5
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFDEC2
+// base address: 0xd0000000
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_LO                                                        0x2ffd0800
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_HI                                                        0x2ffd0801
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_MSG_DATA                                                       0x2ffd0802
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                              5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_CONTROL                                                        0x2ffd0803
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT0_CONTROL_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_LO                                                        0x2ffd0804
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_HI                                                        0x2ffd0805
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_MSG_DATA                                                       0x2ffd0806
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                              5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_CONTROL                                                        0x2ffd0807
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT1_CONTROL_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_LO                                                        0x2ffd0808
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_HI                                                        0x2ffd0809
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_MSG_DATA                                                       0x2ffd080a
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                              5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_CONTROL                                                        0x2ffd080b
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT2_CONTROL_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_LO                                                        0x2ffd080c
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_HI                                                        0x2ffd080d
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_MSG_DATA                                                       0x2ffd080e
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                              5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_CONTROL                                                        0x2ffd080f
+#define regRCC_DEV0_EPF0_1_GFXMSIX_VECT3_CONTROL_BASE_IDX                                               5
+#define regRCC_DEV0_EPF0_1_GFXMSIX_PBA                                                                  0x2ffd0c00
+#define regRCC_DEV0_EPF0_1_GFXMSIX_PBA_BASE_IDX                                                         5
+
+
+// addressBlock: nbif0_rcc_strap_BIFDEC1
+// base address: 0xd0000000
+#define regRCC_STRAP3_RCC_BIF_STRAP0                                                                    0x2ffc0d20
+#define regRCC_STRAP3_RCC_BIF_STRAP0_BASE_IDX                                                           5
+#define regRCC_STRAP3_RCC_BIF_STRAP1                                                                    0x2ffc0d21
+#define regRCC_STRAP3_RCC_BIF_STRAP1_BASE_IDX                                                           5
+#define regRCC_STRAP3_RCC_BIF_STRAP2                                                                    0x2ffc0d26
+#define regRCC_STRAP3_RCC_BIF_STRAP2_BASE_IDX                                                           5
+#define regRCC_STRAP3_RCC_BIF_STRAP3                                                                    0x2ffc0d27
+#define regRCC_STRAP3_RCC_BIF_STRAP3_BASE_IDX                                                           5
+#define regRCC_STRAP3_RCC_BIF_STRAP4                                                                    0x2ffc0d28
+#define regRCC_STRAP3_RCC_BIF_STRAP4_BASE_IDX                                                           5
+#define regRCC_STRAP3_RCC_BIF_STRAP5                                                                    0x2ffc0d29
+#define regRCC_STRAP3_RCC_BIF_STRAP5_BASE_IDX                                                           5
+#define regRCC_STRAP3_RCC_BIF_STRAP6                                                                    0x2ffc0d2a
+#define regRCC_STRAP3_RCC_BIF_STRAP6_BASE_IDX                                                           5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP0                                                              0x2ffc0d2e
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP1                                                              0x2ffc0d2f
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP10                                                             0x2ffc0d30
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP11                                                             0x2ffc0d31
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP12                                                             0x2ffc0d32
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP13                                                             0x2ffc0d33
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP2                                                              0x2ffc0d39
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP3                                                              0x2ffc0d3a
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP4                                                              0x2ffc0d3b
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP5                                                              0x2ffc0d3c
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP6                                                              0x2ffc0d3d
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP7                                                              0x2ffc0d3e
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP8                                                              0x2ffc0d3f
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP9                                                              0x2ffc0d40
+#define regRCC_STRAP3_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP0                                                              0x2ffc0d41
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP1                                                              0x2ffc0d42
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP13                                                             0x2ffc0d43
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP14                                                             0x2ffc0d44
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP15                                                             0x2ffc0d45
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP16                                                             0x2ffc0d46
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP17                                                             0x2ffc0d47
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP18                                                             0x2ffc0d48
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP2                                                              0x2ffc0d4a
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP3                                                              0x2ffc0d4e
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP4                                                              0x2ffc0d4f
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP5                                                              0x2ffc0d50
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP8                                                              0x2ffc0d51
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP9                                                              0x2ffc0d52
+#define regRCC_STRAP3_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP0                                                              0x2ffc0d53
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP2                                                              0x2ffc0d5f
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP3                                                              0x2ffc0d64
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP4                                                              0x2ffc0d65
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP5                                                              0x2ffc0d66
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP6                                                              0x2ffc0d67
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP7                                                              0x2ffc0d68
+#define regRCC_STRAP3_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_bif_bx_pf_BIFPFVFDEC1
+// base address: 0xd0000000
+#define regBIF_BX_PF2_BIF_BME_STATUS                                                                    0x2ffc0e0b
+#define regBIF_BX_PF2_BIF_BME_STATUS_BASE_IDX                                                           5
+#define regBIF_BX_PF2_BIF_ATOMIC_ERR_LOG                                                                0x2ffc0e0c
+#define regBIF_BX_PF2_BIF_ATOMIC_ERR_LOG_BASE_IDX                                                       5
+#define regBIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x2ffc0e13
+#define regBIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                                     5
+#define regBIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x2ffc0e14
+#define regBIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                                      5
+#define regBIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x2ffc0e15
+#define regBIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                          5
+#define regBIF_BX_PF2_HDP_REG_COHERENCY_FLUSH_CNTL                                                      0x2ffc0e16
+#define regBIF_BX_PF2_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                             5
+#define regBIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x2ffc0e17
+#define regBIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                             5
+#define regBIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                                 0x2ffc0e19
+#define regBIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                                        5
+#define regBIF_BX_PF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                            0x2ffc0e1a
+#define regBIF_BX_PF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                                   5
+#define regBIF_BX_PF2_GPU_HDP_FLUSH_REQ                                                                 0x2ffc0e26
+#define regBIF_BX_PF2_GPU_HDP_FLUSH_REQ_BASE_IDX                                                        5
+#define regBIF_BX_PF2_GPU_HDP_FLUSH_DONE                                                                0x2ffc0e27
+#define regBIF_BX_PF2_GPU_HDP_FLUSH_DONE_BASE_IDX                                                       5
+#define regBIF_BX_PF2_BIF_TRANS_PENDING                                                                 0x2ffc0e28
+#define regBIF_BX_PF2_BIF_TRANS_PENDING_BASE_IDX                                                        5
+#define regBIF_BX_PF2_NBIF_GFX_ADDR_LUT_BYPASS                                                          0x2ffc0e32
+#define regBIF_BX_PF2_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFPFVFDEC1
+// base address: 0xd0000000
+#define regRCC_DEV0_EPF0_1_RCC_ERR_LOG                                                                  0x2ffc0da5
+#define regRCC_DEV0_EPF0_1_RCC_ERR_LOG_BASE_IDX                                                         5
+#define regRCC_DEV0_EPF0_1_RCC_DOORBELL_APER_EN                                                         0x2ffc0de0
+#define regRCC_DEV0_EPF0_1_RCC_DOORBELL_APER_EN_BASE_IDX                                                5
+#define regRCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE                                                           0x2ffc0de3
+#define regRCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE_BASE_IDX                                                  5
+#define regRCC_DEV0_EPF0_1_RCC_CONFIG_RESERVED                                                          0x2ffc0de4
+#define regRCC_DEV0_EPF0_1_RCC_CONFIG_RESERVED_BASE_IDX                                                 5
+#define regRCC_DEV0_EPF0_1_RCC_IOV_FUNC_IDENTIFIER                                                      0x2ffc0de5
+#define regRCC_DEV0_EPF0_1_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                             5
+
+
+// addressBlock: nbif0_gdc_GDCDEC
+// base address: 0xd0000000
+#define regGDC1_NGDC_PG_MISC_CTRL                                                                       0x2ffc0eb0
+#define regGDC1_NGDC_PG_MISC_CTRL_BASE_IDX                                                              5
+#define regGDC1_NGDC_PGMST_CTRL                                                                         0x2ffc0eb1
+#define regGDC1_NGDC_PGMST_CTRL_BASE_IDX                                                                5
+#define regGDC1_NGDC_PGSLV_CTRL                                                                         0x2ffc0eb2
+#define regGDC1_NGDC_PGSLV_CTRL_BASE_IDX                                                                5
+#define regGDC2_ATDMA_MISC_CNTL                                                                         0x2ffc0f13
+#define regGDC2_ATDMA_MISC_CNTL_BASE_IDX                                                                5
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_sh_mask.h
new file mode 100644
index 00000000000000..e5dcde9c72be3b
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_11_5_sh_mask.h
@@ -0,0 +1,63248 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _nbio_7_11_5_SH_MASK_HEADER
+#define _nbio_7_11_5_SH_MASK_HEADER
+
+
+// addressBlock: nbif0_bif_cfg_dev0_rc_bifcfgdecp
+//BIF_CFG_DEV0_RC_VENDOR_ID
+#define BIF_CFG_DEV0_RC_VENDOR_ID__VENDOR_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_VENDOR_ID__VENDOR_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_DEVICE_ID
+#define BIF_CFG_DEV0_RC_DEVICE_ID__DEVICE_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_DEVICE_ID__DEVICE_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_COMMAND
+#define BIF_CFG_DEV0_RC_COMMAND__IOEN_DN__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_COMMAND__MEMEN_DN__SHIFT                                                              0x1
+#define BIF_CFG_DEV0_RC_COMMAND__BUS_MASTER_EN__SHIFT                                                         0x2
+#define BIF_CFG_DEV0_RC_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                      0x3
+#define BIF_CFG_DEV0_RC_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_RC_COMMAND__PAL_SNOOP_EN__SHIFT                                                          0x5
+#define BIF_CFG_DEV0_RC_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                 0x6
+#define BIF_CFG_DEV0_RC_COMMAND__AD_STEPPING__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_RC_COMMAND__SERR_EN__SHIFT                                                               0x8
+#define BIF_CFG_DEV0_RC_COMMAND__FAST_B2B_EN__SHIFT                                                           0x9
+#define BIF_CFG_DEV0_RC_COMMAND__INT_DIS__SHIFT                                                               0xa
+#define BIF_CFG_DEV0_RC_COMMAND__IOEN_DN_MASK                                                                 0x0001L
+#define BIF_CFG_DEV0_RC_COMMAND__MEMEN_DN_MASK                                                                0x0002L
+#define BIF_CFG_DEV0_RC_COMMAND__BUS_MASTER_EN_MASK                                                           0x0004L
+#define BIF_CFG_DEV0_RC_COMMAND__SPECIAL_CYCLE_EN_MASK                                                        0x0008L
+#define BIF_CFG_DEV0_RC_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_RC_COMMAND__PAL_SNOOP_EN_MASK                                                            0x0020L
+#define BIF_CFG_DEV0_RC_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                   0x0040L
+#define BIF_CFG_DEV0_RC_COMMAND__AD_STEPPING_MASK                                                             0x0080L
+#define BIF_CFG_DEV0_RC_COMMAND__SERR_EN_MASK                                                                 0x0100L
+#define BIF_CFG_DEV0_RC_COMMAND__FAST_B2B_EN_MASK                                                             0x0200L
+#define BIF_CFG_DEV0_RC_COMMAND__INT_DIS_MASK                                                                 0x0400L
+//BIF_CFG_DEV0_RC_STATUS
+#define BIF_CFG_DEV0_RC_STATUS__IMMEDIATE_READINESS__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_STATUS__INT_STATUS__SHIFT                                                             0x3
+#define BIF_CFG_DEV0_RC_STATUS__CAP_LIST__SHIFT                                                               0x4
+#define BIF_CFG_DEV0_RC_STATUS__PCI_66_CAP__SHIFT                                                             0x5
+#define BIF_CFG_DEV0_RC_STATUS__FAST_BACK_CAPABLE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_RC_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                               0x8
+#define BIF_CFG_DEV0_RC_STATUS__DEVSEL_TIMING__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_RC_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_RC_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                  0xe
+#define BIF_CFG_DEV0_RC_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_RC_STATUS__IMMEDIATE_READINESS_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_RC_STATUS__INT_STATUS_MASK                                                               0x0008L
+#define BIF_CFG_DEV0_RC_STATUS__CAP_LIST_MASK                                                                 0x0010L
+#define BIF_CFG_DEV0_RC_STATUS__PCI_66_CAP_MASK                                                               0x0020L
+#define BIF_CFG_DEV0_RC_STATUS__FAST_BACK_CAPABLE_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_RC_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_RC_STATUS__DEVSEL_TIMING_MASK                                                            0x0600L
+#define BIF_CFG_DEV0_RC_STATUS__SIGNAL_TARGET_ABORT_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_TARGET_ABORT_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_MASTER_ABORT_MASK                                                    0x2000L
+#define BIF_CFG_DEV0_RC_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                    0x4000L
+#define BIF_CFG_DEV0_RC_STATUS__PARITY_ERROR_DETECTED_MASK                                                    0x8000L
+//BIF_CFG_DEV0_RC_REVISION_ID
+#define BIF_CFG_DEV0_RC_REVISION_ID__MINOR_REV_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC_REVISION_ID__MAJOR_REV_ID__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_RC_REVISION_ID__MINOR_REV_ID_MASK                                                        0x0FL
+#define BIF_CFG_DEV0_RC_REVISION_ID__MAJOR_REV_ID_MASK                                                        0xF0L
+//BIF_CFG_DEV0_RC_PROG_INTERFACE
+#define BIF_CFG_DEV0_RC_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PROG_INTERFACE__PROG_INTERFACE_MASK                                                   0xFFL
+//BIF_CFG_DEV0_RC_SUB_CLASS
+#define BIF_CFG_DEV0_RC_SUB_CLASS__SUB_CLASS__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_SUB_CLASS__SUB_CLASS_MASK                                                             0xFFL
+//BIF_CFG_DEV0_RC_BASE_CLASS
+#define BIF_CFG_DEV0_RC_BASE_CLASS__BASE_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_CLASS__BASE_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_RC_CACHE_LINE
+#define BIF_CFG_DEV0_RC_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                      0xFFL
+//BIF_CFG_DEV0_RC_LATENCY
+#define BIF_CFG_DEV0_RC_LATENCY__LATENCY_TIMER__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_LATENCY__LATENCY_TIMER_MASK                                                           0xFFL
+//BIF_CFG_DEV0_RC_HEADER
+#define BIF_CFG_DEV0_RC_HEADER__HEADER_TYPE__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_RC_HEADER__DEVICE_TYPE__SHIFT                                                            0x7
+#define BIF_CFG_DEV0_RC_HEADER__HEADER_TYPE_MASK                                                              0x7FL
+#define BIF_CFG_DEV0_RC_HEADER__DEVICE_TYPE_MASK                                                              0x80L
+//BIF_CFG_DEV0_RC_BIST
+#define BIF_CFG_DEV0_RC_BIST__BIST_COMP__SHIFT                                                                0x0
+#define BIF_CFG_DEV0_RC_BIST__BIST_STRT__SHIFT                                                                0x6
+#define BIF_CFG_DEV0_RC_BIST__BIST_CAP__SHIFT                                                                 0x7
+#define BIF_CFG_DEV0_RC_BIST__BIST_COMP_MASK                                                                  0x0FL
+#define BIF_CFG_DEV0_RC_BIST__BIST_STRT_MASK                                                                  0x40L
+#define BIF_CFG_DEV0_RC_BIST__BIST_CAP_MASK                                                                   0x80L
+//BIF_CFG_DEV0_RC_BASE_ADDR_1
+#define BIF_CFG_DEV0_RC_BASE_ADDR_1__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_ADDR_1__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_BASE_ADDR_2
+#define BIF_CFG_DEV0_RC_BASE_ADDR_2__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_ADDR_2__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                                0x18
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                            0x0000FF00L
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                              0x00FF0000L
+#define BIF_CFG_DEV0_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                  0xFF000000L
+//BIF_CFG_DEV0_RC_IO_BASE_LIMIT
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                        0xc
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                      0x000FL
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_BASE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                     0x0F00L
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT__IO_LIMIT_MASK                                                          0xF000L
+//BIF_CFG_DEV0_RC_SECONDARY_STATUS
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                            0x7
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                                0x9
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                          0xb
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                        0xc
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                        0xd
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                        0xf
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PCI_66_CAP_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                              0x0080L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                       0x0100L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                  0x0600L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                            0x0800L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                          0x1000L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                          0x2000L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                          0x4000L
+#define BIF_CFG_DEV0_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                          0x8000L
+//BIF_CFG_DEV0_RC_MEM_BASE_LIMIT
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                    0x0000000FL
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                   0x0000FFF0L
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_PREF_BASE_LIMIT
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                           0x4
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                              0x0000000FL
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                             0x0000FFF0L
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PREF_BASE_UPPER
+#define BIF_CFG_DEV0_RC_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV0_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                               0x10
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                 0xFFFF0000L
+//BIF_CFG_DEV0_RC_CAP_PTR
+#define BIF_CFG_DEV0_RC_CAP_PTR__CAP_PTR__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_CAP_PTR__CAP_PTR_MASK                                                                 0xFFL
+//BIF_CFG_DEV0_RC_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                           0x1
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                        0x00000001L
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                             0x0000000EL
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                            0x000000F0L
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__BASE_ADDR_MASK                                                         0xFFFFF800L
+//BIF_CFG_DEV0_RC_INTERRUPT_LINE
+#define BIF_CFG_DEV0_RC_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                   0xFFL
+//BIF_CFG_DEV0_RC_INTERRUPT_PIN
+#define BIF_CFG_DEV0_RC_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                     0xFFL
+//BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                        0x3
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                             0x5
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                           0x6
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                       0x9
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                          0xa
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                     0xb
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                          0x0008L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                               0x0020L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                             0x0040L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                           0x0100L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                         0x0200L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                            0x0400L
+#define BIF_CFG_DEV0_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                       0x0800L
+//BIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                   0x01L
+//BIF_CFG_DEV0_RC_PMI_CAP_LIST
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV0_RC_PMI_CAP
+#define BIF_CFG_DEV0_RC_PMI_CAP__VERSION__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_CLOCK__SHIFT                                                             0x3
+#define BIF_CFG_DEV0_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                   0x4
+#define BIF_CFG_DEV0_RC_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_RC_PMI_CAP__AUX_CURRENT__SHIFT                                                           0x6
+#define BIF_CFG_DEV0_RC_PMI_CAP__D1_SUPPORT__SHIFT                                                            0x9
+#define BIF_CFG_DEV0_RC_PMI_CAP__D2_SUPPORT__SHIFT                                                            0xa
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_SUPPORT__SHIFT                                                           0xb
+#define BIF_CFG_DEV0_RC_PMI_CAP__VERSION_MASK                                                                 0x0007L
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_CLOCK_MASK                                                               0x0008L
+#define BIF_CFG_DEV0_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                     0x0010L
+#define BIF_CFG_DEV0_RC_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                       0x0020L
+#define BIF_CFG_DEV0_RC_PMI_CAP__AUX_CURRENT_MASK                                                             0x01C0L
+#define BIF_CFG_DEV0_RC_PMI_CAP__D1_SUPPORT_MASK                                                              0x0200L
+#define BIF_CFG_DEV0_RC_PMI_CAP__D2_SUPPORT_MASK                                                              0x0400L
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_SUPPORT_MASK                                                             0xF800L
+//BIF_CFG_DEV0_RC_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                   0x9
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                 0x16
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                    0x17
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__POWER_STATE_MASK                                                     0x00000003L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_EN_MASK                                                          0x00000100L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                     0x00001E00L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                      0x00006000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_STATUS_MASK                                                      0x00008000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                   0x00400000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                      0x00800000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PMI_DATA_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_CAP__DEVICE_TYPE__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_RC_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_RC_PCIE_CAP__VERSION_MASK                                                                0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_CAP__DEVICE_TYPE_MASK                                                            0x00F0L
+#define BIF_CFG_DEV0_RC_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                       0x0100L
+#define BIF_CFG_DEV0_RC_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                        0x3E00L
+//BIF_CFG_DEV0_RC_DEVICE_CAP
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                              0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                           0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                          0x12
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                          0x1a
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                        0x1c
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                  0x00000007L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__PHANTOM_FUNC_MASK                                                         0x00000018L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__EXTENDED_TAG_MASK                                                         0x00000020L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                               0x000001C0L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                                0x00000E00L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                             0x00008000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                             0x00010000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                            0x03FC0000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                            0x0C000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__FLR_CAPABLE_MASK                                                          0x10000000L
+//BIF_CFG_DEV0_RC_DEVICE_CNTL
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                      0x2
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                   0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                   0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                               0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__CORR_ERR_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                    0x0002L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                        0x0004L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__USR_REPORT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                      0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                    0x00E0L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                     0x0200L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                     0x0400L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                         0x0800L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                               0x7000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                 0x8000L
+//BIF_CFG_DEV0_RC_DEVICE_STATUS
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__CORR_ERR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__FATAL_ERR__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__USR_DETECTED__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__AUX_PWR__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                   0x6
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__CORR_ERR_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__FATAL_ERR_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__USR_DETECTED_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__AUX_PWR_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                     0x0040L
+//BIF_CFG_DEV0_RC_LINK_CAP
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_SPEED__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_WIDTH__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_RC_LINK_CAP__PM_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_RC_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                      0xf
+#define BIF_CFG_DEV0_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                               0x12
+#define BIF_CFG_DEV0_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                          0x13
+#define BIF_CFG_DEV0_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                             0x15
+#define BIF_CFG_DEV0_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                          0x16
+#define BIF_CFG_DEV0_RC_LINK_CAP__PORT_NUMBER__SHIFT                                                          0x18
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_SPEED_MASK                                                             0x0000000FL
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_WIDTH_MASK                                                             0x000003F0L
+#define BIF_CFG_DEV0_RC_LINK_CAP__PM_SUPPORT_MASK                                                             0x00000C00L
+#define BIF_CFG_DEV0_RC_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                       0x00007000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__L1_EXIT_LATENCY_MASK                                                        0x00038000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                 0x00040000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                            0x00100000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                               0x00200000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                            0x00400000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__PORT_NUMBER_MASK                                                            0xFF000000L
+//BIF_CFG_DEV0_RC_LINK_CNTL
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PM_CONTROL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                        0x2
+#define BIF_CFG_DEV0_RC_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_DIS__SHIFT                                                            0x4
+#define BIF_CFG_DEV0_RC_LINK_CNTL__RETRAIN_LINK__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_RC_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                    0x6
+#define BIF_CFG_DEV0_RC_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                         0x9
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                           0xa
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                           0xb
+#define BIF_CFG_DEV0_RC_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                               0xe
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PM_CONTROL_MASK                                                            0x0003L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                          0x0004L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_DIS_MASK                                                              0x0010L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__RETRAIN_LINK_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                      0x0040L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__EXTENDED_SYNC_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                             0x0100L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                           0x0200L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                             0x0400L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                             0x0800L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                 0xC000L
+//BIF_CFG_DEV0_RC_LINK_STATUS
+#define BIF_CFG_DEV0_RC_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_TRAINING__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_RC_LINK_STATUS__DL_ACTIVE__SHIFT                                                         0xd
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                         0xf
+#define BIF_CFG_DEV0_RC_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                               0x03F0L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_TRAINING_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                      0x1000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__DL_ACTIVE_MASK                                                           0x2000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                           0x4000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                           0x8000L
+//BIF_CFG_DEV0_RC_SLOT_CAP
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                               0x3
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                      0x6
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                       0x12
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                    0x13
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                 0x00000002L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                     0x00000004L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                  0x00000010L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                        0x00000040L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                   0x00007F80L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                   0x00018000L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_RC_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                      0xFFF80000L
+//BIF_CFG_DEV0_RC_SLOT_CNTL
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                 0x6
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                          0xb
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                         0xd
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                                0x0001L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                       0x0020L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                   0x00C0L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                    0x0300L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                            0x0800L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                           0x2000L
+#define BIF_CFG_DEV0_RC_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                     0x4000L
+//BIF_CFG_DEV0_RC_SLOT_STATUS
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                                0x1
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                                0x2
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                           0x3
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                 0x0001L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                             0x0008L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                        0x0080L
+#define BIF_CFG_DEV0_RC_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                    0x0100L
+//BIF_CFG_DEV0_RC_ROOT_CNTL
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                             0x1
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                               0x0002L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                            0x0010L
+//BIF_CFG_DEV0_RC_ROOT_CAP
+#define BIF_CFG_DEV0_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                                0x0001L
+//BIF_CFG_DEV0_RC_ROOT_STATUS
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_STATUS__SHIFT                                                        0x10
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_PENDING__SHIFT                                                       0x11
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_STATUS_MASK                                                          0x00010000L
+#define BIF_CFG_DEV0_RC_ROOT_STATUS__PME_PENDING_MASK                                                         0x00020000L
+//BIF_CFG_DEV0_RC_DEVICE_CAP2
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                        0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                         0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                                0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                     0xe
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                   0x11
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                    0x12
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                      0x14
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                      0x15
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                          0x16
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                    0x18
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                     0x1a
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                         0x0000000FL
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                          0x00000040L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                          0x00000100L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                              0x00000200L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                           0x00000400L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                       0x00000800L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                       0x0000C000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                     0x00020000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                      0x000C0000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                        0x00200000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                            0x00C00000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                      0x03000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                       0x04000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_RC_DEVICE_CNTL2
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                                0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                               0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__LTR_EN__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                     0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                     0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__OBFF_EN__SHIFT                                                          0xd
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                  0x0020L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                                0x0040L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                           0x0080L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                              0x0200L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__LTR_EN_MASK                                                             0x0400L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                       0x0800L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                       0x1000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__OBFF_EN_MASK                                                            0x6000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                        0x8000L
+//BIF_CFG_DEV0_RC_DEVICE_STATUS2
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS2__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS2__RESERVED_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_RC_LINK_CAP2
+#define BIF_CFG_DEV0_RC_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                                0x1
+#define BIF_CFG_DEV0_RC_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                           0x17
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                           0x18
+#define BIF_CFG_DEV0_RC_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_RC_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                  0x000000FEL
+#define BIF_CFG_DEV0_RC_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                   0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                              0x0000FE00L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                              0x007F0000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                             0x00800000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                             0x01000000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__DRS_SUPPORTED_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_RC_LINK_CNTL2
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                        0x5
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                               0xa
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                              0xc
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                    0x000FL
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                          0x0020L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                                0x0040L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__XMIT_MARGIN_MASK                                                          0x0380L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                                0xF000L
+//BIF_CFG_DEV0_RC_LINK_STATUS2
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                  0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                  0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                  0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                    0x5
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                                0x7
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                             0x8
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                             0xf
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                               0x0001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                          0x0002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                    0x0004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                    0x0008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                    0x0010L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                      0x0020L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                               0x0300L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                      0x7000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                               0x8000L
+//BIF_CFG_DEV0_RC_SLOT_CAP2
+#define BIF_CFG_DEV0_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                           0x00000001L
+//BIF_CFG_DEV0_RC_SLOT_CNTL2
+#define BIF_CFG_DEV0_RC_SLOT_CNTL2__RESERVED__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_SLOT_CNTL2__RESERVED_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_SLOT_STATUS2
+#define BIF_CFG_DEV0_RC_SLOT_STATUS2__RESERVED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_SLOT_STATUS2__RESERVED_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_CAP_LIST
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV0_RC_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EN__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                    0x1
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                             0x9
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                              0xa
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EN_MASK                                                             0x0001L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                      0x000EL
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                       0x0070L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_64BIT_MASK                                                          0x0080L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                          0x0100L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                               0x0200L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                                0x0400L
+//BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_MSI_MSG_DATA
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA__MSI_DATA__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA__MSI_DATA_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_RC_SSID_CAP_LIST
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC_SSID_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC_SSID_CAP
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                         0x10
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_SSID_CAP__SUBSYSTEM_ID_MASK                                                           0xFFFF0000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                             0x10
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                  0x1
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                       0x9
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                    0x00000002L
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                         0x0000FE00L
+//BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                   0x0000FFFFL
+//BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV0_RC_LINK_CAP_16GT
+#define BIF_CFG_DEV0_RC_LINK_CAP_16GT__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP_16GT__RESERVED_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LINK_CNTL_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                             0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                               0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                               0x00000004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                 0x00000010L
+//BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV0_RC_LINK_CAP_32GT
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                               0x8
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                               0xa
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                                0xb
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                               0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                            0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                 0x00000200L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                 0x00000400L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                  0x0000F800L
+//BIF_CFG_DEV0_RC_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                 0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                       0x00000700L
+//BIF_CFG_DEV0_RC_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                             0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                         0x6
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                  0x9
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                        0xa
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                               0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                               0x00000004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                          0x00000400L
+//BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                             0x3
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                           0x00000007L
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                               0x0000FFF8L
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                             0xFFFF0000L
+//BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT              0x18
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                               0x00FFFFFFL
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK                0x03000000L
+//BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                        0x10
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                        0x00000007L
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                            0x0000FFF8L
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                          0xFFFF0000L
+//BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT           0x18
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                            0x00FFFFFFL
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK             0x03000000L
+//BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_AP_CAP
+#define BIF_CFG_DEV0_RC_AP_CAP__COUNT__SHIFT                                                                  0x0
+#define BIF_CFG_DEV0_RC_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC_AP_CAP__COUNT_MASK                                                                    0x000000FFL
+#define BIF_CFG_DEV0_RC_AP_CAP__SEL_EN_SUPPORTED_MASK                                                         0x00000100L
+//BIF_CFG_DEV0_RC_AP_CNTL
+#define BIF_CFG_DEV0_RC_AP_CNTL__INX_SEL__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_AP_CNTL__INX_SEL_MASK                                                                 0x000000FFL
+#define BIF_CFG_DEV0_RC_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                          0x00000100L
+//BIF_CFG_DEV0_RC_AP_DATA1
+#define BIF_CFG_DEV0_RC_AP_DATA1__USAGE_INFOR__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_AP_DATA1__DETAILS__SHIFT                                                              0x5
+#define BIF_CFG_DEV0_RC_AP_DATA1__VENDORID__SHIFT                                                             0x10
+#define BIF_CFG_DEV0_RC_AP_DATA1__USAGE_INFOR_MASK                                                            0x00000007L
+#define BIF_CFG_DEV0_RC_AP_DATA1__DETAILS_MASK                                                                0x0000FFE0L
+#define BIF_CFG_DEV0_RC_AP_DATA1__VENDORID_MASK                                                               0xFFFF0000L
+//BIF_CFG_DEV0_RC_AP_DATA2
+#define BIF_CFG_DEV0_RC_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                     0x00FFFFFFL
+//BIF_CFG_DEV0_RC_AP_SEL_EN_MASK
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__PCIE__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__OTHERS__SHIFT                                                         0x1
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__PCIE_MASK                                                             0x00000001L
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__OTHERS_MASK                                                           0xFFFFFFFEL
+//BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                    0x0
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                      0x0001L
+//BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                                0x1
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                           0x0001L
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                  0x0002L
+//BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                               0xa
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                 0x00000070L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                 0x00000C00L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                     0x18
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                                0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                 0x000EL
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                         0x0001L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x003F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                     0x1f
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                       0x007FFFFFL
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                       0x80000000L
+//BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                      0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                         0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                          0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                       0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                         0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                        0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                  0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                   0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                  0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                  0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                         0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                          0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                     0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                           0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                            0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                       0x04000000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                              0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                       0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                            0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                             0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                            0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                      0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                       0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                      0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                      0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                             0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                              0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                         0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                               0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                               0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                                0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                             0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                              0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                              0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                        0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                               0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                                0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                           0x04000000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                     0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                  0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                     0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                      0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                 0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                               0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                    0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                     0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                    0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                              0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                               0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                              0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                              0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                     0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                      0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                                0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                                0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                                0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                       0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                        0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                   0x04000000L
+//BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                           0x6
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                               0x8
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                              0xc
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                             0xd
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                      0xe
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                                0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                               0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                        0x00008000L
+//BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                               0x6
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                              0x7
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                   0x8
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                  0xc
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                 0xd
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                          0xe
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                          0xf
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                 0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                 0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                                0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                    0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                   0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                            0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                            0x00008000L
+//BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                            0x5
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                  0xb
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                          0xc
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                             0x0000001FL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                              0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                    0x00000800L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                            0x00001000L
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG0__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG1__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG2__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG3__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                            0x2
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                              0x00000004L
+//BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                  0x2
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                                0x4
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                  0x5
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                     0x6
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                      0x1b
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                    0x00000004L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                  0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                           0x00000180L
+#define BIF_CFG_DEV0_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                        0xF8000000L
+//BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                             0x1
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                             0x2
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                              0x4
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                              0x7
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                               0x0002L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                               0x0004L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                                0x0010L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                              0x0040L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                                0x0080L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                         0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                         0x2
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                      0x3
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                           0x5
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0x8
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0xa
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                 0xc
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                           0x0002L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                           0x0004L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                        0x0008L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                             0x0020L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                            0x0080L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0300L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0C00L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                   0x1000L
+//BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_RTR_DATA1
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RESET_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_RTR_DATA1__DLUP_TIME__SHIFT                                                           0xc
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RTR_VALID__SHIFT                                                           0x1f
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RESET_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_RC_RTR_DATA1__DLUP_TIME_MASK                                                             0x00FFF000L
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RTR_VALID_MASK                                                             0x80000000L
+//BIF_CFG_DEV0_RC_RTR_DATA2
+#define BIF_CFG_DEV0_RC_RTR_DATA2__FLR_TIME__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_RC_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                        0xc
+#define BIF_CFG_DEV0_RC_RTR_DATA2__FLR_TIME_MASK                                                              0x00000FFFL
+#define BIF_CFG_DEV0_RC_RTR_DATA2__D3HOTD0_TIME_MASK                                                          0x00FFF000L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_rc_bifcfgdecp
+//BIF_CFG_DEV1_RC_VENDOR_ID
+#define BIF_CFG_DEV1_RC_VENDOR_ID__VENDOR_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_VENDOR_ID__VENDOR_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV1_RC_DEVICE_ID
+#define BIF_CFG_DEV1_RC_DEVICE_ID__DEVICE_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_DEVICE_ID__DEVICE_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV1_RC_COMMAND
+#define BIF_CFG_DEV1_RC_COMMAND__IOEN_DN__SHIFT                                                               0x0
+#define BIF_CFG_DEV1_RC_COMMAND__MEMEN_DN__SHIFT                                                              0x1
+#define BIF_CFG_DEV1_RC_COMMAND__BUS_MASTER_EN__SHIFT                                                         0x2
+#define BIF_CFG_DEV1_RC_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                      0x3
+#define BIF_CFG_DEV1_RC_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV1_RC_COMMAND__PAL_SNOOP_EN__SHIFT                                                          0x5
+#define BIF_CFG_DEV1_RC_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                 0x6
+#define BIF_CFG_DEV1_RC_COMMAND__AD_STEPPING__SHIFT                                                           0x7
+#define BIF_CFG_DEV1_RC_COMMAND__SERR_EN__SHIFT                                                               0x8
+#define BIF_CFG_DEV1_RC_COMMAND__FAST_B2B_EN__SHIFT                                                           0x9
+#define BIF_CFG_DEV1_RC_COMMAND__INT_DIS__SHIFT                                                               0xa
+#define BIF_CFG_DEV1_RC_COMMAND__IOEN_DN_MASK                                                                 0x0001L
+#define BIF_CFG_DEV1_RC_COMMAND__MEMEN_DN_MASK                                                                0x0002L
+#define BIF_CFG_DEV1_RC_COMMAND__BUS_MASTER_EN_MASK                                                           0x0004L
+#define BIF_CFG_DEV1_RC_COMMAND__SPECIAL_CYCLE_EN_MASK                                                        0x0008L
+#define BIF_CFG_DEV1_RC_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                 0x0010L
+#define BIF_CFG_DEV1_RC_COMMAND__PAL_SNOOP_EN_MASK                                                            0x0020L
+#define BIF_CFG_DEV1_RC_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                   0x0040L
+#define BIF_CFG_DEV1_RC_COMMAND__AD_STEPPING_MASK                                                             0x0080L
+#define BIF_CFG_DEV1_RC_COMMAND__SERR_EN_MASK                                                                 0x0100L
+#define BIF_CFG_DEV1_RC_COMMAND__FAST_B2B_EN_MASK                                                             0x0200L
+#define BIF_CFG_DEV1_RC_COMMAND__INT_DIS_MASK                                                                 0x0400L
+//BIF_CFG_DEV1_RC_STATUS
+#define BIF_CFG_DEV1_RC_STATUS__IMMEDIATE_READINESS__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_RC_STATUS__INT_STATUS__SHIFT                                                             0x3
+#define BIF_CFG_DEV1_RC_STATUS__CAP_LIST__SHIFT                                                               0x4
+#define BIF_CFG_DEV1_RC_STATUS__PCI_66_CAP__SHIFT                                                             0x5
+#define BIF_CFG_DEV1_RC_STATUS__FAST_BACK_CAPABLE__SHIFT                                                      0x7
+#define BIF_CFG_DEV1_RC_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                               0x8
+#define BIF_CFG_DEV1_RC_STATUS__DEVSEL_TIMING__SHIFT                                                          0x9
+#define BIF_CFG_DEV1_RC_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                    0xb
+#define BIF_CFG_DEV1_RC_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                  0xc
+#define BIF_CFG_DEV1_RC_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                  0xd
+#define BIF_CFG_DEV1_RC_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                  0xe
+#define BIF_CFG_DEV1_RC_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                  0xf
+#define BIF_CFG_DEV1_RC_STATUS__IMMEDIATE_READINESS_MASK                                                      0x0001L
+#define BIF_CFG_DEV1_RC_STATUS__INT_STATUS_MASK                                                               0x0008L
+#define BIF_CFG_DEV1_RC_STATUS__CAP_LIST_MASK                                                                 0x0010L
+#define BIF_CFG_DEV1_RC_STATUS__PCI_66_CAP_MASK                                                               0x0020L
+#define BIF_CFG_DEV1_RC_STATUS__FAST_BACK_CAPABLE_MASK                                                        0x0080L
+#define BIF_CFG_DEV1_RC_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                 0x0100L
+#define BIF_CFG_DEV1_RC_STATUS__DEVSEL_TIMING_MASK                                                            0x0600L
+#define BIF_CFG_DEV1_RC_STATUS__SIGNAL_TARGET_ABORT_MASK                                                      0x0800L
+#define BIF_CFG_DEV1_RC_STATUS__RECEIVED_TARGET_ABORT_MASK                                                    0x1000L
+#define BIF_CFG_DEV1_RC_STATUS__RECEIVED_MASTER_ABORT_MASK                                                    0x2000L
+#define BIF_CFG_DEV1_RC_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                    0x4000L
+#define BIF_CFG_DEV1_RC_STATUS__PARITY_ERROR_DETECTED_MASK                                                    0x8000L
+//BIF_CFG_DEV1_RC_REVISION_ID
+#define BIF_CFG_DEV1_RC_REVISION_ID__MINOR_REV_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_RC_REVISION_ID__MAJOR_REV_ID__SHIFT                                                      0x4
+#define BIF_CFG_DEV1_RC_REVISION_ID__MINOR_REV_ID_MASK                                                        0x0FL
+#define BIF_CFG_DEV1_RC_REVISION_ID__MAJOR_REV_ID_MASK                                                        0xF0L
+//BIF_CFG_DEV1_RC_PROG_INTERFACE
+#define BIF_CFG_DEV1_RC_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC_PROG_INTERFACE__PROG_INTERFACE_MASK                                                   0xFFL
+//BIF_CFG_DEV1_RC_SUB_CLASS
+#define BIF_CFG_DEV1_RC_SUB_CLASS__SUB_CLASS__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_SUB_CLASS__SUB_CLASS_MASK                                                             0xFFL
+//BIF_CFG_DEV1_RC_BASE_CLASS
+#define BIF_CFG_DEV1_RC_BASE_CLASS__BASE_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_BASE_CLASS__BASE_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV1_RC_CACHE_LINE
+#define BIF_CFG_DEV1_RC_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_RC_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                      0xFFL
+//BIF_CFG_DEV1_RC_LATENCY
+#define BIF_CFG_DEV1_RC_LATENCY__LATENCY_TIMER__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_LATENCY__LATENCY_TIMER_MASK                                                           0xFFL
+//BIF_CFG_DEV1_RC_HEADER
+#define BIF_CFG_DEV1_RC_HEADER__HEADER_TYPE__SHIFT                                                            0x0
+#define BIF_CFG_DEV1_RC_HEADER__DEVICE_TYPE__SHIFT                                                            0x7
+#define BIF_CFG_DEV1_RC_HEADER__HEADER_TYPE_MASK                                                              0x7FL
+#define BIF_CFG_DEV1_RC_HEADER__DEVICE_TYPE_MASK                                                              0x80L
+//BIF_CFG_DEV1_RC_BIST
+#define BIF_CFG_DEV1_RC_BIST__BIST_COMP__SHIFT                                                                0x0
+#define BIF_CFG_DEV1_RC_BIST__BIST_STRT__SHIFT                                                                0x6
+#define BIF_CFG_DEV1_RC_BIST__BIST_CAP__SHIFT                                                                 0x7
+#define BIF_CFG_DEV1_RC_BIST__BIST_COMP_MASK                                                                  0x0FL
+#define BIF_CFG_DEV1_RC_BIST__BIST_STRT_MASK                                                                  0x40L
+#define BIF_CFG_DEV1_RC_BIST__BIST_CAP_MASK                                                                   0x80L
+//BIF_CFG_DEV1_RC_BASE_ADDR_1
+#define BIF_CFG_DEV1_RC_BASE_ADDR_1__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_BASE_ADDR_1__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_BASE_ADDR_2
+#define BIF_CFG_DEV1_RC_BASE_ADDR_2__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_BASE_ADDR_2__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                          0x8
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                            0x10
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                                0x18
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                              0x000000FFL
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                            0x0000FF00L
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                              0x00FF0000L
+#define BIF_CFG_DEV1_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                  0xFF000000L
+//BIF_CFG_DEV1_RC_IO_BASE_LIMIT
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_BASE__SHIFT                                                         0x4
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                        0xc
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                      0x000FL
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_BASE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                     0x0F00L
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT__IO_LIMIT_MASK                                                          0xF000L
+//BIF_CFG_DEV1_RC_SECONDARY_STATUS
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                   0x5
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                            0x7
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                     0x8
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                                0x9
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                          0xb
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                        0xc
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                        0xd
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                        0xe
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                        0xf
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__PCI_66_CAP_MASK                                                     0x0020L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                              0x0080L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                       0x0100L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                  0x0600L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                            0x0800L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                          0x1000L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                          0x2000L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                          0x4000L
+#define BIF_CFG_DEV1_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                          0x8000L
+//BIF_CFG_DEV1_RC_MEM_BASE_LIMIT
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                 0x4
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                 0x10
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                                0x14
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                    0x0000000FL
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                   0x0000FFF0L
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV1_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV1_RC_PREF_BASE_LIMIT
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                           0x4
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                          0x14
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                              0x0000000FL
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                             0x0000FFF0L
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_RC_PREF_BASE_UPPER
+#define BIF_CFG_DEV1_RC_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV1_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                               0x10
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                 0xFFFF0000L
+//BIF_CFG_DEV1_RC_CAP_PTR
+#define BIF_CFG_DEV1_RC_CAP_PTR__CAP_PTR__SHIFT                                                               0x0
+#define BIF_CFG_DEV1_RC_CAP_PTR__CAP_PTR_MASK                                                                 0xFFL
+//BIF_CFG_DEV1_RC_ROM_BASE_ADDR
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                           0x1
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                          0x4
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                       0xb
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                        0x00000001L
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                             0x0000000EL
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                            0x000000F0L
+#define BIF_CFG_DEV1_RC_ROM_BASE_ADDR__BASE_ADDR_MASK                                                         0xFFFFF800L
+//BIF_CFG_DEV1_RC_INTERRUPT_LINE
+#define BIF_CFG_DEV1_RC_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                   0xFFL
+//BIF_CFG_DEV1_RC_INTERRUPT_PIN
+#define BIF_CFG_DEV1_RC_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                     0xFFL
+//BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                        0x3
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                       0x4
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                             0x5
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                           0x6
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                   0x7
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                         0x8
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                       0x9
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                          0xa
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                     0xb
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                          0x0008L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                         0x0010L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                               0x0020L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                             0x0040L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                     0x0080L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                           0x0100L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                         0x0200L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                            0x0400L
+#define BIF_CFG_DEV1_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                       0x0800L
+//BIF_CFG_DEV1_RC_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV1_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                   0x01L
+//BIF_CFG_DEV1_RC_PMI_CAP_LIST
+#define BIF_CFG_DEV1_RC_PMI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV1_RC_PMI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV1_RC_PMI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV1_RC_PMI_CAP
+#define BIF_CFG_DEV1_RC_PMI_CAP__VERSION__SHIFT                                                               0x0
+#define BIF_CFG_DEV1_RC_PMI_CAP__PME_CLOCK__SHIFT                                                             0x3
+#define BIF_CFG_DEV1_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                   0x4
+#define BIF_CFG_DEV1_RC_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                     0x5
+#define BIF_CFG_DEV1_RC_PMI_CAP__AUX_CURRENT__SHIFT                                                           0x6
+#define BIF_CFG_DEV1_RC_PMI_CAP__D1_SUPPORT__SHIFT                                                            0x9
+#define BIF_CFG_DEV1_RC_PMI_CAP__D2_SUPPORT__SHIFT                                                            0xa
+#define BIF_CFG_DEV1_RC_PMI_CAP__PME_SUPPORT__SHIFT                                                           0xb
+#define BIF_CFG_DEV1_RC_PMI_CAP__VERSION_MASK                                                                 0x0007L
+#define BIF_CFG_DEV1_RC_PMI_CAP__PME_CLOCK_MASK                                                               0x0008L
+#define BIF_CFG_DEV1_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                     0x0010L
+#define BIF_CFG_DEV1_RC_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                       0x0020L
+#define BIF_CFG_DEV1_RC_PMI_CAP__AUX_CURRENT_MASK                                                             0x01C0L
+#define BIF_CFG_DEV1_RC_PMI_CAP__D1_SUPPORT_MASK                                                              0x0200L
+#define BIF_CFG_DEV1_RC_PMI_CAP__D2_SUPPORT_MASK                                                              0x0400L
+#define BIF_CFG_DEV1_RC_PMI_CAP__PME_SUPPORT_MASK                                                             0xF800L
+//BIF_CFG_DEV1_RC_PMI_STATUS_CNTL
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                 0x3
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__PME_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                   0x9
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                    0xd
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                    0xf
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                 0x16
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                    0x17
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                      0x18
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__POWER_STATE_MASK                                                     0x00000003L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                   0x00000008L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__PME_EN_MASK                                                          0x00000100L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                     0x00001E00L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                      0x00006000L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__PME_STATUS_MASK                                                      0x00008000L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                   0x00400000L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                      0x00800000L
+#define BIF_CFG_DEV1_RC_PMI_STATUS_CNTL__PMI_DATA_MASK                                                        0xFF000000L
+//BIF_CFG_DEV1_RC_PCIE_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV1_RC_PCIE_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV1_RC_PCIE_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV1_RC_PCIE_CAP
+#define BIF_CFG_DEV1_RC_PCIE_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV1_RC_PCIE_CAP__DEVICE_TYPE__SHIFT                                                          0x4
+#define BIF_CFG_DEV1_RC_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                      0x9
+#define BIF_CFG_DEV1_RC_PCIE_CAP__VERSION_MASK                                                                0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_CAP__DEVICE_TYPE_MASK                                                            0x00F0L
+#define BIF_CFG_DEV1_RC_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                       0x0100L
+#define BIF_CFG_DEV1_RC_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                        0x3E00L
+//BIF_CFG_DEV1_RC_DEVICE_CAP
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                       0x3
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                       0x5
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                             0x6
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                              0x9
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                           0xf
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                          0x12
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                          0x1a
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                        0x1c
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                  0x00000007L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__PHANTOM_FUNC_MASK                                                         0x00000018L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__EXTENDED_TAG_MASK                                                         0x00000020L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                               0x000001C0L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                                0x00000E00L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                             0x00008000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                             0x00010000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                            0x03FC0000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                            0x0C000000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP__FLR_CAPABLE_MASK                                                          0x10000000L
+//BIF_CFG_DEV1_RC_DEVICE_CNTL
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                  0x1
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                      0x2
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                  0x5
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                   0x9
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                   0xa
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                       0xb
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                             0xc
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                               0xf
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__CORR_ERR_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                    0x0002L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                        0x0004L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__USR_REPORT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                      0x0010L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                    0x00E0L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                     0x0100L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                     0x0200L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                     0x0400L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                         0x0800L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                               0x7000L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                 0x8000L
+//BIF_CFG_DEV1_RC_DEVICE_STATUS
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__CORR_ERR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                   0x1
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__FATAL_ERR__SHIFT                                                       0x2
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__USR_DETECTED__SHIFT                                                    0x3
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__AUX_PWR__SHIFT                                                         0x4
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                               0x5
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                   0x6
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__CORR_ERR_MASK                                                          0x0001L
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                     0x0002L
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__FATAL_ERR_MASK                                                         0x0004L
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__USR_DETECTED_MASK                                                      0x0008L
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__AUX_PWR_MASK                                                           0x0010L
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                 0x0020L
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                     0x0040L
+//BIF_CFG_DEV1_RC_LINK_CAP
+#define BIF_CFG_DEV1_RC_LINK_CAP__LINK_SPEED__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_LINK_CAP__LINK_WIDTH__SHIFT                                                           0x4
+#define BIF_CFG_DEV1_RC_LINK_CAP__PM_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV1_RC_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                     0xc
+#define BIF_CFG_DEV1_RC_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                      0xf
+#define BIF_CFG_DEV1_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                               0x12
+#define BIF_CFG_DEV1_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                          0x13
+#define BIF_CFG_DEV1_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                          0x14
+#define BIF_CFG_DEV1_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                             0x15
+#define BIF_CFG_DEV1_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                          0x16
+#define BIF_CFG_DEV1_RC_LINK_CAP__PORT_NUMBER__SHIFT                                                          0x18
+#define BIF_CFG_DEV1_RC_LINK_CAP__LINK_SPEED_MASK                                                             0x0000000FL
+#define BIF_CFG_DEV1_RC_LINK_CAP__LINK_WIDTH_MASK                                                             0x000003F0L
+#define BIF_CFG_DEV1_RC_LINK_CAP__PM_SUPPORT_MASK                                                             0x00000C00L
+#define BIF_CFG_DEV1_RC_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                       0x00007000L
+#define BIF_CFG_DEV1_RC_LINK_CAP__L1_EXIT_LATENCY_MASK                                                        0x00038000L
+#define BIF_CFG_DEV1_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                 0x00040000L
+#define BIF_CFG_DEV1_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                            0x00080000L
+#define BIF_CFG_DEV1_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                            0x00100000L
+#define BIF_CFG_DEV1_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                               0x00200000L
+#define BIF_CFG_DEV1_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                            0x00400000L
+#define BIF_CFG_DEV1_RC_LINK_CAP__PORT_NUMBER_MASK                                                            0xFF000000L
+//BIF_CFG_DEV1_RC_LINK_CNTL
+#define BIF_CFG_DEV1_RC_LINK_CNTL__PM_CONTROL__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                        0x2
+#define BIF_CFG_DEV1_RC_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                   0x3
+#define BIF_CFG_DEV1_RC_LINK_CNTL__LINK_DIS__SHIFT                                                            0x4
+#define BIF_CFG_DEV1_RC_LINK_CNTL__RETRAIN_LINK__SHIFT                                                        0x5
+#define BIF_CFG_DEV1_RC_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                    0x6
+#define BIF_CFG_DEV1_RC_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                       0x7
+#define BIF_CFG_DEV1_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV1_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                         0x9
+#define BIF_CFG_DEV1_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                           0xa
+#define BIF_CFG_DEV1_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                           0xb
+#define BIF_CFG_DEV1_RC_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                               0xe
+#define BIF_CFG_DEV1_RC_LINK_CNTL__PM_CONTROL_MASK                                                            0x0003L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                          0x0004L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                     0x0008L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__LINK_DIS_MASK                                                              0x0010L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__RETRAIN_LINK_MASK                                                          0x0020L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                      0x0040L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__EXTENDED_SYNC_MASK                                                         0x0080L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                             0x0100L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                           0x0200L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                             0x0400L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                             0x0800L
+#define BIF_CFG_DEV1_RC_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                 0xC000L
+//BIF_CFG_DEV1_RC_LINK_STATUS
+#define BIF_CFG_DEV1_RC_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                             0x4
+#define BIF_CFG_DEV1_RC_LINK_STATUS__LINK_TRAINING__SHIFT                                                     0xb
+#define BIF_CFG_DEV1_RC_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                    0xc
+#define BIF_CFG_DEV1_RC_LINK_STATUS__DL_ACTIVE__SHIFT                                                         0xd
+#define BIF_CFG_DEV1_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                         0xe
+#define BIF_CFG_DEV1_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                         0xf
+#define BIF_CFG_DEV1_RC_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV1_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                               0x03F0L
+#define BIF_CFG_DEV1_RC_LINK_STATUS__LINK_TRAINING_MASK                                                       0x0800L
+#define BIF_CFG_DEV1_RC_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                      0x1000L
+#define BIF_CFG_DEV1_RC_LINK_STATUS__DL_ACTIVE_MASK                                                           0x2000L
+#define BIF_CFG_DEV1_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                           0x4000L
+#define BIF_CFG_DEV1_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                           0x8000L
+//BIF_CFG_DEV1_RC_SLOT_CAP
+#define BIF_CFG_DEV1_RC_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                               0x1
+#define BIF_CFG_DEV1_RC_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                   0x2
+#define BIF_CFG_DEV1_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                               0x3
+#define BIF_CFG_DEV1_RC_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                                0x4
+#define BIF_CFG_DEV1_RC_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                     0x5
+#define BIF_CFG_DEV1_RC_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                      0x6
+#define BIF_CFG_DEV1_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                 0x7
+#define BIF_CFG_DEV1_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                 0xf
+#define BIF_CFG_DEV1_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                        0x11
+#define BIF_CFG_DEV1_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                       0x12
+#define BIF_CFG_DEV1_RC_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                    0x13
+#define BIF_CFG_DEV1_RC_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                    0x00000001L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                 0x00000002L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                     0x00000004L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                 0x00000008L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                  0x00000010L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                       0x00000020L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                        0x00000040L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                   0x00007F80L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                   0x00018000L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                          0x00020000L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                         0x00040000L
+#define BIF_CFG_DEV1_RC_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                      0xFFF80000L
+//BIF_CFG_DEV1_RC_SLOT_CNTL
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                               0x1
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                          0x3
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                     0x5
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                 0x6
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                  0x8
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                 0xa
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                          0xb
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                 0xc
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                         0xd
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                   0xe
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                                0x0001L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                 0x0002L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                            0x0008L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                       0x0020L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                   0x00C0L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                    0x0300L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                   0x0400L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                            0x0800L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                   0x1000L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                           0x2000L
+#define BIF_CFG_DEV1_RC_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                     0x4000L
+//BIF_CFG_DEV1_RC_SLOT_STATUS
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                                0x1
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                                0x2
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                           0x3
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                 0x4
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                  0x5
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                             0x6
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                  0x8
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                 0x0001L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                  0x0002L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                  0x0004L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                             0x0008L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                   0x0010L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                    0x0020L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                               0x0040L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                        0x0080L
+#define BIF_CFG_DEV1_RC_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                    0x0100L
+//BIF_CFG_DEV1_RC_ROOT_CNTL
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                             0x1
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                               0x0002L
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV1_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                            0x0010L
+//BIF_CFG_DEV1_RC_ROOT_CAP
+#define BIF_CFG_DEV1_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                                0x0001L
+//BIF_CFG_DEV1_RC_ROOT_STATUS
+#define BIF_CFG_DEV1_RC_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_ROOT_STATUS__PME_STATUS__SHIFT                                                        0x10
+#define BIF_CFG_DEV1_RC_ROOT_STATUS__PME_PENDING__SHIFT                                                       0x11
+#define BIF_CFG_DEV1_RC_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV1_RC_ROOT_STATUS__PME_STATUS_MASK                                                          0x00010000L
+#define BIF_CFG_DEV1_RC_ROOT_STATUS__PME_PENDING_MASK                                                         0x00020000L
+//BIF_CFG_DEV1_RC_DEVICE_CAP2
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                       0x0
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                         0x4
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                          0x5
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                        0x6
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                        0x7
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                        0x8
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                            0x9
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                         0xa
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                     0xb
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                                0xc
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                     0xe
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                   0x10
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                   0x11
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                    0x12
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                      0x14
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                      0x15
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                          0x16
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                    0x18
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                     0x1a
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                         0x0000000FL
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                           0x00000010L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                            0x00000020L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                          0x00000040L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                          0x00000080L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                          0x00000100L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                              0x00000200L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                           0x00000400L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                       0x00000800L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                  0x00003000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                       0x0000C000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                     0x00010000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                     0x00020000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                      0x000C0000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                        0x00100000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                        0x00200000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                            0x00C00000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                      0x03000000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                       0x04000000L
+#define BIF_CFG_DEV1_RC_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV1_RC_DEVICE_CNTL2
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                  0x4
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                                0x5
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                              0x6
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                         0x7
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                               0x8
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                            0x9
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__LTR_EN__SHIFT                                                           0xa
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                     0xb
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                     0xc
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__OBFF_EN__SHIFT                                                          0xd
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                      0xf
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                  0x000FL
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                    0x0010L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                  0x0020L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                                0x0040L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                           0x0080L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                 0x0100L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                              0x0200L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__LTR_EN_MASK                                                             0x0400L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                       0x0800L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                       0x1000L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__OBFF_EN_MASK                                                            0x6000L
+#define BIF_CFG_DEV1_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                        0x8000L
+//BIF_CFG_DEV1_RC_DEVICE_STATUS2
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS2__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_RC_DEVICE_STATUS2__RESERVED_MASK                                                         0xFFFFL
+//BIF_CFG_DEV1_RC_LINK_CAP2
+#define BIF_CFG_DEV1_RC_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                                0x1
+#define BIF_CFG_DEV1_RC_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                            0x9
+#define BIF_CFG_DEV1_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                            0x10
+#define BIF_CFG_DEV1_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                           0x17
+#define BIF_CFG_DEV1_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                           0x18
+#define BIF_CFG_DEV1_RC_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                       0x1f
+#define BIF_CFG_DEV1_RC_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                  0x000000FEL
+#define BIF_CFG_DEV1_RC_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                   0x00000100L
+#define BIF_CFG_DEV1_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                              0x0000FE00L
+#define BIF_CFG_DEV1_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                              0x007F0000L
+#define BIF_CFG_DEV1_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                             0x00800000L
+#define BIF_CFG_DEV1_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                             0x01000000L
+#define BIF_CFG_DEV1_RC_LINK_CAP2__DRS_SUPPORTED_MASK                                                         0x80000000L
+//BIF_CFG_DEV1_RC_LINK_CNTL2
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                   0x4
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                        0x5
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                              0x6
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                        0x7
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                               0xa
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                     0xb
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                              0xc
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                    0x000FL
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                     0x0010L
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                          0x0020L
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                                0x0040L
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__XMIT_MARGIN_MASK                                                          0x0380L
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                 0x0400L
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                       0x0800L
+#define BIF_CFG_DEV1_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                                0xF000L
+//BIF_CFG_DEV1_RC_LINK_STATUS2
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                        0x1
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                  0x2
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                  0x3
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                  0x4
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                    0x5
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                                0x6
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                                0x7
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                             0x8
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                 0xa
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                    0xc
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                             0xf
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                               0x0001L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                          0x0002L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                    0x0004L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                    0x0008L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                    0x0010L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                      0x0020L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                  0x0040L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                  0x0080L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                               0x0300L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                   0x0400L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                      0x7000L
+#define BIF_CFG_DEV1_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                               0x8000L
+//BIF_CFG_DEV1_RC_SLOT_CAP2
+#define BIF_CFG_DEV1_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                         0x0
+#define BIF_CFG_DEV1_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                           0x00000001L
+//BIF_CFG_DEV1_RC_SLOT_CNTL2
+#define BIF_CFG_DEV1_RC_SLOT_CNTL2__RESERVED__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_SLOT_CNTL2__RESERVED_MASK                                                             0xFFFFL
+//BIF_CFG_DEV1_RC_SLOT_STATUS2
+#define BIF_CFG_DEV1_RC_SLOT_STATUS2__RESERVED__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_SLOT_STATUS2__RESERVED_MASK                                                           0xFFFFL
+//BIF_CFG_DEV1_RC_MSI_CAP_LIST
+#define BIF_CFG_DEV1_RC_MSI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV1_RC_MSI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV1_RC_MSI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV1_RC_MSI_MSG_CNTL
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_EN__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                    0x1
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                     0x4
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                        0x7
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                        0x8
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                             0x9
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                              0xa
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_EN_MASK                                                             0x0001L
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                      0x000EL
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                       0x0070L
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_64BIT_MASK                                                          0x0080L
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                          0x0100L
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                               0x0200L
+#define BIF_CFG_DEV1_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                                0x0400L
+//BIF_CFG_DEV1_RC_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV1_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                               0x2
+#define BIF_CFG_DEV1_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//BIF_CFG_DEV1_RC_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV1_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_MSI_MSG_DATA
+#define BIF_CFG_DEV1_RC_MSI_MSG_DATA__MSI_DATA__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_MSI_MSG_DATA__MSI_DATA_MASK                                                           0xFFFFL
+//BIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                   0xFFFFL
+//BIF_CFG_DEV1_RC_MSI_MSG_DATA_64
+#define BIF_CFG_DEV1_RC_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                     0xFFFFL
+//BIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                             0xFFFFL
+//BIF_CFG_DEV1_RC_SSID_CAP_LIST
+#define BIF_CFG_DEV1_RC_SSID_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV1_RC_SSID_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV1_RC_SSID_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV1_RC_SSID_CAP
+#define BIF_CFG_DEV1_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                         0x10
+#define BIF_CFG_DEV1_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV1_RC_SSID_CAP__SUBSYSTEM_ID_MASK                                                           0xFFFF0000L
+//BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                             0x10
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                          0x14
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV1_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                  0x1
+#define BIF_CFG_DEV1_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                       0x9
+#define BIF_CFG_DEV1_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                            0x00000001L
+#define BIF_CFG_DEV1_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                    0x00000002L
+#define BIF_CFG_DEV1_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                         0x0000FE00L
+//BIF_CFG_DEV1_RC_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV1_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                 0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                   0x0000FFFFL
+//BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV1_RC_LINK_CAP_16GT
+#define BIF_CFG_DEV1_RC_LINK_CAP_16GT__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC_LINK_CAP_16GT__RESERVED_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_LINK_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LINK_CNTL_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_RC_LINK_CNTL_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_LINK_STATUS_16GT
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                   0x0
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                             0x1
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                             0x2
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                             0x3
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                               0x4
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                               0x00000002L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                               0x00000004L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                               0x00000008L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                 0x00000010L
+//BIF_CFG_DEV1_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV1_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV1_RC_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV1_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV1_RC_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV1_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV1_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV1_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV1_RC_LINK_CAP_32GT
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                          0x1
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                               0x8
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                               0xa
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                                0xb
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                               0x00000001L
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                            0x00000002L
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                 0x00000100L
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                 0x00000200L
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                 0x00000400L
+#define BIF_CFG_DEV1_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                  0x0000F800L
+//BIF_CFG_DEV1_RC_LINK_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                  0x0
+#define BIF_CFG_DEV1_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                               0x1
+#define BIF_CFG_DEV1_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                     0x8
+#define BIF_CFG_DEV1_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                    0x00000001L
+#define BIF_CFG_DEV1_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                 0x00000002L
+#define BIF_CFG_DEV1_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                       0x00000700L
+//BIF_CFG_DEV1_RC_LINK_STATUS_32GT
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                   0x0
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                             0x1
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                             0x2
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                             0x3
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                               0x4
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                         0x5
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                         0x6
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                     0x8
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                  0x9
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                        0xa
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                               0x00000002L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                               0x00000004L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                               0x00000008L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                 0x00000010L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                           0x00000020L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                           0x000000C0L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                       0x00000100L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                    0x00000200L
+#define BIF_CFG_DEV1_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                          0x00000400L
+//BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                         0x0
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                             0x3
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                           0x00000007L
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                               0x0000FFF8L
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                             0xFFFF0000L
+//BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT              0x18
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                               0x00FFFFFFL
+#define BIF_CFG_DEV1_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK                0x03000000L
+//BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                      0x0
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                          0x3
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                        0x10
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                        0x00000007L
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                            0x0000FFF8L
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                          0xFFFF0000L
+//BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT           0x18
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                            0x00FFFFFFL
+#define BIF_CFG_DEV1_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK             0x03000000L
+//BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV1_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_RC_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV1_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV1_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV1_RC_AP_CAP
+#define BIF_CFG_DEV1_RC_AP_CAP__COUNT__SHIFT                                                                  0x0
+#define BIF_CFG_DEV1_RC_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_RC_AP_CAP__COUNT_MASK                                                                    0x000000FFL
+#define BIF_CFG_DEV1_RC_AP_CAP__SEL_EN_SUPPORTED_MASK                                                         0x00000100L
+//BIF_CFG_DEV1_RC_AP_CNTL
+#define BIF_CFG_DEV1_RC_AP_CNTL__INX_SEL__SHIFT                                                               0x0
+#define BIF_CFG_DEV1_RC_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV1_RC_AP_CNTL__INX_SEL_MASK                                                                 0x000000FFL
+#define BIF_CFG_DEV1_RC_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                          0x00000100L
+//BIF_CFG_DEV1_RC_AP_DATA1
+#define BIF_CFG_DEV1_RC_AP_DATA1__USAGE_INFOR__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC_AP_DATA1__DETAILS__SHIFT                                                              0x5
+#define BIF_CFG_DEV1_RC_AP_DATA1__VENDORID__SHIFT                                                             0x10
+#define BIF_CFG_DEV1_RC_AP_DATA1__USAGE_INFOR_MASK                                                            0x00000007L
+#define BIF_CFG_DEV1_RC_AP_DATA1__DETAILS_MASK                                                                0x0000FFE0L
+#define BIF_CFG_DEV1_RC_AP_DATA1__VENDORID_MASK                                                               0xFFFF0000L
+//BIF_CFG_DEV1_RC_AP_DATA2
+#define BIF_CFG_DEV1_RC_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                     0x00FFFFFFL
+//BIF_CFG_DEV1_RC_AP_SEL_EN_MASK
+#define BIF_CFG_DEV1_RC_AP_SEL_EN_MASK__PCIE__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC_AP_SEL_EN_MASK__OTHERS__SHIFT                                                         0x1
+#define BIF_CFG_DEV1_RC_AP_SEL_EN_MASK__PCIE_MASK                                                             0x00000001L
+#define BIF_CFG_DEV1_RC_AP_SEL_EN_MASK__OTHERS_MASK                                                           0xFFFFFFFEL
+//BIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_RC_MARGINING_PORT_CAP
+#define BIF_CFG_DEV1_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                    0x0
+#define BIF_CFG_DEV1_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                      0x0001L
+//BIF_CFG_DEV1_RC_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV1_RC_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                         0x0
+#define BIF_CFG_DEV1_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                                0x1
+#define BIF_CFG_DEV1_RC_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                           0x0001L
+#define BIF_CFG_DEV1_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                  0x0002L
+//BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV1_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_RC_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                               0x4
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                               0xa
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                              0x00000007L
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                 0x00000070L
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                   0x00000300L
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                 0x00000C00L
+//BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                     0x18
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                                0x000000FFL
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                       0xFF000000L
+//BIF_CFG_DEV1_RC_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                               0x1
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                             0x0001L
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                 0x000EL
+//BIF_CFG_DEV1_RC_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV1_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                         0x0001L
+//BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x007F0000L
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV1_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x003F0000L
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV1_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                     0x1f
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                       0x007FFFFFL
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                       0x80000000L
+//BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                           0x1f
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV1_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                             0x80000000L
+//BIF_CFG_DEV1_RC_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                         0x4
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                      0x5
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                         0xc
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                          0xd
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                       0x10
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                        0x11
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                         0x12
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                        0x13
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                  0x14
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                   0x15
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                  0x16
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                  0x17
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                         0x18
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                          0x19
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                     0x1a
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                           0x00000010L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                        0x00000020L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                           0x00001000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                            0x00002000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                     0x00008000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                         0x00010000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                          0x00020000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                           0x00040000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                          0x00080000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                    0x00100000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                     0x00200000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                    0x00400000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                    0x00800000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                           0x01000000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                            0x02000000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                       0x04000000L
+//BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                             0x4
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                          0x5
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                             0xc
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                              0xd
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                       0xf
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                            0x11
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                             0x12
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                            0x13
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                      0x14
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                       0x15
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                      0x16
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                      0x17
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                             0x18
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                              0x19
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                         0x1a
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                               0x00000010L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                            0x00000020L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                               0x00001000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                                0x00002000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                         0x00008000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                             0x00010000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                              0x00020000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                               0x00040000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                              0x00080000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                        0x00100000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                         0x00200000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                        0x00400000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                        0x00800000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                               0x01000000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                                0x02000000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                           0x04000000L
+//BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                     0x4
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                  0x5
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                     0xc
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                      0xd
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                 0xe
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                               0xf
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                   0x10
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                    0x11
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                     0x12
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                    0x13
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                              0x14
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                               0x15
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                              0x16
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                              0x17
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                     0x18
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                      0x19
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x1a
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                       0x00000010L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                    0x00000020L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                       0x00001000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                        0x00002000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                   0x00004000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                 0x00008000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                     0x00010000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                      0x00020000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                       0x00040000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                      0x00080000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                                0x00100000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                 0x00200000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                                0x00400000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                                0x00800000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                       0x01000000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                        0x02000000L
+#define BIF_CFG_DEV1_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                   0x04000000L
+//BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                           0x6
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                          0x7
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                               0x8
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                              0xc
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                             0xd
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                      0xe
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                      0xf
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                             0x00000001L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                             0x00000040L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                            0x00000080L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                 0x00000100L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                                0x00001000L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                               0x00002000L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                        0x00004000L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                        0x00008000L
+//BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                               0x6
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                              0x7
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                   0x8
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                  0xc
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                 0xd
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                          0xe
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                          0xf
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                 0x00000001L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                 0x00000040L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                                0x00000080L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                     0x00000100L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                    0x00001000L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                   0x00002000L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                            0x00004000L
+#define BIF_CFG_DEV1_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                            0x00008000L
+//BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                            0x5
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                          0x7
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                      0x9
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                  0xb
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                          0xc
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                             0x0000001FL
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                              0x00000020L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                               0x00000040L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                            0x00000080L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                             0x00000100L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                        0x00000200L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                         0x00000400L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                    0x00000800L
+#define BIF_CFG_DEV1_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                            0x00001000L
+//BIF_CFG_DEV1_RC_PCIE_HDR_LOG0
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG0__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_HDR_LOG1
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG1__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_HDR_LOG2
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG2__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_HDR_LOG3
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC_PCIE_HDR_LOG3__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                            0x2
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                               0x00000001L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                           0x00000002L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                              0x00000004L
+//BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                       0x1
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                  0x2
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                             0x3
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                                0x4
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                  0x5
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                     0x6
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                         0x7
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                      0x1b
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                              0x00000001L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                         0x00000002L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                    0x00000004L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                               0x00000008L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                  0x00000010L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                    0x00000020L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                       0x00000040L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                           0x00000180L
+#define BIF_CFG_DEV1_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                        0xF8000000L
+//BIF_CFG_DEV1_RC_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV1_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                     0x10
+#define BIF_CFG_DEV1_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                      0x0
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                      0x0
+#define BIF_CFG_DEV1_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV1_RC_PCIE_ACS_CAP
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                             0x1
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                             0x2
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                          0x3
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                              0x4
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                               0x5
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                            0x6
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                              0x7
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                       0x8
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                  0x0001L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                               0x0002L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                               0x0004L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                            0x0008L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                                0x0010L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                 0x0020L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                              0x0040L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                                0x0080L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                         0xFF00L
+//BIF_CFG_DEV1_RC_PCIE_ACS_CNTL
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                         0x2
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                      0x3
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                           0x5
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                          0x7
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0x8
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0xa
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                 0xc
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                           0x0002L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                           0x0004L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                        0x0008L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                             0x0020L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                            0x0080L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0300L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0C00L
+#define BIF_CFG_DEV1_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                   0x1000L
+//BIF_CFG_DEV1_RC_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV1_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV1_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV1_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV1_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV1_RC_RTR_DATA1
+#define BIF_CFG_DEV1_RC_RTR_DATA1__RESET_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC_RTR_DATA1__DLUP_TIME__SHIFT                                                           0xc
+#define BIF_CFG_DEV1_RC_RTR_DATA1__RTR_VALID__SHIFT                                                           0x1f
+#define BIF_CFG_DEV1_RC_RTR_DATA1__RESET_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV1_RC_RTR_DATA1__DLUP_TIME_MASK                                                             0x00FFF000L
+#define BIF_CFG_DEV1_RC_RTR_DATA1__RTR_VALID_MASK                                                             0x80000000L
+//BIF_CFG_DEV1_RC_RTR_DATA2
+#define BIF_CFG_DEV1_RC_RTR_DATA2__FLR_TIME__SHIFT                                                            0x0
+#define BIF_CFG_DEV1_RC_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                        0xc
+#define BIF_CFG_DEV1_RC_RTR_DATA2__FLR_TIME_MASK                                                              0x00000FFFL
+#define BIF_CFG_DEV1_RC_RTR_DATA2__D3HOTD0_TIME_MASK                                                          0x00FFF000L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_rc_bifcfgdecp
+//BIF_CFG_DEV2_RC_VENDOR_ID
+#define BIF_CFG_DEV2_RC_VENDOR_ID__VENDOR_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_VENDOR_ID__VENDOR_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV2_RC_DEVICE_ID
+#define BIF_CFG_DEV2_RC_DEVICE_ID__DEVICE_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_DEVICE_ID__DEVICE_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV2_RC_COMMAND
+#define BIF_CFG_DEV2_RC_COMMAND__IOEN_DN__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_RC_COMMAND__MEMEN_DN__SHIFT                                                              0x1
+#define BIF_CFG_DEV2_RC_COMMAND__BUS_MASTER_EN__SHIFT                                                         0x2
+#define BIF_CFG_DEV2_RC_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                      0x3
+#define BIF_CFG_DEV2_RC_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV2_RC_COMMAND__PAL_SNOOP_EN__SHIFT                                                          0x5
+#define BIF_CFG_DEV2_RC_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                 0x6
+#define BIF_CFG_DEV2_RC_COMMAND__AD_STEPPING__SHIFT                                                           0x7
+#define BIF_CFG_DEV2_RC_COMMAND__SERR_EN__SHIFT                                                               0x8
+#define BIF_CFG_DEV2_RC_COMMAND__FAST_B2B_EN__SHIFT                                                           0x9
+#define BIF_CFG_DEV2_RC_COMMAND__INT_DIS__SHIFT                                                               0xa
+#define BIF_CFG_DEV2_RC_COMMAND__IOEN_DN_MASK                                                                 0x0001L
+#define BIF_CFG_DEV2_RC_COMMAND__MEMEN_DN_MASK                                                                0x0002L
+#define BIF_CFG_DEV2_RC_COMMAND__BUS_MASTER_EN_MASK                                                           0x0004L
+#define BIF_CFG_DEV2_RC_COMMAND__SPECIAL_CYCLE_EN_MASK                                                        0x0008L
+#define BIF_CFG_DEV2_RC_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_RC_COMMAND__PAL_SNOOP_EN_MASK                                                            0x0020L
+#define BIF_CFG_DEV2_RC_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                   0x0040L
+#define BIF_CFG_DEV2_RC_COMMAND__AD_STEPPING_MASK                                                             0x0080L
+#define BIF_CFG_DEV2_RC_COMMAND__SERR_EN_MASK                                                                 0x0100L
+#define BIF_CFG_DEV2_RC_COMMAND__FAST_B2B_EN_MASK                                                             0x0200L
+#define BIF_CFG_DEV2_RC_COMMAND__INT_DIS_MASK                                                                 0x0400L
+//BIF_CFG_DEV2_RC_STATUS
+#define BIF_CFG_DEV2_RC_STATUS__IMMEDIATE_READINESS__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_RC_STATUS__INT_STATUS__SHIFT                                                             0x3
+#define BIF_CFG_DEV2_RC_STATUS__CAP_LIST__SHIFT                                                               0x4
+#define BIF_CFG_DEV2_RC_STATUS__PCI_66_CAP__SHIFT                                                             0x5
+#define BIF_CFG_DEV2_RC_STATUS__FAST_BACK_CAPABLE__SHIFT                                                      0x7
+#define BIF_CFG_DEV2_RC_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                               0x8
+#define BIF_CFG_DEV2_RC_STATUS__DEVSEL_TIMING__SHIFT                                                          0x9
+#define BIF_CFG_DEV2_RC_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                    0xb
+#define BIF_CFG_DEV2_RC_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                  0xc
+#define BIF_CFG_DEV2_RC_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                  0xd
+#define BIF_CFG_DEV2_RC_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                  0xe
+#define BIF_CFG_DEV2_RC_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_RC_STATUS__IMMEDIATE_READINESS_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_RC_STATUS__INT_STATUS_MASK                                                               0x0008L
+#define BIF_CFG_DEV2_RC_STATUS__CAP_LIST_MASK                                                                 0x0010L
+#define BIF_CFG_DEV2_RC_STATUS__PCI_66_CAP_MASK                                                               0x0020L
+#define BIF_CFG_DEV2_RC_STATUS__FAST_BACK_CAPABLE_MASK                                                        0x0080L
+#define BIF_CFG_DEV2_RC_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_RC_STATUS__DEVSEL_TIMING_MASK                                                            0x0600L
+#define BIF_CFG_DEV2_RC_STATUS__SIGNAL_TARGET_ABORT_MASK                                                      0x0800L
+#define BIF_CFG_DEV2_RC_STATUS__RECEIVED_TARGET_ABORT_MASK                                                    0x1000L
+#define BIF_CFG_DEV2_RC_STATUS__RECEIVED_MASTER_ABORT_MASK                                                    0x2000L
+#define BIF_CFG_DEV2_RC_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                    0x4000L
+#define BIF_CFG_DEV2_RC_STATUS__PARITY_ERROR_DETECTED_MASK                                                    0x8000L
+//BIF_CFG_DEV2_RC_REVISION_ID
+#define BIF_CFG_DEV2_RC_REVISION_ID__MINOR_REV_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_RC_REVISION_ID__MAJOR_REV_ID__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_RC_REVISION_ID__MINOR_REV_ID_MASK                                                        0x0FL
+#define BIF_CFG_DEV2_RC_REVISION_ID__MAJOR_REV_ID_MASK                                                        0xF0L
+//BIF_CFG_DEV2_RC_PROG_INTERFACE
+#define BIF_CFG_DEV2_RC_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC_PROG_INTERFACE__PROG_INTERFACE_MASK                                                   0xFFL
+//BIF_CFG_DEV2_RC_SUB_CLASS
+#define BIF_CFG_DEV2_RC_SUB_CLASS__SUB_CLASS__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_SUB_CLASS__SUB_CLASS_MASK                                                             0xFFL
+//BIF_CFG_DEV2_RC_BASE_CLASS
+#define BIF_CFG_DEV2_RC_BASE_CLASS__BASE_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_BASE_CLASS__BASE_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV2_RC_CACHE_LINE
+#define BIF_CFG_DEV2_RC_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_RC_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                      0xFFL
+//BIF_CFG_DEV2_RC_LATENCY
+#define BIF_CFG_DEV2_RC_LATENCY__LATENCY_TIMER__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_LATENCY__LATENCY_TIMER_MASK                                                           0xFFL
+//BIF_CFG_DEV2_RC_HEADER
+#define BIF_CFG_DEV2_RC_HEADER__HEADER_TYPE__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_RC_HEADER__DEVICE_TYPE__SHIFT                                                            0x7
+#define BIF_CFG_DEV2_RC_HEADER__HEADER_TYPE_MASK                                                              0x7FL
+#define BIF_CFG_DEV2_RC_HEADER__DEVICE_TYPE_MASK                                                              0x80L
+//BIF_CFG_DEV2_RC_BIST
+#define BIF_CFG_DEV2_RC_BIST__BIST_COMP__SHIFT                                                                0x0
+#define BIF_CFG_DEV2_RC_BIST__BIST_STRT__SHIFT                                                                0x6
+#define BIF_CFG_DEV2_RC_BIST__BIST_CAP__SHIFT                                                                 0x7
+#define BIF_CFG_DEV2_RC_BIST__BIST_COMP_MASK                                                                  0x0FL
+#define BIF_CFG_DEV2_RC_BIST__BIST_STRT_MASK                                                                  0x40L
+#define BIF_CFG_DEV2_RC_BIST__BIST_CAP_MASK                                                                   0x80L
+//BIF_CFG_DEV2_RC_BASE_ADDR_1
+#define BIF_CFG_DEV2_RC_BASE_ADDR_1__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_BASE_ADDR_1__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_BASE_ADDR_2
+#define BIF_CFG_DEV2_RC_BASE_ADDR_2__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_BASE_ADDR_2__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                          0x8
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                            0x10
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                                0x18
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                              0x000000FFL
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                            0x0000FF00L
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                              0x00FF0000L
+#define BIF_CFG_DEV2_RC_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                  0xFF000000L
+//BIF_CFG_DEV2_RC_IO_BASE_LIMIT
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_BASE__SHIFT                                                         0x4
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                        0xc
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                      0x000FL
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_BASE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                     0x0F00L
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT__IO_LIMIT_MASK                                                          0xF000L
+//BIF_CFG_DEV2_RC_SECONDARY_STATUS
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                            0x7
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                     0x8
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                                0x9
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                          0xb
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                        0xc
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                        0xd
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                        0xe
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                        0xf
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__PCI_66_CAP_MASK                                                     0x0020L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                              0x0080L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                       0x0100L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                  0x0600L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                            0x0800L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                          0x1000L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                          0x2000L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                          0x4000L
+#define BIF_CFG_DEV2_RC_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                          0x8000L
+//BIF_CFG_DEV2_RC_MEM_BASE_LIMIT
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                                0x14
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                    0x0000000FL
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                   0x0000FFF0L
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV2_RC_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV2_RC_PREF_BASE_LIMIT
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                           0x4
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                          0x14
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                              0x0000000FL
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                             0x0000FFF0L
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_RC_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_RC_PREF_BASE_UPPER
+#define BIF_CFG_DEV2_RC_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV2_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                               0x10
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_RC_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                 0xFFFF0000L
+//BIF_CFG_DEV2_RC_CAP_PTR
+#define BIF_CFG_DEV2_RC_CAP_PTR__CAP_PTR__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_RC_CAP_PTR__CAP_PTR_MASK                                                                 0xFFL
+//BIF_CFG_DEV2_RC_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                           0x1
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                          0x4
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                        0x00000001L
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                             0x0000000EL
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                            0x000000F0L
+#define BIF_CFG_DEV2_RC_ROM_BASE_ADDR__BASE_ADDR_MASK                                                         0xFFFFF800L
+//BIF_CFG_DEV2_RC_INTERRUPT_LINE
+#define BIF_CFG_DEV2_RC_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                   0xFFL
+//BIF_CFG_DEV2_RC_INTERRUPT_PIN
+#define BIF_CFG_DEV2_RC_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                     0xFFL
+//BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                        0x3
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                             0x5
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                           0x6
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                         0x8
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                       0x9
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                          0xa
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                     0xb
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                          0x0008L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                         0x0010L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                               0x0020L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                             0x0040L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                           0x0100L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                         0x0200L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                            0x0400L
+#define BIF_CFG_DEV2_RC_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                       0x0800L
+//BIF_CFG_DEV2_RC_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV2_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                   0x01L
+//BIF_CFG_DEV2_RC_PMI_CAP_LIST
+#define BIF_CFG_DEV2_RC_PMI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV2_RC_PMI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV2_RC_PMI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV2_RC_PMI_CAP
+#define BIF_CFG_DEV2_RC_PMI_CAP__VERSION__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_RC_PMI_CAP__PME_CLOCK__SHIFT                                                             0x3
+#define BIF_CFG_DEV2_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                   0x4
+#define BIF_CFG_DEV2_RC_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                     0x5
+#define BIF_CFG_DEV2_RC_PMI_CAP__AUX_CURRENT__SHIFT                                                           0x6
+#define BIF_CFG_DEV2_RC_PMI_CAP__D1_SUPPORT__SHIFT                                                            0x9
+#define BIF_CFG_DEV2_RC_PMI_CAP__D2_SUPPORT__SHIFT                                                            0xa
+#define BIF_CFG_DEV2_RC_PMI_CAP__PME_SUPPORT__SHIFT                                                           0xb
+#define BIF_CFG_DEV2_RC_PMI_CAP__VERSION_MASK                                                                 0x0007L
+#define BIF_CFG_DEV2_RC_PMI_CAP__PME_CLOCK_MASK                                                               0x0008L
+#define BIF_CFG_DEV2_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                     0x0010L
+#define BIF_CFG_DEV2_RC_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                       0x0020L
+#define BIF_CFG_DEV2_RC_PMI_CAP__AUX_CURRENT_MASK                                                             0x01C0L
+#define BIF_CFG_DEV2_RC_PMI_CAP__D1_SUPPORT_MASK                                                              0x0200L
+#define BIF_CFG_DEV2_RC_PMI_CAP__D2_SUPPORT_MASK                                                              0x0400L
+#define BIF_CFG_DEV2_RC_PMI_CAP__PME_SUPPORT_MASK                                                             0xF800L
+//BIF_CFG_DEV2_RC_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__PME_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                   0x9
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                    0xd
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                    0xf
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                 0x16
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                    0x17
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__POWER_STATE_MASK                                                     0x00000003L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                   0x00000008L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__PME_EN_MASK                                                          0x00000100L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                     0x00001E00L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                      0x00006000L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__PME_STATUS_MASK                                                      0x00008000L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                   0x00400000L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                      0x00800000L
+#define BIF_CFG_DEV2_RC_PMI_STATUS_CNTL__PMI_DATA_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_RC_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV2_RC_PCIE_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV2_RC_PCIE_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV2_RC_PCIE_CAP
+#define BIF_CFG_DEV2_RC_PCIE_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV2_RC_PCIE_CAP__DEVICE_TYPE__SHIFT                                                          0x4
+#define BIF_CFG_DEV2_RC_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_RC_PCIE_CAP__VERSION_MASK                                                                0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_CAP__DEVICE_TYPE_MASK                                                            0x00F0L
+#define BIF_CFG_DEV2_RC_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                       0x0100L
+#define BIF_CFG_DEV2_RC_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                        0x3E00L
+//BIF_CFG_DEV2_RC_DEVICE_CAP
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                       0x3
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                       0x5
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                             0x6
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                              0x9
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                           0xf
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                          0x12
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                          0x1a
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                        0x1c
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                  0x00000007L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__PHANTOM_FUNC_MASK                                                         0x00000018L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__EXTENDED_TAG_MASK                                                         0x00000020L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                               0x000001C0L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                                0x00000E00L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                             0x00008000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                             0x00010000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                            0x03FC0000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                            0x0C000000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP__FLR_CAPABLE_MASK                                                          0x10000000L
+//BIF_CFG_DEV2_RC_DEVICE_CNTL
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                  0x1
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                      0x2
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                  0x5
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                   0x9
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                   0xa
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                             0xc
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                               0xf
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__CORR_ERR_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                    0x0002L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                        0x0004L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__USR_REPORT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                      0x0010L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                    0x00E0L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                     0x0100L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                     0x0200L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                     0x0400L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                         0x0800L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                               0x7000L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                 0x8000L
+//BIF_CFG_DEV2_RC_DEVICE_STATUS
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__CORR_ERR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                   0x1
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__FATAL_ERR__SHIFT                                                       0x2
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__USR_DETECTED__SHIFT                                                    0x3
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__AUX_PWR__SHIFT                                                         0x4
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                               0x5
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                   0x6
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__CORR_ERR_MASK                                                          0x0001L
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                     0x0002L
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__FATAL_ERR_MASK                                                         0x0004L
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__USR_DETECTED_MASK                                                      0x0008L
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__AUX_PWR_MASK                                                           0x0010L
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                 0x0020L
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                     0x0040L
+//BIF_CFG_DEV2_RC_LINK_CAP
+#define BIF_CFG_DEV2_RC_LINK_CAP__LINK_SPEED__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_LINK_CAP__LINK_WIDTH__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_RC_LINK_CAP__PM_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_RC_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                     0xc
+#define BIF_CFG_DEV2_RC_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                      0xf
+#define BIF_CFG_DEV2_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                               0x12
+#define BIF_CFG_DEV2_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                          0x13
+#define BIF_CFG_DEV2_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                          0x14
+#define BIF_CFG_DEV2_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                             0x15
+#define BIF_CFG_DEV2_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                          0x16
+#define BIF_CFG_DEV2_RC_LINK_CAP__PORT_NUMBER__SHIFT                                                          0x18
+#define BIF_CFG_DEV2_RC_LINK_CAP__LINK_SPEED_MASK                                                             0x0000000FL
+#define BIF_CFG_DEV2_RC_LINK_CAP__LINK_WIDTH_MASK                                                             0x000003F0L
+#define BIF_CFG_DEV2_RC_LINK_CAP__PM_SUPPORT_MASK                                                             0x00000C00L
+#define BIF_CFG_DEV2_RC_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                       0x00007000L
+#define BIF_CFG_DEV2_RC_LINK_CAP__L1_EXIT_LATENCY_MASK                                                        0x00038000L
+#define BIF_CFG_DEV2_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                 0x00040000L
+#define BIF_CFG_DEV2_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                            0x00080000L
+#define BIF_CFG_DEV2_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                            0x00100000L
+#define BIF_CFG_DEV2_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                               0x00200000L
+#define BIF_CFG_DEV2_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                            0x00400000L
+#define BIF_CFG_DEV2_RC_LINK_CAP__PORT_NUMBER_MASK                                                            0xFF000000L
+//BIF_CFG_DEV2_RC_LINK_CNTL
+#define BIF_CFG_DEV2_RC_LINK_CNTL__PM_CONTROL__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                        0x2
+#define BIF_CFG_DEV2_RC_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_RC_LINK_CNTL__LINK_DIS__SHIFT                                                            0x4
+#define BIF_CFG_DEV2_RC_LINK_CNTL__RETRAIN_LINK__SHIFT                                                        0x5
+#define BIF_CFG_DEV2_RC_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                    0x6
+#define BIF_CFG_DEV2_RC_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV2_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                         0x9
+#define BIF_CFG_DEV2_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                           0xa
+#define BIF_CFG_DEV2_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                           0xb
+#define BIF_CFG_DEV2_RC_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                               0xe
+#define BIF_CFG_DEV2_RC_LINK_CNTL__PM_CONTROL_MASK                                                            0x0003L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                          0x0004L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                     0x0008L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__LINK_DIS_MASK                                                              0x0010L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__RETRAIN_LINK_MASK                                                          0x0020L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                      0x0040L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__EXTENDED_SYNC_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                             0x0100L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                           0x0200L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                             0x0400L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                             0x0800L
+#define BIF_CFG_DEV2_RC_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                 0xC000L
+//BIF_CFG_DEV2_RC_LINK_STATUS
+#define BIF_CFG_DEV2_RC_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                             0x4
+#define BIF_CFG_DEV2_RC_LINK_STATUS__LINK_TRAINING__SHIFT                                                     0xb
+#define BIF_CFG_DEV2_RC_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_RC_LINK_STATUS__DL_ACTIVE__SHIFT                                                         0xd
+#define BIF_CFG_DEV2_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                         0xe
+#define BIF_CFG_DEV2_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                         0xf
+#define BIF_CFG_DEV2_RC_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV2_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                               0x03F0L
+#define BIF_CFG_DEV2_RC_LINK_STATUS__LINK_TRAINING_MASK                                                       0x0800L
+#define BIF_CFG_DEV2_RC_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                      0x1000L
+#define BIF_CFG_DEV2_RC_LINK_STATUS__DL_ACTIVE_MASK                                                           0x2000L
+#define BIF_CFG_DEV2_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                           0x4000L
+#define BIF_CFG_DEV2_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                           0x8000L
+//BIF_CFG_DEV2_RC_SLOT_CAP
+#define BIF_CFG_DEV2_RC_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                               0x1
+#define BIF_CFG_DEV2_RC_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                               0x3
+#define BIF_CFG_DEV2_RC_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                                0x4
+#define BIF_CFG_DEV2_RC_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                     0x5
+#define BIF_CFG_DEV2_RC_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                      0x6
+#define BIF_CFG_DEV2_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                 0x7
+#define BIF_CFG_DEV2_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                        0x11
+#define BIF_CFG_DEV2_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                       0x12
+#define BIF_CFG_DEV2_RC_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                    0x13
+#define BIF_CFG_DEV2_RC_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                 0x00000002L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                     0x00000004L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                 0x00000008L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                  0x00000010L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                       0x00000020L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                        0x00000040L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                   0x00007F80L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                   0x00018000L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                         0x00040000L
+#define BIF_CFG_DEV2_RC_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                      0xFFF80000L
+//BIF_CFG_DEV2_RC_SLOT_CNTL
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                               0x1
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                          0x3
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                     0x5
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                 0x6
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                 0xa
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                          0xb
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                         0xd
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                   0xe
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                                0x0001L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                            0x0008L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                       0x0020L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                   0x00C0L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                    0x0300L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                   0x0400L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                            0x0800L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                   0x1000L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                           0x2000L
+#define BIF_CFG_DEV2_RC_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                     0x4000L
+//BIF_CFG_DEV2_RC_SLOT_STATUS
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                                0x1
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                                0x2
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                           0x3
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                  0x5
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                 0x0001L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                  0x0002L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                  0x0004L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                             0x0008L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                   0x0010L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                    0x0020L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                        0x0080L
+#define BIF_CFG_DEV2_RC_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                    0x0100L
+//BIF_CFG_DEV2_RC_ROOT_CNTL
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                             0x1
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                               0x0002L
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV2_RC_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                            0x0010L
+//BIF_CFG_DEV2_RC_ROOT_CAP
+#define BIF_CFG_DEV2_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                                0x0001L
+//BIF_CFG_DEV2_RC_ROOT_STATUS
+#define BIF_CFG_DEV2_RC_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_ROOT_STATUS__PME_STATUS__SHIFT                                                        0x10
+#define BIF_CFG_DEV2_RC_ROOT_STATUS__PME_PENDING__SHIFT                                                       0x11
+#define BIF_CFG_DEV2_RC_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV2_RC_ROOT_STATUS__PME_STATUS_MASK                                                          0x00010000L
+#define BIF_CFG_DEV2_RC_ROOT_STATUS__PME_PENDING_MASK                                                         0x00020000L
+//BIF_CFG_DEV2_RC_DEVICE_CAP2
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                       0x0
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                         0x4
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                          0x5
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                        0x6
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                        0x7
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                        0x8
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                            0x9
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                         0xa
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                     0xb
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                                0xc
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                     0xe
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                   0x10
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                   0x11
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                    0x12
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                      0x14
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                      0x15
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                          0x16
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                    0x18
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                     0x1a
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                         0x0000000FL
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                            0x00000020L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                          0x00000040L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                          0x00000080L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                          0x00000100L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                              0x00000200L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                           0x00000400L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                       0x00000800L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                  0x00003000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                       0x0000C000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                     0x00020000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                      0x000C0000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                        0x00200000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                            0x00C00000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                      0x03000000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                       0x04000000L
+#define BIF_CFG_DEV2_RC_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV2_RC_DEVICE_CNTL2
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                                0x5
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                              0x6
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                         0x7
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                               0x8
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                            0x9
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__LTR_EN__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                     0xb
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                     0xc
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__OBFF_EN__SHIFT                                                          0xd
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                      0xf
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                  0x000FL
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                    0x0010L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                  0x0020L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                                0x0040L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                           0x0080L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                              0x0200L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__LTR_EN_MASK                                                             0x0400L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                       0x0800L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                       0x1000L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__OBFF_EN_MASK                                                            0x6000L
+#define BIF_CFG_DEV2_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                        0x8000L
+//BIF_CFG_DEV2_RC_DEVICE_STATUS2
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS2__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_RC_DEVICE_STATUS2__RESERVED_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_RC_LINK_CAP2
+#define BIF_CFG_DEV2_RC_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                                0x1
+#define BIF_CFG_DEV2_RC_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                            0x9
+#define BIF_CFG_DEV2_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                            0x10
+#define BIF_CFG_DEV2_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                           0x17
+#define BIF_CFG_DEV2_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                           0x18
+#define BIF_CFG_DEV2_RC_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_RC_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                  0x000000FEL
+#define BIF_CFG_DEV2_RC_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                   0x00000100L
+#define BIF_CFG_DEV2_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                              0x0000FE00L
+#define BIF_CFG_DEV2_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                              0x007F0000L
+#define BIF_CFG_DEV2_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                             0x00800000L
+#define BIF_CFG_DEV2_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                             0x01000000L
+#define BIF_CFG_DEV2_RC_LINK_CAP2__DRS_SUPPORTED_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_RC_LINK_CNTL2
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                   0x4
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                        0x5
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                              0x6
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                               0xa
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                     0xb
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                              0xc
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                    0x000FL
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                     0x0010L
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                          0x0020L
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                                0x0040L
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__XMIT_MARGIN_MASK                                                          0x0380L
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                       0x0800L
+#define BIF_CFG_DEV2_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                                0xF000L
+//BIF_CFG_DEV2_RC_LINK_STATUS2
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                        0x1
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                  0x2
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                  0x3
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                  0x4
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                    0x5
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                                0x6
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                                0x7
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                             0x8
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                 0xa
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                    0xc
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                             0xf
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                               0x0001L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                          0x0002L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                    0x0004L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                    0x0008L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                    0x0010L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                      0x0020L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                  0x0080L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                               0x0300L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                   0x0400L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                      0x7000L
+#define BIF_CFG_DEV2_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                               0x8000L
+//BIF_CFG_DEV2_RC_SLOT_CAP2
+#define BIF_CFG_DEV2_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                         0x0
+#define BIF_CFG_DEV2_RC_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                           0x00000001L
+//BIF_CFG_DEV2_RC_SLOT_CNTL2
+#define BIF_CFG_DEV2_RC_SLOT_CNTL2__RESERVED__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_SLOT_CNTL2__RESERVED_MASK                                                             0xFFFFL
+//BIF_CFG_DEV2_RC_SLOT_STATUS2
+#define BIF_CFG_DEV2_RC_SLOT_STATUS2__RESERVED__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_SLOT_STATUS2__RESERVED_MASK                                                           0xFFFFL
+//BIF_CFG_DEV2_RC_MSI_CAP_LIST
+#define BIF_CFG_DEV2_RC_MSI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV2_RC_MSI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV2_RC_MSI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV2_RC_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_EN__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                    0x1
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                        0x8
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                             0x9
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                              0xa
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_EN_MASK                                                             0x0001L
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                      0x000EL
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                       0x0070L
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_64BIT_MASK                                                          0x0080L
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                          0x0100L
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                               0x0200L
+#define BIF_CFG_DEV2_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                                0x0400L
+//BIF_CFG_DEV2_RC_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                               0x2
+#define BIF_CFG_DEV2_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//BIF_CFG_DEV2_RC_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_MSI_MSG_DATA
+#define BIF_CFG_DEV2_RC_MSI_MSG_DATA__MSI_DATA__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_MSI_MSG_DATA__MSI_DATA_MASK                                                           0xFFFFL
+//BIF_CFG_DEV2_RC_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                   0xFFFFL
+//BIF_CFG_DEV2_RC_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_RC_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_RC_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                             0xFFFFL
+//BIF_CFG_DEV2_RC_SSID_CAP_LIST
+#define BIF_CFG_DEV2_RC_SSID_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV2_RC_SSID_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV2_RC_SSID_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV2_RC_SSID_CAP
+#define BIF_CFG_DEV2_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                         0x10
+#define BIF_CFG_DEV2_RC_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV2_RC_SSID_CAP__SUBSYSTEM_ID_MASK                                                           0xFFFF0000L
+//BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                             0x10
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                          0x14
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV2_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                  0x1
+#define BIF_CFG_DEV2_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                       0x9
+#define BIF_CFG_DEV2_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                            0x00000001L
+#define BIF_CFG_DEV2_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                    0x00000002L
+#define BIF_CFG_DEV2_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                         0x0000FE00L
+//BIF_CFG_DEV2_RC_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV2_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                 0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                   0x0000FFFFL
+//BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV2_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV2_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV2_RC_LINK_CAP_16GT
+#define BIF_CFG_DEV2_RC_LINK_CAP_16GT__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC_LINK_CAP_16GT__RESERVED_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_LINK_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LINK_CNTL_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_RC_LINK_CNTL_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_LINK_STATUS_16GT
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                             0x1
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                             0x2
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                             0x3
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                               0x4
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                               0x00000002L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                               0x00000004L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                               0x00000008L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                 0x00000010L
+//BIF_CFG_DEV2_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV2_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV2_RC_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV2_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV2_RC_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV2_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV2_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV2_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV2_RC_LINK_CAP_32GT
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                          0x1
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                               0x8
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                               0xa
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                                0xb
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                               0x00000001L
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                            0x00000002L
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                 0x00000200L
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                 0x00000400L
+#define BIF_CFG_DEV2_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                  0x0000F800L
+//BIF_CFG_DEV2_RC_LINK_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                  0x0
+#define BIF_CFG_DEV2_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                               0x1
+#define BIF_CFG_DEV2_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                     0x8
+#define BIF_CFG_DEV2_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                    0x00000001L
+#define BIF_CFG_DEV2_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                 0x00000002L
+#define BIF_CFG_DEV2_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                       0x00000700L
+//BIF_CFG_DEV2_RC_LINK_STATUS_32GT
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                             0x1
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                             0x2
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                             0x3
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                               0x4
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                         0x5
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                         0x6
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                     0x8
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                  0x9
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                        0xa
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                               0x00000002L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                               0x00000004L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                               0x00000008L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                 0x00000010L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                           0x00000020L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                           0x000000C0L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                       0x00000100L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                          0x00000400L
+//BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                         0x0
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                             0x3
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                           0x00000007L
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                               0x0000FFF8L
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                             0xFFFF0000L
+//BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT              0x18
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                               0x00FFFFFFL
+#define BIF_CFG_DEV2_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK                0x03000000L
+//BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                      0x0
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                          0x3
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                        0x10
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                        0x00000007L
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                            0x0000FFF8L
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                          0xFFFF0000L
+//BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT           0x18
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                            0x00FFFFFFL
+#define BIF_CFG_DEV2_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK             0x03000000L
+//BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV2_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_RC_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV2_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV2_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV2_RC_AP_CAP
+#define BIF_CFG_DEV2_RC_AP_CAP__COUNT__SHIFT                                                                  0x0
+#define BIF_CFG_DEV2_RC_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                       0x8
+#define BIF_CFG_DEV2_RC_AP_CAP__COUNT_MASK                                                                    0x000000FFL
+#define BIF_CFG_DEV2_RC_AP_CAP__SEL_EN_SUPPORTED_MASK                                                         0x00000100L
+//BIF_CFG_DEV2_RC_AP_CNTL
+#define BIF_CFG_DEV2_RC_AP_CNTL__INX_SEL__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_RC_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV2_RC_AP_CNTL__INX_SEL_MASK                                                                 0x000000FFL
+#define BIF_CFG_DEV2_RC_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                          0x00000100L
+//BIF_CFG_DEV2_RC_AP_DATA1
+#define BIF_CFG_DEV2_RC_AP_DATA1__USAGE_INFOR__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC_AP_DATA1__DETAILS__SHIFT                                                              0x5
+#define BIF_CFG_DEV2_RC_AP_DATA1__VENDORID__SHIFT                                                             0x10
+#define BIF_CFG_DEV2_RC_AP_DATA1__USAGE_INFOR_MASK                                                            0x00000007L
+#define BIF_CFG_DEV2_RC_AP_DATA1__DETAILS_MASK                                                                0x0000FFE0L
+#define BIF_CFG_DEV2_RC_AP_DATA1__VENDORID_MASK                                                               0xFFFF0000L
+//BIF_CFG_DEV2_RC_AP_DATA2
+#define BIF_CFG_DEV2_RC_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                     0x00FFFFFFL
+//BIF_CFG_DEV2_RC_AP_SEL_EN_MASK
+#define BIF_CFG_DEV2_RC_AP_SEL_EN_MASK__PCIE__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC_AP_SEL_EN_MASK__OTHERS__SHIFT                                                         0x1
+#define BIF_CFG_DEV2_RC_AP_SEL_EN_MASK__PCIE_MASK                                                             0x00000001L
+#define BIF_CFG_DEV2_RC_AP_SEL_EN_MASK__OTHERS_MASK                                                           0xFFFFFFFEL
+//BIF_CFG_DEV2_RC_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_RC_MARGINING_PORT_CAP
+#define BIF_CFG_DEV2_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                    0x0
+#define BIF_CFG_DEV2_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                      0x0001L
+//BIF_CFG_DEV2_RC_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV2_RC_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                         0x0
+#define BIF_CFG_DEV2_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                                0x1
+#define BIF_CFG_DEV2_RC_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                           0x0001L
+#define BIF_CFG_DEV2_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                  0x0002L
+//BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV2_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_RC_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV2_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV2_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                               0x4
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                               0xa
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                 0x00000070L
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                   0x00000300L
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                 0x00000C00L
+//BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                     0x18
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                                0x000000FFL
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                       0xFF000000L
+//BIF_CFG_DEV2_RC_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                               0x1
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                             0x0001L
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                 0x000EL
+//BIF_CFG_DEV2_RC_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                         0x0001L
+//BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x007F0000L
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV2_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x003F0000L
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV2_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV2_RC_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV2_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                     0x1f
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                       0x007FFFFFL
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                       0x80000000L
+//BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                           0x1f
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV2_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                             0x80000000L
+//BIF_CFG_DEV2_RC_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                         0x4
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                      0x5
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                         0xc
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                          0xd
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                       0x10
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                        0x11
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                         0x12
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                        0x13
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                  0x14
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                   0x15
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                  0x16
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                  0x17
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                         0x18
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                          0x19
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                     0x1a
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                           0x01000000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                            0x02000000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                       0x04000000L
+//BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                             0x4
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                          0x5
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                             0xc
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                              0xd
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                       0xf
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                            0x11
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                             0x12
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                            0x13
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                      0x14
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                       0x15
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                      0x16
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                      0x17
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                             0x18
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                              0x19
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                         0x1a
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                               0x00000010L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                            0x00000020L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                               0x00001000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                                0x00002000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                             0x00010000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                              0x00020000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                               0x00040000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                              0x00080000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                         0x00200000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                        0x00800000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                               0x01000000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                                0x02000000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                           0x04000000L
+//BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                     0x4
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                  0x5
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                     0xc
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                      0xd
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                 0xe
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                               0xf
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                   0x10
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                    0x11
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                     0x12
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                    0x13
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                              0x14
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                               0x15
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                              0x16
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                              0x17
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                     0x18
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                      0x19
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x1a
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                                0x00100000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                                0x00400000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                                0x00800000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                       0x01000000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                        0x02000000L
+#define BIF_CFG_DEV2_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                   0x04000000L
+//BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                           0x6
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                          0x7
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                               0x8
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                              0xc
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                             0xd
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                      0xe
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                      0xf
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                                0x00001000L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                               0x00002000L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                        0x00008000L
+//BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                               0x6
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                              0x7
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                   0x8
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                  0xc
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                 0xd
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                          0xe
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                          0xf
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                 0x00000001L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                 0x00000040L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                                0x00000080L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                     0x00000100L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                    0x00001000L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                   0x00002000L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                            0x00004000L
+#define BIF_CFG_DEV2_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                            0x00008000L
+//BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                            0x5
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                          0x7
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                      0x9
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                  0xb
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                          0xc
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                             0x0000001FL
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                              0x00000020L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                               0x00000040L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                             0x00000100L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                        0x00000200L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                         0x00000400L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                    0x00000800L
+#define BIF_CFG_DEV2_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                            0x00001000L
+//BIF_CFG_DEV2_RC_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG0__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG1__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG2__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC_PCIE_HDR_LOG3__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                            0x2
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                               0x00000001L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                           0x00000002L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                              0x00000004L
+//BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                       0x1
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                  0x2
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                             0x3
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                                0x4
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                  0x5
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                     0x6
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                         0x7
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                      0x1b
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                              0x00000001L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                         0x00000002L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                    0x00000004L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                               0x00000008L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                  0x00000010L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                       0x00000040L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                           0x00000180L
+#define BIF_CFG_DEV2_RC_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                        0xF8000000L
+//BIF_CFG_DEV2_RC_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV2_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                     0x10
+#define BIF_CFG_DEV2_RC_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                      0x0
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                      0x0
+#define BIF_CFG_DEV2_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV2_RC_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV2_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV2_RC_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                             0x1
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                             0x2
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                          0x3
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                              0x4
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                               0x5
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                            0x6
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                              0x7
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                       0x8
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                               0x0002L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                               0x0004L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                            0x0008L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                                0x0010L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                 0x0020L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                              0x0040L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                                0x0080L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                         0xFF00L
+//BIF_CFG_DEV2_RC_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                         0x2
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                      0x3
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                           0x5
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                          0x7
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0x8
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0xa
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                 0xc
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                           0x0002L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                           0x0004L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                        0x0008L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                             0x0020L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                            0x0080L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0300L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0C00L
+#define BIF_CFG_DEV2_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                   0x1000L
+//BIF_CFG_DEV2_RC_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV2_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV2_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV2_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV2_RC_RTR_DATA1
+#define BIF_CFG_DEV2_RC_RTR_DATA1__RESET_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC_RTR_DATA1__DLUP_TIME__SHIFT                                                           0xc
+#define BIF_CFG_DEV2_RC_RTR_DATA1__RTR_VALID__SHIFT                                                           0x1f
+#define BIF_CFG_DEV2_RC_RTR_DATA1__RESET_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV2_RC_RTR_DATA1__DLUP_TIME_MASK                                                             0x00FFF000L
+#define BIF_CFG_DEV2_RC_RTR_DATA1__RTR_VALID_MASK                                                             0x80000000L
+//BIF_CFG_DEV2_RC_RTR_DATA2
+#define BIF_CFG_DEV2_RC_RTR_DATA2__FLR_TIME__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_RC_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                        0xc
+#define BIF_CFG_DEV2_RC_RTR_DATA2__FLR_TIME_MASK                                                              0x00000FFFL
+#define BIF_CFG_DEV2_RC_RTR_DATA2__D3HOTD0_TIME_MASK                                                          0x00FFF000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_COMMAND
+#define BIF_CFG_DEV0_EPF0_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF0_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF0_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF0_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF0_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF0_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF0_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF0_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF0_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF0_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF0_STATUS
+#define BIF_CFG_DEV0_EPF0_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF0_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF0_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF0_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF0_LATENCY
+#define BIF_CFG_DEV0_EPF0_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_HEADER
+#define BIF_CFG_DEV0_EPF0_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF0_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF0_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF0_BIST
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF0_PMI_CAP
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                       0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                          0x000003FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                          0x00001C00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                         0x03FF0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                         0x1C000000L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//PCIE_ATS_ENH_CAP_LIST
+#define PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                                                  0x0
+#define PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                                                 0x10
+#define PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                                0x14
+#define PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                                                    0x0000FFFFL
+#define PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                                                   0x000F0000L
+#define PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                                                  0xFFF00000L
+//PCIE_ATS_CAP
+#define PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                                               0x0
+#define PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                                             0x5
+#define PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                                      0x6
+#define PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                                       0x7
+#define PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                                                     0x9
+#define PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                                                 0x001FL
+#define PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                                               0x0020L
+#define PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                                        0x0040L
+#define PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                                         0x0080L
+#define PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                                       0x0200L
+//PCIE_ATS_CNTL
+#define PCIE_ATS_CNTL__STU__SHIFT                                                                             0x0
+#define PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                      0xf
+#define PCIE_ATS_CNTL__STU_MASK                                                                               0x001FL
+#define PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                        0x8000L
+//PCIE_PAGE_REQ_ENH_CAP_LIST
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID__SHIFT                                                             0x0
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER__SHIFT                                                            0x10
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                           0x14
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID_MASK                                                               0x0000FFFFL
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER_MASK                                                              0x000F0000L
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR_MASK                                                             0xFFF00000L
+//PCIE_PAGE_REQ_CNTL
+#define PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                                                 0x0
+#define PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                                                  0x1
+#define PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                                                   0x0001L
+#define PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                                    0x0002L
+//PCIE_PAGE_REQ_STATUS
+#define PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE__SHIFT                                                         0x0
+#define PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX__SHIFT                                            0x1
+#define PCIE_PAGE_REQ_STATUS__STOPPED__SHIFT                                                                  0x8
+#define PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED__SHIFT                                              0xf
+#define PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE_MASK                                                           0x0001L
+#define PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX_MASK                                              0x0002L
+#define PCIE_PAGE_REQ_STATUS__STOPPED_MASK                                                                    0x0100L
+#define PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED_MASK                                                0x8000L
+//PCIE_OUTSTAND_PAGE_REQ_CAPACITY
+#define PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY__SHIFT                                    0x0
+#define PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY_MASK                                      0xFFFFFFFFL
+//PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                                          0x0
+#define PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                            0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                     0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                              0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                              0x00000002L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                       0x00000004L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                                0xFFE00000L
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                                0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                  0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                  0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                             0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                    0x0010L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                      0x0020L
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                   0x0001L
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_INITIAL_VFS
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                      0xFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_TOTAL_VFS
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                          0xFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_NUM_VFS
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                              0xFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_FUNC_DEP_LINK
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                  0xFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_FIRST_VF_OFFSET
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                              0xFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_STRIDE
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                          0xFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                    0xFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK       0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER__SHIFT                                              0x10
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR__SHIFT                                             0x14
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID_MASK                                                 0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER_MASK                                                0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR_MASK                                               0xFFF00000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID__SHIFT                                                       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV__SHIFT                                                      0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH__SHIFT                                                   0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID_MASK                                                         0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV_MASK                                                        0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH_MASK                                                     0xFFF00000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN__SHIFT                      0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN__SHIFT                    0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN_MASK                        0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN_MASK                      0x02000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS__SHIFT                  0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS__SHIFT                0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS_MASK                    0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS_MASK                  0x02000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR__SHIFT                                     0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID__SHIFT                                     0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR_MASK                                       0x0001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID_MASK                                       0xFF00L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX__SHIFT                                        0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA__SHIFT                                    0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID__SHIFT                                   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK__SHIFT                                     0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX_MASK                                          0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA_MASK                                      0x00000F00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID_MASK                                     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA_MASK                                      0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK_MASK                                       0x01000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK__SHIFT                                     0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID__SHIFT                                   0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK__SHIFT                                     0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID__SHIFT                                   0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK__SHIFT                                     0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID__SHIFT                                   0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK__SHIFT                                     0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID__SHIFT                                   0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK__SHIFT                                     0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID__SHIFT                                   0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK__SHIFT                                     0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID__SHIFT                                   0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK__SHIFT                                     0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID__SHIFT                                   0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK__SHIFT                                     0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID__SHIFT                                   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK__SHIFT                                     0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID__SHIFT                                   0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK__SHIFT                                     0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID__SHIFT                                   0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK__SHIFT                                    0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID__SHIFT                                  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK__SHIFT                                    0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID__SHIFT                                  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK__SHIFT                                    0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID__SHIFT                                  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK__SHIFT                                    0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID__SHIFT                                  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK__SHIFT                                    0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID__SHIFT                                  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK__SHIFT                                    0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID__SHIFT                                  0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK_MASK                                       0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID_MASK                                     0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK_MASK                                       0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID_MASK                                     0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK_MASK                                       0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID_MASK                                     0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK_MASK                                       0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID_MASK                                     0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK_MASK                                       0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID_MASK                                     0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK_MASK                                       0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID_MASK                                     0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK_MASK                                       0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID_MASK                                     0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK_MASK                                       0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID_MASK                                     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK_MASK                                       0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID_MASK                                     0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK_MASK                                       0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID_MASK                                     0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK_MASK                                      0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID_MASK                                    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK_MASK                                      0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID_MASK                                    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK_MASK                                      0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID_MASK                                    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK_MASK                                      0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID_MASK                                    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK_MASK                                      0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID_MASK                                    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK_MASK                                      0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID_MASK                                    0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK__SHIFT                                    0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID__SHIFT                                  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK__SHIFT                                    0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID__SHIFT                                  0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK__SHIFT                                    0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID__SHIFT                                  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK__SHIFT                                    0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID__SHIFT                                  0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK__SHIFT                                    0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID__SHIFT                                  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK__SHIFT                                    0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID__SHIFT                                  0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK__SHIFT                                    0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID__SHIFT                                  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK__SHIFT                                    0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID__SHIFT                                  0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID__SHIFT                                  0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK__SHIFT                                    0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID__SHIFT                                  0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK__SHIFT                                    0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID__SHIFT                                  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK__SHIFT                                    0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID__SHIFT                                  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK__SHIFT                                    0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID__SHIFT                                  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK__SHIFT                                    0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID__SHIFT                                  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK__SHIFT                                    0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID__SHIFT                                  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK__SHIFT                                      0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID__SHIFT                                    0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK_MASK                                      0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID_MASK                                    0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK_MASK                                      0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID_MASK                                    0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK_MASK                                      0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID_MASK                                    0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK_MASK                                      0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID_MASK                                    0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK_MASK                                      0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID_MASK                                    0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK_MASK                                      0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID_MASK                                    0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK_MASK                                      0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID_MASK                                    0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK_MASK                                      0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID_MASK                                    0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK_MASK                                      0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID_MASK                                    0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK_MASK                                      0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID_MASK                                    0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK_MASK                                      0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID_MASK                                    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK_MASK                                      0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID_MASK                                    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK_MASK                                      0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID_MASK                                    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK_MASK                                      0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID_MASK                                    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK_MASK                                      0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID_MASK                                    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK_MASK                                        0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID_MASK                                      0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+//BIF_CFG_DEV0_EPF0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                  0x00000002L
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                       0x0000FE00L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                 0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT__RESERVED_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                             0x00000004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                             0x00000008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                               0x00000010L
+//BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                             0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                             0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                             0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                               0x00000200L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                               0x00000400L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                0x0000F800L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                               0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                     0x00000700L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                      0xa
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                             0x00000004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                             0x00000008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                               0x00000010L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                         0x00000020L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                        0x00000400L
+//BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                           0x3
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                             0x0000FFF8L
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                           0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT            0x18
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                             0x00FFFFFFL
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK              0x03000000L
+//BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                          0x0000FFF8L
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT         0x18
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                          0x00FFFFFFL
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK           0x03000000L
+//BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_AP_CAP
+#define BIF_CFG_DEV0_EPF0_AP_CAP__COUNT__SHIFT                                                                0x0
+#define BIF_CFG_DEV0_EPF0_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_AP_CAP__COUNT_MASK                                                                  0x000000FFL
+#define BIF_CFG_DEV0_EPF0_AP_CAP__SEL_EN_SUPPORTED_MASK                                                       0x00000100L
+//BIF_CFG_DEV0_EPF0_AP_CNTL
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__INX_SEL__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__INX_SEL_MASK                                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                        0x00000100L
+//BIF_CFG_DEV0_EPF0_AP_DATA1
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__USAGE_INFOR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__DETAILS__SHIFT                                                            0x5
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__VENDORID__SHIFT                                                           0x10
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__USAGE_INFOR_MASK                                                          0x00000007L
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__DETAILS_MASK                                                              0x0000FFE0L
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__VENDORID_MASK                                                             0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_AP_DATA2
+#define BIF_CFG_DEV0_EPF0_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                   0x00FFFFFFL
+//BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__PCIE__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__OTHERS__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__PCIE_MASK                                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__OTHERS_MASK                                                         0xFFFFFFFEL
+//BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                    0x0001L
+//BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                0x0002L
+//BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                             0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                               0x00000070L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                               0x00000C00L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                               0x000EL
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                       0x0001L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x003F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                   0x1f
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                     0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                 0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                           0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+//BIF_CFG_DEV0_EPF1_VENDOR_ID
+#define BIF_CFG_DEV0_EPF1_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_DEVICE_ID
+#define BIF_CFG_DEV0_EPF1_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_COMMAND
+#define BIF_CFG_DEV0_EPF1_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF1_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF1_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF1_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF1_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF1_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF1_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF1_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF1_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF1_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF1_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF1_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF1_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF1_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF1_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF1_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF1_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF1_STATUS
+#define BIF_CFG_DEV0_EPF1_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF1_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF1_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF1_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF1_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF1_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF1_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF1_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF1_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF1_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF1_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF1_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF1_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF1_REVISION_ID
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF1_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF1_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_SUB_CLASS
+#define BIF_CFG_DEV0_EPF1_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_BASE_CLASS
+#define BIF_CFG_DEV0_EPF1_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_CACHE_LINE
+#define BIF_CFG_DEV0_EPF1_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF1_LATENCY
+#define BIF_CFG_DEV0_EPF1_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_HEADER
+#define BIF_CFG_DEV0_EPF1_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF1_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF1_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF1_BIST
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF1_CAP_PTR
+#define BIF_CFG_DEV0_EPF1_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF1_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF1_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF1_MIN_GRANT
+#define BIF_CFG_DEV0_EPF1_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF1_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF1_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF1_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF1_PMI_CAP
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF1_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF1_LINK_CAP
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF1_LINK_CNTL
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF1_LINK_STATUS
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_LINK_CAP2
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF1_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF1_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF1_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MASK
+#define BIF_CFG_DEV0_EPF1_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF1_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_PENDING
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF1_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_MSIX_PBA
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_RTR_DATA1
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF1_RTR_DATA2
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf2_bifcfgdecp
+//BIF_CFG_DEV0_EPF2_VENDOR_ID
+#define BIF_CFG_DEV0_EPF2_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF2_DEVICE_ID
+#define BIF_CFG_DEV0_EPF2_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF2_COMMAND
+#define BIF_CFG_DEV0_EPF2_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF2_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF2_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF2_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF2_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF2_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF2_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF2_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF2_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF2_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF2_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF2_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF2_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF2_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF2_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF2_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF2_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF2_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF2_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF2_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF2_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF2_STATUS
+#define BIF_CFG_DEV0_EPF2_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF2_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF2_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF2_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF2_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF2_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF2_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF2_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF2_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF2_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF2_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF2_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF2_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF2_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF2_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF2_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF2_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF2_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF2_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF2_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF2_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF2_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF2_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF2_REVISION_ID
+#define BIF_CFG_DEV0_EPF2_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF2_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF2_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF2_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF2_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF2_SUB_CLASS
+#define BIF_CFG_DEV0_EPF2_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF2_BASE_CLASS
+#define BIF_CFG_DEV0_EPF2_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF2_CACHE_LINE
+#define BIF_CFG_DEV0_EPF2_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF2_LATENCY
+#define BIF_CFG_DEV0_EPF2_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF2_HEADER
+#define BIF_CFG_DEV0_EPF2_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF2_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF2_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF2_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF2_BIST
+#define BIF_CFG_DEV0_EPF2_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF2_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF2_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF2_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF2_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF2_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF2_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF2_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF2_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF2_CAP_PTR
+#define BIF_CFG_DEV0_EPF2_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF2_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF2_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF2_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF2_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF2_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF2_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF2_MIN_GRANT
+#define BIF_CFG_DEV0_EPF2_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF2_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF2_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF2_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF2_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF2_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF2_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF2_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF2_PMI_CAP
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF2_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF2_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF2_SBRN
+#define BIF_CFG_DEV0_EPF2_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV0_EPF2_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV0_EPF2_FLADJ
+#define BIF_CFG_DEV0_EPF2_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF2_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV0_EPF2_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV0_EPF2_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV0_EPF2_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF2_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF2_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF2_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF2_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV0_EPF2_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF2_PCIE_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF2_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF2_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF2_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF2_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF2_LINK_CAP
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF2_LINK_CNTL
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF2_LINK_STATUS
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF2_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF2_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF2_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF2_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF2_LINK_CAP2
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF2_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF2_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF2_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF2_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF2_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF2_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF2_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF2_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_MASK
+#define BIF_CFG_DEV0_EPF2_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF2_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF2_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF2_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF2_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_PENDING
+#define BIF_CFG_DEV0_EPF2_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF2_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF2_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF2_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF2_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF2_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF2_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF2_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF2_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF2_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF2_MSIX_PBA
+#define BIF_CFG_DEV0_EPF2_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF2_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF2_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF2_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF2_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF2_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF2_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF2_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF2_RTR_DATA1
+#define BIF_CFG_DEV0_EPF2_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF2_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF2_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF2_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF2_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF2_RTR_DATA2
+#define BIF_CFG_DEV0_EPF2_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF2_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF2_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF2_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF2_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF2_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF2_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf3_bifcfgdecp
+//BIF_CFG_DEV0_EPF3_VENDOR_ID
+#define BIF_CFG_DEV0_EPF3_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF3_DEVICE_ID
+#define BIF_CFG_DEV0_EPF3_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF3_COMMAND
+#define BIF_CFG_DEV0_EPF3_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF3_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF3_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF3_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF3_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF3_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF3_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF3_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF3_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF3_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF3_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF3_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF3_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF3_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF3_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF3_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF3_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF3_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF3_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF3_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF3_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF3_STATUS
+#define BIF_CFG_DEV0_EPF3_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF3_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF3_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF3_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF3_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF3_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF3_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF3_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF3_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF3_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF3_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF3_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF3_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF3_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF3_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF3_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF3_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF3_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF3_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF3_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF3_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF3_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF3_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF3_REVISION_ID
+#define BIF_CFG_DEV0_EPF3_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF3_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF3_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF3_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF3_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF3_SUB_CLASS
+#define BIF_CFG_DEV0_EPF3_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF3_BASE_CLASS
+#define BIF_CFG_DEV0_EPF3_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF3_CACHE_LINE
+#define BIF_CFG_DEV0_EPF3_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF3_LATENCY
+#define BIF_CFG_DEV0_EPF3_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF3_HEADER
+#define BIF_CFG_DEV0_EPF3_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF3_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF3_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF3_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF3_BIST
+#define BIF_CFG_DEV0_EPF3_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF3_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF3_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF3_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF3_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF3_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF3_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF3_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF3_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF3_CAP_PTR
+#define BIF_CFG_DEV0_EPF3_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF3_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF3_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF3_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF3_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF3_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF3_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF3_MIN_GRANT
+#define BIF_CFG_DEV0_EPF3_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF3_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF3_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF3_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF3_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF3_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF3_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF3_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF3_PMI_CAP
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF3_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF3_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF3_SBRN
+#define BIF_CFG_DEV0_EPF3_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV0_EPF3_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV0_EPF3_FLADJ
+#define BIF_CFG_DEV0_EPF3_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF3_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV0_EPF3_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV0_EPF3_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV0_EPF3_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF3_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF3_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF3_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF3_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV0_EPF3_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF3_PCIE_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF3_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF3_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF3_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF3_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF3_LINK_CAP
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF3_LINK_CNTL
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF3_LINK_STATUS
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF3_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF3_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF3_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF3_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF3_LINK_CAP2
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF3_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF3_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF3_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF3_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF3_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF3_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF3_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF3_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_MASK
+#define BIF_CFG_DEV0_EPF3_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF3_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF3_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF3_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF3_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_PENDING
+#define BIF_CFG_DEV0_EPF3_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF3_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF3_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF3_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF3_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF3_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF3_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF3_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF3_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF3_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF3_MSIX_PBA
+#define BIF_CFG_DEV0_EPF3_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF3_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF3_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF3_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF3_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF3_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF3_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF3_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF3_RTR_DATA1
+#define BIF_CFG_DEV0_EPF3_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF3_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF3_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF3_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF3_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF3_RTR_DATA2
+#define BIF_CFG_DEV0_EPF3_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF3_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF3_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF3_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF3_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF3_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF3_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf5_bifcfgdecp
+//BIF_CFG_DEV0_EPF5_VENDOR_ID
+#define BIF_CFG_DEV0_EPF5_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF5_DEVICE_ID
+#define BIF_CFG_DEV0_EPF5_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF5_COMMAND
+#define BIF_CFG_DEV0_EPF5_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF5_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF5_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF5_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF5_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF5_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF5_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF5_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF5_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF5_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF5_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF5_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF5_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF5_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF5_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF5_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF5_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF5_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF5_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF5_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF5_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF5_STATUS
+#define BIF_CFG_DEV0_EPF5_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF5_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF5_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF5_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF5_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF5_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF5_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF5_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF5_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF5_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF5_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF5_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF5_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF5_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF5_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF5_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF5_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF5_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF5_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF5_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF5_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF5_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF5_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF5_REVISION_ID
+#define BIF_CFG_DEV0_EPF5_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF5_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF5_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF5_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF5_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF5_SUB_CLASS
+#define BIF_CFG_DEV0_EPF5_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF5_BASE_CLASS
+#define BIF_CFG_DEV0_EPF5_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF5_CACHE_LINE
+#define BIF_CFG_DEV0_EPF5_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF5_LATENCY
+#define BIF_CFG_DEV0_EPF5_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF5_HEADER
+#define BIF_CFG_DEV0_EPF5_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF5_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF5_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF5_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF5_BIST
+#define BIF_CFG_DEV0_EPF5_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF5_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF5_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF5_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF5_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF5_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF5_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF5_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF5_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF5_CAP_PTR
+#define BIF_CFG_DEV0_EPF5_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF5_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF5_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF5_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF5_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF5_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF5_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF5_MIN_GRANT
+#define BIF_CFG_DEV0_EPF5_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF5_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF5_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF5_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF5_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF5_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF5_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF5_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF5_PMI_CAP
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF5_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF5_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF5_SBRN
+#define BIF_CFG_DEV0_EPF5_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV0_EPF5_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV0_EPF5_FLADJ
+#define BIF_CFG_DEV0_EPF5_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF5_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV0_EPF5_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV0_EPF5_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV0_EPF5_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF5_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF5_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF5_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF5_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV0_EPF5_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF5_PCIE_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF5_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF5_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF5_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF5_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF5_LINK_CAP
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF5_LINK_CNTL
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF5_LINK_STATUS
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF5_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF5_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF5_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF5_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF5_LINK_CAP2
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF5_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF5_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF5_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF5_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF5_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF5_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF5_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF5_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_MASK
+#define BIF_CFG_DEV0_EPF5_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF5_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF5_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF5_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF5_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_PENDING
+#define BIF_CFG_DEV0_EPF5_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF5_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF5_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF5_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF5_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF5_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF5_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF5_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF5_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF5_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF5_MSIX_PBA
+#define BIF_CFG_DEV0_EPF5_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF5_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF5_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF5_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF5_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF5_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF5_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF5_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF5_RTR_DATA1
+#define BIF_CFG_DEV0_EPF5_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF5_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF5_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF5_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF5_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF5_RTR_DATA2
+#define BIF_CFG_DEV0_EPF5_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF5_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF5_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF5_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF5_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF5_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF5_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf6_bifcfgdecp
+//BIF_CFG_DEV0_EPF6_VENDOR_ID
+#define BIF_CFG_DEV0_EPF6_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF6_DEVICE_ID
+#define BIF_CFG_DEV0_EPF6_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF6_COMMAND
+#define BIF_CFG_DEV0_EPF6_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF6_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF6_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF6_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF6_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF6_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF6_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF6_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF6_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF6_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF6_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF6_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF6_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF6_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF6_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF6_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF6_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF6_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF6_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF6_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF6_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF6_STATUS
+#define BIF_CFG_DEV0_EPF6_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF6_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF6_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF6_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF6_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF6_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF6_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF6_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF6_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF6_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF6_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF6_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF6_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF6_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF6_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF6_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF6_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF6_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF6_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF6_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF6_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF6_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF6_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF6_REVISION_ID
+#define BIF_CFG_DEV0_EPF6_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF6_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF6_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF6_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF6_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF6_SUB_CLASS
+#define BIF_CFG_DEV0_EPF6_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF6_BASE_CLASS
+#define BIF_CFG_DEV0_EPF6_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF6_CACHE_LINE
+#define BIF_CFG_DEV0_EPF6_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF6_LATENCY
+#define BIF_CFG_DEV0_EPF6_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF6_HEADER
+#define BIF_CFG_DEV0_EPF6_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF6_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF6_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF6_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF6_BIST
+#define BIF_CFG_DEV0_EPF6_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF6_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF6_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF6_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF6_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF6_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF6_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF6_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF6_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF6_CAP_PTR
+#define BIF_CFG_DEV0_EPF6_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF6_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF6_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF6_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF6_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF6_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF6_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF6_MIN_GRANT
+#define BIF_CFG_DEV0_EPF6_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF6_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF6_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF6_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF6_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF6_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF6_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF6_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF6_PMI_CAP
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF6_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF6_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF6_SBRN
+#define BIF_CFG_DEV0_EPF6_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV0_EPF6_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV0_EPF6_FLADJ
+#define BIF_CFG_DEV0_EPF6_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF6_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV0_EPF6_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV0_EPF6_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV0_EPF6_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF6_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF6_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF6_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF6_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV0_EPF6_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF6_PCIE_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF6_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF6_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF6_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF6_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF6_LINK_CAP
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF6_LINK_CNTL
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF6_LINK_STATUS
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF6_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF6_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF6_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF6_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF6_LINK_CAP2
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF6_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF6_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF6_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF6_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF6_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF6_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF6_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF6_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_MASK
+#define BIF_CFG_DEV0_EPF6_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF6_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF6_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF6_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF6_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_PENDING
+#define BIF_CFG_DEV0_EPF6_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF6_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF6_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF6_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF6_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF6_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF6_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF6_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF6_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF6_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF6_MSIX_PBA
+#define BIF_CFG_DEV0_EPF6_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF6_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF6_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF6_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF6_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF6_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF6_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF6_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF6_RTR_DATA1
+#define BIF_CFG_DEV0_EPF6_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF6_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF6_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF6_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF6_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF6_RTR_DATA2
+#define BIF_CFG_DEV0_EPF6_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF6_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF6_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF6_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF6_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF6_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF6_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf7_bifcfgdecp
+//BIF_CFG_DEV0_EPF7_VENDOR_ID
+#define BIF_CFG_DEV0_EPF7_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF7_DEVICE_ID
+#define BIF_CFG_DEV0_EPF7_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF7_COMMAND
+#define BIF_CFG_DEV0_EPF7_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF7_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF7_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF7_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF7_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF7_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF7_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF7_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF7_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF7_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF7_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF7_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF7_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF7_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF7_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF7_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF7_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF7_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF7_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF7_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF7_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF7_STATUS
+#define BIF_CFG_DEV0_EPF7_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF7_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF7_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF7_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF7_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF7_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF7_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF7_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF7_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF7_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF7_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF7_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF7_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF7_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF7_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF7_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF7_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF7_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF7_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF7_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF7_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF7_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF7_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF7_REVISION_ID
+#define BIF_CFG_DEV0_EPF7_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF7_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF7_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF7_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF7_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF7_SUB_CLASS
+#define BIF_CFG_DEV0_EPF7_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF7_BASE_CLASS
+#define BIF_CFG_DEV0_EPF7_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF7_CACHE_LINE
+#define BIF_CFG_DEV0_EPF7_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF7_LATENCY
+#define BIF_CFG_DEV0_EPF7_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF7_HEADER
+#define BIF_CFG_DEV0_EPF7_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF7_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF7_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF7_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF7_BIST
+#define BIF_CFG_DEV0_EPF7_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF7_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF7_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF7_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF7_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF7_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF7_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF7_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF7_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF7_CAP_PTR
+#define BIF_CFG_DEV0_EPF7_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF7_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF7_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF7_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF7_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF7_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF7_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF7_MIN_GRANT
+#define BIF_CFG_DEV0_EPF7_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF7_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF7_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF7_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF7_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF7_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF7_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF7_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF7_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF7_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF7_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF7_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF7_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF7_PMI_CAP
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF7_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF7_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF7_SBRN
+#define BIF_CFG_DEV0_EPF7_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV0_EPF7_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV0_EPF7_FLADJ
+#define BIF_CFG_DEV0_EPF7_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF7_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV0_EPF7_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV0_EPF7_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV0_EPF7_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF7_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF7_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF7_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF7_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV0_EPF7_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF7_PCIE_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF7_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV0_EPF7_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV0_EPF7_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF7_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF7_LINK_CAP
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF7_LINK_CNTL
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF7_LINK_STATUS
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF7_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF7_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF7_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF7_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF7_LINK_CAP2
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF7_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF7_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF7_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF7_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF7_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF7_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF7_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF7_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF7_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF7_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF7_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_MASK
+#define BIF_CFG_DEV0_EPF7_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF7_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF7_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF7_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF7_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF7_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_PENDING
+#define BIF_CFG_DEV0_EPF7_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF7_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF7_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF7_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF7_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF7_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF7_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF7_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF7_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF7_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF7_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF7_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF7_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF7_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF7_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF7_MSIX_PBA
+#define BIF_CFG_DEV0_EPF7_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF7_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF7_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF7_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF7_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF7_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF7_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF7_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF7_RTR_DATA1
+#define BIF_CFG_DEV0_EPF7_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF7_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF7_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF7_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF7_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF7_RTR_DATA2
+#define BIF_CFG_DEV0_EPF7_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF7_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF7_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF7_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF7_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF7_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF7_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF7_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF7_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF7_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf0_bifcfgdecp
+//BIF_CFG_DEV2_EPF0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF0_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV2_EPF0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF0_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV2_EPF0_COMMAND
+#define BIF_CFG_DEV2_EPF0_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV2_EPF0_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV2_EPF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV2_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV2_EPF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV2_EPF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV2_EPF0_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV2_EPF0_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV2_EPF0_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV2_EPF0_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV2_EPF0_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV2_EPF0_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV2_EPF0_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV2_EPF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV2_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV2_EPF0_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV2_EPF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV2_EPF0_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV2_EPF0_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV2_EPF0_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV2_EPF0_STATUS
+#define BIF_CFG_DEV2_EPF0_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV2_EPF0_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV2_EPF0_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV2_EPF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF0_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV2_EPF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV2_EPF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF0_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV2_EPF0_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV2_EPF0_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV2_EPF0_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV2_EPF0_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV2_EPF0_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV2_EPF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV2_EPF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV2_EPF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV2_EPF0_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV2_EPF0_REVISION_ID
+#define BIF_CFG_DEV2_EPF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV2_EPF0_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV2_EPF0_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV2_EPF0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF0_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF0_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV2_EPF0_LATENCY
+#define BIF_CFG_DEV2_EPF0_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF0_HEADER
+#define BIF_CFG_DEV2_EPF0_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF0_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV2_EPF0_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV2_EPF0_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV2_EPF0_BIST
+#define BIF_CFG_DEV2_EPF0_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV2_EPF0_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV2_EPF0_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV2_EPF0_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV2_EPF0_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV2_EPF0_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV2_EPF0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV2_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV2_EPF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV2_EPF0_CAP_PTR
+#define BIF_CFG_DEV2_EPF0_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV2_EPF0_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV2_EPF0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV2_EPF0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF0_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF0_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF0_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV2_EPF0_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV2_EPF0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV2_EPF0_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV2_EPF0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV2_EPF0_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV2_EPF0_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV2_EPF0_PMI_CAP
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV2_EPF0_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV2_EPF0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV2_EPF0_SBRN
+#define BIF_CFG_DEV2_EPF0_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV2_EPF0_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV2_EPF0_FLADJ
+#define BIF_CFG_DEV2_EPF0_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV2_EPF0_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV2_EPF0_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV2_EPF0_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV2_EPF0_DBESL_DBESLD
+#define BIF_CFG_DEV2_EPF0_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF0_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV2_EPF0_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV2_EPF0_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV2_EPF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV2_EPF0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV2_EPF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV2_EPF0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV2_EPF0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV2_EPF0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV2_EPF0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV2_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV2_EPF0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV2_EPF0_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV2_EPF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV2_EPF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_MASK
+#define BIF_CFG_DEV2_EPF0_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF0_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF0_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV2_EPF0_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV2_EPF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV2_EPF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV2_EPF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV2_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV2_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV2_EPF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV2_EPF0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV2_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV2_EPF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV2_EPF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//DUAL_BDF_CAP_HDR
+#define DUAL_BDF_CAP_HDR__CAP_ID__SHIFT                                                                       0x0
+#define DUAL_BDF_CAP_HDR__NEXT_PTR__SHIFT                                                                     0x8
+#define DUAL_BDF_CAP_HDR__CAP_LENGTH__SHIFT                                                                   0x10
+#define DUAL_BDF_CAP_HDR__RSVD__SHIFT                                                                         0x18
+#define DUAL_BDF_CAP_HDR__CAP_ID_MASK                                                                         0x000000FFL
+#define DUAL_BDF_CAP_HDR__NEXT_PTR_MASK                                                                       0x0000FF00L
+#define DUAL_BDF_CAP_HDR__CAP_LENGTH_MASK                                                                     0x00FF0000L
+#define DUAL_BDF_CAP_HDR__RSVD_MASK                                                                           0xFF000000L
+//DUAL_BDF_CAP_DW1
+#define DUAL_BDF_CAP_DW1__DVSEC_VEN_ID__SHIFT                                                                 0x0
+#define DUAL_BDF_CAP_DW1__DVSEC_REVISION__SHIFT                                                               0x10
+#define DUAL_BDF_CAP_DW1__DVSEC_LENGTH__SHIFT                                                                 0x14
+#define DUAL_BDF_CAP_DW1__DVSEC_VEN_ID_MASK                                                                   0x0000FFFFL
+#define DUAL_BDF_CAP_DW1__DVSEC_REVISION_MASK                                                                 0x000F0000L
+#define DUAL_BDF_CAP_DW1__DVSEC_LENGTH_MASK                                                                   0xFFF00000L
+//DUAL_BDF_CAP_DW2
+#define DUAL_BDF_CAP_DW2__DVSEC_ID__SHIFT                                                                     0x0
+#define DUAL_BDF_CAP_DW2__ALTER_FUNC_NUM_VEC__SHIFT                                                           0x10
+#define DUAL_BDF_CAP_DW2__PRIM_ALT_DEV_NUM__SHIFT                                                             0x18
+#define DUAL_BDF_CAP_DW2__RSVD__SHIFT                                                                         0x1d
+#define DUAL_BDF_CAP_DW2__DVSEC_ID_MASK                                                                       0x0000FFFFL
+#define DUAL_BDF_CAP_DW2__ALTER_FUNC_NUM_VEC_MASK                                                             0x00FF0000L
+#define DUAL_BDF_CAP_DW2__PRIM_ALT_DEV_NUM_MASK                                                               0x1F000000L
+#define DUAL_BDF_CAP_DW2__RSVD_MASK                                                                           0xE0000000L
+//BIF_CFG_DEV2_EPF0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV2_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV2_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV2_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV2_EPF0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_EPF0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF0_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV2_EPF0_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV2_EPF0_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF0_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV2_EPF0_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV2_EPF0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF0_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV2_EPF0_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV2_EPF0_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV2_EPF0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                        0xa
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                       0x1a
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                          0x000003FFL
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                          0x00001C00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                         0x03FF0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                         0x1C000000L
+//BIF_CFG_DEV2_EPF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV2_EPF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV2_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV2_EPF0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV2_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf0_bifcfgpciedecp
+//BIF_CFG_DEV2_EPF0_LINK_CAP
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV2_EPF0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV2_EPF0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV2_EPF0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV2_EPF0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV2_EPF0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV2_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                          0x00000001L
+#define BIF_CFG_DEV2_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                  0x00000002L
+#define BIF_CFG_DEV2_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                       0x0000FE00L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                               0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                 0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV2_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV2_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV2_EPF0_LINK_CAP_16GT
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_LINK_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_16GT__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_16GT__RESERVED_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                 0x0
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                           0x1
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                           0x2
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                             0x4
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                             0x00000002L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                             0x00000004L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                             0x00000008L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                               0x00000010L
+//BIF_CFG_DEV2_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV2_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV2_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV2_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV2_EPF0_LINK_CAP_32GT
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                        0x1
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                             0x8
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                             0x9
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                             0xa
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                             0x00000001L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                          0x00000002L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                               0x00000100L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                               0x00000200L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                               0x00000400L
+#define BIF_CFG_DEV2_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                0x0000F800L
+//BIF_CFG_DEV2_EPF0_LINK_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                0x0
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                             0x1
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                  0x00000001L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                               0x00000002L
+#define BIF_CFG_DEV2_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                     0x00000700L
+//BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                 0x0
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                           0x1
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                           0x2
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                             0x4
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                       0x6
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                0x9
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                      0xa
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                             0x00000002L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                             0x00000004L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                             0x00000008L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                               0x00000010L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                         0x00000020L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                         0x000000C0L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                     0x00000100L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                  0x00000200L
+#define BIF_CFG_DEV2_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                        0x00000400L
+//BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                           0x3
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                         0x00000007L
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                             0x0000FFF8L
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                           0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT            0x18
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                             0x00FFFFFFL
+#define BIF_CFG_DEV2_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK              0x03000000L
+//BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                    0x0
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                        0x3
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                      0x00000007L
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                          0x0000FFF8L
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                        0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT         0x18
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                          0x00FFFFFFL
+#define BIF_CFG_DEV2_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK           0x03000000L
+//BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV2_EPF0_AP_CAP
+#define BIF_CFG_DEV2_EPF0_AP_CAP__COUNT__SHIFT                                                                0x0
+#define BIF_CFG_DEV2_EPF0_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_AP_CAP__COUNT_MASK                                                                  0x000000FFL
+#define BIF_CFG_DEV2_EPF0_AP_CAP__SEL_EN_SUPPORTED_MASK                                                       0x00000100L
+//BIF_CFG_DEV2_EPF0_AP_CNTL
+#define BIF_CFG_DEV2_EPF0_AP_CNTL__INX_SEL__SHIFT                                                             0x0
+#define BIF_CFG_DEV2_EPF0_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV2_EPF0_AP_CNTL__INX_SEL_MASK                                                               0x000000FFL
+#define BIF_CFG_DEV2_EPF0_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                        0x00000100L
+//BIF_CFG_DEV2_EPF0_AP_DATA1
+#define BIF_CFG_DEV2_EPF0_AP_DATA1__USAGE_INFOR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_AP_DATA1__DETAILS__SHIFT                                                            0x5
+#define BIF_CFG_DEV2_EPF0_AP_DATA1__VENDORID__SHIFT                                                           0x10
+#define BIF_CFG_DEV2_EPF0_AP_DATA1__USAGE_INFOR_MASK                                                          0x00000007L
+#define BIF_CFG_DEV2_EPF0_AP_DATA1__DETAILS_MASK                                                              0x0000FFE0L
+#define BIF_CFG_DEV2_EPF0_AP_DATA1__VENDORID_MASK                                                             0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_AP_DATA2
+#define BIF_CFG_DEV2_EPF0_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                   0x00FFFFFFL
+//BIF_CFG_DEV2_EPF0_AP_SEL_EN_MASK
+#define BIF_CFG_DEV2_EPF0_AP_SEL_EN_MASK__PCIE__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_AP_SEL_EN_MASK__OTHERS__SHIFT                                                       0x1
+#define BIF_CFG_DEV2_EPF0_AP_SEL_EN_MASK__PCIE_MASK                                                           0x00000001L
+#define BIF_CFG_DEV2_EPF0_AP_SEL_EN_MASK__OTHERS_MASK                                                         0xFFFFFFFEL
+//BIF_CFG_DEV2_EPF0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV2_EPF0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV2_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                    0x0001L
+//BIF_CFG_DEV2_EPF0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV2_EPF0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                              0x1
+#define BIF_CFG_DEV2_EPF0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                         0x0001L
+#define BIF_CFG_DEV2_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                0x0002L
+//BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                             0x4
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                             0xa
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                            0x00000007L
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                               0x00000070L
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                               0x00000C00L
+//BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                             0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                               0x000EL
+//BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                       0x0001L
+//BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x003F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV2_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV2_EPF0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV2_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV2_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV2_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                   0x1f
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                     0x007FFFFFL
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                     0x80000000L
+//BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                               0x0
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                         0x1f
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                 0x007FFFFFL
+#define BIF_CFG_DEV2_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                           0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf1_bifcfgdecp
+//BIF_CFG_DEV1_EPF1_VENDOR_ID
+#define BIF_CFG_DEV1_EPF1_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV1_EPF1_DEVICE_ID
+#define BIF_CFG_DEV1_EPF1_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV1_EPF1_COMMAND
+#define BIF_CFG_DEV1_EPF1_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV1_EPF1_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV1_EPF1_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV1_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV1_EPF1_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV1_EPF1_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV1_EPF1_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV1_EPF1_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV1_EPF1_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV1_EPF1_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV1_EPF1_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV1_EPF1_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV1_EPF1_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV1_EPF1_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV1_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV1_EPF1_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV1_EPF1_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV1_EPF1_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV1_EPF1_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV1_EPF1_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV1_EPF1_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV1_EPF1_STATUS
+#define BIF_CFG_DEV1_EPF1_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV1_EPF1_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV1_EPF1_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV1_EPF1_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV1_EPF1_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF1_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV1_EPF1_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV1_EPF1_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV1_EPF1_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV1_EPF1_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV1_EPF1_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV1_EPF1_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV1_EPF1_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV1_EPF1_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV1_EPF1_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV1_EPF1_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV1_EPF1_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV1_EPF1_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV1_EPF1_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV1_EPF1_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV1_EPF1_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV1_EPF1_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV1_EPF1_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV1_EPF1_REVISION_ID
+#define BIF_CFG_DEV1_EPF1_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV1_EPF1_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV1_EPF1_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV1_EPF1_PROG_INTERFACE
+#define BIF_CFG_DEV1_EPF1_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV1_EPF1_SUB_CLASS
+#define BIF_CFG_DEV1_EPF1_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV1_EPF1_BASE_CLASS
+#define BIF_CFG_DEV1_EPF1_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF1_CACHE_LINE
+#define BIF_CFG_DEV1_EPF1_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV1_EPF1_LATENCY
+#define BIF_CFG_DEV1_EPF1_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF1_HEADER
+#define BIF_CFG_DEV1_EPF1_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF1_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV1_EPF1_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV1_EPF1_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV1_EPF1_BIST
+#define BIF_CFG_DEV1_EPF1_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV1_EPF1_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV1_EPF1_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV1_EPF1_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV1_EPF1_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV1_EPF1_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV1_EPF1_BASE_ADDR_1
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_BASE_ADDR_2
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_BASE_ADDR_3
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_BASE_ADDR_4
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_BASE_ADDR_5
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_BASE_ADDR_6
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV1_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_ADAPTER_ID
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV1_EPF1_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV1_EPF1_CAP_PTR
+#define BIF_CFG_DEV1_EPF1_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV1_EPF1_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV1_EPF1_INTERRUPT_LINE
+#define BIF_CFG_DEV1_EPF1_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV1_EPF1_INTERRUPT_PIN
+#define BIF_CFG_DEV1_EPF1_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF1_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV1_EPF1_MIN_GRANT
+#define BIF_CFG_DEV1_EPF1_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF1_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV1_EPF1_MAX_LATENCY
+#define BIF_CFG_DEV1_EPF1_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV1_EPF1_VENDOR_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF1_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF1_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV1_EPF1_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV1_EPF1_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV1_EPF1_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV1_EPF1_ADAPTER_ID_W
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PMI_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_EPF1_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV1_EPF1_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV1_EPF1_PMI_CAP
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV1_EPF1_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV1_EPF1_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV1_EPF1_SBRN
+#define BIF_CFG_DEV1_EPF1_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV1_EPF1_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV1_EPF1_FLADJ
+#define BIF_CFG_DEV1_EPF1_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV1_EPF1_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV1_EPF1_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV1_EPF1_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV1_EPF1_DBESL_DBESLD
+#define BIF_CFG_DEV1_EPF1_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF1_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV1_EPF1_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV1_EPF1_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV1_EPF1_PCIE_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV1_EPF1_PCIE_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV1_EPF1_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV1_EPF1_DEVICE_CAP
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV1_EPF1_DEVICE_CNTL
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV1_EPF1_DEVICE_STATUS
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV1_EPF1_LINK_CAP
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV1_EPF1_LINK_CNTL
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV1_EPF1_LINK_STATUS
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV1_EPF1_DEVICE_CAP2
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV1_EPF1_DEVICE_CNTL2
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV1_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV1_EPF1_DEVICE_STATUS2
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV1_EPF1_LINK_CAP2
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV1_EPF1_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV1_EPF1_LINK_CNTL2
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV1_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV1_EPF1_LINK_STATUS2
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV1_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV1_EPF1_MSI_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_EPF1_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV1_EPF1_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV1_EPF1_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV1_EPF1_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_MSG_DATA
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV1_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_MASK
+#define BIF_CFG_DEV1_EPF1_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF1_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF1_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_MASK_64
+#define BIF_CFG_DEV1_EPF1_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_PENDING
+#define BIF_CFG_DEV1_EPF1_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_MSI_PENDING_64
+#define BIF_CFG_DEV1_EPF1_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_MSIX_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV1_EPF1_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV1_EPF1_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV1_EPF1_MSIX_MSG_CNTL
+#define BIF_CFG_DEV1_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV1_EPF1_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV1_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV1_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV1_EPF1_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV1_EPF1_MSIX_TABLE
+#define BIF_CFG_DEV1_EPF1_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV1_EPF1_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV1_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF1_MSIX_PBA
+#define BIF_CFG_DEV1_EPF1_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV1_EPF1_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV1_EPF1_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV1_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV1_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV1_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG0
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG1
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG2
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG3
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV1_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV1_EPF1_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF1_RTR_DATA1
+#define BIF_CFG_DEV1_EPF1_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV1_EPF1_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV1_EPF1_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV1_EPF1_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV1_EPF1_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV1_EPF1_RTR_DATA2
+#define BIF_CFG_DEV1_EPF1_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF1_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV1_EPF1_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV1_EPF1_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_STATUS
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_ARI_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV1_EPF1_PCIE_ARI_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV1_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR1_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR2_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR3_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR4_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR5_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR6_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_PASID_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV1_EPF1_PCIE_PASID_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV1_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                            0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                     0x2
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                              0x15
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                         0x00000001L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                              0x00000002L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                       0x00000004L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                                0xFFE00000L
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                                0x1
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                           0x2
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                             0x3
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                  0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                  0x0002L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                             0x0004L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                               0x0008L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                    0x0010L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                      0x0020L
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_STATUS
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                                 0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                   0x0001L
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_INITIAL_VFS
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                      0xFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_TOTAL_VFS
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                          0xFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_NUM_VFS
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                              0xFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_FUNC_DEP_LINK
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                                0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                  0xFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                              0xFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_STRIDE
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                          0xFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_DEVICE_ID
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                    0xFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                    0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT     0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT  0x3
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK       0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK  0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV1_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf0_bifcfgdecp
+//BIF_CFG_DEV1_EPF0_VENDOR_ID
+#define BIF_CFG_DEV1_EPF0_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV1_EPF0_DEVICE_ID
+#define BIF_CFG_DEV1_EPF0_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV1_EPF0_COMMAND
+#define BIF_CFG_DEV1_EPF0_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV1_EPF0_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV1_EPF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV1_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV1_EPF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV1_EPF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV1_EPF0_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV1_EPF0_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV1_EPF0_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV1_EPF0_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV1_EPF0_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV1_EPF0_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV1_EPF0_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV1_EPF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV1_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV1_EPF0_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV1_EPF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV1_EPF0_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV1_EPF0_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV1_EPF0_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV1_EPF0_STATUS
+#define BIF_CFG_DEV1_EPF0_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV1_EPF0_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV1_EPF0_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV1_EPF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV1_EPF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF0_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV1_EPF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV1_EPF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV1_EPF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV1_EPF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV1_EPF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV1_EPF0_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV1_EPF0_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV1_EPF0_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV1_EPF0_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV1_EPF0_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV1_EPF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV1_EPF0_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV1_EPF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV1_EPF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV1_EPF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV1_EPF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV1_EPF0_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV1_EPF0_REVISION_ID
+#define BIF_CFG_DEV1_EPF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV1_EPF0_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV1_EPF0_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV1_EPF0_PROG_INTERFACE
+#define BIF_CFG_DEV1_EPF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV1_EPF0_SUB_CLASS
+#define BIF_CFG_DEV1_EPF0_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV1_EPF0_BASE_CLASS
+#define BIF_CFG_DEV1_EPF0_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF0_CACHE_LINE
+#define BIF_CFG_DEV1_EPF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV1_EPF0_LATENCY
+#define BIF_CFG_DEV1_EPF0_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF0_HEADER
+#define BIF_CFG_DEV1_EPF0_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF0_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV1_EPF0_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV1_EPF0_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV1_EPF0_BIST
+#define BIF_CFG_DEV1_EPF0_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV1_EPF0_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV1_EPF0_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV1_EPF0_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV1_EPF0_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV1_EPF0_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV1_EPF0_BASE_ADDR_1
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_BASE_ADDR_2
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_BASE_ADDR_3
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_BASE_ADDR_4
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_BASE_ADDR_5
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_BASE_ADDR_6
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV1_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_ADAPTER_ID
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV1_EPF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV1_EPF0_CAP_PTR
+#define BIF_CFG_DEV1_EPF0_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV1_EPF0_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV1_EPF0_INTERRUPT_LINE
+#define BIF_CFG_DEV1_EPF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV1_EPF0_INTERRUPT_PIN
+#define BIF_CFG_DEV1_EPF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV1_EPF0_MIN_GRANT
+#define BIF_CFG_DEV1_EPF0_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF0_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV1_EPF0_MAX_LATENCY
+#define BIF_CFG_DEV1_EPF0_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV1_EPF0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV1_EPF0_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV1_EPF0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV1_EPF0_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV1_EPF0_ADAPTER_ID_W
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_PMI_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_EPF0_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV1_EPF0_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV1_EPF0_PMI_CAP
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV1_EPF0_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV1_EPF0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV1_EPF0_SBRN
+#define BIF_CFG_DEV1_EPF0_SBRN__SBRN__SHIFT                                                                   0x0
+#define BIF_CFG_DEV1_EPF0_SBRN__SBRN_MASK                                                                     0xFFL
+//BIF_CFG_DEV1_EPF0_FLADJ
+#define BIF_CFG_DEV1_EPF0_FLADJ__FLADJ__SHIFT                                                                 0x0
+#define BIF_CFG_DEV1_EPF0_FLADJ__NFC__SHIFT                                                                   0x6
+#define BIF_CFG_DEV1_EPF0_FLADJ__FLADJ_MASK                                                                   0x3FL
+#define BIF_CFG_DEV1_EPF0_FLADJ__NFC_MASK                                                                     0x40L
+//BIF_CFG_DEV1_EPF0_DBESL_DBESLD
+#define BIF_CFG_DEV1_EPF0_DBESL_DBESLD__DBESL__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF0_DBESL_DBESLD__DBESLD__SHIFT                                                         0x4
+#define BIF_CFG_DEV1_EPF0_DBESL_DBESLD__DBESL_MASK                                                            0x0FL
+#define BIF_CFG_DEV1_EPF0_DBESL_DBESLD__DBESLD_MASK                                                           0xF0L
+//BIF_CFG_DEV1_EPF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV1_EPF0_PCIE_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV1_EPF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+//BIF_CFG_DEV1_EPF0_DEVICE_CAP
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+//BIF_CFG_DEV1_EPF0_DEVICE_CNTL
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV1_EPF0_DEVICE_STATUS
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV1_EPF0_DEVICE_CAP2
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV1_EPF0_DEVICE_CNTL2
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV1_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV1_EPF0_DEVICE_STATUS2
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_EPF0_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV1_EPF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV1_EPF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV1_EPF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_MSG_DATA
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV1_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_MASK
+#define BIF_CFG_DEV1_EPF0_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF0_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_MASK_64
+#define BIF_CFG_DEV1_EPF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_PENDING
+#define BIF_CFG_DEV1_EPF0_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_MSI_PENDING_64
+#define BIF_CFG_DEV1_EPF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV1_EPF0_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV1_EPF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV1_EPF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV1_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV1_EPF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV1_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV1_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV1_EPF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV1_EPF0_MSIX_TABLE
+#define BIF_CFG_DEV1_EPF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV1_EPF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV1_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF0_MSIX_PBA
+#define BIF_CFG_DEV1_EPF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV1_EPF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV1_EPF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+//BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+//BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+//BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV1_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV1_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+//BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV1_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV1_EPF0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF0_RTR_DATA1
+#define BIF_CFG_DEV1_EPF0_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV1_EPF0_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV1_EPF0_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV1_EPF0_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV1_EPF0_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV1_EPF0_RTR_DATA2
+#define BIF_CFG_DEV1_EPF0_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV1_EPF0_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV1_EPF0_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+//BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV1_EPF0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                        0xa
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                       0x1a
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                          0x000003FFL
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                          0x00001C00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                         0x03FF0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                         0x1C000000L
+//BIF_CFG_DEV1_EPF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV1_EPF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV1_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_PASID_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV1_EPF0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV1_EPF0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf0_bifcfgpciedecp
+//BIF_CFG_DEV1_EPF0_LINK_CAP
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV1_EPF0_LINK_CNTL
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV1_EPF0_LINK_STATUS
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV1_EPF0_LINK_CAP2
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV1_EPF0_LINK_CNTL2
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV1_EPF0_LINK_STATUS2
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV1_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                     0x9
+#define BIF_CFG_DEV1_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                          0x00000001L
+#define BIF_CFG_DEV1_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                  0x00000002L
+#define BIF_CFG_DEV1_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                       0x0000FE00L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                               0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                 0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV1_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV1_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV1_EPF0_LINK_CAP_16GT
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_LINK_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_16GT__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_16GT__RESERVED_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                 0x0
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                           0x1
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                           0x2
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                             0x00000002L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                             0x00000004L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                             0x00000008L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                               0x00000010L
+//BIF_CFG_DEV1_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV1_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV1_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV1_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV1_EPF0_LINK_CAP_32GT
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                        0x1
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                             0x8
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                             0x9
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                             0xa
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                              0xb
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                             0x00000001L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                          0x00000002L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                               0x00000100L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                               0x00000200L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                               0x00000400L
+#define BIF_CFG_DEV1_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                0x0000F800L
+//BIF_CFG_DEV1_EPF0_LINK_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                0x0
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                             0x1
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                   0x8
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                  0x00000001L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                               0x00000002L
+#define BIF_CFG_DEV1_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                     0x00000700L
+//BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                 0x0
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                           0x1
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                           0x2
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                       0x5
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                       0x6
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                   0x8
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                0x9
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                      0xa
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                             0x00000002L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                             0x00000004L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                             0x00000008L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                               0x00000010L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                         0x00000020L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                         0x000000C0L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                     0x00000100L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                  0x00000200L
+#define BIF_CFG_DEV1_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                        0x00000400L
+//BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                           0x3
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                         0x00000007L
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                             0x0000FFF8L
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                           0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT            0x18
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                             0x00FFFFFFL
+#define BIF_CFG_DEV1_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK              0x03000000L
+//BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                        0x3
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                      0x10
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                      0x00000007L
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                          0x0000FFF8L
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                        0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT         0x18
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                          0x00FFFFFFL
+#define BIF_CFG_DEV1_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK           0x03000000L
+//BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV1_EPF0_AP_CAP
+#define BIF_CFG_DEV1_EPF0_AP_CAP__COUNT__SHIFT                                                                0x0
+#define BIF_CFG_DEV1_EPF0_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_AP_CAP__COUNT_MASK                                                                  0x000000FFL
+#define BIF_CFG_DEV1_EPF0_AP_CAP__SEL_EN_SUPPORTED_MASK                                                       0x00000100L
+//BIF_CFG_DEV1_EPF0_AP_CNTL
+#define BIF_CFG_DEV1_EPF0_AP_CNTL__INX_SEL__SHIFT                                                             0x0
+#define BIF_CFG_DEV1_EPF0_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV1_EPF0_AP_CNTL__INX_SEL_MASK                                                               0x000000FFL
+#define BIF_CFG_DEV1_EPF0_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                        0x00000100L
+//BIF_CFG_DEV1_EPF0_AP_DATA1
+#define BIF_CFG_DEV1_EPF0_AP_DATA1__USAGE_INFOR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_AP_DATA1__DETAILS__SHIFT                                                            0x5
+#define BIF_CFG_DEV1_EPF0_AP_DATA1__VENDORID__SHIFT                                                           0x10
+#define BIF_CFG_DEV1_EPF0_AP_DATA1__USAGE_INFOR_MASK                                                          0x00000007L
+#define BIF_CFG_DEV1_EPF0_AP_DATA1__DETAILS_MASK                                                              0x0000FFE0L
+#define BIF_CFG_DEV1_EPF0_AP_DATA1__VENDORID_MASK                                                             0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_AP_DATA2
+#define BIF_CFG_DEV1_EPF0_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                   0x00FFFFFFL
+//BIF_CFG_DEV1_EPF0_AP_SEL_EN_MASK
+#define BIF_CFG_DEV1_EPF0_AP_SEL_EN_MASK__PCIE__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_AP_SEL_EN_MASK__OTHERS__SHIFT                                                       0x1
+#define BIF_CFG_DEV1_EPF0_AP_SEL_EN_MASK__PCIE_MASK                                                           0x00000001L
+#define BIF_CFG_DEV1_EPF0_AP_SEL_EN_MASK__OTHERS_MASK                                                         0xFFFFFFFEL
+//BIF_CFG_DEV1_EPF0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV1_EPF0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV1_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                  0x0
+#define BIF_CFG_DEV1_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                    0x0001L
+//BIF_CFG_DEV1_EPF0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV1_EPF0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                              0x1
+#define BIF_CFG_DEV1_EPF0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                         0x0001L
+#define BIF_CFG_DEV1_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                0x0002L
+//BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                               0x8
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                             0xa
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                            0x00000007L
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                               0x00000070L
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                 0x00000300L
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                               0x00000C00L
+//BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                             0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                           0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                               0x000EL
+//BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                       0x0001L
+//BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x007F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x003F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV1_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV1_EPF0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV1_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV1_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV1_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                   0x1f
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                     0x007FFFFFL
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                     0x80000000L
+//BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                               0x0
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                         0x1f
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                 0x007FFFFFL
+#define BIF_CFG_DEV1_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                           0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_rc_bifcfgdecp
+//BIF_CFG_DEV0_RC0_VENDOR_ID
+#define BIF_CFG_DEV0_RC0_VENDOR_ID__VENDOR_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_VENDOR_ID__VENDOR_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_DEVICE_ID
+#define BIF_CFG_DEV0_RC0_DEVICE_ID__DEVICE_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_ID__DEVICE_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_COMMAND
+#define BIF_CFG_DEV0_RC0_COMMAND__IOEN_DN__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_COMMAND__MEMEN_DN__SHIFT                                                             0x1
+#define BIF_CFG_DEV0_RC0_COMMAND__BUS_MASTER_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV0_RC0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_RC0_COMMAND__PAL_SNOOP_EN__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_RC0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC0_COMMAND__AD_STEPPING__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_RC0_COMMAND__SERR_EN__SHIFT                                                              0x8
+#define BIF_CFG_DEV0_RC0_COMMAND__FAST_B2B_EN__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_RC0_COMMAND__INT_DIS__SHIFT                                                              0xa
+#define BIF_CFG_DEV0_RC0_COMMAND__IOEN_DN_MASK                                                                0x0001L
+#define BIF_CFG_DEV0_RC0_COMMAND__MEMEN_DN_MASK                                                               0x0002L
+#define BIF_CFG_DEV0_RC0_COMMAND__BUS_MASTER_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV0_RC0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_RC0_COMMAND__PAL_SNOOP_EN_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_RC0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_RC0_COMMAND__AD_STEPPING_MASK                                                            0x0080L
+#define BIF_CFG_DEV0_RC0_COMMAND__SERR_EN_MASK                                                                0x0100L
+#define BIF_CFG_DEV0_RC0_COMMAND__FAST_B2B_EN_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_RC0_COMMAND__INT_DIS_MASK                                                                0x0400L
+//BIF_CFG_DEV0_RC0_STATUS
+#define BIF_CFG_DEV0_RC0_STATUS__IMMEDIATE_READINESS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_STATUS__INT_STATUS__SHIFT                                                            0x3
+#define BIF_CFG_DEV0_RC0_STATUS__CAP_LIST__SHIFT                                                              0x4
+#define BIF_CFG_DEV0_RC0_STATUS__PCI_66_CAP__SHIFT                                                            0x5
+#define BIF_CFG_DEV0_RC0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_RC0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                              0x8
+#define BIF_CFG_DEV0_RC0_STATUS__DEVSEL_TIMING__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_RC0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_RC0_STATUS__IMMEDIATE_READINESS_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_RC0_STATUS__INT_STATUS_MASK                                                              0x0008L
+#define BIF_CFG_DEV0_RC0_STATUS__CAP_LIST_MASK                                                                0x0010L
+#define BIF_CFG_DEV0_RC0_STATUS__PCI_66_CAP_MASK                                                              0x0020L
+#define BIF_CFG_DEV0_RC0_STATUS__FAST_BACK_CAPABLE_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_RC0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                0x0100L
+#define BIF_CFG_DEV0_RC0_STATUS__DEVSEL_TIMING_MASK                                                           0x0600L
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                   0x4000L
+#define BIF_CFG_DEV0_RC0_STATUS__PARITY_ERROR_DETECTED_MASK                                                   0x8000L
+//BIF_CFG_DEV0_RC0_REVISION_ID
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MINOR_REV_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MINOR_REV_ID_MASK                                                       0x0FL
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MAJOR_REV_ID_MASK                                                       0xF0L
+//BIF_CFG_DEV0_RC0_PROG_INTERFACE
+#define BIF_CFG_DEV0_RC0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_RC0_SUB_CLASS
+#define BIF_CFG_DEV0_RC0_SUB_CLASS__SUB_CLASS__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_SUB_CLASS__SUB_CLASS_MASK                                                            0xFFL
+//BIF_CFG_DEV0_RC0_BASE_CLASS
+#define BIF_CFG_DEV0_RC0_BASE_CLASS__BASE_CLASS__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_CLASS__BASE_CLASS_MASK                                                          0xFFL
+//BIF_CFG_DEV0_RC0_CACHE_LINE
+#define BIF_CFG_DEV0_RC0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                     0xFFL
+//BIF_CFG_DEV0_RC0_LATENCY
+#define BIF_CFG_DEV0_RC0_LATENCY__LATENCY_TIMER__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_LATENCY__LATENCY_TIMER_MASK                                                          0xFFL
+//BIF_CFG_DEV0_RC0_HEADER
+#define BIF_CFG_DEV0_RC0_HEADER__HEADER_TYPE__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC0_HEADER__DEVICE_TYPE__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_RC0_HEADER__HEADER_TYPE_MASK                                                             0x7FL
+#define BIF_CFG_DEV0_RC0_HEADER__DEVICE_TYPE_MASK                                                             0x80L
+//BIF_CFG_DEV0_RC0_BIST
+#define BIF_CFG_DEV0_RC0_BIST__BIST_COMP__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC0_BIST__BIST_STRT__SHIFT                                                               0x6
+#define BIF_CFG_DEV0_RC0_BIST__BIST_CAP__SHIFT                                                                0x7
+#define BIF_CFG_DEV0_RC0_BIST__BIST_COMP_MASK                                                                 0x0FL
+#define BIF_CFG_DEV0_RC0_BIST__BIST_STRT_MASK                                                                 0x40L
+#define BIF_CFG_DEV0_RC0_BIST__BIST_CAP_MASK                                                                  0x80L
+//BIF_CFG_DEV0_RC0_BASE_ADDR_1
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_1__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_BASE_ADDR_2
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_2__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                               0x18
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                           0x0000FF00L
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                             0x00FF0000L
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_RC0_IO_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                     0x000FL
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                    0x0F00L
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_MASK                                                         0xF000L
+//BIF_CFG_DEV0_RC0_SECONDARY_STATUS
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                           0x7
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                               0x9
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                         0xb
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                       0xc
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                       0xd
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                       0xe
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PCI_66_CAP_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                             0x0080L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                      0x0100L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                 0x0600L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                           0x0800L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                         0x1000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                         0x2000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                         0x4000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                         0x8000L
+//BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                   0x0000000FL
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                  0x0000FFF0L
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                             0x0000000FL
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                            0x0000FFF0L
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PREF_BASE_UPPER
+#define BIF_CFG_DEV0_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                              0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                              0x10
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                0xFFFF0000L
+//BIF_CFG_DEV0_RC0_CAP_PTR
+#define BIF_CFG_DEV0_RC0_CAP_PTR__CAP_PTR__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_CAP_PTR__CAP_PTR_MASK                                                                0xFFL
+//BIF_CFG_DEV0_RC0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                          0x1
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                      0xb
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                       0x00000001L
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                            0x0000000EL
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                           0x000000F0L
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                        0xFFFFF800L
+//BIF_CFG_DEV0_RC0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_RC0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_RC0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_RC0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                    0xFFL
+//BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                      0x1
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                         0xa
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                    0xb
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                        0x0002L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                              0x0020L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                          0x0100L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                        0x0200L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                           0x0400L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                      0x0800L
+//BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                  0x01L
+//BIF_CFG_DEV0_RC0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC0_PMI_CAP
+#define BIF_CFG_DEV0_RC0_PMI_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_CLOCK__SHIFT                                                            0x3
+#define BIF_CFG_DEV0_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                  0x4
+#define BIF_CFG_DEV0_RC0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_PMI_CAP__AUX_CURRENT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D1_SUPPORT__SHIFT                                                           0x9
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D2_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_SUPPORT__SHIFT                                                          0xb
+#define BIF_CFG_DEV0_RC0_PMI_CAP__VERSION_MASK                                                                0x0007L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_CLOCK_MASK                                                              0x0008L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                    0x0010L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__AUX_CURRENT_MASK                                                            0x01C0L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D1_SUPPORT_MASK                                                             0x0200L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D2_SUPPORT_MASK                                                             0x0400L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_SUPPORT_MASK                                                            0xF800L
+//BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                0x16
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                   0x17
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                     0x18
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                    0x00000003L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                  0x00000008L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_EN_MASK                                                         0x00000100L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                    0x00001E00L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                     0x00006000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                     0x00008000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                  0x00400000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                     0x00800000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                       0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__VERSION_MASK                                                               0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__DEVICE_TYPE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                      0x0100L
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                       0x3E00L
+//BIF_CFG_DEV0_RC0_DEVICE_CAP
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                      0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                             0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                          0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                         0x1a
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                       0x1c
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                        0x00000018L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__EXTENDED_TAG_MASK                                                        0x00000020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                              0x000001C0L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                               0x00000E00L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                            0x00008000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                            0x00010000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                           0x03FC0000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                           0x0C000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__FLR_CAPABLE_MASK                                                         0x10000000L
+//BIF_CFG_DEV0_RC0_DEVICE_CNTL
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                  0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                      0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                            0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                              0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                   0x00E0L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                    0x0100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                    0x0200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                    0x0400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                        0x0800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                              0x7000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                0x8000L
+//BIF_CFG_DEV0_RC0_DEVICE_STATUS
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__CORR_ERR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                      0x2
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__AUX_PWR__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                              0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__CORR_ERR_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                    0x0002L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__FATAL_ERR_MASK                                                        0x0004L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__USR_DETECTED_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__AUX_PWR_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                0x0020L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                    0x0040L
+//BIF_CFG_DEV0_RC0_LINK_CAP
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_SPEED__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_WIDTH__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PM_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                              0x12
+#define BIF_CFG_DEV0_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                         0x13
+#define BIF_CFG_DEV0_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                            0x15
+#define BIF_CFG_DEV0_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                         0x16
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PORT_NUMBER__SHIFT                                                         0x18
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_SPEED_MASK                                                            0x0000000FL
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_WIDTH_MASK                                                            0x000003F0L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PM_SUPPORT_MASK                                                            0x00000C00L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                      0x00007000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                       0x00038000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                0x00040000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                           0x00080000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                           0x00100000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                              0x00200000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                           0x00400000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PORT_NUMBER_MASK                                                           0xFF000000L
+//BIF_CFG_DEV0_RC0_LINK_CNTL
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PM_CONTROL__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                       0x2
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_DIS__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                   0x6
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                          0xb
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                              0xe
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PM_CONTROL_MASK                                                           0x0003L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                         0x0004L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_DIS_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__RETRAIN_LINK_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                     0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__EXTENDED_SYNC_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                            0x0100L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                            0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                            0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                0xC000L
+//BIF_CFG_DEV0_RC0_LINK_STATUS
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                            0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_TRAINING__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__DL_ACTIVE__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                        0xf
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                 0x000FL
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                              0x03F0L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_TRAINING_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__DL_ACTIVE_MASK                                                          0x2000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                          0x4000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                          0x8000L
+//BIF_CFG_DEV0_RC0_SLOT_CAP
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                              0x3
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                               0x4
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                     0x6
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                      0x12
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                   0x13
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                   0x00000001L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                0x00000002L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                    0x00000004L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                0x00000008L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                 0x00000010L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                      0x00000020L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                       0x00000040L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                  0x00007F80L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                  0x00018000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                         0x00020000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                        0x00040000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                     0xFFF80000L
+//BIF_CFG_DEV0_RC0_SLOT_CNTL
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                              0x2
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                         0x3
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                0xa
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                         0xb
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                0xc
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                        0xd
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                  0xe
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                               0x0001L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                0x0004L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                           0x0008L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                  0x00C0L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                   0x0300L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                  0x0400L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                           0x0800L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                          0x2000L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                    0x4000L
+//BIF_CFG_DEV0_RC0_SLOT_STATUS
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                     0x7
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                0x0001L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                              0x0040L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                       0x0080L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                   0x0100L
+//BIF_CFG_DEV0_RC0_ROOT_CNTL
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                           0x0010L
+//BIF_CFG_DEV0_RC0_ROOT_CAP
+#define BIF_CFG_DEV0_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                               0x0001L
+//BIF_CFG_DEV0_RC0_ROOT_STATUS
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_STATUS__SHIFT                                                       0x10
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_PENDING__SHIFT                                                      0x11
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_STATUS_MASK                                                         0x00010000L
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_PENDING_MASK                                                        0x00020000L
+//BIF_CFG_DEV0_RC0_DEVICE_CAP2
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                        0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                       0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                       0x7
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                        0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                               0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                    0xe
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                  0x10
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                  0x11
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                   0x12
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                     0x15
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                         0x16
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                    0x1f
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                        0x0000000FL
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                          0x00000010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                         0x00000080L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                             0x00000200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                          0x00000400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                      0x00000800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                 0x00003000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                      0x0000C000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                     0x000C0000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                       0x00100000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                           0x00C00000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                     0x03000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                      0x04000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                      0x80000000L
+//BIF_CFG_DEV0_RC0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                              0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__LTR_EN__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                    0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                         0xd
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                 0x000FL
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                          0x0080L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                0x0100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                             0x0200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__LTR_EN_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                      0x0800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                      0x1000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__OBFF_EN_MASK                                                           0x6000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                       0x8000L
+//BIF_CFG_DEV0_RC0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS2__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS2__RESERVED_MASK                                                        0xFFFFL
+//BIF_CFG_DEV0_RC0_LINK_CAP2
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                0x8
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                          0x17
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                          0x18
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                      0x1f
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                 0x000000FEL
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                  0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                             0x0000FE00L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                             0x007F0000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                            0x00800000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                            0x01000000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__DRS_SUPPORTED_MASK                                                        0x80000000L
+//BIF_CFG_DEV0_RC0_LINK_CNTL2
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                       0x5
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                              0xa
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                   0x000FL
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                         0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__XMIT_MARGIN_MASK                                                         0x0380L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                               0xF000L
+//BIF_CFG_DEV0_RC0_LINK_STATUS2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                 0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                 0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                   0x5
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                               0x6
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                               0x7
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                            0x8
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                0xa
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                              0x0001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                         0x0002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                   0x0004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                   0x0008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                   0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                     0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                 0x0080L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                              0x0300L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                  0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                     0x7000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                              0x8000L
+//BIF_CFG_DEV0_RC0_SLOT_CAP2
+#define BIF_CFG_DEV0_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                          0x00000001L
+//BIF_CFG_DEV0_RC0_SLOT_CNTL2
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL2__RESERVED__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL2__RESERVED_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_SLOT_STATUS2
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS2__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS2__RESERVED_MASK                                                          0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EN_MASK                                                            0x0001L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                     0x000EL
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                      0x0070L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                         0x0100L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                              0x0200L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                               0x0400L
+//BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                              0x2
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                0xFFFFFFFCL
+//BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA__MSI_DATA_MASK                                                          0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                  0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                    0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_SSID_CAP_LIST
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_RC0_SSID_CAP
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                        0x10
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_ID_MASK                                                          0xFFFF0000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                 0x1
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                   0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                        0x0000FE00L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                  0x0000FFFFL
+//BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_RC0_LINK_CAP_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                            0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                              0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                              0x00000004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                0x00000010L
+//BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT          0x0
+#define BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK            0x0000FFFFL
+//BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                              0x9
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                              0xa
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                               0xb
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                              0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                0x00000200L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                0x00000400L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                 0x0000F800L
+//BIF_CFG_DEV0_RC0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                      0x00000700L
+//BIF_CFG_DEV0_RC0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                            0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                 0x9
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                       0xa
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                              0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                              0x00000004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                0x00000010L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                          0x000000C0L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                   0x00000200L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                         0x00000400L
+//BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                     0x0001L
+//BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                        0x0
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                          0x0001L
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                0x8
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                              0xa
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                             0x00000007L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                0x00000070L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                  0x00000300L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                0x00000C00L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                    0x18
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                      0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                0x000EL
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                        0x0001L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x007F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x003F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                    0x0
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                    0x1f
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                      0x007FFFFFL
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                      0x80000000L
+//BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                  0x007FFFFFL
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                     0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                         0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                        0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                       0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                  0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                 0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                 0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                        0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                         0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                    0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                          0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                       0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                          0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                           0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                        0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                         0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                          0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                         0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                   0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                   0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                   0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                          0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                           0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                      0x04000000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                            0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                             0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                           0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                            0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                           0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                      0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                     0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                     0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                            0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                             0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                        0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                              0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                              0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                               0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                            0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                             0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                              0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                             0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                       0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                        0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                       0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                              0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                               0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                          0x04000000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                    0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                 0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                     0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                              0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                  0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                   0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                    0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                   0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                             0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                              0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                             0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                             0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                    0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                     0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                      0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                   0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                      0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                       0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                     0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                      0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                     0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                               0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                               0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                               0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                      0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                       0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                  0x04000000L
+//BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                             0xc
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                            0xd
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                            0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                           0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                               0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                              0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                       0x00008000L
+//BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                             0x7
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                 0xc
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                0xd
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                         0xf
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                               0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                           0x00008000L
+//BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                           0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                     0x9
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                      0xa
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                 0xb
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                         0xc
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                            0x0000001FL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                             0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                              0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                           0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                            0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                       0x00000200L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                        0x00000400L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                   0x00000800L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                           0x00001000L
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                           0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                             0x00000004L
+//BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                      0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                 0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                 0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                    0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                     0x1b
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                        0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                   0x00000004L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                   0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                          0x00000180L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                       0xF8000000L
+//BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                      0xFFFF0000L
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                            0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                            0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                         0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                              0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                           0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                             0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                              0x0002L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                              0x0004L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                           0x0008L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                               0x0010L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                0x0020L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                             0x0040L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                               0x0080L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                        0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                        0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                     0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0xa
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                0xc
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                          0x0004L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                       0x0008L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                         0x0040L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                           0x0080L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0300L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0C00L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                  0x1000L
+//BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_RTR_DATA1
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RESET_TIME__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__DLUP_TIME__SHIFT                                                          0xc
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RTR_VALID__SHIFT                                                          0x1f
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RESET_TIME_MASK                                                           0x00000FFFL
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__DLUP_TIME_MASK                                                            0x00FFF000L
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RTR_VALID_MASK                                                            0x80000000L
+//BIF_CFG_DEV0_RC0_RTR_DATA2
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__FLR_TIME__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__FLR_TIME_MASK                                                             0x00000FFFL
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__D3HOTD0_TIME_MASK                                                         0x00FFF000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_COMMAND
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF0_0_STATUS
+#define BIF_CFG_DEV0_EPF0_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF0_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF0_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF0_0_LATENCY
+#define BIF_CFG_DEV0_EPF0_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_0_HEADER
+#define BIF_CFG_DEV0_EPF0_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF0_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF0_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF0_0_BIST
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF0_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                 0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                      0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                     0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                        0x000003FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                        0x00001C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                       0x03FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                       0x1C000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                                 0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                   0x0200L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__STU__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__STU_MASK                                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                               0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX__SHIFT                        0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__STOPPED__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED__SHIFT                          0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX_MASK                          0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__STOPPED_MASK                                                0x0100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED_MASK                            0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY_MASK                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                          0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                            0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                            0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                              0xFFE00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                           0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                           0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                    0x0020L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                 0x0001L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                    0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                        0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                            0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                            0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                        0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                  0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                          0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK     0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER__SHIFT                          0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR__SHIFT                         0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID_MASK                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER_MASK                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR_MASK                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID_MASK                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV_MASK                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN_MASK    0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN_MASK  0x02000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS_MASK  0x02000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR_MASK                   0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID__SHIFT               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX_MASK                      0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA_MASK                  0x00000F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID_MASK                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA_MASK                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK_MASK                   0x01000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID__SHIFT               0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK__SHIFT                 0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID__SHIFT               0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID__SHIFT               0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK__SHIFT                 0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID__SHIFT               0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID__SHIFT               0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK__SHIFT                 0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID__SHIFT               0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK__SHIFT                 0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID__SHIFT               0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK__SHIFT                 0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID__SHIFT               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK__SHIFT                 0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID__SHIFT               0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK__SHIFT                 0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID__SHIFT               0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK__SHIFT                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID__SHIFT              0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK__SHIFT                0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID__SHIFT              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK__SHIFT                0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID__SHIFT              0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK__SHIFT                0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID__SHIFT              0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK__SHIFT                0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID__SHIFT              0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK__SHIFT                0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID__SHIFT              0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK_MASK                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID_MASK                 0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK_MASK                   0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID_MASK                 0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK_MASK                   0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID_MASK                 0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK_MASK                   0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID_MASK                 0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK_MASK                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID_MASK                 0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK_MASK                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID_MASK                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK_MASK                   0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID_MASK                 0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK_MASK                   0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID_MASK                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK_MASK                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID_MASK                 0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK_MASK                   0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID_MASK                 0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK_MASK                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID_MASK                0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK_MASK                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID_MASK                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK_MASK                  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID_MASK                0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK_MASK                  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID_MASK                0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK_MASK                  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID_MASK                0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK_MASK                  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID_MASK                0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID__SHIFT              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK__SHIFT                0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID__SHIFT              0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK__SHIFT                0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID__SHIFT              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK__SHIFT                0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID__SHIFT              0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID__SHIFT              0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK__SHIFT                0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID__SHIFT              0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK__SHIFT                0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID__SHIFT              0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK__SHIFT                0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID__SHIFT              0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID__SHIFT              0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK__SHIFT                0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID__SHIFT              0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK__SHIFT                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID__SHIFT              0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK__SHIFT                0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID__SHIFT              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK__SHIFT                0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID__SHIFT              0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK__SHIFT                0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID__SHIFT              0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK__SHIFT                0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID__SHIFT              0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK__SHIFT                  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID__SHIFT                0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK_MASK                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID_MASK                0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK_MASK                  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID_MASK                0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK_MASK                  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID_MASK                0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK_MASK                  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID_MASK                0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK_MASK                  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID_MASK                0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK_MASK                  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID_MASK                0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK_MASK                  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID_MASK                0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK_MASK                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID_MASK                0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK_MASK                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID_MASK                0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK_MASK                  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID_MASK                0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK_MASK                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID_MASK                0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK_MASK                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID_MASK                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK_MASK                  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID_MASK                0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK_MASK                  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID_MASK                0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK_MASK                  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID_MASK                0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK_MASK                    0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID_MASK                  0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                     0x0000FE00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                               0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT__RESERVED__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT__RESERVED_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT__RESERVED_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                         0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                         0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                           0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                           0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                             0x00000010L
+//BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT       0x0
+#define BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK         0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                           0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                           0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                        0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                             0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                             0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                              0x0000F800L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                   0x00000700L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                         0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                         0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                     0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                     0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                              0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                    0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                           0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                           0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                             0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                       0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                       0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                      0x00000400L
+//BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                  0x0001L
+//BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                            0x1
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                              0x0002L
+//BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                           0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                          0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                             0x00000070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                             0x00000C00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                             0x000EL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                     0x0001L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x003F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                 0x1f
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                               0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                         0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+//BIF_CFG_DEV0_EPF1_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_COMMAND
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF1_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF1_0_STATUS
+#define BIF_CFG_DEV0_EPF1_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF1_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF1_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF1_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF1_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF1_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF1_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF1_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF1_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF1_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF1_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF1_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF1_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF1_0_LATENCY
+#define BIF_CFG_DEV0_EPF1_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF1_0_HEADER
+#define BIF_CFG_DEV0_EPF1_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF1_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF1_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF1_0_BIST
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF1_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF1_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF1_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF1_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF1_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF1_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF1_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF1_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF1_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF1_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF1_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF1_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF1_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF1_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF1_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF1_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                 0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF1_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf2_bifcfgdecp
+//BIF_CFG_DEV0_EPF2_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF2_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF2_0_COMMAND
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF2_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF2_0_STATUS
+#define BIF_CFG_DEV0_EPF2_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF2_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF2_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF2_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF2_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF2_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF2_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF2_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF2_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF2_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF2_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF2_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF2_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF2_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF2_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF2_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF2_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF2_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF2_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF2_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF2_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF2_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF2_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF2_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF2_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF2_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF2_0_LATENCY
+#define BIF_CFG_DEV0_EPF2_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF2_0_HEADER
+#define BIF_CFG_DEV0_EPF2_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF2_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF2_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF2_0_BIST
+#define BIF_CFG_DEV0_EPF2_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF2_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF2_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF2_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF2_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF2_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF2_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF2_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF2_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF2_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF2_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF2_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF2_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF2_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF2_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF2_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF2_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF2_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF2_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF2_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF2_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF2_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF2_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF2_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF2_0_SBRN
+#define BIF_CFG_DEV0_EPF2_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF2_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV0_EPF2_0_FLADJ
+#define BIF_CFG_DEV0_EPF2_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_EPF2_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV0_EPF2_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV0_EPF2_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV0_EPF2_0_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF2_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF2_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV0_EPF2_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV0_EPF2_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF2_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF2_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF2_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF2_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF2_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF2_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF2_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF2_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF2_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF2_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF2_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF2_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF2_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF2_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF2_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF2_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF2_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF2_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF2_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF2_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF2_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF2_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF2_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF2_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF2_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF2_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF2_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF2_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF2_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF2_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF2_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF2_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf3_bifcfgdecp
+//BIF_CFG_DEV0_EPF3_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF3_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF3_0_COMMAND
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF3_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF3_0_STATUS
+#define BIF_CFG_DEV0_EPF3_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF3_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF3_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF3_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF3_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF3_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF3_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF3_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF3_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF3_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF3_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF3_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF3_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF3_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF3_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF3_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF3_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF3_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF3_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF3_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF3_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF3_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF3_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF3_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF3_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF3_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF3_0_LATENCY
+#define BIF_CFG_DEV0_EPF3_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF3_0_HEADER
+#define BIF_CFG_DEV0_EPF3_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF3_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF3_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF3_0_BIST
+#define BIF_CFG_DEV0_EPF3_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF3_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF3_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF3_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF3_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF3_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF3_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF3_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF3_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF3_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF3_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF3_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF3_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF3_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF3_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF3_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF3_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF3_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF3_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF3_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF3_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF3_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF3_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF3_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF3_0_SBRN
+#define BIF_CFG_DEV0_EPF3_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF3_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV0_EPF3_0_FLADJ
+#define BIF_CFG_DEV0_EPF3_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_EPF3_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV0_EPF3_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV0_EPF3_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV0_EPF3_0_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF3_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF3_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV0_EPF3_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV0_EPF3_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF3_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF3_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF3_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF3_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF3_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF3_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF3_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF3_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF3_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF3_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF3_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF3_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF3_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF3_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF3_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF3_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF3_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF3_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF3_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF3_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF3_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF3_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF3_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF3_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF3_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF3_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF3_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF3_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF3_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF3_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF3_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF3_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf4_bifcfgdecp
+//BIF_CFG_DEV0_EPF4_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF4_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF4_0_COMMAND
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF4_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF4_0_STATUS
+#define BIF_CFG_DEV0_EPF4_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF4_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF4_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF4_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF4_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF4_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF4_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF4_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF4_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF4_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF4_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF4_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF4_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF4_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF4_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF4_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF4_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF4_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF4_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF4_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF4_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF4_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF4_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF4_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF4_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF4_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF4_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF4_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF4_0_LATENCY
+#define BIF_CFG_DEV0_EPF4_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF4_0_HEADER
+#define BIF_CFG_DEV0_EPF4_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF4_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF4_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF4_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF4_0_BIST
+#define BIF_CFG_DEV0_EPF4_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF4_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF4_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF4_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF4_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF4_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF4_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF4_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF4_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF4_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF4_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF4_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF4_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF4_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF4_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF4_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF4_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF4_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF4_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF4_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF4_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF4_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF4_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF4_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF4_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF4_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF4_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF4_0_SBRN
+#define BIF_CFG_DEV0_EPF4_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF4_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV0_EPF4_0_FLADJ
+#define BIF_CFG_DEV0_EPF4_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_EPF4_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV0_EPF4_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV0_EPF4_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV0_EPF4_0_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF4_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF4_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF4_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV0_EPF4_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV0_EPF4_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF4_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF4_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF4_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF4_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF4_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF4_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF4_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF4_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF4_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF4_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF4_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF4_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF4_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF4_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF4_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF4_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF4_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF4_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF4_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF4_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF4_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF4_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF4_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF4_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF4_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF4_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF4_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF4_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF4_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF4_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF4_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF4_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF4_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf5_bifcfgdecp
+//BIF_CFG_DEV0_EPF5_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF5_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF5_0_COMMAND
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF5_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF5_0_STATUS
+#define BIF_CFG_DEV0_EPF5_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF5_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF5_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF5_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF5_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF5_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF5_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF5_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF5_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF5_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF5_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF5_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF5_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF5_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF5_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF5_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF5_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF5_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF5_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF5_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF5_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF5_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF5_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF5_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF5_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF5_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF5_0_LATENCY
+#define BIF_CFG_DEV0_EPF5_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF5_0_HEADER
+#define BIF_CFG_DEV0_EPF5_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF5_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF5_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF5_0_BIST
+#define BIF_CFG_DEV0_EPF5_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF5_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF5_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF5_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF5_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF5_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF5_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF5_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF5_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF5_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF5_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF5_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF5_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF5_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF5_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF5_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF5_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF5_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF5_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF5_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF5_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF5_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF5_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF5_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF5_0_SBRN
+#define BIF_CFG_DEV0_EPF5_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF5_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV0_EPF5_0_FLADJ
+#define BIF_CFG_DEV0_EPF5_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_EPF5_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV0_EPF5_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV0_EPF5_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV0_EPF5_0_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF5_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF5_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV0_EPF5_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV0_EPF5_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF5_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF5_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF5_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF5_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF5_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF5_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF5_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF5_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF5_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF5_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF5_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF5_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF5_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF5_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF5_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF5_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF5_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF5_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF5_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF5_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF5_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF5_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF5_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF5_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF5_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF5_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF5_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF5_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF5_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF5_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF5_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF5_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf6_bifcfgdecp
+//BIF_CFG_DEV0_EPF6_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF6_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF6_0_COMMAND
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF6_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF6_0_STATUS
+#define BIF_CFG_DEV0_EPF6_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF6_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF6_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF6_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF6_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF6_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF6_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF6_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF6_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF6_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF6_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF6_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF6_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF6_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF6_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF6_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF6_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF6_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF6_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF6_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF6_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF6_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF6_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF6_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF6_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF6_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF6_0_LATENCY
+#define BIF_CFG_DEV0_EPF6_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF6_0_HEADER
+#define BIF_CFG_DEV0_EPF6_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF6_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF6_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF6_0_BIST
+#define BIF_CFG_DEV0_EPF6_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF6_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF6_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF6_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF6_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF6_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF6_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF6_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF6_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF6_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF6_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF6_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF6_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF6_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF6_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF6_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF6_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF6_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF6_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF6_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF6_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF6_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF6_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF6_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF6_0_SBRN
+#define BIF_CFG_DEV0_EPF6_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF6_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV0_EPF6_0_FLADJ
+#define BIF_CFG_DEV0_EPF6_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_EPF6_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV0_EPF6_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV0_EPF6_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV0_EPF6_0_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF6_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF6_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV0_EPF6_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV0_EPF6_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF6_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF6_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF6_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF6_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF6_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF6_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF6_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF6_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF6_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF6_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF6_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF6_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF6_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF6_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF6_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF6_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF6_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF6_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF6_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF6_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF6_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF6_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF6_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF6_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF6_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF6_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF6_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF6_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF6_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF6_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF6_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF6_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev0_epf7_bifcfgdecp
+//BIF_CFG_DEV0_EPF7_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF7_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF7_0_COMMAND
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF7_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF7_0_STATUS
+#define BIF_CFG_DEV0_EPF7_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF7_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF7_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF7_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF7_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF7_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF7_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF7_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF7_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF7_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF7_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF7_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF7_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF7_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF7_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF7_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF7_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF7_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF7_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF7_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF7_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF7_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF7_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF7_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF7_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF7_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF7_0_LATENCY
+#define BIF_CFG_DEV0_EPF7_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF7_0_HEADER
+#define BIF_CFG_DEV0_EPF7_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF7_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF7_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF7_0_BIST
+#define BIF_CFG_DEV0_EPF7_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF7_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF7_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF7_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF7_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF7_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF7_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF7_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF7_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF7_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF7_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF7_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF7_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF7_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF7_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF7_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF7_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF7_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF7_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF7_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF7_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF7_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF7_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF7_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF7_0_SBRN
+#define BIF_CFG_DEV0_EPF7_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_EPF7_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV0_EPF7_0_FLADJ
+#define BIF_CFG_DEV0_EPF7_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_EPF7_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV0_EPF7_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV0_EPF7_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV0_EPF7_0_DBESL_DBESLD
+#define BIF_CFG_DEV0_EPF7_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF7_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV0_EPF7_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV0_EPF7_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF7_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV0_EPF7_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF7_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF7_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF7_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF7_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF7_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF7_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF7_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF7_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF7_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF7_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF7_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF7_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF7_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF7_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF7_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF7_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF7_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF7_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF7_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF7_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF7_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF7_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF7_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF7_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF7_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF7_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF7_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF7_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF7_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF7_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_rc_bifcfgdecp
+//BIF_CFG_DEV1_RC0_VENDOR_ID
+#define BIF_CFG_DEV1_RC0_VENDOR_ID__VENDOR_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_VENDOR_ID__VENDOR_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV1_RC0_DEVICE_ID
+#define BIF_CFG_DEV1_RC0_DEVICE_ID__DEVICE_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_DEVICE_ID__DEVICE_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV1_RC0_COMMAND
+#define BIF_CFG_DEV1_RC0_COMMAND__IOEN_DN__SHIFT                                                              0x0
+#define BIF_CFG_DEV1_RC0_COMMAND__MEMEN_DN__SHIFT                                                             0x1
+#define BIF_CFG_DEV1_RC0_COMMAND__BUS_MASTER_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV1_RC0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV1_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV1_RC0_COMMAND__PAL_SNOOP_EN__SHIFT                                                         0x5
+#define BIF_CFG_DEV1_RC0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                0x6
+#define BIF_CFG_DEV1_RC0_COMMAND__AD_STEPPING__SHIFT                                                          0x7
+#define BIF_CFG_DEV1_RC0_COMMAND__SERR_EN__SHIFT                                                              0x8
+#define BIF_CFG_DEV1_RC0_COMMAND__FAST_B2B_EN__SHIFT                                                          0x9
+#define BIF_CFG_DEV1_RC0_COMMAND__INT_DIS__SHIFT                                                              0xa
+#define BIF_CFG_DEV1_RC0_COMMAND__IOEN_DN_MASK                                                                0x0001L
+#define BIF_CFG_DEV1_RC0_COMMAND__MEMEN_DN_MASK                                                               0x0002L
+#define BIF_CFG_DEV1_RC0_COMMAND__BUS_MASTER_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV1_RC0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV1_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV1_RC0_COMMAND__PAL_SNOOP_EN_MASK                                                           0x0020L
+#define BIF_CFG_DEV1_RC0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                  0x0040L
+#define BIF_CFG_DEV1_RC0_COMMAND__AD_STEPPING_MASK                                                            0x0080L
+#define BIF_CFG_DEV1_RC0_COMMAND__SERR_EN_MASK                                                                0x0100L
+#define BIF_CFG_DEV1_RC0_COMMAND__FAST_B2B_EN_MASK                                                            0x0200L
+#define BIF_CFG_DEV1_RC0_COMMAND__INT_DIS_MASK                                                                0x0400L
+//BIF_CFG_DEV1_RC0_STATUS
+#define BIF_CFG_DEV1_RC0_STATUS__IMMEDIATE_READINESS__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC0_STATUS__INT_STATUS__SHIFT                                                            0x3
+#define BIF_CFG_DEV1_RC0_STATUS__CAP_LIST__SHIFT                                                              0x4
+#define BIF_CFG_DEV1_RC0_STATUS__PCI_66_CAP__SHIFT                                                            0x5
+#define BIF_CFG_DEV1_RC0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                     0x7
+#define BIF_CFG_DEV1_RC0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                              0x8
+#define BIF_CFG_DEV1_RC0_STATUS__DEVSEL_TIMING__SHIFT                                                         0x9
+#define BIF_CFG_DEV1_RC0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_RC0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                 0xc
+#define BIF_CFG_DEV1_RC0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                 0xd
+#define BIF_CFG_DEV1_RC0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                 0xe
+#define BIF_CFG_DEV1_RC0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                 0xf
+#define BIF_CFG_DEV1_RC0_STATUS__IMMEDIATE_READINESS_MASK                                                     0x0001L
+#define BIF_CFG_DEV1_RC0_STATUS__INT_STATUS_MASK                                                              0x0008L
+#define BIF_CFG_DEV1_RC0_STATUS__CAP_LIST_MASK                                                                0x0010L
+#define BIF_CFG_DEV1_RC0_STATUS__PCI_66_CAP_MASK                                                              0x0020L
+#define BIF_CFG_DEV1_RC0_STATUS__FAST_BACK_CAPABLE_MASK                                                       0x0080L
+#define BIF_CFG_DEV1_RC0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                0x0100L
+#define BIF_CFG_DEV1_RC0_STATUS__DEVSEL_TIMING_MASK                                                           0x0600L
+#define BIF_CFG_DEV1_RC0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                     0x0800L
+#define BIF_CFG_DEV1_RC0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                   0x1000L
+#define BIF_CFG_DEV1_RC0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                   0x2000L
+#define BIF_CFG_DEV1_RC0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                   0x4000L
+#define BIF_CFG_DEV1_RC0_STATUS__PARITY_ERROR_DETECTED_MASK                                                   0x8000L
+//BIF_CFG_DEV1_RC0_REVISION_ID
+#define BIF_CFG_DEV1_RC0_REVISION_ID__MINOR_REV_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_RC0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                     0x4
+#define BIF_CFG_DEV1_RC0_REVISION_ID__MINOR_REV_ID_MASK                                                       0x0FL
+#define BIF_CFG_DEV1_RC0_REVISION_ID__MAJOR_REV_ID_MASK                                                       0xF0L
+//BIF_CFG_DEV1_RC0_PROG_INTERFACE
+#define BIF_CFG_DEV1_RC0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                  0xFFL
+//BIF_CFG_DEV1_RC0_SUB_CLASS
+#define BIF_CFG_DEV1_RC0_SUB_CLASS__SUB_CLASS__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_SUB_CLASS__SUB_CLASS_MASK                                                            0xFFL
+//BIF_CFG_DEV1_RC0_BASE_CLASS
+#define BIF_CFG_DEV1_RC0_BASE_CLASS__BASE_CLASS__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_BASE_CLASS__BASE_CLASS_MASK                                                          0xFFL
+//BIF_CFG_DEV1_RC0_CACHE_LINE
+#define BIF_CFG_DEV1_RC0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                     0xFFL
+//BIF_CFG_DEV1_RC0_LATENCY
+#define BIF_CFG_DEV1_RC0_LATENCY__LATENCY_TIMER__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_LATENCY__LATENCY_TIMER_MASK                                                          0xFFL
+//BIF_CFG_DEV1_RC0_HEADER
+#define BIF_CFG_DEV1_RC0_HEADER__HEADER_TYPE__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC0_HEADER__DEVICE_TYPE__SHIFT                                                           0x7
+#define BIF_CFG_DEV1_RC0_HEADER__HEADER_TYPE_MASK                                                             0x7FL
+#define BIF_CFG_DEV1_RC0_HEADER__DEVICE_TYPE_MASK                                                             0x80L
+//BIF_CFG_DEV1_RC0_BIST
+#define BIF_CFG_DEV1_RC0_BIST__BIST_COMP__SHIFT                                                               0x0
+#define BIF_CFG_DEV1_RC0_BIST__BIST_STRT__SHIFT                                                               0x6
+#define BIF_CFG_DEV1_RC0_BIST__BIST_CAP__SHIFT                                                                0x7
+#define BIF_CFG_DEV1_RC0_BIST__BIST_COMP_MASK                                                                 0x0FL
+#define BIF_CFG_DEV1_RC0_BIST__BIST_STRT_MASK                                                                 0x40L
+#define BIF_CFG_DEV1_RC0_BIST__BIST_CAP_MASK                                                                  0x80L
+//BIF_CFG_DEV1_RC0_BASE_ADDR_1
+#define BIF_CFG_DEV1_RC0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_BASE_ADDR_1__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_BASE_ADDR_2
+#define BIF_CFG_DEV1_RC0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_BASE_ADDR_2__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                         0x8
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                               0x18
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                             0x000000FFL
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                           0x0000FF00L
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                             0x00FF0000L
+#define BIF_CFG_DEV1_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                 0xFF000000L
+//BIF_CFG_DEV1_RC0_IO_BASE_LIMIT
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_BASE__SHIFT                                                        0x4
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                  0x8
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                       0xc
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                     0x000FL
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_BASE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                    0x0F00L
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT__IO_LIMIT_MASK                                                         0xF000L
+//BIF_CFG_DEV1_RC0_SECONDARY_STATUS
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                  0x5
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                           0x7
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                    0x8
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                               0x9
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                         0xb
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                       0xc
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                       0xd
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                       0xe
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                       0xf
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__PCI_66_CAP_MASK                                                    0x0020L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                             0x0080L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                      0x0100L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                 0x0600L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                           0x0800L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                         0x1000L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                         0x2000L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                         0x4000L
+#define BIF_CFG_DEV1_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                         0x8000L
+//BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                0x4
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                0x10
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                               0x14
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                   0x0000000FL
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                  0x0000FFF0L
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV1_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                          0x4
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                          0x10
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                         0x14
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                             0x0000000FL
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                            0x0000FFF0L
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                            0x000F0000L
+#define BIF_CFG_DEV1_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                           0xFFF00000L
+//BIF_CFG_DEV1_RC0_PREF_BASE_UPPER
+#define BIF_CFG_DEV1_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV1_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                              0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                              0x10
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                0xFFFF0000L
+//BIF_CFG_DEV1_RC0_CAP_PTR
+#define BIF_CFG_DEV1_RC0_CAP_PTR__CAP_PTR__SHIFT                                                              0x0
+#define BIF_CFG_DEV1_RC0_CAP_PTR__CAP_PTR_MASK                                                                0xFFL
+//BIF_CFG_DEV1_RC0_ROM_BASE_ADDR
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                          0x1
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                         0x4
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                      0xb
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                       0x00000001L
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                            0x0000000EL
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                           0x000000F0L
+#define BIF_CFG_DEV1_RC0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                        0xFFFFF800L
+//BIF_CFG_DEV1_RC0_INTERRUPT_LINE
+#define BIF_CFG_DEV1_RC0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                  0xFFL
+//BIF_CFG_DEV1_RC0_INTERRUPT_PIN
+#define BIF_CFG_DEV1_RC0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                    0xFFL
+//BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                      0x1
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                       0x3
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                      0x4
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                            0x5
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                          0x6
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                  0x7
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                        0x8
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                      0x9
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                         0xa
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                    0xb
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                        0x0002L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                         0x0008L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                        0x0010L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                              0x0020L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                            0x0040L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                    0x0080L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                          0x0100L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                        0x0200L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                           0x0400L
+#define BIF_CFG_DEV1_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                      0x0800L
+//BIF_CFG_DEV1_RC0_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV1_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                  0x01L
+//BIF_CFG_DEV1_RC0_PMI_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PMI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV1_RC0_PMI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV1_RC0_PMI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV1_RC0_PMI_CAP
+#define BIF_CFG_DEV1_RC0_PMI_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV1_RC0_PMI_CAP__PME_CLOCK__SHIFT                                                            0x3
+#define BIF_CFG_DEV1_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                  0x4
+#define BIF_CFG_DEV1_RC0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                    0x5
+#define BIF_CFG_DEV1_RC0_PMI_CAP__AUX_CURRENT__SHIFT                                                          0x6
+#define BIF_CFG_DEV1_RC0_PMI_CAP__D1_SUPPORT__SHIFT                                                           0x9
+#define BIF_CFG_DEV1_RC0_PMI_CAP__D2_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV1_RC0_PMI_CAP__PME_SUPPORT__SHIFT                                                          0xb
+#define BIF_CFG_DEV1_RC0_PMI_CAP__VERSION_MASK                                                                0x0007L
+#define BIF_CFG_DEV1_RC0_PMI_CAP__PME_CLOCK_MASK                                                              0x0008L
+#define BIF_CFG_DEV1_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                    0x0010L
+#define BIF_CFG_DEV1_RC0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                      0x0020L
+#define BIF_CFG_DEV1_RC0_PMI_CAP__AUX_CURRENT_MASK                                                            0x01C0L
+#define BIF_CFG_DEV1_RC0_PMI_CAP__D1_SUPPORT_MASK                                                             0x0200L
+#define BIF_CFG_DEV1_RC0_PMI_CAP__D2_SUPPORT_MASK                                                             0x0400L
+#define BIF_CFG_DEV1_RC0_PMI_CAP__PME_SUPPORT_MASK                                                            0xF800L
+//BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                0x3
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                  0x9
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                   0xd
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                   0xf
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                0x16
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                   0x17
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                     0x18
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                    0x00000003L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                  0x00000008L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__PME_EN_MASK                                                         0x00000100L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                    0x00001E00L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                     0x00006000L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                     0x00008000L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                  0x00400000L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                     0x00800000L
+#define BIF_CFG_DEV1_RC0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                       0xFF000000L
+//BIF_CFG_DEV1_RC0_PCIE_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_RC0_PCIE_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV1_RC0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV1_RC0_PCIE_CAP
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                         0x4
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                     0x9
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__VERSION_MASK                                                               0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__DEVICE_TYPE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                      0x0100L
+#define BIF_CFG_DEV1_RC0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                       0x3E00L
+//BIF_CFG_DEV1_RC0_DEVICE_CAP
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                      0x3
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                      0x5
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                            0x6
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                             0x9
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                          0xf
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                          0x10
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                         0x12
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                         0x1a
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                       0x1c
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                 0x00000007L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                        0x00000018L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__EXTENDED_TAG_MASK                                                        0x00000020L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                              0x000001C0L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                               0x00000E00L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                            0x00008000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                            0x00010000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                           0x03FC0000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                           0x0C000000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP__FLR_CAPABLE_MASK                                                         0x10000000L
+//BIF_CFG_DEV1_RC0_DEVICE_CNTL
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                 0x1
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                 0x5
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                  0x8
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                  0x9
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                  0xa
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                      0xb
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                            0xc
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                              0xf
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                   0x0002L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                     0x0010L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                   0x00E0L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                    0x0100L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                    0x0200L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                    0x0400L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                        0x0800L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                              0x7000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                0x8000L
+//BIF_CFG_DEV1_RC0_DEVICE_STATUS
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__CORR_ERR__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                  0x1
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                      0x2
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                   0x3
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__AUX_PWR__SHIFT                                                        0x4
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                              0x5
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                  0x6
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__CORR_ERR_MASK                                                         0x0001L
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                    0x0002L
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__FATAL_ERR_MASK                                                        0x0004L
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__USR_DETECTED_MASK                                                     0x0008L
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__AUX_PWR_MASK                                                          0x0010L
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                0x0020L
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                    0x0040L
+//BIF_CFG_DEV1_RC0_LINK_CAP
+#define BIF_CFG_DEV1_RC0_LINK_CAP__LINK_SPEED__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_LINK_CAP__LINK_WIDTH__SHIFT                                                          0x4
+#define BIF_CFG_DEV1_RC0_LINK_CAP__PM_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV1_RC0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                    0xc
+#define BIF_CFG_DEV1_RC0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                     0xf
+#define BIF_CFG_DEV1_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                              0x12
+#define BIF_CFG_DEV1_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                         0x13
+#define BIF_CFG_DEV1_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                         0x14
+#define BIF_CFG_DEV1_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                            0x15
+#define BIF_CFG_DEV1_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                         0x16
+#define BIF_CFG_DEV1_RC0_LINK_CAP__PORT_NUMBER__SHIFT                                                         0x18
+#define BIF_CFG_DEV1_RC0_LINK_CAP__LINK_SPEED_MASK                                                            0x0000000FL
+#define BIF_CFG_DEV1_RC0_LINK_CAP__LINK_WIDTH_MASK                                                            0x000003F0L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__PM_SUPPORT_MASK                                                            0x00000C00L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                      0x00007000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                       0x00038000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                0x00040000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                           0x00080000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                           0x00100000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                              0x00200000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                           0x00400000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP__PORT_NUMBER_MASK                                                           0xFF000000L
+//BIF_CFG_DEV1_RC0_LINK_CNTL
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__PM_CONTROL__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                       0x2
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                  0x3
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__LINK_DIS__SHIFT                                                           0x4
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                       0x5
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                   0x6
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                      0x7
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                          0xb
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                              0xe
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__PM_CONTROL_MASK                                                           0x0003L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                         0x0004L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                    0x0008L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__LINK_DIS_MASK                                                             0x0010L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__RETRAIN_LINK_MASK                                                         0x0020L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                     0x0040L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__EXTENDED_SYNC_MASK                                                        0x0080L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                            0x0100L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                            0x0400L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                            0x0800L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                0xC000L
+//BIF_CFG_DEV1_RC0_LINK_STATUS
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                            0x4
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__LINK_TRAINING__SHIFT                                                    0xb
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                   0xc
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__DL_ACTIVE__SHIFT                                                        0xd
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                        0xe
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                        0xf
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                 0x000FL
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                              0x03F0L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__LINK_TRAINING_MASK                                                      0x0800L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                     0x1000L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__DL_ACTIVE_MASK                                                          0x2000L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                          0x4000L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                          0x8000L
+//BIF_CFG_DEV1_RC0_SLOT_CAP
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                              0x1
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                  0x2
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                              0x3
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                               0x4
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                    0x5
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                     0x6
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                0x7
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                0xf
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                       0x11
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                      0x12
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                   0x13
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                   0x00000001L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                0x00000002L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                    0x00000004L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                0x00000008L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                 0x00000010L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                      0x00000020L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                       0x00000040L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                  0x00007F80L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                  0x00018000L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                         0x00020000L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                        0x00040000L
+#define BIF_CFG_DEV1_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                     0xFFF80000L
+//BIF_CFG_DEV1_RC0_SLOT_CNTL
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                              0x2
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                         0x3
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                0x6
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                0xa
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                         0xb
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                0xc
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                        0xd
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                  0xe
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                               0x0001L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                0x0004L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                           0x0008L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                  0x00C0L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                   0x0300L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                  0x0400L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                           0x0800L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                  0x1000L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                          0x2000L
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                    0x4000L
+//BIF_CFG_DEV1_RC0_SLOT_STATUS
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                               0x1
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                               0x2
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                          0x3
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                0x4
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                 0x5
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                            0x6
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                     0x7
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                0x0001L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                 0x0002L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                 0x0004L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                            0x0008L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                  0x0010L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                   0x0020L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                              0x0040L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                       0x0080L
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                   0x0100L
+//BIF_CFG_DEV1_RC0_ROOT_CNTL
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                  0x0001L
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV1_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                           0x0010L
+//BIF_CFG_DEV1_RC0_ROOT_CAP
+#define BIF_CFG_DEV1_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                               0x0001L
+//BIF_CFG_DEV1_RC0_ROOT_STATUS
+#define BIF_CFG_DEV1_RC0_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_ROOT_STATUS__PME_STATUS__SHIFT                                                       0x10
+#define BIF_CFG_DEV1_RC0_ROOT_STATUS__PME_PENDING__SHIFT                                                      0x11
+#define BIF_CFG_DEV1_RC0_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_ROOT_STATUS__PME_STATUS_MASK                                                         0x00010000L
+#define BIF_CFG_DEV1_RC0_ROOT_STATUS__PME_PENDING_MASK                                                        0x00020000L
+//BIF_CFG_DEV1_RC0_DEVICE_CAP2
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                      0x0
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                        0x4
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                         0x5
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                       0x6
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                       0x7
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                       0x8
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                           0x9
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                        0xa
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                    0xb
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                               0xc
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                    0xe
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                  0x10
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                  0x11
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                   0x12
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                     0x14
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                     0x15
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                         0x16
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                   0x18
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                    0x1a
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                    0x1f
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                        0x0000000FL
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                          0x00000010L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                           0x00000020L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                         0x00000040L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                         0x00000080L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                         0x00000100L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                             0x00000200L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                          0x00000400L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                      0x00000800L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                 0x00003000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                      0x0000C000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                    0x00010000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                    0x00020000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                     0x000C0000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                       0x00100000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                       0x00200000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                           0x00C00000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                     0x03000000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                      0x04000000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                      0x80000000L
+//BIF_CFG_DEV1_RC0_DEVICE_CNTL2
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                 0x4
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                               0x5
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                        0x7
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                              0x8
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                           0x9
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__LTR_EN__SHIFT                                                          0xa
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                    0xb
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                    0xc
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                         0xd
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                     0xf
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                 0x000FL
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                   0x0010L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                 0x0020L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                               0x0040L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                          0x0080L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                0x0100L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                             0x0200L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__LTR_EN_MASK                                                            0x0400L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                      0x0800L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                      0x1000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__OBFF_EN_MASK                                                           0x6000L
+#define BIF_CFG_DEV1_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                       0x8000L
+//BIF_CFG_DEV1_RC0_DEVICE_STATUS2
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS2__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_RC0_DEVICE_STATUS2__RESERVED_MASK                                                        0xFFFFL
+//BIF_CFG_DEV1_RC0_LINK_CAP2
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                               0x1
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                0x8
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                           0x9
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                          0x17
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                          0x18
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                      0x1f
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                 0x000000FEL
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                  0x00000100L
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                             0x0000FE00L
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                             0x007F0000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                            0x00800000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                            0x01000000L
+#define BIF_CFG_DEV1_RC0_LINK_CAP2__DRS_SUPPORTED_MASK                                                        0x80000000L
+//BIF_CFG_DEV1_RC0_LINK_CNTL2
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                  0x4
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                       0x5
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                             0x6
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                       0x7
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                              0xa
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                    0xb
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                             0xc
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                   0x000FL
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                    0x0010L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                         0x0020L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                               0x0040L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__XMIT_MARGIN_MASK                                                         0x0380L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                0x0400L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                      0x0800L
+#define BIF_CFG_DEV1_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                               0xF000L
+//BIF_CFG_DEV1_RC0_LINK_STATUS2
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                       0x1
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                 0x2
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                 0x3
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                 0x4
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                   0x5
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                               0x6
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                               0x7
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                            0x8
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                0xa
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                   0xc
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                            0xf
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                              0x0001L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                         0x0002L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                   0x0004L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                   0x0008L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                   0x0010L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                     0x0020L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                 0x0040L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                 0x0080L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                              0x0300L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                  0x0400L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                     0x7000L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                              0x8000L
+//BIF_CFG_DEV1_RC0_SLOT_CAP2
+#define BIF_CFG_DEV1_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                        0x0
+#define BIF_CFG_DEV1_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                          0x00000001L
+//BIF_CFG_DEV1_RC0_SLOT_CNTL2
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL2__RESERVED__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_SLOT_CNTL2__RESERVED_MASK                                                            0xFFFFL
+//BIF_CFG_DEV1_RC0_SLOT_STATUS2
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS2__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_SLOT_STATUS2__RESERVED_MASK                                                          0xFFFFL
+//BIF_CFG_DEV1_RC0_MSI_CAP_LIST
+#define BIF_CFG_DEV1_RC0_MSI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV1_RC0_MSI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV1_RC0_MSI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV1_RC0_MSI_MSG_CNTL
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                   0x1
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                       0x7
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                       0x8
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                            0x9
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_EN_MASK                                                            0x0001L
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                     0x000EL
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                      0x0070L
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                         0x0080L
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                         0x0100L
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                              0x0200L
+#define BIF_CFG_DEV1_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                               0x0400L
+//BIF_CFG_DEV1_RC0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV1_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                              0x2
+#define BIF_CFG_DEV1_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                0xFFFFFFFCL
+//BIF_CFG_DEV1_RC0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV1_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_MSI_MSG_DATA
+#define BIF_CFG_DEV1_RC0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_MSI_MSG_DATA__MSI_DATA_MASK                                                          0xFFFFL
+//BIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                  0xFFFFL
+//BIF_CFG_DEV1_RC0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV1_RC0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                    0xFFFFL
+//BIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                            0xFFFFL
+//BIF_CFG_DEV1_RC0_SSID_CAP_LIST
+#define BIF_CFG_DEV1_RC0_SSID_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC0_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV1_RC0_SSID_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV1_RC0_SSID_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV1_RC0_SSID_CAP
+#define BIF_CFG_DEV1_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                        0x10
+#define BIF_CFG_DEV1_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_SSID_CAP__SUBSYSTEM_ID_MASK                                                          0xFFFF0000L
+//BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                            0x10
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                         0x14
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                              0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                           0xFFF00000L
+//BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                         0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                 0x1
+#define BIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                      0x9
+#define BIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                           0x00000001L
+#define BIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                   0x00000002L
+#define BIF_CFG_DEV1_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                        0x0000FE00L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                  0x0000FFFFL
+//BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV1_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_RC0_LINK_CAP_16GT
+#define BIF_CFG_DEV1_RC0_LINK_CAP_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_RC0_LINK_CAP_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_LINK_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LINK_CNTL_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_RC0_LINK_CNTL_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_LINK_STATUS_16GT
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                  0x0
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                            0x1
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                            0x2
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                            0x3
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                              0x4
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                              0x00000002L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                              0x00000004L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                              0x00000008L
+#define BIF_CFG_DEV1_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                0x00000010L
+//BIF_CFG_DEV1_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT          0x0
+#define BIF_CFG_DEV1_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK            0x0000FFFFL
+//BIF_CFG_DEV1_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV1_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV1_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV1_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV1_RC0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV1_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV1_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV1_RC0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV1_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV1_RC0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV1_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV1_RC0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV1_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV1_RC0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV1_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV1_RC0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV1_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV1_RC0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV1_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV1_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV1_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV1_RC0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV1_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                   0x0
+#define BIF_CFG_DEV1_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                     0x0001L
+//BIF_CFG_DEV1_RC0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV1_RC0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                        0x0
+#define BIF_CFG_DEV1_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                               0x1
+#define BIF_CFG_DEV1_RC0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                          0x0001L
+#define BIF_CFG_DEV1_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                 0x0002L
+//BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV1_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                              0x4
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                0x8
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                              0xa
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                             0x00000007L
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                0x00000070L
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                  0x00000300L
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                0x00000C00L
+//BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                             0x0
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                    0x18
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                               0x000000FFL
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                      0xFF000000L
+//BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                              0x1
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                0x000EL
+//BIF_CFG_DEV1_RC0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                      0x0
+#define BIF_CFG_DEV1_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                        0x0001L
+//BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x007F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV1_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x003F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV1_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                    0x0
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                    0x1f
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                      0x007FFFFFL
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                      0x80000000L
+//BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                0x0
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                          0x1f
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                  0x007FFFFFL
+#define BIF_CFG_DEV1_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                            0x80000000L
+//BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                        0x4
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                     0x5
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                        0xc
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                         0xd
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                      0x10
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                       0x11
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                        0x12
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                       0x13
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                 0x14
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                  0x15
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                 0x16
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                 0x17
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                        0x18
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                         0x19
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                    0x1a
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                          0x00000010L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                       0x00000020L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                          0x00001000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                           0x00002000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                    0x00008000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                        0x00010000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                         0x00020000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                          0x00040000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                         0x00080000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                   0x00100000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                    0x00200000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                   0x00400000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                   0x00800000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                          0x01000000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                           0x02000000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                      0x04000000L
+//BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                            0x4
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                         0x5
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                            0xc
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                             0xd
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                          0x10
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                           0x11
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                            0x12
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                           0x13
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                     0x14
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                      0x15
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                     0x16
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                     0x17
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                            0x18
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                             0x19
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                        0x1a
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                              0x00000010L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                           0x00000020L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                              0x00001000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                               0x00002000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                        0x00008000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                            0x00010000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                             0x00020000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                              0x00040000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                             0x00080000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                       0x00100000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                        0x00200000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                       0x00400000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                       0x00800000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                              0x01000000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                               0x02000000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                          0x04000000L
+//BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                    0x4
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                 0x5
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                    0xc
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                     0xd
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                0xe
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                              0xf
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                  0x10
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                   0x11
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                    0x12
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                   0x13
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                             0x14
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                              0x15
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                             0x16
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                             0x17
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                    0x18
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                     0x19
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                0x1a
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                      0x00000010L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                   0x00000020L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                      0x00001000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                       0x00002000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                  0x00004000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                0x00008000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                    0x00010000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                     0x00020000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                      0x00040000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                     0x00080000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                               0x00100000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                0x00200000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                               0x00400000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                               0x00800000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                      0x01000000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                       0x02000000L
+#define BIF_CFG_DEV1_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                  0x04000000L
+//BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                          0x6
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                         0x7
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                              0x8
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                             0xc
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                            0xd
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                            0x00000001L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                            0x00000040L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                           0x00000080L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                0x00000100L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                               0x00001000L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                              0x00002000L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                       0x00008000L
+//BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                              0x6
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                             0x7
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                  0x8
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                 0xc
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                0xd
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                         0xf
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                0x00000001L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                0x00000040L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                               0x00000080L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                    0x00000100L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                   0x00001000L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                  0x00002000L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV1_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                           0x00008000L
+//BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                          0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                           0x5
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                         0x7
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                     0x9
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                      0xa
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                 0xb
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                         0xc
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                            0x0000001FL
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                             0x00000020L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                              0x00000040L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                           0x00000080L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                            0x00000100L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                       0x00000200L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                        0x00000400L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                   0x00000800L
+#define BIF_CFG_DEV1_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                           0x00001000L
+//BIF_CFG_DEV1_RC0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_RC0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                           0x2
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                              0x00000001L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                          0x00000002L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                             0x00000004L
+//BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                      0x1
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                 0x2
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                            0x3
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                               0x4
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                 0x5
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                    0x6
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                        0x7
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                     0x1b
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                             0x00000001L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                        0x00000002L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                   0x00000004L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                              0x00000008L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                 0x00000010L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                   0x00000020L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                      0x00000040L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                          0x00000180L
+#define BIF_CFG_DEV1_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                       0xF8000000L
+//BIF_CFG_DEV1_RC0_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV1_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                    0x10
+#define BIF_CFG_DEV1_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                      0xFFFF0000L
+//BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV1_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                     0x0
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                     0x0
+#define BIF_CFG_DEV1_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV1_RC0_PCIE_ACS_CAP
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                               0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                            0x1
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                            0x2
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                         0x3
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                             0x4
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                              0x5
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                           0x6
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                             0x7
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                 0x0001L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                              0x0002L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                              0x0004L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                           0x0008L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                               0x0010L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                0x0020L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                             0x0040L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                               0x0080L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                        0xFF00L
+//BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                        0x2
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                     0x3
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                         0x7
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0x8
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0xa
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                0xc
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                          0x0004L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                       0x0008L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                         0x0040L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                           0x0080L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0300L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0C00L
+#define BIF_CFG_DEV1_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                  0x1000L
+//BIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV1_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV1_RC0_RTR_DATA1
+#define BIF_CFG_DEV1_RC0_RTR_DATA1__RESET_TIME__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_RC0_RTR_DATA1__DLUP_TIME__SHIFT                                                          0xc
+#define BIF_CFG_DEV1_RC0_RTR_DATA1__RTR_VALID__SHIFT                                                          0x1f
+#define BIF_CFG_DEV1_RC0_RTR_DATA1__RESET_TIME_MASK                                                           0x00000FFFL
+#define BIF_CFG_DEV1_RC0_RTR_DATA1__DLUP_TIME_MASK                                                            0x00FFF000L
+#define BIF_CFG_DEV1_RC0_RTR_DATA1__RTR_VALID_MASK                                                            0x80000000L
+//BIF_CFG_DEV1_RC0_RTR_DATA2
+#define BIF_CFG_DEV1_RC0_RTR_DATA2__FLR_TIME__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_RC0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV1_RC0_RTR_DATA2__FLR_TIME_MASK                                                             0x00000FFFL
+#define BIF_CFG_DEV1_RC0_RTR_DATA2__D3HOTD0_TIME_MASK                                                         0x00FFF000L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf0_bifcfgdecp
+//BIF_CFG_DEV1_EPF0_0_VENDOR_ID
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV1_EPF0_0_DEVICE_ID
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV1_EPF0_0_COMMAND
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV1_EPF0_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV1_EPF0_0_STATUS
+#define BIF_CFG_DEV1_EPF0_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV1_EPF0_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV1_EPF0_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV1_EPF0_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV1_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF0_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV1_EPF0_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV1_EPF0_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV1_EPF0_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV1_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV1_EPF0_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV1_EPF0_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV1_EPF0_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV1_EPF0_0_REVISION_ID
+#define BIF_CFG_DEV1_EPF0_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV1_EPF0_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV1_EPF0_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV1_EPF0_0_PROG_INTERFACE
+#define BIF_CFG_DEV1_EPF0_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV1_EPF0_0_SUB_CLASS
+#define BIF_CFG_DEV1_EPF0_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF0_0_BASE_CLASS
+#define BIF_CFG_DEV1_EPF0_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV1_EPF0_0_CACHE_LINE
+#define BIF_CFG_DEV1_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV1_EPF0_0_LATENCY
+#define BIF_CFG_DEV1_EPF0_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV1_EPF0_0_HEADER
+#define BIF_CFG_DEV1_EPF0_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV1_EPF0_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV1_EPF0_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV1_EPF0_0_BIST
+#define BIF_CFG_DEV1_EPF0_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV1_EPF0_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV1_EPF0_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV1_EPF0_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV1_EPF0_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV1_EPF0_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV1_EPF0_0_BASE_ADDR_1
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_BASE_ADDR_2
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_BASE_ADDR_3
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_BASE_ADDR_4
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_BASE_ADDR_5
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_BASE_ADDR_6
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV1_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_ADAPTER_ID
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV1_EPF0_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV1_EPF0_0_CAP_PTR
+#define BIF_CFG_DEV1_EPF0_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV1_EPF0_0_INTERRUPT_LINE
+#define BIF_CFG_DEV1_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV1_EPF0_0_INTERRUPT_PIN
+#define BIF_CFG_DEV1_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV1_EPF0_0_MIN_GRANT
+#define BIF_CFG_DEV1_EPF0_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV1_EPF0_0_MAX_LATENCY
+#define BIF_CFG_DEV1_EPF0_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV1_EPF0_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV1_EPF0_0_ADAPTER_ID_W
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_PMI_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV1_EPF0_0_PMI_CAP
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV1_EPF0_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV1_EPF0_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV1_EPF0_0_SBRN
+#define BIF_CFG_DEV1_EPF0_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV1_EPF0_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV1_EPF0_0_FLADJ
+#define BIF_CFG_DEV1_EPF0_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV1_EPF0_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV1_EPF0_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV1_EPF0_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV1_EPF0_0_DBESL_DBESLD
+#define BIF_CFG_DEV1_EPF0_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV1_EPF0_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV1_EPF0_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV1_EPF0_0_PCIE_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV1_EPF0_0_DEVICE_CAP
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS2
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF0_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV1_EPF0_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_MASK
+#define BIF_CFG_DEV1_EPF0_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_MASK_64
+#define BIF_CFG_DEV1_EPF0_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_PENDING
+#define BIF_CFG_DEV1_EPF0_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSI_PENDING_64
+#define BIF_CFG_DEV1_EPF0_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF0_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV1_EPF0_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV1_EPF0_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV1_EPF0_0_MSIX_TABLE
+#define BIF_CFG_DEV1_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV1_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF0_0_MSIX_PBA
+#define BIF_CFG_DEV1_EPF0_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV1_EPF0_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_RTR_DATA1
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV1_EPF0_0_RTR_DATA2
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV1_EPF0_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                      0xa
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                     0x1a
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                        0x000003FFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                        0x00001C00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                       0x03FF0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                       0x1C000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf0_bifcfgpciedecp
+//BIF_CFG_DEV1_EPF0_0_LINK_CAP
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV1_EPF0_0_LINK_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV1_EPF0_0_LINK_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV1_EPF0_0_LINK_CAP2
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV1_EPF0_0_LINK_CNTL2
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV1_EPF0_0_LINK_STATUS2
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                              0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                   0x9
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                        0x00000001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                0x00000002L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                     0x0000FE00L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                             0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                               0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_LINK_CAP_16GT
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP_16GT__RESERVED__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_CAP_16GT__RESERVED_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_LINK_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL_16GT__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_CNTL_16GT__RESERVED_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                               0x0
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                         0x1
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                         0x2
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                         0x3
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                           0x4
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                           0x00000002L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                           0x00000004L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                           0x00000008L
+#define BIF_CFG_DEV1_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                             0x00000010L
+//BIF_CFG_DEV1_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT       0x0
+#define BIF_CFG_DEV1_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK         0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV1_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV1_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV1_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV1_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV1_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                0x0
+#define BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                  0x0001L
+//BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                            0x1
+#define BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                       0x0001L
+#define BIF_CFG_DEV1_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                              0x0002L
+//BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV1_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                           0x4
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                           0xa
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                          0x00000007L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                             0x00000070L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                               0x00000300L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                             0x00000C00L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                           0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                         0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                             0x000EL
+//BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                   0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                     0x0001L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x007F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x003F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                 0x1f
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                   0x80000000L
+//BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                       0x1f
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                               0x007FFFFFL
+#define BIF_CFG_DEV1_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                         0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev1_epf1_bifcfgdecp
+//BIF_CFG_DEV1_EPF1_0_VENDOR_ID
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_DEVICE_ID
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_COMMAND
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV1_EPF1_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV1_EPF1_0_STATUS
+#define BIF_CFG_DEV1_EPF1_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV1_EPF1_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV1_EPF1_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV1_EPF1_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV1_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF1_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV1_EPF1_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV1_EPF1_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV1_EPF1_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV1_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV1_EPF1_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV1_EPF1_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV1_EPF1_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV1_EPF1_0_REVISION_ID
+#define BIF_CFG_DEV1_EPF1_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV1_EPF1_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV1_EPF1_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV1_EPF1_0_PROG_INTERFACE
+#define BIF_CFG_DEV1_EPF1_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV1_EPF1_0_SUB_CLASS
+#define BIF_CFG_DEV1_EPF1_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF1_0_BASE_CLASS
+#define BIF_CFG_DEV1_EPF1_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV1_EPF1_0_CACHE_LINE
+#define BIF_CFG_DEV1_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV1_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV1_EPF1_0_LATENCY
+#define BIF_CFG_DEV1_EPF1_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV1_EPF1_0_HEADER
+#define BIF_CFG_DEV1_EPF1_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV1_EPF1_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV1_EPF1_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV1_EPF1_0_BIST
+#define BIF_CFG_DEV1_EPF1_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV1_EPF1_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV1_EPF1_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV1_EPF1_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV1_EPF1_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV1_EPF1_0_BASE_ADDR_1
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_BASE_ADDR_2
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_BASE_ADDR_3
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_BASE_ADDR_4
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_BASE_ADDR_5
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_BASE_ADDR_6
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV1_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_ADAPTER_ID
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV1_EPF1_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV1_EPF1_0_CAP_PTR
+#define BIF_CFG_DEV1_EPF1_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV1_EPF1_0_INTERRUPT_LINE
+#define BIF_CFG_DEV1_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV1_EPF1_0_INTERRUPT_PIN
+#define BIF_CFG_DEV1_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV1_EPF1_0_MIN_GRANT
+#define BIF_CFG_DEV1_EPF1_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV1_EPF1_0_MAX_LATENCY
+#define BIF_CFG_DEV1_EPF1_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV1_EPF1_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV1_EPF1_0_ADAPTER_ID_W
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PMI_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV1_EPF1_0_PMI_CAP
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV1_EPF1_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV1_EPF1_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV1_EPF1_0_SBRN
+#define BIF_CFG_DEV1_EPF1_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV1_EPF1_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV1_EPF1_0_FLADJ
+#define BIF_CFG_DEV1_EPF1_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV1_EPF1_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV1_EPF1_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV1_EPF1_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV1_EPF1_0_DBESL_DBESLD
+#define BIF_CFG_DEV1_EPF1_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV1_EPF1_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV1_EPF1_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV1_EPF1_0_PCIE_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV1_EPF1_0_DEVICE_CAP
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV1_EPF1_0_LINK_CAP
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV1_EPF1_0_LINK_CNTL
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV1_EPF1_0_LINK_STATUS
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS2
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF1_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_LINK_CAP2
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV1_EPF1_0_LINK_CNTL2
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV1_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV1_EPF1_0_LINK_STATUS2
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV1_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV1_EPF1_0_MSI_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV1_EPF1_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV1_EPF1_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_MASK
+#define BIF_CFG_DEV1_EPF1_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_MASK_64
+#define BIF_CFG_DEV1_EPF1_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_PENDING
+#define BIF_CFG_DEV1_EPF1_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSI_PENDING_64
+#define BIF_CFG_DEV1_EPF1_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV1_EPF1_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV1_EPF1_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV1_EPF1_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV1_EPF1_0_MSIX_TABLE
+#define BIF_CFG_DEV1_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV1_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF1_0_MSIX_PBA
+#define BIF_CFG_DEV1_EPF1_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV1_EPF1_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_RTR_DATA1
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV1_EPF1_0_RTR_DATA2
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV1_EPF1_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                          0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                            0x15
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                       0x00000001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                            0x00000002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                              0xFFE00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                              0x1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                         0x2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                           0x3
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                  0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                          0x0001L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                0x0002L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                           0x0004L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                             0x0008L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                  0x0010L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                    0x0020L
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_STATUS
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                 0x0001L
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_INITIAL_VFS
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                    0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_TOTAL_VFS
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                        0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_NUM_VFS
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                            0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FUNC_DEP_LINK
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                              0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                0xFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FIRST_VF_OFFSET
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                          0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                            0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_STRIDE
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                      0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                        0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_DEVICE_ID
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                  0xFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                  0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                    0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                          0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_1
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_2
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_3
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV1_EPF1_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_rc_bifcfgdecp
+//BIF_CFG_DEV2_RC0_VENDOR_ID
+#define BIF_CFG_DEV2_RC0_VENDOR_ID__VENDOR_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_VENDOR_ID__VENDOR_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV2_RC0_DEVICE_ID
+#define BIF_CFG_DEV2_RC0_DEVICE_ID__DEVICE_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_DEVICE_ID__DEVICE_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV2_RC0_COMMAND
+#define BIF_CFG_DEV2_RC0_COMMAND__IOEN_DN__SHIFT                                                              0x0
+#define BIF_CFG_DEV2_RC0_COMMAND__MEMEN_DN__SHIFT                                                             0x1
+#define BIF_CFG_DEV2_RC0_COMMAND__BUS_MASTER_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV2_RC0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV2_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV2_RC0_COMMAND__PAL_SNOOP_EN__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_RC0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                0x6
+#define BIF_CFG_DEV2_RC0_COMMAND__AD_STEPPING__SHIFT                                                          0x7
+#define BIF_CFG_DEV2_RC0_COMMAND__SERR_EN__SHIFT                                                              0x8
+#define BIF_CFG_DEV2_RC0_COMMAND__FAST_B2B_EN__SHIFT                                                          0x9
+#define BIF_CFG_DEV2_RC0_COMMAND__INT_DIS__SHIFT                                                              0xa
+#define BIF_CFG_DEV2_RC0_COMMAND__IOEN_DN_MASK                                                                0x0001L
+#define BIF_CFG_DEV2_RC0_COMMAND__MEMEN_DN_MASK                                                               0x0002L
+#define BIF_CFG_DEV2_RC0_COMMAND__BUS_MASTER_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV2_RC0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV2_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV2_RC0_COMMAND__PAL_SNOOP_EN_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_RC0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_RC0_COMMAND__AD_STEPPING_MASK                                                            0x0080L
+#define BIF_CFG_DEV2_RC0_COMMAND__SERR_EN_MASK                                                                0x0100L
+#define BIF_CFG_DEV2_RC0_COMMAND__FAST_B2B_EN_MASK                                                            0x0200L
+#define BIF_CFG_DEV2_RC0_COMMAND__INT_DIS_MASK                                                                0x0400L
+//BIF_CFG_DEV2_RC0_STATUS
+#define BIF_CFG_DEV2_RC0_STATUS__IMMEDIATE_READINESS__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC0_STATUS__INT_STATUS__SHIFT                                                            0x3
+#define BIF_CFG_DEV2_RC0_STATUS__CAP_LIST__SHIFT                                                              0x4
+#define BIF_CFG_DEV2_RC0_STATUS__PCI_66_CAP__SHIFT                                                            0x5
+#define BIF_CFG_DEV2_RC0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                     0x7
+#define BIF_CFG_DEV2_RC0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                              0x8
+#define BIF_CFG_DEV2_RC0_STATUS__DEVSEL_TIMING__SHIFT                                                         0x9
+#define BIF_CFG_DEV2_RC0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_RC0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_RC0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                 0xd
+#define BIF_CFG_DEV2_RC0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_RC0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_RC0_STATUS__IMMEDIATE_READINESS_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_RC0_STATUS__INT_STATUS_MASK                                                              0x0008L
+#define BIF_CFG_DEV2_RC0_STATUS__CAP_LIST_MASK                                                                0x0010L
+#define BIF_CFG_DEV2_RC0_STATUS__PCI_66_CAP_MASK                                                              0x0020L
+#define BIF_CFG_DEV2_RC0_STATUS__FAST_BACK_CAPABLE_MASK                                                       0x0080L
+#define BIF_CFG_DEV2_RC0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                0x0100L
+#define BIF_CFG_DEV2_RC0_STATUS__DEVSEL_TIMING_MASK                                                           0x0600L
+#define BIF_CFG_DEV2_RC0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_RC0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                   0x1000L
+#define BIF_CFG_DEV2_RC0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                   0x2000L
+#define BIF_CFG_DEV2_RC0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                   0x4000L
+#define BIF_CFG_DEV2_RC0_STATUS__PARITY_ERROR_DETECTED_MASK                                                   0x8000L
+//BIF_CFG_DEV2_RC0_REVISION_ID
+#define BIF_CFG_DEV2_RC0_REVISION_ID__MINOR_REV_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_RC0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_RC0_REVISION_ID__MINOR_REV_ID_MASK                                                       0x0FL
+#define BIF_CFG_DEV2_RC0_REVISION_ID__MAJOR_REV_ID_MASK                                                       0xF0L
+//BIF_CFG_DEV2_RC0_PROG_INTERFACE
+#define BIF_CFG_DEV2_RC0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_RC0_SUB_CLASS
+#define BIF_CFG_DEV2_RC0_SUB_CLASS__SUB_CLASS__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_SUB_CLASS__SUB_CLASS_MASK                                                            0xFFL
+//BIF_CFG_DEV2_RC0_BASE_CLASS
+#define BIF_CFG_DEV2_RC0_BASE_CLASS__BASE_CLASS__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_BASE_CLASS__BASE_CLASS_MASK                                                          0xFFL
+//BIF_CFG_DEV2_RC0_CACHE_LINE
+#define BIF_CFG_DEV2_RC0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                     0xFFL
+//BIF_CFG_DEV2_RC0_LATENCY
+#define BIF_CFG_DEV2_RC0_LATENCY__LATENCY_TIMER__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_LATENCY__LATENCY_TIMER_MASK                                                          0xFFL
+//BIF_CFG_DEV2_RC0_HEADER
+#define BIF_CFG_DEV2_RC0_HEADER__HEADER_TYPE__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC0_HEADER__DEVICE_TYPE__SHIFT                                                           0x7
+#define BIF_CFG_DEV2_RC0_HEADER__HEADER_TYPE_MASK                                                             0x7FL
+#define BIF_CFG_DEV2_RC0_HEADER__DEVICE_TYPE_MASK                                                             0x80L
+//BIF_CFG_DEV2_RC0_BIST
+#define BIF_CFG_DEV2_RC0_BIST__BIST_COMP__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_RC0_BIST__BIST_STRT__SHIFT                                                               0x6
+#define BIF_CFG_DEV2_RC0_BIST__BIST_CAP__SHIFT                                                                0x7
+#define BIF_CFG_DEV2_RC0_BIST__BIST_COMP_MASK                                                                 0x0FL
+#define BIF_CFG_DEV2_RC0_BIST__BIST_STRT_MASK                                                                 0x40L
+#define BIF_CFG_DEV2_RC0_BIST__BIST_CAP_MASK                                                                  0x80L
+//BIF_CFG_DEV2_RC0_BASE_ADDR_1
+#define BIF_CFG_DEV2_RC0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_BASE_ADDR_1__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_BASE_ADDR_2
+#define BIF_CFG_DEV2_RC0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_BASE_ADDR_2__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                         0x8
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                               0x18
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                           0x0000FF00L
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                             0x00FF0000L
+#define BIF_CFG_DEV2_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                 0xFF000000L
+//BIF_CFG_DEV2_RC0_IO_BASE_LIMIT
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_BASE__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                     0x000FL
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_BASE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                    0x0F00L
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT__IO_LIMIT_MASK                                                         0xF000L
+//BIF_CFG_DEV2_RC0_SECONDARY_STATUS
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                  0x5
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                           0x7
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                    0x8
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                               0x9
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                         0xb
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                       0xc
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                       0xd
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                       0xe
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                       0xf
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__PCI_66_CAP_MASK                                                    0x0020L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                             0x0080L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                      0x0100L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                 0x0600L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                           0x0800L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                         0x1000L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                         0x2000L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                         0x4000L
+#define BIF_CFG_DEV2_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                         0x8000L
+//BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                0x4
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                0x10
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                               0x14
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                   0x0000000FL
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                  0x0000FFF0L
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV2_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                          0x4
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                          0x10
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                         0x14
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                             0x0000000FL
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                            0x0000FFF0L
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                            0x000F0000L
+#define BIF_CFG_DEV2_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                           0xFFF00000L
+//BIF_CFG_DEV2_RC0_PREF_BASE_UPPER
+#define BIF_CFG_DEV2_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV2_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                              0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                              0x10
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                0xFFFF0000L
+//BIF_CFG_DEV2_RC0_CAP_PTR
+#define BIF_CFG_DEV2_RC0_CAP_PTR__CAP_PTR__SHIFT                                                              0x0
+#define BIF_CFG_DEV2_RC0_CAP_PTR__CAP_PTR_MASK                                                                0xFFL
+//BIF_CFG_DEV2_RC0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                          0x1
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                         0x4
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                      0xb
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                       0x00000001L
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                            0x0000000EL
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                           0x000000F0L
+#define BIF_CFG_DEV2_RC0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                        0xFFFFF800L
+//BIF_CFG_DEV2_RC0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_RC0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_RC0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_RC0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                    0xFFL
+//BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                      0x1
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                       0x3
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                            0x5
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                          0x6
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                        0x8
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                      0x9
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                         0xa
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                    0xb
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                        0x0002L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                         0x0008L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                        0x0010L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                              0x0020L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                            0x0040L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                          0x0100L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                        0x0200L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                           0x0400L
+#define BIF_CFG_DEV2_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                      0x0800L
+//BIF_CFG_DEV2_RC0_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV2_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                  0x01L
+//BIF_CFG_DEV2_RC0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PMI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV2_RC0_PMI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV2_RC0_PMI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV2_RC0_PMI_CAP
+#define BIF_CFG_DEV2_RC0_PMI_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV2_RC0_PMI_CAP__PME_CLOCK__SHIFT                                                            0x3
+#define BIF_CFG_DEV2_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                  0x4
+#define BIF_CFG_DEV2_RC0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_RC0_PMI_CAP__AUX_CURRENT__SHIFT                                                          0x6
+#define BIF_CFG_DEV2_RC0_PMI_CAP__D1_SUPPORT__SHIFT                                                           0x9
+#define BIF_CFG_DEV2_RC0_PMI_CAP__D2_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_RC0_PMI_CAP__PME_SUPPORT__SHIFT                                                          0xb
+#define BIF_CFG_DEV2_RC0_PMI_CAP__VERSION_MASK                                                                0x0007L
+#define BIF_CFG_DEV2_RC0_PMI_CAP__PME_CLOCK_MASK                                                              0x0008L
+#define BIF_CFG_DEV2_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                    0x0010L
+#define BIF_CFG_DEV2_RC0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_RC0_PMI_CAP__AUX_CURRENT_MASK                                                            0x01C0L
+#define BIF_CFG_DEV2_RC0_PMI_CAP__D1_SUPPORT_MASK                                                             0x0200L
+#define BIF_CFG_DEV2_RC0_PMI_CAP__D2_SUPPORT_MASK                                                             0x0400L
+#define BIF_CFG_DEV2_RC0_PMI_CAP__PME_SUPPORT_MASK                                                            0xF800L
+//BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                0x3
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                       0x8
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                   0xd
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                   0xf
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                0x16
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                   0x17
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                     0x18
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                    0x00000003L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                  0x00000008L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__PME_EN_MASK                                                         0x00000100L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                    0x00001E00L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                     0x00006000L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                     0x00008000L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                  0x00400000L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                     0x00800000L
+#define BIF_CFG_DEV2_RC0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                       0xFF000000L
+//BIF_CFG_DEV2_RC0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV2_RC0_PCIE_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV2_RC0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV2_RC0_PCIE_CAP
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                         0x4
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                     0x9
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__VERSION_MASK                                                               0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__DEVICE_TYPE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                      0x0100L
+#define BIF_CFG_DEV2_RC0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                       0x3E00L
+//BIF_CFG_DEV2_RC0_DEVICE_CAP
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                      0x3
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                            0x6
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                             0x9
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                          0xf
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                          0x10
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                         0x12
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                         0x1a
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                       0x1c
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                 0x00000007L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                        0x00000018L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__EXTENDED_TAG_MASK                                                        0x00000020L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                              0x000001C0L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                               0x00000E00L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                            0x00008000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                            0x00010000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                           0x03FC0000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                           0x0C000000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP__FLR_CAPABLE_MASK                                                         0x10000000L
+//BIF_CFG_DEV2_RC0_DEVICE_CNTL
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                 0x1
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                  0xa
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                      0xb
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                            0xc
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                              0xf
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                   0x0002L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                     0x0010L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                   0x00E0L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                    0x0100L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                    0x0200L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                    0x0400L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                        0x0800L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                              0x7000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                0x8000L
+//BIF_CFG_DEV2_RC0_DEVICE_STATUS
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__CORR_ERR__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                  0x1
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                      0x2
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__AUX_PWR__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                              0x5
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                  0x6
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__CORR_ERR_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                    0x0002L
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__FATAL_ERR_MASK                                                        0x0004L
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__USR_DETECTED_MASK                                                     0x0008L
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__AUX_PWR_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                0x0020L
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                    0x0040L
+//BIF_CFG_DEV2_RC0_LINK_CAP
+#define BIF_CFG_DEV2_RC0_LINK_CAP__LINK_SPEED__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_LINK_CAP__LINK_WIDTH__SHIFT                                                          0x4
+#define BIF_CFG_DEV2_RC0_LINK_CAP__PM_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV2_RC0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_RC0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                              0x12
+#define BIF_CFG_DEV2_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                         0x13
+#define BIF_CFG_DEV2_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                         0x14
+#define BIF_CFG_DEV2_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                            0x15
+#define BIF_CFG_DEV2_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                         0x16
+#define BIF_CFG_DEV2_RC0_LINK_CAP__PORT_NUMBER__SHIFT                                                         0x18
+#define BIF_CFG_DEV2_RC0_LINK_CAP__LINK_SPEED_MASK                                                            0x0000000FL
+#define BIF_CFG_DEV2_RC0_LINK_CAP__LINK_WIDTH_MASK                                                            0x000003F0L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__PM_SUPPORT_MASK                                                            0x00000C00L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                      0x00007000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                       0x00038000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                0x00040000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                           0x00080000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                           0x00100000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                              0x00200000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                           0x00400000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP__PORT_NUMBER_MASK                                                           0xFF000000L
+//BIF_CFG_DEV2_RC0_LINK_CNTL
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__PM_CONTROL__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                       0x2
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__LINK_DIS__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                       0x5
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                   0x6
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                      0x7
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                          0xb
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                              0xe
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__PM_CONTROL_MASK                                                           0x0003L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                         0x0004L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__LINK_DIS_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__RETRAIN_LINK_MASK                                                         0x0020L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                     0x0040L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__EXTENDED_SYNC_MASK                                                        0x0080L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                            0x0100L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                            0x0400L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                            0x0800L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                0xC000L
+//BIF_CFG_DEV2_RC0_LINK_STATUS
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                            0x4
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__LINK_TRAINING__SHIFT                                                    0xb
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__DL_ACTIVE__SHIFT                                                        0xd
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                        0xe
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                        0xf
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                 0x000FL
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                              0x03F0L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__LINK_TRAINING_MASK                                                      0x0800L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__DL_ACTIVE_MASK                                                          0x2000L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                          0x4000L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                          0x8000L
+//BIF_CFG_DEV2_RC0_SLOT_CAP
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                              0x1
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                              0x3
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                               0x4
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                     0x6
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                0x7
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                0xf
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                       0x11
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                      0x12
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                   0x13
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                   0x00000001L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                0x00000002L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                    0x00000004L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                0x00000008L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                 0x00000010L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                      0x00000020L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                       0x00000040L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                  0x00007F80L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                  0x00018000L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                         0x00020000L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                        0x00040000L
+#define BIF_CFG_DEV2_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                     0xFFF80000L
+//BIF_CFG_DEV2_RC0_SLOT_CNTL
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                              0x2
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                         0x3
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                0x6
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                0xa
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                         0xb
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                0xc
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                        0xd
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                  0xe
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                               0x0001L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                0x0004L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                           0x0008L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                  0x00C0L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                   0x0300L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                  0x0400L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                           0x0800L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                          0x2000L
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                    0x4000L
+//BIF_CFG_DEV2_RC0_SLOT_STATUS
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                               0x1
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                               0x2
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                          0x3
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                0x4
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                            0x6
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                     0x7
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                0x0001L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                 0x0004L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                            0x0008L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                              0x0040L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                       0x0080L
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                   0x0100L
+//BIF_CFG_DEV2_RC0_ROOT_CNTL
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV2_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                           0x0010L
+//BIF_CFG_DEV2_RC0_ROOT_CAP
+#define BIF_CFG_DEV2_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                               0x0001L
+//BIF_CFG_DEV2_RC0_ROOT_STATUS
+#define BIF_CFG_DEV2_RC0_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_ROOT_STATUS__PME_STATUS__SHIFT                                                       0x10
+#define BIF_CFG_DEV2_RC0_ROOT_STATUS__PME_PENDING__SHIFT                                                      0x11
+#define BIF_CFG_DEV2_RC0_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_ROOT_STATUS__PME_STATUS_MASK                                                         0x00010000L
+#define BIF_CFG_DEV2_RC0_ROOT_STATUS__PME_PENDING_MASK                                                        0x00020000L
+//BIF_CFG_DEV2_RC0_DEVICE_CAP2
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                        0x4
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                         0x5
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                       0x6
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                       0x7
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                       0x8
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                           0x9
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                        0xa
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                    0xb
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                               0xc
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                    0xe
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                  0x10
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                  0x11
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                   0x12
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                     0x14
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                     0x15
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                         0x16
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                   0x18
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                    0x1a
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                    0x1f
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                        0x0000000FL
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                          0x00000010L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                           0x00000020L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                         0x00000080L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                             0x00000200L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                          0x00000400L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                      0x00000800L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                 0x00003000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                      0x0000C000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                    0x00010000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                    0x00020000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                     0x000C0000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                       0x00100000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                       0x00200000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                           0x00C00000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                     0x03000000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                      0x04000000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                      0x80000000L
+//BIF_CFG_DEV2_RC0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                               0x5
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                        0x7
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                              0x8
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                           0x9
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__LTR_EN__SHIFT                                                          0xa
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                    0xb
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                    0xc
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                         0xd
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                     0xf
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                 0x000FL
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                   0x0010L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                 0x0020L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                               0x0040L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                          0x0080L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                0x0100L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                             0x0200L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__LTR_EN_MASK                                                            0x0400L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                      0x0800L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                      0x1000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__OBFF_EN_MASK                                                           0x6000L
+#define BIF_CFG_DEV2_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                       0x8000L
+//BIF_CFG_DEV2_RC0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS2__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_RC0_DEVICE_STATUS2__RESERVED_MASK                                                        0xFFFFL
+//BIF_CFG_DEV2_RC0_LINK_CAP2
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                               0x1
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                0x8
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                           0x9
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                          0x17
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                          0x18
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                      0x1f
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                 0x000000FEL
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                  0x00000100L
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                             0x0000FE00L
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                             0x007F0000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                            0x00800000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                            0x01000000L
+#define BIF_CFG_DEV2_RC0_LINK_CAP2__DRS_SUPPORTED_MASK                                                        0x80000000L
+//BIF_CFG_DEV2_RC0_LINK_CNTL2
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                       0x5
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                             0x6
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                              0xa
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                    0xb
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                             0xc
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                   0x000FL
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                    0x0010L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                         0x0020L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                               0x0040L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__XMIT_MARGIN_MASK                                                         0x0380L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                0x0400L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                      0x0800L
+#define BIF_CFG_DEV2_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                               0xF000L
+//BIF_CFG_DEV2_RC0_LINK_STATUS2
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                       0x1
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                 0x2
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                 0x3
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                 0x4
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                   0x5
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                               0x6
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                               0x7
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                            0x8
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                0xa
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                   0xc
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                            0xf
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                              0x0001L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                         0x0002L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                   0x0004L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                   0x0008L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                   0x0010L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                     0x0020L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                 0x0040L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                 0x0080L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                              0x0300L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                  0x0400L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                     0x7000L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                              0x8000L
+//BIF_CFG_DEV2_RC0_SLOT_CAP2
+#define BIF_CFG_DEV2_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                        0x0
+#define BIF_CFG_DEV2_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                          0x00000001L
+//BIF_CFG_DEV2_RC0_SLOT_CNTL2
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL2__RESERVED__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_SLOT_CNTL2__RESERVED_MASK                                                            0xFFFFL
+//BIF_CFG_DEV2_RC0_SLOT_STATUS2
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS2__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_SLOT_STATUS2__RESERVED_MASK                                                          0xFFFFL
+//BIF_CFG_DEV2_RC0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_RC0_MSI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV2_RC0_MSI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV2_RC0_MSI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV2_RC0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                   0x1
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                       0x8
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                            0x9
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_EN_MASK                                                            0x0001L
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                     0x000EL
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                      0x0070L
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                         0x0100L
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                              0x0200L
+#define BIF_CFG_DEV2_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                               0x0400L
+//BIF_CFG_DEV2_RC0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                              0x2
+#define BIF_CFG_DEV2_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                0xFFFFFFFCL
+//BIF_CFG_DEV2_RC0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_RC0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_MSI_MSG_DATA__MSI_DATA_MASK                                                          0xFFFFL
+//BIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                  0xFFFFL
+//BIF_CFG_DEV2_RC0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_RC0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                    0xFFFFL
+//BIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                            0xFFFFL
+//BIF_CFG_DEV2_RC0_SSID_CAP_LIST
+#define BIF_CFG_DEV2_RC0_SSID_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC0_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV2_RC0_SSID_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV2_RC0_SSID_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV2_RC0_SSID_CAP
+#define BIF_CFG_DEV2_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                        0x10
+#define BIF_CFG_DEV2_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_SSID_CAP__SUBSYSTEM_ID_MASK                                                          0xFFFF0000L
+//BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                            0x10
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                         0x14
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                              0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                           0xFFF00000L
+//BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                         0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                 0x1
+#define BIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                      0x9
+#define BIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                           0x00000001L
+#define BIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                   0x00000002L
+#define BIF_CFG_DEV2_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                        0x0000FE00L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                  0x0000FFFFL
+//BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV2_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_RC0_LINK_CAP_16GT
+#define BIF_CFG_DEV2_RC0_LINK_CAP_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_RC0_LINK_CAP_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_LINK_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LINK_CNTL_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_RC0_LINK_CNTL_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_LINK_STATUS_16GT
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                  0x0
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                            0x1
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                            0x2
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                            0x3
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                              0x00000002L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                              0x00000004L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                              0x00000008L
+#define BIF_CFG_DEV2_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                0x00000010L
+//BIF_CFG_DEV2_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT          0x0
+#define BIF_CFG_DEV2_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK            0x0000FFFFL
+//BIF_CFG_DEV2_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV2_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV2_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV2_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV2_RC0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV2_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV2_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV2_RC0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV2_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV2_RC0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV2_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV2_RC0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV2_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV2_RC0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV2_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV2_RC0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV2_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV2_RC0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV2_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV2_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV2_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV2_RC0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV2_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                   0x0
+#define BIF_CFG_DEV2_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                     0x0001L
+//BIF_CFG_DEV2_RC0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV2_RC0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                        0x0
+#define BIF_CFG_DEV2_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                               0x1
+#define BIF_CFG_DEV2_RC0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                          0x0001L
+#define BIF_CFG_DEV2_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                 0x0002L
+//BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV2_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                              0x4
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                0x8
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                              0xa
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                             0x00000007L
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                0x00000070L
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                  0x00000300L
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                0x00000C00L
+//BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                             0x0
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                    0x18
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                               0x000000FFL
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                      0xFF000000L
+//BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                              0x1
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                0x000EL
+//BIF_CFG_DEV2_RC0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                      0x0
+#define BIF_CFG_DEV2_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                        0x0001L
+//BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x007F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV2_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x003F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV2_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                    0x0
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                    0x1f
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                      0x007FFFFFL
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                      0x80000000L
+//BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                0x0
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                          0x1f
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                  0x007FFFFFL
+#define BIF_CFG_DEV2_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                            0x80000000L
+//BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                        0x4
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                     0x5
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                        0xc
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                         0xd
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                      0x10
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                       0x11
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                        0x12
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                       0x13
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                 0x14
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                  0x15
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                 0x16
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                 0x17
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                        0x18
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                         0x19
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                    0x1a
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                          0x00000010L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                       0x00000020L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                          0x00001000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                           0x00002000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                    0x00008000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                        0x00010000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                         0x00020000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                          0x00040000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                         0x00080000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                   0x00100000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                   0x00400000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                   0x00800000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                          0x01000000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                           0x02000000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                      0x04000000L
+//BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                            0x4
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                         0x5
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                            0xc
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                             0xd
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                          0x10
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                           0x11
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                            0x12
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                           0x13
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                     0x14
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                      0x15
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                     0x16
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                     0x17
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                            0x18
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                             0x19
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                        0x1a
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                              0x00000010L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                           0x00000020L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                              0x00001000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                               0x00002000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                        0x00008000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                            0x00010000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                             0x00020000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                              0x00040000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                             0x00080000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                       0x00100000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                        0x00200000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                       0x00400000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                       0x00800000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                              0x01000000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                               0x02000000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                          0x04000000L
+//BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                    0x4
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                 0x5
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                    0xc
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                     0xd
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                0xe
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                              0xf
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                  0x10
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                   0x11
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                    0x12
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                   0x13
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                             0x14
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                              0x15
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                             0x16
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                             0x17
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                    0x18
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                     0x19
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                0x1a
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                      0x00000010L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                   0x00000020L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                      0x00001000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                       0x00002000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                  0x00004000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                0x00008000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                    0x00010000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                     0x00020000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                      0x00040000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                     0x00080000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                               0x00100000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                0x00200000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                               0x00400000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                               0x00800000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                      0x01000000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                       0x02000000L
+#define BIF_CFG_DEV2_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                  0x04000000L
+//BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                         0x7
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                              0x8
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                             0xc
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                            0xd
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                            0x00000001L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                            0x00000040L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                           0x00000080L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                0x00000100L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                               0x00001000L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                              0x00002000L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                       0x00008000L
+//BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                              0x6
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                             0x7
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                  0x8
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                 0xc
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                0xd
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                         0xf
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                0x00000001L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                0x00000040L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                               0x00000080L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                    0x00000100L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                  0x00002000L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV2_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                           0x00008000L
+//BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                          0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                           0x5
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                         0x7
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                     0x9
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                      0xa
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                 0xb
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                         0xc
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                            0x0000001FL
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                             0x00000020L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                              0x00000040L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                           0x00000080L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                            0x00000100L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                       0x00000200L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                        0x00000400L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                   0x00000800L
+#define BIF_CFG_DEV2_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                           0x00001000L
+//BIF_CFG_DEV2_RC0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_RC0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                           0x2
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                              0x00000001L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                          0x00000002L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                             0x00000004L
+//BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                      0x1
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                 0x2
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                            0x3
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                               0x4
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                 0x5
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                    0x6
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                        0x7
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                     0x1b
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                        0x00000002L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                   0x00000004L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                              0x00000008L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                 0x00000010L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                   0x00000020L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                          0x00000180L
+#define BIF_CFG_DEV2_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                       0xF8000000L
+//BIF_CFG_DEV2_RC0_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV2_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                    0x10
+#define BIF_CFG_DEV2_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                      0xFFFF0000L
+//BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV2_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                     0x0
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                     0x0
+#define BIF_CFG_DEV2_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV2_RC0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                               0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                            0x1
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                            0x2
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                         0x3
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                             0x4
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                              0x5
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                           0x6
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                             0x7
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                 0x0001L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                              0x0002L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                              0x0004L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                           0x0008L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                               0x0010L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                0x0020L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                             0x0040L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                               0x0080L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                        0xFF00L
+//BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                        0x2
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                     0x3
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                         0x7
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0x8
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0xa
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                0xc
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                          0x0004L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                       0x0008L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                         0x0040L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                           0x0080L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0300L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0C00L
+#define BIF_CFG_DEV2_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                  0x1000L
+//BIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV2_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV2_RC0_RTR_DATA1
+#define BIF_CFG_DEV2_RC0_RTR_DATA1__RESET_TIME__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_RC0_RTR_DATA1__DLUP_TIME__SHIFT                                                          0xc
+#define BIF_CFG_DEV2_RC0_RTR_DATA1__RTR_VALID__SHIFT                                                          0x1f
+#define BIF_CFG_DEV2_RC0_RTR_DATA1__RESET_TIME_MASK                                                           0x00000FFFL
+#define BIF_CFG_DEV2_RC0_RTR_DATA1__DLUP_TIME_MASK                                                            0x00FFF000L
+#define BIF_CFG_DEV2_RC0_RTR_DATA1__RTR_VALID_MASK                                                            0x80000000L
+//BIF_CFG_DEV2_RC0_RTR_DATA2
+#define BIF_CFG_DEV2_RC0_RTR_DATA2__FLR_TIME__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_RC0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_RC0_RTR_DATA2__FLR_TIME_MASK                                                             0x00000FFFL
+#define BIF_CFG_DEV2_RC0_RTR_DATA2__D3HOTD0_TIME_MASK                                                         0x00FFF000L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf0_bifcfgdecp
+//BIF_CFG_DEV2_EPF0_0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF0_0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF0_0_COMMAND
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV2_EPF0_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV2_EPF0_0_STATUS
+#define BIF_CFG_DEV2_EPF0_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF0_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_EPF0_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_EPF0_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF0_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_EPF0_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV2_EPF0_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF0_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV2_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF0_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV2_EPF0_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF0_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV2_EPF0_0_REVISION_ID
+#define BIF_CFG_DEV2_EPF0_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF0_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV2_EPF0_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV2_EPF0_0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF0_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF0_0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF0_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF0_0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF0_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF0_0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_EPF0_0_LATENCY
+#define BIF_CFG_DEV2_EPF0_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF0_0_HEADER
+#define BIF_CFG_DEV2_EPF0_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_EPF0_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV2_EPF0_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV2_EPF0_0_BIST
+#define BIF_CFG_DEV2_EPF0_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF0_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV2_EPF0_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV2_EPF0_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV2_EPF0_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV2_EPF0_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV2_EPF0_0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV2_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV2_EPF0_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV2_EPF0_0_CAP_PTR
+#define BIF_CFG_DEV2_EPF0_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF0_0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF0_0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF0_0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF0_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF0_0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF0_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV2_EPF0_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV2_EPF0_0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF0_0_PMI_CAP
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV2_EPF0_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV2_EPF0_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV2_EPF0_0_SBRN
+#define BIF_CFG_DEV2_EPF0_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV2_EPF0_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV2_EPF0_0_FLADJ
+#define BIF_CFG_DEV2_EPF0_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_EPF0_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV2_EPF0_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV2_EPF0_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV2_EPF0_0_DBESL_DBESLD
+#define BIF_CFG_DEV2_EPF0_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF0_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV2_EPF0_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF0_0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV2_EPF0_0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF0_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF0_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_MASK
+#define BIF_CFG_DEV2_EPF0_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF0_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF0_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF0_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF0_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF0_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF0_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV2_EPF0_0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV2_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF0_0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF0_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF0_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_EPF0_0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF0_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                      0xa
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                     0x1a
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                        0x000003FFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                        0x00001C00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                       0x03FF0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                       0x1C000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf0_bifcfgpciedecp
+//BIF_CFG_DEV2_EPF0_0_LINK_CAP
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_EPF0_0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV2_EPF0_0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV2_EPF0_0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF0_0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV2_EPF0_0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                              0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                   0x9
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                        0x00000001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                0x00000002L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                     0x0000FE00L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                             0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                               0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_LINK_CAP_16GT
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP_16GT__RESERVED__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_CAP_16GT__RESERVED_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_LINK_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL_16GT__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_CNTL_16GT__RESERVED_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                               0x0
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                         0x1
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                         0x2
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                         0x3
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                           0x4
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                           0x00000002L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                           0x00000004L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                           0x00000008L
+#define BIF_CFG_DEV2_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                             0x00000010L
+//BIF_CFG_DEV2_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT       0x0
+#define BIF_CFG_DEV2_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK         0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV2_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV2_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV2_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV2_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV2_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                0x0
+#define BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                  0x0001L
+//BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                            0x1
+#define BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                              0x0002L
+//BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV2_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                           0x4
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                           0xa
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                          0x00000007L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                             0x00000070L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                               0x00000300L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                             0x00000C00L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                           0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                         0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                             0x000EL
+//BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                     0x0001L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x007F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x003F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                 0x1f
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                   0x80000000L
+//BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                       0x1f
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                               0x007FFFFFL
+#define BIF_CFG_DEV2_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                         0x80000000L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf1_bifcfgdecp
+//BIF_CFG_DEV2_EPF1_0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF1_0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF1_0_COMMAND
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV2_EPF1_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV2_EPF1_0_STATUS
+#define BIF_CFG_DEV2_EPF1_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF1_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF1_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_EPF1_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_EPF1_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF1_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_EPF1_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV2_EPF1_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF1_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV2_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF1_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV2_EPF1_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF1_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV2_EPF1_0_REVISION_ID
+#define BIF_CFG_DEV2_EPF1_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF1_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV2_EPF1_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV2_EPF1_0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF1_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF1_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF1_0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF1_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF1_0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF1_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF1_0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF1_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_EPF1_0_LATENCY
+#define BIF_CFG_DEV2_EPF1_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF1_0_HEADER
+#define BIF_CFG_DEV2_EPF1_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF1_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_EPF1_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV2_EPF1_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV2_EPF1_0_BIST
+#define BIF_CFG_DEV2_EPF1_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF1_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV2_EPF1_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV2_EPF1_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV2_EPF1_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV2_EPF1_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV2_EPF1_0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV2_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV2_EPF1_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV2_EPF1_0_CAP_PTR
+#define BIF_CFG_DEV2_EPF1_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF1_0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF1_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF1_0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF1_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF1_0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF1_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF1_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF1_0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF1_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV2_EPF1_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV2_EPF1_0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF1_0_PMI_CAP
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV2_EPF1_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV2_EPF1_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV2_EPF1_0_SBRN
+#define BIF_CFG_DEV2_EPF1_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV2_EPF1_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV2_EPF1_0_FLADJ
+#define BIF_CFG_DEV2_EPF1_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_EPF1_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV2_EPF1_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV2_EPF1_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV2_EPF1_0_DBESL_DBESLD
+#define BIF_CFG_DEV2_EPF1_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF1_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF1_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV2_EPF1_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV2_EPF1_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF1_0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV2_EPF1_0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV2_EPF1_0_LINK_CAP
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_EPF1_0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV2_EPF1_0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF1_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_EPF1_0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF1_0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF1_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV2_EPF1_0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV2_EPF1_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV2_EPF1_0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF1_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF1_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_MASK
+#define BIF_CFG_DEV2_EPF1_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF1_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF1_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF1_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF1_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF1_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF1_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF1_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF1_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV2_EPF1_0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV2_EPF1_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF1_0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF1_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF1_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_EPF1_0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF1_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF1_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf2_bifcfgdecp
+//BIF_CFG_DEV2_EPF2_0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF2_0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF2_0_COMMAND
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV2_EPF2_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV2_EPF2_0_STATUS
+#define BIF_CFG_DEV2_EPF2_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF2_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF2_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_EPF2_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_EPF2_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_EPF2_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF2_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_EPF2_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV2_EPF2_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF2_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV2_EPF2_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF2_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV2_EPF2_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF2_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV2_EPF2_0_REVISION_ID
+#define BIF_CFG_DEV2_EPF2_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF2_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV2_EPF2_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV2_EPF2_0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF2_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF2_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF2_0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF2_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF2_0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF2_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF2_0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF2_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF2_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_EPF2_0_LATENCY
+#define BIF_CFG_DEV2_EPF2_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF2_0_HEADER
+#define BIF_CFG_DEV2_EPF2_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF2_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_EPF2_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV2_EPF2_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV2_EPF2_0_BIST
+#define BIF_CFG_DEV2_EPF2_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF2_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV2_EPF2_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV2_EPF2_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV2_EPF2_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV2_EPF2_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV2_EPF2_0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV2_EPF2_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV2_EPF2_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV2_EPF2_0_CAP_PTR
+#define BIF_CFG_DEV2_EPF2_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF2_0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF2_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF2_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF2_0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF2_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF2_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF2_0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF2_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF2_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF2_0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF2_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV2_EPF2_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV2_EPF2_0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF2_0_PMI_CAP
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV2_EPF2_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV2_EPF2_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF2_0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV2_EPF2_0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV2_EPF2_0_LINK_CAP
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_EPF2_0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV2_EPF2_0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF2_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_EPF2_0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF2_0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF2_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV2_EPF2_0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV2_EPF2_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV2_EPF2_0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF2_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF2_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_MASK
+#define BIF_CFG_DEV2_EPF2_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF2_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF2_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF2_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF2_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF2_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF2_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF2_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF2_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV2_EPF2_0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF2_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF2_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV2_EPF2_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF2_0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF2_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF2_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_EPF2_0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF2_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF2_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf3_bifcfgdecp
+//BIF_CFG_DEV2_EPF3_0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF3_0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF3_0_COMMAND
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV2_EPF3_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV2_EPF3_0_STATUS
+#define BIF_CFG_DEV2_EPF3_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF3_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF3_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_EPF3_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_EPF3_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_EPF3_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF3_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_EPF3_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV2_EPF3_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF3_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV2_EPF3_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF3_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV2_EPF3_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF3_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV2_EPF3_0_REVISION_ID
+#define BIF_CFG_DEV2_EPF3_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF3_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV2_EPF3_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV2_EPF3_0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF3_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF3_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF3_0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF3_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF3_0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF3_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF3_0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF3_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF3_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_EPF3_0_LATENCY
+#define BIF_CFG_DEV2_EPF3_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF3_0_HEADER
+#define BIF_CFG_DEV2_EPF3_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF3_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_EPF3_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV2_EPF3_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV2_EPF3_0_BIST
+#define BIF_CFG_DEV2_EPF3_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF3_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV2_EPF3_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV2_EPF3_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV2_EPF3_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV2_EPF3_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV2_EPF3_0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV2_EPF3_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV2_EPF3_0_CAP_PTR
+#define BIF_CFG_DEV2_EPF3_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF3_0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF3_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF3_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF3_0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF3_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF3_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF3_0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF3_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF3_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF3_0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF3_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV2_EPF3_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV2_EPF3_0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF3_0_PMI_CAP
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV2_EPF3_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV2_EPF3_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV2_EPF3_0_SBRN
+#define BIF_CFG_DEV2_EPF3_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV2_EPF3_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV2_EPF3_0_FLADJ
+#define BIF_CFG_DEV2_EPF3_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_EPF3_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV2_EPF3_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV2_EPF3_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV2_EPF3_0_DBESL_DBESLD
+#define BIF_CFG_DEV2_EPF3_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF3_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF3_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV2_EPF3_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV2_EPF3_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF3_0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV2_EPF3_0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV2_EPF3_0_LINK_CAP
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_EPF3_0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV2_EPF3_0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF3_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_EPF3_0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF3_0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF3_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV2_EPF3_0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV2_EPF3_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV2_EPF3_0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF3_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF3_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_MASK
+#define BIF_CFG_DEV2_EPF3_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF3_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF3_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF3_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF3_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF3_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF3_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF3_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF3_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV2_EPF3_0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF3_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF3_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV2_EPF3_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF3_0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF3_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF3_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_EPF3_0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF3_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF3_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf4_bifcfgdecp
+//BIF_CFG_DEV2_EPF4_0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF4_0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF4_0_COMMAND
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV2_EPF4_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV2_EPF4_0_STATUS
+#define BIF_CFG_DEV2_EPF4_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF4_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF4_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_EPF4_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_EPF4_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_EPF4_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF4_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_EPF4_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV2_EPF4_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF4_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV2_EPF4_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF4_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV2_EPF4_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF4_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV2_EPF4_0_REVISION_ID
+#define BIF_CFG_DEV2_EPF4_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF4_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV2_EPF4_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV2_EPF4_0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF4_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF4_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF4_0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF4_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF4_0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF4_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF4_0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF4_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF4_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_EPF4_0_LATENCY
+#define BIF_CFG_DEV2_EPF4_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF4_0_HEADER
+#define BIF_CFG_DEV2_EPF4_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF4_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_EPF4_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV2_EPF4_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV2_EPF4_0_BIST
+#define BIF_CFG_DEV2_EPF4_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF4_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV2_EPF4_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV2_EPF4_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV2_EPF4_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV2_EPF4_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV2_EPF4_0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV2_EPF4_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV2_EPF4_0_CAP_PTR
+#define BIF_CFG_DEV2_EPF4_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF4_0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF4_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF4_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF4_0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF4_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF4_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF4_0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF4_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF4_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF4_0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF4_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV2_EPF4_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV2_EPF4_0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF4_0_PMI_CAP
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV2_EPF4_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV2_EPF4_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV2_EPF4_0_SBRN
+#define BIF_CFG_DEV2_EPF4_0_SBRN__SBRN__SHIFT                                                                 0x0
+#define BIF_CFG_DEV2_EPF4_0_SBRN__SBRN_MASK                                                                   0xFFL
+//BIF_CFG_DEV2_EPF4_0_FLADJ
+#define BIF_CFG_DEV2_EPF4_0_FLADJ__FLADJ__SHIFT                                                               0x0
+#define BIF_CFG_DEV2_EPF4_0_FLADJ__NFC__SHIFT                                                                 0x6
+#define BIF_CFG_DEV2_EPF4_0_FLADJ__FLADJ_MASK                                                                 0x3FL
+#define BIF_CFG_DEV2_EPF4_0_FLADJ__NFC_MASK                                                                   0x40L
+//BIF_CFG_DEV2_EPF4_0_DBESL_DBESLD
+#define BIF_CFG_DEV2_EPF4_0_DBESL_DBESLD__DBESL__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF4_0_DBESL_DBESLD__DBESLD__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF4_0_DBESL_DBESLD__DBESL_MASK                                                          0x0FL
+#define BIF_CFG_DEV2_EPF4_0_DBESL_DBESLD__DBESLD_MASK                                                         0xF0L
+//BIF_CFG_DEV2_EPF4_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF4_0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV2_EPF4_0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV2_EPF4_0_LINK_CAP
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_EPF4_0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV2_EPF4_0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF4_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_EPF4_0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF4_0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF4_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV2_EPF4_0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV2_EPF4_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV2_EPF4_0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF4_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF4_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_MASK
+#define BIF_CFG_DEV2_EPF4_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF4_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF4_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF4_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF4_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF4_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF4_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF4_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF4_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV2_EPF4_0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF4_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF4_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV2_EPF4_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF4_0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF4_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF4_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_EPF4_0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF4_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF4_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf5_bifcfgdecp
+//BIF_CFG_DEV2_EPF5_0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF5_0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF5_0_COMMAND
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV2_EPF5_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV2_EPF5_0_STATUS
+#define BIF_CFG_DEV2_EPF5_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF5_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF5_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_EPF5_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_EPF5_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_EPF5_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF5_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_EPF5_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV2_EPF5_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF5_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV2_EPF5_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF5_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV2_EPF5_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF5_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV2_EPF5_0_REVISION_ID
+#define BIF_CFG_DEV2_EPF5_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF5_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV2_EPF5_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV2_EPF5_0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF5_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF5_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF5_0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF5_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF5_0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF5_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF5_0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF5_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF5_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_EPF5_0_LATENCY
+#define BIF_CFG_DEV2_EPF5_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF5_0_HEADER
+#define BIF_CFG_DEV2_EPF5_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF5_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_EPF5_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV2_EPF5_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV2_EPF5_0_BIST
+#define BIF_CFG_DEV2_EPF5_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF5_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV2_EPF5_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV2_EPF5_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV2_EPF5_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV2_EPF5_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV2_EPF5_0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV2_EPF5_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV2_EPF5_0_CAP_PTR
+#define BIF_CFG_DEV2_EPF5_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF5_0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF5_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF5_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF5_0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF5_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF5_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF5_0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF5_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF5_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF5_0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF5_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV2_EPF5_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV2_EPF5_0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF5_0_PMI_CAP
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV2_EPF5_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV2_EPF5_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF5_0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV2_EPF5_0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV2_EPF5_0_LINK_CAP
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_EPF5_0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV2_EPF5_0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF5_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_EPF5_0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF5_0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF5_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV2_EPF5_0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV2_EPF5_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV2_EPF5_0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF5_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF5_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_MASK
+#define BIF_CFG_DEV2_EPF5_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF5_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF5_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF5_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF5_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF5_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF5_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF5_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF5_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV2_EPF5_0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF5_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF5_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV2_EPF5_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF5_0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF5_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF5_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_EPF5_0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF5_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF5_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_bif_cfg_dev2_epf6_bifcfgdecp
+//BIF_CFG_DEV2_EPF6_0_VENDOR_ID
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF6_0_DEVICE_ID
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV2_EPF6_0_COMMAND
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV2_EPF6_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV2_EPF6_0_STATUS
+#define BIF_CFG_DEV2_EPF6_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF6_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF6_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV2_EPF6_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV2_EPF6_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV2_EPF6_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF6_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV2_EPF6_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV2_EPF6_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF6_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV2_EPF6_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF6_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV2_EPF6_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF6_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV2_EPF6_0_REVISION_ID
+#define BIF_CFG_DEV2_EPF6_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV2_EPF6_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV2_EPF6_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV2_EPF6_0_PROG_INTERFACE
+#define BIF_CFG_DEV2_EPF6_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF6_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF6_0_SUB_CLASS
+#define BIF_CFG_DEV2_EPF6_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF6_0_BASE_CLASS
+#define BIF_CFG_DEV2_EPF6_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF6_0_CACHE_LINE
+#define BIF_CFG_DEV2_EPF6_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV2_EPF6_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV2_EPF6_0_LATENCY
+#define BIF_CFG_DEV2_EPF6_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV2_EPF6_0_HEADER
+#define BIF_CFG_DEV2_EPF6_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF6_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV2_EPF6_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV2_EPF6_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV2_EPF6_0_BIST
+#define BIF_CFG_DEV2_EPF6_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV2_EPF6_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV2_EPF6_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV2_EPF6_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV2_EPF6_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV2_EPF6_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV2_EPF6_0_BASE_ADDR_1
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_BASE_ADDR_2
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_BASE_ADDR_3
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_BASE_ADDR_4
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_BASE_ADDR_5
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_BASE_ADDR_6
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_ADAPTER_ID
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV2_EPF6_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV2_EPF6_0_CAP_PTR
+#define BIF_CFG_DEV2_EPF6_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV2_EPF6_0_INTERRUPT_LINE
+#define BIF_CFG_DEV2_EPF6_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF6_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV2_EPF6_0_INTERRUPT_PIN
+#define BIF_CFG_DEV2_EPF6_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF6_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV2_EPF6_0_MIN_GRANT
+#define BIF_CFG_DEV2_EPF6_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF6_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV2_EPF6_0_MAX_LATENCY
+#define BIF_CFG_DEV2_EPF6_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV2_EPF6_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV2_EPF6_0_ADAPTER_ID_W
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_PMI_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF6_0_PMI_CAP
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV2_EPF6_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV2_EPF6_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF6_0_PCIE_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+//BIF_CFG_DEV2_EPF6_0_DEVICE_CAP
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+//BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV2_EPF6_0_LINK_CAP
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV2_EPF6_0_LINK_CNTL
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV2_EPF6_0_LINK_STATUS
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS2
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF6_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV2_EPF6_0_LINK_CAP2
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV2_EPF6_0_LINK_CNTL2
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV2_EPF6_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV2_EPF6_0_LINK_STATUS2
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV2_EPF6_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV2_EPF6_0_MSI_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV2_EPF6_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV2_EPF6_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_MASK
+#define BIF_CFG_DEV2_EPF6_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_MASK_64
+#define BIF_CFG_DEV2_EPF6_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_PENDING
+#define BIF_CFG_DEV2_EPF6_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSI_PENDING_64
+#define BIF_CFG_DEV2_EPF6_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF6_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF6_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV2_EPF6_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV2_EPF6_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV2_EPF6_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV2_EPF6_0_MSIX_TABLE
+#define BIF_CFG_DEV2_EPF6_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF6_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV2_EPF6_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF6_0_MSIX_PBA
+#define BIF_CFG_DEV2_EPF6_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV2_EPF6_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_RTR_DATA1
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV2_EPF6_0_RTR_DATA2
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV2_EPF6_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV2_EPF6_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+
+
+// addressBlock: nbif0_rcc_dev0_RCCPORTDEC
+//RCC_DEV0_0_RCC_VDM_SUPPORT
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_0_RCC_BUS_CNTL
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_0_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_0_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_0_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_0_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_0_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                            0x10
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                              0x00FF0000L
+//RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_0_RCC_MH_ARB_CNTL
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x0000FFFEL
+//RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+
+
+// addressBlock: nbif0_rcc_ep_dev0_RCCPORTDEC
+//RCC_EP_DEV0_0_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_0_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_0_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_0_EP_PCIE_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_0_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_0_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                  0x5
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                    0x00000020L
+//RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_0_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_0_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_0_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_0_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_0_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                          0x4
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                            0x00000010L
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_RCCPORTDEC
+//RCC_DWN_DEV0_0_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_0_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_0_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_0_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_0_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                 0x5
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                   0x00000020L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_RCCPORTDEC
+//RCC_DWNP_DEV0_0_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_0_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                           0x4
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                             0x00000010L
+//RCC_DWNP_DEV0_0_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_pfc_amdgfx_RCCPFCDEC
+//RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                           0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                           0xa
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                             0xf
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                        0x10
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                        0x1a
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                          0x1f
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                             0x000003FFL
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                             0x00001C00L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                               0x00008000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                          0x03FF0000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                          0x1C000000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                            0x80000000L
+//RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                         0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                     0x8
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                              0x9
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                           0x00000001L
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                       0x00000100L
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                0x00000200L
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                            0x1
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                          0x2
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                              0x3
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                0x4
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                               0x5
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                         0x6
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                  0x7
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                  0x00000001L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                              0x00000002L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                            0x00000004L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                0x00000008L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                  0x00000010L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                 0x00000020L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                           0x00000040L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                    0x00000080L
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                    0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                      0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                       0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                0x3
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                         0x00000007L
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                  0x00000008L
+
+
+// addressBlock: nbif0_rcc_pfc_amdgfxaz_RCCPFCDEC
+//RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                         0xa
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                           0xf
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                      0x10
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                      0x1a
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                        0x1f
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                           0x000003FFL
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                           0x00001C00L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                             0x00008000L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                        0x03FF0000L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                        0x1C000000L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                          0x80000000L
+//RCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                       0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                   0x8
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                            0x9
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                         0x00000001L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                     0x00000100L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                              0x00000200L
+//RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                              0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                          0x1
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                        0x2
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                            0x3
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                              0x4
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                             0x5
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                       0x6
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                0x7
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                0x00000001L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                            0x00000002L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                          0x00000004L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                              0x00000008L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                0x00000010L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                               0x00000020L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                         0x00000040L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                  0x00000080L
+//RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                   0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                     0xFFFFFFFFL
+//RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                   0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                     0xFFFFFFFFL
+//RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                   0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                     0xFFFFFFFFL
+//RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                   0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                     0xFFFFFFFFL
+//RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                  0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                    0xFFFFFFFFL
+//RCC_PFC_AMDGFXAZ_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                     0x0
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                              0x3
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                       0x00000007L
+#define RCC_PFC_AMDGFXAZ_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                0x00000008L
+
+
+// addressBlock: nbif0_rcc_pfc_psp_RCCPFCDEC
+//RCC_PFC_PSP_RCC_PFC_LTR_CNTL
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                              0x0
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                              0xa
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                                0xf
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                           0x10
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                           0x1a
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                             0x1f
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                                0x000003FFL
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                                0x00001C00L
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                                  0x00008000L
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                             0x03FF0000L
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                             0x1C000000L
+#define RCC_PFC_PSP_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                               0x80000000L
+//RCC_PFC_PSP_RCC_PFC_PME_RESTORE
+#define RCC_PFC_PSP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                            0x0
+#define RCC_PFC_PSP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                        0x8
+#define RCC_PFC_PSP_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                                 0x9
+#define RCC_PFC_PSP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                              0x00000001L
+#define RCC_PFC_PSP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                          0x00000100L
+#define RCC_PFC_PSP_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                   0x00000200L
+//RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                   0x0
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                               0x1
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                             0x2
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                                 0x3
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                   0x4
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                                  0x5
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                            0x6
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                     0x7
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                     0x00000001L
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                                 0x00000002L
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                               0x00000004L
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                   0x00000008L
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                     0x00000010L
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                    0x00000020L
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                              0x00000040L
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                       0x00000080L
+//RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                        0x0
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                          0xFFFFFFFFL
+//RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                        0x0
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                          0xFFFFFFFFL
+//RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                        0x0
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                          0xFFFFFFFFL
+//RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                        0x0
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                          0xFFFFFFFFL
+//RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                       0x0
+#define RCC_PFC_PSP_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                         0xFFFFFFFFL
+//RCC_PFC_PSP_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_PSP_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                          0x0
+#define RCC_PFC_PSP_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                   0x3
+#define RCC_PFC_PSP_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                            0x00000007L
+#define RCC_PFC_PSP_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                     0x00000008L
+
+
+// addressBlock: nbif0_rcc_pfc_usb3_1_RCCPFCDEC
+//RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                           0x0
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                           0xa
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                             0xf
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                        0x10
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                        0x1a
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                          0x1f
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                             0x000003FFL
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                             0x00001C00L
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                               0x00008000L
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                          0x03FF0000L
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                          0x1C000000L
+#define RCC_PFC_USB3_1_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                            0x80000000L
+//RCC_PFC_USB3_1_RCC_PFC_PME_RESTORE
+#define RCC_PFC_USB3_1_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                         0x0
+#define RCC_PFC_USB3_1_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                     0x8
+#define RCC_PFC_USB3_1_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                              0x9
+#define RCC_PFC_USB3_1_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                           0x00000001L
+#define RCC_PFC_USB3_1_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                       0x00000100L
+#define RCC_PFC_USB3_1_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                0x00000200L
+//RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                0x0
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                            0x1
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                          0x2
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                              0x3
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                0x4
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                               0x5
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                         0x6
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                  0x7
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                  0x00000001L
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                              0x00000002L
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                            0x00000004L
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                0x00000008L
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                  0x00000010L
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                 0x00000020L
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                           0x00000040L
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                    0x00000080L
+//RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                     0x0
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                     0x0
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                     0x0
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                     0x0
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                    0x0
+#define RCC_PFC_USB3_1_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                      0xFFFFFFFFL
+//RCC_PFC_USB3_1_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_USB3_1_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                       0x0
+#define RCC_PFC_USB3_1_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                0x3
+#define RCC_PFC_USB3_1_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                         0x00000007L
+#define RCC_PFC_USB3_1_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                  0x00000008L
+
+
+// addressBlock: nbif0_rcc_pfc_acp_RCCPFCDEC
+//RCC_PFC_ACP_RCC_PFC_LTR_CNTL
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                              0x0
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                              0xa
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                                0xf
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                           0x10
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                           0x1a
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                             0x1f
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                                0x000003FFL
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                                0x00001C00L
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                                  0x00008000L
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                             0x03FF0000L
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                             0x1C000000L
+#define RCC_PFC_ACP_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                               0x80000000L
+//RCC_PFC_ACP_RCC_PFC_PME_RESTORE
+#define RCC_PFC_ACP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                            0x0
+#define RCC_PFC_ACP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                        0x8
+#define RCC_PFC_ACP_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                                 0x9
+#define RCC_PFC_ACP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                              0x00000001L
+#define RCC_PFC_ACP_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                          0x00000100L
+#define RCC_PFC_ACP_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                   0x00000200L
+//RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                   0x0
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                               0x1
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                             0x2
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                                 0x3
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                   0x4
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                                  0x5
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                            0x6
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                     0x7
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                     0x00000001L
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                                 0x00000002L
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                               0x00000004L
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                   0x00000008L
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                     0x00000010L
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                    0x00000020L
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                              0x00000040L
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                       0x00000080L
+//RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                        0x0
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                          0xFFFFFFFFL
+//RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                        0x0
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                          0xFFFFFFFFL
+//RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                        0x0
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                          0xFFFFFFFFL
+//RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                        0x0
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                          0xFFFFFFFFL
+//RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                       0x0
+#define RCC_PFC_ACP_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                         0xFFFFFFFFL
+//RCC_PFC_ACP_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_ACP_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                          0x0
+#define RCC_PFC_ACP_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                   0x3
+#define RCC_PFC_ACP_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                            0x00000007L
+#define RCC_PFC_ACP_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                     0x00000008L
+
+
+// addressBlock: nbif0_rcc_pfc_az_RCCPFCDEC
+//RCC_PFC_AZ_RCC_PFC_LTR_CNTL
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                               0x0
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                               0xa
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                                 0xf
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                            0x10
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                            0x1a
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                              0x1f
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                                 0x000003FFL
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                                 0x00001C00L
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                                   0x00008000L
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                              0x03FF0000L
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                              0x1C000000L
+#define RCC_PFC_AZ_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                                0x80000000L
+//RCC_PFC_AZ_RCC_PFC_PME_RESTORE
+#define RCC_PFC_AZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                             0x0
+#define RCC_PFC_AZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                         0x8
+#define RCC_PFC_AZ_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                                  0x9
+#define RCC_PFC_AZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                               0x00000001L
+#define RCC_PFC_AZ_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                           0x00000100L
+#define RCC_PFC_AZ_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                    0x00000200L
+//RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                    0x0
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                                0x1
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                              0x2
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                                  0x3
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                    0x4
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                                   0x5
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                             0x6
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                      0x7
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                      0x00000001L
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                                  0x00000002L
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                                0x00000004L
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                    0x00000008L
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                      0x00000010L
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                     0x00000020L
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                               0x00000040L
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                        0x00000080L
+//RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                         0x0
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                           0xFFFFFFFFL
+//RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                         0x0
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                           0xFFFFFFFFL
+//RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                         0x0
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                           0xFFFFFFFFL
+//RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                         0x0
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                           0xFFFFFFFFL
+//RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                        0x0
+#define RCC_PFC_AZ_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                          0xFFFFFFFFL
+//RCC_PFC_AZ_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_AZ_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                           0x0
+#define RCC_PFC_AZ_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                    0x3
+#define RCC_PFC_AZ_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                             0x00000007L
+#define RCC_PFC_AZ_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                      0x00000008L
+
+
+// addressBlock: nbif0_rcc_pfc_mp2_RCCPFCDEC
+//RCC_PFC_MP2_RCC_PFC_LTR_CNTL
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                              0x0
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                              0xa
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                                0xf
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                           0x10
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                           0x1a
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                             0x1f
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                                0x000003FFL
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                                0x00001C00L
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                                  0x00008000L
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                             0x03FF0000L
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                             0x1C000000L
+#define RCC_PFC_MP2_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                               0x80000000L
+//RCC_PFC_MP2_RCC_PFC_PME_RESTORE
+#define RCC_PFC_MP2_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                            0x0
+#define RCC_PFC_MP2_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                        0x8
+#define RCC_PFC_MP2_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                                 0x9
+#define RCC_PFC_MP2_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                              0x00000001L
+#define RCC_PFC_MP2_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                          0x00000100L
+#define RCC_PFC_MP2_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                   0x00000200L
+//RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                   0x0
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                               0x1
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                             0x2
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                                 0x3
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                   0x4
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                                  0x5
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                            0x6
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                     0x7
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                     0x00000001L
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                                 0x00000002L
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                               0x00000004L
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                   0x00000008L
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                     0x00000010L
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                    0x00000020L
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                              0x00000040L
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                       0x00000080L
+//RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                        0x0
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                          0xFFFFFFFFL
+//RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                        0x0
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                          0xFFFFFFFFL
+//RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                        0x0
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                          0xFFFFFFFFL
+//RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                        0x0
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                          0xFFFFFFFFL
+//RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                       0x0
+#define RCC_PFC_MP2_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                         0xFFFFFFFFL
+//RCC_PFC_MP2_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_MP2_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                          0x0
+#define RCC_PFC_MP2_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                   0x3
+#define RCC_PFC_MP2_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                            0x00000007L
+#define RCC_PFC_MP2_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                     0x00000008L
+
+
+// addressBlock: nbif0_rcc_dev1_RCCPORTDEC
+//RCC_DEV1_RCC_VDM_SUPPORT
+#define RCC_DEV1_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                         0x0
+#define RCC_DEV1_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                        0x1
+#define RCC_DEV1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                    0x2
+#define RCC_DEV1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                          0x3
+#define RCC_DEV1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                      0x4
+#define RCC_DEV1_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                           0x00000001L
+#define RCC_DEV1_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                          0x00000002L
+#define RCC_DEV1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                      0x00000004L
+#define RCC_DEV1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                            0x00000008L
+#define RCC_DEV1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                        0x00000010L
+//RCC_DEV1_RCC_BUS_CNTL
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                              0x2
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                             0x3
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                              0x4
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                           0x5
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                          0x6
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                           0x7
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                          0x8
+#define RCC_DEV1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                   0xc
+#define RCC_DEV1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                             0xd
+#define RCC_DEV1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                            0x10
+#define RCC_DEV1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                            0x11
+#define RCC_DEV1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                            0x12
+#define RCC_DEV1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                            0x13
+#define RCC_DEV1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                            0x14
+#define RCC_DEV1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                            0x15
+#define RCC_DEV1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                   0x18
+#define RCC_DEV1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                   0x19
+#define RCC_DEV1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                              0x1c
+#define RCC_DEV1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                              0x1d
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                                0x00000004L
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                               0x00000008L
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                                0x00000010L
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                             0x00000020L
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                            0x00000040L
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                             0x00000080L
+#define RCC_DEV1_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                            0x00000100L
+#define RCC_DEV1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                     0x00001000L
+#define RCC_DEV1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                               0x00002000L
+#define RCC_DEV1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                              0x00010000L
+#define RCC_DEV1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                              0x00020000L
+#define RCC_DEV1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                              0x00040000L
+#define RCC_DEV1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                              0x00080000L
+#define RCC_DEV1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                              0x00100000L
+#define RCC_DEV1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                              0x00200000L
+#define RCC_DEV1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                     0x01000000L
+#define RCC_DEV1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                     0x0E000000L
+#define RCC_DEV1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                                0x10000000L
+#define RCC_DEV1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                                0xE0000000L
+//RCC_DEV1_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                   0x8
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                      0x9
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                      0xa
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                   0xb
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                    0xc
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                        0xd
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                        0xe
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                           0xf
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                   0x10
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                             0x11
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                                 0x12
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                       0x13
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                     0x00000100L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                        0x00000200L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                        0x00000400L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                     0x00000800L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                      0x00001000L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                          0x00002000L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                          0x00004000L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                             0x00008000L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                     0x00010000L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                               0x00020000L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                   0x00040000L
+#define RCC_DEV1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                         0x00080000L
+//RCC_DEV1_RCC_DEV0_LINK_CNTL
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                    0x0
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                   0x8
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                              0x10
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                              0x11
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                      0x00000001L
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                     0x00000100L
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                                0x00010000L
+#define RCC_DEV1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                                0x00020000L
+//RCC_DEV1_RCC_CMN_LINK_CNTL
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                               0x0
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                                0x1
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                               0x2
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                            0x3
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                               0x10
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                                 0x00000001L
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                  0x00000002L
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                                 0x00000004L
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                              0x00000008L
+#define RCC_DEV1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                                 0xFFFF0000L
+//RCC_DEV1_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                              0x0
+#define RCC_DEV1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                              0x8
+#define RCC_DEV1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                              0x10
+#define RCC_DEV1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                                0x000000FFL
+#define RCC_DEV1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                                0x00001F00L
+#define RCC_DEV1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                                0x00FF0000L
+//RCC_DEV1_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                           0x0
+#define RCC_DEV1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                             0x000003FFL
+//RCC_DEV1_RCC_MH_ARB_CNTL
+#define RCC_DEV1_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                          0x0
+#define RCC_DEV1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                  0x1
+#define RCC_DEV1_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                            0x00000001L
+#define RCC_DEV1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                    0x0000FFFEL
+//RCC_DEV1_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                   0x0
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                                0x1
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                  0x2
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                   0x3
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                             0x4
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                    0x5
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                   0xb
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                   0x12
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                  0x19
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                     0x00000001L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                  0x00000002L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                    0x00000004L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                     0x00000008L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                               0x00000010L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                      0x000007E0L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                     0x0003F800L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                     0x01FC0000L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                    0xFE000000L
+//RCC_DEV1_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                               0x0
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                                0x6
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                           0xc
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                        0x11
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                                 0x0000003FL
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                  0x00000FC0L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                             0x0001F000L
+#define RCC_DEV1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                          0x00FE0000L
+
+
+// addressBlock: nbif0_rcc_ep_dev1_RCCPORTDEC
+//RCC_EP_DEV1_EP_PCIE_SCRATCH
+#define RCC_EP_DEV1_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                      0x0
+#define RCC_EP_DEV1_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                        0xFFFFFFFFL
+//RCC_EP_DEV1_EP_PCIE_CNTL
+#define RCC_EP_DEV1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                    0x7
+#define RCC_EP_DEV1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                              0x8
+#define RCC_EP_DEV1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                                 0x1e
+#define RCC_EP_DEV1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                      0x00000080L
+#define RCC_EP_DEV1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                                0x00000100L
+#define RCC_EP_DEV1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                   0x40000000L
+//RCC_EP_DEV1_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                  0x0
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                             0x1
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                                 0x2
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                              0x3
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                  0x4
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                           0x6
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                    0x00000001L
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                               0x00000002L
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                   0x00000004L
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                                0x00000008L
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                    0x00000010L
+#define RCC_EP_DEV1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                             0x00000040L
+//RCC_EP_DEV1_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                            0x0
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                       0x1
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                           0x2
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                        0x3
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                            0x4
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                     0x6
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                  0x7
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                              0x00000001L
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                         0x00000002L
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                             0x00000004L
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                          0x00000008L
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                              0x00000010L
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                       0x00000040L
+#define RCC_EP_DEV1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                    0x00000080L
+//RCC_EP_DEV1_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                     0x0
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                       0x00000001L
+//RCC_EP_DEV1_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                                0x7
+#define RCC_EP_DEV1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                  0x00000080L
+//RCC_EP_DEV1_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                         0x0
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                    0x1
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                    0x2
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                    0x3
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                    0x4
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                    0x5
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                           0x00000001L
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                      0x00000002L
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                      0x00000004L
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                      0x00000008L
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                      0x00000010L
+#define RCC_EP_DEV1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                      0x00000020L
+//RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                        0x0
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                         0x3
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                        0x6
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                       0x7
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                        0xa
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                       0xd
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                                 0xe
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                   0xf
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                              0x10
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                     0x11
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                          0x00000007L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                           0x00000038L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                          0x00000040L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                         0x00000380L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                          0x00001C00L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                         0x00002000L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                   0x00004000L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                     0x00008000L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                                0x00010000L
+#define RCC_EP_DEV1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                       0x00020000L
+//RCC_EP_DEV1_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                             0x1d
+#define RCC_EP_DEV1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                               0x20000000L
+//RCC_EP_DEV1_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                           0x4
+#define RCC_EP_DEV1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                             0x00000010L
+//RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//RCC_EP_DEV1_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0xFFL
+//RCC_EP_DEV1_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                               0x0
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                           0x8
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                                 0x001FL
+#define RCC_EP_DEV1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                             0x0100L
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV1_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                             0x0
+#define RCC_EP_DEV1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                               0x1FL
+//RCC_EP_DEV1_EP_PCIEP_RESERVED
+#define RCC_EP_DEV1_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                  0x0
+#define RCC_EP_DEV1_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                    0xFFFFFFFFL
+//RCC_EP_DEV1_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                   0xa
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                    0xc
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                     0x18
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                     0x19
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                     0x1a
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                     0x00000C00L
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                      0x00003000L
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                       0x01000000L
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                       0x02000000L
+#define RCC_EP_DEV1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                       0x04000000L
+//RCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                  0x0
+#define RCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                    0x3
+#define RCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                       0x8
+#define RCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                    0x00000007L
+#define RCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                      0x000000F8L
+#define RCC_EP_DEV1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                         0x0000FF00L
+//RCC_EP_DEV1_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                                0x0
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                              0x8
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                         0x11
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                                 0x12
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                     0x18
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                     0x19
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                     0x1a
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                     0x1b
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                     0x1c
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                     0x1d
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                     0x1e
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                     0x1f
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                  0x00000001L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                                0x00000700L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                           0x00020000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                   0x00040000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                       0x01000000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                       0x02000000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                       0x04000000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                       0x08000000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                       0x10000000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                       0x20000000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                       0x40000000L
+#define RCC_EP_DEV1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                       0x80000000L
+//RCC_EP_DEV1_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                         0x8
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                  0x9
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                           0x14
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                         0x15
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                           0x16
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                        0x18
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                            0x19
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                        0x1a
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                           0x00000100L
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                    0x00000200L
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                             0x00100000L
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                           0x00200000L
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                             0x00400000L
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                          0x01000000L
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                              0x02000000L
+#define RCC_EP_DEV1_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                          0x04000000L
+//RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                            0x0
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                            0x1
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                            0x2
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                            0x3
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                            0x4
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                              0x00000001L
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                              0x00000002L
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                              0x00000004L
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                              0x00000008L
+#define RCC_EP_DEV1_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                              0x00000010L
+
+
+// addressBlock: nbif0_rcc_dwn_dev1_RCCPORTDEC
+//RCC_DWN_DEV1_DN_PCIE_RESERVED
+#define RCC_DWN_DEV1_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                   0x0
+#define RCC_DWN_DEV1_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV1_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV1_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                     0x0
+#define RCC_DWN_DEV1_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                       0xFFFFFFFFL
+//RCC_DWN_DEV1_DN_PCIE_CNTL
+#define RCC_DWN_DEV1_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                      0x0
+#define RCC_DWN_DEV1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                                0x7
+#define RCC_DWN_DEV1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                                0x1e
+#define RCC_DWN_DEV1_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                        0x00000001L
+#define RCC_DWN_DEV1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                  0x00000080L
+#define RCC_DWN_DEV1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                  0x40000000L
+//RCC_DWN_DEV1_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                  0x19
+#define RCC_DWN_DEV1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                    0x06000000L
+//RCC_DWN_DEV1_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                                 0x1c
+#define RCC_DWN_DEV1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                   0x70000000L
+//RCC_DWN_DEV1_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                               0x7
+#define RCC_DWN_DEV1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                     0x8
+#define RCC_DWN_DEV1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                 0x00000080L
+#define RCC_DWN_DEV1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                       0x00000100L
+//RCC_DWN_DEV1_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                        0x0
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                   0x1
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                   0x2
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                   0x3
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                   0x4
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                   0x5
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                          0x00000001L
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                     0x00000002L
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                     0x00000004L
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                     0x00000008L
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                     0x00000010L
+#define RCC_DWN_DEV1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                     0x00000020L
+//RCC_DWN_DEV1_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                     0x0
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                  0x11
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                          0x15
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                       0x00000001L
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                    0x00020000L
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                            0x00E00000L
+//RCC_DWN_DEV1_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                               0x18
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                            0x1d
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                                 0x01000000L
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                              0x20000000L
+//RCC_DWN_DEV1_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                      0x2
+#define RCC_DWN_DEV1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                        0x00000004L
+
+
+// addressBlock: nbif0_rcc_dwnp_dev1_RCCPORTDEC
+//RCC_DWNP_DEV1_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                                 0x0
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                               0x8
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                      0xb
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                          0x11
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                                 0x12
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                             0x13
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                                0x14
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                   0x00000001L
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                                 0x00000700L
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                        0x00000800L
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                            0x00020000L
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                   0x00040000L
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                               0x00080000L
+#define RCC_DWNP_DEV1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                  0x00100000L
+//RCC_DWNP_DEV1_PCIE_RX_CNTL
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                          0x8
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                                0x9
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                            0x14
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                       0x15
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                             0x1b
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                            0x00000100L
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                  0x00000200L
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                              0x00100000L
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                         0x00200000L
+#define RCC_DWNP_DEV1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                               0x08000000L
+//RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                             0x0
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                             0x1
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                             0x2
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                             0x3
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                             0x4
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                               0x00000001L
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                               0x00000002L
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                               0x00000004L
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                               0x00000008L
+#define RCC_DWNP_DEV1_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                               0x00000010L
+//RCC_DWNP_DEV1_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                                 0x0
+#define RCC_DWNP_DEV1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                       0x1b
+#define RCC_DWNP_DEV1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                   0x00000001L
+#define RCC_DWNP_DEV1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                         0x08000000L
+//RCC_DWNP_DEV1_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                            0xa
+#define RCC_DWNP_DEV1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                              0x00000400L
+//RCC_DWNP_DEV1_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                       0x0
+#define RCC_DWNP_DEV1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                         0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_dev2_RCCPORTDEC
+//RCC_DEV2_RCC_VDM_SUPPORT
+#define RCC_DEV2_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                         0x0
+#define RCC_DEV2_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                        0x1
+#define RCC_DEV2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                    0x2
+#define RCC_DEV2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                          0x3
+#define RCC_DEV2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                      0x4
+#define RCC_DEV2_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                           0x00000001L
+#define RCC_DEV2_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                          0x00000002L
+#define RCC_DEV2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                      0x00000004L
+#define RCC_DEV2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                            0x00000008L
+#define RCC_DEV2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                        0x00000010L
+//RCC_DEV2_RCC_BUS_CNTL
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                              0x2
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                             0x3
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                              0x4
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                           0x5
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                          0x6
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                           0x7
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                          0x8
+#define RCC_DEV2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                   0xc
+#define RCC_DEV2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                             0xd
+#define RCC_DEV2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                            0x10
+#define RCC_DEV2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                            0x11
+#define RCC_DEV2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                            0x12
+#define RCC_DEV2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                            0x13
+#define RCC_DEV2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                            0x14
+#define RCC_DEV2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                            0x15
+#define RCC_DEV2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                   0x18
+#define RCC_DEV2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                   0x19
+#define RCC_DEV2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                              0x1c
+#define RCC_DEV2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                              0x1d
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                                0x00000004L
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                               0x00000008L
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                                0x00000010L
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                             0x00000020L
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                            0x00000040L
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                             0x00000080L
+#define RCC_DEV2_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                            0x00000100L
+#define RCC_DEV2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                     0x00001000L
+#define RCC_DEV2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                               0x00002000L
+#define RCC_DEV2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                              0x00010000L
+#define RCC_DEV2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                              0x00020000L
+#define RCC_DEV2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                              0x00040000L
+#define RCC_DEV2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                              0x00080000L
+#define RCC_DEV2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                              0x00100000L
+#define RCC_DEV2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                              0x00200000L
+#define RCC_DEV2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                     0x01000000L
+#define RCC_DEV2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                     0x0E000000L
+#define RCC_DEV2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                                0x10000000L
+#define RCC_DEV2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                                0xE0000000L
+//RCC_DEV2_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                   0x8
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                      0x9
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                      0xa
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                   0xb
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                    0xc
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                        0xd
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                        0xe
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                           0xf
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                   0x10
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                             0x11
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                                 0x12
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                       0x13
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                     0x00000100L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                        0x00000200L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                        0x00000400L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                     0x00000800L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                      0x00001000L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                          0x00002000L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                          0x00004000L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                             0x00008000L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                     0x00010000L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                               0x00020000L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                   0x00040000L
+#define RCC_DEV2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                         0x00080000L
+//RCC_DEV2_RCC_DEV0_LINK_CNTL
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                    0x0
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                   0x8
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                              0x10
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                              0x11
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                      0x00000001L
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                     0x00000100L
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                                0x00010000L
+#define RCC_DEV2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                                0x00020000L
+//RCC_DEV2_RCC_CMN_LINK_CNTL
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                               0x0
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                                0x1
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                               0x2
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                            0x3
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                               0x10
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                                 0x00000001L
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                  0x00000002L
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                                 0x00000004L
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                              0x00000008L
+#define RCC_DEV2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                                 0xFFFF0000L
+//RCC_DEV2_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                              0x0
+#define RCC_DEV2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                              0x8
+#define RCC_DEV2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                              0x10
+#define RCC_DEV2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                                0x000000FFL
+#define RCC_DEV2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                                0x00001F00L
+#define RCC_DEV2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                                0x00FF0000L
+//RCC_DEV2_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                           0x0
+#define RCC_DEV2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                             0x000003FFL
+//RCC_DEV2_RCC_MH_ARB_CNTL
+#define RCC_DEV2_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                          0x0
+#define RCC_DEV2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                  0x1
+#define RCC_DEV2_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                            0x00000001L
+#define RCC_DEV2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                    0x0000FFFEL
+//RCC_DEV2_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                   0x0
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                                0x1
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                  0x2
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                   0x3
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                             0x4
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                    0x5
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                   0xb
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                   0x12
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                  0x19
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                     0x00000001L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                  0x00000002L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                    0x00000004L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                     0x00000008L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                               0x00000010L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                      0x000007E0L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                     0x0003F800L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                     0x01FC0000L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                    0xFE000000L
+//RCC_DEV2_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                               0x0
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                                0x6
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                           0xc
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                        0x11
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                                 0x0000003FL
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                  0x00000FC0L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                             0x0001F000L
+#define RCC_DEV2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                          0x00FE0000L
+
+
+// addressBlock: nbif0_rcc_ep_dev2_RCCPORTDEC
+//RCC_EP_DEV2_EP_PCIE_SCRATCH
+#define RCC_EP_DEV2_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                      0x0
+#define RCC_EP_DEV2_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                        0xFFFFFFFFL
+//RCC_EP_DEV2_EP_PCIE_CNTL
+#define RCC_EP_DEV2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                    0x7
+#define RCC_EP_DEV2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                              0x8
+#define RCC_EP_DEV2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                                 0x1e
+#define RCC_EP_DEV2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                      0x00000080L
+#define RCC_EP_DEV2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                                0x00000100L
+#define RCC_EP_DEV2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                   0x40000000L
+//RCC_EP_DEV2_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                  0x0
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                             0x1
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                                 0x2
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                              0x3
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                  0x4
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                           0x6
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                    0x00000001L
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                               0x00000002L
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                   0x00000004L
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                                0x00000008L
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                    0x00000010L
+#define RCC_EP_DEV2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                             0x00000040L
+//RCC_EP_DEV2_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                            0x0
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                       0x1
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                           0x2
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                        0x3
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                            0x4
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                     0x6
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                  0x7
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                              0x00000001L
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                         0x00000002L
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                             0x00000004L
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                          0x00000008L
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                              0x00000010L
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                       0x00000040L
+#define RCC_EP_DEV2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                    0x00000080L
+//RCC_EP_DEV2_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                     0x0
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                       0x00000001L
+//RCC_EP_DEV2_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                                0x7
+#define RCC_EP_DEV2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                  0x00000080L
+//RCC_EP_DEV2_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                         0x0
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                    0x1
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                    0x2
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                    0x3
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                    0x4
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                    0x5
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                           0x00000001L
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                      0x00000002L
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                      0x00000004L
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                      0x00000008L
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                      0x00000010L
+#define RCC_EP_DEV2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                      0x00000020L
+//RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                        0x0
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                         0x3
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                        0x6
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                       0x7
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                        0xa
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                       0xd
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                                 0xe
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                   0xf
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                              0x10
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                     0x11
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                          0x00000007L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                           0x00000038L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                          0x00000040L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                         0x00000380L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                          0x00001C00L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                         0x00002000L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                   0x00004000L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                     0x00008000L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                                0x00010000L
+#define RCC_EP_DEV2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                       0x00020000L
+//RCC_EP_DEV2_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                             0x1d
+#define RCC_EP_DEV2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                               0x20000000L
+//RCC_EP_DEV2_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                           0x4
+#define RCC_EP_DEV2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                             0x00000010L
+//RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//RCC_EP_DEV2_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0xFFL
+//RCC_EP_DEV2_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                               0x0
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                           0x8
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                                 0x001FL
+#define RCC_EP_DEV2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                             0x0100L
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                               0x0
+#define RCC_EP_DEV2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                                 0xFFL
+//RCC_EP_DEV2_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                             0x0
+#define RCC_EP_DEV2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                               0x1FL
+//RCC_EP_DEV2_EP_PCIEP_RESERVED
+#define RCC_EP_DEV2_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                  0x0
+#define RCC_EP_DEV2_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                    0xFFFFFFFFL
+//RCC_EP_DEV2_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                   0xa
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                    0xc
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                     0x18
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                     0x19
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                     0x1a
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                     0x00000C00L
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                      0x00003000L
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                       0x01000000L
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                       0x02000000L
+#define RCC_EP_DEV2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                       0x04000000L
+//RCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                  0x0
+#define RCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                    0x3
+#define RCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                       0x8
+#define RCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                    0x00000007L
+#define RCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                      0x000000F8L
+#define RCC_EP_DEV2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                         0x0000FF00L
+//RCC_EP_DEV2_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                                0x0
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                              0x8
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                         0x11
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                                 0x12
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                     0x18
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                     0x19
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                     0x1a
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                     0x1b
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                     0x1c
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                     0x1d
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                     0x1e
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                     0x1f
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                  0x00000001L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                                0x00000700L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                           0x00020000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                   0x00040000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                       0x01000000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                       0x02000000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                       0x04000000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                       0x08000000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                       0x10000000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                       0x20000000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                       0x40000000L
+#define RCC_EP_DEV2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                       0x80000000L
+//RCC_EP_DEV2_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                         0x8
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                  0x9
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                           0x14
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                         0x15
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                           0x16
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                        0x18
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                            0x19
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                        0x1a
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                           0x00000100L
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                    0x00000200L
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                             0x00100000L
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                           0x00200000L
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                             0x00400000L
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                          0x01000000L
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                              0x02000000L
+#define RCC_EP_DEV2_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                          0x04000000L
+//RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                            0x0
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                            0x1
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                            0x2
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                            0x3
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                            0x4
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                              0x00000001L
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                              0x00000002L
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                              0x00000004L
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                              0x00000008L
+#define RCC_EP_DEV2_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                              0x00000010L
+
+
+// addressBlock: nbif0_rcc_dwn_dev2_RCCPORTDEC
+//RCC_DWN_DEV2_DN_PCIE_RESERVED
+#define RCC_DWN_DEV2_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                   0x0
+#define RCC_DWN_DEV2_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV2_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV2_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                     0x0
+#define RCC_DWN_DEV2_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                       0xFFFFFFFFL
+//RCC_DWN_DEV2_DN_PCIE_CNTL
+#define RCC_DWN_DEV2_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                      0x0
+#define RCC_DWN_DEV2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                                0x7
+#define RCC_DWN_DEV2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                                0x1e
+#define RCC_DWN_DEV2_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                        0x00000001L
+#define RCC_DWN_DEV2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                  0x00000080L
+#define RCC_DWN_DEV2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                  0x40000000L
+//RCC_DWN_DEV2_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                  0x19
+#define RCC_DWN_DEV2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                    0x06000000L
+//RCC_DWN_DEV2_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                                 0x1c
+#define RCC_DWN_DEV2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                   0x70000000L
+//RCC_DWN_DEV2_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                               0x7
+#define RCC_DWN_DEV2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                     0x8
+#define RCC_DWN_DEV2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                 0x00000080L
+#define RCC_DWN_DEV2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                       0x00000100L
+//RCC_DWN_DEV2_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                        0x0
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                   0x1
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                   0x2
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                   0x3
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                   0x4
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                   0x5
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                          0x00000001L
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                     0x00000002L
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                     0x00000004L
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                     0x00000008L
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                     0x00000010L
+#define RCC_DWN_DEV2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                     0x00000020L
+//RCC_DWN_DEV2_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                     0x0
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                  0x11
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                          0x15
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                       0x00000001L
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                    0x00020000L
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                            0x00E00000L
+//RCC_DWN_DEV2_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                               0x18
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                            0x1d
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                                 0x01000000L
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                              0x20000000L
+//RCC_DWN_DEV2_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                      0x2
+#define RCC_DWN_DEV2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                        0x00000004L
+
+
+// addressBlock: nbif0_rcc_dwnp_dev2_RCCPORTDEC
+//RCC_DWNP_DEV2_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                                 0x0
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                               0x8
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                      0xb
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                          0x11
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                                 0x12
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                             0x13
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                                0x14
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                   0x00000001L
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                                 0x00000700L
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                        0x00000800L
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                            0x00020000L
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                   0x00040000L
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                               0x00080000L
+#define RCC_DWNP_DEV2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                  0x00100000L
+//RCC_DWNP_DEV2_PCIE_RX_CNTL
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                          0x8
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                                0x9
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                            0x14
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                       0x15
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                             0x1b
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                            0x00000100L
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                  0x00000200L
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                              0x00100000L
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                         0x00200000L
+#define RCC_DWNP_DEV2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                               0x08000000L
+//RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                             0x0
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                             0x1
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                             0x2
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                             0x3
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                             0x4
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                               0x00000001L
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                               0x00000002L
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                               0x00000004L
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                               0x00000008L
+#define RCC_DWNP_DEV2_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                               0x00000010L
+//RCC_DWNP_DEV2_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                                 0x0
+#define RCC_DWNP_DEV2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                       0x1b
+#define RCC_DWNP_DEV2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                   0x00000001L
+#define RCC_DWNP_DEV2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                         0x08000000L
+//RCC_DWNP_DEV2_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                            0xa
+#define RCC_DWNP_DEV2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                              0x00000400L
+//RCC_DWNP_DEV2_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                       0x0
+#define RCC_DWNP_DEV2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                         0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_pfc_usb3_0_RCCPFCDEC
+//RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                           0x0
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                           0xa
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                             0xf
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                        0x10
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                        0x1a
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                          0x1f
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                             0x000003FFL
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                             0x00001C00L
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                               0x00008000L
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                          0x03FF0000L
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                          0x1C000000L
+#define RCC_PFC_USB3_0_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                            0x80000000L
+//RCC_PFC_USB3_0_RCC_PFC_PME_RESTORE
+#define RCC_PFC_USB3_0_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                         0x0
+#define RCC_PFC_USB3_0_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                     0x8
+#define RCC_PFC_USB3_0_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                              0x9
+#define RCC_PFC_USB3_0_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                           0x00000001L
+#define RCC_PFC_USB3_0_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                       0x00000100L
+#define RCC_PFC_USB3_0_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                0x00000200L
+//RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                0x0
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                            0x1
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                          0x2
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                              0x3
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                0x4
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                               0x5
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                         0x6
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                  0x7
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                  0x00000001L
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                              0x00000002L
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                            0x00000004L
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                0x00000008L
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                  0x00000010L
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                 0x00000020L
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                           0x00000040L
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                    0x00000080L
+//RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                     0x0
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                     0x0
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                     0x0
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                     0x0
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                       0xFFFFFFFFL
+//RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                    0x0
+#define RCC_PFC_USB3_0_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                      0xFFFFFFFFL
+//RCC_PFC_USB3_0_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_USB3_0_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                       0x0
+#define RCC_PFC_USB3_0_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                0x3
+#define RCC_PFC_USB3_0_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                         0x00000007L
+#define RCC_PFC_USB3_0_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                  0x00000008L
+
+
+// addressBlock: nbif0_rcc_strap_rcc_strap_internal
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0__SHIFT                                     0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0_MASK                                       0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0__SHIFT                                         0x1e
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0_MASK                                           0x40000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0__SHIFT                                   0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0__SHIFT                           0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0_MASK                                     0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0_MASK                                0x03000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0_MASK                             0x0C000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                  0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                    0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0__SHIFT                                      0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0__SHIFT                                   0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0_MASK                                        0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0_MASK                                     0x00000040L
+//RCC_DEV1_PORT_STRAP0
+#define RCC_DEV1_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV1__SHIFT                                                  0x0
+#define RCC_DEV1_PORT_STRAP0__STRAP_ARI_EN_DN_DEV1__SHIFT                                                     0x10
+#define RCC_DEV1_PORT_STRAP0__STRAP_ACS_EN_DN_DEV1__SHIFT                                                     0x11
+#define RCC_DEV1_PORT_STRAP0__STRAP_AER_EN_DN_DEV1__SHIFT                                                     0x12
+#define RCC_DEV1_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV1__SHIFT                                           0x13
+#define RCC_DEV1_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV1__SHIFT                                                0x14
+#define RCC_DEV1_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV1__SHIFT                                       0x18
+#define RCC_DEV1_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV1__SHIFT                                        0x19
+#define RCC_DEV1_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV1__SHIFT                                        0x1c
+#define RCC_DEV1_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV1__SHIFT                                                 0x1f
+#define RCC_DEV1_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV1_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_PORT_STRAP0__STRAP_ARI_EN_DN_DEV1_MASK                                                       0x00010000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_ACS_EN_DN_DEV1_MASK                                                       0x00020000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_AER_EN_DN_DEV1_MASK                                                       0x00040000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV1_MASK                                             0x00080000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV1_MASK                                                  0x00100000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV1_MASK                                         0x01000000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV1_MASK                                          0x0E000000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV1_MASK                                          0x70000000L
+#define RCC_DEV1_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV1_MASK                                                   0x80000000L
+//RCC_DEV1_PORT_STRAP1
+#define RCC_DEV1_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV1__SHIFT                                                  0x0
+#define RCC_DEV1_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV1__SHIFT                                              0x10
+#define RCC_DEV1_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV1_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV1_MASK                                                0xFFFF0000L
+//RCC_DEV1_PORT_STRAP2
+#define RCC_DEV1_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV1__SHIFT                                            0x0
+#define RCC_DEV1_PORT_STRAP2__STRAP_DSN_EN_DN_DEV1__SHIFT                                                     0x1
+#define RCC_DEV1_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV1__SHIFT                                                 0x2
+#define RCC_DEV1_PORT_STRAP2__STRAP_ECN1P1_EN_DEV1__SHIFT                                                     0x3
+#define RCC_DEV1_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV1__SHIFT                                                 0x4
+#define RCC_DEV1_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV1__SHIFT                                                   0x5
+#define RCC_DEV1_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV1__SHIFT                                        0x7
+#define RCC_DEV1_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV1__SHIFT                                           0x8
+#define RCC_DEV1_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV1__SHIFT                                               0x9
+#define RCC_DEV1_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV1__SHIFT                                         0xc
+#define RCC_DEV1_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV1__SHIFT                                 0xd
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV1__SHIFT                                               0xe
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN2_EN_DEV1__SHIFT                                                       0xf
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV1__SHIFT                                               0x10
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV1__SHIFT                                               0x11
+#define RCC_DEV1_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV1__SHIFT                                        0x14
+#define RCC_DEV1_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV1__SHIFT                                              0x17
+#define RCC_DEV1_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV1__SHIFT                                         0x1a
+#define RCC_DEV1_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV1__SHIFT                                               0x1d
+#define RCC_DEV1_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV1_MASK                                              0x00000001L
+#define RCC_DEV1_PORT_STRAP2__STRAP_DSN_EN_DN_DEV1_MASK                                                       0x00000002L
+#define RCC_DEV1_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV1_MASK                                                   0x00000004L
+#define RCC_DEV1_PORT_STRAP2__STRAP_ECN1P1_EN_DEV1_MASK                                                       0x00000008L
+#define RCC_DEV1_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV1_MASK                                                   0x00000010L
+#define RCC_DEV1_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV1_MASK                                                     0x00000020L
+#define RCC_DEV1_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV1_MASK                                          0x00000080L
+#define RCC_DEV1_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV1_MASK                                             0x00000100L
+#define RCC_DEV1_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV1_MASK                                                 0x00000E00L
+#define RCC_DEV1_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV1_MASK                                           0x00001000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV1_MASK                                   0x00002000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV1_MASK                                                 0x00004000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN2_EN_DEV1_MASK                                                         0x00008000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV1_MASK                                                 0x00010000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV1_MASK                                                 0x00020000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV1_MASK                                          0x00700000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV1_MASK                                                0x03800000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV1_MASK                                           0x1C000000L
+#define RCC_DEV1_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV1_MASK                                                 0xE0000000L
+//RCC_DEV1_PORT_STRAP3
+#define RCC_DEV1_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV1__SHIFT                                0x0
+#define RCC_DEV1_PORT_STRAP3__STRAP_LTR_EN_DEV1__SHIFT                                                        0x1
+#define RCC_DEV1_PORT_STRAP3__STRAP_LTR_EN_DN_DEV1__SHIFT                                                     0x2
+#define RCC_DEV1_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV1__SHIFT                                           0x3
+#define RCC_DEV1_PORT_STRAP3__STRAP_MSI_EN_DN_DEV1__SHIFT                                                     0x6
+#define RCC_DEV1_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV1__SHIFT                                             0x7
+#define RCC_DEV1_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV1__SHIFT                                              0x8
+#define RCC_DEV1_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV1__SHIFT                                                0x9
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV1__SHIFT    0xb
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV1__SHIFT         0xe
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV1__SHIFT      0x12
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV1__SHIFT           0x15
+#define RCC_DEV1_PORT_STRAP3__STRAP_PM_SUPPORT_DEV1__SHIFT                                                    0x19
+#define RCC_DEV1_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV1__SHIFT                                                 0x1b
+#define RCC_DEV1_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV1__SHIFT                                                  0x1d
+#define RCC_DEV1_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV1__SHIFT                                                    0x1f
+#define RCC_DEV1_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV1_MASK                                  0x00000001L
+#define RCC_DEV1_PORT_STRAP3__STRAP_LTR_EN_DEV1_MASK                                                          0x00000002L
+#define RCC_DEV1_PORT_STRAP3__STRAP_LTR_EN_DN_DEV1_MASK                                                       0x00000004L
+#define RCC_DEV1_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV1_MASK                                             0x00000038L
+#define RCC_DEV1_PORT_STRAP3__STRAP_MSI_EN_DN_DEV1_MASK                                                       0x00000040L
+#define RCC_DEV1_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV1_MASK                                               0x00000080L
+#define RCC_DEV1_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV1_MASK                                                0x00000100L
+#define RCC_DEV1_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV1_MASK                                                  0x00000600L
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV1_MASK      0x00003800L
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV1_MASK           0x0003C000L
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV1_MASK        0x001C0000L
+#define RCC_DEV1_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV1_MASK             0x01E00000L
+#define RCC_DEV1_PORT_STRAP3__STRAP_PM_SUPPORT_DEV1_MASK                                                      0x06000000L
+#define RCC_DEV1_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV1_MASK                                                   0x18000000L
+#define RCC_DEV1_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV1_MASK                                                    0x20000000L
+#define RCC_DEV1_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV1_MASK                                                      0x80000000L
+//RCC_DEV1_PORT_STRAP4
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV1__SHIFT                                         0x0
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV1__SHIFT                                         0x8
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV1__SHIFT                                         0x10
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV1__SHIFT                                         0x18
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV1_MASK                                           0x000000FFL
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV1_MASK                                           0x0000FF00L
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV1_MASK                                           0x00FF0000L
+#define RCC_DEV1_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV1_MASK                                           0xFF000000L
+//RCC_DEV1_PORT_STRAP5
+#define RCC_DEV1_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV1__SHIFT                                         0x0
+#define RCC_DEV1_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV1__SHIFT                                         0x8
+#define RCC_DEV1_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV1__SHIFT                                   0x10
+#define RCC_DEV1_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV1__SHIFT                                            0x11
+#define RCC_DEV1_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV1__SHIFT                                             0x12
+#define RCC_DEV1_PORT_STRAP5__STRAP_VC_EN_DN_DEV1__SHIFT                                                      0x13
+#define RCC_DEV1_PORT_STRAP5__STRAP_TwoVC_EN_DEV1__SHIFT                                                      0x14
+#define RCC_DEV1_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV1__SHIFT                                                   0x15
+#define RCC_DEV1_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV1__SHIFT                                           0x16
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV1__SHIFT                                      0x17
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV1__SHIFT                                   0x18
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV1__SHIFT                                   0x19
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV1__SHIFT                                0x1a
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV1__SHIFT                                    0x1b
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV1__SHIFT                                     0x1c
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV1__SHIFT                                  0x1d
+#define RCC_DEV1_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV1__SHIFT                                                    0x1e
+#define RCC_DEV1_PORT_STRAP5__STRAP_SSID_EN_DEV1__SHIFT                                                       0x1f
+#define RCC_DEV1_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV1_MASK                                           0x000000FFL
+#define RCC_DEV1_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV1_MASK                                           0x0000FF00L
+#define RCC_DEV1_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV1_MASK                                     0x00010000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV1_MASK                                              0x00020000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV1_MASK                                               0x00040000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_VC_EN_DN_DEV1_MASK                                                        0x00080000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_TwoVC_EN_DEV1_MASK                                                        0x00100000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV1_MASK                                                     0x00200000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV1_MASK                                             0x00400000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV1_MASK                                        0x00800000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV1_MASK                                     0x01000000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV1_MASK                                     0x02000000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV1_MASK                                  0x04000000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV1_MASK                                      0x08000000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV1_MASK                                       0x10000000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV1_MASK                                    0x20000000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV1_MASK                                                      0x40000000L
+#define RCC_DEV1_PORT_STRAP5__STRAP_SSID_EN_DEV1_MASK                                                         0x80000000L
+//RCC_DEV1_PORT_STRAP6
+#define RCC_DEV1_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV1__SHIFT                                                    0x0
+#define RCC_DEV1_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV1__SHIFT                                    0x1
+#define RCC_DEV1_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV1__SHIFT                                               0x2
+#define RCC_DEV1_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV1__SHIFT                                     0x3
+#define RCC_DEV1_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV1__SHIFT                                     0x4
+#define RCC_DEV1_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV1__SHIFT                                 0x5
+#define RCC_DEV1_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV1__SHIFT                                 0x6
+#define RCC_DEV1_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV1__SHIFT                              0x7
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV1__SHIFT                0x8
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV1__SHIFT                0xc
+#define RCC_DEV1_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV1__SHIFT                                         0x10
+#define RCC_DEV1_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV1__SHIFT                                       0x12
+#define RCC_DEV1_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV1__SHIFT                                0x13
+#define RCC_DEV1_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV1__SHIFT                                               0x14
+#define RCC_DEV1_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV1__SHIFT                                             0x15
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV1__SHIFT                0x18
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV1__SHIFT                0x1c
+#define RCC_DEV1_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV1_MASK                                                      0x00000001L
+#define RCC_DEV1_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV1_MASK                                      0x00000002L
+#define RCC_DEV1_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV1_MASK                                                 0x00000004L
+#define RCC_DEV1_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV1_MASK                                       0x00000008L
+#define RCC_DEV1_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV1_MASK                                       0x00000010L
+#define RCC_DEV1_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV1_MASK                                   0x00000020L
+#define RCC_DEV1_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV1_MASK                                   0x00000040L
+#define RCC_DEV1_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV1_MASK                                0x00000080L
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV1_MASK                  0x00000F00L
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV1_MASK                  0x0000F000L
+#define RCC_DEV1_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV1_MASK                                           0x00030000L
+#define RCC_DEV1_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV1_MASK                                         0x00040000L
+#define RCC_DEV1_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV1_MASK                                  0x00080000L
+#define RCC_DEV1_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV1_MASK                                                 0x00100000L
+#define RCC_DEV1_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV1_MASK                                               0x00E00000L
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV1_MASK                  0x0F000000L
+#define RCC_DEV1_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV1_MASK                  0xF0000000L
+//RCC_DEV1_PORT_STRAP7
+#define RCC_DEV1_PORT_STRAP7__STRAP_PORT_NUMBER_DEV1__SHIFT                                                   0x0
+#define RCC_DEV1_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV1__SHIFT                                               0x8
+#define RCC_DEV1_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV1__SHIFT                                               0xc
+#define RCC_DEV1_PORT_STRAP7__STRAP_RP_BUSNUM_DEV1__SHIFT                                                     0x10
+#define RCC_DEV1_PORT_STRAP7__STRAP_DN_DEVNUM_DEV1__SHIFT                                                     0x18
+#define RCC_DEV1_PORT_STRAP7__STRAP_DN_FUNCID_DEV1__SHIFT                                                     0x1d
+#define RCC_DEV1_PORT_STRAP7__STRAP_PORT_NUMBER_DEV1_MASK                                                     0x000000FFL
+#define RCC_DEV1_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV1_MASK                                                 0x00000F00L
+#define RCC_DEV1_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV1_MASK                                                 0x0000F000L
+#define RCC_DEV1_PORT_STRAP7__STRAP_RP_BUSNUM_DEV1_MASK                                                       0x00FF0000L
+#define RCC_DEV1_PORT_STRAP7__STRAP_DN_DEVNUM_DEV1_MASK                                                       0x1F000000L
+#define RCC_DEV1_PORT_STRAP7__STRAP_DN_FUNCID_DEV1_MASK                                                       0xE0000000L
+//RCC_DEV1_PORT_STRAP8
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV1__SHIFT                                         0x0
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV1__SHIFT                                         0x8
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV1__SHIFT                                         0x10
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV1__SHIFT                                         0x18
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV1_MASK                                           0x000000FFL
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV1_MASK                                           0x0000FF00L
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV1_MASK                                           0x00FF0000L
+#define RCC_DEV1_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV1_MASK                                           0xFF000000L
+//RCC_DEV1_PORT_STRAP9
+#define RCC_DEV1_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV1__SHIFT                                         0x0
+#define RCC_DEV1_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV1__SHIFT                                         0x8
+#define RCC_DEV1_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV1__SHIFT                                                  0x10
+#define RCC_DEV1_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV1_MASK                                           0x000000FFL
+#define RCC_DEV1_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV1_MASK                                           0x0000FF00L
+#define RCC_DEV1_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV1_MASK                                                    0xFFFF0000L
+//RCC_DEV1_PORT_STRAP10
+#define RCC_DEV1_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV1__SHIFT                                         0x0
+#define RCC_DEV1_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV1__SHIFT                                      0x1
+#define RCC_DEV1_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV1__SHIFT                             0x2
+#define RCC_DEV1_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV1__SHIFT                             0x3
+#define RCC_DEV1_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV1__SHIFT                                    0x4
+#define RCC_DEV1_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV1__SHIFT                                  0x5
+#define RCC_DEV1_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV1__SHIFT                                           0x6
+#define RCC_DEV1_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV1__SHIFT                                              0x14
+#define RCC_DEV1_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV1__SHIFT                                         0x18
+#define RCC_DEV1_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV1__SHIFT                                      0x1a
+#define RCC_DEV1_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV1_MASK                                           0x00000001L
+#define RCC_DEV1_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV1_MASK                                        0x00000002L
+#define RCC_DEV1_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV1_MASK                               0x00000004L
+#define RCC_DEV1_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV1_MASK                               0x00000008L
+#define RCC_DEV1_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV1_MASK                                      0x00000010L
+#define RCC_DEV1_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV1_MASK                                    0x00000020L
+#define RCC_DEV1_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV1_MASK                                             0x0007FFC0L
+#define RCC_DEV1_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV1_MASK                                                0x00100000L
+#define RCC_DEV1_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV1_MASK                                           0x03000000L
+#define RCC_DEV1_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV1_MASK                                        0x0C000000L
+//RCC_DEV1_PORT_STRAP11
+#define RCC_DEV1_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV1__SHIFT                                        0x0
+#define RCC_DEV1_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV1__SHIFT                                            0x10
+#define RCC_DEV1_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV1__SHIFT                                                 0x1c
+#define RCC_DEV1_PORT_STRAP11__STRAP_RTR_EN_DN_DEV1__SHIFT                                                    0x1d
+#define RCC_DEV1_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV1__SHIFT                             0x1f
+#define RCC_DEV1_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV1_MASK                                          0x0000FFFFL
+#define RCC_DEV1_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV1_MASK                                              0x0FFF0000L
+#define RCC_DEV1_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV1_MASK                                                   0x10000000L
+#define RCC_DEV1_PORT_STRAP11__STRAP_RTR_EN_DN_DEV1_MASK                                                      0x20000000L
+#define RCC_DEV1_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV1_MASK                               0x80000000L
+//RCC_DEV1_PORT_STRAP12
+#define RCC_DEV1_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV1__SHIFT                                           0x0
+#define RCC_DEV1_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV1_MASK                                             0x00FFFFFFL
+//RCC_DEV1_PORT_STRAP13
+#define RCC_DEV1_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV1__SHIFT                                     0x0
+#define RCC_DEV1_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV1__SHIFT                0x8
+#define RCC_DEV1_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV1__SHIFT                                   0x9
+#define RCC_DEV1_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV1__SHIFT                                          0x14
+#define RCC_DEV1_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV1_MASK                                       0x000000FFL
+#define RCC_DEV1_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV1_MASK                  0x00000100L
+#define RCC_DEV1_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV1_MASK                                     0x000FFE00L
+#define RCC_DEV1_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV1_MASK                                            0xFFF00000L
+//RCC_DEV1_PORT_STRAP14
+#define RCC_DEV1_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV1__SHIFT                                          0x0
+#define RCC_DEV1_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV1__SHIFT                                        0x1
+#define RCC_DEV1_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV1__SHIFT                                            0x2
+#define RCC_DEV1_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV1__SHIFT                                     0x3
+#define RCC_DEV1_PORT_STRAP14__STRAP_DOE_EN_UP_DEV1__SHIFT                                                    0x4
+#define RCC_DEV1_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV1__SHIFT                                                 0x5
+#define RCC_DEV1_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV1__SHIFT                                              0x6
+#define RCC_DEV1_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV1_MASK                                            0x00000001L
+#define RCC_DEV1_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV1_MASK                                          0x00000002L
+#define RCC_DEV1_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV1_MASK                                              0x00000004L
+#define RCC_DEV1_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV1_MASK                                       0x00000008L
+#define RCC_DEV1_PORT_STRAP14__STRAP_DOE_EN_UP_DEV1_MASK                                                      0x00000010L
+#define RCC_DEV1_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV1_MASK                                                   0x00000020L
+#define RCC_DEV1_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV1_MASK                                                0x00000040L
+//RCC_DEV2_PORT_STRAP0
+#define RCC_DEV2_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV2__SHIFT                                                  0x0
+#define RCC_DEV2_PORT_STRAP0__STRAP_ARI_EN_DN_DEV2__SHIFT                                                     0x10
+#define RCC_DEV2_PORT_STRAP0__STRAP_ACS_EN_DN_DEV2__SHIFT                                                     0x11
+#define RCC_DEV2_PORT_STRAP0__STRAP_AER_EN_DN_DEV2__SHIFT                                                     0x12
+#define RCC_DEV2_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV2__SHIFT                                           0x13
+#define RCC_DEV2_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV2__SHIFT                                                0x14
+#define RCC_DEV2_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV2__SHIFT                                       0x18
+#define RCC_DEV2_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV2__SHIFT                                        0x19
+#define RCC_DEV2_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV2__SHIFT                                        0x1c
+#define RCC_DEV2_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV2__SHIFT                                                 0x1f
+#define RCC_DEV2_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV2_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_PORT_STRAP0__STRAP_ARI_EN_DN_DEV2_MASK                                                       0x00010000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_ACS_EN_DN_DEV2_MASK                                                       0x00020000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_AER_EN_DN_DEV2_MASK                                                       0x00040000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV2_MASK                                             0x00080000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV2_MASK                                                  0x00100000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV2_MASK                                         0x01000000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV2_MASK                                          0x0E000000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV2_MASK                                          0x70000000L
+#define RCC_DEV2_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV2_MASK                                                   0x80000000L
+//RCC_DEV2_PORT_STRAP1
+#define RCC_DEV2_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV2__SHIFT                                                  0x0
+#define RCC_DEV2_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV2__SHIFT                                              0x10
+#define RCC_DEV2_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV2_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV2_MASK                                                0xFFFF0000L
+//RCC_DEV2_PORT_STRAP2
+#define RCC_DEV2_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV2__SHIFT                                            0x0
+#define RCC_DEV2_PORT_STRAP2__STRAP_DSN_EN_DN_DEV2__SHIFT                                                     0x1
+#define RCC_DEV2_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV2__SHIFT                                                 0x2
+#define RCC_DEV2_PORT_STRAP2__STRAP_ECN1P1_EN_DEV2__SHIFT                                                     0x3
+#define RCC_DEV2_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV2__SHIFT                                                 0x4
+#define RCC_DEV2_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV2__SHIFT                                                   0x5
+#define RCC_DEV2_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV2__SHIFT                                        0x7
+#define RCC_DEV2_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV2__SHIFT                                           0x8
+#define RCC_DEV2_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV2__SHIFT                                               0x9
+#define RCC_DEV2_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV2__SHIFT                                         0xc
+#define RCC_DEV2_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV2__SHIFT                                 0xd
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV2__SHIFT                                               0xe
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN2_EN_DEV2__SHIFT                                                       0xf
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV2__SHIFT                                               0x10
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV2__SHIFT                                               0x11
+#define RCC_DEV2_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV2__SHIFT                                        0x14
+#define RCC_DEV2_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV2__SHIFT                                              0x17
+#define RCC_DEV2_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV2__SHIFT                                         0x1a
+#define RCC_DEV2_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV2__SHIFT                                               0x1d
+#define RCC_DEV2_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV2_MASK                                              0x00000001L
+#define RCC_DEV2_PORT_STRAP2__STRAP_DSN_EN_DN_DEV2_MASK                                                       0x00000002L
+#define RCC_DEV2_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV2_MASK                                                   0x00000004L
+#define RCC_DEV2_PORT_STRAP2__STRAP_ECN1P1_EN_DEV2_MASK                                                       0x00000008L
+#define RCC_DEV2_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV2_MASK                                                   0x00000010L
+#define RCC_DEV2_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV2_MASK                                                     0x00000020L
+#define RCC_DEV2_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV2_MASK                                          0x00000080L
+#define RCC_DEV2_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV2_MASK                                             0x00000100L
+#define RCC_DEV2_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV2_MASK                                                 0x00000E00L
+#define RCC_DEV2_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV2_MASK                                           0x00001000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV2_MASK                                   0x00002000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV2_MASK                                                 0x00004000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN2_EN_DEV2_MASK                                                         0x00008000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV2_MASK                                                 0x00010000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV2_MASK                                                 0x00020000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV2_MASK                                          0x00700000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV2_MASK                                                0x03800000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV2_MASK                                           0x1C000000L
+#define RCC_DEV2_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV2_MASK                                                 0xE0000000L
+//RCC_DEV2_PORT_STRAP3
+#define RCC_DEV2_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV2__SHIFT                                0x0
+#define RCC_DEV2_PORT_STRAP3__STRAP_LTR_EN_DEV2__SHIFT                                                        0x1
+#define RCC_DEV2_PORT_STRAP3__STRAP_LTR_EN_DN_DEV2__SHIFT                                                     0x2
+#define RCC_DEV2_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV2__SHIFT                                           0x3
+#define RCC_DEV2_PORT_STRAP3__STRAP_MSI_EN_DN_DEV2__SHIFT                                                     0x6
+#define RCC_DEV2_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV2__SHIFT                                             0x7
+#define RCC_DEV2_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV2__SHIFT                                              0x8
+#define RCC_DEV2_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV2__SHIFT                                                0x9
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV2__SHIFT    0xb
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV2__SHIFT         0xe
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV2__SHIFT      0x12
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV2__SHIFT           0x15
+#define RCC_DEV2_PORT_STRAP3__STRAP_PM_SUPPORT_DEV2__SHIFT                                                    0x19
+#define RCC_DEV2_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV2__SHIFT                                                 0x1b
+#define RCC_DEV2_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV2__SHIFT                                                  0x1d
+#define RCC_DEV2_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV2__SHIFT                                                    0x1f
+#define RCC_DEV2_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV2_MASK                                  0x00000001L
+#define RCC_DEV2_PORT_STRAP3__STRAP_LTR_EN_DEV2_MASK                                                          0x00000002L
+#define RCC_DEV2_PORT_STRAP3__STRAP_LTR_EN_DN_DEV2_MASK                                                       0x00000004L
+#define RCC_DEV2_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV2_MASK                                             0x00000038L
+#define RCC_DEV2_PORT_STRAP3__STRAP_MSI_EN_DN_DEV2_MASK                                                       0x00000040L
+#define RCC_DEV2_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV2_MASK                                               0x00000080L
+#define RCC_DEV2_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV2_MASK                                                0x00000100L
+#define RCC_DEV2_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV2_MASK                                                  0x00000600L
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV2_MASK      0x00003800L
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV2_MASK           0x0003C000L
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV2_MASK        0x001C0000L
+#define RCC_DEV2_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV2_MASK             0x01E00000L
+#define RCC_DEV2_PORT_STRAP3__STRAP_PM_SUPPORT_DEV2_MASK                                                      0x06000000L
+#define RCC_DEV2_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV2_MASK                                                   0x18000000L
+#define RCC_DEV2_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV2_MASK                                                    0x20000000L
+#define RCC_DEV2_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV2_MASK                                                      0x80000000L
+//RCC_DEV2_PORT_STRAP4
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV2__SHIFT                                         0x0
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV2__SHIFT                                         0x8
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV2__SHIFT                                         0x10
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV2__SHIFT                                         0x18
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV2_MASK                                           0x000000FFL
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV2_MASK                                           0x0000FF00L
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV2_MASK                                           0x00FF0000L
+#define RCC_DEV2_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV2_MASK                                           0xFF000000L
+//RCC_DEV2_PORT_STRAP5
+#define RCC_DEV2_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV2__SHIFT                                         0x0
+#define RCC_DEV2_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV2__SHIFT                                         0x8
+#define RCC_DEV2_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV2__SHIFT                                   0x10
+#define RCC_DEV2_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV2__SHIFT                                            0x11
+#define RCC_DEV2_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV2__SHIFT                                             0x12
+#define RCC_DEV2_PORT_STRAP5__STRAP_VC_EN_DN_DEV2__SHIFT                                                      0x13
+#define RCC_DEV2_PORT_STRAP5__STRAP_TwoVC_EN_DEV2__SHIFT                                                      0x14
+#define RCC_DEV2_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV2__SHIFT                                                   0x15
+#define RCC_DEV2_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV2__SHIFT                                           0x16
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV2__SHIFT                                      0x17
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV2__SHIFT                                   0x18
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV2__SHIFT                                   0x19
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV2__SHIFT                                0x1a
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV2__SHIFT                                    0x1b
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV2__SHIFT                                     0x1c
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV2__SHIFT                                  0x1d
+#define RCC_DEV2_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV2__SHIFT                                                    0x1e
+#define RCC_DEV2_PORT_STRAP5__STRAP_SSID_EN_DEV2__SHIFT                                                       0x1f
+#define RCC_DEV2_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV2_MASK                                           0x000000FFL
+#define RCC_DEV2_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV2_MASK                                           0x0000FF00L
+#define RCC_DEV2_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV2_MASK                                     0x00010000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV2_MASK                                              0x00020000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV2_MASK                                               0x00040000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_VC_EN_DN_DEV2_MASK                                                        0x00080000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_TwoVC_EN_DEV2_MASK                                                        0x00100000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV2_MASK                                                     0x00200000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV2_MASK                                             0x00400000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV2_MASK                                        0x00800000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV2_MASK                                     0x01000000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV2_MASK                                     0x02000000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV2_MASK                                  0x04000000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV2_MASK                                      0x08000000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV2_MASK                                       0x10000000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV2_MASK                                    0x20000000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV2_MASK                                                      0x40000000L
+#define RCC_DEV2_PORT_STRAP5__STRAP_SSID_EN_DEV2_MASK                                                         0x80000000L
+//RCC_DEV2_PORT_STRAP6
+#define RCC_DEV2_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV2__SHIFT                                                    0x0
+#define RCC_DEV2_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV2__SHIFT                                    0x1
+#define RCC_DEV2_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV2__SHIFT                                               0x2
+#define RCC_DEV2_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV2__SHIFT                                     0x3
+#define RCC_DEV2_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV2__SHIFT                                     0x4
+#define RCC_DEV2_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV2__SHIFT                                 0x5
+#define RCC_DEV2_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV2__SHIFT                                 0x6
+#define RCC_DEV2_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV2__SHIFT                              0x7
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV2__SHIFT                0x8
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV2__SHIFT                0xc
+#define RCC_DEV2_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV2__SHIFT                                         0x10
+#define RCC_DEV2_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV2__SHIFT                                       0x12
+#define RCC_DEV2_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV2__SHIFT                                0x13
+#define RCC_DEV2_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV2__SHIFT                                               0x14
+#define RCC_DEV2_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV2__SHIFT                                             0x15
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV2__SHIFT                0x18
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV2__SHIFT                0x1c
+#define RCC_DEV2_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV2_MASK                                                      0x00000001L
+#define RCC_DEV2_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV2_MASK                                      0x00000002L
+#define RCC_DEV2_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV2_MASK                                                 0x00000004L
+#define RCC_DEV2_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV2_MASK                                       0x00000008L
+#define RCC_DEV2_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV2_MASK                                       0x00000010L
+#define RCC_DEV2_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV2_MASK                                   0x00000020L
+#define RCC_DEV2_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV2_MASK                                   0x00000040L
+#define RCC_DEV2_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV2_MASK                                0x00000080L
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV2_MASK                  0x00000F00L
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV2_MASK                  0x0000F000L
+#define RCC_DEV2_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV2_MASK                                           0x00030000L
+#define RCC_DEV2_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV2_MASK                                         0x00040000L
+#define RCC_DEV2_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV2_MASK                                  0x00080000L
+#define RCC_DEV2_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV2_MASK                                                 0x00100000L
+#define RCC_DEV2_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV2_MASK                                               0x00E00000L
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV2_MASK                  0x0F000000L
+#define RCC_DEV2_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV2_MASK                  0xF0000000L
+//RCC_DEV2_PORT_STRAP7
+#define RCC_DEV2_PORT_STRAP7__STRAP_PORT_NUMBER_DEV2__SHIFT                                                   0x0
+#define RCC_DEV2_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV2__SHIFT                                               0x8
+#define RCC_DEV2_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV2__SHIFT                                               0xc
+#define RCC_DEV2_PORT_STRAP7__STRAP_RP_BUSNUM_DEV2__SHIFT                                                     0x10
+#define RCC_DEV2_PORT_STRAP7__STRAP_DN_DEVNUM_DEV2__SHIFT                                                     0x18
+#define RCC_DEV2_PORT_STRAP7__STRAP_DN_FUNCID_DEV2__SHIFT                                                     0x1d
+#define RCC_DEV2_PORT_STRAP7__STRAP_PORT_NUMBER_DEV2_MASK                                                     0x000000FFL
+#define RCC_DEV2_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV2_MASK                                                 0x00000F00L
+#define RCC_DEV2_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV2_MASK                                                 0x0000F000L
+#define RCC_DEV2_PORT_STRAP7__STRAP_RP_BUSNUM_DEV2_MASK                                                       0x00FF0000L
+#define RCC_DEV2_PORT_STRAP7__STRAP_DN_DEVNUM_DEV2_MASK                                                       0x1F000000L
+#define RCC_DEV2_PORT_STRAP7__STRAP_DN_FUNCID_DEV2_MASK                                                       0xE0000000L
+//RCC_DEV2_PORT_STRAP8
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV2__SHIFT                                         0x0
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV2__SHIFT                                         0x8
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV2__SHIFT                                         0x10
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV2__SHIFT                                         0x18
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV2_MASK                                           0x000000FFL
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV2_MASK                                           0x0000FF00L
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV2_MASK                                           0x00FF0000L
+#define RCC_DEV2_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV2_MASK                                           0xFF000000L
+//RCC_DEV2_PORT_STRAP9
+#define RCC_DEV2_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV2__SHIFT                                         0x0
+#define RCC_DEV2_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV2__SHIFT                                         0x8
+#define RCC_DEV2_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV2__SHIFT                                                  0x10
+#define RCC_DEV2_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV2_MASK                                           0x000000FFL
+#define RCC_DEV2_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV2_MASK                                           0x0000FF00L
+#define RCC_DEV2_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV2_MASK                                                    0xFFFF0000L
+//RCC_DEV2_PORT_STRAP10
+#define RCC_DEV2_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV2__SHIFT                                         0x0
+#define RCC_DEV2_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV2__SHIFT                                      0x1
+#define RCC_DEV2_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV2__SHIFT                             0x2
+#define RCC_DEV2_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV2__SHIFT                             0x3
+#define RCC_DEV2_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV2__SHIFT                                    0x4
+#define RCC_DEV2_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV2__SHIFT                                  0x5
+#define RCC_DEV2_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV2__SHIFT                                           0x6
+#define RCC_DEV2_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV2__SHIFT                                              0x14
+#define RCC_DEV2_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV2__SHIFT                                         0x18
+#define RCC_DEV2_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV2__SHIFT                                      0x1a
+#define RCC_DEV2_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV2_MASK                                           0x00000001L
+#define RCC_DEV2_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV2_MASK                                        0x00000002L
+#define RCC_DEV2_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV2_MASK                               0x00000004L
+#define RCC_DEV2_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV2_MASK                               0x00000008L
+#define RCC_DEV2_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV2_MASK                                      0x00000010L
+#define RCC_DEV2_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV2_MASK                                    0x00000020L
+#define RCC_DEV2_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV2_MASK                                             0x0007FFC0L
+#define RCC_DEV2_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV2_MASK                                                0x00100000L
+#define RCC_DEV2_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV2_MASK                                           0x03000000L
+#define RCC_DEV2_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV2_MASK                                        0x0C000000L
+//RCC_DEV2_PORT_STRAP11
+#define RCC_DEV2_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV2__SHIFT                                        0x0
+#define RCC_DEV2_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV2__SHIFT                                            0x10
+#define RCC_DEV2_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV2__SHIFT                                                 0x1c
+#define RCC_DEV2_PORT_STRAP11__STRAP_RTR_EN_DN_DEV2__SHIFT                                                    0x1d
+#define RCC_DEV2_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV2__SHIFT                             0x1f
+#define RCC_DEV2_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV2_MASK                                          0x0000FFFFL
+#define RCC_DEV2_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV2_MASK                                              0x0FFF0000L
+#define RCC_DEV2_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV2_MASK                                                   0x10000000L
+#define RCC_DEV2_PORT_STRAP11__STRAP_RTR_EN_DN_DEV2_MASK                                                      0x20000000L
+#define RCC_DEV2_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV2_MASK                               0x80000000L
+//RCC_DEV2_PORT_STRAP12
+#define RCC_DEV2_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV2__SHIFT                                           0x0
+#define RCC_DEV2_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV2_MASK                                             0x00FFFFFFL
+//RCC_DEV2_PORT_STRAP13
+#define RCC_DEV2_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV2__SHIFT                                     0x0
+#define RCC_DEV2_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV2__SHIFT                0x8
+#define RCC_DEV2_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV2__SHIFT                                   0x9
+#define RCC_DEV2_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV2__SHIFT                                          0x14
+#define RCC_DEV2_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV2_MASK                                       0x000000FFL
+#define RCC_DEV2_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV2_MASK                  0x00000100L
+#define RCC_DEV2_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV2_MASK                                     0x000FFE00L
+#define RCC_DEV2_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV2_MASK                                            0xFFF00000L
+//RCC_DEV2_PORT_STRAP14
+#define RCC_DEV2_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV2__SHIFT                                          0x0
+#define RCC_DEV2_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV2__SHIFT                                        0x1
+#define RCC_DEV2_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV2__SHIFT                                            0x2
+#define RCC_DEV2_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV2__SHIFT                                     0x3
+#define RCC_DEV2_PORT_STRAP14__STRAP_DOE_EN_UP_DEV2__SHIFT                                                    0x4
+#define RCC_DEV2_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV2__SHIFT                                                 0x5
+#define RCC_DEV2_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV2__SHIFT                                              0x6
+#define RCC_DEV2_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV2_MASK                                            0x00000001L
+#define RCC_DEV2_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV2_MASK                                          0x00000002L
+#define RCC_DEV2_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV2_MASK                                              0x00000004L
+#define RCC_DEV2_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV2_MASK                                       0x00000008L
+#define RCC_DEV2_PORT_STRAP14__STRAP_DOE_EN_UP_DEV2_MASK                                                      0x00000010L
+#define RCC_DEV2_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV2_MASK                                                   0x00000020L
+#define RCC_DEV2_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV2_MASK                                                0x00000040L
+//RCC_STRAP0_RCC_BIF_STRAP0
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x14
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN__SHIFT                         0x15
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GPUIOV_EN__SHIFT                                                     0x16
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE__SHIFT                                          0x17
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN_MASK                           0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GPUIOV_EN_MASK                                                       0x00400000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE_MASK                                            0x00800000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP1
+#define RCC_STRAP0_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP0_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP0_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP2
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS__SHIFT                                               0xc
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS_MASK                                                 0x00001000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP3
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP0_RCC_BIF_STRAP4
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP0_RCC_BIF_STRAP5
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP6
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE__SHIFT                                               0x5
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0xa
+#define RCC_STRAP0_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6__SHIFT                                                 0xf
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE_MASK                                                 0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00007C00L
+#define RCC_STRAP0_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6_MASK                                                   0xFFFF8000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0__SHIFT                      0x1
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0__SHIFT                     0x2
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0__SHIFT                    0x4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0_MASK                        0x00000002L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0_MASK                       0x00000004L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0_MASK                      0x00000010L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0__SHIFT             0x19
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0_MASK               0x02000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0__SHIFT                                  0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0_MASK                                    0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0_MASK                                       0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP17
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0__SHIFT                            0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0_MASK                              0x000FF000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1__SHIFT                    0x4
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1_MASK                      0x00000010L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1__SHIFT             0x19
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1_MASK               0x02000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1__SHIFT                                  0x2
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1__SHIFT                                        0x8
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1__SHIFT                                  0xa
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1__SHIFT                                        0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1__SHIFT                                  0x12
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1__SHIFT                                        0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1__SHIFT                                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1_MASK                                    0x00000004L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1_MASK                                          0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1_MASK                                    0x00000400L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1_MASK                                          0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1_MASK                                    0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1_MASK                                          0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1_MASK                                          0x04000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP7
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP23
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP23__STRAP_DSN_CODE_L_DEV0_F1__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP23__STRAP_DSN_CODE_L_DEV0_F1_MASK                                       0xFFFFFFFFL
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP24
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP24__STRAP_DSN_CODE_H_DEV0_F1__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP24__STRAP_DSN_CODE_H_DEV0_F1_MASK                                       0xFFFFFFFFL
+//RCC_DEV0_EPF2_STRAP0
+#define RCC_DEV0_EPF2_STRAP0__STRAP_DEVICE_ID_DEV0_F2__SHIFT                                                  0x0
+#define RCC_DEV0_EPF2_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F2__SHIFT                                               0x10
+#define RCC_DEV0_EPF2_STRAP0__STRAP_MINOR_REV_ID_DEV0_F2__SHIFT                                               0x14
+#define RCC_DEV0_EPF2_STRAP0__STRAP_FUNC_EN_DEV0_F2__SHIFT                                                    0x1c
+#define RCC_DEV0_EPF2_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F2__SHIFT                                      0x1d
+#define RCC_DEV0_EPF2_STRAP0__STRAP_D1_SUPPORT_DEV0_F2__SHIFT                                                 0x1e
+#define RCC_DEV0_EPF2_STRAP0__STRAP_D2_SUPPORT_DEV0_F2__SHIFT                                                 0x1f
+#define RCC_DEV0_EPF2_STRAP0__STRAP_DEVICE_ID_DEV0_F2_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF2_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F2_MASK                                                 0x000F0000L
+#define RCC_DEV0_EPF2_STRAP0__STRAP_MINOR_REV_ID_DEV0_F2_MASK                                                 0x00F00000L
+#define RCC_DEV0_EPF2_STRAP0__STRAP_FUNC_EN_DEV0_F2_MASK                                                      0x10000000L
+#define RCC_DEV0_EPF2_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F2_MASK                                        0x20000000L
+#define RCC_DEV0_EPF2_STRAP0__STRAP_D1_SUPPORT_DEV0_F2_MASK                                                   0x40000000L
+#define RCC_DEV0_EPF2_STRAP0__STRAP_D2_SUPPORT_DEV0_F2_MASK                                                   0x80000000L
+//RCC_DEV0_EPF2_STRAP2
+#define RCC_DEV0_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F2__SHIFT                               0x4
+#define RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2__SHIFT                                              0x7
+#define RCC_DEV0_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F2__SHIFT                                              0x8
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F2__SHIFT                                            0x9
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F2__SHIFT                                     0xe
+#define RCC_DEV0_EPF2_STRAP2__STRAP_AER_EN_DEV0_F2__SHIFT                                                     0x10
+#define RCC_DEV0_EPF2_STRAP2__STRAP_ACS_EN_DEV0_F2__SHIFT                                                     0x11
+#define RCC_DEV0_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F2__SHIFT                                           0x14
+#define RCC_DEV0_EPF2_STRAP2__STRAP_DPA_EN_DEV0_F2__SHIFT                                                     0x15
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F2__SHIFT                                              0x18
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EN_DEV0_F2__SHIFT                                                   0x1c
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F2__SHIFT                             0x1d
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F2__SHIFT                          0x1e
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F2__SHIFT                                  0x1f
+#define RCC_DEV0_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F2_MASK                                 0x00000010L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F2_MASK                                                0x00000080L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F2_MASK                                                0x00000100L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F2_MASK                                              0x00003E00L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F2_MASK                                       0x00004000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_AER_EN_DEV0_F2_MASK                                                       0x00010000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_ACS_EN_DEV0_F2_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F2_MASK                                             0x00100000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_DPA_EN_DEV0_F2_MASK                                                       0x00200000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F2_MASK                                                0x07000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EN_DEV0_F2_MASK                                                     0x10000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F2_MASK                               0x20000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F2_MASK                            0x40000000L
+#define RCC_DEV0_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F2_MASK                                    0x80000000L
+//RCC_DEV0_EPF2_STRAP3
+#define RCC_DEV0_EPF2_STRAP3__STRAP_SUBSYS_ID_DEV0_F2__SHIFT                                                  0x0
+#define RCC_DEV0_EPF2_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F2__SHIFT                                 0x10
+#define RCC_DEV0_EPF2_STRAP3__STRAP_PWR_EN_DEV0_F2__SHIFT                                                     0x11
+#define RCC_DEV0_EPF2_STRAP3__STRAP_MSI_EN_DEV0_F2__SHIFT                                                     0x12
+#define RCC_DEV0_EPF2_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F2__SHIFT                                         0x13
+#define RCC_DEV0_EPF2_STRAP3__STRAP_MSIX_EN_DEV0_F2__SHIFT                                                    0x14
+#define RCC_DEV0_EPF2_STRAP3__STRAP_PMC_DSI_DEV0_F2__SHIFT                                                    0x18
+#define RCC_DEV0_EPF2_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F2__SHIFT                        0x19
+#define RCC_DEV0_EPF2_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F2__SHIFT                                   0x1a
+#define RCC_DEV0_EPF2_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F2__SHIFT                                  0x1b
+#define RCC_DEV0_EPF2_STRAP3__STRAP_CLK_PM_EN_DEV0_F2__SHIFT                                                  0x1d
+#define RCC_DEV0_EPF2_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F2__SHIFT                                          0x1e
+#define RCC_DEV0_EPF2_STRAP3__STRAP_RTR_EN_DEV0_F2__SHIFT                                                     0x1f
+#define RCC_DEV0_EPF2_STRAP3__STRAP_SUBSYS_ID_DEV0_F2_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF2_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F2_MASK                                   0x00010000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_PWR_EN_DEV0_F2_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_MSI_EN_DEV0_F2_MASK                                                       0x00040000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F2_MASK                                           0x00080000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_MSIX_EN_DEV0_F2_MASK                                                      0x00100000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_PMC_DSI_DEV0_F2_MASK                                                      0x01000000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F2_MASK                          0x02000000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F2_MASK                                     0x04000000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F2_MASK                                    0x08000000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_CLK_PM_EN_DEV0_F2_MASK                                                    0x20000000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F2_MASK                                            0x40000000L
+#define RCC_DEV0_EPF2_STRAP3__STRAP_RTR_EN_DEV0_F2_MASK                                                       0x80000000L
+//RCC_DEV0_EPF2_STRAP4
+#define RCC_DEV0_EPF2_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F2__SHIFT                                            0x14
+#define RCC_DEV0_EPF2_STRAP4__STRAP_ATOMIC_EN_DEV0_F2__SHIFT                                                  0x15
+#define RCC_DEV0_EPF2_STRAP4__STRAP_FLR_EN_DEV0_F2__SHIFT                                                     0x16
+#define RCC_DEV0_EPF2_STRAP4__STRAP_PME_SUPPORT_DEV0_F2__SHIFT                                                0x17
+#define RCC_DEV0_EPF2_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F2__SHIFT                                              0x1c
+#define RCC_DEV0_EPF2_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F2__SHIFT                                             0x1f
+#define RCC_DEV0_EPF2_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F2_MASK                                              0x00100000L
+#define RCC_DEV0_EPF2_STRAP4__STRAP_ATOMIC_EN_DEV0_F2_MASK                                                    0x00200000L
+#define RCC_DEV0_EPF2_STRAP4__STRAP_FLR_EN_DEV0_F2_MASK                                                       0x00400000L
+#define RCC_DEV0_EPF2_STRAP4__STRAP_PME_SUPPORT_DEV0_F2_MASK                                                  0x0F800000L
+#define RCC_DEV0_EPF2_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F2_MASK                                                0x70000000L
+#define RCC_DEV0_EPF2_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F2_MASK                                               0x80000000L
+//RCC_DEV0_EPF2_STRAP5
+#define RCC_DEV0_EPF2_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F2__SHIFT                                              0x0
+#define RCC_DEV0_EPF2_STRAP5__STRAP_AUX_CURRENT_DEV0_F2__SHIFT                                                0x1b
+#define RCC_DEV0_EPF2_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F2__SHIFT                                       0x1e
+#define RCC_DEV0_EPF2_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F2_MASK                                                0x0000FFFFL
+#define RCC_DEV0_EPF2_STRAP5__STRAP_AUX_CURRENT_DEV0_F2_MASK                                                  0x38000000L
+#define RCC_DEV0_EPF2_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F2_MASK                                         0x40000000L
+//RCC_DEV0_EPF2_STRAP6
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER0_EN_DEV0_F2__SHIFT                                                   0x0
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F2__SHIFT                                             0x2
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER1_EN_DEV0_F2__SHIFT                                                   0x8
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F2__SHIFT                                             0xa
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER2_EN_DEV0_F2__SHIFT                                                   0x10
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F2__SHIFT                                             0x12
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER3_EN_DEV0_F2__SHIFT                                                   0x18
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER4_EN_DEV0_F2__SHIFT                                                   0x1a
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER0_EN_DEV0_F2_MASK                                                     0x00000001L
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F2_MASK                                               0x00000004L
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER1_EN_DEV0_F2_MASK                                                     0x00000100L
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F2_MASK                                               0x00000400L
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER2_EN_DEV0_F2_MASK                                                     0x00010000L
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F2_MASK                                               0x00040000L
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER3_EN_DEV0_F2_MASK                                                     0x01000000L
+#define RCC_DEV0_EPF2_STRAP6__STRAP_APER4_EN_DEV0_F2_MASK                                                     0x04000000L
+//RCC_DEV0_EPF2_STRAP7
+//RCC_DEV0_EPF2_STRAP10
+//RCC_DEV0_EPF2_STRAP11
+//RCC_DEV0_EPF2_STRAP12
+//RCC_DEV0_EPF2_STRAP13
+#define RCC_DEV0_EPF2_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F2__SHIFT                                            0x0
+#define RCC_DEV0_EPF2_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F2__SHIFT                                            0x8
+#define RCC_DEV0_EPF2_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F2__SHIFT                                           0x10
+#define RCC_DEV0_EPF2_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F2_MASK                                              0x000000FFL
+#define RCC_DEV0_EPF2_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F2_MASK                                              0x0000FF00L
+#define RCC_DEV0_EPF2_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F2_MASK                                             0x00FF0000L
+//RCC_DEV0_EPF2_STRAP14
+#define RCC_DEV0_EPF2_STRAP14__STRAP_VENDOR_ID_DEV0_F2__SHIFT                                                 0x0
+#define RCC_DEV0_EPF2_STRAP14__STRAP_VENDOR_ID_DEV0_F2_MASK                                                   0x0000FFFFL
+//RCC_DEV0_EPF2_STRAP20
+//RCC_DEV0_EPF3_STRAP0
+#define RCC_DEV0_EPF3_STRAP0__STRAP_DEVICE_ID_DEV0_F3__SHIFT                                                  0x0
+#define RCC_DEV0_EPF3_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F3__SHIFT                                               0x10
+#define RCC_DEV0_EPF3_STRAP0__STRAP_MINOR_REV_ID_DEV0_F3__SHIFT                                               0x14
+#define RCC_DEV0_EPF3_STRAP0__STRAP_FUNC_EN_DEV0_F3__SHIFT                                                    0x1c
+#define RCC_DEV0_EPF3_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F3__SHIFT                                      0x1d
+#define RCC_DEV0_EPF3_STRAP0__STRAP_D1_SUPPORT_DEV0_F3__SHIFT                                                 0x1e
+#define RCC_DEV0_EPF3_STRAP0__STRAP_D2_SUPPORT_DEV0_F3__SHIFT                                                 0x1f
+#define RCC_DEV0_EPF3_STRAP0__STRAP_DEVICE_ID_DEV0_F3_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF3_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F3_MASK                                                 0x000F0000L
+#define RCC_DEV0_EPF3_STRAP0__STRAP_MINOR_REV_ID_DEV0_F3_MASK                                                 0x00F00000L
+#define RCC_DEV0_EPF3_STRAP0__STRAP_FUNC_EN_DEV0_F3_MASK                                                      0x10000000L
+#define RCC_DEV0_EPF3_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F3_MASK                                        0x20000000L
+#define RCC_DEV0_EPF3_STRAP0__STRAP_D1_SUPPORT_DEV0_F3_MASK                                                   0x40000000L
+#define RCC_DEV0_EPF3_STRAP0__STRAP_D2_SUPPORT_DEV0_F3_MASK                                                   0x80000000L
+//RCC_DEV0_EPF3_STRAP2
+#define RCC_DEV0_EPF3_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F3__SHIFT                               0x4
+#define RCC_DEV0_EPF3_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F3__SHIFT                                              0x7
+#define RCC_DEV0_EPF3_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F3__SHIFT                                              0x8
+#define RCC_DEV0_EPF3_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F3__SHIFT                                            0x9
+#define RCC_DEV0_EPF3_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F3__SHIFT                                     0xe
+#define RCC_DEV0_EPF3_STRAP2__STRAP_AER_EN_DEV0_F3__SHIFT                                                     0x10
+#define RCC_DEV0_EPF3_STRAP2__STRAP_ACS_EN_DEV0_F3__SHIFT                                                     0x11
+#define RCC_DEV0_EPF3_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F3__SHIFT                                           0x14
+#define RCC_DEV0_EPF3_STRAP2__STRAP_DPA_EN_DEV0_F3__SHIFT                                                     0x15
+#define RCC_DEV0_EPF3_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F3__SHIFT                                              0x18
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_EN_DEV0_F3__SHIFT                                                   0x1c
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F3__SHIFT                             0x1d
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F3__SHIFT                          0x1e
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F3__SHIFT                                  0x1f
+#define RCC_DEV0_EPF3_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F3_MASK                                 0x00000010L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F3_MASK                                                0x00000080L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F3_MASK                                                0x00000100L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F3_MASK                                              0x00003E00L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F3_MASK                                       0x00004000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_AER_EN_DEV0_F3_MASK                                                       0x00010000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_ACS_EN_DEV0_F3_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F3_MASK                                             0x00100000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_DPA_EN_DEV0_F3_MASK                                                       0x00200000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F3_MASK                                                0x07000000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_EN_DEV0_F3_MASK                                                     0x10000000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F3_MASK                               0x20000000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F3_MASK                            0x40000000L
+#define RCC_DEV0_EPF3_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F3_MASK                                    0x80000000L
+//RCC_DEV0_EPF3_STRAP3
+#define RCC_DEV0_EPF3_STRAP3__STRAP_SUBSYS_ID_DEV0_F3__SHIFT                                                  0x0
+#define RCC_DEV0_EPF3_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F3__SHIFT                                 0x10
+#define RCC_DEV0_EPF3_STRAP3__STRAP_PWR_EN_DEV0_F3__SHIFT                                                     0x11
+#define RCC_DEV0_EPF3_STRAP3__STRAP_MSI_EN_DEV0_F3__SHIFT                                                     0x12
+#define RCC_DEV0_EPF3_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F3__SHIFT                                         0x13
+#define RCC_DEV0_EPF3_STRAP3__STRAP_MSIX_EN_DEV0_F3__SHIFT                                                    0x14
+#define RCC_DEV0_EPF3_STRAP3__STRAP_PMC_DSI_DEV0_F3__SHIFT                                                    0x18
+#define RCC_DEV0_EPF3_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F3__SHIFT                        0x19
+#define RCC_DEV0_EPF3_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F3__SHIFT                                   0x1a
+#define RCC_DEV0_EPF3_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F3__SHIFT                                  0x1b
+#define RCC_DEV0_EPF3_STRAP3__STRAP_CLK_PM_EN_DEV0_F3__SHIFT                                                  0x1d
+#define RCC_DEV0_EPF3_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F3__SHIFT                                          0x1e
+#define RCC_DEV0_EPF3_STRAP3__STRAP_RTR_EN_DEV0_F3__SHIFT                                                     0x1f
+#define RCC_DEV0_EPF3_STRAP3__STRAP_SUBSYS_ID_DEV0_F3_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF3_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F3_MASK                                   0x00010000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_PWR_EN_DEV0_F3_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_MSI_EN_DEV0_F3_MASK                                                       0x00040000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F3_MASK                                           0x00080000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_MSIX_EN_DEV0_F3_MASK                                                      0x00100000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_PMC_DSI_DEV0_F3_MASK                                                      0x01000000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F3_MASK                          0x02000000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F3_MASK                                     0x04000000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F3_MASK                                    0x08000000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_CLK_PM_EN_DEV0_F3_MASK                                                    0x20000000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F3_MASK                                            0x40000000L
+#define RCC_DEV0_EPF3_STRAP3__STRAP_RTR_EN_DEV0_F3_MASK                                                       0x80000000L
+//RCC_DEV0_EPF3_STRAP4
+#define RCC_DEV0_EPF3_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F3__SHIFT                                            0x14
+#define RCC_DEV0_EPF3_STRAP4__STRAP_ATOMIC_EN_DEV0_F3__SHIFT                                                  0x15
+#define RCC_DEV0_EPF3_STRAP4__STRAP_FLR_EN_DEV0_F3__SHIFT                                                     0x16
+#define RCC_DEV0_EPF3_STRAP4__STRAP_PME_SUPPORT_DEV0_F3__SHIFT                                                0x17
+#define RCC_DEV0_EPF3_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F3__SHIFT                                              0x1c
+#define RCC_DEV0_EPF3_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F3__SHIFT                                             0x1f
+#define RCC_DEV0_EPF3_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F3_MASK                                              0x00100000L
+#define RCC_DEV0_EPF3_STRAP4__STRAP_ATOMIC_EN_DEV0_F3_MASK                                                    0x00200000L
+#define RCC_DEV0_EPF3_STRAP4__STRAP_FLR_EN_DEV0_F3_MASK                                                       0x00400000L
+#define RCC_DEV0_EPF3_STRAP4__STRAP_PME_SUPPORT_DEV0_F3_MASK                                                  0x0F800000L
+#define RCC_DEV0_EPF3_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F3_MASK                                                0x70000000L
+#define RCC_DEV0_EPF3_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F3_MASK                                               0x80000000L
+//RCC_DEV0_EPF3_STRAP5
+#define RCC_DEV0_EPF3_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F3__SHIFT                                              0x0
+#define RCC_DEV0_EPF3_STRAP5__STRAP_AUX_CURRENT_DEV0_F3__SHIFT                                                0x1b
+#define RCC_DEV0_EPF3_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F3__SHIFT                                       0x1e
+#define RCC_DEV0_EPF3_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F3_MASK                                                0x0000FFFFL
+#define RCC_DEV0_EPF3_STRAP5__STRAP_AUX_CURRENT_DEV0_F3_MASK                                                  0x38000000L
+#define RCC_DEV0_EPF3_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F3_MASK                                         0x40000000L
+//RCC_DEV0_EPF3_STRAP6
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER0_EN_DEV0_F3__SHIFT                                                   0x0
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F3__SHIFT                                      0x1
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F3__SHIFT                                             0x2
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER1_EN_DEV0_F3__SHIFT                                                   0x8
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F3__SHIFT                                      0x9
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F3__SHIFT                                             0xa
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER2_EN_DEV0_F3__SHIFT                                                   0x10
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F3__SHIFT                                             0x12
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER3_EN_DEV0_F3__SHIFT                                                   0x18
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER4_EN_DEV0_F3__SHIFT                                                   0x1a
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER0_EN_DEV0_F3_MASK                                                     0x00000001L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F3_MASK                                        0x00000002L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F3_MASK                                               0x00000004L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER1_EN_DEV0_F3_MASK                                                     0x00000100L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F3_MASK                                        0x00000200L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F3_MASK                                               0x00000400L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER2_EN_DEV0_F3_MASK                                                     0x00010000L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F3_MASK                                               0x00040000L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER3_EN_DEV0_F3_MASK                                                     0x01000000L
+#define RCC_DEV0_EPF3_STRAP6__STRAP_APER4_EN_DEV0_F3_MASK                                                     0x04000000L
+//RCC_DEV0_EPF3_STRAP7
+//RCC_DEV0_EPF3_STRAP10
+//RCC_DEV0_EPF3_STRAP11
+//RCC_DEV0_EPF3_STRAP12
+//RCC_DEV0_EPF3_STRAP13
+#define RCC_DEV0_EPF3_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F3__SHIFT                                            0x0
+#define RCC_DEV0_EPF3_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F3__SHIFT                                            0x8
+#define RCC_DEV0_EPF3_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F3__SHIFT                                           0x10
+#define RCC_DEV0_EPF3_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F3_MASK                                              0x000000FFL
+#define RCC_DEV0_EPF3_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F3_MASK                                              0x0000FF00L
+#define RCC_DEV0_EPF3_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F3_MASK                                             0x00FF0000L
+//RCC_DEV0_EPF3_STRAP14
+#define RCC_DEV0_EPF3_STRAP14__STRAP_VENDOR_ID_DEV0_F3__SHIFT                                                 0x0
+#define RCC_DEV0_EPF3_STRAP14__STRAP_VENDOR_ID_DEV0_F3_MASK                                                   0x0000FFFFL
+//RCC_DEV0_EPF3_STRAP20
+//RCC_DEV0_EPF4_STRAP0
+#define RCC_DEV0_EPF4_STRAP0__STRAP_DEVICE_ID_DEV0_F4__SHIFT                                                  0x0
+#define RCC_DEV0_EPF4_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F4__SHIFT                                               0x10
+#define RCC_DEV0_EPF4_STRAP0__STRAP_MINOR_REV_ID_DEV0_F4__SHIFT                                               0x14
+#define RCC_DEV0_EPF4_STRAP0__STRAP_FUNC_EN_DEV0_F4__SHIFT                                                    0x1c
+#define RCC_DEV0_EPF4_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F4__SHIFT                                      0x1d
+#define RCC_DEV0_EPF4_STRAP0__STRAP_D1_SUPPORT_DEV0_F4__SHIFT                                                 0x1e
+#define RCC_DEV0_EPF4_STRAP0__STRAP_D2_SUPPORT_DEV0_F4__SHIFT                                                 0x1f
+#define RCC_DEV0_EPF4_STRAP0__STRAP_DEVICE_ID_DEV0_F4_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF4_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F4_MASK                                                 0x000F0000L
+#define RCC_DEV0_EPF4_STRAP0__STRAP_MINOR_REV_ID_DEV0_F4_MASK                                                 0x00F00000L
+#define RCC_DEV0_EPF4_STRAP0__STRAP_FUNC_EN_DEV0_F4_MASK                                                      0x10000000L
+#define RCC_DEV0_EPF4_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F4_MASK                                        0x20000000L
+#define RCC_DEV0_EPF4_STRAP0__STRAP_D1_SUPPORT_DEV0_F4_MASK                                                   0x40000000L
+#define RCC_DEV0_EPF4_STRAP0__STRAP_D2_SUPPORT_DEV0_F4_MASK                                                   0x80000000L
+//RCC_DEV0_EPF4_STRAP2
+#define RCC_DEV0_EPF4_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F4__SHIFT                               0x4
+#define RCC_DEV0_EPF4_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F4__SHIFT                                              0x7
+#define RCC_DEV0_EPF4_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F4__SHIFT                                              0x8
+#define RCC_DEV0_EPF4_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F4__SHIFT                                            0x9
+#define RCC_DEV0_EPF4_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F4__SHIFT                                     0xe
+#define RCC_DEV0_EPF4_STRAP2__STRAP_AER_EN_DEV0_F4__SHIFT                                                     0x10
+#define RCC_DEV0_EPF4_STRAP2__STRAP_ACS_EN_DEV0_F4__SHIFT                                                     0x11
+#define RCC_DEV0_EPF4_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F4__SHIFT                                           0x14
+#define RCC_DEV0_EPF4_STRAP2__STRAP_DPA_EN_DEV0_F4__SHIFT                                                     0x15
+#define RCC_DEV0_EPF4_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F4__SHIFT                                              0x18
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_EN_DEV0_F4__SHIFT                                                   0x1c
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F4__SHIFT                             0x1d
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F4__SHIFT                          0x1e
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F4__SHIFT                                  0x1f
+#define RCC_DEV0_EPF4_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F4_MASK                                 0x00000010L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F4_MASK                                                0x00000080L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F4_MASK                                                0x00000100L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F4_MASK                                              0x00003E00L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F4_MASK                                       0x00004000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_AER_EN_DEV0_F4_MASK                                                       0x00010000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_ACS_EN_DEV0_F4_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F4_MASK                                             0x00100000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_DPA_EN_DEV0_F4_MASK                                                       0x00200000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F4_MASK                                                0x07000000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_EN_DEV0_F4_MASK                                                     0x10000000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F4_MASK                               0x20000000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F4_MASK                            0x40000000L
+#define RCC_DEV0_EPF4_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F4_MASK                                    0x80000000L
+//RCC_DEV0_EPF4_STRAP3
+#define RCC_DEV0_EPF4_STRAP3__STRAP_SUBSYS_ID_DEV0_F4__SHIFT                                                  0x0
+#define RCC_DEV0_EPF4_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F4__SHIFT                                 0x10
+#define RCC_DEV0_EPF4_STRAP3__STRAP_PWR_EN_DEV0_F4__SHIFT                                                     0x11
+#define RCC_DEV0_EPF4_STRAP3__STRAP_MSI_EN_DEV0_F4__SHIFT                                                     0x12
+#define RCC_DEV0_EPF4_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F4__SHIFT                                         0x13
+#define RCC_DEV0_EPF4_STRAP3__STRAP_MSIX_EN_DEV0_F4__SHIFT                                                    0x14
+#define RCC_DEV0_EPF4_STRAP3__STRAP_PMC_DSI_DEV0_F4__SHIFT                                                    0x18
+#define RCC_DEV0_EPF4_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F4__SHIFT                        0x19
+#define RCC_DEV0_EPF4_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F4__SHIFT                                   0x1a
+#define RCC_DEV0_EPF4_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F4__SHIFT                                  0x1b
+#define RCC_DEV0_EPF4_STRAP3__STRAP_CLK_PM_EN_DEV0_F4__SHIFT                                                  0x1d
+#define RCC_DEV0_EPF4_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F4__SHIFT                                          0x1e
+#define RCC_DEV0_EPF4_STRAP3__STRAP_RTR_EN_DEV0_F4__SHIFT                                                     0x1f
+#define RCC_DEV0_EPF4_STRAP3__STRAP_SUBSYS_ID_DEV0_F4_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF4_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F4_MASK                                   0x00010000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_PWR_EN_DEV0_F4_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_MSI_EN_DEV0_F4_MASK                                                       0x00040000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F4_MASK                                           0x00080000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_MSIX_EN_DEV0_F4_MASK                                                      0x00100000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_PMC_DSI_DEV0_F4_MASK                                                      0x01000000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F4_MASK                          0x02000000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F4_MASK                                     0x04000000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F4_MASK                                    0x08000000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_CLK_PM_EN_DEV0_F4_MASK                                                    0x20000000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F4_MASK                                            0x40000000L
+#define RCC_DEV0_EPF4_STRAP3__STRAP_RTR_EN_DEV0_F4_MASK                                                       0x80000000L
+//RCC_DEV0_EPF4_STRAP4
+#define RCC_DEV0_EPF4_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F4__SHIFT                                            0x14
+#define RCC_DEV0_EPF4_STRAP4__STRAP_ATOMIC_EN_DEV0_F4__SHIFT                                                  0x15
+#define RCC_DEV0_EPF4_STRAP4__STRAP_FLR_EN_DEV0_F4__SHIFT                                                     0x16
+#define RCC_DEV0_EPF4_STRAP4__STRAP_PME_SUPPORT_DEV0_F4__SHIFT                                                0x17
+#define RCC_DEV0_EPF4_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F4__SHIFT                                              0x1c
+#define RCC_DEV0_EPF4_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F4__SHIFT                                             0x1f
+#define RCC_DEV0_EPF4_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F4_MASK                                              0x00100000L
+#define RCC_DEV0_EPF4_STRAP4__STRAP_ATOMIC_EN_DEV0_F4_MASK                                                    0x00200000L
+#define RCC_DEV0_EPF4_STRAP4__STRAP_FLR_EN_DEV0_F4_MASK                                                       0x00400000L
+#define RCC_DEV0_EPF4_STRAP4__STRAP_PME_SUPPORT_DEV0_F4_MASK                                                  0x0F800000L
+#define RCC_DEV0_EPF4_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F4_MASK                                                0x70000000L
+#define RCC_DEV0_EPF4_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F4_MASK                                               0x80000000L
+//RCC_DEV0_EPF4_STRAP5
+#define RCC_DEV0_EPF4_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F4__SHIFT                                              0x0
+#define RCC_DEV0_EPF4_STRAP5__STRAP_USB_DBESEL_DEV0_F4__SHIFT                                                 0x10
+#define RCC_DEV0_EPF4_STRAP5__STRAP_USB_DBESELD_DEV0_F4__SHIFT                                                0x14
+#define RCC_DEV0_EPF4_STRAP5__STRAP_AUX_CURRENT_DEV0_F4__SHIFT                                                0x1b
+#define RCC_DEV0_EPF4_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F4__SHIFT                                       0x1e
+#define RCC_DEV0_EPF4_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F4_MASK                                                0x0000FFFFL
+#define RCC_DEV0_EPF4_STRAP5__STRAP_USB_DBESEL_DEV0_F4_MASK                                                   0x000F0000L
+#define RCC_DEV0_EPF4_STRAP5__STRAP_USB_DBESELD_DEV0_F4_MASK                                                  0x00F00000L
+#define RCC_DEV0_EPF4_STRAP5__STRAP_AUX_CURRENT_DEV0_F4_MASK                                                  0x38000000L
+#define RCC_DEV0_EPF4_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F4_MASK                                         0x40000000L
+//RCC_DEV0_EPF4_STRAP6
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER0_EN_DEV0_F4__SHIFT                                                   0x0
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F4__SHIFT                                             0x2
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER1_EN_DEV0_F4__SHIFT                                                   0x8
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F4__SHIFT                                             0xa
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER2_EN_DEV0_F4__SHIFT                                                   0x10
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F4__SHIFT                                             0x12
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER3_EN_DEV0_F4__SHIFT                                                   0x18
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER4_EN_DEV0_F4__SHIFT                                                   0x1a
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER0_EN_DEV0_F4_MASK                                                     0x00000001L
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F4_MASK                                               0x00000004L
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER1_EN_DEV0_F4_MASK                                                     0x00000100L
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F4_MASK                                               0x00000400L
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER2_EN_DEV0_F4_MASK                                                     0x00010000L
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F4_MASK                                               0x00040000L
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER3_EN_DEV0_F4_MASK                                                     0x01000000L
+#define RCC_DEV0_EPF4_STRAP6__STRAP_APER4_EN_DEV0_F4_MASK                                                     0x04000000L
+//RCC_DEV0_EPF4_STRAP7
+#define RCC_DEV0_EPF4_STRAP7__STRAP_MSIX_TABLE_SIZE_DEV0_F4__SHIFT                                            0x5
+#define RCC_DEV0_EPF4_STRAP7__STRAP_MSIX_TABLE_SIZE_DEV0_F4_MASK                                              0x0000FFE0L
+//RCC_DEV0_EPF4_STRAP13
+#define RCC_DEV0_EPF4_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F4__SHIFT                                            0x0
+#define RCC_DEV0_EPF4_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F4__SHIFT                                            0x8
+#define RCC_DEV0_EPF4_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F4__SHIFT                                           0x10
+#define RCC_DEV0_EPF4_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F4_MASK                                              0x000000FFL
+#define RCC_DEV0_EPF4_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F4_MASK                                              0x0000FF00L
+#define RCC_DEV0_EPF4_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F4_MASK                                             0x00FF0000L
+//RCC_DEV0_EPF4_STRAP14
+#define RCC_DEV0_EPF4_STRAP14__STRAP_VENDOR_ID_DEV0_F4__SHIFT                                                 0x0
+#define RCC_DEV0_EPF4_STRAP14__STRAP_VENDOR_ID_DEV0_F4_MASK                                                   0x0000FFFFL
+//RCC_DEV0_EPF5_STRAP0
+#define RCC_DEV0_EPF5_STRAP0__STRAP_DEVICE_ID_DEV0_F5__SHIFT                                                  0x0
+#define RCC_DEV0_EPF5_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F5__SHIFT                                               0x10
+#define RCC_DEV0_EPF5_STRAP0__STRAP_MINOR_REV_ID_DEV0_F5__SHIFT                                               0x14
+#define RCC_DEV0_EPF5_STRAP0__STRAP_FUNC_EN_DEV0_F5__SHIFT                                                    0x1c
+#define RCC_DEV0_EPF5_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F5__SHIFT                                      0x1d
+#define RCC_DEV0_EPF5_STRAP0__STRAP_D1_SUPPORT_DEV0_F5__SHIFT                                                 0x1e
+#define RCC_DEV0_EPF5_STRAP0__STRAP_D2_SUPPORT_DEV0_F5__SHIFT                                                 0x1f
+#define RCC_DEV0_EPF5_STRAP0__STRAP_DEVICE_ID_DEV0_F5_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF5_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F5_MASK                                                 0x000F0000L
+#define RCC_DEV0_EPF5_STRAP0__STRAP_MINOR_REV_ID_DEV0_F5_MASK                                                 0x00F00000L
+#define RCC_DEV0_EPF5_STRAP0__STRAP_FUNC_EN_DEV0_F5_MASK                                                      0x10000000L
+#define RCC_DEV0_EPF5_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F5_MASK                                        0x20000000L
+#define RCC_DEV0_EPF5_STRAP0__STRAP_D1_SUPPORT_DEV0_F5_MASK                                                   0x40000000L
+#define RCC_DEV0_EPF5_STRAP0__STRAP_D2_SUPPORT_DEV0_F5_MASK                                                   0x80000000L
+//RCC_DEV0_EPF5_STRAP2
+#define RCC_DEV0_EPF5_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F5__SHIFT                               0x4
+#define RCC_DEV0_EPF5_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F5__SHIFT                                              0x7
+#define RCC_DEV0_EPF5_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F5__SHIFT                                              0x8
+#define RCC_DEV0_EPF5_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F5__SHIFT                                            0x9
+#define RCC_DEV0_EPF5_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F5__SHIFT                                     0xe
+#define RCC_DEV0_EPF5_STRAP2__STRAP_AER_EN_DEV0_F5__SHIFT                                                     0x10
+#define RCC_DEV0_EPF5_STRAP2__STRAP_ACS_EN_DEV0_F5__SHIFT                                                     0x11
+#define RCC_DEV0_EPF5_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F5__SHIFT                                           0x14
+#define RCC_DEV0_EPF5_STRAP2__STRAP_DPA_EN_DEV0_F5__SHIFT                                                     0x15
+#define RCC_DEV0_EPF5_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F5__SHIFT                                              0x18
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_EN_DEV0_F5__SHIFT                                                   0x1c
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F5__SHIFT                             0x1d
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F5__SHIFT                          0x1e
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F5__SHIFT                                  0x1f
+#define RCC_DEV0_EPF5_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F5_MASK                                 0x00000010L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F5_MASK                                                0x00000080L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F5_MASK                                                0x00000100L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F5_MASK                                              0x00003E00L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F5_MASK                                       0x00004000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_AER_EN_DEV0_F5_MASK                                                       0x00010000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_ACS_EN_DEV0_F5_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F5_MASK                                             0x00100000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_DPA_EN_DEV0_F5_MASK                                                       0x00200000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F5_MASK                                                0x07000000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_EN_DEV0_F5_MASK                                                     0x10000000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F5_MASK                               0x20000000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F5_MASK                            0x40000000L
+#define RCC_DEV0_EPF5_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F5_MASK                                    0x80000000L
+//RCC_DEV0_EPF5_STRAP3
+#define RCC_DEV0_EPF5_STRAP3__STRAP_SUBSYS_ID_DEV0_F5__SHIFT                                                  0x0
+#define RCC_DEV0_EPF5_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F5__SHIFT                                 0x10
+#define RCC_DEV0_EPF5_STRAP3__STRAP_PWR_EN_DEV0_F5__SHIFT                                                     0x11
+#define RCC_DEV0_EPF5_STRAP3__STRAP_MSI_EN_DEV0_F5__SHIFT                                                     0x12
+#define RCC_DEV0_EPF5_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F5__SHIFT                                         0x13
+#define RCC_DEV0_EPF5_STRAP3__STRAP_MSIX_EN_DEV0_F5__SHIFT                                                    0x14
+#define RCC_DEV0_EPF5_STRAP3__STRAP_PMC_DSI_DEV0_F5__SHIFT                                                    0x18
+#define RCC_DEV0_EPF5_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F5__SHIFT                        0x19
+#define RCC_DEV0_EPF5_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F5__SHIFT                                   0x1a
+#define RCC_DEV0_EPF5_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F5__SHIFT                                  0x1b
+#define RCC_DEV0_EPF5_STRAP3__STRAP_CLK_PM_EN_DEV0_F5__SHIFT                                                  0x1d
+#define RCC_DEV0_EPF5_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F5__SHIFT                                          0x1e
+#define RCC_DEV0_EPF5_STRAP3__STRAP_RTR_EN_DEV0_F5__SHIFT                                                     0x1f
+#define RCC_DEV0_EPF5_STRAP3__STRAP_SUBSYS_ID_DEV0_F5_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF5_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F5_MASK                                   0x00010000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_PWR_EN_DEV0_F5_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_MSI_EN_DEV0_F5_MASK                                                       0x00040000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F5_MASK                                           0x00080000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_MSIX_EN_DEV0_F5_MASK                                                      0x00100000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_PMC_DSI_DEV0_F5_MASK                                                      0x01000000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F5_MASK                          0x02000000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F5_MASK                                     0x04000000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F5_MASK                                    0x08000000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_CLK_PM_EN_DEV0_F5_MASK                                                    0x20000000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F5_MASK                                            0x40000000L
+#define RCC_DEV0_EPF5_STRAP3__STRAP_RTR_EN_DEV0_F5_MASK                                                       0x80000000L
+//RCC_DEV0_EPF5_STRAP4
+#define RCC_DEV0_EPF5_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F5__SHIFT                                            0x14
+#define RCC_DEV0_EPF5_STRAP4__STRAP_ATOMIC_EN_DEV0_F5__SHIFT                                                  0x15
+#define RCC_DEV0_EPF5_STRAP4__STRAP_FLR_EN_DEV0_F5__SHIFT                                                     0x16
+#define RCC_DEV0_EPF5_STRAP4__STRAP_PME_SUPPORT_DEV0_F5__SHIFT                                                0x17
+#define RCC_DEV0_EPF5_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F5__SHIFT                                              0x1c
+#define RCC_DEV0_EPF5_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F5__SHIFT                                             0x1f
+#define RCC_DEV0_EPF5_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F5_MASK                                              0x00100000L
+#define RCC_DEV0_EPF5_STRAP4__STRAP_ATOMIC_EN_DEV0_F5_MASK                                                    0x00200000L
+#define RCC_DEV0_EPF5_STRAP4__STRAP_FLR_EN_DEV0_F5_MASK                                                       0x00400000L
+#define RCC_DEV0_EPF5_STRAP4__STRAP_PME_SUPPORT_DEV0_F5_MASK                                                  0x0F800000L
+#define RCC_DEV0_EPF5_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F5_MASK                                                0x70000000L
+#define RCC_DEV0_EPF5_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F5_MASK                                               0x80000000L
+//RCC_DEV0_EPF5_STRAP5
+#define RCC_DEV0_EPF5_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F5__SHIFT                                              0x0
+#define RCC_DEV0_EPF5_STRAP5__STRAP_AUX_CURRENT_DEV0_F5__SHIFT                                                0x1b
+#define RCC_DEV0_EPF5_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F5__SHIFT                                       0x1e
+#define RCC_DEV0_EPF5_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F5_MASK                                                0x0000FFFFL
+#define RCC_DEV0_EPF5_STRAP5__STRAP_AUX_CURRENT_DEV0_F5_MASK                                                  0x38000000L
+#define RCC_DEV0_EPF5_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F5_MASK                                         0x40000000L
+//RCC_DEV0_EPF5_STRAP6
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER0_EN_DEV0_F5__SHIFT                                                   0x0
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F5__SHIFT                                             0x2
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER1_EN_DEV0_F5__SHIFT                                                   0x8
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F5__SHIFT                                             0xa
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER2_EN_DEV0_F5__SHIFT                                                   0x10
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F5__SHIFT                                             0x12
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER3_EN_DEV0_F5__SHIFT                                                   0x18
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER4_EN_DEV0_F5__SHIFT                                                   0x1a
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER0_EN_DEV0_F5_MASK                                                     0x00000001L
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F5_MASK                                               0x00000004L
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER1_EN_DEV0_F5_MASK                                                     0x00000100L
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F5_MASK                                               0x00000400L
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER2_EN_DEV0_F5_MASK                                                     0x00010000L
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F5_MASK                                               0x00040000L
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER3_EN_DEV0_F5_MASK                                                     0x01000000L
+#define RCC_DEV0_EPF5_STRAP6__STRAP_APER4_EN_DEV0_F5_MASK                                                     0x04000000L
+//RCC_DEV0_EPF5_STRAP7
+//RCC_DEV0_EPF5_STRAP13
+#define RCC_DEV0_EPF5_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F5__SHIFT                                            0x0
+#define RCC_DEV0_EPF5_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F5__SHIFT                                            0x8
+#define RCC_DEV0_EPF5_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F5__SHIFT                                           0x10
+#define RCC_DEV0_EPF5_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F5_MASK                                              0x000000FFL
+#define RCC_DEV0_EPF5_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F5_MASK                                              0x0000FF00L
+#define RCC_DEV0_EPF5_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F5_MASK                                             0x00FF0000L
+//RCC_DEV0_EPF5_STRAP14
+#define RCC_DEV0_EPF5_STRAP14__STRAP_VENDOR_ID_DEV0_F5__SHIFT                                                 0x0
+#define RCC_DEV0_EPF5_STRAP14__STRAP_VENDOR_ID_DEV0_F5_MASK                                                   0x0000FFFFL
+//RCC_DEV0_EPF6_STRAP0
+#define RCC_DEV0_EPF6_STRAP0__STRAP_DEVICE_ID_DEV0_F6__SHIFT                                                  0x0
+#define RCC_DEV0_EPF6_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F6__SHIFT                                               0x10
+#define RCC_DEV0_EPF6_STRAP0__STRAP_MINOR_REV_ID_DEV0_F6__SHIFT                                               0x14
+#define RCC_DEV0_EPF6_STRAP0__STRAP_FUNC_EN_DEV0_F6__SHIFT                                                    0x1c
+#define RCC_DEV0_EPF6_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F6__SHIFT                                      0x1d
+#define RCC_DEV0_EPF6_STRAP0__STRAP_D1_SUPPORT_DEV0_F6__SHIFT                                                 0x1e
+#define RCC_DEV0_EPF6_STRAP0__STRAP_D2_SUPPORT_DEV0_F6__SHIFT                                                 0x1f
+#define RCC_DEV0_EPF6_STRAP0__STRAP_DEVICE_ID_DEV0_F6_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF6_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F6_MASK                                                 0x000F0000L
+#define RCC_DEV0_EPF6_STRAP0__STRAP_MINOR_REV_ID_DEV0_F6_MASK                                                 0x00F00000L
+#define RCC_DEV0_EPF6_STRAP0__STRAP_FUNC_EN_DEV0_F6_MASK                                                      0x10000000L
+#define RCC_DEV0_EPF6_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F6_MASK                                        0x20000000L
+#define RCC_DEV0_EPF6_STRAP0__STRAP_D1_SUPPORT_DEV0_F6_MASK                                                   0x40000000L
+#define RCC_DEV0_EPF6_STRAP0__STRAP_D2_SUPPORT_DEV0_F6_MASK                                                   0x80000000L
+//RCC_DEV0_EPF6_STRAP2
+#define RCC_DEV0_EPF6_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F6__SHIFT                               0x4
+#define RCC_DEV0_EPF6_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F6__SHIFT                                              0x7
+#define RCC_DEV0_EPF6_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F6__SHIFT                                              0x8
+#define RCC_DEV0_EPF6_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F6__SHIFT                                            0x9
+#define RCC_DEV0_EPF6_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F6__SHIFT                                     0xe
+#define RCC_DEV0_EPF6_STRAP2__STRAP_AER_EN_DEV0_F6__SHIFT                                                     0x10
+#define RCC_DEV0_EPF6_STRAP2__STRAP_ACS_EN_DEV0_F6__SHIFT                                                     0x11
+#define RCC_DEV0_EPF6_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F6__SHIFT                                           0x14
+#define RCC_DEV0_EPF6_STRAP2__STRAP_DPA_EN_DEV0_F6__SHIFT                                                     0x15
+#define RCC_DEV0_EPF6_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F6__SHIFT                                              0x18
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_EN_DEV0_F6__SHIFT                                                   0x1c
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F6__SHIFT                             0x1d
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F6__SHIFT                          0x1e
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F6__SHIFT                                  0x1f
+#define RCC_DEV0_EPF6_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F6_MASK                                 0x00000010L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F6_MASK                                                0x00000080L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F6_MASK                                                0x00000100L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F6_MASK                                              0x00003E00L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F6_MASK                                       0x00004000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_AER_EN_DEV0_F6_MASK                                                       0x00010000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_ACS_EN_DEV0_F6_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F6_MASK                                             0x00100000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_DPA_EN_DEV0_F6_MASK                                                       0x00200000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F6_MASK                                                0x07000000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_EN_DEV0_F6_MASK                                                     0x10000000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F6_MASK                               0x20000000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F6_MASK                            0x40000000L
+#define RCC_DEV0_EPF6_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F6_MASK                                    0x80000000L
+//RCC_DEV0_EPF6_STRAP3
+#define RCC_DEV0_EPF6_STRAP3__STRAP_SUBSYS_ID_DEV0_F6__SHIFT                                                  0x0
+#define RCC_DEV0_EPF6_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F6__SHIFT                                 0x10
+#define RCC_DEV0_EPF6_STRAP3__STRAP_PWR_EN_DEV0_F6__SHIFT                                                     0x11
+#define RCC_DEV0_EPF6_STRAP3__STRAP_MSI_EN_DEV0_F6__SHIFT                                                     0x12
+#define RCC_DEV0_EPF6_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F6__SHIFT                                         0x13
+#define RCC_DEV0_EPF6_STRAP3__STRAP_MSIX_EN_DEV0_F6__SHIFT                                                    0x14
+#define RCC_DEV0_EPF6_STRAP3__STRAP_PMC_DSI_DEV0_F6__SHIFT                                                    0x18
+#define RCC_DEV0_EPF6_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F6__SHIFT                        0x19
+#define RCC_DEV0_EPF6_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F6__SHIFT                                   0x1a
+#define RCC_DEV0_EPF6_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F6__SHIFT                                  0x1b
+#define RCC_DEV0_EPF6_STRAP3__STRAP_CLK_PM_EN_DEV0_F6__SHIFT                                                  0x1d
+#define RCC_DEV0_EPF6_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F6__SHIFT                                          0x1e
+#define RCC_DEV0_EPF6_STRAP3__STRAP_RTR_EN_DEV0_F6__SHIFT                                                     0x1f
+#define RCC_DEV0_EPF6_STRAP3__STRAP_SUBSYS_ID_DEV0_F6_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF6_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F6_MASK                                   0x00010000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_PWR_EN_DEV0_F6_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_MSI_EN_DEV0_F6_MASK                                                       0x00040000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F6_MASK                                           0x00080000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_MSIX_EN_DEV0_F6_MASK                                                      0x00100000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_PMC_DSI_DEV0_F6_MASK                                                      0x01000000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F6_MASK                          0x02000000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F6_MASK                                     0x04000000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F6_MASK                                    0x08000000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_CLK_PM_EN_DEV0_F6_MASK                                                    0x20000000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F6_MASK                                            0x40000000L
+#define RCC_DEV0_EPF6_STRAP3__STRAP_RTR_EN_DEV0_F6_MASK                                                       0x80000000L
+//RCC_DEV0_EPF6_STRAP4
+#define RCC_DEV0_EPF6_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F6__SHIFT                                            0x14
+#define RCC_DEV0_EPF6_STRAP4__STRAP_ATOMIC_EN_DEV0_F6__SHIFT                                                  0x15
+#define RCC_DEV0_EPF6_STRAP4__STRAP_FLR_EN_DEV0_F6__SHIFT                                                     0x16
+#define RCC_DEV0_EPF6_STRAP4__STRAP_PME_SUPPORT_DEV0_F6__SHIFT                                                0x17
+#define RCC_DEV0_EPF6_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F6__SHIFT                                              0x1c
+#define RCC_DEV0_EPF6_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F6__SHIFT                                             0x1f
+#define RCC_DEV0_EPF6_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F6_MASK                                              0x00100000L
+#define RCC_DEV0_EPF6_STRAP4__STRAP_ATOMIC_EN_DEV0_F6_MASK                                                    0x00200000L
+#define RCC_DEV0_EPF6_STRAP4__STRAP_FLR_EN_DEV0_F6_MASK                                                       0x00400000L
+#define RCC_DEV0_EPF6_STRAP4__STRAP_PME_SUPPORT_DEV0_F6_MASK                                                  0x0F800000L
+#define RCC_DEV0_EPF6_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F6_MASK                                                0x70000000L
+#define RCC_DEV0_EPF6_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F6_MASK                                               0x80000000L
+//RCC_DEV0_EPF6_STRAP5
+#define RCC_DEV0_EPF6_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F6__SHIFT                                              0x0
+#define RCC_DEV0_EPF6_STRAP5__STRAP_AUX_CURRENT_DEV0_F6__SHIFT                                                0x1b
+#define RCC_DEV0_EPF6_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F6__SHIFT                                       0x1e
+#define RCC_DEV0_EPF6_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F6_MASK                                                0x0000FFFFL
+#define RCC_DEV0_EPF6_STRAP5__STRAP_AUX_CURRENT_DEV0_F6_MASK                                                  0x38000000L
+#define RCC_DEV0_EPF6_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F6_MASK                                         0x40000000L
+//RCC_DEV0_EPF6_STRAP6
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER0_EN_DEV0_F6__SHIFT                                                   0x0
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F6__SHIFT                                             0x2
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER1_EN_DEV0_F6__SHIFT                                                   0x8
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F6__SHIFT                                             0xa
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER2_EN_DEV0_F6__SHIFT                                                   0x10
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F6__SHIFT                                             0x12
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER3_EN_DEV0_F6__SHIFT                                                   0x18
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER4_EN_DEV0_F6__SHIFT                                                   0x1a
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER0_EN_DEV0_F6_MASK                                                     0x00000001L
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F6_MASK                                               0x00000004L
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER1_EN_DEV0_F6_MASK                                                     0x00000100L
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F6_MASK                                               0x00000400L
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER2_EN_DEV0_F6_MASK                                                     0x00010000L
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F6_MASK                                               0x00040000L
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER3_EN_DEV0_F6_MASK                                                     0x01000000L
+#define RCC_DEV0_EPF6_STRAP6__STRAP_APER4_EN_DEV0_F6_MASK                                                     0x04000000L
+//RCC_DEV0_EPF6_STRAP13
+#define RCC_DEV0_EPF6_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F6__SHIFT                                            0x0
+#define RCC_DEV0_EPF6_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F6__SHIFT                                            0x8
+#define RCC_DEV0_EPF6_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F6__SHIFT                                           0x10
+#define RCC_DEV0_EPF6_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F6_MASK                                              0x000000FFL
+#define RCC_DEV0_EPF6_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F6_MASK                                              0x0000FF00L
+#define RCC_DEV0_EPF6_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F6_MASK                                             0x00FF0000L
+//RCC_DEV0_EPF6_STRAP14
+#define RCC_DEV0_EPF6_STRAP14__STRAP_VENDOR_ID_DEV0_F6__SHIFT                                                 0x0
+#define RCC_DEV0_EPF6_STRAP14__STRAP_VENDOR_ID_DEV0_F6_MASK                                                   0x0000FFFFL
+//RCC_DEV0_EPF7_STRAP0
+#define RCC_DEV0_EPF7_STRAP0__STRAP_DEVICE_ID_DEV0_F7__SHIFT                                                  0x0
+#define RCC_DEV0_EPF7_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F7__SHIFT                                               0x10
+#define RCC_DEV0_EPF7_STRAP0__STRAP_MINOR_REV_ID_DEV0_F7__SHIFT                                               0x14
+#define RCC_DEV0_EPF7_STRAP0__STRAP_FUNC_EN_DEV0_F7__SHIFT                                                    0x1c
+#define RCC_DEV0_EPF7_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F7__SHIFT                                      0x1d
+#define RCC_DEV0_EPF7_STRAP0__STRAP_D1_SUPPORT_DEV0_F7__SHIFT                                                 0x1e
+#define RCC_DEV0_EPF7_STRAP0__STRAP_D2_SUPPORT_DEV0_F7__SHIFT                                                 0x1f
+#define RCC_DEV0_EPF7_STRAP0__STRAP_DEVICE_ID_DEV0_F7_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF7_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F7_MASK                                                 0x000F0000L
+#define RCC_DEV0_EPF7_STRAP0__STRAP_MINOR_REV_ID_DEV0_F7_MASK                                                 0x00F00000L
+#define RCC_DEV0_EPF7_STRAP0__STRAP_FUNC_EN_DEV0_F7_MASK                                                      0x10000000L
+#define RCC_DEV0_EPF7_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F7_MASK                                        0x20000000L
+#define RCC_DEV0_EPF7_STRAP0__STRAP_D1_SUPPORT_DEV0_F7_MASK                                                   0x40000000L
+#define RCC_DEV0_EPF7_STRAP0__STRAP_D2_SUPPORT_DEV0_F7_MASK                                                   0x80000000L
+//RCC_DEV0_EPF7_STRAP2
+#define RCC_DEV0_EPF7_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F7__SHIFT                               0x4
+#define RCC_DEV0_EPF7_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F7__SHIFT                                              0x7
+#define RCC_DEV0_EPF7_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F7__SHIFT                                              0x8
+#define RCC_DEV0_EPF7_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F7__SHIFT                                            0x9
+#define RCC_DEV0_EPF7_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F7__SHIFT                                     0xe
+#define RCC_DEV0_EPF7_STRAP2__STRAP_AER_EN_DEV0_F7__SHIFT                                                     0x10
+#define RCC_DEV0_EPF7_STRAP2__STRAP_ACS_EN_DEV0_F7__SHIFT                                                     0x11
+#define RCC_DEV0_EPF7_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F7__SHIFT                                           0x14
+#define RCC_DEV0_EPF7_STRAP2__STRAP_DPA_EN_DEV0_F7__SHIFT                                                     0x15
+#define RCC_DEV0_EPF7_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F7__SHIFT                                              0x18
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_EN_DEV0_F7__SHIFT                                                   0x1c
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F7__SHIFT                             0x1d
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F7__SHIFT                          0x1e
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F7__SHIFT                                  0x1f
+#define RCC_DEV0_EPF7_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F7_MASK                                 0x00000010L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F7_MASK                                                0x00000080L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F7_MASK                                                0x00000100L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F7_MASK                                              0x00003E00L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F7_MASK                                       0x00004000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_AER_EN_DEV0_F7_MASK                                                       0x00010000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_ACS_EN_DEV0_F7_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F7_MASK                                             0x00100000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_DPA_EN_DEV0_F7_MASK                                                       0x00200000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F7_MASK                                                0x07000000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_EN_DEV0_F7_MASK                                                     0x10000000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F7_MASK                               0x20000000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F7_MASK                            0x40000000L
+#define RCC_DEV0_EPF7_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F7_MASK                                    0x80000000L
+//RCC_DEV0_EPF7_STRAP3
+#define RCC_DEV0_EPF7_STRAP3__STRAP_SUBSYS_ID_DEV0_F7__SHIFT                                                  0x0
+#define RCC_DEV0_EPF7_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F7__SHIFT                                 0x10
+#define RCC_DEV0_EPF7_STRAP3__STRAP_PWR_EN_DEV0_F7__SHIFT                                                     0x11
+#define RCC_DEV0_EPF7_STRAP3__STRAP_MSI_EN_DEV0_F7__SHIFT                                                     0x12
+#define RCC_DEV0_EPF7_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F7__SHIFT                                         0x13
+#define RCC_DEV0_EPF7_STRAP3__STRAP_MSIX_EN_DEV0_F7__SHIFT                                                    0x14
+#define RCC_DEV0_EPF7_STRAP3__STRAP_PMC_DSI_DEV0_F7__SHIFT                                                    0x18
+#define RCC_DEV0_EPF7_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F7__SHIFT                        0x19
+#define RCC_DEV0_EPF7_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F7__SHIFT                                   0x1a
+#define RCC_DEV0_EPF7_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F7__SHIFT                                  0x1b
+#define RCC_DEV0_EPF7_STRAP3__STRAP_CLK_PM_EN_DEV0_F7__SHIFT                                                  0x1d
+#define RCC_DEV0_EPF7_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F7__SHIFT                                          0x1e
+#define RCC_DEV0_EPF7_STRAP3__STRAP_RTR_EN_DEV0_F7__SHIFT                                                     0x1f
+#define RCC_DEV0_EPF7_STRAP3__STRAP_SUBSYS_ID_DEV0_F7_MASK                                                    0x0000FFFFL
+#define RCC_DEV0_EPF7_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F7_MASK                                   0x00010000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_PWR_EN_DEV0_F7_MASK                                                       0x00020000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_MSI_EN_DEV0_F7_MASK                                                       0x00040000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F7_MASK                                           0x00080000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_MSIX_EN_DEV0_F7_MASK                                                      0x00100000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_PMC_DSI_DEV0_F7_MASK                                                      0x01000000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F7_MASK                          0x02000000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F7_MASK                                     0x04000000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F7_MASK                                    0x08000000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_CLK_PM_EN_DEV0_F7_MASK                                                    0x20000000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F7_MASK                                            0x40000000L
+#define RCC_DEV0_EPF7_STRAP3__STRAP_RTR_EN_DEV0_F7_MASK                                                       0x80000000L
+//RCC_DEV0_EPF7_STRAP4
+#define RCC_DEV0_EPF7_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F7__SHIFT                                            0x14
+#define RCC_DEV0_EPF7_STRAP4__STRAP_ATOMIC_EN_DEV0_F7__SHIFT                                                  0x15
+#define RCC_DEV0_EPF7_STRAP4__STRAP_FLR_EN_DEV0_F7__SHIFT                                                     0x16
+#define RCC_DEV0_EPF7_STRAP4__STRAP_PME_SUPPORT_DEV0_F7__SHIFT                                                0x17
+#define RCC_DEV0_EPF7_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F7__SHIFT                                              0x1c
+#define RCC_DEV0_EPF7_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F7__SHIFT                                             0x1f
+#define RCC_DEV0_EPF7_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F7_MASK                                              0x00100000L
+#define RCC_DEV0_EPF7_STRAP4__STRAP_ATOMIC_EN_DEV0_F7_MASK                                                    0x00200000L
+#define RCC_DEV0_EPF7_STRAP4__STRAP_FLR_EN_DEV0_F7_MASK                                                       0x00400000L
+#define RCC_DEV0_EPF7_STRAP4__STRAP_PME_SUPPORT_DEV0_F7_MASK                                                  0x0F800000L
+#define RCC_DEV0_EPF7_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F7_MASK                                                0x70000000L
+#define RCC_DEV0_EPF7_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F7_MASK                                               0x80000000L
+//RCC_DEV0_EPF7_STRAP5
+#define RCC_DEV0_EPF7_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F7__SHIFT                                              0x0
+#define RCC_DEV0_EPF7_STRAP5__STRAP_AUX_CURRENT_DEV0_F7__SHIFT                                                0x1b
+#define RCC_DEV0_EPF7_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F7__SHIFT                                       0x1e
+#define RCC_DEV0_EPF7_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F7_MASK                                                0x0000FFFFL
+#define RCC_DEV0_EPF7_STRAP5__STRAP_AUX_CURRENT_DEV0_F7_MASK                                                  0x38000000L
+#define RCC_DEV0_EPF7_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F7_MASK                                         0x40000000L
+//RCC_DEV0_EPF7_STRAP6
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER0_EN_DEV0_F7__SHIFT                                                   0x0
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F7__SHIFT                                             0x2
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER1_EN_DEV0_F7__SHIFT                                                   0x8
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F7__SHIFT                                             0xa
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER2_EN_DEV0_F7__SHIFT                                                   0x10
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F7__SHIFT                                             0x12
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER3_EN_DEV0_F7__SHIFT                                                   0x18
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER4_EN_DEV0_F7__SHIFT                                                   0x1a
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER0_EN_DEV0_F7_MASK                                                     0x00000001L
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F7_MASK                                               0x00000004L
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER1_EN_DEV0_F7_MASK                                                     0x00000100L
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F7_MASK                                               0x00000400L
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER2_EN_DEV0_F7_MASK                                                     0x00010000L
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F7_MASK                                               0x00040000L
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER3_EN_DEV0_F7_MASK                                                     0x01000000L
+#define RCC_DEV0_EPF7_STRAP6__STRAP_APER4_EN_DEV0_F7_MASK                                                     0x04000000L
+//RCC_DEV0_EPF7_STRAP13
+#define RCC_DEV0_EPF7_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F7__SHIFT                                            0x0
+#define RCC_DEV0_EPF7_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F7__SHIFT                                            0x8
+#define RCC_DEV0_EPF7_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F7__SHIFT                                           0x10
+#define RCC_DEV0_EPF7_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F7_MASK                                              0x000000FFL
+#define RCC_DEV0_EPF7_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F7_MASK                                              0x0000FF00L
+#define RCC_DEV0_EPF7_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F7_MASK                                             0x00FF0000L
+//RCC_DEV0_EPF7_STRAP14
+#define RCC_DEV0_EPF7_STRAP14__STRAP_VENDOR_ID_DEV0_F7__SHIFT                                                 0x0
+#define RCC_DEV0_EPF7_STRAP14__STRAP_VENDOR_ID_DEV0_F7_MASK                                                   0x0000FFFFL
+//RCC_DEV1_EPF0_STRAP0
+#define RCC_DEV1_EPF0_STRAP0__STRAP_DEVICE_ID_DEV1_F0__SHIFT                                                  0x0
+#define RCC_DEV1_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F0__SHIFT                                               0x10
+#define RCC_DEV1_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV1_F0__SHIFT                                               0x14
+#define RCC_DEV1_EPF0_STRAP0__STRAP_FUNC_EN_DEV1_F0__SHIFT                                                    0x1c
+#define RCC_DEV1_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F0__SHIFT                                      0x1d
+#define RCC_DEV1_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV1_F0__SHIFT                                                 0x1e
+#define RCC_DEV1_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV1_F0__SHIFT                                                 0x1f
+#define RCC_DEV1_EPF0_STRAP0__STRAP_DEVICE_ID_DEV1_F0_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F0_MASK                                                 0x000F0000L
+#define RCC_DEV1_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV1_F0_MASK                                                 0x00F00000L
+#define RCC_DEV1_EPF0_STRAP0__STRAP_FUNC_EN_DEV1_F0_MASK                                                      0x10000000L
+#define RCC_DEV1_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F0_MASK                                        0x20000000L
+#define RCC_DEV1_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV1_F0_MASK                                                   0x40000000L
+#define RCC_DEV1_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV1_F0_MASK                                                   0x80000000L
+//RCC_DEV1_EPF0_STRAP2
+#define RCC_DEV1_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F0__SHIFT                               0x4
+#define RCC_DEV1_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F0__SHIFT                                              0x7
+#define RCC_DEV1_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F0__SHIFT                                              0x8
+#define RCC_DEV1_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F0__SHIFT                                            0x9
+#define RCC_DEV1_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F0__SHIFT                                     0xe
+#define RCC_DEV1_EPF0_STRAP2__STRAP_ARI_EN_DEV1_F0__SHIFT                                                     0xf
+#define RCC_DEV1_EPF0_STRAP2__STRAP_AER_EN_DEV1_F0__SHIFT                                                     0x10
+#define RCC_DEV1_EPF0_STRAP2__STRAP_ACS_EN_DEV1_F0__SHIFT                                                     0x11
+#define RCC_DEV1_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F0__SHIFT                                           0x14
+#define RCC_DEV1_EPF0_STRAP2__STRAP_DPA_EN_DEV1_F0__SHIFT                                                     0x15
+#define RCC_DEV1_EPF0_STRAP2__STRAP_VC_EN_DEV1_F0__SHIFT                                                      0x17
+#define RCC_DEV1_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F0__SHIFT                                              0x18
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_EN_DEV1_F0__SHIFT                                                   0x1c
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F0__SHIFT                             0x1d
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F0__SHIFT                          0x1e
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F0__SHIFT                                  0x1f
+#define RCC_DEV1_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F0_MASK                                 0x00000010L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F0_MASK                                                0x00000080L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F0_MASK                                                0x00000100L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F0_MASK                                              0x00003E00L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F0_MASK                                       0x00004000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_ARI_EN_DEV1_F0_MASK                                                       0x00008000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_AER_EN_DEV1_F0_MASK                                                       0x00010000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_ACS_EN_DEV1_F0_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F0_MASK                                             0x00100000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_DPA_EN_DEV1_F0_MASK                                                       0x00200000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_VC_EN_DEV1_F0_MASK                                                        0x00800000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F0_MASK                                                0x07000000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_EN_DEV1_F0_MASK                                                     0x10000000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F0_MASK                               0x20000000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F0_MASK                            0x40000000L
+#define RCC_DEV1_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F0_MASK                                    0x80000000L
+//RCC_DEV1_EPF0_STRAP3
+#define RCC_DEV1_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV1_F0__SHIFT                                                  0x0
+#define RCC_DEV1_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F0__SHIFT                                 0x10
+#define RCC_DEV1_EPF0_STRAP3__STRAP_PWR_EN_DEV1_F0__SHIFT                                                     0x11
+#define RCC_DEV1_EPF0_STRAP3__STRAP_MSI_EN_DEV1_F0__SHIFT                                                     0x12
+#define RCC_DEV1_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F0__SHIFT                                         0x13
+#define RCC_DEV1_EPF0_STRAP3__STRAP_MSIX_EN_DEV1_F0__SHIFT                                                    0x14
+#define RCC_DEV1_EPF0_STRAP3__STRAP_PMC_DSI_DEV1_F0__SHIFT                                                    0x18
+#define RCC_DEV1_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F0__SHIFT                        0x19
+#define RCC_DEV1_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F0__SHIFT                                   0x1a
+#define RCC_DEV1_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F0__SHIFT                                  0x1b
+#define RCC_DEV1_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV1_F0__SHIFT                                                  0x1d
+#define RCC_DEV1_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F0__SHIFT                                          0x1e
+#define RCC_DEV1_EPF0_STRAP3__STRAP_RTR_EN_DEV1_F0__SHIFT                                                     0x1f
+#define RCC_DEV1_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV1_F0_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F0_MASK                                   0x00010000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_PWR_EN_DEV1_F0_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_MSI_EN_DEV1_F0_MASK                                                       0x00040000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F0_MASK                                           0x00080000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_MSIX_EN_DEV1_F0_MASK                                                      0x00100000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_PMC_DSI_DEV1_F0_MASK                                                      0x01000000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F0_MASK                          0x02000000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F0_MASK                                     0x04000000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F0_MASK                                    0x08000000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV1_F0_MASK                                                    0x20000000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F0_MASK                                            0x40000000L
+#define RCC_DEV1_EPF0_STRAP3__STRAP_RTR_EN_DEV1_F0_MASK                                                       0x80000000L
+//RCC_DEV1_EPF0_STRAP4
+#define RCC_DEV1_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F0__SHIFT                                            0x14
+#define RCC_DEV1_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV1_F0__SHIFT                                                  0x15
+#define RCC_DEV1_EPF0_STRAP4__STRAP_FLR_EN_DEV1_F0__SHIFT                                                     0x16
+#define RCC_DEV1_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV1_F0__SHIFT                                                0x17
+#define RCC_DEV1_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F0__SHIFT                                              0x1c
+#define RCC_DEV1_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F0__SHIFT                                             0x1f
+#define RCC_DEV1_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F0_MASK                                              0x00100000L
+#define RCC_DEV1_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV1_F0_MASK                                                    0x00200000L
+#define RCC_DEV1_EPF0_STRAP4__STRAP_FLR_EN_DEV1_F0_MASK                                                       0x00400000L
+#define RCC_DEV1_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV1_F0_MASK                                                  0x0F800000L
+#define RCC_DEV1_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F0_MASK                                                0x70000000L
+#define RCC_DEV1_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F0_MASK                                               0x80000000L
+//RCC_DEV1_EPF0_STRAP5
+#define RCC_DEV1_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F0__SHIFT                                              0x0
+#define RCC_DEV1_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV1_F0__SHIFT                                                0x1b
+#define RCC_DEV1_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F0__SHIFT                                       0x1e
+#define RCC_DEV1_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F0_MASK                                                0x0000FFFFL
+#define RCC_DEV1_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV1_F0_MASK                                                  0x38000000L
+#define RCC_DEV1_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F0_MASK                                         0x40000000L
+//RCC_DEV1_EPF0_STRAP6
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER0_EN_DEV1_F0__SHIFT                                                   0x0
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F0__SHIFT                                             0x2
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER1_EN_DEV1_F0__SHIFT                                                   0x8
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F0__SHIFT                                             0xa
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER2_EN_DEV1_F0__SHIFT                                                   0x10
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F0__SHIFT                                             0x12
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER3_EN_DEV1_F0__SHIFT                                                   0x18
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER4_EN_DEV1_F0__SHIFT                                                   0x1a
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER0_EN_DEV1_F0_MASK                                                     0x00000001L
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F0_MASK                                               0x00000004L
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER1_EN_DEV1_F0_MASK                                                     0x00000100L
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F0_MASK                                               0x00000400L
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER2_EN_DEV1_F0_MASK                                                     0x00010000L
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F0_MASK                                               0x00040000L
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER3_EN_DEV1_F0_MASK                                                     0x01000000L
+#define RCC_DEV1_EPF0_STRAP6__STRAP_APER4_EN_DEV1_F0_MASK                                                     0x04000000L
+//RCC_DEV1_EPF0_STRAP13
+#define RCC_DEV1_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV1_F0__SHIFT                                            0x0
+#define RCC_DEV1_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV1_F0__SHIFT                                            0x8
+#define RCC_DEV1_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV1_F0__SHIFT                                           0x10
+#define RCC_DEV1_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV1_F0_MASK                                              0x000000FFL
+#define RCC_DEV1_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV1_F0_MASK                                              0x0000FF00L
+#define RCC_DEV1_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV1_F0_MASK                                             0x00FF0000L
+//RCC_DEV1_EPF0_STRAP14
+#define RCC_DEV1_EPF0_STRAP14__STRAP_VENDOR_ID_DEV1_F0__SHIFT                                                 0x0
+#define RCC_DEV1_EPF0_STRAP14__STRAP_VENDOR_ID_DEV1_F0_MASK                                                   0x0000FFFFL
+//RCC_DEV1_EPF1_STRAP0
+#define RCC_DEV1_EPF1_STRAP0__STRAP_DEVICE_ID_DEV1_F1__SHIFT                                                  0x0
+#define RCC_DEV1_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F1__SHIFT                                               0x10
+#define RCC_DEV1_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV1_F1__SHIFT                                               0x14
+#define RCC_DEV1_EPF1_STRAP0__STRAP_FUNC_EN_DEV1_F1__SHIFT                                                    0x1c
+#define RCC_DEV1_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F1__SHIFT                                      0x1d
+#define RCC_DEV1_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV1_F1__SHIFT                                                 0x1e
+#define RCC_DEV1_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV1_F1__SHIFT                                                 0x1f
+#define RCC_DEV1_EPF1_STRAP0__STRAP_DEVICE_ID_DEV1_F1_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F1_MASK                                                 0x000F0000L
+#define RCC_DEV1_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV1_F1_MASK                                                 0x00F00000L
+#define RCC_DEV1_EPF1_STRAP0__STRAP_FUNC_EN_DEV1_F1_MASK                                                      0x10000000L
+#define RCC_DEV1_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F1_MASK                                        0x20000000L
+#define RCC_DEV1_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV1_F1_MASK                                                   0x40000000L
+#define RCC_DEV1_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV1_F1_MASK                                                   0x80000000L
+//RCC_DEV1_EPF1_STRAP2
+#define RCC_DEV1_EPF1_STRAP2__STRAP_SRIOV_EN_DEV1_F1__SHIFT                                                   0x0
+#define RCC_DEV1_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV1_F1__SHIFT                                0x2
+#define RCC_DEV1_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F1__SHIFT                               0x4
+#define RCC_DEV1_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F1__SHIFT                                              0x7
+#define RCC_DEV1_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F1__SHIFT                                              0x8
+#define RCC_DEV1_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F1__SHIFT                                            0x9
+#define RCC_DEV1_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F1__SHIFT                                     0xe
+#define RCC_DEV1_EPF1_STRAP2__STRAP_AER_EN_DEV1_F1__SHIFT                                                     0x10
+#define RCC_DEV1_EPF1_STRAP2__STRAP_ACS_EN_DEV1_F1__SHIFT                                                     0x11
+#define RCC_DEV1_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F1__SHIFT                                           0x14
+#define RCC_DEV1_EPF1_STRAP2__STRAP_DPA_EN_DEV1_F1__SHIFT                                                     0x15
+#define RCC_DEV1_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F1__SHIFT                                              0x18
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_EN_DEV1_F1__SHIFT                                                   0x1c
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F1__SHIFT                             0x1d
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F1__SHIFT                          0x1e
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F1__SHIFT                                  0x1f
+#define RCC_DEV1_EPF1_STRAP2__STRAP_SRIOV_EN_DEV1_F1_MASK                                                     0x00000001L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV1_F1_MASK                                  0x00000004L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F1_MASK                                 0x00000010L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F1_MASK                                                0x00000080L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F1_MASK                                                0x00000100L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F1_MASK                                              0x00003E00L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F1_MASK                                       0x00004000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_AER_EN_DEV1_F1_MASK                                                       0x00010000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_ACS_EN_DEV1_F1_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F1_MASK                                             0x00100000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_DPA_EN_DEV1_F1_MASK                                                       0x00200000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F1_MASK                                                0x07000000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_EN_DEV1_F1_MASK                                                     0x10000000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F1_MASK                               0x20000000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F1_MASK                            0x40000000L
+#define RCC_DEV1_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F1_MASK                                    0x80000000L
+//RCC_DEV1_EPF1_STRAP3
+#define RCC_DEV1_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV1_F1__SHIFT                                                  0x0
+#define RCC_DEV1_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F1__SHIFT                                 0x10
+#define RCC_DEV1_EPF1_STRAP3__STRAP_PWR_EN_DEV1_F1__SHIFT                                                     0x11
+#define RCC_DEV1_EPF1_STRAP3__STRAP_MSI_EN_DEV1_F1__SHIFT                                                     0x12
+#define RCC_DEV1_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F1__SHIFT                                         0x13
+#define RCC_DEV1_EPF1_STRAP3__STRAP_MSIX_EN_DEV1_F1__SHIFT                                                    0x14
+#define RCC_DEV1_EPF1_STRAP3__STRAP_PMC_DSI_DEV1_F1__SHIFT                                                    0x18
+#define RCC_DEV1_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F1__SHIFT                        0x19
+#define RCC_DEV1_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F1__SHIFT                                   0x1a
+#define RCC_DEV1_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F1__SHIFT                                  0x1b
+#define RCC_DEV1_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV1_F1__SHIFT                                           0x1c
+#define RCC_DEV1_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV1_F1__SHIFT                                                  0x1d
+#define RCC_DEV1_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F1__SHIFT                                          0x1e
+#define RCC_DEV1_EPF1_STRAP3__STRAP_RTR_EN_DEV1_F1__SHIFT                                                     0x1f
+#define RCC_DEV1_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV1_F1_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F1_MASK                                   0x00010000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_PWR_EN_DEV1_F1_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_MSI_EN_DEV1_F1_MASK                                                       0x00040000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F1_MASK                                           0x00080000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_MSIX_EN_DEV1_F1_MASK                                                      0x00100000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_PMC_DSI_DEV1_F1_MASK                                                      0x01000000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F1_MASK                          0x02000000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F1_MASK                                     0x04000000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F1_MASK                                    0x08000000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV1_F1_MASK                                             0x10000000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV1_F1_MASK                                                    0x20000000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F1_MASK                                            0x40000000L
+#define RCC_DEV1_EPF1_STRAP3__STRAP_RTR_EN_DEV1_F1_MASK                                                       0x80000000L
+//RCC_DEV1_EPF1_STRAP4
+#define RCC_DEV1_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F1__SHIFT                                            0x14
+#define RCC_DEV1_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV1_F1__SHIFT                                                  0x15
+#define RCC_DEV1_EPF1_STRAP4__STRAP_FLR_EN_DEV1_F1__SHIFT                                                     0x16
+#define RCC_DEV1_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV1_F1__SHIFT                                                0x17
+#define RCC_DEV1_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F1__SHIFT                                              0x1c
+#define RCC_DEV1_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F1__SHIFT                                             0x1f
+#define RCC_DEV1_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F1_MASK                                              0x00100000L
+#define RCC_DEV1_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV1_F1_MASK                                                    0x00200000L
+#define RCC_DEV1_EPF1_STRAP4__STRAP_FLR_EN_DEV1_F1_MASK                                                       0x00400000L
+#define RCC_DEV1_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV1_F1_MASK                                                  0x0F800000L
+#define RCC_DEV1_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F1_MASK                                                0x70000000L
+#define RCC_DEV1_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F1_MASK                                               0x80000000L
+//RCC_DEV1_EPF1_STRAP5
+#define RCC_DEV1_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F1__SHIFT                                              0x0
+#define RCC_DEV1_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV1_F1__SHIFT                                                0x1b
+#define RCC_DEV1_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F1__SHIFT                                       0x1e
+#define RCC_DEV1_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F1_MASK                                                0x0000FFFFL
+#define RCC_DEV1_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV1_F1_MASK                                                  0x38000000L
+#define RCC_DEV1_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F1_MASK                                         0x40000000L
+//RCC_DEV1_EPF1_STRAP6
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER0_EN_DEV1_F1__SHIFT                                                   0x0
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F1__SHIFT                                             0x2
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER0_AP_SIZE_DEV1_F1__SHIFT                                              0x4
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER1_EN_DEV1_F1__SHIFT                                                   0x8
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F1__SHIFT                                             0xa
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER2_EN_DEV1_F1__SHIFT                                                   0x10
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F1__SHIFT                                             0x12
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER3_EN_DEV1_F1__SHIFT                                                   0x18
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER4_EN_DEV1_F1__SHIFT                                                   0x1a
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER0_EN_DEV1_F1_MASK                                                     0x00000001L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F1_MASK                                               0x00000004L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER0_AP_SIZE_DEV1_F1_MASK                                                0x00000070L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER1_EN_DEV1_F1_MASK                                                     0x00000100L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F1_MASK                                               0x00000400L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER2_EN_DEV1_F1_MASK                                                     0x00010000L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F1_MASK                                               0x00040000L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER3_EN_DEV1_F1_MASK                                                     0x01000000L
+#define RCC_DEV1_EPF1_STRAP6__STRAP_APER4_EN_DEV1_F1_MASK                                                     0x04000000L
+//RCC_DEV1_EPF1_STRAP7
+//RCC_DEV1_EPF2_STRAP0
+#define RCC_DEV1_EPF2_STRAP0__STRAP_DEVICE_ID_DEV1_F2__SHIFT                                                  0x0
+#define RCC_DEV1_EPF2_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F2__SHIFT                                               0x10
+#define RCC_DEV1_EPF2_STRAP0__STRAP_MINOR_REV_ID_DEV1_F2__SHIFT                                               0x14
+#define RCC_DEV1_EPF2_STRAP0__STRAP_FUNC_EN_DEV1_F2__SHIFT                                                    0x1c
+#define RCC_DEV1_EPF2_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F2__SHIFT                                      0x1d
+#define RCC_DEV1_EPF2_STRAP0__STRAP_D1_SUPPORT_DEV1_F2__SHIFT                                                 0x1e
+#define RCC_DEV1_EPF2_STRAP0__STRAP_D2_SUPPORT_DEV1_F2__SHIFT                                                 0x1f
+#define RCC_DEV1_EPF2_STRAP0__STRAP_DEVICE_ID_DEV1_F2_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF2_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F2_MASK                                                 0x000F0000L
+#define RCC_DEV1_EPF2_STRAP0__STRAP_MINOR_REV_ID_DEV1_F2_MASK                                                 0x00F00000L
+#define RCC_DEV1_EPF2_STRAP0__STRAP_FUNC_EN_DEV1_F2_MASK                                                      0x10000000L
+#define RCC_DEV1_EPF2_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F2_MASK                                        0x20000000L
+#define RCC_DEV1_EPF2_STRAP0__STRAP_D1_SUPPORT_DEV1_F2_MASK                                                   0x40000000L
+#define RCC_DEV1_EPF2_STRAP0__STRAP_D2_SUPPORT_DEV1_F2_MASK                                                   0x80000000L
+//RCC_DEV1_EPF2_STRAP2
+#define RCC_DEV1_EPF2_STRAP2__STRAP_SRIOV_EN_DEV1_F2__SHIFT                                                   0x0
+#define RCC_DEV1_EPF2_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV1_F2__SHIFT                                 0x1
+#define RCC_DEV1_EPF2_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV1_F2__SHIFT                                0x2
+#define RCC_DEV1_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F2__SHIFT                               0x4
+#define RCC_DEV1_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F2__SHIFT                                              0x7
+#define RCC_DEV1_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F2__SHIFT                                              0x8
+#define RCC_DEV1_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F2__SHIFT                                            0x9
+#define RCC_DEV1_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F2__SHIFT                                     0xe
+#define RCC_DEV1_EPF2_STRAP2__STRAP_AER_EN_DEV1_F2__SHIFT                                                     0x10
+#define RCC_DEV1_EPF2_STRAP2__STRAP_ACS_EN_DEV1_F2__SHIFT                                                     0x11
+#define RCC_DEV1_EPF2_STRAP2__STRAP_ATS_EN_DEV1_F2__SHIFT                                                     0x12
+#define RCC_DEV1_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F2__SHIFT                                           0x14
+#define RCC_DEV1_EPF2_STRAP2__STRAP_DPA_EN_DEV1_F2__SHIFT                                                     0x15
+#define RCC_DEV1_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F2__SHIFT                                              0x18
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PAGE_REQ_EN_DEV1_F2__SHIFT                                                0x1b
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_EN_DEV1_F2__SHIFT                                                   0x1c
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F2__SHIFT                             0x1d
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F2__SHIFT                          0x1e
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F2__SHIFT                                  0x1f
+#define RCC_DEV1_EPF2_STRAP2__STRAP_SRIOV_EN_DEV1_F2_MASK                                                     0x00000001L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV1_F2_MASK                                   0x00000002L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV1_F2_MASK                                  0x00000004L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F2_MASK                                 0x00000010L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F2_MASK                                                0x00000080L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F2_MASK                                                0x00000100L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F2_MASK                                              0x00003E00L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F2_MASK                                       0x00004000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_AER_EN_DEV1_F2_MASK                                                       0x00010000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_ACS_EN_DEV1_F2_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_ATS_EN_DEV1_F2_MASK                                                       0x00040000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F2_MASK                                             0x00100000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_DPA_EN_DEV1_F2_MASK                                                       0x00200000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F2_MASK                                                0x07000000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PAGE_REQ_EN_DEV1_F2_MASK                                                  0x08000000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_EN_DEV1_F2_MASK                                                     0x10000000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F2_MASK                               0x20000000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F2_MASK                            0x40000000L
+#define RCC_DEV1_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F2_MASK                                    0x80000000L
+//RCC_DEV1_EPF2_STRAP3
+#define RCC_DEV1_EPF2_STRAP3__STRAP_SUBSYS_ID_DEV1_F2__SHIFT                                                  0x0
+#define RCC_DEV1_EPF2_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F2__SHIFT                                 0x10
+#define RCC_DEV1_EPF2_STRAP3__STRAP_PWR_EN_DEV1_F2__SHIFT                                                     0x11
+#define RCC_DEV1_EPF2_STRAP3__STRAP_MSI_EN_DEV1_F2__SHIFT                                                     0x12
+#define RCC_DEV1_EPF2_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F2__SHIFT                                         0x13
+#define RCC_DEV1_EPF2_STRAP3__STRAP_MSIX_EN_DEV1_F2__SHIFT                                                    0x14
+#define RCC_DEV1_EPF2_STRAP3__STRAP_PMC_DSI_DEV1_F2__SHIFT                                                    0x18
+#define RCC_DEV1_EPF2_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F2__SHIFT                        0x19
+#define RCC_DEV1_EPF2_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F2__SHIFT                                   0x1a
+#define RCC_DEV1_EPF2_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F2__SHIFT                                  0x1b
+#define RCC_DEV1_EPF2_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV1_F2__SHIFT                                           0x1c
+#define RCC_DEV1_EPF2_STRAP3__STRAP_CLK_PM_EN_DEV1_F2__SHIFT                                                  0x1d
+#define RCC_DEV1_EPF2_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F2__SHIFT                                          0x1e
+#define RCC_DEV1_EPF2_STRAP3__STRAP_RTR_EN_DEV1_F2__SHIFT                                                     0x1f
+#define RCC_DEV1_EPF2_STRAP3__STRAP_SUBSYS_ID_DEV1_F2_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF2_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F2_MASK                                   0x00010000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_PWR_EN_DEV1_F2_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_MSI_EN_DEV1_F2_MASK                                                       0x00040000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F2_MASK                                           0x00080000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_MSIX_EN_DEV1_F2_MASK                                                      0x00100000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_PMC_DSI_DEV1_F2_MASK                                                      0x01000000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F2_MASK                          0x02000000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F2_MASK                                     0x04000000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F2_MASK                                    0x08000000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV1_F2_MASK                                             0x10000000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_CLK_PM_EN_DEV1_F2_MASK                                                    0x20000000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F2_MASK                                            0x40000000L
+#define RCC_DEV1_EPF2_STRAP3__STRAP_RTR_EN_DEV1_F2_MASK                                                       0x80000000L
+//RCC_DEV1_EPF2_STRAP4
+#define RCC_DEV1_EPF2_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F2__SHIFT                                            0x14
+#define RCC_DEV1_EPF2_STRAP4__STRAP_ATOMIC_EN_DEV1_F2__SHIFT                                                  0x15
+#define RCC_DEV1_EPF2_STRAP4__STRAP_FLR_EN_DEV1_F2__SHIFT                                                     0x16
+#define RCC_DEV1_EPF2_STRAP4__STRAP_PME_SUPPORT_DEV1_F2__SHIFT                                                0x17
+#define RCC_DEV1_EPF2_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F2__SHIFT                                              0x1c
+#define RCC_DEV1_EPF2_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F2__SHIFT                                             0x1f
+#define RCC_DEV1_EPF2_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F2_MASK                                              0x00100000L
+#define RCC_DEV1_EPF2_STRAP4__STRAP_ATOMIC_EN_DEV1_F2_MASK                                                    0x00200000L
+#define RCC_DEV1_EPF2_STRAP4__STRAP_FLR_EN_DEV1_F2_MASK                                                       0x00400000L
+#define RCC_DEV1_EPF2_STRAP4__STRAP_PME_SUPPORT_DEV1_F2_MASK                                                  0x0F800000L
+#define RCC_DEV1_EPF2_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F2_MASK                                                0x70000000L
+#define RCC_DEV1_EPF2_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F2_MASK                                               0x80000000L
+//RCC_DEV1_EPF2_STRAP5
+#define RCC_DEV1_EPF2_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F2__SHIFT                                              0x0
+#define RCC_DEV1_EPF2_STRAP5__STRAP_AUX_CURRENT_DEV1_F2__SHIFT                                                0x1b
+#define RCC_DEV1_EPF2_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F2__SHIFT                                       0x1e
+#define RCC_DEV1_EPF2_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F2_MASK                                                0x0000FFFFL
+#define RCC_DEV1_EPF2_STRAP5__STRAP_AUX_CURRENT_DEV1_F2_MASK                                                  0x38000000L
+#define RCC_DEV1_EPF2_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F2_MASK                                         0x40000000L
+//RCC_DEV1_EPF2_STRAP6
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER0_EN_DEV1_F2__SHIFT                                                   0x0
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F2__SHIFT                                             0x2
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER1_EN_DEV1_F2__SHIFT                                                   0x8
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F2__SHIFT                                             0xa
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER2_EN_DEV1_F2__SHIFT                                                   0x10
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F2__SHIFT                                             0x12
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER3_EN_DEV1_F2__SHIFT                                                   0x18
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER4_EN_DEV1_F2__SHIFT                                                   0x1a
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER0_EN_DEV1_F2_MASK                                                     0x00000001L
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F2_MASK                                               0x00000004L
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER1_EN_DEV1_F2_MASK                                                     0x00000100L
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F2_MASK                                               0x00000400L
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER2_EN_DEV1_F2_MASK                                                     0x00010000L
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F2_MASK                                               0x00040000L
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER3_EN_DEV1_F2_MASK                                                     0x01000000L
+#define RCC_DEV1_EPF2_STRAP6__STRAP_APER4_EN_DEV1_F2_MASK                                                     0x04000000L
+//RCC_DEV1_EPF3_STRAP0
+#define RCC_DEV1_EPF3_STRAP0__STRAP_DEVICE_ID_DEV1_F3__SHIFT                                                  0x0
+#define RCC_DEV1_EPF3_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F3__SHIFT                                               0x10
+#define RCC_DEV1_EPF3_STRAP0__STRAP_MINOR_REV_ID_DEV1_F3__SHIFT                                               0x14
+#define RCC_DEV1_EPF3_STRAP0__STRAP_FUNC_EN_DEV1_F3__SHIFT                                                    0x1c
+#define RCC_DEV1_EPF3_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F3__SHIFT                                      0x1d
+#define RCC_DEV1_EPF3_STRAP0__STRAP_D1_SUPPORT_DEV1_F3__SHIFT                                                 0x1e
+#define RCC_DEV1_EPF3_STRAP0__STRAP_D2_SUPPORT_DEV1_F3__SHIFT                                                 0x1f
+#define RCC_DEV1_EPF3_STRAP0__STRAP_DEVICE_ID_DEV1_F3_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF3_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F3_MASK                                                 0x000F0000L
+#define RCC_DEV1_EPF3_STRAP0__STRAP_MINOR_REV_ID_DEV1_F3_MASK                                                 0x00F00000L
+#define RCC_DEV1_EPF3_STRAP0__STRAP_FUNC_EN_DEV1_F3_MASK                                                      0x10000000L
+#define RCC_DEV1_EPF3_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F3_MASK                                        0x20000000L
+#define RCC_DEV1_EPF3_STRAP0__STRAP_D1_SUPPORT_DEV1_F3_MASK                                                   0x40000000L
+#define RCC_DEV1_EPF3_STRAP0__STRAP_D2_SUPPORT_DEV1_F3_MASK                                                   0x80000000L
+//RCC_DEV1_EPF3_STRAP2
+#define RCC_DEV1_EPF3_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F3__SHIFT                               0x4
+#define RCC_DEV1_EPF3_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F3__SHIFT                                              0x7
+#define RCC_DEV1_EPF3_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F3__SHIFT                                              0x8
+#define RCC_DEV1_EPF3_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F3__SHIFT                                            0x9
+#define RCC_DEV1_EPF3_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F3__SHIFT                                     0xe
+#define RCC_DEV1_EPF3_STRAP2__STRAP_AER_EN_DEV1_F3__SHIFT                                                     0x10
+#define RCC_DEV1_EPF3_STRAP2__STRAP_ACS_EN_DEV1_F3__SHIFT                                                     0x11
+#define RCC_DEV1_EPF3_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F3__SHIFT                                           0x14
+#define RCC_DEV1_EPF3_STRAP2__STRAP_DPA_EN_DEV1_F3__SHIFT                                                     0x15
+#define RCC_DEV1_EPF3_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F3__SHIFT                                              0x18
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_EN_DEV1_F3__SHIFT                                                   0x1c
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F3__SHIFT                             0x1d
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F3__SHIFT                          0x1e
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F3__SHIFT                                  0x1f
+#define RCC_DEV1_EPF3_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F3_MASK                                 0x00000010L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F3_MASK                                                0x00000080L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F3_MASK                                                0x00000100L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F3_MASK                                              0x00003E00L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F3_MASK                                       0x00004000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_AER_EN_DEV1_F3_MASK                                                       0x00010000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_ACS_EN_DEV1_F3_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F3_MASK                                             0x00100000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_DPA_EN_DEV1_F3_MASK                                                       0x00200000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F3_MASK                                                0x07000000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_EN_DEV1_F3_MASK                                                     0x10000000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F3_MASK                               0x20000000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F3_MASK                            0x40000000L
+#define RCC_DEV1_EPF3_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F3_MASK                                    0x80000000L
+//RCC_DEV1_EPF3_STRAP3
+#define RCC_DEV1_EPF3_STRAP3__STRAP_SUBSYS_ID_DEV1_F3__SHIFT                                                  0x0
+#define RCC_DEV1_EPF3_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F3__SHIFT                                 0x10
+#define RCC_DEV1_EPF3_STRAP3__STRAP_PWR_EN_DEV1_F3__SHIFT                                                     0x11
+#define RCC_DEV1_EPF3_STRAP3__STRAP_MSI_EN_DEV1_F3__SHIFT                                                     0x12
+#define RCC_DEV1_EPF3_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F3__SHIFT                                         0x13
+#define RCC_DEV1_EPF3_STRAP3__STRAP_MSIX_EN_DEV1_F3__SHIFT                                                    0x14
+#define RCC_DEV1_EPF3_STRAP3__STRAP_PMC_DSI_DEV1_F3__SHIFT                                                    0x18
+#define RCC_DEV1_EPF3_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F3__SHIFT                        0x19
+#define RCC_DEV1_EPF3_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F3__SHIFT                                   0x1a
+#define RCC_DEV1_EPF3_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F3__SHIFT                                  0x1b
+#define RCC_DEV1_EPF3_STRAP3__STRAP_CLK_PM_EN_DEV1_F3__SHIFT                                                  0x1d
+#define RCC_DEV1_EPF3_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F3__SHIFT                                          0x1e
+#define RCC_DEV1_EPF3_STRAP3__STRAP_RTR_EN_DEV1_F3__SHIFT                                                     0x1f
+#define RCC_DEV1_EPF3_STRAP3__STRAP_SUBSYS_ID_DEV1_F3_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF3_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F3_MASK                                   0x00010000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_PWR_EN_DEV1_F3_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_MSI_EN_DEV1_F3_MASK                                                       0x00040000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F3_MASK                                           0x00080000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_MSIX_EN_DEV1_F3_MASK                                                      0x00100000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_PMC_DSI_DEV1_F3_MASK                                                      0x01000000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F3_MASK                          0x02000000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F3_MASK                                     0x04000000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F3_MASK                                    0x08000000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_CLK_PM_EN_DEV1_F3_MASK                                                    0x20000000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F3_MASK                                            0x40000000L
+#define RCC_DEV1_EPF3_STRAP3__STRAP_RTR_EN_DEV1_F3_MASK                                                       0x80000000L
+//RCC_DEV1_EPF3_STRAP4
+#define RCC_DEV1_EPF3_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F3__SHIFT                                            0x14
+#define RCC_DEV1_EPF3_STRAP4__STRAP_ATOMIC_EN_DEV1_F3__SHIFT                                                  0x15
+#define RCC_DEV1_EPF3_STRAP4__STRAP_FLR_EN_DEV1_F3__SHIFT                                                     0x16
+#define RCC_DEV1_EPF3_STRAP4__STRAP_PME_SUPPORT_DEV1_F3__SHIFT                                                0x17
+#define RCC_DEV1_EPF3_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F3__SHIFT                                              0x1c
+#define RCC_DEV1_EPF3_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F3__SHIFT                                             0x1f
+#define RCC_DEV1_EPF3_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F3_MASK                                              0x00100000L
+#define RCC_DEV1_EPF3_STRAP4__STRAP_ATOMIC_EN_DEV1_F3_MASK                                                    0x00200000L
+#define RCC_DEV1_EPF3_STRAP4__STRAP_FLR_EN_DEV1_F3_MASK                                                       0x00400000L
+#define RCC_DEV1_EPF3_STRAP4__STRAP_PME_SUPPORT_DEV1_F3_MASK                                                  0x0F800000L
+#define RCC_DEV1_EPF3_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F3_MASK                                                0x70000000L
+#define RCC_DEV1_EPF3_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F3_MASK                                               0x80000000L
+//RCC_DEV1_EPF3_STRAP5
+#define RCC_DEV1_EPF3_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F3__SHIFT                                              0x0
+#define RCC_DEV1_EPF3_STRAP5__STRAP_USB_DBESEL_DEV1_F3__SHIFT                                                 0x10
+#define RCC_DEV1_EPF3_STRAP5__STRAP_USB_DBESELD_DEV1_F3__SHIFT                                                0x14
+#define RCC_DEV1_EPF3_STRAP5__STRAP_AUX_CURRENT_DEV1_F3__SHIFT                                                0x1b
+#define RCC_DEV1_EPF3_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F3__SHIFT                                       0x1e
+#define RCC_DEV1_EPF3_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F3_MASK                                                0x0000FFFFL
+#define RCC_DEV1_EPF3_STRAP5__STRAP_USB_DBESEL_DEV1_F3_MASK                                                   0x000F0000L
+#define RCC_DEV1_EPF3_STRAP5__STRAP_USB_DBESELD_DEV1_F3_MASK                                                  0x00F00000L
+#define RCC_DEV1_EPF3_STRAP5__STRAP_AUX_CURRENT_DEV1_F3_MASK                                                  0x38000000L
+#define RCC_DEV1_EPF3_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F3_MASK                                         0x40000000L
+//RCC_DEV1_EPF3_STRAP6
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER0_EN_DEV1_F3__SHIFT                                                   0x0
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F3__SHIFT                                             0x2
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER1_EN_DEV1_F3__SHIFT                                                   0x8
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F3__SHIFT                                             0xa
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER2_EN_DEV1_F3__SHIFT                                                   0x10
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F3__SHIFT                                             0x12
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER3_EN_DEV1_F3__SHIFT                                                   0x18
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER4_EN_DEV1_F3__SHIFT                                                   0x1a
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER0_EN_DEV1_F3_MASK                                                     0x00000001L
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F3_MASK                                               0x00000004L
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER1_EN_DEV1_F3_MASK                                                     0x00000100L
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F3_MASK                                               0x00000400L
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER2_EN_DEV1_F3_MASK                                                     0x00010000L
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F3_MASK                                               0x00040000L
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER3_EN_DEV1_F3_MASK                                                     0x01000000L
+#define RCC_DEV1_EPF3_STRAP6__STRAP_APER4_EN_DEV1_F3_MASK                                                     0x04000000L
+//RCC_DEV1_EPF4_STRAP0
+#define RCC_DEV1_EPF4_STRAP0__STRAP_DEVICE_ID_DEV1_F4__SHIFT                                                  0x0
+#define RCC_DEV1_EPF4_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F4__SHIFT                                               0x10
+#define RCC_DEV1_EPF4_STRAP0__STRAP_MINOR_REV_ID_DEV1_F4__SHIFT                                               0x14
+#define RCC_DEV1_EPF4_STRAP0__STRAP_FUNC_EN_DEV1_F4__SHIFT                                                    0x1c
+#define RCC_DEV1_EPF4_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F4__SHIFT                                      0x1d
+#define RCC_DEV1_EPF4_STRAP0__STRAP_D1_SUPPORT_DEV1_F4__SHIFT                                                 0x1e
+#define RCC_DEV1_EPF4_STRAP0__STRAP_D2_SUPPORT_DEV1_F4__SHIFT                                                 0x1f
+#define RCC_DEV1_EPF4_STRAP0__STRAP_DEVICE_ID_DEV1_F4_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF4_STRAP0__STRAP_MAJOR_REV_ID_DEV1_F4_MASK                                                 0x000F0000L
+#define RCC_DEV1_EPF4_STRAP0__STRAP_MINOR_REV_ID_DEV1_F4_MASK                                                 0x00F00000L
+#define RCC_DEV1_EPF4_STRAP0__STRAP_FUNC_EN_DEV1_F4_MASK                                                      0x10000000L
+#define RCC_DEV1_EPF4_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV1_F4_MASK                                        0x20000000L
+#define RCC_DEV1_EPF4_STRAP0__STRAP_D1_SUPPORT_DEV1_F4_MASK                                                   0x40000000L
+#define RCC_DEV1_EPF4_STRAP0__STRAP_D2_SUPPORT_DEV1_F4_MASK                                                   0x80000000L
+//RCC_DEV1_EPF4_STRAP2
+#define RCC_DEV1_EPF4_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F4__SHIFT                               0x4
+#define RCC_DEV1_EPF4_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F4__SHIFT                                              0x7
+#define RCC_DEV1_EPF4_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F4__SHIFT                                              0x8
+#define RCC_DEV1_EPF4_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F4__SHIFT                                            0x9
+#define RCC_DEV1_EPF4_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F4__SHIFT                                     0xe
+#define RCC_DEV1_EPF4_STRAP2__STRAP_AER_EN_DEV1_F4__SHIFT                                                     0x10
+#define RCC_DEV1_EPF4_STRAP2__STRAP_ACS_EN_DEV1_F4__SHIFT                                                     0x11
+#define RCC_DEV1_EPF4_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F4__SHIFT                                           0x14
+#define RCC_DEV1_EPF4_STRAP2__STRAP_DPA_EN_DEV1_F4__SHIFT                                                     0x15
+#define RCC_DEV1_EPF4_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F4__SHIFT                                              0x18
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_EN_DEV1_F4__SHIFT                                                   0x1c
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F4__SHIFT                             0x1d
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F4__SHIFT                          0x1e
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F4__SHIFT                                  0x1f
+#define RCC_DEV1_EPF4_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV1_F4_MASK                                 0x00000010L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_NO_SOFT_RESET_DEV1_F4_MASK                                                0x00000080L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_RESIZE_BAR_EN_DEV1_F4_MASK                                                0x00000100L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_MAX_PASID_WIDTH_DEV1_F4_MASK                                              0x00003E00L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV1_F4_MASK                                       0x00004000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_AER_EN_DEV1_F4_MASK                                                       0x00010000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_ACS_EN_DEV1_F4_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV1_F4_MASK                                             0x00100000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_DPA_EN_DEV1_F4_MASK                                                       0x00200000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_MSI_MULTI_CAP_DEV1_F4_MASK                                                0x07000000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_EN_DEV1_F4_MASK                                                     0x10000000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV1_F4_MASK                               0x20000000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV1_F4_MASK                            0x40000000L
+#define RCC_DEV1_EPF4_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV1_F4_MASK                                    0x80000000L
+//RCC_DEV1_EPF4_STRAP3
+#define RCC_DEV1_EPF4_STRAP3__STRAP_SUBSYS_ID_DEV1_F4__SHIFT                                                  0x0
+#define RCC_DEV1_EPF4_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F4__SHIFT                                 0x10
+#define RCC_DEV1_EPF4_STRAP3__STRAP_PWR_EN_DEV1_F4__SHIFT                                                     0x11
+#define RCC_DEV1_EPF4_STRAP3__STRAP_MSI_EN_DEV1_F4__SHIFT                                                     0x12
+#define RCC_DEV1_EPF4_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F4__SHIFT                                         0x13
+#define RCC_DEV1_EPF4_STRAP3__STRAP_MSIX_EN_DEV1_F4__SHIFT                                                    0x14
+#define RCC_DEV1_EPF4_STRAP3__STRAP_PMC_DSI_DEV1_F4__SHIFT                                                    0x18
+#define RCC_DEV1_EPF4_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F4__SHIFT                        0x19
+#define RCC_DEV1_EPF4_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F4__SHIFT                                   0x1a
+#define RCC_DEV1_EPF4_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F4__SHIFT                                  0x1b
+#define RCC_DEV1_EPF4_STRAP3__STRAP_CLK_PM_EN_DEV1_F4__SHIFT                                                  0x1d
+#define RCC_DEV1_EPF4_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F4__SHIFT                                          0x1e
+#define RCC_DEV1_EPF4_STRAP3__STRAP_RTR_EN_DEV1_F4__SHIFT                                                     0x1f
+#define RCC_DEV1_EPF4_STRAP3__STRAP_SUBSYS_ID_DEV1_F4_MASK                                                    0x0000FFFFL
+#define RCC_DEV1_EPF4_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV1_F4_MASK                                   0x00010000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_PWR_EN_DEV1_F4_MASK                                                       0x00020000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_MSI_EN_DEV1_F4_MASK                                                       0x00040000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV1_F4_MASK                                           0x00080000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_MSIX_EN_DEV1_F4_MASK                                                      0x00100000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_PMC_DSI_DEV1_F4_MASK                                                      0x01000000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV1_F4_MASK                          0x02000000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV1_F4_MASK                                     0x04000000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV1_F4_MASK                                    0x08000000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_CLK_PM_EN_DEV1_F4_MASK                                                    0x20000000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV1_F4_MASK                                            0x40000000L
+#define RCC_DEV1_EPF4_STRAP3__STRAP_RTR_EN_DEV1_F4_MASK                                                       0x80000000L
+//RCC_DEV1_EPF4_STRAP4
+#define RCC_DEV1_EPF4_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F4__SHIFT                                            0x14
+#define RCC_DEV1_EPF4_STRAP4__STRAP_ATOMIC_EN_DEV1_F4__SHIFT                                                  0x15
+#define RCC_DEV1_EPF4_STRAP4__STRAP_FLR_EN_DEV1_F4__SHIFT                                                     0x16
+#define RCC_DEV1_EPF4_STRAP4__STRAP_PME_SUPPORT_DEV1_F4__SHIFT                                                0x17
+#define RCC_DEV1_EPF4_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F4__SHIFT                                              0x1c
+#define RCC_DEV1_EPF4_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F4__SHIFT                                             0x1f
+#define RCC_DEV1_EPF4_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV1_F4_MASK                                              0x00100000L
+#define RCC_DEV1_EPF4_STRAP4__STRAP_ATOMIC_EN_DEV1_F4_MASK                                                    0x00200000L
+#define RCC_DEV1_EPF4_STRAP4__STRAP_FLR_EN_DEV1_F4_MASK                                                       0x00400000L
+#define RCC_DEV1_EPF4_STRAP4__STRAP_PME_SUPPORT_DEV1_F4_MASK                                                  0x0F800000L
+#define RCC_DEV1_EPF4_STRAP4__STRAP_INTERRUPT_PIN_DEV1_F4_MASK                                                0x70000000L
+#define RCC_DEV1_EPF4_STRAP4__STRAP_AUXPWR_SUPPORT_DEV1_F4_MASK                                               0x80000000L
+//RCC_DEV1_EPF4_STRAP5
+#define RCC_DEV1_EPF4_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F4__SHIFT                                              0x0
+#define RCC_DEV1_EPF4_STRAP5__STRAP_AUX_CURRENT_DEV1_F4__SHIFT                                                0x1b
+#define RCC_DEV1_EPF4_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F4__SHIFT                                       0x1e
+#define RCC_DEV1_EPF4_STRAP5__STRAP_SUBSYS_VEN_ID_DEV1_F4_MASK                                                0x0000FFFFL
+#define RCC_DEV1_EPF4_STRAP5__STRAP_AUX_CURRENT_DEV1_F4_MASK                                                  0x38000000L
+#define RCC_DEV1_EPF4_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV1_F4_MASK                                         0x40000000L
+//RCC_DEV1_EPF4_STRAP6
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER0_EN_DEV1_F4__SHIFT                                                   0x0
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV1_F4__SHIFT                                      0x1
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F4__SHIFT                                             0x2
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER1_EN_DEV1_F4__SHIFT                                                   0x8
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV1_F4__SHIFT                                      0x9
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F4__SHIFT                                             0xa
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER2_EN_DEV1_F4__SHIFT                                                   0x10
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV1_F4__SHIFT                                      0x11
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F4__SHIFT                                             0x12
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER3_EN_DEV1_F4__SHIFT                                                   0x18
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER0_EN_DEV1_F4_MASK                                                     0x00000001L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV1_F4_MASK                                        0x00000002L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER0_64BAR_EN_DEV1_F4_MASK                                               0x00000004L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER1_EN_DEV1_F4_MASK                                                     0x00000100L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV1_F4_MASK                                        0x00000200L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER1_64BAR_EN_DEV1_F4_MASK                                               0x00000400L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER2_EN_DEV1_F4_MASK                                                     0x00010000L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV1_F4_MASK                                        0x00020000L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER2_64BAR_EN_DEV1_F4_MASK                                               0x00040000L
+#define RCC_DEV1_EPF4_STRAP6__STRAP_APER3_EN_DEV1_F4_MASK                                                     0x01000000L
+//RCC_DEV1_EPF5_STRAP0
+//RCC_DEV1_EPF5_STRAP2
+//RCC_DEV1_EPF5_STRAP3
+//RCC_DEV1_EPF5_STRAP4
+//RCC_DEV1_EPF5_STRAP5
+//RCC_DEV1_EPF5_STRAP6
+//RCC_DEV2_EPF0_STRAP0
+#define RCC_DEV2_EPF0_STRAP0__STRAP_DEVICE_ID_DEV2_F0__SHIFT                                                  0x0
+#define RCC_DEV2_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV2_F0__SHIFT                                               0x10
+#define RCC_DEV2_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV2_F0__SHIFT                                               0x14
+#define RCC_DEV2_EPF0_STRAP0__STRAP_FUNC_EN_DEV2_F0__SHIFT                                                    0x1c
+#define RCC_DEV2_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV2_F0__SHIFT                                      0x1d
+#define RCC_DEV2_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV2_F0__SHIFT                                                 0x1e
+#define RCC_DEV2_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV2_F0__SHIFT                                                 0x1f
+#define RCC_DEV2_EPF0_STRAP0__STRAP_DEVICE_ID_DEV2_F0_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV2_F0_MASK                                                 0x000F0000L
+#define RCC_DEV2_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV2_F0_MASK                                                 0x00F00000L
+#define RCC_DEV2_EPF0_STRAP0__STRAP_FUNC_EN_DEV2_F0_MASK                                                      0x10000000L
+#define RCC_DEV2_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV2_F0_MASK                                        0x20000000L
+#define RCC_DEV2_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV2_F0_MASK                                                   0x40000000L
+#define RCC_DEV2_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV2_F0_MASK                                                   0x80000000L
+//RCC_DEV2_EPF0_STRAP2
+#define RCC_DEV2_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV2_F0__SHIFT                               0x4
+#define RCC_DEV2_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV2_F0__SHIFT                                              0x7
+#define RCC_DEV2_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV2_F0__SHIFT                                              0x8
+#define RCC_DEV2_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV2_F0__SHIFT                                            0x9
+#define RCC_DEV2_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV2_F0__SHIFT                                     0xe
+#define RCC_DEV2_EPF0_STRAP2__STRAP_ARI_EN_DEV2_F0__SHIFT                                                     0xf
+#define RCC_DEV2_EPF0_STRAP2__STRAP_AER_EN_DEV2_F0__SHIFT                                                     0x10
+#define RCC_DEV2_EPF0_STRAP2__STRAP_ACS_EN_DEV2_F0__SHIFT                                                     0x11
+#define RCC_DEV2_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV2_F0__SHIFT                                           0x14
+#define RCC_DEV2_EPF0_STRAP2__STRAP_DPA_EN_DEV2_F0__SHIFT                                                     0x15
+#define RCC_DEV2_EPF0_STRAP2__STRAP_VC_EN_DEV2_F0__SHIFT                                                      0x17
+#define RCC_DEV2_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV2_F0__SHIFT                                              0x18
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_EN_DEV2_F0__SHIFT                                                   0x1c
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV2_F0__SHIFT                             0x1d
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV2_F0__SHIFT                          0x1e
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV2_F0__SHIFT                                  0x1f
+#define RCC_DEV2_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV2_F0_MASK                                 0x00000010L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV2_F0_MASK                                                0x00000080L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV2_F0_MASK                                                0x00000100L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV2_F0_MASK                                              0x00003E00L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV2_F0_MASK                                       0x00004000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_ARI_EN_DEV2_F0_MASK                                                       0x00008000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_AER_EN_DEV2_F0_MASK                                                       0x00010000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_ACS_EN_DEV2_F0_MASK                                                       0x00020000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV2_F0_MASK                                             0x00100000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_DPA_EN_DEV2_F0_MASK                                                       0x00200000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_VC_EN_DEV2_F0_MASK                                                        0x00800000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV2_F0_MASK                                                0x07000000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_EN_DEV2_F0_MASK                                                     0x10000000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV2_F0_MASK                               0x20000000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV2_F0_MASK                            0x40000000L
+#define RCC_DEV2_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV2_F0_MASK                                    0x80000000L
+//RCC_DEV2_EPF0_STRAP3
+#define RCC_DEV2_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV2_F0__SHIFT                                                  0x0
+#define RCC_DEV2_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV2_F0__SHIFT                                 0x10
+#define RCC_DEV2_EPF0_STRAP3__STRAP_PWR_EN_DEV2_F0__SHIFT                                                     0x11
+#define RCC_DEV2_EPF0_STRAP3__STRAP_MSI_EN_DEV2_F0__SHIFT                                                     0x12
+#define RCC_DEV2_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV2_F0__SHIFT                                         0x13
+#define RCC_DEV2_EPF0_STRAP3__STRAP_MSIX_EN_DEV2_F0__SHIFT                                                    0x14
+#define RCC_DEV2_EPF0_STRAP3__STRAP_PMC_DSI_DEV2_F0__SHIFT                                                    0x18
+#define RCC_DEV2_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV2_F0__SHIFT                        0x19
+#define RCC_DEV2_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV2_F0__SHIFT                                   0x1a
+#define RCC_DEV2_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV2_F0__SHIFT                                  0x1b
+#define RCC_DEV2_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV2_F0__SHIFT                                                  0x1d
+#define RCC_DEV2_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV2_F0__SHIFT                                          0x1e
+#define RCC_DEV2_EPF0_STRAP3__STRAP_RTR_EN_DEV2_F0__SHIFT                                                     0x1f
+#define RCC_DEV2_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV2_F0_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV2_F0_MASK                                   0x00010000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_PWR_EN_DEV2_F0_MASK                                                       0x00020000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_MSI_EN_DEV2_F0_MASK                                                       0x00040000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV2_F0_MASK                                           0x00080000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_MSIX_EN_DEV2_F0_MASK                                                      0x00100000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_PMC_DSI_DEV2_F0_MASK                                                      0x01000000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV2_F0_MASK                          0x02000000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV2_F0_MASK                                     0x04000000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV2_F0_MASK                                    0x08000000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV2_F0_MASK                                                    0x20000000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV2_F0_MASK                                            0x40000000L
+#define RCC_DEV2_EPF0_STRAP3__STRAP_RTR_EN_DEV2_F0_MASK                                                       0x80000000L
+//RCC_DEV2_EPF0_STRAP4
+#define RCC_DEV2_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV2_F0__SHIFT                                            0x14
+#define RCC_DEV2_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV2_F0__SHIFT                                                  0x15
+#define RCC_DEV2_EPF0_STRAP4__STRAP_FLR_EN_DEV2_F0__SHIFT                                                     0x16
+#define RCC_DEV2_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV2_F0__SHIFT                                                0x17
+#define RCC_DEV2_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV2_F0__SHIFT                                              0x1c
+#define RCC_DEV2_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV2_F0__SHIFT                                             0x1f
+#define RCC_DEV2_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV2_F0_MASK                                              0x00100000L
+#define RCC_DEV2_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV2_F0_MASK                                                    0x00200000L
+#define RCC_DEV2_EPF0_STRAP4__STRAP_FLR_EN_DEV2_F0_MASK                                                       0x00400000L
+#define RCC_DEV2_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV2_F0_MASK                                                  0x0F800000L
+#define RCC_DEV2_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV2_F0_MASK                                                0x70000000L
+#define RCC_DEV2_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV2_F0_MASK                                               0x80000000L
+//RCC_DEV2_EPF0_STRAP5
+#define RCC_DEV2_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV2_F0__SHIFT                                              0x0
+#define RCC_DEV2_EPF0_STRAP5__STRAP_USB_DBESEL_DEV2_F0__SHIFT                                                 0x10
+#define RCC_DEV2_EPF0_STRAP5__STRAP_USB_DBESELD_DEV2_F0__SHIFT                                                0x14
+#define RCC_DEV2_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV2_F0__SHIFT                                                0x1b
+#define RCC_DEV2_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV2_F0__SHIFT                                       0x1e
+#define RCC_DEV2_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV2_F0_MASK                                                0x0000FFFFL
+#define RCC_DEV2_EPF0_STRAP5__STRAP_USB_DBESEL_DEV2_F0_MASK                                                   0x000F0000L
+#define RCC_DEV2_EPF0_STRAP5__STRAP_USB_DBESELD_DEV2_F0_MASK                                                  0x00F00000L
+#define RCC_DEV2_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV2_F0_MASK                                                  0x38000000L
+#define RCC_DEV2_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV2_F0_MASK                                         0x40000000L
+//RCC_DEV2_EPF0_STRAP6
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER0_EN_DEV2_F0__SHIFT                                                   0x0
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER0_64BAR_EN_DEV2_F0__SHIFT                                             0x2
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER1_EN_DEV2_F0__SHIFT                                                   0x8
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER1_64BAR_EN_DEV2_F0__SHIFT                                             0xa
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER2_EN_DEV2_F0__SHIFT                                                   0x10
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER2_64BAR_EN_DEV2_F0__SHIFT                                             0x12
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER3_EN_DEV2_F0__SHIFT                                                   0x18
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER4_EN_DEV2_F0__SHIFT                                                   0x1a
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER0_EN_DEV2_F0_MASK                                                     0x00000001L
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER0_64BAR_EN_DEV2_F0_MASK                                               0x00000004L
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER1_EN_DEV2_F0_MASK                                                     0x00000100L
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER1_64BAR_EN_DEV2_F0_MASK                                               0x00000400L
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER2_EN_DEV2_F0_MASK                                                     0x00010000L
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER2_64BAR_EN_DEV2_F0_MASK                                               0x00040000L
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER3_EN_DEV2_F0_MASK                                                     0x01000000L
+#define RCC_DEV2_EPF0_STRAP6__STRAP_APER4_EN_DEV2_F0_MASK                                                     0x04000000L
+//RCC_DEV2_EPF0_STRAP7
+#define RCC_DEV2_EPF0_STRAP7__STRAP_MSIX_TABLE_SIZE_DEV2_F0__SHIFT                                            0x5
+#define RCC_DEV2_EPF0_STRAP7__STRAP_MSIX_TABLE_SIZE_DEV2_F0_MASK                                              0x0000FFE0L
+//RCC_DEV2_EPF0_STRAP13
+#define RCC_DEV2_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV2_F0__SHIFT                                            0x0
+#define RCC_DEV2_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV2_F0__SHIFT                                            0x8
+#define RCC_DEV2_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV2_F0__SHIFT                                           0x10
+#define RCC_DEV2_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV2_F0_MASK                                              0x000000FFL
+#define RCC_DEV2_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV2_F0_MASK                                              0x0000FF00L
+#define RCC_DEV2_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV2_F0_MASK                                             0x00FF0000L
+//RCC_DEV2_EPF0_STRAP14
+#define RCC_DEV2_EPF0_STRAP14__STRAP_VENDOR_ID_DEV2_F0__SHIFT                                                 0x0
+#define RCC_DEV2_EPF0_STRAP14__STRAP_VENDOR_ID_DEV2_F0_MASK                                                   0x0000FFFFL
+//RCC_DEV2_EPF1_STRAP0
+#define RCC_DEV2_EPF1_STRAP0__STRAP_DEVICE_ID_DEV2_F1__SHIFT                                                  0x0
+#define RCC_DEV2_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV2_F1__SHIFT                                               0x10
+#define RCC_DEV2_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV2_F1__SHIFT                                               0x14
+#define RCC_DEV2_EPF1_STRAP0__STRAP_FUNC_EN_DEV2_F1__SHIFT                                                    0x1c
+#define RCC_DEV2_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV2_F1__SHIFT                                      0x1d
+#define RCC_DEV2_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV2_F1__SHIFT                                                 0x1e
+#define RCC_DEV2_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV2_F1__SHIFT                                                 0x1f
+#define RCC_DEV2_EPF1_STRAP0__STRAP_DEVICE_ID_DEV2_F1_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV2_F1_MASK                                                 0x000F0000L
+#define RCC_DEV2_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV2_F1_MASK                                                 0x00F00000L
+#define RCC_DEV2_EPF1_STRAP0__STRAP_FUNC_EN_DEV2_F1_MASK                                                      0x10000000L
+#define RCC_DEV2_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV2_F1_MASK                                        0x20000000L
+#define RCC_DEV2_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV2_F1_MASK                                                   0x40000000L
+#define RCC_DEV2_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV2_F1_MASK                                                   0x80000000L
+//RCC_DEV2_EPF1_STRAP2
+#define RCC_DEV2_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV2_F1__SHIFT                               0x4
+#define RCC_DEV2_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV2_F1__SHIFT                                              0x7
+#define RCC_DEV2_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV2_F1__SHIFT                                              0x8
+#define RCC_DEV2_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV2_F1__SHIFT                                            0x9
+#define RCC_DEV2_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV2_F1__SHIFT                                     0xe
+#define RCC_DEV2_EPF1_STRAP2__STRAP_AER_EN_DEV2_F1__SHIFT                                                     0x10
+#define RCC_DEV2_EPF1_STRAP2__STRAP_ACS_EN_DEV2_F1__SHIFT                                                     0x11
+#define RCC_DEV2_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV2_F1__SHIFT                                           0x14
+#define RCC_DEV2_EPF1_STRAP2__STRAP_DPA_EN_DEV2_F1__SHIFT                                                     0x15
+#define RCC_DEV2_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV2_F1__SHIFT                                              0x18
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_EN_DEV2_F1__SHIFT                                                   0x1c
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV2_F1__SHIFT                             0x1d
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV2_F1__SHIFT                          0x1e
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV2_F1__SHIFT                                  0x1f
+#define RCC_DEV2_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV2_F1_MASK                                 0x00000010L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV2_F1_MASK                                                0x00000080L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV2_F1_MASK                                                0x00000100L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV2_F1_MASK                                              0x00003E00L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV2_F1_MASK                                       0x00004000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_AER_EN_DEV2_F1_MASK                                                       0x00010000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_ACS_EN_DEV2_F1_MASK                                                       0x00020000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV2_F1_MASK                                             0x00100000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_DPA_EN_DEV2_F1_MASK                                                       0x00200000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV2_F1_MASK                                                0x07000000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_EN_DEV2_F1_MASK                                                     0x10000000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV2_F1_MASK                               0x20000000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV2_F1_MASK                            0x40000000L
+#define RCC_DEV2_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV2_F1_MASK                                    0x80000000L
+//RCC_DEV2_EPF1_STRAP3
+#define RCC_DEV2_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV2_F1__SHIFT                                                  0x0
+#define RCC_DEV2_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV2_F1__SHIFT                                 0x10
+#define RCC_DEV2_EPF1_STRAP3__STRAP_PWR_EN_DEV2_F1__SHIFT                                                     0x11
+#define RCC_DEV2_EPF1_STRAP3__STRAP_MSI_EN_DEV2_F1__SHIFT                                                     0x12
+#define RCC_DEV2_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV2_F1__SHIFT                                         0x13
+#define RCC_DEV2_EPF1_STRAP3__STRAP_MSIX_EN_DEV2_F1__SHIFT                                                    0x14
+#define RCC_DEV2_EPF1_STRAP3__STRAP_PMC_DSI_DEV2_F1__SHIFT                                                    0x18
+#define RCC_DEV2_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV2_F1__SHIFT                        0x19
+#define RCC_DEV2_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV2_F1__SHIFT                                   0x1a
+#define RCC_DEV2_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV2_F1__SHIFT                                  0x1b
+#define RCC_DEV2_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV2_F1__SHIFT                                                  0x1d
+#define RCC_DEV2_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV2_F1__SHIFT                                          0x1e
+#define RCC_DEV2_EPF1_STRAP3__STRAP_RTR_EN_DEV2_F1__SHIFT                                                     0x1f
+#define RCC_DEV2_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV2_F1_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV2_F1_MASK                                   0x00010000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_PWR_EN_DEV2_F1_MASK                                                       0x00020000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_MSI_EN_DEV2_F1_MASK                                                       0x00040000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV2_F1_MASK                                           0x00080000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_MSIX_EN_DEV2_F1_MASK                                                      0x00100000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_PMC_DSI_DEV2_F1_MASK                                                      0x01000000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV2_F1_MASK                          0x02000000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV2_F1_MASK                                     0x04000000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV2_F1_MASK                                    0x08000000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV2_F1_MASK                                                    0x20000000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV2_F1_MASK                                            0x40000000L
+#define RCC_DEV2_EPF1_STRAP3__STRAP_RTR_EN_DEV2_F1_MASK                                                       0x80000000L
+//RCC_DEV2_EPF1_STRAP4
+#define RCC_DEV2_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV2_F1__SHIFT                                            0x14
+#define RCC_DEV2_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV2_F1__SHIFT                                                  0x15
+#define RCC_DEV2_EPF1_STRAP4__STRAP_FLR_EN_DEV2_F1__SHIFT                                                     0x16
+#define RCC_DEV2_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV2_F1__SHIFT                                                0x17
+#define RCC_DEV2_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV2_F1__SHIFT                                              0x1c
+#define RCC_DEV2_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV2_F1__SHIFT                                             0x1f
+#define RCC_DEV2_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV2_F1_MASK                                              0x00100000L
+#define RCC_DEV2_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV2_F1_MASK                                                    0x00200000L
+#define RCC_DEV2_EPF1_STRAP4__STRAP_FLR_EN_DEV2_F1_MASK                                                       0x00400000L
+#define RCC_DEV2_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV2_F1_MASK                                                  0x0F800000L
+#define RCC_DEV2_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV2_F1_MASK                                                0x70000000L
+#define RCC_DEV2_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV2_F1_MASK                                               0x80000000L
+//RCC_DEV2_EPF1_STRAP5
+#define RCC_DEV2_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV2_F1__SHIFT                                              0x0
+#define RCC_DEV2_EPF1_STRAP5__STRAP_USB_DBESEL_DEV2_F1__SHIFT                                                 0x10
+#define RCC_DEV2_EPF1_STRAP5__STRAP_USB_DBESELD_DEV2_F1__SHIFT                                                0x14
+#define RCC_DEV2_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV2_F1__SHIFT                                                0x1b
+#define RCC_DEV2_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV2_F1__SHIFT                                       0x1e
+#define RCC_DEV2_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV2_F1_MASK                                                0x0000FFFFL
+#define RCC_DEV2_EPF1_STRAP5__STRAP_USB_DBESEL_DEV2_F1_MASK                                                   0x000F0000L
+#define RCC_DEV2_EPF1_STRAP5__STRAP_USB_DBESELD_DEV2_F1_MASK                                                  0x00F00000L
+#define RCC_DEV2_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV2_F1_MASK                                                  0x38000000L
+#define RCC_DEV2_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV2_F1_MASK                                         0x40000000L
+//RCC_DEV2_EPF1_STRAP6
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER0_EN_DEV2_F1__SHIFT                                                   0x0
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV2_F1__SHIFT                                             0x2
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER1_EN_DEV2_F1__SHIFT                                                   0x8
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV2_F1__SHIFT                                             0xa
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER2_EN_DEV2_F1__SHIFT                                                   0x10
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV2_F1__SHIFT                                             0x12
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER3_EN_DEV2_F1__SHIFT                                                   0x18
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER4_EN_DEV2_F1__SHIFT                                                   0x1a
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER0_EN_DEV2_F1_MASK                                                     0x00000001L
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV2_F1_MASK                                               0x00000004L
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER1_EN_DEV2_F1_MASK                                                     0x00000100L
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV2_F1_MASK                                               0x00000400L
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER2_EN_DEV2_F1_MASK                                                     0x00010000L
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV2_F1_MASK                                               0x00040000L
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER3_EN_DEV2_F1_MASK                                                     0x01000000L
+#define RCC_DEV2_EPF1_STRAP6__STRAP_APER4_EN_DEV2_F1_MASK                                                     0x04000000L
+//RCC_DEV2_EPF1_STRAP13
+#define RCC_DEV2_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV2_F1__SHIFT                                            0x0
+#define RCC_DEV2_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV2_F1__SHIFT                                            0x8
+#define RCC_DEV2_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV2_F1__SHIFT                                           0x10
+#define RCC_DEV2_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV2_F1_MASK                                              0x000000FFL
+#define RCC_DEV2_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV2_F1_MASK                                              0x0000FF00L
+#define RCC_DEV2_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV2_F1_MASK                                             0x00FF0000L
+//RCC_DEV2_EPF1_STRAP14
+#define RCC_DEV2_EPF1_STRAP14__STRAP_VENDOR_ID_DEV2_F1__SHIFT                                                 0x0
+#define RCC_DEV2_EPF1_STRAP14__STRAP_VENDOR_ID_DEV2_F1_MASK                                                   0x0000FFFFL
+//RCC_DEV2_EPF2_STRAP0
+#define RCC_DEV2_EPF2_STRAP0__STRAP_DEVICE_ID_DEV2_F2__SHIFT                                                  0x0
+#define RCC_DEV2_EPF2_STRAP0__STRAP_MAJOR_REV_ID_DEV2_F2__SHIFT                                               0x10
+#define RCC_DEV2_EPF2_STRAP0__STRAP_MINOR_REV_ID_DEV2_F2__SHIFT                                               0x14
+#define RCC_DEV2_EPF2_STRAP0__STRAP_FUNC_EN_DEV2_F2__SHIFT                                                    0x1c
+#define RCC_DEV2_EPF2_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV2_F2__SHIFT                                      0x1d
+#define RCC_DEV2_EPF2_STRAP0__STRAP_D1_SUPPORT_DEV2_F2__SHIFT                                                 0x1e
+#define RCC_DEV2_EPF2_STRAP0__STRAP_D2_SUPPORT_DEV2_F2__SHIFT                                                 0x1f
+#define RCC_DEV2_EPF2_STRAP0__STRAP_DEVICE_ID_DEV2_F2_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_EPF2_STRAP0__STRAP_MAJOR_REV_ID_DEV2_F2_MASK                                                 0x000F0000L
+#define RCC_DEV2_EPF2_STRAP0__STRAP_MINOR_REV_ID_DEV2_F2_MASK                                                 0x00F00000L
+#define RCC_DEV2_EPF2_STRAP0__STRAP_FUNC_EN_DEV2_F2_MASK                                                      0x10000000L
+#define RCC_DEV2_EPF2_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV2_F2_MASK                                        0x20000000L
+#define RCC_DEV2_EPF2_STRAP0__STRAP_D1_SUPPORT_DEV2_F2_MASK                                                   0x40000000L
+#define RCC_DEV2_EPF2_STRAP0__STRAP_D2_SUPPORT_DEV2_F2_MASK                                                   0x80000000L
+//RCC_DEV2_EPF2_STRAP2
+#define RCC_DEV2_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV2_F2__SHIFT                               0x4
+#define RCC_DEV2_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV2_F2__SHIFT                                              0x7
+#define RCC_DEV2_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV2_F2__SHIFT                                              0x8
+#define RCC_DEV2_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV2_F2__SHIFT                                            0x9
+#define RCC_DEV2_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV2_F2__SHIFT                                     0xe
+#define RCC_DEV2_EPF2_STRAP2__STRAP_AER_EN_DEV2_F2__SHIFT                                                     0x10
+#define RCC_DEV2_EPF2_STRAP2__STRAP_ACS_EN_DEV2_F2__SHIFT                                                     0x11
+#define RCC_DEV2_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV2_F2__SHIFT                                           0x14
+#define RCC_DEV2_EPF2_STRAP2__STRAP_DPA_EN_DEV2_F2__SHIFT                                                     0x15
+#define RCC_DEV2_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV2_F2__SHIFT                                              0x18
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_EN_DEV2_F2__SHIFT                                                   0x1c
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV2_F2__SHIFT                             0x1d
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV2_F2__SHIFT                          0x1e
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV2_F2__SHIFT                                  0x1f
+#define RCC_DEV2_EPF2_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV2_F2_MASK                                 0x00000010L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_NO_SOFT_RESET_DEV2_F2_MASK                                                0x00000080L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_RESIZE_BAR_EN_DEV2_F2_MASK                                                0x00000100L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_MAX_PASID_WIDTH_DEV2_F2_MASK                                              0x00003E00L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV2_F2_MASK                                       0x00004000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_AER_EN_DEV2_F2_MASK                                                       0x00010000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_ACS_EN_DEV2_F2_MASK                                                       0x00020000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV2_F2_MASK                                             0x00100000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_DPA_EN_DEV2_F2_MASK                                                       0x00200000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_MSI_MULTI_CAP_DEV2_F2_MASK                                                0x07000000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_EN_DEV2_F2_MASK                                                     0x10000000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV2_F2_MASK                               0x20000000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV2_F2_MASK                            0x40000000L
+#define RCC_DEV2_EPF2_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV2_F2_MASK                                    0x80000000L
+//RCC_DEV2_EPF2_STRAP3
+#define RCC_DEV2_EPF2_STRAP3__STRAP_SUBSYS_ID_DEV2_F2__SHIFT                                                  0x0
+#define RCC_DEV2_EPF2_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV2_F2__SHIFT                                 0x10
+#define RCC_DEV2_EPF2_STRAP3__STRAP_PWR_EN_DEV2_F2__SHIFT                                                     0x11
+#define RCC_DEV2_EPF2_STRAP3__STRAP_MSI_EN_DEV2_F2__SHIFT                                                     0x12
+#define RCC_DEV2_EPF2_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV2_F2__SHIFT                                         0x13
+#define RCC_DEV2_EPF2_STRAP3__STRAP_MSIX_EN_DEV2_F2__SHIFT                                                    0x14
+#define RCC_DEV2_EPF2_STRAP3__STRAP_PMC_DSI_DEV2_F2__SHIFT                                                    0x18
+#define RCC_DEV2_EPF2_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV2_F2__SHIFT                        0x19
+#define RCC_DEV2_EPF2_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV2_F2__SHIFT                                   0x1a
+#define RCC_DEV2_EPF2_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV2_F2__SHIFT                                  0x1b
+#define RCC_DEV2_EPF2_STRAP3__STRAP_CLK_PM_EN_DEV2_F2__SHIFT                                                  0x1d
+#define RCC_DEV2_EPF2_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV2_F2__SHIFT                                          0x1e
+#define RCC_DEV2_EPF2_STRAP3__STRAP_RTR_EN_DEV2_F2__SHIFT                                                     0x1f
+#define RCC_DEV2_EPF2_STRAP3__STRAP_SUBSYS_ID_DEV2_F2_MASK                                                    0x0000FFFFL
+#define RCC_DEV2_EPF2_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV2_F2_MASK                                   0x00010000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_PWR_EN_DEV2_F2_MASK                                                       0x00020000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_MSI_EN_DEV2_F2_MASK                                                       0x00040000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV2_F2_MASK                                           0x00080000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_MSIX_EN_DEV2_F2_MASK                                                      0x00100000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_PMC_DSI_DEV2_F2_MASK                                                      0x01000000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV2_F2_MASK                          0x02000000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV2_F2_MASK                                     0x04000000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV2_F2_MASK                                    0x08000000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_CLK_PM_EN_DEV2_F2_MASK                                                    0x20000000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV2_F2_MASK                                            0x40000000L
+#define RCC_DEV2_EPF2_STRAP3__STRAP_RTR_EN_DEV2_F2_MASK                                                       0x80000000L
+//RCC_DEV2_EPF2_STRAP4
+#define RCC_DEV2_EPF2_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV2_F2__SHIFT                                            0x14
+#define RCC_DEV2_EPF2_STRAP4__STRAP_ATOMIC_EN_DEV2_F2__SHIFT                                                  0x15
+#define RCC_DEV2_EPF2_STRAP4__STRAP_FLR_EN_DEV2_F2__SHIFT                                                     0x16
+#define RCC_DEV2_EPF2_STRAP4__STRAP_PME_SUPPORT_DEV2_F2__SHIFT                                                0x17
+#define RCC_DEV2_EPF2_STRAP4__STRAP_INTERRUPT_PIN_DEV2_F2__SHIFT                                              0x1c
+#define RCC_DEV2_EPF2_STRAP4__STRAP_AUXPWR_SUPPORT_DEV2_F2__SHIFT                                             0x1f
+#define RCC_DEV2_EPF2_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV2_F2_MASK                                              0x00100000L
+#define RCC_DEV2_EPF2_STRAP4__STRAP_ATOMIC_EN_DEV2_F2_MASK                                                    0x00200000L
+#define RCC_DEV2_EPF2_STRAP4__STRAP_FLR_EN_DEV2_F2_MASK                                                       0x00400000L
+#define RCC_DEV2_EPF2_STRAP4__STRAP_PME_SUPPORT_DEV2_F2_MASK                                                  0x0F800000L
+#define RCC_DEV2_EPF2_STRAP4__STRAP_INTERRUPT_PIN_DEV2_F2_MASK                                                0x70000000L
+#define RCC_DEV2_EPF2_STRAP4__STRAP_AUXPWR_SUPPORT_DEV2_F2_MASK                                               0x80000000L
+//RCC_DEV2_EPF2_STRAP5
+#define RCC_DEV2_EPF2_STRAP5__STRAP_SUBSYS_VEN_ID_DEV2_F2__SHIFT                                              0x0
+#define RCC_DEV2_EPF2_STRAP5__STRAP_USB_DBESEL_DEV2_F2__SHIFT                                                 0x10
+#define RCC_DEV2_EPF2_STRAP5__STRAP_USB_DBESELD_DEV2_F2__SHIFT                                                0x14
+#define RCC_DEV2_EPF2_STRAP5__STRAP_AUX_CURRENT_DEV2_F2__SHIFT                                                0x1b
+#define RCC_DEV2_EPF2_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV2_F2__SHIFT                                       0x1e
+#define RCC_DEV2_EPF2_STRAP5__STRAP_SUBSYS_VEN_ID_DEV2_F2_MASK                                                0x0000FFFFL
+#define RCC_DEV2_EPF2_STRAP5__STRAP_USB_DBESEL_DEV2_F2_MASK                                                   0x000F0000L
+#define RCC_DEV2_EPF2_STRAP5__STRAP_USB_DBESELD_DEV2_F2_MASK                                                  0x00F00000L
+#define RCC_DEV2_EPF2_STRAP5__STRAP_AUX_CURRENT_DEV2_F2_MASK                                                  0x38000000L
+#define RCC_DEV2_EPF2_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV2_F2_MASK                                         0x40000000L
+//RCC_DEV2_EPF2_STRAP6
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER0_EN_DEV2_F2__SHIFT                                                   0x0
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER0_64BAR_EN_DEV2_F2__SHIFT                                             0x2
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER1_EN_DEV2_F2__SHIFT                                                   0x8
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER1_64BAR_EN_DEV2_F2__SHIFT                                             0xa
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER2_EN_DEV2_F2__SHIFT                                                   0x10
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER2_64BAR_EN_DEV2_F2__SHIFT                                             0x12
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER3_EN_DEV2_F2__SHIFT                                                   0x18
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER4_EN_DEV2_F2__SHIFT                                                   0x1a
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER0_EN_DEV2_F2_MASK                                                     0x00000001L
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER0_64BAR_EN_DEV2_F2_MASK                                               0x00000004L
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER1_EN_DEV2_F2_MASK                                                     0x00000100L
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER1_64BAR_EN_DEV2_F2_MASK                                               0x00000400L
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER2_EN_DEV2_F2_MASK                                                     0x00010000L
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER2_64BAR_EN_DEV2_F2_MASK                                               0x00040000L
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER3_EN_DEV2_F2_MASK                                                     0x01000000L
+#define RCC_DEV2_EPF2_STRAP6__STRAP_APER4_EN_DEV2_F2_MASK                                                     0x04000000L
+//RCC_DEV2_EPF2_STRAP13
+#define RCC_DEV2_EPF2_STRAP13__STRAP_CLASS_CODE_PIF_DEV2_F2__SHIFT                                            0x0
+#define RCC_DEV2_EPF2_STRAP13__STRAP_CLASS_CODE_SUB_DEV2_F2__SHIFT                                            0x8
+#define RCC_DEV2_EPF2_STRAP13__STRAP_CLASS_CODE_BASE_DEV2_F2__SHIFT                                           0x10
+#define RCC_DEV2_EPF2_STRAP13__STRAP_CLASS_CODE_PIF_DEV2_F2_MASK                                              0x000000FFL
+#define RCC_DEV2_EPF2_STRAP13__STRAP_CLASS_CODE_SUB_DEV2_F2_MASK                                              0x0000FF00L
+#define RCC_DEV2_EPF2_STRAP13__STRAP_CLASS_CODE_BASE_DEV2_F2_MASK                                             0x00FF0000L
+//RCC_DEV2_EPF2_STRAP14
+#define RCC_DEV2_EPF2_STRAP14__STRAP_VENDOR_ID_DEV2_F2__SHIFT                                                 0x0
+#define RCC_DEV2_EPF2_STRAP14__STRAP_VENDOR_ID_DEV2_F2_MASK                                                   0x0000FFFFL
+
+
+// addressBlock: nbif0_bif_rst_bif_rst_regblk
+//HARD_RST_CTRL
+#define HARD_RST_CTRL__DSPT_CFG_RST_EN__SHIFT                                                                 0x0
+#define HARD_RST_CTRL__DSPT_CFG_STICKY_RST_EN__SHIFT                                                          0x1
+#define HARD_RST_CTRL__DSPT_PRV_RST_EN__SHIFT                                                                 0x2
+#define HARD_RST_CTRL__DSPT_PRV_STICKY_RST_EN__SHIFT                                                          0x3
+#define HARD_RST_CTRL__EP_CFG_RST_EN__SHIFT                                                                   0x4
+#define HARD_RST_CTRL__EP_CFG_STICKY_RST_EN__SHIFT                                                            0x5
+#define HARD_RST_CTRL__EP_PRV_RST_EN__SHIFT                                                                   0x6
+#define HARD_RST_CTRL__EP_PRV_STICKY_RST_EN__SHIFT                                                            0x7
+#define HARD_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                               0x9
+#define HARD_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                               0xa
+#define HARD_RST_CTRL__STRAP_RST_EN__SHIFT                                                                    0x17
+#define HARD_RST_CTRL__SWUS_SHADOW_RST_EN__SHIFT                                                              0x1c
+#define HARD_RST_CTRL__CORE_STICKY_RST_EN__SHIFT                                                              0x1d
+#define HARD_RST_CTRL__RELOAD_STRAP_EN__SHIFT                                                                 0x1e
+#define HARD_RST_CTRL__CORE_RST_EN__SHIFT                                                                     0x1f
+#define HARD_RST_CTRL__DSPT_CFG_RST_EN_MASK                                                                   0x00000001L
+#define HARD_RST_CTRL__DSPT_CFG_STICKY_RST_EN_MASK                                                            0x00000002L
+#define HARD_RST_CTRL__DSPT_PRV_RST_EN_MASK                                                                   0x00000004L
+#define HARD_RST_CTRL__DSPT_PRV_STICKY_RST_EN_MASK                                                            0x00000008L
+#define HARD_RST_CTRL__EP_CFG_RST_EN_MASK                                                                     0x00000010L
+#define HARD_RST_CTRL__EP_CFG_STICKY_RST_EN_MASK                                                              0x00000020L
+#define HARD_RST_CTRL__EP_PRV_RST_EN_MASK                                                                     0x00000040L
+#define HARD_RST_CTRL__EP_PRV_STICKY_RST_EN_MASK                                                              0x00000080L
+#define HARD_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                                 0x00000200L
+#define HARD_RST_CTRL__SION_AON_RESET_EN_MASK                                                                 0x00000400L
+#define HARD_RST_CTRL__STRAP_RST_EN_MASK                                                                      0x00800000L
+#define HARD_RST_CTRL__SWUS_SHADOW_RST_EN_MASK                                                                0x10000000L
+#define HARD_RST_CTRL__CORE_STICKY_RST_EN_MASK                                                                0x20000000L
+#define HARD_RST_CTRL__RELOAD_STRAP_EN_MASK                                                                   0x40000000L
+#define HARD_RST_CTRL__CORE_RST_EN_MASK                                                                       0x80000000L
+//RSMU_SOFT_RST_CTRL
+#define RSMU_SOFT_RST_CTRL__DSPT_CFG_RST_EN__SHIFT                                                            0x0
+#define RSMU_SOFT_RST_CTRL__DSPT_CFG_STICKY_RST_EN__SHIFT                                                     0x1
+#define RSMU_SOFT_RST_CTRL__DSPT_PRV_RST_EN__SHIFT                                                            0x2
+#define RSMU_SOFT_RST_CTRL__DSPT_PRV_STICKY_RST_EN__SHIFT                                                     0x3
+#define RSMU_SOFT_RST_CTRL__EP_CFG_RST_EN__SHIFT                                                              0x4
+#define RSMU_SOFT_RST_CTRL__EP_CFG_STICKY_RST_EN__SHIFT                                                       0x5
+#define RSMU_SOFT_RST_CTRL__EP_PRV_RST_EN__SHIFT                                                              0x6
+#define RSMU_SOFT_RST_CTRL__EP_PRV_STICKY_RST_EN__SHIFT                                                       0x7
+#define RSMU_SOFT_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                          0x9
+#define RSMU_SOFT_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                          0xa
+#define RSMU_SOFT_RST_CTRL__STRAP_RST_EN__SHIFT                                                               0x17
+#define RSMU_SOFT_RST_CTRL__SWUS_SHADOW_RST_EN__SHIFT                                                         0x1c
+#define RSMU_SOFT_RST_CTRL__CORE_STICKY_RST_EN__SHIFT                                                         0x1d
+#define RSMU_SOFT_RST_CTRL__RELOAD_STRAP_EN__SHIFT                                                            0x1e
+#define RSMU_SOFT_RST_CTRL__CORE_RST_EN__SHIFT                                                                0x1f
+#define RSMU_SOFT_RST_CTRL__DSPT_CFG_RST_EN_MASK                                                              0x00000001L
+#define RSMU_SOFT_RST_CTRL__DSPT_CFG_STICKY_RST_EN_MASK                                                       0x00000002L
+#define RSMU_SOFT_RST_CTRL__DSPT_PRV_RST_EN_MASK                                                              0x00000004L
+#define RSMU_SOFT_RST_CTRL__DSPT_PRV_STICKY_RST_EN_MASK                                                       0x00000008L
+#define RSMU_SOFT_RST_CTRL__EP_CFG_RST_EN_MASK                                                                0x00000010L
+#define RSMU_SOFT_RST_CTRL__EP_CFG_STICKY_RST_EN_MASK                                                         0x00000020L
+#define RSMU_SOFT_RST_CTRL__EP_PRV_RST_EN_MASK                                                                0x00000040L
+#define RSMU_SOFT_RST_CTRL__EP_PRV_STICKY_RST_EN_MASK                                                         0x00000080L
+#define RSMU_SOFT_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                            0x00000200L
+#define RSMU_SOFT_RST_CTRL__SION_AON_RESET_EN_MASK                                                            0x00000400L
+#define RSMU_SOFT_RST_CTRL__STRAP_RST_EN_MASK                                                                 0x00800000L
+#define RSMU_SOFT_RST_CTRL__SWUS_SHADOW_RST_EN_MASK                                                           0x10000000L
+#define RSMU_SOFT_RST_CTRL__CORE_STICKY_RST_EN_MASK                                                           0x20000000L
+#define RSMU_SOFT_RST_CTRL__RELOAD_STRAP_EN_MASK                                                              0x40000000L
+#define RSMU_SOFT_RST_CTRL__CORE_RST_EN_MASK                                                                  0x80000000L
+//SELF_SOFT_RST
+#define SELF_SOFT_RST__DSPT0_CFG_RST__SHIFT                                                                   0x0
+#define SELF_SOFT_RST__DSPT0_CFG_STICKY_RST__SHIFT                                                            0x1
+#define SELF_SOFT_RST__DSPT0_PRV_RST__SHIFT                                                                   0x2
+#define SELF_SOFT_RST__DSPT0_PRV_STICKY_RST__SHIFT                                                            0x3
+#define SELF_SOFT_RST__EP0_CFG_RST__SHIFT                                                                     0x4
+#define SELF_SOFT_RST__EP0_CFG_STICKY_RST__SHIFT                                                              0x5
+#define SELF_SOFT_RST__EP0_PRV_RST__SHIFT                                                                     0x6
+#define SELF_SOFT_RST__EP0_PRV_STICKY_RST__SHIFT                                                              0x7
+#define SELF_SOFT_RST__DSPT1_CFG_RST__SHIFT                                                                   0x8
+#define SELF_SOFT_RST__DSPT1_CFG_STICKY_RST__SHIFT                                                            0x9
+#define SELF_SOFT_RST__DSPT1_PRV_RST__SHIFT                                                                   0xa
+#define SELF_SOFT_RST__DSPT1_PRV_STICKY_RST__SHIFT                                                            0xb
+#define SELF_SOFT_RST__EP1_CFG_RST__SHIFT                                                                     0xc
+#define SELF_SOFT_RST__EP1_CFG_STICKY_RST__SHIFT                                                              0xd
+#define SELF_SOFT_RST__EP1_PRV_RST__SHIFT                                                                     0xe
+#define SELF_SOFT_RST__EP1_PRV_STICKY_RST__SHIFT                                                              0xf
+#define SELF_SOFT_RST__DSPT2_CFG_RST__SHIFT                                                                   0x10
+#define SELF_SOFT_RST__DSPT2_CFG_STICKY_RST__SHIFT                                                            0x11
+#define SELF_SOFT_RST__DSPT2_PRV_RST__SHIFT                                                                   0x12
+#define SELF_SOFT_RST__DSPT2_PRV_STICKY_RST__SHIFT                                                            0x13
+#define SELF_SOFT_RST__EP2_CFG_RST__SHIFT                                                                     0x14
+#define SELF_SOFT_RST__EP2_CFG_STICKY_RST__SHIFT                                                              0x15
+#define SELF_SOFT_RST__EP2_PRV_RST__SHIFT                                                                     0x16
+#define SELF_SOFT_RST__EP2_PRV_STICKY_RST__SHIFT                                                              0x17
+#define SELF_SOFT_RST__PCIE_NBIF_HST_SDP_PORT_RST__SHIFT                                                      0x18
+#define SELF_SOFT_RST__SST_HST_SDP_PORT_RST__SHIFT                                                            0x19
+#define SELF_SOFT_RST__NBIF_PCIE_DMA_SDP_PORT_RST__SHIFT                                                      0x1a
+#define SELF_SOFT_RST__SST_DMA_SDP_PORT_RST__SHIFT                                                            0x1b
+#define SELF_SOFT_RST__SWUS_SHADOW_RST__SHIFT                                                                 0x1c
+#define SELF_SOFT_RST__CORE_STICKY_RST__SHIFT                                                                 0x1d
+#define SELF_SOFT_RST__RELOAD_STRAP__SHIFT                                                                    0x1e
+#define SELF_SOFT_RST__CORE_RST__SHIFT                                                                        0x1f
+#define SELF_SOFT_RST__DSPT0_CFG_RST_MASK                                                                     0x00000001L
+#define SELF_SOFT_RST__DSPT0_CFG_STICKY_RST_MASK                                                              0x00000002L
+#define SELF_SOFT_RST__DSPT0_PRV_RST_MASK                                                                     0x00000004L
+#define SELF_SOFT_RST__DSPT0_PRV_STICKY_RST_MASK                                                              0x00000008L
+#define SELF_SOFT_RST__EP0_CFG_RST_MASK                                                                       0x00000010L
+#define SELF_SOFT_RST__EP0_CFG_STICKY_RST_MASK                                                                0x00000020L
+#define SELF_SOFT_RST__EP0_PRV_RST_MASK                                                                       0x00000040L
+#define SELF_SOFT_RST__EP0_PRV_STICKY_RST_MASK                                                                0x00000080L
+#define SELF_SOFT_RST__DSPT1_CFG_RST_MASK                                                                     0x00000100L
+#define SELF_SOFT_RST__DSPT1_CFG_STICKY_RST_MASK                                                              0x00000200L
+#define SELF_SOFT_RST__DSPT1_PRV_RST_MASK                                                                     0x00000400L
+#define SELF_SOFT_RST__DSPT1_PRV_STICKY_RST_MASK                                                              0x00000800L
+#define SELF_SOFT_RST__EP1_CFG_RST_MASK                                                                       0x00001000L
+#define SELF_SOFT_RST__EP1_CFG_STICKY_RST_MASK                                                                0x00002000L
+#define SELF_SOFT_RST__EP1_PRV_RST_MASK                                                                       0x00004000L
+#define SELF_SOFT_RST__EP1_PRV_STICKY_RST_MASK                                                                0x00008000L
+#define SELF_SOFT_RST__DSPT2_CFG_RST_MASK                                                                     0x00010000L
+#define SELF_SOFT_RST__DSPT2_CFG_STICKY_RST_MASK                                                              0x00020000L
+#define SELF_SOFT_RST__DSPT2_PRV_RST_MASK                                                                     0x00040000L
+#define SELF_SOFT_RST__DSPT2_PRV_STICKY_RST_MASK                                                              0x00080000L
+#define SELF_SOFT_RST__EP2_CFG_RST_MASK                                                                       0x00100000L
+#define SELF_SOFT_RST__EP2_CFG_STICKY_RST_MASK                                                                0x00200000L
+#define SELF_SOFT_RST__EP2_PRV_RST_MASK                                                                       0x00400000L
+#define SELF_SOFT_RST__EP2_PRV_STICKY_RST_MASK                                                                0x00800000L
+#define SELF_SOFT_RST__PCIE_NBIF_HST_SDP_PORT_RST_MASK                                                        0x01000000L
+#define SELF_SOFT_RST__SST_HST_SDP_PORT_RST_MASK                                                              0x02000000L
+#define SELF_SOFT_RST__NBIF_PCIE_DMA_SDP_PORT_RST_MASK                                                        0x04000000L
+#define SELF_SOFT_RST__SST_DMA_SDP_PORT_RST_MASK                                                              0x08000000L
+#define SELF_SOFT_RST__SWUS_SHADOW_RST_MASK                                                                   0x10000000L
+#define SELF_SOFT_RST__CORE_STICKY_RST_MASK                                                                   0x20000000L
+#define SELF_SOFT_RST__RELOAD_STRAP_MASK                                                                      0x40000000L
+#define SELF_SOFT_RST__CORE_RST_MASK                                                                          0x80000000L
+//BIF_GFX_DRV_VPU_RST
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_RST__SHIFT                                                      0x0
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_FLR_EXC_RST__SHIFT                                              0x1
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_STICKY_RST__SHIFT                                               0x2
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_RST__SHIFT                                                      0x3
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_STICKY_RST__SHIFT                                               0x4
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_RST__SHIFT                                                      0x5
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_STICKY_RST__SHIFT                                               0x6
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_PRV_RST__SHIFT                                                      0x7
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_RST_MASK                                                        0x00000001L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_FLR_EXC_RST_MASK                                                0x00000002L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_STICKY_RST_MASK                                                 0x00000004L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_RST_MASK                                                        0x00000008L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_STICKY_RST_MASK                                                 0x00000010L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_RST_MASK                                                        0x00000020L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_STICKY_RST_MASK                                                 0x00000040L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_PRV_RST_MASK                                                        0x00000080L
+//BIF_RST_MISC_CTRL
+#define BIF_RST_MISC_CTRL__ERRSTATUS_KEPT_IN_PERSTB__SHIFT                                                    0x0
+#define BIF_RST_MISC_CTRL__DRV_RST_MODE__SHIFT                                                                0x2
+#define BIF_RST_MISC_CTRL__DRV_RST_CFG_MASK__SHIFT                                                            0x4
+#define BIF_RST_MISC_CTRL__DRV_RST_BITS_AUTO_CLEAR__SHIFT                                                     0x5
+#define BIF_RST_MISC_CTRL__FLR_RST_BIT_AUTO_CLEAR__SHIFT                                                      0x6
+#define BIF_RST_MISC_CTRL__STRAP_EP_LNK_RST_IOV_EN__SHIFT                                                     0x8
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_MODE__SHIFT                                                          0x9
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_TIMEOUT__SHIFT                                                       0xa
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER_SEL__SHIFT                                                           0xd
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER2_SEL__SHIFT                                                          0xf
+#define BIF_RST_MISC_CTRL__SRIOV_SAVE_VFS_ON_VFENABLE_CLR__SHIFT                                              0x11
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_DIS__SHIFT                                                       0x17
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_RSPSTS__SHIFT                                                    0x18
+#define BIF_RST_MISC_CTRL__HARD_RST_CTRL_SWUS_SHADOW_PRV_RST_EN__SHIFT                                        0x1a
+#define BIF_RST_MISC_CTRL__RSMU_SOFT_RST_CTRL_SWUS_SHADOW_PRV_RST_EN__SHIFT                                   0x1b
+#define BIF_RST_MISC_CTRL__ERRSTATUS_KEPT_IN_PERSTB_MASK                                                      0x00000001L
+#define BIF_RST_MISC_CTRL__DRV_RST_MODE_MASK                                                                  0x0000000CL
+#define BIF_RST_MISC_CTRL__DRV_RST_CFG_MASK_MASK                                                              0x00000010L
+#define BIF_RST_MISC_CTRL__DRV_RST_BITS_AUTO_CLEAR_MASK                                                       0x00000020L
+#define BIF_RST_MISC_CTRL__FLR_RST_BIT_AUTO_CLEAR_MASK                                                        0x00000040L
+#define BIF_RST_MISC_CTRL__STRAP_EP_LNK_RST_IOV_EN_MASK                                                       0x00000100L
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_MODE_MASK                                                            0x00000200L
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_TIMEOUT_MASK                                                         0x00001C00L
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER_SEL_MASK                                                             0x00006000L
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER2_SEL_MASK                                                            0x00018000L
+#define BIF_RST_MISC_CTRL__SRIOV_SAVE_VFS_ON_VFENABLE_CLR_MASK                                                0x000E0000L
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_DIS_MASK                                                         0x00800000L
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_RSPSTS_MASK                                                      0x03000000L
+#define BIF_RST_MISC_CTRL__HARD_RST_CTRL_SWUS_SHADOW_PRV_RST_EN_MASK                                          0x04000000L
+#define BIF_RST_MISC_CTRL__RSMU_SOFT_RST_CTRL_SWUS_SHADOW_PRV_RST_EN_MASK                                     0x08000000L
+//BIF_RST_MISC_CTRL2
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_PROTECT__SHIFT                                                       0x0
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_PROTECT__SHIFT                                                       0x1
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_PROTECT__SHIFT                                                      0x2
+#define BIF_RST_MISC_CTRL2__ENDP1_LNK_RST_PROTECT__SHIFT                                                      0x3
+#define BIF_RST_MISC_CTRL2__ENDP2_LNK_RST_PROTECT__SHIFT                                                      0x4
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT__SHIFT                                                            0xf
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_TRANS_IDLE__SHIFT                                                    0x10
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_TRANS_IDLE__SHIFT                                                    0x11
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_TRANS_IDLE__SHIFT                                                   0x12
+#define BIF_RST_MISC_CTRL2__ENDP1_LNK_RST_TRANS_IDLE__SHIFT                                                   0x13
+#define BIF_RST_MISC_CTRL2__ENDP2_LNK_RST_TRANS_IDLE__SHIFT                                                   0x14
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_DIS__SHIFT                                                        0x1e
+#define BIF_RST_MISC_CTRL2__ALL_RST_TRANS_IDLE__SHIFT                                                         0x1f
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_PROTECT_MASK                                                         0x00000001L
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_PROTECT_MASK                                                         0x00000002L
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_PROTECT_MASK                                                        0x00000004L
+#define BIF_RST_MISC_CTRL2__ENDP1_LNK_RST_PROTECT_MASK                                                        0x00000008L
+#define BIF_RST_MISC_CTRL2__ENDP2_LNK_RST_PROTECT_MASK                                                        0x00000010L
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_MASK                                                              0x00008000L
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_TRANS_IDLE_MASK                                                      0x00010000L
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_TRANS_IDLE_MASK                                                      0x00020000L
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_TRANS_IDLE_MASK                                                     0x00040000L
+#define BIF_RST_MISC_CTRL2__ENDP1_LNK_RST_TRANS_IDLE_MASK                                                     0x00080000L
+#define BIF_RST_MISC_CTRL2__ENDP2_LNK_RST_TRANS_IDLE_MASK                                                     0x00100000L
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_DIS_MASK                                                          0x40000000L
+#define BIF_RST_MISC_CTRL2__ALL_RST_TRANS_IDLE_MASK                                                           0x80000000L
+//BIF_RST_MISC_CTRL3
+#define BIF_RST_MISC_CTRL3__TIMER_SCALE__SHIFT                                                                0x0
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_TIMEOUT__SHIFT                                                        0x4
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_MODE__SHIFT                                                           0x6
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_HARD__SHIFT                                                    0x7
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SOFT__SHIFT                                                    0xa
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SELF__SHIFT                                                    0xd
+#define BIF_RST_MISC_CTRL3__RSMU_SOFT_RST_CYCLE__SHIFT                                                        0x10
+#define BIF_RST_MISC_CTRL3__TIMER_SCALE_MASK                                                                  0x0000000FL
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_TIMEOUT_MASK                                                          0x00000030L
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_MODE_MASK                                                             0x00000040L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_HARD_MASK                                                      0x00000380L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SOFT_MASK                                                      0x00001C00L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SELF_MASK                                                      0x0000E000L
+#define BIF_RST_MISC_CTRL3__RSMU_SOFT_RST_CYCLE_MASK                                                          0x00FF0000L
+//DEV0_PF0_FLR_RST_CTRL
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV0_PF1_FLR_RST_CTRL
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV0_PF2_FLR_RST_CTRL
+#define DEV0_PF2_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF2_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF2_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF2_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF2_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF2_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF2_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF2_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF2_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF2_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF2_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF2_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF2_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF2_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF2_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF2_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF2_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF2_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV0_PF3_FLR_RST_CTRL
+#define DEV0_PF3_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF3_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF3_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF3_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF3_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF3_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF3_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF3_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF3_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF3_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF3_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF3_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF3_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF3_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF3_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF3_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF3_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF3_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV0_PF4_FLR_RST_CTRL
+#define DEV0_PF4_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF4_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF4_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF4_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF4_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF4_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF4_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF4_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF4_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF4_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF4_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF4_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF4_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF4_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF4_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF4_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF4_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF4_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV0_PF5_FLR_RST_CTRL
+#define DEV0_PF5_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF5_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF5_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF5_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF5_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF5_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF5_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF5_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF5_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF5_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF5_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF5_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF5_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF5_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF5_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF5_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF5_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF5_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV0_PF6_FLR_RST_CTRL
+#define DEV0_PF6_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF6_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF6_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF6_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF6_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF6_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF6_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF6_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF6_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF6_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF6_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF6_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF6_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF6_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF6_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF6_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF6_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF6_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV0_PF7_FLR_RST_CTRL
+#define DEV0_PF7_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF7_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF7_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF7_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF7_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF7_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF7_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF7_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF7_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF7_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF7_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF7_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF7_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF7_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF7_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF7_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF7_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF7_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//BIF_INST_RESET_INTR_STS
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_INTR_STS__SHIFT                                               0x0
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_CFG_ONLY_INTR_STS__SHIFT                                      0x1
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M0_INTR_STS__SHIFT                                                 0x2
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M1_INTR_STS__SHIFT                                                 0x3
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M2_INTR_STS__SHIFT                                                 0x4
+#define BIF_INST_RESET_INTR_STS__EP1_LINK_RESET_INTR_STS__SHIFT                                               0x8
+#define BIF_INST_RESET_INTR_STS__EP1_LINK_RESET_CFG_ONLY_INTR_STS__SHIFT                                      0x9
+#define BIF_INST_RESET_INTR_STS__EP2_LINK_RESET_INTR_STS__SHIFT                                               0xa
+#define BIF_INST_RESET_INTR_STS__EP2_LINK_RESET_CFG_ONLY_INTR_STS__SHIFT                                      0xb
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_INTR_STS_MASK                                                 0x00000001L
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_CFG_ONLY_INTR_STS_MASK                                        0x00000002L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M0_INTR_STS_MASK                                                   0x00000004L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M1_INTR_STS_MASK                                                   0x00000008L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M2_INTR_STS_MASK                                                   0x00000010L
+#define BIF_INST_RESET_INTR_STS__EP1_LINK_RESET_INTR_STS_MASK                                                 0x00000100L
+#define BIF_INST_RESET_INTR_STS__EP1_LINK_RESET_CFG_ONLY_INTR_STS_MASK                                        0x00000200L
+#define BIF_INST_RESET_INTR_STS__EP2_LINK_RESET_INTR_STS_MASK                                                 0x00000400L
+#define BIF_INST_RESET_INTR_STS__EP2_LINK_RESET_CFG_ONLY_INTR_STS_MASK                                        0x00000800L
+//BIF_PF_FLR_INTR_STS
+#define BIF_PF_FLR_INTR_STS__DEV0_PF0_FLR_INTR_STS__SHIFT                                                     0x0
+#define BIF_PF_FLR_INTR_STS__DEV0_PF1_FLR_INTR_STS__SHIFT                                                     0x1
+#define BIF_PF_FLR_INTR_STS__DEV0_PF2_FLR_INTR_STS__SHIFT                                                     0x2
+#define BIF_PF_FLR_INTR_STS__DEV0_PF3_FLR_INTR_STS__SHIFT                                                     0x3
+#define BIF_PF_FLR_INTR_STS__DEV0_PF4_FLR_INTR_STS__SHIFT                                                     0x4
+#define BIF_PF_FLR_INTR_STS__DEV0_PF5_FLR_INTR_STS__SHIFT                                                     0x5
+#define BIF_PF_FLR_INTR_STS__DEV0_PF6_FLR_INTR_STS__SHIFT                                                     0x6
+#define BIF_PF_FLR_INTR_STS__DEV0_PF7_FLR_INTR_STS__SHIFT                                                     0x7
+#define BIF_PF_FLR_INTR_STS__DEV1_PF0_FLR_INTR_STS__SHIFT                                                     0x8
+#define BIF_PF_FLR_INTR_STS__DEV1_PF1_FLR_INTR_STS__SHIFT                                                     0x9
+#define BIF_PF_FLR_INTR_STS__DEV1_PF2_FLR_INTR_STS__SHIFT                                                     0xa
+#define BIF_PF_FLR_INTR_STS__DEV1_PF3_FLR_INTR_STS__SHIFT                                                     0xb
+#define BIF_PF_FLR_INTR_STS__DEV1_PF4_FLR_INTR_STS__SHIFT                                                     0xc
+#define BIF_PF_FLR_INTR_STS__DEV2_PF0_FLR_INTR_STS__SHIFT                                                     0x10
+#define BIF_PF_FLR_INTR_STS__DEV2_PF1_FLR_INTR_STS__SHIFT                                                     0x11
+#define BIF_PF_FLR_INTR_STS__DEV2_PF2_FLR_INTR_STS__SHIFT                                                     0x12
+#define BIF_PF_FLR_INTR_STS__DEV2_PF3_FLR_INTR_STS__SHIFT                                                     0x13
+#define BIF_PF_FLR_INTR_STS__DEV2_PF4_FLR_INTR_STS__SHIFT                                                     0x14
+#define BIF_PF_FLR_INTR_STS__DEV2_PF5_FLR_INTR_STS__SHIFT                                                     0x15
+#define BIF_PF_FLR_INTR_STS__DEV2_PF6_FLR_INTR_STS__SHIFT                                                     0x16
+#define BIF_PF_FLR_INTR_STS__DEV0_PF0_FLR_INTR_STS_MASK                                                       0x00000001L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF1_FLR_INTR_STS_MASK                                                       0x00000002L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF2_FLR_INTR_STS_MASK                                                       0x00000004L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF3_FLR_INTR_STS_MASK                                                       0x00000008L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF4_FLR_INTR_STS_MASK                                                       0x00000010L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF5_FLR_INTR_STS_MASK                                                       0x00000020L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF6_FLR_INTR_STS_MASK                                                       0x00000040L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF7_FLR_INTR_STS_MASK                                                       0x00000080L
+#define BIF_PF_FLR_INTR_STS__DEV1_PF0_FLR_INTR_STS_MASK                                                       0x00000100L
+#define BIF_PF_FLR_INTR_STS__DEV1_PF1_FLR_INTR_STS_MASK                                                       0x00000200L
+#define BIF_PF_FLR_INTR_STS__DEV1_PF2_FLR_INTR_STS_MASK                                                       0x00000400L
+#define BIF_PF_FLR_INTR_STS__DEV1_PF3_FLR_INTR_STS_MASK                                                       0x00000800L
+#define BIF_PF_FLR_INTR_STS__DEV1_PF4_FLR_INTR_STS_MASK                                                       0x00001000L
+#define BIF_PF_FLR_INTR_STS__DEV2_PF0_FLR_INTR_STS_MASK                                                       0x00010000L
+#define BIF_PF_FLR_INTR_STS__DEV2_PF1_FLR_INTR_STS_MASK                                                       0x00020000L
+#define BIF_PF_FLR_INTR_STS__DEV2_PF2_FLR_INTR_STS_MASK                                                       0x00040000L
+#define BIF_PF_FLR_INTR_STS__DEV2_PF3_FLR_INTR_STS_MASK                                                       0x00080000L
+#define BIF_PF_FLR_INTR_STS__DEV2_PF4_FLR_INTR_STS_MASK                                                       0x00100000L
+#define BIF_PF_FLR_INTR_STS__DEV2_PF5_FLR_INTR_STS_MASK                                                       0x00200000L
+#define BIF_PF_FLR_INTR_STS__DEV2_PF6_FLR_INTR_STS_MASK                                                       0x00400000L
+//BIF_D3HOTD0_INTR_STS
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF0_D3HOTD0_INTR_STS__SHIFT                                                0x0
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF1_D3HOTD0_INTR_STS__SHIFT                                                0x1
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF2_D3HOTD0_INTR_STS__SHIFT                                                0x2
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF3_D3HOTD0_INTR_STS__SHIFT                                                0x3
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF4_D3HOTD0_INTR_STS__SHIFT                                                0x4
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF5_D3HOTD0_INTR_STS__SHIFT                                                0x5
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF6_D3HOTD0_INTR_STS__SHIFT                                                0x6
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF7_D3HOTD0_INTR_STS__SHIFT                                                0x7
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF0_D3HOTD0_INTR_STS__SHIFT                                                0x8
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF1_D3HOTD0_INTR_STS__SHIFT                                                0x9
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF2_D3HOTD0_INTR_STS__SHIFT                                                0xa
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF3_D3HOTD0_INTR_STS__SHIFT                                                0xb
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF4_D3HOTD0_INTR_STS__SHIFT                                                0xc
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF0_D3HOTD0_INTR_STS__SHIFT                                                0x10
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF1_D3HOTD0_INTR_STS__SHIFT                                                0x11
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF2_D3HOTD0_INTR_STS__SHIFT                                                0x12
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF3_D3HOTD0_INTR_STS__SHIFT                                                0x13
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF4_D3HOTD0_INTR_STS__SHIFT                                                0x14
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF5_D3HOTD0_INTR_STS__SHIFT                                                0x15
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF6_D3HOTD0_INTR_STS__SHIFT                                                0x16
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF0_D3HOTD0_INTR_STS_MASK                                                  0x00000001L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF1_D3HOTD0_INTR_STS_MASK                                                  0x00000002L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF2_D3HOTD0_INTR_STS_MASK                                                  0x00000004L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF3_D3HOTD0_INTR_STS_MASK                                                  0x00000008L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF4_D3HOTD0_INTR_STS_MASK                                                  0x00000010L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF5_D3HOTD0_INTR_STS_MASK                                                  0x00000020L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF6_D3HOTD0_INTR_STS_MASK                                                  0x00000040L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF7_D3HOTD0_INTR_STS_MASK                                                  0x00000080L
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF0_D3HOTD0_INTR_STS_MASK                                                  0x00000100L
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF1_D3HOTD0_INTR_STS_MASK                                                  0x00000200L
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF2_D3HOTD0_INTR_STS_MASK                                                  0x00000400L
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF3_D3HOTD0_INTR_STS_MASK                                                  0x00000800L
+#define BIF_D3HOTD0_INTR_STS__DEV1_PF4_D3HOTD0_INTR_STS_MASK                                                  0x00001000L
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF0_D3HOTD0_INTR_STS_MASK                                                  0x00010000L
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF1_D3HOTD0_INTR_STS_MASK                                                  0x00020000L
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF2_D3HOTD0_INTR_STS_MASK                                                  0x00040000L
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF3_D3HOTD0_INTR_STS_MASK                                                  0x00080000L
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF4_D3HOTD0_INTR_STS_MASK                                                  0x00100000L
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF5_D3HOTD0_INTR_STS_MASK                                                  0x00200000L
+#define BIF_D3HOTD0_INTR_STS__DEV2_PF6_D3HOTD0_INTR_STS_MASK                                                  0x00400000L
+//BIF_POWER_INTR_STS
+#define BIF_POWER_INTR_STS__DEV0_PME_TURN_OFF_INTR_STS__SHIFT                                                 0x0
+#define BIF_POWER_INTR_STS__DEV1_PME_TURN_OFF_INTR_STS__SHIFT                                                 0x1
+#define BIF_POWER_INTR_STS__DEV2_PME_TURN_OFF_INTR_STS__SHIFT                                                 0x2
+#define BIF_POWER_INTR_STS__PORT0_DSTATE_INTR_STS__SHIFT                                                      0x10
+#define BIF_POWER_INTR_STS__PORT1_DSTATE_INTR_STS__SHIFT                                                      0x11
+#define BIF_POWER_INTR_STS__PORT2_DSTATE_INTR_STS__SHIFT                                                      0x12
+#define BIF_POWER_INTR_STS__DEV0_PME_TURN_OFF_INTR_STS_MASK                                                   0x00000001L
+#define BIF_POWER_INTR_STS__DEV1_PME_TURN_OFF_INTR_STS_MASK                                                   0x00000002L
+#define BIF_POWER_INTR_STS__DEV2_PME_TURN_OFF_INTR_STS_MASK                                                   0x00000004L
+#define BIF_POWER_INTR_STS__PORT0_DSTATE_INTR_STS_MASK                                                        0x00010000L
+#define BIF_POWER_INTR_STS__PORT1_DSTATE_INTR_STS_MASK                                                        0x00020000L
+#define BIF_POWER_INTR_STS__PORT2_DSTATE_INTR_STS_MASK                                                        0x00040000L
+//BIF_PF_DSTATE_INTR_STS
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF0_DSTATE_INTR_STS__SHIFT                                               0x0
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF1_DSTATE_INTR_STS__SHIFT                                               0x1
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF2_DSTATE_INTR_STS__SHIFT                                               0x2
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF3_DSTATE_INTR_STS__SHIFT                                               0x3
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF4_DSTATE_INTR_STS__SHIFT                                               0x4
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF5_DSTATE_INTR_STS__SHIFT                                               0x5
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF6_DSTATE_INTR_STS__SHIFT                                               0x6
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF7_DSTATE_INTR_STS__SHIFT                                               0x7
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF0_DSTATE_INTR_STS__SHIFT                                               0x8
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF1_DSTATE_INTR_STS__SHIFT                                               0x9
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF2_DSTATE_INTR_STS__SHIFT                                               0xa
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF3_DSTATE_INTR_STS__SHIFT                                               0xb
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF4_DSTATE_INTR_STS__SHIFT                                               0xc
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF0_DSTATE_INTR_STS__SHIFT                                               0x10
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF1_DSTATE_INTR_STS__SHIFT                                               0x11
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF2_DSTATE_INTR_STS__SHIFT                                               0x12
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF3_DSTATE_INTR_STS__SHIFT                                               0x13
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF4_DSTATE_INTR_STS__SHIFT                                               0x14
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF5_DSTATE_INTR_STS__SHIFT                                               0x15
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF6_DSTATE_INTR_STS__SHIFT                                               0x16
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF0_DSTATE_INTR_STS_MASK                                                 0x00000001L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF1_DSTATE_INTR_STS_MASK                                                 0x00000002L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF2_DSTATE_INTR_STS_MASK                                                 0x00000004L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF3_DSTATE_INTR_STS_MASK                                                 0x00000008L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF4_DSTATE_INTR_STS_MASK                                                 0x00000010L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF5_DSTATE_INTR_STS_MASK                                                 0x00000020L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF6_DSTATE_INTR_STS_MASK                                                 0x00000040L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF7_DSTATE_INTR_STS_MASK                                                 0x00000080L
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF0_DSTATE_INTR_STS_MASK                                                 0x00000100L
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF1_DSTATE_INTR_STS_MASK                                                 0x00000200L
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF2_DSTATE_INTR_STS_MASK                                                 0x00000400L
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF3_DSTATE_INTR_STS_MASK                                                 0x00000800L
+#define BIF_PF_DSTATE_INTR_STS__DEV1_PF4_DSTATE_INTR_STS_MASK                                                 0x00001000L
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF0_DSTATE_INTR_STS_MASK                                                 0x00010000L
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF1_DSTATE_INTR_STS_MASK                                                 0x00020000L
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF2_DSTATE_INTR_STS_MASK                                                 0x00040000L
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF3_DSTATE_INTR_STS_MASK                                                 0x00080000L
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF4_DSTATE_INTR_STS_MASK                                                 0x00100000L
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF5_DSTATE_INTR_STS_MASK                                                 0x00200000L
+#define BIF_PF_DSTATE_INTR_STS__DEV2_PF6_DSTATE_INTR_STS_MASK                                                 0x00400000L
+//SELF_SOFT_RST_2
+#define SELF_SOFT_RST_2__DSPT3_CFG_RST__SHIFT                                                                 0x0
+#define SELF_SOFT_RST_2__DSPT3_CFG_STICKY_RST__SHIFT                                                          0x1
+#define SELF_SOFT_RST_2__DSPT3_PRV_RST__SHIFT                                                                 0x2
+#define SELF_SOFT_RST_2__DSPT3_PRV_STICKY_RST__SHIFT                                                          0x3
+#define SELF_SOFT_RST_2__EP3_CFG_RST__SHIFT                                                                   0x4
+#define SELF_SOFT_RST_2__EP3_CFG_STICKY_RST__SHIFT                                                            0x5
+#define SELF_SOFT_RST_2__EP3_PRV_RST__SHIFT                                                                   0x6
+#define SELF_SOFT_RST_2__EP3_PRV_STICKY_RST__SHIFT                                                            0x7
+#define SELF_SOFT_RST_2__GMISP0_SDP_PORT_RST__SHIFT                                                           0x18
+#define SELF_SOFT_RST_2__STRAP_RST__SHIFT                                                                     0x19
+#define SELF_SOFT_RST_2__SWUS_SHADOW_PRV_RST__SHIFT                                                           0x1a
+#define SELF_SOFT_RST_2__DSPT3_CFG_RST_MASK                                                                   0x00000001L
+#define SELF_SOFT_RST_2__DSPT3_CFG_STICKY_RST_MASK                                                            0x00000002L
+#define SELF_SOFT_RST_2__DSPT3_PRV_RST_MASK                                                                   0x00000004L
+#define SELF_SOFT_RST_2__DSPT3_PRV_STICKY_RST_MASK                                                            0x00000008L
+#define SELF_SOFT_RST_2__EP3_CFG_RST_MASK                                                                     0x00000010L
+#define SELF_SOFT_RST_2__EP3_CFG_STICKY_RST_MASK                                                              0x00000020L
+#define SELF_SOFT_RST_2__EP3_PRV_RST_MASK                                                                     0x00000040L
+#define SELF_SOFT_RST_2__EP3_PRV_STICKY_RST_MASK                                                              0x00000080L
+#define SELF_SOFT_RST_2__GMISP0_SDP_PORT_RST_MASK                                                             0x01000000L
+#define SELF_SOFT_RST_2__STRAP_RST_MASK                                                                       0x02000000L
+#define SELF_SOFT_RST_2__SWUS_SHADOW_PRV_RST_MASK                                                             0x04000000L
+//BIF_INST_RESET_INTR_MASK
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_INTR_MASK__SHIFT                                             0x0
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_CFG_ONLY_INTR_MASK__SHIFT                                    0x1
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M0_INTR_MASK__SHIFT                                               0x2
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M1_INTR_MASK__SHIFT                                               0x3
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M2_INTR_MASK__SHIFT                                               0x4
+#define BIF_INST_RESET_INTR_MASK__EP1_LINK_RESET_INTR_MASK__SHIFT                                             0x8
+#define BIF_INST_RESET_INTR_MASK__EP1_LINK_RESET_CFG_ONLY_INTR_MASK__SHIFT                                    0x9
+#define BIF_INST_RESET_INTR_MASK__EP2_LINK_RESET_INTR_MASK__SHIFT                                             0xa
+#define BIF_INST_RESET_INTR_MASK__EP2_LINK_RESET_CFG_ONLY_INTR_MASK__SHIFT                                    0xb
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_INTR_MASK_MASK                                               0x00000001L
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_CFG_ONLY_INTR_MASK_MASK                                      0x00000002L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M0_INTR_MASK_MASK                                                 0x00000004L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M1_INTR_MASK_MASK                                                 0x00000008L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M2_INTR_MASK_MASK                                                 0x00000010L
+#define BIF_INST_RESET_INTR_MASK__EP1_LINK_RESET_INTR_MASK_MASK                                               0x00000100L
+#define BIF_INST_RESET_INTR_MASK__EP1_LINK_RESET_CFG_ONLY_INTR_MASK_MASK                                      0x00000200L
+#define BIF_INST_RESET_INTR_MASK__EP2_LINK_RESET_INTR_MASK_MASK                                               0x00000400L
+#define BIF_INST_RESET_INTR_MASK__EP2_LINK_RESET_CFG_ONLY_INTR_MASK_MASK                                      0x00000800L
+//BIF_PF_FLR_INTR_MASK
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF0_FLR_INTR_MASK__SHIFT                                                   0x0
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF1_FLR_INTR_MASK__SHIFT                                                   0x1
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF2_FLR_INTR_MASK__SHIFT                                                   0x2
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF3_FLR_INTR_MASK__SHIFT                                                   0x3
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF4_FLR_INTR_MASK__SHIFT                                                   0x4
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF5_FLR_INTR_MASK__SHIFT                                                   0x5
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF6_FLR_INTR_MASK__SHIFT                                                   0x6
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF7_FLR_INTR_MASK__SHIFT                                                   0x7
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF0_FLR_INTR_MASK__SHIFT                                                   0x8
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF1_FLR_INTR_MASK__SHIFT                                                   0x9
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF2_FLR_INTR_MASK__SHIFT                                                   0xa
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF3_FLR_INTR_MASK__SHIFT                                                   0xb
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF4_FLR_INTR_MASK__SHIFT                                                   0xc
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF0_FLR_INTR_MASK__SHIFT                                                   0x10
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF1_FLR_INTR_MASK__SHIFT                                                   0x11
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF2_FLR_INTR_MASK__SHIFT                                                   0x12
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF3_FLR_INTR_MASK__SHIFT                                                   0x13
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF4_FLR_INTR_MASK__SHIFT                                                   0x14
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF5_FLR_INTR_MASK__SHIFT                                                   0x15
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF6_FLR_INTR_MASK__SHIFT                                                   0x16
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF0_FLR_INTR_MASK_MASK                                                     0x00000001L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF1_FLR_INTR_MASK_MASK                                                     0x00000002L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF2_FLR_INTR_MASK_MASK                                                     0x00000004L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF3_FLR_INTR_MASK_MASK                                                     0x00000008L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF4_FLR_INTR_MASK_MASK                                                     0x00000010L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF5_FLR_INTR_MASK_MASK                                                     0x00000020L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF6_FLR_INTR_MASK_MASK                                                     0x00000040L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF7_FLR_INTR_MASK_MASK                                                     0x00000080L
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF0_FLR_INTR_MASK_MASK                                                     0x00000100L
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF1_FLR_INTR_MASK_MASK                                                     0x00000200L
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF2_FLR_INTR_MASK_MASK                                                     0x00000400L
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF3_FLR_INTR_MASK_MASK                                                     0x00000800L
+#define BIF_PF_FLR_INTR_MASK__DEV1_PF4_FLR_INTR_MASK_MASK                                                     0x00001000L
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF0_FLR_INTR_MASK_MASK                                                     0x00010000L
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF1_FLR_INTR_MASK_MASK                                                     0x00020000L
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF2_FLR_INTR_MASK_MASK                                                     0x00040000L
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF3_FLR_INTR_MASK_MASK                                                     0x00080000L
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF4_FLR_INTR_MASK_MASK                                                     0x00100000L
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF5_FLR_INTR_MASK_MASK                                                     0x00200000L
+#define BIF_PF_FLR_INTR_MASK__DEV2_PF6_FLR_INTR_MASK_MASK                                                     0x00400000L
+//BIF_D3HOTD0_INTR_MASK
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF0_D3HOTD0_INTR_MASK__SHIFT                                              0x0
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF1_D3HOTD0_INTR_MASK__SHIFT                                              0x1
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF2_D3HOTD0_INTR_MASK__SHIFT                                              0x2
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF3_D3HOTD0_INTR_MASK__SHIFT                                              0x3
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF4_D3HOTD0_INTR_MASK__SHIFT                                              0x4
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF5_D3HOTD0_INTR_MASK__SHIFT                                              0x5
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF6_D3HOTD0_INTR_MASK__SHIFT                                              0x6
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF7_D3HOTD0_INTR_MASK__SHIFT                                              0x7
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF0_D3HOTD0_INTR_MASK__SHIFT                                              0x8
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF1_D3HOTD0_INTR_MASK__SHIFT                                              0x9
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF2_D3HOTD0_INTR_MASK__SHIFT                                              0xa
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF3_D3HOTD0_INTR_MASK__SHIFT                                              0xb
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF4_D3HOTD0_INTR_MASK__SHIFT                                              0xc
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF0_D3HOTD0_INTR_MASK__SHIFT                                              0x10
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF1_D3HOTD0_INTR_MASK__SHIFT                                              0x11
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF2_D3HOTD0_INTR_MASK__SHIFT                                              0x12
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF3_D3HOTD0_INTR_MASK__SHIFT                                              0x13
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF4_D3HOTD0_INTR_MASK__SHIFT                                              0x14
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF5_D3HOTD0_INTR_MASK__SHIFT                                              0x15
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF6_D3HOTD0_INTR_MASK__SHIFT                                              0x16
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF0_D3HOTD0_INTR_MASK_MASK                                                0x00000001L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF1_D3HOTD0_INTR_MASK_MASK                                                0x00000002L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF2_D3HOTD0_INTR_MASK_MASK                                                0x00000004L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF3_D3HOTD0_INTR_MASK_MASK                                                0x00000008L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF4_D3HOTD0_INTR_MASK_MASK                                                0x00000010L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF5_D3HOTD0_INTR_MASK_MASK                                                0x00000020L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF6_D3HOTD0_INTR_MASK_MASK                                                0x00000040L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF7_D3HOTD0_INTR_MASK_MASK                                                0x00000080L
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF0_D3HOTD0_INTR_MASK_MASK                                                0x00000100L
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF1_D3HOTD0_INTR_MASK_MASK                                                0x00000200L
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF2_D3HOTD0_INTR_MASK_MASK                                                0x00000400L
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF3_D3HOTD0_INTR_MASK_MASK                                                0x00000800L
+#define BIF_D3HOTD0_INTR_MASK__DEV1_PF4_D3HOTD0_INTR_MASK_MASK                                                0x00001000L
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF0_D3HOTD0_INTR_MASK_MASK                                                0x00010000L
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF1_D3HOTD0_INTR_MASK_MASK                                                0x00020000L
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF2_D3HOTD0_INTR_MASK_MASK                                                0x00040000L
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF3_D3HOTD0_INTR_MASK_MASK                                                0x00080000L
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF4_D3HOTD0_INTR_MASK_MASK                                                0x00100000L
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF5_D3HOTD0_INTR_MASK_MASK                                                0x00200000L
+#define BIF_D3HOTD0_INTR_MASK__DEV2_PF6_D3HOTD0_INTR_MASK_MASK                                                0x00400000L
+//BIF_POWER_INTR_MASK
+#define BIF_POWER_INTR_MASK__DEV0_PME_TURN_OFF_INTR_MASK__SHIFT                                               0x0
+#define BIF_POWER_INTR_MASK__DEV1_PME_TURN_OFF_INTR_MASK__SHIFT                                               0x1
+#define BIF_POWER_INTR_MASK__DEV2_PME_TURN_OFF_INTR_MASK__SHIFT                                               0x2
+#define BIF_POWER_INTR_MASK__PORT0_DSTATE_INTR_MASK__SHIFT                                                    0x10
+#define BIF_POWER_INTR_MASK__PORT1_DSTATE_INTR_MASK__SHIFT                                                    0x11
+#define BIF_POWER_INTR_MASK__PORT2_DSTATE_INTR_MASK__SHIFT                                                    0x12
+#define BIF_POWER_INTR_MASK__DEV0_PME_TURN_OFF_INTR_MASK_MASK                                                 0x00000001L
+#define BIF_POWER_INTR_MASK__DEV1_PME_TURN_OFF_INTR_MASK_MASK                                                 0x00000002L
+#define BIF_POWER_INTR_MASK__DEV2_PME_TURN_OFF_INTR_MASK_MASK                                                 0x00000004L
+#define BIF_POWER_INTR_MASK__PORT0_DSTATE_INTR_MASK_MASK                                                      0x00010000L
+#define BIF_POWER_INTR_MASK__PORT1_DSTATE_INTR_MASK_MASK                                                      0x00020000L
+#define BIF_POWER_INTR_MASK__PORT2_DSTATE_INTR_MASK_MASK                                                      0x00040000L
+//BIF_PF_DSTATE_INTR_MASK
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF0_DSTATE_INTR_MASK__SHIFT                                             0x0
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF1_DSTATE_INTR_MASK__SHIFT                                             0x1
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF2_DSTATE_INTR_MASK__SHIFT                                             0x2
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF3_DSTATE_INTR_MASK__SHIFT                                             0x3
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF4_DSTATE_INTR_MASK__SHIFT                                             0x4
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF5_DSTATE_INTR_MASK__SHIFT                                             0x5
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF6_DSTATE_INTR_MASK__SHIFT                                             0x6
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF7_DSTATE_INTR_MASK__SHIFT                                             0x7
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF0_DSTATE_INTR_MASK__SHIFT                                             0x8
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF1_DSTATE_INTR_MASK__SHIFT                                             0x9
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF2_DSTATE_INTR_MASK__SHIFT                                             0xa
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF3_DSTATE_INTR_MASK__SHIFT                                             0xb
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF4_DSTATE_INTR_MASK__SHIFT                                             0xc
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF5_DSTATE_INTR_MASK__SHIFT                                             0xd
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF6_DSTATE_INTR_MASK__SHIFT                                             0xe
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF7_DSTATE_INTR_MASK__SHIFT                                             0xf
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF0_DSTATE_INTR_MASK__SHIFT                                             0x10
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF1_DSTATE_INTR_MASK__SHIFT                                             0x11
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF2_DSTATE_INTR_MASK__SHIFT                                             0x12
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF3_DSTATE_INTR_MASK__SHIFT                                             0x13
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF4_DSTATE_INTR_MASK__SHIFT                                             0x14
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF5_DSTATE_INTR_MASK__SHIFT                                             0x15
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF6_DSTATE_INTR_MASK__SHIFT                                             0x16
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF7_DSTATE_INTR_MASK__SHIFT                                             0x17
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF0_DSTATE_INTR_MASK_MASK                                               0x00000001L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF1_DSTATE_INTR_MASK_MASK                                               0x00000002L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF2_DSTATE_INTR_MASK_MASK                                               0x00000004L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF3_DSTATE_INTR_MASK_MASK                                               0x00000008L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF4_DSTATE_INTR_MASK_MASK                                               0x00000010L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF5_DSTATE_INTR_MASK_MASK                                               0x00000020L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF6_DSTATE_INTR_MASK_MASK                                               0x00000040L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF7_DSTATE_INTR_MASK_MASK                                               0x00000080L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF0_DSTATE_INTR_MASK_MASK                                               0x00000100L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF1_DSTATE_INTR_MASK_MASK                                               0x00000200L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF2_DSTATE_INTR_MASK_MASK                                               0x00000400L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF3_DSTATE_INTR_MASK_MASK                                               0x00000800L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF4_DSTATE_INTR_MASK_MASK                                               0x00001000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF5_DSTATE_INTR_MASK_MASK                                               0x00002000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF6_DSTATE_INTR_MASK_MASK                                               0x00004000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV1_PF7_DSTATE_INTR_MASK_MASK                                               0x00008000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF0_DSTATE_INTR_MASK_MASK                                               0x00010000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF1_DSTATE_INTR_MASK_MASK                                               0x00020000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF2_DSTATE_INTR_MASK_MASK                                               0x00040000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF3_DSTATE_INTR_MASK_MASK                                               0x00080000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF4_DSTATE_INTR_MASK_MASK                                               0x00100000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF5_DSTATE_INTR_MASK_MASK                                               0x00200000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF6_DSTATE_INTR_MASK_MASK                                               0x00400000L
+#define BIF_PF_DSTATE_INTR_MASK__DEV2_PF7_DSTATE_INTR_MASK_MASK                                               0x00800000L
+//BIF_PF_FLR_RST
+#define BIF_PF_FLR_RST__DEV0_PF0_FLR_RST__SHIFT                                                               0x0
+#define BIF_PF_FLR_RST__DEV0_PF1_FLR_RST__SHIFT                                                               0x1
+#define BIF_PF_FLR_RST__DEV0_PF2_FLR_RST__SHIFT                                                               0x2
+#define BIF_PF_FLR_RST__DEV0_PF3_FLR_RST__SHIFT                                                               0x3
+#define BIF_PF_FLR_RST__DEV0_PF4_FLR_RST__SHIFT                                                               0x4
+#define BIF_PF_FLR_RST__DEV0_PF5_FLR_RST__SHIFT                                                               0x5
+#define BIF_PF_FLR_RST__DEV0_PF6_FLR_RST__SHIFT                                                               0x6
+#define BIF_PF_FLR_RST__DEV0_PF7_FLR_RST__SHIFT                                                               0x7
+#define BIF_PF_FLR_RST__DEV1_PF0_FLR_RST__SHIFT                                                               0x8
+#define BIF_PF_FLR_RST__DEV1_PF1_FLR_RST__SHIFT                                                               0x9
+#define BIF_PF_FLR_RST__DEV1_PF2_FLR_RST__SHIFT                                                               0xa
+#define BIF_PF_FLR_RST__DEV1_PF3_FLR_RST__SHIFT                                                               0xb
+#define BIF_PF_FLR_RST__DEV1_PF4_FLR_RST__SHIFT                                                               0xc
+#define BIF_PF_FLR_RST__DEV2_PF0_FLR_RST__SHIFT                                                               0x10
+#define BIF_PF_FLR_RST__DEV2_PF1_FLR_RST__SHIFT                                                               0x11
+#define BIF_PF_FLR_RST__DEV2_PF2_FLR_RST__SHIFT                                                               0x12
+#define BIF_PF_FLR_RST__DEV2_PF3_FLR_RST__SHIFT                                                               0x13
+#define BIF_PF_FLR_RST__DEV2_PF4_FLR_RST__SHIFT                                                               0x14
+#define BIF_PF_FLR_RST__DEV2_PF5_FLR_RST__SHIFT                                                               0x15
+#define BIF_PF_FLR_RST__DEV2_PF6_FLR_RST__SHIFT                                                               0x16
+#define BIF_PF_FLR_RST__DEV0_PF0_FLR_RST_MASK                                                                 0x00000001L
+#define BIF_PF_FLR_RST__DEV0_PF1_FLR_RST_MASK                                                                 0x00000002L
+#define BIF_PF_FLR_RST__DEV0_PF2_FLR_RST_MASK                                                                 0x00000004L
+#define BIF_PF_FLR_RST__DEV0_PF3_FLR_RST_MASK                                                                 0x00000008L
+#define BIF_PF_FLR_RST__DEV0_PF4_FLR_RST_MASK                                                                 0x00000010L
+#define BIF_PF_FLR_RST__DEV0_PF5_FLR_RST_MASK                                                                 0x00000020L
+#define BIF_PF_FLR_RST__DEV0_PF6_FLR_RST_MASK                                                                 0x00000040L
+#define BIF_PF_FLR_RST__DEV0_PF7_FLR_RST_MASK                                                                 0x00000080L
+#define BIF_PF_FLR_RST__DEV1_PF0_FLR_RST_MASK                                                                 0x00000100L
+#define BIF_PF_FLR_RST__DEV1_PF1_FLR_RST_MASK                                                                 0x00000200L
+#define BIF_PF_FLR_RST__DEV1_PF2_FLR_RST_MASK                                                                 0x00000400L
+#define BIF_PF_FLR_RST__DEV1_PF3_FLR_RST_MASK                                                                 0x00000800L
+#define BIF_PF_FLR_RST__DEV1_PF4_FLR_RST_MASK                                                                 0x00001000L
+#define BIF_PF_FLR_RST__DEV2_PF0_FLR_RST_MASK                                                                 0x00010000L
+#define BIF_PF_FLR_RST__DEV2_PF1_FLR_RST_MASK                                                                 0x00020000L
+#define BIF_PF_FLR_RST__DEV2_PF2_FLR_RST_MASK                                                                 0x00040000L
+#define BIF_PF_FLR_RST__DEV2_PF3_FLR_RST_MASK                                                                 0x00080000L
+#define BIF_PF_FLR_RST__DEV2_PF4_FLR_RST_MASK                                                                 0x00100000L
+#define BIF_PF_FLR_RST__DEV2_PF5_FLR_RST_MASK                                                                 0x00200000L
+#define BIF_PF_FLR_RST__DEV2_PF6_FLR_RST_MASK                                                                 0x00400000L
+//BIF_DEV0_PF0_DSTATE_VALUE
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF1_DSTATE_VALUE
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF2_DSTATE_VALUE
+#define BIF_DEV0_PF2_DSTATE_VALUE__DEV0_PF2_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF2_DSTATE_VALUE__DEV0_PF2_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF2_DSTATE_VALUE__DEV0_PF2_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF2_DSTATE_VALUE__DEV0_PF2_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF2_DSTATE_VALUE__DEV0_PF2_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF2_DSTATE_VALUE__DEV0_PF2_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF3_DSTATE_VALUE
+#define BIF_DEV0_PF3_DSTATE_VALUE__DEV0_PF3_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF3_DSTATE_VALUE__DEV0_PF3_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF3_DSTATE_VALUE__DEV0_PF3_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF3_DSTATE_VALUE__DEV0_PF3_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF3_DSTATE_VALUE__DEV0_PF3_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF3_DSTATE_VALUE__DEV0_PF3_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF4_DSTATE_VALUE
+#define BIF_DEV0_PF4_DSTATE_VALUE__DEV0_PF4_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF4_DSTATE_VALUE__DEV0_PF4_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF4_DSTATE_VALUE__DEV0_PF4_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF4_DSTATE_VALUE__DEV0_PF4_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF4_DSTATE_VALUE__DEV0_PF4_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF4_DSTATE_VALUE__DEV0_PF4_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF5_DSTATE_VALUE
+#define BIF_DEV0_PF5_DSTATE_VALUE__DEV0_PF5_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF5_DSTATE_VALUE__DEV0_PF5_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF5_DSTATE_VALUE__DEV0_PF5_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF5_DSTATE_VALUE__DEV0_PF5_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF5_DSTATE_VALUE__DEV0_PF5_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF5_DSTATE_VALUE__DEV0_PF5_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF6_DSTATE_VALUE
+#define BIF_DEV0_PF6_DSTATE_VALUE__DEV0_PF6_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF6_DSTATE_VALUE__DEV0_PF6_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF6_DSTATE_VALUE__DEV0_PF6_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF6_DSTATE_VALUE__DEV0_PF6_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF6_DSTATE_VALUE__DEV0_PF6_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF6_DSTATE_VALUE__DEV0_PF6_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF7_DSTATE_VALUE
+#define BIF_DEV0_PF7_DSTATE_VALUE__DEV0_PF7_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF7_DSTATE_VALUE__DEV0_PF7_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF7_DSTATE_VALUE__DEV0_PF7_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF7_DSTATE_VALUE__DEV0_PF7_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF7_DSTATE_VALUE__DEV0_PF7_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF7_DSTATE_VALUE__DEV0_PF7_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//DEV0_PF0_D3HOTD0_RST_CTRL
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF1_D3HOTD0_RST_CTRL
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF2_D3HOTD0_RST_CTRL
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF2_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF3_D3HOTD0_RST_CTRL
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF3_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF4_D3HOTD0_RST_CTRL
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF4_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF5_D3HOTD0_RST_CTRL
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF5_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF6_D3HOTD0_RST_CTRL
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF6_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF7_D3HOTD0_RST_CTRL
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF7_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV1_PF0_FLR_RST_CTRL
+#define DEV1_PF0_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV1_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV1_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV1_PF0_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV1_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV1_PF0_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV1_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV1_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV1_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV1_PF0_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV1_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV1_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV1_PF0_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV1_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV1_PF0_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV1_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV1_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV1_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV1_PF1_FLR_RST_CTRL
+#define DEV1_PF1_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV1_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV1_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV1_PF1_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV1_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_CFG_EN__SHIFT                                         0x5
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_CFG_STICKY_EN__SHIFT                                  0x6
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_PRV_EN__SHIFT                                         0x7
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_VF_CFG_EN__SHIFT                                      0xd
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN__SHIFT                               0xe
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_VF_PRV_EN__SHIFT                                      0xf
+#define DEV1_PF1_FLR_RST_CTRL__FLR_TWICE_EN__SHIFT                                                            0x10
+#define DEV1_PF1_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV1_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV1_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV1_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV1_PF1_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV1_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV1_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV1_PF1_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV1_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_CFG_EN_MASK                                           0x00000020L
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_CFG_STICKY_EN_MASK                                    0x00000040L
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_PRV_EN_MASK                                           0x00000080L
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_VF_CFG_EN_MASK                                        0x00002000L
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN_MASK                                 0x00004000L
+#define DEV1_PF1_FLR_RST_CTRL__DEV1_PF1_FLR_RST_CTRL_VF_VF_PRV_EN_MASK                                        0x00008000L
+#define DEV1_PF1_FLR_RST_CTRL__FLR_TWICE_EN_MASK                                                              0x00010000L
+#define DEV1_PF1_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV1_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV1_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV1_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV1_PF2_FLR_RST_CTRL
+#define DEV1_PF2_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV1_PF2_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV1_PF2_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV1_PF2_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV1_PF2_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_CFG_EN__SHIFT                                         0x5
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_CFG_STICKY_EN__SHIFT                                  0x6
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_PRV_EN__SHIFT                                         0x7
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_VF_CFG_EN__SHIFT                                      0xd
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN__SHIFT                               0xe
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_VF_PRV_EN__SHIFT                                      0xf
+#define DEV1_PF2_FLR_RST_CTRL__FLR_TWICE_EN__SHIFT                                                            0x10
+#define DEV1_PF2_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV1_PF2_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV1_PF2_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV1_PF2_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV1_PF2_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV1_PF2_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV1_PF2_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV1_PF2_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV1_PF2_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_CFG_EN_MASK                                           0x00000020L
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_CFG_STICKY_EN_MASK                                    0x00000040L
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_PRV_EN_MASK                                           0x00000080L
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_VF_CFG_EN_MASK                                        0x00002000L
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN_MASK                                 0x00004000L
+#define DEV1_PF2_FLR_RST_CTRL__DEV1_PF2_FLR_RST_CTRL_VF_VF_PRV_EN_MASK                                        0x00008000L
+#define DEV1_PF2_FLR_RST_CTRL__FLR_TWICE_EN_MASK                                                              0x00010000L
+#define DEV1_PF2_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV1_PF2_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV1_PF2_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV1_PF2_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV1_PF3_FLR_RST_CTRL
+#define DEV1_PF3_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV1_PF3_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV1_PF3_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV1_PF3_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV1_PF3_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV1_PF3_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV1_PF3_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV1_PF3_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV1_PF3_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV1_PF3_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV1_PF3_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV1_PF3_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV1_PF3_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV1_PF3_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV1_PF3_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV1_PF3_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV1_PF3_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV1_PF3_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV1_PF4_FLR_RST_CTRL
+#define DEV1_PF4_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV1_PF4_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV1_PF4_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV1_PF4_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV1_PF4_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV1_PF4_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV1_PF4_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV1_PF4_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV1_PF4_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV1_PF4_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV1_PF4_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV1_PF4_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV1_PF4_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV1_PF4_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV1_PF4_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV1_PF4_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV1_PF4_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV1_PF4_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//BIF_DEV1_PF0_DSTATE_VALUE
+#define BIF_DEV1_PF0_DSTATE_VALUE__DEV1_PF0_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV1_PF0_DSTATE_VALUE__DEV1_PF0_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV1_PF0_DSTATE_VALUE__DEV1_PF0_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV1_PF0_DSTATE_VALUE__DEV1_PF0_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV1_PF0_DSTATE_VALUE__DEV1_PF0_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV1_PF0_DSTATE_VALUE__DEV1_PF0_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV1_PF1_DSTATE_VALUE
+#define BIF_DEV1_PF1_DSTATE_VALUE__DEV1_PF1_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV1_PF1_DSTATE_VALUE__DEV1_PF1_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV1_PF1_DSTATE_VALUE__DEV1_PF1_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV1_PF1_DSTATE_VALUE__DEV1_PF1_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV1_PF1_DSTATE_VALUE__DEV1_PF1_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV1_PF1_DSTATE_VALUE__DEV1_PF1_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV1_PF2_DSTATE_VALUE
+#define BIF_DEV1_PF2_DSTATE_VALUE__DEV1_PF2_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV1_PF2_DSTATE_VALUE__DEV1_PF2_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV1_PF2_DSTATE_VALUE__DEV1_PF2_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV1_PF2_DSTATE_VALUE__DEV1_PF2_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV1_PF2_DSTATE_VALUE__DEV1_PF2_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV1_PF2_DSTATE_VALUE__DEV1_PF2_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV1_PF3_DSTATE_VALUE
+#define BIF_DEV1_PF3_DSTATE_VALUE__DEV1_PF3_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV1_PF3_DSTATE_VALUE__DEV1_PF3_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV1_PF3_DSTATE_VALUE__DEV1_PF3_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV1_PF3_DSTATE_VALUE__DEV1_PF3_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV1_PF3_DSTATE_VALUE__DEV1_PF3_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV1_PF3_DSTATE_VALUE__DEV1_PF3_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV1_PF4_DSTATE_VALUE
+#define BIF_DEV1_PF4_DSTATE_VALUE__DEV1_PF4_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV1_PF4_DSTATE_VALUE__DEV1_PF4_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV1_PF4_DSTATE_VALUE__DEV1_PF4_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV1_PF4_DSTATE_VALUE__DEV1_PF4_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV1_PF4_DSTATE_VALUE__DEV1_PF4_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV1_PF4_DSTATE_VALUE__DEV1_PF4_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//DEV1_PF0_D3HOTD0_RST_CTRL
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV1_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV1_PF1_D3HOTD0_RST_CTRL
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV1_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV1_PF2_D3HOTD0_RST_CTRL
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV1_PF2_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV1_PF3_D3HOTD0_RST_CTRL
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV1_PF3_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV1_PF4_D3HOTD0_RST_CTRL
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV1_PF4_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV2_PF0_FLR_RST_CTRL
+#define DEV2_PF0_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV2_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV2_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV2_PF0_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV2_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV2_PF0_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV2_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV2_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV2_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV2_PF0_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV2_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV2_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV2_PF0_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV2_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV2_PF0_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV2_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV2_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV2_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV2_PF1_FLR_RST_CTRL
+#define DEV2_PF1_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV2_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV2_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV2_PF1_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV2_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV2_PF1_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV2_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV2_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV2_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV2_PF1_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV2_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV2_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV2_PF1_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV2_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV2_PF1_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV2_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV2_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV2_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV2_PF2_FLR_RST_CTRL
+#define DEV2_PF2_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV2_PF2_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV2_PF2_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV2_PF2_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV2_PF2_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV2_PF2_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV2_PF2_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV2_PF2_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV2_PF2_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV2_PF2_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV2_PF2_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV2_PF2_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV2_PF2_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV2_PF2_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV2_PF2_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV2_PF2_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV2_PF2_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV2_PF2_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV2_PF3_FLR_RST_CTRL
+#define DEV2_PF3_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV2_PF3_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV2_PF3_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV2_PF3_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV2_PF3_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV2_PF3_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV2_PF3_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV2_PF3_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV2_PF3_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV2_PF3_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV2_PF3_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV2_PF3_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV2_PF3_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV2_PF3_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV2_PF3_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV2_PF3_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV2_PF3_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV2_PF3_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV2_PF4_FLR_RST_CTRL
+#define DEV2_PF4_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV2_PF4_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV2_PF4_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV2_PF4_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV2_PF4_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV2_PF4_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV2_PF4_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV2_PF4_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV2_PF4_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV2_PF4_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV2_PF4_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV2_PF4_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV2_PF4_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV2_PF4_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV2_PF4_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV2_PF4_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV2_PF4_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV2_PF4_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV2_PF5_FLR_RST_CTRL
+#define DEV2_PF5_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV2_PF5_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV2_PF5_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV2_PF5_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV2_PF5_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV2_PF5_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV2_PF5_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV2_PF5_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV2_PF5_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV2_PF5_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV2_PF5_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV2_PF5_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV2_PF5_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV2_PF5_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV2_PF5_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV2_PF5_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV2_PF5_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV2_PF5_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//DEV2_PF6_FLR_RST_CTRL
+#define DEV2_PF6_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV2_PF6_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV2_PF6_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV2_PF6_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV2_PF6_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV2_PF6_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV2_PF6_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV2_PF6_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV2_PF6_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV2_PF6_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV2_PF6_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV2_PF6_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV2_PF6_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV2_PF6_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV2_PF6_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV2_PF6_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV2_PF6_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV2_PF6_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//BIF_DEV2_PF0_DSTATE_VALUE
+#define BIF_DEV2_PF0_DSTATE_VALUE__DEV2_PF0_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV2_PF0_DSTATE_VALUE__DEV2_PF0_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV2_PF0_DSTATE_VALUE__DEV2_PF0_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV2_PF0_DSTATE_VALUE__DEV2_PF0_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV2_PF0_DSTATE_VALUE__DEV2_PF0_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV2_PF0_DSTATE_VALUE__DEV2_PF0_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV2_PF1_DSTATE_VALUE
+#define BIF_DEV2_PF1_DSTATE_VALUE__DEV2_PF1_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV2_PF1_DSTATE_VALUE__DEV2_PF1_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV2_PF1_DSTATE_VALUE__DEV2_PF1_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV2_PF1_DSTATE_VALUE__DEV2_PF1_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV2_PF1_DSTATE_VALUE__DEV2_PF1_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV2_PF1_DSTATE_VALUE__DEV2_PF1_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV2_PF2_DSTATE_VALUE
+#define BIF_DEV2_PF2_DSTATE_VALUE__DEV2_PF2_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV2_PF2_DSTATE_VALUE__DEV2_PF2_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV2_PF2_DSTATE_VALUE__DEV2_PF2_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV2_PF2_DSTATE_VALUE__DEV2_PF2_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV2_PF2_DSTATE_VALUE__DEV2_PF2_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV2_PF2_DSTATE_VALUE__DEV2_PF2_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV2_PF3_DSTATE_VALUE
+#define BIF_DEV2_PF3_DSTATE_VALUE__DEV2_PF3_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV2_PF3_DSTATE_VALUE__DEV2_PF3_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV2_PF3_DSTATE_VALUE__DEV2_PF3_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV2_PF3_DSTATE_VALUE__DEV2_PF3_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV2_PF3_DSTATE_VALUE__DEV2_PF3_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV2_PF3_DSTATE_VALUE__DEV2_PF3_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV2_PF4_DSTATE_VALUE
+#define BIF_DEV2_PF4_DSTATE_VALUE__DEV2_PF4_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV2_PF4_DSTATE_VALUE__DEV2_PF4_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV2_PF4_DSTATE_VALUE__DEV2_PF4_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV2_PF4_DSTATE_VALUE__DEV2_PF4_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV2_PF4_DSTATE_VALUE__DEV2_PF4_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV2_PF4_DSTATE_VALUE__DEV2_PF4_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV2_PF5_DSTATE_VALUE
+#define BIF_DEV2_PF5_DSTATE_VALUE__DEV2_PF5_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV2_PF5_DSTATE_VALUE__DEV2_PF5_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV2_PF5_DSTATE_VALUE__DEV2_PF5_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV2_PF5_DSTATE_VALUE__DEV2_PF5_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV2_PF5_DSTATE_VALUE__DEV2_PF5_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV2_PF5_DSTATE_VALUE__DEV2_PF5_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV2_PF6_DSTATE_VALUE
+#define BIF_DEV2_PF6_DSTATE_VALUE__DEV2_PF6_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV2_PF6_DSTATE_VALUE__DEV2_PF6_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV2_PF6_DSTATE_VALUE__DEV2_PF6_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV2_PF6_DSTATE_VALUE__DEV2_PF6_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV2_PF6_DSTATE_VALUE__DEV2_PF6_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV2_PF6_DSTATE_VALUE__DEV2_PF6_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//DEV2_PF0_D3HOTD0_RST_CTRL
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV2_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV2_PF1_D3HOTD0_RST_CTRL
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV2_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV2_PF2_D3HOTD0_RST_CTRL
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV2_PF2_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV2_PF3_D3HOTD0_RST_CTRL
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV2_PF3_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV2_PF4_D3HOTD0_RST_CTRL
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV2_PF4_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV2_PF5_D3HOTD0_RST_CTRL
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV2_PF5_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV2_PF6_D3HOTD0_RST_CTRL
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV2_PF6_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//BIF_PORT0_DSTATE_VALUE
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_TGT_VALUE__SHIFT                                                 0x0
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_ACK_VALUE__SHIFT                                                 0x10
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_TGT_VALUE_MASK                                                   0x00000003L
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_ACK_VALUE_MASK                                                   0x00030000L
+//BIF_PORT1_DSTATE_VALUE
+#define BIF_PORT1_DSTATE_VALUE__PORT1_DSTATE_TGT_VALUE__SHIFT                                                 0x0
+#define BIF_PORT1_DSTATE_VALUE__PORT1_DSTATE_ACK_VALUE__SHIFT                                                 0x10
+#define BIF_PORT1_DSTATE_VALUE__PORT1_DSTATE_TGT_VALUE_MASK                                                   0x00000003L
+#define BIF_PORT1_DSTATE_VALUE__PORT1_DSTATE_ACK_VALUE_MASK                                                   0x00030000L
+//BIF_PORT2_DSTATE_VALUE
+#define BIF_PORT2_DSTATE_VALUE__PORT2_DSTATE_TGT_VALUE__SHIFT                                                 0x0
+#define BIF_PORT2_DSTATE_VALUE__PORT2_DSTATE_ACK_VALUE__SHIFT                                                 0x10
+#define BIF_PORT2_DSTATE_VALUE__PORT2_DSTATE_TGT_VALUE_MASK                                                   0x00000003L
+#define BIF_PORT2_DSTATE_VALUE__PORT2_DSTATE_ACK_VALUE_MASK                                                   0x00030000L
+
+
+// addressBlock: nbif0_bif_misc_bif_misc_regblk
+//REGS_ROM_OFFSET_CTRL
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET__SHIFT                                                               0x0
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET_MASK                                                                 0x7FL
+#define NBIF_HSP_STREAMID__DMA_HSP_ACPI_VTL0_STREAMID__SHIFT                                                  0x0
+#define NBIF_HSP_STREAMID__DMA_HSP_ACPI_VTL1_STREAMID__SHIFT                                                  0x10
+#define NBIF_HSP_STREAMID__DMA_HSP_ACPI_VTL0_STREAMID_MASK                                                    0x0000FFFFL
+#define NBIF_HSP_STREAMID__DMA_HSP_ACPI_VTL1_STREAMID_MASK                                                    0xFFFF0000L
+#define NBIF_MPXSP_STREAMID__DMA_MPXSP_ACPI_VTL0_STREAMID__SHIFT                                              0x0
+#define NBIF_MPXSP_STREAMID__DMA_MPXSP_ACPI_VTL1_STREAMID__SHIFT                                              0x10
+#define NBIF_MPXSP_STREAMID__DMA_MPXSP_ACPI_VTL0_STREAMID_MASK                                                0x0000FFFFL
+#define NBIF_MPXSP_STREAMID__DMA_MPXSP_ACPI_VTL1_STREAMID_MASK                                                0xFFFF0000L
+//NBIF_STRAP_BIOS_CNTL
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_BIOS_EN__SHIFT                                                       0x0
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_PCIE_ID_BIOS_EN__SHIFT                                               0x1
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_BIOS_EN_MASK                                                         0x00000001L
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_PCIE_ID_BIOS_EN_MASK                                                 0x00000002L
+//MISC_SCRATCH
+#define MISC_SCRATCH__MISC_SCRATCH0__SHIFT                                                                    0x0
+#define MISC_SCRATCH__MISC_SCRATCH0_MASK                                                                      0xFFFFFFFFL
+//INTR_LINE_POLARITY
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV0__SHIFT                                                    0x0
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV1__SHIFT                                                    0x8
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV2__SHIFT                                                    0x10
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV0_MASK                                                      0x000000FFL
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV1_MASK                                                      0x0000FF00L
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV2_MASK                                                      0x00FF0000L
+//INTR_LINE_ENABLE
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV0__SHIFT                                                        0x0
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV1__SHIFT                                                        0x8
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV2__SHIFT                                                        0x10
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV0_MASK                                                          0x000000FFL
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV1_MASK                                                          0x0000FF00L
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV2_MASK                                                          0x00FF0000L
+//OUTSTANDING_VC_ALLOC
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC0_ALLOC__SHIFT                                                0x0
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC1_ALLOC__SHIFT                                                0x2
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC2_ALLOC__SHIFT                                                0x4
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC3_ALLOC__SHIFT                                                0x6
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC4_ALLOC__SHIFT                                                0x8
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC5_ALLOC__SHIFT                                                0xa
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC6_ALLOC__SHIFT                                                0xc
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC7_ALLOC__SHIFT                                                0xe
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_THRD__SHIFT                                                     0x10
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC0_ALLOC__SHIFT                                                0x18
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC1_ALLOC__SHIFT                                                0x1a
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_THRD__SHIFT                                                     0x1c
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC0_ALLOC_MASK                                                  0x00000003L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC1_ALLOC_MASK                                                  0x0000000CL
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC2_ALLOC_MASK                                                  0x00000030L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC3_ALLOC_MASK                                                  0x000000C0L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC4_ALLOC_MASK                                                  0x00000300L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC5_ALLOC_MASK                                                  0x00000C00L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC6_ALLOC_MASK                                                  0x00003000L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC7_ALLOC_MASK                                                  0x0000C000L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_THRD_MASK                                                       0x000F0000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC0_ALLOC_MASK                                                  0x03000000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC1_ALLOC_MASK                                                  0x0C000000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_THRD_MASK                                                       0xF0000000L
+//BIFC_MISC_CTRL0
+#define BIFC_MISC_CTRL0__VWIRE_TARG_UNITID_CHECK_EN__SHIFT                                                    0x0
+#define BIFC_MISC_CTRL0__VWIRE_SRC_UNITID_CHECK_EN__SHIFT                                                     0x1
+#define BIFC_MISC_CTRL0__REG_ACTIVE_VLINK_L0_EN__SHIFT                                                        0x3
+#define BIFC_MISC_CTRL0__DMA_VC4_NON_DVM_STS__SHIFT                                                           0x4
+#define BIFC_MISC_CTRL0__DMA_CHAIN_BREAK_IN_RCMODE__SHIFT                                                     0x8
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_P__SHIFT                                                          0x9
+#define BIFC_MISC_CTRL0__GSI_SST_ARB_CHAIN_LOCK__SHIFT                                                        0xa
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_FLUSH_EN__SHIFT                                                   0xb
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_NPWR_DIS__SHIFT                                                   0xc
+#define BIFC_MISC_CTRL0__GSI_SET_PRECEEDINGWR_DIS__SHIFT                                                      0xd
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_NP__SHIFT                                                         0xe
+#define BIFC_MISC_CTRL0__HRP_CHAIN_DISABLE__SHIFT                                                             0xf
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_LENGTH_CHK_DIS__SHIFT                                                     0x10
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_FAILED_STS_SEL__SHIFT                                                     0x11
+#define BIFC_MISC_CTRL0__DMA_FORCE_VF_AS_PF_SRIOIVEN_LOW__SHIFT                                               0x12
+#define BIFC_MISC_CTRL0__DMA_ADDR_KEEP_PH__SHIFT                                                              0x13
+#define BIFC_MISC_CTRL0__RCC_GMI_TD_FORCE_ZERO__SHIFT                                                         0x14
+#define BIFC_MISC_CTRL0__HST_FLUSH_DEFER_EN__SHIFT                                                            0x15
+#define BIFC_MISC_CTRL0__HST_FLUSH_CLR_LOCK_EN__SHIFT                                                         0x16
+#define BIFC_MISC_CTRL0__STFETCH_BLOCK_IN_RST__SHIFT                                                          0x17
+#define BIFC_MISC_CTRL0__PCIE_CAPABILITY_PROT_DIS__SHIFT                                                      0x18
+#define BIFC_MISC_CTRL0__ATS_MSG_BLOCK_IN_RST__SHIFT                                                          0x19
+#define BIFC_MISC_CTRL0__DMA_2ND_REQ_DIS__SHIFT                                                               0x1a
+#define BIFC_MISC_CTRL0__PORT_DSTATE_BYPASS_MODE__SHIFT                                                       0x1b
+#define BIFC_MISC_CTRL0__PME_TURNOFF_MODE__SHIFT                                                              0x1c
+#define BIFC_MISC_CTRL0__DMA_ALL_RST_PROTECT_STS_SEL__SHIFT                                                   0x1d
+#define BIFC_MISC_CTRL0__HDP_P2P_DIRECT_ADD_ADJUST__SHIFT                                                     0x1e
+#define BIFC_MISC_CTRL0__PCIESWUS_SELECTION__SHIFT                                                            0x1f
+#define BIFC_MISC_CTRL0__VWIRE_TARG_UNITID_CHECK_EN_MASK                                                      0x00000001L
+#define BIFC_MISC_CTRL0__VWIRE_SRC_UNITID_CHECK_EN_MASK                                                       0x00000006L
+#define BIFC_MISC_CTRL0__REG_ACTIVE_VLINK_L0_EN_MASK                                                          0x00000008L
+#define BIFC_MISC_CTRL0__DMA_VC4_NON_DVM_STS_MASK                                                             0x000000F0L
+#define BIFC_MISC_CTRL0__DMA_CHAIN_BREAK_IN_RCMODE_MASK                                                       0x00000100L
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_P_MASK                                                            0x00000200L
+#define BIFC_MISC_CTRL0__GSI_SST_ARB_CHAIN_LOCK_MASK                                                          0x00000400L
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_FLUSH_EN_MASK                                                     0x00000800L
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_NPWR_DIS_MASK                                                     0x00001000L
+#define BIFC_MISC_CTRL0__GSI_SET_PRECEEDINGWR_DIS_MASK                                                        0x00002000L
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_NP_MASK                                                           0x00004000L
+#define BIFC_MISC_CTRL0__HRP_CHAIN_DISABLE_MASK                                                               0x00008000L
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_LENGTH_CHK_DIS_MASK                                                       0x00010000L
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_FAILED_STS_SEL_MASK                                                       0x00020000L
+#define BIFC_MISC_CTRL0__DMA_FORCE_VF_AS_PF_SRIOIVEN_LOW_MASK                                                 0x00040000L
+#define BIFC_MISC_CTRL0__DMA_ADDR_KEEP_PH_MASK                                                                0x00080000L
+#define BIFC_MISC_CTRL0__RCC_GMI_TD_FORCE_ZERO_MASK                                                           0x00100000L
+#define BIFC_MISC_CTRL0__HST_FLUSH_DEFER_EN_MASK                                                              0x00200000L
+#define BIFC_MISC_CTRL0__HST_FLUSH_CLR_LOCK_EN_MASK                                                           0x00400000L
+#define BIFC_MISC_CTRL0__STFETCH_BLOCK_IN_RST_MASK                                                            0x00800000L
+#define BIFC_MISC_CTRL0__PCIE_CAPABILITY_PROT_DIS_MASK                                                        0x01000000L
+#define BIFC_MISC_CTRL0__ATS_MSG_BLOCK_IN_RST_MASK                                                            0x02000000L
+#define BIFC_MISC_CTRL0__DMA_2ND_REQ_DIS_MASK                                                                 0x04000000L
+#define BIFC_MISC_CTRL0__PORT_DSTATE_BYPASS_MODE_MASK                                                         0x08000000L
+#define BIFC_MISC_CTRL0__PME_TURNOFF_MODE_MASK                                                                0x10000000L
+#define BIFC_MISC_CTRL0__DMA_ALL_RST_PROTECT_STS_SEL_MASK                                                     0x20000000L
+#define BIFC_MISC_CTRL0__HDP_P2P_DIRECT_ADD_ADJUST_MASK                                                       0x40000000L
+#define BIFC_MISC_CTRL0__PCIESWUS_SELECTION_MASK                                                              0x80000000L
+//BIFC_MISC_CTRL1
+#define BIFC_MISC_CTRL1__THT_HST_CPLD_POISON_REPORT__SHIFT                                                    0x0
+#define BIFC_MISC_CTRL1__DMA_REQ_POISON_REPORT__SHIFT                                                         0x1
+#define BIFC_MISC_CTRL1__DMA_REQ_ACSVIO_REPORT__SHIFT                                                         0x2
+#define BIFC_MISC_CTRL1__DMA_RSP_POISON_CPLD_REPORT__SHIFT                                                    0x3
+#define BIFC_MISC_CTRL1__GSI_SMN_WORST_ERR_STSTUS__SHIFT                                                      0x4
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE1_FOR_ERROR__SHIFT                                           0x5
+#define BIFC_MISC_CTRL1__GSI_RDWR_BALANCE_DIS__SHIFT                                                          0x6
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_DROP__SHIFT                                                        0x7
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_STS__SHIFT                                                      0x8
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_DATASTS__SHIFT                                                  0xa
+#define BIFC_MISC_CTRL1__DROP_OTHER_HT_ADDR_REQ__SHIFT                                                        0xc
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE__SHIFT                                                 0xd
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_VALUE__SHIFT                                           0xe
+#define BIFC_MISC_CTRL1__UPS_SDP_RDY_TIE1__SHIFT                                                              0xf
+#define BIFC_MISC_CTRL1__GMI_RCC_DN_BME_DROP_DIS__SHIFT                                                       0x10
+#define BIFC_MISC_CTRL1__GMI_RCC_EP_BME_DROP_DIS__SHIFT                                                       0x11
+#define BIFC_MISC_CTRL1__GMI_BIH_DN_BME_DROP_DIS__SHIFT                                                       0x12
+#define BIFC_MISC_CTRL1__GMI_BIH_EP_BME_DROP_DIS__SHIFT                                                       0x13
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE0_FOR_ERROR__SHIFT                                           0x14
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE__SHIFT                                                 0x15
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_VALUE__SHIFT                                           0x16
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_FOR_AERLOG__SHIFT                                                  0x17
+#define BIFC_MISC_CTRL1__GMI_RDSIZED_REQATTR_MASK__SHIFT                                                      0x18
+#define BIFC_MISC_CTRL1__GMI_RDSIZEDDW_REQATTR_MASK__SHIFT                                                    0x19
+#define BIFC_MISC_CTRL1__GMI_WRSIZED_REQATTR_MASK__SHIFT                                                      0x1a
+#define BIFC_MISC_CTRL1__GMI_WRSIZEDFL_REQATTR_MASK__SHIFT                                                    0x1b
+#define BIFC_MISC_CTRL1__GMI_FORCE_NOT_SEND_NON_BASEVC_RSPCREDIT__SHIFT                                       0x1c
+#define BIFC_MISC_CTRL1__GMI_CPLBUF_EN__SHIFT                                                                 0x1d
+#define BIFC_MISC_CTRL1__GMI_MSG_BLOCKLVL_SEL__SHIFT                                                          0x1e
+#define BIFC_MISC_CTRL1__THT_HST_CPLD_POISON_REPORT_MASK                                                      0x00000001L
+#define BIFC_MISC_CTRL1__DMA_REQ_POISON_REPORT_MASK                                                           0x00000002L
+#define BIFC_MISC_CTRL1__DMA_REQ_ACSVIO_REPORT_MASK                                                           0x00000004L
+#define BIFC_MISC_CTRL1__DMA_RSP_POISON_CPLD_REPORT_MASK                                                      0x00000008L
+#define BIFC_MISC_CTRL1__GSI_SMN_WORST_ERR_STSTUS_MASK                                                        0x00000010L
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE1_FOR_ERROR_MASK                                             0x00000020L
+#define BIFC_MISC_CTRL1__GSI_RDWR_BALANCE_DIS_MASK                                                            0x00000040L
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_DROP_MASK                                                          0x00000080L
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_STS_MASK                                                        0x00000300L
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_DATASTS_MASK                                                    0x00000C00L
+#define BIFC_MISC_CTRL1__DROP_OTHER_HT_ADDR_REQ_MASK                                                          0x00001000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_MASK                                                   0x00002000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_VALUE_MASK                                             0x00004000L
+#define BIFC_MISC_CTRL1__UPS_SDP_RDY_TIE1_MASK                                                                0x00008000L
+#define BIFC_MISC_CTRL1__GMI_RCC_DN_BME_DROP_DIS_MASK                                                         0x00010000L
+#define BIFC_MISC_CTRL1__GMI_RCC_EP_BME_DROP_DIS_MASK                                                         0x00020000L
+#define BIFC_MISC_CTRL1__GMI_BIH_DN_BME_DROP_DIS_MASK                                                         0x00040000L
+#define BIFC_MISC_CTRL1__GMI_BIH_EP_BME_DROP_DIS_MASK                                                         0x00080000L
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE0_FOR_ERROR_MASK                                             0x00100000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_MASK                                                   0x00200000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_VALUE_MASK                                             0x00400000L
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_FOR_AERLOG_MASK                                                    0x00800000L
+#define BIFC_MISC_CTRL1__GMI_RDSIZED_REQATTR_MASK_MASK                                                        0x01000000L
+#define BIFC_MISC_CTRL1__GMI_RDSIZEDDW_REQATTR_MASK_MASK                                                      0x02000000L
+#define BIFC_MISC_CTRL1__GMI_WRSIZED_REQATTR_MASK_MASK                                                        0x04000000L
+#define BIFC_MISC_CTRL1__GMI_WRSIZEDFL_REQATTR_MASK_MASK                                                      0x08000000L
+#define BIFC_MISC_CTRL1__GMI_FORCE_NOT_SEND_NON_BASEVC_RSPCREDIT_MASK                                         0x10000000L
+#define BIFC_MISC_CTRL1__GMI_CPLBUF_EN_MASK                                                                   0x20000000L
+#define BIFC_MISC_CTRL1__GMI_MSG_BLOCKLVL_SEL_MASK                                                            0xC0000000L
+//BIFC_BME_ERR_LOG_LB
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F0__SHIFT                                                    0x0
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F1__SHIFT                                                    0x1
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F2__SHIFT                                                    0x2
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F3__SHIFT                                                    0x3
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F4__SHIFT                                                    0x4
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F5__SHIFT                                                    0x5
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F6__SHIFT                                                    0x6
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F7__SHIFT                                                    0x7
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F0__SHIFT                                                    0x8
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F1__SHIFT                                                    0x9
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F2__SHIFT                                                    0xa
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F3__SHIFT                                                    0xb
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F4__SHIFT                                                    0xc
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F0__SHIFT                                              0x10
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F1__SHIFT                                              0x11
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F2__SHIFT                                              0x12
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F3__SHIFT                                              0x13
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F4__SHIFT                                              0x14
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F5__SHIFT                                              0x15
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F6__SHIFT                                              0x16
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F7__SHIFT                                              0x17
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F0__SHIFT                                              0x18
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F1__SHIFT                                              0x19
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F2__SHIFT                                              0x1a
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F3__SHIFT                                              0x1b
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F4__SHIFT                                              0x1c
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F0_MASK                                                      0x00000001L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F1_MASK                                                      0x00000002L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F2_MASK                                                      0x00000004L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F3_MASK                                                      0x00000008L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F4_MASK                                                      0x00000010L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F5_MASK                                                      0x00000020L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F6_MASK                                                      0x00000040L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F7_MASK                                                      0x00000080L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F0_MASK                                                      0x00000100L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F1_MASK                                                      0x00000200L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F2_MASK                                                      0x00000400L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F3_MASK                                                      0x00000800L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV1_F4_MASK                                                      0x00001000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F0_MASK                                                0x00010000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F1_MASK                                                0x00020000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F2_MASK                                                0x00040000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F3_MASK                                                0x00080000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F4_MASK                                                0x00100000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F5_MASK                                                0x00200000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F6_MASK                                                0x00400000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F7_MASK                                                0x00800000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F0_MASK                                                0x01000000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F1_MASK                                                0x02000000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F2_MASK                                                0x04000000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F3_MASK                                                0x08000000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV1_F4_MASK                                                0x10000000L
+//BIFC_LC_TIMER_CTRL
+#define BIFC_LC_TIMER_CTRL__ASPM_IDLE_TIMER_SCALE__SHIFT                                                      0x0
+#define BIFC_LC_TIMER_CTRL__L1_EXIT_TIMER_SCALE__SHIFT                                                        0x10
+#define BIFC_LC_TIMER_CTRL__ASPM_IDLE_TIMER_SCALE_MASK                                                        0x0000FFFFL
+#define BIFC_LC_TIMER_CTRL__L1_EXIT_TIMER_SCALE_MASK                                                          0xFFFF0000L
+//BIFC_RCCBIH_BME_ERR_LOG0
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F0__SHIFT                                            0x0
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F1__SHIFT                                            0x1
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F2__SHIFT                                            0x2
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F3__SHIFT                                            0x3
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F4__SHIFT                                            0x4
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F5__SHIFT                                            0x5
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F6__SHIFT                                            0x6
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F7__SHIFT                                            0x7
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F0__SHIFT                                            0x8
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F1__SHIFT                                            0x9
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F2__SHIFT                                            0xa
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F3__SHIFT                                            0xb
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F4__SHIFT                                            0xc
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F0__SHIFT                                      0x10
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F1__SHIFT                                      0x11
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F2__SHIFT                                      0x12
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F3__SHIFT                                      0x13
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F4__SHIFT                                      0x14
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F5__SHIFT                                      0x15
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F6__SHIFT                                      0x16
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F7__SHIFT                                      0x17
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F0__SHIFT                                      0x18
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F1__SHIFT                                      0x19
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F2__SHIFT                                      0x1a
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F3__SHIFT                                      0x1b
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F4__SHIFT                                      0x1c
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F0_MASK                                              0x00000001L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F1_MASK                                              0x00000002L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F2_MASK                                              0x00000004L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F3_MASK                                              0x00000008L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F4_MASK                                              0x00000010L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F5_MASK                                              0x00000020L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F6_MASK                                              0x00000040L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F7_MASK                                              0x00000080L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F0_MASK                                              0x00000100L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F1_MASK                                              0x00000200L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F2_MASK                                              0x00000400L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F3_MASK                                              0x00000800L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV1_F4_MASK                                              0x00001000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F0_MASK                                        0x00010000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F1_MASK                                        0x00020000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F2_MASK                                        0x00040000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F3_MASK                                        0x00080000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F4_MASK                                        0x00100000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F5_MASK                                        0x00200000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F6_MASK                                        0x00400000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F7_MASK                                        0x00800000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F0_MASK                                        0x01000000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F1_MASK                                        0x02000000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F2_MASK                                        0x04000000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F3_MASK                                        0x08000000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV1_F4_MASK                                        0x10000000L
+//BIFC_RCCBIH_BME_ERR_LOG1
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F0__SHIFT                                            0x0
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F1__SHIFT                                            0x1
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F2__SHIFT                                            0x2
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F3__SHIFT                                            0x3
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F4__SHIFT                                            0x4
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F5__SHIFT                                            0x5
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F6__SHIFT                                            0x6
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F0__SHIFT                                      0x10
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F1__SHIFT                                      0x11
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F2__SHIFT                                      0x12
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F3__SHIFT                                      0x13
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F4__SHIFT                                      0x14
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F5__SHIFT                                      0x15
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F6__SHIFT                                      0x16
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F0_MASK                                              0x00000001L
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F1_MASK                                              0x00000002L
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F2_MASK                                              0x00000004L
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F3_MASK                                              0x00000008L
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F4_MASK                                              0x00000010L
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F5_MASK                                              0x00000020L
+#define BIFC_RCCBIH_BME_ERR_LOG1__RCCBIH_ON_BME_LOW_DEV2_F6_MASK                                              0x00000040L
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F0_MASK                                        0x00010000L
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F1_MASK                                        0x00020000L
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F2_MASK                                        0x00040000L
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F3_MASK                                        0x00080000L
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F4_MASK                                        0x00100000L
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F5_MASK                                        0x00200000L
+#define BIFC_RCCBIH_BME_ERR_LOG1__CLEAR_RCCBIH_ON_BME_LOW_DEV2_F6_MASK                                        0x00400000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F0__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F0__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_IDO_DEV0_F0__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F0__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F0__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F0__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F0__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV0_F0__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F1__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F1__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_IDO_DEV0_F1__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F1__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F1__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F1__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F1__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV0_F1__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F0_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F0_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_IDO_DEV0_F0_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F0_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F0_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F0_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F0_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV0_F0_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F1_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F1_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_IDO_DEV0_F1_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F1_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F1_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F1_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F1_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV0_F1_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F2__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F2__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_IDO_DEV0_F2__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F2__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F2__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F2__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F2__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV0_F2__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F3__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F3__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_IDO_DEV0_F3__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F3__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F3__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F3__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F3__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV0_F3__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F2_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F2_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_IDO_DEV0_F2_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F2_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F2_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F2_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F2_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV0_F2_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F3_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F3_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_IDO_DEV0_F3_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F3_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F3_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F3_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F3_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV0_F3_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F4__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F4__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_IDO_DEV0_F4__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F4__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F4__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F4__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F4__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV0_F4__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F5__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F5__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_IDO_DEV0_F5__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F5__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F5__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F5__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F5__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV0_F5__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F4_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F4_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_IDO_DEV0_F4_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F4_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F4_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F4_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F4_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV0_F4_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F5_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F5_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_IDO_DEV0_F5_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F5_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F5_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F5_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F5_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV0_F5_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F6__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F6__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_IDO_DEV0_F6__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F6__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F6__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F6__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F6__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV0_F6__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F7__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F7__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_IDO_DEV0_F7__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F7__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F7__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F7__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F7__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV0_F7__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F6_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F6_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_IDO_DEV0_F6_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F6_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F6_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F6_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F6_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV0_F6_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F7_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F7_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_IDO_DEV0_F7_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F7_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F7_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F7_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F7_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV0_F7_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_P_DEV1_F0__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_NP_DEV1_F0__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_IDO_DEV1_F0__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_P_DEV1_F0__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_NP_DEV1_F0__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_P_DEV1_F0__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_NP_DEV1_F0__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV1_F0__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_P_DEV1_F1__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_NP_DEV1_F1__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_IDO_DEV1_F1__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_P_DEV1_F1__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_NP_DEV1_F1__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_P_DEV1_F1__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_NP_DEV1_F1__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV1_F1__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_P_DEV1_F0_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_NP_DEV1_F0_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_IDO_DEV1_F0_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_P_DEV1_F0_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_NP_DEV1_F0_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_P_DEV1_F0_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_NP_DEV1_F0_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV1_F0_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_P_DEV1_F1_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_IDO_OVERIDE_NP_DEV1_F1_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_IDO_DEV1_F1_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_P_DEV1_F1_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_RO_OVERIDE_NP_DEV1_F1_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_P_DEV1_F1_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__TX_SNR_OVERIDE_NP_DEV1_F1_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV1_F1_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_P_DEV1_F2__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_NP_DEV1_F2__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_IDO_DEV1_F2__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_P_DEV1_F2__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_NP_DEV1_F2__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_P_DEV1_F2__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_NP_DEV1_F2__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV1_F2__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_P_DEV1_F3__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_NP_DEV1_F3__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_IDO_DEV1_F3__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_P_DEV1_F3__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_NP_DEV1_F3__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_P_DEV1_F3__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_NP_DEV1_F3__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV1_F3__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_P_DEV1_F2_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_NP_DEV1_F2_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_IDO_DEV1_F2_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_P_DEV1_F2_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_NP_DEV1_F2_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_P_DEV1_F2_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_NP_DEV1_F2_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV1_F2_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_P_DEV1_F3_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_IDO_OVERIDE_NP_DEV1_F3_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_IDO_DEV1_F3_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_P_DEV1_F3_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_RO_OVERIDE_NP_DEV1_F3_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_P_DEV1_F3_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__TX_SNR_OVERIDE_NP_DEV1_F3_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV1_F3_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_P_DEV1_F4__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_NP_DEV1_F4__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_IDO_DEV1_F4__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_P_DEV1_F4__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_NP_DEV1_F4__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_P_DEV1_F4__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_NP_DEV1_F4__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV1_F4__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_P_DEV1_F5__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_NP_DEV1_F5__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_IDO_DEV1_F5__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_P_DEV1_F5__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_NP_DEV1_F5__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_P_DEV1_F5__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_NP_DEV1_F5__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV1_F5__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_P_DEV1_F4_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_NP_DEV1_F4_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_IDO_DEV1_F4_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_P_DEV1_F4_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_NP_DEV1_F4_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_P_DEV1_F4_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_NP_DEV1_F4_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV1_F4_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_P_DEV1_F5_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_IDO_OVERIDE_NP_DEV1_F5_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_IDO_DEV1_F5_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_P_DEV1_F5_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_RO_OVERIDE_NP_DEV1_F5_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_P_DEV1_F5_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__TX_SNR_OVERIDE_NP_DEV1_F5_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV1_F5_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_P_DEV1_F6__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_NP_DEV1_F6__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_IDO_DEV1_F6__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_P_DEV1_F6__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_NP_DEV1_F6__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_P_DEV1_F6__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_NP_DEV1_F6__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV1_F6__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_P_DEV1_F7__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_NP_DEV1_F7__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_IDO_DEV1_F7__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_P_DEV1_F7__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_NP_DEV1_F7__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_P_DEV1_F7__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_NP_DEV1_F7__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV1_F7__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_P_DEV1_F6_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_NP_DEV1_F6_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_IDO_DEV1_F6_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_P_DEV1_F6_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_NP_DEV1_F6_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_P_DEV1_F6_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_NP_DEV1_F6_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV1_F6_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_P_DEV1_F7_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_IDO_OVERIDE_NP_DEV1_F7_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_IDO_DEV1_F7_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_P_DEV1_F7_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_RO_OVERIDE_NP_DEV1_F7_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_P_DEV1_F7_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__TX_SNR_OVERIDE_NP_DEV1_F7_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV1_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV1_F7_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_P_DEV2_F0__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_NP_DEV2_F0__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_IDO_DEV2_F0__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_P_DEV2_F0__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_NP_DEV2_F0__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_P_DEV2_F0__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_NP_DEV2_F0__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV2_F0__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_P_DEV2_F1__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_NP_DEV2_F1__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_IDO_DEV2_F1__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_P_DEV2_F1__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_NP_DEV2_F1__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_P_DEV2_F1__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_NP_DEV2_F1__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV2_F1__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_P_DEV2_F0_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_NP_DEV2_F0_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_IDO_DEV2_F0_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_P_DEV2_F0_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_NP_DEV2_F0_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_P_DEV2_F0_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_NP_DEV2_F0_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV2_F0_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_P_DEV2_F1_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_IDO_OVERIDE_NP_DEV2_F1_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_IDO_DEV2_F1_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_P_DEV2_F1_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_RO_OVERIDE_NP_DEV2_F1_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_P_DEV2_F1_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__TX_SNR_OVERIDE_NP_DEV2_F1_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F0_F1__VC01_BLKLVL_FOR_NONIDO_DEV2_F1_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_P_DEV2_F2__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_NP_DEV2_F2__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_IDO_DEV2_F2__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_P_DEV2_F2__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_NP_DEV2_F2__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_P_DEV2_F2__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_NP_DEV2_F2__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV2_F2__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_P_DEV2_F3__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_NP_DEV2_F3__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_IDO_DEV2_F3__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_P_DEV2_F3__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_NP_DEV2_F3__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_P_DEV2_F3__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_NP_DEV2_F3__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV2_F3__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_P_DEV2_F2_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_NP_DEV2_F2_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_IDO_DEV2_F2_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_P_DEV2_F2_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_NP_DEV2_F2_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_P_DEV2_F2_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_NP_DEV2_F2_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV2_F2_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_P_DEV2_F3_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_IDO_OVERIDE_NP_DEV2_F3_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_IDO_DEV2_F3_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_P_DEV2_F3_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_RO_OVERIDE_NP_DEV2_F3_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_P_DEV2_F3_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__TX_SNR_OVERIDE_NP_DEV2_F3_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F2_F3__VC01_BLKLVL_FOR_NONIDO_DEV2_F3_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_P_DEV2_F4__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_NP_DEV2_F4__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_IDO_DEV2_F4__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_P_DEV2_F4__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_NP_DEV2_F4__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_P_DEV2_F4__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_NP_DEV2_F4__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV2_F4__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_P_DEV2_F5__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_NP_DEV2_F5__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_IDO_DEV2_F5__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_P_DEV2_F5__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_NP_DEV2_F5__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_P_DEV2_F5__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_NP_DEV2_F5__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV2_F5__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_P_DEV2_F4_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_NP_DEV2_F4_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_IDO_DEV2_F4_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_P_DEV2_F4_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_NP_DEV2_F4_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_P_DEV2_F4_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_NP_DEV2_F4_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV2_F4_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_P_DEV2_F5_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_IDO_OVERIDE_NP_DEV2_F5_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_IDO_DEV2_F5_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_P_DEV2_F5_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_RO_OVERIDE_NP_DEV2_F5_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_P_DEV2_F5_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__TX_SNR_OVERIDE_NP_DEV2_F5_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F4_F5__VC01_BLKLVL_FOR_NONIDO_DEV2_F5_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_P_DEV2_F6__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_NP_DEV2_F6__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_IDO_DEV2_F6__SHIFT                                 0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_P_DEV2_F6__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_NP_DEV2_F6__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_P_DEV2_F6__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_NP_DEV2_F6__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV2_F6__SHIFT                              0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_P_DEV2_F7__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_NP_DEV2_F7__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_IDO_DEV2_F7__SHIFT                                 0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_P_DEV2_F7__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_NP_DEV2_F7__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_P_DEV2_F7__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_NP_DEV2_F7__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV2_F7__SHIFT                              0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_P_DEV2_F6_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_NP_DEV2_F6_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_IDO_DEV2_F6_MASK                                   0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_P_DEV2_F6_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_NP_DEV2_F6_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_P_DEV2_F6_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_NP_DEV2_F6_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV2_F6_MASK                                0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_P_DEV2_F7_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_IDO_OVERIDE_NP_DEV2_F7_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_IDO_DEV2_F7_MASK                                   0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_P_DEV2_F7_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_RO_OVERIDE_NP_DEV2_F7_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_P_DEV2_F7_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__TX_SNR_OVERIDE_NP_DEV2_F7_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV2_F6_F7__VC01_BLKLVL_FOR_NONIDO_DEV2_F7_MASK                                0xC0000000L
+//BIFC_DMA_ATTR_CNTL2_DEV0
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F0__SHIFT                               0x0
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F0__SHIFT                                     0x1
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F1__SHIFT                               0x4
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F1__SHIFT                                     0x5
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F2__SHIFT                               0x8
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F2__SHIFT                                     0x9
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F3__SHIFT                               0xc
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F3__SHIFT                                     0xd
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F4__SHIFT                               0x10
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F4__SHIFT                                     0x11
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F5__SHIFT                               0x14
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F5__SHIFT                                     0x15
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F6__SHIFT                               0x18
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F6__SHIFT                                     0x19
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F7__SHIFT                               0x1c
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F7__SHIFT                                     0x1d
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F0_MASK                                 0x00000001L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F0_MASK                                       0x00000002L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F1_MASK                                 0x00000010L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F1_MASK                                       0x00000020L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F2_MASK                                 0x00000100L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F2_MASK                                       0x00000200L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F3_MASK                                 0x00001000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F3_MASK                                       0x00002000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F4_MASK                                 0x00010000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F4_MASK                                       0x00020000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F5_MASK                                 0x00100000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F5_MASK                                       0x00200000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F6_MASK                                 0x01000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F6_MASK                                       0x02000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F7_MASK                                 0x10000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F7_MASK                                       0x20000000L
+//BIFC_DMA_ATTR_CNTL2_DEV1
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F0__SHIFT                               0x0
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F0__SHIFT                                     0x1
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F1__SHIFT                               0x4
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F1__SHIFT                                     0x5
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F2__SHIFT                               0x8
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F2__SHIFT                                     0x9
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F3__SHIFT                               0xc
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F3__SHIFT                                     0xd
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F4__SHIFT                               0x10
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F4__SHIFT                                     0x11
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F5__SHIFT                               0x14
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F5__SHIFT                                     0x15
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F6__SHIFT                               0x18
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F6__SHIFT                                     0x19
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F7__SHIFT                               0x1c
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F7__SHIFT                                     0x1d
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F0_MASK                                 0x00000001L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F0_MASK                                       0x00000002L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F1_MASK                                 0x00000010L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F1_MASK                                       0x00000020L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F2_MASK                                 0x00000100L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F2_MASK                                       0x00000200L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F3_MASK                                 0x00001000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F3_MASK                                       0x00002000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F4_MASK                                 0x00010000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F4_MASK                                       0x00020000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F5_MASK                                 0x00100000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F5_MASK                                       0x00200000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F6_MASK                                 0x01000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F6_MASK                                       0x02000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV1_F7_MASK                                 0x10000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV1__NBIF_CDMA_ST_OVERRIDE_EN_DEV1_F7_MASK                                       0x20000000L
+//BIFC_DMA_ATTR_CNTL2_DEV2
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F0__SHIFT                               0x0
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F0__SHIFT                                     0x1
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F1__SHIFT                               0x4
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F1__SHIFT                                     0x5
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F2__SHIFT                               0x8
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F2__SHIFT                                     0x9
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F3__SHIFT                               0xc
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F3__SHIFT                                     0xd
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F4__SHIFT                               0x10
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F4__SHIFT                                     0x11
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F5__SHIFT                               0x14
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F5__SHIFT                                     0x15
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F6__SHIFT                               0x18
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F6__SHIFT                                     0x19
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F7__SHIFT                               0x1c
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F7__SHIFT                                     0x1d
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F0_MASK                                 0x00000001L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F0_MASK                                       0x00000002L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F1_MASK                                 0x00000010L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F1_MASK                                       0x00000020L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F2_MASK                                 0x00000100L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F2_MASK                                       0x00000200L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F3_MASK                                 0x00001000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F3_MASK                                       0x00002000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F4_MASK                                 0x00010000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F4_MASK                                       0x00020000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F5_MASK                                 0x00100000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F5_MASK                                       0x00200000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F6_MASK                                 0x01000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F6_MASK                                       0x02000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV2_F7_MASK                                 0x10000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV2__NBIF_CDMA_ST_OVERRIDE_EN_DEV2_F7_MASK                                       0x20000000L
+//BIFC_MISC_CTRL2
+#define BIFC_MISC_CTRL2__DIH_INTR_STFETCH_BLOCK_IN_LINKDOWN__SHIFT                                            0x0
+#define BIFC_MISC_CTRL2__SLOW_GMI_UPS_RSP_CRED_REL_EN__SHIFT                                                  0x1
+#define BIFC_MISC_CTRL2__DMA_RSP_ECRC_REPORT__SHIFT                                                           0x2
+#define BIFC_MISC_CTRL2__SLFR_IGNORE_DATAERR_EN__SHIFT                                                        0x10
+#define BIFC_MISC_CTRL2__DATAERR_OVERRIDE_SLFR_BYTEEN_EN__SHIFT                                               0x11
+#define BIFC_MISC_CTRL2__PH_SUPPORT__SHIFT                                                                    0x12
+#define BIFC_MISC_CTRL2__GMI_FAIL_REQ_RTS_MASK__SHIFT                                                         0x16
+#define BIFC_MISC_CTRL2__NBIF_AERRPT_BACKPERSURE_EN__SHIFT                                                    0x17
+#define BIFC_MISC_CTRL2__DACCFE_SUPPORT_ATOMIC_NR_EN__SHIFT                                                   0x18
+#define BIFC_MISC_CTRL2__DACCBE_SUPPORT_ATOMIC_NR_EN__SHIFT                                                   0x19
+#define BIFC_MISC_CTRL2__DIH_INTR_STFETCH_BLOCK_IN_LINKDOWN_MASK                                              0x00000001L
+#define BIFC_MISC_CTRL2__SLOW_GMI_UPS_RSP_CRED_REL_EN_MASK                                                    0x00000002L
+#define BIFC_MISC_CTRL2__DMA_RSP_ECRC_REPORT_MASK                                                             0x00000004L
+#define BIFC_MISC_CTRL2__SLFR_IGNORE_DATAERR_EN_MASK                                                          0x00010000L
+#define BIFC_MISC_CTRL2__DATAERR_OVERRIDE_SLFR_BYTEEN_EN_MASK                                                 0x00020000L
+#define BIFC_MISC_CTRL2__PH_SUPPORT_MASK                                                                      0x003C0000L
+#define BIFC_MISC_CTRL2__GMI_FAIL_REQ_RTS_MASK_MASK                                                           0x00400000L
+#define BIFC_MISC_CTRL2__NBIF_AERRPT_BACKPERSURE_EN_MASK                                                      0x00800000L
+#define BIFC_MISC_CTRL2__DACCFE_SUPPORT_ATOMIC_NR_EN_MASK                                                     0x01000000L
+#define BIFC_MISC_CTRL2__DACCBE_SUPPORT_ATOMIC_NR_EN_MASK                                                     0x02000000L
+//BME_DUMMY_CNTL_0
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F0__SHIFT                                                     0x0
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F1__SHIFT                                                     0x2
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F2__SHIFT                                                     0x4
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F3__SHIFT                                                     0x6
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F4__SHIFT                                                     0x8
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F5__SHIFT                                                     0xa
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F6__SHIFT                                                     0xc
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F7__SHIFT                                                     0xe
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F0__SHIFT                                                     0x10
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F1__SHIFT                                                     0x12
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F2__SHIFT                                                     0x14
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F3__SHIFT                                                     0x16
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F4__SHIFT                                                     0x18
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F5__SHIFT                                                     0x1a
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F6__SHIFT                                                     0x1c
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F7__SHIFT                                                     0x1e
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F0_MASK                                                       0x00000003L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F1_MASK                                                       0x0000000CL
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F2_MASK                                                       0x00000030L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F3_MASK                                                       0x000000C0L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F4_MASK                                                       0x00000300L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F5_MASK                                                       0x00000C00L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F6_MASK                                                       0x00003000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F7_MASK                                                       0x0000C000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F0_MASK                                                       0x00030000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F1_MASK                                                       0x000C0000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F2_MASK                                                       0x00300000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F3_MASK                                                       0x00C00000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F4_MASK                                                       0x03000000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F5_MASK                                                       0x0C000000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F6_MASK                                                       0x30000000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV1_F7_MASK                                                       0xC0000000L
+//BME_DUMMY_CNTL_1
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F0__SHIFT                                                     0x0
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F1__SHIFT                                                     0x2
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F2__SHIFT                                                     0x4
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F3__SHIFT                                                     0x6
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F4__SHIFT                                                     0x8
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F5__SHIFT                                                     0xa
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F6__SHIFT                                                     0xc
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F7__SHIFT                                                     0xe
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F0_MASK                                                       0x00000003L
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F1_MASK                                                       0x0000000CL
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F2_MASK                                                       0x00000030L
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F3_MASK                                                       0x000000C0L
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F4_MASK                                                       0x00000300L
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F5_MASK                                                       0x00000C00L
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F6_MASK                                                       0x00003000L
+#define BME_DUMMY_CNTL_1__BME_DUMMY_RSPSTS_DEV2_F7_MASK                                                       0x0000C000L
+//BIFC_THT_CNTL
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_RD_VC0__SHIFT                                                         0x0
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC0__SHIFT                                                         0x4
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC1__SHIFT                                                         0x8
+#define BIFC_THT_CNTL__UR_OVRD_FOR_ECRC_EN__SHIFT                                                             0x10
+#define BIFC_THT_CNTL__THT_NTB_VC0_APER0_ADSC_PUSH_DIS__SHIFT                                                 0x18
+#define BIFC_THT_CNTL__THT_NTB_VC0_OTHAPER_ADSC_PUSH_DIS__SHIFT                                               0x19
+#define BIFC_THT_CNTL__THT_NTB_VC1_APER0_ADSC_PUSH_DIS__SHIFT                                                 0x1a
+#define BIFC_THT_CNTL__THT_NTB_VC1_OTHAPER_ADSC_PUSH_DIS__SHIFT                                               0x1b
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_RD_VC0_MASK                                                           0x0000000FL
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC0_MASK                                                           0x000000F0L
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC1_MASK                                                           0x00000F00L
+#define BIFC_THT_CNTL__UR_OVRD_FOR_ECRC_EN_MASK                                                               0x00010000L
+#define BIFC_THT_CNTL__THT_NTB_VC0_APER0_ADSC_PUSH_DIS_MASK                                                   0x01000000L
+#define BIFC_THT_CNTL__THT_NTB_VC0_OTHAPER_ADSC_PUSH_DIS_MASK                                                 0x02000000L
+#define BIFC_THT_CNTL__THT_NTB_VC1_APER0_ADSC_PUSH_DIS_MASK                                                   0x04000000L
+#define BIFC_THT_CNTL__THT_NTB_VC1_OTHAPER_ADSC_PUSH_DIS_MASK                                                 0x08000000L
+//BIFC_HSTARB_CNTL
+#define BIFC_HSTARB_CNTL__SLVARB_MODE__SHIFT                                                                  0x0
+#define BIFC_HSTARB_CNTL__CFG_BLOCK_P_EN__SHIFT                                                               0x8
+#define BIFC_HSTARB_CNTL__SLVARB_MODE_MASK                                                                    0x00000003L
+#define BIFC_HSTARB_CNTL__CFG_BLOCK_P_EN_MASK                                                                 0x00000100L
+//BIFC_GSI_CNTL
+#define BIFC_GSI_CNTL__GSI_SDP_RSP_ARB_MODE__SHIFT                                                            0x0
+#define BIFC_GSI_CNTL__GSI_CPL_RSP_ARB_MODE__SHIFT                                                            0x2
+#define BIFC_GSI_CNTL__GSI_CPL_INTERLEAVING_EN__SHIFT                                                         0x6
+#define BIFC_GSI_CNTL__GSI_CPL_PCR_EP_CAUSE_UR_EN__SHIFT                                                      0x7
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_P_EP_CAUSE_UR_EN__SHIFT                                                    0x8
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_NP_EP_CAUSE_UR_EN__SHIFT                                                   0x9
+#define BIFC_GSI_CNTL__GSI_CPL_SST_EP_CAUSE_UR_EN__SHIFT                                                      0xa
+#define BIFC_GSI_CNTL__GSI_SDP_REQ_ARB_MODE__SHIFT                                                            0xb
+#define BIFC_GSI_CNTL__GSI_SMN_REQ_ARB_MODE__SHIFT                                                            0xd
+#define BIFC_GSI_CNTL__GSI_CPL_SST_ATOMIC_EP_CAUSE_UR_EN__SHIFT                                               0xf
+#define BIFC_GSI_CNTL__GSI_SMN_PARITY_CHK_BE_MSK__SHIFT                                                       0x10
+#define BIFC_GSI_CNTL__GSI_SMN_BURST_EN__SHIFT                                                                0x11
+#define BIFC_GSI_CNTL__GSI_SMN_256B_SPLIT_64B_EN__SHIFT                                                       0x12
+#define BIFC_GSI_CNTL__GSI_CPL_DIH_P_EP_CAUSE_UR_EN__SHIFT                                                    0x13
+#define BIFC_GSI_CNTL__GSI_CPL_DIH_NP_EP_CAUSE_UR_EN__SHIFT                                                   0x14
+#define BIFC_GSI_CNTL__SMN_PP_PIPE_ENABLE__SHIFT                                                              0x1b
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_FBFLUSH__SHIFT                                                    0x1c
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPFLUSH__SHIFT                                                   0x1d
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPRD__SHIFT                                                      0x1e
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_ALL__SHIFT                                                        0x1f
+#define BIFC_GSI_CNTL__GSI_SDP_RSP_ARB_MODE_MASK                                                              0x00000003L
+#define BIFC_GSI_CNTL__GSI_CPL_RSP_ARB_MODE_MASK                                                              0x0000003CL
+#define BIFC_GSI_CNTL__GSI_CPL_INTERLEAVING_EN_MASK                                                           0x00000040L
+#define BIFC_GSI_CNTL__GSI_CPL_PCR_EP_CAUSE_UR_EN_MASK                                                        0x00000080L
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_P_EP_CAUSE_UR_EN_MASK                                                      0x00000100L
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_NP_EP_CAUSE_UR_EN_MASK                                                     0x00000200L
+#define BIFC_GSI_CNTL__GSI_CPL_SST_EP_CAUSE_UR_EN_MASK                                                        0x00000400L
+#define BIFC_GSI_CNTL__GSI_SDP_REQ_ARB_MODE_MASK                                                              0x00001800L
+#define BIFC_GSI_CNTL__GSI_SMN_REQ_ARB_MODE_MASK                                                              0x00006000L
+#define BIFC_GSI_CNTL__GSI_CPL_SST_ATOMIC_EP_CAUSE_UR_EN_MASK                                                 0x00008000L
+#define BIFC_GSI_CNTL__GSI_SMN_PARITY_CHK_BE_MSK_MASK                                                         0x00010000L
+#define BIFC_GSI_CNTL__GSI_SMN_BURST_EN_MASK                                                                  0x00020000L
+#define BIFC_GSI_CNTL__GSI_SMN_256B_SPLIT_64B_EN_MASK                                                         0x00040000L
+#define BIFC_GSI_CNTL__GSI_CPL_DIH_P_EP_CAUSE_UR_EN_MASK                                                      0x00080000L
+#define BIFC_GSI_CNTL__GSI_CPL_DIH_NP_EP_CAUSE_UR_EN_MASK                                                     0x00100000L
+#define BIFC_GSI_CNTL__SMN_PP_PIPE_ENABLE_MASK                                                                0x08000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_FBFLUSH_MASK                                                      0x10000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPFLUSH_MASK                                                     0x20000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPRD_MASK                                                        0x40000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_ALL_MASK                                                          0x80000000L
+//BIFC_PCIEFUNC_CNTL
+#define BIFC_PCIEFUNC_CNTL__DMA_NON_PCIEFUNC_BUSDEVFUNC__SHIFT                                                0x0
+#define BIFC_PCIEFUNC_CNTL__MP1SYSHUBDATA_DRAM_IS_PCIEFUNC__SHIFT                                             0x10
+#define BIFC_PCIEFUNC_CNTL__DMA_NON_PCIEFUNC_BUSDEVFUNC_MASK                                                  0x0000FFFFL
+#define BIFC_PCIEFUNC_CNTL__MP1SYSHUBDATA_DRAM_IS_PCIEFUNC_MASK                                               0x00010000L
+//BIFC_PASID_CHECK_DIS
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F0__SHIFT                                                  0x0
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F1__SHIFT                                                  0x1
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F2__SHIFT                                                  0x2
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F3__SHIFT                                                  0x3
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F4__SHIFT                                                  0x4
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F5__SHIFT                                                  0x5
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F6__SHIFT                                                  0x6
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F7__SHIFT                                                  0x7
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F0__SHIFT                                                  0x8
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F1__SHIFT                                                  0x9
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F2__SHIFT                                                  0xa
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F3__SHIFT                                                  0xb
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F4__SHIFT                                                  0xc
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F0__SHIFT                                                  0x10
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F1__SHIFT                                                  0x11
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F2__SHIFT                                                  0x12
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F3__SHIFT                                                  0x13
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F4__SHIFT                                                  0x14
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F5__SHIFT                                                  0x15
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F6__SHIFT                                                  0x16
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F0_MASK                                                    0x00000001L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F1_MASK                                                    0x00000002L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F2_MASK                                                    0x00000004L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F3_MASK                                                    0x00000008L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F4_MASK                                                    0x00000010L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F5_MASK                                                    0x00000020L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F6_MASK                                                    0x00000040L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F7_MASK                                                    0x00000080L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F0_MASK                                                    0x00000100L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F1_MASK                                                    0x00000200L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F2_MASK                                                    0x00000400L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F3_MASK                                                    0x00000800L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV1_F4_MASK                                                    0x00001000L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F0_MASK                                                    0x00010000L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F1_MASK                                                    0x00020000L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F2_MASK                                                    0x00040000L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F3_MASK                                                    0x00080000L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F4_MASK                                                    0x00100000L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F5_MASK                                                    0x00200000L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV2_F6_MASK                                                    0x00400000L
+//BIFC_SDP_CNTL_0
+#define BIFC_SDP_CNTL_0__HRP_SDP_DISCON_HYSTERESIS__SHIFT                                                     0x0
+#define BIFC_SDP_CNTL_0__GSI_SDP_DISCON_HYSTERESIS__SHIFT                                                     0x8
+#define BIFC_SDP_CNTL_0__GMI_DNS_SDP_DISCON_HYSTERESIS__SHIFT                                                 0x10
+#define BIFC_SDP_CNTL_0__GMI_UPS_SDP_DISCON_HYSTERESIS__SHIFT                                                 0x18
+#define BIFC_SDP_CNTL_0__HRP_SDP_DISCON_HYSTERESIS_MASK                                                       0x000000FFL
+#define BIFC_SDP_CNTL_0__GSI_SDP_DISCON_HYSTERESIS_MASK                                                       0x0000FF00L
+#define BIFC_SDP_CNTL_0__GMI_DNS_SDP_DISCON_HYSTERESIS_MASK                                                   0x00FF0000L
+#define BIFC_SDP_CNTL_0__GMI_UPS_SDP_DISCON_HYSTERESIS_MASK                                                   0xFF000000L
+//BIFC_SDP_CNTL_1
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_DIS__SHIFT                                                            0x0
+#define BIFC_SDP_CNTL_1__GSI_SDP_DISCON_DIS__SHIFT                                                            0x1
+#define BIFC_SDP_CNTL_1__GMI_DNS_SDP_DISCON_DIS__SHIFT                                                        0x2
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_DIS__SHIFT                                                        0x3
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_DEV0__SHIFT                                          0x4
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_DEV1__SHIFT                                          0x5
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_DEV2__SHIFT                                          0x6
+#define BIFC_SDP_CNTL_1__NP_KEEP_GOING_STALL_P__SHIFT                                                         0x7
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_DEV0__SHIFT                                      0x8
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_DEV1__SHIFT                                      0x9
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_DEV2__SHIFT                                      0xa
+#define BIFC_SDP_CNTL_1__ATOMIC_STALL_BY_RDWR_EN__SHIFT                                                       0xb
+#define BIFC_SDP_CNTL_1__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT                                            0xc
+#define BIFC_SDP_CNTL_1__NBIF_OBFF_HW_URGENT_EARLY_WAKEUP_EN__SHIFT                                           0x1e
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_WAKEUP_BY_USB_EARLY_WAKEUP_EN__SHIFT                                     0x1f
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_DIS_MASK                                                              0x00000001L
+#define BIFC_SDP_CNTL_1__GSI_SDP_DISCON_DIS_MASK                                                              0x00000002L
+#define BIFC_SDP_CNTL_1__GMI_DNS_SDP_DISCON_DIS_MASK                                                          0x00000004L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_DIS_MASK                                                          0x00000008L
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_DEV0_MASK                                            0x00000010L
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_DEV1_MASK                                            0x00000020L
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_DEV2_MASK                                            0x00000040L
+#define BIFC_SDP_CNTL_1__NP_KEEP_GOING_STALL_P_MASK                                                           0x00000080L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_DEV0_MASK                                        0x00000100L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_DEV1_MASK                                        0x00000200L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_DEV2_MASK                                        0x00000400L
+#define BIFC_SDP_CNTL_1__ATOMIC_STALL_BY_RDWR_EN_MASK                                                         0x00000800L
+#define BIFC_SDP_CNTL_1__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK                                              0x00001000L
+#define BIFC_SDP_CNTL_1__NBIF_OBFF_HW_URGENT_EARLY_WAKEUP_EN_MASK                                             0x40000000L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_WAKEUP_BY_USB_EARLY_WAKEUP_EN_MASK                                       0x80000000L
+//BIFC_PASID_STS
+#define BIFC_PASID_STS__PASID_STS__SHIFT                                                                      0x0
+#define BIFC_PASID_STS__PASID_STS_MASK                                                                        0x0000000FL
+//BIFC_ATHUB_ACT_CNTL
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_TYPE__SHIFT                                                0x0
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_SLFR_DATAERR_RSP_STS_TYPE__SHIFT                                           0x3
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_REQ_DROP_DIS__SHIFT                                                0x8
+#define BIFC_ATHUB_ACT_CNTL__GSI_ATHUB_ACT_FLUSH_TRIGGER__SHIFT                                               0x9
+#define BIFC_ATHUB_ACT_CNTL__GMI_ATHUB_ACT_FLUSH_TRIGGER__SHIFT                                               0xa
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_SST_PP_REQ_DROP_EN__SHIFT                                          0xb
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_OVRD_EN__SHIFT                                             0xc
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_TYPE_MASK                                                  0x00000007L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_SLFR_DATAERR_RSP_STS_TYPE_MASK                                             0x00000038L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_REQ_DROP_DIS_MASK                                                  0x00000100L
+#define BIFC_ATHUB_ACT_CNTL__GSI_ATHUB_ACT_FLUSH_TRIGGER_MASK                                                 0x00000200L
+#define BIFC_ATHUB_ACT_CNTL__GMI_ATHUB_ACT_FLUSH_TRIGGER_MASK                                                 0x00000400L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_SST_PP_REQ_DROP_EN_MASK                                            0x00000800L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_OVRD_EN_MASK                                               0x00001000L
+//BIFC_PERF_CNTL_0
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_EN__SHIFT                                                          0x0
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_EN__SHIFT                                                          0x1
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_RESET__SHIFT                                                       0x8
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_RESET__SHIFT                                                       0x9
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_SEL__SHIFT                                                         0x10
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_SEL__SHIFT                                                         0x18
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_EN_MASK                                                            0x00000001L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_EN_MASK                                                            0x00000002L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_RESET_MASK                                                         0x00000100L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_RESET_MASK                                                         0x00000200L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_SEL_MASK                                                           0x007F0000L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_SEL_MASK                                                           0x7F000000L
+//BIFC_PERF_CNTL_1
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_SEL__SHIFT                                                          0x0
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_EN__SHIFT                                                           0x9
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_RESET__SHIFT                                                        0xa
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_SEL__SHIFT                                                          0x10
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_EN__SHIFT                                                           0x1a
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_RESET__SHIFT                                                        0x1b
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_SEL_MASK                                                            0x000001FFL
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_EN_MASK                                                             0x00000200L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_RESET_MASK                                                          0x00000400L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_SEL_MASK                                                            0x03FF0000L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_EN_MASK                                                             0x04000000L
+#define BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_RESET_MASK                                                          0x08000000L
+//BIFC_PERF_CNT_MMIO_RD_L32BIT
+#define BIFC_PERF_CNT_MMIO_RD_L32BIT__PERF_CNT_MMIO_RD_VALUE_L32BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_RD_L32BIT__PERF_CNT_MMIO_RD_VALUE_L32BIT_MASK                                      0xFFFFFFFFL
+//BIFC_PERF_CNT_MMIO_WR_L32BIT
+#define BIFC_PERF_CNT_MMIO_WR_L32BIT__PERF_CNT_MMIO_WR_VALUE_L32BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_WR_L32BIT__PERF_CNT_MMIO_WR_VALUE_L32BIT_MASK                                      0xFFFFFFFFL
+//BIFC_PERF_CNT_DMA_RD_L32BIT
+#define BIFC_PERF_CNT_DMA_RD_L32BIT__PERF_CNT_DMA_RD_VALUE_L32BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_RD_L32BIT__PERF_CNT_DMA_RD_VALUE_L32BIT_MASK                                        0xFFFFFFFFL
+//BIFC_PERF_CNT_DMA_WR_L32BIT
+#define BIFC_PERF_CNT_DMA_WR_L32BIT__PERF_CNT_DMA_WR_VALUE_L32BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_WR_L32BIT__PERF_CNT_DMA_WR_VALUE_L32BIT_MASK                                        0xFFFFFFFFL
+//NBIF_REGIF_ERRSET_CTRL
+#define NBIF_REGIF_ERRSET_CTRL__DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                         0x0
+#define NBIF_REGIF_ERRSET_CTRL__DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                           0x00000001L
+//BIFC_SDP_CNTL_2
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS__SHIFT                                                    0x0
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_H__SHIFT                                                  0x8
+#define BIFC_SDP_CNTL_2__HRP_SDP_DISCON_HYSTERESIS_H__SHIFT                                                   0x10
+#define BIFC_SDP_CNTL_2__GSI_SDP_DISCON_HYSTERESIS_H__SHIFT                                                   0x18
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_MASK                                                      0x000000FFL
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_H_MASK                                                    0x00000F00L
+#define BIFC_SDP_CNTL_2__HRP_SDP_DISCON_HYSTERESIS_H_MASK                                                     0x000F0000L
+#define BIFC_SDP_CNTL_2__GSI_SDP_DISCON_HYSTERESIS_H_MASK                                                     0x0F000000L
+//NBIF_PGMST_CTRL
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_HYSTERESIS__SHIFT                                                        0x0
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_EN__SHIFT                                                                0x8
+#define NBIF_PGMST_CTRL__NBIF_CFG_IDLENESS_COUNT_EN__SHIFT                                                    0xa
+#define NBIF_PGMST_CTRL__NBIF_CFG_FW_PG_EXIT_EN__SHIFT                                                        0xe
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_HYSTERESIS_MASK                                                          0x000000FFL
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_EN_MASK                                                                  0x00000100L
+#define NBIF_PGMST_CTRL__NBIF_CFG_IDLENESS_COUNT_EN_MASK                                                      0x00003C00L
+#define NBIF_PGMST_CTRL__NBIF_CFG_FW_PG_EXIT_EN_MASK                                                          0x0000C000L
+//NBIF_PGSLV_CTRL
+#define NBIF_PGSLV_CTRL__NBIF_CFG_IDLE_HYSTERESIS__SHIFT                                                      0x0
+#define NBIF_PGSLV_CTRL__NBIF_CFG_IDLE_HYSTERESIS_MASK                                                        0x0000001FL
+//NBIF_PG_MISC_CTRL
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_0_IDLE_HYSTERESIS__SHIFT                                          0x0
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_1_IDLE_HYSTERESIS__SHIFT                                          0x5
+#define NBIF_PG_MISC_CTRL__NBIF_PG_ENDP_D3_ONLY__SHIFT                                                        0xa
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM1__SHIFT                                                           0xd
+#define NBIF_PG_MISC_CTRL__NBIF_PG_DS_ALLOW_DIS__SHIFT                                                        0xe
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM2__SHIFT                                                           0x10
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_REFCLK_CYCLE_FOR_200NS__SHIFT                                             0x18
+#define NBIF_PG_MISC_CTRL__NBIF_PG_PCIE_NBIF_LD_MASK__SHIFT                                                   0x1e
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_PG_EXIT_OVERRIDE__SHIFT                                                   0x1f
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_0_IDLE_HYSTERESIS_MASK                                            0x0000001FL
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_1_IDLE_HYSTERESIS_MASK                                            0x000003E0L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_ENDP_D3_ONLY_MASK                                                          0x00000400L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM1_MASK                                                             0x00002000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_DS_ALLOW_DIS_MASK                                                          0x00004000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM2_MASK                                                             0x00010000L
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_REFCLK_CYCLE_FOR_200NS_MASK                                               0x3F000000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_PCIE_NBIF_LD_MASK_MASK                                                     0x40000000L
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_PG_EXIT_OVERRIDE_MASK                                                     0x80000000L
+//NBIF_HST_MISC_CTRL
+//SMN_MST_EP_CNTL3
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF0__SHIFT                                                0x0
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF1__SHIFT                                                0x1
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF2__SHIFT                                                0x2
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF3__SHIFT                                                0x3
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF4__SHIFT                                                0x4
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF5__SHIFT                                                0x5
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF6__SHIFT                                                0x6
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF7__SHIFT                                                0x7
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF0__SHIFT                                                0x8
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF1__SHIFT                                                0x9
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF2__SHIFT                                                0xa
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF3__SHIFT                                                0xb
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF4__SHIFT                                                0xc
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF5__SHIFT                                                0xd
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF6__SHIFT                                                0xe
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF7__SHIFT                                                0xf
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF0__SHIFT                                                0x10
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF1__SHIFT                                                0x11
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF2__SHIFT                                                0x12
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF3__SHIFT                                                0x13
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF4__SHIFT                                                0x14
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF5__SHIFT                                                0x15
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF6__SHIFT                                                0x16
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF7__SHIFT                                                0x17
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF0_MASK                                                  0x00000001L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF1_MASK                                                  0x00000002L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF2_MASK                                                  0x00000004L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF3_MASK                                                  0x00000008L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF4_MASK                                                  0x00000010L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF5_MASK                                                  0x00000020L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF6_MASK                                                  0x00000040L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF7_MASK                                                  0x00000080L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF0_MASK                                                  0x00000100L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF1_MASK                                                  0x00000200L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF2_MASK                                                  0x00000400L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF3_MASK                                                  0x00000800L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF4_MASK                                                  0x00001000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF5_MASK                                                  0x00002000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF6_MASK                                                  0x00004000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV1_PF7_MASK                                                  0x00008000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF0_MASK                                                  0x00010000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF1_MASK                                                  0x00020000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF2_MASK                                                  0x00040000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF3_MASK                                                  0x00080000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF4_MASK                                                  0x00100000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF5_MASK                                                  0x00200000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF6_MASK                                                  0x00400000L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV2_PF7_MASK                                                  0x00800000L
+//SMN_MST_EP_CNTL4
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF0__SHIFT                                                0x0
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF1__SHIFT                                                0x1
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF2__SHIFT                                                0x2
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF3__SHIFT                                                0x3
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF4__SHIFT                                                0x4
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF5__SHIFT                                                0x5
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF6__SHIFT                                                0x6
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF7__SHIFT                                                0x7
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF0__SHIFT                                                0x8
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF1__SHIFT                                                0x9
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF2__SHIFT                                                0xa
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF3__SHIFT                                                0xb
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF4__SHIFT                                                0xc
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF5__SHIFT                                                0xd
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF6__SHIFT                                                0xe
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF7__SHIFT                                                0xf
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF0__SHIFT                                                0x10
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF1__SHIFT                                                0x11
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF2__SHIFT                                                0x12
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF3__SHIFT                                                0x13
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF4__SHIFT                                                0x14
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF5__SHIFT                                                0x15
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF6__SHIFT                                                0x16
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF7__SHIFT                                                0x17
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF0_MASK                                                  0x00000001L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF1_MASK                                                  0x00000002L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF2_MASK                                                  0x00000004L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF3_MASK                                                  0x00000008L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF4_MASK                                                  0x00000010L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF5_MASK                                                  0x00000020L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF6_MASK                                                  0x00000040L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF7_MASK                                                  0x00000080L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF0_MASK                                                  0x00000100L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF1_MASK                                                  0x00000200L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF2_MASK                                                  0x00000400L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF3_MASK                                                  0x00000800L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF4_MASK                                                  0x00001000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF5_MASK                                                  0x00002000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF6_MASK                                                  0x00004000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV1_PF7_MASK                                                  0x00008000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF0_MASK                                                  0x00010000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF1_MASK                                                  0x00020000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF2_MASK                                                  0x00040000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF3_MASK                                                  0x00080000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF4_MASK                                                  0x00100000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF5_MASK                                                  0x00200000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF6_MASK                                                  0x00400000L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV2_PF7_MASK                                                  0x00800000L
+//SMN_MST_CNTL1
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_UPS__SHIFT                                                    0x0
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV0__SHIFT                                               0x10
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV1__SHIFT                                               0x11
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV2__SHIFT                                               0x12
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_UPS_MASK                                                      0x00000001L
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV0_MASK                                                 0x00010000L
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV1_MASK                                                 0x00020000L
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV2_MASK                                                 0x00040000L
+//SMN_MST_EP_CNTL5
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF0__SHIFT                                         0x0
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF1__SHIFT                                         0x1
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF2__SHIFT                                         0x2
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF3__SHIFT                                         0x3
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF4__SHIFT                                         0x4
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF5__SHIFT                                         0x5
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF6__SHIFT                                         0x6
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF7__SHIFT                                         0x7
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF0__SHIFT                                         0x8
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF1__SHIFT                                         0x9
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF2__SHIFT                                         0xa
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF3__SHIFT                                         0xb
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF4__SHIFT                                         0xc
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF5__SHIFT                                         0xd
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF6__SHIFT                                         0xe
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF7__SHIFT                                         0xf
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF0__SHIFT                                         0x10
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF1__SHIFT                                         0x11
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF2__SHIFT                                         0x12
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF3__SHIFT                                         0x13
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF4__SHIFT                                         0x14
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF5__SHIFT                                         0x15
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF6__SHIFT                                         0x16
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF7__SHIFT                                         0x17
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF0_MASK                                           0x00000001L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF1_MASK                                           0x00000002L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF2_MASK                                           0x00000004L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF3_MASK                                           0x00000008L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF4_MASK                                           0x00000010L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF5_MASK                                           0x00000020L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF6_MASK                                           0x00000040L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF7_MASK                                           0x00000080L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF0_MASK                                           0x00000100L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF1_MASK                                           0x00000200L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF2_MASK                                           0x00000400L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF3_MASK                                           0x00000800L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF4_MASK                                           0x00001000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF5_MASK                                           0x00002000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF6_MASK                                           0x00004000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV1_PF7_MASK                                           0x00008000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF0_MASK                                           0x00010000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF1_MASK                                           0x00020000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF2_MASK                                           0x00040000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF3_MASK                                           0x00080000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF4_MASK                                           0x00100000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF5_MASK                                           0x00200000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF6_MASK                                           0x00400000L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV2_PF7_MASK                                           0x00800000L
+//BIF_SELFRING_BUFFER_VID
+#define BIF_SELFRING_BUFFER_VID__DOORBELL_MONITOR_CID__SHIFT                                                  0x0
+#define BIF_SELFRING_BUFFER_VID__RAS_CNTLR_INTR_CID__SHIFT                                                    0x8
+#define BIF_SELFRING_BUFFER_VID__RAS_ATHUB_ERR_EVENT_INTR_CID__SHIFT                                          0x10
+#define BIF_SELFRING_BUFFER_VID__DOORBELL_MONITOR_CID_MASK                                                    0x000000FFL
+#define BIF_SELFRING_BUFFER_VID__RAS_CNTLR_INTR_CID_MASK                                                      0x0000FF00L
+#define BIF_SELFRING_BUFFER_VID__RAS_ATHUB_ERR_EVENT_INTR_CID_MASK                                            0x00FF0000L
+//BIF_SELFRING_VECTOR_CNTL
+#define BIF_SELFRING_VECTOR_CNTL__MISC_DB_MNTR_INTR_DIS__SHIFT                                                0x0
+#define BIF_SELFRING_VECTOR_CNTL__DB_MNTR_TS_FROM__SHIFT                                                      0x1
+#define BIF_SELFRING_VECTOR_CNTL__MISC_DB_MNTR_INTR_DIS_MASK                                                  0x00000001L
+#define BIF_SELFRING_VECTOR_CNTL__DB_MNTR_TS_FROM_MASK                                                        0x00000002L
+//NBIF_STRAP_WRITE_CTRL
+#define NBIF_STRAP_WRITE_CTRL__NBIF_STRAP_WRITE_ONCE_ENABLE__SHIFT                                            0x0
+#define NBIF_STRAP_WRITE_CTRL__NBIF_STRAP_WRITE_ONCE_ENABLE_MASK                                              0x00000001L
+//NBIF_INTX_DSTATE_MISC_CNTL
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_EP__SHIFT                                      0x0
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_DN__SHIFT                                      0x1
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_SWUS__SHIFT                                    0x2
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_EP__SHIFT                                         0x3
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_DN__SHIFT                                         0x4
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_EP__SHIFT                                                     0x5
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_DN__SHIFT                                                     0x6
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_SWUS__SHIFT                                                   0x7
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_EP_MASK                                        0x00000001L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_DN_MASK                                        0x00000002L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_SWUS_MASK                                      0x00000004L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_EP_MASK                                           0x00000008L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_DN_MASK                                           0x00000010L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_EP_MASK                                                       0x00000020L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_DN_MASK                                                       0x00000040L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_SWUS_MASK                                                     0x00000080L
+//NBIF_PENDING_MISC_CNTL
+#define NBIF_PENDING_MISC_CNTL__FLR_MST_PEND_CHK_DIS__SHIFT                                                   0x0
+#define NBIF_PENDING_MISC_CNTL__FLR_SLV_PEND_CHK_DIS__SHIFT                                                   0x1
+#define NBIF_PENDING_MISC_CNTL__FLR_MST_PEND_CHK_DIS_MASK                                                     0x00000001L
+#define NBIF_PENDING_MISC_CNTL__FLR_SLV_PEND_CHK_DIS_MASK                                                     0x00000002L
+//BIF_GMI_WRR_WEIGHT
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_COUNTER_MODE__SHIFT                                               0x1d
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_MODE__SHIFT                                                       0x1e
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_SIZE_MODE__SHIFT                                                  0x1f
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_COUNTER_MODE_MASK                                                 0x20000000L
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_MODE_MASK                                                         0x40000000L
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_SIZE_MODE_MASK                                                    0x80000000L
+//BIF_GMI_WRR_WEIGHT2
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY0_WEIGHT__SHIFT                                                     0x0
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY1_WEIGHT__SHIFT                                                     0x8
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY2_WEIGHT__SHIFT                                                     0x10
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY3_WEIGHT__SHIFT                                                     0x18
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY0_WEIGHT_MASK                                                       0x000000FFL
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY1_WEIGHT_MASK                                                       0x0000FF00L
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY2_WEIGHT_MASK                                                       0x00FF0000L
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY3_WEIGHT_MASK                                                       0xFF000000L
+//BIF_GMI_WRR_WEIGHT3
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY4_WEIGHT__SHIFT                                                     0x0
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY5_WEIGHT__SHIFT                                                     0x8
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY6_WEIGHT__SHIFT                                                     0x10
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY7_WEIGHT__SHIFT                                                     0x18
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY4_WEIGHT_MASK                                                       0x000000FFL
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY5_WEIGHT_MASK                                                       0x0000FF00L
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY6_WEIGHT_MASK                                                       0x00FF0000L
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY7_WEIGHT_MASK                                                       0xFF000000L
+//NBIF_PWRBRK_REQUEST
+#define NBIF_PWRBRK_REQUEST__NBIF_PWRBRK_REQUEST__SHIFT                                                       0x0
+#define NBIF_PWRBRK_REQUEST__NBIF_PWRBRK_REQUEST_MASK                                                         0x00000001L
+//BIF_ATOMIC_ERR_LOG_DEV0_F0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_OPCODE_DEV0_F1__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_REQEN_LOW_DEV0_F1__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_LENGTH_DEV0_F1__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_NR_DEV0_F1__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F1__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F1__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F1__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_NR_DEV0_F1__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_OPCODE_DEV0_F1_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_REQEN_LOW_DEV0_F1_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_LENGTH_DEV0_F1_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_NR_DEV0_F1_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F1_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F1_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F1_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_NR_DEV0_F1_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_OPCODE_DEV0_F2__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_REQEN_LOW_DEV0_F2__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_LENGTH_DEV0_F2__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_NR_DEV0_F2__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_OPCODE_DEV0_F2__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F2__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_LENGTH_DEV0_F2__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_NR_DEV0_F2__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_OPCODE_DEV0_F2_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_REQEN_LOW_DEV0_F2_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_LENGTH_DEV0_F2_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__UR_ATOMIC_NR_DEV0_F2_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_OPCODE_DEV0_F2_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F2_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_LENGTH_DEV0_F2_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F2__CLEAR_UR_ATOMIC_NR_DEV0_F2_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_OPCODE_DEV0_F3__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_REQEN_LOW_DEV0_F3__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_LENGTH_DEV0_F3__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_NR_DEV0_F3__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_OPCODE_DEV0_F3__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F3__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_LENGTH_DEV0_F3__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_NR_DEV0_F3__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_OPCODE_DEV0_F3_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_REQEN_LOW_DEV0_F3_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_LENGTH_DEV0_F3_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__UR_ATOMIC_NR_DEV0_F3_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_OPCODE_DEV0_F3_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F3_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_LENGTH_DEV0_F3_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F3__CLEAR_UR_ATOMIC_NR_DEV0_F3_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F4
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_OPCODE_DEV0_F4__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_REQEN_LOW_DEV0_F4__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_LENGTH_DEV0_F4__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_NR_DEV0_F4__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_OPCODE_DEV0_F4__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F4__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_LENGTH_DEV0_F4__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_NR_DEV0_F4__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_OPCODE_DEV0_F4_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_REQEN_LOW_DEV0_F4_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_LENGTH_DEV0_F4_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__UR_ATOMIC_NR_DEV0_F4_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_OPCODE_DEV0_F4_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F4_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_LENGTH_DEV0_F4_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F4__CLEAR_UR_ATOMIC_NR_DEV0_F4_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F5
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_OPCODE_DEV0_F5__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_REQEN_LOW_DEV0_F5__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_LENGTH_DEV0_F5__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_NR_DEV0_F5__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_OPCODE_DEV0_F5__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F5__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_LENGTH_DEV0_F5__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_NR_DEV0_F5__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_OPCODE_DEV0_F5_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_REQEN_LOW_DEV0_F5_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_LENGTH_DEV0_F5_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__UR_ATOMIC_NR_DEV0_F5_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_OPCODE_DEV0_F5_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F5_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_LENGTH_DEV0_F5_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F5__CLEAR_UR_ATOMIC_NR_DEV0_F5_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F6
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_OPCODE_DEV0_F6__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_REQEN_LOW_DEV0_F6__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_LENGTH_DEV0_F6__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_NR_DEV0_F6__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_OPCODE_DEV0_F6__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F6__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_LENGTH_DEV0_F6__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_NR_DEV0_F6__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_OPCODE_DEV0_F6_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_REQEN_LOW_DEV0_F6_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_LENGTH_DEV0_F6_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__UR_ATOMIC_NR_DEV0_F6_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_OPCODE_DEV0_F6_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F6_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_LENGTH_DEV0_F6_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F6__CLEAR_UR_ATOMIC_NR_DEV0_F6_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F7
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_OPCODE_DEV0_F7__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_REQEN_LOW_DEV0_F7__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_LENGTH_DEV0_F7__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_NR_DEV0_F7__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_OPCODE_DEV0_F7__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F7__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_LENGTH_DEV0_F7__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_NR_DEV0_F7__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_OPCODE_DEV0_F7_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_REQEN_LOW_DEV0_F7_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_LENGTH_DEV0_F7_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__UR_ATOMIC_NR_DEV0_F7_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_OPCODE_DEV0_F7_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F7_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_LENGTH_DEV0_F7_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F7__CLEAR_UR_ATOMIC_NR_DEV0_F7_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV1_F0
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_OPCODE_DEV1_F0__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_REQEN_LOW_DEV1_F0__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_LENGTH_DEV1_F0__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_NR_DEV1_F0__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_OPCODE_DEV1_F0__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV1_F0__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_LENGTH_DEV1_F0__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_NR_DEV1_F0__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_OPCODE_DEV1_F0_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_REQEN_LOW_DEV1_F0_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_LENGTH_DEV1_F0_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__UR_ATOMIC_NR_DEV1_F0_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_OPCODE_DEV1_F0_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV1_F0_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_LENGTH_DEV1_F0_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F0__CLEAR_UR_ATOMIC_NR_DEV1_F0_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV1_F1
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_OPCODE_DEV1_F1__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_REQEN_LOW_DEV1_F1__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_LENGTH_DEV1_F1__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_NR_DEV1_F1__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_OPCODE_DEV1_F1__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV1_F1__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_LENGTH_DEV1_F1__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_NR_DEV1_F1__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_OPCODE_DEV1_F1_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_REQEN_LOW_DEV1_F1_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_LENGTH_DEV1_F1_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__UR_ATOMIC_NR_DEV1_F1_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_OPCODE_DEV1_F1_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV1_F1_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_LENGTH_DEV1_F1_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV1_F1__CLEAR_UR_ATOMIC_NR_DEV1_F1_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV2_F0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_OPCODE_DEV2_F0__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_REQEN_LOW_DEV2_F0__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_LENGTH_DEV2_F0__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_NR_DEV2_F0__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_OPCODE_DEV2_F0__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F0__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_LENGTH_DEV2_F0__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_NR_DEV2_F0__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_OPCODE_DEV2_F0_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_REQEN_LOW_DEV2_F0_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_LENGTH_DEV2_F0_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__UR_ATOMIC_NR_DEV2_F0_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_OPCODE_DEV2_F0_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F0_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_LENGTH_DEV2_F0_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F0__CLEAR_UR_ATOMIC_NR_DEV2_F0_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV2_F1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_OPCODE_DEV2_F1__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_REQEN_LOW_DEV2_F1__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_LENGTH_DEV2_F1__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_NR_DEV2_F1__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_OPCODE_DEV2_F1__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F1__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_LENGTH_DEV2_F1__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_NR_DEV2_F1__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_OPCODE_DEV2_F1_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_REQEN_LOW_DEV2_F1_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_LENGTH_DEV2_F1_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__UR_ATOMIC_NR_DEV2_F1_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_OPCODE_DEV2_F1_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F1_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_LENGTH_DEV2_F1_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F1__CLEAR_UR_ATOMIC_NR_DEV2_F1_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV2_F2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_OPCODE_DEV2_F2__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_REQEN_LOW_DEV2_F2__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_LENGTH_DEV2_F2__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_NR_DEV2_F2__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_OPCODE_DEV2_F2__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F2__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_LENGTH_DEV2_F2__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_NR_DEV2_F2__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_OPCODE_DEV2_F2_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_REQEN_LOW_DEV2_F2_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_LENGTH_DEV2_F2_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__UR_ATOMIC_NR_DEV2_F2_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_OPCODE_DEV2_F2_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F2_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_LENGTH_DEV2_F2_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F2__CLEAR_UR_ATOMIC_NR_DEV2_F2_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV2_F3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_OPCODE_DEV2_F3__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_REQEN_LOW_DEV2_F3__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_LENGTH_DEV2_F3__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_NR_DEV2_F3__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_OPCODE_DEV2_F3__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F3__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_LENGTH_DEV2_F3__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_NR_DEV2_F3__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_OPCODE_DEV2_F3_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_REQEN_LOW_DEV2_F3_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_LENGTH_DEV2_F3_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__UR_ATOMIC_NR_DEV2_F3_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_OPCODE_DEV2_F3_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F3_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_LENGTH_DEV2_F3_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F3__CLEAR_UR_ATOMIC_NR_DEV2_F3_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV2_F4
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_OPCODE_DEV2_F4__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_REQEN_LOW_DEV2_F4__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_LENGTH_DEV2_F4__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_NR_DEV2_F4__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_OPCODE_DEV2_F4__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F4__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_LENGTH_DEV2_F4__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_NR_DEV2_F4__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_OPCODE_DEV2_F4_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_REQEN_LOW_DEV2_F4_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_LENGTH_DEV2_F4_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__UR_ATOMIC_NR_DEV2_F4_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_OPCODE_DEV2_F4_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F4_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_LENGTH_DEV2_F4_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F4__CLEAR_UR_ATOMIC_NR_DEV2_F4_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV2_F5
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_OPCODE_DEV2_F5__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_REQEN_LOW_DEV2_F5__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_LENGTH_DEV2_F5__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_NR_DEV2_F5__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_OPCODE_DEV2_F5__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F5__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_LENGTH_DEV2_F5__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_NR_DEV2_F5__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_OPCODE_DEV2_F5_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_REQEN_LOW_DEV2_F5_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_LENGTH_DEV2_F5_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__UR_ATOMIC_NR_DEV2_F5_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_OPCODE_DEV2_F5_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F5_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_LENGTH_DEV2_F5_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F5__CLEAR_UR_ATOMIC_NR_DEV2_F5_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV2_F6
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_OPCODE_DEV2_F6__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_REQEN_LOW_DEV2_F6__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_LENGTH_DEV2_F6__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_NR_DEV2_F6__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_OPCODE_DEV2_F6__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F6__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_LENGTH_DEV2_F6__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_NR_DEV2_F6__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_OPCODE_DEV2_F6_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_REQEN_LOW_DEV2_F6_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_LENGTH_DEV2_F6_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__UR_ATOMIC_NR_DEV2_F6_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_OPCODE_DEV2_F6_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_REQEN_LOW_DEV2_F6_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_LENGTH_DEV2_F6_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV2_F6__CLEAR_UR_ATOMIC_NR_DEV2_F6_MASK                                           0x00080000L
+//BIF_DMA_MP4_ERR_LOG
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_VC4_NON_DVM_ERR__SHIFT                                                    0x0
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_ATOMIC_REQEN_LOW_ERR__SHIFT                                               0x1
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_VC4_NON_DVM_ERR__SHIFT                                              0x10
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_ATOMIC_REQEN_LOW_ERR__SHIFT                                         0x11
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_VC4_NON_DVM_ERR_MASK                                                      0x00000001L
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_ATOMIC_REQEN_LOW_ERR_MASK                                                 0x00000002L
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_VC4_NON_DVM_ERR_MASK                                                0x00010000L
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_ATOMIC_REQEN_LOW_ERR_MASK                                           0x00020000L
+//BIF_PASID_ERR_LOG
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F0__SHIFT                                                           0x0
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F1__SHIFT                                                           0x1
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F2__SHIFT                                                           0x2
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F3__SHIFT                                                           0x3
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F4__SHIFT                                                           0x4
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F5__SHIFT                                                           0x5
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F6__SHIFT                                                           0x6
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F7__SHIFT                                                           0x7
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F0__SHIFT                                                           0x8
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F1__SHIFT                                                           0x9
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F2__SHIFT                                                           0xa
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F3__SHIFT                                                           0xb
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F4__SHIFT                                                           0xc
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F0__SHIFT                                                           0x10
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F1__SHIFT                                                           0x11
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F2__SHIFT                                                           0x12
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F3__SHIFT                                                           0x13
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F4__SHIFT                                                           0x14
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F5__SHIFT                                                           0x15
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F6__SHIFT                                                           0x16
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F0_MASK                                                             0x00000001L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F1_MASK                                                             0x00000002L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F2_MASK                                                             0x00000004L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F3_MASK                                                             0x00000008L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F4_MASK                                                             0x00000010L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F5_MASK                                                             0x00000020L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F6_MASK                                                             0x00000040L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F7_MASK                                                             0x00000080L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F0_MASK                                                             0x00000100L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F1_MASK                                                             0x00000200L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F2_MASK                                                             0x00000400L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F3_MASK                                                             0x00000800L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV1_F4_MASK                                                             0x00001000L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F0_MASK                                                             0x00010000L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F1_MASK                                                             0x00020000L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F2_MASK                                                             0x00040000L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F3_MASK                                                             0x00080000L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F4_MASK                                                             0x00100000L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F5_MASK                                                             0x00200000L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV2_F6_MASK                                                             0x00400000L
+//BIF_PASID_ERR_CLR
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F0__SHIFT                                                       0x0
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F1__SHIFT                                                       0x1
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F2__SHIFT                                                       0x2
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F3__SHIFT                                                       0x3
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F4__SHIFT                                                       0x4
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F5__SHIFT                                                       0x5
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F6__SHIFT                                                       0x6
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F7__SHIFT                                                       0x7
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F0__SHIFT                                                       0x8
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F1__SHIFT                                                       0x9
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F2__SHIFT                                                       0xa
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F3__SHIFT                                                       0xb
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F4__SHIFT                                                       0xc
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F0__SHIFT                                                       0x10
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F1__SHIFT                                                       0x11
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F2__SHIFT                                                       0x12
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F3__SHIFT                                                       0x13
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F4__SHIFT                                                       0x14
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F5__SHIFT                                                       0x15
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F6__SHIFT                                                       0x16
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F0_MASK                                                         0x00000001L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F1_MASK                                                         0x00000002L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F2_MASK                                                         0x00000004L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F3_MASK                                                         0x00000008L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F4_MASK                                                         0x00000010L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F5_MASK                                                         0x00000020L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F6_MASK                                                         0x00000040L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F7_MASK                                                         0x00000080L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F0_MASK                                                         0x00000100L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F1_MASK                                                         0x00000200L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F2_MASK                                                         0x00000400L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F3_MASK                                                         0x00000800L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV1_F4_MASK                                                         0x00001000L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F0_MASK                                                         0x00010000L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F1_MASK                                                         0x00020000L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F2_MASK                                                         0x00040000L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F3_MASK                                                         0x00080000L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F4_MASK                                                         0x00100000L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F5_MASK                                                         0x00200000L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV2_F6_MASK                                                         0x00400000L
+//OBFF_EMU_CFG
+#define OBFF_EMU_CFG__OBFF_EMU_INTR_EN__SHIFT                                                                 0x0
+#define OBFF_EMU_CFG__OBFF_EMU_INTR_EN_MASK                                                                   0x00000001L
+//EP0_INTR_URGENT_CAP
+#define EP0_INTR_URGENT_CAP__EP0_F0_INTR_URGENT_MODE__SHIFT                                                   0x0
+#define EP0_INTR_URGENT_CAP__EP0_F0_INTR_NONOBFFCLIENT__SHIFT                                                 0x2
+#define EP0_INTR_URGENT_CAP__EP0_F1_INTR_URGENT_MODE__SHIFT                                                   0x3
+#define EP0_INTR_URGENT_CAP__EP0_F1_INTR_NONOBFFCLIENT__SHIFT                                                 0x5
+#define EP0_INTR_URGENT_CAP__EP0_F2_INTR_URGENT_MODE__SHIFT                                                   0x6
+#define EP0_INTR_URGENT_CAP__EP0_F2_INTR_NONOBFFCLIENT__SHIFT                                                 0x8
+#define EP0_INTR_URGENT_CAP__EP0_F3_INTR_URGENT_MODE__SHIFT                                                   0x9
+#define EP0_INTR_URGENT_CAP__EP0_F3_INTR_NONOBFFCLIENT__SHIFT                                                 0xb
+#define EP0_INTR_URGENT_CAP__EP0_F4_INTR_URGENT_MODE__SHIFT                                                   0xc
+#define EP0_INTR_URGENT_CAP__EP0_F4_INTR_NONOBFFCLIENT__SHIFT                                                 0xe
+#define EP0_INTR_URGENT_CAP__EP0_F5_INTR_URGENT_MODE__SHIFT                                                   0xf
+#define EP0_INTR_URGENT_CAP__EP0_F5_INTR_NONOBFFCLIENT__SHIFT                                                 0x11
+#define EP0_INTR_URGENT_CAP__EP0_F6_INTR_URGENT_MODE__SHIFT                                                   0x12
+#define EP0_INTR_URGENT_CAP__EP0_F6_INTR_NONOBFFCLIENT__SHIFT                                                 0x14
+#define EP0_INTR_URGENT_CAP__EP0_F7_INTR_URGENT_MODE__SHIFT                                                   0x15
+#define EP0_INTR_URGENT_CAP__EP0_F7_INTR_NONOBFFCLIENT__SHIFT                                                 0x17
+#define EP0_INTR_URGENT_CAP__EP0_F0_INTR_URGENT_MODE_MASK                                                     0x00000003L
+#define EP0_INTR_URGENT_CAP__EP0_F0_INTR_NONOBFFCLIENT_MASK                                                   0x00000004L
+#define EP0_INTR_URGENT_CAP__EP0_F1_INTR_URGENT_MODE_MASK                                                     0x00000018L
+#define EP0_INTR_URGENT_CAP__EP0_F1_INTR_NONOBFFCLIENT_MASK                                                   0x00000020L
+#define EP0_INTR_URGENT_CAP__EP0_F2_INTR_URGENT_MODE_MASK                                                     0x000000C0L
+#define EP0_INTR_URGENT_CAP__EP0_F2_INTR_NONOBFFCLIENT_MASK                                                   0x00000100L
+#define EP0_INTR_URGENT_CAP__EP0_F3_INTR_URGENT_MODE_MASK                                                     0x00000600L
+#define EP0_INTR_URGENT_CAP__EP0_F3_INTR_NONOBFFCLIENT_MASK                                                   0x00000800L
+#define EP0_INTR_URGENT_CAP__EP0_F4_INTR_URGENT_MODE_MASK                                                     0x00003000L
+#define EP0_INTR_URGENT_CAP__EP0_F4_INTR_NONOBFFCLIENT_MASK                                                   0x00004000L
+#define EP0_INTR_URGENT_CAP__EP0_F5_INTR_URGENT_MODE_MASK                                                     0x00018000L
+#define EP0_INTR_URGENT_CAP__EP0_F5_INTR_NONOBFFCLIENT_MASK                                                   0x00020000L
+#define EP0_INTR_URGENT_CAP__EP0_F6_INTR_URGENT_MODE_MASK                                                     0x000C0000L
+#define EP0_INTR_URGENT_CAP__EP0_F6_INTR_NONOBFFCLIENT_MASK                                                   0x00100000L
+#define EP0_INTR_URGENT_CAP__EP0_F7_INTR_URGENT_MODE_MASK                                                     0x00600000L
+#define EP0_INTR_URGENT_CAP__EP0_F7_INTR_NONOBFFCLIENT_MASK                                                   0x00800000L
+//EP1_INTR_URGENT_CAP
+#define EP1_INTR_URGENT_CAP__EP1_F0_INTR_URGENT_MODE__SHIFT                                                   0x0
+#define EP1_INTR_URGENT_CAP__EP1_F0_INTR_NONOBFFCLIENT__SHIFT                                                 0x2
+#define EP1_INTR_URGENT_CAP__EP1_F1_INTR_URGENT_MODE__SHIFT                                                   0x3
+#define EP1_INTR_URGENT_CAP__EP1_F1_INTR_NONOBFFCLIENT__SHIFT                                                 0x5
+#define EP1_INTR_URGENT_CAP__EP1_F2_INTR_URGENT_MODE__SHIFT                                                   0x6
+#define EP1_INTR_URGENT_CAP__EP1_F2_INTR_NONOBFFCLIENT__SHIFT                                                 0x8
+#define EP1_INTR_URGENT_CAP__EP1_F3_INTR_URGENT_MODE__SHIFT                                                   0x9
+#define EP1_INTR_URGENT_CAP__EP1_F3_INTR_NONOBFFCLIENT__SHIFT                                                 0xb
+#define EP1_INTR_URGENT_CAP__EP1_F4_INTR_URGENT_MODE__SHIFT                                                   0xc
+#define EP1_INTR_URGENT_CAP__EP1_F4_INTR_NONOBFFCLIENT__SHIFT                                                 0xe
+#define EP1_INTR_URGENT_CAP__EP1_F0_INTR_URGENT_MODE_MASK                                                     0x00000003L
+#define EP1_INTR_URGENT_CAP__EP1_F0_INTR_NONOBFFCLIENT_MASK                                                   0x00000004L
+#define EP1_INTR_URGENT_CAP__EP1_F1_INTR_URGENT_MODE_MASK                                                     0x00000018L
+#define EP1_INTR_URGENT_CAP__EP1_F1_INTR_NONOBFFCLIENT_MASK                                                   0x00000020L
+#define EP1_INTR_URGENT_CAP__EP1_F2_INTR_URGENT_MODE_MASK                                                     0x000000C0L
+#define EP1_INTR_URGENT_CAP__EP1_F2_INTR_NONOBFFCLIENT_MASK                                                   0x00000100L
+#define EP1_INTR_URGENT_CAP__EP1_F3_INTR_URGENT_MODE_MASK                                                     0x00000600L
+#define EP1_INTR_URGENT_CAP__EP1_F3_INTR_NONOBFFCLIENT_MASK                                                   0x00000800L
+#define EP1_INTR_URGENT_CAP__EP1_F4_INTR_URGENT_MODE_MASK                                                     0x00003000L
+#define EP1_INTR_URGENT_CAP__EP1_F4_INTR_NONOBFFCLIENT_MASK                                                   0x00004000L
+//EP2_INTR_URGENT_CAP
+#define EP2_INTR_URGENT_CAP__EP2_F0_INTR_URGENT_MODE__SHIFT                                                   0x0
+#define EP2_INTR_URGENT_CAP__EP2_F0_INTR_NONOBFFCLIENT__SHIFT                                                 0x2
+#define EP2_INTR_URGENT_CAP__EP2_F1_INTR_URGENT_MODE__SHIFT                                                   0x3
+#define EP2_INTR_URGENT_CAP__EP2_F1_INTR_NONOBFFCLIENT__SHIFT                                                 0x5
+#define EP2_INTR_URGENT_CAP__EP2_F2_INTR_URGENT_MODE__SHIFT                                                   0x6
+#define EP2_INTR_URGENT_CAP__EP2_F2_INTR_NONOBFFCLIENT__SHIFT                                                 0x8
+#define EP2_INTR_URGENT_CAP__EP2_F3_INTR_URGENT_MODE__SHIFT                                                   0x9
+#define EP2_INTR_URGENT_CAP__EP2_F3_INTR_NONOBFFCLIENT__SHIFT                                                 0xb
+#define EP2_INTR_URGENT_CAP__EP2_F4_INTR_URGENT_MODE__SHIFT                                                   0xc
+#define EP2_INTR_URGENT_CAP__EP2_F4_INTR_NONOBFFCLIENT__SHIFT                                                 0xe
+#define EP2_INTR_URGENT_CAP__EP2_F5_INTR_URGENT_MODE__SHIFT                                                   0xf
+#define EP2_INTR_URGENT_CAP__EP2_F5_INTR_NONOBFFCLIENT__SHIFT                                                 0x11
+#define EP2_INTR_URGENT_CAP__EP2_F6_INTR_URGENT_MODE__SHIFT                                                   0x12
+#define EP2_INTR_URGENT_CAP__EP2_F6_INTR_NONOBFFCLIENT__SHIFT                                                 0x14
+#define EP2_INTR_URGENT_CAP__EP2_F0_INTR_URGENT_MODE_MASK                                                     0x00000003L
+#define EP2_INTR_URGENT_CAP__EP2_F0_INTR_NONOBFFCLIENT_MASK                                                   0x00000004L
+#define EP2_INTR_URGENT_CAP__EP2_F1_INTR_URGENT_MODE_MASK                                                     0x00000018L
+#define EP2_INTR_URGENT_CAP__EP2_F1_INTR_NONOBFFCLIENT_MASK                                                   0x00000020L
+#define EP2_INTR_URGENT_CAP__EP2_F2_INTR_URGENT_MODE_MASK                                                     0x000000C0L
+#define EP2_INTR_URGENT_CAP__EP2_F2_INTR_NONOBFFCLIENT_MASK                                                   0x00000100L
+#define EP2_INTR_URGENT_CAP__EP2_F3_INTR_URGENT_MODE_MASK                                                     0x00000600L
+#define EP2_INTR_URGENT_CAP__EP2_F3_INTR_NONOBFFCLIENT_MASK                                                   0x00000800L
+#define EP2_INTR_URGENT_CAP__EP2_F4_INTR_URGENT_MODE_MASK                                                     0x00003000L
+#define EP2_INTR_URGENT_CAP__EP2_F4_INTR_NONOBFFCLIENT_MASK                                                   0x00004000L
+#define EP2_INTR_URGENT_CAP__EP2_F5_INTR_URGENT_MODE_MASK                                                     0x00018000L
+#define EP2_INTR_URGENT_CAP__EP2_F5_INTR_NONOBFFCLIENT_MASK                                                   0x00020000L
+#define EP2_INTR_URGENT_CAP__EP2_F6_INTR_URGENT_MODE_MASK                                                     0x000C0000L
+#define EP2_INTR_URGENT_CAP__EP2_F6_INTR_NONOBFFCLIENT_MASK                                                   0x00100000L
+//EP_PEND_BLOCK_MSK
+#define EP_PEND_BLOCK_MSK__EP0_F0_PEND_BLOCK_MSK__SHIFT                                                       0x0
+#define EP_PEND_BLOCK_MSK__EP0_F1_PEND_BLOCK_MSK__SHIFT                                                       0x1
+#define EP_PEND_BLOCK_MSK__EP0_F2_PEND_BLOCK_MSK__SHIFT                                                       0x2
+#define EP_PEND_BLOCK_MSK__EP0_F3_PEND_BLOCK_MSK__SHIFT                                                       0x3
+#define EP_PEND_BLOCK_MSK__EP0_F4_PEND_BLOCK_MSK__SHIFT                                                       0x4
+#define EP_PEND_BLOCK_MSK__EP0_F5_PEND_BLOCK_MSK__SHIFT                                                       0x5
+#define EP_PEND_BLOCK_MSK__EP0_F6_PEND_BLOCK_MSK__SHIFT                                                       0x6
+#define EP_PEND_BLOCK_MSK__EP0_F7_PEND_BLOCK_MSK__SHIFT                                                       0x7
+#define EP_PEND_BLOCK_MSK__EP1_F0_PEND_BLOCK_MSK__SHIFT                                                       0x8
+#define EP_PEND_BLOCK_MSK__EP1_F1_PEND_BLOCK_MSK__SHIFT                                                       0x9
+#define EP_PEND_BLOCK_MSK__EP1_F2_PEND_BLOCK_MSK__SHIFT                                                       0xa
+#define EP_PEND_BLOCK_MSK__EP1_F3_PEND_BLOCK_MSK__SHIFT                                                       0xb
+#define EP_PEND_BLOCK_MSK__EP1_F4_PEND_BLOCK_MSK__SHIFT                                                       0xc
+#define EP_PEND_BLOCK_MSK__EP2_F0_PEND_BLOCK_MSK__SHIFT                                                       0x10
+#define EP_PEND_BLOCK_MSK__EP2_F1_PEND_BLOCK_MSK__SHIFT                                                       0x11
+#define EP_PEND_BLOCK_MSK__EP2_F2_PEND_BLOCK_MSK__SHIFT                                                       0x12
+#define EP_PEND_BLOCK_MSK__EP2_F3_PEND_BLOCK_MSK__SHIFT                                                       0x13
+#define EP_PEND_BLOCK_MSK__EP2_F4_PEND_BLOCK_MSK__SHIFT                                                       0x14
+#define EP_PEND_BLOCK_MSK__EP2_F5_PEND_BLOCK_MSK__SHIFT                                                       0x15
+#define EP_PEND_BLOCK_MSK__EP2_F6_PEND_BLOCK_MSK__SHIFT                                                       0x16
+#define EP_PEND_BLOCK_MSK__EP0_F0_PEND_BLOCK_MSK_MASK                                                         0x00000001L
+#define EP_PEND_BLOCK_MSK__EP0_F1_PEND_BLOCK_MSK_MASK                                                         0x00000002L
+#define EP_PEND_BLOCK_MSK__EP0_F2_PEND_BLOCK_MSK_MASK                                                         0x00000004L
+#define EP_PEND_BLOCK_MSK__EP0_F3_PEND_BLOCK_MSK_MASK                                                         0x00000008L
+#define EP_PEND_BLOCK_MSK__EP0_F4_PEND_BLOCK_MSK_MASK                                                         0x00000010L
+#define EP_PEND_BLOCK_MSK__EP0_F5_PEND_BLOCK_MSK_MASK                                                         0x00000020L
+#define EP_PEND_BLOCK_MSK__EP0_F6_PEND_BLOCK_MSK_MASK                                                         0x00000040L
+#define EP_PEND_BLOCK_MSK__EP0_F7_PEND_BLOCK_MSK_MASK                                                         0x00000080L
+#define EP_PEND_BLOCK_MSK__EP1_F0_PEND_BLOCK_MSK_MASK                                                         0x00000100L
+#define EP_PEND_BLOCK_MSK__EP1_F1_PEND_BLOCK_MSK_MASK                                                         0x00000200L
+#define EP_PEND_BLOCK_MSK__EP1_F2_PEND_BLOCK_MSK_MASK                                                         0x00000400L
+#define EP_PEND_BLOCK_MSK__EP1_F3_PEND_BLOCK_MSK_MASK                                                         0x00000800L
+#define EP_PEND_BLOCK_MSK__EP1_F4_PEND_BLOCK_MSK_MASK                                                         0x00001000L
+#define EP_PEND_BLOCK_MSK__EP2_F0_PEND_BLOCK_MSK_MASK                                                         0x00010000L
+#define EP_PEND_BLOCK_MSK__EP2_F1_PEND_BLOCK_MSK_MASK                                                         0x00020000L
+#define EP_PEND_BLOCK_MSK__EP2_F2_PEND_BLOCK_MSK_MASK                                                         0x00040000L
+#define EP_PEND_BLOCK_MSK__EP2_F3_PEND_BLOCK_MSK_MASK                                                         0x00080000L
+#define EP_PEND_BLOCK_MSK__EP2_F4_PEND_BLOCK_MSK_MASK                                                         0x00100000L
+#define EP_PEND_BLOCK_MSK__EP2_F5_PEND_BLOCK_MSK_MASK                                                         0x00200000L
+#define EP_PEND_BLOCK_MSK__EP2_F6_PEND_BLOCK_MSK_MASK                                                         0x00400000L
+//NBIF_VWIRE_CTRL
+#define NBIF_VWIRE_CTRL__NBIF_SMN_VWR_DIS__SHIFT                                                              0x0
+#define NBIF_VWIRE_CTRL__SMN_VWR_RESET_DELAY_CNT__SHIFT                                                       0x4
+#define NBIF_VWIRE_CTRL__SMN_VWR_POSTED__SHIFT                                                                0x8
+#define NBIF_VWIRE_CTRL__NBIF_SDP_UPS_VWR_DIS__SHIFT                                                          0x10
+#define NBIF_VWIRE_CTRL__SDP_VWR_RESET_DELAY_CNT__SHIFT                                                       0x14
+#define NBIF_VWIRE_CTRL__SDP_VWR_BLOCKLVL__SHIFT                                                              0x1a
+#define NBIF_VWIRE_CTRL__NBIF_SMN_VWR_DIS_MASK                                                                0x00000001L
+#define NBIF_VWIRE_CTRL__SMN_VWR_RESET_DELAY_CNT_MASK                                                         0x000000F0L
+#define NBIF_VWIRE_CTRL__SMN_VWR_POSTED_MASK                                                                  0x00000100L
+#define NBIF_VWIRE_CTRL__NBIF_SDP_UPS_VWR_DIS_MASK                                                            0x00010000L
+#define NBIF_VWIRE_CTRL__SDP_VWR_RESET_DELAY_CNT_MASK                                                         0x00F00000L
+#define NBIF_VWIRE_CTRL__SDP_VWR_BLOCKLVL_MASK                                                                0x0C000000L
+//NBIF_SMN_VWR_VCHG_DIS_CTRL
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET0_DIS__SHIFT                                              0x0
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET1_DIS__SHIFT                                              0x1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET2_DIS__SHIFT                                              0x2
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET3_DIS__SHIFT                                              0x3
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET4_DIS__SHIFT                                              0x4
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET5_DIS__SHIFT                                              0x5
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET6_DIS__SHIFT                                              0x6
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET7_DIS__SHIFT                                              0x7
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET8_DIS__SHIFT                                              0x8
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET9_DIS__SHIFT                                              0x9
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET10_DIS__SHIFT                                             0xa
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET11_DIS__SHIFT                                             0xb
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET12_DIS__SHIFT                                             0xc
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET13_DIS__SHIFT                                             0xd
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET14_DIS__SHIFT                                             0xe
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET15_DIS__SHIFT                                             0xf
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET16_DIS__SHIFT                                             0x10
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET17_DIS__SHIFT                                             0x11
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET18_DIS__SHIFT                                             0x12
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET19_DIS__SHIFT                                             0x13
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET20_DIS__SHIFT                                             0x14
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET21_DIS__SHIFT                                             0x15
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET22_DIS__SHIFT                                             0x16
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET23_DIS__SHIFT                                             0x17
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET24_DIS__SHIFT                                             0x18
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET25_DIS__SHIFT                                             0x19
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET26_DIS__SHIFT                                             0x1a
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET27_DIS__SHIFT                                             0x1b
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET28_DIS__SHIFT                                             0x1c
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET29_DIS__SHIFT                                             0x1d
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET30_DIS__SHIFT                                             0x1e
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET31_DIS__SHIFT                                             0x1f
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET0_DIS_MASK                                                0x00000001L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET1_DIS_MASK                                                0x00000002L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET2_DIS_MASK                                                0x00000004L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET3_DIS_MASK                                                0x00000008L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET4_DIS_MASK                                                0x00000010L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET5_DIS_MASK                                                0x00000020L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET6_DIS_MASK                                                0x00000040L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET7_DIS_MASK                                                0x00000080L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET8_DIS_MASK                                                0x00000100L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET9_DIS_MASK                                                0x00000200L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET10_DIS_MASK                                               0x00000400L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET11_DIS_MASK                                               0x00000800L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET12_DIS_MASK                                               0x00001000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET13_DIS_MASK                                               0x00002000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET14_DIS_MASK                                               0x00004000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET15_DIS_MASK                                               0x00008000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET16_DIS_MASK                                               0x00010000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET17_DIS_MASK                                               0x00020000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET18_DIS_MASK                                               0x00040000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET19_DIS_MASK                                               0x00080000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET20_DIS_MASK                                               0x00100000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET21_DIS_MASK                                               0x00200000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET22_DIS_MASK                                               0x00400000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET23_DIS_MASK                                               0x00800000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET24_DIS_MASK                                               0x01000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET25_DIS_MASK                                               0x02000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET26_DIS_MASK                                               0x04000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET27_DIS_MASK                                               0x08000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET28_DIS_MASK                                               0x10000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET29_DIS_MASK                                               0x20000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET30_DIS_MASK                                               0x40000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL__SMN_VWR_VCHG_SET31_DIS_MASK                                               0x80000000L
+//NBIF_SMN_VWR_VCHG_RST_CTRL0
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET0_RST_DEF_REV__SHIFT                                     0x0
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET1_RST_DEF_REV__SHIFT                                     0x1
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET2_RST_DEF_REV__SHIFT                                     0x2
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET3_RST_DEF_REV__SHIFT                                     0x3
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET4_RST_DEF_REV__SHIFT                                     0x4
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET5_RST_DEF_REV__SHIFT                                     0x5
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET6_RST_DEF_REV__SHIFT                                     0x6
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET7_RST_DEF_REV__SHIFT                                     0x7
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET8_RST_DEF_REV__SHIFT                                     0x8
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET9_RST_DEF_REV__SHIFT                                     0x9
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET10_RST_DEF_REV__SHIFT                                    0xa
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET11_RST_DEF_REV__SHIFT                                    0xb
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET12_RST_DEF_REV__SHIFT                                    0xc
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET13_RST_DEF_REV__SHIFT                                    0xd
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET14_RST_DEF_REV__SHIFT                                    0xe
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET15_RST_DEF_REV__SHIFT                                    0xf
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET16_RST_DEF_REV__SHIFT                                    0x10
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET17_RST_DEF_REV__SHIFT                                    0x11
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET18_RST_DEF_REV__SHIFT                                    0x12
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET19_RST_DEF_REV__SHIFT                                    0x13
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET20_RST_DEF_REV__SHIFT                                    0x14
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET21_RST_DEF_REV__SHIFT                                    0x15
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET22_RST_DEF_REV__SHIFT                                    0x16
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET23_RST_DEF_REV__SHIFT                                    0x17
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET24_RST_DEF_REV__SHIFT                                    0x18
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET25_RST_DEF_REV__SHIFT                                    0x19
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET26_RST_DEF_REV__SHIFT                                    0x1a
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET27_RST_DEF_REV__SHIFT                                    0x1b
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET28_RST_DEF_REV__SHIFT                                    0x1c
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET29_RST_DEF_REV__SHIFT                                    0x1d
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET30_RST_DEF_REV__SHIFT                                    0x1e
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET31_RST_DEF_REV__SHIFT                                    0x1f
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET0_RST_DEF_REV_MASK                                       0x00000001L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET1_RST_DEF_REV_MASK                                       0x00000002L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET2_RST_DEF_REV_MASK                                       0x00000004L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET3_RST_DEF_REV_MASK                                       0x00000008L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET4_RST_DEF_REV_MASK                                       0x00000010L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET5_RST_DEF_REV_MASK                                       0x00000020L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET6_RST_DEF_REV_MASK                                       0x00000040L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET7_RST_DEF_REV_MASK                                       0x00000080L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET8_RST_DEF_REV_MASK                                       0x00000100L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET9_RST_DEF_REV_MASK                                       0x00000200L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET10_RST_DEF_REV_MASK                                      0x00000400L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET11_RST_DEF_REV_MASK                                      0x00000800L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET12_RST_DEF_REV_MASK                                      0x00001000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET13_RST_DEF_REV_MASK                                      0x00002000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET14_RST_DEF_REV_MASK                                      0x00004000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET15_RST_DEF_REV_MASK                                      0x00008000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET16_RST_DEF_REV_MASK                                      0x00010000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET17_RST_DEF_REV_MASK                                      0x00020000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET18_RST_DEF_REV_MASK                                      0x00040000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET19_RST_DEF_REV_MASK                                      0x00080000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET20_RST_DEF_REV_MASK                                      0x00100000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET21_RST_DEF_REV_MASK                                      0x00200000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET22_RST_DEF_REV_MASK                                      0x00400000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET23_RST_DEF_REV_MASK                                      0x00800000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET24_RST_DEF_REV_MASK                                      0x01000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET25_RST_DEF_REV_MASK                                      0x02000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET26_RST_DEF_REV_MASK                                      0x04000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET27_RST_DEF_REV_MASK                                      0x08000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET28_RST_DEF_REV_MASK                                      0x10000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET29_RST_DEF_REV_MASK                                      0x20000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET30_RST_DEF_REV_MASK                                      0x40000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0__SMN_VWR_VCHG_SET31_RST_DEF_REV_MASK                                      0x80000000L
+//NBIF_SMN_VWR_VCHG_TRIG
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET0_TRIG__SHIFT                                                 0x0
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET1_TRIG__SHIFT                                                 0x1
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET2_TRIG__SHIFT                                                 0x2
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET3_TRIG__SHIFT                                                 0x3
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET4_TRIG__SHIFT                                                 0x4
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET5_TRIG__SHIFT                                                 0x5
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET6_TRIG__SHIFT                                                 0x6
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET7_TRIG__SHIFT                                                 0x7
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET8_TRIG__SHIFT                                                 0x8
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET9_TRIG__SHIFT                                                 0x9
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET10_TRIG__SHIFT                                                0xa
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET11_TRIG__SHIFT                                                0xb
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET12_TRIG__SHIFT                                                0xc
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET13_TRIG__SHIFT                                                0xd
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET14_TRIG__SHIFT                                                0xe
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET15_TRIG__SHIFT                                                0xf
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET16_TRIG__SHIFT                                                0x10
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET17_TRIG__SHIFT                                                0x11
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET18_TRIG__SHIFT                                                0x12
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET19_TRIG__SHIFT                                                0x13
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET20_TRIG__SHIFT                                                0x14
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET21_TRIG__SHIFT                                                0x15
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET22_TRIG__SHIFT                                                0x16
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET23_TRIG__SHIFT                                                0x17
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET24_TRIG__SHIFT                                                0x18
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET25_TRIG__SHIFT                                                0x19
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET26_TRIG__SHIFT                                                0x1a
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET27_TRIG__SHIFT                                                0x1b
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET28_TRIG__SHIFT                                                0x1c
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET29_TRIG__SHIFT                                                0x1d
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET30_TRIG__SHIFT                                                0x1e
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET31_TRIG__SHIFT                                                0x1f
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET0_TRIG_MASK                                                   0x00000001L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET1_TRIG_MASK                                                   0x00000002L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET2_TRIG_MASK                                                   0x00000004L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET3_TRIG_MASK                                                   0x00000008L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET4_TRIG_MASK                                                   0x00000010L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET5_TRIG_MASK                                                   0x00000020L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET6_TRIG_MASK                                                   0x00000040L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET7_TRIG_MASK                                                   0x00000080L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET8_TRIG_MASK                                                   0x00000100L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET9_TRIG_MASK                                                   0x00000200L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET10_TRIG_MASK                                                  0x00000400L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET11_TRIG_MASK                                                  0x00000800L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET12_TRIG_MASK                                                  0x00001000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET13_TRIG_MASK                                                  0x00002000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET14_TRIG_MASK                                                  0x00004000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET15_TRIG_MASK                                                  0x00008000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET16_TRIG_MASK                                                  0x00010000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET17_TRIG_MASK                                                  0x00020000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET18_TRIG_MASK                                                  0x00040000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET19_TRIG_MASK                                                  0x00080000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET20_TRIG_MASK                                                  0x00100000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET21_TRIG_MASK                                                  0x00200000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET22_TRIG_MASK                                                  0x00400000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET23_TRIG_MASK                                                  0x00800000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET24_TRIG_MASK                                                  0x01000000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET25_TRIG_MASK                                                  0x02000000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET26_TRIG_MASK                                                  0x04000000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET27_TRIG_MASK                                                  0x08000000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET28_TRIG_MASK                                                  0x10000000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET29_TRIG_MASK                                                  0x20000000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET30_TRIG_MASK                                                  0x40000000L
+#define NBIF_SMN_VWR_VCHG_TRIG__SMN_VWR_VCHG_SET31_TRIG_MASK                                                  0x80000000L
+//NBIF_SMN_VWR_WTRIG_CNTL
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET0_DIS__SHIFT                                                0x0
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET1_DIS__SHIFT                                                0x1
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET2_DIS__SHIFT                                                0x2
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET3_DIS__SHIFT                                                0x3
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET4_DIS__SHIFT                                                0x4
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET5_DIS__SHIFT                                                0x5
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET6_DIS__SHIFT                                                0x6
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET7_DIS__SHIFT                                                0x7
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET8_DIS__SHIFT                                                0x8
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET9_DIS__SHIFT                                                0x9
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET10_DIS__SHIFT                                               0xa
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET11_DIS__SHIFT                                               0xb
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET12_DIS__SHIFT                                               0xc
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET13_DIS__SHIFT                                               0xd
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET14_DIS__SHIFT                                               0xe
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET15_DIS__SHIFT                                               0xf
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET16_DIS__SHIFT                                               0x10
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET17_DIS__SHIFT                                               0x11
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET18_DIS__SHIFT                                               0x12
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET19_DIS__SHIFT                                               0x13
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET20_DIS__SHIFT                                               0x14
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET21_DIS__SHIFT                                               0x15
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET22_DIS__SHIFT                                               0x16
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET23_DIS__SHIFT                                               0x17
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET24_DIS__SHIFT                                               0x18
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET25_DIS__SHIFT                                               0x19
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET26_DIS__SHIFT                                               0x1a
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET27_DIS__SHIFT                                               0x1b
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET28_DIS__SHIFT                                               0x1c
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET29_DIS__SHIFT                                               0x1d
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET30_DIS__SHIFT                                               0x1e
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET31_DIS__SHIFT                                               0x1f
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET0_DIS_MASK                                                  0x00000001L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET1_DIS_MASK                                                  0x00000002L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET2_DIS_MASK                                                  0x00000004L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET3_DIS_MASK                                                  0x00000008L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET4_DIS_MASK                                                  0x00000010L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET5_DIS_MASK                                                  0x00000020L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET6_DIS_MASK                                                  0x00000040L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET7_DIS_MASK                                                  0x00000080L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET8_DIS_MASK                                                  0x00000100L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET9_DIS_MASK                                                  0x00000200L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET10_DIS_MASK                                                 0x00000400L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET11_DIS_MASK                                                 0x00000800L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET12_DIS_MASK                                                 0x00001000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET13_DIS_MASK                                                 0x00002000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET14_DIS_MASK                                                 0x00004000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET15_DIS_MASK                                                 0x00008000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET16_DIS_MASK                                                 0x00010000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET17_DIS_MASK                                                 0x00020000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET18_DIS_MASK                                                 0x00040000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET19_DIS_MASK                                                 0x00080000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET20_DIS_MASK                                                 0x00100000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET21_DIS_MASK                                                 0x00200000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET22_DIS_MASK                                                 0x00400000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET23_DIS_MASK                                                 0x00800000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET24_DIS_MASK                                                 0x01000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET25_DIS_MASK                                                 0x02000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET26_DIS_MASK                                                 0x04000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET27_DIS_MASK                                                 0x08000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET28_DIS_MASK                                                 0x10000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET29_DIS_MASK                                                 0x20000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET30_DIS_MASK                                                 0x40000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL__SMN_VWR_WTRIG_SET31_DIS_MASK                                                 0x80000000L
+//NBIF_SMN_VWR_VCHG_DIS_CTRL_1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET0_DIFFDET_DEF_REV__SHIFT                                0x0
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET1_DIFFDET_DEF_REV__SHIFT                                0x1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET2_DIFFDET_DEF_REV__SHIFT                                0x2
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET3_DIFFDET_DEF_REV__SHIFT                                0x3
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET4_DIFFDET_DEF_REV__SHIFT                                0x4
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET5_DIFFDET_DEF_REV__SHIFT                                0x5
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET6_DIFFDET_DEF_REV__SHIFT                                0x6
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET7_DIFFDET_DEF_REV__SHIFT                                0x7
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET8_DIFFDET_DEF_REV__SHIFT                                0x8
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET9_DIFFDET_DEF_REV__SHIFT                                0x9
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET10_DIFFDET_DEF_REV__SHIFT                               0xa
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET11_DIFFDET_DEF_REV__SHIFT                               0xb
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET12_DIFFDET_DEF_REV__SHIFT                               0xc
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET13_DIFFDET_DEF_REV__SHIFT                               0xd
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET14_DIFFDET_DEF_REV__SHIFT                               0xe
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET15_DIFFDET_DEF_REV__SHIFT                               0xf
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET16_DIFFDET_DEF_REV__SHIFT                               0x10
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET17_DIFFDET_DEF_REV__SHIFT                               0x11
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET18_DIFFDET_DEF_REV__SHIFT                               0x12
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET19_DIFFDET_DEF_REV__SHIFT                               0x13
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET20_DIFFDET_DEF_REV__SHIFT                               0x14
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET21_DIFFDET_DEF_REV__SHIFT                               0x15
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET22_DIFFDET_DEF_REV__SHIFT                               0x16
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET23_DIFFDET_DEF_REV__SHIFT                               0x17
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET24_DIFFDET_DEF_REV__SHIFT                               0x18
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET25_DIFFDET_DEF_REV__SHIFT                               0x19
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET26_DIFFDET_DEF_REV__SHIFT                               0x1a
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET27_DIFFDET_DEF_REV__SHIFT                               0x1b
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET28_DIFFDET_DEF_REV__SHIFT                               0x1c
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET29_DIFFDET_DEF_REV__SHIFT                               0x1d
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET30_DIFFDET_DEF_REV__SHIFT                               0x1e
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET31_DIFFDET_DEF_REV__SHIFT                               0x1f
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET0_DIFFDET_DEF_REV_MASK                                  0x00000001L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET1_DIFFDET_DEF_REV_MASK                                  0x00000002L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET2_DIFFDET_DEF_REV_MASK                                  0x00000004L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET3_DIFFDET_DEF_REV_MASK                                  0x00000008L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET4_DIFFDET_DEF_REV_MASK                                  0x00000010L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET5_DIFFDET_DEF_REV_MASK                                  0x00000020L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET6_DIFFDET_DEF_REV_MASK                                  0x00000040L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET7_DIFFDET_DEF_REV_MASK                                  0x00000080L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET8_DIFFDET_DEF_REV_MASK                                  0x00000100L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET9_DIFFDET_DEF_REV_MASK                                  0x00000200L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET10_DIFFDET_DEF_REV_MASK                                 0x00000400L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET11_DIFFDET_DEF_REV_MASK                                 0x00000800L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET12_DIFFDET_DEF_REV_MASK                                 0x00001000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET13_DIFFDET_DEF_REV_MASK                                 0x00002000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET14_DIFFDET_DEF_REV_MASK                                 0x00004000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET15_DIFFDET_DEF_REV_MASK                                 0x00008000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET16_DIFFDET_DEF_REV_MASK                                 0x00010000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET17_DIFFDET_DEF_REV_MASK                                 0x00020000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET18_DIFFDET_DEF_REV_MASK                                 0x00040000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET19_DIFFDET_DEF_REV_MASK                                 0x00080000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET20_DIFFDET_DEF_REV_MASK                                 0x00100000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET21_DIFFDET_DEF_REV_MASK                                 0x00200000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET22_DIFFDET_DEF_REV_MASK                                 0x00400000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET23_DIFFDET_DEF_REV_MASK                                 0x00800000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET24_DIFFDET_DEF_REV_MASK                                 0x01000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET25_DIFFDET_DEF_REV_MASK                                 0x02000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET26_DIFFDET_DEF_REV_MASK                                 0x04000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET27_DIFFDET_DEF_REV_MASK                                 0x08000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET28_DIFFDET_DEF_REV_MASK                                 0x10000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET29_DIFFDET_DEF_REV_MASK                                 0x20000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET30_DIFFDET_DEF_REV_MASK                                 0x40000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1__SMN_VWR_VCHG_SET31_DIFFDET_DEF_REV_MASK                                 0x80000000L
+//NBIF_MGCG_CTRL_LCLK
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_EN_LCLK__SHIFT                                                         0x0
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_MODE_LCLK__SHIFT                                                       0x1
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HYSTERESIS_LCLK__SHIFT                                                 0x2
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HST_DIS_LCLK__SHIFT                                                    0xa
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DMA_DIS_LCLK__SHIFT                                                    0xb
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK__SHIFT                                                    0xc
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_AER_DIS_LCLK__SHIFT                                                    0xd
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DBG_DIS_LCLK__SHIFT                                                    0xe
+#define NBIF_MGCG_CTRL_LCLK__NBIF_SRAM_FGCG_EN_LCLK__SHIFT                                                    0xf
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_EN_LCLK_MASK                                                           0x00000001L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_MODE_LCLK_MASK                                                         0x00000002L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HYSTERESIS_LCLK_MASK                                                   0x000003FCL
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HST_DIS_LCLK_MASK                                                      0x00000400L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DMA_DIS_LCLK_MASK                                                      0x00000800L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK_MASK                                                      0x00001000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_AER_DIS_LCLK_MASK                                                      0x00002000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DBG_DIS_LCLK_MASK                                                      0x00004000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_SRAM_FGCG_EN_LCLK_MASK                                                      0x00008000L
+//NBIF_DS_CTRL_LCLK
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_EN__SHIFT                                                             0x0
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_TIMER__SHIFT                                                          0x10
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_EN_MASK                                                               0x00000001L
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_TIMER_MASK                                                            0xFFFF0000L
+//SMN_MST_CNTL0
+#define SMN_MST_CNTL0__SMN_ARB_MODE__SHIFT                                                                    0x0
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_UPS__SHIFT                                                           0x8
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_UPS__SHIFT                                                           0x9
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_UPS__SHIFT                                                            0xa
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_UPS__SHIFT                                                      0xb
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV0__SHIFT                                                      0x10
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV1__SHIFT                                                      0x11
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV2__SHIFT                                                      0x12
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV0__SHIFT                                                      0x14
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV1__SHIFT                                                      0x15
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV2__SHIFT                                                      0x16
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV0__SHIFT                                                       0x18
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV1__SHIFT                                                       0x19
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV2__SHIFT                                                       0x1a
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV0__SHIFT                                                 0x1c
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV1__SHIFT                                                 0x1d
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV2__SHIFT                                                 0x1e
+#define SMN_MST_CNTL0__SMN_ARB_MODE_MASK                                                                      0x00000003L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_UPS_MASK                                                             0x00000100L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_UPS_MASK                                                             0x00000200L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_UPS_MASK                                                              0x00000400L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_UPS_MASK                                                        0x00000800L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV0_MASK                                                        0x00010000L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV1_MASK                                                        0x00020000L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV2_MASK                                                        0x00040000L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV0_MASK                                                        0x00100000L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV1_MASK                                                        0x00200000L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV2_MASK                                                        0x00400000L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV0_MASK                                                         0x01000000L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV1_MASK                                                         0x02000000L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV2_MASK                                                         0x04000000L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV0_MASK                                                   0x10000000L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV1_MASK                                                   0x20000000L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV2_MASK                                                   0x40000000L
+//SMN_MST_EP_CNTL1
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF0__SHIFT                                                 0x0
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF1__SHIFT                                                 0x1
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF2__SHIFT                                                 0x2
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF3__SHIFT                                                 0x3
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF4__SHIFT                                                 0x4
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF5__SHIFT                                                 0x5
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF6__SHIFT                                                 0x6
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF7__SHIFT                                                 0x7
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF0__SHIFT                                                 0x8
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF1__SHIFT                                                 0x9
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF2__SHIFT                                                 0xa
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF3__SHIFT                                                 0xb
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF4__SHIFT                                                 0xc
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF5__SHIFT                                                 0xd
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF6__SHIFT                                                 0xe
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF7__SHIFT                                                 0xf
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF0__SHIFT                                                 0x10
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF1__SHIFT                                                 0x11
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF2__SHIFT                                                 0x12
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF3__SHIFT                                                 0x13
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF4__SHIFT                                                 0x14
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF5__SHIFT                                                 0x15
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF6__SHIFT                                                 0x16
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF7__SHIFT                                                 0x17
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF0_MASK                                                   0x00000001L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF1_MASK                                                   0x00000002L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF2_MASK                                                   0x00000004L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF3_MASK                                                   0x00000008L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF4_MASK                                                   0x00000010L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF5_MASK                                                   0x00000020L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF6_MASK                                                   0x00000040L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF7_MASK                                                   0x00000080L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF0_MASK                                                   0x00000100L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF1_MASK                                                   0x00000200L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF2_MASK                                                   0x00000400L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF3_MASK                                                   0x00000800L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF4_MASK                                                   0x00001000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF5_MASK                                                   0x00002000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF6_MASK                                                   0x00004000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV1_PF7_MASK                                                   0x00008000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF0_MASK                                                   0x00010000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF1_MASK                                                   0x00020000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF2_MASK                                                   0x00040000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF3_MASK                                                   0x00080000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF4_MASK                                                   0x00100000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF5_MASK                                                   0x00200000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF6_MASK                                                   0x00400000L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV2_PF7_MASK                                                   0x00800000L
+//SMN_MST_EP_CNTL2
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF0__SHIFT                                           0x0
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF1__SHIFT                                           0x1
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF2__SHIFT                                           0x2
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF3__SHIFT                                           0x3
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF4__SHIFT                                           0x4
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF5__SHIFT                                           0x5
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF6__SHIFT                                           0x6
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF7__SHIFT                                           0x7
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF0__SHIFT                                           0x8
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF1__SHIFT                                           0x9
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF2__SHIFT                                           0xa
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF3__SHIFT                                           0xb
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF4__SHIFT                                           0xc
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF5__SHIFT                                           0xd
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF6__SHIFT                                           0xe
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF7__SHIFT                                           0xf
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF0__SHIFT                                           0x10
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF1__SHIFT                                           0x11
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF2__SHIFT                                           0x12
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF3__SHIFT                                           0x13
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF4__SHIFT                                           0x14
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF5__SHIFT                                           0x15
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF6__SHIFT                                           0x16
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF7__SHIFT                                           0x17
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF0_MASK                                             0x00000001L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF1_MASK                                             0x00000002L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF2_MASK                                             0x00000004L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF3_MASK                                             0x00000008L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF4_MASK                                             0x00000010L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF5_MASK                                             0x00000020L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF6_MASK                                             0x00000040L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF7_MASK                                             0x00000080L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF0_MASK                                             0x00000100L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF1_MASK                                             0x00000200L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF2_MASK                                             0x00000400L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF3_MASK                                             0x00000800L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF4_MASK                                             0x00001000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF5_MASK                                             0x00002000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF6_MASK                                             0x00004000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV1_PF7_MASK                                             0x00008000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF0_MASK                                             0x00010000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF1_MASK                                             0x00020000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF2_MASK                                             0x00040000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF3_MASK                                             0x00080000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF4_MASK                                             0x00100000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF5_MASK                                             0x00200000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF6_MASK                                             0x00400000L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV2_PF7_MASK                                             0x00800000L
+//NBIF_SDP_VWR_VCHG_DIS_CTRL
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F0_DIS__SHIFT                                           0x0
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F1_DIS__SHIFT                                           0x1
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F2_DIS__SHIFT                                           0x2
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F3_DIS__SHIFT                                           0x3
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F4_DIS__SHIFT                                           0x4
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F5_DIS__SHIFT                                           0x5
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F6_DIS__SHIFT                                           0x6
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F7_DIS__SHIFT                                           0x7
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_SWDS_P0_DIS__SHIFT                                           0x18
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F0_DIS_MASK                                             0x00000001L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F1_DIS_MASK                                             0x00000002L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F2_DIS_MASK                                             0x00000004L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F3_DIS_MASK                                             0x00000008L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F4_DIS_MASK                                             0x00000010L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F5_DIS_MASK                                             0x00000020L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F6_DIS_MASK                                             0x00000040L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F7_DIS_MASK                                             0x00000080L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_SWDS_P0_DIS_MASK                                             0x01000000L
+//NBIF_SDP_VWR_VCHG_RST_CTRL0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_EN__SHIFT                                  0x0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_EN__SHIFT                                  0x1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_EN__SHIFT                                  0x2
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_EN__SHIFT                                  0x3
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_EN__SHIFT                                  0x4
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_EN__SHIFT                                  0x5
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_EN__SHIFT                                  0x6
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_EN__SHIFT                                  0x7
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_EN__SHIFT                                  0x18
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_EN_MASK                                    0x00000001L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_EN_MASK                                    0x00000002L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_EN_MASK                                    0x00000004L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_EN_MASK                                    0x00000008L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_EN_MASK                                    0x00000010L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_EN_MASK                                    0x00000020L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_EN_MASK                                    0x00000040L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_EN_MASK                                    0x00000080L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_EN_MASK                                    0x01000000L
+//NBIF_SDP_VWR_VCHG_RST_CTRL1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_VAL__SHIFT                                 0x0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_VAL__SHIFT                                 0x1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_VAL__SHIFT                                 0x2
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_VAL__SHIFT                                 0x3
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_VAL__SHIFT                                 0x4
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_VAL__SHIFT                                 0x5
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_VAL__SHIFT                                 0x6
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_VAL__SHIFT                                 0x7
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_VAL__SHIFT                                 0x18
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_VAL_MASK                                   0x00000001L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_VAL_MASK                                   0x00000002L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_VAL_MASK                                   0x00000004L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_VAL_MASK                                   0x00000008L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_VAL_MASK                                   0x00000010L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_VAL_MASK                                   0x00000020L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_VAL_MASK                                   0x00000040L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_VAL_MASK                                   0x00000080L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_VAL_MASK                                   0x01000000L
+//NBIF_SDP_VWR_VCHG_TRIG
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F0_TRIG__SHIFT                                              0x0
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F1_TRIG__SHIFT                                              0x1
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F2_TRIG__SHIFT                                              0x2
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F3_TRIG__SHIFT                                              0x3
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F4_TRIG__SHIFT                                              0x4
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F5_TRIG__SHIFT                                              0x5
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F6_TRIG__SHIFT                                              0x6
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F7_TRIG__SHIFT                                              0x7
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_SWDS_P0_TRIG__SHIFT                                              0x18
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F0_TRIG_MASK                                                0x00000001L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F1_TRIG_MASK                                                0x00000002L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F2_TRIG_MASK                                                0x00000004L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F3_TRIG_MASK                                                0x00000008L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F4_TRIG_MASK                                                0x00000010L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F5_TRIG_MASK                                                0x00000020L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F6_TRIG_MASK                                                0x00000040L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F7_TRIG_MASK                                                0x00000080L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_SWDS_P0_TRIG_MASK                                                0x01000000L
+//NBIF_SHUB_TODET_CTRL
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_EN__SHIFT                                                       0x0
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_AER_LOG_EN__SHIFT                                               0x1
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_TIMER_UNIT__SHIFT                                               0x8
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TIMEOUT_COUNT__SHIFT                                                  0x10
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_EN_MASK                                                         0x00000001L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_AER_LOG_EN_MASK                                                 0x00000002L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_TIMER_UNIT_MASK                                                 0x00000700L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TIMEOUT_COUNT_MASK                                                    0xFFFF0000L
+//NBIF_SHUB_TODET_CLIENT_CTRL
+#define NBIF_SHUB_TODET_CLIENT_CTRL__NBIF_SHUB_TODET_SLVERR_EN__SHIFT                                         0x0
+#define NBIF_SHUB_TODET_CLIENT_CTRL__NBIF_SHUB_TODET_SLVERR_EN_MASK                                           0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_STATUS
+#define NBIF_SHUB_TODET_CLIENT_STATUS__NBIF_SHUB_TODET_CLIENT_STATUS__SHIFT                                   0x0
+#define NBIF_SHUB_TODET_CLIENT_STATUS__NBIF_SHUB_TODET_CLIENT_STATUS_MASK                                     0xFFFFFFFFL
+//NBIF_SHUB_TODET_SYNCFLOOD_CTRL
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL__NBIF_SHUB_TODET_SYNCFLOOD_EN__SHIFT                                   0x0
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL__NBIF_SHUB_TODET_SYNCFLOOD_EN_MASK                                     0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_CTRL2
+#define NBIF_SHUB_TODET_CLIENT_CTRL2__NBIF_SHUB_TODET_SLVERR_EN2__SHIFT                                       0x0
+#define NBIF_SHUB_TODET_CLIENT_CTRL2__NBIF_SHUB_TODET_SLVERR_EN2_MASK                                         0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_STATUS2
+#define NBIF_SHUB_TODET_CLIENT_STATUS2__NBIF_SHUB_TODET_CLIENT_STATUS2__SHIFT                                 0x0
+#define NBIF_SHUB_TODET_CLIENT_STATUS2__NBIF_SHUB_TODET_CLIENT_STATUS2_MASK                                   0xFFFFFFFFL
+//NBIF_SHUB_TODET_SYNCFLOOD_CTRL2
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL2__NBIF_SHUB_TODET_SYNCFLOOD_EN2__SHIFT                                 0x0
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL2__NBIF_SHUB_TODET_SYNCFLOOD_EN2_MASK                                   0xFFFFFFFFL
+//BIFC_BME_ERR_LOG_HB
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F0__SHIFT                                                    0x0
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F1__SHIFT                                                    0x1
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F2__SHIFT                                                    0x2
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F3__SHIFT                                                    0x3
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F4__SHIFT                                                    0x4
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F5__SHIFT                                                    0x5
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F6__SHIFT                                                    0x6
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F0__SHIFT                                              0x10
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F1__SHIFT                                              0x11
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F2__SHIFT                                              0x12
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F3__SHIFT                                              0x13
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F4__SHIFT                                              0x14
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F5__SHIFT                                              0x15
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F6__SHIFT                                              0x16
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F0_MASK                                                      0x00000001L
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F1_MASK                                                      0x00000002L
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F2_MASK                                                      0x00000004L
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F3_MASK                                                      0x00000008L
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F4_MASK                                                      0x00000010L
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F5_MASK                                                      0x00000020L
+#define BIFC_BME_ERR_LOG_HB__DMA_ON_BME_LOW_DEV2_F6_MASK                                                      0x00000040L
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F0_MASK                                                0x00010000L
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F1_MASK                                                0x00020000L
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F2_MASK                                                0x00040000L
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F3_MASK                                                0x00080000L
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F4_MASK                                                0x00100000L
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F5_MASK                                                0x00200000L
+#define BIFC_BME_ERR_LOG_HB__CLEAR_DMA_ON_BME_LOW_DEV2_F6_MASK                                                0x00400000L
+//NBIF_SMN_VWR_VCHG_TRIG_HI
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET32_TRIG__SHIFT                                             0x0
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET33_TRIG__SHIFT                                             0x1
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET34_TRIG__SHIFT                                             0x2
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET35_TRIG__SHIFT                                             0x3
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET36_TRIG__SHIFT                                             0x4
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET37_TRIG__SHIFT                                             0x5
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET38_TRIG__SHIFT                                             0x6
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET39_TRIG__SHIFT                                             0x7
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET40_TRIG__SHIFT                                             0x8
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET41_TRIG__SHIFT                                             0x9
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET42_TRIG__SHIFT                                             0xa
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET43_TRIG__SHIFT                                             0xb
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET44_TRIG__SHIFT                                             0xc
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET45_TRIG__SHIFT                                             0xd
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET46_TRIG__SHIFT                                             0xe
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET47_TRIG__SHIFT                                             0xf
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET48_TRIG__SHIFT                                             0x10
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET49_TRIG__SHIFT                                             0x11
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET50_TRIG__SHIFT                                             0x12
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET51_TRIG__SHIFT                                             0x13
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET52_TRIG__SHIFT                                             0x14
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET53_TRIG__SHIFT                                             0x15
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET54_TRIG__SHIFT                                             0x16
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET55_TRIG__SHIFT                                             0x17
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET56_TRIG__SHIFT                                             0x18
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET57_TRIG__SHIFT                                             0x19
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET58_TRIG__SHIFT                                             0x1a
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET59_TRIG__SHIFT                                             0x1b
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET60_TRIG__SHIFT                                             0x1c
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET61_TRIG__SHIFT                                             0x1d
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET62_TRIG__SHIFT                                             0x1e
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET63_TRIG__SHIFT                                             0x1f
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET32_TRIG_MASK                                               0x00000001L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET33_TRIG_MASK                                               0x00000002L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET34_TRIG_MASK                                               0x00000004L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET35_TRIG_MASK                                               0x00000008L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET36_TRIG_MASK                                               0x00000010L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET37_TRIG_MASK                                               0x00000020L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET38_TRIG_MASK                                               0x00000040L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET39_TRIG_MASK                                               0x00000080L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET40_TRIG_MASK                                               0x00000100L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET41_TRIG_MASK                                               0x00000200L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET42_TRIG_MASK                                               0x00000400L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET43_TRIG_MASK                                               0x00000800L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET44_TRIG_MASK                                               0x00001000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET45_TRIG_MASK                                               0x00002000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET46_TRIG_MASK                                               0x00004000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET47_TRIG_MASK                                               0x00008000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET48_TRIG_MASK                                               0x00010000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET49_TRIG_MASK                                               0x00020000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET50_TRIG_MASK                                               0x00040000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET51_TRIG_MASK                                               0x00080000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET52_TRIG_MASK                                               0x00100000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET53_TRIG_MASK                                               0x00200000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET54_TRIG_MASK                                               0x00400000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET55_TRIG_MASK                                               0x00800000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET56_TRIG_MASK                                               0x01000000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET57_TRIG_MASK                                               0x02000000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET58_TRIG_MASK                                               0x04000000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET59_TRIG_MASK                                               0x08000000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET60_TRIG_MASK                                               0x10000000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET61_TRIG_MASK                                               0x20000000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET62_TRIG_MASK                                               0x40000000L
+#define NBIF_SMN_VWR_VCHG_TRIG_HI__SMN_VWR_VCHG_SET63_TRIG_MASK                                               0x80000000L
+//NBIF_SMN_VWR_VCHG_DIS_CTRL_HI
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET32_DIS__SHIFT                                          0x0
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET33_DIS__SHIFT                                          0x1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET34_DIS__SHIFT                                          0x2
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET35_DIS__SHIFT                                          0x3
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET36_DIS__SHIFT                                          0x4
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET37_DIS__SHIFT                                          0x5
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET38_DIS__SHIFT                                          0x6
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET39_DIS__SHIFT                                          0x7
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET40_DIS__SHIFT                                          0x8
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET41_DIS__SHIFT                                          0x9
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET42_DIS__SHIFT                                          0xa
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET43_DIS__SHIFT                                          0xb
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET44_DIS__SHIFT                                          0xc
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET45_DIS__SHIFT                                          0xd
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET46_DIS__SHIFT                                          0xe
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET47_DIS__SHIFT                                          0xf
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET48_DIS__SHIFT                                          0x10
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET49_DIS__SHIFT                                          0x11
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET50_DIS__SHIFT                                          0x12
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET51_DIS__SHIFT                                          0x13
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET52_DIS__SHIFT                                          0x14
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET53_DIS__SHIFT                                          0x15
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET54_DIS__SHIFT                                          0x16
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET55_DIS__SHIFT                                          0x17
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET56_DIS__SHIFT                                          0x18
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET57_DIS__SHIFT                                          0x19
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET58_DIS__SHIFT                                          0x1a
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET59_DIS__SHIFT                                          0x1b
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET60_DIS__SHIFT                                          0x1c
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET61_DIS__SHIFT                                          0x1d
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET62_DIS__SHIFT                                          0x1e
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET63_DIS__SHIFT                                          0x1f
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET32_DIS_MASK                                            0x00000001L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET33_DIS_MASK                                            0x00000002L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET34_DIS_MASK                                            0x00000004L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET35_DIS_MASK                                            0x00000008L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET36_DIS_MASK                                            0x00000010L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET37_DIS_MASK                                            0x00000020L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET38_DIS_MASK                                            0x00000040L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET39_DIS_MASK                                            0x00000080L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET40_DIS_MASK                                            0x00000100L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET41_DIS_MASK                                            0x00000200L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET42_DIS_MASK                                            0x00000400L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET43_DIS_MASK                                            0x00000800L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET44_DIS_MASK                                            0x00001000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET45_DIS_MASK                                            0x00002000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET46_DIS_MASK                                            0x00004000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET47_DIS_MASK                                            0x00008000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET48_DIS_MASK                                            0x00010000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET49_DIS_MASK                                            0x00020000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET50_DIS_MASK                                            0x00040000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET51_DIS_MASK                                            0x00080000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET52_DIS_MASK                                            0x00100000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET53_DIS_MASK                                            0x00200000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET54_DIS_MASK                                            0x00400000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET55_DIS_MASK                                            0x00800000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET56_DIS_MASK                                            0x01000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET57_DIS_MASK                                            0x02000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET58_DIS_MASK                                            0x04000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET59_DIS_MASK                                            0x08000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET60_DIS_MASK                                            0x10000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET61_DIS_MASK                                            0x20000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET62_DIS_MASK                                            0x40000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_HI__SMN_VWR_VCHG_SET63_DIS_MASK                                            0x80000000L
+//NBIF_SMN_VWR_VCHG_RST_CTRL0_HI
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET32_RST_DEF_REV__SHIFT                                 0x0
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET33_RST_DEF_REV__SHIFT                                 0x1
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET34_RST_DEF_REV__SHIFT                                 0x2
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET35_RST_DEF_REV__SHIFT                                 0x3
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET36_RST_DEF_REV__SHIFT                                 0x4
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET37_RST_DEF_REV__SHIFT                                 0x5
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET38_RST_DEF_REV__SHIFT                                 0x6
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET39_RST_DEF_REV__SHIFT                                 0x7
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET40_RST_DEF_REV__SHIFT                                 0x8
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET41_RST_DEF_REV__SHIFT                                 0x9
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET42_RST_DEF_REV__SHIFT                                 0xa
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET43_RST_DEF_REV__SHIFT                                 0xb
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET44_RST_DEF_REV__SHIFT                                 0xc
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET45_RST_DEF_REV__SHIFT                                 0xd
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET46_RST_DEF_REV__SHIFT                                 0xe
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET47_RST_DEF_REV__SHIFT                                 0xf
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET48_RST_DEF_REV__SHIFT                                 0x10
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET49_RST_DEF_REV__SHIFT                                 0x11
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET50_RST_DEF_REV__SHIFT                                 0x12
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET51_RST_DEF_REV__SHIFT                                 0x13
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET52_RST_DEF_REV__SHIFT                                 0x14
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET53_RST_DEF_REV__SHIFT                                 0x15
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET54_RST_DEF_REV__SHIFT                                 0x16
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET55_RST_DEF_REV__SHIFT                                 0x17
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET56_RST_DEF_REV__SHIFT                                 0x18
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET57_RST_DEF_REV__SHIFT                                 0x19
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET58_RST_DEF_REV__SHIFT                                 0x1a
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET59_RST_DEF_REV__SHIFT                                 0x1b
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET60_RST_DEF_REV__SHIFT                                 0x1c
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET61_RST_DEF_REV__SHIFT                                 0x1d
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET62_RST_DEF_REV__SHIFT                                 0x1e
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET63_RST_DEF_REV__SHIFT                                 0x1f
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET32_RST_DEF_REV_MASK                                   0x00000001L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET33_RST_DEF_REV_MASK                                   0x00000002L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET34_RST_DEF_REV_MASK                                   0x00000004L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET35_RST_DEF_REV_MASK                                   0x00000008L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET36_RST_DEF_REV_MASK                                   0x00000010L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET37_RST_DEF_REV_MASK                                   0x00000020L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET38_RST_DEF_REV_MASK                                   0x00000040L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET39_RST_DEF_REV_MASK                                   0x00000080L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET40_RST_DEF_REV_MASK                                   0x00000100L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET41_RST_DEF_REV_MASK                                   0x00000200L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET42_RST_DEF_REV_MASK                                   0x00000400L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET43_RST_DEF_REV_MASK                                   0x00000800L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET44_RST_DEF_REV_MASK                                   0x00001000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET45_RST_DEF_REV_MASK                                   0x00002000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET46_RST_DEF_REV_MASK                                   0x00004000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET47_RST_DEF_REV_MASK                                   0x00008000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET48_RST_DEF_REV_MASK                                   0x00010000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET49_RST_DEF_REV_MASK                                   0x00020000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET50_RST_DEF_REV_MASK                                   0x00040000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET51_RST_DEF_REV_MASK                                   0x00080000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET52_RST_DEF_REV_MASK                                   0x00100000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET53_RST_DEF_REV_MASK                                   0x00200000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET54_RST_DEF_REV_MASK                                   0x00400000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET55_RST_DEF_REV_MASK                                   0x00800000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET56_RST_DEF_REV_MASK                                   0x01000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET57_RST_DEF_REV_MASK                                   0x02000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET58_RST_DEF_REV_MASK                                   0x04000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET59_RST_DEF_REV_MASK                                   0x08000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET60_RST_DEF_REV_MASK                                   0x10000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET61_RST_DEF_REV_MASK                                   0x20000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET62_RST_DEF_REV_MASK                                   0x40000000L
+#define NBIF_SMN_VWR_VCHG_RST_CTRL0_HI__SMN_VWR_VCHG_SET63_RST_DEF_REV_MASK                                   0x80000000L
+//NBIF_SMN_VWR_WTRIG_CNTL_HI
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET32_DIS__SHIFT                                            0x0
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET33_DIS__SHIFT                                            0x1
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET34_DIS__SHIFT                                            0x2
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET35_DIS__SHIFT                                            0x3
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET36_DIS__SHIFT                                            0x4
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET37_DIS__SHIFT                                            0x5
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET38_DIS__SHIFT                                            0x6
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET39_DIS__SHIFT                                            0x7
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET40_DIS__SHIFT                                            0x8
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET41_DIS__SHIFT                                            0x9
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET42_DIS__SHIFT                                            0xa
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET43_DIS__SHIFT                                            0xb
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET44_DIS__SHIFT                                            0xc
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET45_DIS__SHIFT                                            0xd
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET46_DIS__SHIFT                                            0xe
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET47_DIS__SHIFT                                            0xf
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET48_DIS__SHIFT                                            0x10
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET49_DIS__SHIFT                                            0x11
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET50_DIS__SHIFT                                            0x12
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET51_DIS__SHIFT                                            0x13
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET52_DIS__SHIFT                                            0x14
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET53_DIS__SHIFT                                            0x15
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET54_DIS__SHIFT                                            0x16
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET55_DIS__SHIFT                                            0x17
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET56_DIS__SHIFT                                            0x18
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET57_DIS__SHIFT                                            0x19
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET58_DIS__SHIFT                                            0x1a
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET59_DIS__SHIFT                                            0x1b
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET60_DIS__SHIFT                                            0x1c
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET61_DIS__SHIFT                                            0x1d
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET62_DIS__SHIFT                                            0x1e
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET63_DIS__SHIFT                                            0x1f
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET32_DIS_MASK                                              0x00000001L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET33_DIS_MASK                                              0x00000002L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET34_DIS_MASK                                              0x00000004L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET35_DIS_MASK                                              0x00000008L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET36_DIS_MASK                                              0x00000010L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET37_DIS_MASK                                              0x00000020L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET38_DIS_MASK                                              0x00000040L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET39_DIS_MASK                                              0x00000080L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET40_DIS_MASK                                              0x00000100L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET41_DIS_MASK                                              0x00000200L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET42_DIS_MASK                                              0x00000400L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET43_DIS_MASK                                              0x00000800L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET44_DIS_MASK                                              0x00001000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET45_DIS_MASK                                              0x00002000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET46_DIS_MASK                                              0x00004000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET47_DIS_MASK                                              0x00008000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET48_DIS_MASK                                              0x00010000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET49_DIS_MASK                                              0x00020000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET50_DIS_MASK                                              0x00040000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET51_DIS_MASK                                              0x00080000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET52_DIS_MASK                                              0x00100000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET53_DIS_MASK                                              0x00200000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET54_DIS_MASK                                              0x00400000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET55_DIS_MASK                                              0x00800000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET56_DIS_MASK                                              0x01000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET57_DIS_MASK                                              0x02000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET58_DIS_MASK                                              0x04000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET59_DIS_MASK                                              0x08000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET60_DIS_MASK                                              0x10000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET61_DIS_MASK                                              0x20000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET62_DIS_MASK                                              0x40000000L
+#define NBIF_SMN_VWR_WTRIG_CNTL_HI__SMN_VWR_WTRIG_SET63_DIS_MASK                                              0x80000000L
+//NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET32_DIFFDET_DEF_REV__SHIFT                            0x0
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET33_DIFFDET_DEF_REV__SHIFT                            0x1
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET34_DIFFDET_DEF_REV__SHIFT                            0x2
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET35_DIFFDET_DEF_REV__SHIFT                            0x3
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET36_DIFFDET_DEF_REV__SHIFT                            0x4
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET37_DIFFDET_DEF_REV__SHIFT                            0x5
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET38_DIFFDET_DEF_REV__SHIFT                            0x6
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET39_DIFFDET_DEF_REV__SHIFT                            0x7
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET40_DIFFDET_DEF_REV__SHIFT                            0x8
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET41_DIFFDET_DEF_REV__SHIFT                            0x9
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET42_DIFFDET_DEF_REV__SHIFT                            0xa
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET43_DIFFDET_DEF_REV__SHIFT                            0xb
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET44_DIFFDET_DEF_REV__SHIFT                            0xc
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET45_DIFFDET_DEF_REV__SHIFT                            0xd
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET46_DIFFDET_DEF_REV__SHIFT                            0xe
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET47_DIFFDET_DEF_REV__SHIFT                            0xf
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET48_DIFFDET_DEF_REV__SHIFT                            0x10
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET49_DIFFDET_DEF_REV__SHIFT                            0x11
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET50_DIFFDET_DEF_REV__SHIFT                            0x12
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET51_DIFFDET_DEF_REV__SHIFT                            0x13
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET52_DIFFDET_DEF_REV__SHIFT                            0x14
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET53_DIFFDET_DEF_REV__SHIFT                            0x15
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET54_DIFFDET_DEF_REV__SHIFT                            0x16
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET55_DIFFDET_DEF_REV__SHIFT                            0x17
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET56_DIFFDET_DEF_REV__SHIFT                            0x18
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET57_DIFFDET_DEF_REV__SHIFT                            0x19
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET58_DIFFDET_DEF_REV__SHIFT                            0x1a
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET59_DIFFDET_DEF_REV__SHIFT                            0x1b
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET60_DIFFDET_DEF_REV__SHIFT                            0x1c
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET61_DIFFDET_DEF_REV__SHIFT                            0x1d
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET62_DIFFDET_DEF_REV__SHIFT                            0x1e
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET63_DIFFDET_DEF_REV__SHIFT                            0x1f
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET32_DIFFDET_DEF_REV_MASK                              0x00000001L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET33_DIFFDET_DEF_REV_MASK                              0x00000002L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET34_DIFFDET_DEF_REV_MASK                              0x00000004L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET35_DIFFDET_DEF_REV_MASK                              0x00000008L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET36_DIFFDET_DEF_REV_MASK                              0x00000010L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET37_DIFFDET_DEF_REV_MASK                              0x00000020L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET38_DIFFDET_DEF_REV_MASK                              0x00000040L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET39_DIFFDET_DEF_REV_MASK                              0x00000080L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET40_DIFFDET_DEF_REV_MASK                              0x00000100L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET41_DIFFDET_DEF_REV_MASK                              0x00000200L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET42_DIFFDET_DEF_REV_MASK                              0x00000400L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET43_DIFFDET_DEF_REV_MASK                              0x00000800L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET44_DIFFDET_DEF_REV_MASK                              0x00001000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET45_DIFFDET_DEF_REV_MASK                              0x00002000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET46_DIFFDET_DEF_REV_MASK                              0x00004000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET47_DIFFDET_DEF_REV_MASK                              0x00008000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET48_DIFFDET_DEF_REV_MASK                              0x00010000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET49_DIFFDET_DEF_REV_MASK                              0x00020000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET50_DIFFDET_DEF_REV_MASK                              0x00040000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET51_DIFFDET_DEF_REV_MASK                              0x00080000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET52_DIFFDET_DEF_REV_MASK                              0x00100000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET53_DIFFDET_DEF_REV_MASK                              0x00200000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET54_DIFFDET_DEF_REV_MASK                              0x00400000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET55_DIFFDET_DEF_REV_MASK                              0x00800000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET56_DIFFDET_DEF_REV_MASK                              0x01000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET57_DIFFDET_DEF_REV_MASK                              0x02000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET58_DIFFDET_DEF_REV_MASK                              0x04000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET59_DIFFDET_DEF_REV_MASK                              0x08000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET60_DIFFDET_DEF_REV_MASK                              0x10000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET61_DIFFDET_DEF_REV_MASK                              0x20000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET62_DIFFDET_DEF_REV_MASK                              0x40000000L
+#define NBIF_SMN_VWR_VCHG_DIS_CTRL_1_HI__SMN_VWR_VCHG_SET63_DIFFDET_DEF_REV_MASK                              0x80000000L
+//BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+//BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+//BIFC_GMI_SDP_REQ_POOLCRED_ALLOC
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//BIFC_GMI_SDP_DAT_POOLCRED_ALLOC
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                   0x8
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                   0xc
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                   0x10
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                   0x14
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                   0x18
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                   0x1c
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                     0x00000F00L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                     0x0000F000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                     0x000F0000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                     0x00F00000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                     0x0F000000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                     0xF0000000L
+//BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                   0x8
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                   0xc
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                   0x10
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                   0x14
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                   0x18
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                   0x1c
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                     0x00000F00L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                     0x0000F000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                     0x000F0000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                     0x00F00000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                     0x0F000000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                     0xF0000000L
+//DISCON_HYSTERESIS_HEAD_CTRL
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_DNS_SDP_DISCON_HYSTERESIS_H__SHIFT                                   0x0
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_UPS_SDP_DISCON_HYSTERESIS_H__SHIFT                                   0x8
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_DNS_SDP_DISCON_HYSTERESIS_H_MASK                                     0x0000000FL
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_UPS_SDP_DISCON_HYSTERESIS_H_MASK                                     0x00000F00L
+//BIFC_Z10_CTRL0
+#define BIFC_Z10_CTRL0__BIFC_Z10_IDLE_MASK__SHIFT                                                             0x0
+#define BIFC_Z10_CTRL0__REGS_Z10_IDLE_ASSERT__SHIFT                                                           0x10
+#define BIFC_Z10_CTRL0__REGS_Z10_FENCE_INTF_ACK_EN__SHIFT                                                     0x11
+#define BIFC_Z10_CTRL0__REGS_Z10_FENCE_ACK_TIMEOUT_EN__SHIFT                                                  0x12
+#define BIFC_Z10_CTRL0__REGS_Z10_FENCE_ACK_ASSERT__SHIFT                                                      0x13
+#define BIFC_Z10_CTRL0__Z10_DS_ALLOW_EN_IN_PG__SHIFT                                                          0x1e
+#define BIFC_Z10_CTRL0__BIFC_Z10_IDLE_MASK_MASK                                                               0x0000FFFFL
+#define BIFC_Z10_CTRL0__REGS_Z10_IDLE_ASSERT_MASK                                                             0x00010000L
+#define BIFC_Z10_CTRL0__REGS_Z10_FENCE_INTF_ACK_EN_MASK                                                       0x00020000L
+#define BIFC_Z10_CTRL0__REGS_Z10_FENCE_ACK_TIMEOUT_EN_MASK                                                    0x00040000L
+#define BIFC_Z10_CTRL0__REGS_Z10_FENCE_ACK_ASSERT_MASK                                                        0x00080000L
+#define BIFC_Z10_CTRL0__Z10_DS_ALLOW_EN_IN_PG_MASK                                                            0x40000000L
+//BIFC_Z10_CTRL1
+#define BIFC_Z10_CTRL1__REGS_Z10_FENCE_ACK_TIMEOUT__SHIFT                                                     0x0
+#define BIFC_Z10_CTRL1__REGS_Z10_FENCE_ACK_TIMEOUT_MASK                                                       0xFFFFFFFFL
+//BIFC_Z10_STATUS
+#define BIFC_Z10_STATUS__REGS_BIFC_Z10_STATUS__SHIFT                                                          0x0
+#define BIFC_Z10_STATUS__REGS_BIFC_Z10_STATUS_MASK                                                            0xFFFFFFFFL
+//BIFC_EARLY_WAKEUP_CNTL
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_ACTIVE__SHIFT                                     0x0
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_DS_EXIT__SHIFT                                    0x1
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_ALLOW_AER_ACTIVE__SHIFT                                     0x2
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_ACTIVE_MASK                                       0x00000001L
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_DS_EXIT_MASK                                      0x00000002L
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_ALLOW_AER_ACTIVE_MASK                                       0x00000004L
+//BIFC_PERF_CNT_MMIO_RD_H16BIT
+#define BIFC_PERF_CNT_MMIO_RD_H16BIT__PERF_CNT_MMIO_RD_VALUE_H16BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_RD_H16BIT__PERF_CNT_MMIO_RD_VALUE_H16BIT_MASK                                      0x0000FFFFL
+//BIFC_PERF_CNT_MMIO_WR_H16BIT
+#define BIFC_PERF_CNT_MMIO_WR_H16BIT__PERF_CNT_MMIO_WR_VALUE_H16BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_WR_H16BIT__PERF_CNT_MMIO_WR_VALUE_H16BIT_MASK                                      0x0000FFFFL
+//BIFC_PERF_CNT_DMA_RD_H16BIT
+#define BIFC_PERF_CNT_DMA_RD_H16BIT__PERF_CNT_DMA_RD_VALUE_H16BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_RD_H16BIT__PERF_CNT_DMA_RD_VALUE_H16BIT_MASK                                        0x0000FFFFL
+//BIFC_PERF_CNT_DMA_WR_H16BIT
+#define BIFC_PERF_CNT_DMA_WR_H16BIT__PERF_CNT_DMA_WR_VALUE_H16BIT__SHIFT                                      0x0
+#define BIFC_PERF_CNT_DMA_WR_H16BIT__PERF_CNT_DMA_WR_VALUE_H16BIT_MASK                                        0x0000FFFFL
+//NBIF_PERF_COM_COUNT_ENABLE
+#define NBIF_PERF_COM_COUNT_ENABLE__NBIF_COM_COUNT_ENABLE__SHIFT                                              0x0
+#define NBIF_PERF_COM_COUNT_ENABLE__START_COUNT_NOPULS__SHIFT                                                 0x3
+#define NBIF_PERF_COM_COUNT_ENABLE__LEGACY_OUT_REALTIME_SEL__SHIFT                                            0x4
+#define NBIF_PERF_COM_COUNT_ENABLE__PERF_CNT_TRIG_MONI_SEL__SHIFT                                             0x5
+#define NBIF_PERF_COM_COUNT_ENABLE__NBIF_COM_COUNT_ENABLE_MASK                                                0x00000001L
+#define NBIF_PERF_COM_COUNT_ENABLE__START_COUNT_NOPULS_MASK                                                   0x00000008L
+#define NBIF_PERF_COM_COUNT_ENABLE__LEGACY_OUT_REALTIME_SEL_MASK                                              0x00000010L
+#define NBIF_PERF_COM_COUNT_ENABLE__PERF_CNT_TRIG_MONI_SEL_MASK                                               0x00000060L
+//NBIF_BX_PERF_CNT_FSM
+#define NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_SHADOW_TGL_DELAY_COUNT__SHIFT                                         0x0
+#define NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_COUNT__SHIFT                                     0x4
+#define NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_EN__SHIFT                                        0x8
+#define NBIF_BX_PERF_CNT_FSM__BX_PRE_FLD_GLOBAL_SHADOW_WR__SHIFT                                              0x9
+#define NBIF_BX_PERF_CNT_FSM__BX_PERF_CNT_DONE__SHIFT                                                         0xa
+#define NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_SHADOW_TGL_DELAY_COUNT_MASK                                           0x0000000FL
+#define NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_COUNT_MASK                                       0x000000F0L
+#define NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_EN_MASK                                          0x00000100L
+#define NBIF_BX_PERF_CNT_FSM__BX_PRE_FLD_GLOBAL_SHADOW_WR_MASK                                                0x00000200L
+#define NBIF_BX_PERF_CNT_FSM__BX_PERF_CNT_DONE_MASK                                                           0x00000400L
+//NBIF_COM_COUNT_VALUE
+#define NBIF_COM_COUNT_VALUE__NBIF_COM_COUNT_VALUE__SHIFT                                                     0x0
+#define NBIF_COM_COUNT_VALUE__NBIF_COM_COUNT_VALUE_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_bif_ras_bif_ras_regblk
+//BIFL_RAS_CENTRAL_CNTL
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_HST_STALL_DIS__SHIFT                                   0x1b
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_STALL_DIS__SHIFT                                       0x1c
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_DIS__SHIFT                                             0x1d
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_INTR_DIS__SHIFT                                                 0x1e
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_LINKDIS_TRIG_EGRESS_STALL_DIS__SHIFT                                      0x1f
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_HST_STALL_DIS_MASK                                     0x08000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_STALL_DIS_MASK                                         0x10000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_DIS_MASK                                               0x20000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_INTR_DIS_MASK                                                   0x40000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_LINKDIS_TRIG_EGRESS_STALL_DIS_MASK                                        0x80000000L
+//BIFL_RAS_CENTRAL_STATUS
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_EgStall_det__SHIFT                                                  0x0
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_ErrEvent_det__SHIFT                                                 0x1
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_EgStall_det__SHIFT                                                  0x2
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_ErrEvent_det__SHIFT                                                 0x3
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_ErrEvent_Recv__SHIFT                                        0x1d
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_Intr_Recv__SHIFT                                            0x1e
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_LinkDis_Recv__SHIFT                                                     0x1f
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_EgStall_det_MASK                                                    0x00000001L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_ErrEvent_det_MASK                                                   0x00000002L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_EgStall_det_MASK                                                    0x00000004L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_ErrEvent_det_MASK                                                   0x00000008L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_ErrEvent_Recv_MASK                                          0x20000000L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_Intr_Recv_MASK                                              0x40000000L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_LinkDis_Recv_MASK                                                       0x80000000L
+//BIFL_RAS_LEAF0_CTRL
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF0_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF0_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF0_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF0_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF0_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF0_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF0_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF0_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF0_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF0_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF1_CTRL
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF1_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF1_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF1_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF1_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF1_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF1_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF1_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF1_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF1_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF1_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF2_CTRL
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF2_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF2_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF2_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF2_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF2_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF2_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF2_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF2_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF2_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF2_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF3_CTRL
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF3_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF3_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF3_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF3_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF3_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF3_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF3_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF3_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF3_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF3_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF0_STATUS
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF0_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF0_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_RECV_NS__SHIFT                                                       0xc
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF0_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF0_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_RECV_NS_MASK                                                         0x00001000L
+//BIFL_RAS_LEAF1_STATUS
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF1_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF1_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_RECV_NS__SHIFT                                                       0xc
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF1_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF1_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_RECV_NS_MASK                                                         0x00001000L
+//BIFL_RAS_LEAF2_STATUS
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF2_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF2_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_RECV_NS__SHIFT                                                       0xc
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF2_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF2_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_RECV_NS_MASK                                                         0x00001000L
+//BIFL_RAS_LEAF3_STATUS
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF3_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF3_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_RECV_NS__SHIFT                                                       0xc
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF3_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF3_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_RECV_NS_MASK                                                         0x00001000L
+//BIFL_IOHUB_RAS_IH_CNTL
+#define BIFL_IOHUB_RAS_IH_CNTL__BIFL_RAS_IH_INTR_EN__SHIFT                                                    0x0
+#define BIFL_IOHUB_RAS_IH_CNTL__BIFL_RAS_IH_INTR_EN_MASK                                                      0x00000001L
+//BIFL_RAS_VWR_FROM_IOHUB
+#define BIFL_RAS_VWR_FROM_IOHUB__BIFL_RAS_IH_INTR_TRIG__SHIFT                                                 0x0
+#define BIFL_RAS_VWR_FROM_IOHUB__BIFL_RAS_IH_INTR_TRIG_MASK                                                   0x00000001L
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_BIFDEC1
+//RCC_DWN_DEV0_1_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_1_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_1_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_1_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_1_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                 0x5
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                   0x00000020L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_BIFDEC1
+//RCC_DWNP_DEV0_1_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_1_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                           0x4
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                             0x00000010L
+//RCC_DWNP_DEV0_1_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_ep_dev0_BIFDEC1
+//RCC_EP_DEV0_1_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_1_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_1_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_1_EP_PCIE_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_1_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_1_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                  0x5
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                    0x00000020L
+//RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_1_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_1_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_1_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_1_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_1_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                          0x4
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                            0x00000010L
+
+
+// addressBlock: nbif0_rcc_dev0_BIFDEC1
+//RCC_DEV0_0_RCC_ERR_INT_CNTL
+#define RCC_DEV0_0_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN__SHIFT                                0x0
+#define RCC_DEV0_0_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN_MASK                                  0x00000001L
+//RCC_DEV0_0_RCC_BACO_CNTL_MISC
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS__SHIFT                                                 0x0
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS__SHIFT                                                  0x1
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS_MASK                                                   0x00000001L
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS_MASK                                                    0x00000002L
+//RCC_DEV0_0_RCC_RESET_EN
+#define RCC_DEV0_0_RCC_RESET_EN__DB_APER_RESET_EN__SHIFT                                                      0xf
+#define RCC_DEV0_0_RCC_RESET_EN__DB_APER_RESET_EN_MASK                                                        0x00008000L
+//RCC_DEV0_1_RCC_VDM_SUPPORT
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+//RCC_DEV0_0_RCC_GPUIOV_REGION
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__LFB_REGION__SHIFT                                                       0x0
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__MAX_REGION__SHIFT                                                       0x4
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__LFB_REGION_MASK                                                         0x0000000FL
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__MAX_REGION_MASK                                                         0x000000F0L
+//RCC_DEV0_0_RCC_GPU_HOSTVM_EN
+#define RCC_DEV0_0_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN__SHIFT                                                    0x0
+#define RCC_DEV0_0_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN_MASK                                                      0x00000001L
+//RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE__SHIFT                              0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN__SHIFT                                    0x1
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE_MASK                                0x00000001L
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN_MASK                                      0x00000002L
+//RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET__SHIFT                        0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET_MASK                          0xFFFFL
+//RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE__SHIFT                                    0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE_MASK                                      0xFFFFL
+//RCC_DEV0_0_RCC_PEER_REG_RANGE0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_0_RCC_PEER_REG_RANGE1
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_1_RCC_BUS_CNTL
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_0_RCC_CONFIG_CNTL
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT                                               0x2
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GRPH_ADRSEL__SHIFT                                                        0x3
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN_MASK                                                       0x00000001L
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK                                                 0x00000004L
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GRPH_ADRSEL_MASK                                                          0x00000018L
+//RCC_DEV0_0_RCC_CONFIG_F0_BASE
+#define RCC_DEV0_0_RCC_CONFIG_F0_BASE__F0_BASE__SHIFT                                                         0x0
+#define RCC_DEV0_0_RCC_CONFIG_F0_BASE__F0_BASE_MASK                                                           0xFFFFFFFFL
+//RCC_DEV0_0_RCC_CONFIG_APER_SIZE
+#define RCC_DEV0_0_RCC_CONFIG_APER_SIZE__APER_SIZE__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_CONFIG_APER_SIZE__APER_SIZE_MASK                                                       0xFFFFFFFFL
+//RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE
+#define RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT                                             0x0
+#define RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK                                               0x07FFFFFFL
+//RCC_DEV0_0_RCC_XDMA_LO
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_APER_EN__SHIFT                                                       0x1f
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK                                                     0x7FFFFFFFL
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_APER_EN_MASK                                                         0x80000000L
+//RCC_DEV0_0_RCC_XDMA_HI
+#define RCC_DEV0_0_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK                                                     0x7FFFFFFFL
+//RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_0_RCC_BUSNUM_CNTL1
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL1__ID_MASK__SHIFT                                                           0x0
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL1__ID_MASK_MASK                                                             0x000000FFL
+//RCC_DEV0_0_RCC_BUSNUM_LIST0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID0__SHIFT                                                               0x0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID1__SHIFT                                                               0x8
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID2__SHIFT                                                               0x10
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID3__SHIFT                                                               0x18
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID0_MASK                                                                 0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID1_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID2_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID3_MASK                                                                 0xFF000000L
+//RCC_DEV0_0_RCC_BUSNUM_LIST1
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID4__SHIFT                                                               0x0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID5__SHIFT                                                               0x8
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID6__SHIFT                                                               0x10
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID7__SHIFT                                                               0x18
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID4_MASK                                                                 0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID5_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID6_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID7_MASK                                                                 0xFF000000L
+//RCC_DEV0_0_RCC_BUSNUM_CNTL2
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT                                                    0x0
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT                                                     0x8
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT                                           0x11
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK                                                      0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK                                                       0x00000100L
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__HDPREG_CNTL_MASK                                                         0x00010000L
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK                                             0x00020000L
+//RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM
+#define RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN_MASK                                                     0x00000001L
+//RCC_DEV0_0_RCC_HOST_BUSNUM
+#define RCC_DEV0_0_RCC_HOST_BUSNUM__HOST_ID__SHIFT                                                            0x0
+#define RCC_DEV0_0_RCC_HOST_BUSNUM__HOST_ID_MASK                                                              0x0000FFFFL
+//RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT                                                   0x8
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT                                                   0x10
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT                                                   0x18
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK                                                     0x000000FFL
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK                                                     0x0000FF00L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK                                                     0x00FF0000L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK                                                     0xFF000000L
+//RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT                                                   0x8
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT                                                   0x10
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT                                                   0x18
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK                                                     0x000000FFL
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK                                                     0x0000FF00L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK                                                     0x00FF0000L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK                                                     0xFF000000L
+//RCC_DEV0_1_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_1_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                            0x10
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                              0x00FF0000L
+//RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_1_RCC_MH_ARB_CNTL
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x0000FFFEL
+
+
+// addressBlock: nbif0_bif_bx_SYSDEC
+//BIF_BX0_PCIE_INDEX
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX0_PCIE_DATA
+#define BIF_BX0_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX0_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//BIF_BX0_PCIE_INDEX2
+#define BIF_BX0_PCIE_INDEX2__PCIE_INDEX2__SHIFT                                                               0x0
+#define BIF_BX0_PCIE_INDEX2__PCIE_INDEX2_MASK                                                                 0xFFFFFFFFL
+//BIF_BX0_PCIE_DATA2
+#define BIF_BX0_PCIE_DATA2__PCIE_DATA2__SHIFT                                                                 0x0
+#define BIF_BX0_PCIE_DATA2__PCIE_DATA2_MASK                                                                   0xFFFFFFFFL
+//BIF_BX0_PCIE_INDEX_HI
+#define BIF_BX0_PCIE_INDEX_HI__PCIE_INDEX_HI__SHIFT                                                           0x0
+#define BIF_BX0_PCIE_INDEX_HI__PCIE_INDEX_HI_MASK                                                             0x000000FFL
+//BIF_BX0_PCIE_INDEX2_HI
+#define BIF_BX0_PCIE_INDEX2_HI__PCIE_INDEX2_HI__SHIFT                                                         0x0
+#define BIF_BX0_PCIE_INDEX2_HI__PCIE_INDEX2_HI_MASK                                                           0x000000FFL
+//BIF_BX0_SBIOS_SCRATCH_0
+#define BIF_BX0_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_1
+#define BIF_BX0_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_2
+#define BIF_BX0_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_3
+#define BIF_BX0_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_0
+#define BIF_BX0_BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_1
+#define BIF_BX0_BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_2
+#define BIF_BX0_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_3
+#define BIF_BX0_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_4
+#define BIF_BX0_BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_5
+#define BIF_BX0_BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_6
+#define BIF_BX0_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_7
+#define BIF_BX0_BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_8
+#define BIF_BX0_BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_9
+#define BIF_BX0_BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_10
+#define BIF_BX0_BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_11
+#define BIF_BX0_BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_12
+#define BIF_BX0_BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_13
+#define BIF_BX0_BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_14
+#define BIF_BX0_BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_15
+#define BIF_BX0_BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIF_RLC_INTR_CNTL
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX0_BIF_RLC_INTR_CNTL__RLC_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+//BIF_BX0_BIF_VCE_INTR_CNTL
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX0_BIF_VCE_INTR_CNTL__VCE_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+//BIF_BX0_BIF_UVD_INTR_CNTL
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_INST_SEL__SHIFT                                                        0x1c
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+#define BIF_BX0_BIF_UVD_INTR_CNTL__UVD_INST_SEL_MASK                                                          0xF0000000L
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR1
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR2
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR3
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR4
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR5
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR6
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR7
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_CNTL
+#define BIF_BX0_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE_MASK                                                         0x000000FFL
+//BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL__SHIFT                                                 0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL_MASK                                                   0xFFFFFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL__SHIFT                                                   0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL__SHIFT                                 0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL_MASK                                   0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_0
+#define BIF_BX0_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_1
+#define BIF_BX0_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_2
+#define BIF_BX0_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_3
+#define BIF_BX0_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_4
+#define BIF_BX0_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_5
+#define BIF_BX0_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_6
+#define BIF_BX0_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_7
+#define BIF_BX0_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_8
+#define BIF_BX0_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_9
+#define BIF_BX0_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_10
+#define BIF_BX0_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_11
+#define BIF_BX0_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_12
+#define BIF_BX0_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_13
+#define BIF_BX0_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_14
+#define BIF_BX0_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_15
+#define BIF_BX0_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_0
+#define BIF_BX0_FW_SCRATCH_0__FW_SCRATCH_0__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_0__FW_SCRATCH_0_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_1
+#define BIF_BX0_FW_SCRATCH_1__FW_SCRATCH_1__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_1__FW_SCRATCH_1_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_2
+#define BIF_BX0_FW_SCRATCH_2__FW_SCRATCH_2__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_2__FW_SCRATCH_2_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_3
+#define BIF_BX0_FW_SCRATCH_3__FW_SCRATCH_3__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_3__FW_SCRATCH_3_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_4
+#define BIF_BX0_FW_SCRATCH_4__FW_SCRATCH_4__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_4__FW_SCRATCH_4_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_5
+#define BIF_BX0_FW_SCRATCH_5__FW_SCRATCH_5__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_5__FW_SCRATCH_5_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_6
+#define BIF_BX0_FW_SCRATCH_6__FW_SCRATCH_6__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_6__FW_SCRATCH_6_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_7
+#define BIF_BX0_FW_SCRATCH_7__FW_SCRATCH_7__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_7__FW_SCRATCH_7_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_8
+#define BIF_BX0_FW_SCRATCH_8__FW_SCRATCH_8__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_8__FW_SCRATCH_8_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_9
+#define BIF_BX0_FW_SCRATCH_9__FW_SCRATCH_9__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_9__FW_SCRATCH_9_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_10
+#define BIF_BX0_FW_SCRATCH_10__FW_SCRATCH_10__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_10__FW_SCRATCH_10_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_11
+#define BIF_BX0_FW_SCRATCH_11__FW_SCRATCH_11__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_11__FW_SCRATCH_11_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_12
+#define BIF_BX0_FW_SCRATCH_12__FW_SCRATCH_12__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_12__FW_SCRATCH_12_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_13
+#define BIF_BX0_FW_SCRATCH_13__FW_SCRATCH_13__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_13__FW_SCRATCH_13_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_14
+#define BIF_BX0_FW_SCRATCH_14__FW_SCRATCH_14__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_14__FW_SCRATCH_14_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_15
+#define BIF_BX0_FW_SCRATCH_15__FW_SCRATCH_15__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_15__FW_SCRATCH_15_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_4
+#define BIF_BX0_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_5
+#define BIF_BX0_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_6
+#define BIF_BX0_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_7
+#define BIF_BX0_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_8
+#define BIF_BX0_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_9
+#define BIF_BX0_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_10
+#define BIF_BX0_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_11
+#define BIF_BX0_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_12
+#define BIF_BX0_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_13
+#define BIF_BX0_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_14
+#define BIF_BX0_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_15
+#define BIF_BX0_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_bif_bx_pf_SYSPFVFDEC
+//BIF_BX_PF0_MM_INDEX
+#define BIF_BX_PF0_MM_INDEX__MM_OFFSET__SHIFT                                                                 0x0
+#define BIF_BX_PF0_MM_INDEX__MM_APER__SHIFT                                                                   0x1f
+#define BIF_BX_PF0_MM_INDEX__MM_OFFSET_MASK                                                                   0x7FFFFFFFL
+#define BIF_BX_PF0_MM_INDEX__MM_APER_MASK                                                                     0x80000000L
+//BIF_BX_PF0_MM_DATA
+#define BIF_BX_PF0_MM_DATA__MM_DATA__SHIFT                                                                    0x0
+#define BIF_BX_PF0_MM_DATA__MM_DATA_MASK                                                                      0xFFFFFFFFL
+//BIF_BX_PF0_MM_INDEX_HI
+#define BIF_BX_PF0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                           0x0
+#define BIF_BX_PF0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                             0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_INDEX
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX__SHIFT                                                              0x0
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX_MASK                                                                0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_DATA
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA__SHIFT                                                                0x0
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA_MASK                                                                  0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_INDEX_HI
+#define BIF_BX_PF0_RSMU_INDEX_HI__RSMU_INDEX_HI__SHIFT                                                        0x0
+#define BIF_BX_PF0_RSMU_INDEX_HI__RSMU_INDEX_HI_MASK                                                          0x000000FFL
+
+
+// addressBlock: nbif0_bif_bx_BIFDEC1
+//BIF_BX0_BIF_MM_INDACCESS_CNTL
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__WRITE_DIS__SHIFT                                                       0x0
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS__SHIFT                                                0x1
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__WRITE_DIS_MASK                                                         0x00000001L
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS_MASK                                                  0x00000002L
+//BIF_BX0_BUS_CNTL
+#define BIF_BX0_BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT                                                        0x6
+#define BIF_BX0_BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT                                                        0x7
+#define BIF_BX0_BUS_CNTL__SET_AZ_TC__SHIFT                                                                    0xa
+#define BIF_BX0_BUS_CNTL__SET_MC_TC__SHIFT                                                                    0xd
+#define BIF_BX0_BUS_CNTL__ZERO_BE_WR_EN__SHIFT                                                                0x10
+#define BIF_BX0_BUS_CNTL__ZERO_BE_RD_EN__SHIFT                                                                0x11
+#define BIF_BX0_BUS_CNTL__RD_STALL_IO_WR__SHIFT                                                               0x12
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS__SHIFT                                              0x18
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS__SHIFT                                          0x19
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS__SHIFT                                         0x1a
+#define BIF_BX0_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS__SHIFT                                        0x1b
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS__SHIFT                                          0x1c
+#define BIF_BX0_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN__SHIFT                                                     0x1d
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_WR_EN__SHIFT                                                          0x1e
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_RD_EN__SHIFT                                                          0x1f
+#define BIF_BX0_BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK                                                          0x00000040L
+#define BIF_BX0_BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK                                                          0x00000080L
+#define BIF_BX0_BUS_CNTL__SET_AZ_TC_MASK                                                                      0x00001C00L
+#define BIF_BX0_BUS_CNTL__SET_MC_TC_MASK                                                                      0x0000E000L
+#define BIF_BX0_BUS_CNTL__ZERO_BE_WR_EN_MASK                                                                  0x00010000L
+#define BIF_BX0_BUS_CNTL__ZERO_BE_RD_EN_MASK                                                                  0x00020000L
+#define BIF_BX0_BUS_CNTL__RD_STALL_IO_WR_MASK                                                                 0x00040000L
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS_MASK                                                0x01000000L
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS_MASK                                            0x02000000L
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS_MASK                                           0x04000000L
+#define BIF_BX0_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS_MASK                                          0x08000000L
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS_MASK                                            0x10000000L
+#define BIF_BX0_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN_MASK                                                       0x20000000L
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_WR_EN_MASK                                                            0x40000000L
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_RD_EN_MASK                                                            0x80000000L
+//BIF_BX0_BIF_SCRATCH0
+#define BIF_BX0_BIF_SCRATCH0__BIF_SCRATCH0__SHIFT                                                             0x0
+#define BIF_BX0_BIF_SCRATCH0__BIF_SCRATCH0_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_BIF_SCRATCH1
+#define BIF_BX0_BIF_SCRATCH1__BIF_SCRATCH1__SHIFT                                                             0x0
+#define BIF_BX0_BIF_SCRATCH1__BIF_SCRATCH1_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_BX_RESET_EN
+#define BIF_BX0_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN__SHIFT                                                  0x10
+#define BIF_BX0_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN_MASK                                                    0x00010000L
+//BIF_BX0_MM_CFGREGS_CNTL
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT                                                       0x0
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL__SHIFT                                                        0x6
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT                                                       0x1f
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK                                                         0x00000007L
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL_MASK                                                          0x000000C0L
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK                                                         0x80000000L
+//BIF_BX0_BX_RESET_CNTL
+#define BIF_BX0_BX_RESET_CNTL__LINK_TRAIN_EN__SHIFT                                                           0x0
+#define BIF_BX0_BX_RESET_CNTL__LINK_TRAIN_EN_MASK                                                             0x00000001L
+//BIF_BX0_INTERRUPT_CNTL
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+//BIF_BX0_INTERRUPT_CNTL2
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX0_CLKREQB_PAD_CNTL
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT                                                        0x0
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT                                                      0x1
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT                                                     0x2
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT                                                    0x3
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT                                                      0x5
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT                                                      0x6
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT                                                      0x7
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT                                                      0x8
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT                                                    0x9
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT                                                     0xa
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT                                                   0xb
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT                                                  0xc
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y__SHIFT                                                        0xd
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK                                                          0x00000001L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK                                                        0x00000002L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK                                                       0x00000004L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK                                                      0x00000018L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK                                                        0x00000020L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK                                                        0x00000040L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK                                                        0x00000080L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK                                                        0x00000100L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK                                                      0x00000200L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK                                                       0x00000400L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK                                                     0x00000800L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK                                                    0x00001000L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y_MASK                                                          0x00002000L
+//BIF_BX0_BIF_FEATURES_CONTROL_MISC
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT                                          0x0
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT                                          0x1
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT                                          0x2
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT                                          0x3
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS__SHIFT                                          0xd
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS__SHIFT                                       0xe
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT__SHIFT                                           0x10
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR__SHIFT                   0x18
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK                                            0x00000001L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK                                            0x00000002L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK                                            0x00000004L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK                                            0x00000008L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS_MASK                                            0x00002000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS_MASK                                         0x00004000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT_MASK                                             0x00FF0000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR_MASK                     0x01000000L
+//BIF_BX0_HDP_ATOMIC_CONTROL_MISC
+#define BIF_BX0_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT__SHIFT                                      0x0
+#define BIF_BX0_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT_MASK                                        0x000000FFL
+//BIF_BX0_BIF_DOORBELL_CNTL
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS__SHIFT                                                       0x0
+#define BIF_BX0_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS__SHIFT                                                     0x1
+#define BIF_BX0_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN__SHIFT                                                    0x2
+#define BIF_BX0_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS__SHIFT                                         0x3
+#define BIF_BX0_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT                                                 0x4
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS__SHIFT                                                  0x18
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0__SHIFT                                               0x19
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1__SHIFT                                               0x1a
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2__SHIFT                                               0x1b
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS_MASK                                                         0x00000001L
+#define BIF_BX0_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS_MASK                                                       0x00000002L
+#define BIF_BX0_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN_MASK                                                      0x00000004L
+#define BIF_BX0_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS_MASK                                           0x00000008L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK                                                   0x00000010L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS_MASK                                                    0x01000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0_MASK                                                 0x02000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1_MASK                                                 0x04000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2_MASK                                                 0x08000000L
+//BIF_BX0_BIF_FB_EN
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX0_BIF_MST_TRANS_PENDING_VF
+#define BIF_BX0_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX0_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX0_BIF_SLV_TRANS_PENDING_VF
+#define BIF_BX0_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX0_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX0_BACO_CNTL
+#define BIF_BX0_BACO_CNTL__BACO_EN__SHIFT                                                                     0x0
+#define BIF_BX0_BACO_CNTL__BACO_BIF_LCLK_SWITCH__SHIFT                                                        0x1
+#define BIF_BX0_BACO_CNTL__BACO_DUMMY_EN__SHIFT                                                               0x2
+#define BIF_BX0_BACO_CNTL__BACO_POWER_OFF__SHIFT                                                              0x3
+#define BIF_BX0_BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT                                                          0x5
+#define BIF_BX0_BACO_CNTL__BACO_RST_INTR_MASK__SHIFT                                                          0x6
+#define BIF_BX0_BACO_CNTL__BACO_MODE__SHIFT                                                                   0x8
+#define BIF_BX0_BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT                                                         0x9
+#define BIF_BX0_BACO_CNTL__BACO_AUTO_EXIT__SHIFT                                                              0x1f
+#define BIF_BX0_BACO_CNTL__BACO_EN_MASK                                                                       0x00000001L
+#define BIF_BX0_BACO_CNTL__BACO_BIF_LCLK_SWITCH_MASK                                                          0x00000002L
+#define BIF_BX0_BACO_CNTL__BACO_DUMMY_EN_MASK                                                                 0x00000004L
+#define BIF_BX0_BACO_CNTL__BACO_POWER_OFF_MASK                                                                0x00000008L
+#define BIF_BX0_BACO_CNTL__BACO_DSTATE_BYPASS_MASK                                                            0x00000020L
+#define BIF_BX0_BACO_CNTL__BACO_RST_INTR_MASK_MASK                                                            0x00000040L
+#define BIF_BX0_BACO_CNTL__BACO_MODE_MASK                                                                     0x00000100L
+#define BIF_BX0_BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK                                                           0x00000200L
+#define BIF_BX0_BACO_CNTL__BACO_AUTO_EXIT_MASK                                                                0x80000000L
+//BIF_BX0_BIF_BACO_EXIT_TIME0
+#define BIF_BX0_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER__SHIFT                                          0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER_MASK                                            0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER1
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER__SHIFT                                         0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN__SHIFT                                            0x18
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_ENDING_AUTO_BY_RSMU_INTR_CLR__SHIFT                        0x19
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS__SHIFT                                                 0x1a
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH__SHIFT                                           0x1b
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW__SHIFT                                            0x1c
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL__SHIFT                                                    0x1d
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS__SHIFT                                     0x1f
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER_MASK                                           0x000FFFFFL
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN_MASK                                              0x01000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_ENDING_AUTO_BY_RSMU_INTR_CLR_MASK                          0x02000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS_MASK                                                   0x04000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH_MASK                                             0x08000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW_MASK                                              0x10000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL_MASK                                                      0x60000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS_MASK                                       0x80000000L
+//BIF_BX0_BIF_BACO_EXIT_TIMER2
+#define BIF_BX0_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER__SHIFT                                         0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER_MASK                                           0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER3
+#define BIF_BX0_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER__SHIFT                                     0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER_MASK                                       0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER4
+#define BIF_BX0_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER__SHIFT                                      0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER_MASK                                        0x000FFFFFL
+//BIF_BX0_MEM_TYPE_CNTL
+#define BIF_BX0_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3__SHIFT                                                        0x0
+#define BIF_BX0_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3_MASK                                                          0x00000001L
+//BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE__SHIFT                                                     0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE__SHIFT                                                  0x1
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE__SHIFT                                                    0x8
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE_MASK                                                       0x00000001L
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE_MASK                                                    0x00000002L
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE_MASK                                                      0x00000100L
+//BIF_BX0_NBIF_GFX_ADDR_LUT_0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_0__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_0__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_1
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_1__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_1__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_2
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_2__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_2__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_3
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_3__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_3__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_4
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_4__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_4__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_5
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_5__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_5__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_6
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_6__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_6__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_7
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_7__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_7__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_8
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_8__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_8__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_9
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_9__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_9__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_10
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_10__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_10__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_11
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_11__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_11__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_12
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_12__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_12__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_13
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_13__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_13__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_14
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_14__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_14__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_15
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_15__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_15__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX0_MAILBOX_INDEX
+#define BIF_BX0_MAILBOX_INDEX__MAILBOX_INDEX__SHIFT                                                           0x0
+#define BIF_BX0_MAILBOX_INDEX__MAILBOX_INDEX_MASK                                                             0x0000001FL
+//BIF_BX0_BIF_PERSTB_PAD_CNTL
+#define BIF_BX0_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX0_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL_MASK                                                     0x0000FFFFL
+//BIF_BX0_BIF_PX_EN_PAD_CNTL
+#define BIF_BX0_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL__SHIFT                                                     0x0
+#define BIF_BX0_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL_MASK                                                       0x00000FFFL
+//BIF_BX0_BIF_REFPADKIN_PAD_CNTL
+#define BIF_BX0_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL__SHIFT                                             0x0
+#define BIF_BX0_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL_MASK                                               0x000000FFL
+//BIF_BX0_BIF_CLKREQB_PAD_CNTL
+#define BIF_BX0_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL__SHIFT                                                 0x0
+#define BIF_BX0_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_MASK                                                   0x7FFFFFFFL
+//BIF_BX0_BIF_PWRBRK_PAD_CNTL
+#define BIF_BX0_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX0_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL_MASK                                                     0x000000FFL
+
+
+// addressBlock: nbif0_bif_bx_pf_BIFPFVFDEC1
+//BIF_BX_PF0_BIF_BME_STATUS
+#define BIF_BX_PF0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                                      0x0
+#define BIF_BX_PF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                                0x10
+#define BIF_BX_PF0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                                        0x00000001L
+#define BIF_BX_PF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                                  0x00010000L
+//BIF_BX_PF0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                                0x0
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                             0x1
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                                0x2
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                                    0x3
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                          0x10
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                                       0x11
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                          0x12
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                              0x13
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                                  0x00000001L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                               0x00000002L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                                  0x00000004L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                                      0x00000008L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                            0x00010000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                                         0x00020000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                            0x00040000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                                0x00080000L
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                          0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                            0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_PF0_BIF_DOORBELL_INT_CNTL
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS__SHIFT                                    0x0
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS__SHIFT                                   0x1
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS__SHIFT                         0x2
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR__SHIFT                                     0x10
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR__SHIFT                                    0x11
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR__SHIFT                          0x12
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                         0x17
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE__SHIFT                                   0x18
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE__SHIFT                                  0x19
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE__SHIFT                        0x1a
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                            0x1c
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                       0x1d
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                       0x1e
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                           0x1f
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS_MASK                                      0x00000001L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS_MASK                                     0x00000002L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS_MASK                           0x00000004L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR_MASK                                       0x00010000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR_MASK                                      0x00020000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR_MASK                            0x00040000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE_MASK                           0x00800000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE_MASK                                     0x01000000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE_MASK                                    0x02000000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE_MASK                          0x04000000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE_MASK                              0x10000000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                         0x20000000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                         0x40000000L
+#define BIF_BX_PF0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE_MASK                             0x80000000L
+//BIF_BX_PF0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//BIF_BX_PF0_BIF_TRANS_PENDING
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                            0x0
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                            0x1
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                              0x00000001L
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                              0x00000002L
+//BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                                0x0
+#define BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                                  0x00000001L
+//BIF_BX_PF0_BIF_RB_CNTL
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_ENABLE__SHIFT                                                              0x0
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_SIZE__SHIFT                                                                0x1
+#define BIF_BX_PF0_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                  0x8
+#define BIF_BX_PF0_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                   0x9
+#define BIF_BX_PF0_BIF_RB_CNTL__BIF_RB_TRAN__SHIFT                                                            0x11
+#define BIF_BX_PF0_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL__SHIFT                                               0x19
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY__SHIFT                                                   0x1a
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_INTR_ARB_MODE__SHIFT                                                       0x1d
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE__SHIFT                                                  0x1e
+#define BIF_BX_PF0_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                    0x1f
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_ENABLE_MASK                                                                0x00000001L
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_SIZE_MASK                                                                  0x0000003EL
+#define BIF_BX_PF0_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                    0x00000100L
+#define BIF_BX_PF0_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                     0x00003E00L
+#define BIF_BX_PF0_BIF_RB_CNTL__BIF_RB_TRAN_MASK                                                              0x00020000L
+#define BIF_BX_PF0_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL_MASK                                                 0x02000000L
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY_MASK                                                     0x1C000000L
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_INTR_ARB_MODE_MASK                                                         0x20000000L
+#define BIF_BX_PF0_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE_MASK                                                    0x40000000L
+#define BIF_BX_PF0_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                      0x80000000L
+//BIF_BX_PF0_BIF_RB_BASE
+#define BIF_BX_PF0_BIF_RB_BASE__ADDR__SHIFT                                                                   0x0
+#define BIF_BX_PF0_BIF_RB_BASE__ADDR_MASK                                                                     0xFFFFFFFFL
+//BIF_BX_PF0_BIF_RB_RPTR
+#define BIF_BX_PF0_BIF_RB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define BIF_BX_PF0_BIF_RB_RPTR__OFFSET_MASK                                                                   0x0003FFFCL
+//BIF_BX_PF0_BIF_RB_WPTR
+#define BIF_BX_PF0_BIF_RB_WPTR__BIF_RB_OVERFLOW__SHIFT                                                        0x0
+#define BIF_BX_PF0_BIF_RB_WPTR__OFFSET__SHIFT                                                                 0x2
+#define BIF_BX_PF0_BIF_RB_WPTR__BIF_RB_OVERFLOW_MASK                                                          0x00000001L
+#define BIF_BX_PF0_BIF_RB_WPTR__OFFSET_MASK                                                                   0x0003FFFCL
+//BIF_BX_PF0_BIF_RB_WPTR_ADDR_HI
+#define BIF_BX_PF0_BIF_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define BIF_BX_PF0_BIF_RB_WPTR_ADDR_HI__ADDR_MASK                                                             0x000000FFL
+//BIF_BX_PF0_BIF_RB_WPTR_ADDR_LO
+#define BIF_BX_PF0_BIF_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define BIF_BX_PF0_BIF_RB_WPTR_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+
+
+// addressBlock: nbif0_bif_bx_SYSDEC:1
+//BIF_BX1_PCIE_INDEX
+#define BIF_BX1_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX1_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX1_PCIE_DATA
+#define BIF_BX1_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX1_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//BIF_BX1_PCIE_INDEX2
+#define BIF_BX1_PCIE_INDEX2__PCIE_INDEX2__SHIFT                                                               0x0
+#define BIF_BX1_PCIE_INDEX2__PCIE_INDEX2_MASK                                                                 0xFFFFFFFFL
+//BIF_BX1_PCIE_DATA2
+#define BIF_BX1_PCIE_DATA2__PCIE_DATA2__SHIFT                                                                 0x0
+#define BIF_BX1_PCIE_DATA2__PCIE_DATA2_MASK                                                                   0xFFFFFFFFL
+//BIF_BX1_PCIE_INDEX_HI
+#define BIF_BX1_PCIE_INDEX_HI__PCIE_INDEX_HI__SHIFT                                                           0x0
+#define BIF_BX1_PCIE_INDEX_HI__PCIE_INDEX_HI_MASK                                                             0x000000FFL
+//BIF_BX1_PCIE_INDEX2_HI
+#define BIF_BX1_PCIE_INDEX2_HI__PCIE_INDEX2_HI__SHIFT                                                         0x0
+#define BIF_BX1_PCIE_INDEX2_HI__PCIE_INDEX2_HI_MASK                                                           0x000000FFL
+//BIF_BX1_SBIOS_SCRATCH_0
+#define BIF_BX1_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_1
+#define BIF_BX1_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_2
+#define BIF_BX1_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_3
+#define BIF_BX1_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_0
+#define BIF_BX1_BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_1
+#define BIF_BX1_BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_2
+#define BIF_BX1_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_3
+#define BIF_BX1_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_4
+#define BIF_BX1_BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_5
+#define BIF_BX1_BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_6
+#define BIF_BX1_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_7
+#define BIF_BX1_BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_8
+#define BIF_BX1_BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_9
+#define BIF_BX1_BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_10
+#define BIF_BX1_BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_11
+#define BIF_BX1_BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_12
+#define BIF_BX1_BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_13
+#define BIF_BX1_BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_14
+#define BIF_BX1_BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_15
+#define BIF_BX1_BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIF_RLC_INTR_CNTL
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX1_BIF_RLC_INTR_CNTL__RLC_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+//BIF_BX1_BIF_VCE_INTR_CNTL
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX1_BIF_VCE_INTR_CNTL__VCE_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+//BIF_BX1_BIF_UVD_INTR_CNTL
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_INST_SEL__SHIFT                                                        0x1c
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+#define BIF_BX1_BIF_UVD_INTR_CNTL__UVD_INST_SEL_MASK                                                          0xF0000000L
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR1
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR2
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR3
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR4
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR5
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR6
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR7
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_CNTL
+#define BIF_BX1_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE_MASK                                                         0x000000FFL
+//BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL__SHIFT                                                 0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL_MASK                                                   0xFFFFFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL__SHIFT                                                   0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL__SHIFT                                 0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL_MASK                                   0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_0
+#define BIF_BX1_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_1
+#define BIF_BX1_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_2
+#define BIF_BX1_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_3
+#define BIF_BX1_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_4
+#define BIF_BX1_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_5
+#define BIF_BX1_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_6
+#define BIF_BX1_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_7
+#define BIF_BX1_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_8
+#define BIF_BX1_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_9
+#define BIF_BX1_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_10
+#define BIF_BX1_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_11
+#define BIF_BX1_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_12
+#define BIF_BX1_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_13
+#define BIF_BX1_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_14
+#define BIF_BX1_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_15
+#define BIF_BX1_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_0
+#define BIF_BX1_FW_SCRATCH_0__FW_SCRATCH_0__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_0__FW_SCRATCH_0_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_1
+#define BIF_BX1_FW_SCRATCH_1__FW_SCRATCH_1__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_1__FW_SCRATCH_1_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_2
+#define BIF_BX1_FW_SCRATCH_2__FW_SCRATCH_2__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_2__FW_SCRATCH_2_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_3
+#define BIF_BX1_FW_SCRATCH_3__FW_SCRATCH_3__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_3__FW_SCRATCH_3_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_4
+#define BIF_BX1_FW_SCRATCH_4__FW_SCRATCH_4__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_4__FW_SCRATCH_4_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_5
+#define BIF_BX1_FW_SCRATCH_5__FW_SCRATCH_5__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_5__FW_SCRATCH_5_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_6
+#define BIF_BX1_FW_SCRATCH_6__FW_SCRATCH_6__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_6__FW_SCRATCH_6_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_7
+#define BIF_BX1_FW_SCRATCH_7__FW_SCRATCH_7__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_7__FW_SCRATCH_7_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_8
+#define BIF_BX1_FW_SCRATCH_8__FW_SCRATCH_8__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_8__FW_SCRATCH_8_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_9
+#define BIF_BX1_FW_SCRATCH_9__FW_SCRATCH_9__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_9__FW_SCRATCH_9_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_10
+#define BIF_BX1_FW_SCRATCH_10__FW_SCRATCH_10__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_10__FW_SCRATCH_10_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_11
+#define BIF_BX1_FW_SCRATCH_11__FW_SCRATCH_11__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_11__FW_SCRATCH_11_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_12
+#define BIF_BX1_FW_SCRATCH_12__FW_SCRATCH_12__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_12__FW_SCRATCH_12_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_13
+#define BIF_BX1_FW_SCRATCH_13__FW_SCRATCH_13__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_13__FW_SCRATCH_13_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_14
+#define BIF_BX1_FW_SCRATCH_14__FW_SCRATCH_14__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_14__FW_SCRATCH_14_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_15
+#define BIF_BX1_FW_SCRATCH_15__FW_SCRATCH_15__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_15__FW_SCRATCH_15_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_4
+#define BIF_BX1_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_5
+#define BIF_BX1_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_6
+#define BIF_BX1_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_7
+#define BIF_BX1_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_8
+#define BIF_BX1_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_9
+#define BIF_BX1_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_10
+#define BIF_BX1_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_11
+#define BIF_BX1_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_12
+#define BIF_BX1_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_13
+#define BIF_BX1_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_14
+#define BIF_BX1_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_15
+#define BIF_BX1_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_BIFDEC1:1
+//RCC_DWN_DEV0_2_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_2_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_2_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_2_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_2_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                 0x5
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                   0x00000020L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_BIFDEC1:1
+//RCC_DWNP_DEV0_2_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_2_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                           0x4
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                             0x00000010L
+//RCC_DWNP_DEV0_2_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_ep_dev0_BIFDEC1:1
+//RCC_EP_DEV0_2_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_2_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_2_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_2_EP_PCIE_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_2_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_2_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                  0x5
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                    0x00000020L
+//RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_2_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_2_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_2_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_2_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_2_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                          0x4
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                            0x00000010L
+
+
+// addressBlock: nbif0_bif_bx_pf_SYSPFVFDEC:1
+//BIF_BX_PF1_MM_INDEX
+#define BIF_BX_PF1_MM_INDEX__MM_OFFSET__SHIFT                                                                 0x0
+#define BIF_BX_PF1_MM_INDEX__MM_APER__SHIFT                                                                   0x1f
+#define BIF_BX_PF1_MM_INDEX__MM_OFFSET_MASK                                                                   0x7FFFFFFFL
+#define BIF_BX_PF1_MM_INDEX__MM_APER_MASK                                                                     0x80000000L
+//BIF_BX_PF1_MM_DATA
+#define BIF_BX_PF1_MM_DATA__MM_DATA__SHIFT                                                                    0x0
+#define BIF_BX_PF1_MM_DATA__MM_DATA_MASK                                                                      0xFFFFFFFFL
+//BIF_BX_PF1_MM_INDEX_HI
+#define BIF_BX_PF1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                           0x0
+#define BIF_BX_PF1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                             0xFFFFFFFFL
+//BIF_BX_PF1_RSMU_INDEX
+#define BIF_BX_PF1_RSMU_INDEX__RSMU_INDEX__SHIFT                                                              0x0
+#define BIF_BX_PF1_RSMU_INDEX__RSMU_INDEX_MASK                                                                0xFFFFFFFFL
+//BIF_BX_PF1_RSMU_DATA
+#define BIF_BX_PF1_RSMU_DATA__RSMU_DATA__SHIFT                                                                0x0
+#define BIF_BX_PF1_RSMU_DATA__RSMU_DATA_MASK                                                                  0xFFFFFFFFL
+
+
+// addressBlock: nbif0_bif_bx_BIFDEC1:1
+//BIF_BX1_BIF_MM_INDACCESS_CNTL
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__WRITE_DIS__SHIFT                                                       0x0
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS__SHIFT                                                0x1
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__WRITE_DIS_MASK                                                         0x00000001L
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS_MASK                                                  0x00000002L
+//BIF_BX1_BUS_CNTL
+#define BIF_BX1_BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT                                                        0x6
+#define BIF_BX1_BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT                                                        0x7
+#define BIF_BX1_BUS_CNTL__SET_AZ_TC__SHIFT                                                                    0xa
+#define BIF_BX1_BUS_CNTL__SET_MC_TC__SHIFT                                                                    0xd
+#define BIF_BX1_BUS_CNTL__ZERO_BE_WR_EN__SHIFT                                                                0x10
+#define BIF_BX1_BUS_CNTL__ZERO_BE_RD_EN__SHIFT                                                                0x11
+#define BIF_BX1_BUS_CNTL__RD_STALL_IO_WR__SHIFT                                                               0x12
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS__SHIFT                                              0x18
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS__SHIFT                                          0x19
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS__SHIFT                                         0x1a
+#define BIF_BX1_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS__SHIFT                                        0x1b
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS__SHIFT                                          0x1c
+#define BIF_BX1_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN__SHIFT                                                     0x1d
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_WR_EN__SHIFT                                                          0x1e
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_RD_EN__SHIFT                                                          0x1f
+#define BIF_BX1_BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK                                                          0x00000040L
+#define BIF_BX1_BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK                                                          0x00000080L
+#define BIF_BX1_BUS_CNTL__SET_AZ_TC_MASK                                                                      0x00001C00L
+#define BIF_BX1_BUS_CNTL__SET_MC_TC_MASK                                                                      0x0000E000L
+#define BIF_BX1_BUS_CNTL__ZERO_BE_WR_EN_MASK                                                                  0x00010000L
+#define BIF_BX1_BUS_CNTL__ZERO_BE_RD_EN_MASK                                                                  0x00020000L
+#define BIF_BX1_BUS_CNTL__RD_STALL_IO_WR_MASK                                                                 0x00040000L
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS_MASK                                                0x01000000L
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS_MASK                                            0x02000000L
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS_MASK                                           0x04000000L
+#define BIF_BX1_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS_MASK                                          0x08000000L
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS_MASK                                            0x10000000L
+#define BIF_BX1_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN_MASK                                                       0x20000000L
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_WR_EN_MASK                                                            0x40000000L
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_RD_EN_MASK                                                            0x80000000L
+//BIF_BX1_BIF_SCRATCH0
+#define BIF_BX1_BIF_SCRATCH0__BIF_SCRATCH0__SHIFT                                                             0x0
+#define BIF_BX1_BIF_SCRATCH0__BIF_SCRATCH0_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_BIF_SCRATCH1
+#define BIF_BX1_BIF_SCRATCH1__BIF_SCRATCH1__SHIFT                                                             0x0
+#define BIF_BX1_BIF_SCRATCH1__BIF_SCRATCH1_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_BX_RESET_EN
+#define BIF_BX1_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN__SHIFT                                                  0x10
+#define BIF_BX1_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN_MASK                                                    0x00010000L
+//BIF_BX1_MM_CFGREGS_CNTL
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT                                                       0x0
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL__SHIFT                                                        0x6
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT                                                       0x1f
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK                                                         0x00000007L
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL_MASK                                                          0x000000C0L
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK                                                         0x80000000L
+//BIF_BX1_BX_RESET_CNTL
+#define BIF_BX1_BX_RESET_CNTL__LINK_TRAIN_EN__SHIFT                                                           0x0
+#define BIF_BX1_BX_RESET_CNTL__LINK_TRAIN_EN_MASK                                                             0x00000001L
+//BIF_BX1_INTERRUPT_CNTL
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX1_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX1_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX1_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX1_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX1_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX1_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX1_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX1_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX1_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX1_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+//BIF_BX1_INTERRUPT_CNTL2
+#define BIF_BX1_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX1_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX1_CLKREQB_PAD_CNTL
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT                                                        0x0
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT                                                      0x1
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT                                                     0x2
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT                                                    0x3
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT                                                      0x5
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT                                                      0x6
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT                                                      0x7
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT                                                      0x8
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT                                                    0x9
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT                                                     0xa
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT                                                   0xb
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT                                                  0xc
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y__SHIFT                                                        0xd
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK                                                          0x00000001L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK                                                        0x00000002L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK                                                       0x00000004L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK                                                      0x00000018L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK                                                        0x00000020L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK                                                        0x00000040L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK                                                        0x00000080L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK                                                        0x00000100L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK                                                      0x00000200L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK                                                       0x00000400L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK                                                     0x00000800L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK                                                    0x00001000L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y_MASK                                                          0x00002000L
+//BIF_BX1_BIF_FEATURES_CONTROL_MISC
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT                                          0x0
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT                                          0x1
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT                                          0x2
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT                                          0x3
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS__SHIFT                                          0xd
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS__SHIFT                                       0xe
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT__SHIFT                                           0x10
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR__SHIFT                   0x18
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK                                            0x00000001L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK                                            0x00000002L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK                                            0x00000004L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK                                            0x00000008L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS_MASK                                            0x00002000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS_MASK                                         0x00004000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT_MASK                                             0x00FF0000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR_MASK                     0x01000000L
+//BIF_BX1_HDP_ATOMIC_CONTROL_MISC
+#define BIF_BX1_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT__SHIFT                                      0x0
+#define BIF_BX1_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT_MASK                                        0x000000FFL
+//BIF_BX1_BIF_DOORBELL_CNTL
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS__SHIFT                                                       0x0
+#define BIF_BX1_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS__SHIFT                                                     0x1
+#define BIF_BX1_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN__SHIFT                                                    0x2
+#define BIF_BX1_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS__SHIFT                                         0x3
+#define BIF_BX1_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT                                                 0x4
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS__SHIFT                                                  0x18
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0__SHIFT                                               0x19
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1__SHIFT                                               0x1a
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2__SHIFT                                               0x1b
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS_MASK                                                         0x00000001L
+#define BIF_BX1_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS_MASK                                                       0x00000002L
+#define BIF_BX1_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN_MASK                                                      0x00000004L
+#define BIF_BX1_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS_MASK                                           0x00000008L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK                                                   0x00000010L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS_MASK                                                    0x01000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0_MASK                                                 0x02000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1_MASK                                                 0x04000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2_MASK                                                 0x08000000L
+//BIF_BX1_BIF_FB_EN
+#define BIF_BX1_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX1_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX1_BIF_MST_TRANS_PENDING_VF
+#define BIF_BX1_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX1_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX1_BIF_SLV_TRANS_PENDING_VF
+#define BIF_BX1_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX1_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX1_BACO_CNTL
+#define BIF_BX1_BACO_CNTL__BACO_EN__SHIFT                                                                     0x0
+#define BIF_BX1_BACO_CNTL__BACO_BIF_LCLK_SWITCH__SHIFT                                                        0x1
+#define BIF_BX1_BACO_CNTL__BACO_DUMMY_EN__SHIFT                                                               0x2
+#define BIF_BX1_BACO_CNTL__BACO_POWER_OFF__SHIFT                                                              0x3
+#define BIF_BX1_BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT                                                          0x5
+#define BIF_BX1_BACO_CNTL__BACO_RST_INTR_MASK__SHIFT                                                          0x6
+#define BIF_BX1_BACO_CNTL__BACO_MODE__SHIFT                                                                   0x8
+#define BIF_BX1_BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT                                                         0x9
+#define BIF_BX1_BACO_CNTL__BACO_AUTO_EXIT__SHIFT                                                              0x1f
+#define BIF_BX1_BACO_CNTL__BACO_EN_MASK                                                                       0x00000001L
+#define BIF_BX1_BACO_CNTL__BACO_BIF_LCLK_SWITCH_MASK                                                          0x00000002L
+#define BIF_BX1_BACO_CNTL__BACO_DUMMY_EN_MASK                                                                 0x00000004L
+#define BIF_BX1_BACO_CNTL__BACO_POWER_OFF_MASK                                                                0x00000008L
+#define BIF_BX1_BACO_CNTL__BACO_DSTATE_BYPASS_MASK                                                            0x00000020L
+#define BIF_BX1_BACO_CNTL__BACO_RST_INTR_MASK_MASK                                                            0x00000040L
+#define BIF_BX1_BACO_CNTL__BACO_MODE_MASK                                                                     0x00000100L
+#define BIF_BX1_BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK                                                           0x00000200L
+#define BIF_BX1_BACO_CNTL__BACO_AUTO_EXIT_MASK                                                                0x80000000L
+//BIF_BX1_BIF_BACO_EXIT_TIME0
+#define BIF_BX1_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER__SHIFT                                          0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER_MASK                                            0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER1
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER__SHIFT                                         0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN__SHIFT                                            0x18
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_ENDING_AUTO_BY_RSMU_INTR_CLR__SHIFT                        0x19
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS__SHIFT                                                 0x1a
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH__SHIFT                                           0x1b
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW__SHIFT                                            0x1c
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL__SHIFT                                                    0x1d
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS__SHIFT                                     0x1f
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER_MASK                                           0x000FFFFFL
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN_MASK                                              0x01000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_ENDING_AUTO_BY_RSMU_INTR_CLR_MASK                          0x02000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS_MASK                                                   0x04000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH_MASK                                             0x08000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW_MASK                                              0x10000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL_MASK                                                      0x60000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS_MASK                                       0x80000000L
+//BIF_BX1_BIF_BACO_EXIT_TIMER2
+#define BIF_BX1_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER__SHIFT                                         0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER_MASK                                           0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER3
+#define BIF_BX1_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER__SHIFT                                     0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER_MASK                                       0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER4
+#define BIF_BX1_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER__SHIFT                                      0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER_MASK                                        0x000FFFFFL
+//BIF_BX1_MEM_TYPE_CNTL
+#define BIF_BX1_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3__SHIFT                                                        0x0
+#define BIF_BX1_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3_MASK                                                          0x00000001L
+//BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE__SHIFT                                                     0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE__SHIFT                                                  0x1
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE__SHIFT                                                    0x8
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE_MASK                                                       0x00000001L
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE_MASK                                                    0x00000002L
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE_MASK                                                      0x00000100L
+//BIF_BX1_NBIF_GFX_ADDR_LUT_0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_0__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_0__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_1
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_1__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_1__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_2
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_2__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_2__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_3
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_3__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_3__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_4
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_4__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_4__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_5
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_5__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_5__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_6
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_6__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_6__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_7
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_7__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_7__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_8
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_8__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_8__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_9
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_9__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_9__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_10
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_10__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_10__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_11
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_11__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_11__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_12
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_12__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_12__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_13
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_13__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_13__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_14
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_14__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_14__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_15
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_15__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_15__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_VF_REGWR_EN
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF0__SHIFT                                                           0x0
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF1__SHIFT                                                           0x1
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF2__SHIFT                                                           0x2
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF3__SHIFT                                                           0x3
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF4__SHIFT                                                           0x4
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF5__SHIFT                                                           0x5
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF6__SHIFT                                                           0x6
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF7__SHIFT                                                           0x7
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF8__SHIFT                                                           0x8
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF9__SHIFT                                                           0x9
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF10__SHIFT                                                          0xa
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF11__SHIFT                                                          0xb
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF12__SHIFT                                                          0xc
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF13__SHIFT                                                          0xd
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF14__SHIFT                                                          0xe
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF15__SHIFT                                                          0xf
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF16__SHIFT                                                          0x10
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF17__SHIFT                                                          0x11
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF18__SHIFT                                                          0x12
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF19__SHIFT                                                          0x13
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF20__SHIFT                                                          0x14
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF21__SHIFT                                                          0x15
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF22__SHIFT                                                          0x16
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF23__SHIFT                                                          0x17
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF24__SHIFT                                                          0x18
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF25__SHIFT                                                          0x19
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF26__SHIFT                                                          0x1a
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF27__SHIFT                                                          0x1b
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF28__SHIFT                                                          0x1c
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF29__SHIFT                                                          0x1d
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF30__SHIFT                                                          0x1e
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF0_MASK                                                             0x00000001L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF1_MASK                                                             0x00000002L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF2_MASK                                                             0x00000004L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF3_MASK                                                             0x00000008L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF4_MASK                                                             0x00000010L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF5_MASK                                                             0x00000020L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF6_MASK                                                             0x00000040L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF7_MASK                                                             0x00000080L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF8_MASK                                                             0x00000100L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF9_MASK                                                             0x00000200L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF10_MASK                                                            0x00000400L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF11_MASK                                                            0x00000800L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF12_MASK                                                            0x00001000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF13_MASK                                                            0x00002000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF14_MASK                                                            0x00004000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF15_MASK                                                            0x00008000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF16_MASK                                                            0x00010000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF17_MASK                                                            0x00020000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF18_MASK                                                            0x00040000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF19_MASK                                                            0x00080000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF20_MASK                                                            0x00100000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF21_MASK                                                            0x00200000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF22_MASK                                                            0x00400000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF23_MASK                                                            0x00800000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF24_MASK                                                            0x01000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF25_MASK                                                            0x02000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF26_MASK                                                            0x04000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF27_MASK                                                            0x08000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF28_MASK                                                            0x10000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF29_MASK                                                            0x20000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF30_MASK                                                            0x40000000L
+//BIF_BX1_VF_DOORBELL_EN
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0__SHIFT                                                     0x0
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1__SHIFT                                                     0x1
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2__SHIFT                                                     0x2
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3__SHIFT                                                     0x3
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4__SHIFT                                                     0x4
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5__SHIFT                                                     0x5
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6__SHIFT                                                     0x6
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7__SHIFT                                                     0x7
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8__SHIFT                                                     0x8
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9__SHIFT                                                     0x9
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10__SHIFT                                                    0xa
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11__SHIFT                                                    0xb
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12__SHIFT                                                    0xc
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13__SHIFT                                                    0xd
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14__SHIFT                                                    0xe
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15__SHIFT                                                    0xf
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16__SHIFT                                                    0x10
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17__SHIFT                                                    0x11
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18__SHIFT                                                    0x12
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19__SHIFT                                                    0x13
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20__SHIFT                                                    0x14
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21__SHIFT                                                    0x15
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22__SHIFT                                                    0x16
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23__SHIFT                                                    0x17
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24__SHIFT                                                    0x18
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25__SHIFT                                                    0x19
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26__SHIFT                                                    0x1a
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27__SHIFT                                                    0x1b
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28__SHIFT                                                    0x1c
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29__SHIFT                                                    0x1d
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30__SHIFT                                                    0x1e
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS__SHIFT                                                 0x1f
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0_MASK                                                       0x00000001L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1_MASK                                                       0x00000002L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2_MASK                                                       0x00000004L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3_MASK                                                       0x00000008L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4_MASK                                                       0x00000010L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5_MASK                                                       0x00000020L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6_MASK                                                       0x00000040L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7_MASK                                                       0x00000080L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8_MASK                                                       0x00000100L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9_MASK                                                       0x00000200L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10_MASK                                                      0x00000400L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11_MASK                                                      0x00000800L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12_MASK                                                      0x00001000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13_MASK                                                      0x00002000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14_MASK                                                      0x00004000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15_MASK                                                      0x00008000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16_MASK                                                      0x00010000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17_MASK                                                      0x00020000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18_MASK                                                      0x00040000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19_MASK                                                      0x00080000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20_MASK                                                      0x00100000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21_MASK                                                      0x00200000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22_MASK                                                      0x00400000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23_MASK                                                      0x00800000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24_MASK                                                      0x01000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25_MASK                                                      0x02000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26_MASK                                                      0x04000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27_MASK                                                      0x08000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28_MASK                                                      0x10000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29_MASK                                                      0x20000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30_MASK                                                      0x40000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS_MASK                                                   0x80000000L
+//BIF_BX1_VF_FB_EN
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF0__SHIFT                                                                 0x0
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF1__SHIFT                                                                 0x1
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF2__SHIFT                                                                 0x2
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF3__SHIFT                                                                 0x3
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF4__SHIFT                                                                 0x4
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF5__SHIFT                                                                 0x5
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF6__SHIFT                                                                 0x6
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF7__SHIFT                                                                 0x7
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF8__SHIFT                                                                 0x8
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF9__SHIFT                                                                 0x9
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF10__SHIFT                                                                0xa
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF11__SHIFT                                                                0xb
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF12__SHIFT                                                                0xc
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF13__SHIFT                                                                0xd
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF14__SHIFT                                                                0xe
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF15__SHIFT                                                                0xf
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF16__SHIFT                                                                0x10
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF17__SHIFT                                                                0x11
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF18__SHIFT                                                                0x12
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF19__SHIFT                                                                0x13
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF20__SHIFT                                                                0x14
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF21__SHIFT                                                                0x15
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF22__SHIFT                                                                0x16
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF23__SHIFT                                                                0x17
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF24__SHIFT                                                                0x18
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF25__SHIFT                                                                0x19
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF26__SHIFT                                                                0x1a
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF27__SHIFT                                                                0x1b
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF28__SHIFT                                                                0x1c
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF29__SHIFT                                                                0x1d
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF30__SHIFT                                                                0x1e
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF0_MASK                                                                   0x00000001L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF1_MASK                                                                   0x00000002L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF2_MASK                                                                   0x00000004L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF3_MASK                                                                   0x00000008L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF4_MASK                                                                   0x00000010L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF5_MASK                                                                   0x00000020L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF6_MASK                                                                   0x00000040L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF7_MASK                                                                   0x00000080L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF8_MASK                                                                   0x00000100L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF9_MASK                                                                   0x00000200L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF10_MASK                                                                  0x00000400L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF11_MASK                                                                  0x00000800L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF12_MASK                                                                  0x00001000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF13_MASK                                                                  0x00002000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF14_MASK                                                                  0x00004000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF15_MASK                                                                  0x00008000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF16_MASK                                                                  0x00010000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF17_MASK                                                                  0x00020000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF18_MASK                                                                  0x00040000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF19_MASK                                                                  0x00080000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF20_MASK                                                                  0x00100000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF21_MASK                                                                  0x00200000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF22_MASK                                                                  0x00400000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF23_MASK                                                                  0x00800000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF24_MASK                                                                  0x01000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF25_MASK                                                                  0x02000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF26_MASK                                                                  0x04000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF27_MASK                                                                  0x08000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF28_MASK                                                                  0x10000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF29_MASK                                                                  0x20000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF30_MASK                                                                  0x40000000L
+//BIF_BX1_VF_REGWR_STATUS
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0__SHIFT                                                   0x0
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1__SHIFT                                                   0x1
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2__SHIFT                                                   0x2
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3__SHIFT                                                   0x3
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4__SHIFT                                                   0x4
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5__SHIFT                                                   0x5
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6__SHIFT                                                   0x6
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7__SHIFT                                                   0x7
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8__SHIFT                                                   0x8
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9__SHIFT                                                   0x9
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10__SHIFT                                                  0xa
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11__SHIFT                                                  0xb
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12__SHIFT                                                  0xc
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13__SHIFT                                                  0xd
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14__SHIFT                                                  0xe
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15__SHIFT                                                  0xf
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16__SHIFT                                                  0x10
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17__SHIFT                                                  0x11
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18__SHIFT                                                  0x12
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19__SHIFT                                                  0x13
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20__SHIFT                                                  0x14
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21__SHIFT                                                  0x15
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22__SHIFT                                                  0x16
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23__SHIFT                                                  0x17
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24__SHIFT                                                  0x18
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25__SHIFT                                                  0x19
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26__SHIFT                                                  0x1a
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27__SHIFT                                                  0x1b
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28__SHIFT                                                  0x1c
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29__SHIFT                                                  0x1d
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30__SHIFT                                                  0x1e
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0_MASK                                                     0x00000001L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1_MASK                                                     0x00000002L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2_MASK                                                     0x00000004L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3_MASK                                                     0x00000008L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4_MASK                                                     0x00000010L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5_MASK                                                     0x00000020L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6_MASK                                                     0x00000040L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7_MASK                                                     0x00000080L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8_MASK                                                     0x00000100L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9_MASK                                                     0x00000200L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10_MASK                                                    0x00000400L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11_MASK                                                    0x00000800L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12_MASK                                                    0x00001000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13_MASK                                                    0x00002000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14_MASK                                                    0x00004000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15_MASK                                                    0x00008000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16_MASK                                                    0x00010000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17_MASK                                                    0x00020000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18_MASK                                                    0x00040000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19_MASK                                                    0x00080000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20_MASK                                                    0x00100000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21_MASK                                                    0x00200000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22_MASK                                                    0x00400000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23_MASK                                                    0x00800000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24_MASK                                                    0x01000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25_MASK                                                    0x02000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26_MASK                                                    0x04000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27_MASK                                                    0x08000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28_MASK                                                    0x10000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29_MASK                                                    0x20000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30_MASK                                                    0x40000000L
+//BIF_BX1_VF_DOORBELL_STATUS
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0__SHIFT                                             0x0
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1__SHIFT                                             0x1
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2__SHIFT                                             0x2
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3__SHIFT                                             0x3
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4__SHIFT                                             0x4
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5__SHIFT                                             0x5
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6__SHIFT                                             0x6
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7__SHIFT                                             0x7
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8__SHIFT                                             0x8
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9__SHIFT                                             0x9
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10__SHIFT                                            0xa
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11__SHIFT                                            0xb
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12__SHIFT                                            0xc
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13__SHIFT                                            0xd
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14__SHIFT                                            0xe
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15__SHIFT                                            0xf
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16__SHIFT                                            0x10
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17__SHIFT                                            0x11
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18__SHIFT                                            0x12
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19__SHIFT                                            0x13
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20__SHIFT                                            0x14
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21__SHIFT                                            0x15
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22__SHIFT                                            0x16
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23__SHIFT                                            0x17
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24__SHIFT                                            0x18
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25__SHIFT                                            0x19
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26__SHIFT                                            0x1a
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27__SHIFT                                            0x1b
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28__SHIFT                                            0x1c
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29__SHIFT                                            0x1d
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30__SHIFT                                            0x1e
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0_MASK                                               0x00000001L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1_MASK                                               0x00000002L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2_MASK                                               0x00000004L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3_MASK                                               0x00000008L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4_MASK                                               0x00000010L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5_MASK                                               0x00000020L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6_MASK                                               0x00000040L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7_MASK                                               0x00000080L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8_MASK                                               0x00000100L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9_MASK                                               0x00000200L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10_MASK                                              0x00000400L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11_MASK                                              0x00000800L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12_MASK                                              0x00001000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13_MASK                                              0x00002000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14_MASK                                              0x00004000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15_MASK                                              0x00008000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16_MASK                                              0x00010000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17_MASK                                              0x00020000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18_MASK                                              0x00040000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19_MASK                                              0x00080000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20_MASK                                              0x00100000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21_MASK                                              0x00200000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22_MASK                                              0x00400000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23_MASK                                              0x00800000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24_MASK                                              0x01000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25_MASK                                              0x02000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26_MASK                                              0x04000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27_MASK                                              0x08000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28_MASK                                              0x10000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29_MASK                                              0x20000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30_MASK                                              0x40000000L
+//BIF_BX1_VF_FB_STATUS
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF0__SHIFT                                                         0x0
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF1__SHIFT                                                         0x1
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF2__SHIFT                                                         0x2
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF3__SHIFT                                                         0x3
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF4__SHIFT                                                         0x4
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF5__SHIFT                                                         0x5
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF6__SHIFT                                                         0x6
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF7__SHIFT                                                         0x7
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF8__SHIFT                                                         0x8
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF9__SHIFT                                                         0x9
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF10__SHIFT                                                        0xa
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF11__SHIFT                                                        0xb
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF12__SHIFT                                                        0xc
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF13__SHIFT                                                        0xd
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF14__SHIFT                                                        0xe
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF15__SHIFT                                                        0xf
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF16__SHIFT                                                        0x10
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF17__SHIFT                                                        0x11
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF18__SHIFT                                                        0x12
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF19__SHIFT                                                        0x13
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF20__SHIFT                                                        0x14
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF21__SHIFT                                                        0x15
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF22__SHIFT                                                        0x16
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF23__SHIFT                                                        0x17
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF24__SHIFT                                                        0x18
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF25__SHIFT                                                        0x19
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF26__SHIFT                                                        0x1a
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF27__SHIFT                                                        0x1b
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF28__SHIFT                                                        0x1c
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF29__SHIFT                                                        0x1d
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF30__SHIFT                                                        0x1e
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF0_MASK                                                           0x00000001L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF1_MASK                                                           0x00000002L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF2_MASK                                                           0x00000004L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF3_MASK                                                           0x00000008L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF4_MASK                                                           0x00000010L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF5_MASK                                                           0x00000020L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF6_MASK                                                           0x00000040L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF7_MASK                                                           0x00000080L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF8_MASK                                                           0x00000100L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF9_MASK                                                           0x00000200L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF10_MASK                                                          0x00000400L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF11_MASK                                                          0x00000800L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF12_MASK                                                          0x00001000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF13_MASK                                                          0x00002000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF14_MASK                                                          0x00004000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF15_MASK                                                          0x00008000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF16_MASK                                                          0x00010000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF17_MASK                                                          0x00020000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF18_MASK                                                          0x00040000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF19_MASK                                                          0x00080000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF20_MASK                                                          0x00100000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF21_MASK                                                          0x00200000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF22_MASK                                                          0x00400000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF23_MASK                                                          0x00800000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF24_MASK                                                          0x01000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF25_MASK                                                          0x02000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF26_MASK                                                          0x04000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF27_MASK                                                          0x08000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF28_MASK                                                          0x10000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF29_MASK                                                          0x20000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF30_MASK                                                          0x40000000L
+//BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX1_MAILBOX_INDEX
+#define BIF_BX1_MAILBOX_INDEX__MAILBOX_INDEX__SHIFT                                                           0x0
+#define BIF_BX1_MAILBOX_INDEX__MAILBOX_INDEX_MASK                                                             0x0000001FL
+//BIF_BX1_BIF_PERSTB_PAD_CNTL
+#define BIF_BX1_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX1_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL_MASK                                                     0x0000FFFFL
+//BIF_BX1_BIF_PX_EN_PAD_CNTL
+#define BIF_BX1_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL__SHIFT                                                     0x0
+#define BIF_BX1_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL_MASK                                                       0x00000FFFL
+//BIF_BX1_BIF_REFPADKIN_PAD_CNTL
+#define BIF_BX1_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL__SHIFT                                             0x0
+#define BIF_BX1_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL_MASK                                               0x000000FFL
+//BIF_BX1_BIF_CLKREQB_PAD_CNTL
+#define BIF_BX1_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL__SHIFT                                                 0x0
+#define BIF_BX1_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_MASK                                                   0x7FFFFFFFL
+//BIF_BX1_BIF_PWRBRK_PAD_CNTL
+#define BIF_BX1_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX1_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL_MASK                                                     0x000000FFL
+
+
+// addressBlock: nbif0_rcc_dev0_BIFDEC1:1
+//RCC_DEV0_1_RCC_ERR_INT_CNTL
+#define RCC_DEV0_1_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN__SHIFT                                0x0
+#define RCC_DEV0_1_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN_MASK                                  0x00000001L
+//RCC_DEV0_1_RCC_BACO_CNTL_MISC
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS__SHIFT                                                 0x0
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS__SHIFT                                                  0x1
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS_MASK                                                   0x00000001L
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS_MASK                                                    0x00000002L
+//RCC_DEV0_1_RCC_RESET_EN
+#define RCC_DEV0_1_RCC_RESET_EN__DB_APER_RESET_EN__SHIFT                                                      0xf
+#define RCC_DEV0_1_RCC_RESET_EN__DB_APER_RESET_EN_MASK                                                        0x00008000L
+//RCC_DEV0_2_RCC_VDM_SUPPORT
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+//RCC_DEV0_1_RCC_GPUIOV_REGION
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__LFB_REGION__SHIFT                                                       0x0
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__MAX_REGION__SHIFT                                                       0x4
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__LFB_REGION_MASK                                                         0x0000000FL
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__MAX_REGION_MASK                                                         0x000000F0L
+//RCC_DEV0_1_RCC_GPU_HOSTVM_EN
+#define RCC_DEV0_1_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN__SHIFT                                                    0x0
+#define RCC_DEV0_1_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN_MASK                                                      0x00000001L
+//RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE__SHIFT                              0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN__SHIFT                                    0x1
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE_MASK                                0x00000001L
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN_MASK                                      0x00000002L
+//RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET__SHIFT                        0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET_MASK                          0xFFFFL
+//RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE__SHIFT                                    0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE_MASK                                      0xFFFFL
+//RCC_DEV0_1_RCC_PEER_REG_RANGE0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_1_RCC_PEER_REG_RANGE1
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_2_RCC_BUS_CNTL
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_1_RCC_CONFIG_CNTL
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT                                               0x2
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GRPH_ADRSEL__SHIFT                                                        0x3
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN_MASK                                                       0x00000001L
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK                                                 0x00000004L
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GRPH_ADRSEL_MASK                                                          0x00000018L
+//RCC_DEV0_1_RCC_CONFIG_F0_BASE
+#define RCC_DEV0_1_RCC_CONFIG_F0_BASE__F0_BASE__SHIFT                                                         0x0
+#define RCC_DEV0_1_RCC_CONFIG_F0_BASE__F0_BASE_MASK                                                           0xFFFFFFFFL
+//RCC_DEV0_1_RCC_CONFIG_APER_SIZE
+#define RCC_DEV0_1_RCC_CONFIG_APER_SIZE__APER_SIZE__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_CONFIG_APER_SIZE__APER_SIZE_MASK                                                       0xFFFFFFFFL
+//RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE
+#define RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT                                             0x0
+#define RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK                                               0x07FFFFFFL
+//RCC_DEV0_1_RCC_XDMA_LO
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_APER_EN__SHIFT                                                       0x1f
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK                                                     0x7FFFFFFFL
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_APER_EN_MASK                                                         0x80000000L
+//RCC_DEV0_1_RCC_XDMA_HI
+#define RCC_DEV0_1_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK                                                     0x7FFFFFFFL
+//RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_1_RCC_BUSNUM_CNTL1
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL1__ID_MASK__SHIFT                                                           0x0
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL1__ID_MASK_MASK                                                             0x000000FFL
+//RCC_DEV0_1_RCC_BUSNUM_LIST0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID0__SHIFT                                                               0x0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID1__SHIFT                                                               0x8
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID2__SHIFT                                                               0x10
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID3__SHIFT                                                               0x18
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID0_MASK                                                                 0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID1_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID2_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID3_MASK                                                                 0xFF000000L
+//RCC_DEV0_1_RCC_BUSNUM_LIST1
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID4__SHIFT                                                               0x0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID5__SHIFT                                                               0x8
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID6__SHIFT                                                               0x10
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID7__SHIFT                                                               0x18
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID4_MASK                                                                 0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID5_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID6_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID7_MASK                                                                 0xFF000000L
+//RCC_DEV0_1_RCC_BUSNUM_CNTL2
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT                                                    0x0
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT                                                     0x8
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT                                           0x11
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK                                                      0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK                                                       0x00000100L
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__HDPREG_CNTL_MASK                                                         0x00010000L
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK                                             0x00020000L
+//RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM
+#define RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN_MASK                                                     0x00000001L
+//RCC_DEV0_1_RCC_HOST_BUSNUM
+#define RCC_DEV0_1_RCC_HOST_BUSNUM__HOST_ID__SHIFT                                                            0x0
+#define RCC_DEV0_1_RCC_HOST_BUSNUM__HOST_ID_MASK                                                              0x0000FFFFL
+//RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT                                                   0x8
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT                                                   0x10
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT                                                   0x18
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK                                                     0x000000FFL
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK                                                     0x0000FF00L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK                                                     0x00FF0000L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK                                                     0xFF000000L
+//RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT                                                   0x8
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT                                                   0x10
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT                                                   0x18
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK                                                     0x000000FFL
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK                                                     0x0000FF00L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK                                                     0x00FF0000L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK                                                     0xFF000000L
+//RCC_DEV0_2_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_2_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                            0x10
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                              0x00FF0000L
+//RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_2_RCC_MH_ARB_CNTL
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x0000FFFEL
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFDEC2
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_0_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                               0x1
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_2__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_3__SHIFT                                               0x3
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                                 0x00000001L
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                                 0x00000002L
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_2_MASK                                                 0x00000004L
+#define RCC_DEV0_EPF0_0_GFXMSIX_PBA__MSIX_PENDING_BITS_3_MASK                                                 0x00000008L
+
+
+// addressBlock: nbif0_rcc_strap_BIFDEC1
+//RCC_STRAP1_RCC_BIF_STRAP0
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x14
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN__SHIFT                         0x15
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GPUIOV_EN__SHIFT                                                     0x16
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE__SHIFT                                          0x17
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN_MASK                           0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GPUIOV_EN_MASK                                                       0x00400000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE_MASK                                            0x00800000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP1
+#define RCC_STRAP1_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP1_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP1_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP2
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS__SHIFT                                               0xc
+#define RCC_STRAP1_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS_MASK                                                 0x00001000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP3
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP1_RCC_BIF_STRAP4
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP1_RCC_BIF_STRAP5
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP6
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE__SHIFT                                               0x5
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0xa
+#define RCC_STRAP1_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6__SHIFT                                                 0xf
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE_MASK                                                 0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00007C00L
+#define RCC_STRAP1_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6_MASK                                                   0xFFFF8000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0__SHIFT                                     0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0_MASK                                       0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0__SHIFT                                   0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0__SHIFT                           0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0_MASK                                     0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0_MASK                                0x03000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0_MASK                             0x0C000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                  0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                    0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0__SHIFT                                      0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0__SHIFT                                   0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0_MASK                                        0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0_MASK                                     0x00000040L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0__SHIFT                                         0x1e
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0_MASK                                           0x40000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP17
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0__SHIFT                            0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0_MASK                              0x000FF000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0__SHIFT                      0x1
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0__SHIFT                     0x2
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0__SHIFT                    0x4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0_MASK                        0x00000002L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0_MASK                       0x00000004L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0_MASK                      0x00000010L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0__SHIFT             0x19
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0_MASK               0x02000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0__SHIFT                                  0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0_MASK                                    0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0_MASK                                       0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1__SHIFT                    0x4
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1_MASK                      0x00000010L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP23
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP23__STRAP_DSN_CODE_L_DEV0_F1__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP23__STRAP_DSN_CODE_L_DEV0_F1_MASK                                       0xFFFFFFFFL
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP24
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP24__STRAP_DSN_CODE_H_DEV0_F1__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP24__STRAP_DSN_CODE_H_DEV0_F1_MASK                                       0xFFFFFFFFL
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1__SHIFT             0x19
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1_MASK               0x02000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1__SHIFT                                  0x2
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1__SHIFT                                        0x8
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1__SHIFT                                  0xa
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1__SHIFT                                        0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1__SHIFT                                  0x12
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1__SHIFT                                        0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1__SHIFT                                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1_MASK                                    0x00000004L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1_MASK                                          0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1_MASK                                    0x00000400L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1_MASK                                          0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1_MASK                                    0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1_MASK                                          0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1_MASK                                          0x04000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP7
+
+
+// addressBlock: nbif0_bif_bx_pf_BIFPFVFDEC1:1
+//BIF_BX_PF1_BIF_BME_STATUS
+#define BIF_BX_PF1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                                      0x0
+#define BIF_BX_PF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                                0x10
+#define BIF_BX_PF1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                                        0x00000001L
+#define BIF_BX_PF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                                  0x00010000L
+//BIF_BX_PF1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                                0x0
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                             0x1
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                                0x2
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                                    0x3
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                          0x10
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                                       0x11
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                          0x12
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                              0x13
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                                  0x00000001L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                               0x00000002L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                                  0x00000004L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                                      0x00000008L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                            0x00010000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                                         0x00020000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                            0x00040000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                                0x00080000L
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                          0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                            0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_PF1_BIF_DOORBELL_INT_CNTL
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS__SHIFT                                    0x0
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS__SHIFT                                   0x1
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS__SHIFT                         0x2
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR__SHIFT                                     0x10
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR__SHIFT                                    0x11
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR__SHIFT                          0x12
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                         0x17
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE__SHIFT                                   0x18
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE__SHIFT                                  0x19
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE__SHIFT                        0x1a
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                            0x1c
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                       0x1d
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                       0x1e
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                           0x1f
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS_MASK                                      0x00000001L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS_MASK                                     0x00000002L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS_MASK                           0x00000004L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR_MASK                                       0x00010000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR_MASK                                      0x00020000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR_MASK                            0x00040000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE_MASK                           0x00800000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE_MASK                                     0x01000000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE_MASK                                    0x02000000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE_MASK                          0x04000000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE_MASK                              0x10000000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                         0x20000000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                         0x40000000L
+#define BIF_BX_PF1_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE_MASK                             0x80000000L
+//BIF_BX_PF1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//BIF_BX_PF1_BIF_TRANS_PENDING
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                            0x0
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                            0x1
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                              0x00000001L
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                              0x00000002L
+//BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                                0x0
+#define BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                                  0x00000001L
+//BIF_BX_PF1_BIF_RB_CNTL
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_ENABLE__SHIFT                                                              0x0
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_SIZE__SHIFT                                                                0x1
+#define BIF_BX_PF1_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                  0x8
+#define BIF_BX_PF1_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                   0x9
+#define BIF_BX_PF1_BIF_RB_CNTL__BIF_RB_TRAN__SHIFT                                                            0x11
+#define BIF_BX_PF1_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL__SHIFT                                               0x19
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY__SHIFT                                                   0x1a
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_INTR_ARB_MODE__SHIFT                                                       0x1d
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE__SHIFT                                                  0x1e
+#define BIF_BX_PF1_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                    0x1f
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_ENABLE_MASK                                                                0x00000001L
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_SIZE_MASK                                                                  0x0000003EL
+#define BIF_BX_PF1_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                    0x00000100L
+#define BIF_BX_PF1_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                     0x00003E00L
+#define BIF_BX_PF1_BIF_RB_CNTL__BIF_RB_TRAN_MASK                                                              0x00020000L
+#define BIF_BX_PF1_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL_MASK                                                 0x02000000L
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY_MASK                                                     0x1C000000L
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_INTR_ARB_MODE_MASK                                                         0x20000000L
+#define BIF_BX_PF1_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE_MASK                                                    0x40000000L
+#define BIF_BX_PF1_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                      0x80000000L
+//BIF_BX_PF1_BIF_RB_BASE
+#define BIF_BX_PF1_BIF_RB_BASE__ADDR__SHIFT                                                                   0x0
+#define BIF_BX_PF1_BIF_RB_BASE__ADDR_MASK                                                                     0xFFFFFFFFL
+//BIF_BX_PF1_BIF_RB_RPTR
+#define BIF_BX_PF1_BIF_RB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define BIF_BX_PF1_BIF_RB_RPTR__OFFSET_MASK                                                                   0x0003FFFCL
+//BIF_BX_PF1_BIF_RB_WPTR
+#define BIF_BX_PF1_BIF_RB_WPTR__BIF_RB_OVERFLOW__SHIFT                                                        0x0
+#define BIF_BX_PF1_BIF_RB_WPTR__OFFSET__SHIFT                                                                 0x2
+#define BIF_BX_PF1_BIF_RB_WPTR__BIF_RB_OVERFLOW_MASK                                                          0x00000001L
+#define BIF_BX_PF1_BIF_RB_WPTR__OFFSET_MASK                                                                   0x0003FFFCL
+//BIF_BX_PF1_BIF_RB_WPTR_ADDR_HI
+#define BIF_BX_PF1_BIF_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                           0x0
+#define BIF_BX_PF1_BIF_RB_WPTR_ADDR_HI__ADDR_MASK                                                             0x000000FFL
+//BIF_BX_PF1_BIF_RB_WPTR_ADDR_LO
+#define BIF_BX_PF1_BIF_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                           0x2
+#define BIF_BX_PF1_BIF_RB_WPTR_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFPFVFDEC1
+//RCC_DEV0_EPF0_0_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                                0x0
+#define RCC_DEV0_EPF0_0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                       0x1
+#define RCC_DEV0_EPF0_0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                  0x00000001L
+#define RCC_DEV0_EPF0_0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                         0x00000002L
+//RCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                       0x00000001L
+//RCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_0_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_0_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_0_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                       0x0
+#define RCC_DEV0_EPF0_0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                            0x1f
+#define RCC_DEV0_EPF0_0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                         0x00000001L
+#define RCC_DEV0_EPF0_0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                              0x80000000L
+
+
+// addressBlock: nbif0_rcc_strap_BIFDEC1:1
+//RCC_STRAP2_RCC_BIF_STRAP0
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x14
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN__SHIFT                         0x15
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GPUIOV_EN__SHIFT                                                     0x16
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE__SHIFT                                          0x17
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN_MASK                           0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GPUIOV_EN_MASK                                                       0x00400000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE_MASK                                            0x00800000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP1
+#define RCC_STRAP2_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP2_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP2_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP2
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS__SHIFT                                               0xc
+#define RCC_STRAP2_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS_MASK                                                 0x00001000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP3
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP2_RCC_BIF_STRAP4
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP2_RCC_BIF_STRAP5
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP6
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE__SHIFT                                               0x5
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0xa
+#define RCC_STRAP2_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6__SHIFT                                                 0xf
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE_MASK                                                 0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00007C00L
+#define RCC_STRAP2_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6_MASK                                                   0xFFFF8000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0__SHIFT                                     0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0_MASK                                       0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0__SHIFT                                   0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0__SHIFT                           0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0_MASK                                     0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0_MASK                                0x03000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0_MASK                             0x0C000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                  0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                    0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0__SHIFT                                      0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0__SHIFT                                   0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0_MASK                                        0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0_MASK                                     0x00000040L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0__SHIFT                                         0x1e
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0_MASK                                           0x40000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP17
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0__SHIFT                            0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0_MASK                              0x000FF000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0__SHIFT                      0x1
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0__SHIFT                     0x2
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0__SHIFT                    0x4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0_MASK                        0x00000002L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0_MASK                       0x00000004L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0_MASK                      0x00000010L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0__SHIFT             0x19
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0_MASK               0x02000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0__SHIFT                                  0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0_MASK                                    0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0_MASK                                       0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1__SHIFT                    0x4
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1_MASK                      0x00000010L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP23
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP23__STRAP_DSN_CODE_L_DEV0_F1__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP23__STRAP_DSN_CODE_L_DEV0_F1_MASK                                       0xFFFFFFFFL
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP24
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP24__STRAP_DSN_CODE_H_DEV0_F1__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP24__STRAP_DSN_CODE_H_DEV0_F1_MASK                                       0xFFFFFFFFL
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1__SHIFT             0x19
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1_MASK               0x02000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1__SHIFT                                  0x2
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1__SHIFT                                        0x8
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1__SHIFT                                  0xa
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1__SHIFT                                        0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1__SHIFT                                  0x12
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1__SHIFT                                        0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1__SHIFT                                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1_MASK                                    0x00000004L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1_MASK                                          0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1_MASK                                    0x00000400L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1_MASK                                          0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1_MASK                                    0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1_MASK                                          0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1_MASK                                          0x04000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP7
+
+
+// addressBlock: nbif0_gdc_GDCDEC
+//GDC0_NGDC_SDP_PORT_CTRL
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS__SHIFT                                                 0x0
+#define GDC0_NGDC_SDP_PORT_CTRL__NGDC_OBFF_HW_URGENT_EARLY_WAKEUP_EN__SHIFT                                   0xf
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H__SHIFT                                               0x10
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS__SHIFT                                                        0x1f
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_MASK                                                   0x000000FFL
+#define GDC0_NGDC_SDP_PORT_CTRL__NGDC_OBFF_HW_URGENT_EARLY_WAKEUP_EN_MASK                                     0x00008000L
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H_MASK                                                 0x000F0000L
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS_MASK                                                          0x80000000L
+//GDC0_SHUB_REGS_IF_CTL
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                0x0
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS__SHIFT                                             0x1
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                  0x00000001L
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS_MASK                                               0x00000002L
+//GDC0_NGDC_MGCG_CTRL
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_EN__SHIFT                                                              0x0
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_MODE__SHIFT                                                            0x1
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS__SHIFT                                                      0x2
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS__SHIFT                                                         0xa
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS__SHIFT                                                         0xb
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS__SHIFT                                                         0xc
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS__SHIFT                                                         0xd
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS__SHIFT                                                         0xe
+#define GDC0_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN__SHIFT                                                         0xf
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_USB_EARLY_WAKEUP_EN__SHIFT                                         0x10
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_EN_MASK                                                                0x00000001L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_MODE_MASK                                                              0x00000002L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS_MASK                                                        0x000003FCL
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS_MASK                                                           0x00000400L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS_MASK                                                           0x00000800L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS_MASK                                                           0x00001000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS_MASK                                                           0x00002000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS_MASK                                                           0x00004000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN_MASK                                                           0x00008000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_USB_EARLY_WAKEUP_EN_MASK                                           0x00010000L
+//GDC0_S2A_MISC_CNTL
+#define GDC0_S2A_MISC_CNTL__DOORBELL_64BIT_SUPPORT_MPDMADB_DIS__SHIFT                                         0x0
+#define GDC0_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS__SHIFT                                                         0x3
+#define GDC0_S2A_MISC_CNTL__ATM_ARB_MODE__SHIFT                                                               0x8
+#define GDC0_S2A_MISC_CNTL__RB_ARB_MODE__SHIFT                                                                0xa
+#define GDC0_S2A_MISC_CNTL__HSTR_ARB_MODE__SHIFT                                                              0xc
+#define GDC0_S2A_MISC_CNTL__HDP_PERF_ENH_DIS__SHIFT                                                           0xf
+#define GDC0_S2A_MISC_CNTL__WRSP_ARB_MODE__SHIFT                                                              0x10
+#define GDC0_S2A_MISC_CNTL__DOORBELL_64BIT_SUPPORT_MPDMADB_DIS_MASK                                           0x00000001L
+#define GDC0_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS_MASK                                                           0x00000008L
+#define GDC0_S2A_MISC_CNTL__ATM_ARB_MODE_MASK                                                                 0x00000300L
+#define GDC0_S2A_MISC_CNTL__RB_ARB_MODE_MASK                                                                  0x00000C00L
+#define GDC0_S2A_MISC_CNTL__HSTR_ARB_MODE_MASK                                                                0x00003000L
+#define GDC0_S2A_MISC_CNTL__HDP_PERF_ENH_DIS_MASK                                                             0x00008000L
+#define GDC0_S2A_MISC_CNTL__WRSP_ARB_MODE_MASK                                                                0x000F0000L
+//GDC0_NGDC_PG_MISC_CTRL
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY__SHIFT                                                   0xa
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1__SHIFT                                                      0xd
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS__SHIFT                                                   0xe
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2__SHIFT                                                      0x10
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS__SHIFT                                        0x18
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE__SHIFT                                              0x1f
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY_MASK                                                     0x00000400L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1_MASK                                                        0x00002000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS_MASK                                                     0x00004000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2_MASK                                                        0x00010000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS_MASK                                          0x3F000000L
+#define GDC0_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE_MASK                                                0x80000000L
+//GDC0_NGDC_PGMST_CTRL
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS__SHIFT                                                   0x0
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN__SHIFT                                                           0x8
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN__SHIFT                                               0xa
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN__SHIFT                                                   0xe
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS_MASK                                                     0x000000FFL
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN_MASK                                                             0x00000100L
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN_MASK                                                 0x00003C00L
+#define GDC0_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN_MASK                                                     0x0000C000L
+//GDC0_NGDC_PGSLV_CTRL
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS__SHIFT                                       0x0
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS__SHIFT                                       0x5
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS__SHIFT                                          0xa
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_2_IDLE_HYSTERESIS__SHIFT                                       0xf
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS_MASK                                         0x0000001FL
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS_MASK                                         0x000003E0L
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS_MASK                                            0x00007C00L
+#define GDC0_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_2_IDLE_HYSTERESIS_MASK                                         0x000F8000L
+//GDC0_SHUBCLK_DPM_CTRL
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_MODE__SHIFT                                                        0x0
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_ENABLE__SHIFT                                                      0x1
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMAWR_QUANT__SHIFT                                                 0x2
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMARD_QUANT__SHIFT                                                 0xa
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_CLEAR__SHIFT                                                       0x12
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_MODE_MASK                                                          0x00000001L
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_ENABLE_MASK                                                        0x00000002L
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMAWR_QUANT_MASK                                                   0x000003FCL
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMARD_QUANT_MASK                                                   0x0003FC00L
+#define GDC0_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_CLEAR_MASK                                                         0x00040000L
+//GDC0_SHUBCLK_DPM_WR_WEIGHT
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_4B__SHIFT                                           0x0
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_8B__SHIFT                                           0x5
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_16B__SHIFT                                          0xa
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_32B__SHIFT                                          0xf
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_64B__SHIFT                                          0x14
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_4B_MASK                                             0x0000001FL
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_8B_MASK                                             0x000003E0L
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_16B_MASK                                            0x00007C00L
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_32B_MASK                                            0x000F8000L
+#define GDC0_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_64B_MASK                                            0x01F00000L
+//GDC0_SHUBCLK_DPM_RD_WEIGHT
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_4B__SHIFT                                           0x0
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_8B__SHIFT                                           0x5
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_16B__SHIFT                                          0xa
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_32B__SHIFT                                          0xf
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_64B__SHIFT                                          0x14
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_4B_MASK                                             0x0000001FL
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_8B_MASK                                             0x000003E0L
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_16B_MASK                                            0x00007C00L
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_32B_MASK                                            0x000F8000L
+#define GDC0_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_64B_MASK                                            0x01F00000L
+//GDC0_SHUBCLK_DPM_WR_CNT
+#define GDC0_SHUBCLK_DPM_WR_CNT__SHUBCLK_DPM_WR_COUNTER__SHIFT                                                0x0
+#define GDC0_SHUBCLK_DPM_WR_CNT__SHUBCLK_DPM_WR_COUNTER_MASK                                                  0xFFFFFFFFL
+//GDC0_SHUBCLK_DPM_RD_CNT
+#define GDC0_SHUBCLK_DPM_RD_CNT__SHUBCLK_DPM_RD_COUNTER__SHIFT                                                0x0
+#define GDC0_SHUBCLK_DPM_RD_CNT__SHUBCLK_DPM_RD_COUNTER_MASK                                                  0xFFFFFFFFL
+//GDC0_ATDMA_MISC_CNTL
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_ARB_MODE__SHIFT                                                       0x0
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_MISC_CNTL_INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                 0x1
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_RDRSP_ARB_MODE__SHIFT                                                     0x2
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_VC6_WEIGHT__SHIFT                                                     0x8
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_VC0_WEIGHT__SHIFT                                                     0x10
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_VC1_WEIGHT__SHIFT                                                     0x18
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_ARB_MODE_MASK                                                         0x00000001L
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_MISC_CNTL_INSERT_RD_ON_2ND_WDAT_EN_MASK                                   0x00000002L
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_RDRSP_ARB_MODE_MASK                                                       0x0000000CL
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_VC6_WEIGHT_MASK                                                       0x0000FF00L
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_VC0_WEIGHT_MASK                                                       0x00FF0000L
+#define GDC0_ATDMA_MISC_CNTL__ATDMA_WRR_VC1_WEIGHT_MASK                                                       0xFF000000L
+
+
+// addressBlock: nbif0_gdc_s2a_GDCS2A_DEC
+//GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE__SHIFT                            0x1
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS_MASK                       0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE_MASK                              0x00000002L
+//GDC_S2A0_NBIF_GFX_DOORBELL_STATUS
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT__SHIFT                                      0x0
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN__SHIFT                                     0x10
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST__SHIFT                                     0x18
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT_MASK                                        0x0000FFFFL
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN_MASK                                       0x00010000L
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST_MASK                                       0x01000000L
+
+
+// addressBlock: nbif0_gdc_GDC_LINEAR_REGION
+//GDC1_NGDC_SDP_PORT_CTRL
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS__SHIFT                                                 0x0
+#define GDC1_NGDC_SDP_PORT_CTRL__NGDC_OBFF_HW_URGENT_EARLY_WAKEUP_EN__SHIFT                                   0xf
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H__SHIFT                                               0x10
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS__SHIFT                                                        0x1f
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_MASK                                                   0x000000FFL
+#define GDC1_NGDC_SDP_PORT_CTRL__NGDC_OBFF_HW_URGENT_EARLY_WAKEUP_EN_MASK                                     0x00008000L
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H_MASK                                                 0x000F0000L
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS_MASK                                                          0x80000000L
+//GDC1_SHUB_REGS_IF_CTL
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                0x0
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS__SHIFT                                             0x1
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                  0x00000001L
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS_MASK                                               0x00000002L
+//GDC1_NGDC_MGCG_CTRL
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_EN__SHIFT                                                              0x0
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_MODE__SHIFT                                                            0x1
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS__SHIFT                                                      0x2
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS__SHIFT                                                         0xa
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS__SHIFT                                                         0xb
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS__SHIFT                                                         0xc
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS__SHIFT                                                         0xd
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS__SHIFT                                                         0xe
+#define GDC1_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN__SHIFT                                                         0xf
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_USB_EARLY_WAKEUP_EN__SHIFT                                         0x10
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_EN_MASK                                                                0x00000001L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_MODE_MASK                                                              0x00000002L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS_MASK                                                        0x000003FCL
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS_MASK                                                           0x00000400L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS_MASK                                                           0x00000800L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS_MASK                                                           0x00001000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS_MASK                                                           0x00002000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS_MASK                                                           0x00004000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN_MASK                                                           0x00008000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_USB_EARLY_WAKEUP_EN_MASK                                           0x00010000L
+//GDC1_S2A_MISC_CNTL
+#define GDC1_S2A_MISC_CNTL__DOORBELL_64BIT_SUPPORT_MPDMADB_DIS__SHIFT                                         0x0
+#define GDC1_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS__SHIFT                                                         0x3
+#define GDC1_S2A_MISC_CNTL__ATM_ARB_MODE__SHIFT                                                               0x8
+#define GDC1_S2A_MISC_CNTL__RB_ARB_MODE__SHIFT                                                                0xa
+#define GDC1_S2A_MISC_CNTL__HSTR_ARB_MODE__SHIFT                                                              0xc
+#define GDC1_S2A_MISC_CNTL__HDP_PERF_ENH_DIS__SHIFT                                                           0xf
+#define GDC1_S2A_MISC_CNTL__WRSP_ARB_MODE__SHIFT                                                              0x10
+#define GDC1_S2A_MISC_CNTL__DOORBELL_64BIT_SUPPORT_MPDMADB_DIS_MASK                                           0x00000001L
+#define GDC1_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS_MASK                                                           0x00000008L
+#define GDC1_S2A_MISC_CNTL__ATM_ARB_MODE_MASK                                                                 0x00000300L
+#define GDC1_S2A_MISC_CNTL__RB_ARB_MODE_MASK                                                                  0x00000C00L
+#define GDC1_S2A_MISC_CNTL__HSTR_ARB_MODE_MASK                                                                0x00003000L
+#define GDC1_S2A_MISC_CNTL__HDP_PERF_ENH_DIS_MASK                                                             0x00008000L
+#define GDC1_S2A_MISC_CNTL__WRSP_ARB_MODE_MASK                                                                0x000F0000L
+//GDC1_NGDC_EARLY_WAKEUP_CTRL
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE__SHIFT                                0x0
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT__SHIFT                               0x1
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE__SHIFT                                0x2
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE_MASK                                  0x00000001L
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT_MASK                                 0x00000002L
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE_MASK                                  0x00000004L
+//GDC1_SHUBCLK_DPM_CTRL
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_MODE__SHIFT                                                        0x0
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_ENABLE__SHIFT                                                      0x1
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMAWR_QUANT__SHIFT                                                 0x2
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMARD_QUANT__SHIFT                                                 0xa
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_CLEAR__SHIFT                                                       0x12
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_MODE_MASK                                                          0x00000001L
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_ENABLE_MASK                                                        0x00000002L
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMAWR_QUANT_MASK                                                   0x000003FCL
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_DMARD_QUANT_MASK                                                   0x0003FC00L
+#define GDC1_SHUBCLK_DPM_CTRL__SHUBCLK_DPM_CLEAR_MASK                                                         0x00040000L
+//GDC1_SHUBCLK_DPM_WR_WEIGHT
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_4B__SHIFT                                           0x0
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_8B__SHIFT                                           0x5
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_16B__SHIFT                                          0xa
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_32B__SHIFT                                          0xf
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_64B__SHIFT                                          0x14
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_4B_MASK                                             0x0000001FL
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_8B_MASK                                             0x000003E0L
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_16B_MASK                                            0x00007C00L
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_32B_MASK                                            0x000F8000L
+#define GDC1_SHUBCLK_DPM_WR_WEIGHT__SHUBCLK_DPM_WR_WEIGHT_64B_MASK                                            0x01F00000L
+//GDC1_SHUBCLK_DPM_RD_WEIGHT
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_4B__SHIFT                                           0x0
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_8B__SHIFT                                           0x5
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_16B__SHIFT                                          0xa
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_32B__SHIFT                                          0xf
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_64B__SHIFT                                          0x14
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_4B_MASK                                             0x0000001FL
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_8B_MASK                                             0x000003E0L
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_16B_MASK                                            0x00007C00L
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_32B_MASK                                            0x000F8000L
+#define GDC1_SHUBCLK_DPM_RD_WEIGHT__SHUBCLK_DPM_RD_WEIGHT_64B_MASK                                            0x01F00000L
+//GDC1_SHUBCLK_DPM_WR_CNT
+#define GDC1_SHUBCLK_DPM_WR_CNT__SHUBCLK_DPM_WR_COUNTER__SHIFT                                                0x0
+#define GDC1_SHUBCLK_DPM_WR_CNT__SHUBCLK_DPM_WR_COUNTER_MASK                                                  0xFFFFFFFFL
+//GDC1_SHUBCLK_DPM_RD_CNT
+#define GDC1_SHUBCLK_DPM_RD_CNT__SHUBCLK_DPM_RD_COUNTER__SHIFT                                                0x0
+#define GDC1_SHUBCLK_DPM_RD_CNT__SHUBCLK_DPM_RD_COUNTER_MASK                                                  0xFFFFFFFFL
+//GDC1_ATDMA_MISC_CNTL
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_ARB_MODE__SHIFT                                                       0x0
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_MISC_CNTL_INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                 0x1
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_RDRSP_ARB_MODE__SHIFT                                                     0x2
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_VC6_WEIGHT__SHIFT                                                     0x8
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_VC0_WEIGHT__SHIFT                                                     0x10
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_VC1_WEIGHT__SHIFT                                                     0x18
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_ARB_MODE_MASK                                                         0x00000001L
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_MISC_CNTL_INSERT_RD_ON_2ND_WDAT_EN_MASK                                   0x00000002L
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_RDRSP_ARB_MODE_MASK                                                       0x0000000CL
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_VC6_WEIGHT_MASK                                                       0x0000FF00L
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_VC0_WEIGHT_MASK                                                       0x00FF0000L
+#define GDC1_ATDMA_MISC_CNTL__ATDMA_WRR_VC1_WEIGHT_MASK                                                       0xFF000000L
+
+
+// addressBlock: nbif0_gdc_s2a_GDC_LINEAR_REGION
+//GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE__SHIFT                            0x1
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS_MASK                       0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE_MASK                              0x00000002L
+//GDC_S2A1_NBIF_GFX_DOORBELL_STATUS
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT__SHIFT                                      0x0
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN__SHIFT                                     0x10
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST__SHIFT                                     0x18
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT_MASK                                        0x0000FFFFL
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN_MASK                                       0x00010000L
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST_MASK                                       0x01000000L
+
+
+// addressBlock: nbif0_syshub_mmreg_syshubdec
+//SYSHUB_INDEX
+#define SYSHUB_INDEX__INDEX__SHIFT                                                                            0x0
+#define SYSHUB_INDEX__INDEX_MASK                                                                              0xFFFFFFFFL
+//SYSHUB_DATA
+#define SYSHUB_DATA__DATA__SHIFT                                                                              0x0
+#define SYSHUB_DATA__DATA_MASK                                                                                0xFFFFFFFFL
+
+
+// addressBlock: nbif0_bif_bx_SYSDEC
+//BIF_BX2_PCIE_INDEX
+#define BIF_BX2_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX2_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX2_PCIE_DATA
+#define BIF_BX2_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX2_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//BIF_BX2_PCIE_INDEX2
+#define BIF_BX2_PCIE_INDEX2__PCIE_INDEX2__SHIFT                                                               0x0
+#define BIF_BX2_PCIE_INDEX2__PCIE_INDEX2_MASK                                                                 0xFFFFFFFFL
+//BIF_BX2_PCIE_DATA2
+#define BIF_BX2_PCIE_DATA2__PCIE_DATA2__SHIFT                                                                 0x0
+#define BIF_BX2_PCIE_DATA2__PCIE_DATA2_MASK                                                                   0xFFFFFFFFL
+//BIF_BX2_SBIOS_SCRATCH_0
+#define BIF_BX2_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0__SHIFT                                                       0x0
+#define BIF_BX2_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_SBIOS_SCRATCH_1
+#define BIF_BX2_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1__SHIFT                                                       0x0
+#define BIF_BX2_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_SBIOS_SCRATCH_2
+#define BIF_BX2_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2__SHIFT                                                       0x0
+#define BIF_BX2_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_SBIOS_SCRATCH_3
+#define BIF_BX2_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3__SHIFT                                                       0x0
+#define BIF_BX2_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_0
+#define BIF_BX2_BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_1
+#define BIF_BX2_BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_2
+#define BIF_BX2_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_3
+#define BIF_BX2_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_4
+#define BIF_BX2_BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_5
+#define BIF_BX2_BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_6
+#define BIF_BX2_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_7
+#define BIF_BX2_BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_8
+#define BIF_BX2_BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_9
+#define BIF_BX2_BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK                                                           0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_10
+#define BIF_BX2_BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT                                                       0x0
+#define BIF_BX2_BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_11
+#define BIF_BX2_BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT                                                       0x0
+#define BIF_BX2_BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_12
+#define BIF_BX2_BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT                                                       0x0
+#define BIF_BX2_BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_13
+#define BIF_BX2_BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT                                                       0x0
+#define BIF_BX2_BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_14
+#define BIF_BX2_BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT                                                       0x0
+#define BIF_BX2_BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_BIOS_SCRATCH_15
+#define BIF_BX2_BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT                                                       0x0
+#define BIF_BX2_BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK                                                         0xFFFFFFFFL
+//BIF_BX2_BIF_RLC_INTR_CNTL
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX2_BIF_RLC_INTR_CNTL__RLC_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+//BIF_BX2_BIF_VCE_INTR_CNTL
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX2_BIF_VCE_INTR_CNTL__VCE_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+//BIF_BX2_BIF_UVD_INTR_CNTL
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_CMD_COMPLETE__SHIFT                                                    0x0
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_HANG_SELF_RECOVERED__SHIFT                                             0x1
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_HANG_NEED_FLR__SHIFT                                                   0x2
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_VM_BUSY_TRANSITION__SHIFT                                              0x3
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_INST_SEL__SHIFT                                                        0x1c
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_CMD_COMPLETE_MASK                                                      0x00000001L
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_HANG_SELF_RECOVERED_MASK                                               0x00000002L
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_HANG_NEED_FLR_MASK                                                     0x00000004L
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_VM_BUSY_TRANSITION_MASK                                                0x00000008L
+#define BIF_BX2_BIF_UVD_INTR_CNTL__UVD_INST_SEL_MASK                                                          0xF0000000L
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR1
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR1
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR2
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR2
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR3
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR3
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR4
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR4
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR5
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR5
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR6
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR6
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ADDR7
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7_MASK                                                         0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR7
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7__SHIFT                                           0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7_MASK                                             0x000FFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_CNTL
+#define BIF_BX2_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE__SHIFT                                                       0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE_MASK                                                         0x000000FFL
+//BIF_BX2_GFX_MMIOREG_CAM_ZERO_CPL
+#define BIF_BX2_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL__SHIFT                                                 0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL_MASK                                                   0xFFFFFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_ONE_CPL
+#define BIF_BX2_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL__SHIFT                                                   0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL_MASK                                                     0xFFFFFFFFL
+//BIF_BX2_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL
+#define BIF_BX2_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL__SHIFT                                 0x0
+#define BIF_BX2_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL_MASK                                   0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_dwn_dev0_BIFDEC1
+//RCC_DWN_DEV0_3_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_3_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_3_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_3_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_3_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_3_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_3_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_3_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_3_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_3_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_3_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_3_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_3_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_3_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_3_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_3_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_3_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_3_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_3_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_3_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_3_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_3_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_3_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_3_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                 0x5
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+#define RCC_DWN_DEV0_3_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                   0x00000020L
+//RCC_DWN_DEV0_3_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_3_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: nbif0_rcc_dwnp_dev0_BIFDEC1
+//RCC_DWNP_DEV0_3_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_3_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_3_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_3_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                           0x4
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+#define RCC_DWNP_DEV0_3_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                             0x00000010L
+//RCC_DWNP_DEV0_3_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_3_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_3_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_3_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_3_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_3_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_3_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_3_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_3_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_3_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_3_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_rcc_ep_dev0_BIFDEC1
+//RCC_EP_DEV0_3_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_3_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_3_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_3_EP_PCIE_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_3_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_3_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_3_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_3_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_3_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_3_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_3_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_3_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_3_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_3_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_3_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                  0x5
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+#define RCC_EP_DEV0_3_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                    0x00000020L
+//RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_3_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_3_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_3_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_3_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_3_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_3_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_3_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_3_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_3_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_3_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_3_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_3_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_3_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_3_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_3_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_3_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_3_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_3_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_3_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                          0x4
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+#define RCC_EP_DEV0_3_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                            0x00000010L
+
+
+// addressBlock: nbif0_bif_bx_pf_SYSPFVFDEC
+//BIF_BX_PF2_MM_INDEX
+#define BIF_BX_PF2_MM_INDEX__MM_OFFSET__SHIFT                                                                 0x0
+#define BIF_BX_PF2_MM_INDEX__MM_APER__SHIFT                                                                   0x1f
+#define BIF_BX_PF2_MM_INDEX__MM_OFFSET_MASK                                                                   0x7FFFFFFFL
+#define BIF_BX_PF2_MM_INDEX__MM_APER_MASK                                                                     0x80000000L
+//BIF_BX_PF2_MM_DATA
+#define BIF_BX_PF2_MM_DATA__MM_DATA__SHIFT                                                                    0x0
+#define BIF_BX_PF2_MM_DATA__MM_DATA_MASK                                                                      0xFFFFFFFFL
+//BIF_BX_PF2_MM_INDEX_HI
+#define BIF_BX_PF2_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                           0x0
+#define BIF_BX_PF2_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: nbif0_bif_bx_BIFDEC1
+//BIF_BX2_BIF_MM_INDACCESS_CNTL
+#define BIF_BX2_BIF_MM_INDACCESS_CNTL__WRITE_DIS__SHIFT                                                       0x0
+#define BIF_BX2_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS__SHIFT                                                0x1
+#define BIF_BX2_BIF_MM_INDACCESS_CNTL__WRITE_DIS_MASK                                                         0x00000001L
+#define BIF_BX2_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS_MASK                                                  0x00000002L
+//BIF_BX2_BUS_CNTL
+#define BIF_BX2_BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT                                                        0x6
+#define BIF_BX2_BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT                                                        0x7
+#define BIF_BX2_BUS_CNTL__SET_AZ_TC__SHIFT                                                                    0xa
+#define BIF_BX2_BUS_CNTL__SET_MC_TC__SHIFT                                                                    0xd
+#define BIF_BX2_BUS_CNTL__ZERO_BE_WR_EN__SHIFT                                                                0x10
+#define BIF_BX2_BUS_CNTL__ZERO_BE_RD_EN__SHIFT                                                                0x11
+#define BIF_BX2_BUS_CNTL__RD_STALL_IO_WR__SHIFT                                                               0x12
+#define BIF_BX2_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS__SHIFT                                              0x18
+#define BIF_BX2_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS__SHIFT                                          0x19
+#define BIF_BX2_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS__SHIFT                                         0x1a
+#define BIF_BX2_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS__SHIFT                                        0x1b
+#define BIF_BX2_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS__SHIFT                                          0x1c
+#define BIF_BX2_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN__SHIFT                                                     0x1d
+#define BIF_BX2_BUS_CNTL__VGAFB_ZERO_BE_WR_EN__SHIFT                                                          0x1e
+#define BIF_BX2_BUS_CNTL__VGAFB_ZERO_BE_RD_EN__SHIFT                                                          0x1f
+#define BIF_BX2_BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK                                                          0x00000040L
+#define BIF_BX2_BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK                                                          0x00000080L
+#define BIF_BX2_BUS_CNTL__SET_AZ_TC_MASK                                                                      0x00001C00L
+#define BIF_BX2_BUS_CNTL__SET_MC_TC_MASK                                                                      0x0000E000L
+#define BIF_BX2_BUS_CNTL__ZERO_BE_WR_EN_MASK                                                                  0x00010000L
+#define BIF_BX2_BUS_CNTL__ZERO_BE_RD_EN_MASK                                                                  0x00020000L
+#define BIF_BX2_BUS_CNTL__RD_STALL_IO_WR_MASK                                                                 0x00040000L
+#define BIF_BX2_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS_MASK                                                0x01000000L
+#define BIF_BX2_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS_MASK                                            0x02000000L
+#define BIF_BX2_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS_MASK                                           0x04000000L
+#define BIF_BX2_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS_MASK                                          0x08000000L
+#define BIF_BX2_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS_MASK                                            0x10000000L
+#define BIF_BX2_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN_MASK                                                       0x20000000L
+#define BIF_BX2_BUS_CNTL__VGAFB_ZERO_BE_WR_EN_MASK                                                            0x40000000L
+#define BIF_BX2_BUS_CNTL__VGAFB_ZERO_BE_RD_EN_MASK                                                            0x80000000L
+//BIF_BX2_BIF_SCRATCH0
+#define BIF_BX2_BIF_SCRATCH0__BIF_SCRATCH0__SHIFT                                                             0x0
+#define BIF_BX2_BIF_SCRATCH0__BIF_SCRATCH0_MASK                                                               0xFFFFFFFFL
+//BIF_BX2_BIF_SCRATCH1
+#define BIF_BX2_BIF_SCRATCH1__BIF_SCRATCH1__SHIFT                                                             0x0
+#define BIF_BX2_BIF_SCRATCH1__BIF_SCRATCH1_MASK                                                               0xFFFFFFFFL
+//BIF_BX2_BX_RESET_EN
+#define BIF_BX2_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN__SHIFT                                                  0x10
+#define BIF_BX2_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN_MASK                                                    0x00010000L
+//BIF_BX2_MM_CFGREGS_CNTL
+#define BIF_BX2_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT                                                       0x0
+#define BIF_BX2_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL__SHIFT                                                        0x6
+#define BIF_BX2_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT                                                       0x1f
+#define BIF_BX2_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK                                                         0x00000007L
+#define BIF_BX2_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL_MASK                                                          0x000000C0L
+#define BIF_BX2_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK                                                         0x80000000L
+//BIF_BX2_BX_RESET_CNTL
+#define BIF_BX2_BX_RESET_CNTL__LINK_TRAIN_EN__SHIFT                                                           0x0
+#define BIF_BX2_BX_RESET_CNTL__LINK_TRAIN_EN_MASK                                                             0x00000001L
+//BIF_BX2_INTERRUPT_CNTL
+#define BIF_BX2_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX2_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX2_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX2_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX2_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX2_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX2_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX2_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX2_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX2_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX2_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX2_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX2_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX2_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+//BIF_BX2_INTERRUPT_CNTL2
+#define BIF_BX2_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX2_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX2_CLKREQB_PAD_CNTL
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT                                                        0x0
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT                                                      0x1
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT                                                     0x2
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT                                                    0x3
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT                                                      0x5
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT                                                      0x6
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT                                                      0x7
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT                                                      0x8
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT                                                    0x9
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT                                                     0xa
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT                                                   0xb
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT                                                  0xc
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y__SHIFT                                                        0xd
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK                                                          0x00000001L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK                                                        0x00000002L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK                                                       0x00000004L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK                                                      0x00000018L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK                                                        0x00000020L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK                                                        0x00000040L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK                                                        0x00000080L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK                                                        0x00000100L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK                                                      0x00000200L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK                                                       0x00000400L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK                                                     0x00000800L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK                                                    0x00001000L
+#define BIF_BX2_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y_MASK                                                          0x00002000L
+//BIF_BX2_BIF_FEATURES_CONTROL_MISC
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT                                          0x0
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT                                          0x1
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT                                          0x2
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT                                          0x3
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS__SHIFT                                          0xd
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS__SHIFT                                       0xe
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT__SHIFT                                           0x10
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR__SHIFT                   0x18
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK                                            0x00000001L
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK                                            0x00000002L
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK                                            0x00000004L
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK                                            0x00000008L
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS_MASK                                            0x00002000L
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS_MASK                                         0x00004000L
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT_MASK                                             0x00FF0000L
+#define BIF_BX2_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR_MASK                     0x01000000L
+//BIF_BX2_HDP_ATOMIC_CONTROL_MISC
+#define BIF_BX2_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT__SHIFT                                      0x0
+#define BIF_BX2_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT_MASK                                        0x000000FFL
+//BIF_BX2_BIF_DOORBELL_CNTL
+#define BIF_BX2_BIF_DOORBELL_CNTL__SELF_RING_DIS__SHIFT                                                       0x0
+#define BIF_BX2_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS__SHIFT                                                     0x1
+#define BIF_BX2_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN__SHIFT                                                    0x2
+#define BIF_BX2_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS__SHIFT                                         0x3
+#define BIF_BX2_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT                                                 0x4
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS__SHIFT                                                  0x18
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0__SHIFT                                               0x19
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1__SHIFT                                               0x1a
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2__SHIFT                                               0x1b
+#define BIF_BX2_BIF_DOORBELL_CNTL__SELF_RING_DIS_MASK                                                         0x00000001L
+#define BIF_BX2_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS_MASK                                                       0x00000002L
+#define BIF_BX2_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN_MASK                                                      0x00000004L
+#define BIF_BX2_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS_MASK                                           0x00000008L
+#define BIF_BX2_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK                                                   0x00000010L
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS_MASK                                                    0x01000000L
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0_MASK                                                 0x02000000L
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1_MASK                                                 0x04000000L
+#define BIF_BX2_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2_MASK                                                 0x08000000L
+//BIF_BX2_BIF_FB_EN
+#define BIF_BX2_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX2_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX2_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX2_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX2_BIF_MST_TRANS_PENDING_VF
+#define BIF_BX2_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX2_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX2_BIF_SLV_TRANS_PENDING_VF
+#define BIF_BX2_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX2_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX2_BACO_CNTL
+#define BIF_BX2_BACO_CNTL__BACO_EN__SHIFT                                                                     0x0
+#define BIF_BX2_BACO_CNTL__BACO_BIF_LCLK_SWITCH__SHIFT                                                        0x1
+#define BIF_BX2_BACO_CNTL__BACO_DUMMY_EN__SHIFT                                                               0x2
+#define BIF_BX2_BACO_CNTL__BACO_POWER_OFF__SHIFT                                                              0x3
+#define BIF_BX2_BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT                                                          0x5
+#define BIF_BX2_BACO_CNTL__BACO_RST_INTR_MASK__SHIFT                                                          0x6
+#define BIF_BX2_BACO_CNTL__BACO_MODE__SHIFT                                                                   0x8
+#define BIF_BX2_BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT                                                         0x9
+#define BIF_BX2_BACO_CNTL__BACO_AUTO_EXIT__SHIFT                                                              0x1f
+#define BIF_BX2_BACO_CNTL__BACO_EN_MASK                                                                       0x00000001L
+#define BIF_BX2_BACO_CNTL__BACO_BIF_LCLK_SWITCH_MASK                                                          0x00000002L
+#define BIF_BX2_BACO_CNTL__BACO_DUMMY_EN_MASK                                                                 0x00000004L
+#define BIF_BX2_BACO_CNTL__BACO_POWER_OFF_MASK                                                                0x00000008L
+#define BIF_BX2_BACO_CNTL__BACO_DSTATE_BYPASS_MASK                                                            0x00000020L
+#define BIF_BX2_BACO_CNTL__BACO_RST_INTR_MASK_MASK                                                            0x00000040L
+#define BIF_BX2_BACO_CNTL__BACO_MODE_MASK                                                                     0x00000100L
+#define BIF_BX2_BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK                                                           0x00000200L
+#define BIF_BX2_BACO_CNTL__BACO_AUTO_EXIT_MASK                                                                0x80000000L
+//BIF_BX2_BIF_BACO_EXIT_TIME0
+#define BIF_BX2_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER__SHIFT                                          0x0
+#define BIF_BX2_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER_MASK                                            0x000FFFFFL
+//BIF_BX2_BIF_BACO_EXIT_TIMER1
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER__SHIFT                                         0x0
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN__SHIFT                                            0x18
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_ENDING_AUTO_BY_RSMU_INTR_CLR__SHIFT                        0x19
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS__SHIFT                                                 0x1a
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH__SHIFT                                           0x1b
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW__SHIFT                                            0x1c
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL__SHIFT                                                    0x1d
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS__SHIFT                                     0x1f
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER_MASK                                           0x000FFFFFL
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN_MASK                                              0x01000000L
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_ENDING_AUTO_BY_RSMU_INTR_CLR_MASK                          0x02000000L
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS_MASK                                                   0x04000000L
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH_MASK                                             0x08000000L
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW_MASK                                              0x10000000L
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL_MASK                                                      0x60000000L
+#define BIF_BX2_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS_MASK                                       0x80000000L
+//BIF_BX2_BIF_BACO_EXIT_TIMER2
+#define BIF_BX2_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER__SHIFT                                         0x0
+#define BIF_BX2_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER_MASK                                           0x000FFFFFL
+//BIF_BX2_BIF_BACO_EXIT_TIMER3
+#define BIF_BX2_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER__SHIFT                                     0x0
+#define BIF_BX2_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER_MASK                                       0x000FFFFFL
+//BIF_BX2_BIF_BACO_EXIT_TIMER4
+#define BIF_BX2_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER__SHIFT                                      0x0
+#define BIF_BX2_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER_MASK                                        0x000FFFFFL
+//BIF_BX2_MEM_TYPE_CNTL
+#define BIF_BX2_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3__SHIFT                                                        0x0
+#define BIF_BX2_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3_MASK                                                          0x00000001L
+//BIF_BX2_NBIF_GFX_ADDR_LUT_CNTL
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE__SHIFT                                                     0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE__SHIFT                                                  0x1
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE__SHIFT                                                    0x8
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE_MASK                                                       0x00000001L
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE_MASK                                                    0x00000002L
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE_MASK                                                      0x00000100L
+//BIF_BX2_NBIF_GFX_ADDR_LUT_0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_0__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_0__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_1
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_1__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_1__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_2
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_2__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_2__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_3
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_3__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_3__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_4
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_4__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_4__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_5
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_5__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_5__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_6
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_6__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_6__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_7
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_7__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_7__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_8
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_8__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_8__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_9
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_9__ADDR__SHIFT                                                              0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_9__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_10
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_10__ADDR__SHIFT                                                             0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_10__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_11
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_11__ADDR__SHIFT                                                             0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_11__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_12
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_12__ADDR__SHIFT                                                             0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_12__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_13
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_13__ADDR__SHIFT                                                             0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_13__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_14
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_14__ADDR__SHIFT                                                             0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_14__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX2_NBIF_GFX_ADDR_LUT_15
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_15__ADDR__SHIFT                                                             0x0
+#define BIF_BX2_NBIF_GFX_ADDR_LUT_15__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX2_VF_REGWR_EN
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF0__SHIFT                                                           0x0
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF1__SHIFT                                                           0x1
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF2__SHIFT                                                           0x2
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF3__SHIFT                                                           0x3
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF4__SHIFT                                                           0x4
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF5__SHIFT                                                           0x5
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF6__SHIFT                                                           0x6
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF7__SHIFT                                                           0x7
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF8__SHIFT                                                           0x8
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF9__SHIFT                                                           0x9
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF10__SHIFT                                                          0xa
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF11__SHIFT                                                          0xb
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF12__SHIFT                                                          0xc
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF13__SHIFT                                                          0xd
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF14__SHIFT                                                          0xe
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF15__SHIFT                                                          0xf
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF16__SHIFT                                                          0x10
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF17__SHIFT                                                          0x11
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF18__SHIFT                                                          0x12
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF19__SHIFT                                                          0x13
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF20__SHIFT                                                          0x14
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF21__SHIFT                                                          0x15
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF22__SHIFT                                                          0x16
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF23__SHIFT                                                          0x17
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF24__SHIFT                                                          0x18
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF25__SHIFT                                                          0x19
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF26__SHIFT                                                          0x1a
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF27__SHIFT                                                          0x1b
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF28__SHIFT                                                          0x1c
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF29__SHIFT                                                          0x1d
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF30__SHIFT                                                          0x1e
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF0_MASK                                                             0x00000001L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF1_MASK                                                             0x00000002L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF2_MASK                                                             0x00000004L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF3_MASK                                                             0x00000008L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF4_MASK                                                             0x00000010L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF5_MASK                                                             0x00000020L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF6_MASK                                                             0x00000040L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF7_MASK                                                             0x00000080L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF8_MASK                                                             0x00000100L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF9_MASK                                                             0x00000200L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF10_MASK                                                            0x00000400L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF11_MASK                                                            0x00000800L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF12_MASK                                                            0x00001000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF13_MASK                                                            0x00002000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF14_MASK                                                            0x00004000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF15_MASK                                                            0x00008000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF16_MASK                                                            0x00010000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF17_MASK                                                            0x00020000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF18_MASK                                                            0x00040000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF19_MASK                                                            0x00080000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF20_MASK                                                            0x00100000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF21_MASK                                                            0x00200000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF22_MASK                                                            0x00400000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF23_MASK                                                            0x00800000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF24_MASK                                                            0x01000000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF25_MASK                                                            0x02000000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF26_MASK                                                            0x04000000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF27_MASK                                                            0x08000000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF28_MASK                                                            0x10000000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF29_MASK                                                            0x20000000L
+#define BIF_BX2_VF_REGWR_EN__VF_REGWR_EN_VF30_MASK                                                            0x40000000L
+//BIF_BX2_VF_DOORBELL_EN
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0__SHIFT                                                     0x0
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1__SHIFT                                                     0x1
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2__SHIFT                                                     0x2
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3__SHIFT                                                     0x3
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4__SHIFT                                                     0x4
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5__SHIFT                                                     0x5
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6__SHIFT                                                     0x6
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7__SHIFT                                                     0x7
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8__SHIFT                                                     0x8
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9__SHIFT                                                     0x9
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10__SHIFT                                                    0xa
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11__SHIFT                                                    0xb
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12__SHIFT                                                    0xc
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13__SHIFT                                                    0xd
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14__SHIFT                                                    0xe
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15__SHIFT                                                    0xf
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16__SHIFT                                                    0x10
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17__SHIFT                                                    0x11
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18__SHIFT                                                    0x12
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19__SHIFT                                                    0x13
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20__SHIFT                                                    0x14
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21__SHIFT                                                    0x15
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22__SHIFT                                                    0x16
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23__SHIFT                                                    0x17
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24__SHIFT                                                    0x18
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25__SHIFT                                                    0x19
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26__SHIFT                                                    0x1a
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27__SHIFT                                                    0x1b
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28__SHIFT                                                    0x1c
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29__SHIFT                                                    0x1d
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30__SHIFT                                                    0x1e
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS__SHIFT                                                 0x1f
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0_MASK                                                       0x00000001L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1_MASK                                                       0x00000002L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2_MASK                                                       0x00000004L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3_MASK                                                       0x00000008L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4_MASK                                                       0x00000010L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5_MASK                                                       0x00000020L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6_MASK                                                       0x00000040L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7_MASK                                                       0x00000080L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8_MASK                                                       0x00000100L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9_MASK                                                       0x00000200L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10_MASK                                                      0x00000400L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11_MASK                                                      0x00000800L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12_MASK                                                      0x00001000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13_MASK                                                      0x00002000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14_MASK                                                      0x00004000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15_MASK                                                      0x00008000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16_MASK                                                      0x00010000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17_MASK                                                      0x00020000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18_MASK                                                      0x00040000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19_MASK                                                      0x00080000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20_MASK                                                      0x00100000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21_MASK                                                      0x00200000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22_MASK                                                      0x00400000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23_MASK                                                      0x00800000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24_MASK                                                      0x01000000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25_MASK                                                      0x02000000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26_MASK                                                      0x04000000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27_MASK                                                      0x08000000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28_MASK                                                      0x10000000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29_MASK                                                      0x20000000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30_MASK                                                      0x40000000L
+#define BIF_BX2_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS_MASK                                                   0x80000000L
+//BIF_BX2_VF_FB_EN
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF0__SHIFT                                                                 0x0
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF1__SHIFT                                                                 0x1
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF2__SHIFT                                                                 0x2
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF3__SHIFT                                                                 0x3
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF4__SHIFT                                                                 0x4
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF5__SHIFT                                                                 0x5
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF6__SHIFT                                                                 0x6
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF7__SHIFT                                                                 0x7
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF8__SHIFT                                                                 0x8
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF9__SHIFT                                                                 0x9
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF10__SHIFT                                                                0xa
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF11__SHIFT                                                                0xb
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF12__SHIFT                                                                0xc
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF13__SHIFT                                                                0xd
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF14__SHIFT                                                                0xe
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF15__SHIFT                                                                0xf
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF16__SHIFT                                                                0x10
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF17__SHIFT                                                                0x11
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF18__SHIFT                                                                0x12
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF19__SHIFT                                                                0x13
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF20__SHIFT                                                                0x14
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF21__SHIFT                                                                0x15
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF22__SHIFT                                                                0x16
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF23__SHIFT                                                                0x17
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF24__SHIFT                                                                0x18
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF25__SHIFT                                                                0x19
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF26__SHIFT                                                                0x1a
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF27__SHIFT                                                                0x1b
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF28__SHIFT                                                                0x1c
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF29__SHIFT                                                                0x1d
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF30__SHIFT                                                                0x1e
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF0_MASK                                                                   0x00000001L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF1_MASK                                                                   0x00000002L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF2_MASK                                                                   0x00000004L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF3_MASK                                                                   0x00000008L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF4_MASK                                                                   0x00000010L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF5_MASK                                                                   0x00000020L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF6_MASK                                                                   0x00000040L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF7_MASK                                                                   0x00000080L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF8_MASK                                                                   0x00000100L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF9_MASK                                                                   0x00000200L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF10_MASK                                                                  0x00000400L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF11_MASK                                                                  0x00000800L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF12_MASK                                                                  0x00001000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF13_MASK                                                                  0x00002000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF14_MASK                                                                  0x00004000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF15_MASK                                                                  0x00008000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF16_MASK                                                                  0x00010000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF17_MASK                                                                  0x00020000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF18_MASK                                                                  0x00040000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF19_MASK                                                                  0x00080000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF20_MASK                                                                  0x00100000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF21_MASK                                                                  0x00200000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF22_MASK                                                                  0x00400000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF23_MASK                                                                  0x00800000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF24_MASK                                                                  0x01000000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF25_MASK                                                                  0x02000000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF26_MASK                                                                  0x04000000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF27_MASK                                                                  0x08000000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF28_MASK                                                                  0x10000000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF29_MASK                                                                  0x20000000L
+#define BIF_BX2_VF_FB_EN__VF_FB_EN_VF30_MASK                                                                  0x40000000L
+//BIF_BX2_VF_REGWR_STATUS
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0__SHIFT                                                   0x0
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1__SHIFT                                                   0x1
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2__SHIFT                                                   0x2
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3__SHIFT                                                   0x3
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4__SHIFT                                                   0x4
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5__SHIFT                                                   0x5
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6__SHIFT                                                   0x6
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7__SHIFT                                                   0x7
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8__SHIFT                                                   0x8
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9__SHIFT                                                   0x9
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10__SHIFT                                                  0xa
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11__SHIFT                                                  0xb
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12__SHIFT                                                  0xc
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13__SHIFT                                                  0xd
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14__SHIFT                                                  0xe
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15__SHIFT                                                  0xf
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16__SHIFT                                                  0x10
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17__SHIFT                                                  0x11
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18__SHIFT                                                  0x12
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19__SHIFT                                                  0x13
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20__SHIFT                                                  0x14
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21__SHIFT                                                  0x15
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22__SHIFT                                                  0x16
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23__SHIFT                                                  0x17
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24__SHIFT                                                  0x18
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25__SHIFT                                                  0x19
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26__SHIFT                                                  0x1a
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27__SHIFT                                                  0x1b
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28__SHIFT                                                  0x1c
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29__SHIFT                                                  0x1d
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30__SHIFT                                                  0x1e
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0_MASK                                                     0x00000001L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1_MASK                                                     0x00000002L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2_MASK                                                     0x00000004L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3_MASK                                                     0x00000008L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4_MASK                                                     0x00000010L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5_MASK                                                     0x00000020L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6_MASK                                                     0x00000040L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7_MASK                                                     0x00000080L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8_MASK                                                     0x00000100L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9_MASK                                                     0x00000200L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10_MASK                                                    0x00000400L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11_MASK                                                    0x00000800L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12_MASK                                                    0x00001000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13_MASK                                                    0x00002000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14_MASK                                                    0x00004000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15_MASK                                                    0x00008000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16_MASK                                                    0x00010000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17_MASK                                                    0x00020000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18_MASK                                                    0x00040000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19_MASK                                                    0x00080000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20_MASK                                                    0x00100000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21_MASK                                                    0x00200000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22_MASK                                                    0x00400000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23_MASK                                                    0x00800000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24_MASK                                                    0x01000000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25_MASK                                                    0x02000000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26_MASK                                                    0x04000000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27_MASK                                                    0x08000000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28_MASK                                                    0x10000000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29_MASK                                                    0x20000000L
+#define BIF_BX2_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30_MASK                                                    0x40000000L
+//BIF_BX2_VF_DOORBELL_STATUS
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0__SHIFT                                             0x0
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1__SHIFT                                             0x1
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2__SHIFT                                             0x2
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3__SHIFT                                             0x3
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4__SHIFT                                             0x4
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5__SHIFT                                             0x5
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6__SHIFT                                             0x6
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7__SHIFT                                             0x7
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8__SHIFT                                             0x8
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9__SHIFT                                             0x9
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10__SHIFT                                            0xa
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11__SHIFT                                            0xb
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12__SHIFT                                            0xc
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13__SHIFT                                            0xd
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14__SHIFT                                            0xe
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15__SHIFT                                            0xf
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16__SHIFT                                            0x10
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17__SHIFT                                            0x11
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18__SHIFT                                            0x12
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19__SHIFT                                            0x13
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20__SHIFT                                            0x14
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21__SHIFT                                            0x15
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22__SHIFT                                            0x16
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23__SHIFT                                            0x17
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24__SHIFT                                            0x18
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25__SHIFT                                            0x19
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26__SHIFT                                            0x1a
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27__SHIFT                                            0x1b
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28__SHIFT                                            0x1c
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29__SHIFT                                            0x1d
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30__SHIFT                                            0x1e
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0_MASK                                               0x00000001L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1_MASK                                               0x00000002L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2_MASK                                               0x00000004L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3_MASK                                               0x00000008L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4_MASK                                               0x00000010L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5_MASK                                               0x00000020L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6_MASK                                               0x00000040L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7_MASK                                               0x00000080L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8_MASK                                               0x00000100L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9_MASK                                               0x00000200L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10_MASK                                              0x00000400L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11_MASK                                              0x00000800L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12_MASK                                              0x00001000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13_MASK                                              0x00002000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14_MASK                                              0x00004000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15_MASK                                              0x00008000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16_MASK                                              0x00010000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17_MASK                                              0x00020000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18_MASK                                              0x00040000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19_MASK                                              0x00080000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20_MASK                                              0x00100000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21_MASK                                              0x00200000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22_MASK                                              0x00400000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23_MASK                                              0x00800000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24_MASK                                              0x01000000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25_MASK                                              0x02000000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26_MASK                                              0x04000000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27_MASK                                              0x08000000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28_MASK                                              0x10000000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29_MASK                                              0x20000000L
+#define BIF_BX2_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30_MASK                                              0x40000000L
+//BIF_BX2_VF_FB_STATUS
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF0__SHIFT                                                         0x0
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF1__SHIFT                                                         0x1
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF2__SHIFT                                                         0x2
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF3__SHIFT                                                         0x3
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF4__SHIFT                                                         0x4
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF5__SHIFT                                                         0x5
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF6__SHIFT                                                         0x6
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF7__SHIFT                                                         0x7
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF8__SHIFT                                                         0x8
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF9__SHIFT                                                         0x9
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF10__SHIFT                                                        0xa
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF11__SHIFT                                                        0xb
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF12__SHIFT                                                        0xc
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF13__SHIFT                                                        0xd
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF14__SHIFT                                                        0xe
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF15__SHIFT                                                        0xf
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF16__SHIFT                                                        0x10
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF17__SHIFT                                                        0x11
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF18__SHIFT                                                        0x12
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF19__SHIFT                                                        0x13
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF20__SHIFT                                                        0x14
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF21__SHIFT                                                        0x15
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF22__SHIFT                                                        0x16
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF23__SHIFT                                                        0x17
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF24__SHIFT                                                        0x18
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF25__SHIFT                                                        0x19
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF26__SHIFT                                                        0x1a
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF27__SHIFT                                                        0x1b
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF28__SHIFT                                                        0x1c
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF29__SHIFT                                                        0x1d
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF30__SHIFT                                                        0x1e
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF0_MASK                                                           0x00000001L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF1_MASK                                                           0x00000002L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF2_MASK                                                           0x00000004L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF3_MASK                                                           0x00000008L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF4_MASK                                                           0x00000010L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF5_MASK                                                           0x00000020L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF6_MASK                                                           0x00000040L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF7_MASK                                                           0x00000080L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF8_MASK                                                           0x00000100L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF9_MASK                                                           0x00000200L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF10_MASK                                                          0x00000400L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF11_MASK                                                          0x00000800L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF12_MASK                                                          0x00001000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF13_MASK                                                          0x00002000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF14_MASK                                                          0x00004000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF15_MASK                                                          0x00008000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF16_MASK                                                          0x00010000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF17_MASK                                                          0x00020000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF18_MASK                                                          0x00040000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF19_MASK                                                          0x00080000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF20_MASK                                                          0x00100000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF21_MASK                                                          0x00200000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF22_MASK                                                          0x00400000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF23_MASK                                                          0x00800000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF24_MASK                                                          0x01000000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF25_MASK                                                          0x02000000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF26_MASK                                                          0x04000000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF27_MASK                                                          0x08000000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF28_MASK                                                          0x10000000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF29_MASK                                                          0x20000000L
+#define BIF_BX2_VF_FB_STATUS__VF_FB_STATUS_VF30_MASK                                                          0x40000000L
+//BIF_BX2_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX2_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX2_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX2_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX2_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX2_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX2_MAILBOX_INDEX
+#define BIF_BX2_MAILBOX_INDEX__MAILBOX_INDEX__SHIFT                                                           0x0
+#define BIF_BX2_MAILBOX_INDEX__MAILBOX_INDEX_MASK                                                             0x0000001FL
+//BIF_BX2_BIF_PERSTB_PAD_CNTL
+#define BIF_BX2_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX2_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL_MASK                                                     0x0000FFFFL
+//BIF_BX2_BIF_PX_EN_PAD_CNTL
+#define BIF_BX2_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL__SHIFT                                                     0x0
+#define BIF_BX2_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL_MASK                                                       0x00000FFFL
+//BIF_BX2_BIF_REFPADKIN_PAD_CNTL
+#define BIF_BX2_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL__SHIFT                                             0x0
+#define BIF_BX2_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL_MASK                                               0x000000FFL
+//BIF_BX2_BIF_CLKREQB_PAD_CNTL
+#define BIF_BX2_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL__SHIFT                                                 0x0
+#define BIF_BX2_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_MASK                                                   0x7FFFFFFFL
+//BIF_BX2_BIF_PWRBRK_PAD_CNTL
+#define BIF_BX2_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX2_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL_MASK                                                     0x000000FFL
+
+
+// addressBlock: nbif0_rcc_dev0_BIFDEC1
+//RCC_DEV0_2_RCC_ERR_INT_CNTL
+#define RCC_DEV0_2_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN__SHIFT                                0x0
+#define RCC_DEV0_2_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN_MASK                                  0x00000001L
+//RCC_DEV0_2_RCC_BACO_CNTL_MISC
+#define RCC_DEV0_2_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS__SHIFT                                                 0x0
+#define RCC_DEV0_2_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS__SHIFT                                                  0x1
+#define RCC_DEV0_2_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS_MASK                                                   0x00000001L
+#define RCC_DEV0_2_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS_MASK                                                    0x00000002L
+//RCC_DEV0_2_RCC_RESET_EN
+#define RCC_DEV0_2_RCC_RESET_EN__DB_APER_RESET_EN__SHIFT                                                      0xf
+#define RCC_DEV0_2_RCC_RESET_EN__DB_APER_RESET_EN_MASK                                                        0x00008000L
+//RCC_DEV0_3_RCC_VDM_SUPPORT
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_3_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_3_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+//RCC_DEV0_2_RCC_GPUIOV_REGION
+#define RCC_DEV0_2_RCC_GPUIOV_REGION__LFB_REGION__SHIFT                                                       0x0
+#define RCC_DEV0_2_RCC_GPUIOV_REGION__MAX_REGION__SHIFT                                                       0x4
+#define RCC_DEV0_2_RCC_GPUIOV_REGION__LFB_REGION_MASK                                                         0x0000000FL
+#define RCC_DEV0_2_RCC_GPUIOV_REGION__MAX_REGION_MASK                                                         0x000000F0L
+//RCC_DEV0_2_RCC_GPU_HOSTVM_EN
+#define RCC_DEV0_2_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN__SHIFT                                                    0x0
+#define RCC_DEV0_2_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN_MASK                                                      0x00000001L
+//RCC_DEV0_2_RCC_CONSOLE_IOV_MODE_CNTL
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE__SHIFT                              0x0
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN__SHIFT                                    0x1
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE_MASK                                0x00000001L
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN_MASK                                      0x00000002L
+//RCC_DEV0_2_RCC_CONSOLE_IOV_FIRST_VF_OFFSET
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET__SHIFT                        0x0
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET_MASK                          0xFFFFL
+//RCC_DEV0_2_RCC_CONSOLE_IOV_VF_STRIDE
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE__SHIFT                                    0x0
+#define RCC_DEV0_2_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE_MASK                                      0xFFFFL
+//RCC_DEV0_2_RCC_PEER_REG_RANGE0
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE0__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE0__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE0__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE0__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_2_RCC_PEER_REG_RANGE1
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE1__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE1__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE1__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_2_RCC_PEER_REG_RANGE1__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_3_RCC_BUS_CNTL
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_3_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_3_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_3_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_3_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_3_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_3_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_3_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_3_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_2_RCC_CONFIG_CNTL
+#define RCC_DEV0_2_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT                                                     0x0
+#define RCC_DEV0_2_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT                                               0x2
+#define RCC_DEV0_2_RCC_CONFIG_CNTL__GRPH_ADRSEL__SHIFT                                                        0x3
+#define RCC_DEV0_2_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN_MASK                                                       0x00000001L
+#define RCC_DEV0_2_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK                                                 0x00000004L
+#define RCC_DEV0_2_RCC_CONFIG_CNTL__GRPH_ADRSEL_MASK                                                          0x00000018L
+//RCC_DEV0_2_RCC_CONFIG_F0_BASE
+#define RCC_DEV0_2_RCC_CONFIG_F0_BASE__F0_BASE__SHIFT                                                         0x0
+#define RCC_DEV0_2_RCC_CONFIG_F0_BASE__F0_BASE_MASK                                                           0xFFFFFFFFL
+//RCC_DEV0_2_RCC_CONFIG_APER_SIZE
+#define RCC_DEV0_2_RCC_CONFIG_APER_SIZE__APER_SIZE__SHIFT                                                     0x0
+#define RCC_DEV0_2_RCC_CONFIG_APER_SIZE__APER_SIZE_MASK                                                       0xFFFFFFFFL
+//RCC_DEV0_2_RCC_CONFIG_REG_APER_SIZE
+#define RCC_DEV0_2_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT                                             0x0
+#define RCC_DEV0_2_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK                                               0x07FFFFFFL
+//RCC_DEV0_2_RCC_XDMA_LO
+#define RCC_DEV0_2_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_2_RCC_XDMA_LO__BIF_XDMA_APER_EN__SHIFT                                                       0x1f
+#define RCC_DEV0_2_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK                                                     0x7FFFFFFFL
+#define RCC_DEV0_2_RCC_XDMA_LO__BIF_XDMA_APER_EN_MASK                                                         0x80000000L
+//RCC_DEV0_2_RCC_XDMA_HI
+#define RCC_DEV0_2_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_2_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK                                                     0x7FFFFFFFL
+//RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_3_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_2_RCC_BUSNUM_CNTL1
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL1__ID_MASK__SHIFT                                                           0x0
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL1__ID_MASK_MASK                                                             0x000000FFL
+//RCC_DEV0_2_RCC_BUSNUM_LIST0
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID0__SHIFT                                                               0x0
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID1__SHIFT                                                               0x8
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID2__SHIFT                                                               0x10
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID3__SHIFT                                                               0x18
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID0_MASK                                                                 0x000000FFL
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID1_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID2_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_2_RCC_BUSNUM_LIST0__ID3_MASK                                                                 0xFF000000L
+//RCC_DEV0_2_RCC_BUSNUM_LIST1
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID4__SHIFT                                                               0x0
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID5__SHIFT                                                               0x8
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID6__SHIFT                                                               0x10
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID7__SHIFT                                                               0x18
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID4_MASK                                                                 0x000000FFL
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID5_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID6_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_2_RCC_BUSNUM_LIST1__ID7_MASK                                                                 0xFF000000L
+//RCC_DEV0_2_RCC_BUSNUM_CNTL2
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT                                                    0x0
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT                                                     0x8
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT                                                       0x10
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT                                           0x11
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK                                                      0x000000FFL
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK                                                       0x00000100L
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__HDPREG_CNTL_MASK                                                         0x00010000L
+#define RCC_DEV0_2_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK                                             0x00020000L
+//RCC_DEV0_2_RCC_CAPTURE_HOST_BUSNUM
+#define RCC_DEV0_2_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT                                                   0x0
+#define RCC_DEV0_2_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN_MASK                                                     0x00000001L
+//RCC_DEV0_2_RCC_HOST_BUSNUM
+#define RCC_DEV0_2_RCC_HOST_BUSNUM__HOST_ID__SHIFT                                                            0x0
+#define RCC_DEV0_2_RCC_HOST_BUSNUM__HOST_ID_MASK                                                              0x0000FFFFL
+//RCC_DEV0_2_RCC_PEER0_FB_OFFSET_HI
+#define RCC_DEV0_2_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_2_RCC_PEER0_FB_OFFSET_LO
+#define RCC_DEV0_2_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_2_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_2_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_2_RCC_PEER1_FB_OFFSET_HI
+#define RCC_DEV0_2_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_2_RCC_PEER1_FB_OFFSET_LO
+#define RCC_DEV0_2_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_2_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_2_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_2_RCC_PEER2_FB_OFFSET_HI
+#define RCC_DEV0_2_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_2_RCC_PEER2_FB_OFFSET_LO
+#define RCC_DEV0_2_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_2_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_2_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_2_RCC_PEER3_FB_OFFSET_HI
+#define RCC_DEV0_2_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_2_RCC_PEER3_FB_OFFSET_LO
+#define RCC_DEV0_2_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_2_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_2_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_2_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT                                                   0x0
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT                                                   0x8
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT                                                   0x10
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT                                                   0x18
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK                                                     0x000000FFL
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK                                                     0x0000FF00L
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK                                                     0x00FF0000L
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK                                                     0xFF000000L
+//RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT                                                   0x0
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT                                                   0x8
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT                                                   0x10
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT                                                   0x18
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK                                                     0x000000FFL
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK                                                     0x0000FF00L
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK                                                     0x00FF0000L
+#define RCC_DEV0_2_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK                                                     0xFF000000L
+//RCC_DEV0_3_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_3_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_3_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_3_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                            0x10
+#define RCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+#define RCC_DEV0_3_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                              0x00FF0000L
+//RCC_DEV0_3_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_3_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_3_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_3_RCC_MH_ARB_CNTL
+#define RCC_DEV0_3_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_3_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_3_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_3_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x0000FFFEL
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFDEC2
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                             0x2
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                                0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                  0x00000001L
+//RCC_DEV0_EPF0_1_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                               0x1
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_2__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_3__SHIFT                                               0x3
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                                 0x00000001L
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                                 0x00000002L
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_2_MASK                                                 0x00000004L
+#define RCC_DEV0_EPF0_1_GFXMSIX_PBA__MSIX_PENDING_BITS_3_MASK                                                 0x00000008L
+
+
+// addressBlock: nbif0_rcc_strap_BIFDEC1
+//RCC_STRAP3_RCC_BIF_STRAP0
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x14
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN__SHIFT                         0x15
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_GPUIOV_EN__SHIFT                                                     0x16
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE__SHIFT                                          0x17
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00100000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN_MASK                           0x00200000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_GPUIOV_EN_MASK                                                       0x00400000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE_MASK                                            0x00800000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP3_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP3_RCC_BIF_STRAP1
+#define RCC_STRAP3_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP3_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP3_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP3_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP3_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP3_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP3_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP3_RCC_BIF_STRAP2
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN__SHIFT                                          0x7
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS__SHIFT                                               0xc
+#define RCC_STRAP3_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_SWUS_SEC_LVL_OVRD_EN_MASK                                            0x00000080L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_USB_PD_FUNC_DIS_MASK                                                 0x00001000L
+#define RCC_STRAP3_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP3_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP3_RCC_BIF_STRAP3
+#define RCC_STRAP3_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP3_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP3_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP3_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP3_RCC_BIF_STRAP4
+#define RCC_STRAP3_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP3_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP3_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP3_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP3_RCC_BIF_STRAP5
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP3_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP3_RCC_BIF_STRAP6
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE__SHIFT                                               0x5
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0xa
+#define RCC_STRAP3_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6__SHIFT                                                 0xf
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE_MASK                                                 0x00000020L
+#define RCC_STRAP3_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00007C00L
+#define RCC_STRAP3_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6_MASK                                                   0xFFFF8000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0__SHIFT                                     0x14
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0_MASK                                       0x00100000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0__SHIFT                                   0x14
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0__SHIFT                              0x18
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0__SHIFT                           0x1a
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0_MASK                                     0x00100000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0_MASK                                0x03000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0_MASK                             0x0C000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                  0x1f
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                    0x80000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0__SHIFT                                         0x1e
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0_MASK                                           0x40000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP3_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP3_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP17
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0__SHIFT                            0xc
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0_MASK                              0x000FF000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0__SHIFT                      0x1
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0__SHIFT                     0x2
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0__SHIFT                    0x4
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0_MASK                        0x00000002L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0_MASK                       0x00000004L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0_MASK                      0x00000010L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0__SHIFT             0x19
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0_MASK               0x02000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0__SHIFT                                  0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0_MASK                                    0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0_MASK                                       0x38000000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP3_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP3_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+//RCC_STRAP3_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1__SHIFT                    0x4
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1_MASK                      0x00000010L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_ATS_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_DSN_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1__SHIFT             0x19
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1_MASK               0x02000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP3_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP3_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1__SHIFT                                  0x2
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1__SHIFT                                        0x8
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1__SHIFT                                  0xa
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1__SHIFT                                        0x10
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1__SHIFT                                  0x12
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1__SHIFT                                        0x18
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1__SHIFT                                        0x1a
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1_MASK                                    0x00000004L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1_MASK                                          0x00000100L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1_MASK                                    0x00000400L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1_MASK                                          0x00010000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1_MASK                                    0x00040000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1_MASK                                          0x01000000L
+#define RCC_STRAP3_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1_MASK                                          0x04000000L
+//RCC_STRAP3_RCC_DEV0_EPF1_STRAP7
+
+
+// addressBlock: nbif0_bif_bx_pf_BIFPFVFDEC1
+//BIF_BX_PF2_BIF_BME_STATUS
+#define BIF_BX_PF2_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                                      0x0
+#define BIF_BX_PF2_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                                0x10
+#define BIF_BX_PF2_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                                        0x00000001L
+#define BIF_BX_PF2_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                                  0x00010000L
+//BIF_BX_PF2_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                                0x0
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                             0x1
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                                0x2
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                                    0x3
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                          0x10
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                                       0x11
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                          0x12
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                              0x13
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                                  0x00000001L
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                               0x00000002L
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                                  0x00000004L
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                                      0x00000008L
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                            0x00010000L
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                                         0x00020000L
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                            0x00040000L
+#define BIF_BX_PF2_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                                0x00080000L
+//BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF2_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF2_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF2_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF2_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                          0x0
+#define BIF_BX_PF2_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                            0x00000001L
+//BIF_BX_PF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_PF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_PF2_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_PF2_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF2_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF2_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//BIF_BX_PF2_BIF_TRANS_PENDING
+#define BIF_BX_PF2_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                            0x0
+#define BIF_BX_PF2_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                            0x1
+#define BIF_BX_PF2_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                              0x00000001L
+#define BIF_BX_PF2_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                              0x00000002L
+//BIF_BX_PF2_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_PF2_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                                0x0
+#define BIF_BX_PF2_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                                  0x00000001L
+
+
+// addressBlock: nbif0_rcc_dev0_epf0_BIFPFVFDEC1
+//RCC_DEV0_EPF0_1_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_1_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                                0x0
+#define RCC_DEV0_EPF0_1_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                       0x1
+#define RCC_DEV0_EPF0_1_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                  0x00000001L
+#define RCC_DEV0_EPF0_1_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                         0x00000002L
+//RCC_DEV0_EPF0_1_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_1_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_1_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                       0x00000001L
+//RCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_1_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_1_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_1_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_1_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                       0x0
+#define RCC_DEV0_EPF0_1_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                            0x1f
+#define RCC_DEV0_EPF0_1_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                         0x00000001L
+#define RCC_DEV0_EPF0_1_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                              0x80000000L
+
+
+// addressBlock: nbif0_gdc_GDCDEC
+//GDC1_NGDC_PG_MISC_CTRL
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY__SHIFT                                                   0xa
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1__SHIFT                                                      0xd
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS__SHIFT                                                   0xe
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2__SHIFT                                                      0x10
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS__SHIFT                                        0x18
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE__SHIFT                                              0x1f
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_ENDP_D3_ONLY_MASK                                                     0x00000400L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM1_MASK                                                        0x00002000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_DS_ALLOW_DIS_MASK                                                     0x00004000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_PG_CLK_PERM2_MASK                                                        0x00010000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_REFCLK_CYCLE_FOR_200NS_MASK                                          0x3F000000L
+#define GDC1_NGDC_PG_MISC_CTRL__NGDC_CFG_PG_EXIT_OVERRIDE_MASK                                                0x80000000L
+//GDC1_NGDC_PGMST_CTRL
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS__SHIFT                                                   0x0
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN__SHIFT                                                           0x8
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN__SHIFT                                               0xa
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN__SHIFT                                                   0xe
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_HYSTERESIS_MASK                                                     0x000000FFL
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_PG_EN_MASK                                                             0x00000100L
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_IDLENESS_COUNT_EN_MASK                                                 0x00003C00L
+#define GDC1_NGDC_PGMST_CTRL__NGDC_CFG_FW_PG_EXIT_EN_MASK                                                     0x0000C000L
+//GDC1_NGDC_PGSLV_CTRL
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS__SHIFT                                       0x0
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS__SHIFT                                       0x5
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS__SHIFT                                          0xa
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_2_IDLE_HYSTERESIS__SHIFT                                       0xf
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_0_IDLE_HYSTERESIS_MASK                                         0x0000001FL
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_1_IDLE_HYSTERESIS_MASK                                         0x000003E0L
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_GDCCLK_IDLE_HYSTERESIS_MASK                                            0x00007C00L
+#define GDC1_NGDC_PGSLV_CTRL__NGDC_CFG_SHUBCLK_2_IDLE_HYSTERESIS_MASK                                         0x000F8000L
+//GDC2_ATDMA_MISC_CNTL
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_ARB_MODE__SHIFT                                                       0x0
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_MISC_CNTL_INSERT_RD_ON_2ND_WDAT_EN__SHIFT                                 0x1
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_RDRSP_ARB_MODE__SHIFT                                                     0x2
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_VC6_WEIGHT__SHIFT                                                     0x8
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_VC0_WEIGHT__SHIFT                                                     0x10
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_VC1_WEIGHT__SHIFT                                                     0x18
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_ARB_MODE_MASK                                                         0x00000001L
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_MISC_CNTL_INSERT_RD_ON_2ND_WDAT_EN_MASK                                   0x00000002L
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_RDRSP_ARB_MODE_MASK                                                       0x0000000CL
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_VC6_WEIGHT_MASK                                                       0x0000FF00L
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_VC0_WEIGHT_MASK                                                       0x00FF0000L
+#define GDC2_ATDMA_MISC_CNTL__ATDMA_WRR_VC1_WEIGHT_MASK                                                       0xFF000000L
+
+
+// addressBlock: gdc_gs2a_s2a0_gdc_gs2a_regs_indirect_blk
+//S2A0_ROUTING_ID_TABLE_ENTRY0_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY0_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY0_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY1_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY1_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY1_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY2_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY2_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY2_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY3_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY3_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY3_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY4_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY4_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY4_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY5_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY5_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY5_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY6_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY6_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY6_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY7_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY7_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY7_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY8_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY8_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY8_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY9_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__VALID__SHIFT                                                          0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__AXIID__SHIFT                                                          0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__QUEUEID__SHIFT                                                        0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__AXIIFID__SHIFT                                                        0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__ARQOS__SHIFT                                                          0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__AWQOS__SHIFT                                                          0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__RESERVED_0__SHIFT                                                     0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__VALID_MASK                                                            0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__AXIID_MASK                                                            0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__QUEUEID_MASK                                                          0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__AXIIFID_MASK                                                          0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__ARQOS_MASK                                                            0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__AWQOS_MASK                                                            0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_L__RESERVED_0_MASK                                                       0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY9_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_H__RESERVED_1__SHIFT                                                     0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY9_H__RESERVED_1_MASK                                                       0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY10_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY10_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY10_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY11_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY11_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY11_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY12_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY12_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY12_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY13_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY13_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY13_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY14_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY14_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY14_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY15_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY15_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY15_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY16_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY16_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY16_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY17_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY17_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY17_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY18_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY18_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY18_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY19_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY19_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY19_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY20_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY20_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY20_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY21_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY21_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY21_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY22_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY22_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY22_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY23_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY23_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY23_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY24_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY24_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY24_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY25_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY25_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY25_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY26_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY26_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY26_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY27_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY27_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY27_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY28_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY28_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY28_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY29_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY29_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY29_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY30_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY30_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY30_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY31_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY31_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY31_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY32_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY32_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY32_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY33_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY33_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY33_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY34_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY34_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY34_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY35_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY35_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY35_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY36_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY36_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY36_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY37_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY37_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY37_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY38_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY38_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY38_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY39_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY39_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY39_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY40_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY40_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY40_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY41_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY41_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY41_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY42_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY42_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY42_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY43_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY43_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY43_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY44_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY44_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY44_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY45_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY45_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY45_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY46_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY46_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY46_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY47_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY47_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY47_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY48_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY48_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY48_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY49_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY49_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY49_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY50_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY50_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY50_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY51_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY51_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY51_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY52_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY52_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY52_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY53_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY53_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY53_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY54_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY54_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY54_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY55_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY55_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY55_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY56_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY56_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY56_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY57_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY57_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY57_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY58_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY58_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY58_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY59_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY59_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY59_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY60_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY60_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY60_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY61_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY61_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY61_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY62_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY62_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY62_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+//S2A0_ROUTING_ID_TABLE_ENTRY63_L
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__VALID__SHIFT                                                         0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__AXIID__SHIFT                                                         0x1
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__QUEUEID__SHIFT                                                       0x4
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__AXIIFID__SHIFT                                                       0x7
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__ARQOS__SHIFT                                                         0xa
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__AWQOS__SHIFT                                                         0xe
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__RESERVED_0__SHIFT                                                    0x12
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__VALID_MASK                                                           0x00000001L
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__AXIID_MASK                                                           0x0000000EL
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__QUEUEID_MASK                                                         0x00000070L
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__AXIIFID_MASK                                                         0x00000380L
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__ARQOS_MASK                                                           0x00003C00L
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__AWQOS_MASK                                                           0x0003C000L
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_L__RESERVED_0_MASK                                                      0xFFFC0000L
+//S2A0_ROUTING_ID_TABLE_ENTRY63_H
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_H__RESERVED_1__SHIFT                                                    0x0
+#define S2A0_ROUTING_ID_TABLE_ENTRY63_H__RESERVED_1_MASK                                                      0xFFFFFFFFL
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/discovery.h b/drivers/gpu/drm/amd/include/discovery.h
index 76c9f951bc1c86..07199c6abc73ad 100644
--- a/drivers/gpu/drm/amd/include/discovery.h
+++ b/drivers/gpu/drm/amd/include/discovery.h
@@ -33,6 +33,11 @@
 #define MALL_INFO_TABLE_ID              0x4C4C414D
 #define NPS_INFO_TABLE_ID 0x0053504E
 
+/* = ('I' << 16u | 'R' << 8u | 'M') */
+#define MEM_RSV_TABLE_SIGNATURE          0x0049524D
+#define MEM_RSV_TABLE_VERSION            0x01
+#define MAX_MEM_REGION_NUM               16
+
 typedef enum {
 	IP_DISCOVERY = 0,
 	GC,
@@ -40,7 +45,8 @@ typedef enum {
 	VCN_INFO,
 	MALL_INFO,
 	NPS_INFO,
-	TOTAL_TABLES = 6
+	MEM_RESERVED_INFO,
+	TOTAL_TABLES = 7
 } table;
 
 #pragma pack(1)
@@ -272,45 +278,92 @@ struct gc_info_v1_2 {
 };
 
 struct gc_info_v1_3 {
-    struct gpu_info_header header;
-    uint32_t gc_num_se;
-    uint32_t gc_num_wgp0_per_sa;
-    uint32_t gc_num_wgp1_per_sa;
-    uint32_t gc_num_rb_per_se;
-    uint32_t gc_num_gl2c;
-    uint32_t gc_num_gprs;
-    uint32_t gc_num_max_gs_thds;
-    uint32_t gc_gs_table_depth;
-    uint32_t gc_gsprim_buff_depth;
-    uint32_t gc_parameter_cache_depth;
-    uint32_t gc_double_offchip_lds_buffer;
-    uint32_t gc_wave_size;
-    uint32_t gc_max_waves_per_simd;
-    uint32_t gc_max_scratch_slots_per_cu;
-    uint32_t gc_lds_size;
-    uint32_t gc_num_sc_per_se;
-    uint32_t gc_num_sa_per_se;
-    uint32_t gc_num_packer_per_sc;
-    uint32_t gc_num_gl2a;
-    uint32_t gc_num_tcp_per_sa;
-    uint32_t gc_num_sdp_interface;
-    uint32_t gc_num_tcps;
-    uint32_t gc_num_tcp_per_wpg;
-    uint32_t gc_tcp_l1_size;
-    uint32_t gc_num_sqc_per_wgp;
-    uint32_t gc_l1_instruction_cache_size_per_sqc;
-    uint32_t gc_l1_data_cache_size_per_sqc;
-    uint32_t gc_gl1c_per_sa;
-    uint32_t gc_gl1c_size_per_instance;
-    uint32_t gc_gl2c_per_gpu;
-    uint32_t gc_tcp_size_per_cu;
-    uint32_t gc_tcp_cache_line_size;
-    uint32_t gc_instruction_cache_size_per_sqc;
-    uint32_t gc_instruction_cache_line_size;
-    uint32_t gc_scalar_data_cache_size_per_sqc;
-    uint32_t gc_scalar_data_cache_line_size;
-    uint32_t gc_tcc_size;
-    uint32_t gc_tcc_cache_line_size;
+	struct gpu_info_header header;
+	uint32_t gc_num_se;
+	uint32_t gc_num_wgp0_per_sa;
+	uint32_t gc_num_wgp1_per_sa;
+	uint32_t gc_num_rb_per_se;
+	uint32_t gc_num_gl2c;
+	uint32_t gc_num_gprs;
+	uint32_t gc_num_max_gs_thds;
+	uint32_t gc_gs_table_depth;
+	uint32_t gc_gsprim_buff_depth;
+	uint32_t gc_parameter_cache_depth;
+	uint32_t gc_double_offchip_lds_buffer;
+	uint32_t gc_wave_size;
+	uint32_t gc_max_waves_per_simd;
+	uint32_t gc_max_scratch_slots_per_cu;
+	uint32_t gc_lds_size;
+	uint32_t gc_num_sc_per_se;
+	uint32_t gc_num_sa_per_se;
+	uint32_t gc_num_packer_per_sc;
+	uint32_t gc_num_gl2a;
+	uint32_t gc_num_tcp_per_sa;
+	uint32_t gc_num_sdp_interface;
+	uint32_t gc_num_tcps;
+	uint32_t gc_num_tcp_per_wpg;
+	uint32_t gc_tcp_l1_size;
+	uint32_t gc_num_sqc_per_wgp;
+	uint32_t gc_l1_instruction_cache_size_per_sqc;
+	uint32_t gc_l1_data_cache_size_per_sqc;
+	uint32_t gc_gl1c_per_sa;
+	uint32_t gc_gl1c_size_per_instance;
+	uint32_t gc_gl2c_per_gpu;
+	uint32_t gc_tcp_size_per_cu;
+	uint32_t gc_tcp_cache_line_size;
+	uint32_t gc_instruction_cache_size_per_sqc;
+	uint32_t gc_instruction_cache_line_size;
+	uint32_t gc_scalar_data_cache_size_per_sqc;
+	uint32_t gc_scalar_data_cache_line_size;
+	uint32_t gc_tcc_size;
+	uint32_t gc_tcc_cache_line_size;
+};
+
+struct gc_info_v1_5 {
+	struct gpu_info_header header;
+	uint32_t gc_num_se;
+	uint32_t gc_num_wgp0_per_sa;
+	uint32_t gc_num_wgp1_per_sa;
+	uint32_t gc_num_rb_per_se;
+	uint32_t gc_num_gl2c;
+	uint32_t gc_num_gprs;
+	uint32_t gc_num_max_gs_thds;
+	uint32_t gc_gs_table_depth;
+	uint32_t gc_gsprim_buff_depth;
+	uint32_t gc_parameter_cache_depth;
+	uint32_t gc_double_offchip_lds_buffer;
+	uint32_t gc_wave_size;
+	uint32_t gc_max_waves_per_simd;
+	uint32_t gc_max_scratch_slots_per_cu;
+	uint32_t gc_lds_size;
+	uint32_t gc_num_sc_per_se;
+	uint32_t gc_num_sa_per_se;
+	uint32_t gc_num_packer_per_sc;
+	uint32_t gc_num_gl2a;
+	uint32_t gc_num_tcp_per_sa;
+	uint32_t gc_num_sdp_interface;
+	uint32_t gc_num_tcps;
+	uint32_t gc_num_tcp_per_wpg;
+	uint32_t gc_tcp_l1_size;
+	uint32_t gc_num_sqc_per_wgp;
+	uint32_t gc_l1_instruction_cache_size_per_sqc;
+	uint32_t gc_l1_data_cache_size_per_sqc;
+	uint32_t gc_gl1c_per_sa;
+	uint32_t gc_gl1c_size_per_instance;
+	uint32_t gc_gl2c_per_gpu;
+	uint32_t gc_tcp_size_per_cu;
+	uint32_t gc_tcp_cache_line_size;
+	uint32_t gc_instruction_cache_size_per_sqc;
+	uint32_t gc_instruction_cache_line_size;
+	uint32_t gc_scalar_data_cache_size_per_sqc;
+	uint32_t gc_scalar_data_cache_line_size;
+	uint32_t gc_tcc_size;
+	uint32_t gc_tcc_cache_line_size;
+	uint32_t gc_num_wgp0_per_sa1;
+	uint32_t gc_num_wgp1_per_sa1;
+	uint32_t gc_max_num_residency_ways;
+	uint32_t gc_cache_ways_size_in_bytes;
+	uint32_t gc_reserved;
 };
 
 struct gc_info_v2_0 {
@@ -457,6 +510,52 @@ struct nps_info_v1_0 {
 		instance_info[NPS_INFO_TABLE_MAX_NUM_INSTANCES];
 };
 
+typedef enum {
+	NO_RESERVED_REGION_ID		= 0,
+	/* reserved for UEFI GOP/DMUB FW */
+	PRE_OS_DISP_FW_REGION_ID	= 1,
+	/* UMF reserved region for master die MPASP */
+	MASTER_DIE_UMF_REGION_ID	= 2,
+	/* DCC meta data */
+	DCC_META_DATA_REGION_ID		= 3,
+	/* GPUVM page fault region */
+	VM_PAGE_FAULT_REGION_ID		= 4,
+	/* G7 Alternate Pstate Aperture 1 */
+	G7_PSTATE_APERTURE1_ID		= 5,
+	/* G7 Alternate Pstate Aperture 2 */
+	G7_PSTATE_APERTURE2_ID		= 6,
+	/* G7 Alternate Pstate Mirror Region */
+	G7_PSTATE_MIRROR_REGION_ID	= 7,
+	/* G7 Pstate Training Data */
+	G7_TRAINING_DATA_REGION_ID	= 8,
+	/* Specific purpose memory region */
+	SPECIFIC_PURPOSE_REGION_ID	= 9,
+	MEM_RESERVED_REGION_NUM,
+} MEM_RESERVED_REGION_ID;
+
+typedef struct mem_reserved_info_header {
+	/* Table Signature */
+	uint32_t signature;
+	/* Table Version */
+	uint32_t version;
+	uint32_t size;
+} mem_reserved_info_header;
+
+typedef struct mem_reserved_info {
+	/* enum of MEM_RESERVED_REGION_ID */
+	uint32_t reserved_region_id;
+	/* Memory Reserved Region Size */
+	uint64_t reserved_region_size;
+	/* Memory Reserved Region Start Address */
+	uint64_t reserved_region_start;
+} mem_reserved_info;
+
+typedef struct mem_reserved_info_table_v1_0 {
+	mem_reserved_info_header header;
+	uint32_t list_num;
+	mem_reserved_info list[MAX_MEM_REGION_NUM];
+} mem_reserved_info_table_v1_0;
+
 #pragma pack()
 
 #endif
diff --git a/drivers/gpu/drm/amd/include/ivsrcid/mpnht/irqsrcs_mpnht_15_0.h b/drivers/gpu/drm/amd/include/ivsrcid/mpnht/irqsrcs_mpnht_15_0.h
new file mode 100644
index 00000000000000..c960353f107db5
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/ivsrcid/mpnht/irqsrcs_mpnht_15_0.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __IRQSRCS_MPNHT_15_0_H__
+#define __IRQSRCS_MPNHT_15_0_H__
+
+/* MPNHT remote interrupt */
+#define MPNHT_15_0__SRCID__REMOTE_INTERRUPT            0x00
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 965b50c8ca3037..d749669f8e6d1b 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -319,7 +319,8 @@ struct kfd2kgd_calls {
 			uint32_t sch_wave,
 			uint32_t que_sleep,
 			uint32_t *reg_offset,
-			uint32_t *reg_data);
+			uint32_t *reg_data,
+			uint32_t inst);
 	void (*get_cu_occupancy)(struct amdgpu_device *adev,
 				 struct kfd_cu_occupancy *cu_occupancy,
 				 int *max_waves_per_cu, uint32_t inst);
@@ -342,6 +343,7 @@ struct kfd2kgd_calls {
 	int (*hqd_sdma_get_counter)(struct amdgpu_device *adev,
 				    void *mqd, uint32_t num_sdma_queues_per_eng,
 				    uint64_t *val);
+	void (*hqd_gfx_clean_fault)(struct amdgpu_device *adev);
 };
 
 #endif	/* KGD_KFD_INTERFACE_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 606d4811530199..4b863a2e38f1b9 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -172,8 +172,24 @@ enum amd_pp_sensors {
 	AMDGPU_PP_SENSOR_MAXNODEPOWERLIMIT,
 	AMDGPU_PP_SENSOR_UBB_POWER,
 	AMDGPU_PP_SENSOR_UBB_POWER_LIMIT,
+	AMDGPU_PP_SENSOR_NPMSTATUS,
 };
 
+/*
+ * NPM sysfs capability bitmap: two bits per field, R then W.
+ * 00 = unsupported, 01 = read-only, 1x = writable (or RW).
+ */
+enum amdgpu_npm_cap_field {
+	AMDGPU_NPM_CAP_CUR_NODE_POWER_LIMIT,
+	AMDGPU_NPM_CAP_NODE_POWER,
+	AMDGPU_NPM_CAP_GLOBAL_PPT_RESID,
+	AMDGPU_NPM_CAP_MAX_NODE_POWER_LIMIT,
+	AMDGPU_NPM_CAP_NPM_STATUS,
+};
+
+#define AMDGPU_NPM_CAP_R(field)	BIT_ULL((field) * 2)
+#define AMDGPU_NPM_CAP_W(field)	BIT_ULL((field) * 2 + 1)
+
 enum amd_pp_task {
 	AMD_PP_TASK_DISPLAY_CONFIG_CHANGE,
 	AMD_PP_TASK_ENABLE_USER_STATE,
@@ -513,6 +529,7 @@ struct amd_pm_funcs {
 	ssize_t (*get_gpu_metrics)(void *handle, void **table);
 	ssize_t (*get_temp_metrics)(void *handle, enum smu_temp_metric_type type, void *table);
 	bool (*temp_metrics_is_supported)(void *handle, enum smu_temp_metric_type type);
+	u64 (*get_npm_cap)(void *handle);
 	ssize_t (*get_xcp_metrics)(void *handle, int xcp_id, void *table);
 	ssize_t (*get_pm_metrics)(void *handle, void *pmmetrics, size_t size);
 	int (*set_watermarks_for_clock_ranges)(void *handle,
diff --git a/drivers/gpu/drm/amd/include/v12_structs.h b/drivers/gpu/drm/amd/include/v12_structs.h
index 28f4b013268116..7fa0c07b51c678 100644
--- a/drivers/gpu/drm/amd/include/v12_structs.h
+++ b/drivers/gpu/drm/amd/include/v12_structs.h
@@ -1696,7 +1696,7 @@ struct v12_1_compute_mqd {
     uint32_t reserved_506; // offset: 506  (0x1FA)
     uint32_t reserved_507; // offset: 507  (0x1FB)
     uint32_t reserved_508; // offset: 508  (0x1FC)
-    uint32_t reserved_509; // offset: 509  (0x1FD)
+    uint32_t coherent_aql_mtype; // offset: 509  (0x1FD)
     uint32_t reserved_510; // offset: 510  (0x1FE)
     uint32_t reserved_511; // offset: 511  (0x1FF)
     uint32_t reserved_512; // offset: 512  (0x200)
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index 808be6c425bff5..df5bbdab1ea392 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -509,9 +509,6 @@ void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
 		    adev->powerplay.pp_funcs->notify_ac_dc)
 			amdgpu_dpm_notify_ac_dc(adev);
 
-		if (is_support_sw_smu(adev))
-			smu_set_ac_dc(adev->powerplay.pp_handle, true);
-
 		mutex_unlock(&adev->pm.mutex);
 	}
 }
@@ -2104,6 +2101,29 @@ bool amdgpu_dpm_is_temp_metrics_supported(struct amdgpu_device *adev,
 	return support_temp_metrics;
 }
 
+/**
+ * amdgpu_dpm_get_npm_cap - Return NPM sysfs capability bitmap
+ * @adev: Pointer to the device.
+ *
+ * Two bits per field (R then W). Both 0 means the field is unsupported.
+ *
+ * Return: Capability bitmap, or 0 if NPM is not supported.
+ */
+u64 amdgpu_dpm_get_npm_cap(struct amdgpu_device *adev)
+{
+	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+	u64 cap = 0;
+
+	if (!pp_funcs || !pp_funcs->get_npm_cap)
+		return 0;
+
+	mutex_lock(&adev->pm.mutex);
+	cap = pp_funcs->get_npm_cap(adev->powerplay.pp_handle);
+	mutex_unlock(&adev->pm.mutex);
+
+	return cap;
+}
+
 /**
  * amdgpu_dpm_get_xcp_metrics - Retrieve metrics for a specific compute
  * partition
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index c0e677328af574..c6e3748bb11ba6 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -2266,16 +2266,15 @@ static ssize_t amdgpu_show_npm_status(struct device *dev,
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = drm_to_adev(ddev);
-	u32 npower;
+	u32 npm_status;
 	int r;
 
-	/* get the node power */
-	r = amdgpu_pm_get_sensor_generic(adev, AMDGPU_PP_SENSOR_NODEPOWER,
-					 (void *)&npower);
+	r = amdgpu_pm_get_sensor_generic(adev, AMDGPU_PP_SENSOR_NPMSTATUS,
+					 (void *)&npm_status);
 	if (r)
 		return r;
 
-	return sysfs_emit(buf, "%s\n", str_enabled_disabled(npower));
+	return sysfs_emit(buf, "%s\n", str_enabled_disabled(npm_status));
 }
 
 /**
@@ -4726,6 +4725,37 @@ static int amdgpu_od_set_init(struct amdgpu_device *adev)
 	return ret;
 }
 
+static void amdgpu_pm_npm_sysfs_init(struct amdgpu_device *adev)
+{
+	static const struct {
+		struct device_attribute *attr;
+		unsigned int field;
+	} files[] = {
+		{ &dev_attr_cur_node_power_limit, AMDGPU_NPM_CAP_CUR_NODE_POWER_LIMIT },
+		{ &dev_attr_node_power, AMDGPU_NPM_CAP_NODE_POWER },
+		{ &dev_attr_global_ppt_resid, AMDGPU_NPM_CAP_GLOBAL_PPT_RESID },
+		{ &dev_attr_max_node_power_limit, AMDGPU_NPM_CAP_MAX_NODE_POWER_LIMIT },
+		{ &dev_attr_npm_status, AMDGPU_NPM_CAP_NPM_STATUS },
+	};
+	u64 cap = amdgpu_dpm_get_npm_cap(adev);
+	int i;
+
+	if (!cap)
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(files); i++) {
+		unsigned int bits = (cap >> (files[i].field * 2)) & 0x3;
+		struct device_attribute *attr = files[i].attr;
+
+		if (!bits)
+			continue;
+
+		attr->attr.mode = ((bits & BIT(1)) && attr->store) ? 0644 : 0444;
+		sysfs_add_file_to_group(&adev->dev->kobj, &attr->attr,
+					amdgpu_board_attr_group.name);
+	}
+}
+
 int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 {
 	enum amdgpu_sriov_vf_mode mode;
@@ -4797,21 +4827,7 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 					    &amdgpu_board_attr_group);
 		if (ret)
 			goto err_out1;
-		if (amdgpu_pm_get_sensor_generic(adev, AMDGPU_PP_SENSOR_MAXNODEPOWERLIMIT,
-						 (void *)&tmp) != -EOPNOTSUPP) {
-			sysfs_add_file_to_group(&adev->dev->kobj,
-						&dev_attr_cur_node_power_limit.attr,
-						amdgpu_board_attr_group.name);
-			sysfs_add_file_to_group(&adev->dev->kobj, &dev_attr_node_power.attr,
-						amdgpu_board_attr_group.name);
-			sysfs_add_file_to_group(&adev->dev->kobj, &dev_attr_global_ppt_resid.attr,
-						amdgpu_board_attr_group.name);
-			sysfs_add_file_to_group(&adev->dev->kobj,
-						&dev_attr_max_node_power_limit.attr,
-						amdgpu_board_attr_group.name);
-			sysfs_add_file_to_group(&adev->dev->kobj, &dev_attr_npm_status.attr,
-						amdgpu_board_attr_group.name);
-		}
+		amdgpu_pm_npm_sysfs_init(adev);
 		if (amdgpu_pm_get_sensor_generic(adev, AMDGPU_PP_SENSOR_UBB_POWER_LIMIT,
 						 (void *)&tmp) != -EOPNOTSUPP) {
 			sysfs_add_file_to_group(&adev->dev->kobj,
diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
index 8e8a052e1d5317..ed1e7385243db8 100644
--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
@@ -614,5 +614,6 @@ int amdgpu_dpm_reset_vcn(struct amdgpu_device *adev, uint32_t inst_mask);
 bool amdgpu_dpm_reset_vcn_is_supported(struct amdgpu_device *adev);
 bool amdgpu_dpm_is_temp_metrics_supported(struct amdgpu_device *adev,
 					  enum smu_temp_metric_type type);
+u64 amdgpu_dpm_get_npm_cap(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c
index b763fdf43c603b..6dd81a17bc6b45 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c
@@ -3108,11 +3108,6 @@ static int kv_dpm_resume(struct amdgpu_ip_block *ip_block)
 	return ret;
 }
 
-static bool kv_dpm_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return true;
-}
-
 static int kv_dpm_set_interrupt_state(struct amdgpu_device *adev,
 				      struct amdgpu_irq_src *src,
 				      unsigned type,
@@ -3317,7 +3312,6 @@ static const struct amd_ip_funcs kv_dpm_ip_funcs = {
 	.hw_fini = kv_dpm_hw_fini,
 	.suspend = kv_dpm_suspend,
 	.resume = kv_dpm_resume,
-	.is_idle = kv_dpm_is_idle,
 	.set_clockgating_state = kv_dpm_set_clockgating_state,
 	.set_powergating_state = kv_dpm_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
index 012227c7060017..6cc414a8a4dee7 100644
--- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
+++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c
@@ -7928,12 +7928,6 @@ static int si_dpm_resume(struct amdgpu_ip_block *ip_block)
 	return ret;
 }
 
-static bool si_dpm_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	/* XXX */
-	return true;
-}
-
 static int si_dpm_wait_for_idle(struct amdgpu_ip_block *ip_block)
 {
 	/* XXX */
@@ -8134,7 +8128,6 @@ static const struct amd_ip_funcs si_dpm_ip_funcs = {
 	.hw_fini = si_dpm_hw_fini,
 	.suspend = si_dpm_suspend,
 	.resume = si_dpm_resume,
-	.is_idle = si_dpm_is_idle,
 	.wait_for_idle = si_dpm_wait_for_idle,
 	.set_clockgating_state = si_dpm_set_clockgating_state,
 	.set_powergating_state = si_dpm_set_powergating_state,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
index 7c70e228a5bae9..239c64b8edca8f 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
@@ -241,11 +241,6 @@ static void pp_late_fini(struct amdgpu_ip_block *ip_block)
 }
 
 
-static bool pp_is_idle(struct amdgpu_ip_block *ip_block)
-{
-	return false;
-}
-
 static int pp_set_powergating_state(struct amdgpu_ip_block *ip_block,
 				    enum amd_powergating_state state)
 {
@@ -286,7 +281,6 @@ static const struct amd_ip_funcs pp_ip_funcs = {
 	.late_fini = pp_late_fini,
 	.suspend = pp_suspend,
 	.resume = pp_resume,
-	.is_idle = pp_is_idle,
 	.set_clockgating_state = pp_set_clockgating_state,
 	.set_powergating_state = pp_set_powergating_state,
 };
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c
index 4b796d60b03d4a..3217ad3c1d17e3 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c
@@ -1025,6 +1025,9 @@ int atomctrl_get_smc_sclk_range_table(struct pp_hwmgr *hwmgr, struct pp_atom_ctr
 	if (!psmu_info)
 		return -EINVAL;
 
+	if (psmu_info->ucSclkEntryNum > MAX_SCLK_RANGE)
+		return -EINVAL;
+
 	for (i = 0; i < psmu_info->ucSclkEntryNum; i++) {
 		table->entry[i].ucVco_setting = psmu_info->asSclkFcwRangeEntry[i].ucVco_setting;
 		table->entry[i].ucPostdiv = psmu_info->asSclkFcwRangeEntry[i].ucPostdiv;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index cca32f74abf020..a65aa21298455c 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -69,6 +69,7 @@ static int smu_handle_task(struct smu_context *smu,
 			   enum amd_dpm_forced_level level,
 			   enum amd_pp_task task_id);
 static int smu_reset(struct smu_context *smu);
+static int smu_set_ac_dc(struct smu_context *smu);
 static int smu_set_fan_speed_pwm(void *handle, u32 speed);
 static int smu_set_fan_control_mode(void *handle, u32 value);
 static int smu_set_ppt_limit(void *handle, uint32_t limit_type, uint32_t limit);
@@ -641,7 +642,8 @@ bool is_support_cclk_dpm(struct amdgpu_device *adev)
 }
 
 int amdgpu_smu_ras_send_msg(struct amdgpu_device *adev, enum smu_message_type msg,
-			    uint32_t param, uint32_t *read_arg)
+			const uint32_t *params, size_t num_params,
+			uint32_t *read_args, size_t num_read_args)
 {
 	struct smu_context *smu = adev->powerplay.pp_handle;
 	int ret = -EOPNOTSUPP;
@@ -649,8 +651,19 @@ int amdgpu_smu_ras_send_msg(struct amdgpu_device *adev, enum smu_message_type ms
 	if (!smu)
 		return ret;
 
-	if (smu->ppt_funcs && smu->ppt_funcs->ras_send_msg)
-		ret = smu->ppt_funcs->ras_send_msg(smu, msg, param, read_arg);
+	if (smu->ppt_funcs && smu->ppt_funcs->ras_send_msg) {
+		if (num_params && !params)
+			return -EINVAL;
+
+		if (num_read_args && !read_args)
+			return -EINVAL;
+
+		if (num_params > SMU_MSG_MAX_ARGS || num_read_args > SMU_MSG_MAX_ARGS)
+			return -EINVAL;
+
+		ret = smu->ppt_funcs->ras_send_msg(smu, msg,
+				params, num_params, read_args, num_read_args);
+	}
 
 	return ret;
 }
@@ -960,7 +973,7 @@ static int smu_late_init(struct amdgpu_ip_block *ip_block)
 	 * is unnecessary.
 	 */
 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
-	smu_set_ac_dc(smu, false);
+	smu_set_ac_dc(smu);
 
 	if ((amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 1)) ||
 	    (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 3)))
@@ -2774,7 +2787,7 @@ static int smu_set_watermarks_for_clock_ranges(void *handle,
 	return smu_set_watermarks_table(smu, clock_ranges);
 }
 
-int smu_set_ac_dc(struct smu_context *smu, bool restore_ppt_policy)
+static int smu_set_ac_dc(struct smu_context *smu)
 {
 	int ret = 0;
 
@@ -2794,12 +2807,19 @@ int smu_set_ac_dc(struct smu_context *smu, bool restore_ppt_policy)
 		}
 	}
 
-	if (restore_ppt_policy)
-		smu_restore_ppt_limits(smu, true);
-
 	return 0;
 }
 
+static void smu_notify_ac_dc(void *handle)
+{
+	struct smu_context *smu = handle;
+
+	if (smu_set_ac_dc(smu))
+		return;
+
+	smu_restore_ppt_limits(smu, true);
+}
+
 const struct amd_ip_funcs smu_ip_funcs = {
 	.name = "smu",
 	.early_init = smu_early_init,
@@ -2811,7 +2831,6 @@ const struct amd_ip_funcs smu_ip_funcs = {
 	.late_fini = smu_late_fini,
 	.suspend = smu_suspend,
 	.resume = smu_resume,
-	.is_idle = NULL,
 	.wait_for_idle = NULL,
 	.soft_reset = NULL,
 	.set_clockgating_state = smu_set_clockgating_state,
@@ -3966,6 +3985,16 @@ static bool smu_temp_metrics_is_supported(void *handle, enum smu_temp_metric_typ
 	return ret;
 }
 
+static u64 smu_get_npm_cap(void *handle)
+{
+	struct smu_context *smu = handle;
+
+	if (!smu->pm_enabled || !smu->ppt_funcs || !smu->ppt_funcs->get_npm_cap)
+		return 0;
+
+	return smu->ppt_funcs->get_npm_cap(smu);
+}
+
 static ssize_t smu_sys_get_xcp_metrics(void *handle, int xcp_id, void *table)
 {
 	struct smu_context *smu = handle;
@@ -4000,6 +4029,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = {
 	.set_pp_table            = smu_sys_set_pp_table,
 	.switch_power_profile    = smu_switch_power_profile,
 	.pause_power_profile     = smu_pause_power_profile,
+	.notify_ac_dc            = smu_notify_ac_dc,
 	/* export to amdgpu */
 	.dispatch_tasks          = smu_handle_dpm_task,
 	.load_firmware           = smu_load_microcode,
@@ -4039,6 +4069,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = {
 	.get_xcp_metrics                  = smu_sys_get_xcp_metrics,
 	.get_temp_metrics             = smu_sys_get_temp_metrics,
 	.temp_metrics_is_supported      = smu_temp_metrics_is_supported,
+	.get_npm_cap                  = smu_get_npm_cap,
 };
 
 int smu_wait_for_event(struct smu_context *smu, enum smu_event_type event,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index 3ec65630ab99c0..1e5fc6ff76c860 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -486,7 +486,6 @@ struct smu_power_gate {
 
 struct smu_power_context {
 	void *power_context;
-	uint32_t power_context_size;
 	struct smu_power_gate power_gate;
 };
 
@@ -1620,15 +1619,28 @@ struct pptable_funcs {
 	 */
 	ssize_t (*get_xcp_metrics)(struct smu_context *smu, int xcp_id,
 				   void *table);
+	/**
+	 * @get_npm_cap: Return NPM sysfs capability bitmap.
+	 *
+	 * Two bits per field (R then W). Both 0 means the field is
+	 * unsupported.
+	 */
+	u64 (*get_npm_cap)(struct smu_context *smu);
 	/**
 	 * @ras_send_msg: Send a message with a parameter from Ras
 	 * &msg: Type of message.
 	 * &param: Message parameter.
 	 * &read_arg: SMU response (optional).
 	 */
-	int (*ras_send_msg)(struct smu_context *smu,
-			    enum smu_message_type msg, uint32_t param, uint32_t *read_arg);
+	int (*ras_send_msg)(struct smu_context *smu, enum smu_message_type msg,
+				const uint32_t *params, size_t num_params,
+				uint32_t *read_args, size_t num_read_args);
 
+	/**
+	 * @get_ras_smu_drv: Get RAS smu driver interface
+	 * Return: ras_smu_drv *
+	 */
+	int (*get_ras_smu_drv)(struct smu_context *smu, const struct ras_smu_drv **ras_smu_drv);
 	/**
 	 * @set_power_dep: Create or destroy a power dependency link
 	 * from an integrated xHCI controller to the GPU so that the GPU is
@@ -1952,8 +1964,6 @@ int smu_set_soft_freq_range(struct smu_context *smu, enum pp_clock_type clk_type
 
 int smu_set_gfx_power_up_by_imu(struct smu_context *smu);
 
-int smu_set_ac_dc(struct smu_context *smu, bool restore_ppt_policy);
-
 int smu_set_xgmi_plpd_mode(struct smu_context *smu,
 			   enum pp_xgmi_plpd_mode mode);
 
@@ -1985,7 +1995,8 @@ ssize_t smu_get_pm_policy_info(struct smu_context *smu,
 			       enum pp_pm_policy p_type, char *sysbuf);
 
 int amdgpu_smu_ras_send_msg(struct amdgpu_device *adev, enum smu_message_type msg,
-			    uint32_t param, uint32_t *readarg);
+			const uint32_t *params, size_t num_params,
+			uint32_t *read_args, size_t num_read_args);
 int amdgpu_smu_ras_feature_is_enabled(struct amdgpu_device *adev,
 						enum smu_feature_mask mask);
 #endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu15_driver_if_v15_0_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu15_driver_if_v15_0_0.h
index a5d3e8d88860bc..94932d8f3e6193 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu15_driver_if_v15_0_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu15_driver_if_v15_0_0.h
@@ -182,52 +182,6 @@ typedef struct {
 	uint32_t Spare9[8];
 } DpmClocks_t_v15_0_5;
 
-typedef struct {
-  uint16_t CoreFrequency[16];          //Target core frequency [MHz]
-  uint16_t CorePower[16];              //CAC calculated core power [mW]
-  uint16_t CoreTemperature[16];        //TSEN measured core temperature [centi-C]
-  uint16_t GfxTemperature;             //TSEN measured GFX temperature [centi-C]
-  uint16_t SocTemperature;             //TSEN measured SOC temperature [centi-C]
-  uint16_t StapmOpnLimit;              //Maximum IRM defined STAPM power limit [mW]
-  uint16_t StapmCurrentLimit;          //Time filtered STAPM power limit [mW]
-  uint16_t InfrastructureCpuMaxFreq;   //CCLK frequency limit enforced on classic cores [MHz]
-  uint16_t InfrastructureGfxMaxFreq;   //GFXCLK frequency limit enforced on GFX [MHz]
-  uint16_t SkinTemp;                   //Maximum skin temperature reported by APU and HS2 chassis sensors [centi-C]
-  uint16_t GfxclkFrequency;            //Time filtered target GFXCLK frequency [MHz]
-  uint16_t FclkFrequency;              //Time filtered target FCLK frequency [MHz]
-  uint16_t GfxActivity;                //Time filtered GFX busy % [0-100]
-  uint16_t SocclkFrequency;            //Time filtered target SOCCLK frequency [MHz]
-  uint16_t VclkFrequency;              //Time filtered target VCLK frequency [MHz]
-  uint16_t VcnActivity;                //Time filtered VCN busy % [0-100]
-  uint16_t VpeclkFrequency;            //Time filtered target VPECLK frequency [MHz]
-  uint16_t NpuclkFrequency;            //Time filtered target NPUCLK frequency [MHz]
-  uint16_t NpuBusy[8];                 //Time filtered NPU per-column busy % [0-100]
-  uint16_t DRAMReads;                  //Time filtered DRAM read bandwidth [MB/sec]
-  uint16_t DRAMWrites;                 //Time filtered DRAM write bandwidth [MB/sec]
-  uint16_t CoreC0Residency[16];        //Time filtered per-core C0 residency % [0-100]
-  uint16_t NpuPower;                   //Time filtered NPU power [mW]
-  uint32_t ApuPower;                   //Time filtered APU power [mW]
-  uint32_t GfxPower;                   //Time filtered GFX power [mW]
-  uint32_t dGpuPower;                  //Time filtered dGPU power [mW]
-  uint32_t SocketPower;                //Time filtered power used for PPT/STAPM [APU+dGPU] [mW]
-  uint32_t AllCorePower;               //Time filtered sum of core power across all cores in the socket [mW]
-  uint32_t FilterAlphaValue;           //Metrics table alpha filter time constant [us]
-  uint32_t MetricsCounter;             //Counter that is incremented on every metrics table update [PM_TIMER cycles]
-  uint16_t MemclkFrequency;            //Time filtered target MEMCLK frequency [MHz]
-  uint16_t AieclkFrequency;            //Time filtered target AIECLK frequency [MHz]
-  uint16_t NpuReads;                   //Time filtered NPU read bandwidth [MB/sec]
-  uint16_t NpuWrites;                  //Time filtered NPU write bandwidth [MB/sec]
-  uint32_t ThrottleResidency_PROCHOT;  //Counter that is incremented on every metrics table update when PROCHOT was engaged [PM_TIMER cycles]
-  uint32_t ThrottleResidency_SPL;      //Counter that is incremented on every metrics table update when SPL was engaged [PM_TIMER cycles]
-  uint32_t ThrottleResidency_FPPT;     //Counter that is incremented on every metrics table update when fast PPT was engaged [PM_TIMER cycles]
-  uint32_t ThrottleResidency_SPPT;     //Counter that is incremented on every metrics table update when slow PPT was engaged [PM_TIMER cycles]
-  uint32_t ThrottleResidency_THM_VDD;  //Counter that is incremented on every metrics table update when VDD thermal throttling was engaged [PM_TIMER cycles]
-  uint32_t ThrottleResidency_THM_SOC;  //Counter that is incremented on every metrics table update when SOC thermal throttling was engaged [PM_TIMER cycles]
-  uint16_t Psys;                       //Time filtered Psys power [mW]
-  uint16_t spare1;
-  uint32_t spare[6];
-} SmuMetrics_t;
-
 //ISP tile definitions
 typedef enum {
   TILE_XTILE = 0,         //ONO0
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_ppsmc.h
index d09b6ae9827efb..ac7e617e64ad22 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_ppsmc.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_12_ppsmc.h
@@ -105,6 +105,7 @@
 #define PPSMC_MSG_UpdatePccWaitDecMaxStr            0x4C
 #define PPSMC_MSG_ResetSDMA                         0x4D
 #define PPSMC_MSG_GetRasTableVersion                0x4E
+#define PPSMC_MSG_GetRmaStatus                      0x4F
 #define PPSMC_MSG_GetBadPageCount                   0x50
 #define PPSMC_MSG_GetBadPageMcaAddress              0x51
 #define PPSMC_MSG_SetTimestamp                      0x53
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
index d6675b4ee9be47..2631e12b5e5972 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
@@ -285,6 +285,7 @@
 	__SMU_DUMMY_MAP(GetStaticMetricsTable), \
 	__SMU_DUMMY_MAP(GetSystemMetricsTable), \
 	__SMU_DUMMY_MAP(GetRASTableVersion), \
+	__SMU_DUMMY_MAP(GetRmaStatus),    \
 	__SMU_DUMMY_MAP(GetBadPageCount), \
 	__SMU_DUMMY_MAP(GetBadPageMcaAddr), \
 	__SMU_DUMMY_MAP(SetTimestamp), \
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
index ec96dc77548597..0c97bee2c4bbc7 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v15_0.h
@@ -113,8 +113,6 @@ int smu_v15_0_init_microcode(struct smu_context *smu);
 
 void smu_v15_0_fini_microcode(struct smu_context *smu);
 
-int smu_v15_0_load_microcode(struct smu_context *smu);
-
 int smu_v15_0_init_smc_tables(struct smu_context *smu);
 
 int smu_v15_0_fini_smc_tables(struct smu_context *smu);
@@ -183,10 +181,6 @@ int smu_v15_0_set_performance_level(struct smu_context *smu,
 int smu_v15_0_set_power_source(struct smu_context *smu,
 			       enum smu_power_src_type power_src);
 
-int smu_v15_0_set_single_dpm_table(struct smu_context *smu,
-				   enum smu_clk_type clk_type,
-				   struct smu_dpm_table *single_dpm_table);
-
 int smu_v15_0_gfx_ulv_control(struct smu_context *smu,
 			      bool enablement);
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
index f0b2e7da67b1b0..09e20d5ee0446d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c
@@ -345,7 +345,6 @@ int smu_v11_0_init_power(struct smu_context *smu)
 	smu_power->power_context = kzalloc(size, GFP_KERNEL);
 	if (!smu_power->power_context)
 		return -ENOMEM;
-	smu_power->power_context_size = size;
 
 	return 0;
 }
@@ -356,7 +355,6 @@ int smu_v11_0_fini_power(struct smu_context *smu)
 
 	kfree(smu_power->power_context);
 	smu_power->power_context = NULL;
-	smu_power->power_context_size = 0;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index c4b2fe60da62bc..b250c5e41e4531 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -427,13 +427,12 @@ int smu_v13_0_init_power(struct smu_context *smu)
 {
 	struct smu_power_context *smu_power = &smu->smu_power;
 
-	if (smu_power->power_context || smu_power->power_context_size != 0)
+	if (smu_power->power_context)
 		return -EINVAL;
 
 	smu_power->power_context = kzalloc_obj(struct smu_13_0_power_context);
 	if (!smu_power->power_context)
 		return -ENOMEM;
-	smu_power->power_context_size = sizeof(struct smu_13_0_power_context);
 
 	return 0;
 }
@@ -442,12 +441,11 @@ int smu_v13_0_fini_power(struct smu_context *smu)
 {
 	struct smu_power_context *smu_power = &smu->smu_power;
 
-	if (!smu_power->power_context || smu_power->power_context_size == 0)
+	if (!smu_power->power_context)
 		return -EINVAL;
 
 	kfree(smu_power->power_context);
 	smu_power->power_context = NULL;
-	smu_power->power_context_size = 0;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index a29e76b03476bf..6beafabf13065d 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -3077,7 +3077,7 @@ static int smu_v13_0_0_set_ppt_limit(struct smu_context *smu,
 				return ret;
 			}
 		}
-			return smu_v13_0_set_ppt_limit(smu, limit_type, limit);
+		return smu_v13_0_set_ppt_limit(smu, limit_type, limit);
 	} else if (smu->od_enabled) {
 		ret = smu_v13_0_set_ppt_limit(smu, limit_type, msg_limit);
 		if (ret)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
index 0033453f27e5a7..1691a0be4f7dcf 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c
@@ -25,6 +25,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_smu.h"
 #include "smu_v13_0_12_pmfw.h"
 #include "smu_v13_0_6_ppt.h"
@@ -152,6 +153,7 @@ const struct cmn2asic_msg_mapping smu_v13_0_12_message_map[SMU_MSG_MAX_COUNT] =
 	MSG_MAP(GetStaticMetricsTable,               PPSMC_MSG_GetStaticMetricsTable,           1),
 	MSG_MAP(GetSystemMetricsTable,               PPSMC_MSG_GetSystemMetricsTable,           1),
 	MSG_MAP(GetRASTableVersion,                  PPSMC_MSG_GetRasTableVersion,              0),
+	MSG_MAP(GetRmaStatus,                        PPSMC_MSG_GetRmaStatus,                    0),
 	MSG_MAP(GetBadPageCount,                     PPSMC_MSG_GetBadPageCount,                 0),
 	MSG_MAP(GetBadPageMcaAddr,                   PPSMC_MSG_GetBadPageMcaAddress,            0),
 	MSG_MAP(SetTimestamp,                        PPSMC_MSG_SetTimestamp,                    0),
@@ -683,6 +685,9 @@ int smu_v13_0_12_get_npm_data(struct smu_context *smu,
 	case AMDGPU_PP_SENSOR_NODEPOWER:
 		*value = SMUQ10_ROUND(metrics->NodePower);
 		break;
+	case AMDGPU_PP_SENSOR_NPMSTATUS:
+		*value = !!SMUQ10_ROUND(metrics->NodePower);
+		break;
 	case AMDGPU_PP_SENSOR_GPPTRESIDENCY:
 		*value = SMUQ10_ROUND(metrics->GlobalPPTResidencyAcc);
 		break;
@@ -693,6 +698,25 @@ int smu_v13_0_12_get_npm_data(struct smu_context *smu,
 	return ret;
 }
 
+u64 smu_v13_0_12_get_npm_cap(struct smu_context *smu)
+{
+	struct PPTable_t *pptable =
+		(struct PPTable_t *)smu->smu_table.driver_pptable;
+	u64 cap = 0;
+
+	if (!smu_v13_0_6_cap_supported(smu, SMU_CAP(NPM_METRICS)) ||
+	    !pptable->MaxNodePowerLimit)
+		return 0;
+
+	cap |= AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_CUR_NODE_POWER_LIMIT);
+	cap |= AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_NODE_POWER);
+	cap |= AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_GLOBAL_PPT_RESID);
+	cap |= AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_MAX_NODE_POWER_LIMIT);
+	cap |= AMDGPU_NPM_CAP_R(AMDGPU_NPM_CAP_NPM_STATUS);
+
+	return cap;
+}
+
 static ssize_t smu_v13_0_12_get_temp_metrics(struct smu_context *smu,
 					     enum smu_temp_metric_type type, void *table)
 {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index 84c02e6d9673bb..95e288327e455f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -25,6 +25,7 @@
 
 #include <linux/firmware.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 #include "amdgpu_smu.h"
 #include "atomfirmware.h"
 #include "amdgpu_atomfirmware.h"
@@ -1677,6 +1678,7 @@ static int smu_v13_0_6_read_sensor(struct smu_context *smu,
 	case AMDGPU_PP_SENSOR_NODEPOWER:
 	case AMDGPU_PP_SENSOR_GPPTRESIDENCY:
 	case AMDGPU_PP_SENSOR_MAXNODEPOWERLIMIT:
+	case AMDGPU_PP_SENSOR_NPMSTATUS:
 		ret = smu_v13_0_12_get_npm_data(smu, sensor, (uint32_t *)data);
 		if (ret)
 			return ret;
@@ -3215,7 +3217,9 @@ static int smu_v13_0_6_reset_vcn(struct smu_context *smu, uint32_t inst_mask)
 	return ret;
 }
 
-static int smu_v13_0_6_ras_send_msg(struct smu_context *smu, enum smu_message_type msg, uint32_t param, uint32_t *read_arg)
+static int smu_v13_0_6_ras_send_msg(struct smu_context *smu, enum smu_message_type msg,
+			const uint32_t *params, size_t num_params,
+			uint32_t *read_args, size_t num_read_args)
 {
 	struct amdgpu_device *adev = smu->adev;
 	int ret;
@@ -3236,7 +3240,8 @@ static int smu_v13_0_6_ras_send_msg(struct smu_context *smu, enum smu_message_ty
 	case SMU_MSG_GetTimestamp:
 	case SMU_MSG_GetBadPageIpid:
 	case SMU_MSG_EraseRasTable:
-		ret = smu_cmn_send_smc_msg_with_param(smu, msg, param, read_arg);
+		ret = smu_cmn_send_smc_msg_with_params(smu, msg,
+				params, num_params, read_args, num_read_args);
 		break;
 	default:
 		ret = -EPERM;
@@ -3308,6 +3313,7 @@ static const struct pptable_funcs smu_v13_0_6_ppt_funcs = {
 	.get_gpu_metrics = smu_v13_0_6_get_gpu_metrics,
 	.get_pm_metrics = smu_v13_0_6_get_pm_metrics,
 	.get_xcp_metrics = smu_v13_0_6_get_xcp_metrics,
+	.get_npm_cap = smu_v13_0_12_get_npm_cap,
 	.get_thermal_temperature_range = smu_v13_0_6_get_thermal_temperature_range,
 	.mode1_reset_is_support = smu_v13_0_6_is_mode1_reset_supported,
 	.mode1_reset = smu_v13_0_6_mode1_reset,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
index 6554780afff77c..1b6926625cc379 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.h
@@ -111,6 +111,7 @@ void smu_v13_0_12_tables_fini(struct smu_context *smu);
 int smu_v13_0_12_get_npm_data(struct smu_context *smu,
 			      enum amd_pp_sensors sensor,
 			      uint32_t *value);
+u64 smu_v13_0_12_get_npm_cap(struct smu_context *smu);
 int smu_v13_0_12_get_system_power(struct smu_context *smu,
 				  enum amd_pp_sensors sensor,
 				  uint32_t *value);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
index 80dac277d1ae87..5e890b56fd37c7 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0.c
@@ -393,13 +393,12 @@ int smu_v14_0_init_power(struct smu_context *smu)
 {
 	struct smu_power_context *smu_power = &smu->smu_power;
 
-	if (smu_power->power_context || smu_power->power_context_size != 0)
+	if (smu_power->power_context)
 		return -EINVAL;
 
-	smu_power->power_context = kzalloc_obj(struct smu_14_0_dpm_context);
+	smu_power->power_context = kzalloc_obj(struct smu_14_0_power_context);
 	if (!smu_power->power_context)
 		return -ENOMEM;
-	smu_power->power_context_size = sizeof(struct smu_14_0_dpm_context);
 
 	return 0;
 }
@@ -408,12 +407,11 @@ int smu_v14_0_fini_power(struct smu_context *smu)
 {
 	struct smu_power_context *smu_power = &smu->smu_power;
 
-	if (!smu_power->power_context || smu_power->power_context_size == 0)
+	if (!smu_power->power_context)
 		return -EINVAL;
 
 	kfree(smu_power->power_context);
 	smu_power->power_context = NULL;
-	smu_power->power_context_size = 0;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
index 4c0236196de4fb..4be864e4f0e6cd 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
@@ -106,44 +106,6 @@ void smu_v15_0_fini_microcode(struct smu_context *smu)
 	adev->pm.fw_version = 0;
 }
 
-int smu_v15_0_load_microcode(struct smu_context *smu)
-{
-	struct amdgpu_device *adev = smu->adev;
-	const uint32_t *src;
-	const struct smc_firmware_header_v1_0 *hdr;
-	uint32_t addr_start = MP1_SRAM;
-	uint32_t i;
-	uint32_t smc_fw_size;
-	uint32_t mp1_fw_flags;
-
-	hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
-	src = (const uint32_t *)(adev->pm.fw->data +
-				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
-	smc_fw_size = hdr->header.ucode_size_bytes;
-
-	for (i = 1; i < smc_fw_size/4 - 1; i++) {
-		WREG32_PCIE(addr_start, src[i]);
-		addr_start += 4;
-	}
-
-
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (smu->is_apu)
-			mp1_fw_flags = RREG32_PCIE(MP1_Public |
-						   (smnMP1_FIRMWARE_FLAGS & 0xffffffff));
-
-		if ((mp1_fw_flags & MP1_CRU1_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >>
-		    MP1_CRU1_MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT)
-			break;
-		udelay(1);
-	}
-
-	if (i == adev->usec_timeout)
-		return -ETIME;
-
-	return 0;
-}
-
 int smu_v15_0_init_pptable_microcode(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -373,13 +335,12 @@ int smu_v15_0_init_power(struct smu_context *smu)
 {
 	struct smu_power_context *smu_power = &smu->smu_power;
 
-	if (smu_power->power_context || smu_power->power_context_size != 0)
+	if (smu_power->power_context)
 		return -EINVAL;
 
-	smu_power->power_context = kzalloc_obj(struct smu_15_0_dpm_context);
+	smu_power->power_context = kzalloc_obj(struct smu_15_0_power_context);
 	if (!smu_power->power_context)
 		return -ENOMEM;
-	smu_power->power_context_size = sizeof(struct smu_15_0_dpm_context);
 
 	return 0;
 }
@@ -388,12 +349,11 @@ int smu_v15_0_fini_power(struct smu_context *smu)
 {
 	struct smu_power_context *smu_power = &smu->smu_power;
 
-	if (!smu_power->power_context || smu_power->power_context_size == 0)
+	if (!smu_power->power_context)
 		return -EINVAL;
 
 	kfree(smu_power->power_context);
 	smu_power->power_context = NULL;
-	smu_power->power_context_size = 0;
 
 	return 0;
 }
@@ -1161,134 +1121,6 @@ int smu_v15_0_set_power_source(struct smu_context *smu,
 					       NULL);
 }
 
-static int smu_v15_0_get_dpm_freq_by_index(struct smu_context *smu,
-					   enum smu_clk_type clk_type,
-					   uint16_t level,
-					   uint32_t *value)
-{
-	int ret = 0, clk_id = 0;
-	uint32_t param;
-
-	if (!value)
-		return -EINVAL;
-
-	if (!smu_cmn_clk_dpm_is_enabled(smu, clk_type))
-		return 0;
-
-	clk_id = smu_cmn_to_asic_specific_index(smu,
-						CMN2ASIC_MAPPING_CLK,
-						clk_type);
-	if (clk_id < 0)
-		return clk_id;
-
-	param = (uint32_t)(((clk_id & 0xffff) << 16) | (level & 0xffff));
-
-	ret = smu_cmn_send_smc_msg_with_param(smu,
-					      SMU_MSG_GetDpmFreqByIndex,
-					      param,
-					      value);
-	if (ret)
-		return ret;
-
-	*value = *value & 0x7fffffff;
-
-	return ret;
-}
-
-static int smu_v15_0_get_dpm_level_count(struct smu_context *smu,
-					 enum smu_clk_type clk_type,
-					 uint32_t *value)
-{
-	int ret;
-
-	ret = smu_v15_0_get_dpm_freq_by_index(smu, clk_type, 0xff, value);
-
-	return ret;
-}
-
-static int smu_v15_0_get_fine_grained_status(struct smu_context *smu,
-					     enum smu_clk_type clk_type,
-					     bool *is_fine_grained_dpm)
-{
-	int ret = 0, clk_id = 0;
-	uint32_t param;
-	uint32_t value;
-
-	if (!is_fine_grained_dpm)
-		return -EINVAL;
-
-	if (!smu_cmn_clk_dpm_is_enabled(smu, clk_type))
-		return 0;
-
-	clk_id = smu_cmn_to_asic_specific_index(smu,
-						CMN2ASIC_MAPPING_CLK,
-						clk_type);
-	if (clk_id < 0)
-		return clk_id;
-
-	param = (uint32_t)(((clk_id & 0xffff) << 16) | 0xff);
-
-	ret = smu_cmn_send_smc_msg_with_param(smu,
-					      SMU_MSG_GetDpmFreqByIndex,
-					      param,
-					      &value);
-	if (ret)
-		return ret;
-
-	/*
-	 * BIT31:  1 - Fine grained DPM, 0 - Dicrete DPM
-	 * now, we un-support it
-	 */
-	*is_fine_grained_dpm = value & 0x80000000;
-
-	return 0;
-}
-
-int smu_v15_0_set_single_dpm_table(struct smu_context *smu,
-				   enum smu_clk_type clk_type,
-				   struct smu_dpm_table *single_dpm_table)
-{
-	int ret = 0;
-	uint32_t clk;
-	bool is_fine_grained;
-	int i;
-
-	ret = smu_v15_0_get_dpm_level_count(smu,
-					    clk_type,
-					    &single_dpm_table->count);
-	if (ret) {
-		dev_err(smu->adev->dev, "[%s] failed to get dpm levels!\n", __func__);
-		return ret;
-	}
-
-	ret = smu_v15_0_get_fine_grained_status(smu,
-						clk_type,
-						&is_fine_grained);
-	if (ret) {
-		dev_err(smu->adev->dev, "[%s] failed to get fine grained status!\n", __func__);
-		return ret;
-	}
-
-	if (is_fine_grained)
-		single_dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED;
-
-	for (i = 0; i < single_dpm_table->count; i++) {
-		ret = smu_v15_0_get_dpm_freq_by_index(smu,
-						      clk_type,
-						      i,
-						      &clk);
-		if (ret) {
-			dev_err(smu->adev->dev, "[%s] failed to get dpm freq by index!\n", __func__);
-			return ret;
-		}
-
-		single_dpm_table->dpm_levels[i].value = clk;
-		single_dpm_table->dpm_levels[i].enabled = true;
-	}
-
-	return 0;
-}
-
 int smu_v15_0_set_vcn_enable(struct smu_context *smu,
 			      bool enable,
 			      int inst)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
index cbfeb9736e6c40..6e875787434e29 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
@@ -203,8 +203,6 @@ static int smu_v15_0_0_init_smc_tables(struct smu_context *smu)
 		PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
 	SMU_TABLE_INIT(tables, SMU_TABLE_DPMCLOCKS, sizeof(DpmClocks_t_v15_0_5),
 		PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
-	SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, sizeof(SmuMetrics_t),
-		PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
 
 	smu_table->metrics_table = kzalloc_obj(SMU_15_0_0_MetricsInfo_t);
 	if (!smu_table->metrics_table)
@@ -348,36 +346,6 @@ static int smu_v15_0_0_set_default_dpm_tables(struct smu_context *smu)
 				    smu_table->clocks_table, false);
 }
 
-static int smu_v15_0_0_get_gpu_metrics_table(struct smu_context *smu,
-							void *metrics_table,
-							bool bypass_cache)
-{
-	struct smu_table_context *smu_table = &smu->smu_table;
-	uint32_t table_size =
-			smu_table->tables[SMU_TABLE_SMU_METRICS].size;
-	int ret;
-
-	if (bypass_cache ||
-		!smu_table->metrics_time ||
-		time_after(jiffies, smu_table->metrics_time + msecs_to_jiffies(1))) {
-		ret = smu_v15_0_0_update_table(smu,
-						SMU_TABLE_SMU_METRICS,
-						0,
-						smu_table->metrics_table,
-						false);
-		if (ret) {
-			dev_info(smu->adev->dev, "Failed to export SMU15_0_0 metrics table!\n");
-			return ret;
-		}
-		smu_table->metrics_time = jiffies;
-	}
-
-	if (metrics_table)
-		memcpy(metrics_table, smu_table->metrics_table, table_size);
-
-	return 0;
-}
-
 /*
  * Fetch a fresh metrics sample into the inactive buffer.
  * Returns 0 if a new sample was copied, 1 if the cached sample is still
@@ -424,85 +392,76 @@ static int smu_v15_0_0_get_metrics_table(struct smu_context *smu,
 }
 
 /*
- * Accumulators monotonically increase and roll over at their type width.
- * Use the kernel wrapping_sub() API to compute the delta so the subtraction
- * wraps modulo 2^n (correct across a single rollover) without tripping any
- * wrap-around sanitizers.
+ * All accumulators below are Q10 fixed-point values in their native unit
+ * (Q number format: 10 fractional bits, i.e. the real value scaled by 2^10).
+ * The average over the sampling window is (curr - prev) / AccumulationCounter,
+ * still in Q10. We multiply by @scale first (to preserve the sub-unit
+ * fractional bits the scaled unit needs) and only then convert back to an
+ * integer by dropping the 10 fractional bits with a >>10 shift. wrapping_sub()
+ * handles a single accumulator rollover without tripping wrap sanitizers;
+ * typeof() keeps the modular subtraction at the field's real width (the
+ * *_ResidencyAcc fields are u32, the rest u64).
+ *
+ * @scale folds in the unit conversion the consumer expects:
+ *   1    -> native unit  (MHz, %)
+ *   1000 -> W->mW, GHz->MHz, GB/s->MB/s, V->mV
+ *   100  -> C->centi-C
+ */
+#define SMU_V15_IOD_AVG(field, scale)						\
+	((uint32_t)((div64_u64(wrapping_sub(typeof(c->field),			\
+					    c->field, p->field),		\
+			       counter) * (scale)) >> 10))
+#define SMU_V15_CORE_AVG(ccx, core, field, scale)				\
+	((uint32_t)((div64_u64(wrapping_sub(typeof(curr->CCX[ccx].field[core]),	\
+					    curr->CCX[ccx].field[core],		\
+					    prev->CCX[ccx].field[core]),	\
+			       counter) * (scale)) >> 10))
+
+/*
+ * Compute the windowed average of every sensor exposed through read_sensor and
+ * cache it in avg_metric[]. Units here follow the hwmon/sysfs sensor ABI
+ * (milli-C, mW, mV), which differs from the gpu_metrics scaling used in
+ * smu_v15_0_0_get_gpu_metrics(). SMU_V15_IOD_AVG() relies on the c, p and
+ * counter locals declared below.
  */
 static void smu_v15_0_0_compute_all_metrics(
 		uint32_t *avg_metric,
 		MetricsTable_t *prev,
 		MetricsTable_t *curr)
 {
-	uint64_t counter, val;
-	uint32_t mw;
 	MetricsTable_IOD_t *p = &prev->IOD;
 	MetricsTable_IOD_t *c = &curr->IOD;
+	uint64_t counter;
 
 	counter = wrapping_sub(u32, c->AccumulationCounter, p->AccumulationCounter);
 	if (!counter)
 		return;
 
-	/* Accumulator-based clock frequencies (fixed-point /1024) */
-	val = wrapping_sub(u64, c->GfxclkFreqEffAcc, p->GfxclkFreqEffAcc);
-	avg_metric[METRICS_AVERAGE_GFXCLK] = div_u64(div64_u64(val, counter), 1024);
-
-	val = wrapping_sub(u64, c->SocclkFreqEffAcc, p->SocclkFreqEffAcc);
-	avg_metric[METRICS_AVERAGE_SOCCLK] = div_u64(div64_u64(val, counter), 1024);
+	/* Effective clocks: accumulator already in MHz. */
+	avg_metric[METRICS_AVERAGE_GFXCLK] = SMU_V15_IOD_AVG(GfxclkFreqEffAcc, 1);
+	avg_metric[METRICS_AVERAGE_SOCCLK] = SMU_V15_IOD_AVG(SocclkFreqEffAcc, 1);
+	avg_metric[METRICS_AVERAGE_VCLK] = SMU_V15_IOD_AVG(VclkFreqEffAcc, 1);
+	avg_metric[METRICS_AVERAGE_UCLK] = SMU_V15_IOD_AVG(MemclkFreqEffAcc, 1);
+	avg_metric[METRICS_AVERAGE_FCLK] = SMU_V15_IOD_AVG(FclkFreqEffAcc, 1);
+	avg_metric[METRICS_AVERAGE_NPUCLK] = SMU_V15_IOD_AVG(NpuhclkFreqEffAcc, 1);
 
-	val = wrapping_sub(u64, c->VclkFreqEffAcc, p->VclkFreqEffAcc);
-	avg_metric[METRICS_AVERAGE_VCLK] = div_u64(div64_u64(val, counter), 1024);
+	/* Activity: accumulator already in %. */
+	avg_metric[METRICS_AVERAGE_GFXACTIVITY] = SMU_V15_IOD_AVG(GfxBusyAcc, 1);
+	avg_metric[METRICS_AVERAGE_VCNACTIVITY] = SMU_V15_IOD_AVG(VcnBusyAcc, 1);
 
-	val = wrapping_sub(u64, c->MemclkFreqEffAcc, p->MemclkFreqEffAcc);
-	avg_metric[METRICS_AVERAGE_UCLK] = div_u64(div64_u64(val, counter), 1024);
+	/* Power: accumulator in W -> mW (hwmon/debugfs). */
+	avg_metric[METRICS_AVERAGE_SOCKETPOWER] = SMU_V15_IOD_AVG(ApuPowerAcc, 1000);
+	avg_metric[METRICS_CURR_SOCKETPOWER] = SMU_V15_IOD_AVG(SystemPowerAcc, 1000);
 
-	val = wrapping_sub(u64, c->FclkFreqEffAcc, p->FclkFreqEffAcc);
-	avg_metric[METRICS_AVERAGE_FCLK] = div_u64(div64_u64(val, counter), 1024);
-
-	val = wrapping_sub(u64, c->NpuhclkFreqEffAcc, p->NpuhclkFreqEffAcc);
-	avg_metric[METRICS_AVERAGE_NPUCLK] = div_u64(div64_u64(val, counter), 1024);
-
-	/* Activity (fixed-point /1024) */
-	val = wrapping_sub(u64, c->GfxBusyAcc, p->GfxBusyAcc);
-	avg_metric[METRICS_AVERAGE_GFXACTIVITY] = div_u64(div64_u64(val, counter), 1024);
-
-	val = wrapping_sub(u64, c->VcnBusyAcc, p->VcnBusyAcc);
-	avg_metric[METRICS_AVERAGE_VCNACTIVITY] = div_u64(div64_u64(val, counter), 1024);
-
-	/*
-	 * Power: accumulator holds a 1024x fixed-point value in Watts.
-	 * Average it into milliwatts, which is the unit expected by
-	 * power sensor consumers (hwmon/debugfs).
-	 */
-	val = wrapping_sub(u64, c->ApuPowerAcc, p->ApuPowerAcc);
-	mw = div_u64(div64_u64(val, counter) * 1000, 1024);
-	avg_metric[METRICS_AVERAGE_SOCKETPOWER] = mw;
-
-	val = wrapping_sub(u64, c->SystemPowerAcc, p->SystemPowerAcc);
-	mw = div_u64(div64_u64(val, counter) * 1000, 1024);
-	avg_metric[METRICS_CURR_SOCKETPOWER] = mw;
-
-	/*
-	 * Temperature: accumulator holds a 1024x fixed-point value in
-	 * Celsius. Descale by 1024 and convert to millidegrees C as the
-	 * hwmon/sysfs consumers expect (temp*_input is in millidegrees).
-	 */
-	val = wrapping_sub(u64, c->GFX_TempAcc, p->GFX_TempAcc);
+	/* Temperature: accumulator in Celsius -> millidegrees C (temp*_input). */
 	avg_metric[METRICS_TEMPERATURE_VRGFX] =
-		div_u64(div64_u64(val, counter) * SMU_TEMPERATURE_UNITS_PER_CENTIGRADES, 1024);
-
-	val = wrapping_sub(u64, c->STT_APU_HotSpotTempAcc, p->STT_APU_HotSpotTempAcc);
+		SMU_V15_IOD_AVG(GFX_TempAcc, SMU_TEMPERATURE_UNITS_PER_CENTIGRADES);
 	avg_metric[METRICS_TEMPERATURE_HOTSPOT] =
-		div_u64(div64_u64(val, counter) * SMU_TEMPERATURE_UNITS_PER_CENTIGRADES, 1024);
-
-	/* Voltage: accumulator holds a 1024x fixed-point value in Volts;
-	 * convert to millivolts for the hwmon/sysfs consumers.
-	 */
-	val = wrapping_sub(u64, c->VDDCR_GFX_TelemetryVoltage, p->VDDCR_GFX_TelemetryVoltage);
-	avg_metric[METRICS_VOLTAGE_VDDGFX] = div_u64(div64_u64(val, counter) * 1000, 1024);
+		SMU_V15_IOD_AVG(STT_APU_HotSpotTempAcc, SMU_TEMPERATURE_UNITS_PER_CENTIGRADES);
 
-	val = wrapping_sub(u64, c->VDDCR_SOC_TelemetryVoltage, p->VDDCR_SOC_TelemetryVoltage);
-	avg_metric[METRICS_VOLTAGE_VDDSOC] = div_u64(div64_u64(val, counter) * 1000, 1024);
+	/* Voltage: accumulator in Volts -> millivolts. */
+	avg_metric[METRICS_VOLTAGE_VDDGFX] = SMU_V15_IOD_AVG(VDDCR_GFX_TelemetryVoltage, 1000);
+	avg_metric[METRICS_VOLTAGE_VDDSOC] = SMU_V15_IOD_AVG(VDDCR_SOC_TelemetryVoltage, 1000);
 }
 
 static int smu_v15_0_0_get_smu_metrics_data(struct smu_context *smu,
@@ -714,61 +673,99 @@ static ssize_t smu_v15_0_0_get_gpu_metrics(struct smu_context *smu,
 	struct gpu_metrics_v3_0 *gpu_metrics =
 		(struct gpu_metrics_v3_0 *)smu_driver_table_ptr(
 			smu, SMU_DRIVER_TABLE_GPU_METRICS);
-	SmuMetrics_t metrics;
-	int ret = 0;
+	struct smu_table_context *smu_table = &smu->smu_table;
+	SMU_15_0_0_MetricsInfo_t *metrics_info =
+		(SMU_15_0_0_MetricsInfo_t *)smu_table->metrics_table;
+	uint32_t all_core_power = 0;
+	MetricsTable_IOD_t *p, *c;
+	MetricsTable_t *prev, *curr;
+	size_t i, ccx, core;
+	uint32_t coreclk;
+	uint64_t counter;
+	int ret;
 
-	ret = smu_v15_0_0_get_gpu_metrics_table(smu, &metrics, false);
-	if (ret)
+	/* Refresh the accumulator double-buffer (may reuse a cached sample). */
+	ret = smu_v15_0_0_get_metrics_table(smu, metrics_info);
+	if (ret < 0)
 		return ret;
 
+	curr = &metrics_info->metrics[metrics_info->active_idx];
+	prev = &metrics_info->metrics[!metrics_info->active_idx];
+	c = &curr->IOD;
+	p = &prev->IOD;
+
 	smu_cmn_init_soft_gpu_metrics(gpu_metrics, 3, 0);
 
-	gpu_metrics->temperature_gfx = metrics.GfxTemperature;
-	gpu_metrics->temperature_soc = metrics.SocTemperature;
-	memcpy(&gpu_metrics->temperature_core[0],
-		&metrics.CoreTemperature[0],
-		sizeof(uint16_t) * 16);
-	gpu_metrics->temperature_skin = metrics.SkinTemp;
-
-	gpu_metrics->average_gfx_activity = metrics.GfxActivity;
-	gpu_metrics->average_vcn_activity = metrics.VcnActivity;
-
-	memcpy(&gpu_metrics->average_core_c0_activity[0],
-		&metrics.CoreC0Residency[0],
-		sizeof(uint16_t) * 16);
-	gpu_metrics->average_dram_reads = metrics.DRAMReads;
-	gpu_metrics->average_dram_writes = metrics.DRAMWrites;
-
-	gpu_metrics->average_socket_power = metrics.SocketPower;
-	gpu_metrics->average_apu_power = metrics.ApuPower;
-	gpu_metrics->average_gfx_power = metrics.GfxPower;
-	gpu_metrics->average_dgpu_power = metrics.dGpuPower;
-	gpu_metrics->average_all_core_power = metrics.AllCorePower;
-	gpu_metrics->average_sys_power = metrics.Psys;
-	memcpy(&gpu_metrics->average_core_power[0],
-		&metrics.CorePower[0],
-		sizeof(uint16_t) * 16);
-
-	gpu_metrics->average_gfxclk_frequency = metrics.GfxclkFrequency;
-	gpu_metrics->average_socclk_frequency = metrics.SocclkFrequency;
-	gpu_metrics->average_vpeclk_frequency = metrics.VpeclkFrequency;
-	gpu_metrics->average_fclk_frequency = metrics.FclkFrequency;
-	gpu_metrics->average_vclk_frequency = metrics.VclkFrequency;
-	gpu_metrics->average_uclk_frequency = metrics.MemclkFrequency;
-
-	memcpy(&gpu_metrics->current_coreclk[0],
-		&metrics.CoreFrequency[0],
-		sizeof(uint16_t) * 16);
-	gpu_metrics->current_core_maxfreq = metrics.InfrastructureCpuMaxFreq;
-	gpu_metrics->current_gfx_maxfreq = metrics.InfrastructureGfxMaxFreq;
-
-	gpu_metrics->throttle_residency_prochot = metrics.ThrottleResidency_PROCHOT;
-	gpu_metrics->throttle_residency_spl = metrics.ThrottleResidency_SPL;
-	gpu_metrics->throttle_residency_fppt = metrics.ThrottleResidency_FPPT;
-	gpu_metrics->throttle_residency_sppt = metrics.ThrottleResidency_SPPT;
-	gpu_metrics->throttle_residency_thm_soc = metrics.ThrottleResidency_THM_SOC;
-
-	gpu_metrics->time_filter_alphavalue = metrics.FilterAlphaValue;
+	counter = wrapping_sub(u32, c->AccumulationCounter, p->AccumulationCounter);
+	if (counter) {
+		/* Temperatures: accumulator in Celsius -> centi-C. */
+		gpu_metrics->temperature_gfx = SMU_V15_IOD_AVG(GFX_TempAcc, 100);
+		gpu_metrics->temperature_soc = SMU_V15_IOD_AVG(SOC_TempAcc, 100);
+		gpu_metrics->temperature_skin = SMU_V15_IOD_AVG(STT_APU_SkinTempAcc, 100);
+
+		/* Activity: accumulator already in %. */
+		gpu_metrics->average_gfx_activity = SMU_V15_IOD_AVG(GfxBusyAcc, 1);
+		gpu_metrics->average_vcn_activity = SMU_V15_IOD_AVG(VcnBusyAcc, 1);
+
+		/* Bandwidth: accumulator in GB/s -> MB/s. */
+		gpu_metrics->average_dram_reads = SMU_V15_IOD_AVG(DramReadBandwidth, 1000);
+		gpu_metrics->average_dram_writes = SMU_V15_IOD_AVG(DramWriteBandwidth, 1000);
+
+		/* Power: accumulator in W -> mW. */
+		gpu_metrics->average_apu_power = SMU_V15_IOD_AVG(ApuPowerAcc, 1000);
+		gpu_metrics->average_gfx_power = SMU_V15_IOD_AVG(VDDCR_GFX_TelemetryPower, 1000);
+		gpu_metrics->average_dgpu_power = SMU_V15_IOD_AVG(dGpuPowerAcc, 1000);
+		gpu_metrics->average_sys_power = SMU_V15_IOD_AVG(SystemPowerAcc, 1000);
+		/* No dedicated socket accumulator: APU + dGPU, matching PPT/STAPM. */
+		gpu_metrics->average_socket_power =
+			gpu_metrics->average_apu_power + gpu_metrics->average_dgpu_power;
+
+		/* Effective clocks: accumulator already in MHz. */
+		gpu_metrics->average_gfxclk_frequency = SMU_V15_IOD_AVG(GfxclkFreqEffAcc, 1);
+		gpu_metrics->average_socclk_frequency = SMU_V15_IOD_AVG(SocclkFreqEffAcc, 1);
+		gpu_metrics->average_vpeclk_frequency = SMU_V15_IOD_AVG(VpeclkFreqEffAcc, 1);
+		gpu_metrics->average_fclk_frequency = SMU_V15_IOD_AVG(FclkFreqEffAcc, 1);
+		gpu_metrics->average_vclk_frequency = SMU_V15_IOD_AVG(VclkFreqEffAcc, 1);
+		gpu_metrics->average_uclk_frequency = SMU_V15_IOD_AVG(MemclkFreqEffAcc, 1);
+
+		/* Per-core metrics: report only active cores (non-zero effective
+		 * clock), flattening CCX-major into the 16 gpu_metrics slots.
+		 */
+		i = 0;
+		for (ccx = 0; ccx < METRIC_CCX_MAX &&
+			      i < ARRAY_SIZE(gpu_metrics->current_coreclk); ccx++) {
+			for (core = 0; core < NUM_MAX_CORES &&
+				       i < ARRAY_SIZE(gpu_metrics->current_coreclk);
+			     core++) {
+				/* Effective clock gates whether the core is active. */
+				coreclk = SMU_V15_CORE_AVG(ccx, core, Core_FREQEFF, 1000);
+				if (!coreclk)
+					continue;
+
+				gpu_metrics->current_coreclk[i] = coreclk;
+				gpu_metrics->temperature_core[i] =
+					SMU_V15_CORE_AVG(ccx, core, Core_TEMP, 100);
+				gpu_metrics->average_core_c0_activity[i] =
+					SMU_V15_CORE_AVG(ccx, core, Core_C0, 1);
+				gpu_metrics->average_core_power[i] =
+					SMU_V15_CORE_AVG(ccx, core, Core_POWER, 1000);
+				all_core_power += gpu_metrics->average_core_power[i];
+				i++;
+			}
+		}
+		gpu_metrics->average_all_core_power = all_core_power;
+
+		/* Throttle residencies: accumulator in % (semantics differ from
+		 * the PM_TIMER cycle counters exposed by the legacy metrics path).
+		 */
+		gpu_metrics->throttle_residency_prochot =
+			SMU_V15_IOD_AVG(PROCHOT_ResidencyAcc, 1);
+		gpu_metrics->throttle_residency_spl = SMU_V15_IOD_AVG(SPL_ResidencyAcc, 1);
+		gpu_metrics->throttle_residency_fppt = SMU_V15_IOD_AVG(fPPT_ResidencyAcc, 1);
+		gpu_metrics->throttle_residency_sppt = SMU_V15_IOD_AVG(sPPT_ResidencyAcc, 1);
+		gpu_metrics->throttle_residency_thm_soc = SMU_V15_IOD_AVG(THM_ResidencyAcc, 1);
+	}
+
 	gpu_metrics->system_clock_counter = ktime_get_boottime_ns();
 
 	*table = (void *)gpu_metrics;
@@ -778,6 +775,9 @@ static ssize_t smu_v15_0_0_get_gpu_metrics(struct smu_context *smu,
 	return sizeof(struct gpu_metrics_v3_0);
 }
 
+#undef SMU_V15_IOD_AVG
+#undef SMU_V15_CORE_AVG
+
 static int smu_v15_0_0_mode2_reset(struct smu_context *smu)
 {
 	int ret;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
index 4f820db5f8dd2e..93486be4e17bf3 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
@@ -72,6 +72,9 @@
 
 #define NUM_JPEG_RINGS_FW	10
 
+/* Custom 5-second timeout (in us) for unload messages */
+#define SMU_V15_0_8_MSG_TIMEOUT_US	(5 * 1000 * 1000)
+
 #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c))
 
 #define SMU_15_0_8_FEA_MAP(smu_feature, smu_15_0_8_feature)                    \
@@ -123,6 +126,7 @@ static const struct cmn2asic_msg_mapping smu_v15_0_8_message_map[SMU_MSG_MAX_COU
 	MSG_MAP(GetTimestamp,                        PPSMC_MSG_GetTimestamp,                    0),
 	MSG_MAP(GetBadPageIpid,                      PPSMC_MSG_GetBadPageIpIdLoHi,              0),
 	MSG_MAP(EraseRasTable,                       PPSMC_MSG_EraseRasTable,                   0),
+	MSG_MAP(GetRmaStatus,                        PPSMC_MSG_GetRmaStatus,                    0),
 	MSG_MAP(GetStaticMetricsTable,               PPSMC_MSG_GetStaticMetricsTable,		1),
 	MSG_MAP(GetSystemMetricsTable,               PPSMC_MSG_GetSystemMetricsTable,           1),
 	MSG_MAP(GetSystemMetricsVersion,             PPSMC_MSG_GetSystemMetricsVersion,		0),
@@ -614,6 +618,9 @@ static int smu_v15_0_8_get_npm_data(struct smu_context *smu,
 	case AMDGPU_PP_SENSOR_NODEPOWER:
 		*value = SMUQ10_ROUND(metrics->NodePower);
 		break;
+	case AMDGPU_PP_SENSOR_NPMSTATUS:
+		*value = !!SMUQ10_ROUND(metrics->NodePower);
+		break;
 	case AMDGPU_PP_SENSOR_GPPTRESIDENCY:
 		*value = SMUQ10_ROUND(metrics->GlobalPPTResidencyAcc);
 		break;
@@ -677,6 +684,7 @@ static int smu_v15_0_8_read_sensor(struct smu_context *smu,
 	case AMDGPU_PP_SENSOR_NODEPOWER:
 	case AMDGPU_PP_SENSOR_GPPTRESIDENCY:
 	case AMDGPU_PP_SENSOR_MAXNODEPOWERLIMIT:
+	case AMDGPU_PP_SENSOR_NPMSTATUS:
 		ret = smu_v15_0_8_get_npm_data(smu, sensor, (uint32_t *)data);
 		if (ret)
 			return ret;
@@ -1279,12 +1287,18 @@ static int smu_v15_0_8_register_irq_handler(struct smu_context *smu)
 
 static int smu_v15_0_8_notify_unload(struct smu_context *smu)
 {
+	struct smu_msg_ctl *ctl = &smu->msg_ctl;
+	struct smu_msg_args args = {
+		.msg     = SMU_MSG_PrepareMp1ForUnload,
+		.timeout = SMU_V15_0_8_MSG_TIMEOUT_US,
+	};
+
 	if (amdgpu_in_reset(smu->adev))
 		return 0;
 
 	dev_dbg(smu->adev->dev, "Notify PMFW about driver unload");
 	/* Ignore return, just intimate FW that driver is not going to be there */
-	smu_cmn_send_smc_msg(smu, SMU_MSG_PrepareMp1ForUnload, NULL);
+	ctl->ops->send_msg(ctl, &args);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/ras/core/Makefile b/drivers/gpu/drm/amd/ras/core/Makefile
index 5b4cd6a3d3588d..fb16bacaef2fd9 100644
--- a/drivers/gpu/drm/amd/ras/core/Makefile
+++ b/drivers/gpu/drm/amd/ras/core/Makefile
@@ -22,11 +22,15 @@
 RAS_CORE_FILES = core.o \
 			ras_mp1.o \
 			ras_mp1_v13_0.o \
+			ras_mp1_v15_0.o \
 			aca.o  \
 			aca_v1_0.o \
+			aca_v5_0.o \
+			ras_eeprom_mgr.o \
 			eeprom.o \
 			ras_umc.o \
 			ras_umc_v12_0.o \
+			ras_umc_v15_0.o \
 			cmd.o \
 			ras_gfx.o \
 			ras_gfx_v9_0.o \
@@ -34,10 +38,13 @@ RAS_CORE_FILES = core.o \
 			ras_nbio.o \
 			ras_nbio_v7_9.o \
 			log_ring.o \
+			ras_bert.o \
 			ras_cper.o \
 			ras_psp.o \
 			ras_psp_v13_0.o \
-			eeprom_fw.o
+			ras_psp_v15_0.o \
+			eeprom_fw.o \
+			ras_mce.o
 
 
 RAS_CORE = $(addprefix $(AMD_GPU_RAS_PATH)/core/,$(RAS_CORE_FILES))
diff --git a/drivers/gpu/drm/amd/ras/core/aca.c b/drivers/gpu/drm/amd/ras/core/aca.c
index fa7cb327757860..4289e968d331ac 100644
--- a/drivers/gpu/drm/amd/ras/core/aca.c
+++ b/drivers/gpu/drm/amd/ras/core/aca.c
@@ -23,7 +23,9 @@
  */
 #include "ras.h"
 #include "aca.h"
+#include "ras_mce.h"
 #include "aca_v1_0.h"
+#include "aca_v5_0.h"
 #include "ras_mp1_v13_0.h"
 
 #define ACA_MARK_FATAL_FLAG    0x100
@@ -54,24 +56,24 @@ static void aca_report_ecc_info(struct ras_core_context *ras_core,
 				struct aca_bank_ecc *new_ecc)
 {
 	struct aca_ecc_count ecc_count = {0};
+	struct aca_ecc_count *xcd_ecc;
+	int xcd_id;
 
 	ecc_count.new_ue_count = new_ecc->ue_count;
 	ecc_count.new_de_count = new_ecc->de_count;
 	ecc_count.new_ce_count = new_ecc->ce_count;
-	if (blk == RAS_BLOCK_ID__GFX) {
-		struct aca_ecc_count *xcd_ecc;
-		int xcd_id;
 
+	ecc_count.total_ue_count = aid_ecc->ecc_err.total_ue_count;
+	ecc_count.total_de_count = aid_ecc->ecc_err.total_de_count;
+	ecc_count.total_ce_count = aid_ecc->ecc_err.total_ce_count;
+
+	if (blk == RAS_BLOCK_ID__GFX) {
 		for (xcd_id = 0; xcd_id < aid_ecc->xcd.xcd_num; xcd_id++) {
 			xcd_ecc = &aid_ecc->xcd.xcd[xcd_id].ecc_err;
 			ecc_count.total_ue_count += xcd_ecc->total_ue_count;
 			ecc_count.total_de_count += xcd_ecc->total_de_count;
 			ecc_count.total_ce_count += xcd_ecc->total_ce_count;
 		}
-	} else {
-		ecc_count.total_ue_count = aid_ecc->ecc_err.total_ue_count;
-		ecc_count.total_de_count = aid_ecc->ecc_err.total_de_count;
-		ecc_count.total_ce_count = aid_ecc->ecc_err.total_ce_count;
 	}
 
 	if (ecc_count.new_ue_count) {
@@ -126,24 +128,39 @@ static void aca_bank_log(struct ras_core_context *ras_core,
 			"{%llu}" RAS_HW_ERR "ACA[%02d/%02d].%s=0x%016llx\n",
 			bank->seq_no, idx + 1, total,
 			aca_regs[i].name, bank->regs[aca_regs[i].reg_idx]);
+
+	if (ACA_REG_STATUS_SCRUB(bank->regs[ACA_REG_IDX__STATUS]))
+		RAS_DEV_INFO(ras_core->dev,
+			"{%llu}" RAS_HW_ERR "hardware error logged by the scrubber\n",
+			bank->seq_no);
 }
 
 static void aca_log_bank_data(struct ras_core_context *ras_core,
 			struct aca_bank_reg *bank, struct aca_bank_ecc *bank_ecc,
 			struct ras_log_batch_tag *batch)
 {
+	enum ras_log_event event;
+
+	if (batch && bank->timestamp)
+		batch->timestamp = bank->timestamp;
+
 	if (bank_ecc->ue_count)
-		ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_UE, bank->regs, batch);
+		event = RAS_LOG_EVENT_UE;
 	else if (bank_ecc->de_count)
-		ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_DE, bank->regs, batch);
+		event = RAS_LOG_EVENT_DE;
 	else
-		ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_CE, bank->regs, batch);
+		event = RAS_LOG_EVENT_CE;
+
+	ras_log_ring_add_log_event(ras_core,
+		event, bank->regs, sizeof(bank->regs), batch);
 }
 
 static int aca_get_bank_count(struct ras_core_context *ras_core,
 			      enum ras_err_type type, u32 *count)
 {
-	return ras_mp1_get_bank_count(ras_core, type, count);
+	return (type == RAS_ERR_TYPE__MCE) ?
+		ras_mce_get_bank_count(ras_core, type, count) :
+		ras_mp1_get_bank_count(ras_core, type, count);
 }
 
 static bool aca_match_bank(struct aca_block *aca_blk, struct aca_bank_reg *bank)
@@ -170,33 +187,43 @@ static int aca_parse_bank(struct ras_core_context *ras_core,
 	if (!bank_ops || !bank_ops->bank_parse)
 		return -RAS_CORE_NOT_SUPPORTED;
 
+	ecc->block_id = aca_blk->blk_info->ras_block_id;
 	return bank_ops->bank_parse(ras_core, aca_blk, bank, ecc);
 }
 
 static int aca_check_block_ecc_info(struct ras_core_context *ras_core,
 			struct aca_block *aca_blk, struct aca_ecc_info *info)
 {
-	if (info->socket_id >= aca_blk->ecc.socket_num_per_hive) {
+	struct device_system_info dev_info = {0};
+	int ret;
+
+	ret = ras_core_get_device_system_info(ras_core, &dev_info);
+	if (ret) {
+		RAS_DEV_ERR(ras_core->dev, "Failed to get system device info.\n");
+		return ret;
+	}
+
+	if (info->socket_id != dev_info.socket_id) {
 		RAS_DEV_ERR(ras_core->dev,
-			"Socket id (%d) is out of config! max:%u\n",
-			info->socket_id, aca_blk->ecc.socket_num_per_hive);
-		return -ENODATA;
+			"Bank socket id (%d) does not match device socket id:%u\n",
+			info->socket_id, dev_info.socket_id);
+		return -ENXIO;
 	}
 
-	if (info->die_id >= aca_blk->ecc.socket[info->socket_id].aid_num) {
+	if (info->die_id >= aca_blk->ecc.socket.aid_num) {
 		RAS_DEV_ERR(ras_core->dev,
 			"Die id (%d) is out of config! max:%u\n",
-			info->die_id, aca_blk->ecc.socket[info->socket_id].aid_num);
+			info->die_id, aca_blk->ecc.socket.aid_num);
 		return -ENODATA;
 	}
 
-	if ((aca_blk->blk_info->ras_block_id == RAS_BLOCK_ID__GFX) &&
+	if ((aca_blk->blk_info->ras_block_id == RAS_BLOCK_ID__GFX) && info->xcd_valid &&
 	    (info->xcd_id >=
-		 aca_blk->ecc.socket[info->socket_id].aid[info->die_id].xcd.xcd_num)) {
+		 aca_blk->ecc.socket.aid[info->die_id].xcd.xcd_num)) {
 		RAS_DEV_ERR(ras_core->dev,
 			"Xcd id (%d) is out of config! max:%u\n",
 			info->xcd_id,
-			aca_blk->ecc.socket[info->socket_id].aid[info->die_id].xcd.xcd_num);
+			aca_blk->ecc.socket.aid[info->die_id].xcd.xcd_num);
 		return -ENODATA;
 	}
 
@@ -219,11 +246,12 @@ static int aca_log_bad_bank(struct ras_core_context *ras_core,
 		return ret;
 
 	mutex_lock(&ras_core->ras_aca.aca_lock);
-	aid_ecc = &aca_blk->ecc.socket[info->socket_id].aid[info->die_id];
-	if (aca_blk->blk_info->ras_block_id == RAS_BLOCK_ID__GFX)
+	aid_ecc = &aca_blk->ecc.socket.aid[info->die_id];
+	ecc_err = &aid_ecc->ecc_err;
+
+	if ((aca_blk->blk_info->ras_block_id == RAS_BLOCK_ID__GFX) &&
+		bank_ecc->bank_info.xcd_valid)
 		ecc_err = &aid_ecc->xcd.xcd[info->xcd_id].ecc_err;
-	else
-		ecc_err = &aid_ecc->ecc_err;
 
 	ecc_err->new_ce_count += bank_ecc->ce_count;
 	ecc_err->total_ce_count += bank_ecc->ce_count;
@@ -233,36 +261,28 @@ static int aca_log_bad_bank(struct ras_core_context *ras_core,
 	ecc_err->total_de_count += bank_ecc->de_count;
 	mutex_unlock(&ras_core->ras_aca.aca_lock);
 
-	if ((aca_blk->blk_info->ras_block_id == RAS_BLOCK_ID__UMC) &&
+	/* Log ACA bad bank when FW management EEPROM is disabled. */
+	if (!ras_eeprom_mgr_fw_record_enabled(ras_core) &&
+	    (aca_blk->blk_info->ras_block_id == RAS_BLOCK_ID__UMC) &&
 	    bank_ecc->de_count) {
 		struct ras_bank_ecc  ras_ecc = {0};
 
-		if (ras_fw_eeprom_supported(ras_core)) {
-			ret = ras_fw_eeprom_update_record(ras_core, &ras_ecc);
-			if (!ret) {
-				ras_ecc.nps = ras_core_get_curr_nps_mode(ras_core);
-				ras_ecc.status = bank_ecc->bank_info.status;
-				ras_ecc.seq_no = bank->seq_no;
-			}
-		} else {
-			ras_ecc.nps = ras_core_get_curr_nps_mode(ras_core);
-			ras_ecc.addr = bank_ecc->bank_info.addr;
-			ras_ecc.ipid = bank_ecc->bank_info.ipid;
-			ras_ecc.status = bank_ecc->bank_info.status;
-			ras_ecc.seq_no = bank->seq_no;
-		}
+		ras_ecc.nps = ras_core_get_curr_nps_mode(ras_core);
+		ras_ecc.addr = bank_ecc->bank_info.addr;
+		ras_ecc.ipid = bank_ecc->bank_info.ipid;
+		ras_ecc.status = bank_ecc->bank_info.status;
+		ras_ecc.seq_no = bank->seq_no;
+		ras_ecc.timestamp = bank->timestamp;
 
-		if (!ret) {
-			if (ras_core_gpu_in_reset(ras_core))
-				ras_umc_log_bad_bank_pending(ras_core, &ras_ecc);
-			else
-				ras_umc_log_bad_bank(ras_core, &ras_ecc);
-		}
+		if (ras_core_gpu_in_reset(ras_core))
+			ras_umc_log_bad_bank_pending(ras_core, &ras_ecc);
+		else
+			ras_umc_log_bad_bank(ras_core, &ras_ecc);
 	}
 
 	aca_report_ecc_info(ras_core,
 		bank->seq_no, aca_blk->blk_info->ras_block_id, info->socket_id, info->die_id,
-		&aca_blk->ecc.socket[info->socket_id].aid[info->die_id], bank_ecc);
+		&aca_blk->ecc.socket.aid[info->die_id], bank_ecc);
 
 	return 0;
 }
@@ -283,16 +303,11 @@ static int aca_dump_bank(struct ras_core_context *ras_core, u32 ecc_type,
 			 int idx, void *data)
 {
 	struct aca_bank_reg *bank = (struct aca_bank_reg *)data;
-	int i, ret, reg_cnt;
-
-	reg_cnt = min_t(int, 16, ARRAY_SIZE(bank->regs));
-	for (i = 0; i < reg_cnt; i++) {
-		ret = ras_mp1_dump_bank(ras_core, ecc_type, idx, i, &bank->regs[i]);
-		if (ret)
-			return ret;
-	}
 
-	return 0;
+	return (ecc_type == RAS_ERR_TYPE__MCE) ?
+			ras_mce_dump_bank(ras_core, ecc_type, idx, bank) :
+			ras_mp1_dump_bank(ras_core, ecc_type, idx,
+				bank->regs, ARRAY_SIZE(bank->regs));
 }
 
 static uint64_t aca_get_bank_seqno(struct ras_core_context *ras_core,
@@ -379,7 +394,6 @@ static int aca_banks_update(struct ras_core_context *ras_core,
 	if (!count)
 		goto out;
 
-	batch_tag = ras_log_ring_create_batch_tag(ras_core);
 	for (i = 0; i < count; i++) {
 		memset(&bank, 0, sizeof(bank));
 		ret = aca_dump_bank(ras_core, ecc_type, i, &bank);
@@ -390,11 +404,22 @@ static int aca_banks_update(struct ras_core_context *ras_core,
 
 		memset(&bank_ecc, 0, sizeof(bank_ecc));
 		aca_blk = aca_get_bank_aca_block(ras_core, &bank);
+
+		/* Outside of UMC a poison bank is a consumption, not a new error. */
+		if (ecc_type == RAS_ERR_TYPE__UE &&
+		    ACA_REG_STATUS_POISON(bank.regs[ACA_REG_IDX__STATUS]) &&
+		    (!aca_blk || aca_blk->blk_info->ras_block_id != RAS_BLOCK_ID__UMC))
+			continue;
+
 		if (aca_blk)
 			ret = aca_parse_bank(ras_core, aca_blk, &bank, &bank_ecc);
 
 		bank.seq_no = aca_get_bank_seqno(ras_core, ecc_type, aca_blk, &bank_ecc);
 
+		/* Only one MCE error is logged for each batch */
+		if (ecc_type != RAS_ERR_TYPE__MCE && !batch_tag)
+			batch_tag = ras_log_ring_create_batch_tag(ras_core);
+
 		aca_log_bank_data(ras_core, &bank, &bank_ecc, batch_tag);
 		aca_bank_log(ras_core, i, count, &bank, &bank_ecc);
 
@@ -404,8 +429,13 @@ static int aca_banks_update(struct ras_core_context *ras_core,
 		if (ret)
 			break;
 	}
-	ras_log_ring_destroy_batch_tag(ras_core, batch_tag);
 
+	if (batch_tag)
+		ras_log_ring_destroy_batch_tag(ras_core, batch_tag);
+
+	if (!ret)
+		ras_core_event_notify(ras_core,
+			RAS_EVENT_ID__UPDATE_ACA_DATA, NULL);
 out:
 	mutex_unlock(&ras_core->ras_aca.bank_op_lock);
 	return ret;
@@ -422,100 +452,70 @@ static struct aca_block *ras_aca_get_block_handle(struct ras_core_context *ras_c
 	return &ras_core->ras_aca.aca_blk[blk];
 }
 
-static int ras_aca_clear_block_ecc_count(struct ras_core_context *ras_core, u32 blk)
+static void __clear_block_socket_ecc_count(struct ras_core_context *ras_core,
+			enum ras_block_id blk, struct aca_socket_ecc *socket)
 {
-	struct aca_block *aca_blk;
 	struct aca_aid_ecc  *aid_ecc;
-	int skt, aid, xcd;
-
-	mutex_lock(&ras_core->ras_aca.aca_lock);
-	aca_blk = ras_aca_get_block_handle(ras_core, blk);
-	for (skt = 0; skt < aca_blk->ecc.socket_num_per_hive; skt++) {
-		for (aid = 0; aid < aca_blk->ecc.socket[skt].aid_num; aid++) {
-			aid_ecc = &aca_blk->ecc.socket[skt].aid[aid];
-			if (blk == RAS_BLOCK_ID__GFX) {
-				for (xcd = 0; xcd < aid_ecc->xcd.xcd_num; xcd++)
-					memset(&aid_ecc->xcd.xcd[xcd],
+	int aid, xcd;
+
+	for (aid = 0; aid < socket->aid_num; aid++) {
+		aid_ecc = &socket->aid[aid];
+		memset(&aid_ecc->ecc_err, 0, sizeof(aid_ecc->ecc_err));
+		if (blk == RAS_BLOCK_ID__GFX) {
+			for (xcd = 0; xcd < aid_ecc->xcd.xcd_num; xcd++)
+				memset(&aid_ecc->xcd.xcd[xcd],
 						0, sizeof(struct aca_xcd_ecc));
-			} else {
-				memset(&aid_ecc->ecc_err, 0, sizeof(aid_ecc->ecc_err));
-			}
 		}
 	}
-	mutex_unlock(&ras_core->ras_aca.aca_lock);
-
-	return 0;
 }
 
 int ras_aca_clear_all_blocks_ecc_count(struct ras_core_context *ras_core)
 {
+	struct aca_block *aca_blk;
 	enum ras_block_id blk;
-	int ret;
 
+	mutex_lock(&ras_core->ras_aca.aca_lock);
 	for (blk = RAS_BLOCK_ID__UMC; blk < RAS_BLOCK_ID__LAST; blk++) {
-		ret = ras_aca_clear_block_ecc_count(ras_core, blk);
-		if (ret)
-			break;
+		aca_blk = ras_aca_get_block_handle(ras_core, blk);
+		if (aca_blk)
+			__clear_block_socket_ecc_count(ras_core,
+						blk, &aca_blk->ecc.socket);
 	}
+	mutex_unlock(&ras_core->ras_aca.aca_lock);
 
-	return ret;
+	return 0;
 }
 
 int ras_aca_clear_block_new_ecc_count(struct ras_core_context *ras_core, u32 blk)
 {
 	struct aca_block *aca_blk;
-	int skt, aid, xcd;
+	int aid, xcd;
 	struct aca_ecc_count *ecc_err;
 	struct aca_aid_ecc  *aid_ecc;
 
 	mutex_lock(&ras_core->ras_aca.aca_lock);
 	aca_blk = ras_aca_get_block_handle(ras_core, blk);
-	for (skt = 0; skt < aca_blk->ecc.socket_num_per_hive; skt++) {
-		for (aid = 0; aid < aca_blk->ecc.socket[skt].aid_num; aid++) {
-			aid_ecc = &aca_blk->ecc.socket[skt].aid[aid];
-			if (blk == RAS_BLOCK_ID__GFX) {
-				for (xcd = 0; xcd < aid_ecc->xcd.xcd_num; xcd++) {
-					ecc_err = &aid_ecc->xcd.xcd[xcd].ecc_err;
-					ecc_err->new_ce_count = 0;
-					ecc_err->new_ue_count = 0;
-					ecc_err->new_de_count = 0;
-				}
-			} else {
-				ecc_err = &aid_ecc->ecc_err;
+	for (aid = 0; aid < aca_blk->ecc.socket.aid_num; aid++) {
+		aid_ecc = &aca_blk->ecc.socket.aid[aid];
+		if (blk == RAS_BLOCK_ID__GFX) {
+			for (xcd = 0; xcd < aid_ecc->xcd.xcd_num; xcd++) {
+				ecc_err = &aid_ecc->xcd.xcd[xcd].ecc_err;
 				ecc_err->new_ce_count = 0;
 				ecc_err->new_ue_count = 0;
 				ecc_err->new_de_count = 0;
 			}
 		}
+
+		ecc_err = &aid_ecc->ecc_err;
+		ecc_err->new_ce_count = 0;
+		ecc_err->new_ue_count = 0;
+		ecc_err->new_de_count = 0;
 	}
 	mutex_unlock(&ras_core->ras_aca.aca_lock);
 
 	return 0;
 }
 
-static int ras_aca_get_block_each_aid_ecc_count(struct ras_core_context *ras_core,
-						u32 blk, u32 skt, u32 aid, u32 xcd,
-						struct aca_ecc_count *ecc_count)
-{
-	struct aca_block *aca_blk;
-	struct aca_ecc_count *ecc_err;
-
-	aca_blk = ras_aca_get_block_handle(ras_core, blk);
-	if (blk == RAS_BLOCK_ID__GFX)
-		ecc_err = &aca_blk->ecc.socket[skt].aid[aid].xcd.xcd[xcd].ecc_err;
-	else
-		ecc_err = &aca_blk->ecc.socket[skt].aid[aid].ecc_err;
-
-	ecc_count->new_ce_count = ecc_err->new_ce_count;
-	ecc_count->total_ce_count = ecc_err->total_ce_count;
-	ecc_count->new_ue_count = ecc_err->new_ue_count;
-	ecc_count->total_ue_count = ecc_err->total_ue_count;
-	ecc_count->new_de_count = ecc_err->new_de_count;
-	ecc_count->total_de_count = ecc_err->total_de_count;
-
-	return 0;
-}
-
 static inline void _add_ecc_count(struct aca_ecc_count *des, struct aca_ecc_count *src)
 {
 	des->new_ce_count += src->new_ce_count;
@@ -526,12 +526,33 @@ static inline void _add_ecc_count(struct aca_ecc_count *des, struct aca_ecc_coun
 	des->total_de_count += src->total_de_count;
 }
 
+static void __get_block_socket_ecc_count(struct ras_core_context *ras_core,
+		u32 blk, struct aca_socket_ecc *socket, struct aca_ecc_count *ecc_count)
+{
+	struct aca_aid_ecc *aid_ecc;
+	int aid, xcd;
+
+	for (aid = 0; aid < socket->aid_num; aid++) {
+		aid_ecc = &socket->aid[aid];
+
+		/* Add AID error count */
+		_add_ecc_count(ecc_count, &aid_ecc->ecc_err);
+		if (blk == RAS_BLOCK_ID__GFX) {
+			/* Add XCD error count */
+			for (xcd = 0; xcd < aid_ecc->xcd.xcd_num; xcd++)
+				_add_ecc_count(ecc_count, &aid_ecc->xcd.xcd[xcd].ecc_err);
+		}
+	}
+}
+
 static const struct ras_aca_ip_func *aca_get_ip_func(
 				struct ras_core_context *ras_core, uint32_t ip_version)
 {
 	switch (ip_version) {
 	case IP_VERSION(1, 0, 0):
 		return &ras_aca_func_v1_0;
+	case IP_VERSION(5, 0, 0):
+		return &ras_aca_func_v5_0;
 	default:
 		RAS_DEV_ERR(ras_core->dev,
 			"ACA ip version(0x%x) is not supported!\n", ip_version);
@@ -546,9 +567,6 @@ int ras_aca_get_block_ecc_count(struct ras_core_context *ras_core,
 {
 	struct ras_ecc_count *err_data = (struct ras_ecc_count *)data;
 	struct aca_block *aca_blk;
-	int skt, aid, xcd;
-	struct aca_ecc_count ecc_xcd;
-	struct aca_ecc_count ecc_aid;
 	struct aca_ecc_count ecc;
 
 	if (blk >= RAS_BLOCK_ID__LAST)
@@ -561,33 +579,7 @@ int ras_aca_get_block_ecc_count(struct ras_core_context *ras_core,
 	memset(&ecc, 0, sizeof(ecc));
 
 	mutex_lock(&ras_core->ras_aca.aca_lock);
-	if (blk == RAS_BLOCK_ID__GFX) {
-		for (skt = 0; skt < aca_blk->ecc.socket_num_per_hive; skt++) {
-			for (aid = 0; aid < aca_blk->ecc.socket[skt].aid_num; aid++) {
-				memset(&ecc_aid, 0, sizeof(ecc_aid));
-				for (xcd = 0;
-				     xcd < aca_blk->ecc.socket[skt].aid[aid].xcd.xcd_num;
-				     xcd++) {
-					memset(&ecc_xcd, 0, sizeof(ecc_xcd));
-					if (ras_aca_get_block_each_aid_ecc_count(ras_core,
-							blk, skt, aid, xcd, &ecc_xcd))
-						continue;
-					_add_ecc_count(&ecc_aid, &ecc_xcd);
-				}
-				_add_ecc_count(&ecc, &ecc_aid);
-			}
-		}
-	} else {
-		for (skt = 0; skt < aca_blk->ecc.socket_num_per_hive; skt++) {
-			for (aid = 0; aid < aca_blk->ecc.socket[skt].aid_num; aid++) {
-				memset(&ecc_aid, 0, sizeof(ecc_aid));
-				if (ras_aca_get_block_each_aid_ecc_count(ras_core,
-						blk, skt, aid, 0, &ecc_aid))
-					continue;
-				_add_ecc_count(&ecc, &ecc_aid);
-			}
-		}
-	}
+	__get_block_socket_ecc_count(ras_core, blk, &aca_blk->ecc.socket, &ecc);
 
 	err_data->new_ce_count = ecc.new_ce_count;
 	err_data->total_ce_count = ecc.total_ce_count;
@@ -605,36 +597,34 @@ int ras_aca_sw_init(struct ras_core_context *ras_core)
 	struct ras_aca *ras_aca = &ras_core->ras_aca;
 	struct ras_aca_config *aca_cfg = &ras_core->config->aca_cfg;
 	struct aca_block *aca_blk;
-	uint32_t socket_num_per_hive;
 	uint32_t aid_num_per_socket;
 	uint32_t xcd_num_per_aid;
-	int blk, skt, aid;
+	int blk, aid;
 
-	socket_num_per_hive = aca_cfg->socket_num_per_hive;
 	aid_num_per_socket = aca_cfg->aid_num_per_socket;
 	xcd_num_per_aid = aca_cfg->xcd_num_per_aid;
 
 	if (!xcd_num_per_aid || !aid_num_per_socket ||
-		(socket_num_per_hive > MAX_SOCKET_NUM_PER_HIVE) ||
 	    (aid_num_per_socket > MAX_AID_NUM_PER_SOCKET) ||
 	    (xcd_num_per_aid > MAX_XCD_NUM_PER_AID)) {
-		RAS_DEV_ERR(ras_core->dev, "Invalid ACA system configuration: %d, %d, %d\n",
-			socket_num_per_hive, aid_num_per_socket, xcd_num_per_aid);
+		RAS_DEV_ERR(ras_core->dev, "Invalid ACA system configuration: %d, %d\n",
+			aid_num_per_socket, xcd_num_per_aid);
 		return -EINVAL;
 	}
 
 	memset(ras_aca, 0, sizeof(*ras_aca));
 
+	ras_aca->aca_blk = kcalloc(RAS_BLOCK_ID__LAST, sizeof(struct aca_block), GFP_KERNEL);
+	if (!ras_aca->aca_blk)
+		return -ENOMEM;
+
 	for (blk = 0; blk < RAS_BLOCK_ID__LAST; blk++) {
 		aca_blk = &ras_aca->aca_blk[blk];
-		aca_blk->ecc.socket_num_per_hive = socket_num_per_hive;
-		for (skt = 0; skt < aca_blk->ecc.socket_num_per_hive; skt++) {
-			aca_blk->ecc.socket[skt].aid_num = aid_num_per_socket;
-			if (blk == RAS_BLOCK_ID__GFX) {
-				for (aid = 0; aid < aca_blk->ecc.socket[skt].aid_num; aid++)
-					aca_blk->ecc.socket[skt].aid[aid].xcd.xcd_num =
-								xcd_num_per_aid;
-			}
+		aca_blk->ecc.socket.aid_num = aid_num_per_socket;
+		if (blk == RAS_BLOCK_ID__GFX) {
+			for (aid = 0; aid < aca_blk->ecc.socket.aid_num; aid++)
+				aca_blk->ecc.socket.aid[aid].xcd.xcd_num =
+							xcd_num_per_aid;
 		}
 	}
 
@@ -648,6 +638,7 @@ int ras_aca_sw_fini(struct ras_core_context *ras_core)
 {
 	struct ras_aca *ras_aca = &ras_core->ras_aca;
 
+	kfree(ras_aca->aca_blk);
 	mutex_destroy(&ras_aca->aca_lock);
 	mutex_destroy(&ras_aca->bank_op_lock);
 
@@ -671,6 +662,7 @@ int ras_aca_hw_init(struct ras_core_context *ras_core)
 		aca_blk->blk_info = ip_func->block_info[i];
 	}
 
+	ras_aca->ip_func = ip_func;
 	ras_aca->ue_updated_mark = 0;
 
 	return 0;
@@ -684,3 +676,44 @@ int ras_aca_hw_fini(struct ras_core_context *ras_core)
 
 	return 0;
 }
+
+int ras_aca_parse_bank(struct ras_core_context *ras_core,
+				struct aca_bank_reg *bank,
+				struct aca_bank_ecc *ecc)
+{
+	struct aca_block *aca_blk;
+	int ret = -ENODATA;
+
+	if (!ras_core || !bank || !ecc)
+		return -EINVAL;
+
+	aca_blk = aca_get_bank_aca_block(ras_core, bank);
+	if (aca_blk)
+		ret = aca_parse_bank(ras_core, aca_blk, bank, ecc);
+
+	return ret;
+}
+
+u64 ras_aca_get_parser_caps(struct ras_core_context *ras_core)
+{
+	struct ras_aca *ras_aca = &ras_core->ras_aca;
+
+	if (ras_aca->ip_func && ras_aca->ip_func->aca_parse_ras_caps)
+		return ras_aca->ip_func->aca_parse_ras_caps(ras_core);
+
+	return 0;
+}
+
+int ras_aca_fill_rma_bank(struct ras_core_context *ras_core,
+			struct aca_bank_reg *bank)
+{
+	struct ras_aca *ras_aca = &ras_core->ras_aca;
+
+	if (!bank)
+		return -EINVAL;
+
+	if (ras_aca->ip_func && ras_aca->ip_func->fill_rma_bank)
+		return ras_aca->ip_func->fill_rma_bank(ras_core, bank);
+
+	return -EOPNOTSUPP;
+}
diff --git a/drivers/gpu/drm/amd/ras/core/aca.h b/drivers/gpu/drm/amd/ras/core/aca.h
index 3c1e691cd2ee03..e428bf53f3ea89 100644
--- a/drivers/gpu/drm/amd/ras/core/aca.h
+++ b/drivers/gpu/drm/amd/ras/core/aca.h
@@ -26,10 +26,8 @@
 #define __ACA_H__
 #include "ras.h"
 
-#define MAX_SOCKET_NUM_PER_HIVE 8
-#define MAX_AID_NUM_PER_SOCKET 4
-#define MAX_XCD_NUM_PER_AID 2
-#define MAX_ACA_RAS_BLOCK  20
+#define MAX_AID_NUM_PER_SOCKET 16
+#define MAX_XCD_NUM_PER_AID 4
 
 #define ACA_ERROR__UE_MASK			(0x1 << RAS_ERR_TYPE__UE)
 #define ACA_ERROR__CE_MASK			(0x1 << RAS_ERR_TYPE__CE)
@@ -53,7 +51,11 @@ struct ras_core_context;
 struct aca_block;
 
 struct aca_bank_reg {
+	uint64_t timestamp;
+	u32 bank_type;
 	u32 ecc_type;
+	u32 apic_id;
+	u32 bank;
 	u64 seq_no;
 	u64 regs[ACA_REG_MAX_COUNT];
 };
@@ -64,6 +66,14 @@ enum aca_ecc_hwip {
 	ACA_ECC_HWIP__UMC,
 	ACA_ECC_HWIP__SMU,
 	ACA_ECC_HWIP__PCS_XGMI,
+	ACA_ECC_HWIP__GFX,
+	ACA_ECC_HWIP__SDMA,
+	ACA_ECC_HWIP__MMHUB,
+	ACA_ECC_HWIP__MPIFOE,
+	ACA_ECC_HWIP__PCIE_PL,
+	ACA_ECC_HWIP__DACC_BE,
+	ACA_ECC_HWIP__UCIE_PCS,
+	ACA_ECC_HWIP__LSDMA,
 	ACA_ECC_HWIP_COUNT,
 };
 
@@ -73,12 +83,14 @@ struct aca_ecc_info {
 	int xcd_id;
 	int hwid;
 	int mcatype;
+	bool xcd_valid;
 	uint64_t status;
 	uint64_t ipid;
 	uint64_t addr;
 };
 
 struct aca_bank_ecc {
+	u32 block_id;
 	struct aca_ecc_info bank_info;
 	u32 ce_count;
 	u32 ue_count;
@@ -99,13 +111,11 @@ struct aca_xcd_ecc {
 };
 
 struct aca_aid_ecc {
-	union {
-		struct aca_xcd {
-			struct aca_xcd_ecc xcd[MAX_XCD_NUM_PER_AID];
-			u32 xcd_num;
-		} xcd;
-		struct aca_ecc_count ecc_err;
-	};
+	struct aca_xcd {
+		struct aca_xcd_ecc xcd[MAX_XCD_NUM_PER_AID];
+		u32 xcd_num;
+	} xcd;
+	struct aca_ecc_count ecc_err;
 };
 
 struct aca_socket_ecc {
@@ -114,8 +124,7 @@ struct aca_socket_ecc {
 };
 
 struct aca_block_ecc {
-	struct aca_socket_ecc socket[MAX_SOCKET_NUM_PER_HIVE];
-	u32 socket_num_per_hive;
+	struct aca_socket_ecc socket;
 };
 
 struct aca_bank_hw_ops {
@@ -140,6 +149,8 @@ struct aca_block {
 struct ras_aca_ip_func {
 	uint32_t block_num;
 	const struct aca_block_info **block_info;
+	u64 (*aca_parse_ras_caps)(struct ras_core_context *ras_core);
+	int (*fill_rma_bank)(struct ras_core_context *ras_core, struct aca_bank_reg *bank);
 };
 
 struct ras_aca {
@@ -147,7 +158,7 @@ struct ras_aca {
 	const struct ras_aca_ip_func *ip_func;
 	struct mutex  aca_lock;
 	struct mutex  bank_op_lock;
-	struct aca_block aca_blk[MAX_ACA_RAS_BLOCK];
+	struct aca_block *aca_blk;
 	uint32_t ue_updated_mark;
 };
 
@@ -161,4 +172,9 @@ int ras_aca_clear_all_blocks_ecc_count(struct ras_core_context *ras_core);
 int ras_aca_update_ecc(struct ras_core_context *ras_core, u32 ecc_type, void *data);
 void ras_aca_mark_fatal_flag(struct ras_core_context *ras_core);
 void ras_aca_clear_fatal_flag(struct ras_core_context *ras_core);
+int ras_aca_parse_bank(struct ras_core_context *ras_core,
+				struct aca_bank_reg *bank,
+				struct aca_bank_ecc *ecc);
+u64 ras_aca_get_parser_caps(struct ras_core_context *ras_core);
+int ras_aca_fill_rma_bank(struct ras_core_context *ras_core, struct aca_bank_reg *bank);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/aca_v1_0.c b/drivers/gpu/drm/amd/ras/core/aca_v1_0.c
index a8b413270de370..a4145f30f45f32 100644
--- a/drivers/gpu/drm/amd/ras/core/aca_v1_0.c
+++ b/drivers/gpu/drm/amd/ras/core/aca_v1_0.c
@@ -176,7 +176,7 @@ static bool aca_match_mmhub_bank(struct aca_block *aca_blk, void *data)
 
 static bool aca_check_umc_de(struct ras_core_context *ras_core, uint64_t mc_umc_status)
 {
-	return (ras_core->poison_supported &&
+	return (ras_core_poison_supported(ras_core) &&
 		    ACA_REG_STATUS_VAL(mc_umc_status) &&
 		    ACA_REG_STATUS_DEFERRED(mc_umc_status));
 }
@@ -372,7 +372,46 @@ static const struct aca_block_info *aca_block_info_v1_0[] = {
 	&aca_v1_0_xgmi,
 };
 
+static u64 aca_parse_ras_caps_v1_0(struct ras_core_context *ras_core)
+{
+	u64 parser_supported_mask = 0;
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(aca_block_info_v1_0); i++)
+		parser_supported_mask |=
+			BIT_ULL(aca_block_info_v1_0[i]->ras_block_id);
+
+	return parser_supported_mask;
+}
+
+static int aca_fill_rma_bank_v1_0(struct ras_core_context *ras_core, struct aca_bank_reg *bank)
+{
+	struct device_system_info dev_info = {0};
+	u64 socket_id;
+	int ret;
+
+	if (!bank)
+		return -EINVAL;
+
+	ret = ras_core_get_device_system_info(ras_core, &dev_info);
+	if (ret)
+		return ret;
+
+	socket_id = dev_info.socket_id;
+
+	memset(bank->regs, 0, sizeof(bank->regs));
+	bank->regs[ACA_REG_IDX__CTL]    = 0x1ULL;
+	bank->regs[ACA_REG_IDX__STATUS] = 0xB000000000000137ULL;
+	bank->regs[ACA_REG_IDX__CONFG]  = 0x1ff00000002ULL;
+	bank->regs[ACA_REG_IDX__IPID]   = 0x9600000000ULL |
+		((socket_id / 4) & 0x01) | (((socket_id % 4) & 0x3) << 44);
+
+	return 0;
+}
+
 const struct ras_aca_ip_func ras_aca_func_v1_0 = {
 	.block_num = ARRAY_SIZE(aca_block_info_v1_0),
 	.block_info = aca_block_info_v1_0,
+	.aca_parse_ras_caps = aca_parse_ras_caps_v1_0,
+	.fill_rma_bank = aca_fill_rma_bank_v1_0,
 };
diff --git a/drivers/gpu/drm/amd/ras/core/aca_v5_0.c b/drivers/gpu/drm/amd/ras/core/aca_v5_0.c
new file mode 100644
index 00000000000000..30fd1cdef94e8e
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/aca_v5_0.c
@@ -0,0 +1,448 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "ras.h"
+#include "aca.h"
+#include "core_status.h"
+#include "aca_v5_0.h"
+
+#define ACA_HWIP_MAP(ip, hwid, acatypes) \
+	{ACA_ECC_HWIP__##ip, hwid, acatypes, ARRAY_SIZE(acatypes)}
+
+#define ACA_GFX_XCD_HWID  0x2E2
+struct ras_aca_hwip_v5 {
+	enum aca_ecc_hwip hwip;
+	const u32 hwid;
+	const u32 *types;
+	u32 types_sz;
+};
+
+static const u32 smu_aca_types[] = {0x01};
+static const u32 pcs_xgmi_aca_types[] = {0x00};
+static const u32 umc_aca_types[] = {0x00};
+static const u32 gfx_aid_aca_types[] = {0x00, 0x01};
+static const u32 gfx_xcd_aca_types[] = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09,
+	0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x12, 0x13, 0x14,
+};
+static const u32 gfx_sdma_aca_types[] = {0x11};
+static const u32 sdma_aca_types[] =   {0x00};
+static const u32 mmhub_aca_types[] =  {0x00};
+static const u32 mpifoe_aca_types[] = {0x00};
+static const u32 pcie_pl_aca_types[] = {0x00};
+static const u32 dacc_be_aca_types[] = {0x00};
+static const u32 ucie_pcs_aca_types[] = {0x00};
+static const u32 lsdma_aca_types[] = {0x00};
+
+static struct ras_aca_hwip_v5 aca_hwip_maps[] = {
+	ACA_HWIP_MAP(SMU,      0x01,  smu_aca_types),
+	ACA_HWIP_MAP(PCS_XGMI, 0x50,  pcs_xgmi_aca_types),
+	ACA_HWIP_MAP(UMC,      0x96,  umc_aca_types),
+	ACA_HWIP_MAP(GFX,      0x0B,  gfx_aid_aca_types),
+	ACA_HWIP_MAP(GFX,      0x2E2, gfx_xcd_aca_types),
+	ACA_HWIP_MAP(SDMA,     0x2E2, gfx_sdma_aca_types),
+	ACA_HWIP_MAP(SDMA,     0x15A, sdma_aca_types),
+	ACA_HWIP_MAP(MMHUB,    0x73,  mmhub_aca_types),
+	ACA_HWIP_MAP(MPIFOE,   0x1FD, mpifoe_aca_types),
+	ACA_HWIP_MAP(PCIE_PL,  0x1E1, pcie_pl_aca_types),
+	ACA_HWIP_MAP(DACC_BE,  0x164, dacc_be_aca_types),
+	ACA_HWIP_MAP(UCIE_PCS, 0x16C, ucie_pcs_aca_types),
+	ACA_HWIP_MAP(LSDMA,    0x346, lsdma_aca_types),
+};
+
+static void aca_decode_bank_info(struct aca_block *aca_blk,
+			struct aca_bank_reg *bank, struct aca_ecc_info *info)
+{
+	u64 ipid;
+	u32 instidhi, instidlo;
+
+	ipid = bank->regs[ACA_REG_IDX__IPID];
+	info->hwid = ACA_V5_REG_IPID_HARDWAREID(ipid);
+	info->mcatype = ACA_V5_REG_IPID_ACATYPE(ipid);
+
+	instidhi = ACA_V5_REG_IPID_INSTANCEIDHI(ipid);
+	instidlo = ACA_V5_REG_IPID_INSTANCEIDLO(ipid);
+	info->die_id = instidhi & 0xF;
+	info->socket_id = instidlo & 0xFF;
+	/* xcd/aid are unique in die_id, not need per-XCD decode for accounting */
+	info->xcd_valid = false;
+}
+
+static bool aca_check_bank_hwip(struct aca_bank_reg *bank, enum aca_ecc_hwip type)
+{
+	struct ras_aca_hwip_v5 *hwip = NULL;
+	u32 hwid, acatype;
+	u64 ipid;
+	int i, t;
+
+	if (!bank || (type == ACA_ECC_HWIP__UNKNOWN))
+		return false;
+
+	ipid = bank->regs[ACA_REG_IDX__IPID];
+	hwid = ACA_V5_REG_IPID_HARDWAREID(ipid);
+	acatype = ACA_V5_REG_IPID_ACATYPE(ipid);
+
+	for (i = 0; i < ARRAY_SIZE(aca_hwip_maps); i++) {
+		if (aca_hwip_maps[i].hwid != hwid)
+			continue;
+
+		for (t = 0; t < aca_hwip_maps[i].types_sz; t++) {
+			if (aca_hwip_maps[i].types &&
+			    (aca_hwip_maps[i].types[t] == acatype)) {
+				hwip = &aca_hwip_maps[i];
+				goto out;
+			}
+		}
+	}
+
+out:
+	return hwip ? (hwip->hwip == type) : false;
+}
+
+static bool aca_match_bank_default(struct aca_block *aca_blk, void *data)
+{
+	return aca_check_bank_hwip((struct aca_bank_reg *)data, aca_blk->blk_info->hwip);
+}
+
+static bool aca_check_umc_de(struct ras_core_context *ras_core, u64 mc_umc_status)
+{
+	return (ras_core_poison_supported(ras_core) &&
+		    ACA_V5_REG_STATUS_VAL(mc_umc_status) &&
+		    ACA_V5_REG_STATUS_DEFERRED(mc_umc_status));
+}
+
+static bool aca_check_umc_ue(struct ras_core_context *ras_core, u64 mc_umc_status)
+{
+	if (aca_check_umc_de(ras_core, mc_umc_status))
+		return false;
+
+	return (ACA_V5_REG_STATUS_VAL(mc_umc_status) &&
+		    (ACA_V5_REG_STATUS_PCC(mc_umc_status) ||
+		     ACA_V5_REG_STATUS_UC(mc_umc_status) ||
+		     ACA_V5_REG_STATUS_TCC(mc_umc_status)));
+}
+
+static bool aca_check_umc_ce(struct ras_core_context *ras_core, u64 mc_umc_status)
+{
+	if (aca_check_umc_de(ras_core, mc_umc_status))
+		return false;
+
+	return (ACA_V5_REG_STATUS_VAL(mc_umc_status) &&
+		    (ACA_V5_REG_STATUS_CECC(mc_umc_status) ||
+		     (ACA_V5_REG_STATUS_UECC(mc_umc_status) &&
+		      ACA_V5_REG_STATUS_UC(mc_umc_status) == 0) ||
+		/* Identify data parity error in replay mode */
+		     ((ACA_V5_REG_STATUS_ERRORCODEEXT(mc_umc_status) == 0x5 ||
+		      ACA_V5_REG_STATUS_ERRORCODEEXT(mc_umc_status) == 0xb) &&
+		     !(aca_check_umc_ue(ras_core, mc_umc_status)))));
+}
+
+static int aca_parse_umc_bank(struct ras_core_context *ras_core,
+			struct aca_block *aca_blk, void *data, void *buf)
+{
+	struct aca_bank_reg *bank = (struct aca_bank_reg *)data;
+	struct aca_bank_ecc *err = (struct aca_bank_ecc *)buf;
+	u32 ext_error_code, misc0_errcnt;
+	u64 status;
+
+	if (!ras_core || !aca_blk || !data || !buf)
+		return -EINVAL;
+
+	status = bank->regs[ACA_REG_IDX__STATUS];
+	if (!ACA_V5_REG_STATUS_VAL(status))
+		return 0;
+
+	memset(err, 0, sizeof(*err));
+	aca_decode_bank_info(aca_blk, bank, &err->bank_info);
+	err->bank_info.status = bank->regs[ACA_REG_IDX__STATUS];
+	err->bank_info.ipid = bank->regs[ACA_REG_IDX__IPID];
+	err->bank_info.addr = bank->regs[ACA_REG_IDX__ADDR];
+
+	ext_error_code = ACA_V5_REG_STATUS_ERRORCODEEXT(status);
+	misc0_errcnt = ACA_V5_REG_MISC0_ERRCNT(bank->regs[ACA_REG_IDX__MISC0]);
+	if (bank->ecc_type == RAS_ERR_TYPE__MCE &&
+	    ACA_V5_REG_STATUS_DEFERRED(status)) {
+		err->de_count = 1;
+		return 0;
+	}
+
+	if (aca_check_umc_de(ras_core, status))
+		err->de_count = misc0_errcnt ? misc0_errcnt : 1;
+	else if (aca_check_umc_ue(ras_core, status))
+		err->ue_count = ext_error_code ? 1 : misc0_errcnt;
+	else if (aca_check_umc_ce(ras_core, status))
+		err->ce_count = ext_error_code ? 1 : misc0_errcnt;
+
+	if (bank->ecc_type != RAS_ERR_TYPE__MCE || err->de_count ||
+	    err->ue_count || err->ce_count)
+		return 0;
+
+	if (ACA_V5_REG_STATUS_DEFERRED(status))
+		err->de_count = 1;
+	else if (ACA_V5_REG_STATUS_UC(status) || ACA_V5_REG_STATUS_PCC(status) ||
+		 ACA_V5_REG_STATUS_TCC(status))
+		err->ue_count = 1;
+	else if (ACA_V5_REG_STATUS_CECC(status) || ACA_V5_REG_STATUS_UECC(status))
+		err->ce_count = 1;
+
+	return 0;
+}
+
+static bool aca_check_bank_is_de(struct ras_core_context *ras_core,
+				u64 status)
+{
+	return (ACA_V5_REG_STATUS_POISON(status) ||
+				ACA_V5_REG_STATUS_DEFERRED(status));
+}
+
+static int aca_parse_bank_default(struct ras_core_context *ras_core,
+				  struct aca_block *aca_blk,
+				  void *data, void *buf)
+{
+	struct aca_bank_reg *bank = (struct aca_bank_reg *)data;
+	struct aca_bank_ecc *err = (struct aca_bank_ecc *)buf;
+
+	u64 misc0, status;
+
+	if (!ras_core || !aca_blk || !data || !buf)
+		return -EINVAL;
+
+	misc0 = bank->regs[ACA_REG_IDX__MISC0];
+	status = bank->regs[ACA_REG_IDX__STATUS];
+
+	memset(err, 0, sizeof(*err));
+	aca_decode_bank_info(aca_blk, bank, &err->bank_info);
+	err->bank_info.status = status;
+	err->bank_info.ipid = bank->regs[ACA_REG_IDX__IPID];
+	err->bank_info.addr = bank->regs[ACA_REG_IDX__ADDR];
+
+	if (aca_check_bank_is_de(ras_core, status)) {
+		err->de_count = 1;
+	} else {
+		if (bank->ecc_type == RAS_ERR_TYPE__UE)
+			err->ue_count = 1;
+		else if ((bank->ecc_type == RAS_ERR_TYPE__CE) ||
+			(bank->ecc_type == RAS_ERR_TYPE__MCE))
+			err->ce_count = ACA_V5_REG_MISC0_ERRCNT(misc0);
+	}
+
+	return 0;
+}
+
+static int aca_parse_xgmi_bank(struct ras_core_context *ras_core,
+			       struct aca_block *aca_blk,
+			       void *data, void *buf)
+{
+	struct aca_bank_reg *bank = (struct aca_bank_reg *)data;
+	struct aca_bank_ecc *err = (struct aca_bank_ecc *)buf;
+	u64 status, count;
+	int ext_error_code;
+
+	if (!ras_core || !aca_blk || !data || !buf)
+		return -EINVAL;
+
+	memset(err, 0, sizeof(*err));
+	aca_decode_bank_info(aca_blk, bank, &err->bank_info);
+	err->bank_info.status = bank->regs[ACA_REG_IDX__STATUS];
+	err->bank_info.ipid = bank->regs[ACA_REG_IDX__IPID];
+	err->bank_info.addr = bank->regs[ACA_REG_IDX__ADDR];
+
+	status = bank->regs[ACA_REG_IDX__STATUS];
+	ext_error_code = ACA_V5_REG_STATUS_ERRORCODEEXT(status);
+
+	count = ACA_V5_REG_MISC0_ERRCNT(bank->regs[ACA_REG_IDX__MISC0]);
+	if (bank->ecc_type == RAS_ERR_TYPE__UE) {
+		if (ext_error_code != 0 && ext_error_code != 1 && ext_error_code != 9)
+			count = 0ULL;
+		err->ue_count = count;
+	} else if ((bank->ecc_type == RAS_ERR_TYPE__CE) ||
+		(bank->ecc_type == RAS_ERR_TYPE__MCE)) {
+		count = ext_error_code == 6 ? count : 0ULL;
+		err->ce_count = count;
+	}
+
+	return 0;
+}
+
+static const struct aca_block_info aca_v5_0_umc = {
+	.name = "umc",
+	.ras_block_id = RAS_BLOCK_ID__UMC,
+	.hwip = ACA_ECC_HWIP__UMC,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK | ACA_ERROR__DE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_umc_bank,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_gfx = {
+	.name = "gfx",
+	.ras_block_id = RAS_BLOCK_ID__GFX,
+	.hwip = ACA_ECC_HWIP__GFX,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_sdma = {
+	.name = "sdma",
+	.ras_block_id = RAS_BLOCK_ID__SDMA,
+	.hwip = ACA_ECC_HWIP__SDMA,
+	.mask = ACA_ERROR__UE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_mmhub = {
+	.name = "mmhub",
+	.ras_block_id = RAS_BLOCK_ID__MMHUB,
+	.hwip = ACA_ECC_HWIP__MMHUB,
+	.mask = ACA_ERROR__UE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_xgmi = {
+	.name = "xgmi",
+	.ras_block_id = RAS_BLOCK_ID__XGMI_WAFL,
+	.hwip = ACA_ECC_HWIP__PCS_XGMI,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_xgmi_bank,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_pcie_pl = {
+	.name = "pcie_pl",
+	.ras_block_id = RAS_BLOCK_ID__PCIE_PL,
+	.hwip = ACA_ECC_HWIP__PCIE_PL,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_dacc_be = {
+	.name = "dacc_be",
+	.ras_block_id = RAS_BLOCK_ID__DACC_BE,
+	.hwip = ACA_ECC_HWIP__DACC_BE,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_ucie_pcs = {
+	.name = "ucie_pcs",
+	.ras_block_id = RAS_BLOCK_ID__UCIE_PCS,
+	.hwip = ACA_ECC_HWIP__UCIE_PCS,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_lsdma = {
+	.name = "lsdma",
+	.ras_block_id = RAS_BLOCK_ID__LSDMA,
+	.hwip = ACA_ECC_HWIP__LSDMA,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info aca_v5_0_pcs_xgmi = {
+	.name = "pcs_xgmi",
+	.ras_block_id = RAS_BLOCK_ID__PCS_XGMI,
+	.hwip = ACA_ECC_HWIP__PCS_XGMI,
+	.mask = ACA_ERROR__UE_MASK | ACA_ERROR__CE_MASK,
+	.bank_ops = {
+		.bank_match = aca_match_bank_default,
+		.bank_parse = aca_parse_bank_default,
+	},
+};
+
+static const struct aca_block_info *aca_block_info_v5_0[] = {
+	&aca_v5_0_umc,
+	&aca_v5_0_gfx,
+	&aca_v5_0_sdma,
+	&aca_v5_0_mmhub,
+	&aca_v5_0_xgmi,
+	&aca_v5_0_pcie_pl,
+	&aca_v5_0_dacc_be,
+	&aca_v5_0_ucie_pcs,
+	&aca_v5_0_lsdma,
+	&aca_v5_0_pcs_xgmi,
+};
+
+static u64 aca_parse_ras_caps_v5_0(struct ras_core_context *ras_core)
+{
+	u64 parser_supported_mask = 0;
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(aca_block_info_v5_0); i++)
+		parser_supported_mask |=
+			BIT_ULL(aca_block_info_v5_0[i]->ras_block_id);
+
+	return parser_supported_mask;
+}
+
+static int aca_fill_rma_bank_v5_0(struct ras_core_context *ras_core, struct aca_bank_reg *bank)
+{
+	struct device_system_info dev_info = {0};
+	int ret;
+
+	if (!bank)
+		return -EINVAL;
+
+	ret = ras_core_get_device_system_info(ras_core, &dev_info);
+	if (ret)
+		return ret;
+
+	memset(bank->regs, 0, sizeof(bank->regs));
+	bank->regs[ACA_REG_IDX__CTL]    = 0x1ULL;
+	bank->regs[ACA_REG_IDX__STATUS] = 0xB000000000000137ULL;
+	bank->regs[ACA_REG_IDX__CONFG]  = 0x1ff00000002ULL;
+	bank->regs[ACA_REG_IDX__IPID]   = 0x9600000000ULL | (dev_info.socket_id & 0xFF);
+
+	return 0;
+}
+
+const struct ras_aca_ip_func ras_aca_func_v5_0 = {
+	.block_num = ARRAY_SIZE(aca_block_info_v5_0),
+	.block_info = aca_block_info_v5_0,
+	.aca_parse_ras_caps = aca_parse_ras_caps_v5_0,
+	.fill_rma_bank = aca_fill_rma_bank_v5_0,
+};
diff --git a/drivers/gpu/drm/amd/ras/core/aca_v5_0.h b/drivers/gpu/drm/amd/ras/core/aca_v5_0.h
new file mode 100644
index 00000000000000..7f2c296de0b267
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/aca_v5_0.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __ACA_V5_0_H__
+#define __ACA_V5_0_H__
+#include "ras.h"
+
+#define ACA_V5_REG__FIELD(x, h, l)          (((x) & GENMASK_ULL(h, l)) >> l)
+#define ACA_V5_REG_STATUS_VAL(x)            ACA_V5_REG__FIELD(x, 63, 63)
+#define ACA_V5_REG_STATUS_OVERFLOW(x)       ACA_V5_REG__FIELD(x, 62, 62)
+#define ACA_V5_REG_STATUS_UC(x)             ACA_V5_REG__FIELD(x, 61, 61)
+#define ACA_V5_REG_STATUS_EN(x)             ACA_V5_REG__FIELD(x, 60, 60)
+#define ACA_V5_REG_STATUS_MISCV(x)          ACA_V5_REG__FIELD(x, 59, 59)
+#define ACA_V5_REG_STATUS_ADDRV(x)          ACA_V5_REG__FIELD(x, 58, 58)
+#define ACA_V5_REG_STATUS_PCC(x)            ACA_V5_REG__FIELD(x, 57, 57)
+#define ACA_V5_REG_STATUS_ERRCOREIDVAL(x)   ACA_V5_REG__FIELD(x, 56, 56)
+#define ACA_V5_REG_STATUS_TCC(x)            ACA_V5_REG__FIELD(x, 55, 55)
+#define ACA_V5_REG_STATUS_SYNDV(x)          ACA_V5_REG__FIELD(x, 53, 53)
+#define ACA_V5_REG_STATUS_CECC(x)           ACA_V5_REG__FIELD(x, 46, 46)
+#define ACA_V5_REG_STATUS_UECC(x)           ACA_V5_REG__FIELD(x, 45, 45)
+#define ACA_V5_REG_STATUS_DEFERRED(x)       ACA_V5_REG__FIELD(x, 44, 44)
+#define ACA_V5_REG_STATUS_POISON(x)         ACA_V5_REG__FIELD(x, 43, 43)
+#define ACA_V5_REG_STATUS_SCRUB(x)          ACA_V5_REG__FIELD(x, 40, 40)
+#define ACA_V5_REG_STATUS_ERRCOREID(x)      ACA_V5_REG__FIELD(x, 37, 32)
+#define ACA_V5_REG_STATUS_ADDRLSB(x)        ACA_V5_REG__FIELD(x, 29, 24)
+#define ACA_V5_REG_STATUS_ERRORCODEEXT(x)   ACA_V5_REG__FIELD(x, 21, 16)
+#define ACA_V5_REG_STATUS_ERRORCODE(x)      ACA_V5_REG__FIELD(x, 15, 0)
+
+#define ACA_V5_REG_IPID_ACATYPE(x)          ACA_V5_REG__FIELD(x, 63, 48)
+#define ACA_V5_REG_IPID_INSTANCEIDHI(x)     ACA_V5_REG__FIELD(x, 47, 44)
+#define ACA_V5_REG_IPID_HARDWAREID(x)       ACA_V5_REG__FIELD(x, 43, 32)
+#define ACA_V5_REG_IPID_INSTANCEIDLO(x)     ACA_V5_REG__FIELD(x, 31, 0)
+
+#define ACA_V5_REG_MISC0_VALID(x)           ACA_V5_REG__FIELD(x, 63, 63)
+#define ACA_V5_REG_MISC0_OVRFLW(x)          ACA_V5_REG__FIELD(x, 48, 48)
+#define ACA_V5_REG_MISC0_ERRCNT(x)          ACA_V5_REG__FIELD(x, 43, 32)
+
+#define ACA_V5_REG_SYND_ERRORINFORMATION(x)	ACA_V5_REG__FIELD(x, 17, 0)
+
+extern const struct ras_aca_ip_func ras_aca_func_v5_0;
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/cmd.c b/drivers/gpu/drm/amd/ras/core/cmd.c
index 446b935c1fc915..56fcd180b5654e 100644
--- a/drivers/gpu/drm/amd/ras/core/cmd.c
+++ b/drivers/gpu/drm/amd/ras/core/cmd.c
@@ -43,7 +43,8 @@ static int ras_get_block_ecc_info(struct ras_core_context *ras_core,
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	memset(&err_data, 0, sizeof(err_data));
-	ret = ras_aca_get_block_ecc_count(ras_core, input_data->block_id, &err_data);
+	ret = ras_core_query_block_ecc_data(ras_core,
+				input_data->block_id, &err_data, false);
 	if (ret)
 		return RAS_CMD__ERROR_GENERIC;
 
@@ -143,13 +144,8 @@ static int ras_cmd_clear_bad_page_info(struct ras_core_context *ras_core,
 	if (cmd->input_size != sizeof(struct ras_cmd_dev_handle))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
-	if (ras_fw_eeprom_supported(ras_core)) {
-		if (ras_fw_eeprom_reset_table(ras_core))
-			return RAS_CMD__ERROR_GENERIC;
-	} else {
-		if (ras_eeprom_reset_table(ras_core))
-			return RAS_CMD__ERROR_GENERIC;
-	}
+	if (ras_eeprom_mgr_reset_table(ras_core))
+		return RAS_CMD__ERROR_GENERIC;
 
 	if (ras_umc_clean_badpage_data(ras_core))
 		return RAS_CMD__ERROR_GENERIC;
@@ -178,16 +174,18 @@ static int ras_cmd_get_cper_snapshot(struct ras_core_context *ras_core,
 	struct ras_cmd_cper_snapshot_rsp *output_data =
 			(struct ras_cmd_cper_snapshot_rsp *)cmd->output_buff_raw;
 	struct ras_log_batch_overview overview;
+	u64 latest_cper_id;
 
 	if ((cmd->input_size != sizeof(struct ras_cmd_cper_snapshot_req)) ||
 		(cmd->output_buf_size < sizeof(*output_data)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	ras_log_ring_get_batch_overview(ras_core, &overview);
+	latest_cper_id = overview.logged_batch_count ? overview.last_batch_id - 1 : 0;
 
 	output_data->total_cper_num = overview.logged_batch_count;
 	output_data->start_cper_id = overview.first_batch_id;
-	output_data->latest_cper_id = overview.last_batch_id;
+	output_data->latest_cper_id = latest_cper_id;
 
 	output_data->version = 0;
 
@@ -202,13 +200,13 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core,
 			(struct ras_cmd_cper_record_req *)cmd->input_buff_raw;
 	struct ras_cmd_cper_record_rsp *rsp =
 			(struct ras_cmd_cper_record_rsp *)cmd->output_buff_raw;
-	struct ras_log_info *trace = NULL;
-	uint32_t trace_count = MAX_RECORD_PER_BATCH;
+	struct ras_log_info *batch_logs = NULL;
+	uint32_t nr_batch_logs = MAX_RECORD_PER_BATCH;
 	struct ras_log_batch_overview overview;
 	uint32_t offset = 0, real_data_len = 0;
-	uint64_t batch_id;
-	uint8_t *buffer = NULL;
-	int ret = 0, i, count;
+	u64 batch_id, start_batch_id;
+	uint8_t *buf_ptr = (uint8_t *)(uintptr_t)req->buf_ptr;
+	int ret = 0, i, count, read_batch_count = 0;
 
 	if ((cmd->input_size != sizeof(struct ras_cmd_cper_record_req)) ||
 		(cmd->output_buf_size < sizeof(*rsp)))
@@ -218,42 +216,50 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core,
 	    req->buf_size > RAS_CMD_MAX_CPER_BUF_SZ)
 		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
 
-	buffer = kzalloc(req->buf_size, GFP_KERNEL);
-	if (!buffer)
-		return RAS_CMD__ERROR_GENERIC;
-
-	trace = kzalloc_objs(*trace, trace_count);
-	if (!trace) {
+	batch_logs = kcalloc(nr_batch_logs, sizeof(*batch_logs), GFP_KERNEL);
+	if (!batch_logs) {
 		ret = RAS_CMD__ERROR_GENERIC;
 		goto out;
 	}
 
 	ras_log_ring_get_batch_overview(ras_core, &overview);
+
+	start_batch_id = req->cper_start_id;
+
 	for (i = 0; i < req->cper_num; i++) {
-		batch_id = req->cper_start_id + i;
+		batch_id = start_batch_id + i;
 		if (batch_id >= overview.last_batch_id)
 			break;
 
-		count = ras_log_ring_get_batch_records(ras_core, batch_id, trace,
-					trace_count);
+		count = ras_log_ring_get_batch_records(ras_core, batch_id, batch_logs,
+					nr_batch_logs);
 		if (count > 0) {
-			ret = ras_cper_generate_cper(ras_core, trace, count,
-					&buffer[offset], req->buf_size - offset, &real_data_len);
+			if (offset >= req->buf_size) {
+				ret = -ENOMEM;
+				break;
+			}
+
+			ret = ras_cper_generate_batch_cper(ras_core, batch_logs, count,
+					&buf_ptr[offset], req->buf_size - offset, &real_data_len);
 			if (ret)
 				break;
 
 			offset += real_data_len;
 		}
+
+		/* The caller resumes at cper_start_id + real_cper_num, so an id
+		 * that held nothing still has to be counted here.
+		 */
+		read_batch_count++;
 	}
 
-	if ((ret && (ret != -ENOMEM)) ||
-		copy_to_user(u64_to_user_ptr(req->buf_ptr), buffer, offset)) {
+	if ((ret && (ret != -ENOMEM))) {
 		ret = RAS_CMD__ERROR_GENERIC;
 		goto out;
 	}
 
 	rsp->real_data_size = offset;
-	rsp->real_cper_num = i;
+	rsp->real_cper_num = read_batch_count;
 	rsp->remain_num = (ret == -ENOMEM) ? (req->cper_num - i) : 0;
 	rsp->version = 0;
 
@@ -261,8 +267,7 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core,
 	ret = RAS_CMD__SUCCESS;
 
 out:
-	kfree(trace);
-	kfree(buffer);
+	kfree(batch_logs);
 	return ret;
 }
 
@@ -337,8 +342,8 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core,
 				record->seqno = trace_arry[j].seqno;
 				record->timestamp = trace_arry[j].timestamp;
 				record->event = trace_arry[j].event;
-				memcpy(&record->aca_reg,
-					&trace_arry[j].aca_reg, sizeof(trace_arry[j].aca_reg));
+				memcpy(&record->body, &trace_arry[j].body,
+				       sizeof(record->body));
 			}
 		} else {
 			count = 0;
@@ -362,48 +367,6 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core,
 	return RAS_CMD__SUCCESS;
 }
 
-static enum ras_ta_block __get_ras_ta_block(enum ras_block_id block)
-{
-	switch (block) {
-	case RAS_BLOCK_ID__UMC:
-		return RAS_TA_BLOCK__UMC;
-	case RAS_BLOCK_ID__SDMA:
-		return RAS_TA_BLOCK__SDMA;
-	case RAS_BLOCK_ID__GFX:
-		return RAS_TA_BLOCK__GFX;
-	case RAS_BLOCK_ID__MMHUB:
-		return RAS_TA_BLOCK__MMHUB;
-	case RAS_BLOCK_ID__ATHUB:
-		return RAS_TA_BLOCK__ATHUB;
-	case RAS_BLOCK_ID__PCIE_BIF:
-		return RAS_TA_BLOCK__PCIE_BIF;
-	case RAS_BLOCK_ID__HDP:
-		return RAS_TA_BLOCK__HDP;
-	case RAS_BLOCK_ID__XGMI_WAFL:
-		return RAS_TA_BLOCK__XGMI_WAFL;
-	case RAS_BLOCK_ID__DF:
-		return RAS_TA_BLOCK__DF;
-	case RAS_BLOCK_ID__SMN:
-		return RAS_TA_BLOCK__SMN;
-	case RAS_BLOCK_ID__SEM:
-		return RAS_TA_BLOCK__SEM;
-	case RAS_BLOCK_ID__MP0:
-		return RAS_TA_BLOCK__MP0;
-	case RAS_BLOCK_ID__MP1:
-		return RAS_TA_BLOCK__MP1;
-	case RAS_BLOCK_ID__FUSE:
-		return RAS_TA_BLOCK__FUSE;
-	case RAS_BLOCK_ID__MCA:
-		return RAS_TA_BLOCK__MCA;
-	case RAS_BLOCK_ID__VCN:
-		return RAS_TA_BLOCK__VCN;
-	case RAS_BLOCK_ID__JPEG:
-		return RAS_TA_BLOCK__JPEG;
-	default:
-		return RAS_TA_BLOCK__UMC;
-	}
-}
-
 static enum ras_ta_error_type __get_ras_ta_err_type(enum ras_ecc_err_type error)
 {
 	switch (error) {
@@ -431,7 +394,6 @@ static int ras_cmd_inject_error(struct ras_core_context *ras_core,
 		(struct ras_cmd_inject_error_rsp *)cmd->output_buff_raw;
 	int ret = 0;
 	struct ras_ta_trigger_error_input block_info = {
-		.block_id = __get_ras_ta_block(req->block_id),
 		.sub_block_index = req->subblock_id,
 		.inject_error_type = __get_ras_ta_err_type(req->error_type),
 		.address = req->address,
@@ -442,6 +404,12 @@ static int ras_cmd_inject_error(struct ras_core_context *ras_core,
 		(cmd->output_buf_size < sizeof(*output_data)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
+	ret = ras_psp_get_block_ta_id(ras_core, req->block_id, &block_info.block_id);
+	if (ret == -RAS_CORE_NOT_SUPPORTED)
+		return RAS_CMD__ERROR_UNSUPPORT;
+	else if (ret)
+		return ret;
+
 	ret = ras_psp_trigger_error(ras_core, &block_info, req->instance_mask);
 	if (!ret) {
 		output_data->version = 0;
@@ -455,6 +423,34 @@ static int ras_cmd_inject_error(struct ras_core_context *ras_core,
 	return ret;
 }
 
+static int ras_cmd_get_ras_cap(struct ras_core_context *ras_core,
+	struct ras_cmd_ctx *cmd, void *data)
+{
+	struct ras_cmd_get_ras_cap_rsp *output_data =
+			(struct ras_cmd_get_ras_cap_rsp *)cmd->output_buff_raw;
+
+	output_data->version = 0;
+	output_data->poison = ras_core_poison_supported(ras_core);
+	output_data->flex_mca = ras_psp_flex_mca_enabled(ras_core);
+	output_data->ras_block_mask = ras_core_get_ras_caps(ras_core);
+
+	output_data->ecc_type = 0;
+	if (output_data->poison)
+		output_data->ecc_type |= BIT_ULL(RAS_ECC_TYPE_POISON);
+
+	if (output_data->ras_block_mask & BIT_ULL(RAS_BLOCK_ID__UMC))
+		output_data->ecc_type |= BIT_ULL(RAS_ECC_TYPE_MEM);
+
+	if (output_data->ras_block_mask &
+		(BIT_ULL(RAS_BLOCK_ID__GFX) | BIT_ULL(RAS_BLOCK_ID__SDMA)))
+		output_data->ecc_type |= BIT_ULL(RAS_ECC_TYPE_SRAM);
+
+
+	cmd->output_size = sizeof(struct ras_cmd_get_ras_cap_rsp);
+
+	return 0;
+}
+
 static struct ras_cmd_func_map ras_cmd_maps[] = {
 	{RAS_CMD__INJECT_ERROR, ras_cmd_inject_error},
 	{RAS_CMD__GET_BLOCK_ECC_STATUS, ras_get_block_ecc_info},
@@ -465,6 +461,7 @@ static struct ras_cmd_func_map ras_cmd_maps[] = {
 	{RAS_CMD__GET_CPER_RECORD, ras_cmd_get_cper_records},
 	{RAS_CMD__GET_BATCH_TRACE_SNAPSHOT, ras_cmd_get_batch_trace_snapshot},
 	{RAS_CMD__GET_BATCH_TRACE_RECORD, ras_cmd_get_batch_trace_records},
+	{RAS_CMD__GET_RAS_CAP, ras_cmd_get_ras_cap},
 };
 
 int rascore_handle_cmd(struct ras_core_context *ras_core,
diff --git a/drivers/gpu/drm/amd/ras/core/cmd.h b/drivers/gpu/drm/amd/ras/core/cmd.h
index e3b39866455d85..d3f5ec3c8e5c66 100644
--- a/drivers/gpu/drm/amd/ras/core/cmd.h
+++ b/drivers/gpu/drm/amd/ras/core/cmd.h
@@ -77,6 +77,7 @@ enum ras_cmd_id {
 	RAS_CMD__SET_CMD_AUTO_UPDATE,
 	RAS_CMD__CHECK_ADDRESS_VALIDITY,
 	RAS_CMD__CONVERT_RETIRED_ADDRESS,
+	RAS_CMD__GET_RAS_CAP,
 	RAS_CMD__SUPPORTED_MAX = RAS_CMD_ID_COMMON_END,
 };
 
@@ -92,6 +93,13 @@ enum ras_cmd_response {
 	RAS_CMD__ERROR_ACCESS_DENIED,
 	RAS_CMD__ERROR_GENERIC,
 	RAS_CMD__ERROR_TIMEOUT,
+	RAS_CMD__ERROR_UNSUPPORT,
+};
+
+enum ras_ecc_type {
+	RAS_ECC_TYPE_MEM = 0,
+	RAS_ECC_TYPE_SRAM,
+	RAS_ECC_TYPE_POISON,
 };
 
 enum ras_error_type {
@@ -203,12 +211,12 @@ struct ras_cmd_inject_error_rsp {
 struct ras_cmd_dev_info {
 	uint64_t dev_handle;
 	uint32_t location_id;
-	uint32_t ecc_enabled;
-	uint32_t ecc_supported;
+	uint32_t ecc_type;
+	uint64_t block_mask_bits;
 	uint32_t vf_num;
 	uint32_t asic_type;
 	uint32_t oam_id;
-	uint32_t reserved[8];
+	uint32_t reserved[7];
 };
 
 struct ras_cmd_devices_info_rsp {
@@ -351,6 +359,8 @@ struct ras_cmd_cper_snapshot_rsp {
 	uint64_t latest_cper_id;
 };
 
+#define RAS_CMD_MAX_CPER_BUF_SZ	(2 * 1024U * 1024U) /* 2 MiB */
+
 struct ras_cmd_cper_record_req {
 	struct ras_cmd_dev_handle dev;
 	uint64_t cper_start_id;
@@ -444,7 +454,7 @@ struct ras_cmd_convert_retired_address_req {
 	uint32_t reserved[6];
 };
 
-#define RAS_CMD_MAX_RETIRED_ADDR_COUNT  32
+#define RAS_CMD_MAX_RETIRED_ADDR_COUNT  128
 struct ras_cmd_convert_retired_address_rsp {
 	uint32_t version;
 	uint32_t retired_count;
@@ -468,6 +478,20 @@ struct ras_cmd_blocks_ecc_rsp {
 	struct ras_cmd_block_ecc blocks[MAX_RAS_BLOCK_NUM];
 };
 
+struct ras_cmd_get_ras_cap_req {
+	struct ras_cmd_dev_handle dev;
+	uint32_t reserved[4];
+};
+
+struct ras_cmd_get_ras_cap_rsp {
+	uint32_t version;
+	uint32_t ecc_type;
+	uint32_t poison;
+	uint32_t flex_mca;
+	uint64_t ras_block_mask;
+	uint32_t reserved[8];
+};
+
 #pragma pack(pop)
 
 int ras_cmd_init(struct ras_core_context *ras_core);
diff --git a/drivers/gpu/drm/amd/ras/core/core.c b/drivers/gpu/drm/amd/ras/core/core.c
index 0e19cf41bc26ff..34353ac12b923c 100644
--- a/drivers/gpu/drm/amd/ras/core/core.c
+++ b/drivers/gpu/drm/amd/ras/core/core.c
@@ -52,6 +52,26 @@ static const char * const ras_block_name[] = {
 	"jpeg",
 	"ih",
 	"mpio",
+	"mmsch",
+	"mp5",
+	"atu",
+	"dacc_be",
+	"eclr",
+	"kpx_serdes",
+	"lsdma",
+	"mpart",
+	"mpifoe",
+	"mpras",
+	"nbif",
+	"nbio",
+	"oxrp",
+	"pcie_pl",
+	"pcs_xgmi",
+	"pie",
+	"cs",
+	"shub",
+	"ssbdci",
+	"ucie_pcs",
 };
 
 const char *ras_core_get_ras_block_name(enum ras_block_id block_id)
@@ -273,12 +293,9 @@ static int ras_core_eeprom_recovery(struct ras_core_context *ras_core)
 	int count;
 	int ret;
 
-	if (ras_fw_eeprom_supported(ras_core))
-		count = ras_fw_eeprom_get_record_count(ras_core);
-	else
-		count = ras_eeprom_get_record_count(ras_core);
-	if (!count)
-		return 0;
+	count = ras_eeprom_mgr_get_record_count(ras_core);
+	if (count <= 0)
+		return count;
 
 	/* Avoid bad page to be loaded again after gpu reset */
 	if (ras_umc_get_saved_eeprom_count(ras_core) >= count)
@@ -290,10 +307,8 @@ static int ras_core_eeprom_recovery(struct ras_core_context *ras_core)
 		return ret;
 	}
 
-	if (ras_fw_eeprom_supported(ras_core))
-		ras_fw_eeprom_sync_info(ras_core);
-	else
-		ras_eeprom_sync_info(ras_core);
+	if (ras_eeprom_mgr_get_gpu_op_status(ras_core) == RAS_GPU_OP_STATUS_RMA)
+		return -EPERM;
 
 	return ret;
 }
@@ -339,6 +354,12 @@ int ras_core_sw_init(struct ras_core_context *ras_core)
 		return -EINVAL;
 	}
 
+	ras_core->ras_eeprom_supported = ras_core->config->ras_eeprom_supported;
+	ras_core->poison_supported = ras_core->config->poison_supported;
+	ras_core->early_init_service_enabled = ras_core->config->early_init_service_supported;
+
+	ras_core->in_early_init = true;
+
 	ras_core->sys_fn = ras_core->config->sys_fn;
 	if (!ras_core->sys_fn)
 		return -EINVAL;
@@ -375,6 +396,22 @@ int ras_core_sw_init(struct ras_core_context *ras_core)
 	if (ret)
 		return ret;
 
+	ret = ras_mp1_sw_init(ras_core);
+	if (ret)
+		return ret;
+
+	ret = ras_eeprom_mgr_sw_init(ras_core);
+	if (ret)
+		return ret;
+
+	ret = ras_mce_sw_init(ras_core);
+	if (ret)
+		return ret;
+
+	ret = ras_cper_sw_init(ras_core);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
@@ -387,8 +424,11 @@ int ras_core_sw_fini(struct ras_core_context *ras_core)
 	ras_log_ring_sw_fini(ras_core);
 	ras_cmd_fini(ras_core);
 	ras_umc_sw_fini(ras_core);
+	ras_mp1_sw_fini(ras_core);
 	ras_aca_sw_fini(ras_core);
-
+	ras_eeprom_mgr_sw_fini(ras_core);
+	ras_mce_sw_fini(ras_core);
+	ras_cper_sw_fini(ras_core);
 	return 0;
 }
 
@@ -396,11 +436,6 @@ int ras_core_hw_init(struct ras_core_context *ras_core)
 {
 	int ret;
 
-	ras_core->ras_eeprom_supported =
-			ras_core->config->ras_eeprom_supported;
-
-	ras_core->poison_supported = ras_core->config->poison_supported;
-
 	ret = ras_psp_hw_init(ras_core);
 	if (ret)
 		return ret;
@@ -409,10 +444,6 @@ int ras_core_hw_init(struct ras_core_context *ras_core)
 	if (ret)
 		goto init_err1;
 
-	ret = ras_mp1_hw_init(ras_core);
-	if (ret)
-		goto init_err2;
-
 	ret = ras_nbio_hw_init(ras_core);
 	if (ret)
 		goto init_err3;
@@ -425,51 +456,52 @@ int ras_core_hw_init(struct ras_core_context *ras_core)
 	if (ret)
 		goto init_err5;
 
-	ras_fw_init_feature_flags(ras_core);
-
-	if (ras_fw_eeprom_supported(ras_core))
-		ret = ras_fw_eeprom_hw_init(ras_core);
-	else
-		ret = ras_eeprom_hw_init(ras_core);
-	if (ret)
-		goto init_err6;
-
-	ret = ras_core_eeprom_recovery(ras_core);
+	ret = ras_eeprom_mgr_hw_init(ras_core);
+
+	ras_core->in_early_init = false;
+
+	/*
+	 * NOTE:
+	 * For ras_fw_eeprom_hw_init, if it fails without control->ras_tbl_mutex
+	 * initialized, the ras_fw_eeprom_append won't be executed since ras_core
+	 * or sys_func or sys_func->mp1_send_eeprom_msg is null.
+	 *
+	 * For eeprom_hw_init, if it fails without control->ras_tbl_mutex
+	 * initialized, control->ras_max_record_count will be 0 and
+	 * ras_eeprom_append will return -EINVAL since
+	 * (num + control->ras_num_recs) > control->ras_max_record_count),
+	 * and control->ras_tbl_mutex won't be used anymore before
+	 * ras_eeprom_hw_fini
+	 * Thus skip ras_eeprom_hw_fini in failed path is harmless.
+	 */
 	if (ret) {
-		RAS_DEV_ERR(ras_core->dev,
-			"Failed to recovery ras core, ret:%d\n", ret);
-		goto init_err6;
+		RAS_DEV_WARN(ras_core->dev,
+			"RAS EEPROM init failed (%d), bad page persistence disabled\n",
+			ret);
+	} else {
+		ret = ras_core_eeprom_recovery(ras_core);
+		if (ret)
+			RAS_DEV_ERR(ras_core->dev,
+				"Failed to recovery ras core, ret:%d\n", ret);
 	}
 
-	if (ras_fw_eeprom_supported(ras_core))
-		ret = ras_fw_eeprom_check_storage_status(ras_core);
-	else
-		ret = ras_eeprom_check_storage_status(ras_core);
-	if (ret)
-		goto init_err6;
-
 	ret = ras_process_init(ras_core);
 	if (ret)
-		goto init_err7;
+		goto init_err6;
 
 	ras_core->is_initialized = true;
 
 	return 0;
 
-init_err7:
-	if (ras_fw_eeprom_supported(ras_core))
-		ras_fw_eeprom_hw_fini(ras_core);
-	else
-		ras_eeprom_hw_fini(ras_core);
 init_err6:
+	ras_eeprom_mgr_hw_fini(ras_core);
+
 	ras_gfx_hw_fini(ras_core);
 init_err5:
 	ras_umc_hw_fini(ras_core);
 init_err4:
 	ras_nbio_hw_fini(ras_core);
 init_err3:
-	ras_mp1_hw_fini(ras_core);
-init_err2:
 	ras_aca_hw_fini(ras_core);
 init_err1:
 	ras_psp_hw_fini(ras_core);
@@ -481,14 +513,10 @@ int ras_core_hw_fini(struct ras_core_context *ras_core)
 	ras_core->is_initialized = false;
 
 	ras_process_fini(ras_core);
-	if (ras_fw_eeprom_supported(ras_core))
-		ras_fw_eeprom_hw_fini(ras_core);
-	else
-		ras_eeprom_hw_fini(ras_core);
+	ras_eeprom_mgr_hw_fini(ras_core);
 	ras_gfx_hw_fini(ras_core);
 	ras_nbio_hw_fini(ras_core);
 	ras_umc_hw_fini(ras_core);
-	ras_mp1_hw_fini(ras_core);
 	ras_aca_hw_fini(ras_core);
 	ras_psp_hw_fini(ras_core);
 
@@ -514,21 +542,50 @@ int ras_core_handle_fatal_error(struct ras_core_context *ras_core)
 
 uint32_t ras_core_get_curr_nps_mode(struct ras_core_context *ras_core)
 {
-	if (!ras_core)
-		return 0;
+	int ret;
 
-	if (ras_core->ras_nbio.ip_func &&
-	    ras_core->ras_nbio.ip_func->get_memory_partition_mode)
-		return ras_core->ras_nbio.ip_func->get_memory_partition_mode(ras_core);
+	if (!ras_core->sys_fn || !ras_core->sys_fn->get_nps_mode) {
+		RAS_DEV_ERR(ras_core->dev, "Cannot get memory nps mode!\n");
+		return UMC_MEMORY_PARTITION_MODE_UNKNOWN;
+	}
 
-	RAS_DEV_ERR(ras_core->dev, "Failed to get gpu memory nps mode!\n");
-	return 0;
+	ret = ras_core->sys_fn->get_nps_mode(ras_core);
+	if (ret < 0) {
+		RAS_DEV_ERR(ras_core->dev, "Failed to get memory nps mode!\n");
+		return UMC_MEMORY_PARTITION_MODE_UNKNOWN;
+	} else if (!ret) {
+		RAS_DEV_WARN(ras_core->dev, "None nps mode!\n");
+	}
+
+	return ret;
+}
+
+uint32_t ras_core_get_vram_type(struct ras_core_context *ras_core)
+{
+	int ret;
+
+	if (!ras_core->sys_fn || !ras_core->sys_fn->get_vram_type) {
+		RAS_DEV_ERR(ras_core->dev, "Cannot get vram type!\n");
+		return UMC_VRAM_TYPE_UNKNOWN;
+	}
+
+	ret = ras_core->sys_fn->get_vram_type(ras_core);
+	if (ret < 0) {
+		RAS_DEV_ERR(ras_core->dev, "Failed to get vram type!\n");
+		return UMC_VRAM_TYPE_UNKNOWN;
+	} else if (!ret) {
+		RAS_DEV_WARN(ras_core->dev, "None vram type!\n");
+	}
+
+	return ret;
 }
 
 int ras_core_update_ecc_info(struct ras_core_context *ras_core)
 {
 	int ret;
 
+	ras_aca_update_ecc(ras_core, RAS_ERR_TYPE__MCE, NULL);
+
 	ret = ras_aca_update_ecc(ras_core, RAS_ERR_TYPE__CE, NULL);
 	if (!ret)
 		ret = ras_aca_update_ecc(ras_core, RAS_ERR_TYPE__UE, NULL);
@@ -537,7 +594,7 @@ int ras_core_update_ecc_info(struct ras_core_context *ras_core)
 }
 
 int ras_core_query_block_ecc_data(struct ras_core_context *ras_core,
-			enum ras_block_id block, struct ras_ecc_count *ecc_count)
+		enum ras_block_id block, struct ras_ecc_count *ecc_count, bool clear)
 {
 	int ret;
 
@@ -545,7 +602,7 @@ int ras_core_query_block_ecc_data(struct ras_core_context *ras_core,
 		return -EINVAL;
 
 	ret = ras_aca_get_block_ecc_count(ras_core, block, ecc_count);
-	if (!ret)
+	if (!ret && clear)
 		ras_aca_clear_block_new_ecc_count(ras_core, block);
 
 	return ret;
@@ -633,10 +690,7 @@ bool ras_core_is_ready(struct ras_core_context *ras_core)
 
 bool ras_core_check_safety_watermark(struct ras_core_context *ras_core)
 {
-	if (ras_fw_eeprom_supported(ras_core))
-		return ras_fw_eeprom_check_safety_watermark(ras_core);
-
-	return ras_eeprom_check_safety_watermark(ras_core);
+	return ras_eeprom_mgr_check_safety_watermark(ras_core);
 }
 
 int ras_core_down_trylock_gpu_reset_lock(struct ras_core_context *ras_core)
@@ -686,23 +740,14 @@ int ras_core_convert_soc_pa_to_cur_nps_pages(struct ras_core_context *ras_core,
 		uint64_t soc_pa, uint64_t *page_pfn, uint32_t max_pages)
 {
 	struct eeprom_umc_record record;
-	uint32_t cur_nps_mode;
-	int count = 0;
 
 	if (!ras_core || !page_pfn || !max_pages)
 		return -EINVAL;
 
-	cur_nps_mode = ras_core_get_curr_nps_mode(ras_core);
-	if (!cur_nps_mode || cur_nps_mode > UMC_MEMORY_PARTITION_MODE_NPS8)
-		return -EINVAL;
-
 	memset(&record, 0, sizeof(record));
 	record.cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(soc_pa);
 
-	count = ras_umc_convert_record_to_nps_pages(ras_core,
-				&record, cur_nps_mode, page_pfn, max_pages);
-
-	return count;
+	return ras_umc_convert_record_to_row_pages(ras_core, &record, page_pfn, max_pages);
 }
 
 int ras_core_check_address_sanity(struct ras_core_context *ras_core,
@@ -714,3 +759,119 @@ int ras_core_check_address_sanity(struct ras_core_context *ras_core,
 
 	return 0;
 }
+
+int ras_core_get_ip_version(struct ras_core_context *ras_core,
+	enum ras_unit_id unit_id, uint32_t *version)
+{
+	if (!version)
+		return -EINVAL;
+
+	switch (unit_id) {
+	case RAS_UNIT_ID_UMC:
+		*version = ras_core->ras_umc.umc_ip_version;
+		return 0;
+	case RAS_UNIT_ID_GFX:
+		*version = ras_core->ras_gfx.gfx_ip_version;
+		return 0;
+	case RAS_UNIT_ID_MP1:
+		*version = ras_core->ras_mp1.mp1_ip_version;
+		return 0;
+	case RAS_UNIT_ID_PSP:
+		*version = ras_core->ras_psp.psp_ip_version;
+		return 0;
+	case RAS_UNIT_ID_NBIO:
+		*version = ras_core->ras_nbio.nbio_ip_version;
+		return 0;
+	case RAS_UNIT_ID_ACA:
+		*version = ras_core->ras_aca.aca_ip_version;
+		return 0;
+	case RAS_UNIT_ID_EEPROM:
+		return ras_eeprom_mgr_get_version(ras_core, version);
+	default:
+		break;
+	}
+
+	return -RAS_CORE_NOT_SUPPORTED;
+}
+
+int ras_core_get_eeprom_version(struct ras_core_context *ras_core,
+	uint32_t *version)
+{
+	return ras_eeprom_mgr_get_version(ras_core, version);
+}
+
+uint64_t ras_core_get_ras_caps(struct ras_core_context *ras_core)
+{
+	uint64_t ras_hw_caps,  ras_drv_caps;
+	struct ras_module_param param = {0};
+
+	if (!ras_core)
+		return 0;
+
+	if (ras_core_get_module_param(ras_core, &param))
+		return 0;
+
+	if (!param.ras_feature_enable)
+		return 0;
+
+	ras_hw_caps = ras_psp_get_hw_ras_caps(ras_core);
+	ras_drv_caps = ras_aca_get_parser_caps(ras_core);
+
+	return ras_hw_caps & ras_drv_caps & param.ras_feature_mask;
+}
+
+bool ras_core_poison_supported(struct ras_core_context *ras_core)
+{
+	if (!ras_core)
+		return false;
+
+	/* For some ASICs, poison flag is detected externally by uniras module. */
+	return ras_core->poison_supported ? true :
+			ras_psp_poison_supported(ras_core);
+}
+
+bool ras_core_in_early_init(struct ras_core_context *ras_core)
+{
+	if (!ras_core)
+		return true;
+
+	return ras_core->in_early_init;
+}
+
+bool ras_core_early_init_service_enabled(struct ras_core_context *ras_core)
+{
+	return ras_core->early_init_service_enabled &&
+		ras_eeprom_mgr_early_init_service_supported(ras_core);
+}
+
+int ras_core_eeprom_early_init_service(struct ras_core_context *ras_core)
+{
+	if (!ras_core_early_init_service_enabled(ras_core))
+		return -EOPNOTSUPP;
+	else if (!ras_core_in_early_init(ras_core))
+		return -EACCES;
+
+	return ras_core_eeprom_recovery(ras_core);
+}
+
+int ras_core_get_module_param(struct ras_core_context *ras_core,
+		struct ras_module_param *param)
+{
+	if (!ras_core || !ras_core->config || !param)
+		return -EINVAL;
+
+	memcpy(param, &ras_core->config->mod_param, sizeof(*param));
+
+	return 0;
+}
+
+int ras_core_add_log_event(struct ras_core_context *ras_core,
+		uint32_t event, void *data, uint32_t data_sz)
+{
+	if (event >= RAS_LOG_EVENT_COUNT_MAX) {
+		RAS_DEV_ERR(ras_core->dev, "Invalid ras log event(0x%x)!\n", event);
+		return -EINVAL;
+	}
+
+	return ras_log_ring_add_log_event(ras_core, event, data, data_sz, NULL);
+}
diff --git a/drivers/gpu/drm/amd/ras/core/eeprom.c b/drivers/gpu/drm/amd/ras/core/eeprom.c
index 7afaac81f37a1c..e7aad113b98a5b 100644
--- a/drivers/gpu/drm/amd/ras/core/eeprom.c
+++ b/drivers/gpu/drm/amd/ras/core/eeprom.c
@@ -151,32 +151,26 @@
 	 (((_tbl_hdr)->tbl_size - RAS_TABLE_HEADER_SIZE - \
 	   RAS_TABLE_V2_1_INFO_SIZE) / RAS_TABLE_RECORD_SIZE))
 
-#define to_ras_core_context(x) (container_of(x, struct ras_core_context, ras_eeprom))
+#define to_ras_core_context(x) (container_of(x, struct ras_core_context, eeprom_mgr))
 
-static bool __is_ras_eeprom_supported(struct ras_core_context *ras_core)
-{
-	return ras_core->ras_eeprom_supported;
-}
+static int __check_ras_table(struct ras_core_context *ras_core,
+			struct ras_eeprom_control *control);
 
 static bool __get_eeprom_i2c_addr(struct ras_core_context *ras_core,
 				  struct ras_eeprom_control *control)
 {
-	int ret = -EINVAL;
-
-	if (control->sys_func &&
-		control->sys_func->update_eeprom_i2c_config)
-		ret = control->sys_func->update_eeprom_i2c_config(ras_core);
-	else
-		RAS_DEV_WARN(ras_core->dev,
-			"No eeprom i2c system config!\n");
+	if (!control->i2c_address) {
+		RAS_DEV_WARN(ras_core->dev, "Not config eeprom i2c address!\n");
+		return false;
+	}
 
-	return !ret ? true : false;
+	return true;
 }
 
 static int __ras_eeprom_xfer(struct ras_core_context *ras_core, u32 eeprom_addr,
 				u8 *eeprom_buf, u32 buf_size, bool read)
 {
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 	int ret;
 
 	if (control->sys_func && control->sys_func->eeprom_i2c_xfer) {
@@ -205,15 +199,12 @@ static int __ras_eeprom_xfer(struct ras_core_context *ras_core, u32 eeprom_addr,
 static int __eeprom_xfer(struct ras_core_context *ras_core, u32 eeprom_addr,
 			      u8 *eeprom_buf, u32 buf_size, bool read)
 {
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 	u16 limit;
 	u16 ps; /* Partial size */
 	int res = 0, r;
 
-	if (read)
-		limit = ras_core->ras_eeprom.max_read_len;
-	else
-		limit = ras_core->ras_eeprom.max_write_len;
-
+	limit = read ? control->max_read_len : control->max_write_len;
 	if (limit && (limit <= EEPROM_OFFSET_SIZE)) {
 		RAS_DEV_ERR(ras_core->dev,
 				"maddr:0x%04X size:0x%02X:quirk max_%s_len must be > %d",
@@ -294,7 +285,7 @@ __decode_table_header_from_buf(struct ras_eeprom_table_header *hdr,
 static int __write_table_header(struct ras_eeprom_control *control)
 {
 	u8 buf[RAS_TABLE_HEADER_SIZE];
-	struct ras_core_context *ras_core = to_ras_core_context(control);
+	struct ras_core_context *ras_core = to_ras_core_context(control->mgr);
 	int res;
 
 	memset(buf, 0, sizeof(buf));
@@ -333,22 +324,9 @@ __encode_table_ras_info_to_buf(struct ras_eeprom_table_ras_info *rai,
 	pp[0] = cpu_to_le32(tmp);
 }
 
-static void
-__decode_table_ras_info_from_buf(struct ras_eeprom_table_ras_info *rai,
-				 unsigned char *buf)
-{
-	u32 *pp = (uint32_t *)buf;
-	u32 tmp;
-
-	tmp = le32_to_cpu(pp[0]);
-	rai->rma_status = tmp & 0xFF;
-	rai->health_percent = (tmp >> 8) & 0xFF;
-	rai->ecc_page_threshold = (tmp >> 16) & 0xFFFF;
-}
-
 static int __write_table_ras_info(struct ras_eeprom_control *control)
 {
-	struct ras_core_context *ras_core = to_ras_core_context(control);
+	struct ras_core_context *ras_core = to_ras_core_context(control->mgr);
 	u8 *buf;
 	int res;
 
@@ -446,9 +424,9 @@ static void ras_set_eeprom_table_version(struct ras_eeprom_control *control)
 	hdr->version = RAS_TABLE_VER_V3;
 }
 
-int ras_eeprom_reset_table(struct ras_core_context *ras_core)
+static int ras_eeprom_reset_table(struct ras_core_context *ras_core)
 {
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 	struct ras_eeprom_table_header *hdr = &control->tbl_hdr;
 	struct ras_eeprom_table_ras_info *rai = &control->tbl_rai;
 	u8 csum;
@@ -556,38 +534,6 @@ __decode_table_record_from_buf(struct ras_eeprom_control *control,
 	record->retired_row_pfn = (le64_to_cpu(tmp) & 0xffffffffffff);
 }
 
-bool ras_eeprom_check_safety_watermark(struct ras_core_context *ras_core)
-{
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
-	bool ret = false;
-	int bad_page_count;
-
-	if (!__is_ras_eeprom_supported(ras_core) ||
-	    !control->record_threshold_config)
-		return false;
-
-	bad_page_count = ras_umc_get_badpage_count(ras_core);
-	if (control->tbl_hdr.header == RAS_TABLE_HDR_BAD) {
-		if (bad_page_count > control->record_threshold_count)
-			RAS_DEV_WARN(ras_core->dev, "RAS records:%d exceed threshold:%d",
-				bad_page_count, control->record_threshold_count);
-
-		if ((control->record_threshold_config == WARN_NONSTOP_OVER_THRESHOLD) ||
-			(control->record_threshold_config == NONSTOP_OVER_THRESHOLD)) {
-			RAS_DEV_WARN(ras_core->dev,
-				"Please consult AMD Service Action Guide (SAG) for appropriate service procedures.\n");
-			ret = false;
-		} else {
-			ras_core->is_rma = true;
-			RAS_DEV_WARN(ras_core->dev,
-				"Please consider adjusting the customized threshold.\n");
-			ret = true;
-		}
-	}
-
-	return ret;
-}
-
 /**
  * __ras_eeprom_write -- write indexed from buffer to EEPROM
  * @control: pointer to control structure
@@ -601,7 +547,7 @@ bool ras_eeprom_check_safety_watermark(struct ras_core_context *ras_core)
 static int __ras_eeprom_write(struct ras_eeprom_control *control,
 			      u8 *buf, const u32 fri, const u32 num)
 {
-	struct ras_core_context *ras_core = to_ras_core_context(control);
+	struct ras_core_context *ras_core = to_ras_core_context(control->mgr);
 	u32 buf_size;
 	int res;
 
@@ -738,36 +684,21 @@ static int ras_eeprom_append_table(struct ras_eeprom_control *control,
 
 static int ras_eeprom_update_header(struct ras_eeprom_control *control)
 {
-	struct ras_core_context *ras_core = to_ras_core_context(control);
-	int threshold_config = control->record_threshold_config;
+	struct ras_core_context *ras_core = to_ras_core_context(control->mgr);
 	u8 *buf, *pp, csum;
 	u32 buf_size;
-	int bad_page_count;
+	int record_count;
 	int res;
 
-	bad_page_count = ras_umc_get_badpage_count(ras_core);
-	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
-			      &bad_page_count);
-
+	record_count = ras_umc_get_saved_eeprom_count(ras_core);
 	/* Modify the header if it exceeds.
 	 */
-	if (threshold_config != 0 &&
-		bad_page_count > control->record_threshold_count) {
-		RAS_DEV_WARN(ras_core->dev,
-			"Saved bad pages %d reaches threshold value %d\n",
-			bad_page_count, control->record_threshold_count);
+	if (record_count >= control->record_threshold_count) {
 		control->tbl_hdr.header = RAS_TABLE_HDR_BAD;
 		if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1) {
 			control->tbl_rai.rma_status = RAS_GPU_RETIRED__ECC_REACH_THRESHOLD;
 			control->tbl_rai.health_percent = 0;
 		}
-
-		if ((threshold_config != WARN_NONSTOP_OVER_THRESHOLD) &&
-			(threshold_config != NONSTOP_OVER_THRESHOLD))
-			ras_core->is_rma = true;
-
-		/* ignore the -ENOTSUPP return value */
-		ras_core_event_notify(ras_core, RAS_EVENT_ID__DEVICE_RMA, NULL);
 	}
 
 	if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1)
@@ -808,11 +739,10 @@ static int ras_eeprom_update_header(struct ras_eeprom_control *control)
 	 * bad page records have been stored in eeprom,
 	 * now calculate gpu health percent
 	 */
-	if (threshold_config != 0 &&
-	    control->tbl_hdr.version >= RAS_TABLE_VER_V2_1 &&
-	    bad_page_count <= control->record_threshold_count)
+	if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1 &&
+	    record_count <= control->record_threshold_count)
 		control->tbl_rai.health_percent = ((control->record_threshold_count -
-			bad_page_count) * 100) / control->record_threshold_count;
+			record_count) * 100) / control->record_threshold_count;
 
 	/* Recalc the checksum.
 	 */
@@ -847,14 +777,14 @@ static int ras_eeprom_update_header(struct ras_eeprom_control *control)
  *
  * Return 0 on success or if EEPROM is not supported, -errno on error.
  */
-int ras_eeprom_append(struct ras_core_context *ras_core,
+static int ras_eeprom_append(struct ras_core_context *ras_core,
 			   struct eeprom_umc_record *record, const u32 num)
 {
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 	int res;
 
-	if (!__is_ras_eeprom_supported(ras_core))
-		return 0;
+	if (!control)
+		return -EINVAL;
 
 	if (num == 0) {
 		RAS_DEV_ERR(ras_core->dev, "will not append 0 records\n");
@@ -873,6 +803,9 @@ int ras_eeprom_append(struct ras_core_context *ras_core,
 
 	mutex_unlock(&control->ras_tbl_mutex);
 
+	if (!res)
+		res = __check_ras_table(ras_core, control);
+
 	return res;
 }
 
@@ -889,7 +822,7 @@ int ras_eeprom_append(struct ras_core_context *ras_core,
 static int __ras_eeprom_read(struct ras_eeprom_control *control,
 			     u8 *buf, const u32 fri, const u32 num)
 {
-	struct ras_core_context *ras_core = to_ras_core_context(control);
+	struct ras_core_context *ras_core = to_ras_core_context(control->mgr);
 	u32 buf_size;
 	int res;
 
@@ -916,16 +849,16 @@ static int __ras_eeprom_read(struct ras_eeprom_control *control,
 	return res;
 }
 
-int ras_eeprom_read(struct ras_core_context *ras_core,
-			 struct eeprom_umc_record *record, const u32 num)
+static int ras_eeprom_read(struct ras_core_context *ras_core,
+			struct eeprom_umc_record *record, u32 num)
 {
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 	int i, res;
 	u8 *buf, *pp;
 	u32 g0, g1;
 
-	if (!__is_ras_eeprom_supported(ras_core))
-		return 0;
+	if (!control)
+		return -EINVAL;
 
 	if (num == 0) {
 		RAS_DEV_ERR(ras_core->dev, "will not read 0 records\n");
@@ -1004,19 +937,6 @@ int ras_eeprom_read(struct ras_core_context *ras_core,
 	return res;
 }
 
-uint32_t ras_eeprom_max_record_count(struct ras_core_context *ras_core)
-{
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
-
-	/* get available eeprom table version first before eeprom table init */
-	ras_set_eeprom_table_version(control);
-
-	if (control->tbl_hdr.version >= RAS_TABLE_VER_V2_1)
-		return RAS_MAX_RECORD_COUNT_V2_1;
-	else
-		return RAS_MAX_RECORD_COUNT;
-}
-
 /**
  * __verify_ras_table_checksum -- verify the RAS EEPROM table checksum
  * @control: pointer to control structure
@@ -1029,7 +949,7 @@ uint32_t ras_eeprom_max_record_count(struct ras_core_context *ras_core)
  */
 static int __verify_ras_table_checksum(struct ras_eeprom_control *control)
 {
-	struct ras_core_context *ras_core = to_ras_core_context(control);
+	struct ras_core_context *ras_core = to_ras_core_context(control->mgr);
 	int buf_size, res;
 	u8  csum, *buf, *pp;
 
@@ -1070,59 +990,23 @@ static int __verify_ras_table_checksum(struct ras_eeprom_control *control)
 	return res < 0 ? res : csum;
 }
 
-static int __read_table_ras_info(struct ras_eeprom_control *control)
+static int __ras_table_init(struct ras_core_context *ras_core)
 {
-	struct ras_eeprom_table_ras_info *rai = &control->tbl_rai;
-	struct ras_core_context *ras_core = to_ras_core_context(control);
-	unsigned char *buf;
-	int res;
-
-	buf = kzalloc(RAS_TABLE_V2_1_INFO_SIZE, GFP_KERNEL);
-	if (!buf) {
-		RAS_DEV_ERR(ras_core->dev,
-			"Failed to alloc buf to read EEPROM table ras info\n");
-		return -ENOMEM;
-	}
-
-	/**
-	 * EEPROM table V2_1 supports ras info,
-	 * read EEPROM table ras info
-	 */
-	res = __eeprom_read(ras_core,
-			      control->i2c_address + control->ras_info_offset,
-			      buf, RAS_TABLE_V2_1_INFO_SIZE);
-	if (res < RAS_TABLE_V2_1_INFO_SIZE) {
-		RAS_DEV_ERR(ras_core->dev,
-			"Failed to read EEPROM table ras info, res:%d\n", res);
-		res = res >= 0 ? -EIO : res;
-		goto Out;
-	}
-
-	__decode_table_ras_info_from_buf(rai, buf);
-
-Out:
-	kfree(buf);
-	return res == RAS_TABLE_V2_1_INFO_SIZE ? 0 : res;
-}
-
-static int __check_ras_table_status(struct ras_core_context *ras_core)
-{
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 	unsigned char buf[RAS_TABLE_HEADER_SIZE] = { 0 };
 	struct ras_eeprom_table_header *hdr;
 	int res;
 
-	hdr = &control->tbl_hdr;
-
-	if (!__is_ras_eeprom_supported(ras_core))
-		return 0;
+	if (!control)
+		return -EINVAL;
 
 	if (!__get_eeprom_i2c_addr(ras_core, control))
 		return -EINVAL;
 
+	hdr = &control->tbl_hdr;
+
 	control->ras_header_offset = RAS_HDR_START;
 	control->ras_info_offset = RAS_TABLE_V2_1_INFO_START;
-	mutex_init(&control->ras_tbl_mutex);
 
 	/* Read the table header from EEPROM address */
 	res = __eeprom_read(ras_core,
@@ -1193,178 +1077,190 @@ static int __check_ras_table_status(struct ras_core_context *ras_core)
 	return 0;
 }
 
-int ras_eeprom_check_storage_status(struct ras_core_context *ras_core)
+static int ras_eeprom_sw_init(struct ras_core_context *ras_core,
+		struct ras_eeprom_param *param)
 {
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
-	struct ras_eeprom_table_header *hdr;
-	int bad_page_count;
-	int res = 0;
-
-	if (!__is_ras_eeprom_supported(ras_core))
-		return 0;
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	struct ras_eeprom_control *control;
 
-	if (!__get_eeprom_i2c_addr(ras_core, control))
+	if (!param)
 		return -EINVAL;
 
-	hdr = &control->tbl_hdr;
-
-	bad_page_count = ras_umc_get_badpage_count(ras_core);
-	if (hdr->header == RAS_TABLE_HDR_VAL) {
-		RAS_DEV_INFO(ras_core->dev,
-			"Found existing EEPROM table with %d records\n",
-			bad_page_count);
+	control = kzalloc(sizeof(*control), GFP_KERNEL);
+	if (!control)
+		return -ENOMEM;
 
-		if (hdr->version >= RAS_TABLE_VER_V2_1) {
-			res = __read_table_ras_info(control);
-			if (res)
-				return res;
-		}
+	mgr->ras_eeprom = control;
+	memset(control, 0, sizeof(*control));
 
-		res = __verify_ras_table_checksum(control);
-		if (res)
-			RAS_DEV_ERR(ras_core->dev,
-				"RAS table incorrect checksum or error:%d\n", res);
+	control->mgr = mgr;
+	control->max_read_len = param->max_i2c_read_len;
+	control->max_write_len = param->max_i2c_write_len;
+	control->i2c_adapter = param->eeprom_i2c_adapter;
+	control->i2c_port = param->eeprom_i2c_port;
+	control->i2c_address = param->eeprom_i2c_addr;
+	control->record_threshold_count = param->record_threshold;
+	control->sys_func = param->sys_fn;
 
-		/* Warn if we are at 90% of the threshold or above
-		 */
-		if (10 * bad_page_count >= 9 * control->record_threshold_count)
-			RAS_DEV_WARN(ras_core->dev,
-				"RAS records:%u exceeds 90%% of threshold:%d\n",
-				bad_page_count,
-				control->record_threshold_count);
-
-	} else if (hdr->header == RAS_TABLE_HDR_BAD &&
-		   control->record_threshold_config != 0) {
-		if (hdr->version >= RAS_TABLE_VER_V2_1) {
-			res = __read_table_ras_info(control);
-			if (res)
-				return res;
-		}
+	control->update_channel_flag = false;
 
-		res = __verify_ras_table_checksum(control);
-		if (res)
-			RAS_DEV_ERR(ras_core->dev,
-				"RAS Table incorrect checksum or error:%d\n", res);
-
-		if (control->record_threshold_count >= bad_page_count) {
-			/* This means that, the threshold was increased since
-			 * the last time the system was booted, and now,
-			 * ras->record_threshold_count - control->num_recs > 0,
-			 * so that at least one more record can be saved,
-			 * before the page count threshold is reached.
-			 */
-			RAS_DEV_INFO(ras_core->dev,
-				"records:%d threshold:%d, resetting RAS table header signature",
-				bad_page_count,
-				control->record_threshold_count);
-			res = ras_eeprom_correct_header_tag(control, RAS_TABLE_HDR_VAL);
-		} else {
-			RAS_DEV_ERR(ras_core->dev, "RAS records:%d exceed threshold:%d",
-				bad_page_count, control->record_threshold_count);
-			/* send the event when threshold is exceeded, and ignore the
-			 * return value here
-			 */
-			ras_core_event_notify(ras_core, RAS_EVENT_ID__DEVICE_RMA, NULL);
-
-			if ((control->record_threshold_config == WARN_NONSTOP_OVER_THRESHOLD) ||
-				(control->record_threshold_config == NONSTOP_OVER_THRESHOLD)) {
-				RAS_DEV_WARN(ras_core->dev,
-				"Please consult AMD Service Action Guide (SAG) for appropriate service procedures\n");
-				res = 0;
-			} else {
-				ras_core->is_rma = true;
-				RAS_DEV_ERR(ras_core->dev,
-				"User defined threshold is set, runtime service will be halt when threshold is reached\n");
-			}
-		}
-	}
+	mutex_init(&control->ras_tbl_mutex);
 
-	return res < 0 ? res : 0;
+	return 0;
 }
 
-int ras_eeprom_hw_init(struct ras_core_context *ras_core)
+static int ras_eeprom_sw_fini(struct ras_core_context *ras_core)
 {
-	struct ras_eeprom_control *control;
-	struct ras_eeprom_config *eeprom_cfg;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 
-	if (!ras_core)
+	if (!control)
 		return -EINVAL;
 
-	ras_core->is_rma = false;
+	mutex_destroy(&control->ras_tbl_mutex);
 
-	control = &ras_core->ras_eeprom;
+	kfree(control);
+	ras_core->eeprom_mgr.ras_eeprom = NULL;
 
-	memset(control, 0, sizeof(*control));
+	return 0;
+}
+
+static int ras_eeprom_hw_init(struct ras_core_context *ras_core,
+		struct ras_eeprom_param *param)
+{
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
+	int ret;
 
-	eeprom_cfg = &ras_core->config->eeprom_cfg;
-	control->record_threshold_config =
-		eeprom_cfg->eeprom_record_threshold_config;
+	if (!control)
+		return -EINVAL;
 
-	control->record_threshold_count = ras_eeprom_max_record_count(ras_core);
-	if (eeprom_cfg->eeprom_record_threshold_count <
-		control->record_threshold_count)
-		control->record_threshold_count =
-			eeprom_cfg->eeprom_record_threshold_count;
+	control->max_read_len = param->max_i2c_read_len;
+	control->max_write_len = param->max_i2c_write_len;
+	control->i2c_adapter = param->eeprom_i2c_adapter;
+	control->i2c_port = param->eeprom_i2c_port;
+	control->i2c_address = param->eeprom_i2c_addr;
 
-	control->sys_func = eeprom_cfg->eeprom_sys_fn;
-	control->max_read_len = eeprom_cfg->max_i2c_read_len;
-	control->max_write_len = eeprom_cfg->max_i2c_write_len;
-	control->i2c_adapter = eeprom_cfg->eeprom_i2c_adapter;
-	control->i2c_port = eeprom_cfg->eeprom_i2c_port;
-	control->i2c_address = eeprom_cfg->eeprom_i2c_addr;
+	ret = __ras_table_init(ras_core);
+	if (ret)
+		return ret;
 
-	control->update_channel_flag = false;
+	ret = __check_ras_table(ras_core, control);
+	if (ret)
+		RAS_DEV_WARN(ras_core->dev, "Invalid eeprom ras table! ret: %d\n", ret);
 
-	return __check_ras_table_status(ras_core);
+	return 0;
 }
 
-int ras_eeprom_hw_fini(struct ras_core_context *ras_core)
+static int ras_eeprom_hw_fini(struct ras_core_context *ras_core)
 {
-	struct ras_eeprom_control *control;
+	return 0;
+}
+
+static int ras_eeprom_get_record_count(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
 
-	if (!ras_core)
+	if (!control)
 		return -EINVAL;
 
-	control = &ras_core->ras_eeprom;
-	mutex_destroy(&control->ras_tbl_mutex);
+	return control->ras_num_recs;
+}
 
-	return 0;
+static int ras_eeprom_get_records(struct ras_core_context *ras_core, u32 start,
+		struct eeprom_umc_record *record, u32 num)
+{
+	if (!record)
+		return -EINVAL;
+
+	return ras_eeprom_read(ras_core, record, num);
 }
 
-uint32_t ras_eeprom_get_record_count(struct ras_core_context *ras_core)
+static int __check_ras_table(struct ras_core_context *ras_core,
+			struct ras_eeprom_control *control)
 {
-	if (!ras_core)
-		return 0;
+	int res = 0;
 
-	return ras_core->ras_eeprom.ras_num_recs;
+	mutex_lock(&control->ras_tbl_mutex);
+	res = __verify_ras_table_checksum(control);
+	/* Update RAS table status only after an actual table
+	 * validation has been performed.
+	 */
+	if (res >= 0)
+		control->ras_tbl_status_ok = res ? false : true;
+	mutex_unlock(&control->ras_tbl_mutex);
+	if (res) {
+		RAS_DEV_ERR(ras_core->dev,
+			"RAS table validation failed! ret: %d\n", res);
+
+		if (res > 0)
+			res = -EFAULT;
+	}
+
+	return res;
 }
 
-void ras_eeprom_sync_info(struct ras_core_context *ras_core)
+static void __get_eeprom_status(struct ras_core_context *ras_core,
+		u32 *status, bool fast_mode)
 {
-	struct ras_eeprom_control *control;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
+	struct ras_eeprom_table_header *hdr;
 
-	if (!ras_core)
+	if (!control)
 		return;
 
-	control = &ras_core->ras_eeprom;
-	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
-		&control->ras_num_recs);
-	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_CHANNEL_BITMAP,
-		&control->bad_channel_bitmap);
+	hdr = &control->tbl_hdr;
+	if (!control->ras_tbl_status_ok)
+		*status = RAS_EEPROM_FAULT;
+	else if (hdr->header == RAS_TABLE_HDR_VAL)
+		*status = RAS_EEPROM_OK;
+	else if (hdr->header == RAS_TABLE_HDR_BAD)
+		*status = RAS_EEPROM_LOCKED;
+	else
+		*status = RAS_EEPROM_UNKNOWN;
 }
 
-enum ras_gpu_health_status
-	ras_eeprom_check_gpu_status(struct ras_core_context *ras_core)
+static int ras_eeprom_get_eeprom_info(struct ras_core_context *ras_core,
+		struct ras_eeprom_info *eeprom_info, bool fast_mode)
 {
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
-	struct ras_eeprom_table_ras_info *rai = &control->tbl_rai;
+	struct ras_eeprom_control *ctl = ras_core->eeprom_mgr.ras_eeprom;
+	struct ras_eeprom_table_header *hdr;
+
+	if (!ctl || !eeprom_info)
+		return -EINVAL;
+
+	hdr = &ctl->tbl_hdr;
+	__get_eeprom_status(ras_core, &eeprom_info->eeprom_status, fast_mode);
+	eeprom_info->record_format_version = hdr->version;
+	eeprom_info->record_count = ctl->ras_num_recs;
+	eeprom_info->max_record_count = ctl->ras_max_record_count;
+	eeprom_info->bad_channel_bitmap = ctl->bad_channel_bitmap;
 
-	if (!__is_ras_eeprom_supported(ras_core) ||
-	    !control->record_threshold_config)
-		return RAS_GPU_HEALTH_NONE;
+	return 0;
+}
 
-	if (control->tbl_hdr.header == RAS_TABLE_HDR_BAD)
-		return RAS_GPU_IN_BAD_STATUS;
+static int ras_eeprom_unlock(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
+	int ret;
 
-	return rai->rma_status;
+	if (!control)
+		return -EINVAL;
+
+	ret = __check_ras_table(ras_core, control);
+	if (ret)
+		return ret;
+
+	return ras_eeprom_correct_header_tag(control, RAS_TABLE_HDR_VAL);
 }
+
+struct ras_eeprom_ops ras_drv_eeprom_ops = {
+	.sw_init = ras_eeprom_sw_init,
+	.sw_fini = ras_eeprom_sw_fini,
+	.hw_init = ras_eeprom_hw_init,
+	.hw_fini = ras_eeprom_hw_fini,
+	.reset_table = ras_eeprom_reset_table,
+	.get_records = ras_eeprom_get_records,
+	.append_records = ras_eeprom_append,
+	.get_record_count = ras_eeprom_get_record_count,
+	.get_eeprom_info = ras_eeprom_get_eeprom_info,
+	.unlock_eeprom = ras_eeprom_unlock,
+};
diff --git a/drivers/gpu/drm/amd/ras/core/eeprom.h b/drivers/gpu/drm/amd/ras/core/eeprom.h
index 32bd1526bc55bc..1482450c2781f7 100644
--- a/drivers/gpu/drm/amd/ras/core/eeprom.h
+++ b/drivers/gpu/drm/amd/ras/core/eeprom.h
@@ -25,6 +25,7 @@
 #ifndef __EEPROM_H__
 #define __EEPROM_H__
 #include "ras_sys.h"
+#include "ras_eeprom_mgr.h"
 
 #define RAS_TABLE_VER_V1           0x00010000
 #define RAS_TABLE_VER_V2_1         0x00021000
@@ -80,11 +81,11 @@ struct ras_eeprom_table_ras_info {
 } __packed;
 
 struct ras_eeprom_control {
+	void *mgr;
 	struct ras_eeprom_table_header tbl_hdr;
 	struct ras_eeprom_table_ras_info tbl_rai;
 
 	/* record threshold */
-	int record_threshold_config;
 	uint32_t record_threshold_count;
 	bool update_channel_flag;
 
@@ -133,6 +134,8 @@ struct ras_eeprom_control {
 	/* Record channel info which occurred bad pages
 	 */
 	u32 bad_channel_bitmap;
+
+	bool ras_tbl_status_ok;
 };
 
 /*
@@ -162,29 +165,23 @@ struct eeprom_umc_record {
 	/* The following variables will not be saved to eeprom.
 	 */
 	uint64_t cur_nps_retired_row_pfn;
+	/* Combined PA flip bit mask across all bad pages in a row;
+	 * used with cur_nps_retired_row_pfn to derive all bad page addresses
+	 */
+	uint64_t cur_nps_pa_flip_mask;
+	uint32_t cur_nps_valid_page_num;
 	uint32_t cur_nps_bank;
 	uint32_t cur_nps;
+	uint64_t ipid;
 };
 
 struct ras_core_context;
-int ras_eeprom_hw_init(struct ras_core_context *ras_core);
-int ras_eeprom_hw_fini(struct ras_core_context *ras_core);
-
-int ras_eeprom_reset_table(struct ras_core_context *ras_core);
 
 bool ras_eeprom_check_safety_watermark(struct ras_core_context *ras_core);
 
-int ras_eeprom_read(struct ras_core_context *ras_core,
-			 struct eeprom_umc_record *records, const u32 num);
-
-int ras_eeprom_append(struct ras_core_context *ras_core,
-			   struct eeprom_umc_record *records, const u32 num);
-
-uint32_t ras_eeprom_max_record_count(struct ras_core_context *ras_core);
-uint32_t ras_eeprom_get_record_count(struct ras_core_context *ras_core);
-void ras_eeprom_sync_info(struct ras_core_context *ras_core);
-
 int ras_eeprom_check_storage_status(struct ras_core_context *ras_core);
 enum ras_gpu_health_status
 	ras_eeprom_check_gpu_status(struct ras_core_context *ras_core);
+
+extern struct ras_eeprom_ops ras_drv_eeprom_ops;
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/eeprom_fw.c b/drivers/gpu/drm/amd/ras/core/eeprom_fw.c
index 59e195652e427f..02369d54bdffdb 100644
--- a/drivers/gpu/drm/amd/ras/core/eeprom_fw.c
+++ b/drivers/gpu/drm/amd/ras/core/eeprom_fw.c
@@ -23,500 +23,246 @@
  */
 
 #include "ras.h"
+#include "ras_mp1.h"
+#include "eeprom_fw.h"
+#define MAX_EEPROM_ERR_RECORD_NUM 1024
 
-#define RAS_SMU_MESSAGE_TIMEOUT_MS 1000 /* 1s */
-
-void ras_fw_init_feature_flags(struct ras_core_context *ras_core)
-{
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-	uint64_t flags = 0ULL;
-
-	if (!sys_func || !sys_func->mp1_get_ras_enabled_mask)
-		return;
-
-	if (!sys_func->mp1_get_ras_enabled_mask(ras_core, &flags))
-		ras_core->ras_fw_features = flags;
-}
-
-bool ras_fw_eeprom_supported(struct ras_core_context *ras_core)
-{
-	return !!(ras_core->ras_fw_features & RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM);
-}
-
-int ras_fw_get_table_version(struct ras_core_context *ras_core,
-				     uint32_t *table_version)
+static int fw_eeprom_reset_ras_table(struct ras_core_context *ras_core)
 {
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-
-	return sys_func->mp1_send_eeprom_msg(ras_core,
-				RAS_SMU_GetRASTableVersion, 0, table_version);
-}
-
-int ras_fw_get_badpage_count(struct ras_core_context *ras_core,
-				     uint32_t *count, uint32_t timeout)
-{
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-	uint64_t end, now;
-	int ret = 0;
-
-	now = (uint64_t)ktime_to_ms(ktime_get());
-	end = now + timeout;
-
-	do {
-		ret = sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_GetBadPageCount, 0, count);
-		/* eeprom is not ready */
-		if (ret != -EBUSY)
-			return ret;
-
-		usleep_range(10000, 15000);
-		now = (uint64_t)ktime_to_ms(ktime_get());
-	} while (now < end);
-
-	RAS_DEV_ERR(ras_core->dev,
-			"smu get bad page count timeout!\n");
-	return ret;
-}
-
-int ras_fw_get_badpage_mca_addr(struct ras_core_context *ras_core,
-					uint16_t index, uint64_t *mca_addr)
-{
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-	uint32_t temp_arg, temp_addr_lo, temp_addr_high;
-	int ret;
-
-	temp_arg = index | (1 << 16);
-	ret = sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_GetBadPageMcaAddr, temp_arg, &temp_addr_lo);
-	if (ret)
-		return ret;
-
-	temp_arg = index | (2 << 16);
-	ret = sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_GetBadPageMcaAddr, temp_arg, &temp_addr_high);
-
-	if (!ret)
-		*mca_addr = (uint64_t)temp_addr_high << 32 | temp_addr_lo;
-
-	return ret;
-}
-
-int ras_fw_set_timestamp(struct ras_core_context *ras_core,
-				 uint64_t timestamp)
-{
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-
-	return sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_SetTimestamp, (uint32_t)timestamp, 0);
-}
-
-int ras_fw_get_timestamp(struct ras_core_context *ras_core,
-				 uint16_t index, uint64_t *timestamp)
-{
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-	uint32_t temp = 0;
-	int ret;
-
-	ret = sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_GetTimestamp, index, &temp);
-	if (!ret)
-		*timestamp = temp;
-
-	return ret;
-}
-
-int ras_fw_get_badpage_ipid(struct ras_core_context *ras_core,
-				    uint16_t index, uint64_t *ipid)
-{
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-	uint32_t temp_arg, temp_ipid_lo, temp_ipid_high;
-	int ret;
-
-	temp_arg = index | (1 << 16);
-	ret = sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_GetBadPageIpid, temp_arg, &temp_ipid_lo);
-	if (ret)
-		return ret;
-
-	temp_arg = index | (2 << 16);
-	ret = sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_GetBadPageIpid, temp_arg, &temp_ipid_high);
-	if (!ret)
-		*ipid = (uint64_t)temp_ipid_high << 32 | temp_ipid_lo;
-
-	return ret;
-}
-
-int ras_fw_erase_ras_table(struct ras_core_context *ras_core,
-				   uint32_t *result)
-{
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-
-	return sys_func->mp1_send_eeprom_msg(ras_core,
-			RAS_SMU_EraseRasTable, 0, result);
-}
-
-int ras_fw_eeprom_reset_table(struct ras_core_context *ras_core)
-{
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	struct fw_eeprom_control *ctl = ras_core->eeprom_mgr.ras_eeprom;
 	u32 erase_res = 0;
 	int res;
 
-	mutex_lock(&control->ras_tbl_mutex);
-
-	res = ras_fw_erase_ras_table(ras_core, &erase_res);
+	mutex_lock(&ctl->record_lock);
+	res = ras_mp1_reset_ras_table(ras_core, &erase_res);
 	if (res || erase_res) {
-		RAS_DEV_WARN(ras_core->dev, "RAS EEPROM reset failed, res:%d result:%d",
-									res, erase_res);
+		RAS_DEV_WARN(ras_core->dev,
+			"RAS EEPROM reset failed, res:%d result:%d\n", res, erase_res);
 		if (!res)
 			res = -EIO;
+		goto out;
 	}
 
-	control->ras_num_recs = 0;
-	control->bad_channel_bitmap = 0;
-	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
-		&control->ras_num_recs);
-	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_CHANNEL_BITMAP,
-		&control->bad_channel_bitmap);
-	control->update_channel_flag = false;
+	ctl->record_count = 0;
+	memset(ctl->records, 0,
+		sizeof(*ctl->records) * MAX_EEPROM_ERR_RECORD_NUM);
 
-	mutex_unlock(&control->ras_tbl_mutex);
+	ctl->ras_table_format_version = 0;
+	ctl->eeprom_status = 0;
+	ctl->rma_status = 0;
+	ctl->bad_channel_bitmap = 0;
 
+out:
+	mutex_unlock(&ctl->record_lock);
 	return res;
 }
 
-bool ras_fw_eeprom_check_safety_watermark(struct ras_core_context *ras_core)
+static int fw_eeprom_sync_data(struct ras_core_context *ras_core,
+		struct fw_eeprom_control *ctl)
 {
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
-	bool ret = false;
-	int bad_page_count;
-
-	if (!control->record_threshold_config)
-		return false;
+	struct eeprom_err_record err_rec = {0};
+	u32 fw_err_rec_num;
+	u32 idx;
+	int ret = 0;
 
-	bad_page_count = ras_umc_get_badpage_count(ras_core);
+	mutex_lock(&ctl->record_lock);
+	ret = ras_mp1_get_record_count(ras_core, &fw_err_rec_num);
+	if (ret)
+		goto out;
+
+	if (!fw_err_rec_num || fw_err_rec_num == ctl->record_count) {
+		goto out;
+	} else if (fw_err_rec_num < ctl->record_count) {
+		RAS_DEV_ERR(ras_core->dev, "EEPROM ECC error count mismatch!\n");
+		ret = -EFAULT;
+		goto out;
+	} else if (fw_err_rec_num > MAX_EEPROM_ERR_RECORD_NUM) {
+		RAS_DEV_ERR(ras_core->dev,
+			"Invalid EEPROM error count:0x%x\n", fw_err_rec_num);
+		ret = -EOVERFLOW;
+		goto out;
+	}
 
-	if (bad_page_count > control->record_threshold_count)
-		RAS_DEV_WARN(ras_core->dev, "RAS records:%d exceed threshold:%d",
-			bad_page_count, control->record_threshold_count);
+	for (idx = ctl->record_count;
+			idx < fw_err_rec_num; idx++, ctl->record_count = idx) {
+		ret = ras_mp1_get_record(ras_core, idx, &err_rec);
+		if (ret)
+			goto out;
 
-	if ((control->record_threshold_config == WARN_NONSTOP_OVER_THRESHOLD) ||
-		(control->record_threshold_config == NONSTOP_OVER_THRESHOLD)) {
-		RAS_DEV_WARN(ras_core->dev,
-			"Please consult AMD Service Action Guide (SAG) for appropriate service procedures.\n");
-		ret = false;
-	} else {
-		ras_core->is_rma = true;
-		RAS_DEV_WARN(ras_core->dev,
-			"Please consider adjusting the customized threshold.\n");
-		ret = true;
+		memcpy(&ctl->records[idx], &err_rec, sizeof(*ctl->records));
 	}
 
+out:
+	mutex_unlock(&ctl->record_lock);
 	return ret;
 }
 
-int ras_fw_eeprom_append(struct ras_core_context *ras_core,
-			   struct eeprom_umc_record *record, const u32 num)
+static int fw_eeprom_get_record_count(struct ras_core_context *ras_core)
 {
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
-	int threshold_config = control->record_threshold_config;
-	int i, bad_page_count;
-
-	mutex_lock(&control->ras_tbl_mutex);
-
-	for (i = 0; i < num; i++) {
-		/* update bad channel bitmap */
-		if ((record[i].mem_channel < BITS_PER_TYPE(control->bad_channel_bitmap)) &&
-			!(control->bad_channel_bitmap & (1 << record[i].mem_channel))) {
-			control->bad_channel_bitmap |= 1 << record[i].mem_channel;
-			control->update_channel_flag = true;
-		}
-	}
-	control->ras_num_recs += num;
+	struct fw_eeprom_control *ctl;
 
-	bad_page_count = ras_umc_get_badpage_count(ras_core);
+	if (!ras_core->eeprom_mgr.ras_eeprom)
+		return -EINVAL;
 
-	if (threshold_config != 0 &&
-		bad_page_count > control->record_threshold_count) {
-		RAS_DEV_WARN(ras_core->dev,
-			"Saved bad pages %d reaches threshold value %d\n",
-			bad_page_count, control->record_threshold_count);
+	ctl = ras_core->eeprom_mgr.ras_eeprom;
 
-		if ((threshold_config != WARN_NONSTOP_OVER_THRESHOLD) &&
-			(threshold_config != NONSTOP_OVER_THRESHOLD))
-			ras_core->is_rma = true;
+	if (!ras_core_gpu_in_reset(ras_core))
+		fw_eeprom_sync_data(ras_core, ctl);
 
-		/* ignore the -ENOTSUPP return value */
-		ras_core_event_notify(ras_core, RAS_EVENT_ID__DEVICE_RMA, NULL);
-	}
-
-	mutex_unlock(&control->ras_tbl_mutex);
-	return 0;
+	return ctl->record_count;
 }
 
-int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core,
-			 struct eeprom_umc_record *record_umc,
-			 struct ras_bank_ecc *ras_ecc,
-			 u32 rec_idx, const u32 num)
+static int fw_eeprom_get_record(struct ras_core_context *ras_core,
+		u32 idx, struct eeprom_umc_record *rec)
 {
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
-	int i, ret, end_idx;
-	u64 mca, ipid, ts;
-	u32 cu, mem_channel, mcumc_id;
+	struct fw_eeprom_control *ctl;
+	struct ras_bank_ecc bank = {0};
 
-	if (!ras_core->ras_umc.ip_func ||
-	    !ras_core->ras_umc.ip_func->mca_ipid_parse)
-		return -EOPNOTSUPP;
-
-	mutex_lock(&control->ras_tbl_mutex);
-
-	end_idx = rec_idx + num;
-	for (i = rec_idx; i < end_idx; i++) {
-		ret = ras_fw_get_badpage_mca_addr(ras_core, i, &mca);
-		if (ret)
-			goto out;
-
-		ret = ras_fw_get_badpage_ipid(ras_core, i, &ipid);
-		if (ret)
-			goto out;
+	if (!ras_core->eeprom_mgr.ras_eeprom)
+		return -EINVAL;
 
-		ret = ras_fw_get_timestamp(ras_core, i, &ts);
-		if (ret)
-			goto out;
+	ctl = ras_core->eeprom_mgr.ras_eeprom;
 
-		if (record_umc) {
-			record_umc[i - rec_idx].address = mca;
-			/* retired_page (pa) is unused now */
-			record_umc[i - rec_idx].retired_row_pfn = 0x1ULL;
-			record_umc[i - rec_idx].ts = ts;
-			record_umc[i - rec_idx].err_type = RAS_EEPROM_ERR_NON_RECOVERABLE;
-
-			ras_core->ras_umc.ip_func->mca_ipid_parse(ras_core, ipid,
-				&cu, &mem_channel, &mcumc_id, NULL);
-			record_umc[i - rec_idx].cu = (u8)cu;
-			record_umc[i - rec_idx].mem_channel = (u8)mem_channel;
-			record_umc[i - rec_idx].mcumc_id = (u8)mcumc_id;
-
-			/* update bad channel bitmap */
-			if ((record_umc[i - rec_idx].mem_channel < BITS_PER_TYPE(control->bad_channel_bitmap)) &&
-				!(control->bad_channel_bitmap & (1 << record_umc[i - rec_idx].mem_channel))) {
-				control->bad_channel_bitmap |= 1 << record_umc[i - rec_idx].mem_channel;
-				control->update_channel_flag = true;
-			}
-		}
-
-		if (ras_ecc) {
-			ras_ecc[i - rec_idx].addr = mca;
-			ras_ecc[i - rec_idx].ipid = ipid;
-			ras_ecc[i - rec_idx].ts = ts;
-		}
+	if (!rec || idx >= ctl->record_count)
+		return -EINVAL;
 
-	}
+	bank.timestamp = ctl->records[idx].timestamp;
+	bank.status = 0;
+	bank.ipid = ctl->records[idx].ipid;
+	bank.addr = ctl->records[idx].mca_addr;
+	bank.nps = ras_core_get_curr_nps_mode(ras_core);
 
-out:
-	mutex_unlock(&control->ras_tbl_mutex);
-	return ret;
+	return ras_umc_bank_to_umc_record(ras_core, &bank, rec);
 }
 
-uint32_t ras_fw_eeprom_get_record_count(struct ras_core_context *ras_core)
+static int fw_eeprom_sw_init(struct ras_core_context *ras_core,
+			struct ras_eeprom_param *param)
 {
-	if (!ras_core)
-		return 0;
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	struct fw_eeprom_control *ctl;
+	int ret = 0;
 
-	return ras_core->ras_fw_eeprom.ras_num_recs;
-}
+	if (!param)
+		return -EINVAL;
 
-int ras_fw_eeprom_update_record(struct ras_core_context *ras_core,
-				struct ras_bank_ecc *ras_ecc)
-{
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
-	int ret, retry = 20;
-	u32 recs_num_new = control->ras_num_recs;
-
-	do {
-		/* 1000ms timeout is long enough, smu_get_badpage_count won't
-		 * return -EBUSY before timeout.
-		 */
-		ret = ras_fw_get_badpage_count(ras_core,
-			&recs_num_new, RAS_SMU_MESSAGE_TIMEOUT_MS);
-		if (!ret &&
-		    (recs_num_new == control->ras_num_recs)) {
-			/* record number update in PMFW needs some time,
-			 * smu_get_badpage_count may return immediately without
-			 * count update, sleep for a while and retry again.
-			 */
-			msleep(50);
-			retry--;
-		} else {
-			break;
-		}
-	} while (retry);
+	ctl = kzalloc(sizeof(*ctl), GFP_KERNEL);
+	if (!ctl)
+		return -ENOMEM;
 
-	if (ret)
-		return ret;
+	ctl->eeprom_ip_version = param->eeprom_ip_version;
+	ctl->records = kzalloc(sizeof(*ctl->records) * MAX_EEPROM_ERR_RECORD_NUM, GFP_KERNEL);
+	if (!ctl->records) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
-	if (recs_num_new > control->ras_num_recs)
-		ret = ras_fw_eeprom_read_idx(ras_core, 0,
-					ras_ecc, control->ras_num_recs, 1);
-	else
-		ret = -EINVAL;
+	ctl->max_record_count = MAX_EEPROM_ERR_RECORD_NUM;
+	mutex_init(&ctl->record_lock);
+
+	mgr->ras_eeprom = ctl;
+
+	return 0;
 
+out:
+	kfree(ctl);
 	return ret;
 }
 
-static int __check_ras_fw_table_status(struct ras_core_context *ras_core)
+static int fw_eeprom_sw_fini(struct ras_core_context *ras_core)
 {
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
-	uint64_t local_time;
-	int res;
-
-	mutex_init(&control->ras_tbl_mutex);
-
-	res = ras_fw_get_table_version(ras_core, &(control->version));
-	if (res)
-		return res;
+	struct fw_eeprom_control *ctl;
 
-	res = ras_fw_get_badpage_count(ras_core, &(control->ras_num_recs), 100);
-	if (res)
-		return res;
-
-	local_time = (uint64_t)ktime_get_real_seconds();
-	res = ras_fw_set_timestamp(ras_core, local_time);
-	if (res)
-		return res;
+	if (!ras_core->eeprom_mgr.ras_eeprom)
+		return -EINVAL;
 
-	control->ras_max_record_count = 4000;
+	ctl = ras_core->eeprom_mgr.ras_eeprom;
 
+	kfree(ctl->records);
+	mutex_destroy(&ctl->record_lock);
 
-	if (control->ras_num_recs > control->ras_max_record_count) {
-		RAS_DEV_ERR(ras_core->dev,
-			"RAS header invalid, records in header: %u max allowed :%u",
-			control->ras_num_recs, control->ras_max_record_count);
-		return -EINVAL;
-	}
+	kfree(ctl);
+	ras_core->eeprom_mgr.ras_eeprom = NULL;
 
 	return 0;
 }
 
-int ras_fw_eeprom_hw_init(struct ras_core_context *ras_core)
+static int fw_eeprom_hw_init(struct ras_core_context *ras_core,
+			struct ras_eeprom_param *param)
 {
-	struct ras_fw_eeprom_control *control;
-	struct ras_eeprom_config *eeprom_cfg;
-	struct ras_mp1 *mp1;
-	const struct ras_mp1_sys_func *sys_func;
+	struct fw_eeprom_control *ctl;
 
-	if (!ras_core)
+	if (!ras_core->eeprom_mgr.ras_eeprom)
 		return -EINVAL;
 
-	mp1 = &ras_core->ras_mp1;
-	sys_func = mp1->sys_func;
-
-	if (!sys_func || !sys_func->mp1_send_eeprom_msg)
-		return -EINVAL;
-
-	ras_core->is_rma = false;
-
-	control = &ras_core->ras_fw_eeprom;
+	ctl = ras_core->eeprom_mgr.ras_eeprom;
 
-	memset(control, 0, sizeof(*control));
+	ras_mp1_get_table_version(ras_core, &ctl->ras_table_format_version);
 
-	eeprom_cfg = &ras_core->config->eeprom_cfg;
-	control->record_threshold_config =
-		eeprom_cfg->eeprom_record_threshold_config;
-
-	control->record_threshold_count = 4000;
-	if (eeprom_cfg->eeprom_record_threshold_count <
-		control->record_threshold_count)
-		control->record_threshold_count =
-			eeprom_cfg->eeprom_record_threshold_count;
-
-	control->update_channel_flag = false;
-
-	return __check_ras_fw_table_status(ras_core);
+	return 0;
 }
 
-int ras_fw_eeprom_hw_fini(struct ras_core_context *ras_core)
+static int fw_eeprom_hw_fini(struct ras_core_context *ras_core)
 {
-	struct ras_fw_eeprom_control *control;
-
-	if (!ras_core)
-		return -EINVAL;
-
-	control = &ras_core->ras_fw_eeprom;
-	mutex_destroy(&control->ras_tbl_mutex);
-
 	return 0;
 }
 
-int ras_fw_eeprom_check_storage_status(struct ras_core_context *ras_core)
+static int fw_eeprom_get_records(struct ras_core_context *ras_core, u32 start,
+		struct eeprom_umc_record *record, u32 num)
 {
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
-	int bad_page_count;
-
-	bad_page_count = ras_umc_get_badpage_count(ras_core);
-
-	if ((control->record_threshold_count < bad_page_count) &&
-	    (control->record_threshold_config != 0)) {
-		RAS_DEV_ERR(ras_core->dev, "RAS records:%d exceed threshold:%d",
-				bad_page_count, control->record_threshold_count);
-		if ((control->record_threshold_config == WARN_NONSTOP_OVER_THRESHOLD) ||
-			(control->record_threshold_config == NONSTOP_OVER_THRESHOLD)) {
-			RAS_DEV_WARN(ras_core->dev,
-			"Please consult AMD Service Action Guide (SAG) for appropriate service procedures\n");
-		} else {
-			ras_core->is_rma = true;
-			RAS_DEV_ERR(ras_core->dev,
-			"User defined threshold is set, runtime service will be halt when threshold is reached\n");
-		}
-		return 0;
-	}
+	struct fw_eeprom_control *ctl = ras_core->eeprom_mgr.ras_eeprom;
+	int i, ret = 0;
 
-	RAS_DEV_INFO(ras_core->dev,
-			"Found existing EEPROM table with %d records\n",
-			bad_page_count);
-	/* Warn if we are at 90% of the threshold or above
-	 */
-	if (10 * bad_page_count >= 9 * control->record_threshold_count)
-		RAS_DEV_WARN(ras_core->dev,
-			"RAS records:%u exceeds 90%% of threshold:%d\n",
-			bad_page_count,
-			control->record_threshold_count);
+	mutex_lock(&ctl->record_lock);
+	for (i = 0; i < num; i++) {
+		ret = fw_eeprom_get_record(ras_core, start + i, &record[i]);
+		if (ret)
+			break;
+	}
+	mutex_unlock(&ctl->record_lock);
 
-	return 0;
+	return ret;
 }
 
-enum ras_gpu_health_status
-	ras_fw_eeprom_check_gpu_status(struct ras_core_context *ras_core)
+static int fw_eeprom_get_eeprom_info(struct ras_core_context *ras_core,
+		struct ras_eeprom_info *eeprom_info, bool fast_mode)
 {
-	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
+	struct fw_eeprom_control *ctl = ras_core->eeprom_mgr.ras_eeprom;
 
-	if (!control->record_threshold_config)
-		return RAS_GPU_HEALTH_NONE;
+	if (!eeprom_info)
+		return -EINVAL;
 
-	if (ras_core->is_rma)
-		return RAS_GPU_RETIRED__ECC_REACH_THRESHOLD;
+	eeprom_info->record_count = ctl->record_count;
+	eeprom_info->max_record_count = ctl->max_record_count;
+	eeprom_info->bad_channel_bitmap = ctl->bad_channel_bitmap;
 
-	return RAS_GPU_HEALTH_USABLE;
-}
+	if (!fast_mode) {
+		if (!ctl->ras_table_format_version)
+			ras_mp1_get_table_version(ras_core,
+				&ctl->ras_table_format_version);
 
-void ras_fw_eeprom_sync_info(struct ras_core_context *ras_core)
-{
-	struct ras_fw_eeprom_control *control;
+		eeprom_info->rma_status = ras_mp1_rma_detected(ras_core);
+		ctl->rma_status = eeprom_info->rma_status;
+	} else {
+		eeprom_info->rma_status = ctl->rma_status;
+	}
 
-	if (!ras_core)
-		return;
+	eeprom_info->record_format_version = ctl->ras_table_format_version;
 
-	control = &ras_core->ras_fw_eeprom;
-	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
-		&control->ras_num_recs);
-	ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_CHANNEL_BITMAP,
-		&control->bad_channel_bitmap);
+	if (eeprom_info->rma_status)
+		eeprom_info->eeprom_status = RAS_EEPROM_LOCKED;
+	else
+		eeprom_info->eeprom_status = RAS_EEPROM_OK;
+
+	return 0;
 }
+
+struct ras_eeprom_ops ras_fw_eeprom_ops = {
+	.sw_init = fw_eeprom_sw_init,
+	.sw_fini = fw_eeprom_sw_fini,
+	.hw_init = fw_eeprom_hw_init,
+	.hw_fini = fw_eeprom_hw_fini,
+	.reset_table = fw_eeprom_reset_ras_table,
+	.get_records = fw_eeprom_get_records,
+	.get_record_count = fw_eeprom_get_record_count,
+	.get_eeprom_info = fw_eeprom_get_eeprom_info,
+};
diff --git a/drivers/gpu/drm/amd/ras/core/eeprom_fw.h b/drivers/gpu/drm/amd/ras/core/eeprom_fw.h
index 803d3ccdaddbdc..8fc8773732eceb 100644
--- a/drivers/gpu/drm/amd/ras/core/eeprom_fw.h
+++ b/drivers/gpu/drm/amd/ras/core/eeprom_fw.h
@@ -23,65 +23,18 @@
  */
 #ifndef __EEPROM_FW_H__
 #define __EEPROM_FW_H__
+struct fw_eeprom_control {
+	u32 eeprom_ip_version;
+	struct mutex record_lock;
+	struct eeprom_err_record *records;
+	u32 record_count;
 
-struct ras_fw_eeprom_control {
-	uint32_t version;
-	/* record threshold */
-	int record_threshold_config;
-	uint32_t record_threshold_count;
-	bool update_channel_flag;
-
-	/* Number of records in the table.
-	 */
-	u32 ras_num_recs;
-
-	/* Maximum possible number of records
-	 * we could store, i.e. the maximum capacity
-	 * of the table.
-	 */
-	u32 ras_max_record_count;
-
-	/* Protect table access via this mutex.
-	 */
-	struct mutex ras_tbl_mutex;
-
-	/* Record channel info which occurred bad pages
-	 */
+	u32 ras_table_format_version;
+	u32 eeprom_status;
+	u32 rma_status;
+	u32 max_record_count;
 	u32 bad_channel_bitmap;
 };
 
-void ras_fw_init_feature_flags(struct ras_core_context *ras_core);
-bool ras_fw_eeprom_supported(struct ras_core_context *ras_core);
-int ras_fw_get_table_version(struct ras_core_context *ras_core,
-				     uint32_t *table_version);
-int ras_fw_get_badpage_count(struct ras_core_context *ras_core,
-				     uint32_t *count, uint32_t timeout);
-int ras_fw_get_badpage_mca_addr(struct ras_core_context *ras_core,
-					uint16_t index, uint64_t *mca_addr);
-int ras_fw_set_timestamp(struct ras_core_context *ras_core,
-				 uint64_t timestamp);
-int ras_fw_get_timestamp(struct ras_core_context *ras_core,
-				 uint16_t index, uint64_t *timestamp);
-int ras_fw_get_badpage_ipid(struct ras_core_context *ras_core,
-				    uint16_t index, uint64_t *ipid);
-int ras_fw_erase_ras_table(struct ras_core_context *ras_core,
-				   uint32_t *result);
-int ras_fw_eeprom_reset_table(struct ras_core_context *ras_core);
-bool ras_fw_eeprom_check_safety_watermark(struct ras_core_context *ras_core);
-int ras_fw_eeprom_append(struct ras_core_context *ras_core,
-			   struct eeprom_umc_record *record, const u32 num);
-int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core,
-			 struct eeprom_umc_record *record_umc,
-			 struct ras_bank_ecc *ras_ecc,
-			 u32 rec_idx, const u32 num);
-uint32_t ras_fw_eeprom_get_record_count(struct ras_core_context *ras_core);
-int ras_fw_eeprom_update_record(struct ras_core_context *ras_core,
-				struct ras_bank_ecc *ras_ecc);
-int ras_fw_eeprom_hw_init(struct ras_core_context *ras_core);
-int ras_fw_eeprom_hw_fini(struct ras_core_context *ras_core);
-int ras_fw_eeprom_check_storage_status(struct ras_core_context *ras_core);
-enum ras_gpu_health_status
-	ras_fw_eeprom_check_gpu_status(struct ras_core_context *ras_core);
-void ras_fw_eeprom_sync_info(struct ras_core_context *ras_core);
-
+extern struct ras_eeprom_ops ras_fw_eeprom_ops;
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/log_ring.c b/drivers/gpu/drm/amd/ras/core/log_ring.c
index 175c8b3571447a..e57f7ea3f30798 100644
--- a/drivers/gpu/drm/amd/ras/core/log_ring.c
+++ b/drivers/gpu/drm/amd/ras/core/log_ring.c
@@ -32,16 +32,6 @@
 
 #define BATCH_IDX_TO_TREE_IDX(batch_idx, sn) (((batch_idx) << 8) | (sn))
 
-static const uint64_t ras_rma_aca_reg[ACA_REG_MAX_COUNT] = {
-	[ACA_REG_IDX__CTL]    = 0x1,
-	[ACA_REG_IDX__STATUS] = 0xB000000000000137,
-	[ACA_REG_IDX__ADDR]   = 0x0,
-	[ACA_REG_IDX__MISC0]  = 0x0,
-	[ACA_REG_IDX__CONFG] = 0x1ff00000002,
-	[ACA_REG_IDX__IPID]   = 0x9600000000,
-	[ACA_REG_IDX__SYND]   = 0x0,
-};
-
 static uint64_t ras_log_ring_get_logged_ecc_count(struct ras_core_context *ras_core)
 {
 	struct ras_log_ring *log_ring = &ras_core->ras_log_ring;
@@ -212,7 +202,7 @@ struct ras_log_batch_tag *ras_log_ring_create_batch_tag(struct ras_core_context
 	spin_unlock_irqrestore(&log_ring->spin_lock, flags);
 
 	batch_tag->sub_seqno = 0;
-	batch_tag->timestamp = ras_core_get_utc_second_timestamp(ras_core);
+	batch_tag->timestamp = ktime_get_real_ns();
 	return batch_tag;
 }
 
@@ -222,15 +212,21 @@ void ras_log_ring_destroy_batch_tag(struct ras_core_context *ras_core,
 	kfree(batch_tag);
 }
 
-void ras_log_ring_add_log_event(struct ras_core_context *ras_core,
-		enum ras_log_event event, void *data, struct ras_log_batch_tag *batch_tag)
+int ras_log_ring_add_log_event(struct ras_core_context *ras_core,
+		enum ras_log_event event,
+		void *data, uint32_t size, struct ras_log_batch_tag *batch_tag)
 {
 	struct ras_log_ring *log_ring = &ras_core->ras_log_ring;
-	struct device_system_info dev_info = {0};
 	struct ras_log_info *log;
-	uint64_t socket_id;
+	struct aca_bank_reg *rma_bank;
 	void *obj;
 
+	if (size > sizeof(union ras_log_body)) {
+		RAS_DEV_ERR(ras_core->dev,
+			"Log event(0x%x) data size exceeded buffer!\n", event);
+		return -EINVAL;
+	}
+
 	obj = mempool_alloc_preallocated(log_ring->ras_log_mempool);
 	if (!obj ||
 	   (ras_log_ring_get_logged_ecc_count(ras_core) >= RAS_LOG_MEMPOOL_SIZE)) {
@@ -241,28 +237,34 @@ void ras_log_ring_add_log_event(struct ras_core_context *ras_core,
 
 	if (!obj) {
 		RAS_DEV_ERR(ras_core->dev, "ERROR: Failed to alloc ras log buffer!\n");
-		return;
+		return -ENOMEM;
 	}
 
 	log = (struct ras_log_info *)obj;
 
 	memset(log, 0, sizeof(*log));
 	log->timestamp =
-		batch_tag ? batch_tag->timestamp : ras_core_get_utc_second_timestamp(ras_core);
+		batch_tag ? batch_tag->timestamp : ktime_get_real_ns();
 	log->event = event;
 
-	if (data)
-		memcpy(&log->aca_reg, data, sizeof(log->aca_reg));
-
-	if (event == RAS_LOG_EVENT_RMA) {
-		memcpy(&log->aca_reg, ras_rma_aca_reg, sizeof(log->aca_reg));
-		ras_core_get_device_system_info(ras_core, &dev_info);
-		socket_id = dev_info.socket_id;
-		log->aca_reg.regs[ACA_REG_IDX__IPID] |= ((socket_id / 4) & 0x01);
-		log->aca_reg.regs[ACA_REG_IDX__IPID] |= (((socket_id % 4) & 0x3) << 44);
+	if (data && size && size <= sizeof(log->body)) {
+		memcpy(&log->body, data, size);
+		log->size = size;
+	}
+
+	if ((event == RAS_LOG_EVENT_RMA) && !data && !size) {
+		rma_bank = kzalloc_obj(*rma_bank);
+		if (rma_bank && !ras_aca_fill_rma_bank(ras_core, rma_bank)) {
+			memcpy(log->body.aca_reg.regs,
+				rma_bank->regs, sizeof(log->body.aca_reg.regs));
+			log->size = sizeof(log->body.aca_reg);
+		} else {
+			RAS_DEV_WARN(ras_core->dev, "Failed to fill rma bank info!\n");
+		}
+		kfree(rma_bank);
 	}
 
-	ras_log_ring_add_data(ras_core, log, batch_tag);
+	return ras_log_ring_add_data(ras_core, log, batch_tag);
 }
 
 static int ras_log_ring_lookup_data(struct ras_core_context *ras_core,
diff --git a/drivers/gpu/drm/amd/ras/core/log_ring.h b/drivers/gpu/drm/amd/ras/core/log_ring.h
index a1ef2f24130c0c..2d8e40c9710624 100644
--- a/drivers/gpu/drm/amd/ras/core/log_ring.h
+++ b/drivers/gpu/drm/amd/ras/core/log_ring.h
@@ -24,6 +24,7 @@
 #ifndef __LOG_RING_H__
 #define __LOG_RING_H__
 #include "aca.h"
+#include "ras_cper.h"
 
 #define MAX_RECORD_PER_BATCH 32
 
@@ -37,6 +38,9 @@ enum ras_log_event {
 	RAS_LOG_EVENT_POISON_CREATION,
 	RAS_LOG_EVENT_POISON_CONSUMPTION,
 	RAS_LOG_EVENT_RMA,
+	RAS_LOG_EVENT_MCE,
+	RAS_LOG_EVENT_BOOT,
+	RAS_LOG_EVENT_CPU_RAS,
 	RAS_LOG_EVENT_COUNT_MAX,
 };
 
@@ -44,13 +48,44 @@ struct ras_aca_reg {
 	uint64_t regs[ACA_REG_MAX_COUNT];
 };
 
+struct ras_cpu_mce {
+	u64 regs[ACA_REG_MAX_COUNT - 1];
+	u32 apic_id;
+	u16 bank;
+	u16 reserved;
+};
+
+#define RAS_CPER_BOOT_RAW_DATA_SIZE	80
+#define RAS_BOOT_CTX_VALID_APIC_ID	BIT(0)
+
+struct ras_boot_err_ctx {
+	u8 section_type[16];
+	u32 error_severity;
+	u16 reg_ctx_type;
+	u16 reg_arr_size;
+	u32 msr_addr;
+	u16 raw_data_size;
+	u16 flags;
+	u64 apic_id;
+	u64 mm_reg_addr;
+	union {
+		u64 regs[CPER_OAM_MAX_COUNT];
+		u8 raw_data[RAS_CPER_BOOT_RAW_DATA_SIZE];
+	};
+};
+
+union ras_log_body {
+	struct ras_aca_reg aca_reg;
+	struct ras_cpu_mce cpu_mce;
+	struct ras_boot_err_ctx boot_err_ctx;
+};
+
 struct ras_log_info {
 	uint64_t seqno;
 	uint64_t timestamp;
 	enum ras_log_event event;
-	union {
-		struct ras_aca_reg aca_reg;
-	};
+	uint32_t size;
+	union ras_log_body body;
 };
 
 struct ras_log_batch_tag {
@@ -82,8 +117,9 @@ int ras_log_ring_sw_fini(struct ras_core_context *ras_core);
 struct ras_log_batch_tag *ras_log_ring_create_batch_tag(struct ras_core_context *ras_core);
 void ras_log_ring_destroy_batch_tag(struct ras_core_context *ras_core,
 			struct ras_log_batch_tag *tag);
-void ras_log_ring_add_log_event(struct ras_core_context *ras_core,
-		enum ras_log_event event, void *data, struct ras_log_batch_tag *tag);
+int ras_log_ring_add_log_event(struct ras_core_context *ras_core,
+		enum ras_log_event event,
+		void *data, uint32_t size, struct ras_log_batch_tag *batch_tag);
 
 int ras_log_ring_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_idx,
 		struct ras_log_info *log_arr, uint32_t arr_num);
diff --git a/drivers/gpu/drm/amd/ras/core/ras.h b/drivers/gpu/drm/amd/ras/core/ras.h
index b4e40d7cc0e051..73e3960c7c8e41 100644
--- a/drivers/gpu/drm/amd/ras/core/ras.h
+++ b/drivers/gpu/drm/amd/ras/core/ras.h
@@ -27,7 +27,7 @@
 #include "ras_sys.h"
 #include "ras_umc.h"
 #include "aca.h"
-#include "eeprom.h"
+#include "ras_eeprom_mgr.h"
 #include "core_status.h"
 #include "ras_process.h"
 #include "ras_gfx.h"
@@ -37,6 +37,8 @@
 #include "ras_psp.h"
 #include "log_ring.h"
 #include "eeprom_fw.h"
+#include "ras_mce.h"
+#include "ras_bert.h"
 
 #define RAS_HW_ERR		"[Hardware Error]: "
 
@@ -81,8 +83,29 @@ enum ras_block_id {
 	RAS_BLOCK_ID__JPEG,
 	RAS_BLOCK_ID__IH,
 	RAS_BLOCK_ID__MPIO,
-
-	RAS_BLOCK_ID__LAST
+	RAS_BLOCK_ID__MMSCH,
+	RAS_BLOCK_ID__MP5,
+	RAS_BLOCK_ID__ATU,
+	RAS_BLOCK_ID__DACC_BE,
+	RAS_BLOCK_ID__ECLR,
+	RAS_BLOCK_ID__KPX_SERDES,
+	RAS_BLOCK_ID__LSDMA,
+	RAS_BLOCK_ID__MPART,
+	RAS_BLOCK_ID__MPIFOE,
+	RAS_BLOCK_ID__MPRAS,
+	RAS_BLOCK_ID__NBIF,
+	RAS_BLOCK_ID__NBIO,
+	RAS_BLOCK_ID__OXRP,
+	RAS_BLOCK_ID__PCIE_PL,
+	RAS_BLOCK_ID__PCS_XGMI,
+	RAS_BLOCK_ID__PIE,
+	RAS_BLOCK_ID__CS,
+	RAS_BLOCK_ID__SHUB,
+	RAS_BLOCK_ID__SSBDCI,
+	RAS_BLOCK_ID__UCIE_PCS,
+
+	RAS_BLOCK_ID__LAST,
+	MAX_SUPPORTED_RAS_BLOCK_ID  = MAX_RAS_BLOCK_MASK_BITS
 };
 
 enum ras_ecc_err_type {
@@ -97,6 +120,7 @@ enum ras_err_type {
 	RAS_ERR_TYPE__UE = 0,
 	RAS_ERR_TYPE__CE,
 	RAS_ERR_TYPE__DE,
+	RAS_ERR_TYPE__MCE,
 	RAS_ERR_TYPE__LAST
 };
 
@@ -129,6 +153,8 @@ enum ras_notify_event {
 	RAS_EVENT_ID__RESET_VF,
 	RAS_EVENT_ID__RAS_EVENT_PROC_BEGIN,
 	RAS_EVENT_ID__RAS_EVENT_PROC_END,
+	RAS_EVENT_ID__UPDATE_ACA_DATA,
+	RAS_EVENT_ID__EARLY_INIT_RESERVE_PAGE,
 };
 
 enum ras_gpu_status {
@@ -150,6 +176,24 @@ enum ras_fw_eeprom_cmd {
 	RAS_SMU_GetBadPageMcaAddr,
 };
 
+enum ras_unit_id {
+	RAS_UNIT_ID_UMC,
+	RAS_UNIT_ID_GFX,
+	RAS_UNIT_ID_MP1,
+	RAS_UNIT_ID_PSP,
+	RAS_UNIT_ID_NBIO,
+	RAS_UNIT_ID_ACA,
+	RAS_UNIT_ID_EEPROM,
+	RAS_UNIT_ID_MAX
+};
+
+enum ras_work_mode_over_thresh {
+	RAS_WORK_MODE_OVER_THRESH_STRICT,
+	RAS_WORK_MODE_OVER_THRESH_NORMAL,
+	RAS_WORK_MODE_OVER_THRESH_DEBUG,
+	RAS_WORK_MODE_OVER_THRESH_RMA,
+};
+
 struct ras_core_context;
 struct ras_bank_ecc;
 struct ras_umc;
@@ -158,23 +202,31 @@ struct ras_process;
 struct ras_nbio;
 struct ras_log_ring;
 struct ras_psp;
+struct ras_eeprom_mgr;
 
 struct ras_mp1_sys_func {
-	int (*mp1_get_valid_bank_count)(struct ras_core_context *ras_core,
-			u32 msg, u32 *count);
-	int (*mp1_dump_valid_bank)(struct ras_core_context *ras_core,
-			u32 msg, u32 idx, u32 reg_idx, u64 *val);
-	int (*mp1_send_eeprom_msg)(struct ras_core_context *ras_core,
-			enum ras_fw_eeprom_cmd index, uint32_t param, uint32_t *read_arg);
-	int (*mp1_get_ras_enabled_mask)(struct ras_core_context *ras_core,
-			uint64_t *enabled_mask);
-	int (*mp1_set_debug_mode)(struct ras_core_context *ras_core, bool enable);
+	int (*mp1_send_ras_msg)(struct ras_core_context *ras_core, u32 msg_id,
+		u32 *params, u32 num_params, u32 *read_args, u32 num_read_args);
+	int (*check_ras_feature_status)(struct ras_core_context *ras_core,
+		enum ras_mp1_feature_id feature_id, u32 *status);
+};
+
+struct ras_eeprom_param_config {
+	u32 eeprom_ip_version;
+	u64 eeprom_record_threshold_count;
+	enum ras_work_mode_over_thresh work_mode_over_thresh;
+	void *eeprom_i2c_adapter;
+	u32 eeprom_i2c_addr;
+	u32 eeprom_i2c_port;
+	u16 max_i2c_read_len;
+	u16 max_i2c_write_len;
 };
 
 struct ras_eeprom_sys_func {
 	int (*eeprom_i2c_xfer)(struct ras_core_context *ras_core,
 			u32 eeprom_addr, u8 *eeprom_buf, u32 buf_size, bool read);
-	int (*update_eeprom_i2c_config)(struct ras_core_context *ras_core);
+	int (*get_eeprom_config)(struct ras_core_context *ras_core,
+			struct ras_eeprom_param_config *param_cfg);
 };
 
 struct ras_nbio_sys_func {
@@ -204,15 +256,19 @@ enum gpu_mem_type {
 	GPU_MEM_TYPE_RAS_PSP_RING,
 	GPU_MEM_TYPE_RAS_PSP_CMD,
 	GPU_MEM_TYPE_RAS_PSP_FENCE,
-	GPU_MEM_TYPE_RAS_TA_FW,
+	GPU_MEM_TYPE_RAS_FW_BIN,
 	GPU_MEM_TYPE_RAS_TA_CMD,
+	GPU_MEM_TYPE_ALLOC_MEM,
+	GPU_MEM_TYPE_MAX
 };
 
 struct ras_psp_sys_func {
 	int (*get_ras_psp_system_status)(struct ras_core_context *ras_core,
 		struct ras_psp_sys_status *status);
-	int (*get_ras_ta_init_param)(struct ras_core_context *ras_core,
-		struct ras_ta_init_param *ras_ta_param);
+	int (*get_ras_param)(struct ras_core_context *ras_core,
+		struct ras_param *param);
+	int (*psp_translate_addr)(struct ras_core_context *ras_core,
+		struct ras_psp_addr_trans_in *in, struct ras_psp_addr_trans_out *out);
 };
 
 struct ras_sys_func {
@@ -234,6 +290,8 @@ struct ras_sys_func {
 	int (*put_gpu_mem)(struct ras_core_context *ras_core,
 		enum gpu_mem_type mem_type, struct gpu_mem_block *gpu_mem);
 	int (*check_address_sanity)(struct ras_core_context *ras_core, uint64_t addr);
+	int (*get_nps_mode)(struct ras_core_context *ras_core);
+	int (*get_vram_type)(struct ras_core_context *ras_core);
 };
 
 struct ras_ecc_count {
@@ -246,6 +304,7 @@ struct ras_ecc_count {
 };
 
 struct ras_bank_ecc {
+	uint64_t timestamp;
 	uint32_t nps;
 	uint64_t seq_no;
 	uint64_t status;
@@ -260,7 +319,6 @@ struct ras_bank_ecc_node {
 };
 
 struct ras_aca_config {
-	u32 socket_num_per_hive;
 	u32 aid_num_per_socket;
 	u32 xcd_num_per_aid;
 };
@@ -278,19 +336,22 @@ struct ras_psp_config {
 };
 
 struct ras_umc_config {
-	uint32_t umc_vram_type;
 	uint32_t num_umc;
+	/* this node's base in the hive PA space: physical_node_id * lfb_size */
+	uint64_t pa_base;
+	/* local frame buffer size */
+	uint64_t lfb_size;
 };
 
 struct ras_eeprom_config {
 	const struct ras_eeprom_sys_func *eeprom_sys_fn;
-	int eeprom_record_threshold_config;
-	uint32_t eeprom_record_threshold_count;
-	void *eeprom_i2c_adapter;
-	u32 eeprom_i2c_addr;
-	u32 eeprom_i2c_port;
-	u16 max_i2c_read_len;
-	u16 max_i2c_write_len;
+};
+
+struct ras_module_param {
+	/* driver installation option parameter */
+	int ras_feature_enable;
+	u64 ras_feature_mask;
+	int ras_bad_page_threshold;
 };
 
 struct ras_core_config {
@@ -301,6 +362,7 @@ struct ras_core_config {
 	u32 nbio_ip_version;
 	u32 psp_ip_version;
 
+	bool early_init_service_supported;
 	bool poison_supported;
 	bool ras_eeprom_supported;
 	uint ras_debug_mask;
@@ -312,12 +374,15 @@ struct ras_core_config {
 	struct ras_psp_config psp_cfg;
 	struct ras_eeprom_config eeprom_cfg;
 	struct ras_umc_config umc_cfg;
+
+	struct ras_module_param mod_param;
 };
 
 struct ras_core_context {
 	void *dev;
 	struct ras_core_config *config;
-	u32 socket_num_per_hive;
+	bool bert_platform_owner;
+	u32 socket_num_per_node;
 	u32 aid_num_per_socket;
 	u32 xcd_num_per_aid;
 	int max_ue_banks_per_query;
@@ -325,8 +390,7 @@ struct ras_core_context {
 	struct ras_aca ras_aca;
 
 	bool ras_eeprom_supported;
-	struct ras_eeprom_control ras_eeprom;
-	struct ras_fw_eeprom_control ras_fw_eeprom;
+	struct ras_eeprom_mgr eeprom_mgr;
 
 	struct ras_psp ras_psp;
 	struct ras_umc ras_umc;
@@ -335,6 +399,8 @@ struct ras_core_context {
 	struct ras_mp1 ras_mp1;
 	struct ras_process ras_proc;
 	struct ras_log_ring ras_log_ring;
+	struct ras_mce ras_mce;
+	struct ras_cper ras_cper;
 
 	const struct ras_sys_func *sys_fn;
 
@@ -343,6 +409,8 @@ struct ras_core_context {
 
 	bool is_rma;
 	bool is_initialized;
+	bool in_early_init;
+	bool early_init_service_enabled;
 
 	struct kfifo de_seqno_fifo;
 	struct kfifo consumption_seqno_fifo;
@@ -370,7 +438,7 @@ int ras_core_put_seqno(struct ras_core_context *ras_core,
 
 int ras_core_update_ecc_info(struct ras_core_context *ras_core);
 int ras_core_query_block_ecc_data(struct ras_core_context *ras_core,
-		enum ras_block_id block, struct ras_ecc_count *ecc_count);
+		enum ras_block_id block, struct ras_ecc_count *ecc_count, bool clear);
 
 bool ras_core_gpu_in_reset(struct ras_core_context *ras_core);
 bool ras_core_gpu_is_rma(struct ras_core_context *ras_core);
@@ -380,6 +448,7 @@ bool ras_core_handle_nbio_irq(struct ras_core_context *ras_core, void *data);
 int ras_core_handle_fatal_error(struct ras_core_context *ras_core);
 
 uint32_t ras_core_get_curr_nps_mode(struct ras_core_context *ras_core);
+uint32_t ras_core_get_vram_type(struct ras_core_context *ras_core);
 const char *ras_core_get_ras_block_name(enum ras_block_id block_id);
 int ras_core_convert_timestamp_to_time(struct ras_core_context *ras_core,
 			uint64_t timestamp, struct ras_time *tm);
@@ -408,4 +477,17 @@ int ras_core_check_address_sanity(struct ras_core_context *ras_core, uint64_t ad
 
 int ras_core_set_debug_mode(struct ras_core_context *ras_core, bool enable);
 bool ras_core_is_ce_log_disabled(struct ras_core_context *ras_core);
+int ras_core_get_eeprom_version(struct ras_core_context *ras_core,
+	uint32_t *version);
+int ras_core_get_ip_version(struct ras_core_context *ras_core,
+	enum ras_unit_id unit_id, uint32_t *version);
+uint64_t ras_core_get_ras_caps(struct ras_core_context *ras_core);
+bool ras_core_poison_supported(struct ras_core_context *ras_core);
+bool ras_core_in_early_init(struct ras_core_context *ras_core);
+bool ras_core_early_init_service_enabled(struct ras_core_context *ras_core);
+int ras_core_eeprom_early_init_service(struct ras_core_context *ras_core);
+int ras_core_get_module_param(struct ras_core_context *ras_core,
+		struct ras_module_param *param);
+int ras_core_add_log_event(struct ras_core_context *ras_core,
+		uint32_t event, void *data, uint32_t data_sz);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_bert.c b/drivers/gpu/drm/amd/ras/core/ras_bert.c
new file mode 100644
index 00000000000000..cc4140bb976123
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_bert.c
@@ -0,0 +1,546 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "ras.h"
+
+#if defined(CONFIG_X86_MCE_AMD) && defined(CONFIG_ACPI_APEI)
+#define RAS_BERT_ERR_INFO_COUNT(bits)  (((bits) & GENMASK_ULL(7, 2)) >> 2)
+#define RAS_BERT_CONTEXT_COUNT(bits)   (((bits) & GENMASK_ULL(13, 8)) >> 8)
+#define CPER_MEM_VALID_NODE            0x0008
+#define CPER_MEM_VALID_ERROR_STATUS    0x0001
+#define CPER_MEM_VALID_PA              0x0002
+#define CPER_MEM_VALID_PA_MASK         0x0004
+#define CPER_MEM_VALID_REQUESTOR_ID    0x0800
+#define CPER_MEM_VALID_RESPONDER_ID    0x1000
+#define CPER_MEM_VALID_TARGET_ID       0x2000
+#define CPER_MEM_VALID_ERROR_TYPE      0x4000
+#define AMD_CPER_BODY_VALID_MASK       GENMASK_ULL(15, 1)
+#define RAS_BERT_VALID_FRU_STRING       BIT(1)
+#define RAS_BERT_PROC_CTX_TYPE_MSR      1
+
+struct ras_bert_hest_generic_status {
+	u32 block_status;
+	u32 raw_data_offset;
+	u32 raw_data_length;
+	u32 data_length;
+	u32 error_severity;
+};
+
+struct ras_bert_hest_generic_data {
+	u8 section_type[16];
+	u32 error_severity;
+	u16 revision;
+	u8 validation_bits;
+	u8 flags;
+	u32 error_data_length;
+	u8 fru_id[16];
+	u8 fru_text[20];
+};
+
+struct ras_bert_hest_generic_data_v300 {
+	u8 section_type[16];
+	u32 error_severity;
+	u16 revision;
+	u8 validation_bits;
+	u8 flags;
+	u32 error_data_length;
+	u8 fru_id[16];
+	u8 fru_text[20];
+	u64 time_stamp;
+};
+
+struct ras_bert_sec_proc_ia {
+	u64 validation_bits;
+	u64 lapic_id;
+	u8 cpuid[48];
+};
+
+struct ras_bert_ia_err_info {
+	u8 err_type[16];
+	u64 validation_bits;
+	u64 check_info;
+	u64 target_id;
+	u64 requestor_id;
+	u64 responder_id;
+	u64 ip;
+};
+
+struct ras_bert_ia_proc_ctx {
+	u16 reg_ctx_type;
+	u16 reg_arr_size;
+	u32 msr_addr;
+	u64 mm_reg_addr;
+};
+
+struct ras_bert_sec_mem_err_old {
+	u64 validation_bits;
+	u64 error_status;
+	u64 physical_addr;
+	u64 physical_addr_mask;
+	u16 node;
+	u16 card;
+	u16 module;
+	u16 bank;
+	u16 device;
+	u16 row;
+	u16 column;
+	u16 bit_pos;
+	u64 requestor_id;
+	u64 responder_id;
+	u64 target_id;
+	u8 error_type;
+} __packed;
+
+struct ras_bert_sec_mem_err {
+	u64 validation_bits;
+	u64 error_status;
+	u64 physical_addr;
+	u64 physical_addr_mask;
+	u16 node;
+	u16 card;
+	u16 module;
+	u16 bank;
+	u16 device;
+	u16 row;
+	u16 column;
+	u16 bit_pos;
+	u64 requestor_id;
+	u64 responder_id;
+	u64 target_id;
+	u8 error_type;
+	u8 extended;
+	u16 rank;
+	u16 mem_array_handle;
+	u16 mem_dev_handle;
+} __packed;
+
+static inline int ras_bert_get_version(struct ras_bert_hest_generic_data *gdata)
+{
+	return gdata->revision >> 8;
+}
+
+static inline void *ras_bert_get_payload(struct ras_bert_hest_generic_data *gdata)
+{
+	if (ras_bert_get_version(gdata) >= 3)
+		return (void *)(((struct ras_bert_hest_generic_data_v300 *)(gdata)) + 1);
+
+	return gdata + 1;
+}
+
+static inline int ras_bert_get_size(struct ras_bert_hest_generic_data *gdata)
+{
+	if (ras_bert_get_version(gdata) >= 3)
+		return sizeof(struct ras_bert_hest_generic_data_v300);
+
+	return sizeof(struct ras_bert_hest_generic_data);
+}
+
+static inline void *ras_bert_get_next(struct ras_bert_hest_generic_data *gdata)
+{
+	return (void *)gdata + ras_bert_get_size(gdata) + gdata->error_data_length;
+}
+
+static int ras_bert_estatus_check(const struct ras_bert_hest_generic_status *estatus)
+{
+	struct ras_bert_hest_generic_data *gdata;
+	unsigned int data_len, record_size;
+
+	if (estatus->data_length &&
+	    estatus->data_length < sizeof(struct ras_bert_hest_generic_data))
+		return -EINVAL;
+	if (estatus->raw_data_length &&
+	    estatus->raw_data_offset < sizeof(*estatus) + estatus->data_length)
+		return -EINVAL;
+
+	data_len = estatus->data_length;
+	gdata = (struct ras_bert_hest_generic_data *)(estatus + 1);
+	while ((void *)gdata - (void *)(estatus + 1) < estatus->data_length) {
+		if (ras_bert_get_size(gdata) > data_len)
+			return -EINVAL;
+
+		record_size = ras_bert_get_size(gdata) + gdata->error_data_length;
+		if (record_size > data_len)
+			return -EINVAL;
+
+		data_len -= record_size;
+		gdata = ras_bert_get_next(gdata);
+	}
+	if (data_len)
+		return -EINVAL;
+
+	return 0;
+}
+
+typedef int (*ras_bert_section_parser)(struct ras_core_context *ras_core,
+				       struct ras_bert_hest_generic_data *gdata);
+
+struct ras_bert_section_dispatch {
+	const u8 *section_type;
+	ras_bert_section_parser parser;
+};
+
+static const struct ras_cper_guid ras_bert_guid_crashdump = GPU__CRASHDUMP;
+static const struct ras_cper_guid ras_bert_guid_proc_ia = PROC_ERR__SECTION_TYPE;
+static const struct ras_cper_guid ras_bert_guid_platform_mem = PLATFORM_MEM__SECTION_TYPE;
+static const struct ras_cper_guid ras_bert_guid_pcie = PCIE_ERR__SECTION_TYPE;
+static const struct ras_cper_guid ras_bert_guid_smn = SMN_ERR__SECTION_TYPE;
+
+static u64 ras_bert_read_le64(const u8 *data)
+{
+	u64 value = 0;
+	size_t i;
+
+	for (i = 0; i < sizeof(value); i++)
+		value |= (u64)data[i] << (i * 8);
+
+	return value;
+}
+
+static void ras_bert_fill_boot_ctx(struct ras_boot_err_ctx *ctx,
+				   struct ras_bert_hest_generic_data *gdata)
+{
+	memcpy(ctx->section_type, gdata->section_type, sizeof(ctx->section_type));
+	ctx->error_severity = gdata->error_severity;
+}
+
+static bool ras_bert_amd_body_valid(const u8 *body, u32 body_len)
+{
+	u64 valid_bits;
+	u32 context_count;
+	u32 error_count;
+	u32 min_size;
+
+	if (body_len < sizeof(struct crashdump_hdr))
+		return false;
+
+	valid_bits = ras_bert_read_le64(body);
+	if (valid_bits & ~AMD_CPER_BODY_VALID_MASK)
+		return false;
+	error_count = RAS_BERT_ERR_INFO_COUNT(valid_bits);
+	context_count = RAS_BERT_CONTEXT_COUNT(valid_bits);
+	if (!error_count && !context_count)
+		return false;
+
+	min_size = sizeof(struct crashdump_hdr) +
+		(error_count * sizeof(struct crashdump_error_info)) +
+		(context_count * sizeof(struct ras_bert_ia_proc_ctx));
+
+	return min_size <= body_len;
+}
+
+static bool ras_bert_fru_matches_device(struct ras_core_context *ras_core,
+					struct ras_bert_hest_generic_data *gdata)
+{
+	struct device_system_info dev_info = { 0 };
+	char fru_text[sizeof(gdata->fru_text) + 1] = { 0 };
+	unsigned int socket_id;
+
+	if (!(gdata->validation_bits & RAS_BERT_VALID_FRU_STRING))
+		return false;
+	if (ras_core_get_device_system_info(ras_core, &dev_info))
+		return false;
+
+	memcpy(fru_text, gdata->fru_text, sizeof(gdata->fru_text));
+	if (sscanf(fru_text, "OAM%u", &socket_id) != 1 &&
+	    sscanf(fru_text, "EAM%u", &socket_id) != 1)
+		return false;
+
+	return socket_id == dev_info.socket_id;
+}
+
+static void ras_bert_log_processor_section(struct ras_core_context *ras_core,
+					   struct ras_bert_hest_generic_data *gdata,
+					   const struct ras_bert_sec_proc_ia *proc,
+					   const struct ras_bert_ia_proc_ctx *ctx_info)
+{
+	struct ras_boot_err_ctx ctx = { 0 };
+	u16 data_size;
+
+	ras_bert_fill_boot_ctx(&ctx, gdata);
+	ctx.apic_id = proc->lapic_id;
+	if (proc->validation_bits & CPER_PROC_VALID_APIC_ID)
+		ctx.flags |= RAS_BOOT_CTX_VALID_APIC_ID;
+	ctx.reg_ctx_type = ctx_info->reg_ctx_type;
+	ctx.msr_addr = ctx_info->msr_addr;
+	ctx.mm_reg_addr = ctx_info->mm_reg_addr;
+	data_size = min_t(u16, ctx_info->reg_arr_size, sizeof(ctx.regs));
+	ctx.reg_arr_size = data_size;
+	memcpy(ctx.regs, ctx_info + 1, data_size);
+
+	ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_BOOT, &ctx, sizeof(ctx), NULL);
+}
+
+static int ras_bert_record_amd_contexts(struct ras_core_context *ras_core,
+					struct ras_bert_hest_generic_data *gdata,
+					const u8 *body)
+{
+	const struct ras_bert_ia_proc_ctx *context;
+	u64 valid_bits = ras_bert_read_le64(body);
+	u32 error_count = RAS_BERT_ERR_INFO_COUNT(valid_bits);
+	u32 context_count = RAS_BERT_CONTEXT_COUNT(valid_bits);
+	u32 offset = sizeof(struct crashdump_hdr) +
+		(error_count * sizeof(struct crashdump_error_info));
+	u32 i;
+
+	for (i = 0; i < context_count; i++) {
+		struct ras_boot_err_ctx ctx = { 0 };
+		u32 context_size;
+		u16 data_size;
+
+		if (offset > gdata->error_data_length ||
+		    sizeof(*context) > gdata->error_data_length - offset)
+			return -EINVAL;
+
+		context = (const struct ras_bert_ia_proc_ctx *)(body + offset);
+		context_size = ALIGN(sizeof(*context) + context->reg_arr_size, 16);
+		if (context_size > gdata->error_data_length - offset)
+			return -EINVAL;
+
+		ras_bert_fill_boot_ctx(&ctx, gdata);
+		ctx.reg_ctx_type = context->reg_ctx_type;
+		ctx.msr_addr = context->msr_addr;
+		ctx.mm_reg_addr = context->mm_reg_addr;
+		data_size = min_t(u16, context->reg_arr_size, sizeof(ctx.regs));
+		ctx.reg_arr_size = data_size;
+		memcpy(ctx.regs, context + 1, data_size);
+		ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_BOOT,
+					   &ctx, sizeof(ctx), NULL);
+
+		offset += context_size;
+	}
+
+	return 0;
+}
+
+static int ras_bert_record_section(struct ras_core_context *ras_core,
+				   struct ras_bert_hest_generic_data *gdata)
+{
+	struct aca_bank_ecc err = { 0 };
+	struct aca_bank_reg bank = { 0 };
+	struct device_system_info dev_info = { 0 };
+	struct ras_boot_err_ctx ctx = { 0 };
+	u64 *regs = ras_bert_get_payload(gdata);
+	u32 data_size;
+	int ret;
+
+	if (ras_bert_amd_body_valid((u8 *)regs, gdata->error_data_length)) {
+		if (!ras_bert_fru_matches_device(ras_core, gdata))
+			return 0;
+
+		return ras_bert_record_amd_contexts(ras_core, gdata,
+						    (const u8 *)regs);
+	}
+
+	if (gdata->error_data_length < sizeof(u64) * (RAS_CPER_ACA_REG_IPID + 1))
+		return 0;
+
+	bank.bank_type = MCE_BANK_TYPE_GPU;
+	data_size = min_t(u32, gdata->error_data_length, sizeof(bank.regs));
+	memcpy(bank.regs, regs, data_size);
+	ret = ras_aca_parse_bank(ras_core, &bank, &err);
+	if (ret || ras_core_get_device_system_info(ras_core, &dev_info) ||
+	    err.bank_info.socket_id != dev_info.socket_id)
+		return 0;
+
+	ras_bert_fill_boot_ctx(&ctx, gdata);
+	ctx.reg_ctx_type = CPER_CTX_TYPE__BOOT;
+	data_size = min_t(u32, data_size, sizeof(ctx.regs));
+	ctx.reg_arr_size = data_size;
+	memcpy(ctx.regs, regs, data_size);
+
+	ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_BOOT, &ctx, sizeof(ctx), NULL);
+
+	return 0;
+}
+
+static int ras_bert_record_proc_ia_section(struct ras_core_context *ras_core,
+					   struct ras_bert_hest_generic_data *gdata)
+{
+	struct ras_bert_ia_err_info *err_info;
+	struct ras_bert_ia_proc_ctx *ctx_info;
+	struct ras_bert_sec_proc_ia *proc;
+	u8 *section_end;
+	int ctx_count;
+	int err_count;
+	int i;
+
+	if (gdata->error_data_length < sizeof(*proc))
+		return -EINVAL;
+
+	if (!ras_core->bert_platform_owner)
+		return 0;
+
+	proc = ras_bert_get_payload(gdata);
+	section_end = (u8 *)proc + gdata->error_data_length;
+	err_count = RAS_BERT_ERR_INFO_COUNT(proc->validation_bits);
+	ctx_count = RAS_BERT_CONTEXT_COUNT(proc->validation_bits);
+	err_info = (struct ras_bert_ia_err_info *)(proc + 1);
+	if ((u8 *)err_info > section_end ||
+	    err_count * sizeof(*err_info) > section_end - (u8 *)err_info)
+		return -EINVAL;
+
+	err_info += err_count;
+	ctx_info = (struct ras_bert_ia_proc_ctx *)err_info;
+	for (i = 0; i < ctx_count; i++) {
+		size_t remaining;
+		size_t ctx_size;
+
+		if ((u8 *)ctx_info > section_end ||
+		    sizeof(*ctx_info) > section_end - (u8 *)ctx_info)
+			return -EINVAL;
+
+		remaining = section_end - (u8 *)ctx_info;
+		if (ctx_info->reg_arr_size > remaining - sizeof(*ctx_info))
+			return -EINVAL;
+		ctx_size = ALIGN(sizeof(*ctx_info) + ctx_info->reg_arr_size, 16);
+		if (ctx_size > remaining)
+			return -EINVAL;
+		if (ctx_info->reg_ctx_type != RAS_BERT_PROC_CTX_TYPE_MSR ||
+		    ctx_info->msr_addr < CPER_SMCA_MC0_STATUS_MSR ||
+		    (ctx_info->msr_addr - CPER_SMCA_MC0_STATUS_MSR) %
+		    CPER_SMCA_BANK_STRIDE)
+			goto next_ctx;
+		if (ctx_info->reg_arr_size < sizeof(u64))
+			goto next_ctx;
+
+		ras_bert_log_processor_section(ras_core, gdata, proc, ctx_info);
+
+next_ctx:
+		ctx_info = (struct ras_bert_ia_proc_ctx *)((u8 *)ctx_info + ctx_size);
+	}
+
+	return 0;
+}
+
+static int ras_bert_record_platform_mem_section(struct ras_core_context *ras_core,
+						struct ras_bert_hest_generic_data *gdata)
+{
+	struct ras_boot_err_ctx ctx = { 0 };
+	struct ras_bert_sec_mem_err *mem;
+	u16 data_size;
+
+	if (gdata->error_data_length != sizeof(struct ras_bert_sec_mem_err_old) &&
+	    gdata->error_data_length != sizeof(struct ras_bert_sec_mem_err))
+		return -EINVAL;
+
+	if (!ras_core->bert_platform_owner)
+		return 0;
+
+	mem = ras_bert_get_payload(gdata);
+	if (gdata->error_data_length > sizeof(ctx.raw_data))
+		return -EOVERFLOW;
+
+	ras_bert_fill_boot_ctx(&ctx, gdata);
+	data_size = gdata->error_data_length;
+	ctx.raw_data_size = data_size;
+	memcpy(ctx.raw_data, mem, data_size);
+
+	ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_BOOT, &ctx, sizeof(ctx), NULL);
+
+	return 0;
+}
+
+static const struct ras_bert_section_dispatch ras_bert_section_dispatch[] = {
+	{ ras_bert_guid_crashdump.b, ras_bert_record_section },
+	{ ras_bert_guid_pcie.b, ras_bert_record_section },
+	{ ras_bert_guid_smn.b, ras_bert_record_section },
+	{ ras_bert_guid_proc_ia.b, ras_bert_record_proc_ia_section },
+	{ ras_bert_guid_platform_mem.b, ras_bert_record_platform_mem_section },
+};
+
+static const struct ras_bert_section_dispatch *
+ras_bert_find_section_parser(const u8 *section_type)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ras_bert_section_dispatch); i++) {
+		if (!memcmp(section_type, ras_bert_section_dispatch[i].section_type,
+			    CPER_UUID_MAX_SIZE))
+			return &ras_bert_section_dispatch[i];
+	}
+
+	return NULL;
+}
+
+int ras_bert_process_records(struct ras_core_context *ras_core,
+			     const void *bert, u32 bert_len)
+{
+	struct ras_bert_hest_generic_status *estatus;
+	struct ras_bert_hest_generic_data *gdata;
+	u32 estatus_len;
+	int remain;
+	int ret;
+
+	if (!ras_core || !bert || bert_len < sizeof(struct ras_bert_hest_generic_status))
+		return -EINVAL;
+
+	estatus = (struct ras_bert_hest_generic_status *)bert;
+	remain = bert_len;
+
+	while (remain >= sizeof(struct ras_bert_hest_generic_status)) {
+		estatus_len = estatus->raw_data_length ?
+			      estatus->raw_data_offset + estatus->raw_data_length :
+			      sizeof(*estatus) + estatus->data_length;
+		if (remain < estatus_len) {
+			RAS_DEV_ERR(ras_core->dev, "truncated status block (length: %u).\n",
+				    estatus_len);
+			return -EINVAL;
+		}
+		if (!estatus_len)
+			return -EINVAL;
+
+		/*
+		 * The generic APEI BERT path may have already consumed the boot
+		 * error region and cleared block_status. AMDGPU still needs to
+		 * parse the cached BERT payload, so do not use block_status to
+		 * decide whether this status block contains records.
+		 */
+		if (ras_bert_estatus_check(estatus)) {
+			RAS_DEV_ERR(ras_core->dev, "invalid error record.\n");
+			return -EINVAL;
+		}
+
+		gdata = (struct ras_bert_hest_generic_data *)(estatus + 1);
+		while ((void *)gdata - (void *)(estatus + 1) < estatus->data_length) {
+			const struct ras_bert_section_dispatch *parser;
+
+			parser = ras_bert_find_section_parser(gdata->section_type);
+			if (parser) {
+				ret = parser->parser(ras_core, gdata);
+				if (ret)
+					return ret;
+			} else {
+				RAS_DEV_INFO(ras_core->dev, "skip section: %pUl\n",
+					     gdata->section_type);
+			}
+
+			gdata = ras_bert_get_next(gdata);
+		}
+		estatus = (struct ras_bert_hest_generic_status *)((u8 *)estatus + estatus_len);
+		remain -= estatus_len;
+	}
+
+	return 0;
+}
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_bert.h b/drivers/gpu/drm/amd/ras/core/ras_bert.h
new file mode 100644
index 00000000000000..523281afe0d000
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_bert.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __RAS_BERT_H__
+#define __RAS_BERT_H__
+
+struct ras_core_context;
+
+int ras_bert_process_records(struct ras_core_context *ras_core,
+			     const void *bert, u32 bert_len);
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_cper.c b/drivers/gpu/drm/amd/ras/core/ras_cper.c
index 4ffac4289dbdb4..c8506f695d6079 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_cper.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_cper.c
@@ -25,13 +25,15 @@
 #include "core_status.h"
 #include "log_ring.h"
 #include "ras_cper.h"
+#define ns_to_seconds(ns)   div_u64(ns, NSEC_PER_SEC)
 
-static const struct ras_cper_guid MCE	= CPER_NOTIFY__MCE;
-static const struct ras_cper_guid CMC	= CPER_NOTIFY__CMC;
-static const struct ras_cper_guid BOOT	= BOOT__TYPE;
+static const struct ras_cper_guid processor_section_type =
+	PROC_ERR__SECTION_TYPE;
+static const struct ras_cper_guid platform_memory_section_type =
+	PLATFORM_MEM__SECTION_TYPE;
 
-static const struct ras_cper_guid CRASHDUMP = GPU__CRASHDUMP;
-static const struct ras_cper_guid RUNTIME = GPU__NONSTANDARD_ERROR;
+static int ras_cper_get_profile(struct ras_core_context *ras_core,
+		enum ras_log_event event, struct ras_cper_profile **profile);
 
 static void cper_get_timestamp(struct ras_core_context *ras_core,
 		struct ras_cper_timestamp *timestamp, uint64_t utc_second_timestamp)
@@ -50,8 +52,8 @@ static void cper_get_timestamp(struct ras_core_context *ras_core,
 }
 
 static void fill_section_hdr(struct ras_core_context *ras_core,
-				struct cper_section_hdr *hdr, enum ras_cper_type type,
-				enum ras_cper_severity sev, struct ras_log_info *trace)
+		struct cper_section_hdr *hdr, struct ras_cper_guid notify_type,
+		enum ras_cper_severity sev, struct ras_log_info *log)
 {
 	struct device_system_info dev_info = {0};
 	char record_id[32];
@@ -69,10 +71,10 @@ static void fill_section_hdr(struct ras_core_context *ras_core,
 
 	ras_core_get_device_system_info(ras_core, &dev_info);
 
-	cper_get_timestamp(ras_core, &hdr->timestamp, trace->timestamp);
+	cper_get_timestamp(ras_core, &hdr->timestamp, ns_to_seconds(log->timestamp));
 
 	snprintf(record_id, sizeof(record_id), "%d:%llX", dev_info.socket_id,
-		    RAS_LOG_SEQNO_TO_BATCH_IDX(trace->seqno));
+		    RAS_LOG_SEQNO_TO_BATCH_IDX(log->seqno));
 	memcpy(hdr->record_id, record_id, 8);
 
 	snprintf(hdr->platform_id, 16, "0x%04X:0x%04X",
@@ -80,40 +82,20 @@ static void fill_section_hdr(struct ras_core_context *ras_core,
 	/* pmfw version should be part of creator_id according to CPER spec */
 	snprintf(hdr->creator_id, 16, "%s", CPER_CREATOR_ID__AMDGPU);
 
-	switch (type) {
-	case RAS_CPER_TYPE_BOOT:
-		hdr->notify_type = BOOT;
-		break;
-	case RAS_CPER_TYPE_FATAL:
-	case RAS_CPER_TYPE_RMA:
-		hdr->notify_type = MCE;
-		break;
-	case RAS_CPER_TYPE_RUNTIME:
-		if (sev == RAS_CPER_SEV_NON_FATAL_CE)
-			hdr->notify_type = CMC;
-		else
-			hdr->notify_type = MCE;
-		break;
-	default:
-		RAS_DEV_ERR(ras_core->dev, "Unknown CPER Type\n");
-		break;
-	}
+	hdr->notify_type = notify_type;
 }
 
 static int fill_section_descriptor(struct ras_core_context *ras_core,
-					struct cper_section_descriptor *descriptor,
-					enum ras_cper_severity sev,
-					struct ras_cper_guid sec_type,
-					uint32_t section_offset,
-					uint32_t section_length)
+		struct cper_section_desc *descriptor, enum ras_cper_severity sev,
+		struct ras_cper_guid sec_type, uint32_t sec_offset, uint32_t sec_len)
 {
 	struct device_system_info dev_info = {0};
 
 	descriptor->revision_minor		= CPER_SEC__MINOR_REV_1;
 	descriptor->revision_major		= CPER_SEC__MAJOR_REV_22;
-	descriptor->sec_offset		= section_offset;
-	descriptor->sec_length		= section_length;
-	descriptor->valid_bits.fru_text	= 1;
+	descriptor->sec_offset		= sec_offset;
+	descriptor->sec_length		= sec_len;
+	descriptor->valid_bits.fru_text = 1;
 	descriptor->flag_bits.primary	= 1;
 	descriptor->severity = (sev == RAS_CPER_SEV_RMA ? RAS_CPER_SEV_FATAL_UE : sev);
 	descriptor->sec_type			= sec_type;
@@ -131,28 +113,34 @@ static int fill_section_descriptor(struct ras_core_context *ras_core,
 	return 0;
 }
 
-static int fill_section_fatal(struct ras_core_context *ras_core,
-		struct cper_section_fatal *fatal, struct ras_log_info *trace)
+static int fill_section_fatal(struct ras_core_context *ras_core, void *section,
+		struct ras_log_info *log, enum ras_cper_severity sev)
 {
+	struct cper_section_fatal *fatal = section;
+
+	fatal->hdr.valid_bits = CPER_AMD_ERR_INFO_COUNT(1) |
+		CPER_AMD_CONTEXT_COUNT(1);
+	fatal->error_info.error_type = GPU__NONSTANDARD_ERROR;
 	fatal->data.reg_ctx_type = CPER_CTX_TYPE__CRASH;
 	fatal->data.reg_arr_size = sizeof(fatal->data.reg);
 
-	fatal->data.reg.status = trace->aca_reg.regs[RAS_CPER_ACA_REG_STATUS];
-	fatal->data.reg.addr   = trace->aca_reg.regs[RAS_CPER_ACA_REG_ADDR];
-	fatal->data.reg.ipid   = trace->aca_reg.regs[RAS_CPER_ACA_REG_IPID];
-	fatal->data.reg.synd   = trace->aca_reg.regs[RAS_CPER_ACA_REG_SYND];
+	fatal->data.reg.status = log->body.aca_reg.regs[RAS_CPER_ACA_REG_STATUS];
+	fatal->data.reg.addr   = log->body.aca_reg.regs[RAS_CPER_ACA_REG_ADDR];
+	fatal->data.reg.ipid   = log->body.aca_reg.regs[RAS_CPER_ACA_REG_IPID];
+	fatal->data.reg.synd   = log->body.aca_reg.regs[RAS_CPER_ACA_REG_SYND];
 
 	return 0;
 }
 
-static int fill_section_runtime(struct ras_core_context *ras_core,
-		struct cper_section_runtime *runtime, struct ras_log_info *trace,
-		enum ras_cper_severity sev)
+static int fill_section_runtime(struct ras_core_context *ras_core, void *section,
+		struct ras_log_info *log, enum ras_cper_severity sev)
 {
+	struct cper_section_runtime *runtime = section;
+
 	runtime->hdr.valid_bits.err_info_cnt = 1;
 	runtime->hdr.valid_bits.err_context_cnt = 1;
 
-	runtime->descriptor.error_type = RUNTIME;
+	runtime->descriptor.error_type = GPU__NONSTANDARD_ERROR;
 	runtime->descriptor.ms_chk_bits.err_type_valid = 1;
 	if (sev == RAS_CPER_SEV_RMA) {
 		runtime->descriptor.valid_bits.ms_chk = 1;
@@ -163,153 +151,691 @@ static int fill_section_runtime(struct ras_core_context *ras_core,
 	runtime->reg.reg_ctx_type = CPER_CTX_TYPE__CRASH;
 	runtime->reg.reg_arr_size = sizeof(runtime->reg.reg_dump);
 
-	runtime->reg.reg_dump[RAS_CPER_ACA_REG_CTL]    = trace->aca_reg.regs[ACA_REG_IDX__CTL];
-	runtime->reg.reg_dump[RAS_CPER_ACA_REG_STATUS] = trace->aca_reg.regs[ACA_REG_IDX__STATUS];
-	runtime->reg.reg_dump[RAS_CPER_ACA_REG_ADDR]   = trace->aca_reg.regs[ACA_REG_IDX__ADDR];
-	runtime->reg.reg_dump[RAS_CPER_ACA_REG_MISC0]  = trace->aca_reg.regs[ACA_REG_IDX__MISC0];
-	runtime->reg.reg_dump[RAS_CPER_ACA_REG_CONFIG] = trace->aca_reg.regs[ACA_REG_IDX__CONFG];
-	runtime->reg.reg_dump[RAS_CPER_ACA_REG_IPID]   = trace->aca_reg.regs[ACA_REG_IDX__IPID];
-	runtime->reg.reg_dump[RAS_CPER_ACA_REG_SYND]   = trace->aca_reg.regs[ACA_REG_IDX__SYND];
+	runtime->reg.reg_dump[RAS_CPER_ACA_REG_CTL] =
+			log->body.aca_reg.regs[ACA_REG_IDX__CTL];
+	runtime->reg.reg_dump[RAS_CPER_ACA_REG_STATUS] =
+			log->body.aca_reg.regs[ACA_REG_IDX__STATUS];
+	runtime->reg.reg_dump[RAS_CPER_ACA_REG_ADDR] =
+			log->body.aca_reg.regs[ACA_REG_IDX__ADDR];
+	runtime->reg.reg_dump[RAS_CPER_ACA_REG_MISC0] =
+			log->body.aca_reg.regs[ACA_REG_IDX__MISC0];
+	runtime->reg.reg_dump[RAS_CPER_ACA_REG_CONFIG] =
+			log->body.aca_reg.regs[ACA_REG_IDX__CONFG];
+	runtime->reg.reg_dump[RAS_CPER_ACA_REG_IPID] =
+			log->body.aca_reg.regs[ACA_REG_IDX__IPID];
+	runtime->reg.reg_dump[RAS_CPER_ACA_REG_SYND] =
+			log->body.aca_reg.regs[ACA_REG_IDX__SYND];
+
+	return 0;
+}
+
+static int fill_section_boot(struct ras_core_context *ras_core, void *section,
+		struct ras_log_info *log, enum ras_cper_severity sev)
+{
+	struct cper_section_boot *boot = section;
+	struct ras_boot_err_ctx *ctx = &log->body.boot_err_ctx;
+	struct crashdump_boot *data = &boot->data;
+	u16 data_size;
+
+	boot->hdr.valid_bits = CPER_AMD_ERR_INFO_COUNT(1) |
+		CPER_AMD_CONTEXT_COUNT(1);
+	boot->error_info.error_type = GPU__NONSTANDARD_ERROR;
+	data->reg_ctx_type = ctx->reg_ctx_type;
+	data_size = min_t(u16, ctx->reg_arr_size, sizeof(data->msg));
+	data->reg_arr_size = data_size;
+	memcpy(data->msg, ctx->regs, data_size);
+
+	return 0;
+}
+
+static void fill_processor_error_info(struct cper_processor_error_info *error_info,
+		u64 status)
+{
+	u64 check_info = CPER_MS_CHECK_VALID_ERR_TYPE |
+		CPER_MS_CHECK_ERR_TYPE_INTERNAL;
+
+	error_info->error_type = PROC_ERR__MS_CHECK_TYPE;
+	error_info->valid_bits = CPER_PROC_INFO_VALID_CHECK_INFO;
+
+	if (status & CPER_MCA_STATUS_PCC)
+		check_info |= CPER_MS_CHECK_VALID_PCC | CPER_MS_CHECK_PCC;
+	if (status & CPER_MCA_STATUS_UNCORRECTED)
+		check_info |= CPER_MS_CHECK_VALID_UNCORRECTED |
+			CPER_MS_CHECK_UNCORRECTED;
+	if (status & CPER_MCA_STATUS_OVERFLOW)
+		check_info |= CPER_MS_CHECK_VALID_OVERFLOW |
+			CPER_MS_CHECK_OVERFLOW;
+
+	error_info->check_info = check_info;
+}
+
+static int fill_section_processor(struct cper_section_processor *processor,
+		struct ras_log_info *log)
+{
+	const struct ras_cpu_mce *mce = &log->body.cpu_mce;
+	u64 status = mce->regs[RAS_CPER_ACA_REG_STATUS];
+	u64 *reg_dump = processor->context.reg_dump;
+
+	processor->processor.valid_bits = CPER_PROC_VALID_APIC_ID |
+		CPER_PROC_ERR_INFO_COUNT(1) | CPER_PROC_CONTEXT_COUNT(1);
+	processor->processor.apic_id = mce->apic_id;
+	fill_processor_error_info(&processor->error_info, status);
+	processor->context.reg_ctx_type = CPER_CTX_TYPE__CRASH;
+	processor->context.reg_arr_size = sizeof(processor->context.reg_dump);
+	processor->context.msr_addr = CPER_SMCA_MC0_STATUS_MSR +
+		(mce->bank * CPER_SMCA_BANK_STRIDE);
+	/* The IA processor register array starts at MCA_STATUS. */
+	reg_dump[0] = mce->regs[ACA_REG_IDX__STATUS];
+	reg_dump[1] = mce->regs[ACA_REG_IDX__ADDR];
+	reg_dump[2] = mce->regs[ACA_REG_IDX__MISC0];
+	reg_dump[3] = mce->regs[ACA_REG_IDX__CONFG];
+	reg_dump[4] = mce->regs[ACA_REG_IDX__IPID];
+	reg_dump[5] = mce->regs[ACA_REG_IDX__SYND];
+	reg_dump[7] = mce->regs[ACA_REG_IDX__DESTAT];
+	reg_dump[8] = mce->regs[ACA_REG_IDX__DEADDR];
+	reg_dump[9] = mce->regs[ACA_REG_IDX__CTL_MASK];
 
 	return 0;
 }
 
-static int cper_generate_runtime_record(struct ras_core_context *ras_core,
-	struct cper_section_hdr *hdr, struct ras_log_info *trace_arr, uint32_t arr_num,
-		enum ras_cper_severity sev)
+static int fill_section_boot_processor(struct cper_section_processor *processor,
+		struct ras_log_info *log)
+{
+	const struct ras_boot_err_ctx *ctx = &log->body.boot_err_ctx;
+	u64 status = ctx->regs[0];
+
+	processor->processor.valid_bits = CPER_PROC_ERR_INFO_COUNT(1) |
+		CPER_PROC_CONTEXT_COUNT(1);
+	if (ctx->flags & RAS_BOOT_CTX_VALID_APIC_ID)
+		processor->processor.valid_bits |= CPER_PROC_VALID_APIC_ID;
+	processor->processor.apic_id = ctx->apic_id;
+	fill_processor_error_info(&processor->error_info, status);
+	processor->context.reg_ctx_type = ctx->reg_ctx_type;
+	processor->context.reg_arr_size = ctx->reg_arr_size;
+	processor->context.msr_addr = ctx->msr_addr;
+	processor->context.mm_reg_addr = ctx->mm_reg_addr;
+	memcpy(processor->context.reg_dump, ctx->regs,
+		min_t(size_t, ctx->reg_arr_size,
+		      sizeof(processor->context.reg_dump)));
+
+	return 0;
+}
+
+static int cper_boot_get_severity(struct ras_log_info *log,
+			enum ras_cper_severity *sev)
+{
+	*sev = log->body.boot_err_ctx.error_severity;
+	return 0;
+}
+
+static int cper_boot_get_sec_type(struct ras_log_info *log, struct ras_cper_guid *out)
+{
+	memcpy(out, &log->body.boot_err_ctx.section_type,
+		min(sizeof(*out), sizeof(log->body.boot_err_ctx.section_type)));
+	return 0;
+}
+
+static uint32_t cper_get_record_size(const struct ras_cper_profile *profile,
+		uint32_t section_count)
+{
+	if (profile->build_record_per_log)
+		return (RAS_HDR_LEN + RAS_SEC_DESC_LEN + profile->section_size) *
+			section_count;
+
+	return RAS_HDR_LEN +
+		(RAS_SEC_DESC_LEN + profile->section_size) * section_count;
+}
+
+static int cper_build_single_record(struct ras_core_context *ras_core,
+		struct cper_section_hdr *hdr, const struct ras_cper_profile *profile,
+		struct ras_log_info *batch_logs, uint32_t nr_batch_logs)
 {
-	struct cper_section_descriptor *descriptor;
-	struct cper_section_runtime *runtime;
-	int i;
-
-	fill_section_hdr(ras_core, hdr, RAS_CPER_TYPE_RUNTIME, sev, &trace_arr[0]);
-	hdr->record_length =  RAS_HDR_LEN + ((RAS_SEC_DESC_LEN + RAS_NONSTD_SEC_LEN) * arr_num);
-	hdr->sec_cnt = arr_num;
-	for (i = 0; i < arr_num; i++) {
-		descriptor = (struct cper_section_descriptor *)((uint8_t *)hdr +
-			     RAS_SEC_DESC_OFFSET(i));
-		runtime = (struct cper_section_runtime *)((uint8_t *)hdr +
-			  RAS_NONSTD_SEC_OFFSET(hdr->sec_cnt, i));
-
-		fill_section_descriptor(ras_core, descriptor, sev, RUNTIME,
-			RAS_NONSTD_SEC_OFFSET(hdr->sec_cnt, i),
-			sizeof(struct cper_section_runtime));
-		fill_section_runtime(ras_core, runtime, &trace_arr[i], sev);
+	enum ras_cper_severity sev = profile->severity;
+	struct ras_cper_guid sec_type = profile->sec_type;
+	struct ras_cper_guid notify_type = profile->notify_type;
+	struct cper_section_desc *descriptor;
+	uint32_t desc_off, sec_off;
+	void *section;
+	uint32_t i;
+
+	if (profile->get_severity)
+		profile->get_severity(&batch_logs[0], &sev);
+
+	if (profile->get_sec_type)
+		profile->get_sec_type(&batch_logs[0], &sec_type);
+
+	if (profile->get_notify_type)
+		profile->get_notify_type(&batch_logs[0], &notify_type);
+
+	fill_section_hdr(ras_core, hdr, notify_type, sev, &batch_logs[0]);
+	hdr->sec_cnt = nr_batch_logs;
+	hdr->record_length = RAS_HDR_LEN +
+		(RAS_SEC_DESC_LEN + profile->section_size) * nr_batch_logs;
+
+	for (i = 0; i < nr_batch_logs; i++) {
+		desc_off = RAS_HDR_LEN + (RAS_SEC_DESC_LEN * i);
+		sec_off  = RAS_HDR_LEN + (RAS_SEC_DESC_LEN * nr_batch_logs) +
+			   (profile->section_size * i);
+
+		descriptor = (struct cper_section_desc *)((uint8_t *)hdr + desc_off);
+		section = (uint8_t *)hdr + sec_off;
+
+		fill_section_descriptor(ras_core, descriptor, sev,
+			sec_type, sec_off, profile->section_size);
+
+		profile->fill_section(ras_core, section, &batch_logs[i], sev);
 	}
 
 	return 0;
 }
 
-static int cper_generate_fatal_record(struct ras_core_context *ras_core,
-	uint8_t *buffer, struct ras_log_info *trace_arr, uint32_t arr_num)
+static int cper_build_multiple_records(struct ras_core_context *ras_core,
+		uint8_t *buffer, const struct ras_cper_profile *profile,
+		struct ras_log_info *batch_logs, uint32_t nr_batch_logs)
 {
-	struct ras_cper_fatal_record record = {0};
-	int i = 0;
+	uint32_t rec_len = RAS_HDR_LEN + RAS_SEC_DESC_LEN + profile->section_size;
+	uint32_t i;
 
-	for (i = 0; i < arr_num; i++) {
-		fill_section_hdr(ras_core, &record.hdr, RAS_CPER_TYPE_FATAL,
-				 RAS_CPER_SEV_FATAL_UE, &trace_arr[i]);
-		record.hdr.record_length =  RAS_HDR_LEN + RAS_SEC_DESC_LEN + RAS_FATAL_SEC_LEN;
+	for (i = 0; i < nr_batch_logs; i++) {
+		struct ras_log_info *log = &batch_logs[i];
+		enum ras_cper_severity sev = profile->severity;
+		struct ras_cper_guid sec_type = profile->sec_type;
+		struct ras_cper_guid notify_type = profile->notify_type;
+		struct cper_section_hdr *hdr;
+		struct cper_section_desc *descriptor;
+		void *section;
+
+		if (profile->get_severity)
+			profile->get_severity(log, &sev);
+
+		if (profile->get_sec_type)
+			profile->get_sec_type(log, &sec_type);
+
+		if (profile->get_notify_type)
+			profile->get_notify_type(log, &notify_type);
+
+		hdr = (struct cper_section_hdr *)(buffer + (i * rec_len));
+		descriptor = (struct cper_section_desc *)((uint8_t *)hdr + RAS_HDR_LEN);
+		section = (uint8_t *)hdr + RAS_HDR_LEN + RAS_SEC_DESC_LEN;
+
+		fill_section_hdr(ras_core, hdr, notify_type, sev, log);
+		hdr->sec_cnt = 1;
+		hdr->record_length = rec_len;
+
+		fill_section_descriptor(ras_core, descriptor, sev, sec_type,
+			RAS_HDR_LEN + RAS_SEC_DESC_LEN, profile->section_size);
+
+		profile->fill_section(ras_core, section, log, sev);
+	}
+
+	return 0;
+}
+
+static int cper_generate_boot_processor_records(struct ras_core_context *ras_core,
+		u8 *buffer, struct ras_log_info *batch_logs, u32 nr_batch_logs)
+{
+	u32 offset = 0;
+	u32 i;
+
+	for (i = 0; i < nr_batch_logs; i++) {
+		const struct ras_boot_err_ctx *ctx = &batch_logs[i].body.boot_err_ctx;
+		struct ras_cper_processor_record record = { 0 };
+		u16 data_size = min_t(u16, ctx->reg_arr_size,
+					 sizeof(record.processor.context.reg_dump));
+		u32 section_size = offsetof(struct cper_section_processor,
+					    context.reg_dump) + data_size;
+		u32 record_size = RAS_HDR_LEN + RAS_SEC_DESC_LEN + section_size;
+		enum ras_cper_severity sev = ctx->error_severity;
+
+		fill_section_hdr(ras_core, &record.hdr, BOOT__TYPE, sev,
+				 &batch_logs[i]);
+		record.hdr.record_length = record_size;
 		record.hdr.sec_cnt = 1;
+		fill_section_descriptor(ras_core, &record.descriptor, sev,
+				PROC_ERR__SECTION_TYPE,
+				offsetof(struct ras_cper_processor_record, processor),
+				section_size);
+		fill_section_boot_processor(&record.processor, &batch_logs[i]);
+		memcpy(buffer + offset, &record, record_size);
+		offset += record_size;
+	}
 
-		fill_section_descriptor(ras_core, &record.descriptor, RAS_CPER_SEV_FATAL_UE,
-					CRASHDUMP, offsetof(struct ras_cper_fatal_record, fatal),
-					sizeof(struct cper_section_fatal));
+	return offset;
+}
 
-		fill_section_fatal(ras_core, &record.fatal, &trace_arr[i]);
+static int cper_generate_runtime_processor_records(struct ras_core_context *ras_core,
+		u8 *buffer, struct ras_log_info *batch_logs, u32 nr_batch_logs)
+{
+	u32 i;
+
+	for (i = 0; i < nr_batch_logs; i++) {
+		struct ras_cper_processor_record record = { 0 };
+		u64 status = batch_logs[i].body.cpu_mce.regs[ACA_REG_IDX__STATUS];
+		enum ras_cper_severity sev;
+
+		if (!(status & CPER_MCA_STATUS_UNCORRECTED) &&
+		    !(status & CPER_MCA_STATUS_DEFERRED))
+			sev = RAS_CPER_SEV_NON_FATAL_CE;
+		else if ((status & CPER_MCA_STATUS_UNCORRECTED) &&
+			 (status & CPER_MCA_STATUS_PCC))
+			sev = RAS_CPER_SEV_FATAL_UE;
+		else
+			sev = RAS_CPER_SEV_NON_FATAL_UE;
 
-		memcpy(buffer + (i * record.hdr.record_length),
-				&record, record.hdr.record_length);
+		fill_section_hdr(ras_core, &record.hdr, CPER_NOTIFY__MCE, sev,
+				 &batch_logs[i]);
+		record.hdr.revision = CPER_HDR__REV_AMD_CPU;
+		record.hdr.record_length = sizeof(record);
+		record.hdr.sec_cnt = 1;
+		fill_section_descriptor(ras_core, &record.descriptor, sev,
+				PROC_ERR__SECTION_TYPE,
+				offsetof(struct ras_cper_processor_record, processor),
+				RAS_PROC_SEC_LEN);
+		record.descriptor.flag_bits.latent_err =
+			!!(status & CPER_MCA_STATUS_DEFERRED);
+		record.descriptor.revision_minor = CPER_SEC__REV_AMD_CPU & 0xff;
+		record.descriptor.revision_major = CPER_SEC__REV_AMD_CPU >> 8;
+		fill_section_processor(&record.processor, &batch_logs[i]);
+		memcpy(buffer + (i * sizeof(record)), &record, sizeof(record));
 	}
 
 	return 0;
 }
 
-static int cper_get_record_size(enum ras_cper_type type, uint16_t section_count)
+static bool cper_is_processor_record(struct ras_log_info *log)
+{
+	return log->event == RAS_LOG_EVENT_CPU_RAS ||
+		(log->event == RAS_LOG_EVENT_BOOT &&
+		 !memcmp(log->body.boot_err_ctx.section_type,
+			 processor_section_type.b, CPER_UUID_MAX_SIZE));
+}
+
+static int cper_generate_processor_records(struct ras_core_context *ras_core,
+		struct ras_log_info *batch_logs, u32 nr_batch_logs,
+		u8 *buffer, u32 buf_len, u32 *real_data_len)
 {
-	int size = 0;
-
-	size += RAS_HDR_LEN;
-	size += (RAS_SEC_DESC_LEN * section_count);
-
-	switch (type) {
-	case RAS_CPER_TYPE_RUNTIME:
-	case RAS_CPER_TYPE_RMA:
-		size += (RAS_NONSTD_SEC_LEN * section_count);
-		break;
-	case RAS_CPER_TYPE_FATAL:
-		size += (RAS_FATAL_SEC_LEN * section_count);
-		size += (RAS_HDR_LEN * (section_count - 1));
-		break;
-	case RAS_CPER_TYPE_BOOT:
-		size += (RAS_BOOT_SEC_LEN * section_count);
-		break;
-	default:
-		/* should never reach here */
-		break;
+	u32 record_size = 0;
+	u32 i;
+	int ret;
+
+	if (batch_logs[0].event == RAS_LOG_EVENT_CPU_RAS) {
+		if (nr_batch_logs > U32_MAX / sizeof(struct ras_cper_processor_record))
+			return -EOVERFLOW;
+		record_size = sizeof(struct ras_cper_processor_record) * nr_batch_logs;
+		if (record_size > buf_len)
+			return -ENOMEM;
+
+		ret = cper_generate_runtime_processor_records(ras_core, buffer,
+				batch_logs, nr_batch_logs);
+		if (ret)
+			return ret;
+		*real_data_len = record_size;
+		return 0;
+	}
+
+	for (i = 0; i < nr_batch_logs; i++) {
+		const struct ras_boot_err_ctx *ctx = &batch_logs[i].body.boot_err_ctx;
+		u16 data_size;
+		u32 size;
+
+		if (batch_logs[i].event != RAS_LOG_EVENT_BOOT ||
+		    !cper_is_processor_record(&batch_logs[i]))
+			return -EINVAL;
+
+		data_size = min_t(u16, ctx->reg_arr_size,
+				  sizeof(struct cper_processor_context) -
+				  offsetof(struct cper_processor_context, reg_dump));
+		size = RAS_HDR_LEN + RAS_SEC_DESC_LEN +
+			offsetof(struct cper_section_processor, context.reg_dump) +
+			data_size;
+		if (record_size > U32_MAX - size)
+			return -EOVERFLOW;
+		record_size += size;
 	}
+	if (record_size > buf_len)
+		return -ENOMEM;
 
-	return size;
+	ret = cper_generate_boot_processor_records(ras_core, buffer,
+			batch_logs, nr_batch_logs);
+	if (ret < 0)
+		return ret;
+	*real_data_len = ret;
+
+	return 0;
+}
+
+static bool cper_is_platform_memory_record(struct ras_log_info *log)
+{
+	return log->event == RAS_LOG_EVENT_BOOT &&
+		!memcmp(log->body.boot_err_ctx.section_type,
+			platform_memory_section_type.b, CPER_UUID_MAX_SIZE);
 }
 
-static enum ras_cper_type cper_ras_log_event_to_cper_type(enum ras_log_event event)
+static int cper_generate_platform_memory_records(struct ras_core_context *ras_core,
+		struct ras_log_info *batch_logs, u32 nr_batch_logs,
+		u8 *buffer, u32 buf_len, u32 *real_data_len)
 {
-	switch (event) {
-	case RAS_LOG_EVENT_UE:
-		return RAS_CPER_TYPE_FATAL;
-	case RAS_LOG_EVENT_DE:
-	case RAS_LOG_EVENT_CE:
-	case RAS_LOG_EVENT_POISON_CREATION:
-	case RAS_LOG_EVENT_POISON_CONSUMPTION:
-		return RAS_CPER_TYPE_RUNTIME;
-	case RAS_LOG_EVENT_RMA:
-		return RAS_CPER_TYPE_RMA;
-	default:
-		/* should never reach here */
-		return RAS_CPER_TYPE_RUNTIME;
+	u32 record_size = 0;
+	u32 offset = 0;
+	u32 i;
+
+	for (i = 0; i < nr_batch_logs; i++) {
+		const struct ras_boot_err_ctx *ctx = &batch_logs[i].body.boot_err_ctx;
+		u32 size;
+
+		if (!cper_is_platform_memory_record(&batch_logs[i]) ||
+		    !ctx->raw_data_size ||
+		    ctx->raw_data_size > sizeof(ctx->raw_data))
+			return -EINVAL;
+
+		size = ALIGN(RAS_HDR_LEN + RAS_SEC_DESC_LEN +
+			     ctx->raw_data_size, sizeof(u32));
+		if (record_size > U32_MAX - size)
+			return -EOVERFLOW;
+		record_size += size;
 	}
+	if (record_size > buf_len)
+		return -ENOMEM;
+
+	for (i = 0; i < nr_batch_logs; i++) {
+		const struct ras_boot_err_ctx *ctx = &batch_logs[i].body.boot_err_ctx;
+		struct cper_section_desc *descriptor;
+		struct cper_section_hdr *hdr;
+		u32 size = ALIGN(RAS_HDR_LEN + RAS_SEC_DESC_LEN +
+				 ctx->raw_data_size, sizeof(u32));
+
+		hdr = (struct cper_section_hdr *)(buffer + offset);
+		descriptor = (struct cper_section_desc *)(buffer + offset +
+						  RAS_HDR_LEN);
+		fill_section_hdr(ras_core, hdr, BOOT__TYPE,
+				 ctx->error_severity, &batch_logs[i]);
+		hdr->record_length = size;
+		hdr->sec_cnt = 1;
+		fill_section_descriptor(ras_core, descriptor, ctx->error_severity,
+				PLATFORM_MEM__SECTION_TYPE,
+				RAS_HDR_LEN + RAS_SEC_DESC_LEN,
+				ctx->raw_data_size);
+		memcpy(buffer + offset + RAS_HDR_LEN + RAS_SEC_DESC_LEN,
+		       ctx->raw_data, ctx->raw_data_size);
+		memset(buffer + offset + RAS_HDR_LEN + RAS_SEC_DESC_LEN +
+		       ctx->raw_data_size, 0,
+		       size - RAS_HDR_LEN - RAS_SEC_DESC_LEN -
+		       ctx->raw_data_size);
+		offset += size;
+	}
+
+	*real_data_len = offset;
+	return 0;
 }
 
-int ras_cper_generate_cper(struct ras_core_context *ras_core,
-		struct ras_log_info *trace_list, uint32_t count,
+static enum ras_log_event cper_mce_parse_err_type(struct ras_core_context *ras_core,
+						  struct aca_bank_reg *bank)
+{
+	struct aca_bank_ecc bank_err = {0};
+
+	if (ras_aca_parse_bank(ras_core, bank, &bank_err))
+		return RAS_LOG_EVENT_NONE;
+
+	if (bank_err.ue_count)
+		return RAS_LOG_EVENT_UE;
+
+	if (bank_err.ce_count)
+		return RAS_LOG_EVENT_CE;
+
+	if (bank_err.de_count)
+		return RAS_LOG_EVENT_DE;
+
+	return RAS_LOG_EVENT_NONE;
+}
+
+int ras_cper_generate_batch_cper(struct ras_core_context *ras_core,
+		struct ras_log_info *batch_logs, uint32_t nr_batch_logs,
 		uint8_t *buf, uint32_t buf_len, uint32_t *real_data_len)
 {
-	uint8_t *buffer = buf;
-	uint64_t buf_size = buf_len;
-	int record_size, saved_size = 0;
-	struct cper_section_hdr *hdr;
+	struct ras_cper *cper = &ras_core->ras_cper;
+	struct ras_cper_profile *profile;
+	enum ras_log_event event;
+	uint32_t record_size;
+	int ret = 0;
+
+	if (!batch_logs || !nr_batch_logs || !buf || !buf_len || !real_data_len)
+		return -EINVAL;
+
+	*real_data_len = 0;
+	if (cper_is_processor_record(&batch_logs[0]))
+		return cper_generate_processor_records(ras_core, batch_logs,
+			nr_batch_logs, buf, buf_len, real_data_len);
+	if (cper_is_platform_memory_record(&batch_logs[0]))
+		return cper_generate_platform_memory_records(ras_core, batch_logs,
+			nr_batch_logs, buf, buf_len, real_data_len);
+
+	event = batch_logs[0].event;
+	if (event == RAS_LOG_EVENT_MCE) {
+		struct aca_bank_reg bank = { 0 };
+
+		/* MCE is encoded as 1 record each */
+		memcpy(&bank.regs, &batch_logs[0].body.aca_reg.regs, sizeof(bank.regs));
+		bank.ecc_type = RAS_ERR_TYPE__MCE;
+		event = cper_mce_parse_err_type(ras_core, &bank);
+		batch_logs[0].event = event;
+	}
+
+	if (event == RAS_LOG_EVENT_BOOT) {
+		uint32_t i;
+
+		for (i = 1; i < nr_batch_logs; i++) {
+			if (batch_logs[i].event != RAS_LOG_EVENT_BOOT ||
+			    memcmp(batch_logs[i].body.boot_err_ctx.section_type,
+				   batch_logs[0].body.boot_err_ctx.section_type,
+				   CPER_UUID_MAX_SIZE))
+				return -EINVAL;
+		}
+	}
+
+	mutex_lock(&cper->profile_mutex);
+	/* All the batch logs share the same event */
+	ret = ras_cper_get_profile(ras_core, event, &profile);
+	if (ret) {
+		RAS_DEV_ERR(ras_core->dev,
+			"Unprocessed ras log event: %d, ret:%d\n", event, ret);
+		goto out;
+	}
+
+	record_size = cper_get_record_size(profile, nr_batch_logs);
+	if (record_size > buf_len) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	memset(buf, 0, record_size);
+
+	if (profile->build_record_per_log)
+		cper_build_multiple_records(ras_core, buf, profile, batch_logs, nr_batch_logs);
+	else
+		cper_build_single_record(ras_core,
+			(struct cper_section_hdr *)buf, profile, batch_logs, nr_batch_logs);
+
+	*real_data_len = record_size;
+
+out:
+	mutex_unlock(&cper->profile_mutex);
+	return ret;
+}
+
+static int ras_cper_get_profile(struct ras_core_context *ras_core,
+		enum ras_log_event event, struct ras_cper_profile **profile)
+{
+	struct ras_cper *cper = &ras_core->ras_cper;
 
-	/* All the batch traces share the same event */
-	record_size = cper_get_record_size(
-			cper_ras_log_event_to_cper_type(trace_list[0].event), count);
+	if (!profile || event < 0 || event >= cper->nr_profiles)
+		return -EINVAL;
 
-	if ((record_size + saved_size) > buf_size)
+	if (!cper->profiles)
+		return -EPERM;
+
+	if (!cper->profiles[event].fill_section)
+		return -ENOENT;
+
+	*profile = &cper->profiles[event];
+
+	return 0;
+}
+
+static struct ras_cper_profile ras_ue_profile = {
+	.cper_type        = RAS_CPER_TYPE_FATAL,
+	.severity         = RAS_CPER_SEV_FATAL_UE,
+	.notify_type      = CPER_NOTIFY__MCE,
+	.sec_type         = GPU__CRASHDUMP,
+	.section_size     = sizeof(struct cper_section_fatal),
+	.build_record_per_log = true,
+	.fill_section     = fill_section_fatal,
+};
+
+static struct ras_cper_profile ras_de_profile = {
+	.cper_type        = RAS_CPER_TYPE_RUNTIME,
+	.severity         = RAS_CPER_SEV_NON_FATAL_UE,
+	.notify_type      = CPER_NOTIFY__MCE,
+	.sec_type         = GPU__NONSTANDARD_ERROR,
+	.section_size     = sizeof(struct cper_section_runtime),
+	.build_record_per_log = false,
+	.fill_section     = fill_section_runtime,
+};
+
+static struct ras_cper_profile ras_ce_profile = {
+	.cper_type        = RAS_CPER_TYPE_RUNTIME,
+	.severity         = RAS_CPER_SEV_NON_FATAL_CE,
+	.notify_type      = CPER_NOTIFY__CMC,
+	.sec_type         = GPU__NONSTANDARD_ERROR,
+	.section_size     = sizeof(struct cper_section_runtime),
+	.build_record_per_log = false,
+	.fill_section     = fill_section_runtime,
+};
+
+static struct ras_cper_profile ras_rma_profile = {
+	.cper_type        = RAS_CPER_TYPE_RUNTIME,
+	.severity         = RAS_CPER_SEV_RMA,
+	.notify_type      = CPER_NOTIFY__MCE,
+	.sec_type         = GPU__NONSTANDARD_ERROR,
+	.section_size     = sizeof(struct cper_section_runtime),
+	.build_record_per_log = false,
+	.fill_section     = fill_section_runtime,
+};
+
+static struct ras_cper_profile ras_boot_profile = {
+	.cper_type        = RAS_CPER_TYPE_BOOT,
+	.severity         = RAS_CPER_SEV_FATAL_UE,
+	.notify_type      = BOOT__TYPE,
+	.sec_type         = GPU__CRASHDUMP,
+	.section_size     = sizeof(struct cper_section_boot),
+	.build_record_per_log = true,
+	.get_severity     = cper_boot_get_severity,
+	.get_sec_type     = cper_boot_get_sec_type,
+	.fill_section     = fill_section_boot,
+};
+
+static struct ras_event_profile_map ras_event_profile_maps[] = {
+	{RAS_LOG_EVENT_UE,   &ras_ue_profile},
+	{RAS_LOG_EVENT_DE,   &ras_de_profile},
+	{RAS_LOG_EVENT_CE,   &ras_ce_profile},
+	{RAS_LOG_EVENT_RMA,  &ras_rma_profile},
+	{RAS_LOG_EVENT_BOOT, &ras_boot_profile},
+};
+
+int ras_cper_sw_init(struct ras_core_context *ras_core)
+{
+	struct ras_cper *cper = &ras_core->ras_cper;
+	struct ras_cper_profile *profiles;
+	uint32_t nr_profiles = RAS_LOG_EVENT_COUNT_MAX;
+	int i, ret = 0;
+
+	profiles = kcalloc(nr_profiles, sizeof(*profiles), GFP_KERNEL);
+	if (!profiles)
 		return -ENOMEM;
 
-	hdr = (struct cper_section_hdr *)(buffer + saved_size);
-
-	switch (trace_list[0].event) {
-	case RAS_LOG_EVENT_RMA:
-		cper_generate_runtime_record(ras_core, hdr, trace_list, count, RAS_CPER_SEV_RMA);
-		break;
-	case RAS_LOG_EVENT_DE:
-		cper_generate_runtime_record(ras_core,
-			hdr, trace_list, count, RAS_CPER_SEV_NON_FATAL_UE);
-		break;
-	case RAS_LOG_EVENT_CE:
-		cper_generate_runtime_record(ras_core,
-			hdr, trace_list, count, RAS_CPER_SEV_NON_FATAL_CE);
-		break;
-	case RAS_LOG_EVENT_UE:
-		cper_generate_fatal_record(ras_core, buffer + saved_size, trace_list, count);
-		break;
-	default:
-		RAS_DEV_WARN(ras_core->dev, "Unprocessed trace event: %d\n", trace_list[0].event);
-		break;
+	cper->profiles = profiles;
+	cper->nr_profiles = nr_profiles;
+	mutex_init(&cper->profile_mutex);
+
+	for (i = 0; i < ARRAY_SIZE(ras_event_profile_maps); i++) {
+		ret = ras_cper_register_profile(ras_core,
+				ras_event_profile_maps[i].event,
+				ras_event_profile_maps[i].profile);
+		if (ret) {
+			RAS_DEV_ERR(ras_core->dev,
+				"Failed to register %u profile. ret:%d\n",
+				ras_event_profile_maps[i].event, ret);
+			goto out;
+		}
 	}
 
-	saved_size += record_size;
+	return 0;
+
+out:
+	ras_cper_sw_fini(ras_core);
+	return ret;
+}
+
+int ras_cper_sw_fini(struct ras_core_context *ras_core)
+{
+	struct ras_cper *cper = &ras_core->ras_cper;
+
+	if (!cper->profiles)
+		return 0;
+
+	kfree(cper->profiles);
+	cper->profiles = NULL;
+	cper->nr_profiles = 0;
+	mutex_destroy(&cper->profile_mutex);
 
-	*real_data_len = saved_size;
 	return 0;
 }
+
+int ras_cper_register_profile(struct ras_core_context *ras_core,
+	enum ras_log_event event, struct ras_cper_profile *profile)
+{
+	struct ras_cper *cper = &ras_core->ras_cper;
+	int ret = 0;
+
+	if (!profile || event >= cper->nr_profiles)
+		return -EINVAL;
+
+	if (!cper->profiles)
+		return -EPERM;
+
+	mutex_lock(&cper->profile_mutex);
+	if (cper->profiles[event].fill_section) {
+		ret = -EEXIST;
+		goto out;
+	}
+
+	memcpy(&cper->profiles[event], profile, sizeof(struct ras_cper_profile));
+
+out:
+	mutex_unlock(&cper->profile_mutex);
+	return ret;
+}
+
+int ras_cper_unregister_profile(struct ras_core_context *ras_core,
+	enum ras_log_event event)
+{
+	struct ras_cper *cper = &ras_core->ras_cper;
+	int ret = 0;
+
+	if (event >= cper->nr_profiles)
+		return -EINVAL;
+
+	if (!cper->profiles)
+		return -EPERM;
+
+	mutex_lock(&cper->profile_mutex);
+	if (!cper->profiles[event].fill_section) {
+		ret = 0;
+		goto out;
+	}
+
+	memset(&cper->profiles[event], 0, sizeof(struct ras_cper_profile));
+
+out:
+	mutex_unlock(&cper->profile_mutex);
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/ras/core/ras_cper.h b/drivers/gpu/drm/amd/ras/core/ras_cper.h
index e4e3615ecc2e02..58d2955ec08dc3 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_cper.h
+++ b/drivers/gpu/drm/amd/ras/core/ras_cper.h
@@ -23,6 +23,7 @@
  */
 #ifndef __RAS_CPER_H__
 #define __RAS_CPER_H__
+#include "log_ring.h"
 
 #define CPER_UUID_MAX_SIZE 16
 struct ras_cper_guid {
@@ -30,15 +31,17 @@ struct ras_cper_guid {
 };
 
 #define CPER_GUID__INIT(a, b, c, d0, d1, d2, d3, d4, d5, d6, d7)			\
-	((struct ras_cper_guid)								\
+	(struct ras_cper_guid)								\
 	{{ (a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff, \
 		(b) & 0xff, ((b) >> 8) & 0xff,					\
 		(c) & 0xff, ((c) >> 8) & 0xff,					\
-		(d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }})
+		(d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }}
 
 #define CPER_HDR__REV_1          (0x100)
+#define CPER_HDR__REV_AMD_CPU    (0x101)
 #define CPER_SEC__MINOR_REV_1    (0x01)
 #define CPER_SEC__MAJOR_REV_22   (0x22)
+#define CPER_SEC__REV_AMD_CPU    (0x0100)
 #define CPER_OAM_MAX_COUNT      (8)
 
 #define CPER_CTX_TYPE__CRASH     (1)
@@ -46,6 +49,7 @@ struct ras_cper_guid {
 
 #define CPER_CREATOR_ID__AMDGPU	"amdgpu"
 
+/* Cper Notification Type */
 #define CPER_NOTIFY__MCE                                               \
 	CPER_GUID__INIT(0xE8F56FFE, 0x919C, 0x4cc5, 0xBA, 0x88, 0x65, 0xAB, \
 		  0xE1, 0x49, 0x13, 0xBB)
@@ -56,6 +60,7 @@ struct ras_cper_guid {
 	CPER_GUID__INIT(0x3D61A466, 0xAB40, 0x409a, 0xA6, 0x98, 0xF3, 0x62, \
 		  0xD4, 0x64, 0xB3, 0x8F)
 
+/* Cper Section Type */
 #define GPU__CRASHDUMP                                                 \
 	CPER_GUID__INIT(0x32AC0C78, 0x2623, 0x48F6, 0xB0, 0xD0, 0x73, 0x65, \
 		  0x72, 0x5F, 0xD6, 0xAE)
@@ -65,6 +70,43 @@ struct ras_cper_guid {
 #define PROC_ERR__SECTION_TYPE                                         \
 	CPER_GUID__INIT(0xDC3EA0B0, 0xA144, 0x4797, 0xB9, 0x5B, 0x53, 0xFA, \
 		  0x24, 0x2B, 0x6E, 0x1D)
+#define PROC_ERR__MS_CHECK_TYPE                                        \
+	CPER_GUID__INIT(0x48AB7F57, 0xDC34, 0x4F6C, 0xA7, 0xD3, 0xB0, 0xB5, \
+		  0xB0, 0xA7, 0x43, 0x14)
+#define PLATFORM_MEM__SECTION_TYPE                                     \
+	CPER_GUID__INIT(0xA5BC1114, 0x6F64, 0x4EDE, 0xB8, 0x63, 0x3E, 0x83, \
+		  0xED, 0x7C, 0x83, 0xB1)
+#define PCIE_ERR__SECTION_TYPE                                         \
+	CPER_GUID__INIT(0xD995E954, 0xBBC1, 0x430F, 0xAD, 0x91, 0xB4, 0x4D, \
+		  0xCB, 0x3C, 0x6F, 0x35)
+#define SMN_ERR__SECTION_TYPE                                          \
+	CPER_GUID__INIT(0xA2860CC1, 0x8987, 0x4B7C, 0xB8, 0x6A, 0xD5, 0x08, \
+		  0xB1, 0x76, 0xBA, 0x70)
+
+#define CPER_PROC_VALID_APIC_ID		BIT_ULL(0)
+#define CPER_PROC_ERR_INFO_COUNT(x)	(((u64)(x) & 0x3f) << 2)
+#define CPER_PROC_CONTEXT_COUNT(x)	(((u64)(x) & 0x3f) << 8)
+#define CPER_AMD_CONTEXT_COUNT(x)	(((u64)(x) & 0x3f) << 8)
+#define CPER_AMD_ERR_INFO_COUNT(x)	(((u64)(x) & 0x3f) << 2)
+
+#define CPER_PROC_INFO_VALID_CHECK_INFO	BIT_ULL(0)
+
+#define CPER_MS_CHECK_VALID_ERR_TYPE	BIT_ULL(0)
+#define CPER_MS_CHECK_VALID_PCC		BIT_ULL(1)
+#define CPER_MS_CHECK_VALID_UNCORRECTED	BIT_ULL(2)
+#define CPER_MS_CHECK_VALID_OVERFLOW	BIT_ULL(5)
+#define CPER_MS_CHECK_ERR_TYPE_INTERNAL	(5ULL << 16)
+#define CPER_MS_CHECK_PCC		BIT_ULL(19)
+#define CPER_MS_CHECK_UNCORRECTED	BIT_ULL(20)
+#define CPER_MS_CHECK_OVERFLOW		BIT_ULL(23)
+
+#define CPER_MCA_STATUS_OVERFLOW	BIT_ULL(62)
+#define CPER_MCA_STATUS_UNCORRECTED	BIT_ULL(61)
+#define CPER_MCA_STATUS_PCC		BIT_ULL(57)
+#define CPER_MCA_STATUS_DEFERRED	BIT_ULL(44)
+
+#define CPER_SMCA_MC0_STATUS_MSR	(0xc0002001U)
+#define CPER_SMCA_BANK_STRIDE		(0x10U)
 
 enum ras_cper_type {
 	RAS_CPER_TYPE_RUNTIME,
@@ -73,6 +115,7 @@ enum ras_cper_type {
 	RAS_CPER_TYPE_RMA,
 };
 
+/* Cper Error Severity */
 enum ras_cper_severity {
 	RAS_CPER_SEV_NON_FATAL_UE   = 0,
 	RAS_CPER_SEV_FATAL_UE       = 1,
@@ -137,7 +180,7 @@ struct cper_section_hdr {
 	uint8_t			reserved[12];     /* Reserved */
 };
 
-struct cper_section_descriptor {
+struct cper_section_desc {
 	uint32_t sec_offset;     /* Offset from the start of CPER entry */
 	uint32_t sec_length;
 	uint8_t  revision_minor; /* CPER_SEC_MINOR_REV_1 */
@@ -234,10 +277,10 @@ struct cper_section_runtime {
 };
 
 struct crashdump_hdr {
-	uint64_t reserved1;
-	uint64_t reserved2;
+	u64 valid_bits;
+	u32 pcie_device_id;
+	u32 pldm_bundle;
 	char     fw_id[48];
-	uint64_t reserved3[8];
 };
 
 struct fatal_reg_info {
@@ -263,29 +306,121 @@ struct crashdump_boot {
 	uint64_t msg[CPER_OAM_MAX_COUNT];
 };
 
+struct crashdump_error_info {
+	struct ras_cper_guid error_type;
+	u64 valid_bits;
+	u64 check_info;
+	u8 fru_part_number[16];
+	u8 redfish_event_id[16];
+};
+
 struct cper_section_fatal {
-	struct crashdump_hdr    hdr;
-	struct crashdump_fatal  data;
+	struct crashdump_hdr hdr;
+	struct crashdump_error_info error_info;
+	struct crashdump_fatal data;
 };
 
 struct cper_section_boot {
-	struct crashdump_hdr  hdr;
+	struct crashdump_hdr hdr;
+	struct crashdump_error_info error_info;
 	struct crashdump_boot data;
 };
 
+struct cper_processor_section {
+	u64 valid_bits;
+	u64 apic_id;
+	u8 cpuid[48];
+};
+
+struct cper_processor_error_info {
+	struct ras_cper_guid error_type;
+	u64 valid_bits;
+	u64 check_info;
+	u64 target_id;
+	u64 requester_id;
+	u64 responder_id;
+	u64 instruction_pointer;
+};
+
+struct cper_processor_context {
+	u16 reg_ctx_type;
+	u16 reg_arr_size;
+	u32 msr_addr;
+	u64 mm_reg_addr;
+	u64 reg_dump[RAS_CPER_ACA_REG_COUNT];
+};
+
+struct cper_section_processor {
+	struct cper_processor_section processor;
+	struct cper_processor_error_info error_info;
+	struct cper_processor_context context;
+};
+
 struct ras_cper_fatal_record {
 	struct cper_section_hdr hdr;
-	struct cper_section_descriptor descriptor;
+	struct cper_section_desc descriptor;
 	struct cper_section_fatal fatal;
 };
+
+struct ras_cper_boot_record {
+	struct cper_section_hdr hdr;
+	struct cper_section_desc descriptor;
+	struct cper_section_boot boot;
+};
+
+struct ras_cper_processor_record {
+	struct cper_section_hdr hdr;
+	struct cper_section_desc descriptor;
+	struct cper_section_processor processor;
+};
+
 #pragma pack(pop)
 
+struct ras_core_context;
+struct ras_log_info;
+enum ras_log_event;
+/*
+ * Per-event description of how to build its CPER record(s).
+ *
+ * Fixed fields (severity/notify_type/sec_type) are used as-is unless the matching
+ * optional hook (get_severity/get_notify_type/get_sec_type) is provided.
+ */
+struct ras_cper_profile {
+	enum ras_cper_type     cper_type;
+	enum ras_cper_severity severity;
+	struct ras_cper_guid   notify_type;
+	struct ras_cper_guid   sec_type;
+	uint32_t section_size;
+	/* true: one CPER record per log */
+	bool build_record_per_log;
+
+	int (*get_severity)(struct ras_log_info *log, enum ras_cper_severity *out);
+	int (*get_notify_type)(struct ras_log_info *log, struct ras_cper_guid *out);
+	int (*get_sec_type)(struct ras_log_info *log, struct ras_cper_guid *out);
+
+	/* Required: fill the error-type specific section body for one log */
+	int (*fill_section)(struct ras_core_context *ras_core, void *section,
+			    struct ras_log_info *log, enum ras_cper_severity sev);
+};
+
+struct ras_event_profile_map {
+	uint32_t event;
+	struct ras_cper_profile *profile;
+};
+
+struct ras_cper {
+	struct ras_cper_profile *profiles;
+	uint32_t nr_profiles;
+	struct mutex profile_mutex;
+};
+
 #define RAS_HDR_LEN				(sizeof(struct cper_section_hdr))
-#define RAS_SEC_DESC_LEN			(sizeof(struct cper_sec_desc))
+#define RAS_SEC_DESC_LEN			(sizeof(struct cper_section_desc))
 
-#define RAS_BOOT_SEC_LEN			(sizeof(struct cper_sec_crashdump_boot))
-#define RAS_FATAL_SEC_LEN			(sizeof(struct cper_sec_crashdump_fatal))
-#define RAS_NONSTD_SEC_LEN			(sizeof(struct cper_sec_nonstd_err))
+#define RAS_BOOT_SEC_LEN			(sizeof(struct cper_section_boot))
+#define RAS_FATAL_SEC_LEN			(sizeof(struct cper_section_fatal))
+#define RAS_NONSTD_SEC_LEN			(sizeof(struct cper_section_runtime))
+#define RAS_PROC_SEC_LEN			(sizeof(struct cper_section_processor))
 
 #define RAS_SEC_DESC_OFFSET(idx)		(RAS_HDR_LEN + (RAS_SEC_DESC_LEN * idx))
 
@@ -296,9 +431,13 @@ struct ras_cper_fatal_record {
 #define RAS_NONSTD_SEC_OFFSET(count, idx) \
 	(RAS_HDR_LEN + (RAS_SEC_DESC_LEN * count) + (RAS_NONSTD_SEC_LEN * idx))
 
-struct ras_core_context;
-struct ras_log_info;
-int ras_cper_generate_cper(struct ras_core_context *ras_core,
-		struct ras_log_info *trace_list, uint32_t count,
+int ras_cper_sw_init(struct ras_core_context *ras_core);
+int ras_cper_sw_fini(struct ras_core_context *ras_core);
+int ras_cper_generate_batch_cper(struct ras_core_context *ras_core,
+		struct ras_log_info *batch_logs, uint32_t nr_batch_logs,
 		uint8_t *buf, uint32_t buf_len, uint32_t *real_data_len);
+int ras_cper_register_profile(struct ras_core_context *ras_core,
+	enum ras_log_event event, struct ras_cper_profile *profile);
+int ras_cper_unregister_profile(struct ras_core_context *ras_core,
+	enum ras_log_event event);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c b/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c
new file mode 100644
index 00000000000000..1781d037ef13de
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.c
@@ -0,0 +1,418 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "ras_eeprom_mgr.h"
+#include "ras.h"
+#include "eeprom.h"
+#include "eeprom_fw.h"
+
+static bool __ras_eeprom_disabled(struct ras_core_context *ras_core)
+{
+	return !ras_core->ras_eeprom_supported;
+}
+
+static const struct ras_eeprom_ops *ras_eeprom_mgr_get_ip_func(
+				struct ras_core_context *ras_core, uint32_t ip_version)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+
+	switch (ip_version) {
+	case IP_VERSION(13, 0, 6):
+	case IP_VERSION(13, 0, 14):
+	case IP_VERSION(13, 0, 12):
+		mgr->eeprom_early_init_service_supported = false;
+		return &ras_drv_eeprom_ops;
+	case IP_VERSION(15, 0, 8):
+		mgr->eeprom_early_init_service_supported = true;
+		mgr->fw_record_enabled = true;
+		return &ras_fw_eeprom_ops;
+	default:
+		RAS_DEV_ERR(ras_core->dev,
+			"EEPROM(MP1) ip version(0x%x) is not supported!\n", ip_version);
+		break;
+	}
+
+	return NULL;
+}
+
+int ras_eeprom_mgr_sw_init(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	struct ras_eeprom_config *cfg;
+	struct ras_eeprom_param_config  param_cfg = {0};
+	struct ras_eeprom_param eeprom_param = {0};
+	int ret = 0;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	cfg = &ras_core->config->eeprom_cfg;
+	if (!cfg || !cfg->eeprom_sys_fn || !cfg->eeprom_sys_fn->get_eeprom_config) {
+		RAS_DEV_ERR(ras_core->dev, "Ras eeprom not configured!\n");
+		return -EINVAL;
+	}
+
+	memset(mgr, 0, sizeof(*mgr));
+
+	mgr->sys_func = cfg->eeprom_sys_fn;
+	ret = mgr->sys_func->get_eeprom_config(ras_core, &param_cfg);
+	if (ret) {
+		RAS_DEV_ERR(ras_core->dev, "Failed to get ras eeprom param config!\n");
+		return -EPERM;
+	}
+
+	mgr->ras_err_threshold = param_cfg.eeprom_record_threshold_count;
+	mgr->work_mode_over_thresh = param_cfg.work_mode_over_thresh;
+	mgr->eeprom_ip_version = param_cfg.eeprom_ip_version;
+	mgr->eeprom_ops = ras_eeprom_mgr_get_ip_func(ras_core, mgr->eeprom_ip_version);
+	if (!mgr->eeprom_ops)
+		return -ENODEV;
+
+	eeprom_param.sys_fn = mgr->sys_func;
+	eeprom_param.eeprom_ip_version = param_cfg.eeprom_ip_version;
+	eeprom_param.record_threshold = param_cfg.eeprom_record_threshold_count;
+	eeprom_param.max_i2c_read_len = param_cfg.max_i2c_read_len;
+	eeprom_param.max_i2c_write_len = param_cfg.max_i2c_write_len;
+	eeprom_param.eeprom_i2c_adapter = param_cfg.eeprom_i2c_adapter;
+	eeprom_param.eeprom_i2c_port = param_cfg.eeprom_i2c_port;
+	eeprom_param.eeprom_i2c_addr = param_cfg.eeprom_i2c_addr;
+
+	if (mgr->eeprom_ops->sw_init)
+		ret = mgr->eeprom_ops->sw_init(ras_core, &eeprom_param);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_sw_fini(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	int ret = 0;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	if (!mgr->eeprom_ops)
+		return -EINVAL;
+
+	if (mgr->eeprom_ops->sw_fini)
+		ret = mgr->eeprom_ops->sw_fini(ras_core);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_hw_init(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	struct ras_eeprom_param_config  param_cfg = {0};
+	struct ras_eeprom_param eeprom_param = {0};
+	int ret = 0;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	if (!mgr->eeprom_ops)
+		return -EINVAL;
+
+	ret = mgr->sys_func->get_eeprom_config(ras_core, &param_cfg);
+	if (ret) {
+		RAS_DEV_ERR(ras_core->dev, "Failed to get ras eeprom param config!\n");
+		return -EPERM;
+	}
+
+	eeprom_param.max_i2c_read_len = param_cfg.max_i2c_read_len;
+	eeprom_param.max_i2c_write_len = param_cfg.max_i2c_write_len;
+	eeprom_param.eeprom_i2c_adapter = param_cfg.eeprom_i2c_adapter;
+	eeprom_param.eeprom_i2c_port = param_cfg.eeprom_i2c_port;
+	eeprom_param.eeprom_i2c_addr = param_cfg.eeprom_i2c_addr;
+
+	if (mgr->eeprom_ops->hw_init)
+		ret = mgr->eeprom_ops->hw_init(ras_core, &eeprom_param);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_hw_fini(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	int ret = 0;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	if (!mgr->eeprom_ops)
+		return -EINVAL;
+
+	if (mgr->eeprom_ops->hw_fini)
+		ret = mgr->eeprom_ops->hw_fini(ras_core);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_reset_table(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	int ret = -EOPNOTSUPP;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	if (!mgr->eeprom_ops)
+		return -EINVAL;
+
+	if (mgr->eeprom_ops->reset_table)
+		ret = mgr->eeprom_ops->reset_table(ras_core);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_get_record_count(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	int ret = 0;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	if (!mgr->eeprom_ops)
+		return -EINVAL;
+
+	if (mgr->eeprom_ops && mgr->eeprom_ops->get_record_count)
+		ret = mgr->eeprom_ops->get_record_count(ras_core);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_get_records(struct ras_core_context *ras_core, u32 start,
+		struct eeprom_umc_record *record, u32 num)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	int ret = -EOPNOTSUPP;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return -RAS_CORE_NOT_SUPPORTED;
+
+	if (!mgr->eeprom_ops)
+		return -EINVAL;
+
+	if (mgr->eeprom_ops->get_records)
+		ret = mgr->eeprom_ops->get_records(ras_core,
+					start, record, num);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_append_records(struct ras_core_context *ras_core,
+			struct eeprom_umc_record *record, const u32 num)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	int ret = -EOPNOTSUPP;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return -RAS_CORE_NOT_SUPPORTED;
+
+	if (!mgr->eeprom_ops)
+		return -EINVAL;
+
+	if (mgr->eeprom_ops->append_records)
+		ret = mgr->eeprom_ops->append_records(ras_core,
+					record, num);
+
+	return ret;
+}
+
+int ras_eeprom_mgr_get_eeprom_info(struct ras_core_context *ras_core,
+			struct ras_eeprom_info *eeprom_info)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return -EOPNOTSUPP;
+
+	if (!mgr->eeprom_ops || !mgr->eeprom_ops->get_eeprom_info)
+		return -EOPNOTSUPP;
+
+	return mgr->eeprom_ops->get_eeprom_info(ras_core,
+				eeprom_info, ras_core_gpu_in_reset(ras_core));
+}
+
+int ras_eeprom_mgr_check_and_report_status(struct ras_core_context *ras_core, bool chk_hw)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	struct ras_eeprom_info info = {0};
+	int valid_err_count;
+	int ret = 0;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	if (!mgr->eeprom_ops || !mgr->eeprom_ops->get_eeprom_info)
+		return -EOPNOTSUPP;
+
+	ret = mgr->eeprom_ops->get_eeprom_info(ras_core, &info, !chk_hw);
+	if (ret)
+		return ret;
+
+	valid_err_count = ras_umc_get_saved_eeprom_count(ras_core);
+	if (info.eeprom_status == RAS_EEPROM_OK) {
+		RAS_DEV_DBG(ras_core->dev,
+			"Found existing EEPROM table with %d records\n",
+			valid_err_count);
+
+		/* Warn if error count reaches or exceeds 90% of the threshold */
+		if (10 * valid_err_count >= 9 * mgr->ras_err_threshold)
+			RAS_DEV_WARN(ras_core->dev,
+				"RAS records:%u exceeds 90%% of threshold:%d\n",
+				valid_err_count, mgr->ras_err_threshold);
+
+	} else if (info.eeprom_status == RAS_EEPROM_LOCKED) {
+		if (info.rma_status || (valid_err_count >= mgr->ras_err_threshold)) {
+			if (info.rma_status)
+				RAS_DEV_ERR(ras_core->dev, "RAS records:%d exceed error threshold",
+					valid_err_count);
+			else
+				RAS_DEV_ERR(ras_core->dev, "RAS records:%d exceed threshold:%d",
+					valid_err_count, mgr->ras_err_threshold);
+
+			if (mgr->work_mode_over_thresh == RAS_WORK_MODE_OVER_THRESH_STRICT) {
+				RAS_DEV_WARN(ras_core->dev,
+				"Please consult AMD Service Action Guide (SAG) for appropriate service procedures\n");
+			} else if (mgr->work_mode_over_thresh == RAS_WORK_MODE_OVER_THRESH_RMA) {
+				ras_core->is_rma = true;
+				RAS_DEV_ERR(ras_core->dev,
+				"User defined threshold is set, runtime service will be halt when threshold is reached\n");
+			}
+		} else if ((valid_err_count < mgr->ras_err_threshold) &&
+				mgr->eeprom_ops->unlock_eeprom) {
+			/* This means that, the threshold was increased since
+			 * the last time the system was booted, and now,
+			 * the saved error count is less than threshold,
+			 * so that at least one more record can be saved,
+			 * before the page count threshold is reached.
+			 */
+			ret = mgr->eeprom_ops->unlock_eeprom(ras_core);
+			if (ret) {
+				RAS_DEV_ERR(ras_core->dev, "Failed to resize RAS table\n");
+			} else {
+				ras_core->is_rma = false;
+				RAS_DEV_INFO(ras_core->dev,
+					"records:%d threshold:%d, resetting RAS table header signature",
+					valid_err_count, mgr->ras_err_threshold);
+			}
+		} else {
+			RAS_DEV_ERR(ras_core->dev,
+				"records:%d threshold:%d, not support to resize RAS table\n",
+				valid_err_count, mgr->ras_err_threshold);
+		}
+	} else {
+		RAS_DEV_ERR(ras_core->dev, "Error ras eeprom status:0x%x\n", info.eeprom_status);
+	}
+
+	return ret;
+}
+
+enum ras_gpu_op_status
+	ras_eeprom_mgr_get_gpu_op_status(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	struct ras_eeprom_info info = {0};
+	int ret;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return RAS_GPU_OP_STATUS_OK;
+
+	if (!mgr->eeprom_ops || !mgr->eeprom_ops->get_eeprom_info)
+		return RAS_GPU_OP_STATUS_UNKNOWN;
+
+	ret = mgr->eeprom_ops->get_eeprom_info(ras_core,
+				&info, ras_core_gpu_in_reset(ras_core));
+	if (ret)
+		return RAS_GPU_OP_STATUS_UNKNOWN;
+
+	if (info.rma_status || ras_core->is_rma)
+		return RAS_GPU_OP_STATUS_RMA;
+
+	if (info.eeprom_status == RAS_EEPROM_FAULT)
+		return RAS_GPU_OP_STATUS_FAULT;
+	else if (info.eeprom_status == RAS_EEPROM_LOCKED)
+		return RAS_GPU_OP_STATUS_LOCKED;
+	else if (info.eeprom_status == RAS_EEPROM_UNKNOWN)
+		return RAS_GPU_OP_STATUS_UNKNOWN;
+
+	return RAS_GPU_OP_STATUS_OK;
+}
+
+bool ras_eeprom_mgr_check_safety_watermark(struct ras_core_context *ras_core)
+{
+	enum ras_gpu_op_status status;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return true;
+
+	ras_eeprom_mgr_check_and_report_status(ras_core, false);
+
+	status = ras_eeprom_mgr_get_gpu_op_status(ras_core);
+
+	return (status == RAS_GPU_OP_STATUS_OK);
+}
+
+int ras_eeprom_mgr_get_version(struct ras_core_context *ras_core, u32 *version)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+	struct ras_eeprom_info info = {0};
+	int ret = 0;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return 0;
+
+	if (!mgr->eeprom_ops || !mgr->eeprom_ops->get_eeprom_info)
+		return -EOPNOTSUPP;
+
+	ret = mgr->eeprom_ops->get_eeprom_info(ras_core, &info, true);
+	if (ret)
+		return ret;
+
+	*version = info.record_format_version;
+
+	return 0;
+}
+
+bool ras_eeprom_mgr_early_init_service_supported(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return false;
+
+	return mgr->eeprom_early_init_service_supported;
+}
+
+bool ras_eeprom_mgr_fw_record_enabled(struct ras_core_context *ras_core)
+{
+	struct ras_eeprom_mgr *mgr = &ras_core->eeprom_mgr;
+
+	if (__ras_eeprom_disabled(ras_core))
+		return false;
+
+	return mgr->fw_record_enabled;
+}
diff --git a/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.h b/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.h
new file mode 100644
index 00000000000000..b15682b44a6efd
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_eeprom_mgr.h
@@ -0,0 +1,124 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __RAS_EEPROM_MGR_H__
+#define __RAS_EEPROM_MGR_H__
+#include "ras_sys.h"
+
+enum ras_gpu_op_status {
+	RAS_GPU_OP_STATUS_UNKNOWN = 0,
+	RAS_GPU_OP_STATUS_OK      = 1,
+	RAS_GPU_OP_STATUS_LOCKED  = 2,
+	RAS_GPU_OP_STATUS_FAULT   = 3,
+	RAS_GPU_OP_STATUS_RMA     = 4,
+};
+
+enum ras_eeprom_status {
+	RAS_EEPROM_UNKNOWN,
+	/*Read and write */
+	RAS_EEPROM_OK,
+	/* Only read */
+	RAS_EEPROM_LOCKED,
+	/* Invalid data */
+	RAS_EEPROM_FAULT,
+};
+
+struct ras_eeprom_param {
+	u32 eeprom_ip_version;
+	uint32_t record_threshold;
+	void *eeprom_i2c_adapter;
+	u32 eeprom_i2c_addr;
+	u32 eeprom_i2c_port;
+	u16 max_i2c_read_len;
+	u16 max_i2c_write_len;
+	const struct ras_eeprom_sys_func *sys_fn;
+};
+
+struct ras_eeprom_info {
+	u32 record_format_version;
+	u32 eeprom_status;
+	u32 rma_status;
+	/* Raw record count in eeprom, but data may be duplicated,
+	 * therefore not a valid record count
+	 */
+	u32 record_count;
+	u32 max_record_count;
+	u64 bad_channel_bitmap;
+};
+
+struct ras_core_context;
+struct eeprom_umc_record;
+
+struct ras_eeprom_ops {
+	int (*sw_init)(struct ras_core_context *ras_core, struct ras_eeprom_param *param);
+	int (*sw_fini)(struct ras_core_context *ras_core);
+	int (*hw_init)(struct ras_core_context *ras_core, struct ras_eeprom_param *param);
+	int (*hw_fini)(struct ras_core_context *ras_core);
+	int (*reset_table)(struct ras_core_context *ras_core);
+	int (*get_records)(struct ras_core_context *ras_core, u32 start,
+		struct eeprom_umc_record *records, u32 num);
+	int (*append_records)(struct ras_core_context *ras_core,
+		struct eeprom_umc_record *records, u32 num);
+	int (*get_record_count)(struct ras_core_context *ras_core);
+	int (*get_eeprom_info)(struct ras_core_context *ras_core,
+		struct ras_eeprom_info *eeprom_info, bool fast_mode);
+	int (*unlock_eeprom)(struct ras_core_context *ras_core);
+};
+
+struct ras_eeprom_mgr {
+	u32 eeprom_ip_version;
+	const struct ras_eeprom_sys_func *sys_func;
+	void *ras_eeprom;
+	const struct ras_eeprom_ops *eeprom_ops;
+	u32 ras_err_threshold;
+	u32 work_mode_over_thresh;
+	bool eeprom_early_init_service_supported;
+	bool fw_record_enabled;
+};
+
+struct ras_core_context;
+int ras_eeprom_mgr_sw_init(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_sw_fini(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_hw_init(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_hw_fini(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_reset_table(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_get_record_count(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_get_records(struct ras_core_context *ras_core, u32 start,
+		struct eeprom_umc_record *record, const u32 num);
+int ras_eeprom_mgr_append_records(struct ras_core_context *ras_core,
+		struct eeprom_umc_record *record, const u32 num);
+int ras_eeprom_mgr_check_and_report_status(struct ras_core_context *ras_core,
+		bool check_hw);
+enum ras_gpu_op_status
+	ras_eeprom_mgr_get_gpu_op_status(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_get_version(struct ras_core_context *ras_core, u32 *version);
+bool ras_eeprom_mgr_check_safety_watermark(struct ras_core_context *ras_core);
+int ras_eeprom_mgr_get_eeprom_info(struct ras_core_context *ras_core,
+			struct ras_eeprom_info *eeprom_info);
+int ras_core_enable_early_init_service(struct ras_core_context *ras_core,
+		bool enabled);
+bool ras_eeprom_mgr_early_init_service_supported(struct ras_core_context *ras_core);
+bool ras_eeprom_mgr_fw_record_enabled(struct ras_core_context *ras_core);
+int  ras_eeprom_mgr_dump_fw_records(struct ras_core_context *ras_core);
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_gfx.c b/drivers/gpu/drm/amd/ras/core/ras_gfx.c
index 2efe18cbbe5b5b..e553f1364d6c6b 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_gfx.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_gfx.c
@@ -35,6 +35,9 @@ static const struct ras_gfx_ip_func *ras_gfx_get_ip_funcs(
 	case IP_VERSION(9, 4, 4):
 	case IP_VERSION(9, 5, 0):
 		return &gfx_ras_func_v9_0;
+	case IP_VERSION(12, 1, 0):
+		/* TBD */
+		break;
 	default:
 		RAS_DEV_ERR(ras_core->dev,
 			"GFX ip version(0x%x) is not supported!\n", ip_version);
@@ -49,8 +52,11 @@ int ras_gfx_get_ta_subblock(struct ras_core_context *ras_core,
 {
 	struct ras_gfx *gfx = &ras_core->ras_gfx;
 
-	return gfx->ip_func->get_ta_subblock(ras_core,
+	if (gfx->ip_func && gfx->ip_func->get_ta_subblock)
+		return gfx->ip_func->get_ta_subblock(ras_core,
 					error_type, subblock, ta_subblock);
+
+	return -RAS_CORE_NOT_SUPPORTED;
 }
 
 int ras_gfx_hw_init(struct ras_core_context *ras_core)
@@ -61,7 +67,7 @@ int ras_gfx_hw_init(struct ras_core_context *ras_core)
 
 	gfx->ip_func = ras_gfx_get_ip_funcs(ras_core, gfx->gfx_ip_version);
 
-	return gfx->ip_func ? RAS_CORE_OK : -EINVAL;
+	return 0;
 }
 
 int ras_gfx_hw_fini(struct ras_core_context *ras_core)
diff --git a/drivers/gpu/drm/amd/ras/core/ras_mce.c b/drivers/gpu/drm/amd/ras/core/ras_mce.c
new file mode 100644
index 00000000000000..71281917b132a8
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_mce.c
@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "ras.h"
+#include "ras_mce.h"
+#define RAS_MCE_BANK_FIFO_SIZE  (1024 * sizeof(struct aca_bank_reg))
+#define RAS_MCE_BANK_COUNT(x) ((x) / sizeof(struct aca_bank_reg))
+
+struct mce_smca_bank_range {
+	u32 start;
+	u32 end;
+};
+
+static const struct mce_smca_bank_range mce_smca_bank_range_v5[] = {
+	[MCE_BANK_TYPE_GPU] = {32, 63},
+	[MCE_BANK_TYPE_CPU] = {0, 31},
+};
+
+bool ras_mce_check_bank(struct ras_core_context *ras_core,
+		enum mce_bank_type type, u32 bank)
+{
+	const struct mce_smca_bank_range *range;
+	u32 aca_ip_version = 0;
+
+	if (!ras_core || type >= MCE_BANK_TYPE_MAX)
+		return false;
+
+	if (ras_core_get_ip_version(ras_core,
+				RAS_UNIT_ID_ACA, &aca_ip_version))
+		return false;
+
+	switch (aca_ip_version) {
+	case IP_VERSION(5, 0, 0):
+		range = &mce_smca_bank_range_v5[type];
+		break;
+	default:
+		return false;
+	}
+
+	return  ((bank >= range->start) && (bank <= range->end)) ? true : false;
+}
+
+static int ras_mce_add_gpu_bank(struct ras_core_context *ras_core,
+			struct aca_bank_reg *aca_bank)
+{
+	struct ras_mce *mce = &ras_core->ras_mce;
+	int ret;
+
+	ret = kfifo_in_spinlocked(&mce->mce_fifo,
+			aca_bank, sizeof(*aca_bank), &mce->mce_fifo_lock);
+	if (ret)
+		ras_process_add_interrupt_req(ras_core, NULL, false);
+
+	return ret ? 0 : -ENOSPC;
+}
+
+static int ras_mce_log_cpu_bank(struct ras_core_context *ras_core,
+			struct aca_bank_reg *aca_bank)
+{
+	struct ras_cpu_mce cpu_mce = {
+		.apic_id = aca_bank->apic_id,
+		.bank = aca_bank->bank,
+	};
+
+	memcpy(cpu_mce.regs, aca_bank->regs, sizeof(cpu_mce.regs));
+	ras_log_ring_add_log_event(ras_core,
+		RAS_LOG_EVENT_CPU_RAS, &cpu_mce, sizeof(cpu_mce), NULL);
+
+	return 0;
+}
+
+int ras_mce_sw_init(struct ras_core_context *ras_core)
+{
+	struct ras_mce *mce = &ras_core->ras_mce;
+
+	if (kfifo_alloc(&mce->mce_fifo, RAS_MCE_BANK_FIFO_SIZE, GFP_KERNEL))
+		return -ENOMEM;
+
+	spin_lock_init(&mce->mce_fifo_lock);
+
+	return 0;
+}
+
+int ras_mce_sw_fini(struct ras_core_context *ras_core)
+{
+	kfifo_free(&ras_core->ras_mce.mce_fifo);
+
+	return 0;
+}
+
+int ras_mce_get_bank_count(struct ras_core_context *ras_core,
+				enum ras_err_type type, u32 *count)
+{
+	u32 data_count;
+
+	if (!ras_core || !count)
+		return -EINVAL;
+
+	data_count = RAS_MCE_BANK_COUNT(kfifo_len(&ras_core->ras_mce.mce_fifo));
+
+	*count = (data_count <= MAX_RECORD_PER_BATCH) ? data_count : MAX_RECORD_PER_BATCH;
+
+	return 0;
+}
+
+int ras_mce_dump_bank(struct ras_core_context *ras_core,
+			u32 type, u32 idx, struct aca_bank_reg *reg_bank)
+{
+	struct ras_mce *mce = &ras_core->ras_mce;
+	int ret;
+
+	if (!ras_core || !reg_bank)
+		return -EINVAL;
+
+	ret = kfifo_out_spinlocked(&mce->mce_fifo,
+				reg_bank, sizeof(*reg_bank), &mce->mce_fifo_lock);
+
+	return ret ? 0 : -ENOENT;
+}
+
+int ras_mce_add_aca_bank(struct ras_core_context *ras_core,
+		struct aca_bank_reg *aca_bank)
+{
+	if (!ras_core || !aca_bank)
+		return -EINVAL;
+
+	return (aca_bank->bank_type == MCE_BANK_TYPE_GPU) ?
+		ras_mce_add_gpu_bank(ras_core, aca_bank) :
+		ras_mce_log_cpu_bank(ras_core, aca_bank);
+}
diff --git a/drivers/gpu/drm/amd/ras/core/ras_mce.h b/drivers/gpu/drm/amd/ras/core/ras_mce.h
new file mode 100644
index 00000000000000..3b1f690d2eb69e
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_mce.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RAS_MCE_H__
+#define __RAS_MCE_H__
+
+enum mce_bank_type {
+	MCE_BANK_TYPE_GPU,
+	MCE_BANK_TYPE_CPU,
+	MCE_BANK_TYPE_MAX
+};
+
+struct ras_mce {
+	struct kfifo mce_fifo;
+	spinlock_t mce_fifo_lock;
+};
+
+int ras_mce_sw_init(struct ras_core_context *ras_core);
+int ras_mce_sw_fini(struct ras_core_context *ras_core);
+
+int ras_mce_get_bank_count(struct ras_core_context *ras_core,
+	enum ras_err_type type, u32 *count);
+int ras_mce_dump_bank(struct ras_core_context *ras_core,
+	u32 type, u32 idx, struct aca_bank_reg *aca_bank);
+
+bool ras_mce_check_bank(struct ras_core_context *ras_core,
+		enum mce_bank_type, u32 bank);
+int ras_mce_add_aca_bank(struct ras_core_context *ras_core,
+		struct aca_bank_reg *aca_bank);
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_mp1.c b/drivers/gpu/drm/amd/ras/core/ras_mp1.c
index 26af09f3574a85..4c180027850f4e 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_mp1.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_mp1.c
@@ -25,6 +25,7 @@
 #include "ras.h"
 #include "ras_mp1.h"
 #include "ras_mp1_v13_0.h"
+#include "ras_mp1_v15_0.h"
 
 static const struct ras_mp1_ip_func *ras_mp1_get_ip_funcs(
 				struct ras_core_context *ras_core, uint32_t ip_version)
@@ -34,6 +35,8 @@ static const struct ras_mp1_ip_func *ras_mp1_get_ip_funcs(
 	case IP_VERSION(13, 0, 14):
 	case IP_VERSION(13, 0, 12):
 		return &mp1_ras_func_v13_0;
+	case IP_VERSION(15, 0, 8):
+		return &mp1_ras_func_v15_0;
 	default:
 		RAS_DEV_ERR(ras_core->dev,
 			"MP1 ip version(0x%x) is not supported!\n", ip_version);
@@ -47,16 +50,127 @@ int ras_mp1_get_bank_count(struct ras_core_context *ras_core,
 			    enum ras_err_type type, u32 *count)
 {
 	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
 
-	return mp1->ip_func->get_valid_bank_count(ras_core, type, count);
+	if (!mp1->ip_func || !mp1->ip_func->get_valid_bank_count)
+		return 0;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->get_valid_bank_count(ras_core, type, count);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
 }
 
 int ras_mp1_dump_bank(struct ras_core_context *ras_core,
-		u32 type, u32 idx, u32 reg_idx, u64 *val)
+		u32 type, u32 idx, u64 *regs, u32 regs_sz)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
+
+	if (!mp1->ip_func || !mp1->ip_func->dump_valid_bank)
+		return 0;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->dump_valid_bank(ras_core,
+				type, idx, regs, regs_sz);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
+}
+
+int ras_mp1_get_table_version(struct ras_core_context *ras_core,
+		u32 *table_version)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
+
+	if (!mp1->ip_func || !mp1->ip_func->get_table_version)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->get_table_version(ras_core, table_version);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
+}
+
+bool ras_mp1_rma_detected(struct ras_core_context *ras_core)
 {
 	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
 
-	return mp1->ip_func->dump_valid_bank(ras_core, type, idx, reg_idx, val);
+	if (!mp1->ip_func || !mp1->ip_func->rma_detected)
+		return false;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->rma_detected(ras_core);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
+}
+
+int ras_mp1_set_timestamp(struct ras_core_context *ras_core,
+		u64 timestamp)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
+
+	if (!mp1->ip_func || !mp1->ip_func->set_timestamp)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->set_timestamp(ras_core, timestamp);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
+}
+
+int ras_mp1_reset_ras_table(struct ras_core_context *ras_core,
+		u32 *result)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
+
+	if (!result || !mp1->ip_func || !mp1->ip_func->reset_ras_table)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->reset_ras_table(ras_core, result);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
+}
+
+int ras_mp1_get_record_count(struct ras_core_context *ras_core, u32 *count)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
+
+	if (!count || !mp1->ip_func || !mp1->ip_func->get_record_count)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->get_record_count(ras_core, count);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
+}
+
+int ras_mp1_get_record(struct ras_core_context *ras_core,
+		u32 idx, struct eeprom_err_record *rec)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+	int ret;
+
+	if (!rec || !mp1->ip_func || !mp1->ip_func->get_record)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&mp1->op_mutex);
+	ret = mp1->ip_func->get_record(ras_core, idx, rec);
+	mutex_unlock(&mp1->op_mutex);
+
+	return ret;
 }
 
 int ras_mp1_set_debug_mode(struct ras_core_context *ras_core, bool enable)
@@ -69,30 +183,53 @@ int ras_mp1_set_debug_mode(struct ras_core_context *ras_core, bool enable)
 	return mp1->ip_func->set_debug_mode(ras_core, enable);
 }
 
-int ras_mp1_hw_init(struct ras_core_context *ras_core)
+int ras_mp1_check_feature_status(struct ras_core_context *ras_core,
+		enum ras_mp1_feature_id feature_id, u32 *status)
+{
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+
+	if (!mp1->sys_func ||
+	    !mp1->sys_func->check_ras_feature_status)
+		return -EOPNOTSUPP;
+
+	return mp1->sys_func->check_ras_feature_status(ras_core,
+				feature_id, status);
+}
+
+int ras_mp1_sw_init(struct ras_core_context *ras_core)
 {
 	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
 	int ret = 0;
 
-	mp1->mp1_ip_version = ras_core->config->mp1_ip_version;
-	mp1->sys_func = ras_core->config->mp1_cfg.mp1_sys_fn;
-	if (!mp1->sys_func) {
-		RAS_DEV_ERR(ras_core->dev, "RAS mp1 sys function not configured!\n");
+	if (!ras_core->config)
 		return -EINVAL;
-	}
+
+	mp1->mp1_ip_version = ras_core->config->mp1_ip_version;
+
+	if (ras_core->config && ras_core->config->mp1_cfg.mp1_sys_fn)
+		mp1->sys_func = ras_core->config->mp1_cfg.mp1_sys_fn;
 
 	mp1->ip_func = ras_mp1_get_ip_funcs(ras_core, mp1->mp1_ip_version);
 	if (!mp1->ip_func)
 		return -EINVAL;
 
+	mutex_init(&mp1->op_mutex);
+
+	/* Nothing else in the MP1 block depends on the debug mode control. */
 	ret = ras_mp1_set_debug_mode(ras_core, false);
-	if (ret)
-		return -EINVAL;
+	if (ret && ret != -EOPNOTSUPP) {
+		mutex_destroy(&mp1->op_mutex);
+		return ret;
+	}
 
-	return ret;
+	return 0;
 }
 
-int ras_mp1_hw_fini(struct ras_core_context *ras_core)
+int ras_mp1_sw_fini(struct ras_core_context *ras_core)
 {
+	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
+
+	mutex_destroy(&mp1->op_mutex);
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/ras/core/ras_mp1.h b/drivers/gpu/drm/amd/ras/core/ras_mp1.h
index 5bc7c1b7fdab6b..b872fc9758665d 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_mp1.h
+++ b/drivers/gpu/drm/amd/ras/core/ras_mp1.h
@@ -25,29 +25,108 @@
 #define __RAS_MP1_H__
 #include "ras.h"
 
+#define MAX_UE_BANKS_PER_QUERY  12
+#define MAX_CE_BANKS_PER_QUERY  12
+
+enum ras_mp1_feature_id {
+	RAS_MP1_FEATURE_PMFW_EEPROM,
+	RAS_MP1_FEATURE_MAX
+};
+
+enum ras_mp1_msg_id {
+	RAS_MP1_MSG_QueryValidMcaCount,
+	RAS_MP1_MSG_McaBankDumpDW,
+	RAS_MP1_MSG_ClearMcaOnRead,
+	RAS_MP1_MSG_QueryValidMcaCeCount,
+	RAS_MP1_MSG_McaBankCeDumpDW,
+	RAS_MP1_MSG_GetRasTableVersion,
+	RAS_MP1_MSG_GetRmaStatus,
+	RAS_MP1_MSG_GetBadPageCount,
+	RAS_MP1_MSG_GetBadPageMcaAddr,
+	RAS_MP1_MSG_GetBadPagePaAddr,
+	RAS_MP1_MSG_SetTimestamp,
+	RAS_MP1_MSG_GetTimestamp,
+	RAS_MP1_MSG_GetRasPolicy,
+	RAS_MP1_MSG_GetBadPageIpId,
+	RAS_MP1_MSG_EraseRasTable,
+	RAS_MP1_MSG_MAX
+};
+
+struct eeprom_err_record {
+	union {
+		struct {
+			u32 timestamp_low;
+			u32 timestamp_high;
+		};
+		u64 timestamp;
+	};
+
+	union {
+		struct {
+			u32 mca_addr_low;
+			u32 mca_addr_high;
+		};
+		u64 mca_addr;
+	};
+
+	union {
+		struct {
+			u32 ipid_low;
+			u32 ipid_high;
+		};
+		u64 ipid;
+	};
+};
+
 enum ras_err_type;
 struct ras_mp1_ip_func {
 	int (*get_valid_bank_count)(struct ras_core_context *ras_core,
 			enum ras_err_type type, u32 *count);
 	int (*dump_valid_bank)(struct ras_core_context *ras_core,
-		enum ras_err_type type, u32 idx, u32 reg_idx, u64 *val);
+		enum ras_err_type type, u32 idx, u64 *regs, u32 regs_sz);
 	int (*set_debug_mode)(struct ras_core_context *ras_core, bool enable);
+
+	/* The following is used for firmware management of EEPROM */
+	int (*get_table_version)(struct ras_core_context *ras_core,
+			u32 *table_version);
+	bool (*rma_detected)(struct ras_core_context *ras_core);
+	int (*set_timestamp)(struct ras_core_context *ras_core,
+			u64 timestamp);
+	int (*reset_ras_table)(struct ras_core_context *ras_core,
+			u32 *result);
+	int (*get_record_count)(struct ras_core_context *ras_core,
+			u32 *count);
+	int (*get_record)(struct ras_core_context *ras_core,
+			u32 idx, struct eeprom_err_record *rec);
 };
 
 struct ras_mp1 {
 	uint32_t mp1_ip_version;
 	const struct ras_mp1_ip_func *ip_func;
 	const struct ras_mp1_sys_func *sys_func;
+	struct mutex op_mutex;
 };
 
-int ras_mp1_hw_init(struct ras_core_context *ras_core);
-int ras_mp1_hw_fini(struct ras_core_context *ras_core);
+int ras_mp1_sw_init(struct ras_core_context *ras_core);
+int ras_mp1_sw_fini(struct ras_core_context *ras_core);
 
 int ras_mp1_get_bank_count(struct ras_core_context *ras_core,
 			    enum ras_err_type type, u32 *count);
 
 int ras_mp1_dump_bank(struct ras_core_context *ras_core,
-		u32 ecc_type, u32 idx, u32 reg_idx, u64 *val);
+		u32 ecc_type, u32 idx, u64 *regs, u32 regs_sz);
 
 int ras_mp1_set_debug_mode(struct ras_core_context *ras_core, bool enable);
+int ras_mp1_get_table_version(struct ras_core_context *ras_core,
+		u32 *table_version);
+bool ras_mp1_rma_detected(struct ras_core_context *ras_core);
+int ras_mp1_set_timestamp(struct ras_core_context *ras_core,
+		u64 timestamp);
+int ras_mp1_reset_ras_table(struct ras_core_context *ras_core,
+		u32 *result);
+int ras_mp1_get_record_count(struct ras_core_context *ras_core, u32 *count);
+int ras_mp1_get_record(struct ras_core_context *ras_core,
+		u32 idx, struct eeprom_err_record *rec);
+int ras_mp1_check_feature_status(struct ras_core_context *ras_core,
+		enum ras_mp1_feature_id feature_id, u32 *status);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_mp1_v13_0.c b/drivers/gpu/drm/amd/ras/core/ras_mp1_v13_0.c
index b9db91859e5dcd..713c30c5e5a183 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_mp1_v13_0.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_mp1_v13_0.c
@@ -26,30 +26,141 @@
 #include "core_status.h"
 #include "ras_mp1_v13_0.h"
 
-#define RAS_MP1_MSG_QueryValidMcaCount                0x36
-#define RAS_MP1_MSG_McaBankDumpDW                     0x37
-#define RAS_MP1_MSG_ClearMcaOnRead                    0x39
-#define RAS_MP1_MSG_QueryValidMcaCeCount              0x3A
-#define RAS_MP1_MSG_McaBankCeDumpDW                   0x3B
+#define MSG_DATA_LOW32(idx)   (((idx) & 0xFFFFU) | BIT(16))
+#define MSG_DATA_HIGH32(idx)  (((idx) & 0xFFFFU) | BIT(17))
 
-#define MAX_UE_BANKS_PER_QUERY  12
-#define MAX_CE_BANKS_PER_QUERY  12
+#define MP1_V13_ClearMcaOnRead_UE_FLAG_MASK  0x1
+#define MP1_V13_ClearMcaOnRead_CE_POLL_MASK  0x2
+
+static int __send_mp1_msg32(struct ras_core_context *ras_core,
+		enum ras_mp1_msg_id msg_id, u32 input, u32 *output)
+{
+	if (!ras_core->ras_mp1.sys_func ||
+	    !ras_core->ras_mp1.sys_func->mp1_send_ras_msg)
+		return -EOPNOTSUPP;
+
+	return ras_core->ras_mp1.sys_func->mp1_send_ras_msg(ras_core,
+				msg_id, &input, 1, output, output ? 1 : 0);
+}
+
+static int mp1_v13_0_get_table_version(struct ras_core_context *ras_core,
+				     u32 *table_ver)
+{
+	if (!table_ver)
+		return -EINVAL;
+
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetRasTableVersion,
+			0, table_ver);
+}
+
+static bool mp1_v13_0_rma_detected(struct ras_core_context *ras_core)
+{
+	u32 rma = 0;
+
+	if (__send_mp1_msg32(ras_core, RAS_MP1_MSG_GetRmaStatus, 0, &rma))
+		return false;
+
+	return rma;
+}
+
+static int mp1_v13_0_set_timestamp(struct ras_core_context *ras_core,
+			u64 timestamp)
+{
+	if (!timestamp)
+		return -EINVAL;
+
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_SetTimestamp, (u32)timestamp, NULL);
+}
+
+static int mp1_v13_0_reset_ras_table(struct ras_core_context *ras_core,
+				   u32 *result)
+{
+	if (!result)
+		return -EINVAL;
+
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_EraseRasTable, 0, result);
+}
+
+static int mp1_v13_0_get_record_count(struct ras_core_context *ras_core,
+				u32 *count)
+{
+	if (!count)
+		return -EINVAL;
+
+	*count = 0;
+
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetBadPageCount, 0, count);
+}
+
+static int mp1_v13_0_get_record(struct ras_core_context *ras_core,
+			u32 idx, struct eeprom_err_record *rec)
+{
+	int ret;
+
+	if (!rec || (idx > 0xFFFFU))
+		return -EINVAL;
+
+	ret = __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetTimestamp,
+			idx, &rec->timestamp_low);
+	if (ret)
+		return ret;
+
+	ret = __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetBadPageMcaAddr,
+			MSG_DATA_LOW32(idx), &rec->mca_addr_low);
+	if (ret)
+		return ret;
+
+	ret = __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetBadPageMcaAddr,
+			MSG_DATA_HIGH32(idx), &rec->mca_addr_high);
+	if (ret)
+		return ret;
+
+	ret = __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetBadPageIpId,
+			MSG_DATA_LOW32(idx), &rec->ipid_low);
+	if (ret)
+		return ret;
+
+	ret = __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetBadPageIpId,
+			MSG_DATA_HIGH32(idx), &rec->ipid_high);
+
+	return ret;
+}
+
+static int __dump_mp1_bank_reg64(struct ras_core_context *ras_core,
+				     u32 msg, u32 idx, u32 reg_idx, u64 *val)
+{
+	u32 data[2] = {0, 0};
+	u32 param;
+	int ret;
+	u32 i, offset;
+
+	offset = reg_idx * 8;
+	for (i = 0; i < ARRAY_SIZE(data); i++) {
+		param = ((idx & 0xffff) << 16) | ((offset + (i << 2)) & 0xfffc);
+		ret = __send_mp1_msg32(ras_core, msg, param, &data[i]);
+		if (ret) {
+			RAS_DEV_ERR(ras_core->dev,
+				"ACA failed to read register[%u], offset:0x%x\n",
+				reg_idx, offset);
+			return ret;
+		}
+	}
+
+	*val = ((u64)data[1] << 32) | data[0];
+
+	return 0;
+}
 
 static int mp1_v13_0_get_bank_count(struct ras_core_context *ras_core,
 			    enum ras_err_type type, u32 *count)
 {
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-	uint32_t bank_count = 0;
+	u32 bank_count = 0;
 	u32 msg;
 	int ret;
 
 	if (!count)
 		return -EINVAL;
 
-	if (!sys_func || !sys_func->mp1_get_valid_bank_count)
-		return -RAS_CORE_NOT_SUPPORTED;
-
 	switch (type) {
 	case RAS_ERR_TYPE__UE:
 		msg = RAS_MP1_MSG_QueryValidMcaCount;
@@ -62,10 +173,11 @@ static int mp1_v13_0_get_bank_count(struct ras_core_context *ras_core,
 		return -EINVAL;
 	}
 
-	ret = sys_func->mp1_get_valid_bank_count(ras_core, msg, &bank_count);
+	ret = __send_mp1_msg32(ras_core, msg, 0, &bank_count);
 	if (!ret) {
 		if (((type == RAS_ERR_TYPE__UE) && (bank_count >= MAX_UE_BANKS_PER_QUERY)) ||
-			((type == RAS_ERR_TYPE__CE) && (bank_count >= MAX_CE_BANKS_PER_QUERY)))
+			((type == RAS_ERR_TYPE__CE || type == RAS_ERR_TYPE__DE) &&
+			 (bank_count >= MAX_CE_BANKS_PER_QUERY)))
 			return -EINVAL;
 
 		*count = bank_count;
@@ -75,14 +187,13 @@ static int mp1_v13_0_get_bank_count(struct ras_core_context *ras_core,
 }
 
 static int mp1_v13_0_dump_bank(struct ras_core_context *ras_core,
-			enum ras_err_type type, u32 idx, u32 reg_idx, u64 *val)
+			enum ras_err_type type, u32 idx, u64 *regs, u32 regs_sz)
 {
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
+	int i, ret, reg_cnt;
 	u32 msg;
 
-	if (!sys_func || !sys_func->mp1_dump_valid_bank)
-		return -RAS_CORE_NOT_SUPPORTED;
+	if (!regs || !regs_sz || (idx > 0xffff))
+		return -EINVAL;
 
 	switch (type) {
 	case RAS_ERR_TYPE__UE:
@@ -96,18 +207,22 @@ static int mp1_v13_0_dump_bank(struct ras_core_context *ras_core,
 		return -EINVAL;
 	}
 
-	return sys_func->mp1_dump_valid_bank(ras_core, msg, idx, reg_idx, val);
+	reg_cnt = min_t(int, 16, regs_sz);
+	for (i = 0; i < reg_cnt; i++) {
+		ret = __dump_mp1_bank_reg64(ras_core, msg, idx, i, &regs[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int mp1_v13_0_set_debug_mode(struct ras_core_context *ras_core, bool enable)
 {
-	struct ras_mp1 *mp1 = &ras_core->ras_mp1;
-	const struct ras_mp1_sys_func *sys_func = mp1->sys_func;
-
-	if (!sys_func || !sys_func->mp1_set_debug_mode)
-		return -RAS_CORE_NOT_SUPPORTED;
+	u32 param = enable ? 0 :
+		(MP1_V13_ClearMcaOnRead_UE_FLAG_MASK | MP1_V13_ClearMcaOnRead_CE_POLL_MASK);
 
-	return sys_func->mp1_set_debug_mode(ras_core, enable);
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_ClearMcaOnRead, param, NULL);
 }
 
 
@@ -115,4 +230,10 @@ const struct ras_mp1_ip_func mp1_ras_func_v13_0 = {
 	.get_valid_bank_count = mp1_v13_0_get_bank_count,
 	.dump_valid_bank = mp1_v13_0_dump_bank,
 	.set_debug_mode = mp1_v13_0_set_debug_mode,
+	.get_table_version = mp1_v13_0_get_table_version,
+	.rma_detected = mp1_v13_0_rma_detected,
+	.set_timestamp = mp1_v13_0_set_timestamp,
+	.reset_ras_table = mp1_v13_0_reset_ras_table,
+	.get_record_count = mp1_v13_0_get_record_count,
+	.get_record = mp1_v13_0_get_record,
 };
diff --git a/drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.c b/drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.c
new file mode 100644
index 00000000000000..9b3c6651bb852f
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.c
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "ras.h"
+#include "ras_mp1.h"
+#include "core_status.h"
+#include "ras_mp1_v13_0.h"
+
+#define regMP1_SMN_C2PMSG_40                              0x0068
+#define regMP1_SMN_C2PMSG_40_BASE_IDX                     2
+#define regMP1_SMN_C2PMSG_41                              0x0069
+#define regMP1_SMN_C2PMSG_41_BASE_IDX                     2
+#define regMP1_SMN_C2PMSG_42                              0x006a
+#define regMP1_SMN_C2PMSG_42_BASE_IDX                     2
+#define regMP1_SMN_C2PMSG_43                              0x006b
+#define regMP1_SMN_C2PMSG_43_BASE_IDX                     2
+#define regMP1_SMN_C2PMSG_44                              0x006c
+#define regMP1_SMN_C2PMSG_44_BASE_IDX                     2
+#define regMP1_SMN_C2PMSG_45                              0x006d
+#define regMP1_SMN_C2PMSG_45_BASE_IDX                     2
+
+#define MP1_RESP_OK  1
+
+static u32 ras_mp1_msg_codes[RAS_MP1_MSG_MAX] = {
+	[RAS_MP1_MSG_GetRasTableVersion] = 0x1F,
+	[RAS_MP1_MSG_GetRmaStatus] =    0x20,
+	[RAS_MP1_MSG_GetBadPageCount] = 0x21,
+	[RAS_MP1_MSG_GetBadPageMcaAddr] = 0x22,
+	[RAS_MP1_MSG_GetBadPagePaAddr] =  0x23,
+	[RAS_MP1_MSG_SetTimestamp] = 0x24,
+	[RAS_MP1_MSG_GetTimestamp] = 0x25,
+	[RAS_MP1_MSG_GetRasPolicy] = 0x26,
+	[RAS_MP1_MSG_GetBadPageIpId] = 0x27,
+	[RAS_MP1_MSG_EraseRasTable] =  0x28,
+};
+
+static int __direct_send_mp1_msg(struct ras_core_context *ras_core,
+		enum ras_mp1_msg_id msg_id, u32 *inputs, u32 num_inputs,
+		u32 *outputs, u32 num_outputs)
+{
+	u32 msg_code = 0;
+	int timeout = 100000;  //100 ms
+	u32 reg = 0;
+
+	if (num_inputs > 2 || num_outputs > 2)
+		return -EINVAL;
+
+	msg_code = ras_mp1_msg_codes[msg_id];
+	if (!msg_code)
+		return -EOPNOTSUPP;
+
+	/* Send message and parameter to fw */
+	RAS_DEV_WREG32_SOC15(ras_core->dev, MP1, 0, regMP1_SMN_C2PMSG_41, 0);
+	if (num_inputs == 1) {
+		/* Input u32 parameter */
+		RAS_DEV_WREG32_SOC15(ras_core->dev,
+			MP1, 0, regMP1_SMN_C2PMSG_42, inputs[0]);
+	} else if (num_inputs == 2) {
+		/* Input u64 parameter */
+		RAS_DEV_WREG32_SOC15(ras_core->dev,
+			MP1, 0, regMP1_SMN_C2PMSG_42, inputs[0]);
+		RAS_DEV_WREG32_SOC15(ras_core->dev,
+			MP1, 0, regMP1_SMN_C2PMSG_43, inputs[1]);
+	}
+	RAS_DEV_WREG32_SOC15(ras_core->dev, MP1, 0, regMP1_SMN_C2PMSG_40, msg_code);
+
+	/* Poll MP1 response */
+	while (timeout--) {
+		reg = RAS_DEV_RREG32_SOC15(ras_core->dev, MP1, 0, regMP1_SMN_C2PMSG_41);
+		if (reg)
+			break;
+
+		udelay(1);
+	};
+
+	if (reg != MP1_RESP_OK) {
+		RAS_DEV_ERR(ras_core->dev, "MP1 fail to ack 0x%x for msg: 0x%x\n",
+			reg, msg_code);
+		return -EIO;
+	}
+
+	/* Read output data */
+	if (outputs && num_outputs) {
+		if (num_outputs == 1) {
+			/* Output u32 parameter */
+			outputs[0] = RAS_DEV_RREG32_SOC15(ras_core->dev,
+						MP1, 0, regMP1_SMN_C2PMSG_42);
+		} else if (num_outputs == 2) {
+			/* Output u64 parameter */
+			outputs[0] = RAS_DEV_RREG32_SOC15(ras_core->dev,
+						MP1, 0, regMP1_SMN_C2PMSG_42);
+			outputs[1] = RAS_DEV_RREG32_SOC15(ras_core->dev,
+						MP1, 0, regMP1_SMN_C2PMSG_43);
+		}
+	}
+
+	return 0;
+}
+
+static int __sys_send_mp1_msg(struct ras_core_context *ras_core,
+		enum ras_mp1_msg_id msg_id, u32 *inputs, u32 num_inputs,
+		u32 *outputs, u32 num_outputs)
+{
+	if (!ras_core->ras_mp1.sys_func ||
+	    !ras_core->ras_mp1.sys_func->mp1_send_ras_msg)
+		return -EOPNOTSUPP;
+
+	return ras_core->ras_mp1.sys_func->mp1_send_ras_msg(ras_core,
+				msg_id, inputs, num_inputs, outputs, num_outputs);
+}
+
+static int __send_mp1_msg(struct ras_core_context *ras_core,
+		enum ras_mp1_msg_id msg_id, u32 *inputs, u32 num_inputs,
+		u32 *outputs, u32 num_outputs)
+{
+	if (msg_id >= RAS_MP1_MSG_MAX)
+		return -EINVAL;
+
+	if (ras_core_in_early_init(ras_core))
+		return __direct_send_mp1_msg(ras_core, msg_id,
+				inputs, num_inputs, outputs, num_outputs);
+	else
+		return __sys_send_mp1_msg(ras_core, msg_id,
+				inputs, num_inputs, outputs, num_outputs);
+}
+
+static int __send_mp1_msg32(struct ras_core_context *ras_core,
+		enum ras_mp1_msg_id msg_id, u32 input, u32 *output)
+{
+	return __send_mp1_msg(ras_core, msg_id,
+			&input, 1, output, output ? 1 : 0);
+}
+
+static int __send_mp1_msg64(struct ras_core_context *ras_core,
+		enum ras_mp1_msg_id msg_id, u64 input, u64 *output)
+{
+	u32 in[2] = {lower_32_bits(input), upper_32_bits(input)};
+	u32 out[2] = {0};
+	int ret;
+
+	ret = __send_mp1_msg(ras_core, msg_id,
+			in, 2, output ? out : NULL, output ? 2 : 0);
+	if (!ret && output)
+		*output = ((u64)out[1] << 32) | out[0];
+
+	return ret;
+}
+
+static int ras_mp1_v15_get_table_version(struct ras_core_context *ras_core,
+				     u32 *table_ver)
+{
+	if (!table_ver)
+		return -EINVAL;
+
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetRasTableVersion,
+			0, table_ver);
+}
+
+static bool ras_mp1_v15_rma_detected(struct ras_core_context *ras_core)
+{
+	u32 rma = 0;
+
+	if (__send_mp1_msg32(ras_core, RAS_MP1_MSG_GetRmaStatus, 0, &rma))
+		return false;
+
+	return rma;
+}
+
+static int ras_mp1_v15_set_timestamp(struct ras_core_context *ras_core,
+			u64 timestamp)
+{
+	if (!timestamp)
+		return -EINVAL;
+
+	return __send_mp1_msg64(ras_core, RAS_MP1_MSG_SetTimestamp, timestamp, NULL);
+}
+
+static int ras_mp1_v15_reset_ras_table(struct ras_core_context *ras_core,
+				   u32 *result)
+{
+	if (!result)
+		return -EINVAL;
+
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_EraseRasTable, 0, result);
+}
+
+static int ras_mp1_v15_get_record_count(struct ras_core_context *ras_core,
+				u32 *count)
+{
+	if (!count)
+		return -EINVAL;
+
+	*count = 0;
+
+	return __send_mp1_msg32(ras_core, RAS_MP1_MSG_GetBadPageCount, 0, count);
+}
+
+static int ras_mp1_v15_get_record(struct ras_core_context *ras_core,
+			u32 idx, struct eeprom_err_record *rec)
+{
+	int ret;
+
+	if (!rec)
+		return -EINVAL;
+
+	ret = __send_mp1_msg64(ras_core, RAS_MP1_MSG_GetTimestamp,
+			idx, &rec->timestamp);
+	if (ret)
+		return ret;
+
+	ret = __send_mp1_msg64(ras_core, RAS_MP1_MSG_GetBadPageMcaAddr,
+			idx, &rec->mca_addr);
+	if (ret)
+		return ret;
+
+	ret = __send_mp1_msg64(ras_core, RAS_MP1_MSG_GetBadPageIpId,
+			idx, &rec->ipid);
+
+	return ret;
+}
+
+const struct ras_mp1_ip_func mp1_ras_func_v15_0 = {
+	.get_table_version = ras_mp1_v15_get_table_version,
+	.rma_detected = ras_mp1_v15_rma_detected,
+	.set_timestamp = ras_mp1_v15_set_timestamp,
+	.reset_ras_table = ras_mp1_v15_reset_ras_table,
+	.get_record_count = ras_mp1_v15_get_record_count,
+	.get_record = ras_mp1_v15_get_record,
+};
diff --git a/drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.h b/drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.h
new file mode 100644
index 00000000000000..c93b40a604c1ac
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_mp1_v15_0.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RAS_MP1_V15_0_H__
+#define __RAS_MP1_V15_0_H__
+#include "ras_mp1.h"
+
+extern const struct ras_mp1_ip_func mp1_ras_func_v15_0;
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_nbio.c b/drivers/gpu/drm/amd/ras/core/ras_nbio.c
index bfddd104d5486f..ca543b2f367916 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_nbio.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_nbio.c
@@ -33,6 +33,9 @@ static const struct ras_nbio_ip_func *ras_nbio_get_ip_funcs(
 	case IP_VERSION(7, 9, 0):
 	case IP_VERSION(7, 9, 1):
 		return &ras_nbio_v7_9;
+	case IP_VERSION(6, 3, 2):
+		/* TBD for dGPU */
+		break;
 	default:
 		RAS_DEV_ERR(ras_core->dev,
 			"NBIO ip version(0x%x) is not supported!\n", ip_version);
@@ -48,14 +51,8 @@ int ras_nbio_hw_init(struct ras_core_context *ras_core)
 
 	nbio->nbio_ip_version = ras_core->config->nbio_ip_version;
 	nbio->sys_func = ras_core->config->nbio_cfg.nbio_sys_fn;
-	if (!nbio->sys_func) {
-		RAS_DEV_ERR(ras_core->dev, "RAS nbio sys function not configured!\n");
-		return -EINVAL;
-	}
 
 	nbio->ip_func = ras_nbio_get_ip_funcs(ras_core, nbio->nbio_ip_version);
-	if (!nbio->ip_func)
-		return -EINVAL;
 
 	if (nbio->sys_func) {
 		if (nbio->sys_func->set_ras_controller_irq_state)
diff --git a/drivers/gpu/drm/amd/ras/core/ras_process.c b/drivers/gpu/drm/amd/ras/core/ras_process.c
index c001074c8c568d..f313f2021f2438 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_process.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_process.c
@@ -88,7 +88,8 @@ static int ras_process_umc_event(struct ras_core_context *ras_core,
 		if (ret)
 			return ret;
 
-		ret = ras_core_query_block_ecc_data(ras_core, RAS_BLOCK_ID__UMC, &ecc_data);
+		ret = ras_core_query_block_ecc_data(ras_core,
+				RAS_BLOCK_ID__UMC, &ecc_data, true);
 		if (ret)
 			return ret;
 
@@ -170,6 +171,17 @@ int ras_process_handle_ras_event(struct ras_core_context *ras_core)
 	ras_aca_clear_fatal_flag(ras_core);
 	ras_umc_log_pending_bad_bank(ras_core);
 
+	if (ras_eeprom_mgr_fw_record_enabled(ras_core))
+		ras_umc_dump_fw_records(ras_core);
+
+	if (ras_eeprom_mgr_get_gpu_op_status(ras_core) == RAS_GPU_OP_STATUS_RMA) {
+		uint32_t rma_reset = GPU_RESET_CAUSE_RMA;
+
+		RAS_DEV_ERR(ras_core->dev, "Device is RMA, stop gpu work!\n");
+		ras_core_event_notify(ras_core, RAS_EVENT_ID__RESET_GPU, &rma_reset);
+		return -ERESTART;
+	}
+
 	do {
 		umc_event_count = atomic_read(&ras_proc->umc_interrupt_count);
 		ret = ras_process_umc_event(ras_core, umc_event_count);
@@ -207,6 +219,7 @@ static int ras_process_thread(void *context)
 {
 	struct ras_core_context *ras_core = (struct ras_core_context *)context;
 	struct ras_process *ras_proc = &ras_core->ras_proc;
+	int ret;
 
 	while (!kthread_should_stop()) {
 		ras_wait_event_interruptible_timeout(&ras_proc->ras_process_wq,
@@ -225,9 +238,13 @@ static int ras_process_thread(void *context)
 			continue;
 
 		if (ras_core->sys_fn && ras_core->sys_fn->async_handle_ras_event)
-			ras_core->sys_fn->async_handle_ras_event(ras_core, NULL);
+			ret = ras_core->sys_fn->async_handle_ras_event(ras_core, NULL);
 		else
-			ras_process_handle_ras_event(ras_core);
+			ret = ras_process_handle_ras_event(ras_core);
+
+		/* GPU Device is RMA */
+		if (ret == -ERESTART)
+			break;
 	}
 
 	return 0;
@@ -294,13 +311,16 @@ static int ras_process_add_non_umc_interrupt_req(struct ras_core_context *ras_co
 	struct ras_process *ras_proc = &ras_core->ras_proc;
 	int ret;
 
-	ret = ras_process_put_event(ras_core, req);
-	if (!ret) {
-		atomic_inc(&ras_proc->ras_interrupt_req);
-		wake_up(&ras_proc->ras_process_wq);
+	if (req) {
+		ret = ras_process_put_event(ras_core, req);
+		if (ret)
+			return ret;
 	}
 
-	return ret;
+	atomic_inc(&ras_proc->ras_interrupt_req);
+	wake_up(&ras_proc->ras_process_wq);
+
+	return 0;
 }
 
 int ras_process_add_interrupt_req(struct ras_core_context *ras_core,
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp.c b/drivers/gpu/drm/amd/ras/core/ras_psp.c
index 9bf20201649f99..ed53cd25d6bd38 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_psp.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp.c
@@ -25,6 +25,7 @@
 #include "ta_if.h"
 #include "ras_psp.h"
 #include "ras_psp_v13_0.h"
+#include "ras_psp_v15_0.h"
 
 /* position of instance value in sub_block_index of
  * ta_ras_trigger_error_input, the sub block uses lower 12 bits
@@ -32,6 +33,12 @@
 #define RAS_TA_INST_MASK 0xfffff000
 #define RAS_TA_INST_SHIFT 0xc
 
+#define RAS_PSP_RING_SIZE  0x100000
+#define RAS_PSP_CMD_SIZE   0x100000
+#define RAS_PSP_FENCE_SIZE 0x1000
+#define RAS_FW_BIN_SIZE    0x100000
+#define RAS_TA_CMD_SIZE    0x100000
+
 static const struct ras_psp_ip_func *ras_psp_get_ip_funcs(
 			struct ras_core_context *ras_core, uint32_t ip_version)
 {
@@ -40,6 +47,8 @@ static const struct ras_psp_ip_func *ras_psp_get_ip_funcs(
 	case IP_VERSION(13, 0, 14):
 	case IP_VERSION(13, 0, 12):
 		return &ras_psp_v13_0;
+	case IP_VERSION(15, 0, 8):
+		return &ras_psp_v15_0;
 	default:
 		RAS_DEV_ERR(ras_core->dev,
 			"psp ip version(0x%x) is not supported!\n", ip_version);
@@ -49,100 +58,172 @@ static const struct ras_psp_ip_func *ras_psp_get_ip_funcs(
 	return NULL;
 }
 
-static int ras_psp_sync_system_ras_psp_status(struct ras_core_context *ras_core)
+static int ras_psp_get_system_ras_status(struct ras_core_context *ras_core,
+		struct ras_psp_sys_status *status)
 {
 	struct ras_psp *psp = &ras_core->ras_psp;
-	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
-	struct ras_psp_ctx *psp_ctx = &ras_core->ras_psp.psp_ctx;
-	struct ras_psp_sys_status status = {0};
 	int ret;
 
 	if (psp->sys_func && psp->sys_func->get_ras_psp_system_status) {
-		ret = psp->sys_func->get_ras_psp_system_status(ras_core, &status);
+		ret = psp->sys_func->get_ras_psp_system_status(ras_core, status);
 		if (ret)
 			return ret;
-
-		if (status.initialized) {
-			ta_ctx->preload_ras_ta_enabled = true;
-			ta_ctx->ras_ta_initialized = status.initialized;
-			ta_ctx->session_id = status.session_id;
-		}
-
-		psp_ctx->external_mutex = status.psp_cmd_mutex;
 	}
 
 	return 0;
 }
 
-static int ras_psp_get_ras_ta_init_param(struct ras_core_context *ras_core,
-	struct ras_ta_init_param *ras_ta_param)
+static int ras_psp_get_ras_param(struct ras_core_context *ras_core,
+	struct ras_param *param)
 {
 	struct ras_psp *psp = &ras_core->ras_psp;
+	int ret;
 
-	if (psp->sys_func && psp->sys_func->get_ras_ta_init_param)
-		return psp->sys_func->get_ras_ta_init_param(ras_core, ras_ta_param);
+	if (!psp->sys_func || !psp->sys_func->get_ras_param) {
+		RAS_DEV_ERR(ras_core->dev, "Not config get_ras_ta_init_param API!!\n");
+		return -EINVAL;
+	}
+
+	ret = psp->sys_func->get_ras_param(ras_core, param);
+	if (ret)
+		return ret;
 
-	RAS_DEV_ERR(ras_core->dev, "Not config get_ras_ta_init_param API!!\n");
-	return -EACCES;
+	param->ta_param.nps_mode = ras_core_get_curr_nps_mode(ras_core);
+	param->ta_param.vram_type = ras_core_get_vram_type(ras_core);
+	param->ta_param.poison_mode_en = ras_core_poison_supported(ras_core) ? 1 : 0;
+
+	return 0;
 }
 
-static struct gpu_mem_block *ras_psp_get_gpu_mem(struct ras_core_context *ras_core,
-			enum gpu_mem_type mem_type)
+static struct gpu_mem_block *ras_psp_alloc_mem(struct ras_core_context *ras_core,
+			enum gpu_mem_type mem_type, u32 mem_size)
 {
-	struct ras_psp *psp = &ras_core->ras_psp;
-	struct gpu_mem_block *gpu_mem = NULL;
+	struct gpu_mem_block *gpu_mem;
 	int ret;
 
-	switch (mem_type) {
-	case GPU_MEM_TYPE_RAS_PSP_RING:
-		gpu_mem = &psp->psp_ring.ras_ring_gpu_mem;
-		break;
-	case GPU_MEM_TYPE_RAS_PSP_CMD:
-		gpu_mem = &psp->psp_ctx.psp_cmd_gpu_mem;
-		break;
-	case GPU_MEM_TYPE_RAS_PSP_FENCE:
-		gpu_mem = &psp->psp_ctx.out_fence_gpu_mem;
-		break;
-	case GPU_MEM_TYPE_RAS_TA_FW:
-		gpu_mem = &psp->ta_ctx.fw_gpu_mem;
-		break;
-	case GPU_MEM_TYPE_RAS_TA_CMD:
-		gpu_mem = &psp->ta_ctx.cmd_gpu_mem;
-		break;
-	default:
+	if ((mem_type >= GPU_MEM_TYPE_MAX) || (mem_size < PAGE_SIZE))
 		return NULL;
-	}
 
-	if (!gpu_mem->ref_count) {
-		ret = ras_core_get_gpu_mem(ras_core, mem_type, gpu_mem);
-		if (ret)
-			return NULL;
-		gpu_mem->mem_type = mem_type;
-	}
+	gpu_mem = kzalloc(sizeof(*gpu_mem), GFP_KERNEL);
+	if (!gpu_mem)
+		return NULL;
+
+	gpu_mem->mem_size = mem_size;
+	ret = ras_core_get_gpu_mem(ras_core, mem_type, gpu_mem);
+	if (ret)
+		goto err;
 
+	gpu_mem->mem_type = mem_type;
 	gpu_mem->ref_count++;
 
 	return gpu_mem;
+
+err:
+	kfree(gpu_mem);
+	return NULL;
 }
 
-static int ras_psp_put_gpu_mem(struct ras_core_context *ras_core,
+static int ras_psp_free_mem(struct ras_core_context *ras_core,
 			struct gpu_mem_block *gpu_mem)
 {
 	if (!gpu_mem)
 		return 0;
 
-	gpu_mem->ref_count--;
-
-	if (gpu_mem->ref_count > 0) {
+	if (!gpu_mem->ref_count) {
 		return 0;
-	} else if (gpu_mem->ref_count < 0) {
-		RAS_DEV_WARN(ras_core->dev,
-			"Duplicate free gpu memory %u\n", gpu_mem->mem_type);
-	} else {
+	} else if (gpu_mem->ref_count == 1) {
 		ras_core_put_gpu_mem(ras_core, gpu_mem->mem_type, gpu_mem);
 		memset(gpu_mem, 0, sizeof(*gpu_mem));
+		kfree(gpu_mem);
+	} else if (gpu_mem->ref_count > 1) {
+		gpu_mem->ref_count--;
+	}
+
+	return 0;
+}
+
+static int __ras_psp_mem_init(struct ras_core_context *ras_core)
+{
+	struct ras_psp *psp = &ras_core->ras_psp;
+	struct gpu_mem_block *psp_ring = NULL;
+	struct gpu_mem_block *psp_cmd = NULL;
+	struct gpu_mem_block *psp_fence = NULL;
+	struct gpu_mem_block *fw_bin = NULL;
+	struct gpu_mem_block *ta_cmd = NULL;
+
+	/* Avoid repeatedly reinitializing memory */
+	if (psp->psp_ring.ras_ring_gpu_mem)
+		return 0;
+
+	psp_ring = ras_psp_alloc_mem(ras_core,
+				GPU_MEM_TYPE_RAS_PSP_RING, RAS_PSP_RING_SIZE);
+	if (!psp_ring)
+		return -EPIPE;
+
+	psp_cmd = ras_psp_alloc_mem(ras_core,
+				GPU_MEM_TYPE_RAS_PSP_CMD, RAS_PSP_CMD_SIZE);
+	if (!psp_cmd)
+		goto err;
+
+	psp_fence = ras_psp_alloc_mem(ras_core,
+				GPU_MEM_TYPE_RAS_PSP_FENCE, RAS_PSP_FENCE_SIZE);
+	if (!psp_fence)
+		goto err;
+
+	if (psp->use_dedicated_memory) {
+		fw_bin = ras_psp_alloc_mem(ras_core,
+				GPU_MEM_TYPE_RAS_FW_BIN, RAS_FW_BIN_SIZE);
+		if (!fw_bin)
+			goto err;
+
+		ta_cmd = ras_psp_alloc_mem(ras_core,
+				GPU_MEM_TYPE_RAS_TA_CMD, RAS_TA_CMD_SIZE);
+		if (!ta_cmd)
+			goto err;
+	} else {
+		fw_bin = ras_psp_alloc_mem(ras_core,
+				GPU_MEM_TYPE_ALLOC_MEM, RAS_FW_BIN_SIZE);
+		if (!fw_bin)
+			goto err;
+
+		ta_cmd = ras_psp_alloc_mem(ras_core,
+				GPU_MEM_TYPE_ALLOC_MEM, RAS_TA_CMD_SIZE);
+		if (!ta_cmd)
+			goto err;
 	}
 
+	psp->psp_ring.ras_ring_gpu_mem = psp_ring;
+	psp->psp_ctx.psp_cmd_gpu_mem = psp_cmd;
+	psp->psp_ctx.out_fence_gpu_mem = psp_fence;
+	psp->ta_ctx.fw_gpu_mem = fw_bin;
+	psp->ta_ctx.cmd_gpu_mem = ta_cmd;
+
+	return 0;
+
+err:
+	ras_psp_free_mem(ras_core, psp_ring);
+	ras_psp_free_mem(ras_core, psp_cmd);
+	ras_psp_free_mem(ras_core, psp_fence);
+	ras_psp_free_mem(ras_core, fw_bin);
+	ras_psp_free_mem(ras_core, ta_cmd);
+	return -EPIPE;
+}
+
+static int __ras_psp_mem_fini(struct ras_core_context *ras_core)
+{
+	struct ras_psp *psp = &ras_core->ras_psp;
+
+	ras_psp_free_mem(ras_core, psp->psp_ring.ras_ring_gpu_mem);
+	ras_psp_free_mem(ras_core, psp->psp_ctx.psp_cmd_gpu_mem);
+	ras_psp_free_mem(ras_core, psp->psp_ctx.out_fence_gpu_mem);
+	ras_psp_free_mem(ras_core, psp->ta_ctx.fw_gpu_mem);
+	ras_psp_free_mem(ras_core, psp->ta_ctx.cmd_gpu_mem);
+
+	psp->psp_ctx.psp_cmd_gpu_mem = NULL;
+	psp->psp_ctx.out_fence_gpu_mem = NULL;
+	psp->ta_ctx.fw_gpu_mem = NULL;
+	psp->ta_ctx.cmd_gpu_mem = NULL;
+	psp->psp_ring.ras_ring_gpu_mem = NULL;
 	return 0;
 }
 
@@ -188,6 +269,7 @@ static int __set_ring_frame_slot(struct ras_core_context *ras_core,
 static int write_frame_to_ras_psp_ring(struct ras_core_context *ras_core,
 		struct psp_gfx_rb_frame *frame)
 {
+	struct ras_psp *psp = &ras_core->ras_psp;
 	struct gpu_mem_block *ring_mem;
 	struct psp_gfx_rb_frame *rb_frame;
 	uint32_t max_frame_slot;
@@ -195,7 +277,7 @@ static int write_frame_to_ras_psp_ring(struct ras_core_context *ras_core,
 	uint32_t write_flush_read_back = 0;
 	int ret = 0;
 
-	ring_mem = ras_psp_get_gpu_mem(ras_core, GPU_MEM_TYPE_RAS_PSP_RING);
+	ring_mem = psp->psp_ring.ras_ring_gpu_mem;
 	if (!ring_mem)
 		return -ENOMEM;
 
@@ -223,8 +305,7 @@ static int write_frame_to_ras_psp_ring(struct ras_core_context *ras_core,
 			rb_frame[slot_idx].fence_addr_hi,
 			rb_frame[slot_idx].fence_addr_lo,
 			write_flush_read_back, frame->fence_value);
-		ret = -EACCES;
-		goto err;
+		return -EACCES;
 	}
 
 	slot_idx++;
@@ -234,8 +315,6 @@ static int write_frame_to_ras_psp_ring(struct ras_core_context *ras_core,
 
 	__set_ring_frame_slot(ras_core, slot_idx);
 
-err:
-	ras_psp_put_gpu_mem(ras_core, ring_mem);
 	return ret;
 }
 
@@ -258,14 +337,9 @@ static int send_psp_cmd(struct ras_core_context *ras_core,
 
 	__acquire_psp_cmd_lock(ras_core);
 
-	psp_cmd_buf = ras_psp_get_gpu_mem(ras_core, GPU_MEM_TYPE_RAS_PSP_CMD);
-	if (!psp_cmd_buf) {
-		ret = -ENOMEM;
-		goto exit;
-	}
-
-	psp_fence_buf = ras_psp_get_gpu_mem(ras_core, GPU_MEM_TYPE_RAS_PSP_FENCE);
-	if (!psp_fence_buf) {
+	psp_cmd_buf = psp_ctx->psp_cmd_gpu_mem;
+	psp_fence_buf = psp_ctx->out_fence_gpu_mem;
+	if (!psp_cmd_buf || !psp_fence_buf) {
 		ret = -ENOMEM;
 		goto exit;
 	}
@@ -312,9 +386,6 @@ static int send_psp_cmd(struct ras_core_context *ras_core,
 	resp->session_id = gfx_cmd->resp.session_id;
 
 exit:
-	ras_psp_put_gpu_mem(ras_core, psp_cmd_buf);
-	ras_psp_put_gpu_mem(ras_core, psp_fence_buf);
-
 	__release_psp_cmd_lock(ras_core);
 
 	return ret;
@@ -363,6 +434,7 @@ static int send_ras_ta_runtime_cmd(struct ras_core_context *ras_core,
 	struct ras_ta_cmd *ras_cmd;
 	struct psp_gfx_cmd_invoke_cmd invoke_cmd = {0};
 	struct psp_cmd_resp resp = {0};
+	int got_reset_lock = 0;
 	int ret = 0;
 
 	if (!in || (in_size > sizeof(union ras_ta_cmd_input)) ||
@@ -371,15 +443,19 @@ static int send_ras_ta_runtime_cmd(struct ras_core_context *ras_core,
 		return -EINVAL;
 	}
 
-	ras_psp_sync_system_ras_psp_status(ras_core);
+	if (!ta_ctx->ras_ta_initialized) {
+		RAS_DEV_ERR(ras_core->dev, "RAS TA is not initialized, cmd_id: %u\n", cmd_id);
+		return -EACCES;
+	}
 
-	cmd_mem = ras_psp_get_gpu_mem(ras_core, GPU_MEM_TYPE_RAS_TA_CMD);
+	cmd_mem = ta_ctx->cmd_gpu_mem;
 	if (!cmd_mem)
 		return -ENOMEM;
 
-	if (!ras_core_down_trylock_gpu_reset_lock(ras_core)) {
-		ret = -EACCES;
-		goto out;
+	if (!ras_core_gpu_in_reset(ras_core)) {
+		got_reset_lock = ras_core_down_trylock_gpu_reset_lock(ras_core);
+		if (!got_reset_lock)
+			return  -EACCES;
 	}
 
 	ras_cmd = (struct ras_ta_cmd *)cmd_mem->mem_cpu_addr;
@@ -425,9 +501,8 @@ static int send_ras_ta_runtime_cmd(struct ras_core_context *ras_core,
 
 unlock:
 	mutex_unlock(&ta_ctx->ta_mutex);
-	ras_core_up_gpu_reset_lock(ras_core);
-out:
-	ras_psp_put_gpu_mem(ras_core, cmd_mem);
+	if (got_reset_lock)
+		ras_core_up_gpu_reset_lock(ras_core);
 	return ret;
 }
 
@@ -465,53 +540,96 @@ static int trigger_ras_ta_error(struct ras_core_context *ras_core,
 				info, sizeof(*info), NULL, 0);
 }
 
-static int send_load_ta_fw_cmd(struct ras_core_context *ras_core,
-				struct ras_ta_ctx *ta_ctx)
+static int load_ras_rl_fw(struct ras_core_context *ras_core,
+		struct ras_fw_bin *rl_bin)
 {
-	struct ras_ta_fw_bin  *fw_bin = &ta_ctx->fw_bin;
+	struct ras_psp *psp = &ras_core->ras_psp;
 	struct gpu_mem_block *fw_mem;
-	struct gpu_mem_block *cmd_mem;
+	struct psp_gfx_cmd_load_ip_fw psp_load_rl_fw;
+	struct psp_cmd_resp resp = {0};
+	int got_reset_lock = 0;
+	int ret = 0;
+
+	fw_mem = psp->ta_ctx.fw_gpu_mem;
+	if (!fw_mem)
+		return -ENOMEM;
+
+	if (!ras_core_gpu_in_reset(ras_core)) {
+		got_reset_lock = ras_core_down_trylock_gpu_reset_lock(ras_core);
+		if (!got_reset_lock)
+			return -EACCES;
+	}
+
+	mutex_lock(&psp->ta_ctx.ta_mutex);
+	/* copy ras RL to shared gpu memory */
+	memcpy(fw_mem->mem_cpu_addr, rl_bin->bin_addr, rl_bin->bin_size);
+	fw_mem->mem_size = rl_bin->bin_size;
+
+	/* Setup load RL command */
+	memset(&psp_load_rl_fw, 0, sizeof(psp_load_rl_fw));
+	psp_load_rl_fw.fw_phy_addr_lo = lower_32_bits(fw_mem->mem_mc_addr);
+	psp_load_rl_fw.fw_phy_addr_hi = upper_32_bits(fw_mem->mem_mc_addr);
+	psp_load_rl_fw.fw_size = fw_mem->mem_size;
+	psp_load_rl_fw.fw_type = GFX_FW_TYPE_REG_LIST;
+
+	ret = send_psp_cmd(ras_core, GFX_CMD_ID_LOAD_IP_FW,
+			&psp_load_rl_fw, sizeof(psp_load_rl_fw), &resp);
+	if (ret || resp.status) {
+		RAS_DEV_ERR(ras_core->dev,
+			"Failed to load RAS WL. ret:%d, status:%d\n", ret, resp.status);
+		ret = -EPIPE;
+	}
+
+	mutex_unlock(&psp->ta_ctx.ta_mutex);
+	if (got_reset_lock)
+		ras_core_up_gpu_reset_lock(ras_core);
+
+	return ret;
+}
+
+static int load_ras_ta_fw(struct ras_core_context *ras_core,
+		struct ras_fw_bin *ta_bin, struct ras_ta_param *ta_param)
+{
+	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
+	struct gpu_mem_block *fw_mem = NULL;
+	struct gpu_mem_block *cmd_mem = NULL;
 	struct ras_ta_cmd *ta_cmd;
 	struct ras_ta_init_flags *ta_init_flags;
 	struct psp_gfx_cmd_load_ta  psp_load_ta_cmd;
 	struct psp_cmd_resp resp = {0};
 	struct ras_ta_image_header *fw_hdr = NULL;
+	int got_reset_lock = 0;
 	int ret;
 
-	fw_mem = ras_psp_get_gpu_mem(ras_core, GPU_MEM_TYPE_RAS_TA_FW);
-	if (!fw_mem)
+	fw_mem = ta_ctx->fw_gpu_mem;
+	cmd_mem = ta_ctx->cmd_gpu_mem;
+	if (!fw_mem || !cmd_mem)
 		return -ENOMEM;
 
-	cmd_mem = ras_psp_get_gpu_mem(ras_core, GPU_MEM_TYPE_RAS_TA_CMD);
-	if (!cmd_mem) {
-		ret = -ENOMEM;
-		goto err;
+	if (!ras_core_gpu_in_reset(ras_core)) {
+		got_reset_lock = ras_core_down_trylock_gpu_reset_lock(ras_core);
+		if (!got_reset_lock)
+			return -EACCES;
 	}
 
-	ret = ras_psp_get_ras_ta_init_param(ras_core, &ta_ctx->init_param);
-	if (ret)
-		goto err;
-
-	if (!ras_core_down_trylock_gpu_reset_lock(ras_core)) {
-		ret = -EACCES;
-		goto err;
-	}
+	mutex_lock(&ta_ctx->ta_mutex);
 
 	/* copy ras ta binary to shared gpu memory */
-	memcpy(fw_mem->mem_cpu_addr, fw_bin->bin_addr, fw_bin->bin_size);
-	fw_mem->mem_size = fw_bin->bin_size;
+	memcpy(fw_mem->mem_cpu_addr, ta_bin->bin_addr, ta_bin->bin_size);
+	fw_mem->mem_size = ta_bin->bin_size;
 
 	/* Initialize ras ta startup parameter */
 	ta_cmd = (struct ras_ta_cmd *)cmd_mem->mem_cpu_addr;
 	ta_init_flags = &ta_cmd->ras_in_message.init_flags;
 
-	ta_init_flags->poison_mode_en = ta_ctx->init_param.poison_mode_en;
-	ta_init_flags->dgpu_mode = ta_ctx->init_param.dgpu_mode;
-	ta_init_flags->xcc_mask = ta_ctx->init_param.xcc_mask;
-	ta_init_flags->channel_dis_num = ta_ctx->init_param.channel_dis_num;
-	ta_init_flags->nps_mode = ta_ctx->init_param.nps_mode;
-	ta_init_flags->active_umc_mask = ta_ctx->init_param.active_umc_mask;
-	ta_init_flags->vram_type = ta_ctx->init_param.vram_type;
+	ta_init_flags->poison_mode_en = ta_param->poison_mode_en;
+	ta_init_flags->dgpu_mode = ta_param->dgpu_mode;
+	ta_init_flags->xcc_mask = ta_param->xcc_mask;
+	ta_init_flags->channel_dis_num = ta_param->channel_dis_num;
+	ta_init_flags->nps_mode = ta_param->nps_mode;
+	ta_init_flags->active_umc_mask = ta_param->active_umc_mask;
+	ta_init_flags->vram_type = ta_param->vram_type;
+	ta_init_flags->ext_umc_mask = ta_param->ext_umc_mask;
 
 	/* Setup load ras ta command */
 	memset(&psp_load_ta_cmd, 0, sizeof(psp_load_ta_cmd));
@@ -526,7 +644,7 @@ static int send_load_ta_fw_cmd(struct ras_core_context *ras_core,
 			&psp_load_ta_cmd, sizeof(psp_load_ta_cmd), &resp);
 	if (!ret && !resp.status) {
 		/* Read TA version at FW offset 0x60 if TA version not found*/
-		fw_hdr = (struct ras_ta_image_header *)fw_bin->bin_addr;
+		fw_hdr = (struct ras_ta_image_header *)ta_bin->bin_addr;
 		RAS_DEV_INFO(ras_core->dev, "PSP: RAS TA(version:%X.%X.%X.%X) is loaded.\n",
 			(fw_hdr->image_version >> 24) & 0xFF, (fw_hdr->image_version >> 16) & 0xFF,
 			(fw_hdr->image_version >> 8) & 0xFF, fw_hdr->image_version & 0xFF);
@@ -536,46 +654,27 @@ static int send_load_ta_fw_cmd(struct ras_core_context *ras_core,
 	} else {
 		RAS_DEV_ERR(ras_core->dev,
 			"Failed to load RAS TA! ret:%d, status:%d\n", ret, resp.status);
+		ret = -EPIPE;
 	}
 
-	ras_core_up_gpu_reset_lock(ras_core);
-
-err:
-	ras_psp_put_gpu_mem(ras_core, fw_mem);
-	ras_psp_put_gpu_mem(ras_core, cmd_mem);
-	return ret;
-}
-
-static int load_ras_ta_firmware(struct ras_core_context *ras_core,
-		struct ras_psp_ta_load *ras_ta_load)
-{
-	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
-	struct ras_ta_fw_bin  *fw_bin = &ta_ctx->fw_bin;
-	int ret;
-
-	fw_bin->bin_addr = ras_ta_load->bin_addr;
-	fw_bin->bin_size = ras_ta_load->bin_size;
-	fw_bin->fw_version = ras_ta_load->fw_version;
-	fw_bin->feature_version = ras_ta_load->feature_version;
-
-	ret = send_load_ta_fw_cmd(ras_core, ta_ctx);
-	if (!ret) {
-		ras_ta_load->out_session_id = ta_ctx->session_id;
-		ras_ta_load->out_loaded_ta_version = ta_ctx->ta_version;
-	}
+	mutex_unlock(&ta_ctx->ta_mutex);
+	if (got_reset_lock)
+		ras_core_up_gpu_reset_lock(ras_core);
 
 	return ret;
 }
 
-static int unload_ras_ta_firmware(struct ras_core_context *ras_core,
+static int unload_ras_ta_fw(struct ras_core_context *ras_core,
 		struct ras_psp_ta_unload *ras_ta_unload)
 {
 	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
 	struct psp_gfx_cmd_unload_ta  cmd_unload_ta = {0};
 	struct psp_cmd_resp resp = {0};
+	int got_reset_lock = 0;
 	int ret;
 
-	if (!ras_core_down_trylock_gpu_reset_lock(ras_core))
+	got_reset_lock = ras_core_down_trylock_gpu_reset_lock(ras_core);
+	if (!got_reset_lock)
 		return -EACCES;
 
 	cmd_unload_ta.session_id = ta_ctx->session_id;
@@ -588,54 +687,133 @@ static int unload_ras_ta_firmware(struct ras_core_context *ras_core,
 		goto unlock;
 	}
 
-	kfree(ta_ctx->fw_bin.bin_addr);
-	memset(&ta_ctx->fw_bin, 0, sizeof(ta_ctx->fw_bin));
+	RAS_DEV_INFO(ras_core->dev,
+		     "Successfully to unload RAS TA! ret:%d, status:%u\n",
+			ret, resp.status);
+
 	ta_ctx->ta_version = 0;
 	ta_ctx->ras_ta_initialized = false;
 	ta_ctx->session_id = 0;
 
 unlock:
-	ras_core_up_gpu_reset_lock(ras_core);
+	if (got_reset_lock)
+		ras_core_up_gpu_reset_lock(ras_core);
 
 	return ret;
 }
 
-int ras_psp_load_firmware(struct ras_core_context *ras_core,
-	struct ras_psp_ta_load *ras_ta_load)
+static int load_ras_all_fw(struct ras_core_context *ras_core)
 {
-	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
-	struct ras_psp_ta_unload ras_ta_unload = {0};
+	struct ras_param ras_param = {0};
 	int ret;
 
-	if (ta_ctx->preload_ras_ta_enabled)
-		return 0;
+	ret = ras_psp_get_ras_param(ras_core, &ras_param);
+	if (ret)
+		return ret;
+
+	ret = load_ras_rl_fw(ras_core, &ras_param.fw_param.rl_bin);
+	if (ret)
+		return ret;
+
+	return load_ras_ta_fw(ras_core,
+			&ras_param.fw_param.ta_bin, &ras_param.ta_param);
+}
+
+static int unload_ras_all_fw(struct ras_core_context *ras_core)
+{
+	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
+	struct ras_psp_ta_unload ta_unload = {0};
+
+	if (ta_ctx->ras_ta_initialized) {
+		ta_unload.ras_session_id = ta_ctx->session_id;
+		unload_ras_ta_fw(ras_core, &ta_unload);
+		ta_ctx->ras_ta_initialized = false;
+		ta_ctx->session_id = 0;
+		ta_ctx->ta_version = 0;
+	}
+
+	return 0;
+}
 
-	if (!ras_ta_load)
+int ras_psp_sideload_ras_ta(struct ras_core_context *ras_core,
+		struct ras_psp_ta_load *ta_load)
+{
+	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
+	struct ras_psp_ta_unload ras_ta_unload = {0};
+	struct ras_param ras_param = {0};
+	struct ras_fw_bin ta_bin = {0};
+	int ret = 0;
+
+	if (!ta_load || !ta_load->bin_addr || !ta_load->bin_size)
 		return -EINVAL;
 
 	if (ta_ctx->ras_ta_initialized) {
 		ras_ta_unload.ras_session_id = ta_ctx->session_id;
-		ret = unload_ras_ta_firmware(ras_core, &ras_ta_unload);
+		ret = unload_ras_ta_fw(ras_core, &ras_ta_unload);
 		if (ret)
 			return ret;
 	}
 
-	return load_ras_ta_firmware(ras_core, ras_ta_load);
+	ret = ras_psp_get_ras_param(ras_core, &ras_param);
+	if (ret)
+		return ret;
+
+	ta_bin.bin_addr = ta_load->bin_addr;
+	ta_bin.bin_size = ta_load->bin_size;
+	ta_bin.fw_version = ta_load->fw_version;
+	ta_bin.feature_version = ta_load->feature_version;
+
+	ret = load_ras_ta_fw(ras_core, &ta_bin, &ras_param.ta_param);
+	if (ret)
+		return ret;
+
+	ta_load->out_session_id = ta_ctx->session_id;
+	ta_load->out_loaded_ta_version = ta_ctx->ta_version;
+
+	return 0;
 }
 
-int ras_psp_unload_firmware(struct ras_core_context *ras_core,
+int ras_psp_unsideload_ras_ta(struct ras_core_context *ras_core,
 	struct ras_psp_ta_unload *ras_ta_unload)
 {
 	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
 
-	if (ta_ctx->preload_ras_ta_enabled)
-		return 0;
-
 	if ((!ras_ta_unload) ||
 	    (ras_ta_unload->ras_session_id != ta_ctx->session_id))
 		return -EINVAL;
 
-	return unload_ras_ta_firmware(ras_core, ras_ta_unload);
+	return unload_ras_ta_fw(ras_core, ras_ta_unload);
+}
+
+int ras_psp_reload_firmwares(struct ras_core_context *ras_core,
+		uint32_t flags)
+{
+	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
+
+	ta_ctx->ras_ta_initialized = false;
+	ta_ctx->session_id = 0;
+	ta_ctx->ta_version = 0;
+
+	return load_ras_all_fw(ras_core);
+}
+
+int ras_psp_enable_features(struct ras_core_context *ras_core,
+	struct ras_ta_enable_features_input *info, bool enable)
+{
+	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
+
+	if (!info)
+		return -EINVAL;
+
+	if (!ta_ctx->ras_ta_initialized) {
+		RAS_DEV_ERR(ras_core->dev, "RAS: ras firmware not initialized!");
+		return -ENOEXEC;
+	}
+
+	return send_ras_ta_runtime_cmd(ras_core,
+			enable ? RAS_TA_CMD_ID__ENABLE_FEATURES :
+				 RAS_TA_CMD_ID__DISABLE_FEATURES,
+			info, sizeof(*info), NULL, 0);
 }
 
 int ras_psp_trigger_error(struct ras_core_context *ras_core,
@@ -643,7 +821,7 @@ int ras_psp_trigger_error(struct ras_core_context *ras_core,
 {
 	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
 
-	if (!ta_ctx->preload_ras_ta_enabled && !ta_ctx->ras_ta_initialized) {
+	if (!ta_ctx->ras_ta_initialized) {
 		RAS_DEV_ERR(ras_core->dev, "RAS: ras firmware not initialized!");
 		return -ENOEXEC;
 	}
@@ -660,8 +838,7 @@ int ras_psp_query_address(struct ras_core_context *ras_core,
 {
 	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
 
-	if (!ta_ctx->preload_ras_ta_enabled &&
-	    !ta_ctx->ras_ta_initialized) {
+	if (!ta_ctx->ras_ta_initialized) {
 		RAS_DEV_ERR(ras_core->dev, "RAS: ras firmware not initialized!");
 		return -ENOEXEC;
 	}
@@ -705,7 +882,10 @@ int ras_psp_sw_fini(struct ras_core_context *ras_core)
 
 int ras_psp_hw_init(struct ras_core_context *ras_core)
 {
+	int ret = 0;
 	struct ras_psp *psp = &ras_core->ras_psp;
+	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
+	struct ras_psp_sys_status sys_status = {0};
 
 	psp->psp_ip_version = ras_core->config->psp_ip_version;
 
@@ -713,16 +893,47 @@ int ras_psp_hw_init(struct ras_core_context *ras_core)
 	if (!psp->ip_func)
 		return -EINVAL;
 
+	if (psp->ip_func->get_ras_block_maps &&
+		psp->ip_func->get_ras_block_maps(ras_core,
+			&psp->blk_maps, &psp->maps_size))
+		return -EINVAL;
+
+	if (psp->ip_func->get_ras_hw_caps &&
+		psp->ip_func->get_ras_hw_caps(ras_core, &psp->ras_hw_caps))
+		return -EINVAL;
+
 	/* After GPU reset, the system RAS PSP status may change.
 	 * therefore, it is necessary to synchronize the system status again.
 	 */
-	ras_psp_sync_system_ras_psp_status(ras_core);
+	ret = ras_psp_get_system_ras_status(ras_core, &sys_status);
+	if (ret)
+		return ret;
 
-	return 0;
+	if (sys_status.uniras_load_fw) {
+		ta_ctx->ras_ta_initialized = false;
+		psp->load_ras_fw_internal = true;
+	} else {
+		ta_ctx->ras_ta_initialized = true;
+		psp->load_ras_fw_internal = false;
+	}
+
+	psp->use_dedicated_memory = sys_status.use_dedicated_memory;
+	psp->psp_ctx.external_mutex = sys_status.psp_cmd_mutex;
+
+	ret = __ras_psp_mem_init(ras_core);
+	if (ret)
+		return ret;
+
+	if (psp->load_ras_fw_internal)
+		ret = load_ras_all_fw(ras_core);
+
+	return ret;
 }
 
 int ras_psp_hw_fini(struct ras_core_context *ras_core)
 {
+	unload_ras_all_fw(ras_core);
+	__ras_psp_mem_fini(ras_core);
 	return 0;
 }
 
@@ -732,16 +943,12 @@ bool ras_psp_check_supported_cmd(struct ras_core_context *ras_core,
 	struct ras_ta_ctx *ta_ctx = &ras_core->ras_psp.ta_ctx;
 	bool ret = false;
 
-	if (!ta_ctx->preload_ras_ta_enabled && !ta_ctx->ras_ta_initialized)
+	if (!ta_ctx->ras_ta_initialized)
 		return false;
 
 	switch (cmd_id) {
 	case RAS_TA_CMD_ID__QUERY_ADDRESS:
-		/* Currently, querying the address from RAS TA is only supported
-		 * when the RAS TA firmware is loaded during driver installation.
-		 */
-		if (ta_ctx->preload_ras_ta_enabled)
-			ret = true;
+		ret = true;
 		break;
 	case RAS_TA_CMD_ID__TRIGGER_ERROR:
 		ret = true;
@@ -753,3 +960,61 @@ bool ras_psp_check_supported_cmd(struct ras_core_context *ras_core,
 
 	return ret;
 }
+
+int ras_psp_get_block_ta_id(struct ras_core_context *ras_core,
+		uint32_t ras_id, uint32_t *ta_id)
+{
+	struct ras_psp *psp = &ras_core->ras_psp;
+	int i;
+
+	if (!ta_id || !psp->blk_maps || !psp->maps_size) {
+		RAS_DEV_ERR(ras_core->dev, "Invalid ras block parameter\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < psp->maps_size; i++) {
+		if (psp->blk_maps[i].ras_id == ras_id) {
+			*ta_id = psp->blk_maps[i].ta_id;
+			return 0;
+		}
+	}
+
+	RAS_DEV_WARN(ras_core->dev, "Ras block %u is not supported\n", ras_id);
+
+	return -RAS_CORE_NOT_SUPPORTED;
+}
+
+bool ras_psp_poison_supported(struct ras_core_context *ras_core)
+{
+	struct ras_psp *psp = &ras_core->ras_psp;
+
+	return psp->ras_hw_caps.poison_supported;
+}
+
+bool ras_psp_flex_mca_enabled(struct ras_core_context *ras_core)
+{
+	struct ras_psp *psp = &ras_core->ras_psp;
+
+	return psp->ras_hw_caps.flex_mca_enabled;
+}
+
+uint64_t ras_psp_get_hw_ras_caps(struct ras_core_context *ras_core)
+{
+	struct ras_psp *psp = &ras_core->ras_psp;
+
+	return psp->ras_hw_caps.features.block_mask;
+}
+
+int ras_psp_translate_addr(struct ras_core_context *ras_core,
+	struct ras_psp_addr_trans_in *in, struct ras_psp_addr_trans_out *out)
+{
+	struct ras_psp *psp = &ras_core->ras_psp;
+
+	if (!in || !out)
+		return -EINVAL;
+
+	if (!psp->sys_func || !psp->sys_func->psp_translate_addr)
+		return -EOPNOTSUPP;
+
+	return psp->sys_func->psp_translate_addr(ras_core, in, out);
+}
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp.h b/drivers/gpu/drm/amd/ras/core/ras_psp.h
index c899d0ac74cce9..4cc133c6a6d3bd 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_psp.h
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp.h
@@ -26,12 +26,31 @@
 #include "ras.h"
 #include "ta_if.h"
 
+#define MAX_RAS_BLOCK_MASK_BITS 56
+
 struct ras_core_context;
 struct ras_ta_trigger_error_input;
 struct ras_ta_query_address_input;
 struct ras_ta_query_address_output;
 enum ras_ta_cmd_id;
 
+struct ras_psp_addr_trans_in {
+	uint64_t mca_addr;
+	uint64_t ipid;
+	uint32_t nps;
+};
+
+struct ras_psp_addr_trans_out {
+	uint32_t channel_id;
+	uint8_t  socket_id;
+	uint8_t  mem_die_id;
+	uint8_t  dram_entity_id;
+	uint8_t  umc_inst_id;
+	uint64_t row_pa;
+	/* Bitmask of flipping bits across all bad page PAs in a row */
+	uint64_t pa_flip_mask;
+};
+
 struct ras_ta_image_header {
 	uint32_t reserved1[24];
 	uint32_t image_version; /* [0x60] Off Chip Firmware Version */
@@ -39,12 +58,12 @@ struct ras_ta_image_header {
 };
 
 struct ras_psp_sys_status {
-	bool  initialized;
-	uint32_t session_id;
 	void *psp_cmd_mutex;
+	bool use_dedicated_memory;
+	bool uniras_load_fw;
 };
 
-struct ras_ta_init_param {
+struct ras_ta_param {
 	uint8_t poison_mode_en;
 	uint8_t dgpu_mode;
 	uint16_t xcc_mask;
@@ -52,6 +71,24 @@ struct ras_ta_init_param {
 	uint8_t nps_mode;
 	uint32_t active_umc_mask;
 	uint8_t vram_type;
+	uint32_t ext_umc_mask;
+};
+
+struct ras_fw_bin {
+	uint32_t fw_version;
+	uint32_t feature_version;
+	uint32_t bin_size;
+	uint8_t *bin_addr;
+};
+
+struct ras_fw_param {
+	struct ras_fw_bin rl_bin;
+	struct ras_fw_bin ta_bin;
+};
+
+struct ras_param {
+	struct ras_fw_param  fw_param;
+	struct ras_ta_param  ta_param;
 };
 
 struct gpu_mem_block {
@@ -64,13 +101,38 @@ struct gpu_mem_block {
 	void *private;
 };
 
+struct ras_block_map {
+	uint32_t ras_id;
+	uint32_t ta_id;
+};
+
+union ras_feature {
+	struct {
+		uint64_t block_mask : MAX_RAS_BLOCK_MASK_BITS;
+		uint64_t rsv: 4;
+		uint64_t tag: 3;
+		uint64_t en : 1;
+	};
+	uint64_t value;
+};
+
+struct ras_hw_caps {
+	bool poison_supported;
+	bool flex_mca_enabled;
+	union ras_feature features;
+};
+
 struct ras_psp_ip_func {
 	uint32_t (*psp_ras_ring_wptr_get)(struct ras_core_context *ras_core);
 	int (*psp_ras_ring_wptr_set)(struct ras_core_context *ras_core, uint32_t wptr);
+	int (*get_ras_block_maps)(struct ras_core_context *ras_core,
+			struct ras_block_map **blk_maps, uint32_t *maps_size);
+	int (*get_ras_hw_caps)(struct ras_core_context *ras_core,
+			struct ras_hw_caps *ras_cap);
 };
 
 struct ras_psp_ring {
-	struct gpu_mem_block ras_ring_gpu_mem;
+	struct gpu_mem_block *ras_ring_gpu_mem;
 };
 
 struct psp_cmd_resp {
@@ -82,32 +144,28 @@ struct ras_psp_ctx {
 	void *external_mutex;
 	struct mutex internal_mutex;
 	uint64_t in_fence_value;
-	struct gpu_mem_block psp_cmd_gpu_mem;
-	struct gpu_mem_block out_fence_gpu_mem;
-};
-
-struct ras_ta_fw_bin {
-	uint32_t fw_version;
-	uint32_t feature_version;
-	uint32_t bin_size;
-	uint8_t *bin_addr;
+	struct gpu_mem_block *psp_cmd_gpu_mem;
+	struct gpu_mem_block *out_fence_gpu_mem;
 };
 
 struct ras_ta_ctx {
-	bool  preload_ras_ta_enabled;
 	bool  ras_ta_initialized;
 	uint32_t  session_id;
 	uint32_t  resp_status;
 	uint32_t  ta_version;
 	struct mutex ta_mutex;
-	struct ras_ta_fw_bin fw_bin;
-	struct ras_ta_init_param init_param;
-	struct gpu_mem_block fw_gpu_mem;
-	struct gpu_mem_block cmd_gpu_mem;
+	struct ras_fw_bin fw_bin;
+	struct gpu_mem_block *fw_gpu_mem;
+	struct gpu_mem_block *cmd_gpu_mem;
 };
 
 struct ras_psp {
+	bool use_dedicated_memory;
+	bool  load_ras_fw_internal;
 	uint32_t psp_ip_version;
+	struct ras_block_map *blk_maps;
+	uint32_t maps_size;
+	struct ras_hw_caps ras_hw_caps;
 	struct ras_psp_ring psp_ring;
 	struct ras_psp_ctx  psp_ctx;
 	struct ras_ta_ctx   ta_ctx;
@@ -132,10 +190,14 @@ int ras_psp_sw_init(struct ras_core_context *ras_core);
 int ras_psp_sw_fini(struct ras_core_context *ras_core);
 int ras_psp_hw_init(struct ras_core_context *ras_core);
 int ras_psp_hw_fini(struct ras_core_context *ras_core);
-int ras_psp_load_firmware(struct ras_core_context *ras_core,
-		struct ras_psp_ta_load *ras_ta_load);
-int ras_psp_unload_firmware(struct ras_core_context *ras_core,
+
+int ras_psp_reload_firmwares(struct ras_core_context *ras_core, uint32_t flags);
+int ras_psp_sideload_ras_ta(struct ras_core_context *ras_core,
+		struct ras_psp_ta_load *ta_load);
+int ras_psp_unsideload_ras_ta(struct ras_core_context *ras_core,
 		struct ras_psp_ta_unload *ras_ta_unload);
+int ras_psp_enable_features(struct ras_core_context *ras_core,
+	struct ras_ta_enable_features_input *info, bool enable);
 int ras_psp_trigger_error(struct ras_core_context *ras_core,
 	struct ras_ta_trigger_error_input *info, uint32_t instance_mask);
 int ras_psp_query_address(struct ras_core_context *ras_core,
@@ -143,4 +205,11 @@ int ras_psp_query_address(struct ras_core_context *ras_core,
 		struct ras_ta_query_address_output *addr_out);
 bool ras_psp_check_supported_cmd(struct ras_core_context *ras_core,
 		enum ras_ta_cmd_id cmd_id);
+int ras_psp_get_block_ta_id(struct ras_core_context *ras_core,
+		uint32_t ras_id, uint32_t *ta_id);
+bool ras_psp_poison_supported(struct ras_core_context *ras_core);
+bool ras_psp_flex_mca_enabled(struct ras_core_context *ras_core);
+uint64_t ras_psp_get_hw_ras_caps(struct ras_core_context *ras_core);
+int ras_psp_translate_addr(struct ras_core_context *ras_core,
+	struct ras_psp_addr_trans_in *in, struct ras_psp_addr_trans_out *out);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c b/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c
index 626cf39b75acf0..49ba57f58878c3 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp_v13_0.c
@@ -27,6 +27,31 @@
 
 #define regMP0_SMN_C2PMSG_67                           0x0083
 #define regMP0_SMN_C2PMSG_67_BASE_IDX                  0
+#define regMP0_SMN_C2PMSG_127                          0x00bf
+#define regMP0_SMN_C2PMSG_127_BASE_IDX                 0
+
+static struct ras_block_map   ras_block_maps_v13_0[] = {
+	{RAS_BLOCK_ID__UMC,       RAS_TA_BLOCK__UMC},
+	{RAS_BLOCK_ID__SDMA,      RAS_TA_BLOCK__SDMA},
+	{RAS_BLOCK_ID__GFX,       RAS_TA_BLOCK__GFX},
+	{RAS_BLOCK_ID__MMHUB,     RAS_TA_BLOCK__MMHUB},
+	{RAS_BLOCK_ID__ATHUB,     RAS_TA_BLOCK__ATHUB},
+	{RAS_BLOCK_ID__PCIE_BIF,  RAS_TA_BLOCK__PCIE_BIF},
+	{RAS_BLOCK_ID__HDP,       RAS_TA_BLOCK__HDP},
+	{RAS_BLOCK_ID__XGMI_WAFL, RAS_TA_BLOCK__XGMI_WAFL},
+	{RAS_BLOCK_ID__DF,        RAS_TA_BLOCK__DF},
+	{RAS_BLOCK_ID__SMN,       RAS_TA_BLOCK__SMN},
+	{RAS_BLOCK_ID__SEM,       RAS_TA_BLOCK__SEM},
+	{RAS_BLOCK_ID__MP0,       RAS_TA_BLOCK__MP0},
+	{RAS_BLOCK_ID__MP1,       RAS_TA_BLOCK__MP1},
+	{RAS_BLOCK_ID__FUSE,      RAS_TA_BLOCK__FUSE},
+	{RAS_BLOCK_ID__MCA,       RAS_TA_BLOCK__MCA},
+	{RAS_BLOCK_ID__VCN,       RAS_TA_BLOCK__VCN},
+	{RAS_BLOCK_ID__JPEG,      RAS_TA_BLOCK__JPEG},
+	{RAS_BLOCK_ID__IH,        RAS_TA_BLOCK__IH},
+	{RAS_BLOCK_ID__MPIO,      RAS_TA_BLOCK__MPIO},
+	{RAS_BLOCK_ID__MMSCH,     RAS_TA_BLOCK__MMSCH},
+};
 
 static uint32_t ras_psp_v13_0_ring_wptr_get(struct ras_core_context *ras_core)
 {
@@ -40,7 +65,58 @@ static int ras_psp_v13_0_ring_wptr_set(struct ras_core_context *ras_core, uint32
 	return 0;
 }
 
+static int ras_psp_v13_0_get_ras_block_maps(struct ras_core_context *ras_core,
+			struct ras_block_map **blk_maps, uint32_t *maps_size)
+{
+	if (!blk_maps || !maps_size)
+		return -EINVAL;
+
+	*blk_maps = ras_block_maps_v13_0;
+	*maps_size = ARRAY_SIZE(ras_block_maps_v13_0);
+
+	return 0;
+}
+
+static int ras_psp_v13_0_get_ras_hw_caps(struct ras_core_context *ras_core,
+				struct ras_hw_caps *ras_cap)
+{
+	uint32_t ras_hw_cap, bit;
+	uint32_t i;
+
+	if (!ras_cap)
+		return -EINVAL;
+
+	ras_hw_cap = RAS_DEV_RREG32_SOC15(ras_core->dev, MP0, 0, regMP0_SMN_C2PMSG_127);
+
+	ras_cap->poison_supported = true;
+	ras_cap->flex_mca_enabled = false;
+
+	ras_cap->features.value = 0;
+	for (bit = 0; bit <= 29; bit++) {
+		if (!(ras_hw_cap & BIT_ULL(bit)))
+			continue;
+
+		for (i = 0; i < ARRAY_SIZE(ras_block_maps_v13_0); i++) {
+			if (ras_block_maps_v13_0[i].ta_id == bit) {
+				if (ras_block_maps_v13_0[i].ras_id < MAX_SUPPORTED_RAS_BLOCK_ID)
+					ras_cap->features.block_mask |=
+						BIT_ULL(ras_block_maps_v13_0[i].ras_id);
+				else
+					RAS_DEV_WARN(ras_core->dev,
+						"RAS block maps v13 error: invalid ras block id %u\n",
+						ras_block_maps_v13_0[i].ras_id);
+
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
 const struct ras_psp_ip_func ras_psp_v13_0 = {
 	.psp_ras_ring_wptr_get = ras_psp_v13_0_ring_wptr_get,
 	.psp_ras_ring_wptr_set = ras_psp_v13_0_ring_wptr_set,
+	.get_ras_block_maps = ras_psp_v13_0_get_ras_block_maps,
+	.get_ras_hw_caps = ras_psp_v13_0_get_ras_hw_caps,
 };
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.c b/drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.c
new file mode 100644
index 00000000000000..e3cae8c94b28f8
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "ras.h"
+#include "ta_if.h"
+#include "ras_psp_v15_0.h"
+
+#define regMPASP_SMN_C2PMSG_67                           0x0083
+#define regMPASP_SMN_C2PMSG_67_BASE_IDX                  1
+#define regMPASP_SMN_C2PMSG_127                          0x00bf
+#define regMPASP_SMN_C2PMSG_127_BASE_IDX                 1
+
+static struct ras_block_map   ras_block_maps_v15_0[] = {
+	{RAS_BLOCK_ID__UMC,         RAS_TA_MPASP_BLOCK__UMC},
+	{RAS_BLOCK_ID__SDMA,        RAS_TA_MPASP_BLOCK__SDMA},
+	{RAS_BLOCK_ID__GFX,         RAS_TA_MPASP_BLOCK__GFX},
+	{RAS_BLOCK_ID__MMHUB,       RAS_TA_MPASP_BLOCK__MMHUB},
+	{RAS_BLOCK_ID__ATHUB,       RAS_TA_MPASP_BLOCK__ATHUB},
+	{RAS_BLOCK_ID__PCIE_BIF,    RAS_TA_MPASP_BLOCK__PCIE},
+	{RAS_BLOCK_ID__SMN,         RAS_TA_MPASP_BLOCK__SMN},
+	{RAS_BLOCK_ID__MP0,         RAS_TA_MPASP_BLOCK__MP0},
+	{RAS_BLOCK_ID__MP1,         RAS_TA_MPASP_BLOCK__MP1},
+	{RAS_BLOCK_ID__VCN,         RAS_TA_MPASP_BLOCK__VCN},
+	{RAS_BLOCK_ID__IH,          RAS_TA_MPASP_BLOCK__IH},
+	{RAS_BLOCK_ID__MP5,         RAS_TA_MPASP_BLOCK__MP5},
+	{RAS_BLOCK_ID__ATU,         RAS_TA_MPASP_BLOCK__ATU},
+	{RAS_BLOCK_ID__DACC_BE,     RAS_TA_MPASP_BLOCK__DACC_BE},
+	{RAS_BLOCK_ID__ECLR,        RAS_TA_MPASP_BLOCK__ECLR},
+	{RAS_BLOCK_ID__KPX_SERDES,  RAS_TA_MPASP_BLOCK__KPX_SERDES},
+	{RAS_BLOCK_ID__LSDMA,       RAS_TA_MPASP_BLOCK__LSDMA},
+	{RAS_BLOCK_ID__MPART,       RAS_TA_MPASP_BLOCK__MPART},
+	{RAS_BLOCK_ID__MPIFOE,      RAS_TA_MPASP_BLOCK__MPIFOE},
+	{RAS_BLOCK_ID__MPRAS,       RAS_TA_MPASP_BLOCK__MPRAS},
+	{RAS_BLOCK_ID__NBIF,        RAS_TA_MPASP_BLOCK__NBIF},
+	{RAS_BLOCK_ID__NBIO,        RAS_TA_MPASP_BLOCK__NBIO},
+	{RAS_BLOCK_ID__OXRP,        RAS_TA_MPASP_BLOCK__OXRP},
+	{RAS_BLOCK_ID__PCIE_PL,     RAS_TA_MPASP_BLOCK__PCIE_PL},
+	{RAS_BLOCK_ID__PCS_XGMI,    RAS_TA_MPASP_BLOCK__PCS_XGMI},
+	{RAS_BLOCK_ID__PIE,         RAS_TA_MPASP_BLOCK__PIE},
+	{RAS_BLOCK_ID__CS,          RAS_TA_MPASP_BLOCK__CS},
+	{RAS_BLOCK_ID__SHUB,        RAS_TA_MPASP_BLOCK__SHUB},
+	{RAS_BLOCK_ID__SSBDCI,      RAS_TA_MPASP_BLOCK__SSBDCI},
+	{RAS_BLOCK_ID__UCIE_PCS,    RAS_TA_MPASP_BLOCK__UCIE_PCS},
+};
+
+static uint32_t ras_psp_v15_0_ring_wptr_get(struct ras_core_context *ras_core)
+{
+	return RAS_DEV_RREG32_SOC15(ras_core->dev, MP0, 0, regMPASP_SMN_C2PMSG_67);
+}
+
+static int ras_psp_v15_0_ring_wptr_set(struct ras_core_context *ras_core, uint32_t value)
+{
+	RAS_DEV_WREG32_SOC15(ras_core->dev, MP0, 0, regMPASP_SMN_C2PMSG_67, value);
+
+	return 0;
+}
+
+static int ras_psp_v15_0_get_ras_block_maps(struct ras_core_context *ras_core,
+			struct ras_block_map **blk_maps, uint32_t *maps_size)
+{
+	if (!blk_maps || !maps_size)
+		return -EINVAL;
+
+	*blk_maps = ras_block_maps_v15_0;
+	*maps_size = ARRAY_SIZE(ras_block_maps_v15_0);
+
+	return 0;
+}
+
+static int ras_psp_v15_0_get_ras_hw_caps(struct ras_core_context *ras_core,
+				struct ras_hw_caps *ras_cap)
+{
+	uint32_t ras_hw_cap, bit;
+	uint32_t i;
+
+	if (!ras_cap)
+		return -EINVAL;
+
+	ras_hw_cap = RAS_DEV_RREG32_SOC15(ras_core->dev,
+				MP0, 0, regMPASP_SMN_C2PMSG_127);
+
+	ras_cap->poison_supported = (ras_hw_cap & BIT_ULL(31)) ? true : false;
+	ras_cap->flex_mca_enabled = (ras_hw_cap & BIT_ULL(30)) ? true : false;
+
+	ras_cap->features.value = 0;
+	for (bit = 0; bit <= 29; bit++) {
+		if (!(ras_hw_cap & BIT_ULL(bit)))
+			continue;
+
+		for (i = 0; i < ARRAY_SIZE(ras_block_maps_v15_0); i++) {
+			if (ras_block_maps_v15_0[i].ta_id == bit) {
+				if (ras_block_maps_v15_0[i].ras_id < MAX_SUPPORTED_RAS_BLOCK_ID)
+					ras_cap->features.block_mask |=
+						BIT_ULL(ras_block_maps_v15_0[i].ras_id);
+				else
+					RAS_DEV_WARN(ras_core->dev,
+						"RAS block maps v15 error: invalid ras block id %u\n",
+						ras_block_maps_v15_0[i].ras_id);
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
+const struct ras_psp_ip_func ras_psp_v15_0 = {
+	.psp_ras_ring_wptr_get = ras_psp_v15_0_ring_wptr_get,
+	.psp_ras_ring_wptr_set = ras_psp_v15_0_ring_wptr_set,
+	.get_ras_block_maps = ras_psp_v15_0_get_ras_block_maps,
+	.get_ras_hw_caps = ras_psp_v15_0_get_ras_hw_caps,
+};
diff --git a/drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.h b/drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.h
new file mode 100644
index 00000000000000..4149fda38a3749
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_psp_v15_0.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __RAS_PSP_V15_0_H__
+#define __RAS_PSP_V15_0_H__
+#include "ras_psp.h"
+
+extern const struct ras_psp_ip_func ras_psp_v15_0;
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc.c b/drivers/gpu/drm/amd/ras/core/ras_umc.c
index 95a98b411fc79f..b3f30da19a78f7 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc.c
@@ -24,6 +24,7 @@
 #include "ras.h"
 #include "ras_umc.h"
 #include "ras_umc_v12_0.h"
+#include "ras_umc_v15_0.h"
 
 #define MAX_ECC_NUM_PER_RETIREMENT  16
 
@@ -69,10 +70,16 @@ static uint64_t ras_umc_get_eeprom_timestamp(struct ras_core_context *ras_core)
 static const struct ras_umc_ip_func *ras_umc_get_ip_func(
 				struct ras_core_context *ras_core, uint32_t ip_version)
 {
+	struct ras_umc *ras_umc = &ras_core->ras_umc;
+
 	switch (ip_version) {
 	case IP_VERSION(12, 0, 0):
 	case IP_VERSION(12, 5, 0):
+		ras_umc->max_pages_per_row = 16;
 		return &ras_umc_func_v12_0;
+	case IP_VERSION(15, 0, 0):
+		ras_umc->max_pages_per_row = 128;
+		return &ras_umc_func_v15_0;
 	default:
 		RAS_DEV_ERR(ras_core->dev,
 			"UMC ip version(0x%x) is not supported!\n", ip_version);
@@ -82,7 +89,7 @@ static const struct ras_umc_ip_func *ras_umc_get_ip_func(
 	return NULL;
 }
 
-int ras_umc_psp_ma2pa(struct ras_core_context *ras_core,
+int ras_umc_ras_ta_translate_addr(struct ras_core_context *ras_core,
 		struct umc_mca_addr *in, struct umc_phy_addr *out,
 		uint32_t nps)
 {
@@ -121,6 +128,28 @@ int ras_umc_psp_ma2pa(struct ras_core_context *ras_core,
 	return 0;
 }
 
+int ras_umc_psp_translate_addr(struct ras_core_context *ras_core,
+		struct umc_mca_addr *in, struct umc_phy_addr *out,
+		uint32_t nps)
+{
+	struct ras_psp_addr_trans_in psp_in = {0};
+	struct ras_psp_addr_trans_out psp_out = {0};
+	int ret;
+
+	psp_in.mca_addr = in->mca_addr;
+	psp_in.ipid = in->ipid;
+	psp_in.nps = nps;
+
+	ret = ras_psp_translate_addr(ras_core, &psp_in, &psp_out);
+	if (ret)
+		return ret;
+
+	out->pa = psp_out.row_pa;
+	out->pa_flip_mask = psp_out.pa_flip_mask;
+
+	return 0;
+}
+
 static int ras_umc_log_ecc(struct ras_core_context *ras_core,
 		unsigned long idx, void *data)
 {
@@ -129,8 +158,6 @@ static int ras_umc_log_ecc(struct ras_core_context *ras_core,
 
 	mutex_lock(&ras_umc->tree_lock);
 	ret = radix_tree_insert(&ras_umc->root, idx, data);
-	if (!ret)
-		radix_tree_tag_set(&ras_umc->root, idx, UMC_ECC_NEW_DETECTED_TAG);
 	mutex_unlock(&ras_umc->tree_lock);
 
 	return ret;
@@ -154,42 +181,119 @@ int ras_umc_clear_logged_ecc(struct ras_core_context *ras_core)
 	return 0;
 }
 
-int ras_umc_convert_record_to_nps_pages(struct ras_core_context *ras_core,
-		struct eeprom_umc_record *record, uint32_t nps,
-		uint64_t *page_pfn, uint32_t max_pages)
+int ras_umc_alloc_row_pages(struct ras_core_context *ras_core,
+		uint64_t **page_pfns, uint32_t *nr_page_pfns)
 {
-	int count = 0;
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
+	uint64_t *address;
+	uint64_t page_num;
 
-	if (!page_pfn || !max_pages)
+	if (!page_pfns || !nr_page_pfns)
 		return -EINVAL;
 
-	if (ras_umc->ip_func && ras_umc->ip_func->eeprom_record_to_nps_pages)
-		count = ras_umc->ip_func->eeprom_record_to_nps_pages(ras_core,
-					record, nps, page_pfn, max_pages);
+	if (!ras_umc->max_pages_per_row) {
+		RAS_DEV_ERR(ras_core->dev, "max_pages_per_row was not initialized!\n");
+		return -EPERM;
+	}
+
+	page_num = ras_umc->max_pages_per_row;
+
+	address = kcalloc(page_num, sizeof(*address), GFP_KERNEL);
+	if (!address)
+		return -ENOMEM;
+
+	*page_pfns = address;
+	*nr_page_pfns = page_num;
+
+	return 0;
+}
+
+int ras_umc_free_row_pages(struct ras_core_context *ras_core,
+		uint64_t *page_pfns)
+{
+	if (!page_pfns)
+		return -EINVAL;
+
+	kfree(page_pfns);
+
+	return 0;
+}
+
+static int ras_umc_expand_row_pages(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *page_pfns, uint32_t nr_page_pfns)
+{
+	uint64_t retired_addr = RAS_PFN_TO_ADDR(record->cur_nps_retired_row_pfn);
+	uint64_t flip_mask = record->cur_nps_pa_flip_mask;
+	uint64_t subset;
+	uint64_t row_pa, addr;
+	uint32_t count = 0;
+
+	if (!retired_addr || !flip_mask || !page_pfns || !nr_page_pfns)
+		return -ENOEXEC;
+
+	row_pa = retired_addr & ~(flip_mask);
+
+	if ((count < nr_page_pfns) &&
+	    !ras_core_check_address_sanity(ras_core, row_pa))
+		page_pfns[count++] = RAS_ADDR_TO_PFN(row_pa);
+
+	subset = flip_mask;
+	while (subset) {
+		addr = row_pa ^ subset;
+
+		if (count >= nr_page_pfns)
+			break;
+
+		if (!ras_core_check_address_sanity(ras_core, addr))
+			page_pfns[count++] = RAS_ADDR_TO_PFN(addr);
+
+		subset = (subset - 1) & flip_mask;
+	};
 
 	return count;
 }
 
-static void ras_umc_reserve_eeprom_record(struct ras_core_context *ras_core,
-				struct eeprom_umc_record *record)
+int ras_umc_convert_record_to_row_pages(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *page_pfns, uint32_t nr_page_pfns)
 {
-	uint64_t page_pfn[16];
-	int count = 0, i;
+	uint32_t new_nps;
+	int ret, count = 0;
 
-	memset(page_pfn, 0, sizeof(page_pfn));
-	count = ras_umc_convert_record_to_nps_pages(ras_core,
-					record, record->cur_nps, page_pfn, ARRAY_SIZE(page_pfn));
-	if (count <= 0) {
-		RAS_DEV_ERR(ras_core->dev,
-			"Fail to convert error address! count:%d\n", count);
-		return;
+	if (!page_pfns || !nr_page_pfns || !record ||
+	    (record->cur_nps > UMC_MEMORY_PARTITION_MODE_NPS8))
+		return -EINVAL;
+
+	if (!record->cur_nps || !record->cur_nps_retired_row_pfn ||
+	    !record->cur_nps_pa_flip_mask) {
+		new_nps = record->cur_nps ?
+			record->cur_nps : ras_core_get_curr_nps_mode(ras_core);
+		ret = ras_umc_record_to_nps_record(ras_core, record, new_nps);
+		if (ret)
+			return ret;
 	}
 
+	count = ras_umc_expand_row_pages(ras_core, record, page_pfns, nr_page_pfns);
+	if (count >= 0)
+		record->cur_nps_valid_page_num = count;
+
+	return count;
+}
+
+static void ras_umc_reserve_row_pages(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *pages, uint32_t nr_pages)
+{
+	struct ras_umc *ras_umc = &ras_core->ras_umc;
+	int i;
+
+	if (!pages || !nr_pages ||
+		(nr_pages > ras_umc->max_pages_per_row))
+		return;
+
 	/* Reserve memory */
-	for (i = 0; i < count; i++)
-		ras_core_event_notify(ras_core,
-			RAS_EVENT_ID__RESERVE_BAD_PAGE, &page_pfn[i]);
+	for (i = 0; i < nr_pages; i++)
+		ras_core_event_notify(ras_core, ras_core_in_early_init(ras_core) ?
+			RAS_EVENT_ID__EARLY_INIT_RESERVE_PAGE : RAS_EVENT_ID__RESERVE_BAD_PAGE,
+			&pages[i]);
 }
 
 /* When gpu reset is ongoing, ecc logging operations will be pended.
@@ -268,39 +372,24 @@ int ras_umc_log_pending_bad_bank(struct ras_core_context *ras_core)
 int ras_umc_log_bad_bank(struct ras_core_context *ras_core, struct ras_bank_ecc *bank)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
-	struct eeprom_umc_record umc_rec;
-	struct eeprom_umc_record *err_rec;
+	struct eeprom_umc_record umc_rec = {0};
+	uint32_t c = 0;
 	int ret;
 
-	memset(&umc_rec, 0, sizeof(umc_rec));
-
 	mutex_lock(&ras_umc->bank_log_lock);
-	ret = ras_umc->ip_func->bank_to_eeprom_record(ras_core, bank, &umc_rec);
+	ret = ras_umc_bank_to_umc_record(ras_core, bank, &umc_rec);
 	if (ret)
 		goto out;
 
-	err_rec = kzalloc_obj(*err_rec);
-	if (!err_rec) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	memcpy(err_rec, &umc_rec, sizeof(umc_rec));
-	ret = ras_umc_log_ecc(ras_core, err_rec->cur_nps_retired_row_pfn, err_rec);
+	ret = ras_umc_add_bad_pages(ras_core, &umc_rec, 1, &c);
 	if (ret) {
-		if (ret == -EEXIST) {
-			RAS_DEV_INFO(ras_core->dev, "The bad pages have been logged before.\n");
-			ret = 0;
-		}
-
-		kfree(err_rec);
+		RAS_DEV_ERR(ras_core->dev, "Failed to log bad bank! ret:%x\n", ret);
 		goto out;
 	}
 
-	ras_umc_reserve_eeprom_record(ras_core, err_rec);
-
-	ret = ras_core_event_notify(ras_core,
-			RAS_EVENT_ID__BAD_PAGE_DETECTED, NULL);
+	if (c)
+		ret = ras_core_event_notify(ras_core,
+				RAS_EVENT_ID__BAD_PAGE_DETECTED, NULL);
 
 out:
 	mutex_unlock(&ras_umc->bank_log_lock);
@@ -315,7 +404,7 @@ int ras_umc_ma2pa(struct ras_core_context *ras_core,
 	int ret;
 
 	if (ras_psp_check_supported_cmd(ras_core, RAS_TA_CMD_ID__QUERY_ADDRESS)) {
-		ret = ras_umc_psp_ma2pa(ras_core, addr_in, addr_out, nps);
+		ret = ras_umc_ras_ta_translate_addr(ras_core, addr_in, addr_out, nps);
 	} else {
 		if (ras_umc->ip_func && ras_umc->ip_func->ma2pa) {
 			ret = ras_umc->ip_func->ma2pa(ras_core, addr_in, addr_out, nps);
@@ -411,8 +500,9 @@ static int ras_umc_eeprom_rec2nps_addr_legacy(struct ras_core_context *ras_core,
 	 * only ever existed on NPS1 systems)
 	 */
 	if (ras_core->ras_umc.ip_func && ras_core->ras_umc.ip_func->get_die_id) {
-		die_id = ras_core->ras_umc.ip_func->get_die_id(record->address,
-			RAS_PFN_TO_ADDR(EEPROM_RECORD_UMC_ADDR_PFN(record)));
+		die_id = ras_core->ras_umc.ip_func->get_die_id(ras_core,
+				record->address,
+				RAS_PFN_TO_ADDR(EEPROM_RECORD_UMC_ADDR_PFN(record)));
 	} else {
 		RAS_DEV_ERR(ras_core->dev, "get_die_id is not supported!\n");
 		return -EOPNOTSUPP;
@@ -434,7 +524,8 @@ static int ras_umc_eeprom_rec2nps_rec(struct ras_core_context *ras_core,
 	record->cur_nps = nps;
 
 	if (save_nps || ch_idx_v2) {
-		if ((nps == save_nps) && !ras_fw_eeprom_supported(ras_core)) {
+		if ((nps == save_nps) &&
+		    !ras_eeprom_mgr_fw_record_enabled(ras_core)) {
 			record->cur_nps_retired_row_pfn =
 				EEPROM_RECORD_UMC_ADDR_PFN(record);
 		} else {
@@ -466,59 +557,57 @@ static int ras_umc_eeprom_rec2nps_rec(struct ras_core_context *ras_core,
 	return ret;
 }
 
-static int ras_umc_get_new_records(struct ras_core_context *ras_core,
-			struct eeprom_umc_record *records, u32 num)
+static bool ras_umc_check_logged_record(struct ras_core_context *ras_core,
+			struct eeprom_umc_record *record)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
-	struct eeprom_umc_record *entries[MAX_ECC_NUM_PER_RETIREMENT];
-	u32 entry_num = num < MAX_ECC_NUM_PER_RETIREMENT ? num : MAX_ECC_NUM_PER_RETIREMENT;
-	int count = 0;
-	int new_detected, i;
+	void *res = NULL;
 
 	mutex_lock(&ras_umc->tree_lock);
-	new_detected = radix_tree_gang_lookup_tag(&ras_umc->root, (void **)entries,
-			0, entry_num, UMC_ECC_NEW_DETECTED_TAG);
-	for (i = 0; i < new_detected; i++) {
-		if (!entries[i])
-			continue;
-
-		memcpy(&records[i], entries[i], sizeof(struct eeprom_umc_record));
-		count++;
-		radix_tree_tag_clear(&ras_umc->root,
-				entries[i]->cur_nps_retired_row_pfn, UMC_ECC_NEW_DETECTED_TAG);
-	}
+	res = radix_tree_lookup(&ras_umc->root, record->cur_nps_retired_row_pfn);
 	mutex_unlock(&ras_umc->tree_lock);
 
-	return count;
+	return res ? true : false;
 }
 
 static bool ras_umc_check_retired_record(struct ras_core_context *ras_core,
-				struct eeprom_umc_record *record, bool from_eeprom)
+				struct eeprom_umc_record *record)
 {
-	struct ras_umc *ras_umc = &ras_core->ras_umc;
-	struct eeprom_store_record *data = &ras_umc->umc_err_data.rom_data;
 	uint32_t nps = 0;
-	int i, ret;
-
-	if (from_eeprom) {
-		nps = ras_umc->umc_err_data.umc_nps_mode;
-		ret = ras_umc_eeprom_rec2nps_rec(ras_core, record, nps);
-		if (ret)
-			RAS_DEV_WARN_RATELIMITED(ras_core->dev,
-				"Failed to adjust eeprom record, ret:%d", ret);
+	int ret;
 
-		return false;
+	nps = ras_core_get_curr_nps_mode(ras_core);
+	ret = ras_umc_record_to_nps_record(ras_core, record, nps);
+	if (ret) {
+		RAS_DEV_ERR(ras_core->dev, "Failed to translate nps record! ret:%d\n", ret);
+		return true;
 	}
 
-	for (i = 0; i < data->count; i++) {
-		if ((data->bps[i].retired_row_pfn == record->retired_row_pfn) &&
-		    (data->bps[i].cur_nps_retired_row_pfn == record->cur_nps_retired_row_pfn))
-			return true;
-	}
+	if (ras_umc_check_logged_record(ras_core, record))
+		return true;
 
 	return false;
 }
 
+static int ras_umc_log_record(struct ras_core_context *ras_core,
+				struct eeprom_umc_record *record)
+{
+	struct eeprom_umc_record *rec;
+	int ret;
+
+	rec = kzalloc(sizeof(*rec), GFP_KERNEL);
+	if (!rec)
+		return -ENOMEM;
+
+	memcpy(rec, record, sizeof(*rec));
+
+	ret = ras_umc_log_ecc(ras_core, rec->cur_nps_retired_row_pfn, rec);
+	if (ret)
+		kfree(rec);
+
+	return ret;
+}
+
 /* alloc/realloc bps array */
 static int ras_umc_realloc_err_data_space(struct ras_core_context *ras_core,
 		struct eeprom_store_record *data, int pages)
@@ -555,97 +644,127 @@ static int ras_umc_update_eeprom_rom_data(struct ras_core_context *ras_core,
 	memcpy(&data->bps[data->count], bps, sizeof(*data->bps));
 	data->count++;
 	data->space_left--;
+
+	/* update bad channel bitmap */
+	if (bps->mem_channel < BITS_PER_TYPE(data->umc_channel_bitmap))
+		data->umc_channel_bitmap |= 0x1ULL << bps->mem_channel;
+
 	return 0;
 }
 
 static int ras_umc_update_eeprom_ram_data(struct ras_core_context *ras_core,
-				struct eeprom_umc_record *bps)
+		struct eeprom_umc_record *bps, uint64_t *page_pfns, uint32_t nr_page_pfns)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
 	struct eeprom_store_record *data = &ras_umc->umc_err_data.ram_data;
-	uint64_t page_pfn[16];
-	int count = 0, i, j;
+	int j;
+
+	if (!bps || !page_pfns ||
+		(nr_page_pfns > ras_umc->max_pages_per_row))
+		return -EINVAL;
+
+	if (!nr_page_pfns)
+		return 0;
 
 	if (!data->space_left &&
-		ras_umc_realloc_err_data_space(ras_core, data, 256)) {
-		return	-ENOMEM;
-	}
+		ras_umc_realloc_err_data_space(ras_core, data, 256))
+		return -ENOMEM;
 
-	memset(page_pfn, 0, sizeof(page_pfn));
-	count = ras_umc_convert_record_to_nps_pages(ras_core,
-					bps, bps->cur_nps, page_pfn, ARRAY_SIZE(page_pfn));
-	if (count > 0) {
-		for (j = 0; j < count; j++) {
-			if (ras_core_check_address_sanity(ras_core,
-				page_pfn[j] << AMDGPU_GPU_PAGE_SHIFT)) {
-
-				for (i = 0; i < data->count; i++)
-					if (page_pfn[j] == data->bps[i].cur_nps_retired_row_pfn)
-						break;
-				data->bps[data->count].cur_nps_retired_row_pfn = U64_MAX;
-				data->count++;
-				data->space_left--;
-				continue;
-			}
-
-			bps->cur_nps_retired_row_pfn = page_pfn[j];
-			memcpy(&data->bps[data->count], bps, sizeof(*data->bps));
-			data->count++;
-			data->space_left--;
-			data->bad_page_num++;
-		}
-	} else {
-		RAS_DEV_ERR(ras_core->dev, "Failed to convert record to nps pages!");
-		return -EINVAL;
+	for (j = 0; j < nr_page_pfns; j++) {
+		bps->cur_nps_retired_row_pfn = page_pfns[j];
+		memcpy(&data->bps[data->count], bps, sizeof(*data->bps));
+		data->count++;
+		data->space_left--;
 	}
 
+	/* update bad channel bitmap */
+	if (bps->mem_channel < BITS_PER_TYPE(data->umc_channel_bitmap))
+		data->umc_channel_bitmap |= 0x1ULL << bps->mem_channel;
+
 	return 0;
 }
 
-static void ras_umc_update_bad_pages(struct ras_core_context *ras_core)
+void ras_umc_report_badpage_info(struct ras_core_context *ras_core)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
 	struct eeprom_store_record *data = &ras_umc->umc_err_data.ram_data;
 
-	data->bad_page_num_old = data->bad_page_num;
+	if (ras_umc->last_record_count != data->count) {
+		ras_umc->last_record_count = data->count;
+		ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_PAGE_NUM,
+			&ras_umc->last_record_count);
+	}
+
+	if (ras_umc->last_channel_bitmap != data->umc_channel_bitmap) {
+		ras_umc->last_channel_bitmap = data->umc_channel_bitmap;
+		ras_core_event_notify(ras_core, RAS_EVENT_ID__UPDATE_BAD_CHANNEL_BITMAP,
+			&ras_umc->last_channel_bitmap);
+	}
+
+	if (ras_core->is_rma)
+		ras_core_event_notify(ras_core, RAS_EVENT_ID__DEVICE_RMA, NULL);
 }
 
-/* it deal with vram only. */
-static int ras_umc_add_bad_pages(struct ras_core_context *ras_core,
-				 struct eeprom_umc_record *bps,
-				 int pages, bool from_eeprom)
+int ras_umc_add_bad_pages(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *bps, uint32_t bps_sz, uint32_t *valid_sz)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
-	struct ras_umc_err_data *data = &ras_umc->umc_err_data;
-	int i, ret = 0;
+	uint64_t *page_pfns = NULL;
+	uint32_t nr_page_pfns = 0;
+	int nr_valid_pfns = 0;
+	uint32_t i, c = 0;
+	int ret = 0;
 
-	if (!bps || pages <= 0)
-		return 0;
+	if (!bps || !bps_sz || !valid_sz)
+		return -EINVAL;
+
+	ret = ras_umc_alloc_row_pages(ras_core, &page_pfns, &nr_page_pfns);
+	if (ret)
+		return ret;
 
 	mutex_lock(&ras_umc->umc_lock);
-	for (i = 0; i < pages; i++) {
-		if (ras_umc_check_retired_record(ras_core, &bps[i], from_eeprom))
+	for (i = 0; i < bps_sz; i++) {
+		if (ras_umc_check_retired_record(ras_core, &bps[i]))
 			continue;
 
+		nr_valid_pfns = ras_umc_convert_record_to_row_pages(ras_core,
+					&bps[i], page_pfns, nr_page_pfns);
+		if (nr_valid_pfns < 0) {
+			RAS_DEV_ERR(ras_core->dev,
+				"Failed to lookup record bad pages! %d\n", nr_valid_pfns);
+			ret = nr_valid_pfns;
+			goto out;
+		}
+
 		ret = ras_umc_update_eeprom_rom_data(ras_core, &bps[i]);
 		if (ret)
 			goto out;
 
-		if (data->last_retired_pfn == bps[i].cur_nps_retired_row_pfn)
-			continue;
-
-		data->last_retired_pfn = bps[i].cur_nps_retired_row_pfn;
+		ret = ras_umc_log_record(ras_core, &bps[i]);
+		if (ret)
+			goto out;
 
-		if (from_eeprom)
-			ras_umc_reserve_eeprom_record(ras_core, &bps[i]);
+		ras_umc_reserve_row_pages(ras_core,
+			&bps[i], page_pfns, nr_valid_pfns);
 
-		ret = ras_umc_update_eeprom_ram_data(ras_core, &bps[i]);
+		ret = ras_umc_update_eeprom_ram_data(ras_core,
+				&bps[i], page_pfns, nr_valid_pfns);
 		if (ret)
 			goto out;
+		c++;
 	}
+
+	*valid_sz = c;
+
+	if (c) {
+		ras_eeprom_mgr_check_and_report_status(ras_core, true);
+		if (!ras_core_in_early_init(ras_core))
+			ras_umc_report_badpage_info(ras_core);
+	}
+
 out:
 	mutex_unlock(&ras_umc->umc_lock);
-
+	ras_umc_free_row_pages(ras_core, page_pfns);
 	return ret;
 }
 
@@ -656,42 +775,39 @@ static int ras_umc_add_bad_pages(struct ras_core_context *ras_core,
 int ras_umc_load_bad_pages(struct ras_core_context *ras_core)
 {
 	struct eeprom_umc_record *bps;
-	uint32_t ras_num_recs;
-	int ret;
+	uint32_t c = 0;
+	int ras_num_recs, ret;
 
-	if (ras_fw_eeprom_supported(ras_core)) {
-		ras_num_recs = ras_fw_eeprom_get_record_count(ras_core);
-		/* no bad page record, skip eeprom access */
-		if (!ras_num_recs ||
-		    ras_core->ras_fw_eeprom.record_threshold_config == DISABLE_RETIRE_PAGE)
-			return 0;
-	} else {
-		ras_num_recs = ras_eeprom_get_record_count(ras_core);
-		if (!ras_num_recs ||
-		    ras_core->ras_eeprom.record_threshold_config == DISABLE_RETIRE_PAGE)
-			return 0;
-	}
+	ras_num_recs = ras_eeprom_mgr_get_record_count(ras_core);
+	if (ras_num_recs <= 0)
+		return ras_num_recs;
 
 	bps = kzalloc_objs(*bps, ras_num_recs);
 	if (!bps)
 		return -ENOMEM;
 
-	if (ras_fw_eeprom_supported(ras_core))
-		ret = ras_fw_eeprom_read_idx(ras_core, bps, 0, 0, ras_num_recs);
+	ret = ras_eeprom_mgr_get_records(ras_core, 0, bps, ras_num_recs);
+	if (ret)
+		RAS_DEV_ERR(ras_core->dev,
+			"Failed to load EEPROM table records! ret:%d\n", ret);
 	else
-		ret = ras_eeprom_read(ras_core, bps, ras_num_recs);
-	if (ret) {
-		RAS_DEV_ERR(ras_core->dev, "Failed to load EEPROM table records!");
-	} else {
-		ras_core->ras_umc.umc_err_data.last_retired_pfn = UMC_INV_MEM_PFN;
-		ret = ras_umc_add_bad_pages(ras_core, bps, ras_num_recs, true);
-		ras_umc_update_bad_pages(ras_core);
-	}
+		ret = ras_umc_add_bad_pages(ras_core, bps, ras_num_recs, &c);
 
 	kfree(bps);
 	return ret;
 }
 
+static int ras_umc_count_valid_pages(struct ras_core_context *ras_core,
+		struct eeprom_umc_record *records, const u32 nr_records)
+{
+	int count = 0, i;
+
+	for (i = 0; i < nr_records; i++)
+		count += records[i].cur_nps_valid_page_num;
+
+	return count;
+}
+
 /*
  * write error record array to eeprom, the function should be
  * protected by recovery_lock
@@ -701,36 +817,37 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
 	struct eeprom_store_record *data = &ras_umc->umc_err_data.rom_data;
-	struct eeprom_store_record *ram_data = &ras_umc->umc_err_data.ram_data;
-	uint32_t eeprom_record_num, logical_count = 0;
+	int eeprom_record_num;
 	int save_count;
-	int ret = 0;
+	int ret = -ENODATA;
 
-	if (!data->bps)
+	/* Not need to save bad pages when FW management EEPROM is enabled. */
+	if (ras_eeprom_mgr_fw_record_enabled(ras_core))
 		return 0;
 
-	if (ras_fw_eeprom_supported(ras_core))
-		eeprom_record_num = ras_fw_eeprom_get_record_count(ras_core);
-	else
-		eeprom_record_num = ras_eeprom_get_record_count(ras_core);
+	if (!data->bps)
+		return -EINVAL;
+
+	eeprom_record_num = ras_eeprom_mgr_get_record_count(ras_core);
+	if (eeprom_record_num < 0)
+		return eeprom_record_num;
+
 	mutex_lock(&ras_umc->umc_lock);
 	save_count = data->count - eeprom_record_num;
-	logical_count = ram_data->bad_page_num - ram_data->bad_page_num_old;
 	/* only new entries are saved */
 	if (save_count > 0) {
-		if (ras_fw_eeprom_supported(ras_core))
-			ret = ras_fw_eeprom_append(ras_core, &data->bps[eeprom_record_num],
-					save_count);
-		else
-			ret = ras_eeprom_append(ras_core, &data->bps[eeprom_record_num],
-					save_count);
+		ret = ras_eeprom_mgr_append_records(ras_core,
+				&data->bps[eeprom_record_num], save_count);
 		if (ret) {
-			RAS_DEV_ERR(ras_core->dev, "Failed to save EEPROM table data!");
+			RAS_DEV_ERR(ras_core->dev,
+				"Failed to save EEPROM table data! ret:%d\n", ret);
 			ret = -EIO;
 			goto exit;
 		}
-		ras_umc_update_bad_pages(ras_core);
-		RAS_DEV_INFO(ras_core->dev, "Saved %d pages to EEPROM table.\n", logical_count);
+
+		RAS_DEV_INFO(ras_core->dev, "Saved %d pages to EEPROM table.\n",
+			ras_umc_count_valid_pages(ras_core,
+				&data->bps[eeprom_record_num], save_count));
 	}
 
 exit:
@@ -740,39 +857,7 @@ static int ras_umc_save_bad_pages(struct ras_core_context *ras_core)
 
 int ras_umc_handle_bad_pages(struct ras_core_context *ras_core, void *data)
 {
-	struct eeprom_umc_record *records;
-	int count, ret;
-
-	records = kzalloc_objs(*records, MAX_ECC_NUM_PER_RETIREMENT);
-	if (!records)
-		return -ENOMEM;
-
-	count = ras_umc_get_new_records(ras_core, records,
-					MAX_ECC_NUM_PER_RETIREMENT);
-	if (count <= 0) {
-		ret = -ENODATA;
-		goto out;
-	}
-
-	ret = ras_umc_add_bad_pages(ras_core, records, count, false);
-	if (ret) {
-		RAS_DEV_ERR(ras_core->dev, "Failed to add ras bad page!\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	ret = ras_umc_save_bad_pages(ras_core);
-	if (ret) {
-		RAS_DEV_ERR(ras_core->dev, "Failed to save ras bad page\n");
-		ret = -EINVAL;
-		goto out;
-	}
-
-	ret = 0;
-
-out:
-	kfree(records);
-	return ret;
+	return ras_umc_save_bad_pages(ras_core);
 }
 
 int ras_umc_sw_init(struct ras_core_context *ras_core)
@@ -790,6 +875,13 @@ int ras_umc_sw_init(struct ras_core_context *ras_core)
 	mutex_init(&ras_umc->umc_lock);
 	mutex_init(&ras_umc->bank_log_lock);
 
+	ras_umc->umc_ip_version = ras_core->config->umc_ip_version;
+	ras_umc->ip_func = ras_umc_get_ip_func(ras_core, ras_umc->umc_ip_version);
+	if (!ras_umc->ip_func) {
+		RAS_DEV_ERR(ras_core->dev, "Failed to get umc ip function!\n");
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -837,29 +929,17 @@ int ras_umc_sw_fini(struct ras_core_context *ras_core)
 int ras_umc_hw_init(struct ras_core_context *ras_core)
 {
 	struct ras_umc *ras_umc = &ras_core->ras_umc;
-	uint32_t nps;
-
-	nps = ras_core_get_curr_nps_mode(ras_core);
-
-	if (!nps || (nps >= UMC_MEMORY_PARTITION_MODE_UNKNOWN)) {
-		RAS_DEV_ERR(ras_core->dev, "Invalid memory NPS mode: %u!\n", nps);
-		return -ENODATA;
-	}
-
-	ras_umc->umc_err_data.umc_nps_mode = nps;
+	int count = ras_umc_get_badpage_count(ras_core);
 
-	ras_umc->umc_vram_type = ras_core->config->umc_cfg.umc_vram_type;
 	ras_umc->num_umc = ras_core->config->umc_cfg.num_umc;
-	if (!ras_umc->umc_vram_type) {
-		RAS_DEV_ERR(ras_core->dev, "Invalid UMC VRAM Type: %u!\n",
-			ras_umc->umc_vram_type);
-		return -ENODATA;
-	}
+	ras_umc->pa_base = ras_core->config->umc_cfg.pa_base;
+	ras_umc->lfb_size = ras_core->config->umc_cfg.lfb_size;
 
-	ras_umc->umc_ip_version = ras_core->config->umc_ip_version;
-	ras_umc->ip_func = ras_umc_get_ip_func(ras_core, ras_umc->umc_ip_version);
-	if (!ras_umc->ip_func)
-		return -EINVAL;
+	/* For preloaded bad pages case, bad page info is
+	 * deferred to report in hw init.
+	 */
+	if (count > 0)
+		ras_umc_report_badpage_info(ras_core);
 
 	return 0;
 }
@@ -901,6 +981,7 @@ int ras_umc_fill_eeprom_record(struct ras_core_context *ras_core,
 	err_rec->mem_channel = cur_nps_addr->channel_idx;
 	err_rec->mcumc_id = umc_inst;
 	err_rec->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(cur_nps_addr->pa);
+	err_rec->cur_nps_pa_flip_mask = cur_nps_addr->pa_flip_mask;
 	err_rec->cur_nps_bank = cur_nps_addr->bank;
 	err_rec->cur_nps = cur_nps;
 	return 0;
@@ -974,3 +1055,85 @@ uint32_t ras_umc_bit_wise_xor(uint32_t val)
 
 	return result;
 }
+
+int ras_umc_bank_to_umc_record(struct ras_core_context *ras_core,
+		struct ras_bank_ecc *bank, struct eeprom_umc_record *record)
+{
+	struct ras_umc *ras_umc = &ras_core->ras_umc;
+	int ret;
+
+	if (!bank || !record || !ras_umc->ip_func ||
+	    !ras_umc->ip_func->bank_to_eeprom_record)
+		return -EINVAL;
+
+	ret = ras_umc->ip_func->bank_to_eeprom_record(ras_core, bank, record);
+	if (ret)
+		return ret;
+
+	record->ipid = bank->ipid;
+
+	return 0;
+}
+
+int ras_umc_record_to_nps_record(struct ras_core_context *ras_core,
+		struct eeprom_umc_record *record,  uint32_t nps)
+{
+	struct ras_umc *ras_umc = &ras_core->ras_umc;
+	uint64_t ch_idx_v2;
+	uint32_t save_nps;
+
+	if (!record || !nps ||
+		(nps >= UMC_MEMORY_PARTITION_MODE_UNKNOWN))
+		return -EINVAL;
+
+	/* Avoid redundant conversion for the same NPS mode */
+	if ((record->cur_nps == nps) && record->cur_nps_retired_row_pfn &&
+	    record->cur_nps_pa_flip_mask)
+		return 0;
+
+	save_nps = EEPROM_RECORD_UMC_NPS_MODE(record);
+	ch_idx_v2 = record->retired_row_pfn & UMC_CHANNEL_IDX_V2;
+	if (!save_nps && !ch_idx_v2)
+		return ras_umc_eeprom_rec2nps_rec(ras_core, record, nps);
+
+	if (!ras_umc->ip_func || !ras_umc->ip_func->eeprom_record_to_nps_record)
+		return -EOPNOTSUPP;
+
+	return ras_umc->ip_func->eeprom_record_to_nps_record(ras_core, record, nps);
+}
+
+int ras_umc_dump_fw_records(struct ras_core_context *ras_core)
+{
+	struct eeprom_umc_record rec;
+	int eeprom_count, umc_count, new_count = 0;
+	uint32_t c;
+	int i, ret;
+
+	eeprom_count = ras_eeprom_mgr_get_record_count(ras_core);
+	/* no bad page record, skip eeprom access */
+	if (eeprom_count <= 0)
+		return eeprom_count;
+
+	umc_count = ras_umc_get_saved_eeprom_count(ras_core);
+	if (umc_count == eeprom_count) {
+		return 0;
+	} else if (umc_count > eeprom_count) {
+		RAS_DEV_ERR(ras_core->dev, "Invalid error count: eeprom:%d, umc:%d\n",
+			eeprom_count, umc_count);
+		return 0;
+	}
+
+	for (i = umc_count; i < eeprom_count; i++) {
+		memset(&rec, 0, sizeof(rec));
+		ret = ras_eeprom_mgr_get_records(ras_core, i, &rec, 1);
+		if (ret)
+			return 0;
+
+		c = 0;
+		ret = ras_umc_add_bad_pages(ras_core, &rec, 1, &c);
+		if (!ret)
+			new_count += c;
+	}
+
+	return new_count;
+}
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc.h b/drivers/gpu/drm/amd/ras/core/ras_umc.h
index af76de70ddd1d2..b014e531bec5dd 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc.h
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc.h
@@ -104,10 +104,13 @@ struct umc_mca_addr {
 	uint32_t umc_inst;
 	uint32_t node_inst;
 	uint32_t socket_id;
+	uint64_t mca_addr;
+	uint64_t ipid;
 };
 
 struct umc_phy_addr {
 	uint64_t pa;
+	uint64_t pa_flip_mask;
 	uint32_t bank;
 	uint32_t channel_idx;
 };
@@ -125,9 +128,8 @@ struct umc_bank_addr {
 struct ras_umc_ip_func {
 	int (*bank_to_eeprom_record)(struct ras_core_context *ras_core,
 			struct ras_bank_ecc *bank, struct eeprom_umc_record *record);
-	int (*eeprom_record_to_nps_pages)(struct ras_core_context *ras_core,
-			struct eeprom_umc_record *record, uint32_t nps,
-			uint64_t *pfns, uint32_t num);
+	int (*eeprom_record_to_nps_record)(struct ras_core_context *ras_core,
+			struct eeprom_umc_record *record, uint32_t nps);
 	int (*bank_to_soc_pa)(struct ras_core_context *ras_core,
 			struct umc_bank_addr bank_addr, uint64_t *soc_pa);
 	int (*soc_pa_to_bank)(struct ras_core_context *ras_core,
@@ -139,7 +141,8 @@ struct ras_umc_ip_func {
 		uint32_t nps);
 	uint64_t (*nps_pa_to_row_pa)(struct ras_core_context *ras_core,
 		uint64_t pa, enum umc_memory_partition_mode nps, bool zero_pfn_ok);
-	uint32_t (*get_die_id)(uint64_t mca_addr, uint64_t pa);
+	uint32_t (*get_die_id)(struct ras_core_context *ras_core,
+		uint64_t mca_addr, uint64_t pa);
 };
 
 struct eeprom_store_record {
@@ -149,25 +152,21 @@ struct eeprom_store_record {
 	int count;
 	/* the space can place new entries */
 	int space_left;
-	/* logical bad page number */
-	int bad_page_num;
-	/* the bad page number is ras_num_recs or
-	 * ras_num_recs * retire_unit
-	 */
-	int bad_page_num_old;
+	uint64_t umc_channel_bitmap;
 };
 
 struct ras_umc_err_data {
 	struct eeprom_store_record rom_data;
 	struct eeprom_store_record ram_data;
-	enum umc_memory_partition_mode umc_nps_mode;
-	uint64_t last_retired_pfn;
 };
 
 struct ras_umc {
 	u32 umc_ip_version;
-	u32 umc_vram_type;
 	u32 num_umc;
+	/* this node's base in the hive PA space: physical_node_id * lfb_size */
+	u64 pa_base;
+	/* local frame buffer size */
+	u64 lfb_size;
 	const struct ras_umc_ip_func *ip_func;
 	struct radix_tree_root root;
 	struct mutex  tree_lock;
@@ -180,6 +179,11 @@ struct ras_umc {
 	u32 pending_ecc_count;
 	/* number of entries dropped because pending_ecc_list was full */
 	u32 pending_ecc_dropped;
+
+	u32 last_record_count;
+	u64 last_channel_bitmap;
+
+	u32 max_pages_per_row;
 };
 
 /*
@@ -194,7 +198,10 @@ int ras_umc_sw_init(struct ras_core_context *ras);
 int ras_umc_sw_fini(struct ras_core_context *ras);
 int ras_umc_hw_init(struct ras_core_context *ras);
 int ras_umc_hw_fini(struct ras_core_context *ras);
-int ras_umc_psp_ma2pa(struct ras_core_context *ras_core,
+int ras_umc_ras_ta_translate_addr(struct ras_core_context *ras_core,
+		struct umc_mca_addr *in, struct umc_phy_addr *out,
+		uint32_t nps);
+int ras_umc_psp_translate_addr(struct ras_core_context *ras_core,
 		struct umc_mca_addr *in, struct umc_phy_addr *out,
 		uint32_t nps);
 int ras_umc_handle_bad_pages(struct ras_core_context *ras_core, void *data);
@@ -203,6 +210,8 @@ int ras_umc_log_bad_bank_pending(struct ras_core_context *ras_core, struct ras_b
 int ras_umc_log_pending_bad_bank(struct ras_core_context *ras_core);
 int ras_umc_clear_logged_ecc(struct ras_core_context *ras_core);
 int ras_umc_load_bad_pages(struct ras_core_context *ras_core);
+int ras_umc_add_bad_pages(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *bps, uint32_t bps_sz, uint32_t *valid_sz);
 int ras_umc_get_saved_eeprom_count(struct ras_core_context *ras_core);
 int ras_umc_clean_badpage_data(struct ras_core_context *ras_core);
 int ras_umc_fill_eeprom_record(struct ras_core_context *ras_core,
@@ -214,11 +223,20 @@ int ras_umc_get_badpage_record(struct ras_core_context *ras_core, uint32_t index
 bool ras_umc_check_retired_addr(struct ras_core_context *ras_core, uint64_t addr);
 int ras_umc_translate_soc_pa_and_bank(struct ras_core_context *ras_core,
 			uint64_t *soc_pa, struct umc_bank_addr *bank_addr, bool bank_to_pa);
-int ras_umc_convert_record_to_nps_pages(struct ras_core_context *ras_core,
-		struct eeprom_umc_record *record, uint32_t nps,
-		uint64_t *page_pfn, uint32_t max_pages);
+int ras_umc_convert_record_to_row_pages(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *page_addrs, uint32_t nr_page_addrs);
 uint32_t ras_umc_bit_wise_xor(uint32_t val);
 int ras_umc_ma2pa(struct ras_core_context *ras_core,
 	struct umc_mca_addr *addr_in, struct umc_phy_addr *addr_out,
 	uint32_t nps);
+int ras_umc_bank_to_umc_record(struct ras_core_context *ras_core,
+		struct ras_bank_ecc *bank, struct eeprom_umc_record *record);
+int ras_umc_record_to_nps_record(struct ras_core_context *ras_core,
+		struct eeprom_umc_record *record,  uint32_t nps);
+int ras_umc_dump_fw_records(struct ras_core_context *ras_core);
+void ras_umc_report_badpage_info(struct ras_core_context *ras_core);
+int ras_umc_alloc_row_pages(struct ras_core_context *ras_core,
+		uint64_t **page_pfns, uint32_t *nr_page_pfns);
+int ras_umc_free_row_pages(struct ras_core_context *ras_core,
+		uint64_t *page_pfns);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c b/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c
index ccdbc5c4bab1a5..cae7326c1aa052 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.c
@@ -52,7 +52,7 @@ static void __get_nps_pa_flip_bits(struct ras_core_context *ras_core,
 			enum umc_memory_partition_mode nps,
 			struct umc_flip_bits *flip_bits)
 {
-	uint32_t vram_type = ras_core->ras_umc.umc_vram_type;
+	uint32_t vram_type = ras_core_get_vram_type(ras_core);
 	u32 num_umc = ras_core->ras_umc.num_umc;
 
 	/* default setting */
@@ -148,21 +148,31 @@ static void __get_nps_pa_flip_bits(struct ras_core_context *ras_core,
 	}
 }
 
-static uint64_t umc_v12_0_nps_pa_to_row_pa(struct ras_core_context *ras_core,
-		uint64_t pa, enum umc_memory_partition_mode nps, bool zero_pfn_ok)
+static uint64_t __get_nps_pa_flip_mask(struct ras_core_context *ras_core,
+		enum umc_memory_partition_mode nps)
 {
 	struct umc_flip_bits flip_bits = {0};
-	uint64_t row_pa;
+	uint64_t flip_mask = 0;
 	int i;
 
 	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
 
-	row_pa = pa;
-	/* clear loop bits in soc physical address */
 	for (i = 0; i < flip_bits.bit_num; i++)
-		row_pa &= ~BIT_ULL(flip_bits.flip_bits_in_pa[i]);
+		flip_mask |= BIT_ULL(flip_bits.flip_bits_in_pa[i]);
+
+	return flip_mask;
+}
+
+static uint64_t umc_v12_0_nps_pa_to_row_pa(struct ras_core_context *ras_core,
+		uint64_t pa, enum umc_memory_partition_mode nps, bool zero_pfn_ok)
+{
+	struct umc_flip_bits flip_bits = {0};
+	uint64_t row_pa;
+
+	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
+	row_pa = pa & ~__get_nps_pa_flip_mask(ras_core, nps);
 
-	if (!zero_pfn_ok && !RAS_ADDR_TO_PFN(row_pa))
+	if (!zero_pfn_ok && !RAS_ADDR_TO_PFN(row_pa) && flip_bits.bit_num > 2)
 		row_pa |= BIT_ULL(flip_bits.flip_bits_in_pa[2]);
 
 	return row_pa;
@@ -176,13 +186,15 @@ static int lookup_bad_pages_in_a_row(struct ras_core_context *ras_core,
 	uint32_t col, col_lower, row, row_lower, idx, row_high;
 	uint64_t soc_pa, row_pa, column, err_addr;
 	uint64_t retired_addr = RAS_PFN_TO_ADDR(record->cur_nps_retired_row_pfn);
+	uint64_t flip_mask = 0;
 	struct umc_flip_bits flip_bits = {0};
 	uint32_t retire_unit;
 	uint32_t i;
 
 	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
 
-	row_pa = umc_v12_0_nps_pa_to_row_pa(ras_core, retired_addr, nps, true);
+	flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
+	row_pa = retired_addr & ~(flip_mask);
 
 	err_addr = record->address;
 	/* get column bit 0 and 1 in mca address */
@@ -243,6 +255,8 @@ static int umc_v12_0_ma2pa(struct ras_core_context *ras_core,
 	uint32_t socket_id = addr_in->socket_id;
 	uint32_t channel_index;
 	uint64_t err_addr = addr_in->err_addr;
+	uint64_t pa_base = ras_core->ras_umc.pa_base;
+	uint64_t lfb_size = ras_core->ras_umc.lfb_size;
 
 	if (node_inst != UMC_INV_AID_NODE) {
 		if (ch_inst >= UMC_V12_0_CHANNEL_INSTANCE_NUM ||
@@ -318,11 +332,11 @@ static int umc_v12_0_ma2pa(struct ras_core_context *ras_core,
 		OFFSET_IN_256B_BLOCK(na);
 
 	/* calc channel hash based on absolute address */
-	soc_pa += socket_id * SOCKET_LFB_SIZE;
+	soc_pa += pa_base;
 	/* the umc channel bits are not original values, they are hashed */
 	UMC_V12_0_SET_CHANNEL_HASH(channel_index, soc_pa);
 	/* restore pa */
-	soc_pa -= socket_id * SOCKET_LFB_SIZE;
+	soc_pa -= pa_base;
 
 	/* get some channel bits from na_nps directly and
 	 * add nps section offset
@@ -330,16 +344,16 @@ static int umc_v12_0_ma2pa(struct ras_core_context *ras_core,
 	if (nps == UMC_MEMORY_PARTITION_MODE_NPS2) {
 		soc_pa &= ~(0x1ULL << UMC_V12_0_PA_CH5_BIT);
 		soc_pa |= ((na_nps & 0x100) << 5);
-		soc_pa += (node_inst >> 1) * (SOCKET_LFB_SIZE >> 1);
+		soc_pa += (node_inst >> 1) * (lfb_size >> 1);
 	} else if (nps == UMC_MEMORY_PARTITION_MODE_NPS4) {
 		soc_pa &= ~(0x3ULL << UMC_V12_0_PA_CH4_BIT);
 		soc_pa |= ((na_nps & 0x300) << 4);
-		soc_pa += node_inst * (SOCKET_LFB_SIZE >> 2);
+		soc_pa += node_inst * (lfb_size >> 2);
 	} else if (nps == UMC_MEMORY_PARTITION_MODE_NPS8) {
 		soc_pa &= ~(0x7ULL << UMC_V12_0_PA_CH4_BIT);
 		soc_pa |= ((na_nps & 0x700) << 4);
-		soc_pa += node_inst * (SOCKET_LFB_SIZE >> 2) +
-			(channel_index >> 4) * (SOCKET_LFB_SIZE >> 3);
+		soc_pa += node_inst * (lfb_size >> 2) +
+			(channel_index >> 4) * (lfb_size >> 3);
 	}
 
 	addr_out->pa = soc_pa;
@@ -348,7 +362,24 @@ static int umc_v12_0_ma2pa(struct ras_core_context *ras_core,
 
 	return 0;
 }
+static int convert_ma_to_pa(struct ras_core_context *ras_core,
+			struct umc_mca_addr *addr_in, struct umc_phy_addr *addr_out,
+			uint32_t nps)
+{
+	int ret;
+
+	if (ras_psp_check_supported_cmd(ras_core, RAS_TA_CMD_ID__QUERY_ADDRESS))
+		ret = ras_umc_ras_ta_translate_addr(ras_core,
+				addr_in, addr_out, nps);
+	else
+		ret = umc_v12_0_ma2pa(ras_core,
+				addr_in, addr_out, nps);
+
+	if (!addr_out->pa_flip_mask)
+		addr_out->pa_flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
 
+	return ret;
+}
 static int convert_bank_to_nps_addr(struct ras_core_context *ras_core,
 			struct ras_bank_ecc *bank, struct umc_phy_addr *pa_addr, uint32_t nps)
 {
@@ -365,10 +396,10 @@ static int convert_bank_to_nps_addr(struct ras_core_context *ras_core,
 	addr_in.node_inst = ACA_IPID_2_DIE_ID(bank->ipid);
 	addr_in.socket_id = ACA_IPID_2_SOCKET_ID(bank->ipid);
 
-	ret = ras_umc_ma2pa(ras_core, &addr_in, &addr_out, nps);
+	ret = convert_ma_to_pa(ras_core, &addr_in, &addr_out, nps);
 	if (!ret) {
-		pa_addr->pa =
-			umc_v12_0_nps_pa_to_row_pa(ras_core, addr_out.pa, nps, false);
+		pa_addr->pa_flip_mask = addr_out.pa_flip_mask;
+		pa_addr->pa = addr_out.pa | pa_addr->pa_flip_mask;
 		pa_addr->channel_idx = addr_out.channel_idx;
 		pa_addr->bank = addr_out.bank;
 	}
@@ -393,21 +424,77 @@ static int umc_v12_0_bank_to_eeprom_record(struct ras_core_context *ras_core,
 		ACA_ADDR_2_ERR_ADDR(bank->addr), ACA_IPID_2_UMC_INST(bank->ipid),
 		&nps_addr, bank->nps, record);
 
-	if (ras_fw_eeprom_supported(ras_core) && bank->ts)
+	if (bank->ts)
 		record->ts = bank->ts;
 
+	/* If the bank being converted already has a timestamp,
+	 * then use the bank's timestamp.
+	 */
+	if (bank->timestamp)
+		record->ts = bank->timestamp;
+
 	lookup_bad_pages_in_a_row(ras_core, record,
 		bank->nps, NULL, 0, bank->seq_no, true);
 
 	return 0;
 }
 
-static int umc_v12_0_eeprom_record_to_nps_pages(struct ras_core_context *ras_core,
-			struct eeprom_umc_record *record, uint32_t nps,
-			uint64_t *pfns, uint32_t num)
+static int convert_eeprom_record_to_nps_addr(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *pa, uint64_t *pa_flip_mask, uint32_t nps)
 {
-	return lookup_bad_pages_in_a_row(ras_core,
-				record, nps, pfns, num, 0, false);
+	struct device_system_info dev_info = {0};
+	struct umc_mca_addr addr_in;
+	struct umc_phy_addr addr_out;
+	int ret;
+
+	memset(&addr_in, 0, sizeof(addr_in));
+	memset(&addr_out, 0, sizeof(addr_out));
+
+	ras_core_get_device_system_info(ras_core, &dev_info);
+
+	addr_in.err_addr = record->address;
+	addr_in.ch_inst = record->mem_channel;
+	addr_in.umc_inst = record->mcumc_id;
+	addr_in.node_inst = UMC_INV_AID_NODE;
+	addr_in.socket_id = dev_info.socket_id;
+
+	ret = convert_ma_to_pa(ras_core, &addr_in, &addr_out, nps);
+	if (ret)
+		return ret;
+
+	*pa_flip_mask = addr_out.pa_flip_mask;
+	*pa = addr_out.pa | addr_out.pa_flip_mask;
+
+	return 0;
+}
+
+static int umc_v12_0_eeprom_record_to_nps_record(struct ras_core_context *ras_core,
+				struct eeprom_umc_record *record, uint32_t nps)
+{
+	uint64_t pa = 0, flip_mask = 0;
+	uint64_t row_pfn;
+	int ret = 0;
+
+	if (!record || !nps)
+		return -EINVAL;
+
+	if (nps == EEPROM_RECORD_UMC_NPS_MODE(record)) {
+		record->cur_nps_pa_flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
+		row_pfn = EEPROM_RECORD_UMC_ADDR_PFN(record);
+		record->cur_nps_retired_row_pfn =
+			RAS_ADDR_TO_PFN(RAS_PFN_TO_ADDR(row_pfn) | record->cur_nps_pa_flip_mask);
+	} else {
+		ret = convert_eeprom_record_to_nps_addr(ras_core,
+				record, &pa, &flip_mask, nps);
+		if (!ret) {
+			record->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(pa);
+			record->cur_nps_pa_flip_mask = flip_mask;
+		}
+	}
+
+	record->cur_nps = nps;
+
+	return ret;
 }
 
 static int umc_12_0_soc_pa_to_bank(struct ras_core_context *ras_core,
@@ -496,10 +583,14 @@ static void umc_v12_0_mca_ipid_parse(struct ras_core_context *ras_core, uint64_t
  * umc number: 16
  * hbm type: UMC_VRAM_TYPE_HBM
  */
-static uint32_t umc_v12_0_get_die_id(uint64_t mca_addr, uint64_t pa)
+static uint32_t umc_v12_0_get_die_id(struct ras_core_context *ras_core,
+		uint64_t mca_addr, uint64_t pa)
 {
 	uint32_t die = 0;
 
+	/* use absolute PA */
+	pa += ras_core->ras_umc.pa_base;
+
 	/* we only calculate die id for nps1 mode(save_nps == ch_idx_v2 == 0) */
 	die += (((pa >> 12) & 0x1ULL) ^
 			((pa >> 20) & 0x1ULL) ^
@@ -521,7 +612,7 @@ static uint32_t umc_v12_0_get_die_id(uint64_t mca_addr, uint64_t pa)
 
 const struct ras_umc_ip_func ras_umc_func_v12_0 = {
 	.bank_to_eeprom_record = umc_v12_0_bank_to_eeprom_record,
-	.eeprom_record_to_nps_pages = umc_v12_0_eeprom_record_to_nps_pages,
+	.eeprom_record_to_nps_record = umc_v12_0_eeprom_record_to_nps_record,
 	.bank_to_soc_pa = umc_12_0_bank_to_soc_pa,
 	.soc_pa_to_bank = umc_12_0_soc_pa_to_bank,
 	.mca_ipid_parse = umc_v12_0_mca_ipid_parse,
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.h b/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.h
index 034da843a00863..6bddb8d94d43b1 100644
--- a/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.h
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc_v12_0.h
@@ -302,9 +302,6 @@
 #define UMC_V12_0_TOTAL_CHANNEL_NUM \
 	(UMC_V12_0_AID_NUM_MAX * UMC_V12_0_UMC_INSTANCE_NUM * UMC_V12_0_CHANNEL_INSTANCE_NUM)
 
-/* one device has 192GB HBM */
-#define SOCKET_LFB_SIZE   0x3000000000ULL
-
 extern const struct ras_umc_ip_func ras_umc_func_v12_0;
 
 int ras_umc_get_badpage_count(struct ras_core_context *ras_core);
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c b/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c
new file mode 100644
index 00000000000000..8f3bba2621f890
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.c
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "ras.h"
+#include "ras_umc.h"
+#include "core_status.h"
+#include "ras_umc_v15_0.h"
+
+static void __get_nps_pa_flip_bits(struct ras_core_context *ras_core,
+			enum umc_memory_partition_mode nps,
+			struct umc_flip_bits *flip_bits)
+{
+
+}
+
+static uint64_t  __get_nps_pa_flip_mask(struct ras_core_context *ras_core,
+		enum umc_memory_partition_mode nps)
+{
+	struct umc_flip_bits flip_bits = {0};
+	uint64_t flip_mask = 0;
+	int i;
+
+	__get_nps_pa_flip_bits(ras_core, nps, &flip_bits);
+
+	for (i = 0; i < flip_bits.bit_num; i++)
+		flip_mask |= BIT_ULL(flip_bits.flip_bits_in_pa[i]);
+
+	return flip_mask;
+}
+
+static int lookup_bad_pages_in_a_row(struct ras_core_context *ras_core,
+		struct eeprom_umc_record *record, uint32_t nps,
+		uint64_t *pfns, uint32_t num,
+		uint64_t seq_no, bool dump)
+{
+	return 0;
+}
+
+static int umc_v15_convert_ma_to_pa(struct ras_core_context *ras_core,
+			struct umc_mca_addr *addr_in, struct umc_phy_addr *addr_out,
+			uint32_t nps)
+{
+
+	return 0;
+}
+
+static int convert_ma_to_pa(struct ras_core_context *ras_core,
+			struct umc_mca_addr *addr_in, struct umc_phy_addr *addr_out,
+			uint32_t nps)
+{
+	int ret;
+
+	ret = ras_umc_psp_translate_addr(ras_core,
+				addr_in, addr_out, nps);
+	if (ret != -EOPNOTSUPP)
+		return ret;
+
+	if (ras_psp_check_supported_cmd(ras_core, RAS_TA_CMD_ID__QUERY_ADDRESS))
+		ret = ras_umc_ras_ta_translate_addr(ras_core,
+				addr_in, addr_out, nps);
+	else
+		ret = umc_v15_convert_ma_to_pa(ras_core,
+				addr_in, addr_out, nps);
+
+	return ret;
+}
+
+static int convert_bank_to_nps_addr(struct ras_core_context *ras_core,
+			struct ras_bank_ecc *bank, struct umc_phy_addr *pa_addr, uint32_t nps)
+{
+	struct umc_mca_addr addr_in;
+	struct umc_phy_addr addr_out;
+	int ret;
+
+	memset(&addr_in, 0, sizeof(addr_in));
+	memset(&addr_out, 0, sizeof(addr_out));
+
+	addr_in.err_addr = ACA_ADDR_2_ERR_ADDR(bank->addr);
+	addr_in.ch_inst = ACA_IPID_2_UMC_CH(bank->ipid);
+	addr_in.umc_inst = ACA_IPID_2_UMC_INST(bank->ipid);
+	addr_in.node_inst = ACA_IPID_2_DIE_ID(bank->ipid);
+	addr_in.socket_id = ACA_IPID_2_SOCKET_ID(bank->ipid);
+	addr_in.mca_addr = bank->addr;
+	addr_in.ipid = bank->ipid;
+
+	ret = convert_ma_to_pa(ras_core, &addr_in, &addr_out, nps);
+	if (!ret) {
+		pa_addr->pa_flip_mask = addr_out.pa_flip_mask;
+		pa_addr->pa = addr_out.pa | pa_addr->pa_flip_mask;
+		pa_addr->channel_idx = addr_out.channel_idx;
+		pa_addr->bank = addr_out.bank;
+	}
+
+	return ret;
+}
+
+static int umc_v15_0_bank_to_eeprom_record(struct ras_core_context *ras_core,
+		struct ras_bank_ecc *bank, struct eeprom_umc_record *record)
+{
+	struct umc_phy_addr nps_addr;
+	int ret;
+
+	memset(&nps_addr, 0, sizeof(nps_addr));
+
+	ret = convert_bank_to_nps_addr(ras_core, bank,
+			&nps_addr, bank->nps);
+	if (ret)
+		return ret;
+
+	ras_umc_fill_eeprom_record(ras_core,
+		ACA_ADDR_2_ERR_ADDR(bank->addr), ACA_IPID_2_UMC_INST(bank->ipid),
+		&nps_addr, bank->nps, record);
+
+	/* If the bank being converted already has a timestamp,
+	 * then use the bank's timestamp.
+	 */
+	if (bank->timestamp)
+		record->ts = bank->timestamp;
+
+	lookup_bad_pages_in_a_row(ras_core, record,
+		bank->nps, NULL, 0, bank->seq_no, true);
+
+	return 0;
+}
+
+static int convert_eeprom_record_to_nps_addr(struct ras_core_context *ras_core,
+	struct eeprom_umc_record *record, uint64_t *pa, uint64_t *pa_flip_mask, uint32_t nps)
+{
+	struct device_system_info dev_info = {0};
+	struct umc_mca_addr addr_in;
+	struct umc_phy_addr addr_out;
+	int ret;
+
+	memset(&addr_in, 0, sizeof(addr_in));
+	memset(&addr_out, 0, sizeof(addr_out));
+
+	ras_core_get_device_system_info(ras_core, &dev_info);
+
+	addr_in.err_addr = record->address;
+	addr_in.ch_inst = record->mem_channel;
+	addr_in.umc_inst = record->mcumc_id;
+	addr_in.node_inst = UMC_INV_AID_NODE;
+	addr_in.socket_id = dev_info.socket_id;
+
+	ret = convert_ma_to_pa(ras_core, &addr_in, &addr_out, nps);
+	if (ret)
+		return ret;
+
+	*pa_flip_mask = addr_out.pa_flip_mask;
+	*pa = addr_out.pa | addr_out.pa_flip_mask;
+
+	return 0;
+}
+
+static int umc_v15_0_eeprom_record_to_nps_record(struct ras_core_context *ras_core,
+				struct eeprom_umc_record *record, uint32_t nps)
+{
+	uint64_t pa = 0, flip_mask = 0;
+	uint64_t row_pfn;
+	int ret = 0;
+
+	if (nps == EEPROM_RECORD_UMC_NPS_MODE(record)) {
+		record->cur_nps_pa_flip_mask = __get_nps_pa_flip_mask(ras_core, nps);
+		row_pfn = EEPROM_RECORD_UMC_ADDR_PFN(record);
+		record->cur_nps_retired_row_pfn =
+			RAS_ADDR_TO_PFN(RAS_PFN_TO_ADDR(row_pfn) | record->cur_nps_pa_flip_mask);
+	} else {
+		ret = convert_eeprom_record_to_nps_addr(ras_core,
+				record, &pa, &flip_mask, nps);
+		if (!ret) {
+			record->cur_nps_retired_row_pfn = RAS_ADDR_TO_PFN(pa);
+			record->cur_nps_pa_flip_mask = flip_mask;
+		}
+	}
+
+	record->cur_nps = nps;
+
+	return ret;
+}
+
+static int umc_v15_0_soc_pa_to_bank(struct ras_core_context *ras_core,
+			uint64_t soc_pa,
+			struct umc_bank_addr *bank_addr)
+{
+	return 0;
+}
+
+static int umc_v15_0_bank_to_soc_pa(struct ras_core_context *ras_core,
+			struct umc_bank_addr bank_addr,
+			uint64_t *soc_pa)
+{
+	return 0;
+}
+
+const struct ras_umc_ip_func ras_umc_func_v15_0 = {
+	.bank_to_eeprom_record = umc_v15_0_bank_to_eeprom_record,
+	.eeprom_record_to_nps_record = umc_v15_0_eeprom_record_to_nps_record,
+	.bank_to_soc_pa = umc_v15_0_bank_to_soc_pa,
+	.soc_pa_to_bank = umc_v15_0_soc_pa_to_bank,
+};
diff --git a/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.h b/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.h
new file mode 100644
index 00000000000000..5c3a79a48ec00b
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/core/ras_umc_v15_0.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __RAS_UMC_V15_0_H__
+#define __RAS_UMC_V15_0_H__
+#include "ras.h"
+
+/* MCA_UMC_UMC0_MCUMC_ADDRT0 */
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr__SHIFT                0x0
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__Reserved__SHIFT                 0x38
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr_MASK                  0x00FFFFFFFFFFFFFFL
+#define MCA_UMC_UMC0_MCUMC_ADDRT0__Reserved_MASK                   0xFF00000000000000L
+
+/* MCMP1_IPIDT0 */
+#define MCMP1_IPIDT0__InstanceIdLo__SHIFT                          0x0
+#define MCMP1_IPIDT0__HardwareID__SHIFT                            0x20
+#define MCMP1_IPIDT0__InstanceIdHi__SHIFT                          0x2c
+#define MCMP1_IPIDT0__McaType__SHIFT                               0x30
+
+#define MCMP1_IPIDT0__InstanceIdLo_MASK                            0x00000000FFFFFFFFL
+#define MCMP1_IPIDT0__HardwareID_MASK                              0x00000FFF00000000L
+#define MCMP1_IPIDT0__InstanceIdHi_MASK                            0x0000F00000000000L
+#define MCMP1_IPIDT0__McaType_MASK                                 0xFFFF000000000000L
+
+#define ACA_IPID_HI_2_UMC_AID(_ipid_hi) (((_ipid_hi) >> 2) & 0x3)
+#define ACA_IPID_LO_2_UMC_CH(_ipid_lo)  \
+	(((((_ipid_lo) >> 20) & 0x1) * 4) + (((_ipid_lo) >> 12) & 0xF))
+#define ACA_IPID_LO_2_UMC_INST(_ipid_lo) (((_ipid_lo) >> 21) & 0x7)
+
+#define ACA_IPID_2_DIE_ID(ipid)  ((REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi) >> 2) & 0x03)
+#define ACA_IPID_2_UMC_CH(ipid) \
+	(ACA_IPID_LO_2_UMC_CH(REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo)))
+
+#define ACA_IPID_2_UMC_INST(ipid) \
+	(ACA_IPID_LO_2_UMC_INST(REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo)))
+
+#define ACA_IPID_2_SOCKET_ID(ipid) \
+	(((REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo) & 0x1) << 2) | \
+	 (REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi) & 0x03))
+
+#define ACA_ADDR_2_ERR_ADDR(addr) \
+	REG_GET_FIELD(addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr)
+
+/* invalid node instance value */
+#define UMC_INV_AID_NODE 0xffff
+
+extern const struct ras_umc_ip_func ras_umc_func_v15_0;
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/core/ta_if.h b/drivers/gpu/drm/amd/ras/core/ta_if.h
index 6b05c82679c027..30c058a524db7f 100644
--- a/drivers/gpu/drm/amd/ras/core/ta_if.h
+++ b/drivers/gpu/drm/amd/ras/core/ta_if.h
@@ -101,6 +101,40 @@ enum ras_ta_block {
 	RAS_TA_NUM_BLOCK_MAX
 };
 
+enum ras_ta_mpasp_block {
+	RAS_TA_MPASP_BLOCK__UMC = 0,
+	RAS_TA_MPASP_BLOCK__GFX,
+	RAS_TA_MPASP_BLOCK__SDMA,
+	RAS_TA_MPASP_BLOCK__MMHUB,
+	RAS_TA_MPASP_BLOCK__ATHUB,
+	RAS_TA_MPASP_BLOCK__PCIE,
+	RAS_TA_MPASP_BLOCK__SMN,
+	RAS_TA_MPASP_BLOCK__MP0,
+	RAS_TA_MPASP_BLOCK__MP1,
+	RAS_TA_MPASP_BLOCK__VCN,
+	RAS_TA_MPASP_BLOCK__IH,
+	RAS_TA_MPASP_BLOCK__MP5,
+	RAS_TA_MPASP_BLOCK__ATU,
+	RAS_TA_MPASP_BLOCK__DACC_BE,
+	RAS_TA_MPASP_BLOCK__ECLR,
+	RAS_TA_MPASP_BLOCK__KPX_SERDES,
+	RAS_TA_MPASP_BLOCK__LSDMA,
+	RAS_TA_MPASP_BLOCK__MPART,
+	RAS_TA_MPASP_BLOCK__MPIFOE,
+	RAS_TA_MPASP_BLOCK__MPRAS,
+	RAS_TA_MPASP_BLOCK__NBIF,
+	RAS_TA_MPASP_BLOCK__NBIO,
+	RAS_TA_MPASP_BLOCK__OXRP,
+	RAS_TA_MPASP_BLOCK__PCIE_PL,
+	RAS_TA_MPASP_BLOCK__PCS_XGMI,
+	RAS_TA_MPASP_BLOCK__PIE,
+	RAS_TA_MPASP_BLOCK__CS,
+	RAS_TA_MPASP_BLOCK__SHUB,
+	RAS_TA_MPASP_BLOCK__SSBDCI,
+	RAS_TA_MPASP_BLOCK__UCIE_PCS,
+	RAS_TA_MPASP_NUM_BLOCK_MAX
+};
+
 enum ras_ta_mca_block {
 	RAS_TA_MCA_BLOCK__MP0   = 0,
 	RAS_TA_MCA_BLOCK__MP1   = 1,
@@ -168,6 +202,7 @@ struct ras_ta_init_flags {
 	uint8_t nps_mode;
 	uint32_t active_umc_mask;
 	uint8_t vram_type;
+	uint32_t ext_umc_mask;
 };
 
 struct ras_ta_mca_addr {
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/Makefile b/drivers/gpu/drm/amd/ras/ras_mgr/Makefile
index 6759635aadc26c..c70760f10ffda0 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/Makefile
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/Makefile
@@ -21,11 +21,13 @@
 RAS_MGR_FILES = amdgpu_ras_sys.o  \
 		amdgpu_ras_mgr.o \
 		amdgpu_ras_eeprom_i2c.o \
-		amdgpu_ras_mp1_v13_0.o \
+		amdgpu_ras_mp1.o \
 		amdgpu_ras_cmd.o \
 		amdgpu_virt_ras_cmd.o \
 		amdgpu_ras_process.o \
-		amdgpu_ras_nbio_v7_9.o
+		amdgpu_ras_nbio_v7_9.o \
+		amdgpu_ras_bert.o \
+		amdgpu_ras_mce.o
 
 
 RAS_MGR = $(addprefix $(AMD_GPU_RAS_PATH)/ras_mgr/, $(RAS_MGR_FILES))
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.c
new file mode 100644
index 00000000000000..71e9c8d3c8476b
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.c
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_ras_bert.h"
+
+#if defined(CONFIG_X86_MCE_AMD) && defined(CONFIG_ACPI_APEI)
+#include <linux/acpi.h>
+#include <linux/mutex.h>
+
+#include "amdgpu_ras_mgr.h"
+
+struct amdgpu_ras_bert_boot_err_state {
+	struct mutex lock; /* protects cached boot error state */
+	bool polled;
+	bool platform_written;
+	bool written[MAX_GPU_INSTANCE];
+	u32 users;
+	int poll_result;
+	u8 *raw_data;
+	u32 raw_data_len;
+};
+
+static struct amdgpu_ras_bert_boot_err_state boot_err_state = {
+	.lock = __MUTEX_INITIALIZER(boot_err_state.lock),
+};
+
+static int amdgpu_ras_bert_process_boot_errors(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_ras_bert_boot_err_state *state = &boot_err_state;
+	struct acpi_bert_region *boot_error_region;
+	struct acpi_table_header *table;
+	struct acpi_table_bert *bert_tab;
+	unsigned int region_len;
+	acpi_status status;
+	u32 socket_id;
+	int ret;
+
+	if (!ras_mgr || !ras_mgr->ras_core)
+		return -EPERM;
+
+	mutex_lock(&state->lock);
+	state->users++;
+
+	if (!state->polled) {
+		state->polled = true;
+		state->poll_result = 0;
+		ret = 0;
+
+		if (acpi_disabled)
+			goto out_unlock;
+
+		status = acpi_get_table(ACPI_SIG_BERT, 0, &table);
+		if (status == AE_NOT_FOUND)
+			goto out_unlock;
+
+		if (ACPI_FAILURE(status)) {
+			RAS_DEV_ERR(adev, "get table failed, %s.\n", acpi_format_exception(status));
+			state->poll_result = -EINVAL;
+			goto out_unlock;
+		}
+		bert_tab = (struct acpi_table_bert *)table;
+
+		if (bert_tab->header.length < sizeof(struct acpi_table_bert) ||
+		    bert_tab->region_length < sizeof(struct acpi_bert_region)) {
+			RAS_DEV_ERR(adev, "table invalid.\n");
+			state->poll_result = -EINVAL;
+			goto out_put_bert_tab;
+		}
+
+		region_len = bert_tab->region_length;
+		boot_error_region = acpi_os_map_memory(bert_tab->address, region_len);
+		if (boot_error_region) {
+			state->raw_data = kmemdup(boot_error_region, region_len, GFP_KERNEL);
+			acpi_os_unmap_memory(boot_error_region, region_len);
+			if (!state->raw_data) {
+				RAS_DEV_ERR(adev, "failed to cache BERT raw data.\n");
+				state->poll_result = -ENOMEM;
+			} else {
+				state->raw_data_len = region_len;
+			}
+		} else {
+			RAS_DEV_ERR(adev, "failed to map BERT region.\n");
+			state->poll_result = -ENOMEM;
+		}
+
+out_put_bert_tab:
+		acpi_put_table(table);
+	}
+
+	if (state->poll_result) {
+		ret = state->poll_result;
+		goto out_unlock;
+	}
+
+	if (!state->raw_data || !state->raw_data_len ||
+	    !(adev && (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu))) {
+		ret = 0;
+		goto out_unlock;
+	}
+
+	if (!adev->smuio.funcs || !adev->smuio.funcs->get_socket_id) {
+		RAS_DEV_WARN(adev, "No interface to obtain current device socket ID!\n");
+		ret = 0;
+		goto out_unlock;
+	}
+
+	socket_id = adev->smuio.funcs->get_socket_id(adev);
+	if (socket_id >= MAX_GPU_INSTANCE) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
+	if (state->written[socket_id]) {
+		ret = 0;
+		goto out_unlock;
+	}
+
+	ras_mgr->ras_core->bert_platform_owner = !state->platform_written;
+	if (ras_mgr->ras_core->bert_platform_owner)
+		state->platform_written = true;
+	state->written[socket_id] = true;
+	ret = ras_bert_process_records(ras_mgr->ras_core,
+				       state->raw_data, state->raw_data_len);
+	ras_mgr->ras_core->bert_platform_owner = false;
+
+out_unlock:
+	mutex_unlock(&state->lock);
+
+	return ret;
+}
+
+static void amdgpu_ras_bert_release_boot_errors(void)
+{
+	struct amdgpu_ras_bert_boot_err_state *state = &boot_err_state;
+
+	mutex_lock(&state->lock);
+	if (!state->users || --state->users)
+		goto out_unlock;
+
+	memset(state->written, 0, sizeof(state->written));
+	state->platform_written = false;
+	kfree(state->raw_data);
+	state->raw_data = NULL;
+	state->raw_data_len = 0;
+	state->poll_result = 0;
+	state->polled = false;
+
+out_unlock:
+	mutex_unlock(&state->lock);
+}
+#endif
+
+int amdgpu_ras_bert_sw_init(struct amdgpu_device *adev)
+{
+#if defined(CONFIG_X86_MCE_AMD) && defined(CONFIG_ACPI_APEI)
+	return amdgpu_ras_bert_process_boot_errors(adev);
+#else
+	return 0;
+#endif
+}
+
+int amdgpu_ras_bert_sw_fini(struct amdgpu_device *adev)
+{
+#if defined(CONFIG_X86_MCE_AMD) && defined(CONFIG_ACPI_APEI)
+	amdgpu_ras_bert_release_boot_errors();
+#endif
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.h
new file mode 100644
index 00000000000000..e56aa143ad9524
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_bert.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMDGPU_RAS_BERT_H__
+#define __AMDGPU_RAS_BERT_H__
+
+int amdgpu_ras_bert_sw_init(struct amdgpu_device *adev);
+int amdgpu_ras_bert_sw_fini(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
index f9ee297a67450a..49a17b60bf80e8 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
@@ -259,10 +259,48 @@ static int amdgpu_ras_translate_fb_address(struct ras_core_context *ras_core,
 	return RAS_CMD__SUCCESS;
 }
 
+static int amdgpu_ras_get_cper_records(struct ras_core_context *ras_core,
+			struct ras_cmd_ctx *cmd, void *data)
+{
+	struct ras_cmd_cper_record_req *req =
+		(struct ras_cmd_cper_record_req *)cmd->input_buff_raw;
+	uint64_t user_addr = 0;
+	uint8_t *buf_ptr = NULL;
+	int ret;
+
+	if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req))
+		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
+
+	if (!req->buf_size || !req->buf_ptr || !req->cper_num)
+		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
+
+	buf_ptr = kzalloc(req->buf_size, GFP_KERNEL);
+	if (!buf_ptr)
+		return RAS_CMD__ERROR_GENERIC;
+
+	user_addr = req->buf_ptr;
+	req->buf_ptr = (uintptr_t)buf_ptr;
+
+	ret = rascore_handle_cmd(ras_core, cmd, data);
+	if (ret) {
+		kfree(buf_ptr);
+		return ret;
+	}
+
+	if (copy_to_user((void __user *)(uintptr_t)user_addr, buf_ptr, req->buf_size))
+		ret = RAS_CMD__ERROR_GENERIC;
+
+	req->buf_ptr = user_addr;
+	kfree(buf_ptr);
+
+	return ret;
+}
+
 static struct ras_cmd_func_map amdgpu_ras_cmd_maps[] = {
 	{RAS_CMD__INJECT_ERROR, amdgpu_ras_inject_error},
 	{RAS_CMD__GET_SAFE_FB_ADDRESS_RANGES, amdgpu_ras_get_ras_safe_fb_addr_ranges},
 	{RAS_CMD__TRANSLATE_FB_ADDRESS, amdgpu_ras_translate_fb_address},
+	{RAS_CMD__GET_CPER_RECORD, amdgpu_ras_get_cper_records},
 };
 
 int amdgpu_ras_handle_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx *cmd, void *data)
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c
index aa97b58c942274..a56d3116c5fd5c 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_eeprom_i2c.c
@@ -26,6 +26,7 @@
 #include "amdgpu_ras_eeprom.h"
 #include "amdgpu_ras_mgr.h"
 #include "amdgpu_ras_eeprom_i2c.h"
+#include "ras.h"
 #include "eeprom.h"
 
 /* These are memory addresses as would be seen by one or more EEPROM
@@ -61,11 +62,46 @@
 
 #define EEPROM_OFFSET_SIZE 2
 
-static int ras_eeprom_i2c_config(struct ras_core_context *ras_core)
+/* typical ras bad page rate is 1 bad page per 100MB VRAM */
+#define ESTIMATE_BAD_PAGE_THRESHOLD(size) div64_u64(size, 100ULL * SZ_1M)
+
+#define COUNT_BAD_PAGE_THRESHOLD(size) (((size) >> 21) << 4)
+
+/* Reserve 8 physical dram row for possible retirement.
+ * In worst cases, it will lose 8 * 2MB memory in vram domain
+ */
+#define RAS_RESERVED_VRAM_SIZE_DEFAULT	(16ULL << 20)
+
+#define RAS_PAGES_TO_EEPROM_RECORDS(pages, ratio)  div64_u64(pages, ratio)
+
+#define BAD_PAGE_NUM_PER_EEPROM_RECORD_V13  16
+#define BAD_PAGE_NUM_PER_EEPROM_RECORD_V15  128
+
+static u64 ras_eeprom_reserved_vram_size(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	/* Set by amdgpu_ras_init_reserved_vram_size(); 0 means that path
+	 * does not cover this ASIC.
+	 */
+	if (con && con->reserved_pages_in_bytes)
+		return con->reserved_pages_in_bytes;
+
+	return RAS_RESERVED_VRAM_SIZE_DEFAULT;
+}
+
+static int ras_eeprom_i2c_config(struct ras_core_context *ras_core,
+		struct ras_eeprom_param_config *cfg)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	struct ras_eeprom_control *control = &ras_core->ras_eeprom;
+	struct ras_module_param mod_param = {0};
+	u64 badpages, badpages_per_record = 0;
 	u8 i2c_addr;
+	u32 ip_version;
+	int badpage_threshold = 0;
+	int ret;
+
+	ip_version = amdgpu_ip_version(adev, MP1_HWIP, 0);
 
 	if (adev->bios && amdgpu_atomfirmware_ras_rom_addr(adev, &i2c_addr)) {
 		/* The address given by VBIOS is an 8-bit, wire-format
@@ -77,28 +113,99 @@ static int ras_eeprom_i2c_config(struct ras_core_context *ras_core)
 		 * amdgpu_eeprom.c.
 		 */
 		i2c_addr = (i2c_addr & 0x0F) >> 1;
-		control->i2c_address = ((u32) i2c_addr) << 16;
-		return 0;
+		cfg->eeprom_i2c_addr = ((u32) i2c_addr) << 16;
+	} else {
+		switch (ip_version) {
+		case IP_VERSION(13, 0, 5):
+		case IP_VERSION(13, 0, 6):
+		case IP_VERSION(13, 0, 10):
+		case IP_VERSION(13, 0, 12):
+		case IP_VERSION(13, 0, 14):
+			cfg->eeprom_i2c_addr = EEPROM_I2C_MADDR_4;
+			badpages_per_record = BAD_PAGE_NUM_PER_EEPROM_RECORD_V13;
+			break;
+		case IP_VERSION(15, 0, 8):
+			cfg->eeprom_i2c_addr = EEPROM_I2C_MADDR_4;
+			badpages_per_record = BAD_PAGE_NUM_PER_EEPROM_RECORD_V15;
+			break;
+		default:
+			RAS_DEV_ERR(adev, "IP version(0x%x) is not supported!\n", ip_version);
+			return -ENODATA;
+		}
 	}
 
-	switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
-	case IP_VERSION(13, 0, 5):
-	case IP_VERSION(13, 0, 6):
-	case IP_VERSION(13, 0, 10):
-	case IP_VERSION(13, 0, 12):
-	case IP_VERSION(13, 0, 14):
-		control->i2c_address = EEPROM_I2C_MADDR_4;
-		return 0;
-	default:
-		return -ENODATA;
+	if (!badpages_per_record || !cfg->eeprom_i2c_addr) {
+		RAS_DEV_ERR(adev, "EEPROM parameters are not configured!\n");
+		return -EINVAL;
 	}
-	return -ENODATA;
+
+	cfg->eeprom_ip_version = ip_version;
+
+	cfg->eeprom_i2c_adapter = adev->pm.ras_eeprom_i2c_bus;
+	if (cfg->eeprom_i2c_adapter) {
+		const struct i2c_adapter_quirks *quirks =
+			((struct i2c_adapter *)cfg->eeprom_i2c_adapter)->quirks;
+
+		if (quirks) {
+			cfg->max_i2c_read_len = quirks->max_read_len;
+			cfg->max_i2c_write_len = quirks->max_write_len;
+		}
+	}
+
+	ret = ras_core_get_module_param(ras_core, &mod_param);
+	if (ret) {
+		RAS_DEV_ERR(adev, "Failed to get module option parameter.\n");
+		return ret;
+	}
+
+	badpage_threshold = mod_param.ras_bad_page_threshold;
+
+	/*
+	 * badpage_threshold is used to config
+	 * the threshold for the number of bad pages.
+	 * -1:  Threshold is set to default value
+	 *      Driver will issue a warning message when threshold is reached
+	 *      and continue runtime services.
+	 * 0:   Disable bad page retirement
+	 *      Driver will not retire bad pages
+	 *      which is intended for debugging purpose.
+	 * -2:  Threshold is determined by a formula
+	 *      that assumes 1 bad page per 100M of local memory.
+	 *      Driver will continue runtime services when threhold is reached.
+	 * 0 < threshold < max number of bad page records in EEPROM,
+	 *      A user-defined threshold is set
+	 *      Driver will halt runtime services when this custom threshold is reached.
+	 */
+	if (badpage_threshold == NONSTOP_OVER_THRESHOLD) {
+		cfg->work_mode_over_thresh = RAS_WORK_MODE_OVER_THRESH_NORMAL;
+		badpages = ESTIMATE_BAD_PAGE_THRESHOLD(adev->gmc.mc_vram_size);
+	} else if (badpage_threshold == WARN_NONSTOP_OVER_THRESHOLD) {
+		cfg->work_mode_over_thresh = RAS_WORK_MODE_OVER_THRESH_STRICT;
+		badpages = COUNT_BAD_PAGE_THRESHOLD(ras_eeprom_reserved_vram_size(adev));
+	} else if (!badpage_threshold) {
+		cfg->work_mode_over_thresh = RAS_WORK_MODE_OVER_THRESH_DEBUG;
+		badpages = 128;
+	} else if (badpage_threshold > 0) {
+		cfg->work_mode_over_thresh = RAS_WORK_MODE_OVER_THRESH_RMA;
+		badpages = badpage_threshold;
+	} else {
+		RAS_DEV_ERR(adev, "Invalid badpage_threshold value(%d)\n",
+			badpage_threshold);
+		return -EINVAL;
+	}
+
+	/* Convert bad page count to record count as the threshold value */
+	cfg->eeprom_record_threshold_count =
+		RAS_PAGES_TO_EEPROM_RECORDS(badpages, badpages_per_record);
+
+	return 0;
 }
 
 static int ras_eeprom_i2c_xfer(struct ras_core_context *ras_core, u32 eeprom_addr,
 				u8 *eeprom_buf, u32 buf_size, bool read)
 {
-	struct i2c_adapter *i2c_adap = ras_core->ras_eeprom.i2c_adapter;
+	struct ras_eeprom_control *control = ras_core->eeprom_mgr.ras_eeprom;
+	struct i2c_adapter *i2c_adap = control->i2c_adapter;
 	u8 eeprom_offset_buf[EEPROM_OFFSET_SIZE];
 	struct i2c_msg msgs[] = {
 		{
@@ -178,5 +285,5 @@ static int ras_eeprom_i2c_xfer(struct ras_core_context *ras_core, u32 eeprom_add
 
 const struct ras_eeprom_sys_func amdgpu_ras_eeprom_i2c_sys_func = {
 	.eeprom_i2c_xfer = ras_eeprom_i2c_xfer,
-	.update_eeprom_i2c_config = ras_eeprom_i2c_config,
+	.get_eeprom_config = ras_eeprom_i2c_config,
 };
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.c
new file mode 100644
index 00000000000000..0edb3dc06a69c4
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.c
@@ -0,0 +1,267 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_ras_mgr.h"
+#include "amdgpu_ras_mce.h"
+
+#ifdef CONFIG_X86_MCE_AMD
+#include <asm/mce.h>
+
+static int amdgpu_sys_mce_notifier(struct notifier_block *nb, unsigned long val, void *data);
+typedef int (*mce_notifier)(struct amdgpu_device *dev,
+			unsigned int id, unsigned long val, void *data);
+struct ras_mce_dev {
+	void *dev;
+	mce_notifier dev_notifier;
+};
+
+struct ras_mce_mgr {
+	struct ras_mce_dev devs[MAX_GPU_INSTANCE];
+	atomic_t dev_count;
+	atomic_t ref_count;
+	struct notifier_block nb;
+	mce_notifier notifier;
+};
+
+struct ras_mce_mgr  g_ras_mce_mgr = {
+	.nb = {
+		.notifier_call = amdgpu_sys_mce_notifier,
+		.priority = MCE_PRIO_UC,
+	},
+};
+
+static int amdgpu_sys_mce_notifier(struct notifier_block *nb, unsigned long val, void *data)
+{
+	struct ras_mce_mgr *mce_mgr = container_of(nb, struct ras_mce_mgr, nb);
+	struct ras_mce_dev *mce_dev;
+	int i;
+
+	if (!data)
+		return NOTIFY_DONE;
+
+	for (i = 0; i < atomic_read(&mce_mgr->dev_count); i++) {
+		mce_dev = &mce_mgr->devs[i];
+		if (mce_dev->dev && mce_dev->dev_notifier)
+			mce_dev->dev_notifier(mce_dev->dev, i, val, data);
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int amdgpu_ras_register_mce_notifier(struct amdgpu_device *adev, mce_notifier notifier)
+{
+	struct ras_mce_mgr *mce_mgr = &g_ras_mce_mgr;
+	struct ras_mce_dev *mce_dev = NULL;
+	int idx, i;
+
+	if (!adev || !notifier)
+		return -EINVAL;
+
+	if (atomic_read(&mce_mgr->dev_count) >= ARRAY_SIZE(mce_mgr->devs))
+		return -ENOENT;
+
+	for (i = 0; i < atomic_read(&mce_mgr->dev_count); i++) {
+		if (mce_mgr->devs[i].dev == adev)
+			return -EEXIST;
+	}
+
+	idx = atomic_fetch_inc(&mce_mgr->dev_count);
+	mce_dev = &mce_mgr->devs[idx];
+	mce_dev->dev = adev;
+	mce_dev->dev_notifier = notifier;
+	atomic_inc(&mce_mgr->ref_count);
+
+	if (!idx)
+		mce_register_decode_chain(&mce_mgr->nb);
+
+	return 0;
+}
+
+static int amdgpu_ras_unregister_mce_notifier(struct amdgpu_device *adev)
+{
+	struct ras_mce_mgr *mce_mgr = &g_ras_mce_mgr;
+	struct ras_mce_dev *mce_dev;
+	int i;
+
+	if (!adev)
+		return -EINVAL;
+
+	for (i = 0; i < atomic_read(&mce_mgr->dev_count); i++) {
+		mce_dev = &mce_mgr->devs[i];
+		if (mce_dev->dev == adev) {
+			mce_dev->dev = NULL;
+			mce_dev->dev_notifier = NULL;
+			atomic_dec(&mce_mgr->ref_count);
+			break;
+		}
+	}
+
+	if (atomic_read(&mce_mgr->ref_count) <= 0) {
+		atomic_set(&mce_mgr->dev_count, 0);
+		mce_unregister_decode_chain(&mce_mgr->nb);
+	}
+
+	return 0;
+}
+
+static void __fill_mce_to_aca_bank(struct amdgpu_device *adev,
+	enum mce_bank_type bank_type, struct mce *m, struct aca_bank_reg *aca_bank)
+{
+	aca_bank->timestamp = m->time;
+	aca_bank->bank_type = bank_type;
+	aca_bank->ecc_type = RAS_ERR_TYPE__MCE;
+	aca_bank->apic_id = m->apicid;
+	aca_bank->bank = m->bank;
+	aca_bank->regs[ACA_REG_IDX__STATUS] = m->status;
+	aca_bank->regs[ACA_REG_IDX__ADDR] = m->addr;
+	aca_bank->regs[ACA_REG_IDX__MISC0] = m->misc;
+	aca_bank->regs[ACA_REG_IDX__IPID] = m->ipid;
+	aca_bank->regs[ACA_REG_IDX__SYND] = m->synd;
+}
+
+static int amdgpu_ras_mce_notifier_v1(struct amdgpu_device *adev, unsigned int id, struct mce *m)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct aca_bank_reg aca_bank = {0};
+	struct aca_bank_ecc err = {0};
+
+	/*
+	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
+	 * and error occurred in DramECC (Extended error code = 0) then only
+	 * process the error, else bail out.
+	 */
+	#ifdef HAVE_SMCA_UMC_V2
+	if (!((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
+			(XEC(m->status, 0x3f) == 0x0)))
+		return 0;
+	#endif
+
+	if (!adev->smuio.funcs || !adev->smuio.funcs->get_socket_id)
+		return 0;
+
+	__fill_mce_to_aca_bank(adev, MCE_BANK_TYPE_GPU, m, &aca_bank);
+
+	if (ras_aca_parse_bank(ras_mgr->ras_core, &aca_bank, &err))
+		return -EINVAL;
+
+	/* GPU device only record bank data that matches its own socket id.*/
+	if (adev->smuio.funcs->get_socket_id(adev) != err.bank_info.socket_id)
+		return 0;
+
+	return ras_mce_add_aca_bank(ras_mgr->ras_core, &aca_bank);
+}
+
+static int amdgpu_ras_mce_notifier_v5(struct amdgpu_device *adev, unsigned int id, struct mce *m)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct aca_bank_reg aca_bank = {0};
+	struct aca_bank_ecc err = {0};
+	enum mce_bank_type bank_type;
+
+	if (!adev->smuio.funcs || !adev->smuio.funcs->get_socket_id) {
+		RAS_DEV_WARN(adev, "No interface to obtain current device socket ID!\n");
+		return 0;
+	}
+
+	if (ras_mce_check_bank(ras_mgr->ras_core, MCE_BANK_TYPE_GPU, m->bank)) {
+		bank_type = MCE_BANK_TYPE_GPU;
+	} else if (ras_mce_check_bank(ras_mgr->ras_core, MCE_BANK_TYPE_CPU, m->bank)) {
+		/* For CPU bank, only the first registered gpu device needs to record bank */
+		if (id)
+			return 0;
+
+		bank_type = MCE_BANK_TYPE_CPU;
+	} else {
+		RAS_DEV_WARN(adev, "Unsupported mce bank: %u\n",  m->bank);
+		return 0;
+	}
+
+	__fill_mce_to_aca_bank(adev, bank_type, m, &aca_bank);
+
+	if (bank_type == MCE_BANK_TYPE_GPU) {
+
+		if (ras_aca_parse_bank(ras_mgr->ras_core, &aca_bank, &err))
+			return -EINVAL;
+
+		/* GPU device only record bank data that matches its own socket id.*/
+		if (adev->smuio.funcs->get_socket_id(adev) != err.bank_info.socket_id)
+			return 0;
+	}
+
+	return ras_mce_add_aca_bank(ras_mgr->ras_core, &aca_bank);
+}
+
+static int amdgpu_ras_mce_notifier(struct amdgpu_device *adev,
+			unsigned int id, unsigned long val, void *data)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	u32 aca_ip_version = 0;
+
+	if (!data || !ras_mgr)
+		return 0;
+
+	if (ras_core_get_ip_version(ras_mgr->ras_core,
+				RAS_UNIT_ID_ACA, &aca_ip_version))
+		return 0;
+
+	switch (aca_ip_version) {
+	case IP_VERSION(1, 0, 0):
+		return amdgpu_ras_mce_notifier_v1(adev, id, data);
+	case IP_VERSION(5, 0, 0):
+		return amdgpu_ras_mce_notifier_v5(adev, id, data);
+	default:
+		RAS_DEV_WARN(adev, "Invalid aca ip version:0x%x\n", aca_ip_version);
+		break;
+	}
+
+	return 0;
+}
+#endif
+
+int amdgpu_ras_mce_hw_init(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
+int amdgpu_ras_mce_hw_fini(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
+int amdgpu_ras_mce_sw_init(struct amdgpu_device *adev)
+{
+#ifdef CONFIG_X86_MCE_AMD
+	amdgpu_ras_register_mce_notifier(adev, amdgpu_ras_mce_notifier);
+#endif
+	return 0;
+}
+
+int amdgpu_ras_mce_sw_fini(struct amdgpu_device *adev)
+{
+#ifdef CONFIG_X86_MCE_AMD
+	amdgpu_ras_unregister_mce_notifier(adev);
+#endif
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.h
new file mode 100644
index 00000000000000..39e8804f89c417
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mce.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMDGPU_RAS_MCE_H__
+#define __AMDGPU_RAS_MCE_H__
+
+int amdgpu_ras_mce_sw_init(struct amdgpu_device *adev);
+int amdgpu_ras_mce_sw_fini(struct amdgpu_device *adev);
+int amdgpu_ras_mce_hw_init(struct amdgpu_device *adev);
+int amdgpu_ras_mce_hw_fini(struct amdgpu_device *adev);
+#endif
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
index e039249b41afed..eb5722a5978632 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
@@ -30,24 +30,22 @@
 #include "amdgpu_virt_ras_cmd.h"
 #include "amdgpu_ras_process.h"
 #include "amdgpu_ras_eeprom_i2c.h"
-#include "amdgpu_ras_mp1_v13_0.h"
+#include "amdgpu_ras_mp1.h"
 #include "amdgpu_ras_nbio_v7_9.h"
+#include "amdgpu_ras_bert.h"
+#include "amdgpu_ras_mce.h"
 
-#define MAX_SOCKET_NUM_PER_HIVE		8
-#define MAX_AID_NUM_PER_SOCKET		4
-#define MAX_XCD_NUM_PER_AID			2
+#define MAX_AID_NUM_PER_SOCKET_GFX9     4
+#define MAX_XCD_NUM_PER_AID_GFX9        2
 
-/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
-#define TYPICAL_ECC_BAD_PAGE_RATE (100ULL * SZ_1M)
-
-#define COUNT_BAD_PAGE_THRESHOLD(size) (((size) >> 21) << 4)
+#define MAX_AID_NUM_PER_SOCKET_GFX12    16
+#define MAX_XCD_NUM_PER_AID_GFX12       4
 
 /* Reserve 8 physical dram row for possible retirement.
  * In worst cases, it will lose 8 * 2MB memory in vram domain
  */
 #define RAS_RESERVED_VRAM_SIZE_DEFAULT	(16ULL << 20)
 
-
 static void ras_mgr_init_event_mgr(struct ras_event_manager *mgr)
 {
 	struct ras_event_state *event_state;
@@ -83,85 +81,53 @@ static void amdgpu_ras_mgr_init_event_mgr(struct ras_core_context *ras_core)
 		amdgpu_put_xgmi_hive(hive);
 }
 
-static int amdgpu_ras_mgr_init_aca_config(struct amdgpu_device *adev,
-		struct ras_core_config *config)
+static int amdgpu_ras_mgr_init_module_param(struct amdgpu_device *adev,
+		struct ras_core_config *config, struct ras_module_param *param)
 {
-	struct ras_aca_config *aca_cfg = &config->aca_cfg;
+	struct ras_module_param *mod_param = &config->mod_param;
 
-	aca_cfg->socket_num_per_hive = MAX_SOCKET_NUM_PER_HIVE;
-	aca_cfg->aid_num_per_socket = MAX_AID_NUM_PER_SOCKET;
-	aca_cfg->xcd_num_per_aid = MAX_XCD_NUM_PER_AID;
+	if (param) {
+		mod_param->ras_feature_enable = param->ras_feature_enable;
+		mod_param->ras_feature_mask = param->ras_feature_mask;
+		mod_param->ras_bad_page_threshold = param->ras_bad_page_threshold;
+	} else {
+		mod_param->ras_feature_enable = -1;
+		mod_param->ras_feature_mask = U64_MAX;
+		mod_param->ras_bad_page_threshold = -1;
+	}
 
 	return 0;
 }
 
-static uint64_t amdgpu_ras_mgr_reserved_vram_size(struct amdgpu_device *adev)
+static int amdgpu_ras_mgr_init_aca_config(struct amdgpu_device *adev,
+		struct ras_core_config *config)
 {
-	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	uint64_t reserved_pages_in_bytes = 0;
-
-	if (!con || (adev->flags & AMD_IS_APU))
-		return 0;
+	struct ras_aca_config *aca_cfg = &config->aca_cfg;
 
-	switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
-	case IP_VERSION(13, 0, 6):
-	case IP_VERSION(13, 0, 12):
-		reserved_pages_in_bytes = RAS_RESERVED_VRAM_SIZE_DEFAULT;
+	switch (config->gfx_ip_version) {
+	case IP_VERSION(9, 4, 3):
+	case IP_VERSION(9, 4, 4):
+	case IP_VERSION(9, 5, 0):
+		aca_cfg->aid_num_per_socket = MAX_AID_NUM_PER_SOCKET_GFX9;
+		aca_cfg->xcd_num_per_aid = MAX_XCD_NUM_PER_AID_GFX9;
 		break;
-	case IP_VERSION(13, 0, 14):
-		reserved_pages_in_bytes = (RAS_RESERVED_VRAM_SIZE_DEFAULT << 1);
+	case IP_VERSION(12, 1, 0):
+		aca_cfg->aid_num_per_socket = MAX_AID_NUM_PER_SOCKET_GFX12;
+		aca_cfg->xcd_num_per_aid = MAX_XCD_NUM_PER_AID_GFX12;
 		break;
 	default:
-		break;
+		return -EINVAL;
 	}
-	return reserved_pages_in_bytes;
+
+	return 0;
 }
 
 static int amdgpu_ras_mgr_init_eeprom_config(struct amdgpu_device *adev,
 		struct ras_core_config *config)
 {
 	struct ras_eeprom_config *eeprom_cfg = &config->eeprom_cfg;
-	uint64_t ras_reserved_vram_size;
 
-	ras_reserved_vram_size = amdgpu_ras_mgr_reserved_vram_size(adev);
 	eeprom_cfg->eeprom_sys_fn = &amdgpu_ras_eeprom_i2c_sys_func;
-	eeprom_cfg->eeprom_i2c_adapter = adev->pm.ras_eeprom_i2c_bus;
-	if (eeprom_cfg->eeprom_i2c_adapter) {
-		const struct i2c_adapter_quirks *quirks =
-			((struct i2c_adapter *)eeprom_cfg->eeprom_i2c_adapter)->quirks;
-
-		if (quirks) {
-			eeprom_cfg->max_i2c_read_len = quirks->max_read_len;
-			eeprom_cfg->max_i2c_write_len = quirks->max_write_len;
-		}
-	}
-
-	/*
-	 * amdgpu_bad_page_threshold is used to config
-	 * the threshold for the number of bad pages.
-	 * -1:  Threshold is set to default value
-	 *      Driver will issue a warning message when threshold is reached
-	 *      and continue runtime services.
-	 * 0:   Disable bad page retirement
-	 *      Driver will not retire bad pages
-	 *      which is intended for debugging purpose.
-	 * -2:  Threshold is determined by a formula
-	 *      that assumes 1 bad page per 100M of local memory.
-	 *      Driver will continue runtime services when threhold is reached.
-	 * 0 < threshold < max number of bad page records in EEPROM,
-	 *      A user-defined threshold is set
-	 *      Driver will halt runtime services when this custom threshold is reached.
-	 */
-	if (amdgpu_bad_page_threshold == NONSTOP_OVER_THRESHOLD)
-		eeprom_cfg->eeprom_record_threshold_count =
-			div64_u64(adev->gmc.mc_vram_size, TYPICAL_ECC_BAD_PAGE_RATE);
-	else if (amdgpu_bad_page_threshold == WARN_NONSTOP_OVER_THRESHOLD)
-		eeprom_cfg->eeprom_record_threshold_count =
-				COUNT_BAD_PAGE_THRESHOLD(ras_reserved_vram_size);
-	else
-		eeprom_cfg->eeprom_record_threshold_count = amdgpu_bad_page_threshold;
-
-	eeprom_cfg->eeprom_record_threshold_config = amdgpu_bad_page_threshold;
 
 	return 0;
 }
@@ -191,7 +157,8 @@ static int amdgpu_ras_mgr_init_mp1_config(struct amdgpu_device *adev,
 	case IP_VERSION(13, 0, 6):
 	case IP_VERSION(13, 0, 14):
 	case IP_VERSION(13, 0, 12):
-		mp1_cfg->mp1_sys_fn = &amdgpu_ras_mp1_sys_func_v13_0;
+	case IP_VERSION(15, 0, 8):
+		mp1_cfg->mp1_sys_fn = &amdgpu_ras_mp1_sys_func;
 		break;
 	default:
 		RAS_DEV_ERR(adev,
@@ -215,6 +182,9 @@ static int amdgpu_ras_mgr_init_nbio_config(struct amdgpu_device *adev,
 	case IP_VERSION(7, 9, 1):
 		nbio_cfg->nbio_sys_fn = &amdgpu_ras_nbio_sys_func_v7_9;
 		break;
+	case IP_VERSION(6, 3, 2):
+		//TBD for dGPU
+		break;
 	default:
 		RAS_DEV_ERR(adev,
 			"The nbio(0x%x) ras config is not right!\n",
@@ -230,42 +200,50 @@ static int amdgpu_ras_mgr_get_ras_psp_system_status(struct ras_core_context *ras
 			struct ras_psp_sys_status *status)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	struct ta_context *context = &adev->psp.ras_context.context;
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 
-	status->initialized = context->initialized;
-	status->session_id = context->session_id;
 	status->psp_cmd_mutex = &adev->psp.mutex;
+	status->uniras_load_fw = con ? con->uniras_load_ras_fw : false;
+
+	status->use_dedicated_memory = !status->uniras_load_fw;
 
 	return 0;
 }
 
-static int amdgpu_ras_mgr_get_ras_ta_init_param(struct ras_core_context *ras_core,
-	struct ras_ta_init_param *ras_ta_param)
+static int amdgpu_ras_mgr_get_ras_param(struct ras_core_context *ras_core,
+	struct ras_param *param)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	uint32_t nps_mode;
+	struct ta_context *context = &adev->psp.ras_context.context;
 
-	if (amdgpu_ras_is_poison_mode_supported(adev))
-		ras_ta_param->poison_mode_en = 1;
+	if (!param)
+		return -EINVAL;
 
 	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
-		ras_ta_param->dgpu_mode = 1;
+		param->ta_param.dgpu_mode = 1;
+
+	param->ta_param.xcc_mask = adev->gfx.xcc_mask;
+	param->ta_param.channel_dis_num = hweight32(adev->gmc.m_half_use) * 2;
 
-	ras_ta_param->xcc_mask = adev->gfx.xcc_mask;
-	ras_ta_param->channel_dis_num = hweight32(adev->gmc.m_half_use) * 2;
+	param->ta_param.active_umc_mask = lower_32_bits(adev->umc.active_mask);
+	param->ta_param.ext_umc_mask = upper_32_bits(adev->umc.active_mask);
 
-	ras_ta_param->active_umc_mask = adev->umc.active_mask;
-	ras_ta_param->vram_type = (uint8_t)adev->gmc.vram_type;
+	param->fw_param.rl_bin.fw_version = adev->psp.rl.fw_version;
+	param->fw_param.rl_bin.feature_version = adev->psp.rl.feature_version;
+	param->fw_param.rl_bin.bin_size = adev->psp.rl.size_bytes;
+	param->fw_param.rl_bin.bin_addr = adev->psp.rl.start_addr;
 
-	if (!amdgpu_ras_mgr_get_curr_nps_mode(adev, &nps_mode))
-		ras_ta_param->nps_mode = nps_mode;
+	param->fw_param.ta_bin.fw_version = context->bin_desc.fw_version;
+	param->fw_param.ta_bin.feature_version = context->bin_desc.feature_version;
+	param->fw_param.ta_bin.bin_size = context->bin_desc.size_bytes;
+	param->fw_param.ta_bin.bin_addr = context->bin_desc.start_addr;
 
 	return 0;
 }
 
 const struct ras_psp_sys_func amdgpu_ras_psp_sys_func = {
 	.get_ras_psp_system_status = amdgpu_ras_mgr_get_ras_psp_system_status,
-	.get_ras_ta_init_param = amdgpu_ras_mgr_get_ras_ta_init_param,
+	.get_ras_param = amdgpu_ras_mgr_get_ras_param,
 };
 
 static int amdgpu_ras_mgr_init_psp_config(struct amdgpu_device *adev,
@@ -283,13 +261,15 @@ static int amdgpu_ras_mgr_init_umc_config(struct amdgpu_device *adev,
 {
 	struct ras_umc_config *umc_cfg = &config->umc_cfg;
 
-	umc_cfg->umc_vram_type = adev->gmc.vram_type;
 	umc_cfg->num_umc = adev->gmc.num_umc;
+	umc_cfg->lfb_size = adev->gmc.xgmi.node_segment_size;
+	umc_cfg->pa_base = amdgpu_xgmi_get_relative_phy_addr(adev, 0x0ULL);
 
 	return 0;
 }
 
-static struct ras_core_context *amdgpu_ras_mgr_create_ras_core(struct amdgpu_device *adev)
+static struct ras_core_context *amdgpu_ras_mgr_create_ras_core(struct amdgpu_device *adev,
+			struct ras_module_param *param)
 {
 	struct ras_core_config init_config;
 
@@ -299,12 +279,17 @@ static struct ras_core_context *amdgpu_ras_mgr_create_ras_core(struct amdgpu_dev
 	init_config.mp1_ip_version = amdgpu_ip_version(adev, MP1_HWIP, 0);
 	init_config.gfx_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
 	init_config.nbio_ip_version = amdgpu_ip_version(adev, NBIO_HWIP, 0);
-	init_config.psp_ip_version = amdgpu_ip_version(adev, MP1_HWIP, 0);
+	init_config.psp_ip_version = amdgpu_ip_version(adev, MP0_HWIP, 0);
 
-	if (init_config.umc_ip_version == IP_VERSION(12, 0, 0) ||
+	if (init_config.gfx_ip_version == IP_VERSION(12, 1, 0))
+		init_config.aca_ip_version = IP_VERSION(5, 0, 0);
+	else if (init_config.umc_ip_version == IP_VERSION(12, 0, 0) ||
 	    init_config.umc_ip_version == IP_VERSION(12, 5, 0))
 		init_config.aca_ip_version = IP_VERSION(1, 0, 0);
 
+	if (init_config.mp1_ip_version == IP_VERSION(15, 0, 8))
+		init_config.early_init_service_supported = true;
+
 	init_config.sys_fn = &amdgpu_ras_sys_fn;
 	init_config.ras_eeprom_supported =
 		amdgpu_ras_mgr_eeprom_is_supported(adev);
@@ -312,6 +297,7 @@ static struct ras_core_context *amdgpu_ras_mgr_create_ras_core(struct amdgpu_dev
 		amdgpu_ras_is_poison_mode_supported(adev);
 	init_config.ras_debug_mask = amdgpu_debug_mask;
 
+	amdgpu_ras_mgr_init_module_param(adev, &init_config, param);
 	amdgpu_ras_mgr_init_aca_config(adev, &init_config);
 	amdgpu_ras_mgr_init_eeprom_config(adev, &init_config);
 	amdgpu_ras_mgr_init_mp1_config(adev, &init_config);
@@ -322,31 +308,42 @@ static struct ras_core_context *amdgpu_ras_mgr_create_ras_core(struct amdgpu_dev
 	return ras_core_create(&init_config);
 }
 
-static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
+int amdgpu_ras_mgr_sw_init(struct amdgpu_device *adev, struct ras_module_param *param)
 {
-	struct amdgpu_device *adev = ip_block->adev;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct amdgpu_ras_mgr *ras_mgr;
 	int ret = 0;
 
+	if (!con)
+		return 0;
+
 	/* Disabled by default */
 	con->uniras_enabled = false;
 
-	if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) ||
+	/* Disable ras via driver installation option */
+	if (param && !param->ras_feature_enable)
+		return 0;
+	else if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) ||
 	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
-	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6))
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 8))
 		con->uniras_enabled = true;
 	else
 		return 0;
 
+	/* Enable uniras to load RL firmware and TA firmware */
+	con->uniras_load_ras_fw = true;
+
 	ras_mgr = kzalloc_obj(*ras_mgr);
-	if (!ras_mgr)
-		return -EINVAL;
+	if (!ras_mgr) {
+		ret = -EINVAL;
+		goto err1;
+	}
 
 	con->ras_mgr = ras_mgr;
 	ras_mgr->adev = adev;
 
-	ras_mgr->ras_core = amdgpu_ras_mgr_create_ras_core(adev);
+	ras_mgr->ras_core = amdgpu_ras_mgr_create_ras_core(adev, param);
 	if (!ras_mgr->ras_core) {
 		RAS_DEV_ERR(adev, "Failed to create ras core!\n");
 		ret = -EINVAL;
@@ -362,6 +359,8 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 		goto err2;
 	}
 	amdgpu_ras_mgr_init_event_mgr(ras_mgr->ras_core);
+	amdgpu_ras_bert_sw_init(adev);
+	amdgpu_ras_mce_sw_init(adev);
 
 	if (amdgpu_sriov_vf(adev)) {
 		ret = amdgpu_virt_ras_sw_init(adev);
@@ -375,6 +374,8 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 	return 0;
 
 err3:
+	amdgpu_ras_mce_sw_fini(adev);
+	amdgpu_ras_bert_sw_fini(adev);
 	if (ras_mgr->ras_core)
 		ras_core_sw_fini(ras_mgr->ras_core);
 err2:
@@ -384,24 +385,28 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 err1:
 	kfree(ras_mgr);
 	con->ras_mgr = NULL;
+	con->uniras_enabled = false;
+	con->uniras_load_ras_fw = false;
 	return ret;
 }
 
-static int amdgpu_ras_mgr_sw_fini(struct amdgpu_ip_block *ip_block)
+int amdgpu_ras_mgr_sw_fini(struct amdgpu_device *adev)
 {
-	struct amdgpu_device *adev = ip_block->adev;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	struct amdgpu_ras_mgr *ras_mgr = (struct amdgpu_ras_mgr *)con->ras_mgr;
+	struct amdgpu_ras_mgr *ras_mgr;
 
-	if (!con->uniras_enabled)
+	if (!con || !con->uniras_enabled)
 		return 0;
 
+	ras_mgr = amdgpu_ras_mgr_get_context(adev);
 	if (!ras_mgr)
 		return 0;
 
 	if (amdgpu_sriov_vf(adev))
 		amdgpu_virt_ras_sw_fini(adev);
 
+	amdgpu_ras_mce_sw_fini(adev);
+	amdgpu_ras_bert_sw_fini(adev);
 	amdgpu_ras_process_fini(adev);
 	ras_core_sw_fini(ras_mgr->ras_core);
 	ras_core_destroy(ras_mgr->ras_core);
@@ -417,12 +422,13 @@ static int amdgpu_ras_mgr_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_ras_mgr *ras_mgr;
 	int ret;
 
-	if (!con->uniras_enabled)
+	if (!con || !con->uniras_enabled)
 		return 0;
 
+	ras_mgr = amdgpu_ras_mgr_get_context(adev);
 	if (!ras_mgr || !ras_mgr->ras_core)
 		return -EINVAL;
 
@@ -436,6 +442,8 @@ static int amdgpu_ras_mgr_hw_init(struct amdgpu_ip_block *ip_block)
 		return ret;
 	}
 
+	amdgpu_ras_mce_hw_init(adev);
+
 	ras_mgr->ras_is_ready = true;
 
 	amdgpu_enable_uniras(adev, true);
@@ -448,11 +456,12 @@ static int amdgpu_ras_mgr_hw_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
-	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct amdgpu_ras_mgr *ras_mgr;
 
-	if (!con->uniras_enabled)
+	if (!con || !con->uniras_enabled)
 		return 0;
 
+	ras_mgr = amdgpu_ras_mgr_get_context(adev);
 	if (!ras_mgr || !ras_mgr->ras_core)
 		return -EINVAL;
 
@@ -461,6 +470,7 @@ static int amdgpu_ras_mgr_hw_fini(struct amdgpu_ip_block *ip_block)
 	else
 		ras_core_hw_fini(ras_mgr->ras_core);
 
+	amdgpu_ras_mce_hw_fini(adev);
 	ras_mgr->ras_is_ready = false;
 
 	return 0;
@@ -496,12 +506,41 @@ struct amdgpu_ras_mgr *amdgpu_ras_mgr_get_context(struct amdgpu_device *adev)
 	return (struct amdgpu_ras_mgr *)adev->psp.ras_context.ras->ras_mgr;
 }
 
+static int amdgpu_ras_mgr_suspend(struct amdgpu_ip_block *ip_block)
+{
+	return 0;
+}
+
+static int amdgpu_ras_mgr_resume(struct amdgpu_ip_block *ip_block)
+{
+	struct amdgpu_device *adev = ip_block->adev;
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	int ret;
+
+	if (!ras_mgr || !ras_mgr->ras_core)
+		return 0;
+
+	/* Nothing was loaded, so nothing to reload. The hw_init has to run
+	 * from amdgpu_ras_resume_after_reset() instead: it reads the EEPROM,
+	 * which takes the reset domain lock that reset on init still holds.
+	 */
+	if (!ras_mgr->ras_is_ready)
+		return 0;
+
+	ret = ras_psp_reload_firmwares(ras_mgr->ras_core, 0);
+	if (ret)
+		RAS_DEV_ERR(adev,
+			    "Failed to reload RAS TA on resume, ret:%d\n", ret);
+
+	return ret;
+}
+
 static const struct amd_ip_funcs __maybe_unused ras_v1_0_ip_funcs = {
 	.name = "ras_v1_0",
-	.sw_init = amdgpu_ras_mgr_sw_init,
-	.sw_fini = amdgpu_ras_mgr_sw_fini,
 	.hw_init = amdgpu_ras_mgr_hw_init,
 	.hw_fini = amdgpu_ras_mgr_hw_fini,
+	.suspend = amdgpu_ras_mgr_suspend,
+	.resume = amdgpu_ras_mgr_resume,
 };
 
 const struct amdgpu_ip_block_version ras_v1_0_ip_block = {
@@ -582,35 +621,6 @@ uint64_t amdgpu_ras_mgr_gen_ras_event_seqno(struct amdgpu_device *adev,
 	return seq_no;
 }
 
-int amdgpu_ras_mgr_handle_controller_interrupt(struct amdgpu_device *adev, void *data)
-{
-	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
-	struct ras_ih_info *ih_info = (struct ras_ih_info *)data;
-	uint64_t seq_no = 0;
-	int ret = 0;
-
-	if (!amdgpu_ras_mgr_is_ready(adev))
-		return -EPERM;
-
-	if (ih_info && (ih_info->block == AMDGPU_RAS_BLOCK__UMC)) {
-		if (ras_mgr->ras_core->poison_supported) {
-			seq_no = amdgpu_ras_mgr_gen_ras_event_seqno(adev, RAS_SEQNO_TYPE_DE);
-			RAS_DEV_INFO(adev,
-				"{%llu} RAS poison is created, no user action is needed.\n",
-				seq_no);
-		}
-
-		ret = amdgpu_ras_process_handle_umc_interrupt(adev, ih_info);
-	} else if (ras_mgr->ras_core->poison_supported) {
-		ret = amdgpu_ras_process_handle_unexpected_interrupt(adev, ih_info);
-	} else {
-		RAS_DEV_WARN(adev,
-			"No RAS interrupt handler for non-UMC block with poison disabled.\n");
-	}
-
-	return ret;
-}
-
 int amdgpu_ras_mgr_dispatch_interrupt(struct amdgpu_device *adev, struct ras_ih_info *ih_info)
 {
 	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
@@ -624,7 +634,7 @@ int amdgpu_ras_mgr_dispatch_interrupt(struct amdgpu_device *adev, struct ras_ih_
 		return 0;
 
 	if (ih_info->block == RAS_BLOCK_ID__UMC) {
-		if (ras_mgr->ras_core->poison_supported) {
+		if (ras_core_poison_supported(ras_mgr->ras_core)) {
 			seq_no = amdgpu_ras_mgr_gen_ras_event_seqno(adev, RAS_SEQNO_TYPE_DE);
 			RAS_DEV_INFO(adev,
 				"{%llu} RAS poison is created, no user action is needed.\n",
@@ -632,7 +642,7 @@ int amdgpu_ras_mgr_dispatch_interrupt(struct amdgpu_device *adev, struct ras_ih_
 		}
 
 		ret = amdgpu_ras_process_handle_umc_interrupt(adev, ih_info);
-	} else if (ras_mgr->ras_core->poison_supported) {
+	} else if (ras_core_poison_supported(ras_mgr->ras_core)) {
 		ret = amdgpu_ras_process_handle_consumption_interrupt(adev, ih_info);
 	} else {
 		RAS_DEV_WARN(adev,
@@ -642,14 +652,6 @@ int amdgpu_ras_mgr_dispatch_interrupt(struct amdgpu_device *adev, struct ras_ih_
 	return ret;
 }
 
-int amdgpu_ras_mgr_handle_consumer_interrupt(struct amdgpu_device *adev, void *data)
-{
-	if (!amdgpu_ras_mgr_is_ready(adev))
-		return -EPERM;
-
-	return amdgpu_ras_process_handle_consumption_interrupt(adev, data);
-}
-
 int amdgpu_ras_mgr_update_ras_ecc(struct amdgpu_device *adev)
 {
 	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
@@ -678,10 +680,7 @@ bool amdgpu_ras_mgr_check_eeprom_safety_watermark(struct amdgpu_device *adev)
 	if (!amdgpu_ras_mgr_is_ready(adev))
 		return false;
 
-	if (ras_fw_eeprom_supported(ras_mgr->ras_core))
-		return ras_fw_eeprom_check_safety_watermark(ras_mgr->ras_core);
-
-	return ras_eeprom_check_safety_watermark(ras_mgr->ras_core);
+	return ras_core_check_safety_watermark(ras_mgr->ras_core);
 }
 
 int amdgpu_ras_mgr_get_curr_nps_mode(struct amdgpu_device *adev,
@@ -782,6 +781,33 @@ int amdgpu_ras_mgr_post_reset(struct amdgpu_device *adev)
 	return 0;
 }
 
+int amdgpu_ras_mgr_enable_feature(struct amdgpu_device *adev,
+		uint32_t ta_block_id, uint32_t ta_error_type, bool enable)
+{
+	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	struct ras_ta_enable_features_input info = {
+		.block_id = (enum ras_ta_block)ta_block_id,
+		.error_type = (enum ras_ta_error_type)ta_error_type,
+	};
+	uint32_t gfx_ip_version;
+
+	if (!ras_mgr || !ras_mgr->ras_core)
+		return -EINVAL;
+
+	if (ras_core_get_ip_version(ras_mgr->ras_core,
+				    RAS_UNIT_ID_GFX, &gfx_ip_version))
+		return -EPERM;
+
+	/* Newer parts arm RAS features inside the TA, so the driver must not
+	 * toggle them. Older ones still need an explicit request after every
+	 * GPU reset, otherwise GFX stops reporting poison consumption.
+	 */
+	if (gfx_ip_version >= IP_VERSION(12, 1, 0))
+		return 0;
+
+	return ras_psp_enable_features(ras_mgr->ras_core, &info, enable);
+}
+
 int amdgpu_ras_mgr_lookup_bad_pages_in_a_row(struct amdgpu_device *adev,
 		uint64_t addr, uint64_t *nps_page_addr, uint32_t max_page_count)
 {
@@ -802,8 +828,9 @@ int amdgpu_ras_mgr_set_debug_mode(struct amdgpu_device *adev, bool enable)
 	struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev);
 	int ret;
 
-	if (!ras_mgr || !ras_mgr->ras_core || !ras_mgr->ras_is_ready)
-		return false;
+	/* this only talks to PMFW, so it does not wait for the RAS block */
+	if (!ras_mgr || !ras_mgr->ras_core)
+		return -EINVAL;
 
 	ret = ras_core_set_debug_mode(ras_mgr->ras_core, enable);
 	if (!ret)
@@ -821,3 +848,25 @@ bool amdgpu_ras_mgr_get_debug_mode(struct amdgpu_device *adev)
 
 	return ras_mgr->is_debug_mode;
 }
+
+int amdgpu_ras_mgr_early_init_service(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct amdgpu_ras_mgr *ras_mgr;
+	int ret;
+
+	if (!con || !con->uniras_enabled)
+		return 0;
+
+	ras_mgr = amdgpu_ras_mgr_get_context(adev);
+	if (!ras_mgr || !ras_mgr->ras_core) {
+		RAS_DEV_ERR(adev, "amdgpu ras sw is not ready!\n");
+		return -EPERM;
+	}
+
+	ret = ras_core_eeprom_early_init_service(ras_mgr->ras_core);
+	if (ret)
+		RAS_DEV_WARN(adev, "RAS early init service failure! ret:%d\n", ret);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
index b81fd88fca7114..99b3e6995e12b8 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.h
@@ -61,13 +61,15 @@ struct amdgpu_ras_mgr {
 
 extern const struct amdgpu_ip_block_version ras_v1_0_ip_block;
 
+int amdgpu_ras_mgr_sw_init(struct amdgpu_device *adev,
+		struct ras_module_param *param);
+int amdgpu_ras_mgr_sw_fini(struct amdgpu_device *adev);
+int amdgpu_ras_mgr_early_init_service(struct amdgpu_device *adev);
 struct amdgpu_ras_mgr *amdgpu_ras_mgr_get_context(
 			struct amdgpu_device *adev);
 int amdgpu_enable_uniras(struct amdgpu_device *adev, bool enable);
 bool amdgpu_uniras_enabled(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_handle_fatal_interrupt(struct amdgpu_device *adev, void *data);
-int amdgpu_ras_mgr_handle_controller_interrupt(struct amdgpu_device *adev, void *data);
-int amdgpu_ras_mgr_handle_consumer_interrupt(struct amdgpu_device *adev, void *data);
 int amdgpu_ras_mgr_dispatch_interrupt(struct amdgpu_device *adev, struct ras_ih_info *ih_info);
 int amdgpu_ras_mgr_update_ras_ecc(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_reset_gpu(struct amdgpu_device *adev, uint32_t flags);
@@ -83,6 +85,8 @@ int amdgpu_ras_mgr_handle_ras_cmd(struct amdgpu_device *adev,
 		void *output, uint32_t out_size);
 int amdgpu_ras_mgr_pre_reset(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_post_reset(struct amdgpu_device *adev);
+int amdgpu_ras_mgr_enable_feature(struct amdgpu_device *adev,
+		uint32_t ta_block_id, uint32_t ta_error_type, bool enable);
 int amdgpu_ras_mgr_resume_after_reset(struct amdgpu_device *adev);
 int amdgpu_ras_mgr_lookup_bad_pages_in_a_row(struct amdgpu_device *adev,
 		uint64_t addr, uint64_t *nps_page_addr, uint32_t max_page_count);
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.c
new file mode 100644
index 00000000000000..7ca2f33f2cf041
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu_smu.h"
+#include "amdgpu_reset.h"
+#include "ras.h"
+
+static const enum smu_feature_mask ras_smu_feature_masks[RAS_MP1_FEATURE_MAX] = {
+	[RAS_MP1_FEATURE_PMFW_EEPROM] = SMU_FEATURE_HROM_EN_BIT,
+};
+
+static const enum smu_message_type ras_smu_msg_maps[RAS_MP1_MSG_MAX] = {
+	[RAS_MP1_MSG_QueryValidMcaCount] = SMU_MSG_QueryValidMcaCount,
+	[RAS_MP1_MSG_McaBankDumpDW] = SMU_MSG_McaBankDumpDW,
+	[RAS_MP1_MSG_ClearMcaOnRead] = SMU_MSG_ClearMcaOnRead,
+	[RAS_MP1_MSG_QueryValidMcaCeCount] = SMU_MSG_QueryValidMcaCeCount,
+	[RAS_MP1_MSG_McaBankCeDumpDW] = SMU_MSG_McaBankCeDumpDW,
+	[RAS_MP1_MSG_GetRasTableVersion] = SMU_MSG_GetRASTableVersion,
+	[RAS_MP1_MSG_GetRmaStatus] = SMU_MSG_GetRmaStatus,
+	[RAS_MP1_MSG_GetBadPageCount] = SMU_MSG_GetBadPageCount,
+	[RAS_MP1_MSG_GetBadPageMcaAddr] = SMU_MSG_GetBadPageMcaAddr,
+	[RAS_MP1_MSG_SetTimestamp] = SMU_MSG_SetTimestamp,
+	[RAS_MP1_MSG_GetTimestamp] = SMU_MSG_GetTimestamp,
+	[RAS_MP1_MSG_GetBadPageIpId] = SMU_MSG_GetBadPageIpid,
+	[RAS_MP1_MSG_EraseRasTable] = SMU_MSG_EraseRasTable,
+};
+
+static enum smu_message_type
+	__get_ras_smu_msg(struct ras_core_context *ras_core, u32 msg_id)
+{
+	if (msg_id >= RAS_MP1_MSG_MAX)
+		return 0;
+
+	return ras_smu_msg_maps[msg_id];
+}
+
+static int amdgpu_ras_check_ras_feature_status(struct ras_core_context *ras_core,
+		enum ras_mp1_feature_id feature_id, u32 *status)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+	u32 res = 0;
+
+	if (!status || (feature_id >= RAS_MP1_FEATURE_MAX))
+		return -EINVAL;
+
+	if (down_read_trylock(&adev->reset_domain->sem)) {
+		if (amdgpu_smu_ras_feature_is_enabled(adev,
+				ras_smu_feature_masks[feature_id]))
+			res = 1;
+		up_read(&adev->reset_domain->sem);
+	} else {
+		return -RAS_CORE_GPU_IN_MODE1_RESET;
+	}
+
+	*status = res;
+
+	return 0;
+}
+
+static int amdgpu_ras_send_mp1_msg(struct ras_core_context *ras_core, u32 msg_id,
+		u32 *params, u32 num_params, u32 *read_args, u32 num_read_args)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+	enum smu_message_type smu_msg;
+	int ret = 0;
+
+	smu_msg = __get_ras_smu_msg(ras_core, msg_id);
+	if (!smu_msg)
+		return -EOPNOTSUPP;
+
+	if (down_read_trylock(&adev->reset_domain->sem)) {
+		ret = amdgpu_smu_ras_send_msg(adev, smu_msg,
+				params, num_params, read_args, num_read_args);
+		up_read(&adev->reset_domain->sem);
+	} else {
+		ret = -RAS_CORE_GPU_IN_MODE1_RESET;
+	}
+
+	return ret;
+}
+
+const struct ras_mp1_sys_func amdgpu_ras_mp1_sys_func = {
+	.mp1_send_ras_msg = amdgpu_ras_send_mp1_msg,
+	.check_ras_feature_status = amdgpu_ras_check_ras_feature_status,
+};
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.h
new file mode 100644
index 00000000000000..fa7231b03ccdf1
--- /dev/null
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMDGPU_RAS_MP1_H__
+#define __AMDGPU_RAS_MP1_H__
+#include "ras.h"
+
+extern const struct ras_mp1_sys_func amdgpu_ras_mp1_sys_func;
+
+#endif
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
deleted file mode 100644
index 3c4575a5d902d7..00000000000000
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.c
+++ /dev/null
@@ -1,154 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright 2025 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-#include "amdgpu_smu.h"
-#include "amdgpu_reset.h"
-#include "amdgpu_ras_mp1_v13_0.h"
-#include "smu13_driver_if_v13_0_6.h"
-
-#define RAS_MP1_MSG_QueryValidMcaCeCount  0x3A
-#define RAS_MP1_MSG_McaBankCeDumpDW       0x3B
-
-static const enum smu_message_type pmfw_eeprom_msgs[] = {
-	[RAS_SMU_GetRASTableVersion] = SMU_MSG_GetRASTableVersion,
-	[RAS_SMU_GetBadPageCount] = SMU_MSG_GetBadPageCount,
-	[RAS_SMU_SetTimestamp] = SMU_MSG_SetTimestamp,
-	[RAS_SMU_GetTimestamp] = SMU_MSG_GetTimestamp,
-	[RAS_SMU_GetBadPageIpid] = SMU_MSG_GetBadPageIpid,
-	[RAS_SMU_EraseRasTable] = SMU_MSG_EraseRasTable,
-	[RAS_SMU_GetBadPageMcaAddr] = SMU_MSG_GetBadPageMcaAddr,
-};
-
-static int mp1_v13_0_get_valid_bank_count(struct ras_core_context *ras_core,
-					  u32 msg, u32 *count)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	u32 smu_msg;
-	int ret = 0;
-
-	if (!count)
-		return -EINVAL;
-
-	smu_msg = (msg == RAS_MP1_MSG_QueryValidMcaCeCount) ?
-			SMU_MSG_QueryValidMcaCeCount : SMU_MSG_QueryValidMcaCount;
-
-	if (down_read_trylock(&adev->reset_domain->sem)) {
-		ret = amdgpu_smu_ras_send_msg(adev, smu_msg, 0, count);
-		up_read(&adev->reset_domain->sem);
-	} else {
-		ret = -RAS_CORE_GPU_IN_MODE1_RESET;
-	}
-
-	if (ret)
-		*count = 0;
-
-	return ret;
-}
-
-static int mp1_v13_0_dump_valid_bank(struct ras_core_context *ras_core,
-				     u32 msg, u32 idx, u32 reg_idx, u64 *val)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	uint32_t data[2] = {0, 0};
-	uint32_t param;
-	int ret = 0;
-	int i, offset;
-	u32 smu_msg = (msg == RAS_MP1_MSG_McaBankCeDumpDW) ?
-			     SMU_MSG_McaBankCeDumpDW : SMU_MSG_McaBankDumpDW;
-
-	if (down_read_trylock(&adev->reset_domain->sem)) {
-		offset = reg_idx * 8;
-		for (i = 0; i < ARRAY_SIZE(data); i++) {
-			param = ((idx & 0xffff) << 16) | ((offset + (i << 2)) & 0xfffc);
-			ret = amdgpu_smu_ras_send_msg(adev, smu_msg, param, &data[i]);
-			if (ret) {
-				RAS_DEV_ERR(adev, "ACA failed to read register[%d], offset:0x%x\n",
-					reg_idx, offset);
-				break;
-			}
-		}
-		up_read(&adev->reset_domain->sem);
-
-		if (!ret)
-			*val = (uint64_t)data[1] << 32 | data[0];
-	} else {
-		ret = -RAS_CORE_GPU_IN_MODE1_RESET;
-	}
-
-	return ret;
-}
-
-static int mp1_v13_0_eeprom_send_msg(struct ras_core_context *ras_core,
-				enum ras_fw_eeprom_cmd index, uint32_t param, uint32_t *read_arg)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	int ret = 0;
-
-	if (down_read_trylock(&adev->reset_domain->sem)) {
-		ret = amdgpu_smu_ras_send_msg(adev,
-			pmfw_eeprom_msgs[index], param, read_arg);
-		up_read(&adev->reset_domain->sem);
-	} else {
-		ret = -RAS_CORE_GPU_IN_MODE1_RESET;
-	}
-
-	return ret;
-}
-
-static int mp1_v13_0_get_ras_enabled_mask(struct ras_core_context *ras_core,
-					     uint64_t *enabled_mask)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	int ret = 0;
-
-	if (down_read_trylock(&adev->reset_domain->sem)) {
-		if (amdgpu_smu_ras_feature_is_enabled(adev, SMU_FEATURE_HROM_EN_BIT))
-			*enabled_mask |= RAS_CORE_FW_FEATURE_BIT__RAS_EEPROM;
-		up_read(&adev->reset_domain->sem);
-	} else {
-		ret = -RAS_CORE_GPU_IN_MODE1_RESET;
-	}
-
-	return ret;
-}
-
-static int mp1_v13_0_set_debug_mode(struct ras_core_context *ras_core, bool enable)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
-	int ret;
-	u32 smu_msg = SMU_MSG_ClearMcaOnRead;
-
-	ret = amdgpu_smu_ras_send_msg(adev, smu_msg,
-			enable ? 0 : ClearMcaOnRead_UE_FLAG_MASK |
-			ClearMcaOnRead_CE_POLL_MASK, NULL);
-	return ret;
-}
-
-const struct ras_mp1_sys_func amdgpu_ras_mp1_sys_func_v13_0 = {
-	.mp1_get_valid_bank_count = mp1_v13_0_get_valid_bank_count,
-	.mp1_dump_valid_bank = mp1_v13_0_dump_valid_bank,
-	.mp1_send_eeprom_msg = mp1_v13_0_eeprom_send_msg,
-	.mp1_get_ras_enabled_mask = mp1_v13_0_get_ras_enabled_mask,
-	.mp1_set_debug_mode = mp1_v13_0_set_debug_mode,
-};
-
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.h b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.h
deleted file mode 100644
index 71c614ae1ae455..00000000000000
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mp1_v13_0.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright 2025 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef __AMDGPU_RAS_MP1_V13_0_H__
-#define __AMDGPU_RAS_MP1_V13_0_H__
-#include "ras.h"
-
-extern const struct ras_mp1_sys_func amdgpu_ras_mp1_sys_func_v13_0;
-
-#endif
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c
index 285d81a85a99d6..9dd44fb5b885f5 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_process.c
@@ -89,6 +89,7 @@ int amdgpu_ras_process_handle_umc_interrupt(struct amdgpu_device *adev, void *da
 
 int amdgpu_ras_process_handle_unexpected_interrupt(struct amdgpu_device *adev, void *data)
 {
+	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
 	amdgpu_ras_set_fed(adev, true);
 	return amdgpu_ras_mgr_reset_gpu(adev, AMDGPU_RAS_GPU_RESET_MODE1_RESET);
 }
@@ -99,11 +100,17 @@ int amdgpu_ras_process_handle_consumption_interrupt(struct amdgpu_device *adev,
 	struct ras_ih_info *ih_info = (struct ras_ih_info *)data;
 	struct ras_event_req req;
 	uint64_t seqno;
+	uint32_t gfx_ip_version;
 
 	if (!ih_info)
 		return -EINVAL;
 
-	if (amdgpu_sriov_vf(adev)) {
+	if (ras_core_get_ip_version(ras_mgr->ras_core,
+				RAS_UNIT_ID_GFX, &gfx_ip_version))
+		return -EPERM;
+
+	if (amdgpu_sriov_vf(adev) &&
+		gfx_ip_version < IP_VERSION(12, 1, 0)) {
 		if (adev->virt.ops && adev->virt.ops->ras_poison_handler)
 			adev->virt.ops->ras_poison_handler(adev, ih_info->block);
 		else
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
index 0b36b689fcfc13..291b2a96cbb508 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_sys.c
@@ -33,8 +33,14 @@ static int amdgpu_ras_sys_detect_fatal_event(struct ras_core_context *ras_core,
 	uint64_t seq_no;
 
 	ret = amdgpu_ras_global_ras_isr(adev);
-	if (ret)
-		return ret;
+	if (ret) {
+		/* Another device of the hive already asked for the reset, this
+		 * one still has to record that it saw the error.
+		 */
+		kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+		amdgpu_ras_set_fed(adev, true);
+		return ret == -EBUSY ? 0 : ret;
+	}
 
 	seq_no = amdgpu_ras_mgr_gen_ras_event_seqno(adev, RAS_SEQNO_TYPE_UE);
 	RAS_DEV_INFO(adev,
@@ -54,6 +60,8 @@ static int amdgpu_ras_sys_poison_consumption_event(struct ras_core_context *ras_
 	if (!req)
 		return -EINVAL;
 
+	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+
 	if (req->pasid_fn) {
 		pasid_fn = (pasid_notify)req->pasid_fn;
 		pasid_fn(adev, req->pasid, req->data);
@@ -110,6 +118,25 @@ static int amdgpu_ras_sys_gen_seqno(struct ras_core_context *ras_core,
 
 }
 
+static int amdgpu_ras_early_init_reserve_badpage(struct ras_core_context *ras_core,
+			uint64_t pfn)
+{
+	struct amdgpu_bo *bo;
+	void *va;
+	int ret;
+
+	ret = amdgpu_bo_create_kernel_at(ras_core->dev, RAS_PFN_TO_ADDR(pfn),
+			AMDGPU_GPU_PAGE_SIZE, &bo, &va);
+	if (ret) {
+		ras_core->is_rma = true;
+		RAS_DEV_ERR(ras_core->dev,
+			"Early init: RAS failed to reserve: offset=0x%llx size=0x%x ret=%d\n",
+			RAS_PFN_TO_ADDR(pfn), AMDGPU_GPU_PAGE_SIZE, ret);
+	}
+
+	return ret;
+}
+
 static int amdgpu_ras_sys_event_notifier(struct ras_core_context *ras_core,
 				   enum ras_notify_event event_id, void *data)
 {
@@ -126,6 +153,9 @@ static int amdgpu_ras_sys_event_notifier(struct ras_core_context *ras_core,
 	case RAS_EVENT_ID__RESERVE_BAD_PAGE:
 		ret = amdgpu_ras_reserve_page(ras_core->dev, *(uint64_t *)data);
 		break;
+	case RAS_EVENT_ID__EARLY_INIT_RESERVE_PAGE:
+		ret = amdgpu_ras_early_init_reserve_badpage(ras_core, *(uint64_t *)data);
+		break;
 	case RAS_EVENT_ID__FATAL_ERROR_DETECTED:
 		ret = amdgpu_ras_sys_detect_fatal_event(ras_core, data);
 		break;
@@ -136,9 +166,8 @@ static int amdgpu_ras_sys_event_notifier(struct ras_core_context *ras_core,
 		ret = amdgpu_dpm_send_hbm_bad_channel_flag(ras_core->dev, *(uint32_t *)data);
 		break;
 	case RAS_EVENT_ID__DEVICE_RMA:
-		ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_RMA, NULL, NULL);
-		if (!ras_fw_eeprom_supported(ras_core))
-			ret = amdgpu_dpm_send_rma_reason(ras_core->dev);
+		ras_log_ring_add_log_event(ras_core, RAS_LOG_EVENT_RMA, NULL, 0, NULL);
+		ret = amdgpu_dpm_send_rma_reason(ras_core->dev);
 		break;
 	case RAS_EVENT_ID__RESET_GPU:
 		ret = amdgpu_ras_mgr_reset_gpu(ras_core->dev, *(uint32_t *)data);
@@ -149,6 +178,10 @@ static int amdgpu_ras_sys_event_notifier(struct ras_core_context *ras_core,
 	case RAS_EVENT_ID__RAS_EVENT_PROC_END:
 		ret = amdgpu_ras_process_end(ras_core->dev);
 		break;
+	case RAS_EVENT_ID__UPDATE_ACA_DATA:
+		/* TBD: ACA error detected */
+		ret = 0;
+		break;
 	default:
 		RAS_DEV_WARN(ras_core->dev, "Invalid ras notify event:%d\n", event_id);
 		break;
@@ -187,6 +220,9 @@ static int amdgpu_ras_sys_get_device_system_info(struct ras_core_context *ras_co
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
 
+	if (!adev->smuio.funcs || !adev->smuio.funcs->get_socket_id)
+		return -ENOENT;
+
 	dev_info->device_id = adev->pdev->device;
 	dev_info->vendor_id = adev->pdev->vendor;
 	dev_info->socket_id = adev->smuio.funcs->get_socket_id(adev);
@@ -222,6 +258,10 @@ static int amdgpu_ras_sys_get_gpu_mem(struct ras_core_context *ras_core,
 	struct psp_context *psp = &adev->psp;
 	struct psp_ring *psp_ring;
 	struct ta_mem_context *mem_ctx;
+	int ret;
+
+	if (!gpu_mem)
+		return -EINVAL;
 
 	if (mem_type == GPU_MEM_TYPE_RAS_PSP_RING) {
 		psp_ring = &psp->km_ring;
@@ -230,16 +270,16 @@ static int amdgpu_ras_sys_get_gpu_mem(struct ras_core_context *ras_core,
 		gpu_mem->mem_mc_addr = psp_ring->ring_mem_mc_addr;
 		gpu_mem->mem_cpu_addr = psp_ring->ring_mem;
 	} else if (mem_type == GPU_MEM_TYPE_RAS_PSP_CMD) {
-		gpu_mem->mem_bo = psp->cmd_buf_bo;
+		gpu_mem->mem_bo = psp->cmd_resp_buf_bo;
 		gpu_mem->mem_size = PSP_CMD_BUFFER_SIZE;
-		gpu_mem->mem_mc_addr = psp->cmd_buf_mc_addr;
-		gpu_mem->mem_cpu_addr = psp->cmd_buf_mem;
+		gpu_mem->mem_mc_addr = psp->cmd_resp_buf_mc_addr;
+		gpu_mem->mem_cpu_addr = psp->cmd_resp_buf_mem;
 	} else if (mem_type == GPU_MEM_TYPE_RAS_PSP_FENCE) {
 		gpu_mem->mem_bo = psp->fence_buf_bo;
 		gpu_mem->mem_size = PSP_FENCE_BUFFER_SIZE;
 		gpu_mem->mem_mc_addr = psp->fence_buf_mc_addr;
 		gpu_mem->mem_cpu_addr = psp->fence_buf;
-	} else if (mem_type == GPU_MEM_TYPE_RAS_TA_FW) {
+	} else if (mem_type == GPU_MEM_TYPE_RAS_FW_BIN) {
 		gpu_mem->mem_bo = psp->fw_pri_bo;
 		gpu_mem->mem_size = PSP_1_MEG;
 		gpu_mem->mem_mc_addr = psp->fw_pri_mc_addr;
@@ -250,6 +290,25 @@ static int amdgpu_ras_sys_get_gpu_mem(struct ras_core_context *ras_core,
 		gpu_mem->mem_size = mem_ctx->shared_mem_size;
 		gpu_mem->mem_mc_addr = mem_ctx->shared_mc_addr;
 		gpu_mem->mem_cpu_addr = mem_ctx->shared_buf;
+	} else if (mem_type == GPU_MEM_TYPE_ALLOC_MEM) {
+		struct amdgpu_bo *mem_bo = NULL;
+
+		if (!gpu_mem->mem_size)
+			return -EINVAL;
+
+		ret = amdgpu_bo_create_kernel(adev, gpu_mem->mem_size,
+				    PSP_1_MEG, AMDGPU_GEM_DOMAIN_VRAM |
+				    AMDGPU_GEM_DOMAIN_GTT,
+				    &mem_bo,
+				    &gpu_mem->mem_mc_addr,
+				    &gpu_mem->mem_cpu_addr);
+		if (ret) {
+			RAS_DEV_ERR(ras_core->dev, "Failed to alloc shared memory. ret:%d\n", ret);
+			return ret;
+		}
+
+		gpu_mem->mem_bo = mem_bo;
+
 	} else {
 		return -EINVAL;
 	}
@@ -266,6 +325,13 @@ static int amdgpu_ras_sys_get_gpu_mem(struct ras_core_context *ras_core,
 static int amdgpu_ras_sys_put_gpu_mem(struct ras_core_context *ras_core,
 	enum gpu_mem_type mem_type, struct gpu_mem_block *gpu_mem)
 {
+	if ((mem_type == GPU_MEM_TYPE_ALLOC_MEM) && gpu_mem &&
+		gpu_mem->mem_bo && gpu_mem->mem_cpu_addr && gpu_mem->mem_mc_addr) {
+		struct amdgpu_bo *mem_bo = gpu_mem->mem_bo;
+
+		amdgpu_bo_free_kernel(&mem_bo,
+			&gpu_mem->mem_mc_addr, &gpu_mem->mem_cpu_addr);
+	}
 
 	return 0;
 }
@@ -288,6 +354,20 @@ static int amdgpu_ras_sys_check_address_sanity(struct ras_core_context *ras_core
 	return 0;
 }
 
+static int amdgpu_ras_sys_get_nps_mode(struct ras_core_context *ras_core)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+
+	return amdgpu_gmc_query_memory_partition(adev);
+}
+
+static int amdgpu_ras_sys_get_vram_type(struct ras_core_context *ras_core)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)ras_core->dev;
+
+	return adev->gmc.vram_type;
+}
+
 const struct ras_sys_func amdgpu_ras_sys_fn = {
 	.ras_notifier = amdgpu_ras_sys_event_notifier,
 	.get_utc_second_timestamp = amdgpu_ras_sys_get_utc_second_timestamp,
@@ -299,4 +379,6 @@ const struct ras_sys_func amdgpu_ras_sys_fn = {
 	.get_gpu_mem = amdgpu_ras_sys_get_gpu_mem,
 	.put_gpu_mem = amdgpu_ras_sys_put_gpu_mem,
 	.check_address_sanity = amdgpu_ras_sys_check_address_sanity,
+	.get_nps_mode = amdgpu_ras_sys_get_nps_mode,
+	.get_vram_type = amdgpu_ras_sys_get_vram_type,
 };
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
index fd4eea8c3f7baf..1e83cb5064ff63 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
@@ -273,8 +273,8 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
 		(struct ras_cmd_cper_record_rsp *)cmd->output_buff_raw;
 	struct ras_log_batch_overview *overview = &virt_ras->batch_mgr.batch_overview;
 	struct ras_cmd_batch_trace_record_rsp *rsp_cache = &virt_ras->batch_mgr.batch_trace;
-	struct ras_log_info *trace;
-	uint32_t trace_count = MAX_RECORD_PER_BATCH;
+	struct ras_log_info *batch_logs;
+	uint32_t nr_batch_logs = MAX_RECORD_PER_BATCH;
 	uint32_t offset = 0, real_data_len = 0;
 	uint64_t batch_id;
 	uint8_t *out_buf;
@@ -288,13 +288,13 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
 	    req->buf_size > RAS_CMD_MAX_CPER_BUF_SZ)
 		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
 
-	trace = kzalloc_objs(*trace, trace_count);
-	if (!trace)
+	batch_logs = kzalloc_objs(*batch_logs, nr_batch_logs);
+	if (!batch_logs)
 		return RAS_CMD__ERROR_GENERIC;
 
 	out_buf = kzalloc(req->buf_size, GFP_KERNEL);
 	if (!out_buf) {
-		kfree(trace);
+		kfree(batch_logs);
 		return RAS_CMD__ERROR_GENERIC;
 	}
 
@@ -305,10 +305,10 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
 		if (batch_id >= overview->last_batch_id)
 			break;
 		count = amdgpu_virt_ras_get_batch_records(ras_core, batch_id,
-							  trace, trace_count,
+							  batch_logs, nr_batch_logs,
 							  rsp_cache);
 		if (count > 0) {
-			ret = ras_cper_generate_cper(ras_core, trace, count,
+			ret = ras_cper_generate_batch_cper(ras_core, batch_logs, count,
 					&out_buf[offset], req->buf_size - offset, &real_data_len);
 			if (ret)
 				break;
@@ -320,7 +320,7 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
 	if ((ret && (ret != -ENOMEM)) ||
 	    copy_to_user(u64_to_user_ptr(req->buf_ptr), out_buf, offset)) {
 		kfree(out_buf);
-		kfree(trace);
+		kfree(batch_logs);
 		return RAS_CMD__ERROR_GENERIC;
 	}
 
@@ -332,7 +332,7 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
 	cmd->output_size = sizeof(struct ras_cmd_cper_record_rsp);
 
 	kfree(out_buf);
-	kfree(trace);
+	kfree(batch_logs);
 
 	return RAS_CMD__SUCCESS;
 }
@@ -441,27 +441,37 @@ int amdgpu_virt_ras_convert_retired_address(struct amdgpu_device *adev,
 			uint64_t address, uint64_t *pfn, uint32_t max_pfn_sz)
 {
 	struct ras_cmd_convert_retired_address_req req = {0};
-	struct ras_cmd_convert_retired_address_rsp rsp = {0};
+	struct ras_cmd_convert_retired_address_rsp *rsp;
 	int ret = 0, i;
 	int retired_page_count;
 
 	if (!pfn || !max_pfn_sz)
 		return -EINVAL;
 
+	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
+	if (!rsp)
+		return -ENOMEM;
+
 	req.address = address;
 
 	ret = amdgpu_ras_mgr_handle_ras_cmd(adev, RAS_CMD__CONVERT_RETIRED_ADDRESS,
-		&req, sizeof(req), &rsp, sizeof(rsp));
+		&req, sizeof(req), rsp, sizeof(*rsp));
 
-	if (ret || rsp.retired_count == 0)
-		return -EINVAL;
+	if (ret || rsp->retired_count == 0) {
+		ret = -EINVAL;
+		goto out;
+	}
 
-	retired_page_count = rsp.retired_count > max_pfn_sz ? max_pfn_sz : rsp.retired_count;
+	retired_page_count = rsp->retired_count > max_pfn_sz ? max_pfn_sz : rsp->retired_count;
 
 	for (i = 0; i < retired_page_count; i++)
-		pfn[i] = rsp.retired_addr[i] >> AMDGPU_GPU_PAGE_SHIFT;
+		pfn[i] = rsp->retired_addr[i] >> AMDGPU_GPU_PAGE_SHIFT;
+
+	ret = retired_page_count;
 
-	return retired_page_count;
+out:
+	kfree(rsp);
+	return ret;
 }
 
 static struct ras_cmd_func_map amdgpu_virt_ras_cmd_maps[] = {
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h b/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
index 2775c7bf41b7e9..1c5e02de7fca94 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/ras_sys.h
@@ -28,7 +28,10 @@
 #include <linux/printk.h>
 #include <linux/dev_printk.h>
 #include <linux/mempool.h>
+#include <linux/math64.h>
+#include <linux/time.h>
 #include "amdgpu.h"
+#include "amdgpu_ip.h"
 
 /* inject address is 52 bits */
 #define RAS_UMC_INJECT_ADDR_LIMIT       (0x1ULL << 52)
@@ -94,15 +97,13 @@
 #define RAS_GET_INST(dev, ip, inst) \
 ({ \
 	struct amdgpu_device *adev = (struct amdgpu_device *)dev; \
-	adev->ip_map.logical_to_dev_inst ? \
-		adev->ip_map.logical_to_dev_inst(adev, ip##_HWIP, inst) : inst; \
+	amdgpu_ip_map_logical_to_dev_inst(&adev->ip_map, ip##_HWIP, inst); \
 })
 
 #define RAS_GET_MASK(dev, ip, mask) \
 ({ \
 	struct amdgpu_device *adev = (struct amdgpu_device *)dev; \
-	(adev->ip_map.logical_to_dev_mask ? \
-		adev->ip_map.logical_to_dev_mask(adev, ip##_HWIP, mask) : mask); \
+	amdgpu_ip_map_logical_to_dev_mask(&adev->ip_map, ip##_HWIP, mask); \
 })
 
 static inline void *ras_radix_tree_delete_iter(struct radix_tree_root *root, void *iter)
diff --git a/drivers/gpu/drm/arm/display/include/malidp_product.h b/drivers/gpu/drm/arm/display/include/malidp_product.h
index 6f954bcdf40ea7..f9a3ee7ba4b75b 100644
--- a/drivers/gpu/drm/arm/display/include/malidp_product.h
+++ b/drivers/gpu/drm/arm/display/include/malidp_product.h
@@ -8,14 +8,18 @@
 #define _MALIDP_PRODUCT_H_
 
 /* Product identification */
+/* GLB_CORE_ID fields as per HW specification:
+ * MINOR is 8 bits ([11:4]) and STATUS is 4 bits ([3:0]).
+ * Update masks/shifts accordingly.
+ */
 #define MALIDP_CORE_ID(__product, __major, __minor, __status) \
 	((((__product) & 0xFFFF) << 16) | (((__major) & 0xF) << 12) | \
-	(((__minor) & 0xF) << 8) | ((__status) & 0xFF))
+	(((__minor) & 0xFF) << 4) | ((__status) & 0xF))
 
 #define MALIDP_CORE_ID_PRODUCT_ID(__core_id) ((__u32)(__core_id) >> 16)
 #define MALIDP_CORE_ID_MAJOR(__core_id)      (((__u32)(__core_id) >> 12) & 0xF)
-#define MALIDP_CORE_ID_MINOR(__core_id)      (((__u32)(__core_id) >> 8) & 0xF)
-#define MALIDP_CORE_ID_STATUS(__core_id)     (((__u32)(__core_id)) & 0xFF)
+#define MALIDP_CORE_ID_MINOR(__core_id)      (((__u32)(__core_id) >>  4) & 0xFF)
+#define MALIDP_CORE_ID_STATUS(__core_id)     ((__u32)(__core_id) & 0xF)
 
 /* Mali-display product IDs */
 #define MALIDP_D71_PRODUCT_ID	0x0071
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
index e8cb782a6f8ed1..0fd069fc3386d0 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
@@ -11,10 +11,11 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_panel.h>
 
 #include "komeda_dev.h"
 #include "komeda_kms.h"
@@ -493,19 +494,17 @@ static const struct drm_crtc_helper_funcs komeda_crtc_helper_funcs = {
 	.mode_fixup	= komeda_crtc_mode_fixup,
 };
 
-static void komeda_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *komeda_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct komeda_crtc_state *state;
 
-	if (crtc->state)
-		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	kfree(to_kcrtc_st(crtc->state));
-	crtc->state = NULL;
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
 
-	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
+	return &state->base;
 }
 
 static struct drm_crtc_state *
@@ -555,7 +554,7 @@ static const struct drm_crtc_funcs komeda_crtc_funcs = {
 	.destroy		= drm_crtc_cleanup,
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= drm_atomic_helper_page_flip,
-	.reset			= komeda_crtc_reset,
+	.atomic_create_state = komeda_crtc_create_state,
 	.atomic_duplicate_state	= komeda_crtc_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_crtc_atomic_destroy_state,
 	.enable_vblank		= komeda_crtc_vblank_enable,
@@ -635,6 +634,10 @@ static int komeda_attach_bridge(struct device *dev,
 	return err;
 }
 
+static const struct drm_encoder_funcs komeda_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static int komeda_crtc_add(struct komeda_kms_dev *kms,
 			   struct komeda_crtc *kcrtc)
 {
@@ -658,7 +661,8 @@ static int komeda_crtc_add(struct komeda_kms_dev *kms,
 	 * bridge
 	 */
 	kcrtc->encoder.possible_crtcs = drm_crtc_mask(crtc);
-	err = drm_simple_encoder_init(base, encoder, DRM_MODE_ENCODER_TMDS);
+	err = drm_encoder_init(base, encoder, &komeda_encoder_funcs,
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
index 57a4a0c7d404a9..955dbef5d34da0 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
@@ -132,19 +132,17 @@ static void komeda_plane_destroy(struct drm_plane *plane)
 	kfree(to_kplane(plane));
 }
 
-static void komeda_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *komeda_plane_create_state(struct drm_plane *plane)
 {
 	struct komeda_plane_state *state;
 
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	kfree(plane->state);
-	plane->state = NULL;
+	__drm_atomic_helper_plane_state_init(&state->base, plane);
 
-	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_plane_reset(plane, &state->base);
+	return &state->base;
 }
 
 static struct drm_plane_state *
@@ -188,7 +186,7 @@ static const struct drm_plane_funcs komeda_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= komeda_plane_destroy,
-	.reset			= komeda_plane_reset,
+	.atomic_create_state = komeda_plane_create_state,
 	.atomic_duplicate_state	= komeda_plane_atomic_duplicate_state,
 	.atomic_destroy_state	= komeda_plane_atomic_destroy_state,
 	.format_mod_supported	= komeda_plane_format_mod_supported,
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_wb_connector.c b/drivers/gpu/drm/arm/display/komeda/komeda_wb_connector.c
index 41cc3e080dc9ce..4872253a244985 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_wb_connector.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_wb_connector.c
@@ -143,12 +143,17 @@ static int komeda_wb_connector_add(struct komeda_kms_dev *kms,
 	struct komeda_wb_connector *kwb_conn;
 	struct drm_writeback_connector *wb_conn;
 	struct drm_display_info *info;
+	struct drm_encoder *encoder;
+	struct drm_encoder *wb_encoder;
 	u32 *formats, n_formats = 0;
+	u32 crtc_mask;
 	int err;
 
 	if (!kcrtc->master->wb_layer)
 		return 0;
 
+	crtc_mask = drm_crtc_mask(&kcrtc->base);
+
 	kwb_conn = kzalloc_obj(*kwb_conn);
 	if (!kwb_conn)
 		return -ENOMEM;
@@ -176,6 +181,34 @@ static int komeda_wb_connector_add(struct komeda_kms_dev *kms,
 		return err;
 	}
 
+	wb_encoder = &wb_conn->encoder;
+
+	/*
+	 * The writeback connector is associated with a single CRTC. Make its
+	 * encoder clone-compatible only with encoders that can drive that CRTC.
+	 *
+	 * possible_clones must contain the encoder's own bit whenever it is
+	 * non-zero. Add both the encoder itself and the writeback encoder when
+	 * updating the reciprocal clone relationship.
+	 */
+	wb_encoder->possible_clones = drm_encoder_mask(wb_encoder);
+
+	drm_for_each_encoder(encoder, &kms->base) {
+		u32 encoder_mask;
+
+		if (encoder == wb_encoder)
+			continue;
+
+		if (!(encoder->possible_crtcs & crtc_mask))
+			continue;
+
+		encoder_mask = drm_encoder_mask(encoder);
+
+		wb_encoder->possible_clones |= encoder_mask;
+		encoder->possible_clones |= encoder_mask |
+					    drm_encoder_mask(wb_encoder);
+	}
+
 	drm_connector_helper_add(&wb_conn->base, &komeda_wb_conn_helper_funcs);
 
 	info = &kwb_conn->base.base.display_info;
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index 8565749a3e48d2..f288a9c7775da3 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -67,7 +67,7 @@ static const struct drm_crtc_funcs hdlcd_crtc_funcs = {
 	.destroy = hdlcd_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = hdlcd_crtc_enable_vblank,
@@ -297,7 +297,7 @@ static const struct drm_plane_helper_funcs hdlcd_plane_helper_funcs = {
 static const struct drm_plane_funcs hdlcd_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/arm/malidp_crtc.c b/drivers/gpu/drm/arm/malidp_crtc.c
index ebe8e10787772d..ec7b9db8b0b454 100644
--- a/drivers/gpu/drm/arm/malidp_crtc.c
+++ b/drivers/gpu/drm/arm/malidp_crtc.c
@@ -56,6 +56,7 @@ static void malidp_crtc_atomic_enable(struct drm_crtc *crtc,
 
 	if (err < 0) {
 		DRM_DEBUG_DRIVER("Failed to enable runtime power management: %d\n", err);
+		pm_runtime_put_noidle(crtc->dev->dev);
 		return;
 	}
 
@@ -476,17 +477,17 @@ static void malidp_crtc_destroy_state(struct drm_crtc *crtc,
 	kfree(mali_state);
 }
 
-static void malidp_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *malidp_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct malidp_crtc_state *state = kzalloc_obj(*state);
+	struct malidp_crtc_state *state;
 
-	if (crtc->state)
-		malidp_crtc_destroy_state(crtc, crtc->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
 
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	return &state->base;
 }
 
 static int malidp_crtc_enable_vblank(struct drm_crtc *crtc)
@@ -511,7 +512,7 @@ static void malidp_crtc_disable_vblank(struct drm_crtc *crtc)
 static const struct drm_crtc_funcs malidp_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = malidp_crtc_reset,
+	.atomic_create_state = malidp_crtc_create_state,
 	.atomic_duplicate_state = malidp_crtc_duplicate_state,
 	.atomic_destroy_state = malidp_crtc_destroy_state,
 	.enable_vblank = malidp_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index 60b3f2bb153b5f..d971ed6eb6ebab 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -73,17 +73,17 @@
  * allocate a new empty object. We just need enough space to store
  * a malidp_plane_state instead of a drm_plane_state.
  */
-static void malidp_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *malidp_plane_create_state(struct drm_plane *plane)
 {
-	struct malidp_plane_state *state = to_malidp_plane_state(plane->state);
+	struct malidp_plane_state *state;
 
-	if (state)
-		__drm_atomic_helper_plane_destroy_state(&state->base);
-	kfree(state);
-	plane->state = NULL;
 	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_plane_reset(plane, &state->base);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&state->base, plane);
+
+	return &state->base;
 }
 
 static struct
@@ -252,7 +252,7 @@ static bool malidp_format_mod_supported_per_plane(struct drm_plane *plane,
 static const struct drm_plane_funcs malidp_de_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = malidp_plane_reset,
+	.atomic_create_state = malidp_plane_create_state,
 	.atomic_duplicate_state = malidp_duplicate_plane_state,
 	.atomic_destroy_state = malidp_destroy_plane_state,
 	.atomic_print_state = malidp_plane_atomic_print_state,
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index 5e006212fea0d5..d349beaa2990b4 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -817,7 +817,7 @@ static void armada_drm_crtc_disable_vblank(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_funcs armada_crtc_funcs = {
-	.reset		= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.cursor_set	= armada_drm_crtc_cursor_set,
 	.cursor_move	= armada_drm_crtc_cursor_move,
 	.destroy	= armada_drm_crtc_destroy,
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index 3da0b857ef95a0..e75706b85dd129 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -301,30 +301,27 @@ armada_overlay_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 	return ret;
 }
 
-static void armada_overlay_reset(struct drm_plane *plane)
+static struct drm_plane_state *armada_overlay_create_state(struct drm_plane *plane)
 {
 	struct armada_overlay_state *state;
 
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-	kfree(plane->state);
-	plane->state = NULL;
-
 	state = kzalloc_obj(*state);
-	if (state) {
-		state->colorkey_yr = 0xfefefe00;
-		state->colorkey_ug = 0x01010100;
-		state->colorkey_vb = 0x01010100;
-		state->colorkey_mode = CFG_CKMODE(CKMODE_RGB) |
-				       CFG_ALPHAM_GRA | CFG_ALPHA(0);
-		state->colorkey_enable = ADV_GRACOLORKEY;
-		state->brightness = DEFAULT_BRIGHTNESS;
-		state->contrast = DEFAULT_CONTRAST;
-		state->saturation = DEFAULT_SATURATION;
-		__drm_atomic_helper_plane_reset(plane, &state->base.base);
-		state->base.base.color_encoding = DEFAULT_ENCODING;
-		state->base.base.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
-	}
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	state->colorkey_yr = 0xfefefe00;
+	state->colorkey_ug = 0x01010100;
+	state->colorkey_vb = 0x01010100;
+	state->colorkey_mode = CFG_CKMODE(CKMODE_RGB) | CFG_ALPHAM_GRA | CFG_ALPHA(0);
+	state->colorkey_enable = ADV_GRACOLORKEY;
+	state->brightness = DEFAULT_BRIGHTNESS;
+	state->contrast = DEFAULT_CONTRAST;
+	state->saturation = DEFAULT_SATURATION;
+	__drm_atomic_helper_plane_state_init(&state->base.base, plane);
+	state->base.base.color_encoding = DEFAULT_ENCODING;
+	state->base.base.color_range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+
+	return &state->base.base;
 }
 
 static struct drm_plane_state *
@@ -466,7 +463,7 @@ static const struct drm_plane_funcs armada_ovl_plane_funcs = {
 	.update_plane	= armada_overlay_plane_update,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_helper_destroy,
-	.reset		= armada_overlay_reset,
+	.atomic_create_state = armada_overlay_create_state,
 	.atomic_duplicate_state = armada_overlay_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.atomic_set_property = armada_overlay_set_property,
diff --git a/drivers/gpu/drm/armada/armada_plane.c b/drivers/gpu/drm/armada/armada_plane.c
index fe46a8e4508a2c..1f5172d6816b31 100644
--- a/drivers/gpu/drm/armada/armada_plane.c
+++ b/drivers/gpu/drm/armada/armada_plane.c
@@ -256,15 +256,16 @@ static const struct drm_plane_helper_funcs armada_primary_plane_helper_funcs = {
 	.atomic_disable	= armada_drm_primary_plane_atomic_disable,
 };
 
-void armada_plane_reset(struct drm_plane *plane)
+struct drm_plane_state *armada_plane_create_state(struct drm_plane *plane)
 {
 	struct armada_plane_state *st;
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-	kfree(plane->state);
 	st = kzalloc_obj(*st);
-	if (st)
-		__drm_atomic_helper_plane_reset(plane, &st->base);
+	if (!st)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&st->base, plane);
+
+	return &st->base;
 }
 
 struct drm_plane_state *armada_plane_duplicate_state(struct drm_plane *plane)
@@ -285,7 +286,7 @@ static const struct drm_plane_funcs armada_primary_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_helper_destroy,
-	.reset		= armada_plane_reset,
+	.atomic_create_state = armada_plane_create_state,
 	.atomic_duplicate_state = armada_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/armada/armada_plane.h b/drivers/gpu/drm/armada/armada_plane.h
index 69bef09ea02898..be2b1eaafad77a 100644
--- a/drivers/gpu/drm/armada/armada_plane.h
+++ b/drivers/gpu/drm/armada/armada_plane.h
@@ -25,7 +25,7 @@ void armada_drm_plane_cleanup_fb(struct drm_plane *plane,
 	struct drm_plane_state *old_state);
 int armada_drm_plane_atomic_check(struct drm_plane *plane,
 	struct drm_atomic_commit *state);
-void armada_plane_reset(struct drm_plane *plane);
+struct drm_plane_state *armada_plane_create_state(struct drm_plane *plane);
 struct drm_plane_state *armada_plane_duplicate_state(struct drm_plane *plane);
 void armada_plane_destroy_state(struct drm_plane *plane,
 				struct drm_plane_state *state);
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx.h b/drivers/gpu/drm/aspeed/aspeed_gfx.h
index 4e6a442c3886d4..273528d30b4c2c 100644
--- a/drivers/gpu/drm/aspeed/aspeed_gfx.h
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx.h
@@ -2,7 +2,6 @@
 /* Copyright 2018 IBM Corporation */
 
 #include <drm/drm_device.h>
-#include <drm/drm_simple_kms_helper.h>
 
 struct aspeed_gfx {
 	struct drm_device		drm;
@@ -17,10 +16,16 @@ struct aspeed_gfx {
 	u32				throd_val;
 	u32				scan_line_max;
 
-	struct drm_simple_display_pipe	pipe;
+	struct drm_plane		plane;
+	struct drm_crtc			crtc;
+	struct drm_encoder		encoder;
 	struct drm_connector		connector;
 };
-#define to_aspeed_gfx(x) container_of(x, struct aspeed_gfx, drm)
+
+static inline struct aspeed_gfx *to_aspeed_gfx(struct drm_device *drm)
+{
+	return container_of(drm, struct aspeed_gfx, drm);
+}
 
 int aspeed_gfx_create_pipe(struct drm_device *drm);
 int aspeed_gfx_create_output(struct drm_device *drm);
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c b/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
index 7877a57b8e2657..918b8b4dc86347 100644
--- a/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
@@ -5,6 +5,8 @@
 #include <linux/reset.h>
 #include <linux/regmap.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_device.h>
 #include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
@@ -12,22 +14,17 @@
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_panel.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "aspeed_gfx.h"
 
-static struct aspeed_gfx *
-drm_pipe_to_aspeed_gfx(struct drm_simple_display_pipe *pipe)
+static int aspeed_gfx_set_pixel_fmt(struct aspeed_gfx *priv,
+				    struct drm_plane_state *plane_state,
+				    u32 *bpp)
 {
-	return container_of(pipe, struct aspeed_gfx, pipe);
-}
-
-static int aspeed_gfx_set_pixel_fmt(struct aspeed_gfx *priv, u32 *bpp)
-{
-	struct drm_crtc *crtc = &priv->pipe.crtc;
+	struct drm_crtc *crtc = &priv->crtc;
 	struct drm_device *drm = crtc->dev;
-	const u32 format = crtc->primary->state->fb->format->format;
+	const u32 format = plane_state->fb->format->format;
 	u32 ctrl1;
 
 	ctrl1 = readl(priv->base + CRT_CTRL1);
@@ -77,13 +74,15 @@ static void aspeed_gfx_disable_controller(struct aspeed_gfx *priv)
 	regmap_update_bits(priv->scu, priv->dac_reg, BIT(16), 0);
 }
 
-static void aspeed_gfx_crtc_mode_set_nofb(struct aspeed_gfx *priv)
+static void aspeed_gfx_crtc_mode_set_nofb(struct aspeed_gfx *priv,
+					  struct drm_crtc_state *crtc_state,
+					  struct drm_plane_state *plane_state)
 {
-	struct drm_display_mode *m = &priv->pipe.crtc.state->adjusted_mode;
+	struct drm_display_mode *m = &crtc_state->adjusted_mode;
 	u32 ctrl1, d_offset, t_count, bpp;
 	int err;
 
-	err = aspeed_gfx_set_pixel_fmt(priv, &bpp);
+	err = aspeed_gfx_set_pixel_fmt(priv, plane_state, &bpp);
 	if (err)
 		return;
 
@@ -139,48 +138,35 @@ static void aspeed_gfx_crtc_mode_set_nofb(struct aspeed_gfx *priv)
 	writel(priv->throd_val, priv->base + CRT_THROD);
 }
 
-static void aspeed_gfx_pipe_enable(struct drm_simple_display_pipe *pipe,
-			      struct drm_crtc_state *crtc_state,
-			      struct drm_plane_state *plane_state)
+static void aspeed_gfx_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+						 struct drm_atomic_commit *commit)
 {
-	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
-	struct drm_crtc *crtc = &pipe->crtc;
+	struct aspeed_gfx *priv = to_aspeed_gfx(crtc->dev);
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, &priv->plane);
 
-	aspeed_gfx_crtc_mode_set_nofb(priv);
+	aspeed_gfx_crtc_mode_set_nofb(priv, crtc_state, plane_state);
 	aspeed_gfx_enable_controller(priv);
 	drm_crtc_vblank_on(crtc);
 }
 
-static void aspeed_gfx_pipe_disable(struct drm_simple_display_pipe *pipe)
+static void aspeed_gfx_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+						  struct drm_atomic_commit *commit)
 {
-	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
-	struct drm_crtc *crtc = &pipe->crtc;
+	struct aspeed_gfx *priv = to_aspeed_gfx(crtc->dev);
 
 	drm_crtc_vblank_off(crtc);
 	aspeed_gfx_disable_controller(priv);
 }
 
-static void aspeed_gfx_pipe_update(struct drm_simple_display_pipe *pipe,
-				   struct drm_plane_state *plane_state)
+static void aspeed_gfx_plane_helper_atomic_update(struct drm_plane *plane,
+						  struct drm_atomic_commit *commit)
 {
-	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_framebuffer *fb = pipe->plane.state->fb;
-	struct drm_pending_vblank_event *event;
+	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(commit, plane);
+	struct aspeed_gfx *priv = to_aspeed_gfx(plane->dev);
+	struct drm_framebuffer *fb = new_plane_state->fb;
 	struct drm_gem_dma_object *gem;
 
-	spin_lock_irq(&crtc->dev->event_lock);
-	event = crtc->state->event;
-	if (event) {
-		crtc->state->event = NULL;
-
-		if (drm_crtc_vblank_get(crtc) == 0)
-			drm_crtc_arm_vblank_event(crtc, event);
-		else
-			drm_crtc_send_vblank_event(crtc, event);
-	}
-	spin_unlock_irq(&crtc->dev->event_lock);
-
 	if (!fb)
 		return;
 
@@ -190,9 +176,9 @@ static void aspeed_gfx_pipe_update(struct drm_simple_display_pipe *pipe,
 	writel(gem->dma_addr, priv->base + CRT_ADDR);
 }
 
-static int aspeed_gfx_enable_vblank(struct drm_simple_display_pipe *pipe)
+static int aspeed_gfx_crtc_enable_vblank(struct drm_crtc *crtc)
 {
-	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
+	struct aspeed_gfx *priv = to_aspeed_gfx(crtc->dev);
 	u32 reg = readl(priv->base + CRT_CTRL1);
 
 	/* Clear pending VBLANK IRQ */
@@ -204,9 +190,9 @@ static int aspeed_gfx_enable_vblank(struct drm_simple_display_pipe *pipe)
 	return 0;
 }
 
-static void aspeed_gfx_disable_vblank(struct drm_simple_display_pipe *pipe)
+static void aspeed_gfx_crtc_disable_vblank(struct drm_crtc *crtc)
 {
-	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
+	struct aspeed_gfx *priv = to_aspeed_gfx(crtc->dev);
 	u32 reg = readl(priv->base + CRT_CTRL1);
 
 	reg &= ~CRT_CTRL_VERTICAL_INTR_EN;
@@ -216,12 +202,93 @@ static void aspeed_gfx_disable_vblank(struct drm_simple_display_pipe *pipe)
 	writel(reg | CRT_CTRL_VERTICAL_INTR_STS, priv->base + CRT_CTRL1);
 }
 
-static const struct drm_simple_display_pipe_funcs aspeed_gfx_funcs = {
-	.enable		= aspeed_gfx_pipe_enable,
-	.disable	= aspeed_gfx_pipe_disable,
-	.update		= aspeed_gfx_pipe_update,
-	.enable_vblank	= aspeed_gfx_enable_vblank,
-	.disable_vblank	= aspeed_gfx_disable_vblank,
+static int aspeed_gfx_plane_helper_atomic_check(struct drm_plane *plane,
+						struct drm_atomic_commit *commit)
+{
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_crtc_state *crtc_state = NULL;
+
+	if (plane_state->crtc) {
+		crtc_state = drm_atomic_get_crtc_state(commit, plane_state->crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+	}
+
+	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, false);
+}
+
+static const struct drm_plane_helper_funcs aspeed_gfx_plane_helper_funcs = {
+	.prepare_fb	= drm_gem_plane_helper_prepare_fb,
+	.atomic_check	= aspeed_gfx_plane_helper_atomic_check,
+	.atomic_update	= aspeed_gfx_plane_helper_atomic_update,
+};
+
+static const struct drm_plane_funcs aspeed_gfx_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.destroy		= drm_plane_cleanup,
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+};
+
+static int aspeed_gfx_crtc_helper_atomic_check(struct drm_crtc *crtc,
+					       struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static void aspeed_gfx_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+						struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	struct drm_pending_vblank_event *event = new_crtc_state->event;
+
+	if (!event)
+		return;
+
+	new_crtc_state->event = NULL;
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	if (drm_crtc_vblank_get(crtc) == 0)
+		drm_crtc_arm_vblank_event(crtc, event);
+	else
+		drm_crtc_send_vblank_event(crtc, event);
+	spin_unlock_irq(&crtc->dev->event_lock);
+}
+
+static const struct drm_crtc_helper_funcs aspeed_gfx_crtc_helper_funcs = {
+	.atomic_check	= aspeed_gfx_crtc_helper_atomic_check,
+	.atomic_enable	= aspeed_gfx_crtc_helper_atomic_enable,
+	.atomic_disable	= aspeed_gfx_crtc_helper_atomic_disable,
+	.atomic_flush	= aspeed_gfx_crtc_helper_atomic_flush,
+};
+
+static const struct drm_crtc_funcs aspeed_gfx_crtc_funcs = {
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.destroy		= drm_crtc_cleanup,
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+	.enable_vblank		= aspeed_gfx_crtc_enable_vblank,
+	.disable_vblank		= aspeed_gfx_crtc_disable_vblank,
+};
+
+static const struct drm_encoder_funcs aspeed_gfx_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
 static const uint32_t aspeed_gfx_formats[] = {
@@ -232,10 +299,36 @@ static const uint32_t aspeed_gfx_formats[] = {
 int aspeed_gfx_create_pipe(struct drm_device *drm)
 {
 	struct aspeed_gfx *priv = to_aspeed_gfx(drm);
+	struct drm_plane *plane = &priv->plane;
+	struct drm_crtc *crtc = &priv->crtc;
+	struct drm_encoder *encoder = &priv->encoder;
+	int ret;
+
+	ret = drm_universal_plane_init(drm, plane, 0,
+				       &aspeed_gfx_plane_funcs,
+				       aspeed_gfx_formats,
+				       ARRAY_SIZE(aspeed_gfx_formats),
+				       NULL,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+	drm_plane_helper_add(plane, &aspeed_gfx_plane_helper_funcs);
+
+	ret = drm_crtc_init_with_planes(drm, crtc, plane, NULL,
+					&aspeed_gfx_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+	drm_crtc_helper_add(crtc, &aspeed_gfx_crtc_helper_funcs);
+
+	ret = drm_encoder_init(drm, encoder, &aspeed_gfx_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
+	ret = drm_connector_attach_encoder(&priv->connector, encoder);
+	if (ret)
+		return ret;
 
-	return drm_simple_display_pipe_init(drm, &priv->pipe, &aspeed_gfx_funcs,
-					    aspeed_gfx_formats,
-					    ARRAY_SIZE(aspeed_gfx_formats),
-					    NULL,
-					    &priv->connector);
+	return 0;
 }
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
index d4577663a1f0ef..c8746766f25899 100644
--- a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
@@ -20,7 +20,6 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_drv.h>
 
@@ -129,7 +128,7 @@ static irqreturn_t aspeed_gfx_irq_handler(int irq, void *data)
 	reg = readl(priv->base + CRT_CTRL1);
 
 	if (reg & CRT_CTRL_VERTICAL_INTR_STS) {
-		drm_crtc_handle_vblank(&priv->pipe.crtc);
+		drm_crtc_handle_vblank(&priv->crtc);
 		writel(reg, priv->base + priv->int_clr_reg);
 		return IRQ_HANDLED;
 	}
diff --git a/drivers/gpu/drm/ast/ast_dp.c b/drivers/gpu/drm/ast/ast_dp.c
index 5f266a05419baf..f22c0c215a2162 100644
--- a/drivers/gpu/drm/ast/ast_dp.c
+++ b/drivers/gpu/drm/ast/ast_dp.c
@@ -88,8 +88,14 @@ static int ast_astdp_read_edid_block(void *data, u8 *buf, unsigned int block, si
 	int ret = 0;
 	unsigned int i;
 
-	if (block > 0)
-		return -EIO; /* extension headers not supported */
+	if (block > 1) {
+		/* ASPEED DP's EDID buffer holds blocks 0 and 1 only (256
+		 * bytes). Report anything beyond that as an all-zero block
+		 * rather than a read failure.
+		 */
+		memset(buf, 0, len);
+		return 0;
+	}
 
 	/*
 	 * Protect access to I/O registers from concurrent modesetting
@@ -154,20 +160,6 @@ static int ast_astdp_read_edid_block(void *data, u8 *buf, unsigned int block, si
 		ediddata[2] = ast_get_index_reg(ast, AST_IO_VGACRI, 0xda);
 		ediddata[3] = ast_get_index_reg(ast, AST_IO_VGACRI, 0xdb);
 
-		if (i == 31) {
-			/*
-			 * For 128-bytes EDID_1.3,
-			 * 1. Add the value of Bytes-126 to Bytes-127.
-			 *		The Bytes-127 is Checksum. Sum of all 128bytes should
-			 *		equal 0	(mod 256).
-			 * 2. Modify Bytes-126 to be 0.
-			 *		The Bytes-126 indicates the Number of extensions to
-			 *		follow. 0 represents noextensions.
-			 */
-			ediddata[3] = ediddata[3] + ediddata[2];
-			ediddata[2] = 0;
-		}
-
 		memcpy(buf, ediddata, min((len - i), 4));
 		buf += 4;
 	}
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index d5ed8c5c79253a..08db132acae85d 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -43,6 +43,7 @@
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_panic.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -652,6 +653,7 @@ static const struct drm_plane_funcs ast_primary_plane_funcs = {
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
 	DRM_GEM_SHADOW_PLANE_FUNCS,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static int ast_primary_plane_init(struct ast_device *ast)
@@ -888,17 +890,17 @@ static const struct drm_crtc_helper_funcs ast_crtc_helper_funcs = {
 	.atomic_disable = ast_crtc_helper_atomic_disable,
 };
 
-static void ast_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *ast_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct ast_crtc_state *ast_state = kzalloc_obj(*ast_state);
+	struct ast_crtc_state *ast_state;
 
-	if (crtc->state)
-		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
+	ast_state = kzalloc_obj(*ast_state);
+	if (!ast_state)
+		return ERR_PTR(-ENOMEM);
 
-	if (ast_state)
-		__drm_atomic_helper_crtc_reset(crtc, &ast_state->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	__drm_atomic_helper_crtc_state_init(&ast_state->base, crtc);
+
+	return &ast_state->base;
 }
 
 static struct drm_crtc_state *
@@ -934,7 +936,7 @@ static void ast_crtc_atomic_destroy_state(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_funcs ast_crtc_funcs = {
-	.reset = ast_crtc_reset,
+	.atomic_create_state = ast_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
index 9dbac2def333e1..c2c4c9c3086bcf 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
@@ -529,20 +529,17 @@ void atmel_hlcdc_crtc_irq(struct drm_crtc *c)
 	atmel_hlcdc_crtc_finish_page_flip(drm_crtc_to_atmel_hlcdc_crtc(c));
 }
 
-static void atmel_hlcdc_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *atmel_hlcdc_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct atmel_hlcdc_crtc_state *state;
 
-	if (crtc->state) {
-		__drm_atomic_helper_crtc_destroy_state(crtc->state);
-		state = drm_crtc_state_to_atmel_hlcdc_crtc_state(crtc->state);
-		kfree(state);
-		crtc->state = NULL;
-	}
-
 	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
+
+	return &state->base;
 }
 
 static struct drm_crtc_state *
@@ -599,7 +596,7 @@ static void atmel_hlcdc_crtc_disable_vblank(struct drm_crtc *c)
 static const struct drm_crtc_funcs atmel_hlcdc_crtc_funcs = {
 	.page_flip = drm_atomic_helper_page_flip,
 	.set_config = drm_atomic_helper_set_config,
-	.reset = atmel_hlcdc_crtc_reset,
+	.atomic_create_state = atmel_hlcdc_crtc_create_state,
 	.atomic_duplicate_state =  atmel_hlcdc_crtc_duplicate_state,
 	.atomic_destroy_state = atmel_hlcdc_crtc_destroy_state,
 	.enable_vblank = atmel_hlcdc_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index f17a832351e9d7..e09ce6342085dc 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -1183,36 +1183,31 @@ static void atmel_hlcdc_plane_atomic_destroy_state(struct drm_plane *p,
 	kfree(state);
 }
 
-static void atmel_hlcdc_plane_reset(struct drm_plane *p)
+static struct drm_plane_state *atmel_hlcdc_plane_create_state(struct drm_plane *p)
 {
 	struct atmel_hlcdc_plane_state *state;
-	struct atmel_hlcdc_dc *dc = p->dev->dev_private;
-	struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p);
-
-	if (p->state) {
-		atmel_hlcdc_plane_atomic_destroy_state(p, p->state);
-		p->state = NULL;
-	}
+	int ret;
 
 	state = kzalloc_obj(*state);
-	if (state) {
-		if (atmel_hlcdc_plane_alloc_dscrs(p, state)) {
-			kfree(state);
-			drm_err(p->dev,
-				"Failed to allocate initial plane state\n");
-			return;
-		}
-		__drm_atomic_helper_plane_reset(p, &state->base);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	ret = atmel_hlcdc_plane_alloc_dscrs(p, state);
+	if (ret) {
+		kfree(state);
+		drm_err(p->dev, "Failed to allocate initial plane state\n");
+		return ERR_PTR(ret);
 	}
 
-	if (plane->layer.desc->layout.csc)
-		dc->desc->ops->lcdc_csc_init(plane, plane->layer.desc);
+	__drm_atomic_helper_plane_state_init(&state->base, p);
+
+	return &state->base;
 }
 
 static const struct drm_plane_funcs layer_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = atmel_hlcdc_plane_reset,
+	.atomic_create_state = atmel_hlcdc_plane_create_state,
 	.atomic_duplicate_state = atmel_hlcdc_plane_atomic_duplicate_state,
 	.atomic_destroy_state = atmel_hlcdc_plane_atomic_destroy_state,
 };
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 4a57d49b4c6d3a..c7da6b02d7b6b2 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -87,7 +87,6 @@ config DRM_FSL_LDB
 	depends on OF
 	depends on ARCH_MXC || COMPILE_TEST
 	select DRM_KMS_HELPER
-	select DRM_PANEL_BRIDGE
 	help
 	  Support for i.MX8MP DPI-to-LVDS on-SoC encoder.
 
@@ -151,6 +150,7 @@ config DRM_LONTIUM_LT8912B
 config DRM_LONTIUM_LT9211
 	tristate "Lontium LT9211 DSI/LVDS/DPI bridge"
 	depends on OF
+	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
@@ -225,7 +225,6 @@ config DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW
 	tristate "MegaChips stdp4028-ge-b850v3-fw and stdp2690-ge-b850v3-fw"
 	depends on OF
 	select DRM_KMS_HELPER
-	select DRM_PANEL
 	help
 	  This is a driver for the display bridges of
 	  GE B850v3 that convert dual channel LVDS
@@ -260,7 +259,7 @@ config DRM_NXP_PTN3460
 	tristate "NXP PTN3460 DP/LVDS bridge"
 	depends on OF
 	select DRM_KMS_HELPER
-	select DRM_PANEL
+	select DRM_PANEL_BRIDGE
 	help
 	  NXP PTN3460 eDP-LVDS bridge chip driver.
 
@@ -277,7 +276,7 @@ config DRM_OF_DISPLAY_MODE_BRIDGE
 config DRM_PARADE_PS8622
 	tristate "Parade eDP/LVDS bridge"
 	depends on OF
-	select DRM_PANEL
+	select DRM_PANEL_BRIDGE
 	select DRM_KMS_HELPER
 	select BACKLIGHT_CLASS_DEVICE
 	help
@@ -291,7 +290,7 @@ config DRM_PARADE_PS8640
 	select DRM_DISPLAY_DP_AUX_BUS
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
-	select DRM_PANEL
+	select DRM_PANEL_BRIDGE
 	help
 	  Choose this option if you have PS8640 for display
 	  The PS8640 is a high-performance and low-power
@@ -303,7 +302,6 @@ config DRM_SAMSUNG_DSIM
 	depends on OF && HAS_IOMEM
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
-	select DRM_PANEL_BRIDGE
 	select GENERIC_PHY
 	select GENERIC_PHY_MIPI_DPHY
 	help
@@ -351,7 +349,6 @@ config DRM_SOLOMON_SSD2825
 	depends on SPI_MASTER && OF
 	select DRM_MIPI_DSI
 	select DRM_KMS_HELPER
-	select DRM_PANEL
 	help
 	  Say Y here if you want support for the Solomon SSD2825 RGB/DSI
 	  SPI bridge driver.
@@ -389,7 +386,7 @@ config DRM_TOSHIBA_TC358764
 	depends on OF
 	select DRM_MIPI_DSI
 	select DRM_KMS_HELPER
-	select DRM_PANEL
+	select DRM_PANEL_BRIDGE
 	help
 	  Toshiba TC358764 DSI/LVDS bridge driver.
 
@@ -401,7 +398,6 @@ config DRM_TOSHIBA_TC358767
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
 	select DRM_MIPI_DSI
-	select DRM_PANEL
 	help
 	  Toshiba TC358767 eDP bridge chip driver.
 
@@ -410,7 +406,6 @@ config DRM_TOSHIBA_TC358768
 	depends on OF
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
-	select DRM_PANEL
 	select DRM_MIPI_DSI
 	select VIDEOMODE_HELPERS
 	help
@@ -423,16 +418,17 @@ config DRM_TOSHIBA_TC358775
 	select DRM_DISPLAY_HELPER
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
-	select DRM_PANEL
+	select DRM_PANEL_BRIDGE
 	select DRM_MIPI_DSI
 	help
 	  Toshiba TC358775 DSI/LVDS bridge chip driver.
 
 config DRM_TI_DLPC3433
 	tristate "TI DLPC3433 Display controller"
-	depends on DRM && DRM_PANEL
+	depends on DRM
 	depends on OF
 	select DRM_MIPI_DSI
+	select DRM_PANEL_BRIDGE
 	help
 	  TI DLPC3433 is a MIPI DSI based display controller bridge
 	  for processing high resolution DMD based projectors.
@@ -463,7 +459,7 @@ config DRM_TI_SN65DSI83
 	depends on OF
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
-	select DRM_PANEL
+	select DRM_PANEL_BRIDGE
 	select DRM_MIPI_DSI
 	help
 	  Texas Instruments SN65DSI83 and SN65DSI84 DSI to LVDS Bridge driver
@@ -476,10 +472,10 @@ config DRM_TI_SN65DSI86
 	select DRM_BRIDGE_CONNECTOR
 	select DRM_KMS_HELPER
 	select REGMAP_I2C
-	select DRM_PANEL
 	select DRM_MIPI_DSI
 	select AUXILIARY_BUS
 	select DRM_DISPLAY_DP_AUX_BUS
+	select DRM_PANEL_BRIDGE
 	help
 	  Texas Instruments SN65DSI86 DSI to eDP Bridge driver
 
@@ -495,6 +491,7 @@ config DRM_WAVESHARE_BRIDGE
 	tristate "Waveshare DSI bridge"
 	depends on OF
 	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
diff --git a/drivers/gpu/drm/bridge/analogix/Kconfig b/drivers/gpu/drm/bridge/analogix/Kconfig
index 57bb2daa5aaf41..5672d83cb4a13a 100644
--- a/drivers/gpu/drm/bridge/analogix/Kconfig
+++ b/drivers/gpu/drm/bridge/analogix/Kconfig
@@ -6,6 +6,7 @@ config DRM_ANALOGIX_ANX6345
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
 	select DRM_KMS_HELPER
+	select DRM_PANEL
 	select REGMAP_I2C
 	help
 	  ANX6345 is an ultra-low power Full-HD DisplayPort/eDP
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 30c0c0b41d5aae..6df6963740d2c7 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -29,7 +29,6 @@
 #include <drm/drm_device.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -1244,10 +1243,78 @@ static const struct drm_bridge_funcs analogix_dp_bridge_funcs = {
 	.detect = analogix_dp_bridge_detect,
 };
 
+static int analogix_dp_dt_parse_lanes_map(struct analogix_dp_device *dp)
+{
+	struct video_info *video_info = &dp->video_info;
+	struct device_node *endpoint;
+	u32 lane_idx[LANE_COUNT4];
+	u32 map[LANE_COUNT4] = {0, 1, 2, 3};
+	bool used[LANE_COUNT4] = {false};
+	int num_lanes;
+	int ret, i;
+
+	memcpy(video_info->lane_map, map, sizeof(map));
+
+	num_lanes = drm_of_get_data_lanes_count_ep(dp->dev->of_node, 1, 0, 1,
+						   video_info->max_lane_count);
+	if (num_lanes < 0)
+		return -EINVAL;
+
+	endpoint = of_graph_get_endpoint_by_regs(dp->dev->of_node, 1, 0);
+	if (!endpoint)
+		return -EINVAL;
+
+	ret = of_property_read_u32_array(endpoint, "data-lanes", lane_idx, num_lanes);
+	of_node_put(endpoint);
+	if (ret)
+		return -EINVAL;
+
+	for (i = 0; i < num_lanes; i++) {
+		if (lane_idx[i] >= LANE_COUNT4) {
+			dev_dbg(dp->dev, "data-lanes[%d] = %u is out of range\n", i, lane_idx[i]);
+			return -EINVAL;
+		}
+
+		if (used[lane_idx[i]]) {
+			dev_dbg(dp->dev, "data-lanes[%d] = %u is duplicate\n", i, lane_idx[i]);
+			return -EINVAL;
+		}
+
+		used[lane_idx[i]] = true;
+		map[i] = lane_idx[i];
+	}
+
+	/*
+	 * Fill the map[] entries not described by 'data-lanes' with the
+	 * lane indices not used so far, e.g. for 'data-lanes = <3 1>':
+	 *
+	 *	used[]       = {0, 1, 0, 1}  // only lanes 1 and 3 are used
+	 *	map[] before = {3, 1, x, x}  // x = unassigned
+	 *	map[] after  = {3, 1, 0, 2}
+	 *
+	 * Unused entries have no effect on the current link with fewer
+	 * lanes in use, but filling them with distinct indices keeps the
+	 * LANE_MAP register holding a valid permutation, just like its
+	 * reset value (0xe4), so the mapping remains sane if a sink with
+	 * a different lane count is connected later, e.g. via DP hot-plug.
+	 */
+	for (i = 0; i < LANE_COUNT4 && num_lanes < LANE_COUNT4; i++) {
+		if (!used[i])
+			map[num_lanes++] = i;
+	}
+
+	dev_dbg(dp->dev, "Using parsed lane map: <%u %u %u %u>\n", map[0], map[1], map[2], map[3]);
+
+	memcpy(video_info->lane_map, map, sizeof(map));
+
+	return 0;
+}
+
 static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp)
 {
 	struct device_node *dp_node = dp->dev->of_node;
 	struct video_info *video_info = &dp->video_info;
+	u32 val;
 
 	switch (dp->plat_data->dev_type) {
 	case RK3288_DP:
@@ -1269,13 +1336,21 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp)
 		 * NOTE: those property parseing code is used for
 		 * providing backward compatibility for samsung platform.
 		 */
-		of_property_read_u32(dp_node, "samsung,link-rate",
-				     &video_info->max_link_rate);
-		of_property_read_u32(dp_node, "samsung,lane-count",
-				     &video_info->max_lane_count);
+		if (of_property_read_u32(dp_node, "samsung,link-rate", &val))
+			return dev_err_probe(dp->dev, -EINVAL,
+					     "Failed to get samsung,link-rate\n");
+		video_info->max_link_rate = val;
+		if (of_property_read_u32(dp_node, "samsung,lane-count", &val) ||
+		    !drm_dp_lane_count_is_valid(val))
+			return dev_err_probe(dp->dev, -EINVAL,
+					     "Failed to get samsung,lane-count\n");
+		video_info->max_lane_count = val;
 		break;
 	}
 
+	if (analogix_dp_dt_parse_lanes_map(dp))
+		dev_dbg(dp->dev, "No valid data-lanes found, using default lane map\n");
+
 	return 0;
 }
 
@@ -1414,10 +1489,8 @@ analogix_dp_probe(struct device *dev, struct analogix_dp_plat_data *plat_data)
 					analogix_dp_hardirq,
 					analogix_dp_irq_thread,
 					irq_flags, "analogix-dp", dp);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to request irq\n");
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	dp->aux.name = "DP-AUX";
 	dp->aux.transfer = analogix_dpaux_transfer;
@@ -1493,15 +1566,6 @@ int analogix_dp_bind(struct analogix_dp_device *dp, struct drm_device *drm_dev)
 	if (ret)
 		goto err_unregister_aux;
 
-	if (dp->plat_data->panel) {
-		dp->plat_data->next_bridge = devm_drm_panel_bridge_add(dp->dev,
-								       dp->plat_data->panel);
-		if (IS_ERR(dp->plat_data->next_bridge)) {
-			ret = PTR_ERR(dp->plat_data->next_bridge);
-			goto err_unregister_aux;
-		}
-	}
-
 	ret = drm_bridge_attach(dp->encoder, bridge, NULL, DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 	if (ret) {
 		DRM_ERROR("failed to create bridge (%d)\n", ret);
@@ -1573,6 +1637,11 @@ struct drm_dp_aux *analogix_dp_get_aux(struct analogix_dp_device *dp)
 }
 EXPORT_SYMBOL_GPL(analogix_dp_get_aux);
 
+static void analogix_dp_put_bridge(void *data)
+{
+	drm_bridge_put(data);
+}
+
 static int analogix_dp_aux_done_probing(struct drm_dp_aux *aux)
 {
 	struct analogix_dp_device *dp = to_dp(aux);
@@ -1581,14 +1650,25 @@ static int analogix_dp_aux_done_probing(struct drm_dp_aux *aux)
 	int ret;
 
 	/*
-	 * If drm_of_find_panel_or_bridge() returns -ENODEV, there may be no valid panel
-	 * or bridge nodes. The driver should go on for the driver-free bridge or the DP
-	 * mode applications.
+	 * If of_drm_get_bridge_by_endpoint() returns -ENODEV, there may be no
+	 * valid panel or bridge nodes. The driver should go on for the
+	 * driver-free bridge or the DP mode applications.
 	 */
-	ret = drm_of_find_panel_or_bridge(dp->dev->of_node, port, 0,
-					  &plat_data->panel, &plat_data->next_bridge);
-	if (ret && ret != -ENODEV)
-		return ret;
+	plat_data->next_bridge = of_drm_get_bridge_by_endpoint(dp->dev->of_node,
+							       port, 0);
+	if (IS_ERR(plat_data->next_bridge)) {
+		if (PTR_ERR(plat_data->next_bridge) == -ENODEV)
+			plat_data->next_bridge = NULL;
+		else
+			return PTR_ERR(plat_data->next_bridge);
+	}
+
+	if (plat_data->next_bridge) {
+		ret = devm_add_action_or_reset(dp->dev, analogix_dp_put_bridge,
+					       plat_data->next_bridge);
+		if (ret)
+			return ret;
+	}
 
 	return component_add(dp->dev, plat_data->ops);
 }
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
index c7997677a28623..4560597604adb4 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
@@ -137,6 +137,8 @@ struct video_info {
 
 	int max_link_rate;
 	enum link_lane_count_type max_lane_count;
+
+	u32 lane_map[LANE_COUNT4];
 };
 
 struct link_train {
@@ -177,7 +179,7 @@ struct analogix_dp_device {
 /* analogix_dp_reg.c */
 void analogix_dp_enable_video_mute(struct analogix_dp_device *dp, bool enable);
 void analogix_dp_stop_video(struct analogix_dp_device *dp);
-void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable);
+void analogix_dp_lane_mapping(struct analogix_dp_device *dp);
 void analogix_dp_init_analog_param(struct analogix_dp_device *dp);
 void analogix_dp_init_interrupt(struct analogix_dp_device *dp);
 void analogix_dp_reset(struct analogix_dp_device *dp);
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
index ea8401293a23aa..c1344a3f013abe 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
@@ -48,16 +48,15 @@ void analogix_dp_stop_video(struct analogix_dp_device *dp)
 	writel(reg, dp->reg_base + ANALOGIX_DP_VIDEO_CTL_1);
 }
 
-void analogix_dp_lane_swap(struct analogix_dp_device *dp, bool enable)
+void analogix_dp_lane_mapping(struct analogix_dp_device *dp)
 {
+	u32 *lane_map = dp->video_info.lane_map;
 	u32 reg;
 
-	if (enable)
-		reg = LANE3_MAP_LOGIC_LANE_0 | LANE2_MAP_LOGIC_LANE_1 |
-		      LANE1_MAP_LOGIC_LANE_2 | LANE0_MAP_LOGIC_LANE_3;
-	else
-		reg = LANE3_MAP_LOGIC_LANE_3 | LANE2_MAP_LOGIC_LANE_2 |
-		      LANE1_MAP_LOGIC_LANE_1 | LANE0_MAP_LOGIC_LANE_0;
+	reg = lane_map[0] << LANE0_MAP_SHIFT;
+	reg |= lane_map[1] << LANE1_MAP_SHIFT;
+	reg |= lane_map[2] << LANE2_MAP_SHIFT;
+	reg |= lane_map[3] << LANE3_MAP_SHIFT;
 
 	writel(reg, dp->reg_base + ANALOGIX_DP_LANE_MAP);
 }
@@ -140,7 +139,7 @@ void analogix_dp_reset(struct analogix_dp_device *dp)
 
 	usleep_range(20, 30);
 
-	analogix_dp_lane_swap(dp, 0);
+	analogix_dp_lane_mapping(dp);
 
 	writel(0x0, dp->reg_base + ANALOGIX_DP_SYS_CTL_1);
 	writel(0x40, dp->reg_base + ANALOGIX_DP_SYS_CTL_2);
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
index 12735139046ced..ac914e37089ba0 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
@@ -209,6 +209,10 @@
 #define LANE0_MAP_LOGIC_LANE_1			(0x1 << 0)
 #define LANE0_MAP_LOGIC_LANE_2			(0x2 << 0)
 #define LANE0_MAP_LOGIC_LANE_3			(0x3 << 0)
+#define LANE3_MAP_SHIFT				(6)
+#define LANE2_MAP_SHIFT				(4)
+#define LANE1_MAP_SHIFT				(2)
+#define LANE0_MAP_SHIFT				(0)
 
 /* ANALOGIX_DP_ANALOG_CTL_1 */
 #define TX_TERMINAL_CTRL_50_OHM			(0x1 << 4)
diff --git a/drivers/gpu/drm/bridge/aux-bridge.c b/drivers/gpu/drm/bridge/aux-bridge.c
index 169939d2331026..90f76bb529cc2b 100644
--- a/drivers/gpu/drm/bridge/aux-bridge.c
+++ b/drivers/gpu/drm/bridge/aux-bridge.c
@@ -12,6 +12,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/bridge/aux-bridge.h>
+#include <drm/drm_panel.h>
 
 static DEFINE_IDA(drm_aux_bridge_ida);
 
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
index c3e466ded84ab3..1e28ea579b06f7 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
@@ -7,6 +7,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 #include <video/mipi_display.h>
 #include <video/videomode.h>
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
index 504a3186ebb3a3..f47717bd107982 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
@@ -2343,7 +2343,6 @@ static int cdns_mhdp_probe(struct platform_device *pdev)
 					cdns_mhdp_irq_handler, IRQF_ONESHOT,
 					"mhdp8546", mhdp);
 	if (ret) {
-		dev_err(dev, "cannot install IRQ %d\n", irq);
 		ret = -EIO;
 		goto plat_fini;
 	}
diff --git a/drivers/gpu/drm/bridge/chipone-icn6211.c b/drivers/gpu/drm/bridge/chipone-icn6211.c
index 0f6173c615027e..4ec50fb63abab0 100644
--- a/drivers/gpu/drm/bridge/chipone-icn6211.c
+++ b/drivers/gpu/drm/bridge/chipone-icn6211.c
@@ -531,7 +531,7 @@ static int chipone_dsi_attach(struct chipone *icn)
 	dsi->hs_rate = 500000000;
 	dsi->lp_rate = 16000000;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0)
 		dev_err(icn->dev, "failed to attach dsi\n");
 
@@ -763,11 +763,6 @@ static int chipone_i2c_probe(struct i2c_client *client)
 	return chipone_dsi_host_attach(icn);
 }
 
-static void chipone_dsi_remove(struct mipi_dsi_device *dsi)
-{
-	mipi_dsi_detach(dsi);
-}
-
 static const struct of_device_id chipone_of_match[] = {
 	{ .compatible = "chipone,icn6211", },
 	{ /* sentinel */ }
@@ -776,7 +771,6 @@ MODULE_DEVICE_TABLE(of, chipone_of_match);
 
 static struct mipi_dsi_driver chipone_dsi_driver = {
 	.probe = chipone_dsi_probe,
-	.remove = chipone_dsi_remove,
 	.driver = {
 		.name = "chipone-icn6211",
 		.of_match_table = chipone_of_match,
diff --git a/drivers/gpu/drm/bridge/fsl-ldb.c b/drivers/gpu/drm/bridge/fsl-ldb.c
index 26cc72948f316c..7ec84c81cd786a 100644
--- a/drivers/gpu/drm/bridge/fsl-ldb.c
+++ b/drivers/gpu/drm/bridge/fsl-ldb.c
@@ -15,7 +15,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 
 #define LDB_CTRL_CH0_ENABLE			BIT(0)
 #define LDB_CTRL_CH0_DI_SELECT			BIT(1)
@@ -86,7 +85,6 @@ static const struct fsl_ldb_devdata fsl_ldb_devdata[] = {
 struct fsl_ldb {
 	struct device *dev;
 	struct drm_bridge bridge;
-	struct drm_bridge *panel_bridge;
 	struct clk *clk;
 	struct regmap *regmap;
 	const struct fsl_ldb_devdata *devdata;
@@ -119,7 +117,7 @@ static int fsl_ldb_attach(struct drm_bridge *bridge,
 {
 	struct fsl_ldb *fsl_ldb = to_fsl_ldb(bridge);
 
-	return drm_bridge_attach(encoder, fsl_ldb->panel_bridge,
+	return drm_bridge_attach(encoder, fsl_ldb->bridge.next_bridge,
 				 bridge, flags);
 }
 
@@ -298,7 +296,6 @@ static int fsl_ldb_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct device_node *panel_node;
 	struct device_node *remote1, *remote2;
-	struct drm_panel *panel;
 	struct fsl_ldb *fsl_ldb;
 	int dual_link;
 
@@ -339,19 +336,13 @@ static int fsl_ldb_probe(struct platform_device *pdev)
 		fsl_ldb_is_dual(fsl_ldb) ? "dual-link mode" :
 		fsl_ldb->ch0_enabled ? "channel 0" : "channel 1");
 
-	panel = of_drm_find_panel(panel_node);
-	of_node_put(panel_node);
-	if (IS_ERR(panel))
-		return PTR_ERR(panel);
-
 	if (of_property_present(dev->of_node, "nxp,enable-termination-resistor"))
 		fsl_ldb->use_termination_resistor = true;
 
-	fsl_ldb->panel_bridge = devm_drm_panel_bridge_add(dev, panel);
-	drm_panel_put(panel);
-	if (IS_ERR(fsl_ldb->panel_bridge))
-		return PTR_ERR(fsl_ldb->panel_bridge);
-
+	fsl_ldb->bridge.next_bridge = of_drm_find_and_get_bridge(panel_node);
+	of_node_put(panel_node);
+	if (!fsl_ldb->bridge.next_bridge)
+		return -ENODEV;
 
 	if (fsl_ldb_is_dual(fsl_ldb)) {
 		struct device_node *port1, *port2;
diff --git a/drivers/gpu/drm/bridge/imx/imx93-pdfc.c b/drivers/gpu/drm/bridge/imx/imx93-pdfc.c
index 2d6ca95ed5544f..2e5019ed951a8c 100644
--- a/drivers/gpu/drm/bridge/imx/imx93-pdfc.c
+++ b/drivers/gpu/drm/bridge/imx/imx93-pdfc.c
@@ -16,6 +16,7 @@
 
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_panel.h>
 
 #define IMX93_DISPLAY_MUX_REG		0x60
 #define PARALLEL_DISP_FORMAT		GENMASK(10, 8)
diff --git a/drivers/gpu/drm/bridge/ite-it6505.c b/drivers/gpu/drm/bridge/ite-it6505.c
index 025a19c1e53034..e136ac71edbb6a 100644
--- a/drivers/gpu/drm/bridge/ite-it6505.c
+++ b/drivers/gpu/drm/bridge/ite-it6505.c
@@ -3607,10 +3607,8 @@ static int it6505_i2c_probe(struct i2c_client *client)
 					IRQF_TRIGGER_LOW | IRQF_ONESHOT |
 					IRQF_NO_AUTOEN,
 					"it6505-intp", it6505);
-	if (err) {
-		dev_err(dev, "Failed to request INTP threaded IRQ: %d", err);
+	if (err)
 		return err;
-	}
 
 	INIT_WORK(&it6505->link_works, it6505_link_training_work);
 	INIT_WORK(&it6505->hdcp_wait_ksv_list, it6505_hdcp_wait_ksv_list);
diff --git a/drivers/gpu/drm/bridge/lontium-lt8713sx.c b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
index bfff28883280c6..926f9e1e85bf21 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8713sx.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
@@ -572,7 +572,7 @@ static void lt8713sx_remove(struct i2c_client *client)
 }
 
 static struct i2c_device_id lt8713sx_id[] = {
-	{ .name = "lontium,lt8713sx" },
+	{ .name = "lt8713sx" },
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/gpu/drm/bridge/lontium-lt9211.c b/drivers/gpu/drm/bridge/lontium-lt9211.c
index 23682384daf46a..285cda58f6614d 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9211.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9211.c
@@ -773,7 +773,7 @@ static void lt9211_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lt9211_id[] = {
-	{ .name = "lontium,lt9211" },
+	{ .name = "lt9211" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lt9211_id);
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611.c b/drivers/gpu/drm/bridge/lontium-lt9611.c
index fb34f661ee0a0d..cdc2b188e2d87b 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611.c
@@ -1156,10 +1156,8 @@ static int lt9611_probe(struct i2c_client *client)
 	ret = devm_request_threaded_irq(dev, client->irq, NULL,
 					lt9611_irq_thread_handler,
 					IRQF_ONESHOT, "lt9611", lt9611);
-	if (ret) {
-		dev_err(dev, "failed to request irq\n");
+	if (ret)
 		goto err_disable_regulators;
-	}
 
 	i2c_set_clientdata(client, lt9611);
 
@@ -1229,7 +1227,7 @@ static void lt9611_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lt9611_id[] = {
-	{ .name = "lontium,lt9611" },
+	{ .name = "lt9611" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lt9611_id);
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
index 42a6503cd0f9f8..b3bb7f2cebb3bc 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
@@ -899,7 +899,7 @@ static void lt9611uxc_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lt9611uxc_id[] = {
-	{ .name = "lontium,lt9611uxc" },
+	{ .name = "lt9611uxc" },
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c
deleted file mode 100644
index 02388a3de626ee..00000000000000
--- a/drivers/gpu/drm/bridge/panel.c
+++ /dev/null
@@ -1,563 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2016 Laurent Pinchart <laurent.pinchart@ideasonboard.com>
- * Copyright (C) 2017 Broadcom
- */
-
-#include <linux/debugfs.h>
-#include <linux/export.h>
-
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_bridge.h>
-#include <drm/drm_connector.h>
-#include <drm/drm_encoder.h>
-#include <drm/drm_managed.h>
-#include <drm/drm_modeset_helper_vtables.h>
-#include <drm/drm_of.h>
-#include <drm/drm_panel.h>
-#include <drm/drm_print.h>
-#include <drm/drm_probe_helper.h>
-
-struct panel_bridge {
-	struct drm_bridge bridge;
-	struct drm_connector connector;
-	struct drm_panel *panel;
-	u32 connector_type;
-};
-
-static inline struct panel_bridge *
-drm_bridge_to_panel_bridge(struct drm_bridge *bridge)
-{
-	return container_of(bridge, struct panel_bridge, bridge);
-}
-
-static inline struct panel_bridge *
-drm_connector_to_panel_bridge(struct drm_connector *connector)
-{
-	return container_of(connector, struct panel_bridge, connector);
-}
-
-static int panel_bridge_connector_get_modes(struct drm_connector *connector)
-{
-	struct panel_bridge *panel_bridge =
-		drm_connector_to_panel_bridge(connector);
-
-	return drm_panel_get_modes(panel_bridge->panel, connector);
-}
-
-static const struct drm_connector_helper_funcs
-panel_bridge_connector_helper_funcs = {
-	.get_modes = panel_bridge_connector_get_modes,
-};
-
-static const struct drm_connector_funcs panel_bridge_connector_funcs = {
-	.reset = drm_atomic_helper_connector_reset,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = drm_connector_cleanup,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
-static int panel_bridge_attach(struct drm_bridge *bridge,
-			       struct drm_encoder *encoder,
-			       enum drm_bridge_attach_flags flags)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	struct drm_connector *connector = &panel_bridge->connector;
-	int ret;
-
-	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
-		return 0;
-
-	drm_connector_helper_add(connector,
-				 &panel_bridge_connector_helper_funcs);
-
-	ret = drm_connector_init(bridge->dev, connector,
-				 &panel_bridge_connector_funcs,
-				 panel_bridge->connector_type);
-	if (ret) {
-		DRM_ERROR("Failed to initialize connector\n");
-		return ret;
-	}
-
-	drm_panel_bridge_set_orientation(connector, bridge);
-
-	drm_connector_attach_encoder(&panel_bridge->connector,
-					  encoder);
-
-	if (bridge->dev->registered) {
-		if (connector->funcs->reset)
-			connector->funcs->reset(connector);
-		drm_connector_register(connector);
-	}
-
-	return 0;
-}
-
-static void panel_bridge_detach(struct drm_bridge *bridge)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	struct drm_connector *connector = &panel_bridge->connector;
-
-	/*
-	 * Cleanup the connector if we know it was initialized.
-	 *
-	 * FIXME: This wouldn't be needed if the panel_bridge structure was
-	 * allocated with drmm_kzalloc(). This might be tricky since the
-	 * drm_device pointer can only be retrieved when the bridge is attached.
-	 */
-	if (connector->dev)
-		drm_connector_cleanup(connector);
-}
-
-static void panel_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					   struct drm_atomic_commit *atomic_state)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	struct drm_encoder *encoder = bridge->encoder;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *old_crtc_state;
-
-	crtc = drm_atomic_get_new_crtc_for_encoder(atomic_state, encoder);
-	if (!crtc)
-		return;
-
-	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
-	if (old_crtc_state && old_crtc_state->self_refresh_active)
-		return;
-
-	drm_panel_prepare(panel_bridge->panel);
-}
-
-static void panel_bridge_atomic_enable(struct drm_bridge *bridge,
-				       struct drm_atomic_commit *atomic_state)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	struct drm_encoder *encoder = bridge->encoder;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *old_crtc_state;
-
-	crtc = drm_atomic_get_new_crtc_for_encoder(atomic_state, encoder);
-	if (!crtc)
-		return;
-
-	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
-	if (old_crtc_state && old_crtc_state->self_refresh_active)
-		return;
-
-	drm_panel_enable(panel_bridge->panel);
-}
-
-static void panel_bridge_atomic_disable(struct drm_bridge *bridge,
-					struct drm_atomic_commit *atomic_state)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	struct drm_encoder *encoder = bridge->encoder;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *new_crtc_state;
-
-	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state, encoder);
-	if (!crtc)
-		return;
-
-	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
-	if (new_crtc_state && new_crtc_state->self_refresh_active)
-		return;
-
-	drm_panel_disable(panel_bridge->panel);
-}
-
-static void panel_bridge_atomic_post_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_commit *atomic_state)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	struct drm_encoder *encoder = bridge->encoder;
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *new_crtc_state;
-
-	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state, encoder);
-	if (!crtc)
-		return;
-
-	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
-	if (new_crtc_state && new_crtc_state->self_refresh_active)
-		return;
-
-	drm_panel_unprepare(panel_bridge->panel);
-}
-
-static int panel_bridge_get_modes(struct drm_bridge *bridge,
-				  struct drm_connector *connector)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-
-	return drm_panel_get_modes(panel_bridge->panel, connector);
-}
-
-static void panel_bridge_debugfs_init(struct drm_bridge *bridge,
-				      struct dentry *root)
-{
-	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	struct drm_panel *panel = panel_bridge->panel;
-
-	root = debugfs_create_dir("panel", root);
-	if (panel->funcs->debugfs_init)
-		panel->funcs->debugfs_init(panel, root);
-}
-
-static const struct drm_bridge_funcs panel_bridge_bridge_funcs = {
-	.attach = panel_bridge_attach,
-	.detach = panel_bridge_detach,
-	.atomic_pre_enable = panel_bridge_atomic_pre_enable,
-	.atomic_enable = panel_bridge_atomic_enable,
-	.atomic_disable = panel_bridge_atomic_disable,
-	.atomic_post_disable = panel_bridge_atomic_post_disable,
-	.get_modes = panel_bridge_get_modes,
-	.atomic_create_state = drm_atomic_helper_bridge_create_state,
-	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
-	.atomic_get_input_bus_fmts = drm_atomic_helper_bridge_propagate_bus_fmt,
-	.debugfs_init = panel_bridge_debugfs_init,
-};
-
-/**
- * drm_bridge_is_panel - Checks if a drm_bridge is a panel_bridge.
- *
- * @bridge: The drm_bridge to be checked.
- *
- * Returns true if the bridge is a panel bridge, or false otherwise.
- */
-bool drm_bridge_is_panel(const struct drm_bridge *bridge)
-{
-	return bridge->funcs == &panel_bridge_bridge_funcs;
-}
-EXPORT_SYMBOL(drm_bridge_is_panel);
-
-/**
- * drm_panel_bridge_add - Creates a &drm_bridge and &drm_connector that
- * just calls the appropriate functions from &drm_panel.
- *
- * @panel: The drm_panel being wrapped.  Must be non-NULL.
- *
- * For drivers converting from directly using drm_panel: The expected
- * usage pattern is that during either encoder module probe or DSI
- * host attach, a drm_panel will be looked up through
- * drm_of_find_panel_or_bridge().  drm_panel_bridge_add() is used to
- * wrap that panel in the new bridge, and the result can then be
- * passed to drm_bridge_attach().  The drm_panel_prepare() and related
- * functions can be dropped from the encoder driver (they're now
- * called by the KMS helpers before calling into the encoder), along
- * with connector creation.  When done with the bridge (after
- * drm_mode_config_cleanup() if the bridge has already been attached), then
- * drm_panel_bridge_remove() to free it.
- *
- * The connector type is set to @panel->connector_type, which must be set to a
- * known type. Calling this function with a panel whose connector type is
- * DRM_MODE_CONNECTOR_Unknown will return ERR_PTR(-EINVAL).
- *
- * See devm_drm_panel_bridge_add() for an automatically managed version of this
- * function.
- */
-struct drm_bridge *drm_panel_bridge_add(struct drm_panel *panel)
-{
-	if (WARN_ON(panel->connector_type == DRM_MODE_CONNECTOR_Unknown))
-		return ERR_PTR(-EINVAL);
-
-	return drm_panel_bridge_add_typed(panel, panel->connector_type);
-}
-EXPORT_SYMBOL(drm_panel_bridge_add);
-
-/**
- * drm_panel_bridge_add_typed - Creates a &drm_bridge and &drm_connector with
- * an explicit connector type.
- * @panel: The drm_panel being wrapped.  Must be non-NULL.
- * @connector_type: The connector type (DRM_MODE_CONNECTOR_*)
- *
- * This is just like drm_panel_bridge_add(), but forces the connector type to
- * @connector_type instead of infering it from the panel.
- *
- * This function is deprecated and should not be used in new drivers. Use
- * drm_panel_bridge_add() instead, and fix panel drivers as necessary if they
- * don't report a connector type.
- */
-struct drm_bridge *drm_panel_bridge_add_typed(struct drm_panel *panel,
-					      u32 connector_type)
-{
-	struct panel_bridge *panel_bridge;
-
-	if (!panel)
-		return ERR_PTR(-EINVAL);
-
-	panel_bridge = devm_drm_bridge_alloc(panel->dev, struct panel_bridge, bridge,
-					     &panel_bridge_bridge_funcs);
-	if (IS_ERR(panel_bridge))
-		return (void *)panel_bridge;
-
-	panel_bridge->connector_type = connector_type;
-	panel_bridge->panel = drm_panel_get(panel);
-
-	panel_bridge->bridge.of_node = panel->dev->of_node;
-	panel_bridge->bridge.ops = DRM_BRIDGE_OP_MODES;
-	panel_bridge->bridge.type = connector_type;
-	panel_bridge->bridge.pre_enable_prev_first = panel->prepare_prev_first;
-
-	drm_bridge_add(&panel_bridge->bridge);
-
-	return &panel_bridge->bridge;
-}
-EXPORT_SYMBOL(drm_panel_bridge_add_typed);
-
-/**
- * drm_panel_bridge_remove - Unregisters and frees a drm_bridge
- * created by drm_panel_bridge_add().
- *
- * @bridge: The drm_bridge being freed.
- */
-void drm_panel_bridge_remove(struct drm_bridge *bridge)
-{
-	struct panel_bridge *panel_bridge;
-	struct drm_panel *panel;
-
-	if (!bridge)
-		return;
-
-	if (!drm_bridge_is_panel(bridge)) {
-		drm_warn(bridge->dev, "%s: called on non-panel bridge!\n", __func__);
-		return;
-	}
-
-	panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	panel = panel_bridge->panel;
-
-	drm_bridge_remove(bridge);
-	/* TODO remove this after reworking panel_bridge lifetime */
-	devm_drm_put_bridge(panel->dev, bridge);
-	drm_panel_put(panel);
-}
-EXPORT_SYMBOL(drm_panel_bridge_remove);
-
-/**
- * drm_panel_bridge_set_orientation - Set the connector's panel orientation
- * from the bridge that can be transformed to panel bridge.
- *
- * @connector: The connector to be set panel orientation.
- * @bridge: The drm_bridge to be transformed to panel bridge.
- *
- * Returns 0 on success, negative errno on failure.
- */
-int drm_panel_bridge_set_orientation(struct drm_connector *connector,
-				     struct drm_bridge *bridge)
-{
-	struct panel_bridge *panel_bridge;
-
-	panel_bridge = drm_bridge_to_panel_bridge(bridge);
-
-	return drm_connector_set_orientation_from_panel(connector,
-							panel_bridge->panel);
-}
-EXPORT_SYMBOL(drm_panel_bridge_set_orientation);
-
-static void devm_drm_panel_bridge_release(struct device *dev, void *res)
-{
-	struct drm_bridge *bridge = *(struct drm_bridge **)res;
-	struct panel_bridge *panel_bridge;
-	struct drm_panel *panel;
-
-	if (!bridge)
-		return;
-
-	panel_bridge = drm_bridge_to_panel_bridge(bridge);
-	panel = panel_bridge->panel;
-	drm_bridge_remove(bridge);
-	drm_panel_put(panel);
-}
-
-/**
- * devm_drm_panel_bridge_add - Creates a managed &drm_bridge and &drm_connector
- * that just calls the appropriate functions from &drm_panel.
- * @dev: device to tie the bridge lifetime to
- * @panel: The drm_panel being wrapped.  Must be non-NULL.
- *
- * This is the managed version of drm_panel_bridge_add() which automatically
- * calls drm_panel_bridge_remove() when @dev is unbound.
- */
-struct drm_bridge *devm_drm_panel_bridge_add(struct device *dev,
-					     struct drm_panel *panel)
-{
-	if (WARN_ON(panel->connector_type == DRM_MODE_CONNECTOR_Unknown))
-		return ERR_PTR(-EINVAL);
-
-	return devm_drm_panel_bridge_add_typed(dev, panel,
-					       panel->connector_type);
-}
-EXPORT_SYMBOL(devm_drm_panel_bridge_add);
-
-/**
- * devm_drm_panel_bridge_add_typed - Creates a managed &drm_bridge and
- * &drm_connector with an explicit connector type.
- * @dev: device to tie the bridge lifetime to
- * @panel: The drm_panel being wrapped.  Must be non-NULL.
- * @connector_type: The connector type (DRM_MODE_CONNECTOR_*)
- *
- * This is just like devm_drm_panel_bridge_add(), but forces the connector type
- * to @connector_type instead of infering it from the panel.
- *
- * This function is deprecated and should not be used in new drivers. Use
- * devm_drm_panel_bridge_add() instead, and fix panel drivers as necessary if
- * they don't report a connector type.
- */
-struct drm_bridge *devm_drm_panel_bridge_add_typed(struct device *dev,
-						   struct drm_panel *panel,
-						   u32 connector_type)
-{
-	struct drm_bridge **ptr, *bridge;
-
-	ptr = devres_alloc(devm_drm_panel_bridge_release, sizeof(*ptr),
-			   GFP_KERNEL);
-	if (!ptr)
-		return ERR_PTR(-ENOMEM);
-
-	bridge = drm_panel_bridge_add_typed(panel, connector_type);
-	if (IS_ERR(bridge)) {
-		devres_free(ptr);
-		return bridge;
-	}
-
-	*ptr = bridge;
-	devres_add(dev, ptr);
-
-	return bridge;
-}
-EXPORT_SYMBOL(devm_drm_panel_bridge_add_typed);
-
-static void drmm_drm_panel_bridge_release(struct drm_device *drm, void *ptr)
-{
-	struct drm_bridge *bridge = ptr;
-
-	drm_panel_bridge_remove(bridge);
-}
-
-/**
- * drmm_panel_bridge_add - Creates a DRM-managed &drm_bridge and
- *                         &drm_connector that just calls the
- *                         appropriate functions from &drm_panel.
- *
- * @drm: DRM device to tie the bridge lifetime to
- * @panel: The drm_panel being wrapped.  Must be non-NULL.
- *
- * This is the DRM-managed version of drm_panel_bridge_add() which
- * automatically calls drm_panel_bridge_remove() when @dev is cleaned
- * up.
- */
-struct drm_bridge *drmm_panel_bridge_add(struct drm_device *drm,
-					 struct drm_panel *panel)
-{
-	struct drm_bridge *bridge;
-	int ret;
-
-	bridge = drm_panel_bridge_add_typed(panel, panel->connector_type);
-	if (IS_ERR(bridge))
-		return bridge;
-
-	ret = drmm_add_action_or_reset(drm, drmm_drm_panel_bridge_release,
-				       bridge);
-	if (ret)
-		return ERR_PTR(ret);
-
-	return bridge;
-}
-EXPORT_SYMBOL(drmm_panel_bridge_add);
-
-/**
- * drm_panel_bridge_connector - return the connector for the panel bridge
- * @bridge: The drm_bridge.
- *
- * drm_panel_bridge creates the connector.
- * This function gives external access to the connector.
- *
- * Returns: Pointer to drm_connector
- */
-struct drm_connector *drm_panel_bridge_connector(struct drm_bridge *bridge)
-{
-	struct panel_bridge *panel_bridge;
-
-	panel_bridge = drm_bridge_to_panel_bridge(bridge);
-
-	return &panel_bridge->connector;
-}
-EXPORT_SYMBOL(drm_panel_bridge_connector);
-
-#ifdef CONFIG_OF
-/**
- * devm_drm_of_get_bridge - Return next bridge in the chain
- * @dev: device to tie the bridge lifetime to
- * @np: device tree node containing encoder output ports
- * @port: port in the device tree node
- * @endpoint: endpoint in the device tree node
- *
- * Given a DT node's port and endpoint number, finds the connected node
- * and returns the associated bridge if any, or creates and returns a
- * drm panel bridge instance if a panel is connected.
- *
- * Returns a pointer to the bridge if successful, or an error pointer
- * otherwise.
- */
-struct drm_bridge *devm_drm_of_get_bridge(struct device *dev,
-					  struct device_node *np,
-					  u32 port, u32 endpoint)
-{
-	struct drm_bridge *bridge;
-	struct drm_panel *panel;
-	int ret;
-
-	ret = drm_of_find_panel_or_bridge(np, port, endpoint,
-					  &panel, &bridge);
-	if (ret)
-		return ERR_PTR(ret);
-
-	if (panel) {
-		bridge = devm_drm_panel_bridge_add(dev, panel);
-		drm_panel_put(panel);
-	}
-
-	return bridge;
-}
-EXPORT_SYMBOL(devm_drm_of_get_bridge);
-
-/**
- * drmm_of_get_bridge - Return next bridge in the chain
- * @drm: device to tie the bridge lifetime to
- * @np: device tree node containing encoder output ports
- * @port: port in the device tree node
- * @endpoint: endpoint in the device tree node
- *
- * Given a DT node's port and endpoint number, finds the connected node
- * and returns the associated bridge if any, or creates and returns a
- * drm panel bridge instance if a panel is connected.
- *
- * Returns a drmm managed pointer to the bridge if successful, or an error
- * pointer otherwise.
- */
-struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm,
-				      struct device_node *np,
-				      u32 port, u32 endpoint)
-{
-	struct drm_bridge *bridge;
-	struct drm_panel *panel;
-	int ret;
-
-	ret = drm_of_find_panel_or_bridge(np, port, endpoint,
-					  &panel, &bridge);
-	if (ret)
-		return ERR_PTR(ret);
-
-	if (panel) {
-		bridge = drmm_panel_bridge_add(drm, panel);
-		drm_panel_put(panel);
-	}
-
-	return bridge;
-}
-EXPORT_SYMBOL(drmm_of_get_bridge);
-
-#endif
diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c
index e2fc69fc51b60d..480e568dd2c9d5 100644
--- a/drivers/gpu/drm/bridge/samsung-dsim.c
+++ b/drivers/gpu/drm/bridge/samsung-dsim.c
@@ -25,7 +25,6 @@
 #include <video/mipi_display.h>
 
 #include <drm/bridge/samsung-dsim.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 
 /* returns true iff both arguments logically differs */
@@ -1862,7 +1861,7 @@ static int samsung_dsim_register_te_irq(struct samsung_dsim *dsi, struct device
 	int te_gpio_irq;
 	int ret;
 
-	dsi->te_gpio = devm_gpiod_get_optional(dev, "te", GPIOD_IN);
+	dsi->te_gpio = gpiod_get_optional(dev, "te", GPIOD_IN);
 	if (!dsi->te_gpio)
 		return 0;
 	else if (IS_ERR(dsi->te_gpio))
@@ -1898,7 +1897,6 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host,
 	struct device *dev = dsi->dev;
 	struct device_node *np = dev->of_node;
 	struct device_node *remote;
-	struct drm_panel *panel;
 	int ret = 0;
 
 	/*
@@ -1913,7 +1911,7 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host,
 		    of_node_name_eq(remote, "ports"))
 			continue;
 
-		goto of_find_panel_or_bridge;
+		goto of_find_bridge;
 	}
 
 	/*
@@ -1927,25 +1925,13 @@ static int samsung_dsim_host_attach(struct mipi_dsi_host *host,
 
 	remote = of_graph_get_remote_node(np, 1, 0);
 
-of_find_panel_or_bridge:
+of_find_bridge:
 	if (!remote)
 		return -ENODEV;
 
-	panel = of_drm_find_panel(remote);
-	if (!IS_ERR(panel)) {
-		next_bridge = devm_drm_panel_bridge_add(dev, panel);
-		drm_panel_put(panel);
-		if (IS_ERR(next_bridge)) {
-			ret = PTR_ERR(next_bridge);
-			next_bridge = NULL; // Inhibit the cleanup action on an ERR_PTR
-		} else {
-			drm_bridge_get(next_bridge);
-		}
-	} else {
-		next_bridge = of_drm_find_and_get_bridge(remote);
-		if (!next_bridge)
-			ret = -EINVAL;
-	}
+	next_bridge = of_drm_find_and_get_bridge(remote);
+	if (!next_bridge)
+		ret = -EINVAL;
 
 	of_node_put(remote);
 
@@ -2184,10 +2170,8 @@ int samsung_dsim_probe(struct platform_device *pdev)
 					samsung_dsim_irq,
 					IRQF_ONESHOT | IRQF_NO_AUTOEN,
 					dev_name(dev), dsi);
-	if (ret) {
-		dev_err(dev, "failed to request dsi irq\n");
+	if (ret)
 		return ret;
-	}
 
 	ret = samsung_dsim_parse_dt(dsi);
 	if (ret)
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
index b452748ad9dbd0..88f9cd9ba6dbdc 100644
--- a/drivers/gpu/drm/bridge/sil-sii8620.c
+++ b/drivers/gpu/drm/bridge/sil-sii8620.c
@@ -2320,8 +2320,7 @@ static int sii8620_probe(struct i2c_client *client)
 					IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
 					"sii8620", ctx);
 	if (ret < 0)
-		return dev_err_probe(dev, ret,
-				     "failed to install IRQ handler\n");
+		return ret;
 
 	ctx->gpio_reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(ctx->gpio_reset))
diff --git a/drivers/gpu/drm/bridge/ssd2825.c b/drivers/gpu/drm/bridge/ssd2825.c
index b0142d6d40dc89..71877699dfab6b 100644
--- a/drivers/gpu/drm/bridge/ssd2825.c
+++ b/drivers/gpu/drm/bridge/ssd2825.c
@@ -17,7 +17,6 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <video/mipi_display.h>
 
 #define SSD2825_DEVICE_ID_REG			0xb0
@@ -114,7 +113,6 @@ static const struct regulator_bulk_data ssd2825_supplies[] = {
 
 struct ssd2825_dsi_output {
 	struct mipi_dsi_device *dev;
-	struct drm_panel *panel;
 	struct drm_bridge *bridge;
 };
 
@@ -277,7 +275,6 @@ static int ssd2825_dsi_host_attach(struct mipi_dsi_host *host, struct mipi_dsi_d
 {
 	struct ssd2825_priv *priv = dsi_host_to_ssd2825(host);
 	struct drm_bridge *bridge;
-	struct drm_panel *panel;
 	struct device_node *ep;
 	int ret;
 
@@ -295,20 +292,14 @@ static int ssd2825_dsi_host_attach(struct mipi_dsi_host *host, struct mipi_dsi_d
 		return -EOPNOTSUPP;
 	}
 
-	ret = drm_of_find_panel_or_bridge(host->dev->of_node, 1, 0, &panel, &bridge);
-	if (ret)
-		return ret;
+	bridge = of_drm_get_bridge_by_endpoint(host->dev->of_node, 1, 0);
+	if (IS_ERR(bridge))
+		return PTR_ERR(bridge);
 
-	if (panel) {
-		bridge = drm_panel_bridge_add_typed(panel, DRM_MODE_CONNECTOR_DSI);
-		drm_panel_put(panel);
-		if (IS_ERR(bridge))
-			return PTR_ERR(bridge);
-	}
+	bridge->type = DRM_MODE_CONNECTOR_DSI;
 
 	priv->output.dev = dev;
 	priv->output.bridge = bridge;
-	priv->output.panel = panel;
 
 	priv->dsi_lanes = dev->lanes;
 
@@ -333,8 +324,7 @@ static int ssd2825_dsi_host_detach(struct mipi_dsi_host *host, struct mipi_dsi_d
 	struct ssd2825_priv *priv = dsi_host_to_ssd2825(host);
 
 	drm_bridge_remove(&priv->bridge);
-	if (priv->output.panel)
-		drm_panel_bridge_remove(priv->output.bridge);
+	drm_bridge_put(priv->output.bridge);
 
 	return 0;
 }
@@ -579,9 +569,6 @@ static void ssd2825_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 	/* Initial DSI configuration register set */
 	ssd2825_write_reg(priv, SSD2825_CONFIGURATION_REG, config);
 	ssd2825_write_reg(priv, SSD2825_VC_CTRL_REG, 0);
-
-	if (priv->output.panel)
-		drm_panel_enable(priv->output.panel);
 }
 
 static void ssd2825_bridge_atomic_enable(struct drm_bridge *bridge,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
index 8b8529e1723c85..86a20a0b9c0157 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
@@ -2079,10 +2079,8 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder,
 
 	ret = devm_request_threaded_irq(dev, dp->irq, NULL, dw_dp_irq,
 					IRQF_ONESHOT, dev_name(dev), dp);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to request irq\n");
+	if (ret)
 		goto unregister_aux;
-	}
 
 	return dp;
 
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
index cfd1b18c06e62c..c9568aea98dc88 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
@@ -40,88 +40,6 @@
 
 #define SCRAMB_POLL_DELAY_MS	3000
 
-/*
- * Unless otherwise noted, entries in this table are 100% optimization.
- * Values can be obtained from dw_hdmi_qp_compute_n() but that function is
- * slow so we pre-compute values we expect to see.
- *
- * The values for TMDS 25175, 25200, 27000, 54000, 74250 and 148500 kHz are
- * the recommended N values specified in the Audio chapter of the HDMI
- * specification.
- */
-static const struct dw_hdmi_audio_tmds_n {
-	unsigned long tmds;
-	unsigned int n_32k;
-	unsigned int n_44k1;
-	unsigned int n_48k;
-} common_tmds_n_table[] = {
-	{ .tmds = 25175000,  .n_32k = 4576,  .n_44k1 = 7007,  .n_48k = 6864, },
-	{ .tmds = 25200000,  .n_32k = 4096,  .n_44k1 = 6272,  .n_48k = 6144, },
-	{ .tmds = 27000000,  .n_32k = 4096,  .n_44k1 = 6272,  .n_48k = 6144, },
-	{ .tmds = 28320000,  .n_32k = 4096,  .n_44k1 = 5586,  .n_48k = 6144, },
-	{ .tmds = 30240000,  .n_32k = 4096,  .n_44k1 = 5642,  .n_48k = 6144, },
-	{ .tmds = 31500000,  .n_32k = 4096,  .n_44k1 = 5600,  .n_48k = 6144, },
-	{ .tmds = 32000000,  .n_32k = 4096,  .n_44k1 = 5733,  .n_48k = 6144, },
-	{ .tmds = 33750000,  .n_32k = 4096,  .n_44k1 = 6272,  .n_48k = 6144, },
-	{ .tmds = 36000000,  .n_32k = 4096,  .n_44k1 = 5684,  .n_48k = 6144, },
-	{ .tmds = 40000000,  .n_32k = 4096,  .n_44k1 = 5733,  .n_48k = 6144, },
-	{ .tmds = 49500000,  .n_32k = 4096,  .n_44k1 = 5488,  .n_48k = 6144, },
-	{ .tmds = 50000000,  .n_32k = 4096,  .n_44k1 = 5292,  .n_48k = 6144, },
-	{ .tmds = 54000000,  .n_32k = 4096,  .n_44k1 = 6272,  .n_48k = 6144, },
-	{ .tmds = 65000000,  .n_32k = 4096,  .n_44k1 = 7056,  .n_48k = 6144, },
-	{ .tmds = 68250000,  .n_32k = 4096,  .n_44k1 = 5376,  .n_48k = 6144, },
-	{ .tmds = 71000000,  .n_32k = 4096,  .n_44k1 = 7056,  .n_48k = 6144, },
-	{ .tmds = 72000000,  .n_32k = 4096,  .n_44k1 = 5635,  .n_48k = 6144, },
-	{ .tmds = 73250000,  .n_32k = 11648, .n_44k1 = 14112, .n_48k = 6144, },
-	{ .tmds = 74250000,  .n_32k = 4096,  .n_44k1 = 6272,  .n_48k = 6144, },
-	{ .tmds = 75000000,  .n_32k = 4096,  .n_44k1 = 5880,  .n_48k = 6144, },
-	{ .tmds = 78750000,  .n_32k = 4096,  .n_44k1 = 5600,  .n_48k = 6144, },
-	{ .tmds = 78800000,  .n_32k = 4096,  .n_44k1 = 5292,  .n_48k = 6144, },
-	{ .tmds = 79500000,  .n_32k = 4096,  .n_44k1 = 4704,  .n_48k = 6144, },
-	{ .tmds = 83500000,  .n_32k = 4096,  .n_44k1 = 7056,  .n_48k = 6144, },
-	{ .tmds = 85500000,  .n_32k = 4096,  .n_44k1 = 5488,  .n_48k = 6144, },
-	{ .tmds = 88750000,  .n_32k = 4096,  .n_44k1 = 14112, .n_48k = 6144, },
-	{ .tmds = 97750000,  .n_32k = 4096,  .n_44k1 = 14112, .n_48k = 6144, },
-	{ .tmds = 101000000, .n_32k = 4096,  .n_44k1 = 7056,  .n_48k = 6144, },
-	{ .tmds = 106500000, .n_32k = 4096,  .n_44k1 = 4704,  .n_48k = 6144, },
-	{ .tmds = 108000000, .n_32k = 4096,  .n_44k1 = 5684,  .n_48k = 6144, },
-	{ .tmds = 115500000, .n_32k = 4096,  .n_44k1 = 5712,  .n_48k = 6144, },
-	{ .tmds = 119000000, .n_32k = 4096,  .n_44k1 = 5544,  .n_48k = 6144, },
-	{ .tmds = 135000000, .n_32k = 4096,  .n_44k1 = 5488,  .n_48k = 6144, },
-	{ .tmds = 146250000, .n_32k = 11648, .n_44k1 = 6272,  .n_48k = 6144, },
-	{ .tmds = 148500000, .n_32k = 4096,  .n_44k1 = 6272,  .n_48k = 6144, },
-	{ .tmds = 154000000, .n_32k = 4096,  .n_44k1 = 5544,  .n_48k = 6144, },
-	{ .tmds = 162000000, .n_32k = 4096,  .n_44k1 = 5684,  .n_48k = 6144, },
-
-	/* For 297 MHz+ HDMI spec have some other rule for setting N */
-	{ .tmds = 297000000, .n_32k = 3073,  .n_44k1 = 4704,  .n_48k = 5120, },
-	{ .tmds = 594000000, .n_32k = 3073,  .n_44k1 = 9408,  .n_48k = 10240,},
-
-	/* End of table */
-	{ .tmds = 0,         .n_32k = 0,     .n_44k1 = 0,     .n_48k = 0,    },
-};
-
-/*
- * These are the CTS values as recommended in the Audio chapter of the HDMI
- * specification.
- */
-static const struct dw_hdmi_audio_tmds_cts {
-	unsigned long tmds;
-	unsigned int cts_32k;
-	unsigned int cts_44k1;
-	unsigned int cts_48k;
-} common_tmds_cts_table[] = {
-	{ .tmds = 25175000,  .cts_32k = 28125,  .cts_44k1 = 31250,  .cts_48k = 28125,  },
-	{ .tmds = 25200000,  .cts_32k = 25200,  .cts_44k1 = 28000,  .cts_48k = 25200,  },
-	{ .tmds = 27000000,  .cts_32k = 27000,  .cts_44k1 = 30000,  .cts_48k = 27000,  },
-	{ .tmds = 54000000,  .cts_32k = 54000,  .cts_44k1 = 60000,  .cts_48k = 54000,  },
-	{ .tmds = 74250000,  .cts_32k = 74250,  .cts_44k1 = 82500,  .cts_48k = 74250,  },
-	{ .tmds = 148500000, .cts_32k = 148500, .cts_44k1 = 165000, .cts_48k = 148500, },
-
-	/* End of table */
-	{ .tmds = 0,         .cts_32k = 0,      .cts_44k1 = 0,      .cts_48k = 0,      },
-};
-
 struct dw_hdmi_qp_i2c {
 	struct i2c_adapter	adap;
 
@@ -213,130 +131,6 @@ static void dw_hdmi_qp_set_cts_n(struct dw_hdmi_qp *hdmi, unsigned int cts,
 		       AUDPKT_ACR_CONTROL1);
 }
 
-static int dw_hdmi_qp_match_tmds_n_table(struct dw_hdmi_qp *hdmi,
-					 unsigned long pixel_clk,
-					 unsigned long freq)
-{
-	const struct dw_hdmi_audio_tmds_n *tmds_n = NULL;
-	int i;
-
-	for (i = 0; common_tmds_n_table[i].tmds != 0; i++) {
-		if (pixel_clk == common_tmds_n_table[i].tmds) {
-			tmds_n = &common_tmds_n_table[i];
-			break;
-		}
-	}
-
-	if (!tmds_n)
-		return -ENOENT;
-
-	switch (freq) {
-	case 32000:
-		return tmds_n->n_32k;
-	case 44100:
-	case 88200:
-	case 176400:
-		return (freq / 44100) * tmds_n->n_44k1;
-	case 48000:
-	case 96000:
-	case 192000:
-		return (freq / 48000) * tmds_n->n_48k;
-	default:
-		return -ENOENT;
-	}
-}
-
-static u32 dw_hdmi_qp_audio_math_diff(unsigned int freq, unsigned int n,
-				      unsigned int pixel_clk)
-{
-	u64 cts = mul_u32_u32(pixel_clk, n);
-
-	return do_div(cts, 128 * freq);
-}
-
-static unsigned int dw_hdmi_qp_compute_n(struct dw_hdmi_qp *hdmi,
-					 unsigned long pixel_clk,
-					 unsigned long freq)
-{
-	unsigned int min_n = DIV_ROUND_UP((128 * freq), 1500);
-	unsigned int max_n = (128 * freq) / 300;
-	unsigned int ideal_n = (128 * freq) / 1000;
-	unsigned int best_n_distance = ideal_n;
-	unsigned int best_n = 0;
-	u64 best_diff = U64_MAX;
-	int n;
-
-	/* If the ideal N could satisfy the audio math, then just take it */
-	if (dw_hdmi_qp_audio_math_diff(freq, ideal_n, pixel_clk) == 0)
-		return ideal_n;
-
-	for (n = min_n; n <= max_n; n++) {
-		u64 diff = dw_hdmi_qp_audio_math_diff(freq, n, pixel_clk);
-
-		if (diff < best_diff ||
-		    (diff == best_diff && abs(n - ideal_n) < best_n_distance)) {
-			best_n = n;
-			best_diff = diff;
-			best_n_distance = abs(best_n - ideal_n);
-		}
-
-		/*
-		 * The best N already satisfy the audio math, and also be
-		 * the closest value to ideal N, so just cut the loop.
-		 */
-		if (best_diff == 0 && (abs(n - ideal_n) > best_n_distance))
-			break;
-	}
-
-	return best_n;
-}
-
-static unsigned int dw_hdmi_qp_find_n(struct dw_hdmi_qp *hdmi, unsigned long pixel_clk,
-				      unsigned long sample_rate)
-{
-	int n = dw_hdmi_qp_match_tmds_n_table(hdmi, pixel_clk, sample_rate);
-
-	if (n > 0)
-		return n;
-
-	dev_warn(hdmi->dev, "Rate %lu missing; compute N dynamically\n",
-		 pixel_clk);
-
-	return dw_hdmi_qp_compute_n(hdmi, pixel_clk, sample_rate);
-}
-
-static unsigned int dw_hdmi_qp_find_cts(struct dw_hdmi_qp *hdmi, unsigned long pixel_clk,
-					unsigned long sample_rate)
-{
-	const struct dw_hdmi_audio_tmds_cts *tmds_cts = NULL;
-	int i;
-
-	for (i = 0; common_tmds_cts_table[i].tmds != 0; i++) {
-		if (pixel_clk == common_tmds_cts_table[i].tmds) {
-			tmds_cts = &common_tmds_cts_table[i];
-			break;
-		}
-	}
-
-	if (!tmds_cts)
-		return 0;
-
-	switch (sample_rate) {
-	case 32000:
-		return tmds_cts->cts_32k;
-	case 44100:
-	case 88200:
-	case 176400:
-		return tmds_cts->cts_44k1;
-	case 48000:
-	case 96000:
-	case 192000:
-		return tmds_cts->cts_48k;
-	default:
-		return -ENOENT;
-	}
-}
-
 static void dw_hdmi_qp_set_audio_interface(struct dw_hdmi_qp *hdmi,
 					   struct hdmi_codec_daifmt *fmt,
 					   struct hdmi_codec_params *hparms)
@@ -456,8 +250,7 @@ static void dw_hdmi_qp_set_sample_rate(struct dw_hdmi_qp *hdmi, unsigned long lo
 {
 	unsigned int n, cts;
 
-	n = dw_hdmi_qp_find_n(hdmi, tmds_char_rate, sample_rate);
-	cts = dw_hdmi_qp_find_cts(hdmi, tmds_char_rate, sample_rate);
+	drm_hdmi_acr_get_n_cts(tmds_char_rate, sample_rate, &n, &cts);
 
 	dw_hdmi_qp_set_cts_n(hdmi, cts, n);
 }
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index 34c5f583e9108f..937d61c42b58db 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -3353,7 +3353,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev,
 	hdmi->channels = 2;
 	hdmi->disabled = true;
 	hdmi->rxsense = true;
-	hdmi->phy_mask = (u8)~(HDMI_PHY_HPD | HDMI_PHY_RX_SENSE);
+	hdmi->phy_mask = ~(HDMI_PHY_HPD | HDMI_PHY_RX_SENSE) & 0xff;
 	hdmi->mc_clkdis = 0x7f;
 	hdmi->last_connector_result = connector_status_disconnected;
 
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index e3a57f8228dae1..51f4baa62a53e1 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -35,7 +35,6 @@
 #include <drm/drm_edid.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -370,7 +369,6 @@ struct tc_data {
 	struct drm_dp_aux	aux;
 
 	struct drm_bridge	bridge;
-	struct drm_bridge	*panel_bridge;
 	struct drm_connector	connector;
 
 	struct mipi_dsi_device	*dsi;
@@ -1742,8 +1740,8 @@ static int tc_connector_get_modes(struct drm_connector *connector)
 		return 0;
 	}
 
-	if (tc->panel_bridge) {
-		num_modes = drm_bridge_get_modes(tc->panel_bridge, connector);
+	if (tc->bridge.next_bridge) {
+		num_modes = drm_bridge_get_modes(tc->bridge.next_bridge, connector);
 		if (num_modes > 0)
 			return num_modes;
 	}
@@ -1788,7 +1786,7 @@ tc_connector_detect(struct drm_connector *connector, bool force)
 	if (tc->hpd_pin >= 0)
 		return tc_bridge_detect(&tc->bridge, connector);
 
-	if (tc->panel_bridge)
+	if (tc->bridge.next_bridge)
 		return connector_status_connected;
 	else
 		return connector_status_unknown;
@@ -1809,10 +1807,10 @@ static int tc_dpi_bridge_attach(struct drm_bridge *bridge,
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 
-	if (!tc->panel_bridge)
+	if (!tc->bridge.next_bridge)
 		return 0;
 
-	return drm_bridge_attach(tc->bridge.encoder, tc->panel_bridge,
+	return drm_bridge_attach(tc->bridge.encoder, tc->bridge.next_bridge,
 				 &tc->bridge, flags);
 }
 
@@ -1825,9 +1823,9 @@ static int tc_edp_bridge_attach(struct drm_bridge *bridge,
 	struct drm_device *drm = bridge->dev;
 	int ret;
 
-	if (tc->panel_bridge) {
+	if (tc->bridge.next_bridge) {
 		/* If a connector is required then this driver shall create it */
-		ret = drm_bridge_attach(tc->bridge.encoder, tc->panel_bridge,
+		ret = drm_bridge_attach(tc->bridge.encoder, tc->bridge.next_bridge,
 					&tc->bridge, flags | DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 		if (ret)
 			return ret;
@@ -2318,53 +2316,31 @@ static int tc_probe_dpi_bridge_endpoint(struct tc_data *tc)
 {
 	struct device *dev = tc->dev;
 	struct drm_bridge *bridge;
-	struct drm_panel *panel;
-	int ret;
 
 	/* port@1 is the DPI input/output port */
-	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, 0, &panel, &bridge);
-	if (ret && ret != -ENODEV)
-		return dev_err_probe(dev, ret,
-				     "Could not find DPI panel or bridge\n");
-
-	if (panel) {
-		bridge = devm_drm_panel_bridge_add(dev, panel);
-		drm_panel_put(panel);
-		if (IS_ERR(bridge))
-			return PTR_ERR(bridge);
-	}
+	bridge = of_drm_get_bridge_by_endpoint(dev->of_node, 1, 0);
+	if (IS_ERR(bridge))
+		return dev_err_probe(dev, PTR_ERR(bridge),
+				     "Could not find DPI bridge\n");
 
-	if (bridge) {
-		tc->panel_bridge = bridge;
-		tc->bridge.type = DRM_MODE_CONNECTOR_DPI;
+	tc->bridge.next_bridge = bridge;
+	tc->bridge.type = DRM_MODE_CONNECTOR_DPI;
 
-		return 0;
-	}
-
-	return ret;
+	return 0;
 }
 
 static int tc_probe_edp_bridge_endpoint(struct tc_data *tc)
 {
 	struct device *dev = tc->dev;
-	struct drm_panel *panel;
-	int ret;
+	struct drm_bridge *bridge;
 
 	/* port@2 is the output port */
-	ret = drm_of_find_panel_or_bridge(dev->of_node, 2, 0, &panel, NULL);
-	if (ret && ret != -ENODEV)
-		return dev_err_probe(dev, ret,
-				     "Could not find DSI panel or bridge\n");
-
-	if (panel) {
-		struct drm_bridge *panel_bridge;
-
-		panel_bridge = devm_drm_panel_bridge_add(dev, panel);
-		drm_panel_put(panel);
-		if (IS_ERR(panel_bridge))
-			return PTR_ERR(panel_bridge);
+	bridge = of_drm_get_bridge_by_endpoint(dev->of_node, 2, 0);
+	if (IS_ERR(bridge) && PTR_ERR(bridge) != -ENODEV)
+		return dev_err_probe(dev, PTR_ERR(bridge), "Could not find DSI bridge\n");
 
-		tc->panel_bridge = panel_bridge;
+	if (!IS_ERR(bridge)) {
+		tc->bridge.next_bridge = bridge;
 		tc->bridge.type = DRM_MODE_CONNECTOR_eDP;
 	} else {
 		tc->bridge.type = DRM_MODE_CONNECTOR_DisplayPort;
@@ -2530,10 +2506,8 @@ static int tc_probe(struct i2c_client *client)
 						NULL, tc_irq_handler,
 						IRQF_ONESHOT,
 						"tc358767-irq", tc);
-		if (ret) {
-			dev_err(dev, "failed to register dp interrupt\n");
+		if (ret)
 			return ret;
-		}
 
 		tc->have_irq = true;
 	}
diff --git a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c
index 19b43efcf93f65..18f73ad2589eae 100644
--- a/drivers/gpu/drm/bridge/tc358768.c
+++ b/drivers/gpu/drm/bridge/tc358768.c
@@ -22,7 +22,6 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <video/mipi_display.h>
 #include <video/videomode.h>
 
@@ -137,7 +136,6 @@ static const char * const tc358768_supplies[] = {
 
 struct tc358768_dsi_output {
 	struct mipi_dsi_device *dev;
-	struct drm_panel *panel;
 	struct drm_bridge *bridge;
 };
 
@@ -443,7 +441,6 @@ static int tc358768_dsi_host_attach(struct mipi_dsi_host *host,
 {
 	struct tc358768_priv *priv = dsi_host_to_tc358768(host);
 	struct drm_bridge *bridge;
-	struct drm_panel *panel;
 	struct device_node *ep;
 	int ret;
 
@@ -471,24 +468,15 @@ static int tc358768_dsi_host_attach(struct mipi_dsi_host *host,
 		return -ENOTSUPP;
 	}
 
-	ret = drm_of_find_panel_or_bridge(host->dev->of_node, 1, 0, &panel,
-					  &bridge);
-	if (ret)
-		return ret;
+	bridge = of_drm_get_bridge_by_endpoint(host->dev->of_node, 1, 0);
+	if (IS_ERR(bridge))
+		return PTR_ERR(bridge);
 
-	if (panel) {
-		bridge = drm_panel_bridge_add_typed(panel,
-						    DRM_MODE_CONNECTOR_DSI);
-		drm_panel_put(panel);
-		if (IS_ERR(bridge))
-			return PTR_ERR(bridge);
-
-		bridge->pre_enable_prev_first = true;
-	}
+	bridge->type = DRM_MODE_CONNECTOR_DSI;
+	bridge->pre_enable_prev_first = true;
 
 	priv->output.dev = dev;
 	priv->output.bridge = bridge;
-	priv->output.panel = panel;
 
 	priv->dsi_lanes = dev->lanes;
 	priv->dsi_bpp = mipi_dsi_pixel_format_to_bpp(dev->format);
@@ -518,8 +506,7 @@ static int tc358768_dsi_host_detach(struct mipi_dsi_host *host,
 	struct tc358768_priv *priv = dsi_host_to_tc358768(host);
 
 	drm_bridge_remove(&priv->bridge);
-	if (priv->output.panel)
-		drm_panel_bridge_remove(priv->output.bridge);
+	drm_bridge_put(priv->output.bridge);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/bridge/th1520-dw-hdmi.c b/drivers/gpu/drm/bridge/th1520-dw-hdmi.c
index cbea8b14cd4b0f..2fa7f2d46b3dcd 100644
--- a/drivers/gpu/drm/bridge/th1520-dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/th1520-dw-hdmi.c
@@ -135,8 +135,8 @@ static int th1520_dw_hdmi_probe(struct platform_device *pdev)
 	plat_data->priv_data = hdmi;
 
 	hdmi->dw_hdmi = dw_hdmi_probe(pdev, plat_data);
-	if (IS_ERR(hdmi))
-		return PTR_ERR(hdmi);
+	if (IS_ERR(hdmi->dw_hdmi))
+		return PTR_ERR(hdmi->dw_hdmi);
 
 	platform_set_drvdata(pdev, hdmi);
 
@@ -145,9 +145,9 @@ static int th1520_dw_hdmi_probe(struct platform_device *pdev)
 
 static void th1520_dw_hdmi_remove(struct platform_device *pdev)
 {
-	struct dw_hdmi *hdmi = platform_get_drvdata(pdev);
+	struct th1520_hdmi *hdmi = platform_get_drvdata(pdev);
 
-	dw_hdmi_remove(hdmi);
+	dw_hdmi_remove(hdmi->dw_hdmi);
 }
 
 static const struct of_device_id th1520_dw_hdmi_of_table[] = {
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
index 526826ba4524a5..eb5b249cc43c73 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
@@ -11,6 +11,8 @@
  *   = 1x Single-link DSI ~ 2x Single-link or 1x Dual-link LVDS
  *   - Supported
  *   - Dual-link LVDS mode tested
+ *   - Single-link to LVDS Channel A tested
+ *   - Single-link to LVDS Channel B tested
  *   - 2x Single-link LVDS mode unsupported
  *     (should be easy to add by someone who has the HW)
  * - SN65DSI85
@@ -162,7 +164,7 @@ struct sn65dsi83 {
 	struct gpio_desc		*enable_gpio;
 	struct regulator		*vcc;
 	bool				lvds_dual_link;
-	bool				lvds_dual_link_even_odd_swap;
+	bool				lvds_channel_swap;
 	int				lvds_vod_swing_conf[2];
 	int				lvds_term_conf[2];
 	int				irq;
@@ -644,7 +646,7 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 			REG_LVDS_VCOM_CHA_LVDS_VOD_SWING(ctx->lvds_vod_swing_conf[CHANNEL_A]) |
 			REG_LVDS_VCOM_CHB_LVDS_VOD_SWING(ctx->lvds_vod_swing_conf[CHANNEL_B]));
 	regmap_write(ctx->regmap, REG_LVDS_LANE,
-		     (ctx->lvds_dual_link_even_odd_swap ?
+		     (ctx->lvds_channel_swap ?
 		      REG_LVDS_LANE_EVEN_ODD_SWAP : 0) |
 		     (ctx->lvds_term_conf[CHANNEL_A] ?
 			  REG_LVDS_LANE_CHA_LVDS_TERM : 0) |
@@ -827,48 +829,35 @@ static int sn65dsi83_select_lvds_vod_swing(struct device *dev,
 static int sn65dsi83_parse_lvds_endpoint(struct sn65dsi83 *ctx, int channel)
 {
 	struct device *dev = ctx->dev;
-	struct device_node *endpoint;
-	int endpoint_reg;
+	int endpoint_reg = (channel == CHANNEL_A) ? 2 : 3;
+	struct device_node *endpoint __free(device_node) =
+		of_graph_get_endpoint_by_regs(dev->of_node, endpoint_reg, -1);
 	/* Set so the property can be freely selected if not defined */
 	u32 lvds_vod_swing_data[2] = { 0, 1000000 };
 	u32 lvds_vod_swing_clk[2] = { 0, 1000000 };
 	/* Set default near end terminataion to 200 Ohm */
 	u32 lvds_term = 200;
 	int lvds_vod_swing_conf;
-	int ret = 0;
 	int ret_data;
 	int ret_clock;
 
-	if (channel == CHANNEL_A)
-		endpoint_reg = 2;
-	else
-		endpoint_reg = 3;
-
-	endpoint = of_graph_get_endpoint_by_regs(dev->of_node, endpoint_reg, -1);
-
 	of_property_read_u32(endpoint, "ti,lvds-termination-ohms", &lvds_term);
 	if (lvds_term == 100)
 		ctx->lvds_term_conf[channel] = OHM_100;
 	else if (lvds_term == 200)
 		ctx->lvds_term_conf[channel] = OHM_200;
-	else {
-		ret = -EINVAL;
-		goto exit;
-	}
+	else
+		return -EINVAL;
 
 	ret_data = of_property_read_u32_array(endpoint, "ti,lvds-vod-swing-data-microvolt",
 					lvds_vod_swing_data, ARRAY_SIZE(lvds_vod_swing_data));
-	if (ret_data != 0 && ret_data != -EINVAL) {
-		ret = ret_data;
-		goto exit;
-	}
+	if (ret_data != 0 && ret_data != -EINVAL)
+		return ret_data;
 
 	ret_clock = of_property_read_u32_array(endpoint, "ti,lvds-vod-swing-clock-microvolt",
 					lvds_vod_swing_clk, ARRAY_SIZE(lvds_vod_swing_clk));
-	if (ret_clock != 0 && ret_clock != -EINVAL) {
-		ret = ret_clock;
-		goto exit;
-	}
+	if (ret_clock != 0 && ret_clock != -EINVAL)
+		return ret_clock;
 
 	/* Use default value if both properties are NOT defined. */
 	if (ret_data == -EINVAL && ret_clock == -EINVAL)
@@ -878,23 +867,20 @@ static int sn65dsi83_parse_lvds_endpoint(struct sn65dsi83 *ctx, int channel)
 	if (!ret_data || !ret_clock) {
 		lvds_vod_swing_conf = sn65dsi83_select_lvds_vod_swing(dev, lvds_vod_swing_data,
 						lvds_vod_swing_clk, ctx->lvds_term_conf[channel]);
-		if (lvds_vod_swing_conf < 0) {
-			ret = lvds_vod_swing_conf;
-			goto exit;
-		}
+		if (lvds_vod_swing_conf < 0)
+			return lvds_vod_swing_conf;
 	}
 
 	ctx->lvds_vod_swing_conf[channel] = lvds_vod_swing_conf;
-	ret = 0;
-exit:
-	of_node_put(endpoint);
-	return ret;
+
+	return 0;
 }
 
 static int sn65dsi83_parse_dt(struct sn65dsi83 *ctx, enum sn65dsi83_model model)
 {
 	struct drm_bridge *panel_bridge;
 	struct device *dev = ctx->dev;
+	u32 output_port = 2;
 	int ret;
 
 	ret = sn65dsi83_parse_lvds_endpoint(ctx, CHANNEL_A);
@@ -906,29 +892,38 @@ static int sn65dsi83_parse_dt(struct sn65dsi83 *ctx, enum sn65dsi83_model model)
 		return ret;
 
 	ctx->lvds_dual_link = false;
-	ctx->lvds_dual_link_even_odd_swap = false;
+	ctx->lvds_channel_swap = false;
 	if (model != MODEL_SN65DSI83) {
-		struct device_node *port2, *port3;
+		struct device_node *port0, *port1, *port2, *port3;
 		int dual_link;
 
+		port0 = of_graph_get_port_by_id(dev->of_node, 0);
+		port1 = of_graph_get_port_by_id(dev->of_node, 1);
 		port2 = of_graph_get_port_by_id(dev->of_node, 2);
 		port3 = of_graph_get_port_by_id(dev->of_node, 3);
 		dual_link = drm_of_lvds_get_dual_link_pixel_order(port2, port3);
-		of_node_put(port2);
-		of_node_put(port3);
 
 		if (dual_link == DRM_LVDS_DUAL_LINK_ODD_EVEN_PIXELS) {
-			ctx->lvds_dual_link = true;
 			/* Odd pixels to LVDS Channel A, even pixels to B */
-			ctx->lvds_dual_link_even_odd_swap = false;
-		} else if (dual_link == DRM_LVDS_DUAL_LINK_EVEN_ODD_PIXELS) {
 			ctx->lvds_dual_link = true;
+		} else if (dual_link == DRM_LVDS_DUAL_LINK_EVEN_ODD_PIXELS) {
 			/* Even pixels to LVDS Channel A, odd pixels to B */
-			ctx->lvds_dual_link_even_odd_swap = true;
+			ctx->lvds_dual_link = true;
+			ctx->lvds_channel_swap = true;
+		} else if (port0 && !port1 && port2 && !port3) {
+			/* DSI Channel A to LVDS Channel A */
+		} else if (port0 && !port1 && !port2 && port3) {
+			/* DSI Channel A to LVDS Channel B */
+			ctx->lvds_channel_swap = true;
+			output_port = 3;
 		}
+		of_node_put(port0);
+		of_node_put(port1);
+		of_node_put(port2);
+		of_node_put(port3);
 	}
 
-	panel_bridge = devm_drm_of_get_bridge(dev, dev->of_node, 2, 0);
+	panel_bridge = devm_drm_of_get_bridge(dev, dev->of_node, output_port, 0);
 	if (IS_ERR(panel_bridge))
 		return dev_err_probe(dev, PTR_ERR(panel_bridge), "Failed to get panel bridge\n");
 
@@ -1042,7 +1037,7 @@ static int sn65dsi83_probe(struct i2c_client *client)
 		ret = devm_request_threaded_irq(ctx->dev, ctx->irq, NULL, sn65dsi83_irq,
 						IRQF_ONESHOT, dev_name(ctx->dev), ctx);
 		if (ret)
-			return dev_err_probe(dev, ret, "failed to request irq\n");
+			return ret;
 	}
 
 	dev_set_drvdata(dev, ctx);
diff --git a/drivers/gpu/drm/bridge/ti-tdp158.c b/drivers/gpu/drm/bridge/ti-tdp158.c
index 01fc643bd78f18..8e3e99cbe41a71 100644
--- a/drivers/gpu/drm/bridge/ti-tdp158.c
+++ b/drivers/gpu/drm/bridge/ti-tdp158.c
@@ -8,6 +8,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_panel.h>
 
 struct tdp158 {
 	struct drm_bridge bridge;
diff --git a/drivers/gpu/drm/bridge/waveshare-dsi.c b/drivers/gpu/drm/bridge/waveshare-dsi.c
index 420f3b870a747e..2c40bfef2b450a 100644
--- a/drivers/gpu/drm/bridge/waveshare-dsi.c
+++ b/drivers/gpu/drm/bridge/waveshare-dsi.c
@@ -16,11 +16,9 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 
 struct ws_bridge {
 	struct drm_bridge bridge;
-	struct drm_bridge *next_bridge;
 	struct backlight_device *backlight;
 	struct device *dev;
 	struct regmap *reg_map;
@@ -89,7 +87,7 @@ static int ws_bridge_bridge_attach(struct drm_bridge *bridge,
 {
 	struct ws_bridge *ws = bridge_to_ws_bridge(bridge);
 
-	return drm_bridge_attach(encoder, ws->next_bridge,
+	return drm_bridge_attach(encoder, ws->bridge.next_bridge,
 				 &ws->bridge, flags);
 }
 
@@ -150,7 +148,6 @@ static struct backlight_device *ws_bridge_create_backlight(struct ws_bridge *ws)
 static int ws_bridge_probe(struct i2c_client *i2c)
 {
 	struct device *dev = &i2c->dev;
-	struct drm_panel *panel;
 	struct ws_bridge *ws;
 	int ret;
 
@@ -164,14 +161,10 @@ static int ws_bridge_probe(struct i2c_client *i2c)
 	if (IS_ERR(ws->reg_map))
 		return dev_err_probe(dev, PTR_ERR(ws->reg_map), "Failed to allocate regmap\n");
 
-	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, -1, &panel, NULL);
-	if (ret)
-		return dev_err_probe(dev, ret, "Failed to find remote panel\n");
-
-	ws->next_bridge = devm_drm_panel_bridge_add(dev, panel);
-	drm_panel_put(panel);
-	if (IS_ERR(ws->next_bridge))
-		return PTR_ERR(ws->next_bridge);
+	ws->bridge.next_bridge = of_drm_get_bridge_by_endpoint(dev->of_node, 1, -1);
+	if (IS_ERR(ws->bridge.next_bridge))
+		return dev_err_probe(dev, PTR_ERR(ws->bridge.next_bridge),
+				     "Failed to find remote bridge\n");
 
 	ws->backlight = ws_bridge_create_backlight(ws);
 	if (IS_ERR(ws->backlight)) {
diff --git a/drivers/gpu/drm/clients/drm_fbdev_client.c b/drivers/gpu/drm/clients/drm_fbdev_client.c
index 91d196a397cf2e..7574bd36f45eb1 100644
--- a/drivers/gpu/drm/clients/drm_fbdev_client.c
+++ b/drivers/gpu/drm/clients/drm_fbdev_client.c
@@ -42,11 +42,27 @@ static int drm_fbdev_client_restore(struct drm_client_dev *client, bool force)
 {
 	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
 
+	/*
+	 * The client is registered before the initial fbdev probe.
+	 * If probing failed, the client remains registered but there
+	 * is no valid fbdev framebuffer to restore.
+	 */
+	if (!fb_helper->info || !fb_helper->fb)
+		return 0;
+
 	drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper, force);
 
 	return 0;
 }
 
+static void drm_fbdev_client_acquire_outputs(struct drm_client_dev *client)
+{
+	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
+
+	if (fb_helper->info)
+		fb_switch_outputs(fb_helper->info);
+}
+
 static int drm_fbdev_client_hotplug(struct drm_client_dev *client)
 {
 	struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client);
@@ -95,13 +111,14 @@ static int drm_fbdev_client_resume(struct drm_client_dev *client)
 }
 
 static const struct drm_client_funcs drm_fbdev_client_funcs = {
-	.owner		= THIS_MODULE,
-	.free		= drm_fbdev_client_free,
-	.unregister	= drm_fbdev_client_unregister,
-	.restore	= drm_fbdev_client_restore,
-	.hotplug	= drm_fbdev_client_hotplug,
-	.suspend	= drm_fbdev_client_suspend,
-	.resume		= drm_fbdev_client_resume,
+	.owner		 = THIS_MODULE,
+	.free		 = drm_fbdev_client_free,
+	.unregister	 = drm_fbdev_client_unregister,
+	.restore	 = drm_fbdev_client_restore,
+	.acquire_outputs = drm_fbdev_client_acquire_outputs,
+	.hotplug	 = drm_fbdev_client_hotplug,
+	.suspend	 = drm_fbdev_client_suspend,
+	.resume		 = drm_fbdev_client_resume,
 };
 
 /**
diff --git a/drivers/gpu/drm/clients/drm_log.c b/drivers/gpu/drm/clients/drm_log.c
index 99062609d9a6a0..4654091fe11adb 100644
--- a/drivers/gpu/drm/clients/drm_log.c
+++ b/drivers/gpu/drm/clients/drm_log.c
@@ -89,25 +89,21 @@ static void drm_log_blit(struct iosys_map *dst, unsigned int dst_pitch,
 	}
 }
 
-static void drm_log_clear_line(struct drm_log_scanout *scanout, u32 line)
+static void drm_log_clear_line(struct drm_log_scanout *scanout, u32 line,
+			       struct iosys_map map)
 {
 	struct drm_framebuffer *fb = scanout->buffer->fb;
 	unsigned long height = scanout->scaled_font_h;
-	struct iosys_map map;
 	struct drm_rect r = DRM_RECT_INIT(0, line * height, fb->width, height);
 
-	if (drm_client_buffer_vmap_local(scanout->buffer, &map))
-		return;
 	iosys_map_memset(&map, r.y1 * fb->pitches[0], 0, height * fb->pitches[0]);
-	drm_client_buffer_vunmap_local(scanout->buffer);
-	drm_client_buffer_flush(scanout->buffer, &r);
 }
 
 static void drm_log_draw_line(struct drm_log_scanout *scanout, const char *s,
-			      unsigned int len, unsigned int prefix_len)
+			      unsigned int len, unsigned int prefix_len,
+			      struct iosys_map map)
 {
 	struct drm_framebuffer *fb = scanout->buffer->fb;
-	struct iosys_map map;
 	const struct font_desc *font = scanout->font;
 	size_t font_pitch = DIV_ROUND_UP(font->width, 8);
 	const u8 *src;
@@ -116,10 +112,9 @@ static void drm_log_draw_line(struct drm_log_scanout *scanout, const char *s,
 					  fb->width, (scanout->line + 1) * scanout->scaled_font_h);
 	u32 i;
 
-	if (drm_client_buffer_vmap_local(scanout->buffer, &map))
-		return;
 
 	iosys_map_incr(&map, r.y1 * fb->pitches[0]);
+
 	for (i = 0; i < len && i < scanout->columns; i++) {
 		u32 color = (i < prefix_len) ? scanout->prefix_color : scanout->front_color;
 		src = font_data_glyph_buf(font->data, font->width, font->height,
@@ -134,21 +129,40 @@ static void drm_log_draw_line(struct drm_log_scanout *scanout, const char *s,
 	scanout->line++;
 	if (scanout->line >= scanout->rows)
 		scanout->line = 0;
-	drm_client_buffer_vunmap_local(scanout->buffer);
-	drm_client_buffer_flush(scanout->buffer, &r);
 }
 
 static void drm_log_draw_new_line(struct drm_log_scanout *scanout,
-				  const char *s, unsigned int len, unsigned int prefix_len)
+				  const char *s, unsigned int len,
+				  unsigned int prefix_len)
 {
+	struct iosys_map map;
+	struct drm_framebuffer *fb = scanout->buffer->fb;
+	u32 height = scanout->scaled_font_h;
+	u32 line = scanout->line;
+	u32 y2;
+	struct drm_rect dirty;
+
+	if (drm_client_buffer_vmap_local(scanout->buffer, &map))
+		return;
+
 	if (scanout->line == 0) {
-		drm_log_clear_line(scanout, 0);
-		drm_log_clear_line(scanout, 1);
-		drm_log_clear_line(scanout, 2);
-	} else if (scanout->line + 2 < scanout->rows)
-		drm_log_clear_line(scanout, scanout->line + 2);
+		drm_log_clear_line(scanout, 0, map);
+		drm_log_clear_line(scanout, 1, map);
+		drm_log_clear_line(scanout, 2, map);
+		y2 = min(3, scanout->rows) * height;
+	} else if (scanout->line + 2 < scanout->rows) {
+		drm_log_clear_line(scanout, scanout->line + 2, map);
+		y2 = (line + 3) * height;
+	} else {
+		y2 = (line + 1) * height;
+	}
+
+	drm_log_draw_line(scanout, s, len, prefix_len, map);
+
+	drm_client_buffer_vunmap_local(scanout->buffer);
 
-	drm_log_draw_line(scanout, s, len, prefix_len);
+	dirty = DRM_RECT_INIT(0, line * height, fb->width, y2 - line * height);
+	drm_client_buffer_flush(scanout->buffer, &dirty);
 }
 
 /*
diff --git a/drivers/gpu/drm/display/drm_bridge_connector.c b/drivers/gpu/drm/display/drm_bridge_connector.c
index 8b54069fa53a42..81f3e26f6fdf5b 100644
--- a/drivers/gpu/drm/display/drm_bridge_connector.c
+++ b/drivers/gpu/drm/display/drm_bridge_connector.c
@@ -18,6 +18,7 @@
 #include <drm/drm_edid.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/display/drm_hdcp_helper.h>
@@ -208,49 +209,6 @@ static void drm_bridge_connector_disable_hpd(struct drm_connector *connector)
  * Bridge Connector Functions
  */
 
-static enum drm_connector_status
-drm_bridge_connector_detect(struct drm_connector *connector, bool force)
-{
-	struct drm_bridge_connector *bridge_connector =
-		to_drm_bridge_connector(connector);
-	struct drm_bridge *detect = bridge_connector->bridge_detect;
-	struct drm_bridge *hdmi = bridge_connector->bridge_hdmi;
-	enum drm_connector_status status;
-
-	if (detect) {
-		status = detect->funcs->detect(detect, connector);
-
-		if (hdmi)
-			drm_atomic_helper_connector_hdmi_hotplug(connector, status);
-
-		drm_bridge_connector_hpd_notify(connector, status);
-	} else {
-		switch (connector->connector_type) {
-		case DRM_MODE_CONNECTOR_DPI:
-		case DRM_MODE_CONNECTOR_LVDS:
-		case DRM_MODE_CONNECTOR_DSI:
-		case DRM_MODE_CONNECTOR_eDP:
-			status = connector_status_connected;
-			break;
-		default:
-			status = connector_status_unknown;
-			break;
-		}
-	}
-
-	return status;
-}
-
-static void drm_bridge_connector_force(struct drm_connector *connector)
-{
-	struct drm_bridge_connector *bridge_connector =
-		to_drm_bridge_connector(connector);
-	struct drm_bridge *hdmi = bridge_connector->bridge_hdmi;
-
-	if (hdmi)
-		drm_atomic_helper_connector_hdmi_force(connector);
-}
-
 static void drm_bridge_connector_debugfs_init(struct drm_connector *connector,
 					      struct dentry *root)
 {
@@ -263,6 +221,9 @@ static void drm_bridge_connector_debugfs_init(struct drm_connector *connector,
 		if (bridge->funcs->debugfs_init)
 			bridge->funcs->debugfs_init(bridge, root);
 	}
+
+	if (bridge_connector->bridge_hdmi)
+		drm_hdmi_connector_debugfs_init(connector, root);
 }
 
 static struct drm_connector_state *
@@ -307,8 +268,6 @@ drm_bridge_connector_color_format(const struct drm_connector_state *conn_state)
 }
 
 static const struct drm_connector_funcs drm_bridge_connector_funcs = {
-	.detect = drm_bridge_connector_detect,
-	.force = drm_bridge_connector_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.atomic_create_state = drm_bridge_connector_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -322,15 +281,64 @@ static const struct drm_connector_funcs drm_bridge_connector_funcs = {
  * Bridge Connector Helper Functions
  */
 
+static int drm_bridge_connector_detect_ctx(struct drm_connector *connector,
+					   struct drm_modeset_acquire_ctx *ctx,
+					   bool force)
+{
+	struct drm_bridge_connector *bridge_connector =
+		to_drm_bridge_connector(connector);
+	struct drm_bridge *detect = bridge_connector->bridge_detect;
+	struct drm_bridge *hdmi = bridge_connector->bridge_hdmi;
+	enum drm_connector_status status;
+	int ret;
+
+	if (detect) {
+		status = detect->funcs->detect(detect, connector);
+
+		if (hdmi) {
+			ret = drm_atomic_helper_connector_hdmi_hotplug(connector, ctx, status);
+			if (ret == -EDEADLK)
+				return ret;
+		}
+
+		drm_bridge_connector_hpd_notify(connector, status);
+	} else {
+		switch (connector->connector_type) {
+		case DRM_MODE_CONNECTOR_DPI:
+		case DRM_MODE_CONNECTOR_LVDS:
+		case DRM_MODE_CONNECTOR_DSI:
+		case DRM_MODE_CONNECTOR_eDP:
+			status = connector_status_connected;
+			break;
+		default:
+			status = connector_status_unknown;
+			break;
+		}
+	}
+
+	return status;
+}
+
+static int drm_bridge_connector_force_ctx(struct drm_connector *connector,
+					  struct drm_modeset_acquire_ctx *ctx)
+{
+	struct drm_bridge_connector *bridge_connector =
+		to_drm_bridge_connector(connector);
+	struct drm_bridge *hdmi = bridge_connector->bridge_hdmi;
+
+	if (hdmi)
+		return drm_atomic_helper_connector_hdmi_force_ctx(connector, ctx);
+
+	return 0;
+}
+
 static int drm_bridge_connector_get_modes_edid(struct drm_connector *connector,
 					       struct drm_bridge *bridge)
 {
-	enum drm_connector_status status;
 	const struct drm_edid *drm_edid;
 	int n;
 
-	status = drm_bridge_connector_detect(connector, false);
-	if (status != connector_status_connected)
+	if (connector->status != connector_status_connected)
 		goto no_edid;
 
 	drm_edid = drm_bridge_edid_read(bridge, connector);
@@ -415,6 +423,8 @@ static int drm_bridge_connector_atomic_check(struct drm_connector *connector,
 
 static const struct drm_connector_helper_funcs drm_bridge_connector_helper_funcs = {
 	.get_modes = drm_bridge_connector_get_modes,
+	.detect_ctx = drm_bridge_connector_detect_ctx,
+	.force_ctx = drm_bridge_connector_force_ctx,
 	.mode_valid = drm_bridge_connector_mode_valid,
 	.enable_hpd = drm_bridge_connector_enable_hpd,
 	.disable_hpd = drm_bridge_connector_disable_hpd,
@@ -575,6 +585,32 @@ static int drm_bridge_connector_write_spd_infoframe(struct drm_connector *connec
 	return bridge->funcs->hdmi_write_spd_infoframe(bridge, buffer, len);
 }
 
+static int drm_bridge_connector_scrambler_enable(struct drm_connector *connector)
+{
+	struct drm_bridge_connector *bridge_connector =
+		to_drm_bridge_connector(connector);
+	struct drm_bridge *bridge;
+
+	bridge = bridge_connector->bridge_hdmi;
+	if (!bridge)
+		return -EINVAL;
+
+	return bridge->funcs->hdmi_scrambler_enable(bridge);
+}
+
+static int drm_bridge_connector_scrambler_disable(struct drm_connector *connector)
+{
+	struct drm_bridge_connector *bridge_connector =
+		to_drm_bridge_connector(connector);
+	struct drm_bridge *bridge;
+
+	bridge = bridge_connector->bridge_hdmi;
+	if (!bridge)
+		return -EINVAL;
+
+	return bridge->funcs->hdmi_scrambler_disable(bridge);
+}
+
 static const struct drm_edid *
 drm_bridge_connector_read_edid(struct drm_connector *connector)
 {
@@ -590,6 +626,8 @@ drm_bridge_connector_read_edid(struct drm_connector *connector)
 }
 
 static const struct drm_connector_hdmi_funcs drm_bridge_connector_hdmi_funcs = {
+	.supported_formats = BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444),
+	.max_bpc = 8,
 	.tmds_char_rate_valid = drm_bridge_connector_tmds_char_rate_valid,
 	.read_edid = drm_bridge_connector_read_edid,
 	.avi = {
@@ -600,7 +638,7 @@ static const struct drm_connector_hdmi_funcs drm_bridge_connector_hdmi_funcs = {
 		.clear_infoframe = drm_bridge_connector_clear_hdmi_infoframe,
 		.write_infoframe = drm_bridge_connector_write_hdmi_infoframe,
 	},
-	/* audio, hdr_drm and spd are set dynamically during init */
+	/* scrambler, audio, hdr_drm and spd are set dynamically during init */
 };
 
 static const struct drm_connector_infoframe_funcs drm_bridge_connector_hdmi_audio_infoframe = {
@@ -823,8 +861,6 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 	struct drm_connector *connector;
 	struct i2c_adapter *ddc = NULL;
 	struct drm_bridge *panel_bridge __free(drm_bridge_put) = NULL;
-	unsigned int supported_formats = BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444);
-	unsigned int max_bpc = 8;
 	bool support_hdcp = false;
 	int connector_type;
 	int ret;
@@ -906,12 +942,12 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 			     !bridge->funcs->hdmi_clear_spd_infoframe))
 				return ERR_PTR(-EINVAL);
 
-			bridge_connector->bridge_hdmi = drm_bridge_get(bridge);
+			if (bridge->supported_hdmi_ver >= HDMI_VERSION_2_0 &&
+			    (!bridge->funcs->hdmi_scrambler_enable ||
+			     !bridge->funcs->hdmi_scrambler_disable))
+				return ERR_PTR(-EINVAL);
 
-			if (bridge->supported_formats)
-				supported_formats = bridge->supported_formats;
-			if (bridge->max_bpc)
-				max_bpc = bridge->max_bpc;
+			bridge_connector->bridge_hdmi = drm_bridge_get(bridge);
 		}
 
 		if (bridge->ops & DRM_BRIDGE_OP_HDMI_AUDIO) {
@@ -993,10 +1029,30 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 		return ERR_PTR(-EINVAL);
 
 	if (bridge_connector->bridge_hdmi) {
+		bridge_connector->hdmi_funcs = drm_bridge_connector_hdmi_funcs;
+
+		bridge_connector->hdmi_funcs.vendor = bridge_connector->bridge_hdmi->vendor;
+		bridge_connector->hdmi_funcs.product = bridge_connector->bridge_hdmi->product;
+
+		if (bridge_connector->bridge_hdmi->supported_hdmi_ver)
+			bridge_connector->hdmi_funcs.supported_hdmi_ver =
+				bridge_connector->bridge_hdmi->supported_hdmi_ver;
+
+		if (bridge_connector->bridge_hdmi->supported_formats)
+			bridge_connector->hdmi_funcs.supported_formats =
+				bridge_connector->bridge_hdmi->supported_formats;
+
 		if (!connector->ycbcr_420_allowed)
-			supported_formats &= ~BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
+			bridge_connector->hdmi_funcs.supported_formats &=
+				~BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420);
 
-		bridge_connector->hdmi_funcs = drm_bridge_connector_hdmi_funcs;
+		if (bridge_connector->bridge_hdmi->max_tmds_char_rate)
+			bridge_connector->hdmi_funcs.supported_tmds_char_rate =
+				bridge_connector->bridge_hdmi->max_tmds_char_rate;
+
+		if (bridge_connector->bridge_hdmi->max_bpc)
+			bridge_connector->hdmi_funcs.max_bpc =
+				bridge_connector->bridge_hdmi->max_bpc;
 
 		if (bridge_connector->bridge_hdmi->ops & DRM_BRIDGE_OP_HDMI_AUDIO)
 			bridge_connector->hdmi_funcs.audio =
@@ -1010,14 +1066,17 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 			bridge_connector->hdmi_funcs.spd =
 				drm_bridge_connector_hdmi_spd_infoframe;
 
+		if (bridge_connector->bridge_hdmi->supported_hdmi_ver >= HDMI_VERSION_2_0) {
+			bridge_connector->hdmi_funcs.scrambler_enable =
+				drm_bridge_connector_scrambler_enable;
+			bridge_connector->hdmi_funcs.scrambler_disable =
+				drm_bridge_connector_scrambler_disable;
+		}
+
 		ret = drmm_connector_hdmi_init(drm, connector,
-					       bridge_connector->bridge_hdmi->vendor,
-					       bridge_connector->bridge_hdmi->product,
 					       &drm_bridge_connector_funcs,
 					       &bridge_connector->hdmi_funcs,
-					       connector_type, ddc,
-					       supported_formats,
-					       max_bpc);
+					       connector_type, ddc);
 		if (ret)
 			return ERR_PTR(ret);
 	} else {
diff --git a/drivers/gpu/drm/display/drm_dp_cec.c b/drivers/gpu/drm/display/drm_dp_cec.c
index 436bfe9f90819c..476a4aa55fa669 100644
--- a/drivers/gpu/drm/display/drm_dp_cec.c
+++ b/drivers/gpu/drm/display/drm_dp_cec.c
@@ -8,6 +8,7 @@
 #include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/seq_file.h>
 #include <linux/slab.h>
 
 #include <media/cec.h>
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
index 7ce9e212770ad1..b7fce0bf8acc5f 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
@@ -2161,6 +2161,9 @@ ssize_t drm_dp_mst_dpcd_read(struct drm_dp_aux *aux,
 	struct drm_dp_mst_port *port = container_of(aux, struct drm_dp_mst_port,
 						    aux);
 
+	if (!port->ddps)
+		return -EIO;
+
 	return drm_dp_send_dpcd_read(port->mgr, port,
 				     offset, size, buffer);
 }
@@ -2184,6 +2187,9 @@ ssize_t drm_dp_mst_dpcd_write(struct drm_dp_aux *aux,
 	struct drm_dp_mst_port *port = container_of(aux, struct drm_dp_mst_port,
 						    aux);
 
+	if (!port->ddps)
+		return -EIO;
+
 	return drm_dp_send_dpcd_write(port->mgr, port,
 				      offset, size, buffer);
 }
diff --git a/drivers/gpu/drm/display/drm_hdmi_helper.c b/drivers/gpu/drm/display/drm_hdmi_helper.c
index 5cb0b033b1713e..f85cce3d62547a 100644
--- a/drivers/gpu/drm/display/drm_hdmi_helper.c
+++ b/drivers/gpu/drm/display/drm_hdmi_helper.c
@@ -4,12 +4,21 @@
 #include <linux/module.h>
 
 #include <drm/display/drm_hdmi_helper.h>
+#include <drm/display/drm_scdc_helper.h>
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_modeset_lock.h>
 #include <drm/drm_print.h>
 #include <drm/drm_property.h>
 
+#define DRM_HDMI_SCDC_SOURCE_VERSION	1
+#define DRM_HDMI_SCDC_POLL_DELAY_MS	1000
+
 static inline bool is_eotf_supported(u8 output_eotf, u8 sink_eotf)
 {
 	return sink_eotf & BIT(output_eotf);
@@ -426,3 +435,298 @@ drm_hdmi_acr_get_n_cts(unsigned long long tmds_char_rate,
 	*out_cts = cts;
 }
 EXPORT_SYMBOL(drm_hdmi_acr_get_n_cts);
+
+/**
+ * drm_hdmi_mode_needs_scrambling() - Check if an HDMI mode requires scrambling
+ * @mode: Display mode to check
+ * @bpc: Bits per color component
+ * @fmt: Output pixel format
+ *
+ * Computes the TMDS character rate for the given mode, bits per component and
+ * output format, and checks whether it exceeds the HDMI 1.3 maximum TMDS
+ * character rate of 340 MHz.
+ *
+ * Returns:
+ * true if the mode requires scrambling, false otherwise.
+ */
+bool drm_hdmi_mode_needs_scrambling(const struct drm_display_mode *mode,
+				    unsigned int bpc,
+				    enum drm_output_color_format fmt)
+{
+	unsigned long long tmds_char_rate;
+
+	tmds_char_rate = drm_hdmi_compute_mode_clock(mode, bpc, fmt);
+
+	return tmds_char_rate > HDMI_1_3_TMDS_CHAR_RATE_MAX_HZ;
+}
+EXPORT_SYMBOL(drm_hdmi_mode_needs_scrambling);
+
+static bool drm_scdc_sink_supports_scrambling(struct drm_connector *connector)
+{
+	const struct drm_display_info *info = &connector->display_info;
+
+	return info->is_hdmi &&
+	       info->hdmi.scdc.supported &&
+	       info->hdmi.scdc.scrambling.supported;
+}
+
+static int drm_connector_hdmi_try_scrambling_setup(struct drm_connector *connector)
+{
+	bool done;
+
+	done = drm_scdc_set_high_tmds_clock_ratio(connector, true);
+	if (!done)
+		return -EIO;
+
+	done = drm_scdc_set_scrambling(connector, true);
+	if (!done)
+		return -EIO;
+
+	if (READ_ONCE(connector->hdmi.scrambler_enabled))
+		schedule_delayed_work(&connector->hdmi.scdc_work,
+				      msecs_to_jiffies(DRM_HDMI_SCDC_POLL_DELAY_MS));
+
+	return 0;
+}
+
+static void drm_connector_hdmi_scdc_work(struct work_struct *work)
+{
+	struct drm_connector *connector;
+	struct drm_connector_hdmi *hdmi;
+
+	hdmi = container_of(to_delayed_work(work), struct drm_connector_hdmi,
+			    scdc_work);
+	connector = container_of(hdmi, struct drm_connector, hdmi);
+
+	/*
+	 * The connector, and hence this work item, may outlive the source
+	 * driver, e.g. when userspace keeps a DRM file descriptor open past
+	 * unbind. Never poke the DDC bus after the connector got unregistered,
+	 * as the underlying I2C adapter may already be gone.
+	 */
+	if (drm_connector_is_unregistered(connector))
+		return;
+
+	if (READ_ONCE(connector->hdmi.scrambler_enabled) &&
+	    !drm_scdc_get_scrambling_status(connector))
+		drm_connector_hdmi_try_scrambling_setup(connector);
+}
+
+/**
+ * drm_connector_hdmi_enable_scrambling() - enable scrambling and monitor SCDC status
+ * @connector: connector
+ * @conn_state: connector state
+ *
+ * Enables scrambling and high TMDS clock ratio on both source and sink sides.
+ * Additionally, use a delayed work item to monitor the scrambling status on
+ * the sink side and retry the operation, as some displays refuse to set the
+ * scrambling bit right away.
+ *
+ * The work item is cancelled by drm_connector_hdmi_disable_scrambling(), hence
+ * source drivers must ensure the display pipeline is shut down on unbind, e.g.
+ * via drm_atomic_helper_shutdown(). Otherwise the work may outlive the driver's
+ * devres-managed resources, since the connector itself is only cleaned up when
+ * the last reference to the DRM device is dropped.
+ *
+ * Returns:
+ * Zero if scrambling is set successfully, an error code otherwise.
+ */
+int drm_connector_hdmi_enable_scrambling(struct drm_connector *connector,
+					 const struct drm_connector_state *conn_state)
+{
+	struct drm_connector_hdmi *hdmi = &connector->hdmi;
+	struct drm_device *dev = connector->dev;
+	int ret;
+
+	if (!conn_state)
+		return -EINVAL;
+
+	if (!conn_state->hdmi.scrambler_needed)
+		return 0;
+
+	if (!drm_connector_hdmi_scrambler_supported(connector)) {
+		drm_dbg_kms(dev, "Source doesn't support scrambling.\n");
+		return -EINVAL;
+	}
+
+	if (!drm_scdc_sink_supports_scrambling(connector)) {
+		drm_dbg_kms(dev, "Sink doesn't support scrambling.\n");
+		return -EINVAL;
+	}
+
+	if (!connector->ddc)
+		return -EINVAL;
+
+	/*
+	 * Advertise our SCDC source version. This is purely informational and
+	 * does not gate scrambling, so failures are non-fatal.
+	 */
+	drm_scdc_set_source_version(connector, DRM_HDMI_SCDC_SOURCE_VERSION);
+
+	drm_dbg_kms(dev, "Enabling scrambling\n");
+
+	if (!hdmi->scdc_work_initialized) {
+		INIT_DELAYED_WORK(&hdmi->scdc_work,
+				  drm_connector_hdmi_scdc_work);
+		hdmi->scdc_work_initialized = true;
+	}
+
+	WRITE_ONCE(hdmi->scrambler_enabled, true);
+
+	ret = drm_connector_hdmi_try_scrambling_setup(connector);
+	if (!ret)
+		ret = hdmi->funcs->scrambler_enable(connector);
+
+	if (ret) {
+		WRITE_ONCE(hdmi->scrambler_enabled, false);
+		cancel_delayed_work_sync(&hdmi->scdc_work);
+
+		drm_scdc_set_scrambling(connector, false);
+		drm_scdc_set_high_tmds_clock_ratio(connector, false);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_connector_hdmi_enable_scrambling);
+
+/**
+ * drm_connector_hdmi_disable_scrambling() - disable scrambling and SCDC monitoring
+ * @connector: connector
+ *
+ * Disables scrambling and high TMDS clock ratio on both source and sink sides.
+ * Also cancels the SCDC status monitoring work item, if it is still pending.
+ *
+ * Returns:
+ * Zero if scrambling is reset successfully, an error code otherwise.
+ */
+int drm_connector_hdmi_disable_scrambling(struct drm_connector *connector)
+{
+	struct drm_connector_hdmi *hdmi = &connector->hdmi;
+	struct drm_device *dev = connector->dev;
+
+	if (!READ_ONCE(hdmi->scrambler_enabled))
+		return 0;
+
+	drm_dbg_kms(dev, "Disabling scrambling\n");
+
+	WRITE_ONCE(hdmi->scrambler_enabled, false);
+
+	/*
+	 * A driver may force scrambler_enabled at init to trigger a disable
+	 * at boot (e.g. vc4), bypassing the enable path that sets up the work
+	 * item. Only cancel it once it has actually been initialized.
+	 */
+	if (hdmi->scdc_work_initialized)
+		cancel_delayed_work_sync(&hdmi->scdc_work);
+
+	if (connector->ddc &&
+	    connector->status == connector_status_connected &&
+	    drm_scdc_sink_supports_scrambling(connector)) {
+		drm_scdc_set_scrambling(connector, false);
+		drm_scdc_set_high_tmds_clock_ratio(connector, false);
+	}
+
+	return hdmi->funcs->scrambler_disable(connector);
+}
+EXPORT_SYMBOL(drm_connector_hdmi_disable_scrambling);
+
+/**
+ * drm_connector_hdmi_sync_scdc() - resync the sink-side SCDC upon reconnect
+ * @connector: connector
+ * @plugged: connector plugged status event
+ * @ctx: initialized lock acquisition context
+ *
+ * When receiving hotplug disconnect/reconnect event, while the display is
+ * still active (CRTC enabled), the SCDC status on the sink side is reset
+ * and must be explicitly restored.
+ *
+ * The typical solution for this is to trigger an empty modeset in
+ * drm_connector_helper_funcs.detect_ctx(), which is what this helper does
+ * by triggering a CRTC reset on reconnection.
+ *
+ * When making use of the HDMI connector framework, this is automatically
+ * triggered via drm_atomic_helper_connector_hdmi_hotplug().
+ *
+ * Returns:
+ * Zero on success, an error code otherwise, including -EDEADLK.
+ */
+int drm_connector_hdmi_sync_scdc(struct drm_connector *connector, bool plugged,
+				 struct drm_modeset_acquire_ctx *ctx)
+{
+	struct drm_connector_hdmi *hdmi = &connector->hdmi;
+	struct drm_device *dev = connector->dev;
+	struct drm_connector_state *conn_state;
+	struct drm_crtc *crtc;
+	u8 config;
+	int ret;
+
+	if (!connector || !plugged || !ctx)
+		return 0;
+
+	/*
+	 * This is normally part of .detect_ctx() call path, which already holds
+	 * connection_mutex through @ctx.  However, re-acquiring it with the
+	 * same context is a no-op and makes the helper safe under any caller.
+	 */
+	ret = drm_modeset_lock(&connector->dev->mode_config.connection_mutex, ctx);
+	if (ret)
+		return ret;
+
+	/* TODO: Also handle HDMI 2.1 FRL link training */
+
+	if (!drm_connector_hdmi_scrambler_supported(connector) ||
+	    !READ_ONCE(hdmi->scrambler_enabled))
+		return 0;
+
+	conn_state = connector->state;
+	if (!conn_state)
+		return 0;
+
+	crtc = conn_state->crtc;
+	if (!crtc)
+		return 0;
+
+	ret = drm_modeset_lock(&crtc->mutex, ctx);
+	if (ret)
+		return ret;
+
+	if (!crtc->state || !crtc->state->active)
+		return 0;
+
+	if (!connector->ddc || !drm_scdc_sink_supports_scrambling(connector))
+		return 0;
+
+	if (conn_state->commit &&
+	    !try_wait_for_completion(&conn_state->commit->hw_done))
+		return 0;
+
+	ret = drm_scdc_readb(connector->ddc, SCDC_TMDS_CONFIG, &config);
+	if (ret) {
+		drm_dbg_kms(dev, "Failed to read TMDS config: %d\n", ret);
+		return ret;
+	}
+
+	if (config & SCDC_SCRAMBLING_ENABLE)
+		return 0;
+
+	/*
+	 * HDMI 2.0 requires that one should not send scrambled data prior
+	 * to configuring the sink scrambling, and that TMDS clock/data
+	 * transmission should be suspended when changing the TMDS clock rate
+	 * in the sink.  So let's just do a full modeset here, even though some
+	 * sinks would be perfectly happy if were to just reconfigure the SCDC
+	 * settings on the fly.
+	 *
+	 * The disable/re-enable cycle triggered by the reset should call
+	 * drm_connector_hdmi_enable_scrambling() during re-enable, thus
+	 * properly configuring the sink before data transmission resumes.
+	 */
+	drm_dbg_kms(dev, "Resetting CRTC to restore SCDC status\n");
+
+	ret = drm_atomic_helper_reset_crtc(crtc, ctx);
+	if (ret && ret != -EDEADLK)
+		drm_dbg_kms(dev, "Failed to reset CRTC: %d\n", ret);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_connector_hdmi_sync_scdc);
diff --git a/drivers/gpu/drm/display/drm_hdmi_state_helper.c b/drivers/gpu/drm/display/drm_hdmi_state_helper.c
index d6669e36ca1b8c..7cdb7ca3dc12fa 100644
--- a/drivers/gpu/drm/display/drm_hdmi_state_helper.c
+++ b/drivers/gpu/drm/display/drm_hdmi_state_helper.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: MIT
 
+#include <linux/debugfs.h>
 #include <linux/export.h>
 
 #include <drm/drm_atomic.h>
@@ -12,6 +13,7 @@
 #include <drm/display/drm_hdmi_cec_helper.h>
 #include <drm/display/drm_hdmi_helper.h>
 #include <drm/display/drm_hdmi_state_helper.h>
+#include <drm/display/drm_scdc_helper.h>
 
 /**
  * DOC: hdmi helpers
@@ -560,6 +562,10 @@ hdmi_clock_valid(const struct drm_connector *connector,
 	if (info->max_tmds_clock && clock > info->max_tmds_clock * 1000)
 		return MODE_CLOCK_HIGH;
 
+	if (connector->hdmi.max_tmds_char_rate &&
+	    clock > connector->hdmi.max_tmds_char_rate)
+		return MODE_CLOCK_HIGH;
+
 	if (funcs && funcs->tmds_char_rate_valid) {
 		enum drm_mode_status status;
 
@@ -590,6 +596,9 @@ hdmi_compute_clock(const struct drm_connector *connector,
 
 	conn_state->hdmi.tmds_char_rate = clock;
 
+	/* TODO: also check drm_display_info.hdmi.scdc.scrambling.low_rates */
+	conn_state->hdmi.scrambler_needed = (clock > HDMI_1_3_TMDS_CHAR_RATE_MAX_HZ);
+
 	return 0;
 }
 
@@ -965,6 +974,160 @@ drm_hdmi_connector_mode_valid(struct drm_connector *connector,
 }
 EXPORT_SYMBOL(drm_hdmi_connector_mode_valid);
 
+static ssize_t
+audio_infoframe_read(struct file *filp, char __user *ubuf, size_t count, loff_t *ppos)
+{
+	struct drm_connector_hdmi_infoframe *infoframe;
+	struct drm_connector *connector;
+	union hdmi_infoframe *frame;
+	u8 buf[HDMI_INFOFRAME_SIZE(AUDIO)];
+	ssize_t len = 0;
+
+	connector = filp->private_data;
+	mutex_lock(&connector->hdmi.infoframes.lock);
+
+	infoframe = &connector->hdmi.infoframes.audio;
+	if (!infoframe->set)
+		goto out;
+
+	frame = &infoframe->data;
+	len = hdmi_infoframe_pack(frame, buf, sizeof(buf));
+	if (len < 0)
+		goto out;
+
+	len = simple_read_from_buffer(ubuf, count, ppos, buf, len);
+
+out:
+	mutex_unlock(&connector->hdmi.infoframes.lock);
+	return len;
+}
+
+static const struct file_operations audio_infoframe_fops = {
+	.owner   = THIS_MODULE,
+	.open    = simple_open,
+	.read    = audio_infoframe_read,
+};
+
+static int create_hdmi_audio_infoframe_file(struct drm_connector *connector,
+					    struct dentry *parent)
+{
+	struct dentry *file;
+
+	if (!connector->hdmi.funcs ||
+	    !connector->hdmi.funcs->audio.write_infoframe)
+		return 0;
+
+	file = debugfs_create_file("audio", 0400, parent, connector, &audio_infoframe_fops);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	return 0;
+}
+
+#define DEFINE_INFOFRAME_FILE(_f) \
+static ssize_t _f##_read_infoframe(struct file *filp, \
+				   char __user *ubuf, \
+				   size_t count,      \
+				   loff_t *ppos)      \
+{ \
+	struct drm_connector_hdmi_infoframe *infoframe; \
+	struct drm_connector_state *conn_state; \
+	struct drm_connector *connector; \
+	union hdmi_infoframe *frame; \
+	struct drm_device *dev; \
+	u8 buf[HDMI_INFOFRAME_SIZE(MAX)]; \
+	ssize_t len = 0; \
+	\
+	connector = filp->private_data; \
+	dev = connector->dev; \
+	\
+	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); \
+	\
+	conn_state = connector->state; \
+	infoframe = &conn_state->hdmi.infoframes._f; \
+	if (!infoframe->set) \
+		goto out; \
+	\
+	frame = &infoframe->data; \
+	len = hdmi_infoframe_pack(frame, buf, sizeof(buf)); \
+	if (len < 0) \
+		goto out; \
+	\
+	len = simple_read_from_buffer(ubuf, count, ppos, buf, len); \
+	\
+out: \
+	drm_modeset_unlock(&dev->mode_config.connection_mutex); \
+	return len; \
+} \
+\
+static const struct file_operations _f##_infoframe_fops = { \
+	.owner = THIS_MODULE, \
+	.open = simple_open, \
+	.read = _f##_read_infoframe, \
+}; \
+\
+static int create_hdmi_## _f ## _infoframe_file(struct drm_connector *connector, \
+						struct dentry *parent) \
+{ \
+	struct dentry *file; \
+	\
+	if (!connector->hdmi.funcs || \
+	    !connector->hdmi.funcs->_f.write_infoframe) \
+		return 0; \
+	file = debugfs_create_file(#_f, 0400, parent, connector, &_f ## _infoframe_fops); \
+	if (IS_ERR(file)) \
+		return PTR_ERR(file); \
+	\
+	return 0; \
+}
+
+DEFINE_INFOFRAME_FILE(avi);
+DEFINE_INFOFRAME_FILE(hdmi);
+DEFINE_INFOFRAME_FILE(hdr_drm);
+DEFINE_INFOFRAME_FILE(spd);
+
+static int create_hdmi_infoframe_files(struct drm_connector *connector,
+				       struct dentry *parent)
+{
+	int ret;
+
+	ret = create_hdmi_audio_infoframe_file(connector, parent);
+	if (ret)
+		return ret;
+
+	ret = create_hdmi_avi_infoframe_file(connector, parent);
+	if (ret)
+		return ret;
+
+	ret = create_hdmi_hdmi_infoframe_file(connector, parent);
+	if (ret)
+		return ret;
+
+	ret = create_hdmi_hdr_drm_infoframe_file(connector, parent);
+	if (ret)
+		return ret;
+
+	ret = create_hdmi_spd_infoframe_file(connector, parent);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void drm_hdmi_connector_debugfs_init(struct drm_connector *connector,
+				     struct dentry *root)
+{
+	struct dentry *dir;
+
+	dir = debugfs_create_dir("infoframes", root);
+	if (IS_ERR(dir))
+		return;
+
+	create_hdmi_infoframe_files(connector, dir);
+	drm_scdc_debugfs_init(connector, root);
+}
+EXPORT_SYMBOL(drm_hdmi_connector_debugfs_init);
+
 static int clear_infoframe(struct drm_connector *connector,
 			   const struct drm_connector_infoframe_funcs *funcs,
 			   const char *type)
@@ -1196,18 +1359,36 @@ drm_atomic_helper_connector_hdmi_clear_audio_infoframe(struct drm_connector *con
 }
 EXPORT_SYMBOL(drm_atomic_helper_connector_hdmi_clear_audio_infoframe);
 
-static void
+static int
 drm_atomic_helper_connector_hdmi_update(struct drm_connector *connector,
+					struct drm_modeset_acquire_ctx *ctx,
 					enum drm_connector_status status)
 {
 	const struct drm_edid *drm_edid;
+	int ret;
 
 	if (status == connector_status_disconnected) {
-		// TODO: also handle scramber, HDMI sink disconnected.
+		/*
+		 * .detect_ctx() can only ever return a drm_connector_status or
+		 * -EDEADLK. Handle deadlocks, and report any !EDEADLK error.
+		 */
+		ret = drm_connector_hdmi_sync_scdc(connector, false, ctx);
+		if (ret) {
+			if (ret == -EDEADLK)
+				return ret;
+
+			drm_warn(connector->dev, "Ignored SCDC sync error: %d\n", ret);
+		}
+
 		drm_connector_hdmi_audio_plugged_notify(connector, false);
-		drm_edid_connector_update(connector, NULL);
+
+		ret = drm_edid_connector_update(connector, NULL);
+		if (ret)
+			drm_warn(connector->dev, "Ignored EDID update error: %d\n", ret);
+
 		drm_connector_cec_phys_addr_invalidate(connector);
-		return;
+
+		return 0;
 	}
 
 	if (connector->hdmi.funcs->read_edid)
@@ -1220,24 +1401,43 @@ drm_atomic_helper_connector_hdmi_update(struct drm_connector *connector,
 	drm_edid_free(drm_edid);
 
 	if (status == connector_status_connected) {
-		// TODO: also handle scramber, HDMI sink is now connected.
+		/*
+		 * .detect_ctx() can only ever return a drm_connector_status or
+		 * -EDEADLK. Handle deadlocks, and report any !EDEADLK error.
+		 */
+		ret = drm_connector_hdmi_sync_scdc(connector, true, ctx);
+		if (ret) {
+			if (ret == -EDEADLK)
+				return ret;
+
+			drm_warn(connector->dev, "Ignored SCDC sync error: %d\n", ret);
+		}
+
 		drm_connector_hdmi_audio_plugged_notify(connector, true);
 		drm_connector_cec_phys_addr_set(connector);
 	}
+
+	return 0;
 }
 
 /**
  * drm_atomic_helper_connector_hdmi_hotplug - Handle the hotplug event for the HDMI connector
  * @connector: A pointer to the HDMI connector
+ * @ctx: Lock acquisition context to be used for resetting CRTC
  * @status: Connection status
  *
  * This function should be called as a part of the .detect() / .detect_ctx()
  * callbacks for all status changes.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ * If @ctx is set, it might also return -EDEADLK.
  */
-void drm_atomic_helper_connector_hdmi_hotplug(struct drm_connector *connector,
-					      enum drm_connector_status status)
+int drm_atomic_helper_connector_hdmi_hotplug(struct drm_connector *connector,
+					     struct drm_modeset_acquire_ctx *ctx,
+					     enum drm_connector_status status)
 {
-	drm_atomic_helper_connector_hdmi_update(connector, status);
+	return drm_atomic_helper_connector_hdmi_update(connector, ctx, status);
 }
 EXPORT_SYMBOL(drm_atomic_helper_connector_hdmi_hotplug);
 
@@ -1252,6 +1452,27 @@ EXPORT_SYMBOL(drm_atomic_helper_connector_hdmi_hotplug);
  */
 void drm_atomic_helper_connector_hdmi_force(struct drm_connector *connector)
 {
-	drm_atomic_helper_connector_hdmi_update(connector, connector->status);
+	drm_atomic_helper_connector_hdmi_update(connector, NULL, connector->status);
 }
 EXPORT_SYMBOL(drm_atomic_helper_connector_hdmi_force);
+
+/**
+ * drm_atomic_helper_connector_hdmi_force_ctx - HDMI Connector implementation
+ * of the force_ctx callback
+ * @connector: A pointer to the HDMI connector
+ * @ctx: Lock acquisition context to be used for resetting CRTC
+ *
+ * This function implements the .force_ctx() callback for the HDMI connectors.
+ * It can either be used directly as the callback or should be called from
+ * within the .force_ctx() callback implementation to maintain the HDMI-specific
+ * connector's data.
+ *
+ * Returns:
+ * Zero on success, error code on failure, including -EDEADLK.
+ */
+int drm_atomic_helper_connector_hdmi_force_ctx(struct drm_connector *connector,
+					       struct drm_modeset_acquire_ctx *ctx)
+{
+	return drm_atomic_helper_connector_hdmi_update(connector, ctx, connector->status);
+}
+EXPORT_SYMBOL(drm_atomic_helper_connector_hdmi_force_ctx);
diff --git a/drivers/gpu/drm/display/drm_scdc_helper.c b/drivers/gpu/drm/display/drm_scdc_helper.c
index df878aad4a36b2..74388481fc556b 100644
--- a/drivers/gpu/drm/display/drm_scdc_helper.c
+++ b/drivers/gpu/drm/display/drm_scdc_helper.c
@@ -21,10 +21,14 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include <linux/bitfield.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
 #include <linux/export.h>
 #include <linux/i2c.h>
+#include <linux/minmax.h>
+#include <linux/overflow.h>
 #include <linux/slab.h>
-#include <linux/delay.h>
 
 #include <drm/display/drm_scdc_helper.h>
 #include <drm/drm_connector.h>
@@ -55,6 +59,42 @@
 
 #define SCDC_I2C_SLAVE_ADDRESS 0x54
 
+#define drm_scdc_dbg(connector, fmt, ...)					\
+	drm_dbg_kms((connector)->dev, "[CONNECTOR:%d:%s] " fmt,			\
+		    (connector)->base.id, (connector)->name, ##__VA_ARGS__)
+
+static const char *drm_scdc_frl_rate_str(enum drm_scdc_frl_rate rate)
+{
+	switch (rate) {
+	case SCDC_FRL_RATE_OFF:
+		return "Off";
+	case SCDC_FRL_RATE_3X3:
+		return "3 Gbit/s x 3 lanes";
+	case SCDC_FRL_RATE_6X3:
+		return "6 Gbit/s x 3 lanes";
+	case SCDC_FRL_RATE_6X4:
+		return "6 Gbit/s x 4 lanes";
+	case SCDC_FRL_RATE_8X4:
+		return "8 Gbit/s x 4 lanes";
+	case SCDC_FRL_RATE_10X4:
+		return "10 Gbit/s x 4 lanes";
+	case SCDC_FRL_RATE_12X4:
+		return "12 Gbit/s x 4 lanes";
+	case SCDC_FRL_RATE_RESV_7:
+	case SCDC_FRL_RATE_RESV_8:
+	case SCDC_FRL_RATE_RESV_9:
+	case SCDC_FRL_RATE_RESV_10:
+	case SCDC_FRL_RATE_RESV_11:
+	case SCDC_FRL_RATE_RESV_12:
+	case SCDC_FRL_RATE_RESV_13:
+	case SCDC_FRL_RATE_RESV_14:
+	case SCDC_FRL_RATE_RESV_15:
+		return "(Reserved)";
+	default:
+		return NULL;
+	}
+}
+
 /**
  * drm_scdc_read - read a block of data from SCDC
  * @adapter: I2C controller
@@ -67,8 +107,8 @@
  * Returns:
  * 0 on success, negative error code on failure.
  */
-ssize_t drm_scdc_read(struct i2c_adapter *adapter, u8 offset, void *buffer,
-		      size_t size)
+int drm_scdc_read(struct i2c_adapter *adapter, u8 offset, void *buffer,
+		  size_t size)
 {
 	int ret;
 	struct i2c_msg msgs[2] = {
@@ -107,8 +147,8 @@ EXPORT_SYMBOL(drm_scdc_read);
  * Returns:
  * 0 on success, negative error code on failure.
  */
-ssize_t drm_scdc_write(struct i2c_adapter *adapter, u8 offset,
-		       const void *buffer, size_t size)
+int drm_scdc_write(struct i2c_adapter *adapter, u8 offset, const void *buffer,
+		   size_t size)
 {
 	struct i2c_msg msg = {
 		.addr = SCDC_I2C_SLAVE_ADDRESS,
@@ -158,9 +198,7 @@ bool drm_scdc_get_scrambling_status(struct drm_connector *connector)
 
 	ret = drm_scdc_readb(connector->ddc, SCDC_SCRAMBLER_STATUS, &status);
 	if (ret < 0) {
-		drm_dbg_kms(connector->dev,
-			    "[CONNECTOR:%d:%s] Failed to read scrambling status: %d\n",
-			    connector->base.id, connector->name, ret);
+		drm_scdc_dbg(connector, "Failed to read scrambling status: %d\n", ret);
 		return false;
 	}
 
@@ -188,9 +226,7 @@ bool drm_scdc_set_scrambling(struct drm_connector *connector,
 
 	ret = drm_scdc_readb(connector->ddc, SCDC_TMDS_CONFIG, &config);
 	if (ret < 0) {
-		drm_dbg_kms(connector->dev,
-			    "[CONNECTOR:%d:%s] Failed to read TMDS config: %d\n",
-			    connector->base.id, connector->name, ret);
+		drm_scdc_dbg(connector, "Failed to read TMDS config: %d\n", ret);
 		return false;
 	}
 
@@ -201,9 +237,7 @@ bool drm_scdc_set_scrambling(struct drm_connector *connector,
 
 	ret = drm_scdc_writeb(connector->ddc, SCDC_TMDS_CONFIG, config);
 	if (ret < 0) {
-		drm_dbg_kms(connector->dev,
-			    "[CONNECTOR:%d:%s] Failed to enable scrambling: %d\n",
-			    connector->base.id, connector->name, ret);
+		drm_scdc_dbg(connector, "Failed to enable scrambling: %d\n", ret);
 		return false;
 	}
 
@@ -248,9 +282,7 @@ bool drm_scdc_set_high_tmds_clock_ratio(struct drm_connector *connector,
 
 	ret = drm_scdc_readb(connector->ddc, SCDC_TMDS_CONFIG, &config);
 	if (ret < 0) {
-		drm_dbg_kms(connector->dev,
-			    "[CONNECTOR:%d:%s] Failed to read TMDS config: %d\n",
-			    connector->base.id, connector->name, ret);
+		drm_scdc_dbg(connector, "Failed to read TMDS config: %d\n", ret);
 		return false;
 	}
 
@@ -261,9 +293,7 @@ bool drm_scdc_set_high_tmds_clock_ratio(struct drm_connector *connector,
 
 	ret = drm_scdc_writeb(connector->ddc, SCDC_TMDS_CONFIG, config);
 	if (ret < 0) {
-		drm_dbg_kms(connector->dev,
-			    "[CONNECTOR:%d:%s] Failed to set TMDS clock ratio: %d\n",
-			    connector->base.id, connector->name, ret);
+		drm_scdc_dbg(connector, "Failed to set TMDS clock ratio: %d\n", ret);
 		return false;
 	}
 
@@ -276,3 +306,292 @@ bool drm_scdc_set_high_tmds_clock_ratio(struct drm_connector *connector,
 	return true;
 }
 EXPORT_SYMBOL(drm_scdc_set_high_tmds_clock_ratio);
+
+/**
+ * drm_scdc_set_source_version - set SCDC source version on the sink
+ * @connector: connector
+ * @ver: source version to advertise (per spec, 1)
+ *
+ * Reads the sink version for diagnostics and as a guard, then writes
+ * the source version unless the sink reports version 0.
+ *
+ * Returns:
+ * 0 on success or when skipped; a negative error code when either
+ * the read or the write failed.
+ */
+int drm_scdc_set_source_version(struct drm_connector *connector, u8 ver)
+{
+	u8 sink_ver;
+	int ret;
+
+	ret = drm_scdc_readb(connector->ddc, SCDC_SINK_VERSION, &sink_ver);
+	if (ret) {
+		drm_scdc_dbg(connector, "Failed to read SCDC_SINK_VERSION: %d\n", ret);
+		return ret;
+	}
+
+	drm_scdc_dbg(connector, "Sink reported SCDC ver. %u\n", sink_ver);
+
+	/*
+	 * Only advertise our source version to sinks that report a
+	 * non-zero sink version.  A sink reporting version 0 is either
+	 * not SCDC-version-aware or non-conformant; writing the source
+	 * version gains nothing and may upset broken hardware.
+	 */
+	if (sink_ver) {
+		ret = drm_scdc_writeb(connector->ddc, SCDC_SOURCE_VERSION,
+				      min_t(u8, sink_ver, ver));
+		if (ret)
+			drm_scdc_dbg(connector,
+				     "Failed to write SCDC_SOURCE_VERSION: %d\n", ret);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_scdc_set_source_version);
+
+static void
+drm_scdc_parse_status0_flags(u8 val, struct drm_scdc_status_flags *flags)
+{
+	flags->clock_detected = val & SCDC_CLOCK_DETECT;
+	flags->ch0_locked = val & SCDC_CH0_LOCK;
+	flags->ch1_locked = val & SCDC_CH1_LOCK;
+	flags->ch2_locked = val & SCDC_CH2_LOCK;
+	flags->ln3_locked = val & SCDC_LN3_LOCK;
+	flags->flt_ready = val & SCDC_FLT_READY;
+	flags->dsc_fail = val & SCDC_DSC_FAIL;
+}
+
+static void
+drm_scdc_parse_status1_2_flags(u8 val_flag1, u8 val_flag2,
+			       struct drm_scdc_status_flags *flags)
+{
+	flags->ln0_training_pattern = FIELD_GET(SCDC_LN_EVEN_TRAIN_PTRN, val_flag1);
+	flags->ln1_training_pattern = FIELD_GET(SCDC_LN_ODD_TRAIN_PTRN, val_flag1);
+
+	flags->ln2_training_pattern = FIELD_GET(SCDC_LN_EVEN_TRAIN_PTRN, val_flag2);
+	flags->ln3_training_pattern = FIELD_GET(SCDC_LN_ODD_TRAIN_PTRN, val_flag2);
+}
+
+static int drm_scdc_parse_error_counters(const u8 scdc[256], u16 counter[4],
+					 unsigned int num_lanes)
+{
+	u8 end_reg;
+	u8 sum = 0;
+	int i;
+
+	switch (num_lanes) {
+	case 3:
+		end_reg = SCDC_ERR_DET_CHECKSUM;
+		break;
+	case 4:
+		end_reg = SCDC_ERR_DET_3_H;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	for (i = SCDC_ERR_DET_0_L; i <= end_reg; i++)
+		sum = wrapping_add(u8, sum, scdc[i]);
+
+	if (sum)
+		return -EPROTO;
+
+	for (i = 0; i < 3; i++) {
+		if (scdc[SCDC_ERR_DET_0_H + i * 2] & SCDC_CHANNEL_VALID)
+			counter[i] =  (scdc[SCDC_ERR_DET_0_H + i * 2] &
+				       ~SCDC_CHANNEL_VALID) << 8 |
+				       scdc[SCDC_ERR_DET_0_L + i * 2];
+		else
+			counter[i] = 0;
+	}
+
+	if (num_lanes == 4 && scdc[SCDC_ERR_DET_3_H] & SCDC_CHANNEL_VALID)
+		counter[3] =  (scdc[SCDC_ERR_DET_3_H] & ~SCDC_CHANNEL_VALID) << 8 |
+			      scdc[SCDC_ERR_DET_3_L];
+	else
+		counter[3] = 0;
+
+	return 0;
+}
+
+/**
+ * drm_scdc_read_state - Update state from SCDC
+ * @connector: pointer to a &struct drm_connector on which to operate on
+ * @state: pointer to a &struct drm_scdc_state to fill
+ *
+ * Reads the entire 256 byte SCDC state and parses it.
+ *
+ * Returns: %0 on success, negative errno on failure.
+ */
+int drm_scdc_read_state(struct drm_connector *connector, struct drm_scdc_state *state)
+{
+	struct i2c_adapter *ddc;
+	struct drm_scdc *scdc;
+	u8 *buf = state->scdc;
+	int num_lanes;
+	int ret;
+
+	if (!state || !connector)
+		return -ENODEV;
+
+	scdc = &connector->display_info.hdmi.scdc;
+	ddc = connector->ddc;
+
+	if (!scdc->supported || !ddc)
+		return -EOPNOTSUPP;
+
+	/* Read in 128-byte chunks, to work around DP<->HDMI converters with issues. */
+	ret = drm_scdc_read(ddc, 0, buf, 128);
+	if (ret)
+		return ret;
+
+	ret = drm_scdc_read(ddc, 128, &buf[128], 128);
+	if (ret)
+		return ret;
+
+	state->scrambling_enabled = buf[SCDC_TMDS_CONFIG] & SCDC_SCRAMBLING_ENABLE;
+	state->tmds_bclk_x40 = buf[SCDC_TMDS_CONFIG] & SCDC_TMDS_BIT_CLOCK_RATIO_BY_40;
+
+	state->scrambling_detected = buf[SCDC_SCRAMBLER_STATUS] & SCDC_SCRAMBLING_STATUS;
+
+	state->rate = FIELD_GET(SCDC_FRL_RATE, buf[SCDC_CONFIG_1]);
+	num_lanes = drm_scdc_num_frl_lanes(state->rate);
+	if (num_lanes < 0)
+		return num_lanes;
+	if (!num_lanes)
+		num_lanes = 3;
+
+	state->ffe_levels = FIELD_GET(SCDC_FFE_LEVELS, buf[SCDC_CONFIG_1]);
+
+	drm_scdc_parse_status0_flags(buf[SCDC_STATUS_FLAGS_0], &state->stf);
+	drm_scdc_parse_status1_2_flags(buf[SCDC_STATUS_FLAGS_1],
+				       buf[SCDC_STATUS_FLAGS_2], &state->stf);
+	ret = drm_scdc_parse_error_counters(buf, state->error_count, num_lanes);
+	if (ret)
+		return ret;
+
+	if (state->rate && (buf[SCDC_ERR_DET_RS_H] & SCDC_CHANNEL_VALID))
+		state->rs_corrections = (buf[SCDC_ERR_DET_RS_H] & ~SCDC_CHANNEL_VALID) << 8 |
+					buf[SCDC_ERR_DET_RS_L];
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_scdc_read_state);
+
+#define scdc_print_str(_f, key, s) \
+	(seq_printf((_f), "%-30s: %s\n", (key), (s)))
+#define scdc_print_flag(_f, key, val) \
+	(scdc_print_str((_f), (key), str_yes_no((val))))
+#define scdc_print_dec(_f, key, val) \
+	(seq_printf((_f), "%-30s: %d\n", (key), (val)))
+
+static int scdc_status_show(struct seq_file *m, void *data)
+{
+	struct drm_connector *connector = m->private;
+	struct drm_scdc *scdc = &connector->display_info.hdmi.scdc;
+	struct drm_scdc_state *st;
+	int i, ret, frl_lanes;
+
+	drm_connector_get(connector);
+
+	ret = mutex_lock_interruptible(&connector->dev->mode_config.mutex);
+	if (ret)
+		goto err_conn_put;
+
+	if (connector->status != connector_status_connected) {
+		ret = -ENODEV;
+		goto err_unlock;
+	}
+
+	if (scdc->supported) {
+		st = kzalloc_obj(*st);
+		if (!st) {
+			ret = -ENOMEM;
+			goto err_unlock;
+		}
+		ret = drm_scdc_read_state(connector, st);
+		if (ret)
+			goto err_free_state;
+
+		for (i = 0; i < ARRAY_SIZE(st->scdc); i += 16)
+			seq_printf(m, "%*ph\n", 16, &st->scdc[i]);
+
+		seq_puts(m, "\n----------------\n\n");
+	}
+
+	scdc_print_flag(m, "SCDC Supported", scdc->supported);
+	if (!scdc->supported) {
+		ret = 0;
+		goto err_unlock;
+	}
+
+	scdc_print_flag(m, "Sink Read Request Capable", scdc->read_request);
+	scdc_print_flag(m, "Scrambling Supported", scdc->scrambling.supported);
+	scdc_print_flag(m, "Low Rate Scrambling Supported", scdc->scrambling.low_rates);
+
+	mutex_unlock(&connector->dev->mode_config.mutex);
+
+	drm_connector_put(connector);
+
+	frl_lanes = drm_scdc_num_frl_lanes(st->rate);
+
+	scdc_print_flag(m, "Scrambling Enabled", st->scrambling_enabled);
+	scdc_print_flag(m, "Scrambling Detected", st->scrambling_detected);
+	scdc_print_str(m, "FRL Rate", drm_scdc_frl_rate_str(st->rate));
+	scdc_print_dec(m, "FFE Levels", st->ffe_levels);
+
+	if (st->tmds_bclk_x40)
+		scdc_print_str(m, "TMDS Bit Clock Ratio", "1/40");
+	else
+		scdc_print_str(m, "TMDS Bit Clock Ratio", "1/10");
+
+	scdc_print_flag(m, "Clock Detected", st->stf.clock_detected);
+	scdc_print_flag(m, "Channel 0 Locked", st->stf.ch0_locked);
+	scdc_print_flag(m, "Channel 1 Locked", st->stf.ch1_locked);
+	scdc_print_flag(m, "Channel 2 Locked", st->stf.ch2_locked);
+	if (frl_lanes == 4)
+		scdc_print_flag(m, "Lane 3 Locked", st->stf.ln3_locked);
+
+	scdc_print_flag(m, "Sink Ready For Link Training", st->stf.flt_ready);
+	scdc_print_flag(m, "Sink Failed To Decode DSC", st->stf.dsc_fail);
+
+	scdc_print_dec(m, "Channel 0 Errors", st->error_count[0]);
+	scdc_print_dec(m, "Channel 1 Errors", st->error_count[1]);
+	scdc_print_dec(m, "Channel 2 Errors", st->error_count[2]);
+	if (frl_lanes == 4)
+		scdc_print_dec(m, "Lane 3 Errors", st->error_count[3]);
+	if (frl_lanes > 0)
+		scdc_print_dec(m, "Reed-Solomon Corrections", st->rs_corrections);
+
+	kfree(st);
+
+	return 0;
+
+err_free_state:
+	kfree(st);
+err_unlock:
+	mutex_unlock(&connector->dev->mode_config.mutex);
+err_conn_put:
+	drm_connector_put(connector);
+
+	return ret;
+}
+DEFINE_SHOW_ATTRIBUTE(scdc_status);
+
+/**
+ * drm_scdc_debugfs_init - Initialize scdc files in connector debugfs
+ * @connector: pointer to &struct drm_connector to operate on
+ * @root: debugfs &struct dentry for the debugfs root of @connector
+ *
+ * Creates SCDC-related debugfs files for @connector. Must be called after
+ * @root is already created.
+ */
+void drm_scdc_debugfs_init(struct drm_connector *connector, struct dentry *root)
+{
+	if (!root || !connector)
+		return;
+
+	debugfs_create_file("scdc_status", 0444, root, connector, &scdc_status_fops);
+}
+EXPORT_SYMBOL(drm_scdc_debugfs_init);
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index e5c8ef06caeddc..9bb761eda59f4d 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -293,8 +293,10 @@ void drm_atomic_commit_default_clear(struct drm_atomic_commit *state)
 		if (!connector)
 			continue;
 
-		connector->funcs->atomic_destroy_state(connector,
-						       state->connectors[i].state_to_destroy);
+		if (state->connectors[i].state_to_destroy)
+			connector->funcs->atomic_destroy_state(connector,
+							       state->connectors[i].state_to_destroy);
+
 		state->connectors[i].ptr = NULL;
 		state->connectors[i].state_to_destroy = NULL;
 		state->connectors[i].old_state = NULL;
@@ -308,8 +310,9 @@ void drm_atomic_commit_default_clear(struct drm_atomic_commit *state)
 		if (!crtc)
 			continue;
 
-		crtc->funcs->atomic_destroy_state(crtc,
-						  state->crtcs[i].state_to_destroy);
+		if (state->crtcs[i].state_to_destroy)
+			crtc->funcs->atomic_destroy_state(crtc,
+							  state->crtcs[i].state_to_destroy);
 
 		state->crtcs[i].ptr = NULL;
 		state->crtcs[i].state_to_destroy = NULL;
@@ -328,8 +331,10 @@ void drm_atomic_commit_default_clear(struct drm_atomic_commit *state)
 		if (!plane)
 			continue;
 
-		plane->funcs->atomic_destroy_state(plane,
-						   state->planes[i].state_to_destroy);
+		if (state->planes[i].state_to_destroy)
+			plane->funcs->atomic_destroy_state(plane,
+							   state->planes[i].state_to_destroy);
+
 		state->planes[i].ptr = NULL;
 		state->planes[i].state_to_destroy = NULL;
 		state->planes[i].old_state = NULL;
@@ -343,9 +348,9 @@ void drm_atomic_commit_default_clear(struct drm_atomic_commit *state)
 			continue;
 
 		drm_colorop_atomic_destroy_state(colorop,
-						 state->colorops[i].state);
+						 state->colorops[i].state_to_destroy);
 		state->colorops[i].ptr = NULL;
-		state->colorops[i].state = NULL;
+		state->colorops[i].state_to_destroy = NULL;
 		state->colorops[i].old_state = NULL;
 		state->colorops[i].new_state = NULL;
 	}
@@ -353,8 +358,13 @@ void drm_atomic_commit_default_clear(struct drm_atomic_commit *state)
 	for (i = 0; i < state->num_private_objs; i++) {
 		struct drm_private_obj *obj = state->private_objs[i].ptr;
 
-		obj->funcs->atomic_destroy_state(obj,
-						 state->private_objs[i].state_to_destroy);
+		if (!obj)
+			continue;
+
+		if (state->private_objs[i].state_to_destroy)
+			obj->funcs->atomic_destroy_state(obj,
+							 state->private_objs[i].state_to_destroy);
+
 		state->private_objs[i].ptr = NULL;
 		state->private_objs[i].state_to_destroy = NULL;
 		state->private_objs[i].old_state = NULL;
@@ -708,7 +718,7 @@ drm_atomic_get_colorop_state(struct drm_atomic_commit *state,
 	if (!colorop_state)
 		return ERR_PTR(-ENOMEM);
 
-	state->colorops[index].state = colorop_state;
+	state->colorops[index].state_to_destroy = colorop_state;
 	state->colorops[index].ptr = colorop;
 	state->colorops[index].old_state = colorop->state;
 	state->colorops[index].new_state = colorop_state;
@@ -925,6 +935,10 @@ static void drm_atomic_colorop_print_state(struct drm_printer *p,
 				  drm_get_colorop_lut3d_interpolation_name(state->lut3d_interpolation));
 		drm_printf_indent(p, 1, "data blob id=%d\n", state->data ? state->data->base.id : 0);
 		break;
+	case DRM_COLOROP_FIXED_MATRIX:
+		drm_printf_indent(p, 1, "fixed_matrix_type=%s\n",
+				  drm_get_colorop_fixed_matrix_type_name(state->fixed_matrix_type));
+		break;
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 285aac3554dfd6..9d006f98413a58 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -41,7 +41,6 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 #include <drm/drm_self_refresh_helper.h>
 #include <drm/drm_vblank.h>
@@ -49,6 +48,7 @@
 
 #include "drm_crtc_helper_internal.h"
 #include "drm_crtc_internal.h"
+#include "drm_panic_internal.h"
 
 /**
  * DOC: overview
@@ -2037,7 +2037,7 @@ void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_commit *state)
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_tail_rpm);
 
-static void commit_tail(struct drm_atomic_commit *state)
+static void commit_tail(struct drm_atomic_commit *state, bool nonblock)
 {
 	struct drm_device *dev = state->dev;
 	const struct drm_mode_config_helper_funcs *funcs;
@@ -2095,7 +2095,7 @@ static void commit_work(struct work_struct *work)
 	struct drm_atomic_commit *state = container_of(work,
 						      struct drm_atomic_commit,
 						      commit_work);
-	commit_tail(state);
+	commit_tail(state, true);
 }
 
 /**
@@ -2315,7 +2315,7 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 	if (nonblock)
 		queue_work(system_dfl_wq, &state->commit_work);
 	else
-		commit_tail(state);
+		commit_tail(state, false);
 
 	return 0;
 
@@ -3351,7 +3351,7 @@ int drm_atomic_helper_swap_state(struct drm_atomic_commit *state,
 		old_colorop_state->state = state;
 		new_colorop_state->state = NULL;
 
-		state->colorops[i].state = old_colorop_state;
+		state->colorops[i].state_to_destroy = old_colorop_state;
 		colorop->state = new_colorop_state;
 	}
 
diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c
index a2ef272e9f27ce..406157a3660fd1 100644
--- a/drivers/gpu/drm/drm_atomic_state_helper.c
+++ b/drivers/gpu/drm/drm_atomic_state_helper.c
@@ -79,51 +79,6 @@ __drm_atomic_helper_crtc_state_init(struct drm_crtc_state *crtc_state,
 }
 EXPORT_SYMBOL(__drm_atomic_helper_crtc_state_init);
 
-/**
- * __drm_atomic_helper_crtc_reset - reset state on CRTC
- * @crtc: drm CRTC
- * @crtc_state: CRTC state to assign
- *
- * Initializes the newly allocated @crtc_state and assigns it to
- * the &drm_crtc->state pointer of @crtc, usually required when
- * initializing the drivers or when called from the &drm_crtc_funcs.reset
- * hook.
- *
- * This is useful for drivers that subclass the CRTC state.
- */
-void
-__drm_atomic_helper_crtc_reset(struct drm_crtc *crtc,
-			       struct drm_crtc_state *crtc_state)
-{
-	if (crtc_state)
-		__drm_atomic_helper_crtc_state_init(crtc_state, crtc);
-
-	if (drm_dev_has_vblank(crtc->dev))
-		drm_crtc_vblank_reset(crtc);
-
-	crtc->state = crtc_state;
-}
-EXPORT_SYMBOL(__drm_atomic_helper_crtc_reset);
-
-/**
- * drm_atomic_helper_crtc_reset - default &drm_crtc_funcs.reset hook for CRTCs
- * @crtc: drm CRTC
- *
- * Resets the atomic state for @crtc by freeing the state pointer (which might
- * be NULL, e.g. at driver load time) and allocating a new empty state object.
- */
-void drm_atomic_helper_crtc_reset(struct drm_crtc *crtc)
-{
-	struct drm_crtc_state *crtc_state =
-		kzalloc_obj(*crtc->state);
-
-	if (crtc->state)
-		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
-
-	__drm_atomic_helper_crtc_reset(crtc, crtc_state);
-}
-EXPORT_SYMBOL(drm_atomic_helper_crtc_reset);
-
 /**
  * drm_atomic_helper_crtc_create_state - default &drm_crtc_funcs.atomic_create_state hook for crtcs
  * @crtc: crtc object
@@ -331,47 +286,6 @@ void __drm_atomic_helper_plane_state_init(struct drm_plane_state *plane_state,
 }
 EXPORT_SYMBOL(__drm_atomic_helper_plane_state_init);
 
-/**
- * __drm_atomic_helper_plane_reset - reset state on plane
- * @plane: drm plane
- * @plane_state: plane state to assign
- *
- * Initializes the newly allocated @plane_state and assigns it to
- * the &drm_plane->state pointer of @plane, usually required when
- * initializing the drivers or when called from the &drm_plane_funcs.reset
- * hook.
- *
- * This is useful for drivers that subclass the plane state.
- */
-void __drm_atomic_helper_plane_reset(struct drm_plane *plane,
-				     struct drm_plane_state *plane_state)
-{
-	if (plane_state)
-		__drm_atomic_helper_plane_state_init(plane_state, plane);
-
-	plane->state = plane_state;
-}
-EXPORT_SYMBOL(__drm_atomic_helper_plane_reset);
-
-/**
- * drm_atomic_helper_plane_reset - default &drm_plane_funcs.reset hook for planes
- * @plane: drm plane
- *
- * Resets the atomic state for @plane by freeing the state pointer (which might
- * be NULL, e.g. at driver load time) and allocating a new empty state object.
- */
-void drm_atomic_helper_plane_reset(struct drm_plane *plane)
-{
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-
-	kfree(plane->state);
-	plane->state = kzalloc_obj(*plane->state);
-	if (plane->state)
-		__drm_atomic_helper_plane_reset(plane, plane->state);
-}
-EXPORT_SYMBOL(drm_atomic_helper_plane_reset);
-
 /**
  * drm_atomic_helper_plane_create_state - default &drm_plane_funcs.atomic_create_state hook for planes
  * @plane: plane object
diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c
index 68d59e17ffc2d5..1eebcf6f5e05bd 100644
--- a/drivers/gpu/drm/drm_atomic_uapi.c
+++ b/drivers/gpu/drm/drm_atomic_uapi.c
@@ -786,6 +786,11 @@ static int drm_atomic_colorop_set_property(struct drm_colorop *colorop,
 		return drm_atomic_color_set_data_property(colorop, state,
 							  property, val,
 							  replaced);
+	} else if (property == colorop->fixed_matrix_type_property) {
+		if (state->fixed_matrix_type != val) {
+			state->fixed_matrix_type = val;
+			*replaced = true;
+		}
 	} else {
 		drm_dbg_atomic(colorop->dev,
 			       "[COLOROP:%d:%d] unknown property [PROP:%d:%s]\n",
@@ -818,6 +823,8 @@ drm_atomic_colorop_get_property(struct drm_colorop *colorop,
 		*val = state->lut3d_interpolation;
 	else if (property == colorop->data_property)
 		*val = (state->data) ? state->data->base.id : 0;
+	else if (property == colorop->fixed_matrix_type_property)
+		*val = state->fixed_matrix_type;
 	else
 		return -EINVAL;
 
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 83f1809a5d376d..e7f9e6d5892720 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -460,9 +460,13 @@ void drm_bridge_add(struct drm_bridge *bridge)
 	mutex_init(&bridge->hpd_state_mutex);
 	mutex_init(&bridge->hpd_mutex);
 
-	if (bridge->ops & DRM_BRIDGE_OP_HDMI)
+	if (bridge->ops & DRM_BRIDGE_OP_HDMI) {
+		if (bridge->supported_hdmi_ver == HDMI_VERSION_UNKNOWN)
+			DRM_WARN("HDMI bridge misses supported HDMI version\n");
+
 		bridge->ycbcr_420_allowed = !!(bridge->supported_formats &
 					       BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420));
+	}
 
 	mutex_lock(&bridge_lock);
 	list_add_tail(&bridge->list, &bridge_list);
@@ -549,10 +553,31 @@ drm_bridge_atomic_create_priv_state(struct drm_private_obj *obj)
 	return &state->base;
 }
 
+static void
+drm_bridge_atomic_print_priv_state(struct drm_printer *p,
+				   const struct drm_private_state *s)
+{
+	const struct drm_bridge_state *state = drm_priv_to_bridge_state(s);
+	struct drm_bridge *bridge = drm_priv_to_bridge(s->obj);
+
+	if (bridge->of_node)
+		drm_printf(p, "bridge: %ps (%pOFfc)\n", bridge->funcs, bridge->of_node);
+	else
+		drm_printf(p, "bridge: %ps\n", bridge->funcs);
+
+	drm_printf_indent(p, 1, "input bus configuration:");
+	drm_printf_indent(p, 2, "code: %04x", state->input_bus_cfg.format);
+	drm_printf_indent(p, 2, "flags: %08x", state->input_bus_cfg.flags);
+	drm_printf_indent(p, 1, "output bus configuration:");
+	drm_printf_indent(p, 2, "code: %04x", state->output_bus_cfg.format);
+	drm_printf_indent(p, 2, "flags: %08x", state->output_bus_cfg.flags);
+}
+
 static const struct drm_private_state_funcs drm_bridge_priv_state_funcs = {
 	.atomic_create_state = drm_bridge_atomic_create_priv_state,
 	.atomic_duplicate_state = drm_bridge_atomic_duplicate_priv_state,
 	.atomic_destroy_state = drm_bridge_atomic_destroy_priv_state,
+	.atomic_print_state = drm_bridge_atomic_print_priv_state,
 };
 
 /**
@@ -1716,23 +1741,6 @@ struct drm_bridge *of_drm_get_bridge_by_endpoint(const struct device_node *np,
 EXPORT_SYMBOL_GPL(of_drm_get_bridge_by_endpoint);
 #endif
 
-/**
- * devm_drm_put_bridge - Release a bridge reference obtained via devm
- * @dev: device that got the bridge via devm
- * @bridge: pointer to a struct drm_bridge obtained via devm
- *
- * Same as drm_bridge_put() for bridge pointers obtained via devm functions
- * such as devm_drm_bridge_alloc().
- *
- * This function is a temporary workaround and MUST NOT be used. Manual
- * handling of bridge lifetime is inherently unsafe.
- */
-void devm_drm_put_bridge(struct device *dev, struct drm_bridge *bridge)
-{
-	devm_release_action(dev, drm_bridge_put_void, bridge);
-}
-EXPORT_SYMBOL(devm_drm_put_bridge);
-
 static void drm_bridge_debugfs_show_bridge(struct drm_printer *p,
 					   struct drm_bridge *bridge,
 					   unsigned int idx,
diff --git a/drivers/gpu/drm/drm_bridge_helper.c b/drivers/gpu/drm/drm_bridge_helper.c
index 420f29cf3e5435..0a3c8fee66b325 100644
--- a/drivers/gpu/drm/drm_bridge_helper.c
+++ b/drivers/gpu/drm/drm_bridge_helper.c
@@ -50,8 +50,6 @@ int drm_bridge_helper_reset_crtc(struct drm_bridge *bridge,
 
 	crtc = connector->state->crtc;
 	ret = drm_atomic_helper_reset_crtc(crtc, ctx);
-	if (ret)
-		goto out;
 
 out:
 	drm_modeset_unlock(&dev->mode_config.connection_mutex);
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c
index 1536e59c6fe717..02e0460baa7c8b 100644
--- a/drivers/gpu/drm/drm_buddy.c
+++ b/drivers/gpu/drm/drm_buddy.c
@@ -44,7 +44,8 @@ void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p)
 
 	gpu_buddy_driver_lock_held(mm);
 	drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n",
-		   mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
+		   mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20,
+		   gpu_buddy_clear_avail(mm) >> 20);
 
 	for (order = mm->max_order; order >= 0; order--) {
 		u64 free_count = mm->free_scoreboard[order];
diff --git a/drivers/gpu/drm/drm_client_event.c b/drivers/gpu/drm/drm_client_event.c
index 7b3e362f7926cf..f0af584da23c66 100644
--- a/drivers/gpu/drm/drm_client_event.c
+++ b/drivers/gpu/drm/drm_client_event.c
@@ -123,6 +123,24 @@ void drm_client_dev_restore(struct drm_device *dev, bool force)
 	mutex_unlock(&dev->clientlist_mutex);
 }
 
+void drm_client_dev_acquire_outputs(struct drm_device *dev)
+{
+	struct drm_client_dev *client;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return;
+
+	mutex_lock(&dev->clientlist_mutex);
+	list_for_each_entry(client, &dev->clientlist, list) {
+		if (!client->funcs || !client->funcs->acquire_outputs)
+			continue;
+
+		client->funcs->acquire_outputs(client);
+	}
+	mutex_unlock(&dev->clientlist_mutex);
+}
+EXPORT_SYMBOL(drm_client_dev_acquire_outputs);
+
 static int drm_client_suspend(struct drm_client_dev *client)
 {
 	struct drm_device *dev = client->dev;
diff --git a/drivers/gpu/drm/drm_colorop.c b/drivers/gpu/drm/drm_colorop.c
index 682fcc65152594..95da03d6c65c83 100644
--- a/drivers/gpu/drm/drm_colorop.c
+++ b/drivers/gpu/drm/drm_colorop.c
@@ -68,6 +68,7 @@ static const struct drm_prop_enum_list drm_colorop_type_enum_list[] = {
 	{ DRM_COLOROP_CTM_3X4, "3x4 Matrix"},
 	{ DRM_COLOROP_MULTIPLIER, "Multiplier"},
 	{ DRM_COLOROP_3D_LUT, "3D LUT"},
+	{ DRM_COLOROP_FIXED_MATRIX, "Fixed Matrix"},
 };
 
 static const char * const colorop_curve_1d_type_names[] = {
@@ -90,6 +91,15 @@ static const struct drm_prop_enum_list drm_colorop_lut3d_interpolation_list[] =
 	{ DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL, "Tetrahedral" },
 };
 
+static const char * const colorop_fixed_matrix_type_names[] = {
+	[DRM_COLOROP_FM_YCBCR601_FULL_RGB] = "YCbCr 601 Full to RGB",
+	[DRM_COLOROP_FM_YCBCR601_LIMITED_RGB] = "YCbCr 601 Limited to RGB",
+	[DRM_COLOROP_FM_YCBCR709_FULL_RGB] = "YCbCr 709 Full to RGB",
+	[DRM_COLOROP_FM_YCBCR709_LIMITED_RGB] = "YCbCr 709 Limited to RGB",
+	[DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB] = "YCbCr 2020 NC Full to RGB",
+	[DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB] = "YCbCr 2020 NC Limited to RGB",
+};
+
 /* Init Helpers */
 
 static int drm_plane_colorop_init(struct drm_device *dev, struct drm_colorop *colorop,
@@ -453,6 +463,80 @@ int drm_plane_colorop_3dlut_init(struct drm_device *dev, struct drm_colorop *col
 }
 EXPORT_SYMBOL(drm_plane_colorop_3dlut_init);
 
+/**
+ * drm_plane_colorop_fixed_matrix_init - Initialize a DRM_COLOROP_FIXED_MATRIX
+ *
+ * @dev: DRM device
+ * @colorop: The drm_colorop object to initialize
+ * @plane: The associated drm_plane
+ * @funcs: control functions for the new colorop
+ * @supported_fm: A bitfield of supported drm_colorop_fixed_matrix_type enum values,
+ *               created using BIT(fixed_matrix_type) and combined with the OR '|'
+ *               operator.
+ * @flags: bitmask of misc, see DRM_COLOROP_FLAG_* defines.
+ * @return zero on success, -E value on failure
+ */
+int drm_plane_colorop_fixed_matrix_init(struct drm_device *dev, struct drm_colorop *colorop,
+					struct drm_plane *plane,
+					const struct drm_colorop_funcs *funcs,
+					u64 supported_fm, uint32_t flags)
+{
+	struct drm_prop_enum_list enum_list[DRM_COLOROP_FM_COUNT];
+	int i, len;
+	struct drm_property *prop;
+	int ret;
+
+	if (!supported_fm) {
+		drm_err(dev,
+			"No supported FM type op for new Fixed Matrix colorop on [PLANE:%d:%s]\n",
+			plane->base.id, plane->name);
+		return -EINVAL;
+	}
+
+	if ((supported_fm & -BIT(DRM_COLOROP_FM_COUNT)) != 0) {
+		drm_err(dev, "Unknown Fixed Matrix provided on [PLANE:%d:%s]\n",
+			plane->base.id, plane->name);
+		return -EINVAL;
+	}
+
+	ret = drm_plane_colorop_init(dev, colorop, plane, funcs, DRM_COLOROP_FIXED_MATRIX, flags);
+	if (ret)
+		return ret;
+
+	len = 0;
+	for (i = 0; i < DRM_COLOROP_FM_COUNT; i++) {
+		if ((supported_fm & BIT(i)) == 0)
+			continue;
+
+		enum_list[len].type = i;
+		enum_list[len].name = colorop_fixed_matrix_type_names[i];
+		len++;
+	}
+
+	if (WARN_ON(len <= 0))
+		return -EINVAL;
+
+	prop = drm_property_create_enum(dev, DRM_MODE_PROP_ATOMIC, "FIXED_MATRIX_TYPE",
+					enum_list, len);
+
+	if (!prop)
+		return -ENOMEM;
+
+	colorop->fixed_matrix_type_property = prop;
+	/*
+	 * Default to the first supported CSC mode as provided by the driver.
+	 * Intuitively this should be something that keeps the colorop in pixel bypass
+	 * mode but that is already handled via the standard colorop bypass
+	 * property.
+	 */
+	drm_object_attach_property(&colorop->base, colorop->fixed_matrix_type_property,
+				   enum_list[0].type);
+	drm_colorop_reset(colorop);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_plane_colorop_fixed_matrix_init);
+
 static void __drm_atomic_helper_colorop_duplicate_state(struct drm_colorop *colorop,
 							struct drm_colorop_state *state)
 {
@@ -533,6 +617,13 @@ static void __drm_colorop_state_init(struct drm_colorop_state *colorop_state,
 							   &val))
 			colorop_state->lut3d_interpolation = val;
 	}
+
+	if (colorop->fixed_matrix_type_property) {
+		if (!drm_object_property_get_default_value(&colorop->base,
+							   colorop->fixed_matrix_type_property,
+							   &val))
+			colorop_state->fixed_matrix_type = val;
+	}
 }
 
 /**
@@ -596,6 +687,7 @@ static const char * const colorop_type_name[] = {
 	[DRM_COLOROP_CTM_3X4] = "3x4 Matrix",
 	[DRM_COLOROP_MULTIPLIER] = "Multiplier",
 	[DRM_COLOROP_3D_LUT] = "3D LUT",
+	[DRM_COLOROP_FIXED_MATRIX] = "Fixed Matrix",
 };
 
 static const char * const colorop_lu3d_interpolation_name[] = {
@@ -652,6 +744,21 @@ const char *drm_get_colorop_lut3d_interpolation_name(enum drm_colorop_lut3d_inte
 	return colorop_lu3d_interpolation_name[type];
 }
 
+/**
+ * drm_get_colorop_fixed_matrix_type_name: return a string for fixed matrix type
+ * @type: fixed matrix type to compute name of
+ *
+ * In contrast to the other drm_get_*_name functions this one here returns a
+ * const pointer and hence is threadsafe.
+ */
+const char *drm_get_colorop_fixed_matrix_type_name(enum drm_colorop_fixed_matrix_type type)
+{
+	if (WARN_ON(type >= ARRAY_SIZE(colorop_fixed_matrix_type_names)))
+		return "unknown";
+
+	return colorop_fixed_matrix_type_names[type];
+}
+
 /**
  * drm_colorop_set_next_property - sets the next pointer
  * @colorop: drm colorop
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 8b4baed060f3ae..d94c86bfed8662 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -546,6 +546,150 @@ EXPORT_SYMBOL(drmm_connector_init);
  * drmm_connector_hdmi_init - Init a preallocated HDMI connector
  * @dev: DRM device
  * @connector: A pointer to the HDMI connector to init
+ * @funcs: callbacks for this connector
+ * @hdmi_funcs: HDMI-related callbacks and capabilities for this connector
+ * @connector_type: user visible type of the connector
+ * @ddc: optional pointer to the associated ddc adapter
+ *
+ * Initialises a preallocated HDMI connector. Connectors can be
+ * subclassed as part of driver connector objects.
+ *
+ * Cleanup is automatically handled with a call to
+ * drm_connector_cleanup() in a DRM-managed action.
+ *
+ * The connector structure should be allocated with drmm_kzalloc().
+ *
+ * The @drm_connector_funcs.destroy hook must be NULL.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drmm_connector_hdmi_init(struct drm_device *dev,
+			     struct drm_connector *connector,
+			     const struct drm_connector_funcs *funcs,
+			     const struct drm_connector_hdmi_funcs *hdmi_funcs,
+			     int connector_type,
+			     struct i2c_adapter *ddc)
+{
+	int ret;
+
+	if (!hdmi_funcs)
+		return -EINVAL;
+
+	if (!hdmi_funcs->vendor || !hdmi_funcs->product)
+		return -EINVAL;
+
+	if ((strlen(hdmi_funcs->vendor) > DRM_CONNECTOR_HDMI_VENDOR_LEN) ||
+	    (strlen(hdmi_funcs->product) > DRM_CONNECTOR_HDMI_PRODUCT_LEN))
+		return -EINVAL;
+
+	if (!hdmi_funcs->supported_formats ||
+	    !(hdmi_funcs->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444)))
+		return -EINVAL;
+
+	if (connector->ycbcr_420_allowed !=
+	    !!(hdmi_funcs->supported_formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420)))
+		return -EINVAL;
+
+	if (!(hdmi_funcs->max_bpc == 8 ||
+	      hdmi_funcs->max_bpc == 10 ||
+	      hdmi_funcs->max_bpc == 12))
+		return -EINVAL;
+
+	if (!hdmi_funcs->avi.clear_infoframe ||
+	    !hdmi_funcs->avi.write_infoframe ||
+	    !hdmi_funcs->hdmi.clear_infoframe ||
+	    !hdmi_funcs->hdmi.write_infoframe)
+		return -EINVAL;
+
+	if (hdmi_funcs->supported_hdmi_ver >= HDMI_VERSION_2_0 &&
+	    (!hdmi_funcs->scrambler_enable ||
+	     !hdmi_funcs->scrambler_disable)) {
+		drm_err(dev, "Scrambler callbacks missing for HDMI 2.x\n");
+		return -EINVAL;
+	}
+
+	if (!(connector_type == DRM_MODE_CONNECTOR_HDMIA ||
+	      connector_type == DRM_MODE_CONNECTOR_HDMIB))
+		return -EINVAL;
+
+	ret = drmm_connector_init(dev, connector, funcs, connector_type, ddc);
+	if (ret)
+		return ret;
+
+	/* TODO: remove after conversion to new drmm_connector_hdmi_init() */
+	connector->hdmi.supported_formats = hdmi_funcs->supported_formats;
+
+	/*
+	 * The supported HDMI version can be used to determinate the maximum
+	 * TMDS character rate allowed by the specification. Some controllers,
+	 * however, may support a lower rate than that version would imply.
+	 *
+	 * A non-zero caps->max_tmds_char_rate lets drivers override this
+	 * default with the actual controller capability. A value of zero keeps
+	 * the limit inferred from supported_hdmi_ver.
+	 */
+	if (hdmi_funcs->supported_hdmi_ver >= HDMI_VERSION_2_0)
+		connector->hdmi.max_tmds_char_rate = HDMI_2_0_TMDS_CHAR_RATE_MAX_HZ;
+	else if (hdmi_funcs->supported_hdmi_ver >= HDMI_VERSION_1_3)
+		connector->hdmi.max_tmds_char_rate = HDMI_1_3_TMDS_CHAR_RATE_MAX_HZ;
+	else if (hdmi_funcs->supported_hdmi_ver >= HDMI_VERSION_1_0)
+		connector->hdmi.max_tmds_char_rate = HDMI_1_0_TMDS_CHAR_RATE_MAX_HZ;
+
+	if (hdmi_funcs->supported_tmds_char_rate) {
+		if (hdmi_funcs->supported_tmds_char_rate > connector->hdmi.max_tmds_char_rate) {
+			drm_err(dev, "Enforced max_tmds_char_rate exceeds %llu spec limit\n",
+				connector->hdmi.max_tmds_char_rate);
+			return -EINVAL;
+		}
+
+		connector->hdmi.max_tmds_char_rate = hdmi_funcs->supported_tmds_char_rate;
+	}
+
+	/* TODO: remove after conversion to new drmm_connector_hdmi_init() */
+	strtomem_pad(connector->hdmi.vendor, hdmi_funcs->vendor, 0);
+	strtomem_pad(connector->hdmi.product, hdmi_funcs->product, 0);
+
+	/*
+	 * drm_connector_attach_max_bpc_property() requires the
+	 * connector to have a state.
+	 */
+	if (connector->funcs->atomic_create_state) {
+		struct drm_connector_state *state;
+
+		state = connector->funcs->atomic_create_state(connector);
+		if (IS_ERR(state))
+			return PTR_ERR(state);
+
+		connector->state = state;
+	} else if (connector->funcs->reset) {
+		connector->funcs->reset(connector);
+		if (!connector->state)
+			return -ENOMEM;
+	}
+
+	drm_connector_attach_max_bpc_property(connector, 8, hdmi_funcs->max_bpc);
+	/* TODO: remove after conversion to new drmm_connector_hdmi_init() */
+	connector->max_bpc = hdmi_funcs->max_bpc;
+
+	if (hdmi_funcs->max_bpc > 8)
+		drm_connector_attach_hdr_output_metadata_property(connector);
+
+	ret = drm_connector_attach_color_format_property(connector,
+							 hdmi_funcs->supported_formats);
+	if (ret)
+		return ret;
+
+	connector->hdmi.funcs = hdmi_funcs;
+
+	return 0;
+}
+EXPORT_SYMBOL(drmm_connector_hdmi_init);
+
+/**
+ * drmm_connector_hdmi_ini2 - Init a preallocated HDMI connector
+ * @dev: DRM device
+ * @connector: A pointer to the HDMI connector to init
  * @vendor: HDMI Controller Vendor name
  * @product: HDMI Controller Product name
  * @funcs: callbacks for this connector
@@ -568,7 +712,7 @@ EXPORT_SYMBOL(drmm_connector_init);
  * Returns:
  * Zero on success, error code on failure.
  */
-int drmm_connector_hdmi_init(struct drm_device *dev,
+int drmm_connector_hdmi_ini2(struct drm_device *dev,
 			     struct drm_connector *connector,
 			     const char *vendor, const char *product,
 			     const struct drm_connector_funcs *funcs,
@@ -628,6 +772,8 @@ int drmm_connector_hdmi_init(struct drm_device *dev,
 		connector->state = state;
 	} else if (connector->funcs->reset) {
 		connector->funcs->reset(connector);
+		if (!connector->state)
+			return -ENOMEM;
 	}
 
 	drm_connector_attach_max_bpc_property(connector, 8, max_bpc);
@@ -644,7 +790,7 @@ int drmm_connector_hdmi_init(struct drm_device *dev,
 
 	return 0;
 }
-EXPORT_SYMBOL(drmm_connector_hdmi_init);
+EXPORT_SYMBOL(drmm_connector_hdmi_ini2);
 
 /**
  * drm_connector_attach_edid_property - attach edid property.
@@ -780,6 +926,9 @@ void drm_connector_cleanup(struct drm_connector *connector)
 		    DRM_CONNECTOR_REGISTERED))
 		drm_connector_unregister(connector);
 
+	if (connector->hdmi.scdc_work_initialized)
+		cancel_delayed_work_sync(&connector->hdmi.scdc_work);
+
 	platform_device_unregister(connector->hdmi_audio.codec_pdev);
 
 	if (connector->privacy_screen) {
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index 83146ffef00cd2..3492787c36973b 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -322,13 +322,11 @@ bool drm_panic_is_enabled(struct drm_device *dev);
 void drm_panic_register(struct drm_device *dev);
 void drm_panic_unregister(struct drm_device *dev);
 void drm_panic_init(void);
-void drm_panic_exit(void);
 #else
 static inline bool drm_panic_is_enabled(struct drm_device *dev) { return false; }
 static inline void drm_panic_register(struct drm_device *dev) {}
 static inline void drm_panic_unregister(struct drm_device *dev) {}
 static inline void drm_panic_init(void) {}
-static inline void drm_panic_exit(void) {}
 #endif
 
 #endif /* __DRM_CRTC_INTERNAL_H__ */
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index ae1c6126c2c592..a72f9723d3a974 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -633,161 +633,6 @@ static const struct file_operations drm_connector_fops = {
 	.write = connector_write
 };
 
-static ssize_t
-audio_infoframe_read(struct file *filp, char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct drm_connector_hdmi_infoframe *infoframe;
-	struct drm_connector *connector;
-	union hdmi_infoframe *frame;
-	u8 buf[HDMI_INFOFRAME_SIZE(AUDIO)];
-	ssize_t len = 0;
-
-	connector = filp->private_data;
-	mutex_lock(&connector->hdmi.infoframes.lock);
-
-	infoframe = &connector->hdmi.infoframes.audio;
-	if (!infoframe->set)
-		goto out;
-
-	frame = &infoframe->data;
-	len = hdmi_infoframe_pack(frame, buf, sizeof(buf));
-	if (len < 0)
-		goto out;
-
-	len = simple_read_from_buffer(ubuf, count, ppos, buf, len);
-
-out:
-	mutex_unlock(&connector->hdmi.infoframes.lock);
-	return len;
-}
-
-static const struct file_operations audio_infoframe_fops = {
-	.owner   = THIS_MODULE,
-	.open    = simple_open,
-	.read    = audio_infoframe_read,
-};
-
-static int create_hdmi_audio_infoframe_file(struct drm_connector *connector,
-					    struct dentry *parent)
-{
-	struct dentry *file;
-
-	if (!connector->hdmi.funcs ||
-	    !connector->hdmi.funcs->audio.write_infoframe)
-		return 0;
-
-	file = debugfs_create_file("audio", 0400, parent, connector, &audio_infoframe_fops);
-	if (IS_ERR(file))
-		return PTR_ERR(file);
-
-	return 0;
-}
-
-#define DEFINE_INFOFRAME_FILE(_f) \
-static ssize_t _f##_read_infoframe(struct file *filp, \
-				   char __user *ubuf, \
-				   size_t count,      \
-				   loff_t *ppos)      \
-{ \
-	struct drm_connector_hdmi_infoframe *infoframe; \
-	struct drm_connector_state *conn_state; \
-	struct drm_connector *connector; \
-	union hdmi_infoframe *frame; \
-	struct drm_device *dev; \
-	u8 buf[HDMI_INFOFRAME_SIZE(MAX)]; \
-	ssize_t len = 0; \
-	\
-	connector = filp->private_data; \
-	dev = connector->dev; \
-	\
-	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); \
-	\
-	conn_state = connector->state; \
-	infoframe = &conn_state->hdmi.infoframes._f; \
-	if (!infoframe->set) \
-		goto out; \
-	\
-	frame = &infoframe->data; \
-	len = hdmi_infoframe_pack(frame, buf, sizeof(buf)); \
-	if (len < 0) \
-		goto out; \
-	\
-	len = simple_read_from_buffer(ubuf, count, ppos, buf, len); \
-	\
-out: \
-	drm_modeset_unlock(&dev->mode_config.connection_mutex); \
-	return len; \
-} \
-\
-static const struct file_operations _f##_infoframe_fops = { \
-	.owner = THIS_MODULE, \
-	.open = simple_open, \
-	.read = _f##_read_infoframe, \
-}; \
-\
-static int create_hdmi_## _f ## _infoframe_file(struct drm_connector *connector, \
-						struct dentry *parent) \
-{ \
-	struct dentry *file; \
-	\
-	if (!connector->hdmi.funcs || \
-	    !connector->hdmi.funcs->_f.write_infoframe) \
-		return 0; \
-	file = debugfs_create_file(#_f, 0400, parent, connector, &_f ## _infoframe_fops); \
-	if (IS_ERR(file)) \
-		return PTR_ERR(file); \
-	\
-	return 0; \
-}
-
-DEFINE_INFOFRAME_FILE(avi);
-DEFINE_INFOFRAME_FILE(hdmi);
-DEFINE_INFOFRAME_FILE(hdr_drm);
-DEFINE_INFOFRAME_FILE(spd);
-
-static int create_hdmi_infoframe_files(struct drm_connector *connector,
-				       struct dentry *parent)
-{
-	int ret;
-
-	ret = create_hdmi_audio_infoframe_file(connector, parent);
-	if (ret)
-		return ret;
-
-	ret = create_hdmi_avi_infoframe_file(connector, parent);
-	if (ret)
-		return ret;
-
-	ret = create_hdmi_hdmi_infoframe_file(connector, parent);
-	if (ret)
-		return ret;
-
-	ret = create_hdmi_hdr_drm_infoframe_file(connector, parent);
-	if (ret)
-		return ret;
-
-	ret = create_hdmi_spd_infoframe_file(connector, parent);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static void hdmi_debugfs_add(struct drm_connector *connector)
-{
-	struct dentry *dir;
-
-	if (!(connector->connector_type == DRM_MODE_CONNECTOR_HDMIA ||
-	      connector->connector_type == DRM_MODE_CONNECTOR_HDMIB))
-		return;
-
-	dir = debugfs_create_dir("infoframes", connector->debugfs_entry);
-	if (IS_ERR(dir))
-		return;
-
-	create_hdmi_infoframe_files(connector, dir);
-}
-
 void drm_debugfs_connector_add(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
@@ -815,8 +660,6 @@ void drm_debugfs_connector_add(struct drm_connector *connector)
 	debugfs_create_file("output_bpc", 0444, root, connector,
 			    &output_bpc_fops);
 
-	hdmi_debugfs_add(connector);
-
 	if (connector->funcs->debugfs_init)
 		connector->funcs->debugfs_init(connector, root);
 }
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index c808958a2188e2..0cdc606af8d18e 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -36,6 +36,7 @@
 #include <linux/mount.h>
 #include <linux/pseudo_fs.h>
 #include <linux/sched.h>
+#include <linux/seq_buf.h>
 #include <linux/slab.h>
 #include <linux/sprintf.h>
 #include <linux/srcu.h>
@@ -50,13 +51,13 @@
 #include <drm/drm_file.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mode_object.h>
-#include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 #include <drm/drm_privacy_screen_machine.h>
 #include <drm/drm_ras_genl_family.h>
 
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
+#include "drm_panic_internal.h"
 
 MODULE_AUTHOR("Gareth Hughes, Leif Delgass, José Fonseca, Jon Smirl");
 MODULE_DESCRIPTION("DRM shared core routines");
@@ -545,6 +546,8 @@ static const char *drm_get_wedge_recovery(unsigned int opt)
 		return "bus-reset";
 	case DRM_WEDGE_RECOVERY_VENDOR:
 		return "vendor-specific";
+	case DRM_WEDGE_RECOVERY_COLD_RESET:
+		return "cold-reset";
 	default:
 		return NULL;
 	}
@@ -574,27 +577,31 @@ static const char *drm_get_wedge_recovery(unsigned int opt)
 int drm_dev_wedged_event(struct drm_device *dev, unsigned long method,
 			 struct drm_wedge_task_info *info)
 {
-	char event_string[WEDGE_STR_LEN], pid_string[PID_STR_LEN], comm_string[COMM_STR_LEN];
-	char *envp[] = { event_string, NULL, NULL, NULL };
-	const char *recovery = NULL;
-	unsigned int len, opt;
+	DECLARE_SEQ_BUF(event_string, WEDGE_STR_LEN);
+	char pid_string[PID_STR_LEN], comm_string[COMM_STR_LEN];
+	char *envp[4] = { };
+	unsigned int len = 0, opt;
 
-	len = scnprintf(event_string, sizeof(event_string), "%s", "WEDGED=");
+	seq_buf_puts(&event_string, "WEDGED=");
+	envp[0] = event_string.buffer;
 
 	for_each_set_bit(opt, &method, BITS_PER_TYPE(method)) {
-		recovery = drm_get_wedge_recovery(opt);
+		const char *recovery = drm_get_wedge_recovery(opt);
 		if (drm_WARN_ONCE(dev, !recovery, "invalid recovery method %u\n", opt))
 			break;
 
-		len += scnprintf(event_string + len, sizeof(event_string) - len, "%s,", recovery);
+		if (drm_WARN_ON_ONCE(dev, seq_buf_printf(&event_string, "%s,", recovery)))
+			break;
+
+		len = seq_buf_used(&event_string);
 	}
 
-	if (recovery)
-		/* Get rid of trailing comma */
-		event_string[len - 1] = '\0';
+	if (len)
+		/* Strip trailing comma; also discards any partial overflow entry */
+		event_string.buffer[len - 1] = '\0';
 	else
-		/* Caller is unsure about recovery, do the best we can at this point. */
-		snprintf(event_string, sizeof(event_string), "%s", "WEDGED=unknown");
+		/* No complete entry written, do the best we can at this point. */
+		snprintf(event_string.buffer, event_string.size, "%s", "WEDGED=unknown");
 
 	drm_info(dev, "device wedged, %s\n", method == DRM_WEDGE_RECOVERY_NONE ?
 		 "but no recovery needed" : "needs recovery");
@@ -1248,7 +1255,6 @@ static void drm_core_exit(void)
 {
 	drm_ras_genl_family_unregister();
 	drm_privacy_screen_lookup_exit();
-	drm_panic_exit();
 	accel_core_exit();
 	unregister_chrdev(DRM_MAJOR, "drm");
 	drm_debugfs_remove_root();
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 07970e5b5f65e5..3148038747b6eb 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -32,6 +32,7 @@
 #include <linux/byteorder/generic.h>
 #include <linux/cec.h>
 #include <linux/export.h>
+#include <linux/fb.h> /* for KHZ2PICOS() */
 #include <linux/hdmi.h>
 #include <linux/i2c.h>
 #include <linux/kernel.h>
@@ -101,20 +102,42 @@ enum drm_edid_internal_quirk {
 #define MICROSOFT_IEEE_OUI	0xca125c
 #define AMD_IEEE_OUI        0x00001A
 
+#define AMD_VSDB_V1_PAYLOAD_LEN 8
+#define AMD_VSDB_V2_PAYLOAD_LEN 13
 #define AMD_VSDB_V3_PAYLOAD_MIN_LEN 15
-#define AMD_VSDB_V3_PAYLOAD_MAX_LEN 20
+#define AMD_VSDB_V3_PAYLOAD_MAX_LEN 21
 
-struct amd_vsdb_v3_payload {
+#define AMD_VSDB_V3_MAX_VFREQ_EXT2_LEN 21
+
+struct amd_vsdb_common_payload {
 	u8 oui[3];
 	u8 version;
+} __packed;
+
+struct amd_vsdb_v1_payload {
+	struct amd_vsdb_common_payload common;
 	u8 feature_caps;
-	u8 rsvd0[3];
+	u8 min_vfreq;
+	u8 max_vfreq;
+	u8 freesync_vcp_code;
+} __packed;
+
+struct amd_vsdb_v2_payload {
+	struct amd_vsdb_v1_payload v1;
 	u8 cs_eotf_support;
 	u8 lum1_max;
 	u8 lum1_min;
 	u8 lum2_max;
 	u8 lum2_min;
-	u8 rsvd1[2];
+} __packed;
+
+struct amd_vsdb_v3_payload {
+	struct amd_vsdb_v2_payload v2;
+	u8 max_vfreq_ext_low;
+	/*
+	 * byte 14: bits 0-1 extend MSB of max refresh rate
+	 */
+	u8 max_vfreq_ext_high;
 	/*
 	 * Bytes beyond AMD_VSDB_V3_PAYLOAD_MIN_LEN are optional; a
 	 * monitor may provide a payload as short as 15 bytes.  Always
@@ -123,6 +146,12 @@ struct amd_vsdb_v3_payload {
 	u8 extra[AMD_VSDB_V3_PAYLOAD_MAX_LEN - AMD_VSDB_V3_PAYLOAD_MIN_LEN];
 } __packed;
 
+static const u8 amd_vsdb_min_len[] = {
+	[1] = AMD_VSDB_V1_PAYLOAD_LEN,
+	[2] = AMD_VSDB_V2_PAYLOAD_LEN,
+	[3] = AMD_VSDB_V3_PAYLOAD_MIN_LEN,
+};
+
 struct detailed_mode_closure {
 	struct drm_connector *connector;
 	const struct drm_edid *drm_edid;
@@ -5231,7 +5260,7 @@ static bool cea_db_is_microsoft_vsdb(const struct cea_db *db)
 static bool cea_db_is_amd_vsdb(const struct cea_db *db)
 {
 	return cea_db_is_vendor(db, AMD_IEEE_OUI) &&
-		cea_db_payload_len(db) >= AMD_VSDB_V3_PAYLOAD_MIN_LEN &&
+		cea_db_payload_len(db) >= AMD_VSDB_V1_PAYLOAD_LEN &&
 		cea_db_payload_len(db) <= AMD_VSDB_V3_PAYLOAD_MAX_LEN;
 }
 
@@ -6182,6 +6211,33 @@ static void drm_parse_ycbcr420_deep_color_info(struct drm_connector *connector,
 	hdmi->y420_dc_modes = dc_mask;
 }
 
+static void drm_parse_hdmi_gaming_info(struct drm_hdmi_info *hdmi, const u8 *db)
+{
+	struct drm_hdmi_vrr_cap *vrr = &hdmi->vrr_cap;
+
+	if (cea_db_payload_len(db) < 8)
+		return;
+
+	hdmi->fapa_start_location = db[8] & DRM_EDID_FAPA_START_LOCATION;
+	hdmi->allm = db[8] & DRM_EDID_ALLM;
+	vrr->fva = db[8] & DRM_EDID_FVA;
+	vrr->cnmvrr = db[8] & DRM_EDID_CNMVRR;
+	vrr->cinema_vrr = db[8] & DRM_EDID_CINEMA_VRR;
+	vrr->mdelta = db[8] & DRM_EDID_MDELTA;
+
+	if (cea_db_payload_len(db) < 9)
+		return;
+
+	vrr->vrr_min = db[9] & DRM_EDID_VRR_MIN_MASK;
+	vrr->supported = (vrr->vrr_min > 0 && vrr->vrr_min <= 48);
+
+	if (cea_db_payload_len(db) < 10)
+		return;
+
+	vrr->vrr_max = (db[9] & DRM_EDID_VRR_MAX_UPPER_MASK) << 2 | db[10];
+	vrr->supported &= (vrr->vrr_max == 0 || vrr->vrr_max >= 100);
+}
+
 static void drm_parse_dsc_info(struct drm_hdmi_dsc_cap *hdmi_dsc,
 			       const u8 *hf_scds)
 {
@@ -6308,6 +6364,8 @@ static void drm_parse_hdmi_forum_scds(struct drm_connector *connector,
 
 	drm_parse_ycbcr420_deep_color_info(connector, hf_scds);
 
+	drm_parse_hdmi_gaming_info(&connector->display_info.hdmi, hf_scds);
+
 	if (cea_db_payload_len(hf_scds) >= 11 && hf_scds[11]) {
 		drm_parse_dsc_info(hdmi_dsc, hf_scds);
 		dsc_support = true;
@@ -6317,6 +6375,19 @@ static void drm_parse_hdmi_forum_scds(struct drm_connector *connector,
 		    "[CONNECTOR:%d:%s] HF-VSDB: max TMDS clock: %d KHz, HDMI 2.1 support: %s, DSC 1.2 support: %s\n",
 		    connector->base.id, connector->name,
 		    max_tmds_clock, str_yes_no(max_frl_rate), str_yes_no(dsc_support));
+	drm_dbg_kms(connector->dev,
+		    "[CONNECTOR:%d:%s] FAPA in blanking: %s, ALLM support: %s, Fast Vactive support: %s\n",
+		    connector->base.id, connector->name, str_yes_no(hdmi->fapa_start_location),
+		    str_yes_no(hdmi->allm), str_yes_no(hdmi->vrr_cap.fva));
+	drm_dbg_kms(connector->dev,
+		    "[CONNECTOR:%d:%s] Negative M VRR support: %s, CinemaVRR support: %s, Mdelta: %d\n",
+		    connector->base.id, connector->name, str_yes_no(hdmi->vrr_cap.cnmvrr),
+		    str_yes_no(hdmi->vrr_cap.cinema_vrr), hdmi->vrr_cap.mdelta);
+	drm_dbg_kms(connector->dev,
+		    "[CONNECTOR:%d:%s] VRRmin: %u, VRRmax: %u, VRR supported: %s\n",
+		    connector->base.id, connector->name, hdmi->vrr_cap.vrr_min,
+		    hdmi->vrr_cap.vrr_max, str_yes_no(hdmi->vrr_cap.supported));
+
 }
 
 static void drm_parse_hdmi_deep_color_info(struct drm_connector *connector,
@@ -6431,43 +6502,104 @@ static void drm_parse_microsoft_vsdb(struct drm_connector *connector,
 		    connector->base.id, connector->name, version, db[5]);
 }
 
+static void drm_parse_amd_vsdb_common(struct drm_display_info *info,
+				  const struct amd_vsdb_common_payload *data)
+{
+	info->amd_vsdb.version = data->version;
+}
+
+static void drm_parse_amd_vsdb_v1(struct drm_display_info *info,
+				  const struct amd_vsdb_v1_payload *data)
+{
+	info->amd_vsdb.freesync_supported = data->feature_caps & 0x1;
+	info->amd_vsdb.min_frame_rate = data->min_vfreq;
+	info->amd_vsdb.max_frame_rate = data->max_vfreq;
+	info->amd_vsdb.freesync_vcp_code = data->freesync_vcp_code;
+
+	drm_parse_amd_vsdb_common(info, &data->common);
+}
+
+static void drm_parse_amd_vsdb_v2(struct drm_display_info *info,
+				  const struct amd_vsdb_v2_payload *data)
+{
+	info->amd_vsdb.luminance_range1.max_luminance = data->lum1_max;
+	info->amd_vsdb.luminance_range1.min_luminance = data->lum1_min;
+	info->amd_vsdb.luminance_range2.max_luminance = data->lum2_max;
+	info->amd_vsdb.luminance_range2.min_luminance = data->lum2_min;
+
+	drm_parse_amd_vsdb_v1(info, &data->v1);
+}
+
+static void drm_parse_amd_vsdb_v3(struct drm_display_info *info,
+				  const struct amd_vsdb_v3_payload *data,
+				  int payload_len)
+{
+	u16 max_frame_rate;
+
+	info->amd_vsdb.replay_mode = data->v2.v1.feature_caps & 0x40;
+	info->amd_vsdb.panel_type = (data->v2.cs_eotf_support & 0xC0) >> 6;
+
+	drm_parse_amd_vsdb_v2(info, &data->v2);
+
+	/* vfreq is provided in a different set of fields for v3. */
+	max_frame_rate = data->max_vfreq_ext_low |
+			 (data->max_vfreq_ext_high & 0x3) << 8;
+
+	/*
+	 * The AMD VSDB v3 payload length is variable (15..21 bytes).
+	 * All fields through max_vfreq_ext_high (byte 14) are always
+	 * present, but data->extra[] (bytes 15+) may not be, so any access
+	 * to extra[] must be guarded with a runtime length check to
+	 * avoid out-of-bounds reads on shorter (but spec-valid) payloads.
+	 */
+	if (payload_len >= AMD_VSDB_V3_MAX_VFREQ_EXT2_LEN)
+		max_frame_rate |= (data->extra[5] & 0x3) << 10;
+
+	info->amd_vsdb.max_frame_rate = max_frame_rate;
+}
+
 static void drm_parse_amd_vsdb(struct drm_connector *connector,
 			       const struct cea_db *db)
 {
 	struct drm_display_info *info = &connector->display_info;
 	const u8 *data = cea_db_data(db);
-	const struct amd_vsdb_v3_payload *p;
+	int payload_len = cea_db_payload_len(db);
+	u8 version;
 
-	p = (const struct amd_vsdb_v3_payload *)data;
+	version = ((const struct amd_vsdb_common_payload *)data)->version;
 
-	if (p->version != 0x03) {
+	if (version < 1) {
 		drm_dbg_kms(connector->dev,
 			    "[CONNECTOR:%d:%s] Unsupported AMD VSDB version %u\n",
-			    connector->base.id, connector->name, p->version);
+			    connector->base.id, connector->name, version);
 		return;
 	}
 
-	info->amd_vsdb.version = p->version;
-	info->amd_vsdb.replay_mode = p->feature_caps & 0x40;
-	info->amd_vsdb.panel_type = (p->cs_eotf_support & 0xC0) >> 6;
-	info->amd_vsdb.luminance_range1.max_luminance = p->lum1_max;
-	info->amd_vsdb.luminance_range1.min_luminance = p->lum1_min;
-	info->amd_vsdb.luminance_range2.max_luminance = p->lum2_max;
-	info->amd_vsdb.luminance_range2.min_luminance = p->lum2_min;
+	if ((version <= 3 && payload_len < amd_vsdb_min_len[version]) ||
+		(version > 3 && payload_len < AMD_VSDB_V3_PAYLOAD_MIN_LEN)) {
+		drm_dbg_kms(connector->dev,
+			    "[CONNECTOR:%d:%s] AMD VSDB v%u payload too short (%d bytes)\n",
+			    connector->base.id, connector->name, version, payload_len);
+		return;
+	}
 
-	/*
-	 * The AMD VSDB v3 payload length is variable (15..20 bytes).
-	 * All fields through p->rsvd1 (byte 14) are always present,
-	 * but p->extra[] (bytes 15+) may not be.  Any future access to
-	 * extra[] must be guarded with a runtime length check to avoid
-	 * out-of-bounds reads on shorter (but spec-valid) payloads.
-	 * For example:
-	 *
-	 *   int len = cea_db_payload_len(db);
-	 *
-	 *   if (len > AMD_VSDB_V3_PAYLOAD_MIN_LEN)
-	 *       info->amd_vsdb.foo = p->extra[0];
-	 */
+	switch (version) {
+	default:
+		drm_dbg_kms(connector->dev,
+				"[CONNECTOR:%d:%s] AMD VSDB v%u is unsupported, VSDB will be parsed as v3.\n",
+				connector->base.id, connector->name, version);
+		fallthrough;
+	case 3:
+		drm_parse_amd_vsdb_v3(info, (const struct amd_vsdb_v3_payload *)data,
+				      payload_len);
+		break;
+	case 2:
+		drm_parse_amd_vsdb_v2(info, (const struct amd_vsdb_v2_payload *)data);
+		break;
+	case 1:
+		drm_parse_amd_vsdb_v1(info, (const struct amd_vsdb_v1_payload *)data);
+		break;
+	}
 }
 
 static void drm_parse_cea_ext(struct drm_connector *connector,
@@ -7721,3 +7853,50 @@ bool drm_edid_is_digital(const struct drm_edid *drm_edid)
 		drm_edid->edid->input & DRM_EDID_INPUT_DIGITAL;
 }
 EXPORT_SYMBOL(drm_edid_is_digital);
+
+/**
+ * drm_edid_detect_panel_size - Get a panel's size from EDID
+ * @drm_edid: EDID of the panel.
+ * @width: Returns the panel's width in pixels per scanline, if given
+ * @height: Returns the panel's height in scanlines, if given
+ *
+ * This function detects the preferred size of a panel from the given
+ * EDID. There is no such information stored in the EDID block directly,
+ * but the preferred mode often corresponds to the panel's native geometry.
+ *
+ * This helper should only be used during initialization before the
+ * connector is available. For regular use, retrieve the available display
+ * modes with the connector functions.
+ *
+ * Return: Zero on success, or a negative errno code otherwise.
+ */
+int drm_edid_detect_panel_size(const struct drm_edid *drm_edid,
+			       unsigned int *width, unsigned int *height)
+{
+	const struct edid *edid = drm_edid->edid;
+	const struct detailed_timing *dt;
+	const struct detailed_pixel_timing *pt;
+
+	/*
+	 * Use display mode from the Preferred Timing Descriptor. For old
+	 * and obscure displays, we might need better heuristics.
+	 */
+
+	if (edid->revision < 4 || !(edid->features & DRM_EDID_FEATURE_PREFERRED_TIMING))
+		return -EINVAL; /* no Preferred Timing Descriptor */
+
+	dt = &edid->detailed_timings[0];
+
+	if (!is_detailed_timing_descriptor(dt))
+		return -EINVAL;
+
+	pt = &dt->data.pixel_data;
+
+	if (width)
+		*width = (pt->hactive_hblank_hi & 0xf0) << 4 | pt->hactive_lo;
+	if (height)
+		*height = (pt->vactive_vblank_hi & 0xf0) << 4 | pt->vactive_lo;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_edid_detect_panel_size);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index b0e0e4118573d8..3ce38df7b8b85b 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -353,8 +353,11 @@ void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)
 	}
 	mutex_unlock(&dev->object_name_lock);
 
-	if (final)
+	if (final) {
+		if (obj->funcs->handle_free)
+			obj->funcs->handle_free(obj);
 		drm_gem_object_put(obj);
+	}
 }
 
 /*
@@ -1648,15 +1651,13 @@ EXPORT_SYMBOL(drm_gem_lru_move_tail);
  * @nr_to_scan: The number of pages to try to reclaim
  * @remaining: The number of pages left to reclaim, should be initialized by caller
  * @shrink: Callback to try to shrink/reclaim the object.
- * @ticket: Optional ww_acquire_ctx context to use for locking
  */
 unsigned long
 drm_gem_lru_scan(struct drm_device *dev,
 		 struct drm_gem_lru *lru,
 		 unsigned int nr_to_scan,
 		 unsigned long *remaining,
-		 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
-		 struct ww_acquire_ctx *ticket)
+		 bool (*shrink)(struct drm_gem_object *obj))
 {
 	struct drm_gem_lru still_in_lru;
 	struct drm_gem_object *obj;
@@ -1689,20 +1690,17 @@ drm_gem_lru_scan(struct drm_device *dev,
 		 */
 		mutex_unlock(&dev->gem_lru_mutex);
 
-		if (ticket)
-			ww_acquire_init(ticket, &reservation_ww_class);
-
 		/*
 		 * Note that this still needs to be trylock, since we can
 		 * hit shrinker in response to trying to get backing pages
 		 * for this obj (ie. while it's lock is already held)
 		 */
-		if (!ww_mutex_trylock(&obj->resv->lock, ticket)) {
+		if (!ww_mutex_trylock(&obj->resv->lock, NULL)) {
 			*remaining += obj->size >> PAGE_SHIFT;
 			goto tail;
 		}
 
-		if (shrink(obj, ticket)) {
+		if (shrink(obj)) {
 			freed += obj->size >> PAGE_SHIFT;
 
 			/*
@@ -1726,9 +1724,6 @@ drm_gem_lru_scan(struct drm_device *dev,
 
 		dma_resv_unlock(obj->resv);
 
-		if (ticket)
-			ww_acquire_fini(ticket);
-
 tail:
 		drm_gem_object_put(obj);
 		mutex_lock(&dev->gem_lru_mutex);
diff --git a/drivers/gpu/drm/drm_gem_atomic_helper.c b/drivers/gpu/drm/drm_gem_atomic_helper.c
index abef865c5f2c9e..dd9efa7e32167b 100644
--- a/drivers/gpu/drm/drm_gem_atomic_helper.c
+++ b/drivers/gpu/drm/drm_gem_atomic_helper.c
@@ -180,6 +180,48 @@ EXPORT_SYMBOL_GPL(drm_gem_plane_helper_prepare_fb);
  * Shadow-buffered Planes
  */
 
+/**
+ * __drm_gem_shadow_plane_state_init - initializes shadow-buffered plane state
+ * @plane: the plane
+ * @shadow_plane_state: the shadow-buffered plane state
+ *
+ * This function initializes a pre-allocated shadow-buffered plane state.
+ * Helpful for drivers that subclass struct drm_shadow_plane_state.
+ */
+void __drm_gem_shadow_plane_state_init(struct drm_plane *plane,
+				       struct drm_shadow_plane_state *shadow_plane_state)
+{
+	__drm_atomic_helper_plane_state_init(&shadow_plane_state->base, plane);
+	drm_format_conv_state_init(&shadow_plane_state->fmtcnv_state);
+}
+EXPORT_SYMBOL(__drm_gem_shadow_plane_state_init);
+
+/**
+ * drm_gem_create_shadow_plane_state - creates a shadow-buffered plane state
+ * @plane: the plane
+ *
+ * This function implements struct &drm_plane_funcs.atomic_create_state for
+ * shadow-buffered planes. It assumes the new state to be of type
+ * struct drm_shadow_plane_state.
+ *
+ * Returns:
+ * A pointer to a new plane state on success, or an ERR_PTR()-encoded
+ * error code otherwise.
+ */
+struct drm_plane_state *drm_gem_create_shadow_plane_state(struct drm_plane *plane)
+{
+	struct drm_shadow_plane_state *shadow_plane_state;
+
+	shadow_plane_state = kzalloc_obj(*shadow_plane_state);
+	if (!shadow_plane_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_gem_shadow_plane_state_init(plane, shadow_plane_state);
+
+	return &shadow_plane_state->base;
+}
+EXPORT_SYMBOL(drm_gem_create_shadow_plane_state);
+
 /**
  * __drm_gem_duplicate_shadow_plane_state - duplicates shadow-buffered plane state
  * @plane: the plane
@@ -277,49 +319,6 @@ void drm_gem_destroy_shadow_plane_state(struct drm_plane *plane,
 }
 EXPORT_SYMBOL(drm_gem_destroy_shadow_plane_state);
 
-/**
- * __drm_gem_reset_shadow_plane - resets a shadow-buffered plane
- * @plane: the plane
- * @shadow_plane_state: the shadow-buffered plane state
- *
- * This function resets state for shadow-buffered planes. Helpful
- * for drivers that subclass struct drm_shadow_plane_state.
- */
-void __drm_gem_reset_shadow_plane(struct drm_plane *plane,
-				  struct drm_shadow_plane_state *shadow_plane_state)
-{
-	if (shadow_plane_state) {
-		__drm_atomic_helper_plane_reset(plane, &shadow_plane_state->base);
-		drm_format_conv_state_init(&shadow_plane_state->fmtcnv_state);
-	} else {
-		__drm_atomic_helper_plane_reset(plane, NULL);
-	}
-}
-EXPORT_SYMBOL(__drm_gem_reset_shadow_plane);
-
-/**
- * drm_gem_reset_shadow_plane - resets a shadow-buffered plane
- * @plane: the plane
- *
- * This function implements struct &drm_plane_funcs.reset_plane for
- * shadow-buffered planes. It assumes the current plane state to be
- * of type struct drm_shadow_plane and it allocates the new state of
- * this type.
- */
-void drm_gem_reset_shadow_plane(struct drm_plane *plane)
-{
-	struct drm_shadow_plane_state *shadow_plane_state;
-
-	if (plane->state) {
-		drm_gem_destroy_shadow_plane_state(plane, plane->state);
-		plane->state = NULL; /* must be set to NULL here */
-	}
-
-	shadow_plane_state = kzalloc_obj(*shadow_plane_state);
-	__drm_gem_reset_shadow_plane(plane, shadow_plane_state);
-}
-EXPORT_SYMBOL(drm_gem_reset_shadow_plane);
-
 /**
  * drm_gem_begin_shadow_fb_access - prepares shadow framebuffers for CPU access
  * @plane: the plane
@@ -408,17 +407,22 @@ void drm_gem_simple_kms_end_shadow_fb_access(struct drm_simple_display_pipe *pip
 EXPORT_SYMBOL(drm_gem_simple_kms_end_shadow_fb_access);
 
 /**
- * drm_gem_simple_kms_reset_shadow_plane - resets a shadow-buffered plane
+ * drm_gem_simple_kms_create_shadow_plane_state - creates shadow-buffered plane state
  * @pipe: the simple display pipe
  *
- * This function implements struct drm_simple_display_funcs.reset_plane
+ * This function implements struct drm_simple_display_pipe_funcs.create_plane_state
  * for shadow-buffered planes.
+ *
+ * Returns:
+ * A pointer to a new plane state on success, or an ERR_PTR()-encoded
+ * error code otherwise.
  */
-void drm_gem_simple_kms_reset_shadow_plane(struct drm_simple_display_pipe *pipe)
+struct drm_plane_state *
+drm_gem_simple_kms_create_shadow_plane_state(struct drm_simple_display_pipe *pipe)
 {
-	drm_gem_reset_shadow_plane(&pipe->plane);
+	return drm_gem_create_shadow_plane_state(&pipe->plane);
 }
-EXPORT_SYMBOL(drm_gem_simple_kms_reset_shadow_plane);
+EXPORT_SYMBOL(drm_gem_simple_kms_create_shadow_plane_state);
 
 /**
  * drm_gem_simple_kms_duplicate_shadow_plane_state - duplicates shadow-buffered plane state
diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c
index a93eee7ddb9e95..db3aa595fc116f 100644
--- a/drivers/gpu/drm/drm_gpusvm.c
+++ b/drivers/gpu/drm/drm_gpusvm.c
@@ -9,9 +9,9 @@
 #include <linux/dma-mapping.h>
 #include <linux/export.h>
 #include <linux/hmm.h>
-#include <linux/hugetlb_inline.h>
 #include <linux/memremap.h>
 #include <linux/mm_types.h>
+#include <linux/mm.h>
 #include <linux/slab.h>
 
 #include <drm/drm_device.h>
@@ -80,6 +80,13 @@
  *			};
  *		};
  *
+ *		static struct drm_gpusvm_pages *
+ *		driver_pages(struct driver_range *drange)
+ *		{
+ *			return drange->num_pages == 1 ? &drange->inline_pages :
+ *							drange->pages;
+ *		}
+ *
  *	In the N:1 case the driver allocates the pages array with a zeroing
  *	allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with
  *	drm_gpusvm_init_pages(), and frees each entry with
@@ -89,6 +96,28 @@
  *	Each drm_gpusvm_pages must be zero-initialised and initialised with
  *	drm_gpusvm_init_pages(), called once per entry.
  *
+ *	The 1:1 examples below pass @num_pages == 1 and &drange->pages. In the
+ *	N:1 case the driver instead passes the whole array and its count, so a
+ *	single call faults the CPU range once and DMA maps it for every owning
+ *	drm_device, e.g.:
+ *
+ *	.. code-block:: c
+ *
+ *		// GPU fault handler: one fault, one DMA mapping per device
+ *		err = drm_gpusvm_get_pages(gpusvm, driver_pages(drange),
+ *					   drange->num_pages, gpusvm->mm,
+ *					   &range->notifier->notifier,
+ *					   drm_gpusvm_range_start(range),
+ *					   drm_gpusvm_range_end(range), &ctx);
+ *
+ *		// Notifier callback: mark every instance unmapped in one call
+ *		drm_gpusvm_range_set_unmapped(range, driver_pages(drange),
+ *					      drange->num_pages, mmu_range);
+ *
+ *	The unmap and free paths stay per-instance: iterate @num_pages over
+ *	driver_pages(drange) and call drm_gpusvm_unmap_pages() /
+ *	drm_gpusvm_free_pages() for each entry.
+ *
  * - Operations:
  *	Define the interface for driver-specific GPU SVM operations such as
  *	range allocation, notifier allocation, and invalidations.
@@ -232,7 +261,7 @@
  *				goto retry;
  *		}
  *
- *		err = drm_gpusvm_get_pages(gpusvm, &drange->pages,
+ *		err = drm_gpusvm_get_pages(gpusvm, &drange->pages, 1,
  *					   gpusvm->mm, &range->notifier->notifier,
  *					   drm_gpusvm_range_start(range),
  *					   drm_gpusvm_range_end(range), &ctx);
@@ -1141,8 +1170,7 @@ drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
 	 * limitations. If/when migrate_vma_* add more support, this logic will
 	 * have to change.
 	 */
-	migrate_devmem = ctx->devmem_possible &&
-		vma_is_anonymous(vas) && !is_vm_hugetlb_page(vas);
+	migrate_devmem = ctx->devmem_possible && vma_is_anonymous(vas);
 
 	chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas,
 						 fault_addr, gpuva_start,
@@ -1212,6 +1240,8 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
 		struct drm_gpusvm_pages_flags flags = {
 			.__flags = svm_pages->flags.__flags,
 		};
+		const struct drm_pagemap_addr *addrs =
+			drm_gpusvm_pages_first_dma(svm_pages, NULL);
 		bool use_iova = dma_use_iova(&svm_pages->state);
 
 		/*
@@ -1224,12 +1254,20 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
 			if (svm_pages->state_offset)
 				dma_iova_unlink(dev, &svm_pages->state, 0,
 						svm_pages->state_offset,
-						svm_pages->dma_addr[0].dir, 0);
+						addrs[0].dir, 0);
 			dma_iova_free(dev, &svm_pages->state);
 		}
 
-		for (i = 0, j = 0; i < npages; j++) {
-			struct drm_pagemap_addr *addr = &svm_pages->dma_addr[j];
+		/*
+		 * With IOVA and no device page the unlink above tore every
+		 * entry down, and that is also when the range may be folded
+		 * to one entry, which must not be walked per entry. dpagemap
+		 * is set before the first device_map(), so it is also right
+		 * on the error path, where the flags are not published yet.
+		 */
+		for (i = 0, j = 0;
+		     (!use_iova || dpagemap) && i < npages; j++) {
+			const struct drm_pagemap_addr *addr = &addrs[j];
 
 			if (addr->proto == DRM_INTERCONNECT_SYSTEM) {
 				/*
@@ -1270,6 +1308,18 @@ static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
 {
 	lockdep_assert_held(&gpusvm->notifier_lock);
 
+	if (svm_pages->flags.inline_dma_mapping) {
+		struct drm_gpusvm_pages_flags flags = {
+			.__flags = svm_pages->flags.__flags,
+		};
+
+		svm_pages->inline_addr = (struct drm_pagemap_addr){};
+		flags.inline_dma_mapping = false;
+		/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
+		WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
+		return;
+	}
+
 	if (svm_pages->dma_addr) {
 		kvfree(svm_pages->dma_addr);
 		svm_pages->dma_addr = NULL;
@@ -1417,144 +1467,105 @@ EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid);
 /**
  * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked
  * @gpusvm: Pointer to the GPU SVM structure
- * @svm_pages: Pointer to the GPU SVM pages structure
+ * @svm_pages: Array of GPU SVM pages structures
+ * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages
  *
- * This function determines if a GPU SVM pages are valid. Expected be called
- * without holding gpusvm->notifier_lock.
+ * This function determines if every GPU SVM pages instance is valid, resetting
+ * every instance which is not so that get_pages() maps it afresh. It therefore
+ * has to walk them all. Expected be called without holding
+ * gpusvm->notifier_lock.
  *
- * Return: True if GPU SVM pages are valid, False otherwise
+ * Return: True if all GPU SVM pages are valid, False otherwise
  */
 static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm,
-					    struct drm_gpusvm_pages *svm_pages)
+					    struct drm_gpusvm_pages *svm_pages,
+					    unsigned int num_pages)
 {
-	bool pages_valid;
-
-	if (!svm_pages->dma_addr)
-		return false;
+	bool pages_valid = true;
+	unsigned int p;
 
 	drm_gpusvm_notifier_lock(gpusvm);
-	pages_valid = drm_gpusvm_pages_valid(gpusvm, svm_pages);
-	if (!pages_valid)
-		__drm_gpusvm_free_pages(gpusvm, svm_pages);
+	for (p = 0; p < num_pages; ++p) {
+		if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
+			continue;
+		__drm_gpusvm_free_pages(gpusvm, &svm_pages[p]);
+		pages_valid = false;
+	}
 	drm_gpusvm_notifier_unlock(gpusvm);
 
 	return pages_valid;
 }
 
 /**
- * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct
+ * drm_gpusvm_pages_inlinable() - Whether the dma address can be inlined
+ * @svm_pages: The SVM pages instance that was just mapped
+ * @nentries: Number of entries the mapping loop produced
+ * @npages: Number of pages in the CPU range
+ *
+ * A THP maps as one huge page, and an IOVA reservation links every page of
+ * the range at the next offset, so the device addresses run contiguously from
+ * entry 0. Either way one entry describes the whole range, so the dma_addr
+ * array can be freed and the address kept inline.
+ *
+ * state_offset advances only on the IOVA branch, so reaching the full range
+ * length proves no device page was mapped in between. Only single page
+ * entries fold, so the order kept is 0 and describes the range truthfully.
+ * Larger chunks, several huge pages among them, stay an array that is
+ * already short and that a consumer places with one PTE each.
+ *
+ * Return: True if the mapping fits in a single drm_pagemap_addr.
+ */
+static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages,
+				       unsigned long nentries,
+				       unsigned long npages)
+{
+	if (nentries == 1)
+		return true;
+
+	return nentries == npages && dma_use_iova(&svm_pages->state) &&
+	       svm_pages->state_offset == npages * PAGE_SIZE;
+}
+
+/**
+ * drm_gpusvm_dma_map_pages() - DMA map one drm_gpusvm_pages instance
  * @gpusvm: Pointer to the GPU SVM structure
- * @svm_pages: The SVM pages to populate. This will contain the dma-addresses
- * @mm: The mm corresponding to the CPU range
- * @notifier: The corresponding notifier for the given CPU range
- * @pages_start: Start CPU address for the pages
- * @pages_end: End CPU address for the pages (exclusive)
+ * @svm_pages: The SVM pages instance to populate with dma-addresses
+ * @pfns: The already-faulted pfn array (size @npages)
+ * @npages: Number of pages in the CPU range
  * @ctx: GPU SVM context
+ * @dma_dir: DMA data direction for the mappings
  *
- * This function gets and maps pages for CPU range and ensures they are
- * mapped for DMA access.
+ * Map the faulted @pfns into @svm_pages for DMA access through its owning
+ * drm_device. Must be called under the notifier lock and only for an instance
+ * without a live mapping. On failure this unwinds the partial mapping of this
+ * instance before returning.
  *
  * Return: 0 on success, negative error code on failure.
  */
-int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
-			 struct drm_gpusvm_pages *svm_pages,
-			 struct mm_struct *mm,
-			 struct mmu_interval_notifier *notifier,
-			 unsigned long pages_start, unsigned long pages_end,
-			 const struct drm_gpusvm_ctx *ctx)
+static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm,
+				    struct drm_gpusvm_pages *svm_pages,
+				    unsigned long *pfns,
+				    unsigned long npages,
+				    const struct drm_gpusvm_ctx *ctx,
+				    enum dma_data_direction dma_dir)
 {
-	struct hmm_range hmm_range = {
-		.default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 :
-			HMM_PFN_REQ_WRITE),
-		.notifier = notifier,
-		.start = pages_start,
-		.end = pages_end,
-		.dev_private_owner = ctx->device_private_page_owner,
-	};
-	void *zdd;
-	unsigned long timeout =
-		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
-	unsigned long remaining;
+	void *zdd = NULL;
 	unsigned long i, j;
-	unsigned long npages = npages_in_range(pages_start, pages_end);
-	unsigned long num_dma_mapped;
+	unsigned long num_dma_mapped = 0;
 	unsigned int order = 0;
-	unsigned long *pfns;
 	int err = 0;
-	struct dev_pagemap *pagemap;
+	struct dev_pagemap *pagemap = NULL;
 	struct drm_pagemap *dpagemap;
 	struct drm_gpusvm_pages_flags flags;
-	enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE :
-							   DMA_BIDIRECTIONAL;
 	struct dma_iova_state *state = &svm_pages->state;
 
-	if (!svm_pages->drm)
-		return -EINVAL;
-
-retry:
-	remaining = timeout - jiffies;
-
-	if (time_after_eq(jiffies, timeout))
-		return -EBUSY;
-
-	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
-	if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))
-		goto set_seqno;
-
-	pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
-	if (!pfns)
-		return -ENOMEM;
-
-	if (!mmget_not_zero(mm)) {
-		err = -EFAULT;
-		goto err_free;
-	}
-
-	hmm_range.hmm_pfns = pfns;
-	err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining);
-	mmput(mm);
-	if (err)
-		goto err_free;
+	lockdep_assert_held(&gpusvm->notifier_lock);
 
 	*state = (struct dma_iova_state){};
 	svm_pages->state_offset = 0;
 
-map_pages:
-	/*
-	 * Perform all dma mappings under the notifier lock to not
-	 * access freed pages. A notifier will either block on
-	 * the notifier lock or unmap dma.
-	 */
-	drm_gpusvm_notifier_lock(gpusvm);
-
 	flags.__flags = svm_pages->flags.__flags;
-	if (flags.unmapped) {
-		drm_gpusvm_notifier_unlock(gpusvm);
-		err = -EFAULT;
-		goto err_free;
-	}
-
-	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
-		drm_gpusvm_notifier_unlock(gpusvm);
-		kvfree(pfns);
-		goto retry;
-	}
-
-	if (!svm_pages->dma_addr) {
-		/* Unlock and restart mapping to allocate memory. */
-		drm_gpusvm_notifier_unlock(gpusvm);
-		svm_pages->dma_addr =
-			kvzalloc_objs(*svm_pages->dma_addr, npages);
-		if (!svm_pages->dma_addr) {
-			err = -ENOMEM;
-			goto err_free;
-		}
-		goto map_pages;
-	}
 
-	zdd = NULL;
-	pagemap = NULL;
-	num_dma_mapped = 0;
 	for (i = 0, j = 0; i < npages; ++j) {
 		struct page *page = hmm_pfn_to_page(pfns[i]);
 
@@ -1667,20 +1678,186 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 	if (pagemap)
 		flags.has_devmem_pages = true;
 
+	if (drm_gpusvm_pages_inlinable(svm_pages, j, npages)) {
+		struct drm_pagemap_addr addr = svm_pages->dma_addr[0];
+
+		kvfree(svm_pages->dma_addr);
+		svm_pages->inline_addr = addr;
+		flags.inline_dma_mapping = true;
+	}
+
 	/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
 	WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
 
-	drm_gpusvm_notifier_unlock(gpusvm);
-	kvfree(pfns);
-set_seqno:
-	svm_pages->notifier_seq = hmm_range.notifier_seq;
-
 	return 0;
 
 err_unmap:
 	svm_pages->flags.has_dma_mapping = true;
 	__drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped);
+	return err;
+}
+
+/**
+ * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct
+ * @gpusvm: Pointer to the GPU SVM structure
+ * @svm_pages: Array of SVM pages instances to populate with dma addresses
+ * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages, must not be 0
+ * @mm: The mm corresponding to the CPU range
+ * @notifier: The corresponding notifier for the given CPU range
+ * @pages_start: Start CPU address for the pages
+ * @pages_end: End CPU address for the pages (exclusive)
+ * @ctx: GPU SVM context
+ *
+ * This function gets and maps pages for a CPU range and ensures they are
+ * mapped for DMA access. The HMM fault for the CPU range is performed once,
+ * the DMA mapping by drm_gpusvm_dma_map_pages() is then done per instance,
+ * one per owning drm_device. The retry against notifier races is kept here
+ * in common code so drivers never open code it.
+ *
+ * On error the instances mapped before the failing one stay mapped, so the
+ * caller must unmap and free every instance regardless of the return value.
+ *
+ * With &drm_gpusvm_ctx.no_dma_map no mapping state is recorded, so
+ * drm_gpusvm_pages_valid() never returns true and success is only a snapshot:
+ * the caller must recheck mmu_interval_read_retry() against the recorded
+ * &drm_gpusvm_pages.notifier_seq under the notifier lock, and hold it until
+ * its work is visible to invalidation.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
+			 struct drm_gpusvm_pages *svm_pages,
+			 unsigned int num_pages,
+			 struct mm_struct *mm,
+			 struct mmu_interval_notifier *notifier,
+			 unsigned long pages_start, unsigned long pages_end,
+			 const struct drm_gpusvm_ctx *ctx)
+{
+	struct hmm_range hmm_range = {
+		.default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 :
+			HMM_PFN_REQ_WRITE),
+		.notifier = notifier,
+		.start = pages_start,
+		.end = pages_end,
+		.dev_private_owner = ctx->device_private_page_owner,
+	};
+	unsigned long timeout =
+		jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
+	unsigned long remaining;
+	unsigned long npages = npages_in_range(pages_start, pages_end);
+	unsigned long *pfns;
+	int err = 0;
+	enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE :
+							   DMA_BIDIRECTIONAL;
+	const bool map_dma = !ctx->no_dma_map;
+	unsigned int p;
+
+	if (!num_pages)
+		return -EINVAL;
+
+	if (ctx->no_dma_map && ctx->devmem_only)
+		return -EINVAL;
+
+	if (map_dma) {
+		for (p = 0; p < num_pages; ++p) {
+			if (!svm_pages[p].drm)
+				return -EINVAL;
+		}
+	}
+
+retry:
+	remaining = timeout - jiffies;
+
+	if (time_after_eq(jiffies, timeout))
+		return -EBUSY;
+
+	hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
+
+	if (map_dma &&
+	    drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages, num_pages))
+		goto set_seqno;
+
+	pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
+	if (!pfns)
+		return -ENOMEM;
+
+	if (!mmget_not_zero(mm)) {
+		err = -EFAULT;
+		goto err_free;
+	}
+
+	hmm_range.hmm_pfns = pfns;
+	err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining);
+	mmput(mm);
+	if (err)
+		goto err_free;
+
+	if (map_dma) {
+		for (p = 0; p < num_pages; ++p) {
+			if (drm_gpusvm_pages_first_dma(&svm_pages[p], NULL))
+				continue;
+			svm_pages[p].dma_addr =
+				kvzalloc_objs(*svm_pages[p].dma_addr, npages);
+			if (!svm_pages[p].dma_addr) {
+				err = -ENOMEM;
+				goto err_free;
+			}
+		}
+	}
+
+	/*
+	 * Perform all dma mappings under the notifier lock to not
+	 * access freed pages. A notifier will either block on
+	 * the notifier lock or unmap dma.
+	 */
+	drm_gpusvm_notifier_lock(gpusvm);
+
+	/*
+	 * drm_gpusvm_range_set_unmapped() flags the whole array in one go under
+	 * the write lock, so any instance answers for all of them here.
+	 */
+	if (svm_pages[0].flags.unmapped) {
+		drm_gpusvm_notifier_unlock(gpusvm);
+		err = -EFAULT;
+		goto err_free;
+	}
+
+	if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
+		drm_gpusvm_notifier_unlock(gpusvm);
+		kvfree(pfns);
+		goto retry;
+	}
+
+	if (!map_dma)
+		goto done_mapping;
+
+	for (p = 0; p < num_pages; ++p) {
+		if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p]))
+			continue;
+
+		err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns,
+					       npages, ctx, dma_dir);
+		if (err) {
+			/*
+			 * The failing instance was unwound by the helper. Keep
+			 * the ones mapped earlier: the -EAGAIN retry reuses
+			 * them, and the driver unmaps every instance with the
+			 * range on the other error paths.
+			 */
+			drm_gpusvm_notifier_unlock(gpusvm);
+			goto err_free;
+		}
+	}
+
+done_mapping:
 	drm_gpusvm_notifier_unlock(gpusvm);
+	kvfree(pfns);
+set_seqno:
+	for (p = 0; p < num_pages; ++p)
+		svm_pages[p].notifier_seq = hmm_range.notifier_seq;
+
+	return 0;
+
 err_free:
 	kvfree(pfns);
 	if (err == -EAGAIN)
diff --git a/drivers/gpu/drm/drm_kms_helper_common.c b/drivers/gpu/drm/drm_kms_helper_common.c
index 0c7550c0462b5f..b5d5b469b4443c 100644
--- a/drivers/gpu/drm/drm_kms_helper_common.c
+++ b/drivers/gpu/drm/drm_kms_helper_common.c
@@ -27,6 +27,20 @@
 
 #include <linux/module.h>
 
+#include "drm_panic_internal.h"
+
+static int __init drm_kms_helper_init(void)
+{
+	return drm_panic_helper_init();
+}
+module_init(drm_kms_helper_init);
+
+static void __exit drm_kms_helper_exit(void)
+{
+	drm_panic_helper_exit();
+}
+module_exit(drm_kms_helper_exit);
+
 MODULE_AUTHOR("David Airlie, Jesse Barnes");
 MODULE_DESCRIPTION("DRM KMS helper");
 MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index 366f6d8212425e..48a56f23dadbb2 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -30,6 +30,7 @@
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mode_config.h>
+#include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_colorop.h>
 #include <linux/dma-resv.h>
@@ -230,12 +231,22 @@ static int drm_mode_config_crtc_create_state(struct drm_crtc *crtc)
 
 static int drm_mode_config_crtc_reset_with_create_state(struct drm_crtc *crtc)
 {
+	const struct drm_crtc_helper_funcs *helper_funcs = crtc->helper_private;
+	int ret;
+
 	if (crtc->state) {
 		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
 		crtc->state = NULL;
 	}
 
-	return drm_mode_config_crtc_create_state(crtc);
+	ret = drm_mode_config_crtc_create_state(crtc);
+	if (ret)
+		return ret;
+
+	if (helper_funcs && helper_funcs->hw_reset)
+		helper_funcs->hw_reset(crtc);
+
+	return 0;
 }
 
 static int drm_mode_config_connector_create_state(struct drm_connector *connector)
@@ -289,16 +300,12 @@ void drm_mode_config_reset(struct drm_device *dev)
 		drm_colorop_reset(colorop);
 
 	drm_for_each_plane(plane, dev) {
-		if (plane->funcs->reset)
-			plane->funcs->reset(plane);
-		else if (plane->funcs->atomic_create_state)
+		if (plane->funcs->atomic_create_state)
 			drm_mode_config_plane_reset_with_create_state(plane);
 	}
 
 	drm_for_each_crtc(crtc, dev) {
-		if (crtc->funcs->reset)
-			crtc->funcs->reset(crtc);
-		else if (crtc->funcs->atomic_create_state)
+		if (crtc->funcs->atomic_create_state)
 			drm_mode_config_crtc_reset_with_create_state(crtc);
 	}
 
diff --git a/drivers/gpu/drm/drm_modeset_helper.c b/drivers/gpu/drm/drm_modeset_helper.c
index e54584da4c3d16..cca623eb23e53b 100644
--- a/drivers/gpu/drm/drm_modeset_helper.c
+++ b/drivers/gpu/drm/drm_modeset_helper.c
@@ -27,6 +27,7 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_modeset_helper.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -113,6 +114,7 @@ static const uint32_t safe_modeset_formats[] = {
 
 static const struct drm_plane_funcs primary_plane_funcs = {
 	DRM_PLANE_NON_ATOMIC_FUNCS,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 /**
diff --git a/drivers/gpu/drm/drm_of.c b/drivers/gpu/drm/drm_of.c
index e495117735e1f0..411f258d9f9163 100644
--- a/drivers/gpu/drm/drm_of.c
+++ b/drivers/gpu/drm/drm_of.c
@@ -264,73 +264,6 @@ int drm_of_get_panel_orientation(const struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(drm_of_get_panel_orientation);
 
-/**
- * drm_of_find_panel_or_bridge - return connected panel or bridge device
- * @np: device tree node containing encoder output ports
- * @port: port in the device tree node
- * @endpoint: endpoint in the device tree node
- * @panel: pointer to hold returned drm_panel, must not be NULL. On success
- *         the caller must call drm_panel_put() when done with the panel
- * @bridge: pointer to hold returned drm_bridge
- *
- * Given a DT node's port and endpoint number, find the connected node and
- * return either the associated struct drm_panel or drm_bridge device.
- *
- * This function is deprecated and should not be used in new drivers. Use
- * of_drm_get_bridge_by_endpoint() instead when not looking for a panel, or
- * devm_drm_of_get_bridge() otherwise.
- *
- * Returns zero if successful, or one of the standard error codes if it fails.
- */
-int drm_of_find_panel_or_bridge(const struct device_node *np,
-				int port, int endpoint,
-				struct drm_panel **panel,
-				struct drm_bridge **bridge)
-{
-	int ret = -EPROBE_DEFER;
-	struct device_node *remote;
-
-	if (WARN_ON(!panel))
-		return -EINVAL;
-
-	*panel = NULL;
-
-	/*
-	 * of_graph_get_remote_node() produces a noisy error message if port
-	 * node isn't found and the absence of the port is a legit case here,
-	 * so at first we silently check whether a graph is present in the
-	 * device-tree node.
-	 */
-	if (!of_graph_is_present(np))
-		return -ENODEV;
-
-	remote = of_graph_get_remote_node(np, port, endpoint);
-	if (!remote)
-		return -ENODEV;
-
-	*panel = of_drm_find_panel(remote);
-	if (!IS_ERR(*panel))
-		ret = 0;
-	else
-		*panel = NULL;
-
-	if (bridge) {
-		if (ret) {
-			/* No panel found yet, check for a bridge next. */
-			*bridge = of_drm_find_bridge(remote);
-			if (*bridge)
-				ret = 0;
-		} else {
-			*bridge = NULL;
-		}
-
-	}
-
-	of_node_put(remote);
-	return ret;
-}
-EXPORT_SYMBOL_GPL(drm_of_find_panel_or_bridge);
-
 enum drm_of_lvds_pixels {
 	DRM_OF_LVDS_EVEN = BIT(0),
 	DRM_OF_LVDS_ODD = BIT(1),
@@ -550,8 +483,8 @@ EXPORT_SYMBOL_GPL(drm_of_lvds_get_data_mapping);
  *
  * Return:
  * * min..max - positive integer count of "data-lanes" elements
- * * -ve - the "data-lanes" property is missing or invalid
- * * -EINVAL - the "data-lanes" property is unsupported
+ * * -ENODATA - the property does not have a value.
+ * * -EINVAL - the "data-lanes" property is missing or invalid
  */
 int drm_of_get_data_lanes_count(const struct device_node *endpoint,
 				const unsigned int min, const unsigned int max)
@@ -582,8 +515,8 @@ EXPORT_SYMBOL_GPL(drm_of_get_data_lanes_count);
  *
  * Return:
  * * min..max - positive integer count of "data-lanes" elements
- * * -EINVAL - the "data-mapping" property is unsupported
- * * -ENODEV - the "data-mapping" property is missing
+ * * -ENODATA - the property does not have a value.
+ * * -EINVAL - the "data-lanes" property is missing or invalid
  */
 int drm_of_get_data_lanes_count_ep(const struct device_node *port,
 				   int port_reg, int reg,
diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c
index a0546955d0b9f7..743988036a5f84 100644
--- a/drivers/gpu/drm/drm_pagemap.c
+++ b/drivers/gpu/drm/drm_pagemap.c
@@ -1079,7 +1079,7 @@ static void drm_pagemap_release(struct kref *ref)
 	dpagemap->dev_hold = NULL;
 	drm_pagemap_shrinker_add(dpagemap);
 	llist_add(&dev_hold->link, &drm_pagemap_unhold_list);
-	schedule_work(&drm_pagemap_work);
+	queue_work(system_dfl_wq, &drm_pagemap_work);
 	/*
 	 * Here, either the provider device is still alive, since if called from
 	 * page_free(), the caller is holding a reference on the dev_pagemap,
@@ -1106,6 +1106,7 @@ static void drm_pagemap_dev_unhold_work(struct work_struct *work)
 		drm_dev_put(drm);
 		module_put(module);
 		kfree(dev_hold);
+		cond_resched();
 	}
 }
 
diff --git a/drivers/gpu/drm/drm_panel.c b/drivers/gpu/drm/drm_panel.c
index f8f6082e637fd8..c00529bfb706ba 100644
--- a/drivers/gpu/drm/drm_panel.c
+++ b/drivers/gpu/drm/drm_panel.c
@@ -1,36 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * Copyright (C) 2013, NVIDIA Corporation.  All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
+ * Copyright (C) 2016 Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+ * Copyright (C) 2017 Broadcom
  */
 
 #include <linux/backlight.h>
+#include <linux/debugfs.h>
 #include <linux/err.h>
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/of.h>
 
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_connector.h>
 #include <drm/drm_crtc.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
 
 static DEFINE_MUTEX(panel_lock);
 static LIST_HEAD(panel_list);
@@ -46,6 +38,526 @@ static LIST_HEAD(panel_list);
  * take look at drm_panel_bridge_add() and devm_drm_panel_bridge_add().
  */
 
+static inline struct drm_panel *
+drm_bridge_to_panel(const struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct drm_panel, bridge);
+}
+
+static inline struct drm_panel *
+drm_connector_to_panel(const struct drm_connector *connector)
+{
+	return container_of(connector, struct drm_panel, connector);
+}
+
+struct panel_bridge {
+	struct drm_bridge bridge;
+	struct drm_connector connector;
+	struct drm_panel *panel;
+	u32 connector_type;
+};
+
+static int panel_bridge_connector_get_modes(struct drm_connector *connector)
+{
+	struct drm_panel *panel = drm_connector_to_panel(connector);
+
+	return drm_panel_get_modes(panel, connector);
+}
+
+static const struct drm_connector_helper_funcs
+panel_bridge_connector_helper_funcs = {
+	.get_modes = panel_bridge_connector_get_modes,
+};
+
+static const struct drm_connector_funcs panel_bridge_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int panel_bridge_attach(struct drm_bridge *bridge,
+			       struct drm_encoder *encoder,
+			       enum drm_bridge_attach_flags flags)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+	struct drm_connector *connector = &panel->connector;
+	int ret;
+
+	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
+		return 0;
+
+	drm_connector_helper_add(connector,
+				 &panel_bridge_connector_helper_funcs);
+
+	ret = drm_connector_init(bridge->dev, connector,
+				 &panel_bridge_connector_funcs,
+				 panel->connector_type);
+	if (ret) {
+		DRM_ERROR("Failed to initialize connector\n");
+		return ret;
+	}
+
+	drm_panel_bridge_set_orientation(connector, bridge);
+
+	drm_connector_attach_encoder(connector, encoder);
+
+	if (bridge->dev->registered) {
+		if (connector->funcs->reset)
+			connector->funcs->reset(connector);
+		drm_connector_register(connector);
+	}
+
+	return 0;
+}
+
+static void panel_bridge_detach(struct drm_bridge *bridge)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+	struct drm_connector *connector = &panel->connector;
+
+	/* Cleanup the connector if we know it was initialized */
+	if (connector->dev)
+		drm_connector_cleanup(connector);
+}
+
+static void panel_bridge_atomic_pre_enable(struct drm_bridge *bridge,
+					   struct drm_atomic_commit *atomic_state)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+
+	crtc = drm_atomic_get_new_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
+	if (old_crtc_state && old_crtc_state->self_refresh_active)
+		return;
+
+	drm_panel_prepare(panel);
+}
+
+static void panel_bridge_atomic_enable(struct drm_bridge *bridge,
+				       struct drm_atomic_commit *atomic_state)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state;
+
+	crtc = drm_atomic_get_new_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	old_crtc_state = drm_atomic_get_old_crtc_state(atomic_state, crtc);
+	if (old_crtc_state && old_crtc_state->self_refresh_active)
+		return;
+
+	drm_panel_enable(panel);
+}
+
+static void panel_bridge_atomic_disable(struct drm_bridge *bridge,
+					struct drm_atomic_commit *atomic_state)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+
+	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
+	if (new_crtc_state && new_crtc_state->self_refresh_active)
+		return;
+
+	drm_panel_disable(panel);
+}
+
+static void panel_bridge_atomic_post_disable(struct drm_bridge *bridge,
+					     struct drm_atomic_commit *atomic_state)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+	struct drm_encoder *encoder = bridge->encoder;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *new_crtc_state;
+
+	crtc = drm_atomic_get_old_crtc_for_encoder(atomic_state, encoder);
+	if (!crtc)
+		return;
+
+	new_crtc_state = drm_atomic_get_new_crtc_state(atomic_state, crtc);
+	if (new_crtc_state && new_crtc_state->self_refresh_active)
+		return;
+
+	drm_panel_unprepare(panel);
+}
+
+static int panel_bridge_get_modes(struct drm_bridge *bridge,
+				  struct drm_connector *connector)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+
+	return drm_panel_get_modes(panel, connector);
+}
+
+static void panel_bridge_debugfs_init(struct drm_bridge *bridge,
+				      struct dentry *root)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+
+	root = debugfs_create_dir("panel", root);
+	if (panel->funcs->debugfs_init)
+		panel->funcs->debugfs_init(panel, root);
+}
+
+static const struct drm_bridge_funcs panel_bridge_bridge_funcs = {
+	.attach = panel_bridge_attach,
+	.detach = panel_bridge_detach,
+	.atomic_pre_enable = panel_bridge_atomic_pre_enable,
+	.atomic_enable = panel_bridge_atomic_enable,
+	.atomic_disable = panel_bridge_atomic_disable,
+	.atomic_post_disable = panel_bridge_atomic_post_disable,
+	.get_modes = panel_bridge_get_modes,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_get_input_bus_fmts = drm_atomic_helper_bridge_propagate_bus_fmt,
+	.debugfs_init = panel_bridge_debugfs_init,
+};
+
+/**
+ * drm_bridge_is_panel - Checks if a drm_bridge is a panel_bridge.
+ *
+ * @bridge: The drm_bridge to be checked.
+ *
+ * Returns true if the bridge is a panel bridge, or false otherwise.
+ */
+bool drm_bridge_is_panel(const struct drm_bridge *bridge)
+{
+	return bridge->funcs == &panel_bridge_bridge_funcs;
+}
+EXPORT_SYMBOL(drm_bridge_is_panel);
+
+/**
+ * drm_panel_bridge_add - Creates a &drm_bridge and &drm_connector that
+ * just calls the appropriate functions from &drm_panel.
+ *
+ * @panel: The drm_panel being wrapped.  Must be non-NULL.
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel.
+ * Don't use it in new drivers and update your driver to stop using it.
+ *
+ * For drivers converting from directly using drm_panel: The expected
+ * usage pattern is that during either encoder module probe or DSI
+ * host attach, a drm_panel will be looked up through
+ * drm_of_find_panel_or_bridge().  drm_panel_bridge_add() is used to
+ * wrap that panel in the new bridge, and the result can then be
+ * passed to drm_bridge_attach().  The drm_panel_prepare() and related
+ * functions can be dropped from the encoder driver (they're now
+ * called by the KMS helpers before calling into the encoder), along
+ * with connector creation.  When done with the bridge (after
+ * drm_mode_config_cleanup() if the bridge has already been attached), then
+ * drm_panel_bridge_remove() to free it.
+ *
+ * The connector type is set to @panel->connector_type, which must be set to a
+ * known type. Calling this function with a panel whose connector type is
+ * DRM_MODE_CONNECTOR_Unknown will return ERR_PTR(-EINVAL).
+ *
+ * See devm_drm_panel_bridge_add() for an automatically managed version of this
+ * function.
+ */
+struct drm_bridge *drm_panel_bridge_add(struct drm_panel *panel)
+{
+	if (WARN_ON(panel->connector_type == DRM_MODE_CONNECTOR_Unknown))
+		return ERR_PTR(-EINVAL);
+
+	return drm_panel_bridge_add_typed(panel, panel->connector_type);
+}
+EXPORT_SYMBOL(drm_panel_bridge_add);
+
+/**
+ * drm_panel_bridge_add_typed - Pretend to create a &drm_bridge and &drm_connector with
+ * an explicit connector type.
+ * @panel: The drm_panel being wrapped.  Must be non-NULL.
+ * @connector_type: The connector type (DRM_MODE_CONNECTOR_*)
+ *
+ * This is just like drm_panel_bridge_add(), but forces the connector type to
+ * @connector_type instead of infering it from the panel.
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel.
+ * Don't use it in new drivers and update your driver to stop using it.
+ */
+struct drm_bridge *drm_panel_bridge_add_typed(struct drm_panel *panel,
+					      u32 connector_type)
+{
+	if (!panel)
+		return ERR_PTR(-EINVAL);
+
+	return drm_bridge_get(&panel->bridge);
+}
+EXPORT_SYMBOL(drm_panel_bridge_add_typed);
+
+/**
+ * drm_panel_bridge_remove - Unregisters and frees a drm_bridge
+ * created by drm_panel_bridge_add().
+ *
+ * @bridge: The drm_bridge being freed.
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel
+ * and removed when the panel is removed.  Don't use it in new drivers and
+ * update your driver to stop using it.
+ */
+void drm_panel_bridge_remove(struct drm_bridge *bridge)
+{
+	if (!bridge)
+		return;
+
+	if (!drm_bridge_is_panel(bridge)) {
+		drm_warn(bridge->dev, "%s: called on non-panel bridge!\n", __func__);
+		return;
+	}
+
+	drm_bridge_put(bridge);
+}
+EXPORT_SYMBOL(drm_panel_bridge_remove);
+
+/**
+ * drm_panel_bridge_set_orientation - Set the connector's panel orientation
+ * from the bridge that can be transformed to panel bridge.
+ *
+ * @connector: The connector to be set panel orientation.
+ * @bridge: The drm_bridge whose orientation should be set.
+ *
+ * Returns 0 on success, negative errno on failure.
+ */
+int drm_panel_bridge_set_orientation(struct drm_connector *connector,
+				     struct drm_bridge *bridge)
+{
+	struct drm_panel *panel = drm_bridge_to_panel(bridge);
+
+	return drm_connector_set_orientation_from_panel(connector, panel);
+}
+EXPORT_SYMBOL(drm_panel_bridge_set_orientation);
+
+static void devm_drm_panel_bridge_release(struct device *dev, void *res)
+{
+	struct drm_bridge *bridge = *(struct drm_bridge **)res;
+
+	if (!bridge)
+		return;
+
+	drm_bridge_put(bridge);
+}
+
+/**
+ * devm_drm_panel_bridge_add - Creates a managed &drm_bridge and &drm_connector
+ * that just calls the appropriate functions from &drm_panel.
+ * @dev: device to tie the bridge lifetime to
+ * @panel: The drm_panel being wrapped.  Must be non-NULL.
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel.
+ * Don't use it in new drivers and update your driver to stop using it.
+ *
+ * This is the managed version of drm_panel_bridge_add() which automatically
+ * calls drm_panel_bridge_remove() when @dev is unbound.
+ */
+struct drm_bridge *devm_drm_panel_bridge_add(struct device *dev,
+					     struct drm_panel *panel)
+{
+	if (WARN_ON(panel->connector_type == DRM_MODE_CONNECTOR_Unknown))
+		return ERR_PTR(-EINVAL);
+
+	return devm_drm_panel_bridge_add_typed(dev, panel,
+					       panel->connector_type);
+}
+EXPORT_SYMBOL(devm_drm_panel_bridge_add);
+
+/**
+ * devm_drm_panel_bridge_add_typed - Creates a managed &drm_bridge and
+ * &drm_connector with an explicit connector type.
+ * @dev: device to tie the bridge lifetime to
+ * @panel: The drm_panel being wrapped.  Must be non-NULL.
+ * @connector_type: The connector type (DRM_MODE_CONNECTOR_*)
+ *
+ * This is just like devm_drm_panel_bridge_add(), but forces the connector type
+ * to @connector_type instead of infering it from the panel.
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel.
+ * Don't use it in new drivers and update your driver to stop using it.
+ */
+struct drm_bridge *devm_drm_panel_bridge_add_typed(struct device *dev,
+						   struct drm_panel *panel,
+						   u32 connector_type)
+{
+	struct drm_bridge **ptr, *bridge;
+
+	ptr = devres_alloc(devm_drm_panel_bridge_release, sizeof(*ptr),
+			   GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	bridge = drm_panel_bridge_add_typed(panel, connector_type);
+	if (IS_ERR(bridge)) {
+		devres_free(ptr);
+		return bridge;
+	}
+
+	*ptr = bridge;
+	devres_add(dev, ptr);
+
+	return &panel->bridge;
+}
+EXPORT_SYMBOL(devm_drm_panel_bridge_add_typed);
+
+static void drmm_drm_panel_bridge_release(struct drm_device *drm, void *ptr)
+{
+	struct drm_bridge *bridge = ptr;
+
+	drm_panel_bridge_remove(bridge);
+}
+
+/**
+ * drmm_panel_bridge_add - Creates a DRM-managed &drm_bridge and
+ *                         &drm_connector that just calls the
+ *                         appropriate functions from &drm_panel.
+ *
+ * @drm: DRM device to tie the bridge lifetime to
+ * @panel: The drm_panel being wrapped.  Must be non-NULL.
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel.
+ * Don't use it in new drivers and update your driver to stop using it.
+ *
+ * This is the DRM-managed version of drm_panel_bridge_add() which
+ * automatically calls drm_panel_bridge_remove() when @dev is cleaned
+ * up.
+ */
+struct drm_bridge *drmm_panel_bridge_add(struct drm_device *drm,
+					 struct drm_panel *panel)
+{
+	struct drm_bridge *bridge;
+	int ret;
+
+	bridge = drm_panel_bridge_add_typed(panel, panel->connector_type);
+	if (IS_ERR(bridge))
+		return bridge;
+
+	ret = drmm_add_action_or_reset(drm, drmm_drm_panel_bridge_release,
+				       bridge);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return bridge;
+}
+EXPORT_SYMBOL(drmm_panel_bridge_add);
+
+/**
+ * drm_panel_bridge_connector - return the connector for the panel (for
+ * legacy drivers not using DRM_BRIDGE_ATTACH_NO_CONNECTOR)
+ * @bridge: The drm_bridge.
+ *
+ * This function gives external access to the connector.
+ *
+ * Returns: Pointer to drm_connector
+ */
+struct drm_connector *drm_panel_bridge_connector(struct drm_bridge *bridge)
+{
+	struct drm_panel *panel;
+
+	panel = drm_bridge_to_panel(bridge);
+
+	return &panel->connector;
+}
+EXPORT_SYMBOL(drm_panel_bridge_connector);
+
+#ifdef CONFIG_OF
+/**
+ * devm_drm_of_get_bridge - Return next bridge in the chain
+ * @dev: device to tie the bridge lifetime to
+ * @np: device tree node containing encoder output ports
+ * @port: port in the device tree node
+ * @endpoint: endpoint in the device tree node
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel.
+ * Don't use it in new drivers and convert your driver to
+ * of_drm_get_bridge_by_endpoint().
+ *
+ * Given a DT node's port and endpoint number, finds the connected node
+ * and returns the associated bridge if any, or creates and returns a
+ * drm panel bridge instance if a panel is connected.
+ *
+ * Returns a pointer to the bridge if successful, or an error pointer
+ * otherwise.
+ */
+struct drm_bridge *devm_drm_of_get_bridge(struct device *dev,
+					  struct device_node *np,
+					  u32 port, u32 endpoint)
+{
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
+	int ret;
+
+	ret = drm_of_find_panel_or_bridge(np, port, endpoint,
+					  &panel, &bridge);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (panel) {
+		bridge = devm_drm_panel_bridge_add(dev, panel);
+		drm_panel_put(panel);
+	}
+
+	return bridge;
+}
+EXPORT_SYMBOL(devm_drm_of_get_bridge);
+
+/**
+ * drmm_of_get_bridge - Return next bridge in the chain
+ * @drm: device to tie the bridge lifetime to
+ * @np: device tree node containing encoder output ports
+ * @port: port in the device tree node
+ * @endpoint: endpoint in the device tree node
+ *
+ * This function is deprecated. A drm_bridge is embedded into every panel.
+ * Don't use it in new drivers and convert your driver to
+ * of_drm_get_bridge_by_endpoint().
+ *
+ * Given a DT node's port and endpoint number, finds the connected node
+ * and returns the associated bridge if any, or creates and returns a
+ * drm panel bridge instance if a panel is connected.
+ *
+ * Returns a drmm managed pointer to the bridge if successful, or an error
+ * pointer otherwise.
+ */
+struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm,
+				      struct device_node *np,
+				      u32 port, u32 endpoint)
+{
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
+	int ret;
+
+	ret = drm_of_find_panel_or_bridge(np, port, endpoint,
+					  &panel, &bridge);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (panel) {
+		bridge = drmm_panel_bridge_add(drm, panel);
+		drm_panel_put(panel);
+	}
+
+	return bridge;
+}
+EXPORT_SYMBOL(drmm_of_get_bridge);
+
+#endif
+
 /**
  * drm_panel_init - initialize a panel
  * @panel: DRM panel
@@ -86,6 +598,13 @@ void drm_panel_add(struct drm_panel *panel)
 	mutex_lock(&panel_lock);
 	list_add_tail(&panel->list, &panel_list);
 	mutex_unlock(&panel_lock);
+
+	panel->bridge.of_node = panel->dev->of_node;
+	panel->bridge.ops = DRM_BRIDGE_OP_MODES;
+	panel->bridge.type = panel->connector_type;
+	panel->bridge.pre_enable_prev_first = panel->prepare_prev_first;
+
+	drm_bridge_add(&panel->bridge);
 }
 EXPORT_SYMBOL(drm_panel_add);
 
@@ -97,6 +616,7 @@ EXPORT_SYMBOL(drm_panel_add);
  */
 void drm_panel_remove(struct drm_panel *panel)
 {
+	drm_bridge_remove(&panel->bridge);
 	mutex_lock(&panel_lock);
 	list_del_init(&panel->list);
 	mutex_unlock(&panel_lock);
@@ -370,13 +890,6 @@ int drm_panel_get_modes(struct drm_panel *panel,
 }
 EXPORT_SYMBOL(drm_panel_get_modes);
 
-static void __drm_panel_free(struct kref *kref)
-{
-	struct drm_panel *panel = container_of(kref, struct drm_panel, refcount);
-
-	kfree(panel->container);
-}
-
 /**
  * drm_panel_get - Acquire a panel reference
  * @panel: DRM panel
@@ -387,10 +900,8 @@ static void __drm_panel_free(struct kref *kref)
  */
 struct drm_panel *drm_panel_get(struct drm_panel *panel)
 {
-	if (!panel)
-		return panel;
-
-	kref_get(&panel->refcount);
+	if (panel)
+		drm_bridge_get(&panel->bridge);
 
 	return panel;
 }
@@ -406,7 +917,7 @@ EXPORT_SYMBOL(drm_panel_get);
 void drm_panel_put(struct drm_panel *panel)
 {
 	if (panel)
-		kref_put(&panel->refcount, __drm_panel_free);
+		drm_bridge_put(&panel->bridge);
 }
 EXPORT_SYMBOL(drm_panel_put);
 
@@ -429,8 +940,22 @@ void *__devm_drm_panel_alloc(struct device *dev, size_t size, size_t offset,
 			     const struct drm_panel_funcs *funcs,
 			     int connector_type)
 {
-	void *container;
+	/*
+	 * Struct embedding and offsets:
+	 *
+	 *    |--------------- user container struct ------------|
+	 *    :    |---------- struct drm_panel ------------|
+	 *    :    :    |----- struct drm_bridge ------|
+	 *    A    B    C
+	 *
+	 * B - A = offset (passed as argument)
+	 * C - B = panel_bridge_offset
+	 * C - A = alloc_bridge_offset
+	 */
+	const size_t panel_bridge_offset = offsetof(struct drm_panel, bridge);
+	const size_t alloc_bridge_offset = offset + panel_bridge_offset;
 	struct drm_panel *panel;
+	void *container;
 	int err;
 
 	if (!funcs) {
@@ -438,14 +963,16 @@ void *__devm_drm_panel_alloc(struct device *dev, size_t size, size_t offset,
 		return ERR_PTR(-EINVAL);
 	}
 
-	container = kzalloc(size, GFP_KERNEL);
-	if (!container)
-		return ERR_PTR(-ENOMEM);
+	container = __devm_drm_bridge_alloc(dev, size, alloc_bridge_offset,
+					    &panel_bridge_bridge_funcs);
+	if (IS_ERR(container))
+		return container;
 
 	panel = container + offset;
-	panel->container = container;
 	panel->funcs = funcs;
-	kref_init(&panel->refcount);
+	panel->bridge.of_node = dev->of_node;
+
+	drm_panel_get(panel);
 
 	err = devm_add_action_or_reset(dev, drm_panel_put_void, panel);
 	if (err)
@@ -498,6 +1025,69 @@ struct drm_panel *of_drm_find_panel(const struct device_node *np)
 	return ERR_PTR(-EPROBE_DEFER);
 }
 EXPORT_SYMBOL(of_drm_find_panel);
+
+/**
+ * drm_of_find_panel_or_bridge - return connected panel or bridge device
+ * @np: device tree node containing encoder output ports
+ * @port: port in the device tree node
+ * @endpoint: endpoint in the device tree node
+ * @panel: pointer to hold returned drm_panel, must not be NULL. On success
+ *         the caller must call drm_panel_put() when done with the panel
+ * @bridge: pointer to hold returned drm_bridge
+ *
+ * Given a DT node's port and endpoint number, find the connected node and
+ * return either the associated struct drm_panel or drm_bridge device.
+ *
+ * This function is deprecated and should not be used in new drivers. Use
+ * of_drm_get_bridge_by_endpoint() instead when not looking for a panel, or
+ * devm_drm_of_get_bridge() otherwise.
+ *
+ * Returns zero if successful, or one of the standard error codes if it fails.
+ */
+int drm_of_find_panel_or_bridge(const struct device_node *np,
+				int port, int endpoint,
+				struct drm_panel **panel,
+				struct drm_bridge **bridge)
+{
+	if (WARN_ON(!panel))
+		return -EINVAL;
+
+	*panel = NULL;
+	if (bridge)
+		*bridge = NULL;
+
+	/*
+	 * of_graph_get_remote_node() produces a noisy error message if port
+	 * node isn't found and the absence of the port is a legit case here,
+	 * so at first we silently check whether a graph is present in the
+	 * device-tree node.
+	 */
+	if (!of_graph_is_present(np))
+		return -ENODEV;
+
+	struct device_node *remote __free(device_node) =
+		of_graph_get_remote_node(np, port, endpoint);
+	if (!remote)
+		return -ENODEV;
+
+	*panel = of_drm_find_panel(remote);
+	if (!IS_ERR(*panel))
+		return 0;
+
+	*panel = NULL;
+
+	if (bridge) {
+		/* No panel found yet, check for a bridge next. */
+		*bridge = of_drm_find_bridge(remote);
+		if (*bridge)
+			return 0;
+
+		*bridge = NULL;
+	}
+
+	return -EPROBE_DEFER;
+}
+EXPORT_SYMBOL_GPL(drm_of_find_panel_or_bridge);
 #endif
 
 /*
diff --git a/drivers/gpu/drm/drm_panic.c b/drivers/gpu/drm/drm_panic.c
index e576c4791861de..9fe97c4eda6c85 100644
--- a/drivers/gpu/drm/drm_panic.c
+++ b/drivers/gpu/drm/drm_panic.c
@@ -7,33 +7,23 @@
  */
 
 #include <linux/export.h>
-#include <linux/font.h>
-#include <linux/highmem.h>
 #include <linux/init.h>
-#include <linux/iosys-map.h>
 #include <linux/kdebug.h>
 #include <linux/kmsg_dump.h>
-#include <linux/linux_logo.h>
 #include <linux/list.h>
 #include <linux/math.h>
 #include <linux/module.h>
 #include <linux/overflow.h>
 #include <linux/printk.h>
 #include <linux/types.h>
-#include <linux/utsname.h>
-#include <linux/zlib.h>
 
 #include <drm/drm_drv.h>
-#include <drm/drm_fourcc.h>
-#include <drm/drm_framebuffer.h>
-#include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_panic.h>
 #include <drm/drm_plane.h>
 #include <drm/drm_print.h>
-#include <drm/drm_rect.h>
 
 #include "drm_crtc_internal.h"
-#include "drm_draw_internal.h"
+#include "drm_panic_internal.h"
 
 MODULE_AUTHOR("Jocelyn Falempe");
 MODULE_DESCRIPTION("DRM panic handler");
@@ -42,783 +32,25 @@ MODULE_LICENSE("GPL");
 /**
  * DOC: overview
  *
- * To enable DRM panic for a driver, the primary plane must implement a
- * &drm_plane_helper_funcs.get_scanout_buffer helper function. It is then
- * automatically registered to the drm panic handler.
- * When a panic occurs, the &drm_plane_helper_funcs.get_scanout_buffer will be
- * called, and the driver can provide a framebuffer so the panic handler can
- * draw the panic screen on it. Currently only linear buffer and a few color
- * formats are supported.
- * Optionally the driver can also provide a &drm_plane_helper_funcs.panic_flush
- * callback, that will be called after that, to send additional commands to the
- * hardware to make the scanout buffer visible.
- */
-
-/*
- * This module displays a user friendly message on screen when a kernel panic
- * occurs. This is conflicting with fbcon, so you can only enable it when fbcon
- * is disabled.
- * It's intended for end-user, so have minimal technical/debug information.
+ * This module displays a user friendly message on screen when a kernel
+ * panic occurs. It's intended for end users and therefore have minimal
+ * technical/debug information.
  *
- * Implementation details:
+ * To enable DRM panic for a driver, the at least one primary plane must
+ * implement struct &drm_plane_funcs.display_panic_screen. The plane is
+ * then automatically registered to the drm panic handler.
  *
- * It is a panic handler, so it can't take lock, allocate memory, run tasks/irq,
- * or attempt to sleep. It's a best effort, and it may not be able to display
- * the message in all situations (like if the panic occurs in the middle of a
- * modesetting).
- * It will display only one static frame, so performance optimizations are low
- * priority as the machine is already in an unusable state.
+ * When a panic occurs, the DRM panic handler calls struct
+ * &drm_plane_funcs.display_panic_screen. See
+ * drm_plane_helper_display_panic_screen() for a generic implementation.
  */
 
-struct drm_panic_line {
-	u32 len;
-	const char *txt;
-};
-
-#define PANIC_LINE(s) {.len = sizeof(s) - 1, .txt = s}
-
-static struct drm_panic_line panic_msg[] = {
-	PANIC_LINE("KERNEL PANIC!"),
-	PANIC_LINE(""),
-	PANIC_LINE("Please reboot your computer."),
-	PANIC_LINE(""),
-	PANIC_LINE(""), /* will be replaced by the panic description */
-};
-
-static const size_t panic_msg_lines = ARRAY_SIZE(panic_msg);
-
-static const struct drm_panic_line logo_ascii[] = {
-	PANIC_LINE("     .--.        _"),
-	PANIC_LINE("    |o_o |      | |"),
-	PANIC_LINE("    |:_/ |      | |"),
-	PANIC_LINE("   //   \\ \\     |_|"),
-	PANIC_LINE("  (|     | )     _"),
-	PANIC_LINE(" /'\\_   _/`\\    (_)"),
-	PANIC_LINE(" \\___)=(___/"),
-};
-
-static const size_t logo_ascii_lines = ARRAY_SIZE(logo_ascii);
-
-#if defined(CONFIG_LOGO) && !defined(MODULE)
-static const struct linux_logo *logo_mono;
-
-static int drm_panic_setup_logo(void)
-{
-	const struct linux_logo *logo = fb_find_logo(1);
-	const unsigned char *logo_data;
-	struct linux_logo *logo_dup;
-
-	if (!logo || logo->type != LINUX_LOGO_MONO)
-		return 0;
-
-	/* The logo is __init, so we must make a copy for later use */
-	logo_data = kmemdup(logo->data,
-			    size_mul(DIV_ROUND_UP(logo->width, BITS_PER_BYTE), logo->height),
-			    GFP_KERNEL);
-	if (!logo_data)
-		return -ENOMEM;
-
-	logo_dup = kmemdup(logo, sizeof(*logo), GFP_KERNEL);
-	if (!logo_dup) {
-		kfree(logo_data);
-		return -ENOMEM;
-	}
-
-	logo_dup->data = logo_data;
-	logo_mono = logo_dup;
-
-	return 0;
-}
-
-device_initcall(drm_panic_setup_logo);
-#else
-#define logo_mono	((const struct linux_logo *)NULL)
-#endif
-
-/*
- *  Blit & Fill functions
- */
-static void drm_panic_blit_pixel(struct drm_scanout_buffer *sb, struct drm_rect *clip,
-				 const u8 *sbuf8, unsigned int spitch, unsigned int scale,
-				 u32 fg_color)
-{
-	unsigned int y, x;
-
-	for (y = 0; y < drm_rect_height(clip); y++)
-		for (x = 0; x < drm_rect_width(clip); x++)
-			if (drm_draw_is_pixel_fg(sbuf8, spitch, x / scale, y / scale))
-				sb->set_pixel(sb, clip->x1 + x, clip->y1 + y, fg_color);
-}
-
-static void drm_panic_write_pixel16(void *vaddr, unsigned int offset, u16 color)
-{
-	u16 *p = vaddr + offset;
-
-	*p = color;
-}
-
-static void drm_panic_write_pixel24(void *vaddr, unsigned int offset, u32 color)
-{
-	u8 *p = vaddr + offset;
-
-	*p++ = color & 0xff;
-	color >>= 8;
-	*p++ = color & 0xff;
-	color >>= 8;
-	*p = color & 0xff;
-}
-
-/*
- * Special case if the pixel crosses page boundaries
- */
-static void drm_panic_write_pixel24_xpage(void *vaddr, struct page *next_page,
-					  unsigned int offset, u32 color)
-{
-	u8 *vaddr2;
-	u8 *p = vaddr + offset;
-
-	vaddr2 = kmap_local_page_try_from_panic(next_page);
-
-	*p++ = color & 0xff;
-	color >>= 8;
-
-	if (offset == PAGE_SIZE - 1)
-		p = vaddr2;
-
-	*p++ = color & 0xff;
-	color >>= 8;
-
-	if (offset == PAGE_SIZE - 2)
-		p = vaddr2;
-
-	*p = color & 0xff;
-	kunmap_local(vaddr2);
-}
-
-static void drm_panic_write_pixel32(void *vaddr, unsigned int offset, u32 color)
-{
-	u32 *p = vaddr + offset;
-
-	*p = color;
-}
-
-static void drm_panic_write_pixel(void *vaddr, unsigned int offset, u32 color, unsigned int cpp)
-{
-	switch (cpp) {
-	case 2:
-		drm_panic_write_pixel16(vaddr, offset, color);
-		break;
-	case 3:
-		drm_panic_write_pixel24(vaddr, offset, color);
-		break;
-	case 4:
-		drm_panic_write_pixel32(vaddr, offset, color);
-		break;
-	default:
-		pr_debug_once("Can't blit with pixel width %d\n", cpp);
-	}
-}
-
-/*
- * The scanout buffer pages are not mapped, so for each pixel,
- * use kmap_local_page_try_from_panic() to map the page, and write the pixel.
- * Try to keep the map from the previous pixel, to avoid too much map/unmap.
- */
-static void drm_panic_blit_page(struct page **pages, unsigned int dpitch,
-				unsigned int cpp, const u8 *sbuf8,
-				unsigned int spitch, struct drm_rect *clip,
-				unsigned int scale, u32 fg32)
-{
-	unsigned int y, x;
-	unsigned int page = ~0;
-	unsigned int height = drm_rect_height(clip);
-	unsigned int width = drm_rect_width(clip);
-	void *vaddr = NULL;
-
-	for (y = 0; y < height; y++) {
-		for (x = 0; x < width; x++) {
-			if (drm_draw_is_pixel_fg(sbuf8, spitch, x / scale, y / scale)) {
-				unsigned int new_page;
-				unsigned int offset;
-
-				offset = (y + clip->y1) * dpitch + (x + clip->x1) * cpp;
-				new_page = offset >> PAGE_SHIFT;
-				offset = offset % PAGE_SIZE;
-				if (new_page != page) {
-					if (!pages[new_page])
-						continue;
-					if (vaddr)
-						kunmap_local(vaddr);
-					page = new_page;
-					vaddr = kmap_local_page_try_from_panic(pages[page]);
-				}
-				if (!vaddr)
-					continue;
-
-				// Special case for 24bit, as a pixel might cross page boundaries
-				if (cpp == 3 && offset + 3 > PAGE_SIZE)
-					drm_panic_write_pixel24_xpage(vaddr, pages[page + 1],
-								      offset, fg32);
-				else
-					drm_panic_write_pixel(vaddr, offset, fg32, cpp);
-			}
-		}
-	}
-	if (vaddr)
-		kunmap_local(vaddr);
-}
-
-/*
- * drm_panic_blit - convert a monochrome image to a linear framebuffer
- * @sb: destination scanout buffer
- * @clip: destination rectangle
- * @sbuf8: source buffer, in monochrome format, 8 pixels per byte.
- * @spitch: source pitch in bytes
- * @scale: integer scale, source buffer is scale time smaller than destination
- *         rectangle
- * @fg_color: foreground color, in destination format
- *
- * This can be used to draw a font character, which is a monochrome image, to a
- * framebuffer in other supported format.
- */
-static void drm_panic_blit(struct drm_scanout_buffer *sb, struct drm_rect *clip,
-			   const u8 *sbuf8, unsigned int spitch,
-			   unsigned int scale, u32 fg_color)
-
-{
-	struct iosys_map map;
-
-	if (sb->set_pixel)
-		return drm_panic_blit_pixel(sb, clip, sbuf8, spitch, scale, fg_color);
-
-	if (sb->pages)
-		return drm_panic_blit_page(sb->pages, sb->pitch[0], sb->format->cpp[0],
-					   sbuf8, spitch, clip, scale, fg_color);
-
-	map = sb->map[0];
-	iosys_map_incr(&map, clip->y1 * sb->pitch[0] + clip->x1 * sb->format->cpp[0]);
-
-	switch (sb->format->cpp[0]) {
-	case 2:
-		drm_draw_blit16(&map, sb->pitch[0], sbuf8, spitch,
-				drm_rect_height(clip), drm_rect_width(clip), scale, fg_color);
-	break;
-	case 3:
-		drm_draw_blit24(&map, sb->pitch[0], sbuf8, spitch,
-				drm_rect_height(clip), drm_rect_width(clip), scale, fg_color);
-	break;
-	case 4:
-		drm_draw_blit32(&map, sb->pitch[0], sbuf8, spitch,
-				drm_rect_height(clip), drm_rect_width(clip), scale, fg_color);
-	break;
-	default:
-		WARN_ONCE(1, "Can't blit with pixel width %d\n", sb->format->cpp[0]);
-	}
-}
-
-static void drm_panic_fill_pixel(struct drm_scanout_buffer *sb,
-				 struct drm_rect *clip,
-				 u32 color)
-{
-	unsigned int y, x;
-
-	for (y = 0; y < drm_rect_height(clip); y++)
-		for (x = 0; x < drm_rect_width(clip); x++)
-			sb->set_pixel(sb, clip->x1 + x, clip->y1 + y, color);
-}
-
-static void drm_panic_fill_page(struct page **pages, unsigned int dpitch,
-				unsigned int cpp, struct drm_rect *clip,
-				u32 color)
-{
-	unsigned int y, x;
-	unsigned int page = ~0;
-	void *vaddr = NULL;
-
-	for (y = clip->y1; y < clip->y2; y++) {
-		for (x = clip->x1; x < clip->x2; x++) {
-			unsigned int new_page;
-			unsigned int offset;
-
-			offset = y * dpitch + x * cpp;
-			new_page = offset >> PAGE_SHIFT;
-			offset = offset % PAGE_SIZE;
-			if (new_page != page) {
-				if (vaddr)
-					kunmap_local(vaddr);
-				page = new_page;
-				vaddr = kmap_local_page_try_from_panic(pages[page]);
-			}
-			if (!vaddr)
-				continue;
-
-			// Special case for 24bit, as a pixel might cross page boundaries
-			if (cpp == 3 && offset + 3 > PAGE_SIZE)
-				drm_panic_write_pixel24_xpage(vaddr, pages[page + 1],
-							      offset, color);
-			else
-				drm_panic_write_pixel(vaddr, offset, color, cpp);
-		}
-	}
-	if (vaddr)
-		kunmap_local(vaddr);
-}
-
-/*
- * drm_panic_fill - Fill a rectangle with a color
- * @sb: destination scanout buffer
- * @clip: destination rectangle
- * @color: foreground color, in destination format
- *
- * Fill a rectangle with a color, in a linear framebuffer.
- */
-static void drm_panic_fill(struct drm_scanout_buffer *sb, struct drm_rect *clip,
-			   u32 color)
-{
-	struct iosys_map map;
-
-	if (sb->set_pixel)
-		return drm_panic_fill_pixel(sb, clip, color);
-
-	if (sb->pages)
-		return drm_panic_fill_page(sb->pages, sb->pitch[0], sb->format->cpp[0],
-					   clip, color);
-
-	map = sb->map[0];
-	iosys_map_incr(&map, clip->y1 * sb->pitch[0] + clip->x1 * sb->format->cpp[0]);
-
-	switch (sb->format->cpp[0]) {
-	case 2:
-		drm_draw_fill16(&map, sb->pitch[0], drm_rect_height(clip),
-				drm_rect_width(clip), color);
-	break;
-	case 3:
-		drm_draw_fill24(&map, sb->pitch[0], drm_rect_height(clip),
-				drm_rect_width(clip), color);
-	break;
-	case 4:
-		drm_draw_fill32(&map, sb->pitch[0], drm_rect_height(clip),
-				drm_rect_width(clip), color);
-	break;
-	default:
-		WARN_ONCE(1, "Can't fill with pixel width %d\n", sb->format->cpp[0]);
-	}
-}
-
-static unsigned int get_max_line_len(const struct drm_panic_line *lines, int len)
-{
-	int i;
-	unsigned int max = 0;
-
-	for (i = 0; i < len; i++)
-		max = max(lines[i].len, max);
-	return max;
-}
-
-/*
- * Draw a text in a rectangle on a framebuffer. The text is truncated if it overflows the rectangle
- */
-static void draw_txt_rectangle(struct drm_scanout_buffer *sb,
-			       const struct font_desc *font,
-			       const struct drm_panic_line *msg,
-			       unsigned int msg_lines,
-			       bool centered,
-			       struct drm_rect *clip,
-			       u32 color)
-{
-	int i, j;
-	const u8 *src;
-	size_t font_pitch = DIV_ROUND_UP(font->width, 8);
-	struct drm_rect rec;
-
-	msg_lines = min(msg_lines,  drm_rect_height(clip) / font->height);
-	for (i = 0; i < msg_lines; i++) {
-		size_t line_len = min(msg[i].len, drm_rect_width(clip) / font->width);
-
-		rec.y1 = clip->y1 +  i * font->height;
-		rec.y2 = rec.y1 + font->height;
-		rec.x1 = clip->x1;
-
-		if (centered)
-			rec.x1 += (drm_rect_width(clip) - (line_len * font->width)) / 2;
-
-		for (j = 0; j < line_len; j++) {
-			src = font_data_glyph_buf(font->data, font->width, font->height,
-						  (unsigned char)msg[i].txt[j]);
-			rec.x2 = rec.x1 + font->width;
-			if (src)
-				drm_panic_blit(sb, &rec, src, font_pitch, 1, color);
-			rec.x1 += font->width;
-		}
-	}
-}
-
-static void drm_panic_logo_rect(struct drm_rect *rect, const struct font_desc *font)
-{
-	if (logo_mono) {
-		drm_rect_init(rect, 0, 0, logo_mono->width, logo_mono->height);
-	} else {
-		int logo_width = get_max_line_len(logo_ascii, logo_ascii_lines) * font->width;
-
-		drm_rect_init(rect, 0, 0, logo_width, logo_ascii_lines * font->height);
-	}
-}
-
-static void drm_panic_logo_draw(struct drm_scanout_buffer *sb, struct drm_rect *rect,
-				const struct font_desc *font, u32 fg_color)
-{
-	if (rect->x2 > sb->width || rect->y2 > sb->height)
-		return;
-
-	if (logo_mono)
-		drm_panic_blit(sb, rect, logo_mono->data,
-			       DIV_ROUND_UP(drm_rect_width(rect), 8), 1, fg_color);
-	else
-		draw_txt_rectangle(sb, font, logo_ascii, logo_ascii_lines, false, rect,
-				   fg_color);
-}
-
-static void draw_panic_screen_user(struct drm_scanout_buffer *sb)
-{
-	u32 fg_color = drm_draw_color_from_xrgb8888(CONFIG_DRM_PANIC_FOREGROUND_COLOR,
-						    sb->format->format);
-	u32 bg_color = drm_draw_color_from_xrgb8888(CONFIG_DRM_PANIC_BACKGROUND_COLOR,
-						    sb->format->format);
-	const struct font_desc *font = get_default_font(sb->width, sb->height, NULL, NULL);
-	struct drm_rect r_screen, r_logo, r_msg;
-	unsigned int msg_width, msg_height;
-
-	if (!font)
-		return;
-
-	r_screen = DRM_RECT_INIT(0, 0, sb->width, sb->height);
-	drm_panic_logo_rect(&r_logo, font);
-
-	msg_width = min(get_max_line_len(panic_msg, panic_msg_lines) * font->width, sb->width);
-	msg_height = min(panic_msg_lines * font->height, sb->height);
-	r_msg = DRM_RECT_INIT(0, 0, msg_width, msg_height);
-
-	/* Center the panic message */
-	drm_rect_translate(&r_msg, (sb->width - r_msg.x2) / 2, (sb->height - r_msg.y2) / 2);
-
-	/* Fill with the background color, and draw text on top */
-	drm_panic_fill(sb, &r_screen, bg_color);
-
-	if (!drm_rect_overlap(&r_logo, &r_msg))
-		drm_panic_logo_draw(sb, &r_logo, font, fg_color);
-
-	draw_txt_rectangle(sb, font, panic_msg, panic_msg_lines, true, &r_msg, fg_color);
-}
-
-/*
- * Draw one line of kmsg, and handle wrapping if it won't fit in the screen width.
- * Return the y-offset of the next line.
- */
-static int draw_line_with_wrap(struct drm_scanout_buffer *sb, const struct font_desc *font,
-			       struct drm_panic_line *line, int yoffset, u32 fg_color)
-{
-	int chars_per_row = sb->width / font->width;
-	struct drm_rect r_txt = DRM_RECT_INIT(0, yoffset, sb->width, font->height);
-	struct drm_panic_line line_wrap;
-
-	if (line->len > chars_per_row) {
-		line_wrap.len = line->len % chars_per_row;
-		line_wrap.txt = line->txt + line->len - line_wrap.len;
-		draw_txt_rectangle(sb, font, &line_wrap, 1, false, &r_txt, fg_color);
-		r_txt.y1 -= font->height;
-		if (r_txt.y1 < 0)
-			return r_txt.y1;
-		while (line_wrap.txt > line->txt) {
-			line_wrap.txt -= chars_per_row;
-			line_wrap.len = chars_per_row;
-			draw_txt_rectangle(sb, font, &line_wrap, 1, false, &r_txt, fg_color);
-			r_txt.y1 -= font->height;
-			if (r_txt.y1 < 0)
-				return r_txt.y1;
-		}
-	} else {
-		draw_txt_rectangle(sb, font, line, 1, false, &r_txt, fg_color);
-		r_txt.y1 -= font->height;
-	}
-	return r_txt.y1;
-}
-
-/*
- * Draw the kmsg buffer to the screen, starting from the youngest message at the bottom,
- * and going up until reaching the top of the screen.
- */
-static void draw_panic_screen_kmsg(struct drm_scanout_buffer *sb)
-{
-	u32 fg_color = drm_draw_color_from_xrgb8888(CONFIG_DRM_PANIC_FOREGROUND_COLOR,
-						    sb->format->format);
-	u32 bg_color = drm_draw_color_from_xrgb8888(CONFIG_DRM_PANIC_BACKGROUND_COLOR,
-						    sb->format->format);
-	const struct font_desc *font = get_default_font(sb->width, sb->height, NULL, NULL);
-	struct drm_rect r_screen = DRM_RECT_INIT(0, 0, sb->width, sb->height);
-	struct kmsg_dump_iter iter;
-	char kmsg_buf[512];
-	size_t kmsg_len;
-	struct drm_panic_line line;
-	int yoffset;
-
-	if (!font || font->width > sb->width)
-		return;
-
-	yoffset = sb->height - font->height - (sb->height % font->height) / 2;
-
-	/* Fill with the background color, and draw text on top */
-	drm_panic_fill(sb, &r_screen, bg_color);
-
-	kmsg_dump_rewind(&iter);
-	while (kmsg_dump_get_buffer(&iter, false, kmsg_buf, sizeof(kmsg_buf), &kmsg_len)) {
-		char *start;
-		char *end;
-
-		/* ignore terminating NUL and newline */
-		start = kmsg_buf + kmsg_len - 2;
-		end = kmsg_buf + kmsg_len - 1;
-		while (start > kmsg_buf && yoffset >= 0) {
-			while (start > kmsg_buf && *start != '\n')
-				start--;
-			/* don't count the newline character */
-			line.txt = start + (start == kmsg_buf ? 0 : 1);
-			line.len = end - line.txt;
-
-			yoffset = draw_line_with_wrap(sb, font, &line, yoffset, fg_color);
-			end = start;
-			start--;
-		}
-	}
-}
-
 #if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
-/*
- * It is unwise to allocate memory in the panic callback, so the buffers are
- * pre-allocated. Only 2 buffers and the zlib workspace are needed.
- * Two buffers are enough, using the following buffer usage:
- * 1) kmsg messages are dumped in buffer1
- * 2) kmsg is zlib-compressed into buffer2
- * 3) compressed kmsg is encoded as QR-code Numeric stream in buffer1
- * 4) QR-code image is generated in buffer2
- * The Max QR code size is V40, 177x177, 4071 bytes for image, 2956 bytes for
- * data segments.
- *
- * Typically, ~7500 bytes of kmsg, are compressed into 2800 bytes, which fits in
- * a V40 QR-code (177x177).
- *
- * If CONFIG_DRM_PANIC_SCREEN_QR_CODE_URL is not set, the kmsg data will be put
- * directly in the QR code.
- * 1) kmsg messages are dumped in buffer1
- * 2) kmsg message is encoded as byte stream in buffer2
- * 3) QR-code image is generated in buffer1
- */
-
 static uint panic_qr_version = CONFIG_DRM_PANIC_SCREEN_QR_VERSION;
 module_param(panic_qr_version, uint, 0644);
 MODULE_PARM_DESC(panic_qr_version, "maximum version (size) of the QR code");
-
-#define MAX_QR_DATA 2956
-#define MAX_ZLIB_RATIO 3
-#define QR_BUFFER1_SIZE (MAX_ZLIB_RATIO * MAX_QR_DATA) /* Must also be > 4071  */
-#define QR_BUFFER2_SIZE 4096
-#define QR_MARGIN	4	/* 4 modules of foreground color around the qr code */
-
-/* Compression parameters */
-#define COMPR_LEVEL 6
-#define WINDOW_BITS 12
-#define MEM_LEVEL 4
-
-static char *qrbuf1;
-static char *qrbuf2;
-static struct z_stream_s stream;
-
-static void __init drm_panic_qr_init(void)
-{
-	qrbuf1 = kmalloc(QR_BUFFER1_SIZE, GFP_KERNEL);
-	qrbuf2 = kmalloc(QR_BUFFER2_SIZE, GFP_KERNEL);
-	stream.workspace = kmalloc(zlib_deflate_workspacesize(WINDOW_BITS, MEM_LEVEL),
-				   GFP_KERNEL);
-}
-
-static void drm_panic_qr_exit(void)
-{
-	kfree(qrbuf1);
-	qrbuf1 = NULL;
-	kfree(qrbuf2);
-	qrbuf2 = NULL;
-	kfree(stream.workspace);
-	stream.workspace = NULL;
-}
-
-static int drm_panic_get_qr_code_url(u8 **qr_image)
-{
-	struct kmsg_dump_iter iter;
-	char url[256];
-	size_t kmsg_len, max_kmsg_size;
-	char *kmsg;
-	int max_qr_data_size, url_len;
-
-	url_len = snprintf(url, sizeof(url), CONFIG_DRM_PANIC_SCREEN_QR_CODE_URL "?a=%s&v=%s&z=",
-			   utsname()->machine, utsname()->release);
-
-	max_qr_data_size = drm_panic_qr_max_data_size(panic_qr_version, url_len);
-	max_kmsg_size = min(MAX_ZLIB_RATIO * max_qr_data_size, QR_BUFFER1_SIZE);
-
-	/* get kmsg to buffer 1 */
-	kmsg_dump_rewind(&iter);
-	kmsg_dump_get_buffer(&iter, false, qrbuf1, max_kmsg_size, &kmsg_len);
-
-	if (!kmsg_len)
-		return -ENODATA;
-	kmsg = qrbuf1;
-
-try_again:
-	if (zlib_deflateInit2(&stream, COMPR_LEVEL, Z_DEFLATED, WINDOW_BITS,
-			      MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK)
-		return -EINVAL;
-
-	stream.next_in = kmsg;
-	stream.avail_in = kmsg_len;
-	stream.total_in = 0;
-	stream.next_out = qrbuf2;
-	stream.avail_out = QR_BUFFER2_SIZE;
-	stream.total_out = 0;
-
-	if (zlib_deflate(&stream, Z_FINISH) != Z_STREAM_END)
-		return -EINVAL;
-
-	if (zlib_deflateEnd(&stream) != Z_OK)
-		return -EINVAL;
-
-	if (stream.total_out > max_qr_data_size) {
-		/* too much data for the QR code, so skip the first line and try again */
-		kmsg = strchr(kmsg, '\n');
-		if (!kmsg)
-			return -EINVAL;
-		/* skip the first \n */
-		kmsg += 1;
-		kmsg_len = strlen(kmsg);
-		goto try_again;
-	}
-	*qr_image = qrbuf2;
-
-	/* generate qr code image in buffer2 */
-	return drm_panic_qr_generate(url, qrbuf2, stream.total_out, QR_BUFFER2_SIZE,
-				     qrbuf1, QR_BUFFER1_SIZE);
-}
-
-static int drm_panic_get_qr_code_raw(u8 **qr_image)
-{
-	struct kmsg_dump_iter iter;
-	size_t kmsg_len;
-	size_t max_kmsg_size = min(drm_panic_qr_max_data_size(panic_qr_version, 0),
-				   QR_BUFFER1_SIZE);
-
-	kmsg_dump_rewind(&iter);
-	kmsg_dump_get_buffer(&iter, false, qrbuf1, max_kmsg_size, &kmsg_len);
-	if (!kmsg_len)
-		return -ENODATA;
-
-	*qr_image = qrbuf1;
-	return drm_panic_qr_generate(NULL, qrbuf1, kmsg_len, QR_BUFFER1_SIZE,
-				     qrbuf2, QR_BUFFER2_SIZE);
-}
-
-static int drm_panic_get_qr_code(u8 **qr_image)
-{
-	if (strlen(CONFIG_DRM_PANIC_SCREEN_QR_CODE_URL) > 0)
-		return drm_panic_get_qr_code_url(qr_image);
-	else
-		return drm_panic_get_qr_code_raw(qr_image);
-}
-
-/*
- * Draw the panic message at the center of the screen, with a QR Code
- */
-static int _draw_panic_screen_qr_code(struct drm_scanout_buffer *sb)
-{
-	u32 fg_color = drm_draw_color_from_xrgb8888(CONFIG_DRM_PANIC_FOREGROUND_COLOR,
-						    sb->format->format);
-	u32 bg_color = drm_draw_color_from_xrgb8888(CONFIG_DRM_PANIC_BACKGROUND_COLOR,
-						    sb->format->format);
-	const struct font_desc *font = get_default_font(sb->width, sb->height, NULL, NULL);
-	struct drm_rect r_screen, r_logo, r_msg, r_qr, r_qr_canvas;
-	unsigned int max_qr_size, scale;
-	unsigned int msg_width, msg_height;
-	int qr_width, qr_canvas_width, qr_pitch, v_margin;
-	u8 *qr_image;
-
-	if (!font || !qrbuf1 || !qrbuf2 || !stream.workspace)
-		return -ENOMEM;
-
-	r_screen = DRM_RECT_INIT(0, 0, sb->width, sb->height);
-
-	drm_panic_logo_rect(&r_logo, font);
-
-	msg_width = min(get_max_line_len(panic_msg, panic_msg_lines) * font->width, sb->width);
-	msg_height = min(panic_msg_lines * font->height, sb->height);
-	r_msg = DRM_RECT_INIT(0, 0, msg_width, msg_height);
-
-	max_qr_size = min(3 * sb->width / 4, 3 * sb->height / 4);
-
-	qr_width = drm_panic_get_qr_code(&qr_image);
-	if (qr_width <= 0)
-		return -ENOSPC;
-
-	qr_canvas_width = qr_width + QR_MARGIN * 2;
-	scale = max_qr_size / qr_canvas_width;
-	/* QR code is not readable if not scaled at least by 2 */
-	if (scale < 2)
-		return -ENOSPC;
-
-	pr_debug("QR width %d and scale %d\n", qr_width, scale);
-	r_qr_canvas = DRM_RECT_INIT(0, 0, qr_canvas_width * scale, qr_canvas_width * scale);
-
-	v_margin = sb->height - drm_rect_height(&r_qr_canvas) - drm_rect_height(&r_msg);
-	if (v_margin < 0)
-		return -ENOSPC;
-	v_margin /= 5;
-
-	drm_rect_translate(&r_qr_canvas, (sb->width - r_qr_canvas.x2) / 2, 2 * v_margin);
-	r_qr = DRM_RECT_INIT(r_qr_canvas.x1 + QR_MARGIN * scale, r_qr_canvas.y1 + QR_MARGIN * scale,
-			     qr_width * scale, qr_width * scale);
-
-	/* Center the panic message */
-	drm_rect_translate(&r_msg, (sb->width - r_msg.x2) / 2,
-			   3 * v_margin + drm_rect_height(&r_qr_canvas));
-
-	/* Fill with the background color, and draw text on top */
-	drm_panic_fill(sb, &r_screen, bg_color);
-
-	if (!drm_rect_overlap(&r_logo, &r_msg) && !drm_rect_overlap(&r_logo, &r_qr_canvas))
-		drm_panic_logo_draw(sb, &r_logo, font, fg_color);
-
-	draw_txt_rectangle(sb, font, panic_msg, panic_msg_lines, true, &r_msg, fg_color);
-
-	/* Draw the qr code */
-	qr_pitch = DIV_ROUND_UP(qr_width, 8);
-	drm_panic_fill(sb, &r_qr_canvas, fg_color);
-	drm_panic_fill(sb, &r_qr, bg_color);
-	drm_panic_blit(sb, &r_qr, qr_image, qr_pitch, scale, fg_color);
-	return 0;
-}
-
-static void draw_panic_screen_qr_code(struct drm_scanout_buffer *sb)
-{
-	if (_draw_panic_screen_qr_code(sb))
-		draw_panic_screen_user(sb);
-}
-#else
-static void drm_panic_qr_init(void) {};
-static void drm_panic_qr_exit(void) {};
 #endif
 
-enum drm_panic_type {
-	DRM_PANIC_TYPE_KMSG,
-	DRM_PANIC_TYPE_USER,
-	DRM_PANIC_TYPE_QR,
-};
-
 static enum drm_panic_type drm_panic_type = -1;
 
 static const char *drm_panic_type_map[] = {
@@ -863,91 +95,34 @@ MODULE_PARM_DESC(panic_screen,
 #endif
 		 CONFIG_DRM_PANIC_SCREEN "]");
 
-/*
- * drm_panic_is_format_supported()
- * @format: a fourcc color code
- * Returns: true if supported, false otherwise.
- *
- * Check if drm_panic will be able to use this color format.
- */
-static bool drm_panic_is_format_supported(const struct drm_format_info *format)
-{
-	if (format->num_planes != 1)
-		return false;
-	return drm_draw_can_convert_from_xrgb8888(format->format);
-}
+#define drm_panic_trylock(dev, flags) \
+	raw_spin_trylock_irqsave(&(dev)->mode_config.panic_lock, flags)
 
-static void draw_panic_dispatch(struct drm_scanout_buffer *sb)
+static void drm_panic_display_panic_screen(struct drm_plane *plane, const char *description)
 {
-	switch (drm_panic_type) {
-	case DRM_PANIC_TYPE_KMSG:
-		draw_panic_screen_kmsg(sb);
-		break;
-
+#if defined(CONFIG_DRM_PANIC_FOREGROUND_COLOR)
+	u32 fg_color = CONFIG_DRM_PANIC_FOREGROUND_COLOR;
+#else
+	u32 fg_color = 0x00ffffff;
+#endif
+#if defined(CONFIG_DRM_PANIC_BACKGROUND_COLOR)
+	u32 bg_color = CONFIG_DRM_PANIC_BACKGROUND_COLOR;
+#else
+	u32 bg_color = 0x00000000;
+#endif
 #if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
-	case DRM_PANIC_TYPE_QR:
-		draw_panic_screen_qr_code(sb);
-		break;
+	unsigned int qr_version = panic_qr_version;
+#else
+	unsigned int qr_version = 0;
 #endif
-
-	case DRM_PANIC_TYPE_USER:
-	default:
-		draw_panic_screen_user(sb);
-	}
-}
-
-static void drm_panic_set_description(const char *description)
-{
-	u32 len;
-
-	if (description) {
-		struct drm_panic_line *desc_line = &panic_msg[panic_msg_lines - 1];
-
-		desc_line->txt = description;
-		len = strlen(description);
-		/* ignore the last newline character */
-		if (len && description[len - 1] == '\n')
-			len -= 1;
-		desc_line->len = len;
-	}
-}
-
-static void drm_panic_clear_description(void)
-{
-	struct drm_panic_line *desc_line = &panic_msg[panic_msg_lines - 1];
-
-	desc_line->len = 0;
-	desc_line->txt = NULL;
-}
-
-static void draw_panic_plane(struct drm_plane *plane, const char *description)
-{
-	struct drm_scanout_buffer sb = { };
-	int ret;
+	struct drm_device *dev = plane->dev;
 	unsigned long flags;
 
-	if (!drm_panic_trylock(plane->dev, flags))
-		return;
-
-	ret = plane->helper_private->get_scanout_buffer(plane, &sb);
-
-	if (ret || !drm_panic_is_format_supported(sb.format))
-		goto unlock;
-
-	/* One of these should be set, or it can't draw pixels */
-	if (!sb.set_pixel && !sb.pages && iosys_map_is_null(&sb.map[0]))
-		goto unlock;
-
-	drm_panic_set_description(description);
-
-	draw_panic_dispatch(&sb);
-	if (plane->helper_private->panic_flush)
-		plane->helper_private->panic_flush(plane);
-
-	drm_panic_clear_description();
-
-unlock:
-	drm_panic_unlock(plane->dev, flags);
+	if (drm_panic_trylock(dev, flags)) {
+		plane->funcs->display_panic_screen(plane, description, drm_panic_type,
+						   fg_color, bg_color, qr_version);
+		drm_panic_unlock(dev, flags);
+	}
 }
 
 static struct drm_plane *to_drm_plane(struct kmsg_dumper *kd)
@@ -960,10 +135,9 @@ static void drm_panic(struct kmsg_dumper *dumper, struct kmsg_dump_detail *detai
 	struct drm_plane *plane = to_drm_plane(dumper);
 
 	if (detail->reason == KMSG_DUMP_PANIC)
-		draw_panic_plane(plane, detail->description);
+		drm_panic_display_panic_screen(plane, detail->description);
 }
 
-
 /*
  * DEBUG FS, This is currently unsafe.
  * Create one file per plane, so it's possible to debug one plane at a time.
@@ -980,7 +154,7 @@ static ssize_t debugfs_trigger_write(struct file *file, const char __user *user_
 	if (kstrtobool_from_user(user_buf, count, &run) == 0 && run) {
 		struct drm_plane *plane = file->private_data;
 
-		draw_panic_plane(plane, "Test from debugfs");
+		drm_panic_display_panic_screen(plane, "Test from debugfs");
 	}
 	return count;
 }
@@ -1016,9 +190,13 @@ bool drm_panic_is_enabled(struct drm_device *dev)
 	if (!dev->mode_config.num_total_plane)
 		return false;
 
-	drm_for_each_plane(plane, dev)
-		if (plane->helper_private && plane->helper_private->get_scanout_buffer)
-			return true;
+	drm_for_each_plane(plane, dev) {
+		if (plane->type != DRM_PLANE_TYPE_PRIMARY)
+			continue;
+		if (!plane->funcs || !plane->funcs->display_panic_screen)
+			continue;
+		return true;
+	}
 	return false;
 }
 EXPORT_SYMBOL(drm_panic_is_enabled);
@@ -1036,7 +214,9 @@ void drm_panic_register(struct drm_device *dev)
 		return;
 
 	drm_for_each_plane(plane, dev) {
-		if (!plane->helper_private || !plane->helper_private->get_scanout_buffer)
+		if (plane->type != DRM_PLANE_TYPE_PRIMARY)
+			continue;
+		if (!plane->funcs || !plane->funcs->display_panic_screen)
 			continue;
 		plane->kmsg_panic.dump = drm_panic;
 		plane->kmsg_panic.max_reason = KMSG_DUMP_PANIC;
@@ -1063,7 +243,9 @@ void drm_panic_unregister(struct drm_device *dev)
 		return;
 
 	drm_for_each_plane(plane, dev) {
-		if (!plane->helper_private || !plane->helper_private->get_scanout_buffer)
+		if (plane->type != DRM_PLANE_TYPE_PRIMARY)
+			continue;
+		if (!plane->funcs || !plane->funcs->display_panic_screen)
 			continue;
 		kmsg_dump_unregister(&plane->kmsg_panic);
 	}
@@ -1079,17 +261,4 @@ void __init drm_panic_init(void)
 			CONFIG_DRM_PANIC_SCREEN);
 		drm_panic_type = DRM_PANIC_TYPE_USER;
 	}
-	drm_panic_qr_init();
-}
-
-/**
- * drm_panic_exit() - Free the resources taken by drm_panic_exit()
- */
-void drm_panic_exit(void)
-{
-	drm_panic_qr_exit();
 }
-
-#ifdef CONFIG_DRM_KUNIT_TEST
-#include "tests/drm_panic_test.c"
-#endif
diff --git a/drivers/gpu/drm/drm_panic_helper.c b/drivers/gpu/drm/drm_panic_helper.c
new file mode 100644
index 00000000000000..8913c3d22e8d31
--- /dev/null
+++ b/drivers/gpu/drm/drm_panic_helper.c
@@ -0,0 +1,919 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+#include <linux/export.h>
+#include <linux/font.h>
+#include <linux/highmem.h>
+#include <linux/iosys-map.h>
+#include <linux/linux_logo.h>
+#include <linux/utsname.h>
+#include <linux/zlib.h>
+
+#include <kunit/visibility.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_panic.h>
+#include <drm/drm_panic_helper.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_rect.h>
+
+#include "drm_draw_internal.h"
+#include "drm_panic_internal.h"
+
+struct drm_panic_line {
+	u32 len;
+	const char *txt;
+};
+
+#define PANIC_LINE(s) {.len = sizeof(s) - 1, .txt = s}
+
+static struct drm_panic_line panic_msg[] = {
+	PANIC_LINE("KERNEL PANIC!"),
+	PANIC_LINE(""),
+	PANIC_LINE("Please reboot your computer."),
+	PANIC_LINE(""),
+	PANIC_LINE(""), /* will be replaced by the panic description */
+};
+
+static const size_t panic_msg_lines = ARRAY_SIZE(panic_msg);
+
+static const struct drm_panic_line logo_ascii[] = {
+	PANIC_LINE("     .--.        _"),
+	PANIC_LINE("    |o_o |      | |"),
+	PANIC_LINE("    |:_/ |      | |"),
+	PANIC_LINE("   //   \\ \\     |_|"),
+	PANIC_LINE("  (|     | )     _"),
+	PANIC_LINE(" /'\\_   _/`\\    (_)"),
+	PANIC_LINE(" \\___)=(___/"),
+};
+
+static const size_t logo_ascii_lines = ARRAY_SIZE(logo_ascii);
+
+#if defined(CONFIG_LOGO) && !defined(MODULE)
+static const struct linux_logo *logo_mono;
+
+static int __init drm_panic_helper_setup_logo(void)
+{
+	const struct linux_logo *logo = fb_find_logo(1);
+	const unsigned char *logo_data;
+	struct linux_logo *logo_dup;
+
+	if (!logo || logo->type != LINUX_LOGO_MONO)
+		return 0;
+
+	/* The logo is __init, so we must make a copy for later use */
+	logo_data = kmemdup(logo->data,
+			    size_mul(DIV_ROUND_UP(logo->width, BITS_PER_BYTE), logo->height),
+			    GFP_KERNEL);
+	if (!logo_data)
+		return -ENOMEM;
+
+	logo_dup = kmemdup(logo, sizeof(*logo), GFP_KERNEL);
+	if (!logo_dup) {
+		kfree(logo_data);
+		return -ENOMEM;
+	}
+
+	logo_dup->data = logo_data;
+	logo_mono = logo_dup;
+
+	return 0;
+}
+#else
+#define logo_mono	((const struct linux_logo *)NULL)
+static int __init drm_panic_helper_setup_logo(void)
+{
+	return 0;
+}
+#endif
+
+/*
+ *  Blit & Fill functions
+ */
+static void drm_panic_helper_blit_pixel(struct drm_scanout_buffer *sb, struct drm_rect *clip,
+					const u8 *sbuf8, unsigned int spitch, unsigned int scale,
+					u32 fg_color)
+{
+	unsigned int y, x;
+
+	for (y = 0; y < drm_rect_height(clip); y++)
+		for (x = 0; x < drm_rect_width(clip); x++)
+			if (drm_draw_is_pixel_fg(sbuf8, spitch, x / scale, y / scale))
+				sb->set_pixel(sb, clip->x1 + x, clip->y1 + y, fg_color);
+}
+
+static void drm_panic_helper_write_pixel16(void *vaddr, unsigned int offset, u16 color)
+{
+	u16 *p = vaddr + offset;
+
+	*p = color;
+}
+
+static void drm_panic_helper_write_pixel24(void *vaddr, unsigned int offset, u32 color)
+{
+	u8 *p = vaddr + offset;
+
+	*p++ = color & 0xff;
+	color >>= 8;
+	*p++ = color & 0xff;
+	color >>= 8;
+	*p = color & 0xff;
+}
+
+/*
+ * Special case if the pixel crosses page boundaries
+ */
+static void drm_panic_helper_write_pixel24_xpage(void *vaddr, struct page *next_page,
+						 unsigned int offset, u32 color)
+{
+	u8 *vaddr2;
+	u8 *p = vaddr + offset;
+
+	vaddr2 = kmap_local_page_try_from_panic(next_page);
+	if (!vaddr2)
+		return;
+
+	*p++ = color & 0xff;
+	color >>= 8;
+
+	if (offset == PAGE_SIZE - 1)
+		p = vaddr2;
+
+	*p++ = color & 0xff;
+	color >>= 8;
+
+	if (offset == PAGE_SIZE - 2)
+		p = vaddr2;
+
+	*p = color & 0xff;
+	kunmap_local(vaddr2);
+}
+
+static void drm_panic_helper_write_pixel32(void *vaddr, unsigned int offset, u32 color)
+{
+	u32 *p = vaddr + offset;
+
+	*p = color;
+}
+
+static void drm_panic_helper_write_pixel(void *vaddr, unsigned int offset, u32 color,
+					 unsigned int cpp)
+{
+	switch (cpp) {
+	case 2:
+		drm_panic_helper_write_pixel16(vaddr, offset, color);
+		break;
+	case 3:
+		drm_panic_helper_write_pixel24(vaddr, offset, color);
+		break;
+	case 4:
+		drm_panic_helper_write_pixel32(vaddr, offset, color);
+		break;
+	default:
+		pr_debug_once("Can't blit with pixel width %d\n", cpp);
+	}
+}
+
+/*
+ * The scanout buffer pages are not mapped, so for each pixel,
+ * use kmap_local_page_try_from_panic() to map the page, and write the pixel.
+ * Try to keep the map from the previous pixel, to avoid too much map/unmap.
+ */
+static void drm_panic_helper_blit_page(struct page **pages, unsigned int dpitch,
+				       unsigned int cpp, const u8 *sbuf8,
+				       unsigned int spitch, struct drm_rect *clip,
+				       unsigned int scale, u32 fg32)
+{
+	unsigned int y, x;
+	unsigned int page = ~0;
+	unsigned int height = drm_rect_height(clip);
+	unsigned int width = drm_rect_width(clip);
+	void *vaddr = NULL;
+
+	for (y = 0; y < height; y++) {
+		for (x = 0; x < width; x++) {
+			if (drm_draw_is_pixel_fg(sbuf8, spitch, x / scale, y / scale)) {
+				unsigned int new_page;
+				unsigned int offset;
+
+				offset = (y + clip->y1) * dpitch + (x + clip->x1) * cpp;
+				new_page = offset >> PAGE_SHIFT;
+				offset = offset % PAGE_SIZE;
+				if (new_page != page) {
+					if (!pages[new_page])
+						continue;
+					if (vaddr)
+						kunmap_local(vaddr);
+					page = new_page;
+					vaddr = kmap_local_page_try_from_panic(pages[page]);
+				}
+				if (!vaddr)
+					continue;
+
+				// Special case for 24bit, as a pixel might cross page boundaries
+				if (cpp == 3 && offset + 3 > PAGE_SIZE)
+					drm_panic_helper_write_pixel24_xpage(vaddr,
+									     pages[page + 1],
+									     offset, fg32);
+				else
+					drm_panic_helper_write_pixel(vaddr, offset, fg32, cpp);
+			}
+		}
+	}
+	if (vaddr)
+		kunmap_local(vaddr);
+}
+
+/*
+ * drm_panic_helper_blit - convert a monochrome image to a linear framebuffer
+ * @sb: destination scanout buffer
+ * @clip: destination rectangle
+ * @sbuf8: source buffer, in monochrome format, 8 pixels per byte.
+ * @spitch: source pitch in bytes
+ * @scale: integer scale, source buffer is scale time smaller than destination
+ *         rectangle
+ * @fg_color: foreground color, in destination format
+ *
+ * This can be used to draw a font character, which is a monochrome image, to a
+ * framebuffer in other supported format.
+ */
+static void drm_panic_helper_blit(struct drm_scanout_buffer *sb, struct drm_rect *clip,
+				  const u8 *sbuf8, unsigned int spitch,
+				  unsigned int scale, u32 fg_color)
+
+{
+	struct iosys_map map;
+
+	if (sb->set_pixel)
+		return drm_panic_helper_blit_pixel(sb, clip, sbuf8, spitch, scale, fg_color);
+
+	if (sb->pages)
+		return drm_panic_helper_blit_page(sb->pages, sb->pitch[0], sb->format->cpp[0],
+						  sbuf8, spitch, clip, scale, fg_color);
+
+	map = sb->map[0];
+	iosys_map_incr(&map, clip->y1 * sb->pitch[0] + clip->x1 * sb->format->cpp[0]);
+
+	switch (sb->format->cpp[0]) {
+	case 2:
+		drm_draw_blit16(&map, sb->pitch[0], sbuf8, spitch,
+				drm_rect_height(clip), drm_rect_width(clip), scale, fg_color);
+	break;
+	case 3:
+		drm_draw_blit24(&map, sb->pitch[0], sbuf8, spitch,
+				drm_rect_height(clip), drm_rect_width(clip), scale, fg_color);
+	break;
+	case 4:
+		drm_draw_blit32(&map, sb->pitch[0], sbuf8, spitch,
+				drm_rect_height(clip), drm_rect_width(clip), scale, fg_color);
+	break;
+	default:
+		WARN_ONCE(1, "Can't blit with pixel width %d\n", sb->format->cpp[0]);
+	}
+}
+
+static void drm_panic_helper_fill_pixel(struct drm_scanout_buffer *sb,
+					struct drm_rect *clip,
+					u32 color)
+{
+	unsigned int y, x;
+
+	for (y = 0; y < drm_rect_height(clip); y++)
+		for (x = 0; x < drm_rect_width(clip); x++)
+			sb->set_pixel(sb, clip->x1 + x, clip->y1 + y, color);
+}
+
+static void drm_panic_helper_fill_page(struct page **pages, unsigned int dpitch,
+				       unsigned int cpp, struct drm_rect *clip,
+				       u32 color)
+{
+	unsigned int y, x;
+	unsigned int page = ~0;
+	void *vaddr = NULL;
+
+	for (y = clip->y1; y < clip->y2; y++) {
+		for (x = clip->x1; x < clip->x2; x++) {
+			unsigned int new_page;
+			unsigned int offset;
+
+			offset = y * dpitch + x * cpp;
+			new_page = offset >> PAGE_SHIFT;
+			offset = offset % PAGE_SIZE;
+			if (new_page != page) {
+				if (vaddr)
+					kunmap_local(vaddr);
+				page = new_page;
+				vaddr = kmap_local_page_try_from_panic(pages[page]);
+			}
+			if (!vaddr)
+				continue;
+
+			// Special case for 24bit, as a pixel might cross page boundaries
+			if (cpp == 3 && offset + 3 > PAGE_SIZE)
+				drm_panic_helper_write_pixel24_xpage(vaddr, pages[page + 1],
+								     offset, color);
+			else
+				drm_panic_helper_write_pixel(vaddr, offset, color, cpp);
+		}
+	}
+	if (vaddr)
+		kunmap_local(vaddr);
+}
+
+/*
+ * drm_panic_helper_fill - Fill a rectangle with a color
+ * @sb: destination scanout buffer
+ * @clip: destination rectangle
+ * @color: foreground color, in destination format
+ *
+ * Fill a rectangle with a color, in a linear framebuffer.
+ */
+static void drm_panic_helper_fill(struct drm_scanout_buffer *sb, struct drm_rect *clip,
+				  u32 color)
+{
+	struct iosys_map map;
+
+	if (sb->set_pixel)
+		return drm_panic_helper_fill_pixel(sb, clip, color);
+
+	if (sb->pages)
+		return drm_panic_helper_fill_page(sb->pages, sb->pitch[0], sb->format->cpp[0],
+						  clip, color);
+
+	map = sb->map[0];
+	iosys_map_incr(&map, clip->y1 * sb->pitch[0] + clip->x1 * sb->format->cpp[0]);
+
+	switch (sb->format->cpp[0]) {
+	case 2:
+		drm_draw_fill16(&map, sb->pitch[0], drm_rect_height(clip),
+				drm_rect_width(clip), color);
+	break;
+	case 3:
+		drm_draw_fill24(&map, sb->pitch[0], drm_rect_height(clip),
+				drm_rect_width(clip), color);
+	break;
+	case 4:
+		drm_draw_fill32(&map, sb->pitch[0], drm_rect_height(clip),
+				drm_rect_width(clip), color);
+	break;
+	default:
+		WARN_ONCE(1, "Can't fill with pixel width %d\n", sb->format->cpp[0]);
+	}
+}
+
+static unsigned int get_max_line_len(const struct drm_panic_line *lines, int len)
+{
+	int i;
+	unsigned int max = 0;
+
+	for (i = 0; i < len; i++)
+		max = max(lines[i].len, max);
+	return max;
+}
+
+/*
+ * Draw a text in a rectangle on a framebuffer. The text is truncated if it overflows the rectangle
+ */
+static void draw_txt_rectangle(struct drm_scanout_buffer *sb,
+			       const struct font_desc *font,
+			       const struct drm_panic_line *msg,
+			       unsigned int msg_lines,
+			       bool centered,
+			       struct drm_rect *clip,
+			       u32 color)
+{
+	int i, j;
+	const u8 *src;
+	size_t font_pitch = DIV_ROUND_UP(font->width, 8);
+	struct drm_rect rec;
+
+	msg_lines = min(msg_lines,  drm_rect_height(clip) / font->height);
+	for (i = 0; i < msg_lines; i++) {
+		size_t line_len = min(msg[i].len, drm_rect_width(clip) / font->width);
+
+		rec.y1 = clip->y1 +  i * font->height;
+		rec.y2 = rec.y1 + font->height;
+		rec.x1 = clip->x1;
+
+		if (centered)
+			rec.x1 += (drm_rect_width(clip) - (line_len * font->width)) / 2;
+
+		for (j = 0; j < line_len; j++) {
+			src = font_data_glyph_buf(font->data, font->width, font->height,
+						  (unsigned char)msg[i].txt[j]);
+			rec.x2 = rec.x1 + font->width;
+			if (src)
+				drm_panic_helper_blit(sb, &rec, src, font_pitch, 1, color);
+			rec.x1 += font->width;
+		}
+	}
+}
+
+static void drm_panic_helper_logo_rect(struct drm_rect *rect, const struct font_desc *font)
+{
+	if (logo_mono) {
+		drm_rect_init(rect, 0, 0, logo_mono->width, logo_mono->height);
+	} else {
+		int logo_width = get_max_line_len(logo_ascii, logo_ascii_lines) * font->width;
+
+		drm_rect_init(rect, 0, 0, logo_width, logo_ascii_lines * font->height);
+	}
+}
+
+static void drm_panic_helper_logo_draw(struct drm_scanout_buffer *sb, struct drm_rect *rect,
+				       const struct font_desc *font, u32 fg_color)
+{
+	if (rect->x2 > sb->width || rect->y2 > sb->height)
+		return;
+
+	if (logo_mono)
+		drm_panic_helper_blit(sb, rect, logo_mono->data,
+				      DIV_ROUND_UP(drm_rect_width(rect), 8), 1, fg_color);
+	else
+		draw_txt_rectangle(sb, font, logo_ascii, logo_ascii_lines, false, rect,
+				   fg_color);
+}
+
+VISIBLE_IF_KUNIT int drm_panic_helper_draw_screen_user(struct drm_scanout_buffer *sb,
+						       u32 fg_color, u32 bg_color)
+{
+	const struct font_desc *font = get_default_font(sb->width, sb->height, NULL, NULL);
+	struct drm_rect r_screen, r_logo, r_msg;
+	unsigned int msg_width, msg_height;
+
+	if (!font || font->width > sb->width || font->height > sb->height)
+		return -EINVAL;
+
+	fg_color = drm_draw_color_from_xrgb8888(fg_color, sb->format->format);
+	bg_color = drm_draw_color_from_xrgb8888(bg_color, sb->format->format);
+
+	r_screen = DRM_RECT_INIT(0, 0, sb->width, sb->height);
+	drm_panic_helper_logo_rect(&r_logo, font);
+
+	msg_width = min(get_max_line_len(panic_msg, panic_msg_lines) * font->width, sb->width);
+	msg_height = min(panic_msg_lines * font->height, sb->height);
+	r_msg = DRM_RECT_INIT(0, 0, msg_width, msg_height);
+
+	/* Center the panic message */
+	drm_rect_translate(&r_msg, (sb->width - r_msg.x2) / 2, (sb->height - r_msg.y2) / 2);
+
+	/* Fill with the background color, and draw text on top */
+	drm_panic_helper_fill(sb, &r_screen, bg_color);
+
+	if (!drm_rect_overlap(&r_logo, &r_msg))
+		drm_panic_helper_logo_draw(sb, &r_logo, font, fg_color);
+
+	draw_txt_rectangle(sb, font, panic_msg, panic_msg_lines, true, &r_msg, fg_color);
+
+	return 0;
+}
+EXPORT_SYMBOL_IF_KUNIT(drm_panic_helper_draw_screen_user);
+
+/*
+ * Draw one line of kmsg, and handle wrapping if it won't fit in the screen width.
+ * Return the y-offset of the next line.
+ */
+static int draw_line_with_wrap(struct drm_scanout_buffer *sb, const struct font_desc *font,
+			       struct drm_panic_line *line, int yoffset, u32 fg_color)
+{
+	int chars_per_row = sb->width / font->width;
+	struct drm_rect r_txt = DRM_RECT_INIT(0, yoffset, sb->width, font->height);
+	struct drm_panic_line line_wrap;
+
+	if (line->len > chars_per_row) {
+		line_wrap.len = line->len % chars_per_row;
+		line_wrap.txt = line->txt + line->len - line_wrap.len;
+		draw_txt_rectangle(sb, font, &line_wrap, 1, false, &r_txt, fg_color);
+		r_txt.y1 -= font->height;
+		if (r_txt.y1 < 0)
+			return r_txt.y1;
+		while (line_wrap.txt > line->txt) {
+			line_wrap.txt -= chars_per_row;
+			line_wrap.len = chars_per_row;
+			draw_txt_rectangle(sb, font, &line_wrap, 1, false, &r_txt, fg_color);
+			r_txt.y1 -= font->height;
+			if (r_txt.y1 < 0)
+				return r_txt.y1;
+		}
+	} else {
+		draw_txt_rectangle(sb, font, line, 1, false, &r_txt, fg_color);
+		r_txt.y1 -= font->height;
+	}
+	return r_txt.y1;
+}
+
+/*
+ * Draw the kmsg buffer to the screen, starting from the youngest message at the bottom,
+ * and going up until reaching the top of the screen.
+ */
+VISIBLE_IF_KUNIT int drm_panic_helper_draw_screen_kmsg(struct drm_scanout_buffer *sb,
+						       u32 fg_color, u32 bg_color)
+{
+	const struct font_desc *font = get_default_font(sb->width, sb->height, NULL, NULL);
+	struct drm_rect r_screen = DRM_RECT_INIT(0, 0, sb->width, sb->height);
+	struct kmsg_dump_iter iter;
+	char kmsg_buf[512];
+	size_t kmsg_len;
+	struct drm_panic_line line;
+	int yoffset;
+
+	if (!font || font->width > sb->width || font->height > sb->height)
+		return -EINVAL;
+
+	fg_color = drm_draw_color_from_xrgb8888(fg_color, sb->format->format);
+	bg_color = drm_draw_color_from_xrgb8888(bg_color, sb->format->format);
+
+	yoffset = sb->height - font->height - (sb->height % font->height) / 2;
+
+	/* Fill with the background color, and draw text on top */
+	drm_panic_helper_fill(sb, &r_screen, bg_color);
+
+	kmsg_dump_rewind(&iter);
+	while (kmsg_dump_get_buffer(&iter, false, kmsg_buf, sizeof(kmsg_buf), &kmsg_len)) {
+		char *start;
+		char *end;
+
+		/* ignore terminating NUL and newline */
+		start = kmsg_buf + kmsg_len - 2;
+		end = kmsg_buf + kmsg_len - 1;
+		while (start > kmsg_buf && yoffset >= 0) {
+			while (start > kmsg_buf && *start != '\n')
+				start--;
+			/* don't count the newline character */
+			line.txt = start + (start == kmsg_buf ? 0 : 1);
+			line.len = end - line.txt;
+
+			yoffset = draw_line_with_wrap(sb, font, &line, yoffset, fg_color);
+			end = start;
+			start--;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_IF_KUNIT(drm_panic_helper_draw_screen_kmsg);
+
+#if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
+/*
+ * It is unwise to allocate memory in the panic callback, so the buffers are
+ * pre-allocated. Only 2 buffers and the zlib workspace are needed.
+ * Two buffers are enough, using the following buffer usage:
+ * 1) kmsg messages are dumped in buffer1
+ * 2) kmsg is zlib-compressed into buffer2
+ * 3) compressed kmsg is encoded as QR-code Numeric stream in buffer1
+ * 4) QR-code image is generated in buffer2
+ * The Max QR code size is V40, 177x177, 4071 bytes for image, 2956 bytes for
+ * data segments.
+ *
+ * Typically, ~7500 bytes of kmsg, are compressed into 2800 bytes, which fits in
+ * a V40 QR-code (177x177).
+ *
+ * If CONFIG_DRM_PANIC_SCREEN_QR_CODE_URL is not set, the kmsg data will be put
+ * directly in the QR code.
+ * 1) kmsg messages are dumped in buffer1
+ * 2) kmsg message is encoded as byte stream in buffer2
+ * 3) QR-code image is generated in buffer1
+ */
+
+#define MAX_QR_DATA 2956
+#define MAX_ZLIB_RATIO 3
+#define QR_BUFFER1_SIZE (MAX_ZLIB_RATIO * MAX_QR_DATA) /* Must also be > 4071  */
+#define QR_BUFFER2_SIZE 4096
+#define QR_MARGIN	4	/* 4 modules of foreground color around the qr code */
+
+/* Compression parameters */
+#define COMPR_LEVEL 6
+#define WINDOW_BITS 12
+#define MEM_LEVEL 4
+
+static char *qrbuf1;
+static char *qrbuf2;
+static struct z_stream_s stream;
+
+static void __init drm_panic_helper_qr_init(void)
+{
+	qrbuf1 = kmalloc(QR_BUFFER1_SIZE, GFP_KERNEL);
+	qrbuf2 = kmalloc(QR_BUFFER2_SIZE, GFP_KERNEL);
+	stream.workspace = kmalloc(zlib_deflate_workspacesize(WINDOW_BITS, MEM_LEVEL),
+				   GFP_KERNEL);
+}
+
+static void drm_panic_helper_qr_exit(void)
+{
+	kfree(qrbuf1);
+	qrbuf1 = NULL;
+	kfree(qrbuf2);
+	qrbuf2 = NULL;
+	kfree(stream.workspace);
+	stream.workspace = NULL;
+}
+
+static int drm_panic_helper_get_qr_code_url(u8 **qr_image, unsigned int qr_version)
+{
+	struct kmsg_dump_iter iter;
+	char url[256];
+	size_t kmsg_len, max_kmsg_size;
+	char *kmsg;
+	int max_qr_data_size, url_len;
+
+	url_len = snprintf(url, sizeof(url), "%s?a=%s&v=%s&z=",
+			   CONFIG_DRM_PANIC_SCREEN_QR_CODE_URL,
+			   utsname()->machine, utsname()->release);
+
+	max_qr_data_size = drm_panic_helper_qr_max_data_size(qr_version, url_len);
+	max_kmsg_size = min(MAX_ZLIB_RATIO * max_qr_data_size, QR_BUFFER1_SIZE);
+
+	/* get kmsg to buffer 1 */
+	kmsg_dump_rewind(&iter);
+	kmsg_dump_get_buffer(&iter, false, qrbuf1, max_kmsg_size, &kmsg_len);
+
+	if (!kmsg_len)
+		return -ENODATA;
+	kmsg = qrbuf1;
+
+try_again:
+	if (zlib_deflateInit2(&stream, COMPR_LEVEL, Z_DEFLATED, WINDOW_BITS,
+			      MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK)
+		return -EINVAL;
+
+	stream.next_in = kmsg;
+	stream.avail_in = kmsg_len;
+	stream.total_in = 0;
+	stream.next_out = qrbuf2;
+	stream.avail_out = QR_BUFFER2_SIZE;
+	stream.total_out = 0;
+
+	if (zlib_deflate(&stream, Z_FINISH) != Z_STREAM_END)
+		return -EINVAL;
+
+	if (zlib_deflateEnd(&stream) != Z_OK)
+		return -EINVAL;
+
+	if (stream.total_out > max_qr_data_size) {
+		/* too much data for the QR code, so skip the first line and try again */
+		kmsg = strchr(kmsg, '\n');
+		if (!kmsg)
+			return -EINVAL;
+		/* skip the first \n */
+		kmsg += 1;
+		kmsg_len = strlen(kmsg);
+		goto try_again;
+	}
+	*qr_image = qrbuf2;
+
+	/* generate qr code image in buffer2 */
+	return drm_panic_helper_qr_generate(url, qrbuf2, stream.total_out, QR_BUFFER2_SIZE,
+					    qrbuf1, QR_BUFFER1_SIZE);
+}
+
+static int drm_panic_helper_get_qr_code_raw(u8 **qr_image, unsigned int qr_version)
+{
+	struct kmsg_dump_iter iter;
+	size_t kmsg_len;
+	size_t max_kmsg_size = min(drm_panic_helper_qr_max_data_size(qr_version, 0),
+				   QR_BUFFER1_SIZE);
+
+	kmsg_dump_rewind(&iter);
+	kmsg_dump_get_buffer(&iter, false, qrbuf1, max_kmsg_size, &kmsg_len);
+	if (!kmsg_len)
+		return -ENODATA;
+
+	*qr_image = qrbuf1;
+	return drm_panic_helper_qr_generate(NULL, qrbuf1, kmsg_len, QR_BUFFER1_SIZE,
+					    qrbuf2, QR_BUFFER2_SIZE);
+}
+
+static int drm_panic_helper_get_qr_code(u8 **qr_image, unsigned int qr_version)
+{
+	if (strlen(CONFIG_DRM_PANIC_SCREEN_QR_CODE_URL) > 0)
+		return drm_panic_helper_get_qr_code_url(qr_image, qr_version);
+	else
+		return drm_panic_helper_get_qr_code_raw(qr_image, qr_version);
+}
+
+/*
+ * Draw the panic message at the center of the screen, with a QR Code
+ */
+VISIBLE_IF_KUNIT int drm_panic_helper_draw_screen_qr_code(struct drm_scanout_buffer *sb,
+							  u32 fg_color, u32 bg_color,
+							  unsigned int qr_version)
+{
+	const struct font_desc *font = get_default_font(sb->width, sb->height, NULL, NULL);
+	struct drm_rect r_screen, r_logo, r_msg, r_qr, r_qr_canvas;
+	unsigned int max_qr_size, scale;
+	unsigned int msg_width, msg_height;
+	int qr_width, qr_canvas_width, qr_pitch, v_margin;
+	u8 *qr_image;
+
+	if (!qrbuf1 || !qrbuf2 || !stream.workspace)
+		return -ENOMEM;
+	if (!font)
+		return -EINVAL;
+
+	fg_color = drm_draw_color_from_xrgb8888(fg_color, sb->format->format);
+	bg_color = drm_draw_color_from_xrgb8888(bg_color, sb->format->format);
+
+	r_screen = DRM_RECT_INIT(0, 0, sb->width, sb->height);
+
+	drm_panic_helper_logo_rect(&r_logo, font);
+
+	msg_width = min(get_max_line_len(panic_msg, panic_msg_lines) * font->width, sb->width);
+	msg_height = min(panic_msg_lines * font->height, sb->height);
+	r_msg = DRM_RECT_INIT(0, 0, msg_width, msg_height);
+
+	max_qr_size = min(3 * sb->width / 4, 3 * sb->height / 4);
+
+	qr_width = drm_panic_helper_get_qr_code(&qr_image, qr_version);
+	if (qr_width < 0)
+		return qr_width;
+	else if (!qr_width)
+		return -ENOSPC;
+
+	qr_canvas_width = qr_width + QR_MARGIN * 2;
+	scale = max_qr_size / qr_canvas_width;
+	/* QR code is not readable if not scaled at least by 2 */
+	if (scale < 2)
+		return -ENOSPC;
+
+	pr_debug("QR width %d and scale %d\n", qr_width, scale);
+	r_qr_canvas = DRM_RECT_INIT(0, 0, qr_canvas_width * scale, qr_canvas_width * scale);
+
+	v_margin = sb->height - drm_rect_height(&r_qr_canvas) - drm_rect_height(&r_msg);
+	if (v_margin < 0)
+		return -ENOSPC;
+	v_margin /= 5;
+
+	drm_rect_translate(&r_qr_canvas, (sb->width - r_qr_canvas.x2) / 2, 2 * v_margin);
+	r_qr = DRM_RECT_INIT(r_qr_canvas.x1 + QR_MARGIN * scale, r_qr_canvas.y1 + QR_MARGIN * scale,
+			     qr_width * scale, qr_width * scale);
+
+	/* Center the panic message */
+	drm_rect_translate(&r_msg, (sb->width - r_msg.x2) / 2,
+			   3 * v_margin + drm_rect_height(&r_qr_canvas));
+
+	/* Fill with the background color, and draw text on top */
+	drm_panic_helper_fill(sb, &r_screen, bg_color);
+
+	if (!drm_rect_overlap(&r_logo, &r_msg) && !drm_rect_overlap(&r_logo, &r_qr_canvas))
+		drm_panic_helper_logo_draw(sb, &r_logo, font, fg_color);
+
+	draw_txt_rectangle(sb, font, panic_msg, panic_msg_lines, true, &r_msg, fg_color);
+
+	/* Draw the qr code */
+	qr_pitch = DIV_ROUND_UP(qr_width, 8);
+	drm_panic_helper_fill(sb, &r_qr_canvas, fg_color);
+	drm_panic_helper_fill(sb, &r_qr, bg_color);
+	drm_panic_helper_blit(sb, &r_qr, qr_image, qr_pitch, scale, fg_color);
+	return 0;
+}
+EXPORT_SYMBOL_IF_KUNIT(drm_panic_helper_draw_screen_qr_code);
+#else
+static void __init drm_panic_helper_qr_init(void) { };
+static void __exit drm_panic_helper_qr_exit(void) { };
+#endif
+
+/*
+ * drm_panic_helper_format_is_supported()
+ * @format: a fourcc color code
+ * Returns: true if supported, false otherwise.
+ *
+ * Check if drm_panic will be able to use this color format.
+ */
+static bool drm_panic_helper_format_is_supported(const struct drm_format_info *format)
+{
+	if (format->num_planes != 1)
+		return false;
+	return drm_draw_can_convert_from_xrgb8888(format->format);
+}
+
+static int draw_panic_helper_dispatch(struct drm_scanout_buffer *sb,
+				      enum drm_panic_type panic_type,
+				      u32 fg_color, u32 bg_color,
+				      unsigned int qr_version)
+{
+	int ret;
+
+retry:
+	switch (panic_type) {
+	case DRM_PANIC_TYPE_KMSG:
+		ret = drm_panic_helper_draw_screen_kmsg(sb, fg_color, bg_color);
+		if (ret) {
+			panic_type = DRM_PANIC_TYPE_USER;
+			goto retry;
+		}
+		break;
+#if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
+	case DRM_PANIC_TYPE_QR:
+		ret = drm_panic_helper_draw_screen_qr_code(sb, fg_color, bg_color, qr_version);
+		if (ret) {
+			panic_type = DRM_PANIC_TYPE_USER;
+			goto retry;
+		}
+		break;
+#endif
+	case DRM_PANIC_TYPE_USER:
+	default:
+		ret = drm_panic_helper_draw_screen_user(sb, fg_color, bg_color);
+	}
+
+	return ret;
+}
+
+VISIBLE_IF_KUNIT void drm_panic_helper_set_description(const char *description)
+{
+	u32 len;
+
+	if (description) {
+		struct drm_panic_line *desc_line = &panic_msg[panic_msg_lines - 1];
+
+		desc_line->txt = description;
+		len = strlen(description);
+		/* ignore the last newline character */
+		if (len && description[len - 1] == '\n')
+			len -= 1;
+		desc_line->len = len;
+	}
+}
+EXPORT_SYMBOL_IF_KUNIT(drm_panic_helper_set_description);
+
+VISIBLE_IF_KUNIT void drm_panic_helper_clear_description(void)
+{
+	struct drm_panic_line *desc_line = &panic_msg[panic_msg_lines - 1];
+
+	desc_line->len = 0;
+	desc_line->txt = NULL;
+}
+EXPORT_SYMBOL_IF_KUNIT(drm_panic_helper_clear_description);
+
+/**
+ * drm_plane_helper_display_panic_screen - Displays a panic screen according to the given settings
+ * @plane: the DRM plane to display to
+ * @description: error message to display
+ * @panic_type: type of panic screen
+ * @fg_color: text foreground color
+ * @bg_color: text background color
+ * @qr_version: version of the QR code, if any
+ *
+ * This helper display a panic screen on common primary planes. The panic
+ * screen can either display a kernel message, a user message or a QR code.
+ *
+ * The helper uses struct drm_plane_helper_funcs.get_scanout_buffer, where
+ * the plane can provide a scanout buffer that the panic handler can draw to.
+ * Currently only linear buffer and a few color formats are supported.
+ *
+ * Optionally the plane can also provide a &drm_plane_helper_funcs.panic_flush
+ * callback, which the DRM panic handler calls after drawing to send additional
+ * commands to the hardware to make the scanout buffer visible.
+ */
+int drm_plane_helper_display_panic_screen(struct drm_plane *plane, const char *description,
+					  enum drm_panic_type panic_type,
+					  u32 fg_color, u32 bg_color, unsigned int qr_version)
+{
+	struct drm_scanout_buffer sb = { };
+	int ret;
+
+	if (!plane->helper_private || !plane->helper_private->get_scanout_buffer)
+		return -EINVAL;
+
+	ret = plane->helper_private->get_scanout_buffer(plane, &sb);
+	if (ret)
+		return ret;
+
+	if (!drm_panic_helper_format_is_supported(sb.format))
+		return -EINVAL;
+
+	/* One of these should be set, or it can't draw pixels */
+	if (!sb.set_pixel && !sb.pages && iosys_map_is_null(&sb.map[0]))
+		return -EINVAL;
+
+	drm_panic_helper_set_description(description);
+
+	ret = draw_panic_helper_dispatch(&sb, panic_type, fg_color, bg_color, qr_version);
+	if (!ret) {
+		/*
+		 * Only flush if we have a panic screen to display. Otherwise
+		 * it's probably better to leave the display output as-is.
+		 */
+		if (plane->helper_private->panic_flush)
+			plane->helper_private->panic_flush(plane);
+	}
+
+	drm_panic_helper_clear_description();
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_plane_helper_display_panic_screen);
+
+int __init drm_panic_helper_init(void)
+{
+	drm_panic_helper_setup_logo();
+	drm_panic_helper_qr_init();
+
+	return 0;
+}
+
+void __exit drm_panic_helper_exit(void)
+{
+	drm_panic_helper_qr_exit();
+}
diff --git a/drivers/gpu/drm/drm_panic_helper_qr.rs b/drivers/gpu/drm/drm_panic_helper_qr.rs
new file mode 100644
index 00000000000000..4941731fb60fa9
--- /dev/null
+++ b/drivers/gpu/drm/drm_panic_helper_qr.rs
@@ -0,0 +1,1016 @@
+// SPDX-License-Identifier: MIT
+
+//! This is a simple QR encoder for DRM panic.
+//!
+//! It is called from a panic handler, so it should't allocate memory and
+//! does all the work on the stack or on the provided buffers. For
+//! simplification, it only supports low error correction, and applies the
+//! first mask (checkerboard). It will draw the smallest QR code that can
+//! contain the string passed as parameter. To get the most compact
+//! QR code, the start of the URL is encoded as binary, and the
+//! compressed kmsg is encoded as numeric.
+//!
+//! The binary data must be a valid URL parameter, so the easiest way is
+//! to use base64 encoding. But this wastes 25% of data space, so the
+//! whole stack trace won't fit in the QR code. So instead it encodes
+//! every 7 bytes of input into 17 decimal digits, and then uses the
+//! efficient numeric encoding, that encode 3 decimal digits into
+//! 10bits. This makes 168bits of compressed data into 51 decimal digits,
+//! into 170bits in the QR code, so wasting only 1.17%. And the numbers are
+//! valid URL parameter, so the website can do the reverse, to get the
+//! binary data. This is the same algorithm used by Fido v2.2 QR-initiated
+//! authentication specification.
+//!
+//! Inspired by these 3 projects, all under MIT license:
+//!
+//! * <https://github.com/kennytm/qrcode-rust>
+//! * <https://github.com/erwanvivien/fast_qr>
+//! * <https://github.com/bjguillot/qr>
+
+use kernel::prelude::*;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
+struct Version(usize);
+
+// Generator polynomials for ECC, only those that are needed for low quality.
+const P7: [u8; 7] = [87, 229, 146, 149, 238, 102, 21];
+const P10: [u8; 10] = [251, 67, 46, 61, 118, 70, 64, 94, 32, 45];
+const P15: [u8; 15] = [
+    8, 183, 61, 91, 202, 37, 51, 58, 58, 237, 140, 124, 5, 99, 105,
+];
+const P18: [u8; 18] = [
+    215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148, 252, 179, 5, 98, 96, 153,
+];
+const P20: [u8; 20] = [
+    17, 60, 79, 50, 61, 163, 26, 187, 202, 180, 221, 225, 83, 239, 156, 164, 212, 212, 188, 190,
+];
+const P22: [u8; 22] = [
+    210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8, 172, 98, 80, 219, 134, 160, 105,
+    165, 231,
+];
+const P24: [u8; 24] = [
+    229, 121, 135, 48, 211, 117, 251, 126, 159, 180, 169, 152, 192, 226, 228, 218, 111, 0, 117,
+    232, 87, 96, 227, 21,
+];
+const P26: [u8; 26] = [
+    173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165, 53, 161, 21, 245, 142, 13, 102,
+    48, 227, 153, 145, 218, 70,
+];
+const P28: [u8; 28] = [
+    168, 223, 200, 104, 224, 234, 108, 180, 110, 190, 195, 147, 205, 27, 232, 201, 21, 43, 245, 87,
+    42, 195, 212, 119, 242, 37, 9, 123,
+];
+const P30: [u8; 30] = [
+    41, 173, 145, 152, 216, 31, 179, 182, 50, 48, 110, 86, 239, 96, 222, 125, 42, 173, 226, 193,
+    224, 130, 156, 37, 251, 216, 238, 40, 192, 180,
+];
+
+/// QR Code parameters for Low quality ECC:
+/// - Error Correction polynomial.
+/// - Number of blocks in group 1.
+/// - Number of blocks in group 2.
+/// - Block size in group 1.
+///
+/// (Block size in group 2 is one more than group 1).
+struct VersionParameter(&'static [u8], u8, u8, u8);
+const VPARAM: [VersionParameter; 40] = [
+    VersionParameter(&P7, 1, 0, 19),    // V1
+    VersionParameter(&P10, 1, 0, 34),   // V2
+    VersionParameter(&P15, 1, 0, 55),   // V3
+    VersionParameter(&P20, 1, 0, 80),   // V4
+    VersionParameter(&P26, 1, 0, 108),  // V5
+    VersionParameter(&P18, 2, 0, 68),   // V6
+    VersionParameter(&P20, 2, 0, 78),   // V7
+    VersionParameter(&P24, 2, 0, 97),   // V8
+    VersionParameter(&P30, 2, 0, 116),  // V9
+    VersionParameter(&P18, 2, 2, 68),   // V10
+    VersionParameter(&P20, 4, 0, 81),   // V11
+    VersionParameter(&P24, 2, 2, 92),   // V12
+    VersionParameter(&P26, 4, 0, 107),  // V13
+    VersionParameter(&P30, 3, 1, 115),  // V14
+    VersionParameter(&P22, 5, 1, 87),   // V15
+    VersionParameter(&P24, 5, 1, 98),   // V16
+    VersionParameter(&P28, 1, 5, 107),  // V17
+    VersionParameter(&P30, 5, 1, 120),  // V18
+    VersionParameter(&P28, 3, 4, 113),  // V19
+    VersionParameter(&P28, 3, 5, 107),  // V20
+    VersionParameter(&P28, 4, 4, 116),  // V21
+    VersionParameter(&P28, 2, 7, 111),  // V22
+    VersionParameter(&P30, 4, 5, 121),  // V23
+    VersionParameter(&P30, 6, 4, 117),  // V24
+    VersionParameter(&P26, 8, 4, 106),  // V25
+    VersionParameter(&P28, 10, 2, 114), // V26
+    VersionParameter(&P30, 8, 4, 122),  // V27
+    VersionParameter(&P30, 3, 10, 117), // V28
+    VersionParameter(&P30, 7, 7, 116),  // V29
+    VersionParameter(&P30, 5, 10, 115), // V30
+    VersionParameter(&P30, 13, 3, 115), // V31
+    VersionParameter(&P30, 17, 0, 115), // V32
+    VersionParameter(&P30, 17, 1, 115), // V33
+    VersionParameter(&P30, 13, 6, 115), // V34
+    VersionParameter(&P30, 12, 7, 121), // V35
+    VersionParameter(&P30, 6, 14, 121), // V36
+    VersionParameter(&P30, 17, 4, 122), // V37
+    VersionParameter(&P30, 4, 18, 122), // V38
+    VersionParameter(&P30, 20, 4, 117), // V39
+    VersionParameter(&P30, 19, 6, 118), // V40
+];
+
+const MAX_EC_SIZE: usize = 30;
+const MAX_BLK_SIZE: usize = 123;
+
+/// Position of the alignment pattern grid.
+const ALIGNMENT_PATTERNS: [&[u8]; 40] = [
+    &[],
+    &[6, 18],
+    &[6, 22],
+    &[6, 26],
+    &[6, 30],
+    &[6, 34],
+    &[6, 22, 38],
+    &[6, 24, 42],
+    &[6, 26, 46],
+    &[6, 28, 50],
+    &[6, 30, 54],
+    &[6, 32, 58],
+    &[6, 34, 62],
+    &[6, 26, 46, 66],
+    &[6, 26, 48, 70],
+    &[6, 26, 50, 74],
+    &[6, 30, 54, 78],
+    &[6, 30, 56, 82],
+    &[6, 30, 58, 86],
+    &[6, 34, 62, 90],
+    &[6, 28, 50, 72, 94],
+    &[6, 26, 50, 74, 98],
+    &[6, 30, 54, 78, 102],
+    &[6, 28, 54, 80, 106],
+    &[6, 32, 58, 84, 110],
+    &[6, 30, 58, 86, 114],
+    &[6, 34, 62, 90, 118],
+    &[6, 26, 50, 74, 98, 122],
+    &[6, 30, 54, 78, 102, 126],
+    &[6, 26, 52, 78, 104, 130],
+    &[6, 30, 56, 82, 108, 134],
+    &[6, 34, 60, 86, 112, 138],
+    &[6, 30, 58, 86, 114, 142],
+    &[6, 34, 62, 90, 118, 146],
+    &[6, 30, 54, 78, 102, 126, 150],
+    &[6, 24, 50, 76, 102, 128, 154],
+    &[6, 28, 54, 80, 106, 132, 158],
+    &[6, 32, 58, 84, 110, 136, 162],
+    &[6, 26, 54, 82, 110, 138, 166],
+    &[6, 30, 58, 86, 114, 142, 170],
+];
+
+/// Version information for format V7-V40.
+const VERSION_INFORMATION: [u32; 34] = [
+    0b00_0111_1100_1001_0100,
+    0b00_1000_0101_1011_1100,
+    0b00_1001_1010_1001_1001,
+    0b00_1010_0100_1101_0011,
+    0b00_1011_1011_1111_0110,
+    0b00_1100_0111_0110_0010,
+    0b00_1101_1000_0100_0111,
+    0b00_1110_0110_0000_1101,
+    0b00_1111_1001_0010_1000,
+    0b01_0000_1011_0111_1000,
+    0b01_0001_0100_0101_1101,
+    0b01_0010_1010_0001_0111,
+    0b01_0011_0101_0011_0010,
+    0b01_0100_1001_1010_0110,
+    0b01_0101_0110_1000_0011,
+    0b01_0110_1000_1100_1001,
+    0b01_0111_0111_1110_1100,
+    0b01_1000_1110_1100_0100,
+    0b01_1001_0001_1110_0001,
+    0b01_1010_1111_1010_1011,
+    0b01_1011_0000_1000_1110,
+    0b01_1100_1100_0001_1010,
+    0b01_1101_0011_0011_1111,
+    0b01_1110_1101_0111_0101,
+    0b01_1111_0010_0101_0000,
+    0b10_0000_1001_1101_0101,
+    0b10_0001_0110_1111_0000,
+    0b10_0010_1000_1011_1010,
+    0b10_0011_0111_1001_1111,
+    0b10_0100_1011_0000_1011,
+    0b10_0101_0100_0010_1110,
+    0b10_0110_1010_0110_0100,
+    0b10_0111_0101_0100_0001,
+    0b10_1000_1100_0110_1001,
+];
+
+/// Format info for low quality ECC.
+const FORMAT_INFOS_QR_L: [u16; 8] = [
+    0x77c4, 0x72f3, 0x7daa, 0x789d, 0x662f, 0x6318, 0x6c41, 0x6976,
+];
+
+impl Version {
+    /// Returns the smallest QR version than can hold these segments.
+    fn from_segments(segments: &[&Segment<'_>]) -> Option<Version> {
+        (1..=40)
+            .map(Version)
+            .find(|&v| v.max_data() * 8 >= segments.iter().map(|s| s.total_size_bits(v)).sum())
+    }
+
+    fn width(&self) -> u8 {
+        (self.0 as u8) * 4 + 17
+    }
+
+    fn max_data(&self) -> usize {
+        self.g1_blk_size() * self.g1_blocks() + (self.g1_blk_size() + 1) * self.g2_blocks()
+    }
+
+    fn ec_size(&self) -> usize {
+        VPARAM[self.0 - 1].0.len()
+    }
+
+    fn g1_blocks(&self) -> usize {
+        VPARAM[self.0 - 1].1 as usize
+    }
+
+    fn g2_blocks(&self) -> usize {
+        VPARAM[self.0 - 1].2 as usize
+    }
+
+    fn g1_blk_size(&self) -> usize {
+        VPARAM[self.0 - 1].3 as usize
+    }
+
+    fn alignment_pattern(&self) -> &'static [u8] {
+        ALIGNMENT_PATTERNS[self.0 - 1]
+    }
+
+    fn poly(&self) -> &'static [u8] {
+        VPARAM[self.0 - 1].0
+    }
+
+    fn version_info(&self) -> u32 {
+        if *self >= Version(7) {
+            VERSION_INFORMATION[self.0 - 7]
+        } else {
+            0
+        }
+    }
+}
+
+/// Exponential table for Galois Field GF(256).
+const EXP_TABLE: [u8; 256] = [
+    1, 2, 4, 8, 16, 32, 64, 128, 29, 58, 116, 232, 205, 135, 19, 38, 76, 152, 45, 90, 180, 117,
+    234, 201, 143, 3, 6, 12, 24, 48, 96, 192, 157, 39, 78, 156, 37, 74, 148, 53, 106, 212, 181,
+    119, 238, 193, 159, 35, 70, 140, 5, 10, 20, 40, 80, 160, 93, 186, 105, 210, 185, 111, 222, 161,
+    95, 190, 97, 194, 153, 47, 94, 188, 101, 202, 137, 15, 30, 60, 120, 240, 253, 231, 211, 187,
+    107, 214, 177, 127, 254, 225, 223, 163, 91, 182, 113, 226, 217, 175, 67, 134, 17, 34, 68, 136,
+    13, 26, 52, 104, 208, 189, 103, 206, 129, 31, 62, 124, 248, 237, 199, 147, 59, 118, 236, 197,
+    151, 51, 102, 204, 133, 23, 46, 92, 184, 109, 218, 169, 79, 158, 33, 66, 132, 21, 42, 84, 168,
+    77, 154, 41, 82, 164, 85, 170, 73, 146, 57, 114, 228, 213, 183, 115, 230, 209, 191, 99, 198,
+    145, 63, 126, 252, 229, 215, 179, 123, 246, 241, 255, 227, 219, 171, 75, 150, 49, 98, 196, 149,
+    55, 110, 220, 165, 87, 174, 65, 130, 25, 50, 100, 200, 141, 7, 14, 28, 56, 112, 224, 221, 167,
+    83, 166, 81, 162, 89, 178, 121, 242, 249, 239, 195, 155, 43, 86, 172, 69, 138, 9, 18, 36, 72,
+    144, 61, 122, 244, 245, 247, 243, 251, 235, 203, 139, 11, 22, 44, 88, 176, 125, 250, 233, 207,
+    131, 27, 54, 108, 216, 173, 71, 142, 1,
+];
+
+/// Reverse exponential table for Galois Field GF(256).
+const LOG_TABLE: [u8; 256] = [
+    175, 0, 1, 25, 2, 50, 26, 198, 3, 223, 51, 238, 27, 104, 199, 75, 4, 100, 224, 14, 52, 141,
+    239, 129, 28, 193, 105, 248, 200, 8, 76, 113, 5, 138, 101, 47, 225, 36, 15, 33, 53, 147, 142,
+    218, 240, 18, 130, 69, 29, 181, 194, 125, 106, 39, 249, 185, 201, 154, 9, 120, 77, 228, 114,
+    166, 6, 191, 139, 98, 102, 221, 48, 253, 226, 152, 37, 179, 16, 145, 34, 136, 54, 208, 148,
+    206, 143, 150, 219, 189, 241, 210, 19, 92, 131, 56, 70, 64, 30, 66, 182, 163, 195, 72, 126,
+    110, 107, 58, 40, 84, 250, 133, 186, 61, 202, 94, 155, 159, 10, 21, 121, 43, 78, 212, 229, 172,
+    115, 243, 167, 87, 7, 112, 192, 247, 140, 128, 99, 13, 103, 74, 222, 237, 49, 197, 254, 24,
+    227, 165, 153, 119, 38, 184, 180, 124, 17, 68, 146, 217, 35, 32, 137, 46, 55, 63, 209, 91, 149,
+    188, 207, 205, 144, 135, 151, 178, 220, 252, 190, 97, 242, 86, 211, 171, 20, 42, 93, 158, 132,
+    60, 57, 83, 71, 109, 65, 162, 31, 45, 67, 216, 183, 123, 164, 118, 196, 23, 73, 236, 127, 12,
+    111, 246, 108, 161, 59, 82, 41, 157, 85, 170, 251, 96, 134, 177, 187, 204, 62, 90, 203, 89, 95,
+    176, 156, 169, 160, 81, 11, 245, 22, 235, 122, 117, 44, 215, 79, 174, 213, 233, 230, 231, 173,
+    232, 116, 214, 244, 234, 168, 80, 88, 175,
+];
+
+// 4 bits segment header.
+const MODE_STOP: u16 = 0;
+const MODE_NUMERIC: u16 = 1;
+const MODE_BINARY: u16 = 4;
+/// Padding bytes.
+const PADDING: [u8; 2] = [236, 17];
+
+/// Number of bits to encode characters in numeric mode.
+const NUM_CHARS_BITS: [usize; 4] = [0, 4, 7, 10];
+/// Number of decimal digits required to encode n bytes of binary data.
+/// eg: you need 15 decimal digits to fit 6 bytes of binary data.
+const BYTES_TO_DIGITS: [usize; 8] = [0, 3, 5, 8, 10, 13, 15, 17];
+
+enum Segment<'a> {
+    Numeric(&'a [u8]),
+    Binary(&'a [u8]),
+}
+
+impl Segment<'_> {
+    fn get_header(&self) -> (u16, usize) {
+        match self {
+            Segment::Binary(_) => (MODE_BINARY, 4),
+            Segment::Numeric(_) => (MODE_NUMERIC, 4),
+        }
+    }
+
+    /// Returns the size of the length field in bits, depending on QR Version.
+    fn length_bits_count(&self, version: Version) -> usize {
+        let Version(v) = version;
+        match self {
+            Segment::Binary(_) => match v {
+                1..=9 => 8,
+                _ => 16,
+            },
+            Segment::Numeric(_) => match v {
+                1..=9 => 10,
+                10..=26 => 12,
+                _ => 14,
+            },
+        }
+    }
+
+    /// Number of characters in the segment.
+    fn character_count(&self) -> usize {
+        match self {
+            Segment::Binary(data) => data.len(),
+            Segment::Numeric(data) => {
+                let last_chars = BYTES_TO_DIGITS[data.len() % 7];
+                // 17 decimal numbers per 7bytes + remainder.
+                17 * (data.len() / 7) + last_chars
+            }
+        }
+    }
+
+    fn get_length_field(&self, version: Version) -> (u16, usize) {
+        (
+            self.character_count() as u16,
+            self.length_bits_count(version),
+        )
+    }
+
+    fn total_size_bits(&self, version: Version) -> usize {
+        let data_size = match self {
+            Segment::Binary(data) => data.len() * 8,
+            Segment::Numeric(_) => {
+                let digits = self.character_count();
+                10 * (digits / 3) + NUM_CHARS_BITS[digits % 3]
+            }
+        };
+        // header + length + data.
+        4 + self.length_bits_count(version) + data_size
+    }
+
+    fn iter(&self) -> SegmentIterator<'_> {
+        SegmentIterator {
+            segment: self,
+            offset: 0,
+            decfifo: Default::default(),
+        }
+    }
+}
+
+/// Max fifo size is 17 (max push) + 2 (max remaining)
+const MAX_FIFO_SIZE: usize = 19;
+
+/// A simple Decimal digit FIFO
+#[derive(Default)]
+struct DecFifo {
+    decimals: [u8; MAX_FIFO_SIZE],
+    len: usize,
+}
+
+// On arm32 architecture, dividing an `u64` by a constant will generate a call
+// to `__aeabi_uldivmod` which is not present in the kernel.
+// So use the multiply by inverse method for this architecture.
+fn div10(val: u64) -> u64 {
+    if cfg!(target_arch = "arm") {
+        let val_h = val >> 32;
+        let val_l = val & 0xFFFFFFFF;
+        let b_h: u64 = 0x66666666;
+        let b_l: u64 = 0x66666667;
+
+        let tmp1 = val_h * b_l + ((val_l * b_l) >> 32);
+        let tmp2 = val_l * b_h + (tmp1 & 0xffffffff);
+        let tmp3 = val_h * b_h + (tmp1 >> 32) + (tmp2 >> 32);
+
+        tmp3 >> 2
+    } else {
+        val / 10
+    }
+}
+
+impl DecFifo {
+    fn push(&mut self, data: u64, len: usize) {
+        let mut chunk = data;
+        for i in (0..self.len).rev() {
+            self.decimals[i + len] = self.decimals[i];
+        }
+        for decimal in &mut self.decimals[..len] {
+            *decimal = (chunk % 10) as u8;
+            chunk = div10(chunk);
+        }
+        self.len += len;
+    }
+
+    /// Pop 3 decimal digits from the FIFO
+    fn pop3(&mut self) -> Option<(u16, usize)> {
+        if self.len == 0 {
+            None
+        } else {
+            let poplen = 3.min(self.len);
+            self.len -= poplen;
+            let mut out = 0;
+            let mut exp = 1;
+            for i in 0..poplen {
+                out += u16::from(self.decimals[self.len + i]) * exp;
+                exp *= 10;
+            }
+            Some((out, NUM_CHARS_BITS[poplen]))
+        }
+    }
+}
+
+struct SegmentIterator<'a> {
+    segment: &'a Segment<'a>,
+    offset: usize,
+    decfifo: DecFifo,
+}
+
+impl Iterator for SegmentIterator<'_> {
+    type Item = (u16, usize);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.segment {
+            Segment::Binary(data) => {
+                if self.offset < data.len() {
+                    let byte = u16::from(data[self.offset]);
+                    self.offset += 1;
+                    Some((byte, 8))
+                } else {
+                    None
+                }
+            }
+            Segment::Numeric(data) => {
+                if self.decfifo.len < 3 && self.offset < data.len() {
+                    // If there are less than 3 decimal digits in the fifo,
+                    // take the next 7 bytes of input, and push them to the fifo.
+                    let mut buf = [0u8; 8];
+                    let len = 7.min(data.len() - self.offset);
+                    buf[..len].copy_from_slice(&data[self.offset..self.offset + len]);
+                    let chunk = u64::from_le_bytes(buf);
+                    self.decfifo.push(chunk, BYTES_TO_DIGITS[len]);
+                    self.offset += len;
+                }
+                self.decfifo.pop3()
+            }
+        }
+    }
+}
+
+struct EncodedMsg<'a> {
+    data: &'a mut [u8],
+    ec_size: usize,
+    g1_blocks: usize,
+    g2_blocks: usize,
+    g1_blk_size: usize,
+    g2_blk_size: usize,
+    poly: &'static [u8],
+    version: Version,
+}
+
+/// Data to be put in the QR code, with correct segment encoding, padding, and
+/// Error Code Correction.
+impl EncodedMsg<'_> {
+    fn new<'a>(segments: &[&Segment<'_>], data: &'a mut [u8]) -> Option<EncodedMsg<'a>> {
+        let version = Version::from_segments(segments)?;
+        let ec_size = version.ec_size();
+        let g1_blocks = version.g1_blocks();
+        let g2_blocks = version.g2_blocks();
+        let g1_blk_size = version.g1_blk_size();
+        let g2_blk_size = g1_blk_size + 1;
+        let poly = version.poly();
+
+        // clear the output.
+        data.fill(0);
+
+        let mut em = EncodedMsg {
+            data,
+            ec_size,
+            g1_blocks,
+            g2_blocks,
+            g1_blk_size,
+            g2_blk_size,
+            poly,
+            version,
+        };
+        em.encode(segments);
+        Some(em)
+    }
+
+    /// Push bits of data at an offset (in bits).
+    fn push(&mut self, offset: &mut usize, bits: (u16, usize)) {
+        let (number, len_bits) = bits;
+        let byte_off = *offset / 8;
+        let bit_off = *offset % 8;
+        let b = bit_off + len_bits;
+
+        match (bit_off, b) {
+            (0, 0..=8) => {
+                self.data[byte_off] = (number << (8 - b)) as u8;
+            }
+            (0, _) => {
+                self.data[byte_off] = (number >> (b - 8)) as u8;
+                self.data[byte_off + 1] = (number << (16 - b)) as u8;
+            }
+            (_, 0..=8) => {
+                self.data[byte_off] |= (number << (8 - b)) as u8;
+            }
+            (_, 9..=16) => {
+                self.data[byte_off] |= (number >> (b - 8)) as u8;
+                self.data[byte_off + 1] = (number << (16 - b)) as u8;
+            }
+            _ => {
+                self.data[byte_off] |= (number >> (b - 8)) as u8;
+                self.data[byte_off + 1] = (number >> (b - 16)) as u8;
+                self.data[byte_off + 2] = (number << (24 - b)) as u8;
+            }
+        }
+        *offset += len_bits;
+    }
+
+    fn add_segments(&mut self, segments: &[&Segment<'_>]) {
+        let mut offset: usize = 0;
+
+        for s in segments.iter() {
+            self.push(&mut offset, s.get_header());
+            self.push(&mut offset, s.get_length_field(self.version));
+            for bits in s.iter() {
+                self.push(&mut offset, bits);
+            }
+        }
+        self.push(&mut offset, (MODE_STOP, 4));
+
+        let pad_offset = offset.div_ceil(8);
+        for i in pad_offset..self.version.max_data() {
+            self.data[i] = PADDING[(i & 1) ^ (pad_offset & 1)];
+        }
+    }
+
+    fn error_code_for_blocks(&mut self, offset: usize, size: usize, ec_offset: usize) {
+        let mut tmp: [u8; MAX_BLK_SIZE + MAX_EC_SIZE] = [0; MAX_BLK_SIZE + MAX_EC_SIZE];
+
+        tmp[0..size].copy_from_slice(&self.data[offset..offset + size]);
+        for i in 0..size {
+            let lead_coeff = tmp[i] as usize;
+            if lead_coeff == 0 {
+                continue;
+            }
+            let log_lead_coeff = usize::from(LOG_TABLE[lead_coeff]);
+            for (u, &v) in tmp[i + 1..].iter_mut().zip(self.poly.iter()) {
+                *u ^= EXP_TABLE[(usize::from(v) + log_lead_coeff) % 255];
+            }
+        }
+        self.data[ec_offset..ec_offset + self.ec_size]
+            .copy_from_slice(&tmp[size..size + self.ec_size]);
+    }
+
+    fn compute_error_code(&mut self) {
+        let mut offset = 0;
+        let mut ec_offset = self.g1_blocks * self.g1_blk_size + self.g2_blocks * self.g2_blk_size;
+
+        for _ in 0..self.g1_blocks {
+            self.error_code_for_blocks(offset, self.g1_blk_size, ec_offset);
+            offset += self.g1_blk_size;
+            ec_offset += self.ec_size;
+        }
+        for _ in 0..self.g2_blocks {
+            self.error_code_for_blocks(offset, self.g2_blk_size, ec_offset);
+            offset += self.g2_blk_size;
+            ec_offset += self.ec_size;
+        }
+    }
+
+    fn encode(&mut self, segments: &[&Segment<'_>]) {
+        self.add_segments(segments);
+        self.compute_error_code();
+    }
+
+    fn iter(&self) -> EncodedMsgIterator<'_> {
+        EncodedMsgIterator {
+            em: self,
+            offset: 0,
+        }
+    }
+}
+
+/// Iterator, to retrieve the data in the interleaved order needed by QR code.
+struct EncodedMsgIterator<'a> {
+    em: &'a EncodedMsg<'a>,
+    offset: usize,
+}
+
+impl Iterator for EncodedMsgIterator<'_> {
+    type Item = u8;
+
+    /// Send the bytes in interleaved mode, first byte of first block of group1,
+    /// then first byte of second block of group1, ...
+    fn next(&mut self) -> Option<Self::Item> {
+        let em = self.em;
+        let blocks = em.g1_blocks + em.g2_blocks;
+        let g1_end = em.g1_blocks * em.g1_blk_size;
+        let g2_end = g1_end + em.g2_blocks * em.g2_blk_size;
+        let ec_end = g2_end + em.ec_size * blocks;
+
+        if self.offset >= ec_end {
+            return None;
+        }
+
+        let offset = if self.offset < em.g1_blk_size * blocks {
+            // group1 and group2 interleaved
+            let blk = self.offset % blocks;
+            let blk_off = self.offset / blocks;
+            if blk < em.g1_blocks {
+                blk * em.g1_blk_size + blk_off
+            } else {
+                g1_end + em.g2_blk_size * (blk - em.g1_blocks) + blk_off
+            }
+        } else if self.offset < g2_end {
+            // last byte of group2 blocks
+            let blk2 = self.offset - blocks * em.g1_blk_size;
+            em.g1_blk_size * em.g1_blocks + blk2 * em.g2_blk_size + em.g2_blk_size - 1
+        } else {
+            // EC blocks
+            let ec_offset = self.offset - g2_end;
+            let blk = ec_offset % blocks;
+            let blk_off = ec_offset / blocks;
+
+            g2_end + blk * em.ec_size + blk_off
+        };
+        self.offset += 1;
+        Some(em.data[offset])
+    }
+}
+
+/// A QR code image, encoded as a linear binary framebuffer.
+/// 1 bit per module (pixel), each new line start at next byte boundary.
+/// Max width is 177 for V40 QR code, so `u8` is enough for coordinate.
+struct QrImage<'a> {
+    data: &'a mut [u8],
+    width: u8,
+    stride: u8,
+    version: Version,
+}
+
+impl QrImage<'_> {
+    fn new<'a, 'b>(em: &'b EncodedMsg<'b>, qrdata: &'a mut [u8]) -> QrImage<'a> {
+        let width = em.version.width();
+        let stride = width.div_ceil(8);
+        let data = qrdata;
+
+        let mut qr_image = QrImage {
+            data,
+            width,
+            stride,
+            version: em.version,
+        };
+        qr_image.draw_all(em.iter());
+        qr_image
+    }
+
+    fn clear(&mut self) {
+        self.data.fill(0);
+    }
+
+    /// Set pixel to light color.
+    fn set(&mut self, x: u8, y: u8) {
+        let off = y as usize * self.stride as usize + x as usize / 8;
+        let mut v = self.data[off];
+        v |= 0x80 >> (x % 8);
+        self.data[off] = v;
+    }
+
+    /// Invert a module color.
+    fn xor(&mut self, x: u8, y: u8) {
+        let off = y as usize * self.stride as usize + x as usize / 8;
+        self.data[off] ^= 0x80 >> (x % 8);
+    }
+
+    /// Draw a light square at (x, y) top left corner.
+    fn draw_square(&mut self, x: u8, y: u8, size: u8) {
+        for k in 0..size {
+            self.set(x + k, y);
+            self.set(x, y + k + 1);
+            self.set(x + size, y + k);
+            self.set(x + k + 1, y + size);
+        }
+    }
+
+    // Finder pattern: 3 8x8 square at the corners.
+    fn draw_finders(&mut self) {
+        self.draw_square(1, 1, 4);
+        self.draw_square(self.width - 6, 1, 4);
+        self.draw_square(1, self.width - 6, 4);
+        for k in 0..8 {
+            self.set(k, 7);
+            self.set(self.width - k - 1, 7);
+            self.set(k, self.width - 8);
+        }
+        for k in 0..7 {
+            self.set(7, k);
+            self.set(self.width - 8, k);
+            self.set(7, self.width - 1 - k);
+        }
+    }
+
+    fn is_finder(&self, x: u8, y: u8) -> bool {
+        let end = self.width - 8;
+        #[expect(clippy::nonminimal_bool)]
+        {
+            (x < 8 && y < 8) || (x < 8 && y >= end) || (x >= end && y < 8)
+        }
+    }
+
+    // Alignment pattern: 5x5 squares in a grid.
+    fn draw_alignments(&mut self) {
+        let positions = self.version.alignment_pattern();
+        for &x in positions.iter() {
+            for &y in positions.iter() {
+                if !self.is_finder(x, y) {
+                    self.draw_square(x - 1, y - 1, 2);
+                }
+            }
+        }
+    }
+
+    fn is_alignment(&self, x: u8, y: u8) -> bool {
+        let positions = self.version.alignment_pattern();
+        for &ax in positions.iter() {
+            for &ay in positions.iter() {
+                if self.is_finder(ax, ay) {
+                    continue;
+                }
+                if x >= ax - 2 && x <= ax + 2 && y >= ay - 2 && y <= ay + 2 {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    // Timing pattern: 2 dotted line between the finder patterns.
+    fn draw_timing_patterns(&mut self) {
+        let end = self.width - 8;
+
+        for x in (9..end).step_by(2) {
+            self.set(x, 6);
+            self.set(6, x);
+        }
+    }
+
+    fn is_timing(&self, x: u8, y: u8) -> bool {
+        x == 6 || y == 6
+    }
+
+    // Mask info: 15 bits around the finders, written twice for redundancy.
+    fn draw_maskinfo(&mut self) {
+        let info: u16 = FORMAT_INFOS_QR_L[0];
+        let mut skip = 0;
+
+        for k in 0..7 {
+            if k == 6 {
+                skip = 1;
+            }
+            if info & (1 << (14 - k)) == 0 {
+                self.set(k + skip, 8);
+                self.set(8, self.width - 1 - k);
+            }
+        }
+        skip = 0;
+        for k in 0..8 {
+            if k == 2 {
+                skip = 1;
+            }
+            if info & (1 << (7 - k)) == 0 {
+                self.set(8, 8 - skip - k);
+                self.set(self.width - 8 + k, 8);
+            }
+        }
+    }
+
+    fn is_maskinfo(&self, x: u8, y: u8) -> bool {
+        let end = self.width - 8;
+        // Count the dark module as mask info.
+        (x <= 8 && y == 8) || (y <= 8 && x == 8) || (x == 8 && y >= end) || (x >= end && y == 8)
+    }
+
+    // Version info: 18bits written twice, close to the finders.
+    fn draw_version_info(&mut self) {
+        let vinfo = self.version.version_info();
+        let pos = self.width - 11;
+
+        if vinfo != 0 {
+            for x in 0..3 {
+                for y in 0..6 {
+                    if vinfo & (1 << (x + y * 3)) == 0 {
+                        self.set(x + pos, y);
+                        self.set(y, x + pos);
+                    }
+                }
+            }
+        }
+    }
+
+    fn is_version_info(&self, x: u8, y: u8) -> bool {
+        let vinfo = self.version.version_info();
+        let pos = self.width - 11;
+
+        vinfo != 0 && ((x >= pos && x < pos + 3 && y < 6) || (y >= pos && y < pos + 3 && x < 6))
+    }
+
+    /// Returns true if the module is reserved (Not usable for data and EC).
+    fn is_reserved(&self, x: u8, y: u8) -> bool {
+        self.is_alignment(x, y)
+            || self.is_finder(x, y)
+            || self.is_timing(x, y)
+            || self.is_maskinfo(x, y)
+            || self.is_version_info(x, y)
+    }
+
+    /// Last module to draw, at bottom left corner.
+    fn is_last(&self, x: u8, y: u8) -> bool {
+        x == 0 && y == self.width - 1
+    }
+
+    /// Move to the next module according to QR code order.
+    ///
+    /// From bottom right corner, to bottom left corner.
+    fn next(&self, x: u8, y: u8) -> (u8, u8) {
+        let x_adj = if x <= 6 { x + 1 } else { x };
+        let column_type = (self.width - x_adj) % 4;
+
+        match column_type {
+            2 if y > 0 => (x + 1, y - 1),
+            0 if y < self.width - 1 => (x + 1, y + 1),
+            0 | 2 if x == 7 => (x - 2, y),
+            _ => (x - 1, y),
+        }
+    }
+
+    /// Find next module that can hold data.
+    fn next_available(&self, x: u8, y: u8) -> (u8, u8) {
+        let (mut x, mut y) = self.next(x, y);
+        while self.is_reserved(x, y) && !self.is_last(x, y) {
+            (x, y) = self.next(x, y);
+        }
+        (x, y)
+    }
+
+    fn draw_data(&mut self, data: impl Iterator<Item = u8>) {
+        let (mut x, mut y) = (self.width - 1, self.width - 1);
+        for byte in data {
+            for s in 0..8 {
+                if byte & (0x80 >> s) == 0 {
+                    self.set(x, y);
+                }
+                (x, y) = self.next_available(x, y);
+            }
+        }
+        // Set the remaining modules (0, 3 or 7 depending on version).
+        // because 0 correspond to a light module.
+        while !self.is_last(x, y) {
+            if !self.is_reserved(x, y) {
+                self.set(x, y);
+            }
+            (x, y) = self.next(x, y);
+        }
+    }
+
+    /// Apply checkerboard mask to all non-reserved modules.
+    fn apply_mask(&mut self) {
+        for x in 0..self.width {
+            for y in 0..self.width {
+                if (x ^ y) % 2 == 0 && !self.is_reserved(x, y) {
+                    self.xor(x, y);
+                }
+            }
+        }
+    }
+
+    /// Draw the QR code with the provided data iterator.
+    fn draw_all(&mut self, data: impl Iterator<Item = u8>) {
+        // First clear the table, as it may have already some data.
+        self.clear();
+        self.draw_finders();
+        self.draw_alignments();
+        self.draw_timing_patterns();
+        self.draw_version_info();
+        self.draw_data(data);
+        self.draw_maskinfo();
+        self.apply_mask();
+    }
+}
+
+/// C entry point for the rust QR Code generator.
+///
+/// Write the QR code image in the data buffer, and return the QR code width,
+/// or 0, if the data doesn't fit in a QR code.
+///
+/// * `url`: The base URL of the QR code. It will be encoded as Binary segment.
+/// * `data`: A pointer to the binary data, to be encoded. if URL is NULL, it
+///   will be encoded as binary segment, otherwise it will be encoded
+///   efficiently as a numeric segment, and appended to the URL.
+/// * `data_len`: Length of the data, that needs to be encoded, must be less
+///   than `data_size`.
+/// * `data_size`: Size of data buffer, it should be at least 4071 bytes to hold
+///   a V40 QR code. It will then be overwritten with the QR code image.
+/// * `tmp`: A temporary buffer that the QR code encoder will use, to write the
+///   segments and ECC.
+/// * `tmp_size`: Size of the temporary buffer, it must be at least 3706 bytes
+///   long for V40.
+///
+/// # Safety
+///
+/// * `url` must be null or point at a nul-terminated string.
+/// * `data` must be valid for reading and writing for `data_size` bytes.
+/// * `tmp` must be valid for reading and writing for `tmp_size` bytes.
+///
+/// They must remain valid for the duration of the function call.
+#[export]
+pub unsafe extern "C" fn drm_panic_helper_qr_generate(
+    url: *const kernel::ffi::c_char,
+    data: *mut u8,
+    data_len: usize,
+    data_size: usize,
+    tmp: *mut u8,
+    tmp_size: usize,
+) -> u8 {
+    if data_size < 4071 || tmp_size < 3706 || data_len > data_size {
+        return 0;
+    }
+    // SAFETY: The caller ensures that `data` is a valid pointer for reading and
+    // writing `data_size` bytes.
+    let data_slice: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(data, data_size) };
+    // SAFETY: The caller ensures that `tmp` is a valid pointer for reading and
+    // writing `tmp_size` bytes.
+    let tmp_slice: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(tmp, tmp_size) };
+    if url.is_null() {
+        match EncodedMsg::new(&[&Segment::Binary(&data_slice[0..data_len])], tmp_slice) {
+            None => 0,
+            Some(em) => {
+                let qr_image = QrImage::new(&em, data_slice);
+                qr_image.width
+            }
+        }
+    } else {
+        // SAFETY: The caller ensures that `url` is a valid pointer to a
+        // nul-terminated string.
+        let url_cstr: &CStr = unsafe { CStr::from_char_ptr(url) };
+        let segments = &[
+            &Segment::Binary(url_cstr.to_bytes()),
+            &Segment::Numeric(&data_slice[0..data_len]),
+        ];
+        match EncodedMsg::new(segments, tmp_slice) {
+            None => 0,
+            Some(em) => {
+                let qr_image = QrImage::new(&em, data_slice);
+                qr_image.width
+            }
+        }
+    }
+}
+
+/// Returns the maximum data size that can fit in a QR code of this version.
+/// * `version`: QR code version, between 1-40.
+/// * `url_len`: Length of the URL.
+///
+/// * If `url_len` > 0, remove the 2 segments header/length and also count the
+///   conversion to numeric segments.
+/// * If `url_len` = 0, only removes 3 bytes for 1 binary segment.
+///
+/// # Safety
+///
+/// Always safe to call.
+// Required to be unsafe due to the `#[export]` annotation.
+#[export]
+pub unsafe extern "C" fn drm_panic_helper_qr_max_data_size(version: u8, url_len: usize) -> usize {
+    #[expect(clippy::manual_range_contains)]
+    if version < 1 || version > 40 {
+        return 0;
+    }
+    let max_data = Version(version as usize).max_data();
+
+    if url_len > 0 {
+        // Binary segment (URL) 4 + 16 bits, numeric segment (kmsg) 4 + 12 bits => 5 bytes.
+        if url_len + 5 >= max_data {
+            0
+        } else {
+            let max = max_data - url_len - 5;
+            (max * 39) / 40
+        }
+    } else {
+        // Remove 3 bytes for the binary segment (header 4 bits, length 16 bits, stop 4bits).
+        max_data - 3
+    }
+}
diff --git a/drivers/gpu/drm/drm_panic_internal.h b/drivers/gpu/drm/drm_panic_internal.h
new file mode 100644
index 00000000000000..fe6d514a0833e1
--- /dev/null
+++ b/drivers/gpu/drm/drm_panic_internal.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+
+/*
+ * Copyright (c) 2024 Intel
+ * Copyright (c) 2024 Red Hat
+ */
+
+#ifndef __DRM_PANIC_INTERNAL_H__
+#define __DRM_PANIC_INTERNAL_H__
+
+#include <linux/spinlock.h>
+
+struct drm_device;
+
+#ifdef CONFIG_DRM_PANIC
+
+/**
+ * drm_panic_lock - protect panic printing relevant state
+ * @dev: struct drm_device
+ * @flags: unsigned long irq flags you need to pass to the unlock() counterpart
+ *
+ * This function must be called to protect software and hardware state that the
+ * panic printing code must be able to rely on. The protected sections must be
+ * as small as possible. It uses the irqsave/irqrestore variant, and can be
+ * called from irq handler. Examples include:
+ *
+ * - Access to peek/poke or other similar registers, if that is the way the
+ *   driver prints the pixels into the scanout buffer at panic time.
+ *
+ * - Updates to pointers like &drm_plane.state, allowing the panic handler to
+ *   safely deference these. This is done in drm_atomic_helper_swap_state().
+ *
+ * - An state that isn't invariant and that the driver must be able to access
+ *   during panic printing.
+ */
+#define drm_panic_lock(dev, flags) \
+	raw_spin_lock_irqsave(&(dev)->mode_config.panic_lock, flags)
+
+/**
+ * drm_panic_unlock - end of the panic printing critical section
+ * @dev: struct drm_device
+ * @flags: irq flags that were returned when acquiring the lock
+ *
+ * Unlocks the raw spinlock acquired by either drm_panic_lock() or
+ * drm_panic_trylock().
+ */
+#define drm_panic_unlock(dev, flags) \
+	raw_spin_unlock_irqrestore(&(dev)->mode_config.panic_lock, flags)
+
+#else
+static inline void drm_panic_lock(struct drm_device *dev, unsigned long flags) {}
+static inline void drm_panic_unlock(struct drm_device *dev, unsigned long flags) {}
+#endif
+
+#if IS_ENABLED(CONFIG_DRM_PANIC_HELPER)
+/* drm_panic_helper.c */
+int __init drm_panic_helper_init(void);
+void __exit drm_panic_helper_exit(void);
+#else
+static inline int __init drm_panic_helper_init(void)
+{
+	return 0;
+}
+
+static inline void __exit drm_panic_helper_exit(void)
+{ }
+#endif
+
+#endif /* __DRM_PANIC_INTERNAL_H__ */
diff --git a/drivers/gpu/drm/drm_panic_qr.rs b/drivers/gpu/drm/drm_panic_qr.rs
deleted file mode 100644
index 4d7eb75a3afc15..00000000000000
--- a/drivers/gpu/drm/drm_panic_qr.rs
+++ /dev/null
@@ -1,1016 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-//! This is a simple QR encoder for DRM panic.
-//!
-//! It is called from a panic handler, so it should't allocate memory and
-//! does all the work on the stack or on the provided buffers. For
-//! simplification, it only supports low error correction, and applies the
-//! first mask (checkerboard). It will draw the smallest QR code that can
-//! contain the string passed as parameter. To get the most compact
-//! QR code, the start of the URL is encoded as binary, and the
-//! compressed kmsg is encoded as numeric.
-//!
-//! The binary data must be a valid URL parameter, so the easiest way is
-//! to use base64 encoding. But this wastes 25% of data space, so the
-//! whole stack trace won't fit in the QR code. So instead it encodes
-//! every 7 bytes of input into 17 decimal digits, and then uses the
-//! efficient numeric encoding, that encode 3 decimal digits into
-//! 10bits. This makes 168bits of compressed data into 51 decimal digits,
-//! into 170bits in the QR code, so wasting only 1.17%. And the numbers are
-//! valid URL parameter, so the website can do the reverse, to get the
-//! binary data. This is the same algorithm used by Fido v2.2 QR-initiated
-//! authentication specification.
-//!
-//! Inspired by these 3 projects, all under MIT license:
-//!
-//! * <https://github.com/kennytm/qrcode-rust>
-//! * <https://github.com/erwanvivien/fast_qr>
-//! * <https://github.com/bjguillot/qr>
-
-use kernel::prelude::*;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd)]
-struct Version(usize);
-
-// Generator polynomials for ECC, only those that are needed for low quality.
-const P7: [u8; 7] = [87, 229, 146, 149, 238, 102, 21];
-const P10: [u8; 10] = [251, 67, 46, 61, 118, 70, 64, 94, 32, 45];
-const P15: [u8; 15] = [
-    8, 183, 61, 91, 202, 37, 51, 58, 58, 237, 140, 124, 5, 99, 105,
-];
-const P18: [u8; 18] = [
-    215, 234, 158, 94, 184, 97, 118, 170, 79, 187, 152, 148, 252, 179, 5, 98, 96, 153,
-];
-const P20: [u8; 20] = [
-    17, 60, 79, 50, 61, 163, 26, 187, 202, 180, 221, 225, 83, 239, 156, 164, 212, 212, 188, 190,
-];
-const P22: [u8; 22] = [
-    210, 171, 247, 242, 93, 230, 14, 109, 221, 53, 200, 74, 8, 172, 98, 80, 219, 134, 160, 105,
-    165, 231,
-];
-const P24: [u8; 24] = [
-    229, 121, 135, 48, 211, 117, 251, 126, 159, 180, 169, 152, 192, 226, 228, 218, 111, 0, 117,
-    232, 87, 96, 227, 21,
-];
-const P26: [u8; 26] = [
-    173, 125, 158, 2, 103, 182, 118, 17, 145, 201, 111, 28, 165, 53, 161, 21, 245, 142, 13, 102,
-    48, 227, 153, 145, 218, 70,
-];
-const P28: [u8; 28] = [
-    168, 223, 200, 104, 224, 234, 108, 180, 110, 190, 195, 147, 205, 27, 232, 201, 21, 43, 245, 87,
-    42, 195, 212, 119, 242, 37, 9, 123,
-];
-const P30: [u8; 30] = [
-    41, 173, 145, 152, 216, 31, 179, 182, 50, 48, 110, 86, 239, 96, 222, 125, 42, 173, 226, 193,
-    224, 130, 156, 37, 251, 216, 238, 40, 192, 180,
-];
-
-/// QR Code parameters for Low quality ECC:
-/// - Error Correction polynomial.
-/// - Number of blocks in group 1.
-/// - Number of blocks in group 2.
-/// - Block size in group 1.
-///
-/// (Block size in group 2 is one more than group 1).
-struct VersionParameter(&'static [u8], u8, u8, u8);
-const VPARAM: [VersionParameter; 40] = [
-    VersionParameter(&P7, 1, 0, 19),    // V1
-    VersionParameter(&P10, 1, 0, 34),   // V2
-    VersionParameter(&P15, 1, 0, 55),   // V3
-    VersionParameter(&P20, 1, 0, 80),   // V4
-    VersionParameter(&P26, 1, 0, 108),  // V5
-    VersionParameter(&P18, 2, 0, 68),   // V6
-    VersionParameter(&P20, 2, 0, 78),   // V7
-    VersionParameter(&P24, 2, 0, 97),   // V8
-    VersionParameter(&P30, 2, 0, 116),  // V9
-    VersionParameter(&P18, 2, 2, 68),   // V10
-    VersionParameter(&P20, 4, 0, 81),   // V11
-    VersionParameter(&P24, 2, 2, 92),   // V12
-    VersionParameter(&P26, 4, 0, 107),  // V13
-    VersionParameter(&P30, 3, 1, 115),  // V14
-    VersionParameter(&P22, 5, 1, 87),   // V15
-    VersionParameter(&P24, 5, 1, 98),   // V16
-    VersionParameter(&P28, 1, 5, 107),  // V17
-    VersionParameter(&P30, 5, 1, 120),  // V18
-    VersionParameter(&P28, 3, 4, 113),  // V19
-    VersionParameter(&P28, 3, 5, 107),  // V20
-    VersionParameter(&P28, 4, 4, 116),  // V21
-    VersionParameter(&P28, 2, 7, 111),  // V22
-    VersionParameter(&P30, 4, 5, 121),  // V23
-    VersionParameter(&P30, 6, 4, 117),  // V24
-    VersionParameter(&P26, 8, 4, 106),  // V25
-    VersionParameter(&P28, 10, 2, 114), // V26
-    VersionParameter(&P30, 8, 4, 122),  // V27
-    VersionParameter(&P30, 3, 10, 117), // V28
-    VersionParameter(&P30, 7, 7, 116),  // V29
-    VersionParameter(&P30, 5, 10, 115), // V30
-    VersionParameter(&P30, 13, 3, 115), // V31
-    VersionParameter(&P30, 17, 0, 115), // V32
-    VersionParameter(&P30, 17, 1, 115), // V33
-    VersionParameter(&P30, 13, 6, 115), // V34
-    VersionParameter(&P30, 12, 7, 121), // V35
-    VersionParameter(&P30, 6, 14, 121), // V36
-    VersionParameter(&P30, 17, 4, 122), // V37
-    VersionParameter(&P30, 4, 18, 122), // V38
-    VersionParameter(&P30, 20, 4, 117), // V39
-    VersionParameter(&P30, 19, 6, 118), // V40
-];
-
-const MAX_EC_SIZE: usize = 30;
-const MAX_BLK_SIZE: usize = 123;
-
-/// Position of the alignment pattern grid.
-const ALIGNMENT_PATTERNS: [&[u8]; 40] = [
-    &[],
-    &[6, 18],
-    &[6, 22],
-    &[6, 26],
-    &[6, 30],
-    &[6, 34],
-    &[6, 22, 38],
-    &[6, 24, 42],
-    &[6, 26, 46],
-    &[6, 28, 50],
-    &[6, 30, 54],
-    &[6, 32, 58],
-    &[6, 34, 62],
-    &[6, 26, 46, 66],
-    &[6, 26, 48, 70],
-    &[6, 26, 50, 74],
-    &[6, 30, 54, 78],
-    &[6, 30, 56, 82],
-    &[6, 30, 58, 86],
-    &[6, 34, 62, 90],
-    &[6, 28, 50, 72, 94],
-    &[6, 26, 50, 74, 98],
-    &[6, 30, 54, 78, 102],
-    &[6, 28, 54, 80, 106],
-    &[6, 32, 58, 84, 110],
-    &[6, 30, 58, 86, 114],
-    &[6, 34, 62, 90, 118],
-    &[6, 26, 50, 74, 98, 122],
-    &[6, 30, 54, 78, 102, 126],
-    &[6, 26, 52, 78, 104, 130],
-    &[6, 30, 56, 82, 108, 134],
-    &[6, 34, 60, 86, 112, 138],
-    &[6, 30, 58, 86, 114, 142],
-    &[6, 34, 62, 90, 118, 146],
-    &[6, 30, 54, 78, 102, 126, 150],
-    &[6, 24, 50, 76, 102, 128, 154],
-    &[6, 28, 54, 80, 106, 132, 158],
-    &[6, 32, 58, 84, 110, 136, 162],
-    &[6, 26, 54, 82, 110, 138, 166],
-    &[6, 30, 58, 86, 114, 142, 170],
-];
-
-/// Version information for format V7-V40.
-const VERSION_INFORMATION: [u32; 34] = [
-    0b00_0111_1100_1001_0100,
-    0b00_1000_0101_1011_1100,
-    0b00_1001_1010_1001_1001,
-    0b00_1010_0100_1101_0011,
-    0b00_1011_1011_1111_0110,
-    0b00_1100_0111_0110_0010,
-    0b00_1101_1000_0100_0111,
-    0b00_1110_0110_0000_1101,
-    0b00_1111_1001_0010_1000,
-    0b01_0000_1011_0111_1000,
-    0b01_0001_0100_0101_1101,
-    0b01_0010_1010_0001_0111,
-    0b01_0011_0101_0011_0010,
-    0b01_0100_1001_1010_0110,
-    0b01_0101_0110_1000_0011,
-    0b01_0110_1000_1100_1001,
-    0b01_0111_0111_1110_1100,
-    0b01_1000_1110_1100_0100,
-    0b01_1001_0001_1110_0001,
-    0b01_1010_1111_1010_1011,
-    0b01_1011_0000_1000_1110,
-    0b01_1100_1100_0001_1010,
-    0b01_1101_0011_0011_1111,
-    0b01_1110_1101_0111_0101,
-    0b01_1111_0010_0101_0000,
-    0b10_0000_1001_1101_0101,
-    0b10_0001_0110_1111_0000,
-    0b10_0010_1000_1011_1010,
-    0b10_0011_0111_1001_1111,
-    0b10_0100_1011_0000_1011,
-    0b10_0101_0100_0010_1110,
-    0b10_0110_1010_0110_0100,
-    0b10_0111_0101_0100_0001,
-    0b10_1000_1100_0110_1001,
-];
-
-/// Format info for low quality ECC.
-const FORMAT_INFOS_QR_L: [u16; 8] = [
-    0x77c4, 0x72f3, 0x7daa, 0x789d, 0x662f, 0x6318, 0x6c41, 0x6976,
-];
-
-impl Version {
-    /// Returns the smallest QR version than can hold these segments.
-    fn from_segments(segments: &[&Segment<'_>]) -> Option<Version> {
-        (1..=40)
-            .map(Version)
-            .find(|&v| v.max_data() * 8 >= segments.iter().map(|s| s.total_size_bits(v)).sum())
-    }
-
-    fn width(&self) -> u8 {
-        (self.0 as u8) * 4 + 17
-    }
-
-    fn max_data(&self) -> usize {
-        self.g1_blk_size() * self.g1_blocks() + (self.g1_blk_size() + 1) * self.g2_blocks()
-    }
-
-    fn ec_size(&self) -> usize {
-        VPARAM[self.0 - 1].0.len()
-    }
-
-    fn g1_blocks(&self) -> usize {
-        VPARAM[self.0 - 1].1 as usize
-    }
-
-    fn g2_blocks(&self) -> usize {
-        VPARAM[self.0 - 1].2 as usize
-    }
-
-    fn g1_blk_size(&self) -> usize {
-        VPARAM[self.0 - 1].3 as usize
-    }
-
-    fn alignment_pattern(&self) -> &'static [u8] {
-        ALIGNMENT_PATTERNS[self.0 - 1]
-    }
-
-    fn poly(&self) -> &'static [u8] {
-        VPARAM[self.0 - 1].0
-    }
-
-    fn version_info(&self) -> u32 {
-        if *self >= Version(7) {
-            VERSION_INFORMATION[self.0 - 7]
-        } else {
-            0
-        }
-    }
-}
-
-/// Exponential table for Galois Field GF(256).
-const EXP_TABLE: [u8; 256] = [
-    1, 2, 4, 8, 16, 32, 64, 128, 29, 58, 116, 232, 205, 135, 19, 38, 76, 152, 45, 90, 180, 117,
-    234, 201, 143, 3, 6, 12, 24, 48, 96, 192, 157, 39, 78, 156, 37, 74, 148, 53, 106, 212, 181,
-    119, 238, 193, 159, 35, 70, 140, 5, 10, 20, 40, 80, 160, 93, 186, 105, 210, 185, 111, 222, 161,
-    95, 190, 97, 194, 153, 47, 94, 188, 101, 202, 137, 15, 30, 60, 120, 240, 253, 231, 211, 187,
-    107, 214, 177, 127, 254, 225, 223, 163, 91, 182, 113, 226, 217, 175, 67, 134, 17, 34, 68, 136,
-    13, 26, 52, 104, 208, 189, 103, 206, 129, 31, 62, 124, 248, 237, 199, 147, 59, 118, 236, 197,
-    151, 51, 102, 204, 133, 23, 46, 92, 184, 109, 218, 169, 79, 158, 33, 66, 132, 21, 42, 84, 168,
-    77, 154, 41, 82, 164, 85, 170, 73, 146, 57, 114, 228, 213, 183, 115, 230, 209, 191, 99, 198,
-    145, 63, 126, 252, 229, 215, 179, 123, 246, 241, 255, 227, 219, 171, 75, 150, 49, 98, 196, 149,
-    55, 110, 220, 165, 87, 174, 65, 130, 25, 50, 100, 200, 141, 7, 14, 28, 56, 112, 224, 221, 167,
-    83, 166, 81, 162, 89, 178, 121, 242, 249, 239, 195, 155, 43, 86, 172, 69, 138, 9, 18, 36, 72,
-    144, 61, 122, 244, 245, 247, 243, 251, 235, 203, 139, 11, 22, 44, 88, 176, 125, 250, 233, 207,
-    131, 27, 54, 108, 216, 173, 71, 142, 1,
-];
-
-/// Reverse exponential table for Galois Field GF(256).
-const LOG_TABLE: [u8; 256] = [
-    175, 0, 1, 25, 2, 50, 26, 198, 3, 223, 51, 238, 27, 104, 199, 75, 4, 100, 224, 14, 52, 141,
-    239, 129, 28, 193, 105, 248, 200, 8, 76, 113, 5, 138, 101, 47, 225, 36, 15, 33, 53, 147, 142,
-    218, 240, 18, 130, 69, 29, 181, 194, 125, 106, 39, 249, 185, 201, 154, 9, 120, 77, 228, 114,
-    166, 6, 191, 139, 98, 102, 221, 48, 253, 226, 152, 37, 179, 16, 145, 34, 136, 54, 208, 148,
-    206, 143, 150, 219, 189, 241, 210, 19, 92, 131, 56, 70, 64, 30, 66, 182, 163, 195, 72, 126,
-    110, 107, 58, 40, 84, 250, 133, 186, 61, 202, 94, 155, 159, 10, 21, 121, 43, 78, 212, 229, 172,
-    115, 243, 167, 87, 7, 112, 192, 247, 140, 128, 99, 13, 103, 74, 222, 237, 49, 197, 254, 24,
-    227, 165, 153, 119, 38, 184, 180, 124, 17, 68, 146, 217, 35, 32, 137, 46, 55, 63, 209, 91, 149,
-    188, 207, 205, 144, 135, 151, 178, 220, 252, 190, 97, 242, 86, 211, 171, 20, 42, 93, 158, 132,
-    60, 57, 83, 71, 109, 65, 162, 31, 45, 67, 216, 183, 123, 164, 118, 196, 23, 73, 236, 127, 12,
-    111, 246, 108, 161, 59, 82, 41, 157, 85, 170, 251, 96, 134, 177, 187, 204, 62, 90, 203, 89, 95,
-    176, 156, 169, 160, 81, 11, 245, 22, 235, 122, 117, 44, 215, 79, 174, 213, 233, 230, 231, 173,
-    232, 116, 214, 244, 234, 168, 80, 88, 175,
-];
-
-// 4 bits segment header.
-const MODE_STOP: u16 = 0;
-const MODE_NUMERIC: u16 = 1;
-const MODE_BINARY: u16 = 4;
-/// Padding bytes.
-const PADDING: [u8; 2] = [236, 17];
-
-/// Number of bits to encode characters in numeric mode.
-const NUM_CHARS_BITS: [usize; 4] = [0, 4, 7, 10];
-/// Number of decimal digits required to encode n bytes of binary data.
-/// eg: you need 15 decimal digits to fit 6 bytes of binary data.
-const BYTES_TO_DIGITS: [usize; 8] = [0, 3, 5, 8, 10, 13, 15, 17];
-
-enum Segment<'a> {
-    Numeric(&'a [u8]),
-    Binary(&'a [u8]),
-}
-
-impl Segment<'_> {
-    fn get_header(&self) -> (u16, usize) {
-        match self {
-            Segment::Binary(_) => (MODE_BINARY, 4),
-            Segment::Numeric(_) => (MODE_NUMERIC, 4),
-        }
-    }
-
-    /// Returns the size of the length field in bits, depending on QR Version.
-    fn length_bits_count(&self, version: Version) -> usize {
-        let Version(v) = version;
-        match self {
-            Segment::Binary(_) => match v {
-                1..=9 => 8,
-                _ => 16,
-            },
-            Segment::Numeric(_) => match v {
-                1..=9 => 10,
-                10..=26 => 12,
-                _ => 14,
-            },
-        }
-    }
-
-    /// Number of characters in the segment.
-    fn character_count(&self) -> usize {
-        match self {
-            Segment::Binary(data) => data.len(),
-            Segment::Numeric(data) => {
-                let last_chars = BYTES_TO_DIGITS[data.len() % 7];
-                // 17 decimal numbers per 7bytes + remainder.
-                17 * (data.len() / 7) + last_chars
-            }
-        }
-    }
-
-    fn get_length_field(&self, version: Version) -> (u16, usize) {
-        (
-            self.character_count() as u16,
-            self.length_bits_count(version),
-        )
-    }
-
-    fn total_size_bits(&self, version: Version) -> usize {
-        let data_size = match self {
-            Segment::Binary(data) => data.len() * 8,
-            Segment::Numeric(_) => {
-                let digits = self.character_count();
-                10 * (digits / 3) + NUM_CHARS_BITS[digits % 3]
-            }
-        };
-        // header + length + data.
-        4 + self.length_bits_count(version) + data_size
-    }
-
-    fn iter(&self) -> SegmentIterator<'_> {
-        SegmentIterator {
-            segment: self,
-            offset: 0,
-            decfifo: Default::default(),
-        }
-    }
-}
-
-/// Max fifo size is 17 (max push) + 2 (max remaining)
-const MAX_FIFO_SIZE: usize = 19;
-
-/// A simple Decimal digit FIFO
-#[derive(Default)]
-struct DecFifo {
-    decimals: [u8; MAX_FIFO_SIZE],
-    len: usize,
-}
-
-// On arm32 architecture, dividing an `u64` by a constant will generate a call
-// to `__aeabi_uldivmod` which is not present in the kernel.
-// So use the multiply by inverse method for this architecture.
-fn div10(val: u64) -> u64 {
-    if cfg!(target_arch = "arm") {
-        let val_h = val >> 32;
-        let val_l = val & 0xFFFFFFFF;
-        let b_h: u64 = 0x66666666;
-        let b_l: u64 = 0x66666667;
-
-        let tmp1 = val_h * b_l + ((val_l * b_l) >> 32);
-        let tmp2 = val_l * b_h + (tmp1 & 0xffffffff);
-        let tmp3 = val_h * b_h + (tmp1 >> 32) + (tmp2 >> 32);
-
-        tmp3 >> 2
-    } else {
-        val / 10
-    }
-}
-
-impl DecFifo {
-    fn push(&mut self, data: u64, len: usize) {
-        let mut chunk = data;
-        for i in (0..self.len).rev() {
-            self.decimals[i + len] = self.decimals[i];
-        }
-        for decimal in &mut self.decimals[..len] {
-            *decimal = (chunk % 10) as u8;
-            chunk = div10(chunk);
-        }
-        self.len += len;
-    }
-
-    /// Pop 3 decimal digits from the FIFO
-    fn pop3(&mut self) -> Option<(u16, usize)> {
-        if self.len == 0 {
-            None
-        } else {
-            let poplen = 3.min(self.len);
-            self.len -= poplen;
-            let mut out = 0;
-            let mut exp = 1;
-            for i in 0..poplen {
-                out += u16::from(self.decimals[self.len + i]) * exp;
-                exp *= 10;
-            }
-            Some((out, NUM_CHARS_BITS[poplen]))
-        }
-    }
-}
-
-struct SegmentIterator<'a> {
-    segment: &'a Segment<'a>,
-    offset: usize,
-    decfifo: DecFifo,
-}
-
-impl Iterator for SegmentIterator<'_> {
-    type Item = (u16, usize);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        match self.segment {
-            Segment::Binary(data) => {
-                if self.offset < data.len() {
-                    let byte = u16::from(data[self.offset]);
-                    self.offset += 1;
-                    Some((byte, 8))
-                } else {
-                    None
-                }
-            }
-            Segment::Numeric(data) => {
-                if self.decfifo.len < 3 && self.offset < data.len() {
-                    // If there are less than 3 decimal digits in the fifo,
-                    // take the next 7 bytes of input, and push them to the fifo.
-                    let mut buf = [0u8; 8];
-                    let len = 7.min(data.len() - self.offset);
-                    buf[..len].copy_from_slice(&data[self.offset..self.offset + len]);
-                    let chunk = u64::from_le_bytes(buf);
-                    self.decfifo.push(chunk, BYTES_TO_DIGITS[len]);
-                    self.offset += len;
-                }
-                self.decfifo.pop3()
-            }
-        }
-    }
-}
-
-struct EncodedMsg<'a> {
-    data: &'a mut [u8],
-    ec_size: usize,
-    g1_blocks: usize,
-    g2_blocks: usize,
-    g1_blk_size: usize,
-    g2_blk_size: usize,
-    poly: &'static [u8],
-    version: Version,
-}
-
-/// Data to be put in the QR code, with correct segment encoding, padding, and
-/// Error Code Correction.
-impl EncodedMsg<'_> {
-    fn new<'a>(segments: &[&Segment<'_>], data: &'a mut [u8]) -> Option<EncodedMsg<'a>> {
-        let version = Version::from_segments(segments)?;
-        let ec_size = version.ec_size();
-        let g1_blocks = version.g1_blocks();
-        let g2_blocks = version.g2_blocks();
-        let g1_blk_size = version.g1_blk_size();
-        let g2_blk_size = g1_blk_size + 1;
-        let poly = version.poly();
-
-        // clear the output.
-        data.fill(0);
-
-        let mut em = EncodedMsg {
-            data,
-            ec_size,
-            g1_blocks,
-            g2_blocks,
-            g1_blk_size,
-            g2_blk_size,
-            poly,
-            version,
-        };
-        em.encode(segments);
-        Some(em)
-    }
-
-    /// Push bits of data at an offset (in bits).
-    fn push(&mut self, offset: &mut usize, bits: (u16, usize)) {
-        let (number, len_bits) = bits;
-        let byte_off = *offset / 8;
-        let bit_off = *offset % 8;
-        let b = bit_off + len_bits;
-
-        match (bit_off, b) {
-            (0, 0..=8) => {
-                self.data[byte_off] = (number << (8 - b)) as u8;
-            }
-            (0, _) => {
-                self.data[byte_off] = (number >> (b - 8)) as u8;
-                self.data[byte_off + 1] = (number << (16 - b)) as u8;
-            }
-            (_, 0..=8) => {
-                self.data[byte_off] |= (number << (8 - b)) as u8;
-            }
-            (_, 9..=16) => {
-                self.data[byte_off] |= (number >> (b - 8)) as u8;
-                self.data[byte_off + 1] = (number << (16 - b)) as u8;
-            }
-            _ => {
-                self.data[byte_off] |= (number >> (b - 8)) as u8;
-                self.data[byte_off + 1] = (number >> (b - 16)) as u8;
-                self.data[byte_off + 2] = (number << (24 - b)) as u8;
-            }
-        }
-        *offset += len_bits;
-    }
-
-    fn add_segments(&mut self, segments: &[&Segment<'_>]) {
-        let mut offset: usize = 0;
-
-        for s in segments.iter() {
-            self.push(&mut offset, s.get_header());
-            self.push(&mut offset, s.get_length_field(self.version));
-            for bits in s.iter() {
-                self.push(&mut offset, bits);
-            }
-        }
-        self.push(&mut offset, (MODE_STOP, 4));
-
-        let pad_offset = offset.div_ceil(8);
-        for i in pad_offset..self.version.max_data() {
-            self.data[i] = PADDING[(i & 1) ^ (pad_offset & 1)];
-        }
-    }
-
-    fn error_code_for_blocks(&mut self, offset: usize, size: usize, ec_offset: usize) {
-        let mut tmp: [u8; MAX_BLK_SIZE + MAX_EC_SIZE] = [0; MAX_BLK_SIZE + MAX_EC_SIZE];
-
-        tmp[0..size].copy_from_slice(&self.data[offset..offset + size]);
-        for i in 0..size {
-            let lead_coeff = tmp[i] as usize;
-            if lead_coeff == 0 {
-                continue;
-            }
-            let log_lead_coeff = usize::from(LOG_TABLE[lead_coeff]);
-            for (u, &v) in tmp[i + 1..].iter_mut().zip(self.poly.iter()) {
-                *u ^= EXP_TABLE[(usize::from(v) + log_lead_coeff) % 255];
-            }
-        }
-        self.data[ec_offset..ec_offset + self.ec_size]
-            .copy_from_slice(&tmp[size..size + self.ec_size]);
-    }
-
-    fn compute_error_code(&mut self) {
-        let mut offset = 0;
-        let mut ec_offset = self.g1_blocks * self.g1_blk_size + self.g2_blocks * self.g2_blk_size;
-
-        for _ in 0..self.g1_blocks {
-            self.error_code_for_blocks(offset, self.g1_blk_size, ec_offset);
-            offset += self.g1_blk_size;
-            ec_offset += self.ec_size;
-        }
-        for _ in 0..self.g2_blocks {
-            self.error_code_for_blocks(offset, self.g2_blk_size, ec_offset);
-            offset += self.g2_blk_size;
-            ec_offset += self.ec_size;
-        }
-    }
-
-    fn encode(&mut self, segments: &[&Segment<'_>]) {
-        self.add_segments(segments);
-        self.compute_error_code();
-    }
-
-    fn iter(&self) -> EncodedMsgIterator<'_> {
-        EncodedMsgIterator {
-            em: self,
-            offset: 0,
-        }
-    }
-}
-
-/// Iterator, to retrieve the data in the interleaved order needed by QR code.
-struct EncodedMsgIterator<'a> {
-    em: &'a EncodedMsg<'a>,
-    offset: usize,
-}
-
-impl Iterator for EncodedMsgIterator<'_> {
-    type Item = u8;
-
-    /// Send the bytes in interleaved mode, first byte of first block of group1,
-    /// then first byte of second block of group1, ...
-    fn next(&mut self) -> Option<Self::Item> {
-        let em = self.em;
-        let blocks = em.g1_blocks + em.g2_blocks;
-        let g1_end = em.g1_blocks * em.g1_blk_size;
-        let g2_end = g1_end + em.g2_blocks * em.g2_blk_size;
-        let ec_end = g2_end + em.ec_size * blocks;
-
-        if self.offset >= ec_end {
-            return None;
-        }
-
-        let offset = if self.offset < em.g1_blk_size * blocks {
-            // group1 and group2 interleaved
-            let blk = self.offset % blocks;
-            let blk_off = self.offset / blocks;
-            if blk < em.g1_blocks {
-                blk * em.g1_blk_size + blk_off
-            } else {
-                g1_end + em.g2_blk_size * (blk - em.g1_blocks) + blk_off
-            }
-        } else if self.offset < g2_end {
-            // last byte of group2 blocks
-            let blk2 = self.offset - blocks * em.g1_blk_size;
-            em.g1_blk_size * em.g1_blocks + blk2 * em.g2_blk_size + em.g2_blk_size - 1
-        } else {
-            // EC blocks
-            let ec_offset = self.offset - g2_end;
-            let blk = ec_offset % blocks;
-            let blk_off = ec_offset / blocks;
-
-            g2_end + blk * em.ec_size + blk_off
-        };
-        self.offset += 1;
-        Some(em.data[offset])
-    }
-}
-
-/// A QR code image, encoded as a linear binary framebuffer.
-/// 1 bit per module (pixel), each new line start at next byte boundary.
-/// Max width is 177 for V40 QR code, so `u8` is enough for coordinate.
-struct QrImage<'a> {
-    data: &'a mut [u8],
-    width: u8,
-    stride: u8,
-    version: Version,
-}
-
-impl QrImage<'_> {
-    fn new<'a, 'b>(em: &'b EncodedMsg<'b>, qrdata: &'a mut [u8]) -> QrImage<'a> {
-        let width = em.version.width();
-        let stride = width.div_ceil(8);
-        let data = qrdata;
-
-        let mut qr_image = QrImage {
-            data,
-            width,
-            stride,
-            version: em.version,
-        };
-        qr_image.draw_all(em.iter());
-        qr_image
-    }
-
-    fn clear(&mut self) {
-        self.data.fill(0);
-    }
-
-    /// Set pixel to light color.
-    fn set(&mut self, x: u8, y: u8) {
-        let off = y as usize * self.stride as usize + x as usize / 8;
-        let mut v = self.data[off];
-        v |= 0x80 >> (x % 8);
-        self.data[off] = v;
-    }
-
-    /// Invert a module color.
-    fn xor(&mut self, x: u8, y: u8) {
-        let off = y as usize * self.stride as usize + x as usize / 8;
-        self.data[off] ^= 0x80 >> (x % 8);
-    }
-
-    /// Draw a light square at (x, y) top left corner.
-    fn draw_square(&mut self, x: u8, y: u8, size: u8) {
-        for k in 0..size {
-            self.set(x + k, y);
-            self.set(x, y + k + 1);
-            self.set(x + size, y + k);
-            self.set(x + k + 1, y + size);
-        }
-    }
-
-    // Finder pattern: 3 8x8 square at the corners.
-    fn draw_finders(&mut self) {
-        self.draw_square(1, 1, 4);
-        self.draw_square(self.width - 6, 1, 4);
-        self.draw_square(1, self.width - 6, 4);
-        for k in 0..8 {
-            self.set(k, 7);
-            self.set(self.width - k - 1, 7);
-            self.set(k, self.width - 8);
-        }
-        for k in 0..7 {
-            self.set(7, k);
-            self.set(self.width - 8, k);
-            self.set(7, self.width - 1 - k);
-        }
-    }
-
-    fn is_finder(&self, x: u8, y: u8) -> bool {
-        let end = self.width - 8;
-        #[expect(clippy::nonminimal_bool)]
-        {
-            (x < 8 && y < 8) || (x < 8 && y >= end) || (x >= end && y < 8)
-        }
-    }
-
-    // Alignment pattern: 5x5 squares in a grid.
-    fn draw_alignments(&mut self) {
-        let positions = self.version.alignment_pattern();
-        for &x in positions.iter() {
-            for &y in positions.iter() {
-                if !self.is_finder(x, y) {
-                    self.draw_square(x - 1, y - 1, 2);
-                }
-            }
-        }
-    }
-
-    fn is_alignment(&self, x: u8, y: u8) -> bool {
-        let positions = self.version.alignment_pattern();
-        for &ax in positions.iter() {
-            for &ay in positions.iter() {
-                if self.is_finder(ax, ay) {
-                    continue;
-                }
-                if x >= ax - 2 && x <= ax + 2 && y >= ay - 2 && y <= ay + 2 {
-                    return true;
-                }
-            }
-        }
-        false
-    }
-
-    // Timing pattern: 2 dotted line between the finder patterns.
-    fn draw_timing_patterns(&mut self) {
-        let end = self.width - 8;
-
-        for x in (9..end).step_by(2) {
-            self.set(x, 6);
-            self.set(6, x);
-        }
-    }
-
-    fn is_timing(&self, x: u8, y: u8) -> bool {
-        x == 6 || y == 6
-    }
-
-    // Mask info: 15 bits around the finders, written twice for redundancy.
-    fn draw_maskinfo(&mut self) {
-        let info: u16 = FORMAT_INFOS_QR_L[0];
-        let mut skip = 0;
-
-        for k in 0..7 {
-            if k == 6 {
-                skip = 1;
-            }
-            if info & (1 << (14 - k)) == 0 {
-                self.set(k + skip, 8);
-                self.set(8, self.width - 1 - k);
-            }
-        }
-        skip = 0;
-        for k in 0..8 {
-            if k == 2 {
-                skip = 1;
-            }
-            if info & (1 << (7 - k)) == 0 {
-                self.set(8, 8 - skip - k);
-                self.set(self.width - 8 + k, 8);
-            }
-        }
-    }
-
-    fn is_maskinfo(&self, x: u8, y: u8) -> bool {
-        let end = self.width - 8;
-        // Count the dark module as mask info.
-        (x <= 8 && y == 8) || (y <= 8 && x == 8) || (x == 8 && y >= end) || (x >= end && y == 8)
-    }
-
-    // Version info: 18bits written twice, close to the finders.
-    fn draw_version_info(&mut self) {
-        let vinfo = self.version.version_info();
-        let pos = self.width - 11;
-
-        if vinfo != 0 {
-            for x in 0..3 {
-                for y in 0..6 {
-                    if vinfo & (1 << (x + y * 3)) == 0 {
-                        self.set(x + pos, y);
-                        self.set(y, x + pos);
-                    }
-                }
-            }
-        }
-    }
-
-    fn is_version_info(&self, x: u8, y: u8) -> bool {
-        let vinfo = self.version.version_info();
-        let pos = self.width - 11;
-
-        vinfo != 0 && ((x >= pos && x < pos + 3 && y < 6) || (y >= pos && y < pos + 3 && x < 6))
-    }
-
-    /// Returns true if the module is reserved (Not usable for data and EC).
-    fn is_reserved(&self, x: u8, y: u8) -> bool {
-        self.is_alignment(x, y)
-            || self.is_finder(x, y)
-            || self.is_timing(x, y)
-            || self.is_maskinfo(x, y)
-            || self.is_version_info(x, y)
-    }
-
-    /// Last module to draw, at bottom left corner.
-    fn is_last(&self, x: u8, y: u8) -> bool {
-        x == 0 && y == self.width - 1
-    }
-
-    /// Move to the next module according to QR code order.
-    ///
-    /// From bottom right corner, to bottom left corner.
-    fn next(&self, x: u8, y: u8) -> (u8, u8) {
-        let x_adj = if x <= 6 { x + 1 } else { x };
-        let column_type = (self.width - x_adj) % 4;
-
-        match column_type {
-            2 if y > 0 => (x + 1, y - 1),
-            0 if y < self.width - 1 => (x + 1, y + 1),
-            0 | 2 if x == 7 => (x - 2, y),
-            _ => (x - 1, y),
-        }
-    }
-
-    /// Find next module that can hold data.
-    fn next_available(&self, x: u8, y: u8) -> (u8, u8) {
-        let (mut x, mut y) = self.next(x, y);
-        while self.is_reserved(x, y) && !self.is_last(x, y) {
-            (x, y) = self.next(x, y);
-        }
-        (x, y)
-    }
-
-    fn draw_data(&mut self, data: impl Iterator<Item = u8>) {
-        let (mut x, mut y) = (self.width - 1, self.width - 1);
-        for byte in data {
-            for s in 0..8 {
-                if byte & (0x80 >> s) == 0 {
-                    self.set(x, y);
-                }
-                (x, y) = self.next_available(x, y);
-            }
-        }
-        // Set the remaining modules (0, 3 or 7 depending on version).
-        // because 0 correspond to a light module.
-        while !self.is_last(x, y) {
-            if !self.is_reserved(x, y) {
-                self.set(x, y);
-            }
-            (x, y) = self.next(x, y);
-        }
-    }
-
-    /// Apply checkerboard mask to all non-reserved modules.
-    fn apply_mask(&mut self) {
-        for x in 0..self.width {
-            for y in 0..self.width {
-                if (x ^ y) % 2 == 0 && !self.is_reserved(x, y) {
-                    self.xor(x, y);
-                }
-            }
-        }
-    }
-
-    /// Draw the QR code with the provided data iterator.
-    fn draw_all(&mut self, data: impl Iterator<Item = u8>) {
-        // First clear the table, as it may have already some data.
-        self.clear();
-        self.draw_finders();
-        self.draw_alignments();
-        self.draw_timing_patterns();
-        self.draw_version_info();
-        self.draw_data(data);
-        self.draw_maskinfo();
-        self.apply_mask();
-    }
-}
-
-/// C entry point for the rust QR Code generator.
-///
-/// Write the QR code image in the data buffer, and return the QR code width,
-/// or 0, if the data doesn't fit in a QR code.
-///
-/// * `url`: The base URL of the QR code. It will be encoded as Binary segment.
-/// * `data`: A pointer to the binary data, to be encoded. if URL is NULL, it
-///   will be encoded as binary segment, otherwise it will be encoded
-///   efficiently as a numeric segment, and appended to the URL.
-/// * `data_len`: Length of the data, that needs to be encoded, must be less
-///   than `data_size`.
-/// * `data_size`: Size of data buffer, it should be at least 4071 bytes to hold
-///   a V40 QR code. It will then be overwritten with the QR code image.
-/// * `tmp`: A temporary buffer that the QR code encoder will use, to write the
-///   segments and ECC.
-/// * `tmp_size`: Size of the temporary buffer, it must be at least 3706 bytes
-///   long for V40.
-///
-/// # Safety
-///
-/// * `url` must be null or point at a nul-terminated string.
-/// * `data` must be valid for reading and writing for `data_size` bytes.
-/// * `tmp` must be valid for reading and writing for `tmp_size` bytes.
-///
-/// They must remain valid for the duration of the function call.
-#[export]
-pub unsafe extern "C" fn drm_panic_qr_generate(
-    url: *const kernel::ffi::c_char,
-    data: *mut u8,
-    data_len: usize,
-    data_size: usize,
-    tmp: *mut u8,
-    tmp_size: usize,
-) -> u8 {
-    if data_size < 4071 || tmp_size < 3706 || data_len > data_size {
-        return 0;
-    }
-    // SAFETY: The caller ensures that `data` is a valid pointer for reading and
-    // writing `data_size` bytes.
-    let data_slice: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(data, data_size) };
-    // SAFETY: The caller ensures that `tmp` is a valid pointer for reading and
-    // writing `tmp_size` bytes.
-    let tmp_slice: &mut [u8] = unsafe { core::slice::from_raw_parts_mut(tmp, tmp_size) };
-    if url.is_null() {
-        match EncodedMsg::new(&[&Segment::Binary(&data_slice[0..data_len])], tmp_slice) {
-            None => 0,
-            Some(em) => {
-                let qr_image = QrImage::new(&em, data_slice);
-                qr_image.width
-            }
-        }
-    } else {
-        // SAFETY: The caller ensures that `url` is a valid pointer to a
-        // nul-terminated string.
-        let url_cstr: &CStr = unsafe { CStr::from_char_ptr(url) };
-        let segments = &[
-            &Segment::Binary(url_cstr.to_bytes()),
-            &Segment::Numeric(&data_slice[0..data_len]),
-        ];
-        match EncodedMsg::new(segments, tmp_slice) {
-            None => 0,
-            Some(em) => {
-                let qr_image = QrImage::new(&em, data_slice);
-                qr_image.width
-            }
-        }
-    }
-}
-
-/// Returns the maximum data size that can fit in a QR code of this version.
-/// * `version`: QR code version, between 1-40.
-/// * `url_len`: Length of the URL.
-///
-/// * If `url_len` > 0, remove the 2 segments header/length and also count the
-///   conversion to numeric segments.
-/// * If `url_len` = 0, only removes 3 bytes for 1 binary segment.
-///
-/// # Safety
-///
-/// Always safe to call.
-// Required to be unsafe due to the `#[export]` annotation.
-#[export]
-pub unsafe extern "C" fn drm_panic_qr_max_data_size(version: u8, url_len: usize) -> usize {
-    #[expect(clippy::manual_range_contains)]
-    if version < 1 || version > 40 {
-        return 0;
-    }
-    let max_data = Version(version as usize).max_data();
-
-    if url_len > 0 {
-        // Binary segment (URL) 4 + 16 bits, numeric segment (kmsg) 4 + 12 bits => 5 bytes.
-        if url_len + 5 >= max_data {
-            0
-        } else {
-            let max = max_data - url_len - 5;
-            (max * 39) / 40
-        }
-    } else {
-        // Remove 3 bytes for the binary segment (header 4 bits, length 16 bits, stop 4bits).
-        max_data - 3
-    }
-}
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index e26ead5b5e5893..856b40fdaac4a4 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -559,6 +559,7 @@ static int __drm_helper_update_and_validate(struct drm_connector *connector,
 int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
 					    uint32_t maxX, uint32_t maxY)
 {
+	const struct drm_connector_helper_funcs *funcs = connector->helper_private;
 	struct drm_device *dev = connector->dev;
 	struct drm_display_mode *mode;
 	int count = 0, ret;
@@ -592,8 +593,20 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
 			connector->status = connector_status_connected;
 		else
 			connector->status = connector_status_disconnected;
-		if (connector->funcs->force)
+
+		if (funcs && funcs->force_ctx) {
+			ret = funcs->force_ctx(connector, &ctx);
+			if (ret == -EDEADLK) {
+				connector->status = old_status;
+				drm_modeset_backoff(&ctx);
+				goto retry;
+			} else if (ret < 0) {
+				drm_dbg_kms(dev, "[CONNECTOR:%d:%s] force_ctx failed: %d\n",
+					    connector->base.id, connector->name, ret);
+			}
+		} else if (connector->funcs->force) {
 			connector->funcs->force(connector);
+		}
 	} else {
 		ret = drm_helper_probe_detect(connector, &ctx, true);
 
diff --git a/drivers/gpu/drm/drm_ras.c b/drivers/gpu/drm/drm_ras.c
index d6eab29a1394ca..b099eb67836e7e 100644
--- a/drivers/gpu/drm/drm_ras.c
+++ b/drivers/gpu/drm/drm_ras.c
@@ -41,6 +41,18 @@
  *    Userspace must provide Node ID, Error ID.
  *    Clears specific error counter of a node if supported.
  *
+ * 4. ERROR_REPORT: Subscribe to this multicast group to receive error events
+ *
+ * 5. ERROR_EVENT: Report an error event to userspace. The event contains device, node
+ *    and error information that triggered the event.
+ *
+ * 6. GET_ERROR_THRESHOLD: Query error threshold of a given counter.
+ *    Userspace must provide Node ID and Error ID.
+ *    Returns the error threshold of a specific counter.
+ *
+ * 7. SET_ERROR_THRESHOLD: Set error threshold of a given counter.
+ *    Userspace must provide Node ID, Error ID and threshold to be set.
+ *
  * Node registration:
  *
  * - drm_ras_node_register(): Registers a new node and assigns
@@ -61,6 +73,13 @@
  *     + The error counters in the driver doesn't need to be contiguous, but the
  *       driver must return -ENOENT to the query_error_counter as an indication
  *       that the ID should be skipped and not listed in the netlink API.
+ *     + The driver can optionally implement query_error_threshold() and
+ *       set_error_threshold() callbacks to facilitate getting/setting error
+ *       threshold of the counter. Threshold in RAS context means the number of
+ *       errors the hardware is expected to accumulate before it raises them to
+ *       software. This is to have a fine grained control over error notifications
+ *       that are raised by the hardware.
+ *     + The driver is responsible for error threshold bounds checking.
  *
  * Netlink handlers:
  *
@@ -72,6 +91,10 @@
  *   operation, fetching a counter value from a specific node.
  * - drm_ras_nl_clear_error_counter_doit(): Implements the CLEAR_ERROR_COUNTER doit
  *   operation, clearing a counter value from a specific node.
+ * - drm_ras_nl_get_error_threshold_doit(): Implements the GET_ERROR_THRESHOLD doit
+ *   operation, fetching the error threshold of a specific counter.
+ * - drm_ras_nl_set_error_threshold_doit(): Implements the SET_ERROR_THRESHOLD doit
+ *   operation, setting the error threshold of a specific counter.
  */
 
 static DEFINE_XARRAY_ALLOC(drm_ras_xa);
@@ -168,6 +191,40 @@ static int get_node_error_counter(u32 node_id, u32 error_id,
 	return node->query_error_counter(node, error_id, name, value);
 }
 
+static int get_node_error_threshold(u32 node_id, u32 error_id, const char **name, u32 *threshold)
+{
+	struct drm_ras_node *node;
+
+	node = xa_load(&drm_ras_xa, node_id);
+	if (!node)
+		return -ENOENT;
+
+	if (!node->query_error_threshold)
+		return -EOPNOTSUPP;
+
+	if (error_id < node->error_counter_range.first || error_id > node->error_counter_range.last)
+		return -EINVAL;
+
+	return node->query_error_threshold(node, error_id, name, threshold);
+}
+
+static int set_node_error_threshold(u32 node_id, u32 error_id, u32 threshold)
+{
+	struct drm_ras_node *node;
+
+	node = xa_load(&drm_ras_xa, node_id);
+	if (!node)
+		return -ENOENT;
+
+	if (!node->set_error_threshold)
+		return -EOPNOTSUPP;
+
+	if (error_id < node->error_counter_range.first || error_id > node->error_counter_range.last)
+		return -EINVAL;
+
+	return node->set_error_threshold(node, error_id, threshold);
+}
+
 static int msg_reply_value(struct sk_buff *msg, u32 error_id,
 			   const char *error_name, u32 value)
 {
@@ -186,6 +243,50 @@ static int msg_reply_value(struct sk_buff *msg, u32 error_id,
 			   value);
 }
 
+static int msg_put_error_event_attrs(struct sk_buff *msg, struct drm_ras_node *node,
+				     u32 error_id, const char *error_name, u32 value)
+{
+	int ret;
+
+	ret = nla_put_string(msg, DRM_RAS_A_ERROR_EVENT_ATTRS_DEVICE_NAME, node->device_name);
+	if (ret)
+		return ret;
+
+	ret = nla_put_u32(msg, DRM_RAS_A_ERROR_EVENT_ATTRS_NODE_ID, node->id);
+	if (ret)
+		return ret;
+
+	ret = nla_put_string(msg, DRM_RAS_A_ERROR_EVENT_ATTRS_NODE_NAME, node->node_name);
+	if (ret)
+		return ret;
+
+	ret = nla_put_u32(msg, DRM_RAS_A_ERROR_EVENT_ATTRS_ERROR_ID, error_id);
+	if (ret)
+		return ret;
+
+	ret = nla_put_string(msg, DRM_RAS_A_ERROR_EVENT_ATTRS_ERROR_NAME, error_name);
+	if (ret)
+		return ret;
+
+	return nla_put_u32(msg, DRM_RAS_A_ERROR_EVENT_ATTRS_ERROR_VALUE, value);
+}
+
+static int msg_reply_threshold(struct sk_buff *msg, u32 error_id, const char *error_name,
+			       u32 threshold)
+{
+	int ret;
+
+	ret = nla_put_u32(msg, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID, error_id);
+	if (ret)
+		return ret;
+
+	ret = nla_put_string(msg, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_NAME, error_name);
+	if (ret)
+		return ret;
+
+	return nla_put_u32(msg, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_THRESHOLD, threshold);
+}
+
 static int doit_reply_value(struct genl_info *info, u32 node_id,
 			    u32 error_id)
 {
@@ -201,26 +302,134 @@ static int doit_reply_value(struct genl_info *info, u32 node_id,
 
 	hdr = genlmsg_iput(msg, info);
 	if (!hdr) {
-		nlmsg_free(msg);
-		return -EMSGSIZE;
+		ret = -EMSGSIZE;
+		goto free_msg;
 	}
 
 	ret = get_node_error_counter(node_id, error_id,
 				     &error_name, &value);
 	if (ret)
-		return ret;
+		goto cancel_msg;
 
 	ret = msg_reply_value(msg, error_id, error_name, value);
-	if (ret) {
-		genlmsg_cancel(msg, hdr);
-		nlmsg_free(msg);
-		return ret;
+	if (ret)
+		goto cancel_msg;
+
+	genlmsg_end(msg, hdr);
+
+	return genlmsg_reply(msg, info);
+
+cancel_msg:
+	genlmsg_cancel(msg, hdr);
+free_msg:
+	nlmsg_free(msg);
+	return ret;
+}
+
+static int doit_reply_threshold(struct genl_info *info, u32 node_id, u32 error_id)
+{
+	const char *error_name;
+	struct sk_buff *msg;
+	struct nlattr *hdr;
+	u32 threshold;
+	int ret;
+
+	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_iput(msg, info);
+	if (!hdr) {
+		ret = -EMSGSIZE;
+		goto free_msg;
 	}
 
+	ret = get_node_error_threshold(node_id, error_id, &error_name, &threshold);
+	if (ret)
+		goto cancel_msg;
+
+	ret = msg_reply_threshold(msg, error_id, error_name, threshold);
+	if (ret)
+		goto cancel_msg;
+
 	genlmsg_end(msg, hdr);
 
 	return genlmsg_reply(msg, info);
+
+cancel_msg:
+	genlmsg_cancel(msg, hdr);
+free_msg:
+	nlmsg_free(msg);
+	return ret;
+}
+
+/**
+ * drm_ras_nl_error_event() - Report an error event
+ * @node: Node structure
+ * @error_id: ID of the error
+ * @error_name: Name of the error
+ * @value: Value of the error counter
+ *
+ * Report an error-event to userspace using the error-report multicast group.
+ *
+ * Context: Process context only. Uses %GFP_KERNEL and multicasts to all
+ *          netns, which is unbounded work; callers in interrupt handlers
+ *          or other atomic context must defer event.
+ *
+ * Return: 0 on success, or negative errno on failure.
+ */
+int drm_ras_nl_error_event(struct drm_ras_node *node, u32 error_id, const char *error_name,
+			   u32 value)
+{
+	struct genl_info info;
+	struct sk_buff *msg;
+	struct nlattr *hdr;
+	int ret;
+
+	if (!node || !error_name)
+		return -EINVAL;
+
+	/* Check the node is currently registered */
+	if (xa_load(&drm_ras_xa, node->id) != node)
+		return -ENOENT;
+
+	/* Currently only Error Counter events are supported */
+	if (node->type != DRM_RAS_NODE_TYPE_ERROR_COUNTER)
+		return -EOPNOTSUPP;
+
+	/* Check the error ID is within the valid range */
+	if (error_id < node->error_counter_range.first ||
+	    error_id > node->error_counter_range.last)
+		return -EINVAL;
+
+	genl_info_init_ntf(&info, &drm_ras_nl_family, DRM_RAS_CMD_ERROR_EVENT);
+
+	msg = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_iput(msg, &info);
+	if (!hdr) {
+		ret = -EMSGSIZE;
+		goto free_msg;
+	}
+
+	ret = msg_put_error_event_attrs(msg, node, error_id, error_name, value);
+	if (ret)
+		goto cancel_msg;
+
+	genlmsg_end(msg, hdr);
+	genlmsg_multicast_allns(&drm_ras_nl_family, msg, 0, DRM_RAS_NLGRP_ERROR_REPORT);
+
+	return 0;
+
+cancel_msg:
+	genlmsg_cancel(msg, hdr);
+free_msg:
+	nlmsg_free(msg);
+	return ret;
 }
+EXPORT_SYMBOL(drm_ras_nl_error_event);
 
 /**
  * drm_ras_nl_get_error_counter_dumpit() - Dump all Error Counters
@@ -355,6 +564,59 @@ int drm_ras_nl_clear_error_counter_doit(struct sk_buff *skb,
 	return node->clear_error_counter(node, error_id);
 }
 
+/**
+ * drm_ras_nl_get_error_threshold_doit() - Query error threshold of a counter
+ * @skb: Netlink message buffer
+ * @info: Generic Netlink info containing attributes of the request
+ *
+ * Extracts the Node ID and Error ID from the netlink attributes and retrieves
+ * the error threshold of the corresponding counter. Sends the result back to
+ * the requesting user via the standard Genl reply.
+ *
+ * Return: 0 on success, or negative errno on failure.
+ */
+int drm_ras_nl_get_error_threshold_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	u32 node_id, error_id;
+
+	if (!info->attrs ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID) ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID))
+		return -EINVAL;
+
+	node_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID]);
+	error_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID]);
+
+	return doit_reply_threshold(info, node_id, error_id);
+}
+
+/**
+ * drm_ras_nl_set_error_threshold_doit() - Set error threshold of a counter
+ * @skb: Netlink message buffer
+ * @info: Generic Netlink info containing attributes of the request
+ *
+ * Extracts the Node ID, Error ID and threshold from the netlink attributes and
+ * sets the error threshold of the corresponding counter.
+ *
+ * Return: 0 on success, or negative errno on failure.
+ */
+int drm_ras_nl_set_error_threshold_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	u32 node_id, error_id, threshold;
+
+	if (!info->attrs ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID) ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID) ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_THRESHOLD))
+		return -EINVAL;
+
+	node_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID]);
+	error_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID]);
+	threshold = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_THRESHOLD]);
+
+	return set_node_error_threshold(node_id, error_id, threshold);
+}
+
 /**
  * drm_ras_node_register() - Register a new RAS node
  * @node: Node structure to register
diff --git a/drivers/gpu/drm/drm_ras_nl.c b/drivers/gpu/drm/drm_ras_nl.c
index dea1c1b2494e81..c9f5e0ceb3b5fd 100644
--- a/drivers/gpu/drm/drm_ras_nl.c
+++ b/drivers/gpu/drm/drm_ras_nl.c
@@ -28,6 +28,19 @@ static const struct nla_policy drm_ras_clear_error_counter_nl_policy[DRM_RAS_A_E
 	[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID] = { .type = NLA_U32, },
 };
 
+/* DRM_RAS_CMD_GET_ERROR_THRESHOLD - do */
+static const struct nla_policy drm_ras_get_error_threshold_nl_policy[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID + 1] = {
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID] = { .type = NLA_U32, },
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID] = { .type = NLA_U32, },
+};
+
+/* DRM_RAS_CMD_SET_ERROR_THRESHOLD - do */
+static const struct nla_policy drm_ras_set_error_threshold_nl_policy[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_THRESHOLD + 1] = {
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID] = { .type = NLA_U32, },
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID] = { .type = NLA_U32, },
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_THRESHOLD] = { .type = NLA_U32, },
+};
+
 /* Ops table for drm_ras */
 static const struct genl_split_ops drm_ras_nl_ops[] = {
 	{
@@ -56,6 +69,24 @@ static const struct genl_split_ops drm_ras_nl_ops[] = {
 		.maxattr	= DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID,
 		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
 	},
+	{
+		.cmd		= DRM_RAS_CMD_GET_ERROR_THRESHOLD,
+		.doit		= drm_ras_nl_get_error_threshold_doit,
+		.policy		= drm_ras_get_error_threshold_nl_policy,
+		.maxattr	= DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRM_RAS_CMD_SET_ERROR_THRESHOLD,
+		.doit		= drm_ras_nl_set_error_threshold_doit,
+		.policy		= drm_ras_set_error_threshold_nl_policy,
+		.maxattr	= DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_THRESHOLD,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+};
+
+static const struct genl_multicast_group drm_ras_nl_mcgrps[] = {
+	[DRM_RAS_NLGRP_ERROR_REPORT] = { "error-report", },
 };
 
 struct genl_family drm_ras_nl_family __ro_after_init = {
@@ -66,4 +97,6 @@ struct genl_family drm_ras_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.split_ops	= drm_ras_nl_ops,
 	.n_split_ops	= ARRAY_SIZE(drm_ras_nl_ops),
+	.mcgrps		= drm_ras_nl_mcgrps,
+	.n_mcgrps	= ARRAY_SIZE(drm_ras_nl_mcgrps),
 };
diff --git a/drivers/gpu/drm/drm_ras_nl.h b/drivers/gpu/drm/drm_ras_nl.h
index a398643572a5a0..9aef097ea10f39 100644
--- a/drivers/gpu/drm/drm_ras_nl.h
+++ b/drivers/gpu/drm/drm_ras_nl.h
@@ -20,6 +20,14 @@ int drm_ras_nl_get_error_counter_dumpit(struct sk_buff *skb,
 					struct netlink_callback *cb);
 int drm_ras_nl_clear_error_counter_doit(struct sk_buff *skb,
 					struct genl_info *info);
+int drm_ras_nl_get_error_threshold_doit(struct sk_buff *skb,
+					struct genl_info *info);
+int drm_ras_nl_set_error_threshold_doit(struct sk_buff *skb,
+					struct genl_info *info);
+
+enum {
+	DRM_RAS_NLGRP_ERROR_REPORT,
+};
 
 extern struct genl_family drm_ras_nl_family;
 
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
index 8e1d07b9f1e307..5e324c77ad2763 100644
--- a/drivers/gpu/drm/drm_simple_kms_helper.c
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -20,16 +20,6 @@ static const struct drm_encoder_funcs drm_simple_encoder_funcs_cleanup = {
 	.destroy = drm_encoder_cleanup,
 };
 
-int drm_simple_encoder_init(struct drm_device *dev,
-			    struct drm_encoder *encoder,
-			    int encoder_type)
-{
-	return drm_encoder_init(dev, encoder,
-				&drm_simple_encoder_funcs_cleanup,
-				encoder_type, NULL);
-}
-EXPORT_SYMBOL(drm_simple_encoder_init);
-
 void *__drmm_simple_encoder_alloc(struct drm_device *dev, size_t size,
 				  size_t offset, int encoder_type)
 {
@@ -102,17 +92,6 @@ static const struct drm_crtc_helper_funcs drm_simple_kms_crtc_helper_funcs = {
 	.atomic_disable = drm_simple_kms_crtc_disable,
 };
 
-static void drm_simple_kms_crtc_reset(struct drm_crtc *crtc)
-{
-	struct drm_simple_display_pipe *pipe;
-
-	pipe = container_of(crtc, struct drm_simple_display_pipe, crtc);
-	if (!pipe->funcs || !pipe->funcs->reset_crtc)
-		return drm_atomic_helper_crtc_reset(crtc);
-
-	return pipe->funcs->reset_crtc(pipe);
-}
-
 static struct drm_crtc_state *drm_simple_kms_crtc_duplicate_state(struct drm_crtc *crtc)
 {
 	struct drm_simple_display_pipe *pipe;
@@ -158,7 +137,7 @@ static void drm_simple_kms_crtc_disable_vblank(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_funcs drm_simple_kms_crtc_funcs = {
-	.reset = drm_simple_kms_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -281,15 +260,16 @@ static const struct drm_plane_helper_funcs drm_simple_kms_plane_helper_funcs = {
 	.atomic_update = drm_simple_kms_plane_atomic_update,
 };
 
-static void drm_simple_kms_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *
+drm_simple_kms_plane_create_state(struct drm_plane *plane)
 {
 	struct drm_simple_display_pipe *pipe;
 
 	pipe = container_of(plane, struct drm_simple_display_pipe, plane);
-	if (!pipe->funcs || !pipe->funcs->reset_plane)
-		return drm_atomic_helper_plane_reset(plane);
+	if (!pipe->funcs || !pipe->funcs->create_plane_state)
+		return drm_atomic_helper_plane_create_state(plane);
 
-	return pipe->funcs->reset_plane(pipe);
+	return pipe->funcs->create_plane_state(pipe);
 }
 
 static struct drm_plane_state *drm_simple_kms_plane_duplicate_state(struct drm_plane *plane)
@@ -319,7 +299,7 @@ static const struct drm_plane_funcs drm_simple_kms_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_simple_kms_plane_reset,
+	.atomic_create_state	= drm_simple_kms_plane_create_state,
 	.atomic_duplicate_state	= drm_simple_kms_plane_duplicate_state,
 	.atomic_destroy_state	= drm_simple_kms_plane_destroy_state,
 	.format_mod_supported   = drm_simple_kms_format_mod_supported,
@@ -363,7 +343,8 @@ int drm_simple_display_pipe_init(struct drm_device *dev,
 		return ret;
 
 	encoder->possible_crtcs = drm_crtc_mask(crtc);
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_NONE);
+	ret = drm_encoder_init(dev, encoder, &drm_simple_encoder_funcs_cleanup,
+			       DRM_MODE_ENCODER_NONE, NULL);
 	if (ret || !connector)
 		return ret;
 
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index ef4e923a872843..7b16d32b6ff65f 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -589,35 +589,3 @@ struct device *drm_sysfs_minor_alloc(struct drm_minor *minor)
 	put_device(kdev);
 	return ERR_PTR(r);
 }
-
-/**
- * drm_class_device_register - register new device with the DRM sysfs class
- * @dev: device to register
- *
- * Registers a new &struct device within the DRM sysfs class. Essentially only
- * used by ttm to have a place for its global settings. Drivers should never use
- * this.
- */
-int drm_class_device_register(struct device *dev)
-{
-	if (!drm_class || IS_ERR(drm_class))
-		return -ENOENT;
-
-	dev->class = drm_class;
-	return device_register(dev);
-}
-EXPORT_SYMBOL_GPL(drm_class_device_register);
-
-/**
- * drm_class_device_unregister - unregister device with the DRM sysfs class
- * @dev: device to unregister
- *
- * Unregisters a &struct device from the DRM sysfs class. Essentially only used
- * by ttm to have a place for its global settings. Drivers should never use
- * this.
- */
-void drm_class_device_unregister(struct device *dev)
-{
-	return device_unregister(dev);
-}
-EXPORT_SYMBOL_GPL(drm_class_device_unregister);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 5c23182f4d33ad..bfcb2328f5c0d5 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -13,6 +13,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
 #include <linux/reset.h>
+#include <linux/seq_file.h>
 #include <linux/thermal.h>
 
 #include <drm/drm_print.h>
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index e1c9e2946b8ac3..82c23882992126 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -26,7 +26,6 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/exynos_drm.h>
@@ -147,7 +146,6 @@ static const struct component_ops exynos_dp_ops = {
 static int exynos_dp_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *np;
 	struct exynos_dp_device *dp;
 
 	dp = devm_kzalloc(&pdev->dev, sizeof(struct exynos_dp_device),
@@ -163,18 +161,6 @@ static int exynos_dp_probe(struct platform_device *pdev)
 	 */
 	platform_set_drvdata(pdev, dp);
 
-	/* This is for the backward compatibility. */
-	np = of_parse_phandle(dev->of_node, "panel", 0);
-	if (np) {
-		dp->plat_data.panel = of_drm_find_panel(np);
-
-		of_node_put(np);
-		if (IS_ERR(dp->plat_data.panel))
-			return PTR_ERR(dp->plat_data.panel);
-
-		goto out;
-	}
-
 	if (of_get_display_timings(dev->of_node)) {
 		dp->plat_data.next_bridge = devm_drm_of_display_mode_bridge(dp->dev,
 									dp->dev->of_node,
@@ -189,19 +175,11 @@ static int exynos_dp_probe(struct platform_device *pdev)
 	dp->plat_data.power_off = exynos_dp_poweroff;
 	dp->plat_data.ops = &exynos_dp_ops;
 
-out:
 	dp->adp = analogix_dp_probe(dev, &dp->plat_data);
-	if (IS_ERR(dp->adp)) {
-		/*
-		 * The driver core does not invoke remove() for failed probes,
-		 * so release the probe-time panel reference here.
-		 */
-		if (dp->plat_data.panel)
-			drm_panel_put(dp->plat_data.panel);
+	if (IS_ERR(dp->adp))
 		return PTR_ERR(dp->adp);
-	}
 
-	if (dp->plat_data.panel || dp->plat_data.next_bridge)
+	if (dp->plat_data.next_bridge)
 		return component_add(&pdev->dev, &exynos_dp_ops);
 	else
 		return analogix_dp_finish_probe(dp->adp);
@@ -209,16 +187,6 @@ static int exynos_dp_probe(struct platform_device *pdev)
 
 static void exynos_dp_remove(struct platform_device *pdev)
 {
-	struct exynos_dp_device *dp = platform_get_drvdata(pdev);
-
-	/*
-	 * Release the probe-time reference from of_drm_find_panel(). If bind
-	 * ran, the panel_bridge holds a second reference that devm cleanup
-	 * will release when the bridge is destroyed after remove() returns.
-	 */
-	if (dp->plat_data.panel)
-		drm_panel_put(dp->plat_data.panel);
-
 	component_del(&pdev->dev, &exynos_dp_ops);
 }
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index b071c019670de8..458cad64bd073c 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -163,7 +163,7 @@ static const struct drm_crtc_funcs exynos_crtc_funcs = {
 	.set_config	= drm_atomic_helper_set_config,
 	.page_flip	= drm_atomic_helper_page_flip,
 	.destroy	= exynos_drm_crtc_destroy,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = exynos_drm_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dpi.c b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
index 0683236aa5dff9..74d3a2bebb742c 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dpi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
@@ -201,8 +201,7 @@ int exynos_dpi_bind(struct drm_device *dev, struct drm_encoder *encoder)
 	ret = drm_encoder_init(dev, encoder, &exynos_dpi_encoder_funcs,
 			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret) {
-		DRM_DEV_ERROR(encoder_to_dpi(encoder)->dev,
-			      "failed to create encoder ret = %d\n", ret);
+		drm_err(dev, "failed to create encoder ret = %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index 5c04ef87e4007e..020e8c9cf7de23 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -122,23 +122,19 @@ static void exynos_plane_mode_set(struct exynos_drm_plane_state *exynos_state)
 			  exynos_state->crtc.w, exynos_state->crtc.h);
 }
 
-static void exynos_drm_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *exynos_drm_plane_create_state(struct drm_plane *plane)
 {
 	struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
 	struct exynos_drm_plane_state *exynos_state;
 
-	if (plane->state) {
-		exynos_state = to_exynos_plane_state(plane->state);
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-		kfree(exynos_state);
-		plane->state = NULL;
-	}
-
 	exynos_state = kzalloc_obj(*exynos_state);
-	if (exynos_state) {
-		__drm_atomic_helper_plane_reset(plane, &exynos_state->base);
-		plane->state->zpos = exynos_plane->config->zpos;
-	}
+	if (!exynos_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&exynos_state->base, plane);
+	exynos_state->base.zpos = exynos_plane->config->zpos;
+
+	return &exynos_state->base;
 }
 
 static struct drm_plane_state *
@@ -169,7 +165,7 @@ static struct drm_plane_funcs exynos_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_cleanup,
-	.reset		= exynos_drm_plane_reset,
+	.atomic_create_state = exynos_drm_plane_create_state,
 	.atomic_duplicate_state = exynos_drm_plane_duplicate_state,
 	.atomic_destroy_state = exynos_drm_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index 59dea853d3641e..077571eae9d652 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -452,8 +452,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 	ret = drm_encoder_init(drm_dev, encoder, &exynos_vidi_encoder_funcs,
 			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret) {
-		DRM_DEV_ERROR(dev, "failed to initialize encoder ret = %d\n",
-			      ret);
+		drm_err(drm_dev, "failed to initialize encoder ret = %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index d4fce39c2b61b0..055dc04f9b3e57 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -1869,8 +1869,7 @@ static int hdmi_bind(struct device *dev, struct device *master, void *data)
 	ret = drm_encoder_init(drm_dev, encoder, &exynos_hdmi_encoder_funcs,
 			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret) {
-		DRM_DEV_ERROR(dev, "failed to initialize encoder ret = %d\n",
-			      ret);
+		drm_err(drm_dev, "failed to initialize encoder ret = %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
index 9ef16f5eb613e5..73c0675e6c54bd 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
@@ -161,7 +161,7 @@ static const struct drm_crtc_funcs fsl_dcu_drm_crtc_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.destroy = drm_crtc_cleanup,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.set_config = drm_atomic_helper_set_config,
 	.enable_vblank = fsl_dcu_drm_crtc_enable_vblank,
 	.disable_vblank = fsl_dcu_drm_crtc_disable_vblank,
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index b34f880c1e098a..92794b337512cd 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -177,7 +177,7 @@ static const struct drm_plane_funcs fsl_dcu_drm_plane_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.destroy = drm_plane_helper_destroy,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.update_plane = drm_atomic_helper_update_plane,
 };
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
index ec71fbbb0eb892..c64264541fe21b 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
@@ -11,14 +11,18 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "fsl_dcu_drm_drv.h"
 #include "fsl_tcon.h"
 
+static const struct drm_encoder_funcs fsl_dcu_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 int fsl_dcu_drm_encoder_create(struct fsl_dcu_drm_device *fsl_dev,
 			       struct drm_crtc *crtc)
 {
@@ -31,8 +35,8 @@ int fsl_dcu_drm_encoder_create(struct fsl_dcu_drm_device *fsl_dev,
 	if (fsl_dev->tcon)
 		fsl_tcon_bypass_enable(fsl_dev->tcon);
 
-	ret = drm_simple_encoder_init(fsl_dev->drm, encoder,
-				      DRM_MODE_ENCODER_LVDS);
+	ret = drm_encoder_init(fsl_dev->drm, encoder, &fsl_dcu_encoder_funcs,
+			       DRM_MODE_ENCODER_LVDS, NULL);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c
index 0df75a4a773961..f79484b9d593ed 100644
--- a/drivers/gpu/drm/gma500/psb_intel_display.c
+++ b/drivers/gpu/drm/gma500/psb_intel_display.c
@@ -9,8 +9,11 @@
 #include <linux/delay.h>
 #include <linux/i2c.h>
 
+#include <drm/drm_fourcc.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 
 #include "framebuffer.h"
@@ -471,11 +474,25 @@ static void psb_intel_cursor_init(struct drm_device *dev,
 	REG_WRITE(base[gma_crtc->pipe], 0);
 }
 
+/*
+ * The display engine programs the primary plane with
+ * DISPPLANE_32BPP_NO_ALPHA, so no formats with alpha.
+ */
+static const uint32_t gma_primary_formats[] = {
+	DRM_FORMAT_XRGB8888,
+};
+
+static const struct drm_plane_funcs gma_primary_plane_funcs = {
+	.update_plane = drm_plane_helper_update_primary,
+	.disable_plane = drm_plane_helper_disable_primary,
+};
+
 void psb_intel_crtc_init(struct drm_device *dev, int pipe,
 		     struct psb_intel_mode_device *mode_dev)
 {
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 	struct gma_crtc *gma_crtc;
+	struct drm_plane *primary;
 	int i;
 
 	/* We allocate a extra array of drm_connector pointers
@@ -494,7 +511,17 @@ void psb_intel_crtc_init(struct drm_device *dev, int pipe,
 		return;
 	}
 
-	drm_crtc_init(dev, &gma_crtc->base, &gma_crtc_funcs);
+	primary = drmm_universal_plane_alloc(dev, struct drm_plane, dev, 0,
+					     &gma_primary_plane_funcs,
+					     gma_primary_formats,
+					     ARRAY_SIZE(gma_primary_formats),
+					     NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (IS_ERR(primary))
+		goto err_free_crtc;
+
+	if (drm_crtc_init_with_planes(dev, &gma_crtc->base, primary, NULL,
+				      &gma_crtc_funcs, NULL))
+		goto err_free_crtc;
 
 	/* Set the CRTC clock functions from chip specific data */
 	gma_crtc->clock_funcs = dev_priv->ops->clock_funcs;
@@ -524,6 +551,11 @@ void psb_intel_crtc_init(struct drm_device *dev, int pipe,
 
 	/* Set to true so that the pipe is forced off on initial config. */
 	gma_crtc->active = true;
+	return;
+
+err_free_crtc:
+	kfree(gma_crtc->crtc_state);
+	kfree(gma_crtc);
 }
 
 struct drm_crtc *psb_intel_get_crtc_from_pipe(struct drm_device *dev, int pipe)
diff --git a/drivers/gpu/drm/gud/gud_drv.c b/drivers/gpu/drm/gud/gud_drv.c
index 3a1b9e2a2eaa19..680447f426f37b 100644
--- a/drivers/gpu/drm/gud/gud_drv.c
+++ b/drivers/gpu/drm/gud/gud_drv.c
@@ -347,7 +347,7 @@ static const struct drm_crtc_helper_funcs gud_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs gud_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
index b4ab53db1c087d..7849abd0488e6f 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
@@ -20,6 +20,7 @@
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_vblank.h>
+#include <drm/drm_vblank_helper.h>
 
 #include "hibmc_drm_drv.h"
 #include "hibmc_drm_regs.h"
@@ -420,19 +421,6 @@ static void hibmc_crtc_atomic_begin(struct drm_crtc *crtc,
 	/* We can add more initialization as needed. */
 }
 
-static void hibmc_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_commit *state)
-
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&crtc->dev->event_lock, flags);
-	if (crtc->state->event)
-		drm_crtc_send_vblank_event(crtc, crtc->state->event);
-	crtc->state->event = NULL;
-	spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-}
-
 static int hibmc_crtc_enable_vblank(struct drm_crtc *crtc)
 {
 	struct hibmc_drm_private *priv = to_hibmc_drm_private(crtc->dev);
@@ -491,7 +479,7 @@ static const struct drm_crtc_funcs hibmc_crtc_funcs = {
 	.page_flip = drm_atomic_helper_page_flip,
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = drm_crtc_cleanup,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state =  drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = hibmc_crtc_enable_vblank,
@@ -502,7 +490,7 @@ static const struct drm_crtc_funcs hibmc_crtc_funcs = {
 static const struct drm_crtc_helper_funcs hibmc_crtc_helper_funcs = {
 	.mode_set_nofb	= hibmc_crtc_mode_set_nofb,
 	.atomic_begin	= hibmc_crtc_atomic_begin,
-	.atomic_flush	= hibmc_crtc_atomic_flush,
+	.atomic_flush	= drm_crtc_vblank_atomic_flush,
 	.atomic_enable	= hibmc_crtc_atomic_enable,
 	.atomic_disable	= hibmc_crtc_atomic_disable,
 	.mode_valid = hibmc_crtc_mode_valid,
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
index 4d85c89f3f8827..602930964e9309 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
@@ -49,9 +49,10 @@ static irqreturn_t hibmc_interrupt(int irq, void *arg)
 		writel(HIBMC_RAW_INTERRUPT_VBLANK(1),
 		       priv->mmio + HIBMC_RAW_INTERRUPT);
 		drm_handle_vblank(dev, 0);
+		return IRQ_HANDLED;
 	}
 
-	return IRQ_HANDLED;
+	return IRQ_NONE;
 }
 
 static irqreturn_t hibmc_dp_interrupt(int irq, void *arg)
@@ -67,7 +68,7 @@ static irqreturn_t hibmc_dp_interrupt(int irq, void *arg)
 		return IRQ_WAKE_THREAD;
 	}
 
-	return IRQ_HANDLED;
+	return IRQ_NONE;
 }
 
 static int hibmc_dumb_create(struct drm_file *file, struct drm_device *dev,
diff --git a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
index 15042365dec009..62c5bd3277da12 100644
--- a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
+++ b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
@@ -20,11 +20,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_device.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "dw_dsi_reg.h"
 
@@ -687,6 +687,10 @@ static int dsi_encoder_atomic_check(struct drm_encoder *encoder,
 	return 0;
 }
 
+static const struct drm_encoder_funcs dw_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs dw_encoder_helper_funcs = {
 	.atomic_check	= dsi_encoder_atomic_check,
 	.mode_valid	= dsi_encoder_mode_valid,
@@ -708,7 +712,8 @@ static int dw_drm_encoder_init(struct device *dev,
 	}
 
 	encoder->possible_crtcs = crtc_mask;
-	ret = drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_DSI);
+	ret = drm_encoder_init(drm_dev, encoder, &dw_encoder_funcs,
+			       DRM_MODE_ENCODER_DSI, NULL);
 	if (ret) {
 		DRM_ERROR("failed to init dsi encoder\n");
 		return ret;
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
index 902c5dace4ff56..bca839dd600bd3 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
@@ -540,7 +540,7 @@ static const struct drm_crtc_funcs ade_crtc_funcs = {
 	.destroy	= drm_crtc_cleanup,
 	.set_config	= drm_atomic_helper_set_config,
 	.page_flip	= drm_atomic_helper_page_flip,
-	.reset		= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank	= ade_crtc_enable_vblank,
@@ -837,7 +837,7 @@ static struct drm_plane_funcs ade_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/hyperv/Kconfig b/drivers/gpu/drm/hyperv/Kconfig
index 86234f6a73f2aa..e48e35fb7f8b24 100644
--- a/drivers/gpu/drm/hyperv/Kconfig
+++ b/drivers/gpu/drm/hyperv/Kconfig
@@ -8,7 +8,6 @@ config DRM_HYPERV
 	help
 	 This is a KMS driver for Hyper-V synthetic video device. Choose this
 	 option if you would like to enable drm driver for Hyper-V virtual
-	 machine. Unselect Hyper-V framebuffer driver (CONFIG_FB_HYPERV) so
-	 that DRM driver is used by default.
+	 machine.
 
 	 If M is selected the module will be called hyperv_drm.
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm.h b/drivers/gpu/drm/hyperv/hyperv_drm.h
index 78136ec2c2f4e5..29e6f54af15f45 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm.h
+++ b/drivers/gpu/drm/hyperv/hyperv_drm.h
@@ -29,7 +29,6 @@ struct hv_drm_device {
 	unsigned long fb_base;
 	unsigned long fb_size;
 	struct completion wait;
-	u32 synthvid_version;
 	u32 mmio_megabytes;
 	bool dirt_needed;
 
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
index 1855749c1e41f7..523d1c7ea07204 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
@@ -18,6 +18,7 @@
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_panic.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_plane.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
@@ -126,7 +127,7 @@ static const struct drm_crtc_helper_funcs hv_drm_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs hv_drm_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -237,6 +238,7 @@ static const struct drm_plane_funcs hv_drm_plane_funcs = {
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
 	DRM_GEM_SHADOW_PLANE_FUNCS,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static const struct drm_encoder_funcs hv_drm_simple_encoder_funcs_cleanup = {
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
index f0ef627b4898a3..1d09bba1efdd88 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
@@ -17,7 +17,7 @@
 #define SYNTHVID_VER_GET_MAJOR(ver) (ver & 0x0000ffff)
 #define SYNTHVID_VER_GET_MINOR(ver) ((ver & 0xffff0000) >> 16)
 
-/* Support for VERSION_WIN7 is removed. #define is retained for reference. */
+/* Support for WIN7 and WIN8 is removed. #define's retained for reference. */
 #define SYNTHVID_VERSION_WIN7 SYNTHVID_VERSION(3, 0)
 #define SYNTHVID_VERSION_WIN8 SYNTHVID_VERSION(3, 2)
 #define SYNTHVID_VERSION_WIN10 SYNTHVID_VERSION(3, 5)
@@ -181,16 +181,6 @@ struct synthvid_msg {
 	};
 } __packed;
 
-static inline bool hv_drm_version_ge(u32 ver1, u32 ver2)
-{
-	if (SYNTHVID_VER_GET_MAJOR(ver1) > SYNTHVID_VER_GET_MAJOR(ver2) ||
-	    (SYNTHVID_VER_GET_MAJOR(ver1) == SYNTHVID_VER_GET_MAJOR(ver2) &&
-	     SYNTHVID_VER_GET_MINOR(ver1) >= SYNTHVID_VER_GET_MINOR(ver2)))
-		return true;
-
-	return false;
-}
-
 static inline int hv_drm_sendpacket(struct hv_device *hdev, struct synthvid_msg *msg)
 {
 	static atomic64_t request_id = ATOMIC64_INIT(0);
@@ -236,7 +226,6 @@ static int hv_drm_negotiate_version(struct hv_device *hdev, u32 ver)
 		return -ENODEV;
 	}
 
-	hv->synthvid_version = ver;
 	drm_info(dev, "Synthvid Version major %d, minor %d\n",
 		 SYNTHVID_VER_GET_MAJOR(ver), SYNTHVID_VER_GET_MINOR(ver));
 
@@ -557,23 +546,12 @@ int hv_drm_connect_vsp(struct hv_device *hdev)
 		return ret;
 	}
 
-	/* Negotiate the protocol version with host */
-	switch (vmbus_proto_version) {
-	case VERSION_WIN10:
-	case VERSION_WIN10_V5:
-		ret = hv_drm_negotiate_version(hdev, SYNTHVID_VERSION_WIN10);
-		if (!ret)
-			break;
-		fallthrough;
-	case VERSION_WIN8:
-	case VERSION_WIN8_1:
-		ret = hv_drm_negotiate_version(hdev, SYNTHVID_VERSION_WIN8);
-		break;
-	default:
-		ret = hv_drm_negotiate_version(hdev, SYNTHVID_VERSION_WIN10);
-		break;
-	}
-
+	/*
+	 * Negotiate the protocol version with host. Since support for hosts
+	 * older than WIN10 has been removed from Linux, only negotiate the
+	 * WIN10 version.
+	 */
+	ret = hv_drm_negotiate_version(hdev, SYNTHVID_VERSION_WIN10);
 	if (ret) {
 		drm_err(dev, "Synthetic video device version not accepted %d\n", ret);
 		goto error;
@@ -581,11 +559,9 @@ int hv_drm_connect_vsp(struct hv_device *hdev)
 
 	hv->screen_depth = SYNTHVID_DEPTH_WIN8;
 
-	if (hv_drm_version_ge(hv->synthvid_version, SYNTHVID_VERSION_WIN10)) {
-		ret = hv_drm_get_supported_resolution(hdev);
-		if (ret)
-			drm_err(dev, "Failed to get supported resolution from host, use default\n");
-	}
+	ret = hv_drm_get_supported_resolution(hdev);
+	if (ret)
+		drm_err(dev, "Failed to get supported resolution from host, use default\n");
 
 	if (!hv->screen_width_max) {
 		hv->screen_width_max = SYNTHVID_WIDTH_WIN8;
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index a83fa8be0aba1b..d4e7764621e181 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -349,6 +349,7 @@ i915-y += \
 	display/intel_cx0_phy.o \
 	display/intel_ddi.o \
 	display/intel_ddi_buf_trans.o \
+	display/intel_dip.o \
 	display/intel_display_device.o \
 	display/intel_display_trace.o \
 	display/intel_dkl_phy.o \
diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c
index df6da1c6a3593c..0902b17d02693c 100644
--- a/drivers/gpu/drm/i915/display/i9xx_plane.c
+++ b/drivers/gpu/drm/i915/display/i9xx_plane.c
@@ -8,6 +8,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_fourcc.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_print.h>
 
 #include "i9xx_plane.h"
@@ -887,6 +888,7 @@ static const struct drm_plane_funcs i965_plane_funcs = {
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = i965_plane_format_mod_supported,
 	.format_mod_supported_async = intel_plane_format_mod_supported_async,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static const struct drm_plane_funcs i8xx_plane_funcs = {
@@ -897,6 +899,7 @@ static const struct drm_plane_funcs i8xx_plane_funcs = {
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = i8xx_plane_format_mod_supported,
 	.format_mod_supported_async = intel_plane_format_mod_supported_async,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static void i9xx_disable_tiling(struct intel_plane *plane)
diff --git a/drivers/gpu/drm/i915/display/intel_alpm.c b/drivers/gpu/drm/i915/display/intel_alpm.c
index f1383764b7028f..f54b29e193ee93 100644
--- a/drivers/gpu/drm/i915/display/intel_alpm.c
+++ b/drivers/gpu/drm/i915/display/intel_alpm.c
@@ -375,6 +375,142 @@ static u32 get_pr_alpm_as_sdp_transmission_time(const struct intel_crtc_state *c
 	}
 }
 
+/*
+ * Periodic Adaptive-Sync SDP skip frames.
+ *
+ * While Panel Replay is active the transcoder timing generator runs at the
+ * panel's maximum refresh rate. To drive the panel down to its minimum
+ * refresh rate the Adaptive-Sync SDP only needs to reach the panel once per
+ * minimum-rate frame, so transmitting it on every (maximum-rate) frame is
+ * unnecessary and shows up as repeated SDPs on the link. Program the HW skip
+ * counter so that a single AS SDP is followed by
+ * (max_vrefresh / min_vrefresh - 1) idle frames, i.e. one AS SDP per slowest
+ * panel frame.
+ *
+ * The maximum and minimum refresh rates come from the panel's adaptive-sync
+ * monitor range, so this is independent of the current content/flip rate.
+ */
+static u32 intel_pr_as_sdp_skip_frames(struct intel_dp *intel_dp)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+	const struct drm_display_info *info =
+		&intel_dp->attached_connector->base.display_info;
+	int max_vrefresh = info->monitor_range.max_vfreq;
+	int min_vrefresh = info->monitor_range.min_vfreq;
+
+	/* Off by default; gated by the enable_periodic_assdp debugfs knob. */
+	if (!display->params.enable_periodic_assdp)
+		return 0;
+
+	if (min_vrefresh <= 0 || max_vrefresh <= min_vrefresh)
+		return 0;
+
+	return min_t(u32, max_vrefresh / min_vrefresh - 1,
+		     REG_FIELD_MAX(PR_ALPM_CTL_AS_SDP_SKIP_FRAMES_MASK));
+}
+
+/*
+ * Whether periodic AS SDP transmission (AS SDP skip frames) will be programmed
+ * for this Panel Replay config. The skip counter needs the AS SDP to keep
+ * flowing during PR active, which is incompatible with DC3co, so this is used
+ * to keep DC3co disabled while skip frames is enabled.
+ */
+bool intel_alpm_pr_as_sdp_skip_frames_enabled(struct intel_dp *intel_dp,
+					      const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+
+	/*
+	 * The AS SDP skip frames field only exists on Xe3p_LPD+. Periodic AS SDP
+	 * drives the panel down to its minimum refresh rate on its own, so it is
+	 * only used when VRR is not actively driving the refresh rate.
+	 */
+	if (DISPLAY_VER(display) < 35 || !intel_dp->as_sdp_supported ||
+	    !crtc_state->has_panel_replay || crtc_state->vrr.enable)
+		return false;
+
+	return intel_pr_as_sdp_skip_frames(intel_dp) > 0;
+}
+
+/*
+ * Program the AS SDP portion of PR_ALPM_CTL: the transmission position, the
+ * skip-frame counter and the coupled AS SDP transmission / DC3CO idle-protocol
+ * bits. This is a full recompute of those fields (the base value is built from
+ * scratch, not read back), so it can be called both at PR enable time and when
+ * VRR is toggled seamlessly - which changes whether periodic AS SDP is used.
+ *
+ * Caller must hold intel_dp->alpm.lock.
+ */
+static void intel_alpm_configure_pr_as_sdp(struct intel_dp *intel_dp,
+					   const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 pr_alpm_ctl = get_pr_alpm_as_sdp_transmission_time(crtc_state);
+	u32 skip_frames = 0;
+
+	if (intel_alpm_pr_as_sdp_skip_frames_enabled(intel_dp, crtc_state))
+		skip_frames = intel_pr_as_sdp_skip_frames(intel_dp);
+
+	if (crtc_state->link_off_after_as_sdp_when_pr_active)
+		pr_alpm_ctl |= PR_ALPM_CTL_ALLOW_LINK_OFF_BETWEEN_AS_SDP_AND_SU;
+
+	/*
+	 * Skip frames needs the AS SDP to keep flowing during PR active, so it
+	 * is mutually exclusive with disabling AS SDP transmission in active and
+	 * with the DC3CO idle protocol.
+	 */
+	if (skip_frames) {
+		pr_alpm_ctl |= PR_ALPM_CTL_AS_SDP_SKIP_FRAMES(skip_frames);
+		pr_alpm_ctl &= ~PR_ALPM_CTL_AS_SDP_TRANSMISSION_IN_ACTIVE_DISABLE;
+		pr_alpm_ctl &= ~PR_ALPM_CTL_USE_DC3CO_IDLE_PROTOCOL;
+	} else {
+		pr_alpm_ctl &= ~PR_ALPM_CTL_AS_SDP_SKIP_FRAMES_MASK;
+
+		if (crtc_state->disable_as_sdp_when_pr_active)
+			pr_alpm_ctl |= PR_ALPM_CTL_AS_SDP_TRANSMISSION_IN_ACTIVE_DISABLE;
+
+		if (intel_display_power_dc3co_allowed(display))
+			pr_alpm_ctl |= PR_ALPM_CTL_USE_DC3CO_IDLE_PROTOCOL;
+	}
+
+	intel_de_write(display, PR_ALPM_CTL(display, cpu_transcoder), pr_alpm_ctl);
+}
+
+/*
+ * VRR can be enabled or disabled seamlessly, i.e. without a modeset and without
+ * cycling Panel Replay, so the AS SDP skip-frame programming done at PR enable
+ * time would otherwise go stale across a VRR toggle. Reprogram it here so the
+ * skip counter (and the coupled bits) matches the new VRR state.
+ */
+void intel_alpm_pr_as_sdp_update(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	struct intel_encoder *encoder;
+
+	/* AS SDP skip frames field only exists on Xe3p_LPD+ */
+	if (DISPLAY_VER(display) < 35)
+		return;
+
+	for_each_intel_encoder_mask(display->drm, encoder,
+				    crtc_state->uapi.encoder_mask) {
+		struct intel_dp *intel_dp;
+
+		if (!intel_encoder_is_dp(encoder))
+			continue;
+
+		intel_dp = enc_to_intel_dp(encoder);
+
+		if (!intel_dp->as_sdp_supported ||
+		    !intel_alpm_is_alpm_aux_less(intel_dp, crtc_state))
+			continue;
+
+		mutex_lock(&intel_dp->alpm.lock);
+		intel_alpm_configure_pr_as_sdp(intel_dp, crtc_state);
+		mutex_unlock(&intel_dp->alpm.lock);
+	}
+}
+
 static void lnl_alpm_configure(struct intel_dp *intel_dp,
 			       const struct intel_crtc_state *crtc_state)
 {
@@ -397,22 +533,8 @@ static void lnl_alpm_configure(struct intel_dp *intel_dp,
 			ALPM_CTL_AUX_LESS_SLEEP_HOLD_TIME_50_SYMBOLS |
 			ALPM_CTL_AUX_LESS_WAKE_TIME(crtc_state->alpm_state.aux_less_wake_lines);
 
-		if (intel_dp->as_sdp_supported) {
-			u32 pr_alpm_ctl = get_pr_alpm_as_sdp_transmission_time(crtc_state);
-
-			if (crtc_state->link_off_after_as_sdp_when_pr_active)
-				pr_alpm_ctl |= PR_ALPM_CTL_ALLOW_LINK_OFF_BETWEEN_AS_SDP_AND_SU;
-			if (crtc_state->disable_as_sdp_when_pr_active)
-				pr_alpm_ctl |= PR_ALPM_CTL_AS_SDP_TRANSMISSION_IN_ACTIVE_DISABLE;
-
-			if (intel_display_power_dc3co_allowed(display))
-				pr_alpm_ctl |= PR_ALPM_CTL_USE_DC3CO_IDLE_PROTOCOL;
-			else
-				pr_alpm_ctl &= ~PR_ALPM_CTL_USE_DC3CO_IDLE_PROTOCOL;
-
-			intel_de_write(display, PR_ALPM_CTL(display, cpu_transcoder),
-				       pr_alpm_ctl);
-		}
+		if (intel_dp->as_sdp_supported)
+			intel_alpm_configure_pr_as_sdp(intel_dp, crtc_state);
 
 	} else {
 		alpm_ctl = ALPM_CTL_EXTENDED_FAST_WAKE_ENABLE |
@@ -506,7 +628,7 @@ void intel_alpm_enable_sink(struct intel_dp *intel_dp,
 					     intel_alpm_aux_less_wake_supported(intel_dp)))
 		val |= DP_ALPM_MODE_AUX_LESS;
 
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, val);
+	drm_dp_dpcd_write_byte(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, val);
 }
 
 void intel_alpm_lobf_enable(const struct intel_crtc_state *new_crtc_state)
@@ -629,12 +751,12 @@ bool intel_alpm_get_error(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct drm_dp_aux *aux = &intel_dp->aux;
+	int ret;
 	u8 val;
-	int r;
 
-	r = drm_dp_dpcd_readb(aux, DP_RECEIVER_ALPM_STATUS, &val);
-	if (r != 1) {
-		drm_err(display->drm, "Error reading ALPM status\n");
+	ret = drm_dp_dpcd_read_byte(aux, DP_RECEIVER_ALPM_STATUS, &val);
+	if (ret < 0) {
+		drm_err(display->drm, "Error reading ALPM status (%pe)\n", ERR_PTR(ret));
 		return true;
 	}
 
@@ -642,7 +764,7 @@ bool intel_alpm_get_error(struct intel_dp *intel_dp)
 		drm_dbg_kms(display->drm, "ALPM lock timeout error\n");
 
 		/* Clearing error */
-		drm_dp_dpcd_writeb(aux, DP_RECEIVER_ALPM_STATUS, val);
+		drm_dp_dpcd_write_byte(aux, DP_RECEIVER_ALPM_STATUS, val);
 		return true;
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_alpm.h b/drivers/gpu/drm/i915/display/intel_alpm.h
index 1cf70668ab1ba2..f8f605d94f96d8 100644
--- a/drivers/gpu/drm/i915/display/intel_alpm.h
+++ b/drivers/gpu/drm/i915/display/intel_alpm.h
@@ -34,6 +34,9 @@ bool intel_alpm_aux_wake_supported(struct intel_dp *intel_dp);
 bool intel_alpm_aux_less_wake_supported(struct intel_dp *intel_dp);
 bool intel_alpm_is_alpm_aux_less(struct intel_dp *intel_dp,
 				 const struct intel_crtc_state *crtc_state);
+bool intel_alpm_pr_as_sdp_skip_frames_enabled(struct intel_dp *intel_dp,
+					      const struct intel_crtc_state *crtc_state);
+void intel_alpm_pr_as_sdp_update(const struct intel_crtc_state *crtc_state);
 void intel_alpm_disable(struct intel_dp *intel_dp);
 bool intel_alpm_get_error(struct intel_dp *intel_dp);
 void intel_alpm_lobf_compute_config_late(struct intel_dp *intel_dp,
diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c
index 9d0d47c79dd12d..b66c2d4ba2b3f2 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic.c
@@ -276,6 +276,7 @@ intel_crtc_duplicate_state(struct drm_crtc *crtc)
 	crtc_state->dsb_color = NULL;
 	crtc_state->dsb_commit = NULL;
 	crtc_state->use_dsb = false;
+	crtc_state->use_flipq = false;
 
 	return &crtc_state->uapi;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index eae76e9611055e..32f8c5e55319fb 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -697,6 +697,156 @@ static void ibx_audio_codec_enable(struct intel_encoder *encoder,
 	mutex_unlock(&display->audio.mutex);
 }
 
+#define HDMI_TMDS_AUDIO_PACKETS_LINE_MAX	18
+
+static bool hdmi_audio_rate_supported(const struct intel_crtc_state *crtc_state,
+				      int available_tmds,
+				      int audio_rate, int channels)
+{
+	const struct drm_display_mode *mode = &crtc_state->hw.adjusted_mode;
+	int pixel_clk_max_hz;
+	int audio_pkt_factor;
+	u64 audio_pkt_rate_x4_x1000;
+	int audio_packets_line;
+	int hblank_overhead;
+	int hblank_audio_min;
+
+	/*
+	 * Part 2: Calculate TMDS clock cycles required for Audio Bandwidth
+	 *
+	 * Step 1: pixelclk_max = nominal_pixel_rate * (1 + 0.5%)
+	 * crtc_clock (kHz) * 1000 * 1.005 = crtc_clock * 1005 (Hz)
+	 */
+	pixel_clk_max_hz = mode->crtc_clock * 1005;
+
+	/*
+	 * Steps 3-4: Audio Packet Rate.
+	 *   R_AP = (audio_rate * AP + 2 * acrrate_max) * (1 + 1000 / 1e6)
+	 *        = (audio_rate * AP + 2*1500) * 1.001
+	 *
+	 * AP = 0.25 (2ch) or 1.0 (3-8ch); acrrate_max = 1500 Hz (max ACR
+	 * packet transmission rate per HDMI spec)
+	 *
+	 * Scale by 4*1000 to stay integer:
+	 *  x4: eliminates AP=0.25 -> audio_pkt_factor=1(2ch) or 4(3-8ch),
+	 *      scaled acrrate_max: 2 * 1500 * 4 = 12000
+	 *  x1000: eliminates 1.001 -> *1000*1.001 = *1001
+	 *
+	 * R_AP * 4 * 1000 = (audio_rate * audio_pkt_factor + 12000) * 1001
+	 */
+	audio_pkt_factor = (channels <= 2) ? 1 : 4;
+	audio_pkt_rate_x4_x1000 = (u64)(audio_rate * audio_pkt_factor + 12000) * 1001;
+
+	/*
+	 * Steps 2+5-6: Audio packets per line.
+	 *   AudioPackets_Line = CEIL[R_AP * htotal / f_pixelclk_max]
+	 *
+	 * With audio_pkt_rate_x4_x1000 = R_AP * 4 * 1000:
+	 *   = CEIL[audio_pkt_rate_x4_x1000 * htotal / (4 * 1000 * pixel_clk_max_hz)]
+	 */
+	audio_packets_line = DIV64_U64_ROUND_UP(audio_pkt_rate_x4_x1000 * mode->htotal,
+						(u64)4 * 1000 * pixel_clk_max_hz);
+
+	/*
+	 * Steps 7-9: Hblank overhead.
+	 * Standard:  2*dip_guardband + 2*control_period + video_guardband
+	 *          = 2*2 + 2*12 + 2 = 30
+	 * HDCP 1.x:  rekey_period + dip_guardband + control_period + video_guardband
+	 *          = 58 + 2 + 12 + 2 = 74
+	 *
+	 * Always use HDCP 1.x worst case (74) since HDCP can be toggled
+	 * via fastset without compute_config.
+	 */
+	hblank_overhead = 74;
+
+	/*
+	 * Step 10: Required TMDS cycles for Audio.
+	 *  32 TMDS clock cycles per audio packet.
+	 *   Hblank_audio_min = 32 * AudioPackets_Line + Hblank_overhead
+	 */
+	hblank_audio_min = 32 * audio_packets_line + hblank_overhead;
+
+	/*
+	 * Part 3: audio supported if Hblank_audio_min <= TB_blank and
+	 * audio packets per line <= Maximum allowed packets per line
+	 */
+	return hblank_audio_min <= available_tmds &&
+	       audio_packets_line <= HDMI_TMDS_AUDIO_PACKETS_LINE_MAX;
+}
+
+static void intel_audio_hdmi_eld_compute_config(struct intel_crtc_state *crtc_state)
+{
+	static const int sad_freqs[] = {
+		32000, 44100, 48000, 88200, 96000, 176400, 192000
+	};
+	const struct drm_display_mode *mode = &crtc_state->hw.adjusted_mode;
+	int hblank = mode->htotal - mode->hdisplay;
+	int bpc = crtc_state->pipe_bpp / 3;
+	int ycbcr_420_divider = (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) ? 2 : 1;
+	int available_tmds;
+	u8 *eld = crtc_state->eld;
+	int mnl = drm_eld_mnl(eld);
+	int sad_count = drm_eld_sad_count(eld);
+	int src, dst = 0;
+
+	/*
+	 * Part 1: Calculate available TMDS clock cycles (TB_blank).
+	 *
+	 * TB_blank = CEILING[hblank * K_CD / K_420]
+	 *
+	 * K_CD = 1 for YCbCr4:2:2, bpc / 8 otherwise.
+	 * K_420 = 2 for YCbCr4:2:0, 1 otherwise.
+	 * Rearranged: CEILING[hblank * bpc / (8 * K_420)]
+	 *
+	 * TODO: As and when support for YCbCr4:2:2 is added, set bpc = 8
+	 * to achieve K_CD = 1
+	 */
+	available_tmds = DIV_ROUND_UP(hblank * bpc, 8 * ycbcr_420_divider);
+
+	/*
+	 * Walk all SADs once, keeping the ones with at least one supported
+	 * rate and compacting them down to a contiguous [0, dst) range.
+	 */
+	for (src = 0; src < sad_count; src++) {
+		struct cea_sad sad;
+		u8 new_freq_mask = 0;
+		int channels;
+		int bit;
+
+		drm_eld_sad_get(eld, src, &sad);
+		channels = sad.channels + 1;
+
+		for (bit = 0; bit < 7; bit++) {
+			if (!(sad.freq & BIT(bit)))
+				continue;
+			if (hdmi_audio_rate_supported(crtc_state, available_tmds,
+						      sad_freqs[bit], channels))
+				new_freq_mask |= BIT(bit);
+		}
+
+		if (!new_freq_mask)
+			continue;
+
+		sad.freq = new_freq_mask;
+		drm_eld_sad_set(eld, dst, &sad);
+		dst++;
+	}
+
+	/* Clear the now-unused tail */
+	if (dst != sad_count)
+		memset(&eld[DRM_ELD_CEA_SAD(mnl, dst)], 0, (sad_count - dst) * 3);
+
+	sad_count = dst;
+
+	/* Update SAD count in ELD header */
+	eld[DRM_ELD_SAD_COUNT_CONN_TYPE] &= ~DRM_ELD_SAD_COUNT_MASK;
+	eld[DRM_ELD_SAD_COUNT_CONN_TYPE] |= sad_count << DRM_ELD_SAD_COUNT_SHIFT;
+
+	/* Recalculate baseline ELD length (in dwords) */
+	eld[DRM_ELD_BASELINE_ELD_LEN] =
+		DIV_ROUND_UP(drm_eld_calc_baseline_block_size(eld), 4);
+}
+
 static
 bool intel_audio_needs_cpu_transcoder_id(const struct intel_crtc_state *crtc_state)
 {
@@ -725,6 +875,9 @@ bool intel_audio_compute_config(struct intel_encoder *encoder,
 	BUILD_BUG_ON(sizeof(crtc_state->eld) != sizeof(connector->eld));
 	memcpy(crtc_state->eld, connector->eld, sizeof(crtc_state->eld));
 
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
+		intel_audio_hdmi_eld_compute_config(crtc_state);
+
 	crtc_state->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
 	mutex_unlock(&connector->eld_mutex);
 
diff --git a/drivers/gpu/drm/i915/display/intel_audio_regs.h b/drivers/gpu/drm/i915/display/intel_audio_regs.h
index 4c31844d21dff7..f3d2a99c03d09d 100644
--- a/drivers/gpu/drm/i915/display/intel_audio_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_audio_regs.h
@@ -109,12 +109,20 @@
 #define _HSW_AUD_EDID_DATA_B		0x65150
 #define HSW_AUD_EDID_DATA(trans)	_MMIO_TRANS(trans, _HSW_AUD_EDID_DATA_A, _HSW_AUD_EDID_DATA_B)
 
+/*
+ * HSW_AUD_PIN_ELD_CP_VLD has a 4-bit field per transcoder for
+ * TRANSCODER_A..TRANSCODER_D only (bits 0..15).  Any other transcoder
+ * value (TRANSCODER_EDP, TRANSCODER_DSI_*, INVALID_TRANSCODER) is not
+ * valid here.  Mask the index to the low 2 bits so the shift is
+ * always well-defined.
+ */
 #define HSW_AUD_PIPE_CONV_CFG		_MMIO(0x6507c)
 #define HSW_AUD_PIN_ELD_CP_VLD		_MMIO(0x650c0)
-#define   AUDIO_INACTIVE(trans)		((1 << 3) << ((trans) * 4))
-#define   AUDIO_OUTPUT_ENABLE(trans)	((1 << 2) << ((trans) * 4))
-#define   AUDIO_CP_READY(trans)		((1 << 1) << ((trans) * 4))
-#define   AUDIO_ELD_VALID(trans)	((1 << 0) << ((trans) * 4))
+#define   __AUDIO_TRANS_SHIFT(trans)	(((trans) & 0x3) * 4)
+#define   AUDIO_INACTIVE(trans)		REG_BIT(__AUDIO_TRANS_SHIFT(trans) + 3)
+#define   AUDIO_OUTPUT_ENABLE(trans)	REG_BIT(__AUDIO_TRANS_SHIFT(trans) + 2)
+#define   AUDIO_CP_READY(trans)		REG_BIT(__AUDIO_TRANS_SHIFT(trans) + 1)
+#define   AUDIO_ELD_VALID(trans)	REG_BIT(__AUDIO_TRANS_SHIFT(trans))
 
 #define _AUD_TCA_DP_2DOT0_CTRL		0x650bc
 #define _AUD_TCB_DP_2DOT0_CTRL		0x651bc
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index 97cbae2e547e2d..7b6a6f8c3cf196 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -34,6 +34,7 @@
 #include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
 
+#include "intel_ddi_buf_trans.h"
 #include "intel_display.h"
 #include "intel_display_core.h"
 #include "intel_display_rpm.h"
@@ -72,6 +73,8 @@
 struct intel_bios_encoder_data {
 	struct intel_display *display;
 
+	struct intel_ddi_buf_trans *vspeo;
+	union intel_ddi_buf_trans_entry *entries;
 	struct child_device_config child;
 	struct dsc_compression_parameters_entry *dsc;
 	struct list_head node;
@@ -200,6 +203,8 @@ static const struct {
 	  .min_size = sizeof(struct bdb_mipi_sequence) },
 	{ .section_id = BDB_COMPRESSION_PARAMETERS,
 	  .min_size = sizeof(struct bdb_compression_parameters), },
+	{ .section_id = BDB_VSWING_PREEMPH,
+	  .min_size = sizeof(struct bdb_vswing_preemph), },
 	{ .section_id = BDB_GENERIC_DTD,
 	  .min_size = sizeof(struct bdb_generic_dtd), },
 };
@@ -2203,6 +2208,31 @@ parse_compression_parameters(struct intel_display *display)
 	}
 }
 
+static int vswing_preemph_num_rows(struct intel_display *display)
+{
+	return DISPLAY_VER(display) >= 14 ? 16 : 10;
+}
+
+static void
+parse_vswing_preemph_override(struct intel_display *display)
+{
+	const struct bdb_vswing_preemph *block;
+
+	if (display->vbt.version < 218)
+		return;
+
+	block = bdb_find_section(display, BDB_VSWING_PREEMPH);
+
+	/* pre-ICL GOPs don't have VBT #57 */
+	if (!block)
+		return;
+
+	display->vbt.vspeo.tables = block->tables;
+	display->vbt.vspeo.num_tables = block->num_tables;
+	display->vbt.vspeo.num_columns = block->num_columns;
+	display->vbt.vspeo.num_rows = vswing_preemph_num_rows(display);
+}
+
 static u8 translate_iboost(struct intel_display *display, u8 val)
 {
 	static const u8 mapping[] = { 1, 3, 7 }; /* See VBT spec */
@@ -2621,6 +2651,30 @@ static void sanitize_device_type(struct intel_bios_encoder_data *devdata,
 	devdata->child.device_type |= DEVICE_TYPE_NOT_HDMI_OUTPUT;
 }
 
+static void allocate_vswing_preemph_override(struct intel_bios_encoder_data *devdata)
+{
+	int num_rows = devdata->display->vbt.vspeo.num_rows;
+	union intel_ddi_buf_trans_entry *entries;
+	struct intel_ddi_buf_trans *vspeo;
+
+	if (!intel_bios_encoder_requests_vspeo(devdata))
+		return;
+
+	vspeo = kzalloc_obj(*vspeo);
+	if (!vspeo)
+		return;
+
+	entries = kzalloc_objs(*entries, num_rows);
+	if (!entries) {
+		kfree(vspeo);
+		return;
+	}
+
+	devdata->entries = entries;
+	devdata->vspeo = vspeo;
+	devdata->vspeo->num_entries = num_rows;
+}
+
 static void sanitize_hdmi_level_shift(struct intel_bios_encoder_data *devdata,
 				      enum port port)
 {
@@ -2737,7 +2791,6 @@ static bool is_port_valid(struct intel_display *display, enum port port)
 static void print_ddi_port(const struct intel_bios_encoder_data *devdata)
 {
 	struct intel_display *display = devdata->display;
-	const struct child_device_config *child = &devdata->child;
 	bool is_dvi, is_hdmi, is_dp, is_edp, is_dsi, is_crt, supports_typec_usb, supports_tbt;
 	int dp_boost_level, dp_max_link_rate, hdmi_boost_level, hdmi_level_shift, max_tmds_clock;
 	enum port port;
@@ -2774,6 +2827,11 @@ static void print_ddi_port(const struct intel_bios_encoder_data *devdata)
 			    "Port %c supports dynamic DDI allocation in TCSS\n",
 			    port_name(port));
 
+	if (intel_bios_encoder_requests_vspeo(devdata))
+		drm_dbg_kms(display->drm,
+			    "Port %c requests vswing/pre-emphasis override\n",
+			    port_name(port));
+
 	hdmi_level_shift = intel_bios_hdmi_level_shift(devdata);
 	if (hdmi_level_shift >= 0) {
 		drm_dbg_kms(display->drm,
@@ -2805,14 +2863,6 @@ static void print_ddi_port(const struct intel_bios_encoder_data *devdata)
 		drm_dbg_kms(display->drm,
 			    "Port %c VBT DP max link rate: %d\n",
 			    port_name(port), dp_max_link_rate);
-
-	/*
-	 * FIXME need to implement support for VBT
-	 * vswing/preemph tables should this ever trigger.
-	 */
-	drm_WARN(display->drm, child->use_vbt_vswing,
-		 "Port %c asks to use VBT vswing/preemph tables\n",
-		 port_name(port));
 }
 
 static void parse_ddi_port(struct intel_bios_encoder_data *devdata)
@@ -2834,6 +2884,7 @@ static void parse_ddi_port(struct intel_bios_encoder_data *devdata)
 	sanitize_dedicated_external(devdata, port);
 	sanitize_device_type(devdata, port);
 	sanitize_hdmi_level_shift(devdata, port);
+	allocate_vswing_preemph_override(devdata);
 }
 
 static bool has_ddi_port_info(struct intel_display *display)
@@ -3291,6 +3342,7 @@ void intel_bios_init(struct intel_display *display)
 	parse_general_features(display);
 	parse_general_definitions(display);
 	parse_driver_features(display);
+	parse_vswing_preemph_override(display);
 
 	/* Depends on child device list */
 	parse_compression_parameters(display);
@@ -3370,6 +3422,9 @@ void intel_bios_driver_remove(struct intel_display *display)
 	list_for_each_entry_safe(devdata, nd, &display->vbt.display_devices,
 				 node) {
 		list_del(&devdata->node);
+
+		kfree(devdata->entries);
+		kfree(devdata->vspeo);
 		kfree(devdata->dsc);
 		kfree(devdata);
 	}
@@ -3801,6 +3856,11 @@ int intel_bios_hdmi_ddc_pin(const struct intel_bios_encoder_data *devdata)
 	return map_ddc_pin(devdata->display, devdata->child.ddc_pin);
 }
 
+bool intel_bios_encoder_requests_vspeo(const struct intel_bios_encoder_data *devdata)
+{
+	return devdata->display->vbt.version >= 218 && devdata->child.use_vbt_vswing;
+}
+
 bool intel_bios_encoder_supports_typec_usb(const struct intel_bios_encoder_data *devdata)
 {
 	return devdata->display->vbt.version >= 195 && devdata->child.dp_usb_type_c;
@@ -3811,6 +3871,190 @@ bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devda
 	return devdata->display->vbt.version >= 209 && devdata->child.tbt;
 }
 
+static bool
+validate_vspeo(const struct intel_bios_encoder_data *devdata, bool has_dp)
+{
+	struct intel_ddi_buf_trans *vspeo;
+
+	vspeo = devdata->vspeo;
+	if (!vspeo)
+		return false;
+
+	if (!has_dp)
+		return false;
+
+	return true;
+}
+
+const struct intel_ddi_buf_trans *
+intel_bios_get_c20_vspeo(const struct intel_bios_encoder_data *devdata,
+			 bool has_dp, bool is_uhbr)
+{
+	struct intel_display *display = devdata->display;
+	union intel_ddi_buf_trans_entry *entries = devdata->entries;
+	struct intel_ddi_buf_trans *vspeo = devdata->vspeo;
+	const u32 *tables = display->vbt.vspeo.tables;
+	int num_columns = display->vbt.vspeo.num_columns;
+	int num_rows = display->vbt.vspeo.num_rows;
+	int idx = is_uhbr ? 5 : 4;
+	size_t offset = 0;
+	int level;
+
+	if (!validate_vspeo(devdata, has_dp))
+		return NULL;
+
+	offset += idx * num_rows * num_columns;
+
+	for (level = 0; level < num_rows; level++) {
+		u32 vswing = tables[offset];
+		u32 pre_cursor = tables[offset + 1];
+		u32 post_cursor = tables[offset + 2];
+
+		entries[level].snps.vswing = vswing;
+		entries[level].snps.pre_cursor = pre_cursor;
+		entries[level].snps.post_cursor = post_cursor;
+
+		offset += num_columns;
+	}
+
+	vspeo->entries = entries;
+
+	return vspeo;
+}
+
+const struct intel_ddi_buf_trans *
+intel_bios_get_c10_vspeo(const struct intel_bios_encoder_data *devdata,
+			 bool has_dp, int port_clock, bool has_edp)
+{
+	struct intel_display *display = devdata->display;
+	union intel_ddi_buf_trans_entry *entries = devdata->entries;
+	struct intel_ddi_buf_trans *vspeo = devdata->vspeo;
+	const u32 *tables = display->vbt.vspeo.tables;
+	int num_columns = display->vbt.vspeo.num_columns;
+	int num_rows = display->vbt.vspeo.num_rows;
+	size_t offset = 0;
+	int level, idx;
+
+	if (!validate_vspeo(devdata, has_dp))
+		return NULL;
+
+	if (has_edp)
+		idx = port_clock > 540000 ? 3 : 2;
+	else
+		idx = port_clock > 270000 ? 1 : 0;
+
+	offset += idx * num_rows * num_columns;
+
+	for (level = 0; level < num_rows; level++) {
+		u32 vswing = tables[offset];
+		u32 pre_cursor = tables[offset + 1];
+		u32 post_cursor = tables[offset + 2];
+
+		entries[level].snps.vswing = vswing;
+		entries[level].snps.pre_cursor = pre_cursor;
+		entries[level].snps.post_cursor = post_cursor;
+
+		offset += num_columns;
+	}
+
+	vspeo->entries = entries;
+
+	return vspeo;
+}
+
+const struct intel_ddi_buf_trans *
+intel_bios_get_ehl_combo_vspeo(const struct intel_bios_encoder_data *devdata,
+			       bool has_dp, int port_clock, bool low_vswing_edp)
+{
+	struct intel_display *display = devdata->display;
+	union intel_ddi_buf_trans_entry *entries = devdata->entries;
+	struct intel_ddi_buf_trans *vspeo = devdata->vspeo;
+	const u32 *tables = display->vbt.vspeo.tables;
+	int num_columns = display->vbt.vspeo.num_columns;
+	int num_rows = display->vbt.vspeo.num_rows;
+	size_t offset = 0;
+	int level, idx;
+
+	if (!validate_vspeo(devdata, has_dp))
+		return NULL;
+
+	if (low_vswing_edp)
+		idx = port_clock > 540000 ? 2 : 1;
+	else
+		idx = port_clock > 270000 ? 1 : 0;
+
+	offset += idx * num_rows * num_columns;
+
+	for (level = 0; level < num_rows; level++) {
+		u32 dw2_swing_sel = tables[offset];
+		u32 dw7_n_scalar = tables[offset + 1];
+		u32 dw4_cursor_coeff = tables[offset + 2];
+		u32 dw4_post_cursor_2 = tables[offset + 3];
+		u32 dw4_post_cursor_1 = tables[offset + 4];
+
+		entries[level].icl.dw2_swing_sel = dw2_swing_sel;
+		entries[level].icl.dw7_n_scalar = dw7_n_scalar;
+		entries[level].icl.dw4_cursor_coeff = dw4_cursor_coeff;
+		entries[level].icl.dw4_post_cursor_2 = dw4_post_cursor_2;
+		entries[level].icl.dw4_post_cursor_1 = dw4_post_cursor_1;
+
+		offset += num_columns;
+	}
+
+	vspeo->entries = entries;
+
+	return vspeo;
+}
+
+const struct intel_ddi_buf_trans *
+intel_bios_get_jsl_combo_vspeo(const struct intel_bios_encoder_data *devdata,
+			       bool has_dp, int port_clock, bool low_vswing_edp)
+{
+	struct intel_display *display = devdata->display;
+	union intel_ddi_buf_trans_entry *entries = devdata->entries;
+	struct intel_ddi_buf_trans *vspeo = devdata->vspeo;
+	const u32 *tables = display->vbt.vspeo.tables;
+	int num_columns = display->vbt.vspeo.num_columns;
+	int num_rows = display->vbt.vspeo.num_rows;
+	size_t offset = 0;
+	int idx = 0;
+	int level;
+
+	if (!validate_vspeo(devdata, has_dp))
+		return NULL;
+
+	if (low_vswing_edp) {
+		if (port_clock > 540000)
+			idx = 0;
+		else if (port_clock > 270000)
+			idx = 2;
+		else
+			idx = 1;
+	}
+
+	offset += idx * num_rows * num_columns;
+
+	for (level = 0; level < num_rows; level++) {
+		u32 dw2_swing_sel = tables[offset];
+		u32 dw7_n_scalar = tables[offset + 1];
+		u32 dw4_cursor_coeff = tables[offset + 2];
+		u32 dw4_post_cursor_2 = tables[offset + 3];
+		u32 dw4_post_cursor_1 = tables[offset + 4];
+
+		entries[level].icl.dw2_swing_sel = dw2_swing_sel;
+		entries[level].icl.dw7_n_scalar = dw7_n_scalar;
+		entries[level].icl.dw4_cursor_coeff = dw4_cursor_coeff;
+		entries[level].icl.dw4_post_cursor_2 = dw4_post_cursor_2;
+		entries[level].icl.dw4_post_cursor_1 = dw4_post_cursor_1;
+
+		offset += num_columns;
+	}
+
+	vspeo->entries = entries;
+
+	return vspeo;
+}
+
 bool intel_bios_encoder_is_dedicated_external(const struct intel_bios_encoder_data *devdata)
 {
 	return devdata->display->vbt.version >= 264 &&
diff --git a/drivers/gpu/drm/i915/display/intel_bios.h b/drivers/gpu/drm/i915/display/intel_bios.h
index 75dff27b42289d..196b8b3f434f54 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.h
+++ b/drivers/gpu/drm/i915/display/intel_bios.h
@@ -73,6 +73,20 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder,
 const struct intel_bios_encoder_data *
 intel_bios_encoder_data_lookup(struct intel_display *display, enum port port);
 
+const struct intel_ddi_buf_trans *
+intel_bios_get_c20_vspeo(const struct intel_bios_encoder_data *devdata,
+			 bool has_dp, bool is_uhbr);
+const struct intel_ddi_buf_trans *
+intel_bios_get_c10_vspeo(const struct intel_bios_encoder_data *devdata,
+			 bool has_dp, int port_clock, bool has_edp);
+const struct intel_ddi_buf_trans *
+intel_bios_get_ehl_combo_vspeo(const struct intel_bios_encoder_data *devdata,
+			       bool has_dp, int port_clock, bool low_vswing_edp);
+const struct intel_ddi_buf_trans *
+intel_bios_get_jsl_combo_vspeo(const struct intel_bios_encoder_data *devdata,
+			       bool has_dp, int port_clock, bool low_vswing_edp);
+
+bool intel_bios_encoder_requests_vspeo(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_dvi(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_hdmi(const struct intel_bios_encoder_data *devdata);
 bool intel_bios_encoder_supports_dp(const struct intel_bios_encoder_data *devdata);
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index a1a5720996b708..92e6fca861f64a 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -163,6 +163,9 @@ struct intel_cdclk_funcs {
 	void (*set_cdclk)(struct intel_display *display,
 			  const struct intel_cdclk_config *cdclk_config,
 			  enum pipe pipe);
+	int (*pre_notify)(struct intel_display *display);
+	int (*post_notify)(struct intel_display *display,
+			   const struct intel_cdclk_config *cdclk_config);
 	int (*modeset_calc_cdclk)(struct intel_atomic_state *state);
 	u8 (*calc_voltage_level)(int cdclk);
 };
@@ -173,11 +176,45 @@ void intel_cdclk_get_cdclk(struct intel_display *display,
 	display->cdclk.funcs->get_cdclk(display, cdclk_config);
 }
 
+static int intel_cdclk_pre_notify(struct intel_display *display)
+{
+	int ret;
+
+	if (!display->cdclk.funcs->pre_notify)
+		return 0;
+
+	ret = display->cdclk.funcs->pre_notify(display);
+	if (ret)
+		drm_err(display->drm,
+			"Failed to inform system about start of CDCLK change (%d)\n", ret);
+
+	return ret;
+}
+
+static void intel_cdclk_post_notify(struct intel_display *display,
+				    const struct intel_cdclk_config *cdclk_config)
+{
+	int ret;
+
+	if (!display->cdclk.funcs->post_notify)
+		return;
+
+	ret = display->cdclk.funcs->post_notify(display, cdclk_config);
+	if (ret)
+		drm_err(display->drm,
+			"Failed to inform system about end of CDCLK change (%d)\n", ret);
+}
+
 static void intel_cdclk_set_cdclk(struct intel_display *display,
 				  const struct intel_cdclk_config *cdclk_config,
 				  enum pipe pipe)
 {
+	if (intel_cdclk_pre_notify(display))
+		return;
+
 	display->cdclk.funcs->set_cdclk(display, cdclk_config, pipe);
+
+	intel_cdclk_post_notify(display, cdclk_config);
 }
 
 static int intel_cdclk_modeset_calc_cdclk(struct intel_atomic_state *state)
@@ -872,6 +909,19 @@ static u32 bdw_cdclk_freq_sel(int cdclk)
 	}
 }
 
+static int bdw_cdclk_pcode_pre_notify(struct intel_display *display)
+{
+	return intel_parent_pcode_write(display, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ,
+					0x0);
+}
+
+static int bdw_cdclk_pcode_post_notify(struct intel_display *display,
+				       const struct intel_cdclk_config *cdclk_config)
+{
+	return intel_parent_pcode_write(display, HSW_PCODE_DE_WRITE_FREQ_REQ,
+					cdclk_config->voltage_level);
+}
+
 static void bdw_set_cdclk(struct intel_display *display,
 			  const struct intel_cdclk_config *cdclk_config,
 			  enum pipe pipe)
@@ -888,13 +938,6 @@ static void bdw_set_cdclk(struct intel_display *display,
 		     "trying to change cdclk frequency with cdclk not enabled\n"))
 		return;
 
-	ret = intel_parent_pcode_write(display, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
-	if (ret) {
-		drm_err(display->drm,
-			"failed to inform pcode about cdclk change\n");
-		return;
-	}
-
 	intel_de_rmw(display, LCPLL_CTL,
 		     0, LCPLL_CD_SOURCE_FCLK);
 
@@ -918,9 +961,6 @@ static void bdw_set_cdclk(struct intel_display *display,
 	if (ret)
 		drm_err(display->drm, "Switching back to LCPLL failed\n");
 
-	intel_parent_pcode_write(display, HSW_PCODE_DE_WRITE_FREQ_REQ,
-				 cdclk_config->voltage_level);
-
 	intel_de_write(display, CDCLK_FREQ,
 		       DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 
@@ -1155,6 +1195,21 @@ static u32 skl_cdclk_freq_sel(struct intel_display *display,
 	}
 }
 
+static int skl_cdclk_pcode_pre_notify(struct intel_display *display)
+{
+	return intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
+					  SKL_CDCLK_PREPARE_FOR_CHANGE,
+					  SKL_CDCLK_READY_FOR_CHANGE,
+					  SKL_CDCLK_READY_FOR_CHANGE, 3);
+}
+
+static int skl_cdclk_pcode_post_notify(struct intel_display *display,
+				       const struct intel_cdclk_config *cdclk_config)
+{
+	return intel_parent_pcode_write(display, SKL_PCODE_CDCLK_CONTROL,
+					cdclk_config->voltage_level);
+}
+
 static void skl_set_cdclk(struct intel_display *display,
 			  const struct intel_cdclk_config *cdclk_config,
 			  enum pipe pipe)
@@ -1162,7 +1217,6 @@ static void skl_set_cdclk(struct intel_display *display,
 	int cdclk = cdclk_config->cdclk;
 	int vco = cdclk_config->vco;
 	u32 freq_select, cdclk_ctl;
-	int ret;
 
 	/*
 	 * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
@@ -1175,16 +1229,6 @@ static void skl_set_cdclk(struct intel_display *display,
 	drm_WARN_ON_ONCE(display->drm,
 			 display->platform.skylake && vco == 8640000);
 
-	ret = intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
-					 SKL_CDCLK_PREPARE_FOR_CHANGE,
-					 SKL_CDCLK_READY_FOR_CHANGE,
-					 SKL_CDCLK_READY_FOR_CHANGE, 3);
-	if (ret) {
-		drm_err(display->drm,
-			"Failed to inform PCU about cdclk change (%d)\n", ret);
-		return;
-	}
-
 	freq_select = skl_cdclk_freq_sel(display, cdclk, vco);
 
 	if (display->cdclk.hw.vco != 0 &&
@@ -1220,10 +1264,6 @@ static void skl_set_cdclk(struct intel_display *display,
 	intel_de_write(display, CDCLK_CTL, cdclk_ctl);
 	intel_de_posting_read(display, CDCLK_CTL);
 
-	/* inform PCU of the change */
-	intel_parent_pcode_write(display, SKL_PCODE_CDCLK_CONTROL,
-				 cdclk_config->voltage_level);
-
 	intel_update_cdclk(display);
 }
 
@@ -1314,7 +1354,7 @@ static void skl_cdclk_init_hw(struct intel_display *display)
 	cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
 	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
 
-	skl_set_cdclk(display, &cdclk_config, INVALID_PIPE);
+	intel_cdclk_set_cdclk(display, &cdclk_config, INVALID_PIPE);
 }
 
 static void skl_cdclk_uninit_hw(struct intel_display *display)
@@ -1325,7 +1365,7 @@ static void skl_cdclk_uninit_hw(struct intel_display *display)
 	cdclk_config.vco = 0;
 	cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
 
-	skl_set_cdclk(display, &cdclk_config, INVALID_PIPE);
+	intel_cdclk_set_cdclk(display, &cdclk_config, INVALID_PIPE);
 }
 
 struct intel_cdclk_vals {
@@ -2177,6 +2217,11 @@ static bool pll_enable_wa_needed(struct intel_display *display)
 		display->cdclk.hw.vco > 0;
 }
 
+static bool has_cd2x_pipe_select(struct intel_display *display)
+{
+	return IS_DISPLAY_VER(display, 10, 20) || display->platform.broxton;
+}
+
 static u32 bxt_cdclk_ctl(struct intel_display *display,
 			 const struct intel_cdclk_config *cdclk_config,
 			 enum pipe pipe)
@@ -2190,7 +2235,7 @@ static u32 bxt_cdclk_ctl(struct intel_display *display,
 
 	val = bxt_cdclk_cd2x_div_sel(display, cdclk, vco, waveform);
 
-	if (DISPLAY_VER(display) < 30)
+	if (has_cd2x_pipe_select(display))
 		val |= bxt_cdclk_cd2x_pipe(display, pipe);
 
 	/*
@@ -2220,6 +2265,29 @@ static u32 bxt_cdclk_ctl(struct intel_display *display,
 	return val;
 }
 
+static int bxt_cdclk_pcode_pre_notify(struct intel_display *display)
+{
+	/*
+	 * BSpec requires us to wait up to 150usec, but that leads to
+	 * timeouts; the 2ms used here is based on experiment.
+	 */
+	return intel_parent_pcode_write_timeout(display, HSW_PCODE_DE_WRITE_FREQ_REQ,
+						0x80000000, 2);
+}
+
+static int bxt_cdclk_pcode_post_notify(struct intel_display *display,
+				       const struct intel_cdclk_config *cdclk_config)
+{
+	/*
+	 * The timeout isn't specified, the 2ms used here is based on
+	 * experiment.
+	 * FIXME: Waiting for the request completion could be delayed
+	 * until the next PCODE request based on BSpec.
+	 */
+	return intel_parent_pcode_write_timeout(display, HSW_PCODE_DE_WRITE_FREQ_REQ,
+						cdclk_config->voltage_level, 2);
+}
+
 static void _bxt_set_cdclk(struct intel_display *display,
 			   const struct intel_cdclk_config *cdclk_config,
 			   enum pipe pipe)
@@ -2259,36 +2327,6 @@ static void bxt_set_cdclk(struct intel_display *display,
 {
 	struct intel_cdclk_config mid_cdclk_config;
 	int cdclk = cdclk_config->cdclk;
-	int ret = 0;
-
-	/*
-	 * Inform power controller of upcoming frequency change.
-	 * Display versions 14 and beyond do not follow the PUnit
-	 * mailbox communication, skip
-	 * this step.
-	 */
-	if (DISPLAY_VER(display) >= 14 || display->platform.dg2)
-		; /* NOOP */
-	else if (DISPLAY_VER(display) >= 11)
-		ret = intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
-						 SKL_CDCLK_PREPARE_FOR_CHANGE,
-						 SKL_CDCLK_READY_FOR_CHANGE,
-						 SKL_CDCLK_READY_FOR_CHANGE, 3);
-	else
-		/*
-		 * BSpec requires us to wait up to 150usec, but that leads to
-		 * timeouts; the 2ms used here is based on experiment.
-		 */
-		ret = intel_parent_pcode_write_timeout(display,
-						       HSW_PCODE_DE_WRITE_FREQ_REQ,
-						       0x80000000, 2);
-
-	if (ret) {
-		drm_err(display->drm,
-			"Failed to inform PCU about cdclk change (err %d, freq %d)\n",
-			ret, cdclk);
-		return;
-	}
 
 	if (DISPLAY_VER(display) >= 20 && cdclk < display->cdclk.hw.cdclk)
 		xe2lpd_mdclk_cdclk_ratio_program(display, cdclk_config);
@@ -2304,40 +2342,13 @@ static void bxt_set_cdclk(struct intel_display *display,
 	if (DISPLAY_VER(display) >= 20 && cdclk > display->cdclk.hw.cdclk)
 		xe2lpd_mdclk_cdclk_ratio_program(display, cdclk_config);
 
-	if (DISPLAY_VER(display) >= 14)
-		/*
-		 * NOOP - No Pcode communication needed for
-		 * Display versions 14 and beyond
-		 */;
-	else if (DISPLAY_VER(display) >= 11 && !display->platform.dg2)
-		ret = intel_parent_pcode_write(display, SKL_PCODE_CDCLK_CONTROL,
-					       cdclk_config->voltage_level);
-	if (DISPLAY_VER(display) < 11) {
-		/*
-		 * The timeout isn't specified, the 2ms used here is based on
-		 * experiment.
-		 * FIXME: Waiting for the request completion could be delayed
-		 * until the next PCODE request based on BSpec.
-		 */
-		ret = intel_parent_pcode_write_timeout(display,
-						       HSW_PCODE_DE_WRITE_FREQ_REQ,
-						       cdclk_config->voltage_level, 2);
-	}
-	if (ret) {
-		drm_err(display->drm,
-			"PCode CDCLK freq set failed, (err %d, freq %d)\n",
-			ret, cdclk);
-		return;
-	}
-
 	intel_update_cdclk(display);
 
-	if (DISPLAY_VER(display) >= 11)
-		/*
-		 * Can't read out the voltage level :(
-		 * Let's just assume everything is as expected.
-		 */
-		display->cdclk.hw.voltage_level = cdclk_config->voltage_level;
+	/*
+	 * Can't read out the voltage level :(
+	 * Let's just assume everything is as expected.
+	 */
+	display->cdclk.hw.voltage_level = cdclk_config->voltage_level;
 }
 
 static void bxt_sanitize_cdclk(struct intel_display *display)
@@ -2381,7 +2392,7 @@ static void bxt_sanitize_cdclk(struct intel_display *display)
 	 * dividers both syncing to an active pipe, or asynchronously
 	 * (PIPE_NONE).
 	 */
-	if (DISPLAY_VER(display) < 30) {
+	if (has_cd2x_pipe_select(display)) {
 		cdctl &= ~bxt_cdclk_cd2x_pipe_mask(display);
 		cdctl |= bxt_cdclk_cd2x_pipe(display, INVALID_PIPE);
 	}
@@ -2437,7 +2448,7 @@ static void bxt_cdclk_init_hw(struct intel_display *display)
 	cdclk_config.voltage_level =
 		intel_cdclk_calc_voltage_level(display, cdclk_config.cdclk);
 
-	bxt_set_cdclk(display, &cdclk_config, INVALID_PIPE);
+	intel_cdclk_set_cdclk(display, &cdclk_config, INVALID_PIPE);
 }
 
 static void bxt_cdclk_uninit_hw(struct intel_display *display)
@@ -2449,7 +2460,7 @@ static void bxt_cdclk_uninit_hw(struct intel_display *display)
 	cdclk_config.voltage_level =
 		intel_cdclk_calc_voltage_level(display, cdclk_config.cdclk);
 
-	bxt_set_cdclk(display, &cdclk_config, INVALID_PIPE);
+	intel_cdclk_set_cdclk(display, &cdclk_config, INVALID_PIPE);
 }
 
 /**
@@ -2579,8 +2590,7 @@ static bool intel_cdclk_can_cd2x_update(struct intel_display *display,
 					const struct intel_cdclk_config *a,
 					const struct intel_cdclk_config *b)
 {
-	/* Older hw doesn't have the capability */
-	if (DISPLAY_VER(display) < 10 && !display->platform.broxton)
+	if (!has_cd2x_pipe_select(display))
 		return false;
 
 	/*
@@ -2623,19 +2633,15 @@ void intel_cdclk_dump_config(struct intel_display *display,
 		    cdclk_config->voltage_level);
 }
 
-static void intel_pcode_notify(struct intel_display *display,
-			       u8 voltage_level,
-			       u8 active_pipe_count,
-			       u16 cdclk,
-			       bool cdclk_update_valid,
-			       bool pipe_count_update_valid)
+static int dg2_cdclk_pcode_notify(struct intel_display *display,
+				  u8 voltage_level,
+				  u8 active_pipe_count,
+				  u16 cdclk,
+				  bool cdclk_update_valid,
+				  bool pipe_count_update_valid)
 {
-	int ret;
 	u32 update_mask = 0;
 
-	if (!display->platform.dg2)
-		return;
-
 	update_mask = DISPLAY_TO_PCODE_UPDATE_MASK(cdclk, active_pipe_count, voltage_level);
 
 	if (cdclk_update_valid)
@@ -2644,15 +2650,10 @@ static void intel_pcode_notify(struct intel_display *display,
 	if (pipe_count_update_valid)
 		update_mask |= DISPLAY_TO_PCODE_PIPE_COUNT_VALID;
 
-	ret = intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
-					 SKL_CDCLK_PREPARE_FOR_CHANGE |
-					 update_mask,
-					 SKL_CDCLK_READY_FOR_CHANGE,
-					 SKL_CDCLK_READY_FOR_CHANGE, 3);
-	if (ret)
-		drm_err(display->drm,
-			"Failed to inform PCU about display config (err %d)\n",
-			ret);
+	return intel_parent_pcode_request(display, SKL_PCODE_CDCLK_CONTROL,
+					  update_mask,
+					  SKL_CDCLK_READY_FOR_CHANGE,
+					  SKL_CDCLK_READY_FOR_CHANGE, 3);
 }
 
 static void intel_set_cdclk(struct intel_display *display,
@@ -2721,15 +2722,17 @@ static int dg2_power_well_count(struct intel_display *display,
 	return display->platform.dg2 ? hweight8(cdclk_state->active_pipes) : 0;
 }
 
-static void intel_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
+static void dg2_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
 {
 	struct intel_display *display = to_intel_display(state);
 	const struct intel_cdclk_state *old_cdclk_state =
 		intel_atomic_get_old_cdclk_state(state);
 	const struct intel_cdclk_state *new_cdclk_state =
 		intel_atomic_get_new_cdclk_state(state);
-	unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
+	u8 voltage_level, num_active_pipes = 0;
 	bool change_cdclk, update_pipe_count;
+	unsigned int cdclk_mhz = 0;
+	int ret;
 
 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
 				 &new_cdclk_state->actual) &&
@@ -2750,8 +2753,12 @@ static void intel_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
 	 * if CDCLK is decreasing or not changing, set bits 25:16 to current CDCLK,
 	 * which basically means we choose the maximum of old and new CDCLK, if we know both
 	 */
-	if (change_cdclk)
-		cdclk = max(new_cdclk_state->actual.cdclk, old_cdclk_state->actual.cdclk);
+	if (change_cdclk) {
+		unsigned int cdclk = max(new_cdclk_state->actual.cdclk,
+					 old_cdclk_state->actual.cdclk);
+
+		cdclk_mhz = DIV_ROUND_UP(cdclk, 1000);
+	}
 
 	/*
 	 * According to "Sequence For Pipe Count Change",
@@ -2762,19 +2769,24 @@ static void intel_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
 	if (update_pipe_count)
 		num_active_pipes = dg2_power_well_count(display, new_cdclk_state);
 
-	intel_pcode_notify(display, voltage_level, num_active_pipes, cdclk,
-			   change_cdclk, update_pipe_count);
+	ret = dg2_cdclk_pcode_notify(display, voltage_level, num_active_pipes, cdclk_mhz,
+				     change_cdclk, update_pipe_count);
+	if (ret)
+		drm_err(display->drm,
+			"Failed to inform PCODE about start of CDCLK change (%d)\n", ret);
 }
 
-static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
+static void dg2_cdclk_pcode_post_notify(struct intel_atomic_state *state)
 {
 	struct intel_display *display = to_intel_display(state);
 	const struct intel_cdclk_state *new_cdclk_state =
 		intel_atomic_get_new_cdclk_state(state);
 	const struct intel_cdclk_state *old_cdclk_state =
 		intel_atomic_get_old_cdclk_state(state);
-	unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
+	u8 voltage_level, num_active_pipes = 0;
 	bool update_cdclk, update_pipe_count;
+	unsigned int cdclk_mhz = 0;
+	int ret;
 
 	/* According to "Sequence After Frequency Change", set voltage to used level */
 	voltage_level = new_cdclk_state->actual.voltage_level;
@@ -2787,8 +2799,11 @@ static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
 	 * According to "Sequence After Frequency Change",
 	 * set bits 25:16 to current CDCLK
 	 */
-	if (update_cdclk)
-		cdclk = new_cdclk_state->actual.cdclk;
+	if (update_cdclk) {
+		unsigned int cdclk = new_cdclk_state->actual.cdclk;
+
+		cdclk_mhz = DIV_ROUND_UP(cdclk, 1000);
+	}
 
 	/*
 	 * According to "Sequence For Pipe Count Change",
@@ -2799,8 +2814,11 @@ static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
 	if (update_pipe_count)
 		num_active_pipes = dg2_power_well_count(display, new_cdclk_state);
 
-	intel_pcode_notify(display, voltage_level, num_active_pipes, cdclk,
-			   update_cdclk, update_pipe_count);
+	ret = dg2_cdclk_pcode_notify(display, voltage_level, num_active_pipes, cdclk_mhz,
+				     update_cdclk, update_pipe_count);
+	if (ret)
+		drm_err(display->drm,
+			"Failed to inform PCODE about end of CDCLK change (%d)\n", ret);
 }
 
 bool intel_cdclk_is_decreasing_later(struct intel_atomic_state *state)
@@ -2836,12 +2854,11 @@ intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
 		return;
 
 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
-				 &new_cdclk_state->actual))
+				 &new_cdclk_state->actual) &&
+	    dg2_power_well_count(display, old_cdclk_state) ==
+	    dg2_power_well_count(display, new_cdclk_state))
 		return;
 
-	if (display->platform.dg2)
-		intel_cdclk_pcode_pre_notify(state);
-
 	if (new_cdclk_state->disable_pipes) {
 		cdclk_config = new_cdclk_state->actual;
 		pipe = INVALID_PIPE;
@@ -2866,6 +2883,9 @@ intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
 
 	drm_WARN_ON(display->drm, !new_cdclk_state->base.changed);
 
+	if (display->platform.dg2)
+		dg2_cdclk_pcode_pre_notify(state);
+
 	intel_set_cdclk(display, &cdclk_config, pipe,
 			"Pre changing CDCLK to");
 }
@@ -2891,12 +2911,11 @@ intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
 		return;
 
 	if (!intel_cdclk_changed(&old_cdclk_state->actual,
-				 &new_cdclk_state->actual))
+				 &new_cdclk_state->actual) &&
+	    dg2_power_well_count(display, old_cdclk_state) ==
+	    dg2_power_well_count(display, new_cdclk_state))
 		return;
 
-	if (display->platform.dg2)
-		intel_cdclk_pcode_post_notify(state);
-
 	if (!new_cdclk_state->disable_pipes &&
 	    new_cdclk_state->actual.cdclk < old_cdclk_state->actual.cdclk)
 		pipe = new_cdclk_state->pipe;
@@ -2907,6 +2926,9 @@ intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
 
 	intel_set_cdclk(display, &new_cdclk_state->actual, pipe,
 			"Post changing CDCLK to");
+
+	if (display->platform.dg2)
+		dg2_cdclk_pcode_post_notify(state);
 }
 
 /* pixels per CDCLK */
@@ -3942,9 +3964,25 @@ static const struct intel_cdclk_funcs xe3lpd_cdclk_funcs = {
 	.calc_voltage_level = xe3lpd_calc_voltage_level,
 };
 
+static const struct intel_cdclk_funcs mtl_cdclk_funcs = {
+	.get_cdclk = bxt_get_cdclk,
+	.set_cdclk = bxt_set_cdclk,
+	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
+	.calc_voltage_level = rplu_calc_voltage_level,
+};
+
+static const struct intel_cdclk_funcs dg2_cdclk_funcs = {
+	.get_cdclk = bxt_get_cdclk,
+	.set_cdclk = bxt_set_cdclk,
+	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
+	.calc_voltage_level = tgl_calc_voltage_level,
+};
+
 static const struct intel_cdclk_funcs rplu_cdclk_funcs = {
 	.get_cdclk = bxt_get_cdclk,
 	.set_cdclk = bxt_set_cdclk,
+	.pre_notify = skl_cdclk_pcode_pre_notify,
+	.post_notify = skl_cdclk_pcode_post_notify,
 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
 	.calc_voltage_level = rplu_calc_voltage_level,
 };
@@ -3952,6 +3990,8 @@ static const struct intel_cdclk_funcs rplu_cdclk_funcs = {
 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
 	.get_cdclk = bxt_get_cdclk,
 	.set_cdclk = bxt_set_cdclk,
+	.pre_notify = skl_cdclk_pcode_pre_notify,
+	.post_notify = skl_cdclk_pcode_post_notify,
 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
 	.calc_voltage_level = tgl_calc_voltage_level,
 };
@@ -3959,6 +3999,8 @@ static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
 	.get_cdclk = bxt_get_cdclk,
 	.set_cdclk = bxt_set_cdclk,
+	.pre_notify = skl_cdclk_pcode_pre_notify,
+	.post_notify = skl_cdclk_pcode_post_notify,
 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
 	.calc_voltage_level = ehl_calc_voltage_level,
 };
@@ -3966,6 +4008,8 @@ static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
 	.get_cdclk = bxt_get_cdclk,
 	.set_cdclk = bxt_set_cdclk,
+	.pre_notify = skl_cdclk_pcode_pre_notify,
+	.post_notify = skl_cdclk_pcode_post_notify,
 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
 	.calc_voltage_level = icl_calc_voltage_level,
 };
@@ -3973,6 +4017,8 @@ static const struct intel_cdclk_funcs icl_cdclk_funcs = {
 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
 	.get_cdclk = bxt_get_cdclk,
 	.set_cdclk = bxt_set_cdclk,
+	.pre_notify = bxt_cdclk_pcode_pre_notify,
+	.post_notify = bxt_cdclk_pcode_post_notify,
 	.modeset_calc_cdclk = bxt_modeset_calc_cdclk,
 	.calc_voltage_level = bxt_calc_voltage_level,
 };
@@ -3980,12 +4026,16 @@ static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
 	.get_cdclk = skl_get_cdclk,
 	.set_cdclk = skl_set_cdclk,
+	.pre_notify = skl_cdclk_pcode_pre_notify,
+	.post_notify = skl_cdclk_pcode_post_notify,
 	.modeset_calc_cdclk = skl_modeset_calc_cdclk,
 };
 
 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
 	.get_cdclk = bdw_get_cdclk,
 	.set_cdclk = bdw_set_cdclk,
+	.pre_notify = bdw_cdclk_pcode_pre_notify,
+	.post_notify = bdw_cdclk_pcode_post_notify,
 	.modeset_calc_cdclk = bdw_modeset_calc_cdclk,
 };
 
@@ -4091,16 +4141,16 @@ void intel_init_cdclk_hooks(struct intel_display *display)
 		display->cdclk.funcs = &xe3lpd_cdclk_funcs;
 		display->cdclk.table = xe3lpd_cdclk_table;
 	} else if (DISPLAY_VER(display) >= 20) {
-		display->cdclk.funcs = &rplu_cdclk_funcs;
+		display->cdclk.funcs = &mtl_cdclk_funcs;
 		display->cdclk.table = xe2lpd_cdclk_table;
 	} else if (DISPLAY_VERx100(display) >= 1401) {
-		display->cdclk.funcs = &rplu_cdclk_funcs;
+		display->cdclk.funcs = &mtl_cdclk_funcs;
 		display->cdclk.table = xe2hpd_cdclk_table;
 	} else if (DISPLAY_VER(display) >= 14) {
-		display->cdclk.funcs = &rplu_cdclk_funcs;
+		display->cdclk.funcs = &mtl_cdclk_funcs;
 		display->cdclk.table = mtl_cdclk_table;
 	} else if (display->platform.dg2) {
-		display->cdclk.funcs = &tgl_cdclk_funcs;
+		display->cdclk.funcs = &dg2_cdclk_funcs;
 		display->cdclk.table = dg2_cdclk_table;
 	} else if (display->platform.alderlake_p) {
 		/* Wa_22011320316:adl-p[a0] */
diff --git a/drivers/gpu/drm/i915/display/intel_cmtg.c b/drivers/gpu/drm/i915/display/intel_cmtg.c
index c8e0f90af91082..3c0f5b2fb2e701 100644
--- a/drivers/gpu/drm/i915/display/intel_cmtg.c
+++ b/drivers/gpu/drm/i915/display/intel_cmtg.c
@@ -326,7 +326,7 @@ void intel_cmtg_set_vrr_ctl(const struct intel_crtc_state *crtc_state)
 		  XELPD_VRR_CTL_VRR_GUARDBAND(crtc_state->vrr.guardband);
 
 	/* TODO: The code below may need to be revisited once CMRR is enabled */
-	if (crtc_state->cmrr.enable)
+	if (crtc_state->vrr.cmrr.enable)
 		vrr_ctl |= VRR_CTL_CMRR_ENABLE;
 
 	intel_de_write(display, TRANS_VRR_CTL(display, cmtg_transcoder), vrr_ctl);
diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c
index 87ced9f6ff408f..f1df0f9ba76241 100644
--- a/drivers/gpu/drm/i915/display/intel_color.c
+++ b/drivers/gpu/drm/i915/display/intel_color.c
@@ -3998,25 +3998,17 @@ xelpd_program_plane_pre_csc_lut(struct intel_dsb *dsb,
 }
 
 static void
-xelpd_program_plane_post_csc_lut(struct intel_dsb *dsb,
-				 const struct intel_plane_state *plane_state)
+xelpd_load_hdr_post_csc_lut(struct intel_display *display,
+			    struct intel_dsb *dsb,
+			    enum pipe pipe,
+			    enum plane_id plane,
+			    const struct drm_color_lut32 *post_csc_lut)
 {
-	struct intel_display *display = to_intel_display(plane_state);
-	const struct drm_plane_state *state = &plane_state->uapi;
-	enum pipe pipe = to_intel_plane(state->plane)->pipe;
-	enum plane_id plane = to_intel_plane(state->plane)->id;
-	const struct drm_color_lut32 *post_csc_lut = plane_state->hw.gamma_lut->data;
 	int i, lut_size = 32;
 	u32 lut_val;
 
-	if (!icl_is_hdr_plane(display, plane))
-		return;
-
 	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0),
 			   PLANE_PAL_PREC_AUTO_INCREMENT);
-	/* TODO: Add macro */
-	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0),
-			   PLANE_PAL_PREC_AUTO_INCREMENT);
 
 	for (i = 0; i < lut_size + 3; i++) {
 		if (post_csc_lut) {
@@ -4036,8 +4028,62 @@ xelpd_program_plane_post_csc_lut(struct intel_dsb *dsb,
 	}
 
 	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0);
-	intel_de_write_dsb(display, dsb,
-			   PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0), 0);
+}
+
+static void
+xelpd_load_sdr_post_csc_lut(struct intel_display *display,
+			    struct intel_dsb *dsb,
+			    enum pipe pipe,
+			    enum plane_id plane,
+			    const struct drm_color_lut32 *post_csc_lut)
+{
+	int i, lut_size = 32;
+	u32 lut_val;
+
+	/*
+	 * First 3 planes are HDR, so reduce by 3 to get to the right
+	 * SDR plane offset
+	 */
+	plane = plane - 3;
+
+	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX(pipe, plane, 0),
+			   PLANE_PAL_PREC_AUTO_INCREMENT);
+
+	for (i = 0; i < lut_size + 3; i++) {
+		if (post_csc_lut) {
+			if (i < lut_size)
+				lut_val = drm_color_lut32_extract(post_csc_lut[i].green, 16);
+			/* else duplicate last lut_val */
+		} else {
+			if (i < lut_size)
+				lut_val = (i * ((1 << 16) - 1)) / (lut_size - 1);
+			else
+				lut_val = 1 << 16;
+		}
+
+		intel_de_write_dsb(display, dsb,
+				   PLANE_POST_CSC_GAMC_DATA(pipe, plane, 0),
+				   lut_val);
+	}
+
+	intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX(pipe, plane, 0), 0);
+}
+
+static void
+xelpd_program_plane_post_csc_lut(struct intel_dsb *dsb,
+				 const struct intel_plane_state *plane_state)
+{
+	struct intel_display *display = to_intel_display(plane_state);
+	const struct drm_plane_state *state = &plane_state->uapi;
+	enum pipe pipe = to_intel_plane(state->plane)->pipe;
+	enum plane_id plane = to_intel_plane(state->plane)->id;
+	const struct drm_color_lut32 *post_csc_lut = plane_state->hw.gamma_lut ?
+		plane_state->hw.gamma_lut->data : NULL;
+
+	if (icl_is_hdr_plane(display, plane))
+		xelpd_load_hdr_post_csc_lut(display, dsb, pipe, plane, post_csc_lut);
+	else
+		xelpd_load_sdr_post_csc_lut(display, dsb, pipe, plane, post_csc_lut);
 }
 
 static void
diff --git a/drivers/gpu/drm/i915/display/intel_color_pipeline.c b/drivers/gpu/drm/i915/display/intel_color_pipeline.c
index 6cf8080ee80003..7fc31fe891a85d 100644
--- a/drivers/gpu/drm/i915/display/intel_color_pipeline.c
+++ b/drivers/gpu/drm/i915/display/intel_color_pipeline.c
@@ -12,7 +12,7 @@
 #include "skl_universal_plane.h"
 
 #define MAX_COLOR_PIPELINES 1
-#define MAX_COLOROP 4
+#define MAX_COLOROP 5
 #define PLANE_DEGAMMA_SIZE 128
 #define PLANE_GAMMA_SIZE 32
 
@@ -31,6 +31,7 @@ static const struct drm_colorop_funcs intel_colorop_funcs = {
  * the pipeline totally unusable.
  */
 static const enum intel_color_block xe3plpd_primary_plane_pipeline[] = {
+	INTEL_PLANE_CB_CSC_FF,
 	INTEL_PLANE_CB_PRE_CSC_LUT,
 	INTEL_PLANE_CB_CSC,
 	INTEL_PLANE_CB_3DLUT,
@@ -38,11 +39,25 @@ static const enum intel_color_block xe3plpd_primary_plane_pipeline[] = {
 };
 
 static const enum intel_color_block hdr_plane_pipeline[] = {
+	INTEL_PLANE_CB_CSC_FF,
 	INTEL_PLANE_CB_PRE_CSC_LUT,
 	INTEL_PLANE_CB_CSC,
 	INTEL_PLANE_CB_POST_CSC_LUT,
 };
 
+static const enum intel_color_block sdr_plane_yuv_pipeline[] = {
+	INTEL_PLANE_CB_CSC_FF,
+	INTEL_PLANE_CB_POST_CSC_LUT,
+};
+
+static const u64 intel_plane_supported_csc_ff =
+		BIT(DRM_COLOROP_FM_YCBCR601_FULL_RGB) |
+		BIT(DRM_COLOROP_FM_YCBCR601_LIMITED_RGB) |
+		BIT(DRM_COLOROP_FM_YCBCR709_FULL_RGB) |
+		BIT(DRM_COLOROP_FM_YCBCR709_LIMITED_RGB) |
+		BIT(DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB) |
+		BIT(DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB);
+
 static bool plane_has_3dlut(struct intel_display *display, enum pipe pipe,
 			    struct drm_plane *plane)
 {
@@ -92,6 +107,12 @@ struct intel_colorop *intel_color_pipeline_plane_add_colorop(struct drm_plane *p
 							  DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR,
 							  DRM_COLOROP_FLAG_ALLOW_BYPASS);
 		break;
+	case INTEL_PLANE_CB_CSC_FF:
+		ret = drm_plane_colorop_fixed_matrix_init(dev, &colorop->base, plane,
+							  &intel_colorop_funcs,
+							  intel_plane_supported_csc_ff,
+							  DRM_COLOROP_FLAG_ALLOW_BYPASS);
+		break;
 	default:
 		drm_err(plane->dev, "Invalid colorop id [%d]", id);
 		ret = -EINVAL;
@@ -126,9 +147,12 @@ int _intel_color_pipeline_plane_init(struct drm_plane *plane, struct drm_prop_en
 	if (plane_has_3dlut(display, pipe, plane)) {
 		pipeline = xe3plpd_primary_plane_pipeline;
 		pipeline_len = ARRAY_SIZE(xe3plpd_primary_plane_pipeline);
-	} else {
+	} else if (icl_is_hdr_plane(display, to_intel_plane(plane)->id)) {
 		pipeline = hdr_plane_pipeline;
 		pipeline_len = ARRAY_SIZE(hdr_plane_pipeline);
+	} else {
+		pipeline = sdr_plane_yuv_pipeline;
+		pipeline_len = ARRAY_SIZE(sdr_plane_yuv_pipeline);
 	}
 
 	for (i = 0; i < pipeline_len; i++) {
@@ -155,17 +179,11 @@ int _intel_color_pipeline_plane_init(struct drm_plane *plane, struct drm_prop_en
 
 int intel_color_pipeline_plane_init(struct drm_plane *plane, enum pipe pipe)
 {
-	struct drm_device *dev = plane->dev;
-	struct intel_display *display = to_intel_display(dev);
 	struct drm_prop_enum_list pipelines[MAX_COLOR_PIPELINES] = {};
 	int len = 0;
 	int ret = 0;
 	int i;
 
-	/* Currently expose pipeline only for HDR planes */
-	if (!icl_is_hdr_plane(display, to_intel_plane(plane)->id))
-		return 0;
-
 	/* Add pipeline consisting of transfer functions */
 	ret = _intel_color_pipeline_plane_init(plane, &pipelines[len], pipe);
 	if (ret)
diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
index 4493483f10a9e5..92675a88b969be 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
@@ -251,6 +251,15 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 			   str_enabled_disabled(pipe_config->has_panel_replay),
 			   str_enabled_disabled(pipe_config->enable_psr2_sel_fetch));
 		drm_printf(&p, "minimum hblank: %d\n", pipe_config->min_hblank);
+
+		drm_printf(&p, "DIP Transmission Lines: EMP/AS SDP: %u\n",
+			   pipe_config->dip.emp_as_sdp_tl);
+		drm_printf(&p, "DIP Transmission Lines: Common Base SDP: %u, GMP SDP: %u, PPS SDP: %u, VSC SDP: %u, VSC_EXT SDP: %u\n",
+			   pipe_config->dip.cmn_sdp_tl,
+			   pipe_config->dip.gmp_sdp_tl,
+			   pipe_config->dip.pps_sdp_tl,
+			   pipe_config->dip.vsc_sdp_tl,
+			   pipe_config->dip.vsc_ext_sdp_tl);
 	}
 
 	drm_printf(&p, "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
@@ -311,6 +320,9 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 		   pipe_config->vrr.dc_balance.max_increase,
 		   pipe_config->vrr.dc_balance.max_decrease,
 		   pipe_config->vrr.dc_balance.vblank_target);
+	drm_printf(&p, "cmrr: %s, cmrr_m: %llu, cmrr_n: %llu\n",
+		   str_yes_no(pipe_config->vrr.cmrr.enable),
+		   pipe_config->vrr.cmrr.cmrr_m, pipe_config->vrr.cmrr.cmrr_n);
 
 	drm_printf(&p, "requested mode: " DRM_MODE_FMT "\n",
 		   DRM_MODE_ARG(&pipe_config->hw.mode));
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index 0673f16f6fd0dc..4afd275de86eae 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -13,6 +13,7 @@
 #include <drm/drm_vblank.h>
 
 #include "intel_atomic.h"
+#include "intel_crtc.h"
 #include "intel_cursor.h"
 #include "intel_cursor_regs.h"
 #include "intel_de.h"
@@ -536,7 +537,8 @@ static void i9xx_cursor_disable_sel_fetch_arm(struct intel_dsb *dsb,
 	struct intel_display *display = to_intel_display(plane);
 	enum pipe pipe = plane->pipe;
 
-	if (!crtc_state->enable_psr2_sel_fetch)
+	if (!crtc_state->enable_psr2_sel_fetch &&
+	    !crtc_state->clear_psr2_sel_fetch)
 		return;
 
 	intel_de_write_dsb(display, dsb, SEL_FETCH_CUR_CTL(pipe), 0);
@@ -569,8 +571,10 @@ static void i9xx_cursor_update_sel_fetch_arm(struct intel_dsb *dsb,
 	struct intel_display *display = to_intel_display(plane);
 	enum pipe pipe = plane->pipe;
 
-	if (!crtc_state->enable_psr2_sel_fetch)
+	if (!crtc_state->enable_psr2_sel_fetch) {
+		i9xx_cursor_disable_sel_fetch_arm(dsb, plane, crtc_state);
 		return;
+	}
 
 	if (drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0) {
 		if (crtc_state->enable_psr2_su_region_et) {
@@ -796,6 +800,88 @@ void intel_cursor_unpin_work(struct kthread_work *base)
 	intel_plane_destroy_state(&plane->base, &plane_state->uapi);
 }
 
+static int intel_cursor_lock_joined_planes(struct intel_display *display,
+					   const struct intel_crtc_state *crtc_state,
+					   struct drm_modeset_acquire_ctx *ctx)
+{
+	struct intel_crtc *pipe_crtc;
+	int ret;
+
+	for_each_intel_crtc_in_pipe_mask(display, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(crtc_state)) {
+		struct intel_plane *pipe_plane =
+			intel_crtc_get_plane(pipe_crtc, PLANE_CURSOR);
+
+		ret = drm_modeset_lock(&pipe_crtc->base.mutex, ctx);
+		if (ret)
+			return ret;
+
+		ret = drm_modeset_lock(&pipe_plane->base.mutex, ctx);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+static bool
+intel_cursor_joiner_commits_idle(struct intel_display *display,
+				 const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *pipe_crtc;
+
+	for_each_intel_crtc_in_pipe_mask(display, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(crtc_state)) {
+		struct intel_plane *pipe_plane =
+			intel_crtc_get_plane(pipe_crtc, PLANE_CURSOR);
+		struct intel_plane_state *pipe_plane_state =
+			to_intel_plane_state(pipe_plane->base.state);
+
+		if (pipe_plane_state->uapi.commit &&
+		    !try_wait_for_completion(&pipe_plane_state->uapi.commit->hw_done))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+intel_cursor_fastpath_update_plane_state(struct intel_plane_state *plane_state,
+					 const struct intel_plane_state *from_plane_state,
+					 struct drm_framebuffer *fb,
+					 struct intel_crtc *hw_crtc,
+					 int crtc_x, int crtc_y,
+					 unsigned int crtc_w, unsigned int crtc_h,
+					 u32 src_x, u32 src_y,
+					 u32 src_w, u32 src_h)
+{
+	/*
+	 * Only the primary owns its uapi state; a secondary mirrors it, so
+	 * its uapi.crtc/fb stay NULL and hw.crtc comes from hw_crtc.
+	 */
+	if (plane_state == from_plane_state) {
+		drm_atomic_set_fb_for_plane(&plane_state->uapi, fb);
+
+		plane_state->uapi.src_x = src_x;
+		plane_state->uapi.src_y = src_y;
+		plane_state->uapi.src_w = src_w;
+		plane_state->uapi.src_h = src_h;
+		plane_state->uapi.crtc_x = crtc_x;
+		plane_state->uapi.crtc_y = crtc_y;
+		plane_state->uapi.crtc_w = crtc_w;
+		plane_state->uapi.crtc_h = crtc_h;
+	}
+
+	intel_plane_copy_uapi_to_hw_state(NULL, plane_state, from_plane_state, hw_crtc);
+}
+
+struct intel_cursor_joiner_state {
+	struct intel_plane *plane;
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *crtc_state;
+	struct intel_plane_state *old_plane_state;
+	struct intel_plane_state *new_plane_state;
+};
+
 static int
 intel_legacy_cursor_update(struct drm_plane *_plane,
 			   struct drm_crtc *_crtc,
@@ -811,11 +897,13 @@ intel_legacy_cursor_update(struct drm_plane *_plane,
 	struct intel_display *display = to_intel_display(plane);
 	struct intel_plane_state *old_plane_state =
 		to_intel_plane_state(plane->base.state);
-	struct intel_plane_state *new_plane_state;
 	struct intel_crtc_state *crtc_state =
 		to_intel_crtc_state(crtc->base.state);
-	struct intel_crtc_state *new_crtc_state;
 	struct intel_vblank_evade_ctx evade;
+	struct intel_cursor_joiner_state joined_pipe_state[I915_MAX_PIPES] = {};
+	struct intel_crtc *pipe_crtc;
+	int num_pipes = 0;
+	u32 start_vbl_count, end_vbl_count;
 	int ret;
 
 	/*
@@ -824,22 +912,10 @@ intel_legacy_cursor_update(struct drm_plane *_plane,
 	 * PSR2 selective fetch also requires the slow path as
 	 * PSR2 plane and transcoder registers can only be updated during
 	 * vblank.
-	 *
-	 * FIXME joiner fastpath would be good
 	 */
 	if (!crtc_state->hw.active ||
 	    intel_crtc_needs_modeset(crtc_state) ||
-	    intel_crtc_needs_fastset(crtc_state) ||
-	    crtc_state->joiner_pipes)
-		goto slow;
-
-	/*
-	 * Don't do an async update if there is an outstanding commit modifying
-	 * the plane.  This prevents our async update's changes from getting
-	 * overridden by a previous synchronous update's state.
-	 */
-	if (old_plane_state->uapi.commit &&
-	    !try_wait_for_completion(&old_plane_state->uapi.commit->hw_done))
+	    intel_crtc_needs_fastset(crtc_state))
 		goto slow;
 
 	/*
@@ -855,46 +931,89 @@ intel_legacy_cursor_update(struct drm_plane *_plane,
 	    !old_plane_state->uapi.fb != !fb)
 		goto slow;
 
-	new_plane_state = to_intel_plane_state(intel_plane_duplicate_state(&plane->base));
-	if (!new_plane_state)
-		return -ENOMEM;
-
-	new_crtc_state = to_intel_crtc_state(intel_crtc_duplicate_state(&crtc->base));
-	if (!new_crtc_state) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
+	ret = intel_cursor_lock_joined_planes(display, crtc_state, ctx);
+	if (ret == -EDEADLK)
+		return ret;
+	if (ret)
+		goto slow;
 
-	drm_atomic_set_fb_for_plane(&new_plane_state->uapi, fb);
+	/*
+	 * Don't do an async update if there is an outstanding commit modifying
+	 * any of the joined cursor planes. This prevents our async update's
+	 * changes from getting overridden by a previous synchronous update's
+	 * state.
+	 */
+	if (!intel_cursor_joiner_commits_idle(display, crtc_state))
+		goto slow;
 
-	new_plane_state->uapi.src_x = src_x;
-	new_plane_state->uapi.src_y = src_y;
-	new_plane_state->uapi.src_w = src_w;
-	new_plane_state->uapi.src_h = src_h;
-	new_plane_state->uapi.crtc_x = crtc_x;
-	new_plane_state->uapi.crtc_y = crtc_y;
-	new_plane_state->uapi.crtc_w = crtc_w;
-	new_plane_state->uapi.crtc_h = crtc_h;
+	/*
+	 * Iterate over all joined pipes (primary and secondary) uniformly.
+	 * The joined pipe mask includes both the primary pipe and all
+	 * secondary joiner pipes, allowing us to handle them all the same way.
+	 */
+	for_each_intel_crtc_in_pipe_mask(display, pipe_crtc,
+					 intel_crtc_joined_pipe_mask(crtc_state)) {
+		struct intel_cursor_joiner_state *j = &joined_pipe_state[num_pipes];
+
+		j->plane = intel_crtc_get_plane(pipe_crtc, PLANE_CURSOR);
+		j->crtc = pipe_crtc;
+		j->crtc_state = to_intel_crtc_state(pipe_crtc->base.state);
+		j->old_plane_state = to_intel_plane_state(j->plane->base.state);
+		j->new_plane_state =
+			to_intel_plane_state(intel_plane_duplicate_state(&j->plane->base));
+
+		if (!j->new_plane_state) {
+			ret = -ENOMEM;
+			goto out_free;
+		}
 
-	intel_plane_copy_uapi_to_hw_state(NULL, new_plane_state, new_plane_state, crtc);
+		intel_cursor_fastpath_update_plane_state(j->new_plane_state,
+							 joined_pipe_state[0].new_plane_state,
+							 fb, pipe_crtc,
+							 crtc_x, crtc_y,
+							 crtc_w, crtc_h,
+							 src_x, src_y,
+							 src_w, src_h);
+
+		ret = j->plane->check_plane(j->crtc_state, j->new_plane_state);
+		if (ret) {
+			intel_plane_destroy_state(&j->plane->base,
+						  &j->new_plane_state->uapi);
+			goto out_free;
+		}
 
-	ret = intel_plane_atomic_check_with_state(crtc_state, new_crtc_state,
-						  old_plane_state, new_plane_state);
-	if (ret)
-		goto out_free;
+		ret = intel_plane_pin_fb(j->new_plane_state, j->old_plane_state);
+		if (ret) {
+			intel_plane_destroy_state(&j->plane->base,
+						  &j->new_plane_state->uapi);
+			goto out_free;
+		}
 
-	ret = intel_plane_pin_fb(new_plane_state, old_plane_state);
-	if (ret)
-		goto out_free;
+		num_pipes++;
+	}
 
-	intel_frontbuffer_flush(to_intel_frontbuffer(new_plane_state->hw.fb),
+	intel_frontbuffer_flush(to_intel_frontbuffer(joined_pipe_state[0].new_plane_state->hw.fb),
 				ORIGIN_CURSOR_UPDATE);
-	intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
-				to_intel_frontbuffer(new_plane_state->hw.fb),
-				plane->frontbuffer_bit);
 
-	/* Swap plane state */
-	plane->base.state = &new_plane_state->uapi;
+	for (int i = 0; i < num_pipes; i++) {
+		struct intel_frontbuffer *old_front =
+			to_intel_frontbuffer(joined_pipe_state[i].old_plane_state->hw.fb);
+		struct intel_frontbuffer *new_front =
+			to_intel_frontbuffer(joined_pipe_state[i].new_plane_state->hw.fb);
+
+		intel_frontbuffer_track(old_front, new_front,
+					joined_pipe_state[i].plane->frontbuffer_bit);
+	}
+
+	for (int i = 0; i < num_pipes; i++) {
+		joined_pipe_state[i].plane->base.state =
+			&joined_pipe_state[i].new_plane_state->uapi;
+
+		if (joined_pipe_state[i].new_plane_state->uapi.visible)
+			joined_pipe_state[i].crtc_state->active_planes |= BIT(PLANE_CURSOR);
+		else
+			joined_pipe_state[i].crtc_state->active_planes &= ~BIT(PLANE_CURSOR);
+	}
 
 	/*
 	 * We cannot swap crtc_state as it may be in use by an atomic commit or
@@ -906,7 +1025,6 @@ intel_legacy_cursor_update(struct drm_plane *_plane,
 	 * planes atomically. If the cursor was part of the atomic update then
 	 * we would have taken the slowpath.
 	 */
-	crtc_state->active_planes = new_crtc_state->active_planes;
 
 	intel_vblank_evade_init(crtc_state, crtc_state, &evade);
 
@@ -928,37 +1046,74 @@ intel_legacy_cursor_update(struct drm_plane *_plane,
 		local_irq_disable();
 	}
 
-	if (new_plane_state->uapi.visible) {
-		intel_plane_update_noarm(NULL, plane, crtc_state, new_plane_state);
-		intel_plane_update_arm(NULL, plane, crtc_state, new_plane_state);
-	} else {
-		intel_plane_disable_arm(NULL, plane, crtc_state);
+	/*
+	 * Joiner pipes are vblank-synchronized, so sampling only the primary
+	 * pipe is sufficient to detect a straddle across all joined pipes.
+	 * The vblank evasion above also operates on the primary pipe only.
+	 */
+	start_vbl_count = intel_crtc_get_vblank_counter(joined_pipe_state[0].crtc);
+
+	for (int i = 0; i < num_pipes; i++) {
+		if (joined_pipe_state[i].new_plane_state->uapi.visible) {
+			intel_plane_update_noarm(NULL, joined_pipe_state[i].plane,
+						 joined_pipe_state[i].crtc_state,
+						 joined_pipe_state[i].new_plane_state);
+			intel_plane_update_arm(NULL, joined_pipe_state[i].plane,
+					       joined_pipe_state[i].crtc_state,
+					       joined_pipe_state[i].new_plane_state);
+		} else {
+			intel_plane_disable_arm(NULL, joined_pipe_state[i].plane,
+						joined_pipe_state[i].crtc_state);
+		}
 	}
 
-	local_irq_enable();
+	end_vbl_count = intel_crtc_get_vblank_counter(joined_pipe_state[0].crtc);
 
-	intel_psr_unlock(crtc_state);
+	local_irq_enable();
 
-	if (old_plane_state->ggtt_vma != new_plane_state->ggtt_vma) {
-		drm_vblank_work_init(&old_plane_state->unpin_work, &crtc->base,
-				     intel_cursor_unpin_work);
+	if (start_vbl_count != end_vbl_count)
+		drm_err(display->drm,
+			"Atomic update failure on pipe %c (start=%u end=%u)\n",
+			pipe_name(joined_pipe_state[0].crtc->pipe),
+			start_vbl_count, end_vbl_count);
 
-		drm_vblank_work_schedule(&old_plane_state->unpin_work,
-					 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
-					 false);
+	intel_psr_unlock(crtc_state);
 
-		old_plane_state = NULL;
-	} else {
-		intel_plane_unpin_fb(old_plane_state);
+	/*
+	 * Schedule or immediately unpin old framebuffers.
+	 * Protect against concurrent access.
+	 */
+	for (int i = 0; i < num_pipes; i++) {
+		struct intel_plane_state *old = joined_pipe_state[i].old_plane_state;
+
+		if (old->ggtt_vma != joined_pipe_state[i].new_plane_state->ggtt_vma) {
+			drm_vblank_work_init(&old->unpin_work, &crtc->base,
+					     intel_cursor_unpin_work);
+			drm_vblank_work_schedule(&old->unpin_work,
+						 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
+						 false);
+			joined_pipe_state[i].old_plane_state = NULL;
+		} else {
+			intel_plane_unpin_fb(old);
+		}
 	}
 
 out_free:
-	if (new_crtc_state)
-		intel_crtc_destroy_state(&crtc->base, &new_crtc_state->uapi);
-	if (ret)
-		intel_plane_destroy_state(&plane->base, &new_plane_state->uapi);
-	else if (old_plane_state)
-		intel_plane_destroy_state(&plane->base, &old_plane_state->uapi);
+	if (ret) {
+		for (int i = 0; i < num_pipes; i++) {
+			intel_plane_unpin_fb(joined_pipe_state[i].new_plane_state);
+			intel_plane_destroy_state(&joined_pipe_state[i].plane->base,
+						  &joined_pipe_state[i].new_plane_state->uapi);
+		}
+	} else {
+		for (int i = 0; i < num_pipes; i++) {
+			if (!joined_pipe_state[i].old_plane_state)
+				continue;
+
+			intel_plane_destroy_state(&joined_pipe_state[i].plane->base,
+						  &joined_pipe_state[i].old_plane_state->uapi);
+		}
+	}
 	return ret;
 
 slow:
diff --git a/drivers/gpu/drm/i915/display/intel_cursor_regs.h b/drivers/gpu/drm/i915/display/intel_cursor_regs.h
index aaa66331063e14..da2f9630ae2d22 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_cursor_regs.h
@@ -81,7 +81,7 @@
 #define   CUR_WM_EN				REG_BIT(31)
 #define   CUR_WM_IGNORE_LINES			REG_BIT(30)
 #define   CUR_WM_LINES_MASK			REG_GENMASK(26, 14)
-#define   CUR_WM_BLOCKS_MASK			REG_GENMASK(11, 0)
+#define   CUR_WM_BLOCKS_MASK			REG_GENMASK(12, 0)
 
 #define _CUR_WM_SAGV_A		0x70158
 #define _CUR_WM_SAGV_B		0x71158
@@ -98,10 +98,10 @@
 #define _CUR_BUF_CFG_A		0x7017c
 #define _CUR_BUF_CFG_B		0x7117c
 #define CUR_BUF_CFG(pipe)	_MMIO_PIPE((pipe), _CUR_BUF_CFG_A, _CUR_BUF_CFG_B)
-/* skl+: 10 bits, icl+ 11 bits, adlp+ 12 bits */
-#define   CUR_BUF_END_MASK		REG_GENMASK(27, 16)
+/* skl+: 10 bits, icl+ 11 bits, adlp+ 12 bits, xe3plpd+ 13 bits */
+#define   CUR_BUF_END_MASK		REG_GENMASK(28, 16)
 #define   CUR_BUF_END(end)		REG_FIELD_PREP(CUR_BUF_END_MASK, (end))
-#define   CUR_BUF_START_MASK		REG_GENMASK(11, 0)
+#define   CUR_BUF_START_MASK		REG_GENMASK(12, 0)
 #define   CUR_BUF_START(start)		REG_FIELD_PREP(CUR_BUF_START_MASK, (start))
 
 /* tgl+ */
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
index dbebd721084849..b83f0afaa9b523 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
@@ -5,6 +5,7 @@
 
 #include <linux/log2.h>
 #include <linux/math64.h>
+#include <linux/string_choices.h>
 
 #include <drm/drm_print.h>
 
@@ -189,7 +190,7 @@ int intel_cx0_wait_for_ack(struct intel_encoder *encoder,
 		drm_dbg_kms(display->drm,
 			    "PHY %c Error occurred during %s command. Status: 0x%x\n",
 			    phy_name(phy),
-			    command == XELPDP_PORT_P2M_COMMAND_READ_ACK ? "read" : "write", *val);
+			    str_read_write(command == XELPDP_PORT_P2M_COMMAND_READ_ACK), *val);
 		intel_cx0_bus_reset(encoder, lane);
 		return -EINVAL;
 	}
@@ -198,7 +199,7 @@ int intel_cx0_wait_for_ack(struct intel_encoder *encoder,
 		drm_dbg_kms(display->drm,
 			    "PHY %c Not a %s response. MSGBUS Status: 0x%x.\n",
 			    phy_name(phy),
-			    command == XELPDP_PORT_P2M_COMMAND_READ_ACK ? "read" : "write", *val);
+			    str_read_write(command == XELPDP_PORT_P2M_COMMAND_READ_ACK), *val);
 		intel_cx0_bus_reset(encoder, lane);
 		return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index 02a53c9848e137..a46ab5d3d70fc7 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -49,6 +49,7 @@
 #include "intel_ddi.h"
 #include "intel_ddi_buf_trans.h"
 #include "intel_de.h"
+#include "intel_dip.h"
 #include "intel_display_power.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
@@ -2331,15 +2332,17 @@ static void intel_dp_sink_set_msa_timing_par_ignore_state(struct intel_dp *intel
 							  bool enable)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	int ret;
 
 	if (!crtc_state->vrr.enable)
 		return;
 
-	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_DOWNSPREAD_CTRL,
-			       enable ? DP_MSA_TIMING_PAR_IGNORE_EN : 0) <= 0)
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_DOWNSPREAD_CTRL,
+				     enable ? DP_MSA_TIMING_PAR_IGNORE_EN : 0);
+	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Failed to %s MSA_TIMING_PAR_IGNORE in the sink\n",
-			    str_enable_disable(enable));
+			    "Failed to %s MSA_TIMING_PAR_IGNORE in the sink (%pe)\n",
+			    str_enable_disable(enable), ERR_PTR(ret));
 }
 
 static void intel_dp_sink_set_fec_ready(struct intel_dp *intel_dp,
@@ -2347,19 +2350,25 @@ static void intel_dp_sink_set_fec_ready(struct intel_dp *intel_dp,
 					bool enable)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	int ret;
 
 	if (!crtc_state->fec_enable)
 		return;
 
-	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_CONFIGURATION,
-			       enable ? DP_FEC_READY : 0) <= 0)
-		drm_dbg_kms(display->drm, "Failed to set FEC_READY to %s in the sink\n",
-			    str_enabled_disabled(enable));
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_FEC_CONFIGURATION,
+				     enable ? DP_FEC_READY : 0);
+	if (ret < 0)
+		drm_dbg_kms(display->drm, "Failed to set FEC_READY to %s in the sink (%pe)\n",
+			    str_enabled_disabled(enable), ERR_PTR(ret));
 
-	if (enable &&
-	    drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_STATUS,
-			       DP_FEC_DECODE_EN_DETECTED | DP_FEC_DECODE_DIS_DETECTED) <= 0)
-		drm_dbg_kms(display->drm, "Failed to clear FEC detected flags\n");
+	if (enable) {
+		ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_FEC_STATUS,
+					     DP_FEC_DECODE_EN_DETECTED |
+					     DP_FEC_DECODE_DIS_DETECTED);
+		if (ret < 0)
+			drm_dbg_kms(display->drm, "Failed to clear FEC detected flags (%pe)\n",
+				    ERR_PTR(ret));
+	}
 }
 
 static int wait_for_fec_detected(struct drm_dp_aux *aux, bool enabled)
@@ -2736,6 +2745,8 @@ static void mtl_ddi_pre_enable_dp(struct intel_atomic_state *state,
 	/* 6.o Configure and enable FEC if needed */
 	intel_ddi_enable_fec(encoder, crtc_state);
 
+	intel_dip_cmn_sdp_transmission_line_enable(crtc_state);
+
 	/* 7.a 128b/132b SST. */
 	if (!is_mst && intel_dp_is_uhbr(crtc_state)) {
 		/* VCPID 1, start slot 0 for 128b/132b, tu slots */
@@ -3123,6 +3134,7 @@ static void intel_ddi_buf_disable(struct intel_encoder *encoder,
 			     DP_TP_CTL_ENABLE, 0);
 	}
 
+	intel_dip_cmn_sdp_transmission_line_disable(crtc_state);
 	intel_ddi_disable_fec(encoder, crtc_state);
 
 	if (DISPLAY_VER(display) < 14)
@@ -4235,6 +4247,7 @@ static void intel_ddi_get_config(struct intel_encoder *encoder,
 	intel_read_dp_sdp(encoder, pipe_config, HDMI_PACKET_TYPE_GAMUT_METADATA);
 	intel_read_dp_sdp(encoder, pipe_config, DP_SDP_VSC);
 	intel_read_dp_sdp(encoder, pipe_config, DP_SDP_ADAPTIVE_SYNC);
+	intel_dip_sdp_transmission_line_get_config(pipe_config);
 
 	intel_audio_codec_get_config(encoder, pipe_config);
 }
@@ -4325,11 +4338,33 @@ static void mtl_ddi_tc_phy_get_config(struct intel_encoder *encoder,
 				       mtl_port_to_pll_id(display, encoder->port));
 }
 
+static struct intel_dpll *dg2_ddi_get_pll(struct intel_encoder *encoder)
+{
+	struct intel_display *display = to_intel_display(encoder);
+
+	return intel_get_dpll_by_id(display, dg2_port_to_pll_id(encoder->port));
+}
+
 static void dg2_ddi_get_config(struct intel_encoder *encoder,
-				struct intel_crtc_state *crtc_state)
+			       struct intel_crtc_state *crtc_state)
 {
-	intel_mpllb_readout_hw_state(encoder, &crtc_state->dpll_hw_state.mpllb);
-	crtc_state->port_clock = intel_mpllb_calc_port_clock(encoder, &crtc_state->dpll_hw_state.mpllb);
+	struct icl_port_dpll *port_dpll = &crtc_state->icl_port_dplls[ICL_PORT_DPLL_DEFAULT];
+	struct intel_dpll *pll = dg2_ddi_get_pll(encoder);
+
+	if (pll)
+		intel_ddi_get_clock(encoder, crtc_state, pll);
+
+	/*
+	 * Keep the hw readout robust against unexpected NULL PLL lookups,
+	 * so modeset verify always has intel_dpll populated for DG2.
+	 */
+	if (!crtc_state->intel_dpll) {
+		port_dpll->pll = pll;
+		intel_mpllb_readout_hw_state(encoder, &port_dpll->hw_state.mpllb);
+		icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_DEFAULT);
+		crtc_state->port_clock = intel_mpllb_calc_port_clock(encoder,
+								     &port_dpll->hw_state.mpllb);
+	}
 
 	intel_ddi_get_config(encoder, crtc_state);
 }
@@ -4657,8 +4692,8 @@ static int intel_ddi_compute_config_late(struct intel_atomic_state *state,
 
 	if (crtc_state->master_transcoder == crtc_state->cpu_transcoder) {
 		crtc_state->master_transcoder = INVALID_TRANSCODER;
-		crtc_state->sync_mode_slaves_mask =
-			port_sync_transcoders & ~BIT(crtc_state->cpu_transcoder);
+		crtc_state->sync_mode_slaves_mask = port_sync_transcoders &
+			~REG_BIT(crtc_state->cpu_transcoder);
 	}
 
 	return 0;
@@ -5343,8 +5378,6 @@ void intel_ddi_init(struct intel_display *display,
 		else
 			encoder->get_config = mtl_ddi_non_tc_phy_get_config;
 	} else if (display->platform.dg2) {
-		encoder->enable_clock = intel_mpllb_enable;
-		encoder->disable_clock = intel_mpllb_disable;
 		encoder->get_config = dg2_ddi_get_config;
 	} else if (display->platform.alderlake_s) {
 		encoder->enable_clock = adls_ddi_enable_clock;
diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
index 4cd1e4d76c7afa..cc5912178d9cdb 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
@@ -1784,6 +1784,70 @@ xe3plpd_get_lt_buf_trans(struct intel_encoder *encoder,
 		return intel_get_buf_trans(&xe3plpd_lt_trans_dp14, n_entries);
 }
 
+static const struct intel_ddi_buf_trans *
+jsl_get_combo_buf_trans_override(struct intel_encoder *encoder,
+				 const struct intel_crtc_state *crtc_state,
+				 int *n_entries)
+{
+	const struct intel_bios_encoder_data *devdata = encoder->devdata;
+	bool has_edp, has_dp;
+	int port_clock;
+
+	has_edp = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP);
+	has_dp = intel_crtc_has_dp_encoder(crtc_state);
+	port_clock = crtc_state->port_clock;
+
+	return intel_bios_get_jsl_combo_vspeo(devdata, has_dp, port_clock,
+					      has_edp && use_edp_low_vswing(encoder));
+}
+
+static const struct intel_ddi_buf_trans *
+ehl_get_combo_buf_trans_override(struct intel_encoder *encoder,
+				 const struct intel_crtc_state *crtc_state,
+				 int *n_entries)
+{
+	const struct intel_bios_encoder_data *devdata = encoder->devdata;
+	bool has_edp, has_dp;
+	int port_clock;
+
+	has_edp = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP);
+	has_dp = intel_crtc_has_dp_encoder(crtc_state);
+	port_clock = crtc_state->port_clock;
+
+	return intel_bios_get_ehl_combo_vspeo(devdata, has_dp, port_clock,
+					      has_edp && use_edp_low_vswing(encoder));
+}
+
+static const struct intel_ddi_buf_trans *
+mtl_get_c10_buf_trans_override(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       int *n_entries)
+{
+	const struct intel_bios_encoder_data *devdata = encoder->devdata;
+	bool has_edp, has_dp;
+	int port_clock;
+
+	has_edp = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP);
+	has_dp = intel_crtc_has_dp_encoder(crtc_state);
+	port_clock = crtc_state->port_clock;
+
+	return intel_bios_get_c10_vspeo(devdata, has_dp, port_clock, has_edp);
+}
+
+static const struct intel_ddi_buf_trans *
+mtl_get_c20_buf_trans_override(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       int *n_entries)
+{
+	const struct intel_bios_encoder_data *devdata = encoder->devdata;
+	bool has_dp, is_uhbr;
+
+	has_dp = intel_crtc_has_dp_encoder(crtc_state);
+	is_uhbr = intel_dp_is_uhbr(crtc_state);
+
+	return intel_bios_get_c20_vspeo(devdata, has_dp, is_uhbr);
+}
+
 void intel_ddi_buf_trans_init(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
@@ -1791,10 +1855,13 @@ void intel_ddi_buf_trans_init(struct intel_encoder *encoder)
 	if (HAS_LT_PHY(display)) {
 		encoder->get_buf_trans = xe3plpd_get_lt_buf_trans;
 	} else if (DISPLAY_VER(display) >= 14) {
-		if (intel_encoder_is_c10phy(encoder))
+		if (intel_encoder_is_c10phy(encoder)) {
 			encoder->get_buf_trans = mtl_get_c10_buf_trans;
-		else
+			encoder->get_buf_trans_override = mtl_get_c10_buf_trans_override;
+		} else {
 			encoder->get_buf_trans = mtl_get_c20_buf_trans;
+			encoder->get_buf_trans_override = mtl_get_c20_buf_trans_override;
+		}
 	} else if (display->platform.dg2) {
 		encoder->get_buf_trans = dg2_get_snps_buf_trans;
 	} else if (display->platform.alderlake_p) {
@@ -1814,14 +1881,17 @@ void intel_ddi_buf_trans_init(struct intel_encoder *encoder)
 		else
 			encoder->get_buf_trans = tgl_get_dkl_buf_trans;
 	} else if (DISPLAY_VER(display) == 11) {
-		if (display->platform.jasperlake)
+		if (display->platform.jasperlake) {
 			encoder->get_buf_trans = jsl_get_combo_buf_trans;
-		else if (display->platform.elkhartlake)
+			encoder->get_buf_trans_override = jsl_get_combo_buf_trans_override;
+		} else if (display->platform.elkhartlake) {
 			encoder->get_buf_trans = ehl_get_combo_buf_trans;
-		else if (intel_encoder_is_combo(encoder))
+			encoder->get_buf_trans_override = ehl_get_combo_buf_trans_override;
+		} else if (intel_encoder_is_combo(encoder)) {
 			encoder->get_buf_trans = icl_get_combo_buf_trans;
-		else
+		} else {
 			encoder->get_buf_trans = icl_get_mg_buf_trans;
+		}
 	} else if (display->platform.geminilake || display->platform.broxton) {
 		encoder->get_buf_trans = bxt_get_buf_trans;
 	} else if (display->platform.cometlake_ulx ||
@@ -1857,5 +1927,13 @@ const struct intel_ddi_buf_trans *intel_ddi_buf_trans_get(struct intel_encoder *
 							  const struct intel_crtc_state *crtc_state,
 							  int *n_entries)
 {
+	if (encoder->get_buf_trans_override) {
+		const struct intel_ddi_buf_trans *override;
+
+		override = encoder->get_buf_trans_override(encoder, crtc_state, n_entries);
+		if (override)
+			return intel_get_buf_trans(override, n_entries);
+	}
+
 	return encoder->get_buf_trans(encoder, crtc_state, n_entries);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dip.c b/drivers/gpu/drm/i915/display/intel_dip.c
new file mode 100644
index 00000000000000..f447150ff7bc03
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dip.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ *
+ */
+
+#include <drm/drm_print.h>
+
+#include "intel_de.h"
+#include "intel_dip.h"
+#include "intel_dip_regs.h"
+#include "intel_display_types.h"
+#include "intel_hdmi.h"
+
+static int intel_dip_get_as_sdp_transmission_line(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+
+	if (!HAS_EMP_AS_SDP_TL(display))
+		return 0;
+
+	/*
+	 * EMP_AS_SDP_TL defines the T1 position as the default AS SDP
+	 * Transmission Line, which corresponds to the start of the
+	 * VSYNC pulse.
+	 *
+	 * Use the T1 position for now.
+	 */
+	return crtc_state->vrr.vsync_start;
+}
+
+u16 intel_dip_read_emp_as_sdp_tl(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 val;
+
+	if (!HAS_EMP_AS_SDP_TL(display))
+		return 0;
+
+	val = intel_de_read(display, EMP_AS_SDP_TL(display, cpu_transcoder));
+	return REG_FIELD_GET(EMP_AS_SDP_DB_TL_MASK, val);
+}
+
+void intel_dip_write_emp_as_sdp_tl(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+
+	if (!HAS_EMP_AS_SDP_TL(display))
+		return;
+
+	intel_de_write(display,
+		       EMP_AS_SDP_TL(display, cpu_transcoder),
+		       EMP_AS_SDP_DB_TL(crtc_state->dip.emp_as_sdp_tl));
+}
+
+static int intel_dip_sdp_stagger_to_tl(struct intel_crtc_state *crtc_state,
+				       int stagger)
+{
+	return crtc_state->dip.cmn_sdp_tl + stagger;
+}
+
+static
+void intel_dip_cmn_sdp_tl_compute_config_late(struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	bool as_sdp;
+
+	if (!HAS_COMMON_SDP_TL(display))
+		return;
+
+	as_sdp = crtc_state->infoframes.enable &
+		 intel_hdmi_infoframe_enable(DP_SDP_ADAPTIVE_SYNC);
+	/*
+	 * When AS SDP is enabled :
+	 *  - The common SDP Transmission Line matches the EMP SDP Transmission Line.
+	 *
+	 * When AS SDP is disabled:
+	 *  - Bspec mentions the positions as lines of delayed vblank.
+	 *  - Guardband = 1st line of delayed vblank
+	 *  - Common SDP Transmission line is set to 2nd line of delayed vblank.
+	 */
+
+	if (as_sdp)
+		crtc_state->dip.cmn_sdp_tl = crtc_state->dip.emp_as_sdp_tl;
+	else
+		crtc_state->dip.cmn_sdp_tl = crtc_state->vrr.guardband - 1;
+
+	if (drm_WARN_ON(display->drm,
+			crtc_state->dip.cmn_sdp_tl >=
+			crtc_state->vrr.guardband + crtc_state->set_context_latency))
+		return;
+
+	/*
+	 * Currently we are programming the default stagger values, but these
+	 * can be optimized if required, based on number of SDPs enabled.
+	 *
+	 * Default values of the Transmission lines for SDPs other than AS SDP:
+	 * VSC : CMN SDP Transmission line
+	 * GMP : CMN SDP Transmission line
+	 * PPS : CMN SDP Transmission line + 1
+	 * VSC_EXT: CMN SDP Transmission line + 2
+	 */
+	crtc_state->dip.vsc_sdp_tl = crtc_state->dip.cmn_sdp_tl;
+	crtc_state->dip.gmp_sdp_tl =
+		intel_dip_sdp_stagger_to_tl(crtc_state, GMP_STAGGER_DEFAULT);
+	crtc_state->dip.pps_sdp_tl =
+		intel_dip_sdp_stagger_to_tl(crtc_state, PPS_STAGGER_DEFAULT);
+	crtc_state->dip.vsc_ext_sdp_tl =
+		intel_dip_sdp_stagger_to_tl(crtc_state, VSC_EXT_STAGGER_DEFAULT);
+}
+
+void intel_dip_sdp_tl_compute_config_late(struct intel_crtc_state *crtc_state)
+{
+	crtc_state->dip.emp_as_sdp_tl = intel_dip_get_as_sdp_transmission_line(crtc_state);
+
+	intel_dip_cmn_sdp_tl_compute_config_late(crtc_state);
+}
+
+static
+void intel_dip_cmn_sdp_transmission_line_get_config(struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u16 vsc_ext_stagger, pps_stagger, gmp_stagger;
+	u32 val;
+
+	if (!HAS_COMMON_SDP_TL(display))
+		return;
+
+	val = intel_de_read(display, CMN_SDP_TL(display, cpu_transcoder));
+
+	if (!(val & TRANSMISSION_LINE_ENABLE))
+		return;
+
+	crtc_state->dip.cmn_sdp_tl = REG_FIELD_GET(BASE_TRANSMISSION_LINE_MASK, val);
+
+	/* SDP VSC uses same transmission line as CMN base transmission line */
+	crtc_state->dip.vsc_sdp_tl = crtc_state->dip.cmn_sdp_tl;
+
+	val = intel_de_read(display, CMN_SDP_TL_STGR_CTL(display, cpu_transcoder));
+
+	vsc_ext_stagger = REG_FIELD_GET(VSC_EXT_STAGGER_MASK, val);
+	pps_stagger = REG_FIELD_GET(PPS_STAGGER_MASK, val);
+	gmp_stagger = REG_FIELD_GET(GMP_STAGGER_MASK, val);
+
+	crtc_state->dip.vsc_ext_sdp_tl =
+		intel_dip_sdp_stagger_to_tl(crtc_state, vsc_ext_stagger);
+	crtc_state->dip.pps_sdp_tl =
+		intel_dip_sdp_stagger_to_tl(crtc_state, pps_stagger);
+	crtc_state->dip.gmp_sdp_tl =
+		intel_dip_sdp_stagger_to_tl(crtc_state, gmp_stagger);
+}
+
+void intel_dip_sdp_transmission_line_get_config(struct intel_crtc_state *crtc_state)
+{
+	crtc_state->dip.emp_as_sdp_tl = intel_dip_read_emp_as_sdp_tl(crtc_state);
+
+	intel_dip_cmn_sdp_transmission_line_get_config(crtc_state);
+}
+
+static int intel_dip_sdp_tl_to_stagger(const struct intel_crtc_state *crtc_state,
+				       u16 sdp_transmission_line)
+{
+	return sdp_transmission_line - crtc_state->dip.cmn_sdp_tl;
+}
+
+void intel_dip_cmn_sdp_transmission_line_enable(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	int gmp_stagger;
+	int pps_stagger;
+	int vsc_ext_stagger;
+
+	if (!HAS_COMMON_SDP_TL(display))
+		return;
+
+	gmp_stagger = intel_dip_sdp_tl_to_stagger(crtc_state,
+						  crtc_state->dip.gmp_sdp_tl);
+
+	pps_stagger = intel_dip_sdp_tl_to_stagger(crtc_state,
+						  crtc_state->dip.pps_sdp_tl);
+
+	vsc_ext_stagger = intel_dip_sdp_tl_to_stagger(crtc_state,
+						      crtc_state->dip.vsc_ext_sdp_tl);
+
+	if (drm_WARN_ON(display->drm, gmp_stagger < 0))
+		return;
+	if (drm_WARN_ON(display->drm, pps_stagger < 0))
+		return;
+	if (drm_WARN_ON(display->drm, vsc_ext_stagger < 0))
+		return;
+
+	intel_de_write(display, CMN_SDP_TL_STGR_CTL(display, cpu_transcoder),
+		       GMP_STAGGER(gmp_stagger) |
+		       PPS_STAGGER(pps_stagger) |
+		       VSC_EXT_STAGGER(vsc_ext_stagger));
+
+	intel_de_write(display, CMN_SDP_TL(display, cpu_transcoder),
+		       TRANSMISSION_LINE_ENABLE |
+		       BASE_TRANSMISSION_LINE(crtc_state->dip.cmn_sdp_tl));
+}
+
+void intel_dip_cmn_sdp_transmission_line_disable(const struct intel_crtc_state *old_crtc_state)
+{
+	struct intel_display *display = to_intel_display(old_crtc_state);
+	enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
+
+	if (!HAS_COMMON_SDP_TL(display))
+		return;
+
+	intel_de_write(display, CMN_SDP_TL(display, cpu_transcoder), 0);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dip.h b/drivers/gpu/drm/i915/display/intel_dip.h
new file mode 100644
index 00000000000000..7c64c84cb1beb2
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dip.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __INTEL_DIP_H__
+#define __INTEL_DIP_H__
+
+#include "intel_display_device.h"
+
+/*
+ * Video DIP (Data Island Packet) helpers.
+ *
+ * This file contains helpers for programming video DIP related hardware.
+ *
+ * TODO: Currently, this is only used for programming EMP_AS_SDP_TL i.e. to
+ * program Transmission Line for HDMI 2.1 Extended Metadata Packet (EMP) and
+ * DP Adaptive Sync (AS) Secondary Data Packet (SDP). However, all low level
+ * DIP buffer read/write and related helpers should be extracted here later.
+ */
+
+struct intel_crtc_state;
+
+/*
+ * EMP AS SDP TL: Extended Metadata Packet (EMP) Adaptive Sync (AS)
+ * Secondary Data Packet (SDP) Transmission Line (TL).
+ *
+ * Starting with BMG (display ver 14.01) and LNL+ (display ver 20+),
+ * the AS SDP transmission line is programmable via the EMP AS SDP TL
+ * register.
+ */
+#define HAS_EMP_AS_SDP_TL(__display)	(DISPLAY_VERx100(__display) == 1401 || \
+					 DISPLAY_VER(__display) >= 20)
+
+/*
+ * CMN SDP TL: Common Secondary Data Packet Transmission Line.
+ *
+ * Xe3p_lpd introduces new register CMN_SDP_TL to program a common SDP
+ * Transmission line that will be used by the Hardware to position the
+ * SDPs. Along with this, another new register CMN_SDP_TL_STGR_CTL is
+ * also added to stagger the different SDPs.
+ */
+#define HAS_COMMON_SDP_TL(__display)      (DISPLAY_VER(__display) >= 35)
+
+u16 intel_dip_read_emp_as_sdp_tl(const struct intel_crtc_state *crtc_state);
+void intel_dip_write_emp_as_sdp_tl(const struct intel_crtc_state *crtc_state);
+
+void intel_dip_sdp_tl_compute_config_late(struct intel_crtc_state *crtc_state);
+void intel_dip_sdp_transmission_line_get_config(struct intel_crtc_state *crtc_state);
+void intel_dip_cmn_sdp_transmission_line_enable(const struct intel_crtc_state *crtc_state);
+void intel_dip_cmn_sdp_transmission_line_disable(const struct intel_crtc_state *old_crtc_state);
+
+#endif /* __INTEL_DIP_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dip_regs.h b/drivers/gpu/drm/i915/display/intel_dip_regs.h
new file mode 100644
index 00000000000000..d7e40a61391357
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dip_regs.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __INTEL_DIP_REGS_H__
+#define __INTEL_DIP_REGS_H__
+
+#include "intel_display_reg_defs.h"
+
+/* EMP (Extended Metadata Packet) AS (Adaptive Sync) SDP Transmission Line */
+#define _EMP_AS_SDP_TL_A			0x60204
+#define EMP_AS_SDP_TL(display, trans)		_MMIO_TRANS2((display), (trans), _EMP_AS_SDP_TL_A)
+#define   EMP_AS_SDP_DB_TL_MASK			REG_GENMASK(12, 0)
+#define   EMP_AS_SDP_DB_TL(db_transmit_line)	REG_FIELD_PREP(EMP_AS_SDP_DB_TL_MASK, \
+							       (db_transmit_line))
+
+/* COMMON SDP TRANSMISSION LINE */
+#define _CMN_SDP_TL_A			0x6020c
+#define CMN_SDP_TL(display, trans)	_MMIO_TRANS2(display, (trans), _CMN_SDP_TL_A)
+#define  TRANSMISSION_LINE_ENABLE	REG_BIT(31)
+#define  BASE_TRANSMISSION_LINE_MASK	REG_GENMASK(12, 0)
+#define  BASE_TRANSMISSION_LINE(x)	REG_FIELD_PREP(BASE_TRANSMISSION_LINE_MASK, x)
+
+#define _CMN_SDP_TL_STGR_CTL_A			0x60214
+#define CMN_SDP_TL_STGR_CTL(display, trans)	_MMIO_TRANS2(display, (trans), \
+							     _CMN_SDP_TL_STGR_CTL_A)
+#define  VSC_EXT_STAGGER_MASK			REG_GENMASK(11, 8)
+#define  VSC_EXT_STAGGER(x)			REG_FIELD_PREP(VSC_EXT_STAGGER_MASK, x)
+#define  VSC_EXT_STAGGER_DEFAULT		0x2
+#define  PPS_STAGGER_MASK			REG_GENMASK(7, 4)
+#define  PPS_STAGGER(x)				REG_FIELD_PREP(PPS_STAGGER_MASK, x)
+#define  PPS_STAGGER_DEFAULT			0x1
+#define  GMP_STAGGER_MASK			REG_GENMASK(3, 0)
+#define  GMP_STAGGER(x)				REG_FIELD_PREP(GMP_STAGGER_MASK, x)
+#define  GMP_STAGGER_DEFAULT			0x0
+
+#endif /* __INTEL_DIP_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 9dfd095f325f7d..d5d4453f283bb0 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -50,7 +50,6 @@
 #include "g4x_dp.h"
 #include "g4x_hdmi.h"
 #include "hsw_ips.h"
-#include "i915_config.h"
 #include "i9xx_plane.h"
 #include "i9xx_plane_regs.h"
 #include "i9xx_wm.h"
@@ -60,7 +59,6 @@
 #include "intel_bo.h"
 #include "intel_bw.h"
 #include "intel_cdclk.h"
-#include "intel_clock_gating.h"
 #include "intel_cmtg.h"
 #include "intel_color.h"
 #include "intel_crt.h"
@@ -71,6 +69,7 @@
 #include "intel_cx0_phy.h"
 #include "intel_ddi.h"
 #include "intel_de.h"
+#include "intel_dip.h"
 #include "intel_display_driver.h"
 #include "intel_display_power.h"
 #include "intel_display_regs.h"
@@ -107,6 +106,7 @@
 #include "intel_modeset_verify.h"
 #include "intel_overlay.h"
 #include "intel_panel.h"
+#include "intel_parent.h"
 #include "intel_pch_display.h"
 #include "intel_pch_refclk.h"
 #include "intel_pfit.h"
@@ -119,7 +119,6 @@
 #include "intel_sdvo.h"
 #include "intel_snps_phy.h"
 #include "intel_tc.h"
-#include "intel_tdf.h"
 #include "intel_tv.h"
 #include "intel_vblank.h"
 #include "intel_vdsc.h"
@@ -193,6 +192,53 @@ is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 		is_trans_port_sync_slave(crtc_state);
 }
 
+/*
+ * Return a bitmask of all the start indices of consecutive bitfields of size
+ * width in mask.
+ */
+static unsigned long find_consecutive_bits(unsigned long mask, int width)
+{
+	unsigned long bit, out_mask = 0;
+
+	if (!width)
+		return 0;
+
+	for_each_set_bit(bit, &mask, BITS_PER_TYPE(mask)) {
+		/* For each set bit, see if the following bits are set also */
+		unsigned long bitfield = GENMASK(bit + width - 1, bit);
+
+		if ((mask & bitfield) == bitfield)
+			out_mask |= BIT(bit);
+	}
+
+	return out_mask;
+}
+
+/*
+ * Return a bitmask of all valid joiner primary pipes for joining
+ * num_joined_pipes pipes. For completeness, return all valid pipes for
+ * num_joined_pipes == 1.
+ *
+ * Return 0 if the platform doesn't support joining for the requested number of
+ * pipes, or there are not enough consecutive pipes available.
+ */
+u8 intel_joiner_valid_primary_pipe_mask(struct intel_display *display, int num_joined_pipes)
+{
+	if (num_joined_pipes == 1) {
+		return DISPLAY_RUNTIME_INFO(display)->pipe_mask;
+	} else if (num_joined_pipes == 2) {
+		if (!HAS_UNCOMPRESSED_JOINER(display) && !HAS_BIGJOINER(display))
+			return 0;
+	} else if (num_joined_pipes == 4) {
+		if (!HAS_ULTRAJOINER(display))
+			return 0;
+	} else {
+		return 0;
+	}
+
+	return find_consecutive_bits(DISPLAY_RUNTIME_INFO(display)->pipe_mask, num_joined_pipes);
+}
+
 static enum pipe joiner_primary_pipe(const struct intel_crtc_state *crtc_state)
 {
 	return ffs(crtc_state->joiner_pipes) - 1;
@@ -950,8 +996,17 @@ static bool vrr_params_changed(const struct intel_crtc_state *old_crtc_state,
 static bool cmrr_params_changed(const struct intel_crtc_state *old_crtc_state,
 				const struct intel_crtc_state *new_crtc_state)
 {
-	return old_crtc_state->cmrr.cmrr_m != new_crtc_state->cmrr.cmrr_m ||
-		old_crtc_state->cmrr.cmrr_n != new_crtc_state->cmrr.cmrr_n;
+	return old_crtc_state->vrr.cmrr.cmrr_m != new_crtc_state->vrr.cmrr.cmrr_m ||
+		old_crtc_state->vrr.cmrr.cmrr_n != new_crtc_state->vrr.cmrr.cmrr_n;
+}
+
+static bool cmn_sdp_changed(const struct intel_crtc_state *old_crtc_state,
+			    const struct intel_crtc_state *new_crtc_state)
+{
+	return old_crtc_state->dip.cmn_sdp_tl != new_crtc_state->dip.cmn_sdp_tl ||
+		old_crtc_state->dip.gmp_sdp_tl != new_crtc_state->dip.gmp_sdp_tl ||
+		old_crtc_state->dip.pps_sdp_tl != new_crtc_state->dip.pps_sdp_tl ||
+		old_crtc_state->dip.vsc_ext_sdp_tl != new_crtc_state->dip.vsc_ext_sdp_tl;
 }
 
 static bool intel_crtc_vrr_enabling(struct intel_atomic_state *state,
@@ -1186,6 +1241,14 @@ static void intel_pre_plane_update(struct intel_atomic_state *state,
 		intel_vrr_disable(old_crtc_state);
 		intel_vrr_dcb_reset(old_crtc_state, crtc);
 		intel_crtc_update_active_timings(old_crtc_state, false);
+
+		/*
+		 * VRR is being disabled seamlessly (no modeset, Panel Replay
+		 * stays enabled), so re-apply the AS SDP skip-frame programming
+		 * for the new (VRR off) state.
+		 */
+		if (!intel_crtc_needs_modeset(new_crtc_state))
+			intel_alpm_pr_as_sdp_update(new_crtc_state);
 	}
 
 	if (audio_disabling(old_crtc_state, new_crtc_state))
@@ -1817,9 +1880,7 @@ static void hsw_crtc_disable(struct intel_atomic_state *state,
 /* Prefer intel_encoder_is_combo() */
 bool intel_phy_is_combo(struct intel_display *display, enum phy phy)
 {
-	if (phy == PHY_NONE)
-		return false;
-	else if (display->platform.alderlake_s)
+	if (display->platform.alderlake_s)
 		return phy <= PHY_E;
 	else if (display->platform.dg1 || display->platform.rocketlake)
 		return phy <= PHY_D;
@@ -1873,7 +1934,7 @@ bool intel_phy_is_snps(struct intel_display *display, enum phy phy)
 	 * For DG2, and for DG2 only, all four "combo" ports and the TC1 port
 	 * (PHY E) use Synopsis PHYs. See intel_phy_is_tc().
 	 */
-	return display->platform.dg2 && phy > PHY_NONE && phy <= PHY_E;
+	return display->platform.dg2 && phy <= PHY_E;
 }
 
 /* Prefer intel_encoder_to_phy() */
@@ -1891,6 +1952,10 @@ enum phy intel_port_to_phy(struct intel_display *display, enum port port)
 		 port == PORT_D)
 		return PHY_A;
 
+	if (drm_WARN(display->drm, port < 0,
+		     "PHY is invalid if port < 0 (%d), assuming PHY_A\n", port))
+		return PHY_A;
+
 	return PHY_A + port - PORT_A;
 }
 
@@ -5536,9 +5601,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 		PIPE_CONF_CHECK_I(vrr.flipline);
 		PIPE_CONF_CHECK_I(vrr.vsync_start);
 		PIPE_CONF_CHECK_I(vrr.vsync_end);
-		PIPE_CONF_CHECK_LLI(cmrr.cmrr_m);
-		PIPE_CONF_CHECK_LLI(cmrr.cmrr_n);
-		PIPE_CONF_CHECK_BOOL(cmrr.enable);
+		PIPE_CONF_CHECK_LLI(vrr.cmrr.cmrr_m);
+		PIPE_CONF_CHECK_LLI(vrr.cmrr.cmrr_n);
+		PIPE_CONF_CHECK_BOOL(vrr.cmrr.enable);
 		PIPE_CONF_CHECK_I(vrr.dc_balance.vmin);
 		PIPE_CONF_CHECK_I(vrr.dc_balance.vmax);
 		PIPE_CONF_CHECK_I(vrr.dc_balance.guardband);
@@ -5546,6 +5611,12 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 		PIPE_CONF_CHECK_I(vrr.dc_balance.max_increase);
 		PIPE_CONF_CHECK_I(vrr.dc_balance.max_decrease);
 		PIPE_CONF_CHECK_I(vrr.dc_balance.vblank_target);
+		PIPE_CONF_CHECK_I(dip.emp_as_sdp_tl);
+		PIPE_CONF_CHECK_I(dip.gmp_sdp_tl);
+		PIPE_CONF_CHECK_I(dip.pps_sdp_tl);
+		PIPE_CONF_CHECK_I(dip.vsc_sdp_tl);
+		PIPE_CONF_CHECK_I(dip.vsc_ext_sdp_tl);
+		PIPE_CONF_CHECK_I(dip.cmn_sdp_tl);
 	}
 
 	if (!fastset || intel_vrr_always_use_vrr_tg(display)) {
@@ -6863,6 +6934,9 @@ static void intel_pre_update_crtc(struct intel_atomic_state *state,
 		if (vrr_params_changed(old_crtc_state, new_crtc_state) ||
 		    cmrr_params_changed(old_crtc_state, new_crtc_state))
 			intel_vrr_set_transcoder_timings(new_crtc_state);
+
+		if (cmn_sdp_changed(old_crtc_state, new_crtc_state))
+			intel_dip_cmn_sdp_transmission_line_enable(new_crtc_state);
 	}
 
 	intel_fbc_update(state, crtc);
@@ -6923,6 +6997,16 @@ static void intel_update_crtc(struct intel_atomic_state *state,
 		intel_crtc_update_active_timings(new_crtc_state,
 						 new_crtc_state->vrr.enable);
 
+	/*
+	 * VRR is being enabled seamlessly (no modeset, Panel Replay stays
+	 * enabled), so re-apply the AS SDP skip-frame programming for the new
+	 * (VRR on) state. Done here, outside the vblank-evasion critical section
+	 * (which runs with interrupts disabled), because it takes alpm.lock.
+	 */
+	if (intel_crtc_vrr_enabling(state, crtc) &&
+	    !intel_crtc_needs_modeset(new_crtc_state))
+		intel_alpm_pr_as_sdp_update(new_crtc_state);
+
 	if (new_crtc_state->vrr.dc_balance.enable)
 		intel_vrr_dcb_increment_flip_count(new_crtc_state, crtc);
 
@@ -7223,23 +7307,27 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 	drm_WARN_ON(display->drm, update_pipes);
 }
 
-static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
+static void intel_atomic_commit_fence_wait(struct intel_atomic_state *state)
 {
+	struct intel_display *display = to_intel_display(state);
 	struct drm_plane *plane;
 	struct drm_plane_state *new_plane_state;
 	long ret;
 	int i;
 
-	for_each_new_plane_in_state(&intel_state->base, plane, new_plane_state, i) {
-		if (new_plane_state->fence) {
-			ret = dma_fence_wait_timeout(new_plane_state->fence, false,
-						     i915_fence_timeout());
-			if (ret <= 0)
-				break;
+	for_each_new_plane_in_state(&state->base, plane, new_plane_state, i) {
+		if (!new_plane_state->fence)
+			continue;
 
-			dma_fence_put(new_plane_state->fence);
-			new_plane_state->fence = NULL;
+		ret = dma_fence_wait(new_plane_state->fence, false);
+		if (ret < 0) {
+			drm_dbg_kms(display->drm, "[PLANE:%d:%s] fence wait failed (%pe)\n",
+				    plane->base.id, plane->name, ERR_PTR(ret));
+			break;
 		}
+
+		dma_fence_put(new_plane_state->fence);
+		new_plane_state->fence = NULL;
 	}
 }
 
@@ -7506,7 +7594,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 
 	intel_atomic_commit_fence_wait(state);
 
-	intel_td_flush(display);
+	intel_parent_transient_data_flush(display);
 
 	intel_atomic_prepare_plane_clear_colors(state);
 
@@ -8113,9 +8201,9 @@ static int max_dotclock(struct intel_display *display)
 {
 	int max_dotclock = display->cdclk.max_dotclk_freq;
 
-	if (HAS_ULTRAJOINER(display))
+	if (intel_joiner_valid_primary_pipe_mask(display, 4))
 		max_dotclock *= 4;
-	else if (HAS_UNCOMPRESSED_JOINER(display) || HAS_BIGJOINER(display))
+	else if (intel_joiner_valid_primary_pipe_mask(display, 2))
 		max_dotclock *= 2;
 
 	return max_dotclock;
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index 57ea4f2edf2a57..eecafff167d39e 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -136,8 +136,6 @@ enum tc_port {
 };
 
 enum phy {
-	PHY_NONE = -1,
-
 	PHY_A = 0,
 	PHY_B,
 	PHY_C,
@@ -373,6 +371,7 @@ intel_cpu_transcoder_mode_valid(struct intel_display *display,
 enum phy intel_port_to_phy(struct intel_display *display, enum port port);
 bool is_trans_port_sync_mode(const struct intel_crtc_state *state);
 bool is_trans_port_sync_master(const struct intel_crtc_state *state);
+u8 intel_joiner_valid_primary_pipe_mask(struct intel_display *display, int num_joined_pipes);
 u8 intel_crtc_joined_pipe_mask(const struct intel_crtc_state *crtc_state);
 bool intel_crtc_is_joiner_secondary(const struct intel_crtc_state *crtc_state);
 bool intel_crtc_is_joiner_primary(const struct intel_crtc_state *crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_display_clock_gating.c b/drivers/gpu/drm/i915/display/intel_display_clock_gating.c
index d036c7e5863d4a..ef1ee72494dfad 100644
--- a/drivers/gpu/drm/i915/display/intel_display_clock_gating.c
+++ b/drivers/gpu/drm/i915/display/intel_display_clock_gating.c
@@ -27,7 +27,7 @@ static void intel_display_gen9_init_clock_gating(struct intel_display *display)
 	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
 }
 
-void intel_display_skl_init_clock_gating(struct intel_display *display)
+static void intel_display_skl_init_clock_gating(struct intel_display *display)
 {
 	/*
 	 * WaCompressedResourceDisplayNewHashMode:skl,kbl
@@ -47,7 +47,7 @@ void intel_display_skl_init_clock_gating(struct intel_display *display)
 	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 }
 
-void intel_display_kbl_init_clock_gating(struct intel_display *display)
+static void intel_display_kbl_init_clock_gating(struct intel_display *display)
 {
 	/*
 	 * WaCompressedResourceDisplayNewHashMode:skl,kbl
@@ -67,7 +67,7 @@ void intel_display_kbl_init_clock_gating(struct intel_display *display)
 	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 }
 
-void intel_display_cfl_init_clock_gating(struct intel_display *display)
+static void intel_display_cfl_init_clock_gating(struct intel_display *display)
 {
 	/*
 	 * WaCompressedResourceDisplayNewHashMode:skl,kbl (and cfl, cml)
@@ -93,7 +93,7 @@ void intel_display_cfl_init_clock_gating(struct intel_display *display)
 	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 }
 
-void intel_display_bxt_init_clock_gating(struct intel_display *display)
+static void intel_display_bxt_init_clock_gating(struct intel_display *display)
 {
 	intel_display_gen9_init_clock_gating(display);
 
@@ -120,7 +120,7 @@ void intel_display_bxt_init_clock_gating(struct intel_display *display)
 	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 }
 
-void intel_display_glk_init_clock_gating(struct intel_display *display)
+static void intel_display_glk_init_clock_gating(struct intel_display *display)
 {
 	intel_display_gen9_init_clock_gating(display);
 
@@ -134,13 +134,13 @@ void intel_display_glk_init_clock_gating(struct intel_display *display)
 		       PWM1_GATING_DIS | PWM2_GATING_DIS);
 }
 
-void intel_display_bdw_clock_gating_disable_fbcq(struct intel_display *display)
+static void intel_display_bdw_clock_gating_disable_fbcq(struct intel_display *display)
 {
 	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 	intel_de_rmw(display, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 }
 
-void intel_display_bdw_clock_gating_vblank_in_srd(struct intel_display *display)
+static void intel_display_bdw_clock_gating_vblank_in_srd(struct intel_display *display)
 {
 	enum pipe pipe;
 
@@ -154,14 +154,21 @@ void intel_display_bdw_clock_gating_vblank_in_srd(struct intel_display *display)
 	}
 }
 
-void intel_display_bdw_clock_gating_kvm_notif(struct intel_display *display)
+static void intel_display_bdw_clock_gating_kvm_notif(struct intel_display *display)
 {
 	/* WaKVMNotificationOnConfigChange:bdw */
 	intel_de_rmw(display, CHICKEN_PAR2_1, 0,
 		     KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 }
 
-void intel_display_hsw_init_clock_gating(struct intel_display *display)
+static void intel_display_bdw_init_clock_gating(struct intel_display *display)
+{
+	intel_display_bdw_clock_gating_disable_fbcq(display);
+	intel_display_bdw_clock_gating_vblank_in_srd(display);
+	intel_display_bdw_clock_gating_kvm_notif(display);
+}
+
+static void intel_display_hsw_init_clock_gating(struct intel_display *display)
 {
 	enum pipe pipe;
 
@@ -178,7 +185,7 @@ void intel_display_hsw_init_clock_gating(struct intel_display *display)
 	}
 }
 
-void intel_display_disable_trickle_feed(struct intel_display *display)
+static void intel_display_disable_trickle_feed(struct intel_display *display)
 {
 	enum pipe pipe;
 
@@ -191,7 +198,7 @@ void intel_display_disable_trickle_feed(struct intel_display *display)
 	}
 }
 
-void intel_display_ilk_init_clock_gating(struct intel_display *display)
+static void intel_display_ilk_init_clock_gating(struct intel_display *display)
 {
 	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
@@ -223,7 +230,7 @@ void intel_display_ilk_init_clock_gating(struct intel_display *display)
 	intel_display_disable_trickle_feed(display);
 }
 
-void intel_display_gen6_init_clock_gating(struct intel_display *display)
+static void intel_display_gen6_init_clock_gating(struct intel_display *display)
 {
 	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
@@ -244,13 +251,15 @@ void intel_display_gen6_init_clock_gating(struct intel_display *display)
 	intel_display_disable_trickle_feed(display);
 }
 
-void intel_display_ivb_init_clock_gating(struct intel_display *display)
+static void intel_display_ivb_init_clock_gating(struct intel_display *display)
 {
 	intel_de_write(display, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 	intel_de_rmw(display, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
+
+	intel_display_disable_trickle_feed(display);
 }
 
-void intel_display_g4x_init_clock_gating(struct intel_display *display)
+static void intel_display_g4x_init_clock_gating(struct intel_display *display)
 {
 	u32 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 			  OVRUNIT_CLOCK_GATE_DISABLE |
@@ -264,7 +273,35 @@ void intel_display_g4x_init_clock_gating(struct intel_display *display)
 	intel_display_disable_trickle_feed(display);
 }
 
-void intel_display_i965gm_init_clock_gating(struct intel_display *display)
+static void intel_display_i965gm_init_clock_gating(struct intel_display *display)
 {
 	intel_de_write(display, DSPCLK_GATE_D, 0);
 }
+
+void intel_display_init_clock_gating(struct intel_display *display)
+{
+	if (display->platform.skylake)
+		intel_display_skl_init_clock_gating(display);
+	else if (display->platform.kabylake)
+		intel_display_kbl_init_clock_gating(display);
+	else if (display->platform.coffeelake || display->platform.cometlake)
+		intel_display_cfl_init_clock_gating(display);
+	else if (display->platform.broxton)
+		intel_display_bxt_init_clock_gating(display);
+	else if (display->platform.geminilake)
+		intel_display_glk_init_clock_gating(display);
+	else if (display->platform.broadwell)
+		intel_display_bdw_init_clock_gating(display);
+	else if (display->platform.haswell)
+		intel_display_hsw_init_clock_gating(display);
+	else if (display->platform.ivybridge)
+		intel_display_ivb_init_clock_gating(display);
+	else if (display->platform.sandybridge)
+		intel_display_gen6_init_clock_gating(display);
+	else if (display->platform.ironlake)
+		intel_display_ilk_init_clock_gating(display);
+	else if (display->platform.g4x)
+		intel_display_g4x_init_clock_gating(display);
+	else if (display->platform.i965gm)
+		intel_display_i965gm_init_clock_gating(display);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_display_clock_gating.h b/drivers/gpu/drm/i915/display/intel_display_clock_gating.h
index b6dd34ca92ddbb..dbfa5892cffead 100644
--- a/drivers/gpu/drm/i915/display/intel_display_clock_gating.h
+++ b/drivers/gpu/drm/i915/display/intel_display_clock_gating.h
@@ -8,20 +8,6 @@
 
 struct intel_display;
 
-void intel_display_skl_init_clock_gating(struct intel_display *display);
-void intel_display_kbl_init_clock_gating(struct intel_display *display);
-void intel_display_cfl_init_clock_gating(struct intel_display *display);
-void intel_display_bxt_init_clock_gating(struct intel_display *display);
-void intel_display_glk_init_clock_gating(struct intel_display *display);
-void intel_display_bdw_clock_gating_disable_fbcq(struct intel_display *display);
-void intel_display_bdw_clock_gating_vblank_in_srd(struct intel_display *display);
-void intel_display_bdw_clock_gating_kvm_notif(struct intel_display *display);
-void intel_display_hsw_init_clock_gating(struct intel_display *display);
-void intel_display_disable_trickle_feed(struct intel_display *display);
-void intel_display_ilk_init_clock_gating(struct intel_display *display);
-void intel_display_gen6_init_clock_gating(struct intel_display *display);
-void intel_display_ivb_init_clock_gating(struct intel_display *display);
-void intel_display_g4x_init_clock_gating(struct intel_display *display);
-void intel_display_i965gm_init_clock_gating(struct intel_display *display);
+void intel_display_init_clock_gating(struct intel_display *display);
 
 #endif /* __INTEL_DISPLAY_CLOCK_GATING_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index 7e988b7b1fe7a5..c80b4a2f74f9e9 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -244,6 +244,13 @@ struct intel_vbt_data {
 	struct list_head display_devices;
 	struct list_head bdb_blocks;
 
+	struct {
+		const u32 *tables;
+		int num_tables;
+		int num_columns;
+		int num_rows;
+	} vspeo;
+
 	struct sdvo_device_mapping {
 		u8 initialized;
 		u8 dvo_port;
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index 3f02868ef105b7..3e302f23f24749 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -49,6 +49,7 @@
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
 #include "intel_vdsc.h"
+#include "intel_vrr.h"
 #include "intel_wm.h"
 #include "intel_tc.h"
 
@@ -1284,22 +1285,11 @@ static ssize_t i915_joiner_write(struct file *file,
 	if (ret < 0)
 		return ret;
 
-	switch (force_joined_pipes) {
-	case 0:
-	case 1:
-	case 2:
-		connector->force_joined_pipes = force_joined_pipes;
-		break;
-	case 4:
-		if (HAS_ULTRAJOINER(display)) {
-			connector->force_joined_pipes = force_joined_pipes;
-			break;
-		}
-
-		fallthrough;
-	default:
+	if (force_joined_pipes &&
+	    !intel_joiner_valid_primary_pipe_mask(display, force_joined_pipes))
 		return -EINVAL;
-	}
+
+	connector->force_joined_pipes = force_joined_pipes;
 
 	*offp += len;
 
@@ -1395,6 +1385,7 @@ void intel_crtc_debugfs_add(struct intel_crtc *crtc)
 	intel_drrs_crtc_debugfs_add(crtc);
 	intel_fbc_crtc_debugfs_add(crtc);
 	hsw_ips_crtc_debugfs_add(crtc);
+	intel_vrr_crtc_debugfs_add(crtc);
 
 	debugfs_create_file("i915_current_bpc", 0444, root, crtc,
 			    &i915_current_bpc_fops);
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.c b/drivers/gpu/drm/i915/display/intel_display_irq.c
index a59b75830bd137..8ce30112de9743 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.c
@@ -2217,8 +2217,10 @@ static void gen11_display_irq_reset(struct intel_display *display)
 	enum pipe pipe;
 	u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
 		BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
+	bool reset_hpd = !intel_display_rpm_pme_enabled(display);
 
-	intel_de_write(display, GEN11_DISPLAY_INT_CTL, 0);
+	if (reset_hpd)
+		intel_de_write(display, GEN11_DISPLAY_INT_CTL, 0);
 
 	if (DISPLAY_VER(display) >= 12) {
 		enum transcoder trans;
@@ -2250,13 +2252,15 @@ static void gen11_display_irq_reset(struct intel_display *display)
 	irq_reset(display, GEN8_DE_PORT_IRQ_REGS);
 	irq_reset(display, GEN8_DE_MISC_IRQ_REGS);
 
-	if (DISPLAY_VER(display) >= 14)
-		irq_reset(display, PICAINTERRUPT_IRQ_REGS);
-	else
-		irq_reset(display, GEN11_DE_HPD_IRQ_REGS);
+	if (reset_hpd) {
+		if (DISPLAY_VER(display) >= 14)
+			irq_reset(display, PICAINTERRUPT_IRQ_REGS);
+		else
+			irq_reset(display, GEN11_DE_HPD_IRQ_REGS);
 
-	if (INTEL_PCH_TYPE(display) >= PCH_ICP)
-		irq_reset(display, SDE_IRQ_REGS);
+		if (INTEL_PCH_TYPE(display) >= PCH_ICP)
+			irq_reset(display, SDE_IRQ_REGS);
+	}
 }
 
 void gen8_irq_power_well_post_enable(struct intel_display *display,
diff --git a/drivers/gpu/drm/i915/display/intel_display_limits.h b/drivers/gpu/drm/i915/display/intel_display_limits.h
index ea89473c177f41..7ba7360c574e03 100644
--- a/drivers/gpu/drm/i915/display/intel_display_limits.h
+++ b/drivers/gpu/drm/i915/display/intel_display_limits.h
@@ -169,6 +169,7 @@ enum aux_ch {
 enum intel_color_block {
 	INTEL_PLANE_CB_PRE_CSC_LUT,
 	INTEL_PLANE_CB_CSC,
+	INTEL_PLANE_CB_CSC_FF,
 	INTEL_PLANE_CB_POST_CSC_LUT,
 	INTEL_PLANE_CB_3DLUT,
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_params.c b/drivers/gpu/drm/i915/display/intel_display_params.c
index 2aed110c5b090b..ca0ef466bb1039 100644
--- a/drivers/gpu/drm/i915/display/intel_display_params.c
+++ b/drivers/gpu/drm/i915/display/intel_display_params.c
@@ -120,6 +120,10 @@ intel_display_param_named_unsafe(enable_psr, int, 0400,
 	"(0=disabled, 1=enable up to PSR1, 2=enable up to PSR2) "
 	"Default: -1 (use per-chip default)");
 
+intel_display_param_named_unsafe(enable_dc_balance, bool, 0400,
+	"Enable VRR DC balance (0=disabled, 1=enabled). "
+	"Default: 0 (disabled)");
+
 intel_display_param_named_unsafe(enable_panel_replay, int, 0400,
 	"Enable Panel Replay (0=disabled, 1=enabled). Default: -1 (use per-chip default)");
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_params.h b/drivers/gpu/drm/i915/display/intel_display_params.h
index b95ecf728daab7..5c5a1a1358c32a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_params.h
+++ b/drivers/gpu/drm/i915/display/intel_display_params.h
@@ -46,10 +46,16 @@ struct drm_printer;
 	param(bool, enable_dp_mst, true, 0600) \
 	param(int, enable_fbc, -1, 0600) \
 	param(int, enable_psr, -1, 0600) \
+	param(bool, enable_dc_balance, false, 0600) \
 	param(int, enable_panel_replay, -1, 0600) \
 	param(bool, psr_safest_params, false, 0400) \
 	param(bool, enable_psr2_sel_fetch, true, 0400) \
 	param(int, enable_dmc_wl, -1, 0400) \
+	/*
+	 * Debugfs-only knob (per-device): no matching module_param is registered
+	 * in intel_display_params.c on purpose. Runtime-toggle via debugfs.
+	 */ \
+	param(bool, enable_periodic_assdp, false, 0600) \
 
 #define MEMBER(T, member, ...) T member;
 struct intel_display_params {
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index 0ebec6e0c24004..2e534ddb73479e 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -5,11 +5,13 @@
 
 #include <linux/iopoll.h>
 #include <linux/string_helpers.h>
+#include <linux/seq_file.h>
 
 #include <drm/drm_print.h>
 #include <drm/intel/intel_pcode_regs.h>
 #include <drm/intel/step.h>
 
+#include "intel_alpm.h"
 #include "intel_backlight_regs.h"
 #include "intel_cdclk.h"
 #include "intel_clock_gating.h"
@@ -488,6 +490,14 @@ void intel_display_power_dc3co_compute(struct intel_atomic_state *state)
 	if (crtc_state->has_sel_update)
 		trigger |= DC3CO_TRIGGER_PSR2;
 
+	/*
+	 * Periodic AS SDP (skip frames) needs the AS SDP to keep flowing during
+	 * PR active, which is incompatible with DC3co. Keep DC3co disabled while
+	 * skip frames is enabled.
+	 */
+	if (intel_alpm_pr_as_sdp_skip_frames_enabled(intel_dp, crtc_state))
+		trigger = DC3CO_TRIGGER_NONE;
+
 done:
 	intel_display_power_dc3co_update(display, trigger);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_map.c b/drivers/gpu/drm/i915/display/intel_display_power_map.c
index 3400080d78d2a3..e18eb49983fe88 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_map.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_map.c
@@ -1759,6 +1759,44 @@ static const struct i915_power_well_desc_list wcl_power_wells[] = {
 	I915_PW_DESCRIPTORS(wcl_power_wells_aux),
 };
 
+I915_DECL_PW_DOMAINS(xe3plpd_pwdoms_dc_off,
+	POWER_DOMAIN_DC_OFF,
+	XE3LPD_PW_2_POWER_DOMAINS,
+	XE3LPD_PW_C_POWER_DOMAINS,
+	XE3LPD_PW_D_POWER_DOMAINS,
+	POWER_DOMAIN_AUDIO_MMIO,
+	POWER_DOMAIN_AUDIO_PLAYBACK,
+	POWER_DOMAIN_AUX_A,
+	POWER_DOMAIN_AUX_B,
+	POWER_DOMAIN_AUX_USBC1,
+	POWER_DOMAIN_AUX_USBC2,
+	POWER_DOMAIN_AUX_USBC3,
+	POWER_DOMAIN_AUX_USBC4,
+	POWER_DOMAIN_AUX_TBT1,
+	POWER_DOMAIN_AUX_TBT2,
+	POWER_DOMAIN_AUX_TBT3,
+	POWER_DOMAIN_AUX_TBT4,
+	POWER_DOMAIN_INIT);
+
+static const struct i915_power_well_desc xe3plpd_power_wells_dcoff[] = {
+	{
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("DC_off", &xe3plpd_pwdoms_dc_off,
+				.id = SKL_DISP_DC_OFF),
+		),
+		.ops = &gen9_dc_off_power_well_ops,
+	},
+};
+
+static const struct i915_power_well_desc_list xe3plpd_power_wells[] = {
+	I915_PW_DESCRIPTORS(i9xx_power_wells_always_on),
+	I915_PW_DESCRIPTORS(icl_power_wells_pw_1),
+	I915_PW_DESCRIPTORS(xe3plpd_power_wells_dcoff),
+	I915_PW_DESCRIPTORS(xe3lpd_power_wells_main),
+	I915_PW_DESCRIPTORS(xe2lpd_power_wells_pica),
+	I915_PW_DESCRIPTORS(xelpdp_power_wells_aux),
+};
+
 static void init_power_well_domains(const struct i915_power_well_instance *inst,
 				    struct i915_power_well *power_well)
 {
@@ -1865,7 +1903,9 @@ int intel_display_power_map_init(struct i915_power_domains *power_domains)
 		return 0;
 	}
 
-	if (DISPLAY_VERx100(display) == 3002)
+	if (DISPLAY_VER(display) >= 35)
+		return set_power_wells(power_domains, xe3plpd_power_wells);
+	else if (DISPLAY_VERx100(display) == 3002)
 		return set_power_wells(power_domains, wcl_power_wells);
 	else if (DISPLAY_VER(display) >= 30)
 		return set_power_wells(power_domains, xe3lpd_power_wells);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index 02cb4d800e2389..233465a8306ec9 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -325,7 +325,11 @@ static enum phy icl_aux_pw_to_phy(struct intel_display *display,
 {
 	struct intel_encoder *encoder = icl_aux_pw_to_encoder(display, power_well);
 
-	return encoder ? intel_encoder_to_phy(encoder) : PHY_NONE;
+	if (drm_WARN(display->drm, !encoder,
+		     "PHY is invalid if encoder is NULL, assuming PHY_A\n"))
+		return PHY_A;
+
+	return intel_encoder_to_phy(encoder);
 }
 
 static bool icl_aux_pw_is_tc_phy(struct intel_display *display,
diff --git a/drivers/gpu/drm/i915/display/intel_display_regs.h b/drivers/gpu/drm/i915/display/intel_display_regs.h
index 0f5018482497be..8251ec15190bc7 100644
--- a/drivers/gpu/drm/i915/display/intel_display_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_regs.h
@@ -2788,7 +2788,7 @@ enum skl_power_gate {
 #define  BXT_CDCLK_CD2X_PIPE_MASK	REG_GENMASK(21, 20) /* bxt/glk */
 #define  BXT_CDCLK_CD2X_PIPE(pipe)	REG_FIELD_PREP(BXT_CDCLK_CD2X_PIPE_MASK, (pipe))
 #define  BXT_CDCLK_CD2X_PIPE_NONE	REG_FIELD_PREP(BXT_CDCLK_CD2X_PIPE_MASK, 3)
-#define  ICL_CDCLK_CD2X_PIPE_MASK	REG_GENMASK(21, 19) /* icl+ */
+#define  ICL_CDCLK_CD2X_PIPE_MASK	REG_GENMASK(21, 19) /* icl-lnl */
 #define  ICL_CDCLK_CD2X_PIPE(pipe)	REG_FIELD_PREP(ICL_CDCLK_CD2X_PIPE_MASK, (pipe) << 1)
 #define  ICL_CDCLK_CD2X_PIPE_NONE	REG_FIELD_PREP(ICL_CDCLK_CD2X_PIPE_MASK, 7)
 #define  CDCLK_DIVMUX_CD_OVERRIDE	REG_BIT(19) /* pre-icl */
@@ -2874,9 +2874,19 @@ enum skl_power_gate {
 #define  ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port)	(1 << ((tc_port) < TC_PORT_4 ? \
 						       (tc_port) + 12 : \
 						       (tc_port) - TC_PORT_4 + 21))
-#define  ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)	((phy) * 2)
-#define  ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy)	(3 << ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy))
-#define  ICL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll, phy)	((pll) << ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy))
+
+/*
+ * ICL_DPCLKA_CFGCR0 has a 2-bit DDI_CLK_SEL field per combo PHY, for
+ * PHY_A..PHY_D only.  Any other phy value (PHY_NONE, TypeC/SNPS PHYs)
+ * is not valid here.
+ */
+#define ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)	_PICK(phy & 0x3, 0, 2, 4, 6)
+#define ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy) \
+	REG_GENMASK(ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy) + 1, \
+		    ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy))
+#define ICL_DPCLKA_CFGCR0_DDI_CLK_SEL(pll, phy) \
+	((pll) << ICL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy))
+
 #define  RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy)	_PICK(phy, 0, 2, 4, 27)
 #define  RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(phy) \
 	(3 << RKL_DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(phy))
diff --git a/drivers/gpu/drm/i915/display/intel_display_rpm.c b/drivers/gpu/drm/i915/display/intel_display_rpm.c
index 0a331f89b4db5c..9927119a0fd19d 100644
--- a/drivers/gpu/drm/i915/display/intel_display_rpm.c
+++ b/drivers/gpu/drm/i915/display/intel_display_rpm.c
@@ -46,6 +46,13 @@ bool intel_display_rpm_suspended(struct intel_display *display)
 	return display->parent->rpm->suspended(display->drm);
 }
 
+bool intel_display_rpm_pme_enabled(struct intel_display *display)
+{
+	const struct intel_display_rpm_interface *rpm = display->parent->rpm;
+
+	return rpm->pme_enabled && rpm->pme_enabled(display->drm);
+}
+
 void assert_display_rpm_held(struct intel_display *display)
 {
 	display->parent->rpm->assert_held(display->drm);
diff --git a/drivers/gpu/drm/i915/display/intel_display_rpm.h b/drivers/gpu/drm/i915/display/intel_display_rpm.h
index 6ef48515f84bbd..5d9a1cdddfa71f 100644
--- a/drivers/gpu/drm/i915/display/intel_display_rpm.h
+++ b/drivers/gpu/drm/i915/display/intel_display_rpm.h
@@ -21,6 +21,7 @@ void intel_display_rpm_put(struct intel_display *display, struct ref_tracker *wa
 
 /* Only for special cases. */
 bool intel_display_rpm_suspended(struct intel_display *display);
+bool intel_display_rpm_pme_enabled(struct intel_display *display);
 
 void assert_display_rpm_held(struct intel_display *display);
 void intel_display_rpm_assert_block(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index 43d53a98dae755..79f30660c2b69b 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -289,6 +289,9 @@ struct intel_encoder {
 	 */
 	enum icl_port_dpll_id (*port_pll_type)(struct intel_encoder *encoder,
 					       const struct intel_crtc_state *crtc_state);
+	const struct intel_ddi_buf_trans *(*get_buf_trans_override)(struct intel_encoder *encoder,
+								    const struct intel_crtc_state *crtc_state,
+								    int *n_entries);
 	const struct intel_ddi_buf_trans *(*get_buf_trans)(struct intel_encoder *encoder,
 							   const struct intel_crtc_state *crtc_state,
 							   int *n_entries);
@@ -683,6 +686,7 @@ struct intel_plane_state {
 		enum drm_color_range color_range;
 		enum drm_scaling_filter scaling_filter;
 		struct drm_property_blob *ctm, *degamma_lut, *gamma_lut, *lut_3d;
+		bool csc_ff_enable;
 	} hw;
 
 	struct i915_vma *ggtt_vma;
@@ -1003,6 +1007,20 @@ struct intel_casf {
 	bool enable;
 };
 
+struct intel_dip {
+	/*
+	 * DIP Transmission line, relative to the Vtotal.
+	 * The programmed transmit line is (Vtotal - value)
+	 */
+	u16 emp_as_sdp_tl;
+	u16 gmp_sdp_tl;
+	u16 pps_sdp_tl;
+	u16 vsc_sdp_tl;
+	u16 vsc_ext_sdp_tl;
+	/* Common SDP Base transmission line (Xe3p_lpd+) */
+	u16 cmn_sdp_tl;
+};
+
 struct intel_crtc_state {
 	/*
 	 * uapi (drm) state. This is the software state shown to userspace.
@@ -1187,6 +1205,8 @@ struct intel_crtc_state {
 	bool has_sel_update;
 	bool enable_psr2_sel_fetch;
 	bool enable_psr2_su_region_et;
+	/* Drop the stale selective fetch enable bits as selective fetch is turned off */
+	bool clear_psr2_sel_fetch;
 	bool req_psr2_sdp_prior_scanline;
 	bool has_panel_replay;
 	bool link_off_after_as_sdp_when_pr_active;
@@ -1309,6 +1329,8 @@ struct intel_crtc_state {
 		struct drm_dp_as_sdp as_sdp;
 	} infoframes;
 
+	struct intel_dip dip;
+
 	u8 eld[MAX_ELD_BYTES];
 
 	/* HDMI scrambling status */
@@ -1408,13 +1430,13 @@ struct intel_crtc_state {
 			u16 max_increase, max_decrease;
 			u16 vblank_target;
 		} dc_balance;
-	} vrr;
 
-	/* Content Match Refresh Rate state */
-	struct {
-		bool enable;
-		u64 cmrr_n, cmrr_m;
-	} cmrr;
+		/* Content Match Refresh Rate state */
+		struct {
+			bool enable;
+			u64 cmrr_n, cmrr_m;
+		} cmrr;
+	} vrr;
 
 	/* Stream Splitter for eDP MSO */
 	struct {
@@ -1553,6 +1575,11 @@ struct intel_crtc {
 		u64 flip_count;
 	} dc_balance;
 
+	struct {
+		u32 numerator;
+		u32 denominator;
+	} force_cmrr;
+
 	int scanline_offset;
 
 	struct {
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.c b/drivers/gpu/drm/i915/display/intel_display_wa.c
index 3662e0f17c69b4..888676693f5eaf 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.c
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.c
@@ -4,6 +4,7 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
 #include <drm/intel/step.h>
 
 #include "intel_de.h"
@@ -131,7 +132,8 @@ bool __intel_display_wa(struct intel_display *display, enum intel_display_wa wa,
 	case INTEL_DISPLAY_WA_16011863758:
 		return DISPLAY_VER(display) >= 11;
 	case INTEL_DISPLAY_WA_16023588340:
-		return intel_display_needs_wa_16023588340(display);
+		/* This is a GT workaround. */
+		return display->parent->wa && display->parent->wa->wa_16023588340(display->drm);
 	case INTEL_DISPLAY_WA_16025573575:
 		return intel_display_needs_wa_16025573575(display);
 	case INTEL_DISPLAY_WA_16025596647:
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.h b/drivers/gpu/drm/i915/display/intel_display_wa.h
index 338b32e4162d49..425a300840aa31 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.h
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.h
@@ -12,15 +12,6 @@ struct intel_display;
 
 void intel_display_wa_apply(struct intel_display *display);
 
-#ifdef I915
-static inline bool intel_display_needs_wa_16023588340(struct intel_display *display)
-{
-	return false;
-}
-#else
-bool intel_display_needs_wa_16023588340(struct intel_display *display);
-#endif
-
 /*
  * This enum lists display workarounds; each entry here must have a
  * corresponding case in __intel_display_wa().  Keep both sorted by lineage
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 3152122e6aefe9..d8c0d9381471d1 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -61,6 +61,8 @@
 #include "intel_cx0_phy.h"
 #include "intel_ddi.h"
 #include "intel_de.h"
+#include "intel_dip.h"
+#include "intel_dip_regs.h"
 #include "intel_display_driver.h"
 #include "intel_display_jiffies.h"
 #include "intel_display_utils.h"
@@ -1274,17 +1276,7 @@ bool intel_dp_can_join(struct intel_dp *intel_dp,
 	if (num_joined_pipes > 1 && !intel_dp_has_joiner(intel_dp))
 		return false;
 
-	switch (num_joined_pipes) {
-	case 1:
-		return true;
-	case 2:
-		return HAS_BIGJOINER(display) ||
-		       HAS_UNCOMPRESSED_JOINER(display);
-	case 4:
-		return HAS_ULTRAJOINER(display);
-	default:
-		return false;
-	}
+	return intel_joiner_valid_primary_pipe_mask(display, num_joined_pipes);
 }
 
 bool intel_dp_dotclk_valid(struct intel_display *display,
@@ -2920,6 +2912,7 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 			     struct drm_connector_state *conn_state,
 			     bool respect_downstream_limits)
 {
+	struct intel_display *display = to_intel_display(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
@@ -2934,6 +2927,10 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 		return -EINVAL;
 
 	for_each_joiner_candidate(connector, adjusted_mode, num_joined_pipes) {
+		/* If the pipe can't be a joiner primary, skip early. */
+		if (!(intel_joiner_valid_primary_pipe_mask(display, num_joined_pipes) & BIT(crtc->pipe)))
+			continue;
+
 		/*
 		 * NOTE:
 		 * The crtc_state->joiner_pipes should have been set at the end
@@ -3125,6 +3122,7 @@ static void intel_dp_compute_as_sdp(struct intel_dp *intel_dp,
 	struct drm_dp_as_sdp *as_sdp = &crtc_state->infoframes.as_sdp;
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->hw.adjusted_mode;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 
 	/*
 	 * #FIXME: SDP/infoframe updates aren’t truly atomic, and with the new
@@ -3142,10 +3140,14 @@ static void intel_dp_compute_as_sdp(struct intel_dp *intel_dp,
 	as_sdp->revision = 0x2;
 	as_sdp->vtotal = intel_vrr_vmin_vtotal(crtc_state);
 
-	if (crtc_state->cmrr.enable) {
+	if (crtc_state->vrr.cmrr.enable) {
 		as_sdp->mode = DP_AS_SDP_FAVT_TRR_REACHED;
 		as_sdp->target_rr = drm_mode_vrefresh(adjusted_mode);
-		as_sdp->target_rr_divider = true;
+
+		if (crtc->force_cmrr.denominator == 1001)
+			as_sdp->target_rr_divider = true;
+		else
+			as_sdp->target_rr_divider = false;
 	} else if (crtc_state->vrr.enable) {
 		as_sdp->mode = DP_AS_SDP_AVT_DYNAMIC_VTOTAL;
 	} else {
@@ -3734,19 +3736,19 @@ static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp)
 static int
 write_dsc_decompression_flag(struct drm_dp_aux *aux, u8 flag, bool set)
 {
-	int err;
+	int ret;
 	u8 val;
 
-	err = drm_dp_dpcd_readb(aux, DP_DSC_ENABLE, &val);
-	if (err < 0)
-		return err;
+	ret = drm_dp_dpcd_read_byte(aux, DP_DSC_ENABLE, &val);
+	if (ret < 0)
+		return ret;
 
 	if (set)
 		val |= flag;
 	else
 		val &= ~flag;
 
-	return drm_dp_dpcd_writeb(aux, DP_DSC_ENABLE, val);
+	return drm_dp_dpcd_write_byte(aux, DP_DSC_ENABLE, val);
 }
 
 static void
@@ -3910,6 +3912,7 @@ intel_dp_init_source_oui(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	u8 oui[] = { 0x00, 0xaa, 0x01 };
 	u8 buf[3] = {};
+	int ret;
 
 	if (READ_ONCE(intel_dp->oui_valid))
 		return;
@@ -3920,8 +3923,9 @@ intel_dp_init_source_oui(struct intel_dp *intel_dp)
 	 * During driver init, we want to be careful and avoid changing the source OUI if it's
 	 * already set to what we want, so as to avoid clearing any state by accident
 	 */
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_SOURCE_OUI, buf, sizeof(buf)) < 0)
-		drm_dbg_kms(display->drm, "Failed to read source OUI\n");
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_SOURCE_OUI, buf, sizeof(buf));
+	if (ret < 0)
+		drm_dbg_kms(display->drm, "Failed to read source OUI (%pe)\n", ERR_PTR(ret));
 
 	if (memcmp(oui, buf, sizeof(oui)) == 0) {
 		/* Assume the OUI was written now. */
@@ -3929,8 +3933,9 @@ intel_dp_init_source_oui(struct intel_dp *intel_dp)
 		return;
 	}
 
-	if (drm_dp_dpcd_write(&intel_dp->aux, DP_SOURCE_OUI, oui, sizeof(oui)) < 0) {
-		drm_dbg_kms(display->drm, "Failed to write source OUI\n");
+	ret = drm_dp_dpcd_write_data(&intel_dp->aux, DP_SOURCE_OUI, oui, sizeof(oui));
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "Failed to write source OUI (%pe)\n", ERR_PTR(ret));
 		WRITE_ONCE(intel_dp->oui_valid, false);
 	}
 
@@ -3971,7 +3976,7 @@ void intel_dp_set_power(struct intel_dp *intel_dp, u8 mode)
 		if (downstream_hpd_needs_d0(intel_dp))
 			return;
 
-		ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, mode);
+		ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER, mode);
 	} else {
 		struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 
@@ -3985,21 +3990,21 @@ void intel_dp_set_power(struct intel_dp *intel_dp, u8 mode)
 		 * time to wake up.
 		 */
 		for (i = 0; i < 3; i++) {
-			ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, mode);
-			if (ret == 1)
+			ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER, mode);
+			if (!ret)
 				break;
 			msleep(1);
 		}
 
-		if (ret == 1 && intel_lspcon_active(dig_port))
+		if (!ret && intel_lspcon_active(dig_port))
 			intel_lspcon_wait_pcon_mode(dig_port);
 	}
 
-	if (ret != 1)
+	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "[ENCODER:%d:%s] Set power to %s failed\n",
+			    "[ENCODER:%d:%s] Set power to %s failed (%pe)\n",
 			    encoder->base.base.id, encoder->base.name,
-			    mode == DP_SET_POWER_D0 ? "D0" : "D3");
+			    mode == DP_SET_POWER_D0 ? "D0" : "D3", ERR_PTR(ret));
 }
 
 static bool
@@ -4086,6 +4091,7 @@ bool intel_dp_initial_fastset_check(struct intel_encoder *encoder,
 static void intel_dp_get_pcon_dsc_cap(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	int ret;
 
 	/* Clear the cached register set to avoid using stale values */
 
@@ -4094,11 +4100,12 @@ static void intel_dp_get_pcon_dsc_cap(struct intel_dp *intel_dp)
 	if (!drm_dp_is_branch(intel_dp->dpcd))
 		return;
 
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_PCON_DSC_ENCODER,
-			     intel_dp->pcon_dsc_dpcd,
-			     sizeof(intel_dp->pcon_dsc_dpcd)) < 0)
-		drm_err(display->drm, "Failed to read DPCD register 0x%x\n",
-			DP_PCON_DSC_ENCODER);
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_PCON_DSC_ENCODER,
+				    intel_dp->pcon_dsc_dpcd,
+				    sizeof(intel_dp->pcon_dsc_dpcd));
+	if (ret < 0)
+		drm_err(display->drm, "Failed to read DPCD register 0x%x (%pe)\n",
+			DP_PCON_DSC_ENCODER, ERR_PTR(ret));
 
 	drm_dbg_kms(display->drm, "PCON ENCODER DSC DPCD: %*ph\n",
 		    (int)sizeof(intel_dp->pcon_dsc_dpcd), intel_dp->pcon_dsc_dpcd);
@@ -4139,14 +4146,9 @@ static int intel_dp_pcon_set_frl_mask(int max_frl)
 static int intel_dp_hdmi_sink_max_frl(struct intel_dp *intel_dp)
 {
 	struct intel_connector *connector = intel_dp->attached_connector;
-	const struct drm_display_info *info = &connector->base.display_info;
-	int max_frl_rate;
-	int max_lanes, rate_per_lane;
-	int max_dsc_lanes, dsc_rate_per_lane;
-
-	max_lanes = info->hdmi.max_lanes;
-	rate_per_lane = info->hdmi.max_frl_rate_per_lane;
-	max_frl_rate = max_lanes * rate_per_lane;
+	struct drm_connector *drm_connector = &connector->base;
+	int max_frl_rate = intel_hdmi_sink_max_frl_rate(drm_connector);
+	int dsc_max_frl_rate = intel_hdmi_sink_dsc_max_frl_rate(drm_connector);
 
 	/*
 	 * The sink's DSC max FRL rate only applies to compressed video
@@ -4155,12 +4157,8 @@ static int intel_dp_hdmi_sink_max_frl(struct intel_dp *intel_dp)
 	 * the regular max FRL rate is the limit.
 	 */
 	if (drm_dp_pcon_enc_is_dsc_1_2(intel_dp->pcon_dsc_dpcd) &&
-	    info->hdmi.dsc_cap.v_1p2) {
-		max_dsc_lanes = info->hdmi.dsc_cap.max_lanes;
-		dsc_rate_per_lane = info->hdmi.dsc_cap.max_frl_rate_per_lane;
-		if (max_dsc_lanes && dsc_rate_per_lane)
-			max_frl_rate = min(max_frl_rate, max_dsc_lanes * dsc_rate_per_lane);
-	}
+	    dsc_max_frl_rate)
+		return min(max_frl_rate, dsc_max_frl_rate);
 
 	return max_frl_rate;
 }
@@ -4264,13 +4262,13 @@ int intel_dp_pcon_set_tmds_mode(struct intel_dp *intel_dp)
 	/* Set PCON source control mode */
 	buf |= DP_PCON_ENABLE_SOURCE_CTL_MODE;
 
-	ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, buf);
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, buf);
 	if (ret < 0)
 		return ret;
 
 	/* Set HDMI LINK ENABLE */
 	buf |= DP_PCON_ENABLE_HDMI_LINK;
-	ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, buf);
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, buf);
 	if (ret < 0)
 		return ret;
 
@@ -4326,7 +4324,9 @@ intel_dp_pcon_dsc_enc_slices(struct intel_dp *intel_dp,
 	int pcon_max_slices = drm_dp_pcon_dsc_max_slices(intel_dp->pcon_dsc_dpcd);
 	int pcon_max_slice_width = drm_dp_pcon_dsc_max_slice_width(intel_dp->pcon_dsc_dpcd);
 
-	return intel_hdmi_dsc_get_num_slices(crtc_state, pcon_max_slices,
+	return intel_hdmi_dsc_get_num_slices(&crtc_state->hw.adjusted_mode,
+					     crtc_state->output_format,
+					     pcon_max_slices,
 					     pcon_max_slice_width,
 					     hdmi_max_slices, hdmi_throughput);
 }
@@ -4343,9 +4343,10 @@ intel_dp_pcon_dsc_enc_bpp(struct intel_dp *intel_dp,
 	int pcon_fractional_bpp = drm_dp_pcon_dsc_bpp_incr(intel_dp->pcon_dsc_dpcd);
 	int hdmi_max_chunk_bytes =
 		info->hdmi.dsc_cap.total_chunk_kbytes * 1024;
+	int bpc = crtc_state->pipe_bpp / 3;
 
 	return intel_hdmi_dsc_get_bpp(pcon_fractional_bpp, slice_width,
-				      num_slices, output_format, hdmi_all_bpp,
+				      num_slices, output_format, bpc, hdmi_all_bpp,
 				      hdmi_max_chunk_bytes);
 }
 
@@ -4413,6 +4414,7 @@ void intel_dp_configure_protocol_converter(struct intel_dp *intel_dp,
 	bool ycbcr444_to_420 = false;
 	bool rgb_to_ycbcr = false;
 	u8 tmp;
+	int ret;
 
 	if (intel_dp->dpcd[DP_DPCD_REV] < 0x13)
 		return;
@@ -4422,11 +4424,13 @@ void intel_dp_configure_protocol_converter(struct intel_dp *intel_dp,
 
 	tmp = intel_dp_has_hdmi_sink(intel_dp) ? DP_HDMI_DVI_OUTPUT_CONFIG : 0;
 
-	if (drm_dp_dpcd_writeb(&intel_dp->aux,
-			       DP_PROTOCOL_CONVERTER_CONTROL_0, tmp) != 1)
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux,
+				     DP_PROTOCOL_CONVERTER_CONTROL_0, tmp);
+	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Failed to %s protocol converter HDMI mode\n",
-			    str_enable_disable(intel_dp_has_hdmi_sink(intel_dp)));
+			    "Failed to %s protocol converter HDMI mode (%pe)\n",
+			    str_enable_disable(intel_dp_has_hdmi_sink(intel_dp)),
+			    ERR_PTR(ret));
 
 	if (crtc_state->sink_format == INTEL_OUTPUT_FORMAT_YCBCR420) {
 		switch (crtc_state->output_format) {
@@ -4458,26 +4462,29 @@ void intel_dp_configure_protocol_converter(struct intel_dp *intel_dp,
 
 	tmp = ycbcr444_to_420 ? DP_CONVERSION_TO_YCBCR420_ENABLE : 0;
 
-	if (drm_dp_dpcd_writeb(&intel_dp->aux,
-			       DP_PROTOCOL_CONVERTER_CONTROL_1, tmp) != 1)
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux,
+				     DP_PROTOCOL_CONVERTER_CONTROL_1, tmp);
+	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Failed to %s protocol converter YCbCr 4:2:0 conversion mode\n",
-			    str_enable_disable(intel_dp->dfp.ycbcr_444_to_420));
+			    "Failed to %s protocol converter YCbCr 4:2:0 conversion mode (%pe)\n",
+			    str_enable_disable(intel_dp->dfp.ycbcr_444_to_420),
+			    ERR_PTR(ret));
 
 	tmp = rgb_to_ycbcr ? DP_CONVERSION_BT709_RGB_YCBCR_ENABLE : 0;
 
-	if (drm_dp_pcon_convert_rgb_to_ycbcr(&intel_dp->aux, tmp) < 0)
+	ret = drm_dp_pcon_convert_rgb_to_ycbcr(&intel_dp->aux, tmp);
+	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Failed to %s protocol converter RGB->YCbCr conversion mode\n",
-			    str_enable_disable(tmp));
+			    "Failed to %s protocol converter RGB->YCbCr conversion mode (%pe)\n",
+			    str_enable_disable(tmp), ERR_PTR(ret));
 }
 
 static u8 intel_dp_read_dprx_feature_enum(struct intel_dp *intel_dp)
 {
 	u8 dprx = 0;
 
-	drm_dp_dpcd_read_data(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST,
-			      &dprx, sizeof(dprx));
+	drm_dp_dpcd_read_byte(&intel_dp->aux, DP_DPRX_FEATURE_ENUMERATION_LIST, &dprx);
+
 	return dprx;
 }
 
@@ -4496,9 +4503,8 @@ static int intel_dp_read_dsc_dpcd(struct drm_dp_aux *aux,
 {
 	int ret;
 
-	ret = drm_dp_dpcd_read_data(aux, DP_DSC_SUPPORT, dsc_dpcd,
-				    DP_DSC_RECEIVER_CAP_SIZE);
-	if (ret) {
+	ret = drm_dp_dpcd_read_data(aux, DP_DSC_SUPPORT, dsc_dpcd, DP_DSC_RECEIVER_CAP_SIZE);
+	if (ret < 0) {
 		drm_dbg_kms(aux->drm_dev,
 			    "Could not read DSC DPCD register 0x%x Error: %pe\n",
 			    DP_DSC_SUPPORT, ERR_PTR(ret));
@@ -4514,7 +4520,7 @@ static int intel_dp_read_dsc_dpcd(struct drm_dp_aux *aux,
 static void init_dsc_overall_throughput_limits(struct intel_connector *connector, bool is_branch)
 {
 	u8 branch_caps[DP_DSC_BRANCH_CAP_SIZE];
-	int line_width;
+	int line_width, ret;
 
 	connector->dp.dsc_branch_caps.overall_throughput.rgb_yuv444 = INT_MAX;
 	connector->dp.dsc_branch_caps.overall_throughput.yuv422_420 = INT_MAX;
@@ -4523,9 +4529,10 @@ static void init_dsc_overall_throughput_limits(struct intel_connector *connector
 	if (!is_branch)
 		return;
 
-	if (drm_dp_dpcd_read_data(connector->dp.dsc_decompression_aux,
-				  DP_DSC_BRANCH_OVERALL_THROUGHPUT_0, branch_caps,
-				  sizeof(branch_caps)) != 0)
+	ret = drm_dp_dpcd_read_data(connector->dp.dsc_decompression_aux,
+				    DP_DSC_BRANCH_OVERALL_THROUGHPUT_0, branch_caps,
+				    sizeof(branch_caps));
+	if (ret < 0)
 		return;
 
 	connector->dp.dsc_branch_caps.overall_throughput.rgb_yuv444 =
@@ -4543,6 +4550,7 @@ void intel_dp_get_dsc_sink_cap(u8 dpcd_rev,
 			       struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(connector);
+	int ret;
 
 	/*
 	 * Clear the cached register set to avoid using stale values
@@ -4559,13 +4567,16 @@ void intel_dp_get_dsc_sink_cap(u8 dpcd_rev,
 	if (dpcd_rev < DP_DPCD_REV_14)
 		return;
 
-	if (intel_dp_read_dsc_dpcd(connector->dp.dsc_decompression_aux,
-				   connector->dp.dsc_dpcd) < 0)
+	ret = intel_dp_read_dsc_dpcd(connector->dp.dsc_decompression_aux,
+				     connector->dp.dsc_dpcd);
+	if (ret < 0)
 		return;
 
-	if (drm_dp_dpcd_readb(connector->dp.dsc_decompression_aux, DP_FEC_CAPABILITY,
-			      &connector->dp.fec_capability) < 0) {
-		drm_dbg_kms(display->drm, "Could not read FEC DPCD register\n");
+	ret = drm_dp_dpcd_read_byte(connector->dp.dsc_decompression_aux, DP_FEC_CAPABILITY,
+				    &connector->dp.fec_capability);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "Could not read FEC DPCD register (%pe)\n",
+			    ERR_PTR(ret));
 		return;
 	}
 
@@ -4674,13 +4685,15 @@ static void intel_edp_mso_init(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_connector *connector = intel_dp->attached_connector;
 	struct drm_display_info *info = &connector->base.display_info;
+	int ret;
 	u8 mso;
 
 	if (intel_dp->edp_dpcd[0] < DP_EDP_14)
 		return;
 
-	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_MSO_LINK_CAPABILITIES, &mso) != 1) {
-		drm_err(display->drm, "Failed to read MSO cap\n");
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_EDP_MSO_LINK_CAPABILITIES, &mso);
+	if (ret < 0) {
+		drm_err(display->drm, "Failed to read MSO cap (%pe)\n", ERR_PTR(ret));
 		return;
 	}
 
@@ -4785,6 +4798,38 @@ intel_edp_set_sink_rates(struct intel_dp *intel_dp)
 	intel_edp_set_data_override_rates(intel_dp);
 }
 
+static void intel_edp_wake_sink(struct intel_dp *intel_dp)
+{
+	u8 value = 0;
+	int ret;
+
+	/*
+	 * Read the current sink power state. drm_dp_dpcd_read_byte() already
+	 * retries the AUX transaction internally, so a single read suffices.
+	 * First commercial eDP panels are Ver1.0 or 1.1, on which DPCD
+	 * DP_SET_POWER is supported.
+	 */
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_SET_POWER, &value);
+
+	/*
+	 * If the AUX read failed the sink may be asleep and not responding,
+	 * or it read back D3; in either case wake it up to D0.
+	 * In case of AUX read failure which is usually a POR case, the
+	 * remaining bits of register 0x600 is set to '0' on POR. So a bare
+	 * write should be fine.
+	 */
+	if (ret < 0 || value == DP_SET_POWER_D3) {
+		value &= ~DP_SET_POWER_MASK;
+		value |= DP_SET_POWER_D0;
+		drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER,
+				       value);
+		/* After setting to D0 need a min of 1ms to wake (Spec DP2.1 sec 2.3.1.2) */
+		fsleep(1000);
+		drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER,
+				       value);
+	}
+}
+
 static bool
 intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector)
 {
@@ -4795,6 +4840,12 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector
 	/* this function is meant to be called only once */
 	drm_WARN_ON(display->drm, intel_dp->dpcd[DP_DPCD_REV] != 0);
 
+	/*
+	 * Spec DP2.1 Section 3.5.2.16 page 966.
+	 * Also if sink is asleep, this will wake the sink.
+	 */
+	intel_edp_wake_sink(intel_dp);
+
 	if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd) != 0)
 		return false;
 
@@ -4819,9 +4870,9 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector
 	 * method). The display control registers should read zero if they're
 	 * not supported anyway.
 	 */
-	if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV,
-			     intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) ==
-			     sizeof(intel_dp->edp_dpcd)) {
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_EDP_DPCD_REV,
+				    intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd));
+	if (!ret) {
 		drm_dbg_kms(display->drm, "eDP DPCD: %*ph\n",
 			    (int)sizeof(intel_dp->edp_dpcd),
 			    intel_dp->edp_dpcd);
@@ -5050,6 +5101,7 @@ static bool
 intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *esi)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
+	int ret;
 
 	/*
 	 * Display WA for HSD #13013007775: mtl/arl/lnl
@@ -5058,24 +5110,30 @@ intel_dp_get_sink_irq_esi(struct intel_dp *intel_dp, u8 *esi)
 	 * inadvertently.
 	 */
 	if (IS_DISPLAY_VER(display, 14, 20) && !display->platform.battlemage) {
-		if (drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT_ESI, esi, 3) != 3)
+		ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_SINK_COUNT_ESI, esi, 3);
+		if (ret < 0)
 			return false;
 
 		/* DP_SINK_COUNT_ESI + 3 == DP_LINK_SERVICE_IRQ_VECTOR_ESI0 */
-		return drm_dp_dpcd_readb(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0,
-					 &esi[3]) == 1;
+		ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0,
+					    &esi[3]);
+		return ret == 0;
 	}
 
-	return drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_COUNT_ESI, esi, 4) == 4;
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_SINK_COUNT_ESI, esi, 4);
+
+	return ret == 0;
 }
 
 static bool intel_dp_ack_sink_irq_esi(struct intel_dp *intel_dp, u8 esi[4])
 {
 	int retry;
+	int ret;
 
 	for (retry = 0; retry < 3; retry++) {
-		if (drm_dp_dpcd_write(&intel_dp->aux, DP_SINK_COUNT_ESI + 1,
-				      &esi[1], 3) == 3)
+		ret = drm_dp_dpcd_write_data(&intel_dp->aux, DP_SINK_COUNT_ESI + 1,
+					     &esi[1], 3);
+		if (!ret)
 			return true;
 	}
 
@@ -5085,20 +5143,24 @@ static bool intel_dp_ack_sink_irq_esi(struct intel_dp *intel_dp, u8 esi[4])
 /* Return %true if reading the ESI vector succeeded, %false otherwise. */
 static bool intel_dp_get_sink_irq_esi_sst(struct intel_dp *intel_dp, u8 esi[4])
 {
+	int ret;
+
 	memset(esi, 0, 4);
 
 	/*
 	 * TODO: For DP_DPCD_REV >= 0x12 read
 	 * DP_SINK_COUNT_ESI and DP_DEVICE_SERVICE_IRQ_VECTOR_ESI0.
 	 */
-	if (drm_dp_dpcd_read_data(&intel_dp->aux, DP_SINK_COUNT, esi, 2) != 0)
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_SINK_COUNT, esi, 2);
+	if (ret < 0)
 		return false;
 
 	if (intel_dp->dpcd[DP_DPCD_REV] < DP_DPCD_REV_12)
 		return true;
 
 	/* TODO: Read DP_DEVICE_SERVICE_IRQ_VECTOR_ESI1 as well */
-	if (drm_dp_dpcd_read_byte(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0, &esi[3]) != 0)
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0, &esi[3]);
+	if (ret < 0)
 		return false;
 
 	return true;
@@ -5107,18 +5169,22 @@ static bool intel_dp_get_sink_irq_esi_sst(struct intel_dp *intel_dp, u8 esi[4])
 /* Return %true if acking the ESI vector IRQ events succeeded, %false otherwise. */
 static bool intel_dp_ack_sink_irq_esi_sst(struct intel_dp *intel_dp, u8 esi[4])
 {
+	int ret;
+
 	/*
 	 * TODO: For DP_DPCD_REV >= 0x12 write
 	 * DP_DEVICE_SERVICE_IRQ_VECTOR_ESI0
 	 */
-	if (drm_dp_dpcd_write_byte(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR, esi[1]) != 0)
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_DEVICE_SERVICE_IRQ_VECTOR, esi[1]);
+	if (ret < 0)
 		return false;
 
 	if (intel_dp->dpcd[DP_DPCD_REV] < DP_DPCD_REV_12)
 		return true;
 
 	/* TODO: Read DP_DEVICE_SERVICE_IRQ_VECTOR_ESI1 as well */
-	if (drm_dp_dpcd_write_byte(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0, esi[3]) != 0)
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_LINK_SERVICE_IRQ_VECTOR_ESI0, esi[3]);
+	if (ret < 0)
 		return false;
 
 	return true;
@@ -5441,11 +5507,15 @@ static int intel_dp_vsc_sdp_unpack(struct drm_dp_vsc_sdp *vsc,
 		 *   VSC SDP supporting 3D stereo + Panel Replay.
 		 */
 		return 0;
-	} else if (sdp->sdp_header.HB2 == 0x5 && sdp->sdp_header.HB3 == 0x13) {
+	} else if ((sdp->sdp_header.HB2 == 0x5 || sdp->sdp_header.HB2 == 0x7) &&
+			sdp->sdp_header.HB3 == 0x13) {
 		/*
 		 * - HB2 = 0x5, HB3 = 0x13
 		 *   VSC SDP supporting 3D stereo + PSR2 + Pixel Encoding/Colorimetry
 		 *   Format.
+		 * - HB2 = 0x7, HB3 = 0x13
+		 *   VSC SDP supporting 3D stereo + Panel Replay + Pixel Encoding/Colorimetry
+		 *   Format.
 		 */
 		vsc->pixelformat = (sdp->db[16] >> 4) & 0xf;
 		vsc->colorimetry = sdp->db[16] & 0xf;
@@ -5712,14 +5782,17 @@ intel_dp_handle_hdmi_link_status_change(struct intel_dp *intel_dp)
 {
 	bool is_active;
 	u8 buf = 0;
+	int ret;
 
 	is_active = drm_dp_pcon_hdmi_link_active(&intel_dp->aux);
 	if (intel_dp->frl.is_trained && !is_active) {
-		if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, &buf) < 0)
+		ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, &buf);
+		if (ret < 0)
 			return;
 
 		buf &=  ~DP_PCON_ENABLE_HDMI_LINK;
-		if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, buf) < 0)
+		ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_PCON_HDMI_LINK_CONFIG_1, buf);
+		if (ret < 0)
 			return;
 
 		drm_dp_pcon_hdmi_frl_link_error_count(&intel_dp->aux, &intel_dp->attached_connector->base);
@@ -6224,6 +6297,7 @@ static bool
 intel_dp_sink_supports_as_sdp_v2(struct intel_dp *intel_dp)
 {
 	u8 rx_features;
+	int ret;
 
 	/*
 	 * The DP spec does not explicitly provide the AS SDP v2 capability.
@@ -6246,9 +6320,10 @@ intel_dp_sink_supports_as_sdp_v2(struct intel_dp *intel_dp)
 	 * support from Display ID.
 	 */
 
-	if (drm_dp_dpcd_read_byte(&intel_dp->aux,
-				  DP_DPRX_FEATURE_ENUMERATION_LIST_CONT_1,
-				  &rx_features) == 1) {
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux,
+				    DP_DPRX_FEATURE_ENUMERATION_LIST_CONT_1,
+				    &rx_features);
+	if (!ret) {
 		if (rx_features & DP_AS_SDP_FAVT_PAYLOAD_FIELDS_PARSING_SUPPORTED)
 			return true;
 	}
@@ -7330,6 +7405,8 @@ int intel_dp_sdp_compute_config_late(struct intel_crtc_state *crtc_state)
 		return -EINVAL;
 	}
 
+	intel_dip_sdp_tl_compute_config_late(crtc_state);
+
 	return 0;
 }
 
@@ -7351,9 +7428,58 @@ int intel_dp_compute_config_late(struct intel_encoder *encoder,
 	return 0;
 }
 
+static
+int intel_dp_get_lines_for_cmn_sdp_tl(u32 type)
+{
+	u32 stagger_val;
+
+	/*
+	 * Since we are using default stagger values similar to the case
+	 * where CMN SDP TL is not set, the different SDP transmission
+	 * lines are:
+	 * base : 2nd line of delayed vblank:
+	 * GMP : 2 + GMP_STAGGER
+	 * VSC_EXT: 2 + VSC_EXT_STAGGER
+	 * VSC : 2
+	 * PPS : 2 + PPS_STAGGER
+	 *
+	 * SDP Setup = 1 + MAX(GMP, VSC_EXT, VSC, PPS setup lines)
+	 *
+	 * For EMP_AS_SDP_TL guardband should be more than vrr.vsync_start.
+	 */
+
+	switch (type) {
+	case DP_SDP_VSC_EXT_VESA:
+	case DP_SDP_VSC_EXT_CEA:
+		stagger_val = VSC_EXT_STAGGER_DEFAULT;
+		break;
+	case HDMI_PACKET_TYPE_GAMUT_METADATA:
+		stagger_val = GMP_STAGGER_DEFAULT;
+		break;
+	case DP_SDP_PPS:
+		stagger_val = PPS_STAGGER_DEFAULT;
+		break;
+	case DP_SDP_VSC:
+		stagger_val = 0;
+		break;
+	default:
+		return 0;
+	}
+
+	return 1 + 2 + stagger_val;
+}
+
 static
 int intel_dp_get_lines_for_sdp(const struct intel_crtc_state *crtc_state, u32 type)
 {
+	struct intel_display *display = to_intel_display(crtc_state);
+
+	if (type == DP_SDP_ADAPTIVE_SYNC)
+		return crtc_state->vrr.vsync_start + 1;
+
+	if (HAS_COMMON_SDP_TL(display))
+		return intel_dp_get_lines_for_cmn_sdp_tl(type);
+
 	switch (type) {
 	case DP_SDP_VSC_EXT_VESA:
 	case DP_SDP_VSC_EXT_CEA:
@@ -7362,8 +7488,8 @@ int intel_dp_get_lines_for_sdp(const struct intel_crtc_state *crtc_state, u32 ty
 		return 8;
 	case DP_SDP_PPS:
 		return 7;
-	case DP_SDP_ADAPTIVE_SYNC:
-		return crtc_state->vrr.vsync_start + 1;
+	case DP_SDP_VSC:
+		return 3;
 	default:
 		break;
 	}
@@ -7393,6 +7519,11 @@ int intel_dp_sdp_min_guardband(const struct intel_crtc_state *crtc_state,
 		sdp_guardband = max(sdp_guardband,
 				    intel_dp_get_lines_for_sdp(crtc_state, DP_SDP_ADAPTIVE_SYNC));
 
+	if (crtc_state->infoframes.enable &
+	    intel_hdmi_infoframe_enable(DP_SDP_VSC))
+		sdp_guardband = max(sdp_guardband,
+				    intel_dp_get_lines_for_sdp(crtc_state, DP_SDP_VSC));
+
 	return sdp_guardband;
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
index 266e042e00237c..36aebc32971b77 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
@@ -120,8 +120,8 @@ intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
 
 	intel_dp_wait_source_oui(intel_dp);
 
-	ret = drm_dp_dpcd_read(aux, INTEL_EDP_HDR_TCON_CAP0, tcon_cap, sizeof(tcon_cap));
-	if (ret != sizeof(tcon_cap))
+	ret = drm_dp_dpcd_read_data(aux, INTEL_EDP_HDR_TCON_CAP0, tcon_cap, sizeof(tcon_cap));
+	if (ret < 0)
 		return false;
 
 	drm_dbg_kms(display->drm,
@@ -178,11 +178,13 @@ intel_dp_aux_hdr_get_backlight(struct intel_connector *connector, enum pipe pipe
 	struct intel_dp *intel_dp = enc_to_intel_dp(connector->encoder);
 	u8 tmp;
 	u8 buf[2] = {};
+	int ret;
 
-	if (drm_dp_dpcd_readb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, &tmp) != 1) {
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, &tmp);
+	if (ret < 0) {
 		drm_err(display->drm,
-			"[CONNECTOR:%d:%s] Failed to read current backlight mode from DPCD\n",
-			connector->base.base.id, connector->base.name);
+			"[CONNECTOR:%d:%s] Failed to read current backlight mode from DPCD (%pe)\n",
+			connector->base.base.id, connector->base.name, ERR_PTR(ret));
 		return 0;
 	}
 
@@ -197,11 +199,12 @@ intel_dp_aux_hdr_get_backlight(struct intel_connector *connector, enum pipe pipe
 		return panel->backlight.max;
 	}
 
-	if (drm_dp_dpcd_read(&intel_dp->aux, INTEL_EDP_BRIGHTNESS_NITS_LSB, buf,
-			     sizeof(buf)) != sizeof(buf)) {
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, INTEL_EDP_BRIGHTNESS_NITS_LSB, buf,
+				    sizeof(buf));
+	if (ret < 0) {
 		drm_err(display->drm,
-			"[CONNECTOR:%d:%s] Failed to read brightness from DPCD\n",
-			connector->base.base.id, connector->base.name);
+			"[CONNECTOR:%d:%s] Failed to read brightness from DPCD (%pe)\n",
+			connector->base.base.id, connector->base.name, ERR_PTR(ret));
 		return 0;
 	}
 
@@ -215,14 +218,16 @@ intel_dp_aux_hdr_set_aux_backlight(const struct drm_connector_state *conn_state,
 	struct drm_device *dev = connector->base.dev;
 	struct intel_dp *intel_dp = enc_to_intel_dp(connector->encoder);
 	u8 buf[4] = {};
+	int ret;
 
 	buf[0] = level & 0xFF;
 	buf[1] = (level & 0xFF00) >> 8;
 
-	if (drm_dp_dpcd_write(&intel_dp->aux, INTEL_EDP_BRIGHTNESS_NITS_LSB, buf,
-			      sizeof(buf)) != sizeof(buf))
-		drm_err(dev, "[CONNECTOR:%d:%s] Failed to write brightness level to DPCD\n",
-			connector->base.base.id, connector->base.name);
+	ret = drm_dp_dpcd_write_data(&intel_dp->aux, INTEL_EDP_BRIGHTNESS_NITS_LSB, buf,
+				     sizeof(buf));
+	if (ret < 0)
+		drm_err(dev, "[CONNECTOR:%d:%s] Failed to write brightness level to DPCD (%pe)\n",
+			connector->base.base.id, connector->base.name, ERR_PTR(ret));
 }
 
 static void
@@ -258,13 +263,12 @@ intel_dp_aux_write_content_luminance(struct intel_connector *connector,
 	buf[2] = hdr_metadata->hdmi_metadata_type1.max_fall & 0xFF;
 	buf[3] = (hdr_metadata->hdmi_metadata_type1.max_fall & 0xFF00) >> 8;
 
-	ret = drm_dp_dpcd_write(&intel_dp->aux,
-				INTEL_EDP_HDR_CONTENT_LUMINANCE,
-				buf, sizeof(buf));
+	ret = drm_dp_dpcd_write_data(&intel_dp->aux,
+				     INTEL_EDP_HDR_CONTENT_LUMINANCE,
+				     buf, sizeof(buf));
 	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Content Luminance DPCD reg write failed, err:-%d\n",
-			    ret);
+			    "Content Luminance DPCD reg write failed (%pe)\n", ERR_PTR(ret));
 }
 
 static void
@@ -313,11 +317,11 @@ intel_dp_aux_hdr_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	intel_dp_wait_source_oui(intel_dp);
 
-	ret = drm_dp_dpcd_readb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, &old_ctrl);
-	if (ret != 1) {
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, &old_ctrl);
+	if (ret < 0) {
 		drm_err(display->drm,
-			"[CONNECTOR:%d:%s] Failed to read current backlight control mode: %d\n",
-			connector->base.base.id, connector->base.name, ret);
+			"[CONNECTOR:%d:%s] Failed to read current backlight control mode (%pe)\n",
+			connector->base.base.id, connector->base.name, ERR_PTR(ret));
 		return;
 	}
 
@@ -337,11 +341,14 @@ intel_dp_aux_hdr_enable_backlight(const struct intel_crtc_state *crtc_state,
 
 	intel_dp_aux_fill_hdr_tcon_params(conn_state, &ctrl);
 
-	if (ctrl != old_ctrl &&
-	    drm_dp_dpcd_writeb(&intel_dp->aux, INTEL_EDP_HDR_GETSET_CTRL_PARAMS, ctrl) != 1)
-		drm_err(display->drm,
-			"[CONNECTOR:%d:%s] Failed to configure DPCD brightness controls\n",
-			connector->base.base.id, connector->base.name);
+	if (ctrl != old_ctrl) {
+		ret = drm_dp_dpcd_write_byte(&intel_dp->aux,
+					     INTEL_EDP_HDR_GETSET_CTRL_PARAMS, ctrl);
+		if (ret < 0)
+			drm_err(display->drm,
+				"[CONNECTOR:%d:%s] Failed to configure DPCD brightness controls (%pe)\n",
+				connector->base.base.id, connector->base.name, ERR_PTR(ret));
+	}
 
 	if (intel_dp_in_hdr_mode(conn_state)) {
 		hdr_metadata = conn_state->hdr_output_metadata->data;
@@ -392,13 +399,12 @@ intel_dp_aux_write_panel_luminance_override(struct intel_connector *connector)
 	buf[2] = panel->backlight.max & 0xFF;
 	buf[3] = (panel->backlight.max & 0xFF00) >> 8;
 
-	ret = drm_dp_dpcd_write(&intel_dp->aux,
-				INTEL_EDP_HDR_PANEL_LUMINANCE_OVERRIDE,
-				buf, sizeof(buf));
+	ret = drm_dp_dpcd_write_data(&intel_dp->aux,
+				     INTEL_EDP_HDR_PANEL_LUMINANCE_OVERRIDE,
+				     buf, sizeof(buf));
 	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Panel Luminance DPCD reg write failed, err:-%d\n",
-			    ret);
+			    "Panel Luminance DPCD reg write failed (%pe)\n", ERR_PTR(ret));
 }
 
 static int
@@ -456,12 +462,12 @@ static u32 intel_dp_aux_vesa_get_backlight(struct intel_connector *connector, en
 	int ret;
 
 	if (panel->backlight.edp.vesa.luminance_control_support) {
-		ret = drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_PANEL_TARGET_LUMINANCE_VALUE, buf,
-				       sizeof(buf));
+		ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_EDP_PANEL_TARGET_LUMINANCE_VALUE, buf,
+					    sizeof(buf));
 		if (ret < 0) {
 			drm_err(intel_dp->aux.drm_dev,
-				"[CONNECTOR:%d:%s] Failed to read Luminance from DPCD\n",
-				connector->base.base.id, connector->base.name);
+				"[CONNECTOR:%d:%s] Failed to read Luminance from DPCD (%pe)\n",
+				connector->base.base.id, connector->base.name, ERR_PTR(ret));
 			return 0;
 		}
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
index 14ed0ea22dd30f..566a921d18a157 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
@@ -60,16 +60,15 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *dig_port,
 {
 	struct intel_display *display = to_intel_display(dig_port);
 	u8 aksv[DRM_HDCP_KSV_LEN] = {};
-	ssize_t dpcd_ret;
+	int ret;
 
 	/* Output An first, that's easy */
-	dpcd_ret = drm_dp_dpcd_write(&dig_port->dp.aux, DP_AUX_HDCP_AN,
+	ret = drm_dp_dpcd_write_data(&dig_port->dp.aux, DP_AUX_HDCP_AN,
 				     an, DRM_HDCP_AN_LEN);
-	if (dpcd_ret != DRM_HDCP_AN_LEN) {
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Failed to write An over DP/AUX (%zd)\n",
-			    dpcd_ret);
-		return dpcd_ret >= 0 ? -EIO : dpcd_ret;
+			    "Failed to write An over DP/AUX (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
 
 	/*
@@ -79,14 +78,14 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *dig_port,
 	 * the destination address which will tickle the hardware to output the
 	 * Aksv on our behalf after the header is sent.
 	 */
-	dpcd_ret = drm_dp_dpcd_write(&dig_port->dp.aux, DP_AUX_HDCP_AKSV,
+	ret = drm_dp_dpcd_write_data(&dig_port->dp.aux, DP_AUX_HDCP_AKSV,
 				     aksv, DRM_HDCP_KSV_LEN);
-	if (dpcd_ret != DRM_HDCP_KSV_LEN) {
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Failed to write Aksv over DP/AUX (%zd)\n",
-			    dpcd_ret);
-		return dpcd_ret >= 0 ? -EIO : dpcd_ret;
+			    "Failed to write Aksv over DP/AUX (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
+
 	return 0;
 }
 
@@ -94,15 +93,16 @@ static int intel_dp_hdcp_read_bksv(struct intel_digital_port *dig_port,
 				   u8 *bksv)
 {
 	struct intel_display *display = to_intel_display(dig_port);
-	ssize_t ret;
+	int ret;
 
-	ret = drm_dp_dpcd_read(&dig_port->dp.aux, DP_AUX_HDCP_BKSV, bksv,
-			       DRM_HDCP_KSV_LEN);
-	if (ret != DRM_HDCP_KSV_LEN) {
+	ret = drm_dp_dpcd_read_data(&dig_port->dp.aux, DP_AUX_HDCP_BKSV, bksv,
+				    DRM_HDCP_KSV_LEN);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read Bksv from DP/AUX failed (%zd)\n", ret);
-		return ret >= 0 ? -EIO : ret;
+			    "Read Bksv from DP/AUX failed (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
+
 	return 0;
 }
 
@@ -110,20 +110,21 @@ static int intel_dp_hdcp_read_bstatus(struct intel_digital_port *dig_port,
 				      u8 *bstatus)
 {
 	struct intel_display *display = to_intel_display(dig_port);
-	ssize_t ret;
+	int ret;
 
 	/*
 	 * For some reason the HDMI and DP HDCP specs call this register
 	 * definition by different names. In the HDMI spec, it's called BSTATUS,
 	 * but in DP it's called BINFO.
 	 */
-	ret = drm_dp_dpcd_read(&dig_port->dp.aux, DP_AUX_HDCP_BINFO,
-			       bstatus, DRM_HDCP_BSTATUS_LEN);
-	if (ret != DRM_HDCP_BSTATUS_LEN) {
+	ret = drm_dp_dpcd_read_data(&dig_port->dp.aux, DP_AUX_HDCP_BINFO,
+				    bstatus, DRM_HDCP_BSTATUS_LEN);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read bstatus from DP/AUX failed (%zd)\n", ret);
-		return ret >= 0 ? -EIO : ret;
+			    "Read bstatus from DP/AUX failed (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
+
 	return 0;
 }
 
@@ -132,14 +133,13 @@ int intel_dp_hdcp_read_bcaps(struct drm_dp_aux *aux,
 			     struct intel_display *display,
 			     u8 *bcaps)
 {
-	ssize_t ret;
+	int ret;
 
-	ret = drm_dp_dpcd_read(aux, DP_AUX_HDCP_BCAPS,
-			       bcaps, 1);
-	if (ret != 1) {
+	ret = drm_dp_dpcd_read_byte(aux, DP_AUX_HDCP_BCAPS, bcaps);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read bcaps from DP/AUX failed (%zd)\n", ret);
-		return ret >= 0 ? -EIO : ret;
+			    "Read bcaps from DP/AUX failed (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
 
 	return 0;
@@ -166,16 +166,16 @@ int intel_dp_hdcp_read_ri_prime(struct intel_digital_port *dig_port,
 				u8 *ri_prime)
 {
 	struct intel_display *display = to_intel_display(dig_port);
-	ssize_t ret;
+	int ret;
 
-	ret = drm_dp_dpcd_read(&dig_port->dp.aux, DP_AUX_HDCP_RI_PRIME,
-			       ri_prime, DRM_HDCP_RI_LEN);
-	if (ret != DRM_HDCP_RI_LEN) {
+	ret = drm_dp_dpcd_read_data(&dig_port->dp.aux, DP_AUX_HDCP_RI_PRIME,
+				    ri_prime, DRM_HDCP_RI_LEN);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read Ri' from DP/AUX failed (%zd)\n",
-			    ret);
-		return ret >= 0 ? -EIO : ret;
+			    "Read Ri' from DP/AUX failed (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
+
 	return 0;
 }
 
@@ -184,17 +184,18 @@ int intel_dp_hdcp_read_ksv_ready(struct intel_digital_port *dig_port,
 				 bool *ksv_ready)
 {
 	struct intel_display *display = to_intel_display(dig_port);
-	ssize_t ret;
 	u8 bstatus;
+	int ret;
 
-	ret = drm_dp_dpcd_read(&dig_port->dp.aux, DP_AUX_HDCP_BSTATUS,
-			       &bstatus, 1);
-	if (ret != 1) {
+	ret = drm_dp_dpcd_read_byte(&dig_port->dp.aux, DP_AUX_HDCP_BSTATUS,
+				    &bstatus);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read bstatus from DP/AUX failed (%zd)\n", ret);
-		return ret >= 0 ? -EIO : ret;
+			    "Read bstatus from DP/AUX failed (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
 	*ksv_ready = bstatus & DP_BSTATUS_READY;
+
 	return 0;
 }
 
@@ -203,23 +204,24 @@ int intel_dp_hdcp_read_ksv_fifo(struct intel_digital_port *dig_port,
 				int num_downstream, u8 *ksv_fifo)
 {
 	struct intel_display *display = to_intel_display(dig_port);
-	ssize_t ret;
+	int ret;
 	int i;
 
 	/* KSV list is read via 15 byte window (3 entries @ 5 bytes each) */
 	for (i = 0; i < num_downstream; i += 3) {
 		size_t len = min(num_downstream - i, 3) * DRM_HDCP_KSV_LEN;
-		ret = drm_dp_dpcd_read(&dig_port->dp.aux,
-				       DP_AUX_HDCP_KSV_FIFO,
-				       ksv_fifo + i * DRM_HDCP_KSV_LEN,
-				       len);
-		if (ret != len) {
+		ret = drm_dp_dpcd_read_data(&dig_port->dp.aux,
+					    DP_AUX_HDCP_KSV_FIFO,
+					    ksv_fifo + i * DRM_HDCP_KSV_LEN,
+					    len);
+		if (ret < 0) {
 			drm_dbg_kms(display->drm,
-				    "Read ksv[%d] from DP/AUX failed (%zd)\n",
-				    i, ret);
-			return ret >= 0 ? -EIO : ret;
+				    "Read ksv[%d] from DP/AUX failed (%pe)\n",
+				    i, ERR_PTR(ret));
+			return ret;
 		}
 	}
+
 	return 0;
 }
 
@@ -228,19 +230,19 @@ int intel_dp_hdcp_read_v_prime_part(struct intel_digital_port *dig_port,
 				    int i, u32 *part)
 {
 	struct intel_display *display = to_intel_display(dig_port);
-	ssize_t ret;
+	int ret;
 
 	if (i >= DRM_HDCP_V_PRIME_NUM_PARTS)
 		return -EINVAL;
 
-	ret = drm_dp_dpcd_read(&dig_port->dp.aux,
-			       DP_AUX_HDCP_V_PRIME(i), part,
-			       DRM_HDCP_V_PRIME_PART_LEN);
-	if (ret != DRM_HDCP_V_PRIME_PART_LEN) {
+	ret = drm_dp_dpcd_read_data(&dig_port->dp.aux, DP_AUX_HDCP_V_PRIME(i), part,
+				    DRM_HDCP_V_PRIME_PART_LEN);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read v'[%d] from DP/AUX failed (%zd)\n", i, ret);
-		return ret >= 0 ? -EIO : ret;
+			    "Read v'[%d] from DP/AUX failed (%pe)\n", i, ERR_PTR(ret));
+		return ret;
 	}
+
 	return 0;
 }
 
@@ -258,14 +260,13 @@ bool intel_dp_hdcp_check_link(struct intel_digital_port *dig_port,
 			      struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(dig_port);
-	ssize_t ret;
 	u8 bstatus;
+	int ret;
 
-	ret = drm_dp_dpcd_read(&dig_port->dp.aux, DP_AUX_HDCP_BSTATUS,
-			       &bstatus, 1);
-	if (ret != 1) {
+	ret = drm_dp_dpcd_read_byte(&dig_port->dp.aux, DP_AUX_HDCP_BSTATUS, &bstatus);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read bstatus from DP/AUX failed (%zd)\n", ret);
+			    "Read bstatus from DP/AUX failed (%pe)\n", ERR_PTR(ret));
 		return false;
 	}
 
@@ -346,15 +347,14 @@ intel_dp_hdcp2_read_rx_status(struct intel_connector *connector,
 	struct intel_display *display = to_intel_display(connector);
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct drm_dp_aux *aux = &dig_port->dp.aux;
-	ssize_t ret;
+	int ret;
 
-	ret = drm_dp_dpcd_read(aux,
-			       DP_HDCP_2_2_REG_RXSTATUS_OFFSET, rx_status,
-			       HDCP_2_2_DP_RXSTATUS_LEN);
-	if (ret != HDCP_2_2_DP_RXSTATUS_LEN) {
+	ret = drm_dp_dpcd_read_data(aux, DP_HDCP_2_2_REG_RXSTATUS_OFFSET, rx_status,
+				    HDCP_2_2_DP_RXSTATUS_LEN);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Read bstatus from DP/AUX failed (%zd)\n", ret);
-		return ret >= 0 ? -EIO : ret;
+			    "Read bstatus from DP/AUX failed (%pe)\n", ERR_PTR(ret));
+		return ret;
 	}
 
 	return 0;
@@ -474,8 +474,8 @@ int intel_dp_hdcp2_write_msg(struct intel_connector *connector,
 		len = bytes_to_write > DP_AUX_MAX_PAYLOAD_BYTES ?
 				DP_AUX_MAX_PAYLOAD_BYTES : bytes_to_write;
 
-		ret = drm_dp_dpcd_write(aux,
-					offset, (void *)byte, len);
+		/* Note: This may return < len for partial writes. */
+		ret = drm_dp_dpcd_write(aux, offset, byte, len);
 		if (ret < 0)
 			return ret;
 
@@ -487,20 +487,20 @@ int intel_dp_hdcp2_write_msg(struct intel_connector *connector,
 	return size;
 }
 
+/* return number of bytes read on success */
 static
 ssize_t get_receiver_id_list_rx_info(struct intel_connector *connector,
 				     u32 *dev_cnt, u8 *byte)
 {
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
 	struct drm_dp_aux *aux = &dig_port->dp.aux;
-	ssize_t ret;
 	u8 *rx_info = byte;
+	int ret;
 
-	ret = drm_dp_dpcd_read(aux,
-			       DP_HDCP_2_2_REG_RXINFO_OFFSET,
-			       (void *)rx_info, HDCP_2_2_RXINFO_LEN);
-	if (ret != HDCP_2_2_RXINFO_LEN)
-		return ret >= 0 ? -EIO : ret;
+	ret = drm_dp_dpcd_read_data(aux, DP_HDCP_2_2_REG_RXINFO_OFFSET,
+				    rx_info, HDCP_2_2_RXINFO_LEN);
+	if (ret < 0)
+		return ret;
 
 	*dev_cnt = (HDCP_2_2_DEV_COUNT_HI(rx_info[0]) << 4 |
 		   HDCP_2_2_DEV_COUNT_LO(rx_info[1]));
@@ -508,7 +508,7 @@ ssize_t get_receiver_id_list_rx_info(struct intel_connector *connector,
 	if (*dev_cnt > HDCP_2_2_MAX_DEVICE_COUNT)
 		*dev_cnt = HDCP_2_2_MAX_DEVICE_COUNT;
 
-	return ret;
+	return HDCP_2_2_RXINFO_LEN;
 }
 
 static
@@ -566,11 +566,11 @@ int intel_dp_hdcp2_read_msg(struct intel_connector *connector,
 					       hdcp2_msg_data->msg_read_timeout);
 		}
 
-		ret = drm_dp_dpcd_read(aux, offset,
-				       (void *)byte, len);
+		/* Note: This may return < len for partial reads. */
+		ret = drm_dp_dpcd_read(aux, offset, byte, len);
 		if (ret < 0) {
-			drm_dbg_kms(display->drm, "msg_id %d, ret %zd\n",
-				    msg_id, ret);
+			drm_dbg_kms(display->drm, "msg_id %d, ret %pe\n",
+				    msg_id, ERR_PTR(ret));
 			return ret;
 		}
 
@@ -660,11 +660,11 @@ int _intel_dp_hdcp2_get_capability(struct drm_dp_aux *aux,
 	 * declare a monitor not capable of HDCP 2.2.
 	 */
 	for (i = 0; i < 3; i++) {
-		ret = drm_dp_dpcd_read(aux,
-				       DP_HDCP_2_2_REG_RX_CAPS_OFFSET,
-				       rx_caps, HDCP_2_2_RXCAPS_LEN);
-		if (ret != HDCP_2_2_RXCAPS_LEN)
-			return ret >= 0 ? -EIO : ret;
+		ret = drm_dp_dpcd_read_data(aux,
+					    DP_HDCP_2_2_REG_RX_CAPS_OFFSET,
+					    rx_caps, HDCP_2_2_RXCAPS_LEN);
+		if (ret < 0)
+			return ret;
 
 		if (rx_caps[0] == HDCP_2_2_RX_CAPS_VERSION_VAL &&
 		    HDCP_2_2_DP_HDCP_CAPABLE(rx_caps[2])) {
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_caps.c b/drivers/gpu/drm/i915/display/intel_dp_link_caps.c
index 98657aa4d3d580..abec3e2519ca67 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_caps.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_caps.c
@@ -3,6 +3,8 @@
  * Copyright © 2026 Intel Corporation
  */
 
+#include <kunit/visibility.h>
+
 #include <linux/bitops.h>
 #include <linux/debugfs.h>
 #include <linux/log2.h>
@@ -1302,14 +1304,14 @@ void intel_dp_link_caps_cleanup(struct intel_dp_link_caps *link_caps)
 const struct intel_dp_link_caps_test_ops i915_display_dp_link_caps_test_ops = {
 	INTEL_DP_LINK_CAPS_TEST_OPS_INIT
 };
-EXPORT_SYMBOL(i915_display_dp_link_caps_test_ops);
+EXPORT_SYMBOL_IF_KUNIT(i915_display_dp_link_caps_test_ops);
 
 #else
 
 const struct intel_dp_link_caps_test_ops intel_display_dp_link_caps_test_ops = {
 	INTEL_DP_LINK_CAPS_TEST_OPS_INIT
 };
-EXPORT_SYMBOL(intel_display_dp_link_caps_test_ops);
+EXPORT_SYMBOL_IF_KUNIT(intel_display_dp_link_caps_test_ops);
 
 #endif	/* I915 */
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
index cb92cff906146c..ac1aff6edf0528 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
@@ -929,7 +929,7 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
 {
 	int reg = intel_dp_training_pattern_set_reg(intel_dp, dp_phy);
 	u8 buf[sizeof(intel_dp->train_set) + 1];
-	int len;
+	int ret, len;
 
 	intel_dp_program_link_training_pattern(intel_dp, crtc_state,
 					       dp_phy, dp_train_pat);
@@ -939,7 +939,9 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
 	memcpy(buf + 1, intel_dp->train_set, crtc_state->lane_count);
 	len = crtc_state->lane_count + 1;
 
-	return drm_dp_dpcd_write(&intel_dp->aux, reg, buf, len) == len;
+	ret = drm_dp_dpcd_write_data(&intel_dp->aux, reg, buf, len);
+
+	return !ret;
 }
 
 static char dp_training_pattern_name(u8 train_pat)
@@ -1046,10 +1048,10 @@ intel_dp_update_link_train(struct intel_dp *intel_dp,
 
 	intel_dp_set_signal_levels(intel_dp, crtc_state, dp_phy);
 
-	ret = drm_dp_dpcd_write(&intel_dp->aux, reg,
-				intel_dp->train_set, crtc_state->lane_count);
+	ret = drm_dp_dpcd_write_data(&intel_dp->aux, reg,
+				     intel_dp->train_set, crtc_state->lane_count);
 
-	return ret == crtc_state->lane_count;
+	return !ret;
 }
 
 /* 128b/132b */
@@ -1115,7 +1117,7 @@ void intel_dp_link_training_set_mode(struct intel_dp *intel_dp, int link_rate,
 	link_config[0] |= pr_with_as_sdp_enable ? DP_FIXED_VTOTAL_AS_SDP_EN_IN_PR_ACTIVE : 0;
 	link_config[1] = drm_dp_is_uhbr_rate(link_rate) ?
 			 DP_SET_ANSI_128B132B : DP_SET_ANSI_8B10B;
-	drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2);
+	drm_dp_dpcd_write_data(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2);
 }
 
 static bool
@@ -1163,8 +1165,8 @@ void intel_dp_link_training_set_bw(struct intel_dp *intel_dp,
 		/* DP and eDP v1.3 and earlier link bw set method. */
 		u8 link_config[] = { link_bw, lane_count };
 
-		drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config,
-				  ARRAY_SIZE(link_config));
+		drm_dp_dpcd_write_data(&intel_dp->aux, DP_LINK_BW_SET, link_config,
+				       sizeof(link_config));
 	} else {
 		/*
 		 * eDP v1.4 and later link rate set method.
@@ -1175,8 +1177,8 @@ void intel_dp_link_training_set_bw(struct intel_dp *intel_dp,
 		 * eDP v1.5 sinks allow choosing either, and the last choice
 		 * shall be active.
 		 */
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_LANE_COUNT_SET, lane_count);
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_LINK_RATE_SET, rate_select);
+		drm_dp_dpcd_write_byte(&intel_dp->aux, DP_LANE_COUNT_SET, lane_count);
+		drm_dp_dpcd_write_byte(&intel_dp->aux, DP_LINK_RATE_SET, rate_select);
 	}
 }
 
@@ -1286,8 +1288,8 @@ intel_dp_prepare_link_train(struct intel_dp *intel_dp,
 
 		lt_dbg(intel_dp, DP_PHY_DPRX, "Reloading eDP link rates\n");
 
-		drm_dp_dpcd_read(&intel_dp->aux, DP_SUPPORTED_LINK_RATES,
-				 sink_rates, sizeof(sink_rates));
+		drm_dp_dpcd_read_data(&intel_dp->aux, DP_SUPPORTED_LINK_RATES,
+				      sink_rates, sizeof(sink_rates));
 	}
 
 	if (link_bw)
@@ -1612,7 +1614,7 @@ static void intel_dp_stop_post_lt_adj_req(struct intel_dp *intel_dp,
 	if (crtc_state->enhanced_framing)
 		lane_count |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
 
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_LANE_COUNT_SET, lane_count);
+	drm_dp_dpcd_write_byte(&intel_dp->aux, DP_LANE_COUNT_SET, lane_count);
 }
 
 static bool intel_dp_disable_dpcd_training_pattern(struct intel_dp *intel_dp,
@@ -1621,7 +1623,7 @@ static bool intel_dp_disable_dpcd_training_pattern(struct intel_dp *intel_dp,
 	int reg = intel_dp_training_pattern_set_reg(intel_dp, dp_phy);
 	u8 val = DP_TRAINING_PATTERN_DISABLE;
 
-	return drm_dp_dpcd_write(&intel_dp->aux, reg, &val, 1) == 1;
+	return drm_dp_dpcd_write_byte(&intel_dp->aux, reg, val) == 0;
 }
 
 static int
@@ -1631,10 +1633,11 @@ intel_dp_128b132b_intra_hop(struct intel_dp *intel_dp,
 	u8 sink_status;
 	int ret;
 
-	ret = drm_dp_dpcd_readb(&intel_dp->aux, DP_SINK_STATUS, &sink_status);
-	if (ret != 1) {
-		lt_dbg(intel_dp, DP_PHY_DPRX, "Failed to read sink status\n");
-		return ret < 0 ? ret : -EIO;
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_SINK_STATUS, &sink_status);
+	if (ret < 0) {
+		lt_dbg(intel_dp, DP_PHY_DPRX, "Failed to read sink status (%pe)\n",
+		       ERR_PTR(ret));
+		return ret;
 	}
 
 	return sink_status & DP_INTRA_HOP_AUX_REPLY_INDICATION ? 1 : 0;
@@ -2173,10 +2176,13 @@ intel_dp_128b132b_lane_cds(struct intel_dp *intel_dp,
 {
 	u8 link_status[DP_LINK_STATUS_SIZE];
 	unsigned long deadline;
+	int ret;
 
-	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TRAINING_PATTERN_SET,
-			       DP_TRAINING_PATTERN_2_CDS) != 1) {
-		lt_err(intel_dp, DP_PHY_DPRX, "Failed to start 128b/132b TPS2 CDS\n");
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_TRAINING_PATTERN_SET,
+				     DP_TRAINING_PATTERN_2_CDS);
+	if (ret < 0) {
+		lt_err(intel_dp, DP_PHY_DPRX, "Failed to start 128b/132b TPS2 CDS (%pe)\n",
+		       ERR_PTR(ret));
 		return false;
 	}
 
@@ -2363,9 +2369,9 @@ void intel_dp_128b132b_sdp_crc16(struct intel_dp *intel_dp,
 		return;
 
 	/* DP v2.0 SCR on SDP CRC16 for 128b/132b Link Layer */
-	drm_dp_dpcd_writeb(&intel_dp->aux,
-			   DP_SDP_ERROR_DETECTION_CONFIGURATION,
-			   DP_SDP_CRC16_128B132B_EN);
+	drm_dp_dpcd_write_byte(&intel_dp->aux,
+			       DP_SDP_ERROR_DETECTION_CONFIGURATION,
+			       DP_SDP_CRC16_128B132B_EN);
 
 	lt_dbg(intel_dp, DP_PHY_DPRX, "DP2.0 SDP CRC16 for 128b/132b enabled\n");
 }
@@ -2825,14 +2831,14 @@ void intel_dp_link_training_cleanup(struct intel_dp_link_training *link_training
 const struct intel_dp_link_training_test_ops i915_display_dp_link_training_test_ops = {
 	INTEL_DP_LINK_TRAINING_TEST_OPS_INIT
 };
-EXPORT_SYMBOL(i915_display_dp_link_training_test_ops);
+EXPORT_SYMBOL_IF_KUNIT(i915_display_dp_link_training_test_ops);
 
 #else
 
 const struct intel_dp_link_training_test_ops intel_display_dp_link_training_test_ops = {
 	INTEL_DP_LINK_TRAINING_TEST_OPS_INIT
 };
-EXPORT_SYMBOL(intel_display_dp_link_training_test_ops);
+EXPORT_SYMBOL_IF_KUNIT(intel_display_dp_link_training_test_ops);
 
 #endif	/* I915 */
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index 57daed0b0b36b5..7023619aaaba9e 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -766,6 +766,10 @@ static int mst_stream_compute_config(struct intel_atomic_state *state,
 		return ret;
 
 	for_each_joiner_candidate(connector, adjusted_mode, num_joined_pipes) {
+		/* If the pipe can't be a joiner primary, skip early. */
+		if (!(intel_joiner_valid_primary_pipe_mask(display, num_joined_pipes) & BIT(crtc->pipe)))
+			continue;
+
 		if (num_joined_pipes > 1)
 			pipe_config->joiner_pipes = GENMASK(crtc->pipe + num_joined_pipes - 1,
 							    crtc->pipe);
@@ -852,7 +856,8 @@ static u8 get_pipes_downstream_of_mst_port(struct intel_atomic_state *state,
 		if (&connector->mst.dp->mst.mgr != mst_mgr)
 			continue;
 
-		if (connector->mst.port != parent_port &&
+		if (parent_port &&
+		    connector->mst.port != parent_port &&
 		    !drm_dp_mst_port_downstream_of_parent(mst_mgr,
 							  connector->mst.port,
 							  parent_port))
@@ -2167,6 +2172,27 @@ bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state,
 	return false;
 }
 
+bool intel_dp_mst_stream_disconnected(struct intel_atomic_state *state,
+				      const struct intel_crtc *crtc)
+{
+	struct intel_connector *connector;
+
+	connector = get_connector_in_state_for_crtc(state, crtc);
+	if (!connector)
+		return false;
+
+	if (!connector->mst.dp)
+		return false;
+
+	if (!connector->mst.dp->mst.mgr.mst_state)
+		return true;
+
+	if (drm_connector_is_unregistered(&connector->base))
+		return true;
+
+	return false;
+}
+
 /**
  * intel_dp_mst_prepare_probe - Prepare an MST link for topology probing
  * @intel_dp: DP port object
@@ -2226,7 +2252,7 @@ bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp)
 	if (!intel_dp->is_mst)
 		return true;
 
-	ret = drm_dp_dpcd_readb(intel_dp->mst.mgr.aux, DP_MSTM_CTRL, &val);
+	ret = drm_dp_dpcd_read_byte(intel_dp->mst.mgr.aux, DP_MSTM_CTRL, &val);
 
 	/* Adjust the expected register value for SST + SideBand. */
 	if (ret < 0 || val != (DP_MST_EN | DP_UP_REQ_EN | DP_UPSTREAM_IS_SRC)) {
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.h b/drivers/gpu/drm/i915/display/intel_dp_mst.h
index ab09b487c6bb5e..8ce89242c05c95 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.h
@@ -28,6 +28,8 @@ int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state,
 				   struct intel_link_bw_limits *limits);
 bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state,
 				     struct intel_crtc *crtc);
+bool intel_dp_mst_stream_disconnected(struct intel_atomic_state *state,
+				      const struct intel_crtc *crtc);
 void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp);
 bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_test.c b/drivers/gpu/drm/i915/display/intel_dp_test.c
index 0551a1ce60d392..17cf2727312478 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_test.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_test.c
@@ -147,26 +147,25 @@ bool intel_dp_test_compute_config(struct intel_connector *connector,
 static u8 intel_dp_autotest_link_training(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	int status = 0;
+	int ret;
 	int test_link_rate;
 	u8 test_lane_count, test_link_bw;
 	/* (DP CTS 1.2)
 	 * 4.3.1.11
 	 */
 	/* Read the TEST_LANE_COUNT and TEST_LINK_RTAE fields (DP CTS 3.1.4) */
-	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_LANE_COUNT,
-				   &test_lane_count);
-
-	if (status <= 0) {
-		drm_dbg_kms(display->drm, "Lane count read failed\n");
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_TEST_LANE_COUNT,
+				    &test_lane_count);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "Lane count read failed (%pe)\n", ERR_PTR(ret));
 		return DP_TEST_NAK;
 	}
 	test_lane_count &= DP_MAX_LANE_COUNT_MASK;
 
-	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_LINK_RATE,
-				   &test_link_bw);
-	if (status <= 0) {
-		drm_dbg_kms(display->drm, "Link Rate read failed\n");
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_TEST_LINK_RATE,
+				    &test_link_bw);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "Link Rate read failed (%pe)\n", ERR_PTR(ret));
 		return DP_TEST_NAK;
 	}
 	test_link_rate = drm_dp_bw_code_to_link_rate(test_link_bw);
@@ -188,36 +187,34 @@ static u8 intel_dp_autotest_video_pattern(struct intel_dp *intel_dp)
 	u8 test_pattern;
 	u8 test_misc;
 	__be16 h_width, v_height;
-	int status = 0;
+	int ret;
 
 	/* Read the TEST_PATTERN (DP CTS 3.1.5) */
-	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_PATTERN,
-				   &test_pattern);
-	if (status <= 0) {
-		drm_dbg_kms(display->drm, "Test pattern read failed\n");
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_TEST_PATTERN,
+				    &test_pattern);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "Test pattern read failed (%pe)\n", ERR_PTR(ret));
 		return DP_TEST_NAK;
 	}
 	if (test_pattern != DP_COLOR_RAMP)
 		return DP_TEST_NAK;
 
-	status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_H_WIDTH_HI,
-				  &h_width, 2);
-	if (status <= 0) {
-		drm_dbg_kms(display->drm, "H Width read failed\n");
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_TEST_H_WIDTH_HI, &h_width, 2);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "H Width read failed (%pe)\n", ERR_PTR(ret));
 		return DP_TEST_NAK;
 	}
 
-	status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_V_HEIGHT_HI,
-				  &v_height, 2);
-	if (status <= 0) {
-		drm_dbg_kms(display->drm, "V Height read failed\n");
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_TEST_V_HEIGHT_HI,
+				    &v_height, 2);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "V Height read failed (%pe)\n", ERR_PTR(ret));
 		return DP_TEST_NAK;
 	}
 
-	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_MISC0,
-				   &test_misc);
-	if (status <= 0) {
-		drm_dbg_kms(display->drm, "TEST MISC read failed\n");
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_TEST_MISC0, &test_misc);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "TEST MISC read failed (%pe)\n", ERR_PTR(ret));
 		return DP_TEST_NAK;
 	}
 	if ((test_misc & DP_TEST_COLOR_FORMAT_MASK) != DP_COLOR_FORMAT_RGB)
@@ -250,6 +247,7 @@ static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp)
 	u8 test_result = DP_TEST_ACK;
 	struct intel_connector *intel_connector = intel_dp->attached_connector;
 	struct drm_connector *connector = &intel_connector->base;
+	int ret;
 
 	if (!intel_connector->detect_edid || connector->edid_corrupt ||
 	    intel_dp->aux.i2c_defer_count > 6) {
@@ -274,10 +272,11 @@ static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp)
 		/* We have to write the checksum of the last block read */
 		block += block->extensions;
 
-		if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_EDID_CHECKSUM,
-				       block->checksum) <= 0)
+		ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_TEST_EDID_CHECKSUM,
+					     block->checksum);
+		if (ret < 0)
 			drm_dbg_kms(display->drm,
-				    "Failed to write EDID checksum\n");
+				    "Failed to write EDID checksum (%pe)\n", ERR_PTR(ret));
 
 		test_result = DP_TEST_ACK | DP_TEST_EDID_CHECKSUM_WRITE;
 		intel_dp->compliance.test_data.edid = INTEL_DP_RESOLUTION_PREFERRED;
@@ -397,8 +396,8 @@ static void intel_dp_process_phy_request(struct intel_dp *intel_dp,
 
 	intel_dp_phy_pattern_update(intel_dp, crtc_state);
 
-	drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET,
-			  intel_dp->train_set, crtc_state->lane_count);
+	drm_dp_dpcd_write_data(&intel_dp->aux, DP_TRAINING_LANE0_SET,
+			       intel_dp->train_set, crtc_state->lane_count);
 
 	drm_dp_set_phy_test_pattern(&intel_dp->aux, data,
 				    intel_dp->dpcd[DP_DPCD_REV]);
@@ -427,12 +426,12 @@ void intel_dp_test_request(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	u8 response = DP_TEST_NAK;
 	u8 request = 0;
-	int status;
+	int ret;
 
-	status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_REQUEST, &request);
-	if (status <= 0) {
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_TEST_REQUEST, &request);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Could not read test request from sink\n");
+			    "Could not read test request from sink (%pe)\n", ERR_PTR(ret));
 		goto update_status;
 	}
 
@@ -463,10 +462,10 @@ void intel_dp_test_request(struct intel_dp *intel_dp)
 		intel_dp->compliance.test_type = request;
 
 update_status:
-	status = drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_RESPONSE, response);
-	if (status <= 0)
+	ret = drm_dp_dpcd_write_byte(&intel_dp->aux, DP_TEST_RESPONSE, response);
+	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Could not write test response to sink\n");
+			    "Could not write test response to sink (%pe)\n", ERR_PTR(ret));
 }
 
 /* phy test */
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c
index f40807a5566b68..20fd091cb9dbdc 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll.c
@@ -22,7 +22,6 @@
 #include "intel_lvds_regs.h"
 #include "intel_panel.h"
 #include "intel_pps.h"
-#include "intel_snps_phy.h"
 #include "vlv_dpio_phy_regs.h"
 #include "vlv_sideband.h"
 
@@ -1194,24 +1193,6 @@ static int hsw_crtc_get_dpll(struct intel_atomic_state *state,
 	return intel_dpll_reserve(state, crtc, encoder);
 }
 
-static int dg2_crtc_compute_clock(struct intel_atomic_state *state,
-				  struct intel_crtc *crtc)
-{
-	struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	struct intel_encoder *encoder =
-		intel_get_crtc_new_encoder(state, crtc_state);
-	int ret;
-
-	ret = intel_mpllb_calc_state(crtc_state, encoder);
-	if (ret)
-		return ret;
-
-	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
-
-	return 0;
-}
-
 static int ilk_fb_cb_factor(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
@@ -1682,7 +1663,8 @@ static const struct intel_dpll_global_funcs mtl_dpll_funcs = {
 };
 
 static const struct intel_dpll_global_funcs dg2_dpll_funcs = {
-	.crtc_compute_clock = dg2_crtc_compute_clock,
+	.crtc_compute_clock = hsw_crtc_compute_clock,
+	.crtc_get_dpll = hsw_crtc_get_dpll,
 };
 
 static const struct intel_dpll_global_funcs hsw_dpll_funcs = {
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index 0f933101b7dd16..bfd6f0a1972534 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -42,6 +42,7 @@
 #include "intel_lt_phy.h"
 #include "intel_mg_phy_regs.h"
 #include "intel_pch_refclk.h"
+#include "intel_snps_phy.h"
 #include "intel_tc.h"
 
 /**
@@ -4735,6 +4736,214 @@ static const struct intel_dpll_mgr xe3plpd_pll_mgr = {
 	.compare_hw_state = xe3plpd_compare_hw_state,
 };
 
+enum intel_dpll_id dg2_port_to_pll_id(enum port port)
+{
+	switch (port) {
+	case PORT_A:
+		return DPLL_ID_DG2_DPLL_A;
+	case PORT_B:
+		return DPLL_ID_DG2_DPLL_B;
+	case PORT_C:
+		return DPLL_ID_DG2_DPLL_C;
+	case PORT_D_XELPD:
+		return DPLL_ID_DG2_DPLL_D;
+	case PORT_TC1:
+		return DPLL_ID_DG2_DPLL_E;
+	default:
+		MISSING_CASE(port);
+		return DPLL_ID_DG2_DPLL_A;
+	}
+}
+
+static struct intel_encoder *dg2_get_intel_encoder(struct intel_display *display,
+						   const struct intel_dpll *pll)
+{
+	struct intel_encoder *encoder;
+
+	for_each_intel_encoder(display->drm, encoder) {
+		if (dg2_port_to_pll_id(encoder->port) == pll->info->id &&
+		    intel_encoder_is_dig_port(encoder))
+			return encoder;
+	}
+
+	return NULL;
+}
+
+static void dg2_mpllb_enable(struct intel_display *display,
+			     struct intel_dpll *pll,
+			     const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct intel_encoder *encoder = dg2_get_intel_encoder(display, pll);
+
+	if (drm_WARN_ON(display->drm, !encoder))
+		return;
+
+	intel_mpllb_enable_phy(encoder, &dpll_hw_state->mpllb);
+}
+
+static void dg2_mpllb_disable(struct intel_display *display,
+			      struct intel_dpll *pll)
+{
+	struct intel_encoder *encoder = dg2_get_intel_encoder(display, pll);
+
+	if (drm_WARN_ON(display->drm, !encoder))
+		return;
+
+	intel_mpllb_disable(encoder);
+}
+
+static bool dg2_mpllb_get_hw_state(struct intel_display *display,
+				   struct intel_dpll *pll,
+				   struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct intel_encoder *encoder = dg2_get_intel_encoder(display, pll);
+	enum phy phy;
+	i915_reg_t enable_reg;
+	struct ref_tracker *wakeref;
+	bool ret = false;
+	u32 val;
+
+	if (!encoder)
+		return false;
+
+	wakeref = intel_display_power_get_if_enabled(display,
+						     POWER_DOMAIN_DISPLAY_CORE);
+	if (!wakeref)
+		return false;
+
+	phy = intel_encoder_to_phy(encoder);
+	enable_reg = (phy <= PHY_D ? DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0));
+
+	val = intel_de_read(display, enable_reg);
+	if (!(val & PLL_ENABLE))
+		goto out;
+
+	intel_mpllb_readout_hw_state(encoder, &dpll_hw_state->mpllb);
+	ret = true;
+
+out:
+	intel_display_power_put(display, POWER_DOMAIN_DISPLAY_CORE, wakeref);
+	return ret;
+}
+
+static int dg2_mpllb_get_freq(struct intel_display *display,
+			      const struct intel_dpll *pll,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	struct intel_encoder *encoder = dg2_get_intel_encoder(display, pll);
+
+	if (drm_WARN_ON(display->drm, !encoder))
+		return 0;
+
+	return intel_mpllb_calc_port_clock(encoder, &dpll_hw_state->mpllb);
+}
+
+static const struct intel_dpll_funcs mpllb_pll_funcs = {
+	.enable = dg2_mpllb_enable,
+	.disable = dg2_mpllb_disable,
+	.get_hw_state = dg2_mpllb_get_hw_state,
+	.get_freq = dg2_mpllb_get_freq,
+};
+
+static const struct dpll_info dg2_plls[] = {
+	{ .name = "MPLLB A", .funcs = &mpllb_pll_funcs, .id = DPLL_ID_DG2_DPLL_A, },
+	{ .name = "MPLLB B", .funcs = &mpllb_pll_funcs, .id = DPLL_ID_DG2_DPLL_B, },
+	{ .name = "MPLLB C", .funcs = &mpllb_pll_funcs, .id = DPLL_ID_DG2_DPLL_C, },
+	{ .name = "MPLLB D", .funcs = &mpllb_pll_funcs, .id = DPLL_ID_DG2_DPLL_D, },
+	{ .name = "MPLLB E", .funcs = &mpllb_pll_funcs, .id = DPLL_ID_DG2_DPLL_E, },
+	{}
+};
+
+static int dg2_compute_dplls(struct intel_atomic_state *state,
+			     struct intel_crtc *crtc,
+			     struct intel_encoder *encoder)
+{
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	struct icl_port_dpll *port_dpll =
+		&crtc_state->icl_port_dplls[ICL_PORT_DPLL_DEFAULT];
+	int ret;
+
+	ret = intel_mpllb_calc_state(crtc_state, encoder);
+	if (ret)
+		return ret;
+
+	port_dpll->hw_state = crtc_state->dpll_hw_state;
+
+	/* this is mainly for the fastset check */
+	icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_DEFAULT);
+
+	crtc_state->port_clock = intel_mpllb_calc_port_clock(encoder,
+							     &port_dpll->hw_state.mpllb);
+
+	return 0;
+}
+
+static int dg2_get_dplls(struct intel_atomic_state *state,
+			 struct intel_crtc *crtc,
+			 struct intel_encoder *encoder)
+{
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	struct icl_port_dpll *port_dpll =
+		&crtc_state->icl_port_dplls[ICL_PORT_DPLL_DEFAULT];
+	enum intel_dpll_id dpll_id = dg2_port_to_pll_id(encoder->port);
+
+	port_dpll->pll = intel_find_dpll(state, crtc,
+					 &port_dpll->hw_state,
+					 BIT(dpll_id));
+	if (!port_dpll->pll)
+		return -EINVAL;
+
+	intel_reference_dpll(state, crtc,
+			     port_dpll->pll, &port_dpll->hw_state);
+
+	icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_DEFAULT);
+
+	return 0;
+}
+
+static void dg2_dump_hw_state(struct drm_printer *p,
+			      const struct intel_dpll_hw_state *dpll_hw_state)
+{
+	const struct intel_mpllb_state *hw_state = &dpll_hw_state->mpllb;
+
+	drm_printf(p, "dpll_hw_state: mpllb_cp: 0x%x, mpllb_div: 0x%x, "
+		   "mpllb_div2: 0x%x, mpllb_fracn1: 0x%x, "
+		   "mpllb_fracn2: 0x%x, mpllb_sscen: 0x%x, "
+		   "mpllb_sscstep: 0x%x, ref_control: 0x%x\n",
+		   hw_state->mpllb_cp, hw_state->mpllb_div,
+		   hw_state->mpllb_div2, hw_state->mpllb_fracn1,
+		   hw_state->mpllb_fracn2, hw_state->mpllb_sscen,
+		   hw_state->mpllb_sscstep, hw_state->ref_control);
+}
+
+static bool dg2_compare_hw_state(const struct intel_dpll_hw_state *_a,
+				 const struct intel_dpll_hw_state *_b)
+{
+	const struct intel_mpllb_state *a = &_a->mpllb;
+	const struct intel_mpllb_state *b = &_b->mpllb;
+
+	return a->mpllb_cp == b->mpllb_cp &&
+		a->mpllb_div == b->mpllb_div &&
+		a->mpllb_div2 == b->mpllb_div2 &&
+		a->mpllb_fracn1 == b->mpllb_fracn1 &&
+		a->mpllb_fracn2 == b->mpllb_fracn2 &&
+		a->mpllb_sscen == b->mpllb_sscen &&
+		a->mpllb_sscstep == b->mpllb_sscstep;
+}
+
+static const struct intel_dpll_mgr dg2_pll_mgr = {
+	.dpll_info = dg2_plls,
+	.compute_dplls = dg2_compute_dplls,
+	.get_dplls = dg2_get_dplls,
+	.put_dplls = icl_put_dplls,
+	.update_active_dpll = icl_update_active_dpll,
+	.update_ref_clks = icl_update_dpll_ref_clks,
+	.dump_hw_state = dg2_dump_hw_state,
+	.compare_hw_state = dg2_compare_hw_state,
+};
+
 /**
  * intel_dpll_init - Initialize DPLLs
  * @display: intel_display device
@@ -4750,8 +4959,7 @@ void intel_dpll_init(struct intel_display *display)
 	mutex_init(&display->dpll.lock);
 
 	if (display->platform.dg2)
-		/* No shared DPLLs on DG2; port PLLs are part of the PHY */
-		dpll_mgr = NULL;
+		dpll_mgr = &dg2_pll_mgr;
 	else if (DISPLAY_VER(display) >= 35)
 		dpll_mgr = &xe3plpd_pll_mgr;
 	else if (DISPLAY_VER(display) >= 14)
@@ -5120,8 +5328,13 @@ verify_single_dpll_state(struct intel_display *display,
 
 	if (pll->on) {
 		const struct intel_dpll_mgr *dpll_mgr = display->dpll.mgr;
-
-		if (HAS_LT_PHY(display))
+		/*
+		 * Avoid direct struct comparison here. Some hw state fields, such
+		 * as DG2 MPLLB ref_control or LT PHY config[1], are written by
+		 * firmware and may differ from the software state without indicating
+		 * a real mismatch.
+		 */
+		if (HAS_LT_PHY(display) || display->platform.dg2)
 			pll_mismatch = !dpll_mgr->compare_hw_state(&pll->state.hw_state,
 								   &dpll_hw_state);
 		else
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
index d408ccf6f90256..f4b84733a58c42 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
@@ -169,9 +169,31 @@ enum intel_dpll_id {
 	 * @DPLL_ID_DG1_DPLL3: DG1 combo PHY DPLL3
 	 */
 	DPLL_ID_DG1_DPLL3 = 3,
+
+	/**
+	 * @DPLL_ID_DG2_DPLL_A: DG2 port PLL for PHY A (PORT_A)
+	 */
+	DPLL_ID_DG2_DPLL_A = 0,
+	/**
+	 * @DPLL_ID_DG2_DPLL_B: DG2 port PLL for PHY B (PORT_B)
+	 */
+	DPLL_ID_DG2_DPLL_B = 1,
+	/**
+	 * @DPLL_ID_DG2_DPLL_C: DG2 port PLL for PHY C (PORT_C)
+	 */
+	DPLL_ID_DG2_DPLL_C = 2,
+	/**
+	 * @DPLL_ID_DG2_DPLL_D: DG2 port PLL for PHY D (PORT_D_XELPD)
+	 */
+	DPLL_ID_DG2_DPLL_D = 3,
+	/**
+	 * @DPLL_ID_DG2_DPLL_E: DG2 port PLL for PHY F (PORT_TC1)
+	 */
+	DPLL_ID_DG2_DPLL_E = 4,
 };
 
 #define I915_NUM_PLLS 9
+enum intel_dpll_id dg2_port_to_pll_id(enum port port);
 
 enum icl_port_dpll_id {
 	ICL_PORT_DPLL_DEFAULT,
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index 1c0859d5f82986..c3e4c3f6d8cf01 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -596,7 +596,7 @@ u64 *intel_fb_plane_get_modifiers(struct intel_display *display,
 			count++;
 	}
 
-	list = kmalloc_array(count, sizeof(*list), GFP_KERNEL);
+	list = kmalloc_objs(*list, count);
 	if (drm_WARN_ON(display->drm, !list))
 		return NULL;
 
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c
index f61b4a218d6ef4..5694d20773f83c 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.c
+++ b/drivers/gpu/drm/i915/display/intel_fbc.c
@@ -1196,23 +1196,13 @@ xe3p_lpd_fbc_fp16_format_is_valid(const struct intel_plane_state *plane_state)
 
 static bool xe3p_lpd_fbc_pixel_format_is_valid(const struct intel_plane_state *plane_state)
 {
-	const struct drm_framebuffer *fb = plane_state->hw.fb;
-
 	if (lnl_fbc_pixel_format_is_valid(plane_state))
 		return true;
 
 	if (xe3p_lpd_fbc_fp16_format_is_valid(plane_state))
 		return true;
 
-	switch (fb->format->format) {
-	case DRM_FORMAT_XRGB16161616:
-	case DRM_FORMAT_XBGR16161616:
-	case DRM_FORMAT_ARGB16161616:
-	case DRM_FORMAT_ABGR16161616:
-		return true;
-	default:
-		return false;
-	}
+	return false;
 }
 
 bool intel_fbc_need_pixel_normalizer(const struct intel_plane_state *plane_state)
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
index 705742e117caf6..70aae6ffa3fec2 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
@@ -66,7 +66,6 @@
 #include "intel_frontbuffer.h"
 #include "intel_parent.h"
 #include "intel_psr.h"
-#include "intel_tdf.h"
 
 /**
  * frontbuffer_flush - flush frontbuffer
@@ -95,7 +94,7 @@ static void frontbuffer_flush(struct intel_display *display,
 	trace_intel_frontbuffer_flush(display, frontbuffer_bits, origin);
 
 	might_sleep();
-	intel_td_flush(display);
+	intel_parent_transient_data_flush(display);
 	intel_drrs_flush(display, frontbuffer_bits);
 	intel_psr_flush(display, frontbuffer_bits, origin);
 	intel_fbc_flush(display, frontbuffer_bits, origin);
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 8a019d3574df22..b6fa1a1558b82f 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -2399,10 +2399,6 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder,
 	if (!intel_link_bw_compute_pipe_bpp(pipe_config))
 		return -EINVAL;
 
-	pipe_config->has_audio =
-		intel_hdmi_has_audio(encoder, pipe_config, conn_state) &&
-		intel_audio_compute_config(encoder, pipe_config, conn_state);
-
 	/*
 	 * Try to respect downstream TMDS clock limits first, if
 	 * that fails assume the user might know something we don't.
@@ -2417,6 +2413,10 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder,
 		return ret;
 	}
 
+	pipe_config->has_audio =
+		intel_hdmi_has_audio(encoder, pipe_config, conn_state) &&
+		intel_audio_compute_config(encoder, pipe_config, conn_state);
+
 	ret = intel_pfit_compute_config(pipe_config, conn_state);
 	if (ret)
 		return ret;
@@ -3216,7 +3216,8 @@ int intel_hdmi_dsc_get_slice_height(int vactive)
  * intel_hdmi_dsc_get_num_slices - get no. of dsc slices based on dsc encoder
  * and dsc decoder capabilities
  *
- * @crtc_state: intel crtc_state
+ * @mode: drm_display_mode for which num of slices are needed
+ * @output_format : pipe output format
  * @src_max_slices: maximum slices supported by the DSC encoder
  * @src_max_slice_width: maximum slice width supported by DSC encoder
  * @hdmi_max_slices: maximum slices supported by sink DSC decoder
@@ -3226,7 +3227,8 @@ int intel_hdmi_dsc_get_slice_height(int vactive)
  * and decoder.
  */
 int
-intel_hdmi_dsc_get_num_slices(const struct intel_crtc_state *crtc_state,
+intel_hdmi_dsc_get_num_slices(const struct drm_display_mode *mode,
+			      enum intel_output_format output_format,
 			      int src_max_slices, int src_max_slice_width,
 			      int hdmi_max_slices, int hdmi_throughput)
 {
@@ -3248,7 +3250,7 @@ intel_hdmi_dsc_get_num_slices(const struct intel_crtc_state *crtc_state,
 	int max_throughput; /* max clock freq. in khz per slice */
 	int max_slice_width;
 	int slice_width;
-	int pixel_clock = crtc_state->hw.adjusted_mode.crtc_clock;
+	int pixel_clock = mode->crtc_clock;
 
 	if (!hdmi_throughput)
 		return 0;
@@ -3259,8 +3261,8 @@ intel_hdmi_dsc_get_num_slices(const struct intel_crtc_state *crtc_state,
 	 * for 4:4:4 is 1.0. Multiplying these factors by 10 and later
 	 * dividing adjusted clock value by 10.
 	 */
-	if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444 ||
-	    crtc_state->output_format == INTEL_OUTPUT_FORMAT_RGB)
+	if (output_format == INTEL_OUTPUT_FORMAT_YCBCR444 ||
+	    output_format == INTEL_OUTPUT_FORMAT_RGB)
 		kslice_adjust = 10;
 	else
 		kslice_adjust = 5;
@@ -3315,7 +3317,7 @@ intel_hdmi_dsc_get_num_slices(const struct intel_crtc_state *crtc_state,
 		else
 			return 0;
 
-		slice_width = DIV_ROUND_UP(crtc_state->hw.adjusted_mode.hdisplay, target_slices);
+		slice_width = DIV_ROUND_UP(mode->hdisplay, target_slices);
 		if (slice_width >= max_slice_width)
 			min_slices = target_slices + 1;
 	} while (slice_width >= max_slice_width);
@@ -3331,6 +3333,7 @@ intel_hdmi_dsc_get_num_slices(const struct intel_crtc_state *crtc_state,
  * @slice_width: dsc slice width supported by the source and sink
  * @num_slices: num of slices supported by the source and sink
  * @output_format: video output format
+ * @bpc: bits per color
  * @hdmi_all_bpp: sink supports decoding of 1/16th bpp setting
  * @hdmi_max_chunk_bytes: max bytes in a line of chunks supported by sink
  *
@@ -3338,8 +3341,8 @@ intel_hdmi_dsc_get_num_slices(const struct intel_crtc_state *crtc_state,
  */
 int
 intel_hdmi_dsc_get_bpp(int src_fractional_bpp, int slice_width, int num_slices,
-		       int output_format, bool hdmi_all_bpp,
-		       int hdmi_max_chunk_bytes)
+		       enum intel_output_format output_format, int bpc,
+		       bool hdmi_all_bpp, int hdmi_max_chunk_bytes)
 {
 	int max_dsc_bpp, min_dsc_bpp;
 	int target_bytes;
@@ -3356,18 +3359,17 @@ intel_hdmi_dsc_get_bpp(int src_fractional_bpp, int slice_width, int num_slices,
 	 * for each bpp we check if no of bytes can be supported by HDMI sink
 	 */
 
-	/* Assuming: bpc as 8*/
 	if (output_format == INTEL_OUTPUT_FORMAT_YCBCR420) {
 		min_dsc_bpp = 6;
-		max_dsc_bpp = 3 * 4; /* 3*bpc/2 */
+		max_dsc_bpp = 3 * bpc / 2;
 	} else if (output_format == INTEL_OUTPUT_FORMAT_YCBCR444 ||
 		   output_format == INTEL_OUTPUT_FORMAT_RGB) {
 		min_dsc_bpp = 8;
-		max_dsc_bpp = 3 * 8; /* 3*bpc */
+		max_dsc_bpp = 3 * bpc;
 	} else {
 		/* Assuming 4:2:2 encoding */
 		min_dsc_bpp = 7;
-		max_dsc_bpp = 2 * 8; /* 2*bpc */
+		max_dsc_bpp = 2 * bpc;
 	}
 
 	/*
@@ -3420,3 +3422,22 @@ intel_hdmi_dsc_get_bpp(int src_fractional_bpp, int slice_width, int num_slices,
 
 	return 0;
 }
+
+int intel_hdmi_sink_max_frl_rate(struct drm_connector *connector)
+{
+	int max_lanes = connector->display_info.hdmi.max_lanes;
+	int rate_per_lane = connector->display_info.hdmi.max_frl_rate_per_lane;
+
+	return max_lanes * rate_per_lane;
+}
+
+int intel_hdmi_sink_dsc_max_frl_rate(struct drm_connector *connector)
+{
+	int max_lanes = connector->display_info.hdmi.dsc_cap.max_lanes;
+	int rate_per_lane = connector->display_info.hdmi.dsc_cap.max_frl_rate_per_lane;
+
+	if (!max_lanes || !rate_per_lane)
+		return 0;
+
+	return max_lanes * rate_per_lane;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.h b/drivers/gpu/drm/i915/display/intel_hdmi.h
index 0fa3661568e861..c95ed37bcc0c2e 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.h
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.h
@@ -13,6 +13,7 @@ enum intel_output_format;
 enum port;
 struct drm_connector;
 struct drm_connector_state;
+struct drm_display_mode;
 struct drm_encoder;
 struct intel_connector;
 struct intel_crtc_state;
@@ -54,9 +55,10 @@ bool intel_hdmi_bpc_possible(const struct intel_crtc_state *crtc_state,
 			     int bpc, bool has_hdmi_sink);
 int intel_hdmi_tmds_clock(int clock, int bpc, enum intel_output_format sink_format);
 int intel_hdmi_dsc_get_bpp(int src_fractional_bpp, int slice_width,
-			   int num_slices, int output_format, bool hdmi_all_bpp,
-			   int hdmi_max_chunk_bytes);
-int intel_hdmi_dsc_get_num_slices(const struct intel_crtc_state *crtc_state,
+			   int num_slices, enum intel_output_format output_format,
+			   int bpc, bool hdmi_all_bpp, int hdmi_max_chunk_bytes);
+int intel_hdmi_dsc_get_num_slices(const struct drm_display_mode *mode,
+				  enum intel_output_format output_format,
 				  int src_max_slices, int src_max_slice_width,
 				  int hdmi_max_slices, int hdmi_throughput);
 int intel_hdmi_dsc_get_slice_height(int vactive);
@@ -73,4 +75,7 @@ void hsw_read_infoframe(struct intel_encoder *encoder,
 void intel_hdmi_poll_for_scrambling_enable(const struct intel_crtc_state *crtc_state,
 					   struct drm_connector *_connector);
 
+int intel_hdmi_sink_max_frl_rate(struct drm_connector *connector);
+int intel_hdmi_sink_dsc_max_frl_rate(struct drm_connector *connector);
+
 #endif /* __INTEL_HDMI_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c
index 970aa95ee344ab..d095eb13dc06a7 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug.c
@@ -867,6 +867,11 @@ void intel_hpd_poll_enable(struct intel_display *display)
 	if (!HAS_DISPLAY(display) || !intel_display_device_enabled(display))
 		return;
 
+	if (intel_display_rpm_pme_enabled(display)) {
+		drm_dbg_kms(display->drm, "PME wake capable device, skipping HPD polling.\n");
+		return;
+	}
+
 	WRITE_ONCE(display->hotplug.poll_enabled, true);
 
 	/*
diff --git a/drivers/gpu/drm/i915/display/intel_hti.c b/drivers/gpu/drm/i915/display/intel_hti.c
index dc454420c134e6..56602240ceffa0 100644
--- a/drivers/gpu/drm/i915/display/intel_hti.c
+++ b/drivers/gpu/drm/i915/display/intel_hti.c
@@ -23,9 +23,6 @@ void intel_hti_init(struct intel_display *display)
 
 bool intel_hti_uses_phy(struct intel_display *display, enum phy phy)
 {
-	if (drm_WARN_ON(display->drm, phy == PHY_NONE))
-		return false;
-
 	return display->hti.state & HDPORT_ENABLED &&
 		display->hti.state & HDPORT_DDI_USED(phy);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_link_bw.c b/drivers/gpu/drm/i915/display/intel_link_bw.c
index b47474a3e9fec5..e71e76d6fd3e06 100644
--- a/drivers/gpu/drm/i915/display/intel_link_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_link_bw.c
@@ -64,7 +64,8 @@ void intel_link_bw_init_limits(struct intel_atomic_state *state,
 			intel_atomic_get_new_crtc_state(state, crtc);
 		int forced_bpp_x16 = get_forced_link_bpp_x16(state, crtc);
 
-		if (state->base.duplicated && crtc_state) {
+		if ((state->base.duplicated && crtc_state) ||
+		    intel_dp_mst_stream_disconnected(state, crtc)) {
 			limits->max_bpp_x16[pipe] = crtc_state->max_link_bpp_x16;
 			if (intel_dsc_enabled_on_link(crtc_state))
 				limits->link_dsc_pipes |= BIT(pipe);
diff --git a/drivers/gpu/drm/i915/display/intel_lspcon.c b/drivers/gpu/drm/i915/display/intel_lspcon.c
index 9ceabbc981a169..a61224523318c7 100644
--- a/drivers/gpu/drm/i915/display/intel_lspcon.c
+++ b/drivers/gpu/drm/i915/display/intel_lspcon.c
@@ -137,11 +137,10 @@ bool intel_lspcon_detect_hdr_capability(struct intel_digital_port *dig_port)
 	u8 hdr_caps;
 	int ret;
 
-	ret = drm_dp_dpcd_read(&intel_dp->aux, get_hdr_status_reg(lspcon),
-			       &hdr_caps, 1);
-
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, get_hdr_status_reg(lspcon), &hdr_caps);
 	if (ret < 0) {
-		drm_dbg_kms(display->drm, "HDR capability detection failed\n");
+		drm_dbg_kms(display->drm, "HDR capability detection failed (%pe)\n",
+			    ERR_PTR(ret));
 		lspcon->hdr_supported = false;
 	} else if (hdr_caps & 0x1) {
 		drm_dbg_kms(display->drm, "LSPCON capable of HDR\n");
@@ -244,11 +243,12 @@ static bool lspcon_wake_native_aux_ch(struct intel_lspcon *lspcon)
 {
 	struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon);
 	struct intel_display *display = to_intel_display(intel_dp);
+	int ret;
 	u8 rev;
 
-	if (drm_dp_dpcd_readb(&lspcon_to_intel_dp(lspcon)->aux, DP_DPCD_REV,
-			      &rev) != 1) {
-		drm_dbg_kms(display->drm, "Native AUX CH down\n");
+	ret = drm_dp_dpcd_read_byte(&lspcon_to_intel_dp(lspcon)->aux, DP_DPCD_REV, &rev);
+	if (ret < 0) {
+		drm_dbg_kms(display->drm, "Native AUX CH down (%pe)\n", ERR_PTR(ret));
 		return false;
 	}
 
@@ -331,17 +331,17 @@ static bool lspcon_parade_fw_ready(struct drm_dp_aux *aux)
 {
 	u8 avi_if_ctrl;
 	u8 retry;
-	ssize_t ret;
+	int ret;
 
 	/* Check if LSPCON FW is ready for data */
 	for (retry = 0; retry < 5; retry++) {
 		if (retry)
 			usleep_range(200, 300);
 
-		ret = drm_dp_dpcd_read(aux, LSPCON_PARADE_AVI_IF_CTRL,
-				       &avi_if_ctrl, 1);
+		ret = drm_dp_dpcd_read_byte(aux, LSPCON_PARADE_AVI_IF_CTRL, &avi_if_ctrl);
 		if (ret < 0) {
-			drm_err(aux->drm_dev, "Failed to read AVI IF control\n");
+			drm_err(aux->drm_dev, "Failed to read AVI IF control (%pe)\n",
+				ERR_PTR(ret));
 			return false;
 		}
 
@@ -371,10 +371,10 @@ static bool _lspcon_parade_write_infoframe_blocks(struct drm_dp_aux *aux,
 
 		reg = LSPCON_PARADE_AVI_IF_WRITE_OFFSET;
 		data = avi_buf + block_count * 8;
-		ret = drm_dp_dpcd_write(aux, reg, data, 8);
+		ret = drm_dp_dpcd_write_data(aux, reg, data, 8);
 		if (ret < 0) {
-			drm_err(aux->drm_dev, "Failed to write AVI IF block %d\n",
-				block_count);
+			drm_err(aux->drm_dev, "Failed to write AVI IF block %d (%pe)\n",
+				block_count, ERR_PTR(ret));
 			return false;
 		}
 
@@ -386,10 +386,10 @@ static bool _lspcon_parade_write_infoframe_blocks(struct drm_dp_aux *aux,
 		 */
 		reg = LSPCON_PARADE_AVI_IF_CTRL;
 		avi_if_ctrl = LSPCON_PARADE_AVI_IF_KICKOFF | block_count;
-		ret = drm_dp_dpcd_write(aux, reg, &avi_if_ctrl, 1);
+		ret = drm_dp_dpcd_write_byte(aux, reg, avi_if_ctrl);
 		if (ret < 0) {
-			drm_err(aux->drm_dev, "Failed to update (0x%x), block %d\n",
-				reg, block_count);
+			drm_err(aux->drm_dev, "Failed to update (0x%x), block %d (%pe)\n",
+				reg, block_count, ERR_PTR(ret));
 			return false;
 		}
 
@@ -435,35 +435,38 @@ static bool _lspcon_write_avi_infoframe_parade(struct drm_dp_aux *aux,
 static bool _lspcon_write_avi_infoframe_mca(struct drm_dp_aux *aux,
 					    const u8 *buffer, ssize_t len)
 {
-	int ret;
-	u32 val = 0;
+	int ret, written = 0;
 	u32 retry;
 	u16 reg;
 	const u8 *data = buffer;
+	u8 val;
 
 	reg = LSPCON_MCA_AVI_IF_WRITE_OFFSET;
-	while (val < len) {
+	while (written < len) {
 		/* DPCD write for AVI IF can fail on a slow FW day, so retry */
 		for (retry = 0; retry < 5; retry++) {
-			ret = drm_dp_dpcd_write(aux, reg, (void *)data, 1);
-			if (ret == 1) {
+			ret = drm_dp_dpcd_write_byte(aux, reg, *data);
+			if (!ret) {
 				break;
 			} else if (retry < 4) {
 				mdelay(50);
 				continue;
 			} else {
-				drm_err(aux->drm_dev, "DPCD write failed at:0x%x\n", reg);
+				drm_err(aux->drm_dev, "DPCD write failed at:0x%x (%pe)\n",
+					reg, ERR_PTR(ret));
 				return false;
 			}
 		}
-		val++; reg++; data++;
+		written++;
+		reg++;
+		data++;
 	}
 
-	val = 0;
 	reg = LSPCON_MCA_AVI_IF_CTRL;
-	ret = drm_dp_dpcd_read(aux, reg, &val, 1);
+	ret = drm_dp_dpcd_read_byte(aux, reg, &val);
 	if (ret < 0) {
-		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg);
+		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x (%pe)\n",
+			reg, ERR_PTR(ret));
 		return false;
 	}
 
@@ -471,16 +474,17 @@ static bool _lspcon_write_avi_infoframe_mca(struct drm_dp_aux *aux,
 	val &= ~LSPCON_MCA_AVI_IF_HANDLED;
 	val |= LSPCON_MCA_AVI_IF_KICKOFF;
 
-	ret = drm_dp_dpcd_write(aux, reg, &val, 1);
+	ret = drm_dp_dpcd_write_byte(aux, reg, val);
 	if (ret < 0) {
-		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg);
+		drm_err(aux->drm_dev, "DPCD write failed at:0x%x (%pe)\n",
+			reg, ERR_PTR(ret));
 		return false;
 	}
 
-	val = 0;
-	ret = drm_dp_dpcd_read(aux, reg, &val, 1);
+	ret = drm_dp_dpcd_read_byte(aux, reg, &val);
 	if (ret < 0) {
-		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg);
+		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x (%pe)\n",
+			reg, ERR_PTR(ret));
 		return false;
 	}
 
@@ -612,12 +616,13 @@ void lspcon_set_infoframes(struct intel_encoder *encoder,
 static bool _lspcon_read_avi_infoframe_enabled_mca(struct drm_dp_aux *aux)
 {
 	int ret;
-	u32 val = 0;
 	u16 reg = LSPCON_MCA_AVI_IF_CTRL;
+	u8 val;
 
-	ret = drm_dp_dpcd_read(aux, reg, &val, 1);
+	ret = drm_dp_dpcd_read_byte(aux, reg, &val);
 	if (ret < 0) {
-		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg);
+		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x (%pe)\n",
+			reg, ERR_PTR(ret));
 		return false;
 	}
 
@@ -627,12 +632,13 @@ static bool _lspcon_read_avi_infoframe_enabled_mca(struct drm_dp_aux *aux)
 static bool _lspcon_read_avi_infoframe_enabled_parade(struct drm_dp_aux *aux)
 {
 	int ret;
-	u32 val = 0;
 	u16 reg = LSPCON_PARADE_AVI_IF_CTRL;
+	u8 val;
 
-	ret = drm_dp_dpcd_read(aux, reg, &val, 1);
+	ret = drm_dp_dpcd_read_byte(aux, reg, &val);
 	if (ret < 0) {
-		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x\n", reg);
+		drm_err(aux->drm_dev, "DPCD read failed, address 0x%x (%pe)\n",
+			reg, ERR_PTR(ret));
 		return false;
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_verify.c b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
index 2ec17c2bfe0fe2..f0761d8a7437eb 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_verify.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
@@ -245,7 +245,6 @@ void intel_modeset_verify_crtc(struct intel_atomic_state *state,
 	verify_connector_state(state, crtc);
 	verify_crtc_state(state, crtc);
 	intel_dpll_state_verify(state, crtc);
-	intel_mpllb_state_verify(state, crtc);
 }
 
 void intel_modeset_verify_disabled(struct intel_atomic_state *state)
diff --git a/drivers/gpu/drm/i915/display/intel_parent.c b/drivers/gpu/drm/i915/display/intel_parent.c
index a5e41ea6692139..99cebb17763ac1 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.c
+++ b/drivers/gpu/drm/i915/display/intel_parent.c
@@ -281,9 +281,9 @@ void intel_parent_pc8_unblock(struct intel_display *display)
 }
 
 /* pcode */
-int intel_parent_pcode_read(struct intel_display *display, u32 mbox, u32 *val, u32 *val1)
+int intel_parent_pcode_read(struct intel_display *display, u32 mbox, u32 *val0, u32 *val1)
 {
-	return display->parent->pcode->read(display->drm, mbox, val, val1);
+	return display->parent->pcode->read(display->drm, mbox, val0, val1);
 }
 
 int intel_parent_pcode_write_timeout(struct intel_display *display, u32 mbox, u32 val, int timeout_ms)
@@ -451,6 +451,12 @@ bool intel_parent_has_fenced_regions(struct intel_display *display)
 	return display->parent->has_fenced_regions && display->parent->has_fenced_regions(display->drm);
 }
 
+void intel_parent_transient_data_flush(struct intel_display *display)
+{
+	if (display->parent->transient_data_flush)
+		display->parent->transient_data_flush(display->drm);
+}
+
 bool intel_parent_vgpu_active(struct intel_display *display)
 {
 	return display->parent->vgpu_active && display->parent->vgpu_active(display->drm);
diff --git a/drivers/gpu/drm/i915/display/intel_parent.h b/drivers/gpu/drm/i915/display/intel_parent.h
index 595d4148b8eb47..cc4a58f6316637 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.h
+++ b/drivers/gpu/drm/i915/display/intel_parent.h
@@ -115,7 +115,7 @@ void intel_parent_pc8_block(struct intel_display *display);
 void intel_parent_pc8_unblock(struct intel_display *display);
 
 /* pcode */
-int intel_parent_pcode_read(struct intel_display *display, u32 mbox, u32 *val, u32 *val1);
+int intel_parent_pcode_read(struct intel_display *display, u32 mbox, u32 *val0, u32 *val1);
 int intel_parent_pcode_write_timeout(struct intel_display *display, u32 mbox, u32 val, int timeout_ms);
 int intel_parent_pcode_write(struct intel_display *display, u32 mbox, u32 val);
 int intel_parent_pcode_request(struct intel_display *display, u32 mbox, u32 request,
@@ -155,6 +155,7 @@ int intel_parent_vlv_iosf_write(struct intel_display *display, enum vlv_iosf_sb_
 /* generic */
 bool intel_parent_has_auxccs(struct intel_display *display);
 bool intel_parent_has_fenced_regions(struct intel_display *display);
+void intel_parent_transient_data_flush(struct intel_display *display);
 bool intel_parent_vgpu_active(struct intel_display *display);
 void intel_parent_fence_priority_display(struct intel_display *display, struct dma_fence *fence);
 
diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c
index d0f99a87c42e8e..25ca049009efbc 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_plane.c
@@ -462,6 +462,42 @@ intel_plane_colorop_replace_blob(struct intel_plane_state *plane_state,
 	return false;
 }
 
+static u32
+fixedmatrix_colorop_to_encoding(enum drm_colorop_fixed_matrix_type fm_type)
+{
+	switch (fm_type) {
+	case DRM_COLOROP_FM_YCBCR709_FULL_RGB:
+	case DRM_COLOROP_FM_YCBCR709_LIMITED_RGB:
+		return DRM_COLOR_YCBCR_BT709;
+
+	case DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB:
+	case DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB:
+		return DRM_COLOR_YCBCR_BT2020;
+
+	case DRM_COLOROP_FM_YCBCR601_FULL_RGB:
+	case DRM_COLOROP_FM_YCBCR601_LIMITED_RGB:
+	default:
+		return DRM_COLOR_YCBCR_BT601;
+	}
+}
+
+static u32
+fixedmatrix_colorop_to_range(enum drm_colorop_fixed_matrix_type fm_type)
+{
+	switch (fm_type) {
+	case DRM_COLOROP_FM_YCBCR601_FULL_RGB:
+	case DRM_COLOROP_FM_YCBCR709_FULL_RGB:
+	case DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB:
+		return DRM_COLOR_YCBCR_FULL_RANGE;
+
+	case DRM_COLOROP_FM_YCBCR601_LIMITED_RGB:
+	case DRM_COLOROP_FM_YCBCR709_LIMITED_RGB:
+	case DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB:
+	default:
+		return DRM_COLOR_YCBCR_LIMITED_RANGE;
+	}
+}
+
 static void
 intel_plane_color_copy_uapi_to_hw_state(struct intel_atomic_state *state,
 					struct intel_plane_state *plane_state,
@@ -474,6 +510,7 @@ intel_plane_color_copy_uapi_to_hw_state(struct intel_atomic_state *state,
 	struct drm_property_blob *blob;
 	struct intel_crtc_state *new_crtc_state = state ?
 		intel_atomic_get_new_crtc_state(state, crtc) : NULL;
+	enum drm_colorop_fixed_matrix_type fm_type;
 	bool changed = false;
 	int i = 0;
 
@@ -485,11 +522,23 @@ intel_plane_color_copy_uapi_to_hw_state(struct intel_atomic_state *state,
 	while (iter_colorop) {
 		for_each_new_colorop_in_state(&state->base, colorop, new_colorop_state, i) {
 			if (new_colorop_state->colorop == iter_colorop) {
-				blob = new_colorop_state->bypass ? NULL : new_colorop_state->data;
 				intel_colorop = to_intel_colorop(colorop);
-				changed |= intel_plane_colorop_replace_blob(plane_state,
+				if (intel_colorop->id == INTEL_PLANE_CB_CSC_FF) {
+					fm_type = new_colorop_state->fixed_matrix_type;
+
+					plane_state->hw.csc_ff_enable =
+						!new_colorop_state->bypass;
+					plane_state->hw.color_encoding =
+						fixedmatrix_colorop_to_encoding(fm_type);
+					plane_state->hw.color_range =
+						fixedmatrix_colorop_to_range(fm_type);
+				} else {
+					blob = new_colorop_state->bypass ?
+						NULL : new_colorop_state->data;
+					changed |= intel_plane_colorop_replace_blob(plane_state,
 									    intel_colorop,
 									    blob);
+				}
 			}
 		}
 		iter_colorop = iter_colorop->next;
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index 40e3d709599651..00d0146b260024 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -471,13 +471,15 @@ static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	u8 val = 8; /* assume the worst if we can't read the value */
+	int ret;
 
-	if (drm_dp_dpcd_readb(&intel_dp->aux,
-			      DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val) == 1)
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val);
+	if (!ret)
 		val &= DP_MAX_RESYNC_FRAME_COUNT_MASK;
 	else
 		drm_dbg_kms(display->drm,
-			    "Unable to get sink synchronization latency, assuming 8 frames\n");
+			    "Unable to get sink synchronization latency, assuming 8 frames (%pe)\n",
+			    ERR_PTR(ret));
 	return val;
 }
 
@@ -485,7 +487,7 @@ static void _psr_compute_su_granularity(struct intel_dp *intel_dp,
 					struct intel_connector *connector)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	ssize_t r;
+	int ret;
 	__le16 w;
 	u8 y;
 
@@ -500,21 +502,23 @@ static void _psr_compute_su_granularity(struct intel_dp *intel_dp,
 		goto exit;
 	}
 
-	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &w, sizeof(w));
-	if (r != sizeof(w))
+	ret = drm_dp_dpcd_read_data(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &w, sizeof(w));
+	if (ret < 0)
 		drm_dbg_kms(display->drm,
-			    "Unable to read selective update x granularity\n");
+			    "Unable to read selective update x granularity (%pe)\n",
+			    ERR_PTR(ret));
 	/*
 	 * Spec says that if the value read is 0 the default granularity should
 	 * be used instead.
 	 */
-	if (r != sizeof(w) || w == 0)
+	if (ret < 0 || w == 0)
 		w = cpu_to_le16(4);
 
-	r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_Y_GRANULARITY, &y, 1);
-	if (r != 1) {
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_PSR2_SU_Y_GRANULARITY, &y);
+	if (ret < 0) {
 		drm_dbg_kms(display->drm,
-			    "Unable to read selective update y granularity\n");
+			    "Unable to read selective update y granularity (%pe)\n",
+			    ERR_PTR(ret));
 		y = 4;
 	}
 	if (y == 0)
@@ -796,11 +800,11 @@ static void _panel_replay_enable_sink(struct intel_dp *intel_dp,
 		panel_replay_config[1] |=
 			DP_PANEL_REPLAY_SU_REGION_SCANLINE_CAPTURE;
 
-	drm_dp_dpcd_write(&intel_dp->aux, PANEL_REPLAY_CONFIG,
-			  panel_replay_config, sizeof(panel_replay_config));
+	drm_dp_dpcd_write_data(&intel_dp->aux, PANEL_REPLAY_CONFIG,
+			       panel_replay_config, sizeof(panel_replay_config));
 
 	panel_replay_config_3 = intel_dp_as_sdp_transmission_time();
-	drm_dp_dpcd_writeb(&intel_dp->aux, PANEL_REPLAY_CONFIG3, panel_replay_config_3);
+	drm_dp_dpcd_write_byte(&intel_dp->aux, PANEL_REPLAY_CONFIG3, panel_replay_config_3);
 }
 
 static void _psr_enable_sink(struct intel_dp *intel_dp,
@@ -827,10 +831,10 @@ static void _psr_enable_sink(struct intel_dp *intel_dp,
 
 	if (intel_dp->psr.entry_setup_frames > 0)
 		val |= DP_PSR_FRAME_CAPTURE;
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, val);
+	drm_dp_dpcd_write_byte(&intel_dp->aux, DP_PSR_EN_CFG, val);
 
 	val |= DP_PSR_ENABLE;
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, val);
+	drm_dp_dpcd_write_byte(&intel_dp->aux, DP_PSR_EN_CFG, val);
 }
 
 static void intel_psr_enable_sink(struct intel_dp *intel_dp,
@@ -843,7 +847,7 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp,
 		_psr_enable_sink(intel_dp, crtc_state);
 
 	if (intel_dp_is_edp(intel_dp))
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
+		drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 }
 
 void intel_psr_panel_replay_enable_sink(struct intel_dp *intel_dp)
@@ -854,8 +858,8 @@ void intel_psr_panel_replay_enable_sink(struct intel_dp *intel_dp)
 	 * ensure this bit is cleared/set accordingly.
 	 */
 	if (CAN_PANEL_REPLAY(intel_dp) && panel_replay_global_enabled(intel_dp))
-		drm_dp_dpcd_writeb(&intel_dp->aux, PANEL_REPLAY_CONFIG,
-				   DP_PANEL_REPLAY_ENABLE);
+		drm_dp_dpcd_write_byte(&intel_dp->aux, PANEL_REPLAY_CONFIG,
+				       DP_PANEL_REPLAY_ENABLE);
 }
 
 static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
@@ -1491,6 +1495,12 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 	int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
 	int psr_max_h = 0, psr_max_v = 0, max_bpp = 0;
 
+	if (crtc_state->vrr.cmrr.enable) {
+		drm_dbg_kms(display->drm,
+			    "PSR2 cannot be enabled when CMRR is enabled\n");
+		return false;
+	}
+
 	if (!connector->dp.psr_caps.su_support || display->params.enable_psr == 1)
 		return false;
 
@@ -2367,11 +2377,11 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 
 	/* Disable PSR on Sink */
 	if (!intel_dp->psr.panel_replay_enabled) {
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
+		drm_dp_dpcd_write_byte(&intel_dp->aux, DP_PSR_EN_CFG, 0);
 
 		if (intel_dp->psr.sel_update_enabled)
-			drm_dp_dpcd_writeb(&intel_dp->aux,
-					   DP_RECEIVER_ALPM_CONFIG, 0);
+			drm_dp_dpcd_write_byte(&intel_dp->aux,
+					       DP_RECEIVER_ALPM_CONFIG, 0);
 	}
 
 	/* Wa_16025596647 */
@@ -2883,6 +2893,8 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 				struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(state);
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
 	struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_plane_state *new_plane_state, *old_plane_state;
 	struct intel_plane *plane;
@@ -2895,6 +2907,19 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 	bool full_update = false, su_area_changed;
 	int i, ret;
 
+	/*
+	 * Selective fetch is not always usable, for instance it is dropped
+	 * while pipe CRC is active. The planes keep their selective fetch
+	 * enable bit set in hardware over that, and a plane disabled while
+	 * selective fetch is off never gets the bit cleared. Once selective
+	 * fetch comes back the hardware would resume fetching for a plane that
+	 * is no longer enabled and keep its DDB range reserved, so have the
+	 * plane update drop the bit for every plane of the pipe as selective
+	 * fetch is turned off.
+	 */
+	crtc_state->clear_psr2_sel_fetch = old_crtc_state->enable_psr2_sel_fetch &&
+		!crtc_state->enable_psr2_sel_fetch;
+
 	if (!crtc_state->enable_psr2_sel_fetch)
 		return 0;
 
@@ -3496,7 +3521,7 @@ static void intel_psr_handle_irq(struct intel_dp *intel_dp)
 	intel_psr_disable_locked(intel_dp);
 	psr->sink_not_reliable = true;
 	/* let's make sure that sink is awaken */
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
+	drm_dp_dpcd_write_byte(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 }
 
 static void intel_psr_work(struct work_struct *work)
@@ -3775,15 +3800,15 @@ static int psr_get_status_and_error_status(struct intel_dp *intel_dp,
 	offset = intel_dp->psr.panel_replay_enabled ?
 		 DP_SINK_DEVICE_PR_AND_FRAME_LOCK_STATUS : DP_PSR_STATUS;
 
-	ret = drm_dp_dpcd_readb(aux, offset, status);
-	if (ret != 1)
+	ret = drm_dp_dpcd_read_byte(aux, offset, status);
+	if (ret < 0)
 		return ret;
 
 	offset = intel_dp->psr.panel_replay_enabled ?
 		 DP_PANEL_REPLAY_ERROR_STATUS : DP_PSR_ERROR_STATUS;
 
-	ret = drm_dp_dpcd_readb(aux, offset, error_status);
-	if (ret != 1)
+	ret = drm_dp_dpcd_read_byte(aux, offset, error_status);
+	if (ret < 0)
 		return ret;
 
 	*status = *status & DP_PSR_SINK_STATE_MASK;
@@ -3808,12 +3833,12 @@ static void psr_capability_changed_check(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_psr *psr = &intel_dp->psr;
+	int ret;
 	u8 val;
-	int r;
 
-	r = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ESI, &val);
-	if (r != 1) {
-		drm_err(display->drm, "Error reading DP_PSR_ESI\n");
+	ret = drm_dp_dpcd_read_byte(&intel_dp->aux, DP_PSR_ESI, &val);
+	if (ret < 0) {
+		drm_err(display->drm, "Error reading DP_PSR_ESI (%pe)\n", ERR_PTR(ret));
 		return;
 	}
 
@@ -3824,7 +3849,7 @@ static void psr_capability_changed_check(struct intel_dp *intel_dp)
 			    "Sink PSR capability changed, disabling PSR\n");
 
 		/* Clearing it */
-		drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ESI, val);
+		drm_dp_dpcd_write_byte(&intel_dp->aux, DP_PSR_ESI, val);
 	}
 }
 
@@ -3840,6 +3865,7 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_psr *psr = &intel_dp->psr;
 	u8 status, error_status;
+	bool panel_replay_enabled;
 	const u8 errors = DP_PSR_RFB_STORAGE_ERROR |
 			  DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
 			  DP_PSR_LINK_CRC_ERROR;
@@ -3860,6 +3886,12 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp)
 		goto exit;
 	}
 
+	/*
+	 * Save this before intel_psr_disable_locked() clears it; the error
+	 * status is acknowledged to a different DPCD address depending on it.
+	 */
+	panel_replay_enabled = psr->panel_replay_enabled;
+
 	if ((!psr->panel_replay_enabled && status == DP_PSR_SINK_INTERNAL_ERROR) ||
 	    (error_status & errors)) {
 		intel_psr_disable_locked(intel_dp);
@@ -3885,7 +3917,10 @@ void intel_psr_short_pulse(struct intel_dp *intel_dp)
 			"PSR_ERROR_STATUS unhandled errors %x\n",
 			error_status & ~errors);
 	/* clear status register */
-	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, error_status);
+	drm_dp_dpcd_write_byte(&intel_dp->aux,
+			       panel_replay_enabled ?
+			       DP_PANEL_REPLAY_ERROR_STATUS : DP_PSR_ERROR_STATUS,
+			       error_status);
 
 	if (!psr->panel_replay_enabled) {
 		psr_alpm_check(intel_dp);
diff --git a/drivers/gpu/drm/i915/display/intel_psr_regs.h b/drivers/gpu/drm/i915/display/intel_psr_regs.h
index 16a9e3af198d07..bb577e7e3bbda4 100644
--- a/drivers/gpu/drm/i915/display/intel_psr_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_psr_regs.h
@@ -276,6 +276,8 @@
 #define  PR_ALPM_CTL_ADAPTIVE_SYNC_SDP_POSITION_T1_OR_T2	REG_FIELD_PREP(PR_ALPM_CTL_ADAPTIVE_SYNC_SDP_POSITION_MASK, 0)
 #define  PR_ALPM_CTL_ADAPTIVE_SYNC_SDP_POSITION_T1		REG_FIELD_PREP(PR_ALPM_CTL_ADAPTIVE_SYNC_SDP_POSITION_MASK, 1)
 #define  PR_ALPM_CTL_ADAPTIVE_SYNC_SDP_POSITION_T2		REG_FIELD_PREP(PR_ALPM_CTL_ADAPTIVE_SYNC_SDP_POSITION_MASK, 2)
+#define  PR_ALPM_CTL_AS_SDP_SKIP_FRAMES_MASK			REG_GENMASK(27, 16)
+#define  PR_ALPM_CTL_AS_SDP_SKIP_FRAMES(frames)			REG_FIELD_PREP(PR_ALPM_CTL_AS_SDP_SKIP_FRAMES_MASK, (frames))
 
 #define _ALPM_CTL_A	0x60950
 #define ALPM_CTL(dev_priv, tran)	_MMIO_TRANS2(dev_priv, tran, _ALPM_CTL_A)
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c
index 33245f44c0d501..7d7db774d8c7b9 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.c
+++ b/drivers/gpu/drm/i915/display/intel_quirks.c
@@ -257,6 +257,9 @@ static struct intel_quirk intel_quirks[] = {
 	/* Dell XPS 13 7390 2-in-1 */
 	{ 0x8a52, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 },
 
+	/* HP Pavilion Plus Laptop 14-ew1xxx */
+	{ 0x7d55, 0x103c, 0x8c31, quirk_edp_limit_rate_hbr2 },
+
 	/* Xiaomi Book Pro 14 2026 */
 	{ 0xb081, 0x1d72, 0x2424, quirk_disable_psr2 },
 };
diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c
index bf9df566630f01..ce94ac6890642f 100644
--- a/drivers/gpu/drm/i915/display/intel_snps_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c
@@ -1816,11 +1816,10 @@ int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state,
 	return -EINVAL;
 }
 
-void intel_mpllb_enable(struct intel_encoder *encoder,
-			const struct intel_crtc_state *crtc_state)
+void intel_mpllb_enable_phy(struct intel_encoder *encoder,
+			    const struct intel_mpllb_state *pll_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	const struct intel_mpllb_state *pll_state = &crtc_state->dpll_hw_state.mpllb;
 	enum phy phy = intel_encoder_to_phy(encoder);
 	intel_reg_t enable_reg = (phy <= PHY_D ?
 				 DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0));
@@ -1875,6 +1874,12 @@ void intel_mpllb_enable(struct intel_encoder *encoder,
 	 */
 }
 
+void intel_mpllb_enable(struct intel_encoder *encoder,
+			const struct intel_crtc_state *crtc_state)
+{
+	intel_mpllb_enable_phy(encoder, &crtc_state->dpll_hw_state.mpllb);
+}
+
 void intel_mpllb_disable(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
@@ -1977,52 +1982,3 @@ void intel_mpllb_readout_hw_state(struct intel_encoder *encoder,
 	 */
 	pll_state->mpllb_div &= ~SNPS_PHY_MPLLB_FORCE_EN;
 }
-
-void intel_mpllb_state_verify(struct intel_atomic_state *state,
-			      struct intel_crtc *crtc)
-{
-	struct intel_display *display = to_intel_display(state);
-	const struct intel_crtc_state *new_crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	struct intel_mpllb_state mpllb_hw_state = {};
-	const struct intel_mpllb_state *mpllb_sw_state = &new_crtc_state->dpll_hw_state.mpllb;
-	struct intel_encoder *encoder;
-
-	if (!display->platform.dg2)
-		return;
-
-	if (!new_crtc_state->hw.active)
-		return;
-
-	/* intel_get_crtc_new_encoder() only works for modeset/fastset commits */
-	if (!intel_crtc_needs_modeset(new_crtc_state) &&
-	    !intel_crtc_needs_fastset(new_crtc_state))
-		return;
-
-	encoder = intel_get_crtc_new_encoder(state, new_crtc_state);
-	intel_mpllb_readout_hw_state(encoder, &mpllb_hw_state);
-
-#define MPLLB_CHECK(__name)						\
-	INTEL_DISPLAY_STATE_WARN(display, mpllb_sw_state->__name != mpllb_hw_state.__name, \
-				 "[CRTC:%d:%s] mismatch in MPLLB: %s (expected 0x%08x, found 0x%08x)", \
-				 crtc->base.base.id, crtc->base.name,	\
-				 __stringify(__name),			\
-				 mpllb_sw_state->__name, mpllb_hw_state.__name)
-
-	MPLLB_CHECK(mpllb_cp);
-	MPLLB_CHECK(mpllb_div);
-	MPLLB_CHECK(mpllb_div2);
-	MPLLB_CHECK(mpllb_fracn1);
-	MPLLB_CHECK(mpllb_fracn2);
-	MPLLB_CHECK(mpllb_sscen);
-	MPLLB_CHECK(mpllb_sscstep);
-
-	/*
-	 * ref_control is handled by the hardware/firemware and never
-	 * programmed by the software, but the proper values are supplied
-	 * in the bspec for verification purposes.
-	 */
-	MPLLB_CHECK(ref_control);
-
-#undef MPLLB_CHECK
-}
diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.h b/drivers/gpu/drm/i915/display/intel_snps_phy.h
index 7f96da22d02801..2c7a5f2040f175 100644
--- a/drivers/gpu/drm/i915/display/intel_snps_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_snps_phy.h
@@ -22,6 +22,8 @@ void intel_snps_phy_update_psr_power_state(struct intel_encoder *encoder,
 
 int intel_mpllb_calc_state(struct intel_crtc_state *crtc_state,
 			   struct intel_encoder *encoder);
+void intel_mpllb_enable_phy(struct intel_encoder *encoder,
+			    const struct intel_mpllb_state *pll_state);
 void intel_mpllb_enable(struct intel_encoder *encoder,
 			const struct intel_crtc_state *crtc_state);
 void intel_mpllb_disable(struct intel_encoder *encoder);
diff --git a/drivers/gpu/drm/i915/display/intel_tdf.h b/drivers/gpu/drm/i915/display/intel_tdf.h
deleted file mode 100644
index 0862c2bfd9cd36..00000000000000
--- a/drivers/gpu/drm/i915/display/intel_tdf.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2024 Intel Corporation
- */
-
-#ifndef __INTEL_TDF_H__
-#define __INTEL_TDF_H__
-
-/*
- * TDF (Transient-Data-Flush) is needed for Xe2+ where special L3:XD caching can
- * be enabled through various PAT index modes. Idea is to use this caching mode
- * when for example rendering onto the display surface, with the promise that
- * KMD will ensure transient cache entries are always flushed by the time we do
- * the display flip, since display engine is never coherent with CPU/GPU caches.
- */
-
-struct intel_display;
-
-#ifdef I915
-static inline void intel_td_flush(struct intel_display *display) {}
-#else
-void intel_td_flush(struct intel_display *display);
-#endif
-
-#endif
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c
index 51e4f3309b8b16..d43a06907d18a9 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.c
+++ b/drivers/gpu/drm/i915/display/intel_vrr.c
@@ -4,13 +4,21 @@
  *
  */
 
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/string.h>
+
+#include <drm/drm_atomic.h>
 #include <drm/drm_print.h>
 #include <drm/intel/step.h>
 
 #include "intel_alpm.h"
+#include "intel_atomic.h"
 #include "intel_cmtg.h"
 #include "intel_crtc.h"
 #include "intel_de.h"
+#include "intel_dip.h"
+#include "intel_dip_regs.h"
 #include "intel_display_limits.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
@@ -23,9 +31,6 @@
 #include "skl_prefill.h"
 #include "skl_watermark.h"
 
-#define FIXED_POINT_PRECISION		100
-#define CMRR_PRECISION_TOLERANCE	10
-
 /*
  * Tunable parameters for DC Balance correction.
  * These are captured based on experimentations.
@@ -187,69 +192,87 @@ int intel_vrr_vmax_vblank_start(const struct intel_crtc_state *crtc_state)
 	return intel_vrr_vmax_vtotal(crtc_state) - crtc_state->vrr.guardband;
 }
 
-static bool
-is_cmrr_frac_required(struct intel_crtc_state *crtc_state)
+static bool intel_vrr_cmrr_possible(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
-	int calculated_refresh_k, actual_refresh_k, pixel_clock_per_line;
-	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
-
-	/* Avoid CMRR for now till we have VRR with fixed timings working */
-	if (!HAS_CMRR(display) || true)
-		return false;
-
-	actual_refresh_k =
-		drm_mode_vrefresh(adjusted_mode) * FIXED_POINT_PRECISION;
-	pixel_clock_per_line =
-		adjusted_mode->crtc_clock * 1000 / adjusted_mode->crtc_htotal;
-	calculated_refresh_k =
-		pixel_clock_per_line * FIXED_POINT_PRECISION / adjusted_mode->crtc_vtotal;
-
-	if ((actual_refresh_k - calculated_refresh_k) < CMRR_PRECISION_TOLERANCE)
-		return false;
 
-	return true;
+	return HAS_CMRR(display) && intel_vrr_always_use_vrr_tg(display);
 }
 
-static unsigned int
-cmrr_get_vtotal(struct intel_crtc_state *crtc_state, bool video_mode_required)
+static void
+intel_vrr_cmrr_compute_config(struct intel_crtc_state *crtc_state)
 {
-	int multiplier_m = 1, multiplier_n = 1, vtotal, desired_refresh_rate;
-	u64 adjusted_pixel_rate;
+	struct intel_display *display = to_intel_display(crtc_state);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	u64 dividend;
+	u32 mode_rate_mhz;
+	u32 requested_rate_mhz = crtc->force_cmrr.numerator;
+	bool video_mode = crtc->force_cmrr.denominator != 1000;
+	int rr_multiplier = video_mode ? 1000 : 1;
+	int rr_divider = video_mode ? 1001 : 1;
+
+	if (!intel_vrr_cmrr_possible(crtc_state))
+		return;
 
-	desired_refresh_rate = drm_mode_vrefresh(adjusted_mode);
+	/* No CMRR ratio configured through debugfs */
+	if (!requested_rate_mhz)
+		return;
 
-	if (video_mode_required) {
-		multiplier_m = 1001;
-		multiplier_n = 1000;
+	/* Requested rate must match the mode's nominal (integer) refresh rate */
+	if (DIV_ROUND_CLOSEST(requested_rate_mhz, 1000) !=
+			drm_mode_vrefresh(adjusted_mode)) {
+		drm_dbg_kms(display->drm,
+			    "[CRTC:%d:%s] CMRR requested %u.%03u Hz doesn't match mode %d Hz\n",
+			    crtc->base.base.id, crtc->base.name,
+			    requested_rate_mhz / 1000, requested_rate_mhz % 1000,
+			    drm_mode_vrefresh(adjusted_mode));
+		return;
 	}
 
-	crtc_state->cmrr.cmrr_n = mul_u32_u32(desired_refresh_rate * adjusted_mode->crtc_htotal,
-					      multiplier_n);
-	vtotal = DIV_ROUND_UP_ULL(mul_u32_u32(adjusted_mode->crtc_clock * 1000, multiplier_n),
-				  crtc_state->cmrr.cmrr_n);
-	adjusted_pixel_rate = mul_u32_u32(adjusted_mode->crtc_clock * 1000, multiplier_m);
-	crtc_state->cmrr.cmrr_m = do_div(adjusted_pixel_rate, crtc_state->cmrr.cmrr_n);
+	/* Actual rate produced by the current timings, in milli-Hz */
+	mode_rate_mhz =
+		DIV_ROUND_CLOSEST_ULL((u64)adjusted_mode->crtc_clock * 1000 * 1000,
+				      adjusted_mode->crtc_vtotal *
+				      adjusted_mode->crtc_htotal);
 
-	return vtotal;
-}
+	/*
+	 * 1:1 request already satisfied by the mode -> CMRR not needed.
+	 */
+	if (!video_mode && DIV_ROUND_CLOSEST(mode_rate_mhz, 10) ==
+	    DIV_ROUND_CLOSEST(requested_rate_mhz, 10)) {
+		drm_dbg_kms(display->drm,
+			    "[CRTC:%d:%s] %u.%03u Hz can be driven without CMRR\n",
+			    crtc->base.base.id, crtc->base.name,
+			    requested_rate_mhz / 1000, requested_rate_mhz % 1000);
+		return;
+	}
 
-static
-void intel_vrr_compute_cmrr_timings(struct intel_crtc_state *crtc_state)
-{
 	/*
-	 * TODO: Compute precise target refresh rate to determine
-	 * if video_mode_required should be true. Currently set to
-	 * false due to uncertainty about the precise target
-	 * refresh Rate.
+	 * Let pixel_clock_hz = adjusted_mode->crtc_clock * 1000.
+	 *
+	 * cmrr_n = (requested_rate_mhz x htotal x rr_multiplier) / 1000
+	 * cmrr_m = (pixel_clock_hz x rr_divider) % cmrr_n
+	 *
+	 * where requested_rate_mhz is the requested refresh rate in milli-Hz
+	 * and rr_multiplier/rr_divider = 1000/1001 when the video timing
+	 * is required, else 1/1. The integer vtotal term is tracked in SW
+	 * (it is the programmed mode vtotal) while the fractional part
+	 * represented by cmrr_m/cmrr_n is tracked in HW.
+	 *
+	 * TODO: Using the actual desired rate for cmrr_n in video
+	 * mode produces more aggressive vtotal dithering than
+	 * expected; revisit once the Bspec algorithm is clarified.
 	 */
-	crtc_state->vrr.vmax = cmrr_get_vtotal(crtc_state, false);
-	crtc_state->vrr.vmin = crtc_state->vrr.vmax;
-	crtc_state->vrr.flipline = crtc_state->vrr.vmin;
+	crtc_state->vrr.cmrr.cmrr_n =
+		div64_u64((u64)requested_rate_mhz *
+			  adjusted_mode->crtc_htotal * rr_multiplier, 1000);
+	dividend = (u64)adjusted_mode->crtc_clock * rr_divider * 1000;
+	adjusted_mode->crtc_vtotal = div64_u64_rem(dividend,
+						   crtc_state->vrr.cmrr.cmrr_n,
+						   &crtc_state->vrr.cmrr.cmrr_m);
 
-	crtc_state->cmrr.enable = true;
-	crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
+	crtc_state->vrr.cmrr.enable = true;
 }
 
 static
@@ -325,6 +348,36 @@ int intel_vrr_fixed_rr_hw_flipline(const struct intel_crtc_state *crtc_state)
 	return intel_vrr_fixed_rr_hw_vtotal(crtc_state);
 }
 
+static void
+intel_vrr_set_cmrr_timings(const struct intel_crtc_state *crtc_state,
+			   enum transcoder transcoder)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+
+	if (!intel_vrr_cmrr_possible(crtc_state))
+		return;
+
+	intel_de_write(display, TRANS_CMRR_M_HI(display, transcoder),
+		       upper_32_bits(crtc_state->vrr.cmrr.cmrr_m));
+	intel_de_write(display, TRANS_CMRR_M_LO(display, transcoder),
+		       lower_32_bits(crtc_state->vrr.cmrr.cmrr_m));
+	intel_de_write(display, TRANS_CMRR_N_LO(display, transcoder),
+		       lower_32_bits(crtc_state->vrr.cmrr.cmrr_n));
+	intel_de_write(display, TRANS_CMRR_N_HI(display, transcoder),
+		       upper_32_bits(crtc_state->vrr.cmrr.cmrr_n));
+
+	/*
+	 * On always-VRR-TG platforms the fastset path does not rewrite the
+	 * whole TRANS_VRR_CTL, so RMW only the CMRR enable bit here. On a
+	 * modeset the authoritative writes (intel_vrr_tg_enable(),
+	 * intel_cmtg_set_vrr_ctl()) carry the same bit, so this stays
+	 * consistent.
+	 */
+	intel_de_rmw(display, TRANS_VRR_CTL(display, transcoder),
+		     VRR_CTL_CMRR_ENABLE,
+		     crtc_state->vrr.cmrr.enable ? VRR_CTL_CMRR_ENABLE : 0);
+}
+
 void intel_vrr_set_fixed_rr_timings(const struct intel_crtc_state *crtc_state,
 				    enum transcoder transcoder)
 {
@@ -333,6 +386,8 @@ void intel_vrr_set_fixed_rr_timings(const struct intel_crtc_state *crtc_state,
 	if (!intel_vrr_possible(crtc_state))
 		return;
 
+	intel_vrr_set_cmrr_timings(crtc_state, transcoder);
+
 	intel_de_write(display, TRANS_VRR_VMIN(display, transcoder),
 		       intel_vrr_fixed_rr_hw_vmin(crtc_state) - 1);
 	intel_de_write(display, TRANS_VRR_VMAX(display, transcoder),
@@ -386,10 +441,12 @@ static bool intel_vrr_dc_balance_possible(const struct intel_crtc_state *crtc_st
 static void
 intel_vrr_dc_balance_compute_config(struct intel_crtc_state *crtc_state)
 {
+	struct intel_display *display = to_intel_display(crtc_state);
 	int guardband_usec, adjustment_usec;
 	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 
-	if (!intel_vrr_dc_balance_possible(crtc_state) || !crtc_state->vrr.enable)
+	if (!intel_vrr_dc_balance_possible(crtc_state) ||
+	    !crtc_state->vrr.enable || !display->params.enable_dc_balance)
 		return;
 
 	crtc_state->vrr.dc_balance.vmax = crtc_state->vrr.vmax;
@@ -425,8 +482,6 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	bool is_edp = intel_dp_is_edp(intel_dp);
 	struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 	int vmin, vmax;
 
@@ -460,12 +515,17 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
 		vmax = vmin;
 	}
 
-	if (crtc_state->uapi.vrr_enabled && vmin < vmax)
+	if (crtc_state->uapi.vrr_enabled && vmin < vmax) {
 		intel_vrr_compute_vrr_timings(crtc_state, vmin, vmax);
-	else if (is_cmrr_frac_required(crtc_state) && is_edp)
-		intel_vrr_compute_cmrr_timings(crtc_state);
-	else
+	} else {
+		/*
+		 * CMRR is a fixed average Vtotal mode and is only computed on
+		 * the fixed refresh rate path. It is generic across transcoders
+		 * and gated on platform support and a valid debugfs ratio.
+		 */
+		intel_vrr_cmrr_compute_config(crtc_state);
 		intel_vrr_compute_fixed_rr_timings(crtc_state);
+	}
 
 	if (HAS_AS_SDP(display)) {
 		crtc_state->vrr.vsync_start =
@@ -641,17 +701,6 @@ void intel_vrr_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
 		return;
 	}
 
-	if (crtc_state->cmrr.enable) {
-		intel_de_write(display, TRANS_CMRR_M_HI(display, cpu_transcoder),
-			       upper_32_bits(crtc_state->cmrr.cmrr_m));
-		intel_de_write(display, TRANS_CMRR_M_LO(display, cpu_transcoder),
-			       lower_32_bits(crtc_state->cmrr.cmrr_m));
-		intel_de_write(display, TRANS_CMRR_N_HI(display, cpu_transcoder),
-			       upper_32_bits(crtc_state->cmrr.cmrr_n));
-		intel_de_write(display, TRANS_CMRR_N_LO(display, cpu_transcoder),
-			       lower_32_bits(crtc_state->cmrr.cmrr_n));
-	}
-
 	intel_vrr_set_fixed_rr_timings(crtc_state, cpu_transcoder);
 	intel_cmtg_set_vrr_timings(crtc_state);
 
@@ -665,17 +714,7 @@ void intel_vrr_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
 			       VRR_VSYNC_END(crtc_state->vrr.vsync_end) |
 			       VRR_VSYNC_START(crtc_state->vrr.vsync_start));
 
-	/*
-	 * For BMG and LNL+ onwards the EMP_AS_SDP_TL is used for programming
-	 * double buffering point and transmission line for VRR packets for
-	 * HDMI2.1/DP/eDP/DP->HDMI2.1 PCON.
-	 * Since currently we support VRR only for DP/eDP, so this is programmed
-	 * to for Adaptive Sync SDP to Vsync start.
-	 */
-	if (DISPLAY_VERx100(display) == 1401 || DISPLAY_VER(display) >= 20)
-		intel_de_write(display,
-			       EMP_AS_SDP_TL(display, cpu_transcoder),
-			       EMP_AS_SDP_DB_TL(crtc_state->vrr.vsync_start));
+	intel_dip_write_emp_as_sdp_tl(crtc_state);
 }
 
 void
@@ -917,8 +956,7 @@ intel_vrr_disable_dc_balancing(const struct intel_crtc_state *old_crtc_state)
 	intel_de_write(display, TRANS_VRR_CTL(display, cpu_transcoder), vrr_ctl);
 }
 
-static void intel_vrr_tg_enable(const struct intel_crtc_state *crtc_state,
-				bool cmrr_enable)
+static void intel_vrr_tg_enable(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
@@ -930,11 +968,12 @@ static void intel_vrr_tg_enable(const struct intel_crtc_state *crtc_state,
 	vrr_ctl = VRR_CTL_VRR_ENABLE | trans_vrr_ctl(crtc_state);
 
 	/*
-	 * FIXME this might be broken as bspec seems to imply that
-	 * even VRR_CTL_CMRR_ENABLE is armed by TRANS_CMRR_N_HI
-	 * when enabling CMRR (but not when disabling CMRR?).
+	 * This full TRANS_VRR_CTL write is the authoritative one, so it must
+	 * carry VRR_CTL_CMRR_ENABLE when CMRR is in use. Writing TRANS_CMRR_N_HI
+	 * arms the bit in hardware, but this later write would otherwise clear
+	 * it again.
 	 */
-	if (cmrr_enable)
+	if (crtc_state->vrr.cmrr.enable)
 		vrr_ctl |= VRR_CTL_CMRR_ENABLE;
 
 	intel_de_write(display, TRANS_VRR_CTL(display, cpu_transcoder), vrr_ctl);
@@ -970,7 +1009,7 @@ void intel_vrr_enable(const struct intel_crtc_state *crtc_state)
 	intel_vrr_enable_dc_balancing(crtc_state);
 
 	if (!intel_vrr_always_use_vrr_tg(display))
-		intel_vrr_tg_enable(crtc_state, crtc_state->cmrr.enable);
+		intel_vrr_tg_enable(crtc_state);
 }
 
 void intel_vrr_disable(const struct intel_crtc_state *old_crtc_state)
@@ -997,7 +1036,7 @@ void intel_vrr_transcoder_enable(const struct intel_crtc_state *crtc_state)
 		return;
 
 	if (intel_vrr_always_use_vrr_tg(display))
-		intel_vrr_tg_enable(crtc_state, false);
+		intel_vrr_tg_enable(crtc_state);
 }
 
 void intel_vrr_transcoder_disable(const struct intel_crtc_state *old_crtc_state)
@@ -1066,13 +1105,13 @@ void intel_vrr_get_config(struct intel_crtc_state *crtc_state)
 	trans_vrr_ctl = intel_de_read(display,
 				      TRANS_VRR_CTL(display, cpu_transcoder));
 
-	if (HAS_CMRR(display))
-		crtc_state->cmrr.enable = (trans_vrr_ctl & VRR_CTL_CMRR_ENABLE);
+	if (intel_vrr_cmrr_possible(crtc_state))
+		crtc_state->vrr.cmrr.enable = (trans_vrr_ctl & VRR_CTL_CMRR_ENABLE);
 
-	if (crtc_state->cmrr.enable) {
-		crtc_state->cmrr.cmrr_n =
+	if (crtc_state->vrr.cmrr.enable) {
+		crtc_state->vrr.cmrr.cmrr_n =
 			intel_de_read64_2x32(display, TRANS_CMRR_N_LO(display, cpu_transcoder));
-		crtc_state->cmrr.cmrr_m =
+		crtc_state->vrr.cmrr.cmrr_m =
 			intel_de_read64_2x32(display, TRANS_CMRR_M_LO(display, cpu_transcoder));
 	}
 
@@ -1132,11 +1171,6 @@ void intel_vrr_get_config(struct intel_crtc_state *crtc_state)
 
 	intel_vrr_get_dc_balance_config(crtc_state);
 
-	/*
-	 * #TODO: For Both VRR and CMRR the flag I915_MODE_FLAG_VRR is set for mode_flags.
-	 * Since CMRR is currently disabled, set this flag for VRR for now.
-	 * Need to keep this in mind while re-enabling CMRR.
-	 */
 	if (crtc_state->vrr.enable)
 		crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
 
@@ -1231,3 +1265,186 @@ int intel_vrr_dcb_vmax_vblank_start_final(const struct intel_crtc_state *crtc_st
 
 	return intel_vrr_vblank_start(crtc_state, VRR_DCB_VMAX(tmp) + 1);
 }
+
+static
+int intel_vrr_cmrr_parse_ratio(char *str, u32 *numerator, u32 *denominator)
+{
+	char *sep;
+	int ret;
+
+	/*
+	 * Parse a "numerator/denominator" CMRR ratio string. The numerator
+	 * is the requested refresh rate in milli-Hz (refresh rate in Hz * 1000)
+	 * and the denominator selects the timing: 1000 for a 1:1 ratio
+	 * (no video timing) or 1001 for the 1000/1001 video timing.
+	 */
+
+	sep = strchr(str, '/');
+	if (!sep)
+		return -EINVAL;
+
+	*sep = '\0';
+
+	ret = kstrtou32(strim(str), 10, numerator);
+	if (ret)
+		return ret;
+
+	ret = kstrtou32(strim(sep + 1), 10, denominator);
+	if (ret)
+		return ret;
+	/*
+	 * "0/0" clears any previously configured CMRR override.
+	 * A zero numerator already means "CMRR not requested" in
+	 * intel_vrr_cmrr_compute_config(), so just let it through.
+	 */
+	if (*numerator == 0 && *denominator == 0)
+		return 0;
+
+	if (*numerator == 0)
+		return -EINVAL;
+
+	if (*denominator != 1000 && *denominator != 1001)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int intel_vrr_debugfs_target_rr_show(struct seq_file *m, void *data)
+{
+	struct intel_crtc *crtc = m->private;
+
+	seq_printf(m, "%u/%u\n", crtc->force_cmrr.numerator, crtc->force_cmrr.denominator);
+
+	return 0;
+}
+
+static int intel_vrr_debugfs_target_rr_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, intel_vrr_debugfs_target_rr_show, inode->i_private);
+}
+
+/*
+ * Force an internal commit on @crtc so that a CMRR ratio programmed
+ * via debugfs gets recomputed and latched into hardware.
+ *
+ * CMRR only alters the (average) vtotal. Depending on whether the
+ * computed vblank value (with LRR disabled) changes, this commit may or
+ * may not be downgraded to a fastset.
+ */
+static int intel_vrr_cmrr_commit_force(struct intel_crtc *crtc)
+{
+	struct intel_display *display = to_intel_display(crtc);
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_atomic_commit *state;
+	struct intel_crtc_state *crtc_state;
+	int ret = 0;
+
+	state = drm_atomic_commit_alloc(display->drm);
+	if (!state)
+		return -ENOMEM;
+
+	drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
+
+	state->acquire_ctx = &ctx;
+	to_intel_atomic_state(state)->internal = true;
+
+retry:
+	crtc_state = intel_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(crtc_state)) {
+		ret = PTR_ERR(crtc_state);
+		goto out;
+	}
+
+	if (!crtc_state->hw.active)
+		goto out;
+
+	/* Mark mode as changed to trigger a pipe recompute + update() */
+	crtc_state->uapi.mode_changed = true;
+
+	ret = drm_atomic_commit(state);
+out:
+	if (ret == -EDEADLK) {
+		drm_atomic_commit_clear(state);
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry;
+	}
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+	drm_atomic_commit_put(state);
+
+	return ret;
+}
+
+static ssize_t intel_vrr_debugfs_target_rr_write(struct file *file, const char __user *ubuf,
+						 size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct intel_crtc *crtc = m->private;
+	u32 numerator, denominator;
+	u32 old_numerator, old_denominator;
+	char kbuf[32];
+	int ret;
+
+	if (len >= sizeof(kbuf))
+		return -EINVAL;
+
+	if (copy_from_user(kbuf, ubuf, len))
+		return -EFAULT;
+
+	kbuf[len] = '\0';
+
+	ret = intel_vrr_cmrr_parse_ratio(kbuf, &numerator, &denominator);
+	if (ret)
+		return ret;
+
+	if (crtc->force_cmrr.numerator == numerator &&
+	    crtc->force_cmrr.denominator == denominator)
+		return len;
+
+	old_numerator = crtc->force_cmrr.numerator;
+	old_denominator = crtc->force_cmrr.denominator;
+
+	crtc->force_cmrr.numerator = numerator;
+	crtc->force_cmrr.denominator = denominator;
+
+	/*
+	 * The debugfs value is a side channel that is not tracked by the atomic
+	 * state, so kick an internal commit to recompute and latch the
+	 * new CMRR parameters.
+	 */
+	ret = intel_vrr_cmrr_commit_force(crtc);
+	if (ret) {
+		/*
+		 * Restore the last known good ratio so that force_cmrr does not
+		 * hold on to bad values, which would make all subsequent commits
+		 * fail.
+		 */
+		crtc->force_cmrr.numerator = old_numerator;
+		crtc->force_cmrr.denominator = old_denominator;
+		return ret;
+	}
+
+	return len;
+}
+
+static const struct file_operations intel_vrr_debugfs_target_rr_fops = {
+	.owner = THIS_MODULE,
+	.open = intel_vrr_debugfs_target_rr_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = intel_vrr_debugfs_target_rr_write,
+};
+
+void intel_vrr_crtc_debugfs_add(struct intel_crtc *crtc)
+{
+	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
+
+	if (!intel_vrr_cmrr_possible(crtc_state))
+		return;
+
+	debugfs_create_file("intel_vrr_target_refresh_rate", 0600, crtc->base.debugfs_entry,
+			    crtc, &intel_vrr_debugfs_target_rr_fops);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.h b/drivers/gpu/drm/i915/display/intel_vrr.h
index 55e9c429f5793c..19c7990be1b2ad 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.h
+++ b/drivers/gpu/drm/i915/display/intel_vrr.h
@@ -56,4 +56,6 @@ int intel_vrr_dcb_vmax_vblank_start_next(const struct intel_crtc_state *crtc_sta
 int intel_vrr_dcb_vmin_vblank_start_final(const struct intel_crtc_state *crtc_state);
 int intel_vrr_dcb_vmax_vblank_start_final(const struct intel_crtc_state *crtc_state);
 
+void intel_vrr_crtc_debugfs_add(struct intel_crtc *crtc);
+
 #endif /* __INTEL_VRR_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_vrr_regs.h b/drivers/gpu/drm/i915/display/intel_vrr_regs.h
index 9d4d6573a149c9..ba8631cbc672a3 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_vrr_regs.h
@@ -174,12 +174,6 @@
 #define   VRR_VSYNC_START_MASK			REG_GENMASK(12, 0)
 #define   VRR_VSYNC_START(vsync_start)		REG_FIELD_PREP(VRR_VSYNC_START_MASK, (vsync_start))
 
-/* Common register for HDMI EMP and DP AS SDP */
-#define _EMP_AS_SDP_TL_A			0x60204
-#define EMP_AS_SDP_TL(display, trans)		_MMIO_TRANS2((display), (trans), _EMP_AS_SDP_TL_A)
-#define   EMP_AS_SDP_DB_TL_MASK			REG_GENMASK(12, 0)
-#define   EMP_AS_SDP_DB_TL(db_transmit_line)	REG_FIELD_PREP(EMP_AS_SDP_DB_TL_MASK, (db_transmit_line))
-
 #define _TRANS_CMRR_M_LO_A			0x604F0
 #define TRANS_CMRR_M_LO(display, trans)		_MMIO_TRANS2((display), (trans), _TRANS_CMRR_M_LO_A)
 
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index 07a68329335219..8bdf1b8a022999 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -7,6 +7,7 @@
 #include <drm/drm_blend.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_print.h>
 #include <drm/intel/step.h>
 
@@ -885,7 +886,8 @@ static void icl_plane_disable_sel_fetch_arm(struct intel_dsb *dsb,
 	struct intel_display *display = to_intel_display(plane);
 	enum pipe pipe = plane->pipe;
 
-	if (!crtc_state->enable_psr2_sel_fetch)
+	if (!crtc_state->enable_psr2_sel_fetch &&
+	    !crtc_state->clear_psr2_sel_fetch)
 		return;
 
 	intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id), 0);
@@ -1240,37 +1242,47 @@ static u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
 	return plane_color_ctl;
 }
 
-static u32 glk_plane_color_ctl(const struct intel_plane_state *plane_state)
+static u32 glk_plane_color_ctl_input_csc(const struct intel_plane_state *plane_state)
 {
 	struct intel_display *display = to_intel_display(plane_state);
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-	u32 plane_color_ctl = 0;
+	bool color_pipeline = !!plane_state->uapi.color_pipeline;
+	bool needs_csc = color_pipeline ? plane_state->hw.csc_ff_enable : fb->format->is_yuv;
+	u32 ctl = 0;
 
-	plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
-	plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
+	if (!needs_csc)
+		return 0;
 
-	if (fb->format->is_yuv && !icl_is_hdr_plane(display, plane->id)) {
+	if (!icl_is_hdr_plane(display, plane->id)) {
 		switch (plane_state->hw.color_encoding) {
 		case DRM_COLOR_YCBCR_BT709:
-			plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
+			ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
 			break;
 		case DRM_COLOR_YCBCR_BT2020:
-			plane_color_ctl |=
-				PLANE_COLOR_CSC_MODE_YUV2020_TO_RGB2020;
+			ctl |= PLANE_COLOR_CSC_MODE_YUV2020_TO_RGB2020;
 			break;
 		default:
-			plane_color_ctl |=
-				PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601;
+			ctl |= PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601;
 		}
-		if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
-			plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
-	} else if (fb->format->is_yuv) {
-		plane_color_ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
-		if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
-			plane_color_ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
+	} else {
+		ctl |= PLANE_COLOR_INPUT_CSC_ENABLE;
 	}
 
+	if (plane_state->hw.color_range == DRM_COLOR_YCBCR_FULL_RANGE)
+		ctl |= PLANE_COLOR_YUV_RANGE_CORRECTION_DISABLE;
+
+	return ctl;
+}
+
+static u32 glk_plane_color_ctl(const struct intel_plane_state *plane_state)
+{
+	u32 plane_color_ctl = 0;
+
+	plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
+	plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
+	plane_color_ctl |= glk_plane_color_ctl_input_csc(plane_state);
+
 	if (plane_state->force_black)
 		plane_color_ctl |= PLANE_COLOR_PLANE_CSC_ENABLE;
 
@@ -1611,7 +1623,8 @@ icl_plane_update_noarm(struct intel_dsb *dsb,
 	intel_de_write_dsb(display, dsb, PLANE_COLOR_CTL(pipe, plane_id),
 			   plane_color_ctl);
 
-	if (fb->format->is_yuv && icl_is_hdr_plane(display, plane_id))
+	if (icl_is_hdr_plane(display, plane_id) &&
+	    (fb->format->is_yuv || plane_state->hw.csc_ff_enable))
 		icl_program_input_csc(dsb, plane, plane_state);
 
 	skl_write_plane_wm(dsb, plane, crtc_state);
@@ -1634,10 +1647,8 @@ static void icl_plane_update_sel_fetch_arm(struct intel_dsb *dsb,
 	struct intel_display *display = to_intel_display(plane);
 	enum pipe pipe = plane->pipe;
 
-	if (!crtc_state->enable_psr2_sel_fetch)
-		return;
-
-	if (drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0)
+	if (crtc_state->enable_psr2_sel_fetch &&
+	    drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0)
 		intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id),
 				   SEL_FETCH_PLANE_CTL_ENABLE);
 	else
@@ -2707,6 +2718,7 @@ static const struct drm_plane_funcs skl_plane_funcs = {
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = skl_plane_format_mod_supported,
 	.format_mod_supported_async = intel_plane_format_mod_supported_async,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static const struct drm_plane_funcs icl_plane_funcs = {
@@ -2717,6 +2729,7 @@ static const struct drm_plane_funcs icl_plane_funcs = {
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = icl_plane_format_mod_supported,
 	.format_mod_supported_async = intel_plane_format_mod_supported_async,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static const struct drm_plane_funcs tgl_plane_funcs = {
@@ -2727,6 +2740,7 @@ static const struct drm_plane_funcs tgl_plane_funcs = {
 	.atomic_destroy_state = intel_plane_destroy_state,
 	.format_mod_supported = tgl_plane_format_mod_supported,
 	.format_mod_supported_async = intel_plane_format_mod_supported_async,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static void
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index 5172d398265480..9e01f8b2079aba 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -89,6 +89,7 @@ struct drm_i915_gem_object *i915_gem_object_alloc(void)
 
 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 {
+	dma_resv_fini(&obj->base._resv);
 	return kmem_cache_free(slab_objects, obj);
 }
 
@@ -144,7 +145,6 @@ void __i915_gem_object_fini(struct drm_i915_gem_object *obj)
 {
 	mutex_destroy(&obj->mm.get_page.lock);
 	mutex_destroy(&obj->mm.get_dma_page.lock);
-	dma_resv_fini(&obj->base._resv);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
index 043095f93ac605..297fca511afb15 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c
@@ -35,6 +35,7 @@
 
 #include <linux/mmu_context.h>
 #include <linux/mempolicy.h>
+#include <linux/pagemap.h>
 #include <linux/swap.h>
 #include <linux/sched/mm.h>
 
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index c0fd349a4600c6..9dd9665128caa1 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -1607,7 +1607,7 @@ u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
 
 static unsigned long stop_timeout(const struct intel_engine_cs *engine)
 {
-	if (in_atomic() || irqs_disabled()) /* inside atomic preempt-reset? */
+	if (in_atomic() || irqs_disabled() || rcu_preempt_depth()) /* inside atomic preempt-reset? */
 		return 0;
 
 	/*
diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
index e693b0c9d2a3e2..841f7193267565 100644
--- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
@@ -1300,7 +1300,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	 * and context switches) submission.
 	 */
 
-	spin_lock(&sched_engine->lock);
+	spin_lock_irq(&sched_engine->lock);
 
 	/*
 	 * If the queue is higher priority than the last
@@ -1400,7 +1400,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 				 * Even if ELSP[1] is occupied and not worthy
 				 * of timeslices, our queue might be.
 				 */
-				spin_unlock(&sched_engine->lock);
+				spin_unlock_irq(&sched_engine->lock);
 				return;
 			}
 		}
@@ -1426,7 +1426,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 
 		if (last && !can_merge_rq(last, rq)) {
 			spin_unlock(&ve->base.sched_engine->lock);
-			spin_unlock(&engine->sched_engine->lock);
+			spin_unlock_irq(&engine->sched_engine->lock);
 			return; /* leave this for another sibling */
 		}
 
@@ -1588,7 +1588,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	 */
 	sched_engine->queue_priority_hint = queue_prio(sched_engine);
 	i915_sched_engine_reset_on_empty(sched_engine);
-	spin_unlock(&sched_engine->lock);
+	spin_unlock_irq(&sched_engine->lock);
 
 	/*
 	 * We can skip poking the HW if we ended up with exactly the same set
@@ -1614,13 +1614,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	}
 }
 
-static void execlists_dequeue_irq(struct intel_engine_cs *engine)
-{
-	local_irq_disable(); /* Suspend interrupts across request submission */
-	execlists_dequeue(engine);
-	local_irq_enable(); /* flush irq_work (e.g. breadcrumb enabling) */
-}
-
 static void clear_ports(struct i915_request **ports, int count)
 {
 	memset_p((void **)ports, NULL, count);
@@ -2475,7 +2468,7 @@ static void execlists_submission_tasklet(struct tasklet_struct *t)
 	}
 
 	if (!engine->execlists.pending[0]) {
-		execlists_dequeue_irq(engine);
+		execlists_dequeue(engine);
 		start_timeslice(engine);
 	}
 
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index 6053f3e8aa27c1..bd1f26336ac410 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -1433,13 +1433,7 @@ static void intel_gt_reset_global(struct intel_gt *gt,
 			need_display_reset;
 
 		if (reset_display) {
-			if (atomic_read(&i915->pending_fb_pin)) {
-				drm_dbg_kms(&i915->drm,
-					    "Modeset potentially stuck, unbreaking through wedging\n");
-
-				intel_gt_set_wedged(gt);
-			}
-
+			intel_gt_set_wedged(gt);
 			intel_display_reset_prepare(display);
 		}
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c
index 10e556a7eac455..c1eff9edd8a5e9 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_pm.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_pm.c
@@ -277,11 +277,11 @@ static int live_engine_busy_stats(void *arg)
 		st_engine_heartbeat_disable(engine);
 
 		ENGINE_TRACE(engine, "measuring idle time\n");
-		preempt_disable();
+		migrate_disable();
 		de = intel_engine_get_busy_time(engine, &t[0]);
 		udelay(100);
 		de = ktime_sub(intel_engine_get_busy_time(engine, &t[1]), de);
-		preempt_enable();
+		migrate_enable();
 		dt = ktime_sub(t[1], t[0]);
 		if (de < 0 || de > 10) {
 			pr_err("%s: reported %lldns [%d%%] busyness while sleeping [for %lldns]\n",
@@ -316,11 +316,11 @@ static int live_engine_busy_stats(void *arg)
 		}
 
 		ENGINE_TRACE(engine, "measuring busy time\n");
-		preempt_disable();
+		migrate_disable();
 		de = intel_engine_get_busy_time(engine, &t[0]);
 		mdelay(100);
 		de = ktime_sub(intel_engine_get_busy_time(engine, &t[1]), de);
-		preempt_enable();
+		migrate_enable();
 		dt = ktime_sub(t[1], t[0]);
 		if (100 * de < 95 * dt || 95 * de > 100 * dt) {
 			pr_err("%s: reported %lldns [%d%%] busyness while spinning [for %lldns]\n",
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
index 053780f562c1af..b25fa8f4dc4bde 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
@@ -362,7 +362,7 @@ static inline int intel_guc_send_busy_loop(struct intel_guc *guc,
 {
 	int err;
 	unsigned int sleep_period_ms = 1;
-	bool not_atomic = !in_atomic() && !irqs_disabled();
+	bool not_atomic = !in_atomic() && !irqs_disabled() && !rcu_preempt_depth();
 
 	/*
 	 * FIXME: Have caller pass in if we are in an atomic context to avoid
diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c
index 253b41789be910..794fb50e48dddb 100644
--- a/drivers/gpu/drm/i915/gvt/aperture_gm.c
+++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c
@@ -34,6 +34,8 @@
  *
  */
 
+#include <linux/string_choices.h>
+
 #include <drm/drm_print.h>
 
 #include "gt/intel_ggtt_fencing.h"
@@ -76,7 +78,7 @@ static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm)
 	mutex_unlock(&gt->ggtt->vm.mutex);
 	if (ret)
 		gvt_err("fail to alloc %s gm space from host\n",
-			high_gm ? "high" : "low");
+			str_high_low(high_gm));
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 9f61867e24781d..22dd0179863ee3 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -1706,7 +1706,7 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset,
 {
 	u32 value = *(u32 *)p_data;
 	u32 cmd = value & 0xff;
-	u32 *data0 = &vgpu_vreg_t(vgpu, GEN6_PCODE_DATA);
+	u32 *data0 = &vgpu_vreg_t(vgpu, GEN6_PCODE_DATA0);
 
 	switch (cmd) {
 	case GEN9_PCODE_READ_MEM_LATENCY:
diff --git a/drivers/gpu/drm/i915/i915_dpt.c b/drivers/gpu/drm/i915/i915_dpt.c
index fcd7cced771d52..85d872c0f38782 100644
--- a/drivers/gpu/drm/i915/i915_dpt.c
+++ b/drivers/gpu/drm/i915/i915_dpt.c
@@ -6,7 +6,6 @@
 #include <drm/drm_print.h>
 #include <drm/intel/display_parent_interface.h>
 
-#include "display/intel_display_core.h"
 #include "gem/i915_gem_domain.h"
 #include "gem/i915_gem_internal.h"
 #include "gem/i915_gem_lmem.h"
@@ -140,7 +139,6 @@ struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignm
 		pin_flags |= PIN_MAPPABLE;
 
 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
-	atomic_inc(&i915->pending_fb_pin);
 
 	for_i915_gem_ww(&ww, err, true) {
 		err = i915_gem_object_lock(dpt->obj, &ww);
@@ -170,7 +168,6 @@ struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignm
 
 	dpt->obj->mm.dirty = true;
 
-	atomic_dec(&i915->pending_fb_pin);
 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 
 	return err ? ERR_PTR(err) : vma;
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 844ed79e72114e..dafee3dcd1c521 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -315,8 +315,6 @@ struct drm_i915_private {
 	/* The TTM device structure. */
 	struct ttm_device bdev;
 
-	atomic_t pending_fb_pin;
-
 	I915_SELFTEST_DECLARE(struct i915_selftest_stash selftest;)
 
 	/*
diff --git a/drivers/gpu/drm/i915/i915_fb_pin.c b/drivers/gpu/drm/i915/i915_fb_pin.c
index 1034cb767e9f9e..63be136f1a0524 100644
--- a/drivers/gpu/drm/i915/i915_fb_pin.c
+++ b/drivers/gpu/drm/i915/i915_fb_pin.c
@@ -35,8 +35,6 @@ intel_fb_pin_to_dpt(struct drm_gem_object *_obj, struct intel_dpt *dpt,
 	if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
 		return ERR_PTR(-EINVAL);
 
-	atomic_inc(&i915->pending_fb_pin);
-
 	for_i915_gem_ww(&ww, ret, true) {
 		ret = i915_gem_object_lock(obj, &ww);
 		if (ret)
@@ -98,7 +96,6 @@ intel_fb_pin_to_dpt(struct drm_gem_object *_obj, struct intel_dpt *dpt,
 	 */
 	drm_WARN_ON(&i915->drm, i915_dpt_offset(vma));
 err:
-	atomic_dec(&i915->pending_fb_pin);
 
 	return vma;
 }
@@ -132,8 +129,6 @@ intel_fb_pin_to_ggtt(struct drm_gem_object *_obj,
 	 */
 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 
-	atomic_inc(&i915->pending_fb_pin);
-
 	pinctl = 0;
 	/* PIN_MAPPABLE limits the address to GMADR size */
 	if (pin_params->needs_low_address)
@@ -206,7 +201,6 @@ intel_fb_pin_to_ggtt(struct drm_gem_object *_obj,
 	if (ret)
 		vma = ERR_PTR(ret);
 
-	atomic_dec(&i915->pending_fb_pin);
 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 	return vma;
 }
diff --git a/drivers/gpu/drm/i915/i915_overlay.c b/drivers/gpu/drm/i915/i915_overlay.c
index 6de550a1775671..c1a7920c63247f 100644
--- a/drivers/gpu/drm/i915/i915_overlay.c
+++ b/drivers/gpu/drm/i915/i915_overlay.c
@@ -354,14 +354,11 @@ static struct i915_vma *i915_overlay_pin_fb(struct drm_device *drm,
 					    struct drm_gem_object *obj,
 					    u32 *offset)
 {
-	struct drm_i915_private *i915 = to_i915(drm);
 	struct drm_i915_gem_object *new_bo = to_intel_bo(obj);
 	struct i915_gem_ww_ctx ww;
 	struct i915_vma *vma;
 	int ret;
 
-	atomic_inc(&i915->pending_fb_pin);
-
 	i915_gem_ww_ctx_init(&ww, true);
 retry:
 	ret = i915_gem_object_lock(new_bo, &ww);
@@ -377,8 +374,6 @@ static struct i915_vma *i915_overlay_pin_fb(struct drm_device *drm,
 	}
 	i915_gem_ww_ctx_fini(&ww);
 
-	atomic_dec(&i915->pending_fb_pin);
-
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 64e906380131a9..d873072b7a92e9 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -697,7 +697,7 @@
 #define    EDRAM_WAYS_IDX(cap)			(((cap) >> 5) & 0x7)
 #define    EDRAM_SETS_IDX(cap)			(((cap) >> 8) & 0x3)
 
-#define GEN6_PCODE_DATA				_MMIO(0x138128)
+#define GEN6_PCODE_DATA0			_MMIO(0x138128)
 #define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT	8
 #define   GEN6_PCODE_FREQ_RING_RATIO_SHIFT	16
 #define GEN6_PCODE_DATA1			_MMIO(0x13812C)
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index d2c7b1090df081..f66f8efc70629a 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -610,7 +610,6 @@ bool __i915_request_submit(struct i915_request *request)
 
 	RQ_TRACE(request, "\n");
 
-	GEM_BUG_ON(!irqs_disabled());
 	lockdep_assert_held(&engine->sched_engine->lock);
 
 	/*
@@ -719,7 +718,6 @@ void __i915_request_unsubmit(struct i915_request *request)
 	 */
 	RQ_TRACE(request, "\n");
 
-	GEM_BUG_ON(!irqs_disabled());
 	lockdep_assert_held(&engine->sched_engine->lock);
 
 	/*
diff --git a/drivers/gpu/drm/i915/i915_switcheroo.c b/drivers/gpu/drm/i915/i915_switcheroo.c
index 7e0791024282e2..d97057722fde7d 100644
--- a/drivers/gpu/drm/i915/i915_switcheroo.c
+++ b/drivers/gpu/drm/i915/i915_switcheroo.c
@@ -5,6 +5,7 @@
 
 #include <linux/vga_switcheroo.h>
 
+#include <drm/drm_client_event.h>
 #include <drm/drm_print.h>
 
 #include "display/intel_display_device.h"
@@ -58,10 +59,18 @@ static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
 		atomic_read(&i915->drm.open_count) == 0;
 }
 
+static void i915_switcheroo_pre_switch(struct pci_dev *pdev)
+{
+	struct drm_i915_private *i915 = pdev_to_i915(pdev);
+
+	if (i915 && intel_display_device_present(i915->display))
+		drm_client_dev_acquire_outputs(&i915->drm);
+}
+
 static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
 	.set_gpu_state = i915_switcheroo_set_state,
-	.reprobe = NULL,
 	.can_switch = i915_switcheroo_can_switch,
+	.pre_switch = i915_switcheroo_pre_switch,
 };
 
 int i915_switcheroo_register(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.c b/drivers/gpu/drm/i915/intel_clock_gating.c
index 86bdeb20f427d7..c5c4441f3a6151 100644
--- a/drivers/gpu/drm/i915/intel_clock_gating.c
+++ b/drivers/gpu/drm/i915/intel_clock_gating.c
@@ -57,12 +57,12 @@ static void bxt_init_clock_gating(struct drm_i915_private *i915)
 	 */
 	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 
-	intel_display_bxt_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 }
 
 static void glk_init_clock_gating(struct drm_i915_private *i915)
 {
-	intel_display_glk_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 }
 
 static void ilk_init_clock_gating(struct drm_i915_private *i915)
@@ -73,7 +73,7 @@ static void ilk_init_clock_gating(struct drm_i915_private *i915)
 	intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
 			   VFMUNIT_CLOCK_GATE_DISABLE);
 
-	intel_display_ilk_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 	intel_pch_init_clock_gating(i915->display);
 }
 
@@ -90,7 +90,7 @@ static void gen6_check_mch_setup(struct drm_i915_private *i915)
 
 static void gen6_init_clock_gating(struct drm_i915_private *i915)
 {
-	intel_display_gen6_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 
 	intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
 			   intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
@@ -170,7 +170,7 @@ static void cfl_init_clock_gating(struct drm_i915_private *i915)
 	/* WAC6entrylatency:cfl */
 	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
-	intel_display_cfl_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 }
 
 static void kbl_init_clock_gating(struct drm_i915_private *i915)
@@ -188,7 +188,7 @@ static void kbl_init_clock_gating(struct drm_i915_private *i915)
 		intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
 				 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 
-	intel_display_kbl_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 }
 
 static void skl_init_clock_gating(struct drm_i915_private *i915)
@@ -200,18 +200,16 @@ static void skl_init_clock_gating(struct drm_i915_private *i915)
 	/* WAC6entrylatency:skl */
 	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
-	intel_display_skl_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 }
 
 static void bdw_init_clock_gating(struct drm_i915_private *i915)
 {
-	intel_display_bdw_clock_gating_disable_fbcq(i915->display);
+	intel_display_init_clock_gating(i915->display);
 
 	/* WaSwitchSolVfFArbitrationPriority:bdw */
 	intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 
-	intel_display_bdw_clock_gating_vblank_in_srd(i915->display);
-
 	/* WaVSRefCountFullforceMissDisable:bdw */
 	/* WaDSRefCountFullforceMissDisable:bdw */
 	intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
@@ -226,8 +224,6 @@ static void bdw_init_clock_gating(struct drm_i915_private *i915)
 	/* WaProgramL3SqcReg1Default:bdw */
 	gen8_set_l3sqc_credits(i915, 30, 2);
 
-	intel_display_bdw_clock_gating_kvm_notif(i915->display);
-
 	intel_pch_init_clock_gating(i915->display);
 
 	/* WaDisableDopClockGating:bdw
@@ -240,7 +236,7 @@ static void bdw_init_clock_gating(struct drm_i915_private *i915)
 
 static void hsw_init_clock_gating(struct drm_i915_private *i915)
 {
-	intel_display_hsw_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 
 	/* This is required by WaCatErrorRejectionIssue:hsw */
 	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
@@ -256,7 +252,7 @@ static void ivb_init_clock_gating(struct drm_i915_private *i915)
 {
 	struct intel_display *display = i915->display;
 
-	intel_display_ivb_init_clock_gating(display);
+	intel_display_init_clock_gating(display);
 
 	/* WaDisableBackToBackFlipFix:ivb */
 	intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
@@ -285,8 +281,6 @@ static void ivb_init_clock_gating(struct drm_i915_private *i915)
 	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-	intel_display_disable_trickle_feed(display);
-
 	intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
 			 GEN6_MBC_SNPCR_MED);
 
@@ -362,7 +356,7 @@ static void g4x_init_clock_gating(struct drm_i915_private *i915)
 			   GS_UNIT_CLOCK_GATE_DISABLE |
 			   CL_UNIT_CLOCK_GATE_DISABLE);
 	intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
-	intel_display_g4x_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 }
 
 static void i965gm_init_clock_gating(struct drm_i915_private *i915)
@@ -371,7 +365,7 @@ static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 
 	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
-	intel_display_i965gm_init_clock_gating(i915->display);
+	intel_display_init_clock_gating(i915->display);
 	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 	intel_uncore_write16(uncore, DEUC, 0);
 	intel_uncore_write(uncore,
diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
index de118fae0a4973..25b72b20e709c8 100644
--- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
+++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
@@ -611,7 +611,7 @@ static int iterate_generic_mmio(struct intel_gvt_mmio_table_iter *iter)
 	MMIO_D(GEN6_UCGCTL1);
 	MMIO_D(GEN6_UCGCTL2);
 	MMIO_F(_MMIO(0x4f000), 0x90);
-	MMIO_D(GEN6_PCODE_DATA);
+	MMIO_D(GEN6_PCODE_DATA0);
 	MMIO_D(_MMIO(0x13812c));
 	MMIO_D(GEN7_ERR_INT);
 	MMIO_D(HSW_EDRAM_CAP);
diff --git a/drivers/gpu/drm/i915/intel_pcode.c b/drivers/gpu/drm/i915/intel_pcode.c
index c07d48fc1b3550..1c9d1f41375700 100644
--- a/drivers/gpu/drm/i915/intel_pcode.c
+++ b/drivers/gpu/drm/i915/intel_pcode.c
@@ -58,7 +58,7 @@ static int gen7_check_mailbox_status(u32 mbox)
 }
 
 static int __snb_pcode_rw(struct intel_uncore *uncore, u32 mbox,
-			  u32 *val, u32 *val1,
+			  u32 *val0, u32 *val1,
 			  int fast_timeout_us, int slow_timeout_ms,
 			  bool is_read)
 {
@@ -73,7 +73,7 @@ static int __snb_pcode_rw(struct intel_uncore *uncore, u32 mbox,
 	if (intel_uncore_read_fw(uncore, GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY)
 		return -EAGAIN;
 
-	intel_uncore_write_fw(uncore, GEN6_PCODE_DATA, *val);
+	intel_uncore_write_fw(uncore, GEN6_PCODE_DATA0, *val0);
 	intel_uncore_write_fw(uncore, GEN6_PCODE_DATA1, val1 ? *val1 : 0);
 	intel_uncore_write_fw(uncore,
 			      GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
@@ -87,7 +87,7 @@ static int __snb_pcode_rw(struct intel_uncore *uncore, u32 mbox,
 		return -ETIMEDOUT;
 
 	if (is_read)
-		*val = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA);
+		*val0 = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA0);
 	if (is_read && val1)
 		*val1 = intel_uncore_read_fw(uncore, GEN6_PCODE_DATA1);
 
@@ -97,12 +97,12 @@ static int __snb_pcode_rw(struct intel_uncore *uncore, u32 mbox,
 		return gen6_check_mailbox_status(mbox);
 }
 
-int snb_pcode_read(struct intel_uncore *uncore, u32 mbox, u32 *val, u32 *val1)
+int snb_pcode_read(struct intel_uncore *uncore, u32 mbox, u32 *val0, u32 *val1)
 {
 	int err;
 
 	mutex_lock(&uncore->i915->sb_lock);
-	err = __snb_pcode_rw(uncore, mbox, val, val1, 500, 20, true);
+	err = __snb_pcode_rw(uncore, mbox, val0, val1, 500, 20, true);
 	mutex_unlock(&uncore->i915->sb_lock);
 
 	if (err) {
@@ -278,11 +278,11 @@ int snb_pcode_write_p(struct intel_uncore *uncore, u32 mbcmd, u32 p1, u32 p2, u3
 	return err;
 }
 
-static int intel_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1)
+static int intel_pcode_read(struct drm_device *drm, u32 mbox, u32 *val0, u32 *val1)
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 
-	return snb_pcode_read(&i915->uncore, mbox, val, val1);
+	return snb_pcode_read(&i915->uncore, mbox, val0, val1);
 }
 
 static int intel_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms)
diff --git a/drivers/gpu/drm/i915/intel_pcode.h b/drivers/gpu/drm/i915/intel_pcode.h
index 19795ea8172e7a..deecc64bc1b36a 100644
--- a/drivers/gpu/drm/i915/intel_pcode.h
+++ b/drivers/gpu/drm/i915/intel_pcode.h
@@ -11,7 +11,7 @@
 struct drm_device;
 struct intel_uncore;
 
-int snb_pcode_read(struct intel_uncore *uncore, u32 mbox, u32 *val, u32 *val1);
+int snb_pcode_read(struct intel_uncore *uncore, u32 mbox, u32 *val0, u32 *val1);
 int snb_pcode_write_timeout(struct intel_uncore *uncore, u32 mbox, u32 val, int timeout_ms);
 #define snb_pcode_write(uncore, mbox, val) \
 	snb_pcode_write_timeout((uncore), (mbox), (val), 1)
diff --git a/drivers/gpu/drm/i915/selftests/igt_atomic.c b/drivers/gpu/drm/i915/selftests/igt_atomic.c
index fb506b6990956b..8ae39cf570b766 100644
--- a/drivers/gpu/drm/i915/selftests/igt_atomic.c
+++ b/drivers/gpu/drm/i915/selftests/igt_atomic.c
@@ -39,7 +39,14 @@ static void __hardirq_end(void)
 	local_irq_enable();
 }
 
+static void __maybe_unused __nop(void)
+{}
+
 const struct igt_atomic_section igt_atomic_phases[] = {
+#if IS_ENABLED(CONFIG_PREEMPT_RT)
+	{ "sleeping", __nop, __nop },
+	{ },
+#endif
 	{ "preempt", __preempt_begin, __preempt_end },
 	{ "softirq", __softirq_begin, __softirq_end },
 	{ "hardirq", __hardirq_begin, __hardirq_end },
diff --git a/drivers/gpu/drm/i915/vlv_iosf_sb.c b/drivers/gpu/drm/i915/vlv_iosf_sb.c
index 1f0332b4ad0df5..fede1735bc0835 100644
--- a/drivers/gpu/drm/i915/vlv_iosf_sb.c
+++ b/drivers/gpu/drm/i915/vlv_iosf_sb.c
@@ -3,6 +3,8 @@
  * Copyright © 2013-2021 Intel Corporation
  */
 
+#include <linux/string_choices.h>
+
 #include <drm/drm_print.h>
 #include <drm/intel/display_parent_interface.h>
 
@@ -101,7 +103,7 @@ static int vlv_sideband_rw(struct drm_i915_private *i915,
 				    VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
 				    5)) {
 		drm_dbg(&i915->drm, "IOSF sideband idle wait (%s) timed out\n",
-			is_read ? "read" : "write");
+			str_read_write(is_read));
 		return -EAGAIN;
 	}
 
@@ -125,7 +127,7 @@ static int vlv_sideband_rw(struct drm_i915_private *i915,
 		err = 0;
 	} else {
 		drm_dbg(&i915->drm, "IOSF sideband finish wait (%s) timed out\n",
-			is_read ? "read" : "write");
+			str_read_write(is_read));
 		err = -ETIMEDOUT;
 	}
 
diff --git a/drivers/gpu/drm/imagination/pvr_ccb.c b/drivers/gpu/drm/imagination/pvr_ccb.c
index b702d122d7918a..e408df74853d49 100644
--- a/drivers/gpu/drm/imagination/pvr_ccb.c
+++ b/drivers/gpu/drm/imagination/pvr_ccb.c
@@ -17,6 +17,7 @@
 #include <linux/jiffies.h>
 #include <linux/kernel.h>
 #include <linux/mutex.h>
+#include <linux/overflow.h>
 #include <linux/types.h>
 #include <linux/workqueue.h>
 
@@ -82,11 +83,6 @@ pvr_ccb_init(struct pvr_device *pvr_dev, struct pvr_ccb *pvr_ccb,
 	pvr_fw_object_get_fw_addr(pvr_ccb->ctrl_obj, &pvr_ccb->ctrl_fw_addr);
 	pvr_fw_object_get_fw_addr(pvr_ccb->ccb_obj, &pvr_ccb->ccb_fw_addr);
 
-	WRITE_ONCE(pvr_ccb->ctrl->write_offset, 0);
-	WRITE_ONCE(pvr_ccb->ctrl->read_offset, 0);
-	WRITE_ONCE(pvr_ccb->ctrl->wrap_mask, num_cmds - 1);
-	WRITE_ONCE(pvr_ccb->ctrl->cmd_size, cmd_size);
-
 	return 0;
 
 err_free_ctrl:
@@ -527,6 +523,7 @@ void pvr_kccb_wake_up_waiters(struct pvr_device *pvr_dev)
  */
 void pvr_kccb_fini(struct pvr_device *pvr_dev)
 {
+	pvr_fw_object_unmap_and_destroy(pvr_dev->kccb.rtn_obj);
 	pvr_ccb_fini(&pvr_dev->kccb.ccb);
 	WARN_ON(!list_empty(&pvr_dev->kccb.waiters));
 	WARN_ON(pvr_dev->kccb.reserved_count);
@@ -543,14 +540,42 @@ void pvr_kccb_fini(struct pvr_device *pvr_dev)
 int
 pvr_kccb_init(struct pvr_device *pvr_dev)
 {
-	pvr_dev->kccb.slot_count = 1 << ROGUE_FWIF_KCCB_NUMCMDS_LOG2_DEFAULT;
+	const u32 num_slots_log2 = ROGUE_FWIF_KCCB_NUMCMDS_LOG2_DEFAULT;
+	const u32 num_slots = 1 << num_slots_log2;
+	u32 rtn_size;
+	int err;
+
+	/*
+	 * The inputs here are compile-time constants; there's no reason to try
+	 * to gracefully handle overflow at runtime.
+	 */
+	BUILD_BUG_ON(check_mul_overflow(num_slots, sizeof(*pvr_dev->kccb.rtn), &rtn_size));
+
+	pvr_dev->kccb.slot_count = num_slots;
 	INIT_LIST_HEAD(&pvr_dev->kccb.waiters);
 	pvr_dev->kccb.fence_ctx.id = dma_fence_context_alloc(1);
 	spin_lock_init(&pvr_dev->kccb.fence_ctx.lock);
 
-	return pvr_ccb_init(pvr_dev, &pvr_dev->kccb.ccb,
-			    ROGUE_FWIF_KCCB_NUMCMDS_LOG2_DEFAULT,
-			    sizeof(struct rogue_fwif_kccb_cmd));
+	err = pvr_ccb_init(pvr_dev, &pvr_dev->kccb.ccb, num_slots_log2,
+			   sizeof(struct rogue_fwif_kccb_cmd));
+	if (err)
+		return err;
+
+	/* Allocate memory for KCCB return slots. */
+	pvr_dev->kccb.rtn = pvr_fw_object_create_and_map(pvr_dev, rtn_size,
+							 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
+							 NULL, NULL, &pvr_dev->kccb.rtn_obj);
+	if (IS_ERR(pvr_dev->kccb.rtn)) {
+		err = PTR_ERR(pvr_dev->kccb.rtn);
+		goto err_ccb_fini;
+	}
+
+	return 0;
+
+err_ccb_fini:
+	pvr_ccb_fini(&pvr_dev->kccb.ccb);
+
+	return err;
 }
 
 /**
diff --git a/drivers/gpu/drm/imagination/pvr_device.c b/drivers/gpu/drm/imagination/pvr_device.c
index 54fe4180c73c73..35eaa54f75edfe 100644
--- a/drivers/gpu/drm/imagination/pvr_device.c
+++ b/drivers/gpu/drm/imagination/pvr_device.c
@@ -697,25 +697,7 @@ pvr_device_gpu_init(struct pvr_device *pvr_dev)
 	if (err)
 		return err;
 
-	if (pvr_dev->fw_dev.processor_type != PVR_FW_PROCESSOR_TYPE_MIPS) {
-		pvr_dev->kernel_vm_ctx = pvr_vm_create_context(pvr_dev, false);
-		if (IS_ERR(pvr_dev->kernel_vm_ctx))
-			return PTR_ERR(pvr_dev->kernel_vm_ctx);
-	}
-
-	err = pvr_fw_init(pvr_dev);
-	if (err)
-		goto err_vm_ctx_put;
-
-	return 0;
-
-err_vm_ctx_put:
-	if (pvr_dev->fw_dev.processor_type != PVR_FW_PROCESSOR_TYPE_MIPS) {
-		pvr_vm_context_put(pvr_dev->kernel_vm_ctx);
-		pvr_dev->kernel_vm_ctx = NULL;
-	}
-
-	return err;
+	return pvr_fw_init(pvr_dev);
 }
 
 /**
@@ -726,11 +708,6 @@ static void
 pvr_device_gpu_fini(struct pvr_device *pvr_dev)
 {
 	pvr_fw_fini(pvr_dev);
-
-	if (pvr_dev->fw_dev.processor_type != PVR_FW_PROCESSOR_TYPE_MIPS) {
-		WARN_ON(!pvr_vm_context_put(pvr_dev->kernel_vm_ctx));
-		pvr_dev->kernel_vm_ctx = NULL;
-	}
 }
 
 /**
diff --git a/drivers/gpu/drm/imagination/pvr_free_list.c b/drivers/gpu/drm/imagination/pvr_free_list.c
index e85cac83834c63..faf5e586d8dc17 100644
--- a/drivers/gpu/drm/imagination/pvr_free_list.c
+++ b/drivers/gpu/drm/imagination/pvr_free_list.c
@@ -8,6 +8,7 @@
 #include "pvr_vm.h"
 
 #include <drm/drm_gem.h>
+#include <drm/drm_print.h>
 #include <linux/slab.h>
 #include <linux/xarray.h>
 #include <uapi/drm/pvr_drm.h>
@@ -612,13 +613,21 @@ pvr_free_list_process_reconstruct_req(struct pvr_device *pvr_dev,
 	};
 	struct rogue_fwif_freelists_reconstruction_data *resp =
 		&resp_cmd.cmd_data.free_lists_reconstruction_data;
+	u32 count = min_t(u32, req->freelist_count,
+			  ARRAY_SIZE(req->freelist_ids));
 
-	for (u32 i = 0; i < req->freelist_count; i++)
+	if (count != req->freelist_count) {
+		drm_warn_once(from_pvr_device(pvr_dev),
+			      "Requested reconstruction of %u freelists, limiting to %u\n",
+			      req->freelist_count, count);
+	}
+
+	for (u32 i = 0; i < count; i++)
 		pvr_free_list_reconstruct(pvr_dev, req->freelist_ids[i]);
 
-	resp->freelist_count = req->freelist_count;
+	resp->freelist_count = count;
 	memcpy(resp->freelist_ids, req->freelist_ids,
-	       req->freelist_count * sizeof(resp->freelist_ids[0]));
+	       count * sizeof(resp->freelist_ids[0]));
 
 	WARN_ON(pvr_kccb_send_cmd(pvr_dev, &resp_cmd, NULL));
 }
diff --git a/drivers/gpu/drm/imagination/pvr_fw.c b/drivers/gpu/drm/imagination/pvr_fw.c
index 850a3ec8e7750f..c7988278df3f2d 100644
--- a/drivers/gpu/drm/imagination/pvr_fw.c
+++ b/drivers/gpu/drm/imagination/pvr_fw.c
@@ -278,37 +278,6 @@ pvr_fw_find_mmu_segment(struct pvr_device *pvr_dev, u32 addr, u32 size, void *fw
 	return -EINVAL;
 }
 
-static int
-pvr_fw_create_fwif_connection_ctl(struct pvr_device *pvr_dev)
-{
-	struct drm_device *drm_dev = from_pvr_device(pvr_dev);
-	struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
-
-	fw_dev->fwif_connection_ctl =
-		pvr_fw_object_create_and_map_offset(pvr_dev,
-						    fw_dev->fw_heap_info.config_offset +
-						    PVR_ROGUE_FWIF_CONNECTION_CTL_OFFSET,
-						    sizeof(*fw_dev->fwif_connection_ctl),
-						    PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
-						    NULL, NULL,
-						    &fw_dev->mem.fwif_connection_ctl_obj);
-	if (IS_ERR(fw_dev->fwif_connection_ctl)) {
-		drm_err(drm_dev,
-			"Unable to allocate FWIF connection control memory\n");
-		return PTR_ERR(fw_dev->fwif_connection_ctl);
-	}
-
-	return 0;
-}
-
-static void
-pvr_fw_fini_fwif_connection_ctl(struct pvr_device *pvr_dev)
-{
-	struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
-
-	pvr_fw_object_unmap_and_destroy(fw_dev->mem.fwif_connection_ctl_obj);
-}
-
 static void
 fw_osinit_init(void *cpu_ptr, void *priv)
 {
@@ -459,21 +428,34 @@ pvr_fw_create_structures(struct pvr_device *pvr_dev)
 	struct pvr_fw_mem *fw_mem = &fw_dev->mem;
 	int err;
 
+	err = pvr_fw_object_create_offset(pvr_dev,
+					  fw_dev->fw_heap_info.config_offset +
+					  PVR_ROGUE_FWIF_CONNECTION_CTL_OFFSET,
+					  sizeof(struct rogue_fwif_connection_ctl),
+					  PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
+					  NULL, NULL,
+					  &fw_mem->fwif_connection_ctl_obj);
+	if (err) {
+		drm_err(drm_dev,
+			"Unable to allocate FWIF connection control memory\n");
+		return err;
+	}
+
 	fw_dev->power_sync = pvr_fw_object_create_and_map(pvr_dev, sizeof(*fw_dev->power_sync),
 							  PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 							  NULL, NULL, &fw_mem->power_sync_obj);
 	if (IS_ERR(fw_dev->power_sync)) {
 		drm_err(drm_dev, "Unable to allocate FW power_sync structure\n");
-		return PTR_ERR(fw_dev->power_sync);
+		err = PTR_ERR(fw_dev->power_sync);
+		goto err_release_connection_ctl;
 	}
 
-	fw_dev->hwrinfobuf = pvr_fw_object_create_and_map(pvr_dev, sizeof(*fw_dev->hwrinfobuf),
-							  PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
-							  NULL, NULL, &fw_mem->hwrinfobuf_obj);
-	if (IS_ERR(fw_dev->hwrinfobuf)) {
+	err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrinfobuf),
+				   PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
+				   NULL, NULL, &fw_mem->hwrinfobuf_obj);
+	if (err) {
 		drm_err(drm_dev,
 			"Unable to allocate FW hwrinfobuf structure\n");
-		err = PTR_ERR(fw_dev->hwrinfobuf);
 		goto err_release_power_sync;
 	}
 
@@ -536,16 +518,14 @@ pvr_fw_create_structures(struct pvr_device *pvr_dev)
 		goto err_fw_trace_fini;
 	}
 
-	fw_dev->fwif_osinit =
-		pvr_fw_object_create_and_map_offset(pvr_dev,
-						    fw_dev->fw_heap_info.config_offset +
-						    PVR_ROGUE_FWIF_OSINIT_OFFSET,
-						    sizeof(*fw_dev->fwif_osinit),
-						    PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
-						    fw_osinit_init, pvr_dev, &fw_mem->osinit_obj);
-	if (IS_ERR(fw_dev->fwif_osinit)) {
+	err = pvr_fw_object_create_offset(pvr_dev,
+					  fw_dev->fw_heap_info.config_offset +
+					  PVR_ROGUE_FWIF_OSINIT_OFFSET,
+					  sizeof(struct rogue_fwif_osinit),
+					  PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
+					  fw_osinit_init, pvr_dev, &fw_mem->osinit_obj);
+	if (err) {
 		drm_err(drm_dev, "Unable to allocate FW OSINIT structure\n");
-		err = PTR_ERR(fw_dev->fwif_osinit);
 		goto err_release_osdata;
 	}
 
@@ -565,7 +545,7 @@ pvr_fw_create_structures(struct pvr_device *pvr_dev)
 	return 0;
 
 err_release_osinit:
-	pvr_fw_object_unmap_and_destroy(fw_mem->osinit_obj);
+	pvr_fw_object_destroy(fw_mem->osinit_obj);
 
 err_release_osdata:
 	pvr_fw_object_unmap_and_destroy(fw_mem->osdata_obj);
@@ -589,11 +569,14 @@ pvr_fw_create_structures(struct pvr_device *pvr_dev)
 	pvr_fw_object_destroy(fw_mem->mmucache_sync_obj);
 
 err_release_hwrinfobuf:
-	pvr_fw_object_unmap_and_destroy(fw_mem->hwrinfobuf_obj);
+	pvr_fw_object_destroy(fw_mem->hwrinfobuf_obj);
 
 err_release_power_sync:
 	pvr_fw_object_unmap_and_destroy(fw_mem->power_sync_obj);
 
+err_release_connection_ctl:
+	pvr_fw_object_destroy(fw_mem->fwif_connection_ctl_obj);
+
 	return err;
 }
 
@@ -611,10 +594,11 @@ pvr_fw_destroy_structures(struct pvr_device *pvr_dev)
 	pvr_fw_object_unmap_and_destroy(fw_mem->sysinit_obj);
 
 	pvr_fw_object_destroy(fw_mem->mmucache_sync_obj);
-	pvr_fw_object_unmap_and_destroy(fw_mem->hwrinfobuf_obj);
+	pvr_fw_object_destroy(fw_mem->hwrinfobuf_obj);
 	pvr_fw_object_unmap_and_destroy(fw_mem->power_sync_obj);
 	pvr_fw_object_unmap_and_destroy(fw_mem->osdata_obj);
-	pvr_fw_object_unmap_and_destroy(fw_mem->osinit_obj);
+	pvr_fw_object_destroy(fw_mem->osinit_obj);
+	pvr_fw_object_destroy(fw_mem->fwif_connection_ctl_obj);
 }
 
 /**
@@ -755,10 +739,6 @@ pvr_fw_process(struct pvr_device *pvr_dev)
 	pvr_fw_object_vunmap(fw_mem->code_obj);
 	fw_code_ptr = NULL;
 
-	err = pvr_fw_create_fwif_connection_ctl(pvr_dev);
-	if (err)
-		goto err_free_kdata;
-
 	return 0;
 
 err_free_kdata:
@@ -842,8 +822,6 @@ pvr_fw_cleanup(struct pvr_device *pvr_dev)
 {
 	struct pvr_fw_mem *fw_mem = &pvr_dev->fw_dev.mem;
 
-	pvr_fw_fini_fwif_connection_ctl(pvr_dev);
-
 	kfree(fw_mem->core_data);
 	kfree(fw_mem->core_code);
 	kfree(fw_mem->data);
@@ -945,8 +923,6 @@ pvr_fw_init(struct pvr_device *pvr_dev)
 		[PVR_FW_PROCESSOR_TYPE_RISCV] = &pvr_fw_defs_riscv,
 	};
 
-	u32 kccb_size_log2 = ROGUE_FWIF_KCCB_NUMCMDS_LOG2_DEFAULT;
-	u32 kccb_rtn_size = (1 << kccb_size_log2) * sizeof(*pvr_dev->kccb.rtn);
 	struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 	int err;
 
@@ -981,18 +957,9 @@ pvr_fw_init(struct pvr_device *pvr_dev)
 	if (err)
 		goto err_kccb_fini;
 
-	/* Allocate memory for KCCB return slots. */
-	pvr_dev->kccb.rtn = pvr_fw_object_create_and_map(pvr_dev, kccb_rtn_size,
-							 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
-							 NULL, NULL, &pvr_dev->kccb.rtn_obj);
-	if (IS_ERR(pvr_dev->kccb.rtn)) {
-		err = PTR_ERR(pvr_dev->kccb.rtn);
-		goto err_fwccb_fini;
-	}
-
 	err = pvr_fw_create_structures(pvr_dev);
 	if (err)
-		goto err_kccb_rtn_release;
+		goto err_fwccb_fini;
 
 	err = pvr_fw_start(pvr_dev);
 	if (err)
@@ -1014,9 +981,6 @@ pvr_fw_init(struct pvr_device *pvr_dev)
 err_destroy_structures:
 	pvr_fw_destroy_structures(pvr_dev);
 
-err_kccb_rtn_release:
-	pvr_fw_object_unmap_and_destroy(pvr_dev->kccb.rtn_obj);
-
 err_fwccb_fini:
 	pvr_ccb_fini(&pvr_dev->fwccb);
 
@@ -1029,8 +993,7 @@ pvr_fw_init(struct pvr_device *pvr_dev)
 err_mm_takedown:
 	drm_mm_takedown(&fw_dev->fw_mm);
 
-	if (fw_dev->defs->fini)
-		fw_dev->defs->fini(pvr_dev);
+	fw_dev->defs->fini(pvr_dev);
 
 	return err;
 }
@@ -1047,7 +1010,6 @@ pvr_fw_fini(struct pvr_device *pvr_dev)
 	WRITE_ONCE(fw_dev->initialised, false);
 
 	pvr_fw_destroy_structures(pvr_dev);
-	pvr_fw_object_unmap_and_destroy(pvr_dev->kccb.rtn_obj);
 
 	/*
 	 * Ensure FWCCB worker has finished executing before destroying FWCCB. The IRQ handler has
@@ -1063,8 +1025,7 @@ pvr_fw_fini(struct pvr_device *pvr_dev)
 
 	drm_mm_takedown(&fw_dev->fw_mm);
 
-	if (fw_dev->defs->fini)
-		fw_dev->defs->fini(pvr_dev);
+	fw_dev->defs->fini(pvr_dev);
 }
 
 /**
@@ -1419,6 +1380,46 @@ pvr_fw_object_create_and_map_offset(struct pvr_device *pvr_dev,
 						   fw_obj_out);
 }
 
+/**
+ * pvr_fw_object_create_offset() - Create a FW object and map to firmware
+ * at the provided offset.
+ * @pvr_dev: PowerVR device pointer.
+ * @fw_addr: Base address of desired FW mapping, offset from start of FW heap.
+ *           If a specific FW address is not required, pass %U32_MAX.
+ * @size: Size of object, in bytes.
+ * @flags: Options which affect both this operation and future mapping
+ * operations performed on the returned object. Must be a combination of
+ * DRM_PVR_BO_* and/or PVR_BO_* flags.
+ * @init: Initialisation callback.
+ * @init_priv: Private pointer to pass to initialisation callback.
+ * @fw_obj_out: Pointer to location to store created object pointer.
+ *
+ * %DRM_PVR_BO_DEVICE_PM_FW_PROTECT is implied for all FW objects. Consequently,
+ * this function will fail if @flags has %DRM_PVR_BO_CPU_ALLOW_USERSPACE_ACCESS
+ * set.
+ *
+ * Returns:
+ *  * 0 on success, or
+ *  * Any error returned by pvr_fw_object_create_and_map().
+ */
+int
+pvr_fw_object_create_offset(struct pvr_device *pvr_dev, u32 fw_addr,
+			    size_t size, u64 flags,
+			    void (*init)(void *cpu_ptr, void *priv),
+			    void *init_priv, struct pvr_fw_object **fw_obj_out)
+{
+	void *cpu_ptr;
+
+	cpu_ptr = pvr_fw_object_create_and_map_offset(pvr_dev, fw_addr, size, flags,
+						      init, init_priv, fw_obj_out);
+	if (IS_ERR(cpu_ptr))
+		return PTR_ERR(cpu_ptr);
+
+	pvr_fw_object_vunmap(*fw_obj_out);
+
+	return 0;
+}
+
 /**
  * pvr_fw_object_destroy() - Destroy a pvr_fw_object
  * @fw_obj: Pointer to object to destroy.
diff --git a/drivers/gpu/drm/imagination/pvr_fw.h b/drivers/gpu/drm/imagination/pvr_fw.h
index 3390c84e4fd316..626a1fdb6a4d73 100644
--- a/drivers/gpu/drm/imagination/pvr_fw.h
+++ b/drivers/gpu/drm/imagination/pvr_fw.h
@@ -86,7 +86,7 @@ struct pvr_fw_defs {
 	 * FW processor specific finalisation.
 	 * @pvr_dev: Target PowerVR device.
 	 *
-	 * This function is optional.
+	 * This function is mandatory.
 	 */
 	void (*fini)(struct pvr_device *pvr_dev);
 
@@ -349,30 +349,18 @@ struct pvr_fw_device {
 	/** @fw_mm_base: Base address of address space managed by @fw_mm. */
 	u64 fw_mm_base;
 
-	/**
-	 * @fwif_connection_ctl: Pointer to CPU mapping of FWIF connection
-	 *                       control structure.
-	 */
-	struct rogue_fwif_connection_ctl *fwif_connection_ctl;
-
 	/** @fwif_sysinit: Pointer to CPU mapping of FW SYSINIT structure. */
 	struct rogue_fwif_sysinit *fwif_sysinit;
 
 	/** @fwif_sysdata: Pointer to CPU mapping of FW SYSDATA structure. */
 	struct rogue_fwif_sysdata *fwif_sysdata;
 
-	/** @fwif_osinit: Pointer to CPU mapping of FW OSINIT structure. */
-	struct rogue_fwif_osinit *fwif_osinit;
-
 	/** @fwif_osdata: Pointer to CPU mapping of FW OSDATA structure. */
 	struct rogue_fwif_osdata *fwif_osdata;
 
 	/** @power_sync: Pointer to CPU mapping of power sync state. */
 	u32 *power_sync;
 
-	/** @hwrinfobuf: Pointer to CPU mapping of FW HWR info buffer. */
-	struct rogue_fwif_hwrinfobuf *hwrinfobuf;
-
 	/** @fw_trace: Device firmware trace buffer state. */
 	struct pvr_fw_trace fw_trace;
 
@@ -430,6 +418,12 @@ void *pvr_fw_object_create_and_map(struct pvr_device *pvr_dev, size_t size, u64
 				   void (*init)(void *cpu_ptr, void *priv),
 				   void *init_priv, struct pvr_fw_object **pvr_obj_out);
 
+int
+pvr_fw_object_create_offset(struct pvr_device *pvr_dev, u32 fw_addr,
+			    size_t size, u64 flags,
+			    void (*init)(void *cpu_ptr, void *priv),
+			    void *init_priv, struct pvr_fw_object **pvr_obj_out);
+
 void *
 pvr_fw_object_create_and_map_offset(struct pvr_device *pvr_dev, u32 dev_offset, size_t size,
 				    u64 flags, void (*init)(void *cpu_ptr, void *priv),
diff --git a/drivers/gpu/drm/imagination/pvr_fw_meta.c b/drivers/gpu/drm/imagination/pvr_fw_meta.c
index 9ff03bc60a0815..84f568b5510f01 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_meta.c
+++ b/drivers/gpu/drm/imagination/pvr_fw_meta.c
@@ -500,18 +500,33 @@ pvr_meta_init(struct pvr_device *pvr_dev)
 {
 	pvr_fw_heap_info_init(pvr_dev, ROGUE_FW_HEAP_META_SHIFT, 0);
 
+	pvr_dev->kernel_vm_ctx = pvr_vm_create_context(pvr_dev, false);
+	if (IS_ERR(pvr_dev->kernel_vm_ctx))
+		return PTR_ERR(pvr_dev->kernel_vm_ctx);
+
 	return 0;
 }
 
+static void
+pvr_meta_fini(struct pvr_device *pvr_dev)
+{
+	WARN_ON(!pvr_vm_context_put(pvr_dev->kernel_vm_ctx));
+	pvr_dev->kernel_vm_ctx = NULL;
+}
+
 static u32
 pvr_meta_get_fw_addr_with_offset(struct pvr_fw_object *fw_obj, u32 offset)
 {
 	u32 fw_addr = fw_obj->fw_addr_offset + offset + ROGUE_FW_SEGMMU_DATA_BASE_ADDRESS;
 
 	/* META cacheability is determined by address. */
-	if (fw_obj->gem->flags & PVR_BO_FW_FLAGS_DEVICE_UNCACHED)
+	if (fw_obj->gem->flags & PVR_BO_FW_FLAGS_DEVICE_UNCACHED) {
 		fw_addr |= ROGUE_FW_SEGMMU_DATA_META_UNCACHED |
 			   ROGUE_FW_SEGMMU_DATA_VIVT_SLC_UNCACHED;
+	} else {
+		fw_addr |= ROGUE_FW_SEGMMU_DATA_META_CACHED |
+			   ROGUE_FW_SEGMMU_DATA_VIVT_SLC_CACHED;
+	}
 
 	return fw_addr;
 }
@@ -550,6 +565,7 @@ pvr_meta_irq_clear(struct pvr_device *pvr_dev)
 
 const struct pvr_fw_defs pvr_fw_defs_meta = {
 	.init = pvr_meta_init,
+	.fini = pvr_meta_fini,
 	.fw_process = pvr_meta_fw_process,
 	.vm_map = pvr_meta_vm_map,
 	.vm_unmap = pvr_meta_vm_unmap,
diff --git a/drivers/gpu/drm/imagination/pvr_fw_riscv.c b/drivers/gpu/drm/imagination/pvr_fw_riscv.c
index fc13d483be9a3e..cd84635bb88d5f 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_riscv.c
+++ b/drivers/gpu/drm/imagination/pvr_fw_riscv.c
@@ -23,27 +23,33 @@ static int
 pvr_riscv_wrapper_init(struct pvr_device *pvr_dev)
 {
 	const u64 common_opts =
-		((u64)(ROGUE_FW_HEAP_RISCV_SIZE >> FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT)
-		 << ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT) |
+		((u64)(ROGUE_FW_HEAP_RISCV_SIZE
+		       >> ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_SIZE_ALIGNSHIFT)
+		 << ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_SIZE_SHIFT) |
 		((u64)MMU_CONTEXT_MAPPING_FWPRIV
-		 << FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT);
+		 << ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_CBASE_SHIFT);
 
 	u64 code_addr = pvr_fw_obj_get_gpu_addr(pvr_dev->fw_dev.mem.code_obj);
 	u64 data_addr = pvr_fw_obj_get_gpu_addr(pvr_dev->fw_dev.mem.data_obj);
 
 	/* This condition allows us to OR the addresses into the register directly. */
-	static_assert(ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_SHIFT ==
-		      ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSHIFT);
+	static_assert(ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_SHIFT ==
+		      ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_ALIGNSHIFT);
 
-	WARN_ON(code_addr & ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK);
-	WARN_ON(data_addr & ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK);
+	WARN_ON(code_addr & ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_CLRMSK);
+	WARN_ON(data_addr & ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_CLRMSK);
 
-	pvr_cr_write64(pvr_dev, ROGUE_RISCVFW_REGION_REMAP_CR(BOOTLDR_CODE),
-		       code_addr | common_opts | ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN);
+	pvr_cr_write64(pvr_dev,
+		       PVR_CR_REPEAT_X(ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG,
+				       ROGUE_RISCV_REGION_BOOTLDR_CODE),
+		       code_addr | common_opts |
+		       ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_FETCH_EN_EN);
 
-	pvr_cr_write64(pvr_dev, ROGUE_RISCVFW_REGION_REMAP_CR(BOOTLDR_DATA),
+	pvr_cr_write64(pvr_dev,
+		       PVR_CR_REPEAT_X(ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG,
+				       ROGUE_RISCV_REGION_BOOTLDR_DATA),
 		       data_addr | common_opts |
-			       ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN);
+		       ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_LOAD_STORE_EN_EN);
 
 	/* Garten IDLE bit controlled by RISC-V. */
 	pvr_cr_write64(pvr_dev, ROGUE_CR_MTS_GARTEN_WRAPPER_CONFIG,
@@ -103,9 +109,20 @@ pvr_riscv_init(struct pvr_device *pvr_dev)
 {
 	pvr_fw_heap_info_init(pvr_dev, ROGUE_FW_HEAP_RISCV_SHIFT, 0);
 
+	pvr_dev->kernel_vm_ctx = pvr_vm_create_context(pvr_dev, false);
+	if (IS_ERR(pvr_dev->kernel_vm_ctx))
+		return PTR_ERR(pvr_dev->kernel_vm_ctx);
+
 	return 0;
 }
 
+static void
+pvr_riscv_fini(struct pvr_device *pvr_dev)
+{
+	WARN_ON(!pvr_vm_context_put(pvr_dev->kernel_vm_ctx));
+	pvr_dev->kernel_vm_ctx = NULL;
+}
+
 static u32
 pvr_riscv_get_fw_addr_with_offset(struct pvr_fw_object *fw_obj, u32 offset)
 {
@@ -154,6 +171,7 @@ pvr_riscv_irq_clear(struct pvr_device *pvr_dev)
 
 const struct pvr_fw_defs pvr_fw_defs_riscv = {
 	.init = pvr_riscv_init,
+	.fini = pvr_riscv_fini,
 	.fw_process = pvr_riscv_fw_process,
 	.vm_map = pvr_riscv_vm_map,
 	.vm_unmap = pvr_riscv_vm_unmap,
diff --git a/drivers/gpu/drm/imagination/pvr_fw_startstop.c b/drivers/gpu/drm/imagination/pvr_fw_startstop.c
index e24ed6fc4362f3..98ca2692239d0c 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_startstop.c
+++ b/drivers/gpu/drm/imagination/pvr_fw_startstop.c
@@ -43,19 +43,25 @@ rogue_bif_init(struct pvr_device *pvr_dev)
 	pc_dma_addr = pvr_vm_get_page_table_root_addr(pvr_dev->kernel_vm_ctx);
 
 	/* Write the kernel catalogue base. */
-	pc_addr = ((((u64)pc_dma_addr >> ROGUE_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT)
-		    << ROGUE_CR_BIF_CAT_BASE0_ADDR_SHIFT) &
-		   ~ROGUE_CR_BIF_CAT_BASE0_ADDR_CLRMSK);
+	pc_addr = ((((u64)pc_dma_addr >> ROGUE_CR_BIF_CAT_BASEx_ADDR_ALIGNSHIFT)
+		    << ROGUE_CR_BIF_CAT_BASEx_ADDR_SHIFT) &
+		   ~ROGUE_CR_BIF_CAT_BASEx_ADDR_CLRMSK);
 
-	pvr_cr_write64(pvr_dev, BIF_CAT_BASEX(MMU_CONTEXT_MAPPING_FWPRIV),
+	pvr_cr_write64(pvr_dev,
+		       PVR_CR_REPEAT_X(ROGUE_CR_BIF_CAT_BASE,
+				       MMU_CONTEXT_MAPPING_FWPRIV),
 		       pc_addr);
 
 	if (pvr_dev->fw_dev.processor_type == PVR_FW_PROCESSOR_TYPE_RISCV) {
-		pc_addr = (((u64)pc_dma_addr >> ROGUE_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT)
-			   << ROGUE_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT) &
-			  ~ROGUE_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK;
-
-		pvr_cr_write64(pvr_dev, FWCORE_MEM_CAT_BASEX(MMU_CONTEXT_MAPPING_FWPRIV), pc_addr);
+		pc_addr = (((u64)pc_dma_addr
+			    >> ROGUE_CR_FWCORE_MEM_CAT_BASEx_ADDR_ALIGNSHIFT)
+			   << ROGUE_CR_FWCORE_MEM_CAT_BASEx_ADDR_SHIFT) &
+			  ~ROGUE_CR_FWCORE_MEM_CAT_BASEx_ADDR_CLRMSK;
+
+		pvr_cr_write64(pvr_dev,
+			       PVR_CR_REPEAT_X(ROGUE_CR_FWCORE_MEM_CAT_BASE,
+					       MMU_CONTEXT_MAPPING_FWPRIV),
+			       pc_addr);
 	}
 }
 
diff --git a/drivers/gpu/drm/imagination/pvr_fw_trace.c b/drivers/gpu/drm/imagination/pvr_fw_trace.c
index 805d9f9bc1dd9e..6fe478f64f027f 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_trace.c
+++ b/drivers/gpu/drm/imagination/pvr_fw_trace.c
@@ -101,7 +101,6 @@ tracebuf_ctrl_init(void *cpu_ptr, void *priv)
 		pvr_fw_object_get_fw_addr(trace_buffer->buf_obj,
 					  &tracebuf_space->trace_buffer_fw_addr);
 
-		tracebuf_space->trace_buffer = trace_buffer->buf;
 		tracebuf_space->trace_pointer = 0;
 	}
 }
diff --git a/drivers/gpu/drm/imagination/pvr_mmu.c b/drivers/gpu/drm/imagination/pvr_mmu.c
index 3cac482e10347c..62eae7fcd5a26a 100644
--- a/drivers/gpu/drm/imagination/pvr_mmu.c
+++ b/drivers/gpu/drm/imagination/pvr_mmu.c
@@ -12,6 +12,7 @@
 #include "pvr_rogue_mmu_defs.h"
 
 #include <drm/drm_drv.h>
+#include <drm/drm_print.h>
 #include <linux/atomic.h>
 #include <linux/bitops.h>
 #include <linux/dma-mapping.h>
@@ -2335,6 +2336,7 @@ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx)
  * pvr_mmu_op_context_create() - Create an MMU op context.
  * @ctx: MMU context associated with owning VM context.
  * @sgt: Scatter gather table containing pages pinned for use by this context.
+ * @device_addr: Virtual device address at the start of the requested mapping.
  * @sgt_offset: Start offset of the requested device-virtual memory mapping.
  * @size: Size in bytes of the requested device-virtual memory mapping. For an
  * unmapping, this should be zero so that no page tables are allocated.
@@ -2346,8 +2348,9 @@ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx)
  */
 struct pvr_mmu_op_context *
 pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt,
-			  u64 sgt_offset, u64 size)
+			  u64 device_addr, u64 sgt_offset, u64 size)
 {
+	u64 start_addr = device_addr + sgt_offset;
 	int err;
 
 	struct pvr_mmu_op_context *op_ctx = kzalloc_obj(*op_ctx);
@@ -2363,16 +2366,16 @@ pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt,
 	if (size) {
 		/*
 		 * The number of page table objects we need to prealloc is
-		 * indicated by the mapping size, start offset and the sizes
+		 * indicated by the mapping size, start address and the sizes
 		 * of the areas mapped per PT or PD. The range calculation is
 		 * identical to that for the index into a table for a device
 		 * address, so we reuse those functions here.
 		 */
-		const u32 l1_start_idx = pvr_page_table_l2_idx(sgt_offset);
-		const u32 l1_end_idx = pvr_page_table_l2_idx(sgt_offset + size);
+		const u32 l1_start_idx = pvr_page_table_l2_idx(start_addr);
+		const u32 l1_end_idx = pvr_page_table_l2_idx(start_addr + size);
 		const u32 l1_count = l1_end_idx - l1_start_idx + 1;
-		const u32 l0_start_idx = pvr_page_table_l1_idx(sgt_offset);
-		const u32 l0_end_idx = pvr_page_table_l1_idx(sgt_offset + size);
+		const u32 l0_start_idx = pvr_page_table_l1_idx(start_addr);
+		const u32 l0_end_idx = pvr_page_table_l1_idx(start_addr + size);
 		const u32 l0_count = l0_end_idx - l0_start_idx + 1;
 
 		/*
@@ -2553,7 +2556,9 @@ pvr_mmu_map_sgl(struct pvr_mmu_op_context *op_ctx, struct scatterlist *sgl,
 
 err_destroy_pages:
 	memcpy(&op_ctx->curr_page, &ptr_copy, sizeof(op_ctx->curr_page));
-	err = pvr_mmu_op_context_unmap_curr_page(op_ctx, page);
+	if (pvr_mmu_op_context_unmap_curr_page(op_ctx, page))
+		drm_err(from_pvr_device(op_ctx->mmu_ctx->pvr_dev),
+			"%s : Failure in unmapping pages\n", __func__);
 
 	return err;
 }
diff --git a/drivers/gpu/drm/imagination/pvr_mmu.h b/drivers/gpu/drm/imagination/pvr_mmu.h
index a8ecd460168dcd..2c02d61ba0a29f 100644
--- a/drivers/gpu/drm/imagination/pvr_mmu.h
+++ b/drivers/gpu/drm/imagination/pvr_mmu.h
@@ -99,7 +99,7 @@ dma_addr_t pvr_mmu_get_root_table_dma_addr(struct pvr_mmu_context *ctx);
 void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx);
 struct pvr_mmu_op_context *
 pvr_mmu_op_context_create(struct pvr_mmu_context *ctx,
-			  struct sg_table *sgt, u64 sgt_offset, u64 size);
+			  struct sg_table *sgt, u64 device_addr, u64 sgt_offset, u64 size);
 
 int pvr_mmu_map(struct pvr_mmu_op_context *op_ctx, u64 size, u64 flags,
 		u64 device_addr);
diff --git a/drivers/gpu/drm/imagination/pvr_rogue_cr_defs.h b/drivers/gpu/drm/imagination/pvr_rogue_cr_defs.h
index 790c97f80a2ac0..45d784f2a82c55 100644
--- a/drivers/gpu/drm/imagination/pvr_rogue_cr_defs.h
+++ b/drivers/gpu/drm/imagination/pvr_rogue_cr_defs.h
@@ -2344,101 +2344,48 @@
 #define ROGUE_CR_ISP_STATUS_EOR_EN 0x00000001U
 
 /* Register group: ROGUE_CR_ISP_XTP_RESUME, with 64 repeats */
-#define ROGUE_CR_ISP_XTP_RESUME_REPEATCOUNT 64U
-/* Register ROGUE_CR_ISP_XTP_RESUME0 */
-#define ROGUE_CR_ISP_XTP_RESUME0 0x3A00U
-#define ROGUE_CR_ISP_XTP_RESUME0_MASKFULL 0x00000000003FF3FFULL
-#define ROGUE_CR_ISP_XTP_RESUME0_TILE_X_SHIFT 12U
-#define ROGUE_CR_ISP_XTP_RESUME0_TILE_X_CLRMSK 0xFFC00FFFU
-#define ROGUE_CR_ISP_XTP_RESUME0_TILE_Y_SHIFT 0U
-#define ROGUE_CR_ISP_XTP_RESUME0_TILE_Y_CLRMSK 0xFFFFFC00U
+#define ROGUE_CR_ISP_XTP_RESUME_REPEAT_COUNT 64U
+#define ROGUE_CR_ISP_XTP_RESUME_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_ISP_XTP_RESUME_REPEAT_BASE 0x3A00U
+/* Register ROGUE_CR_ISP_XTP_RESUMEx */
+#define ROGUE_CR_ISP_XTP_RESUMEx_MASKFULL 0x00000000003FF3FFULL
+#define ROGUE_CR_ISP_XTP_RESUMEx_TILE_X_SHIFT 12U
+#define ROGUE_CR_ISP_XTP_RESUMEx_TILE_X_CLRMSK 0xFFC00FFFU
+#define ROGUE_CR_ISP_XTP_RESUMEx_TILE_Y_SHIFT 0U
+#define ROGUE_CR_ISP_XTP_RESUMEx_TILE_Y_CLRMSK 0xFFFFFC00U
 
 /* Register group: ROGUE_CR_ISP_XTP_STORE, with 32 repeats */
-#define ROGUE_CR_ISP_XTP_STORE_REPEATCOUNT 32U
-/* Register ROGUE_CR_ISP_XTP_STORE0 */
-#define ROGUE_CR_ISP_XTP_STORE0 0x3C00U
-#define ROGUE_CR_ISP_XTP_STORE0_MASKFULL 0x000000007F3FF3FFULL
-#define ROGUE_CR_ISP_XTP_STORE0_ACTIVE_SHIFT 30U
-#define ROGUE_CR_ISP_XTP_STORE0_ACTIVE_CLRMSK 0xBFFFFFFFU
-#define ROGUE_CR_ISP_XTP_STORE0_ACTIVE_EN 0x40000000U
-#define ROGUE_CR_ISP_XTP_STORE0_EOR_SHIFT 29U
-#define ROGUE_CR_ISP_XTP_STORE0_EOR_CLRMSK 0xDFFFFFFFU
-#define ROGUE_CR_ISP_XTP_STORE0_EOR_EN 0x20000000U
-#define ROGUE_CR_ISP_XTP_STORE0_TILE_LAST_SHIFT 28U
-#define ROGUE_CR_ISP_XTP_STORE0_TILE_LAST_CLRMSK 0xEFFFFFFFU
-#define ROGUE_CR_ISP_XTP_STORE0_TILE_LAST_EN 0x10000000U
-#define ROGUE_CR_ISP_XTP_STORE0_MT_SHIFT 24U
-#define ROGUE_CR_ISP_XTP_STORE0_MT_CLRMSK 0xF0FFFFFFU
-#define ROGUE_CR_ISP_XTP_STORE0_TILE_X_SHIFT 12U
-#define ROGUE_CR_ISP_XTP_STORE0_TILE_X_CLRMSK 0xFFC00FFFU
-#define ROGUE_CR_ISP_XTP_STORE0_TILE_Y_SHIFT 0U
-#define ROGUE_CR_ISP_XTP_STORE0_TILE_Y_CLRMSK 0xFFFFFC00U
+#define ROGUE_CR_ISP_XTP_STORE_REPEAT_COUNT 32U
+#define ROGUE_CR_ISP_XTP_STORE_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_ISP_XTP_STORE_REPEAT_BASE 0x3C00U
+/* Register ROGUE_CR_ISP_XTP_STOREx */
+#define ROGUE_CR_ISP_XTP_STOREx_MASKFULL 0x000000007F3FF3FFULL
+#define ROGUE_CR_ISP_XTP_STOREx_ACTIVE_SHIFT 30U
+#define ROGUE_CR_ISP_XTP_STOREx_ACTIVE_CLRMSK 0xBFFFFFFFU
+#define ROGUE_CR_ISP_XTP_STOREx_ACTIVE_EN 0x40000000U
+#define ROGUE_CR_ISP_XTP_STOREx_EOR_SHIFT 29U
+#define ROGUE_CR_ISP_XTP_STOREx_EOR_CLRMSK 0xDFFFFFFFU
+#define ROGUE_CR_ISP_XTP_STOREx_EOR_EN 0x20000000U
+#define ROGUE_CR_ISP_XTP_STOREx_TILE_LAST_SHIFT 28U
+#define ROGUE_CR_ISP_XTP_STOREx_TILE_LAST_CLRMSK 0xEFFFFFFFU
+#define ROGUE_CR_ISP_XTP_STOREx_TILE_LAST_EN 0x10000000U
+#define ROGUE_CR_ISP_XTP_STOREx_MT_SHIFT 24U
+#define ROGUE_CR_ISP_XTP_STOREx_MT_CLRMSK 0xF0FFFFFFU
+#define ROGUE_CR_ISP_XTP_STOREx_TILE_X_SHIFT 12U
+#define ROGUE_CR_ISP_XTP_STOREx_TILE_X_CLRMSK 0xFFC00FFFU
+#define ROGUE_CR_ISP_XTP_STOREx_TILE_Y_SHIFT 0U
+#define ROGUE_CR_ISP_XTP_STOREx_TILE_Y_CLRMSK 0xFFFFFC00U
 
 /* Register group: ROGUE_CR_BIF_CAT_BASE, with 8 repeats */
-#define ROGUE_CR_BIF_CAT_BASE_REPEATCOUNT 8U
-/* Register ROGUE_CR_BIF_CAT_BASE0 */
-#define ROGUE_CR_BIF_CAT_BASE0 0x1200U
-#define ROGUE_CR_BIF_CAT_BASE0_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE0_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE0_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE0_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE0_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_CAT_BASE1 */
-#define ROGUE_CR_BIF_CAT_BASE1 0x1208U
-#define ROGUE_CR_BIF_CAT_BASE1_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE1_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE1_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE1_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE1_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_CAT_BASE2 */
-#define ROGUE_CR_BIF_CAT_BASE2 0x1210U
-#define ROGUE_CR_BIF_CAT_BASE2_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE2_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE2_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE2_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE2_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_CAT_BASE3 */
-#define ROGUE_CR_BIF_CAT_BASE3 0x1218U
-#define ROGUE_CR_BIF_CAT_BASE3_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE3_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE3_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE3_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE3_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_CAT_BASE4 */
-#define ROGUE_CR_BIF_CAT_BASE4 0x1220U
-#define ROGUE_CR_BIF_CAT_BASE4_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE4_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE4_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE4_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE4_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_CAT_BASE5 */
-#define ROGUE_CR_BIF_CAT_BASE5 0x1228U
-#define ROGUE_CR_BIF_CAT_BASE5_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE5_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE5_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE5_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE5_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_CAT_BASE6 */
-#define ROGUE_CR_BIF_CAT_BASE6 0x1230U
-#define ROGUE_CR_BIF_CAT_BASE6_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE6_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE6_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE6_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE6_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_CAT_BASE7 */
-#define ROGUE_CR_BIF_CAT_BASE7 0x1238U
-#define ROGUE_CR_BIF_CAT_BASE7_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_BIF_CAT_BASE7_ADDR_SHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE7_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_BIF_CAT_BASE7_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_CAT_BASE7_ADDR_ALIGNSIZE 4096U
+#define ROGUE_CR_BIF_CAT_BASE_REPEAT_COUNT 8U
+#define ROGUE_CR_BIF_CAT_BASE_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_BIF_CAT_BASE_REPEAT_BASE 0x1200U
+/* Register ROGUE_CR_BIF_CAT_BASEx */
+#define ROGUE_CR_BIF_CAT_BASEx_MASKFULL 0x000000FFFFFFF000ULL
+#define ROGUE_CR_BIF_CAT_BASEx_ADDR_SHIFT 12U
+#define ROGUE_CR_BIF_CAT_BASEx_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
+#define ROGUE_CR_BIF_CAT_BASEx_ADDR_ALIGNSHIFT 12U
+#define ROGUE_CR_BIF_CAT_BASEx_ADDR_ALIGNSIZE 4096U
 
 /* Register ROGUE_CR_BIF_CAT_BASE_INDEX */
 #define ROGUE_CR_BIF_CAT_BASE_INDEX 0x1240U
@@ -2715,142 +2662,24 @@
 #define ROGUE_CR_BIF_MMU_STATUS_BUSY_EN 0x00000001U
 
 /* Register group: ROGUE_CR_BIF_TILING_CFG, with 8 repeats */
-#define ROGUE_CR_BIF_TILING_CFG_REPEATCOUNT 8U
-/* Register ROGUE_CR_BIF_TILING_CFG0 */
-#define ROGUE_CR_BIF_TILING_CFG0 0x12D8U
-#define ROGUE_CR_BIF_TILING_CFG0_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG0_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG0_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG0_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG0_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG0_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG0_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG0_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG0_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG0_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG0_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG0_MIN_ADDRESS_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_TILING_CFG1 */
-#define ROGUE_CR_BIF_TILING_CFG1 0x12E0U
-#define ROGUE_CR_BIF_TILING_CFG1_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG1_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG1_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG1_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG1_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG1_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG1_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG1_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG1_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG1_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG1_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG1_MIN_ADDRESS_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_TILING_CFG2 */
-#define ROGUE_CR_BIF_TILING_CFG2 0x12E8U
-#define ROGUE_CR_BIF_TILING_CFG2_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG2_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG2_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG2_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG2_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG2_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG2_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG2_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG2_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG2_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG2_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG2_MIN_ADDRESS_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_TILING_CFG3 */
-#define ROGUE_CR_BIF_TILING_CFG3 0x12F0U
-#define ROGUE_CR_BIF_TILING_CFG3_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG3_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG3_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG3_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG3_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG3_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG3_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG3_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG3_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG3_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG3_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG3_MIN_ADDRESS_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_TILING_CFG4 */
-#define ROGUE_CR_BIF_TILING_CFG4 0x12F8U
-#define ROGUE_CR_BIF_TILING_CFG4_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG4_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG4_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG4_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG4_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG4_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG4_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG4_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG4_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG4_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG4_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG4_MIN_ADDRESS_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_TILING_CFG5 */
-#define ROGUE_CR_BIF_TILING_CFG5 0x1300U
-#define ROGUE_CR_BIF_TILING_CFG5_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG5_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG5_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG5_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG5_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG5_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG5_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG5_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG5_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG5_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG5_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG5_MIN_ADDRESS_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_TILING_CFG6 */
-#define ROGUE_CR_BIF_TILING_CFG6 0x1308U
-#define ROGUE_CR_BIF_TILING_CFG6_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG6_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG6_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG6_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG6_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG6_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG6_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG6_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG6_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG6_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG6_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG6_MIN_ADDRESS_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_BIF_TILING_CFG7 */
-#define ROGUE_CR_BIF_TILING_CFG7 0x1310U
-#define ROGUE_CR_BIF_TILING_CFG7_MASKFULL 0xFFFFFFFF0FFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG7_XSTRIDE_SHIFT 61U
-#define ROGUE_CR_BIF_TILING_CFG7_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG7_ENABLE_SHIFT 60U
-#define ROGUE_CR_BIF_TILING_CFG7_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG7_ENABLE_EN 0x1000000000000000ULL
-#define ROGUE_CR_BIF_TILING_CFG7_MAX_ADDRESS_SHIFT 32U
-#define ROGUE_CR_BIF_TILING_CFG7_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
-#define ROGUE_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG7_MAX_ADDRESS_ALIGNSIZE 4096U
-#define ROGUE_CR_BIF_TILING_CFG7_MIN_ADDRESS_SHIFT 0U
-#define ROGUE_CR_BIF_TILING_CFG7_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
-#define ROGUE_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSHIFT 12U
-#define ROGUE_CR_BIF_TILING_CFG7_MIN_ADDRESS_ALIGNSIZE 4096U
+#define ROGUE_CR_BIF_TILING_CFG_REPEAT_COUNT 8U
+#define ROGUE_CR_BIF_TILING_CFG_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_BIF_TILING_CFG_REPEAT_BASE 0x12D8U
+/* Register ROGUE_CR_BIF_TILING_CFGx */
+#define ROGUE_CR_BIF_TILING_CFGx_MASKFULL 0xFFFFFFFF0FFFFFFFULL
+#define ROGUE_CR_BIF_TILING_CFGx_XSTRIDE_SHIFT 61U
+#define ROGUE_CR_BIF_TILING_CFGx_XSTRIDE_CLRMSK 0x1FFFFFFFFFFFFFFFULL
+#define ROGUE_CR_BIF_TILING_CFGx_ENABLE_SHIFT 60U
+#define ROGUE_CR_BIF_TILING_CFGx_ENABLE_CLRMSK 0xEFFFFFFFFFFFFFFFULL
+#define ROGUE_CR_BIF_TILING_CFGx_ENABLE_EN 0x1000000000000000ULL
+#define ROGUE_CR_BIF_TILING_CFGx_MAX_ADDRESS_SHIFT 32U
+#define ROGUE_CR_BIF_TILING_CFGx_MAX_ADDRESS_CLRMSK 0xF0000000FFFFFFFFULL
+#define ROGUE_CR_BIF_TILING_CFGx_MAX_ADDRESS_ALIGNSHIFT 12U
+#define ROGUE_CR_BIF_TILING_CFGx_MAX_ADDRESS_ALIGNSIZE 4096U
+#define ROGUE_CR_BIF_TILING_CFGx_MIN_ADDRESS_SHIFT 0U
+#define ROGUE_CR_BIF_TILING_CFGx_MIN_ADDRESS_CLRMSK 0xFFFFFFFFF0000000ULL
+#define ROGUE_CR_BIF_TILING_CFGx_MIN_ADDRESS_ALIGNSHIFT 12U
+#define ROGUE_CR_BIF_TILING_CFGx_MIN_ADDRESS_ALIGNSIZE 4096U
 
 /* Register ROGUE_CR_BIF_READS_EXT_STATUS */
 #define ROGUE_CR_BIF_READS_EXT_STATUS 0x1320U
@@ -2951,102 +2780,13 @@
 #define ROGUE_CR_MCU_FENCE_ADDR_ALIGNSIZE 32U
 
 /* Register group: ROGUE_CR_SCRATCH, with 16 repeats */
-#define ROGUE_CR_SCRATCH_REPEATCOUNT 16U
-/* Register ROGUE_CR_SCRATCH0 */
-#define ROGUE_CR_SCRATCH0 0x1A00U
-#define ROGUE_CR_SCRATCH0_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH0_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH0_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH1 */
-#define ROGUE_CR_SCRATCH1 0x1A08U
-#define ROGUE_CR_SCRATCH1_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH1_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH1_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH2 */
-#define ROGUE_CR_SCRATCH2 0x1A10U
-#define ROGUE_CR_SCRATCH2_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH2_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH2_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH3 */
-#define ROGUE_CR_SCRATCH3 0x1A18U
-#define ROGUE_CR_SCRATCH3_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH3_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH3_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH4 */
-#define ROGUE_CR_SCRATCH4 0x1A20U
-#define ROGUE_CR_SCRATCH4_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH4_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH4_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH5 */
-#define ROGUE_CR_SCRATCH5 0x1A28U
-#define ROGUE_CR_SCRATCH5_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH5_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH5_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH6 */
-#define ROGUE_CR_SCRATCH6 0x1A30U
-#define ROGUE_CR_SCRATCH6_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH6_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH6_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH7 */
-#define ROGUE_CR_SCRATCH7 0x1A38U
-#define ROGUE_CR_SCRATCH7_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH7_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH7_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH8 */
-#define ROGUE_CR_SCRATCH8 0x1A40U
-#define ROGUE_CR_SCRATCH8_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH8_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH8_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH9 */
-#define ROGUE_CR_SCRATCH9 0x1A48U
-#define ROGUE_CR_SCRATCH9_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH9_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH9_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH10 */
-#define ROGUE_CR_SCRATCH10 0x1A50U
-#define ROGUE_CR_SCRATCH10_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH10_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH10_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH11 */
-#define ROGUE_CR_SCRATCH11 0x1A58U
-#define ROGUE_CR_SCRATCH11_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH11_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH11_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH12 */
-#define ROGUE_CR_SCRATCH12 0x1A60U
-#define ROGUE_CR_SCRATCH12_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH12_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH12_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH13 */
-#define ROGUE_CR_SCRATCH13 0x1A68U
-#define ROGUE_CR_SCRATCH13_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH13_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH13_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH14 */
-#define ROGUE_CR_SCRATCH14 0x1A70U
-#define ROGUE_CR_SCRATCH14_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH14_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH14_DATA_CLRMSK 0x00000000U
-
-/* Register ROGUE_CR_SCRATCH15 */
-#define ROGUE_CR_SCRATCH15 0x1A78U
-#define ROGUE_CR_SCRATCH15_MASKFULL 0x00000000FFFFFFFFULL
-#define ROGUE_CR_SCRATCH15_DATA_SHIFT 0U
-#define ROGUE_CR_SCRATCH15_DATA_CLRMSK 0x00000000U
+#define ROGUE_CR_SCRATCH_REPEAT_COUNT 16U
+#define ROGUE_CR_SCRATCH_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_SCRATCH_REPEAT_BASE 0x1A00U
+/* Register ROGUE_CR_SCRATCHx */
+#define ROGUE_CR_SCRATCHx_MASKFULL 0x00000000FFFFFFFFULL
+#define ROGUE_CR_SCRATCHx_DATA_SHIFT 0U
+#define ROGUE_CR_SCRATCHx_DATA_CLRMSK 0x00000000U
 
 /* Register group: ROGUE_CR_OS0_SCRATCH, with 2 repeats */
 #define ROGUE_CR_OS0_SCRATCH_REPEATCOUNT 2U
@@ -3273,342 +3013,30 @@
 #define ROGUE_CR_SPFILTER_SIGNAL_DESCR_MIN_ADDR_ALIGNSIZE 16U
 
 /* Register group: ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG, with 16 repeats */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG_REPEATCOUNT 16U
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0 0x3000U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1 0x3008U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG1_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2 0x3010U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG2_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3 0x3018U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG3_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4 0x3020U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG4_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5 0x3028U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG5_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6 0x3030U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG6_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7 0x3038U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG7_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8 0x3040U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG8_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9 0x3048U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG9_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10 0x3050U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG10_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11 0x3058U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG11_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12 0x3060U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG12_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13 0x3068U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG13_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14 0x3070U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG14_DEVVADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15 */
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15 0x3078U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_MASKFULL 0x7FFFF7FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_SHIFT 62U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_TRUSTED_EN 0x4000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_SHIFT 61U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_LOAD_STORE_EN_EN 0x2000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_SHIFT 60U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_FETCH_EN_EN 0x1000000000000000ULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_SHIFT 44U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_SHIFT 40U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG15_DEVVADDR_ALIGNSIZE 4096U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG_REPEAT_COUNT 16U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG_REPEAT_BASE 0x3000U
+/* Register ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx */
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_MASKFULL 0x7FFFF7FFFFFFF000ULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_TRUSTED_SHIFT 62U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_TRUSTED_CLRMSK 0xBFFFFFFFFFFFFFFFULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_TRUSTED_EN 0x4000000000000000ULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_LOAD_STORE_EN_SHIFT 61U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_LOAD_STORE_EN_CLRMSK 0xDFFFFFFFFFFFFFFFULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_LOAD_STORE_EN_EN 0x2000000000000000ULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_FETCH_EN_SHIFT 60U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_FETCH_EN_CLRMSK 0xEFFFFFFFFFFFFFFFULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_FETCH_EN_EN 0x1000000000000000ULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_SIZE_SHIFT 44U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_SIZE_CLRMSK 0xF0000FFFFFFFFFFFULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_SIZE_ALIGNSHIFT 12U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_SIZE_ALIGNSIZE 4096U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_CBASE_SHIFT 40U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_CBASE_CLRMSK 0xFFFFF8FFFFFFFFFFULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_SHIFT 12U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_CLRMSK 0xFFFFFF0000000FFFULL
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_ALIGNSHIFT 12U
+#define ROGUE_CR_FWCORE_ADDR_REMAP_CONFIGx_DEVVADDR_ALIGNSIZE 4096U
 
 /* Register ROGUE_CR_FWCORE_BOOT */
 #define ROGUE_CR_FWCORE_BOOT 0x3090U
@@ -3726,70 +3154,15 @@
 #define ROGUE_CR_FWCORE_WRAPPER_FENCE_ID_EN 0x00000001U
 
 /* Register group: ROGUE_CR_FWCORE_MEM_CAT_BASE, with 8 repeats */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE_REPEATCOUNT 8U
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE0 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE0 0x30F0U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE0_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE0_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE0_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE0_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE1 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE1 0x30F8U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE1_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE1_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE1_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE1_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE2 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE2 0x3100U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE2_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE2_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE2_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE2_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE3 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE3 0x3108U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE3_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE3_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE3_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE3_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE4 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE4 0x3110U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE4_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE4_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE4_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE4_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE5 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE5 0x3118U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE5_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE5_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE5_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE5_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE6 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE6 0x3120U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE6_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE6_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE6_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE6_ADDR_ALIGNSIZE 4096U
-
-/* Register ROGUE_CR_FWCORE_MEM_CAT_BASE7 */
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE7 0x3128U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE7_MASKFULL 0x000000FFFFFFF000ULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE7_ADDR_SHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE7_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSHIFT 12U
-#define ROGUE_CR_FWCORE_MEM_CAT_BASE7_ADDR_ALIGNSIZE 4096U
+#define ROGUE_CR_FWCORE_MEM_CAT_BASE_REPEAT_COUNT 8U
+#define ROGUE_CR_FWCORE_MEM_CAT_BASE_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_FWCORE_MEM_CAT_BASE_REPEAT_BASE 0x30F0U
+/* Register ROGUE_CR_FWCORE_MEM_CAT_BASEx */
+#define ROGUE_CR_FWCORE_MEM_CAT_BASEx_MASKFULL 0x000000FFFFFFF000ULL
+#define ROGUE_CR_FWCORE_MEM_CAT_BASEx_ADDR_SHIFT 12U
+#define ROGUE_CR_FWCORE_MEM_CAT_BASEx_ADDR_CLRMSK 0xFFFFFF0000000FFFULL
+#define ROGUE_CR_FWCORE_MEM_CAT_BASEx_ADDR_ALIGNSHIFT 12U
+#define ROGUE_CR_FWCORE_MEM_CAT_BASEx_ADDR_ALIGNSIZE 4096U
 
 /* Register ROGUE_CR_FWCORE_WDT_RESET */
 #define ROGUE_CR_FWCORE_WDT_RESET 0x3130U
@@ -3816,22 +3189,11 @@
 #define ROGUE_CR_FWCORE_WDT_COUNT_VALUE_CLRMSK 0x00000000U
 
 /* Register group: ROGUE_CR_FWCORE_DMI_RESERVED0, with 4 repeats */
-#define ROGUE_CR_FWCORE_DMI_RESERVED0_REPEATCOUNT 4U
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED00 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED00 0x3400U
-#define ROGUE_CR_FWCORE_DMI_RESERVED00_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED01 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED01 0x3408U
-#define ROGUE_CR_FWCORE_DMI_RESERVED01_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED02 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED02 0x3410U
-#define ROGUE_CR_FWCORE_DMI_RESERVED02_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED03 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED03 0x3418U
-#define ROGUE_CR_FWCORE_DMI_RESERVED03_MASKFULL 0x0000000000000000ULL
+#define ROGUE_CR_FWCORE_DMI_RESERVED0_REPEAT_COUNT 4U
+#define ROGUE_CR_FWCORE_DMI_RESERVED0_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_FWCORE_DMI_RESERVED0_REPEAT_BASE 0x3400U
+/* Register ROGUE_CR_FWCORE_DMI_RESERVED0x */
+#define ROGUE_CR_FWCORE_DMI_RESERVED0x_MASKFULL 0x0000000000000000ULL
 
 /* Register ROGUE_CR_FWCORE_DMI_DATA0 */
 #define ROGUE_CR_FWCORE_DMI_DATA0 0x3420U
@@ -3842,26 +3204,11 @@
 #define ROGUE_CR_FWCORE_DMI_DATA1_MASKFULL 0x0000000000000000ULL
 
 /* Register group: ROGUE_CR_FWCORE_DMI_RESERVED1, with 5 repeats */
-#define ROGUE_CR_FWCORE_DMI_RESERVED1_REPEATCOUNT 5U
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED10 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED10 0x3430U
-#define ROGUE_CR_FWCORE_DMI_RESERVED10_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED11 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED11 0x3438U
-#define ROGUE_CR_FWCORE_DMI_RESERVED11_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED12 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED12 0x3440U
-#define ROGUE_CR_FWCORE_DMI_RESERVED12_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED13 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED13 0x3448U
-#define ROGUE_CR_FWCORE_DMI_RESERVED13_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED14 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED14 0x3450U
-#define ROGUE_CR_FWCORE_DMI_RESERVED14_MASKFULL 0x0000000000000000ULL
+#define ROGUE_CR_FWCORE_DMI_RESERVED1_REPEAT_COUNT 5U
+#define ROGUE_CR_FWCORE_DMI_RESERVED1_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_FWCORE_DMI_RESERVED1_REPEAT_BASE 0x3430U
+/* Register ROGUE_CR_FWCORE_DMI_RESERVED1x */
+#define ROGUE_CR_FWCORE_DMI_RESERVED1x_MASKFULL 0x0000000000000000ULL
 
 /* Register ROGUE_CR_FWCORE_DMI_DMCONTROL */
 #define ROGUE_CR_FWCORE_DMI_DMCONTROL 0x3480U
@@ -3872,22 +3219,11 @@
 #define ROGUE_CR_FWCORE_DMI_DMSTATUS_MASKFULL 0x0000000000000000ULL
 
 /* Register group: ROGUE_CR_FWCORE_DMI_RESERVED2, with 4 repeats */
-#define ROGUE_CR_FWCORE_DMI_RESERVED2_REPEATCOUNT 4U
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED20 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED20 0x3490U
-#define ROGUE_CR_FWCORE_DMI_RESERVED20_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED21 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED21 0x3498U
-#define ROGUE_CR_FWCORE_DMI_RESERVED21_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED22 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED22 0x34A0U
-#define ROGUE_CR_FWCORE_DMI_RESERVED22_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED23 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED23 0x34A8U
-#define ROGUE_CR_FWCORE_DMI_RESERVED23_MASKFULL 0x0000000000000000ULL
+#define ROGUE_CR_FWCORE_DMI_RESERVED2_REPEAT_COUNT 4U
+#define ROGUE_CR_FWCORE_DMI_RESERVED2_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_FWCORE_DMI_RESERVED2_REPEAT_BASE 0x3490U
+/* Register ROGUE_CR_FWCORE_DMI_RESERVED2x */
+#define ROGUE_CR_FWCORE_DMI_RESERVED2x_MASKFULL 0x0000000000000000ULL
 
 /* Register ROGUE_CR_FWCORE_DMI_ABSTRACTCS */
 #define ROGUE_CR_FWCORE_DMI_ABSTRACTCS 0x34B0U
@@ -3906,32 +3242,18 @@
 #define ROGUE_CR_FWCORE_DMI_SBADDRESS0_MASKFULL 0x0000000000000000ULL
 
 /* Register group: ROGUE_CR_FWCORE_DMI_RESERVED3, with 2 repeats */
-#define ROGUE_CR_FWCORE_DMI_RESERVED3_REPEATCOUNT 2U
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED30 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED30 0x34D0U
-#define ROGUE_CR_FWCORE_DMI_RESERVED30_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_RESERVED31 */
-#define ROGUE_CR_FWCORE_DMI_RESERVED31 0x34D8U
-#define ROGUE_CR_FWCORE_DMI_RESERVED31_MASKFULL 0x0000000000000000ULL
+#define ROGUE_CR_FWCORE_DMI_RESERVED3_REPEAT_COUNT 2U
+#define ROGUE_CR_FWCORE_DMI_RESERVED3_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_FWCORE_DMI_RESERVED3_REPEAT_BASE 0x34D0U
+/* Register ROGUE_CR_FWCORE_DMI_RESERVED3x */
+#define ROGUE_CR_FWCORE_DMI_RESERVED3x_MASKFULL 0x0000000000000000ULL
 
 /* Register group: ROGUE_CR_FWCORE_DMI_SBDATA, with 4 repeats */
-#define ROGUE_CR_FWCORE_DMI_SBDATA_REPEATCOUNT 4U
-/* Register ROGUE_CR_FWCORE_DMI_SBDATA0 */
-#define ROGUE_CR_FWCORE_DMI_SBDATA0 0x35E0U
-#define ROGUE_CR_FWCORE_DMI_SBDATA0_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_SBDATA1 */
-#define ROGUE_CR_FWCORE_DMI_SBDATA1 0x35E8U
-#define ROGUE_CR_FWCORE_DMI_SBDATA1_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_SBDATA2 */
-#define ROGUE_CR_FWCORE_DMI_SBDATA2 0x35F0U
-#define ROGUE_CR_FWCORE_DMI_SBDATA2_MASKFULL 0x0000000000000000ULL
-
-/* Register ROGUE_CR_FWCORE_DMI_SBDATA3 */
-#define ROGUE_CR_FWCORE_DMI_SBDATA3 0x35F8U
-#define ROGUE_CR_FWCORE_DMI_SBDATA3_MASKFULL 0x0000000000000000ULL
+#define ROGUE_CR_FWCORE_DMI_SBDATA_REPEAT_COUNT 4U
+#define ROGUE_CR_FWCORE_DMI_SBDATA_REPEAT_OFFSET 0x0008U
+#define ROGUE_CR_FWCORE_DMI_SBDATA_REPEAT_BASE 0x35E0U
+/* Register ROGUE_CR_FWCORE_DMI_SBDATAx */
+#define ROGUE_CR_FWCORE_DMI_SBDATAx_MASKFULL 0x0000000000000000ULL
 
 /* Register ROGUE_CR_FWCORE_DMI_HALTSUM0 */
 #define ROGUE_CR_FWCORE_DMI_HALTSUM0 0x3600U
diff --git a/drivers/gpu/drm/imagination/pvr_rogue_defs.h b/drivers/gpu/drm/imagination/pvr_rogue_defs.h
index 932b0168600896..f09ab85e590a0d 100644
--- a/drivers/gpu/drm/imagination/pvr_rogue_defs.h
+++ b/drivers/gpu/drm/imagination/pvr_rogue_defs.h
@@ -93,26 +93,8 @@
 #define MMU_CONTEXT_MAPPING_FWPRIV (0x0) /* FW code/private data */
 #define MMU_CONTEXT_MAPPING_FWIF (0x0) /* Host/FW data */
 
-/*
- * Utility macros to calculate CAT_BASE register addresses
- */
-#define BIF_CAT_BASEX(n)          \
-	(ROGUE_CR_BIF_CAT_BASE0 + \
-	 (n) * (ROGUE_CR_BIF_CAT_BASE1 - ROGUE_CR_BIF_CAT_BASE0))
-
-#define FWCORE_MEM_CAT_BASEX(n)                 \
-	(ROGUE_CR_FWCORE_MEM_CAT_BASE0 +        \
-	 (n) * (ROGUE_CR_FWCORE_MEM_CAT_BASE1 - \
-		ROGUE_CR_FWCORE_MEM_CAT_BASE0))
-
-/*
- * FWCORE wrapper register defines
- */
-#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_SHIFT \
-	ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_SHIFT
-#define FWCORE_ADDR_REMAP_CONFIG0_MMU_CONTEXT_CLRMSK \
-	ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0_CBASE_CLRMSK
-#define FWCORE_ADDR_REMAP_CONFIG0_SIZE_ALIGNSHIFT (12U)
+/* Utility macro for accessing instances of repeated registers */
+#define PVR_CR_REPEAT_X(REG, N) ((REG##_REPEAT_BASE) + (N) * (REG##_REPEAT_OFFSET))
 
 #define ROGUE_MAX_COMPUTE_SHARED_REGISTERS (2 * 1024)
 #define ROGUE_MAX_VERTEX_SHARED_REGISTERS 1024
diff --git a/drivers/gpu/drm/imagination/pvr_rogue_riscv.h b/drivers/gpu/drm/imagination/pvr_rogue_riscv.h
index 9a070e24fa6a8b..d971729e18a423 100644
--- a/drivers/gpu/drm/imagination/pvr_rogue_riscv.h
+++ b/drivers/gpu/drm/imagination/pvr_rogue_riscv.h
@@ -35,7 +35,5 @@ enum rogue_riscvfw_region {
 };
 
 #define ROGUE_RISCVFW_REGION_BASE(r) ((u32)(ROGUE_RISCV_REGION_##r) << ROGUE_RISCVFW_REGION_SHIFT)
-#define ROGUE_RISCVFW_REGION_REMAP_CR(r) \
-	(ROGUE_CR_FWCORE_ADDR_REMAP_CONFIG0 + (u32)(ROGUE_RISCV_REGION_##r) * 8U)
 
 #endif /* PVR_ROGUE_RISCV_H */
diff --git a/drivers/gpu/drm/imagination/pvr_trace.h b/drivers/gpu/drm/imagination/pvr_trace.h
index 943d6b2418cba4..4be7d4b8a35765 100644
--- a/drivers/gpu/drm/imagination/pvr_trace.h
+++ b/drivers/gpu/drm/imagination/pvr_trace.h
@@ -90,21 +90,25 @@ TRACE_EVENT(pvr_job_create,
 		      __entry->sync_op_count)
 );
 
-#undef PVR_JOB_TYPE_TO_STR
 #undef PVR_JOB_GET_HWRT_FW_ADDR
 
 TRACE_EVENT(pvr_job_submit_fw,
 	    TP_PROTO(struct pvr_job *job),
 	    TP_ARGS(job),
 	    TP_STRUCT__entry(__field(struct pvr_job *, job)
-			     __field(u32, done_seqno)),
+			     __field(u32, done_seqno)
+			     __field(enum drm_pvr_job_type, job_type)),
 	    TP_fast_assign(__entry->job = job;
-			   __entry->done_seqno = job->done_fence->seqno;),
-	    TP_printk("job=%p done_seqno=%u",
+			   __entry->done_seqno = job->done_fence->seqno;
+			   __entry->job_type = job->type;),
+	    TP_printk("job=%p done_seqno=%u job_type=%s",
 		      __entry->job,
-		      __entry->done_seqno)
+		      __entry->done_seqno,
+		      PVR_JOB_TYPE_TO_STR(__entry->job_type))
 );
 
+#undef PVR_JOB_TYPE_TO_STR
+
 TRACE_EVENT(pvr_job_done,
 	    TP_PROTO(struct pvr_job *job),
 	    TP_ARGS(job),
diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c
index ceb78694cd9873..55cc999f3708b6 100644
--- a/drivers/gpu/drm/imagination/pvr_vm.c
+++ b/drivers/gpu/drm/imagination/pvr_vm.c
@@ -276,7 +276,7 @@ pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op,
 		goto err_bind_op_fini;
 
 	bind_op->mmu_op_ctx =
-		pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, offset, size);
+		pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, device_addr, offset, size);
 	err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx);
 	if (err) {
 		bind_op->mmu_op_ctx = NULL;
@@ -318,7 +318,7 @@ pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op,
 	}
 
 	bind_op->mmu_op_ctx =
-		pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, 0, 0);
+		pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, device_addr, 0, 0);
 	err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx);
 	if (err) {
 		bind_op->mmu_op_ctx = NULL;
diff --git a/drivers/gpu/drm/imx/dc/dc-crtc.c b/drivers/gpu/drm/imx/dc/dc-crtc.c
index 764510ff90a4a9..0d64186a990184 100644
--- a/drivers/gpu/drm/imx/dc/dc-crtc.c
+++ b/drivers/gpu/drm/imx/dc/dc-crtc.c
@@ -98,7 +98,7 @@ static void dc_crtc_disable_vblank(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_funcs dc_crtc_funcs = {
-	.reset			= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy		= drm_crtc_cleanup,
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/imx/dc/dc-kms.c b/drivers/gpu/drm/imx/dc/dc-kms.c
index 0f8cfaf4c4d15c..0e2ce6a425487e 100644
--- a/drivers/gpu/drm/imx/dc/dc-kms.c
+++ b/drivers/gpu/drm/imx/dc/dc-kms.c
@@ -15,9 +15,9 @@
 #include <drm/drm_encoder.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mode_config.h>
+#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "dc-de.h"
@@ -30,6 +30,10 @@ static const struct drm_mode_config_funcs dc_drm_mode_config_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
+static const struct drm_encoder_funcs dc_kms_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static int dc_kms_init_encoder_per_crtc(struct dc_drm_device *dc_drm,
 					int crtc_index)
 {
@@ -55,7 +59,8 @@ static int dc_kms_init_encoder_per_crtc(struct dc_drm_device *dc_drm,
 	}
 
 	encoder = &dc_drm->encoder[crtc_index];
-	ret = drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_NONE);
+	ret = drm_encoder_init(drm, encoder, &dc_kms_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
 	if (ret) {
 		dev_err(dev, "failed to initialize encoder for CRTC%u: %d\n",
 			crtc->index, ret);
diff --git a/drivers/gpu/drm/imx/dc/dc-plane.c b/drivers/gpu/drm/imx/dc/dc-plane.c
index dea7404fe6590e..62dd0576fadcbb 100644
--- a/drivers/gpu/drm/imx/dc/dc-plane.c
+++ b/drivers/gpu/drm/imx/dc/dc-plane.c
@@ -39,7 +39,7 @@ static const struct drm_plane_funcs dc_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/imx/dcss/Kconfig b/drivers/gpu/drm/imx/dcss/Kconfig
index e014ed3ae66c93..d278bebfc33c40 100644
--- a/drivers/gpu/drm/imx/dcss/Kconfig
+++ b/drivers/gpu/drm/imx/dcss/Kconfig
@@ -6,6 +6,7 @@ config DRM_IMX_DCSS
 	select DRM_DISPLAY_HELPER
 	select DRM_BRIDGE_CONNECTOR
 	select DRM_GEM_DMA_HELPER
+	select DRM_PANEL
 	select VIDEOMODE_HELPERS
 	depends on DRM && ((ARCH_MXC && ARM64) || COMPILE_TEST)
 	help
diff --git a/drivers/gpu/drm/imx/dcss/dcss-crtc.c b/drivers/gpu/drm/imx/dcss/dcss-crtc.c
index ab41759a9f52d2..5e9ee3f09d51f5 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-crtc.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-crtc.c
@@ -45,7 +45,7 @@ static const struct drm_crtc_funcs dcss_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = drm_crtc_cleanup,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = dcss_enable_vblank,
diff --git a/drivers/gpu/drm/imx/dcss/dcss-plane.c b/drivers/gpu/drm/imx/dcss/dcss-plane.c
index 1746dc5d4b6422..303e93fd036c1d 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-plane.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-plane.c
@@ -96,7 +96,7 @@ static const struct drm_plane_funcs dcss_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= dcss_plane_destroy,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 	.format_mod_supported	= dcss_plane_format_mod_supported,
diff --git a/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c
index 1405e50ee56b5e..f4fe1842cfaeb0 100644
--- a/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c
@@ -107,19 +107,17 @@ static void ipu_crtc_atomic_disable(struct drm_crtc *crtc,
 	spin_unlock_irq(&crtc->dev->event_lock);
 }
 
-static void imx_drm_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *imx_drm_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct imx_crtc_state *state;
 
-	if (crtc->state)
-		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	kfree(to_imx_crtc_state(crtc->state));
-	crtc->state = NULL;
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
 
-	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
+	return &state->base;
 }
 
 static struct drm_crtc_state *imx_drm_crtc_duplicate_state(struct drm_crtc *crtc)
@@ -164,7 +162,7 @@ static void ipu_disable_vblank(struct drm_crtc *crtc)
 static const struct drm_crtc_funcs ipu_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = imx_drm_crtc_reset,
+	.atomic_create_state = imx_drm_crtc_create_state,
 	.atomic_duplicate_state = imx_drm_crtc_duplicate_state,
 	.atomic_destroy_state = imx_drm_crtc_destroy_state,
 	.enable_vblank = ipu_enable_vblank,
diff --git a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
index c7ec09e557c19f..67e78a5ac8812e 100644
--- a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
@@ -14,6 +14,7 @@
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_print.h>
 
 #include <video/imx-ipu-v3.h>
@@ -297,21 +298,18 @@ void ipu_plane_disable_deferred(struct drm_plane *plane)
 	}
 }
 
-static void ipu_plane_state_reset(struct drm_plane *plane)
+static struct drm_plane_state *ipu_plane_state_create_state(struct drm_plane *plane)
 {
 	struct ipu_plane_state *ipu_state;
 
-	if (plane->state) {
-		ipu_state = to_ipu_plane_state(plane->state);
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-		kfree(ipu_state);
-		plane->state = NULL;
-	}
-
 	ipu_state = kzalloc_obj(*ipu_state);
 
-	if (ipu_state)
-		__drm_atomic_helper_plane_reset(plane, &ipu_state->base);
+	if (!ipu_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&ipu_state->base, plane);
+
+	return &ipu_state->base;
 }
 
 static struct drm_plane_state *
@@ -358,10 +356,11 @@ static bool ipu_plane_format_mod_supported(struct drm_plane *plane,
 static const struct drm_plane_funcs ipu_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.reset		= ipu_plane_state_reset,
+	.atomic_create_state = ipu_plane_state_create_state,
 	.atomic_duplicate_state	= ipu_plane_duplicate_state,
 	.atomic_destroy_state	= ipu_plane_destroy_state,
 	.format_mod_supported = ipu_plane_format_mod_supported,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static int ipu_plane_atomic_check(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/ingenic/Kconfig b/drivers/gpu/drm/ingenic/Kconfig
index 04ecfb0c5dd67e..987ed42f6f7a59 100644
--- a/drivers/gpu/drm/ingenic/Kconfig
+++ b/drivers/gpu/drm/ingenic/Kconfig
@@ -7,6 +7,7 @@ config DRM_INGENIC
 	depends on COMMON_CLK
 	select DRM_BRIDGE
 	select DRM_CLIENT_SELECTION
+	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
 	select DRM_KMS_HELPER
 	select DRM_DISPLAY_HELPER
diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
index 738a80b2550ffd..625ec76cf04d87 100644
--- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
+++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
@@ -987,7 +987,7 @@ static const struct drm_driver ingenic_drm_driver_data = {
 static const struct drm_plane_funcs ingenic_drm_primary_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.destroy		= drm_plane_cleanup,
 
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
@@ -997,7 +997,7 @@ static const struct drm_plane_funcs ingenic_drm_primary_plane_funcs = {
 static const struct drm_crtc_funcs ingenic_drm_crtc_funcs = {
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= drm_atomic_helper_page_flip,
-	.reset			= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy		= drm_crtc_cleanup,
 
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
diff --git a/drivers/gpu/drm/ingenic/ingenic-ipu.c b/drivers/gpu/drm/ingenic/ingenic-ipu.c
index 56143a191f361f..bf24882e8d9f10 100644
--- a/drivers/gpu/drm/ingenic/ingenic-ipu.c
+++ b/drivers/gpu/drm/ingenic/ingenic-ipu.c
@@ -718,7 +718,7 @@ ingenic_ipu_plane_atomic_set_property(struct drm_plane *plane,
 static const struct drm_plane_funcs ingenic_ipu_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.destroy		= drm_plane_cleanup,
 
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
diff --git a/drivers/gpu/drm/kmb/kmb_crtc.c b/drivers/gpu/drm/kmb/kmb_crtc.c
index 1dcdc55b00696c..06cf5ad60b8106 100644
--- a/drivers/gpu/drm/kmb/kmb_crtc.c
+++ b/drivers/gpu/drm/kmb/kmb_crtc.c
@@ -58,7 +58,7 @@ static const struct drm_crtc_funcs kmb_crtc_funcs = {
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = kmb_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/kmb/kmb_dsi.c b/drivers/gpu/drm/kmb/kmb_dsi.c
index 59d0e856392fc5..13adba96bc7865 100644
--- a/drivers/gpu/drm/kmb/kmb_dsi.c
+++ b/drivers/gpu/drm/kmb/kmb_dsi.c
@@ -14,8 +14,8 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_mipi_dsi.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -1427,6 +1427,10 @@ struct kmb_dsi *kmb_dsi_init(struct platform_device *pdev)
 	return kmb_dsi;
 }
 
+static const struct drm_encoder_funcs kmb_dsi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 int kmb_dsi_encoder_init(struct drm_device *dev, struct kmb_dsi *kmb_dsi)
 {
 	struct drm_encoder *encoder;
@@ -1437,7 +1441,8 @@ int kmb_dsi_encoder_init(struct drm_device *dev, struct kmb_dsi *kmb_dsi)
 	encoder->possible_crtcs = 1;
 	encoder->possible_clones = 0;
 
-	ret = drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_DSI);
+	ret = drm_encoder_init(dev, encoder, &kmb_dsi_encoder_funcs,
+			       DRM_MODE_ENCODER_DSI, NULL);
 	if (ret) {
 		dev_err(kmb_dsi->dev, "Failed to init encoder %d\n", ret);
 		return ret;
diff --git a/drivers/gpu/drm/kmb/kmb_plane.c b/drivers/gpu/drm/kmb/kmb_plane.c
index a0001edb1e21aa..1742c2c6dfd89a 100644
--- a/drivers/gpu/drm/kmb/kmb_plane.c
+++ b/drivers/gpu/drm/kmb/kmb_plane.c
@@ -561,7 +561,7 @@ static const struct drm_plane_funcs kmb_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = kmb_plane_destroy,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/logicvc/Kconfig b/drivers/gpu/drm/logicvc/Kconfig
index 11aae1626199bc..95cdc5cb8cd54d 100644
--- a/drivers/gpu/drm/logicvc/Kconfig
+++ b/drivers/gpu/drm/logicvc/Kconfig
@@ -5,6 +5,7 @@ config DRM_LOGICVC
 	select DRM_CLIENT_SELECTION
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
+	select DRM_PANEL
 	select REGMAP
 	select REGMAP_MMIO
 	help
diff --git a/drivers/gpu/drm/logicvc/logicvc_crtc.c b/drivers/gpu/drm/logicvc/logicvc_crtc.c
index 81e5b80736fa79..af92a573c3f269 100644
--- a/drivers/gpu/drm/logicvc/logicvc_crtc.c
+++ b/drivers/gpu/drm/logicvc/logicvc_crtc.c
@@ -213,10 +213,10 @@ static void logicvc_crtc_disable_vblank(struct drm_crtc *drm_crtc)
 }
 
 static const struct drm_crtc_funcs logicvc_crtc_funcs = {
-	.reset			= drm_atomic_helper_crtc_reset,
 	.destroy		= drm_crtc_cleanup,
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank		= logicvc_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/logicvc/logicvc_layer.c b/drivers/gpu/drm/logicvc/logicvc_layer.c
index 7f5ba000188aba..94741685638113 100644
--- a/drivers/gpu/drm/logicvc/logicvc_layer.c
+++ b/drivers/gpu/drm/logicvc/logicvc_layer.c
@@ -251,7 +251,7 @@ static const struct drm_plane_funcs logicvc_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/loongson/lsdc_crtc.c b/drivers/gpu/drm/loongson/lsdc_crtc.c
index 16b8f36a407107..a47816e3fa81f6 100644
--- a/drivers/gpu/drm/loongson/lsdc_crtc.c
+++ b/drivers/gpu/drm/loongson/lsdc_crtc.c
@@ -388,24 +388,18 @@ static const struct lsdc_crtc_hw_ops ls7a2000_crtc_hw_ops[2] = {
 	},
 };
 
-static void lsdc_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *lsdc_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct lsdc_crtc *lcrtc = to_lsdc_crtc(crtc);
-	const struct lsdc_crtc_hw_ops *ops = lcrtc->hw_ops;
 	struct lsdc_crtc_state *priv_crtc_state;
 
-	if (crtc->state)
-		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
-
 	priv_crtc_state = kzalloc_obj(*priv_crtc_state);
 
 	if (!priv_crtc_state)
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, &priv_crtc_state->base);
+		return ERR_PTR(-ENOMEM);
 
-	/* Reset the CRTC hardware, this is required for S3 support */
-	ops->reset(lcrtc);
+	__drm_atomic_helper_crtc_state_init(&priv_crtc_state->base, crtc);
+
+	return &priv_crtc_state->base;
 }
 
 static void lsdc_crtc_atomic_destroy_state(struct drm_crtc *crtc,
@@ -712,7 +706,7 @@ static void lsdc_crtc_atomic_print_state(struct drm_printer *p,
 }
 
 static const struct drm_crtc_funcs ls7a1000_crtc_funcs = {
-	.reset = lsdc_crtc_reset,
+	.atomic_create_state = lsdc_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -726,7 +720,7 @@ static const struct drm_crtc_funcs ls7a1000_crtc_funcs = {
 };
 
 static const struct drm_crtc_funcs ls7a2000_crtc_funcs = {
-	.reset = lsdc_crtc_reset,
+	.atomic_create_state = lsdc_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -937,6 +931,15 @@ static bool lsdc_crtc_get_scanout_position(struct drm_crtc *crtc,
 	return true;
 }
 
+static void lsdc_crtc_hw_reset(struct drm_crtc *crtc)
+{
+	struct lsdc_crtc *lcrtc = to_lsdc_crtc(crtc);
+
+	/* Reset the CRTC hardware, this is required for S3 support */
+	if (lcrtc->hw_ops->reset)
+		lcrtc->hw_ops->reset(lcrtc);
+}
+
 static const struct drm_crtc_helper_funcs lsdc_crtc_helper_funcs = {
 	.mode_valid = lsdc_crtc_mode_valid,
 	.mode_set_nofb = lsdc_crtc_mode_set_nofb,
@@ -945,6 +948,7 @@ static const struct drm_crtc_helper_funcs lsdc_crtc_helper_funcs = {
 	.atomic_check = lsdc_crtc_helper_atomic_check,
 	.atomic_flush = lsdc_crtc_atomic_flush,
 	.get_scanout_position = lsdc_crtc_get_scanout_position,
+	.hw_reset = lsdc_crtc_hw_reset,
 };
 
 int ls7a1000_crtc_init(struct drm_device *ddev,
diff --git a/drivers/gpu/drm/loongson/lsdc_plane.c b/drivers/gpu/drm/loongson/lsdc_plane.c
index bcc0ffa17bdf2c..35d45e893f2d55 100644
--- a/drivers/gpu/drm/loongson/lsdc_plane.c
+++ b/drivers/gpu/drm/loongson/lsdc_plane.c
@@ -453,7 +453,7 @@ static const struct drm_plane_funcs lsdc_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.atomic_print_state = lsdc_plane_atomic_print_state,
diff --git a/drivers/gpu/drm/mcde/Kconfig b/drivers/gpu/drm/mcde/Kconfig
index 3516c8d2a5d9e6..81e7d3796c0658 100644
--- a/drivers/gpu/drm/mcde/Kconfig
+++ b/drivers/gpu/drm/mcde/Kconfig
@@ -9,7 +9,7 @@ config DRM_MCDE
 	select DRM_CLIENT_SELECTION
 	select DRM_MIPI_DSI
 	select DRM_BRIDGE
-	select DRM_PANEL_BRIDGE
+	select DRM_PANEL
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
 	help
diff --git a/drivers/gpu/drm/mcde/mcde_display.c b/drivers/gpu/drm/mcde/mcde_display.c
index 257a6e84dd5830..281f5b3274bd64 100644
--- a/drivers/gpu/drm/mcde/mcde_display.c
+++ b/drivers/gpu/drm/mcde/mcde_display.c
@@ -10,6 +10,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/media-bus-format.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_device.h>
 #include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
@@ -17,8 +18,8 @@
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_vblank.h>
 #include <video/mipi_display.h>
@@ -132,7 +133,7 @@ void mcde_display_irq(struct mcde *mcde)
 	writel(mispp, mcde->regs + MCDE_RISPP);
 
 	if (vblank)
-		drm_crtc_handle_vblank(&mcde->pipe.crtc);
+		drm_crtc_handle_vblank(&mcde->crtc);
 
 	if (misovl)
 		dev_info(mcde->dev, "some stray overlay IRQ %08x\n", misovl);
@@ -157,40 +158,59 @@ void mcde_display_disable_irqs(struct mcde *mcde)
 	writel(0xFFFFFFFF, mcde->regs + MCDE_RISCHNL);
 }
 
-static int mcde_display_check(struct drm_simple_display_pipe *pipe,
-			      struct drm_plane_state *pstate,
-			      struct drm_crtc_state *cstate)
+static int mcde_plane_helper_atomic_check(struct drm_plane *plane,
+					  struct drm_atomic_commit *commit)
 {
-	const struct drm_display_mode *mode = &cstate->mode;
-	struct drm_framebuffer *old_fb = pipe->plane.state->fb;
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_plane_state *old_pstate = drm_atomic_get_old_plane_state(commit, plane);
+	struct drm_crtc_state *cstate = NULL;
+	const struct drm_display_mode *mode;
+	struct drm_framebuffer *old_fb = old_pstate->fb;
 	struct drm_framebuffer *fb = pstate->fb;
+	int ret;
 
-	if (fb) {
-		u32 offset = drm_fb_dma_get_gem_addr(fb, pstate, 0);
+	if (pstate->crtc) {
+		cstate = drm_atomic_get_crtc_state(commit, pstate->crtc);
+		if (IS_ERR(cstate))
+			return PTR_ERR(cstate);
+	}
 
-		/* FB base address must be dword aligned. */
-		if (offset & 3) {
-			DRM_DEBUG_KMS("FB not 32-bit aligned\n");
-			return -EINVAL;
-		}
+	ret = drm_atomic_helper_check_plane_state(pstate, cstate,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, false);
+	if (ret)
+		return ret;
 
-		/*
-		 * There's no pitch register, the mode's hdisplay
-		 * controls this.
-		 */
-		if (fb->pitches[0] != mode->hdisplay * fb->format->cpp[0]) {
-			DRM_DEBUG_KMS("can't handle pitches\n");
-			return -EINVAL;
-		}
+	if (!pstate->visible || !fb)
+		return 0;
 
-		/*
-		 * We can't change the FB format in a flicker-free
-		 * manner (and only update it during CRTC enable).
-		 */
-		if (old_fb && old_fb->format != fb->format)
-			cstate->mode_changed = true;
+	mode = &cstate->mode;
+
+	u32 offset = drm_fb_dma_get_gem_addr(fb, pstate, 0);
+
+	/* FB base address must be dword aligned. */
+	if (offset & 3) {
+		DRM_DEBUG_KMS("FB not 32-bit aligned\n");
+		return -EINVAL;
 	}
 
+	/*
+	 * There's no pitch register, the mode's hdisplay
+	 * controls this.
+	 */
+	if (fb->pitches[0] != mode->hdisplay * fb->format->cpp[0]) {
+		DRM_DEBUG_KMS("can't handle pitches\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * We can't change the FB format in a flicker-free
+	 * manner (and only update it during CRTC enable).
+	 */
+	if (old_fb && old_fb->format != fb->format)
+		cstate->mode_changed = true;
+
 	return 0;
 }
 
@@ -1149,16 +1169,15 @@ static void mcde_setup_dsi(struct mcde *mcde, const struct drm_display_mode *mod
 	*dsi_formatter_frame = formatter_frame;
 }
 
-static void mcde_display_enable(struct drm_simple_display_pipe *pipe,
-				struct drm_crtc_state *cstate,
-				struct drm_plane_state *plane_state)
+static void mcde_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					   struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_plane *plane = &pipe->plane;
 	struct drm_device *drm = crtc->dev;
 	struct mcde *mcde = to_mcde(drm);
+	struct drm_crtc_state *cstate = drm_atomic_get_new_crtc_state(commit, crtc);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, &mcde->plane);
 	const struct drm_display_mode *mode = &cstate->mode;
-	struct drm_framebuffer *fb = plane->state->fb;
+	struct drm_framebuffer *fb = plane_state->fb;
 	u32 format = fb->format->format;
 	int dsi_pkt_size;
 	int fifo_wtrmrk;
@@ -1298,9 +1317,10 @@ static void mcde_display_enable(struct drm_simple_display_pipe *pipe,
 	dev_info(drm->dev, "MCDE display is enabled\n");
 }
 
-static void mcde_display_disable(struct drm_simple_display_pipe *pipe)
+static void mcde_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					    struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_crtc_state *cstate = drm_atomic_get_new_crtc_state(commit, crtc);
 	struct drm_device *drm = crtc->dev;
 	struct mcde *mcde = to_mcde(drm);
 	struct drm_pending_vblank_event *event;
@@ -1318,9 +1338,9 @@ static void mcde_display_disable(struct drm_simple_display_pipe *pipe)
 		mcde_dsi_disable(mcde->bridge);
 	}
 
-	event = crtc->state->event;
+	event = cstate->event;
 	if (event) {
-		crtc->state->event = NULL;
+		cstate->event = NULL;
 
 		spin_lock_irq(&crtc->dev->event_lock);
 		drm_crtc_send_vblank_event(crtc, event);
@@ -1381,24 +1401,46 @@ static void mcde_set_extsrc(struct mcde *mcde, u32 buffer_address)
 	writel(buffer_address + mcde->stride, mcde->regs + MCDE_EXTSRCXA1);
 }
 
-static void mcde_display_update(struct drm_simple_display_pipe *pipe,
-				struct drm_plane_state *old_pstate)
+static void mcde_plane_helper_atomic_update(struct drm_plane *plane,
+					    struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_device *drm = crtc->dev;
-	struct mcde *mcde = to_mcde(drm);
-	struct drm_pending_vblank_event *event = crtc->state->event;
-	struct drm_plane *plane = &pipe->plane;
-	struct drm_plane_state *pstate = plane->state;
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
+	struct mcde *mcde = to_mcde(plane->dev);
 	struct drm_framebuffer *fb = pstate->fb;
 
+	/*
+	 * We do not start sending framebuffer updates before the
+	 * display is enabled. Update events will however be dispatched
+	 * from the DRM core before the display is enabled.
+	 */
+	if (fb) {
+		mcde_set_extsrc(mcde, drm_fb_dma_get_gem_addr(fb, pstate, 0));
+		dev_info_once(mcde->dev, "first update of display contents\n");
+	} else {
+		/*
+		 * If an update is receieved before the MCDE is enabled
+		 * (before mcde_display_enable() is called) we can't really
+		 * do much with that buffer.
+		 */
+		dev_info(mcde->dev, "ignored a display update\n");
+	}
+}
+
+static void mcde_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+					  struct drm_atomic_commit *commit)
+{
+	struct mcde *mcde = to_mcde(crtc->dev);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, &mcde->plane);
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	struct drm_pending_vblank_event *event = crtc_state->event;
+
 	/*
 	 * Handle any pending event first, we need to arm the vblank
 	 * interrupt before sending any update to the display so we don't
 	 * miss the interrupt.
 	 */
 	if (event) {
-		crtc->state->event = NULL;
+		crtc_state->event = NULL;
 
 		spin_lock_irq(&crtc->dev->event_lock);
 		/*
@@ -1408,7 +1450,7 @@ static void mcde_display_update(struct drm_simple_display_pipe *pipe,
 		 * until we get an update. If MCDE output isn't yet enabled,
 		 * we just send a vblank dummy event back.
 		 */
-		if (crtc->state->active && drm_crtc_vblank_get(crtc) == 0) {
+		if (crtc_state->active && drm_crtc_vblank_get(crtc) == 0) {
 			dev_dbg(mcde->dev, "arm vblank event\n");
 			drm_crtc_arm_vblank_event(crtc, event);
 		} else {
@@ -1419,33 +1461,12 @@ static void mcde_display_update(struct drm_simple_display_pipe *pipe,
 		spin_unlock_irq(&crtc->dev->event_lock);
 	}
 
-	/*
-	 * We do not start sending framebuffer updates before the
-	 * display is enabled. Update events will however be dispatched
-	 * from the DRM core before the display is enabled.
-	 */
-	if (fb) {
-		mcde_set_extsrc(mcde, drm_fb_dma_get_gem_addr(fb, pstate, 0));
-		dev_info_once(mcde->dev, "first update of display contents\n");
-		/*
-		 * Usually the flow is already active, unless we are in
-		 * oneshot mode, then we need to kick the flow right here.
-		 */
-		if (mcde->flow_active == 0)
-			mcde_start_flow(mcde);
-	} else {
-		/*
-		 * If an update is receieved before the MCDE is enabled
-		 * (before mcde_display_enable() is called) we can't really
-		 * do much with that buffer.
-		 */
-		dev_info(mcde->dev, "ignored a display update\n");
-	}
+	if (crtc_state->active && plane_state && plane_state->fb && mcde->flow_active == 0)
+		mcde_start_flow(mcde);
 }
 
-static int mcde_display_enable_vblank(struct drm_simple_display_pipe *pipe)
+static int mcde_crtc_enable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct mcde *mcde = to_mcde(drm);
 	u32 val;
@@ -1462,9 +1483,8 @@ static int mcde_display_enable_vblank(struct drm_simple_display_pipe *pipe)
 	return 0;
 }
 
-static void mcde_display_disable_vblank(struct drm_simple_display_pipe *pipe)
+static void mcde_crtc_disable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct mcde *mcde = to_mcde(drm);
 
@@ -1474,13 +1494,55 @@ static void mcde_display_disable_vblank(struct drm_simple_display_pipe *pipe)
 	writel(0xFFFFFFFF, mcde->regs + MCDE_RISPP);
 }
 
-static struct drm_simple_display_pipe_funcs mcde_display_funcs = {
-	.check = mcde_display_check,
-	.enable = mcde_display_enable,
-	.disable = mcde_display_disable,
-	.update = mcde_display_update,
-	.enable_vblank = mcde_display_enable_vblank,
-	.disable_vblank = mcde_display_disable_vblank,
+static int mcde_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static const struct drm_crtc_funcs mcde_crtc_funcs = {
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.destroy		= drm_crtc_cleanup,
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+	.enable_vblank		= mcde_crtc_enable_vblank,
+	.disable_vblank		= mcde_crtc_disable_vblank,
+};
+
+static const struct drm_crtc_helper_funcs mcde_crtc_helper_funcs = {
+	.atomic_check	= mcde_crtc_helper_atomic_check,
+	.atomic_enable	= mcde_crtc_helper_atomic_enable,
+	.atomic_disable	= mcde_crtc_helper_atomic_disable,
+	.atomic_flush	= mcde_crtc_helper_atomic_flush,
+};
+
+static const struct drm_plane_funcs mcde_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
+	.destroy		= drm_plane_cleanup,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+};
+
+static const struct drm_plane_helper_funcs mcde_plane_helper_funcs = {
+	.prepare_fb	= drm_gem_plane_helper_prepare_fb,
+	.atomic_check	= mcde_plane_helper_atomic_check,
+	.atomic_update	= mcde_plane_helper_atomic_update,
+};
+
+static const struct drm_encoder_funcs mcde_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
 int mcde_display_init(struct drm_device *drm)
@@ -1510,14 +1572,29 @@ int mcde_display_init(struct drm_device *drm)
 	if (ret)
 		return ret;
 
-	ret = drm_simple_display_pipe_init(drm, &mcde->pipe,
-					   &mcde_display_funcs,
-					   formats, ARRAY_SIZE(formats),
-					   NULL,
-					   mcde->connector);
+	ret = drm_universal_plane_init(drm, &mcde->plane, 0,
+				       &mcde_plane_funcs,
+				       formats, ARRAY_SIZE(formats),
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+
+	drm_plane_helper_add(&mcde->plane, &mcde_plane_helper_funcs);
+
+	ret = drm_crtc_init_with_planes(drm, &mcde->crtc, &mcde->plane,
+					NULL, &mcde_crtc_funcs, NULL);
 	if (ret)
 		return ret;
 
+	drm_crtc_helper_add(&mcde->crtc, &mcde_crtc_helper_funcs);
+
+	ret = drm_encoder_init(drm, &mcde->encoder, &mcde_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+
+	mcde->encoder.possible_crtcs = drm_crtc_mask(&mcde->crtc);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mcde_display_init);
diff --git a/drivers/gpu/drm/mcde/mcde_drm.h b/drivers/gpu/drm/mcde/mcde_drm.h
index ecb70b4b737c0e..edbb4580a7d81e 100644
--- a/drivers/gpu/drm/mcde/mcde_drm.h
+++ b/drivers/gpu/drm/mcde/mcde_drm.h
@@ -4,7 +4,7 @@
  * Parts of this file were based on the MCDE driver by Marcus Lorentzon
  * (C) ST-Ericsson SA 2013
  */
-#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_encoder.h>
 
 #ifndef _MCDE_DRM_H_
 #define _MCDE_DRM_H_
@@ -72,7 +72,9 @@ struct mcde {
 	struct drm_panel *panel;
 	struct drm_bridge *bridge;
 	struct drm_connector *connector;
-	struct drm_simple_display_pipe pipe;
+	struct drm_plane plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 	struct mipi_dsi_device *mdsi;
 	bool dpi_output;
 	s16 stride;
@@ -95,7 +97,11 @@ struct mcde {
 	struct regulator *vana;
 };
 
-#define to_mcde(dev) container_of(dev, struct mcde, drm)
+static inline struct mcde *to_mcde(struct drm_device *dev)
+{
+	return container_of(dev, struct mcde, drm);
+}
+
 
 static inline bool mcde_flow_is_video(struct mcde *mcde)
 {
diff --git a/drivers/gpu/drm/mcde/mcde_drv.c b/drivers/gpu/drm/mcde/mcde_drv.c
index 53275b575f0cbb..759e279b645a81 100644
--- a/drivers/gpu/drm/mcde/mcde_drv.c
+++ b/drivers/gpu/drm/mcde/mcde_drv.c
@@ -187,8 +187,7 @@ static int mcde_modeset_init(struct drm_device *drm)
 	}
 
 	/* Attach the bridge. */
-	ret = drm_simple_display_pipe_attach_bridge(&mcde->pipe,
-						    mcde->bridge);
+	ret = drm_bridge_attach(&mcde->encoder, mcde->bridge, NULL, 0);
 	if (ret) {
 		dev_err(drm->dev, "failed to attach display output bridge\n");
 		return ret;
diff --git a/drivers/gpu/drm/mcde/mcde_dsi.c b/drivers/gpu/drm/mcde/mcde_dsi.c
index 6943725818407a..daac8d67913d23 100644
--- a/drivers/gpu/drm/mcde/mcde_dsi.c
+++ b/drivers/gpu/drm/mcde/mcde_dsi.c
@@ -11,17 +11,15 @@
 #include <linux/regulator/consumer.h>
 #include <video/mipi_display.h>
 
-#include <drm/drm_atomic_helper.h>
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_device.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_mipi_dsi.h>
-#include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
-#include <drm/drm_probe_helper.h>
 
 #include "mcde_drm.h"
 #include "mcde_dsi_regs.h"
@@ -39,7 +37,6 @@ struct mcde_dsi {
 	struct device *dev;
 	struct mcde *mcde;
 	struct drm_bridge bridge;
-	struct drm_panel *panel;
 	struct mipi_dsi_host dsi_host;
 	struct mipi_dsi_device *mdsi;
 	const struct drm_display_mode *mode;
@@ -1076,8 +1073,6 @@ static int mcde_dsi_bind(struct device *dev, struct device *master,
 	struct drm_device *drm = data;
 	struct mcde *mcde = to_mcde(drm);
 	struct mcde_dsi *d = dev_get_drvdata(dev);
-	struct device_node *child;
-	struct drm_panel *panel = NULL;
 	struct drm_bridge *bridge __free(drm_bridge_put) = NULL;
 
 	if (!of_get_available_child_count(dev->of_node)) {
@@ -1104,44 +1099,22 @@ static int mcde_dsi_bind(struct device *dev, struct device *master,
 	}
 
 	/* Look for a panel as a child to this node */
-	for_each_available_child_of_node(dev->of_node, child) {
-		panel = of_drm_find_panel(child);
-		if (IS_ERR(panel)) {
-			dev_err(dev, "failed to find panel try bridge (%ld)\n",
-				PTR_ERR(panel));
-			panel = NULL;
-
-			bridge = of_drm_find_and_get_bridge(child);
-			if (!bridge) {
-				dev_err(dev, "failed to find bridge\n");
-				of_node_put(child);
-				return -EINVAL;
-			}
+	for_each_available_child_of_node_scoped(dev->of_node, child) {
+		bridge = of_drm_find_and_get_bridge(child);
+		if (!bridge) {
+			dev_err(dev, "failed to find bridge\n");
+			return -EINVAL;
 		}
 
-		if (panel || bridge) {
-			of_node_put(child);
-			break;
-		}
+		break;
 	}
-	if (panel) {
-		bridge = drm_panel_bridge_add_typed(panel,
-						    DRM_MODE_CONNECTOR_DSI);
-		drm_panel_put(panel);
-		if (IS_ERR(bridge)) {
-			dev_err(dev, "error adding panel bridge\n");
-			return PTR_ERR(bridge);
-		}
-		drm_bridge_get(bridge);
-		dev_info(dev, "connected to panel\n");
-		d->panel = panel;
-	} else if (bridge) {
+	if (!bridge) {
+		dev_err(dev, "no bridge\n");
+		return -ENODEV;
+	} else if (!drm_bridge_is_panel(bridge)) {
 		/* TODO: AV8100 HDMI encoder goes here for example */
 		dev_info(dev, "connected to non-panel bridge (unsupported)\n");
 		return -ENODEV;
-	} else {
-		dev_err(dev, "no panel or bridge\n");
-		return -ENODEV;
 	}
 
 	d->bridge.next_bridge = drm_bridge_get(bridge);
@@ -1163,8 +1136,6 @@ static void mcde_dsi_unbind(struct device *dev, struct device *master,
 {
 	struct mcde_dsi *d = dev_get_drvdata(dev);
 
-	if (d->panel)
-		drm_panel_bridge_remove(d->bridge.next_bridge);
 	regmap_update_bits(d->prcmu, PRCM_DSI_SW_RESET,
 			   PRCM_DSI_SW_RESET_DSI0_SW_RESETN, 0);
 }
diff --git a/drivers/gpu/drm/mediatek/mtk_crtc.c b/drivers/gpu/drm/mediatek/mtk_crtc.c
index 97e3ff412e6ee5..2920f3198ef6ab 100644
--- a/drivers/gpu/drm/mediatek/mtk_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_crtc.c
@@ -150,19 +150,17 @@ static void mtk_crtc_destroy(struct drm_crtc *crtc)
 	drm_crtc_cleanup(crtc);
 }
 
-static void mtk_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *mtk_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct mtk_crtc_state *state;
 
-	if (crtc->state) {
-		__drm_atomic_helper_crtc_destroy_state(crtc->state);
-		kfree(to_mtk_crtc_state(crtc->state));
-	}
-	crtc->state = NULL;
-
 	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
+
+	return &state->base;
 }
 
 static struct drm_crtc_state *mtk_crtc_duplicate_state(struct drm_crtc *crtc)
@@ -886,7 +884,7 @@ static const struct drm_crtc_funcs mtk_crtc_funcs = {
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= drm_atomic_helper_page_flip,
 	.destroy		= mtk_crtc_destroy,
-	.reset			= mtk_crtc_reset,
+	.atomic_create_state	= mtk_crtc_create_state,
 	.atomic_duplicate_state	= mtk_crtc_duplicate_state,
 	.atomic_destroy_state	= mtk_crtc_destroy_state,
 	.enable_vblank		= mtk_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index 7961139333adbc..f82506e10fd59e 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -21,12 +21,12 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "mtk_ddp_comp.h"
 #include "mtk_disp_drv.h"
@@ -914,12 +914,16 @@ void mtk_dsi_ddp_stop(struct device *dev)
 	mtk_dsi_poweroff(dsi);
 }
 
+static const struct drm_encoder_funcs mtk_dsi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static int mtk_dsi_encoder_init(struct drm_device *drm, struct mtk_dsi *dsi)
 {
 	int ret;
 
-	ret = drm_simple_encoder_init(drm, &dsi->encoder,
-				      DRM_MODE_ENCODER_DSI);
+	ret = drm_encoder_init(drm, &dsi->encoder, &mtk_dsi_encoder_funcs,
+			       DRM_MODE_ENCODER_DSI, NULL);
 	if (ret) {
 		drm_err(drm, "Failed to encoder init to drm\n");
 		return ret;
diff --git a/drivers/gpu/drm/mediatek/mtk_plane.c b/drivers/gpu/drm/mediatek/mtk_plane.c
index 2598c8edb443d2..3fad7f2ff00436 100644
--- a/drivers/gpu/drm/mediatek/mtk_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_plane.c
@@ -25,26 +25,21 @@ static const u64 modifiers[] = {
 	DRM_FORMAT_MOD_INVALID,
 };
 
-static void mtk_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *mtk_plane_create_state(struct drm_plane *plane)
 {
 	struct mtk_plane_state *state;
 
-	if (plane->state) {
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-
-		state = to_mtk_plane_state(plane->state);
-		memset(state, 0, sizeof(*state));
-	} else {
-		state = kzalloc_obj(*state);
-		if (!state)
-			return;
-	}
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	__drm_atomic_helper_plane_reset(plane, &state->base);
+	__drm_atomic_helper_plane_state_init(&state->base, plane);
 
 	state->base.plane = plane;
 	state->pending.format = DRM_FORMAT_RGB565;
 	state->pending.modifier = DRM_FORMAT_MOD_LINEAR;
+
+	return &state->base;
 }
 
 static struct drm_plane_state *mtk_plane_duplicate_state(struct drm_plane *plane)
@@ -221,7 +216,7 @@ static const struct drm_plane_funcs mtk_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = mtk_plane_reset,
+	.atomic_create_state = mtk_plane_create_state,
 	.atomic_duplicate_state = mtk_plane_duplicate_state,
 	.atomic_destroy_state = mtk_plane_destroy_state,
 	.format_mod_supported = mtk_plane_format_mod_supported,
diff --git a/drivers/gpu/drm/meson/meson_crtc.c b/drivers/gpu/drm/meson/meson_crtc.c
index b77d2d24ead1c0..056dbea8deecb7 100644
--- a/drivers/gpu/drm/meson/meson_crtc.c
+++ b/drivers/gpu/drm/meson/meson_crtc.c
@@ -74,7 +74,7 @@ static const struct drm_crtc_funcs meson_crtc_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.destroy		= drm_crtc_cleanup,
 	.page_flip		= drm_atomic_helper_page_flip,
-	.reset			= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.set_config             = drm_atomic_helper_set_config,
 	.enable_vblank		= meson_crtc_enable_vblank,
 	.disable_vblank		= meson_crtc_disable_vblank,
diff --git a/drivers/gpu/drm/meson/meson_encoder_cvbs.c b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
index 22cacb1660c4fa..cdb84d2283f832 100644
--- a/drivers/gpu/drm/meson/meson_encoder_cvbs.c
+++ b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
@@ -17,8 +17,8 @@
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_device.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "meson_registers.h"
 #include "meson_vclk.h"
@@ -218,6 +218,10 @@ static const struct drm_bridge_funcs meson_encoder_cvbs_bridge_funcs = {
 	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
+static const struct drm_encoder_funcs meson_encoder_cvbs_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 int meson_encoder_cvbs_probe(struct meson_drm *priv)
 {
 	struct drm_device *drm = priv->drm;
@@ -257,8 +261,9 @@ int meson_encoder_cvbs_probe(struct meson_drm *priv)
 	meson_encoder_cvbs->priv = priv;
 
 	/* Encoder */
-	ret = drm_simple_encoder_init(priv->drm, &meson_encoder_cvbs->encoder,
-				      DRM_MODE_ENCODER_TVDAC);
+	ret = drm_encoder_init(priv->drm, &meson_encoder_cvbs->encoder,
+			       &meson_encoder_cvbs_funcs,
+			       DRM_MODE_ENCODER_TVDAC, NULL);
 	if (ret)
 		return dev_err_probe(priv->dev, ret,
 				     "Failed to init CVBS encoder\n");
diff --git a/drivers/gpu/drm/meson/meson_encoder_dsi.c b/drivers/gpu/drm/meson/meson_encoder_dsi.c
index 3e422b612f74b9..faa309cb97a68c 100644
--- a/drivers/gpu/drm/meson/meson_encoder_dsi.c
+++ b/drivers/gpu/drm/meson/meson_encoder_dsi.c
@@ -10,10 +10,10 @@
 #include <linux/of_graph.h>
 
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_device.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_probe_helper.h>
 
 #include "meson_drv.h"
@@ -99,6 +99,10 @@ static const struct drm_bridge_funcs meson_encoder_dsi_bridge_funcs = {
 	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
+static const struct drm_encoder_funcs meson_encoder_dsi_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 int meson_encoder_dsi_probe(struct meson_drm *priv)
 {
 	struct meson_encoder_dsi *meson_encoder_dsi;
@@ -133,8 +137,9 @@ int meson_encoder_dsi_probe(struct meson_drm *priv)
 	meson_encoder_dsi->priv = priv;
 
 	/* Encoder */
-	ret = drm_simple_encoder_init(priv->drm, &meson_encoder_dsi->encoder,
-				      DRM_MODE_ENCODER_DSI);
+	ret = drm_encoder_init(priv->drm, &meson_encoder_dsi->encoder,
+			       &meson_encoder_dsi_funcs, DRM_MODE_ENCODER_DSI,
+			       NULL);
 	if (ret)
 		return dev_err_probe(priv->dev, ret,
 				     "Failed to init DSI encoder\n");
diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.c b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
index 0c7a72cb514ae4..c4355c5cc340db 100644
--- a/drivers/gpu/drm/meson/meson_encoder_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
@@ -23,8 +23,8 @@
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_device.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include <linux/media-bus-format.h>
 #include <linux/videodev2.h>
@@ -369,6 +369,10 @@ static const struct drm_bridge_funcs meson_encoder_hdmi_bridge_funcs = {
 	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 };
 
+static const struct drm_encoder_funcs meson_encoder_hdmi_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 int meson_encoder_hdmi_probe(struct meson_drm *priv)
 {
 	struct meson_encoder_hdmi *meson_encoder_hdmi;
@@ -407,8 +411,9 @@ int meson_encoder_hdmi_probe(struct meson_drm *priv)
 	meson_encoder_hdmi->priv = priv;
 
 	/* Encoder */
-	ret = drm_simple_encoder_init(priv->drm, &meson_encoder_hdmi->encoder,
-				      DRM_MODE_ENCODER_TMDS);
+	ret = drm_encoder_init(priv->drm, &meson_encoder_hdmi->encoder,
+			       &meson_encoder_hdmi_funcs,
+			       DRM_MODE_ENCODER_TMDS, NULL);
 	if (ret) {
 		dev_err_probe(priv->dev, ret, "Failed to init HDMI encoder\n");
 		goto err_put_node;
diff --git a/drivers/gpu/drm/meson/meson_overlay.c b/drivers/gpu/drm/meson/meson_overlay.c
index 7047807f1a3e2a..930119e18a4f5f 100644
--- a/drivers/gpu/drm/meson/meson_overlay.c
+++ b/drivers/gpu/drm/meson/meson_overlay.c
@@ -802,7 +802,7 @@ static const struct drm_plane_funcs meson_overlay_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 	.format_mod_supported   = meson_overlay_format_mod_supported,
diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c
index 2d88fe87a11acd..cf8c90158fcf85 100644
--- a/drivers/gpu/drm/meson/meson_plane.c
+++ b/drivers/gpu/drm/meson/meson_plane.c
@@ -466,7 +466,7 @@ static const struct drm_plane_funcs meson_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 	.format_mod_supported   = meson_plane_format_mod_supported,
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index 3732c62351bbb3..55ca7cd10e65b1 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -17,6 +17,7 @@
 #include <drm/drm_encoder.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_plane.h>
 
 #include "mgag200_reg.h"
@@ -386,7 +387,8 @@ int mgag200_primary_plane_helper_get_scanout_buffer(struct drm_plane *plane,
 	.update_plane = drm_atomic_helper_update_plane, \
 	.disable_plane = drm_atomic_helper_disable_plane, \
 	.destroy = drm_plane_cleanup, \
-	DRM_GEM_SHADOW_PLANE_FUNCS
+	DRM_GEM_SHADOW_PLANE_FUNCS, \
+	DRM_PANIC_PLANE_FUNCS
 
 void mgag200_crtc_fill_gamma(struct mga_device *mdev, const struct drm_format_info *format);
 void mgag200_crtc_load_gamma(struct mga_device *mdev,
@@ -407,15 +409,15 @@ void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic
 	.atomic_enable = mgag200_crtc_helper_atomic_enable, \
 	.atomic_disable = mgag200_crtc_helper_atomic_disable
 
-void mgag200_crtc_reset(struct drm_crtc *crtc);
+struct drm_crtc_state *mgag200_crtc_atomic_create_state(struct drm_crtc *crtc);
 struct drm_crtc_state *mgag200_crtc_atomic_duplicate_state(struct drm_crtc *crtc);
 void mgag200_crtc_atomic_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state);
 
 #define MGAG200_CRTC_FUNCS \
-	.reset = mgag200_crtc_reset, \
 	.destroy = drm_crtc_cleanup, \
 	.set_config = drm_atomic_helper_set_config, \
 	.page_flip = drm_atomic_helper_page_flip, \
+	.atomic_create_state = mgag200_crtc_atomic_create_state, \
 	.atomic_duplicate_state = mgag200_crtc_atomic_duplicate_state, \
 	.atomic_destroy_state = mgag200_crtc_atomic_destroy_state
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index 7e07fc3f1a6030..4e952655e4f51f 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -679,18 +679,17 @@ void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic
 	mgag200_disable_display(mdev);
 }
 
-void mgag200_crtc_reset(struct drm_crtc *crtc)
+struct drm_crtc_state *mgag200_crtc_atomic_create_state(struct drm_crtc *crtc)
 {
 	struct mgag200_crtc_state *mgag200_crtc_state;
 
-	if (crtc->state)
-		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
-
 	mgag200_crtc_state = kzalloc_obj(*mgag200_crtc_state);
-	if (mgag200_crtc_state)
-		__drm_atomic_helper_crtc_reset(crtc, &mgag200_crtc_state->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	if (!mgag200_crtc_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(&mgag200_crtc_state->base, crtc);
+
+	return &mgag200_crtc_state->base;
 }
 
 struct drm_crtc_state *mgag200_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
index 42d0a529b4d515..77a2d1f6700d0a 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -1140,17 +1140,17 @@ void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
 	DPU_ATRACE_END("crtc_commit");
 }
 
-static void dpu_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *dpu_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct dpu_crtc_state *cstate = kzalloc_obj(*cstate);
+	struct dpu_crtc_state *cstate;
 
-	if (crtc->state)
-		dpu_crtc_destroy_state(crtc, crtc->state);
+	cstate = kzalloc_obj(*cstate);
+	if (!cstate)
+		return ERR_PTR(-ENOMEM);
 
-	if (cstate)
-		__drm_atomic_helper_crtc_reset(crtc, &cstate->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	__drm_atomic_helper_crtc_state_init(&cstate->base, crtc);
+
+	return &cstate->base;
 }
 
 /**
@@ -1843,7 +1843,7 @@ static int dpu_crtc_late_register(struct drm_crtc *crtc)
 static const struct drm_crtc_funcs dpu_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = dpu_crtc_reset,
+	.atomic_create_state = dpu_crtc_create_state,
 	.atomic_duplicate_state = dpu_crtc_duplicate_state,
 	.atomic_destroy_state = dpu_crtc_destroy_state,
 	.atomic_print_state = dpu_crtc_atomic_print_state,
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
index 7b92082d35a642..1a471cdf767b1c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
@@ -1728,32 +1728,28 @@ static void dpu_plane_atomic_print_state(struct drm_printer *p,
 	}
 }
 
-static void dpu_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *dpu_plane_create_state(struct drm_plane *plane)
 {
 	struct dpu_plane *pdpu;
 	struct dpu_plane_state *pstate;
 
 	if (!plane) {
 		DPU_ERROR("invalid plane\n");
-		return;
+		return ERR_PTR(-EINVAL);
 	}
 
 	pdpu = to_dpu_plane(plane);
 	DPU_DEBUG_PLANE(pdpu, "\n");
 
-	/* remove previous state, if present */
-	if (plane->state) {
-		dpu_plane_destroy_state(plane, plane->state);
-		plane->state = NULL;
-	}
-
 	pstate = kzalloc_obj(*pstate);
 	if (!pstate) {
 		DPU_ERROR_PLANE(pdpu, "failed to allocate state\n");
-		return;
+		return ERR_PTR(-ENOMEM);
 	}
 
-	__drm_atomic_helper_plane_reset(plane, &pstate->base);
+	__drm_atomic_helper_plane_state_init(&pstate->base, plane);
+
+	return &pstate->base;
 }
 
 #ifdef CONFIG_DEBUG_FS
@@ -1796,7 +1792,7 @@ static bool dpu_plane_format_mod_supported(struct drm_plane *plane,
 static const struct drm_plane_funcs dpu_plane_funcs = {
 		.update_plane = drm_atomic_helper_update_plane,
 		.disable_plane = drm_atomic_helper_disable_plane,
-		.reset = dpu_plane_reset,
+		.atomic_create_state = dpu_plane_create_state,
 		.atomic_duplicate_state = dpu_plane_duplicate_state,
 		.atomic_destroy_state = dpu_plane_destroy_state,
 		.atomic_print_state = dpu_plane_atomic_print_state,
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
index 57dfce58450b32..93050b3c3548cf 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
@@ -477,7 +477,7 @@ static const struct drm_crtc_funcs mdp4_crtc_funcs = {
 	.page_flip = drm_atomic_helper_page_flip,
 	.cursor_set = mdp4_crtc_cursor_set,
 	.cursor_move = mdp4_crtc_cursor_move,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank  = msm_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
index 9459f70ce0badc..f9067e508f4cbe 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
@@ -71,7 +71,7 @@ static const struct drm_plane_funcs mdp4_plane_funcs = {
 		.update_plane = drm_atomic_helper_update_plane,
 		.disable_plane = drm_atomic_helper_disable_plane,
 		.set_property = mdp4_plane_set_property,
-		.reset = drm_atomic_helper_plane_reset,
+		.atomic_create_state = drm_atomic_helper_plane_create_state,
 		.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 		.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
index 4c4a897fc1eeee..1e1fff37380c76 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
@@ -1128,23 +1128,23 @@ static void mdp5_crtc_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state
 	kfree(mdp5_cstate);
 }
 
-static void mdp5_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *mdp5_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct mdp5_crtc_state *mdp5_cstate = kzalloc_obj(*mdp5_cstate);
+	struct mdp5_crtc_state *mdp5_cstate;
 
-	if (crtc->state)
-		mdp5_crtc_destroy_state(crtc, crtc->state);
+	mdp5_cstate = kzalloc_obj(*mdp5_cstate);
+	if (!mdp5_cstate)
+		return ERR_PTR(-ENOMEM);
 
-	if (mdp5_cstate)
-		__drm_atomic_helper_crtc_reset(crtc, &mdp5_cstate->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	__drm_atomic_helper_crtc_state_init(&mdp5_cstate->base, crtc);
+
+	return &mdp5_cstate->base;
 }
 
 static const struct drm_crtc_funcs mdp5_crtc_no_lm_cursor_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = mdp5_crtc_reset,
+	.atomic_create_state = mdp5_crtc_create_state,
 	.atomic_duplicate_state = mdp5_crtc_duplicate_state,
 	.atomic_destroy_state = mdp5_crtc_destroy_state,
 	.atomic_print_state = mdp5_crtc_atomic_print_state,
@@ -1157,7 +1157,7 @@ static const struct drm_crtc_funcs mdp5_crtc_no_lm_cursor_funcs = {
 static const struct drm_crtc_funcs mdp5_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = mdp5_crtc_reset,
+	.atomic_create_state = mdp5_crtc_create_state,
 	.atomic_duplicate_state = mdp5_crtc_duplicate_state,
 	.atomic_destroy_state = mdp5_crtc_destroy_state,
 	.cursor_set = mdp5_crtc_cursor_set,
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
index 841f444a8d68d3..7e878c69824779 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
@@ -80,19 +80,15 @@ mdp5_plane_atomic_print_state(struct drm_printer *p,
 	drm_printf(p, "\tstage=%s\n", stage2name(pstate->stage));
 }
 
-static void mdp5_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *mdp5_plane_create_state(struct drm_plane *plane)
 {
 	struct mdp5_plane_state *mdp5_state;
-
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-
-	kfree(to_mdp5_plane_state(plane->state));
-	plane->state = NULL;
 	mdp5_state = kzalloc_obj(*mdp5_state);
 	if (!mdp5_state)
-		return;
-	__drm_atomic_helper_plane_reset(plane, &mdp5_state->base);
+		return ERR_PTR(-ENOMEM);
+	__drm_atomic_helper_plane_state_init(&mdp5_state->base, plane);
+
+	return &mdp5_state->base;
 }
 
 static struct drm_plane_state *
@@ -126,7 +122,7 @@ static void mdp5_plane_destroy_state(struct drm_plane *plane,
 static const struct drm_plane_funcs mdp5_plane_funcs = {
 		.update_plane = drm_atomic_helper_update_plane,
 		.disable_plane = drm_atomic_helper_disable_plane,
-		.reset = mdp5_plane_reset,
+		.atomic_create_state = mdp5_plane_create_state,
 		.atomic_duplicate_state = mdp5_plane_duplicate_state,
 		.atomic_destroy_state = mdp5_plane_destroy_state,
 		.atomic_print_state = mdp5_plane_atomic_print_state,
diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c
index 4dcbd9b99d06b8..6abe30bdff120c 100644
--- a/drivers/gpu/drm/msm/dp/dp_display.c
+++ b/drivers/gpu/drm/msm/dp/dp_display.c
@@ -15,6 +15,7 @@
 #include <drm/display/drm_dp_aux_bus.h>
 #include <drm/display/drm_hdmi_audio_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_panel.h>
 
 #include "msm_drv.h"
 #include "msm_kms.h"
diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
index 3c9f01ed627133..c726fc628a087f 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.c
+++ b/drivers/gpu/drm/msm/dsi/dsi.c
@@ -5,6 +5,8 @@
 
 #include "dsi.h"
 
+#include <drm/drm_panel.h>
+
 bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi)
 {
 	unsigned long host_flags = msm_dsi_host_get_mode_flags(msm_dsi->host);
@@ -267,4 +269,3 @@ void msm_dsi_snapshot(struct msm_disp_state *disp_state, struct msm_dsi *msm_dsi
 	msm_dsi_host_snapshot(disp_state, msm_dsi->host);
 	msm_dsi_phy_snapshot(disp_state, msm_dsi->phy);
 }
-
diff --git a/drivers/gpu/drm/msm/msm_fb.c b/drivers/gpu/drm/msm/msm_fb.c
index 60c108d35d2a1c..1e236416e15f08 100644
--- a/drivers/gpu/drm/msm/msm_fb.c
+++ b/drivers/gpu/drm/msm/msm_fb.c
@@ -4,6 +4,8 @@
  * Author: Rob Clark <robdclark@gmail.com>
  */
 
+#include <linux/seq_file.h>
+
 #include <drm/drm_crtc.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c
index 9d2788f79acee1..dba1cbeba11862 100644
--- a/drivers/gpu/drm/msm/msm_gem_shrinker.c
+++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c
@@ -4,6 +4,8 @@
  * Author: Rob Clark <robdclark@gmail.com>
  */
 
+#include <linux/seq_file.h>
+#include <linux/swap.h>
 #include <linux/vmalloc.h>
 #include <linux/sched/mm.h>
 
@@ -102,7 +104,7 @@ with_vm_locks(void (*fn)(struct drm_gem_object *obj),
 }
 
 static bool
-purge(struct drm_gem_object *obj, struct ww_acquire_ctx *unused)
+purge(struct drm_gem_object *obj)
 {
 	if (!is_purgeable(to_msm_bo(obj)))
 		return false;
@@ -114,7 +116,7 @@ purge(struct drm_gem_object *obj, struct ww_acquire_ctx *unused)
 }
 
 static bool
-evict(struct drm_gem_object *obj, struct ww_acquire_ctx *unused)
+evict(struct drm_gem_object *obj)
 {
 	if (is_unevictable(to_msm_bo(obj)))
 		return false;
@@ -133,21 +135,21 @@ wait_for_idle(struct drm_gem_object *obj)
 }
 
 static bool
-active_purge(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
+active_purge(struct drm_gem_object *obj)
 {
 	if (!wait_for_idle(obj))
 		return false;
 
-	return purge(obj, ticket);
+	return purge(obj);
 }
 
 static bool
-active_evict(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
+active_evict(struct drm_gem_object *obj)
 {
 	if (!wait_for_idle(obj))
 		return false;
 
-	return evict(obj, ticket);
+	return evict(obj);
 }
 
 static unsigned long
@@ -156,7 +158,7 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 	struct msm_drm_private *priv = shrinker->private_data;
 	struct {
 		struct drm_gem_lru *lru;
-		bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket);
+		bool (*shrink)(struct drm_gem_object *obj);
 		bool cond;
 		unsigned long freed;
 		unsigned long remaining;
@@ -180,8 +182,7 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 		stages[i].freed =
 			drm_gem_lru_scan(priv->dev, stages[i].lru, nr,
 					 &stages[i].remaining,
-					 stages[i].shrink,
-					 NULL);
+					 stages[i].shrink);
 		nr -= stages[i].freed;
 		freed += stages[i].freed;
 		remaining += stages[i].remaining;
@@ -222,7 +223,7 @@ msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
 static const int vmap_shrink_limit = 15;
 
 static bool
-vmap_shrink(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
+vmap_shrink(struct drm_gem_object *obj)
 {
 	if (!is_vunmapable(to_msm_bo(obj)))
 		return false;
@@ -250,8 +251,7 @@ msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
 		unmapped += drm_gem_lru_scan(priv->dev, lrus[idx],
 					     vmap_shrink_limit - unmapped,
 					     &remaining,
-					     vmap_shrink,
-					     NULL);
+					     vmap_shrink);
 	}
 
 	*(unsigned long *)ptr += unmapped;
diff --git a/drivers/gpu/drm/msm/msm_perfcntr.c b/drivers/gpu/drm/msm/msm_perfcntr.c
index ce65b1160955e2..7fa2e858bd0823 100644
--- a/drivers/gpu/drm/msm/msm_perfcntr.c
+++ b/drivers/gpu/drm/msm/msm_perfcntr.c
@@ -543,8 +543,8 @@ msm_ioctl_perfcntr_config(struct drm_device *dev, void *data, struct drm_file *f
 
 		FD_PREPARE(fdf, O_CLOEXEC,
 			   anon_inode_getfile("[msm_perfcntrs]", &stream_fops, stream, 0));
-		if (fdf.err)
-			return fdf.err;
+		if (fdf->fd < 0)
+			return fdf->fd;
 
 		INIT_WORK(&stream->sel_work, sel_worker);
 		kthread_init_work(&stream->sample_work, sample_worker);
diff --git a/drivers/gpu/drm/mxsfb/lcdif_kms.c b/drivers/gpu/drm/mxsfb/lcdif_kms.c
index 79e0f483ca20d7..c67f58a408d0dd 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_kms.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_kms.c
@@ -596,18 +596,17 @@ static void lcdif_crtc_atomic_destroy_state(struct drm_crtc *crtc,
 	kfree(to_lcdif_crtc_state(state));
 }
 
-static void lcdif_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *lcdif_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct lcdif_crtc_state *state;
 
-	if (crtc->state)
-		lcdif_crtc_atomic_destroy_state(crtc, crtc->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	crtc->state = NULL;
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
 
-	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
+	return &state->base;
 }
 
 static struct drm_crtc_state *
@@ -659,7 +658,7 @@ static const struct drm_crtc_helper_funcs lcdif_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs lcdif_crtc_funcs = {
-	.reset = lcdif_crtc_reset,
+	.atomic_create_state = lcdif_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -724,7 +723,7 @@ static const struct drm_plane_funcs lcdif_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_kms.c b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
index d8ebebc5314bba..6e82f8f767935c 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_kms.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
@@ -484,7 +484,7 @@ static const struct drm_crtc_helper_funcs mxsfb_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs mxsfb_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -495,7 +495,7 @@ static const struct drm_crtc_funcs mxsfb_crtc_funcs = {
 };
 
 static const struct drm_crtc_funcs mxsfb_crtc_with_crc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -643,7 +643,7 @@ static const struct drm_plane_funcs mxsfb_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/nouveau/dispnv04/dfp.c b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
index c9f96ec8455d15..00eb2aacbe932b 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/dfp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
@@ -344,9 +344,8 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
 		regp->fp_control |= (2 << 24);
 	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS) {
 		bool duallink = false, dummy;
-		if (nv_connector->edid &&
-		    nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
-			duallink = (((u8 *)nv_connector->edid)[121] == 2);
+		if (nv_connector->spwg_links) {
+			duallink = nv_connector->spwg_links == 2;
 		} else {
 			nouveau_bios_parse_lvds_table(dev, output_mode->clock,
 						      &duallink, &dummy);
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index 2c66e480b51160..add19d479ebe44 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -1803,8 +1803,8 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_commit *st
 			lvds_dual = bios->fp.dual_link;
 			lvds_8bpc = bios->fp.if_is_24bit;
 		} else {
-			if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
-				if (((u8 *)nv_connector->edid)[121] == 2)
+			if (nv_connector->spwg_links) {
+				if (nv_connector->spwg_links == 2)
 					lvds_dual = true;
 			} else
 			if (mode->clock >= bios->fp.duallink_transition_clk) {
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.c b/drivers/gpu/drm/nouveau/dispnv50/head.c
index 1affa0378bd029..6a3a93de0cbfe7 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/head.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/head.c
@@ -491,18 +491,16 @@ nv50_head_atomic_duplicate_state(struct drm_crtc *crtc)
 	return &asyh->state;
 }
 
-static void
-nv50_head_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *nv50_head_create_state(struct drm_crtc *crtc)
 {
 	struct nv50_head_atom *asyh;
 
 	if (WARN_ON(!(asyh = kzalloc_obj(*asyh))))
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	if (crtc->state)
-		nv50_head_atomic_destroy_state(crtc, crtc->state);
+	__drm_atomic_helper_crtc_state_init(&asyh->state, crtc);
 
-	__drm_atomic_helper_crtc_reset(crtc, &asyh->state);
+	return &asyh->state;
 }
 
 static int
@@ -525,7 +523,7 @@ nv50_head_destroy(struct drm_crtc *crtc)
 
 static const struct drm_crtc_funcs
 nv50_head_func = {
-	.reset = nv50_head_reset,
+	.atomic_create_state = nv50_head_create_state,
 	.destroy = nv50_head_destroy,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -539,7 +537,7 @@ nv50_head_func = {
 
 static const struct drm_crtc_funcs
 nvd9_head_func = {
-	.reset = nv50_head_reset,
+	.atomic_create_state = nv50_head_create_state,
 	.destroy = nv50_head_destroy,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/nouveau/dispnv50/headca7d.c b/drivers/gpu/drm/nouveau/dispnv50/headca7d.c
index eeaeb15aa66469..678cbc78ca9f07 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/headca7d.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/headca7d.c
@@ -219,10 +219,11 @@ headca7d_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
 	struct nvif_push *push = &head->disp->core->chan.push;
 	struct nv50_head_mode *m = &asyh->mode;
+	const u64 hz = (u64)m->clock * 1000;
 	const int i = head->base.index;
 	int ret;
 
-	ret = PUSH_WAIT(push, 11);
+	ret = PUSH_WAIT(push, 14);
 	if (ret)
 		return ret;
 
@@ -245,11 +246,25 @@ headca7d_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
 	PUSH_MTHD(push, NVCA7D, HEAD_SET_CONTROL(i),
 		  NVDEF(NVCA7D, HEAD_SET_CONTROL, STRUCTURE, PROGRESSIVE));
 
+	/* The FREQUENCY methods carry only 31 HERTZ bits; the upper bits
+	 * of anything past 2.147GHz live in the HI methods
+	 * (EvoSetRasterParams9()). Truncating would scan out at pclk modulo 2^31.
+	 */
 	PUSH_MTHD(push, NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY(i),
-		  NVVAL(NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY, HERTZ, m->clock * 1000));
+		  NVVAL(NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY, HERTZ,
+			(u32)(hz & 0x7fffffff)));
 
 	PUSH_MTHD(push, NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY_MAX(i),
-		  NVVAL(NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY_MAX, HERTZ, m->clock * 1000));
+		  NVVAL(NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY_MAX, HERTZ,
+			(u32)(hz & 0x7fffffff)));
+
+	PUSH_MTHD(push, NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI(i),
+		  NVVAL(NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI, HERTZ,
+			(u32)(hz >> 31)),
+
+				HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI_MAX(i),
+		  NVVAL(NVCA7D, HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI_MAX, HERTZ,
+			(u32)(hz >> 31)));
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
index 15a322422f4e6e..74eb1dfcc0437b 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
@@ -39,6 +39,7 @@
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_panic.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/ttm/ttm_bo.h>
 
 #include "nouveau_bo.h"
@@ -752,18 +753,17 @@ nv50_wndw_zpos_default(struct drm_plane *plane)
 	       (plane->type == DRM_PLANE_TYPE_OVERLAY) ? 1 : 255;
 }
 
-static void
-nv50_wndw_reset(struct drm_plane *plane)
+static struct drm_plane_state *nv50_wndw_create_state(struct drm_plane *plane)
 {
 	struct nv50_wndw_atom *asyw;
 
-	if (WARN_ON(!(asyw = kzalloc_obj(*asyw))))
-		return;
+	asyw = kzalloc_obj(*asyw);
+	if (WARN_ON(!asyw))
+		return ERR_PTR(-ENOMEM);
 
-	if (plane->state)
-		plane->funcs->atomic_destroy_state(plane, plane->state);
+	__drm_atomic_helper_plane_state_init(&asyw->state, plane);
 
-	__drm_atomic_helper_plane_reset(plane, &asyw->state);
+	return &asyw->state;
 }
 
 static void
@@ -837,10 +837,11 @@ nv50_wndw = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = nv50_wndw_destroy,
-	.reset = nv50_wndw_reset,
+	.atomic_create_state = nv50_wndw_create_state,
 	.atomic_duplicate_state = nv50_wndw_atomic_duplicate_state,
 	.atomic_destroy_state = nv50_wndw_atomic_destroy_state,
 	.format_mod_supported = nv50_plane_format_mod_supported,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static const u64 nv50_cursor_format_modifiers[] = {
diff --git a/drivers/gpu/drm/nouveau/include/nvhw/class/clca7d.h b/drivers/gpu/drm/nouveau/include/nvhw/class/clca7d.h
index 0fec6fc21d445b..5754878ddc6936 100644
--- a/drivers/gpu/drm/nouveau/include/nvhw/class/clca7d.h
+++ b/drivers/gpu/drm/nouveau/include/nvhw/class/clca7d.h
@@ -734,6 +734,10 @@
 #define NVCA7D_HEAD_SET_CONTROL_CURSOR_COMPOSITION_BYPASS                       20:20
 #define NVCA7D_HEAD_SET_CONTROL_CURSOR_COMPOSITION_BYPASS_DISABLE               (0x00000000)
 #define NVCA7D_HEAD_SET_CONTROL_CURSOR_COMPOSITION_BYPASS_ENABLE                (0x00000001)
+#define NVCA7D_HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI(a)                             (0x000020C0 + (a)*0x00000800)
+#define NVCA7D_HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI_HERTZ                          3:0
+#define NVCA7D_HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI_MAX(a)                         (0x000020C4 + (a)*0x00000800)
+#define NVCA7D_HEAD_SET_PIXEL_CLOCK_FREQUENCY_HI_MAX_HERTZ                      3:0
 #define NVCA7D_HEAD_SET_SURFACE_ADDRESS_HI_CRC(a)                               (0x00002150 + (a)*0x00000800)
 #define NVCA7D_HEAD_SET_SURFACE_ADDRESS_HI_CRC_ADDRESS_HI                       31:0
 #define NVCA7D_HEAD_SET_SURFACE_ADDRESS_LO_CRC(a)                               (0x00002154 + (a)*0x00000800)
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
index ea8267e0d8da27..9e639df1da467a 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
@@ -4,6 +4,7 @@
 
 #define NV_DEVICE_V0_INFO                                                  0x00
 #define NV_DEVICE_V0_TIME                                                  0x01
+#define NV_DEVICE_V0_GCX_READY						   0x02
 
 struct nv_device_info_v0 {
 	__u8  version;
@@ -55,6 +56,15 @@ struct nv_device_time_v0 {
 	__u64 time;
 };
 
+#define NV_DEVICE_GC6_READY	BIT(0)
+#define NV_DEVICE_GCOFF_READY	BIT(1)
+
+struct nv_device_gcx_ready_v0 {
+	__u8 version;
+	__u8 pad01[6];
+	__u8 ready;
+};
+
 #define NV_DEVICE_INFO_UNIT                               (0xffffffffULL << 32)
 #define NV_DEVICE_INFO(n)                          ((n) | (0x00000000ULL << 32))
 #define NV_DEVICE_HOST(n)                          ((n) | (0x00000001ULL << 32))
diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h
index 7877a2a79da9b7..ce2fadcb05d9ef 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/device.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/device.h
@@ -22,4 +22,5 @@ int  nvif_device_ctor(struct nvif_client *, const char *name, struct nvif_device
 void nvif_device_dtor(struct nvif_device *);
 int  nvif_device_map(struct nvif_device *);
 u64  nvif_device_time(struct nvif_device *);
+int  nvif_device_gcx_ready(struct nvif_device *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h
index 64fed208e4cf85..ed5c6e0e68d3c1 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h
@@ -156,6 +156,10 @@ struct nvkm_gsp {
 		struct sg_table fbsr;
 	} sr;
 
+	struct {
+		bool use_raw_mode_comptagline_alloc;
+	} memsys;
+
 	struct {
 		struct nvkm_gsp_mem mem;
 
@@ -495,6 +499,8 @@ nvkm_gsp_event_dtor(struct nvkm_gsp_event *event)
 int nvkm_gsp_intr_stall(struct nvkm_gsp *, enum nvkm_subdev_type, int);
 int nvkm_gsp_intr_nonstall(struct nvkm_gsp *, enum nvkm_subdev_type, int);
 
+int nvkm_gsp_gcx_ready(struct nvkm_gsp *gsp);
+
 int gv100_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **);
 int tu102_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **);
 int tu116_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
index 112b674ed9c8a0..0172e0d200f81a 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
@@ -46,6 +46,7 @@ int nv4c_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct n
 int g84_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **);
 int g92_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **);
 int g94_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **);
+int mcp79_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **);
 int gf100_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **);
 int gf106_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **);
 int gk104_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **);
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 4542d5f4ded89f..e907c1d07887e6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -666,6 +666,10 @@ nouveau_abi16_ioctl_notifierobj_alloc(ABI16_IOCTL_ARGS)
 	if (unlikely(device->info.family >= NV_DEVICE_INFO_V0_FERMI))
 		return nouveau_abi16_put(abi16, -EINVAL);
 
+	/* zero size yields a zero-length node, underflowing args.limit */
+	if (unlikely(!info->size))
+		return nouveau_abi16_put(abi16, -EINVAL);
+
 	chan = nouveau_abi16_chan(abi16, info->channel);
 	if (!chan)
 		return nouveau_abi16_put(abi16, -ENOENT);
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index 21b56cc7605ce3..9e6906490db595 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -360,9 +360,10 @@ void nouveau_unregister_dsm_handler(void) {}
 void nouveau_switcheroo_optimus_dsm(void) {}
 #endif
 
-void *
+const struct drm_edid *
 nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector)
 {
+	const struct drm_edid *drm_edid;
 	struct acpi_device *acpidev;
 	int type, ret;
 	void *edid;
@@ -384,7 +385,34 @@ nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector)
 	if (ret < 0)
 		return NULL;
 
-	return edid;
+	/* Take what _DDC returned as-is first as its length is the only one that
+	 * can describe an HF-EEODB EDID, whose block count is not the byte 126
+	 * one. Only if that does not validate fall back to trimming the buffer
+	 * to the byte 126 size, which is what the padding some firmware appends
+	 * needs.
+	 */
+	drm_edid = drm_edid_alloc(edid, ret);
+	if (drm_edid && !drm_edid_valid(drm_edid)) {
+		drm_edid_free(drm_edid);
+		drm_edid = NULL;
+
+		if (ret > EDID_LENGTH) {
+			int trimmed = EDID_LENGTH *
+				      (1 + ((const struct edid *)edid)->extensions);
+
+			if (trimmed < ret)
+				drm_edid = drm_edid_alloc(edid, trimmed);
+			if (drm_edid && !drm_edid_valid(drm_edid)) {
+				drm_edid_free(drm_edid);
+				drm_edid = NULL;
+			}
+		}
+	}
+	kfree(edid);
+
+	if (!drm_edid)
+		drm_dbg_kms(dev, "Invalid EDID from ACPI _DDC\n");
+	return drm_edid;
 }
 
 bool nouveau_acpi_video_backlight_use_native(void)
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.h b/drivers/gpu/drm/nouveau/nouveau_acpi.h
index e39dd8b94b8b24..bc3ccca7b3389e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.h
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.h
@@ -4,13 +4,15 @@
 
 #define ROM_BIOS_PAGE 4096
 
+struct drm_edid;
+
 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
 bool nouveau_is_optimus(void);
 bool nouveau_is_v1_dsm(void);
 void nouveau_register_dsm_handler(void);
 void nouveau_unregister_dsm_handler(void);
 void nouveau_switcheroo_optimus_dsm(void);
-void *nouveau_acpi_edid(struct drm_device *, struct drm_connector *);
+const struct drm_edid *nouveau_acpi_edid(struct drm_device *, struct drm_connector *);
 bool nouveau_acpi_video_backlight_use_native(void);
 void nouveau_acpi_video_register_backlight(void);
 #else
@@ -19,7 +21,11 @@ static inline bool nouveau_is_v1_dsm(void) { return false; };
 static inline void nouveau_register_dsm_handler(void) {}
 static inline void nouveau_unregister_dsm_handler(void) {}
 static inline void nouveau_switcheroo_optimus_dsm(void) {}
-static inline void *nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector) { return NULL; }
+static inline const struct drm_edid *
+nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector)
+{
+	return NULL;
+}
 static inline bool nouveau_acpi_video_backlight_use_native(void) { return true; }
 static inline void nouveau_acpi_video_register_backlight(void) {}
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 38032bb9582623..10cd606d01b930 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -323,7 +323,7 @@ get_fp_strap(struct drm_device *dev, struct nvbios *bios)
 	/*
 	 * The fp strap is normally dictated by the "User Strap" in
 	 * PEXTDEV_BOOT_0[20:16], but on BMP cards when bit 2 of the
-	 * Internal_Flags struct at 0x48 is set, the user strap gets overriden
+	 * Internal_Flags struct at 0x48 is set, the user strap gets overridden
 	 * by the PCI subsystem ID during POST, but not before the previous user
 	 * strap has been committed to CR58 for CR57=0xf on head A, which may be
 	 * read and used instead
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 0e8de6d4b36f7c..6dcb92575eb466 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -578,8 +578,9 @@ int nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
 				      "0x%08x vs 0x%08x\n", bo,
 				 bo->resource->mem_type, domain);
 			ret = -EBUSY;
+		} else {
+			ttm_bo_pin(&nvbo->bo);
 		}
-		ttm_bo_pin(&nvbo->bo);
 		goto out;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index b0b0ad9a0c248c..cc3caf8de64b02 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -397,7 +397,8 @@ nouveau_connector_destroy(struct drm_connector *connector)
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	nvif_event_dtor(&nv_connector->irq);
 	nvif_event_dtor(&nv_connector->hpd);
-	kfree(nv_connector->edid);
+	cancel_work_sync(&nv_connector->irq_work);
+	drm_edid_free(nv_connector->drm_edid);
 	drm_connector_unregister(connector);
 	drm_connector_cleanup(connector);
 	if (nv_connector->aux.transfer)
@@ -469,6 +470,36 @@ nouveau_connector_ddc_detect(struct drm_connector *connector)
 	return found;
 }
 
+static void
+nouveau_connector_set_edid(struct nouveau_connector *nv_connector,
+			   const struct drm_edid *drm_edid)
+{
+	if (nv_connector->drm_edid == drm_edid)
+		return;
+
+	/* Updates the EDID property and display_info with HF-EEODB-aware
+	 * sizing. The legacy helpers truncate both to what EDID byte 126
+	 * admits, hiding the DisplayID extension blocks that carry the
+	 * high-refresh timings.
+	 */
+	drm_edid_connector_update(&nv_connector->base, drm_edid);
+
+	drm_edid_free(nv_connector->drm_edid);
+	nv_connector->drm_edid = drm_edid;
+
+	/* The SPWG link-count byte lives in a vendor descriptor drm has no
+	 * accessor for. Peek at it once here so nothing else needs the raw
+	 * EDID.
+	 */
+	nv_connector->spwg_links = 0;
+	if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
+		const u8 *raw = (const u8 *)drm_edid_raw(drm_edid);
+
+		if (raw)
+			nv_connector->spwg_links = raw[121] == 2 ? 2 : 1;
+	}
+}
+
 static struct nouveau_encoder *
 nouveau_connector_of_detect(struct drm_connector *connector)
 {
@@ -490,8 +521,17 @@ nouveau_connector_of_detect(struct drm_connector *connector)
 		int idx = name ? name[strlen(name) - 1] - 'A' : 0;
 
 		if (nv_encoder->dcb->i2c_index == idx && edid) {
-			nv_connector->edid =
-				kmemdup(edid, EDID_LENGTH, GFP_KERNEL);
+			const struct drm_edid *drm_edid =
+				drm_edid_alloc(edid, EDID_LENGTH);
+
+			/* Firmware-provided, so validate it like the DDC
+			 * readers would.
+			 */
+			if (drm_edid && !drm_edid_valid(drm_edid)) {
+				drm_edid_free(drm_edid);
+				drm_edid = NULL;
+			}
+			nouveau_connector_set_edid(nv_connector, drm_edid);
 			return nv_encoder;
 		}
 	}
@@ -546,17 +586,23 @@ nouveau_connector_set_encoder(struct drm_connector *connector,
 	}
 }
 
-static void
-nouveau_connector_set_edid(struct nouveau_connector *nv_connector,
-			   struct edid *edid)
+struct nouveau_rm_edid {
+	u8 *data;
+	size_t size;
+};
+
+static int
+nouveau_connector_rm_edid_block(void *context, u8 *buf, unsigned int block,
+				size_t len)
 {
-	if (nv_connector->edid != edid) {
-		struct edid *old_edid = nv_connector->edid;
+	struct nouveau_rm_edid *rm = context;
+	size_t offset = (size_t)block * EDID_LENGTH;
 
-		drm_connector_update_edid_property(&nv_connector->base, edid);
-		kfree(old_edid);
-		nv_connector->edid = edid;
-	}
+	if (offset + len > rm->size)
+		return -EINVAL;
+
+	memcpy(buf, rm->data + offset, len);
+	return 0;
 }
 
 static enum drm_connector_status
@@ -590,22 +636,37 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 
 	nv_encoder = nouveau_connector_ddc_detect(connector);
 	if (nv_encoder) {
-		struct edid *new_edid = NULL;
+		const struct drm_edid *new_edid = NULL;
 
 		if (nv_encoder->i2c) {
 			if ((vga_switcheroo_handler_flags() & VGA_SWITCHEROO_CAN_SWITCH_DDC) &&
 			    nv_connector->type == DCB_CONNECTOR_LVDS)
-				new_edid = drm_get_edid_switcheroo(connector, nv_encoder->i2c);
+				new_edid = drm_edid_read_switcheroo(connector, nv_encoder->i2c);
 			else
-				new_edid = drm_get_edid(connector, nv_encoder->i2c);
+				new_edid = drm_edid_read_ddc(connector, nv_encoder->i2c);
 		} else {
-			ret = nvif_outp_edid_get(&nv_encoder->outp, (u8 **)&new_edid);
-			if (ret < 0)
-				return connector_status_disconnected;
+			struct nouveau_rm_edid rm = {};
+
+			/* RM (which owns the DDC pads on GSP boards) reads the
+			 * EDID whole and returns its true size, which for an
+			 * HF-EEODB EDID exceeds what byte 126 admits. Serve it
+			 * through drm's block reader so EEODB sizing, block
+			 * validation, and the debugfs EDID override all apply.
+			 * A failed read is treated like an empty DDC read,
+			 * which releases the runtime-PM reference.
+			 */
+			ret = nvif_outp_edid_get(&nv_encoder->outp, &rm.data);
+			if (ret >= 0) {
+				rm.size = ret;
+				new_edid = drm_edid_read_custom(connector,
+								nouveau_connector_rm_edid_block,
+								&rm);
+				kfree(rm.data);
+			}
 		}
 
 		nouveau_connector_set_edid(nv_connector, new_edid);
-		if (!nv_connector->edid) {
+		if (!nv_connector->drm_edid) {
 			NV_ERROR(drm, "DDC responded, but no EDID for %s\n",
 				 connector->name);
 			goto detect_analog;
@@ -626,7 +687,7 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 				    nv_partner->dcb->type == DCB_OUTPUT_TMDS) ||
 				   (nv_encoder->dcb->type == DCB_OUTPUT_TMDS &&
 				    nv_partner->dcb->type == DCB_OUTPUT_ANALOG))) {
-			if (nv_connector->edid->input & DRM_EDID_INPUT_DIGITAL)
+			if (drm_edid_is_digital(nv_connector->drm_edid))
 				type = DCB_OUTPUT_TMDS;
 			else
 				type = DCB_OUTPUT_ANALOG;
@@ -638,7 +699,8 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 		conn_status = connector_status_connected;
 
 		if (nv_encoder->dcb->type == DCB_OUTPUT_DP)
-			drm_dp_cec_set_edid(&nv_connector->aux, nv_connector->edid);
+			drm_dp_cec_attach(&nv_connector->aux,
+					  connector->display_info.source_physical_address);
 
 		goto out;
 	} else {
@@ -670,7 +732,7 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 	}
 
  out:
-	if (!nv_connector->edid)
+	if (!nv_connector->drm_edid)
 		drm_dp_cec_unset_edid(&nv_connector->aux);
 
 	pm_runtime_mark_last_busy(dev->dev);
@@ -686,7 +748,7 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder = NULL;
-	struct edid *edid = NULL;
+	const struct drm_edid *edid = NULL;
 	enum drm_connector_status status = connector_status_disconnected;
 
 	nv_encoder = find_encoder(connector, DCB_OUTPUT_LVDS);
@@ -697,7 +759,7 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 	if (!drm->vbios.fp_no_ddc) {
 		status = nouveau_connector_detect(connector, force);
 		if (status == connector_status_connected) {
-			edid = nv_connector->edid;
+			edid = nv_connector->drm_edid;
 			goto out;
 		}
 	}
@@ -733,11 +795,20 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 	 * stored for the panel stored in them.
 	 */
 	if (!drm->vbios.fp_no_ddc) {
-		edid = (struct edid *)nouveau_bios_embedded_edid(dev);
-		if (edid) {
-			edid = kmemdup(edid, EDID_LENGTH, GFP_KERNEL);
+		const void *embedded = nouveau_bios_embedded_edid(dev);
+
+		if (embedded) {
+			edid = drm_edid_alloc(embedded, EDID_LENGTH);
+			/* The panel is there either way, so report it
+			 * connected and let the probe helper fall back to a
+			 * default mode if the EDID itself is unusable.
+			 */
 			if (edid)
 				status = connector_status_connected;
+			if (edid && !drm_edid_valid(edid)) {
+				drm_edid_free(edid);
+				edid = NULL;
+			}
 		}
 	}
 
@@ -886,7 +957,7 @@ nouveau_connector_detect_depth(struct drm_connector *connector)
 	bool duallink;
 
 	/* if the edid is feeling nice enough to provide this info, use it */
-	if (nv_connector->edid && connector->display_info.bpc)
+	if (nv_connector->drm_edid && connector->display_info.bpc)
 		return;
 
 	/* EDID 1.4 is *supposed* to be supported on eDP, but, Apple... */
@@ -913,9 +984,8 @@ nouveau_connector_detect_depth(struct drm_connector *connector)
 	/* LVDS: DDC panel, need to first determine the number of links to
 	 * know which if_is_24bit flag to check...
 	 */
-	if (nv_connector->edid &&
-	    nv_connector->type == DCB_CONNECTOR_LVDS_SPWG)
-		duallink = ((u8 *)nv_connector->edid)[121] == 2;
+	if (nv_connector->spwg_links)
+		duallink = nv_connector->spwg_links == 2;
 	else
 		duallink = mode->clock >= bios->fp.duallink_transition_clk;
 
@@ -973,12 +1043,17 @@ nouveau_connector_get_modes(struct drm_connector *connector)
 		nv_connector->native_mode = NULL;
 	}
 
-	if (nv_connector->edid)
-		ret = drm_add_edid_modes(connector, nv_connector->edid);
-	else
-	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS &&
-	    (nv_encoder->dcb->lvdsconf.use_straps_for_mode ||
-	     drm->vbios.fp_no_ddc) && nouveau_bios_fp_mode(dev, NULL)) {
+	if (nv_connector->drm_edid) {
+		/* The probe helper clears the property and display_info for
+		 * a forced-off connector without calling detect(). Re-sync
+		 * from our copy then, since add_modes() reads the property.
+		 */
+		if (!connector->edid_blob_ptr)
+			drm_edid_connector_update(connector, nv_connector->drm_edid);
+		ret = drm_edid_connector_add_modes(connector);
+	} else if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS &&
+		   (nv_encoder->dcb->lvdsconf.use_straps_for_mode ||
+		    drm->vbios.fp_no_ddc) && nouveau_bios_fp_mode(dev, NULL)) {
 		struct drm_display_mode mode;
 
 		nouveau_bios_fp_mode(dev, &mode);
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index 0608cabed058e3..eb292d2ba4bcde 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -43,7 +43,7 @@
 
 struct nvkm_i2c_port;
 struct dcb_output;
-struct edid;
+struct drm_edid;
 
 #ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
 struct nouveau_backlight {
@@ -121,6 +121,13 @@ struct nouveau_connector {
 	struct drm_connector base;
 	enum dcb_connector_type type;
 	u8 index;
+	/* LVDS_SPWG panels state their link count in EDID descriptor 4 (SPWG
+	 * byte 0x79), cached by nouveau_connector_set_edid() so nothing else
+	 * needs the raw EDID. 0 = unknown (not an SPWG panel, or no EDID) and
+	 * callers fall back to their transition-clock/VBIOS heuristics;
+	 * 1 = single link; 2 = dual link.
+	 */
+	u8 spwg_links;
 
 	struct nvif_conn conn;
 	u64 hpd_pending;
@@ -137,7 +144,8 @@ struct nouveau_connector {
 	int scaling_mode;
 
 	struct nouveau_encoder *detected_encoder;
-	struct edid *edid;
+	/* Owner of the sink's EDID, HF-EEODB-complete. */
+	const struct drm_edid *drm_edid;
 	struct drm_display_mode *native_mode;
 #ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
 	struct nouveau_backlight *backlight;
diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c
index ad4570c50be70c..4402f1c8cee287 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dmem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c
@@ -40,6 +40,7 @@
 #include <linux/hmm.h>
 #include <linux/memremap.h>
 #include <linux/migrate.h>
+#include <linux/pagemap.h>
 
 /*
  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
@@ -339,8 +340,8 @@ nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage,
 	chunk->pagemap.ops = &nouveau_dmem_pagemap_ops;
 	chunk->pagemap.owner = drm->dev;
 
-	ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM, DMEM_CHUNK_SIZE,
-				 &chunk->bo);
+	ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM,
+				 DMEM_CHUNK_SIZE * NR_CHUNKS, &chunk->bo);
 	if (ret)
 		goto out_release;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c
index 55691ec44abaa5..019af307c669aa 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dp.c
@@ -486,7 +486,7 @@ nouveau_dp_irq(struct work_struct *work)
 		container_of(work, typeof(*nv_connector), irq_work);
 	struct drm_connector *connector = &nv_connector->base;
 	struct nouveau_encoder *outp = find_encoder(connector, DCB_OUTPUT_DP);
-	struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev);
+	struct nouveau_drm *drm;
 	struct nv50_mstm *mstm;
 	u64 hpd = 0;
 	int ret;
@@ -494,6 +494,8 @@ nouveau_dp_irq(struct work_struct *work)
 	if (!outp)
 		return;
 
+	drm = nouveau_drm(outp->base.base.dev);
+
 	mstm = outp->dp.mstm;
 	NV_DEBUG(drm, "service %s\n", connector->name);
 
@@ -531,7 +533,7 @@ nv50_dp_mode_valid(struct nouveau_encoder *outp,
 	const unsigned int min_clock = 25000;
 	unsigned int max_rate, mode_rate, ds_max_dotclock, clock = mode->clock;
 	/* Check with the minmum bpc always, so we can advertise better modes.
-	 * In particlar not doing this causes modes to be dropped on HDR
+	 * In particular not doing this causes modes to be dropped on HDR
 	 * displays as we might check with a bpc of 16 even.
 	 */
 	const u8 bpp = 6 * 3;
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 4d1ad718e09b75..5e1336615cf6dc 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -103,17 +103,17 @@ MODULE_PARM_DESC(modeset, "enable driver (default: auto, "
 int nouveau_modeset = -1;
 module_param_named(modeset, nouveau_modeset, int, 0400);
 
-MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)");
-static int nouveau_atomic = 0;
+MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: auto, "
+			 "0 = disabled, 1 = enabled)");
+static int nouveau_atomic = -1;
 module_param_named(atomic, nouveau_atomic, int, 0400);
 
 MODULE_PARM_DESC(runpm, "disable (0), force enable (1), optimus only default (-1)");
 static int nouveau_runtime_pm = -1;
 module_param_named(runpm, nouveau_runtime_pm, int, 0400);
 
-static struct drm_driver driver_stub;
-static struct drm_driver driver_pci;
-static struct drm_driver driver_platform;
+static const struct drm_driver driver_legacy_kms;
+static const struct drm_driver driver_atomic_kms;
 
 #ifdef CONFIG_DEBUG_FS
 struct dentry *nouveau_debugfs_root;
@@ -585,6 +585,7 @@ nouveau_drm_device_fini(struct nouveau_drm *drm)
 	if (nouveau_pmops_runtime()) {
 		pm_runtime_get_sync(dev->dev);
 		pm_runtime_forbid(dev->dev);
+		pm_runtime_dont_use_autosuspend(dev->dev);
 	}
 
 	nouveau_led_fini(dev);
@@ -727,8 +728,7 @@ nouveau_drm_device_del(struct nouveau_drm *drm)
 }
 
 static struct nouveau_drm *
-nouveau_drm_device_new(const struct drm_driver *drm_driver, struct device *parent,
-		       struct nvkm_device *device)
+nouveau_drm_device_new(struct device *parent, struct nvkm_device *device)
 {
 	static const struct nvif_mclass
 	mmus[] = {
@@ -737,19 +737,32 @@ nouveau_drm_device_new(const struct drm_driver *drm_driver, struct device *paren
 		{ NVIF_CLASS_MMU_NV04 , -1 },
 		{}
 	};
+	const struct drm_driver *driver;
 	struct nouveau_drm *drm;
 	int ret;
 
+	if (device->card_type >= NV_50) {
+		if (nouveau_atomic)
+			driver = &driver_atomic_kms;
+		else
+			driver = &driver_legacy_kms;
+	} else {
+		if (nouveau_atomic == 1)
+			dev_warn(parent, "Atomic modesetting not supported (needs nv50+)\n");
+
+		driver = &driver_legacy_kms;
+	}
+
 	drm = kzalloc_obj(*drm);
 	if (!drm)
 		return ERR_PTR(-ENOMEM);
 
 	drm->nvkm = device;
-
-	drm->dev = drm_dev_alloc(drm_driver, parent);
+	drm->dev = drm_dev_alloc(driver, parent);
 	if (IS_ERR(drm->dev)) {
 		ret = PTR_ERR(drm->dev);
-		goto done;
+		kfree(drm);
+		return ERR_PTR(ret);
 	}
 
 	drm->dev->dev_private = drm;
@@ -873,16 +886,13 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
 		return ret;
 
 	/* Remove conflicting drivers (vesafb, efifb etc). */
-	ret = aperture_remove_conflicting_pci_devices(pdev, driver_pci.name);
+	ret = aperture_remove_conflicting_pci_devices(pdev, DRIVER_NAME);
 	if (ret)
 		goto fail_nvkm;
 
 	pci_set_master(pdev);
 
-	if (nouveau_atomic)
-		driver_pci.driver_features |= DRIVER_ATOMIC;
-
-	drm = nouveau_drm_device_new(&driver_pci, &pdev->dev, device);
+	drm = nouveau_drm_device_new(&pdev->dev, device);
 	if (IS_ERR(drm)) {
 		ret = PTR_ERR(drm);
 		goto fail_nvkm;
@@ -1148,6 +1158,7 @@ nouveau_pmops_runtime_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct nouveau_drm *drm = pci_get_drvdata(pdev);
+	struct nvif_device *nvif = &drm->client.device;
 	int ret;
 
 	if (!nouveau_pmops_runtime()) {
@@ -1155,6 +1166,18 @@ nouveau_pmops_runtime_suspend(struct device *dev)
 		return -EBUSY;
 	}
 
+	// Check if the GPU itself is ready for runtime suspend, otherwise mark as busy and check
+	// again in a bit.
+	ret = nvif_device_gcx_ready(nvif);
+	if (ret < 0) {
+		NV_ERROR(drm, "Failed to query GCX readiness (returned %d)\n", ret);
+		return -EBUSY;
+	} else if (!(ret & NV_DEVICE_GCOFF_READY)) {
+		NV_DEBUG(drm, "GPU isn't ready for suspend yet, delaying...\n");
+		pm_runtime_mark_last_busy(dev);
+		return -EBUSY;
+	}
+
 	nouveau_switcheroo_optimus_dsm();
 	ret = nouveau_do_suspend(drm, true);
 	pci_save_state(pdev);
@@ -1250,10 +1273,8 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
 	mutex_unlock(&drm->clients_lock);
 
 done:
-	if (ret && cli) {
-		nouveau_cli_fini(cli);
+	if (ret && cli)
 		kfree(cli);
-	}
 
 	pm_runtime_mark_last_busy(dev->dev);
 	pm_runtime_put_autosuspend(dev->dev);
@@ -1359,35 +1380,54 @@ nouveau_driver_fops = {
 	.fop_flags = FOP_UNSIGNED_OFFSET,
 };
 
-static struct drm_driver
-driver_stub = {
-	.driver_features = DRIVER_GEM |
-			   DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE |
-			   DRIVER_MODESET |
-			   DRIVER_RENDER,
-	.open = nouveau_drm_open,
-	.postclose = nouveau_drm_postclose,
-
-#if defined(CONFIG_DEBUG_FS)
-	.debugfs_init = nouveau_drm_debugfs_init,
+#ifdef CONFIG_DEBUG_FS
+#define NOUVEAU_DEBUGFS_OPS .debugfs_init = nouveau_drm_debugfs_init,
+#else
+#define NOUVEAU_DEBUGFS_OPS
 #endif
 
-	.ioctls = nouveau_ioctls,
-	.num_ioctls = ARRAY_SIZE(nouveau_ioctls),
-	.fops = &nouveau_driver_fops,
-
-	.gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table,
-
-	.dumb_create = nouveau_display_dumb_create,
-	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
-
-	DRM_FBDEV_TTM_DRIVER_OPS,
+#define NOUVEAU_DRIVER_OPS                                              \
+	.open = nouveau_drm_open,                                       \
+	.postclose = nouveau_drm_postclose,                             \
+                                                                        \
+	NOUVEAU_DEBUGFS_OPS						\
+                                                                        \
+	.ioctls = nouveau_ioctls,                                       \
+	.num_ioctls = ARRAY_SIZE(nouveau_ioctls),                       \
+	.fops = &nouveau_driver_fops,                                   \
+                                                                        \
+	.gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table, \
+                                                                        \
+	.dumb_create = nouveau_display_dumb_create,                     \
+	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,                 \
+                                                                        \
+	DRM_FBDEV_TTM_DRIVER_OPS,                                       \
+                                                                        \
+	.name = DRIVER_NAME,                                            \
+	.desc = DRIVER_DESC,                                            \
+	.major = DRIVER_MAJOR,                                          \
+	.minor = DRIVER_MINOR,                                          \
+	.patchlevel = DRIVER_PATCHLEVEL
+
+static const struct drm_driver
+driver_legacy_kms = {
+	.driver_features = DRIVER_GEM
+			 | DRIVER_SYNCOBJ
+			 | DRIVER_SYNCOBJ_TIMELINE
+			 | DRIVER_MODESET
+			 | DRIVER_RENDER,
+	NOUVEAU_DRIVER_OPS,
+};
 
-	.name = DRIVER_NAME,
-	.desc = DRIVER_DESC,
-	.major = DRIVER_MAJOR,
-	.minor = DRIVER_MINOR,
-	.patchlevel = DRIVER_PATCHLEVEL,
+static const struct drm_driver
+driver_atomic_kms = {
+	.driver_features = DRIVER_GEM
+			 | DRIVER_SYNCOBJ
+			 | DRIVER_SYNCOBJ_TIMELINE
+			 | DRIVER_MODESET
+			 | DRIVER_RENDER
+			 | DRIVER_ATOMIC,
+	NOUVEAU_DRIVER_OPS,
 };
 
 static struct pci_device_id
@@ -1416,6 +1456,7 @@ static void nouveau_display_options(void)
 	DRM_DEBUG_DRIVER("... debug        : %s\n", nouveau_debug);
 	DRM_DEBUG_DRIVER("... noaccel      : %d\n", nouveau_noaccel);
 	DRM_DEBUG_DRIVER("... modeset      : %d\n", nouveau_modeset);
+	DRM_DEBUG_DRIVER("... atomic       : %d\n", nouveau_atomic);
 	DRM_DEBUG_DRIVER("... runpm        : %d\n", nouveau_runtime_pm);
 	DRM_DEBUG_DRIVER("... vram_pushbuf : %d\n", nouveau_vram_pushbuf);
 	DRM_DEBUG_DRIVER("... hdmimhz      : %d\n", nouveau_hdmimhz);
@@ -1455,7 +1496,7 @@ nouveau_platform_device_create(const struct nvkm_device_tegra_func *func,
 	if (err)
 		goto err_free;
 
-	drm = nouveau_drm_device_new(&driver_platform, &pdev->dev, *pdevice);
+	drm = nouveau_drm_device_new(&pdev->dev, *pdevice);
 	if (IS_ERR(drm)) {
 		err = PTR_ERR(drm);
 		goto err_free;
@@ -1480,9 +1521,6 @@ nouveau_drm_init(void)
 {
 	int ret;
 
-	driver_pci = driver_stub;
-	driver_platform = driver_stub;
-
 	nouveau_display_options();
 
 	if (nouveau_modeset == -1) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_exec.c b/drivers/gpu/drm/nouveau/nouveau_exec.c
index a08ab1cfea9be8..eea6619cffaa5a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_exec.c
+++ b/drivers/gpu/drm/nouveau/nouveau_exec.c
@@ -23,7 +23,7 @@
  * should be managed by the kernel and which by the UMD.
  *
  * The DRM_NOUVEAU_VM_BIND ioctl provides clients an interface to manage the
- * userspace-managable portion of the VA space. It provides operations to map
+ * userspace-manageable portion of the VA space. It provides operations to map
  * and unmap memory. Mappings may be flagged as sparse. Sparse mappings are not
  * backed by a GEM object and the kernel will ignore GEM handles provided
  * alongside a sparse mapping.
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index edbe9e08ba0fd7..11e95c37ce5069 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -96,7 +96,7 @@ nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error)
 void
 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
 {
-	cancel_work_sync(&fctx->uevent_work);
+	disable_work_sync(&fctx->uevent_work);
 	nouveau_fence_context_kill(fctx, 0);
 	nvif_event_dtor(&fctx->event);
 	fctx->dead = 1;
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 0b7123b163e491..51188be57221ff 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -522,6 +522,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
 			if (unlikely(ret)) {
 				if (ret != -ERESTARTSYS)
 					NV_PRINTK(err, cli, "fail reserve\n");
+				drm_gem_object_put(gem);
 				break;
 			}
 		}
@@ -531,6 +532,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
 			struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
 			if (!vma) {
 				NV_PRINTK(err, cli, "vma not found!\n");
+				drm_gem_object_put(gem);
 				ret = -EINVAL;
 				break;
 			}
diff --git a/drivers/gpu/drm/nouveau/nouveau_led.c b/drivers/gpu/drm/nouveau/nouveau_led.c
index 19e2110904d195..67088a71f6d1c5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_led.c
+++ b/drivers/gpu/drm/nouveau/nouveau_led.c
@@ -65,7 +65,7 @@ nouveau_led_set_brightness(struct led_classdev *led, enum led_brightness value)
 	duty = value * div / LED_FULL;
 
 	/* for now, this is safe to directly poke those registers because:
-	 *  - A: nvidia never puts the logo led to any other PWM controler
+	 *  - A: nvidia never puts the logo led to any other PWM controller
 	 *       than PDISPLAY.SOR[1].PWM.
 	 *  - B: nouveau does not touch these registers anywhere else
 	 */
diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.c b/drivers/gpu/drm/nouveau/nouveau_sched.c
index 8b9f935afe094f..b3f02c490ecb67 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sched.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sched.c
@@ -517,7 +517,7 @@ nouveau_sched_destroy(struct nouveau_sched **psched)
 	struct nouveau_sched *sched = *psched;
 
 	nouveau_sched_fini(sched);
-	kfree(sched);
+	kfree_rcu(sched, rcu);
 
 	*psched = NULL;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.h b/drivers/gpu/drm/nouveau/nouveau_sched.h
index 20cd1da8db73c3..51ce8dcf62858b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sched.h
+++ b/drivers/gpu/drm/nouveau/nouveau_sched.h
@@ -98,6 +98,7 @@ void nouveau_job_free(struct nouveau_job *job);
 
 struct nouveau_sched {
 	struct drm_gpu_scheduler base;
+	struct rcu_head rcu;
 	struct drm_sched_entity entity;
 	struct workqueue_struct *wq;
 	struct mutex mutex;
diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c
index 58735446d7832c..c519c28164fa0c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_svm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_svm.c
@@ -304,6 +304,19 @@ nouveau_svmm_fini(struct nouveau_svmm **psvmm)
 {
 	struct nouveau_svmm *svmm = *psvmm;
 	if (svmm) {
+		struct nouveau_svm *svm = svmm->vmm->cli->drm->svm;
+
+		/* The fault handler caches svmm pointers under svm->mutex and
+		 * dereferences them after dropping it, across blocking faults.
+		 * The instance is already unlinked (nouveau_svmm_part), so drain
+		 * the handler before the free to release any in-flight reference.
+		 *
+		 * On device teardown nouveau_svm_fini() ran first and freed
+		 * drm->svm (NULL) after draining the buffer; nothing to flush.
+		 */
+		if (svm)
+			flush_work(&svm->buffer[0].work);
+
 		mutex_lock(&svmm->mutex);
 		svmm->vmm = NULL;
 		mutex_unlock(&svmm->mutex);
diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
index fc125fd44a9b7e..2026fe6b48c694 100644
--- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
@@ -846,6 +846,9 @@ op_map(struct nouveau_uvma *uvma)
 {
 	struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj);
 
+	if (drm_gpuva_invalidated(&uvma->va))
+		return;
+
 	nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource));
 }
 
@@ -1232,6 +1235,7 @@ bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec,
 
 		drm_gpuva_for_each_op(va_op, op->ops) {
 			struct drm_gem_object *obj = op_gem_obj(va_op);
+			struct nouveau_bo *nvbo;
 
 			if (unlikely(!obj))
 				continue;
@@ -1246,8 +1250,13 @@ bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec,
 			if (va_op->op == DRM_GPUVA_OP_UNMAP)
 				continue;
 
-			ret = nouveau_bo_validate(nouveau_gem_object(obj),
-						  true, false);
+			nvbo = nouveau_gem_object(obj);
+			if (!(nvbo->valid_domains &
+			      (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART)))
+				return -EINVAL;
+
+			nouveau_bo_placement_set(nvbo, nvbo->valid_domains, 0);
+			ret = nouveau_bo_validate(nvbo, true, false);
 			if (ret)
 				return ret;
 		}
diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.c b/drivers/gpu/drm/nouveau/nouveau_vga.c
index a6c375a241542c..29a801124e560a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vga.c
+++ b/drivers/gpu/drm/nouveau/nouveau_vga.c
@@ -54,15 +54,6 @@ nouveau_switcheroo_set_state(struct pci_dev *pdev,
 	}
 }
 
-static void
-nouveau_switcheroo_reprobe(struct pci_dev *pdev)
-{
-	struct nouveau_drm *drm = pci_get_drvdata(pdev);
-	struct drm_device *dev = drm->dev;
-
-	drm_client_dev_hotplug(dev);
-}
-
 static bool
 nouveau_switcheroo_can_switch(struct pci_dev *pdev)
 {
@@ -76,11 +67,28 @@ nouveau_switcheroo_can_switch(struct pci_dev *pdev)
 	return atomic_read(&drm->dev->open_count) == 0;
 }
 
+static void
+nouveau_switcheroo_pre_switch(struct pci_dev *pdev)
+{
+	struct nouveau_drm *drm = pci_get_drvdata(pdev);
+
+	drm_client_dev_acquire_outputs(drm->dev);
+}
+
+static void
+nouveau_switcheroo_post_switch(struct pci_dev *pdev)
+{
+	struct nouveau_drm *drm = pci_get_drvdata(pdev);
+
+	drm_client_dev_hotplug(drm->dev);
+}
+
 static const struct vga_switcheroo_client_ops
 nouveau_switcheroo_ops = {
 	.set_gpu_state = nouveau_switcheroo_set_state,
-	.reprobe = nouveau_switcheroo_reprobe,
 	.can_switch = nouveau_switcheroo_can_switch,
+	.pre_switch = nouveau_switcheroo_pre_switch,
+	.post_switch = nouveau_switcheroo_post_switch,
 };
 
 void
diff --git a/drivers/gpu/drm/nouveau/nvif/device.c b/drivers/gpu/drm/nouveau/nvif/device.c
index 24880931039f64..1be9fbe6cb7036 100644
--- a/drivers/gpu/drm/nouveau/nvif/device.c
+++ b/drivers/gpu/drm/nouveau/nvif/device.c
@@ -38,6 +38,19 @@ nvif_device_time(struct nvif_device *device)
 	return device->user.func->time(&device->user);
 }
 
+int
+nvif_device_gcx_ready(struct nvif_device *device)
+{
+	struct nv_device_gcx_ready_v0 args = {};
+	int ret;
+
+	ret = nvif_object_mthd(&device->object, NV_DEVICE_V0_GCX_READY, &args, sizeof(args));
+	if (ret)
+		return ret;
+
+	return args.ready;
+}
+
 int
 nvif_device_map(struct nvif_device *device)
 {
diff --git a/drivers/gpu/drm/nouveau/nvif/vmm.c b/drivers/gpu/drm/nouveau/nvif/vmm.c
index 65c3e883b11934..579af70766f2db 100644
--- a/drivers/gpu/drm/nouveau/nvif/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvif/vmm.c
@@ -192,6 +192,7 @@ void
 nvif_vmm_dtor(struct nvif_vmm *vmm)
 {
 	kfree(vmm->page);
+	vmm->page = NULL;
 	nvif_object_dtor(&vmm->object);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 96c8a5b299999c..a2ed8c4fe52a0b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -1237,7 +1237,7 @@ nvac_chipset = {
 	.mc       = { 0x00000001, g98_mc_new },
 	.mmu      = { 0x00000001, mcp77_mmu_new },
 	.mxm      = { 0x00000001, nv50_mxm_new },
-	.pci      = { 0x00000001, g94_pci_new },
+	.pci      = { 0x00000001, mcp79_pci_new },
 	.therm    = { 0x00000001, g84_therm_new },
 	.timer    = { 0x00000001, nv41_timer_new },
 	.volt     = { 0x00000001, nv40_volt_new },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c
index f2e9a06263ce33..28702741a88b13 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c
@@ -74,6 +74,7 @@ nvkm_control_mthd_pstate_attr(struct nvkm_control *ctrl, void *data, u32 size)
 	const struct nvkm_domain *domain;
 	struct nvkm_pstate *pstate;
 	struct nvkm_cstate *cstate;
+	bool found = false;
 	int i = 0, j = -1;
 	u32 lo, hi;
 	int ret = -ENOSYS;
@@ -104,10 +105,15 @@ nvkm_control_mthd_pstate_attr(struct nvkm_control *ctrl, void *data, u32 size)
 
 	if (args->v0.state != NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT) {
 		list_for_each_entry(pstate, &clk->states, head) {
-			if (i++ == args->v0.state)
+			if (i++ == args->v0.state) {
+				found = true;
 				break;
+			}
 		}
 
+		if (!found)
+			return -EINVAL;
+
 		lo = pstate->base.domain[domain->name];
 		hi = lo;
 		list_for_each_entry(cstate, &pstate->list, head) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
index 23d11d8221cb6f..f78e6b9b429231 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
@@ -27,6 +27,7 @@
 
 #include <core/client.h>
 #include <subdev/fb.h>
+#include <subdev/gsp.h>
 #include <subdev/instmem.h>
 #include <subdev/timer.h>
 
@@ -189,6 +190,38 @@ nvkm_udevice_time(struct nvkm_udevice *udev, void *data, u32 size)
 	return ret;
 }
 
+static int
+nvkm_udevice_gcx_ready(struct nvkm_udevice *udev, void *data, u32 size)
+{
+	struct nvkm_object *object = &udev->object;
+	struct nvkm_device *device = udev->device;
+	struct nvkm_gsp *gsp = device->gsp;
+	union {
+		struct nv_device_gcx_ready_v0 v0;
+	} *args = data;
+	int ret = -ENOSYS;
+
+	if (!gsp) {
+		args->v0.ready = NV_DEVICE_GC6_READY | NV_DEVICE_GCOFF_READY;
+		return 0;
+	}
+
+	nvif_ioctl(object, "device gcx ready size %d\n", size);
+	ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false);
+	if (!ret) {
+		nvif_ioctl(object, "device gcx ready vers %d\n", args->v0.version);
+
+		ret = nvkm_gsp_gcx_ready(gsp);
+		if (ret < 0)
+			return ret;
+
+		args->v0.ready = ret;
+		ret = 0;
+	}
+
+	return ret;
+}
+
 static int
 nvkm_udevice_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 {
@@ -199,6 +232,8 @@ nvkm_udevice_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)
 		return nvkm_udevice_info(udev, data, size);
 	case NV_DEVICE_V0_TIME:
 		return nvkm_udevice_time(udev, data, size);
+	case NV_DEVICE_V0_GCX_READY:
+		return nvkm_udevice_gcx_ready(udev, data, size);
 	default:
 		break;
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c
index 377d0e0cef8481..9887b3898505b0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c
@@ -253,8 +253,7 @@ nvkm_uoutp_mthd_hdmi(struct nvkm_outp *outp, void *argv, u32 argc)
 
 	if (!ior->func->hdmi ||
 	    args->v0.max_ac_packet > 0x1f ||
-	    args->v0.rekey > 0x7f ||
-	    (args->v0.scdc && !ior->func->hdmi->scdc))
+	    args->v0.rekey > 0x7f)
 		return -EINVAL;
 
 	if (!args->v0.enable) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c
index b54f044c4483b1..3f888fb9791089 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/init.c
@@ -403,7 +403,7 @@ init_table_(struct nvbios_init *init, u16 offset, const char *name)
 #define init_io_condition_table(b) init_table_((b), 0x08, "io condition table")
 #define init_io_flag_condition_table(b) init_table_((b), 0x0a, "io flag condition table")
 #define init_function_table(b) init_table_((b), 0x0c, "function table")
-#define init_xlat_table(b) init_table_((b), 0x10, "xlat table");
+#define init_xlat_table(b) init_table_((b), 0x10, "xlat table")
 
 static u16
 init_script(struct nvkm_bios *bios, int index)
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
index 572e638463159f..1cb83edc78dc4a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c
@@ -199,16 +199,18 @@ nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei)
 	}
 
 	if (volt) {
-		ret = nvkm_volt_set_id(volt, cstate->voltage,
-				       pstate->base.voltage, clk->temp, -1);
-		if (ret && ret != -ENODEV)
-			nvkm_error(subdev, "failed to lower voltage: %d\n", ret);
+		int err = nvkm_volt_set_id(volt, cstate->voltage,
+					   pstate->base.voltage, clk->temp, -1);
+
+		if (err && err != -ENODEV)
+			nvkm_error(subdev, "failed to lower voltage: %d\n", err);
 	}
 
 	if (therm) {
-		ret = nvkm_therm_cstate(therm, pstate->fanspeed, -1);
-		if (ret && ret != -ENODEV)
-			nvkm_error(subdev, "failed to lower fan speed: %d\n", ret);
+		int err = nvkm_therm_cstate(therm, pstate->fanspeed, -1);
+
+		if (err && err != -ENODEV)
+			nvkm_error(subdev, "failed to lower fan speed: %d\n", err);
 	}
 
 	return ret;
@@ -270,13 +272,19 @@ nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei)
 	struct nvkm_fb *fb = subdev->device->fb;
 	struct nvkm_pci *pci = subdev->device->pci;
 	struct nvkm_pstate *pstate;
+	bool found = false;
 	int ret, idx = 0;
 
 	list_for_each_entry(pstate, &clk->states, head) {
-		if (idx++ == pstatei)
+		if (idx++ == pstatei) {
+			found = true;
 			break;
+		}
 	}
 
+	if (!found)
+		return -EINVAL;
+
 	nvkm_debug(subdev, "setting performance state %d\n", pstatei);
 	clk->pstate = pstatei;
 
@@ -473,6 +481,7 @@ static int
 nvkm_clk_ustate_update(struct nvkm_clk *clk, int req)
 {
 	struct nvkm_pstate *pstate;
+	bool found = false;
 	int i = 0;
 
 	if (!clk->allow_reclock)
@@ -480,12 +489,14 @@ nvkm_clk_ustate_update(struct nvkm_clk *clk, int req)
 
 	if (req != -1 && req != -2) {
 		list_for_each_entry(pstate, &clk->states, head) {
-			if (pstate->pstate == req)
+			if (pstate->pstate == req) {
+				found = true;
 				break;
+			}
 			i++;
 		}
 
-		if (pstate->pstate != req)
+		if (!found)
 			return -EINVAL;
 		req = i;
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h
index ea5b0bab4ccec6..7a93f9f4071586 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h
@@ -27,7 +27,7 @@
 #define KHZ (1000)
 #define MHZ (KHZ * 1000)
 
-#define MASK(w)	((1 << (w)) - 1)
+#define MASK(w)	((1U << (w)) - 1)
 
 #define GK20A_CLK_GPC_MDIV 1000
 
@@ -79,7 +79,7 @@
 #define GPC2CLK_OUT			(SYS_GPCPLL_CFG_BASE + 0x250)
 #define GPC2CLK_OUT_SDIV14_INDIV4_WIDTH	1
 #define GPC2CLK_OUT_SDIV14_INDIV4_SHIFT	31
-#define GPC2CLK_OUT_SDIV14_INDIV4_MODE	1
+#define GPC2CLK_OUT_SDIV14_INDIV4_MODE	1U
 #define GPC2CLK_OUT_VCODIV_WIDTH	6
 #define GPC2CLK_OUT_VCODIV_SHIFT	8
 #define GPC2CLK_OUT_VCODIV1		0
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
index 18241c6ba5fa36..4d52a158f3205b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
@@ -51,6 +51,8 @@ nv1a_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
 		mib = ((mem >> 4) & 127) + 1;
 	}
 
+	pci_dev_put(bridge);
+
 	return nvkm_ram_new_(&nv04_ram_func, fb, NVKM_RAM_TYPE_STOLEN,
 			     mib * 1024 * 1024, pram);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c
index 9ba1316831e7e9..e475d0e8fa7bf9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c
@@ -20,6 +20,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "priv.h"
+#include <nvif/cl0080.h>
 
 int
 nvkm_gsp_intr_nonstall(struct nvkm_gsp *gsp, enum nvkm_subdev_type type, int inst)
@@ -47,6 +48,15 @@ nvkm_gsp_intr_stall(struct nvkm_gsp *gsp, enum nvkm_subdev_type type, int inst)
 	return -ENOENT;
 }
 
+int
+nvkm_gsp_gcx_ready(struct nvkm_gsp *gsp)
+{
+	if (!gsp->rm->api->gsp->gcx_ready)
+		return NV_DEVICE_GC6_READY | NV_DEVICE_GCOFF_READY;
+
+	return gsp->rm->api->gsp->gcx_ready(gsp);
+}
+
 static int
 nvkm_gsp_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h
index 71b7203bef5073..b07797813b0490 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h
@@ -93,6 +93,7 @@ void r535_gsp_dtor(struct nvkm_gsp *);
 int r535_gsp_oneinit(struct nvkm_gsp *);
 int r535_gsp_init(struct nvkm_gsp *);
 int r535_gsp_fini(struct nvkm_gsp *, enum nvkm_suspend_state suspend);
+int r535_gsp_gcx_ready(struct nvkm_gsp *gsp);
 
 int nvkm_gsp_new_(const struct nvkm_gsp_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int,
 		  struct nvkm_gsp **);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c
index 94925f1590ea48..63aa30f9474715 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c
@@ -1782,6 +1782,23 @@ r535_gsp_fini(struct nvkm_gsp *gsp, enum nvkm_suspend_state suspend)
 	return 0;
 }
 
+int
+r535_gsp_get_static_memsys_info(struct nvkm_gsp *gsp)
+{
+	NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS *ctrl;
+
+	ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
+				    NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_STATIC_CONFIG,
+				    sizeof(*ctrl));
+	if (IS_ERR(ctrl))
+		return PTR_ERR(ctrl);
+
+	gsp->memsys.use_raw_mode_comptagline_alloc = ctrl->bUseRawModeComptaglineAllocation;
+
+	nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
+	return 0;
+}
+
 int
 r535_gsp_init(struct nvkm_gsp *gsp)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h
index b6683a5bf870ce..7b10b7548c575a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h
@@ -782,6 +782,51 @@ typedef struct NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS {
 
 #define GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE      ((48 << 10) * 2048)   // Support 2048 channels
 
+typedef struct NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS {
+    /*! Determines if RM should use 1 to 1 Comptagline allocation policy */
+    NvBool bOneToOneComptagLineAllocation;
+
+    /*! Determines if RM should use 1 to 4 Comptagline allocation policy */
+    NvBool bUseOneToFourComptagLineAllocation;
+
+    /*! Determines if RM should use raw Comptagline allocation policy */
+    NvBool bUseRawModeComptaglineAllocation;
+
+    /*! Has COMPBIT_BACKING_SIZE been overridden to zero (i.e. disabled)? */
+    NvBool bDisableCompbitBacking;
+
+    /*! Determine if we need to disable post L2 compression */
+    NvBool bDisablePostL2Compression;
+
+    /*! Is ECC DRAM feature supported? */
+    NvBool bEnabledEccFBPA;
+
+    NvBool bL2PreFill;
+
+    /*! L2 cache size */
+    NV_DECLARE_ALIGNED(NvU64 l2CacheSize, 8);
+
+    /*! Indicate whether fpba is present or not */
+    NvBool bFbpaPresent;
+
+    /*! Size covered by one comptag */
+    NvU32  comprPageSize;
+
+    /*! log32(comprPageSize) */
+    NvU32  comprPageShift;
+
+    /*! RAM type */
+    NvU32  ramType;
+
+    /*! LTC count */
+    NvU32  ltcCount;
+
+    /*! LTS per LTC count */
+    NvU32  ltsPerLtcCount;
+} NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS;
+
+#define NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_STATIC_CONFIG        (0x20800a1c) /* finn: Evaluated from "(FINN_NV20_SUBDEVICE_0_INTERNAL_INTERFACE_ID << 8) | NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS_MESSAGE_ID" */
+
 typedef union rpc_message_rpc_union_field_v03_00
 {
     NvU32      spare;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c
index af5aa5065c3ddf..469e7eed1d6f8e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c
@@ -26,6 +26,52 @@ r570_fbsr_suspend_channels(struct nvkm_gsp *gsp, bool suspend)
 	return nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl);
 }
 
+static int
+r570_fb_get_compbit_store_size(struct nvkm_gsp *gsp, u64 *size)
+{
+	NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS *ctrl;
+
+	ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.object,
+				   NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO,
+				   sizeof(*ctrl));
+	if (IS_ERR(ctrl))
+		return PTR_ERR(ctrl);
+
+	*size = ctrl->Size;
+
+	nvkm_gsp_rm_ctrl_done(&gsp->internal.device.object, ctrl);
+	return 0;
+}
+
+static int
+r570_memsys_enable_raw_comp_mode(struct nvkm_gsp *gsp, bool enable)
+{
+	NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS *ctrl;
+	int ret;
+
+	ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice,
+				    NV2080_CTRL_CMD_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE,
+				    sizeof(*ctrl));
+	if (IS_ERR(ctrl))
+		return PTR_ERR(ctrl);
+
+	ctrl->bRawMode = enable;
+
+	ret = nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl);
+	if (!ret)
+		nvkm_debug(&gsp->subdev, "memsys: Raw compression mode %s\n",
+			   str_enabled_disabled(enable));
+
+	return ret;
+}
+
+static bool
+r570_need_raw_comp_war(struct nvkm_gsp *gsp, struct nvkm_device *device)
+{
+	return (device->card_type == GA100 || device->card_type == AD100) &&
+	    gsp->memsys.use_raw_mode_comptagline_alloc;
+}
+
 static void
 r570_fbsr_resume(struct nvkm_gsp *gsp)
 {
@@ -33,6 +79,7 @@ r570_fbsr_resume(struct nvkm_gsp *gsp)
 	struct nvkm_instmem *imem = device->imem;
 	struct nvkm_instobj *iobj;
 	struct nvkm_vmm *vmm;
+	int ret;
 
 	/* Restore BAR2 page tables via BAR0 window, and re-enable BAR2. */
 	list_for_each_entry(iobj, &imem->boot, head) {
@@ -54,6 +101,13 @@ r570_fbsr_resume(struct nvkm_gsp *gsp)
 	vmm = nvkm_bar_bar1_vmm(device);
 	vmm->func->flush(vmm, 0);
 
+	/* Re-enable raw mode if it was previously disabled */
+	if (r570_need_raw_comp_war(gsp, device)) {
+		ret = r570_memsys_enable_raw_comp_mode(gsp, true);
+		if (ret)
+			nvkm_error(&gsp->subdev, "Failed to re-enable raw comp mode\n");
+	}
+
 	/* Resume channel scheduling. */
 	r570_fbsr_suspend_channels(device->gsp, false);
 
@@ -81,7 +135,7 @@ r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size)
 	ctrl->hClient = gsp->internal.client.object.handle;
 	ctrl->hSysMem = memlist.handle;
 	ctrl->sysmemAddrOfSuspendResumeData = gsp->sr.meta.addr;
-	ctrl->bEnteringGcoffState = 0;
+	ctrl->bEnteringGcoffState = 1;
 
 	ret = nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl);
 	if (ret)
@@ -98,12 +152,29 @@ r570_fbsr_suspend(struct nvkm_gsp *gsp)
 	struct nvkm_device *device = subdev->device;
 	struct nvkm_instmem *imem = device->imem;
 	struct nvkm_instobj *iobj;
-	u64 size;
+	u64 size, compbit_store_size;
 	int ret;
 
 	/* Stop channel scheduling. */
 	r570_fbsr_suspend_channels(gsp, true);
 
+	/* Temporarily disable raw mode to prevent FBSR restore operations from corrupting
+	 * compressed surfaces. Required for ampere and ada.
+	 *
+	 * Nvidia bug #3172217
+	 */
+	if (r570_need_raw_comp_war(gsp, device)) {
+		ret = r570_memsys_enable_raw_comp_mode(gsp, false);
+		if (ret)
+			return ret;
+	}
+
+	ret = r570_fb_get_compbit_store_size(gsp, &compbit_store_size);
+	if (ret < 0)
+		return ret;
+	nvkm_debug(&gsp->subdev, "fbsr: Compbit backing store size: 0x%llx bytes\n",
+		   compbit_store_size);
+
 	/* Save BAR2 allocations to system memory. */
 	list_for_each_entry(iobj, &imem->list, head) {
 		if (iobj->preserve) {
@@ -126,6 +197,8 @@ r570_fbsr_suspend(struct nvkm_gsp *gsp)
 	size  = gsp->fb.heap.size;
 	size += gsp->fb.rsvd_size;
 	size += gsp->fb.bios.vga_workspace.size;
+	size += compbit_store_size;
+
 	nvkm_debug(subdev, "fbsr: size: 0x%llx bytes\n", size);
 
 	ret = nvkm_gsp_sg(device, size, &gsp->sr.fbsr);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c
index b45781cd0dfdce..ea38a94211f4b1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c
@@ -10,6 +10,7 @@
 #include "nvrm/gsp.h"
 #include "nvrm/rpcfn.h"
 #include "nvrm/msgfn.h"
+#include "nvif/cl0080.h"
 
 #include <core/pci.h>
 #include <subdev/pci/priv.h>
@@ -137,6 +138,14 @@ r570_gsp_get_static_info(struct nvkm_gsp *gsp)
 		}
 	}
 
+	ret = r535_gsp_get_static_memsys_info(gsp);
+	if (ret) {
+		nvkm_error(&gsp->subdev, "Retrieving static memsys info failed\n");
+		return ret;
+	}
+	nvkm_debug(&gsp->subdev, "memsys: Use raw mode for comptag allocations? %s\n",
+		   str_yes_no(gsp->memsys.use_raw_mode_comptagline_alloc));
+
 	return 0;
 }
 
@@ -215,6 +224,32 @@ r570_gsp_set_rmargs(struct nvkm_gsp *gsp, bool resume)
 	args->bDmemStack = 1;
 }
 
+int
+r570_gsp_gcx_ready(struct nvkm_gsp *gsp)
+{
+	NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS *ctrl;
+	int ret = 0;
+
+	ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,
+				   NV2080_CTRL_CMD_INTERNAL_GCX_ENTRY_PREREQUISITE,
+				   sizeof(*ctrl));
+	if (IS_ERR(ctrl))
+		return PTR_ERR(ctrl);
+
+	if (ctrl->bIsGC6Satisfied)
+		ret |= NV_DEVICE_GC6_READY;
+	if (ctrl->bIsGCOFFSatisfied)
+		ret |= NV_DEVICE_GCOFF_READY;
+
+	nvkm_debug(&gsp->subdev,
+		   "GCX ready status: GC6=%s GCOFF=%s\n",
+		   str_yes_no(ctrl->bIsGC6Satisfied), str_yes_no(ctrl->bIsGCOFFSatisfied));
+
+	nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
+	return ret;
+}
+
+
 const struct nvkm_rm_api_gsp
 r570_gsp = {
 	.set_rmargs = r570_gsp_set_rmargs,
@@ -223,4 +258,5 @@ r570_gsp = {
 	.xlat_mc_engine_idx = r570_gsp_xlat_mc_engine_idx,
 	.drop_post_nocat_record = r570_gsp_drop_post_nocat_record,
 	.sr_data_size = r570_gsp_sr_data_size,
+	.gcx_ready = r570_gsp_gcx_ready,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h
index 8af432375f7a27..cb3e448415b42d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h
@@ -16,4 +16,33 @@ typedef struct NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS {
     NV_DECLARE_ALIGNED(NvU64 sysmemAddrOfSuspendResumeData, 8);
 } NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS;
 
+#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO (0x801306) /* finn: Evaluated from "(FINN_NV01_DEVICE_0_FB_INTERFACE_ID << 8) | NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS_MESSAGE_ID" */
+
+typedef struct NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS {
+    NV_DECLARE_ALIGNED(NvU64 Size, 8);
+    NV_DECLARE_ALIGNED(NvU64 Address, 8);
+    NvU32 AddressSpace;
+    NvU32 MaxCompbitLine;
+    NvU32 comptagsPerCacheLine;
+    NvU32 cacheLineSize;
+    NvU32 cacheLineSizePerSlice;
+    NvU32 cacheLineFetchAlignment;
+    NV_DECLARE_ALIGNED(NvU64 backingStoreBase, 8);
+    NvU32 gobsPerComptagPerSlice;
+    NvU32 backingStoreCbcBase;
+    NvU32 comptaglineAllocationPolicy;
+    NV_DECLARE_ALIGNED(NvU64 privRegionStartOffset, 8);
+    NvU32 cbcCoveragePerSlice;
+} NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS;
+
+#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_UNKNOWN 0 // ADDR_UNKNOWN
+#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_SYSMEM  1 // ADDR_SYSMEM
+#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_FBMEM   2 // ADDR_FBMEM
+
+#define NV2080_CTRL_CMD_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE (0x20800a6f) /* finn: Evaluated from "(FINN_NV20_SUBDEVICE_0_INTERNAL_INTERFACE_ID << 8) | NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS_MESSAGE_ID" */
+
+typedef struct NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS {
+    NvBool bRawMode;
+} NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS;
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h
index c458569af9d720..2814629fddd220 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h
@@ -639,4 +639,11 @@ typedef struct GSP_FMC_BOOT_PARAMS
 } GSP_FMC_BOOT_PARAMS;
 
 #define GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100              (14 << 20)   // Hopper+
+
+#define NV2080_CTRL_CMD_INTERNAL_GCX_ENTRY_PREREQUISITE (0x2080a7d7)
+
+typedef struct NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS {
+    NvBool bIsGC6Satisfied;
+    NvBool bIsGCOFFSatisfied;
+} NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h
index e9ac47d86b69af..17480d4e527ac0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h
@@ -40,6 +40,7 @@ struct nvkm_rm_api {
 		void (*drop_send_user_shared_data)(struct nvkm_gsp *);
 		void (*drop_post_nocat_record)(struct nvkm_gsp *);
 		u32 (*sr_data_size)(struct nvkm_gsp *);
+		int (*gcx_ready)(struct nvkm_gsp *gsp);
 	} *gsp;
 
 	const struct nvkm_rm_api_rpc {
@@ -174,6 +175,8 @@ int r535_gr_chan_new(struct nvkm_gr *, struct nvkm_chan *, const struct nvkm_ocl
 int r535_gr_promote_ctx(struct r535_gr *, bool golden, struct nvkm_vmm *,
 			struct nvkm_memory **pctxbuf_mem, struct nvkm_vma **pctxbuf_vma,
 			struct nvkm_gsp_object *chan);
+int r570_gsp_gcx_ready(struct nvkm_gsp *gsp);
+int r535_gsp_get_static_memsys_info(struct nvkm_gsp *gsp);
 extern const struct nvkm_rm_api_engine r535_nvdec;
 extern const struct nvkm_rm_api_engine r535_nvenc;
 extern const struct nvkm_rm_api_engine r535_nvjpg;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
index a14ea0f7b1c879..90f03baeb22ac9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
@@ -9,6 +9,7 @@ nvkm-y += nvkm/subdev/pci/nv4c.o
 nvkm-y += nvkm/subdev/pci/g84.o
 nvkm-y += nvkm/subdev/pci/g92.o
 nvkm-y += nvkm/subdev/pci/g94.o
+nvkm-y += nvkm/subdev/pci/mcp79.o
 nvkm-y += nvkm/subdev/pci/gf100.o
 nvkm-y += nvkm/subdev/pci/gf106.o
 nvkm-y += nvkm/subdev/pci/gk104.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/mcp79.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/mcp79.c
new file mode 100644
index 00000000000000..e2ae242f1aa771
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/mcp79.c
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Marek Czernohous
+ *
+ * MCP79/MCP7A (NVAC): like g94, but MSI re-arm goes through real PCI
+ * config space.  The MMIO-mirror re-arm is unreliable on this IGP and a
+ * missed re-arm kills the interrupt line (see the nv46 comment; g92
+ * already re-arms through config space for the same reason).
+ */
+#include "priv.h"
+
+static const struct nvkm_pci_func
+mcp79_pci_func = {
+	.cfg = { .addr = 0x088000, .size = 0x1000 },
+
+	.init = g84_pci_init,
+	.msi_rearm = nv46_pci_msi_rearm,
+
+	.pcie.init = g84_pcie_init,
+	.pcie.set_link = g84_pcie_set_link,
+
+	.pcie.max_speed = g84_pcie_max_speed,
+	.pcie.cur_speed = g84_pcie_cur_speed,
+
+	.pcie.set_version = g84_pcie_set_version,
+	.pcie.version = g84_pcie_version,
+	.pcie.version_supported = g92_pcie_version_supported,
+};
+
+int
+mcp79_pci_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
+	      struct nvkm_pci **ppci)
+{
+	return nvkm_pci_new_(&mcp79_pci_func, device, type, inst, ppci);
+}
diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs
index 60752bccddc0bd..46e13fe795eb33 100644
--- a/drivers/gpu/drm/nova/driver.rs
+++ b/drivers/gpu/drm/nova/driver.rs
@@ -18,6 +18,11 @@ use kernel::{
 use crate::file::File;
 use crate::gem::NovaObject;
 
+use nova_core::api::{
+    NovaCoreApi,
+    NovaCoreApiHandle, //
+};
+
 pub(crate) struct NovaDriver;
 
 pub(crate) struct Nova<'bound> {
@@ -26,6 +31,11 @@ pub(crate) struct Nova<'bound> {
     _reg: drm::Registration<'bound, NovaDriver>,
 }
 
+/// DRM registration data, accessible from ioctl handlers via the registration guard.
+pub(crate) struct DrmRegData<'bound> {
+    pub(crate) api: NovaCoreApiHandle<'bound>,
+}
+
 /// Convienence type alias for the DRM device type for this driver
 pub(crate) type NovaDevice<Ctx = drm::Normal> = drm::Device<NovaDriver, Ctx>;
 
@@ -59,9 +69,12 @@ impl auxiliary::Driver for NovaDriver {
         _info: &'bound Self::IdInfo,
     ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
         let drm = drm::UnregisteredDevice::<Self>::new(adev, Ok(()))?;
+        let reg_data = DrmRegData {
+            api: NovaCoreApi::of(adev)?,
+        };
         // SAFETY: `reg` is stored in `Nova` and dropped when the driver is unbound; it is
         // never forgotten.
-        let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, (), 0)? };
+        let reg = unsafe { drm::Registration::new(adev.as_ref(), drm, reg_data, 0)? };
 
         Ok(Nova {
             drm: reg.device().into(),
@@ -73,16 +86,18 @@ impl auxiliary::Driver for NovaDriver {
 #[vtable]
 impl drm::Driver for NovaDriver {
     type Data = ();
-    type RegistrationData<'a> = ();
+    type RegistrationData<'a> = DrmRegData<'a>;
     type File = File;
     type Object = gem::Object<NovaObject>;
     type ParentDevice<Ctx: DeviceContext> = auxiliary::Device<Ctx>;
 
     const INFO: drm::DriverInfo = INFO;
+    const FEAT_RENDER: bool = true;
 
     kernel::declare_drm_ioctls! {
         (NOVA_GETPARAM, drm_nova_getparam, ioctl::RENDER_ALLOW, File::get_param),
         (NOVA_GEM_CREATE, drm_nova_gem_create, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_create),
         (NOVA_GEM_INFO, drm_nova_gem_info, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_info),
+        (NOVA_INFO, drm_nova_info, ioctl::RENDER_ALLOW, File::info),
     }
 }
diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs
index 298c02bacb4bce..766a6ddb7e10a0 100644
--- a/drivers/gpu/drm/nova/file.rs
+++ b/drivers/gpu/drm/nova/file.rs
@@ -1,23 +1,104 @@
 // SPDX-License-Identifier: GPL-2.0
 
-use crate::driver::{NovaDevice, NovaDriver};
+use crate::driver::{
+    DrmRegData,
+    NovaDevice,
+    NovaDriver, //
+};
 use crate::gem::NovaObject;
 use kernel::{
     alloc::flags::*,
-    auxiliary,
-    device::Bound,
     drm::{
         self,
         gem::BaseObject,
         Registered, //
     },
-    pci,
+    num::casts::{
+        arch::IntoSafeCastArch,
+        IntoSafeCast, //
+    },
     prelude::*,
+    transmute::AsBytes,
+    uaccess::UserSlice,
     uapi,
 };
 
 pub(crate) struct File;
 
+/// GPU information returned to userspace.
+#[repr(transparent)]
+struct GpuInfo(uapi::drm_nova_info_gpu);
+
+/// Copies `name` into the zero initialised, fixed size uAPI buffer `dst`, keeping it
+/// NUL-terminated.
+///
+/// Fails with [`ENAMETOOLONG`] if `name` does not fit in `dst` with room for the terminator.
+fn copy_name(dst: &mut [u8], name: &str) -> Result {
+    let bytes = name.as_bytes();
+
+    if bytes.len() >= dst.len() {
+        return Err(ENAMETOOLONG);
+    }
+    dst[..bytes.len()].copy_from_slice(bytes);
+
+    Ok(())
+}
+
+impl GpuInfo {
+    /// Collects the GPU information reported to userspace.
+    ///
+    /// This is fallible so that unexpected GSP behaviour, such as a malformed name string, is
+    /// reported to userspace instead of being silently replaced with a made up value. When adding
+    /// a field, take care that a value the GSP simply does not provide, for example because the
+    /// firmware predates it, does not cause a failure. Such fields must fall back to zero or
+    /// another documented default instead.
+    fn new(reg_data: &DrmRegData<'_>) -> Result<Self> {
+        let spec = reg_data.api.with(|api| api.get_ref().spec());
+        let gsp_static_info = reg_data.api.with(|api| api.get_ref().gsp_static_info());
+
+        let mut info = uapi::drm_nova_info_gpu {
+            architecture: spec.chipset.arch().into(),
+            chipid: spec.chipset.into(),
+            vram_size: gsp_static_info.vram_size(),
+            gpu_gid: gsp_static_info.gpu_gid,
+            ..pin_init::zeroed()
+        };
+
+        copy_name(
+            &mut info.gpu_name,
+            gsp_static_info.gpu_name().map_err(|_| EINVAL)?,
+        )?;
+
+        copy_name(
+            &mut info.gpu_short_name,
+            gsp_static_info.gpu_short_name().map_err(|_| EINVAL)?,
+        )?;
+
+        Ok(Self(info))
+    }
+}
+
+// SAFETY: `GpuInfo` has no interior mutability, and there are no uninitialized bytes.
+unsafe impl AsBytes for GpuInfo {}
+
+fn write_info<T: AsBytes>(info: &mut uapi::drm_nova_info, value: T) -> Result {
+    // A NULL data pointer requests the size of the information structure.
+    if info.data == 0 {
+        info.size = size_of::<T>().try_into()?;
+        return Ok(());
+    }
+
+    let mut writer = UserSlice::new(
+        UserPtr::from_addr(info.data.into_safe_cast_arch()),
+        info.size.into_safe_cast(),
+    )
+    .writer();
+
+    info.size = writer.write_truncated(&value)?.try_into()?;
+
+    Ok(())
+}
+
 impl drm::file::DriverFile for File {
     type Driver = NovaDriver;
 
@@ -29,16 +110,13 @@ impl drm::file::DriverFile for File {
 impl File {
     /// IOCTL: get_param: Query GPU / driver metadata.
     pub(crate) fn get_param(
-        dev: &NovaDevice<Registered>,
-        _reg_data: &(),
+        _dev: &NovaDevice<Registered>,
+        reg_data: &DrmRegData<'_>,
         getparam: &mut uapi::drm_nova_getparam,
         _file: &drm::File<File>,
     ) -> Result<u32> {
-        let adev: &auxiliary::Device<Bound> = dev.as_ref();
-        let pdev: &pci::Device<Bound> = adev.parent().try_into()?;
-
-        let value = match getparam.param as u32 {
-            uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => pdev.resource_len(1)?,
+        let value = match getparam.param.try_into()? {
+            uapi::NOVA_GETPARAM_VRAM_BAR_SIZE => reg_data.api.with(|api| api.bar1_size())?,
             _ => return Err(EINVAL),
         };
 
@@ -50,7 +128,7 @@ impl File {
     /// IOCTL: gem_create: Create a new DRM GEM object.
     pub(crate) fn gem_create(
         dev: &NovaDevice<Registered>,
-        _reg_data: &(),
+        _reg_data: &DrmRegData<'_>,
         req: &mut uapi::drm_nova_gem_create,
         file: &drm::File<File>,
     ) -> Result<u32> {
@@ -64,7 +142,7 @@ impl File {
     /// IOCTL: gem_info: Query GEM metadata.
     pub(crate) fn gem_info(
         _dev: &NovaDevice<Registered>,
-        _reg_data: &(),
+        _reg_data: &DrmRegData<'_>,
         req: &mut uapi::drm_nova_gem_info,
         file: &drm::File<File>,
     ) -> Result<u32> {
@@ -74,4 +152,19 @@ impl File {
 
         Ok(0)
     }
+
+    /// IOCTL: info: Query device information.
+    pub(crate) fn info(
+        _dev: &NovaDevice<Registered>,
+        reg_data: &DrmRegData<'_>,
+        info: &mut uapi::drm_nova_info,
+        _file: &drm::File<File>,
+    ) -> Result<u32> {
+        match info.id {
+            uapi::DRM_NOVA_INFO_GPU => write_info(info, GpuInfo::new(reg_data)?)?,
+            _ => return Err(EINVAL),
+        }
+
+        Ok(0)
+    }
 }
diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index 6da720e8732d79..838b34ea7e31e8 100644
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -38,7 +38,6 @@
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_mipi_dsi.h>
-#include <drm/drm_panel.h>
 #include <video/mipi_display.h>
 
 #include "omapdss.h"
@@ -59,9 +58,6 @@ static void dsi_uninit_dispc(struct dsi_data *dsi);
 
 static int dsi_vc_send_null(struct dsi_data *dsi, int vc, int channel);
 
-static ssize_t _omap_dsi_host_transfer(struct dsi_data *dsi, int vc,
-				       const struct mipi_dsi_msg *msg);
-
 #ifdef DSI_PERF_MEASURE
 static bool dsi_perf;
 module_param(dsi_perf, bool, 0644);
@@ -342,7 +338,7 @@ static void dsi_handle_irq_errors(struct dsi_data *dsi, u32 irqstatus,
 	int i;
 
 	if (irqstatus & DSI_IRQ_ERROR_MASK) {
-		DSSERR("DSI error, irqstatus %x\n", irqstatus);
+		dev_err(dsi->dev, "DSI error, irqstatus %x\n", irqstatus);
 		print_irq_status(irqstatus);
 		spin_lock(&dsi->errors_lock);
 		dsi->errors |= irqstatus & DSI_IRQ_ERROR_MASK;
@@ -353,8 +349,8 @@ static void dsi_handle_irq_errors(struct dsi_data *dsi, u32 irqstatus,
 
 	for (i = 0; i < 4; ++i) {
 		if (vcstatus[i] & DSI_VC_IRQ_ERROR_MASK) {
-			DSSERR("DSI VC(%d) error, vc irqstatus %x\n",
-				       i, vcstatus[i]);
+			dev_err(dsi->dev, "DSI VC(%d) error, vc irqstatus %x\n",
+				i, vcstatus[i]);
 			print_irq_status_vc(i, vcstatus[i]);
 		} else if (debug_irq) {
 			print_irq_status_vc(i, vcstatus[i]);
@@ -362,7 +358,7 @@ static void dsi_handle_irq_errors(struct dsi_data *dsi, u32 irqstatus,
 	}
 
 	if (ciostatus & DSI_CIO_IRQ_ERROR_MASK) {
-		DSSERR("DSI CIO error, cio irqstatus %x\n", ciostatus);
+		dev_err(dsi->dev, "DSI CIO error, cio irqstatus %x\n", ciostatus);
 		print_irq_status_cio(ciostatus);
 	} else if (debug_irq) {
 		print_irq_status_cio(ciostatus);
@@ -698,8 +694,6 @@ static int dsi_runtime_get(struct dsi_data *dsi)
 {
 	int r;
 
-	DSSDBG("dsi_runtime_get\n");
-
 	r = pm_runtime_get_sync(dsi->dev);
 	if (WARN_ON(r < 0)) {
 		pm_runtime_put_noidle(dsi->dev);
@@ -712,8 +706,6 @@ static void dsi_runtime_put(struct dsi_data *dsi)
 {
 	int r;
 
-	DSSDBG("dsi_runtime_put\n");
-
 	r = pm_runtime_put_sync(dsi->dev);
 	WARN_ON(r < 0 && r != -ENOSYS);
 }
@@ -755,13 +747,13 @@ static void _dsi_print_reset_status(struct dsi_data *dsi)
 
 static inline int dsi_if_enable(struct dsi_data *dsi, bool enable)
 {
-	DSSDBG("dsi_if_enable(%d)\n", enable);
+	dev_dbg(dsi->dev, "dsi_if_enable(%d)\n", enable);
 
 	enable = enable ? 1 : 0;
 	REG_FLD_MOD(dsi, DSI_CTRL, enable, 0, 0); /* IF_EN */
 
 	if (!wait_for_bit_change(dsi, DSI_CTRL, 0, enable)) {
-		DSSERR("Failed to set dsi_if_enable to %d\n", enable);
+		dev_err(dsi->dev, "Failed to set dsi_if_enable to %d\n", enable);
 		return -EIO;
 	}
 
@@ -836,7 +828,7 @@ static int dsi_set_lp_clk_divisor(struct dsi_data *dsi)
 
 	lp_clk = dsi_fclk / 2 / lp_clk_div;
 
-	DSSDBG("LP_CLK_DIV %u, LP_CLK %lu\n", lp_clk_div, lp_clk);
+	dev_dbg(dsi->dev, "LP_CLK_DIV %u, LP_CLK %lu\n", lp_clk_div, lp_clk);
 	dsi->current_lp_cinfo.lp_clk = lp_clk;
 	dsi->current_lp_cinfo.lp_clk_div = lp_clk_div;
 
@@ -884,8 +876,8 @@ static int dsi_pll_power(struct dsi_data *dsi, enum dsi_pll_power_state state)
 	/* PLL_PWR_STATUS */
 	while (FLD_GET(dsi_read_reg(dsi, DSI_CLK_CTRL), 29, 28) != state) {
 		if (++t > 1000) {
-			DSSERR("Failed to set DSI PLL power mode to %d\n",
-					state);
+			dev_err(dsi->dev, "Failed to set DSI PLL power mode to %d\n",
+				state);
 			return -ENODEV;
 		}
 		udelay(1);
@@ -911,7 +903,7 @@ static int dsi_pll_enable(struct dss_pll *pll)
 	struct dsi_data *dsi = container_of(pll, struct dsi_data, pll);
 	int r = 0;
 
-	DSSDBG("PLL init\n");
+	dev_dbg(dsi->dev, "PLL init\n");
 
 	r = dsi_runtime_get(dsi);
 	if (r)
@@ -930,7 +922,7 @@ static int dsi_pll_enable(struct dss_pll *pll)
 	dispc_pck_free_enable(dsi->dss->dispc, 1);
 
 	if (!wait_for_bit_change(dsi, DSI_PLL_STATUS, 0, 1)) {
-		DSSERR("PLL not coming out of reset.\n");
+		dev_err(dsi->dev, "PLL not coming out of reset.\n");
 		r = -ENODEV;
 		dispc_pck_free_enable(dsi->dss->dispc, 0);
 		goto err1;
@@ -945,7 +937,7 @@ static int dsi_pll_enable(struct dss_pll *pll)
 	if (r)
 		goto err1;
 
-	DSSDBG("PLL init done\n");
+	dev_dbg(dsi->dev, "PLL init done\n");
 
 	return 0;
 err1:
@@ -967,7 +959,7 @@ static void dsi_pll_disable(struct dss_pll *pll)
 	dsi_disable_scp_clk(dsi);
 	dsi_runtime_put(dsi);
 
-	DSSDBG("PLL disable done\n");
+	dev_dbg(dsi->dev, "PLL disable done\n");
 }
 
 static int dsi_dump_dsi_clocks(struct seq_file *s, void *p)
@@ -1231,7 +1223,7 @@ static int dsi_cio_power(struct dsi_data *dsi, enum dsi_cio_power_state state)
 	while (FLD_GET(dsi_read_reg(dsi, DSI_COMPLEXIO_CFG1),
 			26, 25) != state) {
 		if (++t > 1000) {
-			DSSERR("failed to set complexio power state to "
+			dev_err(dsi->dev, "failed to set complexio power state to "
 					"%d\n", state);
 			return -ENODEV;
 		}
@@ -1372,20 +1364,19 @@ static void dsi_cio_timings(struct dsi_data *dsi)
 	/* min tclk-prepare + tclk-zero = 300ns */
 	tclk_zero = ns2ddr(dsi, 260);
 
-	DSSDBG("ths_prepare %u (%uns), ths_prepare_ths_zero %u (%uns)\n",
+	dev_dbg(dsi->dev, "ths_prepare %u (%uns), ths_prepare_ths_zero %u (%uns)\n",
 		ths_prepare, ddr2ns(dsi, ths_prepare),
 		ths_prepare_ths_zero, ddr2ns(dsi, ths_prepare_ths_zero));
-	DSSDBG("ths_trail %u (%uns), ths_exit %u (%uns)\n",
-			ths_trail, ddr2ns(dsi, ths_trail),
-			ths_exit, ddr2ns(dsi, ths_exit));
-
-	DSSDBG("tlpx_half %u (%uns), tclk_trail %u (%uns), "
-			"tclk_zero %u (%uns)\n",
-			tlpx_half, ddr2ns(dsi, tlpx_half),
-			tclk_trail, ddr2ns(dsi, tclk_trail),
-			tclk_zero, ddr2ns(dsi, tclk_zero));
-	DSSDBG("tclk_prepare %u (%uns)\n",
-			tclk_prepare, ddr2ns(dsi, tclk_prepare));
+	dev_dbg(dsi->dev, "ths_trail %u (%uns), ths_exit %u (%uns)\n",
+		ths_trail, ddr2ns(dsi, ths_trail),
+		ths_exit, ddr2ns(dsi, ths_exit));
+
+	dev_dbg(dsi->dev, "tlpx_half %u (%uns), tclk_trail %u (%uns), tclk_zero %u (%uns)\n",
+		tlpx_half, ddr2ns(dsi, tlpx_half),
+		tclk_trail, ddr2ns(dsi, tclk_trail),
+		tclk_zero, ddr2ns(dsi, tclk_zero));
+	dev_dbg(dsi->dev, "tclk_prepare %u (%uns)\n",
+		tclk_prepare, ddr2ns(dsi, tclk_prepare));
 
 	/* program timings */
 
@@ -1451,8 +1442,8 @@ static int dsi_cio_wait_tx_clk_esc_reset(struct dsi_data *dsi)
 				if (!in_use[i] || (l & (1 << offsets[i])))
 					continue;
 
-				DSSERR("CIO TXCLKESC%d domain not coming " \
-						"out of reset\n", i);
+				dev_err(dsi->dev,
+					"CIO TXCLKESC%d domain not coming out of reset\n", i);
 			}
 			return -EIO;
 		}
@@ -1557,7 +1548,7 @@ static int dsi_cio_init(struct dsi_data *dsi)
 	int r;
 	u32 l;
 
-	DSSDBG("DSI CIO init starts");
+	dev_dbg(dsi->dev, "DSI CIO init starts");
 
 	r = dsi_enable_pads(dsi, dsi_get_lane_mask(dsi));
 	if (r)
@@ -1571,7 +1562,7 @@ static int dsi_cio_init(struct dsi_data *dsi)
 	dsi_read_reg(dsi, DSI_DSIPHY_CFG5);
 
 	if (!wait_for_bit_change(dsi, DSI_DSIPHY_CFG5, 30, 1)) {
-		DSSERR("CIO SCP Clock domain not coming out of reset.\n");
+		dev_err(dsi->dev, "CIO SCP Clock domain not coming out of reset.\n");
 		r = -EIO;
 		goto err_scp_clk_dom;
 	}
@@ -1593,7 +1584,7 @@ static int dsi_cio_init(struct dsi_data *dsi)
 		goto err_cio_pwr;
 
 	if (!wait_for_bit_change(dsi, DSI_COMPLEXIO_CFG1, 29, 1)) {
-		DSSERR("CIO PWR clock domain not coming out of reset.\n");
+		dev_err(dsi->dev, "CIO PWR clock domain not coming out of reset.\n");
 		r = -ENODEV;
 		goto err_cio_pwr_dom;
 	}
@@ -1616,7 +1607,7 @@ static int dsi_cio_init(struct dsi_data *dsi)
 		    !(dsi->dsidev->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS),
 		    13, 13);
 
-	DSSDBG("CIO init done\n");
+	dev_dbg(dsi->dev, "CIO init done\n");
 
 	return 0;
 
@@ -1659,7 +1650,7 @@ static void dsi_config_tx_fifo(struct dsi_data *dsi,
 		int size = dsi->vc[i].tx_fifo_size;
 
 		if (add + size > 4) {
-			DSSERR("Illegal FIFO configuration\n");
+			dev_err(dsi->dev, "Illegal FIFO configuration\n");
 			BUG();
 			return;
 		}
@@ -1691,7 +1682,7 @@ static void dsi_config_rx_fifo(struct dsi_data *dsi,
 		int size = dsi->vc[i].rx_fifo_size;
 
 		if (add + size > 4) {
-			DSSERR("Illegal FIFO configuration\n");
+			dev_err(dsi->dev, "Illegal FIFO configuration\n");
 			BUG();
 			return;
 		}
@@ -1714,7 +1705,7 @@ static int dsi_force_tx_stop_mode_io(struct dsi_data *dsi)
 	dsi_write_reg(dsi, DSI_TIMING1, r);
 
 	if (!wait_for_bit_change(dsi, DSI_TIMING1, 15, 0)) {
-		DSSERR("TX_STOP bit not going down\n");
+		dev_err(dsi->dev, "TX_STOP bit not going down\n");
 		return -EIO;
 	}
 
@@ -1759,7 +1750,7 @@ static int dsi_sync_vc_vp(struct dsi_data *dsi, int vc)
 	if (REG_GET(dsi, DSI_VC_TE(vc), bit, bit)) {
 		if (wait_for_completion_timeout(&completion,
 				msecs_to_jiffies(10)) == 0) {
-			DSSERR("Failed to complete previous frame transfer\n");
+			dev_err(dsi->dev, "Failed to complete previous frame transfer\n");
 			r = -EIO;
 			goto err1;
 		}
@@ -1805,7 +1796,7 @@ static int dsi_sync_vc_l4(struct dsi_data *dsi, int vc)
 	if (REG_GET(dsi, DSI_VC_CTRL(vc), 5, 5)) {
 		if (wait_for_completion_timeout(&completion,
 				msecs_to_jiffies(10)) == 0) {
-			DSSERR("Failed to complete previous l4 transfer\n");
+			dev_err(dsi->dev, "Failed to complete previous l4 transfer\n");
 			r = -EIO;
 			goto err1;
 		}
@@ -1844,15 +1835,15 @@ static int dsi_sync_vc(struct dsi_data *dsi, int vc)
 
 static int dsi_vc_enable(struct dsi_data *dsi, int vc, bool enable)
 {
-	DSSDBG("dsi_vc_enable vc %d, enable %d\n",
-			vc, enable);
+	dev_dbg(dsi->dev, "dsi_vc_enable vc %d, enable %d\n",
+		vc, enable);
 
 	enable = enable ? 1 : 0;
 
 	REG_FLD_MOD(dsi, DSI_VC_CTRL(vc), enable, 0, 0);
 
 	if (!wait_for_bit_change(dsi, DSI_VC_CTRL(vc), 0, enable)) {
-		DSSERR("Failed to set dsi_vc_enable to %d\n", enable);
+		dev_err(dsi->dev, "Failed to set VC_EN to %d\n", enable);
 		return -EIO;
 	}
 
@@ -1863,13 +1854,13 @@ static void dsi_vc_initial_config(struct dsi_data *dsi, int vc)
 {
 	u32 r;
 
-	DSSDBG("Initial config of VC %d", vc);
+	dev_dbg(dsi->dev, "Initial config of VC %d", vc);
 
 	r = dsi_read_reg(dsi, DSI_VC_CTRL(vc));
 
 	if (FLD_GET(r, 15, 15)) /* VC_BUSY */
-		DSSERR("VC(%d) busy when trying to configure it!\n",
-				vc);
+		dev_err(dsi->dev, "VC(%d) busy when trying to configure it!\n",
+			vc);
 
 	r = FLD_MOD(r, 0, 1, 1); /* SOURCE, 0 = L4 */
 	r = FLD_MOD(r, 0, 2, 2); /* BTA_SHORT_EN  */
@@ -1894,7 +1885,7 @@ static void dsi_vc_enable_hs(struct omap_dss_device *dssdev, int vc,
 {
 	struct dsi_data *dsi = to_dsi_data(dssdev);
 
-	DSSDBG("dsi_vc_enable_hs(%d, %d)\n", vc, enable);
+	dev_dbg(dsi->dev, "%s(%d, %d)\n", __func__, vc, enable);
 
 	if (REG_GET(dsi, DSI_VC_CTRL(vc), 9, 9) == enable)
 		return;
@@ -1917,49 +1908,49 @@ static void dsi_vc_flush_long_data(struct dsi_data *dsi, int vc)
 	while (REG_GET(dsi, DSI_VC_CTRL(vc), 20, 20)) {
 		u32 val;
 		val = dsi_read_reg(dsi, DSI_VC_SHORT_PACKET_HEADER(vc));
-		DSSDBG("\t\tb1 %#02x b2 %#02x b3 %#02x b4 %#02x\n",
-				(val >> 0) & 0xff,
-				(val >> 8) & 0xff,
-				(val >> 16) & 0xff,
-				(val >> 24) & 0xff);
+		dev_dbg(dsi->dev, "\t\tb1 %#02x b2 %#02x b3 %#02x b4 %#02x\n",
+			(val >> 0) & 0xff,
+			(val >> 8) & 0xff,
+			(val >> 16) & 0xff,
+			(val >> 24) & 0xff);
 	}
 }
 
-static void dsi_show_rx_ack_with_err(u16 err)
+static void dsi_show_rx_ack_with_err(struct dsi_data *dsi, u16 err)
 {
-	DSSERR("\tACK with ERROR (%#x):\n", err);
+	dev_err(dsi->dev, "\tACK with ERROR (%#x):\n", err);
 	if (err & (1 << 0))
-		DSSERR("\t\tSoT Error\n");
+		dev_err(dsi->dev, "\t\tSoT Error\n");
 	if (err & (1 << 1))
-		DSSERR("\t\tSoT Sync Error\n");
+		dev_err(dsi->dev, "\t\tSoT Sync Error\n");
 	if (err & (1 << 2))
-		DSSERR("\t\tEoT Sync Error\n");
+		dev_err(dsi->dev, "\t\tEoT Sync Error\n");
 	if (err & (1 << 3))
-		DSSERR("\t\tEscape Mode Entry Command Error\n");
+		dev_err(dsi->dev, "\t\tEscape Mode Entry Command Error\n");
 	if (err & (1 << 4))
-		DSSERR("\t\tLP Transmit Sync Error\n");
+		dev_err(dsi->dev, "\t\tLP Transmit Sync Error\n");
 	if (err & (1 << 5))
-		DSSERR("\t\tHS Receive Timeout Error\n");
+		dev_err(dsi->dev, "\t\tHS Receive Timeout Error\n");
 	if (err & (1 << 6))
-		DSSERR("\t\tFalse Control Error\n");
+		dev_err(dsi->dev, "\t\tFalse Control Error\n");
 	if (err & (1 << 7))
-		DSSERR("\t\t(reserved7)\n");
+		dev_err(dsi->dev, "\t\t(reserved7)\n");
 	if (err & (1 << 8))
-		DSSERR("\t\tECC Error, single-bit (corrected)\n");
+		dev_err(dsi->dev, "\t\tECC Error, single-bit (corrected)\n");
 	if (err & (1 << 9))
-		DSSERR("\t\tECC Error, multi-bit (not corrected)\n");
+		dev_err(dsi->dev, "\t\tECC Error, multi-bit (not corrected)\n");
 	if (err & (1 << 10))
-		DSSERR("\t\tChecksum Error\n");
+		dev_err(dsi->dev, "\t\tChecksum Error\n");
 	if (err & (1 << 11))
-		DSSERR("\t\tData type not recognized\n");
+		dev_err(dsi->dev, "\t\tData type not recognized\n");
 	if (err & (1 << 12))
-		DSSERR("\t\tInvalid VC ID\n");
+		dev_err(dsi->dev, "\t\tInvalid VC ID\n");
 	if (err & (1 << 13))
-		DSSERR("\t\tInvalid Transmission Length\n");
+		dev_err(dsi->dev, "\t\tInvalid Transmission Length\n");
 	if (err & (1 << 14))
-		DSSERR("\t\t(reserved14)\n");
+		dev_err(dsi->dev, "\t\t(reserved14)\n");
 	if (err & (1 << 15))
-		DSSERR("\t\tDSI Protocol Violation\n");
+		dev_err(dsi->dev, "\t\tDSI Protocol Violation\n");
 }
 
 static u16 dsi_vc_flush_receive_data(struct dsi_data *dsi, int vc)
@@ -1969,23 +1960,23 @@ static u16 dsi_vc_flush_receive_data(struct dsi_data *dsi, int vc)
 		u32 val;
 		u8 dt;
 		val = dsi_read_reg(dsi, DSI_VC_SHORT_PACKET_HEADER(vc));
-		DSSERR("\trawval %#08x\n", val);
+		dev_err(dsi->dev, "\trawval %#08x\n", val);
 		dt = FLD_GET(val, 5, 0);
 		if (dt == MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT) {
 			u16 err = FLD_GET(val, 23, 8);
-			dsi_show_rx_ack_with_err(err);
+			dsi_show_rx_ack_with_err(dsi, err);
 		} else if (dt == MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE) {
-			DSSERR("\tDCS short response, 1 byte: %#x\n",
-					FLD_GET(val, 23, 8));
+			dev_err(dsi->dev, "\tDCS short response, 1 byte: %#x\n",
+				FLD_GET(val, 23, 8));
 		} else if (dt == MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE) {
-			DSSERR("\tDCS short response, 2 byte: %#x\n",
-					FLD_GET(val, 23, 8));
+			dev_err(dsi->dev, "\tDCS short response, 2 byte: %#x\n",
+				FLD_GET(val, 23, 8));
 		} else if (dt == MIPI_DSI_RX_DCS_LONG_READ_RESPONSE) {
-			DSSERR("\tDCS long response, len %d\n",
-					FLD_GET(val, 23, 8));
+			dev_err(dsi->dev, "\tDCS long response, len %d\n",
+				FLD_GET(val, 23, 8));
 			dsi_vc_flush_long_data(dsi, vc);
 		} else {
-			DSSERR("\tunknown datatype 0x%02x\n", dt);
+			dev_err(dsi->dev, "\tunknown datatype 0x%02x\n", dt);
 		}
 	}
 	return 0;
@@ -1994,13 +1985,13 @@ static u16 dsi_vc_flush_receive_data(struct dsi_data *dsi, int vc)
 static int dsi_vc_send_bta(struct dsi_data *dsi, int vc)
 {
 	if (dsi->debug_write || dsi->debug_read)
-		DSSDBG("dsi_vc_send_bta %d\n", vc);
+		dev_dbg(dsi->dev, "dsi_vc_send_bta %d\n", vc);
 
 	WARN_ON(!dsi_bus_is_locked(dsi));
 
 	/* RX_FIFO_NOT_EMPTY */
 	if (REG_GET(dsi, DSI_VC_CTRL(vc), 20, 20)) {
-		DSSERR("rx fifo not empty when sending BTA, dumping data:\n");
+		dev_err(dsi->dev, "rx fifo not empty when sending BTA, dumping data:\n");
 		dsi_vc_flush_receive_data(dsi, vc);
 	}
 
@@ -2035,14 +2026,14 @@ static int dsi_vc_send_bta_sync(struct omap_dss_device *dssdev, int vc)
 
 	if (wait_for_completion_timeout(&completion,
 				msecs_to_jiffies(500)) == 0) {
-		DSSERR("Failed to receive BTA\n");
+		dev_err(dsi->dev, "Failed to receive BTA\n");
 		r = -EIO;
 		goto err2;
 	}
 
 	err = dsi_get_errors(dsi);
 	if (err) {
-		DSSERR("Error while sending BTA: %x\n", err);
+		dev_err(dsi->dev, "Error while sending BTA: %x\n", err);
 		r = -EIO;
 		goto err2;
 	}
@@ -2096,11 +2087,11 @@ static int dsi_vc_send_long(struct dsi_data *dsi, int vc,
 	u8 b1, b2, b3, b4;
 
 	if (dsi->debug_write)
-		DSSDBG("dsi_vc_send_long, %zu bytes\n", msg->tx_len);
+		dev_dbg(dsi->dev, "%s, %zu bytes\n", __func__, msg->tx_len);
 
 	/* len + header */
 	if (dsi->vc[vc].tx_fifo_size * 32 * 4 < msg->tx_len + 4) {
-		DSSERR("unable to send long packet: packet too long.\n");
+		dev_err(dsi->dev, "unable to send long packet: packet too long.\n");
 		return -EINVAL;
 	}
 
@@ -2109,7 +2100,7 @@ static int dsi_vc_send_long(struct dsi_data *dsi, int vc,
 	p = msg->tx_buf;
 	for (i = 0; i < msg->tx_len >> 2; i++) {
 		if (dsi->debug_write)
-			DSSDBG("\tsending full packet %d\n", i);
+			dev_dbg(dsi->dev, "\tsending full packet %d\n", i);
 
 		b1 = *p++;
 		b2 = *p++;
@@ -2124,7 +2115,7 @@ static int dsi_vc_send_long(struct dsi_data *dsi, int vc,
 		b1 = 0; b2 = 0; b3 = 0;
 
 		if (dsi->debug_write)
-			DSSDBG("\tsending remainder bytes %d\n", i);
+			dev_dbg(dsi->dev, "\tsending remainder bytes %d\n", i);
 
 		switch (i) {
 		case 3:
@@ -2161,11 +2152,11 @@ static int dsi_vc_send_short(struct dsi_data *dsi, int vc,
 	WARN_ON(!dsi_bus_is_locked(dsi));
 
 	if (dsi->debug_write)
-		DSSDBG("dsi_vc_send_short(vc%d, dt %#x, b1 %#x, b2 %#x)\n",
-		       vc, msg->type, pkt.header[1], pkt.header[2]);
+		dev_dbg(dsi->dev, "%s(vc%d, dt %#x, b1 %#x, b2 %#x)\n",
+			__func__, vc, msg->type, pkt.header[1], pkt.header[2]);
 
 	if (FLD_GET(dsi_read_reg(dsi, DSI_VC_CTRL(vc)), 16, 16)) {
-		DSSERR("ERROR FIFO FULL, aborting transfer\n");
+		dev_err(dsi->dev, "ERROR FIFO FULL, aborting transfer\n");
 		return -EINVAL;
 	}
 
@@ -2190,33 +2181,50 @@ static int dsi_vc_send_null(struct dsi_data *dsi, int vc, int channel)
 static int dsi_vc_write_common(struct omap_dss_device *dssdev, int vc,
 			       const struct mipi_dsi_msg *msg)
 {
+	DECLARE_COMPLETION_ONSTACK(completion);
 	struct dsi_data *dsi = to_dsi_data(dssdev);
+	u32 err;
 	int r;
 
+	/* wait for IRQ for packet transmission confirmation */
+	r = dsi_register_isr_vc(dsi, vc, dsi_completion_handler,
+				&completion, DSI_VC_IRQ_PACKET_SENT);
+	if (r)
+		return r;
+
 	if (mipi_dsi_packet_format_is_short(msg->type))
 		r = dsi_vc_send_short(dsi, vc, msg);
 	else
 		r = dsi_vc_send_long(dsi, vc, msg);
 
-	if (r < 0)
+	if ((!r) && wait_for_completion_timeout(&completion,
+				msecs_to_jiffies(500)) == 0)
+		r = -EIO;
+
+	dsi_unregister_isr_vc(dsi, vc, dsi_completion_handler,
+			      &completion, DSI_VC_IRQ_PACKET_SENT);
+	if (r)
 		return r;
 
-	/*
-	 * TODO: we do not always have to do the BTA sync, for example
-	 * we can improve performance by setting the update window
-	 * information without sending BTA sync between the commands.
-	 * In that case we can return early.
-	 */
+	/* TODO: find out if more needs to be done for MIPI_DSI_MSG_REQ_ACK */
 
-	r = dsi_vc_send_bta_sync(dssdev, vc);
-	if (r) {
-		DSSERR("bta sync failed\n");
-		return r;
+	if (msg->flags & MIPI_DSI_MSG_REQ_ACK) {
+		r = dsi_vc_send_bta_sync(dssdev, vc);
+		if (r) {
+			dev_err(dsi->dev, "bta sync failed\n");
+			return r;
+		}
+	} else {
+		err = dsi_get_errors(dsi);
+		if (err) {
+			dev_err(dsi->dev, "Error while sending: %x\n", err);
+			return -EIO;
+		}
 	}
 
 	/* RX_FIFO_NOT_EMPTY */
 	if (REG_GET(dsi, DSI_VC_CTRL(vc), 20, 20)) {
-		DSSERR("rx fifo not empty after write, dumping data:\n");
+		dev_err(dsi->dev, "rx fifo not empty after write, dumping data:\n");
 		dsi_vc_flush_receive_data(dsi, vc);
 		return -EIO;
 	}
@@ -2233,18 +2241,18 @@ static int dsi_vc_read_rx_fifo(struct dsi_data *dsi, int vc, u8 *buf,
 
 	/* RX_FIFO_NOT_EMPTY */
 	if (REG_GET(dsi, DSI_VC_CTRL(vc), 20, 20) == 0) {
-		DSSERR("RX fifo empty when trying to read.\n");
+		dev_err(dsi->dev, "RX fifo empty when trying to read.\n");
 		r = -EIO;
 		goto err;
 	}
 
 	val = dsi_read_reg(dsi, DSI_VC_SHORT_PACKET_HEADER(vc));
 	if (dsi->debug_read)
-		DSSDBG("\theader: %08x\n", val);
+		dev_dbg(dsi->dev, "\theader: %08x\n", val);
 	dt = FLD_GET(val, 5, 0);
 	if (dt == MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT) {
 		u16 err = FLD_GET(val, 23, 8);
-		dsi_show_rx_ack_with_err(err);
+		dsi_show_rx_ack_with_err(dsi, err);
 		r = -EIO;
 		goto err;
 
@@ -2253,7 +2261,7 @@ static int dsi_vc_read_rx_fifo(struct dsi_data *dsi, int vc, u8 *buf,
 			MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE)) {
 		u8 data = FLD_GET(val, 15, 8);
 		if (dsi->debug_read)
-			DSSDBG("\t%s short response, 1 byte: %02x\n",
+			dev_dbg(dsi->dev, "\t%s short response, 1 byte: %02x\n",
 				type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" :
 				"DCS", data);
 
@@ -2270,7 +2278,7 @@ static int dsi_vc_read_rx_fifo(struct dsi_data *dsi, int vc, u8 *buf,
 			MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE)) {
 		u16 data = FLD_GET(val, 23, 8);
 		if (dsi->debug_read)
-			DSSDBG("\t%s short response, 2 byte: %04x\n",
+			dev_dbg(dsi->dev, "\t%s short response, 2 byte: %04x\n",
 				type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" :
 				"DCS", data);
 
@@ -2289,7 +2297,7 @@ static int dsi_vc_read_rx_fifo(struct dsi_data *dsi, int vc, u8 *buf,
 		int w;
 		int len = FLD_GET(val, 23, 8);
 		if (dsi->debug_read)
-			DSSDBG("\t%s long response, len %d\n",
+			dev_dbg(dsi->dev, "\t%s long response, len %d\n",
 				type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" :
 				"DCS", len);
 
@@ -2304,11 +2312,11 @@ static int dsi_vc_read_rx_fifo(struct dsi_data *dsi, int vc, u8 *buf,
 			val = dsi_read_reg(dsi,
 				DSI_VC_SHORT_PACKET_HEADER(vc));
 			if (dsi->debug_read)
-				DSSDBG("\t\t%02x %02x %02x %02x\n",
-						(val >> 0) & 0xff,
-						(val >> 8) & 0xff,
-						(val >> 16) & 0xff,
-						(val >> 24) & 0xff);
+				dev_dbg(dsi->dev, "\t\t%02x %02x %02x %02x\n",
+					(val >> 0) & 0xff,
+					(val >> 8) & 0xff,
+					(val >> 16) & 0xff,
+					(val >> 24) & 0xff);
 
 			for (b = 0; b < 4; ++b) {
 				if (w < len)
@@ -2320,13 +2328,13 @@ static int dsi_vc_read_rx_fifo(struct dsi_data *dsi, int vc, u8 *buf,
 
 		return len;
 	} else {
-		DSSERR("\tunknown datatype 0x%02x\n", dt);
+		dev_err(dsi->dev, "\tunknown datatype 0x%02x\n", dt);
 		r = -EIO;
 		goto err;
 	}
 
 err:
-	DSSERR("dsi_vc_read_rx_fifo(vc %d type %s) failed\n", vc,
+	dev_err(dsi->dev, "%s(vc %d type %s) failed\n", __func__, vc,
 		type == DSS_DSI_CONTENT_GENERIC ? "GENERIC" : "DCS");
 
 	return r;
@@ -2340,7 +2348,7 @@ static int dsi_vc_dcs_read(struct omap_dss_device *dssdev, int vc,
 	int r;
 
 	if (dsi->debug_read)
-		DSSDBG("%s(vc %d, cmd %x)\n", __func__, vc, cmd);
+		dev_dbg(dsi->dev, "%s(vc %d, cmd %x)\n", __func__, vc, cmd);
 
 	r = dsi_vc_send_short(dsi, vc, msg);
 	if (r)
@@ -2362,7 +2370,7 @@ static int dsi_vc_dcs_read(struct omap_dss_device *dssdev, int vc,
 
 	return 0;
 err:
-	DSSERR("%s(vc %d, cmd 0x%02x) failed\n", __func__,  vc, cmd);
+	dev_err(dsi->dev, "%s(vc %d, cmd 0x%02x) failed\n", __func__,  vc, cmd);
 	return r;
 }
 
@@ -2392,7 +2400,7 @@ static int dsi_vc_generic_read(struct omap_dss_device *dssdev, int vc,
 
 	return 0;
 err:
-	DSSERR("%s(vc %d, reqlen %zu) failed\n", __func__,  vc, msg->tx_len);
+	dev_err(dsi->dev, "%s(vc %d, reqlen %zu) failed\n", __func__,  vc, msg->tx_len);
 	return r;
 }
 
@@ -2417,10 +2425,10 @@ static void dsi_set_lp_rx_timeout(struct dsi_data *dsi, unsigned int ticks,
 
 	total_ticks = ticks * (x16 ? 16 : 1) * (x4 ? 4 : 1);
 
-	DSSDBG("LP_RX_TO %lu ticks (%#x%s%s) = %lu ns\n",
-			total_ticks,
-			ticks, x4 ? " x4" : "", x16 ? " x16" : "",
-			(total_ticks * 1000) / (fck / 1000 / 1000));
+	dev_dbg(dsi->dev, "LP_RX_TO %lu ticks (%#x%s%s) = %lu ns\n",
+		total_ticks,
+		ticks, x4 ? " x4" : "", x16 ? " x16" : "",
+		(total_ticks * 1000) / (fck / 1000 / 1000));
 }
 
 static void dsi_set_ta_timeout(struct dsi_data *dsi, unsigned int ticks,
@@ -2444,10 +2452,10 @@ static void dsi_set_ta_timeout(struct dsi_data *dsi, unsigned int ticks,
 
 	total_ticks = ticks * (x16 ? 16 : 1) * (x8 ? 8 : 1);
 
-	DSSDBG("TA_TO %lu ticks (%#x%s%s) = %lu ns\n",
-			total_ticks,
-			ticks, x8 ? " x8" : "", x16 ? " x16" : "",
-			(total_ticks * 1000) / (fck / 1000 / 1000));
+	dev_dbg(dsi->dev, "TA_TO %lu ticks (%#x%s%s) = %lu ns\n",
+		total_ticks,
+		ticks, x8 ? " x8" : "", x16 ? " x16" : "",
+		(total_ticks * 1000) / (fck / 1000 / 1000));
 }
 
 static void dsi_set_stop_state_counter(struct dsi_data *dsi, unsigned int ticks,
@@ -2471,10 +2479,10 @@ static void dsi_set_stop_state_counter(struct dsi_data *dsi, unsigned int ticks,
 
 	total_ticks = ticks * (x16 ? 16 : 1) * (x4 ? 4 : 1);
 
-	DSSDBG("STOP_STATE_COUNTER %lu ticks (%#x%s%s) = %lu ns\n",
-			total_ticks,
-			ticks, x4 ? " x4" : "", x16 ? " x16" : "",
-			(total_ticks * 1000) / (fck / 1000 / 1000));
+	dev_dbg(dsi->dev, "STOP_STATE_COUNTER %lu ticks (%#x%s%s) = %lu ns\n",
+		total_ticks,
+		ticks, x4 ? " x4" : "", x16 ? " x16" : "",
+		(total_ticks * 1000) / (fck / 1000 / 1000));
 }
 
 static void dsi_set_hs_tx_timeout(struct dsi_data *dsi, unsigned int ticks,
@@ -2498,10 +2506,10 @@ static void dsi_set_hs_tx_timeout(struct dsi_data *dsi, unsigned int ticks,
 
 	total_ticks = ticks * (x16 ? 16 : 1) * (x4 ? 4 : 1);
 
-	DSSDBG("HS_TX_TO %lu ticks (%#x%s%s) = %lu ns\n",
-			total_ticks,
-			ticks, x4 ? " x4" : "", x16 ? " x16" : "",
-			(total_ticks * 1000) / (fck / 1000 / 1000));
+	dev_dbg(dsi->dev, "HS_TX_TO %lu ticks (%#x%s%s) = %lu ns\n",
+		total_ticks,
+		ticks, x4 ? " x4" : "", x16 ? " x16" : "",
+		(total_ticks * 1000) / (fck / 1000 / 1000));
 }
 
 static void dsi_config_vp_num_line_buffers(struct dsi_data *dsi)
@@ -2721,11 +2729,11 @@ static void dsi_config_cmd_mode_interleaving(struct dsi_data *dsi)
 					lp_clk_div, dsi_fclk_hsdiv);
 	}
 
-	DSSDBG("DSI HS interleaving(TXBYTECLKHS) HSA %d, HFP %d, HBP %d, BLLP %d\n",
+	dev_dbg(dsi->dev, "DSI HS interleaving(TXBYTECLKHS) HSA %d, HFP %d, HBP %d, BLLP %d\n",
 		hsa_interleave_hs, hfp_interleave_hs, hbp_interleave_hs,
 		bl_interleave_hs);
 
-	DSSDBG("DSI LP interleaving(bytes) HSA %d, HFP %d, HBP %d, BLLP %d\n",
+	dev_dbg(dsi->dev, "DSI LP interleaving(bytes) HSA %d, HFP %d, HBP %d, BLLP %d\n",
 		hsa_interleave_lp, hfp_interleave_lp, hbp_interleave_lp,
 		bl_interleave_lp);
 
@@ -2861,9 +2869,9 @@ static void dsi_proto_timings(struct dsi_data *dsi)
 	r = FLD_MOD(r, ddr_clk_post, 7, 0);
 	dsi_write_reg(dsi, DSI_CLK_TIMING, r);
 
-	DSSDBG("ddr_clk_pre %u, ddr_clk_post %u\n",
-			ddr_clk_pre,
-			ddr_clk_post);
+	dev_dbg(dsi->dev, "ddr_clk_pre %u, ddr_clk_post %u\n",
+		ddr_clk_pre,
+		ddr_clk_post);
 
 	enter_hs_mode_lat = 1 + DIV_ROUND_UP(tlpx, 4) +
 		DIV_ROUND_UP(ths_prepare, 4) +
@@ -2875,8 +2883,8 @@ static void dsi_proto_timings(struct dsi_data *dsi)
 		FLD_VAL(exit_hs_mode_lat, 15, 0);
 	dsi_write_reg(dsi, DSI_VM_TIMING7, r);
 
-	DSSDBG("enter_hs_mode_lat %u, exit_hs_mode_lat %u\n",
-			enter_hs_mode_lat, exit_hs_mode_lat);
+	dev_dbg(dsi->dev, "enter_hs_mode_lat %u, exit_hs_mode_lat %u\n",
+		enter_hs_mode_lat, exit_hs_mode_lat);
 
 	 if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
 		/* TODO: Implement a video mode check_timings function */
@@ -2902,9 +2910,9 @@ static void dsi_proto_timings(struct dsi_data *dsi)
 		tl = DIV_ROUND_UP(4, ndl) + (hsync_end ? hsa : 0) + t_he + hfp +
 			DIV_ROUND_UP(width_bytes + 6, ndl) + hbp;
 
-		DSSDBG("HBP: %d, HFP: %d, HSA: %d, TL: %d TXBYTECLKHS\n", hbp,
+		dev_dbg(dsi->dev, "HBP: %d, HFP: %d, HSA: %d, TL: %d TXBYTECLKHS\n", hbp,
 			hfp, hsync_end ? hsa : 0, tl);
-		DSSDBG("VBP: %d, VFP: %d, VSA: %d, VACT: %d lines\n", vbp, vfp,
+		dev_dbg(dsi->dev, "VBP: %d, VFP: %d, VSA: %d, VACT: %d lines\n", vbp, vfp,
 			vsa, vm->vactive);
 
 		r = dsi_read_reg(dsi, DSI_VM_TIMING1);
@@ -3100,7 +3108,7 @@ static void dsi_update_screen_dispc(struct dsi_data *dsi)
 	u16 w = dsi->vm.hactive;
 	u16 h = dsi->vm.vactive;
 
-	DSSDBG("dsi_update_screen_dispc(%dx%d)\n", w, h);
+	dev_dbg(dsi->dev, "%s(%dx%d)\n", __func__, w, h);
 
 	bytespp	= mipi_dsi_pixel_format_to_bpp(dsi->pix_fmt) / 8;
 	bytespl = w * bytespp;
@@ -3195,7 +3203,7 @@ static void dsi_framedone_timeout_work_callback(struct work_struct *work)
 	 * on the HW is buggy, and would probably require resetting the whole
 	 * DSI */
 
-	DSSERR("Framedone not received for 250ms!\n");
+	dev_err(dsi->dev, "Framedone not received for 250ms!\n");
 
 	dsi_handle_framedone(dsi, -ETIMEDOUT);
 }
@@ -3211,7 +3219,7 @@ static void dsi_framedone_irq_callback(void *data)
 
 	cancel_delayed_work(&dsi->framedone_timeout_work);
 
-	DSSDBG("Framedone received!\n");
+	dev_dbg(dsi->dev, "Framedone received!\n");
 
 	dsi_handle_framedone(dsi, 0);
 }
@@ -3229,21 +3237,6 @@ static int _dsi_update(struct dsi_data *dsi)
 	return 0;
 }
 
-static int _dsi_send_nop(struct dsi_data *dsi, int vc, int channel)
-{
-	const u8 payload[] = { MIPI_DCS_NOP };
-	const struct mipi_dsi_msg msg = {
-		.channel = channel,
-		.type = MIPI_DSI_DCS_SHORT_WRITE,
-		.tx_len = 1,
-		.tx_buf = payload,
-	};
-
-	WARN_ON(!dsi_bus_is_locked(dsi));
-
-	return _omap_dsi_host_transfer(dsi, vc, &msg);
-}
-
 static int dsi_update_channel(struct omap_dss_device *dssdev, int vc)
 {
 	struct dsi_data *dsi = to_dsi_data(dssdev);
@@ -3261,16 +3254,16 @@ static int dsi_update_channel(struct omap_dss_device *dssdev, int vc)
 		goto err;
 	}
 
-	DSSDBG("dsi_update_channel: %d", vc);
+	dev_dbg(dsi->dev, "%s: %d", __func__, vc);
 
 	/*
-	 * Send NOP between the frames. If we don't send something here, the
+	 * Transition to LP here. If we don't send something here, the
 	 * updates stop working. This is probably related to DSI spec stating
 	 * that the DSI host should transition to LP at least once per frame.
 	 */
-	r = _dsi_send_nop(dsi, VC_CMD, dsi->dsidev->channel);
+	r = dsi_vc_send_bta_sync(dssdev, VC_CMD);
 	if (r < 0) {
-		DSSWARN("failed to send nop between frames: %d\n", r);
+		DSSWARN("failed to send bta sync between frames: %d\n", r);
 		goto err;
 	}
 
@@ -3311,7 +3304,7 @@ static int dsi_configure_dispc_clocks(struct dsi_data *dsi)
 
 	r = dispc_calc_clock_rates(dsi->dss->dispc, fck, &dispc_cinfo);
 	if (r) {
-		DSSERR("Failed to calc dispc clocks\n");
+		dev_err(dsi->dev, "Failed to calc dispc clocks\n");
 		return r;
 	}
 
@@ -3333,7 +3326,7 @@ static int dsi_init_dispc(struct dsi_data *dsi)
 		r = dss_mgr_register_framedone_handler(&dsi->output,
 				dsi_framedone_irq_callback, dsi);
 		if (r) {
-			DSSERR("can't register FRAMEDONE handler\n");
+			dev_err(dsi->dev, "can't register FRAMEDONE handler\n");
 			goto err;
 		}
 
@@ -3385,7 +3378,7 @@ static int dsi_configure_dsi_clocks(struct dsi_data *dsi)
 
 	r = dss_pll_set_config(&dsi->pll, &cinfo);
 	if (r) {
-		DSSERR("Failed to set dsi clocks\n");
+		dev_err(dsi->dev, "Failed to set dsi clocks\n");
 		return r;
 	}
 
@@ -3438,7 +3431,7 @@ static int dsi_init_dsi(struct dsi_data *dsi)
 				  dsi->module_id == 0 ?
 				  DSS_CLK_SRC_PLL1_2 : DSS_CLK_SRC_PLL2_2);
 
-	DSSDBG("PLL OK\n");
+	dev_dbg(dsi->dev, "PLL OK\n");
 
 	if (!dsi->vdds_dsi_enabled) {
 		r = regulator_enable(dsi->vdds_dsi_reg);
@@ -3528,7 +3521,7 @@ static void dsi_enable(struct dsi_data *dsi)
 	dsi_runtime_put(dsi);
 err_get_dsi:
 	mutex_unlock(&dsi->lock);
-	DSSDBG("dsi_enable FAILED\n");
+	dev_dbg(dsi->dev, "%s FAILED\n", __func__);
 }
 
 static void dsi_disable(struct dsi_data *dsi)
@@ -4097,7 +4090,7 @@ static int dsi_set_config(struct omap_dss_device *dssdev,
 
 	r = __dsi_calc_config(dsi, mode, &ctx);
 	if (r) {
-		DSSERR("failed to find suitable DSI clock settings\n");
+		dev_err(dsi->dev, "failed to find suitable DSI clock settings\n");
 		goto err;
 	}
 
@@ -4257,7 +4250,7 @@ static int dsi_get_clocks(struct dsi_data *dsi)
 
 	clk = devm_clk_get(dsi->dev, "fck");
 	if (IS_ERR(clk)) {
-		DSSERR("can't get fck\n");
+		dev_err(dsi->dev, "can't get fck\n");
 		return PTR_ERR(clk);
 	}
 
@@ -4363,12 +4356,12 @@ static int omap_dsi_host_attach(struct mipi_dsi_host *host,
 	int r;
 
 	if (dsi->dsidev) {
-		DSSERR("dsi client already attached\n");
+		dev_err(dsi->dev, "dsi client already attached\n");
 		return -EBUSY;
 	}
 
 	if (mipi_dsi_pixel_format_to_bpp(client->format) < 0) {
-		DSSERR("invalid pixel format\n");
+		dev_err(dsi->dev, "invalid pixel format\n");
 		return -EINVAL;
 	}
 
@@ -4529,7 +4522,7 @@ static int dsi_init_pll_data(struct dss_device *dss, struct dsi_data *dsi)
 
 	clk = devm_clk_get(dsi->dev, "sys_clk");
 	if (IS_ERR(clk)) {
-		DSSERR("can't get sys_clk\n");
+		dev_err(dsi->dev, "can't get sys_clk\n");
 		return PTR_ERR(clk);
 	}
 
@@ -4937,21 +4930,21 @@ static int dsi_probe(struct platform_device *pdev)
 
 	dsi->irq = platform_get_irq(pdev, 0);
 	if (dsi->irq < 0) {
-		DSSERR("platform_get_irq failed\n");
+		dev_err(dsi->dev, "platform_get_irq failed\n");
 		return -ENODEV;
 	}
 
 	r = devm_request_irq(dev, dsi->irq, omap_dsi_irq_handler,
 			     IRQF_SHARED, dev_name(dev), dsi);
 	if (r < 0) {
-		DSSERR("request_irq failed\n");
+		dev_err(dsi->dev, "request_irq failed\n");
 		return r;
 	}
 
 	dsi->vdds_dsi_reg = devm_regulator_get(dev, "vdd");
 	if (IS_ERR(dsi->vdds_dsi_reg)) {
 		if (PTR_ERR(dsi->vdds_dsi_reg) != -EPROBE_DEFER)
-			DSSERR("can't get DSI VDD regulator\n");
+			dev_err(dsi->dev, "can't get DSI VDD regulator\n");
 		return PTR_ERR(dsi->vdds_dsi_reg);
 	}
 
@@ -4966,7 +4959,7 @@ static int dsi_probe(struct platform_device *pdev)
 		d++;
 
 	if (d->address == 0) {
-		DSSERR("unsupported DSI module\n");
+		dev_err(dsi->dev, "unsupported DSI module\n");
 		return -ENODEV;
 	}
 
@@ -5016,7 +5009,7 @@ static int dsi_probe(struct platform_device *pdev)
 
 	r = dsi_probe_of(dsi);
 	if (r) {
-		DSSERR("Invalid DSI DT data\n");
+		dev_err(dsi->dev, "Invalid DSI DT data\n");
 		goto err_pm_disable;
 	}
 
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi.h b/drivers/gpu/drm/omapdrm/dss/hdmi.h
index c4a4e07f0b9984..a79d4590611f2f 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi.h
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi.h
@@ -15,6 +15,7 @@
 #include <sound/omap-hdmi-audio.h>
 #include <media/cec.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_connector.h>
 
 #include "omapdss.h"
 #include "dss.h"
@@ -368,6 +369,8 @@ struct omap_hdmi {
 	struct drm_bridge bridge;
 
 	struct platform_device *audio_pdev;
+	struct mutex audio_lock;
+
 	void (*audio_abort_cb)(struct device *dev);
 	int wp_idlemode;
 
@@ -380,6 +383,9 @@ struct omap_hdmi {
 	bool display_enabled;
 };
 
+void hdmi_audio_hpd_notify(struct omap_hdmi *hdmi,
+			   enum drm_connector_status status);
+
 #define drm_bridge_to_hdmi(b) container_of(b, struct omap_hdmi, bridge)
 
 #endif
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index 61cfd003569c6b..ea0d749265d1f9 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -16,6 +16,7 @@
 #include <linux/interrupt.h>
 #include <linux/mutex.h>
 #include <linux/delay.h>
+#include <linux/seq_file.h>
 #include <linux/string.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -433,8 +434,15 @@ static void hdmi4_bridge_hpd_notify(struct drm_bridge *bridge,
 {
 	struct omap_hdmi *hdmi = drm_bridge_to_hdmi(bridge);
 
-	if (status == connector_status_disconnected)
+	hdmi_audio_hpd_notify(hdmi, status);
+
+	if (status == connector_status_disconnected) {
+		if (hdmi_runtime_get(hdmi))
+			return;
+
 		hdmi4_cec_set_phys_addr(&hdmi->core, CEC_PHYS_ADDR_INVALID);
+		hdmi_runtime_put(hdmi);
+	}
 }
 
 static const struct drm_edid *hdmi4_bridge_edid_read(struct drm_bridge *bridge,
@@ -626,12 +634,17 @@ static int hdmi_audio_register(struct omap_hdmi *hdmi)
 		.ops = &hdmi_audio_ops,
 	};
 
+	guard(mutex)(&hdmi->audio_lock);
 	hdmi->audio_pdev = platform_device_register_data(
 		&hdmi->pdev->dev, "omap-hdmi-audio", PLATFORM_DEVID_AUTO,
 		&pdata, sizeof(pdata));
 
-	if (IS_ERR(hdmi->audio_pdev))
-		return PTR_ERR(hdmi->audio_pdev);
+	if (IS_ERR(hdmi->audio_pdev)) {
+		int err = PTR_ERR(hdmi->audio_pdev);
+
+		hdmi->audio_pdev = NULL;
+		return err;
+	}
 
 	return 0;
 }
@@ -688,8 +701,14 @@ static void hdmi4_unbind(struct device *dev, struct device *master, void *data)
 
 	dss_debugfs_remove_file(hdmi->debugfs);
 
-	if (hdmi->audio_pdev)
-		platform_device_unregister(hdmi->audio_pdev);
+	scoped_guard(mutex, &hdmi->audio_lock) {
+		if (hdmi->audio_pdev) {
+			struct platform_device *pdev = hdmi->audio_pdev;
+
+			hdmi->audio_pdev = NULL;
+			platform_device_unregister(pdev);
+		}
+	}
 
 	hdmi4_cec_uninit(&hdmi->core);
 	hdmi_pll_uninit(&hdmi->pll);
@@ -770,6 +789,7 @@ static int hdmi4_probe(struct platform_device *pdev)
 	dev_set_drvdata(&pdev->dev, hdmi);
 
 	mutex_init(&hdmi->lock);
+	mutex_init(&hdmi->audio_lock);
 	spin_lock_init(&hdmi->audio_playing_lock);
 
 	r = hdmi4_probe_of(hdmi);
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index 1ee3bbe205831e..216281a884bdc6 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -20,6 +20,7 @@
 #include <linux/interrupt.h>
 #include <linux/mutex.h>
 #include <linux/delay.h>
+#include <linux/seq_file.h>
 #include <linux/string.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -425,6 +426,15 @@ static void hdmi5_bridge_disable(struct drm_bridge *bridge,
 	mutex_unlock(&hdmi->lock);
 }
 
+static void hdmi5_bridge_hpd_notify(struct drm_bridge *bridge,
+				    struct drm_connector *connector,
+				    enum drm_connector_status status)
+{
+	struct omap_hdmi *hdmi = drm_bridge_to_hdmi(bridge);
+
+	hdmi_audio_hpd_notify(hdmi, status);
+}
+
 static const struct drm_edid *hdmi5_bridge_edid_read(struct drm_bridge *bridge,
 						     struct drm_connector *connector)
 {
@@ -475,6 +485,7 @@ static const struct drm_bridge_funcs hdmi5_bridge_funcs = {
 	.atomic_create_state = drm_atomic_helper_bridge_create_state,
 	.atomic_enable = hdmi5_bridge_enable,
 	.atomic_disable = hdmi5_bridge_disable,
+	.hpd_notify = hdmi5_bridge_hpd_notify,
 	.edid_read = hdmi5_bridge_edid_read,
 };
 
@@ -601,12 +612,18 @@ static int hdmi_audio_register(struct omap_hdmi *hdmi)
 		.ops = &hdmi_audio_ops,
 	};
 
-	hdmi->audio_pdev = platform_device_register_data(
-		&hdmi->pdev->dev, "omap-hdmi-audio", PLATFORM_DEVID_AUTO,
-		&pdata, sizeof(pdata));
+	scoped_guard(mutex, &hdmi->audio_lock) {
+		hdmi->audio_pdev = platform_device_register_data(
+			&hdmi->pdev->dev, "omap-hdmi-audio",
+			PLATFORM_DEVID_AUTO, &pdata, sizeof(pdata));
 
-	if (IS_ERR(hdmi->audio_pdev))
-		return PTR_ERR(hdmi->audio_pdev);
+		if (IS_ERR(hdmi->audio_pdev)) {
+			int err = PTR_ERR(hdmi->audio_pdev);
+
+			hdmi->audio_pdev = NULL;
+			return err;
+		}
+	}
 
 	hdmi_runtime_get(hdmi);
 	hdmi->wp_idlemode =
@@ -654,8 +671,14 @@ static void hdmi5_unbind(struct device *dev, struct device *master, void *data)
 
 	dss_debugfs_remove_file(hdmi->debugfs);
 
-	if (hdmi->audio_pdev)
-		platform_device_unregister(hdmi->audio_pdev);
+	scoped_guard(mutex, &hdmi->audio_lock) {
+		if (hdmi->audio_pdev) {
+			struct platform_device *pdev = hdmi->audio_pdev;
+
+			hdmi->audio_pdev = NULL;
+			platform_device_unregister(pdev);
+		}
+	}
 
 	hdmi_pll_uninit(&hdmi->pll);
 }
@@ -735,6 +758,7 @@ static int hdmi5_probe(struct platform_device *pdev)
 	dev_set_drvdata(&pdev->dev, hdmi);
 
 	mutex_init(&hdmi->lock);
+	mutex_init(&hdmi->audio_lock);
 	spin_lock_init(&hdmi->audio_playing_lock);
 
 	r = hdmi5_probe_of(hdmi);
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_common.c b/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
index 3ecde23ac60406..e9adc5775913c6 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
@@ -147,3 +147,17 @@ int hdmi_compute_acr(u32 pclk, u32 sample_freq, u32 *n, u32 *cts)
 
 	return 0;
 }
+
+void hdmi_audio_hpd_notify(struct omap_hdmi *hdmi,
+			   enum drm_connector_status status)
+{
+	guard(mutex)(&hdmi->audio_lock);
+
+	if (hdmi->audio_pdev) {
+		struct device *dev = &hdmi->audio_pdev->dev;
+		struct omap_hdmi_audio_pdata *ha = dev_get_platdata(dev);
+
+		if (ha->audio_hpd)
+			ha->audio_hpd(dev, status == connector_status_connected);
+	}
+}
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h
index 4c22c09c93d523..e3e685c931f14d 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss.h
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h
@@ -204,7 +204,6 @@ struct omap_dss_device {
 	struct dss_device *dss;
 	struct drm_bridge *bridge;
 	struct drm_bridge *next_bridge;
-	struct drm_panel *panel;
 
 	struct list_head list;
 
diff --git a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
index 6e9bc605ee22f2..36d37deac80263 100644
--- a/drivers/gpu/drm/omapdrm/dss/output.c
+++ b/drivers/gpu/drm/omapdrm/dss/output.c
@@ -12,7 +12,6 @@
 #include <linux/of_graph.h>
 
 #include <drm/drm_bridge.h>
-#include <drm/drm_panel.h>
 
 #include "dss.h"
 #include "omapdss.h"
@@ -20,40 +19,16 @@
 int omapdss_device_init_output(struct omap_dss_device *out,
 			       struct drm_bridge *local_bridge)
 {
-	struct device_node *remote_node;
+	struct drm_bridge *bridge;
 	int ret;
 
-	remote_node = of_graph_get_remote_node(out->dev->of_node,
-					       out->of_port, 0);
-	if (!remote_node) {
+	bridge = of_drm_get_bridge_by_endpoint(out->dev->of_node, out->of_port, 0);
+	if (IS_ERR(bridge)) {
 		dev_dbg(out->dev, "failed to find video sink\n");
 		return 0;
 	}
 
-	out->panel = of_drm_find_panel(remote_node);
-	if (IS_ERR(out->panel))
-		out->panel = NULL;
-
-	if (!out->panel)
-		out->bridge = of_drm_find_and_get_bridge(remote_node);
-
-	of_node_put(remote_node);
-
-	if (out->panel) {
-		struct drm_bridge *bridge;
-
-		bridge = drm_panel_bridge_add(out->panel);
-		drm_panel_put(out->panel);
-		if (IS_ERR(bridge)) {
-			dev_err(out->dev,
-				"unable to create panel bridge (%ld)\n",
-				PTR_ERR(bridge));
-			ret = PTR_ERR(bridge);
-			goto error;
-		}
-
-		out->bridge = drm_bridge_get(bridge);
-	}
+	out->bridge = bridge;
 
 	if (local_bridge) {
 		if (!out->bridge) {
@@ -65,11 +40,6 @@ int omapdss_device_init_output(struct omap_dss_device *out,
 		out->bridge = drm_bridge_get(local_bridge);
 	}
 
-	if (!out->bridge) {
-		ret = -EPROBE_DEFER;
-		goto error;
-	}
-
 	return 0;
 
 error:
@@ -79,10 +49,6 @@ int omapdss_device_init_output(struct omap_dss_device *out,
 
 void omapdss_device_cleanup_output(struct omap_dss_device *out)
 {
-	if (out->bridge && out->panel)
-		drm_panel_bridge_remove(out->next_bridge ?
-					out->next_bridge : out->bridge);
-
 	drm_bridge_put(out->next_bridge);
 	drm_bridge_put(out->bridge);
 }
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index c1b41e010a0045..a9f6a76ebb5e71 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -707,18 +707,17 @@ static int omap_crtc_atomic_get_property(struct drm_crtc *crtc,
 	return 0;
 }
 
-static void omap_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *omap_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct omap_crtc_state *state;
 
-	if (crtc->state)
-		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	kfree(crtc->state);
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
 
-	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
+	return &state->base;
 }
 
 static struct drm_crtc_state *
@@ -745,7 +744,7 @@ omap_crtc_duplicate_state(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_funcs omap_crtc_funcs = {
-	.reset = omap_crtc_reset,
+	.atomic_create_state = omap_crtc_create_state,
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = omap_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index 92d6a1f9c0a1a7..89a6d3b4aa6f7b 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -17,7 +17,6 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_file.h>
 #include <drm/drm_ioctl.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_prime.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index c024cdbbd87afe..e3f60046afcc85 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -403,18 +403,17 @@ void omap_plane_install_properties(struct drm_plane *plane,
 	drm_object_attach_property(obj, priv->zorder_prop, 0);
 }
 
-static void omap_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *omap_plane_create_state(struct drm_plane *plane)
 {
 	struct omap_plane_state *omap_state;
 
-	if (plane->state)
-		drm_atomic_helper_plane_destroy_state(plane, plane->state);
-
 	omap_state = kzalloc_obj(*omap_state);
 	if (!omap_state)
-		return;
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&omap_state->base, plane);
 
-	__drm_atomic_helper_plane_reset(plane, &omap_state->base);
+	return &omap_state->base;
 }
 
 static struct drm_plane_state *
@@ -491,7 +490,7 @@ static int omap_plane_atomic_get_property(struct drm_plane *plane,
 static const struct drm_plane_funcs omap_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = omap_plane_reset,
+	.atomic_create_state = omap_plane_create_state,
 	.destroy = omap_plane_destroy,
 	.atomic_duplicate_state = omap_plane_atomic_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index cbdf7b8f7f7afc..0a79a0558101bc 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config DRM_PANEL
-	bool
+	tristate
 	depends on DRM
 	help
 	  Panel registration and lookup framework.
@@ -298,6 +298,17 @@ config DRM_PANEL_ILITEK_ILI7807S
 
 	  If M is selected the module will be called panel-ilitek-ili7807s.
 
+config DRM_PANEL_ILITEK_ILI7836A
+	tristate "Ilitek ILI7836A-based panels"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y if you want to enable support for panels based on the
+	  Ilitek ILI7836A DDIC.
+
+	  If M is selected the module will be called panel-ilitek-ili7836a.
+
 config DRM_PANEL_ILITEK_IL9322
 	tristate "Ilitek ILI9322 320x240 QVGA panels"
 	depends on OF && SPI
@@ -664,6 +675,20 @@ config DRM_PANEL_NOVATEK_NT36523
 	  around the Novatek NT36523 display controller, such as some
 	  Boe panels used in Xiaomi Mi Pad 5 and 5 Pro tablets.
 
+config DRM_PANEL_NOVATEK_NT36532
+	tristate "Novatek NT36532-based MIPI-DSI panels"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_DISPLAY_DSC_HELPER
+	select DRM_DISPLAY_HELPER
+	select DRM_KMS_HELPER
+	help
+	  Say Y here if you want to enable support for the panels built
+	  around the Novatek NT36532 display controller, such as some
+	  CSOT panels used in OnePlus Pad 2 tablets. These panels are
+	  typically dual-DSI and may use DSC (Display Stream Compression).
+
 config DRM_PANEL_NOVATEK_NT36536
 	tristate "Novatek NT36536 panel driver"
 	depends on OF
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index 3b523cf3783306..f2c9c80a218f08 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_DRM_PANEL_HIMAX_HX83121A) += panel-himax-hx83121a.o
 obj-$(CONFIG_DRM_PANEL_HIMAX_HX8394) += panel-himax-hx8394.o
 obj-$(CONFIG_DRM_PANEL_HYDIS_HV101HD1) += panel-hydis-hv101hd1.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI7807S) += panel-ilitek-ili7807s.o
+obj-$(CONFIG_DRM_PANEL_ILITEK_ILI7836A) += panel-ilitek-ili7836a.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_IL9322) += panel-ilitek-ili9322.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9341) += panel-ilitek-ili9341.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9488) += panel-ilitek-ili9488.o
@@ -64,6 +65,7 @@ obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35532) += panel-novatek-nt35532.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35560) += panel-novatek-nt35560.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35950) += panel-novatek-nt35950.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36523) += panel-novatek-nt36523.o
+obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36532) += panel-novatek-nt36532.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36536) += panel-novatek-nt36536.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36672A) += panel-novatek-nt36672a.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36672E) += panel-novatek-nt36672e.o
diff --git a/drivers/gpu/drm/panel/panel-ebbg-ft8719.c b/drivers/gpu/drm/panel/panel-ebbg-ft8719.c
index fb9f9f42be4f26..1652de70900f19 100644
--- a/drivers/gpu/drm/panel/panel-ebbg-ft8719.c
+++ b/drivers/gpu/drm/panel/panel-ebbg-ft8719.c
@@ -54,8 +54,9 @@ static void ebbg_ft8719_reset(struct ebbg_ft8719 *ctx)
 	usleep_range(15000, 16000);
 }
 
-static int ebbg_ft8719_on(struct ebbg_ft8719 *ctx)
+static int ebbg_ft8719_enable(struct drm_panel *panel)
 {
+	struct ebbg_ft8719 *ctx = to_ebbg_ft8719(panel);
 	struct mipi_dsi_device *dsi = ctx->dsi;
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
@@ -72,19 +73,20 @@ static int ebbg_ft8719_on(struct ebbg_ft8719 *ctx)
 	return dsi_ctx.accum_err;
 }
 
-static int ebbg_ft8719_off(struct ebbg_ft8719 *ctx)
+static int ebbg_ft8719_disable(struct drm_panel *panel)
 {
+	struct ebbg_ft8719 *ctx = to_ebbg_ft8719(panel);
 	struct mipi_dsi_device *dsi = ctx->dsi;
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
-	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+	ctx->dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
 
 	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
 	mipi_dsi_msleep(&dsi_ctx, 90);
 
-	return dsi_ctx.accum_err;
+	return 0;
 }
 
 static int ebbg_ft8719_prepare(struct drm_panel *panel)
@@ -98,12 +100,6 @@ static int ebbg_ft8719_prepare(struct drm_panel *panel)
 
 	ebbg_ft8719_reset(ctx);
 
-	ret = ebbg_ft8719_on(ctx);
-	if (ret < 0) {
-		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
-		return ret;
-	}
-
 	return 0;
 }
 
@@ -111,7 +107,6 @@ static int ebbg_ft8719_unprepare(struct drm_panel *panel)
 {
 	struct ebbg_ft8719 *ctx = to_ebbg_ft8719(panel);
 
-	ebbg_ft8719_off(ctx);
 	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
 	regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
 
@@ -153,6 +148,8 @@ static int ebbg_ft8719_get_modes(struct drm_panel *panel,
 
 static const struct drm_panel_funcs ebbg_ft8719_panel_funcs = {
 	.prepare = ebbg_ft8719_prepare,
+	.enable = ebbg_ft8719_enable,
+	.disable = ebbg_ft8719_disable,
 	.unprepare = ebbg_ft8719_unprepare,
 	.get_modes = ebbg_ft8719_get_modes,
 };
@@ -198,6 +195,8 @@ static int ebbg_ft8719_probe(struct mipi_dsi_device *dsi)
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
 			  MIPI_DSI_CLOCK_NON_CONTINUOUS;
 
+	ctx->panel.prepare_prev_first = true;
+
 	ret = drm_panel_of_backlight(&ctx->panel);
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
@@ -244,4 +243,4 @@ module_mipi_dsi_driver(ebbg_ft8719_driver);
 
 MODULE_AUTHOR("Joel Selvaraj <jo@jsfamily.in>");
 MODULE_DESCRIPTION("DRM driver for EBBG FT8719 video dsi panel");
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index 2b455369ed7005..29f250e6e64dc9 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -2060,6 +2060,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0af2, &delay_200_500_e50_p2e80, "NE140QDM-NX2"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b09, &delay_200_500_e50_po2e200, "NV140FHM-NZ"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b1e, &delay_200_500_e80, "NE140QDM-N6A"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b21, &delay_200_500_e80, "QV116FHB-N81"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b34, &delay_200_500_e80_d50, "NV122WUM-N41"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b43, &delay_200_500_e200, "NV140FHM-T09"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"),
@@ -2083,6 +2084,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x3879, &delay_200_500_e80, "NT116WHM-N21"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x388b, &delay_200_500_e80, "NV116FH1-M31"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x388c, &delay_200_500_e80, "NV116FH1-M30"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x38ea, &delay_200_500_e50, "NV140FHM-T0A"),
 
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1130, &delay_200_500_e50, "N116BGE-EB2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1132, &delay_200_500_e80_d50, "N116BGE-EA2"),
@@ -2139,10 +2141,12 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x144b, &delay_200_500_e80, "MNE001BS1-4"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1457, &delay_80_500_e80_p2e200, "MNE007QS3-8"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1462, &delay_200_500_e50, "MNE007QS5-2"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1465, &delay_200_500_e50, "MNE007QB1-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1468, &delay_200_500_e50, "MNE007QB2-2"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x146e, &delay_80_500_e50_d50, "MNE007QB3-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x147c, &delay_200_500_e50_d100, "MNE007QB3-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1486, &delay_200_500_e50_d100, "MNE001BS6-2"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1487, &delay_200_500_e50, "MNE007QS3-F"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1519, &delay_200_500_e80_d50, "MNF601BS1-3"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1529, &delay_200_500_e80_d50, "MNF307QS3-2"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x153f, &delay_200_500_e200_d50, "MNF601BS4-1"),
@@ -2203,6 +2207,7 @@ static const struct edp_panel_entry edp_panels[] = {
 
 	EDP_PANEL_ENTRY('T', 'M', 'A', 0x0811, &delay_200_500_e80_d50, "TM140VDXP01-04"),
 	EDP_PANEL_ENTRY('T', 'M', 'A', 0x2094, &delay_200_500_e50_d100, "TL140VDMS03-01"),
+	EDP_PANEL_ENTRY('T', 'M', 'A', 0x2137, &delay_200_500_e50, "TM140VDXP15"),
 	EDP_PANEL_ENTRY('T', 'M', 'A', 0x2139, &delay_200_500_e80_pu100, "TM156VDXP25"),
 
 	{ /* sentinal */ }
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83102.c b/drivers/gpu/drm/panel/panel-himax-hx83102.c
index 6a0851ccf9bb0c..3ef052571ce09e 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83102.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83102.c
@@ -1284,9 +1284,7 @@ static int hx83102_panel_add(struct hx83102 *ctx)
 	ctx->base.funcs = &hx83102_drm_funcs;
 	ctx->base.dev = &ctx->dsi->dev;
 
-	drm_panel_add(&ctx->base);
-
-	return 0;
+	return devm_drm_panel_add(dev, &ctx->base);
 }
 
 static int hx83102_probe(struct mipi_dsi_device *dsi)
@@ -1318,24 +1316,7 @@ static int hx83102_probe(struct mipi_dsi_device *dsi)
 
 	mipi_dsi_set_drvdata(dsi, ctx);
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret)
-		drm_panel_remove(&ctx->base);
-
-	return ret;
-}
-
-static void hx83102_remove(struct mipi_dsi_device *dsi)
-{
-	struct hx83102 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
-
-	if (ctx->base.dev)
-		drm_panel_remove(&ctx->base);
+	return devm_mipi_dsi_attach(&dsi->dev, dsi);
 }
 
 static const struct of_device_id hx83102_of_match[] = {
@@ -1369,7 +1350,6 @@ MODULE_DEVICE_TABLE(of, hx83102_of_match);
 
 static struct mipi_dsi_driver hx83102_driver = {
 	.probe	= hx83102_probe,
-	.remove = hx83102_remove,
 	.driver = {
 		.name = "panel-himax-hx83102",
 		.of_match_table = hx83102_of_match,
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83112a.c b/drivers/gpu/drm/panel/panel-himax-hx83112a.c
index 142cb1cc067add..36ebfad7a3e9dc 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83112a.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83112a.c
@@ -303,28 +303,15 @@ static int hx83112a_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
 
-	drm_panel_add(&ctx->panel);
-
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
-		drm_panel_remove(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
 		return ret;
-	}
 
-	return 0;
-}
-
-static void hx83112a_remove(struct mipi_dsi_device *dsi)
-{
-	struct hx83112a_panel *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
+		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
 
-	drm_panel_remove(&ctx->panel);
+	return 0;
 }
 
 static const struct of_device_id hx83112a_of_match[] = {
@@ -335,7 +322,6 @@ MODULE_DEVICE_TABLE(of, hx83112a_of_match);
 
 static struct mipi_dsi_driver hx83112a_driver = {
 	.probe = hx83112a_probe,
-	.remove = hx83112a_remove,
 	.driver = {
 		.name = "panel-himax-hx83112a",
 		.of_match_table = hx83112a_of_match,
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83112b.c b/drivers/gpu/drm/panel/panel-himax-hx83112b.c
index 41f21f8c1373eb..569ed2dec87cb7 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83112b.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83112b.c
@@ -386,29 +386,17 @@ static int hx83112b_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		drm_panel_remove(&ctx->panel);
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0)
 		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
-	}
 
 	return 0;
 }
 
-static void hx83112b_remove(struct mipi_dsi_device *dsi)
-{
-	struct hx83112b_panel *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id hx83112b_of_match[] = {
 	{ .compatible = "djn,98-03057-6598b-i" },
 	{ /* sentinel */ }
@@ -417,7 +405,6 @@ MODULE_DEVICE_TABLE(of, hx83112b_of_match);
 
 static struct mipi_dsi_driver hx83112b_driver = {
 	.probe = hx83112b_probe,
-	.remove = hx83112b_remove,
 	.driver = {
 		.name = "panel-himax-hx83112b",
 		.of_match_table = hx83112b_of_match,
diff --git a/drivers/gpu/drm/panel/panel-himax-hx8394.c b/drivers/gpu/drm/panel/panel-himax-hx8394.c
index 09124610fdc867..4180d4aadf5ebf 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx8394.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx8394.c
@@ -1023,14 +1023,13 @@ static int hx8394_probe(struct mipi_dsi_device *dsi)
 
 	ctx->panel.prepare_prev_first = true;
 
-	drm_panel_add(&ctx->panel);
-
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		dev_err_probe(dev, ret, "mipi_dsi_attach failed\n");
-		drm_panel_remove(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
 		return ret;
-	}
+
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "mipi_dsi_attach failed\n");
 
 	dev_dbg(dev, "%ux%u@%u %ubpp dsi %udl - ready\n",
 		ctx->desc->mode->hdisplay, ctx->desc->mode->vdisplay,
@@ -1040,18 +1039,6 @@ static int hx8394_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static void hx8394_remove(struct mipi_dsi_device *dsi)
-{
-	struct hx8394 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id hx8394_of_match[] = {
 	{ .compatible = "hannstar,hsd060bhw4", .data = &hsd060bhw4_desc },
 	{ .compatible = "huiling,hl055fhav028c", .data = &hl055fhav028c_desc },
@@ -1065,7 +1052,6 @@ MODULE_DEVICE_TABLE(of, hx8394_of_match);
 
 static struct mipi_dsi_driver hx8394_driver = {
 	.probe	= hx8394_probe,
-	.remove = hx8394_remove,
 	.driver = {
 		.name = DRV_NAME,
 		.of_match_table = hx8394_of_match,
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili7836a.c b/drivers/gpu/drm/panel/panel-ilitek-ili7836a.c
new file mode 100644
index 00000000000000..f6331129dadf2c
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili7836a.c
@@ -0,0 +1,297 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+#include <video/mipi_display.h>
+
+static const struct regulator_bulk_data ili7836a_supplies[] = {
+	{ .supply = "vddio", },
+	{ .supply = "avdd", },
+};
+
+struct ili7836a_panel {
+	struct drm_panel panel;
+	struct drm_connector *connector;
+	struct mipi_dsi_device *dsi;
+	struct regulator_bulk_data *supplies;
+	struct gpio_desc *reset_gpio;
+	struct ili7836a_desc *desc;
+	enum drm_panel_orientation orientation;
+};
+
+struct ili7836a_desc {
+	unsigned int width_mm;
+	unsigned int height_mm;
+	unsigned int bpc;
+
+	const struct drm_display_mode *modes;
+	unsigned int num_modes;
+};
+
+static inline struct ili7836a_panel *to_ili7836a_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct ili7836a_panel, panel);
+}
+
+static void ili7836a_reset(struct ili7836a_panel *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	msleep(20);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	msleep(20);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	msleep(20);
+}
+
+static int ili7836a_on(struct ili7836a_panel *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	ctx->dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x5a, 0xa5, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xc8, 0x62);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x5a, 0xa5, 0x21);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xa4, 0x38);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x5a, 0xa5, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x85, 0x15);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x5a, 0xa5, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x53, 0x20);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 20);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x5a, 0xa5, 0x22);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xe1, 0x01);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x5a, 0xa5, 0x00);
+
+	return dsi_ctx.accum_err;
+}
+
+static int ili7836a_disable(struct drm_panel *panel)
+{
+	struct ili7836a_panel *ctx = to_ili7836a_panel(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	ctx->dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 20);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	return dsi_ctx.accum_err;
+}
+
+static int ili7836a_prepare(struct drm_panel *panel)
+{
+	struct ili7836a_panel *ctx = to_ili7836a_panel(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ili7836a_supplies), ctx->supplies);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	ili7836a_reset(ctx);
+
+	ret = ili7836a_on(ctx);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize panel: %d\n", ret);
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+		regulator_bulk_disable(ARRAY_SIZE(ili7836a_supplies), ctx->supplies);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ili7836a_unprepare(struct drm_panel *panel)
+{
+	struct ili7836a_panel *ctx = to_ili7836a_panel(panel);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(ili7836a_supplies), ctx->supplies);
+
+	return 0;
+}
+
+static const struct drm_display_mode nova_modes[] = {
+	{
+		/* 120Hz */
+		.clock = (1280 + 12 + 2 + 8) * (960 + 12 + 2 + 24) * 120 / 1000,
+		.hdisplay = 1280,
+		.hsync_start = 1280 + 12,
+		.hsync_end = 1280 + 12 + 2,
+		.htotal = 1280 + 12 + 2 + 8,
+		.vdisplay = 960,
+		.vsync_start = 960 + 12,
+		.vsync_end = 960 + 12 + 2,
+		.vtotal = 960 + 12 + 2 + 24,
+	}
+};
+
+static struct ili7836a_desc nova_desc = {
+	.modes = nova_modes,
+	.num_modes = ARRAY_SIZE(nova_modes),
+	.width_mm = 91,
+	.height_mm = 68,
+	.bpc = 8,
+};
+
+static int ili7836a_get_modes(struct drm_panel *panel,
+					struct drm_connector *connector)
+{
+	struct ili7836a_panel *ctx = to_ili7836a_panel(panel);
+
+	return drm_connector_helper_get_modes_fixed(connector, ctx->desc->modes);
+}
+
+static enum drm_panel_orientation ili7836a_get_orientation(struct drm_panel *panel)
+{
+	struct ili7836a_panel *ctx = to_ili7836a_panel(panel);
+
+	return ctx->orientation;
+}
+
+static const struct drm_panel_funcs ili7836a_panel_funcs = {
+	.prepare = ili7836a_prepare,
+	.unprepare = ili7836a_unprepare,
+	.disable = ili7836a_disable,
+	.get_modes = ili7836a_get_modes,
+	.get_orientation = ili7836a_get_orientation,
+};
+
+static int ili7836a_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_brightness_large(dsi, brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return 0;
+}
+
+static const struct backlight_ops ili7836a_bl_ops = {
+	.update_status = ili7836a_bl_update_status,
+};
+
+static struct backlight_device *
+ili7836a_create_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct backlight_properties props = {
+		.type = BACKLIGHT_RAW,
+		.brightness = 3445,
+		.max_brightness = 3445,
+	};
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &ili7836a_bl_ops, &props);
+}
+
+static int ili7836a_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct ili7836a_panel *ctx;
+	int ret;
+
+	ctx = devm_drm_panel_alloc(dev, __typeof(*ctx), panel,
+				   &ili7836a_panel_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ctx->desc = (struct ili7836a_desc *)of_device_get_match_data(dev);
+	if (!ctx->desc)
+		return -ENODEV;
+
+	ret = devm_regulator_bulk_get_const(dev,
+					    ARRAY_SIZE(ili7836a_supplies),
+					    ili7836a_supplies,
+					    &ctx->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get regulators\n");
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	ret = drm_of_get_panel_orientation(dev->of_node, &ctx->orientation);
+	if (ret < 0) {
+		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, ret);
+		return ret;
+	}
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_HSE |
+			  MIPI_DSI_MODE_NO_EOT_PACKET | MIPI_DSI_CLOCK_NON_CONTINUOUS;
+
+	ctx->panel.prepare_prev_first = true;
+
+	ctx->panel.backlight = ili7836a_create_backlight(dsi);
+	if (IS_ERR(ctx->panel.backlight))
+		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
+				     "Failed to create backlight\n");
+
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
+
+	return devm_mipi_dsi_attach(dev, dsi);
+}
+
+static const struct of_device_id ili7836a_of_match[] = {
+	{ .compatible = "retroidpocket,nova-panel", .data = &nova_desc },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ili7836a_of_match);
+
+static struct mipi_dsi_driver ili7836a_driver = {
+	.probe = ili7836a_probe,
+	.driver = {
+		.name = "panel-ili7836a-amoled",
+		.of_match_table = ili7836a_of_match,
+	},
+};
+module_mipi_dsi_driver(ili7836a_driver);
+
+MODULE_DESCRIPTION("DRM driver for ILI7836A DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9805.c b/drivers/gpu/drm/panel/panel-ilitek-ili9805.c
index 5610a225f77b20..976305991b7fd4 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9805.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9805.c
@@ -305,31 +305,19 @@ static int ili9805_dsi_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return ret;
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(&dsi->dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(&dsi->dev, dsi);
 	if (ret < 0) {
 		dev_err(&dsi->dev, "mipi_dsi_attach failed: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void ili9805_dsi_remove(struct mipi_dsi_device *dsi)
-{
-	struct ili9805 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n",
-			ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct ili9805_desc gpm1780a0_desc = {
 	.init = gpm1780a0_init,
 	.mode = &gpm1780a0_timing,
@@ -353,7 +341,6 @@ MODULE_DEVICE_TABLE(of, ili9805_of_match);
 
 static struct mipi_dsi_driver ili9805_dsi_driver = {
 	.probe		= ili9805_dsi_probe,
-	.remove		= ili9805_dsi_remove,
 	.driver = {
 		.name		= "ili9805-dsi",
 		.of_match_table	= ili9805_of_match,
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c
index 53e25d1086db89..4645a8cc4a8407 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c
@@ -8,6 +8,7 @@
 #include <drm/drm_panel.h>
 
 #include <linux/delay.h>
+#include <linux/device.h>
 #include <linux/export.h>
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
@@ -112,21 +113,11 @@ int ili9806e_probe(struct device *dev, void *transport,
 	if (set_prepare_prev_first)
 		ctx->panel.prepare_prev_first = true;
 
-	drm_panel_add(&ctx->panel);
-
-	return 0;
+	return devm_drm_panel_add(dev, &ctx->panel);
 
 }
 EXPORT_SYMBOL_GPL(ili9806e_probe);
 
-void ili9806e_remove(struct device *dev)
-{
-	struct ili9806e *ctx = dev_get_drvdata(dev);
-
-	drm_panel_remove(&ctx->panel);
-}
-EXPORT_SYMBOL_GPL(ili9806e_remove);
-
 MODULE_AUTHOR("Dario Binacchi <dario.binacchi@amarulasolutions.com>");
 MODULE_AUTHOR("Gunnar Dibbern <gunnar.dibbern@lht.dlh.de>");
 MODULE_AUTHOR("Michael Walle <mwalle@kernel.org>");
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.h b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.h
index dddece62cf4283..fded4b6793e732 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.h
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.h
@@ -10,6 +10,5 @@ int ili9806e_power_on(struct device *dev);
 int ili9806e_probe(struct device *dev, void *transport,
 		   const struct drm_panel_funcs *funcs,
 		   int connector_type);
-void ili9806e_remove(struct device *dev);
 
 #endif /* _PANEL_ILITEK_ILI9806E_CORE_H */
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c
index 0fb06c42554512..89915d58f69d8a 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-dsi.c
@@ -141,20 +141,7 @@ static int ili9806e_dsi_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return ret;
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
-		ili9806e_remove(dev);
-		return ret;
-	}
-
-	return 0;
-}
-
-static void ili9806e_dsi_remove(struct mipi_dsi_device *dsi)
-{
-	mipi_dsi_detach(dsi);
-	ili9806e_remove(&dsi->dev);
+	return devm_mipi_dsi_attach(dev, dsi);
 }
 
 static void com35h3p70ulc_init(struct mipi_dsi_multi_context *ctx)
@@ -490,7 +477,6 @@ static struct mipi_dsi_driver ili9806e_dsi_driver = {
 		.of_match_table = ili9806e_dsi_of_match,
 	},
 	.probe = ili9806e_dsi_probe,
-	.remove = ili9806e_dsi_remove,
 };
 module_mipi_dsi_driver(ili9806e_dsi_driver);
 
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-spi.c b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-spi.c
index 9d10b0d28f5221..187bcb084dd98a 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-spi.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-spi.c
@@ -124,11 +124,6 @@ static int ili9806e_spi_probe(struct spi_device *spi)
 			      DRM_MODE_CONNECTOR_DPI);
 }
 
-static void ili9806e_spi_remove(struct spi_device *spi)
-{
-	ili9806e_remove(&spi->dev);
-}
-
 static void rk050hr345_ct106a_init(struct ili9806e_spi_panel *ctx)
 {
 	struct mipi_dbi *dbi = &ctx->dbi;
@@ -313,7 +308,6 @@ static struct spi_driver ili9806e_spi_driver = {
 		.of_match_table = ili9806e_spi_of_match,
 	},
 	.probe = ili9806e_spi_probe,
-	.remove = ili9806e_spi_remove,
 	.id_table = ili9806e_spi_ids,
 };
 module_spi_driver(ili9806e_spi_driver);
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
index c3d5ab67121d95..7bf38e96678aca 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
@@ -2594,21 +2594,15 @@ static int ili9881c_dsi_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return ret;
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(&dsi->dev, &ctx->panel);
+	if (ret)
+		return ret;
 
 	dsi->mode_flags = ctx->desc->mode_flags;
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->lanes = ctx->desc->lanes;
 
-	return mipi_dsi_attach(dsi);
-}
-
-static void ili9881c_dsi_remove(struct mipi_dsi_device *dsi)
-{
-	struct ili9881c *ctx = mipi_dsi_get_drvdata(dsi);
-
-	mipi_dsi_detach(dsi);
-	drm_panel_remove(&ctx->panel);
+	return devm_mipi_dsi_attach(&dsi->dev, dsi);
 }
 
 static const struct ili9881c_desc lhr050h41_desc = {
@@ -2722,7 +2716,6 @@ MODULE_DEVICE_TABLE(of, ili9881c_of_match);
 
 static struct mipi_dsi_driver ili9881c_dsi_driver = {
 	.probe		= ili9881c_dsi_probe,
-	.remove		= ili9881c_dsi_remove,
 	.driver = {
 		.name		= "ili9881c-dsi",
 		.of_match_table	= ili9881c_of_match,
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c b/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
index 6d07fe901357eb..bea5ca70c775dd 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9882t.c
@@ -133,6 +133,207 @@ static const struct drm_dsc_config tianma_il79900a_dsc = {
 	.flatness_min_qp = 3,
 };
 
+static int rpi_10inch_init(struct ili9882t *ili)
+{
+	struct mipi_dsi_multi_context ctx = { .dsi = ili->dsi };
+
+	mipi_dsi_usleep_range(&ctx, 5000, 5100);
+
+	il79900a_switch_page(&ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x00, 0x42);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x01, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x02, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x03, 0x00);
+
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x04, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x05, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x06, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x07, 0x00);
+
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x08, 0x80);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x09, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0a, 0x71);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0e, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0d, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0f, 0x00);
+
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x30, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x31, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x32, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x33, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x34, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x35, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x36, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x37, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x38, 0x29);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x39, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x3A, 0x13);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x3B, 0x15);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x3C, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x3D, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x3E, 0x0D);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x3F, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x40, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x41, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x42, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x43, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x44, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x45, 0x02);
+
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x48, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x49, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x4A, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x4B, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x4C, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x4D, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x4E, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x4F, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x50, 0x29);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x51, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x52, 0x12);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x53, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x54, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x55, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x56, 0x0C);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x57, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x58, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x59, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x5A, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x5B, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x5C, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x5D, 0x02);
+
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x2C, 0xB0);
+
+	il79900a_switch_page(&ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x07, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x06, 0xD2);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x53, 0x1F);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x5E, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x84, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x29, 0x3A);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x2A, 0x3B);
+
+	il79900a_switch_page(&ctx, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x20, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x21, 0x3C);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x22, 0xFA);
+
+	il79900a_switch_page(&ctx, 0x12);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x87, 0x2C);
+
+	il79900a_switch_page(&ctx, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x61, 0x8F);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x6D, 0xBB);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x5A, 0x7C);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x67, 0x7C);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x54, 0x92);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x55, 0x92);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x40, 0xD2);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x46, 0x70);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x30, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x7A, 0x89);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xA4, 0x0B);
+
+	il79900a_switch_page(&ctx, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xC0, 0x80);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xC1, 0x7);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xCA, 0x58);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xCB, 0x2);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xCE, 0x58);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xCF, 0x2);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x10, 0x0);
+
+	il79900a_switch_page(&ctx, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x76, 0x55);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x78, 0x68);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x77, 0x0D);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x79, 0x60);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x7A, 0x10);
+
+	il79900a_switch_page(&ctx, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&ctx,
+				     0xE0, 0x00, 0x1E, 0x3F,
+				     0x68, 0x7D, 0x72, 0x86,
+				     0x89, 0x92, 0x8E, 0xA5,
+				     0xA8, 0xAA, 0xBD, 0xB8,
+				     0xBE, 0xC5, 0xD6, 0xCD,
+				     0xC3, 0xDA, 0xEB, 0xED);
+	mipi_dsi_dcs_write_seq_multi(&ctx,
+				     0xE1, 0x00, 0x1E, 0x3F,
+				     0x68, 0x7D, 0x72, 0x86,
+				     0x89, 0x92, 0x8E, 0xA5,
+				     0xA8, 0xAA, 0xBD, 0xB8,
+				     0xBE, 0xC5, 0xD6, 0xCD,
+				     0xC3, 0xDA, 0xEB, 0xED);
+
+	il79900a_switch_page(&ctx, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x85, 0xCC);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x87, 0x91);
+
+	il79900a_switch_page(&ctx, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x11, 0x3);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x12, 0x34);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x10, 0x1);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x13, 0x4);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x14, 0xBB);
+
+	il79900a_switch_page(&ctx, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x94, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x95, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x96, 0x0B);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x97, 0x0B);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x98, 0xC6);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x99, 0xC6);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xA6, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xA7, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xA8, 0x5);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xA9, 0x5);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xAA, 0xBA);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xAB, 0xBA);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xBD, 0x92);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0xBE, 0xE0);
+
+	il79900a_switch_page(&ctx, 0x0e);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0, 0x80);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x14, 0x3);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x12, 0x0);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x13, 0x4);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x11, 0x1);
+
+	il79900a_switch_page(&ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x80, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x82, 0x11);
+	il79900a_switch_page(&ctx, 0x12);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x82, 0x2C);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x85, 0x2C);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x87, 0x2C);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x88, 0xEC);
+	il79900a_switch_page(&ctx, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x4, 0x1);
+	il79900a_switch_page(&ctx, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0C, 0x80);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x3C, 0x80);
+
+	il79900a_switch_page(&ctx, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0E, 0x0);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x0E, 0x3F);
+	mipi_dsi_dcs_write_seq_multi(&ctx, 0x10, 0x0);
+
+	il79900a_switch_page(&ctx, 0x00);
+	mipi_dsi_dcs_set_tear_on_multi(&ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+	mipi_dsi_dcs_exit_sleep_mode_multi(&ctx);
+
+	mipi_dsi_msleep(&ctx, 120);
+
+	mipi_dsi_dcs_set_display_on_multi(&ctx);
+
+	mipi_dsi_msleep(&ctx, 20);
+
+	return ctx.accum_err;
+};
+
 static int starry_ili9882t_init(struct ili9882t *ili)
 {
 	struct mipi_dsi_multi_context ctx = { .dsi = ili->dsi };
@@ -670,6 +871,19 @@ static int ili9882t_enable(struct drm_panel *panel)
 	return 0;
 }
 
+static const struct drm_display_mode rpi_10inch_default_mode = {
+	.clock = 166666,
+	.hdisplay = 1200,
+	.hsync_start = 1200 + 28,
+	.hsync_end = 1200 + 28 + 10,
+	.htotal = 1200 + 28 + 10 + 30,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 238,
+	.vsync_end = 1920 + 238 + 4,
+	.vtotal = 1920 + 238 + 4 + 28,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
 static const struct drm_display_mode starry_ili9882t_default_mode = {
 	.clock = 165280,
 	.hdisplay = 1200,
@@ -696,6 +910,20 @@ static const struct drm_display_mode tianma_il79900a_default_mode = {
 	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
 };
 
+static const struct panel_desc rpi_10inch_desc = {
+	.modes = &rpi_10inch_default_mode,
+	.bpc = 8,
+	.size = {
+		.width_mm = 135,
+		.height_mm = 216,
+	},
+	.lanes = 4,
+	.format = MIPI_DSI_FMT_RGB888,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+		      MIPI_DSI_MODE_LPM,
+	.init = rpi_10inch_init,
+};
+
 static const struct panel_desc starry_ili9882t_desc = {
 	.modes = &starry_ili9882t_default_mode,
 	.bpc = 8,
@@ -787,7 +1015,7 @@ static int ili9882t_add(struct ili9882t *ili)
 	if (IS_ERR(ili->pp1800))
 		return PTR_ERR(ili->pp1800);
 
-	ili->enable_gpio = devm_gpiod_get(dev, "enable", GPIOD_OUT_LOW);
+	ili->enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_LOW);
 	if (IS_ERR(ili->enable_gpio)) {
 		dev_err(dev, "cannot get enable-gpios %ld\n",
 			PTR_ERR(ili->enable_gpio));
@@ -808,10 +1036,9 @@ static int ili9882t_add(struct ili9882t *ili)
 
 	ili->base.funcs = &ili9882t_funcs;
 	ili->base.dev = &ili->dsi->dev;
+	ili->base.prepare_prev_first = true;
 
-	drm_panel_add(&ili->base);
-
-	return 0;
+	return devm_drm_panel_add(dev, &ili->base);
 }
 
 static int ili9882t_probe(struct mipi_dsi_device *dsi)
@@ -844,27 +1071,13 @@ static int ili9882t_probe(struct mipi_dsi_device *dsi)
 
 	mipi_dsi_set_drvdata(dsi, ili);
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret)
-		drm_panel_remove(&ili->base);
-
-	return ret;
-}
-
-static void ili9882t_remove(struct mipi_dsi_device *dsi)
-{
-	struct ili9882t *ili = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
-
-	if (ili->base.dev)
-		drm_panel_remove(&ili->base);
+	return devm_mipi_dsi_attach(&dsi->dev, dsi);
 }
 
 static const struct of_device_id ili9882t_of_match[] = {
+	{ .compatible = "raspberrypi,dsi-10inch",
+	  .data = &rpi_10inch_desc
+	},
 	{ .compatible = "starry,ili9882t",
 	  .data = &starry_ili9882t_desc
 	},
@@ -881,7 +1094,6 @@ static struct mipi_dsi_driver ili9882t_driver = {
 		.of_match_table = ili9882t_of_match,
 	},
 	.probe = ili9882t_probe,
-	.remove = ili9882t_remove,
 };
 module_mipi_dsi_driver(ili9882t_driver);
 
diff --git a/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c b/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
index cbe354b51bce35..caf18d309849d8 100644
--- a/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
+++ b/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
@@ -200,9 +200,11 @@ static int jdi_fhd_r63452_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
 		return ret;
@@ -211,18 +213,6 @@ static int jdi_fhd_r63452_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static void jdi_fhd_r63452_remove(struct mipi_dsi_device *dsi)
-{
-	struct jdi_fhd_r63452 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id jdi_fhd_r63452_of_match[] = {
 	{ .compatible = "jdi,fhd-r63452" },
 	{ /* sentinel */ }
@@ -231,7 +221,6 @@ MODULE_DEVICE_TABLE(of, jdi_fhd_r63452_of_match);
 
 static struct mipi_dsi_driver jdi_fhd_r63452_driver = {
 	.probe = jdi_fhd_r63452_probe,
-	.remove = jdi_fhd_r63452_remove,
 	.driver = {
 		.name = "panel-jdi-fhd-r63452",
 		.of_match_table = jdi_fhd_r63452_of_match,
diff --git a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
index 01bd748aecec7a..e91b41743becf8 100644
--- a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
+++ b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
@@ -345,15 +345,7 @@ static int jdi_panel_add(struct jdi_panel *jdi)
 		return dev_err_probe(dev, PTR_ERR(jdi->backlight),
 				     "failed to register backlight %d\n", ret);
 
-	drm_panel_add(&jdi->base);
-
-	return 0;
-}
-
-static void jdi_panel_del(struct jdi_panel *jdi)
-{
-	if (jdi->base.dev)
-		drm_panel_remove(&jdi->base);
+	return devm_drm_panel_add(dev, &jdi->base);
 }
 
 static int jdi_panel_probe(struct mipi_dsi_device *dsi)
@@ -380,26 +372,7 @@ static int jdi_panel_probe(struct mipi_dsi_device *dsi)
 	if (ret < 0)
 		return ret;
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		jdi_panel_del(jdi);
-		return ret;
-	}
-
-	return 0;
-}
-
-static void jdi_panel_remove(struct mipi_dsi_device *dsi)
-{
-	struct jdi_panel *jdi = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n",
-			ret);
-
-	jdi_panel_del(jdi);
+	return devm_mipi_dsi_attach(&dsi->dev, dsi);
 }
 
 static struct mipi_dsi_driver jdi_panel_driver = {
@@ -408,7 +381,6 @@ static struct mipi_dsi_driver jdi_panel_driver = {
 		.of_match_table = jdi_of_match,
 	},
 	.probe = jdi_panel_probe,
-	.remove = jdi_panel_remove,
 };
 module_mipi_dsi_driver(jdi_panel_driver);
 
diff --git a/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c b/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
index 0856df5a6ee226..8a02e92061f576 100644
--- a/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
+++ b/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
@@ -583,30 +583,19 @@ static int ltk050h3146w_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return ret;
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "mipi_dsi_attach failed: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void ltk050h3146w_remove(struct mipi_dsi_device *dsi)
-{
-	struct ltk050h3146w *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id ltk050h3146w_of_match[] = {
 	{
 		.compatible = "leadtek,ltk050h3146w",
@@ -630,7 +619,6 @@ static struct mipi_dsi_driver ltk050h3146w_driver = {
 		.of_match_table = ltk050h3146w_of_match,
 	},
 	.probe	= ltk050h3146w_probe,
-	.remove = ltk050h3146w_remove,
 };
 module_mipi_dsi_driver(ltk050h3146w_driver);
 
diff --git a/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c b/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c
index 7f19fd5b80600c..ff35b62a327ef0 100644
--- a/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c
+++ b/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c
@@ -649,30 +649,19 @@ static int ltk500hd1829_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return ret;
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "mipi_dsi_attach failed: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void ltk500hd1829_remove(struct mipi_dsi_device *dsi)
-{
-	struct ltk500hd1829 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id ltk500hd1829_of_match[] = {
 	{
 		.compatible = "leadtek,ltk101b4029w",
@@ -692,7 +681,6 @@ static struct mipi_dsi_driver ltk500hd1829_driver = {
 		.of_match_table = ltk500hd1829_of_match,
 	},
 	.probe = ltk500hd1829_probe,
-	.remove = ltk500hd1829_remove,
 };
 module_mipi_dsi_driver(ltk500hd1829_driver);
 
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36532.c b/drivers/gpu/drm/panel/panel-novatek-nt36532.c
new file mode 100644
index 00000000000000..41d2721e0513f0
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36532.c
@@ -0,0 +1,431 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Novatek NT36532 DriverIC panels driver
+ * Based on the template generated by linux-mdss-dsi-panel-driver-generator
+ *
+ * Copyright (c) 2025, 2026 Junjie Cao <junjie.cao@linux.dev>
+ */
+
+#include <linux/delay.h>
+#include <linux/device-id/of.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/display/drm_dsc.h>
+#include <drm/display/drm_dsc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct nt36532 {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi[2];
+	const struct panel_info *panel_info;
+	struct drm_dsc_config dsc;
+	struct gpio_desc *reset_gpio;
+	struct regulator_bulk_data *supplies;
+};
+
+struct panel_info {
+	unsigned int lanes;
+	enum mipi_dsi_pixel_format format;
+	unsigned long mode_flags;
+
+	const struct drm_display_mode display_mode;
+
+	const struct drm_dsc_config *dsc_cfg;
+
+	int (*init_sequence)(struct nt36532 *ctx);
+
+	bool is_dual_dsi;
+};
+
+static const struct regulator_bulk_data nt36532_supplies[] = {
+	{ .supply = "vddio" }, /* 1.8v */
+};
+
+static inline struct nt36532 *to_nt36532(struct drm_panel *panel)
+{
+	return container_of(panel, struct nt36532, panel);
+}
+
+static inline struct mipi_dsi_device *to_primary_dsi(struct nt36532 *ctx)
+{
+	/* Sync on DSI1 for dual dsi */
+	return ctx->panel_info->is_dual_dsi ? ctx->dsi[1] : ctx->dsi[0];
+}
+
+static void nt36532_reset(struct nt36532 *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 10100);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(5000, 5100);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(15000, 15100);
+}
+
+static int nt36532_prepare(struct drm_panel *panel)
+{
+	struct nt36532 *ctx = to_nt36532(panel);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(nt36532_supplies),
+				    ctx->supplies);
+	if (ret < 0)
+		return ret;
+
+	nt36532_reset(ctx);
+
+	ret = ctx->panel_info->init_sequence(ctx);
+	if (ret < 0) {
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+		regulator_bulk_disable(ARRAY_SIZE(nt36532_supplies),
+				       ctx->supplies);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int nt36532_off(struct nt36532 *ctx)
+{
+	struct mipi_dsi_device *dsi = to_primary_dsi(ctx);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_usleep_range(&dsi_ctx, 10000, 10100);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 65);
+
+	return dsi_ctx.accum_err;
+}
+
+static int nt36532_unprepare(struct drm_panel *panel)
+{
+	struct nt36532 *ctx = to_nt36532(panel);
+	struct device *dev = &ctx->dsi[0]->dev;
+	int ret;
+
+	ret = nt36532_off(ctx);
+	if (ret < 0)
+		dev_err(dev, "Failed to un-initialize panel: %d\n", ret);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(nt36532_supplies), ctx->supplies);
+
+	return 0;
+}
+
+static int nt36532_get_modes(struct drm_panel *panel,
+			     struct drm_connector *connector)
+{
+	struct nt36532 *ctx = to_nt36532(panel);
+	const struct panel_info *panel_info = ctx->panel_info;
+
+	return drm_connector_helper_get_modes_fixed(connector,
+						    &panel_info->display_mode);
+}
+
+static const struct drm_panel_funcs nt36532_panel_funcs = {
+	.prepare = nt36532_prepare,
+	.unprepare = nt36532_unprepare,
+	.get_modes = nt36532_get_modes,
+};
+
+static int csot_init_sequence(struct nt36532 *ctx)
+{
+	struct mipi_dsi_device *dsi = to_primary_dsi(ctx);
+	struct drm_dsc_picture_parameter_set pps;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x22);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2d, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x24);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xbc,
+				     0x00, 0x00, 0x03, 0x22, 0x00, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x60);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x5a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x10, 0x0c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x11, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x12, 0xe1);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0xa9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x8e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x8e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x8e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0d, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x0c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x18);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x1c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x24);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x23, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0x2c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x25, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x26, 0x34);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x27, 0x38);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x28, 0x3c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2a, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x30, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x31, 0xfd);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x32, 0xfc);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x33, 0xfa);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x34, 0xf8);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0xf6);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x36, 0xf4);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0xf2);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0xf0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0xee);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0xec);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0xea);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0xe9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0xe8);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0xe7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0xe6);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x46, 0xf9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x47, 0xf6);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x48, 0xf2);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x49, 0xf0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4a, 0xec);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4b, 0xe8);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4c, 0xe4);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4d, 0xe0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4e, 0xde);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4f, 0xd9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x50, 0xd6);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x51, 0xd4);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x52, 0xc2);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x53, 0xd0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x54, 0xcd);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0xf6);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0xf0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0xeb);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0xe8);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0xe5);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0xe3);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0xe0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0xde);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0xda);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0xd7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0xd4);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0xd2);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0xd0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0xcc);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0xc8);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x27);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x30, 0xf0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x31, 0x2a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xd0, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xd1, 0x54);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xde, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xdf, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xc4, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0xf0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfa, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xba, 0xaa, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xbb, 0xa0, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x27);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x8c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_SET_DISPLAY_BRIGHTNESS,
+				     0x07, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY,
+				     0x2c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb9, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb9, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b,
+				     0x03, 0xd2, 0x1a, 0x04, 0x04, 0x00);
+
+	/* Enable DSC */
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x90, 0x03);
+	drm_dsc_pps_payload_pack(&pps, &ctx->dsc);
+	mipi_dsi_picture_parameter_set_multi(&dsi_ctx, &pps);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x9d, 0x01);
+
+	/* Program refresh rate control registers */
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb2, 0x91);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb3, 0x40);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	mipi_dsi_usleep_range(&dsi_ctx, 10000, 10100);
+
+	return dsi_ctx.accum_err;
+}
+
+static const struct drm_display_mode csot_display_mode = {
+	/* 120Hz, the typical value */
+	.clock = (1500 + 118 + 20 + 100) * 2 * (2120 + 26 + 2 + 208) * 120 / 1000,
+	.hdisplay = 1500 * 2,
+	.hsync_start = (1500 + 118) * 2,
+	.hsync_end = (1500 + 118 + 20) * 2,
+	.htotal = (1500 + 118 + 20 + 100) * 2,
+	.vdisplay = 2120,
+	.vsync_start = 2120 + 26,
+	.vsync_end = 2120 + 26 + 2,
+	.vtotal = 2120 + 26 + 2 + 208,
+	.width_mm = 250,
+	.height_mm = 177,
+};
+
+static const struct drm_dsc_config csot_dsc_cfg = {
+	.dsc_version_major = 1,
+	.dsc_version_minor = 2,
+	.slice_height = 20,
+	.slice_width = 750,
+	.slice_count = 2,
+	.bits_per_component = 8,
+	.bits_per_pixel = 8 << 4,
+	.block_pred_enable = true,
+};
+
+static int nt36532_probe(struct mipi_dsi_device *dsi)
+{
+	struct mipi_dsi_device_info dsi_info = {"nt36532-secondary", 0, NULL};
+	const struct panel_info *panel_info;
+	struct mipi_dsi_host *dsi1_host;
+	struct device *dev = &dsi->dev;
+	struct device_node *dsi1;
+	struct nt36532 *ctx;
+	int num_dsi = 1;
+	int ret, i;
+
+	ctx = devm_drm_panel_alloc(dev, struct nt36532, panel,
+				   &nt36532_panel_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ret = devm_regulator_bulk_get_const(&dsi->dev,
+					    ARRAY_SIZE(nt36532_supplies),
+					    nt36532_supplies, &ctx->supplies);
+	if (ret < 0)
+		return ret;
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	ctx->panel_info = of_device_get_match_data(dev);
+	panel_info = ctx->panel_info;
+	if (!panel_info)
+		return -ENODEV;
+
+	ctx->dsc = *panel_info->dsc_cfg;
+
+	if (panel_info->is_dual_dsi) {
+		num_dsi = 2;
+		dsi1 = of_graph_get_remote_node(dsi->dev.of_node, 1, -1);
+		if (!dsi1) {
+			dev_err(dev, "cannot get secondary DSI node.\n");
+			return -ENODEV;
+		}
+
+		dsi1_host = of_find_mipi_dsi_host_by_node(dsi1);
+		of_node_put(dsi1);
+		if (!dsi1_host)
+			return dev_err_probe(dev, -EPROBE_DEFER,
+					     "cannot get secondary DSI host\n");
+
+		ctx->dsi[1] = devm_mipi_dsi_device_register_full(dev, dsi1_host,
+								 &dsi_info);
+		if (IS_ERR(ctx->dsi[1])) {
+			dev_err(dev, "cannot get secondary DSI device\n");
+			return PTR_ERR(ctx->dsi[1]);
+		}
+
+		mipi_dsi_set_drvdata(ctx->dsi[1], ctx);
+	}
+
+	ctx->dsi[0] = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	ctx->panel.prepare_prev_first = true;
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get backlight\n");
+
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to add panel\n");
+
+	for (i = 0; i < num_dsi; i++) {
+		ctx->dsi[i]->lanes = panel_info->lanes;
+		ctx->dsi[i]->format = panel_info->format;
+		ctx->dsi[i]->mode_flags = panel_info->mode_flags;
+		ctx->dsi[i]->dsc = &ctx->dsc;
+		ret = devm_mipi_dsi_attach(dev, ctx->dsi[i]);
+		if (ret < 0) {
+			return dev_err_probe(dev, ret,
+					     "Failed to attach to DSI host\n");
+		}
+	}
+
+	return 0;
+}
+
+static const struct panel_info csot_panel_info = {
+	.lanes = 4,
+	.format = MIPI_DSI_FMT_RGB888,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_CLOCK_NON_CONTINUOUS |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_DSC_ALL_SLICES_IN_PKT,
+	.display_mode = csot_display_mode,
+	.dsc_cfg = &csot_dsc_cfg,
+	.init_sequence = csot_init_sequence,
+	.is_dual_dsi = true,
+};
+
+static const struct of_device_id nt36532_of_match[] = {
+	{ .compatible = "csot,ppc100hb1-1", .data = &csot_panel_info },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, nt36532_of_match);
+
+static struct mipi_dsi_driver nt36532_driver = {
+	.probe = nt36532_probe,
+	.driver = {
+		.name = "panel-novatek-nt36532",
+		.of_match_table = nt36532_of_match,
+	},
+};
+module_mipi_dsi_driver(nt36532_driver);
+
+MODULE_AUTHOR("Junjie Cao <junjie.cao@linux.dev>");
+MODULE_DESCRIPTION("DRM driver for Novatek NT36532 based MIPI DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-samsung-dsi.h b/drivers/gpu/drm/panel/panel-samsung-dsi.h
new file mode 100644
index 00000000000000..65c74dcbb21ba1
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-samsung-dsi.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef _PANEL_SAMSUNG_DSI_H
+#define _PANEL_SAMSUNG_DSI_H
+
+#include <drm/drm_mipi_dsi.h>
+
+static inline void samsung_dsi_test_key_on_lvl1(struct mipi_dsi_multi_context *ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(ctx, 0x9f, 0xa5, 0xa5);
+}
+
+static inline void samsung_dsi_test_key_off_lvl1(struct mipi_dsi_multi_context *ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(ctx, 0x9f, 0x5a, 0x5a);
+}
+
+static inline void samsung_dsi_test_key_on_lvl2(struct mipi_dsi_multi_context *ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0x5a, 0x5a);
+}
+
+static inline void samsung_dsi_test_key_off_lvl2(struct mipi_dsi_multi_context *ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0xa5, 0xa5);
+}
+
+static inline void samsung_dsi_test_key_on_lvl3(struct mipi_dsi_multi_context *ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0x5a, 0x5a);
+}
+
+static inline void samsung_dsi_test_key_off_lvl3(struct mipi_dsi_multi_context *ctx)
+{
+	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0xa5, 0xa5);
+}
+
+#endif /* _PANEL_SAMSUNG_DSI_H */
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
index 54a65abf7e8907..e6ad1e87607081 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
@@ -48,14 +48,10 @@ static int s6d16d0_unprepare(struct drm_panel *panel)
 {
 	struct s6d16d0 *s6 = panel_to_s6d16d0(panel);
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(s6->dev);
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
-	/* Enter sleep mode */
-	ret = mipi_dsi_dcs_enter_sleep_mode(dsi);
-	if (ret) {
-		dev_err(s6->dev, "failed to enter sleep mode (%d)\n", ret);
-		return ret;
-	}
+	/* Enter sleep mode; error does not prevent power-off */
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
 
 	/* Assert RESET */
 	gpiod_set_value_cansleep(s6->reset_gpio, 1);
@@ -68,6 +64,7 @@ static int s6d16d0_prepare(struct drm_panel *panel)
 {
 	struct s6d16d0 *s6 = panel_to_s6d16d0(panel);
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(s6->dev);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 	int ret;
 
 	ret = regulator_enable(s6->supply);
@@ -83,57 +80,39 @@ static int s6d16d0_prepare(struct drm_panel *panel)
 	gpiod_set_value_cansleep(s6->reset_gpio, 0);
 	msleep(120);
 
-	/* Enabe tearing mode: send TE (tearing effect) at VBLANK */
-	ret = mipi_dsi_dcs_set_tear_on(dsi,
+	/* Enable tearing mode: send TE (tearing effect) at VBLANK */
+	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx,
 				       MIPI_DSI_DCS_TEAR_MODE_VBLANK);
-	if (ret) {
-		dev_err(s6->dev, "failed to enable vblank TE (%d)\n", ret);
-		goto err_power_off;
-	}
 	/* Exit sleep mode and power on */
-	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
-	if (ret) {
-		dev_err(s6->dev, "failed to exit sleep mode (%d)\n", ret);
-		goto err_power_off;
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	if (dsi_ctx.accum_err) {
+		gpiod_set_value_cansleep(s6->reset_gpio, 1);
+		regulator_disable(s6->supply);
 	}
 
-	return 0;
-
-err_power_off:
-	gpiod_set_value_cansleep(s6->reset_gpio, 1);
-	regulator_disable(s6->supply);
-
-	return ret;
+	return dsi_ctx.accum_err;
 }
 
 static int s6d16d0_enable(struct drm_panel *panel)
 {
 	struct s6d16d0 *s6 = panel_to_s6d16d0(panel);
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(s6->dev);
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
-	ret = mipi_dsi_dcs_set_display_on(dsi);
-	if (ret) {
-		dev_err(s6->dev, "failed to turn display on (%d)\n", ret);
-		return ret;
-	}
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
 
-	return 0;
+	return dsi_ctx.accum_err;
 }
 
 static int s6d16d0_disable(struct drm_panel *panel)
 {
 	struct s6d16d0 *s6 = panel_to_s6d16d0(panel);
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(s6->dev);
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
 
-	ret = mipi_dsi_dcs_set_display_off(dsi);
-	if (ret) {
-		dev_err(s6->dev, "failed to turn display off (%d)\n", ret);
-		return ret;
-	}
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
 
-	return 0;
+	return dsi_ctx.accum_err;
 }
 
 static int s6d16d0_get_modes(struct drm_panel *panel,
@@ -207,21 +186,11 @@ static int s6d16d0_probe(struct mipi_dsi_device *dsi)
 		return ret;
 	}
 
-	drm_panel_add(&s6->panel);
-
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0)
-		drm_panel_remove(&s6->panel);
-
-	return ret;
-}
-
-static void s6d16d0_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6d16d0 *s6 = mipi_dsi_get_drvdata(dsi);
+	ret = devm_drm_panel_add(dev, &s6->panel);
+	if (ret)
+		return ret;
 
-	mipi_dsi_detach(dsi);
-	drm_panel_remove(&s6->panel);
+	return devm_mipi_dsi_attach(dev, dsi);
 }
 
 static const struct of_device_id s6d16d0_of_match[] = {
@@ -232,7 +201,6 @@ MODULE_DEVICE_TABLE(of, s6d16d0_of_match);
 
 static struct mipi_dsi_driver s6d16d0_driver = {
 	.probe = s6d16d0_probe,
-	.remove = s6d16d0_remove,
 	.driver = {
 		.name = "panel-samsung-s6d16d0",
 		.of_match_table = s6d16d0_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c b/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c
index 692020081524ad..5a4cfb21e2fdc6 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6d7aa0.c
@@ -436,30 +436,19 @@ static int s6d7aa0_probe(struct mipi_dsi_device *dsi)
 					     "Failed to create backlight\n");
 	}
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void s6d7aa0_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6d7aa0 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id s6d7aa0_of_match[] = {
 	{
 		.compatible = "samsung,lsl080al02",
@@ -479,7 +468,6 @@ MODULE_DEVICE_TABLE(of, s6d7aa0_of_match);
 
 static struct mipi_dsi_driver s6d7aa0_driver = {
 	.probe = s6d7aa0_probe,
-	.remove = s6d7aa0_remove,
 	.driver = {
 		.name = "panel-samsung-s6d7aa0",
 		.of_match_table = s6d7aa0_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c b/drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c
index f4d75eca3cdfa2..9cd92d7a823b12 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e3fa7.c
@@ -211,30 +211,19 @@ static int s6e3fa7_panel_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void s6e3fa7_panel_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e3fa7_panel *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id s6e3fa7_panel_of_match[] = {
 	{ .compatible = "samsung,s6e3fa7-ams559nk06" },
 	{ /* sentinel */ }
@@ -243,7 +232,6 @@ MODULE_DEVICE_TABLE(of, s6e3fa7_panel_of_match);
 
 static struct mipi_dsi_driver s6e3fa7_panel_driver = {
 	.probe = s6e3fa7_panel_probe,
-	.remove = s6e3fa7_panel_remove,
 	.driver = {
 		.name = "panel-samsung-s6e3fa7",
 		.of_match_table = s6e3fa7_panel_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e3fc2x01.c b/drivers/gpu/drm/panel/panel-samsung-s6e3fc2x01.c
index e63080204af77e..b8f5e2f4f9e9a7 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e3fc2x01.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e3fc2x01.c
@@ -20,6 +20,8 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
+#include "panel-samsung-dsi.h"
+
 #define MCS_ELVSS_ON            0xb1
 
 struct samsung_s6e3fc2x01 {
@@ -41,19 +43,6 @@ struct samsung_s6e3fc2x01 *to_samsung_s6e3fc2x01(struct drm_panel *panel)
 	return container_of(panel, struct samsung_s6e3fc2x01, panel);
 }
 
-#define s6e3fc2x01_test_key_on_lvl1(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0x9f, 0xa5, 0xa5)
-#define s6e3fc2x01_test_key_off_lvl1(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0x9f, 0x5a, 0x5a)
-#define s6e3fc2x01_test_key_on_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0x5a, 0x5a)
-#define s6e3fc2x01_test_key_off_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0xa5, 0xa5)
-#define s6e3fc2x01_test_key_on_lvl3(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0x5a, 0x5a)
-#define s6e3fc2x01_test_key_off_lvl3(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0xa5, 0xa5)
-
 static void s6e3fc2x01_reset(struct samsung_s6e3fc2x01 *ctx)
 {
 	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
@@ -66,7 +55,7 @@ static int s6e3fc2x01_on(struct samsung_s6e3fc2x01 *ctx)
 {
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
 
-	s6e3fc2x01_test_key_on_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl1(&dsi_ctx);
 
 	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
 
@@ -75,64 +64,64 @@ static int s6e3fc2x01_on(struct samsung_s6e3fc2x01 *ctx)
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x0a);
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 
-	s6e3fc2x01_test_key_off_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl1(&dsi_ctx);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x01);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xcd, 0x01);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
 	mipi_dsi_usleep_range(&dsi_ctx, 15000, 16000);
 
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x0f);
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 
-	s6e3fc2x01_test_key_on_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl1(&dsi_ctx);
 	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
-	s6e3fc2x01_test_key_off_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl1(&dsi_ctx);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xeb, 0x17,
 					       0x41, 0x92,
 					       0x0e, 0x10,
 					       0x82, 0x5a);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
 	/* Column & Page Address Setting */
 	mipi_dsi_dcs_set_column_address_multi(&dsi_ctx, 0x0000, 0x0437);
 	mipi_dsi_dcs_set_page_address_multi(&dsi_ctx, 0x0000, 0x0923);
 
 	/* Horizontal & Vertical sync Setting */
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x09);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xe8, 0x10, 0x30);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
-	s6e3fc2x01_test_key_on_lvl3(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl3(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x01);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xe3, 0x88);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x07);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xed, 0x67);
-	s6e3fc2x01_test_key_off_lvl3(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl3(&dsi_ctx);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x07);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb7, 0x01);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x08);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb7, 0x12);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY, 0x20);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x00);
 	mipi_dsi_usleep_range(&dsi_ctx, 1000, 2000);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MCS_ELVSS_ON, 0x00, 0x01);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb3, 0x00, 0xc1);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xff, 0x78);
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
@@ -140,14 +129,14 @@ static int s6e3fc2x01_on(struct samsung_s6e3fc2x01 *ctx)
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x90);
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x02);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MCS_ELVSS_ON, 0xc6, 0x00, 0x00,
 				     0x21, 0xed, 0x02, 0x08, 0x06, 0xc1, 0x27,
 				     0xfc, 0xdc, 0xe4, 0x00, 0xd9, 0xe6, 0xe7,
 				     0x00, 0xfc, 0xff, 0xea);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MCS_ELVSS_ON, 0x00, 0x00);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 
@@ -159,9 +148,9 @@ static int s6e3fc2x01_enable(struct drm_panel *panel)
 	struct samsung_s6e3fc2x01 *ctx = to_samsung_s6e3fc2x01(panel);
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
 
-	s6e3fc2x01_test_key_on_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl1(&dsi_ctx);
 	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
-	s6e3fc2x01_test_key_off_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl1(&dsi_ctx);
 
 	return dsi_ctx.accum_err;
 }
@@ -170,27 +159,27 @@ static int s6e3fc2x01_off(struct samsung_s6e3fc2x01 *ctx)
 {
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
 
-	s6e3fc2x01_test_key_on_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl1(&dsi_ctx);
 
 	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
 
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_usleep_range(&dsi_ctx, 16000, 17000);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x50);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb9, 0x82);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 	mipi_dsi_usleep_range(&dsi_ctx, 16000, 17000);
 
 	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
 
-	s6e3fc2x01_test_key_off_lvl1(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl1(&dsi_ctx);
 
-	s6e3fc2x01_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x05);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf4, 0x01);
-	s6e3fc2x01_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 	mipi_dsi_msleep(&dsi_ctx, 160);
 
 	return dsi_ctx.accum_err;
@@ -340,30 +329,19 @@ static int s6e3fc2x01_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void s6e3fc2x01_remove(struct mipi_dsi_device *dsi)
-{
-	struct samsung_s6e3fc2x01 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id s6e3fc2x01_of_match[] = {
 	{ .compatible = "samsung,s6e3fc2x01-ams641rw", .data = &ams641rw_mode },
 	{ /* sentinel */ }
@@ -372,7 +350,6 @@ MODULE_DEVICE_TABLE(of, s6e3fc2x01_of_match);
 
 static struct mipi_dsi_driver s6e3fc2x01_driver = {
 	.probe = s6e3fc2x01_probe,
-	.remove = s6e3fc2x01_remove,
 	.driver = {
 		.name = "panel-samsung-s6e3fc2x01",
 		.of_match_table = s6e3fc2x01_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c b/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c
index 1db0c63b1131ee..cede897ff69dee 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c
@@ -722,8 +722,8 @@ static int s6e3ha2_probe(struct mipi_dsi_device *dsi)
 		return PTR_ERR(ctx->enable_gpio);
 	}
 
-	ctx->bl_dev = backlight_device_register("s6e3ha2", dev, ctx,
-						&s6e3ha2_bl_ops, NULL);
+	ctx->bl_dev = devm_backlight_device_register(dev, "s6e3ha2", dev, ctx,
+						     &s6e3ha2_bl_ops, NULL);
 	if (IS_ERR(ctx->bl_dev)) {
 		dev_err(dev, "failed to register backlight device\n");
 		return PTR_ERR(ctx->bl_dev);
@@ -735,28 +735,11 @@ static int s6e3ha2_probe(struct mipi_dsi_device *dsi)
 
 	ctx->panel.prepare_prev_first = true;
 
-	drm_panel_add(&ctx->panel);
-
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0)
-		goto remove_panel;
-
-	return ret;
-
-remove_panel:
-	drm_panel_remove(&ctx->panel);
-	backlight_device_unregister(ctx->bl_dev);
-
-	return ret;
-}
-
-static void s6e3ha2_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e3ha2 *ctx = mipi_dsi_get_drvdata(dsi);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	mipi_dsi_detach(dsi);
-	drm_panel_remove(&ctx->panel);
-	backlight_device_unregister(ctx->bl_dev);
+	return devm_mipi_dsi_attach(dev, dsi);
 }
 
 static const struct of_device_id s6e3ha2_of_match[] = {
@@ -768,7 +751,6 @@ MODULE_DEVICE_TABLE(of, s6e3ha2_of_match);
 
 static struct mipi_dsi_driver s6e3ha2_driver = {
 	.probe = s6e3ha2_probe,
-	.remove = s6e3ha2_remove,
 	.driver = {
 		.name = "panel-samsung-s6e3ha2",
 		.of_match_table = s6e3ha2_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e3ha8.c b/drivers/gpu/drm/panel/panel-samsung-s6e3ha8.c
index 550e9ef9bb71b5..078523df48860e 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e3ha8.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e3ha8.c
@@ -16,6 +16,8 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_panel.h>
 
+#include "panel-samsung-dsi.h"
+
 struct s6e3ha8 {
 	struct drm_panel panel;
 	struct mipi_dsi_device *dsi;
@@ -36,18 +38,6 @@ struct s6e3ha8 *to_s6e3ha8_amb577px01_wqhd(struct drm_panel *panel)
 	return container_of(panel, struct s6e3ha8, panel);
 }
 
-#define s6e3ha8_test_key_on_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0x5a, 0x5a)
-#define s6e3ha8_test_key_off_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0xa5, 0xa5)
-#define s6e3ha8_test_key_on_lvl3(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0x5a, 0x5a)
-#define s6e3ha8_test_key_off_lvl3(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0xa5, 0xa5)
-#define s6e3ha8_test_key_on_lvl1(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0x9f, 0xa5, 0xa5)
-#define s6e3ha8_test_key_off_lvl1(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0x9f, 0x5a, 0x5a)
 #define s6e3ha8_afc_off(ctx) \
 	mipi_dsi_dcs_write_seq_multi(ctx, 0xe2, 0x00, 0x00)
 
@@ -68,26 +58,26 @@ static int s6e3ha8_amb577px01_wqhd_on(struct s6e3ha8 *priv)
 
 	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
 
-	s6e3ha8_test_key_on_lvl1(&ctx);
+	samsung_dsi_test_key_on_lvl1(&ctx);
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	mipi_dsi_compression_mode_multi(&ctx, true);
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
 	mipi_dsi_dcs_exit_sleep_mode_multi(&ctx);
 	usleep_range(5000, 6000);
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xf2, 0x13);
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 	usleep_range(10000, 11000);
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xf2, 0x13);
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
 	/* OMOK setting 1 (Initial setting) - Scaler Latch Setting Guide */
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xb0, 0x07);
 	/* latch setting 1 : Scaler on/off & address setting & PPS setting -> Image update latch */
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xf2, 0x3c, 0x10);
@@ -98,31 +88,31 @@ static int s6e3ha8_amb577px01_wqhd_on(struct s6e3ha8 *priv)
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0x2a, 0x00, 0x00, 0x05, 0x9f); /* CASET */
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0x2b, 0x00, 0x00, 0x0b, 0x8f); /* PASET */
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xba, 0x01); /* scaler setup : scaler off */
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0x35, 0x00); /* TE Vsync ON */
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xed, 0x4c); /* ERR_FG */
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
-	s6e3ha8_test_key_on_lvl3(&ctx);
+	samsung_dsi_test_key_on_lvl3(&ctx);
 	/* FFC Setting 897.6Mbps */
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xc5, 0x0d, 0x10, 0xb4, 0x3e, 0x01);
-	s6e3ha8_test_key_off_lvl3(&ctx);
+	samsung_dsi_test_key_off_lvl3(&ctx);
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xb9,
 				   0x00, 0xb0, 0x81, 0x09, 0x00, 0x00, 0x00,
 				   0x11, 0x03); /* TSP HSYNC Setting */
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xb0, 0x03);
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xf6, 0x43);
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	/* Brightness condition set */
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xca,
 				   0x07, 0x00, 0x00, 0x00, 0x80, 0x80, 0x80,
@@ -138,9 +128,9 @@ static int s6e3ha8_amb577px01_wqhd_on(struct s6e3ha8 *priv)
 				   0x10); /* MPS/ELVSS Setting */
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xf4, 0xeb, 0x28); /* VINT */
 	mipi_dsi_dcs_write_seq_multi(&ctx, 0xf7, 0x03); /* Gamma, LTPS(AID) update */
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
-	s6e3ha8_test_key_off_lvl1(&ctx);
+	samsung_dsi_test_key_off_lvl1(&ctx);
 
 	return ctx.accum_err;
 }
@@ -151,9 +141,9 @@ static int s6e3ha8_enable(struct drm_panel *panel)
 	struct mipi_dsi_device *dsi = priv->dsi;
 	struct mipi_dsi_multi_context ctx = { .dsi = dsi };
 
-	s6e3ha8_test_key_on_lvl1(&ctx);
+	samsung_dsi_test_key_on_lvl1(&ctx);
 	mipi_dsi_dcs_set_display_on_multi(&ctx);
-	s6e3ha8_test_key_off_lvl1(&ctx);
+	samsung_dsi_test_key_off_lvl1(&ctx);
 
 	return ctx.accum_err;
 }
@@ -164,14 +154,14 @@ static int s6e3ha8_disable(struct drm_panel *panel)
 	struct mipi_dsi_device *dsi = priv->dsi;
 	struct mipi_dsi_multi_context ctx = { .dsi = dsi };
 
-	s6e3ha8_test_key_on_lvl1(&ctx);
+	samsung_dsi_test_key_on_lvl1(&ctx);
 	mipi_dsi_dcs_set_display_off_multi(&ctx);
-	s6e3ha8_test_key_off_lvl1(&ctx);
+	samsung_dsi_test_key_off_lvl1(&ctx);
 	mipi_dsi_msleep(&ctx, 20);
 
-	s6e3ha8_test_key_on_lvl2(&ctx);
+	samsung_dsi_test_key_on_lvl2(&ctx);
 	s6e3ha8_afc_off(&ctx);
-	s6e3ha8_test_key_off_lvl2(&ctx);
+	samsung_dsi_test_key_off_lvl2(&ctx);
 
 	mipi_dsi_msleep(&ctx, 160);
 
@@ -200,9 +190,9 @@ static int s6e3ha8_amb577px01_wqhd_prepare(struct drm_panel *panel)
 
 	drm_dsc_pps_payload_pack(&pps, &priv->dsc);
 
-	s6e3ha8_test_key_on_lvl1(&ctx);
+	samsung_dsi_test_key_on_lvl1(&ctx);
 	mipi_dsi_picture_parameter_set_multi(&ctx, &pps);
-	s6e3ha8_test_key_off_lvl1(&ctx);
+	samsung_dsi_test_key_off_lvl1(&ctx);
 
 	mipi_dsi_msleep(&ctx, 28);
 
@@ -283,7 +273,9 @@ static int s6e3ha8_amb577px01_wqhd_probe(struct mipi_dsi_device *dsi)
 
 	priv->panel.prepare_prev_first = true;
 
-	drm_panel_add(&priv->panel);
+	ret = devm_drm_panel_add(dev, &priv->panel);
+	if (ret)
+		return ret;
 
 	/* This panel only supports DSC; unconditionally enable it */
 	dsi->dsc = &priv->dsc;
@@ -299,28 +291,15 @@ static int s6e3ha8_amb577px01_wqhd_probe(struct mipi_dsi_device *dsi)
 	priv->dsc.bits_per_pixel = 8 << 4; /* 4 fractional bits */
 	priv->dsc.block_pred_enable = true;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&priv->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void s6e3ha8_amb577px01_wqhd_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e3ha8 *priv = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&priv->panel);
-}
-
 static const struct of_device_id s6e3ha8_amb577px01_wqhd_of_match[] = {
 	{ .compatible = "samsung,s6e3ha8" },
 	{ /* sentinel */ }
@@ -329,7 +308,6 @@ MODULE_DEVICE_TABLE(of, s6e3ha8_amb577px01_wqhd_of_match);
 
 static struct mipi_dsi_driver s6e3ha8_amb577px01_wqhd_driver = {
 	.probe = s6e3ha8_amb577px01_wqhd_probe,
-	.remove = s6e3ha8_amb577px01_wqhd_remove,
 	.driver = {
 		.name = "panel-s6e3ha8",
 		.of_match_table = s6e3ha8_amb577px01_wqhd_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c b/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c
index e05199ce14eea0..1f825d9956861a 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c
@@ -465,8 +465,8 @@ static int s6e63j0x03_probe(struct mipi_dsi_device *dsi)
 
 	ctx->panel.prepare_prev_first = true;
 
-	ctx->bl_dev = backlight_device_register("s6e63j0x03", dev, ctx,
-						&s6e63j0x03_bl_ops, NULL);
+	ctx->bl_dev = devm_backlight_device_register(dev, "s6e63j0x03", dev, ctx,
+						     &s6e63j0x03_bl_ops, NULL);
 	if (IS_ERR(ctx->bl_dev))
 		return dev_err_probe(dev, PTR_ERR(ctx->bl_dev),
 				     "failed to register backlight device\n");
@@ -475,29 +475,11 @@ static int s6e63j0x03_probe(struct mipi_dsi_device *dsi)
 	ctx->bl_dev->props.brightness = DEFAULT_BRIGHTNESS;
 	ctx->bl_dev->props.power = BACKLIGHT_POWER_OFF;
 
-	drm_panel_add(&ctx->panel);
-
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0)
-		goto remove_panel;
-
-	return ret;
-
-remove_panel:
-	drm_panel_remove(&ctx->panel);
-	backlight_device_unregister(ctx->bl_dev);
-
-	return ret;
-}
-
-static void s6e63j0x03_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e63j0x03 *ctx = mipi_dsi_get_drvdata(dsi);
-
-	mipi_dsi_detach(dsi);
-	drm_panel_remove(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	backlight_device_unregister(ctx->bl_dev);
+	return devm_mipi_dsi_attach(dev, dsi);
 }
 
 static const struct of_device_id s6e63j0x03_of_match[] = {
@@ -508,7 +490,6 @@ MODULE_DEVICE_TABLE(of, s6e63j0x03_of_match);
 
 static struct mipi_dsi_driver s6e63j0x03_driver = {
 	.probe = s6e63j0x03_probe,
-	.remove = s6e63j0x03_remove,
 	.driver = {
 		.name = "panel_samsung_s6e63j0x03",
 		.of_match_table = s6e63j0x03_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c
index 2630975c111b88..c3842449c89547 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c
@@ -105,17 +105,7 @@ static int s6e63m0_dsi_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return ret;
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0)
-		s6e63m0_remove(dev);
-
-	return ret;
-}
-
-static void s6e63m0_dsi_remove(struct mipi_dsi_device *dsi)
-{
-	mipi_dsi_detach(dsi);
-	s6e63m0_remove(&dsi->dev);
+	return devm_mipi_dsi_attach(dev, dsi);
 }
 
 static const struct of_device_id s6e63m0_dsi_of_match[] = {
@@ -126,7 +116,6 @@ MODULE_DEVICE_TABLE(of, s6e63m0_dsi_of_match);
 
 static struct mipi_dsi_driver s6e63m0_dsi_driver = {
 	.probe			= s6e63m0_dsi_probe,
-	.remove			= s6e63m0_dsi_remove,
 	.driver			= {
 		.name		= "panel-samsung-s6e63m0",
 		.of_match_table = s6e63m0_dsi_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-spi.c b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-spi.c
index d99afcc672ca15..c8bec2b1e7c1aa 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-spi.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-spi.c
@@ -62,11 +62,6 @@ static int s6e63m0_spi_probe(struct spi_device *spi)
 			     s6e63m0_spi_dcs_write, false);
 }
 
-static void s6e63m0_spi_remove(struct spi_device *spi)
-{
-	s6e63m0_remove(&spi->dev);
-}
-
 static const struct of_device_id s6e63m0_spi_of_match[] = {
 	{ .compatible = "samsung,s6e63m0" },
 	{ /* sentinel */ }
@@ -75,7 +70,6 @@ MODULE_DEVICE_TABLE(of, s6e63m0_spi_of_match);
 
 static struct spi_driver s6e63m0_spi_driver = {
 	.probe			= s6e63m0_spi_probe,
-	.remove			= s6e63m0_spi_remove,
 	.driver			= {
 		.name		= "panel-samsung-s6e63m0",
 		.of_match_table = s6e63m0_spi_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c b/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c
index 997115b1e541d7..1990a7c232190d 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c
@@ -720,20 +720,10 @@ int s6e63m0_probe(struct device *dev, void *trsp,
 	if (ret < 0)
 		return ret;
 
-	drm_panel_add(&ctx->panel);
-
-	return 0;
+	return devm_drm_panel_add(dev, &ctx->panel);
 }
 EXPORT_SYMBOL_GPL(s6e63m0_probe);
 
-void s6e63m0_remove(struct device *dev)
-{
-	struct s6e63m0 *ctx = dev_get_drvdata(dev);
-
-	drm_panel_remove(&ctx->panel);
-}
-EXPORT_SYMBOL_GPL(s6e63m0_remove);
-
 MODULE_AUTHOR("Paweł Chmiel <pawel.mikolaj.chmiel@gmail.com>");
 MODULE_DESCRIPTION("s6e63m0 LCD Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0.h b/drivers/gpu/drm/panel/panel-samsung-s6e63m0.h
index c926eca1c8176d..7dae6772899c68 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0.h
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0.h
@@ -35,6 +35,5 @@ int s6e63m0_probe(struct device *dev, void *trsp,
 				   const u8 *data,
 				   size_t len),
 		  bool dsi_mode);
-void s6e63m0_remove(struct device *dev);
 
 #endif /* _PANEL_SAMSUNG_S6E63M0_H */
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams427ap24.c b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams427ap24.c
index 77fee36dbb5596..b48a2f9578e775 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams427ap24.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams427ap24.c
@@ -722,30 +722,19 @@ static int s6e88a0_ams427ap24_probe(struct mipi_dsi_device *dsi)
 	if (ret < 0)
 		return ret;
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void s6e88a0_ams427ap24_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e88a0_ams427ap24 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id s6e88a0_ams427ap24_of_match[] = {
 	{ .compatible = "samsung,s6e88a0-ams427ap24" },
 	{ /* sentinel */ },
@@ -754,7 +743,6 @@ MODULE_DEVICE_TABLE(of, s6e88a0_ams427ap24_of_match);
 
 static struct mipi_dsi_driver s6e88a0_ams427ap24_driver = {
 	.probe = s6e88a0_ams427ap24_probe,
-	.remove = s6e88a0_ams427ap24_remove,
 	.driver = {
 		.name = "panel-s6e88a0-ams427ap24",
 		.of_match_table = s6e88a0_ams427ap24_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
index ca5cad41ff1d95..2178e03e477291 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
@@ -13,6 +13,8 @@
 #include <drm/drm_modes.h>
 #include <drm/drm_panel.h>
 
+#include "panel-samsung-dsi.h"
+
 struct s6e88a0_ams452ef01 {
 	struct drm_panel panel;
 	struct mipi_dsi_device *dsi;
@@ -43,7 +45,7 @@ static int s6e88a0_ams452ef01_on(struct s6e88a0_ams452ef01 *ctx)
 
 	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
 
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf0, 0x5a, 0x5a); // enable LEVEL2 commands
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xcc, 0x4c); // set Pixel Clock Divider polarity
 
 	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
@@ -66,7 +68,7 @@ static int s6e88a0_ams452ef01_on(struct s6e88a0_ams452ef01 *ctx)
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb6, 0x2c, 0x0b); // set default elvss voltage
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x00);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf7, 0x03); // gamma/aor update
-	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf0, 0xa5, 0xa5); // disable LEVEL2 commands
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
 	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
 
@@ -194,30 +196,19 @@ static int s6e88a0_ams452ef01_probe(struct mipi_dsi_device *dsi)
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST;
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void s6e88a0_ams452ef01_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e88a0_ams452ef01 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id s6e88a0_ams452ef01_of_match[] = {
 	{ .compatible = "samsung,s6e88a0-ams452ef01" },
 	{ /* sentinel */ },
@@ -226,7 +217,6 @@ MODULE_DEVICE_TABLE(of, s6e88a0_ams452ef01_of_match);
 
 static struct mipi_dsi_driver s6e88a0_ams452ef01_driver = {
 	.probe = s6e88a0_ams452ef01_probe,
-	.remove = s6e88a0_ams452ef01_remove,
 	.driver = {
 		.name = "panel-s6e88a0-ams452ef01",
 		.of_match_table = s6e88a0_ams452ef01_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c b/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c
index 1b5c500d4f4eb7..8f160ca0a94e15 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c
@@ -1018,21 +1018,11 @@ static int s6e8aa0_probe(struct mipi_dsi_device *dsi)
 
 	ctx->panel.prepare_prev_first = true;
 
-	drm_panel_add(&ctx->panel);
-
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0)
-		drm_panel_remove(&ctx->panel);
-
-	return ret;
-}
-
-static void s6e8aa0_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e8aa0 *ctx = mipi_dsi_get_drvdata(dsi);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	mipi_dsi_detach(dsi);
-	drm_panel_remove(&ctx->panel);
+	return devm_mipi_dsi_attach(dev, dsi);
 }
 
 static const struct of_device_id s6e8aa0_of_match[] = {
@@ -1043,7 +1033,6 @@ MODULE_DEVICE_TABLE(of, s6e8aa0_of_match);
 
 static struct mipi_dsi_driver s6e8aa0_driver = {
 	.probe = s6e8aa0_probe,
-	.remove = s6e8aa0_remove,
 	.driver = {
 		.name = "panel-samsung-s6e8aa0",
 		.of_match_table = s6e8aa0_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e8fc0-m1906f9.c b/drivers/gpu/drm/panel/panel-samsung-s6e8fc0-m1906f9.c
index 2fae0dc6c424a1..1d2635c8cc3ba9 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e8fc0-m1906f9.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e8fc0-m1906f9.c
@@ -17,6 +17,8 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
+#include "panel-samsung-dsi.h"
+
 struct s6e8fc0_ctx {
 	struct drm_panel panel;
 	struct mipi_dsi_device *dsi;
@@ -45,20 +47,11 @@ static void s6e8fc0_m1906f9_reset(struct s6e8fc0_ctx *ctx)
 	usleep_range(10000, 11000);
 }
 
-#define s6e8fc0_test_key_on_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0x5a, 0x5a)
-#define s6e8fc0_test_key_off_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0xa5, 0xa5)
-#define s6e8fc0_test_key_on_lvl3(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0x5a, 0x5a)
-#define s6e8fc0_test_key_off_lvl3(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xfc, 0xa5, 0xa5)
-
 static int s6e8fc0_m1906f9_on(struct s6e8fc0_ctx *ctx)
 {
 	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
 
-	s6e8fc0_test_key_on_lvl3(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl3(&dsi_ctx);
 
 	mipi_dsi_dcs_set_display_brightness_multi(&dsi_ctx, 0x0000);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY,
@@ -71,15 +64,15 @@ static int s6e8fc0_m1906f9_on(struct s6e8fc0_ctx *ctx)
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xed,
 				     0xe4, 0x08, 0x96, 0xa4, 0x2a, 0x72, 0xe2,
 				     0xca, 0x00);
-	s6e8fc0_test_key_off_lvl3(&dsi_ctx);
-	s6e8fc0_test_key_on_lvl2(&dsi_ctx);
-	s6e8fc0_test_key_on_lvl3(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl3(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl3(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xe1, 0x93);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x05, 0xf4);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xf4, 0x03);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xed, 0x01, 0x81, 0x04);
-	s6e8fc0_test_key_off_lvl2(&dsi_ctx);
-	s6e8fc0_test_key_off_lvl3(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl3(&dsi_ctx);
 
 	return dsi_ctx.accum_err;
 }
@@ -256,29 +249,17 @@ static int s6e8fc0_m1906f9_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		drm_panel_remove(&ctx->panel);
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0)
 		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
-	}
 
 	return 0;
 }
 
-static void s6e8fc0_remove(struct mipi_dsi_device *dsi)
-{
-	struct s6e8fc0_ctx *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id samsung_s6e8fc0_of_match[] = {
 	{ .compatible = "samsung,s6e8fc0-m1906f9" },
 	{ /* sentinel */ }
@@ -287,7 +268,6 @@ MODULE_DEVICE_TABLE(of, samsung_s6e8fc0_of_match);
 
 static struct mipi_dsi_driver s6e8fc0_driver = {
 	.probe = s6e8fc0_m1906f9_probe,
-	.remove = s6e8fc0_remove,
 	.driver = {
 		.name = "panel-samsung-s6e8fc0-m1906f9",
 		.of_match_table = samsung_s6e8fc0_of_match,
diff --git a/drivers/gpu/drm/panel/panel-samsung-sofef00.c b/drivers/gpu/drm/panel/panel-samsung-sofef00.c
index f07d6533ed697c..913be42e7975c8 100644
--- a/drivers/gpu/drm/panel/panel-samsung-sofef00.c
+++ b/drivers/gpu/drm/panel/panel-samsung-sofef00.c
@@ -18,6 +18,8 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
+#include "panel-samsung-dsi.h"
+
 struct sofef00_panel {
 	struct drm_panel panel;
 	struct mipi_dsi_device *dsi;
@@ -37,11 +39,6 @@ struct sofef00_panel *to_sofef00_panel(struct drm_panel *panel)
 	return container_of(panel, struct sofef00_panel, panel);
 }
 
-#define sofef00_test_key_on_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0x5a, 0x5a)
-#define sofef00_test_key_off_lvl2(ctx) \
-	mipi_dsi_dcs_write_seq_multi(ctx, 0xf0, 0xa5, 0xa5)
-
 static void sofef00_panel_reset(struct sofef00_panel *ctx)
 {
 	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
@@ -62,14 +59,14 @@ static int sofef00_panel_on(struct sofef00_panel *ctx)
 	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
 	mipi_dsi_usleep_range(&dsi_ctx, 10000, 11000);
 
-	sofef00_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
-	sofef00_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
-	sofef00_test_key_on_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_on_lvl2(&dsi_ctx);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb0, 0x07);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xb6, 0x12);
-	sofef00_test_key_off_lvl2(&dsi_ctx);
+	samsung_dsi_test_key_off_lvl2(&dsi_ctx);
 
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_CONTROL_DISPLAY, 0x20);
 	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_WRITE_POWER_SAVE, 0x00);
@@ -250,30 +247,19 @@ static int sofef00_panel_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void sofef00_panel_remove(struct mipi_dsi_device *dsi)
-{
-	struct sofef00_panel *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id sofef00_panel_of_match[] = {
 	{ .compatible = "samsung,sofef00" }, /* legacy */
 	{ .compatible = "samsung,sofef00-ams628nw01" },
@@ -283,7 +269,6 @@ MODULE_DEVICE_TABLE(of, sofef00_panel_of_match);
 
 static struct mipi_dsi_driver sofef00_panel_driver = {
 	.probe = sofef00_panel_probe,
-	.remove = sofef00_panel_remove,
 	.driver = {
 		.name = "panel-samsung-sofef00",
 		.of_match_table = sofef00_panel_of_match,
diff --git a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
index 989b030ea22de5..5077893dab7c75 100644
--- a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
+++ b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
@@ -205,15 +205,7 @@ static int sharp_nt_panel_add(struct sharp_nt_panel *sharp_nt)
 	if (ret)
 		return ret;
 
-	drm_panel_add(&sharp_nt->base);
-
-	return 0;
-}
-
-static void sharp_nt_panel_del(struct sharp_nt_panel *sharp_nt)
-{
-	if (sharp_nt->base.dev)
-		drm_panel_remove(&sharp_nt->base);
+	return devm_drm_panel_add(dev, &sharp_nt->base);
 }
 
 static int sharp_nt_panel_probe(struct mipi_dsi_device *dsi)
@@ -244,25 +236,7 @@ static int sharp_nt_panel_probe(struct mipi_dsi_device *dsi)
 	if (ret < 0)
 		return ret;
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
-		sharp_nt_panel_del(sharp_nt);
-		return ret;
-	}
-
-	return 0;
-}
-
-static void sharp_nt_panel_remove(struct mipi_dsi_device *dsi)
-{
-	struct sharp_nt_panel *sharp_nt = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
-
-	sharp_nt_panel_del(sharp_nt);
+	return devm_mipi_dsi_attach(&dsi->dev, dsi);
 }
 
 static const struct of_device_id sharp_nt_of_match[] = {
@@ -277,7 +251,6 @@ static struct mipi_dsi_driver sharp_nt_panel_driver = {
 		.of_match_table = sharp_nt_of_match,
 	},
 	.probe = sharp_nt_panel_probe,
-	.remove = sharp_nt_panel_remove,
 };
 module_mipi_dsi_driver(sharp_nt_panel_driver);
 
diff --git a/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c b/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c
index 0456f3d705e72e..e59d9d32b1e0a8 100644
--- a/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c
+++ b/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c
@@ -233,30 +233,19 @@ static int sharp_ls060_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void sharp_ls060_remove(struct mipi_dsi_device *dsi)
-{
-	struct sharp_ls060 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id sharp_ls060t1sx01_of_match[] = {
 	{ .compatible = "sharp,ls060t1sx01" },
 	{ /* sentinel */ }
@@ -265,7 +254,6 @@ MODULE_DEVICE_TABLE(of, sharp_ls060t1sx01_of_match);
 
 static struct mipi_dsi_driver sharp_ls060_driver = {
 	.probe = sharp_ls060_probe,
-	.remove = sharp_ls060_remove,
 	.driver = {
 		.name = "panel-sharp-ls060t1sx01",
 		.of_match_table = sharp_ls060t1sx01_of_match,
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index c95964ba25a95d..d7dbd7442919be 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -1593,6 +1593,31 @@ static const struct panel_desc boe_bp101wx1_100 = {
 	.connector_type = DRM_MODE_CONNECTOR_LVDS,
 };
 
+static const struct display_timing boe_dv215fhm_r01_timing = {
+	.pixelclock = { 148500000, 148500000, 148500000 },
+	.hactive = { 1920, 1920, 1920 },
+	.hfront_porch = { 88, 88, 88 },
+	.hback_porch = { 148, 148, 148 },
+	.hsync_len = { 44, 44, 44 },
+	.vactive = { 1080, 1080, 1080 },
+	.vfront_porch = { 4, 4, 4 },
+	.vback_porch = { 36, 36, 36 },
+	.vsync_len = { 5, 5, 5 },
+	.flags = DISPLAY_FLAGS_DE_HIGH,
+};
+
+static const struct panel_desc boe_dv215fhm_r01 = {
+	.timings = &boe_dv215fhm_r01_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 476,
+		.height = 268,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
 static const struct display_timing boe_ev121wxm_n10_1850_timing = {
 	.pixelclock = { 69922000, 71000000, 72293000 },
 	.hactive = { 1280, 1280, 1280 },
@@ -5114,6 +5139,32 @@ static const struct panel_desc tpk_f10a_0102 = {
 	},
 };
 
+static const struct drm_display_mode truly_tft320240_229_e_mode = {
+	.clock = 6410,
+	.hdisplay = 320,
+	.hsync_start = 320 + 8,
+	.hsync_end = 320 + 8 + 39,
+	.htotal = 320 + 8 + 39 + 4,
+	.vdisplay = 240,
+	.vsync_start = 240 + 8,
+	.vsync_end = 240 + 8 + 8,
+	.vtotal = 240 + 8 + 8 + 4,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc truly_tft320240_229_e = {
+	.modes = &truly_tft320240_229_e_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 70,
+		.height = 53,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+};
+
 static const struct display_timing urt_umsh_8596md_timing = {
 	.pixelclock = { 33260000, 33260000, 33260000 },
 	.hactive = { 800, 800, 800 },
@@ -5759,6 +5810,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "boe,bp101wx1-100",
 		.data = &boe_bp101wx1_100,
+	}, {
+		.compatible = "boe,dv215fhm-r01",
+		.data = &boe_dv215fhm_r01,
 	}, {
 		.compatible = "boe,ev121wxm-n10-1850",
 		.data = &boe_ev121wxm_n10_1850,
@@ -6167,6 +6221,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "tpk,f10a-0102",
 		.data = &tpk_f10a_0102,
+	}, {
+		.compatible = "truly,tft320240-229-e",
+		.data = &truly_tft320240_229_e,
 	}, {
 		.compatible = "urt,umsh-8596md-t",
 		.data = &urt_umsh_8596md_parallel,
diff --git a/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
index a14c86c60d19d7..285f0d033efdd8 100644
--- a/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
+++ b/drivers/gpu/drm/panel/panel-sony-td4353-jdi.c
@@ -214,30 +214,19 @@ static int sony_td4353_jdi_probe(struct mipi_dsi_device *dsi)
 
 	ctx->panel.prepare_prev_first = true;
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void sony_td4353_jdi_remove(struct mipi_dsi_device *dsi)
-{
-	struct sony_td4353_jdi *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id sony_td4353_jdi_of_match[] = {
 	{ .compatible = "sony,td4353-jdi-tama", .data = (void *)TYPE_TAMA_60HZ },
 	{ /* sentinel */ }
@@ -246,7 +235,6 @@ MODULE_DEVICE_TABLE(of, sony_td4353_jdi_of_match);
 
 static struct mipi_dsi_driver sony_td4353_jdi_driver = {
 	.probe = sony_td4353_jdi_probe,
-	.remove = sony_td4353_jdi_remove,
 	.driver = {
 		.name = "panel-sony-td4353-jdi",
 		.of_match_table = sony_td4353_jdi_of_match,
diff --git a/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c b/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c
index 216a6ad8696e91..0ebd9836364e3a 100644
--- a/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c
+++ b/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c
@@ -472,30 +472,19 @@ static int truly_nt35521_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
 		return ret;
 	}
 
 	return 0;
 }
 
-static void truly_nt35521_remove(struct mipi_dsi_device *dsi)
-{
-	struct truly_nt35521 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id truly_nt35521_of_match[] = {
 	{ .compatible = "sony,tulip-truly-nt35521" },
 	{ /* sentinel */ }
@@ -504,7 +493,6 @@ MODULE_DEVICE_TABLE(of, truly_nt35521_of_match);
 
 static struct mipi_dsi_driver truly_nt35521_driver = {
 	.probe = truly_nt35521_probe,
-	.remove = truly_nt35521_remove,
 	.driver = {
 		.name = "panel-truly-nt35521",
 		.of_match_table = truly_nt35521_of_match,
diff --git a/drivers/gpu/drm/panel/panel-visionox-r66451.c b/drivers/gpu/drm/panel/panel-visionox-r66451.c
index 690cccedd438fe..1efb139bb7115c 100644
--- a/drivers/gpu/drm/panel/panel-visionox-r66451.c
+++ b/drivers/gpu/drm/panel/panel-visionox-r66451.c
@@ -304,29 +304,17 @@ static int visionox_r66451_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				"Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0) {
+	ret = devm_mipi_dsi_attach(dev, dsi);
+	if (ret < 0)
 		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
-		drm_panel_remove(&ctx->panel);
-	}
 
 	return ret;
 }
 
-static void visionox_r66451_remove(struct mipi_dsi_device *dsi)
-{
-	struct visionox_r66451 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = mipi_dsi_detach(dsi);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to detach DSI host: %d\n", ret);
-
-	drm_panel_remove(&ctx->panel);
-}
-
 static const struct of_device_id visionox_r66451_of_match[] = {
 	{.compatible = "visionox,r66451"},
 	{ /*sentinel*/ }
@@ -335,7 +323,6 @@ MODULE_DEVICE_TABLE(of, visionox_r66451_of_match);
 
 static struct mipi_dsi_driver visionox_r66451_driver = {
 	.probe = visionox_r66451_probe,
-	.remove = visionox_r66451_remove,
 	.driver = {
 		.name = "panel-visionox-r66451",
 		.of_match_table = visionox_r66451_of_match,
diff --git a/drivers/gpu/drm/panel/panel-visionox-rm69299.c b/drivers/gpu/drm/panel/panel-visionox-rm69299.c
index 50f8a84537ca2b..4309e04c35c4b6 100644
--- a/drivers/gpu/drm/panel/panel-visionox-rm69299.c
+++ b/drivers/gpu/drm/panel/panel-visionox-rm69299.c
@@ -382,31 +382,21 @@ static int visionox_rm69299_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
 				     "Failed to create backlight\n");
 
-	drm_panel_add(&ctx->panel);
+	ret = devm_drm_panel_add(dev, &ctx->panel);
+	if (ret)
+		return ret;
 
 	dsi->lanes = 4;
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_LPM |
 			  MIPI_DSI_CLOCK_NON_CONTINUOUS;
-	ret = mipi_dsi_attach(dsi);
+	ret = devm_mipi_dsi_attach(dev, dsi);
 	if (ret < 0) {
 		dev_err(dev, "dsi attach failed ret = %d\n", ret);
-		goto err_dsi_attach;
+		return ret;
 	}
 
 	return 0;
-
-err_dsi_attach:
-	drm_panel_remove(&ctx->panel);
-	return ret;
-}
-
-static void visionox_rm69299_remove(struct mipi_dsi_device *dsi)
-{
-	struct visionox_rm69299 *ctx = mipi_dsi_get_drvdata(dsi);
-
-	mipi_dsi_detach(ctx->dsi);
-	drm_panel_remove(&ctx->panel);
 }
 
 const struct visionox_rm69299_panel_desc visionox_rm69299_1080p_display_desc = {
@@ -438,7 +428,6 @@ static struct mipi_dsi_driver visionox_rm69299_driver = {
 		.of_match_table = visionox_rm69299_of_match,
 	},
 	.probe = visionox_rm69299_probe,
-	.remove = visionox_rm69299_remove,
 };
 module_mipi_dsi_driver(visionox_rm69299_driver);
 
diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
index b51c307788110b..0fe81d259274e6 100644
--- a/drivers/gpu/drm/panfrost/panfrost_devfreq.c
+++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
@@ -78,7 +78,7 @@ static int panfrost_devfreq_get_dev_status(struct device *dev,
 
 	dev_dbg(pfdev->base.dev, "busy %lu total %lu %lu %% freq %lu MHz\n",
 		status->busy_time, status->total_time,
-		status->busy_time / (status->total_time / 100),
+		status->busy_time * 100 / MAX(status->total_time, 1),
 		status->current_frequency / 1000 / 1000);
 
 	return 0;
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c
index d0a23c51be39dd..485349faf25114 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.c
+++ b/drivers/gpu/drm/panfrost/panfrost_device.c
@@ -11,7 +11,6 @@
 
 #include "panfrost_device.h"
 #include "panfrost_devfreq.h"
-#include "panfrost_features.h"
 #include "panfrost_gem.h"
 #include "panfrost_issues.h"
 #include "panfrost_gpu.h"
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index 36cc2e67a3080b..96a66d673ba605 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -662,7 +662,7 @@ static int panfrost_ioctl_query_bo_info(struct drm_device *dev, void *data,
 		if (bo->is_heap)
 			args->create_flags |= PANFROST_BO_HEAP;
 
-		if (!bo->base.map_wc)
+		if (bo->wb_mmap)
 			args->create_flags |= PANFROST_BO_WB_MMAP;
 	}
 
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
index 3a7fce4288987b..68546925f16b0d 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gem.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
@@ -7,6 +7,8 @@
 #include <linux/slab.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-mapping.h>
+#include <linux/highmem.h>
+#include <linux/seq_file.h>
 
 #include <drm/panfrost_drm.h>
 #include <drm/drm_print.h>
diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h
index 0fda64fbe5f229..b55a3f9edd414e 100644
--- a/drivers/gpu/drm/panthor/panthor_device.h
+++ b/drivers/gpu/drm/panthor/panthor_device.h
@@ -92,17 +92,21 @@ struct panthor_irq {
 	u32 mask;
 
 	/**
-	 * @mask_lock: protects modifications to _INT_MASK and @mask.
+	 * @lock: protects modifications to _INT_MASK, @mask and @state.
 	 *
 	 * In paths where _INT_MASK is updated based on a state
 	 * transition/check, it's crucial for the state update/check to be
 	 * inside the locked section, otherwise it introduces a race window
 	 * leading to potential _INT_MASK inconsistencies.
 	 */
-	spinlock_t mask_lock;
+	spinlock_t lock;
 
-	/** @state: one of &enum panthor_irq_state reflecting the current state. */
-	atomic_t state;
+	/**
+	 * @state: one of &enum panthor_irq_state reflecting the current state.
+	 *
+	 * Must be accessed with lock held.
+	 */
+	enum panthor_irq_state state;
 };
 
 /**
@@ -491,150 +495,6 @@ panthor_exception_is_fault(u32 exception_code)
 const char *panthor_exception_name(struct panthor_device *ptdev,
 				   u32 exception_code);
 
-#define INT_RAWSTAT 0x0
-#define INT_CLEAR   0x4
-#define INT_MASK    0x8
-#define INT_STAT    0xc
-
-/**
- * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt
- * registration function.
- *
- * The boiler-plate to gracefully deal with shared interrupts is
- * auto-generated. All you have to do is call PANTHOR_IRQ_HANDLER()
- * just after the actual handler. The handler prototype is:
- *
- * void (*handler)(struct panthor_device *, u32 status);
- */
-#define PANTHOR_IRQ_HANDLER(__name, __handler)							\
-static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data)			\
-{												\
-	struct panthor_irq *pirq = data;							\
-	enum panthor_irq_state old_state;							\
-												\
-	guard(spinlock_irqsave)(&pirq->mask_lock);						\
-	old_state = atomic_cmpxchg(&pirq->state,						\
-				   PANTHOR_IRQ_STATE_ACTIVE,					\
-				   PANTHOR_IRQ_STATE_PROCESSING);				\
-	if (old_state != PANTHOR_IRQ_STATE_ACTIVE)						\
-		return IRQ_NONE;								\
-												\
-	if (!gpu_read(pirq->iomem, INT_STAT)) {							\
-		atomic_cmpxchg(&pirq->state,							\
-			       PANTHOR_IRQ_STATE_PROCESSING,					\
-			       PANTHOR_IRQ_STATE_ACTIVE);					\
-		return IRQ_NONE;								\
-	}											\
-												\
-	gpu_write(pirq->iomem, INT_MASK, 0);							\
-	return IRQ_WAKE_THREAD;									\
-}												\
-												\
-static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *data)		\
-{												\
-	struct panthor_irq *pirq = data;							\
-	struct panthor_device *ptdev = pirq->ptdev;						\
-	irqreturn_t ret = IRQ_NONE;								\
-												\
-	while (true) {										\
-		/* It's safe to access pirq->mask without the lock held here. If a new		\
-		 * event gets added to the mask and the corresponding IRQ is pending,		\
-		 * we'll process it right away instead of adding an extra raw -> threaded	\
-		 * round trip. If an event is removed and the status bit is set, it will	\
-		 * be ignored, just like it would have been if the mask had been adjusted	\
-		 * right before the HW event kicks in. TLDR; it's all expected races we're	\
-		 * covered for.									\
-		 */										\
-		u32 status = gpu_read(pirq->iomem, INT_RAWSTAT) & pirq->mask;			\
-												\
-		if (!status)									\
-			break;									\
-												\
-		__handler(ptdev, status);							\
-		ret = IRQ_HANDLED;								\
-	}											\
-												\
-	scoped_guard(spinlock_irqsave, &pirq->mask_lock) {					\
-		enum panthor_irq_state old_state;						\
-												\
-		old_state = atomic_cmpxchg(&pirq->state,					\
-					   PANTHOR_IRQ_STATE_PROCESSING,			\
-					   PANTHOR_IRQ_STATE_ACTIVE);				\
-		if (old_state == PANTHOR_IRQ_STATE_PROCESSING)					\
-			gpu_write(pirq->iomem, INT_MASK, pirq->mask);				\
-	}											\
-												\
-	return ret;										\
-}												\
-												\
-static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq)			\
-{												\
-	scoped_guard(spinlock_irqsave, &pirq->mask_lock) {					\
-		atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDING);				\
-		gpu_write(pirq->iomem, INT_MASK, 0);						\
-	}											\
-	synchronize_irq(pirq->irq);								\
-	atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDED);					\
-}												\
-												\
-static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq)			\
-{												\
-	guard(spinlock_irqsave)(&pirq->mask_lock);						\
-												\
-	atomic_set(&pirq->state, PANTHOR_IRQ_STATE_ACTIVE);					\
-	gpu_write(pirq->iomem, INT_CLEAR, pirq->mask);						\
-	gpu_write(pirq->iomem, INT_MASK, pirq->mask);						\
-}												\
-												\
-static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev,			\
-					      struct panthor_irq *pirq,				\
-					      int irq, void __iomem *iomem)			\
-{												\
-	pirq->ptdev = ptdev;									\
-	pirq->irq = irq;									\
-	pirq->mask = 0;										\
-	pirq->iomem = iomem;									\
-	spin_lock_init(&pirq->mask_lock);							\
-	atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDED);					\
-	gpu_write(pirq->iomem, INT_MASK, 0);							\
-												\
-	return devm_request_threaded_irq(ptdev->base.dev, irq,					\
-					 panthor_ ## __name ## _irq_raw_handler,		\
-					 panthor_ ## __name ## _irq_threaded_handler,		\
-					 IRQF_SHARED, KBUILD_MODNAME "-" # __name,		\
-					 pirq);							\
-}												\
-												\
-static inline void panthor_ ## __name ## _irq_enable_events(struct panthor_irq *pirq, u32 mask)	\
-{												\
-	guard(spinlock_irqsave)(&pirq->mask_lock);						\
-	pirq->mask |= mask;									\
-												\
-	/* The only situation where we need to write the new mask is if the IRQ is active.	\
-	 * If it's being processed, the mask will be restored for us in _irq_threaded_handler()	\
-	 * on the PROCESSING -> ACTIVE transition.						\
-	 * If the IRQ is suspended/suspending, the mask is restored at resume time.		\
-	 */											\
-	if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE)				\
-		gpu_write(pirq->iomem, INT_MASK, pirq->mask);					\
-}												\
-												\
-static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq *pirq, u32 mask)\
-{												\
-	guard(spinlock_irqsave)(&pirq->mask_lock);						\
-	pirq->mask &= ~mask;									\
-												\
-	/* The only situation where we need to write the new mask is if the IRQ is active.	\
-	 * If it's being processed, the mask will be restored for us in _irq_threaded_handler()	\
-	 * on the PROCESSING -> ACTIVE transition.						\
-	 * If the IRQ is suspended/suspending, the mask is restored at resume time.		\
-	 */											\
-	if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE)				\
-		gpu_write(pirq->iomem, INT_MASK, pirq->mask);					\
-}
-
-extern struct workqueue_struct *panthor_cleanup_wq;
-
 static inline void gpu_write(void __iomem *iomem, u32 reg, u32 data)
 {
 	writel(data, iomem + reg);
@@ -706,4 +566,134 @@ static inline u64 gpu_read64_counter(void __iomem *iomem, u32 reg)
 	read_poll_timeout(gpu_read64_relaxed, val, cond, delay_us, timeout_us,	\
 			  false, iomem, reg)
 
+#define INT_RAWSTAT 0x0
+#define INT_CLEAR   0x4
+#define INT_MASK    0x8
+#define INT_STAT    0xc
+
+static inline irqreturn_t panthor_irq_default_raw_handler(int irq, void *data)
+{
+	struct panthor_irq *pirq = data;
+
+	guard(spinlock_irqsave)(&pirq->lock);
+	if (pirq->state != PANTHOR_IRQ_STATE_ACTIVE)
+		return IRQ_NONE;
+
+	if (!gpu_read(pirq->iomem, INT_STAT))
+		return IRQ_NONE;
+
+	pirq->state = PANTHOR_IRQ_STATE_PROCESSING;
+	gpu_write(pirq->iomem, INT_MASK, 0);
+	return IRQ_WAKE_THREAD;
+}
+
+static __always_inline irqreturn_t
+panthor_irq_default_threaded_handler(void *data,
+				     void (*slow_handler)(struct panthor_irq *, u32))
+{
+	struct panthor_irq *pirq = data;
+	irqreturn_t ret = IRQ_NONE;
+
+	while (true) {
+		/* It's safe to access pirq->mask without the lock held here. If a new
+		 * event gets added to the mask and the corresponding IRQ is pending,
+		 * we'll process it right away instead of adding an extra raw -> threaded
+		 * round trip. If an event is removed and the status bit is set, it will
+		 * be ignored, just like it would have been if the mask had been adjusted
+		 * right before the HW event kicks in. TLDR; it's all expected races we're
+		 * covered for.
+		 */
+		u32 status = gpu_read(pirq->iomem, INT_RAWSTAT) & pirq->mask;
+
+		if (!status)
+			break;
+
+		slow_handler(pirq, status);
+		ret = IRQ_HANDLED;
+	}
+
+	scoped_guard(spinlock_irqsave, &pirq->lock) {
+		if (pirq->state == PANTHOR_IRQ_STATE_PROCESSING) {
+			pirq->state = PANTHOR_IRQ_STATE_ACTIVE;
+			gpu_write(pirq->iomem, INT_MASK, pirq->mask);
+		}
+	}
+
+	return ret;
+}
+
+static inline void panthor_irq_suspend(struct panthor_irq *pirq)
+{
+	scoped_guard(spinlock_irqsave, &pirq->lock) {
+		pirq->state = PANTHOR_IRQ_STATE_SUSPENDING;
+		gpu_write(pirq->iomem, INT_MASK, 0);
+	}
+	synchronize_irq(pirq->irq);
+	scoped_guard(spinlock_irqsave, &pirq->lock)
+		pirq->state = PANTHOR_IRQ_STATE_SUSPENDED;
+}
+
+static inline void panthor_irq_resume(struct panthor_irq *pirq)
+{
+	guard(spinlock_irqsave)(&pirq->lock);
+	pirq->state = PANTHOR_IRQ_STATE_ACTIVE;
+	gpu_write(pirq->iomem, INT_CLEAR, pirq->mask);
+	gpu_write(pirq->iomem, INT_MASK, pirq->mask);
+}
+
+static inline void panthor_irq_enable_events(struct panthor_irq *pirq, u32 mask)
+{
+	guard(spinlock_irqsave)(&pirq->lock);
+	pirq->mask |= mask;
+
+	/* The only situation where we need to write the new mask is if the IRQ is active.
+	 * If it's being processed, the mask will be restored for us in _irq_threaded_handler()
+	 * on the PROCESSING -> ACTIVE transition.
+	 * If the IRQ is suspended/suspending, the mask is restored at resume time.
+	 */
+	if (pirq->state == PANTHOR_IRQ_STATE_ACTIVE)
+		gpu_write(pirq->iomem, INT_MASK, pirq->mask);
+}
+
+static inline void panthor_irq_disable_events(struct panthor_irq *pirq, u32 mask)
+{
+	guard(spinlock_irqsave)(&pirq->lock);
+	pirq->mask &= ~mask;
+
+	/* The only situation where we need to write the new mask is if the IRQ is active.
+	 * If it's being processed, the mask will be restored for us in _irq_threaded_handler()
+	 * on the PROCESSING -> ACTIVE transition.
+	 * If the IRQ is suspended/suspending, the mask is restored at resume time.
+	 */
+	if (pirq->state == PANTHOR_IRQ_STATE_ACTIVE)
+		gpu_write(pirq->iomem, INT_MASK, pirq->mask);
+}
+
+static inline int
+panthor_irq_request(struct panthor_device *ptdev, struct panthor_irq *pirq,
+		    int irq, void __iomem *iomem, const char *name,
+		    irqreturn_t (*threaded_handler)(int, void *data))
+{
+	const char *full_name;
+
+	pirq->ptdev = ptdev;
+	pirq->irq = irq;
+	pirq->mask = 0;
+	pirq->iomem = iomem;
+	spin_lock_init(&pirq->lock);
+	pirq->state = PANTHOR_IRQ_STATE_SUSPENDED;
+
+	full_name = devm_kasprintf(ptdev->base.dev, GFP_KERNEL, KBUILD_MODNAME "-%s", name);
+	if (!full_name)
+		return -ENOMEM;
+
+	gpu_write(pirq->iomem, INT_MASK, 0);
+	return devm_request_threaded_irq(ptdev->base.dev, irq,
+					 panthor_irq_default_raw_handler,
+					 threaded_handler,
+					 IRQF_SHARED, full_name, pirq);
+}
+
+extern struct workqueue_struct *panthor_cleanup_wq;
+
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c
index 46a3080b0b206e..8cdba0c1a14b9e 100644
--- a/drivers/gpu/drm/panthor/panthor_drv.c
+++ b/drivers/gpu/drm/panthor/panthor_drv.c
@@ -1769,6 +1769,7 @@ static void panthor_debugfs_init(struct drm_minor *minor)
 {
 	panthor_mmu_debugfs_init(minor);
 	panthor_gem_debugfs_init(minor);
+	panthor_sched_debugfs_init(minor);
 }
 #endif
 
diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c
index 10b7e12c1f1eda..4f1fab66a13bf1 100644
--- a/drivers/gpu/drm/panthor/panthor_fw.c
+++ b/drivers/gpu/drm/panthor/panthor_fw.c
@@ -1095,8 +1095,9 @@ static void panthor_fw_init_global_iface(struct panthor_device *ptdev)
 			 msecs_to_jiffies(PING_INTERVAL_MS));
 }
 
-static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status)
+static void panthor_job_irq_handler(struct panthor_irq *pirq, u32 status)
 {
+	struct panthor_device *ptdev = pirq->ptdev;
 	u32 duration;
 	u64 start = 0;
 
@@ -1122,7 +1123,11 @@ static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status)
 		trace_gpu_job_irq(ptdev->base.dev, status, duration);
 	}
 }
-PANTHOR_IRQ_HANDLER(job, panthor_job_irq_handler);
+
+static irqreturn_t panthor_job_irq_threaded_handler(int irq, void *data)
+{
+	return panthor_irq_default_threaded_handler(data, panthor_job_irq_handler);
+}
 
 static int panthor_fw_start(struct panthor_device *ptdev)
 {
@@ -1130,8 +1135,8 @@ static int panthor_fw_start(struct panthor_device *ptdev)
 	bool timedout = false;
 
 	ptdev->fw->booted = false;
-	panthor_job_irq_enable_events(&ptdev->fw->irq, ~0);
-	panthor_job_irq_resume(&ptdev->fw->irq);
+	panthor_irq_enable_events(&ptdev->fw->irq, ~0);
+	panthor_irq_resume(&ptdev->fw->irq);
 	gpu_write(fw->iomem, MCU_CONTROL, MCU_CONTROL_AUTO);
 
 	if (!wait_event_timeout(ptdev->fw->req_waitqueue,
@@ -1241,7 +1246,7 @@ void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang)
 			ptdev->reset.fast = true;
 	}
 
-	panthor_job_irq_suspend(&ptdev->fw->irq);
+	panthor_irq_suspend(&ptdev->fw->irq);
 	panthor_fw_stop(ptdev);
 }
 
@@ -1310,7 +1315,7 @@ void panthor_fw_unplug(struct panthor_device *ptdev)
 
 	if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev)) {
 		/* Make sure the IRQ handler cannot be called after that point. */
-		panthor_job_irq_suspend(&ptdev->fw->irq);
+		panthor_irq_suspend(&ptdev->fw->irq);
 		panthor_fw_stop(ptdev);
 	}
 
@@ -1505,8 +1510,9 @@ int panthor_fw_init(struct panthor_device *ptdev)
 	if (irq <= 0)
 		return -ENODEV;
 
-	ret = panthor_request_job_irq(ptdev, &fw->irq, irq,
-				      ptdev->iomem + JOB_INT_BASE);
+	ret = panthor_irq_request(ptdev, &fw->irq, irq,
+				  ptdev->iomem + JOB_INT_BASE, "job",
+				  panthor_job_irq_threaded_handler);
 	if (ret) {
 		drm_err(&ptdev->base, "failed to request job irq");
 		return ret;
diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c
index 72908be5e144f0..ea1230e6e53beb 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.c
+++ b/drivers/gpu/drm/panthor/panthor_gem.c
@@ -9,6 +9,9 @@
 #include <linux/dma-buf.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/pagemap.h>
+#include <linux/swap.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 
@@ -1410,8 +1413,7 @@ panthor_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 	return count ? count : SHRINK_EMPTY;
 }
 
-static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj,
-					       struct ww_acquire_ctx *ticket)
+static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj)
 {
 	/*
 	 * Track last locked entry for unwinding locks in error and
@@ -1497,8 +1499,7 @@ static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj,
 	return ret == 0;
 }
 
-static bool panthor_gem_try_evict(struct drm_gem_object *obj,
-				  struct ww_acquire_ctx *ticket)
+static bool panthor_gem_try_evict(struct drm_gem_object *obj)
 {
 	struct panthor_gem_object *bo = to_panthor_bo(obj);
 
@@ -1506,7 +1507,7 @@ static bool panthor_gem_try_evict(struct drm_gem_object *obj,
 	if (dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_BOOKKEEP, false, 10) <= 0)
 		return false;
 
-	return panthor_gem_try_evict_no_resv_wait(&bo->base, ticket);
+	return panthor_gem_try_evict_no_resv_wait(&bo->base);
 }
 
 static unsigned long
@@ -1521,13 +1522,13 @@ panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 
 	freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.unused,
 				  sc->nr_to_scan - freed, &remaining,
-				  panthor_gem_try_evict_no_resv_wait, NULL);
+				  panthor_gem_try_evict_no_resv_wait);
 	if (freed >= sc->nr_to_scan)
 		goto out;
 
 	freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.mmapped,
 				  sc->nr_to_scan - freed, &remaining,
-				  panthor_gem_try_evict_no_resv_wait, NULL);
+				  panthor_gem_try_evict_no_resv_wait);
 	if (freed >= sc->nr_to_scan)
 		goto out;
 
@@ -1541,7 +1542,7 @@ panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 
 	freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.gpu_mapped_shared,
 				  sc->nr_to_scan - freed, &remaining,
-				  panthor_gem_try_evict, NULL);
+				  panthor_gem_try_evict);
 
 out:
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c
index c013d6bf9a599e..a383b04f101ed1 100644
--- a/drivers/gpu/drm/panthor/panthor_gpu.c
+++ b/drivers/gpu/drm/panthor/panthor_gpu.c
@@ -86,8 +86,9 @@ static void panthor_gpu_l2_config_set(struct panthor_device *ptdev)
 	gpu_write(gpu->iomem, GPU_L2_CONFIG, l2_config);
 }
 
-static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status)
+static void panthor_gpu_irq_handler(struct panthor_irq *pirq, u32 status)
 {
+	struct panthor_device *ptdev = pirq->ptdev;
 	struct panthor_gpu *gpu = ptdev->gpu;
 
 	gpu_write(gpu->irq.iomem, INT_CLEAR, status);
@@ -116,7 +117,11 @@ static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status)
 	}
 	spin_unlock(&ptdev->gpu->reqs_lock);
 }
-PANTHOR_IRQ_HANDLER(gpu, panthor_gpu_irq_handler);
+
+static irqreturn_t panthor_gpu_irq_threaded_handler(int irq, void *data)
+{
+	return panthor_irq_default_threaded_handler(data, panthor_gpu_irq_handler);
+}
 
 /**
  * panthor_gpu_unplug() - Called when the GPU is unplugged.
@@ -128,7 +133,7 @@ void panthor_gpu_unplug(struct panthor_device *ptdev)
 
 	/* Make sure the IRQ handler is not running after that point. */
 	if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev))
-		panthor_gpu_irq_suspend(&ptdev->gpu->irq);
+		panthor_irq_suspend(&ptdev->gpu->irq);
 
 	/* Wake-up all waiters. */
 	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
@@ -169,13 +174,14 @@ int panthor_gpu_init(struct panthor_device *ptdev)
 	if (irq < 0)
 		return irq;
 
-	ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq,
-				      ptdev->iomem + GPU_INT_BASE);
+	ret = panthor_irq_request(ptdev, &ptdev->gpu->irq, irq,
+				  ptdev->iomem + GPU_INT_BASE, "gpu",
+				  panthor_gpu_irq_threaded_handler);
 	if (ret)
 		return ret;
 
-	panthor_gpu_irq_enable_events(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK);
-	panthor_gpu_irq_resume(&ptdev->gpu->irq);
+	panthor_irq_enable_events(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK);
+	panthor_irq_resume(&ptdev->gpu->irq);
 	return 0;
 }
 
@@ -183,7 +189,7 @@ int panthor_gpu_power_changed_on(struct panthor_device *ptdev)
 {
 	guard(pm_runtime_active)(ptdev->base.dev);
 
-	panthor_gpu_irq_enable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
+	panthor_irq_enable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
 
 	return 0;
 }
@@ -192,7 +198,7 @@ void panthor_gpu_power_changed_off(struct panthor_device *ptdev)
 {
 	guard(pm_runtime_active)(ptdev->base.dev);
 
-	panthor_gpu_irq_disable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
+	panthor_irq_disable_events(&ptdev->gpu->irq, GPU_POWER_INTERRUPTS_MASK);
 }
 
 /**
@@ -317,6 +323,21 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev)
 	return panthor_gpu_power_on(ptdev, L2, 1, 20000);
 }
 
+static inline void panthor_gpu_emit_flush_caches_tp(struct panthor_device *ptdev,
+						    u64 start, u32 l2, u32 lsc,
+						    u32 other, int ret)
+{
+	u32 duration;
+
+	if (!tracepoint_enabled(gpu_cache_flush) || !start)
+		return;
+
+	if (check_sub_overflow(ktime_get_ns(), start, &duration))
+		duration = U32_MAX;
+
+	trace_gpu_cache_flush(ptdev->base.dev, l2, lsc, other, duration, ret);
+}
+
 /**
  * panthor_gpu_flush_caches() - Flush caches
  * @ptdev: Device.
@@ -330,36 +351,40 @@ int panthor_gpu_flush_caches(struct panthor_device *ptdev,
 			     u32 l2, u32 lsc, u32 other)
 {
 	struct panthor_gpu *gpu = ptdev->gpu;
-	unsigned long flags;
+	u64 start = 0;
 	int ret = 0;
 
 	/* Serialize cache flush operations. */
 	guard(mutex)(&ptdev->gpu->cache_flush_lock);
 
-	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
-	if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) {
-		ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED;
-		gpu_write(gpu->iomem, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other));
-	} else {
-		ret = -EIO;
+	scoped_guard(spinlock, &ptdev->gpu->reqs_lock) {
+		if (tracepoint_enabled(gpu_cache_flush))
+			start = ktime_get_ns();
+
+		if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) {
+			ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED;
+			gpu_write(gpu->iomem, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other));
+		} else {
+			panthor_gpu_emit_flush_caches_tp(ptdev, start, l2, lsc,
+							 other, -EIO);
+			return -EIO;
+		}
 	}
-	spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
-
-	if (ret)
-		return ret;
 
 	if (!wait_event_timeout(ptdev->gpu->reqs_acked,
 				!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED),
 				msecs_to_jiffies(100))) {
-		spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
-		if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 &&
-		    !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED))
-			ret = -ETIMEDOUT;
-		else
-			ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED;
-		spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
+		scoped_guard(spinlock, &ptdev->gpu->reqs_lock) {
+			if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 &&
+			    !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED))
+				ret = -ETIMEDOUT;
+			else
+				ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED;
+		}
 	}
 
+	panthor_gpu_emit_flush_caches_tp(ptdev, start, l2, lsc, other, ret);
+
 	if (ret) {
 		panthor_device_schedule_reset(ptdev);
 		drm_err(&ptdev->base, "Flush caches timeout");
@@ -378,27 +403,26 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev)
 {
 	struct panthor_gpu *gpu = ptdev->gpu;
 	bool timedout = false;
-	unsigned long flags;
 
-	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
-	if (!drm_WARN_ON(&ptdev->base,
-			 ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) {
-		ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED;
-		gpu_write(gpu->irq.iomem, INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
-		gpu_write(gpu->iomem, GPU_CMD, GPU_SOFT_RESET);
+	scoped_guard(spinlock, &ptdev->gpu->reqs_lock) {
+		if (!drm_WARN_ON(&ptdev->base,
+				ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) {
+			ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED;
+			gpu_write(gpu->irq.iomem, INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
+			gpu_write(gpu->iomem, GPU_CMD, GPU_SOFT_RESET);
+		}
 	}
-	spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
 
 	if (!wait_event_timeout(ptdev->gpu->reqs_acked,
 				!(ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED),
 				msecs_to_jiffies(100))) {
-		spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
-		if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 &&
-		    !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED))
-			timedout = true;
-		else
-			ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED;
-		spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
+		scoped_guard(spinlock, &ptdev->gpu->reqs_lock) {
+			if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 &&
+			    !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED))
+				timedout = true;
+			else
+				ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED;
+		}
 	}
 
 	if (timedout) {
@@ -406,7 +430,10 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev)
 		return -ETIMEDOUT;
 	}
 
-	ptdev->gpu->pending_reqs = 0;
+	scoped_guard(spinlock, &ptdev->gpu->reqs_lock) {
+		ptdev->gpu->pending_reqs = 0;
+	}
+
 	return 0;
 }
 
@@ -425,7 +452,7 @@ void panthor_gpu_suspend(struct panthor_device *ptdev)
 	else
 		panthor_hw_l2_power_off(ptdev);
 
-	panthor_gpu_irq_suspend(&ptdev->gpu->irq);
+	panthor_irq_suspend(&ptdev->gpu->irq);
 }
 
 /**
@@ -437,7 +464,7 @@ void panthor_gpu_suspend(struct panthor_device *ptdev)
  */
 void panthor_gpu_resume(struct panthor_device *ptdev)
 {
-	panthor_gpu_irq_resume(&ptdev->gpu->irq);
+	panthor_irq_resume(&ptdev->gpu->irq);
 	panthor_hw_l2_power_on(ptdev);
 }
 
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
index 9f63a048df61c0..d75d575473da4a 100644
--- a/drivers/gpu/drm/panthor/panthor_mmu.c
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -601,17 +601,13 @@ static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
 	return BIT(as);
 }
 
-/* Forward declaration to call helpers within as_enable/disable */
-static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status);
-PANTHOR_IRQ_HANDLER(mmu, panthor_mmu_irq_handler);
-
 static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
 				 u64 transtab, u64 transcfg, u64 memattr)
 {
 	struct panthor_mmu *mmu = ptdev->mmu;
 
-	panthor_mmu_irq_enable_events(&ptdev->mmu->irq,
-				      panthor_mmu_as_fault_mask(ptdev, as_nr));
+	panthor_irq_enable_events(&ptdev->mmu->irq,
+				  panthor_mmu_as_fault_mask(ptdev, as_nr));
 
 	gpu_write64(mmu->iomem, AS_TRANSTAB(as_nr), transtab);
 	gpu_write64(mmu->iomem, AS_MEMATTR(as_nr), memattr);
@@ -629,8 +625,8 @@ static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr,
 
 	lockdep_assert_held(&ptdev->mmu->as.slots_lock);
 
-	panthor_mmu_irq_disable_events(&ptdev->mmu->irq,
-				       panthor_mmu_as_fault_mask(ptdev, as_nr));
+	panthor_irq_disable_events(&ptdev->mmu->irq,
+				   panthor_mmu_as_fault_mask(ptdev, as_nr));
 
 	/* Flush+invalidate RW caches, invalidate RO ones. */
 	ret = panthor_gpu_flush_caches(ptdev, CACHE_CLEAN | CACHE_INV,
@@ -1859,8 +1855,9 @@ static void panthor_vm_unlock_region(struct panthor_vm *vm)
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
 }
 
-static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
+static void panthor_mmu_irq_handler(struct panthor_irq *pirq, u32 status)
 {
+	struct panthor_device *ptdev = pirq->ptdev;
 	struct panthor_mmu *mmu = ptdev->mmu;
 	bool has_unhandled_faults = false;
 
@@ -1923,6 +1920,11 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 		panthor_sched_report_mmu_fault(ptdev);
 }
 
+static irqreturn_t panthor_mmu_irq_threaded_handler(int irq, void *data)
+{
+	return panthor_irq_default_threaded_handler(data, panthor_mmu_irq_handler);
+}
+
 /**
  * panthor_mmu_suspend() - Suspend the MMU logic
  * @ptdev: Device.
@@ -1947,7 +1949,7 @@ void panthor_mmu_suspend(struct panthor_device *ptdev)
 	}
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
 
-	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+	panthor_irq_suspend(&ptdev->mmu->irq);
 }
 
 /**
@@ -1966,7 +1968,7 @@ void panthor_mmu_resume(struct panthor_device *ptdev)
 	ptdev->mmu->as.faulty_mask = 0;
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
 
-	panthor_mmu_irq_resume(&ptdev->mmu->irq);
+	panthor_irq_resume(&ptdev->mmu->irq);
 }
 
 /**
@@ -1983,7 +1985,7 @@ void panthor_mmu_pre_reset(struct panthor_device *ptdev)
 {
 	struct panthor_vm *vm;
 
-	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+	panthor_irq_suspend(&ptdev->mmu->irq);
 
 	mutex_lock(&ptdev->mmu->vm.lock);
 	ptdev->mmu->vm.reset_in_progress = true;
@@ -2020,7 +2022,7 @@ void panthor_mmu_post_reset(struct panthor_device *ptdev)
 
 	mutex_unlock(&ptdev->mmu->as.slots_lock);
 
-	panthor_mmu_irq_resume(&ptdev->mmu->irq);
+	panthor_irq_resume(&ptdev->mmu->irq);
 
 	/* Restart the VM_BIND queues. */
 	mutex_lock(&ptdev->mmu->vm.lock);
@@ -3278,8 +3280,7 @@ int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm
 unsigned long
 panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
 			     unsigned int nr_to_scan, unsigned long *remaining,
-			     bool (*shrink)(struct drm_gem_object *,
-					    struct ww_acquire_ctx *))
+			     bool (*shrink)(struct drm_gem_object *))
 {
 	unsigned long freed = 0;
 	LIST_HEAD(remaining_vms);
@@ -3305,7 +3306,7 @@ panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
 
 		freed += drm_gem_lru_scan(&ptdev->base, &vm->reclaim.lru,
 					  nr_to_scan - freed,
-					  remaining, shrink, NULL);
+					  remaining, shrink);
 
 		mutex_lock(&ptdev->base.gem_lru_mutex);
 
@@ -3353,7 +3354,7 @@ panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
 void panthor_mmu_unplug(struct panthor_device *ptdev)
 {
 	if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev))
-		panthor_mmu_irq_suspend(&ptdev->mmu->irq);
+		panthor_irq_suspend(&ptdev->mmu->irq);
 
 	mutex_lock(&ptdev->mmu->as.slots_lock);
 	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
@@ -3414,8 +3415,9 @@ int panthor_mmu_init(struct panthor_device *ptdev)
 	if (irq <= 0)
 		return -ENODEV;
 
-	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
-				      ptdev->iomem + MMU_INT_BASE);
+	ret = panthor_irq_request(ptdev, &mmu->irq, irq,
+				  ptdev->iomem + MMU_INT_BASE, "mmu",
+				  panthor_mmu_irq_threaded_handler);
 	if (ret)
 		return ret;
 
@@ -3436,8 +3438,8 @@ int panthor_mmu_init(struct panthor_device *ptdev)
 	if (ret)
 		return ret;
 
-	panthor_mmu_irq_enable_events(&mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
-	panthor_mmu_irq_resume(&mmu->irq);
+	panthor_irq_enable_events(&mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
+	panthor_irq_resume(&mmu->irq);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h
index 3522fbbce369cb..abc36e7204be06 100644
--- a/drivers/gpu/drm/panthor/panthor_mmu.h
+++ b/drivers/gpu/drm/panthor/panthor_mmu.h
@@ -52,8 +52,7 @@ int panthor_vm_evict_bo_mappings_locked(struct panthor_gem_object *bo);
 unsigned long
 panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
 			     unsigned int nr_to_scan, unsigned long *remaining,
-			     bool (*shrink)(struct drm_gem_object *,
-					    struct ww_acquire_ctx *));
+			     bool (*shrink)(struct drm_gem_object *));
 int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec,
 					struct panthor_vm *vm,
 					u32 slot_count);
diff --git a/drivers/gpu/drm/panthor/panthor_pwr.c b/drivers/gpu/drm/panthor/panthor_pwr.c
index f2c2c300059051..dd7b6ef8ea2099 100644
--- a/drivers/gpu/drm/panthor/panthor_pwr.c
+++ b/drivers/gpu/drm/panthor/panthor_pwr.c
@@ -56,8 +56,9 @@ struct panthor_pwr {
 	wait_queue_head_t reqs_acked;
 };
 
-static void panthor_pwr_irq_handler(struct panthor_device *ptdev, u32 status)
+static void panthor_pwr_irq_handler(struct panthor_irq *pirq, u32 status)
 {
+	struct panthor_device *ptdev = pirq->ptdev;
 	struct panthor_pwr *pwr = ptdev->pwr;
 
 	spin_lock(&ptdev->pwr->reqs_lock);
@@ -75,7 +76,11 @@ static void panthor_pwr_irq_handler(struct panthor_device *ptdev, u32 status)
 	}
 	spin_unlock(&ptdev->pwr->reqs_lock);
 }
-PANTHOR_IRQ_HANDLER(pwr, panthor_pwr_irq_handler);
+
+static irqreturn_t panthor_pwr_irq_threaded_handler(int irq, void *data)
+{
+	return panthor_irq_default_threaded_handler(data, panthor_pwr_irq_handler);
+}
 
 static void panthor_pwr_write_command(struct panthor_device *ptdev, u32 command, u64 args)
 {
@@ -454,7 +459,7 @@ void panthor_pwr_unplug(struct panthor_device *ptdev)
 
 	/* Make sure the IRQ handler is not running after that point. */
 	if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev))
-		panthor_pwr_irq_suspend(&ptdev->pwr->irq);
+		panthor_irq_suspend(&ptdev->pwr->irq);
 
 	/* Wake-up all waiters. */
 	spin_lock_irqsave(&ptdev->pwr->reqs_lock, flags);
@@ -484,13 +489,14 @@ int panthor_pwr_init(struct panthor_device *ptdev)
 	if (irq < 0)
 		return irq;
 
-	err = panthor_request_pwr_irq(ptdev, &pwr->irq, irq,
-				      pwr->iomem + PWR_INT_BASE);
+	err = panthor_irq_request(ptdev, &pwr->irq, irq,
+				  pwr->iomem + PWR_INT_BASE, "pwr",
+				  panthor_pwr_irq_threaded_handler);
 	if (err)
 		return err;
 
-	panthor_pwr_irq_enable_events(&pwr->irq, PWR_INTERRUPTS_MASK);
-	panthor_pwr_irq_resume(&pwr->irq);
+	panthor_irq_enable_events(&pwr->irq, PWR_INTERRUPTS_MASK);
+	panthor_irq_resume(&pwr->irq);
 	return 0;
 }
 
@@ -566,7 +572,7 @@ void panthor_pwr_suspend(struct panthor_device *ptdev)
 	if (!ptdev->pwr)
 		return;
 
-	panthor_pwr_irq_suspend(&ptdev->pwr->irq);
+	panthor_irq_suspend(&ptdev->pwr->irq);
 }
 
 void panthor_pwr_resume(struct panthor_device *ptdev)
@@ -574,5 +580,5 @@ void panthor_pwr_resume(struct panthor_device *ptdev)
 	if (!ptdev->pwr)
 		return;
 
-	panthor_pwr_irq_resume(&ptdev->pwr->irq);
+	panthor_irq_resume(&ptdev->pwr->irq);
 }
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c
index 5778e23e20a087..ac6aee554820eb 100644
--- a/drivers/gpu/drm/panthor/panthor_sched.c
+++ b/drivers/gpu/drm/panthor/panthor_sched.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 /* Copyright 2023 Collabora ltd. */
 
+#include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_exec.h>
 #include <drm/drm_file.h>
@@ -177,23 +178,6 @@ struct panthor_scheduler {
 	 */
 	struct work_struct sync_upd_work;
 
-	/**
-	 * @fw_events_work: Work used to process FW events outside the interrupt path.
-	 *
-	 * Even if the interrupt is threaded, we need any event processing
-	 * that require taking the panthor_scheduler::lock to be processed
-	 * outside the interrupt path so we don't block the tick logic when
-	 * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the
-	 * event processing requires taking this lock, we just delegate all
-	 * FW event processing to the scheduler workqueue.
-	 */
-	struct work_struct fw_events_work;
-
-	/**
-	 * @fw_events: Bitmask encoding pending FW events.
-	 */
-	atomic_t fw_events;
-
 	/**
 	 * @resched_target: When the next tick should occur.
 	 *
@@ -254,8 +238,22 @@ struct panthor_scheduler {
 		struct list_head waiting;
 	} groups;
 
+	/**
+	 * @events_lock: Lock taken when processing events.
+	 *
+	 * This also needs to be taken when csg_slots are updated, to make sure
+	 * the event processing logic doesn't touch groups that have left the CSG
+	 * slot.
+	 */
+	spinlock_t events_lock;
+
 	/**
 	 * @csg_slots: FW command stream group slots.
+	 *
+	 * Updates to the group binding must happen with both
+	 * panthor_scheduler::lock and panthor_scheduler::events_lock held.
+	 * The group binding may therefore be read while holding either lock.
+	 * Other slot fields are protected by panthor_scheduler::lock.
 	 */
 	struct panthor_csg_slot csg_slots[MAX_CSGS];
 
@@ -565,11 +563,13 @@ struct panthor_group {
 	/** @idle_queues: Bitmask reflecting the idle queues. */
 	u32 idle_queues;
 
-	/** @fatal_lock: Lock used to protect access to fatal fields. */
-	spinlock_t fatal_lock;
-
-	/** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */
-	u32 fatal_queues;
+	/**
+	 * @fatal_queues: Bitmask reflecting the queues that hit a fatal exception.
+	 *
+	 * This is an atomic because we don't want to acquire the events_lock
+	 * every time we need to check the group state.
+	 */
+	atomic_t fatal_queues;
 
 	/** @tiler_oom: Mask of queues that have a tiler OOM event to process. */
 	atomic_t tiler_oom;
@@ -605,8 +605,14 @@ struct panthor_group {
 	 * any timeout situation is unrecoverable, and the group becomes useless. We
 	 * simply wait for all references to be dropped so we can release the group
 	 * object.
+	 *
+	 * This is an atomic because it can be set from both a scheduling context
+	 * (protected with panthor_scheduler::lock) and an event processing context
+	 * (protected with panthor_scheduler::events_lock). We could protect access
+	 * with the events_lock, but this is simpler to make it an atomic since the
+	 * only allowed transition is false -> true.
 	 */
-	bool timedout;
+	atomic_t timedout;
 
 	/**
 	 * @innocent: True when the group becomes unusable because the group suspension
@@ -676,9 +682,6 @@ struct panthor_group {
 	 */
 	struct panthor_kernel_bo *protm_suspend_buf;
 
-	/** @sync_upd_work: Work used to check/signal job fences. */
-	struct work_struct sync_upd_work;
-
 	/** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */
 	struct work_struct tiler_oom_work;
 
@@ -988,87 +991,6 @@ group_get(struct panthor_group *group)
 	return group;
 }
 
-/**
- * group_bind_locked() - Bind a group to a group slot
- * @group: Group.
- * @csg_id: Slot.
- *
- * Return: 0 on success, a negative error code otherwise.
- */
-static int
-group_bind_locked(struct panthor_group *group, u32 csg_id)
-{
-	struct panthor_device *ptdev = group->ptdev;
-	struct panthor_csg_slot *csg_slot;
-	int ret;
-
-	lockdep_assert_held(&ptdev->scheduler->lock);
-
-	if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS ||
-			ptdev->scheduler->csg_slots[csg_id].group))
-		return -EINVAL;
-
-	ret = panthor_vm_active(group->vm);
-	if (ret)
-		return ret;
-
-	csg_slot = &ptdev->scheduler->csg_slots[csg_id];
-	group_get(group);
-	group->csg_id = csg_id;
-
-	/* Dummy doorbell allocation: doorbell is assigned to the group and
-	 * all queues use the same doorbell.
-	 *
-	 * TODO: Implement LRU-based doorbell assignment, so the most often
-	 * updated queues get their own doorbell, thus avoiding useless checks
-	 * on queues belonging to the same group that are rarely updated.
-	 */
-	for (u32 i = 0; i < group->queue_count; i++)
-		group->queues[i]->doorbell_id = csg_id + 1;
-
-	csg_slot->group = group;
-
-	return 0;
-}
-
-/**
- * group_unbind_locked() - Unbind a group from a slot.
- * @group: Group to unbind.
- *
- * Return: 0 on success, a negative error code otherwise.
- */
-static int
-group_unbind_locked(struct panthor_group *group)
-{
-	struct panthor_device *ptdev = group->ptdev;
-	struct panthor_csg_slot *slot;
-
-	lockdep_assert_held(&ptdev->scheduler->lock);
-
-	if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS))
-		return -EINVAL;
-
-	if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE))
-		return -EINVAL;
-
-	slot = &ptdev->scheduler->csg_slots[group->csg_id];
-	panthor_vm_idle(group->vm);
-	group->csg_id = -1;
-
-	/* Tiler OOM events will be re-issued next time the group is scheduled. */
-	atomic_set(&group->tiler_oom, 0);
-	if (cancel_work(&group->tiler_oom_work))
-		group_put(group);
-
-	for (u32 i = 0; i < group->queue_count; i++)
-		group->queues[i]->doorbell_id = -1;
-
-	slot->group = NULL;
-
-	group_put(group);
-	return 0;
-}
-
 static bool
 group_is_idle(struct panthor_group *group)
 {
@@ -1082,8 +1004,9 @@ group_can_run(struct panthor_group *group)
 {
 	return group->state != PANTHOR_CS_GROUP_TERMINATED &&
 	       group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE &&
-	       !group->destroyed && group->fatal_queues == 0 &&
-	       !group->timedout;
+	       !group->destroyed &&
+	       !atomic_read(&group->fatal_queues) &&
+	       !atomic_read(&group->timedout);
 }
 
 static bool
@@ -1482,7 +1405,7 @@ cs_slot_process_fatal_event_locked(struct panthor_device *ptdev,
 	u32 fatal;
 	u64 info;
 
-	lockdep_assert_held(&sched->lock);
+	lockdep_assert_held(&sched->events_lock);
 
 	cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
 	fatal = cs_iface->output->fatal;
@@ -1492,7 +1415,7 @@ cs_slot_process_fatal_event_locked(struct panthor_device *ptdev,
 		drm_warn(&ptdev->base, "CS_FATAL: pid=%d, comm=%s\n",
 			 group->task_info.pid, group->task_info.comm);
 
-		group->fatal_queues |= BIT(cs_id);
+		atomic_or(BIT(cs_id), &group->fatal_queues);
 	}
 
 	if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) {
@@ -1530,7 +1453,7 @@ cs_slot_process_fault_event_locked(struct panthor_device *ptdev,
 	u32 fault;
 	u64 info;
 
-	lockdep_assert_held(&sched->lock);
+	lockdep_assert_held(&sched->events_lock);
 
 	cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
 	fault = cs_iface->output->fault;
@@ -1622,7 +1545,7 @@ static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id)
 	 */
 	if (ret && ret != -ENOMEM) {
 		drm_warn(&ptdev->base, "Failed to extend the tiler heap\n");
-		group->fatal_queues |= BIT(cs_id);
+		atomic_or(BIT(cs_id), &group->fatal_queues);
 		sched_queue_delayed_work(sched, tick, 0);
 		goto out_put_heap_pool;
 	}
@@ -1682,7 +1605,7 @@ cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev,
 	struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
 	struct panthor_group *group = csg_slot->group;
 
-	lockdep_assert_held(&sched->lock);
+	lockdep_assert_held(&sched->events_lock);
 
 	if (drm_WARN_ON(&ptdev->base, !group))
 		return;
@@ -1703,7 +1626,7 @@ static bool cs_slot_process_irq_locked(struct panthor_device *ptdev,
 	struct panthor_fw_cs_iface *cs_iface;
 	u32 req, ack, events;
 
-	lockdep_assert_held(&ptdev->scheduler->lock);
+	lockdep_assert_held(&ptdev->scheduler->events_lock);
 
 	cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
 	req = cs_iface->input->req;
@@ -1731,9 +1654,7 @@ static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32
 {
 	struct panthor_scheduler *sched = ptdev->scheduler;
 
-	lockdep_assert_held(&sched->lock);
-
-	sched->might_have_idle_groups = true;
+	lockdep_assert_held(&sched->events_lock);
 
 	/* Schedule a tick so we can evict idle groups and schedule non-idle
 	 * ones. This will also update runtime PM and devfreq busy/idle states,
@@ -1742,16 +1663,102 @@ static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32
 	sched_queue_delayed_work(sched, tick, 0);
 }
 
+static void update_fdinfo_stats(struct panthor_job *job)
+{
+	struct panthor_group *group = job->group;
+	struct panthor_queue *queue = group->queues[job->queue_idx];
+	struct panthor_gpu_usage *fdinfo = &group->fdinfo.data;
+	struct panthor_job_profiling_data *slots = queue->profiling.slots->kmap;
+	struct panthor_job_profiling_data *data = &slots[job->profiling.slot];
+
+	scoped_guard(spinlock, &group->fdinfo.lock) {
+		if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_CYCLES)
+			fdinfo->cycles += data->cycles.after - data->cycles.before;
+		if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_TIMESTAMP)
+			fdinfo->time += data->time.after - data->time.before;
+	}
+}
+
+static bool queue_check_job_completion(struct panthor_queue *queue)
+{
+	struct panthor_syncobj_64b *syncobj = NULL;
+	struct panthor_job *job, *job_tmp;
+	bool cookie, progress = false;
+	LIST_HEAD(done_jobs);
+
+	cookie = dma_fence_begin_signalling();
+	scoped_guard(spinlock_irqsave, &queue->fence_ctx.lock) {
+		list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {
+			if (!syncobj) {
+				struct panthor_group *group = job->group;
+
+				syncobj = group->syncobjs->kmap +
+					  (job->queue_idx * sizeof(*syncobj));
+			}
+
+			if (syncobj->seqno < job->done_fence->seqno)
+				break;
+
+			list_move_tail(&job->node, &done_jobs);
+			dma_fence_signal_locked(job->done_fence);
+		}
+
+		if (list_empty(&queue->fence_ctx.in_flight_jobs)) {
+			/* If we have no job left, we cancel the timer, and reset remaining
+			 * time to its default so it can be restarted next time
+			 * queue_resume_timeout() is called.
+			 */
+			queue_suspend_timeout_locked(queue);
+
+			/* If there's no job pending, we consider it progress to avoid a
+			 * spurious timeout if the timeout handler and the sync update
+			 * handler raced.
+			 */
+			progress = true;
+		} else if (!list_empty(&done_jobs)) {
+			queue_reset_timeout_locked(queue);
+			progress = true;
+		}
+	}
+	dma_fence_end_signalling(cookie);
+
+	list_for_each_entry_safe(job, job_tmp, &done_jobs, node) {
+		if (job->profiling.mask)
+			update_fdinfo_stats(job);
+		list_del_init(&job->node);
+		panthor_job_put(&job->base);
+	}
+
+	return progress;
+}
+
+static void group_check_job_completion(struct panthor_group *group)
+{
+	u32 queue_idx;
+	bool cookie;
+
+	cookie = dma_fence_begin_signalling();
+	for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {
+		struct panthor_queue *queue = group->queues[queue_idx];
+
+		if (!queue)
+			continue;
+
+		queue_check_job_completion(queue);
+	}
+	dma_fence_end_signalling(cookie);
+}
+
 static void csg_slot_sync_update_locked(struct panthor_device *ptdev,
 					u32 csg_id)
 {
 	struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
 	struct panthor_group *group = csg_slot->group;
 
-	lockdep_assert_held(&ptdev->scheduler->lock);
+	lockdep_assert_held(&ptdev->scheduler->events_lock);
 
 	if (group)
-		group_queue_work(group, sync_upd);
+		group_check_job_completion(group);
 
 	sched_queue_work(ptdev->scheduler, sync_upd);
 }
@@ -1763,14 +1770,14 @@ csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 c
 	struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
 	struct panthor_group *group = csg_slot->group;
 
-	lockdep_assert_held(&sched->lock);
+	lockdep_assert_held(&sched->events_lock);
 
 	group = csg_slot->group;
 	if (!drm_WARN_ON(&ptdev->base, !group)) {
 		drm_warn(&ptdev->base, "CSG_PROGRESS_TIMER_EVENT: pid=%d, comm=%s\n",
 			 group->task_info.pid, group->task_info.comm);
 
-		group->timedout = true;
+		atomic_set(&group->timedout, true);
 	}
 
 	drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id);
@@ -1784,7 +1791,7 @@ static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_i
 	struct panthor_fw_csg_iface *csg_iface;
 	u32 ring_cs_db_mask = 0;
 
-	lockdep_assert_held(&ptdev->scheduler->lock);
+	lockdep_assert_held(&ptdev->scheduler->events_lock);
 
 	if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
 		return;
@@ -1842,7 +1849,7 @@ static void sched_process_idle_event_locked(struct panthor_device *ptdev)
 {
 	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
 
-	lockdep_assert_held(&ptdev->scheduler->lock);
+	lockdep_assert_held(&ptdev->scheduler->events_lock);
 
 	/* Acknowledge the idle event and schedule a tick. */
 	panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE);
@@ -1858,7 +1865,7 @@ static void sched_process_global_irq_locked(struct panthor_device *ptdev)
 	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
 	u32 req, ack, evts;
 
-	lockdep_assert_held(&ptdev->scheduler->lock);
+	lockdep_assert_held(&ptdev->scheduler->events_lock);
 
 	req = READ_ONCE(glb_iface->input->req);
 	ack = READ_ONCE(glb_iface->output->ack);
@@ -1868,14 +1875,17 @@ static void sched_process_global_irq_locked(struct panthor_device *ptdev)
 		sched_process_idle_event_locked(ptdev);
 }
 
-static void process_fw_events_work(struct work_struct *work)
+/**
+ * panthor_sched_report_fw_events() - Report FW events to the scheduler.
+ * @ptdev: Device.
+ * @events: Bitmask of pending FW events to report.
+ */
+void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)
 {
-	struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
-						      fw_events_work);
-	u32 events = atomic_xchg(&sched->fw_events, 0);
-	struct panthor_device *ptdev = sched->ptdev;
+	if (!ptdev->scheduler)
+		return;
 
-	mutex_lock(&sched->lock);
+	guard(spinlock)(&ptdev->scheduler->events_lock);
 
 	if (events & JOB_INT_GLOBAL_IF) {
 		sched_process_global_irq_locked(ptdev);
@@ -1888,22 +1898,89 @@ static void process_fw_events_work(struct work_struct *work)
 		sched_process_csg_irq_locked(ptdev, csg_id);
 		events &= ~BIT(csg_id);
 	}
+}
 
-	mutex_unlock(&sched->lock);
+/**
+ * group_bind_locked() - Bind a group to a group slot
+ * @group: Group.
+ * @csg_id: Slot.
+ *
+ * Return: 0 on success, a negative error code otherwise.
+ */
+static int
+group_bind_locked(struct panthor_group *group, u32 csg_id)
+{
+	struct panthor_device *ptdev = group->ptdev;
+	int ret;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
+
+	if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS ||
+			ptdev->scheduler->csg_slots[csg_id].group))
+		return -EINVAL;
+
+	ret = panthor_vm_active(group->vm);
+	if (ret)
+		return ret;
+
+	group_get(group);
+
+	/* Dummy doorbell allocation: doorbell is assigned to the group and
+	 * all queues use the same doorbell.
+	 *
+	 * TODO: Implement LRU-based doorbell assignment, so the most often
+	 * updated queues get their own doorbell, thus avoiding useless checks
+	 * on queues belonging to the same group that are rarely updated.
+	 */
+	for (u32 i = 0; i < group->queue_count; i++)
+		group->queues[i]->doorbell_id = csg_id + 1;
+
+	scoped_guard(spinlock, &ptdev->scheduler->events_lock) {
+		ptdev->scheduler->csg_slots[csg_id].group = group;
+		group->csg_id = csg_id;
+	}
+
+	return 0;
 }
 
 /**
- * panthor_sched_report_fw_events() - Report FW events to the scheduler.
- * @ptdev: Device.
- * @events: Bitmask of pending FW events to report.
+ * group_unbind_locked() - Unbind a group from a slot.
+ * @group: Group to unbind.
+ *
+ * Return: 0 on success, a negative error code otherwise.
  */
-void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)
+static int
+group_unbind_locked(struct panthor_group *group)
 {
-	if (!ptdev->scheduler)
-		return;
+	struct panthor_device *ptdev = group->ptdev;
+
+	lockdep_assert_held(&ptdev->scheduler->lock);
 
-	atomic_or(events, &ptdev->scheduler->fw_events);
-	sched_queue_work(ptdev->scheduler, fw_events);
+	if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS))
+		return -EINVAL;
+
+	if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE))
+		return -EINVAL;
+
+	scoped_guard(spinlock, &ptdev->scheduler->events_lock) {
+		/* Process all pending IRQs before returning the slot. */
+		sched_process_csg_irq_locked(ptdev, group->csg_id);
+		ptdev->scheduler->csg_slots[group->csg_id].group = NULL;
+		group->csg_id = -1;
+	}
+
+	panthor_vm_idle(group->vm);
+
+	/* Tiler OOM events will be re-issued next time the group is scheduled. */
+	atomic_set(&group->tiler_oom, 0);
+	if (cancel_work(&group->tiler_oom_work))
+		group_put(group);
+
+	for (u32 i = 0; i < group->queue_count; i++)
+		group->queues[i]->doorbell_id = -1;
+
+	group_put(group);
+	return 0;
 }
 
 static const char *fence_get_driver_name(struct dma_fence *fence)
@@ -2136,11 +2213,12 @@ tick_ctx_init(struct panthor_scheduler *sched,
 		 * CSG IRQs, so we can flag the faulty queue.
 		 */
 		if (panthor_vm_has_unhandled_faults(group->vm)) {
-			sched_process_csg_irq_locked(ptdev, i);
+			scoped_guard(spinlock, &sched->events_lock)
+				sched_process_csg_irq_locked(ptdev, i);
 
 			/* No fatal fault reported, flag all queues as faulty. */
-			if (!group->fatal_queues)
-				group->fatal_queues |= GENMASK(group->queue_count - 1, 0);
+			atomic_cmpxchg(&group->fatal_queues, 0,
+				       GENMASK(group->queue_count - 1, 0));
 		}
 
 		tick_ctx_insert_old_group(sched, ctx, group);
@@ -2173,9 +2251,9 @@ group_term_post_processing(struct panthor_group *group)
 		struct panthor_syncobj_64b *syncobj;
 		int err;
 
-		if (group->fatal_queues & BIT(i))
+		if (atomic_read(&group->fatal_queues) & BIT(i))
 			err = -EINVAL;
-		else if (group->timedout)
+		else if (atomic_read(&group->timedout))
 			err = -ETIMEDOUT;
 		else
 			err = -ECANCELED;
@@ -2331,16 +2409,8 @@ tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *c
 
 	/* Unbind evicted groups. */
 	for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
-		list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
-			/* This group is gone. Process interrupts to clear
-			 * any pending interrupts before we start the new
-			 * group.
-			 */
-			if (group->csg_id >= 0)
-				sched_process_csg_irq_locked(ptdev, group->csg_id);
-
+		list_for_each_entry(group, &ctx->old_groups[prio], run_node)
 			group_unbind_locked(group);
-		}
 	}
 
 	for (i = 0; i < sched->csg_slot_count; i++) {
@@ -2866,7 +2936,7 @@ void panthor_sched_suspend(struct panthor_device *ptdev)
 			/* We consider group suspension failures as fatal and flag the
 			 * group as unusable by setting timedout=true.
 			 */
-			csg_slot->group->timedout = true;
+			atomic_set(&csg_slot->group->timedout, true);
 
 			csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
 						CSG_STATE_TERMINATE,
@@ -2915,10 +2985,12 @@ void panthor_sched_suspend(struct panthor_device *ptdev)
 			u32 csg_id = ffs(slot_mask) - 1;
 			struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
 
-			if (flush_caches_failed)
+			if (flush_caches_failed) {
 				csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
-			else
+			} else {
+				guard(spinlock)(&sched->events_lock);
 				csg_slot_sync_update_locked(ptdev, csg_id);
+			}
 
 			slot_mask &= ~BIT(csg_id);
 		}
@@ -2932,10 +3004,6 @@ void panthor_sched_suspend(struct panthor_device *ptdev)
 			continue;
 
 		group_get(group);
-
-		if (group->csg_id >= 0)
-			sched_process_csg_irq_locked(ptdev, group->csg_id);
-
 		group_unbind_locked(group);
 
 		drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node));
@@ -3018,22 +3086,6 @@ void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed)
 	}
 }
 
-static void update_fdinfo_stats(struct panthor_job *job)
-{
-	struct panthor_group *group = job->group;
-	struct panthor_queue *queue = group->queues[job->queue_idx];
-	struct panthor_gpu_usage *fdinfo = &group->fdinfo.data;
-	struct panthor_job_profiling_data *slots = queue->profiling.slots->kmap;
-	struct panthor_job_profiling_data *data = &slots[job->profiling.slot];
-
-	scoped_guard(spinlock, &group->fdinfo.lock) {
-		if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_CYCLES)
-			fdinfo->cycles += data->cycles.after - data->cycles.before;
-		if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_TIMESTAMP)
-			fdinfo->time += data->time.after - data->time.before;
-	}
-}
-
 void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile)
 {
 	struct panthor_group_pool *gpool = pfile->groups;
@@ -3054,80 +3106,6 @@ void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile)
 	xa_unlock(&gpool->xa);
 }
 
-static bool queue_check_job_completion(struct panthor_queue *queue)
-{
-	struct panthor_syncobj_64b *syncobj = NULL;
-	struct panthor_job *job, *job_tmp;
-	bool cookie, progress = false;
-	LIST_HEAD(done_jobs);
-
-	cookie = dma_fence_begin_signalling();
-	scoped_guard(spinlock_irqsave, &queue->fence_ctx.lock) {
-		list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {
-			if (!syncobj) {
-				struct panthor_group *group = job->group;
-
-				syncobj = group->syncobjs->kmap +
-					  (job->queue_idx * sizeof(*syncobj));
-			}
-
-			if (syncobj->seqno < job->done_fence->seqno)
-				break;
-
-			list_move_tail(&job->node, &done_jobs);
-			dma_fence_signal_locked(job->done_fence);
-		}
-
-		if (list_empty(&queue->fence_ctx.in_flight_jobs)) {
-			/* If we have no job left, we cancel the timer, and reset remaining
-			 * time to its default so it can be restarted next time
-			 * queue_resume_timeout() is called.
-			 */
-			queue_suspend_timeout_locked(queue);
-
-			/* If there's no job pending, we consider it progress to avoid a
-			 * spurious timeout if the timeout handler and the sync update
-			 * handler raced.
-			 */
-			progress = true;
-		} else if (!list_empty(&done_jobs)) {
-			queue_reset_timeout_locked(queue);
-			progress = true;
-		}
-	}
-	dma_fence_end_signalling(cookie);
-
-	list_for_each_entry_safe(job, job_tmp, &done_jobs, node) {
-		if (job->profiling.mask)
-			update_fdinfo_stats(job);
-		list_del_init(&job->node);
-		panthor_job_put(&job->base);
-	}
-
-	return progress;
-}
-
-static void group_sync_upd_work(struct work_struct *work)
-{
-	struct panthor_group *group =
-		container_of(work, struct panthor_group, sync_upd_work);
-	u32 queue_idx;
-	bool cookie;
-
-	cookie = dma_fence_begin_signalling();
-	for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {
-		struct panthor_queue *queue = group->queues[queue_idx];
-
-		if (!queue)
-			continue;
-
-		queue_check_job_completion(queue);
-	}
-	dma_fence_end_signalling(cookie);
-
-	group_put(group);
-}
-
 struct panthor_job_ringbuf_instrs {
 	u64 buffer[MAX_INSTRS_PER_JOB];
 	u32 count;
@@ -3426,7 +3404,7 @@ queue_timedout_job(struct drm_sched_job *sched_job)
 	queue_stop(queue, job);
 
 	mutex_lock(&sched->lock);
-	group->timedout = true;
+	atomic_set(&group->timedout, true);
 	if (group->csg_id >= 0) {
 		sched_queue_delayed_work(ptdev->scheduler, tick, 0);
 	} else {
@@ -3678,7 +3656,6 @@ int panthor_group_create(struct panthor_file *pfile,
 	if (!group)
 		return -ENOMEM;
 
-	spin_lock_init(&group->fatal_lock);
 	kref_init(&group->refcount);
 	group->state = PANTHOR_CS_GROUP_CREATED;
 	group->csg_id = -1;
@@ -3695,7 +3672,6 @@ int panthor_group_create(struct panthor_file *pfile,
 	INIT_LIST_HEAD(&group->wait_node);
 	INIT_LIST_HEAD(&group->run_node);
 	INIT_WORK(&group->term_work, group_term_work);
-	INIT_WORK(&group->sync_upd_work, group_sync_upd_work);
 	INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work);
 	INIT_WORK(&group->release_work, group_release_work);
 
@@ -3850,12 +3826,13 @@ int panthor_group_get_state(struct panthor_file *pfile,
 	memset(get_state, 0, sizeof(*get_state));
 
 	mutex_lock(&sched->lock);
-	if (group->timedout)
+	if (atomic_read(&group->timedout))
 		get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT;
-	if (group->fatal_queues) {
+
+	get_state->fatal_queues = atomic_read(&group->fatal_queues);
+	if (get_state->fatal_queues)
 		get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT;
-		get_state->fatal_queues = group->fatal_queues;
-	}
+
 	if (group->innocent)
 		get_state->state |= DRM_PANTHOR_GROUP_STATE_INNOCENT;
 	mutex_unlock(&sched->lock);
@@ -4066,7 +4043,6 @@ void panthor_sched_unplug(struct panthor_device *ptdev)
 	struct panthor_scheduler *sched = ptdev->scheduler;
 
 	disable_delayed_work_sync(&sched->tick_work);
-	disable_work_sync(&sched->fw_events_work);
 	disable_work_sync(&sched->sync_upd_work);
 
 	mutex_lock(&sched->lock);
@@ -4151,7 +4127,8 @@ int panthor_sched_init(struct panthor_device *ptdev)
 	sched->tick_period = msecs_to_jiffies(10);
 	INIT_DELAYED_WORK(&sched->tick_work, tick_work);
 	INIT_WORK(&sched->sync_upd_work, sync_upd_work);
-	INIT_WORK(&sched->fw_events_work, process_fw_events_work);
+
+	spin_lock_init(&sched->events_lock);
 
 	ret = drmm_mutex_init(&ptdev->base, &sched->lock);
 	if (ret)
@@ -4199,3 +4176,75 @@ int panthor_sched_init(struct panthor_device *ptdev)
 	ptdev->scheduler = sched;
 	return 0;
 }
+
+#ifdef CONFIG_DEBUG_FS
+
+static const char *
+panthor_sched_prio_str(enum panthor_csg_priority prio)
+{
+	switch (prio) {
+	case PANTHOR_CSG_PRIORITY_LOW:
+		return "LOW";
+	case PANTHOR_CSG_PRIORITY_MEDIUM:
+		return "MEDIUM";
+	case PANTHOR_CSG_PRIORITY_HIGH:
+		return "HIGH";
+	case PANTHOR_CSG_PRIORITY_RT:
+		return "REAL-TIME";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static int show_file_groups(struct drm_file *file, struct seq_file *m)
+{
+	struct panthor_file *pfile = file->driver_priv;
+	struct panthor_group *group;
+	unsigned long i;
+
+	if (IS_ERR_OR_NULL(pfile->groups))
+		return -ENOENT;
+
+	xa_lock(&pfile->groups->xa);
+	xa_for_each_marked(&pfile->groups->xa, i, group, GROUP_REGISTERED) {
+		seq_printf(m, "client_id %8llu pid %8d command %s: group %2lu priority %s queue count %u\n",
+			   file->client_id, group->task_info.pid, group->task_info.comm, i,
+			   panthor_sched_prio_str(group->priority), group->queue_count);
+	}
+	xa_unlock(&pfile->groups->xa);
+
+	return 0;
+}
+
+static int show_each_file(struct seq_file *m, void *arg)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *ddev = node->minor->dev;
+	int (*show)(struct drm_file *, struct seq_file *) =
+		node->info_ent->data;
+	struct drm_file *file;
+	int ret;
+
+	scoped_cond_guard(mutex_intr, return -EINTR, &ddev->filelist_mutex) {
+		list_for_each_entry(file, &ddev->filelist, lhead) {
+			ret = show(file, m);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static struct drm_info_list panthor_sched_debugfs_list[] = {
+	{ "sched_groups", show_each_file, 0, show_file_groups },
+};
+
+void panthor_sched_debugfs_init(struct drm_minor *minor)
+{
+	drm_debugfs_create_files(panthor_sched_debugfs_list,
+				 ARRAY_SIZE(panthor_sched_debugfs_list),
+				 minor->debugfs_root, minor);
+}
+
+#endif /* CONFIG_DEBUG_FS */
diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h
index 9a8692de8adedb..d87cbdba4d40b2 100644
--- a/drivers/gpu/drm/panthor/panthor_sched.h
+++ b/drivers/gpu/drm/panthor/panthor_sched.h
@@ -10,6 +10,7 @@ struct drm_file;
 struct drm_gem_object;
 struct drm_sched_job;
 struct drm_memory_stats;
+struct drm_minor;
 struct drm_panthor_group_create;
 struct drm_panthor_queue_create;
 struct drm_panthor_group_get_state;
@@ -55,4 +56,8 @@ void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events);
 
 void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile);
 
+#ifdef CONFIG_DEBUG_FS
+void panthor_sched_debugfs_init(struct drm_minor *minor);
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_trace.h b/drivers/gpu/drm/panthor/panthor_trace.h
index 6ffeb4fe65992b..bd8652549ab463 100644
--- a/drivers/gpu/drm/panthor/panthor_trace.h
+++ b/drivers/gpu/drm/panthor/panthor_trace.h
@@ -76,6 +76,44 @@ TRACE_EVENT(gpu_job_irq,
 		  __entry->events, __entry->duration_ns)
 );
 
+/**
+ * gpu_cache_flush - emitted after cache flush completes
+ * @dev: pointer to the &struct device, for printing the device name
+ * @l2: "l2" flush flags
+ * @lsc: "lsc" flush flags
+ * @other: "other" flush flags
+ * @duration_ns: how long the cache flush operation took, in nanoseconds
+ * @ret: return status, 0 == success, negative errno on error
+ *
+ * Begins measuring after any initial lock contention around the locks needed
+ * for flushing caches, but before the actual cache flush is requested. Stops
+ * measuring and is emitted after flush operation is over.
+ */
+TRACE_EVENT(gpu_cache_flush,
+	    TP_PROTO(const struct device *dev, u32 l2, u32 lsc, u32 other,
+		     u32 duration_ns, int ret),
+	    TP_ARGS(dev, l2, lsc, other, duration_ns, ret),
+	    TP_STRUCT__entry(
+		    __string(dev_name, dev_name(dev))
+		    __field(u32, l2)
+		    __field(u32, lsc)
+		    __field(u32, other)
+		    __field(u32, duration_ns)
+		    __field(int, ret)
+	    ),
+	    TP_fast_assign(
+		    __assign_str(dev_name);
+		    __entry->l2          = l2;
+		    __entry->lsc         = lsc;
+		    __entry->other       = other;
+		    __entry->duration_ns = duration_ns;
+		    __entry->ret         = ret;
+	    ),
+	    TP_printk("%s: l2=0x%x lsc=0x%x other=0x%x duration_ns=%u ret=%d",
+		      __get_str(dev_name), __entry->l2, __entry->lsc,
+		      __entry->other, __entry->duration_ns, __entry->ret)
+);
+
 #endif /* __PANTHOR_TRACE_H__ */
 
 #undef TRACE_INCLUDE_PATH
diff --git a/drivers/gpu/drm/pl111/Kconfig b/drivers/gpu/drm/pl111/Kconfig
index 82e918820950c7..20201ec20aa7de 100644
--- a/drivers/gpu/drm/pl111/Kconfig
+++ b/drivers/gpu/drm/pl111/Kconfig
@@ -9,6 +9,7 @@ config DRM_PL111
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
 	select DRM_BRIDGE
+	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
 	help
 	  Choose this option for DRM support for the PL111 CLCD controller.
diff --git a/drivers/gpu/drm/pl111/pl111_display.c b/drivers/gpu/drm/pl111/pl111_display.c
index 5d10bc5fdf1f64..f5fe5c5e0df318 100644
--- a/drivers/gpu/drm/pl111/pl111_display.c
+++ b/drivers/gpu/drm/pl111/pl111_display.c
@@ -15,6 +15,7 @@
 #include <linux/media-bus-format.h>
 #include <linux/of_graph.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
@@ -37,7 +38,7 @@ irqreturn_t pl111_irq(int irq, void *data)
 		return IRQ_NONE;
 
 	if (irq_stat & CLCD_IRQ_NEXTBASE_UPDATE) {
-		drm_crtc_handle_vblank(&priv->pipe.crtc);
+		drm_crtc_handle_vblank(&priv->crtc);
 
 		status = IRQ_HANDLED;
 	}
@@ -49,10 +50,10 @@ irqreturn_t pl111_irq(int irq, void *data)
 }
 
 static enum drm_mode_status
-pl111_mode_valid(struct drm_simple_display_pipe *pipe,
-		 const struct drm_display_mode *mode)
+pl111_crtc_helper_mode_valid(struct drm_crtc *crtc,
+			     const struct drm_display_mode *mode)
 {
-	struct drm_device *drm = pipe->crtc.dev;
+	struct drm_device *drm = crtc->dev;
 	struct pl111_drm_dev_private *priv = drm->dev_private;
 	u32 cpp = DIV_ROUND_UP(priv->variant->fb_depth, 8);
 	u64 bw;
@@ -83,13 +84,34 @@ pl111_mode_valid(struct drm_simple_display_pipe *pipe,
 	return MODE_OK;
 }
 
-static int pl111_display_check(struct drm_simple_display_pipe *pipe,
-			       struct drm_plane_state *pstate,
-			       struct drm_crtc_state *cstate)
+static int pl111_plane_helper_atomic_check(struct drm_plane *plane,
+					   struct drm_atomic_commit *commit)
 {
-	const struct drm_display_mode *mode = &cstate->mode;
-	struct drm_framebuffer *old_fb = pipe->plane.state->fb;
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_plane_state *old_pstate = drm_atomic_get_old_plane_state(commit, plane);
+	struct drm_crtc_state *cstate = NULL;
+	const struct drm_display_mode *mode;
+	struct drm_framebuffer *old_fb = old_pstate->fb;
 	struct drm_framebuffer *fb = pstate->fb;
+	int ret;
+
+	if (pstate->crtc) {
+		cstate = drm_atomic_get_crtc_state(commit, pstate->crtc);
+		if (IS_ERR(cstate))
+			return PTR_ERR(cstate);
+	}
+
+	ret = drm_atomic_helper_check_plane_state(pstate, cstate,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, false);
+	if (ret)
+		return ret;
+
+	if (!pstate->visible)
+		return 0;
+
+	mode = &cstate->mode;
 
 	if (mode->hdisplay % 16)
 		return -EINVAL;
@@ -117,16 +139,15 @@ static int pl111_display_check(struct drm_simple_display_pipe *pipe,
 	return 0;
 }
 
-static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
-				 struct drm_crtc_state *cstate,
-				 struct drm_plane_state *plane_state)
+static void pl111_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					    struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_plane *plane = &pipe->plane;
 	struct drm_device *drm = crtc->dev;
 	struct pl111_drm_dev_private *priv = drm->dev_private;
+	struct drm_crtc_state *cstate = drm_atomic_get_new_crtc_state(commit, crtc);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, &priv->plane);
 	const struct drm_display_mode *mode = &cstate->mode;
-	struct drm_framebuffer *fb = plane->state->fb;
+	struct drm_framebuffer *fb = plane_state->fb;
 	struct drm_connector *connector = priv->connector;
 	struct drm_bridge *bridge = priv->bridge;
 	bool grayscale = false;
@@ -267,14 +288,12 @@ static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
 		if (priv->variant->st_bitmux_control)
 			cntl |= CNTL_ST_LCDBPP24_PACKED;
 		break;
-	case DRM_FORMAT_ABGR8888:
 	case DRM_FORMAT_XBGR8888:
 		if (priv->variant->st_bitmux_control)
 			cntl |= CNTL_LCDBPP24 | CNTL_BGR;
 		else
 			cntl |= CNTL_LCDBPP24;
 		break;
-	case DRM_FORMAT_ARGB8888:
 	case DRM_FORMAT_XRGB8888:
 		if (priv->variant->st_bitmux_control)
 			cntl |= CNTL_LCDBPP24;
@@ -297,13 +316,11 @@ static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
 		else
 			cntl |= CNTL_LCDBPP16_565 | CNTL_BGR;
 		break;
-	case DRM_FORMAT_ABGR1555:
 	case DRM_FORMAT_XBGR1555:
 		cntl |= CNTL_LCDBPP16;
 		if (priv->variant->st_bitmux_control)
 			cntl |= CNTL_ST_1XBPP_5551 | CNTL_BGR;
 		break;
-	case DRM_FORMAT_ARGB1555:
 	case DRM_FORMAT_XRGB1555:
 		cntl |= CNTL_LCDBPP16;
 		if (priv->variant->st_bitmux_control)
@@ -311,13 +328,11 @@ static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
 		else
 			cntl |= CNTL_BGR;
 		break;
-	case DRM_FORMAT_ABGR4444:
 	case DRM_FORMAT_XBGR4444:
 		cntl |= CNTL_LCDBPP16_444;
 		if (priv->variant->st_bitmux_control)
 			cntl |= CNTL_ST_1XBPP_444 | CNTL_BGR;
 		break;
-	case DRM_FORMAT_ARGB4444:
 	case DRM_FORMAT_XRGB4444:
 		cntl |= CNTL_LCDBPP16_444;
 		if (priv->variant->st_bitmux_control)
@@ -355,9 +370,9 @@ static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
 		drm_crtc_vblank_on(crtc);
 }
 
-static void pl111_display_disable(struct drm_simple_display_pipe *pipe)
+static void pl111_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					     struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct pl111_drm_dev_private *priv = drm->dev_private;
 	u32 cntl;
@@ -387,38 +402,43 @@ static void pl111_display_disable(struct drm_simple_display_pipe *pipe)
 	clk_disable_unprepare(priv->clk);
 }
 
-static void pl111_display_update(struct drm_simple_display_pipe *pipe,
-				 struct drm_plane_state *old_pstate)
+static void pl111_plane_helper_atomic_update(struct drm_plane *plane,
+					     struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_device *drm = crtc->dev;
+	struct drm_device *drm = plane->dev;
 	struct pl111_drm_dev_private *priv = drm->dev_private;
-	struct drm_pending_vblank_event *event = crtc->state->event;
-	struct drm_plane *plane = &pipe->plane;
-	struct drm_plane_state *pstate = plane->state;
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
 	struct drm_framebuffer *fb = pstate->fb;
 
-	if (fb) {
-		u32 addr = drm_fb_dma_get_gem_addr(fb, pstate, 0);
+	if (!fb)
+		return;
 
-		writel(addr, priv->regs + CLCD_UBAS);
-	}
+	u32 addr = drm_fb_dma_get_gem_addr(fb, pstate, 0);
 
-	if (event) {
-		crtc->state->event = NULL;
+	writel(addr, priv->regs + CLCD_UBAS);
+}
 
-		spin_lock_irq(&crtc->dev->event_lock);
-		if (crtc->state->active && drm_crtc_vblank_get(crtc) == 0)
-			drm_crtc_arm_vblank_event(crtc, event);
-		else
-			drm_crtc_send_vblank_event(crtc, event);
-		spin_unlock_irq(&crtc->dev->event_lock);
-	}
+static void pl111_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+					   struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *cstate = drm_atomic_get_new_crtc_state(commit, crtc);
+	struct drm_pending_vblank_event *event = cstate->event;
+
+	if (!event)
+		return;
+
+	cstate->event = NULL;
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	if (cstate->active && drm_crtc_vblank_get(crtc) == 0)
+		drm_crtc_arm_vblank_event(crtc, event);
+	else
+		drm_crtc_send_vblank_event(crtc, event);
+	spin_unlock_irq(&crtc->dev->event_lock);
 }
 
-static int pl111_display_enable_vblank(struct drm_simple_display_pipe *pipe)
+static int pl111_display_enable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct pl111_drm_dev_private *priv = drm->dev_private;
 
@@ -427,21 +447,62 @@ static int pl111_display_enable_vblank(struct drm_simple_display_pipe *pipe)
 	return 0;
 }
 
-static void pl111_display_disable_vblank(struct drm_simple_display_pipe *pipe)
+static void pl111_display_disable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct pl111_drm_dev_private *priv = drm->dev_private;
 
 	writel(0, priv->regs + priv->ienb);
 }
 
-static struct drm_simple_display_pipe_funcs pl111_display_funcs = {
-	.mode_valid = pl111_mode_valid,
-	.check = pl111_display_check,
-	.enable = pl111_display_enable,
-	.disable = pl111_display_disable,
-	.update = pl111_display_update,
+static int pl111_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static struct drm_crtc_funcs pl111_crtc_funcs = {
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.destroy		= drm_crtc_cleanup,
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_crtc_helper_funcs pl111_crtc_helper_funcs = {
+	.mode_valid	= pl111_crtc_helper_mode_valid,
+	.atomic_check	= pl111_crtc_helper_atomic_check,
+	.atomic_enable	= pl111_crtc_helper_atomic_enable,
+	.atomic_disable	= pl111_crtc_helper_atomic_disable,
+	.atomic_flush	= pl111_crtc_helper_atomic_flush,
+};
+
+static const struct drm_plane_funcs pl111_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
+	.destroy		= drm_plane_cleanup,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+};
+
+static const struct drm_plane_helper_funcs pl111_plane_helper_funcs = {
+	.prepare_fb	= drm_gem_plane_helper_prepare_fb,
+	.atomic_check	= pl111_plane_helper_atomic_check,
+	.atomic_update	= pl111_plane_helper_atomic_update,
+};
+
+static const struct drm_encoder_funcs pl111_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
 static int pl111_clk_div_choose_div(struct clk_hw *hw, unsigned long rate,
@@ -583,18 +644,40 @@ int pl111_display_init(struct drm_device *drm)
 		return ret;
 
 	if (!priv->variant->broken_vblank) {
-		pl111_display_funcs.enable_vblank = pl111_display_enable_vblank;
-		pl111_display_funcs.disable_vblank = pl111_display_disable_vblank;
+		pl111_crtc_funcs.enable_vblank = pl111_display_enable_vblank;
+		pl111_crtc_funcs.disable_vblank = pl111_display_disable_vblank;
 	}
 
-	ret = drm_simple_display_pipe_init(drm, &priv->pipe,
-					   &pl111_display_funcs,
-					   priv->variant->formats,
-					   priv->variant->nformats,
-					   NULL,
-					   priv->connector);
+	ret = drm_universal_plane_init(drm, &priv->plane, 0,
+				       &pl111_plane_funcs,
+				       priv->variant->formats,
+				       priv->variant->nformats,
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+
+	drm_plane_helper_add(&priv->plane, &pl111_plane_helper_funcs);
+
+	ret = drm_crtc_init_with_planes(drm, &priv->crtc, &priv->plane,
+					NULL, &pl111_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+
+	drm_crtc_helper_add(&priv->crtc, &pl111_crtc_helper_funcs);
+
+	ret = drm_encoder_init(drm, &priv->encoder, &pl111_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
 	if (ret)
 		return ret;
 
+	priv->encoder.possible_crtcs = drm_crtc_mask(&priv->crtc);
+
+	if (priv->connector) {
+		ret = drm_connector_attach_encoder(priv->connector,
+						   &priv->encoder);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/pl111/pl111_drm.h b/drivers/gpu/drm/pl111/pl111_drm.h
index d1fe756444ee52..ec92a5a180a84d 100644
--- a/drivers/gpu/drm/pl111/pl111_drm.h
+++ b/drivers/gpu/drm/pl111/pl111_drm.h
@@ -21,7 +21,6 @@
 #include <drm/drm_encoder.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_panel.h>
-#include <drm/drm_simple_kms_helper.h>
 
 /*
  * CLCD Controller Internal Register addresses
@@ -135,7 +134,9 @@ struct pl111_drm_dev_private {
 	struct drm_connector *connector;
 	struct drm_panel *panel;
 	struct drm_bridge *bridge;
-	struct drm_simple_display_pipe pipe;
+	struct drm_plane plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 
 	void *regs;
 	u32 memory_bw;
diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c
index 8ec659b3c08eb2..89d44e970510ba 100644
--- a/drivers/gpu/drm/pl111/pl111_drv.c
+++ b/drivers/gpu/drm/pl111/pl111_drv.c
@@ -169,8 +169,7 @@ static int pl111_modeset_init(struct drm_device *dev)
 		goto out_bridge;
 	}
 
-	ret = drm_simple_display_pipe_attach_bridge(&priv->pipe,
-						    bridge);
+	ret = drm_bridge_attach(&priv->encoder, bridge, NULL, 0);
 	if (ret)
 		return ret;
 
@@ -342,13 +341,9 @@ static void pl111_amba_shutdown(struct amba_device *amba_dev)
  * This early variant lacks the 565 and 444 pixel formats.
  */
 static const u32 pl110_pixel_formats[] = {
-	DRM_FORMAT_ABGR8888,
 	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ABGR1555,
 	DRM_FORMAT_XBGR1555,
-	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_XRGB1555,
 };
 
@@ -362,19 +357,13 @@ static const struct pl111_variant_data pl110_variant = {
 
 /* RealView, Versatile Express etc use this modern variant */
 static const u32 pl111_pixel_formats[] = {
-	DRM_FORMAT_ABGR8888,
 	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_BGR565,
 	DRM_FORMAT_RGB565,
-	DRM_FORMAT_ABGR1555,
 	DRM_FORMAT_XBGR1555,
-	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ABGR4444,
 	DRM_FORMAT_XBGR4444,
-	DRM_FORMAT_ARGB4444,
 	DRM_FORMAT_XRGB4444,
 };
 
@@ -388,19 +377,13 @@ static const struct pl111_variant_data pl111_variant = {
 static const u32 pl110_nomadik_pixel_formats[] = {
 	DRM_FORMAT_RGB888,
 	DRM_FORMAT_BGR888,
-	DRM_FORMAT_ABGR8888,
 	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_BGR565,
 	DRM_FORMAT_RGB565,
-	DRM_FORMAT_ABGR1555,
 	DRM_FORMAT_XBGR1555,
-	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ABGR4444,
 	DRM_FORMAT_XBGR4444,
-	DRM_FORMAT_ARGB4444,
 	DRM_FORMAT_XRGB4444,
 };
 
diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c
index 0d8331a3909f4d..9a60f10231b165 100644
--- a/drivers/gpu/drm/pl111/pl111_versatile.c
+++ b/drivers/gpu/drm/pl111/pl111_versatile.c
@@ -209,9 +209,7 @@ static void pl111_versatile_enable(struct drm_device *drm, u32 format)
 	drm_info(drm, "enable Versatile CLCD connectors\n");
 
 	switch (format) {
-	case DRM_FORMAT_ABGR8888:
 	case DRM_FORMAT_XBGR8888:
-	case DRM_FORMAT_ARGB8888:
 	case DRM_FORMAT_XRGB8888:
 		val |= SYS_CLCD_MODE_888;
 		break;
@@ -221,9 +219,7 @@ static void pl111_versatile_enable(struct drm_device *drm, u32 format)
 	case DRM_FORMAT_RGB565:
 		val |= SYS_CLCD_MODE_565_B_LSB;
 		break;
-	case DRM_FORMAT_ABGR1555:
 	case DRM_FORMAT_XBGR1555:
-	case DRM_FORMAT_ARGB1555:
 	case DRM_FORMAT_XRGB1555:
 		val |= SYS_CLCD_MODE_5551;
 		break;
@@ -266,44 +262,30 @@ static void pl111_realview_clcd_enable(struct drm_device *drm, u32 format)
 
 /* PL110 pixel formats for Integrator, vanilla PL110 */
 static const u32 pl110_integrator_pixel_formats[] = {
-	DRM_FORMAT_ABGR8888,
 	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ABGR1555,
 	DRM_FORMAT_XBGR1555,
-	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_XRGB1555,
 };
 
 /* Extended PL110 pixel formats for Integrator and Versatile */
 static const u32 pl110_versatile_pixel_formats[] = {
-	DRM_FORMAT_ABGR8888,
 	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_BGR565, /* Uses external PLD */
 	DRM_FORMAT_RGB565, /* Uses external PLD */
-	DRM_FORMAT_ABGR1555,
 	DRM_FORMAT_XBGR1555,
-	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_XRGB1555,
 };
 
 static const u32 pl111_realview_pixel_formats[] = {
-	DRM_FORMAT_ABGR8888,
 	DRM_FORMAT_XBGR8888,
-	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_BGR565,
 	DRM_FORMAT_RGB565,
-	DRM_FORMAT_ABGR1555,
 	DRM_FORMAT_XBGR1555,
-	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_XRGB1555,
-	DRM_FORMAT_ABGR4444,
 	DRM_FORMAT_XBGR4444,
-	DRM_FORMAT_ARGB4444,
 	DRM_FORMAT_XRGB4444,
 };
 
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 0719fc6a52d537..63fc7c6a7f1a56 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -419,7 +419,7 @@ static const struct drm_crtc_funcs qxl_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = qxl_crtc_destroy,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	DRM_CRTC_VBLANK_TIMER_FUNCS,
@@ -930,7 +930,7 @@ static const struct drm_plane_funcs qxl_cursor_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_helper_destroy,
-	.reset		= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
@@ -952,7 +952,7 @@ static const struct drm_plane_funcs qxl_primary_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_helper_destroy,
-	.reset		= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 8b3f8303a967e8..5b9c72bf409945 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -1646,6 +1646,38 @@ radeon_atom_encoder_dpms_avivo(struct drm_encoder *encoder, int mode)
 	}
 }
 
+/*
+ * Apple's EFI enables the internal eDP panel power sequencer
+ * (LVTMA_PWRSEQ_CNTL) exactly once, at cold boot, and its minimal ATOM tables
+ * never touch it afterwards.  Any GPU re-initialisation - S3 resume or a GPU
+ * reset - clears PWRSEQ_EN, so the panel comes back dark on the DCE3.2 iMacs
+ * (iMac10,1 / iMac11,2, RV730).  Re-assert PWRSEQ_EN when we find it cleared;
+ * the hardware then runs the DIGON->DE->VARY_BL->BLON power-on sequence itself.
+ * DCE3.2 places the register 0x18 above the R600 offset.
+ *
+ * This is restricted to Apple DCE3.2 parts: the register lives at a different
+ * offset on DCE4+ (which Apple also shipped, e.g. the 2011 MacBook Pro), and
+ * non-Apple boards program the sequencer from their own firmware.  The
+ * read-modify-write only fires when the sequencer is actually disabled, so a
+ * lit panel is never disturbed.
+ *
+ * https://gitlab.freedesktop.org/drm/amd/-/issues/5719
+ */
+static void radeon_atom_apple_edp_pwrseq_restore(struct radeon_device *rdev)
+{
+	u32 pwrseq_cntl = R600_LVTMA_PWRSEQ_CNTL + 0x18;
+
+	if (rdev->pdev->subsystem_vendor != PCI_VENDOR_ID_APPLE)
+		return;
+	if (!ASIC_IS_DCE32(rdev) || ASIC_IS_DCE4(rdev))
+		return;
+
+	if (!(RREG32(pwrseq_cntl) & AVIVO_LVTMA_PWRSEQ_EN))
+		WREG32(pwrseq_cntl, AVIVO_LVTMA_PWRSEQ_EN |
+				    AVIVO_LVTMA_SYNCEN_OVRD |
+				    AVIVO_LVTMA_DIGON);
+}
+
 static void
 radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
 {
@@ -1711,6 +1743,10 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
 				atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0);
 		}
 		if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
+			/* restore the panel power sequencer on Apple eDP iMacs
+			 * that lose it across resume/reset (see helper above)
+			 */
+			radeon_atom_apple_edp_pwrseq_restore(rdev);
 			if (rdev->mode_info.bl_encoder)
 				atombios_set_backlight_level(radeon_encoder, dig->backlight_level);
 			else
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 705c012fcf9e0e..9523240110a6a7 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1258,10 +1258,17 @@ static bool radeon_switcheroo_can_switch(struct pci_dev *pdev)
 	return atomic_read(&dev->open_count) == 0;
 }
 
+static void radeon_switcheroo_pre_switch(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+
+	drm_client_dev_acquire_outputs(dev);
+}
+
 static const struct vga_switcheroo_client_ops radeon_switcheroo_ops = {
 	.set_gpu_state = radeon_switcheroo_set_state,
-	.reprobe = NULL,
 	.can_switch = radeon_switcheroo_can_switch,
+	.pre_switch = radeon_switcheroo_pre_switch,
 };
 
 /**
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 299865569252d7..8e9402b8811b6b 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -191,7 +191,7 @@ static int radeon_fence_check_signaled(wait_queue_entry_t *wait,
  * @ring: ring index the fence is associated with
  *
  * Checks the current fence value and calculates the last
- * signalled fence value. Returns true if activity occured
+ * signalled fence value. Returns true if activity occurred
  * on the ring, and the fence_queue should be waken up.
  */
 static bool radeon_fence_activity(struct radeon_device *rdev, int ring)
@@ -262,7 +262,7 @@ static bool radeon_fence_activity(struct radeon_device *rdev, int ring)
  * @work: delayed work item
  *
  * Checks for fence activity and if there is none probe
- * the hardware if a lockup occured.
+ * the hardware if a lockup occurred.
  */
 static void radeon_fence_check_lockup(struct work_struct *work)
 {
diff --git a/drivers/gpu/drm/renesas/rcar-du/Kconfig b/drivers/gpu/drm/renesas/rcar-du/Kconfig
index 840305fdeb49a0..80bd770ae0f68d 100644
--- a/drivers/gpu/drm/renesas/rcar-du/Kconfig
+++ b/drivers/gpu/drm/renesas/rcar-du/Kconfig
@@ -25,6 +25,18 @@ config DRM_RCAR_CMM
 	def_tristate DRM_RCAR_DU
 	depends on DRM_RCAR_USE_CMM
 
+config DRM_RCAR_USE_DSC
+	bool "R-Car DU DSC Encoder Support"
+	depends on DRM_BRIDGE && OF
+	depends on DRM_RCAR_DU || COMPILE_TEST
+	default DRM_RCAR_DU
+	help
+	  Enable support for the R-Car Display Unit embedded DSC encoder.
+
+config DRM_RCAR_DSC
+	def_tristate DRM_RCAR_DU
+	depends on DRM_RCAR_USE_DSC
+
 config DRM_RCAR_DW_HDMI
 	tristate "R-Car Gen3 and RZ/G2 DU HDMI Encoder Support"
 	depends on DRM && OF
diff --git a/drivers/gpu/drm/renesas/rcar-du/Makefile b/drivers/gpu/drm/renesas/rcar-du/Makefile
index 6f132325c8b739..9ab7a0ac45d847 100644
--- a/drivers/gpu/drm/renesas/rcar-du/Makefile
+++ b/drivers/gpu/drm/renesas/rcar-du/Makefile
@@ -10,6 +10,7 @@ rcar-du-drm-$(CONFIG_DRM_RCAR_VSP)	+= rcar_du_vsp.o
 rcar-du-drm-$(CONFIG_DRM_RCAR_WRITEBACK) += rcar_du_writeback.o
 
 obj-$(CONFIG_DRM_RCAR_CMM)		+= rcar_cmm.o
+obj-$(CONFIG_DRM_RCAR_DSC)		+= rcar_dsc.o
 obj-$(CONFIG_DRM_RCAR_DU)		+= rcar-du-drm.o
 obj-$(CONFIG_DRM_RCAR_DW_HDMI)		+= rcar_dw_hdmi.o
 obj-$(CONFIG_DRM_RCAR_LVDS)		+= rcar_lvds.o
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_dsc.c b/drivers/gpu/drm/renesas/rcar-du/rcar_dsc.c
new file mode 100644
index 00000000000000..1882ae34ce8f59
--- /dev/null
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_dsc.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * R-Car DSC Encoder
+ *
+ * Copyright (C) 2025 Marek Vasut <marek.vasut+renesas@mailbox.org>
+ * Copyright (C) 2025 Renesas Electronics Corporation
+ */
+
+#include <linux/container_of.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+
+struct rcar_dsc {
+	struct drm_bridge bridge;
+
+	struct device *dev;
+};
+
+static inline struct rcar_dsc *bridge_to_rcar_dsc(struct drm_bridge *bridge)
+{
+	return container_of(bridge, struct rcar_dsc, bridge);
+}
+
+/* -----------------------------------------------------------------------------
+ * Bridge
+ */
+
+static int rcar_dsc_attach(struct drm_bridge *bridge,
+			   struct drm_encoder *encoder,
+			   enum drm_bridge_attach_flags flags)
+{
+	struct rcar_dsc *dsc = bridge_to_rcar_dsc(bridge);
+
+	if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR))
+		return -EINVAL;
+
+	return drm_bridge_attach(encoder, dsc->bridge.next_bridge, bridge,
+				 DRM_BRIDGE_ATTACH_NO_CONNECTOR);
+}
+
+static void rcar_dsc_atomic_enable(struct drm_bridge *bridge,
+				   struct drm_atomic_commit *commit)
+{
+	struct rcar_dsc *dsc = bridge_to_rcar_dsc(bridge);
+
+	WARN_ON(pm_runtime_get_sync(dsc->dev) < 0);
+}
+
+static void rcar_dsc_atomic_disable(struct drm_bridge *bridge,
+				    struct drm_atomic_commit *commit)
+{
+	struct rcar_dsc *dsc = bridge_to_rcar_dsc(bridge);
+
+	pm_runtime_put(dsc->dev);
+}
+
+static enum drm_mode_status
+rcar_dsc_bridge_mode_valid(struct drm_bridge *bridge,
+			   const struct drm_display_info *info,
+			   const struct drm_display_mode *mode)
+{
+	if (mode->hdisplay < 320 || mode->hdisplay > 8190)
+		return MODE_BAD_HVALUE;
+
+	if (mode->vdisplay < 160 || mode->vdisplay > 8190)
+		return MODE_BAD_VVALUE;
+
+	if (mode->clock > 400000) /* Really 400 Mpixel/s */
+		return MODE_CLOCK_HIGH;
+
+	return MODE_OK;
+}
+
+static const struct drm_bridge_funcs rcar_dsc_bridge_ops = {
+	.attach = rcar_dsc_attach,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_create_state = drm_atomic_helper_bridge_create_state,
+	.atomic_enable = rcar_dsc_atomic_enable,
+	.atomic_disable = rcar_dsc_atomic_disable,
+	.mode_valid = rcar_dsc_bridge_mode_valid,
+};
+
+/* -----------------------------------------------------------------------------
+ * Probe & Remove
+ */
+
+static int rcar_dsc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rcar_dsc *dsc;
+	int ret;
+
+	dsc = devm_drm_bridge_alloc(dev, struct rcar_dsc, bridge,
+				    &rcar_dsc_bridge_ops);
+	if (IS_ERR(dsc))
+		return PTR_ERR(dsc);
+
+	platform_set_drvdata(pdev, dsc);
+
+	dsc->dev = &pdev->dev;
+
+	dsc->bridge.next_bridge = of_drm_get_bridge_by_endpoint(dev->of_node,
+								1, 0);
+	if (IS_ERR(dsc->bridge.next_bridge))
+		return PTR_ERR(dsc->bridge.next_bridge);
+
+	dsc->bridge.of_node = dev->of_node;
+
+	pm_runtime_enable(&pdev->dev);
+
+	ret = devm_drm_bridge_add(dev, &dsc->bridge);
+	if (ret)
+		goto err_runtime_disable;
+
+	return 0;
+
+err_runtime_disable:
+	pm_runtime_disable(&pdev->dev);
+
+	return ret;
+}
+
+static void rcar_dsc_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+}
+
+static const struct of_device_id rcar_dsc_of_table[] = {
+	{ .compatible = "renesas,r8a779g0-dsc" },
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, rcar_dsc_of_table);
+
+static struct platform_driver rcar_dsc_platform_driver = {
+	.probe          = rcar_dsc_probe,
+	.remove		= rcar_dsc_remove,
+	.driver         = {
+		.name   = "rcar-dsc",
+		.of_match_table = rcar_dsc_of_table,
+	},
+};
+
+module_platform_driver(rcar_dsc_platform_driver);
+
+MODULE_DESCRIPTION("Renesas R-Car DSC Encoder Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c
index 6c07c406f72580..59c2a4b761376e 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c
@@ -997,23 +997,20 @@ static void rcar_du_crtc_cleanup(struct drm_crtc *crtc)
 	drm_crtc_cleanup(crtc);
 }
 
-static void rcar_du_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *rcar_du_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct rcar_du_crtc_state *state;
 
-	if (crtc->state) {
-		rcar_du_crtc_atomic_destroy_state(crtc, crtc->state);
-		crtc->state = NULL;
-	}
-
 	state = kzalloc_obj(*state);
 	if (state == NULL)
-		return;
+		return ERR_PTR(-ENOMEM);
 
 	state->crc.source = VSP1_DU_CRC_NONE;
 	state->crc.index = 0;
 
-	__drm_atomic_helper_crtc_reset(crtc, &state->state);
+	__drm_atomic_helper_crtc_state_init(&state->state, crtc);
+
+	return &state->state;
 }
 
 static int rcar_du_crtc_enable_vblank(struct drm_crtc *crtc)
@@ -1155,7 +1152,7 @@ static int rcar_du_crtc_set_crc_source(struct drm_crtc *crtc,
 }
 
 static const struct drm_crtc_funcs crtc_funcs_gen2 = {
-	.reset = rcar_du_crtc_reset,
+	.atomic_create_state = rcar_du_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -1166,7 +1163,7 @@ static const struct drm_crtc_funcs crtc_funcs_gen2 = {
 };
 
 static const struct drm_crtc_funcs crtc_funcs_gen3 = {
-	.reset = rcar_du_crtc_reset,
+	.atomic_create_state = rcar_du_crtc_create_state,
 	.destroy = rcar_du_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c
index d8e7e9877ba863..3f98eae9b6ace3 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c
@@ -90,9 +90,22 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 				drm_bridge_get(bridge);
 
 		if (output == RCAR_DU_OUTPUT_DSI0 ||
-		    output == RCAR_DU_OUTPUT_DSI1)
+		    output == RCAR_DU_OUTPUT_DSI1) {
+			struct drm_bridge *dsi_bridge;
+
+			/*
+			 * When we have a DSC block between the DU and the DSI,
+			 * the "bridge" points to the DSC. Detect the DSC by looking
+			 * at the bridge type, and skip the DSC if the bridge is not
+			 * the DSI bridge.
+			 */
+
+			dsi_bridge = bridge->type == DRM_MODE_CONNECTOR_DSI ?
+				     bridge : bridge->next_bridge;
+
 			rcdu->dsi[output - RCAR_DU_OUTPUT_DSI0] =
-				drm_bridge_get(bridge);
+				drm_bridge_get(dsi_bridge);
+		}
 	}
 
 	/*
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c
index 8870766b9e5411..6ef91fb30985e1 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c
@@ -16,6 +16,7 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_panic_helper.h>
 
 #include "rcar_du_drv.h"
 #include "rcar_du_group.h"
@@ -712,24 +713,21 @@ static void rcar_du_plane_atomic_destroy_state(struct drm_plane *plane,
 	kfree(to_rcar_plane_state(state));
 }
 
-static void rcar_du_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *rcar_du_plane_create_state(struct drm_plane *plane)
 {
 	struct rcar_du_plane_state *state;
 
-	if (plane->state) {
-		rcar_du_plane_atomic_destroy_state(plane, plane->state);
-		plane->state = NULL;
-	}
-
 	state = kzalloc_obj(*state);
-	if (state == NULL)
-		return;
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	__drm_atomic_helper_plane_reset(plane, &state->state);
+	__drm_atomic_helper_plane_state_init(&state->state, plane);
 
 	state->hwindex = -1;
 	state->source = RCAR_DU_PLANE_MEMORY;
 	state->colorkey = RCAR_DU_COLORKEY_NONE;
+
+	return &state->state;
 }
 
 static int rcar_du_plane_atomic_set_property(struct drm_plane *plane,
@@ -767,12 +765,13 @@ static int rcar_du_plane_atomic_get_property(struct drm_plane *plane,
 static const struct drm_plane_funcs rcar_du_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = rcar_du_plane_reset,
+	.atomic_create_state = rcar_du_plane_create_state,
 	.destroy = drm_plane_cleanup,
 	.atomic_duplicate_state = rcar_du_plane_atomic_duplicate_state,
 	.atomic_destroy_state = rcar_du_plane_atomic_destroy_state,
 	.atomic_set_property = rcar_du_plane_atomic_set_property,
 	.atomic_get_property = rcar_du_plane_atomic_get_property,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static const uint32_t formats[] = {
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c
index ae9f381b03c860..cc3213a9dd7612 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c
@@ -421,26 +421,23 @@ static void rcar_du_vsp_plane_atomic_destroy_state(struct drm_plane *plane,
 	kfree(to_rcar_vsp_plane_state(state));
 }
 
-static void rcar_du_vsp_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *rcar_du_vsp_plane_create_state(struct drm_plane *plane)
 {
 	struct rcar_du_vsp_plane_state *state;
 
-	if (plane->state) {
-		rcar_du_vsp_plane_atomic_destroy_state(plane, plane->state);
-		plane->state = NULL;
-	}
-
 	state = kzalloc_obj(*state);
-	if (state == NULL)
-		return;
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&state->state, plane);
 
-	__drm_atomic_helper_plane_reset(plane, &state->state);
+	return &state->state;
 }
 
 static const struct drm_plane_funcs rcar_du_vsp_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = rcar_du_vsp_plane_reset,
+	.atomic_create_state = rcar_du_vsp_plane_create_state,
 	.destroy = drm_plane_cleanup,
 	.atomic_duplicate_state = rcar_du_vsp_plane_atomic_duplicate_state,
 	.atomic_destroy_state = rcar_du_vsp_plane_atomic_destroy_state,
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
index 6e46d6d99f3c0c..02a4bd791745e3 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
@@ -8,6 +8,7 @@
 #include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/dma-mapping.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/math64.h>
@@ -75,6 +76,9 @@ struct rcar_mipi_dsi {
 	unsigned long mode_flags;
 	unsigned int num_data_lanes;
 	unsigned int lanes;
+
+	void *cmd_axi_cpu;
+	dma_addr_t cmd_axi_dma;
 };
 
 struct dsi_setup_info {
@@ -957,6 +961,7 @@ static int rcar_mipi_dsi_host_attach(struct mipi_dsi_host *host,
 
 	/* Initialize the DRM bridge. */
 	dsi->bridge.of_node = dsi->dev->of_node;
+	dsi->bridge.type = DRM_MODE_CONNECTOR_DSI;
 	drm_bridge_add(&dsi->bridge);
 
 	return 0;
@@ -977,6 +982,7 @@ static ssize_t rcar_mipi_dsi_host_tx_transfer(struct mipi_dsi_host *host,
 					      bool is_rx_xfer)
 {
 	const bool is_tx_long = mipi_dsi_packet_format_is_long(msg->type);
+	const bool is_tx_axi = !is_rx_xfer && is_tx_long && (msg->tx_len > 16);
 	struct rcar_mipi_dsi *dsi = host_to_rcar_mipi_dsi(host);
 	struct mipi_dsi_packet packet;
 	u8 payload[16] = { 0 };
@@ -987,9 +993,14 @@ static ssize_t rcar_mipi_dsi_host_tx_transfer(struct mipi_dsi_host *host,
 	if (ret)
 		return ret;
 
-	/* Configure LP or HS command transfer. */
-	rcar_mipi_dsi_write(dsi, TXCMSETR, (msg->flags & MIPI_DSI_MSG_USE_LPM) ?
-					   TXCMSETR_SPDTYP : 0);
+	/* Configure LP or HS and register or AXI command transfer. */
+	rcar_mipi_dsi_write(dsi, TXCMSETR, ((msg->flags & MIPI_DSI_MSG_USE_LPM) ?
+					    TXCMSETR_SPDTYP : 0) |
+					   (is_tx_axi ? TXCMSETR_LPPDACC : 0));
+
+	/* Configure DMA source address for AXI command transfer. */
+	if (is_tx_axi)
+		rcar_mipi_dsi_write(dsi, TXCMADDRSET0R, dsi->cmd_axi_dma);
 
 	/* Register access mode for RX transfer. */
 	if (is_rx_xfer)
@@ -1011,7 +1022,10 @@ static ssize_t rcar_mipi_dsi_host_tx_transfer(struct mipi_dsi_host *host,
 			    TXCMPHDR_DATA1(packet.header[2]) |
 			    TXCMPHDR_DATA0(packet.header[1]));
 
-	if (is_tx_long) {
+	if (is_tx_axi) {
+		memcpy(dsi->cmd_axi_cpu, packet.payload,
+		       min(msg->tx_len, 1024));
+	} else if (is_tx_long) {
 		memcpy(payload, packet.payload,
 		       min(msg->tx_len, sizeof(payload)));
 
@@ -1162,10 +1176,16 @@ static ssize_t rcar_mipi_dsi_host_transfer(struct mipi_dsi_host *host,
 	struct rcar_mipi_dsi *dsi = host_to_rcar_mipi_dsi(host);
 	int ret;
 
-	if (msg->tx_len > 16 || msg->rx_len > 16) {
-		/* ToDo: Implement Memory on AXI bus command mode. */
+	if (msg->tx_len > 1024 || msg->rx_len > 16) {
+		/* ToDo: Implement Memory on AXI bus RX command mode. */
 		dev_warn(dsi->dev,
-			 "Register-based command mode supports only up to 16 Bytes long payload\n");
+			 "Command mode supports only up to 1024B long TX and 16B long RX payload\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((msg->flags & MIPI_DSI_MSG_USE_LPM) && msg->tx_len > 128) {
+		dev_warn(dsi->dev,
+			 "Command mode in LP supports only up to 128B long TX payload\n");
 		return -EOPNOTSUPP;
 	}
 
@@ -1266,6 +1286,10 @@ static int rcar_mipi_dsi_probe(struct platform_device *pdev)
 	struct rcar_mipi_dsi *dsi;
 	int ret;
 
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "No suitable DMA available\n");
+
 	dsi = devm_drm_bridge_alloc(&pdev->dev, struct rcar_mipi_dsi, bridge,
 				    &rcar_mipi_dsi_bridge_ops);
 	if (IS_ERR(dsi))
@@ -1295,12 +1319,21 @@ static int rcar_mipi_dsi_probe(struct platform_device *pdev)
 		return PTR_ERR(dsi->rstc);
 	}
 
+	dsi->cmd_axi_cpu = dma_alloc_coherent(&pdev->dev, SZ_4K, &dsi->cmd_axi_dma,
+					      GFP_KERNEL);
+	if (!dsi->cmd_axi_cpu) {
+		return dev_err_probe(&pdev->dev, -ENOMEM,
+				     "Failed to allocate DSI AXI Access command buffer\n");
+	}
+
 	/* Initialize the DSI host. */
 	dsi->host.dev = dsi->dev;
 	dsi->host.ops = &rcar_mipi_dsi_host_ops;
 	ret = mipi_dsi_host_register(&dsi->host);
-	if (ret < 0)
+	if (ret < 0) {
+		dma_free_coherent(&pdev->dev, SZ_4K, dsi->cmd_axi_cpu, dsi->cmd_axi_dma);
 		return ret;
+	}
 
 	return 0;
 }
@@ -1310,6 +1343,8 @@ static void rcar_mipi_dsi_remove(struct platform_device *pdev)
 	struct rcar_mipi_dsi *dsi = platform_get_drvdata(pdev);
 
 	mipi_dsi_host_unregister(&dsi->host);
+
+	dma_free_coherent(&pdev->dev, SZ_4K, dsi->cmd_axi_cpu, dsi->cmd_axi_dma);
 }
 
 static const struct rcar_mipi_dsi_device_info v3u_data = {
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c
index d0f01aa642a7db..ab2f24cc7683f1 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c
@@ -332,20 +332,17 @@ static void rzg2l_du_crtc_atomic_destroy_state(struct drm_crtc *crtc,
 	kfree(to_rzg2l_crtc_state(state));
 }
 
-static void rzg2l_du_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *rzg2l_du_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct rzg2l_du_crtc_state *state;
 
-	if (crtc->state) {
-		rzg2l_du_crtc_atomic_destroy_state(crtc, crtc->state);
-		crtc->state = NULL;
-	}
-
 	state = kzalloc_obj(*state);
 	if (!state)
-		return;
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(&state->state, crtc);
 
-	__drm_atomic_helper_crtc_reset(crtc, &state->state);
+	return &state->state;
 }
 
 static int rzg2l_du_crtc_enable_vblank(struct drm_crtc *crtc)
@@ -365,7 +362,7 @@ static void rzg2l_du_crtc_disable_vblank(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_funcs crtc_funcs_rz = {
-	.reset = rzg2l_du_crtc_reset,
+	.atomic_create_state = rzg2l_du_crtc_create_state,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
 	.atomic_duplicate_state = rzg2l_du_crtc_atomic_duplicate_state,
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
index 1efa0f0451fe90..944bc9ac9589db 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
@@ -266,26 +266,23 @@ static void rzg2l_du_vsp_plane_atomic_destroy_state(struct drm_plane *plane,
 	kfree(to_rzg2l_vsp_plane_state(state));
 }
 
-static void rzg2l_du_vsp_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *rzg2l_du_vsp_plane_create_state(struct drm_plane *plane)
 {
 	struct rzg2l_du_vsp_plane_state *state;
 
-	if (plane->state) {
-		rzg2l_du_vsp_plane_atomic_destroy_state(plane, plane->state);
-		plane->state = NULL;
-	}
-
 	state = kzalloc_obj(*state);
 	if (!state)
-		return;
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&state->state, plane);
 
-	__drm_atomic_helper_plane_reset(plane, &state->state);
+	return &state->state;
 }
 
 static const struct drm_plane_funcs rzg2l_du_vsp_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = rzg2l_du_vsp_plane_reset,
+	.atomic_create_state = rzg2l_du_vsp_plane_create_state,
 	.atomic_duplicate_state = rzg2l_du_vsp_plane_atomic_duplicate_state,
 	.atomic_destroy_state = rzg2l_du_vsp_plane_atomic_destroy_state,
 };
diff --git a/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c
index 1a2b9b68af6f18..328a36215e3945 100644
--- a/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c
@@ -21,6 +21,7 @@
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
@@ -29,7 +30,6 @@
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
 
 #include <video/videomode.h>
@@ -364,7 +364,7 @@ static void shmob_drm_disable_vblank(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_funcs crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = shmob_drm_crtc_page_flip,
@@ -436,6 +436,10 @@ static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
 	.mode_fixup = shmob_drm_encoder_mode_fixup,
 };
 
+static const struct drm_encoder_funcs shmob_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 /* -----------------------------------------------------------------------------
  * Encoder
  */
@@ -448,8 +452,8 @@ int shmob_drm_encoder_create(struct shmob_drm_device *sdev)
 
 	encoder->possible_crtcs = 1;
 
-	ret = drm_simple_encoder_init(&sdev->ddev, encoder,
-				      DRM_MODE_ENCODER_DPI);
+	ret = drm_encoder_init(&sdev->ddev, encoder, &shmob_encoder_funcs,
+			       DRM_MODE_ENCODER_DPI, NULL);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c b/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c
index 148de34af78528..30f1d9632d4a22 100644
--- a/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c
+++ b/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c
@@ -14,6 +14,7 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_panic_helper.h>
 
 #include "shmob_drm_drv.h"
 #include "shmob_drm_kms.h"
@@ -251,20 +252,17 @@ static void shmob_drm_plane_atomic_destroy_state(struct drm_plane *plane,
 	kfree(to_shmob_plane_state(state));
 }
 
-static void shmob_drm_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *shmob_drm_plane_create_state(struct drm_plane *plane)
 {
 	struct shmob_drm_plane_state *state;
 
-	if (plane->state) {
-		shmob_drm_plane_atomic_destroy_state(plane, plane->state);
-		plane->state = NULL;
-	}
-
 	state = kzalloc_obj(*state);
-	if (state == NULL)
-		return;
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_plane_state_init(&state->base, plane);
 
-	__drm_atomic_helper_plane_reset(plane, &state->base);
+	return &state->base;
 }
 
 static const struct drm_plane_helper_funcs shmob_drm_plane_helper_funcs = {
@@ -283,9 +281,10 @@ static const struct drm_plane_helper_funcs shmob_drm_primary_plane_helper_funcs
 static const struct drm_plane_funcs shmob_drm_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = shmob_drm_plane_reset,
+	.atomic_create_state = shmob_drm_plane_create_state,
 	.atomic_duplicate_state = shmob_drm_plane_atomic_duplicate_state,
 	.atomic_destroy_state = shmob_drm_plane_atomic_destroy_state,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static const uint32_t formats[] = {
diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig
index 4e58685f58ff9e..754d9b977ff196 100644
--- a/drivers/gpu/drm/rockchip/Kconfig
+++ b/drivers/gpu/drm/rockchip/Kconfig
@@ -125,6 +125,7 @@ config ROCKCHIP_LVDS
 	depends on PINCTRL && OF
 	select DRM_DISPLAY_HELPER
 	select DRM_BRIDGE_CONNECTOR
+	select DRM_PANEL
 	help
 	  Choose this option to enable support for Rockchip LVDS controllers.
 	  Rockchip rk3288 SoC has LVDS TX Controller can be used, and it
@@ -137,6 +138,7 @@ config ROCKCHIP_RGB
 	depends on PINCTRL
 	select DRM_DISPLAY_HELPER
 	select DRM_BRIDGE_CONNECTOR
+	select DRM_PANEL
 	help
 	  Choose this option to enable support for Rockchip RGB output.
 	  Some Rockchip CRTCs, like rv1108, can directly output parallel
diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
index efd5a98e80bd21..ccdd4b9ecdb895 100644
--- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
@@ -28,7 +28,6 @@
 #include <drm/drm_bridge_connector.h>
 #include <drm/bridge/analogix_dp.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -487,16 +486,6 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 
 static void rockchip_dp_remove(struct platform_device *pdev)
 {
-	struct rockchip_dp_device *dp = platform_get_drvdata(pdev);
-
-	/*
-	 * Release the probe-time reference from of_drm_find_panel(). If bind
-	 * ran, the panel_bridge holds a second reference that devm cleanup
-	 * will release when the bridge is destroyed after remove() returns.
-	 */
-	if (dp->plat_data.panel)
-		drm_panel_put(dp->plat_data.panel);
-
 	component_del(&pdev->dev, &rockchip_dp_component_ops);
 }
 
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
index 268d4993a04fd5..aab3353945de5c 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
@@ -246,6 +246,7 @@ enum {
 	BIASEXTR_127_7,
 };
 
+struct dw_mipi_dsi_rockchip;
 struct rockchip_dw_dsi_chip_data {
 	u32 reg;
 
@@ -261,12 +262,16 @@ struct rockchip_dw_dsi_chip_data {
 	u32 lanecfg2_grf_reg;
 	u32 lanecfg2;
 
+	int (*dphy_get_timing)(struct dw_mipi_dsi_rockchip *dsi, unsigned int lane_mbps,
+			       struct dw_mipi_dsi_dphy_timing *timing);
+
 	int (*dphy_rx_init)(struct phy *phy);
 	int (*dphy_rx_power_on)(struct phy *phy);
 	int (*dphy_rx_power_off)(struct phy *phy);
 
 	unsigned int flags;
 	unsigned int max_data_lanes;
+	unsigned long max_bit_rate_per_lane;
 };
 
 struct dw_mipi_dsi_rockchip {
@@ -564,7 +569,7 @@ dw_mipi_dsi_get_lane_mbps(void *priv_data, const struct drm_display_mode *mode,
 	int bpp;
 	unsigned long mpclk, tmp;
 	unsigned int target_mbps = 1000;
-	unsigned int max_mbps = dppa_map[ARRAY_SIZE(dppa_map) - 1].max_mbps;
+	unsigned int max_mbps;
 	unsigned long best_freq = 0;
 	unsigned long fvco_min, fvco_max, fin, fout;
 	unsigned int min_prediv, max_prediv;
@@ -572,6 +577,7 @@ dw_mipi_dsi_get_lane_mbps(void *priv_data, const struct drm_display_mode *mode,
 	unsigned long _fbdiv, best_fbdiv;
 	unsigned long min_delta = ULONG_MAX;
 
+	max_mbps = dsi->cdata->max_bit_rate_per_lane / USEC_PER_SEC;
 	dsi->format = format;
 	bpp = mipi_dsi_pixel_format_to_bpp(dsi->format);
 	if (bpp < 0) {
@@ -585,7 +591,7 @@ dw_mipi_dsi_get_lane_mbps(void *priv_data, const struct drm_display_mode *mode,
 	if (mpclk) {
 		/* take 1 / 0.8, since mbps must big than bandwidth of RGB */
 		tmp = mpclk * (bpp / lanes) * 10 / 8;
-		if (tmp < max_mbps)
+		if (tmp <= max_mbps)
 			target_mbps = tmp;
 		else
 			DRM_DEV_ERROR(dsi->dev,
@@ -718,8 +724,9 @@ static struct hstt hstt_table[] = {
 };
 
 static int
-dw_mipi_dsi_phy_get_timing(void *priv_data, unsigned int lane_mbps,
-			   struct dw_mipi_dsi_dphy_timing *timing)
+dw_mipi_dsi_phy_rk3288_get_timing(struct dw_mipi_dsi_rockchip *dsi,
+				  unsigned int lane_mbps,
+				  struct dw_mipi_dsi_dphy_timing *timing)
 {
 	int i;
 
@@ -735,6 +742,35 @@ dw_mipi_dsi_phy_get_timing(void *priv_data, unsigned int lane_mbps,
 	return 0;
 }
 
+static const struct dw_mipi_dsi_dphy_timing dphy_timing_px30 = {
+	.clk_lp2hs = 0x40,
+	.clk_hs2lp = 0x40,
+	.data_lp2hs = 0x10,
+	.data_hs2lp = 0x14,
+};
+
+static int
+dw_mipi_dsi_phy_px30_get_timing(struct dw_mipi_dsi_rockchip *dsi,
+				unsigned int lane_mbps,
+				struct dw_mipi_dsi_dphy_timing *timing)
+{
+	*timing = dphy_timing_px30;
+
+	return 0;
+}
+
+static int
+dw_mipi_dsi_phy_get_timing(void *priv_data, unsigned int lane_mbps,
+			   struct dw_mipi_dsi_dphy_timing *timing)
+{
+	struct dw_mipi_dsi_rockchip *dsi = priv_data;
+
+	if (dsi->cdata->dphy_get_timing)
+		return dsi->cdata->dphy_get_timing(dsi, lane_mbps, timing);
+
+	return dw_mipi_dsi_phy_px30_get_timing(dsi, lane_mbps, timing);
+}
+
 static const struct dw_mipi_dsi_phy_ops dw_mipi_dsi_rockchip_phy_ops = {
 	.init = dw_mipi_dsi_phy_init,
 	.power_on = dw_mipi_dsi_phy_power_on,
@@ -1508,6 +1544,8 @@ static const struct rockchip_dw_dsi_chip_data px30_chip_data[] = {
 						PX30_DSI_FORCETXSTOPMODE), 0),
 
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1000000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_px30_get_timing,
 	},
 	{ /* sentinel */ }
 };
@@ -1520,6 +1558,8 @@ static const struct rockchip_dw_dsi_chip_data rk3128_chip_data[] = {
 						RK3128_DSI_FORCERXMODE |
 						RK3128_DSI_FORCETXSTOPMODE), 0),
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1000000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_px30_get_timing,
 	},
 	{ /* sentinel */ }
 };
@@ -1532,6 +1572,8 @@ static const struct rockchip_dw_dsi_chip_data rk3288_chip_data[] = {
 		.lcdsel_lit = FIELD_PREP_WM16_CONST(RK3288_DSI0_LCDC_SEL, 1),
 
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1500000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_rk3288_get_timing,
 	},
 	{
 		.reg = 0xff964000,
@@ -1540,6 +1582,8 @@ static const struct rockchip_dw_dsi_chip_data rk3288_chip_data[] = {
 		.lcdsel_lit = FIELD_PREP_WM16_CONST(RK3288_DSI1_LCDC_SEL, 1),
 
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1500000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_rk3288_get_timing,
 	},
 	{ /* sentinel */ }
 };
@@ -1552,6 +1596,8 @@ static const struct rockchip_dw_dsi_chip_data rk3368_chip_data[] = {
 						RK3368_DSI_FORCETXSTOPMODE |
 						RK3368_DSI_FORCERXMODE), 0),
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1500000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_px30_get_timing,
 	},
 	{ /* sentinel */ }
 };
@@ -1639,6 +1685,8 @@ static const struct rockchip_dw_dsi_chip_data rk3399_chip_data[] = {
 
 		.flags = DW_MIPI_NEEDS_PHY_CFG_CLK | DW_MIPI_NEEDS_GRF_CLK,
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1500000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_rk3288_get_timing,
 	},
 	{
 		.reg = 0xff968000,
@@ -1663,10 +1711,12 @@ static const struct rockchip_dw_dsi_chip_data rk3399_chip_data[] = {
 
 		.flags = DW_MIPI_NEEDS_PHY_CFG_CLK | DW_MIPI_NEEDS_GRF_CLK,
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1500000000UL,
 
 		.dphy_rx_init = rk3399_dphy_tx1rx1_init,
 		.dphy_rx_power_on = rk3399_dphy_tx1rx1_power_on,
 		.dphy_rx_power_off = rk3399_dphy_tx1rx1_power_off,
+		.dphy_get_timing = dw_mipi_dsi_phy_rk3288_get_timing,
 	},
 	{ /* sentinel */ }
 };
@@ -1679,6 +1729,8 @@ static const struct rockchip_dw_dsi_chip_data rk3506_chip_data[] = {
 			     FIELD_PREP_WM16_CONST(RK3506_DSI_FORCERXMODE, 0) |
 			     FIELD_PREP_WM16_CONST(RK3506_DSI_FORCETXSTOPMODE, 0)),
 		.max_data_lanes = 2,
+		.max_bit_rate_per_lane = 1500000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_px30_get_timing,
 	},
 	{ /* sentinel */ }
 };
@@ -1692,6 +1744,8 @@ static const struct rockchip_dw_dsi_chip_data rk3568_chip_data[] = {
 			     FIELD_PREP_WM16_CONST(RK3568_DSI0_TURNDISABLE, 0) |
 			     FIELD_PREP_WM16_CONST(RK3568_DSI0_FORCERXMODE, 0)),
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1200000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_px30_get_timing,
 	},
 	{
 		.reg = 0xfe070000,
@@ -1701,6 +1755,8 @@ static const struct rockchip_dw_dsi_chip_data rk3568_chip_data[] = {
 			     FIELD_PREP_WM16_CONST(RK3568_DSI1_TURNDISABLE, 0) |
 			     FIELD_PREP_WM16_CONST(RK3568_DSI1_FORCERXMODE, 0)),
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1200000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_px30_get_timing,
 	},
 	{ /* sentinel */ }
 };
@@ -1713,6 +1769,8 @@ static const struct rockchip_dw_dsi_chip_data rv1126_chip_data[] = {
 			     FIELD_PREP_WM16_CONST(RV1126_DSI_FORCERXMODE, 0) |
 			     FIELD_PREP_WM16_CONST(RV1126_DSI_FORCETXSTOPMODE, 0)),
 		.max_data_lanes = 4,
+		.max_bit_rate_per_lane = 1000000000UL,
+		.dphy_get_timing = dw_mipi_dsi_phy_px30_get_timing,
 	},
 	{ /* sentinel */ }
 };
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 0090d8ff0c79e7..b327a475b5bb8e 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -1157,7 +1157,7 @@ static const struct drm_plane_funcs vop_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.format_mod_supported = rockchip_mod_supported,
@@ -1656,17 +1656,17 @@ static void vop_crtc_destroy_state(struct drm_crtc *crtc,
 	kfree(s);
 }
 
-static void vop_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *vop_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct rockchip_crtc_state *crtc_state = kzalloc_obj(*crtc_state);
+	struct rockchip_crtc_state *crtc_state;
 
-	if (crtc->state)
-		vop_crtc_destroy_state(crtc, crtc->state);
+	crtc_state = kzalloc_obj(*crtc_state);
+	if (!crtc_state)
+		return ERR_PTR(-ENOMEM);
 
-	if (crtc_state)
-		__drm_atomic_helper_crtc_reset(crtc, &crtc_state->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	__drm_atomic_helper_crtc_state_init(&crtc_state->base, crtc);
+
+	return &crtc_state->base;
 }
 
 #ifdef CONFIG_DRM_ANALOGIX_DP
@@ -1738,7 +1738,7 @@ static const struct drm_crtc_funcs vop_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
 	.destroy = drm_crtc_cleanup,
-	.reset = vop_crtc_reset,
+	.atomic_create_state = vop_crtc_create_state,
 	.atomic_duplicate_state = vop_crtc_duplicate_state,
 	.atomic_destroy_state = vop_crtc_destroy_state,
 	.enable_vblank = vop_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index 4cce3e336f5b3a..bfb38414256878 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -1497,7 +1497,7 @@ static const struct drm_plane_funcs vop2_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.format_mod_supported = rockchip_vop2_mod_supported,
@@ -2300,24 +2300,24 @@ static void vop2_crtc_destroy_state(struct drm_crtc *crtc,
 	kfree(vcstate);
 }
 
-static void vop2_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *vop2_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct rockchip_crtc_state *vcstate = kzalloc_obj(*vcstate);
+	struct rockchip_crtc_state *vcstate;
 
-	if (crtc->state)
-		vop2_crtc_destroy_state(crtc, crtc->state);
+	vcstate = kzalloc_obj(*vcstate);
+	if (!vcstate)
+		return ERR_PTR(-ENOMEM);
 
-	if (vcstate)
-		__drm_atomic_helper_crtc_reset(crtc, &vcstate->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	__drm_atomic_helper_crtc_state_init(&vcstate->base, crtc);
+
+	return &vcstate->base;
 }
 
 static const struct drm_crtc_funcs vop2_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
 	.destroy = drm_crtc_cleanup,
-	.reset = vop2_crtc_reset,
+	.atomic_create_state = vop2_crtc_create_state,
 	.atomic_duplicate_state = vop2_crtc_duplicate_state,
 	.atomic_destroy_state = vop2_crtc_destroy_state,
 	.enable_vblank = vop2_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c
index 673ca9cbf36283..e4069fcb0272a0 100644
--- a/drivers/gpu/drm/scheduler/sched_entity.c
+++ b/drivers/gpu/drm/scheduler/sched_entity.c
@@ -207,14 +207,18 @@ EXPORT_SYMBOL(drm_sched_entity_modify_sched);
 
 static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity)
 {
-	rmb(); /* for list_empty to work without lock */
+	bool idle = false;
+
+	spin_lock(&entity->lock);
 
 	if (list_empty(&entity->list) ||
 	    spsc_queue_count(&entity->job_queue) == 0 ||
 	    entity->stopped)
-		return true;
+		idle = true;
 
-	return false;
+	spin_unlock(&entity->lock);
+
+	return idle;
 }
 
 /**
diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index 096fe28aa9c9ee..b7a4cb921234e5 100644
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -205,8 +205,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f)
 }
 EXPORT_SYMBOL(to_drm_sched_fence);
 
-struct drm_sched_fence *drm_sched_fence_alloc(struct drm_sched_entity *entity,
-					      void *owner,
+struct drm_sched_fence *drm_sched_fence_alloc(void *owner,
 					      u64 drm_client_id)
 {
 	struct drm_sched_fence *fence = NULL;
diff --git a/drivers/gpu/drm/scheduler/sched_internal.h b/drivers/gpu/drm/scheduler/sched_internal.h
index a901801fce8565..32d3ddb820be6b 100644
--- a/drivers/gpu/drm/scheduler/sched_internal.h
+++ b/drivers/gpu/drm/scheduler/sched_internal.h
@@ -58,8 +58,7 @@ drm_sched_rq_select_entity(struct drm_gpu_scheduler *sched,
 void drm_sched_entity_select_rq(struct drm_sched_entity *entity);
 struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity);
 
-struct drm_sched_fence *drm_sched_fence_alloc(struct drm_sched_entity *s_entity,
-					      void *owner, u64 drm_client_id);
+struct drm_sched_fence *drm_sched_fence_alloc(void *owner, u64 drm_client_id);
 void drm_sched_fence_init(struct drm_sched_fence *fence,
 			  struct drm_sched_entity *entity);
 void drm_sched_fence_free(struct drm_sched_fence *fence);
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 6cb6f95464938c..b9580ce1ddecc9 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -77,7 +77,6 @@
 #include <drm/drm_gem.h>
 #include <drm/drm_syncobj.h>
 #include <drm/gpu_scheduler.h>
-#include <drm/spsc_queue.h>
 
 #include "sched_internal.h"
 
@@ -612,7 +611,7 @@ int drm_sched_job_init(struct drm_sched_job *job,
 
 	job->entity = entity;
 	job->credits = credits;
-	job->s_fence = drm_sched_fence_alloc(entity, owner, drm_client_id);
+	job->s_fence = drm_sched_fence_alloc(owner, drm_client_id);
 	if (!job->s_fence)
 		return -ENOMEM;
 
@@ -1210,9 +1209,6 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched)
 
 	drm_sched_wqueue_stop(sched);
 
-	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++)
-		kfree(sched->sched_rq[i]);
-
 	/* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */
 	wake_up_all(&sched->job_scheduled);
 
@@ -1226,6 +1222,9 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched)
 	if (sched->own_submit_wq)
 		destroy_workqueue(sched->submit_wq);
 	sched->ready = false;
+
+	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++)
+		kfree(sched->sched_rq[i]);
 	kfree(sched->sched_rq);
 	sched->sched_rq = NULL;
 
diff --git a/drivers/gpu/drm/scheduler/tests/tests_basic.c b/drivers/gpu/drm/scheduler/tests/tests_basic.c
index da5c4952780f3e..34bbb2c2791353 100644
--- a/drivers/gpu/drm/scheduler/tests/tests_basic.c
+++ b/drivers/gpu/drm/scheduler/tests/tests_basic.c
@@ -90,7 +90,7 @@ static const struct drm_sched_basic_params drm_sched_basic_cases[] = {
 		.num_entities = 1,
 	},
 	{
-		.description = "A chain of dependent jobs across multiple entities",
+		.description = "A chain of dependent jobs in a single entity",
 		.queue_depth = 100,
 		.job_us = 1000,
 		.num_entities = 1,
diff --git a/drivers/gpu/drm/sitronix/st7571.c b/drivers/gpu/drm/sitronix/st7571.c
index 20954c33eca93c..fc3ba5e8600f62 100644
--- a/drivers/gpu/drm/sitronix/st7571.c
+++ b/drivers/gpu/drm/sitronix/st7571.c
@@ -424,7 +424,7 @@ static const struct drm_crtc_helper_funcs st7571_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs st7571_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/sitronix/st7920.c b/drivers/gpu/drm/sitronix/st7920.c
index d320391801f3f5..9887ed746cebdb 100644
--- a/drivers/gpu/drm/sitronix/st7920.c
+++ b/drivers/gpu/drm/sitronix/st7920.c
@@ -453,17 +453,17 @@ static void st7920_primary_plane_atomic_disable(struct drm_plane *plane,
 }
 
 /* Called during init to allocate the plane's atomic state. */
-static void st7920_primary_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *st7920_primary_plane_create_state(struct drm_plane *plane)
 {
 	struct st7920_plane_state *st7920_state;
 
-	drm_WARN_ON_ONCE(plane->dev, plane->state);
-
 	st7920_state = kzalloc_obj(*st7920_state);
 	if (!st7920_state)
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	__drm_gem_reset_shadow_plane(plane, &st7920_state->base);
+	__drm_gem_shadow_plane_state_init(plane, &st7920_state->base);
+
+	return &(&st7920_state->base)->base;
 }
 
 static struct drm_plane_state *st7920_primary_plane_duplicate_state(struct drm_plane *plane)
@@ -507,7 +507,7 @@ static const struct drm_plane_helper_funcs st7920_primary_plane_helper_funcs = {
 static const struct drm_plane_funcs st7920_primary_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = st7920_primary_plane_reset,
+	.atomic_create_state = st7920_primary_plane_create_state,
 	.atomic_duplicate_state = st7920_primary_plane_duplicate_state,
 	.atomic_destroy_state = st7920_primary_plane_destroy_state,
 	.destroy = drm_plane_cleanup,
@@ -575,17 +575,17 @@ static void st7920_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 /* Called during init to allocate the CRTC's atomic state. */
-static void st7920_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *st7920_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct st7920_crtc_state *st7920_state;
 
-	drm_WARN_ON_ONCE(crtc->dev, crtc->state);
-
 	st7920_state = kzalloc_obj(*st7920_state);
 	if (!st7920_state)
-		return;
+		return ERR_PTR(-ENOMEM);
+
+	__drm_atomic_helper_crtc_state_init(&st7920_state->base, crtc);
 
-	__drm_atomic_helper_crtc_reset(crtc, &st7920_state->base);
+	return &st7920_state->base;
 }
 
 static struct drm_crtc_state *st7920_crtc_duplicate_state(struct drm_crtc *crtc)
@@ -629,7 +629,7 @@ static const struct drm_crtc_helper_funcs st7920_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs st7920_crtc_funcs = {
-	.reset = st7920_crtc_reset,
+	.atomic_create_state = st7920_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/solomon/ssd130x-spi.c b/drivers/gpu/drm/solomon/ssd130x-spi.c
index b52f5fd592a19d..c41064aa8221af 100644
--- a/drivers/gpu/drm/solomon/ssd130x-spi.c
+++ b/drivers/gpu/drm/solomon/ssd130x-spi.c
@@ -146,6 +146,11 @@ static const struct of_device_id ssd130x_of_match[] = {
 		.compatible = "solomon,ssd1331",
 		.data = &ssd130x_variants[SSD1331_ID],
 	},
+	/* ssd135x family */
+	{
+		.compatible = "solomon,ssd1351",
+		.data = &ssd130x_variants[SSD1351_ID],
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, ssd130x_of_match);
@@ -160,17 +165,19 @@ MODULE_DEVICE_TABLE(of, ssd130x_of_match);
  */
 static const struct spi_device_id ssd130x_spi_id[] = {
 	/* ssd130x family */
-	{ "sh1106",  SH1106_ID },
-	{ "ssd1305", SSD1305_ID },
-	{ "ssd1306", SSD1306_ID },
-	{ "ssd1307", SSD1307_ID },
-	{ "ssd1309", SSD1309_ID },
+	{ .name = "sh1106", .driver_data = SH1106_ID },
+	{ .name = "ssd1305", .driver_data = SSD1305_ID },
+	{ .name = "ssd1306", .driver_data = SSD1306_ID },
+	{ .name = "ssd1307", .driver_data = SSD1307_ID },
+	{ .name = "ssd1309", .driver_data = SSD1309_ID },
 	/* ssd132x family */
-	{ "ssd1322", SSD1322_ID },
-	{ "ssd1325", SSD1325_ID },
-	{ "ssd1327", SSD1327_ID },
+	{ .name = "ssd1322", .driver_data = SSD1322_ID },
+	{ .name = "ssd1325", .driver_data = SSD1325_ID },
+	{ .name = "ssd1327", .driver_data = SSD1327_ID },
 	/* ssd133x family */
-	{ "ssd1331", SSD1331_ID },
+	{ .name = "ssd1331", .driver_data = SSD1331_ID },
+	/* ssd135x family */
+	{ .name = "ssd1351", .driver_data = SSD1351_ID },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(spi, ssd130x_spi_id);
diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c
index 0b0fc6fe3df23c..8cbf187544872c 100644
--- a/drivers/gpu/drm/solomon/ssd130x.c
+++ b/drivers/gpu/drm/solomon/ssd130x.c
@@ -146,6 +146,42 @@
 #define SSD133X_DEFAULT_CONTRAST_B		0x50
 #define SSD133X_DEFAULT_CONTRAST_C		0x7d
 
+/* ssd133x remap byte (data of SSD13XX_SET_SEG_REMAP) */
+#define SSD133X_SET_REMAP_COM_SPLIT		BIT(5)
+#define SSD133X_SET_REMAP_COLOR_DEPTH_MASK	GENMASK(7, 6)
+#define SSD133X_COLOR_DEPTH_65K			0x1
+
+/* ssd135x commands */
+#define SSD135X_SET_COL_RANGE			0x15
+#define SSD135X_WRITE_RAM			0x5c
+#define SSD135X_SET_ROW_RANGE			0x75
+#define SSD135X_SET_DISPLAY_START		0xa1
+#define SSD135X_SET_DISPLAY_OFFSET		0xa2
+#define SSD135X_SET_DISPLAY_NORMAL		0xa6
+#define SSD135X_SET_FUNCTION			0xab
+#define SSD135X_SET_PHASE_LENGTH		0xb1
+#define SSD135X_SET_CLOCK_FREQ			0xb3
+#define SSD135X_SET_VSL				0xb4
+#define SSD135X_SET_GPIO			0xb5
+#define SSD135X_SET_PRECHARGE2			0xb6
+#define SSD135X_SET_PRECHARGE_VOLTAGE		0xbb
+#define SSD135X_SET_VCOMH_VOLTAGE		0xbe
+#define SSD135X_SET_CONTRAST			0xc1
+#define SSD135X_SET_CONTRAST_MASTER		0xc7
+#define SSD135X_SET_MUX_RATIO			0xca
+#define SSD135X_SET_COMMAND_LOCK		0xfd
+
+/* ssd135x A/B/C channel contrast at full brightness (white balance) */
+#define SSD135X_DEFAULT_CONTRAST_A		0xc8
+#define SSD135X_DEFAULT_CONTRAST_B		0x80
+#define SSD135X_DEFAULT_CONTRAST_C		0xc8
+
+/* ssd135x remap byte (data of SSD13XX_SET_SEG_REMAP) */
+#define SSD135X_SET_REMAP_COLOR_BGR		BIT(2)
+#define SSD135X_SET_REMAP_COM_SCAN		BIT(4)
+#define SSD135X_SET_REMAP_COM_SPLIT		BIT(5)
+#define SSD135X_SET_REMAP_65K			BIT(6)
+
 #define MAX_CONTRAST 255
 
 const struct ssd130x_deviceinfo ssd130x_variants[] = {
@@ -213,7 +249,14 @@ const struct ssd130x_deviceinfo ssd130x_variants[] = {
 		.default_width = 96,
 		.default_height = 64,
 		.family_id = SSD133X_FAMILY,
-	}
+	},
+	/* ssd135x family */
+	[SSD1351_ID] = {
+		.default_width = 128,
+		.default_height = 128,
+		.family_id = SSD135X_FAMILY,
+		.cmd_params_are_data = true,
+	},
 };
 EXPORT_SYMBOL_NS_GPL(ssd130x_variants, "DRM_SSD130X");
 
@@ -247,61 +290,71 @@ static inline struct ssd130x_device *drm_to_ssd130x(struct drm_device *drm)
 /*
  * Helper to write data (SSD13XX_DATA) to the device.
  */
-static int ssd130x_write_data(struct ssd130x_device *ssd130x, u8 *values, int count)
+static int ssd130x_write_data(struct ssd130x_device *ssd130x, const u8 *values, int count)
 {
 	return regmap_bulk_write(ssd130x->regmap, SSD13XX_DATA, values, count);
 }
 
 /*
- * Helper to write command (SSD13XX_COMMAND). The fist variadic argument
- * is the command to write and the following are the command options.
+ * Helper to write a command (SSD13XX_COMMAND) from a buffer. The first byte
+ * is the command opcode and the following ones are its parameters.
  *
- * Note that the ssd13xx protocol requires each command and option to be
- * written as a SSD13XX_COMMAND device register value. That is why a call
- * to regmap_write(..., SSD13XX_COMMAND, ...) is done for each argument.
+ * By default every byte is written as a SSD13XX_COMMAND register value, hence
+ * the regmap_write() per byte. Controllers that set cmd_params_are_data expect
+ * the parameters on the data path instead, so only the opcode is written as
+ * SSD13XX_COMMAND and the rest as SSD13XX_DATA.
  */
-static int ssd130x_write_cmd(struct ssd130x_device *ssd130x, int count,
-			     /* u8 cmd, u8 option, ... */...)
+static int ssd130x_write_cmds(struct ssd130x_device *ssd130x, const u8 *cmd,
+			      size_t len)
 {
-	va_list ap;
-	u8 value;
+	unsigned int i;
 	int ret;
 
-	va_start(ap, count);
+	if (!len)
+		return 0;
 
-	do {
-		value = va_arg(ap, int);
-		ret = regmap_write(ssd130x->regmap, SSD13XX_COMMAND, value);
-		if (ret)
-			goto out_end;
-	} while (--count);
+	if (ssd130x->device_info->cmd_params_are_data) {
+		ret = regmap_write(ssd130x->regmap, SSD13XX_COMMAND, cmd[0]);
+		/* A command with no parameters is complete after its opcode. */
+		if (ret || len == 1)
+			return ret;
 
-out_end:
-	va_end(ap);
+		return ssd130x_write_data(ssd130x, cmd + 1, len - 1);
+	}
 
-	return ret;
+	for (i = 0; i < len; i++) {
+		ret = regmap_write(ssd130x->regmap, SSD13XX_COMMAND, cmd[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 /*
- * Write a command byte sequence from a buffer.
+ * Variadic wrapper around ssd130x_write_cmds(). The first variadic argument
+ * is the command opcode and the following are its parameters.
  *
- * Like ssd130x_write_cmd() but takes a pre-built byte array instead of
- * variadic arguments, handy when the command is already in an array or
- * when the caller wants to use sizeof() for the length.
+ * The arguments are gathered into a fixed size buffer, so at most 8 bytes
+ * can be sent per call, i.e. an opcode and seven parameters. That covers
+ * every command this driver sends through it. Commands taking more parameters,
+ * such as the grey scale tables, must use ssd130x_write_cmds() instead.
  */
-static int ssd130x_write_cmds(struct ssd130x_device *ssd130x, const u8 *cmd,
-			      size_t len)
+static int ssd130x_write_cmd(struct ssd130x_device *ssd130x, int count,
+			     /* u8 cmd, u8 param, ... */...)
 {
-	unsigned int i;
-	int ret;
+	u8 buf[8];
+	va_list ap;
 
-	for (i = 0; i < len; i++) {
-		ret = regmap_write(ssd130x->regmap, SSD13XX_COMMAND, cmd[i]);
-		if (ret)
-			return ret;
-	}
+	if (drm_WARN_ON(&ssd130x->drm, count > sizeof(buf)))
+		return -EINVAL;
 
-	return 0;
+	va_start(ap, count);
+	for (int i = 0; i < count; i++)
+		buf[i] = va_arg(ap, int);
+	va_end(ap);
+
+	return ssd130x_write_cmds(ssd130x, buf, count);
 }
 
 /*
@@ -362,6 +415,20 @@ static int ssd130x_set_page_range(struct ssd130x_device *ssd130x,
 	return 0;
 }
 
+static int ssd132x_set_col_range(struct ssd130x_device *ssd130x,
+				 u8 col_start, u8 cols)
+{
+	return ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE,
+				 col_start, col_start + cols - 1);
+}
+
+static int ssd132x_set_row_range(struct ssd130x_device *ssd130x,
+				 u8 row_start, u8 rows)
+{
+	return ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE,
+				 row_start, row_start + rows - 1);
+}
+
 /* Set page and column start address for page addressing mode */
 static int ssd130x_set_page_pos(struct ssd130x_device *ssd130x,
 				u8 page_start, u8 col_start)
@@ -615,20 +682,34 @@ static int ssd133x_set_contrast(struct ssd130x_device *ssd130x, u32 brightness)
 	return ssd130x_run_cmd_seq(ssd130x, cmds);
 }
 
+/*
+ * Same white balance calibration as for the ssd133x family, except that the
+ * ssd135x controllers take the three channels as parameters of a single
+ * command instead of one command per channel.
+ */
+static int ssd135x_set_contrast(struct ssd130x_device *ssd130x, u32 brightness)
+{
+	u8 a = ssd130x_scale_contrast(SSD135X_DEFAULT_CONTRAST_A, brightness);
+	u8 b = ssd130x_scale_contrast(SSD135X_DEFAULT_CONTRAST_B, brightness);
+	u8 c = ssd130x_scale_contrast(SSD135X_DEFAULT_CONTRAST_C, brightness);
+
+	return ssd130x_write_cmd(ssd130x, 4, SSD135X_SET_CONTRAST, a, b, c);
+}
+
 static int ssd133x_init(struct ssd130x_device *ssd130x)
 {
 	int ret;
+	/*
+	 * Horizontal address increment, normal SA,SB,SC (e.g. RGB) sub-pixel
+	 * order, COM split odd even and 65k (RGB565) color depth.
+	 */
+	const u8 remap = SSD133X_SET_REMAP_COM_SPLIT |
+			 FIELD_PREP(SSD133X_SET_REMAP_COLOR_DEPTH_MASK, SSD133X_COLOR_DEPTH_65K);
 	const u8 cmds[] = {
 		2, SSD133X_SET_MASTER_CURRENT, 0x06,
 		3, SSD133X_SET_COL_RANGE, 0x00, ssd130x->width - 1,
 		3, SSD133X_SET_ROW_RANGE, 0x00, ssd130x->height - 1,
-		/*
-		 * Horizontal Address Increment
-		 * Normal order SA,SB,SC (e.g. RGB)
-		 * COM Split Odd Even
-		 * 256 color format
-		 */
-		2, SSD13XX_SET_SEG_REMAP, 0x20,
+		2, SSD13XX_SET_SEG_REMAP, remap,
 		2, SSD133X_SET_DISPLAY_START, 0x00,
 		2, SSD133X_SET_DISPLAY_OFFSET, 0x00,
 		1, SSD133X_SET_DISPLAY_NORMAL,
@@ -652,6 +733,47 @@ static int ssd133x_init(struct ssd130x_device *ssd130x)
 	return ssd130x_run_cmd_seq(ssd130x, cmds);
 }
 
+static int ssd135x_init(struct ssd130x_device *ssd130x)
+{
+	/*
+	 * Horizontal address increment, COM split, reversed COM scan direction,
+	 * BGR sub-pixel order and 65k (RGB565) color depth. Rotation is not
+	 * supported, so the remap byte is fixed.
+	 */
+	const u8 remap = SSD135X_SET_REMAP_65K | SSD135X_SET_REMAP_COM_SPLIT |
+			 SSD135X_SET_REMAP_COLOR_BGR | SSD135X_SET_REMAP_COM_SCAN;
+	const u8 cmds[] = {
+		/* Unlock the controller, then the extended command set */
+		2, SSD135X_SET_COMMAND_LOCK, 0x12,
+		2, SSD135X_SET_COMMAND_LOCK, 0xb1,
+		1, SSD13XX_DISPLAY_OFF,
+		2, SSD135X_SET_CLOCK_FREQ, 0xf1,
+		2, SSD135X_SET_MUX_RATIO, ssd130x->height - 1,
+		3, SSD135X_SET_COL_RANGE, 0x00, ssd130x->width - 1,
+		3, SSD135X_SET_ROW_RANGE, 0x00, ssd130x->height - 1,
+		2, SSD135X_SET_DISPLAY_START, 0x00,
+		2, SSD135X_SET_DISPLAY_OFFSET, 0x00,
+		2, SSD135X_SET_GPIO, 0x00,
+		2, SSD135X_SET_FUNCTION, 0x01,
+		2, SSD135X_SET_PHASE_LENGTH, 0x32,
+		4, SSD135X_SET_VSL, 0xa0, 0xb5, 0x55,
+		2, SSD135X_SET_PRECHARGE_VOLTAGE, 0x17,
+		2, SSD135X_SET_VCOMH_VOLTAGE, 0x05,
+		2, SSD135X_SET_CONTRAST_MASTER, 0x0f,
+		2, SSD135X_SET_PRECHARGE2, 0x01,
+		1, SSD135X_SET_DISPLAY_NORMAL,
+		2, SSD13XX_SET_SEG_REMAP, remap,
+		0,
+	};
+	int ret;
+
+	ret = ssd130x_run_cmd_seq(ssd130x, cmds);
+	if (ret < 0)
+		return ret;
+
+	return ssd135x_set_contrast(ssd130x, ssd130x->contrast);
+}
+
 static int ssd130x_update_rect(struct ssd130x_device *ssd130x,
 			       struct drm_rect *rect, u8 *buf,
 			       u8 *data_array)
@@ -788,13 +910,12 @@ static int ssd132x_update_rect(struct ssd130x_device *ssd130x,
 	 * the second byte are SEG2 (D1[3:0]) and SEG3 (D1[7:4]) and so on.
 	 */
 
-	/* Set column start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, col, col + columns - 1);
+	/* Set address range for horizontal addressing mode */
+	ret = ssd132x_set_col_range(ssd130x, col, columns);
 	if (ret < 0)
 		return ret;
 
-	/* Set row start and end */
-	ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, row, row + rows - 1);
+	ret = ssd132x_set_row_range(ssd130x, row, rows);
 	if (ret < 0)
 		return ret;
 
@@ -829,14 +950,20 @@ static int ssd133x_update_rect(struct ssd130x_device *ssd130x,
 	 * COM0 to COM[N - 1] are the rows and SEG0 to SEG[M - 1] are
 	 * the columns.
 	 *
-	 * Each Segment has a 8-bit pixel and each Common output has a
-	 * row of pixels. When using the (default) horizontal address
-	 * increment mode, each byte of data sent to the controller has
-	 * a Segment (e.g: SEG0).
+	 * Each Segment holds one pixel and each Common output has a row
+	 * of pixels. A pixel is 8 bits (one byte) in the 256 color
+	 * (RGB332) format or 16 bits (two bytes) in the 65k color
+	 * (RGB565) format. When using the (default) horizontal address
+	 * increment mode, the pixel data is sent Segment by Segment
+	 * (e.g: SEG0 first).
 	 *
 	 * When using the 256 color depth format, each pixel contains 3
-	 * sub-pixels for color A, B and C. These have 3 bit, 3 bit and
-	 * 2 bits respectively.
+	 * sub-pixels for color A, B and C. These have 3, 3 and 2 bits
+	 * respectively.
+	 *
+	 * When using the 65k color depth format, each pixel contains 3
+	 * sub-pixels for color A, B and C. These have 5, 6 and 5 bits
+	 * respectively.
 	 */
 
 	/* Set column start and end */
@@ -855,6 +982,44 @@ static int ssd133x_update_rect(struct ssd130x_device *ssd130x,
 	return ret;
 }
 
+static int ssd135x_update_rect(struct ssd130x_device *ssd130x,
+			       struct drm_rect *rect, u8 *data_array,
+			       unsigned int pitch)
+{
+	unsigned int x = rect->x1;
+	unsigned int y = rect->y1;
+	unsigned int columns = drm_rect_width(rect);
+	unsigned int rows = drm_rect_height(rect);
+	int ret;
+
+	/*
+	 * The pixel layout is the same as for the ssd133x family: one 65k
+	 * color (RGB565) pixel per Segment, sent Segment by Segment when the
+	 * (default) horizontal address increment mode is used.
+	 *
+	 * But unlike the ssd133x family, which starts accepting pixel data as
+	 * soon as the address window has been programmed, the ssd135x family
+	 * needs an explicit Write RAM command before the data is written.
+	 */
+
+	/* Set column start and end */
+	ret = ssd130x_write_cmd(ssd130x, 3, SSD135X_SET_COL_RANGE, x, x + columns - 1);
+	if (ret < 0)
+		return ret;
+
+	/* Set row start and end */
+	ret = ssd130x_write_cmd(ssd130x, 3, SSD135X_SET_ROW_RANGE, y, y + rows - 1);
+	if (ret < 0)
+		return ret;
+
+	ret = ssd130x_write_cmd(ssd130x, 1, SSD135X_WRITE_RAM);
+	if (ret < 0)
+		return ret;
+
+	/* Write out update in one go since horizontal addressing mode is used */
+	return ssd130x_write_data(ssd130x, data_array, pitch * rows);
+}
+
 static void ssd130x_clear_screen(struct ssd130x_device *ssd130x, u8 *data_array)
 {
 	unsigned int pages = DIV_ROUND_UP(ssd130x->height, SSD130X_PAGE_HEIGHT);
@@ -900,18 +1065,28 @@ static void ssd132x_clear_screen(struct ssd130x_device *ssd130x, u8 *data_array)
 {
 	unsigned int columns = DIV_ROUND_UP(ssd130x->width, SSD132X_SEGMENT_WIDTH);
 	unsigned int height = ssd130x->height;
+	int ret;
 
 	memset(data_array, 0, columns * height);
 
+	ret = ssd132x_set_col_range(ssd130x, 0, columns);
+	if (ret < 0)
+		return;
+
+	ret = ssd132x_set_row_range(ssd130x, 0, height);
+	if (ret < 0)
+		return;
+
 	/* Write out update in one go since horizontal addressing mode is used */
 	ssd130x_write_data(ssd130x, data_array, columns * height);
 }
 
 static void ssd133x_clear_screen(struct ssd130x_device *ssd130x, u8 *data_array)
 {
-	const struct drm_format_info *fi = drm_format_info(DRM_FORMAT_RGB332);
+	const struct drm_format_info *fi;
 	unsigned int pitch;
 
+	fi = drm_format_info(DRM_FORMAT_RGB565);
 	if (!fi)
 		return;
 
@@ -923,6 +1098,23 @@ static void ssd133x_clear_screen(struct ssd130x_device *ssd130x, u8 *data_array)
 	ssd130x_write_data(ssd130x, data_array, pitch * ssd130x->height);
 }
 
+static void ssd135x_clear_screen(struct ssd130x_device *ssd130x, u8 *data_array)
+{
+	struct drm_rect screen = DRM_RECT_INIT(0, 0, ssd130x->width, ssd130x->height);
+	const struct drm_format_info *fi;
+	unsigned int pitch;
+
+	fi = drm_format_info(DRM_FORMAT_RGB565);
+	if (!fi)
+		return;
+
+	pitch = drm_format_info_min_pitch(fi, 0, ssd130x->width);
+
+	memset(data_array, 0, pitch * ssd130x->height);
+
+	ssd135x_update_rect(ssd130x, &screen, data_array, pitch);
+}
+
 static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
 				const struct iosys_map *vmap,
 				struct drm_rect *rect,
@@ -978,23 +1170,48 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb,
 				struct drm_format_conv_state *fmtcnv_state)
 {
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
-	const struct drm_format_info *fi = drm_format_info(DRM_FORMAT_RGB332);
+	const struct drm_format_info *fi;
 	unsigned int dst_pitch;
 	struct iosys_map dst;
 
+	fi = drm_format_info(DRM_FORMAT_RGB565);
 	if (!fi)
 		return -EINVAL;
 
 	dst_pitch = drm_format_info_min_pitch(fi, 0, drm_rect_width(rect));
 
 	iosys_map_set_vaddr(&dst, data_array);
-	drm_fb_xrgb8888_to_rgb332(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
+	drm_fb_xrgb8888_to_rgb565be(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
 
 	ssd133x_update_rect(ssd130x, rect, data_array, dst_pitch);
 
 	return 0;
 }
 
+static int ssd135x_fb_blit_rect(struct drm_framebuffer *fb,
+				const struct iosys_map *vmap,
+				struct drm_rect *rect, u8 *data_array,
+				struct drm_format_conv_state *fmtcnv_state)
+{
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
+	const struct drm_format_info *fi;
+	unsigned int dst_pitch;
+	struct iosys_map dst;
+
+	fi = drm_format_info(DRM_FORMAT_RGB565);
+	if (!fi)
+		return -EINVAL;
+
+	dst_pitch = drm_format_info_min_pitch(fi, 0, drm_rect_width(rect));
+
+	iosys_map_set_vaddr(&dst, data_array);
+	drm_fb_xrgb8888_to_rgb565be(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
+
+	ssd135x_update_rect(ssd130x, rect, data_array, dst_pitch);
+
+	return 0;
+}
+
 static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
 					      struct drm_atomic_commit *state)
 {
@@ -1237,6 +1454,45 @@ static void ssd133x_primary_plane_atomic_update(struct drm_plane *plane,
 	drm_dev_exit(idx);
 }
 
+static void ssd135x_primary_plane_atomic_update(struct drm_plane *plane,
+						struct drm_atomic_commit *state)
+{
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, plane_state->crtc);
+	struct ssd130x_crtc_state *ssd130x_crtc_state =  to_ssd130x_crtc_state(crtc_state);
+	struct drm_framebuffer *fb = plane_state->fb;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_device *drm = plane->dev;
+	struct drm_rect dst_clip;
+	struct drm_rect damage;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	if (drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE))
+		goto out_drm_dev_exit;
+
+	drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
+	drm_atomic_for_each_plane_damage(&iter, &damage) {
+		dst_clip = plane_state->dst;
+
+		if (!drm_rect_intersect(&dst_clip, &damage))
+			continue;
+
+		ssd135x_fb_blit_rect(fb, &shadow_plane_state->data[0], &dst_clip,
+				     ssd130x_crtc_state->data_array,
+				     &shadow_plane_state->fmtcnv_state);
+	}
+
+	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
+
+out_drm_dev_exit:
+	drm_dev_exit(idx);
+}
+
 static void ssd130x_primary_plane_atomic_disable(struct drm_plane *plane,
 						 struct drm_atomic_commit *state)
 {
@@ -1309,18 +1565,43 @@ static void ssd133x_primary_plane_atomic_disable(struct drm_plane *plane,
 	drm_dev_exit(idx);
 }
 
+static void ssd135x_primary_plane_atomic_disable(struct drm_plane *plane,
+						 struct drm_atomic_commit *state)
+{
+	struct drm_device *drm = plane->dev;
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
+	struct drm_plane_state *plane_state;
+	struct drm_crtc_state *crtc_state;
+	struct ssd130x_crtc_state *ssd130x_crtc_state;
+	int idx;
+
+	plane_state = drm_atomic_get_new_plane_state(state, plane);
+	if (!plane_state->crtc)
+		return;
+
+	crtc_state = drm_atomic_get_new_crtc_state(state, plane_state->crtc);
+	ssd130x_crtc_state = to_ssd130x_crtc_state(crtc_state);
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	ssd135x_clear_screen(ssd130x, ssd130x_crtc_state->data_array);
+
+	drm_dev_exit(idx);
+}
+
 /* Called during init to allocate the plane's atomic state. */
-static void ssd130x_primary_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *ssd130x_primary_plane_create_state(struct drm_plane *plane)
 {
 	struct ssd130x_plane_state *ssd130x_state;
 
-	drm_WARN_ON_ONCE(plane->dev, plane->state);
-
 	ssd130x_state = kzalloc_obj(*ssd130x_state);
 	if (!ssd130x_state)
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	__drm_gem_reset_shadow_plane(plane, &ssd130x_state->base);
+	__drm_gem_shadow_plane_state_init(plane, &ssd130x_state->base);
+
+	return &(&ssd130x_state->base)->base;
 }
 
 static struct drm_plane_state *ssd130x_primary_plane_duplicate_state(struct drm_plane *plane)
@@ -1377,13 +1658,19 @@ static const struct drm_plane_helper_funcs ssd130x_primary_plane_helper_funcs[]
 		.atomic_check = ssd133x_primary_plane_atomic_check,
 		.atomic_update = ssd133x_primary_plane_atomic_update,
 		.atomic_disable = ssd133x_primary_plane_atomic_disable,
-	}
+	},
+	[SSD135X_FAMILY] = {
+		DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+		.atomic_check = ssd133x_primary_plane_atomic_check,
+		.atomic_update = ssd135x_primary_plane_atomic_update,
+		.atomic_disable = ssd135x_primary_plane_atomic_disable,
+	},
 };
 
 static const struct drm_plane_funcs ssd130x_primary_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = ssd130x_primary_plane_reset,
+	.atomic_create_state = ssd130x_primary_plane_create_state,
 	.atomic_duplicate_state = ssd130x_primary_plane_duplicate_state,
 	.atomic_destroy_state = ssd130x_primary_plane_destroy_state,
 	.destroy = drm_plane_cleanup,
@@ -1446,10 +1733,11 @@ static int ssd133x_crtc_atomic_check(struct drm_crtc *crtc,
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	struct ssd130x_crtc_state *ssd130x_state = to_ssd130x_crtc_state(crtc_state);
-	const struct drm_format_info *fi = drm_format_info(DRM_FORMAT_RGB332);
+	const struct drm_format_info *fi;
 	unsigned int pitch;
 	int ret;
 
+	fi = drm_format_info(DRM_FORMAT_RGB565);
 	if (!fi)
 		return -EINVAL;
 
@@ -1467,7 +1755,7 @@ static int ssd133x_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 /* Called during init to allocate the CRTC's atomic state. */
-static void ssd130x_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *ssd130x_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct ssd130x_crtc_state *ssd130x_state;
 
@@ -1475,9 +1763,11 @@ static void ssd130x_crtc_reset(struct drm_crtc *crtc)
 
 	ssd130x_state = kzalloc_obj(*ssd130x_state);
 	if (!ssd130x_state)
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	__drm_atomic_helper_crtc_reset(crtc, &ssd130x_state->base);
+	__drm_atomic_helper_crtc_state_init(&ssd130x_state->base, crtc);
+
+	return &ssd130x_state->base;
 }
 
 static struct drm_crtc_state *ssd130x_crtc_duplicate_state(struct drm_crtc *crtc)
@@ -1531,10 +1821,14 @@ static const struct drm_crtc_helper_funcs ssd130x_crtc_helper_funcs[] = {
 		.mode_valid = ssd130x_crtc_mode_valid,
 		.atomic_check = ssd133x_crtc_atomic_check,
 	},
+	[SSD135X_FAMILY] = {
+		.mode_valid = ssd130x_crtc_mode_valid,
+		.atomic_check = ssd133x_crtc_atomic_check,
+	},
 };
 
 static const struct drm_crtc_funcs ssd130x_crtc_funcs = {
-	.reset = ssd130x_crtc_reset,
+	.atomic_create_state = ssd130x_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -1618,6 +1912,31 @@ static void ssd133x_encoder_atomic_enable(struct drm_encoder *encoder,
 	ssd130x_power_off(ssd130x);
 }
 
+static void ssd135x_encoder_atomic_enable(struct drm_encoder *encoder,
+					  struct drm_atomic_commit *state)
+{
+	struct drm_device *drm = encoder->dev;
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
+	int ret;
+
+	ret = ssd130x_power_on(ssd130x);
+	if (ret)
+		return;
+
+	ret = ssd135x_init(ssd130x);
+	if (ret)
+		goto power_off;
+
+	ssd130x_write_cmd(ssd130x, 1, SSD13XX_DISPLAY_ON);
+
+	backlight_enable(ssd130x->bl_dev);
+
+	return;
+
+power_off:
+	ssd130x_power_off(ssd130x);
+}
+
 static void ssd130x_encoder_atomic_disable(struct drm_encoder *encoder,
 					   struct drm_atomic_commit *state)
 {
@@ -1643,7 +1962,11 @@ static const struct drm_encoder_helper_funcs ssd130x_encoder_helper_funcs[] = {
 	[SSD133X_FAMILY] = {
 		.atomic_enable = ssd133x_encoder_atomic_enable,
 		.atomic_disable = ssd130x_encoder_atomic_disable,
-	}
+	},
+	[SSD135X_FAMILY] = {
+		.atomic_enable = ssd135x_encoder_atomic_enable,
+		.atomic_disable = ssd130x_encoder_atomic_disable,
+	},
 };
 
 static const struct drm_encoder_funcs ssd130x_encoder_funcs = {
@@ -1720,6 +2043,15 @@ static int ssd133x_update_bl(struct backlight_device *bdev)
 	return ssd133x_set_contrast(ssd130x, ssd130x->contrast);
 }
 
+static int ssd135x_update_bl(struct backlight_device *bdev)
+{
+	struct ssd130x_device *ssd130x = bl_get_data(bdev);
+
+	ssd130x->contrast = backlight_get_brightness(bdev);
+
+	return ssd135x_set_contrast(ssd130x, ssd130x->contrast);
+}
+
 static const struct backlight_ops ssd130xfb_bl_ops[] = {
 	[SSD130X_FAMILY] = {
 		.update_status	= ssd130x_update_bl,
@@ -1730,6 +2062,9 @@ static const struct backlight_ops ssd130xfb_bl_ops[] = {
 	[SSD133X_FAMILY] = {
 		.update_status	= ssd133x_update_bl,
 	},
+	[SSD135X_FAMILY] = {
+		.update_status	= ssd135x_update_bl,
+	},
 };
 
 static void ssd130x_parse_properties(struct ssd130x_device *ssd130x)
diff --git a/drivers/gpu/drm/solomon/ssd130x.h b/drivers/gpu/drm/solomon/ssd130x.h
index a4554018bb2a49..fd81262f2ceb97 100644
--- a/drivers/gpu/drm/solomon/ssd130x.h
+++ b/drivers/gpu/drm/solomon/ssd130x.h
@@ -26,7 +26,8 @@
 enum ssd130x_family_ids {
 	SSD130X_FAMILY,
 	SSD132X_FAMILY,
-	SSD133X_FAMILY
+	SSD133X_FAMILY,
+	SSD135X_FAMILY,
 };
 
 enum ssd130x_variants {
@@ -42,6 +43,8 @@ enum ssd130x_variants {
 	SSD1327_ID,
 	/* ssd133x family */
 	SSD1331_ID,
+	/* ssd135x family */
+	SSD1351_ID,
 	NR_SSD130X_VARIANTS
 };
 
@@ -54,6 +57,11 @@ struct ssd130x_deviceinfo {
 	bool need_pwm;
 	bool need_chargepump;
 	bool page_mode_only;
+	/*
+	 * Whether the controller takes a command's parameters on the data path
+	 * instead of as further command bytes.
+	 */
+	bool cmd_params_are_data;
 
 	enum ssd130x_family_ids family_id;
 };
diff --git a/drivers/gpu/drm/sprd/sprd_dpu.c b/drivers/gpu/drm/sprd/sprd_dpu.c
index 66990398b90379..8fa3c733cc4f87 100644
--- a/drivers/gpu/drm/sprd/sprd_dpu.c
+++ b/drivers/gpu/drm/sprd/sprd_dpu.c
@@ -578,7 +578,7 @@ static const struct drm_plane_funcs sprd_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
@@ -707,7 +707,7 @@ static const struct drm_crtc_funcs sprd_crtc_funcs = {
 	.destroy	= drm_crtc_cleanup,
 	.set_config	= drm_atomic_helper_set_config,
 	.page_flip	= drm_atomic_helper_page_flip,
-	.reset		= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank	= sprd_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index f0e98653d35a2b..c69612728f2bc9 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -328,7 +328,7 @@ static const struct drm_crtc_funcs sti_crtc_funcs = {
 	.page_flip = drm_atomic_helper_page_flip,
 	.destroy = sti_crtc_destroy,
 	.set_property = sti_crtc_set_property,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.late_register = sti_crtc_late_register,
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index d0b89b28e50cca..6a3ddb7fc17d91 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -240,7 +240,7 @@ static int sti_cursor_atomic_check(struct drm_plane *drm_plane,
 		cursor->pixmap.base = dma_alloc_wc(cursor->dev,
 						   cursor->pixmap.size,
 						   &cursor->pixmap.paddr,
-						   GFP_KERNEL | GFP_DMA);
+						   GFP_KERNEL);
 		if (!cursor->pixmap.base) {
 			DRM_ERROR("Failed to allocate memory for pixmap\n");
 			return -EINVAL;
@@ -356,7 +356,7 @@ static const struct drm_plane_funcs sti_cursor_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.late_register = sti_cursor_late_register,
@@ -380,7 +380,7 @@ struct drm_plane *sti_cursor_create(struct drm_device *drm_dev,
 	/* Allocate clut buffer */
 	size = 0x100 * sizeof(unsigned short);
 	cursor->clut = dma_alloc_wc(dev, size, &cursor->clut_paddr,
-				    GFP_KERNEL | GFP_DMA);
+				    GFP_KERNEL);
 
 	if (!cursor->clut) {
 		DRM_ERROR("Failed to allocate memory for cursor clut\n");
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index 9a17b590aa3d87..89511abd4b392d 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -911,7 +911,7 @@ static const struct drm_plane_funcs sti_gdp_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.late_register = sti_gdp_late_register,
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index cf1ed8a33b8e18..b367f7f84f5fe9 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -860,7 +860,7 @@ static void sti_hqvdp_init(struct sti_hqvdp *hqvdp)
 	size = NB_VDP_CMD * sizeof(struct sti_hqvdp_cmd);
 	hqvdp->hqvdp_cmd = dma_alloc_wc(hqvdp->dev, size,
 					&dma_addr,
-					GFP_KERNEL | GFP_DMA);
+					GFP_KERNEL);
 	if (!hqvdp->hqvdp_cmd) {
 		DRM_ERROR("Failed to allocate memory for VDP cmd\n");
 		return;
@@ -1289,7 +1289,7 @@ static const struct drm_plane_funcs sti_hqvdp_plane_helpers_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.late_register = sti_hqvdp_late_register,
diff --git a/drivers/gpu/drm/stm/Kconfig b/drivers/gpu/drm/stm/Kconfig
index 635be0ac00afae..1e3205aa2c7e84 100644
--- a/drivers/gpu/drm/stm/Kconfig
+++ b/drivers/gpu/drm/stm/Kconfig
@@ -6,6 +6,7 @@ config DRM_STM
 	select DRM_CLIENT_SELECTION
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
+	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
 	select FB_PROVIDE_GET_FB_UNMAPPED_AREA if FB
diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c
index 8818dcf98da796..5311e0b3e8dcdf 100644
--- a/drivers/gpu/drm/stm/ltdc.c
+++ b/drivers/gpu/drm/stm/ltdc.c
@@ -1209,7 +1209,7 @@ static void ltdc_crtc_atomic_print_state(struct drm_printer *p,
 static const struct drm_crtc_funcs ltdc_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = ltdc_crtc_enable_vblank,
@@ -1221,7 +1221,7 @@ static const struct drm_crtc_funcs ltdc_crtc_funcs = {
 static const struct drm_crtc_funcs ltdc_crtc_with_crc_support_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank = ltdc_crtc_enable_vblank,
@@ -1554,7 +1554,7 @@ static void ltdc_plane_atomic_print_state(struct drm_printer *p,
 static const struct drm_plane_funcs ltdc_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 	.atomic_print_state = ltdc_plane_atomic_print_state,
diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c
index 3fa1f4fda0f0d0..e6a20e2f0ade17 100644
--- a/drivers/gpu/drm/sun4i/sun4i_crtc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c
@@ -183,7 +183,7 @@ static const struct drm_crtc_funcs sun4i_crtc_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
 	.destroy		= drm_crtc_cleanup,
 	.page_flip		= drm_atomic_helper_page_flip,
-	.reset			= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.set_config		= drm_atomic_helper_set_config,
 	.enable_vblank		= sun4i_crtc_enable_vblank,
 	.disable_vblank		= sun4i_crtc_disable_vblank,
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
index 97253d631b2bdb..d98b6f12a7bc30 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
@@ -292,6 +292,7 @@ static void sun4i_hdmi_connector_reset(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs sun4i_hdmi_connector_funcs = {
+	.debugfs_init		= drm_hdmi_connector_debugfs_init,
 	.detect			= sun4i_hdmi_connector_detect,
 	.fill_modes		= drm_helper_probe_single_connector_modes,
 	.reset			= sun4i_hdmi_connector_reset,
@@ -653,7 +654,7 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 
 	drm_connector_helper_add(&hdmi->connector,
 				 &sun4i_hdmi_connector_helper_funcs);
-	ret = drmm_connector_hdmi_init(drm, &hdmi->connector,
+	ret = drmm_connector_hdmi_ini2(drm, &hdmi->connector,
 				       /*
 					* NOTE: Those are likely to be
 					* wrong, but I couldn't find the
diff --git a/drivers/gpu/drm/sun4i/sun4i_layer.c b/drivers/gpu/drm/sun4i/sun4i_layer.c
index d65587d96d9eee..7ec6d81319085e 100644
--- a/drivers/gpu/drm/sun4i/sun4i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_layer.c
@@ -16,22 +16,17 @@
 #include "sun4i_layer.h"
 #include "sunxi_engine.h"
 
-static void sun4i_backend_layer_reset(struct drm_plane *plane)
+static struct drm_plane_state *sun4i_backend_layer_create_state(struct drm_plane *plane)
 {
 	struct sun4i_layer_state *state;
 
-	if (plane->state) {
-		state = state_to_sun4i_layer_state(plane->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-		__drm_atomic_helper_plane_destroy_state(&state->state);
+	__drm_atomic_helper_plane_state_init(&state->state, plane);
 
-		kfree(state);
-		plane->state = NULL;
-	}
-
-	state = kzalloc_obj(*state);
-	if (state)
-		__drm_atomic_helper_plane_reset(plane, &state->state);
+	return &state->state;
 }
 
 static struct drm_plane_state *
@@ -133,7 +128,7 @@ static const struct drm_plane_funcs sun4i_backend_layer_funcs = {
 	.atomic_duplicate_state	= sun4i_backend_layer_duplicate_state,
 	.destroy		= drm_plane_cleanup,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= sun4i_backend_layer_reset,
+	.atomic_create_state	= sun4i_backend_layer_create_state,
 	.update_plane		= drm_atomic_helper_update_plane,
 	.format_mod_supported	= sun4i_layer_format_mod_supported,
 };
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
index 00756ab78b4eb2..bad102134726b3 100644
--- a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
@@ -213,7 +213,7 @@ static const struct drm_plane_funcs sun8i_ui_layer_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.destroy		= drm_plane_cleanup,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.update_plane		= drm_atomic_helper_update_plane,
 };
 
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
index 09f668c8af2478..2e9cda45c04ef7 100644
--- a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
@@ -293,7 +293,7 @@ static const struct drm_plane_funcs sun8i_vi_layer_funcs = {
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.destroy		= drm_plane_cleanup,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.update_plane		= drm_atomic_helper_update_plane,
 };
 
diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
index 547f2327af5e2d..c043428e02fd7d 100644
--- a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
+++ b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
@@ -12,6 +12,7 @@
 #include <drm/drm_device.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_panic_helper.h>
 
 struct drm_format_info;
 struct drm_scanout_buffer;
@@ -134,17 +135,18 @@ int drm_sysfb_plane_helper_get_scanout_buffer(struct drm_plane *plane,
 	.atomic_disable = drm_sysfb_plane_helper_atomic_disable, \
 	.get_scanout_buffer = drm_sysfb_plane_helper_get_scanout_buffer
 
-void drm_sysfb_plane_reset(struct drm_plane *plane);
+struct drm_plane_state *drm_sysfb_plane_atomic_create_state(struct drm_plane *plane);
 struct drm_plane_state *drm_sysfb_plane_atomic_duplicate_state(struct drm_plane *plane);
 void drm_sysfb_plane_atomic_destroy_state(struct drm_plane *plane,
 					  struct drm_plane_state *plane_state);
 
 #define DRM_SYSFB_PLANE_FUNCS \
-	.reset = drm_sysfb_plane_reset, \
 	.update_plane = drm_atomic_helper_update_plane, \
 	.disable_plane = drm_atomic_helper_disable_plane, \
+	.atomic_create_state = drm_sysfb_plane_atomic_create_state, \
 	.atomic_duplicate_state = drm_sysfb_plane_atomic_duplicate_state, \
-	.atomic_destroy_state = drm_sysfb_plane_atomic_destroy_state
+	.atomic_destroy_state = drm_sysfb_plane_atomic_destroy_state, \
+	DRM_PANIC_PLANE_FUNCS
 
 /*
  * CRTC
@@ -171,12 +173,12 @@ int drm_sysfb_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_
 	.mode_valid = drm_sysfb_crtc_helper_mode_valid, \
 	.atomic_check = drm_sysfb_crtc_helper_atomic_check
 
-void drm_sysfb_crtc_reset(struct drm_crtc *crtc);
+struct drm_crtc_state *drm_sysfb_crtc_create_state(struct drm_crtc *crtc);
 struct drm_crtc_state *drm_sysfb_crtc_atomic_duplicate_state(struct drm_crtc *crtc);
 void drm_sysfb_crtc_atomic_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state);
 
 #define DRM_SYSFB_CRTC_FUNCS \
-	.reset = drm_sysfb_crtc_reset, \
+	.atomic_create_state = drm_sysfb_crtc_create_state, \
 	.set_config = drm_atomic_helper_set_config, \
 	.page_flip = drm_atomic_helper_page_flip, \
 	.atomic_duplicate_state = drm_sysfb_crtc_atomic_duplicate_state, \
diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c b/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c
index d2de29caf89eb6..13df0925163a56 100644
--- a/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c
+++ b/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c
@@ -375,31 +375,27 @@ void drm_sysfb_plane_helper_atomic_disable(struct drm_plane *plane,
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(dev);
-	struct iosys_map dst = sysfb->fb_addr;
-	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
-	void __iomem *dst_vmap = dst.vaddr_iomem; /* TODO: Use mapping abstraction */
 	unsigned int dst_pitch = sysfb->fb_pitch;
 	const struct drm_format_info *dst_format = sysfb->fb_format;
 	struct drm_rect dst_clip;
-	unsigned long lines, linepixels, i;
+	unsigned long lines, linepixels, dst_size, i;
 	int idx;
 
-	drm_rect_init(&dst_clip,
-		      plane_state->src_x >> 16, plane_state->src_y >> 16,
-		      plane_state->src_w >> 16, plane_state->src_h >> 16);
+	drm_rect_init(&dst_clip, 0, 0,
+		      sysfb->fb_mode.hdisplay, sysfb->fb_mode.vdisplay);
 
 	lines = drm_rect_height(&dst_clip);
 	linepixels = drm_rect_width(&dst_clip);
+	dst_size = min_t(unsigned long,
+			 drm_format_info_min_pitch(dst_format, 0, linepixels),
+			 dst_pitch);
 
 	if (!drm_dev_enter(dev, &idx))
 		return;
 
 	/* Clear buffer to black if disabled */
-	dst_vmap += drm_fb_clip_offset(dst_pitch, dst_format, &dst_clip);
-	for (i = 0; i < lines; ++i) {
-		memset_io(dst_vmap, 0, linepixels * dst_format->cpp[0]);
-		dst_vmap += dst_pitch;
-	}
+	for (i = 0; i < lines; ++i)
+		iosys_map_memset(&sysfb->fb_addr, dst_pitch * i, 0, dst_size);
 
 	drm_dev_exit(idx);
 }
@@ -420,20 +416,32 @@ int drm_sysfb_plane_helper_get_scanout_buffer(struct drm_plane *plane,
 }
 EXPORT_SYMBOL(drm_sysfb_plane_helper_get_scanout_buffer);
 
-void drm_sysfb_plane_reset(struct drm_plane *plane)
+/**
+ * drm_sysfb_plane_atomic_create_state - creates sysfb plane state
+ * @plane: the plane
+ *
+ * This function implements struct &drm_plane_funcs.atomic_create_state for
+ * sysfb planes. It allocates the plane state of type
+ * struct drm_sysfb_plane_state.
+ *
+ * Returns:
+ * A pointer to a new plane state on success, or an ERR_PTR()-encoded
+ * error code otherwise.
+ */
+struct drm_plane_state *
+drm_sysfb_plane_atomic_create_state(struct drm_plane *plane)
 {
 	struct drm_sysfb_plane_state *sysfb_plane_state;
 
-	if (plane->state)
-		drm_sysfb_plane_state_destroy(to_drm_sysfb_plane_state(plane->state));
-
 	sysfb_plane_state = kzalloc_obj(*sysfb_plane_state);
-	if (sysfb_plane_state)
-		__drm_gem_reset_shadow_plane(plane, &sysfb_plane_state->base);
-	else
-		__drm_gem_reset_shadow_plane(plane, NULL);
+	if (!sysfb_plane_state)
+		return ERR_PTR(-ENOMEM);
+
+	__drm_gem_shadow_plane_state_init(plane, &sysfb_plane_state->base);
+
+	return &sysfb_plane_state->base.base;
 }
-EXPORT_SYMBOL(drm_sysfb_plane_reset);
+EXPORT_SYMBOL(drm_sysfb_plane_atomic_create_state);
 
 struct drm_plane_state *drm_sysfb_plane_atomic_duplicate_state(struct drm_plane *plane)
 {
@@ -515,23 +523,21 @@ int drm_sysfb_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_
 }
 EXPORT_SYMBOL(drm_sysfb_crtc_helper_atomic_check);
 
-void drm_sysfb_crtc_reset(struct drm_crtc *crtc)
+struct drm_crtc_state *drm_sysfb_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(crtc->dev);
 	struct drm_sysfb_crtc_state *sysfb_crtc_state;
 
-	if (crtc->state)
-		drm_sysfb_crtc_state_destroy(to_drm_sysfb_crtc_state(crtc->state));
-
 	sysfb_crtc_state = kzalloc_obj(*sysfb_crtc_state);
-	if (sysfb_crtc_state) {
-		sysfb_crtc_state->format = sysfb->fb_format;
-		__drm_atomic_helper_crtc_reset(crtc, &sysfb_crtc_state->base);
-	} else {
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
-	}
+	if (!sysfb_crtc_state)
+		return ERR_PTR(-ENOMEM);
+
+	sysfb_crtc_state->format = sysfb->fb_format;
+	__drm_atomic_helper_crtc_state_init(&sysfb_crtc_state->base, crtc);
+
+	return &sysfb_crtc_state->base;
 }
-EXPORT_SYMBOL(drm_sysfb_crtc_reset);
+EXPORT_SYMBOL(drm_sysfb_crtc_create_state);
 
 struct drm_crtc_state *drm_sysfb_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
 {
diff --git a/drivers/gpu/drm/sysfb/efidrm.c b/drivers/gpu/drm/sysfb/efidrm.c
index 3f9cd5d03efb23..8b19f437308eb9 100644
--- a/drivers/gpu/drm/sysfb/efidrm.c
+++ b/drivers/gpu/drm/sysfb/efidrm.c
@@ -152,6 +152,7 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv,
 	const struct screen_info *si;
 	const struct drm_format_info *format;
 	int width, height, stride;
+	unsigned int panel_width, panel_height;
 	s64 vsize;
 	u64 mem_flags;
 	struct resource resbuf;
@@ -217,6 +218,20 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv,
 	if (drm_edid_header_is_valid(dpy->edid.dummy) == 8)
 		sysfb->edid = dpy->edid.dummy;
 #endif
+
+	panel_width = width;
+	panel_height = height;
+
+	if (sysfb->edid) {
+		const struct drm_edid *drm_edid;
+
+		drm_edid = drm_edid_alloc(sysfb->edid, EDID_LENGTH);
+		if (drm_edid) {
+			drm_edid_detect_panel_size(drm_edid, &panel_width, &panel_height);
+			drm_edid_free(drm_edid);
+		}
+	}
+
 	sysfb->fb_mode = drm_sysfb_mode(width, height, 0, 0);
 	sysfb->fb_format = format;
 	sysfb->fb_pitch = stride;
@@ -340,7 +355,7 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv,
 	drm_connector_helper_add(connector, &efidrm_connector_helper_funcs);
 	drm_connector_set_panel_orientation_with_quirk(connector,
 						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
-						       width, height);
+						       panel_width, panel_height);
 	if (sysfb->edid)
 		drm_connector_attach_edid_property(connector);
 
diff --git a/drivers/gpu/drm/sysfb/ofdrm.c b/drivers/gpu/drm/sysfb/ofdrm.c
index 9d60db45139c3a..c41255569854b2 100644
--- a/drivers/gpu/drm/sysfb/ofdrm.c
+++ b/drivers/gpu/drm/sysfb/ofdrm.c
@@ -229,7 +229,7 @@ static const u8 *display_get_edid_of(struct drm_device *dev, struct device_node
 {
 	int ret = of_property_read_u8_array(of_node, "EDID", buf, EDID_LENGTH);
 
-	if (ret)
+	if (ret || drm_edid_header_is_valid(buf) != 8)
 		return NULL;
 	return buf;
 }
@@ -828,6 +828,7 @@ static struct ofdrm_device *ofdrm_device_create(struct drm_driver *drv,
 	enum ofdrm_model model;
 	bool big_endian;
 	int width, height, depth, linebytes;
+	unsigned int panel_width, panel_height;
 	const struct drm_format_info *format;
 	u64 address;
 	const u8 *edid;
@@ -998,6 +999,19 @@ static struct ofdrm_device *ofdrm_device_create(struct drm_driver *drv,
 		sysfb->fb_gamma_lut_size = OFDRM_GAMMA_LUT_SIZE;
 	sysfb->edid = edid;
 
+	panel_width = width;
+	panel_height = height;
+
+	if (sysfb->edid) {
+		const struct drm_edid *drm_edid;
+
+		drm_edid = drm_edid_alloc(sysfb->edid, EDID_LENGTH);
+		if (drm_edid) {
+			drm_edid_detect_panel_size(drm_edid, &panel_width, &panel_height);
+			drm_edid_free(drm_edid);
+		}
+	}
+
 	drm_dbg(dev, "display mode={" DRM_MODE_FMT "}\n", DRM_MODE_ARG(&sysfb->fb_mode));
 	drm_dbg(dev, "framebuffer format=%p4cc, size=%dx%d, linebytes=%d byte\n",
 		&format->format, width, height, linebytes);
@@ -1069,7 +1083,7 @@ static struct ofdrm_device *ofdrm_device_create(struct drm_driver *drv,
 	drm_connector_helper_add(connector, &ofdrm_connector_helper_funcs);
 	drm_connector_set_panel_orientation_with_quirk(connector,
 						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
-						       width, height);
+						       panel_width, panel_height);
 	if (edid)
 		drm_connector_attach_edid_property(connector);
 
diff --git a/drivers/gpu/drm/sysfb/vesadrm.c b/drivers/gpu/drm/sysfb/vesadrm.c
index 6a67b2d2e451b3..1668aabf8687f7 100644
--- a/drivers/gpu/drm/sysfb/vesadrm.c
+++ b/drivers/gpu/drm/sysfb/vesadrm.c
@@ -24,6 +24,7 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -402,6 +403,7 @@ static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv,
 	const struct screen_info *si;
 	const struct drm_format_info *format;
 	int width, height, stride;
+	unsigned int panel_width, panel_height;
 	s64 vsize;
 	struct resource resbuf;
 	struct resource *res;
@@ -484,6 +486,20 @@ static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv,
 	if (drm_edid_header_is_valid(dpy->edid.dummy) == 8)
 		sysfb->edid = dpy->edid.dummy;
 #endif
+
+	panel_width = width;
+	panel_height = height;
+
+	if (sysfb->edid) {
+		const struct drm_edid *drm_edid;
+
+		drm_edid = drm_edid_alloc(sysfb->edid, EDID_LENGTH);
+		if (drm_edid) {
+			drm_edid_detect_panel_size(drm_edid, &panel_width, &panel_height);
+			drm_edid_free(drm_edid);
+		}
+	}
+
 	sysfb->fb_mode = drm_sysfb_mode(width, height, 0, 0);
 	sysfb->fb_format = format;
 	sysfb->fb_pitch = stride;
@@ -585,7 +601,7 @@ static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv,
 	drm_connector_helper_add(connector, &vesadrm_connector_helper_funcs);
 	drm_connector_set_panel_orientation_with_quirk(connector,
 						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
-						       width, height);
+						       panel_width, panel_height);
 	if (sysfb->edid)
 		drm_connector_attach_edid_property(connector);
 
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index fefc3761a4bc79..b0bfa946e6979f 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -1492,17 +1492,17 @@ static void tegra_dc_destroy(struct drm_crtc *crtc)
 	drm_crtc_cleanup(crtc);
 }
 
-static void tegra_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *tegra_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct tegra_dc_state *state = kzalloc_obj(*state);
+	struct tegra_dc_state *state;
 
-	if (crtc->state)
-		tegra_crtc_atomic_destroy_state(crtc, crtc->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	if (state)
-		__drm_atomic_helper_crtc_reset(crtc, &state->base);
-	else
-		__drm_atomic_helper_crtc_reset(crtc, NULL);
+	__drm_atomic_helper_crtc_state_init(&state->base, crtc);
+
+	return &state->base;
 }
 
 static struct drm_crtc_state *
@@ -1911,7 +1911,7 @@ static const struct drm_crtc_funcs tegra_crtc_funcs = {
 	.page_flip = drm_atomic_helper_page_flip,
 	.set_config = drm_atomic_helper_set_config,
 	.destroy = tegra_dc_destroy,
-	.reset = tegra_crtc_reset,
+	.atomic_create_state = tegra_crtc_create_state,
 	.atomic_duplicate_state = tegra_crtc_atomic_duplicate_state,
 	.atomic_destroy_state = tegra_crtc_atomic_destroy_state,
 	.late_register = tegra_dc_late_register,
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index cb88aafbd36f20..7b595310e73799 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -20,11 +20,11 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_debugfs.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_file.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "dc.h"
 #include "drm.h"
@@ -1055,6 +1055,10 @@ tegra_dsi_encoder_atomic_check(struct drm_encoder *encoder,
 	return err;
 }
 
+static const struct drm_encoder_funcs tegra_dsi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs tegra_dsi_encoder_helper_funcs = {
 	.disable = tegra_dsi_encoder_disable,
 	.enable = tegra_dsi_encoder_enable,
@@ -1078,8 +1082,14 @@ static int tegra_dsi_init(struct host1x_client *client)
 					 &tegra_dsi_connector_helper_funcs);
 		dsi->output.connector.dpms = DRM_MODE_DPMS_OFF;
 
-		drm_simple_encoder_init(drm, &dsi->output.encoder,
-					DRM_MODE_ENCODER_DSI);
+		err = drm_encoder_init(drm, &dsi->output.encoder,
+				       &tegra_dsi_encoder_funcs,
+				       DRM_MODE_ENCODER_DSI, NULL);
+		if (err) {
+			drm_err(drm, "failed to initialize encoder: %d\n", err);
+			return err;
+		}
+
 		drm_encoder_helper_add(&dsi->output.encoder,
 				       &tegra_dsi_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c
index 0cb30910773f27..2148b0c59815c2 100644
--- a/drivers/gpu/drm/tegra/plane.c
+++ b/drivers/gpu/drm/tegra/plane.c
@@ -24,28 +24,24 @@ static void tegra_plane_destroy(struct drm_plane *plane)
 	kfree(p);
 }
 
-static void tegra_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *tegra_plane_atomic_create_state(struct drm_plane *plane)
 {
 	struct tegra_plane *p = to_tegra_plane(plane);
 	struct tegra_plane_state *state;
 	unsigned int i;
 
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
 
-	kfree(plane->state);
-	plane->state = NULL;
+	state->base.plane = plane;
+	state->base.zpos = p->index;
+	state->base.normalized_zpos = p->index;
 
-	state = kzalloc_obj(*state);
-	if (state) {
-		plane->state = &state->base;
-		plane->state->plane = plane;
-		plane->state->zpos = p->index;
-		plane->state->normalized_zpos = p->index;
-
-		for (i = 0; i < 3; i++)
-			state->iova[i] = DMA_MAPPING_ERROR;
-	}
+	for (i = 0; i < 3; i++)
+		state->iova[i] = DMA_MAPPING_ERROR;
+
+	return &state->base;
 }
 
 static struct drm_plane_state *
@@ -131,7 +127,7 @@ const struct drm_plane_funcs tegra_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = tegra_plane_destroy,
-	.reset = tegra_plane_reset,
+	.atomic_create_state = tegra_plane_atomic_create_state,
 	.atomic_duplicate_state = tegra_plane_atomic_duplicate_state,
 	.atomic_destroy_state = tegra_plane_atomic_destroy_state,
 	.format_mod_supported = tegra_plane_format_mod_supported,
diff --git a/drivers/gpu/drm/tegra/rgb.c b/drivers/gpu/drm/tegra/rgb.c
index e67fbb2362e6ec..b6d09a8d8e5ea4 100644
--- a/drivers/gpu/drm/tegra/rgb.c
+++ b/drivers/gpu/drm/tegra/rgb.c
@@ -9,7 +9,9 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge_connector.h>
-#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
 
 #include "drm.h"
 #include "dc.h"
@@ -194,6 +196,10 @@ tegra_rgb_encoder_atomic_check(struct drm_encoder *encoder,
 	return err;
 }
 
+static const struct drm_encoder_funcs tegra_rgb_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs tegra_rgb_encoder_helper_funcs = {
 	.disable = tegra_rgb_encoder_disable,
 	.enable = tegra_rgb_encoder_enable,
@@ -305,7 +311,13 @@ int tegra_dc_rgb_init(struct drm_device *drm, struct tegra_dc *dc)
 	if (!dc->rgb)
 		return -ENODEV;
 
-	drm_simple_encoder_init(drm, &output->encoder, DRM_MODE_ENCODER_LVDS);
+	err = drm_encoder_init(drm, &output->encoder, &tegra_rgb_encoder_funcs,
+			       DRM_MODE_ENCODER_LVDS, NULL);
+	if (err) {
+		drm_err(drm, "failed to initialize encoder: %d\n", err);
+		return err;
+	}
+
 	drm_encoder_helper_add(&output->encoder,
 			       &tegra_rgb_encoder_helper_funcs);
 
diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile
index e7193b49cf91f8..42207df1d1c8b6 100644
--- a/drivers/gpu/drm/tests/Makefile
+++ b/drivers/gpu/drm/tests/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_DRM_KUNIT_TEST) += \
 	drm_managed_test.o \
 	drm_mm_test.o \
 	drm_modes_test.o \
+	drm_panic_helper_test.o \
 	drm_plane_helper_test.o \
 	drm_probe_helper_test.o \
 	drm_rect_test.o \
diff --git a/drivers/gpu/drm/tests/drm_connector_test.c b/drivers/gpu/drm/tests/drm_connector_test.c
index beb1d50a664641..3368aedd4e6699 100644
--- a/drivers/gpu/drm/tests/drm_connector_test.c
+++ b/drivers/gpu/drm/tests/drm_connector_test.c
@@ -669,7 +669,7 @@ static void drm_test_connector_hdmi_init_valid(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -689,7 +689,7 @@ static void drm_test_connector_hdmi_init_null_ddc(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -709,7 +709,7 @@ static void drm_test_connector_hdmi_init_null_vendor(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       NULL, "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -729,7 +729,7 @@ static void drm_test_connector_hdmi_init_null_product(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", NULL,
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -755,7 +755,7 @@ static void drm_test_connector_hdmi_init_product_valid(struct kunit *test)
 
 	KUNIT_ASSERT_LT(test, strlen(product_name), DRM_CONNECTOR_HDMI_PRODUCT_LEN);
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", product_name,
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -788,7 +788,7 @@ static void drm_test_connector_hdmi_init_product_length_exact(struct kunit *test
 
 	KUNIT_ASSERT_EQ(test, strlen(product_name), DRM_CONNECTOR_HDMI_PRODUCT_LEN);
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", product_name,
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -815,7 +815,7 @@ static void drm_test_connector_hdmi_init_product_length_too_long(struct kunit *t
 
 	KUNIT_ASSERT_GT(test, strlen(product_name), DRM_CONNECTOR_HDMI_PRODUCT_LEN);
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", product_name,
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -841,7 +841,7 @@ static void drm_test_connector_hdmi_init_vendor_valid(struct kunit *test)
 
 	KUNIT_ASSERT_LT(test, strlen(vendor_name), DRM_CONNECTOR_HDMI_VENDOR_LEN);
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       vendor_name, "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -873,7 +873,7 @@ static void drm_test_connector_hdmi_init_vendor_length_exact(struct kunit *test)
 
 	KUNIT_ASSERT_EQ(test, strlen(vendor_name), DRM_CONNECTOR_HDMI_VENDOR_LEN);
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       vendor_name, "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -900,7 +900,7 @@ static void drm_test_connector_hdmi_init_vendor_length_too_long(struct kunit *te
 
 	KUNIT_ASSERT_GT(test, strlen(vendor_name), DRM_CONNECTOR_HDMI_VENDOR_LEN);
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       vendor_name, "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -920,7 +920,7 @@ static void drm_test_connector_hdmi_init_bpc_invalid(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -940,7 +940,7 @@ static void drm_test_connector_hdmi_init_bpc_null(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -965,7 +965,7 @@ static void drm_test_connector_hdmi_init_bpc_8(struct kunit *test)
 	uint64_t val;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1006,7 +1006,7 @@ static void drm_test_connector_hdmi_init_bpc_10(struct kunit *test)
 	uint64_t val;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1047,7 +1047,7 @@ static void drm_test_connector_hdmi_init_bpc_12(struct kunit *test)
 	uint64_t val;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1083,7 +1083,7 @@ static void drm_test_connector_hdmi_init_formats_empty(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1103,7 +1103,7 @@ static void drm_test_connector_hdmi_init_formats_no_rgb(struct kunit *test)
 	struct drm_connector_init_priv *priv = test->priv;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1161,7 +1161,7 @@ static void drm_test_connector_hdmi_init_formats_yuv420_allowed(struct kunit *te
 	params = test->param_value;
 	priv->connector.ycbcr_420_allowed = params->yuv420_allowed;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1182,7 +1182,7 @@ static void drm_test_connector_hdmi_init_type_valid(struct kunit *test)
 	unsigned int connector_type = *(unsigned int *)test->param_value;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1217,7 +1217,7 @@ static void drm_test_connector_hdmi_init_type_invalid(struct kunit *test)
 	unsigned int connector_type = *(unsigned int *)test->param_value;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, &priv->connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, &priv->connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
@@ -1494,7 +1494,7 @@ static void drm_test_drm_connector_attach_broadcast_rgb_property_hdmi_connector(
 	struct drm_property *prop;
 	int ret;
 
-	ret = drmm_connector_hdmi_init(&priv->drm, connector,
+	ret = drmm_connector_hdmi_ini2(&priv->drm, connector,
 				       "Vendor", "Product",
 				       &dummy_funcs,
 				       &dummy_hdmi_funcs,
diff --git a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
index 353a261d42dac5..b2e347863a776f 100644
--- a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
@@ -275,7 +275,7 @@ __connector_hdmi_init(struct kunit *test,
 	conn = &priv->connector;
 	conn->ycbcr_420_allowed = !!(formats & BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR420));
 
-	ret = drmm_connector_hdmi_init(drm, conn,
+	ret = drmm_connector_hdmi_ini2(drm, conn,
 				       "Vendor", "Product",
 				       &dummy_connector_funcs,
 				       hdmi_funcs,
@@ -2929,15 +2929,6 @@ static void drm_test_check_infoframes(struct kunit *test)
 
 	old_hdmi_update_failures = priv->hdmi_update_failures;
 
-	ret = drm_atomic_check_only(state);
-	if (ret == -EDEADLK) {
-		drm_atomic_commit_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_crtc_state;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
 		drm_atomic_commit_clear(state);
@@ -3034,15 +3025,6 @@ static void drm_test_check_reject_avi_infoframe(struct kunit *test)
 
 	old_hdmi_update_failures = priv->hdmi_update_failures;
 
-	ret = drm_atomic_check_only(state);
-	if (ret == -EDEADLK) {
-		drm_atomic_commit_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_crtc_state;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
 		drm_atomic_commit_clear(state);
@@ -3153,15 +3135,6 @@ static void drm_test_check_reject_hdr_infoframe_bpc_8(struct kunit *test)
 
 	old_hdmi_update_failures = priv->hdmi_update_failures;
 
-	ret = drm_atomic_check_only(state);
-	if (ret == -EDEADLK) {
-		drm_atomic_commit_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_conn_state;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
 		drm_atomic_commit_clear(state);
@@ -3276,15 +3249,6 @@ static void drm_test_check_reject_hdr_infoframe_bpc_10(struct kunit *test)
 
 	old_hdmi_update_failures = priv->hdmi_update_failures;
 
-	ret = drm_atomic_check_only(state);
-	if (ret == -EDEADLK) {
-		drm_atomic_commit_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_conn_state;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
 		drm_atomic_commit_clear(state);
@@ -3387,15 +3351,6 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 
 	old_hdmi_update_failures = priv->hdmi_update_failures;
 
-	ret = drm_atomic_check_only(state);
-	if (ret == -EDEADLK) {
-		drm_atomic_commit_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_crtc_state;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
 		drm_atomic_commit_clear(state);
@@ -3438,15 +3393,6 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 
 	old_hdmi_update_failures = priv->hdmi_update_failures;
 
-	ret = drm_atomic_check_only(state);
-	if (ret == -EDEADLK) {
-		drm_atomic_commit_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_crtc_state_2;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
 		drm_atomic_commit_clear(state);
diff --git a/drivers/gpu/drm/tests/drm_kunit_helpers.c b/drivers/gpu/drm/tests/drm_kunit_helpers.c
index 727766444d6384..63d02494650053 100644
--- a/drivers/gpu/drm/tests/drm_kunit_helpers.c
+++ b/drivers/gpu/drm/tests/drm_kunit_helpers.c
@@ -142,7 +142,7 @@ static const struct drm_plane_helper_funcs default_plane_helper_funcs = {
 static const struct drm_plane_funcs default_plane_funcs = {
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 };
 
 /**
@@ -217,7 +217,7 @@ static const struct drm_crtc_helper_funcs default_crtc_helper_funcs = {
 static const struct drm_crtc_funcs default_crtc_funcs = {
 	.atomic_destroy_state   = drm_atomic_helper_crtc_destroy_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
-	.reset                  = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 };
 
 /**
diff --git a/drivers/gpu/drm/tests/drm_panic_helper_test.c b/drivers/gpu/drm/tests/drm_panic_helper_test.c
new file mode 100644
index 00000000000000..d0366e10d8c18f
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_panic_helper_test.c
@@ -0,0 +1,248 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+/*
+ * Copyright (c) 2025 Red Hat.
+ * Author: Jocelyn Falempe <jfalempe@redhat.com>
+ *
+ * KUNIT tests for DRM panic helpers
+ */
+
+#include <linux/highmem.h>
+#include <linux/units.h>
+#include <linux/vmalloc.h>
+
+#include <kunit/test.h>
+
+#include <drm/drm_fourcc.h>
+#include <drm/drm_panic.h>
+#include <drm/drm_panic_helper.h>
+
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+static void drm_panic_check_color_byte(struct kunit *test, u8 b)
+{
+	KUNIT_EXPECT_TRUE(test, (b == 0 || b == 0xff));
+}
+
+struct drm_test_mode {
+	const int width;
+	const int height;
+	const u32 format;
+	int (*draw_screen)(struct drm_scanout_buffer *sb);
+	const char *fname;
+};
+
+/*
+ * Run all tests for the 3 panic screens: user, kmsg and qr_code
+ */
+#define DRM_TEST_MODE_LIST(func) \
+	DRM_PANIC_TEST_MODE(1024, 768, DRM_FORMAT_XRGB8888, func) \
+	DRM_PANIC_TEST_MODE(494, 494, DRM_FORMAT_XRGB8888, func) \
+	DRM_PANIC_TEST_MODE(1920, 1080, DRM_FORMAT_XRGB8888, func) \
+	DRM_PANIC_TEST_MODE(1024, 768, DRM_FORMAT_RGB565, func) \
+	DRM_PANIC_TEST_MODE(1024, 768, DRM_FORMAT_RGB888, func) \
+
+#define DRM_PANIC_TEST_MODE(w, h, f, name) { \
+	.width = w, \
+	.height = h, \
+	.format = f, \
+	.draw_screen = drm_panic_helper_draw_screen_ ## name ## _default, \
+	.fname = #name, \
+	}, \
+
+static int drm_panic_helper_draw_screen_user_default(struct drm_scanout_buffer *sb)
+{
+	return drm_panic_helper_draw_screen_user(sb, 0x00ffffff, 0x00000000);
+}
+
+static int drm_panic_helper_draw_screen_kmsg_default(struct drm_scanout_buffer *sb)
+{
+	return drm_panic_helper_draw_screen_kmsg(sb, 0x00ffffff, 0x00000000);
+}
+
+#if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
+static int drm_panic_helper_draw_screen_qr_code_default(struct drm_scanout_buffer *sb)
+{
+	return drm_panic_helper_draw_screen_qr_code(sb, 0x00ffffff, 0x00000000, 40);
+}
+#endif
+
+static const struct drm_test_mode drm_test_modes_cases[] = {
+	DRM_TEST_MODE_LIST(user)
+	DRM_TEST_MODE_LIST(kmsg)
+#if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
+	DRM_TEST_MODE_LIST(qr_code)
+#endif
+};
+
+#undef DRM_PANIC_TEST_MODE
+
+static int drm_test_panic_init(struct kunit *test)
+{
+	struct drm_scanout_buffer *priv;
+
+	priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	test->priv = priv;
+
+	drm_panic_helper_set_description("Kunit testing");
+
+	return 0;
+}
+
+static void drm_test_panic_exit(struct kunit *test)
+{
+	drm_panic_helper_clear_description();
+}
+
+/*
+ * Test drawing the panic screen, using a memory mapped framebuffer
+ * Set the whole buffer to 0xa5, and then check that all pixels have been
+ * written.
+ */
+static void drm_test_panic_screen_user_map(struct kunit *test)
+{
+	struct drm_scanout_buffer *sb = test->priv;
+	const struct drm_test_mode *params = test->param_value;
+	char *fb;
+	int fb_size;
+	int i, ret;
+
+	sb->format = drm_format_info(params->format);
+	fb_size = params->width * params->height * sb->format->cpp[0];
+
+	fb = vmalloc(fb_size);
+	KUNIT_ASSERT_NOT_NULL(test, fb);
+
+	memset(fb, 0xa5, fb_size);
+
+	iosys_map_set_vaddr(&sb->map[0], fb);
+	sb->width = params->width;
+	sb->height = params->height;
+	sb->pitch[0] = params->width * sb->format->cpp[0];
+
+	ret = params->draw_screen(sb);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	for (i = 0; i < fb_size; i++)
+		drm_panic_check_color_byte(test, fb[i]);
+
+	vfree(fb);
+}
+
+/*
+ * Test drawing the panic screen, using a list of pages framebuffer
+ * Set the whole buffer to 0xa5, and then check that all pixels have been
+ * written.
+ */
+static void drm_test_panic_screen_user_page(struct kunit *test)
+{
+	struct drm_scanout_buffer *sb = test->priv;
+	const struct drm_test_mode *params = test->param_value;
+	int fb_size, p, i, npages, ret;
+	struct page **pages;
+	u8 *vaddr;
+
+	sb->format = drm_format_info(params->format);
+	fb_size = params->width * params->height * sb->format->cpp[0];
+	npages = DIV_ROUND_UP(fb_size, PAGE_SIZE);
+
+	pages = kmalloc_objs(struct page *, npages);
+	KUNIT_ASSERT_NOT_NULL(test, pages);
+
+	for (p = 0; p < npages; p++) {
+		pages[p] = alloc_page(GFP_KERNEL);
+		if (!pages[p]) {
+			npages = p - 1;
+			KUNIT_FAIL(test, "Can't allocate page\n");
+			goto free_pages;
+		}
+		vaddr = kmap_local_page(pages[p]);
+		memset(vaddr, 0xa5, PAGE_SIZE);
+		kunmap_local(vaddr);
+	}
+	sb->pages = pages;
+	sb->width = params->width;
+	sb->height = params->height;
+	sb->pitch[0] = params->width * sb->format->cpp[0];
+
+	ret = params->draw_screen(sb);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	for (p = 0; p < npages; p++) {
+		int bytes_in_page = (p == npages - 1) ? fb_size - p * PAGE_SIZE : PAGE_SIZE;
+
+		vaddr = kmap_local_page(pages[p]);
+		for (i = 0; i < bytes_in_page; i++)
+			drm_panic_check_color_byte(test, vaddr[i]);
+
+		kunmap_local(vaddr);
+	}
+
+free_pages:
+	for (p = 0; p < npages; p++)
+		__free_page(pages[p]);
+	kfree(pages);
+}
+
+static void drm_test_panic_set_pixel(struct drm_scanout_buffer *sb,
+				     unsigned int x,
+				     unsigned int y,
+				     u32 color)
+{
+	struct kunit *test = (struct kunit *)sb->private;
+
+	KUNIT_ASSERT_TRUE(test, x < sb->width && y < sb->height);
+}
+
+/*
+ * Test drawing the panic screen, using the set_pixel callback
+ * Check that all calls to set_pixel() are within the framebuffer
+ */
+static void drm_test_panic_screen_user_set_pixel(struct kunit *test)
+{
+	struct drm_scanout_buffer *sb = test->priv;
+	const struct drm_test_mode *params = test->param_value;
+	int ret;
+
+	sb->format = drm_format_info(params->format);
+	sb->set_pixel = drm_test_panic_set_pixel;
+	sb->width = params->width;
+	sb->height = params->height;
+	sb->private = test;
+
+	ret = params->draw_screen(sb);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+}
+
+static void drm_test_panic_desc(const struct drm_test_mode *t, char *desc)
+{
+	sprintf(desc, "Panic screen %s, mode: %d x %d \t%p4cc",
+		t->fname, t->width, t->height, &t->format);
+}
+
+KUNIT_ARRAY_PARAM(drm_test_panic_screen_user_map, drm_test_modes_cases, drm_test_panic_desc);
+KUNIT_ARRAY_PARAM(drm_test_panic_screen_user_page, drm_test_modes_cases, drm_test_panic_desc);
+KUNIT_ARRAY_PARAM(drm_test_panic_screen_user_set_pixel, drm_test_modes_cases, drm_test_panic_desc);
+
+static struct kunit_case drm_panic_screen_user_test[] = {
+	KUNIT_CASE_PARAM(drm_test_panic_screen_user_map,
+			 drm_test_panic_screen_user_map_gen_params),
+	KUNIT_CASE_PARAM(drm_test_panic_screen_user_page,
+			 drm_test_panic_screen_user_page_gen_params),
+	KUNIT_CASE_PARAM(drm_test_panic_screen_user_set_pixel,
+			 drm_test_panic_screen_user_set_pixel_gen_params),
+	{ }
+};
+
+static struct kunit_suite drm_panic_helper_suite = {
+	.name = "drm_panic_helper",
+	.init = drm_test_panic_init,
+	.exit = drm_test_panic_exit,
+	.test_cases = drm_panic_screen_user_test,
+};
+
+kunit_test_suite(drm_panic_helper_suite);
+
+MODULE_DESCRIPTION("KUnit test suite for DRM panic handling");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_panic_test.c b/drivers/gpu/drm/tests/drm_panic_test.c
deleted file mode 100644
index ad2f3a2f93b6a5..00000000000000
--- a/drivers/gpu/drm/tests/drm_panic_test.c
+++ /dev/null
@@ -1,221 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0 or MIT
-/*
- * Copyright (c) 2025 Red Hat.
- * Author: Jocelyn Falempe <jfalempe@redhat.com>
- *
- * KUNIT tests for drm panic
- */
-
-#include <drm/drm_fourcc.h>
-#include <drm/drm_panic.h>
-
-#include <kunit/test.h>
-
-#include <linux/units.h>
-#include <linux/vmalloc.h>
-
-/* Check the framebuffer color only if the panic colors are the default */
-#if (CONFIG_DRM_PANIC_BACKGROUND_COLOR == 0 && \
-	CONFIG_DRM_PANIC_FOREGROUND_COLOR == 0xffffff)
-
-static void drm_panic_check_color_byte(struct kunit *test, u8 b)
-{
-	KUNIT_EXPECT_TRUE(test, (b == 0 || b == 0xff));
-}
-#else
-static void drm_panic_check_color_byte(struct kunit *test, u8 b) {}
-#endif
-
-struct drm_test_mode {
-	const int width;
-	const int height;
-	const u32 format;
-	void (*draw_screen)(struct drm_scanout_buffer *sb);
-	const char *fname;
-};
-
-/*
- * Run all tests for the 3 panic screens: user, kmsg and qr_code
- */
-#define DRM_TEST_MODE_LIST(func) \
-	DRM_PANIC_TEST_MODE(1024, 768, DRM_FORMAT_XRGB8888, func) \
-	DRM_PANIC_TEST_MODE(300, 200, DRM_FORMAT_XRGB8888, func) \
-	DRM_PANIC_TEST_MODE(1920, 1080, DRM_FORMAT_XRGB8888, func) \
-	DRM_PANIC_TEST_MODE(1024, 768, DRM_FORMAT_RGB565, func) \
-	DRM_PANIC_TEST_MODE(1024, 768, DRM_FORMAT_RGB888, func) \
-
-#define DRM_PANIC_TEST_MODE(w, h, f, name) { \
-	.width = w, \
-	.height = h, \
-	.format = f, \
-	.draw_screen = draw_panic_screen_##name, \
-	.fname = #name, \
-	}, \
-
-static const struct drm_test_mode drm_test_modes_cases[] = {
-	DRM_TEST_MODE_LIST(user)
-	DRM_TEST_MODE_LIST(kmsg)
-#if IS_ENABLED(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
-	DRM_TEST_MODE_LIST(qr_code)
-#endif
-};
-
-#undef DRM_PANIC_TEST_MODE
-
-static int drm_test_panic_init(struct kunit *test)
-{
-	struct drm_scanout_buffer *priv;
-
-	priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, priv);
-
-	test->priv = priv;
-
-	drm_panic_set_description("Kunit testing");
-
-	return 0;
-}
-
-/*
- * Test drawing the panic screen, using a memory mapped framebuffer
- * Set the whole buffer to 0xa5, and then check that all pixels have been
- * written.
- */
-static void drm_test_panic_screen_user_map(struct kunit *test)
-{
-	struct drm_scanout_buffer *sb = test->priv;
-	const struct drm_test_mode *params = test->param_value;
-	char *fb;
-	int fb_size;
-	int i;
-
-	sb->format = drm_format_info(params->format);
-	fb_size = params->width * params->height * sb->format->cpp[0];
-
-	fb = vmalloc(fb_size);
-	KUNIT_ASSERT_NOT_NULL(test, fb);
-
-	memset(fb, 0xa5, fb_size);
-
-	iosys_map_set_vaddr(&sb->map[0], fb);
-	sb->width = params->width;
-	sb->height = params->height;
-	sb->pitch[0] = params->width * sb->format->cpp[0];
-
-	params->draw_screen(sb);
-
-	for (i = 0; i < fb_size; i++)
-		drm_panic_check_color_byte(test, fb[i]);
-
-	vfree(fb);
-}
-
-/*
- * Test drawing the panic screen, using a list of pages framebuffer
- * Set the whole buffer to 0xa5, and then check that all pixels have been
- * written.
- */
-static void drm_test_panic_screen_user_page(struct kunit *test)
-{
-	struct drm_scanout_buffer *sb = test->priv;
-	const struct drm_test_mode *params = test->param_value;
-	int fb_size, p, i, npages;
-	struct page **pages;
-	u8 *vaddr;
-
-	sb->format = drm_format_info(params->format);
-	fb_size = params->width * params->height * sb->format->cpp[0];
-	npages = DIV_ROUND_UP(fb_size, PAGE_SIZE);
-
-	pages = kmalloc_objs(struct page *, npages);
-	KUNIT_ASSERT_NOT_NULL(test, pages);
-
-	for (p = 0; p < npages; p++) {
-		pages[p] = alloc_page(GFP_KERNEL);
-		if (!pages[p]) {
-			npages = p - 1;
-			KUNIT_FAIL(test, "Can't allocate page\n");
-			goto free_pages;
-		}
-		vaddr = kmap_local_page(pages[p]);
-		memset(vaddr, 0xa5, PAGE_SIZE);
-		kunmap_local(vaddr);
-	}
-	sb->pages = pages;
-	sb->width = params->width;
-	sb->height = params->height;
-	sb->pitch[0] = params->width * sb->format->cpp[0];
-
-	params->draw_screen(sb);
-
-	for (p = 0; p < npages; p++) {
-		int bytes_in_page = (p == npages - 1) ? fb_size - p * PAGE_SIZE : PAGE_SIZE;
-
-		vaddr = kmap_local_page(pages[p]);
-		for (i = 0; i < bytes_in_page; i++)
-			drm_panic_check_color_byte(test, vaddr[i]);
-
-		kunmap_local(vaddr);
-	}
-
-free_pages:
-	for (p = 0; p < npages; p++)
-		__free_page(pages[p]);
-	kfree(pages);
-}
-
-static void drm_test_panic_set_pixel(struct drm_scanout_buffer *sb,
-				     unsigned int x,
-				     unsigned int y,
-				     u32 color)
-{
-	struct kunit *test = (struct kunit *)sb->private;
-
-	KUNIT_ASSERT_TRUE(test, x < sb->width && y < sb->height);
-}
-
-/*
- * Test drawing the panic screen, using the set_pixel callback
- * Check that all calls to set_pixel() are within the framebuffer
- */
-static void drm_test_panic_screen_user_set_pixel(struct kunit *test)
-{
-	struct drm_scanout_buffer *sb = test->priv;
-	const struct drm_test_mode *params = test->param_value;
-
-	sb->format = drm_format_info(params->format);
-	sb->set_pixel = drm_test_panic_set_pixel;
-	sb->width = params->width;
-	sb->height = params->height;
-	sb->private = test;
-
-	params->draw_screen(sb);
-}
-
-static void drm_test_panic_desc(const struct drm_test_mode *t, char *desc)
-{
-	sprintf(desc, "Panic screen %s, mode: %d x %d \t%p4cc",
-		t->fname, t->width, t->height, &t->format);
-}
-
-KUNIT_ARRAY_PARAM(drm_test_panic_screen_user_map, drm_test_modes_cases, drm_test_panic_desc);
-KUNIT_ARRAY_PARAM(drm_test_panic_screen_user_page, drm_test_modes_cases, drm_test_panic_desc);
-KUNIT_ARRAY_PARAM(drm_test_panic_screen_user_set_pixel, drm_test_modes_cases, drm_test_panic_desc);
-
-static struct kunit_case drm_panic_screen_user_test[] = {
-	KUNIT_CASE_PARAM(drm_test_panic_screen_user_map,
-			 drm_test_panic_screen_user_map_gen_params),
-	KUNIT_CASE_PARAM(drm_test_panic_screen_user_page,
-			 drm_test_panic_screen_user_page_gen_params),
-	KUNIT_CASE_PARAM(drm_test_panic_screen_user_set_pixel,
-			 drm_test_panic_screen_user_set_pixel_gen_params),
-	{ }
-};
-
-static struct kunit_suite drm_panic_suite = {
-	.name = "drm_panic",
-	.init = drm_test_panic_init,
-	.test_cases = drm_panic_screen_user_test,
-};
-
-kunit_test_suite(drm_panic_suite);
diff --git a/drivers/gpu/drm/tidss/Kconfig b/drivers/gpu/drm/tidss/Kconfig
index 31ad582b760238..52a64fa1ad5d0c 100644
--- a/drivers/gpu/drm/tidss/Kconfig
+++ b/drivers/gpu/drm/tidss/Kconfig
@@ -7,6 +7,7 @@ config DRM_TIDSS
 	select DRM_DISPLAY_HELPER
 	select DRM_BRIDGE_CONNECTOR
 	select DRM_GEM_DMA_HELPER
+	select DRM_PANEL
 	help
 	  The TI Keystone family SoCs introduced a new generation of
 	  Display SubSystem. There is currently three Keystone family
diff --git a/drivers/gpu/drm/tidss/tidss_dispc.c b/drivers/gpu/drm/tidss/tidss_dispc.c
index 58d5eb033bdbbf..2a0ec11a6c902f 100644
--- a/drivers/gpu/drm/tidss/tidss_dispc.c
+++ b/drivers/gpu/drm/tidss/tidss_dispc.c
@@ -2157,47 +2157,15 @@ int dispc_plane_check(struct dispc_device *dispc, u32 hw_plane,
 	return 0;
 }
 
-static
-dma_addr_t dispc_plane_state_dma_addr(const struct drm_plane_state *state)
-{
-	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_dma_object *gem;
-	u32 x = state->src_x >> 16;
-	u32 y = state->src_y >> 16;
-
-	gem = drm_fb_dma_get_gem_obj(state->fb, 0);
-
-	return gem->dma_addr + fb->offsets[0] + x * fb->format->cpp[0] +
-		y * fb->pitches[0];
-}
-
-static
-dma_addr_t dispc_plane_state_p_uv_addr(const struct drm_plane_state *state)
-{
-	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_dma_object *gem;
-	u32 x = state->src_x >> 16;
-	u32 y = state->src_y >> 16;
-
-	if (WARN_ON(state->fb->format->num_planes != 2))
-		return 0;
-
-	gem = drm_fb_dma_get_gem_obj(fb, 1);
-
-	return gem->dma_addr + fb->offsets[1] +
-		(x * fb->format->cpp[1] / fb->format->hsub) +
-		(y * fb->pitches[1] / fb->format->vsub);
-}
-
 void dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
-		       const struct drm_plane_state *state,
+		       struct drm_plane_state *state,
 		       u32 hw_videoport)
 {
 	bool lite = dispc->feat->vid_info[hw_plane].is_lite;
 	u32 fourcc = state->fb->format->format;
 	u16 cpp = state->fb->format->cpp[0];
 	u32 fb_width = state->fb->pitches[0] / cpp;
-	dma_addr_t dma_addr = dispc_plane_state_dma_addr(state);
+	dma_addr_t dma_addr = drm_fb_dma_get_gem_addr(state->fb, state, 0);
 	struct dispc_scaling_params scale;
 
 	dispc_vid_calc_scaling(dispc, state, &scale, lite);
@@ -2229,7 +2197,7 @@ void dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
 	if (state->fb->format->num_planes == 2) {
 		u16 cpp_uv = state->fb->format->cpp[1];
 		u32 fb_width_uv = state->fb->pitches[1] / cpp_uv;
-		dma_addr_t p_uv_addr = dispc_plane_state_p_uv_addr(state);
+		dma_addr_t p_uv_addr = drm_fb_dma_get_gem_addr(state->fb, state, 1);
 
 		dispc_vid_write(dispc, hw_plane,
 				DISPC_VID_BA_UV_0, p_uv_addr & 0xffffffff);
diff --git a/drivers/gpu/drm/tidss/tidss_dispc.h b/drivers/gpu/drm/tidss/tidss_dispc.h
index 739d211d0018d1..2183254f1ab714 100644
--- a/drivers/gpu/drm/tidss/tidss_dispc.h
+++ b/drivers/gpu/drm/tidss/tidss_dispc.h
@@ -140,7 +140,7 @@ int dispc_plane_check(struct dispc_device *dispc, u32 hw_plane,
 		      const struct drm_plane_state *state,
 		      u32 hw_videoport);
 void dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
-		       const struct drm_plane_state *state,
+		       struct drm_plane_state *state,
 		       u32 hw_videoport);
 void dispc_plane_enable(struct dispc_device *dispc, u32 hw_plane, bool enable);
 const u32 *dispc_plane_formats(struct dispc_device *dispc, unsigned int *len);
diff --git a/drivers/gpu/drm/tidss/tidss_encoder.c b/drivers/gpu/drm/tidss/tidss_encoder.c
index 698f8d964ca00d..10dbcc6cdf6a89 100644
--- a/drivers/gpu/drm/tidss/tidss_encoder.c
+++ b/drivers/gpu/drm/tidss/tidss_encoder.c
@@ -9,11 +9,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_of.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "tidss_crtc.h"
 #include "tidss_drv.h"
@@ -81,6 +81,10 @@ static const struct drm_bridge_funcs tidss_bridge_funcs = {
 	.atomic_destroy_state		= drm_atomic_helper_bridge_destroy_state,
 };
 
+static const struct drm_encoder_funcs tidss_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 int tidss_encoder_create(struct tidss_device *tidss,
 			 struct drm_bridge *next_bridge,
 			 u32 encoder_type, u32 possible_crtcs)
@@ -95,8 +99,8 @@ int tidss_encoder_create(struct tidss_device *tidss,
 	if (IS_ERR(t_enc))
 		return PTR_ERR(t_enc);
 
-	ret = drm_simple_encoder_init(&tidss->ddev, &t_enc->encoder,
-				      encoder_type);
+	ret = drm_encoder_init(&tidss->ddev, &t_enc->encoder,
+			       &tidss_encoder_funcs, encoder_type, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/tidss/tidss_plane.c b/drivers/gpu/drm/tidss/tidss_plane.c
index 6d82976c2db1b0..b0d5d6959b8856 100644
--- a/drivers/gpu/drm/tidss/tidss_plane.c
+++ b/drivers/gpu/drm/tidss/tidss_plane.c
@@ -12,6 +12,7 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_panic_helper.h>
 
 #include "tidss_crtc.h"
 #include "tidss_dispc.h"
@@ -182,6 +183,7 @@ static const struct drm_plane_funcs tidss_plane_funcs = {
 	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 struct tidss_plane *tidss_plane_create(struct tidss_device *tidss,
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index e07ecb36a27a83..f619b7a6e70d97 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -685,14 +685,12 @@ static void tilcdc_crtc_disable_vblank(struct drm_crtc *crtc)
 	spin_unlock_irqrestore(&tilcdc_crtc->irq_lock, flags);
 }
 
-static void tilcdc_crtc_reset(struct drm_crtc *crtc)
+static void tilcdc_crtc_hw_reset(struct drm_crtc *crtc)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	int ret;
 
-	drm_atomic_helper_crtc_reset(crtc);
-
 	/* Turn the raster off if it for some reason is on. */
 	pm_runtime_get_sync(dev->dev);
 	if (tilcdc_read(dev, LCDC_RASTER_CTRL_REG) & LCDC_RASTER_ENABLE) {
@@ -716,7 +714,7 @@ static void tilcdc_crtc_reset(struct drm_crtc *crtc)
 static const struct drm_crtc_funcs tilcdc_crtc_funcs = {
 	.set_config     = drm_atomic_helper_set_config,
 	.page_flip      = drm_atomic_helper_page_flip,
-	.reset		= tilcdc_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank	= tilcdc_crtc_enable_vblank,
@@ -820,6 +818,7 @@ static const struct drm_crtc_helper_funcs tilcdc_crtc_helper_funcs = {
 	.atomic_enable	= tilcdc_crtc_atomic_enable,
 	.atomic_disable	= tilcdc_crtc_atomic_disable,
 	.atomic_flush	= tilcdc_crtc_atomic_flush,
+	.hw_reset	= tilcdc_crtc_hw_reset,
 };
 
 void tilcdc_crtc_update_clk(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_plane.c b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
index d2678af5dc22c4..eb0a5562421fd5 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_plane.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
@@ -14,7 +14,7 @@
 static const struct drm_plane_funcs tilcdc_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.reset		= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/tiny/appletbdrm.c b/drivers/gpu/drm/tiny/appletbdrm.c
index ef71b995796194..4297d9c1125e49 100644
--- a/drivers/gpu/drm/tiny/appletbdrm.c
+++ b/drivers/gpu/drm/tiny/appletbdrm.c
@@ -505,17 +505,17 @@ static void appletbdrm_primary_plane_helper_atomic_disable(struct drm_plane *pla
 	drm_dev_exit(idx);
 }
 
-static void appletbdrm_primary_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *appletbdrm_primary_plane_create_state(struct drm_plane *plane)
 {
 	struct appletbdrm_plane_state *appletbdrm_state;
 
-	WARN_ON(plane->state);
-
 	appletbdrm_state = kzalloc_obj(*appletbdrm_state);
 	if (!appletbdrm_state)
-		return;
+		return ERR_PTR(-ENOMEM);
+
+	__drm_gem_shadow_plane_state_init(plane, &appletbdrm_state->base);
 
-	__drm_gem_reset_shadow_plane(plane, &appletbdrm_state->base);
+	return &(&appletbdrm_state->base)->base;
 }
 
 static struct drm_plane_state *appletbdrm_primary_plane_duplicate_state(struct drm_plane *plane)
@@ -568,7 +568,7 @@ static const struct drm_plane_helper_funcs appletbdrm_primary_plane_helper_funcs
 static const struct drm_plane_funcs appletbdrm_primary_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = appletbdrm_primary_plane_reset,
+	.atomic_create_state = appletbdrm_primary_plane_create_state,
 	.atomic_duplicate_state = appletbdrm_primary_plane_duplicate_state,
 	.atomic_destroy_state = appletbdrm_primary_plane_destroy_state,
 	.destroy = drm_plane_cleanup,
@@ -605,7 +605,7 @@ static const struct drm_crtc_helper_funcs appletbdrm_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs appletbdrm_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/tiny/arcpgu.c b/drivers/gpu/drm/tiny/arcpgu.c
index c93d61ac0bb755..cdbf28db8fa1db 100644
--- a/drivers/gpu/drm/tiny/arcpgu.c
+++ b/drivers/gpu/drm/tiny/arcpgu.c
@@ -8,6 +8,7 @@
 #include <linux/clk.h>
 
 #include <drm/clients/drm_client_setup.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_debugfs.h>
 #include <drm/drm_device.h>
@@ -17,12 +18,13 @@
 #include <drm/drm_fbdev_dma.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
+#include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_of.h>
+#include <drm/drm_plane.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <linux/dma-mapping.h>
 #include <linux/module.h>
 #include <linux/of_reserved_mem.h>
@@ -52,13 +54,16 @@ struct arcpgu_drm_private {
 	struct drm_device	drm;
 	void __iomem		*regs;
 	struct clk		*clk;
-	struct drm_simple_display_pipe pipe;
+	struct drm_plane	plane;
+	struct drm_crtc		crtc;
+	struct drm_encoder	encoder;
 	struct drm_connector	sim_conn;
 };
 
-#define dev_to_arcpgu(x) container_of(x, struct arcpgu_drm_private, drm)
-
-#define pipe_to_arcpgu_priv(x) container_of(x, struct arcpgu_drm_private, pipe)
+static inline struct arcpgu_drm_private *dev_to_arcpgu(struct drm_device *drm)
+{
+	return container_of(drm, struct arcpgu_drm_private, drm);
+}
 
 static inline void arc_pgu_write(struct arcpgu_drm_private *arcpgu,
 				 unsigned int reg, u32 value)
@@ -115,9 +120,10 @@ static const u32 arc_pgu_supported_formats[] = {
 	DRM_FORMAT_ARGB8888,
 };
 
-static void arc_pgu_set_pxl_fmt(struct arcpgu_drm_private *arcpgu)
+static void arc_pgu_set_pxl_fmt(struct arcpgu_drm_private *arcpgu,
+				const struct drm_plane_state *plane_state)
 {
-	const struct drm_framebuffer *fb = arcpgu->pipe.plane.state->fb;
+	const struct drm_framebuffer *fb = plane_state->fb;
 	uint32_t pixel_format = fb->format->format;
 	u32 format = DRM_FORMAT_INVALID;
 	int i;
@@ -139,10 +145,10 @@ static void arc_pgu_set_pxl_fmt(struct arcpgu_drm_private *arcpgu)
 	arc_pgu_write(arcpgu, ARCPGU_REG_CTRL, reg_ctrl);
 }
 
-static enum drm_mode_status arc_pgu_mode_valid(struct drm_simple_display_pipe *pipe,
-					       const struct drm_display_mode *mode)
+static enum drm_mode_status arcpgu_crtc_helper_mode_valid(struct drm_crtc *crtc,
+							  const struct drm_display_mode *mode)
 {
-	struct arcpgu_drm_private *arcpgu = pipe_to_arcpgu_priv(pipe);
+	struct arcpgu_drm_private *arcpgu = dev_to_arcpgu(crtc->dev);
 	long rate, clk_rate = mode->clock * 1000;
 	long diff = clk_rate / 200; /* +-0.5% allowed by HDMI spec */
 
@@ -153,9 +159,11 @@ static enum drm_mode_status arc_pgu_mode_valid(struct drm_simple_display_pipe *p
 	return MODE_NOCLOCK;
 }
 
-static void arc_pgu_mode_set(struct arcpgu_drm_private *arcpgu)
+static void arc_pgu_mode_set(struct arcpgu_drm_private *arcpgu,
+			     struct drm_crtc_state *crtc_state,
+			     struct drm_plane_state *plane_state)
 {
-	struct drm_display_mode *m = &arcpgu->pipe.crtc.state->adjusted_mode;
+	struct drm_display_mode *m = &crtc_state->adjusted_mode;
 	u32 val;
 
 	arc_pgu_write(arcpgu, ARCPGU_REG_FMT,
@@ -189,18 +197,19 @@ static void arc_pgu_mode_set(struct arcpgu_drm_private *arcpgu)
 	arc_pgu_write(arcpgu, ARCPGU_REG_STRIDE, 0);
 	arc_pgu_write(arcpgu, ARCPGU_REG_START_SET, 1);
 
-	arc_pgu_set_pxl_fmt(arcpgu);
+	arc_pgu_set_pxl_fmt(arcpgu, plane_state);
 
 	clk_set_rate(arcpgu->clk, m->crtc_clock * 1000);
 }
 
-static void arc_pgu_enable(struct drm_simple_display_pipe *pipe,
-			   struct drm_crtc_state *crtc_state,
-			   struct drm_plane_state *plane_state)
+static void arcpgu_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					     struct drm_atomic_commit *commit)
 {
-	struct arcpgu_drm_private *arcpgu = pipe_to_arcpgu_priv(pipe);
+	struct arcpgu_drm_private *arcpgu = dev_to_arcpgu(crtc->dev);
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, &arcpgu->plane);
+	struct drm_crtc_state *cstate = drm_atomic_get_new_crtc_state(commit, crtc);
 
-	arc_pgu_mode_set(arcpgu);
+	arc_pgu_mode_set(arcpgu, cstate, pstate);
 
 	clk_prepare_enable(arcpgu->clk);
 	arc_pgu_write(arcpgu, ARCPGU_REG_CTRL,
@@ -208,9 +217,10 @@ static void arc_pgu_enable(struct drm_simple_display_pipe *pipe,
 		      ARCPGU_CTRL_ENABLE_MASK);
 }
 
-static void arc_pgu_disable(struct drm_simple_display_pipe *pipe)
+static void arcpgu_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					      struct drm_atomic_commit *commit)
 {
-	struct arcpgu_drm_private *arcpgu = pipe_to_arcpgu_priv(pipe);
+	struct arcpgu_drm_private *arcpgu = dev_to_arcpgu(crtc->dev);
 
 	clk_disable_unprepare(arcpgu->clk);
 	arc_pgu_write(arcpgu, ARCPGU_REG_CTRL,
@@ -218,27 +228,21 @@ static void arc_pgu_disable(struct drm_simple_display_pipe *pipe)
 			      ~ARCPGU_CTRL_ENABLE_MASK);
 }
 
-static void arc_pgu_update(struct drm_simple_display_pipe *pipe,
-			   struct drm_plane_state *state)
+static void arcpgu_plane_helper_atomic_update(struct drm_plane *plane,
+					      struct drm_atomic_commit *commit)
 {
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
 	struct arcpgu_drm_private *arcpgu;
 	struct drm_gem_dma_object *gem;
 
-	if (!pipe->plane.state->fb)
+	if (!pstate->fb)
 		return;
 
-	arcpgu = pipe_to_arcpgu_priv(pipe);
-	gem = drm_fb_dma_get_gem_obj(pipe->plane.state->fb, 0);
+	arcpgu = dev_to_arcpgu(plane->dev);
+	gem = drm_fb_dma_get_gem_obj(pstate->fb, 0);
 	arc_pgu_write(arcpgu, ARCPGU_REG_BUF0_ADDR, gem->dma_addr);
 }
 
-static const struct drm_simple_display_pipe_funcs arc_pgu_pipe_funcs = {
-	.update = arc_pgu_update,
-	.mode_valid = arc_pgu_mode_valid,
-	.enable	= arc_pgu_enable,
-	.disable = arc_pgu_disable,
-};
-
 static const struct drm_mode_config_funcs arcpgu_drm_modecfg_funcs = {
 	.fb_create  = drm_gem_fb_create,
 	.atomic_check = drm_atomic_helper_check,
@@ -247,6 +251,74 @@ static const struct drm_mode_config_funcs arcpgu_drm_modecfg_funcs = {
 
 DEFINE_DRM_GEM_DMA_FOPS(arcpgu_drm_ops);
 
+static int arcpgu_plane_helper_atomic_check(struct drm_plane *plane,
+					    struct drm_atomic_commit *commit)
+{
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_crtc_state *crtc_state = NULL;
+
+	if (plane_state->crtc) {
+		crtc_state = drm_atomic_get_crtc_state(commit, plane_state->crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+	}
+
+	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, false);
+}
+
+static const struct drm_plane_helper_funcs arcpgu_plane_helper_funcs = {
+	.prepare_fb	= drm_gem_plane_helper_prepare_fb,
+	.atomic_check	= arcpgu_plane_helper_atomic_check,
+	.atomic_update	= arcpgu_plane_helper_atomic_update,
+};
+
+static const struct drm_plane_funcs arcpgu_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.destroy		= drm_plane_cleanup,
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+};
+
+static int arcpgu_crtc_helper_atomic_check(struct drm_crtc *crtc,
+					   struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static const struct drm_crtc_helper_funcs arcpgu_crtc_helper_funcs = {
+	.mode_valid	= arcpgu_crtc_helper_mode_valid,
+	.atomic_check	= arcpgu_crtc_helper_atomic_check,
+	.atomic_enable	= arcpgu_crtc_helper_atomic_enable,
+	.atomic_disable	= arcpgu_crtc_helper_atomic_disable,
+};
+
+static const struct drm_crtc_funcs arcpgu_crtc_funcs = {
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.destroy		= drm_crtc_cleanup,
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_encoder_funcs arcpgu_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static int arcpgu_load(struct arcpgu_drm_private *arcpgu)
 {
 	struct platform_device *pdev = to_platform_device(arcpgu->drm.dev);
@@ -254,6 +326,9 @@ static int arcpgu_load(struct arcpgu_drm_private *arcpgu)
 	struct device_node *endpoint_node = NULL;
 	struct drm_connector *connector = NULL;
 	struct drm_device *drm = &arcpgu->drm;
+	struct drm_plane *plane;
+	struct drm_encoder *encoder;
+	struct drm_crtc *crtc;
 	int ret;
 
 	arcpgu->clk = devm_clk_get(drm->dev, "pxlclk");
@@ -285,6 +360,30 @@ static int arcpgu_load(struct arcpgu_drm_private *arcpgu)
 	if (dma_set_mask_and_coherent(drm->dev, DMA_BIT_MASK(32)))
 		return -ENODEV;
 
+	plane = &arcpgu->plane;
+	ret = drm_universal_plane_init(drm, plane, 0,
+				       &arcpgu_plane_funcs,
+				       arc_pgu_supported_formats,
+				       ARRAY_SIZE(arc_pgu_supported_formats),
+				       NULL,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+	drm_plane_helper_add(plane, &arcpgu_plane_helper_funcs);
+
+	crtc = &arcpgu->crtc;
+	ret = drm_crtc_init_with_planes(drm, crtc, plane, NULL,
+					&arcpgu_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+	drm_crtc_helper_add(crtc, &arcpgu_crtc_helper_funcs);
+
+	encoder = &arcpgu->encoder;
+	ret = drm_encoder_init(drm, encoder, &arcpgu_encoder_funcs, DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
 	/*
 	 * There is only one output port inside each device. It is linked with
 	 * encoder endpoint.
@@ -293,29 +392,26 @@ static int arcpgu_load(struct arcpgu_drm_private *arcpgu)
 	if (endpoint_node) {
 		encoder_node = of_graph_get_remote_port_parent(endpoint_node);
 		of_node_put(endpoint_node);
-	} else {
-		connector = &arcpgu->sim_conn;
-		dev_info(drm->dev, "no encoder found. Assumed virtual LCD on simulation platform\n");
-		ret = arcpgu_drm_sim_init(drm, connector);
-		if (ret < 0)
-			return ret;
-	}
+		if (!encoder_node)
+			return -ENODEV;
 
-	ret = drm_simple_display_pipe_init(drm, &arcpgu->pipe, &arc_pgu_pipe_funcs,
-					   arc_pgu_supported_formats,
-					   ARRAY_SIZE(arc_pgu_supported_formats),
-					   NULL, connector);
-	if (ret)
-		return ret;
-
-	if (encoder_node) {
 		/* Locate drm bridge from the hdmi encoder DT node */
 		struct drm_bridge *bridge __free(drm_bridge_put) =
 			of_drm_find_and_get_bridge(encoder_node);
 		if (!bridge)
 			return -EPROBE_DEFER;
 
-		ret = drm_simple_display_pipe_attach_bridge(&arcpgu->pipe, bridge);
+		ret = drm_bridge_attach(encoder, bridge, NULL, 0);
+		if (ret)
+			return ret;
+	} else {
+		connector = &arcpgu->sim_conn;
+		dev_info(drm->dev, "no encoder found. Assumed virtual LCD on simulation platform\n");
+		ret = arcpgu_drm_sim_init(drm, connector);
+		if (ret < 0)
+			return ret;
+
+		ret = drm_connector_attach_encoder(connector, encoder);
 		if (ret)
 			return ret;
 	}
@@ -342,7 +438,14 @@ static int arcpgu_show_pxlclock(struct seq_file *m, void *arg)
 	struct drm_device *drm = node->minor->dev;
 	struct arcpgu_drm_private *arcpgu = dev_to_arcpgu(drm);
 	unsigned long clkrate = clk_get_rate(arcpgu->clk);
-	unsigned long mode_clock = arcpgu->pipe.crtc.mode.crtc_clock * 1000;
+	unsigned long mode_clock;
+	int ret;
+
+	ret = drm_modeset_lock_single_interruptible(&arcpgu->crtc.mutex);
+	if (ret)
+		return ret;
+	mode_clock = arcpgu->crtc.state->mode.crtc_clock * 1000;
+	drm_modeset_unlock(&arcpgu->crtc.mutex);
 
 	seq_printf(m, "hw  : %lu\n", clkrate);
 	seq_printf(m, "mode: %lu\n", mode_clock);
diff --git a/drivers/gpu/drm/tiny/bochs.c b/drivers/gpu/drm/tiny/bochs.c
index e2d957e51505cf..8192decddbc1eb 100644
--- a/drivers/gpu/drm/tiny/bochs.c
+++ b/drivers/gpu/drm/tiny/bochs.c
@@ -20,6 +20,7 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_module.h>
 #include <drm/drm_panic.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -501,7 +502,8 @@ static const struct drm_plane_funcs bochs_primary_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = drm_plane_cleanup,
-	DRM_GEM_SHADOW_PLANE_FUNCS
+	DRM_GEM_SHADOW_PLANE_FUNCS,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static void bochs_crtc_helper_mode_set_nofb(struct drm_crtc *crtc)
@@ -550,7 +552,7 @@ static const struct drm_crtc_helper_funcs bochs_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs bochs_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/tiny/cirrus-qemu.c b/drivers/gpu/drm/tiny/cirrus-qemu.c
index 3bf23fcf657496..63c242e2566253 100644
--- a/drivers/gpu/drm/tiny/cirrus-qemu.c
+++ b/drivers/gpu/drm/tiny/cirrus-qemu.c
@@ -419,7 +419,7 @@ static const struct drm_crtc_helper_funcs cirrus_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs cirrus_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/tiny/gm12u320.c b/drivers/gpu/drm/tiny/gm12u320.c
index 4ad074337af07a..ddc19cd3e03c36 100644
--- a/drivers/gpu/drm/tiny/gm12u320.c
+++ b/drivers/gpu/drm/tiny/gm12u320.c
@@ -8,6 +8,7 @@
 #include <linux/usb.h>
 
 #include <drm/clients/drm_client_setup.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_connector.h>
@@ -27,7 +28,6 @@
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 static bool eco_mode;
 module_param(eco_mode, bool, 0644);
@@ -87,7 +87,9 @@ MODULE_PARM_DESC(eco_mode, "Turn on Eco mode (less bright, more silent)");
 
 struct gm12u320_device {
 	struct drm_device	         dev;
-	struct drm_simple_display_pipe   pipe;
+	struct drm_plane	         plane;
+	struct drm_crtc		         crtc;
+	struct drm_encoder	         encoder;
 	struct drm_connector	         conn;
 	unsigned char                   *cmd_buf;
 	unsigned char                   *data_buf[GM12U320_BLOCK_COUNT];
@@ -102,7 +104,10 @@ struct gm12u320_device {
 	} fb_update;
 };
 
-#define to_gm12u320(__dev) container_of(__dev, struct gm12u320_device, dev)
+static inline struct gm12u320_device *to_gm12u320(struct drm_device *__dev)
+{
+	return container_of(__dev, struct gm12u320_device, dev);
+}
 
 static const char cmd_data[CMD_SIZE] = {
 	0x55, 0x53, 0x42, 0x43, 0x00, 0x00, 0x00, 0x00,
@@ -555,43 +560,106 @@ static int gm12u320_conn_init(struct gm12u320_device *gm12u320)
 }
 
 /* ------------------------------------------------------------------ */
-/* gm12u320 (simple) display pipe				      */
+/* gm12u320 display pipe					      */
 
-static void gm12u320_pipe_enable(struct drm_simple_display_pipe *pipe,
-				 struct drm_crtc_state *crtc_state,
-				 struct drm_plane_state *plane_state)
+static void gm12u320_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					       struct drm_atomic_commit *commit)
 {
 	struct drm_rect rect = { 0, 0, GM12U320_USER_WIDTH, GM12U320_HEIGHT };
-	struct gm12u320_device *gm12u320 = to_gm12u320(pipe->crtc.dev);
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct gm12u320_device *gm12u320 = to_gm12u320(crtc->dev);
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, &gm12u320->plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(pstate);
 
 	gm12u320->fb_update.draw_status_timeout = FIRST_FRAME_TIMEOUT;
-	gm12u320_fb_mark_dirty(plane_state->fb, &shadow_plane_state->data[0], &rect);
+	gm12u320_fb_mark_dirty(pstate->fb, &shadow_plane_state->data[0], &rect);
 }
 
-static void gm12u320_pipe_disable(struct drm_simple_display_pipe *pipe)
+static void gm12u320_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+						struct drm_atomic_commit *commit)
 {
-	struct gm12u320_device *gm12u320 = to_gm12u320(pipe->crtc.dev);
+	struct gm12u320_device *gm12u320 = to_gm12u320(crtc->dev);
 
 	gm12u320_stop_fb_update(gm12u320);
 }
 
-static void gm12u320_pipe_update(struct drm_simple_display_pipe *pipe,
-				 struct drm_plane_state *old_state)
+static void gm12u320_plane_helper_atomic_update(struct drm_plane *plane,
+						struct drm_atomic_commit *commit)
 {
-	struct drm_plane_state *state = pipe->plane.state;
+	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(commit, plane);
+	struct drm_plane_state *state = drm_atomic_get_new_plane_state(commit, plane);
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(state);
 	struct drm_rect rect;
 
+	if (!state->fb)
+		return;
+
 	if (drm_atomic_helper_damage_merged(old_state, state, &rect))
 		gm12u320_fb_mark_dirty(state->fb, &shadow_plane_state->data[0], &rect);
 }
 
-static const struct drm_simple_display_pipe_funcs gm12u320_pipe_funcs = {
-	.enable	    = gm12u320_pipe_enable,
-	.disable    = gm12u320_pipe_disable,
-	.update	    = gm12u320_pipe_update,
-	DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS,
+static const struct drm_plane_funcs gm12u320_plane_funcs = {
+	.update_plane	= drm_atomic_helper_update_plane,
+	.disable_plane	= drm_atomic_helper_disable_plane,
+	.destroy	= drm_plane_cleanup,
+	DRM_GEM_SHADOW_PLANE_FUNCS,
+};
+
+static int gm12u320_plane_helper_atomic_check(struct drm_plane *plane,
+					      struct drm_atomic_commit *commit)
+{
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_crtc_state *crtc_state = NULL;
+
+	if (plane_state->crtc) {
+		crtc_state = drm_atomic_get_crtc_state(commit, plane_state->crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+	}
+
+	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, false);
+}
+
+static const struct drm_plane_helper_funcs gm12u320_plane_helper_funcs = {
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+	.atomic_check	= gm12u320_plane_helper_atomic_check,
+	.atomic_update	= gm12u320_plane_helper_atomic_update,
+};
+
+static int gm12u320_crtc_helper_atomic_check(struct drm_crtc *crtc,
+					     struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static const struct drm_crtc_helper_funcs gm12u320_crtc_helper_funcs = {
+	.atomic_check	= gm12u320_crtc_helper_atomic_check,
+	.atomic_enable	= gm12u320_crtc_helper_atomic_enable,
+	.atomic_disable	= gm12u320_crtc_helper_atomic_disable,
+};
+
+static const struct drm_crtc_funcs gm12u320_crtc_funcs = {
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.destroy		= drm_crtc_cleanup,
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_encoder_funcs gm12u320_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
 static const uint32_t gm12u320_pipe_formats[] = {
@@ -678,13 +746,29 @@ static int gm12u320_usb_probe(struct usb_interface *interface,
 	if (ret)
 		return ret;
 
-	ret = drm_simple_display_pipe_init(&gm12u320->dev,
-					   &gm12u320->pipe,
-					   &gm12u320_pipe_funcs,
-					   gm12u320_pipe_formats,
-					   ARRAY_SIZE(gm12u320_pipe_formats),
-					   gm12u320_pipe_modifiers,
-					   &gm12u320->conn);
+	ret = drm_universal_plane_init(dev, &gm12u320->plane, 0,
+				       &gm12u320_plane_funcs,
+				       gm12u320_pipe_formats,
+				       ARRAY_SIZE(gm12u320_pipe_formats),
+				       gm12u320_pipe_modifiers,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+	drm_plane_helper_add(&gm12u320->plane, &gm12u320_plane_helper_funcs);
+
+	ret = drm_crtc_init_with_planes(dev, &gm12u320->crtc, &gm12u320->plane, NULL,
+					&gm12u320_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+	drm_crtc_helper_add(&gm12u320->crtc, &gm12u320_crtc_helper_funcs);
+
+	ret = drm_encoder_init(dev, &gm12u320->encoder, &gm12u320_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+	gm12u320->encoder.possible_crtcs = drm_crtc_mask(&gm12u320->crtc);
+
+	ret = drm_connector_attach_encoder(&gm12u320->conn, &gm12u320->encoder);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/tiny/pixpaper.c b/drivers/gpu/drm/tiny/pixpaper.c
index d02ac26d007c14..b69e7c30607e79 100644
--- a/drivers/gpu/drm/tiny/pixpaper.c
+++ b/drivers/gpu/drm/tiny/pixpaper.c
@@ -963,7 +963,7 @@ static const struct drm_plane_helper_funcs pixpaper_plane_helper_funcs = {
 static const struct drm_crtc_funcs pixpaper_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
diff --git a/drivers/gpu/drm/tiny/repaper.c b/drivers/gpu/drm/tiny/repaper.c
index c8270591afc7f8..82c99d86656f46 100644
--- a/drivers/gpu/drm/tiny/repaper.c
+++ b/drivers/gpu/drm/tiny/repaper.c
@@ -22,6 +22,7 @@
 #include <linux/thermal.h>
 
 #include <drm/clients/drm_client_setup.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_connector.h>
 #include <drm/drm_damage_helper.h>
@@ -38,7 +39,6 @@
 #include <drm/drm_rect.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #define REPAPER_RID_G2_COG_ID	0x12
 
@@ -65,7 +65,9 @@ enum repaper_epd_border_byte {
 
 struct repaper_epd {
 	struct drm_device drm;
-	struct drm_simple_display_pipe pipe;
+	struct drm_plane plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 	const struct drm_display_mode *mode;
 	struct drm_connector connector;
 	struct spi_device *spi;
@@ -622,26 +624,24 @@ static void power_off(struct repaper_epd *epd)
 	gpiod_set_value_cansleep(epd->discharge, 0);
 }
 
-static enum drm_mode_status repaper_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
-						    const struct drm_display_mode *mode)
+static enum drm_mode_status repaper_crtc_helper_mode_valid(struct drm_crtc *crtc,
+							   const struct drm_display_mode *mode)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct repaper_epd *epd = drm_to_epd(crtc->dev);
 
 	return drm_crtc_helper_mode_valid_fixed(crtc, mode, epd->mode);
 }
 
-static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
-				struct drm_crtc_state *crtc_state,
-				struct drm_plane_state *plane_state)
+static void repaper_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					      struct drm_atomic_commit *commit)
 {
-	struct repaper_epd *epd = drm_to_epd(pipe->crtc.dev);
+	struct repaper_epd *epd = drm_to_epd(crtc->dev);
 	struct spi_device *spi = epd->spi;
 	struct device *dev = &spi->dev;
 	bool dc_ok = false;
 	int i, ret, idx;
 
-	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+	if (!drm_dev_enter(crtc->dev, &idx))
 		return;
 
 	DRM_DEBUG_DRIVER("\n");
@@ -771,9 +771,10 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 	drm_dev_exit(idx);
 }
 
-static void repaper_pipe_disable(struct drm_simple_display_pipe *pipe)
+static void repaper_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					       struct drm_atomic_commit *commit)
 {
-	struct repaper_epd *epd = drm_to_epd(pipe->crtc.dev);
+	struct repaper_epd *epd = drm_to_epd(crtc->dev);
 	struct spi_device *spi = epd->spi;
 	unsigned int line;
 
@@ -827,27 +828,90 @@ static void repaper_pipe_disable(struct drm_simple_display_pipe *pipe)
 	power_off(epd);
 }
 
-static void repaper_pipe_update(struct drm_simple_display_pipe *pipe,
-				struct drm_plane_state *old_state)
+static void repaper_plane_helper_atomic_update(struct drm_plane *plane,
+					       struct drm_atomic_commit *commit)
 {
-	struct drm_plane_state *state = pipe->plane.state;
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(state);
+	struct drm_plane_state *old_pstate = drm_atomic_get_old_plane_state(commit, plane);
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(pstate);
+	struct drm_crtc_state *cstate = NULL;
 	struct drm_rect rect;
 
-	if (!pipe->crtc.state->active)
+	if (pstate->crtc)
+		cstate = drm_atomic_get_new_crtc_state(commit, pstate->crtc);
+
+	if (!cstate || !cstate->active)
 		return;
 
-	if (drm_atomic_helper_damage_merged(old_state, state, &rect))
-		repaper_fb_dirty(state->fb, shadow_plane_state->data,
+	if (drm_atomic_helper_damage_merged(old_pstate, pstate, &rect))
+		repaper_fb_dirty(pstate->fb, shadow_plane_state->data,
 				 &shadow_plane_state->fmtcnv_state);
 }
 
-static const struct drm_simple_display_pipe_funcs repaper_pipe_funcs = {
-	.mode_valid = repaper_pipe_mode_valid,
-	.enable = repaper_pipe_enable,
-	.disable = repaper_pipe_disable,
-	.update = repaper_pipe_update,
-	DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS,
+static const struct drm_plane_funcs repaper_plane_funcs = {
+	.update_plane	= drm_atomic_helper_update_plane,
+	.disable_plane	= drm_atomic_helper_disable_plane,
+	.destroy	= drm_plane_cleanup,
+	DRM_GEM_SHADOW_PLANE_FUNCS,
+};
+
+static int repaper_plane_helper_atomic_check(struct drm_plane *plane,
+					     struct drm_atomic_commit *commit)
+{
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_crtc_state *crtc_state = NULL;
+
+	if (plane_state->crtc) {
+		crtc_state = drm_atomic_get_crtc_state(commit, plane_state->crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+	}
+
+	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, false);
+}
+
+static const struct drm_plane_helper_funcs repaper_plane_helper_funcs = {
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+	.atomic_check	= repaper_plane_helper_atomic_check,
+	.atomic_update	= repaper_plane_helper_atomic_update,
+};
+
+static int repaper_crtc_helper_atomic_check(struct drm_crtc *crtc,
+					    struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static const struct drm_crtc_helper_funcs repaper_crtc_helper_funcs = {
+	.mode_valid	= repaper_crtc_helper_mode_valid,
+	.atomic_check	= repaper_crtc_helper_atomic_check,
+	.atomic_enable	= repaper_crtc_helper_atomic_enable,
+	.atomic_disable	= repaper_crtc_helper_atomic_disable,
+};
+
+static const struct drm_crtc_funcs repaper_crtc_funcs = {
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.destroy		= drm_crtc_cleanup,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_encoder_funcs repaper_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
 static int repaper_connector_get_modes(struct drm_connector *connector)
@@ -1102,9 +1166,27 @@ static int repaper_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = drm_simple_display_pipe_init(drm, &epd->pipe, &repaper_pipe_funcs,
-					   repaper_formats, ARRAY_SIZE(repaper_formats),
-					   NULL, &epd->connector);
+	ret = drm_universal_plane_init(drm, &epd->plane, 0,
+				       &repaper_plane_funcs,
+				       repaper_formats, ARRAY_SIZE(repaper_formats),
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+	drm_plane_helper_add(&epd->plane, &repaper_plane_helper_funcs);
+
+	ret = drm_crtc_init_with_planes(drm, &epd->crtc, &epd->plane, NULL,
+					&repaper_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+	drm_crtc_helper_add(&epd->crtc, &repaper_crtc_helper_funcs);
+
+	ret = drm_encoder_init(drm, &epd->encoder, &repaper_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+	epd->encoder.possible_crtcs = drm_crtc_mask(&epd->crtc);
+
+	ret = drm_connector_attach_encoder(&epd->connector, &epd->encoder);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/tiny/sharp-memory.c b/drivers/gpu/drm/tiny/sharp-memory.c
index 7efd7b567f3b21..434e73bd25a581 100644
--- a/drivers/gpu/drm/tiny/sharp-memory.c
+++ b/drivers/gpu/drm/tiny/sharp-memory.c
@@ -345,7 +345,7 @@ static const struct drm_crtc_helper_funcs sharp_memory_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs sharp_memory_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 9b85b5f388d41e..53c68e29ebc214 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -506,7 +506,6 @@ struct ttm_bo_alloc_state {
  *
  * @bo: The buffer to allocate the backing store of
  * @place: The place to attempt allocation in
- * @ctx: ttm_operation_ctx associated with this allocation
  * @force_space: If we should evict buffers to force space
  * @res: On allocation success, the resulting struct ttm_resource.
  * @alloc_state: Object holding allocation state such as charged cgroups.
@@ -1450,7 +1449,7 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo)
 	return ret;
 }
 
-const struct ttm_lru_walk_ops ttm_swap_ops = {
+static const struct ttm_lru_walk_ops ttm_swap_ops = {
 	.process_bo = ttm_bo_swapout_cb,
 };
 
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
index aa137ead5cc596..3daace10593037 100644
--- a/drivers/gpu/drm/ttm/ttm_module.c
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -34,7 +34,6 @@
 #include <linux/pgtable.h>
 #include <linux/sched.h>
 #include <linux/debugfs.h>
-#include <drm/drm_sysfs.h>
 #include <drm/ttm/ttm_caching.h>
 
 #include "ttm_module.h"
diff --git a/drivers/gpu/drm/tve200/Kconfig b/drivers/gpu/drm/tve200/Kconfig
index a9d6fe535d8860..afe243df4ebbe2 100644
--- a/drivers/gpu/drm/tve200/Kconfig
+++ b/drivers/gpu/drm/tve200/Kconfig
@@ -7,7 +7,7 @@ config DRM_TVE200
 	depends on OF
 	select DRM_BRIDGE
 	select DRM_CLIENT_SELECTION
-	select DRM_PANEL_BRIDGE
+	select DRM_PANEL
 	select DRM_KMS_HELPER
 	select DRM_GEM_DMA_HELPER
 	help
diff --git a/drivers/gpu/drm/tve200/tve200_display.c b/drivers/gpu/drm/tve200/tve200_display.c
index 26b6c65ef6fd00..7eb2b01fb5019a 100644
--- a/drivers/gpu/drm/tve200/tve200_display.c
+++ b/drivers/gpu/drm/tve200/tve200_display.c
@@ -15,12 +15,13 @@
 #include <linux/of_graph.h>
 #include <linux/delay.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_dma_helper.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
@@ -52,7 +53,7 @@ irqreturn_t tve200_irq(int irq, void *data)
 		val = readl(priv->regs + TVE200_CTRL);
 		/* We have an actual start of vsync */
 		if (!(val & TVE200_VSTSTYPE_BITS)) {
-			drm_crtc_handle_vblank(&priv->pipe.crtc);
+			drm_crtc_handle_vblank(&priv->crtc);
 			/* Toggle trigger to start of active image */
 			val |= TVE200_VSTSTYPE_VAI;
 		} else {
@@ -69,13 +70,34 @@ irqreturn_t tve200_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static int tve200_display_check(struct drm_simple_display_pipe *pipe,
-			       struct drm_plane_state *pstate,
-			       struct drm_crtc_state *cstate)
+static int tve200_plane_helper_atomic_check(struct drm_plane *plane,
+					    struct drm_atomic_commit *commit)
 {
-	const struct drm_display_mode *mode = &cstate->mode;
-	struct drm_framebuffer *old_fb = pipe->plane.state->fb;
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_plane_state *old_pstate = drm_atomic_get_old_plane_state(commit, plane);
+	struct drm_crtc_state *cstate = NULL;
+	const struct drm_display_mode *mode;
+	struct drm_framebuffer *old_fb = old_pstate->fb;
 	struct drm_framebuffer *fb = pstate->fb;
+	int ret;
+
+	if (pstate->crtc) {
+		cstate = drm_atomic_get_crtc_state(commit, pstate->crtc);
+		if (IS_ERR(cstate))
+			return PTR_ERR(cstate);
+	}
+
+	ret = drm_atomic_helper_check_plane_state(pstate, cstate,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, false);
+	if (ret)
+		return ret;
+
+	if (!pstate->visible || !cstate)
+		return 0;
+
+	mode = &cstate->mode;
 
 	/*
 	 * We support these specific resolutions and nothing else.
@@ -119,16 +141,72 @@ static int tve200_display_check(struct drm_simple_display_pipe *pipe,
 	return 0;
 }
 
-static void tve200_display_enable(struct drm_simple_display_pipe *pipe,
-				 struct drm_crtc_state *cstate,
-				 struct drm_plane_state *plane_state)
+static void tve200_plane_helper_atomic_update(struct drm_plane *plane,
+					      struct drm_atomic_commit *commit)
+{
+	struct tve200_drm_dev_private *priv = plane->dev->dev_private;
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_framebuffer *fb = pstate->fb;
+
+	if (!fb)
+		return;
+
+	/* For RGB, the Y component is used as base address */
+	writel(drm_fb_dma_get_gem_addr(fb, pstate, 0),
+	       priv->regs + TVE200_Y_FRAME_BASE_ADDR);
+
+	/* For three plane YUV we need two more addresses */
+	if (fb->format->format == DRM_FORMAT_YUV420) {
+		writel(drm_fb_dma_get_gem_addr(fb, pstate, 1),
+		       priv->regs + TVE200_U_FRAME_BASE_ADDR);
+		writel(drm_fb_dma_get_gem_addr(fb, pstate, 2),
+		       priv->regs + TVE200_V_FRAME_BASE_ADDR);
+	}
+}
+
+static void tve200_crtc_helper_atomic_flush(struct drm_crtc *crtc,
+					    struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *cstate = drm_atomic_get_new_crtc_state(commit, crtc);
+	struct drm_pending_vblank_event *event = cstate->event;
+
+	if (!event)
+		return;
+
+	cstate->event = NULL;
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	if (cstate->active && drm_crtc_vblank_get(crtc) == 0)
+		drm_crtc_arm_vblank_event(crtc, event);
+	else
+		drm_crtc_send_vblank_event(crtc, event);
+	spin_unlock_irq(&crtc->dev->event_lock);
+}
+
+static const struct drm_plane_helper_funcs tve200_plane_helper_funcs = {
+	.prepare_fb	= drm_gem_plane_helper_prepare_fb,
+	.atomic_check	= tve200_plane_helper_atomic_check,
+	.atomic_update	= tve200_plane_helper_atomic_update,
+};
+
+static const struct drm_plane_funcs tve200_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.destroy		= drm_plane_cleanup,
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+};
+
+static void tve200_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					     struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_plane *plane = &pipe->plane;
 	struct drm_device *drm = crtc->dev;
 	struct tve200_drm_dev_private *priv = drm->dev_private;
+	struct drm_crtc_state *cstate = drm_atomic_get_new_crtc_state(commit, crtc);
 	const struct drm_display_mode *mode = &cstate->mode;
-	struct drm_framebuffer *fb = plane->state->fb;
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, &priv->plane);
+	struct drm_framebuffer *fb = plane_state->fb;
 	struct drm_connector *connector = priv->connector;
 	u32 format = fb->format->format;
 	u32 ctrl1 = 0;
@@ -240,9 +318,9 @@ static void tve200_display_enable(struct drm_simple_display_pipe *pipe,
 	drm_crtc_vblank_on(crtc);
 }
 
-static void tve200_display_disable(struct drm_simple_display_pipe *pipe)
+static void tve200_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					      struct drm_atomic_commit *commit)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct tve200_drm_dev_private *priv = drm->dev_private;
 
@@ -255,46 +333,8 @@ static void tve200_display_disable(struct drm_simple_display_pipe *pipe)
 	clk_disable_unprepare(priv->clk);
 }
 
-static void tve200_display_update(struct drm_simple_display_pipe *pipe,
-				 struct drm_plane_state *old_pstate)
+static int tve200_crtc_enable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_device *drm = crtc->dev;
-	struct tve200_drm_dev_private *priv = drm->dev_private;
-	struct drm_pending_vblank_event *event = crtc->state->event;
-	struct drm_plane *plane = &pipe->plane;
-	struct drm_plane_state *pstate = plane->state;
-	struct drm_framebuffer *fb = pstate->fb;
-
-	if (fb) {
-		/* For RGB, the Y component is used as base address */
-		writel(drm_fb_dma_get_gem_addr(fb, pstate, 0),
-		       priv->regs + TVE200_Y_FRAME_BASE_ADDR);
-
-		/* For three plane YUV we need two more addresses */
-		if (fb->format->format == DRM_FORMAT_YUV420) {
-			writel(drm_fb_dma_get_gem_addr(fb, pstate, 1),
-			       priv->regs + TVE200_U_FRAME_BASE_ADDR);
-			writel(drm_fb_dma_get_gem_addr(fb, pstate, 2),
-			       priv->regs + TVE200_V_FRAME_BASE_ADDR);
-		}
-	}
-
-	if (event) {
-		crtc->state->event = NULL;
-
-		spin_lock_irq(&crtc->dev->event_lock);
-		if (crtc->state->active && drm_crtc_vblank_get(crtc) == 0)
-			drm_crtc_arm_vblank_event(crtc, event);
-		else
-			drm_crtc_send_vblank_event(crtc, event);
-		spin_unlock_irq(&crtc->dev->event_lock);
-	}
-}
-
-static int tve200_display_enable_vblank(struct drm_simple_display_pipe *pipe)
-{
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct tve200_drm_dev_private *priv = drm->dev_private;
 
@@ -304,22 +344,48 @@ static int tve200_display_enable_vblank(struct drm_simple_display_pipe *pipe)
 	return 0;
 }
 
-static void tve200_display_disable_vblank(struct drm_simple_display_pipe *pipe)
+static void tve200_crtc_disable_vblank(struct drm_crtc *crtc)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *drm = crtc->dev;
 	struct tve200_drm_dev_private *priv = drm->dev_private;
 
 	writel(0, priv->regs + TVE200_INT_EN);
 }
 
-static const struct drm_simple_display_pipe_funcs tve200_display_funcs = {
-	.check = tve200_display_check,
-	.enable = tve200_display_enable,
-	.disable = tve200_display_disable,
-	.update = tve200_display_update,
-	.enable_vblank = tve200_display_enable_vblank,
-	.disable_vblank = tve200_display_disable_vblank,
+static int tve200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static const struct drm_crtc_helper_funcs tve200_crtc_helper_funcs = {
+	.atomic_check	= tve200_crtc_helper_atomic_check,
+	.atomic_enable	= tve200_crtc_helper_atomic_enable,
+	.atomic_disable	= tve200_crtc_helper_atomic_disable,
+	.atomic_flush	= tve200_crtc_helper_atomic_flush,
+};
+
+static const struct drm_crtc_funcs tve200_crtc_funcs = {
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.destroy		= drm_crtc_cleanup,
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+	.enable_vblank		= tve200_crtc_enable_vblank,
+	.disable_vblank		= tve200_crtc_disable_vblank,
+};
+
+static const struct drm_encoder_funcs tve200_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
 int tve200_display_init(struct drm_device *drm)
@@ -346,13 +412,31 @@ int tve200_display_init(struct drm_device *drm)
 		DRM_FORMAT_YUV420,
 	};
 
-	ret = drm_simple_display_pipe_init(drm, &priv->pipe,
-					   &tve200_display_funcs,
-					   formats, ARRAY_SIZE(formats),
-					   NULL,
-					   priv->connector);
+	ret = drm_universal_plane_init(drm, &priv->plane, 0,
+				       &tve200_plane_funcs,
+				       formats, ARRAY_SIZE(formats),
+				       NULL,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret)
 		return ret;
 
+	drm_plane_helper_add(&priv->plane, &tve200_plane_helper_funcs);
+
+	ret = drm_crtc_init_with_planes(drm, &priv->crtc,
+					&priv->plane, NULL,
+					&tve200_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+
+	drm_crtc_helper_add(&priv->crtc, &tve200_crtc_helper_funcs);
+
+	ret = drm_encoder_init(drm, &priv->encoder,
+			       &tve200_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+
+	priv->encoder.possible_crtcs = drm_crtc_mask(&priv->crtc);
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/tve200/tve200_drm.h b/drivers/gpu/drm/tve200/tve200_drm.h
index 5420b52ea16b5a..708f8020667a71 100644
--- a/drivers/gpu/drm/tve200/tve200_drm.h
+++ b/drivers/gpu/drm/tve200/tve200_drm.h
@@ -15,8 +15,6 @@
 
 #include <linux/irqreturn.h>
 
-#include <drm/drm_simple_kms_helper.h>
-
 struct clk;
 struct drm_bridge;
 struct drm_connector;
@@ -105,9 +103,10 @@ struct tve200_drm_dev_private {
 	struct drm_device *drm;
 
 	struct drm_connector *connector;
-	struct drm_panel *panel;
 	struct drm_bridge *bridge;
-	struct drm_simple_display_pipe pipe;
+	struct drm_plane plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 
 	void *regs;
 	struct clk *pclk;
diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c
index 8d0e7c7c82e843..2bcb9dc86ffdce 100644
--- a/drivers/gpu/drm/tve200/tve200_drv.c
+++ b/drivers/gpu/drm/tve200/tve200_drv.c
@@ -65,7 +65,6 @@ static int tve200_modeset_init(struct drm_device *dev)
 {
 	struct drm_mode_config *mode_config;
 	struct tve200_drm_dev_private *priv = dev->dev_private;
-	struct drm_panel *panel;
 	struct drm_bridge *bridge;
 	int ret;
 
@@ -77,19 +76,10 @@ static int tve200_modeset_init(struct drm_device *dev)
 	mode_config->min_height = 240;
 	mode_config->max_height = 576;
 
-	ret = drm_of_find_panel_or_bridge(dev->dev->of_node,
-					  0, 0, &panel, &bridge);
-	if (ret && ret != -ENODEV)
-		return ret;
-	if (panel) {
-		bridge = drm_panel_bridge_add_typed(panel,
-						    DRM_MODE_CONNECTOR_Unknown);
-		drm_panel_put(panel);
-		if (IS_ERR(bridge)) {
-			ret = PTR_ERR(bridge);
-			goto out_bridge;
-		}
-	} else {
+	bridge = of_drm_get_bridge_by_endpoint(dev->dev->of_node, 0, 0);
+	if (IS_ERR(bridge) && PTR_ERR(bridge) != -ENODEV)
+		return PTR_ERR(bridge);
+	if (IS_ERR(bridge) || !drm_bridge_is_panel(bridge)) {
 		/*
 		 * TODO: when we are using a different bridge than a panel
 		 * (such as a dumb VGA connector) we need to devise a different
@@ -100,25 +90,27 @@ static int tve200_modeset_init(struct drm_device *dev)
 		goto out_bridge;
 	}
 
+	bridge->type = DRM_MODE_CONNECTOR_Unknown;
+
 	ret = tve200_display_init(dev);
 	if (ret) {
 		dev_err(dev->dev, "failed to init display\n");
 		goto out_bridge;
 	}
 
-	ret = drm_simple_display_pipe_attach_bridge(&priv->pipe,
-						    bridge);
+	priv->bridge = bridge;
+
+	ret = drm_bridge_attach(&priv->encoder, bridge, NULL, 0);
 	if (ret) {
 		dev_err(dev->dev, "failed to attach bridge\n");
 		goto out_bridge;
 	}
 
-	priv->panel = panel;
 	priv->connector = drm_panel_bridge_connector(bridge);
-	priv->bridge = bridge;
-
-	dev_info(dev->dev, "attached to panel %s\n",
-		 dev_name(panel->dev));
+	if (!priv->connector) {
+		ret = -ENODEV;
+		goto out_bridge;
+	}
 
 	ret = drm_vblank_init(dev, 1);
 	if (ret) {
@@ -132,8 +124,6 @@ static int tve200_modeset_init(struct drm_device *dev)
 	goto finish;
 
 out_bridge:
-	if (panel)
-		drm_panel_bridge_remove(bridge);
 	drm_mode_config_cleanup(dev);
 finish:
 	return ret;
@@ -229,8 +219,6 @@ static int tve200_probe(struct platform_device *pdev)
 	return 0;
 
 mode_config_cleanup:
-	if (priv->panel)
-		drm_panel_bridge_remove(priv->bridge);
 	drm_mode_config_cleanup(drm);
 clk_disable:
 	clk_disable_unprepare(priv->pclk);
@@ -244,10 +232,9 @@ static void tve200_remove(struct platform_device *pdev)
 	struct drm_device *drm = platform_get_drvdata(pdev);
 	struct tve200_drm_dev_private *priv = drm->dev_private;
 
+	drm_bridge_put(priv->bridge);
 	drm_dev_unregister(drm);
 	drm_atomic_helper_shutdown(drm);
-	if (priv->panel)
-		drm_panel_bridge_remove(priv->bridge);
 	drm_mode_config_cleanup(drm);
 	clk_disable_unprepare(priv->pclk);
 	drm_dev_put(drm);
diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs
index bfb0ba19caff9f..730b84e37a544c 100644
--- a/drivers/gpu/drm/tyr/driver.rs
+++ b/drivers/gpu/drm/tyr/driver.rs
@@ -46,6 +46,7 @@ use crate::{
 };
 
 pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>;
+pub(crate) type TyrRegisters = kernel::io::Region<SZ_2M>;
 
 pub(crate) struct TyrDrmDriver;
 
diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs
index 47d25c901bd016..7edb5eff17077a 100644
--- a/drivers/gpu/drm/tyr/fw.rs
+++ b/drivers/gpu/drm/tyr/fw.rs
@@ -39,7 +39,8 @@ use kernel::{
 use crate::{
     driver::{
         IoMem,
-        TyrDrmDevice, //
+        TyrDrmDevice,
+        TyrRegisters, //
     },
     fw::parser::{
         FwParser,
@@ -101,6 +102,8 @@ impl From<CacheMode> for Bounded<u32, 2> {
 }
 
 register! {
+    base: TyrRegisters;
+
      #[allow(non_upper_case_globals)]
     pub(super) SectionFlags(u32) @ 0x0 {
         0:0 read => bool;
diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs
index 592b8bb16eba18..e1b04a0c5159b6 100644
--- a/drivers/gpu/drm/tyr/gpu.rs
+++ b/drivers/gpu/drm/tyr/gpu.rs
@@ -74,10 +74,8 @@ impl GpuInfo {
                 io.read(TILER_PRESENT_HI).into_raw(),
             ),
             core_features: io.read(CORE_FEATURES).into_raw(),
-            // Padding must be zero.
-            pad: 0,
-            //GPU_FEATURES register is not available; it was introduced in arch 11.x.
-            gpu_features: 0,
+            // Zero unsupported and newly added UAPI fields.
+            ..pin_init::zeroed()
         })
     }
 
diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs
index a62724378ced2b..68701c39aee2ad 100644
--- a/drivers/gpu/drm/tyr/regs.rs
+++ b/drivers/gpu/drm/tyr/regs.rs
@@ -57,7 +57,11 @@ pub(crate) mod gpu_control {
         uapi, //
     };
 
+    use crate::driver::TyrRegisters;
+
     register! {
+        base: TyrRegisters;
+
         /// GPU identification register.
         pub(crate) GPU_ID(u32) @ 0x0 {
             /// Status of the GPU release.
@@ -315,6 +319,8 @@ pub(crate) mod gpu_control {
     }
 
     register! {
+        base: TyrRegisters;
+
         /// GPU command register.
         ///
         /// Use the constructor methods to create commands:
@@ -380,6 +386,8 @@ pub(crate) mod gpu_control {
     }
 
     register! {
+        base: TyrRegisters;
+
         /// GPU status register. Read only.
         pub(crate) GPU_STATUS(u32) @ 0x34 {
             /// GPU active, a 1-bit boolean flag.
@@ -463,6 +471,8 @@ pub(crate) mod gpu_control {
     }
 
     register! {
+        base: TyrRegisters;
+
         /// GPU fault status register. Read only.
         pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c {
             /// Exception type.
@@ -768,6 +778,8 @@ pub(crate) mod gpu_control {
     }
 
     register! {
+        base: TyrRegisters;
+
         /// Coherency enable. An index of which coherency protocols should be used.
         /// This register only selects the protocol for coherency messages on the
         /// interconnect. This is not to enable or disable coherency controlled by MMU.
@@ -808,6 +820,8 @@ pub(crate) mod gpu_control {
     }
 
     register! {
+        base: TyrRegisters;
+
         /// MCU control.
         pub(crate) MCU_CONTROL(u32) @ 0x700 {
             /// Request MCU state change.
@@ -849,6 +863,8 @@ pub(crate) mod gpu_control {
     }
 
     register! {
+        base: TyrRegisters;
+
         /// MCU status. Read only.
         pub(crate) MCU_STATUS(u32) @ 0x704 {
             /// Read current state of MCU.
@@ -862,7 +878,11 @@ pub(crate) mod gpu_control {
 pub(crate) mod job_control {
     use kernel::register;
 
+    use crate::driver::TyrRegisters;
+
     register! {
+        base: TyrRegisters;
+
         /// Raw status of job interrupts.
         ///
         /// Write to this register to trigger these interrupts.
@@ -912,7 +932,11 @@ pub(crate) mod job_control {
 pub(crate) mod mmu_control {
     use kernel::register;
 
+    use crate::driver::TyrRegisters;
+
     register! {
+        base: TyrRegisters;
+
         /// IRQ sources raw status.
         ///
         /// This register contains the raw unmasked interrupt sources for MMU status and exception
@@ -966,9 +990,10 @@ pub(crate) mod mmu_control {
             prelude::*,
             register, //
         };
-
         use pin_init::Zeroable;
 
+        use crate::driver::TyrRegisters;
+
         /// Maximum number of hardware address space slots.
         /// The actual number of slots available is usually lower.
         pub(crate) const MAX_AS: usize = 16;
@@ -977,6 +1002,8 @@ pub(crate) mod mmu_control {
         const STRIDE: usize = 0x40;
 
         register! {
+            base: TyrRegisters;
+
             /// Translation table base address. A 64-bit pointer.
             ///
             /// This field contains the address of the top level of a translation table structure.
@@ -1104,6 +1131,8 @@ pub(crate) mod mmu_control {
         }
 
         register! {
+            base: TyrRegisters;
+
             /// Stage 1 memory attributes (8-bit bitfield).
             ///
             /// This is not an actual register, but a bitfield definition used by the MEMATTR
@@ -1137,6 +1166,8 @@ pub(crate) mod mmu_control {
         }
 
         register! {
+            base: TyrRegisters;
+
             /// Memory attributes.
             ///
             /// Each address space can configure up to 8 different memory attribute profiles.
@@ -1292,6 +1323,8 @@ pub(crate) mod mmu_control {
         }
 
         register! {
+            base: TyrRegisters;
+
             /// Lock region address for each address space.
             pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 {
                 /// Lock region size.
@@ -1353,6 +1386,8 @@ pub(crate) mod mmu_control {
         }
 
         register! {
+            base: TyrRegisters;
+
             /// MMU command register for each address space. Write only.
             pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 {
                 7:0     command ?=> MmuCommand;
@@ -1480,6 +1515,8 @@ pub(crate) mod mmu_control {
         }
 
         register! {
+            base: TyrRegisters;
+
             /// Fault status register for each address space. Read only.
             pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c {
                 /// Exception type.
@@ -1705,6 +1742,8 @@ pub(crate) mod mmu_control {
         }
 
         register! {
+            base: TyrRegisters;
+
             /// Translation configuration and control.
             pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 {
                 /// Address space mode.
@@ -1760,8 +1799,13 @@ pub(crate) mod mmu_control {
 pub(crate) mod doorbell_block {
     use kernel::register;
 
+    use crate::driver::TyrRegisters;
+
     /// Number of doorbells available.
-    pub(crate) const NUM_DOORBELLS: usize = 64;
+    /// The architectural maximum is 64 but this full array does not
+    /// fit in the 2 MiB iomem region. Panthor currently uses a maximum of
+    /// 17 doorbells so do the same here.
+    pub(crate) const NUM_DOORBELLS: usize = 17;
 
     /// Doorbell block stride (64KiB).
     ///
@@ -1770,6 +1814,8 @@ pub(crate) mod doorbell_block {
     const STRIDE: usize = 0x10000;
 
     register! {
+        base: TyrRegisters;
+
         /// Doorbell request register. Write-only.
         pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 {
             /// Doorbell set. Writing 1 triggers the doorbell.
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index 289711035b67c2..dee085406ac2cc 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -400,7 +400,7 @@ static const struct drm_crtc_helper_funcs udl_crtc_helper_funcs = {
 };
 
 static const struct drm_crtc_funcs udl_crtc_funcs = {
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.destroy = drm_crtc_cleanup,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 29fa14be130b6d..d40610a34db990 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -84,6 +84,8 @@ struct v3d_queue_state {
  * This way, only events related to a specific submission will be counted.
  */
 struct v3d_perfmon {
+	struct v3d_dev *v3d;
+
 	/* Tracks the number of users of the perfmon, when this counter reaches
 	 * zero the perfmon is destroyed.
 	 */
@@ -184,6 +186,11 @@ struct v3d_dev {
 		/* Perfmon currently programmed in HW (or NULL if none). */
 		struct v3d_perfmon *active;
 
+		/* Number of perfmons alive on this device. Jobs are not
+		 * serialized if the number is zero.
+		 */
+		atomic_t nperfmons;
+
 		/* Finished fence of the most recently submitted job that
 		 * opened a serialization window (i.e. a job with a non-global
 		 * perfmon attached).
@@ -191,8 +198,11 @@ struct v3d_dev {
 		struct dma_fence *fence;
 
 		/* Finished fence of the most recently submitted job on each HW
-		 * queue. Used so that a new perfmon-carrying job can depend on
-		 * every job currently in-flight across all queues.
+		 * queue, which is used so that a new perfmon-carrying job can
+		 * depend on every job currently in-flight across all queues.
+		 *
+		 * Finished fences are only tracked if @nperfmons > 0 and no
+		 * global perfmon is set.
 		 */
 		struct dma_fence *last_hw_fence[V3D_MAX_QUEUES];
 	} perfmon_state;
@@ -200,12 +210,6 @@ struct v3d_dev {
 	/* Protects bo_stats */
 	struct mutex bo_lock;
 
-	/* Lock taken when resetting the GPU, to keep multiple
-	 * processes from trying to park the scheduler threads and
-	 * reset at once.
-	 */
-	struct mutex reset_lock;
-
 	/* Ordered workqueue shared by every queue's scheduler timeout work.
 	 * V3D reset is global to all queues, so the timeout handlers must not
 	 * run concurrently.
@@ -703,8 +707,6 @@ void v3d_perfmon_put(struct v3d_perfmon *perfmon);
 void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon);
 void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
 		      bool capture);
-void v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
-			     bool capture);
 void v3d_perfmon_suspend(struct v3d_dev *v3d);
 void v3d_perfmon_resume(struct v3d_dev *v3d);
 struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index bc88ebdcdbeadb..532a92ab2fe763 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -322,9 +322,6 @@ v3d_gem_init(struct drm_device *dev)
 	spin_lock_init(&v3d->mm_lock);
 	spin_lock_init(&v3d->perfmon_state.lock);
 	ret = drmm_mutex_init(dev, &v3d->bo_lock);
-	if (ret)
-		goto err_stats;
-	ret = drmm_mutex_init(dev, &v3d->reset_lock);
 	if (ret)
 		goto err_stats;
 	ret = drmm_mutex_init(dev, &v3d->sched_lock);
diff --git a/drivers/gpu/drm/v3d/v3d_perfmon.c b/drivers/gpu/drm/v3d/v3d_perfmon.c
index 07dab7fb30607d..c00f18d4a4257b 100644
--- a/drivers/gpu/drm/v3d/v3d_perfmon.c
+++ b/drivers/gpu/drm/v3d/v3d_perfmon.c
@@ -217,8 +217,10 @@ void v3d_perfmon_get(struct v3d_perfmon *perfmon)
 
 void v3d_perfmon_put(struct v3d_perfmon *perfmon)
 {
-	if (perfmon && refcount_dec_and_test(&perfmon->refcnt))
+	if (perfmon && refcount_dec_and_test(&perfmon->refcnt)) {
+		atomic_dec(&perfmon->v3d->perfmon_state.nperfmons);
 		kfree(perfmon);
+	}
 }
 
 static void v3d_perfmon_hw_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
@@ -305,8 +307,9 @@ static void v3d_perfmon_capture_locked(struct v3d_dev *v3d,
 	v3d_pm_runtime_put(v3d);
 }
 
-void v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
-			     bool capture)
+static void
+v3d_perfmon_stop_locked(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
+			bool capture)
 {
 	lockdep_assert_held(&v3d->perfmon_state.lock);
 
@@ -434,13 +437,15 @@ int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,
 		perfmon->counters[i] = req->counters[i];
 
 	perfmon->ncounters = req->ncounters;
+	perfmon->v3d = v3d;
 
 	refcount_set(&perfmon->refcnt, 1);
+	atomic_inc(&v3d->perfmon_state.nperfmons);
 
 	ret = xa_alloc(&v3d_priv->perfmons, &id, perfmon, xa_limit_32b,
 		       GFP_KERNEL);
 	if (ret < 0) {
-		kfree(perfmon);
+		v3d_perfmon_put(perfmon);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index e6283a75bcdf0b..1924e3440dc9d2 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -689,8 +689,6 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job,
 	struct v3d_job *job = to_v3d_job(sched_job);
 	enum v3d_queue i;
 
-	mutex_lock(&v3d->reset_lock);
-
 	/* block scheduler */
 	for (i = 0; i < V3D_MAX_QUEUES; i++)
 		drm_sched_stop(&v3d->queue[i].sched, sched_job);
@@ -713,8 +711,6 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job,
 	for (i = 0; i < V3D_MAX_QUEUES; i++)
 		drm_sched_start(&v3d->queue[i].sched, 0);
 
-	mutex_unlock(&v3d->reset_lock);
-
 	return DRM_GPU_SCHED_STAT_RESET;
 }
 
diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c
index 834d52030979dd..7548faab67baad 100644
--- a/drivers/gpu/drm/v3d/v3d_submit.c
+++ b/drivers/gpu/drm/v3d/v3d_submit.c
@@ -357,22 +357,28 @@ v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id)
  *
  * We don't serialize the jobs when using a global perfmon as it's expected to
  * track concurrent activity from all jobs.
+ *
+ * Keeping track of the in-flight jobs costs a fence merge per job, so it is
+ * only done while at least one perfmon is alive, and is skipped for the jobs
+ * carrying no perfmon while a global perfmon is set. Jobs submitted while
+ * tracking is off go untracked and may overlap the first measured job.
  */
 static int
 v3d_serialize_for_perfmon(struct v3d_job *job)
 {
 	struct v3d_dev *v3d = job->v3d;
 	struct dma_fence *merged;
-	bool is_global_perfmon;
 	int ret;
 
 	lockdep_assert_held(&v3d->sched_lock);
 
-	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock)
-		is_global_perfmon = !!v3d->global_perfmon;
+	if (!atomic_read(&v3d->perfmon_state.nperfmons))
+		return 0;
 
-	if (is_global_perfmon)
-		goto publish;
+	scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
+		if (!job->perfmon && v3d->global_perfmon)
+			return 0;
+	}
 
 	if (job->perfmon) {
 		for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) {
@@ -393,7 +399,6 @@ v3d_serialize_for_perfmon(struct v3d_job *job)
 			return ret;
 	}
 
-publish:
 	/*
 	 * Accumulate every in-flight job on this queue into one merged fence.
 	 * A HW queue is fed by several scheduler entities (one per-fd), so jobs
@@ -407,7 +412,7 @@ v3d_serialize_for_perfmon(struct v3d_job *job)
 	dma_fence_put(v3d->perfmon_state.last_hw_fence[job->queue]);
 	v3d->perfmon_state.last_hw_fence[job->queue] = merged;
 
-	if (job->perfmon && !is_global_perfmon) {
+	if (job->perfmon) {
 		dma_fence_put(v3d->perfmon_state.fence);
 		v3d->perfmon_state.fence = dma_fence_get(job->done_fence);
 	}
diff --git a/drivers/gpu/drm/vboxvideo/vbox_mode.c b/drivers/gpu/drm/vboxvideo/vbox_mode.c
index 3c41238a826857..bb3f7f0ad7c4b9 100644
--- a/drivers/gpu/drm/vboxvideo/vbox_mode.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_mode.c
@@ -251,7 +251,7 @@ static const struct drm_crtc_funcs vbox_crtc_funcs = {
 	.page_flip = drm_atomic_helper_page_flip,
 	/* .gamma_set = vbox_crtc_gamma_set, */
 	.destroy = vbox_crtc_destroy,
-	.reset = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
 };
@@ -499,7 +499,7 @@ static const struct drm_plane_funcs vbox_primary_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
 	.destroy	= drm_plane_helper_destroy,
-	.reset		= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/vc4/tests/vc4_mock_crtc.c b/drivers/gpu/drm/vc4/tests/vc4_mock_crtc.c
index ade3309ae042f1..247863f8e2c549 100644
--- a/drivers/gpu/drm/vc4/tests/vc4_mock_crtc.c
+++ b/drivers/gpu/drm/vc4/tests/vc4_mock_crtc.c
@@ -14,7 +14,7 @@ static const struct drm_crtc_helper_funcs vc4_dummy_crtc_helper_funcs = {
 static const struct drm_crtc_funcs vc4_dummy_crtc_funcs = {
 	.atomic_destroy_state	= vc4_crtc_destroy_state,
 	.atomic_duplicate_state	= vc4_crtc_duplicate_state,
-	.reset			= vc4_crtc_reset,
+	.atomic_create_state	= vc4_crtc_create_state,
 };
 
 struct vc4_dummy_crtc *vc4_mock_pv(struct kunit *test,
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 4572b0b9bae1ce..f42cd535323020 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -1135,21 +1135,19 @@ void vc4_crtc_destroy_state(struct drm_crtc *crtc,
 	drm_atomic_helper_crtc_destroy_state(crtc, state);
 }
 
-void vc4_crtc_reset(struct drm_crtc *crtc)
+struct drm_crtc_state *vc4_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct vc4_crtc_state *vc4_crtc_state;
 
-	if (crtc->state)
-		vc4_crtc_destroy_state(crtc, crtc->state);
-
 	vc4_crtc_state = kzalloc_obj(*vc4_crtc_state);
 	if (!vc4_crtc_state) {
-		crtc->state = NULL;
-		return;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	vc4_crtc_state->assigned_channel = VC4_HVS_CHANNEL_DISABLED;
-	__drm_atomic_helper_crtc_reset(crtc, &vc4_crtc_state->base);
+	__drm_atomic_helper_crtc_state_init(&vc4_crtc_state->base, crtc);
+
+	return &vc4_crtc_state->base;
 }
 
 int vc4_crtc_late_register(struct drm_crtc *crtc)
@@ -1170,7 +1168,7 @@ static const struct drm_crtc_funcs vc4_crtc_funcs = {
 	.set_property = NULL,
 	.cursor_set = NULL, /* handled by drm_mode_cursor_universal */
 	.cursor_move = NULL, /* handled by drm_mode_cursor_universal */
-	.reset = vc4_crtc_reset,
+	.atomic_create_state = vc4_crtc_create_state,
 	.atomic_duplicate_state = vc4_crtc_duplicate_state,
 	.atomic_destroy_state = vc4_crtc_destroy_state,
 	.enable_vblank = vc4_enable_vblank,
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 616caf9d9915b9..6930ca20e14b42 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -279,7 +279,6 @@ static const struct of_device_id vc4_dma_range_matches[] = {
 	{ .compatible = "brcm,bcm2835-hvs" },
 	{ .compatible = "brcm,bcm2835-v3d" },
 	{ .compatible = "brcm,cygnus-v3d" },
-	{ .compatible = "brcm,vc4-v3d" },
 	{}
 };
 
@@ -360,8 +359,11 @@ static int vc4_drm_bind(struct device *dev)
 	}
 
 	ret = aperture_remove_all_conflicting_devices(driver->name);
-	if (ret)
+	if (ret) {
+		if (firmware)
+			rpi_firmware_put(firmware);
 		goto err;
+	}
 
 	if (firmware) {
 		ret = rpi_firmware_property(firmware,
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 0f5958c1e6b601..695bc6a29d30c5 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -212,14 +212,9 @@ struct vc4_dev {
 
 	struct work_struct overflow_mem_work;
 
-	int power_refcount;
-
 	/* Set to true when the load tracker is active. */
 	bool load_tracker_enabled;
 
-	/* Mutex controlling the power refcount. */
-	struct mutex power_lock;
-
 	struct {
 		struct timer_list timer;
 		struct work_struct reset_work;
@@ -294,6 +289,13 @@ struct vc4_v3d {
 	struct platform_device *pdev;
 	void __iomem *regs;
 	struct clk *clk;
+
+	/* Reset line for the V3D block, used to recover from a GPU hang.
+	 * NULL if the device tree does not describe one, in which case the
+	 * GPU cannot be reset.
+	 */
+	struct reset_control *reset;
+
 	struct debugfs_regset32 regset;
 };
 
@@ -949,7 +951,7 @@ int vc4_crtc_atomic_check(struct drm_crtc *crtc,
 struct drm_crtc_state *vc4_crtc_duplicate_state(struct drm_crtc *crtc);
 void vc4_crtc_destroy_state(struct drm_crtc *crtc,
 			    struct drm_crtc_state *state);
-void vc4_crtc_reset(struct drm_crtc *crtc);
+struct drm_crtc_state *vc4_crtc_create_state(struct drm_crtc *crtc);
 void vc4_crtc_handle_vblank(struct vc4_crtc *crtc);
 void vc4_crtc_send_vblank(struct drm_crtc *crtc);
 int vc4_crtc_late_register(struct drm_crtc *crtc);
@@ -1056,6 +1058,7 @@ int vc4_v3d_bin_bo_get(struct vc4_dev *vc4, bool *used);
 void vc4_v3d_bin_bo_put(struct vc4_dev *vc4);
 int vc4_v3d_pm_get(struct vc4_dev *vc4);
 void vc4_v3d_pm_put(struct vc4_dev *vc4);
+void vc4_v3d_init_hw(struct drm_device *dev);
 int vc4_v3d_debugfs_init(struct drm_minor *minor);
 
 /* vc4_validate.c */
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index e231c906709c1b..3212b91676201b 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -23,7 +23,7 @@
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
+#include <linux/reset.h>
 #include <linux/device.h>
 #include <linux/io.h>
 #include <linux/sched/signal.h>
@@ -292,19 +292,22 @@ vc4_save_hang_state(struct drm_device *dev)
 static void
 vc4_reset(struct drm_device *dev)
 {
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_v3d *v3d = to_vc4_dev(dev)->v3d;
+	int ret;
 
-	DRM_INFO("Resetting GPU.\n");
+	vc4_irq_disable(dev);
 
-	mutex_lock(&vc4->power_lock);
-	if (vc4->power_refcount) {
-		/* Power the device off and back on the by dropping the
-		 * reference on runtime PM.
-		 */
-		pm_runtime_put_sync_suspend(&vc4->v3d->pdev->dev);
-		pm_runtime_get_sync(&vc4->v3d->pdev->dev);
+	if (v3d->reset) {
+		drm_info(dev, "Resetting GPU.\n");
+
+		ret = reset_control_reset(v3d->reset);
+		if (ret)
+			drm_err(dev, "Failed to reset the GPU: %d\n", ret);
+
+		vc4_v3d_init_hw(dev);
+	} else {
+		drm_info_once(dev, "No reset line; GPU state is not reset.\n");
 	}
-	mutex_unlock(&vc4->power_lock);
 
 	vc4_irq_reset(dev);
 
@@ -320,10 +323,19 @@ vc4_reset_work(struct work_struct *work)
 {
 	struct vc4_dev *vc4 =
 		container_of(work, struct vc4_dev, hangcheck.reset_work);
+	int ret;
 
-	vc4_save_hang_state(&vc4->base);
+	/* Make sure the device is not suspended during the reset. */
+	ret = vc4_v3d_pm_get(vc4);
+	if (ret) {
+		drm_err(&vc4->base, "Failed to resume V3D for GPU reset: %d\n", ret);
+		return;
+	}
 
+	vc4_save_hang_state(&vc4->base);
 	vc4_reset(&vc4->base);
+
+	vc4_v3d_pm_put(vc4);
 }
 
 static void
@@ -1177,10 +1189,6 @@ int vc4_gem_init(struct drm_device *dev)
 
 	INIT_WORK(&vc4->job_done_work, vc4_job_done_work);
 
-	ret = drmm_mutex_init(dev, &vc4->power_lock);
-	if (ret)
-		return ret;
-
 	INIT_LIST_HEAD(&vc4->purgeable.list);
 
 	ret = drmm_mutex_init(dev, &vc4->purgeable.lock);
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index 17c8635c5afae8..3a90dbb61f4ea7 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -375,7 +375,7 @@ static void vc4_hdmi_handle_hotplug(struct vc4_hdmi *vc4_hdmi,
 	 * the lock for now.
 	 */
 
-	drm_atomic_helper_connector_hdmi_hotplug(connector, status);
+	drm_atomic_helper_connector_hdmi_hotplug(connector, ctx, status);
 
 	if (status != connector_status_connected)
 		return;
@@ -513,6 +513,7 @@ static void vc4_hdmi_connector_reset(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs vc4_hdmi_connector_funcs = {
+	.debugfs_init = drm_hdmi_connector_debugfs_init,
 	.force = drm_atomic_helper_connector_hdmi_force,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.reset = vc4_hdmi_connector_reset,
@@ -541,7 +542,7 @@ static int vc4_hdmi_connector_init(struct drm_device *dev,
 	if (vc4_hdmi->variant->supports_hdr)
 		max_bpc = 12;
 
-	ret = drmm_connector_hdmi_init(dev, connector,
+	ret = drmm_connector_hdmi_ini2(dev, connector,
 				       "Broadcom", "Videocore",
 				       &vc4_hdmi_connector_funcs,
 				       &vc4_hdmi_hdmi_connector_funcs,
diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c
index 7877d493d80e62..3a3ea1e62dcb59 100644
--- a/drivers/gpu/drm/vc4/vc4_irq.c
+++ b/drivers/gpu/drm/vc4/vc4_irq.c
@@ -336,10 +336,9 @@ void vc4_irq_reset(struct drm_device *dev)
 	V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);
 
 	/*
-	 * Turn all our interrupts on.  Binner out of memory is the
-	 * only one we expect to trigger at this point, since we've
-	 * just come from poweron and haven't supplied any overflow
-	 * memory yet.
+	 * Turn all our interrupts on. Binner out of memory is the only
+	 * one we expect to trigger at this point, since the reset cleared
+	 * the overflow memory address and none has been supplied yet.
 	 */
 	V3D_WRITE(V3D_INTENA, V3D_DRIVER_IRQS);
 
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 6c433270214d2f..20c9fc9283a88a 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -363,20 +363,17 @@ static void vc4_plane_destroy_state(struct drm_plane *plane,
 }
 
 /* Called during init to allocate the plane's atomic state. */
-static void vc4_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *vc4_plane_create_state(struct drm_plane *plane)
 {
 	struct vc4_plane_state *vc4_state;
 
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-
-	kfree(plane->state);
-
 	vc4_state = kzalloc_obj(*vc4_state);
 	if (!vc4_state)
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	__drm_atomic_helper_plane_reset(plane, &vc4_state->base);
+	__drm_atomic_helper_plane_state_init(&vc4_state->base, plane);
+
+	return &vc4_state->base;
 }
 
 static void vc4_dlist_counter_increment(struct vc4_plane_state *vc4_state)
@@ -2495,7 +2492,7 @@ static bool vc4_format_mod_supported(struct drm_plane *plane,
 static const struct drm_plane_funcs vc4_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.reset = vc4_plane_reset,
+	.atomic_create_state = vc4_plane_create_state,
 	.atomic_duplicate_state = vc4_plane_duplicate_state,
 	.atomic_destroy_state = vc4_plane_destroy_state,
 	.format_mod_supported = vc4_format_mod_supported,
diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c
index bc3f366fc3e687..c469f8b1068b15 100644
--- a/drivers/gpu/drm/vc4/vc4_txp.c
+++ b/drivers/gpu/drm/vc4/vc4_txp.c
@@ -425,7 +425,7 @@ static void vc4_txp_disable_vblank(struct drm_crtc *crtc) {}
 static const struct drm_crtc_funcs vc4_txp_crtc_funcs = {
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= vc4_page_flip,
-	.reset			= vc4_crtc_reset,
+	.atomic_create_state	= vc4_crtc_create_state,
 	.atomic_duplicate_state	= vc4_crtc_duplicate_state,
 	.atomic_destroy_state	= vc4_crtc_destroy_state,
 	.enable_vblank		= vc4_txp_enable_vblank,
diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c
index f32410420d3e4d..d365177c416dde 100644
--- a/drivers/gpu/drm/vc4/vc4_v3d.c
+++ b/drivers/gpu/drm/vc4/vc4_v3d.c
@@ -9,6 +9,8 @@
 #include <linux/component.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/timer.h>
 
 #include <drm/drm_print.h>
 
@@ -122,29 +124,13 @@ static int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused)
 	return 0;
 }
 
-/*
- * Wraps pm_runtime_get_sync() in a refcount, so that we can reliably
- * get the pm_runtime refcount to 0 in vc4_reset().
- */
 int
 vc4_v3d_pm_get(struct vc4_dev *vc4)
 {
 	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 		return -ENODEV;
 
-	mutex_lock(&vc4->power_lock);
-	if (vc4->power_refcount++ == 0) {
-		int ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-
-		if (ret < 0) {
-			vc4->power_refcount--;
-			mutex_unlock(&vc4->power_lock);
-			return ret;
-		}
-	}
-	mutex_unlock(&vc4->power_lock);
-
-	return 0;
+	return pm_runtime_resume_and_get(&vc4->v3d->pdev->dev);
 }
 
 void
@@ -153,15 +139,10 @@ vc4_v3d_pm_put(struct vc4_dev *vc4)
 	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 		return;
 
-	mutex_lock(&vc4->power_lock);
-	if (--vc4->power_refcount == 0) {
-		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
-		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
-	}
-	mutex_unlock(&vc4->power_lock);
+	pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
 }
 
-static void vc4_v3d_init_hw(struct drm_device *dev)
+void vc4_v3d_init_hw(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
@@ -447,6 +428,14 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
 	if (IS_ERR(v3d->clk))
 		return dev_err_probe(dev, PTR_ERR(v3d->clk), "Failed to get V3D clock\n");
 
+	v3d->reset = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(v3d->reset))
+		return dev_err_probe(dev, PTR_ERR(v3d->reset),
+				     "Failed to get reset control\n");
+
+	if (!v3d->reset)
+		drm_warn(drm, "No V3D reset line in the device tree");
+
 	ret = platform_get_irq(pdev, 0);
 	if (ret < 0)
 		return ret;
@@ -497,8 +486,15 @@ static void vc4_v3d_unbind(struct device *dev, struct device *master,
 	struct drm_device *drm = data;
 	struct vc4_dev *vc4 = to_vc4_dev(drm);
 
+	/* A straggler vc4_reset() re-enables the interrupt. */
+	timer_shutdown_sync(&vc4->hangcheck.timer);
+	cancel_work_sync(&vc4->hangcheck.reset_work);
+
 	vc4_irq_uninstall(drm);
 
+	/* Flush rather than cancel, so queued completions release their jobs. */
+	flush_work(&vc4->job_done_work);
+
 	/* Disable the binner's overflow memory address, so the next
 	 * driver probe (if any) doesn't try to reuse our old
 	 * allocation.
@@ -531,7 +527,6 @@ static void vc4_v3d_dev_remove(struct platform_device *pdev)
 const struct of_device_id vc4_v3d_dt_match[] = {
 	{ .compatible = "brcm,bcm2835-v3d" },
 	{ .compatible = "brcm,cygnus-v3d" },
-	{ .compatible = "brcm,vc4-v3d" },
 	{}
 };
 
diff --git a/drivers/gpu/drm/verisilicon/vs_crtc.c b/drivers/gpu/drm/verisilicon/vs_crtc.c
index 0b8a35d09cd2e2..c24c1588cbe695 100644
--- a/drivers/gpu/drm/verisilicon/vs_crtc.c
+++ b/drivers/gpu/drm/verisilicon/vs_crtc.c
@@ -149,7 +149,7 @@ static const struct drm_crtc_funcs vs_crtc_funcs = {
 	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
 	.page_flip		= drm_atomic_helper_page_flip,
-	.reset			= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.set_config		= drm_atomic_helper_set_config,
 	.enable_vblank		= vs_crtc_enable_vblank,
 	.disable_vblank		= vs_crtc_disable_vblank,
diff --git a/drivers/gpu/drm/verisilicon/vs_cursor_plane.c b/drivers/gpu/drm/verisilicon/vs_cursor_plane.c
index da456dced88aad..1fa6b7578b27e9 100644
--- a/drivers/gpu/drm/verisilicon/vs_cursor_plane.c
+++ b/drivers/gpu/drm/verisilicon/vs_cursor_plane.c
@@ -13,6 +13,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
@@ -177,7 +178,8 @@ static void vs_cursor_plane_atomic_update(struct drm_plane *plane,
 		break;
 	}
 
-	dma_addr = vs_fb_get_dma_addr(fb, &state->src);
+	/* hardware handles clipping as seen below */
+	dma_addr = drm_fb_dma_get_gem_addr(fb, state, 0);
 
 	regmap_write(dc->regs, VSDC_CURSOR_ADDRESS(output),
 		     lower_32_bits(dma_addr));
@@ -200,7 +202,7 @@ static void vs_cursor_plane_atomic_update(struct drm_plane *plane,
 	} else {
 		regmap_update_bits(dc->regs, VSDC_CURSOR_CONFIG(output),
 				   VSDC_CURSOR_CONFIG_X_OFF_MASK,
-				   -state->crtc_x);
+				   VSDC_CURSOR_CONFIG_X_OFF(-state->crtc_x));
 		regmap_update_bits(dc->regs, VSDC_CURSOR_LOCATION(output),
 				   VSDC_CURSOR_LOCATION_X_MASK, 0);
 	}
@@ -214,7 +216,7 @@ static void vs_cursor_plane_atomic_update(struct drm_plane *plane,
 	} else {
 		regmap_update_bits(dc->regs, VSDC_CURSOR_CONFIG(output),
 				   VSDC_CURSOR_CONFIG_Y_OFF_MASK,
-				   -state->crtc_y);
+				   VSDC_CURSOR_CONFIG_Y_OFF(-state->crtc_y));
 		regmap_update_bits(dc->regs, VSDC_CURSOR_LOCATION(output),
 				   VSDC_CURSOR_LOCATION_Y_MASK, 0);
 	}
@@ -230,10 +232,10 @@ static const struct drm_plane_helper_funcs vs_cursor_plane_helper_funcs = {
 };
 
 static const struct drm_plane_funcs vs_cursor_plane_funcs = {
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
 	.update_plane		= drm_atomic_helper_update_plane,
 };
 
diff --git a/drivers/gpu/drm/verisilicon/vs_plane.c b/drivers/gpu/drm/verisilicon/vs_plane.c
index 7ddb9d2dcb83dd..03a848ee08750d 100644
--- a/drivers/gpu/drm/verisilicon/vs_plane.c
+++ b/drivers/gpu/drm/verisilicon/vs_plane.c
@@ -107,26 +107,6 @@ int drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format)
 	return 0;
 }
 
-dma_addr_t vs_fb_get_dma_addr(struct drm_framebuffer *fb,
-			      const struct drm_rect *src_rect)
-{
-	struct drm_gem_dma_object *gem;
-	dma_addr_t dma_addr;
-
-	/* Get the physical address of the buffer in memory */
-	gem = drm_fb_dma_get_gem_obj(fb, 0);
-
-	/* Compute the start of the displayed memory */
-	dma_addr = gem->dma_addr + fb->offsets[0];
-
-	/* Fixup framebuffer address for src coordinates */
-	dma_addr += drm_format_info_min_pitch(fb->format, 0,
-					      src_rect->x1 >> 16);
-	dma_addr += (src_rect->y1 >> 16) * fb->pitches[0];
-
-	return dma_addr;
-}
-
 struct drm_plane_state *vs_plane_duplicate_state(struct drm_plane *plane)
 {
 	struct vs_plane_state *vs_state, *vs_state_old;
@@ -156,19 +136,15 @@ void vs_plane_destroy_state(struct drm_plane *plane,
 }
 
 /* Called during init to allocate the plane's atomic state. */
-void vs_plane_reset(struct drm_plane *plane)
+struct drm_plane_state *vs_plane_create_state(struct drm_plane *plane)
 {
 	struct vs_plane_state *vs_state;
 
-	if (plane->state) {
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-		kfree(plane->state);
-		plane->state = NULL;
-	}
-
 	vs_state = kzalloc_obj(*vs_state);
 	if (!vs_state)
-		return;
+		return ERR_PTR(-ENOMEM);
 
-	__drm_atomic_helper_plane_reset(plane, &vs_state->base);
+	__drm_atomic_helper_plane_state_init(&vs_state->base, plane);
+
+	return &vs_state->base;
 }
diff --git a/drivers/gpu/drm/verisilicon/vs_plane.h b/drivers/gpu/drm/verisilicon/vs_plane.h
index 32ed1d70f3b8c6..b4ce14098aa322 100644
--- a/drivers/gpu/drm/verisilicon/vs_plane.h
+++ b/drivers/gpu/drm/verisilicon/vs_plane.h
@@ -75,13 +75,11 @@ static inline struct vs_plane_state *to_vs_plane_state(struct drm_plane_state *s
 }
 
 int drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format);
-dma_addr_t vs_fb_get_dma_addr(struct drm_framebuffer *fb,
-			      const struct drm_rect *src_rect);
 
 struct drm_plane_state *vs_plane_duplicate_state(struct drm_plane *plane);
 void vs_plane_destroy_state(struct drm_plane *plane,
 			    struct drm_plane_state *state);
-void vs_plane_reset(struct drm_plane *plane);
+struct drm_plane_state *vs_plane_create_state(struct drm_plane *plane);
 
 struct drm_plane *vs_primary_plane_init(struct drm_device *dev, struct vs_dc *dc);
 struct drm_plane *vs_cursor_plane_init(struct drm_device *dev, struct vs_dc *dc);
diff --git a/drivers/gpu/drm/verisilicon/vs_primary_plane.c b/drivers/gpu/drm/verisilicon/vs_primary_plane.c
index 8e944916930191..95f4a06039e767 100644
--- a/drivers/gpu/drm/verisilicon/vs_primary_plane.c
+++ b/drivers/gpu/drm/verisilicon/vs_primary_plane.c
@@ -8,6 +8,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
@@ -126,7 +127,11 @@ static void vs_primary_plane_atomic_update(struct drm_plane *plane,
 			   VSDC_FB_CONFIG_UV_SWIZZLE_EN,
 			   vs_state->format.uv_swizzle);
 
-	dma_addr = vs_fb_get_dma_addr(fb, &state->src);
+	/*
+	 * Primary plane cannot be moved, no clipping is involved,
+	 * so the non-clipped framebuffer address can be used.
+	 */
+	dma_addr = drm_fb_dma_get_gem_addr(fb, state, 0);
 
 	regmap_write(dc->regs, VSDC_FB_ADDRESS(output),
 		     lower_32_bits(dma_addr));
@@ -155,10 +160,10 @@ static const struct drm_plane_helper_funcs vs_primary_plane_helper_funcs = {
 };
 
 static const struct drm_plane_funcs vs_primary_plane_funcs = {
+	.atomic_create_state	= vs_plane_create_state,
 	.atomic_destroy_state	= vs_plane_destroy_state,
 	.atomic_duplicate_state	= vs_plane_duplicate_state,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= vs_plane_reset,
 	.update_plane		= drm_atomic_helper_update_plane,
 };
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index a1a875a0c706a7..b60aab07e78d2e 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -28,11 +28,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_encoder.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_vblank_helper.h>
 
@@ -55,7 +55,7 @@ static const struct drm_crtc_funcs virtio_gpu_crtc_funcs = {
 	.destroy                = drm_crtc_cleanup,
 
 	.page_flip              = drm_atomic_helper_page_flip,
-	.reset                  = drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state   = drm_atomic_helper_crtc_destroy_state,
 	DRM_CRTC_VBLANK_TIMER_FUNCS,
@@ -232,6 +232,10 @@ static enum drm_mode_status virtio_gpu_conn_mode_valid(struct drm_connector *con
 	return MODE_BAD;
 }
 
+static const struct drm_encoder_funcs virtio_gpu_enc_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static const struct drm_encoder_helper_funcs virtio_gpu_enc_helper_funcs = {
 	.mode_set   = virtio_gpu_enc_mode_set,
 	.enable     = virtio_gpu_enc_enable,
@@ -306,7 +310,11 @@ static int vgdev_output_init(struct virtio_gpu_device *vgdev, int index)
 	if (vgdev->has_edid)
 		drm_connector_attach_edid_property(connector);
 
-	drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_VIRTUAL);
+	ret = drm_encoder_init(dev, encoder, &virtio_gpu_enc_funcs,
+			       DRM_MODE_ENCODER_VIRTUAL, NULL);
+	if (ret)
+		return ret;
+
 	drm_encoder_helper_add(encoder, &virtio_gpu_enc_helper_funcs);
 	encoder->possible_crtcs = 1 << index;
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 9df4c7117341e7..f3bbbe4468bf1f 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -114,6 +114,8 @@ struct virtio_gpu_object {
 	bool dumb;
 	bool created;
 	bool attached;
+	/* a guest-bound transfer is queued and its mapping not yet synced */
+	bool from_host_pending;
 	bool host3d_blob, guest_blob;
 	uint32_t blob_mem, blob_flags;
 
@@ -196,6 +198,9 @@ struct virtio_gpu_vbuffer {
 	struct list_head list;
 
 	uint32_t seqno;
+
+	/* guest-bound transfer whose shmem backing needs a CPU sync */
+	bool sync_for_cpu;
 };
 
 struct virtio_gpu_output {
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
index 66c3f6f74e9c69..d2f0b8a3f172be 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -45,7 +45,7 @@ static int virtio_gpu_gem_create(struct drm_file *file,
 
 	ret = drm_gem_handle_create(file, &obj->base.base, &handle);
 	if (ret) {
-		drm_gem_object_release(&obj->base.base);
+		drm_gem_object_put(&obj->base.base);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index 3d8e4ccdb7c1f8..81e70a12b35692 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -185,7 +185,7 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data,
 
 	ret = drm_gem_handle_create(file, obj, &handle);
 	if (ret) {
-		drm_gem_object_release(obj);
+		drm_gem_object_put(obj);
 		return ret;
 	}
 
@@ -261,6 +261,27 @@ static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev,
 	if (ret != 0)
 		goto err_put_free;
 
+	if (virtio_gpu_is_shmem(bo) && virtio_gpu_use_dma_api(vgdev->vdev)) {
+		/*
+		 * The sync on completion restores the whole mapping, so an
+		 * earlier transfer has to be done before this one snapshots it.
+		 * Otherwise the snapshot predates anything the CPU wrote once
+		 * that transfer's fence signalled, and the later sync would
+		 * discard it. Nothing can add a fence behind our back here,
+		 * since doing so takes the reservation we already hold.
+		 * This writes the pages, so it waits as a writer does. READ
+		 * usage covers existing readers.
+		 */
+		long wait = dma_resv_wait_timeout(objs->objs[0]->resv,
+						  DMA_RESV_USAGE_READ, true,
+						  MAX_SCHEDULE_TIMEOUT);
+
+		if (wait < 0) {
+			ret = wait;
+			goto err_unlock;
+		}
+	}
+
 	fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0);
 	if (!fence) {
 		ret = -ENOMEM;
@@ -320,6 +341,28 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data,
 		if (ret != 0)
 			goto err_put_free;
 
+		/*
+		 * A transfer the other way may have queued without yet syncing
+		 * its mapping. Pushing the guest pages into it now would
+		 * discard what the device wrote there, so wait for that sync:
+		 * it runs before the fence it belongs to is signalled. The
+		 * flag is only set under this reservation, so it cannot appear
+		 * behind our back, and the acquire pairs with the release in
+		 * that sync, so finding it clear means the pages it wrote are
+		 * visible here too.
+		 */
+		if (smp_load_acquire(&bo->from_host_pending)) {
+			long wait = dma_resv_wait_timeout(objs->objs[0]->resv,
+							  DMA_RESV_USAGE_WRITE,
+							  true,
+							  MAX_SCHEDULE_TIMEOUT);
+
+			if (wait < 0) {
+				ret = wait;
+				goto err_unlock;
+			}
+		}
+
 		ret = -ENOMEM;
 		fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context,
 					       0);
@@ -557,14 +600,14 @@ static int virtio_gpu_resource_create_blob_ioctl(struct drm_device *dev,
 	if (params.blob_flags & VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE) {
 		ret = virtio_gpu_resource_assign_uuid(vgdev, bo);
 		if (ret) {
-			drm_gem_object_release(obj);
+			drm_gem_object_put(obj);
 			return ret;
 		}
 	}
 
 	ret = drm_gem_handle_create(file, obj, &handle);
 	if (ret) {
-		drm_gem_object_release(obj);
+		drm_gem_object_put(obj);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index 640815af409801..10718bc7369399 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -31,6 +31,7 @@
 #include <linux/virtio_dma_buf.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_panic.h>
+#include <drm/drm_panic_helper.h>
 #include <drm/drm_print.h>
 
 #include "virtgpu_drv.h"
@@ -92,9 +93,10 @@ drm_plane_state *virtio_gpu_plane_duplicate_state(struct drm_plane *plane)
 static const struct drm_plane_funcs virtio_gpu_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state = virtio_gpu_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+	DRM_PANIC_PLANE_FUNCS,
 };
 
 static int virtio_gpu_plane_atomic_check(struct drm_plane *plane,
@@ -104,6 +106,7 @@ static int virtio_gpu_plane_atomic_check(struct drm_plane *plane,
 										 plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state,
 										 plane);
+	struct virtio_gpu_object *bo;
 	bool is_cursor = plane->type == DRM_PLANE_TYPE_CURSOR;
 	struct drm_crtc_state *crtc_state;
 	int ret;
@@ -114,10 +117,13 @@ static int virtio_gpu_plane_atomic_check(struct drm_plane *plane,
 	/*
 	 * Ignore damage clips if the framebuffer attached to the plane's state
 	 * has changed since the last plane update (page-flip). In this case, a
-	 * full plane update should happen because uploads are done per-buffer.
+	 * full plane update should happen for dumb buffers because uploads are
+	 * done per-buffer. Rendered resources are already coherent on the host,
+	 * so preserve userspace's accumulated per-buffer damage for those.
 	 */
-	if (old_plane_state->fb != new_plane_state->fb)
-		new_plane_state->ignore_damage_clips = true;
+	bo = gem_to_virtio_gpu_obj(new_plane_state->fb->obj[0]);
+	new_plane_state->ignore_damage_clips =
+		old_plane_state->fb != new_plane_state->fb && bo->dumb;
 
 	crtc_state = drm_atomic_get_crtc_state(state,
 					       new_plane_state->crtc);
@@ -589,6 +595,7 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
 	struct drm_plane *plane;
 	const uint32_t *formats;
 	int nformats;
+	int ret;
 
 	if (type == DRM_PLANE_TYPE_CURSOR) {
 		formats = virtio_gpu_cursor_formats;
@@ -614,5 +621,17 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
 		drm_plane_create_blend_mode_property(plane,
 						     BIT(DRM_MODE_BLEND_PREMULTI));
 
+	if (type == DRM_PLANE_TYPE_CURSOR) {
+		/*
+		 * The cursor plane exposes a format with an alpha channel,
+		 * which requires a blend mode property. The host blends
+		 * premultiplied alpha, matching the property's default.
+		 */
+		ret = drm_plane_create_blend_mode_property(plane,
+							   BIT(DRM_MODE_BLEND_PREMULTI));
+		if (ret)
+			return ERR_PTR(ret);
+	}
+
 	return plane;
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
index 149e6bcb5878c3..ebf471044d06c0 100644
--- a/drivers/gpu/drm/virtio/virtgpu_prime.c
+++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
@@ -349,7 +349,7 @@ struct drm_gem_object *virtgpu_gem_prime_import(struct drm_device *dev,
 		}
 	}
 
-	if (!vgdev->has_resource_blob)
+	if (!vgdev->has_resource_blob || vgdev->has_virgl_3d)
 		return drm_gem_prime_import(dev, buf);
 
 	bo = kzalloc_obj(*bo);
diff --git a/drivers/gpu/drm/virtio/virtgpu_submit.c b/drivers/gpu/drm/virtio/virtgpu_submit.c
index 32cb1e4aa4258f..3d35326dd90406 100644
--- a/drivers/gpu/drm/virtio/virtgpu_submit.c
+++ b/drivers/gpu/drm/virtio/virtgpu_submit.c
@@ -389,10 +389,13 @@ static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit,
 	if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) ||
 	    exbuf->num_out_syncobjs ||
 	    exbuf->num_bo_handles ||
-	    drm_fence_event)
+	    drm_fence_event) {
 		out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx);
-	else
+		if (!out_fence)
+			return -ENOMEM;
+	} else {
 		out_fence = NULL;
+	}
 
 	if (drm_fence_event) {
 		err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx);
@@ -538,6 +541,10 @@ int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
 	virtio_gpu_process_post_deps(&submit);
 	virtio_gpu_complete_submit(&submit);
 cleanup:
+	if (ret && submit.out_fence && submit.out_fence->e) {
+		drm_event_cancel_free(dev, &submit.out_fence->e->base);
+		submit.out_fence->e = NULL;
+	}
 	virtio_gpu_cleanup_submit(&submit);
 
 	return ret;
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index c02c03c10d92e3..d99fb9e326e81e 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -256,6 +256,33 @@ void virtio_gpu_dequeue_ctrl_func(struct work_struct *work)
 	} while (!virtqueue_enable_cb(vgdev->ctrlq.vq));
 	spin_unlock(&vgdev->ctrlq.qlock);
 
+	/*
+	 * Sync guest-bound transfers before signalling anything, so that a
+	 * waiter cannot read the backing pages while what the device wrote is
+	 * still in a bounce buffer. This cannot be folded into the loop below:
+	 * virtio_gpu_fence_event_process() also signals every earlier fence in
+	 * the same context, so any entry there may signal this entry's fence.
+	 */
+	list_for_each_entry(entry, &reclaim_list, list) {
+		if (entry->sync_for_cpu) {
+			struct virtio_gpu_object *bo =
+				gem_to_virtio_gpu_obj(entry->objs->objs[0]);
+
+			dma_sync_sgtable_for_cpu(vgdev->vdev->dev.parent,
+						 bo->base.sgt, DMA_FROM_DEVICE);
+			/*
+			 * Release, so a transfer the other way that skips its
+			 * wait on the strength of this cannot go on to read
+			 * the backing pages before the sync above is visible.
+			 * Nothing orders the two otherwise: where the mapping
+			 * bounces on a coherent device the sync is a plain
+			 * copy, and dma_direct_sync_sg_for_cpu() emits its
+			 * barrier only for the non-coherent case.
+			 */
+			smp_store_release(&bo->from_host_pending, false);
+		}
+	}
+
 	list_for_each_entry(entry, &reclaim_list, list) {
 		resp = (struct virtio_gpu_ctrl_hdr *)entry->resp_buf;
 
@@ -1278,12 +1305,31 @@ void virtio_gpu_cmd_transfer_from_host_3d(struct virtio_gpu_device *vgdev,
 	struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]);
 	struct virtio_gpu_transfer_host_3d *cmd_p;
 	struct virtio_gpu_vbuffer *vbuf;
+	bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev);
 
 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
 	memset(cmd_p, 0, sizeof(*cmd_p));
 
 	vbuf->objs = objs;
 
+	if (virtio_gpu_is_shmem(bo) && use_dma_api) {
+		/*
+		 * The device writes only the requested box, so prime the
+		 * mapping with the current contents: otherwise the sync on
+		 * completion would hand back whatever a bounce buffer held for
+		 * the regions the device does not touch.
+		 */
+		dma_sync_sgtable_for_device(vgdev->vdev->dev.parent,
+					    bo->base.sgt, DMA_TO_DEVICE);
+		vbuf->sync_for_cpu = true;
+		/*
+		 * Set under the reservation the caller holds, so a transfer
+		 * the other way cannot miss it and push the guest pages into
+		 * the mapping while the device still owns it.
+		 */
+		WRITE_ONCE(bo->from_host_pending, true);
+	}
+
 	cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D);
 	cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id);
 	cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle);
diff --git a/drivers/gpu/drm/virtio/virtgpu_vram.c b/drivers/gpu/drm/virtio/virtgpu_vram.c
index 5b4a3ab81cd5b4..01241ce4d07cdc 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vram.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vram.c
@@ -215,16 +215,12 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev,
 
 	/* Create fake offset */
 	ret = drm_gem_create_mmap_offset(obj);
-	if (ret) {
-		kfree(vram);
-		return ret;
-	}
+	if (ret)
+		goto err_release_obj;
 
 	ret = virtio_gpu_resource_id_get(vgdev, &vram->base.hw_res_handle);
-	if (ret) {
-		kfree(vram);
-		return ret;
-	}
+	if (ret)
+		goto err_release_obj;
 
 	virtio_gpu_cmd_resource_create_blob(vgdev, &vram->base, params, NULL,
 					    0);
@@ -240,6 +236,11 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev,
 
 	*bo_ptr = &vram->base;
 	return 0;
+
+err_release_obj:
+	drm_gem_object_release(obj);
+	kfree(vram);
+	return ret;
 }
 
 void virtio_gpu_vram_map_deferred(struct virtio_gpu_object_vram *vram)
diff --git a/drivers/gpu/drm/vkms/tests/gen_yuv_conversion.py b/drivers/gpu/drm/vkms/tests/gen_yuv_conversion.py
new file mode 100755
index 00000000000000..f4d226137cc2f4
--- /dev/null
+++ b/drivers/gpu/drm/vkms/tests/gen_yuv_conversion.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0+
+"""
+Derive the VKMS YCbCr -> RGB conversion constants and the vkms-format KUnit
+reference values.
+
+It produces:
+  1. The S31.32 fixed-point conversion matrices used in
+     drivers/gpu/drm/vkms/vkms_formats.c and asserted by
+     vkms_format_test_conversion_matrix.
+  2. The 16-bit reference YCbCr inputs for the limited-range
+     yuv_u16_to_argb_u16 round-trip cases (the values changed by the fix).
+
+Requires numpy.
+"""
+
+import numpy as np
+
+FP = 1 << 32                       # S31.32 scale: fixed = round(coeff * 2^32)
+
+# ITU-R luma weights (Kr, Kb); Kg = 1 - Kr - Kb.
+ENCODINGS = {
+    "BT601":  (0.299,  0.114),
+    "BT709":  (0.2126, 0.0722),
+    "BT2020": (0.2627, 0.0593),    # non-constant luminance
+}
+
+# 8-bit studio-range levels: luma 16..235 (range 219), chroma 16..240
+# (range 224, neutral 128); full-range maximum is 2^n - 1 = 255.
+FULL_MAX = 255
+Y_RANGE  = 235 - 16                # 219
+C_RANGE  = 240 - 16                # 224
+
+# 16-bit RGB test colors.
+COLORS = {
+    "white": (0xffff, 0xffff, 0xffff),
+    "gray":  (0x8080, 0x8080, 0x8080),
+    "black": (0x0000, 0x0000, 0x0000),
+    "red":   (0xffff, 0x0000, 0x0000),
+    "green": (0x0000, 0xffff, 0x0000),
+    "blue":  (0x0000, 0x0000, 0xffff),
+}
+
+
+def rgb_matrix(kr, kb, limited):
+    """Normalized YCbCr -> RGB (Y in [0,1], Cb/Cr in [-0.5, 0.5])."""
+    kg = 1.0 - kr - kb
+    # Standard full-range relations.
+    m = [[1.0,  0.0,                     2 * (1 - kr)],
+         [1.0, -2 * (1 - kb) * kb / kg, -2 * (1 - kr) * kr / kg],
+         [1.0,  2 * (1 - kb),            0.0]]
+    if limited:
+        # Expand studio input range to full range: luma by 255/219,
+        # chroma by 255/224 (relative to full-range maximum 2^n - 1).
+        ys, cs = FULL_MAX / Y_RANGE, FULL_MAX / C_RANGE
+        m = [[r[0] * ys, r[1] * cs, r[2] * cs] for r in m]
+    return m
+
+
+def ref_yuv(m, y_offset):
+    """16-bit YCbCr that VKMS decodes back to each RGB color.
+
+    VKMS decode: [R,G,B] = M . [Y - y_offset*257, Cb - 128*257, Cr - 128*257]
+    (8-bit offsets lifted to 16-bit by *257 = 65535/255), so invert and
+    re-add the offsets.
+    """
+    inv = np.linalg.inv(np.array(m, float))
+    off = np.array([y_offset * 257, 128 * 257, 128 * 257])
+    return {name: tuple(int(min(max(round(v), 0), 0xffff))
+                        for v in inv @ np.array(rgb, float) + off)
+            for name, rgb in COLORS.items()}
+
+print(f"== RGB REFERENCE VALUES ==")
+for name, rgb in COLORS.items():
+    print(f"    {name:6} {{ {rgb[0]:#06x}, {rgb[1]:#06x}, {rgb[2]:#06x} }}")
+print()
+
+for enc, (kr, kb) in ENCODINGS.items():
+    for limited in (False, True):
+        m = rgb_matrix(kr, kb, limited)
+        y_offset = 16 if limited else 0
+        print(f"== {enc} {'LIMITED' if limited else 'FULL'} (y_offset={y_offset}) ==")
+        for row in m:
+            print("    { " + ", ".join(str(round(v * FP)) for v in row) + " },")
+        for name, yuv in ref_yuv(m, y_offset).items():
+            print(f"    {name:6} {{ {yuv[0]:#06x}, {yuv[1]:#06x}, {yuv[2]:#06x} }}")
+        print()
diff --git a/drivers/gpu/drm/vkms/tests/vkms_format_test.c b/drivers/gpu/drm/vkms/tests/vkms_format_test.c
index a7788fbc45dcc9..bbb1ad18c99ffc 100644
--- a/drivers/gpu/drm/vkms/tests/vkms_format_test.c
+++ b/drivers/gpu/drm/vkms/tests/vkms_format_test.c
@@ -54,6 +54,8 @@ struct yuv_u16_to_argb_u16_case {
  * The YUV color representation were acquired via the colour python framework.
  * Below are the function calls used for generating each case.
  *
+ * The limited-range cases are generated by gen_yuv_conversion.py.
+ *
  * For more information got to the docs:
  * https://colour.readthedocs.io/en/master/generated/colour.RGB_to_YCbCr.html
  */
@@ -101,12 +103,12 @@ static struct yuv_u16_to_argb_u16_case yuv_u16_to_argb_u16_cases[] = {
 		.range = DRM_COLOR_YCBCR_LIMITED_RANGE,
 		.n_colors = 6,
 		.colors = {
-			{ "white", { 0xeb00, 0x8000, 0x8000 }, { 0xffff, 0xffff, 0xffff, 0xffff }},
-			{ "gray",  { 0x7dee, 0x8000, 0x8000 }, { 0xffff, 0x8080, 0x8080, 0x8080 }},
-			{ "black", { 0x1000, 0x8000, 0x8000 }, { 0xffff, 0x0000, 0x0000, 0x0000 }},
-			{ "red",   { 0x517b, 0x5a34, 0xf000 }, { 0xffff, 0xffff, 0x0000, 0x0000 }},
-			{ "green", { 0x908e, 0x35cc, 0x2237 }, { 0xffff, 0x0000, 0xffff, 0x0000 }},
-			{ "blue",  { 0x28f7, 0xf000, 0x6dc9 }, { 0xffff, 0x0000, 0x0000, 0xffff }},
+			{ "white", { 0xebeb, 0x8080, 0x8080 }, { 0xffff, 0xffff, 0xffff, 0xffff }},
+			{ "gray",  { 0x7e6c, 0x8080, 0x8080 }, { 0xffff, 0x8080, 0x8080, 0x8080 }},
+			{ "black", { 0x1010, 0x8080, 0x8080 }, { 0xffff, 0x0000, 0x0000, 0x0000 }},
+			{ "red",   { 0x51cd, 0x5a8e, 0xf0f0 }, { 0xffff, 0xffff, 0x0000, 0x0000 }},
+			{ "green", { 0x911e, 0x3602, 0x2259 }, { 0xffff, 0x0000, 0xffff, 0x0000 }},
+			{ "blue",  { 0x2920, 0xf0f0, 0x6e37 }, { 0xffff, 0x0000, 0x0000, 0xffff }},
 		}
 	},
 	/*
@@ -151,12 +153,12 @@ static struct yuv_u16_to_argb_u16_case yuv_u16_to_argb_u16_cases[] = {
 		.range = DRM_COLOR_YCBCR_LIMITED_RANGE,
 		.n_colors = 6,
 		.colors = {
-			{ "white", { 0xeb00, 0x8000, 0x8000 }, { 0xffff, 0xffff, 0xffff, 0xffff }},
-			{ "gray",  { 0x7dee, 0x8000, 0x8000 }, { 0xffff, 0x8080, 0x8080, 0x8080 }},
-			{ "black", { 0x1000, 0x8000, 0x8000 }, { 0xffff, 0x0000, 0x0000, 0x0000 }},
-			{ "red",   { 0x3e8f, 0x6656, 0xf000 }, { 0xffff, 0xffff, 0x0000, 0x0000 }},
-			{ "green", { 0xaca1, 0x29aa, 0x1a45 }, { 0xffff, 0x0000, 0xffff, 0x0000 }},
-			{ "blue",  { 0x1fd0, 0xf000, 0x75bb }, { 0xffff, 0x0000, 0x0000, 0xffff }},
+			{ "white", { 0xebeb, 0x8080, 0x8080 }, { 0xffff, 0xffff, 0xffff, 0xffff }},
+			{ "gray",  { 0x7e6c, 0x8080, 0x8080 }, { 0xffff, 0x8080, 0x8080, 0x8080 }},
+			{ "black", { 0x1010, 0x8080, 0x8080 }, { 0xffff, 0x0000, 0x0000, 0x0000 }},
+			{ "red",   { 0x3ece, 0x66bc, 0xf0f0 }, { 0xffff, 0xffff, 0x0000, 0x0000 }},
+			{ "green", { 0xad4e, 0x29d4, 0x1a5f }, { 0xffff, 0x0000, 0xffff, 0x0000 }},
+			{ "blue",  { 0x1ff0, 0xf0f0, 0x7631 }, { 0xffff, 0x0000, 0x0000, 0xffff }},
 		}
 	},
 	/*
@@ -201,12 +203,12 @@ static struct yuv_u16_to_argb_u16_case yuv_u16_to_argb_u16_cases[] = {
 		.range = DRM_COLOR_YCBCR_LIMITED_RANGE,
 		.n_colors = 6,
 		.colors = {
-			{ "white", { 0xeb00, 0x8000, 0x8000 }, { 0xffff, 0xffff, 0xffff, 0xffff }},
-			{ "gray",  { 0x7dee, 0x8000, 0x8000 }, { 0xffff, 0x8080, 0x8080, 0x8080 }},
-			{ "black", { 0x1000, 0x8000, 0x8000 }, { 0xffff, 0x0000, 0x0000, 0x0000 }},
-			{ "red",   { 0x4988, 0x60b9, 0xf000 }, { 0xffff, 0xffff, 0x0000, 0x0000 }},
-			{ "green", { 0xa47b, 0x2f47, 0x1902 }, { 0xffff, 0x0000, 0xffff, 0x0000 }},
-			{ "blue",  { 0x1cfd, 0xf000, 0x76fe }, { 0xffff, 0x0000, 0x0000, 0xffff }},
+			{ "white", { 0xebeb, 0x8080, 0x8080 }, { 0xffff, 0xffff, 0xffff, 0xffff }},
+			{ "gray",  { 0x7e6c, 0x8080, 0x8080 }, { 0xffff, 0x8080, 0x8080, 0x8080 }},
+			{ "black", { 0x1010, 0x8080, 0x8080 }, { 0xffff, 0x0000, 0x0000, 0x0000 }},
+			{ "red",   { 0x49d2, 0x611a, 0xf0f0 }, { 0xffff, 0xffff, 0x0000, 0x0000 }},
+			{ "green", { 0xa520, 0x2f76, 0x191b }, { 0xffff, 0x0000, 0xffff, 0x0000 }},
+			{ "blue",  { 0x1d1a, 0xf0f0, 0x7775 }, { 0xffff, 0x0000, 0x0000, 0xffff }},
 		}
 	},
 };
@@ -232,7 +234,7 @@ static void vkms_format_test_yuv_u16_to_argb_u16(struct kunit *test)
 		struct conversion_matrix matrix;
 
 		get_conversion_matrix_to_argb_u16
-			(DRM_FORMAT_NV12, param->encoding, param->range, &matrix);
+			(DRM_FORMAT_NV12, param->encoding, param->range, false, &matrix);
 
 		argb = argb_u16_from_yuv161616(&matrix, color->yuv.y, color->yuv.u,
 					       color->yuv.v);
diff --git a/drivers/gpu/drm/vkms/vkms_colorop.c b/drivers/gpu/drm/vkms/vkms_colorop.c
index 071f3a8d2e7c4d..9424b1994239a6 100644
--- a/drivers/gpu/drm/vkms/vkms_colorop.c
+++ b/drivers/gpu/drm/vkms/vkms_colorop.c
@@ -12,11 +12,19 @@ static const u64 supported_tfs =
 	BIT(DRM_COLOROP_1D_CURVE_SRGB_EOTF) |
 	BIT(DRM_COLOROP_1D_CURVE_SRGB_INV_EOTF);
 
+static const u64 supported_fm =
+	BIT(DRM_COLOROP_FM_YCBCR601_FULL_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR601_LIMITED_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR709_FULL_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR709_LIMITED_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB) |
+	BIT(DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB);
+
 static const struct drm_colorop_funcs vkms_colorop_funcs = {
 	.destroy = drm_colorop_destroy,
 };
 
-#define MAX_COLOR_PIPELINE_OPS 4
+#define MAX_COLOR_PIPELINE_OPS 5
 
 static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_prop_enum_list *list)
 {
@@ -27,7 +35,25 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 
 	memset(ops, 0, sizeof(ops));
 
-	/* 1st op: 1d curve */
+	/* 1st op: Fixed Matrix (YUV to RGB) */
+	ops[i] = kzalloc_obj(*ops[i]);
+	if (!ops[i]) {
+		drm_err(dev, "KMS: Failed to allocate colorop\n");
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	ret = drm_plane_colorop_fixed_matrix_init(dev, ops[i], plane, &vkms_colorop_funcs,
+						  supported_fm,
+						  DRM_COLOROP_FLAG_ALLOW_BYPASS);
+	if (ret)
+		goto cleanup;
+
+	list->type = ops[i]->base.id;
+
+	i++;
+
+	/* 2nd op: 1d curve */
 	ops[i] = kzalloc_obj(*ops[i]);
 	if (!ops[i]) {
 		drm_err(dev, "KMS: Failed to allocate colorop\n");
@@ -41,11 +67,11 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 	if (ret)
 		goto cleanup;
 
-	list->type = ops[i]->base.id;
+	drm_colorop_set_next_property(ops[i - 1], ops[i]);
 
 	i++;
 
-	/* 2nd op: 3x4 matrix */
+	/* 3rd op: 3x4 matrix */
 	ops[i] = kzalloc_obj(*ops[i]);
 	if (!ops[i]) {
 		drm_err(dev, "KMS: Failed to allocate colorop\n");
@@ -62,7 +88,7 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 
 	i++;
 
-	/* 3rd op: 3x4 matrix */
+	/* 4th op: 3x4 matrix */
 	ops[i] = kzalloc_obj(*ops[i]);
 	if (!ops[i]) {
 		drm_err(dev, "KMS: Failed to allocate colorop\n");
@@ -79,7 +105,7 @@ static int vkms_initialize_color_pipeline(struct drm_plane *plane, struct drm_pr
 
 	i++;
 
-	/* 4th op: 1d curve */
+	/* 5th op: 1d curve */
 	ops[i] = kzalloc_obj(*ops[i]);
 	if (!ops[i]) {
 		drm_err(dev, "KMS: Failed to allocate colorop\n");
diff --git a/drivers/gpu/drm/vkms/vkms_composer.c b/drivers/gpu/drm/vkms/vkms_composer.c
index 83d217085ad091..0fc915a954badb 100644
--- a/drivers/gpu/drm/vkms/vkms_composer.c
+++ b/drivers/gpu/drm/vkms/vkms_composer.c
@@ -187,6 +187,12 @@ static void apply_colorop(struct pixel_argb_s32 *pixel, struct drm_colorop *colo
 		if (colorop_state->data)
 			apply_3x4_matrix(pixel,
 					 (struct drm_color_ctm_3x4 *)colorop_state->data->data);
+	} else if (colorop->type == DRM_COLOROP_FIXED_MATRIX) {
+		/*
+		 * Fixed Matrix is a no-op here. YUV-to-RGB conversion
+		 * is handled at format read time via the
+		 * conversion_matrix set in vkms_plane_atomic_update().
+		 */
 	}
 }
 
@@ -305,7 +311,7 @@ static void clamp_line_coordinates(enum pixel_read_direction direction,
 	/* By default the start points are correct */
 	*src_x_start = src_line->x1;
 	*src_y_start = src_line->y1;
-	*dst_x_start = current_plane->frame_info->dst.x1;
+	*dst_x_start = current_plane->frame_info.dst.x1;
 
 	/* Get the correct number of pixel to blend, it depends of the direction */
 	switch (direction) {
@@ -333,8 +339,8 @@ static void clamp_line_coordinates(enum pixel_read_direction direction,
 			*dst_x_start -= *src_x_start;
 			*src_x_start = 0;
 		}
-		if (*src_x_start + *pixel_count > current_plane->frame_info->fb->width)
-			*pixel_count = max(0, (int)current_plane->frame_info->fb->width -
+		if (*src_x_start + *pixel_count > current_plane->frame_info.fb->width)
+			*pixel_count = max(0, (int)current_plane->frame_info.fb->width -
 				*src_x_start);
 		break;
 	case READ_BOTTOM_TO_TOP:
@@ -344,8 +350,8 @@ static void clamp_line_coordinates(enum pixel_read_direction direction,
 			*dst_x_start -= *src_y_start;
 			*src_y_start = 0;
 		}
-		if (*src_y_start + *pixel_count > current_plane->frame_info->fb->height)
-			*pixel_count = max(0, (int)current_plane->frame_info->fb->height -
+		if (*src_y_start + *pixel_count > current_plane->frame_info.fb->height)
+			*pixel_count = max(0, (int)current_plane->frame_info.fb->height -
 				*src_y_start);
 		break;
 	}
@@ -368,8 +374,8 @@ static void blend_line(struct vkms_plane_state *current_plane, int y,
 	struct drm_rect dst_line, tmp_src, src_line;
 
 	/* Avoid rendering useless lines */
-	if (y < current_plane->frame_info->dst.y1 ||
-	    y >= current_plane->frame_info->dst.y2)
+	if (y < current_plane->frame_info.dst.y1 ||
+	    y >= current_plane->frame_info.dst.y2)
 		return;
 
 	/*
@@ -377,11 +383,11 @@ static void blend_line(struct vkms_plane_state *current_plane, int y,
 	 * destination framebuffer, and then drm_rect_* helpers are used to
 	 * compute the correct position into the source framebuffer.
 	 */
-	dst_line = DRM_RECT_INIT(current_plane->frame_info->dst.x1, y,
-				 drm_rect_width(&current_plane->frame_info->dst),
+	dst_line = DRM_RECT_INIT(current_plane->frame_info.dst.x1, y,
+				 drm_rect_width(&current_plane->frame_info.dst),
 				 1);
 
-	drm_rect_fp_to_int(&tmp_src, &current_plane->frame_info->src);
+	drm_rect_fp_to_int(&tmp_src, &current_plane->frame_info.src);
 
 	/*
 	 * [1]: Clamping src_line to the crtc_x_limit to avoid writing outside of
@@ -405,17 +411,17 @@ static void blend_line(struct vkms_plane_state *current_plane, int y,
 	 *   same size, but can be rotated).
 	 * - Apply the offset of the source rectangle to the coordinate.
 	 */
-	drm_rect_translate(&src_line, -current_plane->frame_info->dst.x1,
-			   -current_plane->frame_info->dst.y1);
+	drm_rect_translate(&src_line, -current_plane->frame_info.dst.x1,
+			   -current_plane->frame_info.dst.y1);
 	drm_rect_rotate_inv(&src_line, drm_rect_width(&tmp_src),
 			    drm_rect_height(&tmp_src),
-			    current_plane->frame_info->rotation);
+			    current_plane->frame_info.rotation);
 	drm_rect_translate(&src_line, tmp_src.x1, tmp_src.y1);
 
 	/* Get the correct reading direction in the source buffer. */
 
 	enum pixel_read_direction direction =
-		direction_for_rotation(current_plane->frame_info->rotation);
+		direction_for_rotation(current_plane->frame_info.rotation);
 
 	/* [2]: Compute and clamp the number of pixel to read */
 	clamp_line_coordinates(direction, current_plane, &src_line, &src_x_start, &src_y_start,
@@ -533,7 +539,7 @@ static int check_iosys_map(struct vkms_crtc_state *crtc_state)
 	u32 n_active_planes = crtc_state->num_active_planes;
 
 	for (size_t i = 0; i < n_active_planes; i++)
-		if (iosys_map_is_null(&plane_state[i]->frame_info->map[0]))
+		if (iosys_map_is_null(&plane_state[i]->frame_info.map[0]))
 			return -1;
 
 	return 0;
diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c
index 079abfba427d07..fe17f49305a9e3 100644
--- a/drivers/gpu/drm/vkms/vkms_crtc.c
+++ b/drivers/gpu/drm/vkms/vkms_crtc.c
@@ -86,22 +86,24 @@ static void vkms_atomic_crtc_destroy_state(struct drm_crtc *crtc,
 	kfree(vkms_state);
 }
 
-static void vkms_atomic_crtc_reset(struct drm_crtc *crtc)
+static struct drm_crtc_state *vkms_atomic_crtc_create_state(struct drm_crtc *crtc)
 {
-	struct vkms_crtc_state *vkms_state = kzalloc_obj(*vkms_state);
+	struct vkms_crtc_state *vkms_state;
+
+	vkms_state = kzalloc_obj(*vkms_state);
+	if (!vkms_state)
+		return ERR_PTR(-ENOMEM);
 
-	if (crtc->state)
-		vkms_atomic_crtc_destroy_state(crtc, crtc->state);
+	__drm_atomic_helper_crtc_state_init(&vkms_state->base, crtc);
+	INIT_WORK(&vkms_state->composer_work, vkms_composer_worker);
 
-	__drm_atomic_helper_crtc_reset(crtc, &vkms_state->base);
-	if (vkms_state)
-		INIT_WORK(&vkms_state->composer_work, vkms_composer_worker);
+	return &vkms_state->base;
 }
 
 static const struct drm_crtc_funcs vkms_crtc_funcs = {
 	.set_config             = drm_atomic_helper_set_config,
 	.page_flip              = drm_atomic_helper_page_flip,
-	.reset                  = vkms_atomic_crtc_reset,
+	.atomic_create_state = vkms_atomic_crtc_create_state,
 	.atomic_duplicate_state = vkms_atomic_crtc_duplicate_state,
 	.atomic_destroy_state   = vkms_atomic_crtc_destroy_state,
 	DRM_CRTC_VBLANK_TIMER_FUNCS,
diff --git a/drivers/gpu/drm/vkms/vkms_drv.h b/drivers/gpu/drm/vkms/vkms_drv.h
index 0933e4ce0ff031..381483aa4fb05c 100644
--- a/drivers/gpu/drm/vkms/vkms_drv.h
+++ b/drivers/gpu/drm/vkms/vkms_drv.h
@@ -149,7 +149,7 @@ struct conversion_matrix {
  */
 struct vkms_plane_state {
 	struct drm_shadow_plane_state base;
-	struct vkms_frame_info *frame_info;
+	struct vkms_frame_info frame_info;
 	pixel_read_line_t pixel_read_line;
 	struct conversion_matrix conversion_matrix;
 };
diff --git a/drivers/gpu/drm/vkms/vkms_formats.c b/drivers/gpu/drm/vkms/vkms_formats.c
index dfb8e13cba87a9..7f1f29b589e8df 100644
--- a/drivers/gpu/drm/vkms/vkms_formats.c
+++ b/drivers/gpu/drm/vkms/vkms_formats.c
@@ -321,10 +321,10 @@ static void function_name(const struct vkms_plane_state *plane, int x_start,			\
 			      struct pixel_argb_u16 out_pixel[])				\
 {												\
 	struct pixel_argb_u16 *end = out_pixel + count;						\
-	int step = get_block_step_bytes(plane->frame_info->fb, direction, 0);			\
+	int step = get_block_step_bytes(plane->frame_info.fb, direction, 0);			\
 	u8 *src_pixels;										\
 												\
-	packed_pixels_addr_1x1(plane->frame_info, x_start, y_start, 0, &src_pixels);		\
+	packed_pixels_addr_1x1(&plane->frame_info, x_start, y_start, 0, &src_pixels);		\
 												\
 	while (out_pixel < end) {								\
 		pixel_type *(pixel_name) = (pixel_type *)src_pixels;				\
@@ -379,16 +379,16 @@ static void Rx_read_line(const struct vkms_plane_state *plane, int x_start,
 			 struct pixel_argb_u16 out_pixel[])
 {
 	struct pixel_argb_u16 *end = out_pixel + count;
-	int bits_per_pixel = drm_format_info_bpp(plane->frame_info->fb->format, 0);
+	int bits_per_pixel = drm_format_info_bpp(plane->frame_info.fb->format, 0);
 	u8 *src_pixels;
 	int rem_x, rem_y;
 
-	WARN_ONCE(drm_format_info_block_height(plane->frame_info->fb->format, 0) != 1,
+	WARN_ONCE(drm_format_info_block_height(plane->frame_info.fb->format, 0) != 1,
 		  "%s() only support formats with block_h == 1", __func__);
 
-	packed_pixels_addr(plane->frame_info, x_start, y_start, 0, &src_pixels, &rem_x, &rem_y);
+	packed_pixels_addr(&plane->frame_info, x_start, y_start, 0, &src_pixels, &rem_x, &rem_y);
 	int bit_offset = (8 - bits_per_pixel) - rem_x * bits_per_pixel;
-	int step = get_block_step_bytes(plane->frame_info->fb, direction, 0);
+	int step = get_block_step_bytes(plane->frame_info.fb, direction, 0);
 	int mask = (0x1 << bits_per_pixel) - 1;
 	int lum_per_level = 0xFFFF / mask;
 
@@ -503,15 +503,15 @@ static void function_name(const struct vkms_plane_state *plane, int x_start,			\
 	u8 *plane_1;										\
 	u8 *plane_2;										\
 												\
-	packed_pixels_addr_1x1(plane->frame_info, x_start, y_start, 0,				\
+	packed_pixels_addr_1x1(&plane->frame_info, x_start, y_start, 0,				\
 			       &plane_1);							\
-	packed_pixels_addr_1x1(plane->frame_info,						\
-			       x_start / plane->frame_info->fb->format->hsub,			\
-			       y_start / plane->frame_info->fb->format->vsub, 1,		\
+	packed_pixels_addr_1x1(&plane->frame_info,						\
+			       x_start / plane->frame_info.fb->format->hsub,			\
+			       y_start / plane->frame_info.fb->format->vsub, 1,			\
 			       &plane_2);							\
-	int step_1 = get_block_step_bytes(plane->frame_info->fb, direction, 0);			\
-	int step_2 = get_block_step_bytes(plane->frame_info->fb, direction, 1);			\
-	int subsampling = get_subsampling(plane->frame_info->fb->format, direction);		\
+	int step_1 = get_block_step_bytes(plane->frame_info.fb, direction, 0);			\
+	int step_2 = get_block_step_bytes(plane->frame_info.fb, direction, 1);			\
+	int subsampling = get_subsampling(plane->frame_info.fb->format, direction);		\
 	int subsampling_offset = get_subsampling_offset(direction, x_start, y_start);		\
 	const struct conversion_matrix *conversion_matrix = &plane->conversion_matrix;		\
 												\
@@ -548,20 +548,20 @@ static void planar_yuv_read_line(const struct vkms_plane_state *plane, int x_sta
 	u8 *channel_1_plane;
 	u8 *channel_2_plane;
 
-	packed_pixels_addr_1x1(plane->frame_info, x_start, y_start, 0,
+	packed_pixels_addr_1x1(&plane->frame_info, x_start, y_start, 0,
 			       &y_plane);
-	packed_pixels_addr_1x1(plane->frame_info,
-			       x_start / plane->frame_info->fb->format->hsub,
-			       y_start / plane->frame_info->fb->format->vsub, 1,
+	packed_pixels_addr_1x1(&plane->frame_info,
+			       x_start / plane->frame_info.fb->format->hsub,
+			       y_start / plane->frame_info.fb->format->vsub, 1,
 			       &channel_1_plane);
-	packed_pixels_addr_1x1(plane->frame_info,
-			       x_start / plane->frame_info->fb->format->hsub,
-			       y_start / plane->frame_info->fb->format->vsub, 2,
+	packed_pixels_addr_1x1(&plane->frame_info,
+			       x_start / plane->frame_info.fb->format->hsub,
+			       y_start / plane->frame_info.fb->format->vsub, 2,
 			       &channel_2_plane);
-	int step_y = get_block_step_bytes(plane->frame_info->fb, direction, 0);
-	int step_channel_1 = get_block_step_bytes(plane->frame_info->fb, direction, 1);
-	int step_channel_2 = get_block_step_bytes(plane->frame_info->fb, direction, 2);
-	int subsampling = get_subsampling(plane->frame_info->fb->format, direction);
+	int step_y = get_block_step_bytes(plane->frame_info.fb, direction, 0);
+	int step_channel_1 = get_block_step_bytes(plane->frame_info.fb, direction, 1);
+	int step_channel_2 = get_block_step_bytes(plane->frame_info.fb, direction, 2);
+	int subsampling = get_subsampling(plane->frame_info.fb->format, direction);
 	int subsampling_offset = get_subsampling_offset(direction, x_start, y_start);
 	const struct conversion_matrix *conversion_matrix = &plane->conversion_matrix;
 
@@ -788,15 +788,36 @@ static const struct conversion_matrix yuv_bt601_full = {
 };
 
 /*
- * numpy.around(colour.matrix_YCbCr(K=colour.WEIGHTS_YCBCR["ITU-R BT.601"],
- *                                  is_legal = True,
- *                                  bits = 8) * 2**32).astype(int)
+ * BT.601 limited-/studio-range YCbCr to full-range RGB.
+ *
+ * The coefficients are derived as follows:
+ *
+ *  1. Take the standard ITU-R YCbCr -> RGB relations for luma weights
+ *     Kr, Kb (Kg = 1 - Kr - Kb), with Y in [0, 1] and Cb, Cr in
+ *     [-0.5, 0.5]. For BT.601 Kr = 0.299 and Kb = 0.114:
+ *
+ *       R = Y                                + 2 * (1 - Kr)          * Cr
+ *       G = Y - 2 * (1 - Kb) * Kb / Kg * Cb - 2 * (1 - Kr) * Kr / Kg * Cr
+ *       B = Y + 2 * (1 - Kb)          * Cb
+ *
+ *     These are exactly the yuv_bt601_full coefficients above.
+ *
+ *  2. Expand the studio input range to full range, relative to a
+ *     full-range maximum of 2^n - 1 (255 for 8-bit): the luma
+ *     coefficient is scaled by 255/(235 - 16) and the chroma
+ *     coefficients by 255/(240 - 16). This matches the DRM UAPI
+ *     definition and IGT's igt_ycbcr_to_rgb_matrix(). Note this differs
+ *     from colour.matrix_YCbCr(is_legal=True), which normalises by 2^n
+ *     and is thus off by a factor of 256/255.
+ *
+ *  3. Convert each coefficient to S31.32 fixed point, i.e.
+ *     round(coeff * 2^32).
  */
 static const struct conversion_matrix yuv_bt601_limited = {
 	.matrix = {
-		{ 5020601039, 0,           6881764740 },
-		{ 5020601039, -1689204679, -3505362278 },
-		{ 5020601039, 8697922339,  0 },
+		{ 5000989317, 0,           6854882848 },
+		{ 5000989317, -1682606224, -3491669458 },
+		{ 5000989317, 8663946082,  0 },
 	},
 	.y_offset = 16,
 };
@@ -816,15 +837,14 @@ static const struct conversion_matrix yuv_bt709_full = {
 };
 
 /*
- * numpy.around(colour.matrix_YCbCr(K=colour.WEIGHTS_YCBCR["ITU-R BT.709"],
- *                                  is_legal = True,
- *                                  bits = 8) * 2**32).astype(int)
+ * BT.709 limited-range YCbCr to full-range RGB (Kr = 0.2126, Kb = 0.0722).
+ * Derived as described for yuv_bt601_limited.
  */
 static const struct conversion_matrix yuv_bt709_limited = {
 	.matrix = {
-		{ 5020601039, 0,          7729959424 },
-		{ 5020601039, -919487572, -2297803934 },
-		{ 5020601039, 9108275786, 0 },
+		{ 5000989317, 0,          7699764272 },
+		{ 5000989317, -915895824, -2288828138 },
+		{ 5000989317, 9072696586, 0 },
 	},
 	.y_offset = 16,
 };
@@ -844,15 +864,14 @@ static const struct conversion_matrix yuv_bt2020_full = {
 };
 
 /*
- * numpy.around(colour.matrix_YCbCr(K=colour.WEIGHTS_YCBCR["ITU-R BT.2020"],
- *                                  is_legal = True,
- *                                  bits = 8) * 2**32).astype(int)
+ * BT.2020 non-constant-luminance limited-range YCbCr to full-range RGB
+ * (Kr = 0.2627, Kb = 0.0593). Derived as described for yuv_bt601_limited.
  */
 static const struct conversion_matrix yuv_bt2020_limited = {
 	.matrix = {
-		{ 5020601039, 0,          7238124312 },
-		{ 5020601039, -807714626, -2804506279 },
-		{ 5020601039, 9234915964, 0 },
+		{ 5000989317, 0,          7209850391 },
+		{ 5000989317, -804559491, -2793551177 },
+		{ 5000989317, 9198842076, 0 },
 	},
 	.y_offset = 16,
 };
@@ -876,16 +895,25 @@ static void swap_uv_columns(struct conversion_matrix *matrix)
  * @format: DRM_FORMAT_* value for which to obtain a conversion function (see [drm_fourcc.h])
  * @encoding: DRM_COLOR_* value for which to obtain a conversion matrix
  * @range: DRM_COLOR_*_RANGE value for which to obtain a conversion matrix
+ * @bypass: If true, return an identity (no-op) matrix that passes the samples
+ *          through unchanged, ignoring @encoding and @range. Used when a fixed
+ *          matrix colorop is present but bypassed.
  * @matrix: Pointer to store the value into
  */
 void get_conversion_matrix_to_argb_u16(u32 format,
 				       enum drm_color_encoding encoding,
 				       enum drm_color_range range,
+				       bool bypass,
 				       struct conversion_matrix *matrix)
 {
 	const struct conversion_matrix *matrix_to_copy;
 	bool limited_range;
 
+	if (bypass) {
+		memcpy(matrix, &no_operation, sizeof(no_operation));
+		return;
+	}
+
 	switch (range) {
 	case DRM_COLOR_YCBCR_LIMITED_RANGE:
 		limited_range = true;
diff --git a/drivers/gpu/drm/vkms/vkms_formats.h b/drivers/gpu/drm/vkms/vkms_formats.h
index eeb208cdd6b1be..d969cc669a5e63 100644
--- a/drivers/gpu/drm/vkms/vkms_formats.h
+++ b/drivers/gpu/drm/vkms/vkms_formats.h
@@ -10,7 +10,7 @@ pixel_read_line_t get_pixel_read_line_function(u32 format);
 pixel_write_t get_pixel_write_function(u32 format);
 
 void get_conversion_matrix_to_argb_u16(u32 format, enum drm_color_encoding encoding,
-				       enum drm_color_range range,
+				       enum drm_color_range range, bool bypass,
 				       struct conversion_matrix *matrix);
 
 #if IS_ENABLED(CONFIG_KUNIT)
diff --git a/drivers/gpu/drm/vkms/vkms_plane.c b/drivers/gpu/drm/vkms/vkms_plane.c
index 68cb2a3335e642..fd3d34dcf0fe5b 100644
--- a/drivers/gpu/drm/vkms/vkms_plane.c
+++ b/drivers/gpu/drm/vkms/vkms_plane.c
@@ -6,6 +6,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
+#include <drm/drm_colorop.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -54,21 +55,11 @@ static struct drm_plane_state *
 vkms_plane_duplicate_state(struct drm_plane *plane)
 {
 	struct vkms_plane_state *vkms_state;
-	struct vkms_frame_info *frame_info;
 
 	vkms_state = kzalloc_obj(*vkms_state);
 	if (!vkms_state)
 		return NULL;
 
-	frame_info = kzalloc_obj(*frame_info);
-	if (!frame_info) {
-		DRM_DEBUG_KMS("Couldn't allocate frame_info\n");
-		kfree(vkms_state);
-		return NULL;
-	}
-
-	vkms_state->frame_info = frame_info;
-
 	__drm_gem_duplicate_shadow_plane_state(plane, &vkms_state->base);
 
 	return &vkms_state->base.base;
@@ -80,43 +71,37 @@ static void vkms_plane_destroy_state(struct drm_plane *plane,
 	struct vkms_plane_state *vkms_state = to_vkms_plane_state(old_state);
 	struct drm_crtc *crtc = vkms_state->base.base.crtc;
 
-	if (crtc && vkms_state->frame_info->fb) {
+	if (crtc && vkms_state->frame_info.fb) {
 		/* dropping the reference we acquired in
 		 * vkms_primary_plane_update()
 		 */
-		if (drm_framebuffer_read_refcount(vkms_state->frame_info->fb))
-			drm_framebuffer_put(vkms_state->frame_info->fb);
+		if (drm_framebuffer_read_refcount(vkms_state->frame_info.fb))
+			drm_framebuffer_put(vkms_state->frame_info.fb);
 	}
 
-	kfree(vkms_state->frame_info);
-	vkms_state->frame_info = NULL;
-
 	__drm_gem_destroy_shadow_plane_state(&vkms_state->base);
 	kfree(vkms_state);
 }
 
-static void vkms_plane_reset(struct drm_plane *plane)
+static struct drm_plane_state *vkms_plane_create_state(struct drm_plane *plane)
 {
 	struct vkms_plane_state *vkms_state;
 
-	if (plane->state) {
-		vkms_plane_destroy_state(plane, plane->state);
-		plane->state = NULL; /* must be set to NULL here */
-	}
-
 	vkms_state = kzalloc_obj(*vkms_state);
 	if (!vkms_state) {
 		DRM_ERROR("Cannot allocate vkms_plane_state\n");
-		return;
+		return ERR_PTR(-ENOMEM);
 	}
 
-	__drm_gem_reset_shadow_plane(plane, &vkms_state->base);
+	__drm_gem_shadow_plane_state_init(plane, &vkms_state->base);
+
+	return &(&vkms_state->base)->base;
 }
 
 static const struct drm_plane_funcs vkms_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= vkms_plane_reset,
+	.atomic_create_state = vkms_plane_create_state,
 	.atomic_duplicate_state = vkms_plane_duplicate_state,
 	.atomic_destroy_state	= vkms_plane_destroy_state,
 };
@@ -131,6 +116,9 @@ static void vkms_plane_atomic_update(struct drm_plane *plane,
 	struct drm_framebuffer *fb = new_state->fb;
 	struct vkms_frame_info *frame_info;
 	u32 fmt;
+	enum drm_color_encoding encoding = new_state->color_encoding;
+	enum drm_color_range range = new_state->color_range;
+	bool bypass = false;
 
 	if (!new_state->crtc || !fb)
 		return;
@@ -139,7 +127,7 @@ static void vkms_plane_atomic_update(struct drm_plane *plane,
 	vkms_plane_state = to_vkms_plane_state(new_state);
 	shadow_plane_state = &vkms_plane_state->base;
 
-	frame_info = vkms_plane_state->frame_info;
+	frame_info = &vkms_plane_state->frame_info;
 	memcpy(&frame_info->src, &new_state->src, sizeof(struct drm_rect));
 	memcpy(&frame_info->dst, &new_state->dst, sizeof(struct drm_rect));
 	frame_info->fb = fb;
@@ -148,7 +136,49 @@ static void vkms_plane_atomic_update(struct drm_plane *plane,
 	frame_info->rotation = new_state->rotation;
 
 	vkms_plane_state->pixel_read_line = get_pixel_read_line_function(fmt);
-	get_conversion_matrix_to_argb_u16(fmt, new_state->color_encoding, new_state->color_range,
+
+	if (new_state->color_pipeline) {
+		struct drm_colorop *colorop = new_state->color_pipeline;
+		struct drm_colorop_state *colorop_state;
+
+		colorop_state = drm_atomic_get_new_colorop_state(state, colorop);
+		bypass = !colorop_state || colorop_state->bypass;
+
+		if (!bypass) {
+			switch (colorop_state->fixed_matrix_type) {
+			case DRM_COLOROP_FM_YCBCR601_FULL_RGB:
+				encoding = DRM_COLOR_YCBCR_BT601;
+				range = DRM_COLOR_YCBCR_FULL_RANGE;
+				break;
+			case DRM_COLOROP_FM_YCBCR601_LIMITED_RGB:
+				encoding = DRM_COLOR_YCBCR_BT601;
+				range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+				break;
+			case DRM_COLOROP_FM_YCBCR709_FULL_RGB:
+				encoding = DRM_COLOR_YCBCR_BT709;
+				range = DRM_COLOR_YCBCR_FULL_RANGE;
+				break;
+			case DRM_COLOROP_FM_YCBCR709_LIMITED_RGB:
+				encoding = DRM_COLOR_YCBCR_BT709;
+				range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+				break;
+			case DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB:
+				encoding = DRM_COLOR_YCBCR_BT2020;
+				range = DRM_COLOR_YCBCR_FULL_RANGE;
+				break;
+			case DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB:
+				encoding = DRM_COLOR_YCBCR_BT2020;
+				range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+				break;
+			default:
+				encoding = DRM_COLOR_YCBCR_BT709;
+				range = DRM_COLOR_YCBCR_LIMITED_RANGE;
+				break;
+			}
+		}
+	}
+
+	get_conversion_matrix_to_argb_u16(fmt, encoding, range, bypass,
 					  &vkms_plane_state->conversion_matrix);
 }
 
@@ -175,6 +205,13 @@ static int vkms_plane_atomic_check(struct drm_plane *plane,
 	if (ret != 0)
 		return ret;
 
+	if (new_plane_state->color_pipeline && new_plane_state->fb->format->is_yuv) {
+		struct drm_colorop *colorop = new_plane_state->color_pipeline;
+
+		if (colorop->type != DRM_COLOROP_FIXED_MATRIX)
+			return -EINVAL;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c
index d1e7df50019030..5a69606a0593bc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c
@@ -324,6 +324,7 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
 	unsigned long kmap_num;
 	SVGA3dCopyBox *box;
 	u32 box_count;
+	u64 src_extent;
 	void *virtual;
 	bool is_iomem;
 	struct vmw_dma_cmd {
@@ -335,6 +336,8 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
 		vmw_surface_get_desc(VMW_CURSOR_SNOOP_FORMAT);
 	const u32 image_pitch = VMW_CURSOR_SNOOP_WIDTH * desc->pitchBytesPerBlock;
 
+	BUILD_BUG_ON(sizeof(*srf->snooper.image) != desc->pitchBytesPerBlock);
+
 	cmd = container_of(header, struct vmw_dma_cmd, header);
 
 	/* No snooper installed, nothing to copy */
@@ -372,8 +375,19 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
 		return;
 	}
 
+	if (box->w == 0 || box->h == 0)
+		return;
+
+	src_extent = (u64)(box->h - 1) * cmd->dma.guest.pitch +
+		     (u64)box->w * desc->pitchBytesPerBlock;
+	if (src_extent > bo->base.size) {
+		DRM_ERROR("Cursor snoop source of %llu bytes exceeds the %zu byte buffer\n",
+			  src_extent, bo->base.size);
+		return;
+	}
+
 	kmap_offset = cmd->dma.guest.ptr.offset >> PAGE_SHIFT;
-	kmap_num = (VMW_CURSOR_SNOOP_HEIGHT * image_pitch) >> PAGE_SHIFT;
+	kmap_num = PFN_UP(src_extent);
 
 	ret = ttm_bo_reserve(bo, true, false, NULL);
 	if (unlikely(ret != 0)) {
@@ -387,14 +401,16 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf,
 
 	virtual = ttm_kmap_obj_virtual(&map, &is_iomem);
 
-	if (box->w == VMW_CURSOR_SNOOP_WIDTH && cmd->dma.guest.pitch == image_pitch) {
+	if (box->w == VMW_CURSOR_SNOOP_WIDTH &&
+	    box->h == VMW_CURSOR_SNOOP_HEIGHT &&
+	    cmd->dma.guest.pitch == image_pitch) {
 		memcpy(srf->snooper.image, virtual,
 		       VMW_CURSOR_SNOOP_HEIGHT * image_pitch);
 	} else {
-		/* Image is unsigned pointer. */
+		/* The destination stride is in pixels. */
 		for (i = 0; i < box->h; i++)
-			memcpy(srf->snooper.image + i * image_pitch,
-			       virtual + i * cmd->dma.guest.pitch,
+			memcpy(srf->snooper.image + i * VMW_CURSOR_SNOOP_WIDTH,
+			       virtual + (size_t)i * cmd->dma.guest.pitch,
 			       box->w * desc->pitchBytesPerBlock);
 	}
 	srf->snooper.id++;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 1b407b61f68386..a6c3304e1a67d8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -20,10 +20,18 @@
 #include <drm/drm_rect.h>
 #include <drm/drm_sysfs.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_blend.h>
 
 void vmw_du_init(struct vmw_display_unit *du)
 {
 	vmw_vkms_crtc_init(&du->crtc);
+	drm_plane_create_blend_mode_property(&du->primary,
+					     BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+					     BIT(DRM_MODE_BLEND_PREMULTI));
+	// Check that the cursor plane was actually initialized
+	if (du->crtc.cursor == &du->cursor.base)
+		drm_plane_create_blend_mode_property(&du->cursor.base,
+						     BIT(DRM_MODE_BLEND_PREMULTI));
 }
 
 void vmw_du_cleanup(struct vmw_display_unit *du)
@@ -206,32 +214,24 @@ vmw_du_crtc_duplicate_state(struct drm_crtc *crtc)
 
 
 /**
- * vmw_du_crtc_reset - creates a blank vmw crtc state
+ * vmw_du_crtc_create_state - creates a blank vmw crtc state
  * @crtc: DRM crtc
  *
- * Resets the atomic state for @crtc by freeing the state pointer (which
- * might be NULL, e.g. at driver load time) and allocating a new empty state
- * object.
+ * Allocates a new empty state object for @crtc.
  */
-void vmw_du_crtc_reset(struct drm_crtc *crtc)
+struct drm_crtc_state *vmw_du_crtc_create_state(struct drm_crtc *crtc)
 {
 	struct vmw_crtc_state *vcs;
 
-
-	if (crtc->state) {
-		__drm_atomic_helper_crtc_destroy_state(crtc->state);
-
-		kfree(vmw_crtc_state_to_vcs(crtc->state));
-	}
-
 	vcs = kzalloc_obj(*vcs);
-
 	if (!vcs) {
 		DRM_ERROR("Cannot allocate vmw_crtc_state\n");
-		return;
+		return ERR_PTR(-ENOMEM);
 	}
 
-	__drm_atomic_helper_crtc_reset(crtc, &vcs->base);
+	__drm_atomic_helper_crtc_state_init(&vcs->base, crtc);
+
+	return &vcs->base;
 }
 
 
@@ -287,27 +287,24 @@ vmw_du_plane_duplicate_state(struct drm_plane *plane)
 
 
 /**
- * vmw_du_plane_reset - creates a blank vmw plane state
+ * vmw_du_plane_create_state - creates a blank vmw plane state
  * @plane: drm plane
  *
- * Resets the atomic state for @plane by freeing the state pointer (which might
- * be NULL, e.g. at driver load time) and allocating a new empty state object.
+ * Allocates a new empty state object.
  */
-void vmw_du_plane_reset(struct drm_plane *plane)
+struct drm_plane_state *vmw_du_plane_create_state(struct drm_plane *plane)
 {
 	struct vmw_plane_state *vps;
 
-	if (plane->state)
-		vmw_du_plane_destroy_state(plane, plane->state);
-
 	vps = kzalloc_obj(*vps);
-
 	if (!vps) {
 		DRM_ERROR("Cannot allocate vmw_plane_state\n");
-		return;
+		return ERR_PTR(-ENOMEM);
 	}
 
-	__drm_atomic_helper_plane_reset(plane, &vps->base);
+	__drm_atomic_helper_plane_state_init(&vps->base, plane);
+
+	return &vps->base;
 }
 
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 2224d7d91d1bb2..5085d8bccf0ae9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -411,7 +411,7 @@ int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
 				      struct drm_atomic_commit *state);
 void vmw_du_plane_cleanup_fb(struct drm_plane *plane,
 			     struct drm_plane_state *old_state);
-void vmw_du_plane_reset(struct drm_plane *plane);
+struct drm_plane_state *vmw_du_plane_create_state(struct drm_plane *plane);
 struct drm_plane_state *vmw_du_plane_duplicate_state(struct drm_plane *plane);
 void vmw_du_plane_destroy_state(struct drm_plane *plane,
 				struct drm_plane_state *state);
@@ -421,7 +421,7 @@ int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
 			     struct drm_atomic_commit *state);
 void vmw_du_crtc_atomic_begin(struct drm_crtc *crtc,
 			      struct drm_atomic_commit *state);
-void vmw_du_crtc_reset(struct drm_crtc *crtc);
+struct drm_crtc_state *vmw_du_crtc_create_state(struct drm_crtc *crtc);
 struct drm_crtc_state *vmw_du_crtc_duplicate_state(struct drm_crtc *crtc);
 void vmw_du_crtc_destroy_state(struct drm_crtc *crtc,
 				struct drm_crtc_state *state);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index af3e321745638a..b147f5db9df549 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -249,7 +249,7 @@ static void vmw_ldu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 static const struct drm_crtc_funcs vmw_legacy_crtc_funcs = {
 	.gamma_set = vmw_du_crtc_gamma_set,
 	.destroy = vmw_ldu_crtc_destroy,
-	.reset = vmw_du_crtc_reset,
+	.atomic_create_state = vmw_du_crtc_create_state,
 	.atomic_duplicate_state = vmw_du_crtc_duplicate_state,
 	.atomic_destroy_state = vmw_du_crtc_destroy_state,
 	.set_config = drm_atomic_helper_set_config,
@@ -364,7 +364,7 @@ static const struct drm_plane_funcs vmw_ldu_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = vmw_du_primary_plane_destroy,
-	.reset = vmw_du_plane_reset,
+	.atomic_create_state = vmw_du_plane_create_state,
 	.atomic_duplicate_state = vmw_du_plane_duplicate_state,
 	.atomic_destroy_state = vmw_du_plane_destroy_state,
 };
@@ -373,7 +373,7 @@ static const struct drm_plane_funcs vmw_ldu_cursor_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = vmw_cursor_plane_destroy,
-	.reset = vmw_du_plane_reset,
+	.atomic_create_state = vmw_du_plane_create_state,
 	.atomic_duplicate_state = vmw_du_plane_duplicate_state,
 	.atomic_destroy_state = vmw_du_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index c83061cf74557a..89714d5c235b6d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -305,7 +305,7 @@ static void vmw_sou_crtc_atomic_disable(struct drm_crtc *crtc,
 static const struct drm_crtc_funcs vmw_screen_object_crtc_funcs = {
 	.gamma_set = vmw_du_crtc_gamma_set,
 	.destroy = vmw_sou_crtc_destroy,
-	.reset = vmw_du_crtc_reset,
+	.atomic_create_state = vmw_du_crtc_create_state,
 	.atomic_duplicate_state = vmw_du_crtc_duplicate_state,
 	.atomic_destroy_state = vmw_du_crtc_destroy_state,
 	.set_config = drm_atomic_helper_set_config,
@@ -756,7 +756,7 @@ static const struct drm_plane_funcs vmw_sou_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = vmw_du_primary_plane_destroy,
-	.reset = vmw_du_plane_reset,
+	.atomic_create_state = vmw_du_plane_create_state,
 	.atomic_duplicate_state = vmw_du_plane_duplicate_state,
 	.atomic_destroy_state = vmw_du_plane_destroy_state,
 };
@@ -765,7 +765,7 @@ static const struct drm_plane_funcs vmw_sou_cursor_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = vmw_cursor_plane_destroy,
-	.reset = vmw_du_plane_reset,
+	.atomic_create_state = vmw_du_plane_create_state,
 	.atomic_duplicate_state = vmw_du_plane_duplicate_state,
 	.atomic_destroy_state = vmw_du_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 4139837f4cafbd..cbf55084110adb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -778,7 +778,7 @@ int vmw_kms_stdu_surface_dirty(struct vmw_private *dev_priv,
 static const struct drm_crtc_funcs vmw_stdu_crtc_funcs = {
 	.gamma_set = vmw_du_crtc_gamma_set,
 	.destroy = vmw_stdu_crtc_destroy,
-	.reset = vmw_du_crtc_reset,
+	.atomic_create_state = vmw_du_crtc_create_state,
 	.atomic_duplicate_state = vmw_du_crtc_duplicate_state,
 	.atomic_destroy_state = vmw_du_crtc_destroy_state,
 	.set_config = drm_atomic_helper_set_config,
@@ -1474,7 +1474,7 @@ static const struct drm_plane_funcs vmw_stdu_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = vmw_du_primary_plane_destroy,
-	.reset = vmw_du_plane_reset,
+	.atomic_create_state = vmw_du_plane_create_state,
 	.atomic_duplicate_state = vmw_du_plane_duplicate_state,
 	.atomic_destroy_state = vmw_du_plane_destroy_state,
 };
@@ -1483,7 +1483,7 @@ static const struct drm_plane_funcs vmw_stdu_cursor_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.destroy = vmw_cursor_plane_destroy,
-	.reset = vmw_du_plane_reset,
+	.atomic_create_state = vmw_du_plane_create_state,
 	.atomic_duplicate_state = vmw_du_plane_duplicate_state,
 	.atomic_destroy_state = vmw_du_plane_destroy_state,
 };
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index c92468cb9b8941..563c2f859361f5 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -35,6 +35,7 @@ $(obj)/generated/%_device_wa_oob.c $(obj)/generated/%_device_wa_oob.h: $(obj)/xe
 xe-y += xe_bb.o \
 	xe_bo.o \
 	xe_bo_evict.o \
+	xe_cpu_bind.o \
 	xe_dep_scheduler.o \
 	xe_devcoredump.o \
 	xe_device.o \
@@ -87,6 +88,7 @@ xe-y += xe_bb.o \
 	xe_hw_fence.o \
 	xe_irq.o \
 	xe_late_bind_fw.o \
+	xe_log.o \
 	xe_lrc.o \
 	xe_mem_pool.o \
 	xe_migrate.o \
@@ -230,8 +232,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	display/xe_hdcp_gsc.o \
 	display/xe_initial_plane.o \
 	display/xe_panic.o \
-	display/xe_stolen.o \
-	display/xe_tdf.o
+	display/xe_stolen.o
 
 # Display code shared with i915
 xe-$(CONFIG_DRM_XE_DISPLAY) += \
@@ -259,6 +260,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_ddi.o \
 	i915-display/intel_ddi_buf_trans.o \
 	i915-display/intel_de.o \
+	i915-display/intel_dip.o \
 	i915-display/intel_display.o \
 	i915-display/intel_display_conversion.o \
 	i915-display/intel_display_device.o \
diff --git a/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h
index ce5c595175283f..57baa68bcba409 100644
--- a/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h
@@ -119,6 +119,7 @@ enum slpc_param_id {
 	SLPC_PARAM_STRATEGIES = 26,
 	SLPC_PARAM_POWER_PROFILE = 27,
 	SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY = 28,
+	SLPC_PARAM_SET_IBC_VERSION = 29,
 	SLPC_MAX_PARAM = 32,
 };
 
diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
index 685c4ef17b7329..4bcdc1a96785eb 100644
--- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
@@ -525,6 +525,7 @@ enum xe_guc_klv_ids {
 	GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG			= 0x900e,
 	GUC_WA_KLV_REMAP_RANGED_TLB_INV							= 0x900f,
 	GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64					= 0x9010,
+	GUC_WA_KLV_IGNORE_MULTIQ_CTX_COMPLETE_WITH_PREEMPT_IN_CSB			= 0x9011,
 };
 
 /**
diff --git a/drivers/gpu/drm/xe/abi/xe_log_abi.h b/drivers/gpu/drm/xe/abi/xe_log_abi.h
new file mode 100644
index 00000000000000..526aadf85fe938
--- /dev/null
+++ b/drivers/gpu/drm/xe/abi/xe_log_abi.h
@@ -0,0 +1,200 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _ABI_XE_LOG_ABI_H_
+#define _ABI_XE_LOG_ABI_H_
+
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+
+#include "abi/xe_sigid_abi.h"
+
+/**
+ * enum xe_log_component_bits - bits for components structure definitions
+ *
+ * Component identifiers are structured based on::
+ *
+ *     COMPONENT = CLASS(8b).TYPE(8b)
+ *
+ * and the structure looks like this::
+ *
+ *     ├── SYSTEM(0)
+ *     │   └── ...
+ *     ├── DRIVER(1)
+ *     │   └── ...
+ *     ├── FEATURE(2)
+ *     │   └── ...
+ *     ├── FIRMWARE(4)
+ *     │   └── ...
+ *     └── HARDWARE(8)
+ *         └── ...
+ *
+ * Examples::
+ *
+ *     COMPONENT(0.type) = SYSTEM.type = system component
+ *     COMPONENT(1.type) = DRIVER.type = driver core component
+ *     COMPONENT(3.type) = DRIVER_FEATURE.type = driver feature
+ *     COMPONENT(5.type) = DRIVER_FIRMWARE.type = firmware driver component
+ *     COMPONENT(9.type) = DRIVER_HARDWARE.type = hardware driver component
+ *
+ */
+enum xe_log_component_bits {
+	/* private: */
+	XE_LOG_COMPONENT_CLASS_MASK = GENMASK_U16(7, 0),
+	XE_LOG_COMPONENT_TYPE_MASK = GENMASK_U16(15, 8),
+	/* private: component classes */
+	XE_LOG_COMPONENT_CLASS_SYSTEM = 0u,
+	XE_LOG_COMPONENT_CLASS_DRIVER = 1u,
+	XE_LOG_COMPONENT_CLASS_FEATURE = 2u,
+	XE_LOG_COMPONENT_CLASS_FIRMWARE = 4u,
+	XE_LOG_COMPONENT_CLASS_HARDWARE = 8u,
+	XE_LOG_COMPONENT_CLASS_DRIVER_FEATURE = XE_LOG_COMPONENT_CLASS_DRIVER |
+						XE_LOG_COMPONENT_CLASS_FEATURE,
+	XE_LOG_COMPONENT_CLASS_DRIVER_FIRMWARE = XE_LOG_COMPONENT_CLASS_DRIVER |
+						 XE_LOG_COMPONENT_CLASS_FIRMWARE,
+	XE_LOG_COMPONENT_CLASS_DRIVER_HARDWARE = XE_LOG_COMPONENT_CLASS_DRIVER |
+						 XE_LOG_COMPONENT_CLASS_HARDWARE,
+	/* private: reserved identifiers */
+	XE_LOG_COMPONENT_NONE = 0u,
+};
+
+#define MAKE_XE_LOG_COMPONENT(_CLASS, type) \
+	(FIELD_PREP_CONST(XE_LOG_COMPONENT_CLASS_MASK, \
+			  XE_LOG_COMPONENT_CLASS_##_CLASS) | \
+	 FIELD_PREP_CONST(XE_LOG_COMPONENT_TYPE_MASK, (type)))
+
+/**
+ * enum xe_log_location_bits - bits for location structure definitions
+ *
+ * Location identifiers are structured based on::
+ *
+ *     LOCATION = TYPE(8b).ID(8b)
+ *
+ * and the structure looks like this::
+ *
+ *     ├── DEVICE(0)
+ *     │   └── MBZ(0)
+ *     ├── TILE(1)
+ *     │   ├── Tile0(0)
+ *     │   ├── ...
+ *     │   └── TileN(n)
+ *     ├── GT(1)
+ *     │   ├── GT0(0)
+ *     │   ├── ...
+ *     │   └── GTn(n)
+ *     └── ...
+ *
+ * Examples::
+ *
+ *     LOCATION(0.0) = NONE
+ *     LOCATION(1.0) = DEVICE.0 = "Device"
+ *     LOCATION(2.1) = TILE.1 = "Tile1"
+ *     LOCATION(3.2) = GT.2 = "GT2"
+ *
+ */
+enum xe_log_location_bits {
+	/* private: */
+	XE_LOG_LOCATION_TYPE_MASK = GENMASK_U16(7, 0),
+	XE_LOG_LOCATION_ID_MASK = GENMASK_U16(15, 8),
+	/* private: location types */
+	XE_LOG_LOCATION_TYPE_DEVICE = 1u,
+	XE_LOG_LOCATION_TYPE_TILE = 2u,
+	XE_LOG_LOCATION_TYPE_GT = 3u,
+	/* private: reserved identifiers */
+	XE_LOG_LOCATION_NONE = 0u,
+};
+
+#define PREP_XE_LOG_LOCATION(type, id) \
+	(FIELD_PREP(XE_LOG_LOCATION_TYPE_MASK, (type)) | \
+	 FIELD_PREP(XE_LOG_LOCATION_ID_MASK, (id)))
+
+#define MAKE_XE_LOG_LOCATION(_TYPE, id) \
+	PREP_XE_LOG_LOCATION(XE_LOG_LOCATION_TYPE_##_TYPE, (id))
+
+/**
+ * DEFINE_XE_LOG_COMPONENTS() - Define log components.
+ * @define: name of the inner macro to expand.
+ *
+ * Use this super macro to define custom code for the log components.
+ * The following parameters are available for each component::
+ *
+ *     define(CLASS, ID, TAG, SIGID, NAME)
+ *
+ * where:
+ *
+ *     @CLASS is the component class name (without the XE_LOG_COMPONENT_CLASS_ prefix)
+ *     @ID is the unique component identifier within @CLASS
+ *     @TAG is unique component tag (across all components)
+ *     @SIGID is the default xe_sigid for the component (without the XE_SIGID_ prefix)
+ */
+#define DEFINE_XE_LOG_COMPONENTS(define) \
+	DEFINE_XE_LOG_SOFTWARE_COMPONENTS(define) \
+	DEFINE_XE_LOG_HARDWARE_COMPONENTS(define)
+
+#define DEFINE_XE_LOG_SOFTWARE_COMPONENTS(define) \
+	/* */									\
+	define(SYSTEM, 1, PCI, IO_BUS, "Linux PCI Subsystem")			\
+	define(SYSTEM, 2, DRM, SW, "DRM")					\
+	/* */									\
+	define(DRIVER, 1, XE, SW, "Xe Driver")					\
+	define(DRIVER, 2, PROBE, PROBE, "Driver Initialization")		\
+	define(DRIVER, 3, WEDGED, WEDGED, "Device Malfunction")			\
+	define(DRIVER, 4, RTP, SW, "Register Table Processing")			\
+	define(DRIVER, 5, WA, SW, "Workarounds")				\
+	define(DRIVER, 6, PAGEFAULT, MEM_FAULT, "Page Fault")			\
+	define(DRIVER, 7, GUCSUBMIT, GT_TDR, "GuC Submission")			\
+	/* */									\
+	define(DRIVER_HARDWARE, 1, REGS, IO_BUS, "Registers")			\
+	define(DRIVER_HARDWARE, 2, GGTT, IO_BUS, "Global GTT")			\
+	define(DRIVER_HARDWARE, 3, GT, GT_TDR, "Graphics Technology")		\
+	define(DRIVER_HARDWARE, 4, LMTT, IO_BUS, "LMEM Translation Table")	\
+	define(DRIVER_HARDWARE, 5, MEMIRQ, IO_BUS, "Memory Based IRQ")		\
+	/* */									\
+	define(DRIVER_FEATURE, 1, PF, SW, "SR-IOV Physical Function")		\
+	define(DRIVER_FEATURE, 2, VF, SW, "SR-IOV Virtual Function")		\
+	define(DRIVER_FEATURE, 3, SURVIVABILITY, SURVIVABILITY, "Survivability") \
+	define(DRIVER_FEATURE, 4, RAS, SW, "Reliability, Accessibility, Serviceability") \
+	/* */									\
+	define(DRIVER_FIRMWARE, 1, GUC, RUNTIME_FW, "GuC")			\
+	define(DRIVER_FIRMWARE, 2, HUC, RUNTIME_FW, "HuC")			\
+	define(DRIVER_FIRMWARE, 3, GSC, RUNTIME_FW, "GSC")			\
+	define(DRIVER_FIRMWARE, 16, PCODE, DEVICE_FW, "PCode")			\
+	define(DRIVER_FIRMWARE, 17, SYSCTRL, DEVICE_FW, "System Controller")	\
+
+#define DEFINE_XE_LOG_HARDWARE_COMPONENTS(define) \
+	define(HARDWARE, 1, DEVICE_MEMORY, DEVICE_MEMORY, "Device Memory")	\
+	define(HARDWARE, 2, CORE_COMPUTE, CORE_COMPUTE, "Core Compute")		\
+	/*     HARDWARE, 3, RESERVED */						\
+	define(HARDWARE, 4, PCIE, PCIE, "PCIe Interface")			\
+	define(HARDWARE, 5, FABRIC, FABRIC, "Fabric")				\
+	define(HARDWARE, 6, SOC_INTERNAL, SOC_INTERNAL, "SoC Internal")		\
+	/* eod */
+
+/**
+ * enum xe_log_component_tags - TAGs of all supported components
+ */
+enum xe_log_component_tags {
+	/* private: */
+#define MAKE_XE_LOG_COMPONENT_ENUM(_CLASS, _ID, _TAG, _SIG, _NAME) \
+	XE_LOG_COMPONENT_##_TAG = MAKE_XE_LOG_COMPONENT(_CLASS, (_ID)), \
+	XE_LOG_COMPONENT_##_CLASS##_##_ID = XE_LOG_COMPONENT_##_TAG, \
+	/* eod */
+	DEFINE_XE_LOG_COMPONENTS(MAKE_XE_LOG_COMPONENT_ENUM)
+#undef MAKE_XE_LOG_COMPONENT_ENUM
+};
+
+/**
+ * enum xe_log_component_sigids - SIGIDs of all supported components
+ */
+enum xe_log_component_sigids {
+	/* private: */
+#define MAKE_XE_LOG_COMPONENT_SIGID(_CLASS, _ID, _TAG, _SIG, _NAME) \
+	XE_LOG_COMPONENT_##_TAG##_SIGID = XE_SIGID_##_SIG, \
+	/* eod */
+	DEFINE_XE_LOG_COMPONENTS(MAKE_XE_LOG_COMPONENT_SIGID)
+#undef MAKE_XE_LOG_COMPONENT_SIGID
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/abi/xe_sigid_abi.h b/drivers/gpu/drm/xe/abi/xe_sigid_abi.h
new file mode 100644
index 00000000000000..2f3440505f695e
--- /dev/null
+++ b/drivers/gpu/drm/xe/abi/xe_sigid_abi.h
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _ABI_XE_SIGID_ABI_H_
+#define _ABI_XE_SIGID_ABI_H_
+
+/**
+ * DOC: Xe Error Signatures (SIGID)
+ *
+ * What SIGID stands for
+ * ---------------------
+ *
+ * SIGID is short for *Signature Identifier*. It is a small, stable integer
+ * that names one of the *recognised fault sites* -- nothing more. It is the
+ * primary handle used for triage and maps directly to a specific report site.
+ *
+ * Numbering
+ * ---------
+ *
+ * SIGIDs are a single flat list numbered sequentially within the assigned range,
+ * in the order the fault sites were introduced. Values are stable: once assigned
+ * they are only ever appended, never renumbered or reused. A retired fault site
+ * SIGID value is deprecated in place, never re-purposed.
+ *
+ * Why this exists
+ * ---------------
+ *
+ * Today the driver reports faults with ad-hoc ``xe_err()`` / ``xe_gt_err()``
+ * strings that have no stable shape. That is fine for a human reading dmesg,
+ * but it gives fleet tooling nothing durable to match on: the wording changes
+ * between releases, lines can be rate-limited or dropped under an error storm,
+ * and there is no consistent way to ask "which recognised fault just happened?"
+ *
+ * A SIGID answers exactly that one question, identically across driver and
+ * firmware versions, and (eventually) across other Intel devices in a node.
+ *
+ * What a SIGID is not
+ * -------------------
+ *
+ * SIGID deliberately does not encode the detailed reason or the outcome. Those
+ * are carried alongside it::
+ *
+ *   SIGID    -> which recognised fault site is being reported
+ *   severity -> how serious this instance is
+ *   errno    -> the failing operation's error, if available, shown with %pe
+ *   message  -> free-form human-readable context
+ *
+ * Severity is independent of the SIGID. The same SIGID can be reported at
+ * different severities depending on the instance and the recovery taken.
+ *
+ * When to use SIGID logging
+ * -------------------------
+ *
+ * The xe_log_*() helpers are for these recognised fault sites only --
+ * important, operator-relevant faults and events. The driver's only job is to
+ * emit the right SIGID next to the usual human-readable text.
+
+ * They are not a replacement for ``xe_info()`` / ``xe_dbg()`` / tracing, nor
+ * for one-off diagnostics; using them for ordinary logging would dilute the
+ * fault stream. Not every ``xe_err()`` needs to become a SIGID report -- only
+ * those that correspond to a published fault site.
+ *
+ * SIGID log output (dmesg vs. the machine record)
+ * -----------------------------------------------
+ *
+ * The dmesg line stays close to a normal xe error message so it remains
+ * readable for admins; the only stable, machine-matchable token on it is
+ * ``SIGID=<n>`` (``dmesg | grep SIGID=``).
+ *
+ * The full dmesg line is not an ABI: the surrounding text may change freely,
+ * and lines may be dropped. The durable record for tooling is the CPER record
+ * carrying the same SIGID (generation is a planned follow-up).
+ *
+ * How to pick a SIGID (the uniqueness rule)
+ * -----------------------------------------
+ *
+ * Pick per *report site*, not per incident. Each site emits the single most
+ * specific recognised SIGID *for that site* -- so the question is never
+ * "classify this whole failure", it is "what does this site detect?", which has
+ * one answer. A single underlying failure therefore legitimately produces a
+ * *chain* of reports from different layers, each with its own SIGID -- e.g. a
+ * GuC communication failure is reported as %XE_SIGID_RUNTIME_FW by the firmware
+ * path, the failed recovery as %XE_SIGID_GT_TDR by the reset path, and an
+ * aborted bind as %XE_SIGID_PROBE by the probe path. That chain lets triage
+ * follow a fault from origin to final effect; it is not a duplicate.
+ *
+ * If a site does not match any defined SIGID, keep using the ordinary
+ * ``xe_err()`` / ``xe_gt_err()`` logging rather than forcing a SIGID: a wrong
+ * or over-broad classification is harder to retire than a missing one. When a
+ * new report site is genuinely worth triaging, add it to the list below.
+ *
+ * Usage of the existing SIGID reports must reevaluated according to this section
+ * after making significant changes to the site that emits this SIGID.
+ *
+ * Scope: software vs hardware emitted signatures
+ * ----------------------------------------------
+ *
+ * Some SIGID represents fault sites that the *driver itself* detects and
+ * reports from the software POV: probe abort, wedged, survivability, driver-
+ * detected firmware failures, engine TDR, memory faults and IO/bus faults.
+ * These are the only values the driver assigns on its own.
+ *
+ * Signatures that *originate* in firmware or hardware are a different thing:
+ * they are produced and identified by the firmware or the hardware itself
+ * (e.g. via their own records or error counters), and the driver merely logs
+ * them as they are given to us. They are deliberately enumerated separately.
+ *
+ * The two driver-detected firmware report sites below (%XE_SIGID_RUNTIME_FW,
+ * %XE_SIGID_DEVICE_FW) are software signatures: they mark that *the driver*
+ * observed a firmware problem, not a signature reported by the firmware.
+ */
+
+/*
+ * Top level Intel Error Signature Identifiers.
+ */
+#define INTEL_SIGID_INVALID			0
+#define INTEL_SIGID_BATCH			100
+#define INTEL_SIGID_RANGE_START(n)		((n) * INTEL_SIGID_BATCH)
+#define INTEL_SIGID_RANGE_END(n)		(INTEL_SIGID_RANGE_START((n) + 1) - 1)
+
+/* SIGIDs 1xx are reserved for Xe GPU software and 2xx for Xe GPU hardware */
+#define INTEL_SIGID_GPU_XE_SOFTWARE_START	INTEL_SIGID_RANGE_START(1)
+#define INTEL_SIGID_GPU_XE_SOFTWARE_END		INTEL_SIGID_RANGE_END(1)
+#define INTEL_SIGID_GPU_XE_HARDWARE_START	INTEL_SIGID_RANGE_START(2)
+#define INTEL_SIGID_GPU_XE_HARDWARE_END		INTEL_SIGID_RANGE_END(2)
+
+/**
+ * enum xe_sigid - Stable Xe Error Signature Identifiers (SIGID).
+ * @XE_SIGID_SW: Software component failure.
+ * @XE_SIGID_PROBE: Device probe/bind was aborted.
+ * @XE_SIGID_WEDGED: Device was declared wedged and is no longer usable.
+ * @XE_SIGID_SURVIVABILITY: Device entered survivability mode.
+ * @XE_SIGID_RUNTIME_FW: Driver-detected runtime firmware failure, GuC/HuC/GSC.
+ * @XE_SIGID_DEVICE_FW: Driver-detected device firmware failure, PCODE/sysctrl.
+ * @XE_SIGID_GT_TDR: Engine hang / timeout detection and recovery (reset).
+ * @XE_SIGID_MEM_FAULT: VM bind, page fault or GTT fault.
+ * @XE_SIGID_IO_BUS: Runtime PCIe / IOMMU / MMIO access fault.
+ * @XE_SIGID_HW: Generic hardware failure.
+ * @XE_SIGID_PCIE: PCIe interface errors.
+ * @XE_SIGID_DEVICE_MEMORY: Device memory errors.
+ * @XE_SIGID_CORE_COMPUTE: Compute/shader core errors.
+ * @XE_SIGID_FABRIC: Fabric errors.
+ * @XE_SIGID_SOC_INTERNAL: SoC-internal errors.
+ *
+ * Each SIGID represents the report sites the driver detects and reports.
+ * Values are numbered sequentially, are only ever appended, and are never
+ * renumbered or reused.
+ *
+ * Firmware- and hardware-originated signatures are numbered separately.
+ */
+enum xe_sigid {
+	XE_SIGID_SW			= INTEL_SIGID_GPU_XE_SOFTWARE_START,
+	XE_SIGID_PROBE			= INTEL_SIGID_GPU_XE_SOFTWARE_START + 1,
+	XE_SIGID_WEDGED			= INTEL_SIGID_GPU_XE_SOFTWARE_START + 2,
+	XE_SIGID_SURVIVABILITY		= INTEL_SIGID_GPU_XE_SOFTWARE_START + 3,
+	XE_SIGID_RUNTIME_FW		= INTEL_SIGID_GPU_XE_SOFTWARE_START + 4,
+	XE_SIGID_DEVICE_FW		= INTEL_SIGID_GPU_XE_SOFTWARE_START + 5,
+	XE_SIGID_GT_TDR			= INTEL_SIGID_GPU_XE_SOFTWARE_START + 6,
+	XE_SIGID_MEM_FAULT		= INTEL_SIGID_GPU_XE_SOFTWARE_START + 7,
+	XE_SIGID_IO_BUS			= INTEL_SIGID_GPU_XE_SOFTWARE_START + 8,
+
+	XE_SIGID_HW			= INTEL_SIGID_GPU_XE_HARDWARE_START,
+	XE_SIGID_PCIE			= INTEL_SIGID_GPU_XE_HARDWARE_START + 1,
+	XE_SIGID_DEVICE_MEMORY		= INTEL_SIGID_GPU_XE_HARDWARE_START + 2,
+	XE_SIGID_CORE_COMPUTE		= INTEL_SIGID_GPU_XE_HARDWARE_START + 3,
+	XE_SIGID_FABRIC			= INTEL_SIGID_GPU_XE_HARDWARE_START + 4,
+	XE_SIGID_SOC_INTERNAL		= INTEL_SIGID_GPU_XE_HARDWARE_START + 5,
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/i915_config.h b/drivers/gpu/drm/xe/compat-i915-headers/i915_config.h
deleted file mode 100644
index d4522203e2dd8f..00000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/i915_config.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __I915_CONFIG_H__
-#define __I915_CONFIG_H__
-
-#include <linux/sched.h>
-
-static inline unsigned long i915_fence_timeout(void)
-{
-	return MAX_SCHEDULE_TIMEOUT;
-}
-
-#endif /* __I915_CONFIG_H__ */
diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c
index 8da4c457a4a0f3..506f26e9c59a82 100644
--- a/drivers/gpu/drm/xe/display/xe_display.c
+++ b/drivers/gpu/drm/xe/display/xe_display.c
@@ -6,37 +6,28 @@
 #include "xe_display.h"
 #include "regs/xe_irq_regs.h"
 
-#include <linux/fb.h>
+#include <linux/acpi.h>
 
-#include <drm/drm_client.h>
-#include <drm/drm_client_event.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_managed.h>
-#include <drm/drm_probe_helper.h>
 #include <drm/intel/display_member.h>
 #include <drm/intel/display_parent_interface.h>
-#include <uapi/drm/xe_drm.h>
 
-#include "intel_acpi.h"
-#include "intel_audio.h"
 #include "intel_display.h"
 #include "intel_display_core.h"
 #include "intel_display_device.h"
 #include "intel_display_driver.h"
 #include "intel_display_irq.h"
-#include "intel_display_types.h"
 #include "intel_dmc.h"
 #include "intel_dmc_wl.h"
-#include "intel_dp.h"
 #include "intel_fbdev.h"
-#include "intel_hdcp.h"
 #include "intel_hotplug.h"
 #include "intel_opregion.h"
-#include "skl_watermark.h"
 #include "xe_device.h"
 #include "xe_display_bo.h"
 #include "xe_display_pcode.h"
 #include "xe_display_rpm.h"
+#include "xe_display_wa.h"
 #include "xe_dsb_buffer.h"
 #include "xe_fb_pin.h"
 #include "xe_frontbuffer.h"
@@ -129,9 +120,6 @@ static void xe_display_fini(void *arg)
 	struct xe_device *xe = arg;
 	struct intel_display *display = xe->display;
 
-	intel_hpd_poll_fini(display);
-	intel_hdcp_component_fini(display);
-	intel_audio_deinit(display);
 	intel_display_driver_remove(display);
 }
 
@@ -446,6 +434,20 @@ static bool has_auxccs(struct drm_device *drm)
 	return xe->info.platform == XE_ALDERLAKE_P;
 }
 
+/*
+ * TDF (Transient-Data-Flush) is needed for Xe2+ where special L3:XD caching can
+ * be enabled through various PAT index modes. Idea is to use this caching mode
+ * when for example rendering onto the display surface, with the promise that
+ * KMD will ensure transient cache entries are always flushed by the time we do
+ * the display flip, since display engine is never coherent with CPU/GPU caches.
+ */
+static void transient_data_flush(struct drm_device *drm)
+{
+	struct xe_device *xe = to_xe_device(drm);
+
+	xe_device_td_flush(xe);
+}
+
 static const struct intel_display_parent_interface parent = {
 	.bo = &xe_display_bo_interface,
 	.dsb = &xe_display_dsb_interface,
@@ -458,7 +460,9 @@ static const struct intel_display_parent_interface parent = {
 	.pcode = &xe_display_pcode_interface,
 	.rpm = &xe_display_rpm_interface,
 	.stolen = &xe_display_stolen_interface,
+	.wa = &xe_display_wa_interface,
 	.has_auxccs = has_auxccs,
+	.transient_data_flush = transient_data_flush,
 };
 
 /**
diff --git a/drivers/gpu/drm/xe/display/xe_display_pcode.c b/drivers/gpu/drm/xe/display/xe_display_pcode.c
index f6820ef7e666d3..8dc9cbdb18ec05 100644
--- a/drivers/gpu/drm/xe/display/xe_display_pcode.c
+++ b/drivers/gpu/drm/xe/display/xe_display_pcode.c
@@ -6,12 +6,12 @@
 #include "xe_device.h"
 #include "xe_pcode.h"
 
-static int xe_display_pcode_read(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1)
+static int xe_display_pcode_read(struct drm_device *drm, u32 mbox, u32 *val0, u32 *val1)
 {
 	struct xe_device *xe = to_xe_device(drm);
 	struct xe_tile *tile = xe_device_get_root_tile(xe);
 
-	return xe_pcode_read(tile, mbox, val, val1);
+	return xe_pcode_read(tile, mbox, val0, val1);
 }
 
 static int xe_display_pcode_write_timeout(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms)
diff --git a/drivers/gpu/drm/xe/display/xe_display_rpm.c b/drivers/gpu/drm/xe/display/xe_display_rpm.c
index b3db4003549954..57bcfb41bddab2 100644
--- a/drivers/gpu/drm/xe/display/xe_display_rpm.c
+++ b/drivers/gpu/drm/xe/display/xe_display_rpm.c
@@ -3,8 +3,6 @@
 
 #include <drm/intel/display_parent_interface.h>
 
-#include "intel_display_core.h"
-#include "intel_display_rpm.h"
 #include "xe_device.h"
 #include "xe_device_types.h"
 #include "xe_pm.h"
@@ -47,6 +45,11 @@ static bool xe_display_rpm_suspended(const struct drm_device *drm)
 	return pm_runtime_suspended(xe->drm.dev);
 }
 
+static bool xe_display_rpm_pme_enabled(const struct drm_device *drm)
+{
+	return xe_pm_pme_enabled(to_xe_device(drm));
+}
+
 static void xe_display_rpm_assert_held(const struct drm_device *drm)
 {
 	/* FIXME */
@@ -71,6 +74,7 @@ const struct intel_display_rpm_interface xe_display_rpm_interface = {
 	.put_raw = xe_display_rpm_put,
 	.put_unchecked = xe_display_rpm_put_unchecked,
 	.suspended = xe_display_rpm_suspended,
+	.pme_enabled = xe_display_rpm_pme_enabled,
 	.assert_held = xe_display_rpm_assert_held,
 	.assert_block = xe_display_rpm_assert_block,
 	.assert_unblock = xe_display_rpm_assert_unblock
diff --git a/drivers/gpu/drm/xe/display/xe_display_wa.c b/drivers/gpu/drm/xe/display/xe_display_wa.c
index 2aa1b8c03411fe..46e41e8a3304c5 100644
--- a/drivers/gpu/drm/xe/display/xe_display_wa.c
+++ b/drivers/gpu/drm/xe/display/xe_display_wa.c
@@ -3,17 +3,22 @@
  * Copyright © 2024 Intel Corporation
  */
 
-#include "intel_display_core.h"
-#include "intel_display_wa.h"
+#include <drm/intel/display_parent_interface.h>
+
 #include "xe_device.h"
+#include "xe_display_wa.h"
 #include "xe_wa.h"
 
 #include <generated/xe_wa_oob.h>
 
-bool intel_display_needs_wa_16023588340(struct intel_display *display)
+static bool intel_display_needs_wa_16023588340(struct drm_device *drm)
 {
-	struct xe_device *xe = to_xe_device(display->drm);
+	struct xe_device *xe = to_xe_device(drm);
 	struct xe_gt *wa_gt = xe_root_mmio_gt(xe);
 
 	return wa_gt && XE_GT_WA(wa_gt, 16023588340);
 }
+
+const struct intel_display_wa_interface xe_display_wa_interface = {
+	.wa_16023588340 = intel_display_needs_wa_16023588340,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_display_wa.h b/drivers/gpu/drm/xe/display/xe_display_wa.h
new file mode 100644
index 00000000000000..0262b463916deb
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_display_wa.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __XE_DISPLAY_WA_H__
+#define __XE_DISPLAY_WA_H__
+
+extern const struct intel_display_wa_interface xe_display_wa_interface;
+
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
index a7158c73a14c35..82974e933e8db4 100644
--- a/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
+++ b/drivers/gpu/drm/xe/display/xe_dsb_buffer.c
@@ -88,7 +88,7 @@ static void xe_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
 	 * both for weak ordering archs and discrete cards.
 	 */
 	xe_device_wmb(xe);
-	xe_device_l2_flush(xe);
+	xe_device_l2_flush(xe, false);
 }
 
 const struct intel_display_dsb_interface xe_display_dsb_interface = {
diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c
index 73469ea5f333d3..b46a2c32ac0734 100644
--- a/drivers/gpu/drm/xe/display/xe_fb_pin.c
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c
@@ -203,7 +203,7 @@ static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj,
 	vma->node = dpt->ggtt_node[tile0->id];
 
 	/* Ensure DPT writes are flushed */
-	xe_device_l2_flush(xe);
+	xe_device_l2_flush(xe, false);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/display/xe_panic.c b/drivers/gpu/drm/xe/display/xe_panic.c
index 12c6fb99015dd1..1a6cee25e9d76c 100644
--- a/drivers/gpu/drm/xe/display/xe_panic.c
+++ b/drivers/gpu/drm/xe/display/xe_panic.c
@@ -52,7 +52,8 @@ static void xe_panic_page_set_pixel(struct drm_scanout_buffer *sb, unsigned int
 	if (new_page != panic->page) {
 		if (xe_bo_is_vram(bo)) {
 			/* Display is always mapped on root tile */
-			struct xe_vram_region *vram = xe_bo_device(bo)->mem.vram;
+			struct xe_vram_region *vram =
+				xe_device_get_root_tile(xe_bo_device(bo))->mem.vram;
 
 			if (panic->page < 0 || new_page < panic->page) {
 				xe_res_first(bo->ttm.resource, new_page * PAGE_SIZE,
diff --git a/drivers/gpu/drm/xe/display/xe_tdf.c b/drivers/gpu/drm/xe/display/xe_tdf.c
deleted file mode 100644
index 78bda4c47874ae..00000000000000
--- a/drivers/gpu/drm/xe/display/xe_tdf.c
+++ /dev/null
@@ -1,15 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2024 Intel Corporation
- */
-
-#include "intel_display_core.h"
-#include "intel_tdf.h"
-#include "xe_device.h"
-
-void intel_td_flush(struct intel_display *display)
-{
-	struct xe_device *xe = to_xe_device(display->drm);
-
-	xe_device_td_flush(xe);
-}
diff --git a/drivers/gpu/drm/xe/instructions/xe_mi_commands.h b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h
index ad7d98f2dbbaf7..835ef75bbeb1f9 100644
--- a/drivers/gpu/drm/xe/instructions/xe_mi_commands.h
+++ b/drivers/gpu/drm/xe/instructions/xe_mi_commands.h
@@ -36,6 +36,7 @@
 
 #define MI_SEMAPHORE_WAIT		(__MI_INSTR(0x1c) | XE_INSTR_NUM_DW(5))
 #define   MI_SEMW_GGTT			REG_BIT(22)
+#define   MI_SEMW_REGISTER_POLL		REG_BIT(16)
 #define   MI_SEMW_POLL			REG_BIT(15)
 #define   MI_SEMW_COMPARE_OP_MASK	REG_GENMASK(14, 12)
 #define     COMPARE_OP_SAD_GT_SDD	0
@@ -94,10 +95,4 @@
 #define MI_SET_APPID_SESSION_ID_MASK	REG_GENMASK(6, 0)
 #define MI_SET_APPID_SESSION_ID(x)	REG_FIELD_PREP(MI_SET_APPID_SESSION_ID_MASK, x)
 
-#define MI_SEMAPHORE_WAIT_TOKEN		(__MI_INSTR(0x1c) | XE_INSTR_NUM_DW(5)) /* XeLP+ */
-#define   MI_SEMAPHORE_REGISTER_POLL	REG_BIT(16)
-#define   MI_SEMAPHORE_POLL		REG_BIT(15)
-#define   MI_SEMAPHORE_CMP_OP_MASK	REG_GENMASK(14, 12)
-#define   MI_SEMAPHORE_SAD_EQ_SDD	REG_FIELD_PREP(MI_SEMAPHORE_CMP_OP_MASK, 4)
-
 #endif
diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
index 08251c7a1a4b6b..650f117c3e135b 100644
--- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
@@ -196,6 +196,33 @@
 #define   MSC_MSAA_REODER_BUF_BYPASS_DISABLE	REG_BIT(14)
 #define   FAST_CLEAR_VALIGN_FIX			REG_BIT(13)
 
+#define MSG_IDLE_CS				XE_REG(0x8000)
+#define MSG_IDLE_VCS0				XE_REG(0x8004)
+#define MSG_IDLE_VCS1				XE_REG(0x8008)
+#define MSG_IDLE_VCS2				XE_REG(0x80C0)
+#define MSG_IDLE_VCS3				XE_REG(0x80C4)
+#define MSG_IDLE_VCS4				XE_REG(0x80C8)
+#define MSG_IDLE_VCS5				XE_REG(0x80CC)
+#define MSG_IDLE_VCS6				XE_REG(0x80D0)
+#define MSG_IDLE_VCS7				XE_REG(0x80D4)
+#define MSG_IDLE_VECS0				XE_REG(0x8010)
+#define MSG_IDLE_VECS1				XE_REG(0x80D8)
+#define MSG_IDLE_VECS2				XE_REG(0x80DC)
+#define MSG_IDLE_VECS3				XE_REG(0x80E0)
+#define MSG_IDLE_BCS0				XE_REG(0x800C)
+#define MSG_IDLE_BCS1				XE_REG(0x8680)
+#define MSG_IDLE_BCS2				XE_REG(0x8684)
+#define MSG_IDLE_BCS3				XE_REG(0x8688)
+#define MSG_IDLE_BCS4				XE_REG(0x868C)
+#define MSG_IDLE_BCS5				XE_REG(0x8690)
+#define MSG_IDLE_BCS6				XE_REG(0x8694)
+#define MSG_IDLE_BCS7				XE_REG(0x8698)
+#define MSG_IDLE_BCS8				XE_REG(0x869C)
+#define MSG_IDLE_GSCCS0				XE_REG(0xA62C)
+#define   MSG_IDLE_INDICATION			REG_BIT(0)
+#define   MSG_IDLE_C6_ALLOWED			REG_BIT(1)
+#define   MSG_IDLE_FW_REQ			REG_GENMASK(13, 9)
+
 #define XE2LPM_CCCHKNREG1			XE_REG(0x82a8)
 
 #define VF_PREEMPTION				XE_REG(0x83a4, XE_REG_OPTION_MASKED)
@@ -441,6 +468,7 @@
 #define   L3_SQ_DISABLE_COAMA_2WAY_COH		REG_BIT(30)
 #define   L3_SQ_DISABLE_COAMA			REG_BIT(22)
 #define   COMPMEMRD256BOVRFETCHEN		REG_BIT(20)
+#define   NONCOMPMEMRD256BOVRFETCHEN		REG_BIT(14)
 
 #define L3SQCREG3				XE_REG_MCR(0xb108)
 #define   COMPPWOVERFETCHEN			REG_BIT(28)
@@ -609,7 +637,7 @@
  *   [4-6]     RSVD
  *   [7]       Disabled
  */
-#define CCS_MODE				XE_REG(0x14804, XE_REG_OPTION_MASKED)
+#define CCS_MODE				XE_REG(0x14804)
 #define   CCS_MODE_CSLICE_0_3_MASK		REG_GENMASK(11, 0) /* 3 bits per cslice */
 #define   CCS_MODE_CSLICE_MASK			0x7 /* CCS0-3 + rsvd */
 #define   CCS_MODE_CSLICE_WIDTH			ilog2(CCS_MODE_CSLICE_MASK + 1)
@@ -634,6 +662,12 @@
 #define GT_GFX_RC6_LOCKED			XE_REG(0x138104)
 #define GT_GFX_RC6				XE_REG(0x138108)
 
+#define GT_IA_PERF_BIAS_REG			XE_REG(0x138158)
+#define   GT_BIAS				REG_GENMASK(31, 16)
+#define   IA_BIAS				REG_GENMASK(15, 0)
+#define   GT_BIAS_DEFAULT			0x10
+#define   IA_BIAS_DEFAULT			0x10
+
 #define GT0_PERF_LIMIT_REASONS			XE_REG(0x1381a8)
 /* Common performance limit reason bits - available on all platforms */
 #define   GT0_PERF_LIMIT_REASONS_MASK		0xde3
@@ -651,6 +685,7 @@
 #define   MEM_THERMAL_MASK			REG_BIT(2)
 #define   VR_THERMAL_MASK			REG_BIT(3)
 #define   ICCMAX_MASK				REG_BIT(4)
+#define   PWRBRK_MASK				REG_BIT(5)
 #define   SOC_AVG_THERMAL_MASK			REG_BIT(6)
 #define   FASTVMODE_MASK			REG_BIT(7)
 #define   PSYS_PL1_MASK				REG_BIT(12)
diff --git a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h
index 4ab86fc369fdfa..6f7abc0181b544 100644
--- a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h
+++ b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h
@@ -37,6 +37,8 @@
 #define CTX_QUEUE_TIMESTAMP		(0xd0 + 1)
 #define CTX_QUEUE_TIMESTAMP_UDW		(0xd2 + 1)
 
+#define CTX_CS_CHICKEN1			(0x12e + 1)
+
 #define INDIRECT_CTX_RING_HEAD		(0x02 + 1)
 #define INDIRECT_CTX_RING_TAIL		(0x04 + 1)
 #define INDIRECT_CTX_RING_START		(0x06 + 1)
diff --git a/drivers/gpu/drm/xe/regs/xe_regs.h b/drivers/gpu/drm/xe/regs/xe_regs.h
index ef4746b7b5d342..437485b5a0af9b 100644
--- a/drivers/gpu/drm/xe/regs/xe_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_regs.h
@@ -30,6 +30,8 @@
 #define XEHP_MTCFG_ADDR				XE_REG(0x101800)
 #define   TILE_COUNT				REG_GENMASK(15, 8)
 
+#define CRI_DEVICE_UID				XE_REG(0x102008)
+
 #define GGC					XE_REG(0x108040)
 #define   GMS_MASK				REG_GENMASK(15, 8)
 #define   GGMS_MASK				REG_GENMASK(7, 6)
diff --git a/drivers/gpu/drm/xe/tests/Makefile b/drivers/gpu/drm/xe/tests/Makefile
index f7aa47f11a3687..0b809a252cfa33 100644
--- a/drivers/gpu/drm/xe/tests/Makefile
+++ b/drivers/gpu/drm/xe/tests/Makefile
@@ -7,6 +7,7 @@ xe_live_test-y = xe_live_test_mod.o
 # Normal kunit tests
 obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_test.o
 xe_test-y = xe_test_mod.o \
+	xe_any_kunit.o \
 	xe_args_test.o \
 	xe_pci_test.o \
 	xe_rtp_tables_test.o \
diff --git a/drivers/gpu/drm/xe/tests/xe_any_kunit.c b/drivers/gpu/drm/xe/tests/xe_any_kunit.c
new file mode 100644
index 00000000000000..0a5f28894cd2b5
--- /dev/null
+++ b/drivers/gpu/drm/xe/tests/xe_any_kunit.c
@@ -0,0 +1,213 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <kunit/test.h>
+
+#include "tests/xe_kunit_helpers.h"
+#include "tests/xe_pci_test.h"
+#include "xe_any.h"
+#include "xe_device.h"
+
+static void test_to_xe(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	const struct xe_device *cxe = xe;
+	const struct xe_tile *ctile = tile;
+	const struct xe_gt *cgt = gt;
+
+	KUNIT_EXPECT_PTR_EQ(test, xe, xe_any_to_xe(xe));
+	KUNIT_EXPECT_PTR_EQ(test, xe, xe_any_to_xe(tile));
+	KUNIT_EXPECT_PTR_EQ(test, xe, xe_any_to_xe(gt));
+	KUNIT_EXPECT_PTR_EQ(test, xe, xe_any_to_xe(drm));
+	KUNIT_EXPECT_PTR_EQ(test, xe, xe_any_to_xe(dev));
+	KUNIT_EXPECT_PTR_EQ(test, xe, xe_any_to_xe(pdev));
+
+	KUNIT_EXPECT_PTR_EQ(test, cxe, xe_any_to_xe(cxe));
+	KUNIT_EXPECT_PTR_EQ(test, cxe, xe_any_to_xe(ctile));
+	KUNIT_EXPECT_PTR_EQ(test, cxe, xe_any_to_xe(cgt));
+}
+
+static void test_to_pdev(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(xe));
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(tile));
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(gt));
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(drm));
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(dev));
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(pdev));
+
+	/* mimic early probe stage */
+	dev_set_drvdata(xe->drm.dev, NULL);
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(dev));
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_to_pdev(pdev));
+}
+
+static void test_to_dev(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(xe));
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(tile));
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(gt));
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(drm));
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(dev));
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(pdev));
+
+	/* mimic early probe stage */
+	dev_set_drvdata(xe->drm.dev, NULL);
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(dev));
+	KUNIT_EXPECT_PTR_EQ(test, dev, xe_any_to_dev(pdev));
+}
+
+static void test_to_drm(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	KUNIT_EXPECT_PTR_EQ(test, drm, xe_any_to_drm(xe));
+	KUNIT_EXPECT_PTR_EQ(test, drm, xe_any_to_drm(tile));
+	KUNIT_EXPECT_PTR_EQ(test, drm, xe_any_to_drm(gt));
+	KUNIT_EXPECT_PTR_EQ(test, drm, xe_any_to_drm(drm));
+	KUNIT_EXPECT_PTR_EQ(test, drm, xe_any_to_drm(dev));
+	KUNIT_EXPECT_PTR_EQ(test, drm, xe_any_to_drm(pdev));
+}
+
+static void test_if_pdev(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	KUNIT_EXPECT_TRUE(test, xe && drm && tile && gt && dev && pdev);
+	KUNIT_EXPECT_NULL(test, xe_any_if_pdev(xe));
+	KUNIT_EXPECT_NULL(test, xe_any_if_pdev(tile));
+	KUNIT_EXPECT_NULL(test, xe_any_if_pdev(gt));
+	KUNIT_EXPECT_NULL(test, xe_any_if_pdev(drm));
+	KUNIT_EXPECT_NULL(test, xe_any_if_pdev(dev));
+	KUNIT_EXPECT_PTR_EQ(test, pdev, xe_any_if_pdev(pdev));
+}
+
+static void test_if_xe(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	KUNIT_EXPECT_TRUE(test, xe && drm && tile && gt && dev && pdev);
+	KUNIT_EXPECT_PTR_EQ(test, xe, xe_any_if_xe(xe));
+	KUNIT_EXPECT_NULL(test, xe_any_if_xe(tile));
+	KUNIT_EXPECT_NULL(test, xe_any_if_xe(gt));
+	KUNIT_EXPECT_NULL(test, xe_any_if_xe(drm));
+	KUNIT_EXPECT_NULL(test, xe_any_if_xe(dev));
+	KUNIT_EXPECT_NULL(test, xe_any_if_xe(pdev));
+}
+
+static void test_if_tile(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	KUNIT_EXPECT_TRUE(test, xe && drm && tile && gt && dev && pdev);
+	KUNIT_EXPECT_NULL(test, xe_any_if_tile(xe));
+	KUNIT_EXPECT_PTR_EQ(test, tile, xe_any_if_tile(tile));
+	KUNIT_EXPECT_NULL(test, xe_any_if_tile(gt));
+	KUNIT_EXPECT_NULL(test, xe_any_if_tile(drm));
+	KUNIT_EXPECT_NULL(test, xe_any_if_tile(dev));
+	KUNIT_EXPECT_NULL(test, xe_any_if_tile(pdev));
+}
+
+static void test_if_gt(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct drm_device *drm = &xe->drm;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	KUNIT_EXPECT_TRUE(test, xe && drm && tile && gt && dev && pdev);
+	KUNIT_EXPECT_NULL(test, xe_any_if_gt(xe));
+	KUNIT_EXPECT_NULL(test, xe_any_if_gt(tile));
+	KUNIT_EXPECT_PTR_EQ(test, gt, xe_any_if_gt(gt));
+	KUNIT_EXPECT_NULL(test, xe_any_if_gt(drm));
+	KUNIT_EXPECT_NULL(test, xe_any_if_gt(dev));
+	KUNIT_EXPECT_NULL(test, xe_any_if_gt(pdev));
+}
+
+static void test_to_id(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_gt *gt = tile->primary_gt;
+	struct device *dev = xe->drm.dev;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	const struct xe_device *cxe = xe;
+	const struct xe_tile *ctile = tile;
+	const struct xe_gt *cgt = gt;
+
+	tile->id = 1;
+	gt->info.id = 2;
+
+	KUNIT_EXPECT_EQ(test, 0, xe_any_id(xe));
+	KUNIT_EXPECT_EQ(test, 1, xe_any_id(tile));
+	KUNIT_EXPECT_EQ(test, 2, xe_any_id(gt));
+	KUNIT_EXPECT_EQ(test, 0, xe_any_id(dev));
+	KUNIT_EXPECT_EQ(test, 0, xe_any_id(pdev));
+	KUNIT_EXPECT_EQ(test, 0, xe_any_id(cxe));
+	KUNIT_EXPECT_EQ(test, 1, xe_any_id(ctile));
+	KUNIT_EXPECT_EQ(test, 2, xe_any_id(cgt));
+}
+
+static struct kunit_case xe_any_tests[] = {
+	KUNIT_CASE(test_to_xe),
+	KUNIT_CASE(test_to_dev),
+	KUNIT_CASE(test_to_pdev),
+	KUNIT_CASE(test_to_drm),
+	KUNIT_CASE(test_if_pdev),
+	KUNIT_CASE(test_if_xe),
+	KUNIT_CASE(test_if_tile),
+	KUNIT_CASE(test_if_gt),
+	KUNIT_CASE(test_to_id),
+	{}
+};
+
+static struct kunit_suite xe_any_test_suite = {
+	.name = "xe_any",
+	.test_cases = xe_any_tests,
+	.init = xe_kunit_helper_xe_device_test_init,
+};
+
+kunit_test_suite(xe_any_test_suite);
diff --git a/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c
index 82869363ab7e46..4f7c81c9ebb8d2 100644
--- a/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c
+++ b/drivers/gpu/drm/xe/tests/xe_guc_klv_helpers_kunit.c
@@ -373,6 +373,305 @@ static void test_encode_object_basic(struct kunit *test)
 						     obj_echo_encoder));
 }
 
+static const struct {
+	u32 klvs[4];
+	int expected_err;
+	u16 expected_val;
+	const char *name;
+} decode_u16_params[] = {
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), ~0 },
+		.expected_err = -ENODATA,
+		.expected_val = 0xdead,
+		.name = "empty",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), 1, ~0 },
+		.expected_err = 0,
+		.expected_val = 1,
+		.name = "valid",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), U16_MAX, ~0 },
+		.expected_err = 0,
+		.expected_val = U16_MAX,
+		.name = "max",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), U16_MAX + 1, ~0 },
+		.expected_err = -EOVERFLOW,
+		.expected_val = 0xdead,
+		.name = "big",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), 1, 2, ~0 },
+		.expected_err = -ENOMSG,
+		.expected_val = 0xdead,
+		.name = "long",
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(decode_u16, decode_u16_params, name);
+
+static void test_decode_u16(struct kunit *test)
+{
+	typeof(decode_u16_params[0]) *param = test->param_value;
+	const u32 *klvs = param->klvs;
+	const u32 *value = klvs + GUC_KLV_LEN_MIN;
+	u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
+	u16 data = 0xdead;
+
+	KUNIT_ASSERT_EQ(test, param->expected_err, xe_guc_klv_decode_u16(value, len, &data));
+	KUNIT_ASSERT_EQ(test, param->expected_val, data);
+}
+
+static const struct {
+	u32 klvs[4];
+	int expected_err;
+	u32 expected_val;
+	const char *name;
+} decode_u32_params[] = {
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), ~0 },
+		.expected_err = -ENODATA,
+		.expected_val = 0xdead,
+		.name = "empty",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), 1, ~0 },
+		.expected_err = 0,
+		.expected_val = 1,
+		.name = "valid",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), U32_MAX, ~0 },
+		.expected_err = 0,
+		.expected_val = U32_MAX,
+		.name = "max",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), 1, 2, ~0 },
+		.expected_err = -ENOMSG,
+		.expected_val = 0xdead,
+		.name = "long",
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(decode_u32, decode_u32_params, name);
+
+static void test_decode_u32(struct kunit *test)
+{
+	typeof(decode_u32_params[0]) *param = test->param_value;
+	const u32 *klvs = param->klvs;
+	const u32 *value = klvs + GUC_KLV_LEN_MIN;
+	u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
+	u32 data = 0xdead;
+
+	KUNIT_ASSERT_EQ(test, param->expected_err, xe_guc_klv_decode_u32(value, len, &data));
+	KUNIT_ASSERT_EQ(test, param->expected_val, data);
+}
+
+static const struct {
+	u32 klvs[4];
+	int expected_err;
+	u64 expected_val;
+	const char *name;
+} decode_u64_params[] = {
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), ~0 },
+		.expected_err = -ENODATA,
+		.expected_val = 0xdead,
+		.name = "empty",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 1), 1, ~0, },
+		.expected_err = -ENODATA,
+		.expected_val = 0xdead,
+		.name = "short",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), 1, 2, ~0 },
+		.expected_err = 0,
+		.expected_val = 0x200000001,
+		.name = "valid",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 2), U32_MAX, U32_MAX, 3 },
+		.expected_err = 0,
+		.expected_val = U64_MAX,
+		.name = "max",
+	},
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 3), 1, 2, 3, },
+		.expected_err = -ENOMSG,
+		.expected_val = 0xdead,
+		.name = "long",
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(decode_u64, decode_u64_params, name);
+
+static void test_decode_u64(struct kunit *test)
+{
+	typeof(decode_u64_params[0]) *param = test->param_value;
+	const u32 *klvs = param->klvs;
+	const u32 *value = klvs + GUC_KLV_LEN_MIN;
+	u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
+	u64 data = 0xdead;
+
+	KUNIT_ASSERT_EQ(test, param->expected_err, xe_guc_klv_decode_u64(value, len, &data));
+	KUNIT_ASSERT_EQ(test, param->expected_val, data);
+}
+
+#define make_u32(a, b, c, d) \
+	(FIELD_PREP_CONST(GENMASK_U32(31, 24), (a)) | \
+	 FIELD_PREP_CONST(GENMASK_U32(23, 16), (b)) | \
+	 FIELD_PREP_CONST(GENMASK_U32(15, 8), (c)) | \
+	 FIELD_PREP_CONST(GENMASK_U32(7, 0), (d)))
+
+static const struct {
+	u32 klvs[4];
+	int expected_err;
+	const char *expected_str;
+	const char *name;
+} decode_str_params[] = {
+	{
+		.klvs = { PREP_GUC_KLV_CONST(TEST_KEY, 0), 'a', ~0 },
+		.expected_err = -ENOMSG,
+		.expected_str = NULL,
+		.name = "empty",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 1),
+				make_u32(0, 0, 0, '1'),
+			},
+		.expected_err = 0,
+		.expected_str = "1",
+		.name = "one",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 1),
+				make_u32('3', '2', 0, '1'),
+				'4', ~0
+			},
+		.expected_err = 0,
+		.expected_str = "1",
+		.name = "one_trash",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 1),
+				make_u32(0, 0, '2', '1'),
+				'3', ~0
+			},
+		.expected_err = 0,
+		.expected_str = "12",
+		.name = "two",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 1),
+				make_u32('3', 0, '2', '1'),
+				'4', ~0
+			},
+		.expected_err = 0,
+		.expected_str = "12",
+		.name = "two_trash",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 1),
+				make_u32(0, '3', '2', '1'),
+				'4', ~0
+			},
+		.expected_err = 0,
+		.expected_str = "123",
+		.name = "three",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 1),
+				make_u32('4', '3', '2', '1'),
+				'5', ~0
+			},
+		.expected_err = 0,
+		.expected_str = "1234",
+		.name = "dword",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 2),
+				make_u32('4', '3', '2', '1'),
+				0, ~0
+			},
+		.expected_err = 0,
+		.expected_str = "1234",
+		.name = "four",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 2),
+				make_u32('4', '3', '2', '1'),
+				make_u32(0, 0, 0, '5'),
+				~0,
+			},
+		.expected_err = 0,
+		.expected_str = "12345",
+		.name = "five",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 2),
+				make_u32('4', '3', '2', '1'),
+				make_u32(0, 0, '6', '5'),
+				~0,
+			},
+		.expected_err = 0,
+		.expected_str = "123456",
+		.name = "six",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 2),
+				make_u32('4', '3', '2', '1'),
+				make_u32(0, '7', '6', '5'),
+				~0,
+			},
+		.expected_err = 0,
+		.expected_str = "1234567",
+		.name = "seven",
+	},
+	{
+		.klvs = {
+				PREP_GUC_KLV_CONST(TEST_KEY, 2),
+				make_u32('4', '3', '2', '1'),
+				make_u32('8', '7', '6', '5'),
+				~0,
+			},
+		.expected_err = 0,
+		.expected_str = "12345678",
+		.name = "qword",
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(decode_str, decode_str_params, name);
+
+static void test_decode_str(struct kunit *test)
+{
+	typeof(decode_str_params[0]) *param = test->param_value;
+	const u32 *klvs = param->klvs;
+	const u32 *value = klvs + GUC_KLV_LEN_MIN;
+	u16 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
+	char *data = NULL;
+
+	KUNIT_EXPECT_EQ(test, param->expected_err, xe_guc_klv_decode_string(value, len, &data));
+	if (!param->expected_err)
+		KUNIT_EXPECT_STREQ(test, param->expected_str, data);
+	kfree(data);
+}
+
 static void __drm_printfn_kunit(struct drm_printer *p, struct va_format *vaf)
 {
 	struct kunit *test = p->arg;
@@ -417,6 +716,10 @@ static struct kunit_case guc_klv_helpers_test_cases[] = {
 	KUNIT_CASE(test_encode_object_klv),
 	KUNIT_CASE(test_encode_object_nested),
 	KUNIT_CASE(test_encode_object_basic),
+	KUNIT_CASE_PARAM(test_decode_u16, decode_u16_gen_params),
+	KUNIT_CASE_PARAM(test_decode_u32, decode_u32_gen_params),
+	KUNIT_CASE_PARAM(test_decode_u64, decode_u64_gen_params),
+	KUNIT_CASE_PARAM(test_decode_str, decode_str_gen_params),
 	KUNIT_CASE(test_print),
 	{}
 };
diff --git a/drivers/gpu/drm/xe/tests/xe_kunit_helpers.c b/drivers/gpu/drm/xe/tests/xe_kunit_helpers.c
index bc5156966ce961..27740b40c8ae9f 100644
--- a/drivers/gpu/drm/xe/tests/xe_kunit_helpers.c
+++ b/drivers/gpu/drm/xe/tests/xe_kunit_helpers.c
@@ -39,6 +39,10 @@ struct xe_device *xe_kunit_helper_alloc_xe_device(struct kunit *test,
 					       struct xe_device,
 					       drm, DRIVER_GEM);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xe);
+
+	dev_set_drvdata(xe->drm.dev, &xe->drm);
+	KUNIT_ASSERT_PTR_EQ(test, xe, kdev_to_xe_device(dev));
+
 	return xe;
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_kunit_helper_alloc_xe_device);
diff --git a/drivers/gpu/drm/xe/tests/xe_log_kunit.c b/drivers/gpu/drm/xe/tests/xe_log_kunit.c
new file mode 100644
index 00000000000000..56c36c8a08e183
--- /dev/null
+++ b/drivers/gpu/drm/xe/tests/xe_log_kunit.c
@@ -0,0 +1,553 @@
+// SPDX-License-Identifier: GPL-2.0 AND MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <kunit/static_stub.h>
+#include <kunit/test.h>
+#include <kunit/test-bug.h>
+
+#include "tests/xe_kunit_helpers.h"
+#include "tests/xe_pci_test.h"
+#include "xe_device.h"
+#include "xe_log.h"
+
+static void nop_dmesg_vprintk(struct pci_dev *pdev, int cper_sev, struct va_format *vaf)
+{
+}
+
+static void nop_emit_cper(struct pci_dev *pdev, int cper_sev,
+			  enum xe_sigid sigid, u32 component, u32 location,
+			  const void *data, size_t len, struct va_format *vaf)
+{
+}
+
+static const char *component_name(u32 component)
+{
+	switch (component) {
+#define make_component_tag_case(_CLASS, _ID, _TAG, _SIG, _NAME) \
+	case XE_LOG_COMPONENT_##_TAG: return _NAME;
+	DEFINE_XE_LOG_COMPONENTS(make_component_tag_case)
+#undef make_component_tag_case
+	}
+	return component ? "???" : "";
+}
+
+static const char *location_type(u32 location)
+{
+	u32 type = FIELD_GET(XE_LOG_LOCATION_TYPE_MASK, location);
+
+	return type == XE_LOG_LOCATION_TYPE_DEVICE ? "DEVICE" :
+	       type == XE_LOG_LOCATION_TYPE_TILE ? "TILE" :
+	       type == XE_LOG_LOCATION_TYPE_GT ? "GT" :
+	       location ? "?" : "";
+}
+
+static void fake_emit_cper(struct pci_dev *pdev, int cper_sev,
+			   enum xe_sigid sigid, u32 component, u32 location,
+			   const void *data, size_t len, struct va_format *vaf)
+{
+	char msg[64];
+	int n;
+
+	pr_info("\n");
+	pr_info("CPER SEV=%u SIGID=%u\n", cper_sev, sigid);
+	pr_info("CPER DEVICE=%s\n", dev_name(&pdev->dev));
+	if (location)
+		pr_info("CPER LOCATION=%#x \t# %s.%u\n",
+			location, location_type(location),
+			FIELD_GET(XE_LOG_LOCATION_ID_MASK, location));
+	if (component)
+		pr_info("CPER COMPONENT=%#x \t# %s\n",
+			component, component_name(component));
+	if (IS_ERR(data))
+		pr_info("CPER ERR=%ld \t\t# %pe\n", PTR_ERR(data), data);
+	else if (len)
+		print_hex_dump(KERN_INFO, "CPER BIN=", DUMP_PREFIX_OFFSET,
+			       16, 1, data, len, false);
+
+	n = vscnprintf(msg, sizeof(msg), vaf->fmt, *vaf->va);
+	print_hex_dump(KERN_INFO, "CPER MSG=", DUMP_PREFIX_OFFSET, 16, 1, msg, n, true);
+	pr_info("CPER END\n");
+}
+
+static const u8 blob[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
+static const u8 dead[] = { 0xde, 0xad, 0xbe, 0xef };
+
+static struct xe_tile *to_tile_safe(struct xe_device *xe)
+{
+	return xe ? &xe->tiles[1] : NULL;
+}
+
+static struct xe_gt *to_gt_safe(struct xe_device *xe)
+{
+	return xe ? to_tile_safe(xe)->primary_gt : NULL;
+}
+
+static struct pci_dev *to_pdev_safe(struct xe_device *xe)
+{
+	return xe ? xe_any_to_pdev(xe) : NULL;
+}
+
+static void demo(struct xe_device *xe)
+{
+	struct pci_dev *pdev = xe_any_to_pdev(xe);
+	struct xe_tile *tile = to_tile_safe(xe);
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	/* SW errno */
+	xe_log_emit(pdev, CPER_SEV_FATAL, XE_SIGID_PROBE,
+		    XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+		    ERR_PTR(-ENODEV), 0, "testing %s signature\n", "software");
+
+	xe_log_err(tile, PROBE, -ENODEV, "testing %s signature\n", "software");
+	xe_log_err_corrected(gt, PROBE, -ENODEV, "testing %s signature\n", "software");
+	xe_log_info(gt, PROBE, "testing %s signature\n", "software");
+
+	/* HW data */
+	xe_log_emit(pdev, CPER_SEV_FATAL, XE_SIGID_PCIE,
+		    XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+		    blob, sizeof(blob), "testing %s signature\n", "HARDWARE");
+	xe_log_emit_recoverable(pdev, XE_SIGID_FABRIC,
+				XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+				blob, sizeof(blob), "testing %s signature\n", "HARDWARE");
+	xe_log_from_corrected(tile, XE_SIGID_DEVICE_MEMORY, XE_LOG_COMPONENT_NONE,
+			      blob, sizeof(blob), "testing %s signature\n", "HARDWARE");
+	xe_log_from_info(gt, XE_SIGID_CORE_COMPUTE, XE_LOG_COMPONENT_NONE,
+			 blob, sizeof(blob), "testing %s signature\n", "HARDWARE");
+}
+
+static void demo_dmesg(struct kunit *test)
+{
+	kunit_activate_static_stub(test, log_emit_cper, nop_emit_cper);
+	demo(test->priv);
+}
+
+static void demo_cper(struct kunit *test)
+{
+	kunit_activate_static_stub(test, log_emit_cper, fake_emit_cper);
+	kunit_activate_static_stub(test, log_dmesg_vprintk, nop_dmesg_vprintk);
+	demo(test->priv);
+}
+
+static const char *test_fatal(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_emit_fatal(pdev, XE_SIGID_PROBE,
+				  XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+				  NULL, 0, "testing %d\n", 123);
+	return "SIGID=101 FATAL testing 123\n";
+}
+
+static const char *test_fatal_tile(struct xe_device *xe)
+{
+	struct xe_tile *tile = to_tile_safe(xe);
+
+	if (tile)
+		xe_log_from_fatal(tile, XE_SIGID_PROBE, XE_LOG_COMPONENT_NONE,
+				  ERR_PTR(-ENODEV), 0, "testing %d\n", 123);
+	return "SIGID=101 FATAL (-ENODEV) Tile1: testing 123\n";
+}
+
+static const char *test_fatal_gt(struct xe_device *xe)
+{
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (gt)
+		xe_log_from_fatal(gt, XE_SIGID_PROBE, XE_LOG_COMPONENT_NONE,
+				  ERR_PTR(-ENODEV), 0, "testing %d\n", 123);
+	return "SIGID=101 FATAL (-ENODEV) Tile1: GT1: testing 123\n";
+}
+
+static const char *test_fatal_comp(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_err_fatal(pdev, PROBE, -ENODEV, "testing %d\n", 123);
+	return "SIGID=101 FATAL (-ENODEV) PROBE: testing 123\n";
+}
+
+static const char *test_fatal_comp_tile(struct xe_device *xe)
+{
+	struct xe_tile *tile = to_tile_safe(xe);
+
+	if (tile)
+		xe_log_err_fatal(tile, PROBE, -ENODEV, "testing %d\n", 123);
+	return "SIGID=101 FATAL (-ENODEV) Tile1: PROBE: testing 123\n";
+}
+
+static const char *test_fatal_comp_gt(struct xe_device *xe)
+{
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (gt)
+		xe_log_err_fatal(gt, PROBE, -ENODEV, "testing %d\n", 123);
+	return "SIGID=101 FATAL (-ENODEV) Tile1: GT1: PROBE: testing 123\n";
+}
+
+static const char *test_fatal_all(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (pdev)
+		xe_log_emit(pdev, CPER_SEV_FATAL, XE_SIGID_PROBE,
+			    XE_LOG_COMPONENT_PROBE, MAKE_XE_LOG_LOCATION(GT, gt->info.id),
+			    ERR_PTR(-ENODEV), 0, "testing %d\n", 123);
+	return "SIGID=101 FATAL (-ENODEV) Tile1: GT1: PROBE: testing 123\n";
+}
+
+static const char *test_recoverable(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_emit_recoverable(pdev, XE_SIGID_RUNTIME_FW,
+					XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+					ERR_PTR(-EIO), 0, "testing %d\n", 123);
+	return "SIGID=104 (-EIO) testing 123\n";
+}
+
+static const char *test_recoverable_tile(struct xe_device *xe)
+{
+	struct xe_tile *tile = to_tile_safe(xe);
+
+	if (tile)
+		xe_log_from_recoverable(tile, XE_SIGID_RUNTIME_FW, XE_LOG_COMPONENT_NONE,
+					ERR_PTR(-EIO), 0, "testing %d\n", 123);
+	return "SIGID=104 (-EIO) Tile1: testing 123\n";
+}
+
+static const char *test_recoverable_gt(struct xe_device *xe)
+{
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (gt)
+		xe_log_from_recoverable(gt, XE_SIGID_RUNTIME_FW, XE_LOG_COMPONENT_NONE,
+					ERR_PTR(-EIO), 0, "testing %d\n", 123);
+	return "SIGID=104 (-EIO) Tile1: GT1: testing 123\n";
+}
+
+static const char *test_recoverable_comp(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_err(pdev, GUC, -EIO, "testing %d\n", 123);
+	return "SIGID=104 (-EIO) GUC: testing 123\n";
+}
+
+static const char *test_recoverable_comp_tile(struct xe_device *xe)
+{
+	struct xe_tile *tile = to_tile_safe(xe);
+
+	if (tile)
+		xe_log_err(tile, GUC, -EIO, "testing %d\n", 123);
+	return "SIGID=104 (-EIO) Tile1: GUC: testing 123\n";
+}
+
+static const char *test_recoverable_comp_gt(struct xe_device *xe)
+{
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (gt)
+		xe_log_err(gt, GUC, -EIO, "testing %d\n", 123);
+	return "SIGID=104 (-EIO) Tile1: GT1: GUC: testing 123\n";
+}
+
+static const char *test_recoverable_all(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (pdev)
+		xe_log_emit(pdev, CPER_SEV_RECOVERABLE, XE_SIGID_RUNTIME_FW,
+			    XE_LOG_COMPONENT_GUC, MAKE_XE_LOG_LOCATION(GT, gt->info.id),
+			    ERR_PTR(-EIO), 0, "testing %d\n", 123);
+	return "SIGID=104 (-EIO) Tile1: GT1: GUC: testing 123\n";
+}
+
+static const char *test_info(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_emit(pdev, CPER_SEV_INFORMATIONAL, XE_SIGID_DEVICE_FW,
+			    XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+			    NULL, 0, "testing %d\n", 123);
+	return "SIGID=105 testing 123\n";
+}
+
+static const char *test_info_tile(struct xe_device *xe)
+{
+	struct xe_tile *tile = to_tile_safe(xe);
+
+	if (tile)
+		xe_log_from_info(tile, XE_SIGID_DEVICE_FW, XE_LOG_COMPONENT_NONE,
+				 NULL, 0, "testing %d\n", 123);
+	return "SIGID=105 Tile1: testing 123\n";
+}
+
+static const char *test_info_gt(struct xe_device *xe)
+{
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (gt)
+		xe_log_from_info(gt, XE_SIGID_DEVICE_FW, XE_LOG_COMPONENT_NONE,
+				 NULL, 0, "testing %d\n", 123);
+	return "SIGID=105 Tile1: GT1: testing 123\n";
+}
+
+static const char *test_info_err(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_err_info(pdev, PCODE, -EPROTO, "testing %d\n", 123);
+	return "SIGID=105 (-EPROTO) PCODE: testing 123\n";
+}
+
+static const char *test_info_comp(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_info(pdev, PCODE, "testing %d\n", 123);
+	return "SIGID=105 PCODE: testing 123\n";
+}
+
+static const char *test_info_comp_tile(struct xe_device *xe)
+{
+	struct xe_tile *tile = to_tile_safe(xe);
+
+	if (tile)
+		xe_log_info(tile, PCODE, "testing %d\n", 123);
+	return "SIGID=105 Tile1: PCODE: testing 123\n";
+}
+
+static const char *test_info_comp_gt(struct xe_device *xe)
+{
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (gt)
+		xe_log_info(gt, PCODE, "testing %d\n", 123);
+	return "SIGID=105 Tile1: GT1: PCODE: testing 123\n";
+}
+
+static const char *test_info_all(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+	struct xe_gt *gt = to_gt_safe(xe);
+
+	if (pdev)
+		xe_log_emit(pdev, CPER_SEV_INFORMATIONAL, XE_SIGID_DEVICE_FW,
+			    XE_LOG_COMPONENT_PCODE, MAKE_XE_LOG_LOCATION(GT, gt->info.id),
+			    NULL, 0, "testing %d\n", 123);
+	return "SIGID=105 Tile1: GT1: PCODE: testing 123\n";
+}
+
+static const char *test_hw_fatal(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_emit_fatal(pdev, XE_SIGID_PCIE,
+				  XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+				  dead, sizeof(dead), "testing %d\n", 123);
+	return "SIGID=201 FATAL (deadbeef) " HW_ERR "testing 123\n";
+}
+
+static const char *test_hw_recoverable(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_emit_recoverable(pdev, XE_SIGID_DEVICE_MEMORY,
+					XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+					dead, sizeof(dead), "testing %d\n", 123);
+	return "SIGID=202 (deadbeef) " HW_ERR "testing 123\n";
+}
+
+static const char *test_hw_corrected(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_emit_corrected(pdev, XE_SIGID_DEVICE_MEMORY,
+				      XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+				      dead, sizeof(dead), "testing %d\n", 123);
+	return "SIGID=202 CORRECTED (deadbeef) " HW_ERR "testing 123\n";
+}
+
+static const char *test_hw_informational(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pdev_safe(xe);
+
+	if (pdev)
+		xe_log_emit_info(pdev, XE_SIGID_DEVICE_MEMORY,
+				 XE_LOG_COMPONENT_NONE, XE_LOG_LOCATION_NONE,
+				 dead, sizeof(dead), "testing %d\n", 123);
+	return "SIGID=202 (deadbeef) testing 123\n";
+}
+
+static const struct log_test_param {
+	const char *(*func)(struct xe_device *xe);
+} log_test_params[] = {
+	{ .func = test_fatal },
+	{ .func = test_fatal_tile },
+	{ .func = test_fatal_gt },
+	{ .func = test_fatal_comp },
+	{ .func = test_fatal_comp_tile },
+	{ .func = test_fatal_comp_gt },
+	{ .func = test_fatal_all },
+	{ .func = test_recoverable },
+	{ .func = test_recoverable_tile },
+	{ .func = test_recoverable_gt },
+	{ .func = test_recoverable_comp },
+	{ .func = test_recoverable_comp_tile },
+	{ .func = test_recoverable_comp_gt },
+	{ .func = test_recoverable_all },
+	{ .func = test_info },
+	{ .func = test_info_tile },
+	{ .func = test_info_gt },
+	{ .func = test_info_err },
+	{ .func = test_info_comp },
+	{ .func = test_info_comp_tile },
+	{ .func = test_info_comp_gt },
+	{ .func = test_info_all },
+	{ .func = test_hw_fatal },
+	{ .func = test_hw_recoverable },
+	{ .func = test_hw_corrected },
+	{ .func = test_hw_informational },
+};
+
+static void log_param_get_desc(const struct log_test_param *p, char *desc)
+{
+	snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%ps", p->func);
+}
+
+KUNIT_ARRAY_PARAM(log, log_test_params, log_param_get_desc);
+
+static void check_dmesg_vprintk(struct pci_dev *pdev, int cper_sev, struct va_format *vaf)
+{
+	struct kunit *test = kunit_get_current_test();
+	const struct log_test_param *param = test->param_value;
+	const char *exp = param->func(NULL);
+	char msg[64];
+	int n;
+
+	n = vsnprintf(msg, sizeof(msg), vaf->fmt, *vaf->va);
+	KUNIT_EXPECT_LT(test, n, sizeof(msg));
+	KUNIT_EXPECT_STREQ(test, msg, exp);
+}
+
+static void test_dmesg(struct kunit *test)
+{
+	const struct log_test_param *param = test->param_value;
+
+	kunit_activate_static_stub(test, log_emit_cper, nop_emit_cper);
+	kunit_activate_static_stub(test, log_dmesg_vprintk, check_dmesg_vprintk);
+
+	param->func(test->priv);
+}
+
+#define INVALID_TILEID	(XE_MAX_TILES_PER_DEVICE + 1)
+#define INVALID_GTID	(XE_MAX_TILES_PER_DEVICE * XE_MAX_GT_PER_TILE + 1)
+
+#define PREP_TEST_LOCATION(type, id) \
+	(FIELD_PREP_CONST(XE_LOG_LOCATION_TYPE_MASK, (type)) | \
+	 FIELD_PREP_CONST(XE_LOG_LOCATION_ID_MASK, (id)))
+#define PREP_TEST_COMPONENT(class, type) \
+	(FIELD_PREP_CONST(XE_LOG_COMPONENT_CLASS_MASK, (class)) | \
+	 FIELD_PREP_CONST(XE_LOG_COMPONENT_TYPE_MASK, (type)))
+
+static const struct {
+	u32 comp;
+	u32 loc;
+	const char *name;
+} invalid_params[] = {
+	{	.name = "no-component no-location no-warn" },
+	{	.loc = PREP_TEST_LOCATION(0, 1),
+		.name = "reserved location" },
+	{	.loc = PREP_TEST_LOCATION(255, 0),
+		.name = "unknown location" },
+	{	.loc = PREP_TEST_LOCATION(XE_LOG_LOCATION_TYPE_DEVICE, 1),
+		.name = "nonzero-device-id location" },
+	{	.loc = PREP_TEST_LOCATION(XE_LOG_LOCATION_TYPE_TILE, INVALID_TILEID),
+		.name = "invalid-tile-id location" },
+	{	.loc = PREP_TEST_LOCATION(XE_LOG_LOCATION_TYPE_GT, INVALID_GTID),
+		.name = "invalid-gt-id location" },
+	{	.comp = PREP_TEST_COMPONENT(255, 0),
+		.name = "unknown component class" },
+	{	.comp = PREP_TEST_COMPONENT(XE_LOG_COMPONENT_CLASS_SYSTEM, 255),
+		.name = "unknown system component" },
+	{	.comp = PREP_TEST_COMPONENT(XE_LOG_COMPONENT_CLASS_HARDWARE, 255),
+		.name = "unknown hardware component" },
+	{	.comp = PREP_TEST_COMPONENT(255, 1),
+		.loc = PREP_TEST_LOCATION(255, 1),
+		.name = "unknown component and location" },
+};
+
+KUNIT_ARRAY_PARAM_DESC(invalid_param, invalid_params, name);
+
+static void test_invalid(struct kunit *test)
+{
+	struct xe_device *xe = test->priv;
+	typeof(invalid_params[0]) *param = test->param_value;
+
+	struct pci_dev *pdev = xe_any_to_pdev(xe);
+
+	if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))
+		kunit_skip(test, "requires CONFIG_DRM_XE_DEBUG\n");
+
+	kunit_activate_static_stub(test, log_emit_cper, nop_emit_cper);
+	kunit_activate_static_stub(test, log_dmesg_vprintk, nop_dmesg_vprintk);
+
+	kunit_warning_suppress(test) {
+		xe_log_emit(pdev, CPER_SEV_FATAL, XE_SIGID_PROBE,
+			    param->comp, param->loc,
+			    NULL, 0, "testing %s\n", param->name);
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, !!param->comp + !!param->loc);
+	}
+}
+
+static int xe_log_test_init(struct kunit *test)
+{
+	struct xe_pci_fake_data fake = {
+		.platform = XE_PVC, /* with max_remote_tiles != 0 */
+		.subplatform = XE_SUBPLATFORM_NONE,
+		.graphics_verx100 = 2001,
+		.media_verx100 = 2001,
+	};
+	struct xe_device *xe;
+
+	test->priv = &fake;
+	xe_kunit_helper_xe_device_test_init(test);
+	xe = test->priv;
+
+	KUNIT_ASSERT_NOT_NULL(test, to_tile_safe(xe));
+	KUNIT_ASSERT_NOT_NULL(test, to_gt_safe(xe));
+	KUNIT_EXPECT_EQ(test, 1, xe_any_id(to_tile_safe(xe)));
+	KUNIT_EXPECT_EQ(test, 1, xe_any_id(to_gt_safe(xe)));
+
+	return 0;
+}
+
+static struct kunit_case xe_log_test_cases[] = {
+	KUNIT_CASE(demo_cper),
+	KUNIT_CASE(demo_dmesg),
+	KUNIT_CASE_PARAM(test_dmesg, log_gen_params),
+	KUNIT_CASE_PARAM(test_invalid, invalid_param_gen_params),
+	{}
+};
+
+static struct kunit_suite xe_log_suite = {
+	.name = "xe_log",
+	.test_cases = xe_log_test_cases,
+	.init = xe_log_test_init,
+};
+
+kunit_test_suites(&xe_log_suite);
diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c
index bb0393475524aa..5aefc03c00c7aa 100644
--- a/drivers/gpu/drm/xe/tests/xe_pci.c
+++ b/drivers/gpu/drm/xe/tests/xe_pci.c
@@ -311,10 +311,11 @@ const void *xe_pci_id_gen_param(struct kunit *test, const void *prev, char *desc
 EXPORT_SYMBOL_IF_KUNIT(xe_pci_id_gen_param);
 
 static int fake_probe_info(struct xe_device *xe,
-			   const struct xe_device_desc *desc,
 			   struct xe_pci_fake_data *data,
 			   struct xe_probed_info *probed_info)
 {
+	const struct xe_device_desc *desc = xe->desc;
+
 	probed_info->tile_count = 1 + desc->max_remote_tiles;
 
 	if (!data || desc->pre_gmdid_graphics_ip) {
@@ -351,7 +352,7 @@ int xe_pci_fake_device_init(struct xe_device *xe)
 
 	if (!data) {
 		desc = (const void *)ent->driver_data;
-		subplatform_desc = NULL;
+		subplatform_desc = desc->subplatforms;
 		goto done;
 	}
 
@@ -364,25 +365,37 @@ int xe_pci_fake_device_init(struct xe_device *xe)
 	if (!ent->device)
 		return -ENODEV;
 
+	if (data->subplatform == XE_SUBPLATFORM_NONE) {
+		subplatform_desc = NULL;
+		goto done;
+	}
+
+	if (data->subplatform == XE_SUBPLATFORM_UNINITIALIZED) {
+		subplatform_desc = desc->subplatforms;
+		goto done;
+	}
+
 	for (subplatform_desc = desc->subplatforms;
 	     subplatform_desc && subplatform_desc->subplatform;
 	     subplatform_desc++)
 		if (subplatform_desc->subplatform == data->subplatform)
 			break;
 
-	if (data->subplatform != XE_SUBPLATFORM_NONE && !subplatform_desc)
+	if (!subplatform_desc || !subplatform_desc->subplatform)
 		return -ENODEV;
 
 done:
+	xe->desc = desc;
+	xe->subplatform_desc = subplatform_desc;
 	xe->sriov.__mode = data && data->sriov_mode ?
 			   data->sriov_mode : XE_SRIOV_MODE_NONE;
 
-	err = fake_probe_info(xe, desc, data, &probed_info);
+	err = fake_probe_info(xe, data, &probed_info);
 	if (err)
 		return err;
 
-	xe_info_init_early(xe, desc, subplatform_desc, &probed_info);
-	xe_info_init(xe, desc, &probed_info);
+	xe_info_init_early(xe, &probed_info);
+	xe_info_init(xe, &probed_info);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_any.h b/drivers/gpu/drm/xe/xe_any.h
new file mode 100644
index 00000000000000..5d97afa76915a1
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_any.h
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_ANY_H_
+#define _XE_ANY_H_
+
+#include "xe_device.h"
+
+#define __xe_any_to_self_assoc(type, any) \
+	const type * : (any), \
+	type * : (any)
+
+/**
+ * xe_any_if_type() - Get the pointer only if it is @type pointer.
+ * @any: any pointer
+ * @type: data type to look for
+ *
+ * Return: the @type pointer or NULL.
+ */
+#define xe_any_if_type(any, type)						\
+	_Generic((any),								\
+		 __xe_any_to_self_assoc(type, (any)),				\
+		 default : NULL)
+
+/**
+ * xe_any_if_gt() - Get the pointer only if it is &xe_gt.
+ * @any: any pointer
+ *
+ * Return: the @xe_gt pointer or NULL.
+ */
+#define xe_any_if_gt(any)	xe_any_if_type((any), struct xe_gt)
+
+/**
+ * xe_any_if_tile() - Get the pointer only if it is &xe_tile.
+ * @any: any pointer
+ *
+ * Return: the @xe_tile pointer or NULL.
+ */
+#define xe_any_if_tile(any)	xe_any_if_type((any), struct xe_tile)
+
+/**
+ * xe_any_if_xe() - Get the pointer only if it is &xe_device.
+ * @any: any pointer
+ *
+ * Return: the @xe_device pointer or NULL.
+ */
+#define xe_any_if_xe(any)	xe_any_if_type((any), struct xe_device)
+
+/**
+ * xe_any_if_pdev() - Get the pointer only if it is &pci_dev.
+ * @any: any pointer
+ *
+ * Return: the @pci_dev pointer or NULL.
+ */
+#define xe_any_if_pdev(any)	xe_any_if_type((any), struct pci_dev)
+
+#define __xe_any_to_other_assoc(const, from, other, p) \
+	const struct from * : __##from##_to_##other((const struct from *)(p))
+
+#define __xe_tile_to_xe_device(p)	tile_to_xe(p)
+#define __xe_gt_to_xe_device(p)		gt_to_xe(p)
+#define __pci_dev_to_xe_device(p)	pdev_to_xe_device(p)
+#define __device_to_xe_device(p)	kdev_to_xe_device(p)
+#define __drm_device_to_xe_device(p)	to_xe_device(p)
+#define __pci_dev_to_device(p)		(&(p)->dev)
+
+/**
+ * xe_any_to_xe() - Obtain the &xe_device pointer.
+ * @any: the &pci_dev or the &xe_device or &xe_tile or &xe_gt pointer
+ *
+ * Return: the @xe_device pointer or backpointer.
+ */
+#define xe_any_to_xe(any)							\
+	_Generic((any),								\
+		 __xe_any_to_self_assoc(struct xe_device, (any)),		\
+		 __xe_any_to_other_assoc(/* */, xe_tile, xe_device, (any)),	\
+		 __xe_any_to_other_assoc(const, xe_tile, xe_device, (any)),	\
+		 __xe_any_to_other_assoc(/* */, xe_gt, xe_device, (any)),	\
+		 __xe_any_to_other_assoc(const, xe_gt, xe_device, (any)),	\
+		 __xe_any_to_other_assoc(, drm_device, xe_device, (any)),	\
+		 __xe_any_to_other_assoc(, pci_dev, xe_device, (any)),		\
+		 __xe_any_to_other_assoc(, device, xe_device, (any)))
+
+/**
+ * xe_any_to_drm() - Obtain the &drm_device pointer.
+ * @any: the &pci_dev or the &xe_device or &xe_tile or &xe_gt pointer
+ *
+ * Return: the @drm_device pointer or backpointer.
+ */
+#define xe_any_to_drm(any)							\
+	_Generic((any),								\
+		 __xe_any_to_self_assoc(struct drm_device, (any)),		\
+		 default : &xe_any_to_xe(any)->drm)
+
+/**
+ * xe_any_to_dev() - Obtain the &device pointer.
+ * @any: the &pci_dev or the &xe_device or &xe_tile or &xe_gt pointer
+ *
+ * Return: the @device pointer or backpointer.
+ */
+#define xe_any_to_dev(any)							\
+	_Generic((any),								\
+		 __xe_any_to_self_assoc(struct device, (any)),			\
+		 __xe_any_to_other_assoc(, pci_dev, device, (any)),		\
+		 default : xe_any_to_drm(any)->dev)
+
+/**
+ * xe_any_to_pdev() - Obtain the &pci_dev pointer.
+ * @any: the &pci_dev or the &xe_device or &xe_tile or &xe_gt pointer
+ *
+ * Return: the @pci_dev pointer or backpointer.
+ */
+#define xe_any_to_pdev(any)							\
+	_Generic((any),								\
+		 __xe_any_to_self_assoc(struct pci_dev, (any)),			\
+		 default : to_pci_dev(xe_any_to_dev(any)))
+
+#define __xe_tile_to_id(p)		((p)->id)
+#define __xe_gt_to_id(p)		((p)->info.id)
+
+/**
+ * xe_any_id() - Get the identifier of the underlying object.
+ * @any: the &pci_dev or the &xe_device or &xe_tile or &xe_gt pointer
+ *
+ * Return: the identifier of the object, or 0 if not applicable/available.
+ */
+#define xe_any_id(any)								\
+	_Generic((any),								\
+		 __xe_any_to_other_assoc(/* */, xe_tile, id, (any)),		\
+		 __xe_any_to_other_assoc(const, xe_tile, id, (any)),		\
+		 __xe_any_to_other_assoc(/* */, xe_gt, id, (any)),		\
+		 __xe_any_to_other_assoc(const, xe_gt, id, (any)),		\
+		 default : 0)
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index dde309821237b4..6921b6967330d3 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -28,6 +28,7 @@
 #include "xe_ggtt.h"
 #include "xe_map.h"
 #include "xe_migrate.h"
+#include "xe_mmio_gem.h"
 #include "xe_pat.h"
 #include "xe_pm.h"
 #include "xe_preempt_fence.h"
@@ -104,13 +105,16 @@ static bool resource_is_vram(struct ttm_resource *res)
 
 bool xe_bo_is_vram(struct xe_bo *bo)
 {
-	return resource_is_vram(bo->ttm.resource) ||
-		resource_is_stolen_vram(xe_bo_device(bo), bo->ttm.resource);
+	struct ttm_resource *res = bo->ttm.resource;
+
+	return  res && (resource_is_vram(res) || resource_is_stolen_vram(xe_bo_device(bo), res));
 }
 
 bool xe_bo_is_stolen(struct xe_bo *bo)
 {
-	return bo->ttm.resource->mem_type == XE_PL_STOLEN;
+	struct ttm_resource *res = bo->ttm.resource;
+
+	return res && res->mem_type == XE_PL_STOLEN;
 }
 
 /**
@@ -158,7 +162,13 @@ bool xe_bo_is_vm_bound(struct xe_bo *bo)
 	return !list_empty(&bo->ttm.base.gpuva.list);
 }
 
-static bool xe_bo_is_user(struct xe_bo *bo)
+/**
+ * xe_bo_is_user - Check if BO is user-created
+ * @bo: The BO
+ *
+ * Returns: true if @bo was created by userspace
+ */
+bool xe_bo_is_user(struct xe_bo *bo)
 {
 	return bo->flags & XE_BO_FLAG_USER;
 }
@@ -921,7 +931,7 @@ void xe_bo_set_purgeable_state(struct xe_bo *bo,
  *
  * Return: 0 on success, negative error code on failure
  */
-static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx)
+int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx)
 {
 	struct xe_bo *bo = ttm_to_xe_bo(ttm_bo);
 	struct ttm_placement place = {};
@@ -929,9 +939,6 @@ static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operatio
 
 	xe_bo_assert_held(bo);
 
-	if (!ttm_bo->ttm)
-		return 0;
-
 	if (!xe_bo_madv_is_dontneed(bo))
 		return 0;
 
@@ -1037,6 +1044,13 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
 		} else {
 			drm_dbg(&xe->drm, "Evict system allocator BO failed=%pe\n",
 				ERR_PTR(ret));
+			/*
+			 * The semantic we want upon SVM eviction failure
+			 * because of racing access is keep walking for
+			 * eviction, which is -ENOSPC.
+			 */
+			if (ret == -EBUSY)
+				ret = -ENOSPC;
 		}
 
 		goto out;
@@ -1865,7 +1879,7 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
 		xe_drm_client_remove_bo(bo);
 #endif
 
-	if (bo->vm && xe_bo_is_user(bo))
+	if (bo->vm && (xe_bo_is_user(bo) || bo->flags & XE_BO_FLAG_PAGETABLE))
 		xe_vm_put(bo->vm);
 
 	if (bo->parent_obj)
@@ -2561,7 +2575,7 @@ __xe_bo_create_locked(struct xe_device *xe,
 	 * by having all the vm's bo refereferences released at vm close
 	 * time.
 	 */
-	if (vm && xe_bo_is_user(bo))
+	if (vm && (xe_bo_is_user(bo) || bo->flags & XE_BO_FLAG_PAGETABLE))
 		xe_vm_get(vm);
 	bo->vm = vm;
 
@@ -3261,6 +3275,9 @@ void xe_bo_unpin(struct xe_bo *bo)
 	struct ttm_place *place = &bo->placements[0];
 	struct xe_device *xe = xe_bo_device(bo);
 
+	if (xe_bo_is_purged(bo))
+		return;
+
 	xe_assert(xe, !bo->ttm.base.import_attach);
 	xe_assert(xe, xe_bo_is_pinned(bo));
 
@@ -3655,6 +3672,39 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,
 	return err;
 }
 
+static int xe_gem_pci_barrier_mmap_offset(struct xe_device *xe, struct drm_file *file,
+					  struct drm_xe_gem_mmap_offset *args)
+{
+	struct xe_file *xef = file->driver_priv;
+	struct xe_mmio_gem **barrier = &xef->mmio_gem.pci_barrier;
+
+	if (XE_IOCTL_DBG(xe, !IS_DGFX(xe)))
+		return -EINVAL;
+
+	if (XE_IOCTL_DBG(xe, args->handle))
+		return -EINVAL;
+
+	scoped_guard(mutex, &xef->mmio_gem.lock) {
+		if (!*barrier) {
+			phys_addr_t phys_addr;
+
+#define LAST_DB_PAGE_OFFSET 0x7ff000
+			phys_addr = pci_resource_start(to_pci_dev(xe->drm.dev), 0) +
+				LAST_DB_PAGE_OFFSET;
+			*barrier = xe_mmio_gem_create(xe, file, phys_addr, SZ_4K);
+			if (IS_ERR(*barrier)) {
+				int err = PTR_ERR(*barrier);
+
+				*barrier = NULL;
+				return err;
+			}
+		}
+
+		args->offset = xe_mmio_gem_mmap_offset(*barrier);
+	}
+	return 0;
+}
+
 int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file)
 {
@@ -3670,21 +3720,8 @@ int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
 			 ~DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER))
 		return -EINVAL;
 
-	if (args->flags & DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER) {
-		if (XE_IOCTL_DBG(xe, !IS_DGFX(xe)))
-			return -EINVAL;
-
-		if (XE_IOCTL_DBG(xe, args->handle))
-			return -EINVAL;
-
-		if (XE_IOCTL_DBG(xe, PAGE_SIZE > SZ_4K))
-			return -EINVAL;
-
-		BUILD_BUG_ON(((XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT) +
-			      SZ_4K) >= DRM_FILE_PAGE_OFFSET_START);
-		args->offset = XE_PCI_BARRIER_MMAP_OFFSET;
-		return 0;
-	}
+	if (args->flags & DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER)
+		return xe_gem_pci_barrier_mmap_offset(xe, file, args);
 
 	gem_obj = drm_gem_object_lookup(file, args->handle);
 	if (XE_IOCTL_DBG(xe, !gem_obj))
diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
index e8081af5bfc18e..861b1be231deda 100644
--- a/drivers/gpu/drm/xe/xe_bo.h
+++ b/drivers/gpu/drm/xe/xe_bo.h
@@ -9,6 +9,8 @@
 #include <drm/drm_prime.h>
 #include <drm/ttm/ttm_tt.h>
 
+#include <xen/xen.h>
+
 #include "xe_bo_types.h"
 #include "xe_ggtt.h"
 #include "xe_macros.h"
@@ -87,7 +89,6 @@
 
 #define XE_BO_PROPS_INVALID	(-1)
 
-#define XE_PCI_BARRIER_MMAP_OFFSET	(0x50 << XE_PTE_SHIFT)
 
 /**
  * enum xe_madv_purgeable_state - Buffer object purgeable state enumeration
@@ -495,6 +496,8 @@ void __xe_bo_release_dummy(struct kref *kref);
  * @bo: The bo to put.
  * @deferred: List to which to add the buffer object if we cannot put, or
  * NULL if the function is to put unconditionally.
+ * @added: BO was added to deferred list, written back to caller, can be NULL if
+ * writeback is not needed. Only set to true when added, never set to false.
  *
  * Since the final freeing of an object includes both sleeping and (!)
  * memory allocation in the dma_resv individualization, it's not ok
@@ -514,7 +517,8 @@ void __xe_bo_release_dummy(struct kref *kref);
  * false otherwise.
  */
 static inline bool
-xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)
+xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred,
+		   bool *added)
 {
 	if (!deferred) {
 		xe_bo_put(bo);
@@ -524,6 +528,9 @@ xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)
 	if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy))
 		return false;
 
+	if (added)
+		*added = true;
+
 	return llist_add(&bo->freed, deferred);
 }
 
@@ -540,7 +547,7 @@ xe_bo_put_async(struct xe_bo *bo)
 {
 	struct xe_bo_dev *bo_device = &xe_bo_device(bo)->bo_device;
 
-	if (xe_bo_put_deferred(bo, &bo_device->async_list))
+	if (xe_bo_put_deferred(bo, &bo_device->async_list, NULL))
 		schedule_work(&bo_device->async_free);
 }
 
@@ -575,6 +582,23 @@ static inline unsigned int xe_sg_segment_size(struct device *dev)
 	struct scatterlist __maybe_unused sg;
 	size_t max = BIT_ULL(sizeof(sg.length) * 8) - 1;
 
+	/*
+	 * For Xen PV guests pages aren't contiguous in DMA (machine) address
+	 * space.  The DMA API takes care of that both in dma_alloc_* (by
+	 * calling into the hypervisor to make the pages contiguous) and in
+	 * dma_map_* (by bounce buffering).  But xe (like i915, see commit
+	 * 78a07fe777c4) ignores the coherency aspects of the DMA API and thus
+	 * can't cope with bounce buffering actually happening, so add a hack
+	 * here to force small allocations and mappings when running in PV
+	 * mode on Xen.
+	 *
+	 * Note this will still break if bounce buffering is required for other
+	 * reasons, like confidential computing hypervisors or PCIe root ports
+	 * with addressing limitations.
+	 */
+	if (xen_pv_domain())
+		return PAGE_SIZE;
+
 	max = min_t(size_t, max, dma_max_mapping_size(dev));
 
 	/*
@@ -600,6 +624,8 @@ struct xe_bo_shrink_flags {
 long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
 		  const struct xe_bo_shrink_flags flags,
 		  unsigned long *scanned);
+int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operation_ctx *ctx);
+bool xe_bo_is_user(struct xe_bo *bo);
 
 /**
  * xe_bo_is_mem_type - Whether the bo currently resides in the given
diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h
index e45f24301050ca..8ec4a01a00928f 100644
--- a/drivers/gpu/drm/xe/xe_bo_types.h
+++ b/drivers/gpu/drm/xe/xe_bo_types.h
@@ -20,6 +20,7 @@
 struct xe_device;
 struct xe_mem_pool_node;
 struct xe_vm;
+struct xe_exec_queue;
 
 #define XE_BO_MAX_PLACEMENTS	3
 
@@ -42,6 +43,13 @@ struct xe_bo {
 	u32 flags;
 	/** @vm: VM this BO is attached to, for extobj this will be NULL */
 	struct xe_vm *vm;
+	/**
+	 * @q: Queue this BO is attached to, mostly for LRC BO, NULL otherwise.
+	 * Protected by the BO dma_resv: readers must hold it across both the
+	 * read and xe_exec_queue_get_unless_zero(). The BO holds no reference
+	 * on the queue.
+	 */
+	struct xe_exec_queue *q;
 	/** @tile: Tile this BO is attached to (kernel BO only) */
 	struct xe_tile *tile;
 	/** @placements: valid placements for this BO */
@@ -82,8 +90,6 @@ struct xe_bo {
 
 	/** @freed: List node for delayed put. */
 	struct llist_node freed;
-	/** @update_index: Update index if PT BO */
-	int update_index;
 	/** @created: Whether the bo has passed initial creation */
 	bool created;
 
diff --git a/drivers/gpu/drm/xe/xe_configfs.c b/drivers/gpu/drm/xe/xe_configfs.c
index 052cce96216118..2ddd707a6dbd30 100644
--- a/drivers/gpu/drm/xe/xe_configfs.c
+++ b/drivers/gpu/drm/xe/xe_configfs.c
@@ -61,7 +61,9 @@
  *	    ├── survivability_mode
  *	    ├── gt_types_allowed
  *	    ├── engines_allowed
- *	    └── enable_psmi
+ *	    ├── migrate_ulls_period_ms
+ *	    ├── enable_psmi
+ *	    └── disable_vram_page_offline
  *
  * After configuring the attributes as per next section, the device can be
  * probed with::
@@ -159,6 +161,18 @@
  *
  * This attribute can only be set before binding to the device.
  *
+ * Disable VRAM page offline:
+ * ----------------------------
+ *
+ *  0, n, N, false - Do not disable (Offlining is active - default)
+ *  1, y, Y, true  - Disable vram page offline (Logging only)
+ *
+ *  Example to disable VRAM offline::
+ *
+ *      # echo 1 > /sys/kernel/config/xe/0000:03:00.0/disable_vram_page_offline
+ *
+ * This attribute can only be set on CRI before binding to the device.
+ *
  * Context restore BB
  * ------------------
  *
@@ -249,6 +263,20 @@
  *
  * This attribute can only be set before binding to the device.
  *
+ * Migrate ULLS Period (ms)
+ * ------------------------
+ *
+ * Migrate ULLS period, in milliseconds. This is the delay between entering
+ * migrate ULLS (a continuously running batch) and exiting it. Migrate ULLS is
+ * currently entered during page faults and SVM prefetch operations. Default 5,
+ * zero indicates ULLS is disabled.
+ *
+ * How to disable migration ULLS:
+ *
+ *	# echo 0 > /sys/kernel/config/xe/0000:03:00.0/migrate_ulls_period_ms
+ *
+ * This attribute can only be set before binding to the device.
+ *
  * Remove devices
  * ==============
  *
@@ -270,11 +298,13 @@ struct xe_config_group_device {
 	struct xe_config_device {
 		u64 gt_types_allowed;
 		u64 engines_allowed;
+		u32 migrate_ulls_period_ms;
 		struct wa_bb ctx_restore_post_bb[XE_ENGINE_CLASS_MAX];
 		struct wa_bb ctx_restore_mid_bb[XE_ENGINE_CLASS_MAX];
 		bool survivability_mode;
 		bool enable_psmi;
 		bool enable_multi_queue;
+		bool disable_vram_page_offline;
 		struct {
 			unsigned int max_vfs;
 			bool admin_only_pf;
@@ -292,9 +322,11 @@ struct xe_config_group_device {
 static const struct xe_config_device device_defaults = {
 	.gt_types_allowed = U64_MAX,
 	.engines_allowed = U64_MAX,
+	.migrate_ulls_period_ms = 5,
 	.survivability_mode = false,
 	.enable_psmi = false,
 	.enable_multi_queue = true,
+	.disable_vram_page_offline = false,
 	.sriov = {
 		.max_vfs = XE_DEFAULT_MAX_VFS,
 		.admin_only_pf = XE_DEFAULT_ADMIN_ONLY_PF,
@@ -616,6 +648,60 @@ static ssize_t enable_multi_queue_store(struct config_item *item, const char *pa
 	return len;
 }
 
+static ssize_t migrate_ulls_period_ms_show(struct config_item *item, char *page)
+{
+	struct xe_config_device *dev = to_xe_config_device(item);
+
+	return sprintf(page, "%u\n", dev->migrate_ulls_period_ms);
+}
+
+static ssize_t migrate_ulls_period_ms_store(struct config_item *item,
+					    const char *page, size_t len)
+{
+	struct xe_config_group_device *dev = to_xe_config_group_device(item);
+	u32 val;
+	int ret;
+
+	ret = kstrtou32(page, 0, &val);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&dev->lock);
+	if (is_bound(dev))
+		return -EBUSY;
+
+	dev->config.migrate_ulls_period_ms = val;
+
+	return len;
+}
+
+static ssize_t disable_vram_page_offline_show(struct config_item *item, char *page)
+{
+	struct xe_config_device *dev = to_xe_config_device(item);
+
+	return sprintf(page, "%s\n", str_yes_no(dev->disable_vram_page_offline));
+}
+
+static ssize_t disable_vram_page_offline_store(struct config_item *item,
+					       const char *page, size_t len)
+{
+	struct xe_config_group_device *dev = to_xe_config_group_device(item);
+	bool val;
+	int ret;
+
+	ret = kstrtobool(page, &val);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&dev->lock);
+	if (is_bound(dev))
+		return -EBUSY;
+
+	dev->config.disable_vram_page_offline = val;
+
+	return len;
+}
+
 static bool wa_bb_read_advance(bool dereference, char **p,
 			       const char *append, size_t len,
 			       size_t *max_size)
@@ -854,7 +940,9 @@ static ssize_t ctx_restore_post_bb_store(struct config_item *item,
 CONFIGFS_ATTR(, ctx_restore_mid_bb);
 CONFIGFS_ATTR(, ctx_restore_post_bb);
 CONFIGFS_ATTR(, enable_multi_queue);
+CONFIGFS_ATTR(, migrate_ulls_period_ms);
 CONFIGFS_ATTR(, enable_psmi);
+CONFIGFS_ATTR(, disable_vram_page_offline);
 CONFIGFS_ATTR(, engines_allowed);
 CONFIGFS_ATTR(, gt_types_allowed);
 CONFIGFS_ATTR(, survivability_mode);
@@ -863,7 +951,9 @@ static struct configfs_attribute *xe_config_device_attrs[] = {
 	&attr_ctx_restore_mid_bb,
 	&attr_ctx_restore_post_bb,
 	&attr_enable_multi_queue,
+	&attr_migrate_ulls_period_ms,
 	&attr_enable_psmi,
+	&attr_disable_vram_page_offline,
 	&attr_engines_allowed,
 	&attr_gt_types_allowed,
 	&attr_survivability_mode,
@@ -886,7 +976,7 @@ static struct configfs_item_operations xe_config_device_ops = {
 };
 
 static bool xe_config_device_is_visible(struct config_item *item,
-					struct configfs_attribute *attr, int n)
+					const struct configfs_attribute *attr, int n)
 {
 	struct xe_config_group_device *dev = to_xe_config_group_device(item);
 
@@ -895,6 +985,11 @@ static bool xe_config_device_is_visible(struct config_item *item,
 			return false;
 	}
 
+	if (attr == &attr_disable_vram_page_offline) {
+		if (!dev->desc->is_dgfx || dev->desc->platform != XE_CRESCENTISLAND)
+			return false;
+	}
+
 	return true;
 }
 
@@ -981,7 +1076,7 @@ static struct configfs_attribute *xe_config_sriov_attrs[] = {
 };
 
 static bool xe_config_sriov_is_visible(struct config_item *item,
-				       struct configfs_attribute *attr, int n)
+				       const struct configfs_attribute *attr, int n)
 {
 	struct xe_config_group_device *dev = to_xe_config_group_device(item->ci_parent);
 
@@ -1140,8 +1235,10 @@ static void dump_custom_dev_config(struct pci_dev *pdev,
 
 	PRI_CUSTOM_ATTR("%llx", gt_types_allowed);
 	PRI_CUSTOM_ATTR("%llx", engines_allowed);
+	PRI_CUSTOM_ATTR("%u", migrate_ulls_period_ms);
 	PRI_CUSTOM_ATTR("%d", enable_multi_queue);
 	PRI_CUSTOM_ATTR("%d", enable_psmi);
+	PRI_CUSTOM_ATTR("%d", disable_vram_page_offline);
 	PRI_CUSTOM_ATTR("%d", survivability_mode);
 	PRI_CUSTOM_ATTR("%u", sriov.admin_only_pf);
 
@@ -1290,6 +1387,46 @@ bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev)
 	return ret;
 }
 
+/**
+ * xe_configfs_get_migrate_ulls_period_ms - get configfs migrate_ulls_period_ms setting
+ * @pdev: pci device
+ *
+ * Return: Migrate ULLS period in milliseconds (zero is disabled).
+ */
+u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev)
+{
+	struct xe_config_group_device *dev = find_xe_config_group_device(pdev);
+	u32 ret;
+
+	if (!dev)
+		return device_defaults.migrate_ulls_period_ms;
+
+	ret = dev->config.migrate_ulls_period_ms;
+	config_group_put(&dev->group);
+
+	return ret;
+}
+
+/**
+ * xe_configfs_get_disable_vram_page_offline - get configfs disable_vram_page_offline setting
+ * @pdev: pci device
+ *
+ * Return: disable_vram_page_offline setting in configfs
+ */
+bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev)
+{
+	struct xe_config_group_device *dev = find_xe_config_group_device(pdev);
+	bool ret;
+
+	if (!dev)
+		return device_defaults.disable_vram_page_offline;
+
+	ret = dev->config.disable_vram_page_offline;
+	config_group_put(&dev->group);
+
+	return ret;
+}
+
 /**
  * xe_configfs_get_ctx_restore_mid_bb - get configfs ctx_restore_mid_bb setting
  * @pdev: pci device
diff --git a/drivers/gpu/drm/xe/xe_configfs.h b/drivers/gpu/drm/xe/xe_configfs.h
index 4fbbeafba47349..10a00d6bbf8f46 100644
--- a/drivers/gpu/drm/xe/xe_configfs.h
+++ b/drivers/gpu/drm/xe/xe_configfs.h
@@ -24,6 +24,8 @@ bool xe_configfs_media_gt_allowed(struct pci_dev *pdev);
 u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev);
 bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev);
 bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev);
+u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev);
+bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev);
 u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
 				       enum xe_engine_class class,
 				       const u32 **cs);
@@ -44,6 +46,8 @@ static inline bool xe_configfs_media_gt_allowed(struct pci_dev *pdev) { return t
 static inline u64 xe_configfs_get_engines_allowed(struct pci_dev *pdev) { return U64_MAX; }
 static inline bool xe_configfs_get_psmi_enabled(struct pci_dev *pdev) { return false; }
 static inline bool xe_configfs_get_enable_multi_queue(struct pci_dev *pdev) { return true; }
+static inline u32 xe_configfs_get_migrate_ulls_period_ms(struct pci_dev *pdev) { return 5; }
+static inline bool xe_configfs_get_disable_vram_page_offline(struct pci_dev *pdev) { return false; }
 static inline u32 xe_configfs_get_ctx_restore_mid_bb(struct pci_dev *pdev,
 						     enum xe_engine_class class,
 						     const u32 **cs) { return 0; }
diff --git a/drivers/gpu/drm/xe/xe_cpu_bind.c b/drivers/gpu/drm/xe/xe_cpu_bind.c
new file mode 100644
index 00000000000000..31897829842025
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_cpu_bind.c
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <drm/drm_managed.h>
+#include <linux/mutex.h>
+
+#include "xe_cpu_bind.h"
+#include "xe_device_types.h"
+#include "xe_exec_queue.h"
+#include "xe_pt.h"
+#include "xe_sched_job.h"
+#include "xe_trace_bo.h"
+#include "xe_vm.h"
+
+/**
+ * struct xe_cpu_bind - cpu_bind context.
+ */
+struct xe_cpu_bind {
+	/** @xe: Xe device */
+	struct xe_device *xe;
+	/** @q: Default exec queue used for kernel binds */
+	struct xe_exec_queue *q;
+	/** @job_mutex: Timeline mutex for @q. */
+	struct mutex job_mutex;
+};
+
+static bool is_cpu_bind_queue(struct xe_cpu_bind *cpu_bind,
+			      struct xe_exec_queue *q)
+{
+	return cpu_bind->q == q;
+}
+
+static void xe_cpu_bind_fini(void *arg)
+{
+	struct xe_cpu_bind *cpu_bind = arg;
+
+	mutex_destroy(&cpu_bind->job_mutex);
+	xe_exec_queue_put(cpu_bind->q);
+}
+
+/**
+ * xe_cpu_bind_init() - Initialize a cpu_bind context
+ * @xe: &struct xe_device
+ *
+ * Return: 0 if successful, negative error code on failure
+ */
+int xe_cpu_bind_init(struct xe_device *xe)
+{
+	struct xe_cpu_bind *cpu_bind =
+		drmm_kzalloc(&xe->drm, sizeof(*cpu_bind), GFP_KERNEL);
+	struct xe_exec_queue *q;
+
+	if (!cpu_bind)
+		return -ENOMEM;
+
+	q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), NULL,
+				      EXEC_QUEUE_FLAG_KERNEL |
+				      EXEC_QUEUE_FLAG_MIGRATE, 0);
+	if (IS_ERR(q))
+		return PTR_ERR(q);
+
+	cpu_bind->xe = xe;
+	cpu_bind->q = q;
+	xe->cpu_bind = cpu_bind;
+
+	mutex_init(&cpu_bind->job_mutex);
+
+	fs_reclaim_acquire(GFP_KERNEL);
+	might_lock(&cpu_bind->job_mutex);
+	fs_reclaim_release(GFP_KERNEL);
+
+	return devm_add_action_or_reset(cpu_bind->xe->drm.dev, xe_cpu_bind_fini,
+					cpu_bind);
+}
+
+/**
+ * xe_cpu_bind_queue() - Get the bind queue from cpu_bind context.
+ * @cpu_bind: The cpu bind context.
+ *
+ * Return: Pointer to bind queue.
+ */
+struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind)
+{
+	return cpu_bind->q;
+}
+
+/**
+ * xe_cpu_bind_update_pgtables_execute() - Update a VM's PTEs via the CPU
+ * @vm: The VM being updated
+ * @tile: The tile being updated
+ * @ops: The CPU bind PT update ops
+ * @pt_op: The VM PT update op
+ * @num_ops: The number of The VM PT update ops
+ * @force_clear: Force clear operation
+ *
+ * Execute the VM PT update ops array which results in a VM's PTEs being updated
+ * via the CPU.
+ */
+void
+xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile,
+				    const struct xe_cpu_bind_pt_update_ops *ops,
+				    struct xe_vm_pgtable_update_op *pt_op,
+				    u32 num_ops, bool force_clear)
+{
+	u32 j, i;
+
+	for (j = 0; j < num_ops; ++j, ++pt_op) {
+		for (i = 0; i < pt_op->num_entries; i++) {
+			const struct xe_vm_pgtable_update *update =
+				&pt_op->entries[i];
+
+			xe_assert(vm->xe, !iosys_map_is_null(&update->pt_bo->vmap));
+
+			if (pt_op->bind && !force_clear)
+				ops->populate(tile, &update->pt_bo->vmap,
+					      update);
+			else
+				ops->clear(vm, tile, &update->pt_bo->vmap,
+					   update);
+		}
+	}
+
+	trace_xe_vm_cpu_bind(vm);
+	xe_device_wmb(vm->xe);
+}
+
+static struct dma_fence *
+xe_cpu_bind_update_pgtables_no_job(struct xe_cpu_bind *cpu_bind,
+				   struct xe_cpu_bind_pt_update *pt_update)
+{
+	const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
+	struct xe_vm *vm = pt_update->vops->vm;
+	struct xe_tile *tile;
+	int err, id;
+
+	if (ops->pre_commit) {
+		pt_update->job = NULL;
+		err = ops->pre_commit(pt_update);
+		if (err)
+			return ERR_PTR(err);
+	}
+
+	for_each_tile(tile, vm->xe, id) {
+		struct xe_vm_pgtable_update_ops *pt_update_ops =
+			&pt_update->vops->pt_update_ops[tile->id];
+
+		if (!pt_update_ops->pt_job_ops)
+			continue;
+
+		xe_cpu_bind_update_pgtables_execute(vm, tile, ops,
+						    pt_update_ops->pt_job_ops->ops,
+						    pt_update_ops->pt_job_ops->current_op,
+						    false);
+	}
+
+	return dma_fence_get_stub();
+}
+
+static struct dma_fence *
+xe_cpu_bind_update_pgtables_job(struct xe_cpu_bind *cpu_bind,
+				struct xe_cpu_bind_pt_update *pt_update)
+{
+	const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
+	struct xe_exec_queue *q = pt_update->vops->q;
+	struct xe_device *xe = cpu_bind->xe;
+	struct xe_sched_job *job;
+	struct dma_fence *fence;
+	struct xe_tile *tile;
+	int err, id;
+	bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
+
+	job = xe_sched_job_create(q, NULL);
+	if (IS_ERR(job))
+		return ERR_CAST(job);
+
+	xe_assert(xe, job->is_pt_job);
+
+	if (ops->pre_commit) {
+		pt_update->job = job;
+		err = ops->pre_commit(pt_update);
+		if (err)
+			goto err_job;
+	}
+
+	if (is_cpu_bind)
+		mutex_lock(&cpu_bind->job_mutex);
+
+	job->pt_update[0].vm = pt_update->vops->vm;
+	job->pt_update[0].ops = ops;
+	for_each_tile(tile, xe, id) {
+		struct xe_vm_pgtable_update_ops *pt_update_ops =
+			&pt_update->vops->pt_update_ops[tile->id];
+
+		job->pt_update[0].pt_job_ops[tile->id] =
+			xe_pt_job_ops_get(pt_update_ops->pt_job_ops);
+	}
+
+	xe_sched_job_arm(job);
+	fence = dma_fence_get(&job->drm.s_fence->finished);
+	xe_sched_job_push(job);
+
+	if (is_cpu_bind)
+		mutex_unlock(&cpu_bind->job_mutex);
+
+	return fence;
+
+err_job:
+	xe_sched_job_put(job);
+	return ERR_PTR(err);
+}
+
+/**
+ * xe_cpu_bind_update_pgtables() - Pipelined page-table update
+ * @cpu_bind: The cpu bind context.
+ * @pt_update: PT update arguments
+ *
+ * Perform a pipelined page-table update. The update descriptors are typically
+ * built under the same lock critical section as a call to this function. If
+ * using the default engine for the updates, they will be performed in the
+ * order they grab the job_mutex. If different engines are used, external
+ * synchronization is needed for overlapping updates to maintain page-table
+ * consistency. Note that the meaning of "overlapping" is that the updates
+ * touch the same page-table, which might be a higher-level page-directory.
+ * If no pipelining is needed, then updates may be performed by the cpu.
+ *
+ * Return: A dma_fence that, when signaled, indicates the update completion.
+ */
+struct dma_fence *
+xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind,
+			    struct xe_cpu_bind_pt_update *pt_update)
+{
+	struct dma_fence *fence;
+
+	fence = xe_cpu_bind_update_pgtables_no_job(cpu_bind, pt_update);
+
+	/* -ETIME indicates a job is needed, anything else is legit error */
+	if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
+		return fence;
+
+	return xe_cpu_bind_update_pgtables_job(cpu_bind, pt_update);
+}
+
+/**
+ * xe_cpu_bind_job_lock() - Lock cpu_bind job lock
+ * @cpu_bind: The cpu bind context.
+ * @q: Queue associated with the operation which requires a lock
+ *
+ * Lock the cpu_bind job lock if the queue is a cpu bind queue, otherwise
+ * assert the VM's dma-resv is held (user queue's have own locking).
+ */
+void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind,
+			  struct xe_exec_queue *q)
+{
+	bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
+
+	if (is_cpu_bind)
+		mutex_lock(&cpu_bind->job_mutex);
+	else
+		xe_vm_assert_held(q->user_vm);	/* User queues VM's should be locked */
+}
+
+/**
+ * xe_cpu_bind_job_unlock() - Unlock cpu_bind job lock
+ * @cpu_bind: The cpu bind context.
+ * @q: Queue associated with the operation which requires a lock
+ *
+ * Unlock the cpu_bind job lock if the queue is a cpu bind queue, otherwise
+ * assert the VM's dma-resv is held (user queue's have own locking).
+ */
+void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind,
+			    struct xe_exec_queue *q)
+{
+	bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
+
+	if (is_cpu_bind)
+		mutex_unlock(&cpu_bind->job_mutex);
+	else
+		xe_vm_assert_held(q->user_vm);	/* User queues VM's should be locked */
+}
+
+#if IS_ENABLED(CONFIG_PROVE_LOCKING)
+/**
+ * xe_cpu_bind_job_lock_assert() - Assert cpu_bind job lock held of queue
+ * @q: cpu bind queue
+ */
+void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q)
+{
+	struct xe_device *xe = gt_to_xe(q->gt);
+	struct xe_cpu_bind *cpu_bind = xe->cpu_bind;
+
+	xe_assert(xe, q == cpu_bind->q);
+	lockdep_assert_held(&cpu_bind->job_mutex);
+}
+#endif
diff --git a/drivers/gpu/drm/xe/xe_cpu_bind.h b/drivers/gpu/drm/xe/xe_cpu_bind.h
new file mode 100644
index 00000000000000..69aad5edf2043f
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_cpu_bind.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_CPU_BIND_H_
+#define _XE_CPU_BIND_H_
+
+#include <linux/types.h>
+
+struct dma_fence;
+struct iosys_map;
+struct xe_cpu_bind;
+struct xe_cpu_bind_pt_update;
+struct xe_device;
+struct xe_tlb_inval_job;
+struct xe_tile;
+struct xe_vm;
+struct xe_vm_pgtable_update;
+struct xe_vm_pgtable_update_op;
+struct xe_vma_ops;
+
+/**
+ * struct xe_cpu_bind_pt_update_ops - Callbacks for the
+ * xe_cpu_bind_update_pgtables() function.
+ */
+struct xe_cpu_bind_pt_update_ops {
+	/**
+	 * @populate: Populate a page-table with ptes.
+	 * @tile: The tile for the current operation.
+	 * @map: struct iosys_map into the memory to be populated.
+	 * @update: Information about the PTEs to be inserted.
+	 *
+	 * This interface is intended to be used as a callback into the
+	 * page-table system to populate shared page-tables with PTEs.
+	 */
+	void (*populate)(struct xe_tile *tile, struct iosys_map *map,
+			 const struct xe_vm_pgtable_update *update);
+	/**
+	 * @clear: Clear a page-table's ptes.
+	 * @vm: VM being updated
+	 * @tile: The tile for the current operation.
+	 * @map: struct iosys_map into the page-table to be cleared.
+	 * @update: Information about the PTEs to be cleared.
+	 *
+	 * This interface is intended to be used as a callback into the
+	 * page-table system to clear PTEs from shared page-tables.
+	 */
+	void (*clear)(struct xe_vm *vm, struct xe_tile *tile,
+		      struct iosys_map *map,
+		      const struct xe_vm_pgtable_update *update);
+
+	/**
+	 * @pre_commit: Callback to be called just before arming the
+	 * sched_job.
+	 * @pt_update: Pointer to embeddable callback argument.
+	 *
+	 * Return: 0 on success, negative error code on error.
+	 */
+	int (*pre_commit)(struct xe_cpu_bind_pt_update *pt_update);
+};
+
+/**
+ * struct xe_cpu_bind_pt_update - Argument to the struct
+ * xe_cpu_bind_pt_update_ops callbacks.
+ *
+ * Intended to be subclassed to support additional arguments if necessary.
+ */
+struct xe_cpu_bind_pt_update {
+	/** @ops: Pointer to the struct xe_cpu_bind_pt_update_ops callbacks */
+	const struct xe_cpu_bind_pt_update_ops *ops;
+	/** @vops: VMA operations */
+	struct xe_vma_ops *vops;
+	/** @job: The bind job, or NULL if the update is issued immediatel */
+	struct xe_sched_job *job;
+	/**
+	 * @ijobs: The TLB invalidation jobs, individual instances can be NULL
+	 */
+#define XE_CPU_BIND_INVAL_JOB_COUNT	4
+	struct xe_tlb_inval_job *ijobs[XE_CPU_BIND_INVAL_JOB_COUNT];
+};
+
+int xe_cpu_bind_init(struct xe_device *xe);
+
+struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind);
+
+void
+xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile,
+				    const struct xe_cpu_bind_pt_update_ops *ops,
+				    struct xe_vm_pgtable_update_op *pt_op,
+				    u32 num_ops, bool force_clear);
+
+struct dma_fence *
+xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind,
+			    struct xe_cpu_bind_pt_update *pt_update);
+
+void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind,
+			  struct xe_exec_queue *q);
+
+void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind,
+			    struct xe_exec_queue *q);
+
+#if IS_ENABLED(CONFIG_PROVE_LOCKING)
+void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q);
+#else
+static inline void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q)
+{
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c
index 8de78cd0aa03b2..80f62634fae509 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_debugfs.c
@@ -22,6 +22,7 @@
 #include "xe_guc_ads.h"
 #include "xe_hw_engine.h"
 #include "xe_mmio.h"
+#include "xe_pagefault.h"
 #include "xe_pcode.h"
 #include "xe_pm.h"
 #include "xe_psmi.h"
@@ -31,6 +32,7 @@
 #include "xe_sriov_vf.h"
 #include "xe_step.h"
 #include "xe_tile_debugfs.h"
+#include "xe_ttm_vram_mgr.h"
 #include "xe_vsec.h"
 #include "xe_wa.h"
 
@@ -42,12 +44,19 @@
 
 DECLARE_FAULT_ATTR(gt_reset_failure);
 DECLARE_FAULT_ATTR(inject_csc_hw_error);
+DECLARE_FAULT_ATTR(wedge_cold_reset);
+DECLARE_FAULT_ATTR(inject_mempage_offline);
 
 static bool csc_hw_error_available(struct xe_device *xe)
 {
 	return !IS_SRIOV_VF(xe) && xe->info.platform == XE_BATTLEMAGE;
 }
 
+static bool is_crescent_island_pf(struct xe_device *xe)
+{
+	return !IS_SRIOV_VF(xe) && xe->info.platform == XE_CRESCENTISLAND;
+}
+
 /*
  * Fault injection table.  Each entry registers a debugfs attribute; add a
  * matching FAULT_ACTION() below for every entry added here.
@@ -62,6 +71,11 @@ static struct {
 	{ .name = "inject_csc_hw_error",
 	  .attr = &inject_csc_hw_error,
 	  .is_visible = csc_hw_error_available },
+	{ .name = "wedge_cold_reset",
+	  .attr = &wedge_cold_reset },
+	{ .name = "inject_mempage_offline",
+	  .attr = &inject_mempage_offline,
+	  .is_visible = is_crescent_island_pf },
 };
 
 /*
@@ -76,6 +90,41 @@ bool xe_fault_##name(void)				\
 
 FAULT_ACTION(gt_reset, gt_reset_failure)
 FAULT_ACTION(csc_hw_error, inject_csc_hw_error)
+FAULT_ACTION(wedge_cold_reset, wedge_cold_reset)
+FAULT_ACTION(mempage_offline, inject_mempage_offline)
+
+static ssize_t inject_mempage_offline_trigger(struct file *f,
+					      const char __user *ubuf,
+					      size_t size, loff_t *pos)
+{
+	struct xe_device *xe = file_inode(f)->i_private;
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_vram_region *vr = tile->mem.vram;
+	u64 pfn;
+	int ret;
+
+	if (!vr)
+		return -ENODEV;
+
+	ret = kstrtou64_from_user(ubuf, size, 0, &pfn);
+	if (ret)
+		return ret;
+
+	if (!xe_fault_mempage_offline())
+		return size;
+
+	xe_warn(xe, "Page offlining test interface accessed. Notice: Offlined or reserved memory pages cannot be reclaimed dynamically. A driver rebind (unbind and bind loop) is required post-test to clean up.\n");
+	if (pfn == 0)
+		return xe_ttm_vram_inject_fault(xe) ?: size;
+
+	/* User provided PFN - convert to DPA and inject */
+	return xe_ttm_vram_handle_addr_fault(xe, pfn << PAGE_SHIFT) ?: size;
+}
+
+static const struct file_operations inject_mempage_offline_fops = {
+	.owner = THIS_MODULE,
+	.write = inject_mempage_offline_trigger,
+};
 
 static void xe_fault_inject_debugfs_register(struct xe_device *xe,
 					     struct dentry *root)
@@ -90,6 +139,11 @@ static void xe_fault_inject_debugfs_register(struct xe_device *xe,
 		fault_create_debugfs_attr(xe_fault_inject_entry[i].name, root,
 					  xe_fault_inject_entry[i].attr);
 	}
+
+	if (is_crescent_island_pf(xe)) {
+		debugfs_create_file("inject_mempage_offline_trigger", 0200,
+				    root, xe, &inject_mempage_offline_fops);
+	}
 }
 
 static void read_residency_counter(struct xe_device *xe, struct xe_mmio *mmio,
@@ -194,6 +248,15 @@ static int sriov_info(struct seq_file *m, void *data)
 	return 0;
 }
 
+static int pagefault_info(struct seq_file *m, void *data)
+{
+	struct xe_device *xe = node_to_xe(m->private);
+	struct drm_printer p = drm_seq_file_printer(m);
+
+	xe_pagefault_print_info(xe, &p);
+	return 0;
+}
+
 static int workarounds(struct xe_device *xe, struct drm_printer *p)
 {
 	guard(xe_pm_runtime)(xe);
@@ -285,6 +348,7 @@ static const struct drm_info_list debugfs_list[] = {
 	{"info", info, 0},
 	{ .name = "sriov_info", .show = sriov_info, },
 	{ .name = "workarounds", .show = workaround_info, },
+	{ .name = "pagefault_info", .show = pagefault_info, },
 };
 
 static const struct drm_info_list pcode_info_debugfs[] = {
@@ -758,6 +822,8 @@ void xe_debugfs_register(struct xe_device *xe)
 	if (man)
 		ttm_resource_manager_create_debugfs(man, root, "stolen_mm");
 
+	xe_ttm_vram_debugfs_init(xe, root);
+
 	for_each_tile(tile, xe, tile_id)
 		xe_tile_debugfs_register(tile);
 
diff --git a/drivers/gpu/drm/xe/xe_debugfs.h b/drivers/gpu/drm/xe/xe_debugfs.h
index cd56f7442b9956..88d91c78036b2c 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.h
+++ b/drivers/gpu/drm/xe/xe_debugfs.h
@@ -13,10 +13,14 @@ struct xe_device;
 #ifdef CONFIG_DEBUG_FS
 bool xe_fault_gt_reset(void);
 bool xe_fault_csc_hw_error(void);
+bool xe_fault_wedge_cold_reset(void);
+bool xe_fault_mempage_offline(void);
 void xe_debugfs_register(struct xe_device *xe);
 #else
 static inline bool xe_fault_gt_reset(void) { return false; }
 static inline bool xe_fault_csc_hw_error(void) { return false; }
+static inline bool xe_fault_wedge_cold_reset(void) { return false; }
+static inline bool xe_fault_mempage_offline(void) { return false; }
 static inline void xe_debugfs_register(struct xe_device *xe) { }
 #endif
 
diff --git a/drivers/gpu/drm/xe/xe_defaults.h b/drivers/gpu/drm/xe/xe_defaults.h
index c8ae1d5f3d6012..0884224ef7c7c6 100644
--- a/drivers/gpu/drm/xe/xe_defaults.h
+++ b/drivers/gpu/drm/xe/xe_defaults.h
@@ -22,5 +22,6 @@
 #define XE_DEFAULT_WEDGED_MODE			XE_WEDGED_MODE_UPON_CRITICAL_ERROR
 #define XE_DEFAULT_WEDGED_MODE_STR		"upon-critical-error"
 #define XE_DEFAULT_SVM_NOTIFIER_SIZE		512
+#define XE_DEFAULT_NUM_PF_WORK			2
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c
index 5f2b90b18f97b7..ab681b148869fa 100644
--- a/drivers/gpu/drm/xe/xe_devcoredump.c
+++ b/drivers/gpu/drm/xe/xe_devcoredump.c
@@ -74,11 +74,6 @@ static struct xe_device *coredump_to_xe(const struct xe_devcoredump *coredump)
 	return container_of(coredump, struct xe_device, devcoredump);
 }
 
-static struct xe_guc *exec_queue_to_guc(struct xe_exec_queue *q)
-{
-	return &q->gt->uc.guc;
-}
-
 static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count,
 				     ssize_t start,
 				     struct xe_devcoredump *coredump)
@@ -127,8 +122,10 @@ static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count,
 
 	drm_puts(&p, "\n**** HW Engines ****\n");
 	for (i = 0; i < XE_NUM_HW_ENGINES; i++)
-		if (ss->hwe[i])
+		if (ss->hwe[i]) {
 			xe_engine_snapshot_print(ss->hwe[i], &p);
+			drm_puts(&p, "\n");
+		}
 
 	drm_puts(&p, "\n**** VM state ****\n");
 	xe_vm_snapshot_print(ss->vm, &p);
@@ -323,40 +320,44 @@ static void xe_devcoredump_deferred_snap_work(struct work_struct *work)
 }
 
 static void devcoredump_snapshot(struct xe_devcoredump *coredump,
+				 struct xe_gt *gt,
 				 struct xe_exec_queue *q,
 				 struct xe_sched_job *job)
 {
 	struct xe_devcoredump_snapshot *ss = &coredump->snapshot;
-	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct xe_guc *guc = &gt->uc.guc;
 	const char *process_name = "no process";
 	bool cookie;
 
 	ss->snapshot_time = ktime_get_real();
 	ss->boot_time = ktime_get_boottime();
 
-	if (q->vm && q->vm->xef) {
+	if (q && q->vm && q->vm->xef) {
 		process_name = q->vm->xef->process_name;
 		ss->pid = q->vm->xef->pid;
 	}
 
 	strscpy(ss->process_name, process_name);
 
-	ss->gt = q->gt;
+	ss->gt = gt;
 	INIT_WORK(&ss->work, xe_devcoredump_deferred_snap_work);
 
 	/* keep going if fw fails as we still want to save the memory and SW data */
-	CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL);
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 
 	cookie = dma_fence_begin_signalling();
 
 	ss->guc.log = xe_guc_log_snapshot_capture(&guc->log, true);
 	ss->guc.ct = xe_guc_ct_snapshot_capture(&guc->ct);
-	ss->ge = xe_guc_exec_queue_snapshot_capture(q);
-	if (job)
-		ss->job = xe_sched_job_snapshot_capture(job);
-	ss->vm = xe_vm_snapshot_capture(q->vm);
 
-	xe_engine_snapshot_capture_for_queue(q);
+	if (q) {
+		ss->ge = xe_guc_exec_queue_snapshot_capture(q);
+		if (job)
+			ss->job = xe_sched_job_snapshot_capture(job);
+		ss->vm = xe_vm_snapshot_capture(q->vm);
+
+		xe_engine_snapshot_capture_for_queue(q);
+	}
 
 	queue_work(system_dfl_wq, &ss->work);
 
@@ -364,19 +365,24 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump,
 }
 
 /**
- * xe_devcoredump - Take the required snapshots and initialize coredump device.
- * @q: The faulty xe_exec_queue, where the issue was detected.
- * @job: The faulty xe_sched_job, where the issue was detected.
+ * __xe_devcoredump - Take the required snapshots and initialize coredump device.
+ * @gt: The GT where the issue was detected.
+ * @q: The faulty xe_exec_queue, where the issue was detected, may be NULL for
+ *     hangs that are not tied to an exec queue (e.g. TLB invalidation
+ *     timeouts); in that case only the GT-level state (GuC log and CT state)
+ *     is captured.
+ * @job: The faulty xe_sched_job, where the issue was detected, may be NULL.
  * @fmt: Printf format + args to describe the reason for the core dump
  *
  * This function should be called at the crash time within the serialized
  * gt_reset. It is skipped if we still have the core dump device available
  * with the information of the 'first' snapshot.
  */
-__printf(3, 4)
-void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...)
+__printf(4, 5)
+void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q,
+		      struct xe_sched_job *job, const char *fmt, ...)
 {
-	struct xe_device *xe = gt_to_xe(q->gt);
+	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_devcoredump *coredump = &xe->devcoredump;
 	va_list varg;
 
@@ -394,7 +400,7 @@ void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const cha
 	coredump->snapshot.reason = kvasprintf(GFP_ATOMIC, fmt, varg);
 	va_end(varg);
 
-	devcoredump_snapshot(coredump, q, job);
+	devcoredump_snapshot(coredump, gt, q, job);
 
 	drm_info(&xe->drm, "Xe device coredump has been created\n");
 	drm_info(&xe->drm, "Check your /sys/class/drm/card%d/device/devcoredump/data\n",
diff --git a/drivers/gpu/drm/xe/xe_devcoredump.h b/drivers/gpu/drm/xe/xe_devcoredump.h
index 5391a80a4d1ba1..bc4800b3a3f3f4 100644
--- a/drivers/gpu/drm/xe/xe_devcoredump.h
+++ b/drivers/gpu/drm/xe/xe_devcoredump.h
@@ -11,15 +11,17 @@
 struct drm_printer;
 struct xe_device;
 struct xe_exec_queue;
+struct xe_gt;
 struct xe_sched_job;
 
 #ifdef CONFIG_DEV_COREDUMP
-void xe_devcoredump(struct xe_exec_queue *q, struct xe_sched_job *job, const char *fmt, ...);
+void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q,
+		      struct xe_sched_job *job, const char *fmt, ...);
 int xe_devcoredump_init(struct xe_device *xe);
 #else
-static inline void xe_devcoredump(struct xe_exec_queue *q,
-				  struct xe_sched_job *job,
-				  const char *fmt, ...)
+static inline void __xe_devcoredump(struct xe_gt *gt, struct xe_exec_queue *q,
+				    struct xe_sched_job *job,
+				    const char *fmt, ...)
 {
 }
 
@@ -29,6 +31,11 @@ static inline int xe_devcoredump_init(struct xe_device *xe)
 }
 #endif
 
+#define xe_devcoredump(_q, _job, _fmt, ...) \
+	__xe_devcoredump((_q)->gt, _q, _job, _fmt, ##__VA_ARGS__)
+#define xe_devcoredump_gt(_gt, _fmt, ...) \
+	__xe_devcoredump(_gt, NULL, NULL, _fmt, ##__VA_ARGS__)
+
 void xe_print_blob_ascii85(struct drm_printer *p, const char *prefix, char suffix,
 			   const void *blob, size_t offset, size_t size);
 
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index ee732e5495f7db..e1979bbcf7bc60 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -27,6 +27,7 @@
 #include "xe_bo_evict.h"
 #include "xe_configfs.h"
 #include "xe_debugfs.h"
+#include "xe_cpu_bind.h"
 #include "xe_defaults.h"
 #include "xe_devcoredump.h"
 #include "xe_device_sysfs.h"
@@ -48,7 +49,9 @@
 #include "xe_i2c.h"
 #include "xe_irq.h"
 #include "xe_late_bind_fw.h"
+#include "xe_log.h"
 #include "xe_mmio.h"
+#include "xe_mmio_gem.h"
 #include "xe_module.h"
 #include "xe_nvm.h"
 #include "xe_oa.h"
@@ -110,6 +113,8 @@ static int xe_file_open(struct drm_device *dev, struct drm_file *file)
 	mutex_init(&xef->exec_queue.lock);
 	xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
 
+	mutex_init(&xef->mmio_gem.lock);
+
 	file->driver_priv = xef;
 	kref_init(&xef->refcount);
 
@@ -132,6 +137,8 @@ static void xe_file_destroy(struct kref *ref)
 	xa_destroy(&xef->vm.xa);
 	mutex_destroy(&xef->vm.lock);
 
+	mutex_destroy(&xef->mmio_gem.lock);
+
 	xe_drm_client_put(xef->client);
 	kfree(xef->process_name);
 	kfree(xef);
@@ -188,6 +195,13 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file)
 	xa_for_each(&xef->vm.xa, idx, vm)
 		xe_vm_close_and_put(vm);
 
+	scoped_guard(mutex, &xef->mmio_gem.lock) {
+		if (xef->mmio_gem.pci_barrier) {
+			xe_mmio_gem_destroy(xef->mmio_gem.pci_barrier, file);
+			xef->mmio_gem.pci_barrier = NULL;
+		}
+	}
+
 	xe_file_put(xef);
 }
 
@@ -257,95 +271,6 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo
 #define xe_drm_compat_ioctl NULL
 #endif
 
-static void barrier_open(struct vm_area_struct *vma)
-{
-	drm_dev_get(vma->vm_private_data);
-}
-
-static void barrier_close(struct vm_area_struct *vma)
-{
-	drm_dev_put(vma->vm_private_data);
-}
-
-static void barrier_release_dummy_page(struct drm_device *dev, void *res)
-{
-	struct page *dummy_page = (struct page *)res;
-
-	__free_page(dummy_page);
-}
-
-static vm_fault_t barrier_fault(struct vm_fault *vmf)
-{
-	struct drm_device *dev = vmf->vma->vm_private_data;
-	struct vm_area_struct *vma = vmf->vma;
-	vm_fault_t ret = VM_FAULT_NOPAGE;
-	pgprot_t prot;
-	int idx;
-
-	prot = vma_get_page_prot(vma);
-
-	if (drm_dev_enter(dev, &idx)) {
-		unsigned long pfn;
-
-#define LAST_DB_PAGE_OFFSET 0x7ff001
-		pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
-				LAST_DB_PAGE_OFFSET);
-		ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,
-					  pgprot_noncached(prot));
-		drm_dev_exit(idx);
-	} else {
-		struct page *page;
-
-		/* Allocate new dummy page to map all the VA range in this VMA to it*/
-		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-		if (!page)
-			return VM_FAULT_OOM;
-
-		/* Set the page to be freed using drmm release action */
-		if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
-			return VM_FAULT_OOM;
-
-		ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),
-					  prot);
-	}
-
-	return ret;
-}
-
-static const struct vm_operations_struct vm_ops_barrier = {
-	.open = barrier_open,
-	.close = barrier_close,
-	.fault = barrier_fault,
-};
-
-static int xe_pci_barrier_mmap(struct file *filp,
-			       struct vm_area_struct *vma)
-{
-	struct drm_file *priv = filp->private_data;
-	struct drm_device *dev = priv->minor->dev;
-	struct xe_device *xe = to_xe_device(dev);
-
-	if (!IS_DGFX(xe))
-		return -EINVAL;
-
-	if (vma->vm_end - vma->vm_start > SZ_4K)
-		return -EINVAL;
-
-	if (vma_is_cow_mapping(vma))
-		return -EINVAL;
-
-	if (vma->vm_flags & (VM_READ | VM_EXEC))
-		return -EINVAL;
-
-	vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
-	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
-	vma->vm_ops = &vm_ops_barrier;
-	vma->vm_private_data = dev;
-	drm_dev_get(vma->vm_private_data);
-
-	return 0;
-}
-
 static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct drm_file *priv = filp->private_data;
@@ -354,11 +279,6 @@ static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
 	if (drm_dev_is_unplugged(dev))
 		return -ENODEV;
 
-	switch (vma->vm_pgoff) {
-	case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
-		return xe_pci_barrier_mmap(filp, vma);
-	}
-
 	return drm_gem_mmap(filp, vma);
 }
 
@@ -513,6 +433,17 @@ struct xe_device *xe_device_create(struct pci_dev *pdev)
 }
 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
 
+static void xe_device_parse_modparam(struct xe_device *xe)
+{
+	xe->atomic_svm_timeslice_ms = 5;
+	xe->min_run_period_lr_ms = 5;
+	xe->info.num_pf_work = xe_modparam.num_pf_work;
+	if (xe->info.num_pf_work < 1)
+		xe->info.num_pf_work = 1;
+	else if (xe->info.num_pf_work > XE_PAGEFAULT_WORK_MAX)
+		xe->info.num_pf_work = XE_PAGEFAULT_WORK_MAX;
+}
+
 /**
  * xe_device_init_early() - Initialize a new &xe_device instance
  * @xe: the &xe_device to initialize
@@ -539,8 +470,7 @@ int xe_device_init_early(struct xe_device *xe)
 	if (err)
 		return err;
 
-	xe->atomic_svm_timeslice_ms = 5;
-	xe->min_run_period_lr_ms = 5;
+	xe_device_parse_modparam(xe);
 
 	err = xe_irq_init(xe);
 	if (err)
@@ -742,6 +672,7 @@ static void vf_update_device_info(struct xe_device *xe)
 	xe->info.skip_guc_pc = 1;
 	xe->info.skip_pcode = 1;
 	xe->info.has_drm_ras = false;
+	xe->info.has_device_uid = false;
 }
 
 static int xe_device_vram_alloc(struct xe_device *xe)
@@ -942,6 +873,12 @@ static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe)
 }
 #endif
 
+static void xe_uid_probe(struct xe_device *xe)
+{
+	if (xe->info.has_device_uid)
+		xe->device_uid = xe_mmio_read64_2x32(xe_root_tile_mmio(xe), CRI_DEVICE_UID);
+}
+
 int xe_device_probe(struct xe_device *xe)
 {
 	struct xe_tile *tile;
@@ -949,6 +886,8 @@ int xe_device_probe(struct xe_device *xe)
 	int err;
 	u8 id;
 
+	xe_uid_probe(xe);
+
 	xe_pat_init_early(xe);
 
 	err = xe_sriov_init(xe);
@@ -1040,6 +979,10 @@ int xe_device_probe(struct xe_device *xe)
 	if (err)
 		return err;
 
+	err = xe_vram_reserve_memtest_bo(xe);
+	if (err)
+		return err;
+
 	for_each_tile(tile, xe, id) {
 		err = xe_tile_init(tile);
 		if (err)
@@ -1056,6 +999,14 @@ int xe_device_probe(struct xe_device *xe)
 			return err;
 	}
 
+	err = xe_vram_memtest(xe);
+	if (err)
+		return err;
+
+	err = xe_cpu_bind_init(xe);
+	if (err)
+		return err;
+
 	err = xe_pagefault_init(xe);
 	if (err)
 		return err;
@@ -1259,7 +1210,7 @@ bool xe_device_is_l2_flush_optimized(struct xe_device *xe)
 	return false;
 }
 
-void xe_device_l2_flush(struct xe_device *xe)
+void xe_device_l2_flush(struct xe_device *xe, bool force)
 {
 	struct xe_gt *gt;
 
@@ -1267,7 +1218,7 @@ void xe_device_l2_flush(struct xe_device *xe)
 	if (!gt)
 		return;
 
-	if (!XE_GT_WA(gt, 16023588340))
+	if (!force && !XE_GT_WA(gt, 16023588340))
 		return;
 
 	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
@@ -1322,7 +1273,7 @@ void xe_device_td_flush(struct xe_device *xe)
 
 	if (XE_GT_WA(root_gt, 16023588340)) {
 		/* A transient flush is not sufficient: flush the L2 */
-		xe_device_l2_flush(xe);
+		xe_device_l2_flush(xe, false);
 	} else {
 		xe_guc_pc_apply_flush_freq_limit(&root_gt->uc.guc.pc);
 		tdf_request_sync(xe);
@@ -1432,6 +1383,9 @@ void xe_device_set_wedged_method(struct xe_device *xe, unsigned long method)
 	xe->wedged.method = method;
 }
 
+#define WEDGED_URL	"https://docs.kernel.org/gpu/drm-uapi.html#device-wedging"
+#define XE_BUG_URL	"https://gitlab.freedesktop.org/drm/xe/kernel/issues/new"
+
 /**
  * xe_device_declare_wedged - Declare device wedged
  * @xe: xe device instance
@@ -1463,12 +1417,12 @@ void xe_device_declare_wedged(struct xe_device *xe)
 	if (!atomic_xchg(&xe->wedged.flag, 1)) {
 		xe->needs_flr_on_fini = true;
 		xe_pm_runtime_get_noresume(xe);
-		drm_err(&xe->drm,
-			"CRITICAL: Xe has declared device %s as wedged.\n"
-			"IOCTLs and executions are blocked.\n"
-			"For recovery procedure, refer to https://docs.kernel.org/gpu/drm-uapi.html#device-wedging\n"
-			"Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
-			dev_name(xe->drm.dev));
+
+		xe_log_err_fatal(xe, WEDGED, -EIO, "Device declared wedged!\n");
+		xe_err_once(xe, "IOCTLs and executions are now blocked!\n"
+			    "For recovery procedure, refer to %s\n"
+			    "Please file a _new_ bug report at %s\n",
+			    WEDGED_URL, XE_BUG_URL);
 	}
 
 	for_each_gt(gt, xe, id)
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index 6c4cfaebc44a0e..6d3d6d5eba298e 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -205,7 +205,7 @@ u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);
 
 bool xe_device_is_l2_flush_optimized(struct xe_device *xe);
 void xe_device_td_flush(struct xe_device *xe);
-void xe_device_l2_flush(struct xe_device *xe);
+void xe_device_l2_flush(struct xe_device *xe, bool force);
 
 static inline bool xe_device_wedged(struct xe_device *xe)
 {
diff --git a/drivers/gpu/drm/xe/xe_device_sysfs.c b/drivers/gpu/drm/xe/xe_device_sysfs.c
index a73e0e957cb0bb..18783a286a830e 100644
--- a/drivers/gpu/drm/xe/xe_device_sysfs.c
+++ b/drivers/gpu/drm/xe/xe_device_sysfs.c
@@ -8,6 +8,7 @@
 #include <linux/pci.h>
 #include <linux/sysfs.h>
 
+#include "regs/xe_regs.h"
 #include "xe_device.h"
 #include "xe_device_sysfs.h"
 #include "xe_mmio.h"
@@ -264,6 +265,35 @@ static const struct attribute_group auto_link_downgrade_attr_group = {
 	.attrs = auto_link_downgrade_attrs,
 };
 
+/**
+ * DOC: Device Unique ID
+ *
+ * On supported platforms, Xe driver exposes a unique 64-bit GPU SOC
+ * device identifier through the 'device_uid' sysfs entry.
+ *
+ * See Documentation/ABI/testing/sysfs-driver-intel-xe-gpu for the ABI
+ * specification.
+ */
+
+static ssize_t
+device_uid_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct xe_device *xe = pdev_to_xe_device(pdev);
+
+	return sysfs_emit(buf, "0x%016llx\n", xe->device_uid);
+}
+static DEVICE_ATTR_ADMIN_RO(device_uid);
+
+static struct attribute *device_uid_attrs[] = {
+	&dev_attr_device_uid.attr,
+	NULL
+};
+
+static const struct attribute_group device_uid_attr_group = {
+	.attrs = device_uid_attrs,
+};
+
 int xe_device_sysfs_init(struct xe_device *xe)
 {
 	struct device *dev = xe->drm.dev;
@@ -285,5 +315,11 @@ int xe_device_sysfs_init(struct xe_device *xe)
 			return ret;
 	}
 
+	if (xe->info.has_device_uid) {
+		ret = devm_device_add_group(dev, &device_uid_attr_group);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 03a7bb08adf7bf..41f703a8791696 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -38,8 +38,10 @@
 struct drm_pagemap_shrinker;
 struct intel_display;
 struct intel_dg_nvm_dev;
+struct xe_cpu_bind;
 struct xe_ggtt;
 struct xe_i2c;
+struct xe_mmio_gem;
 struct xe_pat_ops;
 struct xe_pxp;
 struct xe_ttm_stolen_mgr;
@@ -88,9 +90,7 @@ enum xe_page_size_alloc_ctrl_mode {
 
 #define XE_VRAM_FLAGS_NEED64K		BIT(0)
 
-#define XE_GT0		0
-#define XE_GT1		1
-#define XE_MAX_TILES_PER_DEVICE	(XE_GT1 + 1)
+#define XE_MAX_TILES_PER_DEVICE 2
 
 /*
  * Highest GT/tile count for any platform.  Used only for memory allocation
@@ -116,6 +116,12 @@ struct xe_device {
 	/** @devcoredump: device coredump */
 	struct xe_devcoredump devcoredump;
 
+	/** @desc: device descriptor */
+	const struct xe_device_desc *desc;
+
+	/** @subplatform_desc: subplatform descriptor */
+	const struct xe_subplatform_desc *subplatform_desc;
+
 	/** @info: device info */
 	struct intel_device_info {
 		/** @info.platform_name: platform name */
@@ -140,6 +146,8 @@ struct xe_device {
 		u8 revid;
 		/** @info.step: stepping information for each IP */
 		struct xe_step_info step;
+		/** @info.num_pf_work: Number of page fault work thread */
+		int num_pf_work;
 		/** @info.dma_mask_size: DMA address bits */
 		u8 dma_mask_size;
 		/** @info.vram_flags: Vram flags */
@@ -171,6 +179,8 @@ struct xe_device {
 		u8 has_cached_pt:1;
 		/** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */
 		u8 has_device_atomics_on_smem:1;
+		/** @info.has_device_uid: Device supports unique 64-bit GPU SOC ID */
+		u8 has_device_uid:1;
 		/** @info.has_drm_ras: Device supports drm_ras (Reliability, Availability, Serviceability) */
 		u8 has_drm_ras:1;
 		/** @info.has_fan_control: Device supports fan control */
@@ -221,6 +231,8 @@ struct xe_device {
 		u8 has_usm:1;
 		/** @info.has_64bit_timestamp: Device supports 64-bit timestamps */
 		u8 has_64bit_timestamp:1;
+		/** @info.has_pt_mirror: Device has PT mirroring across tiles */
+		u8 has_pt_mirror:1;
 		/** @info.is_dgfx: is discrete device */
 		u8 is_dgfx:1;
 		/** @info.needs_scratch: needs scratch page for oob prefetch to work */
@@ -255,6 +267,9 @@ struct xe_device {
 		bool oob_initialized;
 	} wa_active;
 
+	/** @device_uid: unique 64-bit GPU SOC identifier */
+	u64 device_uid;
+
 	/** @survivability: survivability information for device */
 	struct xe_survivability survivability;
 
@@ -320,18 +335,19 @@ struct xe_device {
 		struct xarray asid_to_vm;
 		/** @usm.next_asid: next ASID, used to cyclical alloc asids */
 		u32 next_asid;
+		/** @usm.current_pf_work: current page fault work item */
+		u32 current_pf_work;
 		/** @usm.lock: protects UM state */
 		struct rw_semaphore lock;
-		/** @usm.pf_wq: page fault work queue, unbound, high priority */
-		struct workqueue_struct *pf_wq;
-		/*
-		 * We pick 4 here because, in the current implementation, it
-		 * yields the best bandwidth utilization of the kernel paging
-		 * engine.
-		 */
-#define XE_PAGEFAULT_QUEUE_COUNT	4
-		/** @usm.pf_queue: Page fault queues */
-		struct xe_pagefault_queue pf_queue[XE_PAGEFAULT_QUEUE_COUNT];
+		/** @usm.pagefault_wq: page fault work queue, unbound, high priority */
+		struct workqueue_struct *pagefault_wq;
+		/** @usm.prefetch_wq: threaded prefetch work queue, unbound */
+		struct workqueue_struct *prefetch_wq;
+#define XE_PAGEFAULT_WORK_MAX	8
+		/** @usm.pf_workers: Page fault workers */
+		struct xe_pagefault_work pf_workers[XE_PAGEFAULT_WORK_MAX];
+		/** @usm.pf_queue: Page fault queue */
+		struct xe_pagefault_queue pf_queue;
 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP)
 		/** @usm.dpagemap_shrinker: Shrinker for unused pagemaps */
 		struct drm_pagemap_shrinker *dpagemap_shrinker;
@@ -450,6 +466,19 @@ struct xe_device {
 		struct mutex lock;
 	} d3cold;
 
+	/** @pme: Encapsulate pme related stuff */
+	struct {
+		/** @pme.capable: Indicates if device is PME capable */
+		bool capable;
+
+		/** @pme.enabled:
+		 *
+		 * Indicates if PME is enabled - depends on user controllable
+		 * sysfs interface as well
+		 */
+		bool enabled;
+	} pme;
+
 	/** @pm_notifier: Our PM notifier to perform actions in response to various PM events. */
 	struct notifier_block pm_notifier;
 	/** @pm_block: Completion to block validating tasks on suspend / hibernate prepare */
@@ -556,6 +585,9 @@ struct xe_device {
 	/** @sc: System Controller */
 	struct xe_sysctrl sc;
 
+	/** @cpu_bind: CPU bind object */
+	struct xe_cpu_bind *cpu_bind;
+
 	/** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */
 	u32 atomic_svm_timeslice_ms;
 
@@ -673,6 +705,18 @@ struct xe_file {
 
 	/** @refcount: ref count of this xe file */
 	struct kref refcount;
+
+	/** @mmio_gem: MMIO GEM objects for this xe file */
+	struct {
+		/**
+		 * @mmio_gem.lock: Protects allocation and attach of MMIO
+		 * GEM objects on first use (singleton). All MMIO GEM access
+		 * should be guarded by this lock. Prefer scoped_guard().
+		 */
+		struct mutex lock;
+		/** @mmio_gem.pci_barrier: MMIO GEM object for PCI barrier mmap. */
+		struct xe_mmio_gem *pci_barrier;
+	} mmio_gem;
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c
index bf0728838ead51..6a85b292dee7b8 100644
--- a/drivers/gpu/drm/xe/xe_dma_buf.c
+++ b/drivers/gpu/drm/xe/xe_dma_buf.c
@@ -7,6 +7,7 @@
 
 #include <kunit/test.h>
 #include <linux/dma-buf.h>
+#include <linux/dynamic_debug.h>
 #include <linux/pci-p2pdma.h>
 
 #include <drm/drm_device.h>
@@ -22,13 +23,29 @@
 
 MODULE_IMPORT_NS("DMA_BUF");
 
+#if defined(CONFIG_DYNAMIC_DEBUG) || \
+	(defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
+static void xe_ddebug_flip_bool(const struct _ddebug *desc, bool *flag)
+{
+	*flag = true;
+}
+
+#define xe_dma_buf_p2p_debug_flip(flag) \
+	_dynamic_func_call("xe dma-buf p2p debug", xe_ddebug_flip_bool, flag)
+#else
+#define xe_dma_buf_p2p_debug_flip(flag) do { } while (0)
+#endif
+
 static int xe_dma_buf_attach(struct dma_buf *dmabuf,
 			     struct dma_buf_attachment *attach)
 {
 	struct drm_gem_object *obj = attach->dmabuf->priv;
+	bool verbose_p2p_dist = drm_debug_enabled(DRM_UT_DRIVER);
+
+	xe_dma_buf_p2p_debug_flip(&verbose_p2p_dist);
 
 	if (attach->peer2peer &&
-	    pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
+	    pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, verbose_p2p_dist) < 0)
 		attach->peer2peer = false;
 
 	if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
@@ -104,6 +121,9 @@ static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
 	struct sg_table *sgt;
 	int r = 0;
 
+	if (xe_bo_is_purged(bo))
+		return ERR_PTR(-ENOENT);
+
 	if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
 		return ERR_PTR(-EOPNOTSUPP);
 
diff --git a/drivers/gpu/drm/xe/xe_drm_client.c b/drivers/gpu/drm/xe/xe_drm_client.c
index e116fb562c4c60..4c424d1c6721f2 100644
--- a/drivers/gpu/drm/xe/xe_drm_client.c
+++ b/drivers/gpu/drm/xe/xe_drm_client.c
@@ -256,7 +256,7 @@ static void show_meminfo(struct drm_printer *p, struct drm_file *file)
 			xe_assert(xef->xe, !list_empty(&bo->client_link));
 		}
 
-		xe_bo_put_deferred(bo, &deferred);
+		xe_bo_put_deferred(bo, &deferred, NULL);
 	}
 	spin_unlock(&client->bos_lock);
 
diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c
index 984fa67b901545..7f3695707611c6 100644
--- a/drivers/gpu/drm/xe/xe_drm_ras.c
+++ b/drivers/gpu/drm/xe/xe_drm_ras.c
@@ -86,6 +86,38 @@ static int clear_correctable_error_counter(struct drm_ras_node *node, u32 error_
 	return clear_error_counter(xe, DRM_XE_RAS_ERR_SEV_CORRECTABLE, error_id);
 }
 
+static int query_correctable_error_threshold(struct drm_ras_node *ep, u32 error_id,
+					     const char **name, u32 *threshold)
+{
+	struct xe_device *xe = ep->priv;
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE];
+
+	if (!info || !info[error_id].name)
+		return -ENOENT;
+
+	if (!xe->info.has_sysctrl)
+		return -EOPNOTSUPP;
+
+	*name = info[error_id].name;
+	return xe_ras_get_threshold(xe, DRM_XE_RAS_ERR_SEV_CORRECTABLE, error_id, threshold);
+}
+
+static int set_correctable_error_threshold(struct drm_ras_node *ep, u32 error_id, u32 threshold)
+{
+	struct xe_device *xe = ep->priv;
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE];
+
+	if (!info || !info[error_id].name)
+		return -ENOENT;
+
+	if (!xe->info.has_sysctrl)
+		return -EOPNOTSUPP;
+
+	return xe_ras_set_threshold(xe, DRM_XE_RAS_ERR_SEV_CORRECTABLE, error_id, threshold);
+}
+
 static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *xe)
 {
 	struct xe_drm_ras_counter *counter;
@@ -134,6 +166,8 @@ static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node,
 	if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) {
 		node->query_error_counter = query_correctable_error_counter;
 		node->clear_error_counter = clear_correctable_error_counter;
+		node->query_error_threshold = query_correctable_error_threshold;
+		node->set_error_threshold = set_correctable_error_threshold;
 	} else {
 		node->query_error_counter = query_uncorrectable_error_counter;
 		node->clear_error_counter = clear_uncorrectable_error_counter;
@@ -185,6 +219,37 @@ static int register_nodes(struct xe_device *xe)
 	return ret;
 }
 
+/**
+ * xe_drm_ras_event() - Report drm-ras error event to userspace
+ * @xe: xe device structure
+ * @component: error component (see &enum drm_xe_ras_error_component)
+ * @severity: error severity (see &enum drm_xe_ras_error_severity)
+ * @value: value of error counter
+ *
+ * Report an error-event to userspace.
+ */
+void xe_drm_ras_event(struct xe_device *xe, u32 component, u32 severity, u32 value)
+{
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[severity];
+	struct drm_ras_node *node;
+	int ret;
+
+	/* Event is supported only if drm-ras is enabled */
+	if (!xe->info.has_drm_ras)
+		return;
+
+	node = &ras->node[severity];
+
+	if (!info || !info[component].name)
+		return;
+
+	ret = drm_ras_nl_error_event(node, component, info[component].name, value);
+	if (ret)
+		drm_err_ratelimited(&xe->drm, "drm-ras error-event failed: %d for %s %s\n", ret,
+				    info[component].name, error_severity[severity]);
+}
+
 /**
  * xe_drm_ras_init() - Initialize DRM RAS
  * @xe: xe device instance
diff --git a/drivers/gpu/drm/xe/xe_drm_ras.h b/drivers/gpu/drm/xe/xe_drm_ras.h
index 365c70e93e82fd..723229b2cffb63 100644
--- a/drivers/gpu/drm/xe/xe_drm_ras.h
+++ b/drivers/gpu/drm/xe/xe_drm_ras.h
@@ -5,11 +5,14 @@
 #ifndef _XE_DRM_RAS_H_
 #define _XE_DRM_RAS_H_
 
+#include <linux/types.h>
+
 struct xe_device;
 
 #define for_each_error_severity(i)	\
 	for (i = 0; i < DRM_XE_RAS_ERR_SEV_MAX; i++)
 
 int xe_drm_ras_init(struct xe_device *xe);
+void xe_drm_ras_event(struct xe_device *xe, u32 component, u32 severity, u32 value);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_drm_ras_types.h b/drivers/gpu/drm/xe/xe_drm_ras_types.h
index 8d729ad6a264dd..0be218ba2db73f 100644
--- a/drivers/gpu/drm/xe/xe_drm_ras_types.h
+++ b/drivers/gpu/drm/xe/xe_drm_ras_types.h
@@ -43,6 +43,9 @@ struct xe_drm_ras {
 
 	/** @info: info array for all types of errors */
 	struct xe_drm_ras_counter *info[DRM_XE_RAS_ERR_SEV_MAX];
+
+	/** @disable_vram_page_offline: cached configfs policy, immutable after init */
+	bool disable_vram_page_offline;
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index cd053de3c6b56b..9a8efa2bf2bfb4 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -14,6 +14,7 @@
 #include <uapi/drm/xe_drm.h>
 
 #include "xe_bo.h"
+#include "xe_cpu_bind.h"
 #include "xe_dep_scheduler.h"
 #include "xe_device.h"
 #include "xe_gt.h"
@@ -142,9 +143,8 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q)
 {
 	int i;
 
-	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i)
-		if (q->tlb_inval[i].dep_scheduler)
-			xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler);
+	for_each_tlb_inval(q, i)
+		xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler);
 
 	if (xe_exec_queue_uses_pxp(q))
 		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
@@ -166,31 +166,34 @@ static void __xe_exec_queue_free(struct xe_exec_queue *q)
 
 static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q)
 {
-	struct xe_tile *tile = gt_to_tile(q->gt);
-	int i;
+	struct xe_tile *tile;
+	int i = 0, j;
+	u8 id;
 
-	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) {
-		struct xe_dep_scheduler *dep_scheduler;
-		struct xe_gt *gt;
-		struct workqueue_struct *wq;
+	for_each_tile(tile, xe, id) {
+		for (j = 0; j < (XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1); ++j, ++i) {
+			struct xe_dep_scheduler *dep_scheduler;
+			struct xe_gt *gt;
+			struct workqueue_struct *wq;
 
-		if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)
-			gt = tile->primary_gt;
-		else
-			gt = tile->media_gt;
+			if (j == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)
+				gt = tile->primary_gt;
+			else
+				gt = tile->media_gt;
 
-		if (!gt)
-			continue;
+			if (!gt)
+				continue;
 
-		wq = gt->tlb_inval.job_wq;
+			wq = gt->tlb_inval.job_wq;
 
 #define MAX_TLB_INVAL_JOBS	16	/* Picking a reasonable value */
-		dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,
-							MAX_TLB_INVAL_JOBS);
-		if (IS_ERR(dep_scheduler))
-			return PTR_ERR(dep_scheduler);
+			dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,
+								MAX_TLB_INVAL_JOBS);
+			if (IS_ERR(dep_scheduler))
+				return PTR_ERR(dep_scheduler);
 
-		q->tlb_inval[i].dep_scheduler = dep_scheduler;
+			q->tlb_inval[i].dep_scheduler = dep_scheduler;
+		}
 	}
 #undef MAX_TLB_INVAL_JOBS
 
@@ -207,9 +210,6 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 	struct xe_gt *gt = hwe->gt;
 	int err;
 
-	/* only kernel queues can be permanent */
-	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL));
-
 	q = kzalloc_flex(*q, lrc, width);
 	if (!q)
 		return ERR_PTR(-ENOMEM);
@@ -227,7 +227,6 @@ static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
 	q->ops = gt->exec_queue_ops;
 	INIT_LIST_HEAD(&q->lr.link);
 	INIT_LIST_HEAD(&q->vm_exec_queue_link);
-	INIT_LIST_HEAD(&q->multi_gt_link);
 	INIT_LIST_HEAD(&q->hw_engine_group_link);
 	INIT_LIST_HEAD(&q->pxp.link);
 	spin_lock_init(&q->multi_queue.lock);
@@ -325,10 +324,66 @@ struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q)
 	return q->lrc[0];
 }
 
+/*
+ * Publish the queue back-pointer in the LRC BOs.
+ *
+ * The BO holds no reference on the queue; the queue owns the LRCs, and
+ * therefore the BOs, instead. The back-pointer is made safe by two rules:
+ *
+ *  - It is published only once the queue is fully constructed and can no
+ *    longer be destroyed by an error path that bypasses the kref (see
+ *    xe_exec_queue_create()), so a reader that successfully takes a
+ *    reference can never be handed a queue that is freed without going
+ *    through xe_exec_queue_destroy().
+ *
+ *  - It is written and cleared under the BO dma_resv. Readers must hold
+ *    the same lock across both the read and
+ *    xe_exec_queue_get_unless_zero(), which serializes them against
+ *    xe_exec_queue_clear_lrc_bo_backpointer() below.
+ *
+ * For a multi-queue group the LRC BOs point at the primary queue, which is
+ * kept alive by the reference every secondary holds on it.
+ */
+static void xe_exec_queue_set_lrc_bo_backpointer(struct xe_exec_queue *q)
+{
+	struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
+	int i;
+
+	for (i = 0; i < q->width; ++i) {
+		struct xe_bo *bo = q->lrc[i]->bo;
+
+		xe_bo_lock(bo, false);
+		bo->q = primary;
+		xe_bo_unlock(bo);
+	}
+}
+
+/*
+ * Drop the queue back-pointer before anything belonging to the queue is
+ * torn down. This must happen before q->ops->fini(), otherwise a reader
+ * could take a reference and then operate on an already destroyed backend.
+ */
+static void xe_exec_queue_clear_lrc_bo_backpointer(struct xe_exec_queue *q)
+{
+	int i;
+
+	for (i = 0; i < q->width; ++i) {
+		struct xe_bo *bo = q->lrc[i] ? q->lrc[i]->bo : NULL;
+
+		if (!bo)
+			continue;
+
+		xe_bo_lock(bo, false);
+		bo->q = NULL;
+		xe_bo_unlock(bo);
+	}
+}
+
 static void __xe_exec_queue_fini(struct xe_exec_queue *q)
 {
 	int i;
 
+	xe_exec_queue_clear_lrc_bo_backpointer(q);
 	q->ops->fini(q);
 
 	for (i = 0; i < q->width; ++i)
@@ -453,6 +508,14 @@ struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *v
 			goto err_post_init;
 	}
 
+	/*
+	 * Publish the LRC BO back-pointers last: past this point the queue can
+	 * only be destroyed through xe_exec_queue_destroy(), so a concurrent
+	 * reader that takes a reference via bo->q cannot race with the
+	 * kref-bypassing error paths below.
+	 */
+	xe_exec_queue_set_lrc_bo_backpointer(q);
+
 	return q;
 
 err_post_init:
@@ -575,7 +638,6 @@ ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
 void xe_exec_queue_destroy(struct kref *ref)
 {
 	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
-	struct xe_exec_queue *eq, *next;
 	int i;
 
 	xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0);
@@ -587,15 +649,9 @@ void xe_exec_queue_destroy(struct kref *ref)
 		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
 
 	xe_exec_queue_last_fence_put_unlocked(q);
-	for_each_tlb_inval(i)
+	for_each_tlb_inval(q, i)
 		xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i);
 
-	if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {
-		list_for_each_entry_safe(eq, next, &q->multi_gt_list,
-					 multi_gt_link)
-			xe_exec_queue_put(eq);
-	}
-
 	if (q->user_vm) {
 		xe_vm_put(q->user_vm);
 		q->user_vm = NULL;
@@ -1121,6 +1177,9 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe,
 	if (!exec_queue_set_property_funcs[idx])
 		return -EINVAL;
 
+	if (XE_IOCTL_DBG(xe, *properties & BIT_ULL(idx)))
+		return -EINVAL;
+
 	*properties |= BIT_ULL(idx);
 	err = exec_queue_user_ext_check(q, *properties);
 	if (XE_IOCTL_DBG(xe, err))
@@ -1279,7 +1338,6 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 		u64_to_user_ptr(args->instances);
 	struct xe_hw_engine *hwe;
 	struct xe_vm *vm;
-	struct xe_tile *tile;
 	struct xe_exec_queue *q = NULL;
 	u32 logical_mask;
 	u32 flags = 0;
@@ -1328,31 +1386,16 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
 			return -ENOENT;
 		}
 
-		for_each_tile(tile, xe, id) {
-			struct xe_exec_queue *new;
-
-			flags |= EXEC_QUEUE_FLAG_VM;
-			if (id)
-				flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD;
-
-			new = xe_exec_queue_create_bind(xe, tile, vm, flags,
-							args->extensions);
-			if (IS_ERR(new)) {
-				up_read(&vm->lock);
-				xe_vm_put(vm);
-				err = PTR_ERR(new);
-				if (q)
-					goto put_exec_queue;
-				return err;
-			}
-			if (id == 0)
-				q = new;
-			else
-				list_add_tail(&new->multi_gt_list,
-					      &q->multi_gt_link);
-		}
+		flags |= EXEC_QUEUE_FLAG_VM;
+
+		q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe),
+					      vm, flags, args->extensions);
 		up_read(&vm->lock);
 		xe_vm_put(vm);
+		if (IS_ERR(q)) {
+			err = PTR_ERR(q);
+			return err;
+		}
 	} else {
 		logical_mask = calc_validate_logical_mask(xe, eci,
 							  args->width,
@@ -1503,6 +1546,7 @@ bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
 /**
  * xe_exec_queue_is_idle() - Whether an exec_queue is idle.
  * @q: The exec_queue
+ * @extra_jobs: Extra jobs on the queue
  *
  * FIXME: Need to determine what to use as the short-lived
  * timeline lock for the exec_queues, so that the return value
@@ -1514,22 +1558,9 @@ bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
  *
  * Return: True if the exec_queue is idle, false otherwise.
  */
-bool xe_exec_queue_is_idle(struct xe_exec_queue *q)
+bool xe_exec_queue_is_idle(struct xe_exec_queue *q, int extra_jobs)
 {
-	if (xe_exec_queue_is_parallel(q)) {
-		int i;
-
-		for (i = 0; i < q->width; ++i) {
-			if (xe_lrc_seqno(q->lrc[i]) !=
-			    q->lrc[i]->fence_ctx.next_seqno - 1)
-				return false;
-		}
-
-		return true;
-	}
-
-	return xe_lrc_seqno(q->lrc[0]) ==
-		q->lrc[0]->fence_ctx.next_seqno - 1;
+	return !(atomic_read(&q->job_cnt) - extra_jobs);
 }
 
 /**
@@ -1566,8 +1597,12 @@ void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
 	 * errors.
 	 */
 	lrc = q->lrc[0];
-	new_ts = xe_lrc_update_timestamp(lrc, &old_ts);
-	q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width;
+	xe_bo_lock(lrc->bo, false);
+	if (!xe_bo_is_purged(lrc->bo)) {
+		new_ts = xe_lrc_update_timestamp(lrc, &old_ts);
+		q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width;
+	}
+	xe_bo_unlock(lrc->bo);
 
 	drm_dev_exit(idx);
 }
@@ -1583,14 +1618,6 @@ void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
  */
 void xe_exec_queue_kill(struct xe_exec_queue *q)
 {
-	struct xe_exec_queue *eq = q, *next;
-
-	list_for_each_entry_safe(eq, next, &eq->multi_gt_list,
-				 multi_gt_link) {
-		q->ops->kill(eq);
-		xe_vm_remove_compute_exec_queue(q->vm, eq);
-	}
-
 	q->ops->kill(q);
 	xe_vm_remove_compute_exec_queue(q->vm, q);
 }
@@ -1641,7 +1668,7 @@ static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q,
 						    struct xe_vm *vm)
 {
 	if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) {
-		xe_migrate_job_lock_assert(q);
+		xe_cpu_bind_job_lock_assert(q);
 	} else if (q->flags & EXEC_QUEUE_FLAG_VM) {
 		lockdep_assert_held(&vm->lock);
 	} else {
@@ -1751,42 +1778,40 @@ void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm,
  * xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence
  * @q: The exec queue
  * @vm: The VM the engine does a bind for
- * @type: Either primary or media GT
+ * @idx: Index of tlb invalidation
  */
 void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
 					    struct xe_vm *vm,
-					    unsigned int type)
+					    unsigned int idx)
 {
 	xe_exec_queue_last_fence_lockdep_assert(q, vm);
-	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
-		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
+	xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
 
-	xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type);
+	xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, idx);
 }
 
 /**
  * xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB
  * invalidation fence unlocked
  * @q: The exec queue
- * @type: Either primary or media GT
+ * @idx: Index of tlb invalidation
  *
  * Only safe to be called from xe_exec_queue_destroy().
  */
 void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
-						     unsigned int type)
+						     unsigned int idx)
 {
-	xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
-		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
+	xe_assert(gt_to_xe(q->gt), idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
 
-	dma_fence_put(q->tlb_inval[type].last_fence);
-	q->tlb_inval[type].last_fence = NULL;
+	dma_fence_put(q->tlb_inval[idx].last_fence);
+	q->tlb_inval[idx].last_fence = NULL;
 }
 
 /**
  * xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation
  * @q: The exec queue
  * @vm: The VM the engine does a bind for
- * @type: Either primary or media GT
+ * @idx: Index of tlb invalidation
  *
  * Get last fence, takes a ref
  *
@@ -1794,22 +1819,21 @@ void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
  */
 struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q,
 							 struct xe_vm *vm,
-							 unsigned int type)
+							 unsigned int idx)
 {
 	struct dma_fence *fence;
 
 	xe_exec_queue_last_fence_lockdep_assert(q, vm);
-	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
-		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
+	xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
 	xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
 				      EXEC_QUEUE_FLAG_MIGRATE));
 
-	if (q->tlb_inval[type].last_fence &&
+	if (q->tlb_inval[idx].last_fence &&
 	    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
-		     &q->tlb_inval[type].last_fence->flags))
-		xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
+		     &q->tlb_inval[idx].last_fence->flags))
+		xe_exec_queue_tlb_inval_last_fence_put(q, vm, idx);
 
-	fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub();
+	fence = q->tlb_inval[idx].last_fence ?: dma_fence_get_stub();
 	dma_fence_get(fence);
 	return fence;
 }
@@ -1819,26 +1843,25 @@ struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q
  * @q: The exec queue
  * @vm: The VM the engine does a bind for
  * @fence: The fence
- * @type: Either primary or media GT
+ * @idx: Index of tlb invalidation
  *
- * Set the last fence for the tlb invalidation type on the queue. Increases
+ * Set the last fence for the tlb invalidation client on the queue. Increases
  * reference count for fence, when closing queue
  * xe_exec_queue_tlb_inval_last_fence_put should be called.
  */
 void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q,
 					    struct xe_vm *vm,
 					    struct dma_fence *fence,
-					    unsigned int type)
+					    unsigned int idx)
 {
 	xe_exec_queue_last_fence_lockdep_assert(q, vm);
-	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
-		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
+	xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
 	xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
 				      EXEC_QUEUE_FLAG_MIGRATE));
 	xe_assert(vm->xe, !dma_fence_is_container(fence));
 
-	xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
-	q->tlb_inval[type].last_fence = dma_fence_get(fence);
+	xe_exec_queue_tlb_inval_last_fence_put(q, vm, idx);
+	q->tlb_inval[idx].last_fence = dma_fence_get(fence);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h
index 0225426c57b085..e8963f85cabd93 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue.h
@@ -14,9 +14,9 @@ struct drm_file;
 struct xe_device;
 struct xe_file;
 
-#define for_each_tlb_inval(__i)	\
-	for (__i = XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT; \
-	     __i <= XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT; ++__i)
+#define for_each_tlb_inval(__q, __i)	\
+	for (__i = 0; __i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++__i)	\
+		for_each_if((__q)->tlb_inval[__i].dep_scheduler)
 
 struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
 					   u32 logical_mask, u16 width,
@@ -116,7 +116,7 @@ static inline struct xe_exec_queue *xe_exec_queue_multi_queue_primary(struct xe_
 
 bool xe_exec_queue_is_lr(struct xe_exec_queue *q);
 
-bool xe_exec_queue_is_idle(struct xe_exec_queue *q);
+bool xe_exec_queue_is_idle(struct xe_exec_queue *q, int extra_jobs);
 
 void xe_exec_queue_kill(struct xe_exec_queue *q);
 
@@ -141,19 +141,19 @@ void xe_exec_queue_last_fence_set(struct xe_exec_queue *e, struct xe_vm *vm,
 
 void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
 					    struct xe_vm *vm,
-					    unsigned int type);
+					    unsigned int idx);
 
 void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
-						     unsigned int type);
+						     unsigned int idx);
 
 struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q,
 							 struct xe_vm *vm,
-							 unsigned int type);
+							 unsigned int idx);
 
 void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q,
 					    struct xe_vm *vm,
 					    struct dma_fence *fence,
-					    unsigned int type);
+					    unsigned int idx);
 
 void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q);
 
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index 53b6c0bf484992..48c05398b0168c 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -70,6 +70,13 @@ struct xe_exec_queue_group {
 	spinlock_t suspend_lock;
 	/** @sync_pending: CGP_SYNC_DONE g2h response pending */
 	bool sync_pending;
+	/**
+	 * @cgp_update_q: Queue that issued the currently outstanding (sent)
+	 * CGP_SYNC or REGISTER_CONTEXT_MULTI_QUEUE; NULL when none is
+	 * outstanding. Used during VF recovery to identify and replay the
+	 * message whose CGP_SYNC_DONE was not received.
+	 */
+	struct xe_exec_queue *cgp_update_q;
 	/** @banned: Group banned */
 	bool banned;
 	/** @stopped: Group is stopped, protected by list_lock */
@@ -128,20 +135,16 @@ struct xe_exec_queue {
 
 /* queue used for kernel submission only */
 #define EXEC_QUEUE_FLAG_KERNEL			BIT(0)
-/* kernel engine only destroyed at driver unload */
-#define EXEC_QUEUE_FLAG_PERMANENT		BIT(1)
 /* for VM jobs. Caller needs to hold rpm ref when creating queue with this flag */
-#define EXEC_QUEUE_FLAG_VM			BIT(2)
-/* child of VM queue for multi-tile VM jobs */
-#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD	BIT(3)
+#define EXEC_QUEUE_FLAG_VM			BIT(1)
 /* kernel exec_queue only, set priority to highest level */
-#define EXEC_QUEUE_FLAG_HIGH_PRIORITY		BIT(4)
+#define EXEC_QUEUE_FLAG_HIGH_PRIORITY		BIT(2)
 /* flag to indicate low latency hint to guc */
-#define EXEC_QUEUE_FLAG_LOW_LATENCY		BIT(5)
+#define EXEC_QUEUE_FLAG_LOW_LATENCY		BIT(3)
 /* for migration (kernel copy, clear, bind) jobs */
-#define EXEC_QUEUE_FLAG_MIGRATE			BIT(6)
+#define EXEC_QUEUE_FLAG_MIGRATE			BIT(4)
 /* for programming COMMON_SLICE_CHICKEN3 on first submission */
-#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX	BIT(7)
+#define EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX	BIT(5)
 
 	/**
 	 * @flags: flags for this exec queue, should statically setup aside from ban
@@ -149,12 +152,8 @@ struct xe_exec_queue {
 	 */
 	unsigned long flags;
 
-	union {
-		/** @multi_gt_list: list head for VM bind engines if multi-GT */
-		struct list_head multi_gt_list;
-		/** @multi_gt_link: link for VM bind engines if multi-GT */
-		struct list_head multi_gt_link;
-	};
+	/** @ban_reason: Bitmask of ban reasons (DRM_XE_EXEC_QUEUE_BAN_REASON_*) */
+	atomic_t ban_reason;
 
 	union {
 		/** @execlist: execlist backend specific state for exec queue */
@@ -222,7 +221,8 @@ struct xe_exec_queue {
 
 #define XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT	0
 #define XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT	1
-#define XE_EXEC_QUEUE_TLB_INVAL_COUNT		(XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT  + 1)
+#define XE_EXEC_QUEUE_TLB_INVAL_COUNT	\
+	((XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT + 1) * 2)
 
 	/** @tlb_inval: TLB invalidations exec queue state */
 	struct {
@@ -343,8 +343,8 @@ struct xe_exec_queue_ops {
 	 * signalled when this function is called.
 	 */
 	void (*resume)(struct xe_exec_queue *q);
-	/** @reset_status: check exec queue reset status */
-	bool (*reset_status)(struct xe_exec_queue *q);
+	/** @reset_status: check exec queue ban status, returns ban reason bitmask */
+	u64 (*reset_status)(struct xe_exec_queue *q);
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c
index 0d0db66c6ea240..a36db39dcda8ad 100644
--- a/drivers/gpu/drm/xe/xe_execlist.c
+++ b/drivers/gpu/drm/xe/xe_execlist.c
@@ -453,10 +453,10 @@ static void execlist_exec_queue_resume(struct xe_exec_queue *q)
 	/* NIY */
 }
 
-static bool execlist_exec_queue_reset_status(struct xe_exec_queue *q)
+static u64 execlist_exec_queue_reset_status(struct xe_exec_queue *q)
 {
 	/* NIY */
-	return false;
+	return 0;
 }
 
 static const struct xe_exec_queue_ops execlist_exec_queue_ops = {
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index 8ec23862477fc5..63e8bf19360587 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -20,6 +20,7 @@
 #include "regs/xe_regs.h"
 #include "xe_assert.h"
 #include "xe_bo.h"
+#include "xe_gt.h"
 #include "xe_gt_printk.h"
 #include "xe_gt_types.h"
 #include "xe_map.h"
@@ -450,6 +451,7 @@ int xe_ggtt_init_early(struct xe_ggtt *ggtt)
 ALLOW_ERROR_INJECTION(xe_ggtt_init_early, ERRNO); /* See xe_pci_probe() */
 
 static void xe_ggtt_invalidate(struct xe_ggtt *ggtt);
+static void xe_ggtt_invalidate_engine(struct xe_ggtt *ggtt);
 
 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt)
 {
@@ -481,8 +483,11 @@ static void ggtt_node_remove(struct xe_ggtt_node *node)
 		xe_ggtt_clear(ggtt, xe_ggtt_node_addr(node), xe_ggtt_node_size(node));
 	drm_mm_remove_node(&node->base);
 	node->base.size = 0;
-	if (bound && node->invalidate_on_remove)
+	if (bound && node->invalidate_on_remove) {
 		xe_ggtt_invalidate(ggtt);
+		/* Drain engine TLBs so a recycled range can't hit a stale entry. */
+		xe_ggtt_invalidate_engine(ggtt);
+	}
 	mutex_unlock(&ggtt->lock);
 
 	ggtt_node_fini(node);
@@ -533,27 +538,36 @@ void xe_ggtt_node_remove(struct xe_ggtt_node *node, bool invalidate)
 int xe_ggtt_init(struct xe_ggtt *ggtt)
 {
 	struct xe_device *xe = tile_to_xe(ggtt->tile);
-	unsigned int flags;
 	int err;
 
 	/*
-	 * So we don't need to worry about 64K GGTT layout when dealing with
-	 * scratch entries, rather keep the scratch page in system memory on
-	 * platforms where 64K pages are needed for VRAM.
+	 * Multi-queue misses engine GGTT TLB invalidations (GuC lite-restore
+	 * skips the full context restore), so skip scratch: a stale translation
+	 * to a freed range then faults instead of silently reading scratch.
 	 */
-	flags = 0;
-	if (ggtt->flags & XE_GGTT_FLAGS_64K)
-		flags |= XE_BO_FLAG_SYSTEM;
-	else
-		flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
+	if (!xe_gt_has_multi_queue(ggtt->tile->primary_gt)) {
+		unsigned int flags = 0;
+
+		/*
+		 * So we don't need to worry about 64K GGTT layout when dealing
+		 * with scratch entries, rather keep the scratch page in system
+		 * memory on platforms where 64K pages are needed for VRAM.
+		 */
+		if (ggtt->flags & XE_GGTT_FLAGS_64K)
+			flags |= XE_BO_FLAG_SYSTEM;
+		else
+			flags |= XE_BO_FLAG_VRAM_IF_DGFX(ggtt->tile);
 
-	ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile, XE_PAGE_SIZE, flags);
-	if (IS_ERR(ggtt->scratch)) {
-		err = PTR_ERR(ggtt->scratch);
-		goto err;
-	}
+		ggtt->scratch = xe_managed_bo_create_pin_map(xe, ggtt->tile,
+							     XE_PAGE_SIZE, flags);
+		if (IS_ERR(ggtt->scratch)) {
+			err = PTR_ERR(ggtt->scratch);
+			goto err;
+		}
 
-	xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, xe_bo_size(ggtt->scratch));
+		xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0,
+			      xe_bo_size(ggtt->scratch));
+	}
 
 	xe_ggtt_initial_clear(ggtt);
 
@@ -591,6 +605,34 @@ static void xe_ggtt_invalidate(struct xe_ggtt *ggtt)
 	ggtt_invalidate_gt_tlb(ggtt->tile->media_gt);
 }
 
+static void ggtt_invalidate_gt_engine_tlb(struct xe_gt *gt)
+{
+	int err;
+
+	/*
+	 * Multi-queue GTs miss engine GGTT TLB invalidations (GuC
+	 * lite-restore skips the full context restore), so the engine
+	 * flush is needed there.
+	 */
+	if (!gt || !xe_gt_has_multi_queue(gt))
+		return;
+
+	err = xe_tlb_inval_ggtt_full(&gt->tlb_inval);
+	xe_gt_WARN(gt, err, "Failed to invalidate engine GGTT TLBs (%pe)",
+		   ERR_PTR(err));
+}
+
+/*
+ * Drain engine-side GGTT TLBs on teardown so a recycled range's next
+ * occupant can't hit a predecessor's cached translation.
+ */
+static void xe_ggtt_invalidate_engine(struct xe_ggtt *ggtt)
+{
+	/* Each GT in a tile has its own engine TLBs to drain */
+	ggtt_invalidate_gt_engine_tlb(ggtt->tile->primary_gt);
+	ggtt_invalidate_gt_engine_tlb(ggtt->tile->media_gt);
+}
+
 /**
  * xe_ggtt_shift_nodes() - Shift GGTT nodes to adjust for a change in usable address range.
  * @ggtt: the &xe_ggtt struct instance
diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c
index aab59dc647fbd1..524ac56bdcc7b0 100644
--- a/drivers/gpu/drm/xe/xe_gsc.c
+++ b/drivers/gpu/drm/xe/xe_gsc.c
@@ -478,8 +478,7 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc)
 
 	q = xe_exec_queue_create(xe, NULL,
 				 BIT(hwe->logical_instance), 1, hwe,
-				 EXEC_QUEUE_FLAG_KERNEL |
-				 EXEC_QUEUE_FLAG_PERMANENT, 0);
+				 EXEC_QUEUE_FLAG_KERNEL, 0);
 	if (IS_ERR(q)) {
 		xe_gt_err(gt, "Failed to create queue for GSC submission\n");
 		return PTR_ERR(q);
diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index dfdacc0f6de975..775c826b68b424 100644
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -48,6 +48,7 @@
 #include "xe_hw_engine_class_sysfs.h"
 #include "xe_irq.h"
 #include "xe_lmtt.h"
+#include "xe_log.h"
 #include "xe_lrc.h"
 #include "xe_map.h"
 #include "xe_migrate.h"
@@ -69,6 +70,8 @@
 #include "xe_wa.h"
 #include "xe_wopcm.h"
 
+#define GRDOM_RESET_TIMEOUT_MS	5
+
 struct xe_gt *xe_gt_alloc(struct xe_tile *tile)
 {
 	struct xe_device *xe = tile_to_xe(tile);
@@ -609,6 +612,12 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt)
 	 */
 	gt->info.gmdid = xe_mmio_read32(&gt->mmio, GMD_ID);
 
+	if (GRAPHICS_VER(gt_to_xe(gt)) >= 20 && xe_gt_is_main_type(gt))
+		gt->info.has_wmtp_disabled = !!(xe_mmio_read32(&gt->mmio, XEHP_FUSE4) &
+			CFEG_WMTP_DISABLE);
+	else
+		gt->info.has_wmtp_disabled = 0;
+
 	/*
 	 * Wa_14026539277 can't be implemented as a regular GT workaround (i.e.
 	 * as an entry in gt_was[]) for two reasons: it is actually a device
@@ -821,10 +830,13 @@ static int do_gt_reset(struct xe_gt *gt)
 	xe_gsc_wa_14015076503(gt, true);
 
 	xe_mmio_write32(&gt->mmio, GDRST, GRDOM_FULL);
-	err = xe_mmio_wait32(&gt->mmio, GDRST, GRDOM_FULL, 0, 5000, NULL, false);
+	err = xe_mmio_wait32(&gt->mmio, GDRST, GRDOM_FULL, 0,
+			     GRDOM_RESET_TIMEOUT_MS * USEC_PER_MSEC,
+			     NULL, false);
 	if (err)
-		xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n",
-			  ERR_PTR(err));
+		xe_log_err(gt, GT, err,
+			   "full graphics reset not completed in %u ms\n",
+			   GRDOM_RESET_TIMEOUT_MS);
 
 	xe_gsc_wa_14015076503(gt, false);
 
@@ -925,7 +937,7 @@ static void gt_reset_worker(struct work_struct *w)
 	if (!xe_device_uc_enabled(gt_to_xe(gt)))
 		goto err_pm_put;
 
-	xe_gt_info(gt, "reset started\n");
+	xe_log_info(gt, GT, "reset started\n");
 
 	if (xe_fault_gt_reset()) {
 		err = -ECANCELED;
@@ -964,7 +976,7 @@ static void gt_reset_worker(struct work_struct *w)
 	/* Pair with get while enqueueing the work in xe_gt_reset_async() */
 	xe_pm_runtime_put(gt_to_xe(gt));
 
-	xe_gt_info(gt, "reset done\n");
+	xe_log_info(gt, GT, "reset done\n");
 
 	return;
 
@@ -973,7 +985,7 @@ static void gt_reset_worker(struct work_struct *w)
 	XE_WARN_ON(xe_uc_start(&gt->uc));
 
 err_fail:
-	xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
+	xe_log_err_fatal(gt, GT, err, "reset failed\n");
 	xe_device_declare_wedged(gt_to_xe(gt));
 err_pm_put:
 	xe_pm_runtime_put(gt_to_xe(gt));
diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h
index 65a4655b0994df..92d934d50244bd 100644
--- a/drivers/gpu/drm/xe/xe_gt.h
+++ b/drivers/gpu/drm/xe/xe_gt.h
@@ -160,4 +160,17 @@ static inline bool xe_gt_supports_multi_queue(const struct xe_gt *gt,
 	return gt->info.multi_queue_engine_class_mask & BIT(class);
 }
 
+/**
+ * xe_gt_has_multi_queue() - Check if gt supports multi-queue on
+ * any engine class.
+ *
+ * @gt: the GT object (may be NULL)
+ *
+ * Return: true if any engine class on @gt supports multi queue, else false
+ */
+static inline bool xe_gt_has_multi_queue(const struct xe_gt *gt)
+{
+	return gt && gt->info.multi_queue_engine_class_mask;
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
index baee1f4a6b013e..224a15675203ad 100644
--- a/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
+++ b/drivers/gpu/drm/xe/xe_gt_ccs_mode.c
@@ -70,12 +70,6 @@ static void __xe_gt_apply_ccs_mode(struct xe_gt *gt, u32 num_engines)
 		}
 	}
 
-	/*
-	 * Mask bits need to be set for the register. Though only Xe2+
-	 * platforms require setting of mask bits, it won't harm for older
-	 * platforms as these bits are unused there.
-	 */
-	mode |= CCS_MODE_CSLICE_0_3_MASK << 16;
 	xe_mmio_write32(&gt->mmio, CCS_MODE, mode);
 
 	xe_gt_dbg(gt, "CCS_MODE=%x config:%08x, num_engines:%d, num_slices:%d\n",
diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c
index c38bcacb27e4a3..405cc076d8431f 100644
--- a/drivers/gpu/drm/xe/xe_gt_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c
@@ -6,22 +6,30 @@
 #include "xe_gt_debugfs.h"
 
 #include <linux/debugfs.h>
+#include <linux/panic.h>
+#include <linux/string.h>
 
 #include <drm/drm_debugfs.h>
 #include <drm/drm_managed.h>
+#include <linux/math.h>
 
+#include "regs/xe_engine_regs.h"
+#include "regs/xe_gt_regs.h"
 #include "xe_device.h"
 #include "xe_force_wake.h"
 #include "xe_gt.h"
 #include "xe_gt_mcr.h"
 #include "xe_gt_idle.h"
+#include "xe_gt_printk.h"
 #include "xe_gt_sriov_pf_debugfs.h"
 #include "xe_gt_sriov_vf_debugfs.h"
 #include "xe_gt_stats.h"
 #include "xe_gt_topology.h"
 #include "xe_guc_hwconfig.h"
+#include "xe_guc_submit.h"
 #include "xe_hw_engine.h"
 #include "xe_lrc.h"
+#include "xe_mmio.h"
 #include "xe_mocs.h"
 #include "xe_pat.h"
 #include "xe_pm.h"
@@ -111,6 +119,38 @@ int xe_gt_debugfs_show_with_rpm(struct seq_file *m, void *data)
 
 static int hw_engines(struct xe_gt *gt, struct drm_printer *p)
 {
+	static const struct {
+		struct xe_reg reg;
+		bool has_fw;
+	} msg_idle_reg[XE_NUM_HW_ENGINES] = {
+		[XE_HW_ENGINE_RCS0] = { MSG_IDLE_CS, true },
+		[XE_HW_ENGINE_BCS0] = { MSG_IDLE_BCS0, true },
+		[XE_HW_ENGINE_BCS1] = { MSG_IDLE_BCS1, false },
+		[XE_HW_ENGINE_BCS2] = { MSG_IDLE_BCS2, false },
+		[XE_HW_ENGINE_BCS3] = { MSG_IDLE_BCS3, false },
+		[XE_HW_ENGINE_BCS4] = { MSG_IDLE_BCS4, false },
+		[XE_HW_ENGINE_BCS5] = { MSG_IDLE_BCS5, false },
+		[XE_HW_ENGINE_BCS6] = { MSG_IDLE_BCS6, false },
+		[XE_HW_ENGINE_BCS7] = { MSG_IDLE_BCS7, false },
+		[XE_HW_ENGINE_BCS8] = { MSG_IDLE_BCS8, false },
+		[XE_HW_ENGINE_VCS0] = { MSG_IDLE_VCS0, true },
+		[XE_HW_ENGINE_VCS1] = { MSG_IDLE_VCS1, true },
+		[XE_HW_ENGINE_VCS2] = { MSG_IDLE_VCS2, true },
+		[XE_HW_ENGINE_VCS3] = { MSG_IDLE_VCS3, true },
+		[XE_HW_ENGINE_VCS4] = { MSG_IDLE_VCS4, true },
+		[XE_HW_ENGINE_VCS5] = { MSG_IDLE_VCS5, true },
+		[XE_HW_ENGINE_VCS6] = { MSG_IDLE_VCS6, true },
+		[XE_HW_ENGINE_VCS7] = { MSG_IDLE_VCS7, true },
+		[XE_HW_ENGINE_VECS0] = { MSG_IDLE_VECS0, true },
+		[XE_HW_ENGINE_VECS1] = { MSG_IDLE_VECS1, true },
+		[XE_HW_ENGINE_VECS2] = { MSG_IDLE_VECS2, true },
+		[XE_HW_ENGINE_VECS3] = { MSG_IDLE_VECS3, true },
+		[XE_HW_ENGINE_CCS0] = { MSG_IDLE_CS, true },
+		[XE_HW_ENGINE_CCS1] = { MSG_IDLE_CS, true },
+		[XE_HW_ENGINE_CCS2] = { MSG_IDLE_CS, true },
+		[XE_HW_ENGINE_CCS3] = { MSG_IDLE_CS, true },
+		[XE_HW_ENGINE_GSCCS0] = { MSG_IDLE_GSCCS0, false },
+	};
 	struct xe_hw_engine *hwe;
 	enum xe_hw_engine_id id;
 
@@ -118,9 +158,69 @@ static int hw_engines(struct xe_gt *gt, struct drm_printer *p)
 	if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL))
 		return -ETIMEDOUT;
 
-	for_each_hw_engine(hwe, gt, id)
+	for_each_hw_engine(hwe, gt, id) {
+		struct xe_reg reg = msg_idle_reg[id].reg;
+		u32 val;
+
 		xe_hw_engine_print(hwe, p);
 
+		val = xe_mmio_read32(&gt->mmio, reg);
+
+		if (msg_idle_reg[id].has_fw)
+			drm_printf(p, "\tpipe %s, C6 %s, fw_req 0x%02x\n",
+				   val & MSG_IDLE_INDICATION ? "idle" : "busy",
+				   val & MSG_IDLE_C6_ALLOWED ? "allowed" : "not allowed",
+				   REG_FIELD_GET(MSG_IDLE_FW_REQ, val));
+		else
+			drm_printf(p, "\tpipe %s, C6 %s\n",
+				   val & MSG_IDLE_INDICATION ? "idle" : "busy",
+				   val & MSG_IDLE_C6_ALLOWED ? "allowed" : "not allowed");
+
+		drm_puts(p, "\n");
+	}
+
+	return 0;
+}
+
+static int multi_queue_active_lrca(struct xe_gt *gt, struct drm_printer *p)
+{
+	struct xe_guc *guc = &gt->uc.guc;
+	struct xe_hw_engine *hwe;
+	enum xe_hw_engine_id id;
+
+	for_each_hw_engine(hwe, gt, id) {
+		u32 cur_lrca, active_id, lrca;
+		unsigned int fw_ref;
+
+		if (!xe_gt_supports_multi_queue(gt, hwe->class))
+			continue;
+
+		/*
+		 * Forcewake is dropped before xe_guc_submit_active_multi_queue_lrca()
+		 * below, which takes guc->submission_state.lock, to avoid holding a
+		 * GT forcewake ref across a mutex acquired elsewhere in the opposite
+		 * order.
+		 */
+		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
+		if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
+			drm_printf(p, "%s\tforcewake failed, skipping\n", hwe->name);
+			xe_force_wake_put(gt_to_fw(gt), fw_ref);
+			continue;
+		}
+
+		cur_lrca = xe_mmio_read32(&gt->mmio,
+					  RING_CURRENT_LRCA(hwe->mmio_base));
+		active_id = xe_lrc_get_multi_queue_active_queue_id(hwe);
+
+		xe_force_wake_put(gt_to_fw(gt), fw_ref);
+
+		lrca = xe_guc_submit_active_multi_queue_lrca(guc, hwe, cur_lrca,
+							     active_id);
+
+		drm_printf(p, "%s\tactive_queue_id %u\tcurrent_lrca 0x%08x\tactive_lrca 0x%08x\n",
+			   hwe->name, active_id, cur_lrca, lrca);
+	}
+
 	return 0;
 }
 
@@ -248,6 +348,11 @@ static const struct drm_info_list pf_only_debugfs_list[] = {
 	{ "steering", .show = xe_gt_debugfs_show_with_rpm, .data = steering },
 };
 
+static const struct drm_info_list multi_queue_debugfs_list[] = {
+	{ "multi_queue_active_lrca",
+		.show = xe_gt_debugfs_show_with_rpm, .data = multi_queue_active_lrca },
+};
+
 static ssize_t write_to_gt_call(const char __user *userbuf, size_t count, loff_t *ppos,
 				void (*call)(struct xe_gt *), struct xe_gt *gt)
 {
@@ -336,6 +441,139 @@ static int force_reset_sync_show(struct seq_file *s, void *unused)
 }
 DEFINE_SHOW_STORE_ATTRIBUTE(force_reset_sync);
 
+#define U1_15_ONE		0x8000
+#define U1_15_INT_BITS		GENMASK(15, 15)
+#define U1_15_FRACTION_BITS	GENMASK(14, 0)
+
+static void u1_15_decode(u16 num, u16 *i, u32 *frac)
+{
+	/*
+	 * In U1.15 format, uppermost bit is integer value and the
+	 * rest 15 are the fraction.
+	 */
+
+	*i = FIELD_GET(U1_15_INT_BITS, num);
+	*frac = FIELD_GET(U1_15_FRACTION_BITS, num);
+}
+
+static void u1_15_decode_decimal(u16 value, u16 *i, u32 *frac, int digits)
+{
+	u1_15_decode(value, i, frac);
+	*frac = (*frac * int_pow(10, digits)) / (FIELD_MAX(U1_15_FRACTION_BITS) + 1);
+}
+
+static int gt_ia_bias_show(struct seq_file *s, void *unused)
+{
+	struct xe_gt *gt = s->private;
+	struct xe_device *xe = gt_to_xe(gt);
+	u32 val;
+	u32 ia_frac, gt_frac;
+	u16 ia_raw, gt_raw;
+	u16 ia_int, gt_int;
+
+	guard(xe_pm_runtime)(xe);
+	val = xe_mmio_read32(&gt->mmio, GT_IA_PERF_BIAS_REG);
+
+	ia_raw = REG_FIELD_GET(IA_BIAS, val);
+	gt_raw = REG_FIELD_GET(GT_BIAS, val);
+
+	u1_15_decode_decimal(ia_raw, &ia_int, &ia_frac, 4);
+	u1_15_decode_decimal(gt_raw, &gt_int, &gt_frac, 4);
+
+	seq_printf(s, "0x%x (GT: %u.%04u, IA: %u.%04u)\n",
+		   val, gt_int, gt_frac, ia_int, ia_frac);
+
+	return 0;
+}
+
+static ssize_t gt_ia_bias_write(struct file *file,
+				const char __user *userbuf,
+				size_t count, loff_t *ppos)
+{
+	struct seq_file *s = file->private_data;
+	struct xe_gt *gt = s->private;
+	struct xe_device *xe = gt_to_xe(gt);
+	u32 val;
+	int ret;
+
+	ret = kstrtou32_from_user(userbuf, count, 0, &val);
+	if (ret)
+		return ret;
+
+	if (REG_FIELD_GET(IA_BIAS, val) > U1_15_ONE ||
+	    REG_FIELD_GET(GT_BIAS, val) > U1_15_ONE)
+		return -EINVAL;
+
+	if (REG_FIELD_GET(IA_BIAS, val) < IA_BIAS_DEFAULT ||
+	    REG_FIELD_GET(GT_BIAS, val) < GT_BIAS_DEFAULT)
+		return -EINVAL;
+
+	guard(xe_pm_runtime)(xe);
+	xe_mmio_write32(&gt->mmio, GT_IA_PERF_BIAS_REG, val);
+
+	return count;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(gt_ia_bias);
+
+static const char * const gpgpu_preemption_level_names[] = {
+	[XE_GPGPU_PREEMPT_DEFAULT]      = "default",
+	[XE_GPGPU_PREEMPT_MID_THREAD]   = "mid-thread",
+	[XE_GPGPU_PREEMPT_THREAD_GROUP] = "thread-group",
+	[XE_GPGPU_PREEMPT_COMMAND]      = "command",
+};
+
+static int gpgpu_preemption_level_show(struct seq_file *m, void *unused)
+{
+	struct xe_gt *gt = m->private;
+
+	seq_printf(m, "%s\n", gpgpu_preemption_level_names[gt->gpgpu_preemption_level]);
+
+	return 0;
+}
+
+static ssize_t gpgpu_preemption_level_write(struct file *file,
+					    const char __user *ubuf,
+					    size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct xe_gt *gt = m->private;
+	enum xe_gpgpu_preempt_level new_level;
+	char buf[16];
+	ssize_t copied;
+	int idx;
+
+	if (*offp)
+		return -EINVAL;
+
+	copied = simple_write_to_buffer(buf, sizeof(buf) - 1, offp, ubuf, len);
+	if (copied < 0)
+		return copied;
+
+	buf[copied] = '\0';
+	idx = sysfs_match_string(gpgpu_preemption_level_names, strim(buf));
+	if (idx < 0)
+		return idx;
+
+	new_level = (enum xe_gpgpu_preempt_level)idx;
+
+	if (new_level == XE_GPGPU_PREEMPT_MID_THREAD && gt->info.has_wmtp_disabled) {
+		xe_gt_warn(gt, "MTP fused off in hardware, cannot select mid-thread\n");
+		return -EINVAL;
+	}
+
+	if (new_level != XE_GPGPU_PREEMPT_DEFAULT) {
+		add_taint(TAINT_USER, LOCKDEP_STILL_OK);
+		xe_gt_notice(gt,
+			     "GPGPU preemption overridden to '%s' (applies to new LRCs only)\n",
+			     gpgpu_preemption_level_names[new_level]);
+	}
+
+	gt->gpgpu_preemption_level = new_level;
+
+	return copied;
+}
+DEFINE_SHOW_STORE_ATTRIBUTE(gpgpu_preemption_level);
+
 void xe_gt_debugfs_register(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);
@@ -369,6 +607,10 @@ void xe_gt_debugfs_register(struct xe_gt *gt)
 	debugfs_create_file("force_reset", 0600, root, gt, &force_reset_fops);
 	debugfs_create_file("force_reset_sync", 0600, root, gt, &force_reset_sync_fops);
 
+	if (GRAPHICS_VER(xe) >= 20 && (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK))
+		debugfs_create_file("gpgpu_preemption_level", 0600, root,
+				    gt, &gpgpu_preemption_level_fops);
+
 	drm_debugfs_create_files(vf_safe_debugfs_list,
 				 ARRAY_SIZE(vf_safe_debugfs_list),
 				 root, minor);
@@ -378,6 +620,14 @@ void xe_gt_debugfs_register(struct xe_gt *gt)
 					 ARRAY_SIZE(pf_only_debugfs_list),
 					 root, minor);
 
+	if (!IS_SRIOV_VF(xe) && xe_gt_has_multi_queue(gt))
+		drm_debugfs_create_files(multi_queue_debugfs_list,
+					 ARRAY_SIZE(multi_queue_debugfs_list),
+					 root, minor);
+
+	if (xe_gt_is_main_type(gt) && !IS_DGFX(xe) && !IS_SRIOV_VF(xe))
+		debugfs_create_file("gt_ia_bias", 0600, root, gt, &gt_ia_bias_fops);
+
 	xe_uc_debugfs_register(&gt->uc, root);
 
 	if (IS_SRIOV_PF(xe))
diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c
index 7dc9873aef545d..2eefd62f8d5927 100644
--- a/drivers/gpu/drm/xe/xe_gt_idle.c
+++ b/drivers/gpu/drm/xe/xe_gt_idle.c
@@ -8,10 +8,12 @@
 #include <drm/drm_managed.h>
 
 #include <generated/xe_wa_oob.h>
+#include <linux/iopoll.h>
 #include "xe_force_wake.h"
 #include "xe_device.h"
 #include "xe_gt.h"
 #include "xe_gt_idle.h"
+#include "xe_gt_printk.h"
 #include "xe_gt_sysfs.h"
 #include "xe_guc_pc.h"
 #include "regs/xe_gt_regs.h"
@@ -208,26 +210,34 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
 	/*
 	 * Media Slices
 	 *
-	 * Slice 0: VCS0, VCS1, VECS0
-	 * Slice 1: VCS2, VCS3, VECS1
-	 * Slice 2: VCS4, VCS5, VECS2
-	 * Slice 3: VCS6, VCS7, VECS3
+	 * Prior to MEDIA_VER 35		Post MEDIA_VER 35
+	 *
+	 * Slice 0: VCS0, VCS1, VECS0, VECS2	Slice 0: VCS0, VCS1, VECS0, VECS1
+	 * Slice 1: VCS2, VCS3, VECS1, VECS3	Slice 1: VCS2, VCS3, VECS2, VECS3
 	 */
-	static const struct {
+	struct media_slice {
 		u64 engines;
 		u32 status_bit;
-	} media_slices[] = {
+	};
+
+	static const struct media_slice media_slices_pre_35[] = {
 		{(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) |
-		  BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS},
+		  BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VECS2)),
+		 MEDIA_SLICE0_AWAKE_STATUS},
 
 		{(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) |
-		   BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS},
+		   BIT(XE_HW_ENGINE_VECS1) | BIT(XE_HW_ENGINE_VECS3)),
+		 MEDIA_SLICE1_AWAKE_STATUS},
+	};
 
-		{(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) |
-		   BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS},
+	static const struct media_slice media_slices_post_35[] = {
+		{(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) |
+		  BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VECS1)),
+		 MEDIA_SLICE0_AWAKE_STATUS},
 
-		{(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) |
-		   BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS},
+		{(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) |
+		   BIT(XE_HW_ENGINE_VECS2) | BIT(XE_HW_ENGINE_VECS3)),
+		 MEDIA_SLICE1_AWAKE_STATUS},
 	};
 
 	if (xe->info.platform == XE_PVC) {
@@ -262,10 +272,21 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
 
 	/* Print media CPG status only if media is present */
 	if (vcs_mask || vecs_mask) {
+		const struct media_slice *media_slices;
+		int num_media_slices;
+
+		if (MEDIA_VERx100(gt_to_xe(gt)) < 3500) {
+			media_slices = media_slices_pre_35;
+			num_media_slices = ARRAY_SIZE(media_slices_pre_35);
+		} else {
+			media_slices = media_slices_post_35;
+			num_media_slices = ARRAY_SIZE(media_slices_post_35);
+		}
+
 		drm_printf(p, "Media Power Gating Enabled: %s\n",
 			   str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE));
 
-		for (n = 0; n < ARRAY_SIZE(media_slices); n++)
+		for (n = 0; n < num_media_slices; n++)
 			if (gt->info.engine_mask & media_slices[n].engines)
 				drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n,
 					   str_up_down(pg_status & media_slices[n].status_bit));
@@ -432,3 +453,40 @@ int xe_gt_idle_disable_c6(struct xe_gt *gt)
 
 	return 0;
 }
+
+static int wait_for_gt_c6_state(struct xe_gt *gt, u32 timeout_ms)
+{
+	struct xe_guc_pc *pc = &gt->uc.guc.pc;
+	enum xe_gt_idle_state state;
+
+	return poll_timeout_us(state = gt->gtidle.idle_status(pc),
+			       state == GT_IDLE_C6,
+			       20,
+			       timeout_ms * USEC_PER_MSEC,
+			       false);
+}
+
+/**
+ * xe_gt_idle_wait_for_c6 - Poll for GT C6
+ * @gt: GT object
+ * @timeout_ms: wait time in ms
+ *
+ * This function waits for GT to enter C6.
+ *
+ * Return: 0 on success, -EAGAIN otherwise
+ */
+int xe_gt_idle_wait_for_c6(struct xe_gt *gt, u32 timeout_ms)
+{
+	if (IS_SRIOV_VF(gt_to_xe(gt)))
+		return 0;
+
+	if (gt_to_xe(gt)->info.platform == XE_PVC)
+		return 0;
+
+	if (wait_for_gt_c6_state(gt, timeout_ms)) {
+		xe_gt_dbg(gt, "GT is not in C6\n");
+		return -EAGAIN;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/xe/xe_gt_idle.h b/drivers/gpu/drm/xe/xe_gt_idle.h
index 9c34a155e10249..2c8e9adf4a6811 100644
--- a/drivers/gpu/drm/xe/xe_gt_idle.h
+++ b/drivers/gpu/drm/xe/xe_gt_idle.h
@@ -18,5 +18,6 @@ void xe_gt_idle_enable_pg(struct xe_gt *gt);
 void xe_gt_idle_disable_pg(struct xe_gt *gt);
 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p);
 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle);
+int xe_gt_idle_wait_for_c6(struct xe_gt *gt, u32 timeout_ms);
 
 #endif /* _XE_GT_IDLE_H_ */
diff --git a/drivers/gpu/drm/xe/xe_gt_printk.h b/drivers/gpu/drm/xe/xe_gt_printk.h
index 1313d32862dbdd..1598098d29873c 100644
--- a/drivers/gpu/drm/xe/xe_gt_printk.h
+++ b/drivers/gpu/drm/xe/xe_gt_printk.h
@@ -35,6 +35,9 @@
 #define xe_gt_dbg(_gt, _fmt, ...) \
 	xe_gt_printk((_gt), dbg, _fmt, ##__VA_ARGS__)
 
+#define xe_gt_dbg_ratelimited(_gt, _fmt, ...) \
+	xe_gt_printk((_gt), dbg_ratelimited, _fmt, ##__VA_ARGS__)
+
 #define xe_gt_WARN_type(_gt, _type, _condition, _fmt, ...) \
 	xe_tile_WARN##_type((_gt)->tile, _condition, _fmt, ## __VA_ARGS__)
 
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
index 2c9b85b84b1be7..6947aeed458ed7 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
@@ -668,7 +668,7 @@ static int pf_config_bulk_set_u64_done(struct xe_gt *gt, unsigned int first, uns
 }
 
 /**
- * xe_gt_sriov_pf_config_bulk_set_ggtt - Provision many VFs with GGTT.
+ * xe_gt_sriov_pf_config_bulk_set_ggtt_locked() - Provision many VFs with GGTT.
  * @gt: the &xe_gt (can't be media)
  * @vfid: starting VF identifier (can't be 0)
  * @num_vfs: number of VFs to provision
@@ -678,31 +678,49 @@ static int pf_config_bulk_set_u64_done(struct xe_gt *gt, unsigned int first, uns
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_config_bulk_set_ggtt(struct xe_gt *gt, unsigned int vfid,
-					unsigned int num_vfs, u64 size)
+int xe_gt_sriov_pf_config_bulk_set_ggtt_locked(struct xe_gt *gt, unsigned int vfid,
+					       unsigned int num_vfs, u64 size)
 {
 	unsigned int n;
 	int err = 0;
 
 	xe_gt_assert(gt, vfid);
 	xe_gt_assert(gt, xe_gt_is_main_type(gt));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
 	if (!num_vfs)
 		return 0;
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
 	for (n = vfid; n < vfid + num_vfs; n++) {
 		err = pf_provision_vf_ggtt(gt, n, size);
 		if (err)
 			break;
 	}
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
 
 	return pf_config_bulk_set_u64_done(gt, vfid, num_vfs, size,
-					   xe_gt_sriov_pf_config_get_ggtt,
+					   pf_get_vf_config_ggtt,
 					   "GGTT", n, err);
 }
 
+/**
+ * xe_gt_sriov_pf_config_bulk_set_ggtt() - Provision many VFs with GGTT.
+ * @gt: the &xe_gt (can't be media)
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision
+ * @size: requested GGTT size
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_ggtt(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs, u64 size)
+{
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_config_bulk_set_ggtt_locked(gt, vfid, num_vfs, size);
+}
+
 /* Return: size of the largest continuous GGTT region */
 static u64 pf_get_max_ggtt(struct xe_gt *gt)
 {
@@ -775,10 +793,9 @@ int xe_gt_sriov_pf_config_set_fair_ggtt(struct xe_gt *gt, unsigned int vfid,
 	xe_gt_assert(gt, num_vfs);
 	xe_gt_assert(gt, xe_gt_is_main_type(gt));
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
-	fair = pf_estimate_fair_ggtt(gt, num_vfs);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
 
+	fair = pf_estimate_fair_ggtt(gt, num_vfs);
 	if (!fair)
 		return -ENOSPC;
 
@@ -787,7 +804,7 @@ int xe_gt_sriov_pf_config_set_fair_ggtt(struct xe_gt *gt, unsigned int vfid,
 		xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %llu vs %llu)\n",
 				 "GGTT", fair, profile);
 
-	return xe_gt_sriov_pf_config_bulk_set_ggtt(gt, vfid, num_vfs, fair);
+	return xe_gt_sriov_pf_config_bulk_set_ggtt_locked(gt, vfid, num_vfs, fair);
 }
 
 /**
@@ -1099,7 +1116,7 @@ static int pf_config_bulk_set_u32_done(struct xe_gt *gt, unsigned int first, uns
 }
 
 /**
- * xe_gt_sriov_pf_config_bulk_set_ctxs - Provision many VFs with GuC context IDs.
+ * xe_gt_sriov_pf_config_bulk_set_ctxs_locked() - Provision many VFs with GuC context IDs.
  * @gt: the &xe_gt
  * @vfid: starting VF identifier
  * @num_vfs: number of VFs to provision
@@ -1109,30 +1126,48 @@ static int pf_config_bulk_set_u32_done(struct xe_gt *gt, unsigned int first, uns
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_config_bulk_set_ctxs(struct xe_gt *gt, unsigned int vfid,
-					unsigned int num_vfs, u32 num_ctxs)
+int xe_gt_sriov_pf_config_bulk_set_ctxs_locked(struct xe_gt *gt, unsigned int vfid,
+					       unsigned int num_vfs, u32 num_ctxs)
 {
 	unsigned int n;
 	int err = 0;
 
 	xe_gt_assert(gt, vfid);
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
 	if (!num_vfs)
 		return 0;
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
 	for (n = vfid; n < vfid + num_vfs; n++) {
 		err = pf_provision_vf_ctxs(gt, n, num_ctxs);
 		if (err)
 			break;
 	}
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
 
 	return pf_config_bulk_set_u32_done(gt, vfid, num_vfs, num_ctxs,
-					   xe_gt_sriov_pf_config_get_ctxs,
+					   pf_get_vf_config_ctxs,
 					   "GuC context IDs", no_unit, n, err);
 }
 
+/**
+ * xe_gt_sriov_pf_config_bulk_set_ctxs() - Provision many VFs with GuC context IDs.
+ * @gt: the &xe_gt
+ * @vfid: starting VF identifier
+ * @num_vfs: number of VFs to provision
+ * @num_ctxs: requested number of GuC contexts IDs (0 to release)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_ctxs(struct xe_gt *gt, unsigned int vfid,
+					unsigned int num_vfs, u32 num_ctxs)
+{
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_config_bulk_set_ctxs_locked(gt, vfid, num_vfs, num_ctxs);
+}
+
 static u32 pf_profile_fair_ctxs(struct xe_gt *gt, unsigned int num_vfs)
 {
 	bool admin_only_pf = xe_sriov_pf_admin_only(gt_to_xe(gt));
@@ -1181,10 +1216,9 @@ int xe_gt_sriov_pf_config_set_fair_ctxs(struct xe_gt *gt, unsigned int vfid,
 	xe_gt_assert(gt, vfid);
 	xe_gt_assert(gt, num_vfs);
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
-	fair = pf_estimate_fair_ctxs(gt, num_vfs);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
 
+	fair = pf_estimate_fair_ctxs(gt, num_vfs);
 	if (!fair)
 		return -ENOSPC;
 
@@ -1193,7 +1227,7 @@ int xe_gt_sriov_pf_config_set_fair_ctxs(struct xe_gt *gt, unsigned int vfid,
 		xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %u vs %u)\n",
 				 "GuC context IDs", fair, profile);
 
-	return xe_gt_sriov_pf_config_bulk_set_ctxs(gt, vfid, num_vfs, fair);
+	return xe_gt_sriov_pf_config_bulk_set_ctxs_locked(gt, vfid, num_vfs, fair);
 }
 
 static u32 pf_get_min_spare_dbs(struct xe_gt *gt)
@@ -1363,7 +1397,7 @@ int xe_gt_sriov_pf_config_set_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_d
 }
 
 /**
- * xe_gt_sriov_pf_config_bulk_set_dbs - Provision many VFs with GuC context IDs.
+ * xe_gt_sriov_pf_config_bulk_set_dbs_locked() - Provision many VFs with GuC doorbells.
  * @gt: the &xe_gt
  * @vfid: starting VF identifier (can't be 0)
  * @num_vfs: number of VFs to provision
@@ -1373,30 +1407,48 @@ int xe_gt_sriov_pf_config_set_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_d
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_config_bulk_set_dbs(struct xe_gt *gt, unsigned int vfid,
-				       unsigned int num_vfs, u32 num_dbs)
+int xe_gt_sriov_pf_config_bulk_set_dbs_locked(struct xe_gt *gt, unsigned int vfid,
+					      unsigned int num_vfs, u32 num_dbs)
 {
 	unsigned int n;
 	int err = 0;
 
 	xe_gt_assert(gt, vfid);
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
 	if (!num_vfs)
 		return 0;
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
 	for (n = vfid; n < vfid + num_vfs; n++) {
 		err = pf_provision_vf_dbs(gt, n, num_dbs);
 		if (err)
 			break;
 	}
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
 
 	return pf_config_bulk_set_u32_done(gt, vfid, num_vfs, num_dbs,
-					   xe_gt_sriov_pf_config_get_dbs,
+					   pf_get_vf_config_dbs,
 					   "GuC doorbell IDs", no_unit, n, err);
 }
 
+/**
+ * xe_gt_sriov_pf_config_bulk_set_dbs() - Provision many VFs with GuC doorbells.
+ * @gt: the &xe_gt
+ * @vfid: starting VF identifier (can't be 0)
+ * @num_vfs: number of VFs to provision
+ * @num_dbs: requested number of GuC doorbell IDs (0 to release)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_dbs(struct xe_gt *gt, unsigned int vfid,
+				       unsigned int num_vfs, u32 num_dbs)
+{
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_config_bulk_set_dbs_locked(gt, vfid, num_vfs, num_dbs);
+}
+
 static u32 pf_profile_fair_dbs(struct xe_gt *gt, unsigned int num_vfs)
 {
 	bool admin_only_pf = xe_sriov_pf_admin_only(gt_to_xe(gt));
@@ -1446,10 +1498,9 @@ int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid,
 	xe_gt_assert(gt, vfid);
 	xe_gt_assert(gt, num_vfs);
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
-	fair = pf_estimate_fair_dbs(gt, num_vfs);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
 
+	fair = pf_estimate_fair_dbs(gt, num_vfs);
 	if (!fair)
 		return -ENOSPC;
 
@@ -1458,7 +1509,7 @@ int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid,
 		xe_gt_sriov_info(gt, "Using non-profile provisioning (%s %u vs %u)\n",
 				 "GuC doorbell IDs", fair, profile);
 
-	return xe_gt_sriov_pf_config_bulk_set_dbs(gt, vfid, num_vfs, fair);
+	return xe_gt_sriov_pf_config_bulk_set_dbs_locked(gt, vfid, num_vfs, fair);
 }
 
 static u64 pf_get_lmem_alignment(struct xe_gt *gt)
@@ -2572,28 +2623,46 @@ static u32 pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid)
 }
 
 /**
- * xe_gt_sriov_pf_config_set_sched_priority() - Configure scheduling priority.
+ * xe_gt_sriov_pf_config_set_sched_priority_locked() - Configure scheduling priority.
  * @gt: the &xe_gt
  * @vfid: the VF identifier
  * @priority: requested scheduling priority
  *
- * This function can only be called on PF.
+ * This function can only be called on PF with the master mutex hold.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority)
+int xe_gt_sriov_pf_config_set_sched_priority_locked(struct xe_gt *gt, unsigned int vfid,
+						    u32 priority)
 {
 	int err;
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
 	err = pf_provision_sched_priority(gt, vfid, priority);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
 
 	return pf_config_set_u32_done(gt, vfid, priority,
-				      xe_gt_sriov_pf_config_get_sched_priority(gt, vfid),
+				      pf_get_sched_priority(gt, vfid),
 				      "scheduling priority", sched_priority_unit, err);
 }
 
+/**
+ * xe_gt_sriov_pf_config_set_sched_priority() - Configure scheduling priority.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier
+ * @priority: requested scheduling priority
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority)
+{
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_config_set_sched_priority_locked(gt, vfid, priority);
+}
+
 /**
  * xe_gt_sriov_pf_config_get_sched_priority - Get VF's scheduling priority.
  * @gt: the &xe_gt
@@ -2715,15 +2784,16 @@ static bool pf_needs_provision_sched(struct xe_gt *gt, unsigned int num_vfs)
 #define XE_ADMIN_PF_SCHED_PRIORITY	GUC_SCHED_PRIORITY_HIGH
 
 /**
- * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling.
+ * xe_gt_sriov_pf_config_set_fair_sched_locked() - Provision PF and VFs with fair scheduling.
  * @gt: the &xe_gt
  * @num_vfs: number of VFs to provision (can't be 0)
  *
+ * The caller must hold the master PF mutex.
  * This function can only be called on PF.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs)
+int xe_gt_sriov_pf_config_set_fair_sched_locked(struct xe_gt *gt, unsigned int num_vfs)
 {
 	int result = 0;
 	int err;
@@ -2732,7 +2802,7 @@ int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs)
 	xe_gt_assert(gt, XE_FAIR_EXEC_QUANTUM_MS);
 	xe_gt_assert(gt, XE_FAIR_PREEMPT_TIMEOUT_US);
 
-	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
 	if (!pf_needs_provision_sched(gt, num_vfs))
 		return 0;
@@ -2753,6 +2823,22 @@ int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs)
 	return result;
 }
 
+/**
+ * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling.
+ * @gt: the &xe_gt
+ * @num_vfs: number of VFs to provision (can't be 0)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs)
+{
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_config_set_fair_sched_locked(gt, num_vfs);
+}
+
 static int pf_provision_threshold(struct xe_gt *gt, unsigned int vfid,
 				  enum xe_guc_klv_threshold_index index, u32 value)
 {
@@ -2863,26 +2949,26 @@ static void pf_release_vf_config(struct xe_gt *gt, unsigned int vfid)
 }
 
 /**
- * xe_gt_sriov_pf_config_release - Release and reset VF configuration.
+ * xe_gt_sriov_pf_config_release_locked() - Release and reset VF configuration.
  * @gt: the &xe_gt
  * @vfid: the VF identifier (can't be PF)
  * @force: force configuration release
  *
+ * Note: The caller must hold the master PF mutex.
  * This function can only be called on PF.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force)
+int xe_gt_sriov_pf_config_release_locked(struct xe_gt *gt, unsigned int vfid, bool force)
 {
 	int err;
 
 	xe_gt_assert(gt, vfid);
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
 	err = pf_send_vf_cfg_reset(gt, vfid);
 	if (!err || force)
 		pf_release_vf_config(gt, vfid);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
 
 	if (unlikely(err)) {
 		xe_gt_sriov_notice(gt, "VF%u unprovisioning failed with error (%pe)%s\n",
@@ -2893,6 +2979,23 @@ int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool forc
 	return force ? 0 : err;
 }
 
+/**
+ * xe_gt_sriov_pf_config_release() - Release and reset VF configuration.
+ * @gt: the &xe_gt
+ * @vfid: the VF identifier (can't be PF)
+ * @force: force configuration release
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force)
+{
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_config_release_locked(gt, vfid, force);
+}
+
 static void pf_sanitize_ggtt(struct xe_ggtt_node *ggtt_region, unsigned int vfid)
 {
 	if (ggtt_region)
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
index 2ec62c12ad5cf0..3e10697d217417 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
@@ -18,18 +18,24 @@ int xe_gt_sriov_pf_config_set_fair_ggtt(struct xe_gt *gt,
 					unsigned int vfid, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_bulk_set_ggtt(struct xe_gt *gt,
 					unsigned int vfid, unsigned int num_vfs, u64 size);
+int xe_gt_sriov_pf_config_bulk_set_ggtt_locked(struct xe_gt *gt,
+					       unsigned int vfid, unsigned int num_vfs, u64 size);
 
 u32 xe_gt_sriov_pf_config_get_ctxs(struct xe_gt *gt, unsigned int vfid);
 int xe_gt_sriov_pf_config_set_ctxs(struct xe_gt *gt, unsigned int vfid, u32 num_ctxs);
 int xe_gt_sriov_pf_config_set_fair_ctxs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_bulk_set_ctxs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs,
 					u32 num_ctxs);
+int xe_gt_sriov_pf_config_bulk_set_ctxs_locked(struct xe_gt *gt, unsigned int vfid,
+					       unsigned int num_vfs, u32 num_ctxs);
 
 u32 xe_gt_sriov_pf_config_get_dbs(struct xe_gt *gt, unsigned int vfid);
 int xe_gt_sriov_pf_config_set_dbs(struct xe_gt *gt, unsigned int vfid, u32 num_dbs);
 int xe_gt_sriov_pf_config_set_fair_dbs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_bulk_set_dbs(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs,
 				       u32 num_dbs);
+int xe_gt_sriov_pf_config_bulk_set_dbs_locked(struct xe_gt *gt, unsigned int vfid,
+					      unsigned int num_vfs, u32 num_dbs);
 
 u64 xe_gt_sriov_pf_config_get_lmem(struct xe_gt *gt, unsigned int vfid);
 int xe_gt_sriov_pf_config_set_lmem(struct xe_gt *gt, unsigned int vfid, u64 size);
@@ -71,6 +77,8 @@ int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned
 
 u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid);
 int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority);
+int xe_gt_sriov_pf_config_set_sched_priority_locked(struct xe_gt *gt, unsigned int vfid,
+						    u32 priority);
 void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority);
 
 u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid,
@@ -79,9 +87,11 @@ int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid,
 					enum xe_guc_klv_threshold_index index, u32 value);
 
 int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs);
+int xe_gt_sriov_pf_config_set_fair_sched_locked(struct xe_gt *gt, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout);
 int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force);
+int xe_gt_sriov_pf_config_release_locked(struct xe_gt *gt, unsigned int vfid, bool force);
 int xe_gt_sriov_pf_config_push(struct xe_gt *gt, unsigned int vfid, bool refresh);
 
 ssize_t xe_gt_sriov_pf_config_save(struct xe_gt *gt, unsigned int vfid, void *buf, size_t size);
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_printk.h b/drivers/gpu/drm/xe/xe_gt_sriov_printk.h
index d3457d608db89b..ada0ee9f45e53f 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_printk.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_printk.h
@@ -27,6 +27,9 @@
 #define xe_gt_sriov_dbg(_gt, _fmt, ...) \
 	__xe_gt_sriov_printk(_gt, dbg, _fmt, ##__VA_ARGS__)
 
+#define xe_gt_sriov_dbg_ratelimited(_gt, _fmt, ...) \
+	__xe_gt_sriov_printk(_gt, dbg_ratelimited, _fmt, ##__VA_ARGS__)
+
 /* for low level noisy debug messages */
 #ifdef CONFIG_DRM_XE_DEBUG_SRIOV
 #define xe_gt_sriov_dbg_verbose(_gt, _fmt, ...) xe_gt_sriov_dbg(_gt, _fmt, ##__VA_ARGS__)
diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c
index 789397514f3ef2..2a40924660eea3 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats.c
+++ b/drivers/gpu/drm/xe/xe_gt_stats.c
@@ -95,12 +95,19 @@ void xe_gt_stats_incr(struct xe_gt *gt, const enum xe_gt_stats_id id, int incr)
 #define DEF_STAT_STR(ID, name) [XE_GT_STATS_ID_##ID] = name
 
 static const char *const stat_description[__XE_GT_STATS_NUM_IDS] = {
+	DEF_STAT_STR(CHAIN_PAGEFAULT_COUNT, "chain_pagefault_count"),
+	DEF_STAT_STR(CHAIN_IRQ_PAGEFAULT_COUNT, "chain_irq_pagefault_count"),
+	DEF_STAT_STR(CHAIN_DRAIN_IRQ_PAGEFAULT_COUNT, "chain_drain_irq_pagefault_count"),
+	DEF_STAT_STR(CHAIN_MISMATCH_PAGEFAULT_COUNT, "chain_mismatch_pagefault_count"),
+	DEF_STAT_STR(PARALLEL_PAGEFAULT_COUNT, "parallel_pagefault_count"),
+	DEF_STAT_STR(LAST_PAGEFAULT_COUNT, "last_pagefault_count"),
 	DEF_STAT_STR(SVM_PAGEFAULT_COUNT, "svm_pagefault_count"),
 	DEF_STAT_STR(TLB_INVAL, "tlb_inval_count"),
 	DEF_STAT_STR(SVM_TLB_INVAL_COUNT, "svm_tlb_inval_count"),
 	DEF_STAT_STR(SVM_TLB_INVAL_US, "svm_tlb_inval_us"),
 	DEF_STAT_STR(VMA_PAGEFAULT_COUNT, "vma_pagefault_count"),
 	DEF_STAT_STR(VMA_PAGEFAULT_KB, "vma_pagefault_kb"),
+	DEF_STAT_STR(PAGEFAULT_US, "pagefault_us"),
 	DEF_STAT_STR(INVALID_PREFETCH_PAGEFAULT_COUNT, "invalid_prefetch_pagefault_count"),
 	DEF_STAT_STR(SVM_4K_PAGEFAULT_COUNT, "svm_4K_pagefault_count"),
 	DEF_STAT_STR(SVM_64K_PAGEFAULT_COUNT, "svm_64K_pagefault_count"),
diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h
index 425491bed6c4eb..24abeb9f213711 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h
@@ -10,6 +10,19 @@
 
 /**
  * enum xe_gt_stats_id - GT statistics identifiers
+ * @XE_GT_STATS_ID_CHAIN_PAGEFAULT_COUNT: Total page faults chained onto an
+ *   in-flight fault instead of being queued separately.
+ * @XE_GT_STATS_ID_CHAIN_IRQ_PAGEFAULT_COUNT: Page faults chained directly from
+ *   the IRQ handler.
+ * @XE_GT_STATS_ID_CHAIN_DRAIN_IRQ_PAGEFAULT_COUNT: IRQ-handler chained faults
+ *   that also drained the fault queue.
+ * @XE_GT_STATS_ID_CHAIN_MISMATCH_PAGEFAULT_COUNT: Chained faults requeued
+ *   because their fault range did not match the fault they were chained onto.
+ * @XE_GT_STATS_ID_PARALLEL_PAGEFAULT_COUNT: Faults dequeued while another page
+ *   fault worker was already handling a fault concurrently.
+ * @XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT: Faults whose range matched the last
+ *   serviced range, allowing an immediate ack.
+ *
  * @XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT: Total SVM page faults handled.
  * @XE_GT_STATS_ID_TLB_INVAL: Total GPU Translation Lookaside Buffer (TLB)
  *   invalidations issued.
@@ -22,6 +35,8 @@
  *   handled.
  * @XE_GT_STATS_ID_VMA_PAGEFAULT_KB: Size (KiB) of VMAs involved in
  *   buffer-object page fault handling.
+ * @XE_GT_STATS_ID_PAGEFAULT_US: Cumulative time (µs) handling of all page
+ *   faults.
  * @XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT: GPU prefetch faults for
  *   addresses with no valid backing.
  *
@@ -127,12 +142,19 @@
  * See Documentation/gpu/xe/xe_gt_stats.rst.
  */
 enum xe_gt_stats_id {
+	XE_GT_STATS_ID_CHAIN_PAGEFAULT_COUNT,
+	XE_GT_STATS_ID_CHAIN_IRQ_PAGEFAULT_COUNT,
+	XE_GT_STATS_ID_CHAIN_DRAIN_IRQ_PAGEFAULT_COUNT,
+	XE_GT_STATS_ID_CHAIN_MISMATCH_PAGEFAULT_COUNT,
+	XE_GT_STATS_ID_PARALLEL_PAGEFAULT_COUNT,
+	XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_TLB_INVAL,
 	XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT,
 	XE_GT_STATS_ID_SVM_TLB_INVAL_US,
 	XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_VMA_PAGEFAULT_KB,
+	XE_GT_STATS_ID_PAGEFAULT_US,
 	XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_SVM_4K_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_SVM_64K_PAGEFAULT_COUNT,
diff --git a/drivers/gpu/drm/xe/xe_gt_throttle.c b/drivers/gpu/drm/xe/xe_gt_throttle.c
index 1e7e3a31aa6982..c0af5484611d1f 100644
--- a/drivers/gpu/drm/xe/xe_gt_throttle.c
+++ b/drivers/gpu/drm/xe/xe_gt_throttle.c
@@ -39,7 +39,7 @@
  * - ``reason_mem_thermal``: Memory thermal
  * - ``reason_vr_thermal``: VR thermal
  * - ``reason_iccmax``: ICCMAX
- * - ``reason_ratl``: RATL thermal algorithm
+ * - ``reason_pwrbrk``: Power brake
  * - ``reason_soc_avg_thermal``: SoC average temp
  * - ``reason_fastvmode``: VR is hitting FastVMode
  * - ``reason_psys_pl1``: PSYS PL1
@@ -200,6 +200,7 @@ static THROTTLE_ATTR_RO(reason_psys_pl1, PSYS_PL1_MASK);
 static THROTTLE_ATTR_RO(reason_psys_pl2, PSYS_PL2_MASK);
 static THROTTLE_ATTR_RO(reason_p0_freq, P0_FREQ_MASK);
 static THROTTLE_ATTR_RO(reason_psys_crit, PSYS_CRIT_MASK);
+static THROTTLE_ATTR_RO(reason_pwrbrk, PWRBRK_MASK);
 
 static struct attribute *cri_throttle_attrs[] = {
 	/* Common */
@@ -209,12 +210,12 @@ static struct attribute *cri_throttle_attrs[] = {
 	&attr_reason_pl2.attr.attr,
 	&attr_reason_pl4.attr.attr,
 	&attr_reason_prochot.attr.attr,
-	&attr_reason_ratl.attr.attr,
 	/* CRI */
 	&attr_reason_vr_thermal.attr.attr,
 	&attr_reason_soc_thermal.attr.attr,
 	&attr_reason_mem_thermal.attr.attr,
 	&attr_reason_iccmax.attr.attr,
+	&attr_reason_pwrbrk.attr.attr,
 	&attr_reason_soc_avg_thermal.attr.attr,
 	&attr_reason_fastvmode.attr.attr,
 	&attr_reason_psys_pl1.attr.attr,
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index a8bbfbdf384940..62891134645549 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -35,6 +35,24 @@ enum xe_gt_eu_type {
 	XE_GT_EU_TYPE_SIMD16,
 };
 
+/**
+ * enum xe_gpgpu_preempt_level - Per-context GPGPU preemption override mode
+ *
+ * Selects the preemption granularity programmed into CS_CHICKEN1[2:1] for
+ * newly created Xe2+ RCS LRCs.
+ *
+ * @XE_GPGPU_PREEMPT_DEFAULT: Keep platform default preemption granularity.
+ * @XE_GPGPU_PREEMPT_MID_THREAD: Force mid-thread preemption level.
+ * @XE_GPGPU_PREEMPT_THREAD_GROUP: Force thread-group preemption level.
+ * @XE_GPGPU_PREEMPT_COMMAND: Force command-level preemption.
+ */
+enum xe_gpgpu_preempt_level {
+	XE_GPGPU_PREEMPT_DEFAULT = 0,
+	XE_GPGPU_PREEMPT_MID_THREAD,
+	XE_GPGPU_PREEMPT_THREAD_GROUP,
+	XE_GPGPU_PREEMPT_COMMAND,
+};
+
 #define XE_MAX_DSS_FUSE_REGS		4
 #define XE_MAX_DSS_FUSE_BITS		(32 * XE_MAX_DSS_FUSE_REGS)
 #define XE_MAX_EU_FUSE_REGS		1
@@ -156,6 +174,8 @@ struct xe_gt {
 		 * feature.
 		 */
 		u8 has_xe2_blt_instructions:1;
+		/** @info.has_wmtp_disabled: hardware fuse indicates WMTP is disabled */
+		u8 has_wmtp_disabled:1;
 		/**
 		 * @info.num_geometry_xecore_fuse_regs: Number of 32b-bit fuse
 		 * registers the geometry XeCore mask spans.
@@ -224,6 +244,15 @@ struct xe_gt {
 	 */
 	u32 ccs_mode;
 
+	/**
+	 * @gpgpu_preemption_level: GPGPU preemption granularity override.
+	 *
+	 * Effective only when FF_SLICE_CS_CHICKEN1[FFSC_PERCTX_PREEMPT_CTRL] is
+	 * enabled via RTP. Affects only LRCs created after the value is
+	 * changed; existing contexts keep their previously programmed value.
+	 */
+	enum xe_gpgpu_preempt_level gpgpu_preemption_level;
+
 	/** @usm: unified shared memory state */
 	struct {
 		/**
diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index 4286bd05c68618..3ab4cb9e4964c9 100644
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -39,6 +39,7 @@
 #include "xe_guc_rc.h"
 #include "xe_guc_relay.h"
 #include "xe_guc_submit.h"
+#include "xe_log.h"
 #include "xe_memirq.h"
 #include "xe_mmio.h"
 #include "xe_platform_types.h"
@@ -1468,6 +1469,22 @@ int xe_guc_suspend(struct xe_guc *guc)
 	return 0;
 }
 
+/**
+ * xe_guc_bo_wa_flags - Extra BO flags for memory shared with the GuC
+ * @gt: the &xe_gt whose GuC the buffer will be shared with
+ *
+ * Wa_22016122933: on the standalone media GT, memory shared between the
+ * CPU and the GuC must not be mapped cached on the CPU side, otherwise
+ * the CPU can read stale data written by the GuC (e.g. G2H CTB writes)
+ * for multiple seconds.
+ *
+ * Return: additional XE_BO_FLAG_* to use when allocating GuC-shared memory
+ */
+u32 xe_guc_bo_wa_flags(struct xe_gt *gt)
+{
+	return XE_GT_WA(gt, 22016122933) ? XE_BO_FLAG_NEEDS_UC : 0;
+}
+
 void xe_guc_notify(struct xe_guc *guc)
 {
 	struct xe_gt *gt = guc_to_gt(guc);
@@ -1542,8 +1559,9 @@ int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
 		/* scratch registers might be cleared during FLR, try once more */
 		if (!header) {
 			if (++lost > MAX_RETRIES_ON_FLR) {
-				xe_gt_err(gt, "GuC mmio request %#x: lost, too many retries %u\n",
-					  request[0], lost);
+				xe_log_err(gt, GUC, -ENOLINK,
+					   "MMIO request %#x: lost, too many retries %u\n",
+					   request[0], lost);
 				return -ENOLINK;
 			}
 			xe_gt_dbg(gt, "GuC mmio request %#x: lost, trying again\n", request[0]);
@@ -1551,8 +1569,8 @@ int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
 			goto retry;
 		}
 timeout:
-		xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n",
-			  request[0], header);
+		xe_log_err(gt, GUC, ret, "MMIO request %#x: no reply %#x\n",
+			   request[0], header);
 		return ret;
 	}
 
@@ -1607,16 +1625,16 @@ int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request,
 			return -EREMCHG;
 		}
 
-		xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n",
-			  request[0], error, hint);
+		xe_log_err(gt, GUC, -ENXIO, "MMIO request %#x: failure %#x hint %#x\n",
+			   request[0], error, hint);
 		return -ENXIO;
 	}
 
 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) !=
 	    GUC_HXG_TYPE_RESPONSE_SUCCESS) {
 proto:
-		xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n",
-			  request[0], header);
+		xe_log_err(gt, GUC, -EPROTO, "MMIO request %#x: unexpected reply %#x\n",
+			   request[0], header);
 		return -EPROTO;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h
index 61e3ee19a59bfd..c4eca40d69cfce 100644
--- a/drivers/gpu/drm/xe/xe_guc.h
+++ b/drivers/gpu/drm/xe/xe_guc.h
@@ -30,6 +30,7 @@
 	xe_guc_fw_version_at_least((guc), MAKE_GUC_VER_ARGS(ver))
 
 struct drm_printer;
+struct xe_gt;
 
 void xe_guc_comm_init_early(struct xe_guc *guc);
 int xe_guc_init_noalloc(struct xe_guc *guc);
@@ -45,6 +46,7 @@ void xe_guc_runtime_suspend(struct xe_guc *guc);
 void xe_guc_runtime_resume(struct xe_guc *guc);
 int xe_guc_suspend(struct xe_guc *guc);
 int xe_guc_softreset(struct xe_guc *guc);
+u32 xe_guc_bo_wa_flags(struct xe_gt *gt);
 void xe_guc_notify(struct xe_guc *guc);
 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr);
 int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len);
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index ff8eee3831aa28..43a2951a66d648 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -383,6 +383,12 @@ static void guc_waklv_init(struct xe_guc_ads *ads)
 		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
 				 GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64);
 
+	/* GuC only applies this WA for scheduled MultiQ contexts; no KMD-side gating needed. */
+	if (XE_GT_WA(gt, 16030224309) &&
+	    GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 73))
+		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
+				 GUC_WA_KLV_IGNORE_MULTIQ_CTX_COMPLETE_WITH_PREEMPT_IN_CSB);
+
 	/*
 	 * On GuC firmware 70.66 and above, use the Feature KLV (shared with the
 	 * WA KLV buffer); older firmware uses GUC_CTL_DISABLE_MULTI_QUEUE in
@@ -435,7 +441,8 @@ int xe_guc_ads_init(struct xe_guc_ads *ads)
 					  XE_BO_FLAG_SYSTEM |
 					  XE_BO_FLAG_GGTT |
 					  XE_BO_FLAG_GGTT_INVALIDATE |
-					  XE_BO_FLAG_PINNED_NORESTORE);
+					  XE_BO_FLAG_PINNED_NORESTORE |
+					  xe_guc_bo_wa_flags(gt));
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -864,7 +871,7 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads,
 		}
 	}
 
-	if (XE_GT_WA(hwe->gt, 16023105232))
+	if (XE_GT_WA(hwe->gt, 16023105232) || XE_GT_WA(hwe->gt, 14025941587))
 		guc_mmio_regset_write_one(ads, regset_map,
 					  RING_IDLEDLY(hwe->mmio_base),
 					  count++);
diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c
index 82df19b304e174..5a4d085660a20f 100644
--- a/drivers/gpu/drm/xe/xe_guc_capture.c
+++ b/drivers/gpu/drm/xe/xe_guc_capture.c
@@ -1868,8 +1868,6 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm
 		snapshot_print_by_list_order(snapshot, p, GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS,
 					     list);
 	}
-
-	drm_puts(p, "\n");
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
index fe70c0fd85c526..5c393aa29de3ef 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.c
+++ b/drivers/gpu/drm/xe/xe_guc_ct.c
@@ -265,7 +265,7 @@ static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
 #define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
 #define CTB_H2G_BUFFER_OFFSET	(CTB_DESC_SIZE * 2)
 #define CTB_G2H_BUFFER_OFFSET	(CTB_DESC_SIZE * 2)
-#define CTB_H2G_BUFFER_SIZE	(SZ_4K)
+#define CTB_H2G_BUFFER_SIZE	(SZ_16K)
 #define CTB_H2G_BUFFER_DWORDS	(CTB_H2G_BUFFER_SIZE / sizeof(u32))
 #define CTB_G2H_BUFFER_SIZE	(SZ_128K)
 #define CTB_G2H_BUFFER_DWORDS	(CTB_G2H_BUFFER_SIZE / sizeof(u32))
@@ -376,7 +376,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
 					  XE_BO_FLAG_SYSTEM |
 					  XE_BO_FLAG_GGTT |
 					  XE_BO_FLAG_GGTT_INVALIDATE |
-					  XE_BO_FLAG_PINNED_NORESTORE);
+					  XE_BO_FLAG_PINNED_NORESTORE |
+					  xe_guc_bo_wa_flags(gt));
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -386,7 +387,8 @@ int xe_guc_ct_init(struct xe_guc_ct *ct)
 					  XE_BO_FLAG_SYSTEM |
 					  XE_BO_FLAG_GGTT |
 					  XE_BO_FLAG_GGTT_INVALIDATE |
-					  XE_BO_FLAG_PINNED_NORESTORE);
+					  XE_BO_FLAG_PINNED_NORESTORE |
+					  xe_guc_bo_wa_flags(gt));
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
@@ -939,7 +941,7 @@ static bool vf_action_can_safely_fail(struct xe_device *xe, u32 action)
 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */
 
 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
-		     u32 ct_fence_value, bool want_response)
+		     u32 ct_fence_value, bool want_response, bool defer_flush)
 {
 	struct xe_device *xe = ct_to_xe(ct);
 	struct xe_gt *gt = ct_to_gt(ct);
@@ -956,7 +958,6 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN);
 
 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
-		u32 desc_tail = desc_read(xe, h2g, tail);
 		u32 desc_head = desc_read(xe, h2g, head);
 		u32 desc_status;
 
@@ -966,12 +967,6 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 			goto corrupted;
 		}
 
-		if (tail != desc_tail) {
-			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH);
-			xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail);
-			goto corrupted;
-		}
-
 		if (tail > h2g->info.size) {
 			desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW);
 			xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n",
@@ -993,7 +988,10 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 			      (h2g->info.size - tail) * sizeof(u32));
 		h2g_reserve_space(ct, (h2g->info.size - tail));
 		h2g->info.tail = 0;
-		desc_write(xe, h2g, tail, h2g->info.tail);
+		if (!defer_flush) {
+			xe_device_wmb(xe);
+			desc_write(xe, h2g, tail, h2g->info.tail);
+		}
 
 		return -EAGAIN;
 	}
@@ -1024,14 +1022,15 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	/* Write H2G ensuring visible before descriptor update */
 	xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));
 	xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));
-	xe_device_wmb(xe);
-
 	/* Update local copies */
 	h2g->info.tail = (tail + full_len) % h2g->info.size;
 	h2g_reserve_space(ct, full_len);
 
 	/* Update descriptor */
-	desc_write(xe, h2g, tail, h2g->info.tail);
+	if (!defer_flush) {
+		xe_device_wmb(xe);
+		desc_write(xe, h2g, tail, h2g->info.tail);
+	}
 
 	/*
 	 * desc_read() performs an VRAM read which serializes the CPU and drains
@@ -1052,7 +1051,7 @@ static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
 
 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
 				u32 len, u32 g2h_len, u32 num_g2h,
-				struct g2h_fence *g2h_fence)
+				struct g2h_fence *g2h_fence, bool defer_flush)
 {
 	struct xe_gt *gt = ct_to_gt(ct);
 	u16 seqno;
@@ -1112,7 +1111,7 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
 	if (unlikely(ret))
 		goto out_unlock;
 
-	ret = h2g_write(ct, action, len, seqno, !!g2h_fence);
+	ret = h2g_write(ct, action, len, seqno, !!g2h_fence, defer_flush);
 	if (unlikely(ret)) {
 		if (ret == -EAGAIN)
 			goto retry;
@@ -1120,7 +1119,8 @@ static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
 	}
 
 	__g2h_reserve_space(ct, g2h_len, num_g2h);
-	xe_guc_notify(ct_to_guc(ct));
+	if (!defer_flush)
+		xe_guc_notify(ct_to_guc(ct));
 out_unlock:
 	if (g2h_len)
 		spin_unlock_irq(&ct->fast_lock);
@@ -1196,7 +1196,7 @@ static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len,
 
 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
 			      u32 g2h_len, u32 num_g2h,
-			      struct g2h_fence *g2h_fence)
+			      struct g2h_fence *g2h_fence, bool defer_flush)
 {
 	struct xe_gt *gt = ct_to_gt(ct);
 	unsigned int sleep_period_ms = 1;
@@ -1209,9 +1209,10 @@ static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
 
 try_again:
 	ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,
-				   g2h_fence);
+				   g2h_fence, defer_flush);
 
 	if (unlikely(ret == -EBUSY)) {
+		xe_guc_ct_send_flush(ct);
 		if (!guc_ct_send_wait_for_retry(ct, len, g2h_len, g2h_fence,
 						&sleep_period_ms, &sleep_total_ms))
 			goto broken;
@@ -1235,7 +1236,8 @@ static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence);
 
 	mutex_lock(&ct->lock);
-	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
+	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence,
+				 false);
 	mutex_unlock(&ct->lock);
 
 	return ret;
@@ -1283,25 +1285,76 @@ int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	return ret;
 }
 
+/**
+ * xe_guc_ct_send_locked() - submit a GuC CT H2G message with CT lock held
+ * @ct: GuC CT object
+ * @action: payload dwords (HxG header dword is expected at @action[-1])
+ * @len: number of payload dwords in @action
+ * @defer_flush: defer publishing/doorbell for batching
+ *
+ * Sends a single H2G message to the GuC CT buffer while the caller already
+ * holds @ct->lock.
+ *
+ * If @defer_flush is false, the function completes the submission immediately:
+ * it makes the payload visible to the device, updates the H2G descriptor and
+ * rings the GuC doorbell.
+ *
+ * If @defer_flush is true, the message payload is copied into the H2G ring and
+ * the software tail is advanced, but the descriptor update and doorbell are
+ * deferred so multiple messages can be batched. In this mode, the caller must
+ * eventually call xe_guc_ct_send_flush() (still holding @ct->lock) to publish
+ * the descriptor and notify the GuC. On internal retry paths (-EBUSY), the
+ * implementation may force a flush to ensure forward progress.
+ *
+ * Return: 0 on success, negative errno on failure.
+ *
+ * Locking:
+ *   Must be called with @ct->lock held.
+ */
 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
-			  u32 g2h_len, u32 num_g2h)
+			  bool defer_flush)
 {
 	int ret;
 
-	ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);
+	ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL, defer_flush);
 	if (ret == -EDEADLK)
 		kick_reset(ct);
 
 	return ret;
 }
 
+/**
+ * xe_guc_ct_send_flush() - flush pending GuC CT H2G writes
+ * @ct: GuC CT instance
+ *
+ * Some callers batch multiple H2G writes using xe_guc_ct_send_locked() in
+ * "write-only" mode (i.e., queue the message payloads but defer ringing the
+ * doorbell / updating the CT descriptor). This helper completes the submission
+ * by ensuring the payload writes are visible to the device, updating the H2G
+ * descriptor, and ringing the GuC CT doorbell.
+ *
+ * Locking:
+ *   Must be called with @ct->lock held.
+ */
+void xe_guc_ct_send_flush(struct xe_guc_ct *ct)
+{
+	struct xe_device *xe = ct_to_xe(ct);
+	struct guc_ctb *h2g = &ct->ctbs.h2g;
+
+	lockdep_assert_held(&ct->lock);
+
+	xe_device_wmb(xe);
+	desc_write(xe, h2g, tail, h2g->info.tail);
+	xe_guc_notify(ct_to_guc(ct));
+}
+
 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)
 {
 	int ret;
 
 	lockdep_assert_held(&ct->lock);
 
-	ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);
+	ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL, false);
 	if (ret == -EDEADLK)
 		kick_reset(ct);
 
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h
index 767365a33deeac..3ddc665ab84ad1 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.h
+++ b/drivers/gpu/drm/xe/xe_guc_ct.h
@@ -6,6 +6,8 @@
 #ifndef _XE_GUC_CT_H_
 #define _XE_GUC_CT_H_
 
+#include <linux/mutex.h>
+
 #include "xe_guc_ct_types.h"
 
 struct drm_printer;
@@ -54,7 +56,7 @@ static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct)
 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
 		   u32 g2h_len, u32 num_g2h);
 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
-			  u32 g2h_len, u32 num_g2h);
+			  bool defer_flush);
 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
 			u32 *response_buffer);
 static inline int
@@ -63,6 +65,8 @@ xe_guc_ct_send_block(struct xe_guc_ct *ct, const u32 *action, u32 len)
 	return xe_guc_ct_send_recv(ct, action, len, NULL);
 }
 
+void xe_guc_ct_send_flush(struct xe_guc_ct *ct);
+
 /* This is only version of the send CT you can call from a G2H handler */
 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action,
 			       u32 len);
@@ -87,4 +91,36 @@ static inline void xe_guc_ct_wake_waiters(struct xe_guc_ct *ct)
 	wake_up_all(&ct->wq);
 }
 
+/**
+ * xe_guc_ct_lock() - take the GuC CT mutex
+ * @ct: GuC CT object
+ *
+ * Wrapper around mutex_lock(&ct->lock) for cases where CT operations need to be
+ * performed from contexts that want an explicit "CT locked" pair without
+ * exporting the lock itself.
+ *
+ * Return/Locking:
+ *   Acquires @ct->lock.
+ */
+static inline void xe_guc_ct_lock(struct xe_guc_ct *ct)
+__acquires(&ct->lock)
+{
+	mutex_lock(&ct->lock);
+}
+
+/**
+ * xe_guc_ct_unlock() - release the GuC CT mutex
+ * @ct: GuC CT object
+ *
+ * Counterpart to xe_guc_ct_lock().
+ *
+ * Locking:
+ *   Releases @ct->lock.
+ */
+static inline void xe_guc_ct_unlock(struct xe_guc_ct *ct)
+__releases(&ct->lock)
+{
+	mutex_unlock(&ct->lock);
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc_engine_activity.c b/drivers/gpu/drm/xe/xe_guc_engine_activity.c
index a782be57caadbb..729ce8ac140b6d 100644
--- a/drivers/gpu/drm/xe/xe_guc_engine_activity.c
+++ b/drivers/gpu/drm/xe/xe_guc_engine_activity.c
@@ -97,7 +97,8 @@ static int allocate_engine_activity_buffers(struct xe_guc *guc,
 
 	metadata_bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(metadata_size),
 						ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM |
-						XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE,
+						XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE |
+						xe_guc_bo_wa_flags(gt),
 						false);
 
 	if (IS_ERR(metadata_bo))
@@ -105,7 +106,8 @@ static int allocate_engine_activity_buffers(struct xe_guc *guc,
 
 	bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(size),
 				       ttm_bo_type_kernel, XE_BO_FLAG_VRAM_IF_DGFX(tile) |
-				       XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE, false);
+				       XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE |
+				       xe_guc_bo_wa_flags(gt), false);
 
 	if (IS_ERR(bo)) {
 		xe_bo_unpin_map_no_vm(metadata_bo);
diff --git a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
index acdc24d1a6bd21..d27826b36649f9 100644
--- a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h
@@ -36,7 +36,7 @@ struct xe_guc_exec_queue {
 	 * a message needs to sent through the GPU scheduler but memory
 	 * allocations are not allowed.
 	 */
-#define MAX_STATIC_MSG_TYPE	3
+#define MAX_STATIC_MSG_TYPE	4
 	struct xe_sched_msg static_msgs[MAX_STATIC_MSG_TYPE];
 	/** @destroy_async: do final destroy async from this worker */
 	struct work_struct destroy_async;
@@ -76,6 +76,36 @@ struct xe_guc_exec_queue {
 	 * recovery.
 	 */
 	bool needs_resume;
+	/** @multi_queue: multi-queue group CGP state for VF post migration recovery */
+	struct {
+		/**
+		 * @multi_queue.needs_cgp_sync: Needs a CGP_SYNC (dynamic CGP
+		 * update) message replayed during recovery.
+		 */
+		u8 needs_cgp_sync:1;
+		/**
+		 * @multi_queue.re_register: A registration-time CGP update was
+		 * interrupted by recovery; the queue must be re-registered.
+		 */
+		u8 re_register:1;
+		/**
+		 * @multi_queue.re_update: A dynamic CGP update was interrupted
+		 * by recovery; the CGP update must be replayed.
+		 */
+		u8 re_update:1;
+		/**
+		 * @multi_queue.registering_cgp: This queue's currently
+		 * outstanding CGP_SYNC is a registration (matched against
+		 * group->cgp_update_q in revert).
+		 */
+		u8 registering_cgp:1;
+		/**
+		 * @multi_queue.updating_cgp: This queue's currently outstanding
+		 * CGP_SYNC is a dynamic update (matched against
+		 * group->cgp_update_q in revert).
+		 */
+		u8 updating_cgp:1;
+	} multi_queue;
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.c b/drivers/gpu/drm/xe/xe_guc_hwconfig.c
index b300901dbb8e1e..dc667a71b438dc 100644
--- a/drivers/gpu/drm/xe/xe_guc_hwconfig.c
+++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.c
@@ -65,7 +65,7 @@ int xe_guc_hwconfig_init(struct xe_guc *guc)
 	/*
 	 * All hwconfig the same across GTs so only GT0 needs to be configured
 	 */
-	if (gt->info.id != XE_GT0)
+	if (gt != xe_root_mmio_gt(xe))
 		return 0;
 
 	/* ADL_P, DG2+ supports hwconfig table */
diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
index dc8612d761e3cf..dafb5b74e4509a 100644
--- a/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
+++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.c
@@ -372,6 +372,84 @@ int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj,
 	return total;
 }
 
+/**
+ * xe_guc_klv_decode_u16() - Decode u16 data from KLV.
+ * @value: array of dwords
+ * @len: number of dwords in @value
+ * @data: pointer where to store the data
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_guc_klv_decode_u16(const u32 *value, u16 len, u16 *data)
+{
+	u32 tmp;
+	int err;
+
+	err = xe_guc_klv_decode_u32(value, len, &tmp);
+	if (err)
+		return err;
+	if (upper_16_bits(tmp))
+		return -EOVERFLOW;
+	*data = lower_16_bits(tmp);
+	return 0;
+}
+
+/**
+ * xe_guc_klv_decode_u32() - Decode u32 data from KLV.
+ * @value: array of dwords
+ * @len: number of dwords in @value
+ * @data: pointer where to store the data
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_guc_klv_decode_u32(const u32 *value, u16 len, u32 *data)
+{
+	if (len < to_num_dwords(sizeof(u32)))
+		return -ENODATA;
+	if (len > to_num_dwords(sizeof(u32)))
+		return -ENOMSG;
+	*data = *value;
+	return 0;
+}
+
+/**
+ * xe_guc_klv_decode_u64() - Decode u64 data from KLV.
+ * @value: array of dwords
+ * @len: number of dwords in @value
+ * @data: pointer where to store the data
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_guc_klv_decode_u64(const u32 *value, u16 len, u64 *data)
+{
+	if (len < to_num_dwords(sizeof(u64)))
+		return -ENODATA;
+	if (len != to_num_dwords(sizeof(u64)))
+		return -ENOMSG;
+	*data = make_u64(value[1], value[0]);
+	return 0;
+}
+
+/**
+ * xe_guc_klv_decode_string() - Decode string data from KLV.
+ * @value: array of dwords
+ * @len: number of dwords in @value
+ * @data: pointer where to store the data
+ *
+ * Note: caller must use kfree() to release decoded string.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_guc_klv_decode_string(const u32 *value, u16 len, char **data)
+{
+	const char *src = (const void *)value;
+
+	if (!len)
+		return -ENOMSG;
+	*data = kstrndup(src, to_num_bytes(len), GFP_KERNEL);
+	return *data ? 0 : -ENOMEM;
+}
+
 #if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
 #include "tests/xe_guc_klv_helpers_kunit.c"
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
index cf3b29adc9d6b3..10af545a0f7921 100644
--- a/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
+++ b/drivers/gpu/drm/xe/xe_guc_klv_helpers.h
@@ -26,6 +26,11 @@ u32 *xe_guc_klv_encode_object(u32 *klvs, u32 avail, u16 key, const void *obj,
 int xe_guc_klv_parser(const u32 *klvs, u32 num_dwords, void *obj,
 		      int (*decoder)(void *obj, u16 key, u16 len, const u32 *value));
 
+int xe_guc_klv_decode_u16(const u32 *value, u16 len, u16 *data);
+int xe_guc_klv_decode_u32(const u32 *value, u16 len, u32 *data);
+int xe_guc_klv_decode_u64(const u32 *value, u16 len, u64 *data);
+int xe_guc_klv_decode_string(const u32 *value, u16 len, char **data);
+
 /**
  * PREP_GUC_KLV - Prepare KLV header value based on provided key and len.
  * @key: KLV key
diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c
index 538d4df0f7aa76..7d006268ce916a 100644
--- a/drivers/gpu/drm/xe/xe_guc_log.c
+++ b/drivers/gpu/drm/xe/xe_guc_log.c
@@ -17,6 +17,7 @@
 #include "xe_force_wake.h"
 #include "xe_gt_printk.h"
 #include "xe_gt_types.h"
+#include "xe_guc.h"
 #include "xe_map.h"
 #include "xe_mmio.h"
 #include "xe_module.h"
@@ -624,14 +625,16 @@ void xe_guc_log_print_lfd(struct xe_guc_log *log, struct drm_printer *p)
 int xe_guc_log_init(struct xe_guc_log *log)
 {
 	struct xe_device *xe = log_to_xe(log);
-	struct xe_tile *tile = gt_to_tile(log_to_gt(log));
+	struct xe_gt *gt = log_to_gt(log);
+	struct xe_tile *tile = gt_to_tile(gt);
 	struct xe_bo *bo;
 
 	bo = xe_managed_bo_create_pin_map(xe, tile, GUC_LOG_SIZE,
 					  XE_BO_FLAG_SYSTEM |
 					  XE_BO_FLAG_GGTT |
 					  XE_BO_FLAG_GGTT_INVALIDATE |
-					  XE_BO_FLAG_PINNED_NORESTORE);
+					  XE_BO_FLAG_PINNED_NORESTORE |
+					  xe_guc_bo_wa_flags(gt));
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
diff --git a/drivers/gpu/drm/xe/xe_guc_pagefault.c b/drivers/gpu/drm/xe/xe_guc_pagefault.c
index 607e32392f46f9..df237fd4055191 100644
--- a/drivers/gpu/drm/xe/xe_guc_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_guc_pagefault.c
@@ -10,6 +10,22 @@
 #include "xe_pagefault.h"
 #include "xe_pagefault_types.h"
 
+#define XE_GUC_PAGEFAULT_FLUSH_PERIOD	BIT(4)	/* Sixteen */
+
+static void guc_ack_fault_begin(void *private)
+{
+	struct xe_guc *guc = private;
+
+	xe_guc_ct_lock(&guc->ct);
+
+	BUILD_BUG_ON(((XE_GUC_PAGEFAULT_FLUSH_PERIOD - 1) &
+		     XE_GUC_PAGEFAULT_FLUSH_PERIOD) != 0);
+
+	/* Ack the 2nd, then 18th, etc... */
+	guc->pagefault_ack_counter =
+		XE_GUC_PAGEFAULT_FLUSH_PERIOD - 1;
+}
+
 static void guc_ack_fault(struct xe_pagefault *pf, int err)
 {
 	u32 vfid = FIELD_GET(PFD_VFID, pf->producer.msg[2]);
@@ -36,12 +52,26 @@ static void guc_ack_fault(struct xe_pagefault *pf, int err)
 		FIELD_PREP(PFR_PDATA, pdata),
 	};
 	struct xe_guc *guc = pf->producer.private;
+	bool write_only = guc->pagefault_ack_counter++ &
+		(XE_GUC_PAGEFAULT_FLUSH_PERIOD - 1);
 
-	xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
+	xe_guc_ct_send_locked(&guc->ct, action, ARRAY_SIZE(action),
+			      write_only);
+}
+
+static void guc_ack_fault_end(void *private)
+{
+	struct xe_guc *guc = private;
+
+	if ((guc->pagefault_ack_counter & (XE_GUC_PAGEFAULT_FLUSH_PERIOD - 1)) != 1)
+		xe_guc_ct_send_flush(&guc->ct);
+	xe_guc_ct_unlock(&guc->ct);
 }
 
 static const struct xe_pagefault_ops guc_pagefault_ops = {
+	.ack_fault_begin = guc_ack_fault_begin,
 	.ack_fault = guc_ack_fault,
+	.ack_fault_end = guc_ack_fault_end,
 };
 
 /**
@@ -78,7 +108,13 @@ int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
 				      << PFD_VIRTUAL_ADDR_HI_SHIFT) |
 		(FIELD_GET(PFD_VIRTUAL_ADDR_LO, msg[2]) <<
 		 PFD_VIRTUAL_ADDR_LO_SHIFT);
-	pf.consumer.asid = FIELD_GET(PFD_ASID, msg[1]);
+
+	BUILD_BUG_ON(XE_MAX_ASID > XE_PAGEFAULT_ASID_MASK);
+
+	pf.consumer.id = FIELD_PREP(XE_PAGEFAULT_ASID_MASK,
+				    FIELD_GET(PFD_ASID, msg[1])) |
+			 FIELD_PREP(XE_PAGEFAULT_SRCID_MASK,
+				    FIELD_GET(PFD_SRC_ID, msg[0]));
 	pf.consumer.access_type = FIELD_GET(PFD_ACCESS_TYPE, msg[2]) |
 		(FIELD_GET(PFD_PREFETCH, msg[2]) ? XE_PAGEFAULT_ACCESS_PREFETCH : 0);
 	if (FIELD_GET(XE2_PFD_TRVA_FAULT, msg[0]))
@@ -89,8 +125,11 @@ int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len)
 				   FIELD_GET(PFD_FAULT_LEVEL, msg[0])) |
 			FIELD_PREP(XE_PAGEFAULT_TYPE_MASK,
 				   FIELD_GET(PFD_FAULT_TYPE, msg[2]));
-	pf.consumer.engine_class = FIELD_GET(PFD_ENG_CLASS, msg[0]);
-	pf.consumer.engine_instance = FIELD_GET(PFD_ENG_INSTANCE, msg[0]);
+	pf.consumer.engine_class_instance =
+		FIELD_PREP(XE_PAGEFAULT_ENGINE_CLASS_MASK,
+			   FIELD_GET(PFD_ENG_CLASS, msg[0])) |
+		FIELD_PREP(XE_PAGEFAULT_ENGINE_INSTANCE_MASK,
+			   FIELD_GET(PFD_ENG_INSTANCE, msg[0]));
 
 	pf.producer.private = guc;
 	pf.producer.ops = &guc_pagefault_ops;
diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c
index 7cf8f485859886..e0105222a2c7ef 100644
--- a/drivers/gpu/drm/xe/xe_guc_pc.c
+++ b/drivers/gpu/drm/xe/xe_guc_pc.c
@@ -1241,8 +1241,21 @@ static int pc_modify_defaults(struct xe_guc_pc *pc)
 
 	if (xe->info.platform == XE_PANTHERLAKE) {
 		ret = pc_action_set_dcc(pc, false);
-		if (unlikely(ret))
+		if (unlikely(ret)) {
 			xe_gt_err(gt, "Failed to modify DCC default: %pe\n", ERR_PTR(ret));
+			return ret;
+		}
+
+		if (xe_gt_is_main_type(gt) &&
+		    GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 48, 0)) {
+			ret = pc_action_set_param(pc,
+						  SLPC_PARAM_SET_IBC_VERSION,
+						  3);
+			if (unlikely(ret)) {
+				xe_gt_err(gt, "Failed to modify IBC version: %pe\n", ERR_PTR(ret));
+				return ret;
+			}
+		}
 	}
 
 	return ret;
@@ -1378,7 +1391,8 @@ int xe_guc_pc_init(struct xe_guc_pc *pc)
 					  XE_BO_FLAG_VRAM_IF_DGFX(tile) |
 					  XE_BO_FLAG_GGTT |
 					  XE_BO_FLAG_GGTT_INVALIDATE |
-					  XE_BO_FLAG_PINNED_NORESTORE);
+					  XE_BO_FLAG_PINNED_NORESTORE |
+					  xe_guc_bo_wa_flags(gt));
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index 864859bd0ccc9e..cd9bb9f9e28c82 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -6,6 +6,7 @@
 #include "xe_guc_submit.h"
 
 #include <linux/bitfield.h>
+#include <uapi/drm/xe_drm.h>
 #include <linux/bitmap.h>
 #include <linux/circ_buf.h>
 #include <linux/dma-fence-array.h>
@@ -18,6 +19,7 @@
 #include "abi/guc_klvs_abi.h"
 #include "xe_assert.h"
 #include "xe_bo.h"
+#include "xe_cpu_bind.h"
 #include "xe_devcoredump.h"
 #include "xe_device.h"
 #include "xe_exec_queue.h"
@@ -34,12 +36,14 @@
 #include "xe_guc_klv_helpers.h"
 #include "xe_guc_submit_types.h"
 #include "xe_hw_engine.h"
+#include "xe_log.h"
 #include "xe_lrc.h"
 #include "xe_macros.h"
 #include "xe_map.h"
 #include "xe_mocs.h"
 #include "xe_module.h"
 #include "xe_pm.h"
+#include "xe_pt.h"
 #include "xe_ring_ops_types.h"
 #include "xe_sched_job.h"
 #include "xe_sleep.h"
@@ -800,9 +804,12 @@ static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe,
 	}
 }
 
+#define CGP_SYNC_REGISTRATION			BIT(0)
+
 static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
 					     struct xe_exec_queue *q,
-					     const u32 *action, u32 len)
+					     const u32 *action, u32 len,
+					     unsigned int flags)
 {
 	struct xe_exec_queue_group *group = q->multi_queue.group;
 	struct xe_device *xe = guc_to_xe(guc);
@@ -815,13 +822,12 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
 	 * Hence, no locking is required here.
 	 * Wait for any pending CGP_SYNC_DONE response before updating the
 	 * CGP page and sending CGP_SYNC message.
-	 *
-	 * FIXME: Support VF migration
 	 */
 	ret = wait_event_timeout(guc->ct.wq,
 				 !READ_ONCE(group->sync_pending) ||
-				 xe_guc_read_stopped(guc), HZ);
-	if (!ret || xe_guc_read_stopped(guc)) {
+				 xe_guc_read_stopped(guc) || vf_recovery(guc),
+				 HZ);
+	if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) {
 		/* CGP_SYNC failed. Reset gt, cleanup the group */
 		xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n");
 		set_exec_queue_group_banned(q);
@@ -830,17 +836,45 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
 		return;
 	}
 
+	/*
+	 * If woken by VF migration recovery, do not touch the CGP or send: the
+	 * message would be lost and, for a registration, GuC must (re-)register
+	 * the context before its CGP entry may be read. Flag the queue so revert
+	 * replays it - a registration by re-registration, a dynamic update by a
+	 * replayed CGP_SYNC - and bail.
+	 */
+	if (vf_recovery(guc)) {
+		if (flags & CGP_SYNC_REGISTRATION)
+			q->guc->multi_queue.re_register = true;
+		else
+			q->guc->multi_queue.re_update = true;
+		return;
+	}
+
 	scoped_guard(spinlock, &q->multi_queue.lock)
 		priority = q->multi_queue.priority;
 
 	xe_lrc_set_multi_queue_priority(q->lrc[0], priority);
 	xe_guc_exec_queue_group_cgp_update(xe, q);
 
+	/*
+	 * Record the nature of this outstanding sync so revert can replay it if
+	 * its CGP_SYNC_DONE is lost across a migration: a registration is
+	 * recovered by re-registration, a dynamic update by a replayed CGP_SYNC.
+	 */
+	if (flags & CGP_SYNC_REGISTRATION) {
+		q->guc->multi_queue.registering_cgp = true;
+		q->guc->multi_queue.updating_cgp = false;
+	} else {
+		q->guc->multi_queue.updating_cgp = true;
+		q->guc->multi_queue.registering_cgp = false;
+	}
+	WRITE_ONCE(group->cgp_update_q, q);
 	WRITE_ONCE(group->sync_pending, true);
 	xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1);
 }
 
-static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q)
+static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q, unsigned int flags)
 {
 #define MAX_MULTI_QUEUE_CGP_SYNC_SIZE	(2)
 	struct xe_guc *guc = exec_queue_to_guc(q);
@@ -854,7 +888,7 @@ static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q)
 	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
 #undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
 
-	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
+	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len, flags);
 }
 
 static void __register_exec_queue_group(struct xe_exec_queue *q,
@@ -882,7 +916,8 @@ static void __register_exec_queue_group(struct xe_exec_queue *q,
 	 * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
 	 * from guc.
 	 */
-	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
+	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len,
+					 CGP_SYNC_REGISTRATION);
 }
 
 static void __register_mlrc_exec_queue(struct xe_guc *guc,
@@ -1042,7 +1077,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
 		init_policies(guc, q);
 
 	if (xe_exec_queue_is_multi_queue_secondary(q))
-		guc_exec_queue_send_cgp_sync(q);
+		guc_exec_queue_send_cgp_sync(q, CGP_SYNC_REGISTRATION);
 }
 
 static u32 wq_space_until_wrap(struct xe_exec_queue *q)
@@ -1155,9 +1190,10 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
 	xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q));
 
 	if (!job->restore_replay || job->last_replay) {
+		/* A ULLS job past the first publishes its own ring tail */
 		if (xe_exec_queue_is_parallel(q))
 			wq_item_append(q);
-		else
+		else if (!xe_sched_job_ulls_is_chained(job))
 			xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
 		job->last_replay = false;
 	}
@@ -1174,6 +1210,9 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
 	if (exec_queue_suspended(q))
 		return;
 
+	if (xe_sched_job_ulls_is_chained(job))
+		xe_lrc_set_ulls_semaphore(lrc, xe_sched_job_lrc_seqno(job));
+
 	if (!exec_queue_enabled(q)) {
 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
 		action[len++] = q->guc->id;
@@ -1187,13 +1226,14 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
 		set_exec_queue_pending_enable(q);
 		set_exec_queue_enabled(q);
 		trace_xe_exec_queue_scheduling_enable(q);
-	} else {
+	} else if (!xe_sched_job_ulls_is_chained(job)) {
 		action[len++] = XE_GUC_ACTION_SCHED_CONTEXT;
 		action[len++] = q->guc->id;
 		trace_xe_exec_queue_submit(q);
 	}
 
-	xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);
+	if (!xe_sched_job_ulls_is_chained(job) || num_g2h)
+		xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);
 
 	if (extra_submit) {
 		len = 0;
@@ -1205,21 +1245,70 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job)
 	}
 }
 
+static bool is_pt_job(struct xe_sched_job *job)
+{
+	return job->is_pt_job;
+}
+
+static void run_pt_job(struct xe_device *xe, struct xe_sched_job *job,
+		       bool force_clear)
+{
+	struct xe_tile *tile;
+	int id;
+
+	for_each_tile(tile, xe, id) {
+		struct xe_pt_job_ops *pt_job_ops =
+			job->pt_update[0].pt_job_ops[id];
+
+		if (!pt_job_ops || !pt_job_ops->current_op)
+			continue;
+
+		xe_cpu_bind_update_pgtables_execute(job->pt_update[0].vm, tile,
+						    job->pt_update[0].ops,
+						    pt_job_ops->ops,
+						    pt_job_ops->current_op,
+						    force_clear);
+	}
+}
+
+static void put_pt_job(struct xe_device *xe, struct xe_sched_job *job)
+{
+	struct xe_tile *tile;
+	int id;
+
+	for_each_tile(tile, xe, id) {
+		struct xe_pt_job_ops *pt_job_ops =
+			job->pt_update[0].pt_job_ops[id];
+
+		xe_pt_job_ops_put(pt_job_ops);
+	}
+}
+
 static struct dma_fence *
 guc_exec_queue_run_job(struct drm_sched_job *drm_job)
 {
 	struct xe_sched_job *job = to_xe_sched_job(drm_job);
 	struct xe_exec_queue *q = job->q;
 	struct xe_guc *guc = exec_queue_to_guc(q);
-	bool killed_or_banned_or_wedged =
-		exec_queue_killed_or_banned_or_wedged(q);
+	bool killed_or_banned_or_wedged_or_error  =
+		exec_queue_killed_or_banned_or_wedged(q) ||
+		xe_sched_job_is_error(job);
 
 	xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) ||
 		     exec_queue_banned(q) || exec_queue_suspended(q));
 
 	trace_xe_sched_job_run(job);
 
-	if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) {
+	if (is_pt_job(job)) {
+		xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q));
+
+		run_pt_job(guc_to_xe(guc), job,
+			   killed_or_banned_or_wedged_or_error);
+		put_pt_job(guc_to_xe(guc), job);
+		dma_fence_put(job->fence);	/* Drop ref from xe_sched_job_arm */
+
+		return NULL;
+	} else if (!killed_or_banned_or_wedged_or_error) {
 		if (xe_exec_queue_is_multi_queue_secondary(q)) {
 			struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
 
@@ -1568,6 +1657,12 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 	else
 		wedged = xe_device_wedged(xe);
 
+	/*
+	 * Only tag as GPU hang if this is the original timeout, not a
+	 * consequence of a prior kill (e.g., page-offline).
+	 */
+	if (!exec_queue_killed(q))
+		atomic_or(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG, &q->ban_reason);
 	set_exec_queue_banned(q);
 
 	/* Kick job / queue off hardware */
@@ -1651,6 +1746,9 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
 		if (timeout_needs_gt_reset(q, job, skip_timeout_check)) {
 			if (!xe_sched_invalidate_job(job, 2)) {
 				clear_exec_queue_banned(q);
+				/* protect concurrent page offline reasons */
+				atomic_andnot(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG,
+					      &q->ban_reason);
 				xe_gt_reset_async(q->gt);
 				goto rearm;
 			}
@@ -1787,32 +1885,15 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w)
 static void guc_exec_queue_destroy_async(struct xe_exec_queue *q)
 {
 	INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async);
-
-	/* We must block on kernel engines so slabs are empty on driver unload */
-	if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q))
-		guc_exec_queue_do_destroy(q);
-	else
-		xe_destroy_wq_queue(&q->guc->destroy_async);
-}
-
-static void __guc_exec_queue_destroy(struct xe_guc *guc, struct xe_exec_queue *q)
-{
-	/*
-	 * Might be done from within the GPU scheduler, need to do async as we
-	 * fini the scheduler when the engine is fini'd, the scheduler can't
-	 * complete fini within itself (circular dependency). Async resolves
-	 * this we and don't really care when everything is fini'd, just that it
-	 * is.
-	 */
-	guc_exec_queue_destroy_async(q);
+	xe_destroy_wq_queue(&q->guc->destroy_async);
 }
 
-static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg)
+static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg,
+						 bool bound)
 {
 	struct xe_exec_queue *q = msg->private_data;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 
-	xe_gt_assert(guc_to_gt(guc), !(q->flags & EXEC_QUEUE_FLAG_PERMANENT));
 	trace_xe_exec_queue_cleanup_entity(q);
 
 	/*
@@ -1825,10 +1906,12 @@ static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg)
 	 *   it is safe to directly destroy the exec queue on driver side, as the GuC
 	 *   will not process further requests and all resources must be cleaned up locally.
 	 */
-	if (exec_queue_registered(q) && xe_uc_fw_is_running(&guc->fw))
+	/* A wedged GuC won't answer the H2G, so tear down on the driver side. */
+	if (bound && !exec_queue_wedged(q) && exec_queue_registered(q) &&
+	    xe_uc_fw_is_running(&guc->fw))
 		disable_scheduling_deregister(guc, q);
 	else
-		__guc_exec_queue_destroy(guc, q);
+		guc_exec_queue_destroy_async(q);
 }
 
 static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q)
@@ -1836,12 +1919,13 @@ static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q)
 	return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q);
 }
 
-static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg)
+static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg,
+							 bool bound)
 {
 	struct xe_exec_queue *q = msg->private_data;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 
-	if (guc_exec_queue_allowed_to_change_state(q))
+	if (guc_exec_queue_allowed_to_change_state(q) && bound)
 		init_policies(guc, q);
 	kfree(msg);
 }
@@ -1879,13 +1963,14 @@ static void suspend_fence_signal(struct xe_exec_queue *q)
 	__suspend_fence_signal(q);
 }
 
-static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
+static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg,
+						 bool bound)
 {
 	struct xe_exec_queue *q = msg->private_data;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 
 	if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) &&
-	    exec_queue_enabled(q)) {
+	    exec_queue_enabled(q) && bound) {
 		wait_event(guc->ct.wq, vf_recovery(guc) ||
 			   ((q->guc->resume_time != RESUME_PENDING ||
 			   xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)));
@@ -1909,11 +1994,12 @@ static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg)
 	}
 }
 
-static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
+static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg,
+						bool bound)
 {
 	struct xe_exec_queue *q = msg->private_data;
 
-	if (guc_exec_queue_allowed_to_change_state(q)) {
+	if (guc_exec_queue_allowed_to_change_state(q) && bound) {
 		clear_exec_queue_suspended(q);
 		if (!exec_queue_enabled(q)) {
 			q->guc->resume_time = RESUME_PENDING;
@@ -1925,52 +2011,79 @@ static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg)
 	}
 }
 
-static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg)
+static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg,
+								  bool bound)
 {
 	struct xe_exec_queue *q = msg->private_data;
 
-	if (guc_exec_queue_allowed_to_change_state(q))
-		guc_exec_queue_send_cgp_sync(q);
+	if (guc_exec_queue_allowed_to_change_state(q) && bound)
+		guc_exec_queue_send_cgp_sync(q, 0);
 
 	kfree(msg);
 }
 
+static void __guc_exec_queue_process_msg_cgp_sync(struct xe_sched_msg *msg,
+						  bool bound)
+{
+	struct xe_exec_queue *q = msg->private_data;
+
+	/*
+	 * Replay a dynamic CGP update lost across VF migration by re-issuing the
+	 * CGP update + CGP_SYNC (re-applies the current priority from
+	 * q->multi_queue.priority).
+	 */
+	if (guc_exec_queue_allowed_to_change_state(q) && bound)
+		guc_exec_queue_send_cgp_sync(q, 0);
+}
+
 #define CLEANUP				1	/* Non-zero values to catch uninitialized msg */
 #define SET_SCHED_PROPS			2
 #define SUSPEND				3
 #define RESUME				4
 #define SET_MULTI_QUEUE_PRIORITY	5
+#define CGP_SYNC_MSG			6
 #define OPCODE_MASK	0xf
 #define MSG_LOCKED	BIT(8)
 #define MSG_HEAD	BIT(9)
+#define MSG_PM_REF	BIT(10)
 
 static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)
 {
 	struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data));
+	int idx;
+	bool pm_ref = !!(msg->opcode & MSG_PM_REF);
+	bool bound = drm_dev_enter(&xe->drm, &idx);
 
 	trace_xe_sched_msg_recv(msg);
 
-	switch (msg->opcode) {
+	switch (msg->opcode & OPCODE_MASK) {
 	case CLEANUP:
-		__guc_exec_queue_process_msg_cleanup(msg);
+		__guc_exec_queue_process_msg_cleanup(msg, bound);
 		break;
 	case SET_SCHED_PROPS:
-		__guc_exec_queue_process_msg_set_sched_props(msg);
+		__guc_exec_queue_process_msg_set_sched_props(msg, bound);
 		break;
 	case SUSPEND:
-		__guc_exec_queue_process_msg_suspend(msg);
+		__guc_exec_queue_process_msg_suspend(msg, bound);
 		break;
 	case RESUME:
-		__guc_exec_queue_process_msg_resume(msg);
+		__guc_exec_queue_process_msg_resume(msg, bound);
 		break;
 	case SET_MULTI_QUEUE_PRIORITY:
-		__guc_exec_queue_process_msg_set_multi_queue_priority(msg);
+		__guc_exec_queue_process_msg_set_multi_queue_priority(msg, bound);
+		break;
+	case CGP_SYNC_MSG:
+		__guc_exec_queue_process_msg_cgp_sync(msg, bound);
 		break;
 	default:
 		XE_WARN_ON("Unknown message type");
 	}
 
-	xe_pm_runtime_put(xe);
+	if (pm_ref)
+		xe_pm_runtime_put(xe);
+
+	if (bound)
+		drm_dev_exit(idx);
 }
 
 static const struct drm_sched_backend_ops drm_sched_ops = {
@@ -1992,6 +2105,7 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 	struct xe_guc_exec_queue *ge;
 	long timeout;
 	int err, i;
+	int max_jobs = (xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES);
 
 	xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc)));
 
@@ -2031,8 +2145,11 @@ static int guc_exec_queue_init(struct xe_exec_queue *q)
 		submit_wq = primary->guc->sched.base.submit_wq;
 	}
 
+	if (q->vm && q->vm->flags & XE_VM_FLAG_MIGRATION)
+		max_jobs = min(max_jobs, LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1);
+
 	err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops,
-			    submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64,
+			    submit_wq, max_jobs, 64,
 			    timeout, guc_to_gt(guc)->ordered_wq, NULL,
 			    ge->name, gt_to_xe(q->gt)->drm.dev);
 	if (err)
@@ -2095,10 +2212,16 @@ static void guc_exec_queue_kill(struct xe_exec_queue *q)
 static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg,
 				   u32 opcode)
 {
-	xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q)));
+	struct xe_device *xe = guc_to_xe(exec_queue_to_guc(q));
+	int idx;
+	bool bound = drm_dev_enter(&xe->drm, &idx);
 
 	INIT_LIST_HEAD(&msg->link);
 	msg->opcode = opcode & OPCODE_MASK;
+	if (bound) {
+		xe_pm_runtime_get_noresume(xe);
+		msg->opcode |= MSG_PM_REF;
+	}
 	msg->private_data = q;
 
 	trace_xe_sched_msg_add(msg);
@@ -2108,6 +2231,9 @@ static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg
 		xe_sched_add_msg_locked(&q->guc->sched, msg);
 	else
 		xe_sched_add_msg(&q->guc->sched, msg);
+
+	if (bound)
+		drm_dev_exit(idx);
 }
 
 static void guc_exec_queue_try_add_msg_head(struct xe_exec_queue *q,
@@ -2135,14 +2261,12 @@ static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q,
 #define STATIC_MSG_CLEANUP	0
 #define STATIC_MSG_SUSPEND	1
 #define STATIC_MSG_RESUME	2
+#define STATIC_MSG_CGP_SYNC	3
 static void guc_exec_queue_destroy(struct xe_exec_queue *q)
 {
 	struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;
 
-	if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q))
-		guc_exec_queue_add_msg(q, msg, CLEANUP);
-	else
-		__guc_exec_queue_destroy(exec_queue_to_guc(q), q);
+	guc_exec_queue_add_msg(q, msg, CLEANUP);
 }
 
 static int guc_exec_queue_set_priority(struct xe_exec_queue *q,
@@ -2527,13 +2651,29 @@ static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q)
 	}
 }
 
-static bool guc_exec_queue_reset_status(struct xe_exec_queue *q)
+static u64 guc_exec_queue_reset_status(struct xe_exec_queue *q)
 {
-	if (xe_exec_queue_is_multi_queue_secondary(q) &&
-	    guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q)))
-		return true;
+	/* TODO: In case of multiqueue, if a secondary queue is banned due to
+	 * page offlining, checking only the primary queue's GuC reset status
+	 * may mask the true reason or race with it.
+	 */
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		u64 status = guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q));
+
+		if (status)
+			return status;
+	}
+
+	if (exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q)) {
+		u64 reason = atomic_read_acquire(&q->ban_reason);
+
+		/* If no specific reason was recorded, default to GPU hang */
+		if (!reason)
+			reason = DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG;
+		return reason;
+	}
 
-	return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q);
+	return 0;
 }
 
 /*
@@ -2607,7 +2747,7 @@ static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
 	}
 
 	if (do_destroy)
-		__guc_exec_queue_destroy(guc, q);
+		guc_exec_queue_destroy_async(q);
 }
 
 static int guc_submit_reset_prepare(struct xe_guc *guc)
@@ -2723,6 +2863,57 @@ static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc,
 			  q->guc->id);
 	}
 
+	/*
+	 * A registration time CGP update that bailed when woken by VF recovery.
+	 * Re-register the queue.
+	 */
+	if (q->guc->multi_queue.re_register) {
+		clear_exec_queue_registered(q);
+		q->guc->multi_queue.re_register = false;
+		xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER (cgp) - guc_id=%d",
+			  q->guc->id);
+	}
+
+	/*
+	 * If a CGP update gets dropped during migration, CGP_SYNC_DONE will not
+	 * be received (sync_pending still set and this queue owns it). Recover
+	 * it the same way and clear the stuck sync_pending.
+	 */
+	if (xe_exec_queue_is_multi_queue(q)) {
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+
+		if (q == READ_ONCE(group->cgp_update_q) &&
+		    READ_ONCE(group->sync_pending)) {
+			if (q->guc->multi_queue.registering_cgp) {
+				clear_exec_queue_registered(q);
+				xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER (cgp sync) - guc_id=%d",
+					  q->guc->id);
+			} else if (q->guc->multi_queue.updating_cgp) {
+				q->guc->multi_queue.needs_cgp_sync = true;
+				xe_gt_dbg(guc_to_gt(guc), "Replay CGP_SYNC - guc_id=%d",
+					  q->guc->id);
+			}
+			q->guc->multi_queue.registering_cgp = false;
+			q->guc->multi_queue.updating_cgp = false;
+			WRITE_ONCE(group->cgp_update_q, NULL);
+			WRITE_ONCE(group->sync_pending, false);
+		}
+	}
+
+	/*
+	 * A dynamic-time CGP update that bailed when woken by VF recovery.
+	 * Replay the dynamic CGP update unless the queue is registered or being
+	 * re-registered, which re-does the CGP anyway.
+	 */
+	if (q->guc->multi_queue.re_update) {
+		q->guc->multi_queue.re_update = false;
+		if (exec_queue_registered(q)) {
+			q->guc->multi_queue.needs_cgp_sync = true;
+			xe_gt_dbg(guc_to_gt(guc), "Replay CGP_SYNC (re-update) - guc_id=%d",
+				  q->guc->id);
+		}
+	}
+
 	q->guc->resume_time = 0;
 }
 
@@ -2743,17 +2934,24 @@ static void lrc_parallel_clear(struct xe_lrc *lrc)
  * during VF resume flows. The function scans the queue state, make adjustments
  * as needed, and queues jobs / messages which replayed upon unpause.
  */
-static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q)
+static void guc_exec_queue_pause_prepare(struct xe_guc *guc, struct xe_exec_queue *q)
 {
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
-	struct xe_sched_job *job;
-	int i;
 
 	lockdep_assert_held(&guc->submission_state.lock);
 
 	/* Stop scheduling + flush any DRM scheduler operations */
 	xe_sched_submission_stop(sched);
 	cancel_delayed_work_sync(&sched->base.work_tdr);
+}
+
+static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q)
+{
+	struct xe_gpu_scheduler *sched = &q->guc->sched;
+	struct xe_sched_job *job;
+	int i;
+
+	lockdep_assert_held(&guc->submission_state.lock);
 
 	guc_exec_queue_revert_pending_state_change(guc, q);
 
@@ -2812,6 +3010,19 @@ void xe_guc_submit_pause_vf(struct xe_guc *guc)
 	xe_gt_assert(guc_to_gt(guc), vf_recovery(guc));
 
 	mutex_lock(&guc->submission_state.lock);
+	/*
+	 * Stop all schedulers before reverting any queue: in a multi-queue
+	 * group a secondary's run_job() can register the primary, which must
+	 * not race an in-progress revert.
+	 */
+	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
+		/* Prevent redundant attempts to stop parallel queues */
+		if (q->guc->id != index)
+			continue;
+
+		guc_exec_queue_pause_prepare(guc, q);
+	}
+
 	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
 		/* Prevent redundant attempts to stop parallel queues */
 		if (q->guc->id != index)
@@ -2928,6 +3139,16 @@ static void guc_exec_queue_replay_pending_state_change(struct xe_exec_queue *q)
 	struct xe_gpu_scheduler *sched = &q->guc->sched;
 	struct xe_sched_msg *msg;
 
+	if (q->guc->multi_queue.needs_cgp_sync) {
+		msg = q->guc->static_msgs + STATIC_MSG_CGP_SYNC;
+
+		xe_sched_msg_lock(sched);
+		guc_exec_queue_try_add_msg_head(q, msg, CGP_SYNC_MSG);
+		xe_sched_msg_unlock(sched);
+
+		q->guc->multi_queue.needs_cgp_sync = false;
+	}
+
 	if (q->guc->needs_cleanup) {
 		msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;
 
@@ -3172,7 +3393,7 @@ static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q)
 	trace_xe_exec_queue_deregister_done(q);
 
 	clear_exec_queue_registered(q);
-	__guc_exec_queue_destroy(guc, q);
+	guc_exec_queue_destroy_async(q);
 }
 
 int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
@@ -3360,8 +3581,9 @@ int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 le
 	reason = msg[2];
 
 	/* Unexpected failure of a hardware feature, log an actual error */
-	xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X",
-		  guc_class, instance, reason);
+	xe_log_err(gt, GUCSUBMIT, -EIO,
+		   "engine reset failed on %u:%u, reason=%#x\n",
+		   guc_class, instance, reason);
 
 	xe_gt_reset_async(gt);
 
@@ -3411,7 +3633,8 @@ int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg,
 int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
 {
 	struct xe_device *xe = guc_to_xe(guc);
-	struct xe_exec_queue *q;
+	struct xe_exec_queue_group *group;
+	struct xe_exec_queue *q, *upd_q;
 	u32 guc_id = msg[0];
 
 	if (unlikely(len < 1)) {
@@ -3428,8 +3651,20 @@ int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 le
 		return -EPROTO;
 	}
 
+	/*
+	 * The outstanding CGP update is now confirmed; clear the owning queue's
+	 * tracking so a later migration does not needlessly replay it.
+	 */
+	group = q->multi_queue.group;
+	upd_q = READ_ONCE(group->cgp_update_q);
+	if (upd_q) {
+		upd_q->guc->multi_queue.registering_cgp = false;
+		upd_q->guc->multi_queue.updating_cgp = false;
+		WRITE_ONCE(group->cgp_update_q, NULL);
+	}
+
 	/* Wakeup the serialized cgp update wait */
-	WRITE_ONCE(q->multi_queue.group->sync_pending, false);
+	WRITE_ONCE(group->sync_pending, false);
 	xe_guc_ct_wake_waiters(&guc->ct);
 
 	return 0;
@@ -3511,6 +3746,7 @@ xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q)
 	snapshot->logical_mask = q->logical_mask;
 	snapshot->width = q->width;
 	snapshot->refcount = kref_read(&q->refcount);
+	snapshot->jobcount = atomic_read(&q->job_cnt);
 	snapshot->sched_timeout = sched->base.timeout;
 	snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us;
 	snapshot->sched_props.preempt_timeout_us =
@@ -3583,6 +3819,7 @@ xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snaps
 	drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask);
 	drm_printf(p, "\tWidth: %d\n", snapshot->width);
 	drm_printf(p, "\tRef: %d\n", snapshot->refcount);
+	drm_printf(p, "\tJob count: %u\n", snapshot->jobcount);
 	drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout);
 	drm_printf(p, "\tTimeslice: %u (us)\n",
 		   snapshot->sched_props.timeslice_us);
@@ -3706,6 +3943,79 @@ bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc)
 	return false;
 }
 
+/**
+ * xe_guc_submit_active_multi_queue_lrca() - Resolve the LRCA of the active
+ * queue in the multi-queue group currently running on an engine.
+ * @guc: the &xe_guc managing the exec queues
+ * @hwe: the &xe_hw_engine whose active queue is being resolved
+ * @cur_lrca: value read from RING_CURRENT_LRCA, identifies the running group
+ * @active_id: current Active Queue ID read from CSMQDEBUG (position in group)
+ *
+ * The running group is identified by matching @cur_lrca against the group's
+ * primary LRCA; @active_id then selects the active queue within that group.
+ *
+ * Return: the LRCA of the active queue, or 0 if no matching queue is found.
+ */
+u32 xe_guc_submit_active_multi_queue_lrca(struct xe_guc *guc,
+					  struct xe_hw_engine *hwe,
+					  u32 cur_lrca, u32 active_id)
+{
+	struct xe_exec_queue *q;
+	unsigned long index;
+	u32 lrca = 0;
+
+	/*
+	 * submission_state.lock also protects exec_queue teardown: an exec
+	 * queue is removed from exec_queue_lookup before its group/primary
+	 * are freed, so any q found in the xarray below has a live group
+	 * and primary for as long as we hold the lock.
+	 */
+	guard(mutex)(&guc->submission_state.lock);
+
+	xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) {
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+		struct xe_lrc *active_lrc;
+		struct xe_lrc *primary_lrc;
+
+		if (!q->multi_queue.valid || !group || !group->primary)
+			continue;
+		/*
+		 * Multi-queue exec queues are bound to a hw engine class;
+		 * GuC dynamically schedules them onto one of the class's
+		 * physical instances, so there is no fixed queue-to-instance
+		 * mapping to filter on here.
+		 */
+		if (q->class != hwe->class)
+			continue;
+		if (q->multi_queue.pos != active_id)
+			continue;
+		/*
+		 * LRCAs are page-aligned (4K) addresses in GGTT; the low
+		 * bits reported by RING_CURRENT_LRCA are not meaningful, so
+		 * only compare bits [31:12].
+		 */
+		primary_lrc = xe_exec_queue_get_lrc(group->primary, 0);
+		if (!primary_lrc)
+			continue;
+
+		if ((xe_lrc_ggtt_addr(primary_lrc) ^ cur_lrca) & GENMASK(31, 12)) {
+			xe_lrc_put(primary_lrc);
+			continue;
+		}
+
+		active_lrc = xe_exec_queue_get_lrc(q, 0);
+		xe_lrc_put(primary_lrc);
+		if (!active_lrc)
+			continue;
+
+		lrca = xe_lrc_ggtt_addr(active_lrc);
+		xe_lrc_put(active_lrc);
+		break;
+	}
+
+	return lrca;
+}
+
 /**
  * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all
  * exec queues registered to given GuC.
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h
index ccade320dc6957..29abf07d8f041d 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit.h
@@ -11,6 +11,7 @@
 struct drm_printer;
 struct xe_exec_queue;
 struct xe_guc;
+struct xe_hw_engine;
 
 int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids);
 int xe_guc_submit_enable(struct xe_guc *guc);
@@ -55,6 +56,10 @@ void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type);
 
 bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc);
 
+u32 xe_guc_submit_active_multi_queue_lrca(struct xe_guc *guc,
+					  struct xe_hw_engine *hwe,
+					  u32 cur_lrca, u32 active_id);
+
 int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h
index 7824f61b12906b..8271702e692e85 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h
@@ -77,6 +77,8 @@ struct xe_guc_submit_exec_queue_snapshot {
 	u16 width;
 	/** @refcount: ref count of this exec queue */
 	u32 refcount;
+	/** @jobcount: job count of this exec queue */
+	u32 jobcount;
 	/**
 	 * @sched_timeout: the time after which a job is removed from the
 	 * scheduler.
diff --git a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
index 046d0655122f1f..111fef781b2abb 100644
--- a/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_guc_tlb_inval.c
@@ -98,6 +98,33 @@ static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
 	return -ECANCELED;
 }
 
+/*
+ * Emit INVAL_FULL (intra vf) to invalidate engine TLBs across all engines
+ * within the VF.
+ */
+static int send_tlb_inval_ggtt_full(struct xe_tlb_inval *tlb_inval, u32 seqno)
+{
+	struct xe_guc *guc = tlb_inval->private;
+	u32 action[] = {
+		XE_GUC_ACTION_TLB_INVALIDATION,
+		seqno,
+		MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
+	};
+	int ret;
+
+	ret = send_tlb_inval(guc, action, ARRAY_SIZE(action));
+
+	/*
+	 * send_tlb_inval() may return -ENODEV (CT disabled at cold boot) or
+	 * -ENOTRECOVERABLE (device wedged); nothing runs on the engines in
+	 * either case, so treat as cancelled.
+	 */
+	if (ret == -ENODEV || ret == -ENOTRECOVERABLE)
+		return -ECANCELED;
+
+	return ret;
+}
+
 static int send_page_reclaim(struct xe_guc *guc, u32 seqno,
 			     u64 gpu_addr)
 {
@@ -346,6 +373,7 @@ static long tlb_inval_timeout_delay(struct xe_tlb_inval *tlb_inval)
 static const struct xe_tlb_inval_ops guc_tlb_inval_asid_ops = {
 	.all = send_tlb_inval_all,
 	.ggtt = send_tlb_inval_ggtt,
+	.ggtt_full = send_tlb_inval_ggtt_full,
 	.ppgtt = send_tlb_inval_asid_ppgtt,
 	.initialized = tlb_inval_initialized,
 	.flush = tlb_inval_flush,
@@ -354,6 +382,7 @@ static const struct xe_tlb_inval_ops guc_tlb_inval_asid_ops = {
 
 static const struct xe_tlb_inval_ops guc_tlb_inval_ctx_ops = {
 	.ggtt = send_tlb_inval_ggtt,
+	.ggtt_full = send_tlb_inval_ggtt_full,
 	.all = send_tlb_inval_all,
 	.ppgtt = send_tlb_inval_ctx_ppgtt,
 	.initialized = tlb_inval_initialized,
diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h
index 31a2acb63ac34d..3dbcb1331690ea 100644
--- a/drivers/gpu/drm/xe/xe_guc_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_types.h
@@ -122,6 +122,12 @@ struct xe_guc {
 	struct xe_reg notify_reg;
 	/** @params: Control params for fw initialization */
 	u32 params[GUC_CTL_MAX_DWORDS];
+
+	/**
+	 * @pagefault_ack_counter: Counter to determine when periodically ack
+	 * pagefaults in a batch.
+	 */
+	u32 pagefault_ack_counter;
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c
index 010499766fce28..9680d2a5adec7e 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine.c
@@ -585,28 +585,102 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe,
 	xe_reg_whitelist_process_engine(hwe);
 }
 
+static u32 idledly_floor_ticks(u32 idledly_ns, u32 idledly_units_ps)
+{
+	return DIV_ROUND_DOWN_ULL((u64)idledly_ns * 1000, idledly_units_ps);
+}
+
 static void adjust_idledly(struct xe_hw_engine *hwe)
 {
 	struct xe_gt *gt = hwe->gt;
-	u32 idledly, maxcnt;
+	u32 idledly, idledly_hw, idledly_reg_val, maxcnt;
 	u32 idledly_units_ps = 8 * gt->info.timestamp_base;
 	u32 maxcnt_units_ns = 640;
-	bool inhibit_switch = 0;
+	bool inhibit_switch = false;
+	bool wa_applied = false;
+	bool clamped_below_maxcnt = false;
+
+	if ((!IS_SRIOV_VF(gt_to_xe(gt)) && XE_GT_WA(gt, 16023105232)) ||
+	    XE_GT_WA(gt, 14025941587)) {
+		u32 mincnt_idledly_ns = 5000;
+
+		/* xe_gt_clock_init() warns and zeroes timestamp_base on unknown crystal clock. */
+		if (!idledly_units_ps)
+			return;
 
-	if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_GT_WA(gt, 16023105232)) {
-		idledly = xe_mmio_read32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base));
+		idledly_reg_val = xe_mmio_read32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base));
 		maxcnt = xe_mmio_read32(&gt->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base));
 
-		inhibit_switch = idledly & INHIBIT_SWITCH_UNTIL_PREEMPTED;
-		idledly = REG_FIELD_GET(IDLE_DELAY, idledly);
-		idledly = DIV_ROUND_CLOSEST(idledly * idledly_units_ps, 1000);
+		inhibit_switch = idledly_reg_val & INHIBIT_SWITCH_UNTIL_PREEMPTED;
+		idledly = REG_FIELD_GET(IDLE_DELAY, idledly_reg_val);
+		idledly = DIV_ROUND_CLOSEST_ULL((u64)idledly * idledly_units_ps, 1000);
+		idledly_hw = idledly;
 		maxcnt = REG_FIELD_GET(IDLE_WAIT_TIME, maxcnt);
 		maxcnt *= maxcnt_units_ns;
 
-		if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) {
-			idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000),
-						    idledly_units_ps);
-			xe_mmio_write32(&gt->mmio, RING_IDLEDLY(hwe->mmio_base), idledly);
+		/*
+		 * Wa_14025941587 is applied before Wa_16023105232, which takes
+		 * priority if the two ever conflict (not expected in practice).
+		 */
+		if (XE_GT_WA(gt, 14025941587) &&
+		    idledly < mincnt_idledly_ns) {
+			idledly = mincnt_idledly_ns;
+			wa_applied = true;
+		}
+
+		if (XE_GT_WA(gt, 16023105232)) {
+			/* Clear the inhibit switch without disturbing a valid delay. */
+			if (inhibit_switch) {
+				idledly_reg_val &= ~INHIBIT_SWITCH_UNTIL_PREEMPTED;
+				wa_applied = true;
+			}
+
+			/* Warn only on the value read from hardware. */
+			xe_gt_WARN_ON(gt, idledly_hw >= maxcnt);
+
+			if (idledly >= maxcnt) {
+				/* maxcnt may be 0 if IDLE_WAIT_TIME is unprogrammed. */
+				idledly = maxcnt ? maxcnt - 1 : 0;
+				clamped_below_maxcnt = true;
+				wa_applied = true;
+			}
+		}
+
+		if (wa_applied) {
+			u32 idledly_ticks;
+
+			/*
+			 * Wa_16023105232 requires idledly < maxcnt, so floor
+			 * that clamp; otherwise round up to guarantee the
+			 * Wa_14025941587 minimum survives tick quantization.
+			 */
+			if (clamped_below_maxcnt)
+				idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps);
+			else
+				idledly_ticks = DIV_ROUND_UP_ULL((u64)idledly * 1000,
+								 idledly_units_ps);
+
+			/*
+			 * Tick quantization can still push the rounded-up value
+			 * to/above maxcnt; re-floor here so Wa_16023105232 keeps
+			 * priority even in that case.
+			 */
+			if (!clamped_below_maxcnt && XE_GT_WA(gt, 16023105232) &&
+			    (u64)idledly_ticks * idledly_units_ps >= (u64)maxcnt * 1000) {
+				xe_gt_dbg(gt, "idledly %s: %u ticks would exceed maxcnt=%u, so flooring\n",
+					  hwe->name, idledly_ticks, maxcnt);
+				idledly = maxcnt ? maxcnt - 1 : 0;
+				idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps);
+			}
+
+			idledly_reg_val &= ~IDLE_DELAY;
+			idledly_reg_val |= REG_FIELD_PREP(IDLE_DELAY, idledly_ticks);
+			xe_gt_dbg(gt, "idledly %s: set %u max=%u inh=%u ts=%u\n",
+				  hwe->name, idledly, maxcnt,
+				  !!inhibit_switch, gt->info.timestamp_base);
+			xe_mmio_write32(&gt->mmio,
+					RING_IDLEDLY(hwe->mmio_base),
+					idledly_reg_val);
 		}
 	}
 }
diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c
index de3f2aeffc3fba..5edeac961ec33e 100644
--- a/drivers/gpu/drm/xe/xe_hwmon.c
+++ b/drivers/gpu/drm/xe/xe_hwmon.c
@@ -101,11 +101,6 @@ enum sensor_attr_power {
 
 #define PWR_ATTR_TO_STR(attr)	(((attr) == hwmon_power_max) ? "PL1" : "PL2")
 
-/*
- * Timeout for power limit write mailbox command.
- */
-#define PL_WRITE_MBX_TIMEOUT_MS	(1)
-
 /* Index of memory controller in READ_THERMAL_DATA output */
 #define TEMP_INDEX_MCTRL	2
 
@@ -252,7 +247,7 @@ static int xe_hwmon_pcode_rmw_power_limit(const struct xe_hwmon *hwmon, u32 attr
 							     (channel == CHANNEL_CARD) ?
 							     WRITE_PSYSGPU_POWER_LIMIT :
 							     WRITE_PACKAGE_POWER_LIMIT, 0),
-							     val0, val1, PL_WRITE_MBX_TIMEOUT_MS);
+							     val0, val1, PCODE_DEFAULT_TIMEOUT_MS);
 	if (ret)
 		drm_dbg(&hwmon->xe->drm, "write failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n",
 			channel, val0, val1, ret);
@@ -266,7 +261,7 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg
 
 	switch (hwmon_reg) {
 	case REG_TEMP:
-		if (xe->info.platform == XE_BATTLEMAGE) {
+		if (xe->info.platform == XE_BATTLEMAGE || xe->info.platform == XE_CRESCENTISLAND) {
 			if (channel == CHANNEL_PKG)
 				return BMG_PACKAGE_TEMPERATURE;
 			else if (channel == CHANNEL_VRAM)
@@ -575,23 +570,36 @@ xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *at
 
 	mutex_unlock(&hwmon->hwmon_lock);
 
-	x = REG_FIELD_GET(PWR_LIM_TIME_X, reg_val);
-	y = REG_FIELD_GET(PWR_LIM_TIME_Y, reg_val);
+	if (hwmon->xe->info.platform >= XE_CRESCENTISLAND) {
+		/**
+		 * On CRI and newer platforms, the interval encoding changed.
+		 * The value is now stored directly in milliseconds as U5.2,
+		 * replacing the older 1.x * 2^y representation.
+		 * Bits [6:2] hold the integer part and bits [1:0] the fraction,
+		 * so convert to milliseconds by extracting integer/fractional parts.
+		 * Round fraction value to 1 when it is >= 0.5.
+		 */
+		reg_val = REG_FIELD_GET(PWR_LIM_TIME, reg_val);
+		out = (u64)((reg_val >> 2) + ((reg_val & 0x3) >= 2));
+	} else {
+		x = REG_FIELD_GET(PWR_LIM_TIME_X, reg_val);
+		y = REG_FIELD_GET(PWR_LIM_TIME_Y, reg_val);
 
-	/*
-	 * tau = (1 + (x / 4)) * power(2,y), x = bits(23:22), y = bits(21:17)
-	 *     = (4 | x) << (y - 2)
-	 *
-	 * Here (y - 2) ensures a 1.x fixed point representation of 1.x
-	 * As x is 2 bits so 1.x can be 1.0, 1.25, 1.50, 1.75
-	 *
-	 * As y can be < 2, we compute tau4 = (4 | x) << y
-	 * and then add 2 when doing the final right shift to account for units
-	 */
-	tau4 = (u64)((1 << x_w) | x) << y;
+		/*
+		 * tau = (1 + (x / 4)) * power(2,y), x = bits(23:22), y = bits(21:17)
+		 *     = (4 | x) << (y - 2)
+		 *
+		 * Here (y - 2) ensures a 1.x fixed point representation of 1.x
+		 * As x is 2 bits so 1.x can be 1.0, 1.25, 1.50, 1.75
+		 *
+		 * As y can be < 2, we compute tau4 = (4 | x) << y
+		 * and then add 2 when doing the final right shift to account for units
+		 */
+		tau4 = (u64)((1 << x_w) | x) << y;
 
-	/* val in hwmon interface units (millisec) */
-	out = mul_u64_u32_shr(tau4, SF_TIME, hwmon->scl_shift_time + x_w);
+		/* val in hwmon interface units (millisec) */
+		out = mul_u64_u32_shr(tau4, SF_TIME, hwmon->scl_shift_time + x_w);
+	}
 
 	return sysfs_emit(buf, "%llu\n", out);
 }
@@ -637,24 +645,35 @@ xe_hwmon_power_max_interval_store(struct device *dev, struct device_attribute *a
 	if (val > max_win)
 		return -EINVAL;
 
-	/* val in hw units */
-	val = DIV_ROUND_CLOSEST_ULL((u64)val << hwmon->scl_shift_time, SF_TIME) + 1;
-
-	/*
-	 * Convert val to 1.x * power(2,y)
-	 * y = ilog2(val)
-	 * x = (val - (1 << y)) >> (y - 2)
-	 */
-	if (!val) {
-		y = 0;
-		x = 0;
+	if (hwmon->xe->info.platform >= XE_CRESCENTISLAND) {
+		/**
+		 * On CRI and newer platforms, the interval encoding changed.
+		 * The value is now stored directly in milliseconds as U5.2,
+		 * replacing the older 1.x * 2^y representation.
+		 * Bits [6:2] hold the integer part and bits [1:0] the fraction,so
+		 * convert from milliseconds by shifting the value left by 2 to fit into the field.
+		 */
+		rxy = REG_FIELD_PREP(PWR_LIM_TIME, (val << 2));
 	} else {
-		y = ilog2(val);
-		x = (val - (1ul << y)) << x_w >> y;
-	}
+		/* val in hw units */
+		val = DIV_ROUND_CLOSEST_ULL((u64)val << hwmon->scl_shift_time, SF_TIME) + 1;
 
-	rxy = REG_FIELD_PREP(PWR_LIM_TIME_X, x) |
-			       REG_FIELD_PREP(PWR_LIM_TIME_Y, y);
+		/*
+		 * Convert val to 1.x * power(2,y)
+		 * y = ilog2(val)
+		 * x = (val - (1 << y)) >> (y - 2)
+		 */
+		if (!val) {
+			y = 0;
+			x = 0;
+		} else {
+			y = ilog2(val);
+			x = (val - (1ul << y)) << x_w >> y;
+		}
+
+		rxy = REG_FIELD_PREP(PWR_LIM_TIME_X, x) |
+				       REG_FIELD_PREP(PWR_LIM_TIME_Y, y);
+	}
 
 	guard(xe_pm_runtime)(hwmon->xe);
 
diff --git a/drivers/gpu/drm/xe/xe_log.c b/drivers/gpu/drm/xe/xe_log.c
new file mode 100644
index 00000000000000..29eb16db332057
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_log.c
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <kunit/static_stub.h>
+#include <kunit/visibility.h>
+
+#include "abi/xe_log_abi.h"
+
+#include "xe_device.h"
+#include "xe_log.h"
+#include "xe_pci_types.h"
+#include "xe_printk.h"
+
+static void log_emit_cper(struct pci_dev *pdev, int cper_sev, enum xe_sigid sigid,
+			  u32 component, u32 location, const void *data, size_t len,
+			  struct va_format *vaf)
+{
+	KUNIT_STATIC_STUB_REDIRECT(log_emit_cper, pdev, cper_sev, sigid,
+				   component, location, data, len, vaf);
+	/* TODO */
+}
+
+static const char *log_unknown_component_prefix(u32 component)
+{
+	u32 class = FIELD_GET(XE_LOG_COMPONENT_CLASS_MASK, component);
+	u32 type = FIELD_GET(XE_LOG_COMPONENT_TYPE_MASK, component);
+
+	WARN(IS_ENABLED(CONFIG_DRM_XE_DEBUG), "LOG: unrecognized component %u.%u\n", class, type);
+	switch (class) {
+#define MAKE_XE_LOG_COMPONENT_CLASS_PREFIX(_CLASS) \
+	case XE_LOG_COMPONENT_CLASS_##_CLASS: return #_CLASS "? ";
+	MAKE_XE_LOG_COMPONENT_CLASS_PREFIX(SYSTEM)
+	MAKE_XE_LOG_COMPONENT_CLASS_PREFIX(DRIVER)
+	MAKE_XE_LOG_COMPONENT_CLASS_PREFIX(FEATURE)
+	MAKE_XE_LOG_COMPONENT_CLASS_PREFIX(FIRMWARE)
+	MAKE_XE_LOG_COMPONENT_CLASS_PREFIX(HARDWARE)
+#undef MAKE_XE_LOG_COMPONENT_CLASS_PREFIX
+	}
+	return "COMP? ";
+}
+
+static const char *log_component_prefix(u32 component)
+{
+	switch (component) {
+#define MAKE_XE_LOG_COMPONENT_CASE_PREFIX(_CLASS, _ID, _TAG, _SIG, _NAME) \
+	case XE_LOG_COMPONENT_##_TAG: return #_TAG ": ";
+	DEFINE_XE_LOG_COMPONENTS(MAKE_XE_LOG_COMPONENT_CASE_PREFIX)
+#undef MAKE_XE_LOG_COMPONENT_CASE_PREFIX
+	}
+
+	return component ? log_unknown_component_prefix(component) : "";
+}
+
+static bool allowed_tile_id(struct xe_device *xe, u8 tile_id)
+{
+	return tile_id < 1 + xe->desc->max_remote_tiles;
+}
+
+static bool allowed_gt_id(struct xe_device *xe, u8 gt_id)
+{
+	return gt_id < (1 + xe->desc->max_remote_tiles) * xe->desc->max_gt_per_tile;
+}
+
+static u8 gt_id_to_tile_id(struct xe_device *xe, u8 gt_id)
+{
+	return gt_id / xe->desc->max_gt_per_tile;
+}
+
+static const char *location_suffix(bool valid)
+{
+	return valid ? ":" : "?";
+}
+
+static const char *log_location_prefix(struct pci_dev *pdev, u32 location, char *buf, size_t size)
+{
+	u32 type = FIELD_GET(XE_LOG_LOCATION_TYPE_MASK, location);
+	u32 id = FIELD_GET(XE_LOG_LOCATION_ID_MASK, location);
+
+	if (!location || type == XE_LOG_LOCATION_TYPE_DEVICE) {
+		if (id)
+			goto unrecognized;
+		strscpy(buf, "", size);
+	} else if (type == XE_LOG_LOCATION_TYPE_TILE) {
+		struct xe_device *xe = xe_any_to_xe(pdev);
+		bool valid = xe ? allowed_tile_id(xe, id) : false;
+		const char *pad = location_suffix(valid);
+
+		pci_WARN(pdev, !valid && IS_ENABLED(CONFIG_DRM_XE_DEBUG),
+			 "LOG: invalid tile identifier: %u\n", id);
+		snprintf(buf, size, "Tile%u%s ", id, pad);
+	} else if (type == XE_LOG_LOCATION_TYPE_GT) {
+		struct xe_device *xe = xe_any_to_xe(pdev);
+		bool valid = xe ? allowed_gt_id(xe, id) : false;
+		const char *pad = location_suffix(valid);
+		u8 tile_id = xe ? gt_id_to_tile_id(xe, id) : 0;
+
+		pci_WARN(pdev, !valid && IS_ENABLED(CONFIG_DRM_XE_DEBUG),
+			 "LOG: invalid GT identifier: %u\n", id);
+		snprintf(buf, size, "Tile%u%s GT%u%s ", tile_id, pad, id, pad);
+	} else {
+		goto unrecognized;
+	}
+
+	return buf;
+
+unrecognized:
+	pci_WARN(pdev, IS_ENABLED(CONFIG_DRM_XE_DEBUG),
+		 "LOG: unrecognized location %u.%u\n", type, id);
+	snprintf(buf, size, "LOC%u.%u? ", type, id);
+	return buf;
+}
+
+static bool is_hw_sigid(enum xe_sigid sigid)
+{
+	return (int)sigid >= INTEL_SIGID_GPU_XE_HARDWARE_START;
+}
+
+static bool is_sev_error(int cper_sev)
+{
+	return cper_sev != CPER_SEV_INFORMATIONAL;
+}
+
+static const char *log_hwe_prefix(int cper_sev, enum xe_sigid sigid)
+{
+	return is_sev_error(cper_sev) && is_hw_sigid(sigid) ? HW_ERR : "";
+}
+
+static const char *log_sev_prefix(int cper_sev)
+{
+	switch (cper_sev) {
+	case CPER_SEV_FATAL:
+		return "FATAL ";
+	case CPER_SEV_RECOVERABLE:
+		return "";
+	case CPER_SEV_CORRECTED:
+		return "CORRECTED ";
+	case CPER_SEV_INFORMATIONAL:
+		return "";
+	default:
+		WARN(IS_ENABLED(CONFIG_DRM_XE_DEBUG), "LOG: unknown severity %d\n", cper_sev);
+		return "";
+	}
+}
+
+#define __LOG_DRM_PRINTK_FMT(fmt, args...)	"[drm] " fmt, ##args
+#define __LOG_DRM_PRINTK_ERR_FMT(fmt, args...)	__LOG_DRM_PRINTK_FMT("*ERROR* " fmt, args)
+
+static void log_dmesg_vprintk(struct pci_dev *pdev, int cper_sev, struct va_format *vaf)
+{
+	KUNIT_STATIC_STUB_REDIRECT(log_dmesg_vprintk, pdev, cper_sev, vaf);
+
+	if (cper_sev == CPER_SEV_INFORMATIONAL)
+		pci_info(pdev, __LOG_DRM_PRINTK_FMT("%pV", vaf));
+	else
+		pci_err(pdev, __LOG_DRM_PRINTK_ERR_FMT("%pV", vaf));
+}
+
+static void log_dmesg_printf(struct pci_dev *pdev, int cper_sev, const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, fmt);
+	vaf.fmt = fmt;
+	vaf.va = &args;
+
+	log_dmesg_vprintk(pdev, cper_sev, &vaf);
+
+	va_end(args);
+}
+
+static void log_emit_dmesg(struct pci_dev *pdev, int cper_sev, enum xe_sigid sigid,
+			   u32 component, u32 location, const void *data, size_t len,
+			   struct va_format *vaf)
+{
+	char buf[32];
+	const char *loc_prefix = log_location_prefix(pdev, location, buf, sizeof(buf));
+	const char *comp_prefix = log_component_prefix(component);
+	const char *hwe_prefix = log_hwe_prefix(cper_sev, sigid);
+	const char *sev_prefix = log_sev_prefix(cper_sev);
+
+	if (IS_ERR(data))
+		log_dmesg_printf(pdev, cper_sev, "SIGID=%u %s(%pe) %s%s%s%pV",
+				 sigid, sev_prefix, data, hwe_prefix,
+				 loc_prefix, comp_prefix, vaf);
+	else if (data && len)
+		log_dmesg_printf(pdev, cper_sev, "SIGID=%u %s(%*phN) %s%s%s%pV",
+				 sigid, sev_prefix, (int)len, data, hwe_prefix,
+				 loc_prefix, comp_prefix, vaf);
+	else
+		log_dmesg_printf(pdev, cper_sev, "SIGID=%u %s%s%s%s%pV",
+				 sigid, sev_prefix, hwe_prefix,
+				 loc_prefix, comp_prefix, vaf);
+}
+
+/**
+ * __xe_log_emit() - Emit a structured SIGID log entry
+ * @pdev: the &pci_dev device
+ * @cper_sev: CPER severity (CPER_SEV_FATAL, CPER_SEV_RECOVERABLE, ...)
+ * @sigid: signature identifier, see &enum xe_sigid
+ * @component: component identifier
+ * @location: location details of the @component
+ * @data: pointer to the additional details, or ERR_PTR, or NULL if not applicable
+ * @len: length of the @data in bytes, or 0 if not applicable
+ * @fmt: printf-style format string
+ * @...: format arguments
+ *
+ * Emits a dmesg line that includes a single stable, machine-matchable token
+ * ``SIGID=<n>`` followed by the optional severity token (like ``FATAL``) and,
+ * when @data pointer is set, either the error printed with %pe or a packed hex
+ * dump of the @data binary blob. The dmesg line will also include printf-style
+ * text message.
+ *
+ * Note that the full dmesg line, with the free text message, is only a debugging
+ * aid, not an interface! Only the ``SIGID=<n>`` token is stable there.
+ * The durable machine record is the CPER carrying the same SIGID.
+ *
+ * Note: generation of the CPER record is a planned follow-up.
+ *
+ * Examples::
+ *
+ *   <3> xe 0000:03:00.0: [drm] *ERROR* SIGID=104 FATAL (-EPROTO) Invalid GuC reply
+ *   <3> xe 0000:03:00.0: [drm] *ERROR* SIGID=106 (-ETIMEDOUT) Engine 'rcs0' hung
+ *   <6> xe 0000:03:00.0: [drm] SIGID=103 In survivability mode
+ */
+void __xe_log_emit(struct pci_dev *pdev, int cper_sev, enum xe_sigid sigid,
+		   u32 component, u32 location, const void *data, size_t len,
+		   const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, fmt);
+	vaf.fmt = fmt;
+	vaf.va = &args;
+
+	log_emit_dmesg(pdev, cper_sev, sigid, component, location, data, len, &vaf);
+	log_emit_cper(pdev, cper_sev, sigid, component, location, data, len, &vaf);
+
+	va_end(args);
+}
+
+#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
+#include "tests/xe_log_kunit.c"
+#endif
diff --git a/drivers/gpu/drm/xe/xe_log.h b/drivers/gpu/drm/xe/xe_log.h
new file mode 100644
index 00000000000000..bb1185143589c7
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_log.h
@@ -0,0 +1,194 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_LOG_H_
+#define _XE_LOG_H_
+
+#include <linux/cper.h>
+#include <linux/err.h>
+
+#include "abi/xe_log_abi.h"
+#include "abi/xe_sigid_abi.h"
+#include "xe_any.h"
+
+struct pci_dev;
+
+__printf(8, 9)
+void __xe_log_emit(struct pci_dev *pdev, int cper_sev, enum xe_sigid sigid,
+		   u32 component, u32 location, const void *data, size_t len,
+		   const char *fmt, ...);
+
+#define __xe_log_const_sev_to_level(sev) \
+	(__builtin_constant_p(sev) ? (sev) == CPER_SEV_INFORMATIONAL ? KERN_INFO : KERN_ERR : NULL)
+
+#define __xe_log_emit_printk_index(level, fmt) \
+	dev_printk_index_emit(level, "%s SIGID=%u %s" fmt);
+
+#define xe_log_emit(pdev, sev, sig, comp, loc, data, len, fmt, args...) do {		\
+	__xe_log_emit_printk_index(__xe_log_const_sev_to_level(sev), fmt);		\
+	__xe_log_emit((pdev), (sev), (sig), (comp), (loc), (data), (len), fmt, ##args); \
+} while (0)
+
+#define xe_log_emit_fatal(pdev, sig, comp, loc, data, len, fmt, args...) \
+	xe_log_emit((pdev), CPER_SEV_FATAL, (sig), (comp), (loc), \
+		    (data), (len), fmt, ##args)
+
+#define xe_log_emit_recoverable(pdev, sig, comp, loc, data, len, fmt, args...) \
+	xe_log_emit((pdev), CPER_SEV_RECOVERABLE, (sig), (comp), (loc), \
+		    (data), (len), fmt, ##args)
+
+#define xe_log_emit_corrected(pdev, sig, comp, loc, data, len, fmt, args...) \
+	xe_log_emit((pdev), CPER_SEV_CORRECTED, (sig), (comp), (loc), \
+		    (data), (len), fmt, ##args)
+
+#define xe_log_emit_info(pdev, sig, comp, loc, data, len, fmt, args...) \
+	xe_log_emit((pdev), CPER_SEV_INFORMATIONAL, (sig), (comp), (loc), \
+		    (data), (len), fmt, ##args)
+
+#define xe_log_location_type(any)							\
+	_Generic((any),									\
+		 struct xe_gt * : XE_LOG_LOCATION_TYPE_GT,				\
+		 const struct xe_gt * : XE_LOG_LOCATION_TYPE_GT,			\
+		 struct xe_tile * : XE_LOG_LOCATION_TYPE_TILE,				\
+		 const struct xe_tile * : XE_LOG_LOCATION_TYPE_TILE,			\
+		 struct xe_device * : XE_LOG_LOCATION_TYPE_DEVICE,			\
+		 const struct xe_device * : XE_LOG_LOCATION_TYPE_DEVICE,		\
+		 struct pci_dev * : XE_LOG_LOCATION_TYPE_DEVICE,			\
+		 struct device * : XE_LOG_LOCATION_TYPE_DEVICE)
+
+#define xe_log_location(any) \
+	PREP_XE_LOG_LOCATION(xe_log_location_type(any), xe_any_id(any))
+
+/**
+ * xe_log_from() - Emit a structured SIGID log entry using @any pointer as location.
+ * @any: the &xe_device or &xe_tile or &xe_gt pointer this report relates to
+ * @cper_sev: CPER severity (CPER_SEV_FATAL, CPER_SEV_RECOVERABLE, ...)
+ * @sigid: signature identifier, see &enum xe_sigid
+ * @component: component identifer
+ * @data: pointer to the additional details, or ERR_PTR, or NULL if not applicable
+ * @len: length of the @data in bytes, or 0 if not applicable
+ * @fmt: printf-style format string
+ * @args: arguments for the @fmt format string
+ *
+ * The location used to emit SIGID entry will be based on the @any pointer type.
+ * See xe_log_emit() for more details.
+ */
+#define xe_log_from(any, cper_sev, sigid, component, data, len, fmt, args...) do {	\
+	typeof(any) ___any = (any);							\
+	xe_log_emit(xe_any_to_pdev(___any), (cper_sev), (sigid), (component),		\
+		    xe_log_location(___any), (data), (len), fmt, ##args);		\
+} while (0)
+
+#define xe_log_from_fatal(any, sig, comp, data, len, fmt, args...) \
+	xe_log_from((any), CPER_SEV_FATAL, (sig), (comp), \
+		    (data), (len), fmt, ##args)
+
+#define xe_log_from_recoverable(any, sig, comp, data, len, fmt, args...) \
+	xe_log_from((any), CPER_SEV_RECOVERABLE, (sig), (comp), \
+		    (data), (len), fmt, ##args)
+
+#define xe_log_from_corrected(any, sig, comp, data, len, fmt, args...) \
+	xe_log_from((any), CPER_SEV_CORRECTED, (sig), (comp), \
+		    (data), (len), fmt, ##args)
+
+#define xe_log_from_info(any, sig, comp, data, len, fmt, args...) \
+	xe_log_from((any), CPER_SEV_INFORMATIONAL, (sig), (comp), \
+		    (data), (len), fmt, ##args)
+
+/**
+ * xe_log_comp() - Emit a structured SIGID log entry on the component behalf.
+ * @any: the &xe_device or &xe_tile or &xe_gt pointer this report relates to
+ * @cper_sev: CPER severity (CPER_SEV_FATAL, CPER_SEV_RECOVERABLE, ...)
+ * @TAG: the component tag to use
+ * @data: pointer to the additional details, or ERR_PTR, or NULL if not applicable
+ * @len: length of the @data in bytes, or 0 if not applicable
+ * @fmt: printf-style free text format string (not a stable interface)
+ * @args: arguments for the @fmt format string
+ *
+ * The SIGID will be determined from the component's @TAG.
+ * The component identifier will be determined from the component's @TAG.
+ * The location used to emit SIGID entry will be based on the @any pointer type.
+ */
+#define xe_log_comp(any, cper_sev, TAG, data, len, fmt, args...) \
+	xe_log_from((any), (cper_sev), (int)XE_LOG_COMPONENT_##TAG##_SIGID, \
+		    XE_LOG_COMPONENT_##TAG, (data), (len), fmt, ##args)
+
+#define xe_log_comp_fatal(any, TAG, data, len, fmt, args...) \
+	xe_log_comp((any), CPER_SEV_FATAL, TAG, (data), (len), fmt, ##args)
+
+#define xe_log_comp_recoverable(any, TAG, data, len, fmt, args...) \
+	xe_log_comp((any), CPER_SEV_RECOVERABLE, TAG, (data), (len), fmt, ##args)
+
+#define xe_log_comp_corrected(any, TAG, data, len, fmt, args...) \
+	xe_log_comp((any), CPER_SEV_CORRECTED, TAG, (data), (len), fmt, ##args)
+
+#define xe_log_comp_info(any, TAG, data, len, fmt, args...) \
+	xe_log_comp((any), CPER_SEV_INFORMATIONAL, TAG, (data), (len), fmt, ##args)
+
+/**
+ * xe_log_err() - Emit a structured SIGID error log entry on the component behalf.
+ * @any: the &xe_device or &xe_tile or &xe_gt pointer this report relates to
+ * @TAG: the component tag to use
+ * @err: negative errno for the failing operation, or 0 if not applicable
+ * @fmt: printf-style free text format string (not a stable interface)
+ * @args: arguments for the @fmt format string
+ *
+ * The log entry will be emitted with @CPER_SEV_RECOVERABLE severity.
+ */
+#define xe_log_err(any, TAG, err, fmt, args...) \
+	xe_log_comp_recoverable((any), TAG, ERR_PTR(err), 0, fmt, ##args)
+
+/**
+ * xe_log_err_fatal() - Emit a structured SIGID error log entry on the component behalf.
+ * @any: the &xe_device or &xe_tile or &xe_gt pointer this report relates to
+ * @TAG: the component tag to use
+ * @err: negative errno for the failing operation, or 0 if not applicable
+ * @fmt: printf-style free text format string (not a stable interface)
+ * @args: arguments for the @fmt format string
+ *
+ * The log entry will be emitted with @CPER_SEV_FATAL severity.
+ */
+#define xe_log_err_fatal(any, TAG, err, fmt, args...) \
+	xe_log_comp_fatal((any), TAG, ERR_PTR(err), 0, fmt, ##args)
+
+/**
+ * xe_log_err_corrected() - Emit a structured SIGID error log entry on the component behalf.
+ * @any: the &xe_device or &xe_tile or &xe_gt pointer this report relates to
+ * @TAG: the component tag to use
+ * @err: negative errno for the failing operation, or 0 if not applicable
+ * @fmt: printf-style free text format string (not a stable interface)
+ * @args: arguments for the @fmt format string
+ *
+ * The log entry will be emitted with @CPER_SEV_CORRECTED severity.
+ */
+#define xe_log_err_corrected(any, TAG, err, fmt, args...) \
+	xe_log_comp_corrected((any), TAG, ERR_PTR(err), 0, fmt, ##args)
+
+/**
+ * xe_log_err_info() - Emit a structured SIGID error log entry on the component behalf.
+ * @any: the &xe_device or &xe_tile or &xe_gt pointer this report relates to
+ * @TAG: the component tag to use
+ * @err: negative errno for the failing operation, or 0 if not applicable
+ * @fmt: printf-style free text format string (not a stable interface)
+ * @args: arguments for the @fmt format string
+ *
+ * The log entry will be emitted with @CPER_SEV_INFORMATIONAL severity.
+ */
+#define xe_log_err_info(any, TAG, err, fmt, args...) \
+	xe_log_comp_info((any), TAG, ERR_PTR(err), 0, fmt, ##args)
+
+/**
+ * xe_log_info() - Emit a structured SIGID information log entry on the component behalf.
+ * @any: the &xe_device or &xe_tile or &xe_gt pointer this report relates to
+ * @TAG: the component tag to use
+ * @fmt: printf-style free text format string (not a stable interface)
+ * @args: arguments for the @fmt format string
+ *
+ * The log entry will be emitted with @CPER_SEV_INFORMATIONAL severity.
+ */
+#define xe_log_info(any, TAG, fmt, args...) \
+	xe_log_err_info((any), TAG, 0, fmt, ##args)
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index 35b4e8289b5fee..bf5c2351aed3d1 100644
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -706,6 +706,7 @@ u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc)
 #define LRC_CTX_JOB_TIMESTAMP_OFFSET 512
 #define LRC_ENGINE_ID_PPHWSP_OFFSET 1024
 #define LRC_PARALLEL_PPHWSP_OFFSET 2048
+#define LRC_ULLS_PPHWSP_OFFSET 2048	/* Mutually exclusive with parallel */
 
 #define LRC_SEQNO_OFFSET 0
 #define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8)
@@ -768,6 +769,12 @@ static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc)
 	return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET;
 }
 
+static u32 __xe_lrc_ulls_offset(struct xe_lrc *lrc)
+{
+	/* The ulls is stored in the driver-defined portion of PPHWSP */
+	return xe_lrc_pphwsp_offset(lrc) + LRC_ULLS_PPHWSP_OFFSET;
+}
+
 static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc)
 {
 	return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32);
@@ -835,6 +842,7 @@ DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo)
 DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo)
 DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo)
 DECL_MAP_ADDR_HELPERS(parallel, lrc->bo)
+DECL_MAP_ADDR_HELPERS(ulls, lrc->bo)
 DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo)
 DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo)
 DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo)
@@ -1486,6 +1494,36 @@ void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_pri
 	lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority));
 }
 
+static void xe_lrc_set_gpgpu_preemption_level(struct xe_lrc *lrc, struct xe_gt *gt)
+{
+	enum xe_gpgpu_preempt_level level = gt->gpgpu_preemption_level;
+	u32 level_bits;
+	u32 val;
+
+	if (level == XE_GPGPU_PREEMPT_DEFAULT)
+		return;
+
+	switch (level) {
+	case XE_GPGPU_PREEMPT_MID_THREAD:
+		level_bits = PREEMPT_GPGPU_MID_THREAD_LEVEL;
+		break;
+	case XE_GPGPU_PREEMPT_THREAD_GROUP:
+		level_bits = PREEMPT_GPGPU_THREAD_GROUP_LEVEL;
+		break;
+	case XE_GPGPU_PREEMPT_COMMAND:
+		level_bits = PREEMPT_GPGPU_COMMAND_LEVEL;
+		break;
+	default:
+		xe_gt_WARN(gt, true, "Invalid GPGPU preemption level: %d\n", level);
+		return;
+	}
+
+	val = xe_lrc_read_ctx_reg(lrc, CTX_CS_CHICKEN1);
+	val &= ~PREEMPT_GPGPU_LEVEL_MASK;
+	val |= REG_MASKED_FIELD(PREEMPT_GPGPU_LEVEL_MASK, level_bits);
+	xe_lrc_write_ctx_reg(lrc, CTX_CS_CHICKEN1, val);
+}
+
 static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm,
 			   void *replay_state, u16 msix_vec, u32 init_flags)
 {
@@ -1589,6 +1627,9 @@ static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct
 	if (xe->info.has_asid && vm)
 		xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid);
 
+	if (GRAPHICS_VER(xe) >= 20 && hwe->class == XE_ENGINE_CLASS_RENDER)
+		xe_lrc_set_gpgpu_preemption_level(lrc, gt);
+
 	lrc->desc = LRC_VALID;
 	lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT);
 	/* TODO: Priority */
@@ -1772,6 +1813,26 @@ void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail)
 		xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail);
 }
 
+/**
+ * xe_lrc_ring_tail_ggtt_addr() - Saved ring tail GGTT address
+ * @lrc: Pointer to the lrc.
+ *
+ * GGTT address of the ring tail as saved for this context - in the indirect
+ * ring state on platforms which have it, otherwise in the context image.
+ * This is what a context restore loads the tail register from, so a ring
+ * which advances the tail register itself must keep this in sync.
+ *
+ * Returns: saved ring tail GGTT address
+ */
+u32 xe_lrc_ring_tail_ggtt_addr(struct xe_lrc *lrc)
+{
+	if (xe_lrc_has_indirect_ring_state(lrc))
+		return __xe_lrc_indirect_ring_ggtt_addr(lrc) +
+			INDIRECT_CTX_RING_TAIL * sizeof(u32);
+
+	return __xe_lrc_regs_ggtt_addr(lrc) + CTX_RING_TAIL * sizeof(u32);
+}
+
 u32 xe_lrc_ring_tail(struct xe_lrc *lrc)
 {
 	if (xe_lrc_has_indirect_ring_state(lrc))
@@ -1951,6 +2012,51 @@ static u32 xe_lrc_engine_id(struct xe_lrc *lrc)
 	return xe_map_read32(xe, &map);
 }
 
+#define semaphore_offset(seqno) \
+	(sizeof(u32) * ((seqno) % LRC_MIGRATION_ULLS_SEMAPHORE_COUNT))
+
+/**
+ * xe_lrc_ulls_semaphore_ggtt_addr() - ULLS semaphore GGTT address
+ * @lrc: Pointer to the lrc.
+ * @seqno: seqno of current job.
+ *
+ * Calculate ULLS semaphore GGTT address based on input seqno
+ *
+ * Returns: ULLS semaphore GGTT address
+ */
+u32 xe_lrc_ulls_semaphore_ggtt_addr(struct xe_lrc *lrc, u32 seqno)
+{
+	xe_assert(lrc_to_xe(lrc), semaphore_offset(seqno) <
+		  LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET);
+
+	return __xe_lrc_ulls_ggtt_addr(lrc) + semaphore_offset(seqno);
+}
+
+/**
+ * xe_lrc_set_ulls_semaphore() - Set ULLS semaphore
+ * @lrc: Pointer to the lrc.
+ * @seqno: seqno of current job.
+ *
+ * Set ULLS semaphore based on input seqno
+ */
+void xe_lrc_set_ulls_semaphore(struct xe_lrc *lrc, u32 seqno)
+{
+	struct xe_device *xe = lrc_to_xe(lrc);
+	struct iosys_map map = __xe_lrc_ulls_map(lrc);
+
+	xe_assert(xe, semaphore_offset(seqno) <
+		  LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET);
+
+	/*
+	 * The ring contents this semaphore releases are ordered by the
+	 * xe_device_wmb() at the end of xe_lrc_write_ring().
+	 */
+	iosys_map_incr(&map, semaphore_offset(seqno));
+	xe_map_write32(xe, &map, LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL);
+
+	xe_device_wmb(xe);	/* Flush write to hardware */
+}
+
 static int instr_dw(u32 cmd_header)
 {
 	/* GFXPIPE "SINGLE_DW" opcodes are a single dword */
@@ -2672,7 +2778,7 @@ static u64 get_queue_timestamp(struct xe_hw_engine *hwe)
 				   RING_QUEUE_TIMESTAMP(hwe->mmio_base));
 }
 
-static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe)
+u32 xe_lrc_get_multi_queue_active_queue_id(struct xe_hw_engine *hwe)
 {
 	u32 val = xe_mmio_read32(&hwe->gt->mmio,
 				 RING_CSMQDEBUG(hwe->mmio_base));
@@ -2706,14 +2812,14 @@ static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc)
 	if (!hwe)
 		return xe_lrc_queue_timestamp(lrc);
 
-	if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos)
+	if (xe_lrc_get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos)
 		return xe_lrc_queue_timestamp(lrc);
 
 	/* queue is active, so store the queue timestamp register */
 	reg_queue_ts = get_queue_timestamp(hwe);
 
 	/* double check queue and primary queue are both still active */
-	if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos ||
+	if (xe_lrc_get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos ||
 	    !context_active(primary_lrc))
 		return xe_lrc_queue_timestamp(lrc);
 
diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h
index 7be5e3da8bc800..f2e829e621684d 100644
--- a/drivers/gpu/drm/xe/xe_lrc.h
+++ b/drivers/gpu/drm/xe/xe_lrc.h
@@ -112,6 +112,7 @@ u32 xe_lrc_regs_offset(struct xe_lrc *lrc);
 
 void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail);
 u32 xe_lrc_ring_tail(struct xe_lrc *lrc);
+u32 xe_lrc_ring_tail_ggtt_addr(struct xe_lrc *lrc);
 void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head);
 u32 xe_lrc_ring_head(struct xe_lrc *lrc);
 u32 xe_lrc_ring_space(struct xe_lrc *lrc);
@@ -127,6 +128,9 @@ void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe);
 void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
 					    u32 *regs);
 
+u32 xe_lrc_ulls_semaphore_ggtt_addr(struct xe_lrc *lrc, u32 seqno);
+void xe_lrc_set_ulls_semaphore(struct xe_lrc *lrc, u32 seqno);
+
 u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr);
 void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val);
 
@@ -158,6 +162,7 @@ int xe_lrc_lookup_default_reg_value(struct xe_gt *gt,
 u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs);
 
 void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority);
+u32 xe_lrc_get_multi_queue_active_queue_id(struct xe_hw_engine *hwe);
 
 struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc);
 void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot);
diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h
index 53ef48feebfc90..b18daf1092e6c7 100644
--- a/drivers/gpu/drm/xe/xe_lrc_types.h
+++ b/drivers/gpu/drm/xe/xe_lrc_types.h
@@ -12,6 +12,10 @@
 
 struct xe_bo;
 
+#define LRC_MIGRATION_ULLS_SEMAPHORE_COUNT	64	/* Must be pow2 */
+#define LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR	0
+#define LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL	1
+
 /**
  * struct xe_lrc - Logical ring context (LRC) and submission ring object
  */
diff --git a/drivers/gpu/drm/xe/xe_mert.c b/drivers/gpu/drm/xe/xe_mert.c
index f637df95418b03..32700c19a1df31 100644
--- a/drivers/gpu/drm/xe/xe_mert.c
+++ b/drivers/gpu/drm/xe/xe_mert.c
@@ -77,7 +77,7 @@ static void mert_handle_cat_error(struct xe_device *xe)
 		xe_device_declare_wedged(xe);
 		break;
 	case CATERR_LMTT_FAULT:
-		xe_sriov_dbg(xe, "MERT: CAT_ERR: VF%u LMTT fault!\n", vfid);
+		xe_sriov_dbg_ratelimited(xe, "MERT: CAT_ERR: VF%u LMTT fault!\n", vfid);
 		/* XXX: track/report malicious VF activity */
 		break;
 	default:
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index f79d0047bec69b..0dfc54ba3b8f9e 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -8,6 +8,7 @@
 #include <linux/bitfield.h>
 #include <linux/sizes.h>
 
+#include <drm/drm_drv.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_pagemap.h>
 #include <drm/ttm/ttm_tt.h>
@@ -22,6 +23,7 @@
 #include "xe_assert.h"
 #include "xe_bb.h"
 #include "xe_bo.h"
+#include "xe_configfs.h"
 #include "xe_exec_queue.h"
 #include "xe_ggtt.h"
 #include "xe_gt.h"
@@ -32,6 +34,7 @@
 #include "xe_mem_pool.h"
 #include "xe_mocs.h"
 #include "xe_pat.h"
+#include "xe_pm.h"
 #include "xe_printk.h"
 #include "xe_pt.h"
 #include "xe_res_cursor.h"
@@ -45,6 +48,155 @@
 #include "xe_vm.h"
 #include "xe_vram.h"
 
+/**
+ * DOC: ULLS (Ultra Low Latency Submission) for migration jobs
+ *
+ * Migration jobs issued on behalf of GPU page faults and SVM prefetches sit
+ * directly in the critical path of a stalled GPU workload. The dominant cost
+ * of such a job is not the copy or clear itself but the submission latency:
+ * the H2G round trip to GuC, the GuC scheduling decision, and the hardware
+ * context switch required to place the migration LRC on an engine.
+ *
+ * ULLS removes that cost by keeping the migration context resident and
+ * *running* on the hardware engine across jobs. Instead of the ring going
+ * empty and the context being switched out between jobs, the tail of every
+ * ULLS job parks the engine on a semaphore wait for the *next* job's
+ * semaphore, and then advances the ring tail itself. Submitting the next job
+ * therefore costs the CPU a single write to signal that semaphore - no H2G,
+ * no GuC round trip, no context switch, no MMIO.
+ *
+ * Requirements
+ * ------------
+ *
+ * ULLS is only used on dGFX platforms with USM support, where a hardware
+ * engine is reserved exclusively for migration jobs. Because the engine
+ * spins on a semaphore while ULLS is active, it cannot be shared with
+ * user submissions.
+ *
+ * ULLS can be disabled by setting the ``migrate_ulls_period_ms`` configfs
+ * attribute to 0. Otherwise, the same attribute controls how long ULLS
+ * remains active before exiting, in milliseconds.
+ *
+ * Fixed size jobs
+ * ---------------
+ *
+ * A job updates the ring tail to cover its successor, but it is emitted long
+ * before that successor exists, so it can not know how much ring the
+ * successor will occupy. Every ULLS job is therefore padded out to exactly
+ * ULLS_JOB_SIZE_BYTES, which lets the next tail be computed arithmetically
+ * from where the current job started.
+ *
+ * This is why the shorter jobs still have to reach the same size: the "last"
+ * job skips the batch buffers and the postamble, and pads the difference with
+ * MI_NOOP. The "first" job is not covered by any predecessor's tail update
+ * and so is unconstrained, but is padded anyway to keep the arithmetic
+ * uniform.
+ *
+ * Leaving ULLS mode always goes through a "last" job, which emits no tail
+ * update, so an ordinary variable length migration job never follows a
+ * prediction.
+ *
+ * Semaphores
+ * ----------
+ *
+ * The semaphores live in the driver-defined portion of the migration LRC's
+ * PPHWSP (see LRC_ULLS_PPHWSP_OFFSET, mutually exclusive with the parallel
+ * submission area). There are LRC_MIGRATION_ULLS_SEMAPHORE_COUNT of them and
+ * a job's semaphore is selected by ``seqno % COUNT``, so the semaphore ring
+ * wraps with the job seqnos. To guarantee a job can never overwrite the
+ * semaphore of a job still in flight, the GuC backend caps the migration
+ * queue's scheduler job count at LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1.
+ *
+ * Ring layout of a ULLS job
+ * -------------------------
+ *
+ * Emitted by emit_migration_job_gen12() in xe_ring_ops.c::
+ *
+ *	preamble:	clear semaphore[seqno]	(reuse for a later wrap)
+ *	<copy timestamp, start seqno store>
+ *	<batch buffer start(s)>			(skipped on first/last job)
+ *	<seqno write + user interrupt>
+ *	postamble:	SDI saved ring tail = end of next job
+ *			LRI RING_TAIL = end of next job
+ *			wait on semaphore[seqno + 1]
+ *						(skipped on the last job)
+ *	pad:		MI_NOOP up to ULLS_JOB_SIZE_DW
+ *
+ * The preamble clears the current job's semaphore so it can be reused once
+ * the seqno space wraps. The postamble is what keeps the engine busy: it
+ * advances the ring tail over the next job and then blocks on that job's
+ * semaphore, which is only signaled when the job is actually submitted. It
+ * advances the saved tail as well as the tail register, keeping the two in
+ * step without any help from the CPU, so a context save and restore can not
+ * rewind the tail behind work which has already been published.
+ *
+ * The tail register write must be non-posted, i.e. it must not carry
+ * MI_LRI_FORCE_POSTED. Posted, the new tail is free to land after the command
+ * streamer has already drained the rest of the job, at which point the command
+ * streamer sees head == the old tail and parks as though the ring were empty.
+ * A parked context can be switched off the hardware, and the fast path below
+ * has no H2G with which to ask GuC to bring it back.
+ *
+ * The tail is published ahead of the semaphore wait rather than after it so
+ * that the non-posted write drains while the engine is parked anyway, keeping
+ * a register round trip off the path between the semaphore being signaled and
+ * the next job running.
+ *
+ * Submission fast path
+ * --------------------
+ *
+ * In submit_exec_queue() (xe_guc_submit.c), a ULLS job that is not the first
+ * one reduces to::
+ *
+ *	xe_lrc_set_ulls_semaphore(lrc, seqno);		release previous job
+ *
+ * The XE_GUC_ACTION_SCHED_CONTEXT H2G is suppressed, and so is the write of
+ * the saved ring tail: the previous job's postamble has already published
+ * this job's tail both in the tail register and in the context image, so the
+ * semaphore signal is all that is left. The previous job's semaphore wait is
+ * satisfied and the engine walks straight into this job.
+ *
+ * This does assume the context stays resident for as long as ULLS mode is
+ * active. Nothing else is scheduled on the reserved engine, so the only ways
+ * off the hardware are the "last" job below, or a reset - and a migration job
+ * failing already wedges the device.
+ *
+ * Enter / exit
+ * ------------
+ *
+ * xe_migrate_ulls_enter() is called from the page fault handler and from the
+ * SVM prefetch path, i.e. exactly where low latency migration matters. It
+ * takes a PM runtime reference (the device must not suspend while the engine
+ * spins), then submits a "first" ULLS job. That first job carries no batch
+ * buffer; it exists only to get the context onto the hardware through the
+ * normal GuC path and to leave the engine waiting on the next semaphore,
+ * pipelining the GuC/HW context switch out of the critical path.
+ *
+ * No forcewake reference is required. Nothing in the fast path touches MMIO,
+ * and the engine keeps itself awake for as long as it is executing the ring.
+ * Not needing host MMIO access is also what lets ULLS run on SRIOV VFs.
+ *
+ * Keeping an engine spinning costs power, so ULLS is not left enabled
+ * indefinitely. Every enter and every ULLS job submission re-arms
+ * @xe_migrate.ulls.exit_work with a ULLS_EXIT_JIFFIES delay. When it fires
+ * with the queue idle, it submits a "last" ULLS job - again with no batch
+ * buffer and, crucially, with no postamble semaphore wait or tail update -
+ * which lets the ring drain so the context can be switched off the hardware.
+ * The PM reference is then dropped. If the queue was not idle, the worker
+ * simply re-arms itself.
+ *
+ * Job state
+ * ---------
+ *
+ * The state above is communicated to the ring ops and GuC backend via
+ * @xe_sched_job.ulls, set under @xe_migrate.job_mutex:
+ *
+ * - %ULLS_NONE: job submitted outside of ULLS mode
+ * - %ULLS_ENTER: job that enters ULLS mode
+ * - %ULLS_ACTIVE: job submitted while in ULLS mode
+ * - %ULLS_EXIT: job that exits ULLS mode
+ */
+
 /**
  * struct xe_migrate - migrate context.
  */
@@ -75,19 +227,24 @@ struct xe_migrate {
 	 * Protected by @job_mutex.
 	 */
 	struct dma_fence *fence;
-	/**
-	 * @vm_update_sa: For integrated, used to suballocate page-tables
-	 * out of the pt_bo.
-	 */
-	struct drm_suballoc_manager vm_update_sa;
 	/** @min_chunk_size: For dgfx, Minimum chunk size */
 	u64 min_chunk_size;
+	/** @ulls: ULLS support */
+	struct {
+		/** @ulls.exit_ms: ULLS exit period milliseconds */
+		u32 exit_ms;
+		/** @ulls.enabled: ULLS is enabled, protected by job_mutex */
+		bool enabled;
+		/** @ulls.exit_work: ULLS exit worker */
+		struct delayed_work exit_work;
+	} ulls;
 };
 
+#define ULLS_EXIT_JIFFIES(_m)	msecs_to_jiffies((_m)->ulls.exit_ms)
+
 #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */
 #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */
-#define NUM_KERNEL_PDE 15
-#define NUM_PT_SLOTS 32
+#define NUM_PT_SLOTS 48
 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M
 #define MAX_NUM_PTE 512
 #define IDENTITY_OFFSET 256ULL
@@ -101,9 +258,30 @@ struct xe_migrate {
  */
 #define MAX_PTE_PER_SDI 0x1FEU
 
+static bool xe_migrate_ulls_enabled(struct xe_migrate *m)
+{
+	lockdep_assert_held(&m->job_mutex);
+	return m->ulls.enabled;
+}
+
+static void xe_migrate_ulls_toggle_enable(struct xe_migrate *m, bool enabled)
+{
+	lockdep_assert_held(&m->job_mutex);
+	m->ulls.enabled = enabled;
+}
+
 static void xe_migrate_fini(void *arg)
 {
 	struct xe_migrate *m = arg;
+	struct xe_device *xe = tile_to_xe(m->tile);
+
+	disable_delayed_work_sync(&m->ulls.exit_work);
+	scoped_guard(mutex, &m->job_mutex) {
+		if (xe_migrate_ulls_enabled(m)) {
+			xe_pm_runtime_put(xe);
+			xe_migrate_ulls_toggle_enable(m, false);
+		}
+	}
 
 	xe_vm_lock(m->q->vm, false);
 	xe_bo_unpin(m->pt_bo);
@@ -111,7 +289,6 @@ static void xe_migrate_fini(void *arg)
 
 	dma_fence_put(m->fence);
 	xe_bo_put(m->pt_bo);
-	drm_suballoc_manager_fini(&m->vm_update_sa);
 	mutex_destroy(&m->job_mutex);
 	xe_vm_close_and_put(m->q->vm);
 	xe_exec_queue_put(m->q);
@@ -163,22 +340,20 @@ static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte)
 }
 
 static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo,
-					u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs)
+					u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs,
+					u64 pt_4k_ofs)
 {
 	struct xe_vram_region *vram = xe->mem.vram;
 	resource_size_t dpa_base = xe_vram_region_dpa_base(vram);
 	u64 pos, ofs, flags;
 	u64 entry;
-	/* XXX: Unclear if this should be usable_size? */
-	u64 vram_limit = xe_vram_region_actual_physical_size(vram) + dpa_base;
+	u64 vram_limit = xe_vram_region_usable_size(vram) + dpa_base;
 	u32 level = 2;
 
 	ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8;
 	flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,
 					    true, 0);
 
-	xe_assert(xe, IS_ALIGNED(xe_vram_region_usable_size(vram), SZ_2M));
-
 	/*
 	 * Use 1GB pages when possible, last chunk always use 2M
 	 * pages as mixing reserved memory (stolen, WOCPM) with a single
@@ -196,8 +371,24 @@ static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm,
 							    true, 0);
 
 			for (ofs = pt_2m_ofs; pos < vram_limit;
-			     pos += SZ_2M, ofs += 8)
+			     pos += SZ_2M, ofs += 8) {
+				if (pos + SZ_2M > vram_limit) {
+					entry = vm->pt_ops->pde_encode_bo(bo, pt_4k_ofs);
+					xe_map_wr(xe, &bo->vmap, ofs, u64, entry);
+
+					flags = vm->pt_ops->pte_encode_addr(xe, 0,
+									    pat_index,
+									    level - 2,
+									    true, 0);
+
+					for (ofs = pt_4k_ofs; pos < vram_limit;
+					     pos += SZ_4K, ofs += 8)
+						xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
+					break;
+				}
+
 				xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags);
+			}
 			break;	/* Ensure pos == vram_limit assert correct */
 		}
 
@@ -217,8 +408,6 @@ static int xe_migrate_pt_bo_alloc(struct xe_tile *tile, struct xe_migrate *m,
 	BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE);
 	/* Must be a multiple of 64K to support all platforms */
 	BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K);
-	/* And one slot reserved for the 4KiB page table updates */
-	BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1));
 
 	/* Need to be sure everything fits in the first PT, or create more */
 	xe_tile_assert(tile, m->batch_base_ofs + xe_bo_size(batch) < SZ_2M);
@@ -241,18 +430,20 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 	struct xe_device *xe = tile_to_xe(tile);
 	u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
 	u8 id = tile->id;
-	u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
-#define VRAM_IDENTITY_MAP_COUNT	2
-	u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT;
-#undef VRAM_IDENTITY_MAP_COUNT
+	u32 num_entries = NUM_PT_SLOTS, num_level =
+		xe_vm_pt_root(vm, id)->level;
+#define VRAM_IDENTITY_MAP_PT_COUNT	4
+	u32 num_setup = num_level + VRAM_IDENTITY_MAP_PT_COUNT;
+#undef VRAM_IDENTITY_MAP_PT_COUNT
 	u32 map_ofs, level, i;
 	struct xe_bo *bo = m->pt_bo, *batch = tile->mem.kernel_bb_pool->bo;
-	u64 entry, pt29_ofs;
+	u64 entry;
+
+	/* PT44..PT47 reserved for 4K and 2M identity map */
+	u64 l1_pt_ofs = xe_bo_size(bo) - 5 * XE_PAGE_SIZE;
 
-	/* PT30 & PT31 reserved for 2M identity map */
-	pt29_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE;
-	entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs);
-	xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry);
+	entry = vm->pt_ops->pde_encode_bo(bo, l1_pt_ofs);
+	xe_pt_write(xe, &xe_vm_pt_root(vm, id)->bo->vmap, 0, entry);
 
 	map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE;
 
@@ -347,11 +538,12 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 
 	/* Identity map the entire vram at 256GiB offset */
 	if (IS_DGFX(xe)) {
-		u64 pt30_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE;
+		u64 pt46_ofs = xe_bo_size(bo) - 2 * XE_PAGE_SIZE;
 		resource_size_t actual_phy_size = xe_vram_region_actual_physical_size(xe->mem.vram);
 
+		u64 pt44_ofs = xe_bo_size(bo) - 4 * XE_PAGE_SIZE;
 		xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET,
-					    pat_index, pt30_ofs);
+					    pat_index, pt46_ofs, pt44_ofs);
 		xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G);
 
 		/*
@@ -362,46 +554,33 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 			u16 comp_pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION);
 			u64 vram_offset = IDENTITY_OFFSET +
 				DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G);
-			u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE;
+			u64 pt47_ofs = xe_bo_size(bo) - XE_PAGE_SIZE;
 
 			xe_assert(xe, actual_phy_size <= (MAX_NUM_PTE - IDENTITY_OFFSET -
 							  IDENTITY_OFFSET / 2) * SZ_1G);
+			u64 pt45_ofs = xe_bo_size(bo) - 3 * XE_PAGE_SIZE;
 			xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset,
-						    comp_pat_index, pt31_ofs);
+						    comp_pat_index, pt47_ofs, pt45_ofs);
 		}
 	}
 
-	if (ofs)
-		*ofs = map_ofs;
-}
-
-static void xe_migrate_suballoc_manager_init(struct xe_migrate *m, u32 map_ofs)
-{
 	/*
 	 * Example layout created above, with root level = 3:
 	 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's
-	 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's
-	 * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's
-	 * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map]
+	 * [PT41 = PDE 0] [PT44...PT47 = 4K and 2M vram identity maps]
 	 *
 	 * This makes the lowest part of the VM point to the pagetables.
 	 * Hence the lowest 2M in the vm should point to itself, with a few writes
 	 * and flushes, other parts of the VM can be used either for copying and
 	 * clearing.
 	 *
-	 * For performance, the kernel reserves PDE's, so about 20 are left
-	 * for async VM updates.
-	 *
 	 * To make it easier to work, each scratch PT is put in slot (1 + PT #)
 	 * everywhere, this allows lockless updates to scratch pages by using
 	 * the different addresses in VM.
 	 */
-#define NUM_VMUSA_UNIT_PER_PAGE	32
-#define VM_SA_UPDATE_UNIT_SIZE		(XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE)
-#define NUM_VMUSA_WRITES_PER_UNIT	(VM_SA_UPDATE_UNIT_SIZE / sizeof(u64))
-	drm_suballoc_manager_init(&m->vm_update_sa,
-				  (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) *
-				  NUM_VMUSA_UNIT_PER_PAGE, 0);
+
+	if (ofs)
+		*ofs = map_ofs;
 }
 
 static bool xe_migrate_needs_ccs_emit(struct xe_device *xe)
@@ -446,7 +625,6 @@ static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m
 			return err;
 
 		xe_migrate_prepare_vm(tile, m, vm, &map_ofs);
-		xe_migrate_suballoc_manager_init(m, map_ofs);
 		drm_exec_retry_on_contention(&exec);
 		xe_validation_retry_on_oom(&ctx, &err);
 	}
@@ -454,6 +632,150 @@ static int xe_migrate_lock_prepare_vm(struct xe_tile *tile, struct xe_migrate *m
 	return err;
 }
 
+static struct dma_fence *__xe_migrate_job_push(struct xe_migrate *m,
+					       struct xe_sched_job *job,
+					       enum xe_ulls_state ulls)
+{
+	struct dma_fence *fence;
+
+	lockdep_assert_held(&m->job_mutex);
+	xe_tile_assert(m->tile, m->q == job->q);
+
+	job->ulls = ulls;
+	xe_sched_job_arm(job);
+	fence = dma_fence_get(&job->drm.s_fence->finished);
+	xe_sched_job_push(job);
+
+	return fence;
+}
+
+/*
+ * Arm and push a migration job, tagging it as a ULLS job and deferring the
+ * ULLS exit while ULLS mode is active.
+ *
+ * Returns a reference to the job's finished fence.
+ */
+static struct dma_fence *xe_migrate_job_push(struct xe_migrate *m,
+					     struct xe_sched_job *job)
+{
+	enum xe_ulls_state ulls = ULLS_NONE;
+
+	lockdep_assert_held(&m->job_mutex);
+
+	if (xe_migrate_ulls_enabled(m)) {
+		ulls = ULLS_ACTIVE;
+		mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
+				 ULLS_EXIT_JIFFIES(m));
+	}
+
+	return __xe_migrate_job_push(m, job, ulls);
+}
+
+/**
+ * xe_migrate_ulls_enter() - Enter ULLS mode
+ * @m: The migration context.
+ *
+ * If DGFX, enter ULLS mode bypassing GuC / HW context switches by utilizing
+ * semaphore and continuously running batches.
+ */
+void xe_migrate_ulls_enter(struct xe_migrate *m)
+{
+	struct xe_device *xe = tile_to_xe(m->tile);
+	struct xe_sched_job *job = NULL;
+	u64 batch_addr[2] = { 0, 0 };
+	bool alloc = false;
+
+	xe_assert(xe, xe->info.has_usm);
+
+	if (!IS_DGFX(xe) || !m->ulls.exit_ms)
+		return;
+
+job_alloc:
+	if (alloc) {
+		/*
+		 * Must be done outside job_mutex as that lock is tainted with
+		 * reclaim.
+		 */
+		job = xe_sched_job_create(m->q, batch_addr);
+		if (WARN_ON_ONCE(IS_ERR(job)))
+			return;		/* Not fatal */
+	}
+
+	mutex_lock(&m->job_mutex);
+	if (!xe_migrate_ulls_enabled(m)) {
+		struct dma_fence *fence;
+
+		if (!job) {
+			alloc = true;
+			mutex_unlock(&m->job_mutex);
+			goto job_alloc;
+		}
+
+		/* Pairs with PM put on ULLS exit */
+		xe_pm_runtime_get_noresume(xe);
+
+		xe_sched_job_get(job);
+		fence = __xe_migrate_job_push(m, job, ULLS_ENTER);
+		dma_fence_put(fence);
+
+		xe_dbg(xe, "Migrate ULLS mode enter");
+		xe_migrate_ulls_toggle_enable(m, true);
+	}
+	if (job)
+		xe_sched_job_put(job);
+	if (xe_migrate_ulls_enabled(m))
+		mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
+				 ULLS_EXIT_JIFFIES(m));
+	mutex_unlock(&m->job_mutex);
+}
+
+static void xe_migrate_ulls_exit(struct work_struct *work)
+{
+	struct xe_migrate *m = container_of(work, struct xe_migrate,
+					    ulls.exit_work.work);
+	struct xe_device *xe = tile_to_xe(m->tile);
+	struct xe_sched_job *job = NULL;
+	struct dma_fence *fence = NULL;
+	u64 batch_addr[2] = { 0, 0 };
+	int idx;
+
+	xe_assert(xe, m->ulls.enabled);
+
+	if (!drm_dev_enter(&xe->drm, &idx))
+		return;
+
+	/*
+	 * Must be done outside job_mutex as that lock is tainted with
+	 * reclaim and must be done holding a pm ref.
+	 */
+	job = xe_sched_job_create(m->q, batch_addr);
+	if (WARN_ON_ONCE(IS_ERR(job))) {
+		drm_dev_exit(idx);
+		mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
+				 ULLS_EXIT_JIFFIES(m));
+		return;		/* Not fatal */
+	}
+
+	scoped_guard(mutex, &m->job_mutex) {
+		if (xe_exec_queue_is_idle(m->q, 1)) {
+			fence = __xe_migrate_job_push(m, job, ULLS_EXIT);
+			dma_fence_put(fence);
+
+			xe_pm_runtime_put(xe);	/* Pairs with PM get in enter */
+			xe_migrate_ulls_toggle_enable(m, false);
+			cancel_delayed_work(&m->ulls.exit_work);
+
+			xe_dbg(xe, "Migrate ULLS mode exit");
+		} else {
+			xe_sched_job_put(job);
+			mod_delayed_work(system_percpu_wq, &m->ulls.exit_work,
+					 ULLS_EXIT_JIFFIES(m));
+		}
+	}
+
+	drm_dev_exit(idx);
+}
+
 /**
  * xe_migrate_init() - Initialize a migrate context
  * @m: The migration context
@@ -493,7 +815,6 @@ int xe_migrate_init(struct xe_migrate *m)
 		 */
 		m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0,
 					    EXEC_QUEUE_FLAG_KERNEL |
-					    EXEC_QUEUE_FLAG_PERMANENT |
 					    EXEC_QUEUE_FLAG_HIGH_PRIORITY |
 					    EXEC_QUEUE_FLAG_MIGRATE |
 					    EXEC_QUEUE_FLAG_LOW_LATENCY, 0);
@@ -501,7 +822,6 @@ int xe_migrate_init(struct xe_migrate *m)
 		m->q = xe_exec_queue_create_class(xe, primary_gt, vm,
 						  XE_ENGINE_CLASS_COPY,
 						  EXEC_QUEUE_FLAG_KERNEL |
-						  EXEC_QUEUE_FLAG_PERMANENT |
 						  EXEC_QUEUE_FLAG_MIGRATE, 0);
 	}
 	if (IS_ERR(m->q)) {
@@ -514,6 +834,8 @@ int xe_migrate_init(struct xe_migrate *m)
 	might_lock(&m->job_mutex);
 	fs_reclaim_release(GFP_KERNEL);
 
+	INIT_DELAYED_WORK(&m->ulls.exit_work, xe_migrate_ulls_exit);
+
 	err = devm_add_action_or_reset(xe->drm.dev, xe_migrate_fini, m);
 	if (err)
 		return err;
@@ -531,6 +853,9 @@ int xe_migrate_init(struct xe_migrate *m)
 			(unsigned long long)m->min_chunk_size);
 	}
 
+	m->ulls.exit_ms =
+		xe_configfs_get_migrate_ulls_period_ms(to_pci_dev(xe->drm.dev));
+
 	return err;
 
 err_out:
@@ -1041,10 +1366,8 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
 		}
 
 		mutex_lock(&m->job_mutex);
-		xe_sched_job_arm(job);
 		dma_fence_put(fence);
-		fence = dma_fence_get(&job->drm.s_fence->finished);
-		xe_sched_job_push(job);
+		fence = xe_migrate_job_push(m, job);
 
 		dma_fence_put(m->fence);
 		m->fence = dma_fence_get(fence);
@@ -1131,6 +1454,9 @@ struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate)
 	return migrate->q->lrc[0];
 }
 
+/* XXX: With CPU binds this can be removed in a follow up */
+#define NUM_KERNEL_PDE 15
+
 static u64 migrate_vm_ppgtt_addr_tlb_inval(void)
 {
 	/*
@@ -1469,10 +1795,8 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off
 						     DMA_RESV_USAGE_BOOKKEEP));
 
 		scoped_guard(mutex, &m->job_mutex) {
-			xe_sched_job_arm(job);
 			dma_fence_put(fence);
-			fence = dma_fence_get(&job->drm.s_fence->finished);
-			xe_sched_job_push(job);
+			fence = xe_migrate_job_push(m, job);
 
 			dma_fence_put(m->fence);
 			m->fence = dma_fence_get(fence);
@@ -1706,10 +2030,8 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
 		}
 
 		mutex_lock(&m->job_mutex);
-		xe_sched_job_arm(job);
 		dma_fence_put(fence);
-		fence = dma_fence_get(&job->drm.s_fence->finished);
-		xe_sched_job_push(job);
+		fence = xe_migrate_job_push(m, job);
 
 		dma_fence_put(m->fence);
 		m->fence = dma_fence_get(fence);
@@ -1739,55 +2061,6 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
 	return fence;
 }
 
-static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
-			  const struct xe_vm_pgtable_update_op *pt_op,
-			  const struct xe_vm_pgtable_update *update,
-			  struct xe_migrate_pt_update *pt_update)
-{
-	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
-	u32 chunk;
-	u32 ofs = update->ofs, size = update->qwords;
-
-	/*
-	 * If we have 512 entries (max), we would populate it ourselves,
-	 * and update the PDE above it to the new pointer.
-	 * The only time this can only happen if we have to update the top
-	 * PDE. This requires a BO that is almost vm->size big.
-	 *
-	 * This shouldn't be possible in practice.. might change when 16K
-	 * pages are used. Hence the assert.
-	 */
-	xe_tile_assert(tile, update->qwords < MAX_NUM_PTE);
-	if (!ppgtt_ofs)
-		ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
-						xe_bo_addr(update->pt_bo, 0,
-							   XE_PAGE_SIZE), false);
-
-	do {
-		u64 addr = ppgtt_ofs + ofs * 8;
-
-		chunk = min(size, MAX_PTE_PER_SDI);
-
-		/* Ensure populatefn can do memset64 by aligning bb->cs */
-		if (!(bb->len & 1))
-			bb->cs[bb->len++] = MI_NOOP;
-
-		bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
-		bb->cs[bb->len++] = lower_32_bits(addr);
-		bb->cs[bb->len++] = upper_32_bits(addr);
-		if (pt_op->bind)
-			ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
-				      ofs, chunk, update);
-		else
-			ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
-				   ofs, chunk, update);
-
-		bb->len += chunk * 2;
-		ofs += chunk;
-		size -= chunk;
-	} while (size);
-}
-
 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m)
 {
 	return xe_vm_get(m->q->vm);
@@ -1803,284 +2076,6 @@ struct migrate_test_params {
 	container_of(_priv, struct migrate_test_params, base)
 #endif
 
-static struct dma_fence *
-xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
-			       struct xe_migrate_pt_update *pt_update)
-{
-	XE_TEST_DECLARE(struct migrate_test_params *test =
-			to_migrate_test_params
-			(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));)
-	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
-	struct xe_vm *vm = pt_update->vops->vm;
-	struct xe_vm_pgtable_update_ops *pt_update_ops =
-		&pt_update->vops->pt_update_ops[pt_update->tile_id];
-	int err;
-	u32 i, j;
-
-	if (XE_TEST_ONLY(test && test->force_gpu))
-		return ERR_PTR(-ETIME);
-
-	if (ops->pre_commit) {
-		pt_update->job = NULL;
-		err = ops->pre_commit(pt_update);
-		if (err)
-			return ERR_PTR(err);
-	}
-
-	for (i = 0; i < pt_update_ops->num_ops; ++i) {
-		const struct xe_vm_pgtable_update_op *pt_op =
-			&pt_update_ops->ops[i];
-
-		for (j = 0; j < pt_op->num_entries; j++) {
-			const struct xe_vm_pgtable_update *update =
-				&pt_op->entries[j];
-
-			if (pt_op->bind)
-				ops->populate(pt_update, m->tile,
-					      &update->pt_bo->vmap, NULL,
-					      update->ofs, update->qwords,
-					      update);
-			else
-				ops->clear(pt_update, m->tile,
-					   &update->pt_bo->vmap, NULL,
-					   update->ofs, update->qwords, update);
-		}
-	}
-
-	trace_xe_vm_cpu_bind(vm);
-	xe_device_wmb(vm->xe);
-
-	return dma_fence_get_stub();
-}
-
-static struct dma_fence *
-__xe_migrate_update_pgtables(struct xe_migrate *m,
-			     struct xe_migrate_pt_update *pt_update,
-			     struct xe_vm_pgtable_update_ops *pt_update_ops)
-{
-	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
-	struct xe_tile *tile = m->tile;
-	struct xe_gt *gt = tile->primary_gt;
-	struct xe_device *xe = tile_to_xe(tile);
-	struct xe_sched_job *job;
-	struct dma_fence *fence;
-	struct drm_suballoc *sa_bo = NULL;
-	struct xe_bb *bb;
-	u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0;
-	u32 num_updates = 0, current_update = 0;
-	u64 addr;
-	int err = 0;
-	bool is_migrate = pt_update_ops->q == m->q;
-	bool usm = is_migrate && xe->info.has_usm;
-
-	for (i = 0; i < pt_update_ops->num_ops; ++i) {
-		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
-		struct xe_vm_pgtable_update *updates = pt_op->entries;
-
-		num_updates += pt_op->num_entries;
-		for (j = 0; j < pt_op->num_entries; ++j) {
-			u32 num_cmds = DIV_ROUND_UP(updates[j].qwords,
-						    MAX_PTE_PER_SDI);
-
-			/* align noop + MI_STORE_DATA_IMM cmd prefix */
-			batch_size += 4 * num_cmds + updates[j].qwords * 2;
-		}
-	}
-
-	/* fixed + PTE entries */
-	if (IS_DGFX(xe))
-		batch_size += 2;
-	else
-		batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) +
-			num_updates * 2;
-
-	bb = xe_bb_new(gt, batch_size, usm);
-	if (IS_ERR(bb))
-		return ERR_CAST(bb);
-
-	/* For sysmem PTE's, need to map them in our hole.. */
-	if (!IS_DGFX(xe)) {
-		u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
-		u32 ptes, ofs;
-
-		ppgtt_ofs = NUM_KERNEL_PDE - 1;
-		if (!is_migrate) {
-			u32 num_units = DIV_ROUND_UP(num_updates,
-						     NUM_VMUSA_WRITES_PER_UNIT);
-
-			if (num_units > m->vm_update_sa.size) {
-				err = -ENOBUFS;
-				goto err_bb;
-			}
-			sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units,
-						 GFP_KERNEL, true, 0);
-			if (IS_ERR(sa_bo)) {
-				err = PTR_ERR(sa_bo);
-				goto err_bb;
-			}
-
-			ppgtt_ofs = NUM_KERNEL_PDE +
-				(drm_suballoc_soffset(sa_bo) /
-				 NUM_VMUSA_UNIT_PER_PAGE);
-			page_ofs = (drm_suballoc_soffset(sa_bo) %
-				    NUM_VMUSA_UNIT_PER_PAGE) *
-				VM_SA_UPDATE_UNIT_SIZE;
-		}
-
-		/* Map our PT's to gtt */
-		i = 0;
-		j = 0;
-		ptes = num_updates;
-		ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs;
-		while (ptes) {
-			u32 chunk = min(MAX_PTE_PER_SDI, ptes);
-			u32 idx = 0;
-
-			bb->cs[bb->len++] = MI_STORE_DATA_IMM |
-				MI_SDI_NUM_QW(chunk);
-			bb->cs[bb->len++] = ofs;
-			bb->cs[bb->len++] = 0; /* upper_32_bits */
-
-			for (; i < pt_update_ops->num_ops; ++i) {
-				struct xe_vm_pgtable_update_op *pt_op =
-					&pt_update_ops->ops[i];
-				struct xe_vm_pgtable_update *updates = pt_op->entries;
-
-				for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) {
-					struct xe_vm *vm = pt_update->vops->vm;
-					struct xe_bo *pt_bo = updates[j].pt_bo;
-
-					if (idx == chunk)
-						goto next_cmd;
-
-					xe_tile_assert(tile, xe_bo_size(pt_bo) == SZ_4K);
-
-					/* Map a PT at most once */
-					if (pt_bo->update_index < 0)
-						pt_bo->update_index = current_update;
-
-					addr = vm->pt_ops->pte_encode_bo(pt_bo, 0,
-									 pat_index, 0);
-					bb->cs[bb->len++] = lower_32_bits(addr);
-					bb->cs[bb->len++] = upper_32_bits(addr);
-				}
-
-				j = 0;
-			}
-
-next_cmd:
-			ptes -= chunk;
-			ofs += chunk * sizeof(u64);
-		}
-
-		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
-		update_idx = bb->len;
-
-		addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
-			(page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
-		for (i = 0; i < pt_update_ops->num_ops; ++i) {
-			struct xe_vm_pgtable_update_op *pt_op =
-				&pt_update_ops->ops[i];
-			struct xe_vm_pgtable_update *updates = pt_op->entries;
-
-			for (j = 0; j < pt_op->num_entries; ++j) {
-				struct xe_bo *pt_bo = updates[j].pt_bo;
-
-				write_pgtable(tile, bb, addr +
-					      pt_bo->update_index * XE_PAGE_SIZE,
-					      pt_op, &updates[j], pt_update);
-			}
-		}
-	} else {
-		/* phys pages, no preamble required */
-		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
-		update_idx = bb->len;
-
-		for (i = 0; i < pt_update_ops->num_ops; ++i) {
-			struct xe_vm_pgtable_update_op *pt_op =
-				&pt_update_ops->ops[i];
-			struct xe_vm_pgtable_update *updates = pt_op->entries;
-
-			for (j = 0; j < pt_op->num_entries; ++j)
-				write_pgtable(tile, bb, 0, pt_op, &updates[j],
-					      pt_update);
-		}
-	}
-
-	job = xe_bb_create_migration_job(pt_update_ops->q, bb,
-					 xe_migrate_batch_base(m, usm),
-					 update_idx);
-	if (IS_ERR(job)) {
-		err = PTR_ERR(job);
-		goto err_sa;
-	}
-
-	xe_sched_job_add_migrate_flush(job, MI_INVALIDATE_TLB);
-
-	if (ops->pre_commit) {
-		pt_update->job = job;
-		err = ops->pre_commit(pt_update);
-		if (err)
-			goto err_job;
-	}
-	if (is_migrate)
-		mutex_lock(&m->job_mutex);
-
-	xe_sched_job_arm(job);
-	fence = dma_fence_get(&job->drm.s_fence->finished);
-	xe_sched_job_push(job);
-
-	if (is_migrate)
-		mutex_unlock(&m->job_mutex);
-
-	xe_bb_free(bb, fence);
-	drm_suballoc_free(sa_bo, fence);
-
-	return fence;
-
-err_job:
-	xe_sched_job_put(job);
-err_sa:
-	drm_suballoc_free(sa_bo, NULL);
-err_bb:
-	xe_bb_free(bb, NULL);
-	return ERR_PTR(err);
-}
-
-/**
- * xe_migrate_update_pgtables() - Pipelined page-table update
- * @m: The migrate context.
- * @pt_update: PT update arguments
- *
- * Perform a pipelined page-table update. The update descriptors are typically
- * built under the same lock critical section as a call to this function. If
- * using the default engine for the updates, they will be performed in the
- * order they grab the job_mutex. If different engines are used, external
- * synchronization is needed for overlapping updates to maintain page-table
- * consistency. Note that the meaning of "overlapping" is that the updates
- * touch the same page-table, which might be a higher-level page-directory.
- * If no pipelining is needed, then updates may be performed by the cpu.
- *
- * Return: A dma_fence that, when signaled, indicates the update completion.
- */
-struct dma_fence *
-xe_migrate_update_pgtables(struct xe_migrate *m,
-			   struct xe_migrate_pt_update *pt_update)
-
-{
-	struct xe_vm_pgtable_update_ops *pt_update_ops =
-		&pt_update->vops->pt_update_ops[pt_update->tile_id];
-	struct dma_fence *fence;
-
-	fence =  xe_migrate_update_pgtables_cpu(m, pt_update);
-
-	/* -ETIME indicates a job is needed, anything else is legit error */
-	if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
-		return fence;
-
-	return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops);
-}
-
 /**
  * xe_migrate_wait() - Complete all operations using the xe_migrate context
  * @m: Migrate context to wait for.
@@ -2312,9 +2307,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
 	}
 
 	mutex_lock(&m->job_mutex);
-	xe_sched_job_arm(job);
-	fence = dma_fence_get(&job->drm.s_fence->finished);
-	xe_sched_job_push(job);
+	fence = xe_migrate_job_push(m, job);
 
 	dma_fence_put(m->fence);
 	m->fence = dma_fence_get(fence);
@@ -2582,56 +2575,67 @@ int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo,
 	return IS_ERR(fence) ? PTR_ERR(fence) : 0;
 }
 
-/**
- * xe_migrate_job_lock() - Lock migrate job lock
- * @m: The migration context.
- * @q: Queue associated with the operation which requires a lock
- *
- * Lock the migrate job lock if the queue is a migration queue, otherwise
- * assert the VM's dma-resv is held (user queue's have own locking).
- */
-void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q)
+#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
+#include "tests/xe_migrate.c"
+#endif
+
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
+int xe_migrate_debug_ccs_overlap(struct xe_migrate *m,
+				 struct xe_bo *scratch_bo,
+				 bool write_to_ccs)
 {
-	bool is_migrate = q == m->q;
+	struct xe_device *xe = tile_to_xe(m->tile);
+	struct xe_gt *gt = m->tile->primary_gt;
+	struct dma_fence *fence;
+	struct xe_bb *bb;
+	struct xe_sched_job *job;
+	u64 first_page_dpa, clear_L0_ofs, scratch_dpa, scratch_L0_ofs;
 
-	if (is_migrate)
-		mutex_lock(&m->job_mutex);
-	else
-		xe_vm_assert_held(q->user_vm);	/* User queues VM's should be locked */
-}
+	if (!xe_device_has_flat_ccs(xe))
+		return -EINVAL;
 
-/**
- * xe_migrate_job_unlock() - Unlock migrate job lock
- * @m: The migration context.
- * @q: Queue associated with the operation which requires a lock
- *
- * Unlock the migrate job lock if the queue is a migration queue, otherwise
- * assert the VM's dma-resv is held (user queue's have own locking).
- */
-void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q)
-{
-	bool is_migrate = q == m->q;
+	first_page_dpa = xe_vram_region_dpa_base(m->tile->mem.vram);
+	clear_L0_ofs = xe_migrate_vram_ofs(xe, first_page_dpa, true);
 
-	if (is_migrate)
+	scratch_dpa = xe_bo_addr(scratch_bo, 0, XE_PAGE_SIZE);
+	scratch_L0_ofs = xe_migrate_vram_ofs(xe, scratch_dpa, false);
+
+	bb = xe_bb_new(gt, EMIT_COPY_CCS_DW + 1, xe->info.has_usm);
+	if (IS_ERR(bb)) {
+		drm_warn(&xe->drm, "Failed to create bb for VRAM overlap check\n");
+		return PTR_ERR(bb);
+	}
+
+	/* 4MB payload = 8KB CCS metadata */
+	if (write_to_ccs) {
+		emit_copy_ccs(gt, bb, clear_L0_ofs, true,
+			      scratch_L0_ofs, false, SZ_4M);
+	} else {
+		emit_copy_ccs(gt, bb, scratch_L0_ofs, false,
+			      clear_L0_ofs, true, SZ_4M);
+	}
+
+	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
+
+	job = xe_bb_create_migration_job(m->q, bb,
+					 xe_migrate_batch_base(m, xe->info.has_usm),
+					 0);
+	if (!IS_ERR(job)) {
+		xe_sched_job_add_migrate_flush(job, MI_FLUSH_DW_CCS);
+
+		mutex_lock(&m->job_mutex);
+		fence = xe_migrate_job_push(m, job);
 		mutex_unlock(&m->job_mutex);
-	else
-		xe_vm_assert_held(q->user_vm);	/* User queues VM's should be locked */
-}
 
-#if IS_ENABLED(CONFIG_PROVE_LOCKING)
-/**
- * xe_migrate_job_lock_assert() - Assert migrate job lock held of queue
- * @q: Migrate queue
- */
-void xe_migrate_job_lock_assert(struct xe_exec_queue *q)
-{
-	struct xe_migrate *m = gt_to_tile(q->gt)->migrate;
+		dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+	} else {
+		drm_warn(&xe->drm, "Failed to create job for VRAM overlap check\n");
+		xe_bb_free(bb, NULL);
+		return PTR_ERR(job);
+	}
 
-	xe_gt_assert(q->gt, q == m->q);
-	lockdep_assert_held(&m->job_mutex);
+	xe_bb_free(bb, NULL);
+	return 0;
 }
 #endif
-
-#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
-#include "tests/xe_migrate.c"
-#endif
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index c3a268b01768ae..2e8be10fdb71ff 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -24,6 +24,7 @@ struct xe_pt;
 struct xe_tile;
 struct xe_vm;
 struct xe_vm_pgtable_update;
+struct xe_vm_pgtable_update_op;
 struct xe_vma;
 
 enum xe_sriov_vf_ccs_rw_ctxs;
@@ -33,83 +34,6 @@ enum xe_migrate_copy_dir {
 	XE_MIGRATE_COPY_TO_SRAM,
 };
 
-/**
- * struct xe_migrate_pt_update_ops - Callbacks for the
- * xe_migrate_update_pgtables() function.
- */
-struct xe_migrate_pt_update_ops {
-	/**
-	 * @populate: Populate a command buffer or page-table with ptes.
-	 * @pt_update: Embeddable callback argument.
-	 * @tile: The tile for the current operation.
-	 * @map: struct iosys_map into the memory to be populated.
-	 * @pos: If @map is NULL, map into the memory to be populated.
-	 * @ofs: qword offset into @map, unused if @map is NULL.
-	 * @num_qwords: Number of qwords to write.
-	 * @update: Information about the PTEs to be inserted.
-	 *
-	 * This interface is intended to be used as a callback into the
-	 * page-table system to populate command buffers or shared
-	 * page-tables with PTEs.
-	 */
-	void (*populate)(struct xe_migrate_pt_update *pt_update,
-			 struct xe_tile *tile, struct iosys_map *map,
-			 void *pos, u32 ofs, u32 num_qwords,
-			 const struct xe_vm_pgtable_update *update);
-	/**
-	 * @clear: Clear a command buffer or page-table with ptes.
-	 * @pt_update: Embeddable callback argument.
-	 * @tile: The tile for the current operation.
-	 * @map: struct iosys_map into the memory to be populated.
-	 * @pos: If @map is NULL, map into the memory to be populated.
-	 * @ofs: qword offset into @map, unused if @map is NULL.
-	 * @num_qwords: Number of qwords to write.
-	 * @update: Information about the PTEs to be inserted.
-	 *
-	 * This interface is intended to be used as a callback into the
-	 * page-table system to populate command buffers or shared
-	 * page-tables with PTEs.
-	 */
-	void (*clear)(struct xe_migrate_pt_update *pt_update,
-		      struct xe_tile *tile, struct iosys_map *map,
-		      void *pos, u32 ofs, u32 num_qwords,
-		      const struct xe_vm_pgtable_update *update);
-
-	/**
-	 * @pre_commit: Callback to be called just before arming the
-	 * sched_job.
-	 * @pt_update: Pointer to embeddable callback argument.
-	 *
-	 * Return: 0 on success, negative error code on error.
-	 */
-	int (*pre_commit)(struct xe_migrate_pt_update *pt_update);
-};
-
-/**
- * struct xe_migrate_pt_update - Argument to the
- * struct xe_migrate_pt_update_ops callbacks.
- *
- * Intended to be subclassed to support additional arguments if necessary.
- */
-struct xe_migrate_pt_update {
-	/** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */
-	const struct xe_migrate_pt_update_ops *ops;
-	/** @vops: VMA operations */
-	struct xe_vma_ops *vops;
-	/** @job: The job if a GPU page-table update. NULL otherwise */
-	struct xe_sched_job *job;
-	/**
-	 * @ijob: The TLB invalidation job for primary GT. NULL otherwise
-	 */
-	struct xe_tlb_inval_job *ijob;
-	/**
-	 * @mjob: The TLB invalidation job for media GT. NULL otherwise
-	 */
-	struct xe_tlb_inval_job *mjob;
-	/** @tile_id: Tile ID of the update */
-	u8 tile_id;
-};
-
 struct xe_migrate *xe_migrate_alloc(struct xe_tile *tile);
 int xe_migrate_init(struct xe_migrate *m);
 
@@ -165,21 +89,14 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
 
 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
 
-struct dma_fence *
-xe_migrate_update_pgtables(struct xe_migrate *m,
-			   struct xe_migrate_pt_update *pt_update);
-
 void xe_migrate_wait(struct xe_migrate *m);
 
-#if IS_ENABLED(CONFIG_PROVE_LOCKING)
-void xe_migrate_job_lock_assert(struct xe_exec_queue *q);
-#else
-static inline void xe_migrate_job_lock_assert(struct xe_exec_queue *q)
-{
-}
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
+int xe_migrate_debug_ccs_overlap(struct xe_migrate *m,
+				 struct xe_bo *scratch_bo,
+				 bool write_to_ccs);
 #endif
 
-void xe_migrate_job_lock(struct xe_migrate *m, struct xe_exec_queue *q);
-void xe_migrate_job_unlock(struct xe_migrate *m, struct xe_exec_queue *q);
+void xe_migrate_ulls_enter(struct xe_migrate *m);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_mmio_gem.c b/drivers/gpu/drm/xe/xe_mmio_gem.c
index 5ffe03d3619028..3cdc9538957d27 100644
--- a/drivers/gpu/drm/xe/xe_mmio_gem.c
+++ b/drivers/gpu/drm/xe/xe_mmio_gem.c
@@ -177,9 +177,12 @@ static int xe_mmio_gem_mmap(struct drm_gem_object *base, struct vm_area_struct *
 	if ((vma->vm_flags & VM_SHARED) == 0)
 		return -EINVAL;
 
+	if (vma->vm_flags & VM_EXEC)
+		return -EINVAL;
+
 	vma->vm_page_prot = pgprot_noncached(vma_get_page_prot(vma));
-	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
-		     VM_DONTCOPY | VM_NORESERVE);
+	vm_flags_mod(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
+		     VM_NORESERVE, VM_MAYEXEC);
 
 	/* Defer actual mapping to the fault handler. */
 	return 0;
diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c
index 848d652654438e..4bc28dfc19921c 100644
--- a/drivers/gpu/drm/xe/xe_module.c
+++ b/drivers/gpu/drm/xe/xe_module.c
@@ -29,6 +29,7 @@ struct xe_modparam xe_modparam = {
 	.max_vfs =		XE_DEFAULT_MAX_VFS,
 #endif
 	.wedged_mode =		XE_DEFAULT_WEDGED_MODE,
+	.num_pf_work =		XE_DEFAULT_NUM_PF_WORK,
 	.svm_notifier_size =	XE_DEFAULT_SVM_NOTIFIER_SIZE,
 	/* the rest are 0 by default */
 };
@@ -81,6 +82,9 @@ MODULE_PARM_DESC(wedged_mode,
 		 "Module's default policy for the wedged mode (0=never, 1=upon-critical-error, 2=upon-any-hang-no-reset "
 		 "[default=" XE_DEFAULT_WEDGED_MODE_STR "])");
 
+module_param_named(num_pf_work, xe_modparam.num_pf_work, uint, 0600);
+MODULE_PARM_DESC(num_pf_work, "Number of page fault work threads, default=2, min=1, max=8");
+
 static int xe_check_nomodeset(void)
 {
 	if (drm_firmware_drivers_only())
diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h
index a0eb7db0777042..6272d9e412077e 100644
--- a/drivers/gpu/drm/xe/xe_module.h
+++ b/drivers/gpu/drm/xe/xe_module.h
@@ -23,6 +23,7 @@ struct xe_modparam {
 	unsigned int max_vfs;
 #endif
 	unsigned int wedged_mode;
+	unsigned int num_pf_work;
 	u32 svm_notifier_size;
 };
 
diff --git a/drivers/gpu/drm/xe/xe_nvm.c b/drivers/gpu/drm/xe/xe_nvm.c
index 1ea67eaeae2437..51a1a431ad85f1 100644
--- a/drivers/gpu/drm/xe/xe_nvm.c
+++ b/drivers/gpu/drm/xe/xe_nvm.c
@@ -28,6 +28,7 @@
 static const struct intel_dg_nvm_region regions[INTEL_DG_NVM_REGIONS] = {
 	[0] = { .name = "DESCRIPTOR", },
 	[2] = { .name = "GSC", },
+	[5] = { .name = "SYSCTL", },
 	[9] = { .name = "PADDING", },
 	[11] = { .name = "OptionROM", },
 	[12] = { .name = "DAM", },
diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c
index ab09dcff58609d..a3484e943c553c 100644
--- a/drivers/gpu/drm/xe/xe_oa.c
+++ b/drivers/gpu/drm/xe/xe_oa.c
@@ -2583,10 +2583,11 @@ static u32 __hwe_oam_unit(struct xe_hw_engine *hwe)
 		return XE_OA_UNIT_INVALID;
 	else if (!IS_DGFX(gt_to_xe(hwe->gt)))
 		return XE_OAM_UNIT_SCMI_0;
-	else if (hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE)
-		return (hwe->instance / 2 & 0x1) + 1;
-	else if (hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE)
+	else if (hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE &&
+		 MEDIA_VERx100(gt_to_xe(hwe->gt)) < 3500)
 		return (hwe->instance & 0x1) + 1;
+	else
+		return (hwe->instance / 2 & 0x1) + 1;
 
 	return XE_OA_UNIT_INVALID;
 }
diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c
index a4986df8328d60..f219e9e3b73edc 100644
--- a/drivers/gpu/drm/xe/xe_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_pagefault.c
@@ -14,6 +14,8 @@
 #include "xe_gt_types.h"
 #include "xe_gt_stats.h"
 #include "xe_hw_engine.h"
+#include "xe_log.h"
+#include "xe_migrate.h"
 #include "xe_pagefault.h"
 #include "xe_pagefault_types.h"
 #include "xe_pm.h"
@@ -36,6 +38,73 @@
  * xe_pagefault.c implements the consumer layer.
  */
 
+/**
+ * DOC: Xe page fault cache
+ *
+ * Some Xe hardware can trigger “fault storms,” which are many page faults to
+ * the same address within a short period of time. An example is many EU threads
+ * faulting on the same page simultaneously. With the current page fault locking
+ * structure, only one page fault for a given address range can be processed at
+ * a time. This causes head-of-queue blocking across workers, killing
+ * parallelism. If the page fault handler must repeatedly look up resources
+ * (VMAs, ranges) to determine that the pages are valid for each fault in the
+ * storm, the time complexity grows rapidly.
+ *
+ * To address this, each page fault worker maintains a cache of the active fault
+ * being processed. Subsequent faults that hit in the cache are chained to the
+ * pending fault, and all chained faults are acknowledged once the initial fault
+ * completes. This alleviates head-of-queue blocking and quickly chains faults
+ * in the upper layers, avoiding expensive lookups in the main fault-handling
+ * path.
+ *
+ * Faults are buffered in the page fault queue in a way that provides stable
+ * storage for outstanding faults. In particular, faults may be chained directly
+ * while still resident in the queue storage (i.e., outside the worker’s current
+ * head/tail dequeue position). This allows the IRQ handler to match newly
+ * arrived faults against the per-worker cache and immediately chain cache hits
+ * onto the active fault under the queue lock, without allocating memory or
+ * waiting for the worker to pop the fault first.
+ *
+ * A per-fault state field is used to assert correctness of these invariants.
+ * The state tracks whether an entry is free, queued, chained, or currently
+ * active. Transitions are performed under the page fault queue lock, and the
+ * worker acknowledges faults by walking the chain and returning entries to the
+ * free state once they are complete.
+ */
+
+/**
+ * enum xe_pagefault_alloc_state - lifetime state for a page fault queue entry
+ * @XE_PAGEFAULT_ALLOC_STATE_FREE:
+ *	Entry is unused and may be overwritten by the producer, consumer retry
+ *	or requeue..
+ * @XE_PAGEFAULT_ALLOC_STATE_QUEUED:
+ *	Entry has been enqueued and may be dequeued by a worker.
+ * @XE_PAGEFAULT_ALLOC_STATE_ACTIVE:
+ *	Entry has been dequeued and is the worker's currently serviced fault.
+ *	The worker may attach additional faults to it via consumer.next.
+ * @XE_PAGEFAULT_ALLOC_STATE_CHAINED:
+ *	Entry is not independently serviced; it has been chained onto an
+ *	ACTIVE entry via consumer.next and will be acknowledged when the
+ *	leading fault completes.
+ * @XE_PAGEFAULT_ALLOC_STATE_COUNT:
+ *	Count of allocation states.
+ *
+ * The page fault queue provides stable storage for outstanding faults so the
+ * IRQ handler can chain new cache hits directly onto a worker's active fault.
+ * Because entries may remain referenced outside the consumer dequeue window,
+ * the producer must only write into entries in the FREE state.
+ *
+ * State transitions are protected by the page fault queue lock. Workers return
+ * entries to FREE after acknowledging the fault (either as ACTIVE or CHAINED).
+ */
+enum xe_pagefault_alloc_state {
+	XE_PAGEFAULT_ALLOC_STATE_FREE		= 0,
+	XE_PAGEFAULT_ALLOC_STATE_QUEUED		= 1,
+	XE_PAGEFAULT_ALLOC_STATE_CHAINED	= 2,
+	XE_PAGEFAULT_ALLOC_STATE_ACTIVE		= 3,
+	XE_PAGEFAULT_ALLOC_STATE_COUNT		= 4,
+};
+
 static int xe_pagefault_entry_size(void)
 {
 	/*
@@ -78,20 +147,26 @@ static int xe_pagefault_begin(struct drm_exec *exec, struct xe_vma *vma,
 }
 
 static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma,
-				   bool atomic)
+				   struct xe_pagefault *pf, bool atomic)
 {
 	struct xe_vm *vm = xe_vma_vm(vma);
 	struct xe_tile *tile = gt_to_tile(gt);
 	struct xe_validation_ctx ctx;
 	struct drm_exec exec;
 	struct dma_fence *fence;
-	int err, needs_vram;
+	int err = 0, needs_vram;
 
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 
 	needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic);
-	if (needs_vram < 0 || (needs_vram && xe_vma_is_userptr(vma)))
-		return needs_vram < 0 ? needs_vram : -EACCES;
+	if (needs_vram < 0) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK);
+		return needs_vram;
+	}
+	if (needs_vram && xe_vma_is_userptr(vma)) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR);
+		return -EACCES;
+	}
 
 	xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1);
 	xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB,
@@ -99,50 +174,66 @@ static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma,
 
 	trace_xe_vma_pagefault(vma);
 
+	guard(mutex)(&vma->fault_lock);
+
 	/* Check if VMA is valid, opportunistic check only */
 	if (xe_vm_has_valid_gpu_mapping(tile, vma->tile_present,
-					vma->tile_invalidated) && !atomic)
+					vma->tile_invalidated) && !atomic) {
+		xe_pagefault_set_start_addr(pf, xe_vma_start(vma));
+		xe_pagefault_set_end_addr(pf, xe_vma_end(vma));
 		return 0;
+	}
 
-retry_userptr:
-	if (xe_vma_is_userptr(vma) &&
-	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
-		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
+	do {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_NONE);
 
-		err = xe_vma_userptr_pin_pages(uvma);
-		if (err)
-			return err;
-	}
+		if (xe_vma_is_userptr(vma) &&
+		    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
+			struct xe_userptr_vma *uvma = to_userptr_vma(vma);
+
+			err = xe_vma_userptr_pin_pages(uvma);
+			if (err) {
+				xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN);
+				return err;
+			}
+		}
 
-	/* Lock VM and BOs dma-resv */
-	xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {});
-	drm_exec_until_all_locked(&exec) {
-		err = xe_pagefault_begin(&exec, vma, tile->mem.vram,
-					 needs_vram == 1);
-		drm_exec_retry_on_contention(&exec);
-		xe_validation_retry_on_oom(&ctx, &err);
-		if (err)
-			goto unlock_dma_resv;
-
-		/* Bind VMA only to the GT that has faulted */
-		trace_xe_vma_pf_bind(vma);
-		xe_vm_set_validation_exec(vm, &exec);
-		fence = xe_vma_rebind(vm, vma, BIT(tile->id));
-		xe_vm_set_validation_exec(vm, NULL);
-		if (IS_ERR(fence)) {
-			err = PTR_ERR(fence);
+		/* Lock VM and BOs dma-resv */
+		xe_validation_ctx_init(&ctx, &vm->xe->val, &exec,
+				       (struct xe_val_flags) {});
+		drm_exec_until_all_locked(&exec) {
+			err = xe_pagefault_begin(&exec, vma, tile->mem.vram,
+						 needs_vram == 1);
+			drm_exec_retry_on_contention(&exec);
 			xe_validation_retry_on_oom(&ctx, &err);
-			goto unlock_dma_resv;
+			if (err) {
+				xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_VALIDATE);
+				break;
+			}
+
+			/* Bind VMA only to the GT that has faulted */
+			trace_xe_vma_pf_bind(vma);
+			xe_vm_set_validation_exec(vm, &exec);
+			fence = xe_vma_rebind(vm, vma, BIT(tile->id));
+			xe_vm_set_validation_exec(vm, NULL);
+			if (IS_ERR(fence)) {
+				err = PTR_ERR(fence);
+				xe_validation_retry_on_oom(&ctx, &err);
+				xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_REBIND);
+				break;
+			}
 		}
-	}
+		xe_validation_ctx_fini(&ctx);
+	} while (err == -EAGAIN);
 
-	dma_fence_wait(fence, false);
-	dma_fence_put(fence);
+	if (!err) {
+		/* Give hint to immediately ack faults */
+		xe_pagefault_set_start_addr(pf, xe_vma_start(vma));
+		xe_pagefault_set_end_addr(pf, xe_vma_end(vma));
 
-unlock_dma_resv:
-	xe_validation_ctx_fini(&ctx);
-	if (err == -EAGAIN)
-		goto retry_userptr;
+		dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+	}
 
 	return err;
 }
@@ -153,16 +244,22 @@ xe_pagefault_access_is_atomic(enum xe_pagefault_access_type access_type)
 	return (access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK) == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC;
 }
 
-static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid)
+static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_pagefault *pf, u32 asid)
 {
+	struct xe_device *xe = gt_to_xe(pf->gt);
 	struct xe_vm *vm;
 
 	down_read(&xe->usm.lock);
 	vm = xa_load(&xe->usm.asid_to_vm, asid);
-	if (vm && xe_vm_in_fault_mode(vm))
-		xe_vm_get(vm);
-	else
+	if (!vm) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_NOT_FOUND);
+		vm = ERR_PTR(-EINVAL);
+	} else if (!xe_vm_in_fault_mode(vm)) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE);
 		vm = ERR_PTR(-EINVAL);
+	} else {
+		xe_vm_get(vm);
+	}
 	up_read(&xe->usm.lock);
 
 	return vm;
@@ -171,97 +268,378 @@ static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_device *xe, u32 asid)
 static int xe_pagefault_service(struct xe_pagefault *pf)
 {
 	struct xe_gt *gt = pf->gt;
-	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_vm *vm;
 	struct xe_vma *vma = NULL;
 	int err;
 	bool atomic;
+	u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id);
 
 	/* Producer flagged this fault to be nacked */
 	if (pf->consumer.fault_type_level == XE_PAGEFAULT_TYPE_LEVEL_NACK)
 		return -EFAULT;
 
-	vm = xe_pagefault_asid_to_vm(xe, pf->consumer.asid);
+	vm = xe_pagefault_asid_to_vm(pf, asid);
 	if (IS_ERR(vm))
 		return PTR_ERR(vm);
 
-	/*
-	 * TODO: Change to read lock? Using write lock for simplicity.
-	 */
-	down_write(&vm->lock);
+	xe_migrate_ulls_enter(gt_to_tile(gt)->migrate);
+
+	down_read(&vm->lock);
 
 	if (xe_vm_is_closed(vm)) {
 		err = -ENOENT;
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_CLOSED);
 		goto unlock_vm;
 	}
 
-	vma = xe_vm_find_vma_by_addr(vm, pf->consumer.page_addr);
+	vma = xe_vm_find_vma_by_addr(vm, xe_pagefault_addr(pf));
 	if (!vma) {
 		err = -EINVAL;
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NOT_FOUND);
 		goto unlock_vm;
 	}
 
 	if (xe_vma_read_only(vma) &&
 	    pf->consumer.access_type != XE_PAGEFAULT_ACCESS_TYPE_READ) {
 		err = -EPERM;
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION);
 		goto unlock_vm;
 	}
 
 	atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type);
 
 	if (xe_vma_is_cpu_addr_mirror(vma))
-		err = xe_svm_handle_pagefault(vm, vma, gt,
-					      pf->consumer.page_addr, atomic);
+		err = xe_svm_handle_pagefault(vm, vma, pf, gt,
+					      xe_pagefault_addr(pf), atomic);
 	else
-		err = xe_pagefault_handle_vma(gt, vma, atomic);
+		err = xe_pagefault_handle_vma(gt, vma, pf, atomic);
 
 unlock_vm:
-	if (!err)
-		vm->usm.last_fault_vma = vma;
-	up_write(&vm->lock);
+	up_read(&vm->lock);
 	xe_vm_put(vm);
 
 	return err;
 }
 
-static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue,
-				   struct xe_pagefault *pf)
+#define XE_PAGEFAULT_CACHE_START_INVALID	U64_MAX
+#define xe_pagefault_cache_start_invalidate(val)	\
+	(val = XE_PAGEFAULT_CACHE_START_INVALID)
+
+static void
+xe_pagefault_cache_invalidate(struct xe_pagefault_queue *pf_queue,
+			      struct xe_pagefault_work *pf_work)
 {
-	bool found_fault = false;
+	lockdep_assert_held(&pf_queue->lock);
 
-	spin_lock_irq(&pf_queue->lock);
-	if (pf_queue->tail != pf_queue->head) {
-		memcpy(pf, pf_queue->data + pf_queue->tail, sizeof(*pf));
-		pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) %
+	xe_pagefault_cache_start_invalidate(pf_work->cache.start);
+}
+
+static bool xe_pagefault_queue_full(struct xe_pagefault_queue *pf_queue)
+{
+	lockdep_assert_held(&pf_queue->lock);
+
+	return CIRC_SPACE(pf_queue->head, pf_queue->tail,
+			  pf_queue->size) <= xe_pagefault_entry_size();
+}
+
+static struct xe_pagefault *
+xe_pagefault_queue_add(struct xe_pagefault_queue *pf_queue,
+		       struct xe_pagefault *pf)
+{
+	struct xe_device *xe = container_of(pf_queue, typeof(*xe),
+					    usm.pf_queue);
+	struct xe_pagefault *lpf;
+
+	lockdep_assert_held(&pf_queue->lock);
+
+	do {
+		/* Not possible, warn on and drop page fault */
+		if (WARN_ON_ONCE(xe_pagefault_queue_full(pf_queue))) {
+			xe_log_err(xe, PAGEFAULT, -ENOSPC, "Queue full!\n");
+			return NULL;
+		}
+
+		lpf = (pf_queue->data + pf_queue->head);
+		pf_queue->head = (pf_queue->head + xe_pagefault_entry_size()) %
 			pf_queue->size;
-		found_fault = true;
+	} while (lpf->consumer.alloc_state != XE_PAGEFAULT_ALLOC_STATE_FREE);
+
+	xe_assert(xe, lpf != pf);
+	memcpy(lpf, pf, sizeof(*pf));
+	lpf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_QUEUED;
+
+	return lpf;
+}
+
+static struct xe_pagefault *
+xe_pagefault_queue_unchain_requeue(struct xe_pagefault_queue *pf_queue,
+				   struct xe_pagefault *pf, struct xe_gt *gt)
+{
+	struct xe_device *xe = container_of(pf_queue, typeof(*xe),
+					    usm.pf_queue);
+	struct xe_pagefault *next = pf->consumer.next, *lpf;
+
+	lockdep_assert_held(&pf_queue->lock);
+	xe_assert(xe, pf->consumer.alloc_state ==
+		  XE_PAGEFAULT_ALLOC_STATE_CHAINED);
+
+	xe_gt_stats_incr(gt, XE_GT_STATS_ID_CHAIN_MISMATCH_PAGEFAULT_COUNT, 1);
+
+	pf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_FREE;
+	lpf = xe_pagefault_queue_add(pf_queue, pf);
+	if (lpf) {
+		lpf->consumer.next = NULL;
+		lpf->consumer.fault_type_level |= XE_PAGEFAULT_REQUEUE_MASK;
+	}
+
+	return next;
+}
+
+static bool xe_pagefault_match(struct xe_pagefault *pf, u64 start,
+			       u64 end, u64 cache_asid)
+{
+	struct xe_device *xe = gt_to_xe(pf->gt);
+	u64 page_addr = xe_pagefault_addr(pf);
+	u32 pf_asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id);
+
+	xe_assert(xe, pf->consumer.alloc_state !=
+		  XE_PAGEFAULT_ALLOC_STATE_FREE);
+
+	return page_addr >= start && page_addr < end &&
+		pf_asid == cache_asid;
+}
+
+static bool xe_pagefault_try_chain(struct xe_pagefault_queue *pf_queue,
+				   struct xe_pagefault *pf)
+{
+	struct xe_device *xe = container_of(pf_queue, typeof(*xe),
+					    usm.pf_queue);
+	struct xe_pagefault_work *pf_work;
+	bool requeue = FIELD_GET(XE_PAGEFAULT_REQUEUE_MASK,
+				 pf->consumer.fault_type_level);
+	int i;
+
+	lockdep_assert_held(&pf_queue->lock);
+	xe_assert(xe, pf->consumer.alloc_state ==
+		  XE_PAGEFAULT_ALLOC_STATE_QUEUED);
+
+	/*
+	 * If this is a retry, we may already have a chain attached. In that
+	 * case, we cannot hit in the cache because chains cannot easily be
+	 * combined.
+	 */
+	if (pf->consumer.next)
+		return false;
+
+	for (i = 0, pf_work = xe->usm.pf_workers;
+	     i < xe->info.num_pf_work; ++i, ++pf_work) {
+		u64 start = pf_work->cache.start;
+		u64 end = requeue ? start + SZ_4K : pf_work->cache.end;
+		u32 asid = pf_work->cache.asid;
+
+		if (xe_pagefault_match(pf, start, end, asid)) {
+			xe_assert(xe, pf_work->cache.pf->consumer.alloc_state ==
+				  XE_PAGEFAULT_ALLOC_STATE_ACTIVE);
+
+			if (pf->producer.private !=
+			    pf_work->cache.pf->producer.private)
+				continue;
+
+			xe_gt_stats_incr(pf->gt,
+					 XE_GT_STATS_ID_CHAIN_PAGEFAULT_COUNT,
+					 1);
+
+			pf->consumer.alloc_state =
+				XE_PAGEFAULT_ALLOC_STATE_CHAINED;
+			pf->consumer.next = pf_work->cache.pf->consumer.next;
+			pf_work->cache.pf->consumer.next = pf;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static void xe_pagefault_queue_advance(struct xe_pagefault_queue *pf_queue)
+{
+	lockdep_assert_held(&pf_queue->lock);
+
+	pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) %
+		pf_queue->size;
+}
+
+static struct xe_pagefault *
+xe_pagefault_queue_tail_fault(struct xe_pagefault_queue *pf_queue)
+{
+	lockdep_assert_held(&pf_queue->lock);
+
+	return pf_queue->data + pf_queue->tail;
+}
+
+static bool xe_pagefault_queue_empty(struct xe_pagefault_queue *pf_queue)
+{
+	lockdep_assert_held(&pf_queue->lock);
+
+	return pf_queue->head == pf_queue->tail;
+}
+
+static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue,
+				   struct xe_pagefault **pf, int id)
+{
+	struct xe_device *xe = container_of(pf_queue, typeof(*xe),
+					    usm.pf_queue);
+	struct xe_pagefault_work *pf_work, *__pf_work;
+	struct xe_pagefault *lpf;
+	size_t align = SZ_2M;
+	int i;
+
+	guard(spinlock_irq)(&pf_queue->lock);
+
+	for (*pf = NULL; !*pf;) {
+		if (xe_pagefault_queue_empty(pf_queue))
+			return false;
+
+		lpf = xe_pagefault_queue_tail_fault(pf_queue);
+		xe_pagefault_queue_advance(pf_queue);
+
+		if (lpf->consumer.alloc_state !=
+		    XE_PAGEFAULT_ALLOC_STATE_QUEUED)
+			continue;
+
+		if (xe_pagefault_try_chain(pf_queue, lpf))
+			continue;
+
+		*pf = lpf;	/* Hand back page fault for processing */
 	}
-	spin_unlock_irq(&pf_queue->lock);
 
-	return found_fault;
+	/*
+	 * No cache hit; allocate a new cache entry. We assume most faults
+	 * within a 2M range will hit the same pages. If this assumption proves
+	 * false, the mismatched fault is requeued after the initial fault is
+	 * acknowledged.
+	 */
+	pf_work = xe->usm.pf_workers + id;
+	if (FIELD_GET(XE_PAGEFAULT_REQUEUE_MASK,
+		      lpf->consumer.fault_type_level))
+		align = SZ_4K;
+	pf_work->cache.start = ALIGN_DOWN(xe_pagefault_addr(lpf), align);
+	pf_work->cache.end = pf_work->cache.start + align;
+	pf_work->cache.asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, lpf->consumer.id);
+	pf_work->cache.pf = lpf;
+	lpf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_ACTIVE;
+
+	for (i = 0, __pf_work = xe->usm.pf_workers;
+	     i < xe->info.num_pf_work; ++i, ++__pf_work) {
+		u64 cache_start = __pf_work->cache.start;
+
+		if (__pf_work == pf_work)
+			continue;
+
+		if (cache_start != XE_PAGEFAULT_CACHE_START_INVALID) {
+			xe_gt_stats_incr(xe_root_mmio_gt(xe),
+					 XE_GT_STATS_ID_PARALLEL_PAGEFAULT_COUNT,
+					 1);
+			break;
+		}
+	}
+
+	/* Drain queue until empty or new fault found */
+	while (1) {
+		if (xe_pagefault_queue_empty(pf_queue))
+			break;
+
+		lpf = xe_pagefault_queue_tail_fault(pf_queue);
+
+		if (lpf->consumer.alloc_state !=
+		    XE_PAGEFAULT_ALLOC_STATE_QUEUED) {
+			xe_pagefault_queue_advance(pf_queue);
+			continue;
+		}
+
+		if (!xe_pagefault_try_chain(pf_queue, lpf))
+			break;
+
+		xe_pagefault_queue_advance(pf_queue);
+	}
+
+	return true;
+}
+
+static const char *xe_pagefault_error_to_str(enum xe_pagefault_error error)
+{
+	switch (error) {
+	case XE_PAGEFAULT_ERROR_NONE:
+		return "NONE";
+	case XE_PAGEFAULT_ERROR_VM_NOT_FOUND:
+		return "VM_NOT_FOUND";
+	case XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE:
+		return "VM_NOT_IN_FAULT_MODE";
+	case XE_PAGEFAULT_ERROR_VM_CLOSED:
+		return "VM_CLOSED";
+	case XE_PAGEFAULT_ERROR_VMA_NOT_FOUND:
+		return "VMA_NOT_FOUND";
+	case XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION:
+		return "READ_ONLY_VIOLATION";
+	case XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK:
+		return "VMA_NEEDS_VRAM_CHECK";
+	case XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR:
+		return "VMA_ATOMIC_USERPTR";
+	case XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN:
+		return "VMA_USERPTR_PIN";
+	case XE_PAGEFAULT_ERROR_VMA_VALIDATE:
+		return "VMA_VALIDATE";
+	case XE_PAGEFAULT_ERROR_VMA_REBIND:
+		return "VMA_REBIND";
+	case XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR:
+		return "SVM_GARBAGE_COLLECTOR";
+	case XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND:
+		return "SVM_RANGE_NOT_FOUND";
+	case XE_PAGEFAULT_ERROR_SVM_REBIND:
+		return "SVM_REBIND";
+	case XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK:
+		return "SVM_NEEDS_VRAM_CHECK";
+	case XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND:
+		return "SVM_VMA_NOT_FOUND";
+	case XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED:
+		return "SVM_SERVICE_FAILED";
+	default:
+		return "UNKNOWN";
+	}
 }
 
 static void xe_pagefault_print(struct xe_pagefault *pf)
 {
-	xe_gt_info(pf->gt, "\n\tASID: %d\n"
+	u8 engine_class = FIELD_GET(XE_PAGEFAULT_ENGINE_CLASS_MASK,
+				    pf->consumer.engine_class_instance);
+	u64 addr = xe_pagefault_addr(pf);
+
+	xe_gt_info(pf->gt, "\n\tASID: %lu\n"
 		   "\tFaulted Address: 0x%08x%08x\n"
 		   "\tFaultType: %lu\n"
 		   "\tAccessType: %lu\n"
 		   "\tFaultLevel: %lu\n"
 		   "\tEngineClass: %d %s\n"
-		   "\tEngineInstance: %d\n",
-		   pf->consumer.asid,
-		   upper_32_bits(pf->consumer.page_addr),
-		   lower_32_bits(pf->consumer.page_addr),
+		   "\tEngineInstance: %lu\n"
+		   "\tSRCID: 0x%02lx\n"
+		   "\tError: %s\n",
+		   FIELD_GET(XE_PAGEFAULT_ASID_MASK,
+			     pf->consumer.id),
+		   upper_32_bits(addr),
+		   lower_32_bits(addr),
 		   FIELD_GET(XE_PAGEFAULT_TYPE_MASK,
 			     pf->consumer.fault_type_level),
 		   FIELD_GET(XE_PAGEFAULT_ACCESS_TYPE_MASK,
 			     pf->consumer.access_type),
 		   FIELD_GET(XE_PAGEFAULT_LEVEL_MASK,
 			     pf->consumer.fault_type_level),
-		   pf->consumer.engine_class,
-		   xe_hw_engine_class_to_str(pf->consumer.engine_class),
-		   pf->consumer.engine_instance);
+		   engine_class,
+		   xe_hw_engine_class_to_str(engine_class),
+		   FIELD_GET(XE_PAGEFAULT_ENGINE_INSTANCE_MASK,
+			     pf->consumer.engine_class_instance),
+		   FIELD_GET(XE_PAGEFAULT_SRCID_MASK,
+			     pf->consumer.id),
+		   xe_pagefault_error_to_str(xe_pagefault_get_error(pf)));
 }
 
 static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *pf)
@@ -274,7 +652,8 @@ static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *p
 	 * mode, return VM anyways.
 	 */
 	down_read(&xe->usm.lock);
-	vm = xa_load(&xe->usm.asid_to_vm, pf->consumer.asid);
+	vm = xa_load(&xe->usm.asid_to_vm,
+		     FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id));
 	if (vm)
 		xe_vm_get(vm);
 	else
@@ -291,11 +670,22 @@ static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *p
 
 static void xe_pagefault_queue_work(struct work_struct *w)
 {
-	struct xe_pagefault_queue *pf_queue =
-		container_of(w, typeof(*pf_queue), worker);
-	struct xe_device *xe = pf_queue->xe;
-	struct xe_pagefault pf;
+	struct xe_pagefault_work *pf_work =
+		container_of(w, typeof(*pf_work), work);
+	struct xe_device *xe = pf_work->xe;
+	struct xe_pagefault_queue *pf_queue = &xe->usm.pf_queue;
+	struct xe_pagefault *pf;
+	ktime_t start = xe_gt_stats_ktime_get();
 	unsigned long threshold;
+	u64 cache_start = XE_PAGEFAULT_CACHE_START_INVALID, cache_end = 0;
+	u32 cache_asid = 0;
+
+	/*
+	 * A live VM holds a PM reference, but a torn-down VM does not.
+	 * Guard the entire worker loop to safely drain stale faults and
+	 * prevent autosuspends from desyncing batched CT flushes.
+	 */
+	guard(xe_pm_runtime)(xe);
 
 	/*
 	 * A live VM holds a PM reference, but a torn-down VM does not.
@@ -307,34 +697,91 @@ static void xe_pagefault_queue_work(struct work_struct *w)
 #define USM_QUEUE_MAX_RUNTIME_MS      20
 	threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
 
-	while (xe_pagefault_queue_pop(pf_queue, &pf)) {
-		int err;
+	while (xe_pagefault_queue_pop(pf_queue, &pf, pf_work->id)) {
+		const struct xe_pagefault_ops *ops = pf->producer.ops;
+		void *private = pf->producer.private;
+		struct xe_gt *gt = pf->gt;
+		u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id);
+		int err = 0;
+		bool invalidated = false;
+
+		/* Last fault same address, ack immediately */
+		if (xe_pagefault_match(pf, cache_start, cache_end, cache_asid)) {
+			xe_gt_stats_incr(gt, XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT, 1);
+			goto ack_fault;
+		}
 
-		if (!pf.gt)	/* Fault squashed during reset */
-			continue;
+		err = xe_pagefault_service(pf);
 
-		err = xe_pagefault_service(&pf);
 		if (err) {
-			xe_pagefault_save_to_vm(gt_to_xe(pf.gt), &pf);
-			if (!(pf.consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) {
-				xe_pagefault_print(&pf);
-				xe_gt_info(pf.gt, "Fault response: Unsuccessful %pe\n",
-					   ERR_PTR(err));
+			if (!(pf->consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) {
+				xe_pagefault_save_to_vm(gt_to_xe(gt), pf);
+				xe_pagefault_cache_start_invalidate(cache_start);
+				xe_pagefault_print(pf);
+				xe_log_err_info(pf->gt, PAGEFAULT, err, "Unsuccessful response\n");
 			} else {
-				xe_gt_stats_incr(pf.gt, XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1);
-				xe_gt_dbg(pf.gt, "Prefetch Fault response: Unsuccessful %pe\n",
+				xe_gt_stats_incr(pf->gt, XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1);
+				xe_gt_dbg(pf->gt, "Prefetch Fault response: Unsuccessful %pe\n",
 					  ERR_PTR(err));
 			}
+		} else {
+			/* Cache valid fault locally */
+			cache_start = xe_pagefault_start_addr(pf);
+			cache_end = xe_pagefault_end_addr(pf);
+			cache_asid = asid;
 		}
 
-		pf.producer.ops->ack_fault(&pf, err);
+ack_fault:
+		xe_assert(xe, pf->consumer.alloc_state ==
+			  XE_PAGEFAULT_ALLOC_STATE_ACTIVE);
+		xe_assert(xe, pf == pf_work->cache.pf);
+
+		ops->ack_fault_begin(private);
+		while (pf) {
+			xe_assert(xe, pf->consumer.alloc_state ==
+				  XE_PAGEFAULT_ALLOC_STATE_ACTIVE);
+			xe_assert(xe, ops == pf->producer.ops);
+			xe_assert(xe, gt == pf->gt);
+
+			ops->ack_fault(pf, err);
+
+			spin_lock_irq(&pf_queue->lock);
+
+			if (!invalidated) {
+				invalidated = true;
+				xe_pagefault_cache_invalidate(pf_queue,
+							      pf_work);
+			}
+
+			pf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_FREE;
+			pf = pf->consumer.next;
+
+			/*
+			 * Requeue chained faults which do not match the last
+			 * fault processed
+			 */
+			while (pf && !xe_pagefault_match(pf, cache_start,
+							 cache_end, cache_asid))
+				pf = xe_pagefault_queue_unchain_requeue(pf_queue, pf, gt);
+
+			/* Ensure resets are safe */
+			if (pf)
+				pf->consumer.alloc_state =
+					XE_PAGEFAULT_ALLOC_STATE_ACTIVE;
+
+			spin_unlock_irq(&pf_queue->lock);
+		}
+		ops->ack_fault_end(private);
 
 		if (time_after(jiffies, threshold)) {
-			queue_work(gt_to_xe(pf.gt)->usm.pf_wq, w);
+			queue_work(xe->usm.pagefault_wq, w);
 			break;
 		}
 	}
 #undef USM_QUEUE_MAX_RUNTIME_MS
+
+	xe_gt_stats_incr(xe_root_mmio_gt(xe), XE_GT_STATS_ID_PAGEFAULT_US,
+			 xe_gt_stats_ktime_us_delta(start));
 }
 
 static int xe_pagefault_queue_init(struct xe_device *xe,
@@ -376,7 +823,6 @@ static int xe_pagefault_queue_init(struct xe_device *xe,
 
 	pf_queue->xe = xe;
 	spin_lock_init(&pf_queue->lock);
-	INIT_WORK(&pf_queue->worker, xe_pagefault_queue_work);
 
 	pf_queue->data = drmm_kzalloc(&xe->drm, pf_queue->size, GFP_KERNEL);
 	if (!pf_queue->data)
@@ -389,7 +835,8 @@ static void xe_pagefault_fini(void *arg)
 {
 	struct xe_device *xe = arg;
 
-	destroy_workqueue(xe->usm.pf_wq);
+	destroy_workqueue(xe->usm.prefetch_wq);
+	destroy_workqueue(xe->usm.pagefault_wq);
 }
 
 /**
@@ -407,22 +854,39 @@ int xe_pagefault_init(struct xe_device *xe)
 	if (!xe->info.has_usm)
 		return 0;
 
-	xe->usm.pf_wq = alloc_workqueue("xe_page_fault_work_queue",
-					WQ_UNBOUND | WQ_HIGHPRI,
-					XE_PAGEFAULT_QUEUE_COUNT);
-	if (!xe->usm.pf_wq)
+	xe->usm.pagefault_wq = alloc_workqueue("xe_page_fault_work_queue",
+					       WQ_UNBOUND | WQ_HIGHPRI,
+					       xe->info.num_pf_work);
+	if (!xe->usm.pagefault_wq)
 		return -ENOMEM;
 
-	for (i = 0; i < XE_PAGEFAULT_QUEUE_COUNT; ++i) {
-		err = xe_pagefault_queue_init(xe, xe->usm.pf_queue + i);
-		if (err)
-			goto err_out;
+	xe->usm.prefetch_wq = alloc_workqueue("xe_prefetch_work_queue",
+					      WQ_UNBOUND,
+					      xe->info.num_pf_work);
+	if (!xe->usm.prefetch_wq) {
+		err = -ENOMEM;
+		goto err_pagefault_wq;
+	}
+
+	err = xe_pagefault_queue_init(xe, &xe->usm.pf_queue);
+	if (err)
+		goto err_out;
+
+	for (i = 0; i < xe->info.num_pf_work; ++i) {
+		struct xe_pagefault_work *pf_work = xe->usm.pf_workers + i;
+
+		pf_work->xe = xe;
+		pf_work->id = i;
+		xe_pagefault_cache_start_invalidate(pf_work->cache.start);
+		INIT_WORK(&pf_work->work, xe_pagefault_queue_work);
 	}
 
 	return devm_add_action_or_reset(xe->drm.dev, xe_pagefault_fini, xe);
 
 err_out:
-	destroy_workqueue(xe->usm.pf_wq);
+	destroy_workqueue(xe->usm.prefetch_wq);
+err_pagefault_wq:
+	destroy_workqueue(xe->usm.pagefault_wq);
 	return err;
 }
 
@@ -437,15 +901,23 @@ static void xe_pagefault_queue_reset(struct xe_device *xe, struct xe_gt *gt,
 
 	/* Squash all pending faults on the GT */
 
-	spin_lock_irq(&pf_queue->lock);
-	for (i = pf_queue->tail; i != pf_queue->head;
-	     i = (i + xe_pagefault_entry_size()) % pf_queue->size) {
+	guard(spinlock_irq)(&pf_queue->lock);
+
+	for (i = 0; i < pf_queue->size; i += xe_pagefault_entry_size()) {
 		struct xe_pagefault *pf = pf_queue->data + i;
+		bool active = pf->consumer.alloc_state ==
+			XE_PAGEFAULT_ALLOC_STATE_ACTIVE;
 
-		if (pf->gt == gt)
-			pf->gt = NULL;
+		if (pf->gt != gt || active) {
+			if (active)
+				pf->consumer.next = NULL;
+			continue;
+		}
+
+		pf->consumer.alloc_state =
+			XE_PAGEFAULT_ALLOC_STATE_FREE;
+		pf->consumer.next = NULL;
 	}
-	spin_unlock_irq(&pf_queue->lock);
 }
 
 /**
@@ -458,18 +930,18 @@ static void xe_pagefault_queue_reset(struct xe_device *xe, struct xe_gt *gt,
  */
 void xe_pagefault_reset(struct xe_device *xe, struct xe_gt *gt)
 {
-	int i;
-
-	for (i = 0; i < XE_PAGEFAULT_QUEUE_COUNT; ++i)
-		xe_pagefault_queue_reset(xe, gt, xe->usm.pf_queue + i);
+	xe_pagefault_queue_reset(xe, gt, &xe->usm.pf_queue);
 }
 
-static bool xe_pagefault_queue_full(struct xe_pagefault_queue *pf_queue)
+/*
+ * This function can race with multiple page fault producers, but worst case we
+ * stick a page fault on the same queue for consumption.
+ */
+static int xe_pagefault_work_index(struct xe_device *xe)
 {
-	lockdep_assert_held(&pf_queue->lock);
+	lockdep_assert_held(&xe->usm.pf_queue.lock);
 
-	return CIRC_SPACE(pf_queue->head, pf_queue->tail, pf_queue->size) <=
-		xe_pagefault_entry_size();
+	return xe->usm.current_pf_work++ % xe->info.num_pf_work;
 }
 
 /**
@@ -484,24 +956,95 @@ static bool xe_pagefault_queue_full(struct xe_pagefault_queue *pf_queue)
  */
 int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf)
 {
-	struct xe_pagefault_queue *pf_queue = xe->usm.pf_queue +
-		(pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT);
-	unsigned long flags;
-	bool full;
-
-	spin_lock_irqsave(&pf_queue->lock, flags);
-	full = xe_pagefault_queue_full(pf_queue);
-	if (!full) {
-		memcpy(pf_queue->data + pf_queue->head, pf, sizeof(*pf));
-		pf_queue->head = (pf_queue->head + xe_pagefault_entry_size()) %
-			pf_queue->size;
-		queue_work(xe->usm.pf_wq, &pf_queue->worker);
+	struct xe_pagefault_queue *pf_queue = &xe->usm.pf_queue;
+	struct xe_pagefault *lpf;
+	bool empty;
+
+	guard(spinlock_irqsave)(&pf_queue->lock);
+
+	empty = xe_pagefault_queue_empty(pf_queue);
+	lpf = xe_pagefault_queue_add(pf_queue, pf);
+	if (!lpf)
+		return -ENOSPC;
+
+	lpf->consumer.next = NULL;
+	if (xe_pagefault_try_chain(pf_queue, lpf)) {
+		xe_gt_stats_incr(pf->gt, XE_GT_STATS_ID_CHAIN_IRQ_PAGEFAULT_COUNT, 1);
+		if (empty) {
+			xe_gt_stats_incr(pf->gt,
+					 XE_GT_STATS_ID_CHAIN_DRAIN_IRQ_PAGEFAULT_COUNT, 1);
+			xe_pagefault_queue_advance(pf_queue);
+		}
 	} else {
-		drm_warn(&xe->drm,
-			 "PageFault Queue (%d) full, shouldn't be possible\n",
-			 pf->consumer.asid % XE_PAGEFAULT_QUEUE_COUNT);
+		int work_index = xe_pagefault_work_index(xe);
+
+		queue_work(xe->usm.pagefault_wq,
+			   &xe->usm.pf_workers[work_index].work);
 	}
-	spin_unlock_irqrestore(&pf_queue->lock, flags);
 
-	return full ? -ENOSPC : 0;
+	return 0;
+}
+
+/**
+ * xe_pagefault_print_info() - dump page fault queue/cache debug information
+ * @xe: Xe device
+ * @p: DRM printer to emit output to
+ *
+ * Print a snapshot of the page fault queue state for debugging. The output
+ * includes queue parameters (entry size, total size, head/tail), a histogram
+ * of per-entry allocation state values, and the validity of each per-worker
+ * page fault cache.
+ *
+ * This function is intended for debugfs and similar diagnostics. It acquires
+ * the page fault queue spinlock internally to serialize against IRQ-side
+ * producers and the worker consumer path, so callers must not hold the queue
+ * lock.
+ */
+void xe_pagefault_print_info(struct xe_device *xe, struct drm_printer *p)
+{
+	struct xe_pagefault_queue *pf_queue = &xe->usm.pf_queue;
+	struct xe_pagefault_work *pf_work;
+	static const char * const alloc_state_names[] = {
+		[XE_PAGEFAULT_ALLOC_STATE_FREE] = "free",
+		[XE_PAGEFAULT_ALLOC_STATE_QUEUED] = "queued",
+		[XE_PAGEFAULT_ALLOC_STATE_CHAINED] = "chained",
+		[XE_PAGEFAULT_ALLOC_STATE_ACTIVE] = "active",
+	};
+	u32 i, counts[XE_PAGEFAULT_ALLOC_STATE_COUNT] = {};
+
+	/* Driver load failure guard / USM not enabled guard */
+	if (!pf_queue->data)
+		return;
+
+	guard(spinlock_irq)(&pf_queue->lock);
+
+	drm_printf(p, "pagefault size: %u\n", xe_pagefault_entry_size());
+	drm_printf(p, "pagefault queue size: %u\n", pf_queue->size);
+	drm_printf(p, "pagefault queue head: %u\n", pf_queue->head);
+	drm_printf(p, "pagefault queue tail: %u\n", pf_queue->tail);
+
+	for (i = 0; i < pf_queue->size; i += xe_pagefault_entry_size()) {
+		struct xe_pagefault *pf = pf_queue->data + i;
+
+		if (pf->consumer.alloc_state >=
+		    XE_PAGEFAULT_ALLOC_STATE_COUNT) {
+			drm_printf(p, "pagefault[%u] corrupted alloc_state=%u\n",
+				   i, pf->consumer.alloc_state);
+			continue;
+		}
+
+		counts[pf->consumer.alloc_state]++;
+	}
+
+	for (i = 0; i < XE_PAGEFAULT_ALLOC_STATE_COUNT; ++i)
+		drm_printf(p, "pagefault queue %s count: %u\n",
+			   alloc_state_names[i], counts[i]);
+
+	for (i = 0, pf_work = xe->usm.pf_workers;
+	     i < xe->info.num_pf_work; ++i, ++pf_work) {
+		if (pf_work->cache.start == XE_PAGEFAULT_CACHE_START_INVALID)
+			drm_printf(p, "pagefault work[%u] cache invalid\n", i);
+		else
+			drm_printf(p, "pagefault work[%u] cache valid\n", i);
+	}
 }
diff --git a/drivers/gpu/drm/xe/xe_pagefault.h b/drivers/gpu/drm/xe/xe_pagefault.h
index bd0cdf9ed37f13..799c984dee84f5 100644
--- a/drivers/gpu/drm/xe/xe_pagefault.h
+++ b/drivers/gpu/drm/xe/xe_pagefault.h
@@ -6,6 +6,11 @@
 #ifndef _XE_PAGEFAULT_H_
 #define _XE_PAGEFAULT_H_
 
+#include <linux/bitfield.h>
+
+#include "xe_pagefault_types.h"
+
+struct drm_printer;
 struct xe_device;
 struct xe_gt;
 struct xe_pagefault;
@@ -16,4 +21,136 @@ void xe_pagefault_reset(struct xe_device *xe, struct xe_gt *gt);
 
 int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf);
 
+void xe_pagefault_print_info(struct xe_device *xe, struct drm_printer *p);
+
+/*
+ * consumer.page_addr is always page (4K) aligned, so the low bits are
+ * reserved and unused by the real address. Steal a byte of those bits to
+ * record an &enum xe_pagefault_error describing the high-level point at
+ * which servicing of the fault failed, so it can be reported by
+ * xe_pagefault_print(). All real address consumers of page_addr must go
+ * through xe_pagefault_addr() to mask off these reserved bits.
+ */
+#define XE_PAGEFAULT_ERROR_MASK		GENMASK_ULL(7, 0)
+
+/**
+ * xe_pagefault_set_error() - record the failure reason for a pagefault
+ * @pf: Pagefault entry
+ * @error: Failure reason
+ *
+ * Encodes @error into the reserved low bits of consumer.page_addr. Should be
+ * called at the high-level point a pagefault fails to service so
+ * xe_pagefault_print() can later report a mnemonic failure reason.
+ */
+static inline void
+xe_pagefault_set_error(struct xe_pagefault *pf, enum xe_pagefault_error error)
+{
+	pf->consumer.page_addr &= ~XE_PAGEFAULT_ERROR_MASK;
+	pf->consumer.page_addr |= FIELD_PREP(XE_PAGEFAULT_ERROR_MASK, error);
+}
+
+/**
+ * xe_pagefault_get_error() - read the failure reason for a pagefault
+ * @pf: Pagefault entry
+ *
+ * Return: The &enum xe_pagefault_error previously recorded via
+ * xe_pagefault_set_error(), or %XE_PAGEFAULT_ERROR_NONE if none was recorded.
+ */
+static inline enum xe_pagefault_error
+xe_pagefault_get_error(struct xe_pagefault *pf)
+{
+	return FIELD_GET(XE_PAGEFAULT_ERROR_MASK, pf->consumer.page_addr);
+}
+
+/**
+ * xe_pagefault_addr() - read the real faulted address for a pagefault
+ * @pf: Pagefault entry
+ *
+ * consumer.page_addr may have failure reason bits encoded into its reserved
+ * low bits by xe_pagefault_set_error(). This masks those bits off, returning
+ * the real page address. All accesses to the faulted address must go through
+ * this helper rather than reading consumer.page_addr directly.
+ *
+ * Return: The real (page aligned) faulted address.
+ */
+static inline u64 xe_pagefault_addr(struct xe_pagefault *pf)
+{
+	return pf->consumer.page_addr & ~XE_PAGEFAULT_ERROR_MASK;
+}
+
+#define XE_PAGEFAULT_END_ADDR_MASK	(~0xfffull)
+
+/**
+ * xe_pagefault_set_end_addr() - store serviced range end for a pagefault
+ * @pf: Pagefault entry
+ * @end_addr: Inclusive end address of the serviced fault range
+ *
+ * The pagefault consumer stores the resolved fault range so subsequent faults
+ * hitting the same range can be immediately acknowledged without re-running
+ * the full fault handling path.
+ *
+ * The end address shares storage with other consumer metadata and therefore
+ * must be masked with %XE_PAGEFAULT_END_ADDR_MASK before storing. Bits outside
+ * the mask are reserved for internal state tracking and must be preserved.
+ */
+static inline void
+xe_pagefault_set_end_addr(struct xe_pagefault *pf, u64 end_addr)
+{
+	pf->consumer.end_addr &= ~XE_PAGEFAULT_END_ADDR_MASK;
+	pf->consumer.end_addr |= end_addr;
+}
+
+/**
+ * xe_pagefault_end_addr() - read serviced range end for a pagefault
+ * @pf: Pagefault entry
+ *
+ * Returns the inclusive end address of the range previously recorded by
+ * xe_pagefault_set_end_addr(). Only the bits covered by
+ * %XE_PAGEFAULT_END_ADDR_MASK are returned; other bits in the storage are
+ * reserved for internal state.
+ *
+ * Return: End address of the serviced fault range.
+ */
+static inline u64 xe_pagefault_end_addr(struct xe_pagefault *pf)
+{
+	return pf->consumer.end_addr & XE_PAGEFAULT_END_ADDR_MASK;
+}
+
+#undef XE_PAGEFAULT_END_ADDR_MASK
+
+/**
+ * xe_pagefault_set_start_addr() - store serviced range start for a pagefault
+ * @pf: Pagefault entry
+ * @start_addr: Start address of the serviced fault range
+ *
+ * The pagefault consumer stores the resolved fault range so subsequent faults
+ * hitting the same range can be immediately acknowledged without re-running
+ * the full fault handling path.
+ *
+ * The start address shares storage with the failure reason recorded by
+ * xe_pagefault_set_error() and therefore must be masked with
+ * %XE_PAGEFAULT_ERROR_MASK before storing so any previously recorded error is
+ * preserved.
+ */
+static inline void
+xe_pagefault_set_start_addr(struct xe_pagefault *pf, u64 start_addr)
+{
+	pf->consumer.page_addr &= XE_PAGEFAULT_ERROR_MASK;
+	pf->consumer.page_addr |= (start_addr & ~XE_PAGEFAULT_ERROR_MASK);
+}
+
+/**
+ * xe_pagefault_start_addr() - read serviced range start for a pagefault
+ * @pf: Pagefault entry
+ *
+ * Returns the inclusive start address of the range previously recorded by
+ * xe_pagefault_set_start_addr().
+ *
+ * Return: Start address of the serviced fault range.
+ */
+static inline u64 xe_pagefault_start_addr(struct xe_pagefault *pf)
+{
+	return xe_pagefault_addr(pf);
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h
index f63a12aa0d4f7a..8ae6b9848bdc47 100644
--- a/drivers/gpu/drm/xe/xe_pagefault_types.h
+++ b/drivers/gpu/drm/xe/xe_pagefault_types.h
@@ -32,8 +32,97 @@ enum xe_pagefault_type {
 	XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION	= 2,
 };
 
+/**
+ * enum xe_pagefault_error - Xe page fault servicing error
+ *
+ * Uniquely identifies the high-level point at which servicing of a page
+ * fault failed. Encoded into the reserved low bits of
+ * &xe_pagefault.consumer.page_addr (which is always page aligned) so the
+ * failure reason can be threaded back up to xe_pagefault_print() without
+ * growing the size of struct xe_pagefault. See xe_pagefault_set_error() and
+ * xe_pagefault_error_to_str().
+ */
+enum xe_pagefault_error {
+	/** @XE_PAGEFAULT_ERROR_NONE: No error recorded */
+	XE_PAGEFAULT_ERROR_NONE = 0,
+	/**
+	 * @XE_PAGEFAULT_ERROR_VM_NOT_FOUND: VM lookup by ASID failed, e.g.
+	 * the VM's file descriptor was already closed and the ASID has been
+	 * torn down
+	 */
+	XE_PAGEFAULT_ERROR_VM_NOT_FOUND,
+	/**
+	 * @XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE: VM found by ASID lookup
+	 * but is not in fault mode
+	 */
+	XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE,
+	/** @XE_PAGEFAULT_ERROR_VM_CLOSED: VM found but already closed */
+	XE_PAGEFAULT_ERROR_VM_CLOSED,
+	/** @XE_PAGEFAULT_ERROR_VMA_NOT_FOUND: No VMA covers the faulted address */
+	XE_PAGEFAULT_ERROR_VMA_NOT_FOUND,
+	/**
+	 * @XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION: Write/atomic fault on a
+	 * read-only VMA
+	 */
+	XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION,
+	/**
+	 * @XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK: Failed determining if VMA
+	 * requires VRAM for an atomic access
+	 */
+	XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK,
+	/**
+	 * @XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR: Atomic access requires VRAM
+	 * but VMA is a userptr, which is unsupported
+	 */
+	XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR,
+	/** @XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN: Userptr page pin/repin failed */
+	XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN,
+	/**
+	 * @XE_PAGEFAULT_ERROR_VMA_VALIDATE: Failed to lock/validate VMA's BO
+	 * or migrate it to VRAM
+	 */
+	XE_PAGEFAULT_ERROR_VMA_VALIDATE,
+	/** @XE_PAGEFAULT_ERROR_VMA_REBIND: Failed to rebind VMA into page tables */
+	XE_PAGEFAULT_ERROR_VMA_REBIND,
+	/**
+	 * @XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR: Failed processing
+	 * pending SVM garbage collection (unmaps) prior to servicing the
+	 * fault
+	 */
+	XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR,
+	/**
+	 * @XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND: Failed to find or insert
+	 * an SVM range covering the faulted address
+	 */
+	XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND,
+	/** @XE_PAGEFAULT_ERROR_SVM_REBIND: Failed to rebind an SVM range into page tables */
+	XE_PAGEFAULT_ERROR_SVM_REBIND,
+	/**
+	 * @XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK: Failed determining if SVM
+	 * VMA requires VRAM for an atomic access
+	 */
+	XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK,
+	/**
+	 * @XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND: SVM VMA re-lookup after a
+	 * range split failed to find a covering VMA
+	 */
+	XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND,
+	/**
+	 * @XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED: SVM range population,
+	 * migration, or bind failed
+	 */
+	XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED,
+};
+
 /** struct xe_pagefault_ops - Xe pagefault ops (producer) */
 struct xe_pagefault_ops {
+	/**
+	 * @ack_fault_begin: Ack fault begin
+	 * @private: producer private data
+	 *
+	 * Page fault producer begins acknowledgment from the consumer.
+	 */
+	void (*ack_fault_begin)(void *private);
 	/**
 	 * @ack_fault: Ack fault
 	 * @pf: Page fault
@@ -43,6 +132,13 @@ struct xe_pagefault_ops {
 	 * sends the result to the HW/FW interface.
 	 */
 	void (*ack_fault)(struct xe_pagefault *pf, int err);
+	/**
+	 * @ack_fault_end: Ack fault end
+	 * @private: producer private data
+	 *
+	 * Page fault producer ends acknowledgment from the consumer.
+	 */
+	void (*ack_fault_end)(void *private);
 };
 
 /**
@@ -61,34 +157,63 @@ struct xe_pagefault {
 	/**
 	 * @consumer: State for the software handling the fault. Populated by
 	 * the producer and may be modified by the consumer to communicate
-	 * information back to the producer upon fault acknowledgment.
+	 * information back to the producer upon fault acknowledgment. After
+	 * fault acknowledgment, the producer should only access consumer fields
+	 * via well defined helpers.
 	 */
 	struct {
-		/** @consumer.page_addr: address of page fault */
-		u64 page_addr;
-		/** @consumer.asid: address space ID */
-		u32 asid;
 		/**
-		 * @consumer.access_type: access type and prefetch flag packed
-		 * into a u8.
+		 * @consumer.page_addr: address of page fault, populated by
+		 * consumer after fault completion
 		 */
-		u8 access_type;
+		u64 page_addr;
+		union {
+			struct {
+				/**
+				 * @consumer.alloc_state: page fault allocation
+				 * state
+				 */
+				u8 alloc_state;
+				/**
+				 * @consumer.access_type: access type, u8 rather
+				 * than enum to keep size compact
+				 */
+				u8 access_type;
 #define XE_PAGEFAULT_ACCESS_TYPE_MASK	GENMASK(1, 0)
 #define XE_PAGEFAULT_ACCESS_PREFETCH	BIT(7)
+				/**
+				 * @consumer.fault_type_level: fault type and
+				 * level, u8 rather than enum to keep size
+				 * compact
+				 */
+				u8 fault_type_level;
+#define XE_PAGEFAULT_TYPE_LEVEL_NACK		0xff	/* Producer indicates nack fault */
+#define XE_PAGEFAULT_LEVEL_MASK			GENMASK(2, 0)
+#define XE_PAGEFAULT_TYPE_MASK			GENMASK(6, 3)
+#define XE_PAGEFAULT_REQUEUE_MASK		BIT(7)
+				/** @consumer.engine_class_instance: engine class and instance */
+				u8 engine_class_instance;
+#define XE_PAGEFAULT_ENGINE_CLASS_MASK		GENMASK(3, 0)
+#define XE_PAGEFAULT_ENGINE_INSTANCE_MASK	GENMASK(7, 4)
+				/**
+				 * @consumer.id: address space ID and SRCID, folded into one
+				 * to keep size compact
+				 */
+				u32 id;
+#define XE_PAGEFAULT_ASID_MASK	GENMASK(23, 0)
+#define XE_PAGEFAULT_SRCID_MASK	GENMASK(31, 24)
+			};
+			/**
+			 * @consumer.end_addr: end address of page fault,
+			 * populated by consumer after fault completion
+			 */
+			u64 end_addr;
+		};
 		/**
-		 * @consumer.fault_type_level: fault type and level, u8 rather
-		 * than enum to keep size compact
+		 * @consumer.next: next pagefault chained to this fault,
+		 * protected by pf_queue lock
 		 */
-		u8 fault_type_level;
-#define XE_PAGEFAULT_TYPE_LEVEL_NACK		0xff	/* Producer indicates nack fault */
-#define XE_PAGEFAULT_LEVEL_MASK			GENMASK(3, 0)
-#define XE_PAGEFAULT_TYPE_MASK			GENMASK(7, 4)
-		/** @consumer.engine_class: engine class */
-		u8 engine_class;
-		/** @consumer.engine_instance: engine instance */
-		u8 engine_instance;
-		/** @consumer.reserved: reserved bits for future expansion */
-		u64 reserved;
+		struct xe_pagefault *next;
 	} consumer;
 	/**
 	 * @producer: State for the producer (i.e., HW/FW interface). Populated
@@ -132,10 +257,38 @@ struct xe_pagefault_queue {
 	u32 head;
 	/** @tail: Tail pointer in bytes, moved by consumer, protected by @lock */
 	u32 tail;
-	/** @lock: protects page fault queue */
+	/** @lock: protects page fault queue, workers caches */
 	spinlock_t lock;
-	/** @worker: to process page faults */
-	struct work_struct worker;
+};
+
+/**
+ * struct xe_pagefault_work - Xe page fault work item (consumer)
+ *
+ * Represents a worker that pops a &struct xe_pagefault from the page fault
+ * queue and processes it.
+ */
+struct xe_pagefault_work {
+	/** @xe: Back-pointer to the Xe device */
+	struct xe_device *xe;
+	/** @id: Identifier for this work item */
+	int id;
+	/**
+	 * @cache: Page fault cache for the currently processed fault
+	 *
+	 * Protected by the page fault queue lock.
+	 */
+	struct {
+		/** @cache.start: Start address of the current page fault */
+		u64 start;
+		/** @cache.end: End address of the current page fault */
+		u64 end;
+		/** @cache.asid: Address space ID of the current page fault */
+		u32 asid;
+		/** @cache.pf: Pointer to the current page fault */
+		struct xe_pagefault *pf;
+	} cache;
+	/** @work: Work item used to process the page fault */
+	struct work_struct work;
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index 5f2a0b19839d16..50e4920e8e08cc 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -25,6 +25,7 @@
 #include "xe_gt_printk.h"
 #include "xe_gt_sriov_vf.h"
 #include "xe_guc.h"
+#include "xe_log.h"
 #include "xe_mmio.h"
 #include "xe_module.h"
 #include "xe_pci_error.h"
@@ -370,6 +371,7 @@ static const __maybe_unused struct xe_device_desc pvc_desc = {
 	.has_display = false,
 	.has_drm_ras = true,
 	.has_gsc_nvm = 1,
+	.has_pt_mirror = 1,
 	.has_heci_gscfi = 1,
 	.max_gt_per_tile = 1,
 	.max_remote_tiles = 1,
@@ -459,6 +461,7 @@ static const struct xe_device_desc nvls_desc = {
 	.has_flat_ccs = 1,
 	.has_pre_prod_wa = 1,
 	.has_sriov = true,
+	.has_pxp = true,
 	.max_gt_per_tile = 2,
 	MULTI_LRC_MASK,
 	.va_bits = 48,
@@ -470,6 +473,7 @@ static const struct xe_device_desc cri_desc = {
 	PLATFORM(CRESCENTISLAND),
 	.dma_mask_size = 52,
 	.has_display = false,
+	.has_device_uid = true,
 	.has_drm_ras = true,
 	.has_flat_ccs = false,
 	.has_gsc_nvm = 1,
@@ -751,7 +755,6 @@ struct xe_probed_info {
  * Probe from the hardware the info required by xe_info_init_early().
  */
 static int xe_probe_info_early(struct xe_device *xe,
-			       const struct xe_device_desc *desc,
 			       struct xe_probed_info *probed_info)
 {
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
@@ -759,7 +762,7 @@ static int xe_probe_info_early(struct xe_device *xe,
 	probed_info->devid = pdev->device;
 	probed_info->revid = pdev->revision;
 
-	xe_step_platform_get(desc->platform, probed_info->revid, &probed_info->step);
+	xe_step_platform_get(xe->desc->platform, probed_info->revid, &probed_info->step);
 
 	return 0;
 }
@@ -769,10 +772,10 @@ static int xe_probe_info_early(struct xe_device *xe,
  * passed to the driver at probe time from PCI ID table.
  */
 static int xe_info_init_early(struct xe_device *xe,
-			      const struct xe_device_desc *desc,
-			      const struct xe_subplatform_desc *subplatform_desc,
 			      struct xe_probed_info *probed_info)
 {
+	const struct xe_subplatform_desc *subplatform_desc = xe->subplatform_desc;
+	const struct xe_device_desc *desc = xe->desc;
 	int err;
 
 	xe->info.devid = probed_info->devid;
@@ -791,6 +794,7 @@ static int xe_info_init_early(struct xe_device *xe,
 
 	xe->info.is_dgfx = desc->is_dgfx;
 	xe->info.has_cached_pt = desc->has_cached_pt;
+	xe->info.has_device_uid = desc->has_device_uid;
 	xe->info.has_drm_ras = desc->has_drm_ras;
 	xe->info.has_fan_control = desc->has_fan_control;
 	/* runtime fusing may force flat_ccs to disabled later */
@@ -806,6 +810,7 @@ static int xe_info_init_early(struct xe_device *xe,
 	xe->info.has_mert = desc->has_mert;
 	xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist;
 	xe->info.has_pre_prod_wa = desc->has_pre_prod_wa;
+	xe->info.has_pt_mirror = desc->has_pt_mirror;
 	xe->info.has_pxp = desc->has_pxp;
 	xe->info.has_soc_remapper_sysctrl = desc->has_soc_remapper_sysctrl;
 	xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem;
@@ -835,14 +840,13 @@ static int xe_info_init_early(struct xe_device *xe,
 }
 
 static void xe_probe_tile_count(struct xe_device *xe,
-				const struct xe_device_desc *desc,
 				struct xe_probed_info *probed_info)
 {
 	struct xe_mmio *mmio;
 	u8 tile_count;
 	u32 mtcfg;
 
-	probed_info->tile_count = 1 + desc->max_remote_tiles;
+	probed_info->tile_count = 1 + xe->desc->max_remote_tiles;
 
 	/*
 	 * Probe for tile count only for platforms that support multiple
@@ -945,9 +949,10 @@ static struct xe_gt *alloc_media_gt(struct xe_tile *tile,
 }
 
 static int xe_probe_ips(struct xe_device *xe,
-			const struct xe_device_desc *desc,
 			struct xe_probed_info *probed_info)
 {
+	const struct xe_device_desc *desc = xe->desc;
+
 	/*
 	 * If this platform supports GMD_ID, we'll detect the proper IP
 	 * descriptor to use from hardware registers.
@@ -988,14 +993,13 @@ static int xe_probe_ips(struct xe_device *xe,
  * Probe from the hardware the info required by xe_info_init().
  */
 static int xe_probe_info(struct xe_device *xe,
-			 const struct xe_device_desc *desc,
 			 struct xe_probed_info *probed_info)
 {
 	int err;
 
-	xe_probe_tile_count(xe, desc, probed_info);
+	xe_probe_tile_count(xe, probed_info);
 
-	err = xe_probe_ips(xe, desc, probed_info);
+	err = xe_probe_ips(xe, probed_info);
 	if (err)
 		return err;
 
@@ -1009,7 +1013,6 @@ static int xe_probe_info(struct xe_device *xe,
  * present in device info.
  */
 static int xe_info_init(struct xe_device *xe,
-			const struct xe_device_desc *desc,
 			struct xe_probed_info *probed_info)
 {
 	const struct xe_ip *graphics_ip;
@@ -1145,17 +1148,12 @@ static void xe_pci_remove(struct pci_dev *pdev)
  * caller. Therefore there is no consequence on those specific callers when
  * function error injection skips the whole function.
  */
+static int __xe_pci_probe(struct pci_dev *pdev, const struct xe_device_desc *desc);
 static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
-	struct xe_probed_info probed_info = {};
 	const struct xe_device_desc *desc = (const void *)ent->driver_data;
-	const struct xe_subplatform_desc *subplatform_desc;
-	struct xe_device *xe;
-	void *group;
 	int err;
 
-	subplatform_desc = find_subplatform(desc, pdev->device);
-
 	xe_configfs_check_device(pdev);
 
 	if (desc->require_force_probe && !id_forced(pdev->device)) {
@@ -1171,14 +1169,34 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	if (id_blocked(pdev->device)) {
-		dev_info(&pdev->dev, "Probe blocked for device [%04x:%04x].\n",
-			 pdev->vendor, pdev->device);
+		xe_log_info(pdev, PROBE, "driver loading blocked for device '%04x'\n",
+			    pdev->device);
 		return -ENODEV;
 	}
 
 	if (xe_display_driver_probe_defer(pdev))
 		return -EPROBE_DEFER;
 
+	err = __xe_pci_probe(pdev, desc);
+	if (err) {
+		xe_log_err_fatal(pdev, PROBE, err, "driver loading failed for device '%04x'\n",
+				 pdev->device);
+		return err;
+	}
+
+	return 0;
+}
+
+static int __xe_pci_probe(struct pci_dev *pdev, const struct xe_device_desc *desc)
+{
+	const struct xe_subplatform_desc *subplatform_desc;
+	struct xe_probed_info probed_info = {};
+	struct xe_device *xe;
+	void *group;
+	int err;
+
+	subplatform_desc = find_subplatform(desc, pdev->device);
+
 	/* Group all devres so xe_pci_error_slot_reset() can release them as a unit. */
 	group = devres_open_group(&pdev->dev, NULL, GFP_KERNEL);
 	if (!group)
@@ -1192,6 +1210,8 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (IS_ERR(xe))
 		return PTR_ERR(xe);
 
+	xe->desc = desc;
+	xe->subplatform_desc = subplatform_desc;
 	xe->devres_group = group;
 
 	pci_set_drvdata(pdev, &xe->drm);
@@ -1200,11 +1220,11 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_master(pdev);
 
-	err = xe_probe_info_early(xe, desc, &probed_info);
+	err = xe_probe_info_early(xe, &probed_info);
 	if (err)
 		return err;
 
-	err = xe_info_init_early(xe, desc, subplatform_desc, &probed_info);
+	err = xe_info_init_early(xe, &probed_info);
 	if (err)
 		return err;
 
@@ -1223,11 +1243,11 @@ static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		return err;
 
-	err = xe_probe_info(xe, desc, &probed_info);
+	err = xe_probe_info(xe, &probed_info);
 	if (err)
 		return err;
 
-	err = xe_info_init(xe, desc, &probed_info);
+	err = xe_info_init(xe, &probed_info);
 	if (err)
 		return err;
 
@@ -1364,7 +1384,9 @@ static int xe_pci_runtime_suspend(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct xe_device *xe = pdev_to_xe_device(pdev);
-	int err;
+	unsigned int flags;
+	bool pme_enabled;
+	int err, ret;
 
 	/*
 	 * We hold an additional reference to the runtime PM to keep PF in D0
@@ -1375,9 +1397,36 @@ static int xe_pci_runtime_suspend(struct device *dev)
 	xe_assert(xe, !IS_SRIOV_VF(xe));
 	xe_assert(xe, !pci_num_vf(pdev));
 
+	flags = memalloc_noreclaim_save();
+	pme_enabled = xe->pme.capable && !xe->d3cold.allowed &&
+		      pci_enable_wake(pdev, PCI_D3hot, true) == 0;
+	memalloc_noreclaim_restore(flags);
+
+	xe_pm_update_pme_enabled(xe, pme_enabled);
+
 	err = xe_pm_runtime_suspend(xe);
-	if (err)
+	if (err) {
+		if (pme_enabled) {
+			flags = memalloc_noreclaim_save();
+			pci_enable_wake(pdev, PCI_D3hot, false);
+			memalloc_noreclaim_restore(flags);
+
+			xe_pm_update_pme_enabled(xe, false);
+		}
+
 		return err;
+	}
+
+	err = xe_pm_wait_all_c6(xe);
+	if (err) {
+		drm_dbg(&xe->drm, "Resuming - GT C6 check failed!");
+		ret = xe_pm_runtime_resume(xe);
+		if (ret) {
+			drm_err(&xe->drm, "Resume failed after suspend was canceled");
+			return ret;
+		}
+		return err;
+	}
 
 	pci_save_state(pdev);
 
@@ -1406,6 +1455,8 @@ static int xe_pci_runtime_resume(struct device *dev)
 
 	pci_restore_state(pdev);
 
+	xe_pm_update_pme_enabled(xe, false);
+
 	if (xe->d3cold.allowed) {
 		err = pci_enable_device(pdev);
 		if (err)
diff --git a/drivers/gpu/drm/xe/xe_pci_error.c b/drivers/gpu/drm/xe/xe_pci_error.c
index e41af2ac7f23b0..79ce0c671549a9 100644
--- a/drivers/gpu/drm/xe/xe_pci_error.c
+++ b/drivers/gpu/drm/xe/xe_pci_error.c
@@ -7,6 +7,7 @@
 
 #include "xe_device.h"
 #include "xe_gt.h"
+#include "xe_log.h"
 #include "xe_pci.h"
 #include "xe_pm.h"
 #include "xe_printk.h"
@@ -83,6 +84,12 @@ static pci_ers_result_t xe_pci_error_mmio_enabled(struct pci_dev *pdev)
 	xe_info(xe, "PCI error: MMIO enabled\n");
 	action = xe_ras_process_errors(xe);
 
+	/* User wants to debug the error, prevent reset */
+	if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) {
+		xe_device_declare_wedged(xe);
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
 	return ras_action_to_pci_result(pdev, action);
 }
 
@@ -90,13 +97,15 @@ static pci_ers_result_t xe_pci_error_slot_reset(struct pci_dev *pdev)
 {
 	const struct pci_device_id *ent = pci_match_id(pdev->driver->id_table, pdev);
 	struct xe_device *xe = pdev_to_xe_device(pdev);
+	int err;
 
 	xe_info(xe, "PCI error: slot reset\n");
 
 	pci_restore_state(pdev);
 
-	if (pci_enable_device(pdev)) {
-		xe_err(xe, "Cannot re-enable PCI device after reset\n");
+	err = pci_enable_device(pdev);
+	if (err) {
+		xe_log_err_fatal(xe, PCI, err, "Cannot re-enable PCI device after reset\n");
 		return PCI_ERS_RESULT_DISCONNECT;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c
index 35e6b53e18ce23..31abcee5521150 100644
--- a/drivers/gpu/drm/xe/xe_pci_sriov.c
+++ b/drivers/gpu/drm/xe/xe_pci_sriov.c
@@ -82,16 +82,33 @@ static void pf_engine_activity_stats(struct xe_device *xe, unsigned int num_vfs,
 	}
 }
 
-static int resize_vf_vram_bar(struct xe_device *xe, int num_vfs)
+static int pf_resize_vf_vram_bar(struct xe_device *xe, int num_vfs)
 {
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+	char buf[10];
 	u32 sizes;
+	int size;
+	int err;
+
+	if (!IS_DGFX(xe))
+		return 0;
 
 	sizes = pci_iov_vf_bar_get_sizes(pdev, VF_LMEM_BAR, num_vfs);
 	if (!sizes)
 		return 0;
 
-	return pci_iov_vf_bar_set_size(pdev, VF_LMEM_BAR, __fls(sizes));
+	size = __fls(sizes);
+	string_get_size(pci_rebar_size_to_bytes(size), 1, STRING_UNITS_2, buf, sizeof(buf));
+
+	err = pci_iov_vf_bar_set_size(pdev, VF_LMEM_BAR, size);
+	if (err) {
+		xe_sriov_notice(xe, "Failed to resize VF BAR %u to %s (%pe)\n",
+				VF_LMEM_BAR, buf, ERR_PTR(err));
+		return err;
+	}
+
+	xe_sriov_dbg(xe, "VF LMEM BAR resized to %s\n", buf);
+	return 0;
 }
 
 static int pf_prepare_vfs_enabling(struct xe_device *xe)
@@ -142,11 +159,7 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs)
 	if (err < 0)
 		goto failed;
 
-	if (IS_DGFX(xe)) {
-		err = resize_vf_vram_bar(xe, num_vfs);
-		if (err)
-			xe_sriov_info(xe, "Failed to set VF LMEM BAR size: %d\n", err);
-	}
+	pf_resize_vf_vram_bar(xe, num_vfs);
 
 	err = pci_enable_sriov(pdev, num_vfs);
 	if (err < 0)
@@ -164,6 +177,7 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs)
 	return num_vfs;
 
 failed:
+	pf_resize_vf_vram_bar(xe, xe->sriov.pf.device_total_vfs);
 	xe_sriov_pf_unprovision_vfs(xe, num_vfs);
 	xe_pm_runtime_put(xe);
 	pf_finish_vfs_enabling(xe);
@@ -191,6 +205,8 @@ static int pf_disable_vfs(struct xe_device *xe)
 
 	pci_disable_sriov(pdev);
 
+	pf_resize_vf_vram_bar(xe, xe->sriov.pf.device_total_vfs);
+
 	xe_sriov_pf_reprovision_default(xe);
 
 	pf_reset_vfs(xe, num_vfs);
diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h
index fed509ff601e4b..448bbe5ba1d77b 100644
--- a/drivers/gpu/drm/xe/xe_pci_types.h
+++ b/drivers/gpu/drm/xe/xe_pci_types.h
@@ -40,6 +40,7 @@ struct xe_device_desc {
 
 	u8 has_cached_pt:1;
 	u8 has_display:1;
+	u8 has_device_uid:1;
 	u8 has_drm_ras:1;
 	u8 has_fan_control:1;
 	u8 has_flat_ccs:1;
@@ -52,8 +53,9 @@ struct xe_device_desc {
 	u8 has_mbx_power_limits:1;
 	u8 has_mbx_thermal_info:1;
 	u8 has_mert:1;
-	u8 has_pre_prod_wa:1;
 	u8 has_page_reclaim_hw_assist:1;
+	u8 has_pre_prod_wa:1;
+	u8 has_pt_mirror:1;
 	u8 has_pxp:1;
 	u8 has_soc_remapper_sysctrl:1;
 	u8 has_soc_remapper_telem:1;
diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c
index ccc3bdeed6bb22..266deecbb100a8 100644
--- a/drivers/gpu/drm/xe/xe_pcode.c
+++ b/drivers/gpu/drm/xe/xe_pcode.c
@@ -14,6 +14,7 @@
 #include "regs/xe_pmt.h"
 #include "xe_assert.h"
 #include "xe_device.h"
+#include "xe_log.h"
 #include "xe_mmio.h"
 #include "xe_pcode_api.h"
 #include "xe_pm.h"
@@ -61,9 +62,7 @@ static int pcode_mailbox_status(struct xe_tile *tile)
 	}
 
 	if (err) {
-		drm_err(&tile_to_xe(tile)->drm, "PCODE Mailbox failed: %d %s",
-			err_decode, err_str);
-
+		xe_log_err(tile, PCODE, err_decode, "Mailbox failed: %s\n", err_str);
 		return err_decode;
 	}
 
@@ -136,12 +135,12 @@ int xe_pcode_write64_timeout(struct xe_tile *tile, u32 mbox, u32 data0, u32 data
 	return err;
 }
 
-int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val, u32 *val1)
+int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val0, u32 *val1)
 {
 	int err;
 
 	mutex_lock(&tile->pcode.lock);
-	err = pcode_mailbox_rw(tile, mbox, val, val1, 1, true, false);
+	err = pcode_mailbox_rw(tile, mbox, val0, val1, PCODE_DEFAULT_TIMEOUT_MS, true, false);
 	mutex_unlock(&tile->pcode.lock);
 
 	return err;
@@ -149,7 +148,7 @@ int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val, u32 *val1)
 
 static int pcode_try_request(struct xe_tile *tile, u32 mbox,
 			     u32 request, u32 reply_mask, u32 reply,
-			     u32 *status, bool atomic, int timeout_us, bool locked)
+			     int *status, bool atomic, int timeout_us, bool locked)
 {
 	int slept, wait = 10;
 
@@ -197,8 +196,7 @@ static int pcode_try_request(struct xe_tile *tile, u32 mbox,
 int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request,
 		     u32 reply_mask, u32 reply, int timeout_base_ms)
 {
-	u32 status;
-	int ret;
+	int status, ret;
 
 	xe_tile_assert(tile, timeout_base_ms <= 3);
 
@@ -219,8 +217,7 @@ int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request,
 	 * requests, and for any quirks of the PCODE firmware that delays
 	 * the request completion.
 	 */
-	drm_err(&tile_to_xe(tile)->drm,
-		"PCODE timeout, retrying with preemption disabled\n");
+	xe_log_err(tile, PCODE, ret, "timeout, retrying with preemption disabled\n");
 	preempt_disable();
 	ret = pcode_try_request(tile, mbox, request, reply_mask, reply, &status,
 				true, 50 * 1000, true);
@@ -297,10 +294,10 @@ int xe_pcode_init_min_freq_table(struct xe_tile *tile, u32 min_gt_freq,
  */
 int xe_pcode_ready(struct xe_device *xe, bool locked)
 {
-	u32 status, request = DGFX_GET_INIT_STATUS;
 	struct xe_tile *tile = xe_device_get_root_tile(xe);
-	int timeout_us = 180000000; /* 3 min */
-	int ret;
+	long timeout_us = 3 * 60 * USEC_PER_SEC; /* 3 min */
+	u32 request = DGFX_GET_INIT_STATUS;
+	int status, ret;
 
 	if (xe->info.skip_pcode)
 		return 0;
@@ -320,8 +317,8 @@ int xe_pcode_ready(struct xe_device *xe, bool locked)
 		mutex_unlock(&tile->pcode.lock);
 
 	if (ret)
-		drm_err(&xe->drm,
-			"PCODE initialization timedout after: 3 min\n");
+		xe_log_err(tile, PCODE, ret, "initialization timedout after %ld seconds\n",
+			   timeout_us / USEC_PER_SEC);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h
index 8fb3e4ba13a6ed..7e43792b0037a6 100644
--- a/drivers/gpu/drm/xe/xe_pcode.h
+++ b/drivers/gpu/drm/xe/xe_pcode.h
@@ -18,12 +18,14 @@ struct xe_pcode_version {
 	u32 engg;
 };
 
+#define PCODE_DEFAULT_TIMEOUT_MS	10
+
 int xe_pcode_init_early(struct xe_tile *tile);
 int xe_pcode_probe_early(struct xe_device *xe);
 int xe_pcode_ready(struct xe_device *xe, bool locked);
 int xe_pcode_init_min_freq_table(struct xe_tile *tile, u32 min_gt_freq,
 				 u32 max_gt_freq);
-int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val, u32 *val1);
+int xe_pcode_read(struct xe_tile *tile, u32 mbox, u32 *val0, u32 *val1);
 int xe_pcode_write_timeout(struct xe_tile *tile, u32 mbox, u32 val,
 			   int timeout_ms);
 int xe_pcode_write64_timeout(struct xe_tile *tile, u32 mbox, u32 data0,
@@ -31,7 +33,7 @@ int xe_pcode_write64_timeout(struct xe_tile *tile, u32 mbox, u32 data0,
 int xe_get_pcode_version(struct xe_device *xe, struct xe_pcode_version *version);
 
 #define xe_pcode_write(tile, mbox, val) \
-	xe_pcode_write_timeout(tile, mbox, val, 1)
+	xe_pcode_write_timeout(tile, mbox, val, PCODE_DEFAULT_TIMEOUT_MS)
 
 int xe_pcode_request(struct xe_tile *tile, u32 mbox, u32 request,
 		     u32 reply_mask, u32 reply, int timeout_ms);
diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c
index a5289a9df8d257..daa7ba2df317a6 100644
--- a/drivers/gpu/drm/xe/xe_pm.c
+++ b/drivers/gpu/drm/xe/xe_pm.c
@@ -78,6 +78,8 @@
  * management (RPS).
  */
 
+#define HAS_PM_PME_SUPPORT(xe) (GRAPHICS_VERx100(xe) >= 3500)
+
 #ifdef CONFIG_LOCKDEP
 static struct lockdep_map xe_pm_runtime_d3cold_map = {
 	.name = "xe_rpm_d3cold_map"
@@ -384,6 +386,14 @@ int xe_pm_init_early(struct xe_device *xe)
 }
 ALLOW_ERROR_INJECTION(xe_pm_init_early, ERRNO); /* See xe_pci_probe() */
 
+static bool xe_pm_pci_pme_capable(struct xe_device *xe)
+{
+	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+
+	return HAS_PM_PME_SUPPORT(xe) ?
+		pci_pme_capable(pdev, PCI_D3hot) : false;
+}
+
 /**
  * xe_pm_probe() - Initialize Xe Power Management
  * @xe: the &xe_device instance
@@ -397,6 +407,16 @@ int xe_pm_probe(struct xe_device *xe)
 	xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe);
 	xe_dbg(xe, "d3cold: capable=%s\n", str_yes_no(xe->d3cold.capable));
 
+	xe->pme.capable = xe_pm_pci_pme_capable(xe);
+	xe_dbg(xe, "pme: capable=%s\n", str_yes_no(xe->pme.capable));
+
+	if (xe->pme.capable) {
+		int err = devm_device_init_wakeup(xe->drm.dev);
+
+		if (err)
+			return err;
+	}
+
 	return 0;
 }
 
@@ -650,6 +670,7 @@ int xe_pm_runtime_suspend(struct xe_device *xe)
 	return 0;
 
 out_resume:
+	xe_pm_update_pme_enabled(xe, false);
 	xe_display_pm_runtime_resume(xe);
 	xe_pxp_pm_resume(xe->pxp);
 out:
@@ -905,6 +926,11 @@ static bool xe_pm_suspending_or_resuming(struct xe_device *xe)
  * break scope-based handling, or when the lifetime of the runtime PM reference
  * does not match a specific scope (e.g., runtime PM obtained in one function
  * and released in a different one).
+ *
+ * This helper assumes the caller already holds a runtime PM reference and
+ * only warns when it cannot see one. After hot-unplug runtime PM is disabled
+ * and the check fails even when a reference is held, so callers that may run
+ * after unplug must guard it with drm_dev_enter()/drm_dev_exit() instead.
  */
 void xe_pm_runtime_get_noresume(struct xe_device *xe)
 {
@@ -988,6 +1014,30 @@ int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold)
 	return 0;
 }
 
+/**
+ * xe_pm_pme_enabled - get the current status of PME enabled
+ * @xe: xe device instance
+ *
+ * Returns: True if PME is enabled, false otherwise.
+ */
+bool xe_pm_pme_enabled(struct xe_device *xe)
+{
+	return xe->pme.enabled;
+}
+
+/**
+ * xe_pm_update_pme_enabled - Update the PME enabled state
+ * @xe: xe device instance
+ * @status: New PME enabled status
+ *
+ * Called during runtime suspend / resume. Status is set to True if PME is
+ * enabled during runtime_suspend. Cleared on runtime_resume.
+ */
+void xe_pm_update_pme_enabled(struct xe_device *xe, bool status)
+{
+	xe->pme.enabled = status;
+}
+
 /**
  * xe_pm_d3cold_allowed_toggle - Check conditions to toggle d3cold.allowed
  * @xe: xe device instance
@@ -1025,6 +1075,24 @@ void xe_pm_d3cold_allowed_toggle(struct xe_device *xe)
 	mutex_unlock(&xe->d3cold.lock);
 }
 
+/**
+ * xe_pm_wait_all_c6() - Wait for all GTs to enter C6.
+ * @xe: xe device instance
+ *
+ * Return: 0 on success, -EAGAIN on failure
+ */
+int xe_pm_wait_all_c6(struct xe_device *xe)
+{
+	struct xe_gt *gt;
+	u8 id;
+
+	for_each_gt(gt, xe, id)
+		if (xe_gt_idle_wait_for_c6(gt, 200))
+			return -EAGAIN;
+
+	return 0;
+}
+
 /**
  * xe_pm_module_init() - Perform xe_pm specific module initialization.
  *
diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h
index 6d5ab09cb7696f..16d4e6c629f11a 100644
--- a/drivers/gpu/drm/xe/xe_pm.h
+++ b/drivers/gpu/drm/xe/xe_pm.h
@@ -33,9 +33,12 @@ bool xe_pm_runtime_resume_and_get(struct xe_device *xe);
 void xe_pm_assert_unbounded_bridge(struct xe_device *xe);
 int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold);
 void xe_pm_d3cold_allowed_toggle(struct xe_device *xe);
+bool xe_pm_pme_enabled(struct xe_device *xe);
+void xe_pm_update_pme_enabled(struct xe_device *xe, bool status);
 bool xe_rpm_reclaim_safe(const struct xe_device *xe);
 struct task_struct *xe_pm_read_callback_task(struct xe_device *xe);
 int xe_pm_block_on_suspend(struct xe_device *xe);
+int xe_pm_wait_all_c6(struct xe_device *xe);
 void xe_pm_might_block_on_suspend(void);
 int xe_pm_module_init(void);
 
diff --git a/drivers/gpu/drm/xe/xe_printk.h b/drivers/gpu/drm/xe/xe_printk.h
index c5be2385aa9509..87f51d7edfa8b1 100644
--- a/drivers/gpu/drm/xe/xe_printk.h
+++ b/drivers/gpu/drm/xe/xe_printk.h
@@ -36,6 +36,9 @@
 #define xe_dbg(_xe, _fmt, ...) \
 	xe_printk((_xe), dbg, _fmt, ##__VA_ARGS__)
 
+#define xe_dbg_ratelimited(_xe, _fmt, ...) \
+	xe_printk((_xe), dbg_ratelimited, _fmt, ##__VA_ARGS__)
+
 #define xe_WARN_type(_xe, _type, _condition, _fmt, ...) \
 	drm_WARN##_type(&(_xe)->drm, _condition, _fmt, ## __VA_ARGS__)
 
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index a07316a45d79b7..6ad4bbaa186007 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -7,12 +7,12 @@
 
 #include "regs/xe_gtt_defs.h"
 #include "xe_bo.h"
+#include "xe_cpu_bind.h"
 #include "xe_device.h"
 #include "xe_drm_client.h"
 #include "xe_exec_queue.h"
 #include "xe_gt.h"
 #include "xe_gt_stats.h"
-#include "xe_migrate.h"
 #include "xe_page_reclaim.h"
 #include "xe_pat.h"
 #include "xe_pt_types.h"
@@ -206,6 +206,7 @@ unsigned int xe_pt_shift(unsigned int level)
  */
 void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
 {
+	bool added = false;
 	int i;
 
 	if (!pt)
@@ -213,7 +214,9 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
 
 	XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list));
 	xe_bo_unpin(pt->bo);
-	xe_bo_put_deferred(pt->bo, deferred);
+	xe_bo_put_deferred(pt->bo, deferred, &added);
+	if (added)
+		xe_assert(pt->bo->vm->xe, !kref_read(&pt->bo->ttm.base.refcount));
 
 	if (pt->level > 0 && pt->num_live) {
 		struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
@@ -236,9 +239,11 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
  */
 void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt)
 {
-	struct iosys_map *map = &pt->bo->vmap;
+	struct xe_bo *bo = pt->bo;
 
-	xe_map_memset(xe, map, 0, 0, SZ_4K);
+	xe_bo_assert_held(bo);
+	if (!iosys_map_is_null(&bo->vmap))
+		xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K);
 }
 
 /**
@@ -374,7 +379,7 @@ xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
 	entry->pt = parent;
 	entry->flags = 0;
 	entry->qwords = 0;
-	entry->pt_bo->update_index = -1;
+	entry->level = parent->level;
 
 	if (alloc_entries) {
 		entry->pt_entries = kmalloc_objs(*entry->pt_entries, XE_PDES);
@@ -799,7 +804,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 		.wupd.entries = entries,
 		.clear_pt = clear_pt,
 	};
-	struct xe_pt *pt = vm->pt_root[tile->id];
+	struct xe_pt *pt = xe_vm_pt_root(vm, tile->id);
 	int ret;
 	bool is_purged = false;
 
@@ -831,9 +836,12 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 			return -EAGAIN;
 		}
 		if (xe_svm_range_has_dma_mapping(range)) {
-			xe_res_first_dma(range->pages.dma_addr, 0,
-					 xe_svm_range_size(range),
-					 &curs);
+			const struct drm_pagemap_addr *addr;
+			bool contiguous;
+
+			addr = xe_svm_range_first_dma(range, &contiguous);
+			xe_res_first_dma(addr, 0, xe_svm_range_size(range),
+					 contiguous, &curs);
 			xe_svm_range_debug(range, "BIND PREPARE - MIXED");
 		} else {
 			xe_assert(xe, false);
@@ -865,10 +873,15 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 		xe_bo_assert_held(bo);
 
 	if (!xe_vma_is_null(vma) && !range && !is_purged) {
-		if (xe_vma_is_userptr(vma))
-			xe_res_first_dma(to_userptr_vma(vma)->userptr.pages.dma_addr, 0,
-					 xe_vma_size(vma), &curs);
-		else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
+		if (xe_vma_is_userptr(vma)) {
+			const struct drm_pagemap_addr *addr;
+			bool contiguous;
+
+			addr = drm_gpusvm_pages_first_dma(&to_userptr_vma(vma)->userptr.pages,
+							  &contiguous);
+			xe_res_first_dma(addr, 0, xe_vma_size(vma), contiguous,
+					 &curs);
+		} else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
 			xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
 				     xe_vma_size(vma), &curs);
 		else
@@ -992,6 +1005,11 @@ static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset,
 	return 0;
 }
 
+static bool pt_mirroring_disabled_for_tile(struct xe_vm *vm, u8 tile_id)
+{
+	return !vm->xe->info.has_pt_mirror && tile_id;
+}
+
 static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
 	.pt_entry = xe_pt_zap_ptes_entry,
 };
@@ -1033,6 +1051,9 @@ bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
 	if (!(pt_mask & BIT(tile->id)))
 		return false;
 
+	if (pt_mirroring_disabled_for_tile(xe_vma_vm(vma), tile->id))
+		return true;
+
 	(void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
 				xe_vma_end(vma), &xe_walk.base);
 
@@ -1085,6 +1106,9 @@ bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
 	if (!(pt_mask & BIT(tile->id)))
 		return false;
 
+	if (pt_mirroring_disabled_for_tile(vm, tile->id))
+		return true;
+
 	(void)xe_pt_walk_shared(&pt->base, pt->level, xe_svm_range_start(range),
 				xe_svm_range_end(range), &xe_walk.base);
 
@@ -1092,32 +1116,18 @@ bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
 }
 
 static void
-xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
-		       struct iosys_map *map, void *data,
-		       u32 qword_ofs, u32 num_qwords,
+xe_vm_populate_pgtable(struct xe_tile *tile, struct iosys_map *map,
 		       const struct xe_vm_pgtable_update *update)
 {
 	struct xe_pt_entry *ptes = update->pt_entries;
-	u64 *ptr = data;
 	u32 i;
 
-	/*
-	 * @qword_ofs is the absolute entry offset within the page table, while
-	 * @ptes is indexed relative to @update->ofs (its first entry). The GPU
-	 * path (write_pgtable) splits a single update into MAX_PTE_PER_SDI-sized
-	 * chunks, calling this with an advancing @qword_ofs but a fresh @data
-	 * pointer per chunk, so translate back into a @ptes index rather than
-	 * assuming the chunk starts at ptes[0].
-	 */
-	for (i = 0; i < num_qwords; i++) {
-		u32 idx = qword_ofs - update->ofs + i;
+	xe_assert(tile_to_xe(tile), map);
+	xe_assert(tile_to_xe(tile), !iosys_map_is_null(map));
 
-		if (map)
-			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
-				  sizeof(u64), u64, ptes[idx].pte);
-		else
-			ptr[i] = ptes[idx].pte;
-	}
+	for (i = 0; i < update->qwords; i++)
+		xe_map_wr(tile_to_xe(tile), map, (update->ofs + i) *
+			  sizeof(u64), u64, ptes[i].pte);
 }
 
 static void xe_pt_cancel_bind(struct xe_vma *vma,
@@ -1401,11 +1411,9 @@ static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
 }
 
 static int xe_pt_vm_dependencies(struct xe_sched_job *job,
-				 struct xe_tlb_inval_job *ijob,
-				 struct xe_tlb_inval_job *mjob,
+				 struct xe_tlb_inval_job **ijobs,
 				 struct xe_vm *vm,
 				 struct xe_vma_ops *vops,
-				 struct xe_vm_pgtable_update_ops *pt_update_ops,
 				 struct xe_range_fence_tree *rftree)
 {
 	struct xe_range_fence *rtfence;
@@ -1418,20 +1426,22 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job,
 	if (!job && !no_in_syncs(vops->syncs, vops->num_syncs))
 		return -ETIME;
 
-	if (!job && !xe_exec_queue_is_idle(pt_update_ops->q))
+	if (!job && !xe_exec_queue_is_idle(vops->q, 0))
 		return -ETIME;
 
-	if (pt_update_ops->wait_vm_bookkeep || pt_update_ops->wait_vm_kernel) {
-		err = job_test_add_deps(job, xe_vm_resv(vm),
-					pt_update_ops->wait_vm_bookkeep ?
-					DMA_RESV_USAGE_BOOKKEEP :
-					DMA_RESV_USAGE_KERNEL);
+	if (vops->flags & (XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP |
+			   XE_VMA_OPS_FLAG_WAIT_VM_KERNEL)) {
+		enum dma_resv_usage usage = DMA_RESV_USAGE_KERNEL;
+
+		if (vops->flags & XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP)
+			usage = DMA_RESV_USAGE_BOOKKEEP;
+
+		err = job_test_add_deps(job, xe_vm_resv(vm), usage);
 		if (err)
 			return err;
 	}
 
-	rtfence = xe_range_fence_tree_first(rftree, pt_update_ops->start,
-					    pt_update_ops->last);
+	rtfence = xe_range_fence_tree_first(rftree, vops->start, vops->last);
 	while (rtfence) {
 		fence = rtfence->fence;
 
@@ -1449,9 +1459,8 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job,
 				return err;
 		}
 
-		rtfence = xe_range_fence_tree_next(rtfence,
-						   pt_update_ops->start,
-						   pt_update_ops->last);
+		rtfence = xe_range_fence_tree_next(rtfence, vops->start,
+						   vops->last);
 	}
 
 	list_for_each_entry(op, &vops->list, link) {
@@ -1464,14 +1473,11 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job,
 		err = xe_sync_entry_add_deps(&vops->syncs[i], job);
 
 	if (job) {
-		if (ijob) {
-			err = xe_tlb_inval_job_alloc_dep(ijob);
-			if (err)
-				return err;
-		}
+		for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) {
+			if (!ijobs[i])
+				continue;
 
-		if (mjob) {
-			err = xe_tlb_inval_job_alloc_dep(mjob);
+			err = xe_tlb_inval_job_alloc_dep(ijobs[i]);
 			if (err)
 				return err;
 		}
@@ -1480,17 +1486,14 @@ static int xe_pt_vm_dependencies(struct xe_sched_job *job,
 	return err;
 }
 
-static int xe_pt_pre_commit(struct xe_migrate_pt_update *pt_update)
+static int xe_pt_pre_commit(struct xe_cpu_bind_pt_update *pt_update)
 {
 	struct xe_vma_ops *vops = pt_update->vops;
 	struct xe_vm *vm = vops->vm;
-	struct xe_range_fence_tree *rftree = &vm->rftree[pt_update->tile_id];
-	struct xe_vm_pgtable_update_ops *pt_update_ops =
-		&vops->pt_update_ops[pt_update->tile_id];
+	struct xe_range_fence_tree *rftree = &vm->rftree;
 
-	return xe_pt_vm_dependencies(pt_update->job, pt_update->ijob,
-				     pt_update->mjob, vm, pt_update->vops,
-				     pt_update_ops, rftree);
+	return xe_pt_vm_dependencies(pt_update->job, pt_update->ijobs,
+				     vm, vops, rftree);
 }
 
 #if IS_ENABLED(CONFIG_DRM_GPUSVM)
@@ -1550,8 +1553,7 @@ static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
 
 #endif
 
-static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
-			     struct xe_vm_pgtable_update_ops *pt_update)
+static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma)
 {
 	struct xe_userptr_vma *uvma;
 	unsigned long notifier_seq;
@@ -1581,8 +1583,7 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
 	return 0;
 }
 
-static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op,
-				struct xe_vm_pgtable_update_ops *pt_update)
+static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op)
 {
 	int err = 0;
 
@@ -1593,13 +1594,13 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op,
 		if (!op->map.immediate && xe_vm_in_fault_mode(vm))
 			break;
 
-		err = vma_check_userptr(vm, op->map.vma, pt_update);
+		err = vma_check_userptr(vm, op->map.vma);
 		break;
 	case DRM_GPUVA_OP_REMAP:
 		if (op->remap.prev && !op->remap.skip_prev)
-			err = vma_check_userptr(vm, op->remap.prev, pt_update);
+			err = vma_check_userptr(vm, op->remap.prev);
 		if (!err && op->remap.next && !op->remap.skip_next)
-			err = vma_check_userptr(vm, op->remap.next, pt_update);
+			err = vma_check_userptr(vm, op->remap.next);
 		break;
 	case DRM_GPUVA_OP_UNMAP:
 		break;
@@ -1619,7 +1620,7 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op,
 				}
 			}
 		} else {
-			err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va), pt_update);
+			err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va));
 		}
 		break;
 #if IS_ENABLED(CONFIG_DRM_XE_GPUSVM)
@@ -1645,12 +1646,10 @@ static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op,
 	return err;
 }
 
-static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
+static int xe_pt_svm_userptr_pre_commit(struct xe_cpu_bind_pt_update *pt_update)
 {
 	struct xe_vm *vm = pt_update->vops->vm;
 	struct xe_vma_ops *vops = pt_update->vops;
-	struct xe_vm_pgtable_update_ops *pt_update_ops =
-		&vops->pt_update_ops[pt_update->tile_id];
 	struct xe_vma_op *op;
 	int err;
 
@@ -1661,7 +1660,7 @@ static int xe_pt_svm_userptr_pre_commit(struct xe_migrate_pt_update *pt_update)
 	xe_pt_svm_userptr_notifier_lock(vm);
 
 	list_for_each_entry(op, &vops->list, link) {
-		err = op_check_svm_userptr(vm, op, pt_update_ops);
+		err = op_check_svm_userptr(vm, op);
 		if (err) {
 			xe_pt_svm_userptr_notifier_unlock(vm);
 			break;
@@ -1999,7 +1998,7 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
 		.wupd.entries = entries,
 		.prl = pt_update_op->prl,
 	};
-	struct xe_pt *pt = vm->pt_root[tile->id];
+	struct xe_pt *pt = xe_vm_pt_root(vm, tile->id);
 
 	(void)xe_pt_walk_shared(&pt->base, pt->level, start, end,
 				&xe_walk.base);
@@ -2008,24 +2007,23 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
 }
 
 static void
-xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
-				  struct xe_tile *tile, struct iosys_map *map,
-				  void *ptr, u32 qword_ofs, u32 num_qwords,
-				  const struct xe_vm_pgtable_update *update)
+xe_pt_clear_pgtable_callback(struct xe_vm *vm, struct xe_tile *tile,
+			     struct iosys_map *map,
+			     const struct xe_vm_pgtable_update *update)
 {
-	struct xe_vm *vm = pt_update->vops->vm;
-	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
+	u64 empty = __xe_pt_empty_pte(tile, vm, update->level);
 	int i;
 
-	if (map && map->is_iomem)
-		for (i = 0; i < num_qwords; ++i)
-			xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) *
+	xe_assert(vm->xe, map);
+	xe_assert(vm->xe, !iosys_map_is_null(map));
+
+	if (map->is_iomem)
+		for (i = 0; i < update->qwords; ++i)
+			xe_map_wr(tile_to_xe(tile), map, (update->ofs + i) *
 				  sizeof(u64), u64, empty);
-	else if (map)
-		memset64(map->vaddr + qword_ofs * sizeof(u64), empty,
-			 num_qwords);
 	else
-		memset64(ptr, empty, num_qwords);
+		memset64(map->vaddr + update->ofs * sizeof(u64), empty,
+			 update->qwords);
 }
 
 static void xe_pt_abort_unbind(struct xe_vma *vma,
@@ -2080,13 +2078,40 @@ xe_pt_commit_prepare_unbind(struct xe_vma *vma,
 	}
 }
 
+static struct xe_vm_pgtable_update_op *
+to_pt_op(struct xe_vm_pgtable_update_ops *pt_update_ops, u32 op_idx)
+{
+	return &pt_update_ops->pt_job_ops->ops[op_idx];
+}
+
+static u32
+get_current_op(struct xe_vm_pgtable_update_ops *pt_update_ops)
+{
+	if (!pt_update_ops->pt_job_ops)
+		return 0;
+
+	return pt_update_ops->pt_job_ops->current_op;
+}
+
+static struct xe_vm_pgtable_update_op *
+to_current_pt_op(struct xe_vm_pgtable_update_ops *pt_update_ops)
+{
+	return to_pt_op(pt_update_ops, get_current_op(pt_update_ops));
+}
+
+static void
+incr_current_op(struct xe_vm_pgtable_update_ops *pt_update_ops)
+{
+	++pt_update_ops->pt_job_ops->current_op;
+}
+
 static void
 xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
 				 u64 start, u64 end)
 {
 	u64 last;
-	u32 current_op = pt_update_ops->current_op;
-	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+	struct xe_vm_pgtable_update_op *pt_op =
+		to_current_pt_op(pt_update_ops);
 	int i, level = 0;
 
 	for (i = 0; i < pt_op->num_entries; i++) {
@@ -2121,8 +2146,8 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 			   struct xe_vm_pgtable_update_ops *pt_update_ops,
 			   struct xe_vma *vma, bool invalidate_on_bind)
 {
-	u32 current_op = pt_update_ops->current_op;
-	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+	struct xe_vm_pgtable_update_op *pt_op =
+		to_current_pt_op(pt_update_ops);
 	int err;
 
 	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
@@ -2151,7 +2176,7 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 		xe_pt_update_ops_rfence_interval(pt_update_ops,
 						 xe_vma_start(vma),
 						 xe_vma_end(vma));
-		++pt_update_ops->current_op;
+		incr_current_op(pt_update_ops);
 		pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
 
 		/*
@@ -2194,8 +2219,8 @@ static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile,
 			      struct xe_vm_pgtable_update_ops *pt_update_ops,
 			      struct xe_vma *vma, struct xe_svm_range *range)
 {
-	u32 current_op = pt_update_ops->current_op;
-	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+	struct xe_vm_pgtable_update_op *pt_op =
+		to_current_pt_op(pt_update_ops);
 	int err;
 
 	xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma));
@@ -2219,7 +2244,7 @@ static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile,
 		xe_pt_update_ops_rfence_interval(pt_update_ops,
 						 xe_svm_range_start(range),
 						 xe_svm_range_end(range));
-		++pt_update_ops->current_op;
+		incr_current_op(pt_update_ops);
 		pt_update_ops->needs_svm_lock = true;
 
 		pt_op->vma = vma;
@@ -2237,8 +2262,8 @@ static int unbind_op_prepare(struct xe_tile *tile,
 			     struct xe_vma *vma)
 {
 	struct xe_device *xe = tile_to_xe(tile);
-	u32 current_op = pt_update_ops->current_op;
-	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+	struct xe_vm_pgtable_update_op *pt_op =
+		to_current_pt_op(pt_update_ops);
 	int err;
 
 	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
@@ -2277,7 +2302,7 @@ static int unbind_op_prepare(struct xe_tile *tile,
 				pt_op->num_entries, false);
 	xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma),
 					 xe_vma_end(vma));
-	++pt_update_ops->current_op;
+	incr_current_op(pt_update_ops);
 	pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
 	pt_update_ops->needs_invalidation = true;
 
@@ -2317,8 +2342,8 @@ static int unbind_range_prepare(struct xe_vm *vm,
 				struct xe_vm_pgtable_update_ops *pt_update_ops,
 				struct xe_svm_range *range)
 {
-	u32 current_op = pt_update_ops->current_op;
-	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
+	struct xe_vm_pgtable_update_op *pt_op =
+		to_current_pt_op(pt_update_ops);
 
 	if (!(range->tile_present & BIT(tile->id)))
 		return 0;
@@ -2339,7 +2364,7 @@ static int unbind_range_prepare(struct xe_vm *vm,
 				pt_op->num_entries, false);
 	xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range),
 					 xe_svm_range_end(range));
-	++pt_update_ops->current_op;
+	incr_current_op(pt_update_ops);
 	pt_update_ops->needs_svm_lock = true;
 	pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) ||
 		xe_vm_has_valid_gpu_mapping(tile, range->tile_present,
@@ -2354,6 +2379,7 @@ static int unbind_range_prepare(struct xe_vm *vm,
 
 static int op_prepare(struct xe_vm *vm,
 		      struct xe_tile *tile,
+		      struct xe_vma_ops *vops,
 		      struct xe_vm_pgtable_update_ops *pt_update_ops,
 		      struct xe_vma_op *op)
 {
@@ -2370,7 +2396,7 @@ static int op_prepare(struct xe_vm *vm,
 
 		err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma,
 				      op->map.invalidate_on_bind);
-		pt_update_ops->wait_vm_kernel = true;
+		vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_KERNEL;
 		break;
 	case DRM_GPUVA_OP_REMAP:
 	{
@@ -2384,12 +2410,12 @@ static int op_prepare(struct xe_vm *vm,
 		if (!err && op->remap.prev && !op->remap.skip_prev) {
 			err = bind_op_prepare(vm, tile, pt_update_ops,
 					      op->remap.prev, false);
-			pt_update_ops->wait_vm_bookkeep = true;
+			vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
 		}
 		if (!err && op->remap.next && !op->remap.skip_next) {
 			err = bind_op_prepare(vm, tile, pt_update_ops,
 					      op->remap.next, false);
-			pt_update_ops->wait_vm_bookkeep = true;
+			vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
 		}
 		break;
 	}
@@ -2414,12 +2440,18 @@ static int op_prepare(struct xe_vm *vm,
 			xa_for_each(&op->prefetch_range.range, i, range) {
 				err = bind_range_prepare(vm, tile, pt_update_ops,
 							 vma, range);
+				/*
+				 * Don't tell user space to retry, rather let
+				 * page faults fixup the pages.
+				 */
+				if (err == -EAGAIN)
+					err = -ENODATA;
 				if (err)
 					return err;
 			}
 		} else {
 			err = bind_op_prepare(vm, tile, pt_update_ops, vma, false);
-			pt_update_ops->wait_vm_kernel = true;
+			vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_KERNEL;
 		}
 		break;
 	}
@@ -2445,8 +2477,7 @@ static int op_prepare(struct xe_vm *vm,
 static void
 xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
 {
-	init_llist_head(&pt_update_ops->deferred);
-	pt_update_ops->current_op = 0;
+	pt_update_ops->pt_job_ops->current_op = 0;
 	pt_update_ops->start = ~0x0ull;
 	pt_update_ops->last = 0x0ull;
 	pt_update_ops->needs_svm_lock = false;
@@ -2454,18 +2485,8 @@ xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
 	xe_page_reclaim_list_init(&pt_update_ops->prl);
 }
 
-/**
- * xe_pt_update_ops_prepare() - Prepare PT update operations
- * @tile: Tile of PT update operations
- * @vops: VMA operationa
- *
- * Prepare PT update operations which includes updating internal PT state,
- * allocate memory for page tables, populate page table being pruned in, and
- * create PT update operations for leaf insertion / removal.
- *
- * Return: 0 on success, negative error code on error.
- */
-int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
+static int __xe_pt_update_ops_prepare(struct xe_tile *tile,
+				      struct xe_vma_ops *vops)
 {
 	struct xe_vm_pgtable_update_ops *pt_update_ops =
 		&vops->pt_update_ops[tile->id];
@@ -2484,15 +2505,25 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
 		return err;
 
 	list_for_each_entry(op, &vops->list, link) {
-		err = op_prepare(vops->vm, tile, pt_update_ops, op);
+		err = op_prepare(vops->vm, tile, vops, pt_update_ops, op);
 
 		if (err)
 			return err;
 	}
 
-	xe_tile_assert(tile, pt_update_ops->current_op <=
+	xe_tile_assert(tile, get_current_op(pt_update_ops) <=
 		       pt_update_ops->num_ops);
 
+	/* Propagate individual tile state up to VMA operation */
+	if (pt_update_ops->start < vops->start)
+		vops->start = pt_update_ops->start;
+	if (pt_update_ops->last > vops->last)
+		vops->last = pt_update_ops->last;
+	if (pt_update_ops->needs_invalidation)
+		vops->flags |= XE_VMA_OPS_FLAG_NEEDS_INVALIDATION;
+	if (pt_update_ops->needs_svm_lock)
+		vops->flags |= XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK;
+
 #ifdef TEST_VM_OPS_ERROR
 	if (vops->inject_error &&
 	    vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
@@ -2501,35 +2532,80 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
 
 	return 0;
 }
-ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO);
 
-static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
-			   struct xe_vm_pgtable_update_ops *pt_update_ops,
-			   struct xe_vma *vma, struct dma_fence *fence,
-			   struct dma_fence *fence2, bool invalidate_on_bind)
+/**
+ * xe_pt_update_ops_prepare() - Prepare PT update operations
+ * @xe: xe device.
+ * @vops: VMA operationa
+ *
+ * Prepare PT update operations which includes updating internal PT state,
+ * allocate memory for page tables, populate page table being pruned in, and
+ * create PT update operations for leaf insertion / removal.
+ *
+ * Return: 0 on success, negative error code on error.
+ */
+int xe_pt_update_ops_prepare(struct xe_device *xe, struct xe_vma_ops *vops)
 {
-	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
+	struct xe_tile *tile;
+	int id, err;
+
+	for_each_tile(tile, xe, id) {
+		struct xe_vm_pgtable_update_ops *pt_update_ops =
+			&vops->pt_update_ops[id];
+
+		if (!vops->pt_update_ops[id].num_ops)
+			continue;
 
-	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
-		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
-				   pt_update_ops->wait_vm_bookkeep ?
-				   DMA_RESV_USAGE_KERNEL :
-				   DMA_RESV_USAGE_BOOKKEEP);
-		if (fence2)
-			dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
-					   pt_update_ops->wait_vm_bookkeep ?
-					   DMA_RESV_USAGE_KERNEL :
-					   DMA_RESV_USAGE_BOOKKEEP);
+		if (pt_mirroring_disabled_for_tile(vops->vm, id)) {
+			struct xe_page_reclaim_list *prl = &pt_update_ops->prl;
+
+			/* Transfer root PT update ops PRL to current */
+			*prl = vops->pt_update_ops[0].prl;
+			xe_page_reclaim_entries_get(prl->entries);
+			continue;
+		}
+
+		err = __xe_pt_update_ops_prepare(tile, vops);
+		if (err)
+			return err;
 	}
+
+	return 0;
+}
+ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO);
+
+static void vma_add_fences(struct xe_vma *vma, struct dma_fence **fences,
+			   int fence_count, enum dma_resv_usage usage)
+{
+	int i;
+
+	if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm)
+		return;
+
+	for (i = 0; i < fence_count; ++i)
+		if (fences[i])
+			dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv,
+					   fences[i], usage);
+}
+
+static void bind_op_commit(struct xe_vm *vm, struct xe_vma *vma,
+			   struct dma_fence **fences, int fence_count,
+			   enum dma_resv_usage usage, u8 tile_mask,
+			   bool invalidate_on_bind)
+{
+	xe_assert(vm->xe, !xe_vma_is_cpu_addr_mirror(vma));
+
+	vma_add_fences(vma, fences, fence_count, usage);
+
 	/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
-	WRITE_ONCE(vma->tile_present, vma->tile_present | BIT(tile->id));
+	WRITE_ONCE(vma->tile_present, vma->tile_present | tile_mask);
 	if (invalidate_on_bind)
 		WRITE_ONCE(vma->tile_invalidated,
-			   vma->tile_invalidated | BIT(tile->id));
+			   vma->tile_invalidated | tile_mask);
 	else
 		WRITE_ONCE(vma->tile_invalidated,
-			   vma->tile_invalidated & ~BIT(tile->id));
-	vma->tile_staged &= ~BIT(tile->id);
+			   vma->tile_invalidated & ~tile_mask);
+	vma->tile_staged &= ~tile_mask;
 	if (xe_vma_is_userptr(vma)) {
 		xe_svm_assert_held_read_or_inject_write(vm);
 		to_userptr_vma(vma)->userptr.initial_bind = true;
@@ -2539,31 +2615,21 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
 	 * Kick rebind worker if this bind triggers preempt fences and not in
 	 * the rebind worker
 	 */
-	if (pt_update_ops->wait_vm_bookkeep &&
+	if (usage == DMA_RESV_USAGE_KERNEL &&
 	    xe_vm_in_preempt_fence_mode(vm) &&
 	    !current->mm)
 		xe_vm_queue_rebind_worker(vm);
 }
 
-static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
-			     struct xe_vm_pgtable_update_ops *pt_update_ops,
-			     struct xe_vma *vma, struct dma_fence *fence,
-			     struct dma_fence *fence2)
+static void unbind_op_commit(struct xe_vm *vm, struct xe_vma *vma,
+			     struct dma_fence **fences, int fence_count,
+			     enum dma_resv_usage usage, u8 tile_mask)
 {
-	xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
+	xe_assert(vm->xe, !xe_vma_is_cpu_addr_mirror(vma));
+
+	vma_add_fences(vma, fences, fence_count, usage);
 
-	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) {
-		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
-				   pt_update_ops->wait_vm_bookkeep ?
-				   DMA_RESV_USAGE_KERNEL :
-				   DMA_RESV_USAGE_BOOKKEEP);
-		if (fence2)
-			dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence2,
-					   pt_update_ops->wait_vm_bookkeep ?
-					   DMA_RESV_USAGE_KERNEL :
-					   DMA_RESV_USAGE_BOOKKEEP);
-	}
-	vma->tile_present &= ~BIT(tile->id);
+	vma->tile_present &= ~tile_mask;
 	if (!vma->tile_present) {
 		list_del_init(&vma->combined_links.rebind);
 		if (xe_vma_is_userptr(vma)) {
@@ -2578,21 +2644,19 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
 
 static void range_present_and_invalidated_tile(struct xe_vm *vm,
 					       struct xe_svm_range *range,
-					       u8 tile_id)
+					       u8 tile_mask)
 {
 	/* All WRITE_ONCE pair with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
 
 	lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);
 
-	WRITE_ONCE(range->tile_present, range->tile_present | BIT(tile_id));
-	WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~BIT(tile_id));
+	WRITE_ONCE(range->tile_present, range->tile_present | tile_mask);
+	WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~tile_mask);
 }
 
-static void op_commit(struct xe_vm *vm,
-		      struct xe_tile *tile,
-		      struct xe_vm_pgtable_update_ops *pt_update_ops,
-		      struct xe_vma_op *op, struct dma_fence *fence,
-		      struct dma_fence *fence2)
+static void op_commit(struct xe_vm *vm, struct xe_vma_op *op,
+		      struct dma_fence **fences, int fence_count,
+		      enum dma_resv_usage usage, u8 tile_mask)
 {
 	xe_vm_assert_held(vm);
 
@@ -2602,8 +2666,8 @@ static void op_commit(struct xe_vm *vm,
 		    (op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
 			break;
 
-		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence,
-			       fence2, op->map.invalidate_on_bind);
+		bind_op_commit(vm, op->map.vma, fences, fence_count, usage,
+			       tile_mask, op->map.invalidate_on_bind);
 		break;
 	case DRM_GPUVA_OP_REMAP:
 	{
@@ -2612,14 +2676,15 @@ static void op_commit(struct xe_vm *vm,
 		if (xe_vma_is_cpu_addr_mirror(old))
 			break;
 
-		unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2);
+		unbind_op_commit(vm, old, fences, fence_count, usage,
+				 tile_mask);
 
 		if (op->remap.prev && !op->remap.skip_prev)
-			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
-				       fence, fence2, false);
+			bind_op_commit(vm, op->remap.prev, fences, fence_count,
+				       usage, tile_mask, false);
 		if (op->remap.next && !op->remap.skip_next)
-			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
-				       fence, fence2, false);
+			bind_op_commit(vm, op->remap.next, fences, fence_count,
+				       usage, tile_mask, false);
 		break;
 	}
 	case DRM_GPUVA_OP_UNMAP:
@@ -2627,8 +2692,8 @@ static void op_commit(struct xe_vm *vm,
 		struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
 
 		if (!xe_vma_is_cpu_addr_mirror(vma))
-			unbind_op_commit(vm, tile, pt_update_ops, vma, fence,
-					 fence2);
+			unbind_op_commit(vm, vma, fences, fence_count,
+					 usage, tile_mask);
 		break;
 	}
 	case DRM_GPUVA_OP_PREFETCH:
@@ -2640,10 +2705,11 @@ static void op_commit(struct xe_vm *vm,
 			unsigned long i;
 
 			xa_for_each(&op->prefetch_range.range, i, range)
-				range_present_and_invalidated_tile(vm, range, tile->id);
+				range_present_and_invalidated_tile(vm, range,
+								   tile_mask);
 		} else {
-			bind_op_commit(vm, tile, pt_update_ops, vma, fence,
-				       fence2, false);
+			bind_op_commit(vm, vma, fences, fence_count, usage,
+				       tile_mask, false);
 		}
 		break;
 	}
@@ -2651,11 +2717,12 @@ static void op_commit(struct xe_vm *vm,
 	{
 		/* WRITE_ONCE pairs with READ_ONCE in xe_vm_has_valid_gpu_mapping() */
 		if (op->subop == XE_VMA_SUBOP_MAP_RANGE)
-			range_present_and_invalidated_tile(vm, op->map_range.range, tile->id);
+			range_present_and_invalidated_tile(vm, op->map_range.range,
+							   tile_mask);
 		else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE)
 			WRITE_ONCE(op->unmap_range.range->tile_present,
 				   op->unmap_range.range->tile_present &
-				   ~BIT(tile->id));
+				   ~tile_mask);
 
 		break;
 	}
@@ -2664,34 +2731,25 @@ static void op_commit(struct xe_vm *vm,
 	}
 }
 
-static const struct xe_migrate_pt_update_ops migrate_ops = {
+static const struct xe_cpu_bind_pt_update_ops cpu_bind_ops = {
 	.populate = xe_vm_populate_pgtable,
-	.clear = xe_migrate_clear_pgtable_callback,
+	.clear = xe_pt_clear_pgtable_callback,
 	.pre_commit = xe_pt_pre_commit,
 };
 
 #if IS_ENABLED(CONFIG_DRM_GPUSVM)
-static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops = {
+static const struct xe_cpu_bind_pt_update_ops svm_userptr_cpu_bind_ops = {
 	.populate = xe_vm_populate_pgtable,
-	.clear = xe_migrate_clear_pgtable_callback,
+	.clear = xe_pt_clear_pgtable_callback,
 	.pre_commit = xe_pt_svm_userptr_pre_commit,
 };
 #else
-static const struct xe_migrate_pt_update_ops svm_userptr_migrate_ops;
+static const struct xe_cpu_bind_pt_update_ops svm_userptr_cpu_bind_ops;
 #endif
 
-static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q,
-						 struct xe_gt *gt)
-{
-	if (xe_gt_is_media_type(gt))
-		return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT].dep_scheduler;
-
-	return q->tlb_inval[XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT].dep_scheduler;
-}
-
 /**
  * xe_pt_update_ops_run() - Run PT update operations
- * @tile: Tile of PT update operations
+ * @xe: xe device.
  * @vops: VMA operationa
  *
  * Run PT update operations which includes committing internal PT state changes,
@@ -2701,80 +2759,83 @@ static struct xe_dep_scheduler *to_dep_scheduler(struct xe_exec_queue *q,
  * Return: fence on success, negative ERR_PTR on error.
  */
 struct dma_fence *
-xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
+xe_pt_update_ops_run(struct xe_device *xe, struct xe_vma_ops *vops)
 {
 	struct xe_vm *vm = vops->vm;
-	struct xe_vm_pgtable_update_ops *pt_update_ops =
-		&vops->pt_update_ops[tile->id];
-	struct xe_exec_queue *q = pt_update_ops->q;
-	struct dma_fence *fence, *ifence = NULL, *mfence = NULL;
-	struct xe_tlb_inval_job *ijob = NULL, *mjob = NULL;
+	struct xe_exec_queue *q = vops->q;
+	struct dma_fence *fence;
+	struct dma_fence *ifences[XE_CPU_BIND_INVAL_JOB_COUNT] = {};
 	struct xe_range_fence *rfence;
+	enum dma_resv_usage usage = DMA_RESV_USAGE_BOOKKEEP;
 	struct xe_vma_op *op;
-	int err = 0, i;
-	struct xe_migrate_pt_update update = {
-		.ops = pt_update_ops->needs_svm_lock ?
-			&svm_userptr_migrate_ops :
-			&migrate_ops,
+	struct xe_tile *tile;
+	int err = 0, total_ops = 0, i, j;
+	u8 tile_mask = 0;
+	bool needs_invalidation = vops->flags &
+		XE_VMA_OPS_FLAG_NEEDS_INVALIDATION;
+	bool needs_svm_lock = vops->flags &
+		XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK;
+	struct xe_cpu_bind_pt_update update = {
+		.ops = needs_svm_lock ? &svm_userptr_cpu_bind_ops :
+			&cpu_bind_ops,
 		.vops = vops,
-		.tile_id = tile->id,
 	};
 
 	lockdep_assert_held(&vm->lock);
 	xe_vm_assert_held(vm);
 
-	if (!pt_update_ops->current_op) {
-		xe_tile_assert(tile, xe_vm_in_fault_mode(vm));
+	for_each_tile(tile, xe, j) {
+		struct xe_vm_pgtable_update_ops *pt_update_ops =
+			&vops->pt_update_ops[j];
 
+		total_ops += get_current_op(pt_update_ops);
+	}
+	if (!total_ops) {
+		xe_assert(xe, xe_vm_in_fault_mode(vm));
 		return dma_fence_get_stub();
 	}
 
 #ifdef TEST_VM_OPS_ERROR
 	if (vops->inject_error &&
-	    vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
+	    xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
 		return ERR_PTR(-ENOSPC);
 #endif
 
-	if (pt_update_ops->needs_invalidation) {
-		struct xe_dep_scheduler *dep_scheduler =
-			to_dep_scheduler(q, tile->primary_gt);
-
-		ijob = xe_tlb_inval_job_create(q, &tile->primary_gt->tlb_inval,
-					       dep_scheduler, vm,
-					       pt_update_ops->start,
-					       pt_update_ops->last,
-					       XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
-		if (IS_ERR(ijob)) {
-			err = PTR_ERR(ijob);
-			goto kill_vm_tile1;
-		}
-		update.ijob = ijob;
-		/*
-		 * Only add page reclaim for the primary GT. Media GT does not have
-		 * any PPC to flush, so enabling the PPC flush bit for media is
-		 * effectively a NOP and provides no performance benefit nor
-		 * interfere with primary GT.
-		 */
-		if (xe_page_reclaim_list_valid(&pt_update_ops->prl)) {
-			xe_tlb_inval_job_add_page_reclaim(ijob, &pt_update_ops->prl);
-			/* Release ref from alloc, job will now handle it */
-			xe_page_reclaim_list_invalidate(&pt_update_ops->prl);
-		}
-
-		if (tile->media_gt) {
-			dep_scheduler = to_dep_scheduler(q, tile->media_gt);
-
-			mjob = xe_tlb_inval_job_create(q,
-						       &tile->media_gt->tlb_inval,
-						       dep_scheduler, vm,
-						       pt_update_ops->start,
-						       pt_update_ops->last,
-						       XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT);
-			if (IS_ERR(mjob)) {
-				err = PTR_ERR(mjob);
+	if (needs_invalidation) {
+		for_each_tlb_inval(q, i) {
+			struct xe_dep_scheduler *dep_scheduler =
+				q->tlb_inval[i].dep_scheduler;
+			struct xe_tile *tile =
+				&xe->tiles[i / XE_MAX_GT_PER_TILE];
+			struct xe_vm_pgtable_update_ops *pt_update_ops =
+				&vops->pt_update_ops[tile->id];
+			struct xe_page_reclaim_list *prl = &pt_update_ops->prl;
+			struct xe_tlb_inval_job *ijob;
+			struct xe_gt *gt = i % XE_MAX_GT_PER_TILE ?
+				tile->media_gt : tile->primary_gt;
+
+			ijob = xe_tlb_inval_job_create(q, &gt->tlb_inval,
+						       dep_scheduler,
+						       vm, vops->start,
+						       vops->last, i);
+			if (IS_ERR(ijob)) {
+				err = PTR_ERR(ijob);
 				goto free_ijob;
 			}
-			update.mjob = mjob;
+
+			update.ijobs[i] = ijob;
+
+			/*
+			 * Only add page reclaim for the primary GT. Media GT
+			 * does not have any PPC to flush, so enabling the PPC
+			 * flush bit for media is effectively a NOP and provides
+			 * no performance benefit nor interfere with primary GT.
+			 */
+			if (xe_page_reclaim_list_valid(prl)) {
+				xe_tlb_inval_job_add_page_reclaim(ijob, prl);
+				/* Release ref from alloc, job will now handle it */
+				xe_page_reclaim_list_invalidate(prl);
+			}
 		}
 	}
 
@@ -2784,65 +2845,67 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 		goto free_ijob;
 	}
 
-	fence = xe_migrate_update_pgtables(tile->migrate, &update);
+	fence = xe_cpu_bind_update_pgtables(xe->cpu_bind, &update);
 	if (IS_ERR(fence)) {
 		err = PTR_ERR(fence);
 		goto free_rfence;
 	}
 
 	/* Point of no return - VM killed if failure after this */
-	for (i = 0; i < pt_update_ops->current_op; ++i) {
-		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
+	for_each_tile(tile, xe, j) {
+		struct xe_vm_pgtable_update_ops *pt_update_ops =
+			&vops->pt_update_ops[j];
+
+		if (pt_mirroring_disabled_for_tile(vm, j)) {
+			xe_tile_assert(tile, !get_current_op(pt_update_ops));
+			tile_mask |= BIT(tile->id);
+			continue;
+		}
+
+		for (i = 0; i < get_current_op(pt_update_ops); ++i) {
+			struct xe_vm_pgtable_update_op *pt_op =
+				to_pt_op(pt_update_ops, i);
 
-		xe_pt_commit(pt_op->vma, pt_op->entries,
-			     pt_op->num_entries, &pt_update_ops->deferred);
-		pt_op->vma = NULL;	/* skip in xe_pt_update_ops_abort */
+			xe_pt_commit(pt_op->vma, pt_op->entries,
+				     pt_op->num_entries,
+				     &pt_update_ops->pt_job_ops->deferred);
+			pt_op->vma = NULL;	/* skip in xe_pt_update_ops_abort */
+			tile_mask |= BIT(tile->id);
+		}
 	}
 
-	if (xe_range_fence_insert(&vm->rftree[tile->id], rfence,
+	if (xe_range_fence_insert(&vm->rftree, rfence,
 				  &xe_range_fence_kfree_ops,
-				  pt_update_ops->start,
-				  pt_update_ops->last, fence))
+				  vops->start, vops->last, fence))
 		dma_fence_wait(fence, false);
 
-	if (ijob)
-		ifence = xe_tlb_inval_job_push(ijob, tile->migrate, fence);
-	if (mjob)
-		mfence = xe_tlb_inval_job_push(mjob, tile->migrate, fence);
+	if (vops->flags & XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP)
+		usage = DMA_RESV_USAGE_KERNEL;
 
-	if (!mjob && !ijob) {
-		dma_resv_add_fence(xe_vm_resv(vm), fence,
-				   pt_update_ops->wait_vm_bookkeep ?
-				   DMA_RESV_USAGE_KERNEL :
-				   DMA_RESV_USAGE_BOOKKEEP);
+	if (!needs_invalidation) {
+		dma_resv_add_fence(xe_vm_resv(vm), fence, usage);
 
 		list_for_each_entry(op, &vops->list, link)
-			op_commit(vops->vm, tile, pt_update_ops, op, fence, NULL);
-	} else if (ijob && !mjob) {
-		dma_resv_add_fence(xe_vm_resv(vm), ifence,
-				   pt_update_ops->wait_vm_bookkeep ?
-				   DMA_RESV_USAGE_KERNEL :
-				   DMA_RESV_USAGE_BOOKKEEP);
-
-		list_for_each_entry(op, &vops->list, link)
-			op_commit(vops->vm, tile, pt_update_ops, op, ifence, NULL);
+			op_commit(vops->vm, op, &fence, 1, usage, tile_mask);
 	} else {
-		dma_resv_add_fence(xe_vm_resv(vm), ifence,
-				   pt_update_ops->wait_vm_bookkeep ?
-				   DMA_RESV_USAGE_KERNEL :
-				   DMA_RESV_USAGE_BOOKKEEP);
+		for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) {
+			if (!update.ijobs[i])
+				continue;
 
-		dma_resv_add_fence(xe_vm_resv(vm), mfence,
-				   pt_update_ops->wait_vm_bookkeep ?
-				   DMA_RESV_USAGE_KERNEL :
-				   DMA_RESV_USAGE_BOOKKEEP);
+			ifences[i] = xe_tlb_inval_job_push(update.ijobs[i],
+							   fence);
+			xe_assert(xe, !IS_ERR_OR_NULL(ifences[i]));
+
+			dma_resv_add_fence(xe_vm_resv(vm), ifences[i], usage);
+		}
 
 		list_for_each_entry(op, &vops->list, link)
-			op_commit(vops->vm, tile, pt_update_ops, op, ifence,
-				  mfence);
+			op_commit(vops->vm, op, ifences,
+				  XE_CPU_BIND_INVAL_JOB_COUNT, usage,
+				  tile_mask);
 	}
 
-	if (pt_update_ops->needs_svm_lock)
+	if (needs_svm_lock)
 		xe_pt_svm_userptr_notifier_unlock(vm);
 
 	/*
@@ -2852,21 +2915,18 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 	if (!(q->flags & EXEC_QUEUE_FLAG_MIGRATE))
 		xe_exec_queue_last_fence_set(q, vm, fence);
 
-	xe_tlb_inval_job_put(mjob);
-	xe_tlb_inval_job_put(ijob);
-	dma_fence_put(ifence);
-	dma_fence_put(mfence);
+	for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) {
+		xe_tlb_inval_job_put(update.ijobs[i]);
+		dma_fence_put(ifences[i]);
+	}
 
 	return fence;
 
 free_rfence:
 	kfree(rfence);
 free_ijob:
-	xe_tlb_inval_job_put(mjob);
-	xe_tlb_inval_job_put(ijob);
-kill_vm_tile1:
-	if (err != -EAGAIN && err != -ENODATA && tile->id)
-		xe_vm_kill(vops->vm, false);
+	for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i)
+		xe_tlb_inval_job_put(update.ijobs[i]);
 
 	return ERR_PTR(err);
 }
@@ -2874,61 +2934,164 @@ ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO);
 
 /**
  * xe_pt_update_ops_fini() - Finish PT update operations
- * @tile: Tile of PT update operations
+ * @xe: xe device.
  * @vops: VMA operations
  *
  * Finish PT update operations by committing to destroy page table memory
  */
-void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
+void xe_pt_update_ops_fini(struct xe_device *xe, struct xe_vma_ops *vops)
 {
-	struct xe_vm_pgtable_update_ops *pt_update_ops =
-		&vops->pt_update_ops[tile->id];
-	int i;
-
-	xe_page_reclaim_entries_put(pt_update_ops->prl.entries);
+	struct xe_tile *tile;
+	int id;
 
-	lockdep_assert_held(&vops->vm->lock);
-	xe_vm_assert_held(vops->vm);
+	for_each_tile(tile, xe, id) {
+		struct xe_vm_pgtable_update_ops *pt_update_ops =
+			&vops->pt_update_ops[id];
 
-	for (i = 0; i < pt_update_ops->current_op; ++i) {
-		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
+		if (!pt_update_ops->num_ops)
+			continue;
 
-		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+		xe_page_reclaim_entries_put(pt_update_ops->prl.entries);
 	}
-	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
 }
 
 /**
  * xe_pt_update_ops_abort() - Abort PT update operations
- * @tile: Tile of PT update operations
+ * @xe: xe device.
  * @vops: VMA operationa
  *
  *  Abort PT update operations by unwinding internal PT state
  */
-void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
+void xe_pt_update_ops_abort(struct xe_device *xe, struct xe_vma_ops *vops)
 {
-	struct xe_vm_pgtable_update_ops *pt_update_ops =
-		&vops->pt_update_ops[tile->id];
-	int i;
+	struct xe_tile *tile;
+	int id;
 
 	lockdep_assert_held(&vops->vm->lock);
 	xe_vm_assert_held(vops->vm);
 
-	for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
-		struct xe_vm_pgtable_update_op *pt_op =
-			&pt_update_ops->ops[i];
+	for_each_tile(tile, xe, id) {
+		struct xe_vm_pgtable_update_ops *pt_update_ops =
+			&vops->pt_update_ops[id];
+		int i;
 
-		if (!pt_op->vma || i >= pt_update_ops->current_op)
+		if (pt_mirroring_disabled_for_tile(vops->vm, id))
 			continue;
 
-		if (pt_op->bind)
-			xe_pt_abort_bind(pt_op->vma, pt_op->entries,
-					 pt_op->num_entries,
-					 pt_op->rebind);
-		else
-			xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
-					   pt_op->num_entries);
+		for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
+			struct xe_vm_pgtable_update_op *pt_op =
+				to_pt_op(pt_update_ops, i);
+
+			if (!pt_op->vma || i >= get_current_op(pt_update_ops))
+				continue;
+
+			if (pt_op->bind)
+				xe_pt_abort_bind(pt_op->vma, pt_op->entries,
+						 pt_op->num_entries,
+						 pt_op->rebind);
+			else
+				xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
+						   pt_op->num_entries);
+		}
 	}
 
-	xe_pt_update_ops_fini(tile, vops);
+	xe_pt_update_ops_fini(xe, vops);
+}
+
+/**
+ * xe_pt_job_ops_alloc() - Allocate PT job ops
+ * @num_ops: Number of VM PT update ops
+ *
+ * Allocate PT job ops and internal array of VM PT update ops.
+ *
+ * Return: Pointer to PT job ops or NULL
+ */
+struct xe_pt_job_ops *xe_pt_job_ops_alloc(u32 num_ops)
+{
+	struct xe_pt_job_ops *pt_job_ops;
+
+	pt_job_ops = kmalloc_obj(*pt_job_ops);
+	if (!pt_job_ops)
+		return NULL;
+
+	pt_job_ops->ops = kvmalloc_array(num_ops, sizeof(*pt_job_ops->ops),
+					 GFP_KERNEL);
+	if (!pt_job_ops->ops) {
+		kfree(pt_job_ops);
+		return NULL;
+	}
+
+	pt_job_ops->current_op = 0;
+	kref_init(&pt_job_ops->refcount);
+	init_llist_head(&pt_job_ops->deferred);
+
+	return pt_job_ops;
+}
+
+/**
+ * xe_pt_job_ops_get() - Get PT job ops
+ * @pt_job_ops: PT job ops to get
+ *
+ * Take a reference to PT job ops
+ *
+ * Return: Pointer to PT job ops or NULL
+ */
+struct xe_pt_job_ops *xe_pt_job_ops_get(struct xe_pt_job_ops *pt_job_ops)
+{
+	if (pt_job_ops)
+		kref_get(&pt_job_ops->refcount);
+
+	return pt_job_ops;
+}
+
+static void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op,
+				  u32 num_ops)
+{
+	u32 i;
+
+	for (i = 0; i < num_ops; ++i, ++pt_op)
+		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+}
+
+static void xe_pt_job_ops_destroy(struct kref *ref)
+{
+	struct xe_pt_job_ops *pt_job_ops =
+		container_of(ref, struct xe_pt_job_ops, refcount);
+	struct llist_node *freed;
+	struct xe_bo *bo, *next;
+
+	xe_pt_update_ops_free(pt_job_ops->ops,
+			      pt_job_ops->current_op);
+
+	freed = llist_del_all(&pt_job_ops->deferred);
+	if (freed) {
+		llist_for_each_entry_safe(bo, next, freed, freed) {
+			struct xe_bo_dev *bo_device =
+				&xe_bo_device(bo)->bo_device;
+			/*
+			 * If called from run_job, we are in the dma-fencing
+			 * path and cannot take dma-resv locks so use an async
+			 * put.
+			 */
+			if (llist_add(&bo->freed, &bo_device->async_list))
+				schedule_work(&bo_device->async_free);
+		}
+	}
+
+	kvfree(pt_job_ops->ops);
+	kfree(pt_job_ops);
+}
+
+/**
+ * xe_pt_job_ops_put() - Put PT job ops
+ * @pt_job_ops: PT job ops to put
+ *
+ * Drop a reference to PT job ops
+ */
+void xe_pt_job_ops_put(struct xe_pt_job_ops *pt_job_ops)
+{
+	if (!pt_job_ops)
+		return;
+
+	kref_put(&pt_job_ops->refcount, xe_pt_job_ops_destroy);
 }
diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h
index 4daeebaab5a1f2..cd78141fb81cea 100644
--- a/drivers/gpu/drm/xe/xe_pt.h
+++ b/drivers/gpu/drm/xe/xe_pt.h
@@ -39,14 +39,18 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred);
 
 void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt);
 
-int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops);
-struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
+int xe_pt_update_ops_prepare(struct xe_device *xe, struct xe_vma_ops *vops);
+struct dma_fence *xe_pt_update_ops_run(struct xe_device *xe,
 				       struct xe_vma_ops *vops);
-void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
-void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
+void xe_pt_update_ops_fini(struct xe_device *xe, struct xe_vma_ops *vops);
+void xe_pt_update_ops_abort(struct xe_device *xe, struct xe_vma_ops *vops);
 
 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma);
 bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
 			  struct xe_svm_range *range);
 
+struct xe_pt_job_ops *xe_pt_job_ops_alloc(u32 num_ops);
+struct xe_pt_job_ops *xe_pt_job_ops_get(struct xe_pt_job_ops *pt_job_ops);
+void xe_pt_job_ops_put(struct xe_pt_job_ops *pt_job_ops);
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h
index a7d1bb708b69e7..0d4bac22ee6c7c 100644
--- a/drivers/gpu/drm/xe/xe_pt_types.h
+++ b/drivers/gpu/drm/xe/xe_pt_types.h
@@ -65,12 +65,18 @@ struct xe_vm_pgtable_update {
 	/** @qwords: number of PTE's to write */
 	u32 qwords;
 
-	/** @pt: opaque pointer useful for the caller of xe_migrate_update_pgtables */
+	/**
+	 * @pt: opaque pointer useful for PT building in the bind IOCTL. Only
+	 * safe to touch during the bind IOCTL (i.e., not in bind jobs).
+	 */
 	struct xe_pt *pt;
 
 	/** @pt_entries: Newly added pagetable entries */
 	struct xe_pt_entry *pt_entries;
 
+	/** @level: level of update */
+	unsigned int level;
+
 	/** @flags: Target flags */
 	u32 flags;
 };
@@ -91,14 +97,29 @@ struct xe_vm_pgtable_update_op {
 	bool rebind;
 };
 
+/**
+ * struct xe_pt_job_ops - Page-table update operations (dynamically allocated)
+ *
+ * This is the portion of &struct xe_vma_ops and
+ * &struct xe_vm_pgtable_update_ops that is dynamically allocated, as it
+ * must remain valid until the associated bind job completes. A reference
+ * count controls its lifetime.
+ */
+struct xe_pt_job_ops {
+	/** @current_op: current page-table update operation */
+	u32 current_op;
+	/** @refcount: reference count */
+	struct kref refcount;
+	/** @deferred: list of deferred PT entries to destroy */
+	struct llist_head deferred;
+	/** @ops: page-table update operations */
+	struct xe_vm_pgtable_update_op *ops;
+};
+
 /** struct xe_vm_pgtable_update_ops: page table update operations */
 struct xe_vm_pgtable_update_ops {
-	/** @ops: operations */
-	struct xe_vm_pgtable_update_op *ops;
-	/** @deferred: deferred list to destroy PT entries */
-	struct llist_head deferred;
-	/** @q: exec queue for PT operations */
-	struct xe_exec_queue *q;
+	/** @pt_job_ops: PT update operations dynamic allocation*/
+	struct xe_pt_job_ops *pt_job_ops;
 	/** @prl: embedded page reclaim list */
 	struct xe_page_reclaim_list prl;
 	/** @start: start address of ops */
@@ -107,24 +128,10 @@ struct xe_vm_pgtable_update_ops {
 	u64 last;
 	/** @num_ops: number of operations */
 	u32 num_ops;
-	/** @current_op: current operations */
-	u32 current_op;
 	/** @needs_svm_lock: Needs SVM lock */
 	bool needs_svm_lock;
 	/** @needs_invalidation: Needs invalidation */
 	bool needs_invalidation;
-	/**
-	 * @wait_vm_bookkeep: PT operations need to wait until VM is idle
-	 * (bookkeep dma-resv slots are idle) and stage all future VM activity
-	 * behind these operations (install PT operations into VM kernel
-	 * dma-resv slot).
-	 */
-	bool wait_vm_bookkeep;
-	/**
-	 * @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv
-	 * slots are idle.
-	 */
-	bool wait_vm_kernel;
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pxp_submit.c b/drivers/gpu/drm/xe/xe_pxp_submit.c
index e60526e3003094..5de86a8cf27d95 100644
--- a/drivers/gpu/drm/xe/xe_pxp_submit.c
+++ b/drivers/gpu/drm/xe/xe_pxp_submit.c
@@ -46,7 +46,7 @@ static int allocate_vcs_execution_resources(struct xe_pxp *pxp)
 		return -ENODEV;
 
 	q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1, hwe,
-				 EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_PERMANENT, 0);
+				 EXEC_QUEUE_FLAG_KERNEL, 0);
 	if (IS_ERR(q))
 		return PTR_ERR(q);
 
@@ -144,8 +144,7 @@ static int allocate_gsc_client_resources(struct xe_gt *gt,
 	}
 
 	q = xe_exec_queue_create(xe, vm, BIT(hwe->logical_instance), 1, hwe,
-				 EXEC_QUEUE_FLAG_KERNEL |
-				 EXEC_QUEUE_FLAG_PERMANENT, 0);
+				 EXEC_QUEUE_FLAG_KERNEL, 0);
 	if (IS_ERR(q)) {
 		err = PTR_ERR(q);
 		goto bo_out;
diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c
index d98ff9453f609a..7a85735c57d58b 100644
--- a/drivers/gpu/drm/xe/xe_ras.c
+++ b/drivers/gpu/drm/xe/xe_ras.c
@@ -3,8 +3,11 @@
  * Copyright © 2026 Intel Corporation
  */
 
+#include "xe_configfs.h"
+#include "xe_debugfs.h"
 #include "xe_device.h"
 #include "xe_drm_ras.h"
+#include "xe_log.h"
 #include "xe_pm.h"
 #include "xe_printk.h"
 #include "xe_ras.h"
@@ -45,6 +48,16 @@ enum xe_ras_component {
 	XE_RAS_COMP_MAX
 };
 
+#define CHECK_COMPONENT(RAS_COMP, LOG_COMP) \
+	static_assert(MAKE_XE_LOG_COMPONENT(HARDWARE, (RAS_COMP)) == (LOG_COMP))
+	/* make sure components definitions maintain stable relation */
+	CHECK_COMPONENT(XE_RAS_COMP_DEVICE_MEMORY, XE_LOG_COMPONENT_DEVICE_MEMORY);
+	CHECK_COMPONENT(XE_RAS_COMP_CORE_COMPUTE, XE_LOG_COMPONENT_CORE_COMPUTE);
+	CHECK_COMPONENT(XE_RAS_COMP_PCIE, XE_LOG_COMPONENT_PCIE);
+	CHECK_COMPONENT(XE_RAS_COMP_FABRIC, XE_LOG_COMPONENT_FABRIC);
+	CHECK_COMPONENT(XE_RAS_COMP_SOC_INTERNAL, XE_LOG_COMPONENT_SOC_INTERNAL);
+#undef CHECK_COMPONENT
+
 /* RAS response status codes */
 enum xe_ras_response_status {
 	XE_RAS_STATUS_SUCCESS = 0,
@@ -90,6 +103,8 @@ static const char * const gpu_health_states[] = {
 };
 static_assert(ARRAY_SIZE(gpu_health_states) == XE_RAS_HEALTH_MAX);
 
+static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, u32 *value);
+
 static u8 drm_to_xe_ras_severity(u8 severity)
 {
 	switch (severity) {
@@ -102,6 +117,18 @@ static u8 drm_to_xe_ras_severity(u8 severity)
 	}
 }
 
+static u8 xe_to_drm_ras_severity(u8 severity)
+{
+	switch (severity) {
+	case XE_RAS_SEV_CORRECTABLE:
+		return DRM_XE_RAS_ERR_SEV_CORRECTABLE;
+	case XE_RAS_SEV_UNCORRECTABLE:
+		return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE;
+	default:
+		return DRM_XE_RAS_ERR_SEV_MAX;
+	}
+}
+
 static u8 drm_to_xe_ras_component(u8 component)
 {
 	switch (component) {
@@ -120,6 +147,24 @@ static u8 drm_to_xe_ras_component(u8 component)
 	}
 }
 
+static u8 xe_to_drm_ras_component(u8 component)
+{
+	switch (component) {
+	case XE_RAS_COMP_DEVICE_MEMORY:
+		return DRM_XE_RAS_ERR_COMP_DEVICE_MEMORY;
+	case XE_RAS_COMP_CORE_COMPUTE:
+		return DRM_XE_RAS_ERR_COMP_CORE_COMPUTE;
+	case XE_RAS_COMP_PCIE:
+		return DRM_XE_RAS_ERR_COMP_PCIE;
+	case XE_RAS_COMP_FABRIC:
+		return DRM_XE_RAS_ERR_COMP_FABRIC;
+	case XE_RAS_COMP_SOC_INTERNAL:
+		return DRM_XE_RAS_ERR_COMP_SOC_INTERNAL;
+	default:
+		return DRM_XE_RAS_ERR_COMP_MAX;
+	}
+}
+
 static int ras_status_to_errno(u32 status)
 {
 	switch (status) {
@@ -156,6 +201,24 @@ static inline const char *comp_to_str(u8 component)
 	return xe_ras_components[component];
 }
 
+static bool ras_counter_is_valid(struct xe_device *xe, struct xe_ras_error_class *counter)
+{
+	u8 severity = counter->common.severity;
+	u8 component = counter->common.component;
+
+	if (!in_range(severity, XE_RAS_SEV_NOT_SUPPORTED + 1, XE_RAS_SEV_MAX - 1)) {
+		xe_err(xe, "sysctrl: unexpected severity %u\n", severity);
+		return false;
+	}
+
+	if (!in_range(component, XE_RAS_COMP_NOT_SUPPORTED + 1, XE_RAS_COMP_MAX - 1)) {
+		xe_err(xe, "sysctrl: unexpected component %u\n", component);
+		return false;
+	}
+
+	return true;
+}
+
 static struct pci_dev *find_usp_dev(struct pci_dev *pdev)
 {
 	struct pci_dev *vsp;
@@ -218,6 +281,26 @@ static void ras_usp_aer_init(struct xe_device *xe)
 	dev_dbg(&usp->dev, "Uncorrectable Internal Errors downgraded and unmasked\n");
 }
 
+static void ras_send_error_event(struct xe_device *xe, u8 severity, u8 component)
+{
+	struct xe_ras_error_class counter = {0};
+	u8 drm_severity, drm_component;
+	u32 value;
+	int ret;
+
+	counter.common.severity = severity;
+	counter.common.component = component;
+
+	ret = get_counter(xe, &counter, &value);
+	if (ret)
+		return;
+
+	drm_severity = xe_to_drm_ras_severity(severity);
+	drm_component = xe_to_drm_ras_component(component);
+
+	xe_drm_ras_event(xe, drm_component, drm_severity, value);
+}
+
 static u8 handle_core_compute_errors(struct xe_ras_error_array *arr)
 {
 	struct xe_ras_compute_error *error_info = (void *)arr->details;
@@ -236,6 +319,12 @@ static u8 handle_core_compute_errors(struct xe_ras_error_array *arr)
 	return XE_RAS_RECOVERY_ACTION_RECOVERED;
 }
 
+static void punit_error_handler(struct xe_device *xe)
+{
+	xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_COLD_RESET);
+	xe_device_declare_wedged(xe);
+}
+
 static u8 handle_soc_internal_errors(struct xe_device *xe, struct xe_ras_error_array *arr)
 {
 	struct xe_ras_soc_error *info = (void *)arr->details;
@@ -267,7 +356,7 @@ static u8 handle_soc_internal_errors(struct xe_device *xe, struct xe_ras_error_a
 			xe_err(xe, "[RAS]: PUNIT %s detected: 0x%x\n",
 			       sev_to_str(counter->common.severity),
 			       ieh_error->global_error_status);
-			/* TODO: Add PUNIT error handling */
+			punit_error_handler(xe);
 			return XE_RAS_RECOVERY_ACTION_DISCONNECT;
 		}
 	}
@@ -312,8 +401,10 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe,
 	struct xe_ras_threshold_crossed *pending = (void *)&response->data;
 	struct xe_ras_error_class *errors = pending->counters;
 	u32 id, ncounters = pending->ncounters;
+	u8 sent = 0;
 
 	BUILD_BUG_ON(sizeof(response->data) < sizeof(*pending));
+	BUILD_BUG_ON(BITS_PER_TYPE(sent) < XE_RAS_COMP_MAX);
 	xe_device_assert_mem_access(xe);
 
 	if (!ncounters || ncounters > XE_RAS_NUM_COUNTERS)
@@ -327,8 +418,18 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe,
 		severity = errors[id].common.severity;
 		component = errors[id].common.component;
 
+		if (!ras_counter_is_valid(xe, &errors[id]))
+			continue;
+
 		xe_warn(xe, "[RAS]: %s %s detected\n",
 			comp_to_str(component), sev_to_str(severity));
+
+		/* Send event once per component */
+		if (sent & BIT(component))
+			continue;
+		sent |= BIT(component);
+
+		ras_send_error_event(xe, severity, component);
 	}
 }
 
@@ -358,6 +459,9 @@ static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter,
 		return -EIO;
 	}
 
+	if (!ras_counter_is_valid(xe, &response.counter))
+		return -EBADMSG;
+
 	common = &response.counter.common;
 	*value = response.value;
 
@@ -382,12 +486,20 @@ enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe)
 	enum xe_ras_recovery_action final_action;
 	u32 remaining = XE_SYSCTRL_FLOOD_LIMIT;
 	struct xe_ras_get_soc_error response;
+	u8 sent = 0;
 	size_t rlen;
 	int ret;
 
+	if (xe_fault_wedge_cold_reset()) {
+		xe_err(xe, "[RAS]: cold-reset wedge injected\n");
+		punit_error_handler(xe);
+		return XE_RAS_RECOVERY_ACTION_DISCONNECT;
+	}
+
 	if (!xe->info.has_sysctrl)
 		return XE_RAS_RECOVERY_ACTION_RESET;
 
+	BUILD_BUG_ON(BITS_PER_TYPE(sent) < XE_RAS_COMP_MAX);
 	/* Default action */
 	final_action = XE_RAS_RECOVERY_ACTION_RECOVERED;
 
@@ -422,9 +534,18 @@ enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe)
 			component = arr->counter.common.component;
 			severity = arr->counter.common.severity;
 
+			if (!ras_counter_is_valid(xe, &arr->counter))
+				continue;
+
 			xe_info(xe, "[RAS]: %s %s detected\n", comp_to_str(component),
 				sev_to_str(severity));
 
+			/* Send event once per component */
+			if (!(sent & BIT(component))) {
+				sent |= BIT(component);
+				ras_send_error_event(xe, severity, component);
+			}
+
 			switch (component) {
 			case XE_RAS_COMP_CORE_COMPUTE:
 				action = handle_core_compute_errors(arr);
@@ -532,12 +653,126 @@ int xe_ras_clear_counter(struct xe_device *xe, u8 severity, u8 component)
 
 	counter = &response.counter;
 
+	if (!ras_counter_is_valid(xe, counter))
+		return -EBADMSG;
+
 	xe_dbg(xe, "[RAS]: clear counter for %s %s\n", comp_to_str(counter->common.component),
 	       sev_to_str(counter->common.severity));
 
 	return 0;
 }
 
+/**
+ * xe_ras_get_threshold() - Get error counter threshold
+ * @xe: Xe device instance
+ * @severity: Error severity to be queried (&enum drm_xe_ras_error_severity)
+ * @component: Error component to be queried (&enum drm_xe_ras_error_component)
+ * @threshold: Counter threshold
+ *
+ * This function retrieves the error threshold of a specific counter based on
+ * severity and component.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int xe_ras_get_threshold(struct xe_device *xe, u8 severity, u8 component, u32 *threshold)
+{
+	struct xe_ras_get_threshold_response response = {};
+	struct xe_ras_get_threshold_request request = {};
+	struct xe_sysctrl_mailbox_command command = {};
+	struct xe_ras_error_class *counter;
+	size_t len;
+	int ret;
+
+	counter = &request.counter;
+	counter->common.severity = drm_to_xe_ras_severity(severity);
+	counter->common.component = drm_to_xe_ras_component(component);
+
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_THRESHOLD,
+				  &request, sizeof(request), &response, sizeof(response));
+
+	guard(xe_pm_runtime)(xe);
+	ret = xe_sysctrl_send_command(&xe->sc, &command, &len);
+	if (ret) {
+		xe_err(xe, "sysctrl: failed to get threshold %d\n", ret);
+		return ret;
+	}
+
+	if (len != sizeof(response)) {
+		xe_err(xe, "sysctrl: unexpected get threshold response length %zu (expected %zu)\n",
+		       len, sizeof(response));
+		return -EIO;
+	}
+
+	if (!ras_counter_is_valid(xe, &response.counter))
+		return -EBADMSG;
+
+	counter = &response.counter;
+	*threshold = response.threshold;
+
+	xe_dbg(xe, "[RAS]: get threshold %u for %s %s\n", *threshold,
+	       comp_to_str(counter->common.component), sev_to_str(counter->common.severity));
+	return 0;
+}
+
+/**
+ * xe_ras_set_threshold() - Set error counter threshold
+ * @xe: Xe device instance
+ * @severity: Error severity to be set (&enum drm_xe_ras_error_severity)
+ * @component: Error component to be set (&enum drm_xe_ras_error_component)
+ * @threshold: Counter threshold
+ *
+ * This function sets the error threshold of a specific counter based on
+ * severity and component.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int xe_ras_set_threshold(struct xe_device *xe, u8 severity, u8 component, u32 threshold)
+{
+	struct xe_ras_set_threshold_response response = {};
+	struct xe_ras_set_threshold_request request = {};
+	struct xe_sysctrl_mailbox_command command = {};
+	struct xe_ras_error_class *counter;
+	size_t len;
+	int ret;
+
+	counter = &request.counter;
+	counter->common.severity = drm_to_xe_ras_severity(severity);
+	counter->common.component = drm_to_xe_ras_component(component);
+	request.threshold = threshold;
+
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_SET_THRESHOLD,
+				  &request, sizeof(request), &response, sizeof(response));
+
+	guard(xe_pm_runtime)(xe);
+	ret = xe_sysctrl_send_command(&xe->sc, &command, &len);
+	if (ret) {
+		xe_err(xe, "sysctrl: failed to set threshold %d\n", ret);
+		return ret;
+	}
+
+	if (len != sizeof(response)) {
+		xe_err(xe, "sysctrl: unexpected set threshold response length %zu (expected %zu)\n",
+		       len, sizeof(response));
+		return -EIO;
+	}
+
+	ret = ras_status_to_errno(response.status);
+	if (ret) {
+		xe_err(xe, "sysctrl: set threshold command failed with status %#x\n",
+		       response.status);
+		return ret;
+	}
+
+	counter = &response.counter;
+
+	if (!ras_counter_is_valid(xe, counter))
+		return -EBADMSG;
+
+	xe_dbg(xe, "[RAS]: set threshold %u for %s %s\n", response.threshold,
+	       comp_to_str(counter->common.component), sev_to_str(counter->common.severity));
+	return 0;
+}
+
 static ssize_t gpu_health_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct xe_ras_get_health_response response = {0};
@@ -681,6 +916,14 @@ void xe_ras_init(struct xe_device *xe)
 {
 	int ret;
 
+	/*
+	 * TODO: Replace platform check with xe->info.has_disable_vram_page_offline
+	 * once the feature flag is plumbed through device info.
+	 */
+	if (xe->info.platform == XE_CRESCENTISLAND)
+		xe->ras.disable_vram_page_offline =
+			xe_configfs_get_disable_vram_page_offline(to_pci_dev(xe->drm.dev));
+
 	xe_drm_ras_init(xe);
 
 	if (!xe->info.has_sysctrl)
diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h
index 6183647340430d..0b8669f28d5621 100644
--- a/drivers/gpu/drm/xe/xe_ras.h
+++ b/drivers/gpu/drm/xe/xe_ras.h
@@ -16,6 +16,8 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe,
 				      struct xe_sysctrl_event_response *response);
 int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *value);
 int xe_ras_clear_counter(struct xe_device *xe, u8 severity, u8 component);
+int xe_ras_get_threshold(struct xe_device *xe, u8 severity, u8 component, u32 *threshold);
+int xe_ras_set_threshold(struct xe_device *xe, u8 severity, u8 component, u32 threshold);
 void xe_ras_init(struct xe_device *xe);
 enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe);
 
diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h
index 99b2466e2062c9..fe6f3658a2a442 100644
--- a/drivers/gpu/drm/xe/xe_ras_types.h
+++ b/drivers/gpu/drm/xe/xe_ras_types.h
@@ -147,6 +147,56 @@ struct xe_ras_clear_counter_response {
 	u32 reserved1[3];
 } __packed;
 
+/**
+ * struct xe_ras_get_threshold_request - Request structure for get threshold
+ */
+struct xe_ras_get_threshold_request {
+	/** @counter: Counter to get threshold for */
+	struct xe_ras_error_class counter;
+	/** @reserved: Reserved for future use */
+	u32 reserved;
+} __packed;
+
+/**
+ * struct xe_ras_get_threshold_response - Response structure for get threshold
+ */
+struct xe_ras_get_threshold_response {
+	/** @counter: Counter ID */
+	struct xe_ras_error_class counter;
+	/** @threshold: Current threshold of the counter */
+	u32 threshold;
+	/** @reserved: Reserved for future use */
+	u32 reserved[4];
+} __packed;
+
+/**
+ * struct xe_ras_set_threshold_request - Request structure for set threshold
+ */
+struct xe_ras_set_threshold_request {
+	/** @counter: Counter to set threshold for */
+	struct xe_ras_error_class counter;
+	/** @threshold: Threshold to be set */
+	u32 threshold;
+	/** @reserved: Reserved for future use */
+	u32 reserved;
+} __packed;
+
+/**
+ * struct xe_ras_set_threshold_response - Response structure for set threshold
+ */
+struct xe_ras_set_threshold_response {
+	/** @counter: Counter ID */
+	struct xe_ras_error_class counter;
+	/** @reserved: Reserved */
+	u32 reserved;
+	/** @threshold: Updated threshold */
+	u32 threshold;
+	/** @status: Operation status */
+	u32 status;
+	/** @reserved1: Reserved for future use */
+	u32 reserved1[2];
+} __packed;
+
 /**
  * struct xe_ras_error_array - Details of the error types
  */
diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h
index 0522caafd89d7e..c3a037e5f34e2f 100644
--- a/drivers/gpu/drm/xe/xe_res_cursor.h
+++ b/drivers/gpu/drm/xe/xe_res_cursor.h
@@ -233,12 +233,13 @@ static inline void xe_res_first_sg(const struct sg_table *sg,
  * @dma_addr: struct drm_pagemap_addr array to walk
  * @start: Start of the range
  * @size: Size of the range
+ * @contiguous: Whether one entry describes the whole range
  * @cur: cursor object to initialize
  *
  * Start walking over the range of allocations between @start and @size.
  */
 static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr,
-				    u64 start, u64 size,
+				    u64 start, u64 size, bool contiguous,
 				    struct xe_res_cursor *cur)
 {
 	XE_WARN_ON(!dma_addr);
@@ -248,7 +249,7 @@ static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr,
 	cur->node = NULL;
 	cur->start = start;
 	cur->remaining = size;
-	cur->dma_seg_size = PAGE_SIZE << dma_addr->order;
+	cur->dma_seg_size = contiguous ? start + size : PAGE_SIZE << dma_addr->order;
 	cur->dma_start = 0;
 	cur->size = 0;
 	cur->dma_addr = dma_addr;
diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c
index 08b4a4283e96a2..bc4dea606b381f 100644
--- a/drivers/gpu/drm/xe/xe_ring_ops.c
+++ b/drivers/gpu/drm/xe/xe_ring_ops.c
@@ -55,8 +55,8 @@ __emit_aux_table_inv(u32 *cmd, const struct xe_reg reg, u32 adj_offset)
 		 MI_LRI_MMIO_REMAP_EN;
 	*cmd++ = reg.addr + adj_offset;
 	*cmd++ = AUX_INV;
-	*cmd++ = MI_SEMAPHORE_WAIT_TOKEN | MI_SEMAPHORE_REGISTER_POLL |
-		 MI_SEMAPHORE_POLL | MI_SEMAPHORE_SAD_EQ_SDD;
+	*cmd++ = MI_SEMAPHORE_WAIT | MI_SEMW_REGISTER_POLL |
+		 MI_SEMW_POLL | MI_SEMW_COMPARE(SAD_EQ_SDD);
 	*cmd++ = 0;
 	*cmd++ = reg.addr + adj_offset;
 	*cmd++ = 0;
@@ -143,9 +143,8 @@ static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
 
 static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i)
 {
-	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW |
-		  MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0;
-
+	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
+		  (flush_flags & MI_INVALIDATE_TLB);
 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
 	dw[i++] = 0;
 	dw[i++] = val;
@@ -505,6 +504,68 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
 }
 
+static int emit_ulls_preamble(struct xe_lrc *lrc, u32 *dw, int i, u32 seqno)
+{
+	u32 addr = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno);
+
+	return emit_store_imm_ggtt(addr, LRC_MIGRATION_ULLS_SEMAPHORE_CLEAR,
+				   dw, i);
+}
+
+/*
+ * Advance the ring tail from within the ring, so submitting the next ULLS job
+ * needs nothing from the CPU beyond signalling the semaphore. All ULLS jobs
+ * occupy exactly ULLS_JOB_SIZE_BYTES, so the tail the next job ends at is two
+ * job slots on from where this job started, even though that job has not been
+ * emitted yet. Both LRC and MMIO ring tail advanced in step.
+ */
+static int emit_ulls_ring_tail(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw,
+			       int i, u32 head)
+{
+	u32 next_tail = (head + 2 * ULLS_JOB_SIZE_BYTES) & (lrc->ring.size - 1);
+
+	xe_gt_assert(gt, IS_ALIGNED(next_tail, 8));
+
+	i = emit_store_imm_ggtt(xe_lrc_ring_tail_ggtt_addr(lrc), next_tail,
+				dw, i);
+
+	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) |
+		MI_LRI_LRM_CS_MMIO;
+	dw[i++] = RING_TAIL(0).addr;
+	dw[i++] = next_tail;
+
+	return i;
+}
+
+/* Publish the next job's tail, then park the engine on its semaphore */
+static int emit_ulls_postamble(struct xe_gt *gt, struct xe_lrc *lrc, u32 *dw,
+			       int i, u32 seqno, u32 head)
+{
+	i = emit_ulls_ring_tail(gt, lrc, dw, i, head);
+
+	dw[i++] = MI_SEMAPHORE_WAIT |
+		MI_SEMW_GGTT |
+		MI_SEMW_POLL |
+		MI_SEMW_COMPARE(SAD_EQ_SDD);
+	dw[i++] = LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL;
+	dw[i++] = xe_lrc_ulls_semaphore_ggtt_addr(lrc, seqno + 1);
+	dw[i++] = 0;
+	dw[i++] = 0;
+
+	return i;
+}
+
+/* Pad out to the fixed ULLS job size */
+static int emit_ulls_pad(struct xe_gt *gt, u32 *dw, int i)
+{
+	xe_gt_assert(gt, i <= ULLS_JOB_SIZE_DW);
+
+	while (i < ULLS_JOB_SIZE_DW)
+		dw[i++] = MI_NOOP;
+
+	return i;
+}
+
 static void emit_migration_job_gen12(struct xe_sched_job *job,
 				     struct xe_lrc *lrc, u32 *head,
 				     u32 seqno)
@@ -518,10 +579,16 @@ static void emit_migration_job_gen12(struct xe_sched_job *job,
 
 	xe_gt_assert(gt, !job->ring_ops_force_reset);
 
+	if (xe_sched_job_is_ulls(job))
+		i = emit_ulls_preamble(lrc, dw, i, seqno);
+
 	i = emit_copy_timestamp(xe, lrc, dw, i);
 
 	i = emit_store_imm_ggtt(saddr, seqno, dw, i);
 
+	if (!xe_sched_job_ulls_has_batch(job))
+		goto seqno_write;
+
 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
 
 	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
@@ -532,12 +599,19 @@ static void emit_migration_job_gen12(struct xe_sched_job *job,
 
 	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
 
+seqno_write:
 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno,
 				job->migrate_flush_flags,
 				dw, i);
 
 	i = emit_user_interrupt(dw, i);
 
+	if (xe_sched_job_ulls_parks(job))
+		i = emit_ulls_postamble(gt, lrc, dw, i, seqno, *head);
+
+	if (xe_sched_job_is_ulls(job))
+		i = emit_ulls_pad(gt, dw, i);
+
 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
 
 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
diff --git a/drivers/gpu/drm/xe/xe_ring_ops_types.h b/drivers/gpu/drm/xe/xe_ring_ops_types.h
index 52ff96bc410043..ea4af321dd7c4d 100644
--- a/drivers/gpu/drm/xe/xe_ring_ops_types.h
+++ b/drivers/gpu/drm/xe/xe_ring_ops_types.h
@@ -6,6 +6,7 @@
 #ifndef _XE_RING_OPS_TYPES_H_
 #define _XE_RING_OPS_TYPES_H_
 
+#include <linux/build_bug.h>
 #include <linux/types.h>
 
 struct xe_gt;
@@ -14,6 +15,29 @@ struct xe_sched_job;
 #define MAX_JOB_SIZE_DW 74
 #define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4)
 
+/*
+ * ULLS migration jobs advance the ring tail from within the ring itself, so a
+ * job has to know where its successor will end before that successor has been
+ * emitted. Every ULLS job is therefore padded to a fixed size, letting the
+ * next tail be derived arithmetically.
+ *
+ * Sized for the largest such job, emitted by emit_migration_job_gen12():
+ * preamble (4), copy timestamp (8, its size on an SRIOV VF), start seqno
+ * store (4), arbitration off (1), batch buffer starts (2 * 3), pre-parser
+ * bracketed flush invalidate (6), seqno flush (4), user interrupt (3) and
+ * postamble (7 + 5).
+ */
+#define ULLS_JOB_SIZE_DW 48
+#define ULLS_JOB_SIZE_BYTES (ULLS_JOB_SIZE_DW * 4)
+
+/*
+ * RING_TAIL only encodes a qword aligned offset, and xe_lrc_write_ring()
+ * appends a NOP to anything shorter, either of which would desynchronise the
+ * ring from the tail a job predicts for its successor.
+ */
+static_assert(ULLS_JOB_SIZE_BYTES % 8 == 0);
+static_assert(ULLS_JOB_SIZE_DW <= MAX_JOB_SIZE_DW);
+
 /**
  * struct xe_ring_ops - Ring operations
  */
diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c
index a4fa00632a3033..59f7215f27b17f 100644
--- a/drivers/gpu/drm/xe/xe_sched_job.c
+++ b/drivers/gpu/drm/xe/xe_sched_job.c
@@ -26,19 +26,22 @@ static struct kmem_cache *xe_sched_job_parallel_slab;
 
 int __init xe_sched_job_module_init(void)
 {
+	struct xe_sched_job *job;
+	size_t size;
+
+	size = struct_size(job, ptrs, 1);
 	xe_sched_job_slab =
-		kmem_cache_create("xe_sched_job",
-				  sizeof(struct xe_sched_job) +
-				  sizeof(struct xe_job_ptrs), 0,
+		kmem_cache_create("xe_sched_job", size, 0,
 				  SLAB_HWCACHE_ALIGN, NULL);
 	if (!xe_sched_job_slab)
 		return -ENOMEM;
 
+	size = max_t(size_t,
+		     struct_size(job, ptrs,
+				 XE_HW_ENGINE_MAX_INSTANCE),
+		     struct_size(job, pt_update, 1));
 	xe_sched_job_parallel_slab =
-		kmem_cache_create("xe_sched_job_parallel",
-				  sizeof(struct xe_sched_job) +
-				  sizeof(struct xe_job_ptrs) *
-				  XE_HW_ENGINE_MAX_INSTANCE, 0,
+		kmem_cache_create("xe_sched_job_parallel", size, 0,
 				  SLAB_HWCACHE_ALIGN, NULL);
 	if (!xe_sched_job_parallel_slab) {
 		kmem_cache_destroy(xe_sched_job_slab);
@@ -70,8 +73,9 @@ static void job_free(struct xe_sched_job *job)
 	struct xe_exec_queue *q = job->q;
 	bool is_migration = xe_sched_job_is_migration(q);
 
-	kmem_cache_free(xe_exec_queue_is_parallel(job->q) || is_migration ?
-			xe_sched_job_parallel_slab : xe_sched_job_slab, job);
+	kmem_cache_free(job->is_pt_job || xe_exec_queue_is_parallel(job->q) ||
+			is_migration ? xe_sched_job_parallel_slab :
+			xe_sched_job_slab, job);
 }
 
 static struct xe_device *job_to_xe(struct xe_sched_job *job)
@@ -84,6 +88,9 @@ static void xe_sched_job_free_fences(struct xe_sched_job *job)
 {
 	int i;
 
+	if (job->is_pt_job)
+		return;
+
 	for (i = 0; i < job->q->width; ++i) {
 		struct xe_job_ptrs *ptrs = &job->ptrs[i];
 
@@ -93,10 +100,23 @@ static void xe_sched_job_free_fences(struct xe_sched_job *job)
 	}
 }
 
+/**
+ * xe_sched_job_create() - Create a scheduler job
+ * @q: exec queue to create the scheduler job for
+ * @batch_addr: array of batch addresses for the job; must match the width of
+ * @q, or NULL to indicate a PT job that does not require a batch address
+ *
+ * Create a scheduler job for submission.
+ *
+ * Context: Reclaim
+ *
+ * Return: a &xe_sched_job object on success, or an ERR_PTR on failure.
+ */
 struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
 					 u64 *batch_addr)
 {
 	bool is_migration = xe_sched_job_is_migration(q);
+	struct xe_device *xe = gt_to_xe(q->gt);
 	struct xe_sched_job *job;
 	int err;
 	int i;
@@ -105,10 +125,15 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
 	/* only a kernel context can submit a vm-less job */
 	XE_WARN_ON(!q->vm && !(q->flags & EXEC_QUEUE_FLAG_KERNEL));
 
-	job = job_alloc(xe_exec_queue_is_parallel(q) || is_migration);
+	xe_assert(xe, batch_addr ||
+		  q->flags & (EXEC_QUEUE_FLAG_VM | EXEC_QUEUE_FLAG_MIGRATE));
+
+	job = job_alloc(!batch_addr || xe_exec_queue_is_parallel(q) ||
+			is_migration);
 	if (!job)
 		return ERR_PTR(-ENOMEM);
 
+	job->is_pt_job = !batch_addr;
 	job->q = q;
 	job->sample_timestamp = U64_MAX;
 	kref_init(&job->refcount);
@@ -119,34 +144,38 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
 	if (err)
 		goto err_free;
 
-	for (i = 0; i < q->width; ++i) {
-		struct dma_fence *fence = xe_lrc_alloc_seqno_fence();
-		struct dma_fence_chain *chain;
-
-		if (IS_ERR(fence)) {
-			err = PTR_ERR(fence);
-			goto err_sched_job;
+	if (!batch_addr) {
+		job->fence = dma_fence_get_stub();
+	} else {
+		for (i = 0; i < q->width; ++i) {
+			struct dma_fence *fence = xe_lrc_alloc_seqno_fence();
+			struct dma_fence_chain *chain;
+
+			if (IS_ERR(fence)) {
+				err = PTR_ERR(fence);
+				goto err_sched_job;
+			}
+			job->ptrs[i].lrc_fence = fence;
+
+			if (i + 1 == q->width)
+				continue;
+
+			chain = dma_fence_chain_alloc();
+			if (!chain) {
+				err = -ENOMEM;
+				goto err_sched_job;
+			}
+			job->ptrs[i].chain_fence = chain;
 		}
-		job->ptrs[i].lrc_fence = fence;
 
-		if (i + 1 == q->width)
-			continue;
+		width = q->width;
+		if (is_migration)
+			width = 2;
 
-		chain = dma_fence_chain_alloc();
-		if (!chain) {
-			err = -ENOMEM;
-			goto err_sched_job;
-		}
-		job->ptrs[i].chain_fence = chain;
+		for (i = 0; i < width; ++i)
+			job->ptrs[i].batch_addr = batch_addr[i];
 	}
 
-	width = q->width;
-	if (is_migration)
-		width = 2;
-
-	for (i = 0; i < width; ++i)
-		job->ptrs[i].batch_addr = batch_addr[i];
-
 	atomic_inc(&q->job_cnt);
 	xe_pm_runtime_get_noresume(job_to_xe(job));
 	trace_xe_sched_job_create(job);
@@ -246,7 +275,7 @@ bool xe_sched_job_completed(struct xe_sched_job *job)
 void xe_sched_job_arm(struct xe_sched_job *job)
 {
 	struct xe_exec_queue *q = job->q;
-	struct dma_fence *fence, *prev;
+	struct dma_fence *fence = job->fence, *prev;
 	struct xe_vm *vm = q->vm;
 	u64 seqno = 0;
 	int i;
@@ -259,6 +288,9 @@ void xe_sched_job_arm(struct xe_sched_job *job)
 			xe_vm_assert_held(q->vm);
 	}
 
+	if (job->is_pt_job)
+		goto arm;
+
 	if (vm && !xe_sched_job_is_migration(q) && !xe_vm_in_lr_mode(vm) &&
 	    (vm->batch_invalidate_tlb || vm->tlb_flush_seqno != q->tlb_flush_seqno)) {
 		xe_vm_assert_held(vm);
@@ -286,6 +318,7 @@ void xe_sched_job_arm(struct xe_sched_job *job)
 		fence = &chain->base;
 	}
 
+arm:
 	job->fence = dma_fence_get(fence);	/* Pairs with put in scheduler */
 	drm_sched_job_arm(&job->drm);
 }
@@ -329,7 +362,7 @@ xe_sched_job_snapshot_capture(struct xe_sched_job *job)
 
 	snapshot->batch_addr_len = q->width;
 	for (i = 0; i < q->width; i++)
-		snapshot->batch_addr[i] =
+		snapshot->batch_addr[i] = job->is_pt_job ? 0 :
 			xe_device_uncanonicalize_addr(xe, job->ptrs[i].batch_addr);
 
 	return snapshot;
diff --git a/drivers/gpu/drm/xe/xe_sched_job.h b/drivers/gpu/drm/xe/xe_sched_job.h
index 1c1cb44216c3c6..bbf5e3864ba373 100644
--- a/drivers/gpu/drm/xe/xe_sched_job.h
+++ b/drivers/gpu/drm/xe/xe_sched_job.h
@@ -83,6 +83,62 @@ xe_sched_job_add_migrate_flush(struct xe_sched_job *job, u32 flags)
 	job->migrate_flush_flags = flags;
 }
 
+/**
+ * xe_sched_job_is_ulls() - Is a ULLS job
+ * @job: Xe schedule job object
+ *
+ * Return: True if @job is submitted as part of a ULLS sequence, False
+ * otherwise.
+ */
+static inline bool xe_sched_job_is_ulls(struct xe_sched_job *job)
+{
+	return job->ulls != ULLS_NONE;
+}
+
+/**
+ * xe_sched_job_ulls_has_batch() - Does a job carry batch buffers
+ * @job: Xe schedule job object
+ *
+ * The ULLS jobs which enter and exit ULLS mode exist only to move the
+ * migration context on and off the hardware, and carry no batch buffers.
+ *
+ * Return: True if @job carries batch buffers, False otherwise.
+ */
+static inline bool xe_sched_job_ulls_has_batch(struct xe_sched_job *job)
+{
+	return job->ulls == ULLS_NONE || job->ulls == ULLS_ACTIVE;
+}
+
+/**
+ * xe_sched_job_ulls_parks() - Does a job park the engine for its successor
+ * @job: Xe schedule job object
+ *
+ * A ULLS job which is not the last one emits a postamble, parking the engine
+ * on its successor's semaphore and publishing that successor's ring tail.
+ *
+ * Return: True if @job emits a ULLS postamble, False otherwise.
+ */
+static inline bool xe_sched_job_ulls_parks(struct xe_sched_job *job)
+{
+	return job->ulls == ULLS_ENTER || job->ulls == ULLS_ACTIVE;
+}
+
+/**
+ * xe_sched_job_ulls_is_chained() - Has a job's predecessor already published it
+ * @job: Xe schedule job object
+ *
+ * A ULLS job which is not the first one has had its ring tail published by its
+ * predecessor's postamble, which also left the engine parked on this job's
+ * semaphore. Submitting it is a semaphore write alone - no H2G and no ring
+ * tail write.
+ *
+ * Return: True if @job was published by its predecessor, False otherwise.
+ */
+static inline bool xe_sched_job_ulls_is_chained(struct xe_sched_job *job)
+{
+	return job->ulls == ULLS_ACTIVE || job->ulls == ULLS_EXIT;
+}
+
 bool xe_sched_job_is_migration(struct xe_exec_queue *q);
 
 struct xe_sched_job_snapshot *xe_sched_job_snapshot_capture(struct xe_sched_job *job);
diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h
index 0490b1247a6e91..c8cdbe68843f68 100644
--- a/drivers/gpu/drm/xe/xe_sched_job_types.h
+++ b/drivers/gpu/drm/xe/xe_sched_job_types.h
@@ -10,10 +10,28 @@
 
 #include <drm/gpu_scheduler.h>
 
-struct xe_exec_queue;
 struct dma_fence;
 struct dma_fence_chain;
 
+struct xe_exec_queue;
+struct xe_cpu_bind_pt_update_ops;
+struct xe_pt_job_ops;
+struct xe_tile;
+struct xe_vm;
+
+/**
+ * struct xe_pt_update_args - PT update arguments
+ */
+struct xe_pt_update_args {
+	/** @vm: VM which is being bound */
+	struct xe_vm *vm;
+	/** @ops: CPU bind PT update ops */
+	const struct xe_cpu_bind_pt_update_ops *ops;
+#define XE_PT_UPDATE_JOB_OPS_COUNT	2
+	/** @pt_job_ops: PT job ops state */
+	struct xe_pt_job_ops *pt_job_ops[XE_PT_UPDATE_JOB_OPS_COUNT];
+};
+
 /**
  * struct xe_job_ptrs - Per hw engine instance data
  */
@@ -31,6 +49,23 @@ struct xe_job_ptrs {
 	u32 head;
 };
 
+/**
+ * enum xe_ulls_state - ULLS state of a migration job
+ *
+ * Describes where a job sits in a ULLS (Ultra Low Latency Submission)
+ * sequence. See the ULLS documentation in xe_migrate.c.
+ */
+enum xe_ulls_state {
+	/** @ULLS_NONE: Not a ULLS job */
+	ULLS_NONE = 0,
+	/** @ULLS_ENTER: Job which enters ULLS mode */
+	ULLS_ENTER,
+	/** @ULLS_ACTIVE: Job submitted while in ULLS mode */
+	ULLS_ACTIVE,
+	/** @ULLS_EXIT: Job which exits ULLS mode */
+	ULLS_EXIT,
+};
+
 /**
  * struct xe_sched_job - Xe schedule job (batch buffer tracking)
  */
@@ -61,6 +96,8 @@ struct xe_sched_job {
 	u32 migrate_flush_flags;
 	/** @sample_timestamp: Sampling of job timestamp in TDR */
 	u64 sample_timestamp;
+	/** @ulls: ULLS state of this job */
+	enum xe_ulls_state ulls;
 	/** @ring_ops_flush_tlb: The ring ops need to flush TLB before payload. */
 	bool ring_ops_flush_tlb;
 	/** @ring_ops_force_reset: The ring ops need to trigger a reset before payload. */
@@ -71,8 +108,14 @@ struct xe_sched_job {
 	bool restore_replay;
 	/** @last_replay: last job being replayed */
 	bool last_replay;
-	/** @ptrs: per instance pointers. */
-	struct xe_job_ptrs ptrs[];
+	/** @is_pt_job: is a PT job */
+	bool is_pt_job;
+	union {
+		/** @ptrs: per instance pointers. */
+		DECLARE_FLEX_ARRAY(struct xe_job_ptrs, ptrs);
+		/** @pt_update: PT update arguments */
+		DECLARE_FLEX_ARRAY(struct xe_pt_update_args, pt_update);
+	};
 };
 
 struct xe_sched_job_snapshot {
diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c
index e9ae9c9744eab3..f50dc4e3a611b3 100644
--- a/drivers/gpu/drm/xe/xe_sriov_packet.c
+++ b/drivers/gpu/drm/xe/xe_sriov_packet.c
@@ -394,24 +394,43 @@ static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
 static int descriptor_decoder(void *arg, u16 key, u16 len, const u32 *value)
 {
 	struct xe_device *xe = arg;
+	int err;
 
 	xe_sriov_dbg_verbose(xe, "found KLV %#x %s\n", key, xe_guc_klv_key_to_string(key));
 
 	switch (key) {
-	case MIGRATION_KLV_DEVICE_DEVID_KEY:
-		if (*value != xe->info.devid) {
+	case MIGRATION_KLV_DEVICE_DEVID_KEY: {
+		u16 devid;
+
+		err = xe_guc_klv_decode_u16(value, len, &devid);
+		if (err) {
+			xe_sriov_warn(xe, "Aborting migration, corrupted key %#x (%pe)\n",
+				      key, ERR_PTR(err));
+			return err;
+		}
+		if (devid != xe->info.devid) {
 			xe_sriov_warn(xe, "Aborting migration, devid mismatch %#06x!=%#06x\n",
-				      *value, xe->info.devid);
+				      devid, xe->info.devid);
 			return -ENODEV;
 		}
 		break;
-	case MIGRATION_KLV_DEVICE_REVID_KEY:
-		if (*value != xe->info.revid) {
+	}
+	case MIGRATION_KLV_DEVICE_REVID_KEY: {
+		u16 revid;
+
+		err = xe_guc_klv_decode_u16(value, len, &revid);
+		if (err) {
+			xe_sriov_warn(xe, "Aborting migration, corrupted key %#x (%pe)\n",
+				      key, ERR_PTR(err));
+			return err;
+		}
+		if (revid != xe->info.revid) {
 			xe_sriov_warn(xe, "Aborting migration, revid mismatch %#06x!=%#06x\n",
-				      *value, xe->info.revid);
+				      revid, xe->info.revid);
 			return -ENODEV;
 		}
 		break;
+	}
 	default:
 		if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
 			struct drm_printer p = xe_dbg_printer(xe);
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
index 0ec7ea83f12acf..c37e422b5a5cdd 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
@@ -112,7 +112,7 @@ static int pf_reprovision_default(struct xe_device *xe)
 	int result = 0;
 	int err;
 
-	guard(mutex)(xe_sriov_pf_master_mutex(xe));
+	lockdep_assert_held(xe_sriov_pf_master_mutex(xe));
 
 	for_each_gt(gt, xe, id) {
 		err = xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, false);
@@ -138,6 +138,8 @@ int xe_sriov_pf_reprovision_default(struct xe_device *xe)
 {
 	xe_assert(xe, IS_SRIOV_PF(xe));
 
+	guard(mutex)(xe_sriov_pf_master_mutex(xe));
+
 	if (!pf_auto_provisioning_mode(xe))
 		return 0;
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_printk.h b/drivers/gpu/drm/xe/xe_sriov_printk.h
index 4c6b5c3d2190b5..5931c918655ff2 100644
--- a/drivers/gpu/drm/xe/xe_sriov_printk.h
+++ b/drivers/gpu/drm/xe/xe_sriov_printk.h
@@ -36,6 +36,9 @@
 #define xe_sriov_dbg(xe, fmt, ...) \
 	xe_sriov_printk((xe), dbg, fmt, ##__VA_ARGS__)
 
+#define xe_sriov_dbg_ratelimited(xe, fmt, ...) \
+	xe_sriov_printk((xe), dbg_ratelimited, fmt, ##__VA_ARGS__)
+
 /* for low level noisy debug messages */
 #ifdef CONFIG_DRM_XE_DEBUG_SRIOV
 #define xe_sriov_dbg_verbose(xe, fmt, ...) xe_sriov_dbg(xe, fmt, ##__VA_ARGS__)
diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
index a8c831fbee3b5a..a54138461f4426 100644
--- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
+++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c
@@ -354,7 +354,6 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe)
 		ctx->ctx_id = ctx_id;
 
 		flags = EXEC_QUEUE_FLAG_KERNEL |
-			EXEC_QUEUE_FLAG_PERMANENT |
 			EXEC_QUEUE_FLAG_MIGRATE;
 		q = xe_exec_queue_create_bind(xe, tile, NULL, flags, 0);
 		if (IS_ERR(q)) {
diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c
index 4c506027fa94c9..45d44ebe288b4f 100644
--- a/drivers/gpu/drm/xe/xe_survivability_mode.c
+++ b/drivers/gpu/drm/xe/xe_survivability_mode.c
@@ -14,9 +14,11 @@
 #include "xe_device.h"
 #include "xe_heci_gsc.h"
 #include "xe_i2c.h"
+#include "xe_log.h"
 #include "xe_mmio.h"
 #include "xe_nvm.h"
 #include "xe_pcode_api.h"
+#include "xe_printk.h"
 #include "xe_vsec.h"
 
 /**
@@ -172,18 +174,32 @@ static void populate_survivability_info(struct xe_device *xe)
 	}
 }
 
-static void log_survivability_info(struct pci_dev *pdev)
+static const char *boot_status_str(u8 boot_status)
+{
+	switch (boot_status) {
+	case CRITICAL_FAILURE:
+		return "Critical Failure";
+	case NON_CRITICAL_FAILURE:
+		return "Non Critical Failure";
+	default:
+		return "Other";
+	}
+}
+
+static void log_survivability_info(struct xe_device *xe)
 {
-	struct xe_device *xe = pdev_to_xe_device(pdev);
 	struct xe_survivability *survivability = &xe->survivability;
 	u32 *info = survivability->info;
 	int id;
 
-	dev_info(&pdev->dev, "Survivability Boot Status : Critical Failure (%d)\n",
-		 survivability->boot_status);
+	xe_log_info(xe, SURVIVABILITY, "Boot Status: %#x (%s)\n",
+		    survivability->boot_status,
+		    boot_status_str(survivability->boot_status));
+
 	for (id = 0; id < MAX_SCRATCH_REG; id++) {
-		if (info[id])
-			dev_info(&pdev->dev, "%s: 0x%x\n", reg_map[id], info[id]);
+		if (!info[id])
+			continue;
+		xe_log_info(xe, SURVIVABILITY, "%s: %#x\n", reg_map[id], info[id]);
 	}
 }
 
@@ -289,41 +305,46 @@ static const struct attribute_group survivability_info_group = {
 
 static int create_survivability_sysfs(struct pci_dev *pdev)
 {
-	struct device *dev = &pdev->dev;
+	/* Survivability info is required if not enabled via configfs */
+	bool needs_info = !xe_configfs_get_survivability_mode(pdev);
 	struct xe_device *xe = pdev_to_xe_device(pdev);
+	struct device *dev = &pdev->dev;
 	int ret;
 
 	ret = device_create_file(dev, &dev_attr_survivability_mode);
-	if (ret) {
-		dev_warn(dev, "Failed to create survivability sysfs files\n");
-		return ret;
-	}
+	if (ret)
+		goto failed;
 
 	ret = devm_add_action_or_reset(xe->drm.dev,
 				       xe_survivability_mode_fini, xe);
 	if (ret)
-		return ret;
+		goto failed;
 
-	/* Survivability info is not required if enabled via configfs */
-	if (!xe_configfs_get_survivability_mode(pdev)) {
+	if (needs_info) {
 		ret = devm_device_add_group(dev, &survivability_info_group);
 		if (ret)
-			return ret;
+			goto failed;
 	}
 
 	return 0;
+
+failed:
+	xe_err(xe, "Failed to create survivability sysfs files: %pe\n", ERR_PTR(ret));
+	/* no sysfs, dump Survivability info to dmesg instead */
+	if (needs_info)
+		log_survivability_info(xe);
+	return ret;
 }
 
 static int enable_boot_survivability_mode(struct pci_dev *pdev)
 {
-	struct device *dev = &pdev->dev;
 	struct xe_device *xe = pdev_to_xe_device(pdev);
 	struct xe_survivability *survivability = &xe->survivability;
-	int ret = 0;
+	int ret;
 
 	ret = create_survivability_sysfs(pdev);
 	if (ret)
-		return ret;
+		goto failed;
 
 	/* Make sure xe_heci_gsc_init() and xe_i2c_probe() are aware of survivability */
 	survivability->mode = true;
@@ -335,19 +356,22 @@ static int enable_boot_survivability_mode(struct pci_dev *pdev)
 	if (survivability->fdo_mode) {
 		ret = xe_nvm_init(xe);
 		if (ret)
-			goto err;
+			goto failed;
 	}
 
 	ret = xe_i2c_probe(xe);
 	if (ret)
-		goto err;
+		goto failed;
 
-	dev_err(dev, "In Survivability Mode\n");
+	if (check_boot_failure(xe))
+		xe_log_err_fatal(pdev, SURVIVABILITY, 0, "Boot Mode enabled!\n");
+	else
+		xe_log_info(pdev, SURVIVABILITY, "Boot Mode enabled!\n");
 
 	return 0;
 
-err:
-	dev_err(dev, "Failed to enable Survivability Mode\n");
+failed:
+	xe_log_err_fatal(pdev, SURVIVABILITY, ret, "Failed to enable Boot Mode!\n");
 	survivability->mode = false;
 	return ret;
 }
@@ -412,21 +436,22 @@ void xe_survivability_mode_runtime_enable(struct xe_device *xe)
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 
 	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) {
-		dev_err(&pdev->dev, "Runtime Survivability Mode not supported\n");
+		xe_log_err(xe, SURVIVABILITY, -EOPNOTSUPP, "Runtime Mode not supported!\n");
 		return;
 	}
 
 	populate_survivability_info(xe);
-
-	if (create_survivability_sysfs(pdev))
-		dev_err(&pdev->dev, "Failed to create survivability sysfs\n");
+	create_survivability_sysfs(pdev);
 
 	survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME;
-	dev_err(&pdev->dev, "Runtime Survivability mode enabled\n");
+	xe_log_err(xe, SURVIVABILITY, 0, "Runtime Mode enabled!\n");
 
 	xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR);
 	xe_device_declare_wedged(xe);
-	dev_err(&pdev->dev, "Firmware flash required, Please refer to the userspace documentation for more details!\n");
+
+	xe_log_err(xe, SURVIVABILITY, 0, "Firmware flash required!\n");
+	xe_info(xe, "Please refer to the userspace documentation for more details how to flash the firmware on %s!\n",
+		xe->info.platform_name);
 }
 
 /**
@@ -452,7 +477,7 @@ int xe_survivability_mode_boot_enable(struct xe_device *xe)
 	 * v2 supports survivability mode for critical errors
 	 */
 	if (survivability->version < 2  && survivability->boot_status == CRITICAL_FAILURE) {
-		log_survivability_info(pdev);
+		log_survivability_info(xe);
 		return -ENXIO;
 	}
 
diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index b228a737cfd6d3..f39e647512ad93 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -13,8 +13,10 @@
 #include "xe_bo.h"
 #include "xe_exec_queue_types.h"
 #include "xe_gt_stats.h"
+#include "xe_log.h"
 #include "xe_migrate.h"
 #include "xe_module.h"
+#include "xe_pagefault.h"
 #include "xe_pm.h"
 #include "xe_pt.h"
 #include "xe_svm.h"
@@ -115,6 +117,7 @@ xe_svm_range_alloc(struct drm_gpusvm *gpusvm)
 		return NULL;
 
 	INIT_LIST_HEAD(&range->garbage_collector_link);
+	mutex_init(&range->lock);
 	drm_gpusvm_init_pages(&range->pages, &gpusvm_to_vm(gpusvm)->xe->drm);
 	xe_vm_get(gpusvm_to_vm(gpusvm));
 
@@ -125,6 +128,7 @@ static void xe_svm_range_free(struct drm_gpusvm_range *range)
 {
 	drm_gpusvm_free_pages(range->gpusvm, &(to_xe_range(range)->pages),
 			      drm_gpusvm_range_size(range) >> PAGE_SHIFT);
+	mutex_destroy(&to_xe_range(range)->lock);
 	xe_vm_put(range_to_vm(range));
 	kfree(to_xe_range(range));
 }
@@ -140,13 +144,13 @@ xe_svm_garbage_collector_add_range(struct xe_vm *vm, struct xe_svm_range *range,
 	drm_gpusvm_range_set_unmapped(&range->base, &range->pages, 1,
 				      mmu_range);
 
-	spin_lock(&vm->svm.garbage_collector.lock);
+	spin_lock(&vm->svm.garbage_collector.list_lock);
 	if (list_empty(&range->garbage_collector_link))
 		list_add_tail(&range->garbage_collector_link,
 			      &vm->svm.garbage_collector.range_list);
-	spin_unlock(&vm->svm.garbage_collector.lock);
+	spin_unlock(&vm->svm.garbage_collector.list_lock);
 
-	queue_work(xe->usm.pf_wq, &vm->svm.garbage_collector.work);
+	queue_work(xe->usm.pagefault_wq, &vm->svm.garbage_collector.work);
 }
 
 static void xe_svm_tlb_inval_count_stats_incr(struct xe_gt *gt)
@@ -309,18 +313,30 @@ static int __xe_svm_garbage_collector(struct xe_vm *vm,
 
 	range_debug(range, "GARBAGE COLLECTOR");
 
-	xe_vm_lock(vm, false);
-	fence = xe_vm_range_unbind(vm, range);
-	xe_vm_unlock(vm);
-	if (IS_ERR(fence))
-		return PTR_ERR(fence);
-	dma_fence_put(fence);
+	scoped_guard(mutex, &range->lock) {
+		drm_gpusvm_range_get(&range->base);
+		range->removed = true;
+
+		range_debug(range, "GARBAGE COLLECTOR");
+
+		xe_vm_lock(vm, false);
+		fence = xe_vm_range_unbind(vm, range);
+		xe_vm_unlock(vm);
+		if (IS_ERR(fence)) {
+			drm_gpusvm_range_put(&range->base);
+			return PTR_ERR(fence);
+		}
+		dma_fence_put(fence);
 
-	drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &range->pages,
-			       drm_gpusvm_range_size(&range->base) >> PAGE_SHIFT,
-			       &ctx);
+		drm_gpusvm_unmap_pages(&vm->svm.gpusvm, &range->pages,
+				       drm_gpusvm_range_size(&range->base) >> PAGE_SHIFT,
+				       &ctx);
 
-	drm_gpusvm_range_remove(&vm->svm.gpusvm, &range->base);
+		scoped_guard(mutex, &vm->svm.range_lock)
+			drm_gpusvm_range_remove(&vm->svm.gpusvm, &range->base);
+	}
+
+	drm_gpusvm_range_put(&range->base);
 
 	return 0;
 }
@@ -393,13 +409,15 @@ static int xe_svm_garbage_collector(struct xe_vm *vm)
 	u64 range_end;
 	int err, ret = 0;
 
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 
 	if (xe_vm_is_closed_or_banned(vm))
 		return -ENOENT;
 
+	guard(mutex)(&vm->svm.garbage_collector.lock);
+
 	for (;;) {
-		spin_lock(&vm->svm.garbage_collector.lock);
+		spin_lock(&vm->svm.garbage_collector.list_lock);
 		range = list_first_entry_or_null(&vm->svm.garbage_collector.range_list,
 						 typeof(*range),
 						 garbage_collector_link);
@@ -410,7 +428,7 @@ static int xe_svm_garbage_collector(struct xe_vm *vm)
 		range_end = xe_svm_range_end(range);
 
 		list_del(&range->garbage_collector_link);
-		spin_unlock(&vm->svm.garbage_collector.lock);
+		spin_unlock(&vm->svm.garbage_collector.list_lock);
 
 		err = __xe_svm_garbage_collector(vm, range);
 		if (err) {
@@ -429,7 +447,7 @@ static int xe_svm_garbage_collector(struct xe_vm *vm)
 				return err;
 		}
 	}
-	spin_unlock(&vm->svm.garbage_collector.lock);
+	spin_unlock(&vm->svm.garbage_collector.list_lock);
 
 	return ret;
 }
@@ -439,9 +457,8 @@ static void xe_svm_garbage_collector_work_func(struct work_struct *w)
 	struct xe_vm *vm = container_of(w, struct xe_vm,
 					svm.garbage_collector.work);
 
-	down_write(&vm->lock);
+	guard(rwsem_read)(&vm->lock);
 	xe_svm_garbage_collector(vm);
-	up_write(&vm->lock);
 }
 
 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP)
@@ -893,8 +910,11 @@ int xe_svm_init(struct xe_vm *vm)
 {
 	int err;
 
+	mutex_init(&vm->svm.range_lock);
+	mutex_init(&vm->svm.garbage_collector.lock);
+
 	if (vm->flags & XE_VM_FLAG_FAULT_MODE) {
-		spin_lock_init(&vm->svm.garbage_collector.lock);
+		spin_lock_init(&vm->svm.garbage_collector.list_lock);
 		INIT_LIST_HEAD(&vm->svm.garbage_collector.range_list);
 		INIT_WORK(&vm->svm.garbage_collector.work,
 			  xe_svm_garbage_collector_work_func);
@@ -903,12 +923,12 @@ int xe_svm_init(struct xe_vm *vm)
 		err = drm_pagemap_acquire_owner(&vm->svm.peer, &xe_owner_list,
 						xe_has_interconnect);
 		if (err)
-			return err;
+			goto out_err;
 
 		err = xe_svm_get_pagemaps(vm);
 		if (err) {
 			drm_pagemap_release_owner(&vm->svm.peer);
-			return err;
+			goto out_err;
 		}
 
 		err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM",
@@ -916,19 +936,27 @@ int xe_svm_init(struct xe_vm *vm)
 				      xe_modparam.svm_notifier_size * SZ_1M,
 				      &gpusvm_ops, fault_chunk_sizes,
 				      ARRAY_SIZE(fault_chunk_sizes));
-		drm_gpusvm_driver_set_lock(&vm->svm.gpusvm, &vm->lock);
+		drm_gpusvm_driver_set_lock(&vm->svm.gpusvm, &vm->svm.range_lock);
 
 		if (err) {
 			xe_svm_put_pagemaps(vm);
 			drm_pagemap_release_owner(&vm->svm.peer);
-			return err;
+			goto out_err;
 		}
 	} else {
 		err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)",
 				      NULL, 0, 0, 0, NULL,
 				      NULL, 0);
+		if (err)
+			goto out_err;
 	}
 
+	return 0;
+
+out_err:
+	mutex_destroy(&vm->svm.range_lock);
+	mutex_destroy(&vm->svm.garbage_collector.lock);
+
 	return err;
 }
 
@@ -969,7 +997,10 @@ void xe_svm_fini(struct xe_vm *vm)
 					       &ctx);
 	}
 
-	drm_gpusvm_fini(&vm->svm.gpusvm);
+	scoped_guard(mutex, &vm->svm.range_lock)
+		drm_gpusvm_fini(&vm->svm.gpusvm);
+	mutex_destroy(&vm->svm.range_lock);
+	mutex_destroy(&vm->svm.garbage_collector.lock);
 }
 
 static bool xe_svm_range_has_pagemap_locked(const struct xe_svm_range *range,
@@ -1022,6 +1053,7 @@ void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range)
  * @tile_mask: Mask representing the tiles to be checked
  * @dpagemap: if !%NULL, the range is expected to be present
  * in device memory identified by this parameter.
+ * @valid_pages: Pages are valid, result written back to caller
  *
  * The xe_svm_range_validate() function checks if a range is
  * valid and located in the desired memory region.
@@ -1030,7 +1062,8 @@ void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range)
  */
 bool xe_svm_range_validate(struct xe_vm *vm,
 			   struct xe_svm_range *range,
-			   u8 tile_mask, const struct drm_pagemap *dpagemap)
+			   u8 tile_mask, const struct drm_pagemap *dpagemap,
+			   bool *valid_pages)
 {
 	bool ret;
 
@@ -1042,6 +1075,8 @@ bool xe_svm_range_validate(struct xe_vm *vm,
 	else
 		ret = ret && !range->pages.dpagemap;
 
+	*valid_pages = xe_svm_range_pages_valid(range);
+
 	xe_svm_notifier_unlock(vm);
 
 	return ret;
@@ -1231,8 +1266,8 @@ DECL_SVM_RANGE_US_STATS(bind, BIND)
 DECL_SVM_RANGE_US_STATS(fault, PAGEFAULT)
 
 static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
-				     struct xe_gt *gt, u64 fault_addr,
-				     bool need_vram)
+				     struct xe_pagefault *pf, struct xe_gt *gt,
+				     u64 fault_addr, bool need_vram)
 {
 	int devmem_possible = IS_DGFX(vm->xe) &&
 		IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
@@ -1246,40 +1281,57 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 	};
 	struct xe_validation_ctx vctx;
 	struct drm_exec exec;
-	struct xe_svm_range *range;
+	struct xe_svm_range *range = NULL;
 	struct drm_gpusvm_range_flags range_flags;
 	struct dma_fence *fence;
 	struct drm_pagemap *dpagemap;
 	struct xe_tile *tile = gt_to_tile(gt);
 	int migrate_try_count = ctx.devmem_only ? 3 : 1;
 	ktime_t start = xe_gt_stats_ktime_get(), bind_start, get_pages_start;
-	int err;
+	int err = 0;
 
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 	xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma));
 
 	xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT, 1);
 
 retry:
+	/* Release old range */
+	if (range) {
+		mutex_unlock(&range->lock);
+		drm_gpusvm_range_put(&range->base);
+	}
+
 	/* Always process UNMAPs first so view SVM ranges is current */
 	err = xe_svm_garbage_collector(vm);
-	if (err)
+	if (err) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR);
 		return err;
+	}
 
 	dpagemap = ctx.devmem_only ? xe_tile_local_pagemap(tile) :
 		xe_vma_resolve_pagemap(vma, tile);
 	ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap);
 	range = xe_svm_range_find_or_insert(vm, fault_addr, vma, &ctx);
 
-	if (IS_ERR(range))
+	if (IS_ERR(range)) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND);
 		return PTR_ERR(range);
+	}
 
 	xe_svm_range_fault_count_stats_incr(gt, range);
 
+	mutex_lock(&range->lock);
+
+	if (xe_svm_range_is_removed(range))
+		goto retry;
+
 	/* READ_ONCE pairs with WRITE_ONCE in drm_gpusvm_range_set_unmapped() */
 	range_flags.__flags = READ_ONCE(range->base.flags.__flags);
-	if (ctx.devmem_only && !range_flags.migrate_devmem)
-		return -EACCES;
+	if (ctx.devmem_only && !range_flags.migrate_devmem) {
+		err = -EACCES;
+		goto err_out;
+	}
 
 	if (xe_svm_range_is_valid(range, tile, ctx.devmem_only, dpagemap)) {
 		xe_svm_range_valid_fault_count_stats_incr(gt, range);
@@ -1314,10 +1366,10 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 				else
 					goto retry;
 			} else {
-				drm_err(&vm->xe->drm,
-					"VRAM allocation failed, retry count exceeded, asid=%u, errno=%pe\n",
-					vm->usm.asid, ERR_PTR(err));
-				return err;
+				xe_log_err(gt, PAGEFAULT, err,
+					   "VRAM allocation failed, retry count exceeded, ASID=%u\n",
+					   vm->usm.asid);
+				goto err_out;
 			}
 		}
 	}
@@ -1337,14 +1389,14 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 			range_debug(range, "PAGE FAULT - RETRY PAGES");
 			goto retry;
 		} else {
-			drm_err(&vm->xe->drm,
-				"Get pages failed, retry count exceeded, asid=%u, gpusvm=%p, errno=%pe\n",
-				vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
+			xe_log_err(gt, PAGEFAULT, err,
+				   "Get pages failed, retry count exceeded, ASID=%u, GPUVM=%s\n",
+				   vm->usm.asid, vm->svm.gpusvm.name);
 		}
 	}
 	if (err) {
 		range_debug(range, "PAGE FAULT - FAIL PAGE COLLECT");
-		goto out;
+		goto err_out;
 	} else if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) {
 		drm_dbg(&vm->xe->drm, "After page collect data location is %sin \"%s\".\n",
 			xe_svm_range_has_pagemap(range, dpagemap) ? "" : "NOT ",
@@ -1367,6 +1419,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 			err = PTR_ERR(fence);
 			xe_validation_retry_on_oom(&vctx, &err);
 			xe_svm_range_bind_us_stats_incr(gt, range, bind_start);
+			xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_REBIND);
 			break;
 		}
 	}
@@ -1378,16 +1431,26 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 	xe_svm_range_bind_us_stats_incr(gt, range, bind_start);
 
 out:
+	/* Give hint to immediately ack faults */
+	xe_pagefault_set_start_addr(pf,  xe_svm_range_start(range));
+	xe_pagefault_set_end_addr(pf, xe_svm_range_end(range));
+
 	xe_svm_range_fault_us_stats_incr(gt, range, start);
+	mutex_unlock(&range->lock);
+	drm_gpusvm_range_put(&range->base);
 	return 0;
 
 err_out:
 	if (err == -EAGAIN) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_NONE);
 		ctx.timeslice_ms <<= 1;	/* Double timeslice if we have to retry */
 		range_debug(range, "PAGE FAULT - RETRY BIND");
 		goto retry;
 	}
 
+	mutex_unlock(&range->lock);
+	drm_gpusvm_range_put(&range->base);
+
 	return err;
 }
 
@@ -1395,6 +1458,7 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
  * xe_svm_handle_pagefault() - SVM handle page fault
  * @vm: The VM.
  * @vma: The CPU address mirror VMA.
+ * @pf: Pagefault structure
  * @gt: The gt upon the fault occurred.
  * @fault_addr: The GPU fault address.
  * @atomic: The fault atomic access bit.
@@ -1405,16 +1469,18 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
  * Return: 0 on success, negative error code on error.
  */
 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
-			    struct xe_gt *gt, u64 fault_addr,
-			    bool atomic)
+			    struct xe_pagefault *pf, struct xe_gt *gt,
+			    u64 fault_addr, bool atomic)
 {
 	int need_vram, ret;
 retry:
 	need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic);
-	if (need_vram < 0)
+	if (need_vram < 0) {
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK);
 		return need_vram;
+	}
 
-	ret =  __xe_svm_handle_pagefault(vm, vma, gt, fault_addr,
+	ret =  __xe_svm_handle_pagefault(vm, vma, pf, gt, fault_addr,
 					 need_vram ? true : false);
 	if (ret == -EAGAIN) {
 		/*
@@ -1422,11 +1488,16 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
 		 * may have been split by xe_svm_range_set_default_attr.
 		 */
 		vma = xe_vm_find_vma_by_addr(vm, fault_addr);
-		if (!vma)
+		if (!vma) {
+			xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND);
 			return -EINVAL;
+		}
 
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_NONE);
 		goto retry;
 	}
+	if (ret && xe_pagefault_get_error(pf) == XE_PAGEFAULT_ERROR_NONE)
+		xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED);
 	return ret;
 }
 
@@ -1470,9 +1541,9 @@ void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end)
 				drm_gpusvm_range_get(range);
 				__xe_svm_garbage_collector(vm, to_xe_range(range));
 				if (!list_empty(&to_xe_range(range)->garbage_collector_link)) {
-					spin_lock(&vm->svm.garbage_collector.lock);
+					spin_lock(&vm->svm.garbage_collector.list_lock);
 					list_del(&to_xe_range(range)->garbage_collector_link);
-					spin_unlock(&vm->svm.garbage_collector.lock);
+					spin_unlock(&vm->svm.garbage_collector.list_lock);
 				}
 				drm_gpusvm_range_put(range);
 			}
@@ -1502,7 +1573,7 @@ int xe_svm_bo_evict(struct xe_bo *bo)
  * @ctx: GPU SVM context
  *
  * This function finds or inserts a newly allocated a SVM range based on the
- * address.
+ * address. Take a reference to SVM range on success.
  *
  * Return: Pointer to the SVM range on success, ERR_PTR() on failure.
  */
@@ -1511,11 +1582,15 @@ struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr,
 {
 	struct drm_gpusvm_range *r;
 
+	guard(mutex)(&vm->svm.range_lock);
+
 	r = drm_gpusvm_range_find_or_insert(&vm->svm.gpusvm, max(addr, xe_vma_start(vma)),
 					    xe_vma_start(vma), xe_vma_end(vma), ctx);
 	if (IS_ERR(r))
 		return ERR_CAST(r);
 
+	drm_gpusvm_range_get(r);
+
 	return to_xe_range(r);
 }
 
@@ -1535,7 +1610,9 @@ int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range,
 {
 	int err = 0;
 
-	err = drm_gpusvm_get_pages(&vm->svm.gpusvm, &range->pages,
+	lockdep_assert_held(&range->lock);
+
+	err = drm_gpusvm_get_pages(&vm->svm.gpusvm, &range->pages, 1,
 				   vm->svm.gpusvm.mm,
 				   &range->base.notifier->notifier,
 				   drm_gpusvm_range_start(&range->base),
@@ -1659,6 +1736,7 @@ int xe_svm_alloc_vram(struct xe_svm_range *range, const struct drm_gpusvm_ctx *c
 		.__flags = READ_ONCE(range->base.flags.__flags),
 	};
 
+	lockdep_assert_held(&range->lock);
 	xe_assert(range_to_vm(&range->base)->xe, flags.migrate_devmem);
 	range_debug(range, "ALLOCATE VRAM");
 
diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h
index a921556d346621..2ef4ef026ccdc7 100644
--- a/drivers/gpu/drm/xe/xe_svm.h
+++ b/drivers/gpu/drm/xe/xe_svm.h
@@ -21,6 +21,7 @@ struct drm_file;
 struct xe_bo;
 struct xe_gt;
 struct xe_device;
+struct xe_pagefault;
 struct xe_vram_region;
 struct xe_tile;
 struct xe_vm;
@@ -38,6 +39,13 @@ struct xe_svm_range {
 	 * list. Protected by VM's garbage collect lock.
 	 */
 	struct list_head garbage_collector_link;
+	/**
+	 * @lock: Protects fault handler, garbage collector, and prefetch
+	 * critical sections, ensuring only one thread operates on a range at a
+	 * time. Locking order: inside vm->lock and garbage collector, outside
+	 * dma-resv locks, vm->svm.range_lock.
+	 */
+	struct mutex lock;
 	/**
 	 * @tile_present: Tile mask of binding is present for this range.
 	 * Protected by GPU SVM notifier lock.
@@ -48,8 +56,22 @@ struct xe_svm_range {
 	 * range. Protected by GPU SVM notifier lock.
 	 */
 	u8 tile_invalidated;
+	/**
+	 * @removed: Range has been removed from GPU SVM tree, protected by
+	 * @lock.
+	 */
+	bool removed;
 };
 
+/**
+ * xe_svm_range_put() - SVM range put
+ * @range: SVM range
+ */
+static inline void xe_svm_range_put(struct xe_svm_range *range)
+{
+	drm_gpusvm_range_put(&range->base);
+}
+
 /**
  * struct xe_pagemap - Manages xe device_private memory for SVM.
  * @pagemap: The struct dev_pagemap providing the struct pages.
@@ -88,8 +110,8 @@ void xe_svm_fini(struct xe_vm *vm);
 void xe_svm_close(struct xe_vm *vm);
 
 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
-			    struct xe_gt *gt, u64 fault_addr,
-			    bool atomic);
+			    struct xe_pagefault *pf, struct xe_gt *gt,
+			    u64 fault_addr, bool atomic);
 
 bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end);
 
@@ -113,7 +135,8 @@ void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range);
 
 bool xe_svm_range_validate(struct xe_vm *vm,
 			   struct xe_svm_range *range,
-			   u8 tile_mask, const struct drm_pagemap *dpagemap);
+			   u8 tile_mask, const struct drm_pagemap *dpagemap,
+			   bool *valid_pages);
 
 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 addr, u64 end,  struct xe_vma *vma);
 
@@ -137,6 +160,19 @@ static inline bool xe_svm_range_has_dma_mapping(struct xe_svm_range *range)
 	return range->pages.flags.has_dma_mapping;
 }
 
+/**
+ * xe_svm_range_is_removed() - SVM range is removed from GPU SVM tree
+ * @range: SVM range
+ *
+ * Return: True if SVM range is removed from GPU SVM tree, False otherwise
+ */
+static inline bool xe_svm_range_is_removed(struct xe_svm_range *range)
+{
+	lockdep_assert_held(&range->lock);
+
+	return range->removed;
+}
+
 /**
  * to_xe_range - Convert a drm_gpusvm_range pointer to a xe_svm_range
  * @r: Pointer to the drm_gpusvm_range structure
@@ -184,6 +220,19 @@ static inline unsigned long xe_svm_range_size(struct xe_svm_range *range)
 	return drm_gpusvm_range_size(&range->base);
 }
 
+/**
+ * xe_svm_range_first_dma() - Resolve the device address array of a SVM range
+ * @range: SVM range
+ * @contiguous: Where to store whether one entry spans the whole range
+ *
+ * Return: Pointer to the first device address, NULL if none is populated.
+ */
+static inline const struct drm_pagemap_addr *
+xe_svm_range_first_dma(struct xe_svm_range *range, bool *contiguous)
+{
+	return drm_gpusvm_pages_first_dma(&range->pages, contiguous);
+}
+
 void xe_svm_flush(struct xe_vm *vm);
 
 int xe_pagemap_shrinker_create(struct xe_device *xe);
@@ -216,10 +265,15 @@ struct xe_svm_range {
 	struct {
 		const struct drm_pagemap_addr *dma_addr;
 	} pages;
+	struct mutex lock;
 	u32 tile_present;
 	u32 tile_invalidated;
 };
 
+static inline void xe_svm_range_put(struct xe_svm_range *range)
+{
+}
+
 static inline bool xe_svm_range_pages_valid(struct xe_svm_range *range)
 {
 	return false;
@@ -258,8 +312,8 @@ void xe_svm_close(struct xe_vm *vm)
 
 static inline
 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,
-			    struct xe_gt *gt, u64 fault_addr,
-			    bool atomic)
+			    struct xe_pagefault *pf, struct xe_gt *gt,
+			    u64 fault_addr, bool atomic)
 {
 	return 0;
 }
@@ -337,7 +391,8 @@ void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range)
 static inline
 bool xe_svm_range_validate(struct xe_vm *vm,
 			   struct xe_svm_range *range,
-			   u8 tile_mask, bool devmem_preferred)
+			   u8 tile_mask, const struct drm_pagemap *dpagemap,
+			   bool *valid_pages)
 {
 	return false;
 }
@@ -389,6 +444,18 @@ static inline struct drm_pagemap *xe_drm_pagemap_from_fd(int fd, u32 region_inst
 	return ERR_PTR(-ENOENT);
 }
 
+static inline bool xe_svm_range_is_removed(struct xe_svm_range *range)
+{
+	return false;
+}
+
+static inline const struct drm_pagemap_addr *
+xe_svm_range_first_dma(struct xe_svm_range *range, bool *contiguous)
+{
+	*contiguous = false;
+	return NULL;
+}
+
 #define xe_svm_range_has_dma_mapping(...) false
 #endif /* CONFIG_DRM_XE_GPUSVM */
 
diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
index 37866768d64c23..06b1c913588ab5 100644
--- a/drivers/gpu/drm/xe/xe_sync.c
+++ b/drivers/gpu/drm/xe/xe_sync.c
@@ -345,15 +345,9 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
 			return ERR_PTR(-EOPNOTSUPP);
 
 	if (q->flags & EXEC_QUEUE_FLAG_VM) {
-		struct xe_exec_queue *__q;
-		struct xe_tile *tile;
-		u8 id;
-
-		for_each_tile(tile, vm->xe, id) {
+		num_fence++;
+		for_each_tlb_inval(q, i)
 			num_fence++;
-			for_each_tlb_inval(i)
-				num_fence++;
-		}
 
 		fences = kmalloc_objs(*fences, num_fence);
 		if (!fences)
@@ -361,17 +355,9 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
 
 		fences[current_fence++] =
 			xe_exec_queue_last_fence_get(q, vm);
-		for_each_tlb_inval(i)
+		for_each_tlb_inval(q, i)
 			fences[current_fence++] =
 				xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
-		list_for_each_entry(__q, &q->multi_gt_list,
-				    multi_gt_link) {
-			fences[current_fence++] =
-				xe_exec_queue_last_fence_get(__q, vm);
-			for_each_tlb_inval(i)
-				fences[current_fence++] =
-					xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i);
-		}
 
 		xe_assert(vm->xe, current_fence == num_fence);
 		cf = dma_fence_array_create(num_fence, fences,
diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c
index 62ccc9be71b45a..4067e1dfdcd50d 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl.c
+++ b/drivers/gpu/drm/xe/xe_sysctrl.c
@@ -13,9 +13,11 @@
 #include "xe_device.h"
 #include "xe_mmio.h"
 #include "xe_pm.h"
+#include "xe_printk.h"
 #include "xe_soc_remapper.h"
 #include "xe_sysctrl.h"
 #include "xe_sysctrl_mailbox.h"
+#include "xe_sysctrl_mailbox_types.h"
 #include "xe_sysctrl_types.h"
 
 /**
@@ -29,6 +31,21 @@
  * This module provides initialization and support code for interacting
  * with System Controller through the mailbox interface.
  */
+
+/* Application status flags reported in xe_sysctrl_app_status_resp.flags */
+#define XE_SYSCTRL_APP_RESP_VALID		BIT(0)
+#define XE_SYSCTRL_APP_RESP_BOOTED		BIT(1)
+#define XE_SYSCTRL_APP_RESP_INITIALIZED	BIT(2)
+
+/*
+ * Known System Controller application identifiers, keyed by firmware
+ * application ID.
+ */
+enum xe_sysctrl_app_id {
+	XE_SYSCTRL_APP_OCODE	= 0x0C,
+	XE_SYSCTRL_APP_DIAG	= 0x0D,
+};
+
 static void sysctrl_fini(void *arg)
 {
 	struct xe_device *xe = arg;
@@ -125,3 +142,78 @@ void xe_sysctrl_pm_resume(struct xe_device *xe)
 
 	xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX);
 }
+
+static enum xe_sysctrl_fw_status
+xe_sysctrl_check_app_status(struct xe_device *xe, enum xe_sysctrl_app_id app_id)
+{
+	struct xe_sysctrl_app_status_req req = {};
+	struct xe_sysctrl_app_status_resp resp = {};
+	struct xe_sysctrl_mailbox_command cmd = {};
+	size_t out_len = 0;
+	u32 flags;
+	int ret;
+
+	req.app_id = (u8)app_id;
+
+	xe_sysctrl_create_command(&cmd, XE_SYSCTRL_GROUP_CORE, XE_SYSCTRL_CMD_GET_APP_STATUS_BY_ID,
+				  &req, sizeof(req), &resp, sizeof(resp));
+
+	ret = xe_sysctrl_send_command(&xe->sc, &cmd, &out_len);
+	if (ret)
+		return XE_SYSCTRL_FIRMWARE_COMM_FAILURE;
+
+	if (out_len != sizeof(resp)) {
+		xe_err(xe, "sysctrl: unexpected get app status response length %zu (expected %zu)\n",
+		       out_len, sizeof(resp));
+		return XE_SYSCTRL_FIRMWARE_COMM_FAILURE;
+	}
+
+	flags = resp.flags;
+
+	if (!(flags & XE_SYSCTRL_APP_RESP_VALID))
+		return XE_SYSCTRL_FIRMWARE_APP_INVALID;
+
+	if (!(flags & XE_SYSCTRL_APP_RESP_BOOTED))
+		return XE_SYSCTRL_FIRMWARE_APP_NOT_LOADED;
+
+	if (!(flags & XE_SYSCTRL_APP_RESP_INITIALIZED))
+		return XE_SYSCTRL_FIRMWARE_APP_BOOTED;
+
+	return XE_SYSCTRL_FIRMWARE_APP_INITIALIZED;
+}
+
+/**
+ * xe_sysctrl_is_oobmsm_fw_ready() - Check if oCode firmware is fully initialized
+ * @xe: xe device instance
+ *
+ * Returns true if oCode firmware has reached the initialized state, indicating
+ * it is ready to handle requests.
+ *
+ * Callers must only invoke this on platforms where System Controller is
+ * present (xe->info.has_sysctrl).
+ *
+ * Return: true if oCode firmware is initialized, false otherwise
+ */
+bool xe_sysctrl_is_oobmsm_fw_ready(struct xe_device *xe)
+{
+	return xe_sysctrl_check_app_status(xe, XE_SYSCTRL_APP_OCODE) ==
+	       XE_SYSCTRL_FIRMWARE_APP_INITIALIZED;
+}
+
+/**
+ * xe_sysctrl_is_diag_fw_ready() - Check if diag firmware is fully initialized
+ * @xe: xe device instance
+ *
+ * Returns true if diag firmware has reached the initialized state, indicating
+ * it is ready to handle requests.
+ *
+ * Callers must only invoke this on platforms where System Controller is
+ * present (xe->info.has_sysctrl).
+ *
+ * Return: true if diag firmware is initialized, false otherwise
+ */
+bool xe_sysctrl_is_diag_fw_ready(struct xe_device *xe)
+{
+	return xe_sysctrl_check_app_status(xe, XE_SYSCTRL_APP_DIAG) ==
+	       XE_SYSCTRL_FIRMWARE_APP_INITIALIZED;
+}
diff --git a/drivers/gpu/drm/xe/xe_sysctrl.h b/drivers/gpu/drm/xe/xe_sysctrl.h
index 090dffb6d55f60..b69a3f47423659 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl.h
+++ b/drivers/gpu/drm/xe/xe_sysctrl.h
@@ -20,5 +20,7 @@ void xe_sysctrl_event(struct xe_sysctrl *sc);
 int xe_sysctrl_init(struct xe_device *xe);
 void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl);
 void xe_sysctrl_pm_resume(struct xe_device *xe);
+bool xe_sysctrl_is_oobmsm_fw_ready(struct xe_device *xe);
+bool xe_sysctrl_is_diag_fw_ready(struct xe_device *xe);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event.c b/drivers/gpu/drm/xe/xe_sysctrl_event.c
index da395148ee9dc0..15ddd2d8bc6d2d 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_event.c
+++ b/drivers/gpu/drm/xe/xe_sysctrl_event.c
@@ -49,18 +49,6 @@ static void get_pending_event(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_c
 	} while (response->count);
 }
 
-static void event_request_prepare(struct xe_device *xe, struct xe_sysctrl_app_msg_hdr *header,
-				  struct xe_sysctrl_event_request *request)
-{
-	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
-
-	header->data = REG_FIELD_PREP(APP_HDR_GROUP_ID_MASK, XE_SYSCTRL_GROUP_GFSP) |
-		       REG_FIELD_PREP(APP_HDR_COMMAND_MASK, XE_SYSCTRL_CMD_GET_PENDING_EVENT);
-
-	request->vector = xe_device_has_msix(xe) ? XE_IRQ_DEFAULT_MSIX : 0;
-	request->fn = PCI_FUNC(pdev->devfn);
-}
-
 /**
  * xe_sysctrl_event() - Handler for System Controller events
  * @sc: System Controller instance
@@ -72,16 +60,16 @@ void xe_sysctrl_event(struct xe_sysctrl *sc)
 	struct xe_sysctrl_mailbox_command command = {};
 	struct xe_sysctrl_event_response response = {};
 	struct xe_sysctrl_event_request request = {};
-	struct xe_sysctrl_app_msg_hdr header = {};
+	struct xe_device *xe = sc_to_xe(sc);
+	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+
+	xe_device_assert_mem_access(xe);
 
-	xe_device_assert_mem_access(sc_to_xe(sc));
-	event_request_prepare(sc_to_xe(sc), &header, &request);
+	request.vector = xe_device_has_msix(xe) ? XE_IRQ_DEFAULT_MSIX : 0;
+	request.fn = PCI_FUNC(pdev->devfn);
 
-	command.header = header;
-	command.data_in = &request;
-	command.data_in_len = sizeof(request);
-	command.data_out = &response;
-	command.data_out_len = sizeof(response);
+	xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_PENDING_EVENT,
+				  &request, sizeof(request), &response, sizeof(response));
 
 	guard(mutex)(&sc->event_lock);
 	get_pending_event(sc, &command);
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
index 72baf1aa4b3ad6..e21848ca7594b3 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
@@ -334,6 +334,9 @@ void xe_sysctrl_create_command(struct xe_sysctrl_mailbox_command *command, u8 gr
  * If a response is expected, @cmd->data_out must point to a buffer of
  * size @cmd->data_out_len supplied by caller.
  *
+ * @cmd->timeout_ms specifies the response timeout in milliseconds.
+ * A value of 0 uses %XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS.
+ *
  * On success, @rdata_len is updated with number of valid response bytes
  * returned by firmware, bounded by @cmd->data_out_len.
  *
@@ -347,6 +350,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc,
 	u8 group_id, command_code;
 	u8 *mbox_cmd = NULL;
 	size_t cmd_size = 0;
+	unsigned int timeout_ms;
 	int ret;
 
 	guard(xe_pm_runtime_noresume)(xe);
@@ -360,6 +364,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc,
 
 	group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header);
 	command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header);
+	timeout_ms = cmd->timeout_ms ?: XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS;
 
 	might_sleep();
 
@@ -375,7 +380,7 @@ int xe_sysctrl_send_command(struct xe_sysctrl *sc,
 
 	ret = sysctrl_send_command(sc, mbox_cmd, cmd_size,
 				   cmd->data_out, cmd->data_out_len, rdata_len,
-				   XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS);
+				   timeout_ms);
 	if (ret)
 		xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret));
 
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
index d0341538ad0584..e98fa6877fec37 100644
--- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
@@ -14,9 +14,11 @@
  * enum xe_sysctrl_group - System Controller command groups
  *
  * @XE_SYSCTRL_GROUP_GFSP: GFSP group
+ * @XE_SYSCTRL_GROUP_CORE: Core group
  */
 enum xe_sysctrl_group {
 	XE_SYSCTRL_GROUP_GFSP			= 0x01,
+	XE_SYSCTRL_GROUP_CORE			= 0xFF,
 };
 
 /**
@@ -25,6 +27,8 @@ enum xe_sysctrl_group {
  * @XE_SYSCTRL_CMD_GET_SOC_ERROR: Retrieve basic error information
  * @XE_SYSCTRL_CMD_GET_COUNTER: Get error counter value
  * @XE_SYSCTRL_CMD_CLEAR_COUNTER: Clear error counter value
+ * @XE_SYSCTRL_CMD_GET_THRESHOLD: Retrieve error threshold
+ * @XE_SYSCTRL_CMD_SET_THRESHOLD: Set error threshold
  * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event
  * @XE_SYSCTRL_CMD_GET_HEALTH: Retrieve gpu health
  * @XE_SYSCTRL_CMD_SET_HEALTH: Set gpu health
@@ -33,11 +37,56 @@ enum xe_sysctrl_gfsp_cmd {
 	XE_SYSCTRL_CMD_GET_SOC_ERROR		= 0x01,
 	XE_SYSCTRL_CMD_GET_COUNTER		= 0x03,
 	XE_SYSCTRL_CMD_CLEAR_COUNTER		= 0x04,
+	XE_SYSCTRL_CMD_GET_THRESHOLD		= 0x05,
+	XE_SYSCTRL_CMD_SET_THRESHOLD		= 0x06,
 	XE_SYSCTRL_CMD_GET_PENDING_EVENT	= 0x07,
 	XE_SYSCTRL_CMD_GET_HEALTH		= 0x0B,
 	XE_SYSCTRL_CMD_SET_HEALTH		= 0x0C,
 };
 
+/**
+ * enum xe_sysctrl_core_cmd - Commands supported by Core group
+ *
+ * @XE_SYSCTRL_CMD_GET_APP_STATUS_BY_ID: Retrieve application status by ID
+ */
+enum xe_sysctrl_core_cmd {
+	XE_SYSCTRL_CMD_GET_APP_STATUS_BY_ID		= 0x05,
+};
+
+/**
+ * struct xe_sysctrl_app_status_req - Get application status request
+ *
+ * @app_id: Application ID for which to retrieve status
+ */
+struct xe_sysctrl_app_status_req {
+	u8 app_id;
+} __packed;
+
+/**
+ * struct xe_sysctrl_app_status_resp - Get application status response
+ * @flags: Application status flags interpreted by xe_sysctrl_check_app_status()
+ */
+struct xe_sysctrl_app_status_resp {
+	u32 flags;
+} __packed;
+
+/**
+ * enum xe_sysctrl_fw_status - System Controller firmware application lifecycle states
+ *
+ * @XE_SYSCTRL_FIRMWARE_APP_INVALID: app_id is not recognized by firmware
+ * @XE_SYSCTRL_FIRMWARE_APP_NOT_LOADED: application is known but has not yet booted
+ * @XE_SYSCTRL_FIRMWARE_APP_BOOTED: boot sequence completed, post-boot init pending
+ * @XE_SYSCTRL_FIRMWARE_APP_INITIALIZED: application fully operational
+ * @XE_SYSCTRL_FIRMWARE_COMM_FAILURE: communication with System Controller firmware failed
+ */
+enum xe_sysctrl_fw_status {
+	XE_SYSCTRL_FIRMWARE_APP_INVALID,
+	XE_SYSCTRL_FIRMWARE_APP_NOT_LOADED,
+	XE_SYSCTRL_FIRMWARE_APP_BOOTED,
+	XE_SYSCTRL_FIRMWARE_APP_INITIALIZED,
+	XE_SYSCTRL_FIRMWARE_COMM_FAILURE,
+};
+
 /**
  * struct xe_sysctrl_mailbox_command - System Controller mailbox command
  */
@@ -56,6 +105,9 @@ struct xe_sysctrl_mailbox_command {
 
 	/** @data_out_len: Size of output buffer in bytes (0 if no response expected) */
 	size_t data_out_len;
+
+	/** @timeout_ms: Response timeout in ms, or 0 for %XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS */
+	unsigned int timeout_ms;
 };
 
 /* Modify as needed */
diff --git a/drivers/gpu/drm/xe/xe_tile_printk.h b/drivers/gpu/drm/xe/xe_tile_printk.h
index 738433a764bdb3..c87c07e14fd1a3 100644
--- a/drivers/gpu/drm/xe/xe_tile_printk.h
+++ b/drivers/gpu/drm/xe/xe_tile_printk.h
@@ -34,6 +34,9 @@
 #define xe_tile_dbg(_tile, _fmt, ...) \
 	xe_tile_printk((_tile), dbg, _fmt, ##__VA_ARGS__)
 
+#define xe_tile_dbg_ratelimited(_tile, _fmt, ...) \
+	xe_tile_printk((_tile), dbg_ratelimited, _fmt, ##__VA_ARGS__)
+
 #define xe_tile_WARN_type(_tile, _type, _condition, _fmt, ...) \
 	xe_WARN##_type((_tile)->xe, _condition, _fmt, ## __VA_ARGS__)
 
diff --git a/drivers/gpu/drm/xe/xe_tile_sriov_printk.h b/drivers/gpu/drm/xe/xe_tile_sriov_printk.h
index 68323512872cf1..c23e698bc8db2f 100644
--- a/drivers/gpu/drm/xe/xe_tile_sriov_printk.h
+++ b/drivers/gpu/drm/xe/xe_tile_sriov_printk.h
@@ -27,6 +27,9 @@
 #define xe_tile_sriov_dbg(_tile, _fmt, ...) \
 	xe_tile_sriov_printk(_tile, dbg, _fmt, ##__VA_ARGS__)
 
+#define xe_tile_sriov_dbg_ratelimited(_tile, _fmt, ...) \
+	xe_tile_sriov_printk(_tile, dbg_ratelimited, _fmt, ##__VA_ARGS__)
+
 #define xe_tile_sriov_dbg_verbose(_tile, _fmt, ...) \
 	xe_tile_sriov_printk(_tile, dbg_verbose, _fmt, ##__VA_ARGS__)
 
diff --git a/drivers/gpu/drm/xe/xe_tile_types.h b/drivers/gpu/drm/xe/xe_tile_types.h
index 0048100ccb723d..e1368c04846aee 100644
--- a/drivers/gpu/drm/xe/xe_tile_types.h
+++ b/drivers/gpu/drm/xe/xe_tile_types.h
@@ -97,6 +97,10 @@ struct xe_tile {
 		 * Only main GT has page reclaim list allocations.
 		 */
 		struct xe_sa_manager *reclaim_pool;
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
+		/** @mem.memtest_bo: VRAM overlap check BO */
+		struct xe_bo *memtest_bo;
+#endif
 	} mem;
 
 	/** @sriov: tile level virtualization data */
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c
index bbd21d393062fd..c9cc5842d65b65 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.c
@@ -5,6 +5,7 @@
 
 #include <drm/drm_managed.h>
 
+#include "xe_devcoredump.h"
 #include "xe_device_types.h"
 #include "xe_force_wake.h"
 #include "xe_gt_stats.h"
@@ -13,6 +14,7 @@
 #include "xe_guc_tlb_inval.h"
 #include "xe_mmio.h"
 #include "xe_pm.h"
+#include "xe_printk.h"
 #include "xe_tlb_inval.h"
 #include "xe_trace.h"
 
@@ -29,6 +31,12 @@
 
 #define FENCE_STACK_BIT		DMA_FENCE_FLAG_USER_BITS
 
+/* The frontend is only ever embedded in a GT */
+static struct xe_gt *tlb_inval_to_gt(struct xe_tlb_inval *tlb_inval)
+{
+	return container_of(tlb_inval, struct xe_gt, tlb_inval);
+}
+
 static void xe_tlb_inval_fence_fini(struct xe_tlb_inval_fence *fence)
 {
 	if (WARN_ON_ONCE(!fence->tlb_inval))
@@ -73,6 +81,7 @@ static void xe_tlb_inval_fence_timeout(struct work_struct *work)
 	struct xe_device *xe = tlb_inval->xe;
 	struct xe_tlb_inval_fence *fence, *next;
 	long timeout_delay = tlb_inval->ops->timeout_delay(tlb_inval);
+	int timedout_seqno = 0, seqno_recv = 0;
 
 	tlb_inval->ops->flush(tlb_inval);
 
@@ -90,13 +99,45 @@ static void xe_tlb_inval_fence_timeout(struct work_struct *work)
 			"TLB invalidation fence timeout, seqno=%d recv=%d",
 			fence->seqno, tlb_inval->seqno_recv);
 
+		if (!timedout_seqno) {
+			/*
+			 * Hold a PM reference across the capture below. Every
+			 * pending fence holds one, so the device is awake
+			 * here, but signalling them may drop the last
+			 * reference and let it autosuspend before the
+			 * snapshot touches the hardware.
+			 */
+			xe_pm_runtime_get_noresume(xe);
+		}
+
+		timedout_seqno = fence->seqno;
+		if (!tlb_inval->timedout_seqno) {
+			tlb_inval->timedout_seqno = fence->seqno;
+			tlb_inval->timedout_inval_time = fence->inval_time;
+			tlb_inval->timedout_time = ktime_get();
+		}
+
 		fence->base.error = -ETIME;
 		xe_tlb_inval_fence_signal(fence);
 	}
 	if (!list_empty(&tlb_inval->pending_fences))
 		queue_delayed_work(tlb_inval->timeout_wq, &tlb_inval->fence_tdr,
 				   timeout_delay);
+	seqno_recv = tlb_inval->seqno_recv;
 	spin_unlock_irq(&tlb_inval->pending_lock);
+
+	/*
+	 * Capture the GuC log and CT state so the firmware side of the hang
+	 * can be inspected; there is no queue or job to blame here. Must be
+	 * outside pending_lock as the capture takes sleeping locks, hence
+	 * @seqno_recv is sampled above while the lock is still held.
+	 */
+	if (timedout_seqno) {
+		xe_devcoredump_gt(tlb_inval_to_gt(tlb_inval),
+				  "TLB invalidation fence timeout, seqno=%d recv=%d",
+				  timedout_seqno, seqno_recv);
+		xe_pm_runtime_put(xe);
+	}
 }
 
 /**
@@ -207,6 +248,7 @@ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval)
 	else
 		pending_seqno = tlb_inval->seqno - 1;
 	WRITE_ONCE(tlb_inval->seqno_recv, pending_seqno);
+	tlb_inval->timedout_seqno = 0;
 
 	list_for_each_entry_safe(fence, next,
 				 &tlb_inval->pending_fences, link)
@@ -280,7 +322,8 @@ static void xe_tlb_inval_fence_prep(struct xe_tlb_inval_fence *fence)
 		xe_tlb_inval_fence_signal_unlocked((__fence));	\
 	mutex_unlock(&(__tlb_inval)->seqno_lock);		\
 								\
-	__ret == -ECANCELED ? 0 : __ret;			\
+	/* Undelivered: fence already signalled, report done */	\
+	(__ret == -ECANCELED || __ret == -ENOTRECOVERABLE) ? 0 : __ret;	\
 })
 
 /**
@@ -301,26 +344,56 @@ int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
 }
 
 /**
- * xe_tlb_inval_ggtt() - Issue a TLB invalidation for the GGTT
+ * xe_tlb_inval_issue_op_wait() - Issue a TLB invalidation and wait
  * @tlb_inval: TLB invalidation client
- *
- * Issue a TLB invalidation for the GGTT. Completion of TLB is asynchronous and
- * caller can use the invalidation fence to wait for completion.
+ * @op: backend invalidation hook to issue
  *
  * Return: 0 on success, negative error code on error
  */
-int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
+static int xe_tlb_inval_issue_op_wait(struct xe_tlb_inval *tlb_inval,
+				      int (*op)(struct xe_tlb_inval *tlb_inval, u32 seqno))
 {
 	struct xe_tlb_inval_fence fence, *fence_ptr = &fence;
 	int ret;
 
 	xe_tlb_inval_fence_init(tlb_inval, fence_ptr, true);
-	ret = xe_tlb_inval_issue(tlb_inval, fence_ptr, tlb_inval->ops->ggtt);
+	ret = xe_tlb_inval_issue(tlb_inval, fence_ptr, op);
 	xe_tlb_inval_fence_wait(fence_ptr);
 
 	return ret;
 }
 
+/**
+ * xe_tlb_inval_ggtt() - Issue a TLB invalidation for the GGTT
+ * @tlb_inval: TLB invalidation client
+ *
+ * Issue a TLB invalidation for the GGTT. Completion of TLB is asynchronous and
+ * caller can use the invalidation fence to wait for completion.
+ *
+ * Return: 0 on success, negative error code on error
+ */
+int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval)
+{
+	return xe_tlb_inval_issue_op_wait(tlb_inval, tlb_inval->ops->ggtt);
+}
+
+/**
+ * xe_tlb_inval_ggtt_full() - Full engine TLB invalidation within a VF
+ * @tlb_inval: TLB invalidation client
+ *
+ * Issue INVAL_FULL (intra vf) to flush engine TLBs across all engines
+ * within the requesting VF.
+ *
+ * Return: 0 on success, negative error code on error
+ */
+int xe_tlb_inval_ggtt_full(struct xe_tlb_inval *tlb_inval)
+{
+	if (!tlb_inval->ops->ggtt_full)
+		return -EOPNOTSUPP;
+
+	return xe_tlb_inval_issue_op_wait(tlb_inval, tlb_inval->ops->ggtt_full);
+}
+
 /**
  * xe_tlb_inval_range() - Issue a TLB invalidation for an address range
  * @tlb_inval: TLB invalidation client
@@ -404,6 +477,18 @@ void xe_tlb_inval_done_handler(struct xe_tlb_inval *tlb_inval, int seqno)
 
 	WRITE_ONCE(tlb_inval->seqno_recv, seqno);
 
+	if (tlb_inval->timedout_seqno &&
+	    xe_tlb_inval_seqno_past(tlb_inval, tlb_inval->timedout_seqno)) {
+		ktime_t now = ktime_get();
+
+		xe_warn(xe,
+			"TLB invalidation late ack: seqno=%d recv=%d, request-to-ack=%lldms, timeout-to-ack=%lldms",
+			tlb_inval->timedout_seqno, seqno,
+			ktime_ms_delta(now, tlb_inval->timedout_inval_time),
+			ktime_ms_delta(now, tlb_inval->timedout_time));
+		tlb_inval->timedout_seqno = 0;
+	}
+
 	list_for_each_entry_safe(fence, next,
 				 &tlb_inval->pending_fences, link) {
 		trace_xe_tlb_inval_fence_recv(xe, fence);
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.h b/drivers/gpu/drm/xe/xe_tlb_inval.h
index a76b7823a5f2e2..665072b4b81dca 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.h
@@ -20,6 +20,7 @@ void xe_tlb_inval_reset(struct xe_tlb_inval *tlb_inval);
 int xe_tlb_inval_all(struct xe_tlb_inval *tlb_inval,
 		     struct xe_tlb_inval_fence *fence);
 int xe_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval);
+int xe_tlb_inval_ggtt_full(struct xe_tlb_inval *tlb_inval);
 void xe_tlb_inval_vm(struct xe_tlb_inval *tlb_inval, struct xe_vm *vm);
 int xe_tlb_inval_range(struct xe_tlb_inval *tlb_inval,
 		       struct xe_tlb_inval_fence *fence,
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.c b/drivers/gpu/drm/xe/xe_tlb_inval_job.c
index 04d21015cd5dcf..7378cfe6e85526 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_job.c
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.c
@@ -4,6 +4,7 @@
  */
 
 #include "xe_assert.h"
+#include "xe_cpu_bind.h"
 #include "xe_dep_job_types.h"
 #include "xe_dep_scheduler.h"
 #include "xe_exec_queue.h"
@@ -12,7 +13,6 @@
 #include "xe_page_reclaim.h"
 #include "xe_tlb_inval.h"
 #include "xe_tlb_inval_job.h"
-#include "xe_migrate.h"
 #include "xe_pm.h"
 #include "xe_vm.h"
 
@@ -39,8 +39,8 @@ struct xe_tlb_inval_job {
 	u64 start;
 	/** @end: End address to invalidate */
 	u64 end;
-	/** @type: GT type */
-	int type;
+	/** @idx: Index of tlb invalidation */
+	int idx;
 	/** @fence_armed: Fence has been armed */
 	bool fence_armed;
 };
@@ -87,7 +87,7 @@ static const struct xe_dep_job_ops dep_job_ops = {
  * @vm: VM which TLB invalidation is being issued for
  * @start: Start address to invalidate
  * @end: End address to invalidate
- * @type: GT type
+ * @idx: Index of tlb invalidation
  *
  * Create a TLB invalidation job and initialize internal fields. The caller is
  * responsible for releasing the creation reference.
@@ -97,7 +97,7 @@ static const struct xe_dep_job_ops dep_job_ops = {
 struct xe_tlb_inval_job *
 xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 			struct xe_dep_scheduler *dep_scheduler,
-			struct xe_vm *vm, u64 start, u64 end, int type)
+			struct xe_vm *vm, u64 start, u64 end, int idx)
 {
 	struct xe_tlb_inval_job *job;
 	struct drm_sched_entity *entity =
@@ -105,8 +105,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 	struct xe_tlb_inval_fence *ifence;
 	int err;
 
-	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
-		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
+	xe_assert(vm->xe, idx < XE_EXEC_QUEUE_TLB_INVAL_COUNT);
 
 	job = kmalloc_obj(*job);
 	if (!job)
@@ -120,7 +119,7 @@ xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 	job->fence_armed = false;
 	xe_page_reclaim_list_init(&job->prl);
 	job->dep.ops = &dep_job_ops;
-	job->type = type;
+	job->idx = idx;
 	kref_init(&job->refcount);
 	xe_exec_queue_get(q);	/* Pairs with put in xe_tlb_inval_job_destroy */
 	xe_vm_get(vm);		/* Pairs with put in xe_tlb_inval_job_destroy */
@@ -219,7 +218,6 @@ int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job)
 /**
  * xe_tlb_inval_job_push() - TLB invalidation job push
  * @job: TLB invalidation job to push
- * @m: The migration object being used
  * @fence: Dependency for TLB invalidation job
  *
  * Pushes a TLB invalidation job for execution, using @fence as a dependency.
@@ -231,11 +229,11 @@ int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job)
  * Return: Job's finished fence on success, cannot fail
  */
 struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
-					struct xe_migrate *m,
 					struct dma_fence *fence)
 {
 	struct xe_tlb_inval_fence *ifence =
 		container_of(job->fence, typeof(*ifence), base);
+	struct xe_cpu_bind *cpu_bind = gt_to_xe(job->q->gt)->cpu_bind;
 
 	if (!dma_fence_is_signaled(fence)) {
 		void *ptr;
@@ -259,11 +257,11 @@ struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
 	job->fence_armed = true;
 
 	/*
-	 * We need the migration lock to protect the job's seqno and the spsc
-	 * queue, only taken on migration queue, user queues protected dma-resv
+	 * We need the cpu_bind lock to protect the job's seqno and the spsc
+	 * queue, only taken on cpu_bind queue, user queues protected dma-resv
 	 * VM lock.
 	 */
-	xe_migrate_job_lock(m, job->q);
+	xe_cpu_bind_job_lock(cpu_bind, job->q);
 
 	/* Creation ref pairs with put in xe_tlb_inval_job_destroy */
 	xe_tlb_inval_fence_init(job->tlb_inval, ifence, false);
@@ -280,9 +278,9 @@ struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
 	/* Let the upper layers fish this out */
 	xe_exec_queue_tlb_inval_last_fence_set(job->q, job->vm,
 					       &job->dep.drm.s_fence->finished,
-					       job->type);
+					       job->idx);
 
-	xe_migrate_job_unlock(m, job->q);
+	xe_cpu_bind_job_unlock(cpu_bind, job->q);
 
 	/*
 	 * Not using job->fence, as it has its own dma-fence context, which does
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_job.h b/drivers/gpu/drm/xe/xe_tlb_inval_job.h
index 03d6e21cd6113f..97e032ea21c378 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_job.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_job.h
@@ -11,7 +11,6 @@
 struct dma_fence;
 struct xe_dep_scheduler;
 struct xe_exec_queue;
-struct xe_migrate;
 struct xe_page_reclaim_list;
 struct xe_tlb_inval;
 struct xe_tlb_inval_job;
@@ -20,7 +19,7 @@ struct xe_vm;
 struct xe_tlb_inval_job *
 xe_tlb_inval_job_create(struct xe_exec_queue *q, struct xe_tlb_inval *tlb_inval,
 			struct xe_dep_scheduler *dep_scheduler,
-			struct xe_vm *vm, u64 start, u64 end, int type);
+			struct xe_vm *vm, u64 start, u64 end, int idx);
 
 void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
 				       struct xe_page_reclaim_list *prl);
@@ -28,7 +27,6 @@ void xe_tlb_inval_job_add_page_reclaim(struct xe_tlb_inval_job *job,
 int xe_tlb_inval_job_alloc_dep(struct xe_tlb_inval_job *job);
 
 struct dma_fence *xe_tlb_inval_job_push(struct xe_tlb_inval_job *job,
-					struct xe_migrate *m,
 					struct dma_fence *fence);
 
 void xe_tlb_inval_job_get(struct xe_tlb_inval_job *job);
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval_types.h b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
index 3d1797d186fd79..80d2019fa20b48 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval_types.h
+++ b/drivers/gpu/drm/xe/xe_tlb_inval_types.h
@@ -36,6 +36,16 @@ struct xe_tlb_inval_ops {
 	 */
 	int (*ggtt)(struct xe_tlb_inval *tlb_inval, u32 seqno);
 
+	/**
+	 * @ggtt_full: Full engine TLB invalidation within a VF
+	 * @tlb_inval: TLB invalidation client
+	 * @seqno: Seqno of TLB invalidation
+	 *
+	 * Return 0 on success, -ECANCELED if backend is mid-reset, error on
+	 * failure
+	 */
+	int (*ggtt_full)(struct xe_tlb_inval *tlb_inval, u32 seqno);
+
 	/**
 	 * @ppgtt: Invalidate per-process translation TLBs
 	 * @tlb_inval: TLB invalidation client
@@ -102,6 +112,23 @@ struct xe_tlb_inval {
 	 * @pending_lock: protects @pending_fences and updating @seqno_recv.
 	 */
 	spinlock_t pending_lock;
+	/**
+	 * @timedout_seqno: seqno of the most recent timed out TLB
+	 * invalidation, 0 if none. Used to measure how late the ack for a
+	 * timed out invalidation actually arrives. Protected by
+	 * @pending_lock.
+	 */
+	int timedout_seqno;
+	/**
+	 * @timedout_inval_time: request time of @timedout_seqno. Protected by
+	 * @pending_lock.
+	 */
+	ktime_t timedout_inval_time;
+	/**
+	 * @timedout_time: time @timedout_seqno was signaled with -ETIME.
+	 * Protected by @pending_lock.
+	 */
+	ktime_t timedout_time;
 	/**
 	 * @fence_tdr: schedules a delayed call to xe_tlb_fence_timeout after
 	 * the timeout interval is over.
diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h
index 2fe8f89a1e34f0..d4e9d91f6f7f02 100644
--- a/drivers/gpu/drm/xe/xe_trace.h
+++ b/drivers/gpu/drm/xe/xe_trace.h
@@ -261,7 +261,7 @@ DECLARE_EVENT_CLASS(xe_sched_job,
 			   __entry->flags = job->q->flags;
 			   __entry->error = job->fence ? job->fence->error : 0;
 			   __entry->fence = job->fence;
-			   __entry->batch_addr = (u64)job->ptrs[0].batch_addr;
+			   __entry->batch_addr = job->is_pt_job ? 0 : (u64)job->ptrs[0].batch_addr;
 			   ),
 
 		    TP_printk("dev=%s, fence=%p, seqno=%u, lrc_seqno=%u, gt=%u, guc_id=%d, batch_addr=0x%012llx, guc_state=0x%x, flags=0x%x, error=%d",
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 05911904c1f975..9a514d983e90be 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -5,18 +5,27 @@
  */
 
 #include <linux/cgroup_dmem.h>
+#include <linux/debugfs.h>
 
 #include <drm/drm_managed.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_buddy.h>
+#include <uapi/drm/xe_drm.h>
 
 #include <drm/ttm/ttm_placement.h>
 #include <drm/ttm/ttm_range_manager.h>
 
+#include "regs/xe_regs.h"
 #include "xe_bo.h"
+#include "xe_configfs.h"
 #include "xe_device.h"
+#include "xe_exec_queue.h"
+#include "xe_lrc.h"
+#include "xe_mmio.h"
 #include "xe_pm.h"
+#include "xe_printk.h"
 #include "xe_res_cursor.h"
+#include "xe_ttm_stolen_mgr.h"
 #include "xe_ttm_vram_mgr.h"
 #include "xe_vram_types.h"
 
@@ -49,6 +58,48 @@ static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm,
 	return true;
 }
 
+static int xe_ttm_vram_buddy_alloc(struct xe_ttm_vram_mgr *mgr, u64 start,
+				   u64 end, u64 size, u64 min_page_size,
+				   struct list_head *blocks, unsigned long flags,
+				   struct ttm_resource *res, u64 *used_visible)
+{
+	struct gpu_buddy *mm = &mgr->mm;
+	struct gpu_buddy_block *block;
+	int err;
+
+	err = gpu_buddy_alloc_blocks(mm, start, end, size, min_page_size, blocks, flags);
+	if (err)
+		return err;
+
+	/*
+	 * Track the owning resource, never the owning BO. A BO backpointer
+	 * cached here goes stale the moment TTM hands the resource to a ghost
+	 * object (ttm_buffer_object_transfer()), which happens on every
+	 * accelerated move and on pipelined gutting. The resource, in
+	 * contrast, has exactly the same lifetime as these blocks and TTM
+	 * keeps &ttm_resource.bo pointing at the current owner for us.
+	 */
+	list_for_each_entry(block, blocks, link)
+		block->private = res;
+
+	if (end <= mgr->visible_size) {
+		*used_visible = size;
+	} else {
+		list_for_each_entry(block, blocks, link) {
+			u64 blk_start = gpu_buddy_block_offset(block);
+
+			if (blk_start < mgr->visible_size) {
+				u64 blk_end = blk_start + gpu_buddy_block_size(mm, block);
+
+				*used_visible += min(blk_end, mgr->visible_size) - blk_start;
+			}
+		}
+	}
+
+	mgr->visible_avail -= *used_visible;
+	return 0;
+}
+
 static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 			       struct ttm_buffer_object *tbo,
 			       const struct ttm_place *place,
@@ -117,30 +168,12 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 		goto error_unlock;
 	}
 
-	err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
-				     (u64)lpfn << PAGE_SHIFT, size,
-				     min_page_size, &vres->blocks, vres->flags);
+	err = xe_ttm_vram_buddy_alloc(mgr, (u64)place->fpfn << PAGE_SHIFT,
+				      (u64)lpfn << PAGE_SHIFT, size,
+				      min_page_size, &vres->blocks, vres->flags,
+				      &vres->base, &vres->used_visible_size);
 	if (err)
 		goto error_unlock;
-
-	if (lpfn <= mgr->visible_size >> PAGE_SHIFT) {
-		vres->used_visible_size = size;
-	} else {
-		struct gpu_buddy_block *block;
-
-		list_for_each_entry(block, &vres->blocks, link) {
-			u64 start = gpu_buddy_block_offset(block);
-
-			if (start < mgr->visible_size) {
-				u64 end = start + gpu_buddy_block_size(mm, block);
-
-				vres->used_visible_size +=
-					min(end, mgr->visible_size) - start;
-			}
-		}
-	}
-
-	mgr->visible_avail -= vres->used_visible_size;
 	mutex_unlock(&mgr->lock);
 
 	if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) &&
@@ -172,17 +205,61 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man,
 	return err;
 }
 
+static void xe_ttm_vram_buddy_free(struct xe_ttm_vram_mgr *mgr,
+				   struct list_head *blocks,
+				   u64 used_visible)
+{
+	struct gpu_buddy_block *block;
+
+	list_for_each_entry(block, blocks, link)
+		block->private = NULL;
+	gpu_buddy_free_list(&mgr->mm, blocks, 0);
+	mgr->visible_avail += used_visible;
+}
+
+/*
+ * Retry pending page-offline reservations.
+ *
+ * A reservation can fail because the blocks backing the bad page are still
+ * allocated: either the owning BO could not be purged, or the purge was
+ * pipelined and TTM handed the resource to a ghost object which frees it
+ * only once the move fences signal. Rather than giving up, entries stay on
+ * @queued_pages and are retried here every time VRAM blocks come back.
+ *
+ * Called with @mgr->lock held.
+ */
+static void xe_ttm_vram_retry_queued_pages(struct xe_ttm_vram_mgr *mgr)
+{
+	struct xe_ttm_vram_offline_resource *pos, *n;
+
+	lockdep_assert_held(&mgr->lock);
+
+	list_for_each_entry_safe(pos, n, &mgr->queued_pages, queued_link) {
+		if (xe_ttm_vram_buddy_alloc(mgr, pos->addr, pos->addr + PAGE_SIZE,
+					    PAGE_SIZE, PAGE_SIZE, &pos->blocks,
+					    GPU_BUDDY_RANGE_ALLOCATION, NULL,
+					    &pos->used_visible_size)) {
+			pos->status = XE_PAGE_RESERVE_FAIL;
+			continue;
+		}
+		--mgr->n_queued_pages;
+		list_del_rcu(&pos->queued_link);
+		++mgr->n_offlined_pages;
+		list_add_rcu(&pos->offlined_link, &mgr->offlined_pages);
+	}
+}
+
 static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man,
 				struct ttm_resource *res)
 {
 	struct xe_ttm_vram_mgr_resource *vres =
 		to_xe_ttm_vram_mgr_resource(res);
 	struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man);
-	struct gpu_buddy *mm = &mgr->mm;
 
 	mutex_lock(&mgr->lock);
-	gpu_buddy_free_list(mm, &vres->blocks, 0);
-	mgr->visible_avail += vres->used_visible_size;
+	xe_ttm_vram_buddy_free(mgr, &vres->blocks, vres->used_visible_size);
+	if (unlikely(!list_empty(&mgr->queued_pages)))
+		xe_ttm_vram_retry_queued_pages(mgr);
 	mutex_unlock(&mgr->lock);
 
 	ttm_resource_fini(man, res);
@@ -312,12 +389,35 @@ static void xe_ttm_vram_mgr_set_unused(struct drm_device *dev, void *arg)
 	ttm_resource_manager_set_used(man, false);
 }
 
+static void xe_ttm_vram_free_bad_pages(struct xe_ttm_vram_mgr *mgr)
+{
+	struct xe_ttm_vram_offline_resource *pos, *n;
+
+	list_for_each_entry_safe(pos, n, &mgr->offlined_pages, offlined_link) {
+		list_del_rcu(&pos->offlined_link);
+		xe_ttm_vram_buddy_free(mgr, &pos->blocks, pos->used_visible_size);
+		--mgr->n_offlined_pages;
+		kfree_rcu(pos, rcu);
+	}
+	list_for_each_entry_safe(pos, n, &mgr->queued_pages, queued_link) {
+		list_del_rcu(&pos->queued_link);
+		/* queued entries have no buddy reservation yet */
+		xe_ttm_vram_buddy_free(mgr, &pos->blocks, 0);
+		--mgr->n_queued_pages;
+		kfree_rcu(pos, rcu);
+	}
+}
+
 static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
 {
 	struct xe_device *xe = to_xe_device(dev);
 	struct xe_ttm_vram_mgr *mgr = arg;
 	struct ttm_resource_manager *man = &mgr->manager;
 
+	mutex_lock(&mgr->lock);
+	xe_ttm_vram_free_bad_pages(mgr);
+	mutex_unlock(&mgr->lock);
+
 	if (ttm_resource_manager_evict_all(&xe->ttm, man))
 		return;
 
@@ -344,6 +444,8 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
 	err = drmm_mutex_init(&xe->drm, &mgr->lock);
 	if (err)
 		return err;
+	INIT_LIST_HEAD(&mgr->offlined_pages);
+	INIT_LIST_HEAD(&mgr->queued_pages);
 	mgr->default_page_size = default_page_size;
 	mgr->visible_size = io_size;
 	mgr->visible_avail = io_size;
@@ -521,3 +623,451 @@ u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man)
 
 	return avail;
 }
+
+static int xe_ttm_vram_purge_page(struct xe_device *xe, struct xe_bo *bo)
+{
+	u32 q_flag = DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE;
+	struct ttm_operation_ctx ctx = {};
+	struct xe_exec_queue *q_to_put = NULL;
+	struct xe_exec_queue *q = NULL;
+	struct xe_vm *vm = NULL;
+	u32	flags;
+	int ret = 0;
+
+	xe_bo_lock(bo, false);
+	if (bo->vm)
+		vm = xe_vm_get(bo->vm);
+	flags = bo->flags;
+	xe_bo_unlock(bo);
+	/*  Ban VM if BO is PPGTT */
+	if (vm && (flags & XE_BO_FLAG_PAGETABLE)) {
+		struct xe_exec_queue *eq;
+		int id;
+
+		down_write(&vm->lock);
+		if (xe->info.has_ctx_tlb_inval) {
+			/*
+			 * Must be the write lock: send_tlb_inval_ctx_ppgtt()
+			 * mutates this list (list_move_tail() onto an on-stack
+			 * head) while holding only the read lock, relying on
+			 * tlb_inval->seqno_lock to keep itself the sole
+			 * mutator. Traversing it under down_read() would let
+			 * this walk follow entries onto that stack list.
+			 */
+			down_write(&vm->exec_queues.lock);
+			for (id = 0; id < ARRAY_SIZE(vm->exec_queues.list); id++)
+				list_for_each_entry(eq, &vm->exec_queues.list[id],
+						    vm_exec_queue_link)
+					atomic_or(q_flag, &eq->ban_reason);
+			up_write(&vm->exec_queues.lock);
+		} else {
+			list_for_each_entry(eq, &vm->preempt.exec_queues, lr.link)
+				atomic_or(q_flag, &eq->ban_reason);
+		}
+		smp_wmb(); /* Force all queue bits to be visible before killing the VM */
+		xe_vm_kill(vm, true);
+		up_write(&vm->lock);
+	}
+	if (vm)
+		xe_vm_put(vm);
+
+	xe_bo_lock(bo, false);
+	q = READ_ONCE(bo->q);
+	/*  Ban exec queue if BO is lrc */
+	if (q && xe_exec_queue_get_unless_zero(q)) {
+		/* ban queue */
+		atomic_or(q_flag, &q->ban_reason);
+		smp_wmb(); /* Force bit change to finish before state change triggers */
+		q_to_put = q;
+	}
+
+	if (bo->purgeable.state == XE_MADV_PURGEABLE_PURGED) {
+		/* Already purged by shrinker during unlocked window — nothing to do */
+		xe_bo_unlock(bo);
+		goto out;
+	}
+
+	xe_bo_set_purgeable_state(bo, XE_MADV_PURGEABLE_DONTNEED);
+	ttm_bo_unmap_virtual(&bo->ttm);   /* nuke CPU mmap + VRAM IO mappings */
+	if (xe_bo_is_pinned(bo))
+		xe_bo_unpin(bo);
+	ret = xe_ttm_bo_purge(&bo->ttm, &ctx);
+	xe_bo_unlock(bo);
+
+out:
+	if (q_to_put) {
+		xe_exec_queue_kill(q_to_put);
+		xe_exec_queue_put(q_to_put);
+	}
+
+	return ret;
+}
+
+static bool xe_ttm_vram_page_already_processed(struct xe_ttm_vram_mgr *mgr,
+					       u64 addr)
+{
+	struct xe_ttm_vram_offline_resource *pos;
+
+	lockdep_assert_held(&mgr->lock);
+
+	list_for_each_entry(pos, &mgr->offlined_pages, offlined_link) {
+		if (pos->addr == addr)
+			return true;
+	}
+
+	list_for_each_entry(pos, &mgr->queued_pages, queued_link) {
+		if (pos->addr == addr)
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * Resolve the BO currently owning @block and take a reference on it.
+ *
+ * Called with @mgr->lock held, which serializes against
+ * xe_ttm_vram_buddy_free() clearing block->private.
+ *
+ * Returns NULL when there is no xe_bo we can act on: either the block is
+ * free, or the resource is temporarily owned by a TTM ghost object because
+ * a move or a pipelined gutting is still in flight. In both cases the
+ * blocks will hit xe_ttm_vram_mgr_del() on their own and the pending
+ * reservation is retried from there.
+ */
+static struct xe_bo *xe_ttm_vram_block_owner_get(struct xe_device *xe,
+						 struct gpu_buddy_block *block)
+{
+	struct ttm_resource *res = block->private;
+	struct ttm_buffer_object *tbo;
+	struct xe_bo *bo;
+
+	if (!res)
+		return NULL;
+
+	guard(spinlock)(&xe->ttm.lru_lock);
+
+	/*
+	 * res->bo is updated under bdev->lru_lock by ttm_resource_set_bo().
+	 * Racing with a ghost transfer here is benign: we either see the old
+	 * owner (whose purge is a no-op and the retry path recovers) or the
+	 * ghost (rejected below).
+	 *
+	 * A ghost is a bare ttm_transfer_obj, not an xe_bo, so ttm_to_xe_bo()
+	 * on one would be out of bounds. xe_bo_is_xe_bo() rejects it since
+	 * only our own BOs carry xe_ttm_bo_destroy().
+	 */
+	tbo = READ_ONCE(res->bo);
+	if (!tbo || !xe_bo_is_xe_bo(tbo))
+		return NULL;
+
+	bo = ttm_to_xe_bo(tbo);
+
+	/* The BO may already be in teardown with a zero refcount */
+	return xe_bo_get_unless_zero(bo) ? bo : NULL;
+}
+
+static int xe_ttm_vram_reserve_page_at_addr(struct xe_device *xe, u64 addr,
+					    struct xe_ttm_vram_mgr *vram_mgr, struct gpu_buddy *mm)
+{
+	struct xe_ttm_vram_offline_resource *nentry;
+	struct xe_bo *pbo_to_put = NULL;
+	struct xe_bo *pbo = NULL;
+	struct gpu_buddy_block *block;
+	u64 size = PAGE_SIZE;
+	int ret = 0;
+
+	scoped_guard(mutex, &vram_mgr->lock) {
+		if (xe_ttm_vram_page_already_processed(vram_mgr, addr))
+			return -EEXIST;
+		block = gpu_buddy_allocated_addr_to_block(mm, addr);
+		if (WARN_ON(IS_ERR(block)))
+			return PTR_ERR(block);
+
+		nentry = kzalloc_obj(*nentry);
+		if (!nentry)
+			return -ENOMEM;
+		INIT_LIST_HEAD(&nentry->blocks);
+		nentry->status = XE_PAGE_RESERVE_PENDING;
+		nentry->addr = addr;
+
+		if (block) {
+			pbo = xe_ttm_vram_block_owner_get(xe, block);
+
+			/*
+			 * Critical kernel BO? Best-effort check without resv lock;
+			 * worst case a concurrent pin causes reset path unnecessarily.
+			 */
+			if (pbo && ((pbo->ttm.type == ttm_bo_type_kernel &&
+				     !(pbo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE)) ||
+				    (xe_bo_is_user(pbo) && xe_bo_is_pinned(pbo)))) {
+				kfree(nentry);
+				pbo_to_put = pbo;
+				drm_err(&xe->drm,
+					"%s: addr: 0x%llx is critical kernel bo, requesting SBR\n",
+					__func__, addr);
+				break;
+			}
+			/* Queue free(to-be-purged) pages */
+			++vram_mgr->n_queued_pages;
+			list_add_rcu(&nentry->queued_link, &vram_mgr->queued_pages);
+		} else {
+			/* Immediately offline unoccupied pages */
+			/* Queue free(to-be-reserved) pages */
+			ret = xe_ttm_vram_buddy_alloc(vram_mgr, addr, addr + size,
+						      size, size, &nentry->blocks,
+						      GPU_BUDDY_RANGE_ALLOCATION,
+						      NULL, &nentry->used_visible_size);
+			if (ret) {
+				nentry->status = XE_PAGE_RESERVE_FAIL;
+				drm_dbg(&xe->drm,
+					"Page at addr:0x%llx still busy (%d), deferring reservation\n",
+					addr, ret);
+				++vram_mgr->n_queued_pages;
+				list_add_rcu(&nentry->queued_link, &vram_mgr->queued_pages);
+				return 0;
+			}
+			++vram_mgr->n_offlined_pages;
+			list_add_rcu(&nentry->offlined_link, &vram_mgr->offlined_pages);
+			return ret;
+		}
+	}
+
+	/* Deferred put outside lock to avoid recursive deadlock */
+	if (pbo_to_put) {
+		xe_bo_put(pbo_to_put);
+		/* Hint System controller driver for reset with -EIO  */
+		return -EIO;
+	}
+
+	if (pbo) {
+		/*
+		 * Purge BO containing address - reference held from above.
+		 * This does not necessarily free the blocks synchronously: if
+		 * the BO is not idle, ttm_bo_pipeline_gutting() hands the
+		 * resource to a ghost object and it is released only once the
+		 * move fences signal. The reservation below then fails and is
+		 * retried from xe_ttm_vram_mgr_del().
+		 */
+		ret = xe_ttm_vram_purge_page(xe, pbo);
+		xe_bo_put(pbo);
+		if (ret)
+			drm_warn(&xe->drm, "Purge failed at addr:0x%llx, ret:%d\n", addr, ret);
+	}
+
+	return 0;
+}
+
+static struct xe_vram_region *xe_ttm_vram_addr_to_region(struct xe_device *xe, u64 addr)
+{
+	struct xe_tile *tile;
+	u8 id;
+
+	for_each_tile(tile, xe, id) {
+		struct xe_vram_region *vr = tile->mem.vram;
+
+		if (!vr)
+			continue;
+
+		if (addr >= vr->dpa_base && addr < (vr->dpa_base + vr->usable_size))
+			return vr;
+
+		/* CCS, GSM, or DSM — infrastructure zone, needs reset */
+		if (addr >= (vr->dpa_base + vr->usable_size) &&
+		    addr < (vr->dpa_base + vr->actual_physical_size))
+			return NULL;
+	}
+
+	/*
+	 * Return an explicit error pointer so the caller knows the addr
+	 * is invalid and should be ignored, NOT SBR.
+	 */
+	return ERR_PTR(-EOPNOTSUPP);
+}
+
+/**
+ * xe_ttm_vram_handle_addr_fault - Handle vram physical address error flaged
+ * @xe: pointer to parent device
+ * @addr: physical faulty address
+ *
+ * Handle the physcial faulty address error on specific tile.
+ *
+ * Returns 0 for success, negative error code otherwise as follow:
+ * * %-EIO - critical BO or address outside any VRAM region; next action is reset.
+ * * %-EOPNOTSUPP - log-only policy or unknown address; no further action.
+ * * %-ENOMEM - allocation failure; next action is reset.
+ * * %-ENXIO - address not found in buddy; no further action.
+ * * %-EEXIST - address already processed; no further action.
+ *
+ * A return of 0 means the page is tracked. It may still be listed as
+ * pending if the blocks backing it could not be freed immediately; the
+ * reservation is then completed from xe_ttm_vram_mgr_del().
+ */
+int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr)
+{
+	struct xe_ttm_vram_mgr *vram_mgr;
+	struct xe_vram_region *vr;
+	struct gpu_buddy *mm;
+
+	/* Assert that the address is PAGE_SIZE aligned */
+	if (WARN_ON_ONCE(!IS_ALIGNED(addr, PAGE_SIZE))) {
+		drm_err(&xe->drm, "Address %llx is not %lu aligned!\n", addr, PAGE_SIZE);
+		return -EINVAL;
+	}
+
+	vr = xe_ttm_vram_addr_to_region(xe, addr);
+	if (IS_ERR(vr)) {
+		/*
+		 * The addr is outside VRAM and GSM.
+		 * Log a debug message if needed, and safely exit/ignore.
+		 */
+		drm_dbg(&xe->drm, "Address %llx is out of bounds, ignoring fault.\n", addr);
+		return PTR_ERR(vr);
+	}
+	if (!vr) {
+		drm_err(&xe->drm, "%s:%d GSM addr:%llx error requesting SBR\n",
+			__func__, __LINE__, addr);
+		/* Hint System controller driver for reset with -EIO  */
+		return -EIO;
+	}
+	vram_mgr = &vr->ttm;
+	mm = &vram_mgr->mm;
+
+	if (xe->ras.disable_vram_page_offline) {
+		xe_err(xe, "0x%llx is reported as corrupted address by HW\n",
+		       addr);
+		return -EOPNOTSUPP;
+	}
+
+	/* Reserve page at address */
+	return xe_ttm_vram_reserve_page_at_addr(xe, addr - vr->dpa_base, vram_mgr, mm);
+}
+EXPORT_SYMBOL(xe_ttm_vram_handle_addr_fault);
+
+/**
+ * xe_ttm_vram_inject_fault - Inject a VRAM page fault for testing
+ * @xe: xe device instance
+ *
+ * Picks the last unallocated VRAM page and reports it as faulted
+ * via xe_ttm_vram_handle_addr_fault(). Used by the fault-inject
+ * debugfs interface for testing page offlining.
+ *
+ * Note: Executing this test will permanently retire the allocated
+ * memory tracking pages. The driver must be rebinded (unbind and bind)
+ * post-test execution to reclaim the reserved space, as these pages
+ * cannot be freed or reclaimed dynamically while the current instance
+ * remains active.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int xe_ttm_vram_inject_fault(struct xe_device *xe)
+{
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_vram_region *vr = tile->mem.vram;
+	struct xe_ttm_vram_mgr *vram_mgr = &vr->ttm;
+	struct gpu_buddy *mm = &vram_mgr->mm;
+	u64 addr;
+
+	if (vr->actual_physical_size < PAGE_SIZE)
+		return -ENOSPC;
+
+	addr = vr->actual_physical_size - PAGE_SIZE;
+	while (addr < vr->actual_physical_size) {
+		struct gpu_buddy_block *block;
+		bool found = false;
+
+		scoped_guard(mutex, &vram_mgr->lock) {
+			block = gpu_buddy_allocated_addr_to_block(mm, addr);
+			if (!block)
+				found = true;
+		}
+
+		/*
+		 * Intentional race window: xe_ttm_vram_handle_addr_fault()
+		 * re-acquires vram_mgr->lock internally, so we cannot hold
+		 * it here. A concurrent allocation claiming this page between
+		 * the two calls is an acceptable false negative for this
+		 * test-only path.
+		 */
+		if (found)
+			return xe_ttm_vram_handle_addr_fault(xe, addr + vr->dpa_base);
+
+		cond_resched();
+		if (addr == 0)
+			break;
+		addr -= PAGE_SIZE;
+	}
+
+	return -ENOSPC;
+}
+EXPORT_SYMBOL(xe_ttm_vram_inject_fault);
+
+static int vram_bad_pages_show(struct seq_file *m, void *unused)
+{
+	struct xe_device *xe = m->private;
+	struct xe_ttm_vram_offline_resource *pos;
+	struct ttm_resource_manager *man;
+	struct xe_ttm_vram_mgr *mgr;
+	struct xe_tile *tile;
+	u8 id;
+
+	man = ttm_manager_type(&xe->ttm, XE_PL_VRAM0);
+	if (man)
+		/* TODO Hook with RAS to show max_pages fetched from FW */
+		seq_printf(m, "max_pages: %d\n",
+			   to_xe_ttm_vram_mgr(man)->max_pages);
+
+	for_each_tile(tile, xe, id) {
+		struct xe_vram_region *vr = tile->mem.vram;
+
+		man = ttm_manager_type(&xe->ttm, XE_PL_VRAM0 + id);
+		if (!man || !vr)
+			continue;
+		mgr = to_xe_ttm_vram_mgr(man);
+
+		rcu_read_lock();
+
+		list_for_each_entry_rcu(pos, &mgr->offlined_pages, offlined_link) {
+			u64 pfn;
+
+			pfn = (pos->addr + vr->dpa_base) >> PAGE_SHIFT;
+			seq_printf(m, "0x%016llx : 0x%016lx : R\n", pfn, PAGE_SIZE);
+		}
+
+		list_for_each_entry_rcu(pos, &mgr->queued_pages, queued_link) {
+			u64 pfn;
+
+			pfn = (pos->addr + vr->dpa_base) >> PAGE_SHIFT;
+			seq_printf(m, "0x%016llx : 0x%016lx : %c\n",
+				   pfn, PAGE_SIZE, pos->status ? 'F' : 'P');
+		}
+
+		rcu_read_unlock();
+	}
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(vram_bad_pages);
+
+/**
+ * xe_ttm_vram_debugfs_init - Initialize VRAM debugfs interfaces
+ * @xe: The xe device structure pointer
+ * @root: The root dentry of the debugfs directory
+ *
+ * This function registers platform-specific VRAM debugfs files used for
+ * testing and debugging. Currently, it exposes the "vram_bad_pages" interface
+ * to inspect marked faulty memory pages, restricted specifically to the
+ * %XE_CRESCENTISLAND platform.
+ *
+ * Return: Void.
+ */
+void xe_ttm_vram_debugfs_init(struct xe_device *xe, struct dentry *root)
+{
+	/*
+	 * TODO: Replace platform check with xe->info
+	 * once the feature flag is plumbed through device info.
+	 */
+	if (xe->info.platform != XE_CRESCENTISLAND)
+		return;
+	debugfs_create_file("vram_bad_pages", 0444, root, xe, &vram_bad_pages_fops);
+}
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h
index 87b7fae5edba1a..d77f067d197b2b 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h
@@ -17,6 +17,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
 			   u32 mem_type, u64 size, u64 io_size,
 			   u64 default_page_size);
 int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram);
+void xe_ttm_vram_debugfs_init(struct xe_device *xe, struct dentry *root);
 int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe,
 			      struct ttm_resource *res,
 			      u64 offset, u64 length,
@@ -30,6 +31,8 @@ u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man);
 u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man);
 void xe_ttm_vram_get_used(struct ttm_resource_manager *man,
 			  u64 *used, u64 *used_visible);
+int xe_ttm_vram_handle_addr_fault(struct xe_device *xe, u64 addr);
+int xe_ttm_vram_inject_fault(struct xe_device *xe);
 
 static inline struct xe_ttm_vram_mgr_resource *
 to_xe_ttm_vram_mgr_resource(struct ttm_resource *res)
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
index 9106da056b4904..efcf3e1d4e80c7 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h
@@ -19,6 +19,14 @@ struct xe_ttm_vram_mgr {
 	struct ttm_resource_manager manager;
 	/** @mm: DRM buddy allocator which manages the VRAM */
 	struct gpu_buddy mm;
+	/** @offlined_pages: List of offlined pages */
+	struct list_head offlined_pages;
+	/** @n_offlined_pages: Number of offlined pages */
+	u16 n_offlined_pages;
+	/** @queued_pages: List of queued pages */
+	struct list_head queued_pages;
+	/** @n_queued_pages: Number of queued pages */
+	u16 n_queued_pages;
 	/** @visible_size: Proped size of the CPU visible portion */
 	u64 visible_size;
 	/** @visible_avail: CPU visible portion still unallocated */
@@ -29,6 +37,8 @@ struct xe_ttm_vram_mgr {
 	struct mutex lock;
 	/** @mem_type: The TTM memory type */
 	u32 mem_type;
+	/** @max_pages: max pages that can be in offline queue retrieved from FW */
+	u16 max_pages;
 };
 
 /**
@@ -45,4 +55,34 @@ struct xe_ttm_vram_mgr_resource {
 	unsigned long flags;
 };
 
+/**
+ * enum xe_page_reserve_status - Buddy reservation status
+ * @XE_PAGE_RESERVE_PENDING: reservation in progress
+ * @XE_PAGE_RESERVE_FAIL: reservation failed
+ */
+enum xe_page_reserve_status {
+	XE_PAGE_RESERVE_PENDING = 0,
+	XE_PAGE_RESERVE_FAIL,
+};
+
+/**
+ * struct xe_ttm_vram_offline_resource - Tracks a single offlined VRAM page
+ */
+struct xe_ttm_vram_offline_resource {
+	/** @offlined_link: Link into mgr->offlined_pages */
+	struct list_head offlined_link;
+	/** @queued_link: Link into mgr->queued_pages */
+	struct list_head queued_link;
+	/** @blocks: Buddy blocks reserved for this page */
+	struct list_head blocks;
+	/** @used_visible_size: CPU-visible bytes consumed */
+	u64 used_visible_size;
+	/** @addr: Faulty DPA reported by HW */
+	u64 addr;
+	/** @status: buddy reservation status */
+	enum xe_page_reserve_status status;
+	/** @rcu: RCU head for deferred freeing */
+	struct rcu_head rcu;
+};
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c
index bcec40ca2d35f3..a9bd3f8969a545 100644
--- a/drivers/gpu/drm/xe/xe_tuning.c
+++ b/drivers/gpu/drm/xe/xe_tuning.c
@@ -100,6 +100,10 @@ VISIBLE_IF_KUNIT const struct xe_rtp_table_sr gt_tunings = XE_RTP_TABLE_SR(
 	  XE_RTP_ACTIONS(FIELD_SET(GAMSTLB_CTRL, BANK_HASH_MODE,
 				   BANK_HASH_4KB_MODE))
 	},
+	{ XE_RTP_NAME("Tuning: Force All Non-Compressible Memory Reads to be 256B Overfetches"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED)),
+	  XE_RTP_ACTIONS(SET(L3SQCREG2, NONCOMPMEMRD256BOVRFETCHEN))
+	},
 );
 EXPORT_SYMBOL_IF_KUNIT(gt_tunings);
 
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c
index a8e6f18cc9b4aa..f61910431cf25e 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.c
+++ b/drivers/gpu/drm/xe/xe_uc_fw.c
@@ -141,6 +141,7 @@ struct fw_blobs_by_type {
 
 /* for the GSC FW we match the compatibility version and not the release one */
 #define XE_GSC_FIRMWARE_DEFS(fw_def, major_ver)		\
+	fw_def(NOVALAKE_S,	GT_TYPE_ANY,	major_ver(xe,	gsc,	nvl,	108, 0, 0))	\
 	fw_def(PANTHERLAKE,	GT_TYPE_ANY,	major_ver(xe,	gsc,	ptl,	105, 1, 0))	\
 	fw_def(LUNARLAKE,	GT_TYPE_ANY,	major_ver(xe,	gsc,	lnl,	104, 1, 0))	\
 	fw_def(METEORLAKE,	GT_TYPE_ANY,	major_ver(i915,	gsc,	mtl,	102, 1, 0))
diff --git a/drivers/gpu/drm/xe/xe_userptr.c b/drivers/gpu/drm/xe/xe_userptr.c
index 8b2d461ea0b28d..9c1dac0fce6f1c 100644
--- a/drivers/gpu/drm/xe/xe_userptr.c
+++ b/drivers/gpu/drm/xe/xe_userptr.c
@@ -57,6 +57,23 @@ int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
 		list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
 }
 
+#if IS_ENABLED(CONFIG_PROVE_LOCKING)
+static bool __xe_vma_userptr_lockdep(struct xe_userptr_vma *uvma)
+{
+	struct xe_vma *vma = &uvma->vma;
+	struct xe_vm *vm = xe_vma_vm(vma);
+
+	return lockdep_is_held_type(&vm->lock, 0) ||
+		(lockdep_is_held_type(&vm->lock, 1) &&
+		 lockdep_is_held_type(&vma->fault_lock, 0));
+}
+
+#define xe_vma_userptr_lockdep(uvma)	\
+	lockdep_assert(__xe_vma_userptr_lockdep(uvma))
+#else
+#define xe_vma_userptr_lockdep(uvma)
+#endif
+
 int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
 {
 	struct xe_vma *vma = &uvma->vma;
@@ -68,13 +85,13 @@ int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
 		.allow_mixed = true,
 	};
 
-	lockdep_assert_held(&vm->lock);
+	xe_vma_userptr_lockdep(uvma);
 	xe_assert(xe, xe_vma_is_userptr(vma));
 
 	if (vma->gpuva.flags & XE_VMA_DESTROYED)
 		return 0;
 
-	return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages,
+	return drm_gpusvm_get_pages(&vm->svm.gpusvm, &uvma->userptr.pages, 1,
 				    uvma->userptr.notifier.mm,
 				    &uvma->userptr.notifier,
 				    xe_vma_userptr(vma),
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 23952ad8951e28..425c678f4480ac 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -24,11 +24,13 @@
 #include "regs/xe_gtt_defs.h"
 #include "xe_assert.h"
 #include "xe_bo.h"
+#include "xe_cpu_bind.h"
 #include "xe_device.h"
 #include "xe_drm_client.h"
 #include "xe_exec_queue.h"
 #include "xe_gt.h"
 #include "xe_migrate.h"
+#include "xe_pagefault.h"
 #include "xe_pat.h"
 #include "xe_pm.h"
 #include "xe_preempt_fence.h"
@@ -147,7 +149,7 @@ static bool xe_vm_is_idle(struct xe_vm *vm)
 
 	xe_vm_assert_held(vm);
 	list_for_each_entry(q, &vm->preempt.exec_queues, lr.link) {
-		if (!xe_exec_queue_is_idle(q))
+		if (!xe_exec_queue_is_idle(q, 0))
 			return false;
 	}
 
@@ -615,10 +617,13 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf)
 {
 	struct xe_vm_fault_entry *e;
 	struct xe_hw_engine *hwe;
+	u8 engine_class = FIELD_GET(XE_PAGEFAULT_ENGINE_CLASS_MASK,
+				    pf->consumer.engine_class_instance);
+	u8 engine_instance = FIELD_GET(XE_PAGEFAULT_ENGINE_INSTANCE_MASK,
+				       pf->consumer.engine_class_instance);
 
 	/* Do not report faults on reserved engines */
-	hwe = xe_gt_hw_engine(pf->gt, pf->consumer.engine_class,
-			      pf->consumer.engine_instance, false);
+	hwe = xe_gt_hw_engine(pf->gt, engine_class, engine_instance, false);
 	if (!hwe || xe_hw_engine_is_reserved(hwe))
 		return;
 
@@ -640,7 +645,7 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf)
 		return;
 	}
 
-	e->address = pf->consumer.page_addr;
+	e->address = xe_pagefault_addr(pf);
 	/*
 	 * TODO:
 	 * Address precision is currently always SZ_4K, but this may change
@@ -652,6 +657,7 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf)
 				  pf->consumer.fault_type_level);
 	e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK,
 				   pf->consumer.fault_type_level);
+	e->srcid = FIELD_GET(XE_PAGEFAULT_SRCID_MASK, pf->consumer.id);
 
 	list_add_tail(&e->list, &vm->faults.list);
 	vm->faults.len++;
@@ -677,11 +683,9 @@ static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)
 		if (!vops->pt_update_ops[i].num_ops)
 			continue;
 
-		vops->pt_update_ops[i].ops =
-			kmalloc_objs(*vops->pt_update_ops[i].ops,
-				     vops->pt_update_ops[i].num_ops,
-				     GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
-		if (!vops->pt_update_ops[i].ops)
+		vops->pt_update_ops[i].pt_job_ops =
+			xe_pt_job_ops_alloc(vops->pt_update_ops[i].num_ops);
+		if (!vops->pt_update_ops[i].pt_job_ops)
 			return array_of_binds ? -ENOBUFS : -ENOMEM;
 	}
 
@@ -689,6 +693,17 @@ static int xe_vma_ops_alloc(struct xe_vma_ops *vops, bool array_of_binds)
 }
 ALLOW_ERROR_INJECTION(xe_vma_ops_alloc, ERRNO);
 
+static void xe_vma_svm_prefetch_ranges_fini(struct xe_vma_op *op)
+{
+	struct xe_svm_range *svm_range;
+	unsigned long i;
+
+	xa_for_each(&op->prefetch_range.range, i, svm_range)
+		xe_svm_range_put(svm_range);
+
+	xa_destroy(&op->prefetch_range.range);
+}
+
 static void xe_vma_svm_prefetch_op_fini(struct xe_vma_op *op)
 {
 	struct xe_vma *vma;
@@ -696,7 +711,7 @@ static void xe_vma_svm_prefetch_op_fini(struct xe_vma_op *op)
 	vma = gpuva_to_vma(op->base.prefetch.va);
 
 	if (op->base.op == DRM_GPUVA_OP_PREFETCH && xe_vma_is_cpu_addr_mirror(vma))
-		xa_destroy(&op->prefetch_range.range);
+		xe_vma_svm_prefetch_ranges_fini(op);
 }
 
 static void xe_vma_svm_prefetch_ops_fini(struct xe_vma_ops *vops)
@@ -717,7 +732,7 @@ static void xe_vma_ops_fini(struct xe_vma_ops *vops)
 	xe_vma_svm_prefetch_ops_fini(vops);
 
 	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
-		kfree(vops->pt_update_ops[i].ops);
+		xe_pt_job_ops_put(vops->pt_update_ops[i].pt_job_ops);
 }
 
 static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask, int inc_val)
@@ -782,16 +797,16 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 	struct xe_vma *vma, *next;
 	struct xe_vma_ops vops;
 	struct xe_vma_op *op, *next_op;
-	int err, i;
+	int err;
 
 	lockdep_assert_held(&vm->lock);
 	if ((xe_vm_in_lr_mode(vm) && !rebind_worker) ||
 	    list_empty(&vm->rebind_list))
 		return 0;
 
-	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
-	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
-		vops.pt_update_ops[i].wait_vm_bookkeep = true;
+	xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
+			NULL, 0);
+	vops.flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
 
 	xe_vm_assert_held(vm);
 	list_for_each_entry(vma, &vm->rebind_list, combined_links.rebind) {
@@ -831,28 +846,32 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 	return err;
 }
 
+static u8 adjust_rebind_tile_mask(struct xe_vm *vm, u8 tile_mask)
+{
+	if (vm->xe->info.has_pt_mirror)
+		return tile_mask;
+
+	return (0x1 << vm->xe->info.tile_count) - 1;
+}
+
 struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask)
 {
 	struct dma_fence *fence = NULL;
 	struct xe_vma_ops vops;
 	struct xe_vma_op *op, *next_op;
-	struct xe_tile *tile;
-	u8 id;
 	int err;
 
 	lockdep_assert_held(&vm->lock);
 	xe_vm_assert_held(vm);
 	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
 
-	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
-	vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
-	for_each_tile(tile, vm->xe, id) {
-		vops.pt_update_ops[id].wait_vm_bookkeep = true;
-		vops.pt_update_ops[tile->id].q =
-			xe_migrate_exec_queue(tile->migrate);
-	}
+	xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
+			NULL, 0);
+	vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT |
+		XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
 
-	err = xe_vm_ops_add_rebind(&vops, vma, tile_mask);
+	err = xe_vm_ops_add_rebind(&vops, vma,
+				   adjust_rebind_tile_mask(vm, tile_mask));
 	if (err)
 		return ERR_PTR(err);
 
@@ -926,24 +945,21 @@ struct dma_fence *xe_vm_range_rebind(struct xe_vm *vm,
 	struct dma_fence *fence = NULL;
 	struct xe_vma_ops vops;
 	struct xe_vma_op *op, *next_op;
-	struct xe_tile *tile;
-	u8 id;
 	int err;
 
+	lockdep_assert_held(&range->lock);
 	lockdep_assert_held(&vm->lock);
 	xe_vm_assert_held(vm);
 	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
 	xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma));
 
-	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
-	vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT;
-	for_each_tile(tile, vm->xe, id) {
-		vops.pt_update_ops[id].wait_vm_bookkeep = true;
-		vops.pt_update_ops[tile->id].q =
-			xe_migrate_exec_queue(tile->migrate);
-	}
+	xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
+			NULL, 0);
+	vops.flags |= XE_VMA_OPS_FLAG_SKIP_TLB_WAIT |
+		XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
 
-	err = xe_vm_ops_add_range_rebind(&vops, vma, range, tile_mask);
+	err = xe_vm_ops_add_range_rebind(&vops, vma, range,
+					 adjust_rebind_tile_mask(vm, tile_mask));
 	if (err)
 		return ERR_PTR(err);
 
@@ -1008,10 +1024,9 @@ struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm,
 	struct dma_fence *fence = NULL;
 	struct xe_vma_ops vops;
 	struct xe_vma_op *op, *next_op;
-	struct xe_tile *tile;
-	u8 id;
 	int err;
 
+	lockdep_assert_held(&range->lock);
 	lockdep_assert_held(&vm->lock);
 	xe_vm_assert_held(vm);
 	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
@@ -1019,12 +1034,9 @@ struct dma_fence *xe_vm_range_unbind(struct xe_vm *vm,
 	if (!range->tile_present)
 		return dma_fence_get_stub();
 
-	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
-	for_each_tile(tile, vm->xe, id) {
-		vops.pt_update_ops[id].wait_vm_bookkeep = true;
-		vops.pt_update_ops[tile->id].q =
-			xe_migrate_exec_queue(tile->migrate);
-	}
+	xe_vma_ops_init(&vops, vm, xe_cpu_bind_queue(vm->xe->cpu_bind),
+			NULL, 0);
+	vops.flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
 
 	err = xe_vm_ops_add_range_unbind(&vops, range);
 	if (err)
@@ -1187,6 +1199,8 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm,
 		xe_vm_get(vm);
 	}
 
+	mutex_init(&vma->fault_lock);
+
 	return vma;
 }
 
@@ -1211,6 +1225,7 @@ static void xe_vma_destroy_late(struct xe_vma *vma)
 		xe_bo_put(bo);
 	}
 
+	mutex_destroy(&vma->fault_lock);
 	xe_vma_free(vma);
 }
 
@@ -1231,12 +1246,19 @@ static void vma_destroy_cb(struct dma_fence *fence,
 	queue_work(system_dfl_wq, &vma->destroy_work);
 }
 
+static void xe_vm_assert_write_mode_or_garbage_collector(struct xe_vm *vm)
+{
+	lockdep_assert(lockdep_is_held_type(&vm->lock, 0) ||
+		       (lockdep_is_held_type(&vm->lock, 1) &&
+			lockdep_is_held_type(&vm->svm.garbage_collector.lock, 0)));
+}
+
 static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence)
 {
 	struct xe_vm *vm = xe_vma_vm(vma);
 	struct xe_bo *bo = xe_vma_bo(vma);
 
-	lockdep_assert_held_write(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 	xe_assert(vm->xe, list_empty(&vma->combined_links.destroy));
 
 	if (xe_vma_is_userptr(vma)) {
@@ -1320,7 +1342,9 @@ xe_vm_find_overlapping_vma(struct xe_vm *vm, u64 start, u64 range)
 
 	xe_assert(vm->xe, start + range <= vm->size);
 
+	mutex_lock(&vm->snap_mutex);
 	gpuva = drm_gpuva_find_first(&vm->gpuvm, start, range);
+	mutex_unlock(&vm->snap_mutex);
 
 	return gpuva ? gpuva_to_vma(gpuva) : NULL;
 }
@@ -1330,7 +1354,7 @@ static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma)
 	int err;
 
 	xe_assert(vm->xe, xe_vma_vm(vma) == vm);
-	lockdep_assert_held(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	mutex_lock(&vm->snap_mutex);
 	err = drm_gpuva_insert(&vm->gpuvm, &vma->gpuva);
@@ -1343,13 +1367,11 @@ static int xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma)
 static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma)
 {
 	xe_assert(vm->xe, xe_vma_vm(vma) == vm);
-	lockdep_assert_held(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	mutex_lock(&vm->snap_mutex);
 	drm_gpuva_remove(&vma->gpuva);
 	mutex_unlock(&vm->snap_mutex);
-	if (vm->usm.last_fault_vma == vma)
-		vm->usm.last_fault_vma = NULL;
 }
 
 static struct drm_gpuva_op *xe_vm_op_alloc(void)
@@ -1685,9 +1707,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 
 	init_rwsem(&vm->exec_queues.lock);
 	xe_vm_init_prove_locking(xe, vm);
-
-	for_each_tile(tile, xe, id)
-		xe_range_fence_tree_init(&vm->rftree[id]);
+	xe_range_fence_tree_init(&vm->rftree);
 
 	vm->pt_ops = &xelp_pt_ops;
 
@@ -1729,7 +1749,8 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 
 		for_each_tile(tile, xe, id) {
 			if (flags & XE_VM_FLAG_MIGRATION &&
-			    tile->id != XE_VM_FLAG_TILE_ID(flags))
+			    tile->id != XE_VM_FLAG_TILE_ID(flags) &&
+			    (vm->xe->info.has_pt_mirror || id))
 				continue;
 
 			vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level,
@@ -1785,7 +1806,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 			struct xe_exec_queue *q;
 			u32 create_flags = EXEC_QUEUE_FLAG_VM;
 
-			if (!vm->pt_root[id])
+			if (!vm->pt_root[id] || vm->q)
 				continue;
 
 			if (!xef) /* Not from userspace */
@@ -1796,7 +1817,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 				err = PTR_ERR(q);
 				goto err_close;
 			}
-			vm->q[id] = q;
+			vm->q = q;
 		}
 	}
 
@@ -1829,8 +1850,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef)
 	xe_svm_fini(vm);
 err_no_resv:
 	mutex_destroy(&vm->snap_mutex);
-	for_each_tile(tile, xe, id)
-		xe_range_fence_tree_fini(&vm->rftree[id]);
+	xe_range_fence_tree_fini(&vm->rftree);
 	ttm_lru_bulk_move_fini(&xe->ttm, &vm->lru_bulk_move);
 	if (vm->xef)
 		xe_file_put(vm->xef);
@@ -1849,6 +1869,8 @@ static void xe_vm_close(struct xe_vm *vm)
 	bound = drm_dev_enter(&xe->drm, &idx);
 
 	down_write(&vm->lock);
+	xe_vm_lock(vm, false);
+
 	if (xe_vm_in_fault_mode(vm))
 		xe_svm_notifier_lock(vm);
 
@@ -1876,6 +1898,8 @@ static void xe_vm_close(struct xe_vm *vm)
 
 	if (xe_vm_in_fault_mode(vm))
 		xe_svm_notifier_unlock(vm);
+
+	xe_vm_unlock(vm);
 	up_write(&vm->lock);
 
 	if (bound)
@@ -1886,10 +1910,8 @@ void xe_vm_close_and_put(struct xe_vm *vm)
 {
 	LIST_HEAD(contested);
 	struct xe_device *xe = vm->xe;
-	struct xe_tile *tile;
 	struct xe_vma *vma, *next_vma;
 	struct drm_gpuva *gpuva, *next;
-	u8 id;
 
 	xe_assert(xe, !vm->preempt.num_exec_queues);
 
@@ -1903,24 +1925,18 @@ void xe_vm_close_and_put(struct xe_vm *vm)
 	if (xe_vm_in_fault_mode(vm))
 		xe_svm_close(vm);
 
-	down_write(&vm->lock);
-	for_each_tile(tile, xe, id) {
-		if (vm->q[id]) {
-			int i;
+	if (vm->q) {
+		int i;
 
-			xe_exec_queue_last_fence_put(vm->q[id], vm);
-			for_each_tlb_inval(i)
-				xe_exec_queue_tlb_inval_last_fence_put(vm->q[id], vm, i);
-		}
-	}
-	up_write(&vm->lock);
+		down_write(&vm->lock);
+		xe_exec_queue_last_fence_put(vm->q, vm);
+		for_each_tlb_inval(vm->q, i)
+			xe_exec_queue_tlb_inval_last_fence_put(vm->q, vm, i);
+		up_write(&vm->lock);
 
-	for_each_tile(tile, xe, id) {
-		if (vm->q[id]) {
-			xe_exec_queue_kill(vm->q[id]);
-			xe_exec_queue_put(vm->q[id]);
-			vm->q[id] = NULL;
-		}
+		xe_exec_queue_kill(vm->q);
+		xe_exec_queue_put(vm->q);
+		vm->q = NULL;
 	}
 
 	down_write(&vm->lock);
@@ -1947,21 +1963,13 @@ void xe_vm_close_and_put(struct xe_vm *vm)
 		vma->gpuva.flags |= XE_VMA_DESTROYED;
 	}
 
-	/*
-	 * All vm operations will add shared fences to resv.
-	 * The only exception is eviction for a shared object,
-	 * but even so, the unbind when evicted would still
-	 * install a fence to resv. Hence it's safe to
-	 * destroy the pagetables immediately.
-	 */
-	xe_vm_free_scratch(vm);
-	xe_vm_pt_destroy(vm);
 	xe_vm_unlock(vm);
 
 	/*
-	 * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL
-	 * Since we hold a refcount to the bo, we can remove and free
-	 * the members safely without locking.
+	 * Unlink and destroy all contested external-BO VMAs before destroying
+	 * the page tables. Otherwise, concurrent eviction holding only bo->resv
+	 * can walk the BO's VMAs and attempt to invalidate/zap page tables that
+	 * have already been freed.
 	 */
 	list_for_each_entry_safe(vma, next_vma, &contested,
 				 combined_links.destroy) {
@@ -1969,6 +1977,11 @@ void xe_vm_close_and_put(struct xe_vm *vm)
 		xe_vma_destroy_unlocked(vma);
 	}
 
+	xe_vm_lock(vm, false);
+	xe_vm_free_scratch(vm);
+	xe_vm_pt_destroy(vm);
+	xe_vm_unlock(vm);
+
 	xe_svm_fini(vm);
 
 	up_write(&vm->lock);
@@ -1987,8 +2000,7 @@ void xe_vm_close_and_put(struct xe_vm *vm)
 
 	xe_vm_clear_fault_entries(vm);
 
-	for_each_tile(tile, xe, id)
-		xe_range_fence_tree_fini(&vm->rftree[id]);
+	xe_range_fence_tree_fini(&vm->rftree);
 
 	xe_vm_put(vm);
 }
@@ -2048,13 +2060,13 @@ struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id)
 
 u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile)
 {
-	return vm->pt_ops->pde_encode_bo(vm->pt_root[tile->id]->bo, 0);
+	return vm->pt_ops->pde_encode_bo(xe_vm_pt_root(vm, tile->id)->bo, 0);
 }
 
 static struct xe_exec_queue *
 to_wait_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 {
-	return q ? q : vm->q[0];
+	return q ? q : vm->q;
 }
 
 static struct xe_user_fence *
@@ -2185,7 +2197,7 @@ static int xe_vm_query_vmas(struct xe_vm *vm, u64 start, u64 end)
 	struct drm_gpuva *gpuva;
 	u32 num_vmas = 0;
 
-	lockdep_assert_held(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 	drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end)
 		num_vmas++;
 
@@ -2198,7 +2210,7 @@ static int get_mem_attrs(struct xe_vm *vm, u32 *num_vmas, u64 start,
 	struct drm_gpuva *gpuva;
 	int i = 0;
 
-	lockdep_assert_held(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) {
 		struct xe_vma *vma = gpuva_to_vma(gpuva);
@@ -2244,7 +2256,7 @@ int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_
 	if (XE_IOCTL_DBG(xe, !vm))
 		return -EINVAL;
 
-	err = down_read_interruptible(&vm->lock);
+	err = down_write_killable(&vm->lock);
 	if (err)
 		goto put_vm;
 
@@ -2278,21 +2290,12 @@ int xe_vm_query_vmas_attrs_ioctl(struct drm_device *dev, void *data, struct drm_
 free_mem_attrs:
 	kvfree(mem_attrs);
 unlock_vm:
-	up_read(&vm->lock);
+	up_write(&vm->lock);
 put_vm:
 	xe_vm_put(vm);
 	return err;
 }
 
-static bool vma_matches(struct xe_vma *vma, u64 page_addr)
-{
-	if (page_addr > xe_vma_end(vma) - 1 ||
-	    page_addr + SZ_4K - 1 < xe_vma_start(vma))
-		return false;
-
-	return true;
-}
-
 /**
  * xe_vm_find_vma_by_addr() - Find a VMA by its address
  *
@@ -2301,16 +2304,7 @@ static bool vma_matches(struct xe_vma *vma, u64 page_addr)
  */
 struct xe_vma *xe_vm_find_vma_by_addr(struct xe_vm *vm, u64 page_addr)
 {
-	struct xe_vma *vma = NULL;
-
-	if (vm->usm.last_fault_vma) {   /* Fast lookup */
-		if (vma_matches(vm->usm.last_fault_vma, page_addr))
-			vma = vm->usm.last_fault_vma;
-	}
-	if (!vma)
-		vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
-
-	return vma;
+	return xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);
 }
 
 static const u32 region_to_mem_type[] = {
@@ -2423,7 +2417,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 	u64 range_end = addr + range;
 	int err;
 
-	lockdep_assert_held_write(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	vm_dbg(&vm->xe->drm,
 	       "op=%d, addr=0x%016llx, range=0x%016llx, bo_offset_or_userptr=0x%016llx",
@@ -2446,10 +2440,12 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 			.map.gem.offset = bo_offset_or_userptr,
 		};
 
+		vops->flags |= XE_VMA_OPS_FLAG_MODIFIES_GPUVA;
 		ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, &map_req);
 		break;
 	}
 	case DRM_XE_VM_BIND_OP_UNMAP:
+		vops->flags |= XE_VMA_OPS_FLAG_MODIFIES_GPUVA;
 		ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range);
 		break;
 	case DRM_XE_VM_BIND_OP_PREFETCH:
@@ -2458,6 +2454,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 	case DRM_XE_VM_BIND_OP_UNMAP_ALL:
 		xe_assert(vm->xe, bo);
 
+		vops->flags |= XE_VMA_OPS_FLAG_MODIFIES_GPUVA;
 		err = xe_bo_lock(bo, true);
 		if (err)
 			return ERR_PTR(err);
@@ -2479,6 +2476,16 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 	if (IS_ERR(ops))
 		return ops;
 
+	/* Setup safe unwind */
+	drm_gpuva_for_each_op(__op, ops) {
+		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
+
+		if (__op->op == DRM_GPUVA_OP_PREFETCH) {
+			xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC);
+			op->prefetch_range.ranges_count = 0;
+		}
+	}
+
 	drm_gpuva_for_each_op(__op, ops) {
 		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
 
@@ -2507,21 +2514,24 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 			struct drm_pagemap *dpagemap = NULL;
 			u8 id, tile_mask = 0;
 			u32 i;
+			bool need_put, valid_pages;
+
+			if (xe_vma_is_userptr(vma))
+				vops->flags |= XE_VMA_OPS_FLAG_MODIFIES_GPUVA;
 
 			if (!xe_vma_is_cpu_addr_mirror(vma)) {
 				op->prefetch.region = prefetch_region;
-				break;
+				continue;
 			}
 
 			ctx.read_only = xe_vma_read_only(vma);
 			ctx.devmem_possible = IS_DGFX(vm->xe) &&
 					      IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
 
-			for_each_tile(tile, vm->xe, id)
+			for_each_tile(tile, vm->xe, id) {
+				xe_migrate_ulls_enter(tile->migrate);
 				tile_mask |= 0x1 << id;
-
-			xa_init_flags(&op->prefetch_range.range, XA_FLAGS_ALLOC);
-			op->prefetch_range.ranges_count = 0;
+			}
 
 			if (prefetch_region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) {
 				dpagemap = xe_vma_resolve_pagemap(vma,
@@ -2534,6 +2544,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 
 			op->prefetch_range.dpagemap = dpagemap;
 alloc_next_range:
+			need_put = false;
 			svm_range = xe_svm_range_find_or_insert(vm, addr, vma, &ctx);
 
 			if (PTR_ERR(svm_range) == -ENOENT) {
@@ -2551,8 +2562,11 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 				goto unwind_prefetch_ops;
 			}
 
-			if (xe_svm_range_validate(vm, svm_range, tile_mask, dpagemap)) {
+			if (xe_svm_range_validate(vm, svm_range, tile_mask,
+						  dpagemap, &valid_pages)) {
 				xe_svm_range_debug(svm_range, "PREFETCH - RANGE IS VALID");
+				xe_assert(vm->xe, valid_pages);
+				need_put = true;
 				goto check_next_range;
 			}
 
@@ -2560,18 +2574,27 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_vma_ops *vops,
 				       &i, svm_range, xa_limit_32b,
 				       GFP_KERNEL);
 
-			if (err)
+			if (err) {
+				xe_svm_range_put(svm_range);
 				goto unwind_prefetch_ops;
+			}
 
 			op->prefetch_range.ranges_count++;
 			vops->flags |= XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH;
+			if (valid_pages)
+				vops->flags |= XE_VMA_OPS_FLAG_HAS_SVM_VALID_RANGE;
 			xe_svm_range_debug(svm_range, "PREFETCH - RANGE CREATED");
 check_next_range:
 			if (range_end > xe_svm_range_end(svm_range) &&
 			    xe_svm_range_end(svm_range) < xe_vma_end(vma)) {
 				addr = xe_svm_range_end(svm_range);
+				if (need_put)
+					xe_svm_range_put(svm_range);
 				goto alloc_next_range;
 			}
+			if (need_put)
+				xe_svm_range_put(svm_range);
+
 		}
 print_op_label:
 		print_op(vm->xe, __op);
@@ -2596,7 +2619,7 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
 	struct xe_vma *vma;
 	int err = 0;
 
-	lockdep_assert_held_write(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	if (bo) {
 		err = 0;
@@ -2693,10 +2716,12 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
 {
 	int err = 0;
 
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
+		xe_vm_assert_write_mode_or_garbage_collector(vm);
+
 		err |= xe_vm_insert_vma(vm, op->map.vma);
 		if (!err)
 			op->flags |= XE_VMA_OP_COMMITTED;
@@ -2706,6 +2731,8 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
 		u8 tile_present =
 			gpuva_to_vma(op->base.remap.unmap->va)->tile_present;
 
+		xe_vm_assert_write_mode_or_garbage_collector(vm);
+
 		prep_vma_destroy(vm, gpuva_to_vma(op->base.remap.unmap->va),
 				 true);
 		op->flags |= XE_VMA_OP_COMMITTED;
@@ -2740,6 +2767,8 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
 		break;
 	}
 	case DRM_GPUVA_OP_UNMAP:
+		xe_vm_assert_write_mode_or_garbage_collector(vm);
+
 		prep_vma_destroy(vm, gpuva_to_vma(op->base.unmap.va), true);
 		op->flags |= XE_VMA_OP_COMMITTED;
 		break;
@@ -2785,7 +2814,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops,
 	u8 id, tile_mask = 0;
 	int err = 0;
 
-	lockdep_assert_held_write(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	for_each_tile(tile, vm->xe, id)
 		tile_mask |= 0x1 << id;
@@ -2964,10 +2993,12 @@ static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
 			     bool post_commit, bool prev_post_commit,
 			     bool next_post_commit)
 {
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
+		xe_vm_assert_write_mode_or_garbage_collector(vm);
+
 		if (op->map.vma) {
 			prep_vma_destroy(vm, op->map.vma, post_commit);
 			xe_vma_destroy_unlocked(op->map.vma);
@@ -2977,6 +3008,8 @@ static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
 	{
 		struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
 
+		xe_vm_assert_write_mode_or_garbage_collector(vm);
+
 		if (vma) {
 			xe_svm_notifier_lock(vm);
 			vma->gpuva.flags &= ~XE_VMA_DESTROYED;
@@ -2990,6 +3023,8 @@ static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
 	{
 		struct xe_vma *vma = gpuva_to_vma(op->base.remap.unmap->va);
 
+		xe_vm_assert_write_mode_or_garbage_collector(vm);
+
 		if (op->remap.prev) {
 			prep_vma_destroy(vm, op->remap.prev, prev_post_commit);
 			xe_vma_destroy_unlocked(op->remap.prev);
@@ -3118,16 +3153,87 @@ static int check_ufence(struct xe_vma *vma)
 	return 0;
 }
 
-static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_op *op)
+struct prefetch_thread {
+	struct work_struct work;
+	struct drm_gpusvm_ctx *ctx;
+	struct xe_vma *vma;
+	struct xe_svm_range *svm_range;
+	struct drm_pagemap *dpagemap;
+	int err;
+};
+
+static void prefetch_thread_func(struct prefetch_thread *thread)
+{
+	struct xe_vma *vma = thread->vma;
+	struct xe_vm *vm = xe_vma_vm(vma);
+	struct xe_svm_range *svm_range = thread->svm_range;
+	struct drm_pagemap *dpagemap = thread->dpagemap;
+	int err = 0;
+
+	guard(mutex)(&svm_range->lock);
+
+	if (xe_svm_range_is_removed(svm_range))
+		return;
+
+	if (!dpagemap)
+		xe_svm_range_migrate_to_smem(vm, svm_range);
+
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) {
+		drm_dbg(&vm->xe->drm,
+			"Prefetch pagemap is %s start 0x%016lx end 0x%016lx\n",
+			dpagemap ? dpagemap->drm->unique : "system",
+			xe_svm_range_start(svm_range), xe_svm_range_end(svm_range));
+	}
+
+	if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, dpagemap)) {
+		err = xe_svm_alloc_vram(svm_range, thread->ctx, dpagemap);
+		if (err) {
+			drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n",
+				vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
+			/*
+			 * We intentionally return -ENODATA on any races to
+			 * commit any VMA updates from other ops without
+			 * updating any page tables deferring to page faults to
+			 * page updates skipped in the IOCTL.
+			 */
+			thread->err = -ENODATA;
+			return;
+		}
+		xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM");
+	}
+
+	err = xe_svm_range_get_pages(vm, svm_range, thread->ctx);
+	if (err) {
+		drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n",
+			vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
+		if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM)
+			err = -ENODATA;
+		thread->err = err;
+		return;
+	}
+	xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE");
+}
+
+static void prefetch_work_func(struct work_struct *w)
+{
+	struct prefetch_thread *thread =
+		container_of(w, struct prefetch_thread, work);
+
+	prefetch_thread_func(thread);
+}
+
+static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops,
+			   struct xe_vma_op *op)
 {
 	bool devmem_possible = IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_PAGEMAP);
 	struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
 	struct drm_pagemap *dpagemap = op->prefetch_range.dpagemap;
-	int err = 0;
-
 	struct xe_svm_range *svm_range;
 	struct drm_gpusvm_ctx ctx = {};
+	struct prefetch_thread stack_thread, *thread, *prefetches;
 	unsigned long i;
+	int err = 0, idx = 0;
+	bool skip_threads;
 
 	if (!xe_vma_is_cpu_addr_mirror(vma))
 		return 0;
@@ -3137,37 +3243,50 @@ static int prefetch_ranges(struct xe_vm *vm, struct xe_vma_op *op)
 	ctx.check_pages_threshold = devmem_possible ? SZ_64K : 0;
 	ctx.device_private_page_owner = xe_svm_private_page_owner(vm, !dpagemap);
 
-	/* TODO: Threading the migration */
+	skip_threads =  op->prefetch_range.ranges_count == 1 ||
+		(!dpagemap && !(vops->flags &
+				XE_VMA_OPS_FLAG_HAS_SVM_VALID_RANGE)) ||
+		!(vops->flags & XE_VMA_OPS_FLAG_DOWNGRADE_LOCK) ||
+		vm->xe->info.num_pf_work == 1;
+	thread = skip_threads ? &stack_thread : NULL;
+
+	if (!skip_threads) {
+		prefetches = kvmalloc_array(op->prefetch_range.ranges_count,
+					    sizeof(*prefetches), GFP_KERNEL);
+		if (!prefetches)
+			return -ENOMEM;
+	}
+
 	xa_for_each(&op->prefetch_range.range, i, svm_range) {
-		if (!dpagemap)
-			xe_svm_range_migrate_to_smem(vm, svm_range);
-
-		if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) {
-			drm_dbg(&vm->xe->drm,
-				"Prefetch pagemap is %s start 0x%016lx end 0x%016lx\n",
-				dpagemap ? dpagemap->drm->unique : "system",
-				xe_svm_range_start(svm_range), xe_svm_range_end(svm_range));
+		if (!skip_threads) {
+			thread = prefetches + idx++;
+			INIT_WORK(&thread->work, prefetch_work_func);
 		}
 
-		if (xe_svm_range_needs_migrate_to_vram(svm_range, vma, dpagemap)) {
-			err = xe_svm_alloc_vram(svm_range, &ctx, dpagemap);
-			if (err) {
-				drm_dbg(&vm->xe->drm, "VRAM allocation failed, retry from userspace, asid=%u, gpusvm=%p, errno=%pe\n",
-					vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
-				return -ENODATA;
-			}
-			xe_svm_range_debug(svm_range, "PREFETCH - RANGE MIGRATED TO VRAM");
+		thread->ctx = &ctx;
+		thread->vma = vma;
+		thread->svm_range = svm_range;
+		thread->dpagemap = dpagemap;
+		thread->err = 0;
+
+		if (skip_threads) {
+			prefetch_thread_func(thread);
+			if (thread->err)
+				return thread->err;
+		} else {
+			queue_work(vm->xe->usm.prefetch_wq, &thread->work);
 		}
+	}
 
-		err = xe_svm_range_get_pages(vm, svm_range, &ctx);
-		if (err) {
-			drm_dbg(&vm->xe->drm, "Get pages failed, asid=%u, gpusvm=%p, errno=%pe\n",
-				vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err));
-			if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM)
-				err = -ENODATA;
-			return err;
+	if (!skip_threads) {
+		for (i = 0; i < idx; ++i) {
+			thread = prefetches + i;
+
+			flush_work(&thread->work);
+			if (thread->err && !err)
+				err = thread->err;
 		}
-		xe_svm_range_debug(svm_range, "PREFETCH - RANGE GET PAGES DONE");
+		kvfree(prefetches);
 	}
 
 	return err;
@@ -3298,7 +3417,8 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm,
 	return err;
 }
 
-static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops *vops)
+static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm *vm,
+					     struct xe_vma_ops *vops)
 {
 	struct xe_vma_op *op;
 	int err;
@@ -3308,7 +3428,7 @@ static int vm_bind_ioctl_ops_prefetch_ranges(struct xe_vm *vm, struct xe_vma_ops
 
 	list_for_each_entry(op, &vops->list, link) {
 		if (op->base.op  == DRM_GPUVA_OP_PREFETCH) {
-			err = prefetch_ranges(vm, op);
+			err = prefetch_ranges(vm, vops, op);
 			if (err)
 				return err;
 		}
@@ -3377,28 +3497,18 @@ static void trace_xe_vm_ops_execute(struct xe_vma_ops *vops)
 		op_trace(op);
 }
 
-static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops)
+static int vm_ops_setup(struct xe_vm *vm, struct xe_vma_ops *vops)
 {
-	struct xe_exec_queue *q = vops->q;
 	struct xe_tile *tile;
 	int number_tiles = 0;
 	u8 id;
 
-	for_each_tile(tile, vm->xe, id) {
+	for_each_tile(tile, vm->xe, id)
 		if (vops->pt_update_ops[id].num_ops)
 			++number_tiles;
 
-		if (vops->pt_update_ops[id].q)
-			continue;
-
-		if (q) {
-			vops->pt_update_ops[id].q = q;
-			if (vm->pt_root[id] && !list_empty(&q->multi_gt_list))
-				q = list_next_entry(q, multi_gt_list);
-		} else {
-			vops->pt_update_ops[id].q = vm->q[id];
-		}
-	}
+	if (!vops->q)
+		vops->q = vm->q;
 
 	return number_tiles;
 }
@@ -3406,26 +3516,21 @@ static int vm_ops_setup_tile_args(struct xe_vm *vm, struct xe_vma_ops *vops)
 static struct dma_fence *ops_execute(struct xe_vm *vm,
 				     struct xe_vma_ops *vops)
 {
-	struct xe_tile *tile;
+	struct xe_device *xe = vm->xe;
 	struct dma_fence *fence = NULL;
 	struct dma_fence **fences = NULL;
 	struct dma_fence_array *cf = NULL;
-	int number_tiles = 0, current_fence = 0, n_fence = 0, err, i;
-	u8 id;
+	int current_fence = 0, n_fence = 1, err, i;
 
-	number_tiles = vm_ops_setup_tile_args(vm, vops);
-	if (number_tiles == 0)
+	if (!vm_ops_setup(vm, vops))
 		return ERR_PTR(-ENODATA);
 
-	for_each_tile(tile, vm->xe, id) {
-		++n_fence;
-
-		if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT))
-			for_each_tlb_inval(i)
-				++n_fence;
+	if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) {
+		for_each_tlb_inval(vops->q, i)
+			++n_fence;
 	}
 
-	fences = kmalloc_objs(*fences, n_fence);
+	fences = kcalloc(n_fence, sizeof(*fences), GFP_KERNEL);
 	if (!fences) {
 		fence = ERR_PTR(-ENOMEM);
 		goto err_trace;
@@ -3437,63 +3542,39 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
 		goto err_out;
 	}
 
-	for_each_tile(tile, vm->xe, id) {
-		if (!vops->pt_update_ops[id].num_ops)
-			continue;
-
-		err = xe_pt_update_ops_prepare(tile, vops);
-		if (err) {
-			fence = ERR_PTR(err);
-			goto err_out;
-		}
+	err = xe_pt_update_ops_prepare(xe, vops);
+	if (err) {
+		fence = ERR_PTR(err);
+		goto err_out;
 	}
 
 	trace_xe_vm_ops_execute(vops);
 
-	for_each_tile(tile, vm->xe, id) {
-		struct xe_exec_queue *q = vops->pt_update_ops[tile->id].q;
-
-		fence = NULL;
-		if (!vops->pt_update_ops[id].num_ops)
-			goto collect_fences;
-
-		fence = xe_pt_update_ops_run(tile, vops);
-		if (IS_ERR(fence))
-			goto err_out;
-
-collect_fences:
-		fences[current_fence++] = fence ?: dma_fence_get_stub();
-		if (vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)
-			continue;
+	fence = xe_pt_update_ops_run(xe, vops);
+	if (IS_ERR(fence))
+		goto err_out;
+	fences[current_fence++] = fence;
 
-		xe_migrate_job_lock(tile->migrate, q);
-		for_each_tlb_inval(i)
+	if (!(vops->flags & XE_VMA_OPS_FLAG_SKIP_TLB_WAIT)) {
+		xe_cpu_bind_job_lock(xe->cpu_bind, vops->q);
+		for_each_tlb_inval(vops->q, i)
 			fences[current_fence++] =
-				xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
-		xe_migrate_job_unlock(tile->migrate, q);
+				xe_exec_queue_tlb_inval_last_fence_get(vops->q,
+								       vm, i);
+		xe_cpu_bind_job_unlock(xe->cpu_bind, vops->q);
 	}
 
-	xe_assert(vm->xe, current_fence == n_fence);
+	xe_assert(xe, current_fence == n_fence);
 	dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1),
 			     1);
 	fence = &cf->base;
 
-	for_each_tile(tile, vm->xe, id) {
-		if (!vops->pt_update_ops[id].num_ops)
-			continue;
-
-		xe_pt_update_ops_fini(tile, vops);
-	}
+	xe_pt_update_ops_fini(xe, vops);
 
 	return fence;
 
 err_out:
-	for_each_tile(tile, vm->xe, id) {
-		if (!vops->pt_update_ops[id].num_ops)
-			continue;
-
-		xe_pt_update_ops_abort(tile, vops);
-	}
+	xe_pt_update_ops_abort(xe, vops);
 	while (current_fence)
 		dma_fence_put(fences[--current_fence]);
 	kfree(fences);
@@ -3569,7 +3650,7 @@ static struct dma_fence *vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 	struct dma_fence *fence;
 	int err = 0;
 
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 
 	xe_validation_guard(&ctx, &vm->xe->val, &exec,
 			    ((struct xe_val_flags) {
@@ -3805,6 +3886,8 @@ static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
 	vops->syncs = syncs;
 	vops->num_syncs = num_syncs;
 	vops->flags = 0;
+	vops->start = ~0x0ull;
+	vops->last = 0x0ull;
 }
 
 static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo,
@@ -3892,7 +3975,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	u32 num_syncs, num_ufence = 0;
 	struct xe_sync_entry *syncs = NULL;
 	struct drm_xe_vm_bind_op *bind_ops = NULL;
-	struct xe_vma_ops vops;
+	struct xe_vma_ops vops = { .flags = 0, };
 	struct dma_fence *fence;
 	int err;
 	int i;
@@ -3999,7 +4082,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 
 	syncs_user = u64_to_user_ptr(args->syncs);
 	for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
-		struct xe_exec_queue *__q = q ?: vm->q[0];
+		struct xe_exec_queue *__q = q ?: vm->q;
 
 		err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
 					  &syncs_user[num_syncs],
@@ -4067,6 +4150,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		goto unwind_ops;
 	}
 
+	if (!(vops.flags & XE_VMA_OPS_FLAG_MODIFIES_GPUVA)) {
+		vops.flags |= XE_VMA_OPS_FLAG_DOWNGRADE_LOCK;
+		downgrade_write(&vm->lock);
+	}
+
 	err = xe_vma_ops_alloc(&vops, args->num_binds > 1);
 	if (err)
 		goto unwind_ops;
@@ -4103,7 +4191,10 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 free_bos:
 	kvfree(bos);
 release_vm_lock:
-	up_write(&vm->lock);
+	if (vops.flags & XE_VMA_OPS_FLAG_DOWNGRADE_LOCK)
+		up_read(&vm->lock);
+	else
+		up_write(&vm->lock);
 put_exec_queue:
 	if (q)
 		xe_exec_queue_put(q);
@@ -4137,6 +4228,11 @@ static u8 xe_to_user_fault_level(u8 fault_level)
 	return fault_level;
 }
 
+static u8 xe_to_user_srcid(u8 srcid)
+{
+	return srcid;
+}
+
 static int fill_faults(struct xe_vm *vm,
 		       struct drm_xe_vm_get_property *args)
 {
@@ -4164,6 +4260,8 @@ static int fill_faults(struct xe_vm *vm,
 		fault_entry.fault_type = xe_to_user_fault_type(entry->fault_type);
 		fault_entry.fault_level = xe_to_user_fault_level(entry->fault_level);
 
+		fault_entry.srcid = xe_to_user_srcid(entry->srcid);
+
 		memcpy(&fault_list[i], &fault_entry, entry_size);
 
 		i++;
@@ -4735,7 +4833,7 @@ static int xe_vm_alloc_vma(struct xe_vm *vm,
 	u16 default_pat;
 	int err;
 
-	lockdep_assert_held_write(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	if (is_madvise)
 		ops = drm_gpuvm_madvise_ops_create(&vm->gpuvm, map_req);
@@ -4869,7 +4967,7 @@ int xe_vm_alloc_madvise_vma(struct xe_vm *vm, uint64_t start, uint64_t range)
 		.map.va.range = range,
 	};
 
-	lockdep_assert_held_write(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	vm_dbg(&vm->xe->drm, "MADVISE_OPS_CREATE: addr=0x%016llx, size=0x%016llx", start, range);
 
@@ -4901,7 +4999,7 @@ void xe_vm_find_cpu_addr_mirror_vma_range(struct xe_vm *vm, u64 *start, u64 *end
 {
 	struct xe_vma *prev, *next;
 
-	lockdep_assert_held(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	if (*start >= SZ_4K) {
 		prev = xe_vm_find_vma_by_addr(vm, *start - SZ_4K);
@@ -4933,7 +5031,7 @@ int xe_vm_alloc_cpu_addr_mirror_vma(struct xe_vm *vm, uint64_t start, uint64_t r
 		.map.va.range = range,
 	};
 
-	lockdep_assert_held_write(&vm->lock);
+	xe_vm_assert_write_mode_or_garbage_collector(vm);
 
 	vm_dbg(&vm->xe->drm, "CPU_ADDR_MIRROR_VMA_OPS_CREATE: addr=0x%016llx, size=0x%016llx",
 	       start, range);
@@ -4955,7 +5053,6 @@ void xe_vm_add_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 
 	/* User VMs and queues only */
 	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_KERNEL));
-	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_PERMANENT));
 	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
 	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_MIGRATE));
 	xe_assert(xe, vm->xef);
@@ -4990,3 +5087,21 @@ void xe_vm_remove_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q)
 	}
 	up_write(&vm->exec_queues.lock);
 }
+
+/**
+ * xe_vm_pt_root() - Retrieve VM page-table root
+ * @vm: The VM.
+ * @tile_id: Tile ID
+ *
+ * Retrieve VM page-table root for a tile ID, used to abstract if PT mirroring is
+ * enabled across tiles.
+ *
+ * Return: VM page-table root for a tile ID
+ */
+struct xe_pt *xe_vm_pt_root(struct xe_vm *vm, u8 tile_id)
+{
+	if (vm->xe->info.has_pt_mirror)
+		return vm->pt_root[tile_id];
+
+	return vm->pt_root[0];
+}
diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index c5b900f38dedc1..89c4f95984a20b 100644
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -436,4 +436,7 @@ static inline struct drm_exec *xe_vm_validation_exec(struct xe_vm *vm)
 	((READ_ONCE(tile_present) & ~READ_ONCE(tile_invalidated)) & BIT((tile)->id))
 
 void xe_vma_mem_attr_copy(struct xe_vma_mem_attr *to, struct xe_vma_mem_attr *from);
+
+struct xe_pt *xe_vm_pt_root(struct xe_vm *vm, u8 tile_id);
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c
index 0474768a38aaa7..5c7aefc641ee8a 100644
--- a/drivers/gpu/drm/xe/xe_vm_madvise.c
+++ b/drivers/gpu/drm/xe/xe_vm_madvise.c
@@ -713,12 +713,6 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil
 
 	attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs));
 
-	/* Ensure the madvise function exists for this type */
-	if (!madvise_funcs[attr_type]) {
-		err = -EINVAL;
-		goto err_fini;
-	}
-
 	madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args,
 				 &details);
 
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index 635ed29b9a69b0..fb9305124679e2 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -132,6 +132,12 @@ struct xe_vma {
 		struct work_struct destroy_work;
 	};
 
+	/**
+	 * @fault_lock: Synchronizes fault processing. Locking order: inside
+	 * vm->lock, outside dma-resv.
+	 */
+	struct mutex fault_lock;
+
 	/**
 	 * @tile_invalidated: Tile mask of binding are invalidated for this VMA.
 	 * protected by BO's resv and for userptrs, vm->svm.gpusvm.notifier_lock in
@@ -196,6 +202,7 @@ struct xe_device;
  * @access_type: type of address access that resulted in fault
  * @fault_type: type of fault reported
  * @fault_level: fault level of the fault
+ * @srcid: ID of the faulting hardware unit
  */
 struct xe_vm_fault_entry {
 	struct list_head list;
@@ -204,6 +211,7 @@ struct xe_vm_fault_entry {
 	u8 access_type;
 	u8 fault_type;
 	u8 fault_level;
+	u8 srcid;
 };
 
 struct xe_vm {
@@ -214,13 +222,27 @@ struct xe_vm {
 	struct {
 		/** @svm.gpusvm: base GPUSVM used to track fault allocations */
 		struct drm_gpusvm gpusvm;
+		/**
+		 * @svm.range_lock: Protects insertion and removal of ranges
+		 * from GPU SVM tree.
+		 */
+		struct mutex range_lock;
 		/**
 		 * @svm.garbage_collector: Garbage collector which is used unmap
 		 * SVM range's GPU bindings and destroy the ranges.
 		 */
 		struct {
-			/** @svm.garbage_collector.lock: Protect's range list */
-			spinlock_t lock;
+			/**
+			 * @svm.garbage_collector.lock: Ensures only one thread
+			 * runs the garbage collector at a time. Locking order:
+			 * inside vm->lock, outside range->lock and dma-resv.
+			 */
+			struct mutex lock;
+			/**
+			 * @svm.garbage_collector.list_lock: Protect's range
+			 * list
+			 */
+			spinlock_t list_lock;
 			/**
 			 * @svm.garbage_collector.range_list: List of SVM ranges
 			 * in the garbage collector.
@@ -240,7 +262,7 @@ struct xe_vm {
 	struct xe_device *xe;
 
 	/* exec queue used for (un)binding vma's */
-	struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE];
+	struct xe_exec_queue *q;
 
 	/** @lru_bulk_move: Bulk LRU move list for this VM's BOs */
 	struct ttm_lru_bulk_move lru_bulk_move;
@@ -295,7 +317,7 @@ struct xe_vm {
 	 * @rftree: range fence tree to track updates to page table structure.
 	 * Used to implement conflict tracking between independent bind engines.
 	 */
-	struct xe_range_fence_tree rftree[XE_MAX_TILES_PER_DEVICE];
+	struct xe_range_fence_tree rftree;
 
 	const struct xe_pt_ops *pt_ops;
 
@@ -350,11 +372,6 @@ struct xe_vm {
 	struct {
 		/** @asid: address space ID, unique to each VM */
 		u32 asid;
-		/**
-		 * @last_fault_vma: Last fault VMA, used for fast lookup when we
-		 * get a flood of faults to the same VMA
-		 */
-		struct xe_vma *last_fault_vma;
 	} usm;
 
 	/** @error_capture: allow to track errors */
@@ -540,12 +557,23 @@ struct xe_vma_ops {
 	u32 num_syncs;
 	/** @pt_update_ops: page table update operations */
 	struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
+	/** @start: start address of ops */
+	u64 start;
+	/** @last: last address of ops */
+	u64 last;
 	/** @flag: signify the properties within xe_vma_ops*/
-#define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH BIT(0)
-#define XE_VMA_OPS_FLAG_MADVISE          BIT(1)
-#define XE_VMA_OPS_ARRAY_OF_BINDS	 BIT(2)
-#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT	 BIT(3)
-#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP  BIT(4)
+#define XE_VMA_OPS_FLAG_HAS_SVM_PREFETCH	BIT(0)
+#define XE_VMA_OPS_FLAG_MADVISE			BIT(1)
+#define XE_VMA_OPS_ARRAY_OF_BINDS		BIT(2)
+#define XE_VMA_OPS_FLAG_SKIP_TLB_WAIT		BIT(3)
+#define XE_VMA_OPS_FLAG_ALLOW_SVM_UNMAP		BIT(4)
+#define XE_VMA_OPS_FLAG_MODIFIES_GPUVA		BIT(5)
+#define XE_VMA_OPS_FLAG_DOWNGRADE_LOCK		BIT(6)
+#define XE_VMA_OPS_FLAG_HAS_SVM_VALID_RANGE	BIT(7)
+#define XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP	BIT(8)
+#define XE_VMA_OPS_FLAG_WAIT_VM_KERNEL		BIT(9)
+#define XE_VMA_OPS_FLAG_NEEDS_INVALIDATION	BIT(10)
+#define XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK		BIT(11)
 	u32 flags;
 #ifdef TEST_VM_OPS_ERROR
 	/** @inject_error: inject error to test error handling */
diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c
index 56cff1e44530b0..dcedd8cfd731ea 100644
--- a/drivers/gpu/drm/xe/xe_vram.c
+++ b/drivers/gpu/drm/xe/xe_vram.c
@@ -17,8 +17,11 @@
 #include "xe_device.h"
 #include "xe_force_wake.h"
 #include "xe_gt_mcr.h"
+#include "xe_map.h"
+#include "xe_migrate.h"
 #include "xe_mmio.h"
 #include "xe_sriov.h"
+#include "xe_tile.h"
 #include "xe_tile_sriov_vf.h"
 #include "xe_ttm_vram_mgr.h"
 #include "xe_vram.h"
@@ -55,9 +58,6 @@ static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region *
 	/* XXX: Need to change when xe link code is ready */
 	lmem_bar->dpa_base = 0;
 
-	/* set up a map to the total memory area. */
-	lmem_bar->mapping = devm_ioremap_wc(&pdev->dev, lmem_bar->io_start, lmem_bar->io_size);
-
 	return 0;
 }
 
@@ -196,7 +196,7 @@ static void vram_fini(void *arg)
 	struct xe_tile *tile;
 	int id;
 
-	xe->mem.vram->mapping = NULL;
+	xe_assert(xe, !xe->mem.vram->mapping);
 
 	for_each_tile(tile, xe, id) {
 		tile->mem.vram->mapping = NULL;
@@ -257,8 +257,15 @@ static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram,
 		return -ENODEV;
 	}
 
+	if (vram != xe->mem.vram) {
+		struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+
+		vram->mapping = devm_ioremap_wc(&pdev->dev, vram->io_start, vram->io_size);
+		if (!vram->mapping)
+			return -ENOMEM;
+	}
+
 	vram->dpa_base = lmem_bar->dpa_base + offset;
-	vram->mapping = lmem_bar->mapping + offset;
 	vram->usable_size = usable_size;
 
 	print_vram_region_info(xe, vram);
@@ -403,3 +410,241 @@ resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region
 	return vram ? vram->actual_physical_size : 0;
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_vram_region_actual_physical_size);
+
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
+static void memtest_bo_cleanup(void *arg)
+{
+	struct xe_device *xe = arg;
+
+	xe_vram_free_memtest_bos(xe);
+}
+
+int xe_vram_reserve_memtest_bo(struct xe_device *xe)
+{
+	struct xe_tile *tile;
+	u8 id;
+
+	if (IS_SRIOV_VF(xe))
+		return 0;
+
+	for_each_tile(tile, xe, id) {
+		u64 vram_size;
+
+		if (!tile->mem.vram)
+			continue;
+
+		if (tile->mem.vram->io_size < tile->mem.vram->usable_size) {
+			drm_info(&xe->drm,
+				 "Tile %d: Small-BAR system detected, skipping VRAM memtest\n",
+				 id);
+			continue;
+		}
+
+		vram_size = tile->mem.vram->usable_size;
+
+		tile->mem.memtest_bo = xe_bo_create_pin_map_at_novm(xe, tile, SZ_64K,
+								    vram_size - SZ_64K,
+								    ttm_bo_type_kernel,
+								    XE_BO_FLAG_VRAM_IF_DGFX(tile),
+								    0, false);
+		if (IS_ERR(tile->mem.memtest_bo)) {
+			drm_warn(&xe->drm, "Tile %d: Failed to reserve memtest BO\n", id);
+			tile->mem.memtest_bo = NULL;
+			continue;
+		}
+
+		drm_info(&xe->drm, "Tile %d: Reserved memtest BO at offset 0x%llx\n",
+			 id, vram_size - SZ_64K);
+	}
+
+	return devm_add_action_or_reset(xe->drm.dev, memtest_bo_cleanup, xe);
+}
+
+void xe_vram_free_memtest_bos(struct xe_device *xe)
+{
+	struct xe_tile *tile;
+	u8 id;
+
+	for_each_tile(tile, xe, id) {
+		if (tile->mem.memtest_bo) {
+			xe_bo_unpin_map_no_vm(tile->mem.memtest_bo);
+			tile->mem.memtest_bo = NULL;
+		}
+	}
+}
+
+int xe_vram_memtest(struct xe_device *xe)
+{
+	struct xe_tile *tile;
+	u8 id;
+	int err = 0;
+
+	if (IS_SRIOV_VF(xe))
+		return 0;
+
+	for_each_tile(tile, xe, id) {
+		struct xe_bo *last_page_bo = tile->mem.memtest_bo;
+		struct dma_fence *fence;
+		bool overlap = false;
+		int i;
+		u8 val;
+
+		if (!last_page_bo || !tile->migrate)
+			continue;
+
+		drm_info(&xe->drm, "Tile %d: Running VRAM memtest...\n", id);
+
+		/* CPU write and readback first and last byte of the last page */
+		xe_map_wr(xe, &last_page_bo->vmap, 0, u8, 0xA5);
+		xe_map_wr(xe, &last_page_bo->vmap, SZ_64K - 1, u8, 0x5A);
+
+		val = xe_map_rd(xe, &last_page_bo->vmap, 0, u8);
+		if (drm_WARN(&xe->drm, val != 0xA5,
+			     "Tile %d: CPU memtest failed at offset 0 (expected 0xA5, got 0x%02x)\n",
+			     id, val)) {
+			err = -EIO;
+			goto unpin;
+		}
+
+		val = xe_map_rd(xe, &last_page_bo->vmap, SZ_64K - 1, u8);
+		if (drm_WARN(&xe->drm, val != 0x5A,
+			     "Tile %d: CPU memtest failed at offset 65535 (expected 0x5A, got 0x%02x)\n",
+			     id, val)) {
+			err = -EIO;
+			goto unpin;
+		}
+
+		/* Non-CCS access via GPU on the last page */
+		xe_bo_lock(last_page_bo, false);
+		fence = xe_migrate_clear(tile->migrate, last_page_bo,
+					 last_page_bo->ttm.resource,
+					 XE_MIGRATE_CLEAR_FLAG_BO_DATA);
+		xe_bo_unlock(last_page_bo);
+
+		if (!IS_ERR(fence)) {
+			dma_fence_wait(fence, false);
+			dma_fence_put(fence);
+		} else {
+			err = PTR_ERR(fence);
+			goto unpin;
+		}
+
+		val = xe_map_rd(xe, &last_page_bo->vmap, 0, u8);
+		if (drm_WARN(&xe->drm, val != 0x00,
+			     "Tile %d: GPU memtest clear failed at offset 0 (expected 0x00, got 0x%02x)\n",
+			     id, val)) {
+			err = -EIO;
+			goto unpin;
+		}
+
+		/*
+		 * Check for CCS overlap on the root tile.
+		 *
+		 * TODO: maybe extend if we ever get multi-tile + CCS. Pay
+		 * special attention to the l2 flush below. Currently that is
+		 * hard coded to the root tile.
+		 */
+		if (!id && xe_device_has_flat_ccs(xe) &&
+		    GRAPHICS_VERx100(xe) >= 2000) {
+			struct xe_bo *scratch_bo_before;
+			struct xe_bo *scratch_bo_after;
+
+			scratch_bo_before = xe_bo_create_pin_map_novm(xe, tile, SZ_64K,
+								      ttm_bo_type_kernel,
+								      XE_BO_FLAG_VRAM_IF_DGFX(tile),
+								      false);
+			if (IS_ERR(scratch_bo_before)) {
+				err = PTR_ERR(scratch_bo_before);
+				goto unpin;
+			}
+
+			scratch_bo_after = xe_bo_create_pin_map_novm(xe, tile, SZ_64K,
+								     ttm_bo_type_kernel,
+								     XE_BO_FLAG_VRAM_IF_DGFX(tile),
+								     false);
+			if (IS_ERR(scratch_bo_after)) {
+				xe_bo_unpin_map_no_vm(scratch_bo_before);
+				err = PTR_ERR(scratch_bo_after);
+				goto unpin;
+			}
+
+			/* Save original CCS metadata for PA 0 + */
+			err = xe_migrate_debug_ccs_overlap(tile->migrate, scratch_bo_before, false);
+			if (err) {
+				xe_bo_unpin_map_no_vm(scratch_bo_before);
+				xe_bo_unpin_map_no_vm(scratch_bo_after);
+				goto unpin;
+			}
+
+			/*
+			 * Fill last page. If there is CCS overlap in the last
+			 * page this will snag the raw CCS storage.
+			 */
+			xe_map_memset(xe, &last_page_bo->vmap, 0, 0x5A, SZ_64K);
+			xe_device_wmb(xe);
+
+			/*
+			 * Global invalidation. Some BMG SKUs will cache the BAR
+			 * writes in the GPU side VRAM cache. Make sure above
+			 * writes are fully flushed out to VRAM, so this is
+			 * hopefully more well behaved with the CCS unit, if
+			 * there is indeed CCS overlap with normal VRAM. Since
+			 * there is a separate CCS cache, the CCS unit might not
+			 * respect the GPU VRAM cache for CCS accesses, so opt
+			 * for being super careful here.
+			 */
+			xe_device_l2_flush(xe, true);
+
+			/* Use GPU to clear CCS state for PA 0 */
+			xe_map_memset(xe, &scratch_bo_after->vmap, 0, 0x00, SZ_64K);
+			err = xe_migrate_debug_ccs_overlap(tile->migrate, scratch_bo_after, true);
+			if (err) {
+				xe_bo_unpin_map_no_vm(scratch_bo_before);
+				xe_bo_unpin_map_no_vm(scratch_bo_after);
+				goto unpin;
+			}
+			/*
+			 * Global invalidation. Ensure CCS caches really are
+			 * nuked and the raw CCS data is visible in VRAM, for
+			 * the below access.
+			 */
+			xe_device_l2_flush(xe, true);
+
+			/* Check if last_page_bo was corrupted by the GPU CCS clear */
+			for (i = 0; i < SZ_64K; i += 8) {
+				u64 payload = xe_map_rd(xe, &last_page_bo->vmap, i, u64);
+
+				if (payload != 0x5A5A5A5A5A5A5A5AULL) {
+					overlap = true;
+					break;
+				}
+			}
+
+			/* Restore original CCS metadata for PA 0 + */
+			err = xe_migrate_debug_ccs_overlap(tile->migrate, scratch_bo_before, true);
+			if (err)
+				drm_warn(&xe->drm, "Failed to restore CCS metadata\n");
+
+			xe_bo_unpin_map_no_vm(scratch_bo_before);
+			xe_bo_unpin_map_no_vm(scratch_bo_after);
+		}
+
+		if (drm_WARN(&xe->drm, overlap,
+			     "Tile %d: VRAM bounds overlap CCS region! VRAM sizing is incorrect.\n",
+			     id)) {
+			err = -EINVAL;
+			goto unpin;
+		}
+
+		drm_info(&xe->drm, "Tile %d: VRAM memtest completed.\n", id);
+
+unpin:
+		if (err)
+			break;
+	}
+
+	xe_vram_free_memtest_bos(xe);
+
+	return err;
+}
+#endif
diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h
index dd1c8bf1792234..38425d81f777f5 100644
--- a/drivers/gpu/drm/xe/xe_vram.h
+++ b/drivers/gpu/drm/xe/xe_vram.h
@@ -23,4 +23,14 @@ resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram);
 resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram);
 resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram);
 
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM)
+int xe_vram_reserve_memtest_bo(struct xe_device *xe);
+void xe_vram_free_memtest_bos(struct xe_device *xe);
+int xe_vram_memtest(struct xe_device *xe);
+#else
+static inline int xe_vram_reserve_memtest_bo(struct xe_device *xe) { return 0; }
+static inline void xe_vram_free_memtest_bos(struct xe_device *xe) {}
+static inline int xe_vram_memtest(struct xe_device *xe) { return 0; }
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index 139434946f8f82..c94fe7753f9a93 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -343,7 +343,9 @@ static const struct xe_rtp_table_sr engine_was = XE_RTP_TABLE_SR(
 	  XE_RTP_ACTIONS(SET(SAMPLER_MODE, ENABLE_SMALLPL))
 	},
 	{ XE_RTP_NAME("FtrPerCtxtPreemptionGranularityControl"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1250), ENGINE_CLASS(RENDER)),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1250), ENGINE_CLASS(RENDER), OR,
+		       GRAPHICS_VERSION_RANGE(2000, XE_RTP_END_VERSION_UNDEFINED),
+		       ENGINE_CLASS(RENDER)),
 	  XE_RTP_ACTIONS(SET(FF_SLICE_CS_CHICKEN1(RENDER_RING_BASE),
 			     FFSC_PERCTX_PREEMPT_CTRL))
 	},
diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules
index dd69ad07f7a9a7..f35181fc4e49b6 100644
--- a/drivers/gpu/drm/xe/xe_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_wa_oob.rules
@@ -14,6 +14,7 @@
 16017236439	PLATFORM(PVC)
 14019821291	MEDIA_VERSION_RANGE(1300, 2000)
 14015076503	MEDIA_VERSION(1300)
+22016122933	MEDIA_VERSION(1300)
 14018913170	GRAPHICS_VERSION_RANGE(1270, 1274)
 		MEDIA_VERSION(1300)
 		PLATFORM(DG2)
@@ -72,3 +73,6 @@
 16029897822	MEDIA_VERSION(3500)
 		GRAPHICS_VERSION(3510)
 14027054324	GRAPHICS_VERSION(3511)
+14025941587	GRAPHICS_VERSION_RANGE(2001, 3511), FUNC(xe_rtp_match_not_sriov_vf)
+		MEDIA_VERSION_RANGE(1301, 3503), FUNC(xe_rtp_match_not_sriov_vf)
+16030224309	GRAPHICS_VERSION(3510)
diff --git a/drivers/gpu/drm/xen/xen_drm_front.h b/drivers/gpu/drm/xen/xen_drm_front.h
index a987c78abe412b..02138b5abef512 100644
--- a/drivers/gpu/drm/xen/xen_drm_front.h
+++ b/drivers/gpu/drm/xen/xen_drm_front.h
@@ -14,7 +14,7 @@
 #include <linux/scatterlist.h>
 
 #include <drm/drm_connector.h>
-#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_encoder.h>
 
 #include "xen_drm_front_cfg.h"
 
@@ -100,7 +100,9 @@ struct xen_drm_front_drm_pipeline {
 
 	int index;
 
-	struct drm_simple_display_pipe pipe;
+	struct drm_plane plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 
 	struct drm_connector conn;
 	/* These are only for connector mode checking */
diff --git a/drivers/gpu/drm/xen/xen_drm_front_kms.c b/drivers/gpu/drm/xen/xen_drm_front_kms.c
index 48772b5fe71cf8..d193f60fad2218 100644
--- a/drivers/gpu/drm/xen/xen_drm_front_kms.c
+++ b/drivers/gpu/drm/xen/xen_drm_front_kms.c
@@ -30,10 +30,16 @@
  */
 #define FRAME_DONE_TO_MS	(XEN_DRM_FRONT_WAIT_BACK_MS + 100)
 
-static struct xen_drm_front_drm_pipeline *
-to_xen_drm_pipeline(struct drm_simple_display_pipe *pipe)
+static inline struct xen_drm_front_drm_pipeline *
+xen_drm_pipeline_from_crtc(struct drm_crtc *crtc)
 {
-	return container_of(pipe, struct xen_drm_front_drm_pipeline, pipe);
+	return container_of(crtc, struct xen_drm_front_drm_pipeline, crtc);
+}
+
+static inline struct xen_drm_front_drm_pipeline *
+xen_drm_pipeline_from_plane(struct drm_plane *plane)
+{
+	return container_of(plane, struct xen_drm_front_drm_pipeline, plane);
 }
 
 static void fb_destroy(struct drm_framebuffer *fb)
@@ -94,7 +100,7 @@ static const struct drm_mode_config_funcs mode_config_funcs = {
 
 static void send_pending_event(struct xen_drm_front_drm_pipeline *pipeline)
 {
-	struct drm_crtc *crtc = &pipeline->pipe.crtc;
+	struct drm_crtc *crtc = &pipeline->crtc;
 	struct drm_device *dev = crtc->dev;
 	unsigned long flags;
 
@@ -105,24 +111,21 @@ static void send_pending_event(struct xen_drm_front_drm_pipeline *pipeline)
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 }
 
-static void display_enable(struct drm_simple_display_pipe *pipe,
-			   struct drm_crtc_state *crtc_state,
-			   struct drm_plane_state *plane_state)
+static void xen_drm_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+					      struct drm_atomic_commit *commit)
 {
-	struct xen_drm_front_drm_pipeline *pipeline =
-			to_xen_drm_pipeline(pipe);
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_framebuffer *fb = plane_state->fb;
+	struct xen_drm_front_drm_pipeline *pipeline = xen_drm_pipeline_from_crtc(crtc);
+	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(commit, &pipeline->plane);
+	struct drm_framebuffer *fb = pstate->fb;
 	int ret, idx;
 
-	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+	if (!drm_dev_enter(crtc->dev, &idx))
 		return;
 
-	ret = xen_drm_front_mode_set(pipeline, crtc->x, crtc->y,
+	ret = xen_drm_front_mode_set(pipeline, pstate->crtc_x, pstate->crtc_y,
 				     fb->width, fb->height,
 				     fb->format->cpp[0] * 8,
 				     xen_drm_front_fb_to_cookie(fb));
-
 	if (ret) {
 		DRM_ERROR("Failed to enable display: %d\n", ret);
 		pipeline->conn_connected = false;
@@ -131,13 +134,13 @@ static void display_enable(struct drm_simple_display_pipe *pipe,
 	drm_dev_exit(idx);
 }
 
-static void display_disable(struct drm_simple_display_pipe *pipe)
+static void xen_drm_crtc_helper_atomic_disable(struct drm_crtc *crtc,
+					       struct drm_atomic_commit *commit)
 {
-	struct xen_drm_front_drm_pipeline *pipeline =
-			to_xen_drm_pipeline(pipe);
+	struct xen_drm_front_drm_pipeline *pipeline = xen_drm_pipeline_from_crtc(crtc);
 	int ret = 0, idx;
 
-	if (drm_dev_enter(pipe->crtc.dev, &idx)) {
+	if (drm_dev_enter(crtc->dev, &idx)) {
 		ret = xen_drm_front_mode_set(pipeline, 0, 0, 0, 0, 0,
 					     xen_drm_front_fb_to_cookie(NULL));
 		drm_dev_exit(idx);
@@ -177,12 +180,13 @@ static void pflip_to_worker(struct work_struct *work)
 	send_pending_event(pipeline);
 }
 
-static bool display_send_page_flip(struct drm_simple_display_pipe *pipe,
+static bool display_send_page_flip(struct xen_drm_front_drm_pipeline *pipeline,
+				   struct drm_atomic_commit *commit,
 				   struct drm_plane_state *old_plane_state)
 {
 	struct drm_plane_state *plane_state =
-			drm_atomic_get_new_plane_state(old_plane_state->state,
-						       &pipe->plane);
+			drm_atomic_get_new_plane_state(commit,
+						       &pipeline->plane);
 
 	/*
 	 * If old_plane_state->fb is NULL and plane_state->fb is not,
@@ -193,8 +197,6 @@ static bool display_send_page_flip(struct drm_simple_display_pipe *pipe,
 	 * sent to the backend as a part of display_set_config call.
 	 */
 	if (old_plane_state->fb && plane_state->fb) {
-		struct xen_drm_front_drm_pipeline *pipeline =
-				to_xen_drm_pipeline(pipe);
 		struct xen_drm_front_drm_info *drm_info = pipeline->drm_info;
 		int ret;
 
@@ -224,10 +226,29 @@ static bool display_send_page_flip(struct drm_simple_display_pipe *pipe,
 	return false;
 }
 
-static int display_check(struct drm_simple_display_pipe *pipe,
-			 struct drm_plane_state *plane_state,
-			 struct drm_crtc_state *crtc_state)
+static int xen_drm_plane_helper_atomic_check(struct drm_plane *plane,
+					     struct drm_atomic_commit *commit)
 {
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(commit, plane);
+	struct drm_crtc_state *crtc_state = NULL;
+	int ret;
+
+	if (plane_state->crtc) {
+		crtc_state = drm_atomic_get_crtc_state(commit, plane_state->crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+	}
+
+	ret = drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, false);
+	if (ret)
+		return ret;
+
+	if (!plane_state->visible || !crtc_state)
+		return 0;
+
 	/*
 	 * Xen doesn't initialize vblanking via drm_vblank_init(), so
 	 * DRM helpers assume that it doesn't handle vblanking and start
@@ -242,16 +263,17 @@ static int display_check(struct drm_simple_display_pipe *pipe,
 	return 0;
 }
 
-static void display_update(struct drm_simple_display_pipe *pipe,
-			   struct drm_plane_state *old_plane_state)
+static void xen_drm_plane_helper_atomic_update(struct drm_plane *plane,
+					       struct drm_atomic_commit *commit)
 {
-	struct xen_drm_front_drm_pipeline *pipeline =
-			to_xen_drm_pipeline(pipe);
-	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(commit, plane);
+	struct xen_drm_front_drm_pipeline *pipeline = xen_drm_pipeline_from_plane(plane);
+	struct drm_crtc *crtc = &pipeline->crtc;
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
 	struct drm_pending_vblank_event *event;
 	int idx;
 
-	event = crtc->state->event;
+	event = crtc_state ? crtc_state->event : NULL;
 	if (event) {
 		struct drm_device *dev = crtc->dev;
 		unsigned long flags;
@@ -259,13 +281,13 @@ static void display_update(struct drm_simple_display_pipe *pipe,
 		WARN_ON(pipeline->pending_event);
 
 		spin_lock_irqsave(&dev->event_lock, flags);
-		crtc->state->event = NULL;
+		crtc_state->event = NULL;
 
 		pipeline->pending_event = event;
 		spin_unlock_irqrestore(&dev->event_lock, flags);
 	}
 
-	if (!drm_dev_enter(pipe->crtc.dev, &idx)) {
+	if (!drm_dev_enter(crtc->dev, &idx)) {
 		send_pending_event(pipeline);
 		return;
 	}
@@ -278,19 +300,17 @@ static void display_update(struct drm_simple_display_pipe *pipe,
 	 * If this is not a page flip, e.g. no flip done event from the backend
 	 * is expected, then send now.
 	 */
-	if (!display_send_page_flip(pipe, old_plane_state))
+	if (!display_send_page_flip(pipeline, commit, old_plane_state))
 		send_pending_event(pipeline);
 
 	drm_dev_exit(idx);
 }
 
 static enum drm_mode_status
-display_mode_valid(struct drm_simple_display_pipe *pipe,
-		   const struct drm_display_mode *mode)
+xen_drm_crtc_helper_mode_valid(struct drm_crtc *crtc,
+			       const struct drm_display_mode *mode)
 {
-	struct xen_drm_front_drm_pipeline *pipeline =
-			container_of(pipe, struct xen_drm_front_drm_pipeline,
-				     pipe);
+	struct xen_drm_front_drm_pipeline *pipeline = xen_drm_pipeline_from_crtc(crtc);
 
 	if (mode->hdisplay != pipeline->width)
 		return MODE_ERROR;
@@ -301,12 +321,53 @@ display_mode_valid(struct drm_simple_display_pipe *pipe,
 	return MODE_OK;
 }
 
-static const struct drm_simple_display_pipe_funcs display_funcs = {
-	.mode_valid = display_mode_valid,
-	.enable = display_enable,
-	.disable = display_disable,
-	.check = display_check,
-	.update = display_update,
+static int xen_drm_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit)
+{
+	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(commit, crtc);
+	int ret;
+
+	if (crtc_state->enable) {
+		ret = drm_atomic_helper_check_crtc_primary_plane(crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return drm_atomic_add_affected_planes(commit, crtc);
+}
+
+static const struct drm_plane_helper_funcs display_plane_helper_funcs = {
+	.prepare_fb	= drm_gem_plane_helper_prepare_fb,
+	.atomic_check	= xen_drm_plane_helper_atomic_check,
+	.atomic_update	= xen_drm_plane_helper_atomic_update,
+};
+
+static const struct drm_plane_funcs display_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.destroy		= drm_plane_cleanup,
+	.atomic_create_state	= drm_atomic_helper_plane_create_state,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+};
+
+static const struct drm_crtc_helper_funcs display_crtc_helper_funcs = {
+	.mode_valid	= xen_drm_crtc_helper_mode_valid,
+	.atomic_check	= xen_drm_crtc_helper_atomic_check,
+	.atomic_enable	= xen_drm_crtc_helper_atomic_enable,
+	.atomic_disable	= xen_drm_crtc_helper_atomic_disable,
+};
+
+static const struct drm_crtc_funcs display_crtc_funcs = {
+	.atomic_create_state	= drm_atomic_helper_crtc_create_state,
+	.destroy		= drm_crtc_cleanup,
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_encoder_funcs display_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
 static int display_pipe_init(struct xen_drm_front_drm_info *drm_info,
@@ -331,10 +392,37 @@ static int display_pipe_init(struct xen_drm_front_drm_info *drm_info,
 
 	formats = xen_drm_front_conn_get_formats(&format_count);
 
-	return drm_simple_display_pipe_init(dev, &pipeline->pipe,
-					    &display_funcs, formats,
-					    format_count, NULL,
-					    &pipeline->conn);
+	ret = drm_universal_plane_init(dev, &pipeline->plane, 0,
+				       &display_plane_funcs,
+				       formats, format_count,
+				       NULL,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret)
+		return ret;
+
+	drm_plane_helper_add(&pipeline->plane, &display_plane_helper_funcs);
+
+	ret = drm_crtc_init_with_planes(dev, &pipeline->crtc,
+					&pipeline->plane, NULL,
+					&display_crtc_funcs, NULL);
+	if (ret)
+		return ret;
+
+	drm_crtc_helper_add(&pipeline->crtc, &display_crtc_helper_funcs);
+
+	ret = drm_encoder_init(dev, &pipeline->encoder,
+			       &display_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret)
+		return ret;
+
+	pipeline->encoder.possible_crtcs = drm_crtc_mask(&pipeline->crtc);
+
+	ret = drm_connector_attach_encoder(&pipeline->conn, &pipeline->encoder);
+	if (ret)
+		return ret;
+
+	return 0;
 }
 
 int xen_drm_front_kms_init(struct xen_drm_front_drm_info *drm_info)
diff --git a/drivers/gpu/drm/xlnx/zynqmp_kms.c b/drivers/gpu/drm/xlnx/zynqmp_kms.c
index d5f92245056554..5274bd3f907889 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_kms.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_kms.c
@@ -29,8 +29,8 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_mode_config.h>
 #include <drm/drm_plane.h>
+#include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
 
 #include <linux/clk.h>
@@ -141,7 +141,7 @@ static const struct drm_plane_funcs zynqmp_dpsub_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
 	.destroy		= drm_plane_cleanup,
-	.reset			= drm_atomic_helper_plane_reset,
+	.atomic_create_state = drm_atomic_helper_plane_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
 };
@@ -308,7 +308,7 @@ static const struct drm_crtc_funcs zynqmp_dpsub_crtc_funcs = {
 	.destroy		= drm_crtc_cleanup,
 	.set_config		= drm_atomic_helper_set_config,
 	.page_flip		= drm_atomic_helper_page_flip,
-	.reset			= drm_atomic_helper_crtc_reset,
+	.atomic_create_state = drm_atomic_helper_crtc_create_state,
 	.atomic_duplicate_state	= drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_crtc_destroy_state,
 	.enable_vblank		= zynqmp_dpsub_crtc_enable_vblank,
@@ -417,6 +417,10 @@ static const struct drm_driver zynqmp_dpsub_drm_driver = {
 	.minor				= 0,
 };
 
+static const struct drm_encoder_funcs zynqmp_dpsub_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
 static int zynqmp_dpsub_kms_init(struct zynqmp_dpsub *dpsub)
 {
 	struct drm_encoder *encoder = &dpsub->drm->encoder;
@@ -436,7 +440,13 @@ static int zynqmp_dpsub_kms_init(struct zynqmp_dpsub *dpsub)
 
 	/* Create the encoder and attach the bridge. */
 	encoder->possible_crtcs |= drm_crtc_mask(&dpsub->drm->crtc);
-	drm_simple_encoder_init(&dpsub->drm->dev, encoder, DRM_MODE_ENCODER_NONE);
+	ret = drm_encoder_init(&dpsub->drm->dev, encoder,
+			       &zynqmp_dpsub_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret) {
+		drm_err(&dpsub->drm->dev, "failed to initialize encoder\n");
+		return ret;
+	}
 
 	ret = drm_bridge_attach(encoder, dpsub->bridge, NULL,
 				DRM_BRIDGE_ATTACH_NO_CONNECTOR);
diff --git a/drivers/gpu/ipu-v3/ipu-image-convert.c b/drivers/gpu/ipu-v3/ipu-image-convert.c
index 29b6d36c9bb674..493be4d4b091e3 100644
--- a/drivers/gpu/ipu-v3/ipu-image-convert.c
+++ b/drivers/gpu/ipu-v3/ipu-image-convert.c
@@ -371,7 +371,7 @@ static int alloc_dma_buf(struct ipu_image_convert_priv *priv,
 {
 	buf->len = PAGE_ALIGN(size);
 	buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys,
-				       GFP_DMA | GFP_KERNEL);
+				       GFP_KERNEL);
 	if (!buf->virt) {
 		dev_err(priv->ipu->dev, "failed to alloc dma buffer\n");
 		return -ENOMEM;
diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig
index f918f69e059968..1934f17baa8b6a 100644
--- a/drivers/gpu/nova-core/Kconfig
+++ b/drivers/gpu/nova-core/Kconfig
@@ -5,6 +5,7 @@ config NOVA_CORE
 	depends on RUST
 	depends on !CPU_BIG_ENDIAN
 	select AUXILIARY_BUS
+	select GPU_BUDDY
 	select RUST_FW_LOADER_ABSTRACTIONS
 	default n
 	help
@@ -15,3 +16,12 @@ config NOVA_CORE
 	  This driver is work in progress and may not be functional.
 
 	  If M is selected, the module will be called nova-core.
+
+config NOVA_CORE_SELFTESTS
+	bool "Nova Core driver self-tests"
+	depends on NOVA_CORE
+	default n
+	help
+	  Build the driver self-tests and run them when the GPU is probed.
+
+	  If unsure, say N.
diff --git a/drivers/gpu/nova-core/api.rs b/drivers/gpu/nova-core/api.rs
new file mode 100644
index 00000000000000..f02c6c7c7e51df
--- /dev/null
+++ b/drivers/gpu/nova-core/api.rs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Nova-core auxbus data. Contains all the methods used by the auxbus drivers
+//! to interact with nova-core.
+
+use core::pin::Pin;
+
+use kernel::{
+    auxiliary,
+    device::Bound,
+    pci,
+    prelude::*,
+    types::ForLt, //
+};
+
+pub use crate::gpu::Spec;
+
+use crate::gpu::Gpu;
+use crate::gsp::commands::GetGspStaticInfoReply;
+
+/// API handle for the auxiliary bus child drivers to interact with nova-core.
+pub struct NovaCoreApi<'bound> {
+    pub(crate) gpu: Pin<&'bound Gpu<'bound>>,
+    pub(crate) pdev: &'bound pci::Device<Bound>,
+}
+
+impl NovaCoreApi<'_> {
+    /// Obtain a [`NovaCoreApi`] handle from an auxiliary device registered
+    /// by nova-core.
+    pub fn of(adev: &auxiliary::Device<Bound>) -> Result<NovaCoreApiHandle<'_>> {
+        NovaCoreApiHandle::of(adev)
+    }
+
+    /// Returns the GPU [`GetGspStaticInfoReply`].
+    pub fn gsp_static_info(&self) -> &GetGspStaticInfoReply {
+        &self.gpu.gsp_static_info
+    }
+
+    /// Returns the GPU [`Spec`].
+    pub fn spec(&self) -> &Spec {
+        &self.gpu.spec
+    }
+
+    /// Returns the size of the PCIe BAR used for accessing VRAM, typically
+    /// BAR1.
+    pub fn bar1_size(&self) -> Result<u64> {
+        self.pdev.resource_len(1)
+    }
+}
+
+/// Expose a handle to nova-core API
+pub struct NovaCoreApiHandle<'a> {
+    adev: &'a auxiliary::Device<Bound>,
+}
+
+impl<'a> NovaCoreApiHandle<'a> {
+    fn of(adev: &'a auxiliary::Device<Bound>) -> Result<Self> {
+        adev.registration_data_with::<ForLt!(NovaCoreApi<'_>), ()>(|_| ())?;
+        Ok(Self { adev })
+    }
+
+    /// Access the [`NovaCoreApi`] through a closure.
+    pub fn with<R>(&self, f: impl for<'b> FnOnce(Pin<&'a NovaCoreApi<'b>>) -> R) -> R {
+        self.adev
+            .registration_data_with::<ForLt!(NovaCoreApi<'_>), R>(f)
+            .expect("TypeId was validated in NovaCoreApiHandle::of()")
+    }
+}
diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index bbd93959e0b2f3..291a047e4d8682 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -2,7 +2,11 @@
 
 use kernel::{
     auxiliary,
-    device::Core,
+    device::{
+        Bound,
+        Core, //
+    },
+    io::resource,
     pci,
     pci::{
         Class,
@@ -15,21 +19,25 @@ use kernel::{
         Atomic,
         Relaxed, //
     },
-    types::CovariantForLt,
+    types::ForLt,
 };
 
-use crate::gpu::Gpu;
+use crate::{
+    api::NovaCoreApi,
+    gpu::Gpu, //
+};
 
 /// Counter for generating unique auxiliary device IDs.
 static AUXILIARY_ID_COUNTER: Atomic<u32> = Atomic::new(0);
 
 #[pin_data]
 pub(crate) struct NovaCore<'bound> {
+    #[allow(clippy::type_complexity)]
+    _reg: auxiliary::Registration<'bound, ForLt!(NovaCoreApi<'_>)>,
     #[pin]
     pub(crate) gpu: Gpu<'bound>,
     bar: pci::Bar<'bound, BAR0_SIZE>,
-    #[allow(clippy::type_complexity)]
-    _reg: auxiliary::Registration<'bound, CovariantForLt!(())>,
+    bar1: pci::Bar<'bound>,
 }
 
 pub(crate) struct NovaCoreDriver;
@@ -37,6 +45,27 @@ pub(crate) struct NovaCoreDriver;
 const BAR0_SIZE: usize = SZ_16M;
 
 pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>;
+pub(crate) type NovaRegisters = kernel::io::Region<BAR0_SIZE>;
+pub(crate) type Bar1<'a> = &'a pci::Bar<'a>;
+
+/// Returns the Linux PCI resource index that holds BAR1 for an NVIDIA GPU.
+///
+/// On Maxwell through Ada, BAR0 is a 32-bit memory BAR occupying a single
+/// Linux PCI resource slot, so BAR1 lives at index 1. Starting with Blackwell
+/// (and on some Ampere GA100 / Hopper SKUs) BAR0 is a 64-bit memory BAR that
+/// consumes two consecutive resource slots: index 0 holds the low 32 bits and
+/// index 1 holds the high 32 bits (with no `flags` / or size of its own),
+/// shifting BAR1 to index 2.
+pub(crate) fn bar1_resource_index(pdev: &pci::Device<Bound>) -> Result<u32> {
+    // Probe the `IORESOURCE_MEM_64` flag of BAR0 as a robust way of exposing
+    // if BAR0 and hence BAR1 is 64-bit.
+    let flags0 = pdev.resource_flags(0)?;
+    if flags0.contains(resource::Flags::IORESOURCE_MEM_64) {
+        Ok(2)
+    } else {
+        Ok(1)
+    }
+}
 
 kernel::pci_device_table!(
     PCI_TABLE,
@@ -71,30 +100,58 @@ impl pci::Driver for NovaCoreDriver {
         pdev: &'bound pci::Device<Core<'_>>,
         _info: Option<&'bound Self::IdInfo>,
     ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
-        pin_init::pin_init_scope(move || {
-            dev_dbg!(pdev, "Probe Nova Core GPU driver.\n");
+        try_pin_init!(NovaCore {
+            _: {
+                pdev.enable_device_mem()?;
+                pdev.set_master();
+            },
+
+            bar: pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0")?,
 
-            pdev.enable_device_mem()?;
-            pdev.set_master();
+            bar1: {
+                let bar1_idx = bar1_resource_index(pdev)?;
+                pdev.iomap_region(bar1_idx, c"nova-core/bar1")?
+            },
 
-            Ok(try_pin_init!(NovaCore {
-                bar: pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0")?,
-                // TODO: Use `&bar` self-referential pin-init syntax once available.
+            // TODO: Use self-referential pin-init syntax once available.
+            gpu <- Gpu::new(
+                pdev,
+                // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`.
+                unsafe { &*core::ptr::from_ref(bar) },
+                // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`.
+                unsafe { &*core::ptr::from_ref(bar1) },
+            ).pin_chain(|_gpu| {
+                #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+                _gpu.run_selftests(pdev);
+                Ok(())
+            }),
+
+            _reg: {
+                // TODO: Use `&gpu` self-referential pin-init syntax once available.
                 //
-                // SAFETY: `bar` is initialized before this expression is evaluated
-                // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned
-                // stable address, and is dropped after `gpu` (struct field drop order).
-                gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }),
-                _reg: auxiliary::Registration::new(
-                    pdev.as_ref(),
-                    c"nova-drm",
-                    // TODO[XARR]: Use XArray or perhaps IDA for proper ID allocation/recycling. For
-                    // now, use a simple atomic counter that never recycles IDs.
-                    AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed),
-                    crate::MODULE_NAME,
-                    (),
-                )?,
-            }))
+                // SAFETY: `gpu` is initialized before this expression is evaluated
+                // (`try_pin_init!()` initializes fields in initializer order), lives at
+                // a pinned stable address, and is dropped after `_reg` (struct field
+                // drop order).
+                let gpu = unsafe {
+                    Pin::new_unchecked(&*core::ptr::from_ref(gpu.as_ref().get_ref()))
+                };
+
+                // SAFETY: `NovaCore` is dropped when the device is unbound;
+                // i.e. `mem::forget()` is never called on it.
+                unsafe {
+                    auxiliary::Registration::new_with_lt(
+                        pdev.as_ref(),
+                        c"nova-drm",
+                        // TODO[XARR]: Use XArray or perhaps IDA for proper ID
+                        // allocation/recycling. For now, use a simple atomic counter that
+                        // never recycles IDs.
+                        AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed),
+                        crate::MODULE_NAME,
+                        NovaCoreApi { gpu, pdev },
+                    )?
+                }
+            },
         })
     }
 }
diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs
index 65cb12d26e2b94..9015de965a5320 100644
--- a/drivers/gpu/nova-core/falcon.rs
+++ b/drivers/gpu/nova-core/falcon.rs
@@ -14,13 +14,12 @@ use kernel::{
     io::{
         io_project,
         poll::read_poll_timeout,
-        register::{
-            RegisterBase,
-            WithBase, //
-        },
+        register::Array,
         Io,
+        Mmio, //
     },
     prelude::*,
+    sizes::SZ_4K,
     time::Delta,
 };
 
@@ -165,18 +164,25 @@ bounded_enum! {
     }
 }
 
-/// Type used to represent the `PFALCON` registers address base for a given falcon engine.
-pub(crate) struct PFalconBase(());
+const PFALCON_REGION_SIZE: usize = SZ_4K;
+const PFALCON2_REGION_SIZE: usize = SZ_4K;
 
-/// Type used to represent the `PFALCON2` registers address base for a given falcon engine.
-pub(crate) struct PFalcon2Base(());
+/// Type used to represent the `PFALCON` registers.
+#[repr(align(4))]
+#[derive(FromBytes, IntoBytes)]
+pub(crate) struct PFalconRegisters([u8; PFALCON_REGION_SIZE]);
+
+/// Type used to represent the `PFALCON2` registers.
+#[repr(align(4))]
+#[derive(FromBytes, IntoBytes)]
+pub(crate) struct PFalcon2Registers([u8; PFALCON2_REGION_SIZE]);
 
 /// Trait defining the parameters of a given Falcon engine.
 ///
 /// Each engine provides one base for `PFALCON` and `PFALCON2` registers.
-pub(crate) trait FalconEngine:
-    Send + Sync + RegisterBase<PFalconBase> + RegisterBase<PFalcon2Base> + Sized
-{
+pub(crate) trait FalconEngine: Send + Sync + Sized {
+    fn pfalcon(io: Bar0<'_>) -> Mmio<'_, PFalconRegisters>;
+    fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, PFalcon2Registers>;
 }
 
 /// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM)
@@ -358,6 +364,9 @@ pub(crate) struct Falcon<'a, E: FalconEngine> {
     hal: KBox<dyn FalconHal<E>>,
     dev: &'a device::Device<device::Bound>,
     bar: Bar0<'a>,
+    // TODO: make private
+    pub(crate) pfalcon: Mmio<'a, PFalconRegisters>,
+    pfalcon2: Mmio<'a, PFalcon2Registers>,
 }
 
 impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
@@ -371,19 +380,19 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
             hal: hal::falcon_hal(chipset)?,
             dev,
             bar,
+            pfalcon: E::pfalcon(bar),
+            pfalcon2: E::pfalcon2(bar),
         })
     }
 
     /// Resets DMA-related registers.
     pub(crate) fn dma_reset(&self) {
-        self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
+        self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| {
             v.with_allow_phys_no_ctx(true)
         });
 
-        self.bar.write(
-            WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_DMACTL::zeroed(),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed());
     }
 
     /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete.
@@ -392,10 +401,10 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
         self.hal.select_core(self)?;
         self.hal.reset_wait_mem_scrubbing(self)?;
 
-        self.bar.write(
-            WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_RM::from(crate::gpu::boot_0_raw(
+                self.bar,
+            )));
 
         Ok(())
     }
@@ -413,8 +422,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
             return Err(EINVAL);
         }
 
-        self.bar.write(
-            WithBase::of::<E>().at(Self::PIO_PORT),
+        self.pfalcon.write(
+            Array::at(Self::PIO_PORT),
             regs::NV_PFALCON_FALCON_IMEMC::zeroed()
                 .with_secure(load_offsets.secure)
                 .with_aincw(true)
@@ -424,14 +433,14 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
         for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() {
             let n = u16::try_from(n)?;
             let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?;
-            self.bar.write(
-                WithBase::of::<E>().at(Self::PIO_PORT),
+            self.pfalcon.write(
+                Array::at(Self::PIO_PORT),
                 regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag),
             );
             for word in block.chunks_exact(4) {
                 let w = [word[0], word[1], word[2], word[3]];
-                self.bar.write(
-                    WithBase::of::<E>().at(Self::PIO_PORT),
+                self.pfalcon.write(
+                    Array::at(Self::PIO_PORT),
                     regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)),
                 );
             }
@@ -450,8 +459,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
             return Err(EINVAL);
         }
 
-        self.bar.write(
-            WithBase::of::<E>().at(Self::PIO_PORT),
+        self.pfalcon.write(
+            Array::at(Self::PIO_PORT),
             regs::NV_PFALCON_FALCON_DMEMC::zeroed()
                 .with_aincw(true)
                 .with_offs(load_offsets.dst_start),
@@ -459,8 +468,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
 
         for word in load_offsets.data.chunks_exact(4) {
             let w = [word[0], word[1], word[2], word[3]];
-            self.bar.write(
-                WithBase::of::<E>().at(Self::PIO_PORT),
+            self.pfalcon.write(
+                Array::at(Self::PIO_PORT),
                 regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)),
             );
         }
@@ -473,14 +482,12 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
         &self,
         fw: &F,
     ) -> Result {
-        self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| {
+        self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| {
             v.with_allow_phys_no_ctx(true)
         });
 
-        self.bar.write(
-            WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_DMACTL::zeroed(),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed());
 
         if let Some(imem_ns) = fw.imem_ns_load_params() {
             self.pio_wr_imem_slice(imem_ns)?;
@@ -492,10 +499,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
 
         self.hal.program_brom(self, &fw.brom_params());
 
-        self.bar.write(
-            WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()));
 
         Ok(())
     }
@@ -506,7 +511,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
     /// `sec` is set if the loaded firmware is expected to run in secure mode.
     fn dma_wr(
         &self,
-        dma_obj: &Coherent<[u8]>,
+        dma_obj: &Coherent<'_, [u8]>,
         target_mem: FalconMem,
         load_offsets: FalconDmaLoadTarget,
     ) -> Result {
@@ -547,16 +552,13 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
 
         // Set up the base source DMA address.
 
-        self.bar.write(
-            WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base(
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base(
                 // CAST: `as u32` is used on purpose since we do want to strip the upper bits,
                 // which will be written to `NV_PFALCON_FALCON_DMATRFBASE1`.
                 (dma_address >> 8) as u32,
-            ),
-        );
-        self.bar.write(
-            WithBase::of::<E>(),
+            ));
+        self.pfalcon.write_reg(
             regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_address >> 40)?,
         );
 
@@ -566,23 +568,21 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
 
         for pos in (0..num_transfers).map(|i| i * DMA_LEN) {
             // Perform a transfer of size `DMA_LEN`.
-            self.bar.write(
-                WithBase::of::<E>(),
+            self.pfalcon.write_reg(
                 regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed()
                     .try_with_offs(load_offsets.dst_start + pos)?,
             );
-            self.bar.write(
-                WithBase::of::<E>(),
+            self.pfalcon.write_reg(
                 regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos),
             );
 
-            self.bar.write(WithBase::of::<E>(), cmd);
+            self.pfalcon.write_reg(cmd);
 
             // Wait for the transfer to complete.
             // TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories
             // should ever take that long.
             read_poll_timeout(
-                || Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::<E>())),
+                || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_DMATRFCMD)),
                 |r| r.idle(),
                 Delta::ZERO,
                 Delta::from_secs(2),
@@ -614,8 +614,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
         };
 
         self.dma_reset();
-        self.bar
-            .update(regs::NV_PFALCON_FBIF_TRANSCFG::of::<E>().at(0), |v| {
+        self.pfalcon
+            .update(regs::NV_PFALCON_FBIF_TRANSCFG::at(0), |v| {
                 v.with_target(FalconFbifTarget::CoherentSysmem)
                     .with_mem_type(FalconFbifMemType::Physical)
             });
@@ -626,10 +626,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
         self.hal.program_brom(self, &fw.brom_params());
 
         // Set `BootVec` to start of non-secure code.
-        self.bar.write(
-            WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()));
 
         Ok(())
     }
@@ -638,7 +636,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
     pub(crate) fn wait_till_halted(&self) -> Result<()> {
         // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds.
         read_poll_timeout(
-            || Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())),
+            || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL)),
             |r| r.halted(),
             Delta::ZERO,
             Delta::from_secs(2),
@@ -649,19 +647,13 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
 
     /// Start the falcon CPU.
     pub(crate) fn start(&self) -> Result<()> {
-        match self
-            .bar
-            .read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())
-            .alias_en()
-        {
-            true => self.bar.write(
-                WithBase::of::<E>(),
-                regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true),
-            ),
-            false => self.bar.write(
-                WithBase::of::<E>(),
-                regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true),
-            ),
+        match self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL).alias_en() {
+            true => self
+                .pfalcon
+                .write_reg(regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true)),
+            false => self
+                .pfalcon
+                .write_reg(regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true)),
         }
 
         Ok(())
@@ -670,32 +662,24 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
     /// Writes values to the mailbox registers if provided.
     pub(crate) fn write_mailboxes(&self, mbox0: Option<u32>, mbox1: Option<u32>) {
         if let Some(mbox0) = mbox0 {
-            self.bar.write(
-                WithBase::of::<E>(),
-                regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0),
-            );
+            self.pfalcon
+                .write_reg(regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0));
         }
 
         if let Some(mbox1) = mbox1 {
-            self.bar.write(
-                WithBase::of::<E>(),
-                regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1),
-            );
+            self.pfalcon
+                .write_reg(regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1));
         }
     }
 
     /// Reads the value from `mbox0` register.
     pub(crate) fn read_mailbox0(&self) -> u32 {
-        self.bar
-            .read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>())
-            .value()
+        self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX0).value()
     }
 
     /// Reads the value from `mbox1` register.
     pub(crate) fn read_mailbox1(&self) -> u32 {
-        self.bar
-            .read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>())
-            .value()
+        self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX1).value()
     }
 
     /// Reads values from both mailbox registers.
@@ -760,9 +744,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
 
     /// Write the application version to the OS register.
     pub(crate) fn write_os_version(&self, app_version: u32) {
-        self.bar.write(
-            WithBase::of::<E>(),
-            regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version));
     }
 }
diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs
index 0437180b882959..85f9c8c5d60e8c 100644
--- a/drivers/gpu/nova-core/falcon/fsp.rs
+++ b/drivers/gpu/nova-core/falcon/fsp.rs
@@ -8,13 +8,12 @@
 
 use kernel::{
     io::{
+        io_project,
         poll::read_poll_timeout,
-        register::{
-            Array,
-            RegisterBase,
-            WithBase, //
-        },
-        Io, //
+        register,
+        register::Array,
+        Io,
+        Mmio, //
     },
     prelude::*,
     sizes::SZ_1K,
@@ -22,11 +21,13 @@ use kernel::{
 };
 
 use crate::{
+    driver::{
+        Bar0,
+        NovaRegisters, //
+    },
     falcon::{
         Falcon,
-        FalconEngine,
-        PFalcon2Base,
-        PFalconBase, //
+        FalconEngine, //
     },
     num,
     regs, //
@@ -41,15 +42,24 @@ const FSP_EMEM_CHANNEL_0_SIZE: usize = SZ_1K;
 /// Type specifying the `Fsp` falcon engine. Cannot be instantiated.
 pub(crate) struct Fsp(());
 
-impl RegisterBase<PFalconBase> for Fsp {
-    const BASE: usize = 0x8f2000;
-}
+register! {
+    base: NovaRegisters;
 
-impl RegisterBase<PFalcon2Base> for Fsp {
-    const BASE: usize = 0x8f3000;
+    PFALCON: super::PFalconRegisters @ 0x8f2000;
+    PFALCON2: super::PFalcon2Registers @ 0x8f3000;
 }
 
-impl FalconEngine for Fsp {}
+impl FalconEngine for Fsp {
+    #[inline]
+    fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> {
+        io_project!(io, build: PFALCON)
+    }
+
+    #[inline]
+    fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> {
+        io_project!(io, build: PFALCON2)
+    }
+}
 
 impl<'a> Falcon<'a, Fsp> {
     /// Writes `data` to FSP external memory at offset `0`.
@@ -62,19 +72,15 @@ impl<'a> Falcon<'a, Fsp> {
         }
 
         // Begin a write burst at offset `0`, auto-incrementing on each write.
-        self.bar.write(
-            WithBase::of::<Fsp>(),
-            regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true));
 
         for chunk in data.chunks_exact(4) {
             let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
 
             // Write the next 32-bit `value`; hardware advances the offset.
-            self.bar.write(
-                WithBase::of::<Fsp>(),
-                regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value),
-            );
+            self.pfalcon
+                .write_reg(regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value));
         }
 
         Ok(())
@@ -90,17 +96,12 @@ impl<'a> Falcon<'a, Fsp> {
         }
 
         // Begin a read burst at offset `0`, auto-incrementing on each read.
-        self.bar.write(
-            WithBase::of::<Fsp>(),
-            regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true));
 
         for chunk in data.chunks_exact_mut(4) {
             // Read the next 32-bit word; hardware advances the offset.
-            let value = self
-                .bar
-                .read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>())
-                .data();
+            let value = self.pfalcon.read(regs::NV_PFALCON_FALCON_EMEMD).data();
             chunk.copy_from_slice(&value.to_le_bytes());
         }
 
diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs
index ae32f401aeb03e..4c96ae325fda26 100644
--- a/drivers/gpu/nova-core/falcon/gsp.rs
+++ b/drivers/gpu/nova-core/falcon/gsp.rs
@@ -2,23 +2,24 @@
 
 use kernel::{
     io::{
+        io_project,
         poll::read_poll_timeout,
-        register::{
-            RegisterBase,
-            WithBase, //
-        },
+        register,
         Io,
+        Mmio, //
     },
     prelude::*,
     time::Delta, //
 };
 
 use crate::{
+    driver::{
+        Bar0,
+        NovaRegisters, //
+    },
     falcon::{
         Falcon,
-        FalconEngine,
-        PFalcon2Base,
-        PFalconBase, //
+        FalconEngine, //
     },
     regs,
 };
@@ -26,24 +27,31 @@ use crate::{
 /// Type specifying the `Gsp` falcon engine. Cannot be instantiated.
 pub(crate) struct Gsp(());
 
-impl RegisterBase<PFalconBase> for Gsp {
-    const BASE: usize = 0x00110000;
-}
+register! {
+    base: NovaRegisters;
 
-impl RegisterBase<PFalcon2Base> for Gsp {
-    const BASE: usize = 0x00111000;
+    PFALCON: super::PFalconRegisters @ 0x00110000;
+    PFALCON2: super::PFalcon2Registers @ 0x00111000;
 }
 
-impl FalconEngine for Gsp {}
+impl FalconEngine for Gsp {
+    #[inline]
+    fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> {
+        io_project!(io, build: PFALCON)
+    }
+
+    #[inline]
+    fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> {
+        io_project!(io, build: PFALCON2)
+    }
+}
 
 impl<'a> Falcon<'a, Gsp> {
     /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to
     /// allow GSP to signal CPU for processing new messages in message queue.
     pub(crate) fn clear_swgen0_intr(&self) {
-        self.bar.write(
-            WithBase::of::<Gsp>(),
-            regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true),
-        );
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true));
     }
 
     /// Checks if GSP reload/resume has completed during the boot process.
@@ -59,8 +67,8 @@ impl<'a> Falcon<'a, Gsp> {
 
     /// Returns whether the RISC-V branch privilege lockdown bit is set.
     pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool {
-        self.bar
-            .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>())
+        self.pfalcon
+            .read(regs::NV_PFALCON_FALCON_HWCFG2)
             .riscv_br_priv_lockdown()
     }
 
@@ -71,10 +79,7 @@ impl<'a> Falcon<'a, Gsp> {
         const LOCKED_PATTERN: u32 = 0xbadf_4100;
         const LOCKED_MASK: u32 = 0xffff_ff00;
 
-        let hwcfg2 = self
-            .bar
-            .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>())
-            .into_raw();
+        let hwcfg2 = self.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2).into_raw();
 
         hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN
     }
diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs
index 7600ee07ca2ebc..f9a8444cf84094 100644
--- a/drivers/gpu/nova-core/falcon/hal/ga102.rs
+++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs
@@ -6,11 +6,9 @@ use kernel::{
     device,
     io::{
         poll::read_poll_timeout,
-        register::{
-            Array,
-            WithBase, //
-        },
-        Io, //
+        register::Array,
+        Io,
+        Mmio, //
     },
     prelude::*,
     time::Delta, //
@@ -24,6 +22,7 @@ use crate::{
         FalconBromParams,
         FalconEngine,
         FalconModSelAlgo,
+        PFalcon2Registers,
         PeregrineCoreSelect, //
     },
     regs,
@@ -31,17 +30,16 @@ use crate::{
 
 use super::FalconHal;
 
-fn select_core_ga102<E: FalconEngine>(bar: Bar0<'_>) -> Result {
-    let bcr_ctrl = bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::<E>());
+fn select_core_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>) -> Result {
+    let bcr_ctrl = pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL);
     if bcr_ctrl.core_select() != PeregrineCoreSelect::Falcon {
-        bar.write(
-            WithBase::of::<E>(),
+        pfalcon2.write_reg(
             regs::NV_PRISCV_RISCV_BCR_CTRL::zeroed().with_core_select(PeregrineCoreSelect::Falcon),
         );
 
         // TIMEOUT: falcon core should take less than 10ms to report being enabled.
         read_poll_timeout(
-            || Ok(bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::<E>())),
+            || Ok(pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL)),
             |r| r.valid(),
             Delta::ZERO,
             Delta::from_millis(10),
@@ -86,24 +84,20 @@ fn signature_reg_fuse_version_ga102(
     Ok(u16::BITS - reg_fuse_version.leading_zeros())
 }
 
-fn program_brom_ga102<E: FalconEngine>(bar: Bar0<'_>, params: &FalconBromParams) {
-    bar.write(
-        WithBase::of::<E>().at(0),
+fn program_brom_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>, params: &FalconBromParams) {
+    pfalcon2.write(
+        Array::at(0),
         regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset),
     );
-    bar.write(
-        WithBase::of::<E>(),
+    pfalcon2.write_reg(
         regs::NV_PFALCON2_FALCON_BROM_ENGIDMASK::zeroed()
             .with_value(u32::from(params.engine_id_mask)),
     );
-    bar.write(
-        WithBase::of::<E>(),
+    pfalcon2.write_reg(
         regs::NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID::zeroed().with_ucode_id(params.ucode_id),
     );
-    bar.write(
-        WithBase::of::<E>(),
-        regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k),
-    );
+    pfalcon2
+        .write_reg(regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k));
 }
 
 pub(super) struct Ga102<E: FalconEngine>(PhantomData<E>);
@@ -116,7 +110,7 @@ impl<E: FalconEngine> Ga102<E> {
 
 impl<E: FalconEngine> FalconHal<E> for Ga102<E> {
     fn select_core(&self, falcon: &Falcon<'_, E>) -> Result {
-        select_core_ga102::<E>(falcon.bar)
+        select_core_ga102(falcon.pfalcon2)
     }
 
     fn signature_reg_fuse_version(
@@ -129,27 +123,24 @@ impl<E: FalconEngine> FalconHal<E> for Ga102<E> {
     }
 
     fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) {
-        program_brom_ga102::<E>(falcon.bar, params);
+        program_brom_ga102(falcon.pfalcon2, params);
     }
 
     fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool {
         falcon
-            .bar
-            .read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>())
+            .pfalcon2
+            .read(regs::NV_PRISCV_RISCV_CPUCTL)
             .active_stat()
     }
 
     fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result<bool> {
-        Ok(falcon
-            .bar
-            .read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>())
-            .halted())
+        Ok(falcon.pfalcon2.read(regs::NV_PRISCV_RISCV_CPUCTL).halted())
     }
 
     fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
         // TIMEOUT: memory scrubbing should complete in less than 20ms.
         read_poll_timeout(
-            || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())),
+            || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)),
             |r| r.mem_scrubbing_done(),
             Delta::ZERO,
             Delta::from_millis(20),
@@ -158,20 +149,18 @@ impl<E: FalconEngine> FalconHal<E> for Ga102<E> {
     }
 
     fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result {
-        let bar = falcon.bar;
-
-        let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>());
+        let _ = falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2);
 
         // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set
         // RESET_READY so a non-failing timeout is used.
         let _ = read_poll_timeout(
-            || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())),
+            || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)),
             |r| r.reset_ready(),
             Delta::ZERO,
             Delta::from_micros(150),
         );
 
-        regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar);
+        regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon);
         self.reset_wait_mem_scrubbing(falcon)?;
 
         Ok(())
diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs
index 5291598fedf783..7fc6e83c2566d5 100644
--- a/drivers/gpu/nova-core/falcon/hal/tu102.rs
+++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs
@@ -5,7 +5,6 @@ use core::marker::PhantomData;
 use kernel::{
     io::{
         poll::read_poll_timeout,
-        register::WithBase,
         Io, //
     },
     prelude::*,
@@ -50,8 +49,8 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> {
 
     fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool {
         falcon
-            .bar
-            .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>())
+            .pfalcon2
+            .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS)
             .active_stat()
     }
 
@@ -62,7 +61,7 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> {
     fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result {
         // TIMEOUT: memory scrubbing should complete in less than 10ms.
         read_poll_timeout(
-            || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())),
+            || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_DMACTL)),
             |r| r.mem_scrubbing_done(),
             Delta::ZERO,
             Delta::from_millis(10),
@@ -71,7 +70,7 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> {
     }
 
     fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result {
-        regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(falcon.bar);
+        regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon);
         self.reset_wait_mem_scrubbing(falcon)?;
 
         Ok(())
diff --git a/drivers/gpu/nova-core/falcon/sec2.rs b/drivers/gpu/nova-core/falcon/sec2.rs
index 91ec7d49c1f576..6648a397d38a7d 100644
--- a/drivers/gpu/nova-core/falcon/sec2.rs
+++ b/drivers/gpu/nova-core/falcon/sec2.rs
@@ -1,22 +1,37 @@
 // SPDX-License-Identifier: GPL-2.0
 
-use kernel::io::register::RegisterBase;
+use kernel::io::{
+    io_project,
+    register,
+    Mmio, //
+};
 
-use crate::falcon::{
-    FalconEngine,
-    PFalcon2Base,
-    PFalconBase, //
+use crate::{
+    driver::{
+        Bar0,
+        NovaRegisters, //
+    },
+    falcon::FalconEngine, //
 };
 
 /// Type specifying the `Sec2` falcon engine. Cannot be instantiated.
 pub(crate) struct Sec2(());
 
-impl RegisterBase<PFalconBase> for Sec2 {
-    const BASE: usize = 0x00840000;
-}
+register! {
+    base: NovaRegisters;
 
-impl RegisterBase<PFalcon2Base> for Sec2 {
-    const BASE: usize = 0x00841000;
+    PFALCON: super::PFalconRegisters @ 0x00840000;
+    PFALCON2: super::PFalcon2Registers @ 0x00841000;
 }
 
-impl FalconEngine for Sec2 {}
+impl FalconEngine for Sec2 {
+    #[inline]
+    fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> {
+        io_project!(io, build: PFALCON)
+    }
+
+    #[inline]
+    fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> {
+        io_project!(io, build: PFALCON2)
+    }
+}
diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs
index 1576399389b192..b0c0b8fe60085c 100644
--- a/drivers/gpu/nova-core/fb.rs
+++ b/drivers/gpu/nova-core/fb.rs
@@ -49,7 +49,7 @@ pub(crate) struct SysmemFlush<'sys> {
     device: &'sys device::Device,
     bar: Bar0<'sys>,
     /// Keep the page alive as long as we need it.
-    page: CoherentHandle,
+    page: CoherentHandle<'sys>,
 }
 
 impl<'sys> SysmemFlush<'sys> {
@@ -159,7 +159,7 @@ pub(crate) struct FbRanges {
     /// Memory area containing the GSP bootloader image.
     pub(crate) boot: FbRange,
     /// Memory area containing the GSP firmware image.
-    pub(crate) elf: FbRange,
+    pub(crate) fw_image: FbRange,
     /// WPR2 heap.
     pub(crate) wpr2_heap: FbRange,
     /// WPR2 region range, starting with an instance of `GspFwWprMeta`.
@@ -177,7 +177,7 @@ impl FbRanges {
     pub(crate) fn new(
         chipset: Chipset,
         bar: Bar0<'_>,
-        gsp_fw: &GspFirmware,
+        gsp_fw: &GspFirmware<'_>,
         vgpu_state: VgpuState,
     ) -> Result<Self> {
         let hal = hal::fb_hal(chipset);
@@ -234,25 +234,25 @@ impl FbRanges {
             FbRange(bootloader_base..bootloader_base + bootloader_size)
         };
 
-        let elf = {
-            const ELF_DOWN_ALIGN: Alignment = Alignment::new::<SZ_64K>();
-            let elf_size = u64::from_safe_cast(gsp_fw.size);
-            let elf_addr = (boot.start - elf_size).align_down(ELF_DOWN_ALIGN);
+        let fw_image = {
+            const FW_IMAGE_DOWN_ALIGN: Alignment = Alignment::new::<SZ_64K>();
+            let fw_image_size = u64::from_safe_cast(gsp_fw.size());
+            let fw_image_addr = (boot.start - fw_image_size).align_down(FW_IMAGE_DOWN_ALIGN);
 
-            FbRange(elf_addr..elf_addr + elf_size)
+            FbRange(fw_image_addr..fw_image_addr + fw_image_size)
         };
 
         let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb.end)?;
 
         let wpr2_heap = {
             const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>();
-            let wpr2_heap_addr = elf
+            let wpr2_heap_addr = fw_image
                 .start
                 .checked_sub(wpr2_heap_size)
                 .ok_or(EOVERFLOW)?
                 .align_down(WPR2_HEAP_DOWN_ALIGN);
 
-            FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN))
+            FbRange(wpr2_heap_addr..(fw_image.start).align_down(WPR2_HEAP_DOWN_ALIGN))
         };
 
         let wpr2 = {
@@ -273,7 +273,7 @@ impl FbRanges {
             vga_workspace,
             frts,
             boot,
-            elf,
+            fw_image,
             wpr2_heap,
             wpr2,
             non_wpr_heap,
diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs
index d9e4d62ae63218..9fa0949396009b 100644
--- a/drivers/gpu/nova-core/fb/hal/gb100.rs
+++ b/drivers/gpu/nova-core/fb/hal/gb100.rs
@@ -5,11 +5,10 @@
 
 use kernel::{
     io::{
-        register::{
-            RegisterBase,
-            WithBase, //
-        },
-        Io, //
+        io_project,
+        register,
+        Io,
+        Mmio, //
     },
     num::Bounded,
     prelude::*,
@@ -21,7 +20,10 @@ use kernel::{
 };
 
 use crate::{
-    driver::Bar0,
+    driver::{
+        Bar0,
+        NovaRegisters, //
+    },
     fb::{
         hal::FbHal,
         regs, //
@@ -31,17 +33,26 @@ use crate::{
 
 struct Gb100;
 
-impl RegisterBase<regs::Hshub0Base> for Gb100 {
-    const BASE: usize = 0x0087_0000;
+register! {
+    base: NovaRegisters;
+
+    HSHUB0: regs::Hshub0Registers @ 0x0087_0000;
+}
+
+#[inline]
+fn hshub0(bar: Bar0<'_>) -> Mmio<'_, regs::Hshub0Registers> {
+    io_project!(bar, build: HSHUB0)
 }
 
-fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 {
+fn read_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>) -> u64 {
     let lo = u64::from(
-        bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>())
+        hshub0
+            .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO)
             .adr(),
     );
     let hi = u64::from(
-        bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>())
+        hshub0
+            .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI)
             .adr(),
     );
 
@@ -52,7 +63,7 @@ fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 {
 ///
 /// Both the primary and EG (egress) register pairs must be programmed to the same address,
 /// as required by hardware.
-fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) {
+fn write_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>, addr: Bounded<u64, 52>) {
     // CAST: lower 32 bits. Hardware ignores bits 7:0.
     let addr_lo = *addr as u32;
     let addr_hi = addr.shr::<32, 20>().cast::<u32>();
@@ -60,24 +71,12 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) {
     // Write HI first. The hardware will trigger the flush on the LO write.
 
     // Primary HSHUB pair.
-    bar.write(
-        regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(),
-        regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi),
-    );
-    bar.write(
-        regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(),
-        regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo),
-    );
+    hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi));
+    hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo));
 
     // EG (egress) pair -- must match the primary pair.
-    bar.write(
-        regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(),
-        regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi),
-    );
-    bar.write(
-        regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(),
-        regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo),
-    );
+    hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi));
+    hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo));
 }
 
 // This PMU reservation size is r570-specific.
@@ -88,13 +87,13 @@ pub(super) const fn pmu_reserved_size_gb100() -> u32 {
 
 impl FbHal for Gb100 {
     fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 {
-        read_sysmem_flush_page_gb100(bar)
+        read_sysmem_flush_page_gb100(hshub0(bar))
     }
 
     fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result {
         let addr = Bounded::<u64, 52>::try_new(addr).ok_or(EINVAL)?;
 
-        write_sysmem_flush_page_gb100(bar, addr);
+        write_sysmem_flush_page_gb100(hshub0(bar), addr);
 
         Ok(())
     }
diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs
index 95adbe124a300d..131787996a249b 100644
--- a/drivers/gpu/nova-core/fb/regs.rs
+++ b/drivers/gpu/nova-core/fb/regs.rs
@@ -2,12 +2,20 @@
 
 use kernel::{
     io::register,
-    sizes::SizeConstants, //
+    prelude::*,
+    sizes::{
+        SizeConstants,
+        SZ_4K, //
+    }, //
 };
 
+use crate::driver::NovaRegisters;
+
 // PDISP
 
 register! {
+    base: NovaRegisters;
+
     pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 {
         /// VGA workspace base address divided by 0x10000.
         31:8    addr;
@@ -30,6 +38,8 @@ impl NV_PDISP_VGA_WORKSPACE_BASE {
 // PFB
 
 register! {
+    base: NovaRegisters;
+
     /// Low bits of the physical system memory address used by the GPU to perform sysmembar
     /// operations (see [`crate::fb::SysmemFlush`]).
     pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 {
@@ -59,34 +69,42 @@ register! {
     }
 }
 
-/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM).
+const HSHUB0_REGION_SIZE: usize = SZ_4K;
+
+/// The GB10x HSHUB0 register window (Base defined as `NV_HSHUB0_PRIV_BASE` in Open RM).
 ///
 /// The base is provided by the GB10x framebuffer HAL.
-pub(super) struct Hshub0Base(());
+#[repr(align(4))]
+#[derive(FromBytes, IntoBytes)]
+pub(super) struct Hshub0Registers([u8; HSHUB0_REGION_SIZE]);
 
 register! {
+    base: Hshub0Registers;
+
     // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar
     // through a primary and an EG (egress) pair that must both be programmed to the same
     // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed
     // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here.
-    pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 {
+    pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00000e50 {
         31:0    adr => u32;
     }
 
-    pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 {
+    pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00000e54 {
         19:0    adr;
     }
 
-    pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 {
+    pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x000006c0 {
         31:0    adr => u32;
     }
 
-    pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 {
+    pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x000006c4 {
         19:0    adr;
     }
 }
 
 register! {
+    base: NovaRegisters;
+
     // GB20x FBHUB0 sysmem flush registers. Unlike the older
     // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an
     // 8-bit right-shift, these take the raw address split into lower and upper
@@ -101,6 +119,8 @@ register! {
 }
 
 register! {
+    base: NovaRegisters;
+
     /// Low bits of the physical system memory address used by the GPU to perform
     /// sysmembar operations on Hopper.
     ///
diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs
index b49613a90bf069..d8f6509a35d178 100644
--- a/drivers/gpu/nova-core/firmware.rs
+++ b/drivers/gpu/nova-core/firmware.rs
@@ -23,9 +23,10 @@ use crate::{
 };
 
 pub(crate) mod booter;
-pub(crate) mod fsp;
 pub(crate) mod fwsec;
 pub(crate) mod gsp;
+pub(crate) mod gsp_fmc;
+pub(crate) mod radix3;
 pub(crate) mod riscv;
 pub(crate) mod tlv;
 
diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs
index dc071edba3315f..aa4458bb33127d 100644
--- a/drivers/gpu/nova-core/firmware/booter.rs
+++ b/drivers/gpu/nova-core/firmware/booter.rs
@@ -186,7 +186,7 @@ impl BooterFirmware {
         &self,
         dev: &device::Device<device::Bound>,
         sec2_falcon: &Falcon<'_, Sec2>,
-        wpr_meta: &Coherent<T>,
+        wpr_meta: &Coherent<'_, T>,
     ) -> Result {
         sec2_falcon.reset()?;
         sec2_falcon.load(self)?;
diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs
deleted file mode 100644
index 5462e318410af1..00000000000000
--- a/drivers/gpu/nova-core/firmware/fsp.rs
+++ /dev/null
@@ -1,122 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-
-//! FSP is a hardware unit that runs FMC firmware.
-
-use kernel::{
-    device,
-    dma::Coherent,
-    prelude::*, //
-};
-
-use crate::{
-    firmware::tlv::{
-        request_tlv, //
-        Tlv,
-    },
-    gpu::Chipset, //
-};
-
-/// Size of the FSP SHA-384 hash, in bytes.
-const FSP_HASH_SIZE: usize = 48;
-/// Maximum size of the FSP public key (RSA-3072), in bytes.
-///
-/// The FMC `PKEY` tag may be shorter, so the remaining bytes are zero-padded.
-const FSP_PKEY_SIZE: usize = 384;
-/// Maximum size of the FSP signature (RSA-3072), in bytes.
-///
-/// The FMC `SIGN` tag may be shorter, so the remaining bytes are zero-padded.
-const FSP_SIG_SIZE: usize = 384;
-
-/// Structure to hold FMC signatures.
-///
-/// C representation is used because this type is used for communication with the FSP.
-#[derive(Debug, Clone, Copy, Zeroable)]
-#[repr(C)]
-pub(crate) struct FmcSignatures {
-    pub(crate) hash384: [u8; FSP_HASH_SIZE],
-    pub(crate) public_key: [u8; FSP_PKEY_SIZE],
-    pub(crate) signature: [u8; FSP_SIG_SIZE],
-}
-
-pub(crate) struct FspFirmware {
-    /// FMC firmware image data
-    pub(crate) fmc_image: Coherent<[u8]>,
-    /// FMC firmware signatures.
-    pub(crate) fmc_sigs: KBox<FmcSignatures>,
-}
-
-impl FspFirmware {
-    pub(crate) fn new(dev: &device::Device<device::Bound>, chipset: Chipset) -> Result<Self> {
-        let fw = request_tlv(dev, chipset, "fmc")?;
-        let tlv = Tlv::new(fw.data())?;
-        dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?);
-
-        let fmc_image_data = tlv.get_bytes(b"BLOB")?;
-        let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?;
-
-        Ok(Self {
-            fmc_image,
-            fmc_sigs: Self::extract_fmc_signatures(&tlv, dev)?,
-        })
-    }
-
-    /// Extract FMC firmware signatures for Chain of Trust verification.
-    ///
-    /// Extracts real cryptographic signatures from FMC TLV firmware tags.
-    /// Returns signatures in a heap-allocated structure to prevent stack overflow.
-    fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result<KBox<FmcSignatures>> {
-        let hash_section = tlv.get_bytes(b"HASH")?;
-        let pkey_section = tlv.get_bytes(b"PKEY")?;
-        let sig_section = tlv.get_bytes(b"SIGN")?;
-
-        // The hash section is a SHA-384 output: it must be exactly FSP_HASH_SIZE bytes.
-        if hash_section.len() != FSP_HASH_SIZE {
-            dev_err!(
-                dev,
-                "FMC hash section size {} != expected {}\n",
-                hash_section.len(),
-                FSP_HASH_SIZE
-            );
-            return Err(EINVAL);
-        }
-
-        // The key and signature sections are zero-padded to a fixed maximum, so they may be
-        // shorter, but must not exceed the destination buffers.
-        if pkey_section.len() > FSP_PKEY_SIZE {
-            dev_err!(
-                dev,
-                "FMC public key section size {} > maximum {}\n",
-                pkey_section.len(),
-                FSP_PKEY_SIZE
-            );
-            return Err(EINVAL);
-        }
-        if sig_section.len() > FSP_SIG_SIZE {
-            dev_err!(
-                dev,
-                "FMC signature section size {} > maximum {}\n",
-                sig_section.len(),
-                FSP_SIG_SIZE
-            );
-            return Err(EINVAL);
-        }
-
-        // Initialize the signatures in place to avoid building the large `FmcSignatures` on the
-        // stack, then fill each section from the firmware.
-        let signatures = KBox::init(
-            pin_init::init_zeroed::<FmcSignatures>().chain(|sigs| {
-                // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above).
-                sigs.hash384.copy_from_slice(hash_section);
-                // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE (verified above).
-                sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section);
-                // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE (verified above).
-                sigs.signature[..sig_section.len()].copy_from_slice(sig_section);
-                Ok(())
-            }),
-            GFP_KERNEL,
-        )?;
-
-        Ok(signatures)
-    }
-}
diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
index ec4d92317a93f7..a87878fe2aec75 100644
--- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
+++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs
@@ -12,7 +12,10 @@ use kernel::{
         Device, //
     },
     dma::Coherent,
-    io::{register::WithBase, Io},
+    io::{
+        register::Array,
+        Io, //
+    },
     prelude::*,
     ptr::{
         Alignable,
@@ -23,7 +26,6 @@ use kernel::{
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         self,
         gsp::Gsp,
@@ -98,9 +100,9 @@ unsafe impl AsBytes for BootloaderDmemDescV2 {}
 
 /// Wrapper for [`FwsecFirmware`] that includes the bootloader performing the actual load
 /// operation.
-pub(crate) struct FwsecFirmwareWithBl {
+pub(crate) struct FwsecFirmwareWithBl<'a> {
     /// DMA object the bootloader will copy the firmware from.
-    _firmware_dma: Coherent<[u8]>,
+    _firmware_dma: Coherent<'a, [u8]>,
     /// Code of the bootloader to be loaded into non-secure IMEM.
     ucode: KVec<u8>,
     /// Descriptor to be loaded into DMEM for the bootloader to read.
@@ -113,12 +115,12 @@ pub(crate) struct FwsecFirmwareWithBl {
     start_tag: u16,
 }
 
-impl FwsecFirmwareWithBl {
+impl<'a> FwsecFirmwareWithBl<'a> {
     /// Loads the bootloader firmware for `dev` and `chipset`, and wrap `firmware` so it can be
     /// loaded using it.
     pub(crate) fn new(
         firmware: FwsecFirmware,
-        dev: &Device<device::Bound>,
+        dev: &'a Device<device::Bound>,
         chipset: Chipset,
     ) -> Result<Self> {
         let fw = request_tlv(dev, chipset, "gen_bootloader")?;
@@ -235,12 +237,7 @@ impl FwsecFirmwareWithBl {
     ///
     /// The bootloader will load the FWSEC firmware and then execute it. This function returns
     /// after FWSEC has reached completion.
-    pub(crate) fn run(
-        &self,
-        dev: &Device<device::Bound>,
-        falcon: &Falcon<'_, Gsp>,
-        bar: Bar0<'_>,
-    ) -> Result<()> {
+    pub(crate) fn run(&self, dev: &Device<device::Bound>, falcon: &Falcon<'_, Gsp>) -> Result<()> {
         // Reset falcon, load the firmware, and run it.
         falcon
             .reset()
@@ -250,9 +247,8 @@ impl FwsecFirmwareWithBl {
             .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?;
 
         // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory.
-        bar.update(
-            regs::NV_PFALCON_FBIF_TRANSCFG::of::<Gsp>()
-                .try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma))
+        falcon.pfalcon.update(
+            regs::NV_PFALCON_FBIF_TRANSCFG::try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma))
                 .ok_or(EINVAL)?,
             |v| {
                 v.with_target(FalconFbifTarget::CoherentSysmem)
@@ -272,7 +268,7 @@ impl FwsecFirmwareWithBl {
     }
 }
 
-impl FalconFirmware for FwsecFirmwareWithBl {
+impl FalconFirmware for FwsecFirmwareWithBl<'_> {
     type Target = Gsp;
 
     fn brom_params(&self) -> FalconBromParams {
@@ -286,7 +282,7 @@ impl FalconFirmware for FwsecFirmwareWithBl {
     }
 }
 
-impl FalconPioLoadable for FwsecFirmwareWithBl {
+impl FalconPioLoadable for FwsecFirmwareWithBl<'_> {
     fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> {
         None
     }
diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs
index e8f9491e84cc70..341a8b19aa3802 100644
--- a/drivers/gpu/nova-core/firmware/gsp.rs
+++ b/drivers/gpu/nova-core/firmware/gsp.rs
@@ -4,21 +4,16 @@ use kernel::{
     device,
     dma::{
         Coherent,
-        CoherentBox,
-        DataDirection,
         DmaAddress, //
     },
     firmware,
     prelude::*,
-    scatterlist::{
-        Owned,
-        SGTable, //
-    },
     str::CString,
 };
 
 use crate::{
     firmware::{
+        radix3::Radix3,
         riscv::RiscvFirmware, //
         tlv::{
             request_tlv, //
@@ -26,48 +21,25 @@ use crate::{
         },
     },
     gpu::Chipset,
-    gsp::GSP_PAGE_SIZE,
     num::FromSafeCast,
 };
 
-/// GSP firmware with 3-level radix page tables for the GSP bootloader.
-///
-/// The bootloader expects firmware to be mapped starting at address 0 in GSP's virtual address
-/// space:
-///
-/// ```text
-/// Level 0:  1 page, 1 entry         -> points to first level 1 page
-/// Level 1:  Multiple pages/entries  -> each entry points to a level 2 page
-/// Level 2:  Multiple pages/entries  -> each entry points to a firmware page
-/// ```
-///
-/// Each page is 4KB, each entry is 8 bytes (64-bit DMA address).
-/// Also known as "Radix3" firmware.
+/// The GSP firmware image, its signatures, and the GSP bootloader.
 #[pin_data]
-pub(crate) struct GspFirmware {
-    /// The GSP firmware inside a [`VVec`], device-mapped via a SG table.
-    #[pin]
-    fw: SGTable<Owned<VVec<u8>>>,
-    /// Level 2 page table whose entries contain DMA addresses of firmware pages.
+pub(crate) struct GspFirmware<'a> {
+    /// The firmware image and the radix3 table that maps it.
     #[pin]
-    level2: SGTable<Owned<VVec<u8>>>,
-    /// Level 1 page table whose entries contain DMA addresses of level 2 pages.
-    #[pin]
-    level1: SGTable<Owned<VVec<u8>>>,
-    /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page.
-    level0: Coherent<[u64]>,
-    /// Size in bytes of the firmware contained in [`Self::fw`].
-    pub(crate) size: usize,
+    radix3: Radix3<'a>,
     /// Device-mapped GSP signatures matching the GPU's [`Chipset`].
-    pub(crate) signatures: Coherent<[u8]>,
+    pub(crate) signatures: Coherent<'a, [u8]>,
     /// GSP bootloader, verifies the GSP firmware before loading and running it.
-    pub(crate) bootloader: RiscvFirmware,
+    pub(crate) bootloader: RiscvFirmware<'a>,
 }
 
-impl GspFirmware {
+impl<'a> GspFirmware<'a> {
     /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page
     /// tables expected by the GSP bootloader to load it.
-    pub(crate) fn new<'a>(
+    pub(crate) fn new(
         dev: &'a device::Device<device::Bound>,
         chipset: Chipset,
     ) -> impl PinInit<Self, Error> + 'a {
@@ -87,49 +59,7 @@ impl GspFirmware {
             let signatures = Coherent::from_slice(dev, tlv.get_bytes(b"SIGN")?, GFP_KERNEL)?;
 
             Ok(try_pin_init!(Self {
-                fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL),
-                level2 <- {
-                    // Allocate the level 2 page table, map the firmware onto it, and map it into
-                    // the device address space.
-                    VVec::<u8>::with_capacity(
-                        fw.iter().count() * core::mem::size_of::<u64>(),
-                        GFP_KERNEL,
-                    )
-                    .map_err(|_| ENOMEM)
-                    .and_then(|level2| map_into_lvl(&fw, level2))
-                    .map(|level2| SGTable::new(dev, level2, DataDirection::ToDevice, GFP_KERNEL))?
-                },
-                level1 <- {
-                    // Allocate the level 1 page table, map the level 2 page table onto it, and map
-                    // it into the device address space.
-                    VVec::<u8>::with_capacity(
-                        level2.iter().count() * core::mem::size_of::<u64>(),
-                        GFP_KERNEL,
-                    )
-                    .map_err(|_| ENOMEM)
-                    .and_then(|level1| map_into_lvl(&level2, level1))
-                    .map(|level1| SGTable::new(dev, level1, DataDirection::ToDevice, GFP_KERNEL))?
-                },
-                level0: {
-                    // Allocate the level 0 page table as a device-visible DMA object, and map the
-                    // level 1 page table onto it.
-
-                    // Fill level 1 page entry.
-                    let level1_entry = level1.iter().next().ok_or(EINVAL)?;
-                    let level1_entry_addr = level1_entry.dma_address();
-
-                    // Create level 0 page table data and fill its first entry with the level 1
-                    // table.
-                    let mut level0 = CoherentBox::<[u64]>::zeroed_slice(
-                        dev,
-                        GSP_PAGE_SIZE / size_of::<u64>(),
-                        GFP_KERNEL
-                    )?;
-                    level0[0] = level1_entry_addr.to_le();
-
-                    level0.into()
-                },
-                size,
+                radix3 <- Radix3::new(dev, fw_vvec),
                 signatures,
                 bootloader: {
                     let bl = request_tlv(dev, chipset, "gsp_bootloader")?;
@@ -140,28 +70,13 @@ impl GspFirmware {
         })
     }
 
-    /// Returns the DMA address of the radix3 level 0 page table.
-    pub(crate) fn radix3_dma_address(&self) -> DmaAddress {
-        self.level0.dma_address()
+    /// Returns the size of the firmware image, in bytes.
+    pub(crate) fn size(&self) -> usize {
+        self.radix3.size()
     }
-}
 
-/// Build a page table from a scatter-gather list.
-///
-/// Takes each DMA-mapped region from `sg_table` and writes page table entries
-/// for all 4KB pages within that region. For example, a 16KB SG entry becomes
-/// 4 consecutive page table entries.
-fn map_into_lvl(sg_table: &SGTable<Owned<VVec<u8>>>, mut dst: VVec<u8>) -> Result<VVec<u8>> {
-    for sg_entry in sg_table.iter() {
-        // Number of pages we need to map.
-        let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE);
-
-        for i in 0..num_pages {
-            let entry = sg_entry.dma_address()
-                + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE));
-            dst.extend_from_slice(&entry.to_le_bytes(), GFP_KERNEL)?;
-        }
+    /// Returns the DMA address of the radix3 table that maps the firmware image.
+    pub(crate) fn radix3_dma_address(&self) -> DmaAddress {
+        self.radix3.dma_address()
     }
-
-    Ok(dst)
 }
diff --git a/drivers/gpu/nova-core/firmware/gsp_fmc.rs b/drivers/gpu/nova-core/firmware/gsp_fmc.rs
new file mode 100644
index 00000000000000..94fcc86c7dff1f
--- /dev/null
+++ b/drivers/gpu/nova-core/firmware/gsp_fmc.rs
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! GSP-FMC (First Mutable Code) is loaded by FSP into GSP to serve as the loader and verifier of
+//! GSP-RM.
+
+use kernel::{
+    device,
+    dma::Coherent,
+    prelude::*, //
+};
+
+use crate::{
+    firmware::tlv::{
+        request_tlv, //
+        Tlv,
+    },
+    gpu::Chipset, //
+};
+
+/// Size of the GSP-FMC SHA-384 hash, in bytes.
+const FMC_HASH_SIZE: usize = 48;
+/// Maximum size of the GSP-FMC public key (RSA-3072), in bytes.
+///
+/// The `PKEY` tag may be shorter, so the remaining bytes are zero-padded.
+const FMC_PKEY_SIZE: usize = 384;
+/// Maximum size of the GSP-FMC signature (RSA-3072), in bytes.
+///
+/// The `SIGN` tag may be shorter, so the remaining bytes are zero-padded.
+const FMC_SIG_SIZE: usize = 384;
+
+/// Structure to hold FMC signatures.
+///
+/// C representation is used because this type is used for communication with the FSP.
+#[derive(Debug, Clone, Copy, Zeroable)]
+#[repr(C)]
+pub(crate) struct FmcSignatures {
+    pub(crate) hash384: [u8; FMC_HASH_SIZE],
+    pub(crate) public_key: [u8; FMC_PKEY_SIZE],
+    pub(crate) signature: [u8; FMC_SIG_SIZE],
+}
+
+pub(crate) struct GspFmcFirmware<'a> {
+    /// FMC firmware image data
+    pub(crate) fmc_image: Coherent<'a, [u8]>,
+    /// FMC firmware signatures.
+    pub(crate) fmc_sigs: KBox<FmcSignatures>,
+}
+
+impl<'a> GspFmcFirmware<'a> {
+    pub(crate) fn new(dev: &'a device::Device<device::Bound>, chipset: Chipset) -> Result<Self> {
+        let fw = request_tlv(dev, chipset, "fmc")?;
+        let tlv = Tlv::new(fw.data())?;
+        dev_dbg!(
+            dev,
+            "loaded GSP-FMC firmware v{}\n",
+            tlv.get_string(b"VERS")?
+        );
+
+        let fmc_image_data = tlv.get_bytes(b"BLOB")?;
+        let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?;
+
+        Ok(Self {
+            fmc_image,
+            fmc_sigs: Self::extract_fmc_signatures(&tlv, dev)?,
+        })
+    }
+
+    /// Extract FMC firmware signatures for Chain of Trust verification.
+    ///
+    /// Extracts real cryptographic signatures from FMC TLV firmware tags.
+    /// Returns signatures in a heap-allocated structure to prevent stack overflow.
+    fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result<KBox<FmcSignatures>> {
+        let hash_section = tlv.get_bytes(b"HASH")?;
+        let pkey_section = tlv.get_bytes(b"PKEY")?;
+        let sig_section = tlv.get_bytes(b"SIGN")?;
+
+        // The hash section is a SHA-384 output: it must be exactly `FMC_HASH_SIZE` bytes.
+        if hash_section.len() != FMC_HASH_SIZE {
+            dev_err!(
+                dev,
+                "FMC hash section size {} != expected {}\n",
+                hash_section.len(),
+                FMC_HASH_SIZE
+            );
+            return Err(EINVAL);
+        }
+
+        // The key and signature sections are zero-padded to a fixed maximum, so they may be
+        // shorter, but must not exceed the destination buffers.
+        if pkey_section.len() > FMC_PKEY_SIZE {
+            dev_err!(
+                dev,
+                "FMC public key section size {} > maximum {}\n",
+                pkey_section.len(),
+                FMC_PKEY_SIZE
+            );
+            return Err(EINVAL);
+        }
+        if sig_section.len() > FMC_SIG_SIZE {
+            dev_err!(
+                dev,
+                "FMC signature section size {} > maximum {}\n",
+                sig_section.len(),
+                FMC_SIG_SIZE
+            );
+            return Err(EINVAL);
+        }
+
+        // Initialize the signatures in place to avoid building the large `FmcSignatures` on the
+        // stack, then fill each section from the firmware.
+        let signatures = KBox::init(
+            pin_init::init_zeroed::<FmcSignatures>().chain(|sigs| {
+                // PANIC: src and dst lengths are both `FMC_HASH_SIZE` (verified above).
+                sigs.hash384.copy_from_slice(hash_section);
+                // PANIC: dst is sliced to src.len(); src.len() <= `FMC_PKEY_SIZE` (verified above).
+                sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section);
+                // PANIC: dst is sliced to src.len(); src.len() <= `FMC_SIG_SIZE` (verified above).
+                sigs.signature[..sig_section.len()].copy_from_slice(sig_section);
+                Ok(())
+            }),
+            GFP_KERNEL,
+        )?;
+
+        Ok(signatures)
+    }
+}
diff --git a/drivers/gpu/nova-core/firmware/radix3.rs b/drivers/gpu/nova-core/firmware/radix3.rs
new file mode 100644
index 00000000000000..94235bd1209de9
--- /dev/null
+++ b/drivers/gpu/nova-core/firmware/radix3.rs
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! The radix3 page table, through which firmware running on the GPU reads a buffer in system
+//! memory. LibOS, the operating system of the GSP, defines the format: three levels of
+//! [`GSP_PAGE_SIZE`] pages, each entry the little-endian DMA address of one page.
+//!
+//! ```text
+//! Level 0:  one page, one entry  ->  the first level 1 page
+//! Level 1:  pages of entries     ->  each entry a level 2 page
+//! Level 2:  pages of entries     ->  each entry a page of the buffer
+//! ```
+
+use kernel::{
+    device,
+    dma::{
+        Coherent,
+        CoherentBox,
+        DataDirection,
+        DmaAddress, //
+    },
+    prelude::*,
+    scatterlist::{
+        Owned,
+        SGTable, //
+    },
+};
+
+use crate::{
+    gsp::GSP_PAGE_SIZE,
+    num::FromSafeCast, //
+};
+
+/// A radix3 page table and the data it maps.
+#[pin_data]
+pub(crate) struct Radix3<'a> {
+    /// The mapped data.
+    #[pin]
+    data: SGTable<Owned<VVec<u8>>>,
+    /// Level 2: one entry per page of `data`.
+    #[pin]
+    level2: SGTable<Owned<VVec<u8>>>,
+    /// Level 1: one entry per page of `level2`.
+    #[pin]
+    level1: SGTable<Owned<VVec<u8>>>,
+    /// Level 0: one page, whose single entry is the DMA address of the first `level1` page.
+    level0: Coherent<'a, [u64]>,
+    /// Length of `data`, in bytes.
+    size: usize,
+}
+
+impl<'a> Radix3<'a> {
+    /// Builds a radix3 page table over `data`, mapped for `dev` to read. May sleep.
+    pub(crate) fn new(
+        dev: &'a device::Device<device::Bound>,
+        data: VVec<u8>,
+    ) -> impl PinInit<Self, Error> + 'a {
+        let size = data.len();
+
+        pin_init::pin_init_scope(move || {
+            Ok(try_pin_init!(Self {
+                data <- SGTable::new(dev, data, DataDirection::ToDevice, GFP_KERNEL),
+                level2 <- {
+                    build_lvl(&data)
+                        .map(|l2| SGTable::new(dev, l2, DataDirection::ToDevice, GFP_KERNEL))?
+                },
+                level1 <- {
+                    build_lvl(&level2)
+                        .map(|l1| SGTable::new(dev, l1, DataDirection::ToDevice, GFP_KERNEL))?
+                },
+                level0: {
+                    // Allocate the level 0 page table as a device-visible DMA object, and map the
+                    // level 1 page table onto it.
+
+                    // Fill level 1 page entry.
+                    let level1_entry = level1.iter().next().ok_or(EINVAL)?;
+                    let level1_entry_addr = level1_entry.dma_address();
+
+                    // Create level 0 page table data and fill its first entry with the level 1
+                    // table.
+                    let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice(
+                        dev,
+                        GSP_PAGE_SIZE / size_of::<u64>(),
+                        GFP_KERNEL
+                    )?;
+                    level0[0] = level1_entry_addr.to_le();
+
+                    level0.into()
+                },
+                size,
+            }))
+        })
+    }
+
+    /// Returns the DMA address of the level 0 page, which is the address of the table.
+    pub(crate) fn dma_address(&self) -> DmaAddress {
+        self.level0.dma_address()
+    }
+
+    /// Returns the length of the mapped data, in bytes.
+    pub(crate) fn size(&self) -> usize {
+        self.size
+    }
+}
+
+/// Returns the size of the level that maps `sg_table`: one `u64` entry per [`GSP_PAGE_SIZE`]
+/// page of each DMA-mapped region, rounded up to whole pages.
+fn lvl_size(sg_table: &SGTable<Owned<VVec<u8>>>) -> usize {
+    let entries: usize = sg_table
+        .iter()
+        .map(|sg_entry| usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE))
+        .sum();
+
+    (entries * size_of::<u64>()).next_multiple_of(GSP_PAGE_SIZE)
+}
+
+/// Builds one level of the table over `sg_table`: one entry per [`GSP_PAGE_SIZE`] page of each
+/// DMA-mapped region, in region order. The level is a whole number of pages and every byte past
+/// the last entry is zero, because the booter reads a level a whole page at a time.
+///
+/// # Errors
+///
+/// - `ENOMEM` if the level cannot be allocated.
+/// - `EINVAL` if `sg_table` yields more entries than [`lvl_size`] sized the level for.
+fn build_lvl(sg_table: &SGTable<Owned<VVec<u8>>>) -> Result<VVec<u8>> {
+    let mut dst = VVec::<u8>::zeroed(lvl_size(sg_table), GFP_KERNEL).map_err(|_| ENOMEM)?;
+    let mut entries = dst.chunks_exact_mut(size_of::<u64>());
+
+    for sg_entry in sg_table.iter() {
+        let num_pages = usize::from_safe_cast(sg_entry.dma_len()).div_ceil(GSP_PAGE_SIZE);
+
+        for i in 0..num_pages {
+            let entry = sg_entry.dma_address()
+                + (u64::from_safe_cast(i) * u64::from_safe_cast(GSP_PAGE_SIZE));
+
+            entries
+                .next()
+                .ok_or(EINVAL)?
+                .copy_from_slice(&entry.to_le_bytes());
+        }
+    }
+
+    Ok(dst)
+}
diff --git a/drivers/gpu/nova-core/firmware/riscv.rs b/drivers/gpu/nova-core/firmware/riscv.rs
index 1403f05a730572..f05cfb1c65daa2 100644
--- a/drivers/gpu/nova-core/firmware/riscv.rs
+++ b/drivers/gpu/nova-core/firmware/riscv.rs
@@ -13,7 +13,7 @@ use kernel::{
 use crate::firmware::tlv::Tlv;
 
 /// A parsed firmware for a RISC-V core, ready to be loaded and run.
-pub(crate) struct RiscvFirmware {
+pub(crate) struct RiscvFirmware<'a> {
     /// Offset at which the code starts in the firmware image.
     pub(crate) code_offset: u32,
     /// Offset at which the data starts in the firmware image.
@@ -23,12 +23,12 @@ pub(crate) struct RiscvFirmware {
     /// Application version.
     pub(crate) app_version: u32,
     /// Device-mapped firmware image.
-    pub(crate) ucode: Coherent<[u8]>,
+    pub(crate) ucode: Coherent<'a, [u8]>,
 }
 
-impl RiscvFirmware {
+impl<'a> RiscvFirmware<'a> {
     /// Parses the RISC-V firmware image contained in `fw`.
-    pub(crate) fn new(dev: &device::Device<device::Bound>, fw: &Firmware) -> Result<Self> {
+    pub(crate) fn new(dev: &'a device::Device<device::Bound>, fw: &Firmware) -> Result<Self> {
         let tlv = Tlv::new(fw.data())?;
         dev_dbg!(
             dev,
diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs
index ab685fb4168f2f..b738dcabcdef88 100644
--- a/drivers/gpu/nova-core/fsp.rs
+++ b/drivers/gpu/nova-core/fsp.rs
@@ -3,9 +3,12 @@
 
 //! FSP (Foundation Security Processor) interface for Hopper/Blackwell GPUs.
 //!
-//! Hopper/Blackwell use a simplified firmware boot sequence: FMC, then FSP, then GSP.
+//! Hopper/Blackwell use a simplified firmware boot sequence: FSP secure-boots independently before
+//! the driver starts. The driver then sends FSP a Chain-of-Trust request containing the GSP-FMC
+//! image. FSP authenticates the image and launches GSP-FMC on the GSP RISC-V core; GSP-FMC
+//! subsequently authenticates and boots GSP-RM.
+//!
 //! Unlike Turing/Ampere/Ada, there is no SEC2 (Security Engine 2) usage.
-//! FSP handles secure boot directly using FMC firmware and Chain of Trust.
 
 use kernel::{
     device,
@@ -32,9 +35,9 @@ use crate::{
         Falcon, //
     },
     fb::FbSizes,
-    firmware::fsp::{
+    firmware::gsp_fmc::{
         FmcSignatures,
-        FspFirmware, //
+        GspFmcFirmware, //
     },
     gpu::Chipset,
     gsp::{
@@ -267,7 +270,7 @@ impl FspCotMessage {
     /// Returns an in-place initializer for [`FspCotMessage`].
     fn new<'a>(
         fb_info: &FbSizes,
-        fsp_fw: &'a FspFirmware,
+        fmc_fw: &'a GspFmcFirmware<'_>,
         args: &'a FmcBootArgs<'_>,
     ) -> Result<impl Init<Self> + 'a> {
         let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?;
@@ -296,13 +299,13 @@ impl FspCotMessage {
         .chain(move |msg| {
             msg.cot.version = version;
             msg.cot.size = size;
-            msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_address();
+            msg.cot.gsp_fmc_sysmem_offset = fmc_fw.fmc_image.dma_address();
             msg.cot.frts_vidmem_offset = frts_vidmem_offset;
             msg.cot.frts_vidmem_size = frts_size;
             // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem
             // fields point to an FRTS buffer in sysmem instead, for systems without VRAM.
             msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_address();
-            msg.cot.sigs = *fsp_fw.fmc_sigs;
+            msg.cot.sigs = *fmc_fw.fmc_sigs;
 
             Ok(())
         }))
@@ -345,28 +348,28 @@ impl MessageToFsp for FspPrcMessage {
 /// Bundled arguments for FMC boot via FSP Chain of Trust.
 pub(crate) struct FmcBootArgs<'a> {
     chipset: Chipset,
-    fmc_boot_params: Coherent<GspFmcBootParams>,
+    fmc_boot_params: Coherent<'a, GspFmcBootParams>,
     resume: bool,
     // Additional dependencies required to be kept alive for FMC boot.
-    _wpr_meta: Coherent<GspFwWprMeta>,
-    _libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
+    _wpr_meta: Coherent<'a, GspFwWprMeta>,
+    _libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
 }
 
 impl<'a> FmcBootArgs<'a> {
     /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter
     /// structure that FSP will read.
     pub(crate) fn new(
-        dev: &device::Device<device::Bound>,
+        dev: &'a device::Device<device::Bound>,
         chipset: Chipset,
-        wpr_meta: Coherent<GspFwWprMeta>,
-        libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
+        wpr_meta: Coherent<'a, GspFwWprMeta>,
+        libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
         resume: bool,
     ) -> Result<Self> {
         let init = GspFmcBootParams::new(wpr_meta.dma_address(), libos.dma_address());
 
         Ok(Self {
             chipset,
-            fmc_boot_params: Coherent::<GspFmcBootParams>::init(dev, GFP_KERNEL, init)?,
+            fmc_boot_params: Coherent::init(dev, GFP_KERNEL, init)?,
             resume,
             _wpr_meta: wpr_meta,
             _libos: libos,
@@ -374,19 +377,19 @@ impl<'a> FmcBootArgs<'a> {
     }
 
     /// Returns the FMC boot parameters allocation.
-    pub(crate) fn boot_params(&self) -> &Coherent<GspFmcBootParams> {
+    pub(crate) fn boot_params(&self) -> &Coherent<'_, GspFmcBootParams> {
         &self.fmc_boot_params
     }
 }
 
 /// FSP interface for Hopper/Blackwell GPUs.
 ///
-/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot
-/// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent
+/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot has
+/// completed. It owns the FSP falcon and the GSP-FMC firmware, which are used for the subsequent
 /// Chain of Trust boot.
 pub(crate) struct Fsp<'a> {
     falcon: Falcon<'a, FspEngine>,
-    fsp_fw: FspFirmware,
+    fmc_fw: GspFmcFirmware<'a>,
 }
 
 impl<'a> Fsp<'a> {
@@ -422,7 +425,7 @@ impl<'a> Fsp<'a> {
         const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000;
 
         let falcon = Falcon::<FspEngine>::new(dev, chipset, bar)?;
-        let fsp_fw = FspFirmware::new(dev, chipset)?;
+        let fmc_fw = GspFmcFirmware::new(dev, chipset)?;
 
         read_poll_timeout(
             || Ok(hal.fsp_boot_status(bar)),
@@ -434,7 +437,7 @@ impl<'a> Fsp<'a> {
             dev_err!(dev, "FSP secure boot completion error: {:?}\n", e);
         })?;
 
-        Ok(Fsp { falcon, fsp_fw })
+        Ok(Fsp { falcon, fmc_fw })
     }
 
     /// Sends a message to FSP and waits for the response.
@@ -540,7 +543,7 @@ impl<'a> Fsp<'a> {
     ) -> Result {
         dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset);
 
-        let msg = KBox::init(FspCotMessage::new(fb_info, &self.fsp_fw, args)?, GFP_KERNEL)?;
+        let msg = KBox::init(FspCotMessage::new(fb_info, &self.fmc_fw, args)?, GFP_KERNEL)?;
 
         let _response_buf = self.send_sync_fsp(dev, &*msg)?;
 
diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
index fd1414004dd08f..fb6f8a86a5036b 100644
--- a/drivers/gpu/nova-core/gpu.rs
+++ b/drivers/gpu/nova-core/gpu.rs
@@ -6,16 +6,25 @@ use kernel::{
     device,
     dma::Device,
     fmt,
+    gpu::buddy::GpuBuddyParams,
     io::Io,
     num::Bounded,
     pci,
     prelude::*,
-    sizes::SizeConstants, //
+    ptr::Alignment,
+    sizes::{
+        SizeConstants,
+        SZ_4K, //
+    },
+    uapi, //
 };
 
 use crate::{
     bounded_enum,
-    driver::Bar0,
+    driver::{
+        Bar0,
+        Bar1, //
+    },
     falcon::{
         gsp::Gsp as GspFalcon,
         sec2::Sec2 as Sec2Falcon,
@@ -29,27 +38,36 @@ use crate::{
         Gsp,
         GspBootContext, //
     },
-    regs,
+    mm::{
+        bar_user::BarUser,
+        pagetable::MmuVersion,
+        GpuMm,
+        VramAddress, //
+    },
     vgpu::VgpuManager, //
 };
 
 mod hal;
+mod regs;
 
 macro_rules! define_chipset {
-    ({ $($variant:ident = $value:expr),* $(,)* }) =>
+    ({ $($variant:ident),* $(,)* }) =>
     {
+        ::kernel::macros::paste!(
         /// Enum representation of the GPU chipset.
         #[derive(fmt::Debug, Copy, Clone, PartialOrd, Ord, PartialEq, Eq)]
-        pub(crate) enum Chipset {
-            $($variant = $value),*,
+        #[repr(u32)]
+        #[allow(missing_docs)]
+        pub enum Chipset {
+            $($variant = uapi::[<drm_nova_chipid_NOVA_DRM_CHIPID_ $variant:upper>]),*,
         }
 
         impl Chipset {
-            pub(crate) const ALL: &'static [Chipset] = &[
+            /// All chipsets known to the driver.
+            pub const ALL: &'static [Chipset] = &[
                 $( Chipset::$variant, )*
             ];
 
-            ::kernel::macros::paste!(
             /// Returns the name of this chipset, in lowercase.
             ///
             /// # Examples
@@ -65,7 +83,6 @@ macro_rules! define_chipset {
                 )*
                 }
             }
-            );
         }
 
         // TODO[FPRI]: replace with something like derive(FromPrimitive)
@@ -74,49 +91,55 @@ macro_rules! define_chipset {
 
             fn try_from(value: u32) -> Result<Self, Self::Error> {
                 match value {
-                    $( $value => Ok(Chipset::$variant), )*
+                    $(
+                        uapi::[<drm_nova_chipid_NOVA_DRM_CHIPID_ $variant:upper>] => {
+                            Ok(Chipset::$variant)
+                        }
+                    )*
                     _ => Err(ENODEV),
                 }
             }
         }
+    );
     }
 }
 
 define_chipset!({
     // Turing
-    TU102 = 0x162,
-    TU104 = 0x164,
-    TU106 = 0x166,
-    TU117 = 0x167,
-    TU116 = 0x168,
+    TU102,
+    TU104,
+    TU106,
+    TU117,
+    TU116,
     // Ampere
-    GA100 = 0x170,
-    GA102 = 0x172,
-    GA103 = 0x173,
-    GA104 = 0x174,
-    GA106 = 0x176,
-    GA107 = 0x177,
+    GA100,
+    GA102,
+    GA103,
+    GA104,
+    GA106,
+    GA107,
     // Hopper
-    GH100 = 0x180,
+    GH100,
     // Ada
-    AD102 = 0x192,
-    AD103 = 0x193,
-    AD104 = 0x194,
-    AD106 = 0x196,
-    AD107 = 0x197,
+    AD102,
+    AD103,
+    AD104,
+    AD106,
+    AD107,
     // Blackwell GB10x
-    GB100 = 0x1a0,
-    GB102 = 0x1a2,
+    GB100,
+    GB102,
     // Blackwell GB20x
-    GB202 = 0x1b2,
-    GB203 = 0x1b3,
-    GB205 = 0x1b5,
-    GB206 = 0x1b6,
-    GB207 = 0x1b7,
+    GB202,
+    GB203,
+    GB205,
+    GB206,
+    GB207,
 });
 
 impl Chipset {
-    pub(crate) const fn arch(self) -> Architecture {
+    /// Returns the [`Architecture`] generation of this chipset.
+    pub const fn arch(self) -> Architecture {
         match self {
             Self::TU102 | Self::TU104 | Self::TU106 | Self::TU117 | Self::TU116 => {
                 Architecture::Turing
@@ -139,6 +162,19 @@ impl Chipset {
     pub(crate) fn pci_config_mirror_range(self) -> Range<u32> {
         hal::gpu_hal(self).pci_config_mirror_range()
     }
+
+    /// Returns the MMU version for this chipset.
+    pub(crate) fn mmu_version(self) -> MmuVersion {
+        MmuVersion::from(self.arch())
+    }
+}
+
+impl From<Chipset> for u32 {
+    #[inline]
+    fn from(value: Chipset) -> Self {
+        // CAST: `Chipset` is `repr(u32)` and can thus be cast losslessly.
+        value as u32
+    }
 }
 
 // TODO
@@ -158,13 +194,30 @@ impl fmt::Display for Chipset {
 bounded_enum! {
     /// Enum representation of the GPU generation.
     #[derive(fmt::Debug, Copy, Clone)]
-    pub(crate) enum Architecture with TryFrom<Bounded<u32, 6>> {
-        Turing = 0x16,
-        Ampere = 0x17,
-        Hopper = 0x18,
-        Ada = 0x19,
-        BlackwellGB10x = 0x1a,
-        BlackwellGB20x = 0x1b,
+    #[repr(u32)]
+    pub enum Architecture with TryFrom<Bounded<u32, 6>> {
+        /// Turing (TU1xx).
+        Turing = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_TURING,
+        /// Ampere (GA10x).
+        Ampere = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_AMPERE,
+        /// Hopper (GH100).
+        Hopper = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_HOPPER,
+        /// Ada Lovelace (AD10x).
+        Ada = uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_ADA,
+        /// Blackwell (GB10x).
+        BlackwellGB10x =
+            uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB10X,
+        /// Blackwell (GB20x).
+        BlackwellGB20x =
+            uapi::drm_nova_architecture_NOVA_DRM_ARCHITECTURE_BLACKWELL_GB20X,
+    }
+}
+
+impl From<Architecture> for u32 {
+    #[inline]
+    fn from(value: Architecture) -> Self {
+        // CAST: `Architecture` is `repr(u32)` and can thus be cast losslessly.
+        value as u32
     }
 }
 
@@ -191,8 +244,9 @@ impl fmt::Display for Revision {
 
 /// Structure holding a basic description of the GPU: `Chipset` and `Revision`.
 #[derive(Clone, Copy)]
-pub(crate) struct Spec {
-    chipset: Chipset,
+pub struct Spec {
+    /// The GPU chipset.
+    pub chipset: Chipset,
     revision: Revision,
 }
 
@@ -272,17 +326,25 @@ struct GspResources<'gpu> {
     vgpu: VgpuManager,
     /// GSP runtime data.
     #[pin]
-    gsp: Gsp,
+    gsp: Gsp<'gpu>,
     /// GSP unload firmware bundle, if any.
-    unload_bundle: Option<gsp::UnloadBundle>,
+    unload_bundle: Option<gsp::UnloadBundle<'gpu>>,
 }
 
 /// Structure holding the resources required to operate the GPU.
 #[pin_data]
 pub(crate) struct Gpu<'gpu> {
-    spec: Spec,
+    pub(crate) spec: Spec,
     /// Static GPU information as provided by the GSP.
-    gsp_static_info: GetGspStaticInfoReply,
+    pub(crate) gsp_static_info: GetGspStaticInfoReply,
+    /// GPU memory manager owning memory management resources.
+    ///
+    /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while
+    /// the GSP is still operational.
+    mm: GpuMm<'gpu>,
+    /// BAR1 user interface for CPU access to GPU virtual memory.
+    #[pin]
+    bar_user: BarUser<'gpu>,
     /// GSP and its resources.
     #[pin]
     gsp_resources: GspResources<'gpu>,
@@ -326,6 +388,7 @@ impl<'gpu> Gpu<'gpu> {
     pub(crate) fn new<'a>(
         pdev: &'gpu pci::Device<device::Core<'a>>,
         bar: Bar0<'gpu>,
+        bar1: Bar1<'gpu>,
     ) -> impl PinInit<Self, Error> + use<'gpu, 'a> {
         let dev = pdev.as_ref();
 
@@ -343,6 +406,13 @@ impl<'gpu> Gpu<'gpu> {
                 // still constructing it, so no concurrent DMA allocations can exist.
                 unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? };
 
+                // Nova walks SG segments to build page tables, so their length is
+                // irrelevant to the device.
+                //
+                // SAFETY: `Gpu` owns all DMA allocations for this device, and we are
+                // still constructing it, so no concurrent DMA allocations can exist.
+                unsafe { pdev.dma_set_max_seg_size(u32::MAX) };
+
                 hal.wait_gfw_boot_completion(bar)
                     .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))?;
             },
@@ -370,7 +440,7 @@ impl<'gpu> Gpu<'gpu> {
 
                 vgpu: VgpuManager::new(pdev, spec.chipset, fsp.as_mut()),
 
-                gsp <- Gsp::new(pdev),
+                gsp <- Gsp::new(pdev, bar),
 
                 // This member must be initialized last, so the `UnloadBundle` can never be dropped
                 // from outside of the constructed `GspResources`, ensuring that the unload sequence
@@ -388,7 +458,7 @@ impl<'gpu> Gpu<'gpu> {
 
             gsp_static_info: {
                 // Obtain and display basic GPU information.
-                let info = gsp_resources.gsp.get_static_info(bar)?;
+                let info = gsp_resources.gsp.get_static_info()?;
                 match info.gpu_name() {
                     Ok(name) => dev_info!(dev, "GPU name: {}\n", name),
                     Err(e) => dev_warn!(dev, "GPU name unavailable: {:?}\n", e),
@@ -403,14 +473,67 @@ impl<'gpu> Gpu<'gpu> {
                     dev_dbg!(
                         dev,
                         "Total usable VRAM: {} MiB\n",
-                        info.usable_fb_regions.iter().fold(0u64, |res, region| res
-                            .saturating_add(region.end - region.start))
-                            / u64::SZ_1M
+                        info.vram_size() / u64::SZ_1M
                     );
                 }
 
                 info
-            }
+            },
+
+            // Create GPU memory manager owning memory management resources.
+            mm: {
+                let usable_vram = gsp_static_info.usable_fb_regions.first().ok_or(ENODEV)?;
+                let buddy_params = GpuBuddyParams {
+                    base_offset: usable_vram.start,
+                    size: usable_vram.end - usable_vram.start,
+                    chunk_size: Alignment::new::<SZ_4K>(),
+                };
+
+                GpuMm::new(
+                    bar,
+                    gsp_resources.spec.chipset,
+                    buddy_params,
+                    VramAddress::from_raw(gsp_static_info.total_fb_end),
+                )?
+            },
+
+            // Create BAR1 user interface for CPU access to GPU virtual memory.
+            bar_user <- {
+                let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base);
+                let bar1_idx = crate::driver::bar1_resource_index(pdev)?;
+                let bar1_size = pdev.resource_len(bar1_idx)?;
+
+                BarUser::new(
+                    pdb_addr,
+                    gsp_resources.spec.chipset,
+                    bar1_size,
+                    bar1,
+                )
+            },
         })
     }
+
+    /// Runs self-tests on the constructed [`Gpu`], logging failures without failing probe.
+    #[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+    pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device<device::Bound>) {
+        let this = self.project();
+        let dev = pdev.as_ref();
+        let regions = &this.gsp_static_info.usable_fb_regions;
+
+        if let Err(err) = crate::mm::selftest::run(
+            dev,
+            this.mm,
+            regions,
+            &this.bar_user,
+            this.gsp_static_info.bar1_pde_base,
+            this.spec.chipset,
+        ) {
+            dev_err!(dev, "self-tests failed: {:?}\n", err);
+        }
+    }
+}
+
+/// Reads the boot0 register and returns its raw value.
+pub(crate) fn boot_0_raw(bar: Bar0<'_>) -> u32 {
+    bar.read(regs::NV_PMC_BOOT_0).into_raw()
 }
diff --git a/drivers/gpu/nova-core/gpu/regs.rs b/drivers/gpu/nova-core/gpu/regs.rs
new file mode 100644
index 00000000000000..54e740d847cd7b
--- /dev/null
+++ b/drivers/gpu/nova-core/gpu/regs.rs
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use kernel::{
+    io::register,
+    prelude::*, //
+};
+
+use super::{
+    Architecture,
+    Chipset, //
+};
+
+use crate::driver::NovaRegisters;
+
+// PMC
+
+register! {
+    base: NovaRegisters;
+
+    /// Basic revision information about the GPU.
+    pub(super) NV_PMC_BOOT_0(u32) @ 0x00000000 {
+        /// Lower bits of the architecture.
+        28:24   architecture_0;
+        /// Implementation version of the architecture.
+        23:20   implementation;
+        /// MSB of the architecture.
+        8:8     architecture_1;
+        /// Major revision of the chip.
+        7:4     major_revision;
+        /// Minor revision of the chip.
+        3:0     minor_revision;
+    }
+
+    /// Extended architecture information.
+    pub(super) NV_PMC_BOOT_42(u32) @ 0x00000a00 {
+        /// Architecture value.
+        29:24   architecture ?=> Architecture;
+        /// Implementation version of the architecture.
+        23:20   implementation;
+        /// Major revision of the chip.
+        19:16   major_revision;
+        /// Minor revision of the chip.
+        15:12   minor_revision;
+    }
+}
+
+impl NV_PMC_BOOT_0 {
+    pub(super) fn is_older_than_fermi(self) -> bool {
+        // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals :
+        const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc;
+
+        // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than
+        // GF100, means "older than Fermi".
+        self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100
+    }
+}
+
+impl NV_PMC_BOOT_42 {
+    /// Combines `architecture` and `implementation` to obtain a code unique to the chipset.
+    pub(super) fn chipset(self) -> Result<Chipset> {
+        self.architecture()
+            .map(|arch| {
+                ((arch as u32) << Self::IMPLEMENTATION_RANGE.len())
+                    | u32::from(self.implementation())
+            })
+            .and_then(Chipset::try_from)
+    }
+
+    /// Returns the raw architecture value from the register.
+    fn architecture_raw(self) -> u8 {
+        ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start())
+            & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8
+    }
+}
+
+impl kernel::fmt::Display for NV_PMC_BOOT_42 {
+    fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
+        write!(
+            f,
+            "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})",
+            self.inner,
+            self.architecture_raw(),
+            self.implementation()
+        )
+    }
+}
diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs
index 13f361406a6c3a..dda58095f40bd6 100644
--- a/drivers/gpu/nova-core/gsp.rs
+++ b/drivers/gpu/nova-core/gsp.rs
@@ -115,11 +115,11 @@ impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> {
 /// then pp points to index into the buffer where the next logging entry will
 /// be written. Therefore, the logging data is valid if:
 ///   1 <= pp < sizeof(buffer)/sizeof(u64)
-struct LogBuffer(Coherent<[u8; LOG_BUFFER_SIZE]>);
+struct LogBuffer<'a>(Coherent<'a, [u8; LOG_BUFFER_SIZE]>);
 
-impl LogBuffer {
+impl<'a> LogBuffer<'a> {
     /// Creates a new `LogBuffer` mapped on `dev`.
-    fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
+    fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> {
         let obj = Self(Coherent::zeroed(dev, GFP_KERNEL)?);
 
         let start_addr = obj.0.dma_address();
@@ -135,33 +135,36 @@ impl LogBuffer {
     }
 }
 
-struct LogBuffers {
+struct LogBuffers<'a> {
     /// Init log buffer.
-    loginit: LogBuffer,
+    loginit: LogBuffer<'a>,
     /// Interrupts log buffer.
-    logintr: LogBuffer,
+    logintr: LogBuffer<'a>,
     /// RM log buffer.
-    logrm: LogBuffer,
+    logrm: LogBuffer<'a>,
 }
 
 /// GSP runtime data.
 #[pin_data]
-pub(crate) struct Gsp {
+pub(crate) struct Gsp<'gsp> {
     /// Libos arguments.
-    pub(crate) libos: Coherent<[LibosMemoryRegionInitArgument]>,
+    pub(crate) libos: Coherent<'gsp, [LibosMemoryRegionInitArgument]>,
     /// Log buffers, optionally exposed via debugfs.
     #[pin]
-    logs: debugfs::Scope<LogBuffers>,
+    logs: debugfs::Scope<LogBuffers<'gsp>>,
     /// Command queue.
     #[pin]
-    pub(crate) cmdq: Cmdq,
+    pub(crate) cmdq: Cmdq<'gsp>,
     /// RM arguments.
-    rmargs: Coherent<GspArgumentsPadded>,
+    rmargs: Coherent<'gsp, GspArgumentsPadded>,
 }
 
-impl Gsp {
+impl<'gsp> Gsp<'gsp> {
     // Creates an in-place initializer for a `Gsp` manager for `pdev`.
-    pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ {
+    pub(crate) fn new(
+        pdev: &'gsp pci::Device<device::Bound>,
+        bar: Bar0<'gsp>,
+    ) -> impl PinInit<Self, Error> + 'gsp {
         pin_init::pin_init_scope(move || {
             let dev = pdev.as_ref();
 
@@ -173,7 +176,7 @@ impl Gsp {
             // _kgspInitLibosLoggingStructures (allocates memory for buffers)
             // kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
             Ok(try_pin_init!(Self {
-                cmdq <- Cmdq::new(dev),
+                cmdq <- Cmdq::new(dev, bar),
                 rmargs: Coherent::init(dev, GFP_KERNEL, GspArgumentsPadded::new(&cmdq))?,
                 libos: {
                     let mut libos = CoherentBox::zeroed_slice(
@@ -217,10 +220,10 @@ impl Gsp {
     }
 
     /// Query the GSP for the static GPU information.
-    pub(crate) fn get_static_info(&self, bar: Bar0<'_>) -> Result<commands::GetGspStaticInfoReply> {
-        self.cmdq.send_command(bar, commands::GetGspStaticInfo)
+    pub(crate) fn get_static_info(&self) -> Result<commands::GetGspStaticInfoReply> {
+        self.cmdq.send_command(commands::GetGspStaticInfo)
     }
 }
 
 /// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`].
-pub(crate) struct UnloadBundle(KBox<dyn hal::UnloadBundle>);
+pub(crate) struct UnloadBundle<'a>(KBox<dyn hal::UnloadBundle + 'a>);
diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs
index e03700ee7beaf4..4fb1b69ac9d5c6 100644
--- a/drivers/gpu/nova-core/gsp/boot.rs
+++ b/drivers/gpu/nova-core/gsp/boot.rs
@@ -10,7 +10,6 @@ use kernel::{
 };
 
 use crate::{
-    driver::Bar0,
     falcon::{
         gsp::Gsp,
         Falcon, //
@@ -22,7 +21,7 @@ use crate::{
     },
 };
 
-impl super::Gsp {
+impl<'gsp> super::Gsp<'gsp> {
     /// Attempt to boot the GSP.
     ///
     /// This is a GPU-dependent and complex procedure that involves loading firmware files from
@@ -33,10 +32,9 @@ impl super::Gsp {
     /// [`Self::unload`]) returned.
     pub(crate) fn boot(
         self: Pin<&mut Self>,
-        mut ctx: super::GspBootContext<'_, '_>,
-    ) -> Result<Option<super::UnloadBundle>> {
+        mut ctx: super::GspBootContext<'_, 'gsp>,
+    ) -> Result<Option<super::UnloadBundle<'gsp>>> {
         let pdev = ctx.pdev;
-        let bar = ctx.bar;
         let chipset = ctx.chipset;
         let gsp_falcon = ctx.gsp_falcon;
         let dev = pdev.as_ref();
@@ -44,6 +42,11 @@ impl super::Gsp {
 
         let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset), GFP_KERNEL)?;
 
+        self.cmdq
+            .send_command_no_wait(commands::SetSystemInfo::new(pdev, chipset))?;
+        self.cmdq
+            .send_command_no_wait(commands::SetRegistry::new(ctx.vgpu.state())?)?;
+
         // Perform the chipset-specific boot sequence, and retrieve the unload bundle.
         let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| {
             dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n");
@@ -73,11 +76,6 @@ impl super::Gsp {
 
         dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),);
 
-        self.cmdq
-            .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?;
-        self.cmdq
-            .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?;
-
         hal.post_boot(&self, ctx, &gsp_fw)?;
 
         // Wait until GSP is fully initialized.
@@ -88,13 +86,12 @@ impl super::Gsp {
 
     /// Shut down the GSP and wait until it is offline.
     fn shutdown_gsp(
-        cmdq: &Cmdq,
-        bar: Bar0<'_>,
+        cmdq: &Cmdq<'_>,
         gsp_falcon: &Falcon<'_, Gsp>,
         mode: commands::PowerStateLevel,
     ) -> Result {
         // Command to shut the GSP down.
-        cmdq.send_command(bar, commands::UnloadingGuestDriver::new(mode))?;
+        cmdq.send_command(commands::UnloadingGuestDriver::new(mode))?;
 
         // Wait until GSP signals it is suspended.
         const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31);
@@ -113,14 +110,13 @@ impl super::Gsp {
     pub(crate) fn unload(
         &self,
         mut ctx: super::GspBootContext<'_, '_>,
-        unload_bundle: Option<super::UnloadBundle>,
+        unload_bundle: Option<super::UnloadBundle<'_>>,
     ) -> Result {
         let dev = ctx.dev();
 
         // Shut down the GSP. Keep going even in case of error.
         let mut res = Self::shutdown_gsp(
             &self.cmdq,
-            ctx.bar,
             ctx.gsp_falcon,
             commands::PowerStateLevel::Level0,
         )
diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs
index 6da728201281ba..a5595da23407b0 100644
--- a/drivers/gpu/nova-core/gsp/cmdq.rs
+++ b/drivers/gpu/nova-core/gsp/cmdq.rs
@@ -2,13 +2,7 @@
 
 mod continuation;
 
-use core::{
-    mem,
-    sync::atomic::{
-        fence,
-        Ordering, //
-    },
-};
+use core::mem;
 
 use kernel::{
     device,
@@ -26,7 +20,12 @@ use kernel::{
     prelude::*,
     ptr,
     sync::{
-        aref::ARef,
+        barrier::{
+            dma_mb,
+            Full,
+            Read,
+            Write, //
+        },
         Mutex, //
     },
     time::Delta,
@@ -230,19 +229,19 @@ unsafe impl FromBytes for GspMem {}
 ///   pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`].
 /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read
 ///   pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`].
-struct DmaGspMem(Coherent<GspMem>);
+struct DmaGspMem<'a>(Coherent<'a, GspMem>);
 
-impl DmaGspMem {
+impl<'a> DmaGspMem<'a> {
     /// Allocate a new instance and map it for `dev`.
-    fn new(dev: &device::Device<device::Bound>) -> Result<Self> {
+    fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> {
         const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>();
         const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>();
 
-        let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?;
+        let mut gsp_mem = CoherentBox::<'_, GspMem>::zeroed(dev, GFP_KERNEL)?;
         gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES);
         gsp_mem.cpuq.rx = MsgqRxHeader::new();
 
-        let gsp_mem: Coherent<_> = gsp_mem.into();
+        let gsp_mem: Coherent<'_, _> = gsp_mem.into();
         PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?;
 
         Ok(Self(gsp_mem))
@@ -404,7 +403,12 @@ impl DmaGspMem {
     //
     // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn gsp_write_ptr(&self) -> u32 {
-        MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES
+        let ptr = MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES;
+
+        // ORDERING: LOAD->LOAD ordering needed to order `gsp_write_ptr` read before data read.
+        dma_mb(Read);
+
+        ptr
     }
 
     // Returns the index of the memory page the GSP will read the next command from.
@@ -413,7 +417,12 @@ impl DmaGspMem {
     //
     // - The returned value is within `0..MSGQ_NUM_PAGES`.
     fn gsp_read_ptr(&self) -> u32 {
-        MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES
+        let ptr = MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES;
+
+        // ORDERING: LOAD->STORE ordering needed to order `gsp_read_ptr` read before data write.
+        dma_mb(Full);
+
+        ptr
     }
 
     // Returns the index of the memory page the CPU can read the next message from.
@@ -427,12 +436,11 @@ impl DmaGspMem {
 
     // Informs the GSP that it can send `elem_count` new pages into the message queue.
     fn advance_cpu_read_ptr(&mut self, elem_count: u32) {
+        // ORDERING: LOAD->STORE ordering needed to order `cpu_read_ptr` write after data read.
+        dma_mb(Full);
+
         let rx = io_project!(self.0, .cpuq.rx);
         let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
-
-        // Ensure read pointer is properly ordered.
-        fence(Ordering::SeqCst);
-
         MsgqRxHeader::set_read_ptr(rx, rptr)
     }
 
@@ -447,12 +455,12 @@ impl DmaGspMem {
 
     // Informs the GSP that it can process `elem_count` new pages from the command queue.
     fn advance_cpu_write_ptr(&mut self, elem_count: u32) {
+        // ORDERING: STORE->STORE ordering needed to order `cpu_write_ptr` write after data write.
+        dma_mb(Write);
+
         let tx = io_project!(self.0, .cpuq.tx);
         let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES;
         MsgqTxHeader::set_write_ptr(tx, wptr);
-
-        // Ensure all command data is visible before triggering the GSP read.
-        fence(Ordering::SeqCst);
     }
 }
 
@@ -469,7 +477,7 @@ struct GspCommand<'a> {
 
 /// A message ready to be processed from the message queue.
 ///
-/// This is the type returned by [`Cmdq::wait_for_msg`].
+/// This is the type returned by [`CmdqInner::wait_for_msg`].
 struct GspMessage<'a> {
     // Reference to the header of the message.
     header: &'a GspMsgElement,
@@ -483,15 +491,15 @@ struct GspMessage<'a> {
 /// Provides the ability to send commands and receive messages from the GSP using a shared memory
 /// area.
 #[pin_data]
-pub(crate) struct Cmdq {
+pub(crate) struct Cmdq<'cmdq> {
     /// Inner mutex-protected state.
     #[pin]
-    inner: Mutex<CmdqInner>,
+    inner: Mutex<CmdqInner<'cmdq>>,
     /// DMA address of the command queue's shared memory region.
     pub(super) dma_addr: DmaAddress,
 }
 
-impl Cmdq {
+impl<'cmdq> Cmdq<'cmdq> {
     /// Offset of the data after the PTEs.
     const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq);
 
@@ -512,14 +520,18 @@ impl Cmdq {
     pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5);
 
     /// Creates a new command queue for `dev`.
-    pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ {
+    pub(crate) fn new(
+        dev: &'cmdq device::Device<device::Bound>,
+        bar: Bar0<'cmdq>,
+    ) -> impl PinInit<Self, Error> + 'cmdq {
         pin_init_scope(move || {
             let gsp_mem = DmaGspMem::new(dev)?;
 
             Ok(try_pin_init!(Self {
                 dma_addr: gsp_mem.0.dma_address(),
                 inner <- new_mutex!(CmdqInner {
-                    dev: dev.into(),
+                    dev,
+                    bar,
                     gsp_mem,
                     seq: 0,
                 }),
@@ -561,7 +573,7 @@ impl Cmdq {
     ///   written to by its [`CommandToGsp::init_variable_payload`] method.
     ///
     /// Error codes returned by the command and reply initializers are propagated as-is.
-    pub(crate) fn send_command<M>(&self, bar: Bar0<'_>, command: M) -> Result<M::Reply>
+    pub(crate) fn send_command<M>(&self, command: M) -> Result<M::Reply>
     where
         M: CommandToGsp,
         M::Reply: MessageFromGsp,
@@ -569,7 +581,7 @@ impl Cmdq {
         Error: From<<M::Reply as MessageFromGsp>::InitError>,
     {
         let mut inner = self.inner.lock();
-        inner.send_command(bar, command)?;
+        inner.send_command(command)?;
 
         loop {
             match inner.receive_msg::<M::Reply>(Self::RECEIVE_TIMEOUT) {
@@ -589,12 +601,12 @@ impl Cmdq {
     ///   written to by its [`CommandToGsp::init_variable_payload`] method.
     ///
     /// Error codes returned by the command initializers are propagated as-is.
-    pub(crate) fn send_command_no_wait<M>(&self, bar: Bar0<'_>, command: M) -> Result
+    pub(crate) fn send_command_no_wait<M>(&self, command: M) -> Result
     where
         M: CommandToGsp<Reply = NoReply>,
         Error: From<M::InitError>,
     {
-        self.inner.lock().send_command(bar, command)
+        self.inner.lock().send_command(command)
     }
 
     /// Receive a message from the GSP.
@@ -610,16 +622,18 @@ impl Cmdq {
 }
 
 /// Inner mutex protected state of [`Cmdq`].
-struct CmdqInner {
+struct CmdqInner<'a> {
     /// Device this command queue belongs to.
-    dev: ARef<device::Device>,
+    dev: &'a device::Device,
+    /// MMIO mapping of PCI BAR0, for writing the GSP doorbell.
+    bar: Bar0<'a>,
     /// Current command sequence number.
     seq: u32,
     /// Memory area shared with the GSP for communicating commands and messages.
-    gsp_mem: DmaGspMem,
+    gsp_mem: DmaGspMem<'a>,
 }
 
-impl CmdqInner {
+impl CmdqInner<'_> {
     /// Timeout for waiting for space on the command queue.
     const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1);
 
@@ -633,7 +647,7 @@ impl CmdqInner {
     ///   written to by its [`CommandToGsp::init_variable_payload`] method.
     ///
     /// Error codes returned by the command initializers are propagated as-is.
-    fn send_single_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result
+    fn send_single_command<M>(&mut self, command: M) -> Result
     where
         M: CommandToGsp,
         // This allows all error types, including `Infallible`, to be used for `M::InitError`.
@@ -687,7 +701,7 @@ impl CmdqInner {
         let elem_count = dst.header.element_count();
         self.seq += 1;
         self.gsp_mem.advance_cpu_write_ptr(elem_count);
-        Cmdq::notify_gsp(bar);
+        Cmdq::notify_gsp(self.bar);
 
         Ok(())
     }
@@ -703,19 +717,19 @@ impl CmdqInner {
     ///   written to by its [`CommandToGsp::init_variable_payload`] method.
     ///
     /// Error codes returned by the command initializers are propagated as-is.
-    fn send_command<M>(&mut self, bar: Bar0<'_>, command: M) -> Result
+    fn send_command<M>(&mut self, command: M) -> Result
     where
         M: CommandToGsp,
         Error: From<M::InitError>,
     {
         match SplitState::new(command)? {
-            SplitState::Single(command) => self.send_single_command(bar, command),
+            SplitState::Single(command) => self.send_single_command(command),
             SplitState::Split(command, mut continuations) => {
-                self.send_single_command(bar, command)?;
+                self.send_single_command(command)?;
 
                 while let Some(continuation) = continuations.next() {
                     // Turbofish needed because the compiler cannot infer M here.
-                    self.send_single_command::<ContinuationRecord<'_>>(bar, continuation)?;
+                    self.send_single_command::<ContinuationRecord<'_>>(continuation)?;
                 }
 
                 Ok(())
diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs
index ffc25fd8c47b85..59d7d7fb15e89c 100644
--- a/drivers/gpu/nova-core/gsp/commands.rs
+++ b/drivers/gpu/nova-core/gsp/commands.rs
@@ -187,7 +187,7 @@ impl MessageFromGsp for GspInitDone {
 }
 
 /// Waits for GSP initialization to complete.
-pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result {
+pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq<'_>) -> Result {
     loop {
         match cmdq.receive_msg::<GspInitDone>(Cmdq::RECEIVE_TIMEOUT) {
             Ok(_) => break Ok(()),
@@ -212,10 +212,17 @@ impl CommandToGsp for GetGspStaticInfo {
 }
 
 /// The reply from the GSP to the [`GetGspStaticInfo`] command.
-pub(crate) struct GetGspStaticInfoReply {
+pub struct GetGspStaticInfoReply {
     gpu_name: [u8; 64],
+    gpu_short_name: [u8; 64],
+    /// The 16-byte SHA-1 based GPU identifier (GID) reported by GSP-RM.
+    pub gpu_gid: [u8; 16],
+    /// BAR1 Page Directory Entry base address.
+    pub(crate) bar1_pde_base: u64,
     /// Usable FB (VRAM) regions for driver memory allocation.
     pub(crate) usable_fb_regions: KVec<Range<u64>>,
+    /// Exclusive end of the FB physical address space.
+    pub(crate) total_fb_end: u64,
 }
 
 impl MessageFromGsp for GetGspStaticInfoReply {
@@ -231,22 +238,26 @@ impl MessageFromGsp for GetGspStaticInfoReply {
         for region in msg.usable_fb_regions() {
             usable_fb_regions.push(region, GFP_KERNEL)?;
         }
+        let total_fb_end = msg.total_fb_end().ok_or(EINVAL)?;
 
         Ok(GetGspStaticInfoReply {
             gpu_name: msg.gpu_name_str(),
+            gpu_short_name: msg.gpu_short_name_str(),
+            gpu_gid: msg.gpu_gid(),
+            bar1_pde_base: msg.bar1_pde_base(),
             usable_fb_regions,
+            total_fb_end,
         })
     }
 }
 
 /// Error type for [`GetGspStaticInfoReply::gpu_name`].
 #[derive(Debug)]
-pub(crate) enum GpuNameError {
+pub enum GpuNameError {
     /// The GPU name string does not contain a null terminator.
     NoNullTerminator(FromBytesUntilNulError),
 
     /// The GPU name string contains invalid UTF-8.
-    #[expect(dead_code)]
     InvalidUtf8(Utf8Error),
 }
 
@@ -255,12 +266,31 @@ impl GetGspStaticInfoReply {
     ///
     /// Returns an error if the string given by the GSP does not contain a null terminator or
     /// contains invalid UTF-8.
-    pub(crate) fn gpu_name(&self) -> core::result::Result<&str, GpuNameError> {
+    pub fn gpu_name(&self) -> Result<&str, GpuNameError> {
         CStr::from_bytes_until_nul(&self.gpu_name)
             .map_err(GpuNameError::NoNullTerminator)?
             .to_str()
             .map_err(GpuNameError::InvalidUtf8)
     }
+
+    /// Returns the short name of the GPU as a string.
+    ///
+    /// Returns an error if the string given by the GSP does not contain a null terminator or
+    /// contains invalid UTF-8.
+    pub fn gpu_short_name(&self) -> core::result::Result<&str, GpuNameError> {
+        CStr::from_bytes_until_nul(&self.gpu_short_name)
+            .map_err(GpuNameError::NoNullTerminator)?
+            .to_str()
+            .map_err(GpuNameError::InvalidUtf8)
+    }
+
+    /// Returns the total usable VRAM size in bytes, i.e. the summed lengths of all usable FB
+    /// regions.
+    pub fn vram_size(&self) -> u64 {
+        self.usable_fb_regions.iter().fold(0, |size, region| {
+            size.saturating_add(region.end - region.start)
+        })
+    }
 }
 
 pub(crate) use fw::commands::PowerStateLevel;
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index 05f54fee618666..918a7ae809eb84 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -179,7 +179,7 @@ impl GspFwWprMeta {
     /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the
     /// framebuffer ranges `ranges`.
     pub(crate) fn from_ranges<'a>(
-        gsp_firmware: &'a GspFirmware,
+        gsp_firmware: &'a GspFirmware<'_>,
         ranges: &'a FbRanges,
     ) -> impl Init<Self> + 'a {
         let init_inner = init!(bindings::GspFwWprMeta {
@@ -187,7 +187,7 @@ impl GspFwWprMeta {
             magic: bindings::GSP_FW_WPR_META_MAGIC as u64,
             revision: u64::from(bindings::GSP_FW_WPR_META_REVISION),
             sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(),
-            sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size),
+            sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size()),
             sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(),
             sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()),
             bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset),
@@ -205,7 +205,7 @@ impl GspFwWprMeta {
             gspFwWprStart: ranges.wpr2.start,
             gspFwHeapOffset: ranges.wpr2_heap.start,
             gspFwHeapSize: ranges.wpr2_heap.len(),
-            gspFwOffset: ranges.elf.start,
+            gspFwOffset: ranges.fw_image.start,
             bootBinOffset: ranges.boot.start,
             frtsOffset: ranges.frts.start,
             frtsSize: ranges.frts.len(),
@@ -231,7 +231,7 @@ impl GspFwWprMeta {
     ///
     /// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2.
     pub(crate) fn from_sizes<'a>(
-        gsp_firmware: &'a GspFirmware,
+        gsp_firmware: &'a GspFirmware<'_>,
         sizes: &'a FbSizes,
     ) -> impl Init<Self> + 'a {
         /// VGA workspace size to reserve at the end of the framebuffer, in bytes.
@@ -242,7 +242,7 @@ impl GspFwWprMeta {
             magic: bindings::GSP_FW_WPR_META_MAGIC as u64,
             revision: u64::from(bindings::GSP_FW_WPR_META_REVISION),
             sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(),
-            sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size),
+            sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size()),
             sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(),
             sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()),
             bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset),
@@ -665,7 +665,7 @@ unsafe impl FromBytes for LibosMemoryRegionInitArgument {}
 impl LibosMemoryRegionInitArgument {
     pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>(
         name: &'static str,
-        obj: &'a Coherent<A>,
+        obj: &'a Coherent<'_, A>,
     ) -> impl Init<Self> + 'a {
         /// Generates the `ID8` identifier required for some GSP objects.
         fn id8(name: &str) -> u64 {
@@ -897,7 +897,7 @@ pub(crate) struct GspArgumentsCached {
 
 impl GspArgumentsCached {
     /// Creates the arguments for starting the GSP up using `cmdq` as its command queue.
-    pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
+    pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> {
         let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED {
             messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq),
             bDmemStack: 1,
@@ -924,7 +924,7 @@ pub(crate) struct GspArgumentsPadded {
 }
 
 impl GspArgumentsPadded {
-    pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
+    pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> {
         init!(GspArgumentsPadded {
             inner <- GspArgumentsCached::new(cmdq),
             ..Zeroable::init_zeroed()
@@ -944,7 +944,7 @@ type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS;
 
 impl MessageQueueInitArguments {
     /// Creates a new init arguments structure for `cmdq`.
-    fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ {
+    fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> {
         init!(MessageQueueInitArguments {
             sharedMemPhysAddr: cmdq.dma_addr,
             pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(),
diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs
index 6dc31d1bf5ae45..567c56fbbfe39e 100644
--- a/drivers/gpu/nova-core/gsp/fw/commands.rs
+++ b/drivers/gpu/nova-core/gsp/fw/commands.rs
@@ -131,6 +131,29 @@ impl GspStaticConfigInfo {
         self.0.gpuNameString
     }
 
+    /// Returns a bytes array containing the (hopefully) zero-terminated short name of this GPU.
+    pub(crate) fn gpu_short_name_str(&self) -> [u8; 64] {
+        self.0.gpuShortNameString
+    }
+
+    /// Returns the 16-byte SHA-1 GPU identifier supplied by GSP-RM.
+    ///
+    /// GSP-RM reports the GID in binary SHA-1 form, which occupies the first 16 bytes of the
+    /// GID info payload.
+    pub(crate) fn gpu_gid(&self) -> [u8; 16] {
+        let mut gid = [0u8; 16];
+        gid.copy_from_slice(&self.0.gidInfo.data[..16]);
+        gid
+    }
+
+    /// Returns the BAR1 Page Directory Entry base address.
+    ///
+    /// This is the root page table address for BAR1 virtual memory,
+    /// set up by GSP-RM firmware.
+    pub(crate) fn bar1_pde_base(&self) -> u64 {
+        self.0.bar1PdeBase
+    }
+
     /// Returns an iterator over valid FB regions from GSP firmware data.
     fn fb_regions(
         &self,
@@ -165,6 +188,11 @@ impl GspStaticConfigInfo {
             }
         })
     }
+
+    /// Computes the exclusive end of the FB physical address space.
+    pub(crate) fn total_fb_end(&self) -> Option<u64> {
+        self.fb_regions().map(|reg| reg.limit).max()?.checked_add(1)
+    }
 }
 
 // SAFETY: Padding is explicit and will not contain uninitialized data.
diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs
index 5850fa0fe0e9a3..a0117d6d16a204 100644
--- a/drivers/gpu/nova-core/gsp/hal.rs
+++ b/drivers/gpu/nova-core/gsp/hal.rs
@@ -24,7 +24,7 @@ use crate::{
 /// The GSP unload code might run in a situation where we cannot load firmware dynamically (e.g.
 /// because we are in shutdown and the file system is not accessible anymore). Thus, the firmware
 /// required for unloading is prepared at load time, and stored here until it needs to be run.
-pub(super) trait UnloadBundle: Send {
+pub(super) trait UnloadBundle: Send + Sync {
     /// Performs the steps required to properly reset the GSP after it has been stopped.
     fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result;
 }
@@ -35,12 +35,12 @@ pub(super) trait GspHal: Send {
     ///
     /// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one
     /// could be created.
-    fn boot(
+    fn boot<'gpu>(
         &self,
-        gsp: &Gsp,
-        ctx: &mut GspBootContext<'_, '_>,
-        gsp_fw: &GspFirmware,
-    ) -> Result<Option<crate::gsp::UnloadBundle>>;
+        gsp: &Gsp<'gpu>,
+        ctx: &mut GspBootContext<'_, 'gpu>,
+        gsp_fw: &GspFirmware<'gpu>,
+    ) -> Result<Option<super::UnloadBundle<'gpu>>>;
 
     /// Performs HAL-specific post-GSP boot tasks.
     ///
@@ -48,9 +48,9 @@ pub(super) trait GspHal: Send {
     /// after the initialization commands have been pushed onto its queue.
     fn post_boot(
         &self,
-        _gsp: &Gsp,
+        _gsp: &Gsp<'_>,
         _ctx: &mut GspBootContext<'_, '_>,
-        _gsp_fw: &GspFirmware,
+        _gsp_fw: &GspFirmware<'_>,
     ) -> Result {
         Ok(())
     }
diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs
index e283429a95ddae..91201b51030e05 100644
--- a/drivers/gpu/nova-core/gsp/hal/gh100.rs
+++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs
@@ -58,7 +58,7 @@ impl GspMbox {
     fn lockdown_released_or_error(
         &self,
         gsp_falcon: &Falcon<'_, GspEngine>,
-        fmc_boot_params: &Coherent<GspFmcBootParams>,
+        fmc_boot_params: &Coherent<'_, GspFmcBootParams>,
     ) -> bool {
         // GSP-FMC normally clears the boot parameters address from the mailboxes early during
         // boot. If the address is still there, keep polling rather than treating it as an error.
@@ -75,7 +75,7 @@ impl GspMbox {
 fn wait_for_gsp_lockdown_release(
     dev: &device::Device<device::Bound>,
     gsp_falcon: &Falcon<'_, GspEngine>,
-    fmc_boot_params: &Coherent<GspFmcBootParams>,
+    fmc_boot_params: &Coherent<'_, GspFmcBootParams>,
 ) -> Result {
     dev_dbg!(dev, "Waiting for GSP lockdown release\n");
 
@@ -141,12 +141,12 @@ impl GspHal for Gh100 {
     ///
     /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles
     /// the GSP boot internally - no manual GSP reset/boot is needed.
-    fn boot(
+    fn boot<'gpu>(
         &self,
-        gsp: &Gsp,
-        ctx: &mut GspBootContext<'_, '_>,
-        gsp_fw: &GspFirmware,
-    ) -> Result<Option<crate::gsp::UnloadBundle>> {
+        gsp: &Gsp<'gpu>,
+        ctx: &mut GspBootContext<'_, 'gpu>,
+        gsp_fw: &GspFirmware<'gpu>,
+    ) -> Result<Option<crate::gsp::UnloadBundle<'gpu>>> {
         let dev = ctx.dev();
         let chipset = ctx.chipset;
         let gsp_falcon = ctx.gsp_falcon;
@@ -159,7 +159,7 @@ impl GspHal for Gh100 {
         let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?;
 
         let unload_bundle = crate::gsp::UnloadBundle(
-            KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle>
+            KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle + 'gpu>
         );
 
         // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args`
diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs
index a5c0ca35549309..e90db1a230325c 100644
--- a/drivers/gpu/nova-core/gsp/hal/tu102.rs
+++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs
@@ -52,34 +52,33 @@ use crate::{
 //
 // Since there are two variants of the prepared firmware (with and without a bootloader), this type
 // abstracts the difference.
-enum FwsecUnloadFirmware {
+enum FwsecUnloadFirmware<'a> {
     WithoutBl(FwsecFirmware),
-    WithBl(FwsecFirmwareWithBl),
+    WithBl(FwsecFirmwareWithBl<'a>),
 }
 
-impl FwsecUnloadFirmware {
+impl FwsecUnloadFirmware<'_> {
     /// Runs the FWSEC SB firmware.
     fn run(
         &self,
         dev: &device::Device<device::Bound>,
-        bar: Bar0<'_>,
         gsp_falcon: &Falcon<'_, GspEngine>,
     ) -> Result {
         match self {
             Self::WithoutBl(fw) => fw.run(dev, gsp_falcon),
-            Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar),
+            Self::WithBl(fw) => fw.run(dev, gsp_falcon),
         }
     }
 }
 
 // Contains the firmware required to fully reset GSP on chipsets where the GSP is started using
 // FWSEC/Booter.
-struct Sec2UnloadBundle {
-    fwsec_sb: FwsecUnloadFirmware,
+struct Sec2UnloadBundle<'a> {
+    fwsec_sb: FwsecUnloadFirmware<'a>,
     booter_unloader: BooterFirmware,
 }
 
-impl UnloadBundle for Sec2UnloadBundle {
+impl UnloadBundle for Sec2UnloadBundle<'_> {
     fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result {
         let dev = ctx.dev();
         let bar = ctx.bar;
@@ -88,7 +87,7 @@ impl UnloadBundle for Sec2UnloadBundle {
         // Log errors but keep going if it fails.
         let fwsec_sb_res = self
             .fwsec_sb
-            .run(dev, bar, ctx.gsp_falcon)
+            .run(dev, ctx.gsp_falcon)
             .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e));
 
         // Remove WPR2 region if set.
@@ -168,7 +167,7 @@ impl Tu102 {
         if self.needs_fwsec_bootloader {
             let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?;
             // Load and run the bootloader, which will load FWSEC-FRTS and run it.
-            fwsec_frts_bl.run(dev, falcon, bar)?;
+            fwsec_frts_bl.run(dev, falcon)?;
         } else {
             // Load and run FWSEC-FRTS directly.
             fwsec_frts.run(dev, falcon)?;
@@ -213,14 +212,14 @@ impl Tu102 {
     }
 
     /// Load and prepare the resources required to properly reset the GSP after it has been stopped.
-    fn build_unload_bundle(
+    fn build_unload_bundle<'gpu>(
         &self,
-        dev: &device::Device<device::Bound>,
+        dev: &'gpu device::Device<device::Bound>,
         chipset: Chipset,
         bios: &Vbios,
         gsp_falcon: &Falcon<'_, GspEngine>,
         sec2_falcon: &Falcon<'_, Sec2>,
-    ) -> Result<crate::gsp::UnloadBundle> {
+    ) -> Result<crate::gsp::UnloadBundle<'gpu>> {
         // Load the FWSEC SB firmware, as well as its bootloader if required.
         let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?;
         let fwsec_sb = if self.needs_fwsec_bootloader {
@@ -241,18 +240,18 @@ impl Tu102 {
             },
             GFP_KERNEL,
         )
-        .map(|b| crate::gsp::UnloadBundle(b))
+        .map(|b| crate::gsp::UnloadBundle(b as KBox<dyn UnloadBundle + 'gpu>))
         .map_err(Into::into)
     }
 }
 
 impl GspHal for Tu102 {
-    fn boot(
+    fn boot<'gpu>(
         &self,
-        gsp: &Gsp,
-        ctx: &mut GspBootContext<'_, '_>,
-        gsp_fw: &GspFirmware,
-    ) -> Result<Option<crate::gsp::UnloadBundle>> {
+        gsp: &Gsp<'gpu>,
+        ctx: &mut GspBootContext<'_, 'gpu>,
+        gsp_fw: &GspFirmware<'gpu>,
+    ) -> Result<Option<crate::gsp::UnloadBundle<'gpu>>> {
         let dev = ctx.dev();
         let bar = ctx.bar;
         let chipset = ctx.chipset;
@@ -317,9 +316,9 @@ impl GspHal for Tu102 {
 
     fn post_boot(
         &self,
-        gsp: &Gsp,
+        gsp: &Gsp<'_>,
         ctx: &mut GspBootContext<'_, '_>,
-        gsp_fw: &GspFirmware,
+        gsp_fw: &GspFirmware<'_>,
     ) -> Result {
         GspSequencer::run(&gsp.cmdq, ctx, &gsp.libos, gsp_fw.bootloader.app_version)?;
 
diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs
index 9a48aa87e7fb19..3c410d65e8e4d2 100644
--- a/drivers/gpu/nova-core/gsp/regs.rs
+++ b/drivers/gpu/nova-core/gsp/regs.rs
@@ -2,11 +2,16 @@
 
 use kernel::io::register;
 
-use crate::regs::NV_PBUS_SW_SCRATCH;
+use crate::{
+    driver::NovaRegisters,
+    regs::NV_PBUS_SW_SCRATCH, //
+};
 
 // PGSP
 
 register! {
+    base: NovaRegisters;
+
     pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 {
         31:0    address;
     }
@@ -15,6 +20,8 @@ register! {
 // PBUS
 
 register! {
+    base: NovaRegisters;
+
     /// Scratch register 0xe used as FRTS firmware error code.
     pub(super) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] {
         31:16   frts_err_code;
diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs
index bcad1421953a74..dae34c11eb05de 100644
--- a/drivers/gpu/nova-core/gsp/sequencer.rs
+++ b/drivers/gpu/nova-core/gsp/sequencer.rs
@@ -138,7 +138,7 @@ pub(crate) struct GspSequencer<'a> {
     /// GSP falcon for core operations.
     gsp_falcon: &'a Falcon<'a, Gsp>,
     /// LibOS memory region init arguments.
-    libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
+    libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
     /// Bootloader application version.
     bootloader_app_version: u32,
     /// Device for logging.
@@ -338,9 +338,9 @@ impl<'a> Iterator for GspSeqIter<'a> {
 
 impl<'a> GspSequencer<'a> {
     pub(crate) fn run(
-        cmdq: &Cmdq,
+        cmdq: &Cmdq<'_>,
         ctx: &'a GspBootContext<'_, '_>,
-        libos: &'a Coherent<[LibosMemoryRegionInitArgument]>,
+        libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>,
         bootloader_app_version: u32,
     ) -> Result {
         let seq_info = loop {
diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs
index 90c642c91a7247..f62e79f56b05a4 100644
--- a/drivers/gpu/nova-core/mctp.rs
+++ b/drivers/gpu/nova-core/mctp.rs
@@ -42,13 +42,25 @@ bitfield! {
         29:28 seq;
         /// Source endpoint ID.
         23:16 seid;
+        /// Destination endpoint ID.
+        15:8 deid;
+        /// MCTP header version.
+        3:0 version;
     }
 }
 
 impl MctpHeader {
-    /// Builds a single-packet MCTP header (`SOM=1`, `EOM=1`, `SEQ=0`, `SEID=0`).
+    /// The MCTP header version that this driver uses.
+    const VERSION: u32 = 1;
+
+    /// Builds the MCTP header of a message that fits in one packet: `SOM` and `EOM` set, the
+    /// version set, and every other field zero.
     pub(crate) fn single_packet() -> Self {
-        Self::zeroed().with_som(true).with_eom(true)
+        Self::zeroed()
+            .with_const_version::<{ Self::VERSION }>()
+            .with_const_deid::<0>()
+            .with_som(true)
+            .with_eom(true)
     }
 
     /// Returns whether this is a complete single-packet message (`SOM=1` and `EOM=1`).
diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs
new file mode 100644
index 00000000000000..8a934c55169fdd
--- /dev/null
+++ b/drivers/gpu/nova-core/mm.rs
@@ -0,0 +1,335 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Memory management subsystems.
+
+#![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))]
+
+/// Implements `From` conversions between a frame-number type and `Bounded<u64, N>`.
+///
+/// Each MMU version module should invoke this for the specific bit widths used by that version's
+/// PTE/PDE bitfield definitions.
+macro_rules! impl_frame_number_bounded {
+    ($type:ty, $bits:literal) => {
+        impl From<Bounded<u64, $bits>> for $type {
+            fn from(val: Bounded<u64, $bits>) -> Self {
+                Self::new(val.get())
+            }
+        }
+
+        impl From<$type> for Bounded<u64, $bits> {
+            fn from(v: $type) -> Self {
+                Bounded::from_expr(v.raw() & ::kernel::bits::genmask_u64(0..=($bits - 1)))
+            }
+        }
+    };
+}
+
+/// Implements `From` conversions between [`Pfn`] and `Bounded<u64, N>` for bitfield interop.
+macro_rules! impl_pfn_bounded {
+    ($bits:literal) => {
+        impl_frame_number_bounded!(Pfn, $bits);
+    };
+}
+
+use core::{
+    fmt::LowerHex,
+    ops, //
+};
+
+use kernel::{
+    bitfield,
+    fmt,
+    gpu::buddy::{
+        GpuBuddy,
+        GpuBuddyParams, //
+    },
+    num::Bounded,
+    prelude::*,
+    ptr::{
+        Alignable,
+        Alignment, //
+    },
+    sizes::SZ_4K, //
+};
+
+use crate::{
+    driver::Bar0,
+    gpu::Chipset, //
+};
+
+pub(crate) use tlb::Tlb;
+
+pub(crate) mod bar_user;
+mod hal;
+pub(super) mod pagetable;
+mod pramin;
+mod regs;
+pub(super) mod tlb;
+pub(super) mod vmm;
+
+/// GPU Memory Manager - owns all core MM components.
+///
+/// Provides centralized ownership of memory management resources:
+/// - [`GpuBuddy`] allocator for VRAM page table allocation.
+/// - [`pramin::Pramin`] for direct VRAM access.
+/// - [`Tlb`] manager for translation buffer flush operations.
+pub(crate) struct GpuMm<'gpu> {
+    buddy: GpuBuddy,
+    pramin: pramin::Pramin<'gpu>,
+    tlb: Pin<KBox<Tlb<'gpu>>>,
+}
+
+impl<'gpu> GpuMm<'gpu> {
+    /// Creates the GPU memory manager.
+    pub(crate) fn new(
+        bar: Bar0<'gpu>,
+        chipset: Chipset,
+        buddy_params: GpuBuddyParams,
+        total_fb_end: VramAddress,
+    ) -> Result<Self> {
+        // PRAMIN covers all physical VRAM (including GSP-reserved areas
+        // above the usable region, e.g. the BAR1 page directory).
+        let vram_region = VramAddress::ZERO..total_fb_end;
+
+        Ok(Self {
+            buddy: GpuBuddy::new(buddy_params)?,
+            pramin: pramin::Pramin::new(bar, chipset, vram_region)?,
+            tlb: KBox::pin_init(Tlb::new(bar), GFP_KERNEL)?,
+        })
+    }
+
+    /// Access the [`GpuBuddy`] allocator.
+    pub(crate) fn buddy(&self) -> &GpuBuddy {
+        &self.buddy
+    }
+
+    /// Access the [`pramin::Pramin`].
+    fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> {
+        &mut self.pramin
+    }
+
+    /// Access the [`Tlb`] manager.
+    pub(crate) fn tlb(&self) -> &Tlb<'gpu> {
+        self.tlb.as_ref().get_ref()
+    }
+}
+
+/// Page size in bytes (4 KiB).
+pub(crate) const PAGE_SIZE: usize = SZ_4K;
+
+/// Physical VRAM address in GPU video memory.
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+#[repr(transparent)]
+pub(crate) struct VramAddress(u64);
+
+impl VramAddress {
+    /// The zero address.
+    pub(crate) const ZERO: Self = Self::from_raw(0);
+
+    /// Creates an address from a raw value.
+    pub(crate) const fn from_raw(addr: u64) -> Self {
+        Self(addr)
+    }
+
+    /// Returns the address as a raw value.
+    pub(crate) const fn into_raw(self) -> u64 {
+        self.0
+    }
+
+    /// Adds `rhs` to this address, returning [`None`] on overflow.
+    pub(crate) const fn checked_add(self, rhs: u64) -> Option<Self> {
+        match self.into_raw().checked_add(rhs) {
+            Some(addr) => Some(Self::from_raw(addr)),
+            None => None,
+        }
+    }
+}
+
+impl Alignable for VramAddress {
+    fn align_down(self, alignment: Alignment) -> Self {
+        Self::from_raw(self.into_raw().align_down(alignment))
+    }
+
+    fn align_up(self, alignment: Alignment) -> Option<Self> {
+        self.into_raw().align_up(alignment).map(Self::from_raw)
+    }
+}
+
+impl LowerHex for VramAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        LowerHex::fmt(&self.into_raw(), f)
+    }
+}
+
+impl fmt::Debug for VramAddress {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_fmt(fmt!("{:#x}", self))
+    }
+}
+
+impl ops::Add<u64> for VramAddress {
+    type Output = Self;
+
+    fn add(self, rhs: u64) -> Self::Output {
+        Self::from_raw(self.into_raw() + rhs)
+    }
+}
+
+impl ops::Sub for VramAddress {
+    type Output = u64;
+
+    fn sub(self, rhs: Self) -> Self::Output {
+        self.into_raw() - rhs.into_raw()
+    }
+}
+
+impl From<Pfn> for VramAddress {
+    fn from(pfn: Pfn) -> Self {
+        Self::from_raw(pfn.raw() << 12)
+    }
+}
+
+bitfield! {
+    /// Virtual address in GPU address space.
+    pub(crate) struct VirtualAddress(u64) {
+        /// Offset within 4KB page.
+        11:0    offset;
+        /// Virtual frame number.
+        63:12   frame_number => Vfn;
+    }
+}
+
+impl VirtualAddress {
+    /// Create a new virtual address from a raw value.
+    #[expect(dead_code)]
+    pub(crate) const fn new(addr: u64) -> Self {
+        Self::from_raw(addr)
+    }
+}
+
+impl From<Vfn> for VirtualAddress {
+    fn from(vfn: Vfn) -> Self {
+        Self::zeroed().with_frame_number(vfn)
+    }
+}
+
+/// Physical Frame Number.
+///
+/// Represents a physical page in VRAM.
+#[repr(transparent)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub(crate) struct Pfn(u64);
+
+impl Pfn {
+    /// Create a new PFN from a frame number.
+    pub(crate) const fn new(frame_number: u64) -> Self {
+        Self(frame_number)
+    }
+
+    /// Get the raw frame number.
+    pub(crate) const fn raw(self) -> u64 {
+        self.0
+    }
+}
+
+impl From<VramAddress> for Pfn {
+    fn from(addr: VramAddress) -> Self {
+        Self::new(addr.into_raw() >> 12)
+    }
+}
+
+impl From<u64> for Pfn {
+    fn from(val: u64) -> Self {
+        Self(val)
+    }
+}
+
+impl From<Pfn> for u64 {
+    fn from(pfn: Pfn) -> Self {
+        pfn.0
+    }
+}
+
+impl_pfn_bounded!(52);
+
+/// Virtual Frame Number.
+///
+/// Represents a virtual page in GPU address space.
+#[repr(transparent)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub(crate) struct Vfn(u64);
+
+impl Vfn {
+    /// Create a new VFN from a frame number.
+    pub(crate) const fn new(frame_number: u64) -> Self {
+        Self(frame_number)
+    }
+
+    /// Get the raw frame number.
+    pub(crate) const fn raw(self) -> u64 {
+        self.0
+    }
+}
+
+impl From<VirtualAddress> for Vfn {
+    fn from(addr: VirtualAddress) -> Self {
+        addr.frame_number()
+    }
+}
+
+impl From<u64> for Vfn {
+    fn from(val: u64) -> Self {
+        Self(val)
+    }
+}
+
+impl From<Vfn> for u64 {
+    fn from(vfn: Vfn) -> Self {
+        vfn.0
+    }
+}
+
+impl_frame_number_bounded!(Vfn, 52);
+
+#[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+pub(crate) mod selftest {
+    use core::ops::Range;
+
+    use kernel::{
+        device,
+        sizes::SizeConstants, //
+    };
+
+    use super::*;
+
+    /// Run MM subsystem self-tests during probe.
+    pub(crate) fn run(
+        dev: &device::Device<device::Bound>,
+        mm: &mut GpuMm<'_>,
+        usable_fb_regions: &[Range<u64>],
+        bar_user: &bar_user::BarUser<'_>,
+        bar1_pdb: u64,
+        chipset: Chipset,
+    ) -> Result {
+        // VRAM span the self-tests are free to overwrite, from the chosen test base.
+        const SELFTEST_SPAN: u64 = u64::SZ_64M;
+
+        let base = usable_fb_regions.iter().find_map(|region| {
+            // Tests rely on this being 8 byte aligned for checking misalignment handling.
+            let base = region.start.align_up(Alignment::new::<8>())?;
+            (base.checked_add(SELFTEST_SPAN)? <= region.end).then_some(base)
+        });
+        let Some(base) = base else {
+            dev_warn!(
+                dev,
+                "PRAMIN: skipping self-tests, no usable VRAM region of {:#x} bytes\n",
+                SELFTEST_SPAN
+            );
+            return Ok(());
+        };
+
+        pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base))?;
+        bar_user::run_self_test(dev, mm, bar_user, bar1_pdb, chipset)
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs
new file mode 100644
index 00000000000000..6e9384f06f43cd
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/bar_user.rs
@@ -0,0 +1,420 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD
+//! for GPU work submission, and applications to access GPU buffers via mmap().
+
+use kernel::{
+    io::Io,
+    new_mutex,
+    prelude::*,
+    sync::Mutex, //
+};
+
+use crate::{
+    driver::Bar1,
+    gpu::Chipset,
+    mm::{
+        vmm::{
+            MappedRange,
+            Vmm, //
+        },
+        GpuMm,
+        Pfn,
+        Vfn,
+        VirtualAddress,
+        VramAddress,
+        PAGE_SIZE, //
+    },
+    num::IntoSafeCast,
+};
+
+#[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+use kernel::device;
+
+/// BAR1 user interface for virtual memory mappings.
+///
+/// Owns the [`Vmm`] for the BAR1 address space.
+#[pin_data]
+pub(crate) struct BarUser<'a> {
+    #[pin]
+    vmm: Mutex<Vmm>,
+    bar1: Bar1<'a>,
+}
+
+impl<'a> BarUser<'a> {
+    /// Create a pin-initializer for [`BarUser`].
+    pub(crate) fn new(
+        pdb_addr: VramAddress,
+        chipset: Chipset,
+        va_size: u64,
+        bar1: Bar1<'a>,
+    ) -> impl PinInit<Self, Error> + 'a {
+        try_pin_init!(Self {
+            vmm <- new_mutex!(
+                Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?,
+                "bar_user_vmm",
+            ),
+            bar1,
+        })
+    }
+
+    /// Map physical pages to a contiguous BAR1 virtual range.
+    pub(crate) fn map(
+        &self,
+        mm: &mut GpuMm<'_>,
+        pfns: &[Pfn],
+        writable: bool,
+    ) -> Result<BarUserAccess<'_>> {
+        if pfns.is_empty() {
+            return Err(EINVAL);
+        }
+        let mut vmm = self.vmm.lock();
+        let mapped = vmm.map_pages(mm, pfns, None, writable)?;
+
+        Ok(BarUserAccess {
+            bar_user: self,
+            mapped: Some(mapped),
+        })
+    }
+}
+
+/// Access object for a mapped BAR1 region.
+///
+/// Borrows the [`BarUser`] managing the mapping for the lifetime of the access object.
+pub(crate) struct BarUserAccess<'a> {
+    bar_user: &'a BarUser<'a>,
+    /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at
+    /// drop time means `release()` was never called.
+    mapped: Option<MappedRange>,
+}
+
+#[expect(dead_code)]
+impl BarUserAccess<'_> {
+    /// Tear down the BAR1 mapping.
+    pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result {
+        let mapped = self.mapped.take().ok_or(EINVAL)?;
+        let mut vmm = self.bar_user.vmm.lock();
+        vmm.unmap_pages(mm, mapped)?;
+        Ok(())
+    }
+
+    /// Returns the active mapping.
+    fn mapped(&self) -> &MappedRange {
+        // `mapped` is only `None` after `take()` in `release`; hence unwrap()
+        // cannot panic here.
+        self.mapped.as_ref().unwrap()
+    }
+
+    /// Get the base virtual address of this mapping.
+    pub(crate) fn base(&self) -> VirtualAddress {
+        VirtualAddress::from(self.mapped().vfn_start)
+    }
+
+    /// Get the total size of the mapped region in bytes.
+    pub(crate) fn size(&self) -> usize {
+        self.mapped().num_pages * PAGE_SIZE
+    }
+
+    /// Get the starting virtual frame number.
+    pub(crate) fn vfn_start(&self) -> Vfn {
+        self.mapped().vfn_start
+    }
+
+    /// Get the number of pages in this mapping.
+    pub(crate) fn num_pages(&self) -> usize {
+        self.mapped().num_pages
+    }
+
+    /// Translate an offset within this mapping to a BAR1 aperture offset.
+    fn bar_offset(&self, offset: usize) -> Result<usize> {
+        if offset >= self.size() {
+            return Err(EINVAL);
+        }
+
+        let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast();
+        let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?;
+        base.checked_add(offset).ok_or(EOVERFLOW)
+    }
+
+    // Fallible accessors with runtime bounds checking.
+
+    /// Read a 32-bit value at the given offset.
+    pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_read32(off)
+    }
+
+    /// Write a 32-bit value at the given offset.
+    pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_write32(value, off)
+    }
+
+    /// Read a 64-bit value at the given offset.
+    pub(crate) fn try_read64(&self, offset: usize) -> Result<u64> {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_read64(off)
+    }
+
+    /// Write a 64-bit value at the given offset.
+    pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result {
+        let off = self.bar_offset(offset)?;
+        self.bar_user.bar1.try_write64(value, off)
+    }
+}
+
+impl Drop for BarUserAccess<'_> {
+    fn drop(&mut self) {
+        if self.mapped.is_some() {
+            kernel::pr_warn!(
+                "BarUserAccess dropped without calling release(). BarUser address space will leak.\n"
+            );
+        }
+        // The inner `MappedRange`'s own `MustUnmapGuard` will also fire,
+        // identifying the leaked VA range.
+    }
+}
+
+/// Run MM subsystem self-tests during probe.
+///
+/// Tests page table infrastructure and `BAR1` MMIO access using the `BAR1`
+/// address space. Uses the `GpuMm`'s buddy allocator to allocate page tables
+/// and test pages as needed.
+#[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+pub(crate) fn run_self_test(
+    dev: &device::Device<device::Bound>,
+    mm: &mut GpuMm<'_>,
+    bar_user: &BarUser<'_>,
+    bar1_pdb: u64,
+    chipset: Chipset,
+) -> Result {
+    use kernel::{
+        gpu::buddy::{
+            GpuBuddyAllocFlags,
+            GpuBuddyAllocMode, //
+        },
+        ptr::Alignment,
+        sizes::{
+            SZ_16K,
+            SZ_32K,
+            SZ_4K,
+            SZ_64K, //
+        },
+    };
+
+    // Test patterns.
+    const PATTERN_PRAMIN: u32 = 0xDEAD_BEEF;
+    const PATTERN_BAR1: u32 = 0xCAFE_BABE;
+
+    let bar1 = bar_user.bar1;
+    dev_info!(dev, "MM: Starting self-test...\n");
+
+    let pdb_addr = VramAddress::from_raw(bar1_pdb);
+
+    // Check if initial page tables are in VRAM.
+    if crate::mm::pagetable::check_pdb_valid(mm.pramin_mut(), pdb_addr, chipset).is_err() {
+        dev_info!(dev, "MM: Self-test SKIPPED - no valid VRAM page tables\n");
+        return Ok(());
+    }
+
+    // Set up a test page from the buddy allocator.
+    let test_page_blocks = KBox::pin_init(
+        mm.buddy().alloc_blocks(
+            GpuBuddyAllocMode::Simple,
+            SZ_4K.into_safe_cast(),
+            Alignment::new::<SZ_4K>(),
+            GpuBuddyAllocFlags::default(),
+        ),
+        GFP_KERNEL,
+    )?;
+    let test_vram_offset = test_page_blocks.iter().next().ok_or(ENOMEM)?.offset();
+    let test_vram = VramAddress::from_raw(test_vram_offset);
+    let test_pfn = Pfn::from(test_vram);
+
+    // Create a VMM of size 64K to track virtual memory mappings.
+    let mut vmm = Vmm::new(pdb_addr, chipset.mmu_version(), SZ_64K.into_safe_cast())?;
+
+    // Create a test mapping.
+    let mapped = vmm.map_pages(mm, &[test_pfn], None, true)?;
+    let test_vfn = mapped.vfn_start;
+
+    // Pre-compute test addresses for the PRAMIN to BAR1 read test.
+    let vfn_offset: usize = test_vfn.raw().into_safe_cast();
+    let bar1_base_offset = vfn_offset.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?;
+    let bar1_read_offset: usize = bar1_base_offset + 0x100;
+    let vram_read_addr = test_vram + 0x100;
+
+    // Test 1: Write via PRAMIN, read via BAR1.
+    mm.pramin_mut()
+        .window_at::<u32>(vram_read_addr)?
+        .view()
+        .write_val(PATTERN_PRAMIN);
+
+    // Read back via BAR1 aperture.
+    let bar1_value = bar1.try_read32(bar1_read_offset)?;
+
+    let test1_passed = if bar1_value == PATTERN_PRAMIN {
+        true
+    } else {
+        dev_err!(
+            dev,
+            "MM: Test 1 FAILED - Expected {:#010x}, got {:#010x}\n",
+            PATTERN_PRAMIN,
+            bar1_value
+        );
+        false
+    };
+
+    // Cleanup - invalidate PTE.
+    vmm.unmap_pages(mm, mapped)?;
+
+    // Test 2: Two-phase prepare/execute API.
+    let prepared = vmm.prepare_map(mm, 1, None)?;
+    let mapped2 = vmm.execute_map(mm, prepared, &[test_pfn], true)?;
+    let readback = vmm.read_mapping(mm, mapped2.vfn_start)?;
+    let test2_passed = if readback == Some(test_pfn) {
+        true
+    } else {
+        dev_err!(dev, "MM: Test 2 FAILED - Two-phase map readback mismatch\n");
+        false
+    };
+    vmm.unmap_pages(mm, mapped2)?;
+
+    // Test 3: Range-constrained allocation with a hole — exercises block.size()-driven
+    // BAR1 mapping. A 4K hole is punched at base+16K, then a single 32K allocation
+    // is requested within [base, base+36K). The buddy allocator must split around the
+    // hole, returning multiple blocks (expected: {16K, 4K, 8K, 4K} = 32K total).
+    // Each block is mapped into BAR1 and verified via PRAMIN read-back.
+    //
+    // Address layout (base = 0x10000):
+    //   [    16K    ] [HOLE 4K] [4K] [ 8K ] [4K]
+    //   0x10000       0x14000  0x15000 0x16000 0x18000 0x19000
+    let range_base: u64 = SZ_64K.into_safe_cast();
+    let sz_4k: u64 = SZ_4K.into_safe_cast();
+    let sz_16k: u64 = SZ_16K.into_safe_cast();
+    let sz_32k_4k: u64 = (SZ_32K + SZ_4K).into_safe_cast();
+
+    // Punch a 4K hole at base+16K so the subsequent 32K allocation must split.
+    let _hole = KBox::pin_init(
+        mm.buddy().alloc_blocks(
+            GpuBuddyAllocMode::Range(range_base + sz_16k..range_base + sz_16k + sz_4k),
+            SZ_4K.into_safe_cast(),
+            Alignment::new::<SZ_4K>(),
+            GpuBuddyAllocFlags::default(),
+        ),
+        GFP_KERNEL,
+    )?;
+
+    // Allocate 32K within [base, base+36K). The hole forces the allocator to return
+    // split blocks whose sizes are determined by buddy alignment.
+    let blocks = KBox::pin_init(
+        mm.buddy().alloc_blocks(
+            GpuBuddyAllocMode::Range(range_base..range_base + sz_32k_4k),
+            SZ_32K.into_safe_cast(),
+            Alignment::new::<SZ_4K>(),
+            GpuBuddyAllocFlags::default(),
+        ),
+        GFP_KERNEL,
+    )?;
+
+    let mut test3_passed = true;
+    let mut total_size = 0usize;
+
+    for block in blocks.iter() {
+        total_size += IntoSafeCast::<usize>::into_safe_cast(block.size());
+
+        // Map all pages of this block.
+        let page_size: u64 = PAGE_SIZE.into_safe_cast();
+        let num_pages: usize = (block.size() / page_size).into_safe_cast();
+
+        let mut pfns = KVec::new();
+        for j in 0..num_pages {
+            let j_u64: u64 = j.into_safe_cast();
+            pfns.push(
+                Pfn::from(VramAddress::from_raw(
+                    block.offset() + j_u64.checked_mul(page_size).ok_or(EOVERFLOW)?,
+                )),
+                GFP_KERNEL,
+            )?;
+        }
+
+        let mapped = vmm.map_pages(mm, &pfns, None, true)?;
+        let bar1_base_vfn: usize = mapped.vfn_start.raw().into_safe_cast();
+        let bar1_base = bar1_base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?;
+
+        for j in 0..num_pages {
+            let page_bar1_off = bar1_base + j * PAGE_SIZE;
+            let j_u64: u64 = j.into_safe_cast();
+            let page_phys = block.offset()
+                + j_u64
+                    .checked_mul(PAGE_SIZE.into_safe_cast())
+                    .ok_or(EOVERFLOW)?;
+
+            bar1.try_write32(PATTERN_BAR1, page_bar1_off)?;
+
+            let pramin_val = mm
+                .pramin_mut()
+                .window_at::<u32>(VramAddress::from_raw(page_phys))?
+                .view()
+                .read_val();
+
+            if pramin_val != PATTERN_BAR1 {
+                dev_err!(
+                    dev,
+                    "MM: Test 3 FAILED block offset {:#x} page {} (val={:#x})\n",
+                    block.offset(),
+                    j,
+                    pramin_val
+                );
+                test3_passed = false;
+            }
+        }
+
+        vmm.unmap_pages(mm, mapped)?;
+    }
+
+    // Verify aggregate: all returned block sizes must sum to allocation size.
+    if total_size != SZ_32K {
+        dev_err!(
+            dev,
+            "MM: Test 3 FAILED - total size {} != expected {}\n",
+            total_size,
+            SZ_32K
+        );
+        test3_passed = false;
+    }
+
+    // Release Tests 1-3's Vmm before Test 4 uses the GPU's BarUser on the same PDB.
+    drop(vmm);
+
+    // Test 4: Exercise `BarUser::map()` end-to-end.
+    let access = bar_user.map(mm, &[test_pfn], true)?;
+
+    // Write pattern via PRAMIN, read via BarUserAccess.
+    mm.pramin_mut()
+        .window_at::<u32>(test_vram)?
+        .view()
+        .write_val(PATTERN_BAR1);
+
+    let readback = access.try_read32(0)?;
+    let test4_passed = if readback == PATTERN_BAR1 {
+        true
+    } else {
+        dev_err!(
+            dev,
+            "MM: Test 4 FAILED - Expected {:#010x}, got {:#010x}\n",
+            PATTERN_BAR1,
+            readback
+        );
+        false
+    };
+    access.release(mm)?;
+
+    if test1_passed && test2_passed && test3_passed && test4_passed {
+        dev_info!(dev, "MM: All self-tests PASSED\n");
+        Ok(())
+    } else {
+        dev_err!(dev, "MM: Self-tests FAILED\n");
+        Err(EIO)
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/hal.rs b/drivers/gpu/nova-core/mm/hal.rs
new file mode 100644
index 00000000000000..e7fd1e38bd3815
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/hal.rs
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Memory management HAL.
+
+use kernel::{
+    num::Bounded,
+    prelude::*, //
+};
+
+use crate::{
+    driver::Bar0,
+    gpu::{
+        Architecture,
+        Chipset, //
+    },
+    mm::VramAddress, //
+};
+
+mod gb100;
+mod gh100;
+mod tu102;
+
+/// Trait implemented by per-architecture MM HALs.
+///
+/// `Sync` is required so that the `&'static dyn MmHal` references can be stored in `Send`
+/// structures.
+pub(super) trait MmHal: Sync {
+    /// Positions the PRAMIN window at `base`.
+    ///
+    /// This fails if `base` is not aligned to the 64 KiB window alignment or is too large for
+    /// the receiving register.
+    fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result;
+}
+
+/// Returns the HAL corresponding to `chipset`.
+pub(super) fn mm_hal(chipset: Chipset) -> &'static dyn MmHal {
+    match chipset.arch() {
+        Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL,
+        Architecture::Hopper => gh100::GH100_HAL,
+        Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => gb100::GB100_HAL,
+    }
+}
+
+/// Converts `base` into the value of the window-base register field.
+///
+/// Fails with [`EINVAL`] if `base` is not aligned to the window alignment required by the register
+/// field's shift, or if the shifted value does not fit within `RES` bits.
+fn window_base<const RES: u32>(base: VramAddress) -> Result<Bounded<u64, RES>> {
+    const WINDOW_BASE_SHIFT: u32 = 16;
+
+    Bounded::<u64, 64>::from(base.into_raw())
+        .shr_exact::<WINDOW_BASE_SHIFT, { 64 - WINDOW_BASE_SHIFT }>()
+        .and_then(Bounded::try_shrink)
+        .ok_or(EINVAL)
+}
diff --git a/drivers/gpu/nova-core/mm/hal/gb100.rs b/drivers/gpu/nova-core/mm/hal/gb100.rs
new file mode 100644
index 00000000000000..3781e143dea7f2
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/hal/gb100.rs
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Blackwell GB10x/GB20x memory management HAL.
+
+use kernel::{
+    io::Io,
+    prelude::*, //
+};
+
+use crate::{
+    driver::Bar0,
+    mm::{
+        hal::{
+            window_base,
+            MmHal, //
+        },
+        regs,
+        VramAddress, //
+    },
+};
+
+struct Gb100;
+
+impl MmHal for Gb100 {
+    fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result {
+        bar.write_reg(
+            regs::gb100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()),
+        );
+        Ok(())
+    }
+}
+
+const GB100: Gb100 = Gb100;
+pub(super) const GB100_HAL: &dyn MmHal = &GB100;
diff --git a/drivers/gpu/nova-core/mm/hal/gh100.rs b/drivers/gpu/nova-core/mm/hal/gh100.rs
new file mode 100644
index 00000000000000..8af384db2921d5
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/hal/gh100.rs
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Hopper memory management HAL.
+
+use kernel::{
+    io::Io,
+    prelude::*, //
+};
+
+use crate::{
+    driver::Bar0,
+    mm::{
+        hal::{
+            window_base,
+            MmHal, //
+        },
+        regs,
+        VramAddress, //
+    },
+};
+
+struct Gh100;
+
+impl MmHal for Gh100 {
+    fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result {
+        bar.write_reg(
+            regs::gh100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()),
+        );
+        Ok(())
+    }
+}
+
+const GH100: Gh100 = Gh100;
+pub(super) const GH100_HAL: &dyn MmHal = &GH100;
diff --git a/drivers/gpu/nova-core/mm/hal/tu102.rs b/drivers/gpu/nova-core/mm/hal/tu102.rs
new file mode 100644
index 00000000000000..e4fe7561223cf4
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/hal/tu102.rs
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Turing, Ampere and Ada memory management HAL.
+
+use kernel::{
+    io::Io,
+    prelude::*, //
+};
+
+use crate::{
+    driver::Bar0,
+    mm::{
+        hal::{
+            window_base,
+            MmHal, //
+        },
+        regs,
+        VramAddress, //
+    },
+};
+
+struct Tu102;
+
+impl MmHal for Tu102 {
+    fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result {
+        bar.write_reg(
+            regs::NV_PBUS_BAR0_WINDOW::zeroed()
+                .with_target(regs::Bar0WindowTarget::VidMem)
+                .with_base(window_base(base)?.cast()),
+        );
+        Ok(())
+    }
+}
+
+const TU102: Tu102 = Tu102;
+pub(super) const TU102_HAL: &dyn MmHal = &TU102;
diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs
new file mode 100644
index 00000000000000..63a7e1855caa30
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/pagetable.rs
@@ -0,0 +1,424 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Common page table types shared between MMU v2 and v3.
+//!
+//! This module provides foundational types used by both MMU versions:
+//! - Page table level hierarchy
+//! - Memory aperture types for PDEs and PTEs
+
+#![expect(dead_code)]
+
+pub(super) mod map;
+pub(super) mod ver2;
+pub(super) mod ver3;
+pub(super) mod walk;
+
+use kernel::{
+    io::Io,
+    num::Bounded,
+    prelude::*, //
+};
+
+use crate::{
+    gpu::Architecture,
+    mm::{
+        pramin,
+        Pfn,
+        VirtualAddress,
+        VramAddress, //
+    },
+};
+
+/// Extracts the page table index at a given level from a virtual address.
+pub(super) trait VaLevelIndex {
+    /// Return the page table index at `level` for this virtual address.
+    fn level_index(&self, level: u64) -> u64;
+}
+
+/// MMU version enumeration.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum MmuVersion {
+    /// MMU v2 for Turing/Ampere/Ada.
+    V2,
+    /// MMU v3 for Hopper and later.
+    V3,
+}
+
+impl From<Architecture> for MmuVersion {
+    fn from(arch: Architecture) -> Self {
+        match arch {
+            Architecture::Turing | Architecture::Ampere | Architecture::Ada => Self::V2,
+            Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => {
+                Self::V3
+            }
+        }
+    }
+}
+
+/// Page Table Level hierarchy for MMU v2/v3.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum PageTableLevel {
+    /// Level 0 - Page Directory Base (root).
+    Pdb,
+    /// Level 1 - Intermediate page directory.
+    L1,
+    /// Level 2 - Intermediate page directory.
+    L2,
+    /// Level 3 - Intermediate page directory or dual PDE (version-dependent).
+    L3,
+    /// Level 4 - PTE level for v2, intermediate page directory for v3.
+    L4,
+    /// Level 5 - PTE level used for MMU v3 only.
+    L5,
+}
+
+impl PageTableLevel {
+    /// Number of entries per page table (512 for 4KB pages).
+    pub(super) const ENTRIES_PER_TABLE: usize = 512;
+
+    /// Get the next level in the hierarchy.
+    pub(super) const fn next(&self) -> Option<PageTableLevel> {
+        match self {
+            Self::Pdb => Some(Self::L1),
+            Self::L1 => Some(Self::L2),
+            Self::L2 => Some(Self::L3),
+            Self::L3 => Some(Self::L4),
+            Self::L4 => Some(Self::L5),
+            Self::L5 => None,
+        }
+    }
+
+    /// Convert level to index.
+    pub(super) const fn as_index(&self) -> u64 {
+        match self {
+            Self::Pdb => 0,
+            Self::L1 => 1,
+            Self::L2 => 2,
+            Self::L3 => 3,
+            Self::L4 => 4,
+            Self::L5 => 5,
+        }
+    }
+}
+
+// Trait abstractions for page table operations.
+
+/// Operations on Page Table Entries (`PTE`s).
+pub(super) trait PteOps: Copy + core::fmt::Debug + Into<u64> {
+    /// Create a `PTE` from a raw `u64` value.
+    fn from_raw(val: u64) -> Self;
+
+    /// Create an invalid `PTE`.
+    fn invalid() -> Self;
+
+    /// Create a valid `PTE` for the given memory aperture.
+    fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self;
+
+    /// Check if this `PTE` is valid.
+    fn is_valid(&self) -> bool;
+
+    /// Get the physical frame number.
+    fn frame_number(&self) -> Pfn;
+
+    /// Read a `PTE` from VRAM.
+    fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result<Self> {
+        let val = pramin.window_at::<u64>(addr)?.view().read_val();
+        Ok(Self::from_raw(val))
+    }
+
+    /// Write this `PTE` to VRAM.
+    fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result {
+        pramin
+            .window_at::<u64>(addr)?
+            .view()
+            .write_val((*self).into());
+        Ok(())
+    }
+}
+
+/// Operations on Page Directory Entries (`PDE`s).
+pub(super) trait PdeOps: Copy + core::fmt::Debug + Into<u64> {
+    /// Create a `PDE` from a raw `u64` value.
+    fn from_raw(val: u64) -> Self;
+
+    /// Create a valid `PDE` pointing to a page table in the given aperture.
+    fn new(aperture: AperturePde, table_pfn: Pfn) -> Self;
+
+    /// Create an invalid `PDE`.
+    fn invalid() -> Self;
+
+    /// Check if this `PDE` is valid.
+    fn is_valid(&self) -> bool;
+
+    /// Get the memory aperture of this `PDE`.
+    fn aperture(&self) -> AperturePde;
+
+    /// Get the VRAM address of the page table.
+    fn table_vram_address(&self) -> VramAddress;
+
+    /// Read a `PDE` from VRAM.
+    fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result<Self> {
+        let val = pramin.window_at::<u64>(addr)?.view().read_val();
+        Ok(Self::from_raw(val))
+    }
+
+    /// Write this `PDE` to VRAM.
+    fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result {
+        pramin
+            .window_at::<u64>(addr)?
+            .view()
+            .write_val((*self).into());
+        Ok(())
+    }
+
+    /// Check if this `PDE` is valid and points to video memory.
+    fn is_valid_vram(&self) -> bool {
+        self.is_valid() && self.aperture() == AperturePde::VideoMemory
+    }
+}
+
+/// Operations on Dual Page Directory Entries (128-bit `DualPde`s).
+pub(super) trait DualPdeOps: Copy + core::fmt::Debug {
+    /// Create a `DualPde` from raw 128-bit value (two `u64`s).
+    fn from_raw(big: u64, small: u64) -> Self;
+
+    /// Create a `DualPde` with only the small page table pointer set.
+    fn new_small(table_pfn: Pfn) -> Self;
+
+    /// Check if the small page table pointer is valid.
+    fn has_small(&self) -> bool;
+
+    /// Get the small page table VRAM address.
+    fn small_vram_address(&self) -> VramAddress;
+
+    /// Get the raw `u64` value of the big PDE.
+    fn big_raw_u64(&self) -> u64;
+
+    /// Get the raw `u64` value of the small PDE.
+    fn small_raw_u64(&self) -> u64;
+
+    /// Read a dual PDE (128-bit) from VRAM.
+    fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result<Self> {
+        let lo = pramin.window_at::<u64>(addr)?.view().read_val();
+        let hi = pramin.window_at::<u64>(addr + 8)?.view().read_val();
+        Ok(Self::from_raw(lo, hi))
+    }
+
+    /// Write this dual PDE (128-bit) to VRAM.
+    fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result {
+        pramin
+            .window_at::<u64>(addr)?
+            .view()
+            .write_val(self.big_raw_u64());
+        pramin
+            .window_at::<u64>(addr + 8)?
+            .view()
+            .write_val(self.small_raw_u64());
+        Ok(())
+    }
+}
+
+/// MMU configuration trait -- encodes version-specific constants and types.
+pub(super) trait MmuConfig: 'static {
+    /// Page Table Entry type.
+    type Pte: PteOps;
+    /// Page Directory Entry type.
+    type Pde: PdeOps;
+    /// Dual Page Directory Entry type (128-bit).
+    type DualPde: DualPdeOps;
+
+    /// PDE levels (excluding PTE level) for page table walking.
+    const PDE_LEVELS: &'static [PageTableLevel];
+    /// PTE level for this MMU version.
+    const PTE_LEVEL: PageTableLevel;
+    /// Dual PDE level (128-bit entries) for this MMU version.
+    const DUAL_PDE_LEVEL: PageTableLevel;
+
+    /// Get the number of entries per page table page for a given level.
+    fn entries_per_page(level: PageTableLevel) -> usize;
+
+    /// Extract the page table index at `level` from `va`.
+    fn level_index(va: VirtualAddress, level: u64) -> u64;
+
+    /// Get the entry size in bytes for a given level.
+    fn entry_size(level: PageTableLevel) -> usize {
+        if level == Self::DUAL_PDE_LEVEL {
+            16 // 128-bit dual PDE
+        } else {
+            8 // 64-bit PDE/PTE
+        }
+    }
+
+    /// Compute upper bound on page table pages needed for `num_virt_pages`.
+    ///
+    /// Walks from PTE level up through PDE levels, accumulating the tree.
+    fn pt_pages_upper_bound(num_virt_pages: usize) -> usize {
+        let mut total = 0;
+
+        // PTE pages at the leaf level.
+        let pte_epp = Self::entries_per_page(Self::PTE_LEVEL);
+        let mut pages_at_level = num_virt_pages.div_ceil(pte_epp);
+        total += pages_at_level;
+
+        // Walk PDE levels bottom-up (reverse of PDE_LEVELS).
+        for &level in Self::PDE_LEVELS.iter().rev() {
+            let epp = Self::entries_per_page(level);
+
+            // How many pages at this level do we need to point to
+            // the previous pages_at_level?
+            pages_at_level = pages_at_level.div_ceil(epp);
+            total += pages_at_level;
+        }
+
+        total
+    }
+}
+
+/// Marker struct for MMU v2 (Turing/Ampere/Ada).
+pub(super) struct MmuV2;
+
+impl MmuConfig for MmuV2 {
+    type Pte = ver2::Pte;
+    type Pde = ver2::Pde;
+    type DualPde = ver2::DualPde;
+
+    const PDE_LEVELS: &'static [PageTableLevel] = ver2::PDE_LEVELS;
+    const PTE_LEVEL: PageTableLevel = ver2::PTE_LEVEL;
+    const DUAL_PDE_LEVEL: PageTableLevel = ver2::DUAL_PDE_LEVEL;
+
+    fn entries_per_page(level: PageTableLevel) -> usize {
+        // TODO: Calculate these values from the bitfield dynamically
+        // instead of hardcoding them.
+        match level {
+            PageTableLevel::Pdb => 4,  // PD3 root: bits [48:47] = 2 bits
+            PageTableLevel::L3 => 256, // PD0 dual: bits [28:21] = 8 bits
+            _ => 512,                  // PD2, PD1, PT: 9 bits each
+        }
+    }
+
+    fn level_index(va: VirtualAddress, level: u64) -> u64 {
+        ver2::VirtualAddressV2::new(va).level_index(level)
+    }
+}
+
+/// Marker struct for MMU v3 (Hopper and later).
+pub(super) struct MmuV3;
+
+impl MmuConfig for MmuV3 {
+    type Pte = ver3::Pte;
+    type Pde = ver3::Pde;
+    type DualPde = ver3::DualPde;
+
+    const PDE_LEVELS: &'static [PageTableLevel] = ver3::PDE_LEVELS;
+    const PTE_LEVEL: PageTableLevel = ver3::PTE_LEVEL;
+    const DUAL_PDE_LEVEL: PageTableLevel = ver3::DUAL_PDE_LEVEL;
+
+    fn entries_per_page(level: PageTableLevel) -> usize {
+        match level {
+            PageTableLevel::Pdb => 2,  // PDE4 root: bit [56] = 1 bit, 2 entries
+            PageTableLevel::L4 => 256, // PDE0 dual: bits [28:21] = 8 bits
+            _ => 512,                  // PDE3, PDE2, PDE1, PT: 9 bits each
+        }
+    }
+
+    fn level_index(va: VirtualAddress, level: u64) -> u64 {
+        ver3::VirtualAddressV3::new(va).level_index(level)
+    }
+}
+
+/// Memory aperture for Page Table Entries (`PTE`s).
+///
+/// Determines which memory region the `PTE` points to.
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub(super) enum AperturePte {
+    /// Local video memory (VRAM).
+    #[default]
+    VideoMemory = 0,
+    /// Peer GPU's video memory.
+    PeerMemory = 1,
+    /// System memory with cache coherence.
+    SystemCoherent = 2,
+    /// System memory without cache coherence.
+    SystemNonCoherent = 3,
+}
+
+// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available.
+impl From<Bounded<u64, 2>> for AperturePte {
+    fn from(val: Bounded<u64, 2>) -> Self {
+        match *val {
+            0 => Self::VideoMemory,
+            1 => Self::PeerMemory,
+            2 => Self::SystemCoherent,
+            3 => Self::SystemNonCoherent,
+            _ => Self::VideoMemory,
+        }
+    }
+}
+
+// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available.
+impl From<AperturePte> for Bounded<u64, 2> {
+    fn from(val: AperturePte) -> Self {
+        Bounded::from_expr(val as u64 & 0x3)
+    }
+}
+
+/// Memory aperture for Page Directory Entries (`PDE`s).
+///
+/// Note: For `PDE`s, `Invalid` (0) means the entry is not valid.
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub(super) enum AperturePde {
+    /// Invalid/unused entry.
+    #[default]
+    Invalid = 0,
+    /// Page table is in video memory.
+    VideoMemory = 1,
+    /// Page table is in system memory with coherence.
+    SystemCoherent = 2,
+    /// Page table is in system memory without coherence.
+    SystemNonCoherent = 3,
+}
+
+// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available.
+impl From<Bounded<u64, 2>> for AperturePde {
+    fn from(val: Bounded<u64, 2>) -> Self {
+        match *val {
+            1 => Self::VideoMemory,
+            2 => Self::SystemCoherent,
+            3 => Self::SystemNonCoherent,
+            _ => Self::Invalid,
+        }
+    }
+}
+
+// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available.
+impl From<AperturePde> for Bounded<u64, 2> {
+    fn from(val: AperturePde) -> Self {
+        Bounded::from_expr(val as u64 & 0x3)
+    }
+}
+
+/// Check if the PDB has valid, VRAM-backed page tables.
+#[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+fn check_pdb_inner<M: MmuConfig>(pramin: &mut pramin::Pramin<'_>, pdb_addr: VramAddress) -> Result {
+    let raw = pramin.window_at::<u64>(pdb_addr)?.view().read_val();
+
+    if !M::Pde::from_raw(raw).is_valid_vram() {
+        return Err(ENOENT);
+    }
+    Ok(())
+}
+
+/// Check if the PDB has valid, VRAM-backed page tables, dispatching by MMU version.
+#[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+pub(super) fn check_pdb_valid(
+    pramin: &mut pramin::Pramin<'_>,
+    pdb_addr: VramAddress,
+    chipset: crate::gpu::Chipset,
+) -> Result {
+    match MmuVersion::from(chipset.arch()) {
+        MmuVersion::V2 => check_pdb_inner::<MmuV2>(pramin, pdb_addr),
+        MmuVersion::V3 => check_pdb_inner::<MmuV3>(pramin, pdb_addr),
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/pagetable/map.rs b/drivers/gpu/nova-core/mm/pagetable/map.rs
new file mode 100644
index 00000000000000..77431c509a8958
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/pagetable/map.rs
@@ -0,0 +1,345 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Page table mapping operations for NVIDIA GPUs.
+
+use core::marker::PhantomData;
+
+use kernel::{
+    gpu::buddy::{
+        AllocatedBlocks,
+        GpuBuddyAllocFlags,
+        GpuBuddyAllocMode, //
+    },
+    io::io_write,
+    prelude::*,
+    ptr::Alignment,
+    rbtree::{
+        RBTree,
+        RBTreeNode, //
+    },
+    sizes::SZ_4K, //
+};
+
+use super::{
+    walk::{
+        PtWalkInner,
+        WalkPdeResult,
+        WalkResult, //
+    },
+    AperturePde,
+    AperturePte,
+    DualPdeOps,
+    MmuConfig,
+    MmuV2,
+    MmuV3,
+    MmuVersion,
+    PageTableLevel,
+    PdeOps,
+    PteOps, //
+};
+use crate::{
+    mm::{
+        GpuMm,
+        Pfn,
+        Vfn,
+        VramAddress,
+        PAGE_SIZE, //
+    },
+    num::{
+        IntoSafeCast, //
+    },
+};
+
+/// A pre-allocated and zeroed page table page.
+///
+/// Created during the mapping prepare phase and consumed during the execute phase.
+/// Stored in an [`RBTree`] keyed by the PDE slot address (`install_addr`).
+pub(in crate::mm) struct PreparedPtPage {
+    /// The allocated and zeroed page table page.
+    pub(in crate::mm) alloc: Pin<KBox<AllocatedBlocks>>,
+    /// Page table level -- needed to determine if this PT page is for a dual PDE.
+    pub(in crate::mm) level: PageTableLevel,
+}
+
+/// Page table mapper.
+pub(in crate::mm) struct PtMapInner<M: MmuConfig> {
+    walker: PtWalkInner<M>,
+    pdb_addr: VramAddress,
+    _phantom: PhantomData<M>,
+}
+
+impl<M: MmuConfig> PtMapInner<M> {
+    /// Create a new [`PtMapInner`].
+    pub(super) fn new(pdb_addr: VramAddress) -> Self {
+        Self {
+            walker: PtWalkInner::<M>::new(pdb_addr),
+            pdb_addr,
+            _phantom: PhantomData,
+        }
+    }
+
+    /// Allocate and zero a physical page table page.
+    fn alloc_and_zero_page(mm: &mut GpuMm<'_>, level: PageTableLevel) -> Result<PreparedPtPage> {
+        let blocks = KBox::pin_init(
+            mm.buddy().alloc_blocks(
+                GpuBuddyAllocMode::Simple,
+                SZ_4K.into_safe_cast(),
+                Alignment::new::<SZ_4K>(),
+                GpuBuddyAllocFlags::default(),
+            ),
+            GFP_KERNEL,
+        )?;
+
+        let page_vram = VramAddress::from_raw(blocks.iter().next().ok_or(ENOMEM)?.offset());
+
+        // Zero via PRAMIN.
+        let window = mm
+            .pramin_mut()
+            .window_at::<[u64; PAGE_SIZE / 8]>(page_vram)?;
+        for i in 0..PAGE_SIZE / 8 {
+            io_write!(window.view(), [build: i], 0);
+        }
+
+        Ok(PreparedPtPage {
+            alloc: blocks,
+            level,
+        })
+    }
+
+    /// Ensure all intermediate page table pages exist for a single VFN.
+    ///
+    /// The mutable PRAMIN borrow ends before each allocation.
+    fn ensure_single_pte_path(
+        &self,
+        mm: &mut GpuMm<'_>,
+        vfn: Vfn,
+        pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
+    ) -> Result {
+        let max_iter = 2 * M::PDE_LEVELS.len();
+
+        for _ in 0..max_iter {
+            let result = self
+                .walker
+                .walk_pde_levels(mm.pramin_mut(), vfn, |install_addr| {
+                    pt_pages.get(&install_addr).and_then(|p| {
+                        p.alloc
+                            .iter()
+                            .next()
+                            .map(|b| VramAddress::from_raw(b.offset()))
+                    })
+                })?;
+
+            match result {
+                WalkPdeResult::Complete { .. } => {
+                    return Ok(());
+                }
+                WalkPdeResult::Missing {
+                    install_addr,
+                    level,
+                } => {
+                    let page = Self::alloc_and_zero_page(mm, level)?;
+                    let node = RBTreeNode::new(install_addr, page, GFP_KERNEL)?;
+                    let old = pt_pages.insert(node);
+                    if old.is_some() {
+                        kernel::pr_warn_once!(
+                            "VMM: duplicate install_addr in pt_pages (internal consistency error)\n"
+                        );
+                        return Err(EIO);
+                    }
+                }
+            }
+        }
+
+        kernel::pr_warn!(
+            "VMM: ensure_pte_path: loop exhausted after {} iters (VFN {:?})\n",
+            max_iter,
+            vfn
+        );
+        Err(EIO)
+    }
+
+    /// Prepare page table resources for mapping `num_pages` pages starting at `vfn_start`.
+    ///
+    /// Reserves capacity in `page_table_allocs`, then walks the hierarchy
+    /// per-VFN to prepare pages for all missing PDEs.
+    pub(super) fn prepare_map(
+        &self,
+        mm: &mut GpuMm<'_>,
+        vfn_start: Vfn,
+        num_pages: usize,
+        page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
+        pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
+    ) -> Result {
+        // Pre-reserve so install_mappings() can use push_within_capacity (no alloc
+        // in fence signalling critical path).
+        let pt_upper_bound = M::pt_pages_upper_bound(num_pages);
+        page_table_allocs.reserve(pt_upper_bound, GFP_KERNEL)?;
+
+        // Walk the hierarchy per-VFN to prepare pages for all missing PDEs.
+        for i in 0..num_pages {
+            let i_u64: u64 = i.into_safe_cast();
+            let vfn = Vfn::new(vfn_start.raw() + i_u64);
+            self.ensure_single_pte_path(mm, vfn, pt_pages)?;
+        }
+        Ok(())
+    }
+
+    /// Install prepared PDEs and write PTEs, then flush TLB.
+    ///
+    /// Drains `pt_pages` and moves allocations into `page_table_allocs`.
+    pub(super) fn install_mappings(
+        &self,
+        mm: &mut GpuMm<'_>,
+        pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
+        page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
+        vfn_start: Vfn,
+        pfns: &[Pfn],
+        writable: bool,
+    ) -> Result {
+        {
+            let pramin = mm.pramin_mut();
+
+            // Drain prepared PT pages, install all pending PDEs.
+            let mut cursor = pt_pages.cursor_front_mut();
+            while let Some(c) = cursor {
+                let (next, node) = c.remove_current();
+                let (install_addr, page) = node.to_key_value();
+                let page_vram =
+                    VramAddress::from_raw(page.alloc.iter().next().ok_or(ENOMEM)?.offset());
+
+                if page.level == M::DUAL_PDE_LEVEL {
+                    let new_dpde = M::DualPde::new_small(Pfn::from(page_vram));
+                    new_dpde.write(pramin, install_addr)?;
+                } else {
+                    let new_pde = M::Pde::new(AperturePde::VideoMemory, Pfn::from(page_vram));
+                    new_pde.write(pramin, install_addr)?;
+                }
+
+                page_table_allocs
+                    .push_within_capacity(page.alloc)
+                    .map_err(|_| ENOMEM)?;
+
+                cursor = next;
+            }
+
+            // Write PTEs (all PDEs now installed in HW).
+            for (i, &pfn) in pfns.iter().enumerate() {
+                let i_u64: u64 = i.into_safe_cast();
+                let vfn = Vfn::new(vfn_start.raw() + i_u64);
+                let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?;
+
+                match result {
+                    WalkResult::Unmapped { pte_addr } | WalkResult::Mapped { pte_addr, .. } => {
+                        let pte = M::Pte::new(AperturePte::VideoMemory, pfn, writable);
+                        pte.write(pramin, pte_addr)?;
+                    }
+                    WalkResult::PageTableMissing => {
+                        kernel::pr_warn_once!("VMM: page table missing for VFN {vfn:?}\n");
+                        return Err(EIO);
+                    }
+                }
+            }
+        }
+
+        // Flush TLB.
+        mm.tlb().flush(self.pdb_addr)
+    }
+
+    /// Invalidate PTEs for a range and flush TLB.
+    pub(super) fn invalidate_ptes(
+        &self,
+        mm: &mut GpuMm<'_>,
+        vfn_start: Vfn,
+        num_pages: usize,
+    ) -> Result {
+        let invalid_pte = M::Pte::invalid();
+
+        {
+            let pramin = mm.pramin_mut();
+            for i in 0..num_pages {
+                let i_u64: u64 = i.into_safe_cast();
+                let vfn = Vfn::new(vfn_start.raw() + i_u64);
+                let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?;
+
+                match result {
+                    WalkResult::Mapped { pte_addr, .. } | WalkResult::Unmapped { pte_addr } => {
+                        invalid_pte.write(pramin, pte_addr)?;
+                    }
+                    WalkResult::PageTableMissing => {
+                        continue;
+                    }
+                }
+            }
+        }
+
+        mm.tlb().flush(self.pdb_addr)
+    }
+}
+
+macro_rules! pt_map_dispatch {
+    ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => {
+        match $self {
+            PtMap::V2(inner) => inner.$method($($arg),*),
+            PtMap::V3(inner) => inner.$method($($arg),*),
+        }
+    };
+}
+
+/// Page table mapper dispatch.
+pub(in crate::mm) enum PtMap {
+    /// MMU v2 (Turing/Ampere/Ada).
+    V2(PtMapInner<MmuV2>),
+    /// MMU v3 (Hopper+).
+    V3(PtMapInner<MmuV3>),
+}
+
+impl PtMap {
+    /// Create a new page table mapper for the given MMU version.
+    pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self {
+        match version {
+            MmuVersion::V2 => Self::V2(PtMapInner::<MmuV2>::new(pdb_addr)),
+            MmuVersion::V3 => Self::V3(PtMapInner::<MmuV3>::new(pdb_addr)),
+        }
+    }
+
+    /// Prepare page table resources for a mapping.
+    pub(in crate::mm) fn prepare_map(
+        &self,
+        mm: &mut GpuMm<'_>,
+        vfn_start: Vfn,
+        num_pages: usize,
+        page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
+        pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
+    ) -> Result {
+        pt_map_dispatch!(
+            self,
+            prepare_map(mm, vfn_start, num_pages, page_table_allocs, pt_pages)
+        )
+    }
+
+    /// Install prepared PDEs and write PTEs, then flush TLB.
+    pub(in crate::mm) fn install_mappings(
+        &self,
+        mm: &mut GpuMm<'_>,
+        pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
+        page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
+        vfn_start: Vfn,
+        pfns: &[Pfn],
+        writable: bool,
+    ) -> Result {
+        pt_map_dispatch!(
+            self,
+            install_mappings(mm, pt_pages, page_table_allocs, vfn_start, pfns, writable)
+        )
+    }
+
+    /// Invalidate PTEs for a range and flush TLB.
+    pub(in crate::mm) fn invalidate_ptes(
+        &self,
+        mm: &mut GpuMm<'_>,
+        vfn_start: Vfn,
+        num_pages: usize,
+    ) -> Result {
+        pt_map_dispatch!(self, invalidate_ptes(mm, vfn_start, num_pages))
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/pagetable/ver2.rs b/drivers/gpu/nova-core/mm/pagetable/ver2.rs
new file mode 100644
index 00000000000000..d7169a0fcff978
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/pagetable/ver2.rs
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! MMU v2 page table types for Turing, Ampere and Ada GPUs.
+//!
+//! This module defines MMU version 2 specific types (Turing, Ampere and Ada GPUs).
+//!
+//! Bit field layouts derived from the NVIDIA OpenRM documentation:
+//! `open-gpu-kernel-modules/src/common/inc/swref/published/turing/tu102/dev_mmu.h`
+
+#![allow(dead_code)]
+
+use kernel::{
+    bitfield,
+    num::Bounded, //
+};
+
+use pin_init::Zeroable;
+
+use super::{
+    AperturePde,
+    AperturePte,
+    DualPdeOps,
+    PageTableLevel,
+    PdeOps,
+    PteOps,
+    VaLevelIndex, //
+};
+
+use crate::mm::{
+    Pfn,
+    VirtualAddress,
+    VramAddress, //
+};
+
+// Bounded to version 2 Pfn bitfield conversions:
+// 25 bits for video memory frame numbers (bits 32:8).
+impl_pfn_bounded!(25);
+// 46 bits for system memory frame numbers (bits 53:8).
+impl_pfn_bounded!(46);
+
+bitfield! {
+    /// MMU v2 49-bit virtual address layout.
+    pub(super) struct VirtualAddressV2(u64) {
+        /// Page offset [11:0].
+        11:0    offset;
+        /// PT index [20:12].
+        20:12   pt_idx;
+        /// PDE0 index [28:21].
+        28:21   pde0_idx;
+        /// PDE1 index [37:29].
+        37:29   pde1_idx;
+        /// PDE2 index [46:38].
+        46:38   pde2_idx;
+        /// PDE3 index [48:47].
+        48:47   pde3_idx;
+    }
+}
+
+impl VirtualAddressV2 {
+    /// Create a [`VirtualAddressV2`] from a [`VirtualAddress`].
+    pub(super) fn new(va: VirtualAddress) -> Self {
+        Self::from_raw(va.into_raw())
+    }
+}
+
+impl VaLevelIndex for VirtualAddressV2 {
+    fn level_index(&self, level: u64) -> u64 {
+        match level {
+            0 => *self.pde3_idx(),
+            1 => *self.pde2_idx(),
+            2 => *self.pde1_idx(),
+            3 => *self.pde0_idx(),
+            4 => *self.pt_idx(),
+            _ => 0,
+        }
+    }
+}
+
+/// `PDE` levels for MMU v2 (5-level hierarchy: `PDB` -> `L1` -> `L2` -> `L3` -> `L4`).
+pub(super) const PDE_LEVELS: &[PageTableLevel] = &[
+    PageTableLevel::Pdb,
+    PageTableLevel::L1,
+    PageTableLevel::L2,
+    PageTableLevel::L3,
+];
+
+/// `PTE` level for MMU v2.
+pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L4;
+
+/// Dual `PDE` level for MMU v2 (128-bit entries).
+pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L3;
+
+// Page Table Entry (PTE) for MMU v2 - 64-bit entry at level 4.
+bitfield! {
+    /// Page Table Entry for MMU v2.
+    pub(in crate::mm) struct Pte(u64) {
+        /// Entry is valid.
+        0:0     valid;
+        /// Memory aperture type.
+        2:1     aperture => AperturePte;
+        /// Volatile (bypass L2 cache).
+        3:3     volatile;
+        /// Encryption enabled (Confidential Computing).
+        4:4     encrypted;
+        /// Privileged access only.
+        5:5     privilege;
+        /// Write protection.
+        6:6     read_only;
+        /// Atomic operations disabled.
+        7:7     atomic_disable;
+        /// Frame number for system memory.
+        53:8    frame_number_sys => Pfn;
+        /// Frame number for video memory.
+        32:8    frame_number_vid => Pfn;
+        /// Peer GPU ID for peer memory (0-7).
+        35:33   peer_id;
+        /// Compression tag line bits.
+        53:36   comptagline;
+        /// Surface kind/format.
+        63:56   kind;
+    }
+}
+
+impl PteOps for Pte {
+    fn from_raw(val: u64) -> Self {
+        Self::from_raw(val)
+    }
+
+    fn invalid() -> Self {
+        Self::zeroed()
+    }
+
+    fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self {
+        let base = Self::zeroed()
+            .with_valid(true)
+            .with_aperture(aperture)
+            .with_read_only(!writable);
+        match aperture {
+            AperturePte::VideoMemory => base.with_frame_number_vid(pfn),
+            // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency.
+            AperturePte::SystemCoherent => base.with_frame_number_sys(pfn).with_volatile(true),
+            AperturePte::PeerMemory | AperturePte::SystemNonCoherent => {
+                kernel::pr_warn!("MMU v2 PTE aperture {:?} not supported\n", aperture);
+                Self::invalid()
+            }
+        }
+    }
+
+    fn is_valid(&self) -> bool {
+        self.valid().into_bool()
+    }
+
+    fn frame_number(&self) -> Pfn {
+        match self.aperture() {
+            AperturePte::VideoMemory => self.frame_number_vid(),
+            _ => self.frame_number_sys(),
+        }
+    }
+}
+
+// Page Directory Entry (PDE) for MMU v2 - 64-bit entry at levels 0-2.
+bitfield! {
+    /// Page Directory Entry for MMU v2.
+    pub(in crate::mm) struct Pde(u64) {
+        /// Valid bit (inverted logic).
+        0:0     valid_inverted;
+        /// Memory aperture type.
+        2:1     aperture => AperturePde;
+        /// Volatile (bypass L2 cache).
+        3:3     volatile;
+        /// Disable Address Translation Services.
+        5:5     no_ats;
+        /// Table frame number for system memory.
+        53:8    table_frame_sys => Pfn;
+        /// Table frame number for video memory.
+        32:8    table_frame_vid => Pfn;
+        /// Peer GPU ID (0-7).
+        35:33   peer_id;
+    }
+}
+
+impl PdeOps for Pde {
+    fn from_raw(val: u64) -> Self {
+        Self::from_raw(val)
+    }
+
+    fn new(aperture: AperturePde, table_pfn: Pfn) -> Self {
+        let base = Self::zeroed()
+            .with_valid_inverted(false) // 0 = valid
+            .with_aperture(aperture);
+        match aperture {
+            AperturePde::VideoMemory => base.with_table_frame_vid(table_pfn),
+            // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency.
+            AperturePde::SystemCoherent => base.with_table_frame_sys(table_pfn).with_volatile(true),
+            AperturePde::Invalid | AperturePde::SystemNonCoherent => {
+                kernel::pr_warn!("MMU v2 PDE aperture {:?} not supported\n", aperture);
+                Self::invalid()
+            }
+        }
+    }
+
+    fn invalid() -> Self {
+        Self::zeroed()
+            .with_valid_inverted(true)
+            .with_aperture(AperturePde::Invalid)
+    }
+
+    fn is_valid(&self) -> bool {
+        !self.valid_inverted().into_bool() && self.aperture() != AperturePde::Invalid
+    }
+
+    fn aperture(&self) -> AperturePde {
+        Pde::aperture(*self)
+    }
+
+    fn table_vram_address(&self) -> VramAddress {
+        debug_assert!(
+            Pde::aperture(*self) == AperturePde::VideoMemory,
+            "table_vram_address called on non-VRAM PDE (aperture: {:?})",
+            Pde::aperture(*self)
+        );
+        VramAddress::from(self.table_frame_vid())
+    }
+}
+
+/// Dual `PDE` at Level 3 - 128-bit entry of Large/Small Page Table pointers.
+///
+/// The dual `PDE` supports both large (64KB) and small (4KB) page tables.
+#[repr(C)]
+#[derive(Debug, Clone, Copy)]
+pub(in crate::mm) struct DualPde {
+    /// Large/Big Page Table pointer (lower 64 bits).
+    pub(super) big: Pde,
+    /// Small Page Table pointer (upper 64 bits).
+    pub(super) small: Pde,
+}
+
+impl DualPde {
+    /// Check if the big page table pointer is valid.
+    fn has_big(&self) -> bool {
+        PdeOps::is_valid(&self.big)
+    }
+}
+
+impl DualPdeOps for DualPde {
+    fn from_raw(big: u64, small: u64) -> Self {
+        Self {
+            big: PdeOps::from_raw(big),
+            small: PdeOps::from_raw(small),
+        }
+    }
+
+    fn new_small(table_pfn: Pfn) -> Self {
+        Self {
+            big: PdeOps::from_raw(0),
+            small: PdeOps::new(AperturePde::VideoMemory, table_pfn),
+        }
+    }
+
+    fn has_small(&self) -> bool {
+        PdeOps::is_valid(&self.small)
+    }
+
+    fn small_vram_address(&self) -> VramAddress {
+        PdeOps::table_vram_address(&self.small)
+    }
+
+    fn big_raw_u64(&self) -> u64 {
+        self.big.into_raw()
+    }
+
+    fn small_raw_u64(&self) -> u64 {
+        self.small.into_raw()
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/pagetable/ver3.rs b/drivers/gpu/nova-core/mm/pagetable/ver3.rs
new file mode 100644
index 00000000000000..47ed3339026b6b
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/pagetable/ver3.rs
@@ -0,0 +1,421 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! MMU v3 page table types for Hopper and later GPUs.
+//!
+//! This module defines MMU version 3 specific types (Hopper and later GPUs).
+//!
+//! Key differences from MMU v2:
+//! - Unified 40-bit address field for all apertures (v2 had separate sys/vid fields).
+//! - PCF (Page Classification Field) replaces separate privilege/RO/atomic/cache bits.
+//! - KIND field is 4 bits (not 8).
+//! - IS_PTE bit in PDE to support large pages directly.
+//! - No COMPTAGLINE field (compression handled differently in v3).
+//! - No separate ENCRYPTED bit.
+//!
+//! Bit field layouts derived from the NVIDIA OpenRM documentation:
+//! `open-gpu-kernel-modules/src/common/inc/swref/published/hopper/gh100/dev_mmu.h`
+
+#![allow(dead_code)]
+
+use kernel::{
+    bitfield,
+    num::Bounded,
+    prelude::*, //
+};
+
+use pin_init::Zeroable;
+
+use super::{
+    AperturePde,
+    AperturePte,
+    DualPdeOps,
+    PageTableLevel,
+    PdeOps,
+    PteOps,
+    VaLevelIndex, //
+};
+
+use crate::mm::{
+    Pfn,
+    VirtualAddress,
+    VramAddress, //
+};
+
+// Bounded to version 3 Pfn conversion.
+impl_pfn_bounded!(40);
+
+bitfield! {
+    /// MMU v3 57-bit virtual address layout.
+    pub(super) struct VirtualAddressV3(u64) {
+        /// Page offset [11:0].
+        11:0    offset;
+        /// PT index [20:12].
+        20:12   pt_idx;
+        /// PDE0 index [28:21].
+        28:21   pde0_idx;
+        /// PDE1 index [37:29].
+        37:29   pde1_idx;
+        /// PDE2 index [46:38].
+        46:38   pde2_idx;
+        /// PDE3 index [55:47].
+        55:47   pde3_idx;
+        /// PDE4 index [56].
+        56:56   pde4_idx;
+    }
+}
+
+impl VirtualAddressV3 {
+    /// Create a [`VirtualAddressV3`] from a [`VirtualAddress`].
+    pub(super) fn new(va: VirtualAddress) -> Self {
+        Self::from_raw(va.into_raw())
+    }
+}
+
+impl VaLevelIndex for VirtualAddressV3 {
+    fn level_index(&self, level: u64) -> u64 {
+        match level {
+            0 => *self.pde4_idx(),
+            1 => *self.pde3_idx(),
+            2 => *self.pde2_idx(),
+            3 => *self.pde1_idx(),
+            4 => *self.pde0_idx(),
+            5 => *self.pt_idx(),
+            _ => 0,
+        }
+    }
+}
+
+/// PDE levels for MMU v3 (6-level hierarchy).
+pub(super) const PDE_LEVELS: &[PageTableLevel] = &[
+    PageTableLevel::Pdb,
+    PageTableLevel::L1,
+    PageTableLevel::L2,
+    PageTableLevel::L3,
+    PageTableLevel::L4,
+];
+
+/// PTE level for MMU v3.
+pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L5;
+
+/// Dual PDE level for MMU v3 (128-bit entries).
+pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L4;
+
+bitfield! {
+    /// Page Classification Field for PTEs (5 bits) in MMU v3.
+    pub(in crate::mm) struct PtePcf(u8) {
+        /// Bypass L2 cache (0=cached, 1=bypass).
+        0:0     uncached;
+        /// Access counting disabled (0=enabled, 1=disabled).
+        1:1     acd;
+        /// Read-only access (0=read-write, 1=read-only).
+        2:2     read_only;
+        /// Atomics disabled (0=enabled, 1=disabled).
+        3:3     no_atomic;
+        /// Privileged access only (0=regular, 1=privileged).
+        4:4     privileged;
+    }
+}
+
+impl PtePcf {
+    /// Create PCF for read-write mapping (cached, no atomics, regular mode).
+    fn rw() -> Self {
+        Self::zeroed().with_no_atomic(true)
+    }
+
+    /// Create PCF for read-only mapping (cached, no atomics, regular mode).
+    fn ro() -> Self {
+        Self::zeroed().with_read_only(true).with_no_atomic(true)
+    }
+
+    /// Get the raw `u8` value.
+    fn raw_u8(&self) -> u8 {
+        self.into_raw()
+    }
+}
+
+impl From<Bounded<u64, 5>> for PtePcf {
+    fn from(val: Bounded<u64, 5>) -> Self {
+        Self::from_raw(u8::from(val))
+    }
+}
+
+impl From<PtePcf> for Bounded<u64, 5> {
+    fn from(pcf: PtePcf) -> Self {
+        Bounded::from_expr(u64::from(pcf.into_raw()) & 0x1F)
+    }
+}
+
+bitfield! {
+    /// Page Classification Field for PDEs (3 bits) in MMU v3.
+    ///
+    /// Controls Address Translation Services (ATS) and caching.
+    pub(in crate::mm) struct PdePcf(u8) {
+        /// Bypass L2 cache (0=cached, 1=bypass).
+        0:0     uncached;
+        /// ATS disabled (0=enabled, 1=disabled).
+        1:1     no_ats;
+    }
+}
+
+impl PdePcf {
+    /// Create PCF for cached mapping with ATS enabled (default).
+    fn cached() -> Self {
+        Self::zeroed()
+    }
+
+    /// Get the raw `u8` value.
+    fn raw_u8(&self) -> u8 {
+        self.into_raw()
+    }
+}
+
+impl From<Bounded<u64, 3>> for PdePcf {
+    fn from(val: Bounded<u64, 3>) -> Self {
+        Self::from_raw(u8::from(val))
+    }
+}
+
+impl From<PdePcf> for Bounded<u64, 3> {
+    fn from(pcf: PdePcf) -> Self {
+        Bounded::from_expr(u64::from(pcf.into_raw()) & 0x7)
+    }
+}
+
+bitfield! {
+    /// Page Table Entry for MMU v3.
+    pub(in crate::mm) struct Pte(u64) {
+        /// Entry is valid.
+        0:0     valid;
+        /// Memory aperture type.
+        2:1     aperture => AperturePte;
+        /// Page Classification Field.
+        7:3     pcf => PtePcf;
+        /// Surface kind (4 bits, 0x0=pitch, 0xF=invalid).
+        11:8    kind;
+        /// Physical frame number (for all apertures).
+        51:12   frame_number => Pfn;
+        /// Peer GPU ID for peer memory (0-7).
+        63:61   peer_id;
+    }
+}
+
+impl PteOps for Pte {
+    fn from_raw(val: u64) -> Self {
+        Self::from_raw(val)
+    }
+
+    fn invalid() -> Self {
+        Self::zeroed()
+    }
+
+    fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self {
+        let pcf = match (aperture, writable) {
+            (AperturePte::VideoMemory, true) => PtePcf::rw(),
+            (AperturePte::VideoMemory, false) => PtePcf::ro(),
+            // Sysmem PTEs use uncached+no_atomic PCF for cache coherency.
+            (AperturePte::SystemCoherent, true) => {
+                PtePcf::zeroed().with_uncached(true).with_no_atomic(true)
+            }
+            (AperturePte::SystemCoherent, false) => PtePcf::zeroed()
+                .with_uncached(true)
+                .with_no_atomic(true)
+                .with_read_only(true),
+            (AperturePte::PeerMemory | AperturePte::SystemNonCoherent, _) => {
+                kernel::pr_warn!("MMU v3 PTE aperture {:?} not supported\n", aperture);
+                return Self::invalid();
+            }
+        };
+        Self::zeroed()
+            .with_valid(true)
+            .with_aperture(aperture)
+            .with_pcf(pcf)
+            .with_frame_number(pfn)
+    }
+
+    fn is_valid(&self) -> bool {
+        self.valid().into_bool()
+    }
+
+    fn frame_number(&self) -> Pfn {
+        Pte::frame_number(*self)
+    }
+}
+
+bitfield! {
+    /// Page Directory Entry for MMU v3 (Hopper+).
+    ///
+    /// ## Note
+    ///
+    /// v3 uses a unified 40-bit address field (v2 had separate sys/vid address fields).
+    pub(in crate::mm) struct Pde(u64) {
+        /// Entry is a PTE (0=PDE, 1=large page PTE).
+        0:0     is_pte;
+        /// Memory aperture type.
+        2:1     aperture => AperturePde;
+        /// Page Classification Field (3 bits for PDE).
+        5:3     pcf => PdePcf;
+        /// Table frame number (40-bit unified address).
+        51:12   table_frame => Pfn;
+    }
+}
+
+impl PdeOps for Pde {
+    fn from_raw(val: u64) -> Self {
+        Self::from_raw(val)
+    }
+
+    fn new(aperture: AperturePde, table_pfn: Pfn) -> Self {
+        match aperture {
+            AperturePde::VideoMemory => Self::zeroed()
+                .with_is_pte(false)
+                .with_aperture(aperture)
+                .with_table_frame(table_pfn),
+            AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => {
+                kernel::pr_warn!("MMU v3 PDE aperture {:?} not supported\n", aperture);
+                Self::invalid()
+            }
+        }
+    }
+
+    fn invalid() -> Self {
+        Self::zeroed().with_aperture(AperturePde::Invalid)
+    }
+
+    fn is_valid(&self) -> bool {
+        Pde::aperture(*self) != AperturePde::Invalid
+    }
+
+    fn aperture(&self) -> AperturePde {
+        Pde::aperture(*self)
+    }
+
+    fn table_vram_address(&self) -> VramAddress {
+        debug_assert!(
+            Pde::aperture(*self) == AperturePde::VideoMemory,
+            "table_vram_address called on non-VRAM PDE (aperture: {:?})",
+            Pde::aperture(*self)
+        );
+        VramAddress::from(self.table_frame())
+    }
+}
+
+bitfield! {
+    /// Big Page Table pointer in Dual PDE (MMU v3).
+    ///
+    /// 64-bit lower word of the 128-bit Dual PDE.
+    pub(super) struct DualPdeBig(u64) {
+        /// Entry is a PTE (for large pages).
+        0:0     is_pte;
+        /// Memory aperture type.
+        2:1     aperture => AperturePde;
+        /// Page Classification Field.
+        5:3     pcf => PdePcf;
+        /// Table frame (table address 256-byte aligned).
+        51:8    table_frame;
+    }
+}
+
+impl DualPdeBig {
+    /// Create an invalid big page table pointer.
+    fn invalid() -> Self {
+        Self::zeroed().with_aperture(AperturePde::Invalid)
+    }
+
+    /// Create a valid big PDE pointing to a page table in the given aperture.
+    fn new(aperture: AperturePde, table_addr: VramAddress) -> Result<Self> {
+        // Big page table addresses must be 256-byte aligned (shift 8).
+        if table_addr.into_raw() & 0xFF != 0 {
+            return Err(EINVAL);
+        }
+        let table_frame = Bounded::from_expr(table_addr.into_raw() >> 8);
+        match aperture {
+            AperturePde::VideoMemory => Ok(Self::zeroed()
+                .with_is_pte(false)
+                .with_aperture(aperture)
+                .with_table_frame(table_frame)),
+            AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => {
+                kernel::pr_warn!("MMU v3 DualPdeBig aperture {:?} not supported\n", aperture);
+                Ok(Self::invalid())
+            }
+        }
+    }
+
+    /// Check if this big PDE is valid.
+    fn is_valid(&self) -> bool {
+        self.aperture() != AperturePde::Invalid
+    }
+
+    /// Get the VRAM address of the big page table.
+    fn table_vram_address(&self) -> VramAddress {
+        debug_assert!(
+            self.aperture() == AperturePde::VideoMemory,
+            "table_vram_address called on non-VRAM DualPdeBig (aperture: {:?})",
+            self.aperture()
+        );
+        VramAddress::from_raw(*self.table_frame() << 8)
+    }
+}
+
+/// Dual PDE at Level 4 for MMU v3 - 128-bit entry.
+///
+/// Contains both big (64KB) and small (4KB) page table pointers:
+/// - Lower 64 bits: Big Page Table pointer.
+/// - Upper 64 bits: Small Page Table pointer.
+///
+/// ## Note
+///
+/// The big and small page table pointers have different address layouts:
+/// - Big address = field value << 8 (256-byte alignment).
+/// - Small address = field value << 12 (4KB alignment).
+///
+/// This is why `DualPdeBig` is a separate type from `Pde`.
+#[repr(C)]
+#[derive(Debug, Clone, Copy)]
+pub(in crate::mm) struct DualPde {
+    /// Big Page Table pointer.
+    pub(super) big: DualPdeBig,
+    /// Small Page Table pointer.
+    pub(super) small: Pde,
+}
+
+// SAFETY: Both `DualPdeBig` and `Pde` fields are `Zeroable` (bitfield types are Zeroable).
+unsafe impl Zeroable for DualPde {}
+
+impl DualPde {
+    /// Check if the big page table pointer is valid.
+    fn has_big(&self) -> bool {
+        self.big.is_valid()
+    }
+}
+
+impl DualPdeOps for DualPde {
+    fn from_raw(big: u64, small: u64) -> Self {
+        Self {
+            big: DualPdeBig::from_raw(big),
+            small: PdeOps::from_raw(small),
+        }
+    }
+
+    fn new_small(table_pfn: Pfn) -> Self {
+        Self {
+            big: DualPdeBig::invalid(),
+            small: PdeOps::new(AperturePde::VideoMemory, table_pfn),
+        }
+    }
+
+    fn has_small(&self) -> bool {
+        PdeOps::is_valid(&self.small)
+    }
+
+    fn small_vram_address(&self) -> VramAddress {
+        PdeOps::table_vram_address(&self.small)
+    }
+
+    fn big_raw_u64(&self) -> u64 {
+        self.big.into_raw()
+    }
+
+    fn small_raw_u64(&self) -> u64 {
+        self.small.into_raw()
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/pagetable/walk.rs b/drivers/gpu/nova-core/mm/pagetable/walk.rs
new file mode 100644
index 00000000000000..76c1729971f53d
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/pagetable/walk.rs
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Page table walker implementation for NVIDIA GPUs.
+//!
+//! This module provides page table walking functionality for MMU v2 and v3.
+//! The walker traverses the page table hierarchy to resolve virtual addresses
+//! to physical addresses or to find PTE locations.
+//!
+//! # Page Table Hierarchy
+//!
+//! ## MMU v2 (Turing/Ampere/Ada) - 5 levels
+//!
+//! ```text
+//!     +-------+     +-------+     +-------+     +---------+     +-------+
+//!     | PDB   |---->|  L1   |---->|  L2   |---->| L3 Dual |---->|  L4   |
+//!     | (L0)  |     |       |     |       |     | PDE     |     | (PTE) |
+//!     +-------+     +-------+     +-------+     +---------+     +-------+
+//!       64-bit        64-bit        64-bit        128-bit         64-bit
+//!        PDE           PDE           PDE        (big+small)        PTE
+//! ```
+//!
+//! ## MMU v3 (Hopper+) - 6 levels
+//!
+//! ```text
+//!     +-------+     +-------+     +-------+     +-------+     +---------+     +-------+
+//!     | PDB   |---->|  L1   |---->|  L2   |---->|  L3   |---->| L4 Dual |---->|  L5   |
+//!     | (L0)  |     |       |     |       |     |       |     | PDE     |     | (PTE) |
+//!     +-------+     +-------+     +-------+     +-------+     +---------+     +-------+
+//!       64-bit        64-bit        64-bit        64-bit        128-bit         64-bit
+//!        PDE           PDE           PDE           PDE        (big+small)        PTE
+//! ```
+//!
+//! # Result of a page table walk
+//!
+//! The walker returns a [`WalkResult`] indicating the outcome.
+
+use core::marker::PhantomData;
+
+use kernel::prelude::*;
+
+use super::{
+    DualPdeOps,
+    MmuConfig,
+    MmuV2,
+    MmuV3,
+    MmuVersion,
+    PageTableLevel,
+    PdeOps,
+    PteOps, //
+};
+use crate::{
+    mm::{
+        pramin,
+        GpuMm,
+        Pfn,
+        Vfn,
+        VirtualAddress,
+        VramAddress, //
+    },
+    num::{
+        IntoSafeCast, //
+    },
+};
+
+/// Result of walking to a PTE.
+#[derive(Debug, Clone, Copy)]
+pub(in crate::mm) enum WalkResult {
+    /// Intermediate page tables are missing (only returned in lookup mode).
+    PageTableMissing,
+    /// PTE exists but is invalid (page not mapped).
+    Unmapped { pte_addr: VramAddress },
+    /// PTE exists and is valid (page is mapped).
+    Mapped { pte_addr: VramAddress, pfn: Pfn },
+}
+
+/// Result of walking PDE levels only.
+///
+/// Returned by [`PtWalkInner::walk_pde_levels()`] to indicate whether all PDE
+/// levels resolved or a PDE is missing.
+#[derive(Debug, Clone, Copy)]
+pub(in crate::mm) enum WalkPdeResult {
+    /// All PDE levels resolved -- returns PTE page table address.
+    Complete {
+        /// VRAM address of the PTE-level page table.
+        pte_table: VramAddress,
+    },
+    /// A PDE is missing and no prepared page was provided by the closure.
+    Missing {
+        /// PDE slot address in the parent page table (where to install).
+        install_addr: VramAddress,
+        /// The page table level that is missing.
+        level: PageTableLevel,
+    },
+}
+
+/// Page table walker.
+pub(in crate::mm) struct PtWalkInner<M: MmuConfig> {
+    pdb_addr: VramAddress,
+    _phantom: PhantomData<M>,
+}
+
+impl<M: MmuConfig> PtWalkInner<M> {
+    /// Calculate the VRAM address of an entry within a page table.
+    fn entry_addr(table: VramAddress, level: PageTableLevel, index: u64) -> VramAddress {
+        let entry_size: u64 = M::entry_size(level).into_safe_cast();
+        table + index * entry_size
+    }
+
+    /// Create a new page table walker.
+    pub(super) fn new(pdb_addr: VramAddress) -> Self {
+        Self {
+            pdb_addr,
+            _phantom: PhantomData,
+        }
+    }
+
+    /// Walk PDE levels with closure-based resolution for missing PDEs.
+    ///
+    /// Traverses all PDE levels for the MMU version. At each level, reads the PDE.
+    /// If valid, extracts the child table address and continues. If missing, calls
+    /// `resolve_prepared(install_addr)` to resolve the missing PDE.
+    pub(super) fn walk_pde_levels(
+        &self,
+        pramin: &mut pramin::Pramin<'_>,
+        vfn: Vfn,
+        resolve_prepared: impl Fn(VramAddress) -> Option<VramAddress>,
+    ) -> Result<WalkPdeResult> {
+        let va = VirtualAddress::from(vfn);
+        let mut cur_table = self.pdb_addr;
+
+        for &level in M::PDE_LEVELS {
+            let idx = M::level_index(va, level.as_index());
+            let install_addr = Self::entry_addr(cur_table, level, idx);
+
+            if level == M::DUAL_PDE_LEVEL {
+                // 128-bit dual PDE with big+small page table pointers.
+                let dpde = M::DualPde::read(pramin, install_addr)?;
+                if dpde.has_small() {
+                    cur_table = dpde.small_vram_address();
+                    continue;
+                }
+            } else {
+                // Regular 64-bit PDE. Use `is_valid_vram()` because
+                // `table_vram_address()` only reads the VRAM frame-number
+                // bitfield; system-memory PDEs store the address in a
+                // different (wider) field and would be silently truncated.
+                let pde = M::Pde::read(pramin, install_addr)?;
+                if pde.is_valid_vram() {
+                    cur_table = pde.table_vram_address();
+                    continue;
+                }
+            }
+
+            // PDE missing in HW. Ask caller for resolution.
+            if let Some(prepared_addr) = resolve_prepared(install_addr) {
+                cur_table = prepared_addr;
+                continue;
+            }
+
+            return Ok(WalkPdeResult::Missing {
+                install_addr,
+                level,
+            });
+        }
+
+        Ok(WalkPdeResult::Complete {
+            pte_table: cur_table,
+        })
+    }
+
+    /// Walk to PTE for lookup only (no allocation).
+    ///
+    /// Returns [`WalkResult::PageTableMissing`] if intermediate tables don't exist.
+    pub(super) fn walk_to_pte_lookup(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result<WalkResult> {
+        self.walk_to_pte_lookup_with_window(mm.pramin_mut(), vfn)
+    }
+
+    /// Walk to PTE using a caller-provided PRAMIN manager (lookup only).
+    pub(super) fn walk_to_pte_lookup_with_window(
+        &self,
+        pramin: &mut pramin::Pramin<'_>,
+        vfn: Vfn,
+    ) -> Result<WalkResult> {
+        match self.walk_pde_levels(pramin, vfn, |_| None)? {
+            WalkPdeResult::Complete { pte_table } => {
+                Self::read_pte_at_level(pramin, vfn, pte_table)
+            }
+            WalkPdeResult::Missing { .. } => Ok(WalkResult::PageTableMissing),
+        }
+    }
+
+    /// Read the PTE at the PTE level given the PTE table address.
+    fn read_pte_at_level(
+        pramin: &mut pramin::Pramin<'_>,
+        vfn: Vfn,
+        pte_table: VramAddress,
+    ) -> Result<WalkResult> {
+        let va = VirtualAddress::from(vfn);
+        let pte_level = M::PTE_LEVEL;
+        let pte_idx = M::level_index(va, pte_level.as_index());
+        let pte_addr = Self::entry_addr(pte_table, pte_level, pte_idx);
+        let pte = M::Pte::read(pramin, pte_addr)?;
+
+        if pte.is_valid() {
+            return Ok(WalkResult::Mapped {
+                pte_addr,
+                pfn: pte.frame_number(),
+            });
+        }
+        Ok(WalkResult::Unmapped { pte_addr })
+    }
+}
+
+macro_rules! pt_walk_dispatch {
+    ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => {
+        match $self {
+            PtWalk::V2(inner) => inner.$method($($arg),*),
+            PtWalk::V3(inner) => inner.$method($($arg),*),
+        }
+    };
+}
+
+/// Page table walker dispatch.
+pub(in crate::mm) enum PtWalk {
+    /// MMU v2 (Turing/Ampere/Ada).
+    V2(PtWalkInner<MmuV2>),
+    /// MMU v3 (Hopper+).
+    V3(PtWalkInner<MmuV3>),
+}
+
+impl PtWalk {
+    /// Create a new page table walker for the given MMU version.
+    pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self {
+        match version {
+            MmuVersion::V2 => Self::V2(PtWalkInner::<MmuV2>::new(pdb_addr)),
+            MmuVersion::V3 => Self::V3(PtWalkInner::<MmuV3>::new(pdb_addr)),
+        }
+    }
+
+    /// Walk to PTE for lookup.
+    pub(in crate::mm) fn walk_to_pte(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result<WalkResult> {
+        pt_walk_dispatch!(self, walk_to_pte_lookup(mm, vfn))
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/pramin.rs b/drivers/gpu/nova-core/mm/pramin.rs
new file mode 100644
index 00000000000000..7f89c093d59155
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/pramin.rs
@@ -0,0 +1,312 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Utilities for accessing VRAM through the PRAMIN window.
+
+use core::ops::Range;
+
+use kernel::{
+    io::{
+        io_project,
+        register,
+        register::OffsetLoc,
+        Io,
+        Mmio, //
+    },
+    prelude::*,
+    ptr::{
+        Alignable,
+        Alignment, //
+    },
+    sizes::{
+        SZ_1M,
+        SZ_64K, //
+    },
+};
+
+use crate::{
+    driver::{
+        Bar0,
+        NovaRegisters, //
+    },
+    gpu::Chipset,
+    mm::{
+        hal::{
+            self,
+            MmHal, //
+        },
+        VramAddress, //
+    },
+    num::IntoSafeCast, //
+};
+
+/// Size of the PRAMIN window (1 MiB).
+const WINDOW_SIZE: usize = SZ_1M;
+
+/// The PRAMIN window, which is a 1 MiB window into VRAM at a fixed BAR0 offset.
+#[derive(FromBytes, IntoBytes)]
+struct PraminWindow([u8; WINDOW_SIZE]);
+
+register! {
+    base: NovaRegisters;
+
+    /// Location of the window inside BAR0.
+    PRAMIN: PraminWindow @ 0x700000;
+}
+
+/// Owner of the PRAMIN window state.
+///
+/// [`Pramin::window_at()`] repositions the window as needed and returns a typed MMIO view into
+/// it, holding the manager borrowed for the lifetime of the view.
+pub(super) struct Pramin<'gpu> {
+    bar: Bar0<'gpu>,
+    hal: &'static dyn MmHal,
+    /// MMIO view of the PRAMIN window in BAR0.
+    window: Mmio<'gpu, PraminWindow>,
+    /// VRAM range to keep the PRAMIN window inside.
+    vram_range: Range<VramAddress>,
+    /// Cached window position.
+    window_range: Range<VramAddress>,
+}
+
+/// Typed view of VRAM through the PRAMIN window.
+///
+/// Inserts an ordering point after previous writes through the window on drop. Views returned
+/// by [`PraminAccess::view()`] cannot outlive this access, so the ordering point covers every
+/// write made through them.
+pub(super) struct PraminAccess<'a, T>
+where
+    T: FromBytes + IntoBytes,
+{
+    view: Mmio<'a, T>,
+}
+
+impl<T> PraminAccess<'_, T>
+where
+    T: FromBytes + IntoBytes,
+{
+    /// Returns the MMIO view of the accessed location.
+    pub(super) fn view(&self) -> Mmio<'_, T> {
+        self.view
+    }
+}
+
+impl<T> Drop for PraminAccess<'_, T>
+where
+    T: FromBytes + IntoBytes,
+{
+    fn drop(&mut self) {
+        // Insert an ordering point after previous writes through this window.
+        self.view.cast::<u8>().read_val();
+    }
+}
+
+impl<'gpu> Pramin<'gpu> {
+    /// Alignment required by the PRAMIN window.
+    const BASE_ALIGN: Alignment = Alignment::new::<SZ_64K>();
+
+    /// Creates the window manager for the given VRAM region.
+    pub(super) fn new(
+        bar: Bar0<'gpu>,
+        chipset: Chipset,
+        vram_range: Range<VramAddress>,
+    ) -> Result<Self> {
+        let hal = hal::mm_hal(chipset);
+        let window = io_project!(bar, build: PRAMIN);
+        let base = vram_range.start.align_down(Self::BASE_ALIGN);
+        let window_range = Self::window_range(base)?;
+        hal.write_pramin_window_base(bar, base)?;
+
+        Ok(Self {
+            bar,
+            hal,
+            window,
+            vram_range,
+            window_range,
+        })
+    }
+
+    /// Returns the VRAM range a window based at `base` exposes.
+    fn window_range(base: VramAddress) -> Result<Range<VramAddress>> {
+        let end = base
+            .checked_add(WINDOW_SIZE.into_safe_cast())
+            .ok_or(EINVAL)?;
+        Ok(base..end)
+    }
+
+    /// Check the window covers `len` bytes at `addr`, moving it if needed.
+    ///
+    /// Returns the window offset at which to perform the access.
+    fn window_offset(&mut self, addr: VramAddress, len: usize) -> Result<usize> {
+        let end = addr.checked_add(len.into_safe_cast()).ok_or(EINVAL)?;
+
+        let inside = |r: &Range<VramAddress>| r.contains(&addr) && end <= r.end;
+        if !inside(&self.vram_range) {
+            return Err(EINVAL);
+        }
+
+        // Reposition the window if the access falls outside it.
+        if !inside(&self.window_range) {
+            let base = addr.align_down(Self::BASE_ALIGN);
+            let window_range = Self::window_range(base)?;
+            if !inside(&window_range) {
+                return Err(EINVAL);
+            }
+            self.hal.write_pramin_window_base(self.bar, base)?;
+            self.window_range = window_range;
+        }
+
+        Ok((addr - self.window_range.start).into_safe_cast())
+    }
+
+    /// Return a typed MMIO view of a `T` at `vram_addr`.
+    ///
+    /// Returns an error if `vram_addr` is not aligned to `T`'s alignment, or if
+    /// a `T` at `vram_addr` does not fit within the VRAM region.
+    pub(super) fn window_at<'a, T>(
+        &'a mut self,
+        vram_addr: VramAddress,
+    ) -> Result<PraminAccess<'a, T>>
+    where
+        T: FromBytes + IntoBytes,
+    {
+        let offset = self.window_offset(vram_addr, size_of::<T>())?;
+        let view = io_project!(self.window, try: OffsetLoc::new(offset));
+
+        Ok(PraminAccess { view })
+    }
+}
+
+#[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+pub(super) mod selftest {
+    use kernel::{
+        device,
+        io::io_read,
+        sizes::SizeConstants, //
+    };
+
+    use super::*;
+    use crate::{
+        selftest_assert,
+        selftest_assert_eq, //
+    };
+
+    /// Test read/write at byte granularity, at unaligned addresses.
+    fn test_byte_readwrite(
+        dev: &device::Device<device::Bound>,
+        pramin: &mut Pramin<'_>,
+        base: VramAddress,
+    ) -> Result {
+        for i in 0u8..4 {
+            let addr = base + 1 + u64::from(i);
+            pramin.window_at::<u8>(addr)?.view().write_val(0xA0 + i);
+        }
+
+        for i in 0u8..4 {
+            let addr = base + 1 + u64::from(i);
+            selftest_assert_eq!(
+                dev,
+                pramin.window_at::<u8>(addr)?.view().read_val(),
+                0xA0 + i
+            );
+        }
+        Ok(())
+    }
+
+    /// Test writing a `u32` and reading back as individual `u8`s.
+    fn test_u32_as_bytes(
+        dev: &device::Device<device::Bound>,
+        pramin: &mut Pramin<'_>,
+        base: VramAddress,
+    ) -> Result {
+        let addr = base + 0x10;
+        let val: u32 = 0xDEADBEEF;
+        pramin.window_at::<u32>(addr)?.view().write_val(val);
+
+        let window = pramin.window_at::<[u8; 4]>(addr)?;
+        for (i, &expected) in val.to_le_bytes().iter().enumerate() {
+            selftest_assert_eq!(dev, io_read!(window.view(), [build: i]), expected);
+        }
+        Ok(())
+    }
+
+    /// Test window repositioning across 1 MiB boundaries.
+    fn test_window_reposition(
+        dev: &device::Device<device::Bound>,
+        pramin: &mut Pramin<'_>,
+        base: VramAddress,
+    ) -> Result {
+        let addr_a = base;
+        let addr_b = base + u64::SZ_2M; // base + 2 MiB (different 1 MiB region).
+        let val_a: u32 = 0x11111111;
+        let val_b: u32 = 0x22222222;
+
+        pramin.window_at::<u32>(addr_a)?.view().write_val(val_a);
+        pramin.window_at::<u32>(addr_b)?.view().write_val(val_b);
+
+        selftest_assert_eq!(
+            dev,
+            pramin.window_at::<u32>(addr_a)?.view().read_val(),
+            val_a
+        );
+        selftest_assert_eq!(
+            dev,
+            pramin.window_at::<u32>(addr_b)?.view().read_val(),
+            val_b
+        );
+        Ok(())
+    }
+
+    /// Test that offsets outside the VRAM region are rejected.
+    fn test_invalid_offset(
+        dev: &device::Device<device::Bound>,
+        pramin: &mut Pramin<'_>,
+        vram_end: VramAddress,
+    ) -> Result {
+        selftest_assert!(dev, pramin.window_at::<u32>(vram_end).is_err());
+        Ok(())
+    }
+
+    /// Test that misaligned accesses are rejected.
+    fn test_misaligned_access(
+        dev: &device::Device<device::Bound>,
+        pramin: &mut Pramin<'_>,
+        base: VramAddress,
+    ) -> Result {
+        // `u16` at odd offset (not 2-byte aligned).
+        selftest_assert!(dev, pramin.window_at::<u16>(base + 0x21).is_err());
+
+        // `u32` at 2-byte-aligned (not 4-byte-aligned) offset.
+        selftest_assert!(dev, pramin.window_at::<u32>(base + 2).is_err());
+
+        // `u64` at a 4-byte-aligned (not 8-byte-aligned) address.
+        selftest_assert!(dev, pramin.window_at::<u64>(base + 0x44).is_err());
+
+        // A `u16` view at an even address is allowed.
+        pramin.window_at::<u16>(base + 0x22)?;
+        Ok(())
+    }
+
+    /// Run PRAMIN self-tests during probe.
+    ///
+    /// `base` is the start of a driver-usable VRAM span that the tests are free to
+    /// overwrite.
+    pub(crate) fn run(
+        dev: &device::Device<device::Bound>,
+        pramin: &mut Pramin<'_>,
+        base: VramAddress,
+    ) -> Result {
+        dev_dbg!(dev, "PRAMIN: starting self-tests\n");
+
+        let vram_end = pramin.vram_range.end;
+
+        test_byte_readwrite(dev, pramin, base)?;
+        test_u32_as_bytes(dev, pramin, base)?;
+        test_window_reposition(dev, pramin, base)?;
+        test_invalid_offset(dev, pramin, vram_end)?;
+        test_misaligned_access(dev, pramin, base)?;
+
+        dev_info!(dev, "PRAMIN: self-tests passed\n");
+        Ok(())
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/regs.rs b/drivers/gpu/nova-core/mm/regs.rs
new file mode 100644
index 00000000000000..82de6dfa4e8b72
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/regs.rs
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Registers used by the memory management subsystems: the BAR0 PRAMIN window.
+
+use kernel::io::register;
+
+use crate::{
+    bounded_enum,
+    driver::NovaRegisters, //
+};
+
+// PRAMIN window
+
+bounded_enum! {
+    /// Target memory type for the BAR0 window register.
+    ///
+    /// Only VRAM is needed by the driver. Pre-Hopper window registers also define
+    /// system-memory targets that are unused here; Hopper+ uses a separate register
+    /// without a target field.
+    #[derive(Debug, Copy, Clone)]
+    pub(super) enum Bar0WindowTarget with TryFrom<Bounded<u32, 2>> {
+        /// Video memory (GPU framebuffer memory).
+        VidMem = 0,
+    }
+}
+
+register! {
+    base: NovaRegisters;
+
+    /// BAR0 window control for PRAMIN access.
+    pub(super) NV_PBUS_BAR0_WINDOW(u32) @ 0x00001700 {
+        /// Target memory aperture for the window.
+        25:24   target ?=> Bar0WindowTarget;
+        /// PRAMIN window base bits 39:16.
+        23:0    base;
+    }
+}
+
+pub(super) mod gh100 {
+    use kernel::io::register;
+
+    use crate::driver::NovaRegisters;
+
+    register! {
+        base: NovaRegisters;
+
+        /// Hopper register for PRAMIN window.
+        pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 {
+            /// PRAMIN window base bits 37:16.
+            21:0    base;
+        }
+    }
+}
+
+pub(super) mod gb100 {
+    use kernel::io::register;
+
+    use crate::driver::NovaRegisters;
+
+    register! {
+        base: NovaRegisters;
+
+        /// Blackwell GB10x/GB20x register for PRAMIN window.
+        pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 {
+            /// PRAMIN window base bits 38:16.
+            22:0    base;
+        }
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/tlb.rs b/drivers/gpu/nova-core/mm/tlb.rs
new file mode 100644
index 00000000000000..cc862e8159a19a
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/tlb.rs
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! TLB (Translation Lookaside Buffer) flush support for GPU MMU.
+//!
+//! After modifying page table entries, the GPU's TLB must be flushed to
+//! ensure the new mappings take effect. This module provides TLB flush
+//! functionality for virtual memory managers.
+//!
+//! # Examples
+//!
+//! ```ignore
+//! use crate::mm::tlb::Tlb;
+//!
+//! fn page_table_update(tlb: &Tlb, pdb_addr: VramAddress) -> Result<()> {
+//!     // ... modify page tables ...
+//!
+//!     // Flush TLB to make changes visible (polls for completion).
+//!     tlb.flush(pdb_addr)?;
+//!
+//!     Ok(())
+//! }
+//! ```
+
+use kernel::{
+    io::poll::read_poll_timeout,
+    io::Io,
+    new_mutex,
+    prelude::*,
+    sync::Mutex,
+    time::Delta, //
+};
+
+use crate::{
+    bounded_enum,
+    driver::Bar0,
+    mm::VramAddress,
+    regs, //
+};
+
+bounded_enum! {
+    /// TLB invalidation acknowledgment scope.
+    ///
+    /// Controls how far the hardware waits for the invalidation to propagate
+    /// before clearing the `trigger` bit of `NV_TLB_FLUSH_CTRL`.
+    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+    pub(crate) enum TlbAckMode with TryFrom<Bounded<u32, 2>> {
+        /// Fire-and-forget: no acknowledgment required.
+        None = 0,
+        /// Wait for acknowledgment from all consumers, including remote GPUs
+        /// reachable over NVLink.
+        ///
+        /// Globally is strictly required only during unmap or permission
+        /// tightening, because the backing memory may be reassigned after the
+        /// flush returns and a stale TLB entry could let the GPU access freed
+        /// memory. For new mapping or relaxing permissions, a stale entry would
+        /// merely cause a redundant fault and retry, so [`TlbAckMode::None`]
+        /// would suffice.
+        Globally = 1,
+        /// Wait for acknowledgment from consumers within the local NVLink
+        /// fabric node only; skip cross-node ack.
+        Intranode = 2,
+    }
+}
+
+/// TLB manager for GPU translation buffer operations.
+#[pin_data]
+pub(crate) struct Tlb<'gpu> {
+    bar: Bar0<'gpu>,
+    /// TLB flush serialization lock: This lock is designed to be acquired during
+    /// the DMA fence signalling critical path. It should NEVER be held across any
+    /// reclaimable CPU memory allocations because the memory reclaim path can
+    /// call `dma_fence_wait()` (when implemented), which would deadlock if lock held.
+    #[pin]
+    lock: Mutex<()>,
+}
+
+impl<'gpu> Tlb<'gpu> {
+    /// Create a new TLB manager.
+    pub(super) fn new(bar: Bar0<'gpu>) -> impl PinInit<Self> {
+        pin_init!(Self {
+            bar,
+            lock <- new_mutex!((), "tlb_flush"),
+        })
+    }
+
+    /// Flush the GPU TLB for a specific page directory base.
+    ///
+    /// This invalidates all TLB entries associated with the given PDB address.
+    /// Must be called after modifying page table entries to ensure the GPU sees
+    /// the updated mappings.
+    pub(super) fn flush(&self, pdb_addr: VramAddress) -> Result {
+        let _guard = self.lock.lock();
+
+        // Write PDB address.
+        self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_LO::from_pdb_addr(
+            pdb_addr.into_raw(),
+        ));
+        self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_HI::from_pdb_addr(
+            pdb_addr.into_raw(),
+        ));
+
+        // Trigger flush.
+        self.bar.write_reg(
+            regs::NV_TLB_FLUSH_CTRL::zeroed()
+                .with_all_va(true)
+                .with_ack(TlbAckMode::None)
+                .with_trigger(true),
+        );
+
+        // Poll for completion.
+        read_poll_timeout(
+            || Ok(self.bar.read(regs::NV_TLB_FLUSH_CTRL)),
+            |ctrl: &regs::NV_TLB_FLUSH_CTRL| !ctrl.trigger(),
+            Delta::ZERO,
+            Delta::from_secs(2),
+        )?;
+
+        Ok(())
+    }
+}
diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs
new file mode 100644
index 00000000000000..51b500a2723391
--- /dev/null
+++ b/drivers/gpu/nova-core/mm/vmm.rs
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Virtual Memory Manager for NVIDIA GPU page table management.
+//!
+//! The [`Vmm`] provides high-level page mapping and unmapping operations for GPU
+//! virtual address spaces (Channels, BAR1, BAR2).
+
+use kernel::{
+    gpu::buddy::AllocatedBlocks,
+    maple_tree::MapleTreeAlloc,
+    prelude::*,
+    rbtree::RBTree, //
+};
+
+use core::{
+    cell::Cell,
+    ops::Range, //
+};
+
+use crate::{
+    mm::{
+        pagetable::{
+            map::{
+                PtMap, //
+            },
+            walk::{
+                PtWalk,
+                WalkResult, //
+            },
+            MmuVersion, //
+        },
+        GpuMm,
+        Pfn,
+        Vfn,
+        VramAddress,
+        PAGE_SIZE, //
+    },
+    num::{
+        IntoSafeCast, //
+    },
+};
+
+/// Multi-page prepared mapping -- VA range allocated, ready for execute.
+///
+/// Produced by [`Vmm::prepare_map()`], consumed by [`Vmm::execute_map()`].
+/// The VA space allocation is tracked in the [`Vmm`]'s maple tree and freed
+/// on error or via [`Vmm::unmap_pages()`].
+///
+/// Dropping without calling [`Vmm::execute_map()`] logs a warning and leaks
+/// the VA range in the maple tree.
+pub(crate) struct PreparedMapping {
+    vfn_start: Vfn,
+    num_pages: usize,
+    /// Logs a warning if dropped without executing.
+    _drop_guard: MustExecuteGuard,
+}
+
+/// Result of a mapping operation -- tracks the active mapped range.
+///
+/// Returned by [`Vmm::execute_map()`] and [`Vmm::map_pages()`].
+/// Callers must call [`Vmm::unmap_pages()`] before dropping to invalidate
+/// PTEs and free the VA range. Dropping without unmapping logs a warning
+/// and leaks the VA range in the maple tree.
+pub(crate) struct MappedRange {
+    pub(super) vfn_start: Vfn,
+    pub(super) num_pages: usize,
+    /// Logs a warning if dropped without unmapping.
+    _drop_guard: MustUnmapGuard,
+}
+
+/// Guard that logs a warning if a [`PreparedMapping`] is dropped without
+/// being consumed by [`Vmm::execute_map()`].
+struct MustExecuteGuard {
+    armed: Cell<bool>,
+}
+
+impl MustExecuteGuard {
+    const fn new() -> Self {
+        Self {
+            armed: Cell::new(true),
+        }
+    }
+
+    fn disarm(&self) {
+        self.armed.set(false);
+    }
+}
+
+impl Drop for MustExecuteGuard {
+    fn drop(&mut self) {
+        if self.armed.get() {
+            kernel::pr_warn!("PreparedMapping dropped without calling execute_map()\n");
+        }
+    }
+}
+
+/// Guard that logs a warning if a [`MappedRange`] is dropped without
+/// calling [`Vmm::unmap_pages()`].
+struct MustUnmapGuard {
+    armed: Cell<bool>,
+}
+
+impl MustUnmapGuard {
+    const fn new() -> Self {
+        Self {
+            armed: Cell::new(true),
+        }
+    }
+
+    fn disarm(&self) {
+        self.armed.set(false);
+    }
+}
+
+impl Drop for MustUnmapGuard {
+    fn drop(&mut self) {
+        if self.armed.get() {
+            kernel::pr_warn!("MappedRange dropped without calling unmap_pages()\n");
+        }
+    }
+}
+
+/// Virtual Memory Manager for a GPU address space.
+///
+/// Each [`Vmm`] instance manages a single address space identified by its Page
+/// Directory Base (`PDB`) address. Used for Channel, BAR1 and BAR2 mappings.
+pub(crate) struct Vmm {
+    /// Page Directory Base address for this address space.
+    #[expect(dead_code)]
+    pdb_addr: VramAddress,
+    /// Page table walker for reading existing mappings.
+    pt_walk: PtWalk,
+    /// Page table mapper for prepare/execute operations.
+    pt_map: PtMap,
+    /// Page table allocations required for mappings.
+    page_table_allocs: KVec<Pin<KBox<AllocatedBlocks>>>,
+    /// Maple tree allocator for virtual address range tracking.
+    virt_alloc: Pin<KBox<MapleTreeAlloc<()>>>,
+    /// Total number of pages in the virtual address space.
+    va_pages: usize,
+    /// Prepared PT pages pending PDE installation, keyed by `install_addr`.
+    ///
+    /// Populated during prepare phase and drained in execute phase. Shared by all
+    /// pending maps, preventing races on the same PDE slot.
+    pt_pages: RBTree<VramAddress, super::pagetable::map::PreparedPtPage>,
+}
+
+impl Vmm {
+    /// Create a new [`Vmm`] for the given Page Directory Base address.
+    ///
+    /// The [`Vmm`] will manage a virtual address space of `va_size` bytes.
+    pub(crate) fn new(
+        pdb_addr: VramAddress,
+        mmu_version: MmuVersion,
+        va_size: u64,
+    ) -> Result<Self> {
+        let page_size: u64 = PAGE_SIZE.into_safe_cast();
+        let va_pages: usize = (va_size / page_size).into_safe_cast();
+        let virt_alloc = KBox::pin_init(MapleTreeAlloc::<()>::new(), GFP_KERNEL)?;
+
+        Ok(Self {
+            pdb_addr,
+            pt_walk: PtWalk::new(pdb_addr, mmu_version),
+            pt_map: PtMap::new(pdb_addr, mmu_version),
+            page_table_allocs: KVec::new(),
+            virt_alloc,
+            va_pages,
+            pt_pages: RBTree::new(),
+        })
+    }
+
+    /// Allocate a contiguous virtual frame number range.
+    fn alloc_vfn_range(&self, num_pages: usize, va_range: Option<Range<u64>>) -> Result<Vfn> {
+        let page_size: u64 = PAGE_SIZE.into_safe_cast();
+
+        let start_vfn = match va_range {
+            Some(r) => {
+                let num_pages_u64: u64 = num_pages.into_safe_cast();
+                let size = num_pages_u64.checked_mul(page_size).ok_or(EOVERFLOW)?;
+                let range_size = r.end.checked_sub(r.start).ok_or(EOVERFLOW)?;
+                if range_size != size {
+                    return Err(EINVAL);
+                }
+                let start_vfn: usize = (r.start / page_size).into_safe_cast();
+                let end_vfn: usize = (r.end / page_size).into_safe_cast();
+                self.virt_alloc
+                    .insert_range(start_vfn..end_vfn, (), GFP_KERNEL)?;
+                start_vfn
+            }
+            None => self
+                .virt_alloc
+                .alloc_range(num_pages, (), ..self.va_pages, GFP_KERNEL)?,
+        };
+
+        Ok(Vfn::new(start_vfn.into_safe_cast()))
+    }
+
+    /// Free a virtual frame number range back to the maple tree.
+    fn free_vfn(&self, vfn: Vfn) {
+        let vfn_index: usize = vfn.raw().into_safe_cast();
+        if self.virt_alloc.erase(vfn_index).is_none() {
+            kernel::pr_warn!("free_vfn: VFN {} not found in maple tree\n", vfn_index);
+        }
+    }
+
+    /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped.
+    pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result<Option<Pfn>> {
+        match self.pt_walk.walk_to_pte(mm, vfn)? {
+            WalkResult::Mapped { pfn, .. } => Ok(Some(pfn)),
+            WalkResult::Unmapped { .. } | WalkResult::PageTableMissing => Ok(None),
+        }
+    }
+
+    /// Prepare resources for mapping `num_pages` pages.
+    ///
+    /// Allocates a contiguous VA range, then walks the hierarchy per-VFN to prepare pages
+    /// for all missing PDEs. Returns a [`PreparedMapping`] with the VA allocation.
+    ///
+    /// If `va_range` is not `None`, the VA range is constrained to the given range. Safe
+    /// to call outside the fence signalling critical path.
+    pub(crate) fn prepare_map(
+        &mut self,
+        mm: &mut GpuMm<'_>,
+        num_pages: usize,
+        va_range: Option<Range<u64>>,
+    ) -> Result<PreparedMapping> {
+        if num_pages == 0 {
+            return Err(EINVAL);
+        }
+
+        // Allocate contiguous VA range.
+        let vfn_start = self.alloc_vfn_range(num_pages, va_range)?;
+
+        if let Err(e) = self.pt_map.prepare_map(
+            mm,
+            vfn_start,
+            num_pages,
+            &mut self.page_table_allocs,
+            &mut self.pt_pages,
+        ) {
+            self.free_vfn(vfn_start);
+            return Err(e);
+        }
+
+        Ok(PreparedMapping {
+            vfn_start,
+            num_pages,
+            _drop_guard: MustExecuteGuard::new(),
+        })
+    }
+
+    /// Execute a prepared multi-page mapping.
+    ///
+    /// Installs all prepared PDEs and writes PTEs into the page table, then flushes TLB.
+    pub(crate) fn execute_map(
+        &mut self,
+        mm: &mut GpuMm<'_>,
+        prepared: PreparedMapping,
+        pfns: &[Pfn],
+        writable: bool,
+    ) -> Result<MappedRange> {
+        if pfns.len() != prepared.num_pages {
+            self.free_vfn(prepared.vfn_start);
+            return Err(EINVAL);
+        }
+
+        let PreparedMapping {
+            vfn_start,
+            num_pages,
+            _drop_guard,
+        } = prepared;
+        _drop_guard.disarm();
+
+        if let Err(e) = self.pt_map.install_mappings(
+            mm,
+            &mut self.pt_pages,
+            &mut self.page_table_allocs,
+            vfn_start,
+            pfns,
+            writable,
+        ) {
+            self.free_vfn(vfn_start);
+            return Err(e);
+        }
+
+        Ok(MappedRange {
+            vfn_start,
+            num_pages,
+            _drop_guard: MustUnmapGuard::new(),
+        })
+    }
+
+    /// Map pages doing prepare and execute in the same call.
+    ///
+    /// This is a convenience wrapper for callers outside the fence signalling critical
+    /// path (e.g., BAR mappings). For DRM usecases, [`Vmm::prepare_map()`] and
+    /// [`Vmm::execute_map()`] will be called separately.
+    pub(crate) fn map_pages(
+        &mut self,
+        mm: &mut GpuMm<'_>,
+        pfns: &[Pfn],
+        va_range: Option<Range<u64>>,
+        writable: bool,
+    ) -> Result<MappedRange> {
+        if pfns.is_empty() {
+            return Err(EINVAL);
+        }
+
+        // Check if provided VA range is sufficient (if provided).
+        if let Some(ref range) = va_range {
+            let required: u64 = pfns
+                .len()
+                .checked_mul(PAGE_SIZE)
+                .ok_or(EOVERFLOW)?
+                .into_safe_cast();
+            let available = range.end.checked_sub(range.start).ok_or(EINVAL)?;
+            if available < required {
+                return Err(EINVAL);
+            }
+        }
+
+        let prepared = self.prepare_map(mm, pfns.len(), va_range)?;
+        self.execute_map(mm, prepared, pfns, writable)
+    }
+
+    /// Unmap all pages in a [`MappedRange`] with a single TLB flush.
+    pub(crate) fn unmap_pages(&mut self, mm: &mut GpuMm<'_>, range: MappedRange) -> Result {
+        let result = self
+            .pt_map
+            .invalidate_ptes(mm, range.vfn_start, range.num_pages);
+
+        // TODO: Internal page table pages (PDE, PTE pages) are still kept around.
+        // This is by design as repeated maps/unmaps will be fast. As a future TODO,
+        // we can add a reclaimer here to reclaim if VRAM is short. For now, the PT
+        // pages are dropped once the `Vmm` is dropped.
+
+        // Free the VA range regardless of PTE invalidation success, so that the VA
+        // range is recovered even on failure (PTEs may be stale, but that is better
+        // than leaking both PTEs and VA range).
+        self.free_vfn(range.vfn_start);
+
+        // Unmap complete, safe to drop `MappedRange`.
+        range._drop_guard.disarm();
+        result
+    }
+}
diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs
index 35a8b1214b0e81..4f8ce4e4c18751 100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -10,6 +10,7 @@ use kernel::{
     InPlaceModule, //
 };
 
+pub mod api;
 mod driver;
 mod falcon;
 mod fb;
@@ -18,10 +19,13 @@ mod fsp;
 mod gpu;
 mod gsp;
 mod mctp;
+mod mm;
 #[macro_use]
 mod num;
 mod regs;
 mod sbuffer;
+#[cfg(CONFIG_NOVA_CORE_SELFTESTS)]
+mod selftest;
 mod vbios;
 mod vgpu;
 
diff --git a/drivers/gpu/nova-core/num.rs b/drivers/gpu/nova-core/num.rs
index 6eb174d136ab3e..f4169235bc24c2 100644
--- a/drivers/gpu/nova-core/num.rs
+++ b/drivers/gpu/nova-core/num.rs
@@ -263,7 +263,7 @@ macro_rules! bounded_enum {
             ) -> kernel::error::Result<Self> {
                 match value.get() {
                     $(
-                        $value => Ok($enum_type::$variant),
+                        value if value == $value => Ok($enum_type::$variant),
                     )*
                     _ => Err(kernel::error::code::EINVAL),
                 }
diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs
index caeef4d8587466..9978fb2803b0df 100644
--- a/drivers/gpu/nova-core/regs.rs
+++ b/drivers/gpu/nova-core/regs.rs
@@ -4,110 +4,37 @@
 use kernel::{
     io::{
         register,
-        register::WithBase,
-        Io, //
+        Io,
+        Mmio, //
     },
-    prelude::*,
     sizes::SizeConstants,
     time, //
 };
+use pin_init::Zeroable;
 
 use crate::{
-    driver::Bar0,
+    driver::NovaRegisters,
     falcon::{
         DmaTrfCmdSize,
         FalconCoreRev,
         FalconCoreRevSubversion,
-        FalconEngine,
         FalconFbifMemType,
         FalconFbifTarget,
         FalconMem,
         FalconModSelAlgo,
         FalconSecurityModel,
-        PFalcon2Base,
-        PFalconBase,
+        PFalcon2Registers,
+        PFalconRegisters,
         PeregrineCoreSelect, //
     },
-    gpu::{
-        Architecture,
-        Chipset, //
-    },
+    mm::tlb::TlbAckMode, //
 };
 
-// PMC
-
-register! {
-    /// Basic revision information about the GPU.
-    pub(crate) NV_PMC_BOOT_0(u32) @ 0x00000000 {
-        /// Lower bits of the architecture.
-        28:24   architecture_0;
-        /// Implementation version of the architecture.
-        23:20   implementation;
-        /// MSB of the architecture.
-        8:8     architecture_1;
-        /// Major revision of the chip.
-        7:4     major_revision;
-        /// Minor revision of the chip.
-        3:0     minor_revision;
-    }
-
-    /// Extended architecture information.
-    pub(crate) NV_PMC_BOOT_42(u32) @ 0x00000a00 {
-        /// Architecture value.
-        29:24   architecture ?=> Architecture;
-        /// Implementation version of the architecture.
-        23:20   implementation;
-        /// Major revision of the chip.
-        19:16   major_revision;
-        /// Minor revision of the chip.
-        15:12   minor_revision;
-    }
-}
-
-impl NV_PMC_BOOT_0 {
-    pub(crate) fn is_older_than_fermi(self) -> bool {
-        // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals :
-        const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc;
-
-        // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than
-        // GF100, means "older than Fermi".
-        self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100
-    }
-}
-
-impl NV_PMC_BOOT_42 {
-    /// Combines `architecture` and `implementation` to obtain a code unique to the chipset.
-    pub(crate) fn chipset(self) -> Result<Chipset> {
-        self.architecture()
-            .map(|arch| {
-                ((arch as u32) << Self::IMPLEMENTATION_RANGE.len())
-                    | u32::from(self.implementation())
-            })
-            .and_then(Chipset::try_from)
-    }
-
-    /// Returns the raw architecture value from the register.
-    fn architecture_raw(self) -> u8 {
-        ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start())
-            & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8
-    }
-}
-
-impl kernel::fmt::Display for NV_PMC_BOOT_42 {
-    fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
-        write!(
-            f,
-            "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})",
-            self.inner,
-            self.architecture_raw(),
-            self.implementation()
-        )
-    }
-}
-
 // PBUS
 
 register! {
+    base: NovaRegisters;
+
     pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {}
 }
 
@@ -121,6 +48,8 @@ register! {
 // number.
 
 register! {
+    base: NovaRegisters;
+
     /// Boot Sequence Interface (BSI) register used to determine
     /// if GSP reload/resume has completed during the boot process.
     pub(crate) NV_PGC6_BSI_SECURE_SCRATCH_14(u32) @ 0x001180f8 {
@@ -175,6 +104,8 @@ impl NV_USABLE_FB_SIZE_IN_MB {
 pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16;
 
 register! {
+    base: NovaRegisters;
+
     pub(crate) NV_FUSE_OPT_FPF_NVDEC_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x00824100 {
         15:0    data => u16;
     }
@@ -191,30 +122,32 @@ register! {
 // PFALCON
 
 register! {
-    pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ PFalconBase + 0x00000004 {
+    base: PFalconRegisters;
+
+    pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ 0x00000004 {
         6:6     swgen0 => bool;
         4:4     halt => bool;
     }
 
-    pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ PFalconBase + 0x00000040 {
+    pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ 0x00000040 {
         31:0    value => u32;
     }
 
-    pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ PFalconBase + 0x00000044 {
+    pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ 0x00000044 {
         31:0    value => u32;
     }
 
     /// Used to store version information about the firmware running
     /// on the Falcon processor.
-    pub(crate) NV_PFALCON_FALCON_OS(u32) @ PFalconBase + 0x00000080 {
+    pub(crate) NV_PFALCON_FALCON_OS(u32) @ 0x00000080 {
         31:0    value => u32;
     }
 
-    pub(crate) NV_PFALCON_FALCON_RM(u32) @ PFalconBase + 0x00000084 {
+    pub(crate) NV_PFALCON_FALCON_RM(u32) @ 0x00000084 {
         31:0    value => u32;
     }
 
-    pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ PFalconBase + 0x000000f4 {
+    pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ 0x000000f4 {
         /// Signal indicating that reset is completed (GA102+).
         31:31   reset_ready => bool;
         /// RISC-V branch privilege lockdown bit.
@@ -224,17 +157,17 @@ register! {
         10:10   riscv => bool;
     }
 
-    pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ PFalconBase + 0x00000100 {
+    pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ 0x00000100 {
         6:6     alias_en => bool;
         4:4     halted => bool;
         1:1     startcpu => bool;
     }
 
-    pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ PFalconBase + 0x00000104 {
+    pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ 0x00000104 {
         31:0    value => u32;
     }
 
-    pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ PFalconBase + 0x0000010c {
+    pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ 0x0000010c {
         7:7     secure_stat => bool;
         6:3     dmaq_num;
         2:2     imem_scrubbing => bool;
@@ -242,15 +175,15 @@ register! {
         0:0     require_ctx => bool;
     }
 
-    pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ PFalconBase + 0x00000110 {
+    pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ 0x00000110 {
         31:0    base => u32;
     }
 
-    pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ PFalconBase + 0x00000114 {
+    pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ 0x00000114 {
         23:0    offs;
     }
 
-    pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ PFalconBase + 0x00000118 {
+    pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ 0x00000118 {
         16:16   set_dmtag;
         14:12   ctxdma;
         10:8    size ?=> DmaTrfCmdSize;
@@ -261,15 +194,15 @@ register! {
         0:0     full => bool;
     }
 
-    pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ PFalconBase + 0x0000011c {
+    pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ 0x0000011c {
         31:0    offs => u32;
     }
 
-    pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ PFalconBase + 0x00000128 {
+    pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ 0x00000128 {
         8:0     base;
     }
 
-    pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ PFalconBase + 0x0000012c {
+    pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ 0x0000012c {
         /// Core revision subversion.
         7:6     core_rev_subversion => FalconCoreRevSubversion;
         /// Security model.
@@ -278,12 +211,12 @@ register! {
         3:0     core_rev ?=> FalconCoreRev;
     }
 
-    pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ PFalconBase + 0x00000130 {
+    pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ 0x00000130 {
         1:1     startcpu => bool;
     }
 
     /// IMEM access control register. Up to 4 ports are available for IMEM access.
-    pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ PFalconBase + 0x00000180 {
+    pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ 0x00000180 {
         /// Access secure IMEM.
         28:28     secure => bool;
         /// Auto-increment on write.
@@ -294,17 +227,17 @@ register! {
 
     /// IMEM data register. Reading/writing this register accesses IMEM at the address
     /// specified by the corresponding IMEMC register.
-    pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ PFalconBase + 0x00000184 {
+    pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ 0x00000184 {
         31:0      data;
     }
 
     /// IMEM tag register. Used to set the tag for the current IMEM block.
-    pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ PFalconBase + 0x00000188 {
+    pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ 0x00000188 {
         15:0      tag;
     }
 
     /// DMEM access control register. Up to 8 ports are available for DMEM access.
-    pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ PFalconBase + 0x000001c0 {
+    pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ 0x000001c0 {
         /// Auto-increment on write.
         24:24     aincw => bool;
         /// DMEM block and word offset.
@@ -313,29 +246,29 @@ register! {
 
     /// DMEM data register. Reading/writing this register accesses DMEM at the address
     /// specified by the corresponding DMEMC register.
-    pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ PFalconBase + 0x000001c4 {
+    pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ 0x000001c4 {
         31:0      data;
     }
 
     /// Actually known as `NV_PSEC_FALCON_ENGINE` and `NV_PGSP_FALCON_ENGINE` depending on the
     /// falcon instance.
-    pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ PFalconBase + 0x000003c0 {
+    pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ 0x000003c0 {
         0:0     reset => bool;
     }
 
-    pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ PFalconBase + 0x00000600 {
+    pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 {
         2:2     mem_type => FalconFbifMemType;
         1:0     target ?=> FalconFbifTarget;
     }
 
-    pub(crate) NV_PFALCON_FBIF_CTL(u32) @ PFalconBase + 0x00000624 {
+    pub(crate) NV_PFALCON_FBIF_CTL(u32) @ 0x00000624 {
         7:7     allow_phys_no_ctx => bool;
     }
 
     // Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell).
     // These provide the falcon external memory communication interface.
 
-    pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ PFalconBase + 0x00000ac0 {
+    pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ 0x00000ac0 {
         /// EMEM byte offset (4-byte aligned) within the block.
         7:2     offs;
         /// EMEM block to access.
@@ -346,7 +279,7 @@ register! {
         25:25   aincr => bool;
     }
 
-    pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ PFalconBase + 0x00000ac4 {
+    pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ 0x00000ac4 {
         31:0    data => u32;
     }
 }
@@ -372,13 +305,13 @@ impl NV_PFALCON_FALCON_DMATRFCMD {
 
 impl NV_PFALCON_FALCON_ENGINE {
     /// Resets the falcon
-    pub(crate) fn reset_engine<E: FalconEngine>(bar: Bar0<'_>) {
-        bar.update(Self::of::<E>(), |r| r.with_reset(true));
+    pub(crate) fn reset_engine(pfalcon: Mmio<'_, PFalconRegisters>) {
+        pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(true));
 
         // TIMEOUT: falcon engine should not take more than 10us to reset.
         time::delay::fsleep(time::Delta::from_micros(10));
 
-        bar.update(Self::of::<E>(), |r| r.with_reset(false));
+        pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(false));
     }
 }
 
@@ -392,21 +325,23 @@ impl NV_PFALCON_FALCON_HWCFG2 {
 /* PFALCON2 */
 
 register! {
-    pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ PFalcon2Base + 0x00000180 {
+    base: PFalcon2Registers;
+
+    pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ 0x00000180 {
         7:0     algo ?=> FalconModSelAlgo;
     }
 
-    pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ PFalcon2Base + 0x00000198 {
+    pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ 0x00000198 {
         7:0    ucode_id => u8;
     }
 
-    pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ PFalcon2Base + 0x0000019c {
+    pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ 0x0000019c {
         31:0    value => u32;
     }
 
     /// OpenRM defines this as a register array, but doesn't specify its size and only uses its
     /// first element. Be conservative until we know the actual size or need to use more registers.
-    pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ PFalcon2Base + 0x00000210 {
+    pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ 0x00000210 {
         31:0    value => u32;
     }
 }
@@ -414,21 +349,23 @@ register! {
 // PRISCV
 
 register! {
+    base: PFalcon2Registers;
+
     /// RISC-V status register for debug (Turing and GA100 only).
     /// Reflects current RISC-V core status.
-    pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ PFalcon2Base + 0x00000240 {
+    pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ 0x00000240 {
         /// RISC-V core active/inactive status.
         0:0     active_stat => bool;
     }
 
     /// GA102 and later.
-    pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 {
+    pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ 0x00000388 {
         7:7     active_stat => bool;
         4:4     halted => bool;
     }
 
     /// GA102 and later.
-    pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ PFalcon2Base + 0x00000668 {
+    pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ 0x00000668 {
         8:8     br_fetch => bool;
         4:4     core_select => PeregrineCoreSelect;
         0:0     valid => bool;
@@ -439,6 +376,8 @@ register! {
 // These registers manage falcon EMEM communication queues.
 
 register! {
+    base: NovaRegisters;
+
     pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 {
         31:0    address => u32;
     }
@@ -462,9 +401,13 @@ register! {
 pub(crate) mod gm107 {
     use kernel::io::register;
 
+    use crate::driver::NovaRegisters;
+
     // FUSE
 
     register! {
+        base: NovaRegisters;
+
         pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00021c04 {
             0:0     display_disabled => bool;
         }
@@ -474,9 +417,13 @@ pub(crate) mod gm107 {
 pub(crate) mod ga100 {
     use kernel::io::register;
 
+    use crate::driver::NovaRegisters;
+
     // FUSE
 
     register! {
+        base: NovaRegisters;
+
         pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00820c04 {
             0:0     display_disabled => bool;
         }
@@ -488,9 +435,13 @@ pub(crate) const NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS: u32 = 0
 pub(crate) mod gh100 {
     use kernel::io::register;
 
+    use crate::driver::NovaRegisters;
+
     // PTHERM
 
     register! {
+        base: NovaRegisters;
+
         pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc {
             31:0    data;
         }
@@ -505,9 +456,13 @@ pub(crate) mod gh100 {
 pub(crate) mod gb202 {
     use kernel::io::register;
 
+    use crate::driver::NovaRegisters;
+
     // PTHERM
 
     register! {
+        base: NovaRegisters;
+
         pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc {
             31:0    data;
         }
@@ -518,3 +473,69 @@ pub(crate) mod gb202 {
         }
     }
 }
+
+// MMU TLB
+
+register! {
+    base: NovaRegisters;
+
+    /// TLB flush register: PDB address lower bits.
+    pub(crate) NV_TLB_FLUSH_PDB_LO(u32) @ 0x00b830a0 {
+        /// PDB address bits [39:8].
+        31:0    pdb_lo => u32;
+    }
+
+    /// TLB flush register: PDB address higher bits.
+    pub(crate) NV_TLB_FLUSH_PDB_HI(u32) @ 0x00b830a4 {
+        /// PDB address bits [47:40].
+        7:0     pdb_hi => u8;
+    }
+
+    /// TLB flush control register.
+    pub(crate) NV_TLB_FLUSH_CTRL(u32) @ 0x00b830b0 {
+        /// Invalidate every VA in the PDB selected by `NV_TLB_FLUSH_PDB_LO/HI`.
+        0:0     all_va => bool;
+        /// Invalidate TLBs for all PDBs (ignores `NV_TLB_FLUSH_PDB_LO/HI`).
+        1:1     all_pdb => bool;
+        /// Restrict the flush to the HUB MMU's TLBs; skip broadcasting to the
+        /// per-GPC L2 TLBs.
+        ///
+        /// The GPU MMU has a two-level TLB hierarchy:
+        /// 1. The *HUB MMU* sits at the top and serves memory requests from
+        ///    "host-side" engines: the host/channel interface, copy engines,
+        ///    display, and BAR1/BAR2 accesses.
+        /// 2. Each GPC (Graphics Processing Cluster — the block that houses
+        ///    shader cores / SMs) has its own L2 TLB that serves requests from
+        ///    the compute and graphics engines inside the cluster.
+        ///
+        /// When set, only the HUB TLBs are invalidated. This is a performance
+        /// optimization for flushes that only affect HUB-side mappings (e.g.
+        /// BAR1/BAR2 windows), where fanning the invalidation out to every
+        /// GPC's L2 TLB would be wasted work. Must be false when flushing
+        /// mappings that may be cached by compute/graphics engines.
+        2:2     hubtlb_only => bool;
+        /// Invalidation acknowledgment scope. See [`TlbAckMode`] for details.
+        8:7     ack ?=> TlbAckMode;
+        /// Write 1 to kick off the flush. Hardware clears this bit when the
+        /// flush completes; reads as 1 while the flush is in progress.
+        31:31   trigger => bool;
+    }
+}
+
+impl NV_TLB_FLUSH_PDB_LO {
+    /// Create a register value from a PDB address.
+    ///
+    /// Extracts bits [39:8] of the address and shifts it right by 8 bits.
+    pub(crate) fn from_pdb_addr(addr: u64) -> Self {
+        Self::zeroed().with_pdb_lo(((addr >> 8) & 0xFFFF_FFFF) as u32)
+    }
+}
+
+impl NV_TLB_FLUSH_PDB_HI {
+    /// Create a register value from a PDB address.
+    ///
+    /// Extracts bits [47:40] of the address and shifts it right by 40 bits.
+    pub(crate) fn from_pdb_addr(addr: u64) -> Self {
+        Self::zeroed().with_pdb_hi(((addr >> 40) & 0xFF) as u8)
+    }
+}
diff --git a/drivers/gpu/nova-core/selftest.rs b/drivers/gpu/nova-core/selftest.rs
new file mode 100644
index 00000000000000..f5b5965b7e6adb
--- /dev/null
+++ b/drivers/gpu/nova-core/selftest.rs
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+//! Assertion macros for driver self-tests.
+//!
+//! Self-tests run against live hardware during probe, so a failed assertion should not panic. These
+//! macros log the failure on the device and fail the enclosing test by returning
+//! [`EIO`](kernel::error::code::EIO) instead.
+
+/// Like [`assert!`], but logs the failure via `dev` and fails the enclosing test instead of
+/// panicking.
+///
+/// As with [`assert!`], a custom message with format arguments can follow the condition.
+#[macro_export]
+macro_rules! selftest_assert {
+    ($dev:expr, $cond:expr $(,)?) => {
+        $crate::selftest_assert!($dev, $cond, "assertion failed: {}", ::core::stringify!($cond))
+    };
+    ($dev:expr, $cond:expr, $($arg:tt)+) => {{
+        if !$cond {
+            ::kernel::dev_err!(
+                $dev,
+                "Selftest: {}:{}: {}\n",
+                ::core::file!(),
+                ::core::line!(),
+                ::kernel::prelude::fmt!($($arg)+)
+            );
+            return Err(::kernel::error::code::EIO);
+        }
+    }};
+}
+
+/// Like [`assert_eq!`], but logs the failure via `dev` and fails the enclosing test instead of
+/// panicking.
+///
+/// As with [`assert_eq!`], a custom message with format arguments can follow the compared values.
+#[macro_export]
+macro_rules! selftest_assert_eq {
+    ($dev:expr, $left:expr, $right:expr $(,)?) => {
+        match (&$left, &$right) {
+            (left, right) => $crate::selftest_assert!(
+                $dev,
+                left == right,
+                "assertion `{} == {}` failed: left {:?}, right {:?}",
+                ::core::stringify!($left),
+                ::core::stringify!($right),
+                left,
+                right
+            ),
+        }
+    };
+    ($dev:expr, $left:expr, $right:expr, $($arg:tt)+) => {
+        match (&$left, &$right) {
+            (left, right) => $crate::selftest_assert!(
+                $dev,
+                left == right,
+                "assertion `left == right` failed: {}: left {:?}, right {:?}",
+                ::kernel::prelude::fmt!($($arg)+),
+                left,
+                right
+            ),
+        }
+    };
+}
diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs
index c03650ee522653..9c214b9f4dd9d2 100644
--- a/drivers/gpu/nova-core/vbios.rs
+++ b/drivers/gpu/nova-core/vbios.rs
@@ -16,7 +16,10 @@ use kernel::{
 };
 
 use crate::{
-    driver::Bar0,
+    driver::{
+        Bar0,
+        NovaRegisters, //
+    },
     firmware::{
         fwsec::Bcrt30Rsa3kSignature,
         FalconUCodeDesc,
@@ -92,12 +95,16 @@ impl<'a> VbiosIterator<'a> {
     fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result<usize> {
         // IFR Header in VBIOS.
         register! {
+            base: NovaRegisters;
+
             NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 {
                 31:0    signature;
             }
         }
 
         register! {
+            base: NovaRegisters;
+
             NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 {
                 30:16   fixed_data_size;
                 15:8    version => u8;
@@ -105,6 +112,8 @@ impl<'a> VbiosIterator<'a> {
         }
 
         register! {
+            base: NovaRegisters;
+
             NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 {
                 19:0 total_data_size;
             }
diff --git a/drivers/gpu/tests/gpu_buddy_test.c b/drivers/gpu/tests/gpu_buddy_test.c
index ed4c1c3acb3c8d..b75d32ca6ca0a6 100644
--- a/drivers/gpu/tests/gpu_buddy_test.c
+++ b/drivers/gpu/tests/gpu_buddy_test.c
@@ -38,7 +38,7 @@ static void gpu_test_buddy_subtree_offset_alignment_stress(struct kunit *test)
 	};
 	struct list_head allocated[ARRAY_SIZE(alignments)];
 	unsigned int i, max_subtree_align = 0;
-	int ret, tree, order;
+	int ret, order;
 	struct gpu_buddy mm;
 
 	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K),
@@ -78,15 +78,11 @@ static void gpu_test_buddy_subtree_offset_alignment_stress(struct kunit *test)
 		}
 
 		for (order = mm.max_order; order >= 0 && !root; order--) {
-			for (tree = 0; tree < 2; tree++) {
-				node = mm.free_trees[tree][order].rb_node;
-				if (node) {
-					root = container_of(node,
-							    struct gpu_buddy_block,
-							    rb);
-					break;
-				}
-			}
+			node = mm.free_tree[order].rb_node;
+			if (node)
+				root = container_of(node,
+						    struct gpu_buddy_block,
+						    rb);
 		}
 
 		KUNIT_ASSERT_NOT_NULL(test, root);
@@ -97,15 +93,13 @@ static void gpu_test_buddy_subtree_offset_alignment_stress(struct kunit *test)
 		gpu_buddy_free_list(&mm, &allocated[i], 0);
 
 		for (order = 0; order <= mm.max_order; order++) {
-			for (tree = 0; tree < 2; tree++) {
-				node = mm.free_trees[tree][order].rb_node;
-				if (!node)
-					continue;
-
-				block = container_of(node, struct gpu_buddy_block, rb);
-				max_subtree_align = max(max_subtree_align,
-							block->subtree_max_alignment);
-			}
+			node = mm.free_tree[order].rb_node;
+			if (!node)
+				continue;
+
+			block = container_of(node, struct gpu_buddy_block, rb);
+			max_subtree_align = max(max_subtree_align,
+						block->subtree_max_alignment);
 		}
 
 		KUNIT_EXPECT_GE(test, max_subtree_align, ilog2(alignments[i]));
@@ -289,6 +283,121 @@ static void gpu_test_buddy_fragmentation_performance(struct kunit *test)
 	gpu_buddy_fini(&mm);
 }
 
+static void gpu_test_buddy_dirty_tracker_performance(struct kunit *test)
+{
+	struct gpu_buddy_block *block, *tmp;
+	unsigned long elapsed_ms;
+	LIST_HEAD(clear_blocks);
+	LIST_HEAD(dirty_blocks);
+	LIST_HEAD(allocated);
+	struct gpu_buddy mm;
+	LIST_HEAD(results);
+	ktime_t start, end;
+	int i, count;
+
+	/*
+	 * Contiguous alloc latency after alternating clear/dirty fragmentation
+	 *
+	 * Fill a 4 GiB pool with 4 KiB allocations, partition them into
+	 * alternating cleared and dirty sets, then free both.  In the old
+	 * dual-tree design every adjacent buddy pair has one cleared half and
+	 * one dirty half, so the pair sits on opposite sides of the clear/dirty
+	 * merge barrier and cannot be coalesced at free() time.  The pool
+	 * stays fully fragmented and the subsequent contiguous 2 GiB allocation
+	 * (smaller than the 4 GiB span so it routes through the contiguous
+	 * allocator rather than the exact-range fast path) has to invoke
+	 * __force_merge() to rebuild a 2 GiB block before it can succeed.  With
+	 * the dirty-tracker design buddy pairs coalesce unconditionally during
+	 * free(), so the pool is already at max_order before the timed alloc
+	 * begins and __force_merge() is not needed.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, SZ_4G, SZ_4K),
+			       "buddy_init failed\n");
+
+	for (i = 0; i < SZ_4G / SZ_4K; i++)
+		KUNIT_ASSERT_FALSE_MSG(test,
+				       gpu_buddy_alloc_blocks(&mm, 0, SZ_4G, SZ_4K, SZ_4K,
+							      &allocated, 0),
+				       "buddy_alloc hit an error size=%u\n", SZ_4K);
+
+	count = 0;
+	list_for_each_entry_safe(block, tmp, &allocated, link) {
+		if (count++ % 2 == 0)
+			list_move_tail(&block->link, &clear_blocks);
+		else
+			list_move_tail(&block->link, &dirty_blocks);
+	}
+
+	gpu_buddy_free_list(&mm, &clear_blocks, GPU_BUDDY_CLEARED);
+	gpu_buddy_free_list(&mm, &dirty_blocks, 0);
+
+	start = ktime_get();
+	KUNIT_ASSERT_FALSE_MSG(test,
+			       gpu_buddy_alloc_blocks(&mm, 0, SZ_4G, SZ_2G, SZ_2G,
+						      &results,
+						      GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+			       "contiguous alloc failed\n");
+	end = ktime_get();
+	elapsed_ms = ktime_to_ms(ktime_sub(end, start));
+
+	kunit_info(test, "Contiguous alloc after fragmentation: %lu ms\n",
+		   elapsed_ms);
+
+	gpu_buddy_free_list(&mm, &results, 0);
+	gpu_buddy_fini(&mm);
+
+	/*
+	 * Repeated alloc throughput from a maximally fragmented pool
+	 *
+	 * Fill a 4 GiB pool with 4 KiB allocations, free even-indexed blocks
+	 * as cleared and odd-indexed blocks as dirty.  The alternating pattern
+	 * ensures every adjacent buddy pair has one cleared half and one dirty
+	 * half, so each pair lands on opposite sides of the old merge barrier.
+	 * Each of the 16 384 x 256 KiB allocations in the timed loop asks for a
+	 * single contiguous block (size == min_block_size, CONTIGUOUS flag), so
+	 * it cannot be satisfied by 64 stray 4 KiB blocks: under the old design
+	 * every alloc has to pay the __force_merge() cost to rebuild a 256 KiB
+	 * block.  With the dirty-tracker design the pool collapses to one
+	 * max_order block during free(), so each alloc is a simple O(log N)
+	 * split.
+	 */
+	KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, SZ_4G, SZ_4K),
+			       "buddy_init failed\n");
+
+	for (i = 0; i < SZ_4G / SZ_4K; i++)
+		KUNIT_ASSERT_FALSE_MSG(test,
+				       gpu_buddy_alloc_blocks(&mm, 0, SZ_4G, SZ_4K, SZ_4K,
+							      &allocated, 0),
+				       "buddy_alloc hit an error size=%u\n", SZ_4K);
+
+	count = 0;
+	list_for_each_entry_safe(block, tmp, &allocated, link) {
+		if (count++ % 2 == 0)
+			list_move_tail(&block->link, &clear_blocks);
+		else
+			list_move_tail(&block->link, &dirty_blocks);
+	}
+
+	gpu_buddy_free_list(&mm, &clear_blocks, GPU_BUDDY_CLEARED);
+	gpu_buddy_free_list(&mm, &dirty_blocks, 0);
+
+	start = ktime_get();
+	for (i = 0; i < SZ_4G / SZ_256K; i++)
+		KUNIT_ASSERT_FALSE_MSG(test,
+				       gpu_buddy_alloc_blocks(&mm, 0, SZ_4G, SZ_256K, SZ_256K,
+							      &results,
+							      GPU_BUDDY_CONTIGUOUS_ALLOCATION),
+				       "buddy_alloc hit an error size=%u\n", SZ_256K);
+	end = ktime_get();
+	elapsed_ms = ktime_to_ms(ktime_sub(end, start));
+
+	kunit_info(test, "Repeated 256 KiB allocs from fragmented pool: %lu ms\n",
+		   elapsed_ms);
+
+	gpu_buddy_free_list(&mm, &results, 0);
+	gpu_buddy_fini(&mm);
+}
+
 static void gpu_test_buddy_alloc_range_bias(struct kunit *test)
 {
 	u32 mm_size, size, ps, bias_size, bias_start, bias_end, bias_rem;
@@ -931,10 +1040,12 @@ static void gpu_test_buddy_alloc_clear(struct kunit *test)
 	/*
 	 * Create a new mm. Intentionally fragment the address space by creating
 	 * two alternating lists. Free both lists, one as dirty the other as clean.
-	 * Try to allocate double the previous size with matching min_page_size. The
-	 * allocation should never fail as it calls the force_merge. Also check that
-	 * the page is always dirty after force_merge. Free the page as dirty, then
-	 * repeat the whole thing, increment the order until we hit the max_order.
+	 * Try to allocate double the previous size with matching min_page_size.
+	 * The allocation should never fail because buddy pairs coalesce
+	 * unconditionally at free() time, rebuilding the larger block. Also check
+	 * that the page is always dirty (a merged block spanning a dirty half is
+	 * dirty). Free the page as dirty, then repeat the whole thing, increment
+	 * the order until we hit the max_order.
 	 */
 
 	i = 0;
@@ -1027,7 +1138,7 @@ static void gpu_test_buddy_alloc_clear(struct kunit *test)
 	gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
 	gpu_buddy_free_list(&mm, &dirty, 0);
 	gpu_buddy_reset_clear(&mm, false);
-	KUNIT_EXPECT_EQ(test, mm.clear_avail, 0);
+	KUNIT_EXPECT_EQ(test, gpu_buddy_clear_avail(&mm), 0);
 	gpu_buddy_fini(&mm);
 
 	/*
@@ -1050,7 +1161,7 @@ static void gpu_test_buddy_alloc_clear(struct kunit *test)
 	gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED);
 	gpu_buddy_free_list(&mm, &dirty, 0);
 	gpu_buddy_reset_clear(&mm, true);
-	KUNIT_EXPECT_EQ(test, mm.clear_avail, mm_size);
+	KUNIT_EXPECT_EQ(test, gpu_buddy_clear_avail(&mm), mm_size);
 	gpu_buddy_fini(&mm);
 }
 
@@ -1496,6 +1607,7 @@ static struct kunit_case gpu_buddy_tests[] = {
 	KUNIT_CASE(gpu_test_buddy_alloc_range),
 	KUNIT_CASE(gpu_test_buddy_alloc_range_bias),
 	KUNIT_CASE_SLOW(gpu_test_buddy_fragmentation_performance),
+	KUNIT_CASE_SLOW(gpu_test_buddy_dirty_tracker_performance),
 	KUNIT_CASE(gpu_test_buddy_alloc_exceeds_max_order),
 	KUNIT_CASE(gpu_test_buddy_offset_aligned_allocation),
 	KUNIT_CASE(gpu_test_buddy_subtree_offset_alignment_stress),
diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c
index 22cf52b78b7552..9431d83b38a978 100644
--- a/drivers/gpu/vga/vga_switcheroo.c
+++ b/drivers/gpu/vga/vga_switcheroo.c
@@ -32,9 +32,9 @@
 
 #include <linux/apple-gmux.h>
 #include <linux/debugfs.h>
-#include <linux/fb.h>
+#include <linux/export.h>
 #include <linux/fs.h>
-#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
@@ -90,7 +90,6 @@
 /**
  * struct vga_switcheroo_client - registered client
  * @pdev: client pci device
- * @fb_info: framebuffer to which console is remapped on switching
  * @pwr_state: current power state if manual power control is used.
  *	For driver power control, call vga_switcheroo_pwr_state().
  * @ops: client callbacks
@@ -105,12 +104,11 @@
  * @vga_dev: pci device, indicate which GPU is bound to current audio client
  *
  * Registered client. A client can be either a GPU or an audio device on a GPU.
- * For audio clients, the @fb_info and @active members are bogus. For GPU
- * clients, the @vga_dev is bogus.
+ * For audio clients, the @active member is bogus. For GPU clients, the @vga_dev
+ * is bogus.
  */
 struct vga_switcheroo_client {
 	struct pci_dev *pdev;
-	struct fb_info *fb_info;
 	enum vga_switcheroo_state pwr_state;
 	const struct vga_switcheroo_client_ops *ops;
 	enum vga_switcheroo_client_id id;
@@ -514,27 +512,6 @@ void vga_switcheroo_unregister_client(struct pci_dev *pdev)
 }
 EXPORT_SYMBOL(vga_switcheroo_unregister_client);
 
-/**
- * vga_switcheroo_client_fb_set() - set framebuffer of a given client
- * @pdev: client pci device
- * @info: framebuffer
- *
- * Set framebuffer of a given client. The console will be remapped to this
- * on switching.
- */
-void vga_switcheroo_client_fb_set(struct pci_dev *pdev,
-				 struct fb_info *info)
-{
-	struct vga_switcheroo_client *client;
-
-	mutex_lock(&vgasr_mutex);
-	client = find_client_from_pci(&vgasr_priv.clients, pdev);
-	if (client)
-		client->fb_info = info;
-	mutex_unlock(&vgasr_mutex);
-}
-EXPORT_SYMBOL(vga_switcheroo_client_fb_set);
-
 /**
  * vga_switcheroo_lock_ddc() - temporarily switch DDC lines to a given client
  * @pdev: client pci device
@@ -733,10 +710,8 @@ static int vga_switchto_stage2(struct vga_switcheroo_client *new_client)
 	if (!active->driver_power_control)
 		set_audio_state(active->id, VGA_SWITCHEROO_OFF);
 
-#if defined(CONFIG_FB)
-	if (new_client->fb_info)
-		fb_switch_outputs(new_client->fb_info);
-#endif
+	if (new_client->ops->pre_switch)
+		new_client->ops->pre_switch(new_client->pdev);
 
 	mutex_lock(&vgasr_priv.mux_hw_lock);
 	ret = vgasr_priv.handler->switchto(new_client->id);
@@ -744,8 +719,8 @@ static int vga_switchto_stage2(struct vga_switcheroo_client *new_client)
 	if (ret)
 		return ret;
 
-	if (new_client->ops->reprobe)
-		new_client->ops->reprobe(new_client->pdev);
+	if (new_client->ops->post_switch)
+		new_client->ops->post_switch(new_client->pdev);
 
 	if (vga_switcheroo_pwr_state(active) == VGA_SWITCHEROO_ON)
 		vga_switchoff(active);
diff --git a/drivers/greybus/svc_watchdog.c b/drivers/greybus/svc_watchdog.c
index 16e6de5e9eff70..b318eb34bccaa1 100644
--- a/drivers/greybus/svc_watchdog.c
+++ b/drivers/greybus/svc_watchdog.c
@@ -7,6 +7,7 @@
 
 #include <linux/delay.h>
 #include <linux/suspend.h>
+#include <linux/umh.h>
 #include <linux/workqueue.h>
 #include <linux/greybus.h>
 
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index a81bf51cbcf103..0e3a0ccd6901e8 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -527,15 +527,6 @@ config HID_KYE
 	- MousePen i608X tablet
 	- EasyPen M610X tablet
 
-config HID_KYSONA
-	tristate "Kysona devices"
-	depends on USB_HID
-	help
-	Support for Kysona mice.
-
-	Say Y here if you have a Kysona M600 mouse
-	and want to be able to read its battery capacity.
-
 config HID_UCLOGIC
 	tristate "UC-Logic"
 	depends on USB_HID
@@ -1210,7 +1201,8 @@ config HID_STEELSERIES
 	depends on USB_HID
 	help
 	Support for Steelseries SRW-S1 steering wheel, and the Steelseries
-	Arctis 1 Wireless for XBox headset.
+	Arctis 1 Wireless for XBox, Arctis 7 (2018), Arctis 9, and Arctis
+	Nova headsets.
 
 config HID_SUNPLUS
 	tristate "Sunplus wireless desktop"
@@ -1358,6 +1350,18 @@ config HID_UNIVERSAL_PIDFF
 
 	  Supports Moza Racing, Cammus, VRS, FFBeast and more.
 
+config HID_PULSAR
+	tristate "Pulsar gaming mouse support"
+	depends on USB_HID
+	select POWER_SUPPLY
+	help
+	  Support for Pulsar gaming mice (X2 V2, X2H, X2A, Xlite V3)
+	  connected via 1kHz/4kHz USB dongle or wired.
+	  Provides battery level, voltage, and charging status
+	  monitoring via the power supply framework.
+
+	  Additional supported devices: Kysona M600, ATK VXE R1 SE+
+
 config HID_WACOM
 	tristate "Wacom Intuos/Graphire tablet support (USB)"
 	depends on USB_HID
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index 48a863b245eed3..79384d905e0021 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -74,7 +74,6 @@ obj-$(CONFIG_HID_JABRA)		+= hid-jabra.o
 obj-$(CONFIG_HID_KENSINGTON)	+= hid-kensington.o
 obj-$(CONFIG_HID_KEYTOUCH)	+= hid-keytouch.o
 obj-$(CONFIG_HID_KYE)		+= hid-kye.o
-obj-$(CONFIG_HID_KYSONA)	+= hid-kysona.o
 obj-$(CONFIG_HID_LCPOWER)	+= hid-lcpower.o
 obj-$(CONFIG_HID_LENOVO)	+= hid-lenovo.o
 obj-$(CONFIG_HID_LENOVO_GO)	+= hid-lenovo-go.o
@@ -117,6 +116,7 @@ hid-picolcd-$(CONFIG_DEBUG_FS)		+= hid-picolcd_debugfs.o
 obj-$(CONFIG_HID_PLANTRONICS)	+= hid-plantronics.o
 obj-$(CONFIG_HID_PLAYSTATION)	+= hid-playstation.o
 obj-$(CONFIG_HID_PRIMAX)	+= hid-primax.o
+obj-$(CONFIG_HID_PULSAR)	+= hid-pulsar.o
 obj-$(CONFIG_HID_PXRC)		+= hid-pxrc.o
 obj-$(CONFIG_HID_RAKK)		+= hid-rakk.o
 obj-$(CONFIG_HID_RAPOO) += hid-rapoo.o
diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_common.h b/drivers/hid/amd-sfh-hid/amd_sfh_common.h
index f8c6b7fc34fb12..e8cc7f1d289cca 100644
--- a/drivers/hid/amd-sfh-hid/amd_sfh_common.h
+++ b/drivers/hid/amd-sfh-hid/amd_sfh_common.h
@@ -13,10 +13,15 @@
 #include <linux/auxiliary_bus.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
+#include <linux/sizes.h>
 #include "amd_sfh_hid.h"
 
 #define PCI_DEVICE_ID_AMD_MP2		0x15E4
 #define PCI_DEVICE_ID_AMD_MP2_1_1	0x164A
+#define PCI_DEVICE_ID_AMD_MP2_1_2	0x1152
+
+/* The BAR 2 size must cover the highest register offset (0x10958) */
+#define AMD_SFH_MIN_BAR_SIZE		SZ_128K
 
 #define AMD_C2P_MSG(regno) (0x10500 + ((regno) * 4))
 #define AMD_P2C_MSG(regno) (0x10680 + ((regno) * 4))
@@ -39,6 +44,7 @@ enum cmd_id {
 enum amd_mp2_version {
 	MP2_VER_V2 = 1,
 	MP2_VER_1_1 = 2,
+	MP2_VER_1_2 = 3,
 };
 
 struct amd_mp2_sensor_info {
diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
index eda26a094d3f17..aa09a9538f4f96 100644
--- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
+++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c
@@ -41,6 +41,8 @@ module_param_named(sensor_mask, sensor_mask_override, int, 0444);
 MODULE_PARM_DESC(sensor_mask, "override the detected sensors mask");
 
 static bool intr_disable = true;
+module_param_named(intr_disable, intr_disable, bool, 0444);
+MODULE_PARM_DESC(intr_disable, "override the interrupt disable sensor bit");
 
 static int amd_sfh_wait_response_v2(struct amd_mp2_dev *mp2, u8 sid, u32 sensor_sts)
 {
@@ -313,6 +315,13 @@ static const struct dmi_system_id dmi_sfh_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "HP ProBook x360 435 G7"),
 		},
 	},
+	{
+		.callback = mp2_disable_intr,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "VivoBook_ASUSLaptop TP420UA_TM420UA"),
+		},
+	},
 	{}
 };
 
@@ -497,6 +506,16 @@ static int amd_mp2_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i
 	if (rc)
 		return rc;
 
+	if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
+		dev_err(&pdev->dev, "BAR 2 is not IORESOURCE_MEM\n");
+		return -ENODEV;
+	}
+
+	if (pci_resource_len(pdev, 2) < AMD_SFH_MIN_BAR_SIZE) {
+		dev_err(&pdev->dev, "BAR 2 is too small\n");
+		return -EINVAL;
+	}
+
 	rc = pcim_iomap_regions(pdev, BIT(2), DRIVER_NAME);
 	if (rc)
 		return rc;
@@ -588,6 +607,7 @@ static SIMPLE_DEV_PM_OPS(amd_mp2_pm_ops, amd_mp2_pci_suspend,
 static const struct pci_device_id amd_mp2_pci_tbl[] = {
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_MP2) },
 	{ PCI_DEVICE_DATA(AMD, MP2_1_1, MP2_VER_1_1) },
+	{ PCI_DEVICE_DATA(AMD, MP2_1_2, MP2_VER_1_2) },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, amd_mp2_pci_tbl);
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c
index c8916afefa626f..b81a97dbb1b014 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_desc.c
@@ -187,6 +187,7 @@ static u8 get_input_rep(u8 current_index, int sensor_idx, int report_id,
 	struct sfh_gyro_data gyro_data;
 	struct sfh_mag_data mag_data;
 	struct sfh_als_data als_data;
+	struct sfh_hpd_data hpd_data;
 	struct hpd_status hpdstatus;
 	struct sfh_base_info binfo;
 	void __iomem *sensoraddr;
@@ -251,8 +252,16 @@ static u8 get_input_rep(u8 current_index, int sensor_idx, int report_id,
 		break;
 	case HPD_IDX:
 		get_common_inputs(&hpd_input.common_property, report_id);
-		hpdstatus.val = readl(mp2->mmio + amd_get_c2p_val(mp2, 4));
-		hpd_input.human_presence = hpdstatus.shpd.presence;
+		if (mp2->mp2_ver >= MP2_VER_1_2) {
+			sensoraddr = mp2->vsbase +
+				(HPD_IDX * SENSOR_DATA_MEM_SIZE_DEFAULT) +
+				OFFSET_SENSOR_DATA_DEFAULT;
+			memcpy_fromio(&hpd_data, sensoraddr, sizeof(struct sfh_hpd_data));
+			hpd_input.human_presence = hpd_data.status.shpd.presence;
+		} else {
+			hpdstatus.val = readl(mp2->mmio + amd_get_c2p_val(mp2, 4));
+			hpd_input.human_presence = hpdstatus.shpd.presence;
+		}
 		report_size = sizeof(hpd_input);
 		memcpy(input_report, &hpd_input, sizeof(hpd_input));
 		break;
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
index 60c185a9cd501b..bc0adb789896bb 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.c
@@ -98,6 +98,25 @@ void sfh_interface_init(struct amd_mp2_dev *mp2)
 	emp2 = mp2;
 }
 
+static unsigned int amd_sfh_read_c2p3(struct amd_mp2_dev *mp2)
+{
+	struct sfh_accel_data accel_data;
+	void __iomem *sensoraddr;
+	u32 val;
+
+	if (mp2->mp2_ver >= MP2_VER_1_2) {
+		sensoraddr = mp2->vsbase +
+			(SRA_IDX * SENSOR_DATA_MEM_SIZE_DEFAULT) +
+			OFFSET_SENSOR_DATA_DEFAULT;
+		memcpy_fromio(&accel_data, sensoraddr, sizeof(struct sfh_accel_data));
+		val = accel_data.c2p3_data;
+	} else {
+		val = readl(mp2->mmio + amd_get_c2p_val(mp2, 3));
+	}
+
+	return val;
+}
+
 static int amd_sfh_op_mode_info(u32 *op_mode)
 {
 	struct sfh_op_mode mode;
@@ -114,7 +133,7 @@ static int amd_sfh_op_mode_info(u32 *op_mode)
 	if (!present)
 		return -ENODEV;
 
-	mode.val = readl(emp2->mmio + amd_get_c2p_val(emp2, 3));
+	mode.val = amd_sfh_read_c2p3(emp2);
 	dev_dbg(&emp2->pdev->dev, "op-mode: %s (mode=%u)\n",
 		mode.op_mode.mode == SFH_MODE_TABLET ? "tablet" : "laptop",
 		mode.op_mode.mode);
@@ -133,7 +152,7 @@ static int amd_sfh_mode_info(u32 *platform_type, u32 *laptop_placement)
 	if (!emp2 || !emp2->dev_en.is_sra_present)
 		return -ENODEV;
 
-	mode.val = readl(emp2->mmio + amd_get_c2p_val(emp2, 3));
+	mode.val = amd_sfh_read_c2p3(emp2);
 
 	*platform_type = mode.op_mode.devicemode;
 
@@ -160,7 +179,9 @@ static int amd_sfh_mode_info(u32 *platform_type, u32 *laptop_placement)
 
 static int amd_sfh_hpd_info(u8 *user_present)
 {
+	struct sfh_hpd_data hpd_data;
 	struct hpd_status hpdstatus;
+	void __iomem *sensoraddr;
 
 	if (!user_present)
 		return -EINVAL;
@@ -168,8 +189,16 @@ static int amd_sfh_hpd_info(u8 *user_present)
 	if (!emp2 || !emp2->dev_en.is_hpd_present || !emp2->dev_en.is_hpd_enabled)
 		return -ENODEV;
 
-	hpdstatus.val = readl(emp2->mmio + amd_get_c2p_val(emp2, 4));
-	*user_present = hpdstatus.shpd.presence;
+	if (emp2->mp2_ver >= MP2_VER_1_2) {
+		sensoraddr = emp2->vsbase +
+			(HPD_IDX * SENSOR_DATA_MEM_SIZE_DEFAULT) +
+			OFFSET_SENSOR_DATA_DEFAULT;
+		memcpy_fromio(&hpd_data, sensoraddr, sizeof(struct sfh_hpd_data));
+		*user_present = hpd_data.status.shpd.presence;
+	} else {
+		hpdstatus.val = readl(emp2->mmio + amd_get_c2p_val(emp2, 4));
+		*user_present = hpdstatus.shpd.presence;
+	}
 
 	return 0;
 }
diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h
index 56258c4d1b3ade..eaa90c68904973 100644
--- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h
+++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_interface.h
@@ -130,6 +130,7 @@ struct sfh_accel_data {
 	struct sfh_common_data commondata;
 	struct sfh_float32 acceldata;
 	u32 accelstatus;
+	u32 c2p3_data;
 };
 
 struct sfh_gyro_data {
@@ -165,6 +166,12 @@ struct hpd_status {
 	};
 };
 
+struct sfh_hpd_data {
+	struct sfh_common_data commondata;
+	u32 reserved;
+	struct hpd_status status;
+};
+
 struct sfh_op_mode {
 	union {
 		u32 val;
diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c
index 44671dbdeca8e1..6b467c74601cb3 100644
--- a/drivers/hid/bpf/hid_bpf_dispatch.c
+++ b/drivers/hid/bpf/hid_bpf_dispatch.c
@@ -359,7 +359,7 @@ hid_bpf_release_context(struct hid_bpf_ctx *ctx)
 
 static int
 __hid_bpf_hw_check_params(struct hid_bpf_ctx *ctx, __u8 *buf, size_t *buf__sz,
-			  enum hid_report_type rtype)
+			  enum hid_report_type rtype, bool hw_request)
 {
 	struct hid_report_enum *report_enum;
 	struct hid_report *report;
@@ -388,6 +388,10 @@ __hid_bpf_hw_check_params(struct hid_bpf_ctx *ctx, __u8 *buf, size_t *buf__sz,
 
 	report_len = hid_report_len(report);
 
+	/* unnumbered reports need to have a report ID reserved in the first byte */
+	if (hw_request && report_enum->numbered == 0)
+		report_len += 1;
+
 	if (*buf__sz > report_len)
 		*buf__sz = report_len;
 
@@ -420,7 +424,7 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz,
 		return -EDEADLOCK;
 
 	/* check arguments */
-	ret = __hid_bpf_hw_check_params(ctx, buf, &size, rtype);
+	ret = __hid_bpf_hw_check_params(ctx, buf, &size, rtype, true);
 	if (ret)
 		return ret;
 
@@ -449,7 +453,7 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz,
 					      (u64)(long)ctx,
 					      true); /* prevent infinite recursions */
 
-	if (ret > size)
+	if (ret > (ssize_t)size)
 		ret = size;
 	if (ret > 0)
 		memcpy(buf, dma_data, ret);
@@ -480,7 +484,7 @@ hid_bpf_hw_output_report(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz)
 		return -EDEADLOCK;
 
 	/* check arguments */
-	ret = __hid_bpf_hw_check_params(ctx, buf, &size, HID_OUTPUT_REPORT);
+	ret = __hid_bpf_hw_check_params(ctx, buf, &size, HID_OUTPUT_REPORT, true);
 	if (ret)
 		return ret;
 
@@ -506,7 +510,7 @@ __hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum hid_report_type type, u8 *b
 		return -EDEADLOCK;
 
 	/* check arguments */
-	ret = __hid_bpf_hw_check_params(ctx, buf, &size, type);
+	ret = __hid_bpf_hw_check_params(ctx, buf, &size, type, false);
 	if (ret)
 		return ret;
 
diff --git a/drivers/hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c b/drivers/hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c
index e6ba2295dc7750..d081b7c0b771bb 100644
--- a/drivers/hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c
+++ b/drivers/hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c
@@ -15,6 +15,12 @@
 #define PID_L610			0x2012
 
 #define PEN_RDESC_SIZE			125
+/*
+ * The L610 pen descriptor is also seen 2 bytes shorter (123 bytes); the
+ * anchor bytes checked below (offsets 0-3, 16-17) are the same in both
+ * variants, so accept either length.
+ */
+#define PEN_RDESC_SIZE_SHORT		123
 #define SECONDARY_SWITCH_OFFSET		17
 
 HID_BPF_CONFIG(
@@ -44,7 +50,7 @@ int BPF_PROG(fix_secondary_barrel_rdesc, struct hid_bpf_ctx *hctx)
 	if (!data)
 		return 0; /* EPERM check */
 
-	if (hctx->size != PEN_RDESC_SIZE)
+	if (hctx->size != PEN_RDESC_SIZE && hctx->size != PEN_RDESC_SIZE_SHORT)
 		return 0;
 
 	if (data[0] != 0x05 || data[1] != 0x0d || /* Usage Page (Digitizers) */
@@ -65,7 +71,8 @@ HID_BPF_OPS(fix_secondary_barrel) = {
 SEC("syscall")
 int probe(struct hid_bpf_probe_args *ctx)
 {
-	ctx->retval = ctx->rdesc_size != PEN_RDESC_SIZE;
+	ctx->retval = ctx->rdesc_size != PEN_RDESC_SIZE &&
+		      ctx->rdesc_size != PEN_RDESC_SIZE_SHORT;
 	if (ctx->retval) {
 		ctx->retval = -EINVAL;
 		return 0;
diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c
index 67179e3fe39b7d..0556cb5645ebbf 100644
--- a/drivers/hid/hid-alps.c
+++ b/drivers/hid/hid-alps.c
@@ -407,6 +407,8 @@ static int u1_raw_event(struct alps_dev *hdata, u8 *data, int size)
 		return 1;
 
 	case U1_SP_ABSOLUTE_REPORT_ID:
+		if (!hdata->input2)
+			return 0;
 		sp_x = get_unaligned_le16(data+2);
 		sp_y = get_unaligned_le16(data+4);
 
@@ -738,7 +740,6 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi)
 			goto exit;
 		}
 
-		data->input2 = input2;
 		input2->phys = input->phys;
 		input2->name = "DualPoint Stick";
 		input2->id.bustype = BUS_I2C;
@@ -762,11 +763,12 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi)
 		__set_bit(INPUT_PROP_POINTER, input2->propbit);
 		__set_bit(INPUT_PROP_POINTING_STICK, input2->propbit);
 
-		if (input_register_device(data->input2)) {
+		if (input_register_device(input2)) {
 			input_free_device(input2);
 			ret = -ENOENT;
 			goto exit;
 		}
+		data->input2 = input2;
 	}
 
 exit:
@@ -823,6 +825,24 @@ static int alps_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	return 0;
 }
 
+static void alps_remove(struct hid_device *hdev)
+{
+	struct alps_dev *data = hid_get_drvdata(hdev);
+
+	/*
+	 * input2 ("DualPoint Stick") is allocated separately and is not
+	 * tracked in hdev->inputs, so the default remove path
+	 * (hid_hw_stop -> hidinput_disconnect) does not unregister it.
+	 *
+	 * Stop the device first so that no URB callback can touch input2
+	 * while it is being unregistered, then drop it explicitly.
+	 */
+	hid_hw_stop(hdev);
+
+	if (data->input2)
+		input_unregister_device(data->input2);
+}
+
 static const struct hid_device_id alps_id[] = {
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY,
 		USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_U1_DUAL) },
@@ -845,6 +865,7 @@ static struct hid_driver alps_driver = {
 	.input_configured	= alps_input_configured,
 	.resume			= pm_ptr(alps_post_resume),
 	.reset_resume		= pm_ptr(alps_post_reset),
+	.remove			= alps_remove,
 };
 
 module_hid_driver(alps_driver);
diff --git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c
index 5cc27066f60262..3cc3a13b71a63d 100644
--- a/drivers/hid/hid-appletb-kbd.c
+++ b/drivers/hid/hid-appletb-kbd.c
@@ -56,6 +56,12 @@ static int appletb_tb_idle_timeout = 15;
 module_param_named(idle_timeout, appletb_tb_idle_timeout, int, 0644);
 MODULE_PARM_DESC(idle_timeout, "Idle timeout in sec");
 
+static int appletb_tb_double_press_switch_time;
+module_param_named(double_press_switch_time,
+		   appletb_tb_double_press_switch_time, int, 0644);
+MODULE_PARM_DESC(double_press_switch_time,
+		 "Fn double-press interval in ms (0 disables layer switching)");
+
 struct appletb_kbd {
 	struct hid_field *mode_field;
 	struct input_handler inp_handler;
@@ -68,6 +74,7 @@ struct appletb_kbd {
 	bool has_turned_off;
 	u8 saved_mode;
 	u8 current_mode;
+	unsigned long last_fn_press;
 };
 
 static const struct key_entry appletb_kbd_keymap[] = {
@@ -87,29 +94,37 @@ static const struct key_entry appletb_kbd_keymap[] = {
 	{ KE_END, 0 }
 };
 
-static int appletb_kbd_set_mode(struct appletb_kbd *kbd, u8 mode)
+static int appletb_kbd_set_mode_active(struct appletb_kbd *kbd, u8 mode)
 {
 	struct hid_report *report = kbd->mode_field->report;
 	struct hid_device *hdev = report->device;
 	int ret;
 
-	ret = hid_hw_power(hdev, PM_HINT_FULLON);
-	if (ret) {
-		hid_err(hdev, "Device didn't resume (%pe)\n", ERR_PTR(ret));
-		return ret;
-	}
-
 	ret = hid_set_field(kbd->mode_field, 0, mode);
 	if (ret) {
 		hid_err(hdev, "Failed to set mode field to %u (%pe)\n", mode, ERR_PTR(ret));
-		goto power_normal;
+		return ret;
 	}
 
 	hid_hw_request(hdev, report, HID_REQ_SET_REPORT);
 
 	kbd->current_mode = mode;
+	return 0;
+}
+
+static int appletb_kbd_set_mode(struct appletb_kbd *kbd, u8 mode)
+{
+	struct hid_device *hdev = kbd->mode_field->report->device;
+	int ret;
+
+	ret = hid_hw_power(hdev, PM_HINT_FULLON);
+	if (ret) {
+		hid_err(hdev, "Device didn't resume (%pe)\n", ERR_PTR(ret));
+		return ret;
+	}
+
+	ret = appletb_kbd_set_mode_active(kbd, mode);
 
-power_normal:
 	hid_hw_power(hdev, PM_HINT_NORMAL);
 
 	return ret;
@@ -243,6 +258,18 @@ static int appletb_kbd_hid_event(struct hid_device *hdev, struct hid_field *fiel
 	return kbd->current_mode == APPLETB_KBD_MODE_OFF;
 }
 
+static u8 appletb_switch_mode(u8 mode)
+{
+	switch (mode) {
+	case APPLETB_KBD_MODE_SPCL:
+		return APPLETB_KBD_MODE_FN;
+	case APPLETB_KBD_MODE_FN:
+		return APPLETB_KBD_MODE_SPCL;
+	default:
+		return mode;
+	}
+}
+
 static void appletb_kbd_inp_event(struct input_handle *handle, unsigned int type,
 			      unsigned int code, int value)
 {
@@ -250,15 +277,36 @@ static void appletb_kbd_inp_event(struct input_handle *handle, unsigned int type
 
 	reset_inactivity_timer(kbd);
 
-	if (type == EV_KEY && code == KEY_FN && appletb_tb_fn_toggle &&
-		(kbd->current_mode == APPLETB_KBD_MODE_SPCL ||
-		 kbd->current_mode == APPLETB_KBD_MODE_FN)) {
+	if (type == EV_KEY && code == KEY_FN &&
+	    (kbd->current_mode == APPLETB_KBD_MODE_SPCL ||
+	     kbd->current_mode == APPLETB_KBD_MODE_FN)) {
 		if (value == 1) {
-			kbd->saved_mode = kbd->current_mode;
-			appletb_kbd_set_mode(kbd, kbd->current_mode == APPLETB_KBD_MODE_SPCL
-						? APPLETB_KBD_MODE_FN : APPLETB_KBD_MODE_SPCL);
+			if (appletb_tb_double_press_switch_time > 0) {
+				unsigned long now = jiffies;
+
+				if (time_before(now, kbd->last_fn_press +
+					msecs_to_jiffies(appletb_tb_double_press_switch_time))) {
+					appletb_tb_def_mode =
+						appletb_switch_mode(appletb_tb_def_mode);
+
+					appletb_kbd_set_mode(kbd, appletb_tb_def_mode);
+
+					kbd->saved_mode = appletb_tb_def_mode;
+					kbd->last_fn_press = 0;
+
+					return;
+				}
+
+				kbd->last_fn_press = now;
+			}
+			if (appletb_tb_fn_toggle) {
+				kbd->saved_mode = kbd->current_mode;
+
+				appletb_kbd_set_mode(kbd, appletb_switch_mode(kbd->current_mode));
+			}
 		} else if (value == 0) {
-			if (kbd->saved_mode != kbd->current_mode)
+			if (appletb_tb_fn_toggle &&
+			    kbd->saved_mode != kbd->current_mode)
 				appletb_kbd_set_mode(kbd, kbd->saved_mode);
 		}
 	}
@@ -487,18 +535,19 @@ static int appletb_kbd_suspend(struct hid_device *hdev, pm_message_t msg)
 	struct appletb_kbd *kbd = hid_get_drvdata(hdev);
 
 	kbd->saved_mode = kbd->current_mode;
-	appletb_kbd_set_mode(kbd, APPLETB_KBD_MODE_OFF);
 
-	return 0;
+	/* This callback already runs as part of a USB PM transition. Taking a
+	 * nested runtime-PM reference would wait for that transition to complete
+	 * and deadlock the USB PM state machine.
+	 */
+	return appletb_kbd_set_mode_active(kbd, APPLETB_KBD_MODE_OFF);
 }
 
 static int appletb_kbd_resume(struct hid_device *hdev)
 {
 	struct appletb_kbd *kbd = hid_get_drvdata(hdev);
 
-	appletb_kbd_set_mode(kbd, kbd->saved_mode);
-
-	return 0;
+	return appletb_kbd_set_mode_active(kbd, kbd->saved_mode);
 }
 
 static const struct hid_device_id appletb_kbd_hid_ids[] = {
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index c38a2b3b33d9e8..4112c0fc7aaf97 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -1267,6 +1267,7 @@ static int asus_input_mapping(struct hid_device *hdev,
 		case 0xa6: asus_map_key_clear(KEY_F16);		break; /* ROG Ally QAM button */
 		case 0xa7: asus_map_key_clear(KEY_F17);		break; /* ROG Ally ROG long-press */
 		case 0xa8: asus_map_key_clear(KEY_F18);		break; /* ROG Ally ROG long-press-release */
+		case 0x9c: asus_map_key_clear(KEY_F19);		break; /* Zephyrus Duo tent mode */
 
 		default:
 			/* ASUS lazily declares 256 usages, ignore the rest,
@@ -1480,6 +1481,14 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 			is_vendor = true;
 	}
 
+	/*
+	 * A vendor collection may be the only application collection on the
+	 * interface, which hidinput_connect() otherwise skips, leaving the
+	 * hotkey usages unmapped. Unpopulated inputs are dropped later.
+	 */
+	if (is_vendor && (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD))
+		hdev->quirks |= HID_QUIRK_HIDINPUT_FORCE;
+
 	ret = asus_worker_create(hdev, drvdata);
 	if (ret) {
 		hid_warn(hdev, "Failed to initialize worker: %d\n", ret);
@@ -1493,12 +1502,14 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		return ret;
 	}
 
-	for (int r = 0; r < ARRAY_SIZE(asus_report_id_init); r++) {
-		if (asus_has_report_id(hdev, asus_report_id_init[r])) {
-			ret = asus_kbd_init(hdev, asus_report_id_init[r]);
-			if (ret < 0)
-				hid_warn(hdev, "Failed to initialize 0x%x: %d.\n",
-					 asus_report_id_init[r], ret);
+	if (!drvdata->tp) {
+		for (int r = 0; r < ARRAY_SIZE(asus_report_id_init); r++) {
+			if (asus_has_report_id(hdev, asus_report_id_init[r])) {
+				ret = asus_kbd_init(hdev, asus_report_id_init[r]);
+				if (ret < 0)
+					hid_warn(hdev, "Failed to initialize 0x%x: %d.\n",
+						 asus_report_id_init[r], ret);
+			}
 		}
 	}
 
@@ -1693,6 +1704,9 @@ static const struct hid_device_id asus_devices[] = {
 	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK,
 	    USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2),
 	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD | QUIRK_HID_FN_LOCK },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
+	    USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3),
+	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
 	    USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR),
 	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD },
@@ -1731,6 +1745,9 @@ static const struct hid_device_id asus_devices[] = {
 	{ HID_DEVICE(BUS_USB, HID_GROUP_GENERIC,
 		USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO),
 	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD },
+	{ HID_DEVICE(BUS_BLUETOOTH, HID_GROUP_GENERIC,
+		USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3_BT),
+	  QUIRK_USE_KBD_BACKLIGHT | QUIRK_ROG_NKEY_KEYBOARD },
 	{ HID_DEVICE(BUS_USB, HID_GROUP_GENERIC,
 		USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T101HA_KEYBOARD) },
 	{ }
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index a3ff0514f9cdf8..ec7c2860c93efb 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -944,7 +944,7 @@ static int hid_scan_report(struct hid_device *hid)
 		hid_parser_reserved
 	};
 
-	struct hid_parser *parser __free(kvfree) = vzalloc(sizeof(*parser));
+	struct hid_parser *parser __free(vfree) = vzalloc(sizeof(*parser));
 	if (!parser)
 		return -ENOMEM;
 
@@ -1265,7 +1265,7 @@ static int hid_parse_collections(struct hid_device *device)
 		hid_parser_reserved
 	};
 
-	struct hid_parser *parser __free(kvfree) = vzalloc(sizeof(*parser));
+	struct hid_parser *parser __free(vfree) = vzalloc(sizeof(*parser));
 	if (!parser)
 		return -ENOMEM;
 
diff --git a/drivers/hid/hid-corsair-void.c b/drivers/hid/hid-corsair-void.c
index 071a663a6c2686..7be8c974d11e92 100644
--- a/drivers/hid/hid-corsair-void.c
+++ b/drivers/hid/hid-corsair-void.c
@@ -51,20 +51,23 @@
 /* Receiver report information: (ID 102)                                      */
 /* -------------------------------------------------------------------------- */
 /*
- * When queried, the recevier responds with 4 bytes to describe the firmware
- * The first 2 bytes are for the receiver, the second 2 are the headset
+ * When queried, the receiver responds with 5 bytes to describe the firmware
+ * The first byte is the ID, then 2 bytes for the receiver, then 2 for the headset
  * The headset firmware version will be 0 if no headset is connected
  *
- * 0: Recevier firmware major version
+ * 0: Report ID
+ *  102 for the firmware packet
+ *
+ * 1: Receiver firmware major version
  *  Major version of the receiver's firmware
  *
- * 1: Recevier firmware minor version
+ * 2: Receiver firmware minor version
  *  Minor version of the receiver's firmware
  *
- * 2: Headset firmware major version
+ * 3: Headset firmware major version
  *  Major version of the headset's firmware
  *
- * 3: Headset firmware minor version
+ * 4: Headset firmware minor version
  *  Minor version of the headset's firmware
  */
 /* -------------------------------------------------------------------------- */
diff --git a/drivers/hid/hid-elecom.c b/drivers/hid/hid-elecom.c
index 37d88ce57f6718..d21ead40bb076b 100644
--- a/drivers/hid/hid-elecom.c
+++ b/drivers/hid/hid-elecom.c
@@ -79,7 +79,7 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 	case USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB:
 	case USB_DEVICE_ID_ELECOM_M_XT3URBK_018F:
 	case USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC:
-	case USB_DEVICE_ID_ELECOM_M_XT4DRBK:
+	case USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD:
 		/*
 		 * Report descriptor format:
 		 * 12: button bit count
@@ -104,6 +104,7 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		mouse_button_fixup(hdev, rdesc, *rsize, 12, 30, 14, 20, 8);
 		break;
 	case USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C:
+	case USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E:
 		/*
 		 * Report descriptor format:
 		 * 22: button bit count
@@ -148,7 +149,8 @@ static const struct hid_device_id elecom_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1URBK) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1DRBK) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT2DRBK) },
diff --git a/drivers/hid/hid-google-hammer.c b/drivers/hid/hid-google-hammer.c
index c99c3c0d442e1b..847b1ddefeeca9 100644
--- a/drivers/hid/hid-google-hammer.c
+++ b/drivers/hid/hid-google-hammer.c
@@ -268,7 +268,7 @@ static void cbas_ec_remove(struct platform_device *pdev)
 
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id cbas_ec_acpi_ids[] = {
-	{ "GOOG000B", 0 },
+	{ .id = "GOOG000B" },
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, cbas_ec_acpi_ids);
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index b3aca5aa917677..c9b2c7782d9a1d 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -227,6 +227,8 @@
 #define USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD3 0x1822
 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD	0x1866
 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD2	0x19b6
+#define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3	0x1ce6
+#define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_KEYBOARD3_BT	0x1ce7
 #define USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO		0x1a30
 #define USB_DEVICE_ID_ASUSTEK_ROG_Z13_LIGHTBAR		0x18c6
 #define USB_DEVICE_ID_ASUSTEK_ROG_NKEY_ALLY		0x1abe
@@ -252,6 +254,12 @@
 #define USB_VENDOR_ID_ATMEL_V_USB	0x16c0
 #define USB_DEVICE_ID_ATMEL_V_USB	0x05df
 
+#define USB_VENDOR_ID_ATK	0x3554
+#define USB_DEVICE_ID_ATK_VXE_R1_SE_WIRED	0xF58F
+
+#define USB_VENDOR_ID_ATK_ALT		0x373B
+#define USB_DEVICE_ID_ATK_VXE_R1_SE_DONGLE	0x1085
+
 #define USB_VENDOR_ID_AUREAL		0x0755
 #define USB_DEVICE_ID_AUREAL_W01RN	0x2626
 
@@ -477,7 +485,8 @@
 #define USB_DEVICE_ID_ELECOM_M_XT3URBK_018F	0x018f
 #define USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC	0x00fc
 #define USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C	0x018c
-#define USB_DEVICE_ID_ELECOM_M_XT4DRBK	0x00fd
+#define USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD	0x00fd
+#define USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E	0x018e
 #define USB_DEVICE_ID_ELECOM_M_DT1URBK	0x00fe
 #define USB_DEVICE_ID_ELECOM_M_DT1DRBK	0x00ff
 #define USB_DEVICE_ID_ELECOM_M_DT2DRBK	0x018d
@@ -765,6 +774,7 @@
 #define I2C_DEVICE_ID_ITE_LENOVO_LEGION_Y720	0x837a
 #define USB_DEVICE_ID_ITE_LENOVO_YOGA900	0x8396
 #define I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_KEYBOARD	0x8987
+#define I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_G11_KEYBOARD	0x83db
 #define USB_DEVICE_ID_ITE8595		0x8595
 #define USB_DEVICE_ID_ITE_MEDION_E1239T	0xce50
 
@@ -1167,6 +1177,7 @@
 #define USB_DEVICE_ID_PDP_PS4_JAGUAR	0x0173
 #define USB_DEVICE_ID_PDP_PS4_RIFFMASTER	0x024a
 #define USB_DEVICE_ID_PDP_PS5_RIFFMASTER	0x0249
+#define USB_DEVICE_ID_PDP_SWITCH_AFTERGLOW	0x018c
 
 #define USB_VENDOR_ID_PLANTRONICS	0x047f
 #define USB_DEVICE_ID_PLANTRONICS_BLACKWIRE_3210_SERIES	0xc055
@@ -1219,6 +1230,11 @@
 #define USB_VENDOR_ID_PRODIGE		0x05af
 #define USB_DEVICE_ID_PRODIGE_CORDLESS	0x3062
 
+#define USB_VENDOR_ID_PULSAR	0x3554
+#define USB_DEVICE_ID_PULSAR_WIRED	0xf507
+#define USB_DEVICE_ID_PULSAR_1KHZ	0xf508
+#define USB_DEVICE_ID_PULSAR_4KHZ	0xf509
+
 #define I2C_VENDOR_ID_QTEC              0x6243
 
 #define USB_VENDOR_ID_QUANTA		0x0408
@@ -1296,6 +1312,9 @@
 #define USB_DEVICE_ID_SAMSUNG_WIRELESS_UNIVERSAL_KBD	0xa006
 #define USB_DEVICE_ID_SAMSUNG_WIRELESS_MULTI_HOGP_KBD	0xa064
 
+#define USB_VENDOR_ID_SDINNOVATION		0x36ae
+#define USB_DEVICE_ID_SDINNOVATION_GAMING_KBD	0xfeab
+
 #define USB_VENDOR_ID_SEMICO			0x1a2c
 #define USB_DEVICE_ID_SEMICO_USB_KEYKOARD	0x0023
 #define USB_DEVICE_ID_SEMICO_USB_KEYKOARD2	0x0027
@@ -1395,6 +1414,7 @@
 #define USB_VENDOR_ID_STEELSERIES	0x1038
 #define USB_DEVICE_ID_STEELSERIES_SRWS1	0x1410
 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_1_X	0x12b6
+#define USB_DEVICE_ID_STEELSERIES_ARCTIS_7_2018	0x12ad
 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_9	0x12c2
 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_5_X	0x2253
 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_7	0x2202
@@ -1504,6 +1524,9 @@
 #define USB_DEVICE_ID_ASUS_MD_5110	0x5110
 #define USB_DEVICE_ID_TURBOX_TOUCHSCREEN_MOSART	0x7100
 
+#define USB_VENDOR_ID_TURTLE_BEACH	0x10f5
+#define USB_DEVICE_ID_TB_VELOCITYONE_RACE	0x7077
+
 #define USB_VENDOR_ID_TWINHAN		0x6253
 #define USB_DEVICE_ID_TWINHAN_IR_REMOTE	0x0100
 
diff --git a/drivers/hid/hid-kysona.c b/drivers/hid/hid-kysona.c
deleted file mode 100644
index ccbd8380064ea5..00000000000000
--- a/drivers/hid/hid-kysona.c
+++ /dev/null
@@ -1,292 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *  USB HID driver for Kysona
- *  Kysona M600 mice.
- *
- *  Copyright (c) 2024 Lode Willems <me@lodewillems.com>
- */
-
-#include <linux/device.h>
-#include <linux/hid.h>
-#include <linux/usb.h>
-
-#include "hid-ids.h"
-
-#define BATTERY_TIMEOUT_MS 5000
-
-#define ONLINE_REPORT_ID 3
-#define BATTERY_REPORT_ID 4
-
-struct kysona_drvdata {
-	struct hid_device *hdev;
-	bool online;
-
-	struct power_supply_desc battery_desc;
-	struct power_supply *battery;
-	u8 battery_capacity;
-	bool battery_charging;
-	u16 battery_voltage;
-	struct delayed_work battery_work;
-};
-
-static enum power_supply_property kysona_battery_props[] = {
-	POWER_SUPPLY_PROP_STATUS,
-	POWER_SUPPLY_PROP_PRESENT,
-	POWER_SUPPLY_PROP_CAPACITY,
-	POWER_SUPPLY_PROP_SCOPE,
-	POWER_SUPPLY_PROP_MODEL_NAME,
-	POWER_SUPPLY_PROP_VOLTAGE_NOW,
-	POWER_SUPPLY_PROP_ONLINE
-};
-
-static int kysona_battery_get_property(struct power_supply *psy,
-				       enum power_supply_property psp,
-				       union power_supply_propval *val)
-{
-	struct kysona_drvdata *drv_data = power_supply_get_drvdata(psy);
-	int ret = 0;
-
-	switch (psp) {
-	case POWER_SUPPLY_PROP_PRESENT:
-		val->intval = 1;
-		break;
-	case POWER_SUPPLY_PROP_ONLINE:
-		val->intval = drv_data->online;
-		break;
-	case POWER_SUPPLY_PROP_STATUS:
-		if (drv_data->online)
-			val->intval = drv_data->battery_charging ?
-					POWER_SUPPLY_STATUS_CHARGING :
-					POWER_SUPPLY_STATUS_DISCHARGING;
-		else
-			val->intval = POWER_SUPPLY_STATUS_UNKNOWN;
-		break;
-	case POWER_SUPPLY_PROP_SCOPE:
-		val->intval = POWER_SUPPLY_SCOPE_DEVICE;
-		break;
-	case POWER_SUPPLY_PROP_CAPACITY:
-		val->intval = drv_data->battery_capacity;
-		break;
-	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
-		/* hardware reports voltage in mV. sysfs expects uV */
-		val->intval = drv_data->battery_voltage * 1000;
-		break;
-	case POWER_SUPPLY_PROP_MODEL_NAME:
-		val->strval = drv_data->hdev->name;
-		break;
-	default:
-		ret = -EINVAL;
-		break;
-	}
-	return ret;
-}
-
-static const char kysona_online_request[] = {
-	0x08, ONLINE_REPORT_ID, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a
-};
-
-static const char kysona_battery_request[] = {
-	0x08, BATTERY_REPORT_ID, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x49
-};
-
-static int kysona_m600_fetch_online(struct hid_device *hdev)
-{
-	u8 *write_buf;
-	int ret;
-
-	/* Request online information */
-	write_buf = kmemdup(kysona_online_request, sizeof(kysona_online_request), GFP_KERNEL);
-	if (!write_buf)
-		return -ENOMEM;
-
-	ret = hid_hw_raw_request(hdev, kysona_online_request[0],
-				 write_buf, sizeof(kysona_online_request),
-				 HID_OUTPUT_REPORT, HID_REQ_SET_REPORT);
-	if (ret < (int)sizeof(kysona_online_request)) {
-		hid_err(hdev, "hid_hw_raw_request() failed with %d\n", ret);
-		ret = -ENODATA;
-	}
-	kfree(write_buf);
-	return ret;
-}
-
-static void kysona_fetch_online(struct hid_device *hdev)
-{
-	int ret = kysona_m600_fetch_online(hdev);
-
-	if (ret < 0)
-		hid_dbg(hdev,
-			"Online query failed (err: %d)\n", ret);
-}
-
-static int kysona_m600_fetch_battery(struct hid_device *hdev)
-{
-	u8 *write_buf;
-	int ret;
-
-	/* Request battery information */
-	write_buf = kmemdup(kysona_battery_request, sizeof(kysona_battery_request), GFP_KERNEL);
-	if (!write_buf)
-		return -ENOMEM;
-
-	ret = hid_hw_raw_request(hdev, kysona_battery_request[0],
-				 write_buf, sizeof(kysona_battery_request),
-				 HID_OUTPUT_REPORT, HID_REQ_SET_REPORT);
-	if (ret < (int)sizeof(kysona_battery_request)) {
-		hid_err(hdev, "hid_hw_raw_request() failed with %d\n", ret);
-		ret = -ENODATA;
-	}
-	kfree(write_buf);
-	return ret;
-}
-
-static void kysona_fetch_battery(struct hid_device *hdev)
-{
-	int ret = kysona_m600_fetch_battery(hdev);
-
-	if (ret < 0)
-		hid_dbg(hdev,
-			"Battery query failed (err: %d)\n", ret);
-}
-
-static void kysona_battery_timer_tick(struct work_struct *work)
-{
-	struct kysona_drvdata *drv_data = container_of(work,
-		struct kysona_drvdata, battery_work.work);
-	struct hid_device *hdev = drv_data->hdev;
-
-	kysona_fetch_online(hdev);
-	kysona_fetch_battery(hdev);
-	schedule_delayed_work(&drv_data->battery_work,
-			      msecs_to_jiffies(BATTERY_TIMEOUT_MS));
-}
-
-static int kysona_battery_probe(struct hid_device *hdev)
-{
-	struct kysona_drvdata *drv_data = hid_get_drvdata(hdev);
-	struct power_supply_config pscfg = { .drv_data = drv_data };
-	int ret = 0;
-
-	drv_data->online = false;
-	drv_data->battery_capacity = 100;
-	drv_data->battery_voltage = 4200;
-
-	drv_data->battery_desc.properties = kysona_battery_props;
-	drv_data->battery_desc.num_properties = ARRAY_SIZE(kysona_battery_props);
-	drv_data->battery_desc.get_property = kysona_battery_get_property;
-	drv_data->battery_desc.type = POWER_SUPPLY_TYPE_BATTERY;
-	drv_data->battery_desc.use_for_apm = 0;
-	drv_data->battery_desc.name = devm_kasprintf(&hdev->dev, GFP_KERNEL,
-						     "kysona-%s-battery",
-						     strlen(hdev->uniq) ?
-						     hdev->uniq : dev_name(&hdev->dev));
-	if (!drv_data->battery_desc.name)
-		return -ENOMEM;
-
-	drv_data->battery = devm_power_supply_register(&hdev->dev,
-						       &drv_data->battery_desc, &pscfg);
-	if (IS_ERR(drv_data->battery)) {
-		ret = PTR_ERR(drv_data->battery);
-		drv_data->battery = NULL;
-		hid_err(hdev, "Unable to register battery device\n");
-		return ret;
-	}
-
-	power_supply_powers(drv_data->battery, &hdev->dev);
-
-	INIT_DELAYED_WORK(&drv_data->battery_work, kysona_battery_timer_tick);
-	kysona_fetch_online(hdev);
-	kysona_fetch_battery(hdev);
-	schedule_delayed_work(&drv_data->battery_work,
-			      msecs_to_jiffies(BATTERY_TIMEOUT_MS));
-
-	return ret;
-}
-
-static int kysona_probe(struct hid_device *hdev, const struct hid_device_id *id)
-{
-	int ret;
-	struct kysona_drvdata *drv_data;
-	struct usb_interface *usbif;
-
-	if (!hid_is_usb(hdev))
-		return -EINVAL;
-
-	usbif = to_usb_interface(hdev->dev.parent);
-
-	drv_data = devm_kzalloc(&hdev->dev, sizeof(*drv_data), GFP_KERNEL);
-	if (!drv_data)
-		return -ENOMEM;
-
-	hid_set_drvdata(hdev, drv_data);
-	drv_data->hdev = hdev;
-
-	ret = hid_parse(hdev);
-	if (ret)
-		return ret;
-
-	ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT);
-	if (ret)
-		return ret;
-
-	if (usbif->cur_altsetting->desc.bInterfaceNumber == 1) {
-		if (kysona_battery_probe(hdev) < 0)
-			hid_err(hdev, "Kysona hid battery_probe failed: %d\n", ret);
-	}
-
-	return 0;
-}
-
-static int kysona_raw_event(struct hid_device *hdev,
-			    struct hid_report *report, u8 *data, int size)
-{
-	struct kysona_drvdata *drv_data = hid_get_drvdata(hdev);
-
-	if (size == sizeof(kysona_online_request) &&
-	    data[0] == 8 && data[1] == ONLINE_REPORT_ID) {
-		drv_data->online = data[6];
-	}
-
-	if (size == sizeof(kysona_battery_request) &&
-	    data[0] == 8 && data[1] == BATTERY_REPORT_ID) {
-		drv_data->battery_capacity = data[6];
-		drv_data->battery_charging = data[7];
-		drv_data->battery_voltage = (data[8] << 8) | data[9];
-	}
-
-	return 0;
-}
-
-static void kysona_remove(struct hid_device *hdev)
-{
-	struct kysona_drvdata *drv_data = hid_get_drvdata(hdev);
-
-	if (drv_data->battery)
-		cancel_delayed_work_sync(&drv_data->battery_work);
-
-	hid_hw_stop(hdev);
-}
-
-static const struct hid_device_id kysona_devices[] = {
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYSONA, USB_DEVICE_ID_KYSONA_M600_DONGLE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYSONA, USB_DEVICE_ID_KYSONA_M600_WIRED) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_VXE, USB_DEVICE_ID_VXE_DRAGONFLY_R1_PRO_DONGLE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_VXE, USB_DEVICE_ID_VXE_DRAGONFLY_R1_PRO_WIRED) },
-	{ }
-};
-MODULE_DEVICE_TABLE(hid, kysona_devices);
-
-static struct hid_driver kysona_driver = {
-	.name			= "kysona",
-	.id_table		= kysona_devices,
-	.probe			= kysona_probe,
-	.raw_event		= kysona_raw_event,
-	.remove			= kysona_remove
-};
-module_hid_driver(kysona_driver);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("HID driver for Kysona devices");
-MODULE_AUTHOR("Lode Willems <me@lodewillems.com>");
diff --git a/drivers/hid/hid-lenovo.c b/drivers/hid/hid-lenovo.c
index 3976d7b53b14b8..617bba6626592e 100644
--- a/drivers/hid/hid-lenovo.c
+++ b/drivers/hid/hid-lenovo.c
@@ -476,6 +476,16 @@ static int lenovo_input_mapping_x1_tab_kbd(struct hid_device *hdev,
 	return 0;
 }
 
+static int lenovo_input_mapping_yoga7x_g11_kbd(struct hid_device *hdev,
+		struct hid_input *hi, struct hid_field *field,
+		struct hid_usage *usage, unsigned long **bit, int *max)
+{
+	if (field->application == HID_GD_WIRELESS_RADIO_CTLS)
+		return -1;
+
+	return 0;
+}
+
 static int lenovo_input_mapping(struct hid_device *hdev,
 		struct hid_input *hi, struct hid_field *field,
 		struct hid_usage *usage, unsigned long **bit, int *max)
@@ -510,6 +520,8 @@ static int lenovo_input_mapping(struct hid_device *hdev,
 	case USB_DEVICE_ID_LENOVO_X1_TAB2:
 	case USB_DEVICE_ID_LENOVO_X1_TAB3:
 		return lenovo_input_mapping_x1_tab_kbd(hdev, hi, field, usage, bit, max);
+	case I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_G11_KEYBOARD:
+		return lenovo_input_mapping_yoga7x_g11_kbd(hdev, hi, field, usage, bit, max);
 	default:
 		return 0;
 	}
@@ -1568,6 +1580,8 @@ static const struct hid_device_id lenovo_devices[] = {
 		     USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X13_TAB) },
 	{ HID_DEVICE(BUS_I2C, HID_GROUP_GENERIC,
 		     USB_VENDOR_ID_ITE, I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_KEYBOARD) },
+	{ HID_DEVICE(BUS_I2C, HID_GROUP_GENERIC,
+		     USB_VENDOR_ID_ITE, I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_G11_KEYBOARD) },
 	{ }
 };
 
diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c
index 1d619d2345e12f..390d106d5dc131 100644
--- a/drivers/hid/hid-logitech-dj.c
+++ b/drivers/hid/hid-logitech-dj.c
@@ -121,7 +121,6 @@ enum recvr_type {
 	recvr_type_27mhz,
 	recvr_type_bluetooth,
 	recvr_type_dinovo,
-	recvr_type_bolt,
 };
 
 struct dj_report {
@@ -1157,10 +1156,6 @@ static void logi_hidpp_recv_queue_notif(struct hid_device *hdev,
 		logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
 		workitem.reports_supported |= STD_KEYBOARD;
 		break;
-	case 0x10:
-		device_type = "Bolt";
-		logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem);
-		break;
 	}
 
 	/* custom receiver device (eg. powerplay) */
@@ -1750,24 +1745,6 @@ static int logi_dj_hidpp_event(struct hid_device *hdev,
 
 	dj_dev = djrcv_dev->paired_dj_devices[device_index];
 
-	/*
-	 * Bolt receivers send explicit unpair notifications as HID++ events;
-	 * queue device removal when we receive one.
-	 */
-	if (djrcv_dev->type == recvr_type_bolt &&
-	    hidpp_report->report_id == REPORT_ID_HIDPP_SHORT &&
-	    hidpp_report->sub_id == REPORT_TYPE_NOTIF_DEVICE_UNPAIRED) {
-		struct dj_workitem workitem = {
-			.device_index = device_index,
-			.type = WORKITEM_TYPE_UNPAIRED,
-		};
-
-		kfifo_in(&djrcv_dev->notif_fifo, &workitem, sizeof(workitem));
-		schedule_work(&djrcv_dev->work);
-		spin_unlock_irqrestore(&djrcv_dev->lock, flags);
-		return false;
-	}
-
 	/*
 	 * With 27 MHz receivers, we do not get an explicit unpair event,
 	 * remove the old device if the user has paired a *different* device.
@@ -1907,9 +1884,6 @@ static int logi_dj_probe(struct hid_device *hdev,
 	 * treat these as logitech-dj interfaces then this causes input events
 	 * reported through this extra interface to not be reported correctly.
 	 * To avoid this, we treat these as generic-hid devices.
-	 *
-	 * Bolt receivers only use LOGITECH_DJ_INTERFACE_NUMBER for receiver
-	 * reporting. Treat all other Bolt interfaces as generic-hid devices.
 	 */
 	switch (id->driver_data) {
 	case recvr_type_dj:		no_dj_interfaces = 3; break;
@@ -1923,20 +1897,10 @@ static int logi_dj_probe(struct hid_device *hdev,
 	}
 	if (hid_is_usb(hdev)) {
 		intf = to_usb_interface(hdev->dev.parent);
-		if (intf) {
-			bool generic_hid_interface;
-
-			if (id->driver_data == recvr_type_bolt)
-				generic_hid_interface =
-					intf->altsetting->desc.bInterfaceNumber !=
-					LOGITECH_DJ_INTERFACE_NUMBER;
-			else
-				generic_hid_interface =
-					intf->altsetting->desc.bInterfaceNumber >= no_dj_interfaces;
-			if (generic_hid_interface) {
-				hdev->quirks |= HID_QUIRK_INPUT_PER_APP;
-				return hid_hw_start(hdev, HID_CONNECT_DEFAULT);
-			}
+		if (intf && intf->altsetting->desc.bInterfaceNumber >=
+							no_dj_interfaces) {
+			hdev->quirks |= HID_QUIRK_INPUT_PER_APP;
+			return hid_hw_start(hdev, HID_CONNECT_DEFAULT);
 		}
 	}
 
@@ -2139,10 +2103,6 @@ static const struct hid_device_id logi_dj_receivers[] = {
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
 		USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_3),
 	 .driver_data = recvr_type_gaming_hidpp_ls_1_3},
-	{ /* Logitech Bolt receiver (0xc548) */
-	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
-			 USB_DEVICE_ID_LOGITECH_BOLT_RECEIVER),
-	 .driver_data = recvr_type_bolt},
 	{ /* Logitech lightspeed receiver (0xc54d) */
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
 		USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_4),
diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c
index 1504de32b1c84b..9646380febacaf 100644
--- a/drivers/hid/hid-logitech-hidpp.c
+++ b/drivers/hid/hid-logitech-hidpp.c
@@ -988,8 +988,7 @@ static int hidpp_root_get_protocol_version(struct hidpp_device *hidpp)
 	}
 
 	/* the device might not be connected */
-	if (ret == HIDPP_ERROR_CONNECT_FAIL ||
-	    ret == HIDPP_ERROR_RESOURCE_ERROR ||
+	if (ret == HIDPP_ERROR_RESOURCE_ERROR ||
 	    ret == HIDPP_ERROR_UNKNOWN_DEVICE)
 		return -EIO;
 
@@ -4405,50 +4404,8 @@ static int hidpp_initialize_battery(struct hidpp_device *hidpp)
 	return ret;
 }
 
-static bool hidpp_is_bolt_child(struct hid_device *hdev)
-{
-	struct device *parent = hdev->dev.parent;
-	struct hid_device *receiver_hdev;
-
-	if (!parent)
-		return false;
-
-	receiver_hdev = to_hid_device(parent);
-	return receiver_hdev->vendor == USB_VENDOR_ID_LOGITECH &&
-	       receiver_hdev->product == USB_DEVICE_ID_LOGITECH_BOLT_RECEIVER;
-}
-
-static int hidpp_bolt_init(struct hidpp_device *hidpp)
-{
-	struct hid_device *hdev = hidpp->hid_dev;
-	char *name;
-	int ret;
-
-	ret = hidpp_serial_init(hidpp);
-	if (ret)
-		return ret;
-
-	name = hidpp_get_device_name(hidpp);
-	if (!name)
-		return -EIO;
-
-	snprintf(hdev->name, sizeof(hdev->name), "%s", name);
-	dbg_hid("HID++ Bolt: Got name: %s\n", name);
-
-	kfree(name);
-	return 0;
-}
-
-static int hidpp_receiver_init(struct hidpp_device *hidpp)
-{
-	if (hidpp_is_bolt_child(hidpp->hid_dev))
-		return hidpp_bolt_init(hidpp);
-
-	return hidpp_unifying_init(hidpp);
-}
-
 /* Get name + serial for USB and Bluetooth HID++ devices */
-static void hidpp_non_receiver_init(struct hidpp_device *hidpp)
+static void hidpp_non_unifying_init(struct hidpp_device *hidpp)
 {
 	struct hid_device *hdev = hidpp->hid_dev;
 	char *name;
@@ -4800,9 +4757,9 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 	/* Get name + serial, store in hdev->name + hdev->uniq */
 	if (id->group == HID_GROUP_LOGITECH_DJ_DEVICE)
-		hidpp_receiver_init(hidpp);
+		hidpp_unifying_init(hidpp);
 	else
-		hidpp_non_receiver_init(hidpp);
+		hidpp_non_unifying_init(hidpp);
 
 	if (hidpp->quirks & HIDPP_QUIRK_DELAYED_INIT)
 		connect_mask &= ~HID_CONNECT_HIDINPUT;
@@ -4924,6 +4881,8 @@ static const struct hid_device_id hidpp_devices[] = {
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC08D) },
 	{ /* Logitech G502 X Plus Wireless Gaming Mouse over USB */
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC095) },
+	{ /* Logitech G502 X Lightspeed Wireless Gaming Mouse over USB */
+	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC098) },
 	{ /* Logitech G703 Gaming Mouse over USB */
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC087) },
 	{ /* Logitech G703 Hero Gaming Mouse over USB */
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index 451c7324e6a061..4e19a0c4d2381d 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -79,6 +79,7 @@ MODULE_LICENSE("GPL");
 #define MT_QUIRK_APPLE_TOUCHBAR		BIT(23)
 #define MT_QUIRK_YOGABOOK9I		BIT(24)
 #define MT_QUIRK_KEEP_LATENCY_ON_CLOSE	BIT(25)
+#define MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH	BIT(26)
 
 #define MT_INPUTMODE_TOUCHSCREEN	0x02
 #define MT_INPUTMODE_TOUCHPAD		0x03
@@ -235,6 +236,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app);
 #define MT_CLS_APPLE_TOUCHBAR			0x0114
 #define MT_CLS_YOGABOOK9I			0x0115
 #define MT_CLS_EGALAX_P80H84			0x0116
+#define MT_CLS_ASUS_ROG_Z13_FOLIO		0x0117
 #define MT_CLS_SIS				0x0457
 
 #define MT_DEFAULT_MAXCONTACT	10
@@ -405,6 +407,16 @@ static const struct mt_class mt_classes[] = {
 		.quirks = MT_QUIRK_ALWAYS_VALID |
 			MT_QUIRK_CONTACT_CNT_ACCURATE |
 			MT_QUIRK_ASUS_CUSTOM_UP },
+	{ .name = MT_CLS_ASUS_ROG_Z13_FOLIO,
+		.quirks = MT_QUIRK_ALWAYS_VALID |
+			MT_QUIRK_IGNORE_DUPLICATES |
+			MT_QUIRK_HOVERING |
+			MT_QUIRK_CONTACT_CNT_ACCURATE |
+			MT_QUIRK_STICKY_FINGERS |
+			MT_QUIRK_WIN8_PTP_BUTTONS |
+			MT_QUIRK_CONFIDENCE |
+			MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH,
+		.export_all_inputs = true },
 	{ .name = MT_CLS_VTL,
 		.quirks = MT_QUIRK_ALWAYS_VALID |
 			MT_QUIRK_CONTACT_CNT_ACCURATE |
@@ -507,6 +519,7 @@ static const struct attribute_group mt_attribute_group = {
 
 static void mt_get_feature(struct hid_device *hdev, struct hid_report *report)
 {
+	struct mt_device *td = hid_get_drvdata(hdev);
 	int ret;
 	u32 size = hid_report_len(report);
 	u8 *buf;
@@ -528,8 +541,14 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report)
 		dev_warn(&hdev->dev, "failed to fetch feature %d\n",
 			 report->id);
 	} else {
-		/* The report ID in the request and the response should match */
-		if (report->id != buf[0]) {
+		/*
+		 * The report ID in the request and the response should match.
+		 * Some firmware (e.g. the ASUS ROG Z13 Folio
+		 * touchpad) returns a mismatched ID on this specific fetch;
+		 * tolerate it only for devices explicitly flagged as such.
+		 */
+		if (report->id != buf[0] &&
+		    !(td->mtclass.quirks & MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH)) {
 			hid_err(hdev, "Returned feature report did not match the request\n");
 			goto free;
 		}
@@ -2230,7 +2249,7 @@ static void mt_remove(struct hid_device *hdev)
 {
 	struct mt_device *td = hid_get_drvdata(hdev);
 
-	timer_delete_sync(&td->release_timer);
+	timer_shutdown_sync(&td->release_timer);
 
 	sysfs_remove_group(&hdev->dev.kobj, &mt_attribute_group);
 	hid_hw_stop(hdev);
@@ -2715,6 +2734,12 @@ static const struct hid_device_id mt_devices[] = {
 		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
 			   I2C_VENDOR_ID_HANTICK, I2C_PRODUCT_ID_HANTICK_5288) },
 
+	/* Asus ROG Flow Z13 (2025) GZ302EA keyboard-cover touchpad */
+	{ .driver_data = MT_CLS_ASUS_ROG_Z13_FOLIO,
+		HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8,
+			USB_VENDOR_ID_ASUSTEK,
+			USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO) },
+
 	/* Generic MT device */
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH, HID_ANY_ID, HID_ANY_ID) },
 
diff --git a/drivers/hid/hid-nintendo.c b/drivers/hid/hid-nintendo.c
index 43e0f2aaea3b5a..b1fcf3e5767ba9 100644
--- a/drivers/hid/hid-nintendo.c
+++ b/drivers/hid/hid-nintendo.c
@@ -434,8 +434,8 @@ static const struct joycon_ctlr_button_mapping procon_button_mappings[] = {
 	{ /* sentinel */ },
 };
 
-/* Licensed Pro Controllers (e.g. HORI) swap X/Y bits in the report */
-static const struct joycon_ctlr_button_mapping lic_procon_button_mappings[] = {
+/* HORI Licensed Pro Controllers swap X/Y bits in the report */
+static const struct joycon_ctlr_button_mapping lic_hori_procon_button_mappings[] = {
 	{ BTN_EAST,	JC_BTN_A,	},
 	{ BTN_SOUTH,	JC_BTN_B,	},
 	{ BTN_NORTH,	JC_BTN_Y,	},
@@ -1767,8 +1767,8 @@ static void joycon_parse_report(struct joycon_ctlr *ctlr,
 		joycon_report_left_stick(ctlr, rep);
 		joycon_report_right_stick(ctlr, rep);
 		joycon_report_dpad(ctlr, rep);
-		if (ctlr->ctlr_type == JOYCON_CTLR_TYPE_LIC_PRO)
-			joycon_report_buttons(ctlr, rep, lic_procon_button_mappings);
+		if (ctlr->hdev->vendor == USB_VENDOR_ID_HORI)
+			joycon_report_buttons(ctlr, rep, lic_hori_procon_button_mappings);
 		else
 			joycon_report_buttons(ctlr, rep, procon_button_mappings);
 	} else if (joycon_type_is_any_nescon(ctlr)) {
@@ -2212,8 +2212,8 @@ static int joycon_input_create(struct joycon_ctlr *ctlr)
 		joycon_config_left_stick(ctlr->input);
 		joycon_config_right_stick(ctlr->input);
 		joycon_config_dpad(ctlr->input);
-		if (ctlr->ctlr_type == JOYCON_CTLR_TYPE_LIC_PRO)
-			joycon_config_buttons(ctlr->input, lic_procon_button_mappings);
+		if (ctlr->hdev->vendor == USB_VENDOR_ID_HORI)
+			joycon_config_buttons(ctlr->input, lic_hori_procon_button_mappings);
 		else
 			joycon_config_buttons(ctlr->input, procon_button_mappings);
 	} else if (joycon_type_is_any_nescon(ctlr)) {
@@ -2905,6 +2905,8 @@ static const struct hid_device_id nintendo_hid_devices[] = {
 			 USB_DEVICE_ID_NINTENDO_N64CON) },
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_HORI,
 			 USB_DEVICE_ID_HORI_WIRELESS_SWITCH_PAD) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_PDP,
+			 USB_DEVICE_ID_PDP_SWITCH_AFTERGLOW) },
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, nintendo_hid_devices);
diff --git a/drivers/hid/hid-oxp.c b/drivers/hid/hid-oxp.c
index d2ded6b08ce9e2..1e691ebc1199e6 100644
--- a/drivers/hid/hid-oxp.c
+++ b/drivers/hid/hid-oxp.c
@@ -1552,9 +1552,9 @@ static int oxp_hid_probe(struct hid_device *hdev,
 
 static void oxp_hid_remove(struct hid_device *hdev)
 {
-	cancel_delayed_work(&drvdata.oxp_rgb_queue);
-	cancel_delayed_work(&drvdata.oxp_btn_queue);
-	cancel_delayed_work(&drvdata.oxp_mcu_init);
+	cancel_delayed_work_sync(&drvdata.oxp_rgb_queue);
+	cancel_delayed_work_sync(&drvdata.oxp_btn_queue);
+	cancel_delayed_work_sync(&drvdata.oxp_mcu_init);
 	hid_hw_close(hdev);
 	hid_hw_stop(hdev);
 }
diff --git a/drivers/hid/hid-pulsar.c b/drivers/hid/hid-pulsar.c
new file mode 100644
index 00000000000000..e6e3db5e916daf
--- /dev/null
+++ b/drivers/hid/hid-pulsar.c
@@ -0,0 +1,765 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * HID driver for pulsar mice
+ *
+ * Supported pulsar devices:
+ *	- Pulsar
+ *		- X2 V2
+ *		- X2H
+ *		- X2A
+ *		- Xlite V3
+ *	- Kysona
+ *		-M600
+ *	- ATK
+ *		- VXE R1 SE+
+ *	- VXE
+ *		- Dragonfly R1 Pro
+ *
+ * Copyright (c) 2026 Nikolas Koesling
+ */
+
+#include <linux/hid.h>
+#include <linux/usb.h>
+#include <linux/power_supply.h>
+#include "hid-ids.h"
+
+/* ----- driver settings ----- */
+#define CMD_TIMEOUT_MSEC 100
+#define MAX_BATTERY_AGE_NS 60000000000ULL	/* 60s */
+#define MAX_UNAVAIL_AGE_NS 5000000000ULL	/* 5s */
+#define INIT_RETRIES 1
+#define INIT_DELAY_MSEC 1000
+
+/* ----- constants ----- */
+#define USB_INTERFACE 1
+#define USB_PAYLOAD_LEN 17
+#define CMD_HID_REPORT_ID 0x08
+#define CHECKSUM_MAGIC 0x55
+#define DEV_INFO_LEN 4
+#define CON_1K 0x00
+#define CON_4K 0x01
+#define CON_WIRED 0x02
+
+/* ----- device commands ----- */
+enum pulsar_cmd {
+	CMD_NONE = 0,
+	CMD_INFO = 0x01,
+	CMD_STATUS = 0x03,
+	CMD_POWER = 0x04,
+	CMD_EVENT = 0x0a,	/* recv only */
+};
+
+#define EVENT_PWR 0x40		/* power status change */
+#define EVENT_PWR_CHK 0xf9
+
+/* ----- device types ----- */
+enum dev_type {
+	TYPE_UNKNOWN,
+	TYPE_PULSAR,
+	TYPE_KYSONA,
+	TYPE_ATK,
+	TYPE_VXE,
+};
+
+/* ----- structs ----- */
+struct pulsar_battery {
+	struct power_supply *ps;
+	struct power_supply_desc desc;
+	char name[48];
+	char model[MAX(32, sizeof((struct hid_device){}).name)];
+	u8 level;		/* percent */
+	u16 voltage;		/* millivolts */
+	bool conn;
+	bool available;
+	bool refresh;		/* cache invalid, read regardless of age */
+	u64 last_read;
+	u64 last_status;
+};
+
+struct pulsar_data {
+	struct hid_device *hdev;
+
+	enum dev_type type;
+
+	spinlock_t raw_event_lock;	/* protects response_buf, pending_event */
+	struct mutex lock_cmd;		/* serializes device command execution */
+	struct rw_semaphore lock_bat;	/* protects battery state */
+
+	struct completion response_ready;
+	u8 response_buf[USB_PAYLOAD_LEN];
+	u8 pending_event;
+	struct work_struct power_uevent_work;
+	struct delayed_work init_work;
+	unsigned int init_retries;
+	atomic_t device_verified;
+	atomic_t stopping;
+
+	struct pulsar_battery battery;
+};
+
+static u8 calc_checksum(const u8 *data, size_t len)
+{
+	u8 sum = 0;
+
+	for (size_t i = 0; i < len - 1; i++)
+		sum += data[i];
+
+	return (u8)CHECKSUM_MAGIC - sum;
+}
+
+static int send_cmd(struct hid_device *hdev, const u8 *buf, size_t len)
+{
+	int ret;
+	u8 *dmabuf;
+
+	hid_dbg(hdev, "send command: %*ph\n", (int)len, buf);
+
+	dmabuf = kmemdup(buf, len, GFP_KERNEL);
+	if (!dmabuf)
+		return -ENOMEM;
+
+	/* device listens only to control transfers */
+	ret = hid_hw_raw_request(hdev, dmabuf[0], dmabuf, len,
+				 HID_OUTPUT_REPORT, HID_REQ_SET_REPORT);
+
+	kfree(dmabuf);
+
+	if (ret < 0)
+		return ret;
+	if (ret != len)
+		return -EIO;
+
+	return 0;
+}
+
+static int exec_cmd(struct pulsar_data *drvdata, const u8 *payload,
+		    u8 *response, unsigned int timeout_msec)
+{
+	struct hid_device *hdev = drvdata->hdev;
+	unsigned long flags;
+	int ret;
+	unsigned long timeout;
+	u8 checksum;
+
+	if (atomic_read(&drvdata->stopping))
+		return -ENODEV;
+
+	mutex_lock(&drvdata->lock_cmd);
+
+	if (atomic_read(&drvdata->stopping)) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	spin_lock_irqsave(&drvdata->raw_event_lock, flags);
+	reinit_completion(&drvdata->response_ready);
+	drvdata->pending_event = payload[1];
+	spin_unlock_irqrestore(&drvdata->raw_event_lock, flags);
+
+	ret = send_cmd(hdev, payload, USB_PAYLOAD_LEN);
+
+	if (ret < 0) {
+		spin_lock_irqsave(&drvdata->raw_event_lock, flags);
+		drvdata->pending_event = CMD_NONE;
+		spin_unlock_irqrestore(&drvdata->raw_event_lock, flags);
+		hid_err(hdev, "failed to send command 0x%02x: %d\n",
+			payload[1], ret);
+		goto out;
+	}
+
+	timeout = wait_for_completion_timeout(&drvdata->response_ready,
+					      msecs_to_jiffies(timeout_msec));
+
+	if (timeout == 0) {
+		spin_lock_irqsave(&drvdata->raw_event_lock, flags);
+		drvdata->pending_event = CMD_NONE;
+		spin_unlock_irqrestore(&drvdata->raw_event_lock, flags);
+		ret = -ETIMEDOUT;
+		goto out;
+	}
+
+	spin_lock_irqsave(&drvdata->raw_event_lock, flags);
+	memcpy(response, drvdata->response_buf, USB_PAYLOAD_LEN);
+	spin_unlock_irqrestore(&drvdata->raw_event_lock, flags);
+
+	/* validate checksum */
+	checksum = calc_checksum(response, USB_PAYLOAD_LEN);
+
+	if (response[USB_PAYLOAD_LEN - 1] != checksum) {
+		hid_err(hdev,
+			"invalid checksum in response: 0x%02x (expected 0x%02x)\n",
+			response[USB_PAYLOAD_LEN - 1], checksum);
+		ret = -EIO;
+		goto out;
+	}
+
+	ret = 0;
+out:
+	mutex_unlock(&drvdata->lock_cmd);
+	return ret;
+}
+
+static inline void finalize_payload(u8 *payload, u8 cmd)
+{
+	payload[0] = CMD_HID_REPORT_ID;
+	payload[1] = cmd;
+	payload[USB_PAYLOAD_LEN - 1] = calc_checksum(payload, USB_PAYLOAD_LEN);
+}
+
+static int read_status(struct pulsar_data *drvdata)
+{
+	int ret;
+	u8 payload[USB_PAYLOAD_LEN] = { 0 };
+	u8 response[USB_PAYLOAD_LEN];
+
+	finalize_payload(payload, CMD_STATUS);
+
+	ret = exec_cmd(drvdata, payload, response, CMD_TIMEOUT_MSEC);
+	if (ret < 0)
+		return ret;
+	if (response[6] > 0x01)
+		return -EIO;
+
+	return (int)response[6];	/* 1: available, 0: not available */
+}
+
+static int read_device_info(struct pulsar_data *drvdata, u8 *data)
+{
+	int ret;
+	u8 payload[USB_PAYLOAD_LEN] = { 0 };
+	u8 response[USB_PAYLOAD_LEN];
+
+	payload[5] = DEV_INFO_LEN * 2;
+	get_random_bytes(payload + 6, DEV_INFO_LEN);
+	finalize_payload(payload, CMD_INFO);
+
+	ret = exec_cmd(drvdata, payload, response, CMD_TIMEOUT_MSEC);
+	if (ret < 0)
+		return ret;
+
+	if (data)
+		memcpy(data, response + 6 + DEV_INFO_LEN, DEV_INFO_LEN);
+
+	response[8 + DEV_INFO_LEN] = 0;
+	response[9 + DEV_INFO_LEN] = 0;
+
+	/*
+	 * Verify challenge-response. Response layout from offset 6:
+	 *   [0..3] encoded response   [4..7] device info (ID + conn type)
+	 *
+	 * resp[i] = challenge[i] * (i+1) + challenge[(i+1) % 4] + device_id[i]
+	 *
+	 * bytes 6..7 are zeroed for verification.
+	 */
+	for (int i = 0; i < DEV_INFO_LEN; i++) {
+		u8 expect = response[6 + DEV_INFO_LEN + i];
+		u8 actual = response[6 + i] - (i + 1) * payload[6 + i] -
+		    payload[6 + (i + 1) % DEV_INFO_LEN];
+
+		if (expect != actual) {
+			hid_warn(drvdata->hdev,
+				 "device info[%d] mismatch: %02x != %02x\n",
+				 i, expect, actual);
+			return -EIO;
+		}
+	}
+
+	return 0;
+}
+
+static int read_power(struct pulsar_data *drvdata)
+{
+	u64 now;
+	bool need_status, need_power;
+	int ret = 0;
+	u8 payload[USB_PAYLOAD_LEN] = { 0 };
+	u8 response[USB_PAYLOAD_LEN];
+	struct pulsar_battery *battery = &drvdata->battery;
+
+	now = ktime_get_ns();
+
+	down_write(&drvdata->lock_bat);
+
+	need_status = battery->refresh ||
+	    (now - battery->last_status >= MAX_UNAVAIL_AGE_NS);
+	need_power = battery->available &&
+	    (battery->refresh ||
+	     now - battery->last_read >= MAX_BATTERY_AGE_NS);
+
+	if (!need_status && !need_power)
+		goto unlock;
+
+	if (need_status) {
+		const int status = read_status(drvdata);
+
+		if (status < 0) {
+			ret = status;
+			hid_err(drvdata->hdev,
+				"%s: failed to read status: %d\n",
+				__func__, ret);
+			goto unlock;
+		}
+
+		battery->last_status = now;
+
+		if (!status) {
+			battery->available = false;
+			battery->refresh = false;
+			goto unlock;
+		}
+
+		/* device just became available, force power read */
+		if (!battery->available)
+			need_power = true;
+	}
+
+	if (!need_power)
+		goto unlock;
+
+	finalize_payload(payload, CMD_POWER);
+
+	ret = exec_cmd(drvdata, payload, response, CMD_TIMEOUT_MSEC);
+	if (ret < 0) {
+		hid_err(drvdata->hdev, "%s: failed to read power: %d\n",
+			__func__, ret);
+		goto unlock;
+	}
+
+	if (response[6] > 100 || response[7] > 0x01) {
+		ret = -EIO;
+		goto unlock;
+	}
+
+	battery->available = true;
+	battery->level = response[6];
+	battery->conn = response[7] == 1;
+	battery->voltage = (response[8] << 8) | response[9];
+	battery->last_read = now;
+	battery->refresh = false;
+
+	hid_dbg(drvdata->hdev, "%s: level=%d, conn=%d, voltage=%d\n",
+		__func__, battery->level, battery->conn, battery->voltage);
+
+unlock:
+	up_write(&drvdata->lock_bat);
+	return ret;
+}
+
+static int battery_get_property(struct power_supply *psy,
+				enum power_supply_property psp,
+				union power_supply_propval *val)
+{
+	struct pulsar_data *drvdata;
+	int ret;
+
+	drvdata = power_supply_get_drvdata(psy);
+
+	ret = read_power(drvdata);
+	if (ret)
+		return ret;
+
+	down_read(&drvdata->lock_bat);
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_STATUS:
+		if (!drvdata->battery.available)
+			val->intval = POWER_SUPPLY_STATUS_UNKNOWN;
+		else if (drvdata->battery.conn && drvdata->battery.level < 100)
+			val->intval = POWER_SUPPLY_STATUS_CHARGING;
+		else if (drvdata->battery.conn && drvdata->battery.level >= 100)
+			val->intval = POWER_SUPPLY_STATUS_FULL;
+		else
+			val->intval = POWER_SUPPLY_STATUS_DISCHARGING;
+		break;
+	case POWER_SUPPLY_PROP_CAPACITY:
+		val->intval = drvdata->battery.level;
+		break;
+	case POWER_SUPPLY_PROP_VOLTAGE_NOW:
+		val->intval = drvdata->battery.voltage * 1000;
+		break;
+	case POWER_SUPPLY_PROP_PRESENT:
+	case POWER_SUPPLY_PROP_ONLINE:
+		val->intval = drvdata->battery.available;
+		break;
+	case POWER_SUPPLY_PROP_SCOPE:
+		val->intval = POWER_SUPPLY_SCOPE_DEVICE;
+		break;
+	case POWER_SUPPLY_PROP_MODEL_NAME:
+		val->strval = drvdata->battery.model;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	up_read(&drvdata->lock_bat);
+	return ret;
+}
+
+static void power_uevent_work_handler(struct work_struct *work)
+{
+	struct pulsar_data *drvdata;
+	int ret;
+
+	drvdata = container_of(work, struct pulsar_data, power_uevent_work);
+
+	if (atomic_read(&drvdata->stopping))
+		return;
+
+	down_write(&drvdata->lock_bat);
+	drvdata->battery.refresh = true;
+	up_write(&drvdata->lock_bat);
+
+	ret = read_power(drvdata);
+	if (ret < 0) {
+		hid_err(drvdata->hdev, "%s: failed to read power: %d\n",
+			__func__, ret);
+		return;
+	}
+
+	power_supply_changed(drvdata->battery.ps);
+}
+
+static int pulsar_raw_event(struct hid_device *hdev,
+			    struct hid_report *report, u8 *data, int size)
+{
+	struct pulsar_data *drvdata;
+
+	drvdata = hid_get_drvdata(hdev);
+	if (!drvdata)
+		return 0;
+
+	hid_dbg(hdev, "received raw event: %*ph\n", size, data);
+
+	if (size != USB_PAYLOAD_LEN || data[0] != CMD_HID_REPORT_ID)
+		return 0;
+
+	if (data[1] != CMD_EVENT) {
+		spin_lock(&drvdata->raw_event_lock);
+		if (drvdata->pending_event != data[1]) {
+			spin_unlock(&drvdata->raw_event_lock);
+			return 0;
+		}
+		memcpy(drvdata->response_buf, data, size);
+		drvdata->pending_event = CMD_NONE;
+		complete(&drvdata->response_ready);
+		spin_unlock(&drvdata->raw_event_lock);
+		return 1;
+	}
+
+	if (!atomic_read(&drvdata->device_verified))
+		return 0;
+
+	if (data[6] == EVENT_PWR && data[USB_PAYLOAD_LEN - 1] == EVENT_PWR_CHK) {
+		schedule_work(&drvdata->power_uevent_work);
+		hid_dbg(hdev, "received power event\n");
+		return 1;
+	}
+
+	return 0;
+}
+
+static const enum power_supply_property pulsar_battery_props[] = {
+	POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_CAPACITY,
+	POWER_SUPPLY_PROP_VOLTAGE_NOW, POWER_SUPPLY_PROP_ONLINE,
+	POWER_SUPPLY_PROP_MODEL_NAME, POWER_SUPPLY_PROP_SCOPE,
+	POWER_SUPPLY_PROP_PRESENT
+};
+
+static void init_power_supply_desc(struct pulsar_data *drvdata)
+{
+	drvdata->battery.desc.name = drvdata->battery.name;
+	drvdata->battery.desc.type = POWER_SUPPLY_TYPE_BATTERY;
+	drvdata->battery.desc.properties = pulsar_battery_props;
+	drvdata->battery.desc.num_properties = ARRAY_SIZE(pulsar_battery_props);
+	drvdata->battery.desc.get_property = battery_get_property;
+}
+
+static void model_pulsar(u8 *device_id, struct pulsar_data *drvdata)
+{
+	u16 model_id;
+	const char *con_type = "unknown";
+
+	model_id = device_id[0] << 8 | device_id[1];
+
+	switch (device_id[2]) {
+	case CON_1K:
+		con_type = "1kHz";
+		break;
+	case CON_4K:
+		con_type = "4kHz";
+		break;
+	case CON_WIRED:
+		con_type = "wired";
+		break;
+	}
+
+	switch (model_id) {
+	case 0x060a:
+	case 0x060b:
+	case 0x0612:
+	case 0x0613:
+	case 0x0614:
+	case 0x0615:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "Pulsar X2 V2 (%s)", con_type);
+		break;
+	case 0x060c:
+	case 0x060d:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "Pulsar X2H (%s)", con_type);
+		break;
+	case 0x0607:
+	case 0x060e:
+	case 0x060f:
+	case 0x0610:
+	case 0x0611:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "Pulsar Xlite V3 (%s)", con_type);
+		break;
+	case 0x0608:
+	case 0x0609:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "Pulsar X2A (%s)", con_type);
+		break;
+	default:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "Pulsar unknown (%s)", con_type);
+	}
+}
+
+static void model_atk(u8 *device_id, struct pulsar_data *drvdata)
+{
+	u16 model_id;
+	const char *con_type = "unknown";
+
+	model_id = device_id[0] << 8 | device_id[1];
+
+	switch (device_id[2]) {
+	case CON_1K:
+		con_type = "1kHz";
+		break;
+	case CON_4K:
+		con_type = "4kHz";
+		break;
+	case CON_WIRED:
+		con_type = "wired";
+		break;
+	}
+
+	switch (model_id) {
+	case 0x0220:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "ATK VXE R1 SE+ (%s)", con_type);
+		break;
+	default:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "Unknown ATK (%s)", con_type);
+	}
+}
+
+static void pulsar_init_work(struct work_struct *work)
+{
+	struct pulsar_data *drvdata;
+	struct hid_device *hdev;
+	struct power_supply_config psy_cfg;
+	int ret;
+	u8 data[DEV_INFO_LEN];
+
+	drvdata = container_of(work, struct pulsar_data, init_work.work);
+	hdev = drvdata->hdev;
+
+	ret = read_device_info(drvdata, data);
+	if (ret == -ETIMEDOUT) {
+		if (drvdata->init_retries--) {
+			hid_dbg(hdev,
+				"device info read timed out, retrying (%u left)\n",
+				drvdata->init_retries);
+			schedule_delayed_work(&drvdata->init_work,
+					      msecs_to_jiffies
+					      (INIT_DELAY_MSEC));
+			return;
+		}
+		hid_err(hdev, "device info read timed out, giving up\n");
+		return;
+	}
+	if (ret < 0) {
+		if (drvdata->type == TYPE_PULSAR) {
+			hid_err(hdev, "failed to read device info: %d\n", ret);
+			return;
+		}
+		hid_dbg(hdev, "failed to read device info: %d\n", ret);
+		snprintf(drvdata->battery.model,
+			 sizeof(drvdata->battery.model), "%s", hdev->name);
+		goto register_battery;
+	}
+
+	hid_dbg(hdev, "device info: %*ph (%d)\n", DEV_INFO_LEN, data, ret);
+
+	switch (drvdata->type) {
+	case TYPE_PULSAR:
+		model_pulsar(data, drvdata);
+		break;
+	case TYPE_ATK:
+		model_atk(data, drvdata);
+		break;
+	default:
+		snprintf(drvdata->battery.model, sizeof(drvdata->battery.model),
+			 "%s", hdev->name);
+	}
+
+register_battery:
+	init_power_supply_desc(drvdata);
+
+	psy_cfg = (struct power_supply_config) {.drv_data = drvdata };
+	drvdata->battery.ps =
+	    devm_power_supply_register(&hdev->dev, &drvdata->battery.desc,
+				       &psy_cfg);
+	if (IS_ERR(drvdata->battery.ps)) {
+		hid_err(hdev, "failed to register battery: %ld\n",
+			PTR_ERR(drvdata->battery.ps));
+		drvdata->battery.ps = NULL;
+		return;
+	}
+
+	atomic_set(&drvdata->device_verified, 1);
+	hid_info(hdev, "device verified, battery registered\n");
+}
+
+static int pulsar_probe(struct hid_device *hdev, const struct hid_device_id *id)
+{
+	int ret;
+	struct usb_interface *intf;
+	struct usb_device *usbdev;
+	struct pulsar_data *drvdata;
+	struct hid_report *report_in;
+	struct hid_report *report_out;
+
+	if (!hid_is_usb(hdev))
+		return -ENODEV;
+
+	ret = hid_parse(hdev);
+	if (ret < 0) {
+		hid_err(hdev, "hid_parse failed: %d\n", ret);
+		return ret;
+	}
+
+	intf = to_usb_interface(hdev->dev.parent);
+	report_in =
+	    hdev->report_enum[HID_INPUT_REPORT].report_id_hash[CMD_HID_REPORT_ID];
+	report_out =
+	    hdev->report_enum[HID_OUTPUT_REPORT].report_id_hash[CMD_HID_REPORT_ID];
+
+	if (!report_in || !report_out ||
+	    hid_report_len(report_in) != USB_PAYLOAD_LEN ||
+	    hid_report_len(report_out) != USB_PAYLOAD_LEN ||
+	    intf->cur_altsetting->desc.bInterfaceNumber != USB_INTERFACE)
+		return hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+
+	drvdata = devm_kzalloc(&hdev->dev, sizeof(*drvdata), GFP_KERNEL);
+	if (!drvdata)
+		return -ENOMEM;
+
+	drvdata->hdev = hdev;
+	drvdata->type = id->driver_data;
+
+	mutex_init(&drvdata->lock_cmd);
+	init_rwsem(&drvdata->lock_bat);
+
+	usbdev = interface_to_usbdev(intf);
+
+	spin_lock_init(&drvdata->raw_event_lock);
+	hid_set_drvdata(hdev, drvdata);
+	init_completion(&drvdata->response_ready);
+	INIT_WORK(&drvdata->power_uevent_work, power_uevent_work_handler);
+	INIT_DELAYED_WORK(&drvdata->init_work, pulsar_init_work);
+	drvdata->init_retries = INIT_RETRIES;
+	drvdata->battery.refresh = true;
+
+	snprintf(drvdata->battery.name, sizeof(drvdata->battery.name),
+		 "pulsar_%s_battery", dev_name(&usbdev->dev));
+
+	ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+	if (ret < 0) {
+		hid_err(hdev, "hw start failed\n");
+		return ret;
+	}
+
+	ret = hid_hw_open(hdev);
+	if (ret < 0) {
+		hid_err(hdev, "hw open failed\n");
+		goto err_open;
+	}
+
+	schedule_delayed_work(&drvdata->init_work, 0);
+
+	return 0;
+
+err_open:
+	hid_hw_stop(hdev);
+	cancel_work_sync(&drvdata->power_uevent_work);
+	return ret;
+}
+
+static void pulsar_remove(struct hid_device *hdev)
+{
+	struct pulsar_data *drvdata;
+
+	drvdata = hid_get_drvdata(hdev);
+	if (!drvdata) {
+		hid_hw_stop(hdev);
+		return;
+	}
+
+	atomic_set(&drvdata->stopping, 1);
+
+	/*
+	 * Drain init_work before the device is torn down.
+	 * It cannot requeue itself because exec_cmd() fails with -ENODEV.
+	 */
+	cancel_delayed_work_sync(&drvdata->init_work);
+
+	/* wait for active device i/o (exec_cmd) */
+	mutex_lock(&drvdata->lock_cmd);
+	hid_hw_close(hdev);
+	mutex_unlock(&drvdata->lock_cmd);
+
+	hid_hw_stop(hdev);
+	cancel_work_sync(&drvdata->power_uevent_work);
+}
+
+static const struct hid_device_id pulsar_table[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PULSAR, USB_DEVICE_ID_PULSAR_WIRED),
+	  .driver_data = TYPE_PULSAR },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PULSAR, USB_DEVICE_ID_PULSAR_1KHZ),
+	  .driver_data = TYPE_PULSAR },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PULSAR, USB_DEVICE_ID_PULSAR_4KHZ),
+	  .driver_data = TYPE_PULSAR },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYSONA, USB_DEVICE_ID_KYSONA_M600_DONGLE),
+	  .driver_data = TYPE_KYSONA },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYSONA, USB_DEVICE_ID_KYSONA_M600_WIRED),
+	  .driver_data = TYPE_KYSONA },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATK, USB_DEVICE_ID_ATK_VXE_R1_SE_WIRED),
+	  .driver_data = TYPE_ATK },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATK_ALT, USB_DEVICE_ID_ATK_VXE_R1_SE_DONGLE),
+	  .driver_data = TYPE_ATK },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_VXE, USB_DEVICE_ID_VXE_DRAGONFLY_R1_PRO_DONGLE),
+	  .driver_data = TYPE_VXE },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_VXE, USB_DEVICE_ID_VXE_DRAGONFLY_R1_PRO_WIRED),
+	  .driver_data = TYPE_VXE },
+	{ }
+};
+
+static struct hid_driver pulsar_driver = {
+	.name = "pulsar",
+	.id_table = pulsar_table,
+	.probe = pulsar_probe,
+	.remove = pulsar_remove,
+	.raw_event = pulsar_raw_event,
+};
+
+module_hid_driver(pulsar_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("HID driver for pulsar mice");
+MODULE_AUTHOR("Nikolas Koesling");
+MODULE_DEVICE_TABLE(hid, pulsar_table);
diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
index 8a0b51d47040c3..96a36c5ba04f79 100644
--- a/drivers/hid/hid-quirks.c
+++ b/drivers/hid/hid-quirks.c
@@ -186,6 +186,7 @@ static const struct hid_device_id hid_quirks[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X52_2), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X52_PRO), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X65), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SDINNOVATION, USB_DEVICE_ID_SDINNOVATION_GAMING_KBD), HID_QUIRK_ALWAYS_POLL },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD2), HID_QUIRK_NO_INIT_REPORTS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD), HID_QUIRK_NO_INIT_REPORTS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_SENNHEISER, USB_DEVICE_ID_SENNHEISER_BTD500USB), HID_QUIRK_NOGET },
@@ -425,13 +426,14 @@ static const struct hid_device_id hid_have_special_driver[] = {
 #endif
 #if IS_ENABLED(CONFIG_HID_ELECOM)
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) },
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1URBK) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1DRBK) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT2DRBK) },
@@ -751,6 +753,7 @@ static const struct hid_device_id hid_have_special_driver[] = {
 #if IS_ENABLED(CONFIG_HID_STEELSERIES)
 	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_SRWS1) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_1_X) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_7_2018) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_9) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_5_X) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_7) },
diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c
index 4f15eb95103968..5deb6da8d4f746 100644
--- a/drivers/hid/hid-roccat.c
+++ b/drivers/hid/hid-roccat.c
@@ -344,8 +344,6 @@ int roccat_connect(const struct class *klass, struct hid_device *hid, int report
 		return temp;
 	}
 
-	mutex_unlock(&devices_lock);
-
 	init_waitqueue_head(&device->wait);
 	INIT_LIST_HEAD(&device->readers);
 	mutex_init(&device->readers_lock);
@@ -356,6 +354,7 @@ int roccat_connect(const struct class *klass, struct hid_device *hid, int report
 	device->cbuf_end = 0;
 	device->report_size = report_size;
 
+	mutex_unlock(&devices_lock);
 	return minor;
 }
 EXPORT_SYMBOL_GPL(roccat_connect);
@@ -369,15 +368,12 @@ void roccat_disconnect(int minor)
 
 	mutex_lock(&devices_lock);
 	device = devices[minor];
-	mutex_unlock(&devices_lock);
 
 	device->exist = 0; /* TODO exist maybe not needed */
 
 	device_destroy(device->dev->class, MKDEV(roccat_major, minor));
 
-	mutex_lock(&devices_lock);
 	devices[minor] = NULL;
-	mutex_unlock(&devices_lock);
 
 	if (device->open) {
 		hid_hw_close(device->hid);
@@ -385,6 +381,8 @@ void roccat_disconnect(int minor)
 	} else {
 		roccat_free_device(device);
 	}
+
+	mutex_unlock(&devices_lock);
 }
 EXPORT_SYMBOL_GPL(roccat_disconnect);
 
diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c
index 6470a290ebfc5b..a9bd72218c070c 100644
--- a/drivers/hid/hid-sensor-hub.c
+++ b/drivers/hid/hid-sensor-hub.c
@@ -334,6 +334,8 @@ int sensor_hub_input_attr_read_values(struct hid_sensor_hub_device *hsdev,
 			ret = -ETIMEDOUT;
 		else if (cycles < 0)
 			ret = cycles;
+		else if (hsdev->pending.shutdown)
+			ret = -ENODEV;
 
 		hsdev->pending.status = false;
 	}
@@ -805,8 +807,10 @@ static int sensor_hub_finalize_pending_fn(struct device *dev, void *data)
 {
 	struct hid_sensor_hub_device *hsdev = dev->platform_data;
 
-	if (hsdev->pending.status)
+	if (hsdev->pending.status) {
+		hsdev->pending.shutdown = true;
 		complete(&hsdev->pending.ready);
+	}
 
 	return 0;
 }
diff --git a/drivers/hid/hid-steelseries-arctis.c b/drivers/hid/hid-steelseries-arctis.c
index 7a855b66730a6c..df61fa33d4c687 100644
--- a/drivers/hid/hid-steelseries-arctis.c
+++ b/drivers/hid/hid-steelseries-arctis.c
@@ -101,6 +101,19 @@ static int steelseries_arctis_1_request_status(struct hid_device *hdev)
 	return steelseries_send_output_report(hdev, data, sizeof(data));
 }
 
+static int steelseries_arctis_7_2018_request_status(struct hid_device *hdev)
+{
+	const u8 connection[] = { 0x06, 0x14 };
+	const u8 battery[] = { 0x06, 0x18 };
+	int ret;
+
+	ret = steelseries_send_output_report(hdev, connection, sizeof(connection));
+	if (ret)
+		return ret;
+
+	return steelseries_send_output_report(hdev, battery, sizeof(battery));
+}
+
 static int steelseries_arctis_9_request_status(struct hid_device *hdev)
 {
 	const u8 data[] = { 0x00, 0x20 };
@@ -152,6 +165,27 @@ static void steelseries_arctis_1_parse_status(struct steelseries_device *sd,
 	sd->battery_capacity = data[3];
 }
 
+static void steelseries_arctis_7_2018_parse_status(struct steelseries_device *sd,
+						      u8 *data, int size)
+{
+	if (size < 3 || data[0] != 0x06)
+		return;
+
+	switch (data[1]) {
+	case 0x14:
+		/* 0x03 means that the headset is connected to the transmitter. */
+		sd->headset_connected = data[2] == 0x03;
+		break;
+	case 0x18:
+		if (!sd->headset_connected)
+			break;
+
+		/* The Arctis 7 sometimes overreports battery. Cap to 100. */
+		sd->battery_capacity = steelseries_map_capacity(data[2], 0, 100);
+		break;
+	}
+}
+
 static void steelseries_arctis_9_parse_status(struct steelseries_device *sd,
 					      u8 *data, int size)
 {
@@ -232,6 +266,13 @@ static const struct steelseries_device_info arctis_1_info = {
 	.parse_status = steelseries_arctis_1_parse_status,
 };
 
+static const struct steelseries_device_info arctis_7_2018_info = {
+	.sync_interface = 5,
+	.capabilities = SS_CAP_BATTERY,
+	.request_status = steelseries_arctis_7_2018_request_status,
+	.parse_status = steelseries_arctis_7_2018_parse_status,
+};
+
 static const struct steelseries_device_info arctis_9_info = {
 	.sync_interface = 0,
 	.capabilities = SS_CAP_BATTERY,
@@ -653,6 +694,9 @@ static const struct hid_device_id steelseries_arctis_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES,
 			 USB_DEVICE_ID_STEELSERIES_ARCTIS_1_X),
 	  .driver_data = (unsigned long)&arctis_1_info },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES,
+			 USB_DEVICE_ID_STEELSERIES_ARCTIS_7_2018),
+	  .driver_data = (unsigned long)&arctis_7_2018_info },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES,
 			 USB_DEVICE_ID_STEELSERIES_ARCTIS_9),
 	  .driver_data = (unsigned long)&arctis_9_info },
diff --git a/drivers/hid/hid-universal-pidff.c b/drivers/hid/hid-universal-pidff.c
index 60180467a0bfd9..f1854cfe42d0d9 100644
--- a/drivers/hid/hid-universal-pidff.c
+++ b/drivers/hid/hid-universal-pidff.c
@@ -187,6 +187,7 @@ static const struct hid_device_id universal_pidff_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASETEK, USB_DEVICE_ID_ASETEK_FORTE) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASETEK, USB_DEVICE_ID_ASETEK_LA_PRIMA) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASETEK, USB_DEVICE_ID_ASETEK_TONY_KANAAN) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TURTLE_BEACH, USB_DEVICE_ID_TB_VELOCITYONE_RACE)},
 	{}
 };
 MODULE_DEVICE_TABLE(hid, universal_pidff_devices);
diff --git a/drivers/hid/hid-winwing.c b/drivers/hid/hid-winwing.c
index 9cd25a77999e65..19b92c2c6579a2 100644
--- a/drivers/hid/hid-winwing.c
+++ b/drivers/hid/hid-winwing.c
@@ -315,7 +315,8 @@ static void winwing_haptic_rumble_cb(struct work_struct *work)
 static int winwing_play_effect(struct input_dev *dev, void *context,
 		struct ff_effect *effect)
 {
-	struct winwing_drv_data *data = (struct winwing_drv_data *) context;
+	struct hid_device *hdev = input_get_drvdata(dev);
+	struct winwing_drv_data *data = hid_get_drvdata(hdev);
 
 	if (effect->type != FF_RUMBLE)
 		return 0;
@@ -342,7 +343,12 @@ static int winwing_init_ff(struct hid_device *hdev, struct hid_input *hidinput)
 
 	input_set_capability(hidinput->input, EV_FF, FF_RUMBLE);
 
-	return input_ff_create_memless(hidinput->input, data,
+	/*
+	 * input_ff_create_memless() takes ownership of the context pointer
+	 * and frees it on teardown; do not hand it the devm-managed drvdata.
+	 * winwing_play_effect() fetches it from the input device instead.
+	 */
+	return input_ff_create_memless(hidinput->input, NULL,
 			winwing_play_effect);
 }
 
diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.c b/drivers/hid/i2c-hid/i2c-hid-acpi.c
index 13f977d6aab615..9371db200fc4da 100644
--- a/drivers/hid/i2c-hid/i2c-hid-acpi.c
+++ b/drivers/hid/i2c-hid/i2c-hid-acpi.c
@@ -39,12 +39,12 @@ static const struct acpi_device_id i2c_hid_acpi_blacklist[] = {
 	 * The CHPN0001 ACPI device, which is used to describe the Chipone
 	 * ICN8505 controller, has a _CID of PNP0C50 but is not HID compatible.
 	 */
-	{ "CHPN0001" },
+	{ .id = "CHPN0001" },
 	/*
 	 * The IDEA5002 ACPI device causes high interrupt usage and spurious
 	 * wakeups from suspend.
 	 */
-	{ "IDEA5002" },
+	{ .id = "IDEA5002" },
 	{ }
 };
 
@@ -93,8 +93,8 @@ static int i2c_hid_acpi_probe(struct i2c_client *client)
 }
 
 static const struct acpi_device_id i2c_hid_acpi_match[] = {
-	{ "ACPI0C50" },
-	{ "PNP0C50" },
+	{ .id = "ACPI0C50" },
+	{ .id = "PNP0C50" },
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, i2c_hid_acpi_match);
diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c
index 0ff07fdab442a3..a8a9854854f523 100644
--- a/drivers/hid/i2c-hid/i2c-hid-core.c
+++ b/drivers/hid/i2c-hid/i2c-hid-core.c
@@ -123,9 +123,11 @@ static const struct i2c_hid_quirks {
 	__u32 quirks;
 } i2c_hid_quirks[] = {
 	{ I2C_VENDOR_ID_HANTICK, I2C_PRODUCT_ID_HANTICK_5288,
-		I2C_HID_QUIRK_NO_IRQ_AFTER_RESET },
+		I2C_HID_QUIRK_NO_IRQ_AFTER_RESET | I2C_HID_QUIRK_BAD_INPUT_SIZE },
 	{ I2C_VENDOR_ID_ITE, I2C_DEVICE_ID_ITE_VOYO_WINPAD_A15,
 		I2C_HID_QUIRK_NO_IRQ_AFTER_RESET },
+	{ USB_VENDOR_ID_ITE, I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_G11_KEYBOARD,
+		I2C_HID_QUIRK_NO_IRQ_AFTER_RESET },
 	{ I2C_VENDOR_ID_RAYDIUM, I2C_PRODUCT_ID_RAYDIUM_3118,
 		I2C_HID_QUIRK_NO_IRQ_AFTER_RESET },
 	{ USB_VENDOR_ID_ALPS_JP, HID_ANY_ID,
@@ -1355,7 +1357,16 @@ void i2c_hid_core_shutdown(struct i2c_client *client)
 {
 	struct i2c_hid *ihid = i2c_get_clientdata(client);
 
-	i2c_hid_set_power(ihid, I2C_HID_PWR_SLEEP);
+	/*
+	 * Like in i2c_hid_core_remove(): if we're a follower, the act of
+	 * unfollowing will cause us to be powered down. Otherwise we need to
+	 * manually do it.
+	 */
+	if (ihid->is_panel_follower)
+		drm_panel_remove_follower(&ihid->panel_follower);
+	else
+		i2c_hid_set_power(ihid, I2C_HID_PWR_SLEEP);
+
 	free_irq(client->irq, ihid);
 
 	i2c_hid_core_shutdown_tail(ihid);
diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c
index 0d2ad7bc3648a2..7a913f717efaad 100644
--- a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c
+++ b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c
@@ -315,7 +315,7 @@ static int handle_input_report(struct quicki2c_device *qcdev)
 		if (qcdev->state != QUICKI2C_ENABLED)
 			continue;
 
-		quicki2c_hid_send_report(qcdev, pkt->data,
+		quicki2c_hid_send_report(qcdev, pkt->data, qcdev->input_len,
 					 HIDI2C_DATA_LEN(le16_to_cpu(pkt->len)));
 	}
 
@@ -606,17 +606,21 @@ static int quicki2c_alloc_report_buf(struct quicki2c_device *qcdev)
 	 */
 	max_report_len = max(le16_to_cpu(qcdev->dev_desc.max_input_len), SZ_4K);
 
-	qcdev->input_buf = devm_kzalloc(qcdev->dev, max_report_len, GFP_KERNEL);
+	qcdev->input_buf = devm_kzalloc(qcdev->dev,
+					HIDI2C_PACKET_LEN(max_report_len), GFP_KERNEL);
 	if (!qcdev->input_buf)
 		return -ENOMEM;
 
+	qcdev->input_len = max_report_len;
+
 	if (!le16_to_cpu(qcdev->dev_desc.max_output_len))
 		qcdev->dev_desc.max_output_len = cpu_to_le16(SZ_4K);
 
 	max_report_len = max(le16_to_cpu(qcdev->dev_desc.max_output_len),
 			     max_report_len);
 
-	qcdev->report_buf = devm_kzalloc(qcdev->dev, max_report_len, GFP_KERNEL);
+	qcdev->report_buf = devm_kzalloc(qcdev->dev,
+					 HIDI2C_PACKET_LEN(max_report_len), GFP_KERNEL);
 	if (!qcdev->report_buf)
 		return -ENOMEM;
 
diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h
index 6d25a846153e66..e9ff0440fb5464 100644
--- a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h
+++ b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-dev.h
@@ -181,6 +181,7 @@ struct acpi_device;
  * @low_power_ltr_val: THC low power LTR value
  * @report_descriptor: Store a copy of device report descriptor
  * @input_buf: Store a copy of latest input report data
+ * @input_len: The data size of the input report buffer
  * @report_buf: Store a copy of latest input/output report packet from set/get feature
  * @report_len: The length of input/output report packet
  * @reset_ack_wq: Workqueue for waiting reset response from device
@@ -212,6 +213,7 @@ struct quicki2c_device {
 
 	u8 *report_descriptor;
 	u8 *input_buf;
+	size_t input_len;
 	u8 *report_buf;
 	size_t report_len;
 
diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c
index 8075992e8732ef..1a220a50d934ca 100644
--- a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c
+++ b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.c
@@ -157,16 +157,18 @@ void quicki2c_hid_remove(struct quicki2c_device *qcdev)
  *
  * @qcdev: point to quicki2c device
  * @data: point to input report data buffer
+ * @buf_size: the allocated size of the input report data buffer
  * @data_len: the length of input report data
  *
  * Return: 0 on success, non zero on error.
  */
 int quicki2c_hid_send_report(struct quicki2c_device *qcdev,
-			     void *data, size_t data_len)
+			     void *data, size_t buf_size, size_t data_len)
 {
 	int ret;
 
-	ret = hid_input_report(qcdev->hid_dev, HID_INPUT_REPORT, data, data_len, 1);
+	ret = hid_safe_input_report(qcdev->hid_dev, HID_INPUT_REPORT, data,
+				    buf_size, data_len, 1);
 	if (ret)
 		dev_err(qcdev->dev, "Failed to send HID input report, ret = %d.\n", ret);
 
diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.h b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.h
index e80df5f339fefb..4c6757eca18e15 100644
--- a/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.h
+++ b/drivers/hid/intel-thc-hid/intel-quicki2c/quicki2c-hid.h
@@ -7,7 +7,7 @@
 struct quicki2c_device;
 
 int quicki2c_hid_send_report(struct quicki2c_device *qcdev,
-			     void *data, size_t data_size);
+			     void *data, size_t buf_size, size_t data_size);
 int quicki2c_hid_probe(struct quicki2c_device *qcdev);
 void quicki2c_hid_remove(struct quicki2c_device *qcdev);
 
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c
index 89226f5ce45e09..53023e30cf8d47 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c
@@ -552,10 +552,13 @@ static int quickspi_alloc_report_buf(struct quickspi_device *qsdev)
 	max_input_len = max(le16_to_cpu(qsdev->dev_desc.rep_desc_len),
 			    le16_to_cpu(qsdev->dev_desc.max_input_len));
 
-	qsdev->input_buf = devm_kzalloc(qsdev->dev, max_input_len, GFP_KERNEL);
+	qsdev->input_buf = devm_kzalloc(qsdev->dev,
+					HIDSPI_INPUT_BODY_SIZE(max_input_len), GFP_KERNEL);
 	if (!qsdev->input_buf)
 		return -ENOMEM;
 
+	qsdev->input_len = max_input_len;
+
 	max_report_len = max(le16_to_cpu(qsdev->dev_desc.max_output_len),
 			     le16_to_cpu(qsdev->dev_desc.max_input_len));
 
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h
index dc67a39ad546b6..fe51ff51d8c0e9 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h
@@ -113,6 +113,7 @@ struct acpi_device;
  * @low_power_ltr_val: THC low power LTR value
  * @report_descriptor: store a copy of device report descriptor
  * @input_buf: store a copy of latest input report data
+ * @input_len: the data size of the input report buffer
  * @report_buf: store a copy of latest input/output report packet from set/get feature
  * @report_len: the length of input/output report packet
  * @reset_ack_wq: workqueue for waiting reset response from device
@@ -159,6 +160,7 @@ struct quickspi_device {
 
 	u8 *report_descriptor;
 	u8 *input_buf;
+	u32 input_len;
 	u8 *report_buf;
 	u32 report_buf_size;
 	u32 report_len;
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c
index a60a0a7f16aad3..ed1c40b8c4fa1d 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c
@@ -148,16 +148,18 @@ void quickspi_hid_remove(struct quickspi_device *qsdev)
  *
  * @qsdev: point to quickspi device
  * @data: point to input report data buffer
+ * @buf_size: the allocated size of the input report data buffer
  * @data_len: the length of input report data
  *
  * Return: 0 on success, non zero on error.
  */
 int quickspi_hid_send_report(struct quickspi_device *qsdev,
-			     void *data, size_t data_len)
+			     void *data, size_t buf_size, size_t data_len)
 {
 	int ret;
 
-	ret = hid_input_report(qsdev->hid_dev, HID_INPUT_REPORT, data, data_len, 1);
+	ret = hid_safe_input_report(qsdev->hid_dev, HID_INPUT_REPORT, data,
+				    buf_size, data_len, 1);
 	if (ret)
 		dev_err(qsdev->dev, "Failed to send HID input report, ret = %d.\n", ret);
 
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.h
index f640fa876a405a..61b7c6be6c5160 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.h
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.h
@@ -7,7 +7,7 @@
 struct quickspi_device;
 
 int quickspi_hid_send_report(struct quickspi_device *qsdev,
-			     void *data, size_t data_size);
+			     void *data, size_t buf_size, size_t data_size);
 int quickspi_hid_probe(struct quickspi_device *qsdev);
 void quickspi_hid_remove(struct quickspi_device *qsdev);
 
diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c
index 847c5ec55569a2..0df33c44fbffe5 100644
--- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c
+++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c
@@ -240,7 +240,7 @@ void quickspi_handle_input_data(struct quickspi_device *qsdev, u32 buf_len)
 		input_len = sizeof(body_hdr->content_id) + input_len;
 		input_report = input_body->content - sizeof(body_hdr->content_id);
 
-		ret = quickspi_hid_send_report(qsdev, input_report, input_len);
+		ret = quickspi_hid_send_report(qsdev, input_report, qsdev->input_len, input_len);
 		if (ret)
 			dev_err_once(qsdev->dev, "Failed to send HID input report: %d\n", ret);
 
diff --git a/drivers/hid/surface-hid/surface_kbd.c b/drivers/hid/surface-hid/surface_kbd.c
index 0be01b5e742585..036829ba47d427 100644
--- a/drivers/hid/surface-hid/surface_kbd.c
+++ b/drivers/hid/surface-hid/surface_kbd.c
@@ -277,8 +277,8 @@ static void surface_kbd_remove(struct platform_device *pdev)
 }
 
 static const struct acpi_device_id surface_kbd_match[] = {
-	{ "MSHW0096" },
-	{ },
+	{ .id = "MSHW0096" },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, surface_kbd_match);
 
diff --git a/drivers/hid/usbhid/hiddev.c b/drivers/hid/usbhid/hiddev.c
index 6378801b22c6d8..8430655c00fe67 100644
--- a/drivers/hid/usbhid/hiddev.c
+++ b/drivers/hid/usbhid/hiddev.c
@@ -46,6 +46,12 @@ struct hiddev_list {
 	struct mutex thread_lock;
 };
 
+static void hiddev_put(struct hiddev *hiddev)
+{
+	if (refcount_dec_and_test(&hiddev->refcount))
+		kfree(hiddev);
+}
+
 /*
  * Find a report, given the report's type and ID.  The ID can be specified
  * indirectly by REPORT_ID_FIRST (which returns the first report of the given
@@ -216,26 +222,21 @@ static int hiddev_fasync(int fd, struct file *file, int on)
 static int hiddev_release(struct inode * inode, struct file * file)
 {
 	struct hiddev_list *list = file->private_data;
+	struct hiddev *hiddev = list->hiddev;
 	unsigned long flags;
 
-	spin_lock_irqsave(&list->hiddev->list_lock, flags);
+	spin_lock_irqsave(&hiddev->list_lock, flags);
 	list_del(&list->node);
-	spin_unlock_irqrestore(&list->hiddev->list_lock, flags);
+	spin_unlock_irqrestore(&hiddev->list_lock, flags);
 
-	mutex_lock(&list->hiddev->existancelock);
-	if (!--list->hiddev->open) {
-		if (list->hiddev->exist) {
-			hid_hw_close(list->hiddev->hid);
-			hid_hw_power(list->hiddev->hid, PM_HINT_NORMAL);
-		} else {
-			mutex_unlock(&list->hiddev->existancelock);
-			kfree(list->hiddev);
-			vfree(list);
-			return 0;
-		}
+	mutex_lock(&hiddev->existancelock);
+	if (!--hiddev->open && hiddev->exist) {
+		hid_hw_close(hiddev->hid);
+		hid_hw_power(hiddev->hid, PM_HINT_NORMAL);
 	}
 
-	mutex_unlock(&list->hiddev->existancelock);
+	mutex_unlock(&hiddev->existancelock);
+	hiddev_put(hiddev);
 	vfree(list);
 
 	return 0;
@@ -270,6 +271,7 @@ static int __hiddev_open(struct hiddev *hiddev, struct file *file)
 	spin_unlock_irq(&hiddev->list_lock);
 
 	file->private_data = list;
+	refcount_inc(&hiddev->refcount);
 
 	return 0;
 
@@ -897,9 +899,12 @@ int hiddev_connect(struct hid_device *hid, unsigned int force)
 	INIT_LIST_HEAD(&hiddev->list);
 	spin_lock_init(&hiddev->list_lock);
 	mutex_init(&hiddev->existancelock);
+	refcount_set(&hiddev->refcount, 1);
 	hid->hiddev = hiddev;
 	hiddev->hid = hid;
 	hiddev->exist = 1;
+	/* Honor NO_INIT_REPORTS before the character device becomes visible. */
+	hiddev->initialized = hid->quirks & HID_QUIRK_NO_INIT_REPORTS;
 	retval = usb_register_dev(usbhid->intf, &hiddev_class);
 	if (retval) {
 		hid_err(hid, "Not able to get a minor for this device\n");
@@ -908,12 +913,6 @@ int hiddev_connect(struct hid_device *hid, unsigned int force)
 		return retval;
 	}
 
-	/*
-	 * If HID_QUIRK_NO_INIT_REPORTS is set, make sure we don't initialize
-	 * the reports.
-	 */
-	hiddev->initialized = hid->quirks & HID_QUIRK_NO_INIT_REPORTS;
-
 	hiddev->minor = usbhid->intf->minor;
 
 	return 0;
@@ -937,9 +936,7 @@ void hiddev_disconnect(struct hid_device *hid)
 	if (hiddev->open) {
 		hid_hw_close(hiddev->hid);
 		wake_up_interruptible(&hiddev->wait);
-		mutex_unlock(&hiddev->existancelock);
-	} else {
-		mutex_unlock(&hiddev->existancelock);
-		kfree(hiddev);
 	}
+	mutex_unlock(&hiddev->existancelock);
+	hiddev_put(hiddev);
 }
diff --git a/drivers/hid/wacom.h b/drivers/hid/wacom.h
index 1deacb4568cb92..d2398b1cdbb2cf 100644
--- a/drivers/hid/wacom.h
+++ b/drivers/hid/wacom.h
@@ -82,7 +82,7 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include <linux/mod_devicetable.h>
+#include <linux/device-id/hid.h>
 #include <linux/hid.h>
 #include <linux/kfifo.h>
 #include <linux/leds.h>
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index 0eafa483b7f757..43d13c59554c21 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -113,8 +113,9 @@ static int wacom_wac_pen_serial_enforce(struct hid_device *hdev,
 
 	/* Queue events which have invalid tool type or serial number */
 	for (i = 0; i < report->maxfield; i++) {
-		for (j = 0; j < report->field[i]->maxusage; j++) {
-			struct hid_field *field = report->field[i];
+		struct hid_field *field = report->field[i];
+
+		for (j = 0; j < field->report_count; j++) {
 			struct hid_usage *usage = &field->usage[j];
 			unsigned int equivalent_usage = wacom_equivalent_usage(usage->hid);
 			unsigned int offset;
@@ -757,27 +758,47 @@ static void wacom_retrieve_hid_descriptor(struct hid_device *hdev,
 struct wacom_hdev_data {
 	struct list_head list;
 	struct kref kref;
-	struct hid_device *dev;
+	char phys[64];
+	__u32 vendor;
+	__u32 product;
+	__u32 device_type;
 	struct wacom_shared shared;
 };
 
+static bool wacom_compare_device_paths(struct hid_device *hdev_a,
+				       const char *phys_b, char separator)
+{
+	const char *p1 = strrchr(hdev_a->phys, separator);
+	const char *p2 = strrchr(phys_b, separator);
+	int n1, n2;
+
+	if (!p1 || !p2)
+		return false;
+
+	n1 = p1 - hdev_a->phys;
+	n2 = p2 - phys_b;
+
+	if (n1 != n2 || n1 <= 0 || n2 <= 0)
+		return false;
+
+	return !strncmp(hdev_a->phys, phys_b, n1);
+}
+
 static LIST_HEAD(wacom_udev_list);
 static DEFINE_MUTEX(wacom_udev_list_lock);
 
 static bool wacom_are_sibling(struct hid_device *hdev,
-		struct hid_device *sibling)
+		struct wacom_hdev_data *data)
 {
 	struct wacom *wacom = hid_get_drvdata(hdev);
 	struct wacom_features *features = &wacom->wacom_wac.features;
-	struct wacom *sibling_wacom = hid_get_drvdata(sibling);
-	struct wacom_features *sibling_features = &sibling_wacom->wacom_wac.features;
 	__u32 oVid = features->oVid ? features->oVid : hdev->vendor;
 	__u32 oPid = features->oPid ? features->oPid : hdev->product;
 
 	/* The defined oVid/oPid must match that of the sibling */
-	if (features->oVid != HID_ANY_ID && sibling->vendor != oVid)
+	if (features->oVid != HID_ANY_ID && data->vendor != oVid)
 		return false;
-	if (features->oPid != HID_ANY_ID && sibling->product != oPid)
+	if (features->oPid != HID_ANY_ID && data->product != oPid)
 		return false;
 
 	/*
@@ -785,11 +806,11 @@ static bool wacom_are_sibling(struct hid_device *hdev,
 	 * device path, while those with different VID/PID must share
 	 * the same physical parent device path.
 	 */
-	if (hdev->vendor == sibling->vendor && hdev->product == sibling->product) {
-		if (!hid_compare_device_paths(hdev, sibling, '/'))
+	if (hdev->vendor == data->vendor && hdev->product == data->product) {
+		if (!wacom_compare_device_paths(hdev, data->phys, '/'))
 			return false;
 	} else {
-		if (!hid_compare_device_paths(hdev, sibling, '.'))
+		if (!wacom_compare_device_paths(hdev, data->phys, '.'))
 			return false;
 	}
 
@@ -802,7 +823,7 @@ static bool wacom_are_sibling(struct hid_device *hdev,
 	 * devices.
 	 */
 	if ((features->device_type & WACOM_DEVICETYPE_DIRECT) &&
-	    !(sibling_features->device_type & WACOM_DEVICETYPE_DIRECT))
+	    !(data->device_type & WACOM_DEVICETYPE_DIRECT))
 		return false;
 
 	/*
@@ -810,17 +831,17 @@ static bool wacom_are_sibling(struct hid_device *hdev,
 	 * devices.
 	 */
 	if (!(features->device_type & WACOM_DEVICETYPE_DIRECT) &&
-	    (sibling_features->device_type & WACOM_DEVICETYPE_DIRECT))
+	    (data->device_type & WACOM_DEVICETYPE_DIRECT))
 		return false;
 
 	/* Pen devices may only be siblings of touch devices */
 	if ((features->device_type & WACOM_DEVICETYPE_PEN) &&
-	    !(sibling_features->device_type & WACOM_DEVICETYPE_TOUCH))
+	    !(data->device_type & WACOM_DEVICETYPE_TOUCH))
 		return false;
 
 	/* Touch devices may only be siblings of pen devices */
 	if ((features->device_type & WACOM_DEVICETYPE_TOUCH) &&
-	    !(sibling_features->device_type & WACOM_DEVICETYPE_PEN))
+	    !(data->device_type & WACOM_DEVICETYPE_PEN))
 		return false;
 
 	/*
@@ -836,7 +857,7 @@ static struct wacom_hdev_data *wacom_get_hdev_data(struct hid_device *hdev)
 
 	/* Try to find an already-probed interface from the same device */
 	list_for_each_entry(data, &wacom_udev_list, list) {
-		if (hid_compare_device_paths(hdev, data->dev, '/')) {
+		if (wacom_compare_device_paths(hdev, data->phys, '/')) {
 			kref_get(&data->kref);
 			return data;
 		}
@@ -844,7 +865,7 @@ static struct wacom_hdev_data *wacom_get_hdev_data(struct hid_device *hdev)
 
 	/* Fallback to finding devices that appear to be "siblings" */
 	list_for_each_entry(data, &wacom_udev_list, list) {
-		if (wacom_are_sibling(hdev, data->dev)) {
+		if (wacom_are_sibling(hdev, data)) {
 			kref_get(&data->kref);
 			return data;
 		}
@@ -858,29 +879,41 @@ static void wacom_release_shared_data(struct kref *kref)
 	struct wacom_hdev_data *data =
 		container_of(kref, struct wacom_hdev_data, kref);
 
-	mutex_lock(&wacom_udev_list_lock);
 	list_del(&data->list);
-	mutex_unlock(&wacom_udev_list_lock);
-
 	kfree(data);
 }
 
 static void wacom_remove_shared_data(void *res)
 {
-	struct wacom *wacom = res;
+	struct wacom *res_wacom = res;
 	struct wacom_hdev_data *data;
-	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_wac *wacom_wac = &res_wacom->wacom_wac;
 
 	if (wacom_wac->shared) {
 		data = container_of(wacom_wac->shared, struct wacom_hdev_data,
 				    shared);
 
-		if (wacom_wac->shared->touch == wacom->hdev)
-			wacom_wac->shared->touch = NULL;
-		else if (wacom_wac->shared->pen == wacom->hdev)
-			wacom_wac->shared->pen = NULL;
+		scoped_guard(mutex, &wacom_udev_list_lock) {
+			struct hid_device *touch =
+				rcu_dereference_protected(wacom_wac->shared->touch,
+							  lockdep_is_held(&wacom_udev_list_lock));
+			struct hid_device *pen =
+				rcu_dereference_protected(wacom_wac->shared->pen,
+							  lockdep_is_held(&wacom_udev_list_lock));
+
+			if (touch == res_wacom->hdev) {
+				rcu_assign_pointer(wacom_wac->shared->touch, NULL);
+				rcu_assign_pointer(wacom_wac->shared->touch_input, NULL);
+			} else if (pen == res_wacom->hdev) {
+				rcu_assign_pointer(wacom_wac->shared->pen, NULL);
+			}
+		}
+
+		synchronize_rcu();
+
+		if (kref_put_mutex(&data->kref, wacom_release_shared_data, &wacom_udev_list_lock))
+			mutex_unlock(&wacom_udev_list_lock);
 
-		kref_put(&data->kref, wacom_release_shared_data);
 		wacom_wac->shared = NULL;
 	}
 }
@@ -903,22 +936,21 @@ static int wacom_add_shared_data(struct hid_device *hdev)
 		}
 
 		kref_init(&data->kref);
-		data->dev = hdev;
+		strscpy(data->phys, hdev->phys, sizeof(data->phys));
+		data->vendor = hdev->vendor;
+		data->product = hdev->product;
+		data->device_type = wacom_wac->features.device_type;
 		list_add_tail(&data->list, &wacom_udev_list);
 	}
 
-	mutex_unlock(&wacom_udev_list_lock);
-
 	wacom_wac->shared = &data->shared;
 
-	retval = devm_add_action_or_reset(&hdev->dev, wacom_remove_shared_data, wacom);
-	if (retval)
-		return retval;
+	if (wacom_wac->has_mute_touch_switch)
+		WRITE_ONCE(wacom_wac->shared->has_mute_touch_switch, true);
 
-	if (wacom_wac->features.device_type & WACOM_DEVICETYPE_TOUCH)
-		wacom_wac->shared->touch = hdev;
-	else if (wacom_wac->features.device_type & WACOM_DEVICETYPE_PEN)
-		wacom_wac->shared->pen = hdev;
+	mutex_unlock(&wacom_udev_list_lock);
+
+	retval = devm_add_action_or_reset(&hdev->dev, wacom_remove_shared_data, wacom);
 
 	return retval;
 }
@@ -2343,13 +2375,12 @@ static void wacom_release_resources(struct wacom *wacom)
 
 static void wacom_set_shared_values(struct wacom_wac *wacom_wac)
 {
-	if (wacom_wac->features.device_type & WACOM_DEVICETYPE_TOUCH) {
-		wacom_wac->shared->type = wacom_wac->features.type;
-		wacom_wac->shared->touch_input = wacom_wac->touch_input;
-	}
+	struct wacom *wacom = container_of(wacom_wac, struct wacom, wacom_wac);
+
+	guard(mutex)(&wacom_udev_list_lock);
 
 	if (wacom_wac->has_mute_touch_switch) {
-		wacom_wac->shared->has_mute_touch_switch = true;
+		WRITE_ONCE(wacom_wac->shared->has_mute_touch_switch, true);
 		/* Hardware touch switch may be off. Wait until
 		 * we know the switch state to decide is_touch_on.
 		 * Softkey state should be initialized to "on" to
@@ -2359,14 +2390,73 @@ static void wacom_set_shared_values(struct wacom_wac *wacom_wac)
 			wacom_wac->shared->is_touch_on = true;
 	}
 
-	if (wacom_wac->shared->has_mute_touch_switch &&
-	    wacom_wac->shared->touch_input) {
-		set_bit(EV_SW, wacom_wac->shared->touch_input->evbit);
-		input_set_capability(wacom_wac->shared->touch_input, EV_SW,
-				     SW_MUTE_DEVICE);
+	if (wacom_wac->features.device_type & WACOM_DEVICETYPE_TOUCH) {
+		struct hid_device *touch =
+			rcu_dereference_protected(wacom_wac->shared->touch,
+						  lockdep_is_held(&wacom_udev_list_lock));
+
+		if (touch == wacom->hdev || !touch) {
+			wacom_wac->shared->type = wacom_wac->features.type;
+			rcu_assign_pointer(wacom_wac->shared->touch, wacom->hdev);
+			rcu_assign_pointer(wacom_wac->shared->touch_input, wacom_wac->touch_input);
+		}
+	} else if (wacom_wac->features.device_type & WACOM_DEVICETYPE_PEN) {
+		rcu_assign_pointer(wacom_wac->shared->pen, wacom->hdev);
 	}
 }
 
+static bool wacom_sibling_pending(struct wacom *wacom)
+{
+	const struct wacom_features *features = &wacom->wacom_wac.features;
+	struct hid_device *hdev = wacom->hdev;
+	struct usb_interface *sibling_intf;
+	int ifnum;
+
+	if (features->type != HID_GENERIC ||
+	    !(features->device_type & WACOM_DEVICETYPE_TOUCH))
+		return false;
+
+	if (wacom->wacom_wac.shared) {
+		if (rcu_access_pointer(wacom->wacom_wac.shared->pen))
+			return false;
+	}
+
+	if (!hid_is_usb(hdev) || !wacom->usbdev || !wacom->intf ||
+	    !wacom->intf->cur_altsetting)
+		return false;
+
+	ifnum = wacom->intf->cur_altsetting->desc.bInterfaceNumber;
+
+	/*
+	 * On composite Wacom devices, the Pen interface is always interface 0.
+	 * If the Touch interface is interface 0, there is no sibling Pen
+	 * interface on this device (standalone touch device).
+	 */
+	if (ifnum == 0)
+		return false;
+
+	/* Look for the sibling Pen interface at interface 0 */
+	sibling_intf = usb_ifnum_to_if(wacom->usbdev, 0);
+	if (!sibling_intf || !sibling_intf->cur_altsetting)
+		return false;
+
+	if (sibling_intf->cur_altsetting->desc.bInterfaceClass !=
+	    USB_INTERFACE_CLASS_HID)
+		return false;
+
+	if (sibling_intf->cur_altsetting->desc.bInterfaceSubClass == 1 &&
+	    (sibling_intf->cur_altsetting->desc.bInterfaceProtocol == 1 ||
+	     sibling_intf->cur_altsetting->desc.bInterfaceProtocol == 2))
+		return false;
+
+	/*
+	 * Interface 0 is a candidate HID interface on this composite device
+	 * whose probe has not completed yet (shared->pen is NULL). Defer until
+	 * interface 0 finishes probing and registers shared values.
+	 */
+	return true;
+}
+
 static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 {
 	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
@@ -2444,6 +2534,11 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 	if (error)
 		goto fail;
 
+	if (wacom_sibling_pending(wacom)) {
+		error = -EPROBE_DEFER;
+		goto fail;
+	}
+
 	error = wacom_setup_inputs(wacom);
 	if (error)
 		goto fail;
@@ -2501,6 +2596,7 @@ static int wacom_parse_and_register(struct wacom *wacom, bool wireless)
 
 fail_hw_stop:
 	hid_hw_stop(hdev);
+	cancel_delayed_work_sync(&wacom->init_work);
 fail:
 	wacom_release_resources(wacom);
 	return error;
@@ -2788,16 +2884,19 @@ static void wacom_mode_change_work(struct work_struct *work)
 	bool is_direct = wacom->wacom_wac.is_direct_mode;
 	int error = 0;
 
-	if (shared->pen) {
-		wacom1 = hid_get_drvdata(shared->pen);
+	struct hid_device *pen = rcu_access_pointer(shared->pen);
+	struct hid_device *touch = rcu_access_pointer(shared->touch);
+
+	if (pen) {
+		wacom1 = hid_get_drvdata(pen);
 		wacom_release_resources(wacom1);
 		hid_hw_stop(wacom1->hdev);
 		wacom1->wacom_wac.has_mode_change = true;
 		wacom1->wacom_wac.is_direct_mode = is_direct;
 	}
 
-	if (shared->touch) {
-		wacom2 = hid_get_drvdata(shared->touch);
+	if (touch) {
+		wacom2 = hid_get_drvdata(touch);
 		wacom_release_resources(wacom2);
 		hid_hw_stop(wacom2->hdev);
 		wacom2->wacom_wac.has_mode_change = true;
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 8feb8027be95e7..c4cd87b781c451 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -510,6 +510,18 @@ static void wacom_intuos_schedule_prox_event(struct wacom_wac *wacom_wac)
 	}
 }
 
+static void wacom_report_touch_mute(struct wacom_wac *wacom_wac, bool mute)
+{
+	struct input_dev *touch_input;
+
+	guard(rcu)();
+	touch_input = rcu_dereference(wacom_wac->shared->touch_input);
+	if (touch_input) {
+		input_report_switch(touch_input, SW_MUTE_DEVICE, mute);
+		input_sync(touch_input);
+	}
+}
+
 static int wacom_intuos_pad(struct wacom_wac *wacom)
 {
 	struct wacom_features *features = &wacom->features;
@@ -650,12 +662,8 @@ static int wacom_intuos_pad(struct wacom_wac *wacom)
 	input_report_key(input, KEY_CONTROLPANEL, menu);
 	input_report_key(input, KEY_INFO, info);
 
-	if (wacom->shared && wacom->shared->touch_input) {
-		input_report_switch(wacom->shared->touch_input,
-				    SW_MUTE_DEVICE,
-				    !wacom->shared->is_touch_on);
-		input_sync(wacom->shared->touch_input);
-	}
+	if (wacom->shared)
+		wacom_report_touch_mute(wacom, !wacom->shared->is_touch_on);
 
 	input_report_abs(input, ABS_RX, strip1);
 	input_report_abs(input, ABS_RY, strip2);
@@ -2168,7 +2176,7 @@ static void wacom_wac_pad_event(struct hid_device *hdev, struct hid_field *field
 	 */
 	if ((equivalent_usage == WACOM_HID_WD_MUTE_DEVICE) ||
 	   (equivalent_usage == WACOM_HID_WD_TOUCHONOFF)) {
-		if (wacom_wac->shared->touch_input) {
+		if (wacom_wac->shared) {
 			bool *is_touch_on = &wacom_wac->shared->is_touch_on;
 
 			if (equivalent_usage == WACOM_HID_WD_MUTE_DEVICE && value)
@@ -2176,9 +2184,7 @@ static void wacom_wac_pad_event(struct hid_device *hdev, struct hid_field *field
 			else if (equivalent_usage == WACOM_HID_WD_TOUCHONOFF)
 				*is_touch_on = value;
 
-			input_report_switch(wacom_wac->shared->touch_input,
-					    SW_MUTE_DEVICE, !(*is_touch_on));
-			input_sync(wacom_wac->shared->touch_input);
+			wacom_report_touch_mute(wacom_wac, !(*is_touch_on));
 		}
 		return;
 	}
@@ -3301,6 +3307,7 @@ static int wacom_bpt_irq(struct wacom_wac *wacom, size_t len)
 static void wacom_bamboo_pad_pen_event(struct wacom_wac *wacom,
 		unsigned char *data)
 {
+	struct hid_device *pen;
 	unsigned char prefix;
 
 	/*
@@ -3313,13 +3320,11 @@ static void wacom_bamboo_pad_pen_event(struct wacom_wac *wacom,
 	prefix = data[0];
 	data[0] = WACOM_REPORT_BPAD_PEN;
 
-	/*
-	 * actually reroute the event.
-	 * No need to check if wacom->shared->pen is valid, hid_input_report()
-	 * will check for us.
-	 */
-	hid_input_report(wacom->shared->pen, HID_INPUT_REPORT, data,
-			 WACOM_PKGLEN_PENABLED, 1);
+	guard(rcu)();
+	pen = rcu_dereference(wacom->shared->pen);
+	if (pen)
+		hid_input_report(pen, HID_INPUT_REPORT, data,
+				 WACOM_PKGLEN_PENABLED, 1);
 
 	data[0] = prefix;
 }
@@ -3396,11 +3401,8 @@ static int wacom_wireless_irq(struct wacom_wac *wacom, size_t len)
 
 		if ((wacom->shared->type == INTUOSHT ||
 		    wacom->shared->type == INTUOSHT2) &&
-		    wacom->shared->touch_input &&
 		    wacom->shared->touch_max) {
-			input_report_switch(wacom->shared->touch_input,
-					SW_MUTE_DEVICE, data[5] & 0x40);
-			input_sync(wacom->shared->touch_input);
+			wacom_report_touch_mute(wacom, data[5] & 0x40);
 		}
 
 		pid = get_unaligned_be16(&data[6]);
@@ -3435,11 +3437,8 @@ static int wacom_status_irq(struct wacom_wac *wacom_wac, size_t len)
 
 	if ((features->type == INTUOSHT ||
 	    features->type == INTUOSHT2) &&
-	    wacom_wac->shared->touch_input &&
 	    features->touch_max) {
-		input_report_switch(wacom_wac->shared->touch_input,
-				    SW_MUTE_DEVICE, data[8] & 0x40);
-		input_sync(wacom_wac->shared->touch_input);
+		wacom_report_touch_mute(wacom_wac, data[8] & 0x40);
 	}
 
 	if (data[9] & 0x02) { /* wireless module is attached */
@@ -3966,8 +3965,8 @@ int wacom_setup_pen_input_capabilities(struct input_dev *input_dev,
 int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 					 struct wacom_wac *wacom_wac)
 {
+	struct hid_device *hdev = container_of(wacom_wac, struct wacom, wacom_wac)->hdev;
 	struct wacom_features *features = &wacom_wac->features;
-
 	if (!(features->device_type & WACOM_DEVICETYPE_TOUCH))
 		return -ENODEV;
 
@@ -3976,9 +3975,16 @@ int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 	else
 		__set_bit(INPUT_PROP_POINTER, input_dev->propbit);
 
-	if (features->type == HID_GENERIC)
+	if (features->type == HID_GENERIC) {
+		hid_dbg(hdev, "generic touch setup\n");
+		if (wacom_wac->has_mute_touch_switch ||
+		    (wacom_wac->shared && READ_ONCE(wacom_wac->shared->has_mute_touch_switch))) {
+			input_set_capability(input_dev, EV_SW, SW_MUTE_DEVICE);
+			wacom_wac->has_mute_touch_switch = true;
+		}
 		/* setup has already been done */
 		return 0;
+	}
 
 	input_dev->evbit[0] |= BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS);
 	__set_bit(BTN_TOUCH, input_dev->keybit);
@@ -4010,19 +4016,17 @@ int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 		input_dev->evbit[0] |= BIT_MASK(EV_SW);
 		__set_bit(SW_MUTE_DEVICE, input_dev->swbit);
 
-		if (wacom_wac->shared->touch->product == 0x361) {
+		if (hdev->product == 0x361) {
 			input_set_abs_params(input_dev, ABS_MT_POSITION_X,
 					     0, 12440, 4, 0);
 			input_set_abs_params(input_dev, ABS_MT_POSITION_Y,
 					     0, 8640, 4, 0);
-		}
-		else if (wacom_wac->shared->touch->product == 0x360) {
+		} else if (hdev->product == 0x360) {
 			input_set_abs_params(input_dev, ABS_MT_POSITION_X,
 					     0, 8960, 4, 0);
 			input_set_abs_params(input_dev, ABS_MT_POSITION_Y,
 					     0, 5920, 4, 0);
-		}
-		else if (wacom_wac->shared->touch->product == 0x393) {
+		} else if (hdev->product == 0x393) {
 			input_set_abs_params(input_dev, ABS_MT_POSITION_X,
 					     0, 6400, 4, 0);
 			input_set_abs_params(input_dev, ABS_MT_POSITION_Y,
@@ -4052,8 +4056,8 @@ int wacom_setup_touch_input_capabilities(struct input_dev *input_dev,
 		fallthrough;
 
 	case WACOM_27QHDT:
-		if (wacom_wac->shared->touch->product == 0x32C ||
-		    wacom_wac->shared->touch->product == 0xF6) {
+		if (hdev->product == 0x32C ||
+		    hdev->product == 0xF6) {
 			input_dev->evbit[0] |= BIT_MASK(EV_SW);
 			__set_bit(SW_MUTE_DEVICE, input_dev->swbit);
 			wacom_wac->has_mute_touch_switch = true;
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index 126bec6e5c0c42..170d6adbe02aa4 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -285,9 +285,9 @@ struct wacom_shared {
 	/* for wireless device to access USB interfaces */
 	unsigned touch_max;
 	int type;
-	struct input_dev *touch_input;
-	struct hid_device *pen;
-	struct hid_device *touch;
+	struct input_dev __rcu *touch_input;
+	struct hid_device __rcu *pen;
+	struct hid_device __rcu *touch;
 	bool has_mute_touch_switch;
 	bool is_touch_on;
 };
diff --git a/drivers/hsi/clients/cmt_speech.c b/drivers/hsi/clients/cmt_speech.c
index 7226677ebde7ac..801697b74d4f8b 100644
--- a/drivers/hsi/clients/cmt_speech.c
+++ b/drivers/hsi/clients/cmt_speech.c
@@ -1084,22 +1084,6 @@ static void cs_hsi_stop(struct cs_hsi_iface *hi)
 	kfree(hi);
 }
 
-static vm_fault_t cs_char_vma_fault(struct vm_fault *vmf)
-{
-	struct cs_char *csdata = vmf->vma->vm_private_data;
-	struct page *page;
-
-	page = virt_to_page((void *)csdata->mmap_base);
-	get_page(page);
-	vmf->page = page;
-
-	return 0;
-}
-
-static const struct vm_operations_struct cs_char_vm_ops = {
-	.fault	= cs_char_vma_fault,
-};
-
 static int cs_char_fasync(int fd, struct file *file, int on)
 {
 	struct cs_char *csdata = file->private_data;
@@ -1256,18 +1240,19 @@ static long cs_char_ioctl(struct file *file, unsigned int cmd,
 	return r;
 }
 
-static int cs_char_mmap(struct file *file, struct vm_area_struct *vma)
+static int cs_char_mmap_prepare(struct vm_area_desc *desc)
 {
-	if (vma->vm_end < vma->vm_start)
-		return -EINVAL;
+	struct file *file = desc->file;
+	struct cs_char *csdata = file->private_data;
+	struct page **pages = (struct page **)&desc->private_data;
 
-	if (vma_pages(vma) != 1)
+	if (vma_desc_pages(desc) != 1)
 		return -EINVAL;
 
-	vm_flags_set(vma, VM_IO | VM_DONTDUMP | VM_DONTEXPAND);
-	vma->vm_ops = &cs_char_vm_ops;
-	vma->vm_private_data = file->private_data;
+	vma_desc_set_flags(desc, VMA_DONTDUMP_BIT, VMA_DONTEXPAND_BIT);
 
+	*pages = virt_to_page((void *)csdata->mmap_base);
+	mmap_action_map_kernel_pages_full(desc, pages);
 	return 0;
 }
 
@@ -1353,7 +1338,7 @@ static const struct file_operations cs_char_fops = {
 	.write		= cs_char_write,
 	.poll		= cs_char_poll,
 	.unlocked_ioctl	= cs_char_ioctl,
-	.mmap		= cs_char_mmap,
+	.mmap_prepare	= cs_char_mmap_prepare,
 	.open		= cs_char_open,
 	.release	= cs_char_release,
 	.fasync		= cs_char_fasync,
diff --git a/drivers/hte/hte-tegra194-test.c b/drivers/hte/hte-tegra194-test.c
index 32907d951ec9ab..cca33868fb03dc 100644
--- a/drivers/hte/hte-tegra194-test.c
+++ b/drivers/hte/hte-tegra194-test.c
@@ -217,10 +217,10 @@ static void tegra_hte_test_remove(struct platform_device *pdev)
 {
 	(void)pdev;
 
+	timer_shutdown_sync(&hte.timer);
 	free_irq(hte.gpio_in_irq, &hte);
 	gpiod_put(hte.gpio_in);
 	gpiod_put(hte.gpio_out);
-	timer_delete_sync(&hte.timer);
 }
 
 static struct platform_driver tegra_hte_test_driver = {
diff --git a/drivers/hte/hte-tegra194.c b/drivers/hte/hte-tegra194.c
index 4a7702b32b24fe..189f7fcffbe2a6 100644
--- a/drivers/hte/hte-tegra194.c
+++ b/drivers/hte/hte-tegra194.c
@@ -856,10 +856,8 @@ static int tegra_hte_probe(struct platform_device *pdev)
 	hte_dev->hte_irq = ret;
 	ret = devm_request_irq(dev, hte_dev->hte_irq, tegra_hte_isr, 0,
 			       dev_name(dev), hte_dev);
-	if (ret < 0) {
-		dev_err(dev, "request irq failed.\n");
+	if (ret < 0)
 		return ret;
-	}
 
 	gc->nlines = nlines;
 	gc->ops = &g_ops;
@@ -910,17 +908,17 @@ static int tegra_hte_probe(struct platform_device *pdev)
 
 	hte_dev->chip = gc;
 
+	for (i = 0; i < slices; i++) {
+		hte_dev->sl[i].flags = 0;
+		spin_lock_init(&hte_dev->sl[i].s_lock);
+	}
+
 	ret = devm_hte_register_chip(hte_dev->chip);
 	if (ret) {
 		dev_err(gc->dev, "hte chip register failed");
 		return ret;
 	}
 
-	for (i = 0; i < slices; i++) {
-		hte_dev->sl[i].flags = 0;
-		spin_lock_init(&hte_dev->sl[i].s_lock);
-	}
-
 	val = HTE_TECTRL_ENABLE_ENABLE |
 	      (HTE_TECTRL_INTR_ENABLE << HTE_TECTRL_INTR_SHIFT) |
 	      (hte_dev->itr_thrshld << HTE_TECTRL_OCCU_SHIFT);
diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig
index aa11bcefddf2c0..36445c2619ed1c 100644
--- a/drivers/hv/Kconfig
+++ b/drivers/hv/Kconfig
@@ -4,8 +4,7 @@ menu "Microsoft Hyper-V guest support"
 
 config HYPERV
 	bool "Microsoft Hyper-V core hypervisor support"
-	depends on (X86 && X86_LOCAL_APIC && HYPERVISOR_GUEST) \
-		|| (ARM64 && !CPU_BIG_ENDIAN)
+	depends on (X86 && X86_LOCAL_APIC && HYPERVISOR_GUEST) || ARM64
 	select PARAVIRT
 	select X86_HV_CALLBACK_VECTOR if X86
 	select OF_EARLY_FLATTREE if OF
diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c
index 7e4cc6f55237a8..a0883ddf32e2fe 100644
--- a/drivers/hv/channel.c
+++ b/drivers/hv/channel.c
@@ -79,7 +79,7 @@ static inline u32 hv_ring_gpadl_send_hvpgoffset(u32 offset)
 	 * therefore leaving a (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap.
 	 *
 	 * And to calculate the effective send offset in gpadl, we need to
-	 * substract this gap.
+	 * subtract this gap.
 	 */
 	return (offset - (PAGE_SIZE - HV_HYP_PAGE_SIZE)) >> HV_HYP_PAGE_SHIFT;
 }
diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
index a044fd3b3c4e7d..c42def04ac417b 100644
--- a/drivers/hv/channel_mgmt.c
+++ b/drivers/hv/channel_mgmt.c
@@ -752,13 +752,14 @@ static void init_vp_index(struct vmbus_channel *channel)
 	u32 i, ncpu = num_online_cpus();
 	cpumask_var_t available_mask;
 	struct cpumask *allocated_mask;
+	const struct cpumask *node_mask;
 	const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
 	u32 target_cpu;
 	int numa_node;
 
 	if (!perf_chn ||
 	    !alloc_cpumask_var(&available_mask, GFP_KERNEL) ||
-	    cpumask_empty(hk_mask)) {
+	    !cpumask_intersects(hk_mask, cpu_online_mask)) {
 		/*
 		 * If the channel is not a performance critical
 		 * channel, bind it to VMBUS_CONNECT_CPU.
@@ -780,14 +781,19 @@ static void init_vp_index(struct vmbus_channel *channel)
 				next_numa_node_id = 0;
 				continue;
 			}
-			if (cpumask_empty(cpumask_of_node(numa_node)))
+			/*
+			 * Try next NUMA node if current NUMA node has no CPU
+			 * or doesn't contain any housekeeping CPU.
+			 */
+			node_mask = cpumask_of_node(numa_node);
+			if (!cpumask_intersects(node_mask, hk_mask))
 				continue;
 			break;
 		}
 		allocated_mask = &hv_context.hv_numa_map[numa_node];
 
 retry:
-		cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node));
+		cpumask_xor(available_mask, allocated_mask, node_mask);
 		cpumask_and(available_mask, available_mask, hk_mask);
 
 		if (cpumask_empty(available_mask)) {
diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c
index 1ab3581b096a4c..818667ef9c74dc 100644
--- a/drivers/hv/connection.c
+++ b/drivers/hv/connection.c
@@ -346,7 +346,6 @@ int vmbus_connect(void)
 cleanup:
 	pr_err("Unable to connect to host\n");
 
-	vmbus_connection.conn_state = DISCONNECTED;
 	vmbus_disconnect();
 
 	kfree(msginfo);
diff --git a/drivers/hv/mshv_vtl_main.c b/drivers/hv/mshv_vtl_main.c
index 6e3c11c681717b..d8225242c73f83 100644
--- a/drivers/hv/mshv_vtl_main.c
+++ b/drivers/hv/mshv_vtl_main.c
@@ -819,6 +819,9 @@ static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf)
 		return VM_FAULT_NOPAGE;
 	}
 
+	if (!page)
+		return VM_FAULT_SIGBUS;
+
 	get_page(page);
 	vmf->page = page;
 
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index 5ebdbe24b5a1e7..4692687cdbf3b5 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -2923,23 +2923,19 @@ static void hv_crash_handler(struct pt_regs *regs)
 static int hv_synic_suspend(void *data)
 {
 	/*
-	 * When we reach here, all the non-boot CPUs have been offlined.
-	 * If we're in a legacy configuration where stimer Direct Mode is
-	 * not enabled, the stimers on the non-boot CPUs have been unbound
-	 * in hv_synic_cleanup() -> hv_stimer_legacy_cleanup() ->
-	 * hv_stimer_cleanup() -> clockevents_unbind_device().
+	 * When we reach here, all the non-boot CPUs have been offlined,
+	 * and their stimers have been cleaned up by the cpuhp teardown
+	 * callback hv_stimer_cleanup().
 	 *
 	 * hv_synic_suspend() only runs on CPU0 with interrupts disabled.
-	 * Here we do not call hv_stimer_legacy_cleanup() on CPU0 because:
-	 * 1) it's unnecessary as interrupts remain disabled between
-	 * syscore_suspend() and syscore_resume(): see create_image() and
+	 * The stimer on CPU0 is not explicitly cleaned up here because:
+	 * 1) it's harmless as interrupts remain disabled between
+	 * syscore_suspend() and syscore_resume(), so the stimer cannot
+	 * fire during this window: see create_image() and
 	 * resume_target_kernel()
 	 * 2) the stimer on CPU0 is automatically disabled later by
 	 * syscore_suspend() -> timekeeping_suspend() -> tick_suspend() -> ...
 	 * -> clockevents_shutdown() -> ... -> hv_ce_shutdown()
-	 * 3) a warning would be triggered if we call
-	 * clockevents_unbind_device(), which may sleep, in an
-	 * interrupts-disabled context.
 	 */
 
 	hv_hyp_synic_disable_regs(0);
@@ -3043,7 +3039,6 @@ static void __exit vmbus_exit(void)
 
 	hv_remove_kexec_handler();
 	hv_remove_crash_handler();
-	vmbus_connection.conn_state = DISCONNECTED;
 	hv_stimer_global_cleanup();
 	vmbus_disconnect();
 	if (vmbus_irq == -1)
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index fecff8610ea8ba..18c682e1a4756a 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -311,6 +311,29 @@ config SENSORS_AXI_FAN_CONTROL
 	  This driver can also be built as a module. If so, the module
 	  will be called axi-fan-control
 
+config SENSORS_AXIADO_PWM_FAN
+	tristate "Axiado PWM fan controller"
+	depends on ARCH_AXIADO || COMPILE_TEST
+	depends on PWM
+	help
+	  This driver supports the Axiado PWM fan controller. It uses a PWM
+	  output to control fan speed and a tachometer interrupt to report fan
+	  speed as RPM through the hardware monitoring sysfs interface.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called axiado-pwm-fan.
+
+config SENSORS_AXIADO_TSADC
+	tristate "Axiado AX3000 TSADC temperature sensor"
+	depends on ARCH_AXIADO || COMPILE_TEST
+	help
+	  If you say yes here you get support for the on-chip temperature
+	  sensor (TSADC) found on Axiado AX3000 and AX3005 SoCs. The die
+	  temperature is reported through the hwmon sysfs interface.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called axiado-tsadc.
+
 config SENSORS_K8TEMP
 	tristate "AMD Athlon64/FX or Opteron temperature sensor"
 	depends on X86 && PCI
@@ -1490,6 +1513,22 @@ config SENSORS_MENF21BMC_HWMON
 	  This driver can also be built as a module. If so the module
 	  will be called menf21bmc_hwmon.
 
+config SENSORS_MINISFORUM_UM780XTX
+	tristate "Minisforum UM780 XTX EC hardware monitoring"
+	depends on ACPI_EC && DMI && X86
+	help
+	  If you say yes here you get support for the hardware monitoring
+	  features of the embedded controller in the Minisforum UM780 XTX.
+	  This includes CPU and system fan speeds, their control temperatures,
+	  two CPU fan profiles, system fan temperature thresholds, and
+	  preservation of the selected settings across suspend and resume.
+
+	  This driver only binds to explicitly supported board and firmware
+	  versions.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called minisforum-um780xtx.
+
 config SENSORS_MR75203
 	tristate "Moortec Semiconductor MR75203 PVT Controller"
 	select REGMAP_MMIO
@@ -1521,10 +1560,10 @@ config SENSORS_LM63
 	depends on I2C
 	help
 	  If you say yes here you get support for the National
-	  Semiconductor LM63, LM64, and LM96163 remote diode digital temperature
-	  sensors with integrated fan control.  Such chips are found
-	  on the Tyan S4882 (Thunder K8QS Pro) motherboard, among
-	  others.
+	  Semiconductor LM63, LM64, LM96163, and Sensylink CTF2301
+	  remote diode digital temperature sensors with integrated fan
+	  control. Such chips are found on the Tyan S4882 (Thunder K8QS
+	  Pro) motherboard, among others.
 
 	  This driver can also be built as a module. If so, the module
 	  will be called lm63.
@@ -2827,6 +2866,16 @@ config SENSORS_ASUS_EC
 	  This driver can also be built as a module. If so, the module
 	  will be called asus_ec_sensors.
 
+config SENSORS_HONOR_FMI
+	tristate "HONOR FMI-XX fan monitor"
+	depends on X86 && ACPI
+	help
+	  If you say yes here, you get support for fan speed monitoring on
+	  the HONOR FMI-XX laptop through its firmware ACPI method.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called honor-fmi.
+
 config SENSORS_HP_WMI
 	tristate "HP WMI Sensors"
 	depends on ACPI_WMI
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 1229b6b3996d79..d4c345ed7261f9 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_SENSORS_ACPI_POWER) += acpi_power_meter.o
 obj-$(CONFIG_SENSORS_ATK0110)	+= asus_atk0110.o
 obj-$(CONFIG_SENSORS_ASUS_EC)	+= asus-ec-sensors.o
 obj-$(CONFIG_SENSORS_ASUS_WMI)	+= asus_wmi_sensors.o
+obj-$(CONFIG_SENSORS_HONOR_FMI)	+= honor-fmi.o
 obj-$(CONFIG_SENSORS_HP_WMI)	+= hp-wmi-sensors.o
 
 # Native drivers
@@ -59,6 +60,8 @@ obj-$(CONFIG_SENSORS_ASPEED_G6) += aspeed-g6-pwm-tach.o
 obj-$(CONFIG_SENSORS_ASUS_ROG_RYUJIN)	+= asus_rog_ryujin.o
 obj-$(CONFIG_SENSORS_ATXP1)	+= atxp1.o
 obj-$(CONFIG_SENSORS_AXI_FAN_CONTROL) += axi-fan-control.o
+obj-$(CONFIG_SENSORS_AXIADO_PWM_FAN) += axiado-pwm-fan.o
+obj-$(CONFIG_SENSORS_AXIADO_TSADC)	+= axiado-tsadc.o
 obj-$(CONFIG_SENSORS_CGBC)	+= cgbc-hwmon.o
 obj-$(CONFIG_SENSORS_CHIPCAP2) += chipcap2.o
 obj-$(CONFIG_SENSORS_CORETEMP)	+= coretemp.o
@@ -180,6 +183,7 @@ obj-$(CONFIG_SENSORS_TC654)	+= tc654.o
 obj-$(CONFIG_SENSORS_TPS23861)	+= tps23861.o
 obj-$(CONFIG_SENSORS_MLXREG_FAN) += mlxreg-fan.o
 obj-$(CONFIG_SENSORS_MENF21BMC_HWMON) += menf21bmc_hwmon.o
+obj-$(CONFIG_SENSORS_MINISFORUM_UM780XTX) += minisforum-um780xtx.o
 obj-$(CONFIG_SENSORS_MR75203)	+= mr75203.o
 obj-$(CONFIG_SENSORS_NCT6683)	+= nct6683.o
 obj-$(CONFIG_SENSORS_NCT6694)	+= nct6694-hwmon.o
diff --git a/drivers/hwmon/abituguru.c b/drivers/hwmon/abituguru.c
index ba8c68ae45953b..3bb959d16dbf53 100644
--- a/drivers/hwmon/abituguru.c
+++ b/drivers/hwmon/abituguru.c
@@ -475,7 +475,7 @@ abituguru_detect_bank1_sensor_type(struct abituguru_data *data,
 	u8 val, test_flag, buf[3];
 	int i, ret = -ENODEV; /* error is the most common used retval :| */
 
-	/* If overriden by the user return the user selected type */
+	/* If overridden by the user return the user selected type */
 	if (bank1_types[sensor_addr] >= ABIT_UGURU_IN_SENSOR &&
 			bank1_types[sensor_addr] <= ABIT_UGURU_NC) {
 		ABIT_UGURU_DEBUG(2, "assuming sensor type %d for bank1 sensor "
@@ -1262,7 +1262,7 @@ static int abituguru_probe(struct platform_device *pdev)
 
 	/*
 	 * El weirdo probe order, to keep the sysfs order identical to the
-	 * BIOS and window-appliction listing order.
+	 * BIOS and window-application listing order.
 	 */
 	static const u8 probe_order[ABIT_UGURU_MAX_BANK1_SENSORS] = {
 		0x00, 0x01, 0x03, 0x04, 0x0A, 0x08, 0x0E, 0x02,
diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c
index 8a539e8d1334b3..d8e8715094055d 100644
--- a/drivers/hwmon/acpi_power_meter.c
+++ b/drivers/hwmon/acpi_power_meter.c
@@ -54,8 +54,8 @@ static int can_cap_in_hardware(void)
 }
 
 static const struct acpi_device_id power_meter_ids[] = {
-	{"ACPI000D", 0},
-	{"", 0},
+	{ .id = "ACPI000D" },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, power_meter_ids);
 
diff --git a/drivers/hwmon/adm1177.c b/drivers/hwmon/adm1177.c
index dc4d8a214d1b48..85ad4a3009bd46 100644
--- a/drivers/hwmon/adm1177.c
+++ b/drivers/hwmon/adm1177.c
@@ -105,7 +105,7 @@ static int adm1177_read(struct device *dev, enum hwmon_sensor_types type,
 			return ret;
 		dummy = (data[0] << 4) | (data[2] >> 4);
 		/*
-		 * convert to millivolts based on resistor devision
+		 * convert to millivolts based on resistor division
 		 * (V_fullscale / 4096) * raw
 		 */
 		if (st->vrange_high)
diff --git a/drivers/hwmon/adt7x10.c b/drivers/hwmon/adt7x10.c
index d003ee3ebf062e..f99d38e82d0bb4 100644
--- a/drivers/hwmon/adt7x10.c
+++ b/drivers/hwmon/adt7x10.c
@@ -338,7 +338,7 @@ int adt7x10_probe(struct device *dev, const char *name, int irq,
 	data->oldconfig = config;
 
 	/*
-	 * Set to 16 bit resolution, continous conversion and comparator mode.
+	 * Set to 16 bit resolution, continuous conversion and comparator mode.
 	 */
 	data->config = data->oldconfig;
 	data->config &= ~(ADT7X10_MODE_MASK | ADT7X10_CT_POLARITY |
diff --git a/drivers/hwmon/arctic_fan_controller.c b/drivers/hwmon/arctic_fan_controller.c
index dbe84cd93c0837..9eb83a149cd63d 100644
--- a/drivers/hwmon/arctic_fan_controller.c
+++ b/drivers/hwmon/arctic_fan_controller.c
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
+// SPDX-License-Identifier: GPL-2.0-or-later OR BSD-2-Clause
 /*
  * Linux hwmon driver for ARCTIC Fan Controller
  *
@@ -35,6 +35,8 @@
  * Measured over 500 iterations: max ~563 ms. Keep 1 s as margin.
  */
 #define ARCTIC_ACK_TIMEOUT_MS		1000
+/* MCU factory default; 40% of 0-255 is 102. */
+#define ARCTIC_PWM_DEFAULT		102
 
 struct arctic_fan_data {
 	struct hid_device *hdev;
@@ -164,7 +166,7 @@ static int arctic_fan_write(struct device *dev, enum hwmon_sensor_types type,
 
 	/*
 	 * Build the buffer and arm write_pending under in_report_lock so that
-	 * reset_resume() cannot clear pwm_duty[] between the pwm_duty[] read
+	 * reset_resume() cannot replace pwm_duty[] between the pwm_duty[] read
 	 * and the buffer write, and raw_event() cannot deliver a stale ACK
 	 * from a previous write into this write's completion.
 	 *
@@ -256,14 +258,14 @@ static int arctic_fan_reset_resume(struct hid_device *hdev)
 	unsigned long flags;
 
 	/*
-	 * The device resets its PWM channels to hardware defaults on power
-	 * loss during suspend. Clear the cached duty values so they reflect
-	 * the unknown hardware state, consistent with probe-time behaviour
-	 * (the device has no GET_REPORT support). Hold in_report_lock so
-	 * this does not race with a concurrent pwm read or write callback.
+	 * The device resets its PWM channels to the MCU factory default
+	 * (40%) on power loss during suspend. Restore the cache to that
+	 * same default, consistent with probe-time behaviour (the device
+	 * has no GET_REPORT support). Hold in_report_lock so this does
+	 * not race with a concurrent pwm read or write callback.
 	 */
 	spin_lock_irqsave(&priv->in_report_lock, flags);
-	memset(priv->pwm_duty, 0, sizeof(priv->pwm_duty));
+	memset(priv->pwm_duty, ARCTIC_PWM_DEFAULT, sizeof(priv->pwm_duty));
 	spin_unlock_irqrestore(&priv->in_report_lock, flags);
 	return 0;
 }
@@ -288,6 +290,8 @@ static int arctic_fan_probe(struct hid_device *hdev,
 	priv->hdev = hdev;
 	spin_lock_init(&priv->in_report_lock);
 	init_completion(&priv->in_report_received);
+	/* Same MCU factory default as reset_resume(); see ARCTIC_PWM_DEFAULT above. */
+	memset(priv->pwm_duty, ARCTIC_PWM_DEFAULT, sizeof(priv->pwm_duty));
 	hid_set_drvdata(hdev, priv);
 
 	ret = hid_hw_start(hdev, HID_CONNECT_DRIVER);
@@ -371,4 +375,4 @@ module_hid_driver(arctic_fan_driver);
 
 MODULE_AUTHOR("Aureo Serrano de Souza <aureo.serrano@arctic.de>");
 MODULE_DESCRIPTION("HID hwmon driver for ARCTIC Fan Controller");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c
index e43645e884fdb6..19f423e45ce5e4 100644
--- a/drivers/hwmon/asus-ec-sensors.c
+++ b/drivers/hwmon/asus-ec-sensors.c
@@ -962,6 +962,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_strix_x570_f_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-I GAMING",
 					&board_info_strix_x570_i_gaming),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X670E-A GAMING WIFI",
+					&board_info_strix_x670e_e_gaming_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X670E-E GAMING WIFI",
 					&board_info_strix_x670e_e_gaming_wifi),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X670E-I GAMING WIFI",
@@ -980,6 +982,8 @@ static const struct dmi_system_id dmi_table[] = {
 					&board_info_strix_z390_f_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z390-F GAMING",
 					&board_info_strix_z390_f_gaming),
+	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z490-A GAMING",
+					&board_info_strix_z490_f_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z490-F GAMING",
 					&board_info_strix_z490_f_gaming),
 	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z690-A GAMING WIFI D4",
diff --git a/drivers/hwmon/asus_atk0110.c b/drivers/hwmon/asus_atk0110.c
index 92afb64c09df8d..868c4e14d26acf 100644
--- a/drivers/hwmon/asus_atk0110.c
+++ b/drivers/hwmon/asus_atk0110.c
@@ -141,8 +141,8 @@ typedef ssize_t (*sysfs_show_func)(struct device *dev,
 			struct device_attribute *attr, char *buf);
 
 static const struct acpi_device_id atk_ids[] = {
-	{ATK_HID, 0},
-	{"", 0},
+	{ .id = ATK_HID },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, atk_ids);
 
diff --git a/drivers/hwmon/asus_rog_ryujin.c b/drivers/hwmon/asus_rog_ryujin.c
index e297557ca34638..5ced69e94ead45 100644
--- a/drivers/hwmon/asus_rog_ryujin.c
+++ b/drivers/hwmon/asus_rog_ryujin.c
@@ -20,6 +20,7 @@
 #define USB_PRODUCT_ID_RYUJIN_III_EXTREME	0x1bcb
 #define USB_PRODUCT_ID_RYUJIN_III_EVA		0x1ade
 #define USB_PRODUCT_ID_RYUJIN_III_WHITE		0x1ada
+#define USB_PRODUCT_ID_RYUJIN_III		0x1aa2
 
 struct rog_ryujin_device_info {
 	u8 temp_offset;
@@ -613,6 +614,8 @@ static const struct hid_device_id rog_ryujin_table[] = {
 	  .driver_data = (kernel_ulong_t)&rog_ryujin_iii_info },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUS_ROG, USB_PRODUCT_ID_RYUJIN_III_WHITE),
 	  .driver_data = (kernel_ulong_t)&rog_ryujin_iii_info },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUS_ROG, USB_PRODUCT_ID_RYUJIN_III),
+	  .driver_data = (kernel_ulong_t)&rog_ryujin_iii_info },
 	{ }
 };
 
diff --git a/drivers/hwmon/asus_wmi_sensors.c b/drivers/hwmon/asus_wmi_sensors.c
index c2dd7ff882f24b..621e3506972d47 100644
--- a/drivers/hwmon/asus_wmi_sensors.c
+++ b/drivers/hwmon/asus_wmi_sensors.c
@@ -573,7 +573,7 @@ static int asus_wmi_configure_sensor_setup(struct device *dev,
 
 		sensor_data->wmi.info[type] = devm_kcalloc(dev,
 							   nr_count[type],
-							   sizeof(*sensor_data->wmi.info),
+							   sizeof(*sensor_data->wmi.info[type]),
 							   GFP_KERNEL);
 		if (!sensor_data->wmi.info[type])
 			return -ENOMEM;
diff --git a/drivers/hwmon/axiado-pwm-fan.c b/drivers/hwmon/axiado-pwm-fan.c
new file mode 100644
index 00000000000000..6fca5292eb25b8
--- /dev/null
+++ b/drivers/hwmon/axiado-pwm-fan.c
@@ -0,0 +1,416 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2021-2026 Axiado Corporation.
+ */
+
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/hwmon.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/pwm.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#define AX_TACH_PWM_VAL_MAX	255
+
+/* TACH Register offsets */
+#define AX_TACH_CTRL_REG	0x00
+#define AX_TACH_TIMER_COUNT_REG	0x04
+#define AX_TACH_COUNT_REG	0x08
+#define AX_TACH_INT_STATUS_REG	0x0c
+
+#define AX_TACH_CTRL_ENABLE	BIT(0)
+#define AX_TACH_CTRL_INT_ENABLE	BIT(1)
+
+#define AX_TACH_INT_PENDING	BIT(0)
+
+struct axiado_pwm_fan_tach {
+	int irq;
+	u32 pulses_per_revolution;
+	u32 timer_count;
+	u32 count;
+};
+
+struct axiado_pwm_fan_ctx {
+	/* Serializes PWM updates with PM and shutdown operations. */
+	struct mutex pwm_lock;
+	/* Protects tachometer count updated from interrupt context. */
+	spinlock_t tach_lock;
+	struct pwm_device *pwm;
+	struct pwm_state pwm_state;
+	void __iomem *tach_base;
+	struct axiado_pwm_fan_tach tach;
+	unsigned int pwm_value;
+};
+
+static const struct hwmon_channel_info * const pwm_fan_info[] = {
+	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT),
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT),
+	NULL
+};
+
+static irqreturn_t axiado_tach_irq_handler(int irq, void *dev)
+{
+	struct axiado_pwm_fan_ctx *ctx = dev;
+	u32 status;
+
+	status = ioread32(ctx->tach_base + AX_TACH_INT_STATUS_REG);
+
+	if (!(status & AX_TACH_INT_PENDING))
+		return IRQ_NONE;
+
+	scoped_guard(spinlock_irqsave, &ctx->tach_lock) {
+		ctx->tach.count = ioread32(ctx->tach_base + AX_TACH_COUNT_REG);
+	}
+
+	iowrite32(AX_TACH_INT_PENDING, ctx->tach_base + AX_TACH_INT_STATUS_REG);
+
+	return IRQ_HANDLED;
+}
+
+static int axiado_set_pwm(struct axiado_pwm_fan_ctx *ctx, unsigned int pwm)
+{
+	struct pwm_state state;
+	int ret;
+
+	guard(mutex)(&ctx->pwm_lock);
+
+	if (ctx->pwm_value == pwm)
+		return 0;
+
+	state = ctx->pwm_state;
+	state.duty_cycle = DIV_ROUND_UP_ULL((u64)pwm * state.period,
+					    AX_TACH_PWM_VAL_MAX);
+	/*
+	 * Keep the PWM enabled even at zero duty cycle so the output stays
+	 * driven low instead of being left floating by the PWM hardware.
+	 */
+	state.enabled = true;
+
+	ret = pwm_apply_might_sleep(ctx->pwm, &state);
+	if (ret)
+		return ret;
+
+	ctx->pwm_state = state;
+	ctx->pwm_value = pwm;
+
+	return 0;
+}
+
+static int axiado_pwm_fan_write(struct device *dev,
+				enum hwmon_sensor_types type, u32 attr,
+				int channel, long val)
+{
+	struct axiado_pwm_fan_ctx *ctx = dev_get_drvdata(dev);
+
+	if (type != hwmon_pwm || attr != hwmon_pwm_input)
+		return -EOPNOTSUPP;
+
+	if (val < 0 || val > AX_TACH_PWM_VAL_MAX)
+		return -EINVAL;
+
+	return axiado_set_pwm(ctx, val);
+}
+
+static unsigned int axiado_tach_get_rpm(struct axiado_pwm_fan_ctx *ctx)
+{
+	u32 pulses_per_revolution = ctx->tach.pulses_per_revolution;
+	u64 pulses, rpm;
+
+	scoped_guard(spinlock_irqsave, &ctx->tach_lock)
+		pulses = ctx->tach.count;
+
+	if (!pulses_per_revolution)
+		return 0;
+
+	rpm = pulses * 60;
+	do_div(rpm, pulses_per_revolution);
+
+	return rpm;
+}
+
+static int axiado_pwm_fan_read(struct device *dev, enum hwmon_sensor_types type,
+			       u32 attr, int channel, long *val)
+{
+	struct axiado_pwm_fan_ctx *ctx = dev_get_drvdata(dev);
+
+	switch (type) {
+	case hwmon_pwm:
+		if (attr != hwmon_pwm_input)
+			return -EOPNOTSUPP;
+
+		*val = ctx->pwm_value;
+
+		return 0;
+
+	case hwmon_fan:
+		if (attr != hwmon_fan_input)
+			return -EOPNOTSUPP;
+
+		*val = axiado_tach_get_rpm(ctx);
+
+		return 0;
+
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static umode_t axiado_pwm_fan_is_visible(const void *data,
+					 enum hwmon_sensor_types type, u32 attr,
+					 int channel)
+{
+	if (type == hwmon_fan && attr == hwmon_fan_input)
+		return 0444;
+
+	if (type == hwmon_pwm && attr == hwmon_pwm_input)
+		return 0644;
+
+	return 0;
+}
+
+static const struct hwmon_ops pwm_fan_hwmon_ops = {
+	.is_visible = axiado_pwm_fan_is_visible,
+	.read = axiado_pwm_fan_read,
+	.write = axiado_pwm_fan_write,
+};
+
+static const struct hwmon_chip_info pwm_fan_chip_info = {
+	.ops = &pwm_fan_hwmon_ops,
+	.info = pwm_fan_info,
+};
+
+static int axiado_pwm_apply_disabled(struct axiado_pwm_fan_ctx *ctx)
+{
+	struct pwm_state state;
+
+	guard(mutex)(&ctx->pwm_lock);
+
+	if (!ctx->pwm_state.enabled)
+		return 0;
+
+	state = ctx->pwm_state;
+	state.duty_cycle = 0;
+	state.enabled = false;
+
+	return pwm_apply_might_sleep(ctx->pwm, &state);
+}
+
+static void axiado_pwm_disable(void *data)
+{
+	struct axiado_pwm_fan_ctx *ctx = data;
+
+	axiado_pwm_apply_disabled(ctx);
+}
+
+static void axiado_tach_enable(struct axiado_pwm_fan_ctx *ctx)
+{
+	iowrite32(AX_TACH_INT_PENDING,
+		  ctx->tach_base + AX_TACH_INT_STATUS_REG);
+	iowrite32(ctx->tach.timer_count,
+		  ctx->tach_base + AX_TACH_TIMER_COUNT_REG);
+	iowrite32(AX_TACH_CTRL_ENABLE | AX_TACH_CTRL_INT_ENABLE,
+		  ctx->tach_base + AX_TACH_CTRL_REG);
+}
+
+static void axiado_tach_disable(void *data)
+{
+	struct axiado_pwm_fan_ctx *ctx = data;
+
+	iowrite32(0, ctx->tach_base + AX_TACH_CTRL_REG);
+	iowrite32(AX_TACH_INT_PENDING,
+		  ctx->tach_base + AX_TACH_INT_STATUS_REG);
+}
+
+static int axiado_pwm_fan_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct fwnode_handle *fan_node;
+	struct axiado_pwm_fan_ctx *ctx;
+	unsigned long tach_clk_rate;
+	struct device *hwmon;
+	struct clk *tach_clk;
+	int ret;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	mutex_init(&ctx->pwm_lock);
+	spin_lock_init(&ctx->tach_lock);
+
+	ctx->tach_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ctx->tach_base))
+		return PTR_ERR(ctx->tach_base);
+
+	fan_node = device_get_named_child_node(dev, "fan");
+	if (!fan_node)
+		return dev_err_probe(dev, -EINVAL,
+				     "Missing fan child node\n");
+
+	ctx->pwm = devm_fwnode_pwm_get(dev, fan_node, NULL);
+	if (IS_ERR(ctx->pwm)) {
+		ret = dev_err_probe(dev, PTR_ERR(ctx->pwm),
+				    "Could not get PWM\n");
+		goto put_fan_node;
+	}
+
+	ctx->tach.pulses_per_revolution = 2;
+
+	ret = fwnode_property_read_u32(fan_node, "pulses-per-revolution",
+				       &ctx->tach.pulses_per_revolution);
+	if (ret && ret != -EINVAL) {
+		ret = dev_err_probe(dev, ret,
+				    "Failed to read pulses-per-revolution\n");
+		goto put_fan_node;
+	}
+
+	if (!ctx->tach.pulses_per_revolution) {
+		ret = dev_err_probe(dev, -EINVAL,
+				    "pulses-per-revolution cannot be zero\n");
+		goto put_fan_node;
+	}
+
+	fwnode_handle_put(fan_node);
+
+	platform_set_drvdata(pdev, ctx);
+
+	pwm_init_state(ctx->pwm, &ctx->pwm_state);
+
+	if (!ctx->pwm_state.period)
+		return dev_err_probe(dev, -EINVAL, "PWM period is zero\n");
+
+	tach_clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(tach_clk))
+		return dev_err_probe(dev, PTR_ERR(tach_clk),
+				     "Failed to get tachometer clock\n");
+
+	tach_clk_rate = clk_get_rate(tach_clk);
+	if (!tach_clk_rate || tach_clk_rate > U32_MAX)
+		return dev_err_probe(dev, -EINVAL,
+				     "Invalid tachometer clock rate: %lu\n",
+				     tach_clk_rate);
+
+	ctx->tach.timer_count = tach_clk_rate;
+
+	ctx->tach.irq = platform_get_irq(pdev, 0);
+	if (ctx->tach.irq < 0)
+		return dev_err_probe(dev, ctx->tach.irq,
+				     "Failed to get tachometer IRQ\n");
+
+	ret = devm_request_irq(dev, ctx->tach.irq, axiado_tach_irq_handler, 0,
+			       dev_name(dev), ctx);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to request tach IRQ\n");
+
+	ret = devm_add_action_or_reset(dev, axiado_tach_disable, ctx);
+	if (ret)
+		return ret;
+
+	dev_dbg(dev, "Fan tachometer: irq=%d, pulses_per_revolution=%u\n",
+		ctx->tach.irq, ctx->tach.pulses_per_revolution);
+
+	axiado_tach_enable(ctx);
+
+	ret = axiado_set_pwm(ctx, AX_TACH_PWM_VAL_MAX);
+
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to configure PWM\n");
+
+	ret = devm_add_action_or_reset(dev, axiado_pwm_disable, ctx);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to add PWM disable action\n");
+
+	hwmon = devm_hwmon_device_register_with_info(dev, "axpwmfan", ctx,
+						     &pwm_fan_chip_info, NULL);
+	if (IS_ERR(hwmon))
+		return dev_err_probe(dev, PTR_ERR(hwmon),
+				     "Failed to register hwmon device\n");
+
+	return 0;
+
+put_fan_node:
+	fwnode_handle_put(fan_node);
+
+	return ret;
+}
+
+static int axiado_pwm_fan_disable(struct device *dev)
+{
+	struct axiado_pwm_fan_ctx *ctx = dev_get_drvdata(dev);
+	int ret;
+
+	ret = axiado_pwm_apply_disabled(ctx);
+	if (ret)
+		return ret;
+
+	axiado_tach_disable(ctx);
+	synchronize_irq(ctx->tach.irq);
+
+	return 0;
+}
+
+static void axiado_pwm_fan_shutdown(struct platform_device *pdev)
+{
+	struct axiado_pwm_fan_ctx *ctx = platform_get_drvdata(pdev);
+
+	/* Best effort during shutdown. */
+	axiado_pwm_apply_disabled(ctx);
+
+	axiado_tach_disable(ctx);
+	synchronize_irq(ctx->tach.irq);
+}
+
+static int axiado_pwm_fan_suspend(struct device *dev)
+{
+	return axiado_pwm_fan_disable(dev);
+}
+
+static int axiado_pwm_fan_resume(struct device *dev)
+{
+	struct axiado_pwm_fan_ctx *ctx = dev_get_drvdata(dev);
+	int ret;
+
+	axiado_tach_enable(ctx);
+
+	scoped_guard(mutex, &ctx->pwm_lock)
+		ret = pwm_apply_might_sleep(ctx->pwm, &ctx->pwm_state);
+
+	if (ret)
+		axiado_tach_disable(ctx);
+
+	return ret;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(axiado_pwm_fan_pm,
+				axiado_pwm_fan_suspend,
+				axiado_pwm_fan_resume);
+
+static const struct of_device_id axiado_pwm_fan_match[] = {
+	{ .compatible = "axiado,ax3000-pwm-fan" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, axiado_pwm_fan_match);
+
+static struct platform_driver axiado_pwm_fan_driver = {
+	.probe		= axiado_pwm_fan_probe,
+	.shutdown	= axiado_pwm_fan_shutdown,
+	.driver	= {
+		.name		= "axiado-pwm-fan",
+		.pm		= pm_sleep_ptr(&axiado_pwm_fan_pm),
+		.of_match_table	= axiado_pwm_fan_match,
+	},
+};
+module_platform_driver(axiado_pwm_fan_driver);
+
+MODULE_AUTHOR("Axiado Corporation");
+MODULE_DESCRIPTION("Axiado PWM fan controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/axiado-tsadc.c b/drivers/hwmon/axiado-tsadc.c
new file mode 100644
index 00000000000000..c111df6f00447f
--- /dev/null
+++ b/drivers/hwmon/axiado-tsadc.c
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021-2026 Axiado Corporation
+ *
+ * Driver for the Axiado on-chip temperature sensor (TSADC)
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+/* TSADC register offsets */
+#define AXIADO_TSADC_MODE	0x0000
+#define AXIADO_TSADC_FUNC_CTRL	0x0004
+#define AXIADO_TSADC_AVG_DOUT	0x000c
+#define AXIADO_TSADC_INTR_MASK	0x0010
+#define AXIADO_TSADC_ANA_REG	0x0020
+#define AXIADO_TSADC_SEL	0x0028
+#define AXIADO_TSADC_TSEN_EN	0x002c
+#define AXIADO_TSADC_EN	0x0030
+
+/* TSADC mode register: continuous conversion */
+#define AXIADO_TSADC_MODE_CONT	0x2
+
+/*
+ * TSADC functional control register: bit 0 starts the conversion and
+ * bits [31:16] specify the number of samples. Average over 16 samples.
+ */
+#define AXIADO_TSADC_FUNC_CTRL_START		BIT(0)
+#define AXIADO_TSADC_FUNC_CTRL_NUM_SAMPLES	GENMASK(31, 16)
+#define AXIADO_TSADC_FUNC_CTRL_CLR		0x0
+#define AXIADO_TSADC_FUNC_CTRL_SET				\
+	(FIELD_PREP(AXIADO_TSADC_FUNC_CTRL_NUM_SAMPLES, 16) |	\
+	 AXIADO_TSADC_FUNC_CTRL_START)
+
+/* TSADC average output register: 12-bit ADC result */
+#define AXIADO_TSADC_AVG_DOUT_MASK	GENMASK(11, 0)
+
+/* Interrupt mask bits */
+#define AXIADO_TSADC_INTR_BIST_DONE		BIT(0)
+#define AXIADO_TSADC_INTR_LOOP_DONE		BIT(1)
+#define AXIADO_TSADC_INTR_THRESHOLD		BIT(2)
+#define AXIADO_TSADC_INTR_MASK_ALL		\
+	(AXIADO_TSADC_INTR_BIST_DONE |		\
+	 AXIADO_TSADC_INTR_LOOP_DONE |		\
+	 AXIADO_TSADC_INTR_THRESHOLD)
+
+/* TSADC analog block configuration required for temperature measurement */
+#define AXIADO_TSADC_ANA_REG_INIT	0x6
+
+/* TSADC SEL: select temperature measurement */
+#define AXIADO_TSADC_SEL_TEMP	0x00
+
+/* TSADC EN / TSEN_EN field values */
+#define AXIADO_TSADC_DIS	0x00
+#define AXIADO_TSADC_ENA	0x01
+
+struct axiado_tsadc {
+	void __iomem *regs;
+};
+
+/*
+ * Lookup table: raw ADC output (12-bit) vs temperature in millidegrees
+ * Celsius. Raw values decrease as temperature increases, from -40°C
+ * to 125°C in 5°C steps.
+ */
+struct axiado_tsadc_point {
+	u32 raw;
+	int temp_mc;
+};
+
+static const struct axiado_tsadc_point axiado_tsadc_table[] = {
+	{ .raw = 2776, .temp_mc = -40000 },
+	{ .raw = 2748, .temp_mc = -35000 },
+	{ .raw = 2721, .temp_mc = -30000 },
+	{ .raw = 2694, .temp_mc = -25000 },
+	{ .raw = 2667, .temp_mc = -20000 },
+	{ .raw = 2640, .temp_mc = -15000 },
+	{ .raw = 2612, .temp_mc = -10000 },
+	{ .raw = 2584, .temp_mc =  -5000 },
+	{ .raw = 2556, .temp_mc =      0 },
+	{ .raw = 2528, .temp_mc =   5000 },
+	{ .raw = 2500, .temp_mc =  10000 },
+	{ .raw = 2472, .temp_mc =  15000 },
+	{ .raw = 2444, .temp_mc =  20000 },
+	{ .raw = 2416, .temp_mc =  25000 },
+	{ .raw = 2389, .temp_mc =  30000 },
+	{ .raw = 2362, .temp_mc =  35000 },
+	{ .raw = 2335, .temp_mc =  40000 },
+	{ .raw = 2307, .temp_mc =  45000 },
+	{ .raw = 2279, .temp_mc =  50000 },
+	{ .raw = 2251, .temp_mc =  55000 },
+	{ .raw = 2223, .temp_mc =  60000 },
+	{ .raw = 2195, .temp_mc =  65000 },
+	{ .raw = 2167, .temp_mc =  70000 },
+	{ .raw = 2138, .temp_mc =  75000 },
+	{ .raw = 2110, .temp_mc =  80000 },
+	{ .raw = 2081, .temp_mc =  85000 },
+	{ .raw = 2052, .temp_mc =  90000 },
+	{ .raw = 2024, .temp_mc =  95000 },
+	{ .raw = 1996, .temp_mc = 100000 },
+	{ .raw = 1967, .temp_mc = 105000 },
+	{ .raw = 1938, .temp_mc = 110000 },
+	{ .raw = 1909, .temp_mc = 115000 },
+	{ .raw = 1880, .temp_mc = 120000 },
+	{ .raw = 1852, .temp_mc = 125000 },
+};
+
+/* Convert raw ADC code to millidegrees Celsius with linear interpolation. */
+static int axiado_tsadc_raw_to_mc(u32 raw, long *val)
+{
+	int i, n = ARRAY_SIZE(axiado_tsadc_table);
+	long num, denom;
+
+	if (raw >= axiado_tsadc_table[0].raw) {
+		*val = axiado_tsadc_table[0].temp_mc;
+		return 0;
+	}
+
+	if (raw <= axiado_tsadc_table[n - 1].raw) {
+		*val = axiado_tsadc_table[n - 1].temp_mc;
+		return 0;
+	}
+
+	/* Find i such that axiado_tsadc_table[i].raw >= raw > axiado_tsadc_table[i+1].raw */
+	for (i = 0; i < n - 1; i++) {
+		if (raw >= axiado_tsadc_table[i + 1].raw)
+			break;
+	}
+
+	if (raw == axiado_tsadc_table[i].raw) {
+		*val = axiado_tsadc_table[i].temp_mc;
+		return 0;
+	}
+
+	/* Linear interpolation within the interval */
+	num = (long)(axiado_tsadc_table[i + 1].temp_mc - axiado_tsadc_table[i].temp_mc) *
+	      (axiado_tsadc_table[i].raw - raw);
+	denom = axiado_tsadc_table[i].raw - axiado_tsadc_table[i + 1].raw;
+	*val = axiado_tsadc_table[i].temp_mc + num / denom;
+	return 0;
+}
+
+static int axiado_tsadc_read(struct device *dev, enum hwmon_sensor_types type,
+			     u32 attr, int channel, long *val)
+{
+	struct axiado_tsadc *tsadc = dev_get_drvdata(dev);
+	u32 raw;
+
+	if (type != hwmon_temp)
+		return -EOPNOTSUPP;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		if (ioread32(tsadc->regs + AXIADO_TSADC_SEL) != AXIADO_TSADC_SEL_TEMP ||
+		    ioread32(tsadc->regs + AXIADO_TSADC_EN) != AXIADO_TSADC_ENA)
+			return -ENODATA;
+
+		raw = ioread32(tsadc->regs + AXIADO_TSADC_AVG_DOUT) &
+			       AXIADO_TSADC_AVG_DOUT_MASK;
+
+		return axiado_tsadc_raw_to_mc(raw, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static umode_t axiado_tsadc_is_visible(const void *data,
+				       enum hwmon_sensor_types type, u32 attr,
+				       int channel)
+{
+	if (type == hwmon_temp) {
+		switch (attr) {
+		case hwmon_temp_input:
+			return 0444;
+		default:
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static const struct hwmon_channel_info * const axiado_tsadc_info[] = {
+	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT),
+	NULL
+};
+
+static const struct hwmon_ops axiado_tsadc_hwmon_ops = {
+	.is_visible = axiado_tsadc_is_visible,
+	.read = axiado_tsadc_read,
+};
+
+static const struct hwmon_chip_info axiado_tsadc_chip_info = {
+	.ops = &axiado_tsadc_hwmon_ops,
+	.info = axiado_tsadc_info,
+};
+
+static void axiado_tsadc_disable(void *data)
+{
+	struct axiado_tsadc *tsadc = data;
+
+	iowrite32(AXIADO_TSADC_DIS, tsadc->regs + AXIADO_TSADC_TSEN_EN);
+	iowrite32(AXIADO_TSADC_DIS, tsadc->regs + AXIADO_TSADC_EN);
+}
+
+static int axiado_tsadc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct axiado_tsadc *tsadc;
+	struct device *hwmon_dev;
+	struct clk *refclk;
+	int ret;
+
+	tsadc = devm_kzalloc(dev, sizeof(*tsadc), GFP_KERNEL);
+	if (!tsadc)
+		return -ENOMEM;
+
+	tsadc->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(tsadc->regs))
+		return dev_err_probe(dev, PTR_ERR(tsadc->regs),
+				     "failed to map registers\n");
+
+	refclk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(refclk))
+		return dev_err_probe(dev, PTR_ERR(refclk),
+				     "failed to enable reference clock\n");
+
+	/* Mask all interrupts before setup */
+	iowrite32(AXIADO_TSADC_INTR_MASK_ALL, tsadc->regs + AXIADO_TSADC_INTR_MASK);
+
+	iowrite32(AXIADO_TSADC_ANA_REG_INIT, tsadc->regs + AXIADO_TSADC_ANA_REG);
+	iowrite32(AXIADO_TSADC_ENA, tsadc->regs + AXIADO_TSADC_EN);
+	iowrite32(AXIADO_TSADC_SEL_TEMP, tsadc->regs + AXIADO_TSADC_SEL);
+	iowrite32(AXIADO_TSADC_MODE_CONT, tsadc->regs + AXIADO_TSADC_MODE);
+	iowrite32(AXIADO_TSADC_FUNC_CTRL_CLR, tsadc->regs + AXIADO_TSADC_FUNC_CTRL);
+	iowrite32(AXIADO_TSADC_FUNC_CTRL_SET, tsadc->regs + AXIADO_TSADC_FUNC_CTRL);
+
+	/* Enable the bandgap circuit once the block is fully configured */
+	iowrite32(AXIADO_TSADC_ENA, tsadc->regs + AXIADO_TSADC_TSEN_EN);
+
+	ret = devm_add_action_or_reset(dev, axiado_tsadc_disable, tsadc);
+	if (ret)
+		return ret;
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, "axiado_tsadc",
+							 tsadc,
+							 &axiado_tsadc_chip_info,
+							 NULL);
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static const struct of_device_id axiado_tsadc_match[] = {
+	{ .compatible = "axiado,ax3000-tsadc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, axiado_tsadc_match);
+
+static struct platform_driver axiado_tsadc_driver = {
+	.driver = {
+		.name = "axiado-tsadc",
+		.of_match_table	= axiado_tsadc_match,
+	},
+	.probe = axiado_tsadc_probe,
+};
+module_platform_driver(axiado_tsadc_driver);
+
+MODULE_DESCRIPTION("Axiado TSADC temperature sensor driver");
+MODULE_AUTHOR("Axiado Corporation");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index 5095eb0576805d..0ab6bbff5637d8 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -402,9 +402,9 @@ static ssize_t show_temp(struct device *dev,
 
 	mutex_lock(&tdata->update_lock);
 
-	tjmax = get_tjmax(tdata, dev);
 	/* Check whether the time interval has elapsed */
 	if (time_after(jiffies, tdata->last_updated + HZ)) {
+		tjmax = get_tjmax(tdata, dev);
 		rdmsrq_on_cpu(tdata->cpu, tdata->status_reg, &val.q);
 		/*
 		 * Ignore the valid bit. In all observed cases the register
@@ -511,6 +511,11 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag)
 	tdata->cpu = cpu;
 	tdata->cpu_core_id = topology_core_id(cpu);
 	tdata->attr_size = MAX_CORE_ATTRS;
+	/*
+	 * A zero timestamp does not look stale on 32-bit, where jiffies
+	 * starts just short of wrapping. Backdate it instead.
+	 */
+	tdata->last_updated = jiffies - HZ - 1;
 	mutex_init(&tdata->update_lock);
 	return tdata;
 }
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index 20920d4bd81e27..88837f0284051e 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -1543,6 +1543,14 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = {
 		},
 		.driver_data = (void *)&i8k_fan_control_data[I8K_FAN_34A3_35A3],
 	},
+	{
+		.ident = "Dell Latitude 5420",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Latitude 5420"),
+		},
+		.driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3],
+	},
 	{
 		.ident = "Dell Latitude 5480",
 		.matches = {
@@ -1631,6 +1639,22 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = {
 		},
 		.driver_data = (void *)&i8k_fan_control_data[I8K_FAN_34A3_35A3],
 	},
+	{
+		.ident = "Dell Precision 3650 Tower",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Precision 3650 Tower"),
+		},
+		.driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3],
+	},
+	{
+		.ident = "Dell Optiplex 7090",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7090"),
+		},
+		.driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3],
+	},
 	{
 		.ident = "Dell XPS 9315",
 		.matches = {
@@ -1655,6 +1679,14 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = {
 		},
 		.driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3],
 	},
+	{
+		.ident = "Dell 16 DC16250",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Dell 16 DC16250"),
+		},
+		.driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3],
+	},
 	{ }
 };
 
diff --git a/drivers/hwmon/emc2103.c b/drivers/hwmon/emc2103.c
index 27dc149a3ed995..4adc23ffaf720d 100644
--- a/drivers/hwmon/emc2103.c
+++ b/drivers/hwmon/emc2103.c
@@ -32,7 +32,7 @@ static const u8 REG_TEMP_MAX[4] = { 0x34, 0x30, 0x31, 0x32 };
 #define REG_PRODUCT_ID		0xfd
 #define REG_MFG_ID		0xfe
 
-/* equation 4 from datasheet: rpm = (3932160 * multipler) / count */
+/* equation 4 from datasheet: rpm = (3932160 * multiplier) / count */
 #define FAN_RPM_FACTOR		3932160
 
 /*
diff --git a/drivers/hwmon/hih6130.c b/drivers/hwmon/hih6130.c
index 7984be1e706d55..6a79aab5d1285a 100644
--- a/drivers/hwmon/hih6130.c
+++ b/drivers/hwmon/hih6130.c
@@ -168,7 +168,7 @@ static ssize_t hih6130_temperature_show(struct device *dev,
 	ret = hih6130_update_measurements(dev);
 	if (ret < 0)
 		return ret;
-	return sprintf(buf, "%d\n", hih6130->temperature);
+	return sysfs_emit(buf, "%d\n", hih6130->temperature);
 }
 
 /**
@@ -189,7 +189,7 @@ static ssize_t hih6130_humidity_show(struct device *dev,
 	ret = hih6130_update_measurements(dev);
 	if (ret < 0)
 		return ret;
-	return sprintf(buf, "%d\n", hih6130->humidity);
+	return sysfs_emit(buf, "%d\n", hih6130->humidity);
 }
 
 /* sysfs attributes */
diff --git a/drivers/hwmon/honor-fmi.c b/drivers/hwmon/honor-fmi.c
new file mode 100644
index 00000000000000..3421e40a132f83
--- /dev/null
+++ b/drivers/hwmon/honor-fmi.c
@@ -0,0 +1,178 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Read-only fan monitoring for the HONOR FMI-XX.
+ *
+ * The firmware-provided \GFNS ACPI method accepts a three-byte buffer.
+ * Byte 2 selects fan 0 or 1. It returns a status byte followed by a
+ * little-endian 16-bit fan speed in RPM. The method owns all Embedded
+ * Controller access and serialization; this driver deliberately exposes no
+ * fan control interface.
+ */
+
+#include <linux/acpi.h>
+#include <linux/dmi.h>
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#define HONOR_FMI_GFNS_RESULT_SIZE	3
+
+struct honor_fmi_data {
+	acpi_handle gfns;
+};
+
+static const struct dmi_system_id honor_fmi_dmi_table[] = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "HONOR"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "FMI-XX"),
+		},
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(dmi, honor_fmi_dmi_table);
+
+static int honor_fmi_read_rpm(struct honor_fmi_data *data, int channel,
+			      long *rpm)
+{
+	union acpi_object input = {
+		.buffer = {
+			.type = ACPI_TYPE_BUFFER,
+			.length = 3,
+		},
+	};
+	struct acpi_object_list arguments = {
+		.count = 1,
+		.pointer = &input,
+	};
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *result;
+	u8 input_bytes[3] = { 0, 0, channel };
+	acpi_status status;
+	int ret = 0;
+
+	input.buffer.pointer = input_bytes;
+
+	status = acpi_evaluate_object(data->gfns, NULL, &arguments, &output);
+	if (ACPI_FAILURE(status))
+		return -EIO;
+
+	result = output.pointer;
+	if (!result || result->type != ACPI_TYPE_BUFFER ||
+	    result->buffer.length < HONOR_FMI_GFNS_RESULT_SIZE) {
+		ret = -EPROTO;
+		goto out_free;
+	}
+
+	if (result->buffer.pointer[0]) {
+		ret = -EIO;
+		goto out_free;
+	}
+
+	*rpm = result->buffer.pointer[1] |
+	       (result->buffer.pointer[2] << 8);
+
+out_free:
+	kfree(output.pointer);
+	return ret;
+}
+
+static umode_t honor_fmi_is_visible(const void *data,
+				    enum hwmon_sensor_types type, u32 attr,
+				    int channel)
+{
+	return 0444;
+}
+
+static int honor_fmi_read(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int channel, long *value)
+{
+	struct honor_fmi_data *data = dev_get_drvdata(dev);
+
+	return honor_fmi_read_rpm(data, channel, value);
+}
+
+static const struct hwmon_ops honor_fmi_hwmon_ops = {
+	.is_visible = honor_fmi_is_visible,
+	.read = honor_fmi_read,
+};
+
+static const struct hwmon_channel_info * const honor_fmi_hwmon_info[] = {
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT, HWMON_F_INPUT),
+	NULL
+};
+
+static const struct hwmon_chip_info honor_fmi_chip_info = {
+	.ops = &honor_fmi_hwmon_ops,
+	.info = honor_fmi_hwmon_info,
+};
+
+static int honor_fmi_probe(struct platform_device *pdev)
+{
+	struct honor_fmi_data *data;
+	struct device *hwmon_dev;
+	acpi_status status;
+
+	if (!dmi_check_system(honor_fmi_dmi_table))
+		return -ENODEV;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	status = acpi_get_handle(NULL, "\\GFNS", &data->gfns);
+	if (ACPI_FAILURE(status))
+		return dev_err_probe(&pdev->dev, -ENODEV,
+				     "firmware does not provide \\GFNS\n");
+
+	hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev, "honor_fmi",
+							 data,
+							 &honor_fmi_chip_info,
+							 NULL);
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static struct platform_driver honor_fmi_driver = {
+	.probe = honor_fmi_probe,
+	.driver = {
+		.name = "honor-fmi-hwmon",
+	},
+};
+
+static struct platform_device *honor_fmi_device;
+
+static int __init honor_fmi_init(void)
+{
+	int ret;
+
+	if (!dmi_check_system(honor_fmi_dmi_table))
+		return -ENODEV;
+
+	ret = platform_driver_register(&honor_fmi_driver);
+	if (ret)
+		return ret;
+
+	honor_fmi_device = platform_device_register_simple("honor-fmi-hwmon",
+							   PLATFORM_DEVID_NONE,
+							   NULL, 0);
+	if (IS_ERR(honor_fmi_device)) {
+		platform_driver_unregister(&honor_fmi_driver);
+		return PTR_ERR(honor_fmi_device);
+	}
+
+	return 0;
+}
+
+static void __exit honor_fmi_exit(void)
+{
+	platform_device_unregister(honor_fmi_device);
+	platform_driver_unregister(&honor_fmi_driver);
+}
+
+module_init(honor_fmi_init);
+module_exit(honor_fmi_exit);
+
+MODULE_AUTHOR("Nikita Dubrovskih <testname142@gmail.com>");
+MODULE_DESCRIPTION("HONOR FMI-XX fan speed monitor");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/hp-wmi-sensors.c b/drivers/hwmon/hp-wmi-sensors.c
index cfa0d6ad0fb3f1..d8bcfeb8d5b290 100644
--- a/drivers/hwmon/hp-wmi-sensors.c
+++ b/drivers/hwmon/hp-wmi-sensors.c
@@ -1804,7 +1804,7 @@ static int init_numeric_sensors(struct hp_wmi_sensors *state,
 
 		if (!info_map[type]) {
 			info_map[type] = devm_kcalloc(dev, channel_count[type],
-						      sizeof(*info_map),
+						      sizeof(*info_map[type]),
 						      GFP_KERNEL);
 			if (!info_map[type])
 				return -ENOMEM;
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index 10d2df3efdfaee..3f32b0e83b532a 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -625,6 +625,8 @@ static const char * const hwmon_in_attr_templates[] = {
 	[hwmon_in_max] = "in%d_max",
 	[hwmon_in_lcrit] = "in%d_lcrit",
 	[hwmon_in_crit] = "in%d_crit",
+	[hwmon_in_lemergency] = "in%d_lemergency",
+	[hwmon_in_emergency] = "in%d_emergency",
 	[hwmon_in_average] = "in%d_average",
 	[hwmon_in_lowest] = "in%d_lowest",
 	[hwmon_in_highest] = "in%d_highest",
@@ -635,6 +637,8 @@ static const char * const hwmon_in_attr_templates[] = {
 	[hwmon_in_max_alarm] = "in%d_max_alarm",
 	[hwmon_in_lcrit_alarm] = "in%d_lcrit_alarm",
 	[hwmon_in_crit_alarm] = "in%d_crit_alarm",
+	[hwmon_in_lemergency_alarm] = "in%d_lemergency_alarm",
+	[hwmon_in_emergency_alarm] = "in%d_emergency_alarm",
 	[hwmon_in_rated_min] = "in%d_rated_min",
 	[hwmon_in_rated_max] = "in%d_rated_max",
 	[hwmon_in_beep] = "in%d_beep",
@@ -648,6 +652,7 @@ static const char * const hwmon_curr_attr_templates[] = {
 	[hwmon_curr_max] = "curr%d_max",
 	[hwmon_curr_lcrit] = "curr%d_lcrit",
 	[hwmon_curr_crit] = "curr%d_crit",
+	[hwmon_curr_emergency] = "curr%d_emergency",
 	[hwmon_curr_average] = "curr%d_average",
 	[hwmon_curr_lowest] = "curr%d_lowest",
 	[hwmon_curr_highest] = "curr%d_highest",
@@ -658,6 +663,7 @@ static const char * const hwmon_curr_attr_templates[] = {
 	[hwmon_curr_max_alarm] = "curr%d_max_alarm",
 	[hwmon_curr_lcrit_alarm] = "curr%d_lcrit_alarm",
 	[hwmon_curr_crit_alarm] = "curr%d_crit_alarm",
+	[hwmon_curr_emergency_alarm] = "curr%d_emergency_alarm",
 	[hwmon_curr_rated_min] = "curr%d_rated_min",
 	[hwmon_curr_rated_max] = "curr%d_rated_max",
 	[hwmon_curr_beep] = "curr%d_beep",
diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c
index 87edb1b6048bb5..2757d428a23dba 100644
--- a/drivers/hwmon/it87.c
+++ b/drivers/hwmon/it87.c
@@ -36,6 +36,7 @@
  *            IT8790E  Super I/O chip w/LPC interface
  *            IT8792E  Super I/O chip w/LPC interface
  *            IT87952E  Super I/O chip w/LPC interface
+ *            IT8613E  Super I/O chip w/LPC interface
  *            Sis950   A clone of the IT8705F
  *
  *  Copyright (C) 2001 Chris Gauthron
@@ -65,7 +66,7 @@
 
 enum chips { it87, it8712, it8716, it8718, it8720, it8721, it8728, it8732,
 	     it8771, it8772, it8781, it8782, it8783, it8786, it8790,
-	     it8792, it8603, it8620, it8622, it8628, it8689, it87952 };
+	     it8792, it8603, it8613, it8620, it8622, it8628, it8689, it87952 };
 
 static struct platform_device *it87_pdev[2];
 
@@ -159,6 +160,7 @@ static inline void superio_exit(int ioreg, bool noexit)
 #define IT8786E_DEVID 0x8786
 #define IT8790E_DEVID 0x8790
 #define IT8603E_DEVID 0x8603
+#define IT8613E_DEVID 0x8613
 #define IT8620E_DEVID 0x8620
 #define IT8622E_DEVID 0x8622
 #define IT8623E_DEVID 0x8623
@@ -252,6 +254,7 @@ static const u8 IT87_REG_TEMP_OFFSET[] = { 0x56, 0x57, 0x59 };
 #define IT87_REG_FAN_MAIN_CTRL 0x13
 #define IT87_REG_FAN_CTL       0x14
 static const u8 IT87_REG_PWM[]         = { 0x15, 0x16, 0x17, 0x7f, 0xa7, 0xaf };
+static const u8 IT87_REG_PWM_8665[]    = { 0x15, 0x16, 0x17, 0x1e, 0x1f, 0x92 };
 static const u8 IT87_REG_PWM_DUTY[]    = { 0x63, 0x6b, 0x73, 0x7b, 0xa3, 0xab };
 
 static const u8 IT87_REG_VIN[]	= { 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26,
@@ -283,6 +286,7 @@ static const u8 IT87_REG_AUTO_BASE[] = { 0x60, 0x68, 0x70, 0x78, 0xa0, 0xa8 };
 #define NUM_TEMP		6
 #define NUM_TEMP_OFFSET		ARRAY_SIZE(IT87_REG_TEMP_OFFSET)
 #define NUM_TEMP_LIMIT		3
+#define IT87_PWM_OLD_NUM_TEMP	3
 #define NUM_FAN			ARRAY_SIZE(IT87_REG_FAN)
 #define NUM_FAN_DIV		3
 #define NUM_PWM			ARRAY_SIZE(IT87_REG_PWM)
@@ -292,6 +296,8 @@ struct it87_devices {
 	const char *name;
 	const char * const model;
 	u32 features;
+	const u8 *reg_pwm;
+	u8 num_temp_map;
 	u8 peci_mask;
 	u8 old_peci_mask;
 	u8 smbus_bitmap;	/* SMBus enable bits in extra config register */
@@ -328,6 +334,8 @@ struct it87_devices {
 #define FEAT_FOUR_PWM		BIT(21)	/* Supports four fan controls */
 #define FEAT_FOUR_TEMP		BIT(22)
 #define FEAT_FANCTL_ONOFF	BIT(23)	/* chip has FAN_CTL ON/OFF */
+#define FEAT_NEW_TEMPMAP	BIT(24)	/* PWM uses extended temp map */
+#define FEAT_11MV_ADC		BIT(25)
 
 static const struct it87_devices it87_devices[] = {
 	[it87] = {
@@ -335,12 +343,16 @@ static const struct it87_devices it87_devices[] = {
 		.model = "IT87F",
 		.features = FEAT_OLD_AUTOPWM | FEAT_FANCTL_ONOFF,
 		/* may need to overwrite */
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 	},
 	[it8712] = {
 		.name = "it8712",
 		.model = "IT8712F",
 		.features = FEAT_OLD_AUTOPWM | FEAT_VID | FEAT_FANCTL_ONOFF,
 		/* may need to overwrite */
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 	},
 	[it8716] = {
 		.name = "it8716",
@@ -348,6 +360,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_16BIT_FANS | FEAT_TEMP_OFFSET | FEAT_VID
 		  | FEAT_FAN16_CONFIG | FEAT_FIVE_FANS | FEAT_PWM_FREQ2
 		  | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 	},
 	[it8718] = {
 		.name = "it8718",
@@ -355,6 +369,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_16BIT_FANS | FEAT_TEMP_OFFSET | FEAT_VID
 		  | FEAT_TEMP_OLD_PECI | FEAT_FAN16_CONFIG | FEAT_FIVE_FANS
 		  | FEAT_PWM_FREQ2 | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.old_peci_mask = 0x4,
 	},
 	[it8720] = {
@@ -363,6 +379,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_16BIT_FANS | FEAT_TEMP_OFFSET | FEAT_VID
 		  | FEAT_TEMP_OLD_PECI | FEAT_FAN16_CONFIG | FEAT_FIVE_FANS
 		  | FEAT_PWM_FREQ2 | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.old_peci_mask = 0x4,
 	},
 	[it8721] = {
@@ -372,6 +390,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_OLD_PECI | FEAT_TEMP_PECI
 		  | FEAT_FAN16_CONFIG | FEAT_FIVE_FANS | FEAT_IN7_INTERNAL
 		  | FEAT_PWM_FREQ2 | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x05,
 		.old_peci_mask = 0x02,	/* Actually reports PCH */
 	},
@@ -382,6 +402,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_FIVE_FANS
 		  | FEAT_IN7_INTERNAL | FEAT_PWM_FREQ2
 		  | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 	},
 	[it8732] = {
@@ -391,6 +413,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_OLD_PECI | FEAT_TEMP_PECI
 		  | FEAT_10_9MV_ADC | FEAT_IN7_INTERNAL | FEAT_FOUR_FANS
 		  | FEAT_FOUR_PWM | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 		.old_peci_mask = 0x02,	/* Actually reports PCH */
 	},
@@ -404,6 +428,8 @@ static const struct it87_devices it87_devices[] = {
 				/* 12mV ADC (OHM) */
 				/* 16 bit fans (OHM) */
 				/* three fans, always 16 bit (guesswork) */
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 	},
 	[it8772] = {
@@ -416,6 +442,8 @@ static const struct it87_devices it87_devices[] = {
 				/* 12mV ADC (HWSensors4, OHM) */
 				/* 16 bit fans (HWSensors4, OHM) */
 				/* three fans, always 16 bit (datasheet) */
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 	},
 	[it8781] = {
@@ -424,6 +452,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_16BIT_FANS | FEAT_TEMP_OFFSET
 		  | FEAT_TEMP_OLD_PECI | FEAT_FAN16_CONFIG | FEAT_PWM_FREQ2
 		  | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.old_peci_mask = 0x4,
 	},
 	[it8782] = {
@@ -432,6 +462,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_16BIT_FANS | FEAT_TEMP_OFFSET
 		  | FEAT_TEMP_OLD_PECI | FEAT_FAN16_CONFIG | FEAT_PWM_FREQ2
 		  | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.old_peci_mask = 0x4,
 	},
 	[it8783] = {
@@ -440,6 +472,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_16BIT_FANS | FEAT_TEMP_OFFSET
 		  | FEAT_TEMP_OLD_PECI | FEAT_FAN16_CONFIG | FEAT_PWM_FREQ2
 		  | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.old_peci_mask = 0x4,
 	},
 	[it8786] = {
@@ -448,6 +482,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_NEWER_AUTOPWM | FEAT_12MV_ADC | FEAT_16BIT_FANS
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_IN7_INTERNAL
 		  | FEAT_PWM_FREQ2 | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 	},
 	[it8790] = {
@@ -456,6 +492,8 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_NEWER_AUTOPWM | FEAT_12MV_ADC | FEAT_16BIT_FANS
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_IN7_INTERNAL
 		  | FEAT_PWM_FREQ2 | FEAT_FANCTL_ONOFF | FEAT_NOCONF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 	},
 	[it8792] = {
@@ -465,6 +503,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_OLD_PECI | FEAT_TEMP_PECI
 		  | FEAT_10_9MV_ADC | FEAT_IN7_INTERNAL | FEAT_FANCTL_ONOFF
 		  | FEAT_NOCONF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 		.old_peci_mask = 0x02,	/* Actually reports PCH */
 	},
@@ -474,6 +514,20 @@ static const struct it87_devices it87_devices[] = {
 		.features = FEAT_NEWER_AUTOPWM | FEAT_12MV_ADC | FEAT_16BIT_FANS
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_IN7_INTERNAL
 		  | FEAT_AVCC3 | FEAT_PWM_FREQ2,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 4,
+		.peci_mask = 0x07,
+	},
+	[it8613] = {
+		.name = "it8613",
+		.model = "IT8613E",
+		/* Only three temperature inputs are currently known. */
+		.features = FEAT_NEWER_AUTOPWM | FEAT_11MV_ADC | FEAT_16BIT_FANS
+		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_FIVE_FANS
+		  | FEAT_FIVE_PWM | FEAT_IN7_INTERNAL | FEAT_PWM_FREQ2
+		  | FEAT_AVCC3 | FEAT_NEW_TEMPMAP,
+		.reg_pwm = IT87_REG_PWM_8665,
+		.num_temp_map = 6,
 		.peci_mask = 0x07,
 	},
 	[it8620] = {
@@ -483,6 +537,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_SIX_FANS
 		  | FEAT_IN7_INTERNAL | FEAT_SIX_PWM | FEAT_PWM_FREQ2
 		  | FEAT_SIX_TEMP | FEAT_VIN3_5V | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 	},
 	[it8622] = {
@@ -492,6 +548,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_FIVE_FANS
 		  | FEAT_FIVE_PWM | FEAT_IN7_INTERNAL | FEAT_PWM_FREQ2
 		  | FEAT_AVCC3 | FEAT_VIN3_5V | FEAT_FOUR_TEMP,
+		.reg_pwm = IT87_REG_PWM_8665,
+		.num_temp_map = 4,
 		.peci_mask = 0x07,
 		.smbus_bitmap = BIT(1) | BIT(2),
 	},
@@ -502,6 +560,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_PECI | FEAT_SIX_FANS
 		  | FEAT_IN7_INTERNAL | FEAT_SIX_PWM | FEAT_PWM_FREQ2
 		  | FEAT_SIX_TEMP | FEAT_VIN3_5V | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 	},
 	[it8689] = {
@@ -511,6 +571,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_SIX_FANS | FEAT_IN7_INTERNAL
 		  | FEAT_SIX_PWM | FEAT_PWM_FREQ2 | FEAT_SIX_TEMP | FEAT_AVCC3
 		  | FEAT_FANCTL_ONOFF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.smbus_bitmap = BIT(1) | BIT(2),
 	},
 	[it87952] = {
@@ -520,6 +582,8 @@ static const struct it87_devices it87_devices[] = {
 		  | FEAT_TEMP_OFFSET | FEAT_TEMP_OLD_PECI | FEAT_TEMP_PECI
 		  | FEAT_10_9MV_ADC | FEAT_IN7_INTERNAL | FEAT_FANCTL_ONOFF
 		  | FEAT_NOCONF,
+		.reg_pwm = IT87_REG_PWM,
+		.num_temp_map = 3,
 		.peci_mask = 0x07,
 		.old_peci_mask = 0x02,	/* Actually reports PCH */
 	},
@@ -528,6 +592,7 @@ static const struct it87_devices it87_devices[] = {
 #define has_16bit_fans(data)	((data)->features & FEAT_16BIT_FANS)
 #define has_12mv_adc(data)	((data)->features & FEAT_12MV_ADC)
 #define has_10_9mv_adc(data)	((data)->features & FEAT_10_9MV_ADC)
+#define has_11mv_adc(data)	((data)->features & FEAT_11MV_ADC)
 #define has_newer_autopwm(data)	((data)->features & FEAT_NEWER_AUTOPWM)
 #define has_old_autopwm(data)	((data)->features & FEAT_OLD_AUTOPWM)
 #define has_temp_offset(data)	((data)->features & FEAT_TEMP_OFFSET)
@@ -558,8 +623,10 @@ static const struct it87_devices it87_devices[] = {
 #define has_vin3_5v(data)	((data)->features & FEAT_VIN3_5V)
 #define has_noconf(data)	((data)->features & FEAT_NOCONF)
 #define has_scaling(data)	((data)->features & (FEAT_12MV_ADC | \
-						     FEAT_10_9MV_ADC))
+						     FEAT_10_9MV_ADC | \
+						     FEAT_11MV_ADC))
 #define has_fanctl_onoff(data)	((data)->features & FEAT_FANCTL_ONOFF)
+#define has_new_tempmap(data)	((data)->features & FEAT_NEW_TEMPMAP)
 
 struct it87_sio_data {
 	int sioaddr;
@@ -589,6 +656,8 @@ struct it87_data {
 	int sioaddr;
 	enum chips type;
 	u32 features;
+	const u8 *reg_pwm;
+	u8 num_temp_map;
 	u8 peci_mask;
 	u8 old_peci_mask;
 
@@ -635,7 +704,9 @@ struct it87_data {
 	u8 has_pwm;		/* Bitfield, pwm control enabled */
 	u8 pwm_ctrl[NUM_PWM];	/* Register value */
 	u8 pwm_duty[NUM_PWM];	/* Manual PWM value set by user */
-	u8 pwm_temp_map[NUM_PWM];/* PWM to temp. chan. mapping (bits 1-0) */
+	u8 pwm_temp_map[NUM_PWM];/* PWM to temp. chan. mapping */
+	u8 pwm_temp_map_mask;
+	u8 pwm_temp_map_shift;
 
 	/* Automatic fan speed control registers */
 	u8 auto_pwm[NUM_AUTO_PWM][4];	/* [nr][3] is hard-coded */
@@ -658,6 +729,8 @@ static int adc_lsb(const struct it87_data *data, int nr)
 		lsb = 120;
 	else if (has_10_9mv_adc(data))
 		lsb = 109;
+	else if (has_11mv_adc(data))
+		lsb = 110;
 	else
 		lsb = 160;
 	if (data->in_scaled & BIT(nr))
@@ -717,6 +790,77 @@ static int pwm_from_reg(const struct it87_data *data, u8 reg)
 		return (reg & 0x7f) << 1;
 }
 
+static inline u8 pwm_temp_map_get(const struct it87_data *data, u8 ctrl)
+{
+	return (ctrl >> data->pwm_temp_map_shift) &
+		data->pwm_temp_map_mask;
+}
+
+static inline u8 pwm_temp_map_set(const struct it87_data *data, u8 ctrl,
+				  u8 map)
+{
+	ctrl &= ~(data->pwm_temp_map_mask << data->pwm_temp_map_shift);
+	return ctrl | ((map & data->pwm_temp_map_mask)
+		       << data->pwm_temp_map_shift);
+}
+
+static inline u8 pwm_num_temp_map(const struct it87_data *data)
+{
+	return data->num_temp_map;
+}
+
+static inline bool uses_global_temp_map(const struct it87_data *data)
+{
+	return has_new_tempmap(data) ||
+		pwm_num_temp_map(data) != IT87_PWM_OLD_NUM_TEMP;
+}
+
+static unsigned int pwm_temp_channel(const struct it87_data *data,
+				     int nr, u8 map)
+{
+	if (uses_global_temp_map(data)) {
+		u8 num = pwm_num_temp_map(data);
+
+		if (map >= num)
+			map = 0;
+		return map;
+	}
+
+	if (map >= IT87_PWM_OLD_NUM_TEMP)
+		map = 0;
+
+	if (nr >= IT87_PWM_OLD_NUM_TEMP)
+		map += IT87_PWM_OLD_NUM_TEMP;
+
+	return map;
+}
+
+static int pwm_temp_map_from_channel(const struct it87_data *data, int nr,
+				     unsigned int channel, u8 *map)
+{
+	if (uses_global_temp_map(data)) {
+		u8 num = pwm_num_temp_map(data);
+
+		if (channel >= num)
+			return -EINVAL;
+		*map = channel;
+		return 0;
+	}
+
+	if (nr >= IT87_PWM_OLD_NUM_TEMP) {
+		if (channel < IT87_PWM_OLD_NUM_TEMP ||
+		    channel >= 2 * IT87_PWM_OLD_NUM_TEMP)
+			return -EINVAL;
+		channel -= IT87_PWM_OLD_NUM_TEMP;
+	} else {
+		if (channel >= IT87_PWM_OLD_NUM_TEMP)
+			return -EINVAL;
+	}
+
+	*map = channel;
+	return 0;
+}
+
 static int DIV_TO_REG(int val)
 {
 	int answer = 0;
@@ -728,6 +872,11 @@ static int DIV_TO_REG(int val)
 
 #define DIV_FROM_REG(val) BIT(val)
 
+static inline u16 it87_reg_pwm(const struct it87_data *data, int nr)
+{
+	return data->reg_pwm[nr];
+}
+
 /*
  * PWM base frequencies. The frequency has to be divided by either 128 or 256,
  * depending on the chip type, to calculate the actual PWM frequency.
@@ -808,16 +957,23 @@ static void it87_write_value(struct it87_data *data, u8 reg, u8 value)
 
 static void it87_update_pwm_ctrl(struct it87_data *data, int nr)
 {
-	data->pwm_ctrl[nr] = it87_read_value(data, IT87_REG_PWM[nr]);
+	data->pwm_ctrl[nr] = it87_read_value(data, it87_reg_pwm(data, nr));
 	if (has_newer_autopwm(data)) {
-		data->pwm_temp_map[nr] = data->pwm_ctrl[nr] & 0x03;
+		data->pwm_temp_map[nr] =
+			pwm_temp_map_get(data, data->pwm_ctrl[nr]);
+		if (uses_global_temp_map(data) &&
+		    data->pwm_temp_map[nr] >= pwm_num_temp_map(data))
+			data->pwm_temp_map[nr] = 0;
 		data->pwm_duty[nr] = it87_read_value(data,
 						     IT87_REG_PWM_DUTY[nr]);
-	} else {
-		if (data->pwm_ctrl[nr] & 0x80)	/* Automatic mode */
-			data->pwm_temp_map[nr] = data->pwm_ctrl[nr] & 0x03;
-		else				/* Manual mode */
-			data->pwm_duty[nr] = data->pwm_ctrl[nr] & 0x7f;
+	} else if (data->pwm_ctrl[nr] & 0x80) {	/* Automatic mode */
+		data->pwm_temp_map[nr] =
+			pwm_temp_map_get(data, data->pwm_ctrl[nr]);
+		if (uses_global_temp_map(data) &&
+		    data->pwm_temp_map[nr] >= pwm_num_temp_map(data))
+			data->pwm_temp_map[nr] = 0;
+	} else {				/* Manual mode */
+		data->pwm_duty[nr] = data->pwm_ctrl[nr] & 0x7f;
 	}
 
 	if (has_old_autopwm(data)) {
@@ -1567,27 +1723,32 @@ static ssize_t set_pwm_enable(struct device *dev, struct device_attribute *attr,
 					 data->pwm_duty[nr]);
 			/* and set manual mode */
 			if (has_newer_autopwm(data)) {
-				ctrl = (data->pwm_ctrl[nr] & 0x7c) |
-					data->pwm_temp_map[nr];
+				ctrl = pwm_temp_map_set(data,
+							data->pwm_ctrl[nr] &
+							~0x80,
+							data->pwm_temp_map[nr]);
 			} else {
 				ctrl = data->pwm_duty[nr];
 			}
 			data->pwm_ctrl[nr] = ctrl;
-			it87_write_value(data, IT87_REG_PWM[nr], ctrl);
+			it87_write_value(data, it87_reg_pwm(data, nr), ctrl);
 		}
 	} else {
 		u8 ctrl;
 
 		if (has_newer_autopwm(data)) {
-			ctrl = (data->pwm_ctrl[nr] & 0x7c) |
-				data->pwm_temp_map[nr];
+			ctrl = pwm_temp_map_set(data,
+						data->pwm_ctrl[nr] & ~0x80,
+						data->pwm_temp_map[nr]);
 			if (val != 1)
 				ctrl |= 0x80;
 		} else {
-			ctrl = (val == 1 ? data->pwm_duty[nr] : 0x80);
+			ctrl = val == 1 ? data->pwm_duty[nr] :
+				pwm_temp_map_set(data, 0x80,
+						 data->pwm_temp_map[nr]);
 		}
 		data->pwm_ctrl[nr] = ctrl;
-		it87_write_value(data, IT87_REG_PWM[nr], ctrl);
+		it87_write_value(data, it87_reg_pwm(data, nr), ctrl);
 
 		if (has_fanctl_onoff(data) && nr < 3) {
 			/* set SmartGuardian mode */
@@ -1638,7 +1799,7 @@ static ssize_t set_pwm(struct device *dev, struct device_attribute *attr,
 		 */
 		if (!(data->pwm_ctrl[nr] & 0x80)) {
 			data->pwm_ctrl[nr] = data->pwm_duty[nr];
-			it87_write_value(data, IT87_REG_PWM[nr],
+			it87_write_value(data, it87_reg_pwm(data, nr),
 					 data->pwm_ctrl[nr]);
 		}
 	}
@@ -1693,18 +1854,14 @@ static ssize_t show_pwm_temp_map(struct device *dev,
 	struct sensor_device_attribute *sensor_attr = to_sensor_dev_attr(attr);
 	struct it87_data *data = it87_update_device(dev);
 	int nr = sensor_attr->index;
-	int map;
+	unsigned int channel;
 
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
-	map = data->pwm_temp_map[nr];
-	if (map >= 3)
-		map = 0;	/* Should never happen */
-	if (nr >= 3)		/* pwm channels 3..6 map to temp4..6 */
-		map += 3;
+	channel = pwm_temp_channel(data, nr, data->pwm_temp_map[nr]);
 
-	return sprintf(buf, "%d\n", (int)BIT(map));
+	return sprintf(buf, "%d\n", (int)BIT(channel));
 }
 
 static ssize_t set_pwm_temp_map(struct device *dev,
@@ -1716,42 +1873,33 @@ static ssize_t set_pwm_temp_map(struct device *dev,
 	int nr = sensor_attr->index;
 	long val;
 	int err;
-	u8 reg;
+	unsigned int channel;
+	u8 map;
 
-	if (kstrtol(buf, 10, &val) < 0)
+	if (kstrtol(buf, 10, &val) < 0 || val <= 0 || !is_power_of_2(val))
 		return -EINVAL;
 
-	if (nr >= 3)
-		val -= 3;
-
-	switch (val) {
-	case BIT(0):
-		reg = 0x00;
-		break;
-	case BIT(1):
-		reg = 0x01;
-		break;
-	case BIT(2):
-		reg = 0x02;
-		break;
-	default:
+	channel = __ffs(val);
+	if (pwm_temp_map_from_channel(data, nr, channel, &map))
 		return -EINVAL;
-	}
 
 	err = it87_lock(data);
 	if (err)
 		return err;
 
 	it87_update_pwm_ctrl(data, nr);
-	data->pwm_temp_map[nr] = reg;
+	data->pwm_temp_map[nr] = map;
 	/*
-	 * If we are in automatic mode, write the temp mapping immediately;
-	 * otherwise, just store it for later use.
+	 * Newer controllers keep the duty cycle in a separate register, so
+	 * their temperature mapping can be updated in any mode. On older
+	 * controllers, defer the update until automatic mode is enabled.
 	 */
-	if (data->pwm_ctrl[nr] & 0x80) {
-		data->pwm_ctrl[nr] = (data->pwm_ctrl[nr] & 0xfc) |
-						data->pwm_temp_map[nr];
-		it87_write_value(data, IT87_REG_PWM[nr], data->pwm_ctrl[nr]);
+	if (has_newer_autopwm(data) || (data->pwm_ctrl[nr] & 0x80)) {
+		data->pwm_ctrl[nr] = pwm_temp_map_set(data,
+						      data->pwm_ctrl[nr],
+						      data->pwm_temp_map[nr]);
+		it87_write_value(data, it87_reg_pwm(data, nr),
+				 data->pwm_ctrl[nr]);
 	}
 	it87_unlock(data);
 	return count;
@@ -2790,6 +2938,9 @@ static int __init it87_find(int sioaddr, unsigned short *address,
 	case IT8623E_DEVID:
 		sio_data->type = it8603;
 		break;
+	case IT8613E_DEVID:
+		sio_data->type = it8613;
+		break;
 	case IT8620E_DEVID:
 		sio_data->type = it8620;
 		break;
@@ -2967,6 +3118,43 @@ static int __init it87_find(int sioaddr, unsigned short *address,
 		sio_data->skip_in |= BIT(5); /* No VIN5 */
 		sio_data->skip_in |= BIT(6); /* No VIN6 */
 
+		sio_data->beep_pin = superio_inb(sioaddr,
+						 IT87_SIO_BEEP_PIN_REG) & 0x3f;
+	} else if (sio_data->type == it8613) {
+		int reg27, reg29, reg2a;
+
+		superio_select(sioaddr, GPIO);
+
+		/* Check for pwm3, fan3, pwm5, fan5 */
+		reg27 = superio_inb(sioaddr, IT87_SIO_GPIO3_REG);
+		if (!(reg27 & BIT(1)))
+			sio_data->skip_fan |= BIT(4);
+		if (reg27 & BIT(3))
+			sio_data->skip_pwm |= BIT(4);
+		if (reg27 & BIT(6))
+			sio_data->skip_pwm |= BIT(2);
+		if (reg27 & BIT(7))
+			sio_data->skip_fan |= BIT(2);
+
+		/* Check for pwm2, fan2 */
+		reg29 = superio_inb(sioaddr, IT87_SIO_GPIO5_REG);
+		if (reg29 & BIT(1))
+			sio_data->skip_pwm |= BIT(1);
+		if (reg29 & BIT(2))
+			sio_data->skip_fan |= BIT(1);
+
+		/* Check for pwm4, fan4 */
+		reg2a = superio_inb(sioaddr, IT87_SIO_PINX1_REG);
+		if (!(reg2a & BIT(0)) || (reg29 & BIT(7))) {
+			sio_data->skip_fan |= BIT(3);
+			sio_data->skip_pwm |= BIT(3);
+		}
+
+		sio_data->skip_pwm |= BIT(0); /* No pwm1 */
+		sio_data->skip_fan |= BIT(0); /* No fan1 */
+		sio_data->skip_in |= BIT(3);  /* No VIN3 */
+		sio_data->skip_in |= BIT(6);  /* No VIN6 */
+
 		sio_data->beep_pin = superio_inb(sioaddr,
 						 IT87_SIO_BEEP_PIN_REG) & 0x3f;
 	} else if (sio_data->type == it8620 || sio_data->type == it8628) {
@@ -3350,7 +3538,10 @@ static void it87_init_device(struct platform_device *pdev)
 	 * manual duty cycle.
 	 */
 	for (i = 0; i < NUM_AUTO_PWM; i++) {
-		data->pwm_temp_map[i] = i;
+		if (uses_global_temp_map(data))
+			data->pwm_temp_map[i] = 0;
+		else
+			data->pwm_temp_map[i] = i % IT87_PWM_OLD_NUM_TEMP;
 		data->pwm_duty[i] = 0x7f;	/* Full speed */
 		data->auto_pwm[i][3] = 0x7f;	/* Full speed, hard-coded */
 	}
@@ -3461,6 +3652,7 @@ static int it87_probe(struct platform_device *pdev)
 	struct resource *res;
 	struct device *dev = &pdev->dev;
 	struct it87_sio_data *sio_data = dev_get_platdata(dev);
+	const struct it87_devices *chip;
 	int enable_pwm_interface;
 	struct device *hwmon_dev;
 	int err;
@@ -3483,9 +3675,19 @@ static int it87_probe(struct platform_device *pdev)
 	data->type = sio_data->type;
 	data->smbus_bitmap = sio_data->smbus_bitmap;
 	data->ec_special_config = sio_data->ec_special_config;
-	data->features = it87_devices[sio_data->type].features;
-	data->peci_mask = it87_devices[sio_data->type].peci_mask;
-	data->old_peci_mask = it87_devices[sio_data->type].old_peci_mask;
+	chip = &it87_devices[sio_data->type];
+	data->features = chip->features;
+	data->reg_pwm = chip->reg_pwm;
+	data->peci_mask = chip->peci_mask;
+	data->old_peci_mask = chip->old_peci_mask;
+	data->num_temp_map = chip->num_temp_map;
+	if (has_new_tempmap(data)) {
+		data->pwm_temp_map_mask = 0x07;
+		data->pwm_temp_map_shift = 3;
+	} else {
+		data->pwm_temp_map_mask = 0x03;
+		data->pwm_temp_map_shift = 0;
+	}
 	/*
 	 * IT8705F Datasheet 0.4.1, 3h == Version G.
 	 * IT8712F Datasheet 0.9.1, section 8.3.5 indicates 8h == Version J.
diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c
index 3e7e63edc6a300..76f8829dd63d37 100644
--- a/drivers/hwmon/k10temp.c
+++ b/drivers/hwmon/k10temp.c
@@ -64,7 +64,7 @@ MODULE_PARM_DESC(force, "force loading on processors with erratum 319");
 #define F15H_M60H_HARDWARE_TEMP_CTRL_OFFSET	0xd8200c64
 #define F15H_M60H_REPORTED_TEMP_CTRL_OFFSET	0xd8200ca4
 
-/* Common for Zen CPU families (Family 17h and 18h and 19h and 1Ah) */
+/* Common for Zen CPU families (Family 17h, 19h and 1Ah) */
 #define ZEN_REPORTED_TEMP_CTRL_BASE		0x00059800
 
 #define ZEN_CCD_TEMP(offset, x)			(ZEN_REPORTED_TEMP_CTRL_BASE + \
@@ -94,6 +94,7 @@ MODULE_PARM_DESC(force, "force loading on processors with erratum 319");
  * Defining locally as IDs are not shared.
  */
 #define PCI_DEVICE_ID_AMD_1AH_M50H_DF_F3	0x12cb
+#define PCI_DEVICE_ID_AMD_1AH_M80H_DF_F3	0x1243
 #define PCI_DEVICE_ID_AMD_1AH_M90H_DF_F3	0x127b
 
 struct k10temp_data {
@@ -475,7 +476,7 @@ static int k10temp_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		data->read_tempreg = read_tempreg_pci;
 	}
 
-	if (boot_cpu_data.x86 == 0x17 || boot_cpu_data.x86 == 0x18) {
+	if (boot_cpu_data.x86 == 0x17) {
 		switch (boot_cpu_data.x86_model) {
 		case 0x1:	/* Zen */
 		case 0x8:	/* Zen+ */
@@ -582,8 +583,8 @@ static const struct pci_device_id k10temp_id_table[] = {
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M50H_DF_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M60H_DF_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M70H_DF_F3) },
+	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M80H_DF_F3) },
 	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_1AH_M90H_DF_F3) },
-	{ PCI_VDEVICE(HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) },
 	{}
 };
 MODULE_DEVICE_TABLE(pci, k10temp_id_table);
diff --git a/drivers/hwmon/lm63.c b/drivers/hwmon/lm63.c
index da09770c05e7c3..607758bf7c58c2 100644
--- a/drivers/hwmon/lm63.c
+++ b/drivers/hwmon/lm63.c
@@ -35,13 +35,14 @@
 #include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/sysfs.h>
+#include <linux/thermal.h>
 #include <linux/types.h>
 
 /*
  * Addresses to scan
  * Address is fully defined internally and cannot be changed except for
  * LM64 which has one pin dedicated to address selection.
- * LM63 and LM96163 have address 0x4c.
+ * LM63, LM96163, and CTF2301 have address 0x4c.
  * LM64 can have address 0x18 or 0x4e.
  */
 
@@ -70,6 +71,9 @@ static const unsigned short normal_i2c[] = { 0x18, 0x4c, 0x4e, I2C_CLIENT_END };
 #define LM63_REG_LOCAL_TEMP		0x00
 #define LM63_REG_LOCAL_HIGH		0x05
 
+#define CTF2301_REG_LOCAL_TEMP_LSB	0x15
+#define CTF2301_REG_LOCAL_HIGH_LSB	0x06
+
 #define LM63_REG_REMOTE_TEMP_MSB	0x01
 #define LM63_REG_REMOTE_TEMP_LSB	0x10
 #define LM63_REG_REMOTE_OFFSET_MSB	0x11
@@ -94,6 +98,7 @@ static const unsigned short normal_i2c[] = { 0x18, 0x4c, 0x4e, I2C_CLIENT_END };
 
 #define LM63_PWM_BASE_FAST_HZ		180000
 #define LM63_PWM_BASE_SLOW_HZ		700
+#define LM63_MAX_PWM			255
 
 #define LM63_MAX_CONVRATE		9
 
@@ -121,6 +126,9 @@ static const unsigned short normal_i2c[] = { 0x18, 0x4c, 0x4e, I2C_CLIENT_END };
 							    127000), 1000)
 #define TEMP8U_TO_REG(val)	DIV_ROUND_CLOSEST(clamp_val((val), 0, \
 							    255000), 1000)
+#define TEMP12_FROM_REG(reg)	DIV_ROUND_CLOSEST((s16)(reg) * 1000, 256)
+#define TEMP12_TO_REG(val)	(DIV_ROUND_CLOSEST(clamp_val((val), -128000, \
+							     127937) * 16, 1000) * 16)
 #define TEMP11_FROM_REG(reg)	((reg) / 32 * 125)
 #define TEMP11_TO_REG(val)	(DIV_ROUND_CLOSEST(clamp_val((val), -128000, \
 							     127875), 125) * 32)
@@ -132,7 +140,11 @@ static const unsigned short normal_i2c[] = { 0x18, 0x4c, 0x4e, I2C_CLIENT_END };
 #define UPDATE_INTERVAL(max, rate) \
 			((1000 << (LM63_MAX_CONVRATE - (rate))) / (max))
 
-enum chips { lm63, lm64, lm96163 };
+enum chips { ctf2301, lm63, lm64, lm96163 };
+
+static const unsigned int ctf2301_update_intervals[] = {
+	20000, 10000, 4902, 2463, 1000, 615, 307, 154, 107
+};
 
 /*
  * Client data (each client gets its own)
@@ -164,6 +176,8 @@ struct lm63_data {
 			   1: local high limit
 			   2: remote critical limit
 			   3-14: lookup table */
+	s16 temp1_input; /* local input for chips with extended resolution */
+	s16 temp1_max; /* local limit for chips with extended resolution */
 	s16 temp11[4];	/* 0: remote input
 			   1: remote low limit
 			   2: remote high limit
@@ -229,6 +243,7 @@ static struct lm63_data *lm63_update_device(struct device *dev)
 	struct lm63_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
 	unsigned long next_update;
+	int msb, lsb;
 
 	mutex_lock(&data->update_lock);
 
@@ -253,10 +268,26 @@ static struct lm63_data *lm63_update_device(struct device *dev)
 		data->pwm1[0] = i2c_smbus_read_byte_data(client,
 				LM63_REG_PWM_VALUE);
 
-		data->temp8[0] = i2c_smbus_read_byte_data(client,
-				 LM63_REG_LOCAL_TEMP);
-		data->temp8[1] = i2c_smbus_read_byte_data(client,
-				 LM63_REG_LOCAL_HIGH);
+		if (data->kind == ctf2301) {
+			msb = i2c_smbus_read_byte_data(client,
+						       LM63_REG_LOCAL_TEMP);
+			lsb = i2c_smbus_read_byte_data(client,
+						       CTF2301_REG_LOCAL_TEMP_LSB);
+			if (msb >= 0 && lsb >= 0)
+				data->temp1_input = ((u16)(u8)msb << 8) | lsb;
+
+			msb = i2c_smbus_read_byte_data(client,
+						       LM63_REG_LOCAL_HIGH);
+			lsb = i2c_smbus_read_byte_data(client,
+						       CTF2301_REG_LOCAL_HIGH_LSB);
+			if (msb >= 0 && lsb >= 0)
+				data->temp1_max = ((u16)(u8)msb << 8) | lsb;
+		} else {
+			data->temp8[0] = i2c_smbus_read_byte_data(client,
+								  LM63_REG_LOCAL_TEMP);
+			data->temp8[1] = i2c_smbus_read_byte_data(client,
+								  LM63_REG_LOCAL_HIGH);
+		}
 
 		/* order matters for temp2_input */
 		data->temp11[0] = i2c_smbus_read_byte_data(client,
@@ -276,7 +307,7 @@ static struct lm63_data *lm63_update_device(struct device *dev)
 				| i2c_smbus_read_byte_data(client,
 				  LM63_REG_REMOTE_OFFSET_LSB);
 
-		if (data->kind == lm96163)
+		if (data->kind == lm96163 || data->kind == ctf2301)
 			data->temp11u = (i2c_smbus_read_byte_data(client,
 					LM96163_REG_REMOTE_TEMP_U_MSB) << 8)
 				      | i2c_smbus_read_byte_data(client,
@@ -366,6 +397,49 @@ static ssize_t set_fan(struct device *dev, struct device_attribute *dummy,
 	return count;
 }
 
+static int lm63_pwm_from_reg(struct lm63_data *data, u8 reg)
+{
+	if (data->pwm_highres)
+		return reg;
+
+	return reg >= 2 * data->pwm1_freq ?
+	       LM63_MAX_PWM :
+	       (reg * LM63_MAX_PWM + data->pwm1_freq) / (2 * data->pwm1_freq);
+}
+
+static u8 lm63_pwm_to_reg(struct lm63_data *data, unsigned long val)
+{
+	val = clamp_val(val, 0, LM63_MAX_PWM);
+
+	if (data->pwm_highres)
+		return val;
+
+	return (val * data->pwm1_freq * 2 + LM63_MAX_PWM / 2) / LM63_MAX_PWM;
+}
+
+static void lm63_update_pwm_highres(struct lm63_data *data,
+				    int config_enhanced)
+{
+	data->pwm_highres = !(data->config_fan & 0x08) &&
+			     data->pwm1_freq == 8 &&
+			     (config_enhanced & 0x10);
+}
+
+static int lm63_write_pwm(struct lm63_data *data, int nr, unsigned long val)
+{
+	struct i2c_client *client = data->client;
+	u8 reg = nr ? LM63_REG_LUT_PWM(nr - 1) : LM63_REG_PWM_VALUE;
+	u8 regval = lm63_pwm_to_reg(data, val);
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(client, reg, regval);
+	if (ret)
+		return ret;
+	data->pwm1[nr] = regval;
+
+	return 0;
+}
+
 static ssize_t show_pwm1(struct device *dev, struct device_attribute *devattr,
 			 char *buf)
 {
@@ -375,12 +449,7 @@ static ssize_t show_pwm1(struct device *dev, struct device_attribute *devattr,
 	int pwm;
 
 	mutex_lock(&data->update_lock);
-	if (data->pwm_highres)
-		pwm = data->pwm1[nr];
-	else
-		pwm = data->pwm1[nr] >= 2 * data->pwm1_freq ?
-		       255 : (data->pwm1[nr] * 255 + data->pwm1_freq) /
-		       (2 * data->pwm1_freq);
+	pwm = lm63_pwm_from_reg(data, data->pwm1[nr]);
 	mutex_unlock(&data->update_lock);
 
 	return sprintf(buf, "%d\n", pwm);
@@ -391,11 +460,9 @@ static ssize_t set_pwm1(struct device *dev, struct device_attribute *devattr,
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct lm63_data *data = dev_get_drvdata(dev);
-	struct i2c_client *client = data->client;
 	int nr = attr->index;
 	unsigned long val;
 	int err;
-	u8 reg;
 
 	if (!(data->config_fan & 0x20)) /* register is read-only */
 		return -EPERM;
@@ -404,15 +471,11 @@ static ssize_t set_pwm1(struct device *dev, struct device_attribute *devattr,
 	if (err)
 		return err;
 
-	reg = nr ? LM63_REG_LUT_PWM(nr - 1) : LM63_REG_PWM_VALUE;
-	val = clamp_val(val, 0, 255);
-
 	mutex_lock(&data->update_lock);
-	data->pwm1[nr] = data->pwm_highres ? val :
-			(val * data->pwm1_freq * 2 + 127) / 255;
-	i2c_smbus_write_byte_data(client, reg, data->pwm1[nr]);
+	err = lm63_write_pwm(data, nr, val);
 	mutex_unlock(&data->update_lock);
-	return count;
+
+	return err ? err : count;
 }
 
 static ssize_t pwm1_enable_show(struct device *dev,
@@ -485,7 +548,9 @@ static ssize_t pwm1_freq_store(struct device *dev,
 	struct lm63_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
 	unsigned long val, pfr_fast, pfr_slow, err_fast, err_slow, pfr;
+	u8 config_fan;
 	bool slow_clock;
+	int config_enhanced = 0;
 	int ret;
 
 	ret = kstrtoul(buf, 10, &val);
@@ -511,35 +576,58 @@ static ssize_t pwm1_freq_store(struct device *dev,
 
 	mutex_lock(&data->update_lock);
 	ret = i2c_smbus_read_byte_data(client, LM63_REG_CONFIG_FAN);
-	if (ret < 0) {
-		mutex_unlock(&data->update_lock);
-		return ret;
-	}
-	data->config_fan = ret;
+	if (ret < 0)
+		goto unlock;
+	config_fan = ret;
+	data->config_fan = config_fan;
 
-	if (!(data->config_fan & 0x20)) { /* register is read-only */
-		mutex_unlock(&data->update_lock);
-		return -EPERM;
+	if (!(config_fan & 0x20)) { /* register is read-only */
+		ret = -EPERM;
+		goto unlock;
 	}
 
-	if (data->kind == lm96163) {
-		ret = i2c_smbus_read_byte_data(client, LM96163_REG_CONFIG_ENHANCED);
-		if (ret < 0) {
-			mutex_unlock(&data->update_lock);
-			return ret;
+	if (data->kind == lm96163 || data->kind == ctf2301) {
+		ret = i2c_smbus_read_byte_data(client,
+					       LM96163_REG_CONFIG_ENHANCED);
+		if (ret < 0)
+			goto unlock;
+		config_enhanced = ret;
+		lm63_update_pwm_highres(data, config_enhanced);
+
+		if (data->kind == ctf2301) {
+			if (!slow_clock && pfr == 8)
+				config_enhanced |= 0x10;
+			else
+				config_enhanced &= ~0x10;
+			ret = i2c_smbus_write_byte_data(client,
+							LM96163_REG_CONFIG_ENHANCED,
+							config_enhanced);
+			if (ret)
+				goto unlock;
+			lm63_update_pwm_highres(data, config_enhanced);
 		}
-		data->pwm_highres = !slow_clock && pfr == 8 && (ret & 0x10);
 	}
 
 	if (slow_clock)
-		data->config_fan |= 0x08;
+		config_fan |= 0x08;
 	else
-		data->config_fan &= ~0x08;
-	i2c_smbus_write_byte_data(client, LM63_REG_CONFIG_FAN, data->config_fan);
-	i2c_smbus_write_byte_data(client, LM63_REG_PWM_FREQ, pfr);
+		config_fan &= ~0x08;
+	ret = i2c_smbus_write_byte_data(client, LM63_REG_PWM_FREQ, pfr);
+	if (ret)
+		goto unlock;
 	data->pwm1_freq = pfr;
+	lm63_update_pwm_highres(data, config_enhanced);
+	ret = i2c_smbus_write_byte_data(client, LM63_REG_CONFIG_FAN,
+					config_fan);
+	if (ret)
+		goto unlock;
+
+	data->config_fan = config_fan;
+	lm63_update_pwm_highres(data, config_enhanced);
+unlock:
 	mutex_unlock(&data->update_lock);
-	return count;
+
+	return ret ? ret : count;
 }
 
 /*
@@ -554,7 +642,17 @@ static ssize_t show_local_temp8(struct device *dev,
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct lm63_data *data = lm63_update_device(dev);
-	return sprintf(buf, "%d\n", TEMP8_FROM_REG(data->temp8[attr->index]));
+	int temp;
+
+	mutex_lock(&data->update_lock);
+	if (data->kind == ctf2301)
+		temp = TEMP12_FROM_REG(attr->index ? data->temp1_max :
+				       data->temp1_input);
+	else
+		temp = TEMP8_FROM_REG(data->temp8[attr->index]);
+	mutex_unlock(&data->update_lock);
+
+	return sprintf(buf, "%d\n", temp);
 }
 
 static ssize_t show_remote_temp8(struct device *dev,
@@ -594,6 +692,21 @@ static ssize_t set_temp8(struct device *dev, struct device_attribute *devattr,
 		return err;
 
 	mutex_lock(&data->update_lock);
+	if (data->kind == ctf2301 && nr == 1) {
+		temp = TEMP12_TO_REG(val);
+		err = i2c_smbus_write_byte_data(client, LM63_REG_LOCAL_HIGH,
+						temp >> 8);
+		if (!err)
+			err = i2c_smbus_write_byte_data(client,
+							CTF2301_REG_LOCAL_HIGH_LSB,
+							temp & 0xf0);
+		if (!err)
+			data->temp1_max = temp;
+		mutex_unlock(&data->update_lock);
+
+		return err ? err : count;
+	}
+
 	switch (nr) {
 	case 2:
 		reg = LM63_REG_REMOTE_TCRIT;
@@ -610,10 +723,12 @@ static ssize_t set_temp8(struct device *dev, struct device_attribute *devattr,
 		reg = LM63_REG_LUT_TEMP(nr - 3);
 		temp = lut_temp_to_reg(data, val);
 	}
-	data->temp8[nr] = temp;
-	i2c_smbus_write_byte_data(client, reg, temp);
+	err = i2c_smbus_write_byte_data(client, reg, temp);
+	if (!err)
+		data->temp8[nr] = temp;
 	mutex_unlock(&data->update_lock);
-	return count;
+
+	return err ? err : count;
 }
 
 static ssize_t show_temp11(struct device *dev, struct device_attribute *devattr,
@@ -787,11 +902,40 @@ static ssize_t temp2_crit_hyst_store(struct device *dev,
  * Set conversion rate.
  * client->update_lock must be held when calling this function.
  */
-static void lm63_set_convrate(struct lm63_data *data, unsigned int interval)
+static unsigned int lm63_update_interval(struct lm63_data *data, u8 rate)
+{
+	if (data->kind == ctf2301)
+		return ctf2301_update_intervals[min_t(unsigned int, rate,
+					ARRAY_SIZE(ctf2301_update_intervals) - 1)];
+
+	return UPDATE_INTERVAL(data->max_convrate_hz, rate);
+}
+
+static int lm63_set_convrate(struct lm63_data *data, unsigned int interval)
 {
 	struct i2c_client *client = data->client;
-	unsigned int update_interval;
-	int i;
+	unsigned int update_interval, error, best_error = ~0U;
+	unsigned int best = 0;
+	unsigned int i;
+	int ret;
+
+	if (data->kind == ctf2301) {
+		for (i = 0; i < ARRAY_SIZE(ctf2301_update_intervals); i++) {
+			error = abs_diff(interval, ctf2301_update_intervals[i]);
+			if (error < best_error) {
+				best_error = error;
+				best = i;
+			}
+		}
+
+		ret = i2c_smbus_write_byte_data(client, LM63_REG_CONVRATE,
+						best);
+		if (ret)
+			return ret;
+		data->update_interval = ctf2301_update_intervals[best];
+
+		return 0;
+	}
 
 	/* Shift calculations to avoid rounding errors */
 	interval <<= 6;
@@ -803,8 +947,12 @@ static void lm63_set_convrate(struct lm63_data *data, unsigned int interval)
 		if (interval >= update_interval * 3 / 4)
 			break;
 
-	i2c_smbus_write_byte_data(client, LM63_REG_CONVRATE, i);
+	ret = i2c_smbus_write_byte_data(client, LM63_REG_CONVRATE, i);
+	if (ret)
+		return ret;
 	data->update_interval = UPDATE_INTERVAL(data->max_convrate_hz, i);
+
+	return 0;
 }
 
 static ssize_t update_interval_show(struct device *dev,
@@ -828,10 +976,10 @@ static ssize_t update_interval_store(struct device *dev,
 		return err;
 
 	mutex_lock(&data->update_lock);
-	lm63_set_convrate(data, clamp_val(val, 0, 100000));
+	err = lm63_set_convrate(data, clamp_val(val, 0, 100000));
 	mutex_unlock(&data->update_lock);
 
-	return count;
+	return err ? err : count;
 }
 
 static ssize_t temp2_type_show(struct device *dev,
@@ -996,6 +1144,75 @@ static DEVICE_ATTR_RO(alarms);
 
 static DEVICE_ATTR_RW(update_interval);
 
+static int lm63_get_max_state(struct thermal_cooling_device *cdev,
+			      unsigned long *state)
+{
+	*state = LM63_MAX_PWM;
+
+	return 0;
+}
+
+static int lm63_get_cur_state(struct thermal_cooling_device *cdev,
+			      unsigned long *state)
+{
+	struct lm63_data *data = cdev->devdata;
+	int ret;
+
+	mutex_lock(&data->update_lock);
+	ret = i2c_smbus_read_byte_data(data->client, LM63_REG_PWM_VALUE);
+	if (ret < 0)
+		goto unlock;
+
+	data->pwm1[0] = ret;
+	*state = lm63_pwm_from_reg(data, data->pwm1[0]);
+unlock:
+	mutex_unlock(&data->update_lock);
+
+	return ret < 0 ? ret : 0;
+}
+
+static int lm63_set_cur_state(struct thermal_cooling_device *cdev,
+			      unsigned long state)
+{
+	struct lm63_data *data = cdev->devdata;
+	int config_fan;
+	int ret;
+
+	if (state > LM63_MAX_PWM)
+		return -EINVAL;
+
+	mutex_lock(&data->update_lock);
+	config_fan = i2c_smbus_read_byte_data(data->client,
+					      LM63_REG_CONFIG_FAN);
+	if (config_fan < 0) {
+		ret = config_fan;
+		goto unlock;
+	}
+	data->config_fan = config_fan;
+
+	if (!(config_fan & 0x20)) {
+		config_fan |= 0x20;
+		ret = i2c_smbus_write_byte_data(data->client,
+						LM63_REG_CONFIG_FAN,
+						config_fan);
+		if (ret)
+			goto unlock;
+		data->config_fan = config_fan;
+	}
+
+	ret = lm63_write_pwm(data, 0, state);
+unlock:
+	mutex_unlock(&data->update_lock);
+
+	return ret;
+}
+
+static const struct thermal_cooling_device_ops lm63_cooling_ops = {
+	.get_max_state = lm63_get_max_state,
+	.get_cur_state = lm63_get_cur_state,
+	.set_cur_state = lm63_set_cur_state,
+};
+
 static struct attribute *lm63_attributes[] = {
 	&sensor_dev_attr_pwm1.dev_attr.attr,
 	&dev_attr_pwm1_enable.attr,
@@ -1078,8 +1295,8 @@ static const struct attribute_group lm63_group_extra_lut = {
  * On LM63, temp2_crit can be set only once, which should be job
  * of the bootloader.
  * On LM64, temp2_crit can always be set.
- * On LM96163, temp2_crit can be set if bit 1 of the configuration
- * register is true.
+ * On LM96163 and CTF2301, temp2_crit can be set if bit 1 of the
+ * configuration register is true.
  */
 static umode_t lm63_attribute_mode(struct kobject *kobj,
 				   struct attribute *attr, int index)
@@ -1089,7 +1306,8 @@ static umode_t lm63_attribute_mode(struct kobject *kobj,
 
 	if (attr == &sensor_dev_attr_temp2_crit.dev_attr.attr
 	    && (data->kind == lm64 ||
-		(data->kind == lm96163 && (data->config & 0x02))))
+		((data->kind == lm96163 || data->kind == ctf2301) &&
+		 (data->config & 0x02))))
 		return attr->mode | S_IWUSR;
 
 	return attr->mode;
@@ -1164,29 +1382,48 @@ static int lm63_detect(struct i2c_client *client,
  * Ideally we shouldn't have to initialize anything, since the BIOS
  * should have taken care of everything
  */
-static void lm63_init_client(struct lm63_data *data)
+static int lm63_init_client(struct lm63_data *data)
 {
 	struct i2c_client *client = data->client;
 	struct device *dev = &client->dev;
 	u8 convrate;
+	int ret;
 
-	data->config = i2c_smbus_read_byte_data(client, LM63_REG_CONFIG1);
-	data->config_fan = i2c_smbus_read_byte_data(client,
-						    LM63_REG_CONFIG_FAN);
+	ret = i2c_smbus_read_byte_data(client, LM63_REG_CONFIG1);
+	if (ret < 0)
+		return ret;
+	data->config = ret;
+	ret = i2c_smbus_read_byte_data(client, LM63_REG_CONFIG_FAN);
+	if (ret < 0)
+		return ret;
+	data->config_fan = ret;
 
 	/* Start converting if needed */
 	if (data->config & 0x40) { /* standby */
 		dev_dbg(dev, "Switching to operational mode\n");
 		data->config &= 0xA7;
-		i2c_smbus_write_byte_data(client, LM63_REG_CONFIG1,
-					  data->config);
+		ret = i2c_smbus_write_byte_data(client, LM63_REG_CONFIG1,
+						data->config);
+		if (ret)
+			return ret;
 	}
 	/* Tachometer is always enabled on LM64 */
 	if (data->kind == lm64)
 		data->config |= 0x04;
+	if (data->kind == ctf2301 && !client->irq && !(data->config & 0x04)) {
+		dev_dbg(dev, "Configuring ALERT/TACH pin for tachometer input\n");
+		data->config |= 0x04;
+		ret = i2c_smbus_write_byte_data(client, LM63_REG_CONFIG1,
+						data->config);
+		if (ret)
+			return ret;
+	}
 
 	/* We may need pwm1_freq before ever updating the client data */
-	data->pwm1_freq = i2c_smbus_read_byte_data(client, LM63_REG_PWM_FREQ) & 0x1f;
+	ret = i2c_smbus_read_byte_data(client, LM63_REG_PWM_FREQ);
+	if (ret < 0)
+		return ret;
+	data->pwm1_freq = ret & 0x1f;
 	if (data->pwm1_freq == 0)
 		data->pwm1_freq = 1;
 
@@ -1199,25 +1436,38 @@ static void lm63_init_client(struct lm63_data *data)
 	case lm96163:
 		data->max_convrate_hz = LM96163_MAX_CONVRATE_HZ;
 		data->lut_size = 12;
-		data->trutherm
-		  = i2c_smbus_read_byte_data(client,
-					     LM96163_REG_TRUTHERM) & 0x02;
+		ret = i2c_smbus_read_byte_data(client, LM96163_REG_TRUTHERM);
+		if (ret < 0)
+			return ret;
+		data->trutherm = ret & 0x02;
+		break;
+	case ctf2301:
+		data->lut_size = 12;
 		break;
 	}
-	convrate = i2c_smbus_read_byte_data(client, LM63_REG_CONVRATE);
-	if (unlikely(convrate > LM63_MAX_CONVRATE))
+	ret = i2c_smbus_read_byte_data(client, LM63_REG_CONVRATE);
+	if (ret < 0)
+		return ret;
+	convrate = ret;
+	if (data->kind == ctf2301 &&
+	    convrate >= ARRAY_SIZE(ctf2301_update_intervals))
+		convrate = ARRAY_SIZE(ctf2301_update_intervals) - 1;
+	else if (unlikely(convrate > LM63_MAX_CONVRATE))
 		convrate = LM63_MAX_CONVRATE;
-	data->update_interval = UPDATE_INTERVAL(data->max_convrate_hz,
-						convrate);
+	data->update_interval = lm63_update_interval(data, convrate);
 
 	/*
-	 * For LM96163, check if high resolution PWM
+	 * For LM96163 and CTF2301, check if high resolution PWM
 	 * and unsigned temperature format is enabled.
 	 */
-	if (data->kind == lm96163) {
-		u8 config_enhanced
-		  = i2c_smbus_read_byte_data(client,
-					     LM96163_REG_CONFIG_ENHANCED);
+	if (data->kind == lm96163 || data->kind == ctf2301) {
+		u8 config_enhanced;
+
+		ret = i2c_smbus_read_byte_data(client,
+					       LM96163_REG_CONFIG_ENHANCED);
+		if (ret < 0)
+			return ret;
+		config_enhanced = ret;
 		if (config_enhanced & 0x20)
 			data->lut_temp_highres = true;
 		if ((config_enhanced & 0x10)
@@ -1238,6 +1488,8 @@ static void lm63_init_client(struct lm63_data *data)
 	dev_dbg(dev, "PWM output active %s, %s mode\n",
 		(data->config_fan & 0x10) ? "low" : "high",
 		(data->config_fan & 0x20) ? "manual" : "auto");
+
+	return 0;
 }
 
 static const struct i2c_device_id lm63_id[];
@@ -1245,9 +1497,11 @@ static const struct i2c_device_id lm63_id[];
 static int lm63_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
+	struct thermal_cooling_device *cdev;
 	struct device *hwmon_dev;
 	struct lm63_data *data;
 	int groups = 0;
+	int ret;
 
 	data = devm_kzalloc(dev, sizeof(struct lm63_data), GFP_KERNEL);
 	if (!data)
@@ -1262,7 +1516,9 @@ static int lm63_probe(struct i2c_client *client)
 		data->temp2_offset = 16000;
 
 	/* Initialize chip */
-	lm63_init_client(data);
+	ret = lm63_init_client(data);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to initialize device\n");
 
 	/* Register sysfs hooks */
 	data->groups[groups++] = &lm63_group;
@@ -1272,11 +1528,26 @@ static int lm63_probe(struct i2c_client *client)
 	if (data->kind == lm96163) {
 		data->groups[groups++] = &lm63_group_temp2_type;
 		data->groups[groups++] = &lm63_group_extra_lut;
+	} else if (data->kind == ctf2301) {
+		data->groups[groups++] = &lm63_group_extra_lut;
 	}
 
 	hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
 							   data, data->groups);
-	return PTR_ERR_OR_ZERO(hwmon_dev);
+	if (IS_ERR(hwmon_dev))
+		return PTR_ERR(hwmon_dev);
+
+	if (IS_ENABLED(CONFIG_THERMAL_OF) &&
+	    of_property_present(dev->of_node, "#cooling-cells")) {
+		cdev = devm_thermal_of_cooling_device_register(dev, 0,
+							       client->name,
+							       data,
+							       &lm63_cooling_ops);
+		if (IS_ERR(cdev))
+			return PTR_ERR(cdev);
+	}
+
+	return 0;
 }
 
 /*
@@ -1287,6 +1558,7 @@ static const struct i2c_device_id lm63_id[] = {
 	{ .name = "lm63", .driver_data = lm63 },
 	{ .name = "lm64", .driver_data = lm64 },
 	{ .name = "lm96163", .driver_data = lm96163 },
+	{ .name = "ctf2301", .driver_data = ctf2301 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm63_id);
@@ -1304,6 +1576,10 @@ static const struct of_device_id __maybe_unused lm63_of_match[] = {
 		.compatible = "national,lm96163",
 		.data = (void *)lm96163
 	},
+	{
+		.compatible = "sensylink,ctf2301",
+		.data = (void *)ctf2301
+	},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, lm63_of_match);
diff --git a/drivers/hwmon/lm70.c b/drivers/hwmon/lm70.c
index 8b525725fef4f1..b29bac7a4cc14c 100644
--- a/drivers/hwmon/lm70.c
+++ b/drivers/hwmon/lm70.c
@@ -96,21 +96,21 @@ static ssize_t temp1_input_show(struct device *dev,
 	 */
 	switch (p_lm70->chip) {
 	case LM70_CHIP_LM70:
-		val = ((int)raw / 32) * 250;
+		val = ((int)raw >> 5) * 250;
 		break;
 
 	case LM70_CHIP_TMP121:
 	case LM70_CHIP_TMP122:
 	case LM70_CHIP_LM74:
-		val = ((int)raw / 8) * 625 / 10;
+		val = ((int)raw >> 3) * 625 / 10;
 		break;
 
 	case LM70_CHIP_LM71:
-		val = ((int)raw / 4) * 3125 / 100;
+		val = ((int)raw >> 2) * 3125 / 100;
 		break;
 
 	case LM70_CHIP_TMP125:
-		val = (sign_extend32(raw, 14) / 32) * 250;
+		val = (sign_extend32(raw, 14) >> 5) * 250;
 		break;
 	}
 
diff --git a/drivers/hwmon/lm85.c b/drivers/hwmon/lm85.c
index c56e164d61c185..27ef127ccc2c47 100644
--- a/drivers/hwmon/lm85.c
+++ b/drivers/hwmon/lm85.c
@@ -1445,7 +1445,7 @@ static void lm85_init_client(struct i2c_client *client)
 static int lm85_is_fake(struct i2c_client *client)
 {
 	/*
-	 * Differenciate between real LM96000 and Winbond WPCD377I. The latter
+	 * Differentiate between real LM96000 and Winbond WPCD377I. The latter
 	 * emulate the former except that it has no hardware monitoring function
 	 * so the readings are always 0.
 	 */
diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c
index 1c603272538aa1..3faeb2c6ab01b2 100644
--- a/drivers/hwmon/lm90.c
+++ b/drivers/hwmon/lm90.c
@@ -2712,6 +2712,11 @@ static int lm90_probe_channel(struct i2c_client *client,
 		return -EINVAL;
 	}
 
+	if (id == 2 && !(data->flags & LM90_HAVE_TEMP3)) {
+		dev_err(dev, "channel %d is not supported for this chip in %pfw\n", id, child);
+		return -EINVAL;
+	}
+
 	err = fwnode_property_read_string(child, "label", &data->channel_label[id]);
 	if (err == -ENODATA || err == -EILSEQ) {
 		dev_err(dev, "invalid label property in %pfw\n", child);
diff --git a/drivers/hwmon/lm95245.c b/drivers/hwmon/lm95245.c
index 11553391f54b1e..925574b8b0ca4d 100644
--- a/drivers/hwmon/lm95245.c
+++ b/drivers/hwmon/lm95245.c
@@ -96,9 +96,9 @@ static int temp_from_reg_unsigned(u8 val_h, u8 val_l)
 
 static int temp_from_reg_signed(u8 val_h, u8 val_l)
 {
-	if (val_h & 0x80)
-		return (val_h - 0x100) * 1000;
-	return temp_from_reg_unsigned(val_h, val_l);
+	s16 val_hl = (val_h << 8) | val_l;
+
+	return val_hl * 1000 / 256;
 }
 
 static int lm95245_read_conversion_rate(struct lm95245_data *data)
diff --git a/drivers/hwmon/minisforum-um780xtx.c b/drivers/hwmon/minisforum-um780xtx.c
new file mode 100644
index 00000000000000..f5d6e18cb651f5
--- /dev/null
+++ b/drivers/hwmon/minisforum-um780xtx.c
@@ -0,0 +1,501 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Minisforum UM780 XTX (F7BSD) embedded-controller hwmon driver.
+ *
+ * Copyright (C) 2026 Sebastián Peyrott <speyrott@gmail.com>
+ */
+
+#include <linux/acpi.h>
+#include <linux/cleanup.h>
+#include <linux/dmi.h>
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pm.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#define UM780XTX_EC_TEMP_SYS	0x05
+#define UM780XTX_EC_TEMP_CPU	0x09
+#define UM780XTX_EC_CPU_PROFILE	0x2f
+#define UM780XTX_EC_SYS_POINT1	0x31
+#define UM780XTX_EC_SYS_POINT2	0x34
+#define UM780XTX_EC_SYS_POINT3	0x37
+
+#define UM780XTX_EC_PROFILE_B1	0xb1
+#define UM780XTX_EC_PROFILE_B2	0xb2
+
+#define UM780XTX_EC_FAN1_LO	0xb6
+#define UM780XTX_EC_FAN1_HI	0xb7
+#define UM780XTX_EC_FAN2_LO	0xb9
+#define UM780XTX_EC_FAN2_HI	0xba
+
+#define UM780XTX_EC_MAX_RPM	9000
+#define UM780XTX_EC_RPM_RETRIES	3
+
+static struct platform_device *um780xtx_pdev;
+
+struct um780xtx_data {
+	struct device *hwmon_dev;
+	u8 saved_profile;
+	u8 saved_sys_point1;
+	u8 saved_sys_point2;
+	u8 sys_point3;
+};
+
+static const struct dmi_system_id um780xtx_dmi_table[] = {
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR,
+					"Micro Computer (HK) Tech Limited"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Venus series"),
+			DMI_EXACT_MATCH(DMI_BOARD_VENDOR,
+					"Shenzhen Meigao Electronic Equipment Co.,Ltd"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "F7BSD"),
+		},
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(dmi, um780xtx_dmi_table);
+
+static bool um780xtx_firmware_match(void)
+{
+	const char *board_version = dmi_get_system_info(DMI_BOARD_VERSION);
+	const char *bios_version = dmi_get_system_info(DMI_BIOS_VERSION);
+
+	return board_version && bios_version &&
+		!strcmp(board_version, "1.1") && !strcmp(bios_version, "1.06");
+}
+
+static int um780xtx_oem_read(u8 command, u8 *value)
+{
+	return ec_transaction(command, NULL, 0, value, 1);
+}
+
+static int um780xtx_read_rpm(u8 command_hi, u8 command_lo, long *rpm)
+{
+	u8 hi_before;
+	u8 hi_after;
+	u8 lo;
+	unsigned int value;
+	int attempt;
+	int ret;
+
+	for (attempt = 0; attempt < UM780XTX_EC_RPM_RETRIES; attempt++) {
+		ret = um780xtx_oem_read(command_hi, &hi_before);
+		if (ret)
+			return ret;
+		ret = um780xtx_oem_read(command_lo, &lo);
+		if (ret)
+			return ret;
+		ret = um780xtx_oem_read(command_hi, &hi_after);
+		if (ret)
+			return ret;
+		if (hi_before != hi_after)
+			continue;
+
+		value = (hi_after << 8) | lo;
+		if (value > UM780XTX_EC_MAX_RPM)
+			continue;
+
+		*rpm = value;
+		return 0;
+	}
+
+	return -EIO;
+}
+
+static int um780xtx_read_profile(long *mode)
+{
+	u8 profile;
+	int ret;
+
+	ret = ec_read(UM780XTX_EC_CPU_PROFILE, &profile);
+	if (ret)
+		return ret;
+	if (profile == UM780XTX_EC_PROFILE_B1) {
+		*mode = 2;
+		return 0;
+	}
+	if (profile == UM780XTX_EC_PROFILE_B2) {
+		*mode = 3;
+		return 0;
+	}
+
+	return -ENODATA;
+}
+
+static int um780xtx_write_profile(struct um780xtx_data *data, long mode)
+{
+	u8 expected;
+	u8 profile;
+	int ret;
+
+	if (mode != 2 && mode != 3)
+		return -EINVAL;
+	expected = mode == 2 ? UM780XTX_EC_PROFILE_B1 : UM780XTX_EC_PROFILE_B2;
+
+	ret = ec_transaction(expected, NULL, 0, NULL, 0);
+	if (ret)
+		return ret;
+	ret = ec_read(UM780XTX_EC_CPU_PROFILE, &profile);
+	if (ret)
+		return ret;
+	if (profile != expected)
+		return -EIO;
+
+	data->saved_profile = profile;
+	return 0;
+}
+
+static const u8 um780xtx_sys_point_offsets[] = {
+	UM780XTX_EC_SYS_POINT1,
+	UM780XTX_EC_SYS_POINT2,
+};
+
+static ssize_t um780xtx_sys_point_temp_show(struct device *dev,
+					    struct device_attribute *attr,
+					    char *buf)
+{
+	struct sensor_device_attribute *sattr = to_sensor_dev_attr(attr);
+	u8 value;
+	int ret;
+
+	guard(hwmon_lock)(dev);
+	ret = ec_read(um780xtx_sys_point_offsets[sattr->index], &value);
+	if (ret)
+		return ret;
+
+	return sysfs_emit(buf, "%u\n", value * 1000);
+}
+
+static ssize_t um780xtx_sys_point_temp_store(struct device *dev,
+					     struct device_attribute *attr,
+					     const char *buf, size_t count)
+{
+	struct um780xtx_data *data = dev_get_drvdata(dev);
+	struct sensor_device_attribute *sattr = to_sensor_dev_attr(attr);
+	u8 points[2];
+	u8 readback;
+	long value;
+	int ret;
+
+	ret = kstrtol(buf, 10, &value);
+	if (ret)
+		return ret;
+
+	guard(hwmon_lock)(dev);
+	/* Validate and clamp against the live peer threshold. */
+	ret = ec_read(UM780XTX_EC_SYS_POINT1, &points[0]);
+	if (ret)
+		return ret;
+	ret = ec_read(UM780XTX_EC_SYS_POINT2, &points[1]);
+	if (ret)
+		return ret;
+	if (points[0] >= points[1] || points[1] >= data->sys_point3)
+		return -EIO;
+
+	value = DIV_ROUND_CLOSEST(value, 1000);
+	if (!sattr->index)
+		value = clamp_val(value, 0, points[1] - 1);
+	else
+		value = clamp_val(value, points[0] + 1,
+				  data->sys_point3 - 1);
+	points[sattr->index] = value;
+
+	ret = ec_write(um780xtx_sys_point_offsets[sattr->index],
+		       points[sattr->index]);
+	if (ret)
+		return ret;
+	ret = ec_read(um780xtx_sys_point_offsets[sattr->index], &readback);
+	if (ret)
+		return ret;
+	if (readback != points[sattr->index])
+		return -EIO;
+
+	data->saved_sys_point1 = points[0];
+	data->saved_sys_point2 = points[1];
+	return count;
+}
+
+static SENSOR_DEVICE_ATTR_RW(pwm2_auto_point1_temp,
+			     um780xtx_sys_point_temp, 0);
+static SENSOR_DEVICE_ATTR_RW(pwm2_auto_point2_temp,
+			     um780xtx_sys_point_temp, 1);
+
+static struct attribute *um780xtx_extra_attrs[] = {
+	&sensor_dev_attr_pwm2_auto_point1_temp.dev_attr.attr,
+	&sensor_dev_attr_pwm2_auto_point2_temp.dev_attr.attr,
+	NULL
+};
+
+static const struct attribute_group um780xtx_extra_group = {
+	.attrs = um780xtx_extra_attrs,
+};
+
+static const struct attribute_group *um780xtx_extra_groups[] = {
+	&um780xtx_extra_group,
+	NULL
+};
+
+static umode_t um780xtx_is_visible(const void *data,
+				   enum hwmon_sensor_types type,
+				   u32 attr, int channel)
+{
+	if (type == hwmon_temp || type == hwmon_fan)
+		return 0444;
+	if (type == hwmon_pwm && attr == hwmon_pwm_enable)
+		return 0644;
+	return 0;
+}
+
+static int um780xtx_read(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long *value)
+{
+	u8 raw;
+	int ret;
+
+	if (type == hwmon_temp) {
+		ret = ec_read(channel ? UM780XTX_EC_TEMP_SYS :
+			      UM780XTX_EC_TEMP_CPU, &raw);
+		if (ret)
+			return ret;
+		*value = raw * 1000L;
+		return 0;
+	}
+	if (type == hwmon_fan) {
+		if (!channel)
+			ret = um780xtx_read_rpm(UM780XTX_EC_FAN1_HI,
+						UM780XTX_EC_FAN1_LO, value);
+		else
+			ret = um780xtx_read_rpm(UM780XTX_EC_FAN2_HI,
+						UM780XTX_EC_FAN2_LO, value);
+		return ret;
+	}
+	if (type == hwmon_pwm)
+		return um780xtx_read_profile(value);
+
+	return -EOPNOTSUPP;
+}
+
+static int um780xtx_write(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int channel, long value)
+{
+	struct um780xtx_data *data = dev_get_drvdata(dev);
+
+	if (type == hwmon_pwm && attr == hwmon_pwm_enable)
+		return um780xtx_write_profile(data, value);
+
+	return -EOPNOTSUPP;
+}
+
+static int um780xtx_read_string(struct device *dev,
+				enum hwmon_sensor_types type,
+				u32 attr, int channel, const char **str)
+{
+	static const char * const labels[] = { "CPU", "SYS" };
+
+	if (type == hwmon_temp && attr == hwmon_temp_label)
+		*str = labels[channel];
+	else if (type == hwmon_fan && attr == hwmon_fan_label)
+		*str = labels[channel];
+	else
+		return -EOPNOTSUPP;
+	return 0;
+}
+
+static const struct hwmon_ops um780xtx_hwmon_ops = {
+	.is_visible = um780xtx_is_visible,
+	.read = um780xtx_read,
+	.write = um780xtx_write,
+	.read_string = um780xtx_read_string,
+};
+
+static const struct hwmon_channel_info * const um780xtx_info[] = {
+	HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL),
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_LABEL,
+			   HWMON_F_INPUT | HWMON_F_LABEL),
+	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_ENABLE),
+	NULL
+};
+
+static const struct hwmon_chip_info um780xtx_chip_info = {
+	.ops = &um780xtx_hwmon_ops,
+	.info = um780xtx_info,
+};
+
+static int um780xtx_write_sys_point(u8 offset, u8 value)
+{
+	u8 readback;
+	int ret;
+
+	ret = ec_write(offset, value);
+	if (ret)
+		return ret;
+	ret = ec_read(offset, &readback);
+	if (ret)
+		return ret;
+
+	return readback == value ? 0 : -EIO;
+}
+
+static int um780xtx_restore_sys_points(struct um780xtx_data *data,
+				       u8 current_point2)
+{
+	u8 point1 = data->saved_sys_point1;
+	u8 point2 = data->saved_sys_point2;
+	int ret;
+
+	/* Keep strict ordering valid after each individual EC write. */
+	if (point1 >= current_point2) {
+		ret = um780xtx_write_sys_point(UM780XTX_EC_SYS_POINT2, point2);
+		if (ret)
+			return ret;
+		return um780xtx_write_sys_point(UM780XTX_EC_SYS_POINT1,
+						 point1);
+	}
+
+	ret = um780xtx_write_sys_point(UM780XTX_EC_SYS_POINT1, point1);
+	if (ret)
+		return ret;
+	return um780xtx_write_sys_point(UM780XTX_EC_SYS_POINT2, point2);
+}
+
+static int um780xtx_read_initial_state(struct um780xtx_data *data)
+{
+	u8 profile;
+	u8 point1;
+	u8 point2;
+	int ret;
+
+	ret = ec_read(UM780XTX_EC_CPU_PROFILE, &profile);
+	if (ret)
+		return ret;
+	if (profile != UM780XTX_EC_PROFILE_B1 &&
+	    profile != UM780XTX_EC_PROFILE_B2)
+		return -EINVAL;
+
+	ret = ec_read(UM780XTX_EC_SYS_POINT1, &point1);
+	if (ret)
+		return ret;
+	ret = ec_read(UM780XTX_EC_SYS_POINT2, &point2);
+	if (ret)
+		return ret;
+	if (point1 >= point2 || point2 >= data->sys_point3)
+		return -EINVAL;
+
+	data->saved_profile = profile;
+	data->saved_sys_point1 = point1;
+	data->saved_sys_point2 = point2;
+	return 0;
+}
+
+static int um780xtx_restore_state(struct um780xtx_data *data)
+{
+	u8 readback;
+	u8 point2;
+	int ret;
+
+	ret = ec_transaction(data->saved_profile, NULL, 0, NULL, 0);
+	if (ret)
+		return ret;
+	ret = ec_read(UM780XTX_EC_CPU_PROFILE, &readback);
+	if (ret)
+		return ret;
+	if (readback != data->saved_profile)
+		return -EIO;
+
+	ret = ec_read(UM780XTX_EC_SYS_POINT2, &point2);
+	if (ret)
+		return ret;
+
+	return um780xtx_restore_sys_points(data, point2);
+}
+
+static int um780xtx_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct um780xtx_data *data;
+	struct device *hwmon;
+	int ret;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, data);
+
+	ret = ec_read(UM780XTX_EC_SYS_POINT3, &data->sys_point3);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to read system fan limit\n");
+	ret = um780xtx_read_initial_state(data);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to read initial fan settings\n");
+
+	hwmon = devm_hwmon_device_register_with_info(dev, "um780xtx_ec", data,
+						     &um780xtx_chip_info,
+						     um780xtx_extra_groups);
+	if (IS_ERR(hwmon))
+		return PTR_ERR(hwmon);
+	data->hwmon_dev = hwmon;
+
+	return 0;
+}
+
+static int um780xtx_resume(struct device *dev)
+{
+	struct um780xtx_data *data = dev_get_drvdata(dev);
+
+	guard(hwmon_lock)(data->hwmon_dev);
+	return um780xtx_restore_state(data);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(um780xtx_pm_ops, NULL, um780xtx_resume);
+
+static struct platform_driver um780xtx_driver = {
+	.driver = {
+		.name = "um780xtx-ec-hwmon",
+		.pm = pm_sleep_ptr(&um780xtx_pm_ops),
+	},
+	.probe = um780xtx_probe,
+};
+
+static int __init um780xtx_init(void)
+{
+	int ret;
+
+	if (!dmi_check_system(um780xtx_dmi_table) ||
+	    !um780xtx_firmware_match() || !ec_get_handle())
+		return -ENODEV;
+	ret = platform_driver_register(&um780xtx_driver);
+	if (ret)
+		return ret;
+
+	um780xtx_pdev = platform_device_register_simple("um780xtx-ec-hwmon",
+							PLATFORM_DEVID_NONE,
+						       NULL, 0);
+	if (IS_ERR(um780xtx_pdev)) {
+		ret = PTR_ERR(um780xtx_pdev);
+		platform_driver_unregister(&um780xtx_driver);
+		return ret;
+	}
+	return 0;
+}
+
+static void __exit um780xtx_exit(void)
+{
+	platform_device_unregister(um780xtx_pdev);
+	platform_driver_unregister(&um780xtx_driver);
+}
+
+module_init(um780xtx_init);
+module_exit(um780xtx_exit);
+
+MODULE_AUTHOR("Sebastián Peyrott <speyrott@gmail.com>");
+MODULE_DESCRIPTION("Minisforum UM780 XTX EC hwmon driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/nct6775-core.c b/drivers/hwmon/nct6775-core.c
index 759ea3d1286e35..cf27b9c7a89ed4 100644
--- a/drivers/hwmon/nct6775-core.c
+++ b/drivers/hwmon/nct6775-core.c
@@ -21,6 +21,8 @@
  * Chip        #vin    #fan    #pwm    #temp  chip IDs       man ID
  * nct6106d     9      3       3       6+3    0xc450 0xc1    0x5ca3
  * nct6116d     9      5       5       3+3    0xd280 0xc1    0x5ca3
+ * nct6126d    10      5       5       3+3    0xd283 0xc1    0x5ca3
+ *                                           (0xd284)
  * nct6775f     9      4       3       6+3    0xb470 0xc1    0x5ca3
  * nct6776f     9      5       3       6+3    0xc330 0xc1    0x5ca3
  * nct6779d    15      5       5       2+6    0xc560 0xc1    0x5ca3
@@ -66,6 +68,7 @@
 static const char * const nct6775_device_names[] = {
 	[nct6106] = "nct6106",
 	[nct6116] = "nct6116",
+	[nct6126] = "nct6126",
 	[nct6775] = "nct6775",
 	[nct6776] = "nct6776",
 	[nct6779] = "nct6779",
@@ -897,6 +900,31 @@ static const s8 NCT6116_BEEP_BITS[NUM_BEEP_BITS] = {
 
 static const u16 NCT6116_REG_TSI_TEMP[] = { 0x59, 0x5b };
 
+/* NCT6122D/NCT6126D specific data */
+
+static const u16 NCT6126_REG_IN[] = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09 };
+static const u16 NCT6126_REG_IN_MAX[] = {
+	0x90, 0x92, 0x94, 0x96, 0x98, 0x9a, 0x9c, 0x9e, 0xa0, 0xa2 };
+static const u16 NCT6126_REG_IN_MIN[] = {
+	0x91, 0x93, 0x95, 0x97, 0x99, 0x9b, 0x9d, 0x9f, 0xa1, 0xa3 };
+
+static const s8 NCT6126_ALARM_BITS[NUM_ALARM_BITS] = {
+	 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, -1, -1,	  /* in0-in11     */
+	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,	  /* in12-in23    */
+	32, 33, 34, 35, 36, -1, -1, -1, -1, -1, -1, -1,	  /* fan1-fan12   */
+	16, 17, 18, -1, -1, -1, -1, -1, -1, -1, -1, -1,	  /* temp1-temp12 */
+	48, -1,						  /* intr0-intr1  */
+};
+
+static const s8 NCT6126_BEEP_BITS[NUM_BEEP_BITS] = {
+	 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11,	  /* in0-in11     */
+	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,	  /* in12-in23    */
+	24, 25, 26, 27, 28, -1, -1, -1, -1, -1, -1, -1,	  /* fan1-fan12   */
+	16, 17, 18, -1, -1, -1, -1, -1, -1, -1, -1, -1,	  /* temp1-temp12 */
+	34, -1, 32					  /* intr0-intr1, beep_en */
+};
+
 static enum pwm_enable reg_to_pwm_enable(int pwm, int mode)
 {
 	if (mode == 0 && pwm == 255)
@@ -998,6 +1026,14 @@ static const u16 scale_in_6798[NUM_IN] = {
 	800, 800,  800, 1600, 800
 };
 
+/*
+ * NCT6126 scaling:
+ * CPUVCORE, VIN0, AVSB, 3VCC, VIN1, VIN2, VHIF, 3VSB, VBAT,  VTT
+ */
+static const u16 scale_in_6126[10] = {
+	800, 800, 1600, 1600, 800, 800, 1600, 1600, 1600, 800
+};
+
 static inline long in_from_reg(u8 reg, u8 nr, const u16 *scales)
 {
 	return DIV_ROUND_CLOSEST(reg * scales[nr], 100);
@@ -1156,6 +1192,7 @@ bool nct6775_reg_is_word_sized(struct nct6775_data *data, u16 reg)
 		  reg == 0xe0 || reg == 0xe2 || reg == 0xe4 ||
 		  reg == 0x111 || reg == 0x121 || reg == 0x131;
 	case nct6116:
+	case nct6126:
 		return reg == 0x20 || reg == 0x22 || reg == 0x24 ||
 		  reg == 0x26 || reg == 0x28 || reg == 0x59 || reg == 0x5b ||
 		  reg == 0xe0 || reg == 0xe2 || reg == 0xe4 || reg == 0xe6 ||
@@ -1538,6 +1575,7 @@ static int nct6775_update_pwm_limits(struct device *dev)
 			break;
 		case nct6106:
 		case nct6116:
+		case nct6126:
 		case nct6779:
 		case nct6791:
 		case nct6792:
@@ -3203,6 +3241,7 @@ store_auto_pwm(struct device *dev, struct device_attribute *attr,
 			break; /* always enabled, nothing to do */
 		case nct6106:
 		case nct6116:
+		case nct6126:
 		case nct6779:
 		case nct6791:
 		case nct6792:
@@ -3677,6 +3716,83 @@ int nct6775_probe(struct device *dev, struct nct6775_data *data,
 		reg_temp_crit_l = NCT6106_REG_TEMP_CRIT_L;
 		reg_temp_crit_h = NCT6106_REG_TEMP_CRIT_H;
 
+		break;
+	case nct6126:
+		data->in_num = 10;
+		data->scale_in = scale_in_6126;
+		data->pwm_num = 5;
+		data->auto_pwm_num = 4;
+		data->temp_fixed_num = 3;
+		data->num_temp_alarms = 3;
+		data->num_temp_beeps = 3;
+
+		data->fan_from_reg = fan_from_reg13;
+		data->fan_from_reg_min = fan_from_reg13;
+
+		data->temp_label = nct6776_temp_label;
+		data->temp_mask = NCT6776_TEMP_MASK;
+		data->virt_temp_mask = NCT6776_VIRT_TEMP_MASK;
+
+		data->REG_VBAT = NCT6106_REG_VBAT;
+		data->REG_DIODE = NCT6106_REG_DIODE;
+		data->DIODE_MASK = NCT6106_DIODE_MASK;
+		data->REG_VIN = NCT6126_REG_IN;
+		data->REG_IN_MINMAX[0] = NCT6126_REG_IN_MIN;
+		data->REG_IN_MINMAX[1] = NCT6126_REG_IN_MAX;
+		data->REG_TARGET = NCT6116_REG_TARGET;
+		data->REG_FAN = NCT6116_REG_FAN;
+		data->REG_FAN_MODE = NCT6116_REG_FAN_MODE;
+		data->REG_FAN_MIN = NCT6116_REG_FAN_MIN;
+		data->REG_FAN_PULSES = NCT6116_REG_FAN_PULSES;
+		data->FAN_PULSE_SHIFT = NCT6116_FAN_PULSE_SHIFT;
+		data->REG_FAN_TIME[0] = NCT6116_REG_FAN_STOP_TIME;
+		data->REG_FAN_TIME[1] = NCT6116_REG_FAN_STEP_UP_TIME;
+		data->REG_FAN_TIME[2] = NCT6116_REG_FAN_STEP_DOWN_TIME;
+		data->REG_TOLERANCE_H = NCT6116_REG_TOLERANCE_H;
+		data->REG_PWM[0] = NCT6116_REG_PWM;
+		data->REG_PWM[1] = NCT6116_REG_FAN_START_OUTPUT;
+		data->REG_PWM[2] = NCT6116_REG_FAN_STOP_OUTPUT;
+		data->REG_PWM[5] = NCT6106_REG_WEIGHT_DUTY_STEP;
+		data->REG_PWM[6] = NCT6106_REG_WEIGHT_DUTY_BASE;
+		data->REG_PWM_READ = NCT6106_REG_PWM_READ;
+		data->REG_PWM_MODE = NCT6106_REG_PWM_MODE;
+		data->PWM_MODE_MASK = NCT6106_PWM_MODE_MASK;
+		data->REG_AUTO_TEMP = NCT6116_REG_AUTO_TEMP;
+		data->REG_AUTO_PWM = NCT6116_REG_AUTO_PWM;
+		data->REG_CRITICAL_TEMP = NCT6116_REG_CRITICAL_TEMP;
+		data->REG_CRITICAL_TEMP_TOLERANCE =
+			NCT6116_REG_CRITICAL_TEMP_TOLERANCE;
+		data->REG_CRITICAL_PWM_ENABLE = NCT6116_REG_CRITICAL_PWM_ENABLE;
+		data->CRITICAL_PWM_ENABLE_MASK =
+			NCT6106_CRITICAL_PWM_ENABLE_MASK;
+		data->REG_CRITICAL_PWM = NCT6116_REG_CRITICAL_PWM;
+		data->REG_TEMP_OFFSET = NCT6106_REG_TEMP_OFFSET;
+		data->REG_TEMP_SOURCE = NCT6106_REG_TEMP_SOURCE;
+		data->REG_TEMP_SEL = NCT6116_REG_TEMP_SEL;
+		data->REG_WEIGHT_TEMP_SEL = NCT6106_REG_WEIGHT_TEMP_SEL;
+		data->REG_WEIGHT_TEMP[0] = NCT6106_REG_WEIGHT_TEMP_STEP;
+		data->REG_WEIGHT_TEMP[1] = NCT6106_REG_WEIGHT_TEMP_STEP_TOL;
+		data->REG_WEIGHT_TEMP[2] = NCT6106_REG_WEIGHT_TEMP_BASE;
+		data->REG_ALARM = NCT6106_REG_ALARM;
+		data->ALARM_BITS = NCT6126_ALARM_BITS;
+		data->REG_BEEP = NCT6106_REG_BEEP;
+		data->BEEP_BITS = NCT6126_BEEP_BITS;
+		data->REG_TSI_TEMP = NCT6116_REG_TSI_TEMP;
+
+		reg_temp = NCT6106_REG_TEMP;
+		reg_temp_mon = NCT6106_REG_TEMP_MON;
+		num_reg_temp = 3;
+		num_reg_temp_mon = ARRAY_SIZE(NCT6106_REG_TEMP_MON);
+		num_reg_tsi_temp = ARRAY_SIZE(NCT6116_REG_TSI_TEMP);
+		reg_temp_over = NCT6106_REG_TEMP_OVER;
+		reg_temp_hyst = NCT6106_REG_TEMP_HYST;
+		reg_temp_config = NCT6106_REG_TEMP_CONFIG;
+		num_reg_temp_config = 3;
+		reg_temp_alternate = NCT6106_REG_TEMP_ALTERNATE;
+		reg_temp_crit = NCT6106_REG_TEMP_CRIT;
+		reg_temp_crit_l = NCT6106_REG_TEMP_CRIT_L;
+		reg_temp_crit_h = NCT6106_REG_TEMP_CRIT_H;
+
 		break;
 	case nct6775:
 		data->in_num = 9;
diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c
index d394376e430686..d359c28f3e0f32 100644
--- a/drivers/hwmon/nct6775-platform.c
+++ b/drivers/hwmon/nct6775-platform.c
@@ -25,6 +25,7 @@ enum sensor_access { access_direct, access_asuswmi };
 static const char * const nct6775_sio_names[] __initconst = {
 	[nct6106] = "NCT6106D",
 	[nct6116] = "NCT6116D",
+	[nct6126] = "NCT6122D/NCT6126D",
 	[nct6775] = "NCT6775F",
 	[nct6776] = "NCT6776D/F",
 	[nct6779] = "NCT6779D",
@@ -76,6 +77,9 @@ MODULE_PARM_DESC(fan_debounce, "Enable debouncing for fan RPM signal");
 
 #define SIO_NCT6106_ID		0xc450
 #define SIO_NCT6116_ID		0xd280
+#define SIO_NCT6122_ID		0xd2a0
+#define SIO_NCT6126_A_ID	0xd283
+#define SIO_NCT6126_B_ID	0xd284
 #define SIO_NCT6775_ID		0xb470
 #define SIO_NCT6776_ID		0xc330
 #define SIO_NCT6779_ID		0xc560
@@ -565,6 +569,25 @@ nct6775_check_fan_inputs(struct nct6775_data *data, struct nct6775_sio_data *sio
 		pwm3pin = fan3pin && (cr24 & 0x08);
 		pwm4pin = fan4pin;
 		pwm5pin = fan5pin;
+	} else if (data->kind == nct6126) {
+		int cr1a = sio_data->sio_inb(sio_data, 0x1a);
+		int cr1b = sio_data->sio_inb(sio_data, 0x1b);
+		int cr24 = sio_data->sio_inb(sio_data, 0x24);
+		int cr2a = sio_data->sio_inb(sio_data, 0x2a);
+		int cr2b = sio_data->sio_inb(sio_data, 0x2b);
+		int cr2f = sio_data->sio_inb(sio_data, 0x2f);
+
+		fan3pin = !(cr2b & 0x10);
+		fan4pin = (cr2b & 0x80) ||			/* pin 1(2) */
+			(!(cr2f & 0x10) && (cr1a & 0x04));	/* pin 65(66) */
+		fan5pin = (cr2b & 0x80) ||			/* pin 126(127) */
+			(!(cr1b & 0x03) && (cr2a & 0x02));	/* pin 94(96) */
+
+		pwm3pin = fan3pin && (cr24 & 0x08);
+		pwm4pin = fan4pin;
+		pwm5pin = fan5pin;
+
+		fan4min = fan4pin;
 	} else {
 		/*
 		 * NCT6779D, NCT6791D, NCT6792D, NCT6793D, NCT6795D, NCT6796D,
@@ -861,6 +884,7 @@ static int nct6775_platform_probe_init(struct nct6775_data *data)
 		break;
 	case nct6106:
 	case nct6116:
+	case nct6126:
 	case nct6779:
 	case nct6791:
 	case nct6792:
@@ -892,6 +916,7 @@ static int nct6775_platform_probe_init(struct nct6775_data *data)
 		switch (data->kind) {
 		case nct6106:
 		case nct6116:
+		case nct6126:
 			tmp |= 0xe0;
 			break;
 		case nct6775:
@@ -1006,7 +1031,13 @@ static int __init nct6775_find(int sioaddr, struct nct6775_sio_data *sio_data)
 		sio_data->kind = nct6106;
 		break;
 	case SIO_NCT6116_ID:
-		sio_data->kind = nct6116;
+		if (val == SIO_NCT6126_A_ID || val == SIO_NCT6126_B_ID)
+			sio_data->kind = nct6126;
+		else
+			sio_data->kind = nct6116;
+		break;
+	case SIO_NCT6122_ID:
+		sio_data->kind = nct6126;
 		break;
 	case SIO_NCT6775_ID:
 		sio_data->kind = nct6775;
diff --git a/drivers/hwmon/nct6775.h b/drivers/hwmon/nct6775.h
index 296eff99d00383..d18ac91be49718 100644
--- a/drivers/hwmon/nct6775.h
+++ b/drivers/hwmon/nct6775.h
@@ -4,8 +4,8 @@
 
 #include <linux/types.h>
 
-enum kinds { nct6106, nct6116, nct6775, nct6776, nct6779, nct6791, nct6792,
-	     nct6793, nct6795, nct6796, nct6797, nct6798, nct6799 };
+enum kinds { nct6106, nct6116, nct6126, nct6775, nct6776, nct6779, nct6791,
+	     nct6792, nct6793, nct6795, nct6796, nct6797, nct6798, nct6799 };
 enum pwm_enable { off, manual, thermal_cruise, speed_cruise, sf3, sf4 };
 
 #define NUM_TEMP	12	/* Max number of temp attribute sets w/ limits*/
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index eb71a0ac044b26..fd6641316454f0 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -393,6 +393,25 @@ config SENSORS_MAX20751
 	  This driver can also be built as a module. If so, the module will
 	  be called max20751.
 
+config SENSORS_MAX20826
+	tristate "Analog Devices MAX20826 and similar devices"
+	help
+	  If you say yes here you get hardware monitoring support for Analog
+	  Devices MAX20826 and similar devices.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called max20826.
+
+config SENSORS_MAX20826_REGULATOR
+	bool "Regulator support for Analog Devices MAX20826 and similar devices"
+	depends on SENSORS_MAX20826 && REGULATOR
+	help
+	  If you say yes here you get regulator support for Analog Devices
+	  MAX20826 and similar sensors.
+
+	  This enables the MAX20826 to be used as a regulator device,
+	  providing voltage control through the regulator framework.
+
 config SENSORS_MAX20830
 	tristate "Analog Devices MAX20830 and compatibles"
 	help
@@ -435,8 +454,8 @@ config SENSORS_MAX34440
 	help
 	  If you say yes here you get hardware monitoring support for Maxim
 	  MAX34440, MAX34441, MAX34446, MAX34451, MAX34452, MAX34460, and
-	  MAX34461. Other compatible devices are ADPM12160, ADPM12200, and
-	  ADPM12250.
+	  MAX34461. Other compatible devices are ADPM12160, ADPM12200,
+	  ADPM12250, and ADPM12300.
 
 	  This driver can also be built as a module. If so, the module will
 	  be called max34440.
@@ -763,6 +782,15 @@ config SENSORS_TDA38640_REGULATOR
 	  If you say yes here you get regulator support for Infineon
 	  TDA38640 as regulator.
 
+config SENSORS_TDA38740
+	tristate "Infineon TDA38725/TDA38740"
+	help
+	  If you say yes here you get hardware monitoring support for Infineon
+	  TDA38725 and TDA38740.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called tda38740.
+
 config SENSORS_TPS25990
 	tristate "TI TPS25990"
 	help
@@ -790,10 +818,11 @@ config SENSORS_TPS40422
 	  be called tps40422.
 
 config SENSORS_TPS53679
-	tristate "TI TPS53647, TPS53667, TPS53676, TPS53679, TPS53681, TPS53688"
+	tristate "TI TPS536xx family"
 	help
 	  If you say yes here you get hardware monitoring support for TI
-	  TPS53647, TPS53667, TPS53676, TPS53679, TPS53681, and TPS53688.
+	  TPS53622, TPS53647, TPS53659, TPS53667, TPS53676, TPS53679, TPS53681,
+	  TPS53685, TPS53688, and TPS536C7.
 
 	  This driver can also be built as a module. If so, the module will
 	  be called tps53679.
@@ -867,7 +896,7 @@ config SENSORS_XDPE1A2G7B
 	tristate "Infineon XDPE1A2G7B"
 	help
 	  If you say yes here you get hardware monitoring support for Infineon
-	  XDPE1A2G5B and XDPE1A2G7B.
+	  XDPE1A2G5B,XDPE1A2G7B and XDPE1A2G7C.
 
 	  This driver can also be built as a module. If so, the module will
 	  be called xdpe1a2g7b.
@@ -877,7 +906,7 @@ config SENSORS_XDPE1A2G7B_REGULATOR
 	depends on SENSORS_XDPE1A2G7B && REGULATOR
 	help
 	  If you say yes here you get regulator support for Infineon
-	  XDPE1A2G5B and XDPE1A2G7B multi-phase digital controllers.
+	  XDPE1A2G5B,XDPE1A2G7B and XDPE1A2G7C multi-phase digital controllers.
 
 	  This enables the controllers to be used as regulator devices,
 	  providing voltage control through the regulator framework.
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index e288fe72a43726..b67ec382328ee7 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -38,6 +38,7 @@ obj-$(CONFIG_SENSORS_MAX16601)	+= max16601.o
 obj-$(CONFIG_SENSORS_MAX17616)	+= max17616.o
 obj-$(CONFIG_SENSORS_MAX20730)	+= max20730.o
 obj-$(CONFIG_SENSORS_MAX20751)	+= max20751.o
+obj-$(CONFIG_SENSORS_MAX20826)	+= max20826.o
 obj-$(CONFIG_SENSORS_MAX20830)	+= max20830.o
 obj-$(CONFIG_SENSORS_MAX20860A)	+= max20860a.o
 obj-$(CONFIG_SENSORS_MAX31785)	+= max31785.o
@@ -70,6 +71,7 @@ obj-$(CONFIG_SENSORS_STEF48H28)	+= stef48h28.o
 obj-$(CONFIG_SENSORS_SQ24860)	+= sq24860.o
 obj-$(CONFIG_SENSORS_STPDDC60)	+= stpddc60.o
 obj-$(CONFIG_SENSORS_TDA38640)	+= tda38640.o
+obj-$(CONFIG_SENSORS_TDA38740)	+= tda38740.o
 obj-$(CONFIG_SENSORS_TPS25990)	+= tps25990.o
 obj-$(CONFIG_SENSORS_TPS40422)	+= tps40422.o
 obj-$(CONFIG_SENSORS_TPS53679)	+= tps53679.o
diff --git a/drivers/hwmon/pmbus/ibm-cffps.c b/drivers/hwmon/pmbus/ibm-cffps.c
index aad94bcb9ceb5f..2498c6f4778ecc 100644
--- a/drivers/hwmon/pmbus/ibm-cffps.c
+++ b/drivers/hwmon/pmbus/ibm-cffps.c
@@ -166,7 +166,7 @@ static ssize_t ibm_cffps_debugfs_read(struct file *file, char __user *buf,
 				if (rc < 0)
 					goto unlock;
 
-				snprintf(&data[i * 2], 3, "%02X", rc);
+				snprintf(&data[i * 2], 3, "%02X", (u8)rc);
 			}
 
 			rc = i * 2;
@@ -177,7 +177,7 @@ static ssize_t ibm_cffps_debugfs_read(struct file *file, char __user *buf,
 				if (rc < 0)
 					goto unlock;
 
-				snprintf(&data[i * 4], 5, "%04X", rc);
+				snprintf(&data[i * 4], 5, "%04X", (u16)rc);
 			}
 
 			rc = i * 4;
diff --git a/drivers/hwmon/pmbus/isl68137.c b/drivers/hwmon/pmbus/isl68137.c
index 2f7f825bfb69e0..15b4e3f15f11f2 100644
--- a/drivers/hwmon/pmbus/isl68137.c
+++ b/drivers/hwmon/pmbus/isl68137.c
@@ -456,6 +456,8 @@ static const struct i2c_device_id raa_dmpvr_id[] = {
 	{ .name = "raa229004", .driver_data = raa_dmpvr2_2rail },
 	{ .name = "raa229141", .driver_data = raa_dmpvr2_2rail_pmbus },
 	{ .name = "raa229621", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "raa229639", .driver_data = raa_dmpvr2_2rail_nontc },
+	{ .name = "raa229640", .driver_data = raa_dmpvr2_2rail_nontc },
 	{ }
 };
 
@@ -506,6 +508,8 @@ static const struct of_device_id isl68137_of_match[] = {
 	{ .compatible = "renesas,raa229001", .data = (void *)raa_dmpvr2_2rail },
 	{ .compatible = "renesas,raa229004", .data = (void *)raa_dmpvr2_2rail },
 	{ .compatible = "renesas,raa229621", .data = (void *)raa_dmpvr2_2rail },
+	{ .compatible = "renesas,raa229639", .data = (void *)raa_dmpvr2_2rail_nontc },
+	{ .compatible = "renesas,raa229640", .data = (void *)raa_dmpvr2_2rail_nontc },
 	{ },
 };
 
diff --git a/drivers/hwmon/pmbus/ltc4286.c b/drivers/hwmon/pmbus/ltc4286.c
index 8715d380784a02..8190706d1bebb9 100644
--- a/drivers/hwmon/pmbus/ltc4286.c
+++ b/drivers/hwmon/pmbus/ltc4286.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include <linux/err.h>
+#include <linux/hwmon-sysfs.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -16,6 +17,85 @@
 
 #define LTC4286_MFR_ID_SIZE	3
 
+struct ltc4286_data {
+	struct pmbus_driver_info info;
+	u32 rsense;
+	bool vrange_low_enable;
+};
+
+#define to_ltc4286_data(x) container_of((x), struct ltc4286_data, info)
+
+static ssize_t ltc4286_rsense_show(struct device *dev,
+				   struct device_attribute *devattr,
+				   char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev->parent);
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct ltc4286_data *data = to_ltc4286_data(info);
+	u32 rsense;
+	int ret;
+
+	ret = pmbus_lock_interruptible(client);
+	if (ret)
+		return ret;
+
+	rsense = data->rsense;
+
+	pmbus_unlock(client);
+
+	return sysfs_emit(buf, "%u\n", rsense);
+}
+
+static ssize_t ltc4286_rsense_store(struct device *dev,
+				    struct device_attribute *devattr,
+				    const char *buf, size_t count)
+{
+	struct i2c_client *client = to_i2c_client(dev->parent);
+	const struct pmbus_driver_info *info_ro = pmbus_get_driver_info(client);
+	struct ltc4286_data *data = to_ltc4286_data(info_ro);
+	struct pmbus_driver_info *info = &data->info;
+	u32 rsense;
+	int ret;
+
+	ret = kstrtou32(buf, 10, &rsense);
+	if (ret)
+		return ret;
+
+	if (rsense == 0)
+		return -EINVAL;
+
+	if (rsense > (INT_MAX / 1024))
+		return -EINVAL;
+
+	ret = pmbus_lock_interruptible(client);
+	if (ret)
+		return ret;
+
+	data->rsense = rsense;
+	info->m[PSC_CURRENT_OUT] = 1024 * rsense;
+	info->m[PSC_POWER] = data->vrange_low_enable ? 4 * rsense : rsense;
+
+	pmbus_unlock(client);
+
+	return count;
+}
+
+static SENSOR_DEVICE_ATTR_RW(shunt_resistor, ltc4286_rsense, 0);
+
+static struct attribute *ltc4286_attrs[] = {
+	&sensor_dev_attr_shunt_resistor.dev_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group ltc4286_group = {
+	.attrs = ltc4286_attrs,
+};
+
+static const struct attribute_group *ltc4286_attribute_groups[] = {
+	&ltc4286_group,
+	NULL,
+};
+
 /*
  * Initialize the MBR as default settings which is referred to LTC4286 datasheet
  * (March 22, 2022 version) table 3 page 16
@@ -55,6 +135,7 @@ static struct pmbus_driver_info ltc4286_info = {
 	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT |
 		   PMBUS_HAVE_PIN | PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_VOUT |
 		   PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_STATUS_TEMP,
+	.groups = ltc4286_attribute_groups,
 };
 
 static const struct i2c_device_id ltc4286_id[] = {
@@ -69,6 +150,7 @@ static int ltc4286_probe(struct i2c_client *client)
 	int ret;
 	const struct i2c_device_id *mid;
 	u8 block_buffer[I2C_SMBUS_BLOCK_MAX + 1];
+	struct ltc4286_data *data;
 	struct pmbus_driver_info *info;
 	u32 rsense;
 	int vrange_nval, vrange_oval;
@@ -114,11 +196,14 @@ static int ltc4286_probe(struct i2c_client *client)
 	if (rsense > (INT_MAX / 1024))
 		return -EINVAL;
 
-	info = devm_kmemdup(&client->dev, &ltc4286_info, sizeof(*info),
-			    GFP_KERNEL);
-	if (!info)
+	data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
 		return -ENOMEM;
 
+	data->info = ltc4286_info;
+	data->rsense = rsense;
+	info = &data->info;
+
 	/* Check MFR1 CONFIG register bit 1 VRANGE_SELECT before driver loading */
 	vrange_oval = i2c_smbus_read_word_data(client, LTC4286_MFR_CONFIG1);
 	if (vrange_oval < 0)
@@ -126,7 +211,10 @@ static int ltc4286_probe(struct i2c_client *client)
 				     "Failed to read manufacturer configuration one\n");
 	vrange_nval = vrange_oval;
 
-	if (device_property_read_bool(&client->dev, "adi,vrange-low-enable")) {
+	data->vrange_low_enable =
+		device_property_read_bool(&client->dev, "adi,vrange-low-enable");
+
+	if (data->vrange_low_enable) {
 		vrange_nval &=
 			~VRANGE_SELECT_BIT; /* VRANGE_SELECT = 0, 25.6 volts */
 
diff --git a/drivers/hwmon/pmbus/max20826.c b/drivers/hwmon/pmbus/max20826.c
new file mode 100644
index 00000000000000..013a97189fde7a
--- /dev/null
+++ b/drivers/hwmon/pmbus/max20826.c
@@ -0,0 +1,1044 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Hardware monitoring driver for Analog Devices MAX20826 PMBus device
+ *
+ * Copyright 2026 Analog Devices Inc.
+ */
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/pmbus.h>
+#include <linux/property.h>
+#include <linux/regulator/driver.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+
+#include "pmbus.h"
+
+#define MAX20826_REG_RAIL_PHASE_CFG	0xB1
+#define MAX20826_REG_CTRL_MISC		0xCB
+#define MAX20826_OPL_EN_MSK		BIT(7)
+
+#define MAX20826_REG_VOUT_RES		0xDB
+#define MAX20826_VOUT_RES_MSK		BIT(7)
+
+#define MAX20855B_REG_VOUT_VRM		0xD1
+#define MAX20855B_REG_VOUT_VRM_MASK	BIT(4)
+
+#define MAX20826_REG_C_MODEAB		0xDC
+/* Byte 1 bits 7 and 6*/
+#define MAX20826_C_MODEAB_MASK		GENMASK(15, 14)
+
+#define MAX20826_REG_ADDR_MODE		0xEC
+#define MAX20826_PAGE_MODE_MSK		BIT(7)
+#define MAX20826_DIRECT_ADDR_MSK	GENMASK(6, 0)
+
+#define MAX20826_REG_OVERRIDE		0xED
+#define MAX20826_OVERRIDE_MASK		BIT(7)
+
+#define MAX20826_REG_PHASE_DETECT	0xF3
+#define MAX20826_REG_PHASE_READ		0xF4
+
+#define MAX20826_REG_STATUS_MON		0xF9
+#define MAX20826_PHASES_NUM_MASK	GENMASK(7, 3)
+
+#define MAX20826_MAX_PAGES		2
+#define MAX20826_MAX_PHASES		16
+#define MAX20826_PHASES_PER_PAGE	8
+#define MAX20826_INTF_PWMVID		1
+#define MAX20826_INTF_AVSBUS		3
+
+#define MAX20855B_PHASES_NUM_MASK	GENMASK(7, 4)
+#define MAX20855B_MAX_PHASES		8
+#define MAX20908_MAX_PHASES		8
+#define MAX20912_MAX_PHASES		12
+#define MAX20916_MAX_PHASES		16
+
+struct max20826_chip_info {
+	const char *vendor_bus_name;
+	u8 max_phases;
+	unsigned int phase_num_mask;
+	u8 start_index_iin;
+	u8 start_index_iout;
+	bool is_reg_addr_mode_block;
+	bool is_vout_direct;
+	bool select_vrm;
+	bool has_avsbus;
+	bool has_opl;
+	u8 (*count_phases)(const u8 *config, int page);
+};
+
+struct max20826 {
+	const struct max20826_chip_info *chip_info;
+	struct pmbus_driver_info info;
+	struct i2c_client *client;
+	/* RAIL-B direct mode */
+	struct i2c_client *client_b;
+	struct i2c_client *curr_client;
+	struct gpio_desc *avren;
+	struct gpio_desc *bvren;
+	bool vendor_bus;
+	bool high_speed[MAX20826_MAX_PAGES];
+	bool on_off_ctrl[MAX20826_MAX_PAGES];
+	bool opl_enabled[MAX20826_MAX_PAGES];
+};
+
+struct max20826_debugfs_entry {
+	struct max20826 *st;
+	u8 page;
+};
+
+static u8 __max20826_count_phases(const u8 *config, int page)
+{
+	if (page)
+		return hweight8(config[4]);
+
+	return hweight8(config[0]) + hweight8(config[1]) -
+	       hweight8(config[4]);
+}
+
+static u8 __max20855b_count_phases(const u8 *config, int page)
+{
+	if (page)
+		return hweight8(config[3] & 0x3F);
+
+	return hweight8(config[0]) + hweight8(config[1] & 0x0F) -
+	       hweight8(config[3] & 0x3F);
+}
+
+static u8 __max20908_count_phases(const u8 *config, int page)
+{
+	if (page)
+		return hweight8(config[4]);
+
+	return hweight8(config[0]) + hweight8(config[1] & 0xF0) -
+	       hweight8(config[4]);
+}
+
+static u8 __max20912_count_phases(const u8 *config, int page)
+{
+	if (page)
+		return hweight8(config[4]);
+
+	return hweight8(config[0]) + hweight8(config[1] & 0xFC) -
+	       hweight8(config[4]);
+}
+
+#define to_max20826(p)	container_of(p, struct max20826, info)
+
+enum {
+	RAIL_A,
+	RAIL_B,
+};
+
+static const struct regulator_desc __maybe_unused max20826_reg_desc[] = {
+	PMBUS_REGULATOR("vout", 0),
+	PMBUS_REGULATOR("vout", 1),
+};
+
+static struct i2c_client *max20826_select_rail(struct max20826 *st,
+					       int page, bool probing)
+{
+	int ret;
+
+	/*
+	 * If in direct mode and we want RAIL_B (page 1) just return client_b.
+	 * Otherwise, set the proper page (if page mode) and return RAIL_A.
+	 */
+	if (st->client_b) {
+		/* if 0xff just return the last client */
+		if (page < 0)
+			return st->curr_client;
+		if (page)
+			st->curr_client = st->client_b;
+		else
+			st->curr_client = st->client;
+
+		return st->curr_client;
+	}
+
+	if (!probing)
+		ret = pmbus_set_page(st->client, page, 0xff);
+	else
+		ret = i2c_smbus_write_byte_data(st->client, PMBUS_PAGE, page);
+	if (ret < 0)
+		return ERR_PTR(ret);
+
+	return st->client;
+}
+
+static int max20826_rail_read_block_data(struct i2c_client *rail, u8 reg,
+					 u8 *block, u8 min_len)
+{
+	int ret;
+
+	ret = pmbus_read_smbus_i2c_block_data(rail, reg, block);
+	if (ret < 0)
+		return ret;
+	if (ret < min_len)
+		return -EIO;
+
+	return ret;
+}
+
+static int __max20826_read_block_data(struct max20826 *st, int page, u8 reg,
+				      u8 *block, u8 min_len)
+{
+	struct i2c_client *rail;
+
+	rail = max20826_select_rail(st, page, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	return max20826_rail_read_block_data(rail, reg, block, min_len);
+}
+
+static int __max20826_read_byte_data(struct max20826 *st, int page, int reg)
+{
+	struct i2c_client *rail;
+
+	rail = max20826_select_rail(st, page, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	return i2c_smbus_read_byte_data(rail, reg);
+}
+
+static int __max20826_write_byte_data(struct max20826 *st, int page, int reg,
+				      u8 value)
+{
+	struct i2c_client *rail;
+
+	rail = max20826_select_rail(st, page, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	return i2c_smbus_write_byte_data(rail, reg, value);
+}
+
+static int __max20826_read_word_data(struct max20826 *st, int page, int reg)
+{
+	struct i2c_client *rail;
+
+	rail = max20826_select_rail(st, page, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	return i2c_smbus_read_word_data(rail, reg);
+}
+
+/*
+ * @rail is the already selected rail client and is only given during probe,
+ * where the PMBus core cannot be used yet. Otherwise, pass NULL and let the
+ * core dispatch through our own read_byte_data()/write_byte_data() callbacks,
+ * which take care of selecting the rail.
+ */
+static int max20826_set_high_speed(struct max20826 *st, struct i2c_client *rail,
+				   u8 page, bool high_speed)
+{
+	u8 reg, mask, val;
+	int ret;
+
+	if (st->vendor_bus) {
+		reg = MAX20826_REG_OVERRIDE;
+		mask = MAX20826_OVERRIDE_MASK;
+		val = high_speed ? 0 : FIELD_PREP(MAX20826_OVERRIDE_MASK, 1);
+	} else {
+		reg = PMBUS_OPERATION;
+		mask = PB_OPERATION_CONTROL_V_SRC;
+		val = high_speed ? FIELD_PREP(PB_OPERATION_CONTROL_V_SRC, MAX20826_INTF_AVSBUS) : 0;
+	}
+
+	if (!rail) {
+		ret = pmbus_update_byte_data(st->client, page, reg, mask, val);
+	} else {
+		ret = i2c_smbus_read_byte_data(rail, reg);
+		if (ret < 0)
+			return ret;
+
+		ret = i2c_smbus_write_byte_data(rail, reg, (ret & ~mask) | val);
+	}
+
+	if (ret)
+		return ret;
+
+	st->high_speed[page] = high_speed;
+
+	return 0;
+}
+
+static int max20826_read_curr(struct max20826 *st, int page, int phase, int reg)
+{
+	u8 val_out[I2C_SMBUS_BLOCK_MAX], start_byte;
+	struct i2c_client *rail;
+	int ret;
+
+	if (phase == 0xff)
+		return __max20826_read_word_data(st, page, reg);
+
+	/*
+	 * On Rail_A phases are ascending (from 1) while on RAIL_B they
+	 * are descending (from 16).
+	 */
+	if (!page)
+		phase += 1;
+	else
+		phase = st->chip_info->max_phases - phase;
+
+	/* phase detect and read are only available on RAIL_A */
+	rail = max20826_select_rail(st, RAIL_A, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	ret = i2c_smbus_write_byte_data(rail, MAX20826_REG_PHASE_DETECT, phase);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * For MAX20826: Byte 2:3 is for phase IOUT 4:5 is for phase IIN.
+	 * For other chips: Byte 0:1 is for phase IOUT 2:3 is for phase IIN.
+	 */
+	if (reg == PMBUS_READ_IIN)
+		start_byte = st->chip_info->start_index_iin;
+	else
+		start_byte = st->chip_info->start_index_iout;
+
+	ret = max20826_rail_read_block_data(rail, MAX20826_REG_PHASE_READ,
+					    val_out, start_byte + 2);
+	if (ret < 0)
+		return ret;
+
+	return get_unaligned_le16(&val_out[start_byte]);
+}
+
+static int max20826_read_word_data(struct i2c_client *client, int page,
+				   int phase, int reg)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct max20826 *st = to_max20826(info);
+
+	switch (reg) {
+	case PMBUS_READ_IIN:
+	case PMBUS_READ_IOUT:
+		return max20826_read_curr(st, page, phase, reg);
+	case PMBUS_IOUT_OC_FAULT_LIMIT:
+		if (!st->chip_info->has_opl || !st->opl_enabled[page])
+			return __max20826_read_word_data(st, page, reg);
+		return -EIO;
+	case PMBUS_POUT_OP_FAULT_LIMIT:
+		/*
+		 * If Over Power Limit is enabled, PMBUS_IOUT_OC_FAULT_LIMIT
+		 * shows the power limit and hence we need to report it
+		 * properly in Watts.
+		 */
+		if (!st->chip_info->has_opl)
+			return __max20826_read_word_data(st, page, reg);
+
+		if (st->opl_enabled[page])
+			return __max20826_read_word_data(st, page,
+							 PMBUS_IOUT_OC_FAULT_LIMIT);
+		return -EIO;
+	default:
+		if (reg >= PMBUS_VIRT_BASE)
+			return -EOPNOTSUPP;
+		return __max20826_read_word_data(st, page, reg);
+	}
+}
+
+static int max20826_write_word_data(struct i2c_client *client, int page,
+				    int reg, u16 word)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct max20826 *st = to_max20826(info);
+	struct i2c_client *rail;
+
+	rail = max20826_select_rail(st, page, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	return i2c_smbus_write_word_data(rail, reg, word);
+}
+
+static int max20826_regulator_do_enable(struct max20826 *st,
+					struct i2c_client *rail, u8 page,
+					u8 byte)
+{
+	/*
+	 * If AVSBus is enabled (bits 5 and 4 set) the device refuses to set bit
+	 * 7 of the OPERATION register. Hence, to workaround this, we first
+	 * clear the bits and then set them all together.
+	 */
+	if (st->chip_info->has_avsbus && !st->vendor_bus && st->high_speed[page] &&
+	    PB_OPERATION_CONTROL_ON & byte) {
+		u8 __byte = byte & ~(PB_OPERATION_CONTROL_V_SRC | PB_OPERATION_CONTROL_ON);
+		int ret;
+
+		ret = i2c_smbus_write_byte_data(rail, PMBUS_OPERATION, __byte);
+		if (ret < 0)
+			return ret;
+	}
+
+	return i2c_smbus_write_byte_data(rail, PMBUS_OPERATION, byte);
+}
+
+static int max20826_regulator_enable(struct max20826 *st, int page, u8 byte)
+{
+	struct i2c_client *rail;
+
+	rail = max20826_select_rail(st, page, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	if (page)
+		gpiod_set_value_cansleep(st->bvren, !!(PB_OPERATION_CONTROL_ON & byte));
+	else
+		gpiod_set_value_cansleep(st->avren, !!(PB_OPERATION_CONTROL_ON & byte));
+
+	return max20826_regulator_do_enable(st, rail, page, byte);
+}
+
+static int max20826_write_byte_data(struct i2c_client *client, int page,
+				    int reg, u8 byte)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct max20826 *st = to_max20826(info);
+
+	switch (reg) {
+	case PMBUS_OPERATION:
+		return max20826_regulator_enable(st, page, byte);
+	default:
+		return __max20826_write_byte_data(st, page, reg, byte);
+	}
+}
+
+static int max20826_write_byte(struct i2c_client *client, int page, u8 byte)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct max20826 *st = to_max20826(info);
+	struct i2c_client *rail;
+
+	rail = max20826_select_rail(st, page, false);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	return i2c_smbus_write_byte(rail, byte);
+}
+
+static int max20826_regulator_enabled(struct max20826 *st, int page)
+{
+	struct gpio_desc *gpio = page ? st->bvren : st->avren;
+	int on, ret;
+
+	if (gpio) {
+		on = gpiod_get_value_cansleep(gpio);
+		if (on < 0)
+			return on;
+	} else {
+		/* If the gpios are not given, just assume it's on */
+		on = 1;
+	}
+
+	ret = __max20826_read_byte_data(st, page, PMBUS_OPERATION);
+	if (ret < 0)
+		return ret;
+
+	if (st->on_off_ctrl[page])
+		on = (PB_OPERATION_CONTROL_ON & ret) && on;
+
+	ret &= ~PB_OPERATION_CONTROL_ON;
+	return ret | FIELD_PREP(PB_OPERATION_CONTROL_ON, on);
+}
+
+static int max20826_iout_status(struct max20826 *st, int page)
+{
+	int status;
+
+	status = __max20826_read_byte_data(st, page, PMBUS_STATUS_IOUT);
+	if (status < 0 || !st->chip_info->has_opl)
+		return status;
+
+	/*
+	 * If Over power limit is on, the fault condition is still set on the OC
+	 * bit
+	 */
+	if (st->opl_enabled[page] && (status & PB_IOUT_OC_FAULT))
+		return status | PB_POUT_OP_FAULT;
+
+	return status;
+}
+
+static int max20826_read_byte_data(struct i2c_client *client, int page, int reg)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct max20826 *st = to_max20826(info);
+
+	switch (reg) {
+	case PMBUS_OPERATION:
+		return max20826_regulator_enabled(st, page);
+	case PMBUS_STATUS_IOUT:
+		return max20826_iout_status(st, page);
+	default:
+		return __max20826_read_byte_data(st, page, reg);
+	}
+}
+
+static int max20826_read_block_data(struct i2c_client *client, int page, u8 reg,
+				    char *data_buf)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct max20826 *st = to_max20826(info);
+
+	switch (reg) {
+	case PMBUS_MFR_ID:
+	case PMBUS_MFR_MODEL:
+		return __max20826_read_block_data(st, page, reg, data_buf, 16);
+	case PMBUS_MFR_REVISION:
+		return __max20826_read_block_data(st, page, reg, data_buf, 2);
+	case PMBUS_MFR_DATE:
+		return __max20826_read_block_data(st, page, reg, data_buf, 8);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static struct pmbus_driver_info max20826_default_info = {
+	.pages = 1,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+		   PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+		   PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP |
+		   PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_PIN | PMBUS_HAVE_POUT |
+		   PMBUS_PHASE_VIRTUAL,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_VOLTAGE_OUT] = direct,
+	.format[PSC_TEMPERATURE] = linear,
+	.format[PSC_CURRENT_IN] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_POWER] = linear,
+	.m[PSC_VOLTAGE_OUT] = 2,
+	.R[PSC_VOLTAGE_OUT] = 3,
+	.write_byte_data = max20826_write_byte_data,
+	.write_byte = max20826_write_byte,
+	.read_byte_data = max20826_read_byte_data,
+	.read_word_data = max20826_read_word_data,
+	.write_word_data = max20826_write_word_data,
+	.read_block_data = max20826_read_block_data,
+#if IS_ENABLED(CONFIG_SENSORS_MAX20826_REGULATOR)
+	.num_regulators = 1,
+	.reg_desc = max20826_reg_desc,
+#endif
+};
+
+static int max20826_detect_addr_mode(struct max20826 *st)
+{
+	u8 val_buf[I2C_SMBUS_BLOCK_MAX];
+	struct device *dev = &st->client->dev;
+	int ret, val;
+
+	/*
+	 * The idea is that after a POR, we are in page0 in which case we can
+	 * read MAX20826_REG_ADDR_MODE. If we fail to read
+	 * MAX20826_REG_ADDR_MODE, it might be due to a soft reset or unbinding
+	 * the device in which case the device could be left in page 1. Hence,
+	 * let's just try to change to page 0 and error out if we can't.
+	 *
+	 * OTOH, if the addressing mode is direct, we should be able to
+	 * read MAX20826_REG_ADDR_MODE. If not, we'll fail setting the page.
+	 *
+	 * REG_ADDR_MODE, MAX20855B has 6 bytes, other devices have 1. Byte 3 is
+	 * the equivalent of the single byte of the others.
+	 */
+	if (st->chip_info->is_reg_addr_mode_block)
+		val = max20826_rail_read_block_data(st->client,
+						    MAX20826_REG_ADDR_MODE,
+						    val_buf, 3);
+	else
+		val = i2c_smbus_read_byte_data(st->client, MAX20826_REG_ADDR_MODE);
+
+	if (val < 0) {
+		ret = i2c_smbus_write_byte_data(st->client, PMBUS_PAGE, 0);
+		if (ret < 0)
+			return dev_err_probe(dev, ret,
+					     "Failed to change to page 0\n");
+
+		/* try again! */
+		if (st->chip_info->is_reg_addr_mode_block)
+			val = max20826_rail_read_block_data(st->client,
+							    MAX20826_REG_ADDR_MODE,
+							    val_buf, 3);
+		else
+			val = i2c_smbus_read_byte_data(st->client,
+						       MAX20826_REG_ADDR_MODE);
+		if (val < 0)
+			return dev_err_probe(dev, val,
+					     "Failed to read MAX20826_REG_ADDR_MODE\n");
+	}
+
+	if (st->chip_info->is_reg_addr_mode_block)
+		val = val_buf[2];
+
+	if (val & MAX20826_PAGE_MODE_MSK)
+		return 0;
+
+	/*
+	 * If in direct mode, rail b will be accessible from the addr of RAIL_A
+	 * +1
+	 */
+	st->client_b = devm_i2c_new_dummy_device(dev, st->client->adapter,
+						 st->client->addr + 1);
+	if (IS_ERR(st->client_b))
+		return PTR_ERR(st->client_b);
+
+	return 0;
+}
+
+static int max20826_set_vout_interface(struct max20826 *st,
+				       struct i2c_client *rail, u8 page)
+{
+	static const char * const props[MAX20826_MAX_PAGES] = {
+		"adi,rail-a-high-speed",
+		"adi,rail-b-high-speed",
+	};
+	bool high_speed;
+	int ret;
+
+	/*
+	 * Rail A 0xDC_1[7:6] sets the operating mode for both rails, so only look
+	 * at it once.
+	 */
+	if (page == RAIL_A) {
+		if (st->chip_info->has_avsbus) {
+			ret = i2c_smbus_read_word_data(st->client,
+						       MAX20826_REG_C_MODEAB);
+			if (ret < 0)
+				return ret;
+
+			if (FIELD_GET(MAX20826_C_MODEAB_MASK, ret) == MAX20826_INTF_PWMVID)
+				st->vendor_bus = true;
+		} else {
+			st->vendor_bus = true;
+		}
+	}
+
+	high_speed = device_property_read_bool(&st->client->dev, props[page]);
+
+	return max20826_set_high_speed(st, rail, page, high_speed);
+}
+
+static int max20826_detect_phases(struct max20826 *st, struct i2c_client *rail,
+				  struct pmbus_driver_info *info, u8 page,
+				  u8 expected)
+{
+	u8 status_mon[I2C_SMBUS_BLOCK_MAX];
+	int ret, ret2 = 0, oper_save = -1;
+	u8 n_phases;
+
+	/*
+	 * In order to detect the number of phases, we need to be regulating.
+	 * It is also assumed we're already in the page we want to detect the
+	 * phases.
+	 *
+	 * If the vendor high speed interface controls the output voltage, the
+	 * enable state is not ours to control (it is given by that interface
+	 * together with the AVREN/BVREN pins) and writing PMBUS_OPERATION is a
+	 * no-op, so just assume the rail is already regulating.
+	 */
+	if (!st->vendor_bus || !st->high_speed[page]) {
+		ret = i2c_smbus_read_byte_data(rail, PMBUS_OPERATION);
+		if (ret < 0)
+			return ret;
+
+		if (!(ret & PB_OPERATION_CONTROL_ON)) {
+			oper_save = ret;
+			ret = max20826_regulator_do_enable(st, rail, page,
+							   ret | PB_OPERATION_CONTROL_ON);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	ret = max20826_rail_read_block_data(rail, MAX20826_REG_STATUS_MON,
+					    status_mon, 2);
+	if (ret < 0)
+		goto out_restore_oper;
+
+	n_phases = field_get(st->chip_info->phase_num_mask, status_mon[1]);
+	if (n_phases != expected) {
+		ret = dev_err_probe(&st->client->dev, -EIO,
+				    "Number of phases mismatch: expected=%u, detected=%u\n",
+				    expected, n_phases);
+		goto out_restore_oper;
+	}
+
+	info->phases[page] = n_phases;
+	for (unsigned int phase = 0; phase < n_phases; phase++)
+		info->pfunc[phase] = PMBUS_HAVE_IOUT | PMBUS_HAVE_IIN;
+
+out_restore_oper:
+	if (oper_save >= 0)
+		ret2 = max20826_regulator_do_enable(st, rail, page, oper_save);
+
+	return ret < 0 ? ret : ret2;
+}
+
+static int max20826_get_rail_config(struct max20826 *st,
+				    struct i2c_client *rail, u8 page)
+{
+	u8 ctrl_misc[I2C_SMBUS_BLOCK_MAX];
+	int ret;
+
+	/*
+	 * Check if we need to control PMBUS_OPERATION in addition to the
+	 * CONTROL pin.
+	 */
+	ret = i2c_smbus_read_byte_data(rail, PMBUS_ON_OFF_CONFIG);
+	if (ret < 0)
+		return ret;
+
+	st->on_off_ctrl[page] = !!(ret & PB_ON_OFF_CONFIG_OPERATION_REQ);
+
+	/*
+	 * See if Over Power Limit is enabled. This will impact how
+	 * OC_FAULT_LIMIT is handled.
+	 */
+	if (st->chip_info->has_opl) {
+		ret = max20826_rail_read_block_data(rail, MAX20826_REG_CTRL_MISC,
+						    ctrl_misc, 1);
+		if (ret < 0)
+			return ret;
+
+		st->opl_enabled[page] = !!(ctrl_misc[0] & MAX20826_OPL_EN_MSK);
+	}
+
+	return 0;
+}
+
+static int max20826_setup_vout_format(struct max20826 *st)
+{
+	struct pmbus_driver_info *info = &st->info;
+	struct i2c_client *rail;
+	int ret;
+
+	if (st->chip_info->is_vout_direct) {
+		u8 vout_res[I2C_SMBUS_BLOCK_MAX];
+
+		rail = max20826_select_rail(st, RAIL_A, true);
+		if (IS_ERR(rail))
+			return PTR_ERR(rail);
+
+		ret = max20826_rail_read_block_data(rail, MAX20826_REG_VOUT_RES,
+						    vout_res, 2);
+		if (ret < 0)
+			return ret;
+
+		/* check for 1 mv/LSB */
+		if (MAX20826_VOUT_RES_MSK & vout_res[1])
+			info->m[PSC_VOLTAGE_OUT] = 1;
+
+		return 0;
+	}
+
+	st->info.format[PSC_VOLTAGE_OUT] = vid;
+	for (unsigned int page = 0; page < info->pages; page++) {
+		if (!st->chip_info->select_vrm) {
+			/* only vr12 in this case */
+			st->info.vrm_version[page] = vr12;
+			continue;
+		}
+
+		rail = max20826_select_rail(st, page, true);
+		if (IS_ERR(rail))
+			return PTR_ERR(rail);
+
+		ret = i2c_smbus_read_byte_data(rail, MAX20855B_REG_VOUT_VRM);
+		if (ret < 0)
+			return ret;
+
+		if (ret & MAX20855B_REG_VOUT_VRM_MASK)
+			st->info.vrm_version[page] = vr13;
+		else
+			st->info.vrm_version[page] = vr12;
+	}
+
+	return 0;
+}
+
+static int max20826_setup(struct max20826 *st)
+{
+	struct pmbus_driver_info *info = &st->info;
+	u8 config[I2C_SMBUS_BLOCK_MAX], expected_phases;
+	struct device *dev = &st->client->dev;
+	struct i2c_client *rail;
+	int ret;
+
+	ret = max20826_detect_addr_mode(st);
+	if (ret < 0)
+		return ret;
+
+	ret = max20826_set_vout_interface(st, st->client, RAIL_A);
+	if (ret < 0)
+		return ret;
+
+	ret = max20826_rail_read_block_data(st->client,
+					    MAX20826_REG_RAIL_PHASE_CFG,
+					    config, 5);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * If we have phases in RAIL_B, we need to subtract them on config[1] as
+	 * those bits are also set in case RAIL_B has phases.
+	 */
+	expected_phases = st->chip_info->count_phases(config, RAIL_A);
+	ret = max20826_detect_phases(st, st->client, info, RAIL_A,
+				     expected_phases);
+	if (ret < 0)
+		return ret;
+
+	ret = max20826_get_rail_config(st, st->client, RAIL_A);
+	if (ret < 0)
+		return ret;
+
+	/* Let's see if RAIL_B is present */
+	rail = max20826_select_rail(st, RAIL_B, true);
+	if (IS_ERR(rail))
+		return PTR_ERR(rail);
+
+	/* Let's see if there's something on RAIL_B */
+	st->bvren = devm_gpiod_get_optional(dev, "bvren", GPIOD_OUT_HIGH);
+	if (IS_ERR(st->bvren))
+		return PTR_ERR(st->bvren);
+
+	ret = max20826_set_vout_interface(st, rail, RAIL_B);
+	if (ret < 0)
+		return ret;
+
+	expected_phases = st->chip_info->count_phases(config, RAIL_B);
+	ret = max20826_detect_phases(st, rail, info, RAIL_B, expected_phases);
+	if (ret < 0)
+		return ret;
+
+	if (info->phases[RAIL_B]) {
+		info->pages = MAX20826_MAX_PAGES;
+		info->func[RAIL_B] = PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+				     PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+				     PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP |
+				     PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_PIN | PMBUS_HAVE_POUT |
+				     PMBUS_PHASE_VIRTUAL;
+
+		if (IS_ENABLED(CONFIG_SENSORS_MAX20826_REGULATOR))
+			info->num_regulators = 2;
+
+		ret = max20826_get_rail_config(st, rail, RAIL_B);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = max20826_setup_vout_format(st);
+	if (ret < 0)
+		return ret;
+
+	if (info->pages == 1) {
+		/*
+		 * In this case we can be left in RAIL_B and given that we on
+		 * only have one page, pmbus_set_page() will refuse to comeback
+		 * to RAIL_A which is obviously problematic. Hence, let's just
+		 * select RAIL_A here.
+		 */
+		rail = max20826_select_rail(st, RAIL_A, true);
+		if (IS_ERR(rail))
+			return PTR_ERR(rail);
+	}
+
+	return 0;
+}
+
+static int max20826_high_speed_en_get(void *arg, u64 *val)
+{
+	const struct max20826_debugfs_entry *entry = arg;
+
+	*val = entry->st->high_speed[entry->page];
+
+	return 0;
+}
+
+static int max20826_high_speed_en_set(void *arg, u64 val)
+{
+	const struct max20826_debugfs_entry *entry = arg;
+	struct max20826 *st = entry->st;
+
+	guard(pmbus_lock)(st->client);
+
+	return max20826_set_high_speed(st, NULL, entry->page, !!val);
+}
+DEFINE_DEBUGFS_ATTRIBUTE(max20826_high_speed_en_fops,
+			 max20826_high_speed_en_get,
+			 max20826_high_speed_en_set, "%llu\n");
+
+static int max20826_high_speed_bus_show(struct seq_file *s, void *data)
+{
+	struct device *dev = s->private;
+	struct i2c_client *client = to_i2c_client(dev);
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct max20826 *st = to_max20826(info);
+
+	seq_printf(s, "%s\n", st->vendor_bus ? st->chip_info->vendor_bus_name : "AVSBus");
+
+	return 0;
+}
+
+static void max20826_init_debugfs(struct max20826 *st)
+{
+	struct device *dev = &st->client->dev;
+	struct dentry *dir;
+
+	dir = pmbus_get_debugfs_dir(st->client);
+	if (!dir)
+		return;
+
+	for (unsigned int page = 0; page < st->info.pages; page++) {
+		struct max20826_debugfs_entry *entry;
+		char name[32];
+
+		entry = devm_kzalloc(dev, sizeof(*entry), GFP_KERNEL);
+		if (!entry)
+			return;
+
+		entry->st = st;
+		entry->page = page;
+
+		scnprintf(name, sizeof(name), "in%u_high_speed_en", page + 2);
+		debugfs_create_file_unsafe(name, 0644, dir, entry,
+					   &max20826_high_speed_en_fops);
+	}
+
+	debugfs_create_devm_seqfile(dev, "in_high_speed_bus", dir,
+				    max20826_high_speed_bus_show);
+}
+
+static int max20826_probe(struct i2c_client *client)
+{
+	struct max20826 *st;
+	int ret;
+
+	st = devm_kzalloc(&client->dev, sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	st->client = client;
+	memcpy(&st->info, &max20826_default_info, sizeof(st->info));
+
+	st->chip_info = i2c_get_match_data(client);
+	if (!st->chip_info)
+		return -EINVAL;
+
+	st->avren = devm_gpiod_get_optional(&client->dev, "avren", GPIOD_OUT_HIGH);
+	if (IS_ERR(st->avren))
+		return PTR_ERR(st->avren);
+
+	ret = max20826_setup(st);
+	if (ret)
+		return ret;
+
+	ret = pmbus_do_probe(client, &st->info);
+	if (ret)
+		return ret;
+
+	max20826_init_debugfs(st);
+
+	return 0;
+}
+
+static const struct max20826_chip_info chip_info_max20826 = {
+	.vendor_bus_name = "Nvidia PWMVID",
+	.max_phases = MAX20826_MAX_PHASES,
+	.phase_num_mask = MAX20826_PHASES_NUM_MASK,
+	.start_index_iin = 4,
+	.start_index_iout = 2,
+	.is_vout_direct = true,
+	.has_avsbus = true,
+	.has_opl = true,
+	.count_phases = __max20826_count_phases,
+};
+
+static const struct max20826_chip_info chip_info_max20855b = {
+	.vendor_bus_name = "Intel SVID",
+	.max_phases = MAX20855B_MAX_PHASES,
+	.phase_num_mask = MAX20855B_PHASES_NUM_MASK,
+	.start_index_iin = 2,
+	.start_index_iout = 0,
+	.is_reg_addr_mode_block = true,
+	.select_vrm = true,
+	.count_phases = __max20855b_count_phases,
+};
+
+static const struct max20826_chip_info chip_info_max20908 = {
+	.vendor_bus_name = "AMD SVI3",
+	.max_phases = MAX20908_MAX_PHASES,
+	.phase_num_mask = MAX20826_PHASES_NUM_MASK,
+	.start_index_iin = 2,
+	.start_index_iout = 0,
+	.count_phases = __max20908_count_phases,
+};
+
+static const struct max20826_chip_info chip_info_max20912 = {
+	.vendor_bus_name = "AMD SVI3",
+	.max_phases = MAX20912_MAX_PHASES,
+	.phase_num_mask = MAX20826_PHASES_NUM_MASK,
+	.start_index_iin = 2,
+	.start_index_iout = 0,
+	.count_phases = __max20912_count_phases,
+};
+
+static const struct max20826_chip_info chip_info_max20916 = {
+	.vendor_bus_name = "AMD SVI3",
+	.max_phases = MAX20916_MAX_PHASES,
+	.phase_num_mask = MAX20826_PHASES_NUM_MASK,
+	.start_index_iin = 2,
+	.start_index_iout = 0,
+	.count_phases = __max20826_count_phases,
+};
+
+static const struct i2c_device_id max20826_id[] = {
+	{ "max20826", (kernel_ulong_t)&chip_info_max20826 },
+	{ "max20855b", (kernel_ulong_t)&chip_info_max20855b },
+	{ "max20908", (kernel_ulong_t)&chip_info_max20908 },
+	{ "max20912", (kernel_ulong_t)&chip_info_max20912 },
+	{ "max20916", (kernel_ulong_t)&chip_info_max20916 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, max20826_id);
+
+static const struct of_device_id max20826_of_match[] = {
+	{ .compatible = "adi,max20826", .data = &chip_info_max20826 },
+	{ .compatible = "adi,max20855b", .data = &chip_info_max20855b },
+	{ .compatible = "adi,max20908", .data = &chip_info_max20908 },
+	{ .compatible = "adi,max20912", .data = &chip_info_max20912 },
+	{ .compatible = "adi,max20916", .data = &chip_info_max20916 },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, max20826_of_match);
+
+static struct i2c_driver max20826_driver = {
+	.driver = {
+		.name = "max20826",
+		.of_match_table = max20826_of_match,
+	},
+	.probe = max20826_probe,
+	.id_table = max20826_id,
+};
+module_i2c_driver(max20826_driver);
+
+MODULE_AUTHOR("Nuno Sá <nuno.sa@analog.com>");
+MODULE_DESCRIPTION("PMBus driver for MAX20826");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/max34440.c b/drivers/hwmon/pmbus/max34440.c
index 866d25ae8971c4..7f71dfe4864387 100644
--- a/drivers/hwmon/pmbus/max34440.c
+++ b/drivers/hwmon/pmbus/max34440.c
@@ -812,10 +812,12 @@ static int max34440_probe(struct i2c_client *client)
 	return pmbus_do_probe(client, &data->info);
 }
 
+/* ADPM12300 is just ADPM12250 with higher power capacity. Reuses driver data */
 static const struct i2c_device_id max34440_id[] = {
 	{ .name = "adpm12160", .driver_data = adpm12160 },
 	{ .name = "adpm12200", .driver_data = adpm12200 },
 	{ .name = "adpm12250", .driver_data = adpm12250 },
+	{ .name = "adpm12300", .driver_data = adpm12250 },
 	{ .name = "max34440", .driver_data = max34440 },
 	{ .name = "max34441", .driver_data = max34441 },
 	{ .name = "max34446", .driver_data = max34446 },
diff --git a/drivers/hwmon/pmbus/mp2888.c b/drivers/hwmon/pmbus/mp2888.c
index c5f35daa3fe11e..7a2ecffba960e6 100644
--- a/drivers/hwmon/pmbus/mp2888.c
+++ b/drivers/hwmon/pmbus/mp2888.c
@@ -272,14 +272,14 @@ static int mp2888_write_word_data(struct i2c_client *client, int page, int reg,
 		word = clamp_val(word, 0, GENMASK(7, 0));
 		break;
 	case PMBUS_IOUT_OC_WARN_LIMIT:
-		/* Fix limit according to total curent resolution. */
+		/* Fix limit according to total current resolution. */
 		word = data->total_curr_resolution ? DIV_ROUND_CLOSEST(word, 8) :
 		       DIV_ROUND_CLOSEST(word, 4);
 		/* Drop unused bits 15:10. */
 		word = clamp_val(word, 0, GENMASK(9, 0));
 		break;
 	case PMBUS_POUT_OP_WARN_LIMIT:
-		/* Fix limit according to total curent resolution. */
+		/* Fix limit according to total current resolution. */
 		word = data->total_curr_resolution ? DIV_ROUND_CLOSEST(word, 4) :
 		       DIV_ROUND_CLOSEST(word, 2);
 		/* Drop unused bits 15:10. */
diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h
index 920c1102ab6df5..5525c048a81058 100644
--- a/drivers/hwmon/pmbus/pmbus.h
+++ b/drivers/hwmon/pmbus/pmbus.h
@@ -461,6 +461,13 @@ struct pmbus_driver_info {
 	int (*read_byte_data)(struct i2c_client *client, int page, int reg);
 	int (*read_word_data)(struct i2c_client *client, int page, int phase,
 			      int reg);
+	/*
+	 * data_buf is at least I2C_SMBUS_BLOCK_MAX bytes long. The callback
+	 * must never return more than I2C_SMBUS_BLOCK_MAX bytes of data and
+	 * returns the number of bytes written into data_buf.
+	 */
+	int (*read_block_data)(struct i2c_client *client, int page, u8 reg,
+			       char *data_buf);
 	int (*write_byte_data)(struct i2c_client *client, int page, int reg,
 			      u8 byte);
 	int (*write_word_data)(struct i2c_client *client, int page, int reg,
diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
index 5f69c1420b4e15..369a3bed8c87a4 100644
--- a/drivers/hwmon/pmbus/pmbus_core.c
+++ b/drivers/hwmon/pmbus/pmbus_core.c
@@ -14,6 +14,7 @@
 #include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
@@ -616,6 +617,26 @@ static int pmbus_read_block_data(struct i2c_client *client, int page, u8 reg,
 	return pmbus_read_smbus_i2c_block_data(client, reg, data_buf);
 }
 
+/*
+ * _pmbus_read_block_data() is similar to pmbus_read_block_data(), but checks if
+ * a device specific mapping function exists and calls it if necessary.
+ */
+static int _pmbus_read_block_data(struct i2c_client *client, int page, u8 reg,
+				  char *data_buf)
+{
+	struct pmbus_data *data = i2c_get_clientdata(client);
+	const struct pmbus_driver_info *info = data->info;
+	int status;
+
+	if (info->read_block_data) {
+		status = info->read_block_data(client, page, reg, data_buf);
+		if (status != -ENODATA)
+			return status;
+	}
+
+	return pmbus_read_block_data(client, page, reg, data_buf);
+}
+
 static struct pmbus_sensor *pmbus_find_sensor(struct pmbus_data *data, int page,
 					      int reg)
 {
@@ -761,7 +782,7 @@ static bool __maybe_unused pmbus_check_block_register(struct i2c_client *client,
 	struct pmbus_data *data = i2c_get_clientdata(client);
 	char data_buf[I2C_SMBUS_BLOCK_MAX + 2];
 
-	rv = pmbus_read_block_data(client, page, reg, data_buf);
+	rv = _pmbus_read_block_data(client, page, reg, data_buf);
 	if (rv >= 0 && !(data->flags & PMBUS_SKIP_STATUS_CHECK))
 		rv = pmbus_check_status_cml(client);
 	if (rv < 0 && (data->flags & PMBUS_READ_STATUS_AFTER_FAILED_CHECK))
@@ -2962,6 +2983,11 @@ static int pmbus_init_common(struct i2c_client *client, struct pmbus_data *data,
 	}
 
 	for (page = 0; page < info->pages; page++) {
+		if (info->phases[page] > PMBUS_PHASES) {
+			dev_err(dev, "Bad number of PMBus phases for page %d: %d\n",
+				page, info->phases[page]);
+			return -ENODEV;
+		}
 		ret = pmbus_identify_common(client, data, page);
 		if (ret < 0) {
 			dev_err(dev, "Failed to identify chip capabilities\n");
@@ -3679,7 +3705,7 @@ static ssize_t pmbus_debugfs_block_read(struct file *file, char __user *buf,
 	char data[I2C_SMBUS_BLOCK_MAX + 2] = { 0 };
 
 	scoped_guard(pmbus_lock, client) {
-		rc = pmbus_read_block_data(client, entry->page, entry->reg, data);
+		rc = _pmbus_read_block_data(client, entry->page, entry->reg, data);
 		if (rc < 0)
 			return rc;
 	}
diff --git a/drivers/hwmon/pmbus/tda38740.c b/drivers/hwmon/pmbus/tda38740.c
new file mode 100644
index 00000000000000..067de07ba9fb4a
--- /dev/null
+++ b/drivers/hwmon/pmbus/tda38740.c
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Hardware monitoring driver for Infineon TDA38725/TDA38740
+ *
+ * Copyright (c) 2023 9elements GmbH
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include "pmbus.h"
+
+static struct pmbus_driver_info tda38740_info = {
+	.pages = 1,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_VOLTAGE_OUT] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_CURRENT_IN] = linear,
+	.format[PSC_POWER] = linear,
+	.format[PSC_TEMPERATURE] = linear,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_STATUS_INPUT
+	    | PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP
+	    | PMBUS_HAVE_IIN
+	    | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT
+	    | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT
+	    | PMBUS_HAVE_POUT | PMBUS_HAVE_PIN,
+};
+
+static int tda38740_probe(struct i2c_client *client)
+{
+	return pmbus_do_probe(client, &tda38740_info);
+}
+
+static const struct i2c_device_id tda38740_id[] = {
+	{ .name = "tda38725"},
+	{ .name = "tda38740"},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, tda38740_id);
+
+static const struct of_device_id tda38740_of_match[] = {
+	{ .compatible = "infineon,tda38725"},
+	{ .compatible = "infineon,tda38740"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, tda38740_of_match);
+
+static struct i2c_driver tda38740_driver = {
+	.driver = {
+		.name = "tda38740",
+		.of_match_table = tda38740_of_match,
+	},
+	.probe = tda38740_probe,
+	.id_table = tda38740_id,
+};
+
+module_i2c_driver(tda38740_driver);
+
+MODULE_DESCRIPTION("PMBus driver for Infineon TDA38725/TDA38740");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/tps25990.c b/drivers/hwmon/pmbus/tps25990.c
index 9d318e6509abf6..0d5c99053f23f1 100644
--- a/drivers/hwmon/pmbus/tps25990.c
+++ b/drivers/hwmon/pmbus/tps25990.c
@@ -47,6 +47,22 @@
 					 PK_MIN_AVG_RST_AVG  | \
 					 PK_MIN_AVG_RST_MIN)
 
+#define TPS1689_VIN_OV_RANGE_SEL_MASK	GENMASK(7, 6)
+#define TPS1689_VIN_VOV_MASK		GENMASK(5, 0)
+#define TPS1689_VIN_SCALING		251
+#define TPS1689_VIN_VOV_STEP_MV		250
+#define TPS1689_VIN_RANGE_SPAN_MV	16000
+
+enum chips {
+	tps1689,
+	tps25990,
+};
+
+struct tps25990_data {
+	struct pmbus_driver_info info;
+	enum chips chip_id;
+};
+
 /*
  * Arbitrary default Rimon value: 1kOhm
  * This correspond to an overcurrent limit of 55A, close to the specified limit
@@ -96,6 +112,8 @@ static int tps25990_mfr_write_protect_get(struct i2c_client *client)
 static int tps25990_read_word_data(struct i2c_client *client,
 				   int page, int phase, int reg)
 {
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct tps25990_data *data = container_of(info, struct tps25990_data, info);
 	int ret;
 
 	switch (reg) {
@@ -184,9 +202,18 @@ static int tps25990_read_word_data(struct i2c_client *client,
 		ret = pmbus_read_word_data(client, page, phase, reg);
 		if (ret < 0)
 			break;
-		ret = DIV_ROUND_CLOSEST(ret * TPS25990_VIN_OVF_NUM,
-					TPS25990_VIN_OVF_DIV);
-		ret += TPS25990_VIN_OVF_OFF;
+		if (data->chip_id == tps25990) {
+			ret = DIV_ROUND_CLOSEST(ret * TPS25990_VIN_OVF_NUM,
+						TPS25990_VIN_OVF_DIV);
+			ret += TPS25990_VIN_OVF_OFF;
+		} else if (data->chip_id == tps1689) {
+			int rng = (FIELD_GET(TPS1689_VIN_OV_RANGE_SEL_MASK, ret) + 1) *
+				   TPS1689_VIN_RANGE_SPAN_MV;
+			int vov = FIELD_GET(TPS1689_VIN_VOV_MASK, ret) * TPS1689_VIN_VOV_STEP_MV;
+
+			ret = DIV_ROUND_CLOSEST(rng + vov - TPS1689_VIN_RANGE_SPAN_MV,
+						TPS1689_VIN_SCALING);
+		}
 		break;
 
 	case PMBUS_IIN_OC_FAULT_LIMIT:
@@ -229,6 +256,8 @@ static int tps25990_read_word_data(struct i2c_client *client,
 static int tps25990_write_word_data(struct i2c_client *client,
 				    int page, int reg, u16 value)
 {
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct tps25990_data *data = container_of(info, struct tps25990_data, info);
 	int ret;
 
 	switch (reg) {
@@ -240,26 +269,52 @@ static int tps25990_write_word_data(struct i2c_client *client,
 	case PMBUS_OT_WARN_LIMIT:
 	case PMBUS_OT_FAULT_LIMIT:
 	case PMBUS_PIN_OP_WARN_LIMIT:
-		value >>= TPS25990_8B_SHIFT;
+		value = clamp_val((s16)value, 0, S16_MAX) >> TPS25990_8B_SHIFT;
 		value = clamp_val(value, 0, 0xff);
 		ret = pmbus_write_word_data(client, page, reg, value);
 		break;
 
 	case PMBUS_VIN_OV_FAULT_LIMIT:
-		value -= TPS25990_VIN_OVF_OFF;
-		value = DIV_ROUND_CLOSEST(((unsigned int)value) * TPS25990_VIN_OVF_DIV,
-					  TPS25990_VIN_OVF_NUM);
-		value = clamp_val(value, 0, 0xf);
+		if ((s16)value < 0)
+			return -EINVAL;
+
+		if (data->chip_id == tps25990) {
+			int tmp = (int)value - TPS25990_VIN_OVF_OFF;
+
+			tmp = clamp_val(tmp, 0, INT_MAX);
+			value = DIV_ROUND_CLOSEST((unsigned int)tmp * TPS25990_VIN_OVF_DIV,
+						  TPS25990_VIN_OVF_NUM);
+			value = clamp_val(value, 0, 0xf);
+		} else if (data->chip_id == tps1689) {
+			u32 scaled_value = value * TPS1689_VIN_SCALING + TPS1689_VIN_RANGE_SPAN_MV;
+			u32 rng_idx = scaled_value / TPS1689_VIN_RANGE_SPAN_MV;
+			u32 ov_set;
+
+			rng_idx = clamp_val(rng_idx, 1,
+					    FIELD_MAX(TPS1689_VIN_OV_RANGE_SEL_MASK) + 1);
+			ov_set = scaled_value - (TPS1689_VIN_RANGE_SPAN_MV * rng_idx);
+			ov_set = min_t(u32, ov_set / TPS1689_VIN_VOV_STEP_MV,
+				       FIELD_MAX(TPS1689_VIN_VOV_MASK));
+			value = FIELD_PREP(TPS1689_VIN_OV_RANGE_SEL_MASK, rng_idx - 1) |
+						FIELD_PREP(TPS1689_VIN_VOV_MASK, ov_set);
+		}
 		ret = pmbus_write_word_data(client, page, reg, value);
 		break;
 
-	case PMBUS_IIN_OC_FAULT_LIMIT:
-		value -= TPS25990_IIN_OCF_OFF;
-		value = DIV_ROUND_CLOSEST(((unsigned int)value) * TPS25990_IIN_OCF_DIV,
+	case PMBUS_IIN_OC_FAULT_LIMIT: {
+		int tmp;
+
+		if ((s16)value < 0)
+			return -EINVAL;
+
+		tmp = (int)value - TPS25990_IIN_OCF_OFF;
+		tmp = clamp_val(tmp, 0, INT_MAX);
+		value = DIV_ROUND_CLOSEST((unsigned int)tmp * TPS25990_IIN_OCF_DIV,
 					  TPS25990_IIN_OCF_NUM);
 		value = clamp_val(value, 0, 0x3f);
 		ret = pmbus_write_byte_data(client, page, TPS25990_VIREF, value);
 		break;
+	}
 
 	case PMBUS_VIRT_SAMPLES:
 		value = clamp_val(value, 1, 1 << PK_MIN_AVG_AVG_CNT);
@@ -337,63 +392,120 @@ static const struct regulator_desc tps25990_reg_desc[] = {
 };
 #endif
 
-static const struct pmbus_driver_info tps25990_base_info = {
-	.pages = 1,
-	.format[PSC_VOLTAGE_IN] = direct,
-	.m[PSC_VOLTAGE_IN] = 5251,
-	.b[PSC_VOLTAGE_IN] = 0,
-	.R[PSC_VOLTAGE_IN] = -2,
-	.format[PSC_VOLTAGE_OUT] = direct,
-	.m[PSC_VOLTAGE_OUT] = 5251,
-	.b[PSC_VOLTAGE_OUT] = 0,
-	.R[PSC_VOLTAGE_OUT] = -2,
-	.format[PSC_TEMPERATURE] = direct,
-	.m[PSC_TEMPERATURE] = 140,
-	.b[PSC_TEMPERATURE] = 32100,
-	.R[PSC_TEMPERATURE] = -2,
-	/*
-	 * Current and Power measurement depends on the ohm value
-	 * of Rimon. m is multiplied by 1000 below to have an integer
-	 * and -3 is added to R to compensate.
-	 */
-	.format[PSC_CURRENT_IN] = direct,
-	.m[PSC_CURRENT_IN] = 9538,
-	.b[PSC_CURRENT_IN] = 0,
-	.R[PSC_CURRENT_IN] = -6,
-	.format[PSC_POWER] = direct,
-	.m[PSC_POWER] = 4901,
-	.b[PSC_POWER] = 0,
-	.R[PSC_POWER] = -7,
-	.func[0] = (PMBUS_HAVE_VIN |
-		    PMBUS_HAVE_VOUT |
-		    PMBUS_HAVE_VMON |
-		    PMBUS_HAVE_IIN |
-		    PMBUS_HAVE_PIN |
-		    PMBUS_HAVE_TEMP |
-		    PMBUS_HAVE_STATUS_VOUT |
-		    PMBUS_HAVE_STATUS_IOUT |
-		    PMBUS_HAVE_STATUS_INPUT |
-		    PMBUS_HAVE_STATUS_TEMP |
-		    PMBUS_HAVE_SAMPLES),
-	.read_word_data = tps25990_read_word_data,
-	.write_word_data = tps25990_write_word_data,
-	.read_byte_data = tps25990_read_byte_data,
-	.write_byte_data = tps25990_write_byte_data,
+static const struct pmbus_driver_info tps25990_base_info[] = {
+	[tps1689] = {
+		.pages = 1,
+		.format[PSC_VOLTAGE_IN] = direct,
+		.m[PSC_VOLTAGE_IN] = 3984,
+		.b[PSC_VOLTAGE_IN] = -63750,
+		.R[PSC_VOLTAGE_IN] = -3,
+		.format[PSC_VOLTAGE_OUT] = direct,
+		.m[PSC_VOLTAGE_OUT] = 1166,
+		.b[PSC_VOLTAGE_OUT] = 0,
+		.R[PSC_VOLTAGE_OUT] = -2,
+		.format[PSC_TEMPERATURE] = direct,
+		.m[PSC_TEMPERATURE] = 140,
+		.b[PSC_TEMPERATURE] = 32103,
+		.R[PSC_TEMPERATURE] = -2,
+		/*
+		 * Current and Power measurement depends on the ohm value
+		 * of Rimon. m is multiplied by 1000 below to have an integer
+		 * and -3 is added to R to compensate.
+		 */
+		.format[PSC_CURRENT_IN] = direct,
+		.m[PSC_CURRENT_IN] = 9548,
+		.b[PSC_CURRENT_IN] = 0,
+		.R[PSC_CURRENT_IN] = -6,
+		.format[PSC_CURRENT_OUT] = direct,
+		.m[PSC_CURRENT_OUT] = 24347,
+		.b[PSC_CURRENT_OUT] = 0,
+		.R[PSC_CURRENT_OUT] = -3,
+		.format[PSC_POWER] = direct,
+		.m[PSC_POWER] = 2775,
+		.b[PSC_POWER] = 0,
+		.R[PSC_POWER] = -4,
+		.func[0] = (PMBUS_HAVE_VIN |
+			    PMBUS_HAVE_VOUT |
+			    PMBUS_HAVE_VMON |
+			    PMBUS_HAVE_IIN |
+			    PMBUS_HAVE_IOUT |
+			    PMBUS_HAVE_PIN |
+			    PMBUS_HAVE_TEMP |
+			    PMBUS_HAVE_STATUS_VOUT |
+			    PMBUS_HAVE_STATUS_IOUT |
+			    PMBUS_HAVE_STATUS_INPUT |
+			    PMBUS_HAVE_STATUS_TEMP |
+			    PMBUS_HAVE_SAMPLES),
+
+		.read_word_data = tps25990_read_word_data,
+		.write_word_data = tps25990_write_word_data,
+		.read_byte_data = tps25990_read_byte_data,
+		.write_byte_data = tps25990_write_byte_data,
 
 #if IS_ENABLED(CONFIG_SENSORS_TPS25990_REGULATOR)
-	.reg_desc = tps25990_reg_desc,
-	.num_regulators = ARRAY_SIZE(tps25990_reg_desc),
+		.reg_desc = tps25990_reg_desc,
+		.num_regulators = ARRAY_SIZE(tps25990_reg_desc),
+#endif
+		},
+	[tps25990] = {
+		.pages = 1,
+		.format[PSC_VOLTAGE_IN] = direct,
+		.m[PSC_VOLTAGE_IN] = 5251,
+		.b[PSC_VOLTAGE_IN] = 0,
+		.R[PSC_VOLTAGE_IN] = -2,
+		.format[PSC_VOLTAGE_OUT] = direct,
+		.m[PSC_VOLTAGE_OUT] = 5251,
+		.b[PSC_VOLTAGE_OUT] = 0,
+		.R[PSC_VOLTAGE_OUT] = -2,
+		.format[PSC_TEMPERATURE] = direct,
+		.m[PSC_TEMPERATURE] = 140,
+		.b[PSC_TEMPERATURE] = 32100,
+		.R[PSC_TEMPERATURE] = -2,
+		/*
+		 * Current and Power measurement depends on the ohm value
+		 * of Rimon. m is multiplied by 1000 below to have an integer
+		 * and -3 is added to R to compensate.
+		 */
+		.format[PSC_CURRENT_IN] = direct,
+		.m[PSC_CURRENT_IN] = 9538,
+		.b[PSC_CURRENT_IN] = 0,
+		.R[PSC_CURRENT_IN] = -6,
+		.format[PSC_POWER] = direct,
+		.m[PSC_POWER] = 4901,
+		.b[PSC_POWER] = 0,
+		.R[PSC_POWER] = -7,
+		.func[0] = (PMBUS_HAVE_VIN |
+			    PMBUS_HAVE_VOUT |
+			    PMBUS_HAVE_VMON |
+			    PMBUS_HAVE_IIN |
+			    PMBUS_HAVE_PIN |
+			    PMBUS_HAVE_TEMP |
+			    PMBUS_HAVE_STATUS_VOUT |
+			    PMBUS_HAVE_STATUS_IOUT |
+			    PMBUS_HAVE_STATUS_INPUT |
+			    PMBUS_HAVE_STATUS_TEMP |
+			    PMBUS_HAVE_SAMPLES),
+		.read_word_data = tps25990_read_word_data,
+		.write_word_data = tps25990_write_word_data,
+		.read_byte_data = tps25990_read_byte_data,
+		.write_byte_data = tps25990_write_byte_data,
+#if IS_ENABLED(CONFIG_SENSORS_TPS25990_REGULATOR)
+		.reg_desc = tps25990_reg_desc,
+		.num_regulators = ARRAY_SIZE(tps25990_reg_desc),
 #endif
+	},
 };
 
 static const struct i2c_device_id tps25990_i2c_id[] = {
-	{ .name = "tps25990" },
-	{ }
+	{ .name = "tps1689", .driver_data = tps1689 },
+	{ .name = "tps25990", .driver_data = tps25990 },
+	{}
 };
 MODULE_DEVICE_TABLE(i2c, tps25990_i2c_id);
 
 static const struct of_device_id tps25990_of_match[] = {
-	{ .compatible = "ti,tps25990" },
+	{ .compatible = "ti,tps1689", .data = (void *)tps1689 },
+	{ .compatible = "ti,tps25990", .data = (void *)tps25990 },
 	{}
 };
 MODULE_DEVICE_TABLE(of, tps25990_of_match);
@@ -401,8 +513,9 @@ MODULE_DEVICE_TABLE(of, tps25990_of_match);
 static int tps25990_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
-	struct pmbus_driver_info *info;
+	struct tps25990_data *data;
 	const char *propname;
+	enum chips chip_id;
 	u32 rimon;
 	int ret;
 
@@ -415,15 +528,21 @@ static int tps25990_probe(struct i2c_client *client)
 		rimon = TPS25990_DEFAULT_RIMON;
 	}
 
-	info = devm_kmemdup(dev, &tps25990_base_info, sizeof(*info), GFP_KERNEL);
-	if (!info)
+	chip_id = (enum chips)(unsigned long)i2c_get_match_data(client);
+
+	data = devm_kzalloc(dev, sizeof(struct tps25990_data), GFP_KERNEL);
+	if (!data)
 		return -ENOMEM;
 
+	data->info = tps25990_base_info[chip_id];
+	data->chip_id = chip_id;
+
 	/* Adapt the current and power scale for each instance */
-	tps25990_set_m(&info->m[PSC_CURRENT_IN], rimon);
-	tps25990_set_m(&info->m[PSC_POWER], rimon);
+	tps25990_set_m(&data->info.m[PSC_CURRENT_IN], rimon);
+	tps25990_set_m(&data->info.m[PSC_CURRENT_OUT], rimon);
+	tps25990_set_m(&data->info.m[PSC_POWER], rimon);
 
-	return pmbus_do_probe(client, info);
+	return pmbus_do_probe(client, &data->info);
 }
 
 static struct i2c_driver tps25990_driver = {
diff --git a/drivers/hwmon/pmbus/tps53679.c b/drivers/hwmon/pmbus/tps53679.c
index 6c25701b36fcec..a04a632a509174 100644
--- a/drivers/hwmon/pmbus/tps53679.c
+++ b/drivers/hwmon/pmbus/tps53679.c
@@ -16,13 +16,21 @@
 #include "pmbus.h"
 
 enum chips {
-	tps53647, tps53667, tps53676, tps53679, tps53681, tps53685, tps53688
+	tps53622, tps53647, tps53659, tps53667, tps53676, tps53679, tps53681,
+	tps53685, tps53688, tps536c7
 };
 
 #define TPS53647_PAGE_NUM		1
 
-#define TPS53676_USER_DATA_03		0xb3
+#define TPS536XX_PHASE_CONFIG		0xb3
 #define TPS53676_MAX_PHASES		7
+#define TPS536C7_MAX_PHASES		12
+
+#define TPS536XX_PHASE_ENABLE		BIT(7)
+#define TPS536XX_PHASE_PAGE		BIT(4)
+
+#define TPS53676_DEVICE_ID		"TI\x53\x67\x60\x00"
+#define TPS536C7_DEVICE_ID		"TI\x53\x6c\x70\x00"
 
 #define TPS53679_PROT_VR12_5MV		0x01 /* VR12.0 mode, 5-mV DAC */
 #define TPS53679_PROT_VR12_5_10MV	0x02 /* VR12.5 mode, 10-mV DAC */
@@ -165,34 +173,72 @@ static int tps53681_identify(struct i2c_client *client,
 					    TPS53681_DEVICE_ID);
 }
 
-static int tps53676_identify(struct i2c_client *client,
-			     struct pmbus_driver_info *info)
+static int tps536xx_read_block(struct i2c_client *client, u8 reg, u8 *buf,
+			       bool retry_without_pec)
+{
+	int ret = i2c_smbus_read_block_data(client, reg, buf);
+
+	/*
+	 * Some TPS536C7 samples return an invalid PEC on the device-ID and
+	 * phase-configuration block reads. Retry once without PEC; ordinary
+	 * telemetry keeps using PEC.
+	 */
+	if (ret == -EBADMSG && retry_without_pec &&
+	    (client->flags & I2C_CLIENT_PEC)) {
+		client->flags &= ~I2C_CLIENT_PEC;
+		ret = i2c_smbus_read_block_data(client, reg, buf);
+		client->flags |= I2C_CLIENT_PEC;
+	}
+	return ret;
+}
+
+static int tps536xx_read_phases(struct i2c_client *client,
+				const char *device_id, int max_phases,
+				bool retry_without_pec, int *phases_a,
+				int *phases_b)
 {
 	u8 buf[I2C_SMBUS_BLOCK_MAX];
-	int phases_a = 0, phases_b = 0;
 	int i, ret;
 
-	ret = i2c_smbus_read_block_data(client, PMBUS_IC_DEVICE_ID, buf);
+	ret = tps536xx_read_block(client, PMBUS_IC_DEVICE_ID, buf,
+				  retry_without_pec);
 	if (ret < 0)
 		return ret;
-	if (ret != 6 || memcmp(buf, "TI\x53\x67\x60\x00", 6)) {
+	if (ret != 6 || memcmp(buf, device_id, 6)) {
 		dev_err(&client->dev, "Unexpected device ID: %*ph\n", ret, buf);
 		return -ENODEV;
 	}
 
-	ret = i2c_smbus_read_block_data(client, TPS53676_USER_DATA_03, buf);
+	ret = tps536xx_read_block(client, TPS536XX_PHASE_CONFIG, buf,
+				  retry_without_pec);
 	if (ret < 0)
 		return ret;
 	if (ret != 24)
 		return -EIO;
-	for (i = 0; i < 2 * TPS53676_MAX_PHASES; i += 2) {
-		if (buf[i + 1] & 0x80) {
-			if (buf[i] & BIT(4))
-				phases_b++;
+
+	*phases_a = 0;
+	*phases_b = 0;
+	for (i = 0; i < 2 * max_phases; i += 2) {
+		if (buf[i + 1] & TPS536XX_PHASE_ENABLE) {
+			if (buf[i] & TPS536XX_PHASE_PAGE)
+				(*phases_b)++;
 			else
-				phases_a++;
+				(*phases_a)++;
 		}
 	}
+	return 0;
+}
+
+static int tps53676_identify(struct i2c_client *client,
+			     struct pmbus_driver_info *info)
+{
+	int phases_a, phases_b, ret;
+
+	ret = tps536xx_read_phases(client, TPS53676_DEVICE_ID,
+				   TPS53676_MAX_PHASES, false,
+				   &phases_a, &phases_b);
+	if (ret)
+		return ret;
 
 	info->format[PSC_VOLTAGE_OUT] = linear;
 	info->pages = 1;
@@ -213,6 +259,69 @@ static int tps53676_identify(struct i2c_client *client,
 	return 0;
 }
 
+static int tps536c7_identify(struct i2c_client *client,
+			     struct pmbus_driver_info *info)
+{
+	int phases_a, phases_b, page, ret;
+
+	ret = tps536xx_read_phases(client, TPS536C7_DEVICE_ID,
+				   TPS536C7_MAX_PHASES, true,
+				   &phases_a, &phases_b);
+	if (ret)
+		return ret;
+	if (!phases_a) {
+		dev_err(&client->dev,
+			"TPS536C7 without channel A is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	info->format[PSC_VOLTAGE_OUT] = linear;
+	/*
+	 * TPS536C7 can place up to 12 phases on channel A, which exceeds
+	 * PMBUS_PHASES. Report aggregate per-channel telemetry only and do
+	 * not populate info->phases[].
+	 */
+	info->pages = phases_b ? 2 : 1;
+
+	/*
+	 * pmbus_set_page() does not update the PAGE register on single-page
+	 * devices, so select page 0 explicitly in case the boot firmware
+	 * left the device on another page.
+	 */
+	if (info->pages == 1) {
+		ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0);
+		if (ret < 0)
+			return ret;
+	}
+
+	/*
+	 * With info->phases[] left unset the PMBus core never programs the
+	 * PHASE selector, so make sure each page reports the aggregate
+	 * current (PHASE = 0xff) rather than whatever a previous boot left.
+	 */
+	for (page = 0; page < info->pages; page++) {
+		ret = pmbus_read_byte_data(client, page, PMBUS_PHASE);
+		if (ret < 0)
+			return ret;
+		if (ret == 0xff)
+			continue;
+		ret = pmbus_write_byte_data(client, page, PMBUS_PHASE, 0xff);
+		if (ret < 0)
+			return ret;
+		/* PHASE may be write-protected; confirm it actually changed. */
+		ret = pmbus_read_byte_data(client, page, PMBUS_PHASE);
+		if (ret < 0)
+			return ret;
+		if (ret != 0xff) {
+			dev_err(&client->dev,
+				"failed to select aggregate PHASE on page %d\n",
+				page);
+			return -EIO;
+		}
+	}
+	return 0;
+}
+
 static int tps53681_read_word_data(struct i2c_client *client, int page,
 				   int phase, int reg)
 {
@@ -277,6 +386,8 @@ static int tps53679_probe(struct i2c_client *client)
 	case tps53676:
 		info->identify = tps53676_identify;
 		break;
+	case tps53622:
+	case tps53659:
 	case tps53679:
 	case tps53688:
 		info->pages = TPS53679_PAGE_NUM;
@@ -292,6 +403,9 @@ static int tps53679_probe(struct i2c_client *client)
 		info->pages = TPS53679_PAGE_NUM;
 		info->identify = tps53685_identify;
 		break;
+	case tps536c7:
+		info->identify = tps536c7_identify;
+		break;
 	default:
 		return -ENODEV;
 	}
@@ -301,26 +415,32 @@ static int tps53679_probe(struct i2c_client *client)
 
 static const struct i2c_device_id tps53679_id[] = {
 	{ .name = "bmr474", .driver_data = tps53676 },
+	{ .name = "tps53622", .driver_data = tps53622 },
 	{ .name = "tps53647", .driver_data = tps53647 },
+	{ .name = "tps53659", .driver_data = tps53659 },
 	{ .name = "tps53667", .driver_data = tps53667 },
 	{ .name = "tps53676", .driver_data = tps53676 },
 	{ .name = "tps53679", .driver_data = tps53679 },
 	{ .name = "tps53681", .driver_data = tps53681 },
 	{ .name = "tps53685", .driver_data = tps53685 },
 	{ .name = "tps53688", .driver_data = tps53688 },
+	{ .name = "tps536c7", .driver_data = tps536c7 },
 	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tps53679_id);
 
 static const struct of_device_id __maybe_unused tps53679_of_match[] = {
+	{.compatible = "ti,tps53622", .data = (void *)tps53622},
 	{.compatible = "ti,tps53647", .data = (void *)tps53647},
+	{.compatible = "ti,tps53659", .data = (void *)tps53659},
 	{.compatible = "ti,tps53667", .data = (void *)tps53667},
 	{.compatible = "ti,tps53676", .data = (void *)tps53676},
 	{.compatible = "ti,tps53679", .data = (void *)tps53679},
 	{.compatible = "ti,tps53681", .data = (void *)tps53681},
 	{.compatible = "ti,tps53685", .data = (void *)tps53685},
 	{.compatible = "ti,tps53688", .data = (void *)tps53688},
+	{.compatible = "ti,tps536c7", .data = (void *)tps536c7},
 	{}
 };
 MODULE_DEVICE_TABLE(of, tps53679_of_match);
diff --git a/drivers/hwmon/pmbus/xdpe1a2g7b.c b/drivers/hwmon/pmbus/xdpe1a2g7b.c
index 9455ef443b2b35..48f6f8081c4b5b 100644
--- a/drivers/hwmon/pmbus/xdpe1a2g7b.c
+++ b/drivers/hwmon/pmbus/xdpe1a2g7b.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0+
 /*
- * Hardware monitoring driver for Infineon Multi-phase Digital XDPE1A2G5B
- * and XDPE1A2G7B Controllers
+ * Hardware monitoring driver for Infineon Multi-phase Digital XDPE1A2G5B,
+ * XDPE1A2G7B and XDPE1A2G7C Controllers
  *
  * Copyright (c) 2026 Infineon Technologies. All rights reserved.
  */
@@ -23,7 +23,7 @@ static int xdpe1a2g7b_identify(struct i2c_client *client,
 	int vout_mode;
 
 	/*
-	 * XDPE1A2G5B and XDPE1A2G7B support both Linear and NVIDIA PWM VID data
+	 * XDPE1A2G5B and XDPE1A2G7B/7C support both Linear and NVIDIA PWM VID data
 	 * formats via VOUT_MODE. Note that the device pages/loops are not fully
 	 * independent: configuration is shared, so programming each page/loop
 	 * separately is not supported.
@@ -100,6 +100,7 @@ static int xdpe1a2g7b_probe(struct i2c_client *client)
 static const struct i2c_device_id xdpe1a2g7b_id[] = {
 	{ .name = "xdpe1a2g5b" },
 	{ .name = "xdpe1a2g7b" },
+	{ .name = "xdpe1a2g7c" },
 	{ }
 };
 
@@ -108,6 +109,7 @@ MODULE_DEVICE_TABLE(i2c, xdpe1a2g7b_id);
 static const struct of_device_id __maybe_unused xdpe1a2g7b_of_match[] = {
 	{ .compatible = "infineon,xdpe1a2g5b" },
 	{ .compatible = "infineon,xdpe1a2g7b" },
+	{ .compatible = "infineon,xdpe1a2g7c" },
 	{}
 };
 
@@ -125,6 +127,6 @@ static struct i2c_driver xdpe1a2g7b_driver = {
 module_i2c_driver(xdpe1a2g7b_driver);
 
 MODULE_AUTHOR("Ashish Yadav <ashish.yadav@infineon.com>");
-MODULE_DESCRIPTION("PMBus driver for Infineon XDPE1A2G5B/7B");
+MODULE_DESCRIPTION("PMBus driver for Infineon XDPE1A2G5B/7B/7C");
 MODULE_LICENSE("GPL");
 MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pt5161l.c b/drivers/hwmon/pt5161l.c
index 2b408a69b085c1..1638f54f8eaebf 100644
--- a/drivers/hwmon/pt5161l.c
+++ b/drivers/hwmon/pt5161l.c
@@ -611,8 +611,8 @@ static const struct of_device_id __maybe_unused pt5161l_of_match[] = {
 MODULE_DEVICE_TABLE(of, pt5161l_of_match);
 
 static const struct acpi_device_id __maybe_unused pt5161l_acpi_match[] = {
-	{ "PT5161L", 0 },
-	{},
+	{ .id = "PT5161L" },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, pt5161l_acpi_match);
 
diff --git a/drivers/hwmon/pwm-fan.c b/drivers/hwmon/pwm-fan.c
index c633d7f6464c79..9cabbe9f8462b0 100644
--- a/drivers/hwmon/pwm-fan.c
+++ b/drivers/hwmon/pwm-fan.c
@@ -619,7 +619,7 @@ static int pwm_fan_probe(struct platform_device *pdev)
 	}
 
 	channels = devm_kcalloc(dev, channel_count + 1,
-				sizeof(struct hwmon_channel_info *), GFP_KERNEL);
+				sizeof(*channels), GFP_KERNEL);
 	if (!channels)
 		return -ENOMEM;
 
diff --git a/drivers/hwmon/scmi-hwmon.c b/drivers/hwmon/scmi-hwmon.c
index eec223d174c074..9ae8f774297e5e 100644
--- a/drivers/hwmon/scmi-hwmon.c
+++ b/drivers/hwmon/scmi-hwmon.c
@@ -305,7 +305,8 @@ static int scmi_hwmon_probe(struct scmi_device *sdev)
 
 		scmi_sensors->info[type] =
 			devm_kcalloc(dev, nr_count[type],
-				     sizeof(*scmi_sensors->info), GFP_KERNEL);
+				     sizeof(*scmi_sensors->info[type]),
+				     GFP_KERNEL);
 		if (!scmi_sensors->info[type])
 			return -ENOMEM;
 	}
diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c
index a97dda9e92dc5d..ebe072c1676e1b 100644
--- a/drivers/hwmon/sht4x.c
+++ b/drivers/hwmon/sht4x.c
@@ -43,6 +43,7 @@
 #define SHT4X_CRC8_LEN		1
 #define SHT4X_WORD_LEN		2
 #define SHT4X_RESPONSE_LENGTH	6
+#define STS4X_RESPONSE_LENGTH	3
 #define SHT4X_CRC8_POLYNOMIAL	0x31
 #define SHT4X_CRC8_INIT		0xff
 #define SHT4X_MIN_TEMPERATURE	-45000
@@ -52,9 +53,15 @@
 
 DECLARE_CRC8_TABLE(sht4x_crc8_table);
 
+enum sht4x_chips {
+	sht4x,
+	sts4x,
+};
+
 /**
  * struct sht4x_data - All the data required to operate an SHT4X chip
  * @client: the i2c client associated with the SHT4X
+ * @chip_id: the chip type (sht4x or sts4x)
  * @heating_complete: the time that the last heating finished
  * @data_pending: true if and only if there are measurements to retrieve after heating
  * @heater_power: the power at which the heater will be started
@@ -67,7 +74,8 @@ DECLARE_CRC8_TABLE(sht4x_crc8_table);
  */
 struct sht4x_data {
 	struct i2c_client	*client;
-	unsigned long		heating_complete;	/* in jiffies */
+	enum sht4x_chips	chip_id;
+	u64			heating_complete;	/* in jiffies */
 	bool			data_pending;
 	u32			heater_power;	/* in milli-watts */
 	u32			heater_time;	/* in milli-seconds */
@@ -92,14 +100,18 @@ static int sht4x_read_values(struct sht4x_data *data)
 	u8 crc;
 	u8 cmd[SHT4X_CMD_LEN] = {SHT4X_CMD_MEASURE_HPM};
 	u8 raw_data[SHT4X_RESPONSE_LENGTH];
-	unsigned long curr_jiffies;
+	size_t response_length = data->chip_id == sts4x ?
+				 STS4X_RESPONSE_LENGTH : SHT4X_RESPONSE_LENGTH;
 
-	curr_jiffies = jiffies;
-	if (time_before(curr_jiffies, data->heating_complete))
-		msleep(jiffies_to_msecs(data->heating_complete - curr_jiffies));
+	if (data->chip_id != sts4x) {
+		u64 curr_jiffies = get_jiffies_64();
+
+		if (time_before64(curr_jiffies, data->heating_complete))
+			msleep(jiffies_to_msecs(data->heating_complete - curr_jiffies));
+	}
 
 	if (data->data_pending &&
-	    time_before(jiffies, data->heating_complete + data->update_interval)) {
+	    time_before64(get_jiffies_64(), data->heating_complete + data->update_interval)) {
 		data->data_pending = false;
 	} else {
 		next_update = data->last_updated +
@@ -115,15 +127,14 @@ static int sht4x_read_values(struct sht4x_data *data)
 		usleep_range(SHT4X_MEAS_DELAY_HPM, SHT4X_MEAS_DELAY_HPM + SHT4X_DELAY_EXTRA);
 	}
 
-	ret = i2c_master_recv(client, raw_data, SHT4X_RESPONSE_LENGTH);
-	if (ret != SHT4X_RESPONSE_LENGTH) {
+	ret = i2c_master_recv(client, raw_data, response_length);
+	if (ret != response_length) {
 		if (ret >= 0)
 			ret = -ENODATA;
 		return ret;
 	}
 
 	t_ticks = raw_data[0] << 8 | raw_data[1];
-	rh_ticks = raw_data[3] << 8 | raw_data[4];
 
 	crc = crc8(sht4x_crc8_table, &raw_data[0], SHT4X_WORD_LEN, CRC8_INIT_VALUE);
 	if (crc != raw_data[2]) {
@@ -131,14 +142,19 @@ static int sht4x_read_values(struct sht4x_data *data)
 		return -EIO;
 	}
 
-	crc = crc8(sht4x_crc8_table, &raw_data[3], SHT4X_WORD_LEN, CRC8_INIT_VALUE);
-	if (crc != raw_data[5]) {
-		dev_err(&client->dev, "data integrity check failed\n");
-		return -EIO;
+	data->temperature = ((21875 * (int32_t)t_ticks) >> 13) - 45000;
+
+	if (data->chip_id != sts4x) {
+		rh_ticks = raw_data[3] << 8 | raw_data[4];
+		crc = crc8(sht4x_crc8_table, &raw_data[3], SHT4X_WORD_LEN, CRC8_INIT_VALUE);
+		if (crc != raw_data[5]) {
+			dev_err(&client->dev, "data integrity check failed\n");
+			return -EIO;
+		}
+
+		data->humidity = ((15625 * (int32_t)rh_ticks) >> 13) - 6000;
 	}
 
-	data->temperature = ((21875 * (int32_t)t_ticks) >> 13) - 45000;
-	data->humidity = ((15625 * (int32_t)rh_ticks) >> 13) - 6000;
 	data->last_updated = jiffies;
 	data->valid = true;
 	return 0;
@@ -190,9 +206,14 @@ static umode_t sht4x_hwmon_visible(const void *data,
 				   enum hwmon_sensor_types type,
 				   u32 attr, int channel)
 {
+	const struct sht4x_data *chip_data = data;
+
 	switch (type) {
 	case hwmon_temp:
+		return 0444;
 	case hwmon_humidity:
+		if (chip_data->chip_id == sts4x)
+			return 0;
 		return 0444;
 	case hwmon_chip:
 		return 0644;
@@ -237,7 +258,7 @@ static ssize_t heater_enable_show(struct device *dev,
 {
 	struct sht4x_data *data = dev_get_drvdata(dev);
 
-	return sysfs_emit(buf, "%u\n", time_before(jiffies, data->heating_complete));
+	return sysfs_emit(buf, "%u\n", time_before64(get_jiffies_64(), data->heating_complete));
 }
 
 static ssize_t heater_enable_store(struct device *dev,
@@ -279,14 +300,14 @@ static ssize_t heater_enable_store(struct device *dev,
 
 	guard(hwmon_lock)(dev);
 
-	if (time_before(jiffies, data->heating_complete))
+	if (time_before64(get_jiffies_64(), data->heating_complete))
 		return -EBUSY;
 
 	ret = i2c_master_send(data->client, &cmd, SHT4X_CMD_LEN);
 	if (ret < 0)
 		return ret;
 
-	data->heating_complete = jiffies + msecs_to_jiffies(heating_time_bound);
+	data->heating_complete = get_jiffies_64() + msecs_to_jiffies(heating_time_bound);
 	data->data_pending = true;
 	return count;
 }
@@ -388,6 +409,7 @@ static const struct hwmon_chip_info sht4x_chip_info = {
 
 static int sht4x_probe(struct i2c_client *client)
 {
+	const struct attribute_group **groups = NULL;
 	struct device *device = &client->dev;
 	struct device *hwmon_dev;
 	struct sht4x_data *data;
@@ -406,11 +428,15 @@ static int sht4x_probe(struct i2c_client *client)
 	if (!data)
 		return -ENOMEM;
 
+	data->chip_id = (uintptr_t)i2c_get_match_data(client);
 	data->update_interval = SHT4X_MIN_POLL_INTERVAL;
 	data->client = client;
-	data->heater_power = 200;
-	data->heater_time = 1000;
-	data->heating_complete = jiffies;
+	if (data->chip_id != sts4x) {
+		data->heater_power = 200;
+		data->heater_time = 1000;
+		data->heating_complete = get_jiffies_64();
+		groups = sht4x_groups;
+	}
 
 	crc8_populate_msb(sht4x_crc8_table, SHT4X_CRC8_POLYNOMIAL);
 
@@ -424,19 +450,21 @@ static int sht4x_probe(struct i2c_client *client)
 							 client->name,
 							 data,
 							 &sht4x_chip_info,
-							 sht4x_groups);
+							 groups);
 
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
 static const struct i2c_device_id sht4x_id[] = {
-	{ .name = "sht4x" },
+	{ .name = "sht4x", .driver_data = sht4x },
+	{ .name = "sts4x", .driver_data = sts4x },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sht4x_id);
 
 static const struct of_device_id sht4x_of_match[] = {
-	{ .compatible = "sensirion,sht4x" },
+	{ .compatible = "sensirion,sht4x", .data = (void *)sht4x },
+	{ .compatible = "sensirion,sts40", .data = (void *)sts4x },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sht4x_of_match);
diff --git a/drivers/hwmon/socfpga-hwmon.c b/drivers/hwmon/socfpga-hwmon.c
index 5b43274d0aa2d3..5fe5ccab1e4f73 100644
--- a/drivers/hwmon/socfpga-hwmon.c
+++ b/drivers/hwmon/socfpga-hwmon.c
@@ -433,6 +433,26 @@ static const struct socfpga_hwmon_board_data agilex_hwmon_board = {
 	.num_volt = ARRAY_SIZE(agilex_hwmon_volt_channels),
 };
 
+/*
+ * Agilex 5 exposes the SDM and three corner temperature sensors. Channel 2
+ * (top-left corner on Agilex) is not present in hardware, so the SDM channel
+ * numbering keeps the gap (0, 1, 3, 4) rather than renumbering.
+ */
+static const struct socfpga_hwmon_channel agilex5_hwmon_temp_channels[] = {
+	{ SOCFPGA_HWMON_CHAN(0, 0), "Main Die SDM" },
+	{ SOCFPGA_HWMON_CHAN(1, 0), "Main Die corner bottom left max" },
+	{ SOCFPGA_HWMON_CHAN(3, 0), "Main Die corner bottom right max" },
+	{ SOCFPGA_HWMON_CHAN(4, 0), "Main Die corner top right max" },
+};
+
+/* Agilex 5 reuses the Agilex voltage SDM page/channel encoding and labels. */
+static const struct socfpga_hwmon_board_data agilex5_hwmon_board = {
+	.temp = agilex5_hwmon_temp_channels,
+	.num_temp = ARRAY_SIZE(agilex5_hwmon_temp_channels),
+	.volt = agilex_hwmon_volt_channels,
+	.num_volt = ARRAY_SIZE(agilex_hwmon_volt_channels),
+};
+
 static const struct socfpga_hwmon_board_data *
 socfpga_hwmon_get_board(struct device *dev)
 {
@@ -443,6 +463,8 @@ socfpga_hwmon_get_board(struct device *dev)
 
 	if (of_device_is_compatible(np, "intel,stratix10-svc"))
 		return &s10_hwmon_board;
+	if (of_device_is_compatible(np, "intel,agilex5-svc"))
+		return &agilex5_hwmon_board;
 	if (of_device_is_compatible(np, "intel,agilex-svc"))
 		return &agilex_hwmon_board;
 
diff --git a/drivers/hwmon/spd5118.c b/drivers/hwmon/spd5118.c
index 9724cf70b61d4d..52f35448934116 100644
--- a/drivers/hwmon/spd5118.c
+++ b/drivers/hwmon/spd5118.c
@@ -16,6 +16,7 @@
 
 #include <linux/bitops.h>
 #include <linux/bits.h>
+#include <linux/dev_printk.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/i3c/device.h>
@@ -637,44 +638,54 @@ static int spd5118_i2c_init(struct i2c_client *client)
 				     I2C_FUNC_SMBUS_WORD_DATA))
 		return -ENODEV;
 
+	/* Early check to avoid obviously unsupported I2C devices */
 	regval = i2c_smbus_read_word_swapped(client, SPD5118_REG_TYPE);
-	if (regval < 0 || (regval && regval != 0x5118))
+	if (regval < 0)
+		return regval;
+
+	/*
+	 * Some SPD5118 devices report 0x0 when page 0 is not selected,
+	 * so we only fail here if the register value is not 0x0 or 0x5118.
+	 */
+	if (regval && regval != 0x5118)
 		return -ENODEV;
 
 	/*
-	 * If the device type registers return 0, it is possible that the chip
-	 * has a non-zero page selected and takes the specification literally,
+	 * We must select page 0 to ensure that we can reliably read the volatile
+	 * registers on chips that take the specification literally,
 	 * i.e. disables access to volatile registers besides the page register
 	 * if the page is not 0. The Renesas/ITD SPD5118 Hub Controller is known
-	 * to show this behavior. Try to identify such chips.
+	 * to show this behavior.
+	 *
+	 * We must also perform an unconditional register write to detect if
+	 * the i2c controller blocks write accesses to the SPD device. Some Intel
+	 * controllers might be configured by the BIOS to do this.
 	 */
-	if (!regval) {
-		/* Vendor ID registers must also be 0 */
-		regval = i2c_smbus_read_word_data(client, SPD5118_REG_VENDOR);
-		if (regval)
-			return -ENODEV;
-
-		/* The selected page in MR11 must not be 0 */
-		mode = i2c_smbus_read_byte_data(client, SPD5118_REG_I2C_LEGACY_MODE);
-		if (mode < 0 || (mode & ~SPD5118_LEGACY_MODE_MASK) ||
-		    !(mode & SPD5118_LEGACY_PAGE_MASK))
-			return -ENODEV;
-
-		err = i2c_smbus_write_byte_data(client, SPD5118_REG_I2C_LEGACY_MODE,
-						mode & SPD5118_LEGACY_MODE_ADDR);
-		if (err)
-			return -ENODEV;
-
-		/*
-		 * If the device type registers are still bad after selecting
-		 * page 0, this is not a SPD5118 device. Restore original
-		 * legacy mode register value and abort.
-		 */
-		regval = i2c_smbus_read_word_swapped(client, SPD5118_REG_TYPE);
-		if (regval != 0x5118) {
-			i2c_smbus_write_byte_data(client, SPD5118_REG_I2C_LEGACY_MODE, mode);
-			return -ENODEV;
-		}
+	mode = i2c_smbus_read_byte_data(client, SPD5118_REG_I2C_LEGACY_MODE);
+	if (mode < 0)
+		return mode;
+
+	/* 16-bit addressing is not supported */
+	if (mode & SPD5118_LEGACY_MODE_ADDR) {
+		dev_notice(&client->dev,
+			   "Unable to access device due to 16-bit addressing being enabled\n");
+		return -ENODEV;
+	}
+
+	err = i2c_smbus_write_byte_data(client, SPD5118_REG_I2C_LEGACY_MODE,
+					mode & ~SPD5118_LEGACY_PAGE_MASK);
+	if (err < 0)
+		return err;
+
+	/* We only need to access SPD5118_REG_TYPE again if regval was 0x0 */
+	if (regval == 0x5118)
+		return 0;
+
+	regval = i2c_smbus_read_word_swapped(client, SPD5118_REG_TYPE);
+	if (regval != 0x5118) {
+		/* Restore original register content */
+		i2c_smbus_write_byte_data(client, SPD5118_REG_I2C_LEGACY_MODE, mode);
+		return -ENODEV;
 	}
 
 	/* We are reasonably sure that this is really a SPD5118 hub controller */
diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c
index 50a8b050d8da1b..8c8463ddb6c453 100644
--- a/drivers/hwmon/tmp102.c
+++ b/drivers/hwmon/tmp102.c
@@ -58,7 +58,7 @@ struct tmp102 {
 	const char *label;
 	struct regmap *regmap;
 	u16 config_orig;
-	unsigned long ready_time;
+	u64 ready_time;
 	u16 sample_time;
 };
 
@@ -106,7 +106,7 @@ static int tmp102_read_temp(struct device *dev, u32 attr, long *val)
 	switch (attr) {
 	case hwmon_temp_input:
 		/* Is it too early to return a conversion ? */
-		if (time_before(jiffies, tmp102->ready_time)) {
+		if (time_before64(get_jiffies_64(), tmp102->ready_time)) {
 			dev_dbg(dev, "%s: Conversion not ready yet..\n", __func__);
 			return -EAGAIN;
 		}
@@ -354,7 +354,7 @@ static int tmp102_probe(struct i2c_client *client)
 	 * Mark that we are not ready with data until the first
 	 * conversion is complete
 	 */
-	tmp102->ready_time = jiffies + msecs_to_jiffies(CONVERSION_TIME_MS);
+	tmp102->ready_time = get_jiffies_64() + msecs_to_jiffies(CONVERSION_TIME_MS);
 
 	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
 							 tmp102,
@@ -387,7 +387,7 @@ static int tmp102_resume(struct device *dev)
 	err = regmap_update_bits(tmp102->regmap, TMP102_CONF_REG,
 				 TMP102_CONF_SD, 0);
 
-	tmp102->ready_time = jiffies + msecs_to_jiffies(CONVERSION_TIME_MS);
+	tmp102->ready_time = get_jiffies_64() + msecs_to_jiffies(CONVERSION_TIME_MS);
 
 	return err;
 }
diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c
index 9fa31bd66ff694..91d20a1ce11204 100644
--- a/drivers/hwmon/tmp108.c
+++ b/drivers/hwmon/tmp108.c
@@ -78,7 +78,7 @@
 struct tmp108 {
 	struct regmap *regmap;
 	u16 orig_config;
-	unsigned long ready_time;
+	u64 ready_time;
 	const struct tmp108_params *params;
 };
 
@@ -138,7 +138,7 @@ static int tmp108_read(struct device *dev, enum hwmon_sensor_types type,
 	switch (attr) {
 	case hwmon_temp_input:
 		/* Is it too early to return a conversion ? */
-		if (time_before(jiffies, tmp108->ready_time)) {
+		if (time_before64(get_jiffies_64(), tmp108->ready_time)) {
 			dev_dbg(dev, "%s: Conversion not ready yet..\n",
 				__func__);
 			return -EAGAIN;
@@ -477,7 +477,7 @@ static int tmp108_common_probe(struct device *dev, struct regmap *regmap, char *
 		return err;
 	}
 
-	tmp108->ready_time = jiffies;
+	tmp108->ready_time = get_jiffies_64();
 	if ((tmp108->orig_config & TMP108_CONF_MODE_MASK) ==
 	    TMP108_MODE_SHUTDOWN)
 		tmp108->ready_time +=
@@ -528,7 +528,7 @@ static int tmp108_resume(struct device *dev)
 
 	err = regmap_update_bits(tmp108->regmap, TMP108_REG_CONF,
 				 TMP108_CONF_MODE_MASK, TMP108_MODE_CONTINUOUS);
-	tmp108->ready_time = jiffies +
+	tmp108->ready_time = get_jiffies_64() +
 			     msecs_to_jiffies(TMP108_CONVERSION_TIME_MS);
 	return err;
 }
diff --git a/drivers/hwmon/xgene-hwmon.c b/drivers/hwmon/xgene-hwmon.c
index 38b140c23c88eb..eebbb741deac63 100644
--- a/drivers/hwmon/xgene-hwmon.c
+++ b/drivers/hwmon/xgene-hwmon.c
@@ -595,9 +595,9 @@ static void xgene_hwmon_tx_done(struct mbox_client *cl, void *msg, int ret)
 
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id xgene_hwmon_acpi_match[] = {
-	{"APMC0D29", XGENE_HWMON_V1},
-	{"APMC0D8A", XGENE_HWMON_V2},
-	{},
+	{ .id = "APMC0D29", .driver_data = XGENE_HWMON_V1 },
+	{ .id = "APMC0D8A", .driver_data = XGENE_HWMON_V2 },
+	{ }
 };
 MODULE_DEVICE_TABLE(acpi, xgene_hwmon_acpi_match);
 #endif
diff --git a/drivers/hwmon/yogafan.c b/drivers/hwmon/yogafan.c
index 278cb089b0fd17..aa328456071563 100644
--- a/drivers/hwmon/yogafan.c
+++ b/drivers/hwmon/yogafan.c
@@ -53,6 +53,11 @@ struct yoga_fan_data {
 };
 
 /* Specific configurations mapped via DMI */
+static const struct yogafan_config yoga_740_15iml_cfg = {
+	.multiplier = 100,
+	.fan_count = 1,
+	.paths = { "\\_SB.PCI0.LPCB.EC0.FANS", NULL }
+};
 
 static const struct yogafan_config yoga_8bit_fans_cfg = {
 	.multiplier = 100,
@@ -90,6 +95,17 @@ static const struct yogafan_config yoga_pro_7_14iah10_cfg = {
 	.paths = { "\\_SB.PC00.LPCB.EC0.FANS", NULL }
 };
 
+/*
+ * Lenovo Yoga Pro 9 16IMH9 (83DN) uses the PC00 namespace and has two
+ * 8-bit fan tachometer fields.
+ */
+static const struct yogafan_config yoga_pro_83dn_cfg = {
+	.multiplier = 100,
+	.fan_count = 2,
+	.paths = { "\\_SB.PC00.LPCB.EC0.FANS",
+		   "\\_SB.PC00.LPCB.EC0.FA2S" }
+};
+
 static void apply_rllag_filter(struct yoga_fan_data *data, int idx, long raw_rpm)
 {
 	ktime_t now = ktime_get_boottime();
@@ -189,6 +205,30 @@ static const struct hwmon_chip_info yoga_fan_chip_info = {
 };
 
 static const struct dmi_system_id yogafan_quirks[] = {
+	{
+		.ident = "Lenovo Yoga 740-15IML",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_FAMILY, "Yoga 740-15IML"),
+		},
+		.driver_data = (void *)&yoga_740_15iml_cfg,
+	},
+	{
+		.ident = "Lenovo IdeaPad 3 15ALC6 Ub",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_FAMILY, "IdeaPad 3 15ALC6 Ub"),
+		},
+		.driver_data = (void *)&xiaoxin_8bit_dual_cfg,
+	},
+	{
+		.ident = "Yoga 14cACN 2021",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_FAMILY, "Yoga 14cACN 2021"),
+		},
+		.driver_data = (void *)&yoga_8bit_fans_cfg,
+	},
 	{
 		.ident = "Lenovo LOQ 15IAX9",
 		.matches = {
@@ -237,6 +277,14 @@ static const struct dmi_system_id yogafan_quirks[] = {
 		},
 		.driver_data = (void *)&xiaoxin_8bit_dual_cfg,
 	},
+	{
+		.ident = "Lenovo Yoga Pro 9 16IMH9 (83DN)",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "83DN"),
+		},
+		.driver_data = (void *)&yoga_pro_83dn_cfg,
+	},
 	{
 		.ident = "Lenovo Yoga",
 		.matches = {
diff --git a/drivers/i2c/Kconfig b/drivers/i2c/Kconfig
index c232054fddd6fd..1d9691ec0e7ea8 100644
--- a/drivers/i2c/Kconfig
+++ b/drivers/i2c/Kconfig
@@ -136,6 +136,19 @@ config I2C_SLAVE_TESTUNIT
 
 endif
 
+config I2C_DYNAMIC_TIMEOUT
+	bool "Dynamic per-transfer timeout based on message length"
+	depends on I2C
+	help
+	  When enabled, the I2C core computes a per-transfer timeout from the
+	  message length and bus frequency instead of using a static 1-second
+	  default.  A timeout explicitly configured via the I2C_TIMEOUT userspace
+	  interface always takes precedence over the computed value.
+
+	  When disabled, the existing static 1-second timeout is preserved.
+
+	  If unsure, say N.
+
 config I2C_DEBUG_CORE
 	bool "I2C Core debugging messages"
 	help
diff --git a/drivers/i2c/busses/i2c-amd-asf-plat.c b/drivers/i2c/busses/i2c-amd-asf-plat.c
index b0b40fe1d79bfe..8266267caa126a 100644
--- a/drivers/i2c/busses/i2c-amd-asf-plat.c
+++ b/drivers/i2c/busses/i2c-amd-asf-plat.c
@@ -15,6 +15,7 @@
 #include <linux/errno.h>
 #include <linux/gfp_types.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/module.h>
diff --git a/drivers/i2c/busses/i2c-at91-core.c b/drivers/i2c/busses/i2c-at91-core.c
index b64adef778d4b8..8ca4556d966491 100644
--- a/drivers/i2c/busses/i2c-at91-core.c
+++ b/drivers/i2c/busses/i2c-at91-core.c
@@ -255,6 +255,7 @@ static int at91_twi_probe(struct platform_device *pdev)
 	if (rc) {
 		pm_runtime_disable(dev->dev);
 		pm_runtime_set_suspended(dev->dev);
+		at91_twi_dma_release(dev);
 
 		return rc;
 	}
@@ -270,6 +271,8 @@ static void at91_twi_remove(struct platform_device *pdev)
 
 	i2c_del_adapter(&dev->adapter);
 
+	at91_twi_dma_release(dev);
+
 	pm_runtime_disable(dev->dev);
 	pm_runtime_set_suspended(dev->dev);
 }
diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c
index 894cedbca99f57..68238cc8aee0d1 100644
--- a/drivers/i2c/busses/i2c-at91-master.c
+++ b/drivers/i2c/busses/i2c-at91-master.c
@@ -817,11 +817,21 @@ static int at91_twi_configure_dma(struct at91_twi_dev *dev, u32 phy_addr)
 error:
 	if (ret != -EPROBE_DEFER)
 		dev_info(dev->dev, "can't get DMA channel, continue without DMA support\n");
+	at91_twi_dma_release(dev);
+	return ret;
+}
+
+void at91_twi_dma_release(struct at91_twi_dev *dev)
+{
+	struct at91_twi_dma *dma = &dev->dma;
+
 	if (dma->chan_rx)
 		dma_release_channel(dma->chan_rx);
 	if (dma->chan_tx)
 		dma_release_channel(dma->chan_tx);
-	return ret;
+	dma->chan_rx = NULL;
+	dma->chan_tx = NULL;
+	dev->use_dma = false;
 }
 
 static int at91_init_twi_recovery_gpio(struct platform_device *pdev,
diff --git a/drivers/i2c/busses/i2c-at91.h b/drivers/i2c/busses/i2c-at91.h
index 942e9c3973bb57..d68fcbbc3e0f5d 100644
--- a/drivers/i2c/busses/i2c-at91.h
+++ b/drivers/i2c/busses/i2c-at91.h
@@ -172,6 +172,7 @@ void at91_twi_irq_restore(struct at91_twi_dev *dev);
 void at91_init_twi_bus(struct at91_twi_dev *dev);
 
 void at91_init_twi_bus_master(struct at91_twi_dev *dev);
+void at91_twi_dma_release(struct at91_twi_dev *dev);
 int at91_twi_probe_master(struct platform_device *pdev, u32 phy_addr,
 			  struct at91_twi_dev *dev);
 
diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c
index 0d7e2654a534e9..b30f7e0e45f160 100644
--- a/drivers/i2c/busses/i2c-bcm2835.c
+++ b/drivers/i2c/busses/i2c-bcm2835.c
@@ -166,7 +166,7 @@ static struct clk *bcm2835_i2c_register_div(struct device *dev,
 					struct clk *mclk,
 					struct bcm2835_i2c_dev *i2c_dev)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_bcm2835_i2c *priv;
 	char name[32];
 	const char *mclk_name;
diff --git a/drivers/i2c/busses/i2c-gxp.c b/drivers/i2c/busses/i2c-gxp.c
index 6a5cef7e438397..ab80e83ccc12dd 100644
--- a/drivers/i2c/busses/i2c-gxp.c
+++ b/drivers/i2c/busses/i2c-gxp.c
@@ -4,6 +4,7 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c
index 1cfd7a4c823701..49aabef7f115c1 100644
--- a/drivers/i2c/busses/i2c-imx-lpi2c.c
+++ b/drivers/i2c/busses/i2c-imx-lpi2c.c
@@ -1555,23 +1555,13 @@ static int lpi2c_imx_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 
 	/*
-	 * Reset all internal controller registers to avoid effects of any
-	 * state left over from a previous stage (e.g. the bootloader).
-	 *
-	 * The Master block (MCR) is present on every controller, so reset it
-	 * unconditionally. VERID shows whether the target feature is supported.
-	 * Do not touch the Target block (SCR) on a master-only controller to
-	 * avoid an asynchronous SError.
+	 * Reset all internal controller registers of both Master and Target
+	 * to avoid effects of previous status.
 	 */
 	writel(MCR_RST, lpi2c_imx->base + LPI2C_MCR);
+	writel(SCR_RST, lpi2c_imx->base + LPI2C_SCR);
 	writel(0, lpi2c_imx->base + LPI2C_MCR);
-
-	lpi2c_imx->target_supported = !!(readl(lpi2c_imx->base + LPI2C_VERID) &
-					 VERID_FEATURE_TARGET_PRESENT);
-	if (lpi2c_imx->target_supported) {
-		writel(SCR_RST, lpi2c_imx->base + LPI2C_SCR);
-		writel(0, lpi2c_imx->base + LPI2C_SCR);
-	}
+	writel(0, lpi2c_imx->base + LPI2C_SCR);
 
 	ret = devm_request_irq(&pdev->dev, lpi2c_imx->irq, lpi2c_imx_isr, IRQF_NO_SUSPEND,
 			       pdev->name, lpi2c_imx);
diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
index 19ec056b00afeb..ff5a91158df98d 100644
--- a/drivers/i2c/busses/i2c-imx.c
+++ b/drivers/i2c/busses/i2c-imx.c
@@ -1880,6 +1880,8 @@ static int i2c_imx_probe(struct platform_device *pdev)
 
 clk_notifier_unregister:
 	clk_notifier_unregister(i2c_imx->clk, &i2c_imx->clk_change_nb);
+	if (i2c_imx->dma)
+		i2c_imx_dma_free(i2c_imx);
 	free_irq(irq, i2c_imx);
 rpm_disable:
 	pm_runtime_put_noidle(&pdev->dev);
@@ -1920,6 +1922,7 @@ static void i2c_imx_remove(struct platform_device *pdev)
 
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 }
 
 static int i2c_imx_runtime_suspend(struct device *dev)
diff --git a/drivers/i2c/busses/i2c-k1.c b/drivers/i2c/busses/i2c-k1.c
index 4e1d6754113220..13b396b5d3db1b 100644
--- a/drivers/i2c/busses/i2c-k1.c
+++ b/drivers/i2c/busses/i2c-k1.c
@@ -8,6 +8,7 @@
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
diff --git a/drivers/i2c/busses/i2c-ljca.c b/drivers/i2c/busses/i2c-ljca.c
index 93274f0c2d7216..da3370968c8ed1 100644
--- a/drivers/i2c/busses/i2c-ljca.c
+++ b/drivers/i2c/busses/i2c-ljca.c
@@ -297,7 +297,6 @@ static int ljca_i2c_probe(struct auxiliary_device *auxdev,
 	device_set_node(&ljca_i2c->adap.dev, dev_fwnode(&auxdev->dev));
 
 	i2c_set_adapdata(&ljca_i2c->adap, ljca_i2c);
-	auxiliary_set_drvdata(auxdev, ljca_i2c);
 
 	ret = ljca_i2c_init(ljca_i2c, ljca_i2c->i2c_info->id);
 	if (ret)
@@ -315,13 +314,6 @@ static int ljca_i2c_probe(struct auxiliary_device *auxdev,
 	return 0;
 }
 
-static void ljca_i2c_remove(struct auxiliary_device *auxdev)
-{
-	struct ljca_i2c_dev *ljca_i2c = auxiliary_get_drvdata(auxdev);
-
-	i2c_del_adapter(&ljca_i2c->adap);
-}
-
 static const struct auxiliary_device_id ljca_i2c_id_table[] = {
 	{ "usb_ljca.ljca-i2c", 0 },
 	{ /* sentinel */ }
@@ -330,7 +322,6 @@ MODULE_DEVICE_TABLE(auxiliary, ljca_i2c_id_table);
 
 static struct auxiliary_driver ljca_i2c_driver = {
 	.probe = ljca_i2c_probe,
-	.remove = ljca_i2c_remove,
 	.id_table = ljca_i2c_id_table,
 };
 module_auxiliary_driver(ljca_i2c_driver);
diff --git a/drivers/i2c/busses/i2c-pasemi-platform.c b/drivers/i2c/busses/i2c-pasemi-platform.c
index a486a37d386345..f6d98647f9df9c 100644
--- a/drivers/i2c/busses/i2c-pasemi-platform.c
+++ b/drivers/i2c/busses/i2c-pasemi-platform.c
@@ -7,6 +7,7 @@
 
 #include <linux/clk.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/drivers/i2c/busses/i2c-qcom-cci.c b/drivers/i2c/busses/i2c-qcom-cci.c
index 25b6e4e9e3fa31..873e901a23d7a2 100644
--- a/drivers/i2c/busses/i2c-qcom-cci.c
+++ b/drivers/i2c/busses/i2c-qcom-cci.c
@@ -497,10 +497,14 @@ static const struct dev_pm_ops qcom_cci_pm = {
 	SET_RUNTIME_PM_OPS(cci_suspend_runtime, cci_resume_runtime, NULL)
 };
 
+static void cci_put_of_node(void *data)
+{
+	of_node_put(data);
+}
+
 static int cci_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *child;
 	struct resource *r;
 	struct cci *cci;
 	int ret, i;
@@ -516,7 +520,7 @@ static int cci_probe(struct platform_device *pdev)
 	if (!cci->data)
 		return -ENOENT;
 
-	for_each_available_child_of_node(dev->of_node, child) {
+	for_each_available_child_of_node_scoped(dev->of_node, child) {
 		struct cci_master *master;
 		u32 idx;
 
@@ -537,6 +541,9 @@ static int cci_probe(struct platform_device *pdev)
 		master->adap.algo = &cci_algo;
 		master->adap.dev.parent = dev;
 		master->adap.dev.of_node = of_node_get(child);
+		ret = devm_add_action_or_reset(dev, cci_put_of_node, child);
+		if (ret)
+			return ret;
 		master->master = idx;
 		master->cci = cci;
 
@@ -606,10 +613,8 @@ static int cci_probe(struct platform_device *pdev)
 			continue;
 
 		ret = i2c_add_adapter(&cci->master[i].adap);
-		if (ret < 0) {
-			of_node_put(cci->master[i].adap.dev.of_node);
+		if (ret < 0)
 			goto error_i2c;
-		}
 	}
 
 	return 0;
@@ -617,10 +622,8 @@ static int cci_probe(struct platform_device *pdev)
 error_i2c:
 
 	for (--i ; i >= 0; i--) {
-		if (cci->master[i].cci) {
+		if (cci->master[i].cci)
 			i2c_del_adapter(&cci->master[i].adap);
-			of_node_put(cci->master[i].adap.dev.of_node);
-		}
 	}
 disable_clocks:
 	cci_disable_clocks(cci);
@@ -636,7 +639,6 @@ static void cci_remove(struct platform_device *pdev)
 	for (i = 0; i < cci->data->num_masters; i++) {
 		if (cci->master[i].cci) {
 			i2c_del_adapter(&cci->master[i].adap);
-			of_node_put(cci->master[i].adap.dev.of_node);
 			cci_halt(cci, i);
 		}
 	}
diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c
index 00013b41a6f597..0f3f717f194e42 100644
--- a/drivers/i2c/busses/i2c-qcom-geni.c
+++ b/drivers/i2c/busses/i2c-qcom-geni.c
@@ -79,9 +79,17 @@ enum geni_i2c_err_code {
 
 #define ABORT_TIMEOUT		HZ
 #define CANCEL_TIMEOUT		HZ
-#define XFER_TIMEOUT		HZ
 #define RST_TIMEOUT		HZ
 
+/* 9 bits per byte (8 data + 1 ACK), 10x safety margin */
+#define I2C_TIMEOUT_SAFETY_COEFFICIENT	10
+
+/* 300ms floor: budget for clock stretching; slave may hold SCL low indefinitely */
+#define I2C_TIMEOUT_MIN_USEC		300000
+
+#define GENI_SE_CLK_32MHZ	(32 * HZ_PER_MHZ)
+#define GENI_SE_CLK_19P2MHZ	19200000UL
+
 struct geni_i2c_desc {
 	bool no_dma_support;
 	unsigned int tx_fifo_depth;
@@ -127,6 +135,7 @@ struct geni_i2c_dev {
 	spinlock_t lock;
 	u32 clk_freq_out;
 	const struct geni_i2c_clk_fld *clk_fld;
+	u32 clk_idx;
 	void *dma_buf;
 	size_t xfer_len;
 	dma_addr_t dma_addr;
@@ -197,19 +206,44 @@ static const struct geni_i2c_clk_fld geni_i2c_clk_map_32mhz[] = {
 static int geni_i2c_clk_map_idx(struct geni_i2c_dev *gi2c)
 {
 	const struct geni_i2c_clk_fld *itr;
+	unsigned long res_freq;
 
-	if (clk_get_rate(gi2c->se.clk) == 32 * HZ_PER_MHZ)
+	/*
+	 * Frequency counter tables are calibrated for a specific source
+	 * clock frequency and are not valid for any multiple of it
+	 * (e.g. 64 MHz, 128 MHz).
+	 * Use exact=true and verify res_freq matches req_freq literally
+	 * to reject harmonics: a 64 MHz clock that divides evenly to
+	 * 32 MHz would pass exact matching but produce double the intended
+	 * I2C frequency with these counter values.
+	 */
+	if (!geni_se_clk_freq_match(&gi2c->se, GENI_SE_CLK_32MHZ,
+				    &gi2c->clk_idx, &res_freq, true) &&
+	    res_freq == GENI_SE_CLK_32MHZ) {
 		itr = geni_i2c_clk_map_32mhz;
-	else
+	} else if (!geni_se_clk_freq_match(&gi2c->se, GENI_SE_CLK_19P2MHZ,
+					   &gi2c->clk_idx, &res_freq, true) &&
+		   res_freq == GENI_SE_CLK_19P2MHZ) {
 		itr = geni_i2c_clk_map_19p2mhz;
+	} else {
+		dev_err(gi2c->se.dev,
+			"Unsupported SE source clock: must be exactly 32 MHz or 19.2 MHz\n");
+		return -EINVAL;
+	}
 
 	while (itr->clk_freq_out != 0) {
 		if (itr->clk_freq_out == gi2c->clk_freq_out) {
 			gi2c->clk_fld = itr;
+			dev_dbg(gi2c->se.dev,
+				"I2C clk selected: freq: %u Hz, clk_idx: %u\n",
+				gi2c->clk_freq_out, gi2c->clk_idx);
 			return 0;
 		}
 		itr++;
 	}
+
+	dev_err(gi2c->se.dev, "Unsupported I2C output frequency %u Hz\n", gi2c->clk_freq_out);
+
 	return -EINVAL;
 }
 
@@ -219,7 +253,7 @@ static int qcom_geni_i2c_conf(struct geni_se *se, unsigned long freq)
 	const struct geni_i2c_clk_fld *itr = gi2c->clk_fld;
 	u32 val;
 
-	writel_relaxed(0, gi2c->se.base + SE_GENI_CLK_SEL);
+	writel_relaxed(gi2c->clk_idx, gi2c->se.base + SE_GENI_CLK_SEL);
 
 	val = (itr->clk_div << CLK_DIV_SHFT) | SER_CLK_EN;
 	writel_relaxed(val, gi2c->se.base + GENI_SER_M_CLK_CFG);
@@ -513,7 +547,10 @@ static int geni_i2c_rx_one_msg(struct geni_i2c_dev *gi2c, struct i2c_msg *msg,
 	}
 
 	cur = gi2c->cur;
-	time_left = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT);
+	i2c_update_timeout(&gi2c->adap, gi2c->clk_freq_out, len,
+			   I2C_TIMEOUT_SAFETY_COEFFICIENT,
+			   I2C_TIMEOUT_MIN_USEC);
+	time_left = wait_for_completion_timeout(&gi2c->done, gi2c->adap.timeout);
 	if (!time_left || (gi2c->err && gi2c->err != gi2c_log[ADDR_NACK].err))
 		geni_i2c_cancel_xfer(gi2c);
 
@@ -555,7 +592,10 @@ static int geni_i2c_tx_one_msg(struct geni_i2c_dev *gi2c, struct i2c_msg *msg,
 		writel_relaxed(1, se->base + SE_GENI_TX_WATERMARK_REG);
 
 	cur = gi2c->cur;
-	time_left = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT);
+	i2c_update_timeout(&gi2c->adap, gi2c->clk_freq_out, len,
+			   I2C_TIMEOUT_SAFETY_COEFFICIENT,
+			   I2C_TIMEOUT_MIN_USEC);
+	time_left = wait_for_completion_timeout(&gi2c->done, gi2c->adap.timeout);
 	if (!time_left || (gi2c->err && gi2c->err != gi2c_log[ADDR_NACK].err))
 		geni_i2c_cancel_xfer(gi2c);
 
@@ -633,7 +673,7 @@ static void geni_i2c_gpi_multi_desc_unmap(struct geni_i2c_dev *gi2c, struct i2c_
  * geni_i2c_gpi_multi_xfer_timeout_handler() - Handles multi message transfer timeout
  * @dev: Pointer to the corresponding dev node
  * @multi_xfer: Pointer to the geni_i2c_gpi_multi_desc_xfer
- * @transfer_timeout_msecs: Timeout value in milliseconds
+ * @timeout_jiffies: Per-message completion timeout in jiffies
  * @transfer_comp: Completion object of the transfer
  *
  * This function waits for the completion of each processed transfer messages
@@ -643,18 +683,18 @@ static void geni_i2c_gpi_multi_desc_unmap(struct geni_i2c_dev *gi2c, struct i2c_
  */
 static int geni_i2c_gpi_multi_xfer_timeout_handler(struct device *dev,
 						   struct geni_i2c_gpi_multi_desc_xfer *multi_xfer,
-						   u32 transfer_timeout_msecs,
+						   unsigned long timeout_jiffies,
 						   struct completion *transfer_comp)
 {
 	int i;
-	u32 time_left;
+	unsigned long time_left;
 
 	for (i = 0; i < multi_xfer->msg_idx_cnt - 1; i++) {
 		reinit_completion(transfer_comp);
 
 		if (multi_xfer->msg_idx_cnt != multi_xfer->irq_cnt) {
 			time_left = wait_for_completion_timeout(transfer_comp,
-								transfer_timeout_msecs);
+								timeout_jiffies);
 			if (!time_left) {
 				dev_err(dev, "%s: Transfer timeout\n", __func__);
 				return -ETIMEDOUT;
@@ -778,8 +818,24 @@ static int geni_i2c_gpi(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[],
 		dma_async_issue_pending(gi2c->tx_c);
 
 		if ((msg_idx == (gi2c->num_msgs - 1)) || flags & DMA_PREP_INTERRUPT) {
+			size_t total_len = 0;
+			int j;
+
+			/*
+			 * All TREs except the last carry the BEI bit, so a single
+			 * completion interrupt fires only after the entire batch has
+			 * drained on the wire. The timeout budget must therefore cover
+			 * the combined wire time of every message in the batch.
+			 */
+			for (j = 0; j < gi2c->num_msgs; j++)
+				total_len += msgs[j].len;
+
+			i2c_update_timeout(&gi2c->adap, gi2c->clk_freq_out, total_len,
+					   I2C_TIMEOUT_SAFETY_COEFFICIENT,
+					   I2C_TIMEOUT_MIN_USEC);
 			ret = geni_i2c_gpi_multi_xfer_timeout_handler(gi2c->se.dev, gi2c_gpi_xfer,
-								      XFER_TIMEOUT, &gi2c->done);
+								      gi2c->adap.timeout,
+								      &gi2c->done);
 			if (ret) {
 				dev_err(gi2c->se.dev,
 					"I2C multi write msg transfer timeout: %d\n",
@@ -899,7 +955,10 @@ static int geni_i2c_gpi_xfer(struct geni_i2c_dev *gi2c, struct i2c_msg msgs[], i
 
 		if (!gi2c->is_tx_multi_desc_xfer) {
 			dma_async_issue_pending(gi2c->tx_c);
-			time_left = wait_for_completion_timeout(&gi2c->done, XFER_TIMEOUT);
+			i2c_update_timeout(&gi2c->adap, gi2c->clk_freq_out, msgs[i].len,
+					   I2C_TIMEOUT_SAFETY_COEFFICIENT,
+					   I2C_TIMEOUT_MIN_USEC);
+			time_left = wait_for_completion_timeout(&gi2c->done, gi2c->adap.timeout);
 			if (!time_left) {
 				dev_err(gi2c->se.dev, "%s:I2C timeout\n", __func__);
 				gi2c->err = -ETIMEDOUT;
@@ -963,10 +1022,9 @@ static int geni_i2c_xfer(struct i2c_adapter *adap,
 	struct geni_i2c_dev *gi2c = i2c_get_adapdata(adap);
 	int ret;
 
-	ret = pm_runtime_get_sync(gi2c->se.dev);
+	ret = pm_runtime_resume_and_get(gi2c->se.dev);
 	if (ret < 0) {
 		dev_err(gi2c->se.dev, "error turning SE resources:%d\n", ret);
-		pm_runtime_put_noidle(gi2c->se.dev);
 		/* Set device in suspended since resume failed */
 		pm_runtime_set_suspended(gi2c->se.dev);
 		return ret;
@@ -1041,10 +1099,8 @@ static int geni_i2c_init(struct geni_i2c_dev *gi2c)
 	int ret;
 
 	ret = pm_runtime_resume_and_get(gi2c->se.dev);
-	if (ret < 0) {
-		dev_err(gi2c->se.dev, "error turning on device :%d\n", ret);
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(gi2c->se.dev, ret, "error turning on device\n");
 
 	proto = geni_se_read_proto(&gi2c->se);
 	if (proto == GENI_SE_INVALID_PROTO) {
@@ -1111,8 +1167,7 @@ static int geni_i2c_resources_init(struct geni_se *se)
 
 	ret = geni_i2c_clk_map_idx(gi2c);
 	if (ret)
-		return dev_err_probe(gi2c->se.dev, ret, "Invalid clk frequency %d Hz\n",
-				     gi2c->clk_freq_out);
+		return ret;
 
 	return geni_icc_set_bw_ab(&gi2c->se, GENI_DEFAULT_BW, GENI_DEFAULT_BW,
 				  Bps_to_icc(gi2c->clk_freq_out));
@@ -1145,9 +1200,6 @@ static int geni_i2c_probe(struct platform_device *pdev)
 		gi2c->clk_freq_out = I2C_MAX_STANDARD_MODE_FREQ;
 	}
 
-	if (has_acpi_companion(dev))
-		ACPI_COMPANION_SET(&gi2c->adap.dev, ACPI_COMPANION(dev));
-
 	gi2c->irq = platform_get_irq(pdev, 0);
 	if (gi2c->irq < 0)
 		return gi2c->irq;
@@ -1173,7 +1225,7 @@ static int geni_i2c_probe(struct platform_device *pdev)
 
 	i2c_set_adapdata(&gi2c->adap, gi2c);
 	gi2c->adap.dev.parent = dev;
-	gi2c->adap.dev.of_node = dev->of_node;
+	device_set_node(&gi2c->adap.dev, dev_fwnode(dev));
 	strscpy(gi2c->adap.name, "Geni-I2C", sizeof(gi2c->adap.name));
 
 	pm_runtime_set_suspended(dev);
@@ -1189,8 +1241,10 @@ static int geni_i2c_probe(struct platform_device *pdev)
 		return ret;
 
 	ret = i2c_add_adapter(&gi2c->adap);
-	if (ret)
+	if (ret) {
+		release_gpi_dma(gi2c);
 		return dev_err_probe(dev, ret, "Error adding i2c adapter\n");
+	}
 
 	dev_dbg(dev, "Geni-I2C adaptor successfully added\n");
 
@@ -1213,7 +1267,7 @@ static void geni_i2c_shutdown(struct platform_device *pdev)
 	i2c_mark_adapter_suspended(&gi2c->adap);
 }
 
-static int __maybe_unused geni_i2c_runtime_suspend(struct device *dev)
+static int geni_i2c_runtime_suspend(struct device *dev)
 {
 	int ret = 0;
 	struct geni_i2c_dev *gi2c = dev_get_drvdata(dev);
@@ -1231,7 +1285,7 @@ static int __maybe_unused geni_i2c_runtime_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused geni_i2c_runtime_resume(struct device *dev)
+static int geni_i2c_runtime_resume(struct device *dev)
 {
 	int ret = 0;
 	struct geni_i2c_dev *gi2c = dev_get_drvdata(dev);
@@ -1247,7 +1301,7 @@ static int __maybe_unused geni_i2c_runtime_resume(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused geni_i2c_suspend_noirq(struct device *dev)
+static int geni_i2c_suspend_noirq(struct device *dev)
 {
 	struct geni_i2c_dev *gi2c = dev_get_drvdata(dev);
 	int ret;
@@ -1261,7 +1315,7 @@ static int __maybe_unused geni_i2c_suspend_noirq(struct device *dev)
 	return ret;
 }
 
-static int __maybe_unused geni_i2c_resume_noirq(struct device *dev)
+static int geni_i2c_resume_noirq(struct device *dev)
 {
 	struct geni_i2c_dev *gi2c = dev_get_drvdata(dev);
 	int ret;
@@ -1275,9 +1329,10 @@ static int __maybe_unused geni_i2c_resume_noirq(struct device *dev)
 }
 
 static const struct dev_pm_ops geni_i2c_pm_ops = {
-	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(geni_i2c_suspend_noirq, geni_i2c_resume_noirq)
-	SET_RUNTIME_PM_OPS(geni_i2c_runtime_suspend, geni_i2c_runtime_resume,
-									NULL)
+	NOIRQ_SYSTEM_SLEEP_PM_OPS(geni_i2c_suspend_noirq,
+				  geni_i2c_resume_noirq)
+	RUNTIME_PM_OPS(geni_i2c_runtime_suspend, geni_i2c_runtime_resume,
+		       NULL)
 };
 
 static const struct geni_i2c_desc geni_i2c = {
diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c
index 3e7735e1dae009..5e397a7e63f6ad 100644
--- a/drivers/i2c/busses/i2c-xiic.c
+++ b/drivers/i2c/busses/i2c-xiic.c
@@ -73,6 +73,9 @@ enum i2c_scl_freq {
  * @prev_msg_tx: Previous message is Tx
  * @quirks: To hold platform specific bug info
  * @smbus_block_read: Flag to handle block read
+ * @smbus_actual_len: Valid byte count (length + payload + optional PEC) of a
+ *	padded SMBus block read. msg->len is trimmed to this on completion.
+ *	Zero when no trimming is needed.
  * @input_clk: Input clock to I2C controller
  * @i2c_clk: I2C SCL frequency
  * @atomic: Mode of transfer
@@ -98,6 +101,7 @@ struct xiic_i2c {
 	bool prev_msg_tx;
 	u32 quirks;
 	bool smbus_block_read;
+	unsigned int smbus_actual_len;
 	unsigned long input_clk;
 	unsigned int i2c_clk;
 	bool atomic;
@@ -539,30 +543,38 @@ static void xiic_smbus_block_read_setup(struct xiic_i2c *i2c)
 
 	/* Check if received length is valid */
 	if (rxmsg_len <= I2C_SMBUS_BLOCK_MAX) {
+		unsigned int pec_len = i2c->rx_msg->len - 1;
+
 		/* Set Receive fifo depth */
-		if (rxmsg_len > IIC_RX_FIFO_DEPTH) {
+		if (rxmsg_len + pec_len > IIC_RX_FIFO_DEPTH) {
 			/*
-			 * When Rx msg len greater than or equal to Rx fifo capacity
-			 * Receive fifo depth should set to Rx fifo capacity minus 1
+			 * Trailing bytes (payload plus any PEC) exceed Rx FIFO
+			 * capacity, so drain in chunks. This also keeps the
+			 * else branch below from pushing rfd_set past the
+			 * 4-bit XIIC_RFD_REG_OFFSET field.
 			 */
 			rfd_set = IIC_RX_FIFO_DEPTH - 1;
-			i2c->rx_msg->len = rxmsg_len + 1;
-		} else if ((rxmsg_len == 1) ||
-			(rxmsg_len == 0)) {
+			i2c->rx_msg->len = rxmsg_len + 1 + pec_len;
+		} else if (1 + rxmsg_len + pec_len < SMBUS_BLOCK_READ_MIN_LEN) {
 			/*
-			 * Minimum of 3 bytes required to exit cleanly. 1 byte
-			 * already received, Second byte is being received. Have
-			 * to set NACK in read_rx before receiving the last byte
+			 * The HW needs SMBUS_BLOCK_READ_MIN_LEN bytes on the
+			 * bus to exit cleanly: by the time the ISR reads the
+			 * length byte the second byte is already being clocked
+			 * in, too late to NACK. Pad the drain target and record
+			 * the real length, trimmed back on completion so the
+			 * PEC check sees the right byte.
 			 */
 			rfd_set = 0;
 			i2c->rx_msg->len = SMBUS_BLOCK_READ_MIN_LEN;
+			i2c->smbus_actual_len = 1 + rxmsg_len + pec_len;
 		} else {
 			/*
-			 * When Rx msg len less than Rx fifo capacity
-			 * Receive fifo depth should set to Rx msg len minus 2
+			 * All trailing bytes fit in the Rx FIFO. Defer RX_FULL
+			 * until every one of them is buffered, so the drain
+			 * takes xiic_read_rx()'s bytes_rem == 0 path.
 			 */
-			rfd_set = rxmsg_len - 2;
-			i2c->rx_msg->len = rxmsg_len + 1;
+			rfd_set = rxmsg_len + pec_len - 1;
+			i2c->rx_msg->len = rxmsg_len + 1 + pec_len;
 		}
 		xiic_setreg8(i2c, XIIC_RFD_REG_OFFSET, rfd_set);
 
@@ -575,6 +587,16 @@ static void xiic_smbus_block_read_setup(struct xiic_i2c *i2c)
 	dev_err(i2c->adap.dev.parent, "smbus_block_read Invalid msg length\n");
 }
 
+/*
+ * Undo the setup-time padding of a short SMBus block read before rx_msg is
+ * cleared, so the PEC check sees the right byte.
+ */
+static void xiic_smbus_trim_len(struct xiic_i2c *i2c)
+{
+	if (i2c->rx_msg && i2c->smbus_actual_len)
+		i2c->rx_msg->len = i2c->smbus_actual_len;
+}
+
 static void xiic_read_rx(struct xiic_i2c *i2c)
 {
 	u8 bytes_in_fifo, cr = 0, bytes_to_read = 0;
@@ -797,6 +819,8 @@ static irqreturn_t xiic_process(int irq, void *dev_id)
 
 		xiic_read_rx(i2c);
 		if (xiic_rx_space(i2c) == 0) {
+			xiic_smbus_trim_len(i2c);
+
 			/* this is the last part of the message */
 			i2c->rx_msg = NULL;
 
@@ -865,8 +889,11 @@ static irqreturn_t xiic_process(int irq, void *dev_id)
 
 		if (i2c->tx_msg && i2c->smbus_block_read) {
 			i2c->smbus_block_read = false;
-			/* Set requested message len=1 to indicate STATE_DONE */
-			i2c->tx_msg->len = 1;
+			/*
+			 * Drive xiic_tx_space() to 0 to signal STATE_DONE
+			 * without truncating the rx_msg length.
+			 */
+			i2c->tx_pos = i2c->tx_msg->len;
 		}
 
 		if (!i2c->tx_msg)
@@ -938,6 +965,7 @@ static void xiic_recv_atomic(struct xiic_i2c *i2c)
 			return;
 	}
 
+	xiic_smbus_trim_len(i2c);
 	i2c->rx_msg = NULL;
 	xiic_irq_clr_en(i2c, XIIC_INTR_TX_ERROR_MASK);
 
@@ -955,6 +983,9 @@ static void xiic_start_recv(struct xiic_i2c *i2c)
 	u8 cr = 0, rfd_set = 0;
 	struct i2c_msg *msg = i2c->rx_msg = i2c->tx_msg;
 
+	/* A stale value from an aborted block read would truncate this msg. */
+	i2c->smbus_actual_len = 0;
+
 	if (!i2c->atomic)
 		dev_dbg(i2c->adap.dev.parent, "%s entry, ISR: 0x%x, CR: 0x%x\n",
 			__func__, xiic_getreg32(i2c, XIIC_IISR_OFFSET),
diff --git a/drivers/i2c/i2c-atr.c b/drivers/i2c/i2c-atr.c
index e6d2af659d8101..ca29633dcd623a 100644
--- a/drivers/i2c/i2c-atr.c
+++ b/drivers/i2c/i2c-atr.c
@@ -855,6 +855,7 @@ int i2c_atr_add_adapter(struct i2c_atr *atr, struct i2c_atr_adap_desc *desc)
 
 	ret = i2c_add_adapter(&chan->adap);
 	if (ret) {
+		atr->adapter[chan_id] = NULL;
 		dev_err(dev, "failed to add atr-adapter %u (error=%d)\n",
 			chan_id, ret);
 		goto err_free_alias_pool;
diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c
index 8f3bdd50186e39..3af0e05511851e 100644
--- a/drivers/i2c/i2c-core-acpi.c
+++ b/drivers/i2c/i2c-core-acpi.c
@@ -371,6 +371,7 @@ static const struct acpi_device_id i2c_acpi_force_100khz_device_ids[] = {
 	 * the device works without issues on Windows at what is expected to be
 	 * a 400KHz frequency. The root cause of the issue is not known.
 	 */
+	{ "DELL0A86", 0 },
 	{ "DLL0945", 0 },
 	{ "ELAN0678", 0 },
 	{ "ELAN06FA", 0 },
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index ddaacf876dada9..a9aa876090b8c3 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -29,6 +29,7 @@
 #include <linux/irq.h>
 #include <linux/jump_label.h>
 #include <linux/kernel.h>
+#include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/of_device.h>
@@ -2007,6 +2008,48 @@ void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_de
 }
 EXPORT_SYMBOL_GPL(i2c_parse_fw_timings);
 
+#ifdef CONFIG_I2C_DYNAMIC_TIMEOUT
+/**
+ * i2c_update_timeout - compute and set a dynamic transfer timeout on an adapter
+ * @adap: the i2c_adapter whose timeout field will be updated
+ * @bus_freq_hz: I2C bus clock frequency in Hz
+ * @len: transfer length in bytes
+ * @safety_coeff: multiplier applied over the theoretical wire time
+ * @min_usec: minimum timeout floor in microseconds
+ *
+ * Computes a transfer-specific timeout from the message length and bus
+ * frequency, applies a safety multiplier and a minimum floor, then stores
+ * the result in adap->timeout (in jiffies).  The caller supplies the policy
+ * constants so they remain internal to the driver.
+ *
+ * A timeout explicitly configured by userspace via the I2C_TIMEOUT ioctl
+ * always takes precedence; the computed value is used only when userspace
+ * has not configured one.
+ */
+void i2c_update_timeout(struct i2c_adapter *adap, u32 bus_freq_hz,
+			size_t len, unsigned int safety_coeff,
+			unsigned int min_usec)
+{
+	u64 bit_usec, total_usec;
+	unsigned long jiffies_val;
+
+	if (adap->user_timeout > 0) {
+		adap->timeout = adap->user_timeout;
+		return;
+	}
+
+	if (WARN_ON_ONCE(!bus_freq_hz))
+		return;
+
+	bit_usec = mul_u64_u32_div(len * 9, USEC_PER_SEC, bus_freq_hz);
+	total_usec = bit_usec * safety_coeff + min_usec;
+
+	jiffies_val = usecs_to_jiffies((unsigned int)min_t(u64, total_usec, UINT_MAX));
+	adap->timeout = (int)min_t(unsigned long, jiffies_val, INT_MAX);
+}
+EXPORT_SYMBOL_GPL(i2c_update_timeout);
+#endif /* CONFIG_I2C_DYNAMIC_TIMEOUT */
+
 /* ------------------------------------------------------------------------- */
 
 int i2c_for_each_dev(void *data, int (*fn)(struct device *dev, void *data))
diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c
index fa63bee0b345dc..57f55bbab8a1a8 100644
--- a/drivers/i2c/i2c-core-smbus.c
+++ b/drivers/i2c/i2c-core-smbus.c
@@ -208,11 +208,11 @@ s32 i2c_smbus_write_word_data(const struct i2c_client *client, u8 command,
 EXPORT_SYMBOL(i2c_smbus_write_word_data);
 
 /**
- * i2c_smbus_read_block_data - SMBus "block read" protocol
+ * __i2c_smbus_read_block_data - SMBus "block read" protocol
  * @client: Handle to slave device
  * @command: Byte interpreted by slave
- * @values: Byte array into which data will be read; big enough to hold
- *	the data returned by the slave.  SMBus allows at most 32 bytes.
+ * @length: size of the @values array. SMBus allows at most 32 bytes
+ * @values: Byte array into which data will be read
  *
  * This executes the SMBus "block read" protocol, returning negative errno
  * else the number of data bytes in the slave's response.
@@ -222,22 +222,30 @@ EXPORT_SYMBOL(i2c_smbus_write_word_data);
  * support this; its emulation through I2C messaging relies on a specific
  * mechanism (I2C_M_RECV_LEN) which may not be implemented.
  */
-s32 i2c_smbus_read_block_data(const struct i2c_client *client, u8 command,
-			      u8 *values)
+s32 __i2c_smbus_read_block_data(const struct i2c_client *client, u8 command,
+				u8 length, u8 *values)
 {
 	union i2c_smbus_data data;
+	int ret_len;
 	int status;
 
+	if (length > I2C_SMBUS_BLOCK_MAX)
+		return -EINVAL;
+
 	status = i2c_smbus_xfer(client->adapter, client->addr, client->flags,
 				I2C_SMBUS_READ, command,
 				I2C_SMBUS_BLOCK_DATA, &data);
 	if (status)
 		return status;
 
-	memcpy(values, &data.block[1], data.block[0]);
-	return data.block[0];
+	ret_len = data.block[0];
+	if (ret_len > length)
+		return -EMSGSIZE;
+
+	memcpy(values, &data.block[1], ret_len);
+	return ret_len;
 }
-EXPORT_SYMBOL(i2c_smbus_read_block_data);
+EXPORT_SYMBOL(__i2c_smbus_read_block_data);
 
 /**
  * i2c_smbus_write_block_data - SMBus "block write" protocol
diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c
index ccaac5e29f906b..d219eb89f978b6 100644
--- a/drivers/i2c/i2c-dev.c
+++ b/drivers/i2c/i2c-dev.c
@@ -495,6 +495,9 @@ static long i2cdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			return -EINVAL;
 
 		client->adapter->timeout = msecs_to_jiffies(arg * 10);
+#ifdef CONFIG_I2C_DYNAMIC_TIMEOUT
+		client->adapter->user_timeout = client->adapter->timeout;
+#endif
 		break;
 	default:
 		/* NOTE:  returning a fault code here could cause trouble
diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c
index 68a4c34b5987d5..53428098ddc7c3 100644
--- a/drivers/i2c/i2c-mux.c
+++ b/drivers/i2c/i2c-mux.c
@@ -24,7 +24,7 @@
 #include <linux/i2c-mux.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/slab.h>
 #include <linux/sysfs.h>
 
@@ -33,6 +33,7 @@ struct i2c_mux_priv {
 	struct i2c_adapter adap;
 	struct i2c_algorithm algo;
 	struct i2c_mux_core *muxc;
+	struct fwnode_handle *channel_node;
 	u32 chan_id;
 };
 
@@ -264,6 +265,51 @@ static const struct i2c_lock_operations i2c_parent_lock_ops = {
 	.unlock_bus =  i2c_parent_unlock_bus,
 };
 
+static struct fwnode_handle *
+i2c_mux_get_channel_node(struct i2c_mux_core *muxc, u32 chan_id)
+{
+	struct fwnode_handle *dev_node;
+	struct fwnode_handle *mux_node;
+	struct fwnode_handle *child;
+	u32 reg;
+	int ret;
+
+	dev_node = dev_fwnode(muxc->dev);
+	if (!dev_node)
+		return NULL;
+
+	if (muxc->arbitrator)
+		mux_node = fwnode_get_named_child_node(dev_node, "i2c-arb");
+	else if (muxc->gate)
+		mux_node = fwnode_get_named_child_node(dev_node, "i2c-gate");
+	else
+		mux_node = fwnode_get_named_child_node(dev_node, "i2c-mux");
+
+	if (mux_node) {
+		/* A "reg" property indicates an old-style firmware entry. */
+		if (!fwnode_property_read_u32(mux_node, "reg", &reg)) {
+			fwnode_handle_put(mux_node);
+			mux_node = NULL;
+		}
+	}
+
+	if (!mux_node)
+		mux_node = fwnode_handle_get(dev_node);
+	else if (muxc->arbitrator || muxc->gate)
+		return mux_node;
+
+	fwnode_for_each_child_node(mux_node, child) {
+		ret = fwnode_property_read_u32(child, "reg", &reg);
+		if (ret)
+			continue;
+		if (chan_id == reg)
+			break;
+	}
+
+	fwnode_handle_put(mux_node);
+	return child;
+}
+
 int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 			u32 force_nr, u32 chan_id)
 {
@@ -323,48 +369,9 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 	else
 		priv->adap.lock_ops = &i2c_parent_lock_ops;
 
-	/*
-	 * Try to populate the mux adapter's of_node, expands to
-	 * nothing if !CONFIG_OF.
-	 */
-	if (muxc->dev->of_node) {
-		struct device_node *dev_node = muxc->dev->of_node;
-		struct device_node *mux_node, *child = NULL;
-		u32 reg;
-
-		if (muxc->arbitrator)
-			mux_node = of_get_child_by_name(dev_node, "i2c-arb");
-		else if (muxc->gate)
-			mux_node = of_get_child_by_name(dev_node, "i2c-gate");
-		else
-			mux_node = of_get_child_by_name(dev_node, "i2c-mux");
-
-		if (mux_node) {
-			/* A "reg" property indicates an old-style DT entry */
-			if (!of_property_read_u32(mux_node, "reg", &reg)) {
-				of_node_put(mux_node);
-				mux_node = NULL;
-			}
-		}
-
-		if (!mux_node)
-			mux_node = of_node_get(dev_node);
-		else if (muxc->arbitrator || muxc->gate)
-			child = of_node_get(mux_node);
-
-		if (!child) {
-			for_each_child_of_node(mux_node, child) {
-				ret = of_property_read_u32(child, "reg", &reg);
-				if (ret)
-					continue;
-				if (chan_id == reg)
-					break;
-			}
-		}
-
-		priv->adap.dev.of_node = child;
-		of_node_put(mux_node);
-	}
+	/* Associate the mux adapter with its channel firmware node. */
+	priv->channel_node = i2c_mux_get_channel_node(muxc, chan_id);
+	device_set_node(&priv->adap.dev, priv->channel_node);
 
 	/*
 	 * Associate the mux channel with an ACPI node.
@@ -408,7 +415,7 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 	return 0;
 
 err_free_priv:
-	of_node_put(priv->adap.dev.of_node);
+	fwnode_handle_put(priv->channel_node);
 	kfree(priv);
 	return ret;
 }
@@ -421,7 +428,6 @@ void i2c_mux_del_adapters(struct i2c_mux_core *muxc)
 	while (muxc->num_adapters) {
 		struct i2c_adapter *adap = muxc->adapter[--muxc->num_adapters];
 		struct i2c_mux_priv *priv = adap->algo_data;
-		struct device_node *np = adap->dev.of_node;
 
 		muxc->adapter[muxc->num_adapters] = NULL;
 
@@ -431,7 +437,7 @@ void i2c_mux_del_adapters(struct i2c_mux_core *muxc)
 
 		sysfs_remove_link(&priv->adap.dev.kobj, "mux_device");
 		i2c_del_adapter(adap);
-		of_node_put(np);
+		fwnode_handle_put(priv->channel_node);
 		kfree(priv);
 	}
 }
diff --git a/drivers/i3c/device.c b/drivers/i3c/device.c
index f1ba363b22a17c..4df4682af15045 100644
--- a/drivers/i3c/device.c
+++ b/drivers/i3c/device.c
@@ -301,16 +301,23 @@ i3c_device_match_id(struct i3c_device *i3cdev,
 EXPORT_SYMBOL_GPL(i3c_device_match_id);
 
 /**
- * i3c_device_get_supported_xfer_mode - Returns the supported transfer mode by
- *					connected master controller.
+ * i3c_device_get_supported_xfer_mode - Returns the transfer modes supported by
+ *					the connected master controller and @dev,
+ *					and allowed by the bus configuration.
  * @dev: I3C device
  *
  * Return: a bit mask, which supported transfer mode, bit position is defined at
- *	   enum i3c_hdr_mode
+ *	   enum i3c_xfer_mode
  */
 u32 i3c_device_get_supported_xfer_mode(struct i3c_device *dev)
 {
-	return i3c_bus_to_i3c_master(dev->bus)->this->info.hdr_cap | BIT(I3C_SDR);
+	u32 modes;
+
+	i3c_bus_normaluse_lock(dev->bus);
+	modes = i3c_dev_supported_xfer_modes_locked(dev->desc);
+	i3c_bus_normaluse_unlock(dev->bus);
+
+	return modes;
 }
 EXPORT_SYMBOL_GPL(i3c_device_get_supported_xfer_mode);
 
diff --git a/drivers/i3c/internals.h b/drivers/i3c/internals.h
index 86a36b951e0d57..9a6936e2c6608a 100644
--- a/drivers/i3c/internals.h
+++ b/drivers/i3c/internals.h
@@ -18,6 +18,8 @@ bool i3c_bus_rpm_ibi_allowed(struct i3c_bus *bus);
 void i3c_bus_normaluse_lock(struct i3c_bus *bus);
 void i3c_bus_normaluse_unlock(struct i3c_bus *bus);
 
+u32 i3c_dev_supported_xfer_modes_locked(struct i3c_dev_desc *dev);
+
 int i3c_dev_setdasa_locked(struct i3c_dev_desc *dev);
 int i3c_dev_do_xfers_locked(struct i3c_dev_desc *dev,
 			    struct i3c_xfer *xfers,
diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c
index afcd7a21a3e66e..2cdbffd33fd7f4 100644
--- a/drivers/i3c/master.c
+++ b/drivers/i3c/master.c
@@ -8,6 +8,7 @@
 #include <dt-bindings/i3c/i3c.h>
 #include <linux/acpi.h>
 #include <linux/atomic.h>
+#include <linux/bitfield.h>
 #include <linux/bitmap.h>
 #include <linux/bug.h>
 #include <linux/delay.h>
@@ -937,6 +938,61 @@ static int i3c_bus_set_mode(struct i3c_bus *i3cbus, enum i3c_bus_mode mode,
 	return 0;
 }
 
+/*
+ * I3C v1.1.1 Section 5.1.2.4 Table 9 lists the HDR Modes each Bus
+ * Configuration allows.  A Mixed Slow / Limited Bus has Legacy I2C Devices
+ * without a 50 ns spike filter, so there is no way to hide any HDR Mode from
+ * them.  Of the two Ternary Modes, only HDR-TSL is defined for a Bus that
+ * also has Legacy I2C Devices; HDR-TSP is defined for a Pure Bus.
+ */
+static u32 i3c_bus_hdr_modes(struct i3c_bus *bus)
+{
+	switch (bus->mode) {
+	case I3C_BUS_MODE_PURE:
+		return BIT(I3C_HDR_DDR) | BIT(I3C_HDR_TSP) | BIT(I3C_HDR_TSL);
+	case I3C_BUS_MODE_MIXED_FAST:
+		return BIT(I3C_HDR_DDR) | BIT(I3C_HDR_TSL);
+	case I3C_BUS_MODE_MIXED_LIMITED:
+	case I3C_BUS_MODE_MIXED_SLOW:
+		break;
+	}
+
+	return 0;
+}
+
+static u32 i3c_dev_hdr_modes(struct i3c_dev_desc *dev)
+{
+	if (!(dev->info.bcr & I3C_BCR_HDR_CAP))
+		return 0;
+
+	return dev->info.hdr_cap;
+}
+
+/**
+ * i3c_dev_supported_xfer_modes_locked() - Get the transfer modes usable with a
+ *					   device
+ * @dev: I3C device descriptor
+ *
+ * The HDR Modes the controller and @dev both support, restricted to those the
+ * bus configuration allows.  SDR is always supported.
+ *
+ * The bus lock must be held in normal use mode.
+ *
+ * Return: a bit mask of &enum i3c_xfer_mode values.
+ */
+u32 i3c_dev_supported_xfer_modes_locked(struct i3c_dev_desc *dev)
+{
+	struct i3c_master_controller *master = i3c_dev_get_master(dev);
+
+	/*
+	 * master->this->info.bcr is ignored because it describes the master's
+	 * target capability, not its controller capability.
+	 */
+	return (master->this->info.hdr_cap &
+		i3c_bus_hdr_modes(&master->bus) &
+		i3c_dev_hdr_modes(dev)) | BIT(I3C_SDR);
+}
+
 static struct i3c_master_controller *
 i2c_adapter_to_i3c_master(struct i2c_adapter *adap)
 {
@@ -1776,11 +1832,6 @@ static int i3c_master_getstatus_locked(struct i3c_master_controller *master,
 	if (ret)
 		goto out;
 
-	if (dest.payload.len != sizeof(*getstatus)) {
-		ret = -EIO;
-		goto out;
-	}
-
 	if (status)
 		*status = be16_to_cpu(getstatus->status);
 out:
@@ -2216,12 +2267,11 @@ struct i3c_dma *i3c_master_dma_map_single(struct device *dev, void *buf,
 
 	if (force_bounce) {
 		dma_xfer->map_len = ALIGN(len, cache_line_size());
-		if (dir == DMA_FROM_DEVICE)
-			bounce = kzalloc(dma_xfer->map_len, GFP_KERNEL);
-		else
-			bounce = kmemdup(buf, dma_xfer->map_len, GFP_KERNEL);
+		bounce = kzalloc(dma_xfer->map_len, GFP_KERNEL);
 		if (!bounce)
 			return NULL;
+		if (dir != DMA_FROM_DEVICE)
+			memcpy(bounce, buf, len);
 		dma_buf = bounce;
 	}
 
@@ -3077,6 +3127,13 @@ static int i3c_master_add_of_dev(struct i3c_master_controller *master,
 	return ret;
 }
 
+#define I3C_ACPI_ADR_PID GENMASK_U64(47, 0)
+/*
+ * Zero-based instance number of the Bus Controller to which the Target is
+ * connected.
+ */
+#define I3C_ACPI_ADR_INSTANCE GENMASK_U64(51, 48)
+
 #ifdef CONFIG_ACPI
 static int i3c_master_add_acpi_dev(struct i3c_master_controller *master,
 				   struct fwnode_handle *fwnode)
@@ -3084,6 +3141,7 @@ static int i3c_master_add_acpi_dev(struct i3c_master_controller *master,
 	struct acpi_device *adev = to_acpi_device_node(fwnode);
 	acpi_bus_address adr;
 	u32 reg[3] = { 0 };
+	u64 pid;
 	int ret;
 
 	/*
@@ -3101,9 +3159,15 @@ static int i3c_master_add_acpi_dev(struct i3c_master_controller *master,
 
 	adr = acpi_device_adr(adev);
 
+	/* Match the multi-bus instance number */
+	if (FIELD_GET(I3C_ACPI_ADR_INSTANCE, adr) != master->instance)
+		return 0;
+
 	/* For I3C devices, _ADR will have the 48 bit PID of the device  */
-	reg[1] = upper_32_bits(adr);
-	reg[2] = lower_32_bits(adr);
+	pid = FIELD_GET(I3C_ACPI_ADR_PID, adr);
+
+	reg[1] = upper_32_bits(pid);
+	reg[2] = lower_32_bits(pid);
 
 	fwnode_property_read_u32(fwnode, "mipi-i3c-static-address", &reg[0]);
 
@@ -3847,7 +3911,7 @@ int i3c_dev_do_xfers_locked(struct i3c_dev_desc *dev, struct i3c_xfer *xfers,
 	if (!master || !xfers)
 		return -EINVAL;
 
-	if (mode != I3C_SDR && !(master->this->info.hdr_cap & BIT(mode)))
+	if (mode != I3C_SDR && !(i3c_dev_supported_xfer_modes_locked(dev) & BIT(mode)))
 		return -EOPNOTSUPP;
 
 	return master->ops->i3c_xfers(dev, xfers, nxfers, mode);
@@ -3945,7 +4009,8 @@ int i3c_dev_request_ibi_locked(struct i3c_dev_desc *dev,
 	if (!ibi)
 		return -ENOMEM;
 
-	ibi->wq = alloc_ordered_workqueue(dev_name(i3cdev_to_dev(dev->dev)), WQ_MEM_RECLAIM);
+	ibi->wq = alloc_ordered_workqueue(dev_name(i3cdev_to_dev(dev->dev)),
+					  WQ_MEM_RECLAIM | WQ_HIGHPRI);
 	if (!ibi->wq) {
 		kfree(ibi);
 		return -ENOMEM;
diff --git a/drivers/i3c/master/amd-i3c-master.c b/drivers/i3c/master/amd-i3c-master.c
index e4da9c38eccb47..b62157729a5b82 100644
--- a/drivers/i3c/master/amd-i3c-master.c
+++ b/drivers/i3c/master/amd-i3c-master.c
@@ -806,8 +806,8 @@ static int xi3c_master_send_direct_ccc_cmd(struct xi3c_master *master,
 
 	/* Report actual byte count so the core sees the right length on short reads */
 	if (!ret && ccc->rnw)
-		ccc->dests[0].payload.len = min(xfer->cmds[1].rx_actual,
-						ccc->dests[0].payload.len);
+		ccc->dests[0].payload.actual_len = min(xfer->cmds[1].rx_actual,
+						       ccc->dests[0].payload.len);
 
 	return ret;
 }
diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd.h b/drivers/i3c/master/mipi-i3c-hci/cmd.h
index 7bada7b4b2de5b..fa3107df7c5133 100644
--- a/drivers/i3c/master/mipi-i3c-hci/cmd.h
+++ b/drivers/i3c/master/mipi-i3c-hci/cmd.h
@@ -24,7 +24,7 @@
  */
 #define RESP_STATUS(resp)		FIELD_GET(GENMASK(31, 28), resp)
 #define RESP_TID(resp)			FIELD_GET(GENMASK(27, 24), resp)
-#define RESP_DATA_LENGTH(resp)		FIELD_GET(GENMASK(21,  0), resp)
+#define RESP_DATA_LENGTH(resp)		FIELD_GET(GENMASK(15,  0), resp)
 
 #define RESP_ERR_FIELD			GENMASK(31, 28)
 
@@ -55,7 +55,7 @@ struct hci_cmd_ops {
 	int (*prep_ccc)(struct i3c_hci *hci, struct hci_xfer *xfer,
 			u8 ccc_addr, u8 ccc_cmd, bool raw);
 	void (*prep_i3c_xfer)(struct i3c_hci *hci, struct i3c_dev_desc *dev,
-			      struct hci_xfer *xfer);
+			      struct hci_xfer *xfer, enum i3c_xfer_mode mode);
 	void (*prep_i2c_xfer)(struct i3c_hci *hci, struct i2c_dev_desc *dev,
 			      struct hci_xfer *xfer);
 	int (*perform_daa)(struct i3c_hci *hci);
diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c
index 3b9345718d273d..78ad2ec2bd9360 100644
--- a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c
+++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c
@@ -108,8 +108,6 @@ enum hci_cmd_mode {
 	MODE_I3C_SDR4		= 0x4,
 	MODE_I3C_HDR_TSx	= 0x5,
 	MODE_I3C_HDR_DDR	= 0x6,
-	MODE_I3C_HDR_BT		= 0x7,
-	MODE_I3C_Fm_FmP		= 0x8,
 	MODE_I2C_Fm		= 0x0,
 	MODE_I2C_FmP		= 0x1,
 	MODE_I2C_UD1		= 0x2,
@@ -217,14 +215,30 @@ static int hci_cmd_v1_prep_ccc(struct i3c_hci *hci,
 
 static void hci_cmd_v1_prep_i3c_xfer(struct i3c_hci *hci,
 				     struct i3c_dev_desc *dev,
-				     struct hci_xfer *xfer)
+				     struct hci_xfer *xfer,
+				     enum i3c_xfer_mode xfer_mode)
 {
 	struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
 	unsigned int dat_idx = dev_data->dat_idx;
-	enum hci_cmd_mode mode = get_i3c_mode(hci);
+	enum hci_cmd_mode mode;
 	u8 *data = xfer->data;
 	unsigned int data_len = xfer->data_len;
 	bool rnw = xfer->rnw;
+	u32 cp_cmd = 0;
+
+	if (xfer_mode == I3C_SDR) {
+		mode = get_i3c_mode(hci);
+	} else {
+		/*
+		 * HDR-DDR is the only advertised HDR Mode, so a non-SDR
+		 * transfer uses the HDR-DDR Command Code encoding: CP marks CMD
+		 * as valid, CMD carries bits[6:0], and RNW carries bit[7]. CP
+		 * and CMD occupy the same descriptor bits in both command
+		 * formats.
+		 */
+		mode = MODE_I3C_HDR_DDR;
+		cp_cmd = CMD_R0_CP | CMD_R0_CMD(xfer->hdr_cmd & I3C_HDR_CMD_CODE);
+	}
 
 	xfer->cmd_tid = hci_get_tid();
 
@@ -235,7 +249,8 @@ static void hci_cmd_v1_prep_i3c_xfer(struct i3c_hci *hci,
 			CMD_I0_TID(xfer->cmd_tid) |
 			CMD_I0_DEV_INDEX(dat_idx) |
 			CMD_I0_DTT(data_len) |
-			CMD_I0_MODE(mode);
+			CMD_I0_MODE(mode) |
+			cp_cmd;
 		fill_data_bytes(xfer, data, data_len);
 	} else {
 		/* we use a Regular Data Transfer Command */
@@ -244,7 +259,8 @@ static void hci_cmd_v1_prep_i3c_xfer(struct i3c_hci *hci,
 			CMD_R0_TID(xfer->cmd_tid) |
 			CMD_R0_DEV_INDEX(dat_idx) |
 			CMD_R0_MODE(mode) |
-			(rnw ? CMD_R0_RNW : 0);
+			(rnw ? CMD_R0_RNW : 0) |
+			cp_cmd;
 		xfer->cmd_desc[1] =
 			CMD_R1_DATA_LENGTH(data_len);
 	}
diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c
index 8d93748e858dd3..b5e25f05f03e17 100644
--- a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c
+++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c
@@ -212,9 +212,11 @@ static int hci_cmd_v2_prep_ccc(struct i3c_hci *hci, struct hci_xfer *xfer,
 	return 0;
 }
 
+/* For cmd_v2, only SDR capability is advertised */
 static void hci_cmd_v2_prep_i3c_xfer(struct i3c_hci *hci,
 				     struct i3c_dev_desc *dev,
-				     struct hci_xfer *xfer)
+				     struct hci_xfer *xfer,
+				     enum i3c_xfer_mode xfer_mode)
 {
 	unsigned int mode = XFERMODE_IDX_I3C_SDR;
 	unsigned int rate = get_i3c_rate_idx(hci);
diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c
index dadf049bd4b54a..1b75df2fe34cd2 100644
--- a/drivers/i3c/master/mipi-i3c-hci/core.c
+++ b/drivers/i3c/master/mipi-i3c-hci/core.c
@@ -51,12 +51,10 @@
 #define HC_CAP_SG_DC_EN			BIT(30)
 #define HC_CAP_SG_IBI_EN		BIT(29)
 #define HC_CAP_SG_CR_EN			BIT(28)
-#define HC_CAP_MAX_DATA_LENGTH		GENMASK(24, 22)
 #define HC_CAP_CMD_SIZE			GENMASK(21, 20)
 #define HC_CAP_DIRECT_COMMANDS_EN	BIT(18)
 #define HC_CAP_MULTI_LANE_EN		BIT(15)
 #define HC_CAP_CMD_CCC_DEFBYTE		BIT(10)
-#define HC_CAP_HDR_BT_EN		BIT(8)
 #define HC_CAP_HDR_TS_EN		BIT(7)
 #define HC_CAP_HDR_DDR_EN		BIT(6)
 #define HC_CAP_NON_CURRENT_MASTER_CAP	BIT(5)	/* master handoff capable */
@@ -118,6 +116,13 @@ static inline struct i3c_hci *to_i3c_hci(struct i3c_master_controller *m)
 	return container_of(m, struct i3c_hci, master);
 }
 
+/* HDR support has been added for cmd_v1 only */
+static bool i3c_hci_hdr_ddr_capable(struct i3c_hci *hci)
+{
+	return hci->cmd == &mipi_i3c_hci_cmd_v1 &&
+	       hci->caps & HC_CAP_HDR_DDR_EN;
+}
+
 /**
  * i3c_hci_sysdev() - Get the device to use for DMA and system PM
  * @dev: Device the HCI controller is bound to
@@ -160,6 +165,8 @@ static int i3c_hci_bus_init(struct i3c_master_controller *m)
 	i3c_hci_set_master_dyn_addr(hci);
 	memset(&info, 0, sizeof(info));
 	info.dyn_addr = hci->dyn_addr;
+	if (i3c_hci_hdr_ddr_capable(hci))
+		info.hdr_cap = BIT(I3C_HDR_DDR);
 	ret = i3c_master_set_info(m, &info);
 	if (ret)
 		return ret;
@@ -465,6 +472,27 @@ static int i3c_hci_daa(struct i3c_master_controller *m)
 	return ret;
 }
 
+static bool i3c_hci_rnw(struct i3c_xfer *i3c_xfer, enum i3c_xfer_mode mode)
+{
+	if (mode == I3C_SDR)
+		return i3c_xfer->rnw;
+
+	return i3c_xfer->cmd & I3C_HDR_CMD_RNW;
+}
+
+static int i3c_hci_check_hdr_ddr_xfers(struct i3c_xfer *i3c_xfers, int nxfers)
+{
+	/*
+	 * HDR-DDR frames 16-bit Data Words, and at least one Data Word must
+	 * follow the Command Word.
+	 */
+	for (int i = 0; i < nxfers; i++)
+		if (i3c_xfers[i].len < 2 || i3c_xfers[i].len % 2)
+			return -EINVAL;
+
+	return 0;
+}
+
 static int i3c_hci_i3c_xfers(struct i3c_dev_desc *dev,
 			     struct i3c_xfer *i3c_xfers, int nxfers,
 			     enum i3c_xfer_mode mode)
@@ -473,30 +501,31 @@ static int i3c_hci_i3c_xfers(struct i3c_dev_desc *dev,
 	struct i3c_hci *hci = to_i3c_hci(m);
 	struct hci_xfer *xfer;
 	DECLARE_COMPLETION_ONSTACK(done);
-	unsigned int size_limit;
 	int i, last, ret = 0;
 
 	dev_dbg(&hci->master.dev, "nxfers = %d", nxfers);
 
+	if (mode == I3C_HDR_DDR) {
+		ret = i3c_hci_check_hdr_ddr_xfers(i3c_xfers, nxfers);
+		if (ret)
+			return ret;
+	}
+
 	xfer = hci_alloc_xfer(nxfers);
 	if (!xfer)
 		return -ENOMEM;
 
-	size_limit = 1U << (16 + FIELD_GET(HC_CAP_MAX_DATA_LENGTH, hci->caps));
-
 	for (i = 0; i < nxfers; i++) {
 		xfer[i].data_len = i3c_xfers[i].len;
-		ret = -EFBIG;
-		if (xfer[i].data_len >= size_limit)
-			goto out;
-		xfer[i].rnw = i3c_xfers[i].rnw;
-		if (i3c_xfers[i].rnw) {
+		xfer[i].rnw = i3c_hci_rnw(i3c_xfers + i, mode);
+		xfer[i].hdr_cmd = i3c_xfers[i].cmd;
+		if (xfer[i].rnw) {
 			xfer[i].data = i3c_xfers[i].data.in;
 		} else {
 			/* silence the const qualifier warning with a cast */
 			xfer[i].data = (void *) i3c_xfers[i].data.out;
 		}
-		hci->cmd->prep_i3c_xfer(hci, dev, &xfer[i]);
+		hci->cmd->prep_i3c_xfer(hci, dev, &xfer[i], mode);
 		xfer[i].cmd_desc[0] |= CMD_0_ROC;
 	}
 	last = i - 1;
@@ -508,7 +537,7 @@ static int i3c_hci_i3c_xfers(struct i3c_dev_desc *dev,
 	if (ret)
 		goto out;
 	for (i = 0; i < nxfers; i++) {
-		if (i3c_xfers[i].rnw)
+		if (xfer[i].rnw)
 			i3c_xfers[i].len = RESP_DATA_LENGTH(xfer[i].response);
 		if (RESP_STATUS(xfer[i].response) != RESP_SUCCESS) {
 			ret = -EIO;
@@ -563,6 +592,11 @@ static int i3c_hci_i2c_xfers(struct i2c_dev_desc *dev,
 	return ret;
 }
 
+static void i3c_hci_dat_v1_set_curr_nack_retry(struct i3c_hci *hci, unsigned int dat_idx)
+{
+	mipi_i3c_hci_dat_v1.set_nack_retry(hci, dat_idx, hci->master.dev_nack_retry_count);
+}
+
 static int i3c_hci_attach_i3c_dev(struct i3c_dev_desc *dev)
 {
 	struct i3c_master_controller *m = i3c_dev_get_master(dev);
@@ -581,6 +615,7 @@ static int i3c_hci_attach_i3c_dev(struct i3c_dev_desc *dev)
 		}
 		mipi_i3c_hci_dat_v1.set_dynamic_addr(hci, ret,
 						     dev->info.dyn_addr ?: dev->info.static_addr);
+		i3c_hci_dat_v1_set_curr_nack_retry(hci, ret);
 		dev_data->dat_idx = ret;
 	}
 	i3c_dev_set_master_data(dev, dev_data);
@@ -630,6 +665,7 @@ static int i3c_hci_attach_i2c_dev(struct i2c_dev_desc *dev)
 	}
 	mipi_i3c_hci_dat_v1.set_static_addr(hci, ret, dev->addr);
 	mipi_i3c_hci_dat_v1.set_flags(hci, ret, DAT_0_I2C_DEVICE, 0);
+	i3c_hci_dat_v1_set_curr_nack_retry(hci, ret);
 	dev_data->dat_idx = ret;
 	i2c_dev_set_master_data(dev, dev_data);
 	return 0;
@@ -675,11 +711,13 @@ static void __i3c_hci_disable_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev)
 
 static void i3c_hci_free_ibi(struct i3c_dev_desc *dev)
 {
+	struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
 	struct i3c_master_controller *m = i3c_dev_get_master(dev);
 	struct i3c_hci *hci = to_i3c_hci(m);
 
-	/* Must ensure the IBI has been disabled */
-	__i3c_hci_disable_ibi(hci, dev);
+	/* Must ensure IBIs for this device will no longer be processed */
+	scoped_guard(spinlock_irqsave, &hci->lock)
+		hci->ibi_devs[dev_data->dat_idx] = NULL;
 	hci->io->free_ibi(hci, dev);
 }
 
@@ -703,11 +741,15 @@ static int i3c_hci_enable_ibi(struct i3c_dev_desc *dev)
 	struct i3c_master_controller *m = i3c_dev_get_master(dev);
 	struct i3c_hci *hci = to_i3c_hci(m);
 	struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
+	int ret;
 
 	mipi_i3c_hci_dat_v1.clear_flags(hci, dev_data->dat_idx, DAT_0_SIR_REJECT, 0);
 	scoped_guard(spinlock_irqsave, &hci->lock)
 		hci->ibi_devs[dev_data->dat_idx] = dev;
-	return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
+	ret = i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
+	if (ret)
+		__i3c_hci_disable_ibi(hci, dev);
+	return ret;
 }
 
 static int i3c_hci_disable_ibi(struct i3c_dev_desc *dev)
@@ -715,13 +757,14 @@ static int i3c_hci_disable_ibi(struct i3c_dev_desc *dev)
 	struct i3c_master_controller *m = i3c_dev_get_master(dev);
 	struct i3c_hci *hci = to_i3c_hci(m);
 
-	__i3c_hci_disable_ibi(hci, dev);
 	/*
-	 * The DAT entry is now set to NACK and DISEC this target's IBIs, so
+	 * The DAT entry will be set to NACK and DISEC this target's IBIs, so
 	 * the IBI teardown can proceed even if DISEC below fails, so ignore
 	 * errors.
 	 */
 	i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
+
+	__i3c_hci_disable_ibi(hci, dev);
 	return 0;
 }
 
@@ -734,6 +777,23 @@ static void i3c_hci_recycle_ibi_slot(struct i3c_dev_desc *dev,
 	hci->io->recycle_ibi_slot(hci, dev, slot);
 }
 
+static int i3c_hci_set_dev_nack_retry(struct i3c_master_controller *m, unsigned int cnt)
+{
+	struct i3c_hci *hci = to_i3c_hci(m);
+	unsigned int dat_idx;
+
+	if (hci->cmd != &mipi_i3c_hci_cmd_v1)
+		return -EOPNOTSUPP;
+
+	if (cnt > FIELD_MAX(DAT_0_DEV_NACK_RETRY_CNT))
+		return -ERANGE;
+
+	for_each_set_bit(dat_idx, hci->DAT_data, hci->DAT_entries)
+		mipi_i3c_hci_dat_v1.set_nack_retry(hci, dat_idx, cnt);
+
+	return 0;
+}
+
 static const struct i3c_master_controller_ops i3c_hci_ops = {
 	.bus_init		= i3c_hci_bus_init,
 	.bus_cleanup		= i3c_hci_bus_cleanup,
@@ -753,6 +813,7 @@ static const struct i3c_master_controller_ops i3c_hci_ops = {
 	.recycle_ibi_slot	= i3c_hci_recycle_ibi_slot,
 	.enable_hotjoin		= i3c_hci_enable_hotjoin,
 	.disable_hotjoin	= i3c_hci_disable_hotjoin,
+	.set_dev_nack_retry	= i3c_hci_set_dev_nack_retry,
 };
 
 static irqreturn_t i3c_hci_irq_handler(int irq, void *dev_id)
@@ -1142,8 +1203,10 @@ static int i3c_hci_probe(struct platform_device *pdev)
 	 * necessarily in separate contiguous sub-ranges. To avoid overlapping
 	 * mappings, provide base_regs from the parent mapping.
 	 */
-	if (pdata)
+	if (pdata) {
 		hci->base_regs = pdata->base_regs;
+		hci->master.instance = pdata->instance;
+	}
 
 	if (!hci->base_regs) {
 		hci->base_regs = devm_platform_ioremap_resource(pdev, 0);
@@ -1185,6 +1248,13 @@ static int i3c_hci_probe(struct platform_device *pdev)
 	if (device_can_wakeup(i3c_hci_sysdev(&pdev->dev)))
 		hci->master.ibi_wakeup = true;
 
+	/*
+	 * HCI v1.1 onward does 1 retry for Direct CCCs anyway, so for v1.0 to
+	 * be consistent, promote 0 to 1.
+	 */
+	if (!hci->master.dev_nack_retry_count)
+		hci->master.dev_nack_retry_count = 1;
+
 	return i3c_master_register(&hci->master, &pdev->dev, &i3c_hci_ops, false);
 }
 
diff --git a/drivers/i3c/master/mipi-i3c-hci/dat.h b/drivers/i3c/master/mipi-i3c-hci/dat.h
index 6881f19da77f20..31becc6f9aec41 100644
--- a/drivers/i3c/master/mipi-i3c-hci/dat.h
+++ b/drivers/i3c/master/mipi-i3c-hci/dat.h
@@ -12,6 +12,7 @@
 
 /* Global DAT flags */
 #define DAT_0_I2C_DEVICE		W0_BIT_(31)
+#define DAT_0_DEV_NACK_RETRY_CNT	W0_MASK(30, 29)
 #define DAT_0_SIR_REJECT		W0_BIT_(13)
 #define DAT_0_IBI_PAYLOAD		W0_BIT_(12)
 
@@ -25,6 +26,7 @@ struct hci_dat_ops {
 	void (*clear_flags)(struct i3c_hci *hci, unsigned int dat_idx, u32 w0, u32 w1);
 	int (*get_index)(struct i3c_hci *hci, u8 address);
 	void (*restore)(struct i3c_hci *hci);
+	void (*set_nack_retry)(struct i3c_hci *hci, unsigned int dat_idx, unsigned int cnt);
 };
 
 extern const struct hci_dat_ops mipi_i3c_hci_dat_v1;
diff --git a/drivers/i3c/master/mipi-i3c-hci/dat_v1.c b/drivers/i3c/master/mipi-i3c-hci/dat_v1.c
index 852966aa20d980..ec0764a27c621f 100644
--- a/drivers/i3c/master/mipi-i3c-hci/dat_v1.c
+++ b/drivers/i3c/master/mipi-i3c-hci/dat_v1.c
@@ -24,7 +24,7 @@
 #define DAT_1_AUTOCMD_VALUE		W1_MASK(47, 40)
 #define DAT_1_AUTOCMD_MASK		W1_MASK(39, 32)
 /*	DAT_0_I2C_DEVICE		W0_BIT_(31) */
-#define DAT_0_DEV_NACK_RETRY_CNT	W0_MASK(30, 29)
+/*	DAT_0_DEV_NACK_RETRY_CNT	W0_MASK(30, 29) */
 #define DAT_0_RING_ID			W0_MASK(28, 26)
 #define DAT_0_DYNADDR_PARITY		W0_BIT_(23)
 #define DAT_0_DYNAMIC_ADDRESS		W0_MASK(22, 16)
@@ -189,6 +189,16 @@ static void hci_dat_v1_restore(struct i3c_hci *hci)
 	}
 }
 
+static void hci_dat_v1_set_nack_retry(struct i3c_hci *hci, unsigned int dat_idx, unsigned int cnt)
+{
+	u32 dat_w0;
+
+	dat_w0 = dat_w0_read(dat_idx);
+	dat_w0 &= ~DAT_0_DEV_NACK_RETRY_CNT;
+	dat_w0 |= FIELD_PREP(DAT_0_DEV_NACK_RETRY_CNT, cnt);
+	dat_w0_write(dat_idx, dat_w0);
+}
+
 const struct hci_dat_ops mipi_i3c_hci_dat_v1 = {
 	.init			= hci_dat_v1_init,
 	.alloc_entry		= hci_dat_v1_alloc_entry,
@@ -199,4 +209,5 @@ const struct hci_dat_ops mipi_i3c_hci_dat_v1 = {
 	.clear_flags		= hci_dat_v1_clear_flags,
 	.get_index		= hci_dat_v1_get_index,
 	.restore		= hci_dat_v1_restore,
+	.set_nack_retry		= hci_dat_v1_set_nack_retry,
 };
diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c
index 7c2b2047413042..b8fcea0870ab5e 100644
--- a/drivers/i3c/master/mipi-i3c-hci/dma.c
+++ b/drivers/i3c/master/mipi-i3c-hci/dma.c
@@ -15,6 +15,7 @@
 #include <linux/errno.h>
 #include <linux/i3c/master.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 
 #include "hci.h"
 #include "cmd.h"
@@ -273,29 +274,6 @@ static void hci_dma_init_rings(struct i3c_hci *hci)
 		hci_dma_init_rh(hci, &rings->headers[i], i);
 }
 
-static void hci_dma_suspend(struct i3c_hci *hci)
-{
-	struct hci_rings_data *rings = hci->io_data;
-	int n = rings ? rings->total : 0;
-
-	for (int i = 0; i < n; i++) {
-		struct hci_rh_data *rh = &rings->headers[i];
-
-		rh_reg_write(INTR_SIGNAL_ENABLE, 0);
-		rh_reg_write(RING_CONTROL, 0);
-	}
-
-	i3c_hci_sync_irq_inactive(hci);
-}
-
-static void hci_dma_resume(struct i3c_hci *hci)
-{
-	struct hci_rings_data *rings = hci->io_data;
-
-	if (rings)
-		hci_dma_init_rings(hci);
-}
-
 static int hci_dma_init(struct i3c_hci *hci)
 {
 	struct hci_rings_data *rings;
@@ -428,7 +406,7 @@ static void hci_dma_unmap_xfer(struct i3c_hci *hci,
 static struct i3c_dma *hci_dma_map_xfer(struct device *dev, struct hci_xfer *xfer)
 {
 	enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
-	bool need_bounce = device_iommu_mapped(dev) && xfer->rnw && (xfer->data_len & 3);
+	bool need_bounce = xfer->rnw && (xfer->data_len & 3);
 
 	return i3c_master_dma_map_single(dev, xfer->data, xfer->data_len, need_bounce, dir);
 }
@@ -868,25 +846,24 @@ static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci,
 	i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot);
 }
 
-static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)
+static bool hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh,
+				u32 *op1_val, unsigned int enq_ptr)
 {
 	struct hci_rings_data *rings = hci->io_data;
 	struct i3c_dev_desc *dev;
 	struct i3c_hci_dev_data *dev_data;
 	struct hci_dma_dev_ibi_data *dev_ibi;
 	struct i3c_ibi_slot *slot;
-	u32 op1_val, op2_val, ibi_status_error;
-	unsigned int ptr, enq_ptr, deq_ptr;
+	u32 ibi_status_error;
+	unsigned int ptr, deq_ptr;
 	unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part;
 	int ibi_addr, last_ptr;
 	void *ring_ibi_data;
 	dma_addr_t ring_ibi_data_dma;
 
-	op1_val = rh_reg_read(RING_OPERATION1);
-	deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val);
-
-	op2_val = rh_reg_read(RING_OPERATION2);
-	enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val);
+	deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, *op1_val);
+	if (deq_ptr == enq_ptr)
+		return false;
 
 	ibi_status_error = 0;
 	ibi_addr = -1;
@@ -936,7 +913,7 @@ static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)
 		dev_dbg(&hci->master.dev,
 			"no LAST_STATUS available (e=%d d=%d)",
 			enq_ptr, deq_ptr);
-		return;
+		return false;
 	}
 	deq_ptr = last_ptr + 1;
 	deq_ptr %= rh->ibi_status_entries;
@@ -1015,10 +992,9 @@ static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)
 	i3c_master_queue_ibi(dev, slot);
 
 done:
-	op1_val = rh_reg_read(RING_OPERATION1);
-	op1_val &= ~RING_OP1_IBI_DEQ_PTR;
-	op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr);
-	rh_reg_write(RING_OPERATION1, op1_val);
+	*op1_val &= ~RING_OP1_IBI_DEQ_PTR;
+	*op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr);
+	rh_reg_write(RING_OPERATION1, *op1_val);
 
 	/* update the chunk pointer */
 	rh->ibi_chunk_ptr += ibi_chunks;
@@ -1026,6 +1002,19 @@ static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)
 
 	/* and tell the hardware about freed chunks */
 	rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks);
+
+	return true;
+}
+
+static void hci_dma_drain_ibi_ring(struct i3c_hci *hci, struct hci_rh_data *rh)
+{
+	u32 op1_val = rh_reg_read(RING_OPERATION1);
+	u32 op2_val = rh_reg_read(RING_OPERATION2);
+	unsigned int enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val);
+
+	/* Loop is bounded by enq_ptr. Further IBIs will re-assert INTR_IBI_READY */
+	while (hci_dma_process_ibi(hci, rh, &op1_val, enq_ptr))
+		;
 }
 
 static bool hci_dma_irq_handler(struct i3c_hci *hci)
@@ -1047,7 +1036,7 @@ static bool hci_dma_irq_handler(struct i3c_hci *hci)
 		rh_reg_write(INTR_STATUS, status);
 
 		if (status & INTR_IBI_READY)
-			hci_dma_process_ibi(hci, rh);
+			hci_dma_drain_ibi_ring(hci, rh);
 		if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR))
 			hci_dma_xfer_done(hci, rh);
 		if (status & INTR_RING_OP)
@@ -1064,6 +1053,77 @@ static bool hci_dma_irq_handler(struct i3c_hci *hci)
 	return handled;
 }
 
+/*
+ * With the bus disabled, a ring should stop within a few microseconds. The
+ * timeout is therefore only expected to expire if the hardware is stuck.
+ * Allow sufficient margin for slow systems, but keep the delay acceptable
+ * during suspend.
+ */
+#define RING_STOP_TIMEOUT_US	(100 * USEC_PER_MSEC)
+/*
+ * The ring is usually already stopped, so polling typically completes on the
+ * first iteration. Use a modest sleep interval to avoid busy-waiting without
+ * adding excessive latency.
+ */
+#define RING_STOP_SLEEP_US	100
+
+static void hci_dma_suspend(struct i3c_hci *hci)
+{
+	struct hci_rings_data *rings = hci->io_data;
+	int n = rings ? rings->total : 0;
+	struct hci_rh_data *rh;
+	u32 regval;
+
+	/* Gracefully stop the rings */
+	scoped_guard(spinlock_irqsave, &hci->lock) {
+		for (int i = 0; i < n; i++) {
+			rh = &rings->headers[i];
+			regval = rh_reg_read(RING_CONTROL);
+			if (regval & RING_CTRL_RUN_STOP)
+				rh_reg_write(RING_CONTROL, regval & ~RING_CTRL_RUN_STOP);
+		}
+	}
+
+	/* Wait for actual stop */
+	for (int i = 0; i < n; i++) {
+		rh = &rings->headers[i];
+		if (readx_poll_timeout(readl, rh->regs + RH_RING_STATUS, regval,
+				       !(regval & RING_STATUS_RUNNING),
+				       RING_STOP_SLEEP_US, RING_STOP_TIMEOUT_US))
+			dev_err(&hci->master.dev, "%s: Ring did not stop, status %#x\n",
+				__func__, regval);
+	}
+
+	/*
+	 * With the rings stopped, no more IBIs can be received. Flush and make
+	 * the interrupt handler inactive.
+	 */
+	i3c_hci_sync_irq_inactive(hci);
+
+	/* Disable interrupt signals and disable the rings */
+	scoped_guard(spinlock_irqsave, &hci->lock)
+		for (int i = 0; i < n; i++) {
+			rh = &rings->headers[i];
+			rh_reg_write(INTR_SIGNAL_ENABLE, 0);
+			/*
+			 * Be absolutely certain there is no unprocessed IBI.
+			 * hci_dma_drain_ibi_ring() will do nothing if there is
+			 * none.
+			 */
+			if (i < IBI_RINGS)
+				hci_dma_drain_ibi_ring(hci, rh);
+			rh_reg_write(RING_CONTROL, 0);
+		}
+}
+
+static void hci_dma_resume(struct i3c_hci *hci)
+{
+	struct hci_rings_data *rings = hci->io_data;
+
+	if (rings)
+		hci_dma_init_rings(hci);
+}
+
 const struct hci_io_ops mipi_i3c_hci_dma = {
 	.init			= hci_dma_init,
 	.cleanup		= hci_dma_cleanup,
diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h
index ee73f6e6756a12..4288a1b5b49136 100644
--- a/drivers/i3c/master/mipi-i3c-hci/hci.h
+++ b/drivers/i3c/master/mipi-i3c-hci/hci.h
@@ -85,10 +85,10 @@ struct i3c_hci {
 
 /*
  * Structure to represent a master initiated transfer.
- * The rnw, data and data_len fields must be initialized before calling any
- * hci->cmd->*() method. The cmd method will initialize cmd_desc[] and
- * possibly modify (clear) the data field. Then xfer->cmd_desc[0] can
- * be augmented with CMD_0_ROC and/or CMD_0_TOC.
+ * The rnw, data and data_len (and hdr_cmd for HDR) fields must be initialized
+ * before calling any hci->cmd->*() method. The cmd method will initialize
+ * cmd_desc[] and possibly modify (clear) the data field. Then xfer->cmd_desc[0]
+ * can be augmented with CMD_0_ROC and/or CMD_0_TOC.
  * The completion field needs to be initialized before queueing with
  * hci->io->queue_xfer(), and requires CMD_0_ROC to be set.
  */
@@ -97,6 +97,7 @@ struct hci_xfer {
 	u32 response;
 	bool rnw;
 	bool started;
+	u8 hdr_cmd;
 	void *data;
 	unsigned int data_len;
 	unsigned int cmd_tid;
diff --git a/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c b/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c
index ab595661db1e8f..5b8e10d05ceeaa 100644
--- a/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c
+++ b/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c
@@ -390,6 +390,7 @@ static void mipi_i3c_hci_pci_setup_cell(struct mipi_i3c_hci_pci *hci, int idx,
 					struct mfd_cell *cell)
 {
 	data->pdata.base_regs = hci->base + hci->info->instance_offset[idx];
+	data->pdata.instance = idx;
 
 	data->res = DEFINE_RES_IRQ(0);
 
@@ -445,6 +446,8 @@ static int mipi_i3c_hci_pci_probe(struct pci_dev *pci,
 	if (ret < 0)
 		return ret;
 
+	pci_set_drvdata(pci, hci);
+
 	hci->info = (const struct mipi_i3c_hci_pci_info *)id->driver_data;
 
 	ret = hci->info->init ? hci->info->init(hci) : 0;
@@ -455,8 +458,6 @@ static int mipi_i3c_hci_pci_probe(struct pci_dev *pci,
 	if (ret)
 		goto err_exit;
 
-	pci_set_drvdata(pci, hci);
-
 	mipi_i3c_hci_pci_rpm_allow(&pci->dev);
 
 	return 0;
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index 651408df9c24ac..a045a384b949d2 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -1792,9 +1792,7 @@ static bool __init intel_idle_cst_usable(void)
 {
 	int cstate, limit;
 
-	limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
-		      acpi_state_table.count);
-
+	limit = min3(CPUIDLE_STATE_MAX, max_cstate + 1, acpi_state_table.count);
 	for (cstate = 1; cstate < limit; cstate++) {
 		struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
 
@@ -1906,7 +1904,7 @@ static void __init intel_idle_init_cstates_acpi_lpi(struct cpuidle_driver *drv)
 		state = &drv->states[drv->state_count++];
 
 		scnprintf(state->name, CPUIDLE_NAME_LEN, "C%d_LPI", index + 1);
-		strscpy(state->desc, lpi_state->desc, CPUIDLE_DESC_LEN);
+		strscpy(state->desc, lpi_state->desc);
 		state->exit_latency = lpi_state->wake_latency;
 		state->target_residency = lpi_state->min_residency;
 		state->flags = MWAIT2flg(lpi_state->address);
@@ -1938,7 +1936,7 @@ static void __init intel_idle_init_cstates_acpi_lpi(struct cpuidle_driver *drv)
 
 static void __init intel_idle_init_cstates_acpi_cst(struct cpuidle_driver *drv)
 {
-	int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
+	int cstate, limit = min(CPUIDLE_STATE_MAX, acpi_state_table.count);
 
 	/*
 	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
@@ -1956,7 +1954,7 @@ static void __init intel_idle_init_cstates_acpi_cst(struct cpuidle_driver *drv)
 		state = &drv->states[drv->state_count++];
 
 		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
-		strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
+		strscpy(state->desc, cx->desc);
 		state->exit_latency = cx->latency;
 		/*
 		 * For C1-type C-states use the same number for both the exit
@@ -2018,7 +2016,7 @@ static bool __init intel_idle_off_by_default_lpi(unsigned int flags, u32 mwait_h
 
 static bool __init intel_idle_off_by_default_cst(unsigned int flags, u32 mwait_hint)
 {
-	int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
+	int cstate, limit = min(CPUIDLE_STATE_MAX, acpi_state_table.count);
 
 	/*
 	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig
index 4094299e2ed813..141ac984247cdd 100644
--- a/drivers/iio/accel/Kconfig
+++ b/drivers/iio/accel/Kconfig
@@ -7,13 +7,14 @@
 menu "Accelerometers"
 
 config ADIS16201
-	tristate "Analog Devices ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer"
+	tristate "ADIS16201 Digital Inclinometer and similar"
 	depends on SPI
 	select IIO_ADIS_LIB
 	select IIO_ADIS_LIB_BUFFER if IIO_BUFFER
 	help
-	  Say Y here to build support for Analog Devices adis16201 dual-axis
-	  digital inclinometer and accelerometer.
+	  Say Y here to build support for Analog Devices Digital Inclinometers:
+	  - ADIS16201
+	  - ADIS16203
 
 	  To compile this driver as a module, say M here: the module will
 	  be called adis16201.
diff --git a/drivers/iio/accel/adis16201.c b/drivers/iio/accel/adis16201.c
index ba0f97944c6d5f..6d5a7b69611211 100644
--- a/drivers/iio/accel/adis16201.c
+++ b/drivers/iio/accel/adis16201.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer
+ * ADIS16201 Digital Inclinometer and similar
  *
  * Copyright 2010 Analog Devices Inc.
  */
@@ -63,6 +63,7 @@
 #define ADIS16201_DIAG_STAT_REG				0x3C
 #define  ADIS16201_DIAG_STAT_ALARM2			BIT(9)
 #define  ADIS16201_DIAG_STAT_ALARM1			BIT(8)
+#define  ADIS16203_DIAG_STAT_SELFTEST_FAIL_BIT		5
 #define  ADIS16201_DIAG_STAT_SPI_FAIL_BIT		3
 #define  ADIS16201_DIAG_STAT_FLASH_UPT_FAIL_BIT		2
 /* Power supply above 3.625 V */
@@ -87,6 +88,21 @@ enum adis16201_scan {
 	ADIS16201_SCAN_TEMP,
 };
 
+struct adis16201_chip_info {
+	const char *name;
+	const struct iio_chan_spec *arr_channels;
+	const struct adis_data *data;
+	unsigned int incli_scale_val2;
+	unsigned int read_bits_incli;
+	unsigned int num_channels;
+	u16 write_mask_incli;
+};
+
+struct adis16201_state {
+	struct adis adis;
+	const struct adis16201_chip_info *info;
+};
+
 static const u8 adis16201_addresses[] = {
 	[ADIS16201_SCAN_ACC_X] = ADIS16201_XACCL_OFFS_REG,
 	[ADIS16201_SCAN_ACC_Y] = ADIS16201_YACCL_OFFS_REG,
@@ -99,7 +115,7 @@ static int adis16201_read_raw(struct iio_dev *indio_dev,
 			      int *val, int *val2,
 			      long mask)
 {
-	struct adis *st = iio_priv(indio_dev);
+	struct adis16201_state *st = iio_priv(indio_dev);
 	int ret;
 	int bits;
 	u8 addr;
@@ -137,7 +153,7 @@ static int adis16201_read_raw(struct iio_dev *indio_dev,
 			return IIO_VAL_INT_PLUS_NANO;
 		case IIO_INCLI:
 			*val = 0;
-			*val2 = 100000;
+			*val2 = st->info->incli_scale_val2;
 			return IIO_VAL_INT_PLUS_MICRO;
 		default:
 			return -EINVAL;
@@ -157,13 +173,13 @@ static int adis16201_read_raw(struct iio_dev *indio_dev,
 			bits = 12;
 			break;
 		case IIO_INCLI:
-			bits = 9;
+			bits = st->info->read_bits_incli;
 			break;
 		default:
 			return -EINVAL;
 		}
 		addr = adis16201_addresses[chan->scan_index];
-		ret = adis_read_reg_16(st, addr, &val16);
+		ret = adis_read_reg_16(&st->adis, addr, &val16);
 		if (ret)
 			return ret;
 
@@ -180,41 +196,49 @@ static int adis16201_write_raw(struct iio_dev *indio_dev,
 			       int val2,
 			       long mask)
 {
-	struct adis *st = iio_priv(indio_dev);
-	int m;
+	struct adis16201_state *st = iio_priv(indio_dev);
 
 	if (mask != IIO_CHAN_INFO_CALIBBIAS)
 		return -EINVAL;
 
 	switch (chan->type) {
 	case IIO_ACCEL:
-		m = GENMASK(11, 0);
+		val &= GENMASK(11, 0);
 		break;
 	case IIO_INCLI:
-		m = GENMASK(8, 0);
+		val &= st->info->write_mask_incli;
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	return adis_write_reg_16(st, adis16201_addresses[chan->scan_index],
-				 val & m);
+	return adis_write_reg_16(&st->adis, adis16201_addresses[chan->scan_index],
+				 val);
 }
 
 static const struct iio_chan_spec adis16201_channels[] = {
-	ADIS_SUPPLY_CHAN(ADIS16201_SUPPLY_OUT_REG, ADIS16201_SCAN_SUPPLY, 0,
-			 12),
-	ADIS_TEMP_CHAN(ADIS16201_TEMP_OUT_REG, ADIS16201_SCAN_TEMP, 0, 12),
 	ADIS_ACCEL_CHAN(X, ADIS16201_XACCL_OUT_REG, ADIS16201_SCAN_ACC_X,
 			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
 	ADIS_ACCEL_CHAN(Y, ADIS16201_YACCL_OUT_REG, ADIS16201_SCAN_ACC_Y,
 			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	ADIS_AUX_ADC_CHAN(ADIS16201_AUX_ADC_REG, ADIS16201_SCAN_AUX_ADC, 0, 12),
 	ADIS_INCLI_CHAN(X, ADIS16201_XINCL_OUT_REG, ADIS16201_SCAN_INCLI_X,
 			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 12),
 	ADIS_INCLI_CHAN(Y, ADIS16201_YINCL_OUT_REG, ADIS16201_SCAN_INCLI_Y,
 			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 12),
-	IIO_CHAN_SOFT_TIMESTAMP(7)
+	ADIS_SUPPLY_CHAN(ADIS16201_SUPPLY_OUT_REG, ADIS16201_SCAN_SUPPLY, 0,
+			 12),
+	ADIS_AUX_ADC_CHAN(ADIS16201_AUX_ADC_REG, ADIS16201_SCAN_AUX_ADC, 0, 12),
+	ADIS_TEMP_CHAN(ADIS16201_TEMP_OUT_REG, ADIS16201_SCAN_TEMP, 0, 12),
+	IIO_CHAN_SOFT_TIMESTAMP(7),
+};
+
+static const struct iio_chan_spec adis16203_channels[] = {
+	ADIS_INCLI_CHAN(X, ADIS16201_XINCL_OUT_REG, ADIS16201_SCAN_INCLI_X,
+			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
+	ADIS_SUPPLY_CHAN(ADIS16201_SUPPLY_OUT_REG, ADIS16201_SCAN_SUPPLY, 0, 12),
+	ADIS_AUX_ADC_CHAN(ADIS16201_AUX_ADC_REG, ADIS16201_SCAN_AUX_ADC, 0, 12),
+	ADIS_TEMP_CHAN(ADIS16201_TEMP_OUT_REG, ADIS16201_SCAN_TEMP, 0, 12),
+	IIO_CHAN_SOFT_TIMESTAMP(7),
 };
 
 static const struct iio_info adis16201_info = {
@@ -224,6 +248,7 @@ static const struct iio_info adis16201_info = {
 };
 
 static const char * const adis16201_status_error_msgs[] = {
+	[ADIS16203_DIAG_STAT_SELFTEST_FAIL_BIT] = "Self test failure",
 	[ADIS16201_DIAG_STAT_SPI_FAIL_BIT] = "SPI failure",
 	[ADIS16201_DIAG_STAT_FLASH_UPT_FAIL_BIT] = "Flash update failed",
 	[ADIS16201_DIAG_STAT_POWER_HIGH_BIT] = "Power supply above 3.625V",
@@ -254,10 +279,49 @@ static const struct adis_data adis16201_data = {
 		BIT(ADIS16201_DIAG_STAT_POWER_LOW_BIT),
 };
 
+static const struct adis_data adis16203_data = {
+	.read_delay = 20,
+	.msc_ctrl_reg = ADIS16201_MSC_CTRL_REG,
+	.glob_cmd_reg = ADIS16201_GLOB_CMD_REG,
+	.diag_stat_reg = ADIS16201_DIAG_STAT_REG,
+
+	.self_test_mask = ADIS16201_MSC_CTRL_SELF_TEST_EN,
+	.self_test_reg = ADIS16201_MSC_CTRL_REG,
+	.self_test_no_autoclear = true,
+	.timeouts = &adis16201_timeouts,
+
+	.status_error_msgs = adis16201_status_error_msgs,
+	.status_error_mask = BIT(ADIS16203_DIAG_STAT_SELFTEST_FAIL_BIT) |
+		BIT(ADIS16201_DIAG_STAT_SPI_FAIL_BIT) |
+		BIT(ADIS16201_DIAG_STAT_FLASH_UPT_FAIL_BIT) |
+		BIT(ADIS16201_DIAG_STAT_POWER_HIGH_BIT) |
+		BIT(ADIS16201_DIAG_STAT_POWER_LOW_BIT),
+};
+
+static const struct adis16201_chip_info adis16201_chip_data = {
+	.arr_channels = adis16201_channels,
+	.incli_scale_val2 = 100000,
+	.write_mask_incli = GENMASK(8, 0),
+	.read_bits_incli = 9,
+	.num_channels = ARRAY_SIZE(adis16201_channels),
+	.name = "adis16201",
+	.data = &adis16201_data,
+};
+
+static const struct adis16201_chip_info adis16203_chip_data = {
+	.arr_channels = adis16203_channels,
+	.incli_scale_val2 = 25000,
+	.write_mask_incli = GENMASK(13, 0),
+	.read_bits_incli = 14,
+	.num_channels = ARRAY_SIZE(adis16203_channels),
+	.name = "adis16203",
+	.data = &adis16203_data,
+};
+
 static int adis16201_probe(struct spi_device *spi)
 {
 	struct iio_dev *indio_dev;
-	struct adis *st;
+	struct adis16201_state *st;
 	int ret;
 
 	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
@@ -266,38 +330,57 @@ static int adis16201_probe(struct spi_device *spi)
 
 	st = iio_priv(indio_dev);
 
-	indio_dev->name = spi->dev.driver->name;
+	st->info = spi_get_device_match_data(spi);
+	if (!st->info)
+		return -ENODATA;
+
+	indio_dev->name = st->info->name;
 	indio_dev->info = &adis16201_info;
 
-	indio_dev->channels = adis16201_channels;
-	indio_dev->num_channels = ARRAY_SIZE(adis16201_channels);
+	indio_dev->channels = st->info->arr_channels;
+	indio_dev->num_channels = st->info->num_channels;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
-	ret = adis_init(st, indio_dev, spi, &adis16201_data);
+	ret = adis_init(&st->adis, indio_dev, spi, st->info->data);
 	if (ret)
 		return ret;
 
-	ret = devm_adis_setup_buffer_and_trigger(st, indio_dev, NULL);
+	ret = devm_adis_setup_buffer_and_trigger(&st->adis, indio_dev, NULL);
 	if (ret)
 		return ret;
 
-	ret = __adis_initial_startup(st);
+	ret = __adis_initial_startup(&st->adis);
 	if (ret)
 		return ret;
 
 	return devm_iio_device_register(&spi->dev, indio_dev);
 }
 
+static const struct of_device_id adis16201_of_match[] = {
+	{ .compatible = "adi,adis16201", .data = &adis16201_chip_data },
+	{ .compatible = "adi,adis16203", .data = &adis16203_chip_data },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, adis16201_of_match);
+
+static const struct spi_device_id adis16201_ids[] = {
+	{ .name = "adis16201", .driver_data = (kernel_ulong_t)&adis16201_chip_data },
+	{ .name = "adis16203", .driver_data = (kernel_ulong_t)&adis16203_chip_data },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, adis16201_ids);
+
 static struct spi_driver adis16201_driver = {
 	.driver = {
 		.name = "adis16201",
+		.of_match_table = adis16201_of_match,
 	},
 	.probe = adis16201_probe,
+	.id_table = adis16201_ids,
 };
 module_spi_driver(adis16201_driver);
 
 MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADIS16201 Dual-Axis Digital Inclinometer and Accelerometer");
+MODULE_DESCRIPTION("Analog Devices ADIS16201 Inclinometer and similar");
 MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("spi:adis16201");
 MODULE_IMPORT_NS("IIO_ADISLIB");
diff --git a/drivers/iio/accel/adxl313_spi.c b/drivers/iio/accel/adxl313_spi.c
index a61295de104f66..0f0da44461f741 100644
--- a/drivers/iio/accel/adxl313_spi.c
+++ b/drivers/iio/accel/adxl313_spi.c
@@ -79,6 +79,8 @@ static int adxl313_spi_probe(struct spi_device *spi)
 		return ret;
 
 	chip_data = spi_get_device_match_data(spi);
+	if (!chip_data)
+		return -ENODATA;
 
 	regmap = devm_regmap_init_spi(spi,
 				      &adxl31x_spi_regmap_config[chip_data->type]);
diff --git a/drivers/iio/accel/adxl367.c b/drivers/iio/accel/adxl367.c
index 4ff1c7a0988b89..026db73057c549 100644
--- a/drivers/iio/accel/adxl367.c
+++ b/drivers/iio/accel/adxl367.c
@@ -13,6 +13,7 @@
 #include <linux/iio/sysfs.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 #include <linux/unaligned.h>
@@ -81,6 +82,7 @@
 #define ADXL367_SAMPLES_L_MASK		GENMASK(7, 0)
 
 #define ADXL367_REG_INT1_MAP		0x2A
+#define ADXL367_REG_INT2_MAP		0x2B
 #define ADXL367_INT_INACT_MASK		BIT(5)
 #define ADXL367_INT_ACT_MASK		BIT(4)
 #define ADXL367_INT_FIFO_WATERMARK_MASK	BIT(2)
@@ -168,6 +170,8 @@ struct adxl367_state {
 	enum adxl367_odr	odr;
 	enum adxl367_range	range;
 
+	u8		int_map_reg;
+
 	unsigned int	act_threshold;
 	unsigned int	act_time_ms;
 	unsigned int	inact_threshold;
@@ -366,8 +370,7 @@ static int adxl367_set_act_interrupt_en(struct adxl367_state *st,
 {
 	unsigned int mask = adxl367_act_int_mask_tbl[act];
 
-	return regmap_update_bits(st->regmap, ADXL367_REG_INT1_MAP,
-				  mask, en ? mask : 0);
+	return regmap_assign_bits(st->regmap, st->int_map_reg, mask, en);
 }
 
 static int adxl367_get_act_interrupt_en(struct adxl367_state *st,
@@ -378,7 +381,7 @@ static int adxl367_get_act_interrupt_en(struct adxl367_state *st,
 	unsigned int val;
 	int ret;
 
-	ret = regmap_read(st->regmap, ADXL367_REG_INT1_MAP, &val);
+	ret = regmap_read(st->regmap, st->int_map_reg, &val);
 	if (ret)
 		return ret;
 
@@ -401,9 +404,8 @@ static int adxl367_set_act_en(struct adxl367_state *st,
 static int adxl367_set_fifo_watermark_interrupt_en(struct adxl367_state *st,
 						   bool en)
 {
-	return regmap_update_bits(st->regmap, ADXL367_REG_INT1_MAP,
-				  ADXL367_INT_FIFO_WATERMARK_MASK,
-				  en ? ADXL367_INT_FIFO_WATERMARK_MASK : 0);
+	return regmap_assign_bits(st->regmap, st->int_map_reg,
+				  ADXL367_INT_FIFO_WATERMARK_MASK, en);
 }
 
 static int adxl367_get_fifo_mode(struct adxl367_state *st,
@@ -631,8 +633,7 @@ static int adxl367_set_odr(struct iio_dev *indio_dev, enum adxl367_odr odr)
 static int adxl367_set_temp_adc_en(struct adxl367_state *st, unsigned int reg,
 				   bool en)
 {
-	return regmap_update_bits(st->regmap, reg, ADXL367_ADC_EN_MASK,
-				  en ? ADXL367_ADC_EN_MASK : 0);
+	return regmap_assign_bits(st->regmap, reg, ADXL367_ADC_EN_MASK, en);
 }
 
 static int adxl367_set_temp_adc_reg_en(struct adxl367_state *st,
@@ -787,6 +788,13 @@ static bool adxl367_push_fifo_data(struct iio_dev *indio_dev, u8 status,
 	if (!FIELD_GET(ADXL367_STATUS_FIFO_FULL_MASK, status))
 		return false;
 
+	if (fifo_entries > ADXL367_FIFO_SIZE) {
+		dev_err_ratelimited(st->dev,
+				    "FIFO entry count %u exceeds FIFO size %u\n",
+				    fifo_entries, ADXL367_FIFO_SIZE);
+		return true;
+	}
+
 	fifo_entries -= fifo_entries % st->fifo_set_size;
 
 	ret = st->ops->read_fifo(st->context, st->fifo_buf, fifo_entries);
@@ -1426,6 +1434,36 @@ static int adxl367_setup(struct adxl367_state *st)
 	return adxl367_set_measure_en(st, true);
 }
 
+static int adxl367_set_int_map_reg(struct adxl367_state *st, int irq)
+{
+	int ret;
+
+	/*
+	 * Route the mappings to whichever of INT1/INT2 is named in the
+	 * interrupt-names property. -EINVAL (property absent) and -ENODATA
+	 * (name not listed) are not errors; anything else is.
+	 */
+	ret = fwnode_irq_get_byname(dev_fwnode(st->dev), "INT1");
+	if (ret > 0) {
+		st->int_map_reg = ADXL367_REG_INT1_MAP;
+		return ret;
+	}
+	if (ret != -ENODATA && ret != -EINVAL)
+		return ret;
+
+	ret = fwnode_irq_get_byname(dev_fwnode(st->dev), "INT2");
+	if (ret > 0) {
+		st->int_map_reg = ADXL367_REG_INT2_MAP;
+		return ret;
+	}
+	if (ret != -ENODATA && ret != -EINVAL)
+		return ret;
+
+	/* No INT1/INT2 named: default to INT1 and the bus interrupt. */
+	st->int_map_reg = ADXL367_REG_INT1_MAP;
+	return irq;
+}
+
 int adxl367_probe(struct device *dev, const struct adxl367_ops *ops,
 		  void *context, struct regmap *regmap, int irq)
 {
@@ -1455,6 +1493,11 @@ int adxl367_probe(struct device *dev, const struct adxl367_ops *ops,
 	indio_dev->info = &adxl367_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
+	ret = adxl367_set_int_map_reg(st, irq);
+	if (ret < 0)
+		return dev_err_probe(st->dev, ret, "Failed to get interrupt\n");
+	irq = ret;
+
 	ret = devm_regulator_bulk_get_enable(st->dev,
 					     ARRAY_SIZE(regulator_names),
 					     regulator_names);
diff --git a/drivers/iio/accel/adxl372_spi.c b/drivers/iio/accel/adxl372_spi.c
index 128e148c4e96b0..d54544d6df6f42 100644
--- a/drivers/iio/accel/adxl372_spi.c
+++ b/drivers/iio/accel/adxl372_spi.c
@@ -25,6 +25,8 @@ static int adxl372_spi_probe(struct spi_device *spi)
 	struct regmap *regmap;
 
 	chip_info = spi_get_device_match_data(spi);
+	if (!chip_info)
+		return -ENODATA;
 
 	regmap = devm_regmap_init_spi(spi, &adxl372_spi_regmap_config);
 	if (IS_ERR(regmap))
diff --git a/drivers/iio/accel/adxl380.c b/drivers/iio/accel/adxl380.c
index 7dca5523091fc4..8518ee23e11490 100644
--- a/drivers/iio/accel/adxl380.c
+++ b/drivers/iio/accel/adxl380.c
@@ -966,6 +966,13 @@ static irqreturn_t adxl380_irq_handler(int irq, void  *p)
 	if (ret)
 		return IRQ_HANDLED;
 
+	if (fifo_entries > ADXL380_FIFO_SAMPLES) {
+		dev_err_ratelimited(st->dev,
+				    "FIFO entry count %u exceeds FIFO size %lu\n",
+				    fifo_entries, ADXL380_FIFO_SAMPLES);
+		return IRQ_HANDLED;
+	}
+
 	fifo_entries = rounddown(fifo_entries, st->fifo_set_size);
 	ret = regmap_noinc_read(st->regmap, ADXL380_FIFO_DATA, &st->fifo_buf,
 				sizeof(*st->fifo_buf) * fifo_entries);
diff --git a/drivers/iio/accel/adxl380_spi.c b/drivers/iio/accel/adxl380_spi.c
index ae8467f4e6c01e..8ffb3a8690a63e 100644
--- a/drivers/iio/accel/adxl380_spi.c
+++ b/drivers/iio/accel/adxl380_spi.c
@@ -25,6 +25,8 @@ static int adxl380_spi_probe(struct spi_device *spi)
 	struct regmap *regmap;
 
 	chip_data = spi_get_device_match_data(spi);
+	if (!chip_data)
+		return -ENODATA;
 
 	regmap = devm_regmap_init_spi(spi, &adxl380_spi_regmap_config);
 	if (IS_ERR(regmap))
diff --git a/drivers/iio/accel/bma220_spi.c b/drivers/iio/accel/bma220_spi.c
index d6c1087cc3c0f8..d7a95289df4810 100644
--- a/drivers/iio/accel/bma220_spi.c
+++ b/drivers/iio/accel/bma220_spi.c
@@ -28,12 +28,13 @@ static const struct spi_device_id bma220_spi_id[] = {
 	{ .name = "bma220" },
 	{ }
 };
+MODULE_DEVICE_TABLE(spi, bma220_spi_id);
 
 static const struct acpi_device_id bma220_acpi_id[] = {
 	{ "BMA0220", 0 },
 	{ }
 };
-MODULE_DEVICE_TABLE(spi, bma220_spi_id);
+MODULE_DEVICE_TABLE(acpi, bma220_acpi_id);
 
 static const struct of_device_id bma220_of_spi_match[] = {
 	{ .compatible = "bosch,bma220" },
diff --git a/drivers/iio/accel/bma400_core.c b/drivers/iio/accel/bma400_core.c
index 7e3cff4cca91c6..6bc64ca87f4bd2 100644
--- a/drivers/iio/accel/bma400_core.c
+++ b/drivers/iio/accel/bma400_core.c
@@ -6,8 +6,6 @@
  *
  * TODO:
  *  - Support for power management
- *  - Support events and interrupts
- *  - Create channel for step count
  *  - Create channel for sensor time
  */
 
diff --git a/drivers/iio/accel/fxls8962af-core.c b/drivers/iio/accel/fxls8962af-core.c
index 18d7b09bddd2b4..55183c69465d12 100644
--- a/drivers/iio/accel/fxls8962af-core.c
+++ b/drivers/iio/accel/fxls8962af-core.c
@@ -16,6 +16,7 @@
 #include <linux/bitfield.h>
 #include <linux/i2c.h>
 #include <linux/irq.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
 #include <linux/property.h>
diff --git a/drivers/iio/accel/kionix-kx022a.c b/drivers/iio/accel/kionix-kx022a.c
index 02dd1db7a6463c..add89512af79c3 100644
--- a/drivers/iio/accel/kionix-kx022a.c
+++ b/drivers/iio/accel/kionix-kx022a.c
@@ -301,11 +301,10 @@ struct kx022a_data {
 	__le16 *fifo_buffer;
 
 	/* 3 x 16bit accel data + timestamp */
-	__le16 buffer[8] __aligned(IIO_DMA_MINALIGN);
 	struct {
 		__le16 channels[3];
 		aligned_s64 ts;
-	} scan;
+	} scan __aligned(IIO_DMA_MINALIGN);
 };
 
 static const struct iio_mount_matrix *
@@ -611,14 +610,15 @@ static int kx022a_get_axis(struct kx022a_data *data,
 			   struct iio_chan_spec const *chan,
 			   int *val)
 {
+	__le16 *buf = &data->scan.channels[0];
 	int ret;
 
-	ret = regmap_bulk_read(data->regmap, chan->address, &data->buffer[0],
-			       sizeof(__le16));
+	ret = regmap_bulk_read(data->regmap, chan->address,
+			       buf, sizeof(*buf));
 	if (ret)
 		return ret;
 
-	*val = (s16)le16_to_cpu(data->buffer[0]);
+	*val = (s16)le16_to_cpup(buf);
 
 	return IIO_VAL_INT;
 }
@@ -649,7 +649,7 @@ static int kx022a_read_raw(struct iio_dev *idev,
 		if (ret)
 			return ret;
 
-		if ((regval & KX022A_MASK_ODR) >
+		if ((regval & KX022A_MASK_ODR) >=
 		    ARRAY_SIZE(kx022a_accel_samp_freq_table)) {
 			dev_err(data->dev, "Invalid ODR\n");
 			return -EINVAL;
@@ -809,7 +809,7 @@ static int __kx022a_fifo_flush(struct iio_dev *idev, unsigned int samples,
 		 * the timestamps. If we are ran from IRQ, then the
 		 * IRQF_ONESHOT has us covered - but if we are ran by the
 		 * user-space read we need to disable the IRQ to be on a safe
-		 * side. We do this usng synchronous disable so that if the
+		 * side. We do this using synchronous disable so that if the
 		 * IRQ thread is being ran on other CPU we wait for it to be
 		 * finished.
 		 */
@@ -864,7 +864,8 @@ static int __kx022a_fifo_flush(struct iio_dev *idev, unsigned int samples,
 		for_each_set_bit(bit, idev->active_scan_mask, AXIS_MAX)
 			chs[bit] = sam[bit];
 
-		iio_push_to_buffers_with_timestamp(idev, &data->scan, tstamp);
+		iio_push_to_buffers_with_ts(idev, &data->scan,
+					    sizeof(data->scan), tstamp);
 
 		tstamp += sample_period;
 	}
@@ -980,26 +981,44 @@ static int kx022a_fifo_enable(struct kx022a_data *data)
 	guard(mutex)(&data->mutex);
 	ret = __kx022a_turn_on_off(data, false);
 	if (ret)
-		return ret;
+		goto err_free_out;
 
 	/* Update watermark to HW */
 	ret = kx022a_fifo_set_wmi(data);
 	if (ret)
-		return ret;
+		goto err_turn_on_out;
 
 	/* Enable buffer */
 	ret = regmap_set_bits(data->regmap, data->chip_info->buf_cntl2,
 			      KX022A_MASK_BUF_EN);
 	if (ret)
-		return ret;
+		goto err_turn_on_out;
 
 	data->state |= KX022A_STATE_FIFO;
 	ret = regmap_set_bits(data->regmap, data->ien_reg,
 			      KX022A_MASK_WMI);
 	if (ret)
-		return ret;
+		goto err_buf_disable_out;
 
-	return __kx022a_turn_on_off(data, true);
+	ret = __kx022a_turn_on_off(data, true);
+	if (ret)
+		goto err_wmi_clear_out;
+
+	return ret;
+
+err_wmi_clear_out:
+	regmap_clear_bits(data->regmap, data->ien_reg,
+			  KX022A_MASK_WMI);
+err_buf_disable_out:
+	regmap_clear_bits(data->regmap, data->chip_info->buf_cntl2,
+			  KX022A_MASK_BUF_EN);
+	data->state &= ~KX022A_STATE_FIFO;
+err_turn_on_out:
+	__kx022a_turn_on_off(data, true);
+err_free_out:
+	kfree(data->fifo_buffer);
+
+	return ret;
 }
 
 static int kx022a_buffer_postenable(struct iio_dev *idev)
@@ -1029,12 +1048,13 @@ static irqreturn_t kx022a_trigger_handler(int irq, void *p)
 	struct kx022a_data *data = iio_priv(idev);
 	int ret;
 
-	ret = regmap_bulk_read(data->regmap, data->chip_info->xout_l, data->buffer,
-			       KX022A_FIFO_SAMPLES_SIZE_BYTES);
+	ret = regmap_bulk_read(data->regmap, data->chip_info->xout_l,
+			       data->scan.channels, KX022A_FIFO_SAMPLES_SIZE_BYTES);
 	if (ret < 0)
 		goto err_read;
 
-	iio_push_to_buffers_with_timestamp(idev, data->buffer, data->timestamp);
+	iio_push_to_buffers_with_ts(idev, &data->scan, sizeof(data->scan),
+				    data->timestamp);
 err_read:
 	iio_trigger_notify_done(idev->trig);
 
diff --git a/drivers/iio/accel/kionix-kx022a.h b/drivers/iio/accel/kionix-kx022a.h
index 0ed54f58422359..a2d122c1e23477 100644
--- a/drivers/iio/accel/kionix-kx022a.h
+++ b/drivers/iio/accel/kionix-kx022a.h
@@ -65,7 +65,7 @@
 #define KX022A_MASK_IEN		BIT(5)
 #define KX022A_MASK_IPOL	BIT(4)
 #define KX022A_IPOL_LOW		0
-#define KX022A_IPOL_HIGH	KX022A_MASK_IPOL1
+#define KX022A_IPOL_HIGH	KX022A_MASK_IPOL
 #define KX022A_MASK_ITYP	BIT(3)
 #define KX022A_ITYP_PULSE	KX022A_MASK_ITYP
 #define KX022A_ITYP_LEVEL	0
diff --git a/drivers/iio/accel/kxcjk-1013.c b/drivers/iio/accel/kxcjk-1013.c
index 166fb786425f8d..b3d753bd94761d 100644
--- a/drivers/iio/accel/kxcjk-1013.c
+++ b/drivers/iio/accel/kxcjk-1013.c
@@ -1030,7 +1030,7 @@ static int kxcjk1013_write_event_config(struct iio_dev *indio_dev,
 	struct kxcjk1013_data *data = iio_priv(indio_dev);
 	int ret;
 
-	if (state && data->ev_enable_state)
+	if (state == data->ev_enable_state)
 		return 0;
 
 	mutex_lock(&data->mutex);
@@ -1547,6 +1547,7 @@ static void kxcjk1013_remove(struct i2c_client *client)
 
 	iio_device_unregister(indio_dev);
 
+	pm_runtime_dont_use_autosuspend(&client->dev);
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
 
diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c
index 7d683686dd9d61..fe62a903f0e228 100644
--- a/drivers/iio/accel/mma8452.c
+++ b/drivers/iio/accel/mma8452.c
@@ -18,6 +18,8 @@
  * TODO: orientation events
  */
 
+#include <linux/array_size.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
@@ -110,8 +112,7 @@ struct mma8452_data {
 	u8 data_cfg;
 	const struct mma_chip_info *chip_info;
 	int sleep_val;
-	struct regulator *vdd_reg;
-	struct regulator *vddio_reg;
+	struct regulator_bulk_data regs[2];
 
 	/* Ensure correct alignment of time stamp when present */
 	struct {
@@ -222,15 +223,15 @@ static int mma8452_drdy(struct mma8452_data *data)
 static int mma8452_set_runtime_pm_state(struct i2c_client *client, bool on)
 {
 #ifdef CONFIG_PM
+	struct device *dev = &client->dev;
 	int ret;
 
 	if (on)
-		ret = pm_runtime_resume_and_get(&client->dev);
+		ret = pm_runtime_resume_and_get(dev);
 	else
-		ret = pm_runtime_put_autosuspend(&client->dev);
+		ret = pm_runtime_put_autosuspend(dev);
 	if (ret < 0) {
-		dev_err(&client->dev,
-			"failed to change power state to %d\n", on);
+		dev_err(dev, "failed to change power state to %d\n", on);
 
 		return ret;
 	}
@@ -495,14 +496,14 @@ static int mma8452_read_raw(struct iio_dev *indio_dev,
 	int i, ret;
 
 	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		if (!iio_device_claim_direct(indio_dev))
+	case IIO_CHAN_INFO_RAW: {
+		IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+		if (IIO_DEV_ACQUIRE_FAILED(claim))
 			return -EBUSY;
 
-		mutex_lock(&data->lock);
+		guard(mutex)(&data->lock);
+
 		ret = mma8452_read(data, buffer);
-		mutex_unlock(&data->lock);
-		iio_device_release_direct(indio_dev);
 		if (ret < 0)
 			return ret;
 
@@ -511,6 +512,7 @@ static int mma8452_read_raw(struct iio_dev *indio_dev,
 			chan->scan_type.realbits - 1);
 
 		return IIO_VAL_INT;
+	}
 	case IIO_CHAN_INFO_SCALE:
 		i = data->data_cfg & MMA8452_DATA_CFG_FS_MASK;
 		*val = data->chip_info->mma_scales[i][0];
@@ -599,36 +601,30 @@ static int mma8452_change_config(struct mma8452_data *data, u8 reg, u8 val)
 	int ret;
 	int is_active;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
 
 	is_active = mma8452_is_active(data);
-	if (is_active < 0) {
-		ret = is_active;
-		goto fail;
-	}
+	if (is_active < 0)
+		return is_active;
 
 	/* config can only be changed when in standby */
 	if (is_active > 0) {
 		ret = mma8452_standby(data);
 		if (ret < 0)
-			goto fail;
+			return ret;
 	}
 
 	ret = i2c_smbus_write_byte_data(data->client, reg, val);
 	if (ret < 0)
-		goto fail;
+		return ret;
 
 	if (is_active > 0) {
 		ret = mma8452_active(data);
 		if (ret < 0)
-			goto fail;
+			return ret;
 	}
 
-	ret = 0;
-fail:
-	mutex_unlock(&data->lock);
-
-	return ret;
+	return 0;
 }
 
 static int mma8452_set_power_mode(struct mma8452_data *data, u8 mode)
@@ -777,14 +773,11 @@ static int mma8452_write_raw(struct iio_dev *indio_dev,
 			     struct iio_chan_spec const *chan,
 			     int val, int val2, long mask)
 {
-	int ret;
-
-	if (!iio_device_claim_direct(indio_dev))
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
 		return -EBUSY;
 
-	ret = __mma8452_write_raw(indio_dev, chan, val, val2, mask);
-	iio_device_release_direct(indio_dev);
-	return ret;
+	return __mma8452_write_raw(indio_dev, chan, val, val2, mask);
 }
 
 static int mma8452_get_event_regs(struct mma8452_data *data,
@@ -1552,7 +1545,7 @@ static int mma8452_probe(struct i2c_client *client)
 	struct iio_dev *indio_dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -1562,32 +1555,21 @@ static int mma8452_probe(struct i2c_client *client)
 
 	data->chip_info = i2c_get_match_data(client);
 	if (!data->chip_info)
-		return dev_err_probe(&client->dev, -ENODEV,
-				     "unknown device model\n");
+		return dev_err_probe(dev, -ENODEV, "unknown device model\n");
 
-	ret = iio_read_mount_matrix(&client->dev, &data->orientation);
+	ret = iio_read_mount_matrix(dev, &data->orientation);
 	if (ret)
 		return ret;
 
-	data->vdd_reg = devm_regulator_get(&client->dev, "vdd");
-	if (IS_ERR(data->vdd_reg))
-		return dev_err_probe(&client->dev, PTR_ERR(data->vdd_reg),
-				     "failed to get VDD regulator!\n");
-
-	data->vddio_reg = devm_regulator_get(&client->dev, "vddio");
-	if (IS_ERR(data->vddio_reg))
-		return dev_err_probe(&client->dev, PTR_ERR(data->vddio_reg),
-				     "failed to get VDDIO regulator!\n");
-
-	ret = regulator_enable(data->vdd_reg);
+	data->regs[0].supply = "vdd";
+	data->regs[1].supply = "vddio";
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(data->regs), data->regs);
 	if (ret)
-		return dev_err_probe(dev, ret, "failed to enable VDD regulator!\n");
+		return dev_err_probe(dev, ret, "failed to get regulators\n");
 
-	ret = regulator_enable(data->vddio_reg);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to enable VDDIO regulator!\n");
-		goto disable_regulator_vdd;
-	}
+	ret = regulator_bulk_enable(ARRAY_SIZE(data->regs), data->regs);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable regulators\n");
 
 	ret = i2c_smbus_read_byte_data(client, MMA8452_WHO_AM_I);
 	if (ret < 0)
@@ -1608,7 +1590,7 @@ static int mma8452_probe(struct i2c_client *client)
 		goto disable_regulators;
 	}
 
-	dev_info(&client->dev, "registering %s accelerometer; ID 0x%x\n",
+	dev_info(dev, "registering %s accelerometer; ID 0x%x\n",
 		 data->chip_info->name, data->chip_info->chip_id);
 
 	i2c_set_clientdata(client, indio_dev);
@@ -1641,10 +1623,10 @@ static int mma8452_probe(struct i2c_client *client)
 	if (client->irq) {
 		int irq2;
 
-		irq2 = fwnode_irq_get_byname(dev_fwnode(&client->dev), "INT2");
+		irq2 = fwnode_irq_get_byname(dev_fwnode(dev), "INT2");
 
 		if (irq2 == client->irq) {
-			dev_dbg(&client->dev, "using interrupt line INT2\n");
+			dev_dbg(dev, "using interrupt line INT2\n");
 		} else {
 			ret = i2c_smbus_write_byte_data(client,
 							MMA8452_CTRL_REG5,
@@ -1652,7 +1634,7 @@ static int mma8452_probe(struct i2c_client *client)
 			if (ret < 0)
 				goto disable_regulators;
 
-			dev_dbg(&client->dev, "using interrupt line INT1\n");
+			dev_dbg(dev, "using interrupt line INT1\n");
 		}
 
 		ret = i2c_smbus_write_byte_data(client,
@@ -1689,14 +1671,13 @@ static int mma8452_probe(struct i2c_client *client)
 			goto buffer_cleanup;
 	}
 
-	ret = pm_runtime_set_active(&client->dev);
+	ret = pm_runtime_set_active(dev);
 	if (ret < 0)
 		goto free_irq;
 
-	pm_runtime_enable(&client->dev);
-	pm_runtime_set_autosuspend_delay(&client->dev,
-					 MMA8452_AUTO_SUSPEND_DELAY_MS);
-	pm_runtime_use_autosuspend(&client->dev);
+	pm_runtime_enable(dev);
+	pm_runtime_set_autosuspend_delay(dev, MMA8452_AUTO_SUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(dev);
 
 	ret = iio_device_register(indio_dev);
 	if (ret < 0)
@@ -1722,10 +1703,7 @@ static int mma8452_probe(struct i2c_client *client)
 	mma8452_trigger_cleanup(indio_dev);
 
 disable_regulators:
-	regulator_disable(data->vddio_reg);
-
-disable_regulator_vdd:
-	regulator_disable(data->vdd_reg);
+	regulator_bulk_disable(ARRAY_SIZE(data->regs), data->regs);
 
 	return ret;
 }
@@ -1734,11 +1712,12 @@ static void mma8452_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mma8452_data *data = iio_priv(indio_dev);
+	struct device *dev = &client->dev;
 
 	iio_device_unregister(indio_dev);
 
-	pm_runtime_disable(&client->dev);
-	pm_runtime_set_suspended(&client->dev);
+	pm_runtime_disable(dev);
+	pm_runtime_set_suspended(dev);
 
 	if (client->irq)
 		free_irq(client->irq, indio_dev);
@@ -1747,8 +1726,7 @@ static void mma8452_remove(struct i2c_client *client)
 	mma8452_trigger_cleanup(indio_dev);
 	mma8452_standby(iio_priv(indio_dev));
 
-	regulator_disable(data->vddio_reg);
-	regulator_disable(data->vdd_reg);
+	regulator_bulk_disable(ARRAY_SIZE(data->regs), data->regs);
 }
 
 #ifdef CONFIG_PM
@@ -1758,23 +1736,16 @@ static int mma8452_runtime_suspend(struct device *dev)
 	struct mma8452_data *data = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&data->lock);
-	ret = mma8452_standby(data);
-	mutex_unlock(&data->lock);
+	scoped_guard(mutex, &data->lock)
+		ret = mma8452_standby(data);
 	if (ret < 0) {
-		dev_err(&data->client->dev, "powering off device failed\n");
+		dev_err(dev, "powering off device failed\n");
 		return -EAGAIN;
 	}
 
-	ret = regulator_disable(data->vddio_reg);
-	if (ret) {
-		dev_err(dev, "failed to disable VDDIO regulator\n");
-		return ret;
-	}
-
-	ret = regulator_disable(data->vdd_reg);
+	ret = regulator_bulk_disable(ARRAY_SIZE(data->regs), data->regs);
 	if (ret) {
-		dev_err(dev, "failed to disable VDD regulator\n");
+		dev_err(dev, "failed to disable regulators\n");
 		return ret;
 	}
 
@@ -1787,16 +1758,9 @@ static int mma8452_runtime_resume(struct device *dev)
 	struct mma8452_data *data = iio_priv(indio_dev);
 	int ret, sleep_val;
 
-	ret = regulator_enable(data->vdd_reg);
-	if (ret) {
-		dev_err(dev, "failed to enable VDD regulator\n");
-		return ret;
-	}
-
-	ret = regulator_enable(data->vddio_reg);
+	ret = regulator_bulk_enable(ARRAY_SIZE(data->regs), data->regs);
 	if (ret) {
-		dev_err(dev, "failed to enable VDDIO regulator\n");
-		regulator_disable(data->vdd_reg);
+		dev_err(dev, "failed to enable regulators\n");
 		return ret;
 	}
 
@@ -1814,8 +1778,7 @@ static int mma8452_runtime_resume(struct device *dev)
 	return 0;
 
 runtime_resume_failed:
-	regulator_disable(data->vddio_reg);
-	regulator_disable(data->vdd_reg);
+	regulator_bulk_disable(ARRAY_SIZE(data->regs), data->regs);
 
 	return ret;
 }
diff --git a/drivers/iio/accel/mma9551.c b/drivers/iio/accel/mma9551.c
index 7e01427fd9c4d7..182f488edab1c8 100644
--- a/drivers/iio/accel/mma9551.c
+++ b/drivers/iio/accel/mma9551.c
@@ -500,6 +500,7 @@ static void mma9551_remove(struct i2c_client *client)
 
 	iio_device_unregister(indio_dev);
 
+	pm_runtime_dont_use_autosuspend(&client->dev);
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
 
diff --git a/drivers/iio/accel/mma9553.c b/drivers/iio/accel/mma9553.c
index 8e7aaac89d4739..5a36d415a8372d 100644
--- a/drivers/iio/accel/mma9553.c
+++ b/drivers/iio/accel/mma9553.c
@@ -1136,6 +1136,7 @@ static void mma9553_remove(struct i2c_client *client)
 
 	iio_device_unregister(indio_dev);
 
+	pm_runtime_dont_use_autosuspend(&client->dev);
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
 
diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c
index a92a563f6a4bf0..acd0890c78a581 100644
--- a/drivers/iio/accel/sca3000.c
+++ b/drivers/iio/accel/sca3000.c
@@ -638,7 +638,7 @@ static int sca3000_write_raw_samp_freq(struct sca3000_state *st, int val)
 
 	if (val == base_freq / 2)
 		ctrlval |= SCA3000_REG_OUT_CTRL_BUF_DIV_2;
-	if (val == base_freq / 4)
+	else if (val == base_freq / 4)
 		ctrlval |= SCA3000_REG_OUT_CTRL_BUF_DIV_4;
 	else if (val != base_freq)
 		return -EINVAL;
@@ -1462,6 +1462,8 @@ static int sca3000_probe(struct spi_device *spi)
 	st->us = spi;
 	mutex_init(&st->lock);
 	st->info = spi_get_device_match_data(spi);
+	if (!st->info)
+		return -ENODATA;
 
 	indio_dev->name = st->info->name;
 	indio_dev->info = &sca3000_info;
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 415e519ad4eb48..a802d8ee4dd9cd 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -105,7 +105,7 @@ config AD4080
 	  called ad4080.
 
 config AD4130
-	tristate "Analog Device AD4130 ADC Driver"
+	tristate "Analog Devices AD4130 ADC Driver"
 	depends on SPI
 	depends on GPIOLIB
 	select IIO_BUFFER
@@ -121,7 +121,7 @@ config AD4130
 	  called ad4130.
 
 config AD4134
-	tristate "Analog Device AD4134 ADC Driver"
+	tristate "Analog Devices AD4134 ADC Driver"
 	depends on SPI
 	select REGMAP_SPI
 	select CRC8
@@ -133,7 +133,7 @@ config AD4134
 	  called ad4134_spi.
 
 config AD4170_4
-	tristate "Analog Device AD4170-4 ADC Driver"
+	tristate "Analog Devices AD4170-4 ADC Driver"
 	depends on SPI
 	select REGMAP_SPI
 	select IIO_BUFFER
@@ -164,7 +164,7 @@ config AD4691
 	  called ad4691.
 
 config AD4695
-	tristate "Analog Device AD4695 ADC Driver"
+	tristate "Analog Devices AD4695 ADC Driver"
 	depends on SPI
 	select IIO_BUFFER
 	select IIO_BUFFER_DMAENGINE
@@ -179,7 +179,7 @@ config AD4695
 	  called ad4695.
 
 config AD4851
-	tristate "Analog Device AD4851 DAS Driver"
+	tristate "Analog Devices AD4851 DAS Driver"
 	depends on SPI
 	depends on PWM
 	select REGMAP_SPI
@@ -347,7 +347,7 @@ config AD7380
 	  called ad7380.
 
 config AD7405
-	tristate "Analog Device AD7405 ADC Driver"
+	tristate "Analog Devices AD7405 ADC Driver"
 	depends on IIO_BACKEND
 	help
 	  Say yes here to build support for Analog Devices AD7405, ADUM7701,
@@ -431,6 +431,21 @@ config AD7766
 	  To compile this driver as a module, choose M here: the module will be
 	  called ad7766.
 
+config AD7768
+	tristate "Analog Devices AD7768/AD7768-4 ADC driver"
+	depends on SPI
+	select REGULATOR
+	select AUXILIARY_BUS
+	select IIO_BUFFER
+	select IIO_BACKEND
+	select REGMAP
+	help
+	  Say yes here to build support for Analog Devices AD7768 and AD7768-4
+	  SPI analog to digital converters.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ad7768.
+
 config AD7768_1
 	tristate "Analog Devices AD7768-1 ADC driver"
 	depends on SPI
@@ -576,20 +591,22 @@ config AD9467
 	  called ad9467.
 
 config ADE9000
-	tristate "Analog Devices ADE9000 Multiphase Energy, and Power Quality Monitoring IC Driver"
+	tristate "Analog Devices Multiphase Energy and Power Quality Monitoring IC Driver"
 	depends on SPI
 	select REGMAP_SPI
 	select IIO_BUFFER
 	select IIO_KFIFO_BUF
 	help
-	  Say yes here to build support for the Analog Devices ADE9000,
-	  a highly accurate, multiphase energy and power quality monitoring
-	  integrated circuit.
+	  Say yes here to build support for the following Analog Devices
+	  highly accurate, multiphase energy and power quality monitoring
+	  integrated circuits:
+	    - ADE9000
+	    - ADE9078
 
-	  The device features high-precision analog-to-digital converters
+	  The devices feature high-precision analog-to-digital converters
 	  and digital signal processing to compute RMS values, power factor,
-	  frequency, and harmonic analysis. It supports SPI communication
-	  and provides buffered data output through the IIO framework.
+	  frequency, and harmonic analysis. They support SPI communication
+	  and provide buffered data output through the IIO framework.
 
 	  To compile this driver as a module, choose M here: the module will
 	  be called ade9000.
@@ -653,6 +670,17 @@ config AT91_SAMA5D2_ADC
 	  To compile this driver as a module, choose M here: the module will be
 	  called at91-sama5d2_adc.
 
+config AXIADO_SARADC
+	tristate "Axiado SARADC driver"
+	depends on ARCH_AXIADO || COMPILE_TEST
+	select REGMAP_MMIO
+	help
+	  Say yes here to build support for the SARADC found in Axiado
+	  SoCs.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called axiado_saradc.
+
 config AXP20X_ADC
 	tristate "X-Powers AXP20X and AXP22X ADC driver"
 	depends on MFD_AXP20X
@@ -1116,6 +1144,17 @@ config MAX34408
 	  To compile this driver as a module, choose M here: the module will be
 	  called max34408.
 
+config MAX40080
+	tristate "Analog Devices MAX40080 Current Sense Amplifier"
+	depends on I2C
+	help
+	  Say yes here to build support for the Analog Devices MAX40080
+	  bidirectional current-sense amplifier with a 12-bit ADC and an I2C
+	  interface.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called max40080.
+
 config MAX77541_ADC
 	tristate "Analog Devices MAX77541 ADC driver"
 	depends on MFD_MAX77541
@@ -1551,6 +1590,8 @@ config RZT2H_ADC
 	tristate "Renesas RZ/T2H / RZ/N2H ADC driver"
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select IIO_ADC_HELPER
+	select IIO_BUFFER
+	select IIO_KFIFO_BUF
 	help
 	  Say yes here to build support for the ADC found in Renesas
 	  RZ/T2H / RZ/N2H SoCs.
@@ -1801,11 +1842,14 @@ config TI_ADS1018
          called ti-ads1018.
 
 config TI_ADS1100
-	tristate "Texas Instruments ADS1100 and ADS1000 ADC"
+	tristate "Texas Instruments ADS1100 and similar single channel I2C ADC"
 	depends on I2C
 	help
-	  If you say yes here you get support for Texas Instruments ADS1100 and
-	  ADS1000 ADC chips.
+	  If you say yes here you get support for TI single channel I2C Analog
+	  Devices.
+	  * ADS1000 12-Bit, 128 SPS Analog-to-Digital Converter
+	  * ADS1100 16-Bit, 128 SPS Analog-to-Digital Converter
+	  * ADS1110 16-Bit, 240 SPS Analog-to-Digital Converter
 
 	  This driver can also be built as a module. If so, the module will be
 	  called ti-ads1100.
@@ -1822,6 +1866,16 @@ config TI_ADS1119
          This driver can also be built as a module. If so, the module will be
          called ti-ads1119.
 
+config TI_ADS112C04
+	tristate "Texas Instruments ADS112C04 ADC"
+	depends on I2C
+	help
+	  If you say yes here you get support for Texas Instruments
+	  ADS112C04 (16-bit) I2C analog to digital converters.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called ti-ads112c04.
+
 config TI_ADS112C14
 	tristate "Texas Instruments ADS112C14/ADS122C14"
 	depends on I2C
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index dcec0abb03b73b..9e82c21dbd3740 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -39,6 +39,7 @@ obj-$(CONFIG_AD7606_IFACE_SPI) += ad7606_spi.o
 obj-$(CONFIG_AD7606) += ad7606.o
 obj-$(CONFIG_AD7625) += ad7625.o
 obj-$(CONFIG_AD7766) += ad7766.o
+obj-$(CONFIG_AD7768) += ad7768.o
 obj-$(CONFIG_AD7768_1) += ad7768-1.o
 obj-$(CONFIG_AD7779) += ad7779.o
 obj-$(CONFIG_AD7780) += ad7780.o
@@ -55,6 +56,7 @@ obj-$(CONFIG_ADI_AXI_ADC) += adi-axi-adc.o
 obj-$(CONFIG_ASPEED_ADC) += aspeed_adc.o
 obj-$(CONFIG_AT91_ADC) += at91_adc.o
 obj-$(CONFIG_AT91_SAMA5D2_ADC) += at91-sama5d2_adc.o
+obj-$(CONFIG_AXIADO_SARADC) += axiado_saradc.o
 obj-$(CONFIG_AXP20X_ADC) += axp20x_adc.o
 obj-$(CONFIG_AXP288_ADC) += axp288_adc.o
 obj-$(CONFIG_BCM_IPROC_ADC) += bcm_iproc_adc.o
@@ -95,6 +97,7 @@ obj-$(CONFIG_MAX1241) += max1241.o
 obj-$(CONFIG_MAX1363) += max1363.o
 obj-$(CONFIG_MAX14001) += max14001.o
 obj-$(CONFIG_MAX34408) += max34408.o
+obj-$(CONFIG_MAX40080) += max40080.o
 obj-$(CONFIG_MAX77541_ADC) += max77541-adc.o
 obj-$(CONFIG_MAX9611) += max9611.o
 obj-$(CONFIG_MCP320X) += mcp320x.o
@@ -155,6 +158,7 @@ obj-$(CONFIG_TI_ADS1015) += ti-ads1015.o
 obj-$(CONFIG_TI_ADS1018) += ti-ads1018.o
 obj-$(CONFIG_TI_ADS1100) += ti-ads1100.o
 obj-$(CONFIG_TI_ADS1119) += ti-ads1119.o
+obj-$(CONFIG_TI_ADS112C04) += ti-ads112c04.o
 obj-$(CONFIG_TI_ADS112C14) += ti-ads112c14.o
 obj-$(CONFIG_TI_ADS124S08) += ti-ads124s08.o
 obj-$(CONFIG_TI_ADS1298) += ti-ads1298.o
diff --git a/drivers/iio/adc/ad4030.c b/drivers/iio/adc/ad4030.c
index e97400a1aa4812..877b06007396f0 100644
--- a/drivers/iio/adc/ad4030.c
+++ b/drivers/iio/adc/ad4030.c
@@ -746,14 +746,21 @@ static int ad4030_set_chan_calibbias(struct iio_dev *indio_dev,
 static int ad4030_set_avg_frame_len(struct iio_dev *dev, int avg_val)
 {
 	struct ad4030_state *st = iio_priv(dev);
-	unsigned int avg_log2 = ilog2(avg_val);
 	unsigned int last_avg_idx = ARRAY_SIZE(ad4030_average_modes) - 1;
+	unsigned int avg_log2;
 	int freq_hz;
 	int ret;
 
-	if (avg_val < 0 || avg_val > ad4030_average_modes[last_avg_idx])
+	/* Reject unsupported modes */
+	if (avg_val > ad4030_average_modes[last_avg_idx])
+		return -EINVAL;
+
+	/* Avoid invalid values for logarithm since it's undefined */
+	if (avg_val < 1)
 		return -EINVAL;
 
+	avg_log2 = ilog2(avg_val);
+
 	if (st->offload_trigger) {
 		/*
 		 * The sample averaging and sampling frequency configurations
diff --git a/drivers/iio/adc/ad4080.c b/drivers/iio/adc/ad4080.c
index 04cd6628ebffb3..4ed82aa42400bc 100644
--- a/drivers/iio/adc/ad4080.c
+++ b/drivers/iio/adc/ad4080.c
@@ -137,6 +137,7 @@
 #define AD4880_CHIP_ID						0x0059
 #define AD4883_CHIP_ID						0x005B
 #define AD4884_CHIP_ID						0x005C
+#define AD4885_CHIP_ID						0x005D
 
 #define AD4080_MAX_CHANNELS					2
 
@@ -145,6 +146,10 @@
 #define AD4080_MAX_SAMP_FREQ					40000000
 #define AD4080_MIN_SAMP_FREQ					1250000
 
+/* debugfs direct_reg_access channel windowing: 0x0RR = ch0, 0x1RR = ch1 */
+#define AD4080_DEBUGFS_REG_CH_MSK				GENMASK(15, 8)
+#define AD4080_DEBUGFS_REG_OFFSET_MSK				GENMASK(7, 0)
+
 enum ad4080_filter_type {
 	FILTER_NONE,
 	SINC_1,
@@ -210,11 +215,16 @@ static int ad4080_reg_access(struct iio_dev *indio_dev, unsigned int reg,
 			     unsigned int writeval, unsigned int *readval)
 {
 	struct ad4080_state *st = iio_priv(indio_dev);
+	unsigned int ch = FIELD_GET(AD4080_DEBUGFS_REG_CH_MSK, reg);
+	unsigned int offset = FIELD_GET(AD4080_DEBUGFS_REG_OFFSET_MSK, reg);
+
+	if (ch >= st->info->num_channels)
+		return -EINVAL;
 
 	if (readval)
-		return regmap_read(st->regmap[0], reg, readval);
+		return regmap_read(st->regmap[ch], offset, readval);
 
-	return regmap_write(st->regmap[0], reg, writeval);
+	return regmap_write(st->regmap[ch], offset, writeval);
 }
 
 static int ad4080_get_scale(struct ad4080_state *st, int *val, int *val2)
@@ -552,6 +562,11 @@ static const struct iio_chan_spec ad4884_channels[] = {
 	AD4880_CHANNEL_DEFINE(16, 16, 1),
 };
 
+static const struct iio_chan_spec ad4885_channels[] = {
+	AD4880_CHANNEL_DEFINE(16, 16, 0),
+	AD4880_CHANNEL_DEFINE(16, 16, 1),
+};
+
 static const struct ad4080_chip_info ad4080_chip_info = {
 	.name = "ad4080",
 	.product_id = AD4080_CHIP_ID,
@@ -672,6 +687,16 @@ static const struct ad4080_chip_info ad4884_chip_info = {
 	.lvds_cnv_clk_cnt_max = 2,
 };
 
+static const struct ad4080_chip_info ad4885_chip_info = {
+	.name = "ad4885",
+	.product_id = AD4885_CHIP_ID,
+	.scale_table = ad4080_scale_table,
+	.num_scales = ARRAY_SIZE(ad4080_scale_table),
+	.num_channels = 2,
+	.channels = ad4885_channels,
+	.lvds_cnv_clk_cnt_max = 8,
+};
+
 static int ad4080_setup_channel(struct ad4080_state *st, unsigned int ch)
 {
 	struct device *dev = regmap_get_device(st->regmap[ch]);
@@ -685,7 +710,8 @@ static int ad4080_setup_channel(struct ad4080_state *st, unsigned int ch)
 		return ret;
 
 	ret = regmap_write(st->regmap[ch], AD4080_REG_INTERFACE_CONFIG_A,
-			   AD4080_INTERFACE_CONFIG_A_SDO_ENABLE);
+			   AD4080_INTERFACE_CONFIG_A_SDO_ENABLE |
+			   AD4080_INTERFACE_CONFIG_A_ADDR_ASC);
 	if (ret)
 		return ret;
 
@@ -881,6 +907,7 @@ static const struct spi_device_id ad4080_id[] = {
 	{ .name = "ad4880", .driver_data = (kernel_ulong_t)&ad4880_chip_info },
 	{ .name = "ad4883", .driver_data = (kernel_ulong_t)&ad4883_chip_info },
 	{ .name = "ad4884", .driver_data = (kernel_ulong_t)&ad4884_chip_info },
+	{ .name = "ad4885", .driver_data = (kernel_ulong_t)&ad4885_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4080_id);
@@ -898,6 +925,7 @@ static const struct of_device_id ad4080_of_match[] = {
 	{ .compatible = "adi,ad4880", &ad4880_chip_info },
 	{ .compatible = "adi,ad4883", &ad4883_chip_info },
 	{ .compatible = "adi,ad4884", &ad4884_chip_info },
+	{ .compatible = "adi,ad4885", &ad4885_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ad4080_of_match);
diff --git a/drivers/iio/adc/ad4134.c b/drivers/iio/adc/ad4134.c
index 0490218bb0e9b1..70f80bf1825f84 100644
--- a/drivers/iio/adc/ad4134.c
+++ b/drivers/iio/adc/ad4134.c
@@ -496,4 +496,3 @@ module_spi_driver(ad4134_driver);
 MODULE_AUTHOR("Marcelo Schmitt <marcelo.schmitt@analog.com>");
 MODULE_DESCRIPTION("Analog Devices AD4134 SPI driver");
 MODULE_LICENSE("GPL");
-MODULE_IMPORT_NS("IIO_AD4134");
diff --git a/drivers/iio/adc/ad7173.c b/drivers/iio/adc/ad7173.c
index eb47175a052870..24904be5090a46 100644
--- a/drivers/iio/adc/ad7173.c
+++ b/drivers/iio/adc/ad7173.c
@@ -775,9 +775,9 @@ static int ad7173_load_config(struct ad7173_state *st,
 
 	return ad_sd_write_reg(&st->sd, AD7173_REG_FILTER(free_cfg_slot), 2,
 			       FIELD_PREP(AD7173_FILTER_SINC3_MAP, 0) |
-			       FIELD_PREP(AD7173_FILTER_ENHFILT_MASK,
-					  post_filter_enable) |
 			       FIELD_PREP(AD7173_FILTER_ENHFILTEN,
+					  post_filter_enable) |
+			       FIELD_PREP(AD7173_FILTER_ENHFILT_MASK,
 					  post_filter_select) |
 			       FIELD_PREP(AD7173_FILTER_ORDER, 0) |
 			       FIELD_PREP(AD7173_FILTER_ODR_MASK,
@@ -1801,8 +1801,7 @@ static int ad7173_fw_parse_channel_config(struct iio_dev *indio_dev)
 	indio_dev->num_channels = num_channels;
 	st->num_channels = num_channels;
 
-	chan_arr = devm_kcalloc(dev, sizeof(*indio_dev->channels),
-				st->num_channels, GFP_KERNEL);
+	chan_arr = devm_kcalloc(dev, sizeof(*chan_arr), st->num_channels, GFP_KERNEL);
 	if (!chan_arr)
 		return -ENOMEM;
 
diff --git a/drivers/iio/adc/ad7192.c b/drivers/iio/adc/ad7192.c
index d8f5886fcf6947..8bd81054631112 100644
--- a/drivers/iio/adc/ad7192.c
+++ b/drivers/iio/adc/ad7192.c
@@ -1406,6 +1406,9 @@ static int ad7192_probe(struct spi_device *spi)
 	st->int_vref_mv = ret == -ENODEV ? avdd_mv : ret / MILLI;
 
 	st->chip_info = spi_get_device_match_data(spi);
+	if (!st->chip_info)
+		return -ENODATA;
+
 	indio_dev->name = st->chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->info = st->chip_info->info;
diff --git a/drivers/iio/adc/ad7606_spi.c b/drivers/iio/adc/ad7606_spi.c
index 1ebdfb8580ab52..8c479fa50a2376 100644
--- a/drivers/iio/adc/ad7606_spi.c
+++ b/drivers/iio/adc/ad7606_spi.c
@@ -456,7 +456,11 @@ static const struct ad7606_bus_info ad7616_bus_info = {
 
 static int ad7606_spi_probe(struct spi_device *spi)
 {
-	const struct ad7606_bus_info *bus_info = spi_get_device_match_data(spi);
+	const struct ad7606_bus_info *bus_info;
+
+	bus_info = spi_get_device_match_data(spi);
+	if (!bus_info)
+		return -ENODATA;
 
 	return ad7606_probe(&spi->dev, spi->irq, NULL,
 			    bus_info->chip_info, bus_info->bops);
diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c
index e9060c1bbe6fbe..3972821c295a13 100644
--- a/drivers/iio/adc/ad7768-1.c
+++ b/drivers/iio/adc/ad7768-1.c
@@ -1829,6 +1829,9 @@ static int ad7768_probe(struct spi_device *spi)
 	}
 
 	st->chip = spi_get_device_match_data(spi);
+	if (!st->chip)
+		return -ENODATA;
+
 	st->spi = spi;
 
 	st->regmap = devm_regmap_init_spi(spi, &ad7768_regmap_config);
diff --git a/drivers/iio/adc/ad7768.c b/drivers/iio/adc/ad7768.c
new file mode 100644
index 00000000000000..aa2db87a1b9f1a
--- /dev/null
+++ b/drivers/iio/adc/ad7768.c
@@ -0,0 +1,1921 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Analog Devices AD7768 ADC driver
+ *
+ * Copyright 2018-2026 Analog Devices Inc.
+ */
+
+#include <linux/array_size.h>
+#include <linux/auxiliary_bus.h>
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/log2.h>
+#include <linux/math.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/regulator/driver.h>
+#include <linux/reset.h>
+#include <linux/spi/spi.h>
+#include <linux/time.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+#include <linux/units.h>
+
+#include <linux/iio/backend.h>
+#include <linux/iio/iio.h>
+
+#define AD7768_REG_CH_STANDBY				0x00
+
+#define AD7768_REG_CH_MODE(x)				(0x01 + (x))
+#define   AD7768_CH_MODE_FILTER_TYPE_MSK		BIT(3)
+#define   AD7768_CH_MODE_DEC_RATE_MSK			GENMASK(2, 0)
+
+#define AD7768_REG_CH_MODE_SEL				0x03
+
+#define AD7768_REG_POWER_MODE				0x04
+#define   AD7768_SLEEP_MODE_MSK				BIT(7)
+#define   AD7768_POWER_MODE_POWER_MODE_MSK		GENMASK(5, 4)
+#define     AD7768_POWER_MODE_POWER_MODE_LOW		0x0
+#define     AD7768_POWER_MODE_POWER_MODE_MEDIAN		0x2
+#define     AD7768_POWER_MODE_POWER_MODE_FAST		0x3
+#define   AD7768_POWER_MODE_LVDS_ENABLE			BIT(3)
+#define   AD7768_POWER_MODE_MCLK_DIV_MSK		GENMASK(1, 0)
+
+#define AD7768_REG_GENERAL_CONFIG			0x05
+#define   AD7768_GEN_CONFIG_VCM_SEL_MSK			GENMASK(1, 0)
+#define   AD7768_GEN_CONFIG_VCM_PD			BIT(4)
+
+#define AD7768_REG_DATA_CONTROL				0x06
+#define   AD7768_DATA_CONTROL_SPI_RESET_1		0x03
+#define   AD7768_DATA_CONTROL_SPI_RESET_2		0x02
+#define   AD7768_DATA_CONTROL_SPI_SYNC			BIT(7)
+
+#define AD7768_REG_INTERFACE_CFG			0x07
+#define   AD7768_INTERFACE_CFG_CRC_SELECT_MSK		GENMASK(3, 2)
+#define     AD7768_INTERFACE_CFG_CRC_SELECT_4		0x1
+#define   AD7768_INTERFACE_CFG_DCLK_DIV_MSK		GENMASK(1, 0)
+#define     AD7768_INTERFACE_CFG_DCLK_DIV_8		0x0
+#define     AD7768_INTERFACE_CFG_DCLK_DIV_4		0x1
+#define     AD7768_INTERFACE_CFG_DCLK_DIV_2		0x2
+#define     AD7768_INTERFACE_CFG_DCLK_DIV_1		0x3
+
+#define AD7768_REG_REV_ID				0x0A
+#define   AD7768_REV_ID_VAL				0x06
+
+#define AD7768_REG_GPIO_CONTROL				0x0E
+#define   AD7768_GPIO_UGPIO_ENABLE			BIT(7)
+#define   AD7768_GPIO4_OUTPUT_ENABLE			BIT(4)
+
+#define AD7768_REG_GPIO_WRITE				0x0F
+
+#define AD7768_REG_PRECHARGE_BUF1			0x11
+#define AD7768_REG_PRECHARGE_BUF2			0x12
+#define   AD7768_PREBUF_POS_EN(ch)			BIT((ch) * 2 + 0)
+#define   AD7768_PREBUF_NEG_EN(ch)			BIT((ch) * 2 + 1)
+
+#define AD7768_REG_REFP_BUF				0x13
+#define AD7768_REG_REFN_BUF				0x14
+
+#define AD7768_REG_OFFSET_BASE				0x1E
+#define AD7768_REG_GAIN_BASE				0x36
+#define AD7768_REG_PHASE_BASE				0x4E
+#define AD7768_REG_OFFSET(ch)	(AD7768_REG_OFFSET_BASE + 3 * (ch))
+#define AD7768_REG_GAIN(ch)	(AD7768_REG_GAIN_BASE + 3 * (ch))
+#define AD7768_REG_PHASE(ch)	(AD7768_REG_PHASE_BASE + (ch))
+
+#define AD7768_REG_DIAGNOSTIC_RX			0x56
+
+#define AD7768_REG_CHOP_CTRL				0x59
+
+#define AD7768_SPI_READ_CMD				BIT(15)
+#define AD7768_SPI_REG_MASK				GENMASK(14, 8)
+#define AD7768_SPI_DATA_MASK				GENMASK(7, 0)
+
+#define AD7768_SAMPLE_SIZE				32
+#define AD7768_MAX_DCLK_DIV				8
+#define AD7768_MIN_MCLK_FREQ_HZ				(1150 * HZ_PER_KHZ)
+#define AD7768_MIN_XTAL_FREQ_HZ				(8 * HZ_PER_MHZ)
+#define AD7768_MAX_MCLK_FREQ_HZ				(34 * HZ_PER_MHZ)
+#define AD7768_MAX_FREQ_PER_MODE			6
+#define AD7768_MAX_CHANNEL				8
+#define AD7768_NUM_CHANNEL_MODES			2
+#define AD7768_CALIB_SIGN_BIT				23
+#define AD7768_CALIB_REG_MSK				GENMASK(AD7768_CALIB_SIGN_BIT, 0)
+#define AD7768_WIDEBAND_SETTLING_SAMPLES		68
+#define AD7768_SINC5_SETTLING_SAMPLES			7
+
+enum ad7768_filter_type {
+	AD7768_FILTER_TYPE_WIDEBAND,
+	AD7768_FILTER_TYPE_SINC5,
+};
+
+enum ad7768_clock_source {
+	AD7768_CLOCK_SOURCE_MCLK,
+	AD7768_CLOCK_SOURCE_XTAL,
+	AD7768_CLOCK_SOURCE_LVDS,
+};
+
+struct ad7768_power_mode_info {
+	unsigned int mode;
+	unsigned int mclk_div;
+};
+
+static const struct ad7768_power_mode_info ad7768_power_modes[] = {
+	{ .mode = AD7768_POWER_MODE_POWER_MODE_LOW, .mclk_div = 32 },
+	{ .mode = AD7768_POWER_MODE_POWER_MODE_MEDIAN, .mclk_div = 8 },
+	{ .mode = AD7768_POWER_MODE_POWER_MODE_FAST, .mclk_div = 4 },
+};
+
+#define AD7768_MAX_FREQS \
+	(AD7768_MAX_FREQ_PER_MODE + ARRAY_SIZE(ad7768_power_modes))
+
+struct ad7768_precharge_config {
+	bool prebufp_en;
+	bool prebufn_en;
+	bool refbufp;
+	bool refbufn;
+};
+
+struct ad7768_freq_config {
+	unsigned int freq_hz;
+	unsigned int dec_rate;
+};
+
+struct ad7768_convdelay_params {
+	unsigned int shift;
+	unsigned int max_raw;
+	u64 step_ps;
+};
+
+struct ad7768_avail_freq {
+	unsigned int n_freqs;
+	struct ad7768_freq_config freq_cfg[AD7768_MAX_FREQ_PER_MODE];
+};
+
+struct ad7768_chip_info {
+	const char *name;
+	const struct regmap_config *regmap_config;
+	const unsigned int *available_datalines;
+	unsigned int num_datalines;
+	unsigned int num_channels;
+	const u8 *chan_map;
+	u8 prebuf_split;
+};
+
+struct ad7768_state {
+	struct regmap *regmap;
+	/* Protects device register access and configuration. */
+	struct mutex lock;
+	struct clk *mclk;
+	unsigned int datalines;
+	enum ad7768_clock_source clock_source;
+	const struct ad7768_chip_info *chip_info;
+	struct ad7768_avail_freq avail_freq[ARRAY_SIZE(ad7768_power_modes)];
+	unsigned int n_freqs;
+	int freqs[AD7768_MAX_FREQS];
+	u64 ch_convdelay_ps[AD7768_MAX_CHANNEL];
+	unsigned int ch_freq[AD7768_MAX_CHANNEL];
+	unsigned int power_mode_idx;
+	enum ad7768_filter_type ch_filter[AD7768_MAX_CHANNEL];
+	struct iio_backend *back;
+	struct regulator_dev *vcm_rdev;
+	unsigned int avdd1_uV;
+	unsigned int vref_uV[2];
+
+	/* Used only in regmap_read() callback, hence guarded by regmap lock. */
+	__be16 d16 __aligned(IIO_DMA_MINALIGN);
+};
+
+static const unsigned int ad7768_vcm_voltage_table[] = {
+	0, 1650000, 2500000, 2140000,
+};
+
+static int ad7768_vcm_list_voltage(struct regulator_dev *rdev,
+				   unsigned int selector)
+{
+	struct ad7768_state *st = rdev_get_drvdata(rdev);
+
+	if (selector >= ARRAY_SIZE(ad7768_vcm_voltage_table))
+		return -EINVAL;
+
+	if (!selector)
+		return DIV_ROUND_CLOSEST(st->avdd1_uV, 2);
+
+	return ad7768_vcm_voltage_table[selector];
+}
+
+static int ad7768_vcm_enable(struct regulator_dev *rdev)
+{
+	struct ad7768_state *st = rdev_get_drvdata(rdev);
+	int ret;
+
+	ret = pm_runtime_resume_and_get(regmap_get_device(st->regmap));
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_clear_bits(st->regmap, AD7768_REG_GENERAL_CONFIG,
+				AD7768_GEN_CONFIG_VCM_PD);
+	if (ret)
+		pm_runtime_put_autosuspend(regmap_get_device(st->regmap));
+
+	return ret;
+}
+
+static int ad7768_vcm_disable(struct regulator_dev *rdev)
+{
+	struct ad7768_state *st = rdev_get_drvdata(rdev);
+	int ret;
+
+	ret = regmap_set_bits(st->regmap, AD7768_REG_GENERAL_CONFIG,
+			      AD7768_GEN_CONFIG_VCM_PD);
+	if (ret)
+		return ret;
+
+	pm_runtime_put_autosuspend(regmap_get_device(st->regmap));
+
+	return 0;
+}
+
+static int ad7768_vcm_is_enabled(struct regulator_dev *rdev)
+{
+	struct ad7768_state *st = rdev_get_drvdata(rdev);
+	int ret;
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(regmap_get_device(st->regmap), pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = regmap_test_bits(st->regmap, AD7768_REG_GENERAL_CONFIG,
+			       AD7768_GEN_CONFIG_VCM_PD);
+	if (ret < 0)
+		return ret;
+
+	return !ret;
+}
+
+static int ad7768_vcm_set_voltage_sel(struct regulator_dev *rdev,
+				      unsigned int selector)
+{
+	struct ad7768_state *st = rdev_get_drvdata(rdev);
+	int ret;
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(regmap_get_device(st->regmap), pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(st->regmap, AD7768_REG_GENERAL_CONFIG,
+				  AD7768_GEN_CONFIG_VCM_SEL_MSK,
+				  FIELD_PREP(AD7768_GEN_CONFIG_VCM_SEL_MSK,
+					     selector));
+}
+
+static int ad7768_vcm_get_voltage_sel(struct regulator_dev *rdev)
+{
+	struct ad7768_state *st = rdev_get_drvdata(rdev);
+	unsigned int val;
+	int ret;
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(regmap_get_device(st->regmap), pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(st->regmap, AD7768_REG_GENERAL_CONFIG, &val);
+	if (ret)
+		return ret;
+
+	return FIELD_GET(AD7768_GEN_CONFIG_VCM_SEL_MSK, val);
+}
+
+static const struct regulator_ops ad7768_vcm_ops = {
+	.enable = ad7768_vcm_enable,
+	.disable = ad7768_vcm_disable,
+	.is_enabled = ad7768_vcm_is_enabled,
+	.list_voltage = ad7768_vcm_list_voltage,
+	.set_voltage_sel = ad7768_vcm_set_voltage_sel,
+	.get_voltage_sel = ad7768_vcm_get_voltage_sel,
+};
+
+static const struct regulator_desc ad7768_vcm_desc = {
+	.name = "vcm",
+	.of_match = "vcm-output",
+	.regulators_node = "regulators",
+	.n_voltages = ARRAY_SIZE(ad7768_vcm_voltage_table),
+	.ops = &ad7768_vcm_ops,
+	.type = REGULATOR_VOLTAGE,
+	.owner = THIS_MODULE,
+};
+
+static int ad7768_register_vcm_regulator(struct device *dev,
+					 struct ad7768_state *st)
+{
+	struct regulator_config config = {
+		.dev = dev,
+		.driver_data = st,
+	};
+	int ret;
+
+	/*
+	 * Start with VCM disabled so each enabled state is paired with the
+	 * runtime PM reference acquired by ad7768_vcm_enable().
+	 */
+	ret = regmap_set_bits(st->regmap, AD7768_REG_GENERAL_CONFIG,
+			      AD7768_GEN_CONFIG_VCM_PD);
+	if (ret)
+		return ret;
+
+	st->vcm_rdev = devm_regulator_register(dev, &ad7768_vcm_desc, &config);
+	return PTR_ERR_OR_ZERO(st->vcm_rdev);
+}
+
+static const unsigned int ad7768_dec_rate[AD7768_MAX_FREQ_PER_MODE] = {
+	32, 64, 128, 256, 512, 1024,
+};
+
+static const unsigned int ad7768_available_datalines[] = {
+	1, 2, 8,
+};
+
+static const unsigned int ad7768_4_available_datalines[] = {
+	1, 4,
+};
+
+static const u8 ad7768_chan_map[] = {
+	0, 1, 2, 3, 4, 5, 6, 7,
+};
+
+static const u8 ad7768_4_chan_map[] = {
+	0, 1, 4, 5,
+};
+
+static const char * const ad7768_supply_names[] = {
+	"avdd2", "iovdd",
+};
+
+static const char * const ad7768_vref_supply_names[][2] = {
+	{ "ref1p", "ref1n" },
+	{ "ref2p", "ref2n" },
+};
+
+static const char * const ad7768_clock_names[] = {
+	[AD7768_CLOCK_SOURCE_MCLK] = "mclk",
+	[AD7768_CLOCK_SOURCE_XTAL] = "xtal",
+	[AD7768_CLOCK_SOURCE_LVDS] = "lvds",
+};
+
+static u8 ad7768_channel_mask(const struct ad7768_state *st, u8 ch)
+{
+	return BIT(st->chip_info->chan_map[ch]);
+}
+
+static u8 ad7768_channel_mode_mask(const struct ad7768_state *st, u8 ch)
+{
+	/*
+	 * AD7768-4 channels 2 and 3 use bits 4 and 5 for CH_x_MODE, while
+	 * their CH_x_MDE_EN fields are in bits 2 and 3 and must be programmed
+	 * to the same value. On AD7768, chan_map[] is an identity map, so the
+	 * OR produces a single bit.
+	 */
+	return BIT(ch) | ad7768_channel_mask(st, ch);
+}
+
+static u8 ad7768_all_standby_mask(const struct ad7768_state *st)
+{
+	return GENMASK(st->chip_info->num_channels - 1, 0);
+}
+
+static void ad7768_keep_xtal_channel_active(const struct ad7768_state *st,
+					    unsigned long *standby_mask)
+{
+	/*
+	 * Crystal excitation requires channel 4 on AD7768 or channel 2 on
+	 * AD7768-4 to remain active.
+	 */
+	if (st->clock_source == AD7768_CLOCK_SOURCE_XTAL)
+		__clear_bit(st->chip_info->num_channels / 2, standby_mask);
+}
+
+static unsigned int ad7768_offset_reg(const struct ad7768_state *st,
+				      unsigned int ch)
+{
+	return AD7768_REG_OFFSET(st->chip_info->chan_map[ch]);
+}
+
+static unsigned int ad7768_gain_reg(const struct ad7768_state *st,
+				    unsigned int ch)
+{
+	return AD7768_REG_GAIN(st->chip_info->chan_map[ch]);
+}
+
+static unsigned int ad7768_phase_reg(const struct ad7768_state *st,
+				     unsigned int ch)
+{
+	return AD7768_REG_PHASE(st->chip_info->chan_map[ch]);
+}
+
+static u8 ad7768_precharge_buf1_mask(const struct ad7768_state *st, u16 val)
+{
+	return val & GENMASK(st->chip_info->prebuf_split - 1, 0);
+}
+
+static u8 ad7768_precharge_buf2_mask(const struct ad7768_state *st, u16 val)
+{
+	unsigned int split = st->chip_info->prebuf_split;
+
+	return field_get(GENMASK(2 * split - 1, split), val);
+}
+
+static int ad7768_regmap_read(void *context, const void *reg_buf,
+			      size_t reg_size, void *val_buf, size_t val_size)
+{
+	struct ad7768_state *st = spi_get_drvdata(context);
+	struct spi_device *spi = context;
+	struct spi_transfer t[] = {
+		{
+			.tx_buf = &st->d16,
+			.len = sizeof(st->d16),
+			.cs_change = 1,
+		}, {
+			/*
+			 * Register responses are delayed by one CS frame. While
+			 * receiving the response to this read, the device also
+			 * decodes another command on SDI. Repeat the read
+			 * command to avoid sending an unspecified dummy
+			 * command.
+			 */
+			.tx_buf = &st->d16,
+			.rx_buf = &st->d16,
+			.len = sizeof(st->d16),
+		},
+	};
+	u8 *data_val = val_buf;
+	unsigned int reg;
+	int ret;
+
+	reg = *(const u8 *)reg_buf;
+
+	st->d16 = be16_replace_bits(cpu_to_be16(AD7768_SPI_READ_CMD), reg,
+				    AD7768_SPI_REG_MASK);
+
+	ret = spi_sync_transfer(spi, t, ARRAY_SIZE(t));
+	if (ret)
+		return ret;
+
+	*data_val = be16_get_bits(st->d16, AD7768_SPI_DATA_MASK);
+
+	return 0;
+}
+
+static int ad7768_regmap_write(void *spi, const void *data, size_t count)
+{
+	return spi_write(spi, data, count);
+}
+
+static const struct regmap_bus ad7768_regmap_bus = {
+	.read = ad7768_regmap_read,
+	.write = ad7768_regmap_write,
+	.reg_format_endian_default = REGMAP_ENDIAN_BIG,
+	.val_format_endian_default = REGMAP_ENDIAN_BIG,
+};
+
+static bool ad7768_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case AD7768_REG_CH_STANDBY ... AD7768_REG_REV_ID:
+	case AD7768_REG_GPIO_CONTROL ... AD7768_REG_REFN_BUF:
+	case AD7768_REG_OFFSET(0) ... AD7768_REG_OFFSET(7) + 2:
+	case AD7768_REG_GAIN(0) ... AD7768_REG_GAIN(7) + 2:
+	case AD7768_REG_PHASE(0) ... AD7768_REG_PHASE(7):
+	case AD7768_REG_DIAGNOSTIC_RX ... AD7768_REG_CHOP_CTRL:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool ad7768_4_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case AD7768_REG_CH_STANDBY ... AD7768_REG_REV_ID:
+	case AD7768_REG_GPIO_CONTROL ... AD7768_REG_REFN_BUF:
+	case AD7768_REG_OFFSET(0) ... AD7768_REG_OFFSET(1) + 2:
+	case AD7768_REG_OFFSET(4) ... AD7768_REG_OFFSET(5) + 2:
+	case AD7768_REG_GAIN(0) ... AD7768_REG_GAIN(1) + 2:
+	case AD7768_REG_GAIN(4) ... AD7768_REG_GAIN(5) + 2:
+	case AD7768_REG_PHASE(0) ... AD7768_REG_PHASE(1):
+	case AD7768_REG_PHASE(4) ... AD7768_REG_PHASE(5):
+	case AD7768_REG_DIAGNOSTIC_RX ... AD7768_REG_CHOP_CTRL:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config ad7768_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = AD7768_REG_CHOP_CTRL,
+	/*
+	 * Regmap bulk transfers use one starting address, while each register
+	 * access requires a separate 16-bit SPI frame and reads use an off-frame
+	 * response. Split bulk transfers into individual register accesses.
+	 */
+	.use_single_read = true,
+	.use_single_write = true,
+	.readable_reg = ad7768_readable_reg,
+};
+
+static const struct regmap_config ad7768_4_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = AD7768_REG_CHOP_CTRL,
+	/*
+	 * Regmap bulk transfers use one starting address, while each register
+	 * access requires a separate 16-bit SPI frame and reads use an off-frame
+	 * response. Split bulk transfers into individual register accesses. Do
+	 * not use a register cache because some readable registers report live
+	 * hardware state.
+	 */
+	.use_single_read = true,
+	.use_single_write = true,
+	.readable_reg = ad7768_4_readable_reg,
+};
+
+static unsigned int ad7768_get_calib_reg_base(struct ad7768_state *st,
+					      const struct iio_chan_spec *chan,
+					      bool is_gain)
+{
+	if (is_gain)
+		return ad7768_gain_reg(st, chan->channel);
+
+	return ad7768_offset_reg(st, chan->channel);
+}
+
+static int ad7768_read_calib_value(struct ad7768_state *st,
+				   unsigned int base_reg, bool is_signed,
+				   int *val)
+{
+	unsigned int regval;
+	u8 data[3];
+	int ret;
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(regmap_get_device(st->regmap), pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&st->lock);
+
+	ret = regmap_bulk_read(st->regmap, base_reg, data, sizeof(data));
+	if (ret)
+		return ret;
+
+	regval = get_unaligned_be24(data);
+	*val = is_signed ? sign_extend32(regval, AD7768_CALIB_SIGN_BIT) : regval;
+
+	return 0;
+}
+
+static int ad7768_write_calib_value(struct ad7768_state *st,
+				    unsigned int base_reg, bool is_signed,
+				    int val)
+{
+	u8 data[3];
+	int ret;
+
+	if (is_signed &&
+	    val != sign_extend32(val, AD7768_CALIB_SIGN_BIT))
+		return -EINVAL;
+
+	if (!is_signed && (val < 0 || val > AD7768_CALIB_REG_MSK))
+		return -EINVAL;
+
+	put_unaligned_be24(val, data);
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(regmap_get_device(st->regmap), pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&st->lock);
+
+	return regmap_bulk_write(st->regmap, base_reg, data, sizeof(data));
+}
+
+static int ad7768_reg_access(struct iio_dev *indio_dev,
+			     unsigned int reg,
+			     unsigned int writeval,
+			     unsigned int *readval)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+	int ret;
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(regmap_get_device(st->regmap), pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	if (readval)
+		return regmap_read(st->regmap, reg, readval);
+
+	return regmap_write(st->regmap, reg, writeval);
+}
+
+static int ad7768_sync(struct ad7768_state *st)
+{
+	int ret;
+
+	ret = regmap_clear_bits(st->regmap, AD7768_REG_DATA_CONTROL,
+				AD7768_DATA_CONTROL_SPI_SYNC);
+	if (ret)
+		return ret;
+
+	return regmap_set_bits(st->regmap, AD7768_REG_DATA_CONTROL,
+			       AD7768_DATA_CONTROL_SPI_SYNC);
+}
+
+static int ad7768_set_power_mode(struct ad7768_state *st,
+				 unsigned int mode_idx)
+{
+	const struct ad7768_power_mode_info *mode_info;
+	int ret;
+
+	mode_info = &ad7768_power_modes[mode_idx];
+	ret = regmap_update_bits(st->regmap, AD7768_REG_POWER_MODE,
+				 AD7768_POWER_MODE_POWER_MODE_MSK |
+				 AD7768_POWER_MODE_MCLK_DIV_MSK,
+				 FIELD_PREP(AD7768_POWER_MODE_POWER_MODE_MSK,
+					    mode_info->mode) |
+				 FIELD_PREP(AD7768_POWER_MODE_MCLK_DIV_MSK,
+					    mode_info->mode));
+	if (ret)
+		return ret;
+
+	ret = ad7768_sync(st);
+	if (ret)
+		return ret;
+
+	st->power_mode_idx = mode_idx;
+
+	return 0;
+}
+
+static const struct ad7768_freq_config *
+ad7768_find_freq_config(const struct ad7768_state *st,
+			unsigned int mode_idx, unsigned int freq)
+{
+	const struct ad7768_avail_freq *avail_freq = &st->avail_freq[mode_idx];
+
+	for (unsigned int i = 0; i < avail_freq->n_freqs; i++) {
+		if (freq == avail_freq->freq_cfg[i].freq_hz)
+			return &avail_freq->freq_cfg[i];
+	}
+
+	return NULL;
+}
+
+static int ad7768_set_clk_divs(struct ad7768_state *st, unsigned int freq)
+{
+	const struct ad7768_freq_config *freq_cfg;
+	unsigned int mclk, dclk, dclk_div;
+	unsigned int chan_per_doutx;
+	unsigned int dclk_div_reg;
+
+	freq_cfg = ad7768_find_freq_config(st, st->power_mode_idx, freq);
+	if (!freq_cfg)
+		return -EINVAL;
+
+	mclk = clk_get_rate(st->mclk);
+	chan_per_doutx = st->chip_info->num_channels / st->datalines;
+	if (!chan_per_doutx)
+		return -EINVAL;
+
+	dclk = freq_cfg->freq_hz * AD7768_SAMPLE_SIZE * chan_per_doutx;
+	if (dclk > mclk)
+		return -EINVAL;
+
+	dclk_div = DIV_ROUND_CLOSEST(mclk, dclk);
+
+	/* Set the divider to the next-lowest supported power of two. */
+	dclk_div = rounddown_pow_of_two(min(dclk_div, AD7768_MAX_DCLK_DIV));
+
+	switch (dclk_div) {
+	case 1:
+		dclk_div_reg = AD7768_INTERFACE_CFG_DCLK_DIV_1;
+		break;
+	case 2:
+		dclk_div_reg = AD7768_INTERFACE_CFG_DCLK_DIV_2;
+		break;
+	case 4:
+		dclk_div_reg = AD7768_INTERFACE_CFG_DCLK_DIV_4;
+		break;
+	case 8:
+		dclk_div_reg = AD7768_INTERFACE_CFG_DCLK_DIV_8;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(st->regmap, AD7768_REG_INTERFACE_CFG,
+				  AD7768_INTERFACE_CFG_DCLK_DIV_MSK,
+				  FIELD_PREP(AD7768_INTERFACE_CFG_DCLK_DIV_MSK,
+					     dclk_div_reg));
+}
+
+static int ad7768_set_mode_decimation(struct ad7768_state *st,
+				      unsigned int freq, unsigned int mode)
+{
+	const struct ad7768_freq_config *freq_cfg;
+
+	freq_cfg = ad7768_find_freq_config(st, st->power_mode_idx, freq);
+	if (!freq_cfg)
+		return -EINVAL;
+
+	return regmap_update_bits(st->regmap, AD7768_REG_CH_MODE(mode),
+				  AD7768_CH_MODE_DEC_RATE_MSK,
+				  FIELD_PREP(AD7768_CH_MODE_DEC_RATE_MSK,
+					     freq_cfg->dec_rate));
+}
+
+static int ad7768_set_lowest_noise_mode(struct ad7768_state *st,
+					const unsigned long *scan_mask)
+{
+	unsigned int mode_idx = ARRAY_SIZE(ad7768_power_modes);
+
+	/*
+	 * The output data rate ranges overlap between the power modes. At a
+	 * common ODR, the faster mode has lower noise, so prefer the fastest
+	 * mode that supports every enabled channel.
+	 */
+	while (mode_idx--) {
+		unsigned int channel;
+
+		for_each_set_bit(channel, scan_mask, st->chip_info->num_channels) {
+			if (!ad7768_find_freq_config(st, mode_idx,
+						     st->ch_freq[channel]))
+				break;
+		}
+
+		/* All enabled channels support their requested frequency. */
+		if (channel == st->chip_info->num_channels)
+			return ad7768_set_power_mode(st, mode_idx);
+	}
+
+	return -EINVAL;
+}
+
+static bool ad7768_freq_supported_in_any_mode(const struct ad7768_state *st,
+					      unsigned int freq)
+{
+	for (unsigned int mode_idx = 0;
+	     mode_idx < ARRAY_SIZE(ad7768_power_modes); mode_idx++) {
+		if (ad7768_find_freq_config(st, mode_idx, freq))
+			return true;
+	}
+
+	return false;
+}
+
+static int ad7768_set_sampling_freq(struct iio_dev *indio_dev,
+				    unsigned int freq, unsigned int channel)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+
+	if (!freq || !ad7768_freq_supported_in_any_mode(st, freq))
+		return -EINVAL;
+
+	guard(mutex)(&st->lock);
+	st->ch_freq[channel] = freq;
+
+	return 0;
+}
+
+static int ad7768_set_filter_mode(struct iio_dev *indio_dev,
+				  const struct iio_chan_spec *chan,
+				  unsigned int mode)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	guard(mutex)(&st->lock);
+	st->ch_filter[chan->channel] = mode;
+
+	return 0;
+}
+
+static int ad7768_get_filter_mode(struct iio_dev *indio_dev,
+				  const struct iio_chan_spec *chan)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+
+	guard(mutex)(&st->lock);
+
+	return st->ch_filter[chan->channel];
+}
+
+static const char *const ad7768_filter_types[] = {
+	[AD7768_FILTER_TYPE_WIDEBAND] = "wideband",
+	[AD7768_FILTER_TYPE_SINC5] = "sinc5",
+};
+
+static const struct iio_enum ad7768_filter_types_enum = {
+	.items = ad7768_filter_types,
+	.num_items = ARRAY_SIZE(ad7768_filter_types),
+	.set = ad7768_set_filter_mode,
+	.get = ad7768_get_filter_mode,
+};
+
+static struct iio_chan_spec_ext_info ad7768_ext_info[] = {
+	IIO_ENUM("filter_type", IIO_SEPARATE, &ad7768_filter_types_enum),
+	IIO_ENUM_AVAILABLE("filter_type", IIO_SEPARATE, &ad7768_filter_types_enum),
+	{ }
+};
+
+static int ad7768_find_matching_mode(const bool *mode_used,
+				     const unsigned int *mode_freq,
+				     const enum ad7768_filter_type *mode_filter,
+				     unsigned int freq,
+				     enum ad7768_filter_type filter)
+{
+	for (unsigned int mode = 0; mode < AD7768_NUM_CHANNEL_MODES; mode++) {
+		if (!mode_used[mode] ||
+		    (mode_freq[mode] == freq && mode_filter[mode] == filter))
+			return mode;
+	}
+
+	return -EINVAL;
+}
+
+static int ad7768_get_convdelay_params(struct ad7768_state *st, unsigned int ch,
+				       struct ad7768_convdelay_params *params)
+{
+	const struct ad7768_freq_config *freq_cfg;
+	unsigned int dec_rate;
+	unsigned int mclk_div;
+	unsigned long mclk;
+	u64 mult;
+
+	freq_cfg = ad7768_find_freq_config(st, st->power_mode_idx,
+					   st->ch_freq[ch]);
+	if (!freq_cfg)
+		return -EINVAL;
+
+	dec_rate = ad7768_dec_rate[freq_cfg->dec_rate];
+	switch (dec_rate) {
+	case 32:
+		params->shift = 3;
+		params->max_raw = 31;
+		mult = 1;
+		break;
+	case 64:
+		params->shift = 2;
+		params->max_raw = 63;
+		mult = 1;
+		break;
+	case 128:
+		params->shift = 1;
+		params->max_raw = 127;
+		mult = 1;
+		break;
+	case 256:
+		params->shift = 0;
+		params->max_raw = 255;
+		mult = 1;
+		break;
+	case 512:
+		params->shift = 0;
+		params->max_raw = 255;
+		mult = 2;
+		break;
+	case 1024:
+		params->shift = 0;
+		params->max_raw = 255;
+		mult = 4;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	mclk = clk_get_rate(st->mclk);
+	if (!mclk || mclk > U32_MAX)
+		return -EINVAL;
+
+	mclk_div = ad7768_power_modes[st->power_mode_idx].mclk_div;
+	params->step_ps =
+		DIV_ROUND_CLOSEST_ULL(mult * PSEC_PER_SEC * mclk_div, (u32)mclk);
+	if (params->step_ps > U32_MAX)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int ad7768_set_channel_convdelay(struct ad7768_state *st,
+					unsigned int ch)
+{
+	u64 delay_ps = st->ch_convdelay_ps[ch];
+	struct ad7768_convdelay_params params;
+	u64 max_delay_ps;
+	u64 raw;
+	int ret;
+
+	ret = ad7768_get_convdelay_params(st, ch, &params);
+	if (ret)
+		return ret;
+
+	/*
+	 * Validate the requested delay before rounding so an out-of-range value
+	 * cannot round down into the valid range. Then verify that the rounded
+	 * value fits in the hardware register.
+	 */
+	max_delay_ps = (u64)params.max_raw * params.step_ps;
+	if (delay_ps > max_delay_ps)
+		return -EINVAL;
+
+	raw = DIV_ROUND_CLOSEST_ULL(delay_ps, (u32)params.step_ps);
+	if (raw > params.max_raw)
+		return -EINVAL;
+
+	return regmap_write(st->regmap, ad7768_phase_reg(st, ch),
+			    raw << params.shift);
+}
+
+static void ad7768_filter_wait(const unsigned int *mode_freq,
+			       const enum ad7768_filter_type *mode_filter,
+			       const bool *mode_used)
+{
+	unsigned int t_settle_us = 0;
+
+	for (unsigned int mode = 0; mode < AD7768_NUM_CHANNEL_MODES; mode++) {
+		unsigned int t_mode_us;
+		u8 settling_samples;
+
+		if (!mode_used[mode] || !mode_freq[mode])
+			continue;
+
+		if (mode_filter[mode] == AD7768_FILTER_TYPE_SINC5)
+			settling_samples = AD7768_SINC5_SETTLING_SAMPLES;
+		else
+			settling_samples = AD7768_WIDEBAND_SETTLING_SAMPLES;
+
+		t_mode_us = DIV_ROUND_UP(settling_samples * USEC_PER_SEC,
+					 mode_freq[mode]);
+		t_settle_us = max(t_settle_us, t_mode_us);
+	}
+
+	if (t_settle_us)
+		fsleep(t_settle_us);
+}
+
+static int ad7768_apply_channel_modes(struct iio_dev *indio_dev,
+				      const unsigned long *scan_mask)
+{
+	enum ad7768_filter_type mode_filter[AD7768_NUM_CHANNEL_MODES] = { };
+	unsigned int mode_freq[AD7768_NUM_CHANNEL_MODES] = { };
+	bool mode_used[AD7768_NUM_CHANNEL_MODES] = { };
+	struct ad7768_state *st = iio_priv(indio_dev);
+	struct device *dev = indio_dev->dev.parent;
+	unsigned long channel_mask;
+	unsigned long standby_mask;
+	unsigned int max_freq;
+	unsigned int c;
+	int ret;
+
+	guard(mutex)(&st->lock);
+
+	ret = ad7768_set_lowest_noise_mode(st, scan_mask);
+	if (ret == -EINVAL)
+		return dev_err_probe(dev, ret,
+				     "No power mode supports all ODRs\n");
+	if (ret)
+		return ret;
+
+	channel_mask = ad7768_all_standby_mask(st);
+	standby_mask = channel_mask & ~*scan_mask;
+	ad7768_keep_xtal_channel_active(st, &standby_mask);
+
+	for_each_set_bit(c, scan_mask, st->chip_info->num_channels) {
+		unsigned int mask;
+		int mode;
+
+		mode = ad7768_find_matching_mode(mode_used, mode_freq,
+						 mode_filter, st->ch_freq[c],
+						 st->ch_filter[c]);
+		if (mode < 0)
+			return dev_err_probe(dev, -EINVAL,
+					     "Over %d channel modes required\n",
+					     AD7768_NUM_CHANNEL_MODES);
+
+		mode_freq[mode] = st->ch_freq[c];
+		mode_filter[mode] = st->ch_filter[c];
+		mode_used[mode] = true;
+
+		mask = ad7768_channel_mode_mask(st, c);
+		ret = regmap_assign_bits(st->regmap, AD7768_REG_CH_MODE_SEL,
+					 mask, mode);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_update_bits(st->regmap, AD7768_REG_CH_STANDBY,
+				 channel_mask, standby_mask);
+	if (ret)
+		return ret;
+
+	max_freq = 0;
+	for (unsigned int mode = 0; mode < AD7768_NUM_CHANNEL_MODES; mode++) {
+		if (!mode_used[mode])
+			continue;
+
+		ret = ad7768_set_mode_decimation(st, mode_freq[mode], mode);
+		if (ret)
+			return ret;
+
+		ret = regmap_assign_bits(st->regmap, AD7768_REG_CH_MODE(mode),
+					 AD7768_CH_MODE_FILTER_TYPE_MSK,
+					 mode_filter[mode]);
+		if (ret)
+			return ret;
+
+		max_freq = max(max_freq, mode_freq[mode]);
+	}
+
+	for_each_set_bit(c, scan_mask, st->chip_info->num_channels) {
+		ret = ad7768_set_channel_convdelay(st, c);
+		if (ret == -EINVAL)
+			return dev_err_probe(regmap_get_device(st->regmap), ret,
+					     "Invalid conversion delay for channel %u\n",
+					     c);
+		if (ret)
+			return ret;
+	}
+
+	ret = ad7768_set_clk_divs(st, max_freq);
+	if (ret)
+		return ret;
+
+	ret = ad7768_sync(st);
+	if (ret)
+		return ret;
+
+	/* Apply a filter settling time (datasheet Tables 28 and 29). */
+	ad7768_filter_wait(mode_freq, mode_filter, mode_used);
+
+	return 0;
+}
+
+static int ad7768_update_scan_mode(struct iio_dev *indio_dev,
+				   const unsigned long *scan_mask)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+	int ret;
+
+	ret = ad7768_apply_channel_modes(indio_dev, scan_mask);
+	if (ret)
+		return ret;
+
+	for (unsigned int ch = 0; ch < st->chip_info->num_channels; ch++) {
+		if (test_bit(ch, scan_mask))
+			ret = iio_backend_chan_enable(st->back, ch);
+		else
+			ret = iio_backend_chan_disable(st->back, ch);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static const struct ad7768_chip_info ad7768_chip_info = {
+	.name = "ad7768",
+	.num_channels = 8,
+	.regmap_config = &ad7768_regmap_config,
+	.available_datalines = ad7768_available_datalines,
+	.num_datalines = ARRAY_SIZE(ad7768_available_datalines),
+	.chan_map = ad7768_chan_map,
+	.prebuf_split = 8,
+};
+
+static const struct ad7768_chip_info ad7768_4_chip_info = {
+	.name = "ad7768-4",
+	.num_channels = 4,
+	.regmap_config = &ad7768_4_regmap_config,
+	.available_datalines = ad7768_4_available_datalines,
+	.num_datalines = ARRAY_SIZE(ad7768_4_available_datalines),
+	.chan_map = ad7768_4_chan_map,
+	.prebuf_split = 4,
+};
+
+static int ad7768_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+
+	return pm_runtime_resume_and_get(regmap_get_device(st->regmap));
+}
+
+static int ad7768_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+
+	pm_runtime_put_autosuspend(regmap_get_device(st->regmap));
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops ad7768_buffer_ops = {
+	.preenable = ad7768_buffer_preenable,
+	.postdisable = ad7768_buffer_postdisable,
+};
+
+static int ad7768_read_scale(struct ad7768_state *st,
+			     const struct iio_chan_spec *chan,
+			     int *val, int *val2)
+{
+	unsigned int vref_idx;
+
+	vref_idx = chan->channel >= st->chip_info->num_channels / 2;
+	*val = 2 * st->vref_uV[vref_idx] / (MICRO / MILLI);
+	*val2 = chan->scan_type.realbits;
+
+	return IIO_VAL_FRACTIONAL_LOG2;
+}
+
+static int ad7768_read_raw(struct iio_dev *indio_dev,
+			   const struct iio_chan_spec *chan,
+			   int *val, int *val2, long info)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+	int ret;
+
+	switch (info) {
+	case IIO_CHAN_INFO_SCALE:
+		return ad7768_read_scale(st, chan, val, val2);
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		guard(mutex)(&st->lock);
+		*val = st->ch_freq[chan->channel];
+
+		return IIO_VAL_INT;
+	}
+
+	case IIO_CHAN_INFO_CALIBBIAS:
+	case IIO_CHAN_INFO_CALIBSCALE: {
+		bool is_gain = info == IIO_CHAN_INFO_CALIBSCALE;
+		unsigned int base_reg;
+
+		base_reg = ad7768_get_calib_reg_base(st, chan, is_gain);
+		ret = ad7768_read_calib_value(st, base_reg, !is_gain, val);
+		if (ret)
+			return ret;
+
+		return IIO_VAL_INT;
+	}
+	case IIO_CHAN_INFO_CONVDELAY: {
+		guard(mutex)(&st->lock);
+		iio_val_s64_decompose(st->ch_convdelay_ps[chan->channel],
+				      val, val2);
+
+		return IIO_VAL_DECIMAL64_PICO;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad7768_write_raw_get_fmt(struct iio_dev *indio_dev,
+				    struct iio_chan_spec const *chan, long info)
+{
+	switch (info) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+	case IIO_CHAN_INFO_CALIBBIAS:
+	case IIO_CHAN_INFO_CALIBSCALE:
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_CONVDELAY:
+		return IIO_VAL_DECIMAL64_PICO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad7768_write_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int val, int val2, long info)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+	unsigned int base_reg;
+	s64 delay_ps;
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	switch (info) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return ad7768_set_sampling_freq(indio_dev, val, chan->channel);
+	case IIO_CHAN_INFO_CALIBBIAS:
+		base_reg = ad7768_get_calib_reg_base(st, chan, false);
+		return ad7768_write_calib_value(st, base_reg, true, val);
+	case IIO_CHAN_INFO_CALIBSCALE:
+		base_reg = ad7768_get_calib_reg_base(st, chan, true);
+		return ad7768_write_calib_value(st, base_reg, false, val);
+	case IIO_CHAN_INFO_CONVDELAY: {
+		delay_ps = iio_val_s64_compose(val, val2);
+		if (delay_ps < 0)
+			return -EINVAL;
+
+		guard(mutex)(&st->lock);
+		st->ch_convdelay_ps[chan->channel] = delay_ps;
+		return 0;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad7768_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type, int *length,
+			     long info)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+
+	if (info != IIO_CHAN_INFO_SAMP_FREQ)
+		return -EINVAL;
+
+	*vals = st->freqs;
+	*type = IIO_VAL_INT;
+	*length = st->n_freqs;
+
+	return IIO_AVAIL_LIST;
+}
+
+static const struct iio_info ad7768_info = {
+	.debugfs_reg_access = ad7768_reg_access,
+	.read_raw = ad7768_read_raw,
+	.write_raw_get_fmt = ad7768_write_raw_get_fmt,
+	.write_raw = ad7768_write_raw,
+	.read_avail = ad7768_read_avail,
+	.update_scan_mode = ad7768_update_scan_mode,
+};
+
+static int ad7768_gpio_adev_init(struct ad7768_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	struct spi_device *spi = to_spi_device(dev);
+	struct auxiliary_device *adev;
+	int id;
+
+	if (!device_property_read_bool(dev, "gpio-controller"))
+		return 0;
+
+	/*
+	 * Use the SPI bus number and chip select to derive a stable per-device
+	 * ID.
+	 */
+	id = (spi->controller->bus_num << 8) | spi_get_chipselect(spi, 0);
+	adev = __devm_auxiliary_device_create(dev, KBUILD_MODNAME, "gpio",
+					      NULL, id);
+	if (!adev)
+		return dev_err_probe(dev, -ENODEV,
+				     "Failed to create GPIO auxiliary device\n");
+
+	return 0;
+}
+
+static int ad7768_configure_precharge_buffers(struct iio_dev *indio_dev,
+					      struct ad7768_precharge_config *precharge_cfg)
+{
+	struct ad7768_state *st = iio_priv(indio_dev);
+	u8 prebuf1_val, prebuf2_val;
+	u16 prebuf_mask = 0;
+	u8 refbufp_val = 0;
+	u8 refbufn_val = 0;
+	int ret;
+
+	for (unsigned int ch = 0; ch < indio_dev->num_channels; ch++) {
+		u8 channel = indio_dev->channels[ch].channel;
+
+		if (precharge_cfg[channel].prebufp_en)
+			prebuf_mask |= AD7768_PREBUF_POS_EN(channel);
+
+		if (precharge_cfg[channel].prebufn_en)
+			prebuf_mask |= AD7768_PREBUF_NEG_EN(channel);
+
+		if (precharge_cfg[channel].refbufp)
+			refbufp_val |= ad7768_channel_mask(st, channel);
+
+		if (precharge_cfg[channel].refbufn)
+			refbufn_val |= ad7768_channel_mask(st, channel);
+	}
+
+	prebuf1_val = ad7768_precharge_buf1_mask(st, ~prebuf_mask);
+	prebuf2_val = ad7768_precharge_buf2_mask(st, ~prebuf_mask);
+
+	ret = regmap_write(st->regmap, AD7768_REG_PRECHARGE_BUF1, prebuf1_val);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(st->regmap, AD7768_REG_PRECHARGE_BUF2, prebuf2_val);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(st->regmap, AD7768_REG_REFP_BUF, refbufp_val);
+	if (ret)
+		return ret;
+
+	return regmap_write(st->regmap, AD7768_REG_REFN_BUF, refbufn_val);
+}
+
+static void ad7768_set_available_sampling_freqs(struct ad7768_state *st)
+{
+	unsigned int n_power_modes = ARRAY_SIZE(ad7768_power_modes);
+	unsigned int mclk = clk_get_rate(st->mclk);
+
+	for (unsigned int mode_idx = 0; mode_idx < n_power_modes; mode_idx++) {
+		unsigned int dec = ARRAY_SIZE(ad7768_dec_rate);
+		struct ad7768_avail_freq *avail_freq;
+		unsigned int div;
+
+		avail_freq = &st->avail_freq[mode_idx];
+		div = ad7768_power_modes[mode_idx].mclk_div;
+		while (dec--) {
+			struct ad7768_freq_config *freq_cfg;
+
+			freq_cfg = &avail_freq->freq_cfg[avail_freq->n_freqs++];
+			freq_cfg->dec_rate = dec;
+			freq_cfg->freq_hz = mclk / (ad7768_dec_rate[dec] * div);
+		}
+	}
+
+	/* One DOUT line cannot carry the AD7768 fast-mode x32 output rate. */
+	if (st->datalines == 1 &&
+	    st->chip_info->num_channels == AD7768_MAX_CHANNEL)
+		st->avail_freq[n_power_modes - 1].n_freqs--;
+
+	for (unsigned int mode_idx = 0; mode_idx < n_power_modes; mode_idx++) {
+		const struct ad7768_avail_freq *avail_freq;
+
+		avail_freq = &st->avail_freq[mode_idx];
+		for (unsigned int i = 0; i < avail_freq->n_freqs; i++) {
+			unsigned int freq = avail_freq->freq_cfg[i].freq_hz;
+
+			if (st->n_freqs && st->freqs[st->n_freqs - 1] >= freq)
+				continue;
+
+			st->freqs[st->n_freqs++] = freq;
+		}
+	}
+}
+
+static int ad7768_init_sampling_freqs(struct ad7768_state *st)
+{
+	const struct ad7768_avail_freq *avail_freq;
+	unsigned int default_freq;
+	int ret;
+
+	ret = ad7768_set_power_mode(st, ARRAY_SIZE(ad7768_power_modes) - 1);
+	if (ret)
+		return ret;
+
+	avail_freq = &st->avail_freq[st->power_mode_idx];
+	default_freq = avail_freq->freq_cfg[avail_freq->n_freqs - 1].freq_hz;
+	for (unsigned int channel = 0;
+	     channel < st->chip_info->num_channels; channel++) {
+		st->ch_freq[channel] = default_freq;
+		st->ch_filter[channel] = AD7768_FILTER_TYPE_WIDEBAND;
+	}
+
+	return 0;
+}
+
+static int ad7768_parse_config(struct iio_dev *indio_dev,
+			       struct device *dev)
+{
+	struct ad7768_precharge_config precharge_cfg[AD7768_MAX_CHANNEL] = { };
+	struct ad7768_state *st = iio_priv(indio_dev);
+	const unsigned int *available_datalines;
+	const char *propname;
+	bool datalines_valid = false;
+	struct iio_chan_spec *chan;
+	unsigned int num_channels;
+	unsigned long standby_mask;
+	unsigned int len;
+	int chan_idx = 0;
+	int ret;
+
+	propname = "adi,data-lines-number";
+	num_channels = device_get_named_child_node_count(dev, "channel");
+	if (num_channels == 0)
+		return dev_err_probe(dev, -ENOENT, "No channel specified\n");
+	if (num_channels > st->chip_info->num_channels)
+		return dev_err_probe(dev, -ENOSPC, "Invalid number of channels\n");
+
+	chan = devm_kcalloc(dev, num_channels, sizeof(*chan), GFP_KERNEL);
+	if (!chan)
+		return -ENOMEM;
+
+	indio_dev->channels = chan;
+	indio_dev->num_channels = num_channels;
+
+	standby_mask = ad7768_all_standby_mask(st);
+	ad7768_keep_xtal_channel_active(st, &standby_mask);
+
+	ret = regmap_write(st->regmap, AD7768_REG_CH_STANDBY, standby_mask);
+	if (ret)
+		return ret;
+
+	device_for_each_named_child_node_scoped(dev, child, "channel") {
+		u32 channel;
+
+		ret = fwnode_property_read_u32(child, "reg", &channel);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to parse reg of %pfwP\n",
+					     child);
+
+		if (channel >= st->chip_info->num_channels)
+			return dev_err_probe(dev, -ECHRNG,
+					     "Invalid channel %u in firmware\n",
+					     channel);
+
+		ret = regmap_clear_bits(st->regmap, AD7768_REG_CH_STANDBY,
+					BIT(channel));
+		if (ret)
+			return ret;
+
+		precharge_cfg[channel].prebufp_en =
+			fwnode_property_read_bool(child, "adi,prechargebuf-pos-enable");
+		precharge_cfg[channel].prebufn_en =
+			fwnode_property_read_bool(child, "adi,prechargebuf-neg-enable");
+		precharge_cfg[channel].refbufp =
+			fwnode_property_read_bool(child, "adi,refbuf-pos-enable");
+		precharge_cfg[channel].refbufn =
+			fwnode_property_read_bool(child, "adi,refbuf-neg-enable");
+
+		chan[chan_idx++] = (struct iio_chan_spec) {
+			.type = IIO_VOLTAGE,
+			.info_mask_separate = BIT(IIO_CHAN_INFO_SCALE) |
+					      BIT(IIO_CHAN_INFO_CALIBBIAS) |
+					      BIT(IIO_CHAN_INFO_CALIBSCALE) |
+					      BIT(IIO_CHAN_INFO_CONVDELAY) |
+					      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+			.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_SAMP_FREQ),
+			.indexed = 1,
+			.channel = channel,
+			.scan_index = channel,
+			.scan_type = {
+				.sign = 's',
+				.realbits = 24,
+				.storagebits = 32,
+			},
+			.ext_info = ad7768_ext_info,
+		};
+	}
+
+	ret = ad7768_configure_precharge_buffers(indio_dev, precharge_cfg);
+	if (ret)
+		return ret;
+
+	available_datalines = st->chip_info->available_datalines;
+	len = st->chip_info->num_datalines;
+	if (device_property_present(dev, propname)) {
+		ret = device_property_read_u32(dev, propname, &st->datalines);
+		if (ret)
+			return dev_err_probe(dev, ret, "Invalid %s property\n",
+					     propname);
+	} else {
+		st->datalines = available_datalines[len - 1];
+	}
+
+	for (unsigned int i = 0; i < len; i++) {
+		if (available_datalines[i] == st->datalines) {
+			datalines_valid = true;
+			break;
+		}
+	}
+
+	if (!datalines_valid)
+		return dev_err_probe(dev, -EINVAL,
+				     "Invalid %s %u for %s\n", propname,
+				     st->datalines, st->chip_info->name);
+
+	ad7768_set_available_sampling_freqs(st);
+
+	ret = ad7768_init_sampling_freqs(st);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to set power mode\n");
+
+	return 0;
+}
+
+static int ad7768_reset(struct ad7768_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	struct reset_control *reset_ctrl;
+	unsigned long reset_low_us;
+	unsigned long mclk;
+	int ret;
+
+	reset_ctrl = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(reset_ctrl))
+		return PTR_ERR(reset_ctrl);
+
+	if (reset_ctrl) {
+		mclk = clk_get_rate(st->mclk);
+		if (!mclk)
+			return -EINVAL;
+
+		/*
+		 * Minimum RESET low pulse width: 2 x tMCLK
+		 * (datasheet Table 1).
+		 */
+		reset_low_us = DIV_ROUND_UP_ULL(2ULL * USEC_PER_SEC, mclk);
+
+		ret = reset_control_assert(reset_ctrl);
+		if (ret)
+			return ret;
+
+		fsleep(max(1UL, reset_low_us));
+
+		ret = reset_control_deassert(reset_ctrl);
+		if (ret)
+			return ret;
+	} else {
+		ret = regmap_write(st->regmap, AD7768_REG_DATA_CONTROL,
+				   AD7768_DATA_CONTROL_SPI_RESET_1);
+		if (ret)
+			return ret;
+
+		ret = regmap_write(st->regmap, AD7768_REG_DATA_CONTROL,
+				   AD7768_DATA_CONTROL_SPI_RESET_2);
+		if (ret)
+			return ret;
+	}
+
+	/* ADC start-up time after reset: 1.66 ms max (datasheet Table 1) */
+	fsleep(1660);
+
+	return 0;
+}
+
+static void ad7768_disable_clk(void *clk)
+{
+	clk_disable_unprepare(clk);
+}
+
+static int ad7768_configure_xtal_clock(struct ad7768_state *st)
+{
+	int ret;
+
+	ret = regmap_set_bits(st->regmap, AD7768_REG_GPIO_WRITE, BIT(4));
+	if (ret)
+		return ret;
+
+	return regmap_set_bits(st->regmap, AD7768_REG_GPIO_CONTROL,
+			       AD7768_GPIO_UGPIO_ENABLE |
+			       AD7768_GPIO4_OUTPUT_ENABLE);
+}
+
+static int ad7768_enable_lvds_clock(struct ad7768_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	int ret;
+
+	ret = regmap_clear_bits(st->regmap, AD7768_REG_GPIO_WRITE, BIT(4));
+	if (ret)
+		return ret;
+
+	ret = regmap_set_bits(st->regmap, AD7768_REG_GPIO_CONTROL,
+			      AD7768_GPIO_UGPIO_ENABLE |
+			      AD7768_GPIO4_OUTPUT_ENABLE);
+	if (ret)
+		return ret;
+
+	ret = regmap_set_bits(st->regmap, AD7768_REG_POWER_MODE,
+			      AD7768_POWER_MODE_LVDS_ENABLE);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(st->mclk);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, ad7768_disable_clk, st->mclk);
+}
+
+static int ad7768_get_enable_vref(struct device *dev, unsigned int index)
+{
+	const char * const *supply = ad7768_vref_supply_names[index];
+	int refp_uV;
+	int refn_uV;
+
+	refp_uV = devm_regulator_get_enable_read_voltage(dev, supply[0]);
+	if (refp_uV < 0)
+		return dev_err_probe(dev, refp_uV,
+				     "Failed to get %s supply voltage\n", supply[0]);
+
+	refn_uV = devm_regulator_get_enable_read_voltage(dev, supply[1]);
+	if (refn_uV == -ENODEV)
+		refn_uV = 0;
+	else if (refn_uV < 0)
+		return dev_err_probe(dev, refn_uV,
+				     "Failed to get %s supply voltage\n", supply[1]);
+
+	if (refp_uV <= refn_uV)
+		return dev_err_probe(dev, -EINVAL,
+				     "Invalid reference %u voltage\n", index + 1);
+
+	return refp_uV - refn_uV;
+}
+
+static int ad7768_enter_sleep(struct ad7768_state *st)
+{
+	return regmap_set_bits(st->regmap, AD7768_REG_POWER_MODE,
+			       AD7768_SLEEP_MODE_MSK);
+}
+
+static void ad7768_power_off(void *data)
+{
+	struct ad7768_state *st = data;
+	struct device *dev = regmap_get_device(st->regmap);
+	int ret;
+
+	ret = ad7768_enter_sleep(st);
+	if (ret)
+		dev_err(dev, "Failed to put device to sleep\n");
+}
+
+static int ad7768_validate_mclk_rate(struct device *dev,
+				     const struct ad7768_state *st)
+{
+	unsigned long min_rate = AD7768_MIN_MCLK_FREQ_HZ;
+	unsigned long rate = clk_get_rate(st->mclk);
+
+	if (st->clock_source == AD7768_CLOCK_SOURCE_XTAL)
+		min_rate = AD7768_MIN_XTAL_FREQ_HZ;
+
+	if (rate < min_rate || rate > AD7768_MAX_MCLK_FREQ_HZ)
+		return dev_err_probe(dev, -EINVAL,
+				     "MCLK rate %lu Hz outside %lu-%lu Hz\n",
+				     rate, min_rate, AD7768_MAX_MCLK_FREQ_HZ);
+
+	return 0;
+}
+
+static int ad7768_probe(struct spi_device *spi)
+{
+	unsigned int spi_readback, rev_id;
+	struct device *dev = &spi->dev;
+	struct iio_dev *indio_dev;
+	struct ad7768_state *st;
+	const char *clock_name;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	spi_set_drvdata(spi, st);
+
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
+	st->chip_info = spi_get_device_match_data(spi);
+	if (!st->chip_info)
+		return dev_err_probe(dev, -ENODATA, "Failed to get match data\n");
+
+	ret = devm_regulator_get_enable_optional(dev, "avss");
+	if (ret == -ENODEV) {
+		/* AVSS may be connected directly to ground instead of a regulator. */
+	} else if (ret) {
+		return dev_err_probe(dev, ret, "Failed to enable AVSS supply\n");
+	}
+
+	ret = devm_regulator_get_enable_read_voltage(dev, "avdd1");
+	if (ret < 0)
+		return dev_err_probe(dev, ret,
+				     "Failed to enable AVDD1 supply\n");
+
+	st->avdd1_uV = ret;
+
+	ret = devm_regulator_bulk_get_enable(dev,
+					     ARRAY_SIZE(ad7768_supply_names),
+					     ad7768_supply_names);
+	if (ret)
+		return ret;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(ad7768_vref_supply_names); i++) {
+		ret = ad7768_get_enable_vref(dev, i);
+		if (ret < 0)
+			return ret;
+
+		st->vref_uV[i] = ret;
+	}
+
+	ret = device_property_match_property_string(dev, "clock-names",
+						    ad7768_clock_names,
+						    ARRAY_SIZE(ad7768_clock_names));
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Invalid clock source\n");
+
+	st->clock_source = ret;
+	clock_name = ad7768_clock_names[st->clock_source];
+
+	/*
+	 * The device must start on its internal clock. Keep the LVDS clock
+	 * disabled until GPIO4 is driven low and the LVDS input is enabled in
+	 * the power mode register.
+	 */
+	switch (st->clock_source) {
+	case AD7768_CLOCK_SOURCE_MCLK:
+	case AD7768_CLOCK_SOURCE_XTAL:
+		st->mclk = devm_clk_get_enabled(dev, clock_name);
+		break;
+	case AD7768_CLOCK_SOURCE_LVDS:
+		st->mclk = devm_clk_get(dev, clock_name);
+		break;
+	}
+
+	if (IS_ERR(st->mclk))
+		return dev_err_probe(dev, PTR_ERR(st->mclk),
+				     "Failed to get master clock\n");
+
+	ret = ad7768_validate_mclk_rate(dev, st);
+	if (ret)
+		return ret;
+
+	st->regmap = devm_regmap_init(dev, &ad7768_regmap_bus, spi,
+				      st->chip_info->regmap_config);
+	if (IS_ERR(st->regmap))
+		return PTR_ERR(st->regmap);
+
+	ret = ad7768_reset(st);
+	if (ret)
+		return ret;
+
+	/* Discard the reset response with a dummy SPI register read. */
+	ret = regmap_read(st->regmap, AD7768_REG_REV_ID, &spi_readback);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(st->regmap, AD7768_REG_REV_ID, &rev_id);
+	if (ret)
+		return ret;
+
+	if (rev_id != AD7768_REV_ID_VAL)
+		dev_info(dev, "Unexpected revision ID 0x%02x\n", rev_id);
+
+	switch (st->clock_source) {
+	case AD7768_CLOCK_SOURCE_MCLK:
+		break;
+	case AD7768_CLOCK_SOURCE_XTAL:
+		ret = ad7768_configure_xtal_clock(st);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to configure crystal clock\n");
+		break;
+	case AD7768_CLOCK_SOURCE_LVDS:
+		ret = ad7768_enable_lvds_clock(st);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to enable LVDS clock\n");
+		break;
+	}
+
+	ret = devm_add_action_or_reset(dev, ad7768_power_off, st);
+	if (ret)
+		return ret;
+
+	ret = ad7768_parse_config(indio_dev, dev);
+	if (ret)
+		return ret;
+
+	/*
+	 * Hardware supports CRC every 4 or 16 samples; the backend supports only
+	 * 4-sample CRC.
+	 */
+	ret = regmap_update_bits(st->regmap, AD7768_REG_INTERFACE_CFG,
+				 AD7768_INTERFACE_CFG_CRC_SELECT_MSK,
+				 FIELD_PREP(AD7768_INTERFACE_CFG_CRC_SELECT_MSK,
+					    AD7768_INTERFACE_CFG_CRC_SELECT_4));
+	if (ret)
+		return ret;
+
+	indio_dev->name = st->chip_info->name;
+	indio_dev->info = &ad7768_info;
+	indio_dev->setup_ops = &ad7768_buffer_ops;
+
+	st->back = devm_iio_backend_get(dev, NULL);
+	if (IS_ERR(st->back))
+		return PTR_ERR(st->back);
+
+	ret = devm_iio_backend_request_buffer(dev, st->back, indio_dev);
+	if (ret)
+		return ret;
+
+	ret = iio_backend_num_lanes_set(st->back, st->datalines);
+	if (ret)
+		return ret;
+
+	ret = iio_backend_crc_enable(st->back);
+	if (ret)
+		return ret;
+
+	ret = devm_iio_backend_enable(dev, st->back);
+	if (ret)
+		return ret;
+
+	pm_runtime_set_autosuspend_delay(dev, 2000);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_set_active_enabled(dev);
+	if (ret)
+		return ret;
+
+	ret = ad7768_register_vcm_regulator(dev, st);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to register VCM regulator\n");
+
+	ret = ad7768_gpio_adev_init(st);
+	if (ret)
+		return ret;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static int ad7768_runtime_suspend(struct device *dev)
+{
+	struct ad7768_state *st = dev_get_drvdata(dev);
+
+	return ad7768_enter_sleep(st);
+}
+
+static int ad7768_runtime_resume(struct device *dev)
+{
+	struct ad7768_state *st = dev_get_drvdata(dev);
+	int ret;
+
+	ret = regmap_clear_bits(st->regmap, AD7768_REG_POWER_MODE,
+				AD7768_SLEEP_MODE_MSK);
+	if (ret)
+		return ret;
+
+	/*
+	 * The datasheet does not specify a wake-up time. Allow 20 ms for the
+	 * ADC and digital clocks to restart.
+	 */
+	fsleep(20 * USEC_PER_MSEC);
+
+	return 0;
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(ad7768_pm_ops, ad7768_runtime_suspend,
+				 ad7768_runtime_resume, NULL);
+
+static const struct of_device_id ad7768_of_match[] = {
+	{ .compatible = "adi,ad7768", .data = &ad7768_chip_info },
+	{ .compatible = "adi,ad7768-4", .data = &ad7768_4_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ad7768_of_match);
+
+static const struct spi_device_id ad7768_spi_id[] = {
+	{ .name = "ad7768", .driver_data = (kernel_ulong_t)&ad7768_chip_info },
+	{ .name = "ad7768-4", .driver_data = (kernel_ulong_t)&ad7768_4_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, ad7768_spi_id);
+
+static struct spi_driver ad7768_driver = {
+	.probe = ad7768_probe,
+	.driver = {
+		.name = "ad7768",
+		.of_match_table = ad7768_of_match,
+		.pm = pm_ptr(&ad7768_pm_ops),
+	},
+	.id_table = ad7768_spi_id,
+};
+module_spi_driver(ad7768_driver);
+
+MODULE_AUTHOR("Stefan Popa <stefan.popa@analog.com>");
+MODULE_AUTHOR("Janani Sunil <janani.sunil@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD7768 ADC driver");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_BACKEND");
diff --git a/drivers/iio/adc/ad_sigma_delta.c b/drivers/iio/adc/ad_sigma_delta.c
index 1b410291da5378..119ddc5a13e27c 100644
--- a/drivers/iio/adc/ad_sigma_delta.c
+++ b/drivers/iio/adc/ad_sigma_delta.c
@@ -498,7 +498,6 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev)
 	const struct iio_scan_type *scan_type = &indio_dev->channels[0].scan_type;
 	struct spi_transfer *xfer = sigma_delta->sample_xfer;
 	unsigned int i, slot, channel;
-	u8 *samples_buf;
 	int ret;
 
 	if (sigma_delta->num_slots == 1) {
@@ -530,7 +529,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev)
 		xfer[1].bits_per_word = scan_type->realbits;
 		xfer[1].len = spi_bpw_to_bytes(scan_type->realbits);
 	} else {
-		unsigned int samples_buf_size, scan_size;
+		unsigned int scan_size;
 
 		if (sigma_delta->active_slots > 1) {
 			ret = ad_sigma_delta_append_status(sigma_delta, true);
@@ -538,17 +537,6 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev)
 				return ret;
 		}
 
-		samples_buf_size =
-			ALIGN(slot * BITS_TO_BYTES(scan_type->storagebits),
-			      sizeof(s64));
-		samples_buf_size += sizeof(s64);
-		samples_buf = devm_krealloc(&sigma_delta->spi->dev,
-					    sigma_delta->samples_buf,
-					    samples_buf_size, GFP_KERNEL);
-		if (!samples_buf)
-			return -ENOMEM;
-
-		sigma_delta->samples_buf = samples_buf;
 		scan_size = BITS_TO_BYTES(scan_type->realbits + scan_type->shift);
 		/* For 24-bit data, there is an extra byte of padding. */
 		xfer[1].rx_buf = &sigma_delta->rx_buf[scan_size == 3 ? 1 : 0];
@@ -855,6 +843,23 @@ int devm_ad_sd_setup_buffer_and_trigger(struct device *dev, struct iio_dev *indi
 
 		indio_dev->setup_ops = &ad_sd_buffer_setup_ops;
 	} else {
+		const struct iio_scan_type *scan_type =
+			&indio_dev->channels[0].scan_type;
+		unsigned int samples_buf_size;
+
+		/*
+		 * Worst-case size: all sequencer slots can be active, capped
+		 * at num_slots by ad_sd_validate_scan_mask().
+		 */
+		samples_buf_size =
+			ALIGN(sigma_delta->num_slots *
+			      BITS_TO_BYTES(scan_type->storagebits),
+			      sizeof(s64));
+		samples_buf_size += sizeof(s64);
+		sigma_delta->samples_buf = devm_kzalloc(dev, samples_buf_size, GFP_KERNEL);
+		if (!sigma_delta->samples_buf)
+			return -ENOMEM;
+
 		ret = devm_iio_triggered_buffer_setup(dev, indio_dev,
 						      &iio_pollfunc_store_time,
 						      &ad_sd_trigger_handler,
diff --git a/drivers/iio/adc/ade9000.c b/drivers/iio/adc/ade9000.c
index c6c3ea953feaee..c4450f03889840 100644
--- a/drivers/iio/adc/ade9000.c
+++ b/drivers/iio/adc/ade9000.c
@@ -219,7 +219,7 @@
 #define ADE9000_ST1_ERROR2_BIT		BIT(30)
 #define ADE9000_ST1_ERROR3_BIT		BIT(31)
 #define ADE9000_ST1_CROSSING_FIRST	6
-#define ADE9000_ST1_CROSSING_DEPTH	25
+#define ADE9000_ST1_CROSSING_DEPTH	26
 
 #define ADE9000_WFB_TRG_DIP_BIT		BIT(0)
 #define ADE9000_WFB_TRG_SWELL_BIT	BIT(1)
@@ -242,14 +242,6 @@
 #define ADE9000_LAST_PAGE_BIT		BIT(15)
 #define ADE9000_MIDDLE_PAGE_BIT		BIT(7)
 
-/*
- * Full scale Codes referred from Datasheet. Respective digital codes are
- * produced when ADC inputs are at full scale.
- */
-#define ADE9000_RMS_FULL_SCALE_CODES	52866837
-#define ADE9000_WATT_FULL_SCALE_CODES	20694066
-#define ADE9000_PCF_FULL_SCALE_CODES	74770000
-
 /* Phase and channel definitions */
 #define ADE9000_PHASE_A_NR		0
 #define ADE9000_PHASE_B_NR		1
@@ -290,7 +282,29 @@ enum ade9000_wfb_cfg {
 #define ADE9000_ADDR_ADJUST(addr, chan)					\
 	(((chan) == 0 ? 0 : (chan) == 1 ? 2 : 4) << 4 | (addr))
 
+/**
+ * struct ade9000_chip_info - part-specific configuration
+ * @name: IIO device name reported to userspace
+ * @channels: channel specification for this part
+ * @num_channels: number of entries in @channels
+ * @rms_full_scale_codes: digital code produced at full-scale RMS input
+ * @watt_full_scale_codes: digital code produced at full-scale power input
+ * @pcf_full_scale_codes: digital code produced at full-scale xI_PCF/xV_PCF input
+ *
+ * The full-scale codes are taken from the respective datasheets and are used to
+ * derive the IIO scale of the raw measurement channels.
+ */
+struct ade9000_chip_info {
+	const char *name;
+	const struct iio_chan_spec *channels;
+	unsigned int num_channels;
+	unsigned int rms_full_scale_codes;
+	unsigned int watt_full_scale_codes;
+	unsigned int pcf_full_scale_codes;
+};
+
 struct ade9000_state {
+	const struct ade9000_chip_info *info;
 	struct completion reset_completion;
 	struct mutex lock; /* Protects SPI transactions */
 	u8 wf_src;
@@ -444,7 +458,7 @@ static const struct iio_chan_spec_ext_info ade9000_ext_info[] = {
 			      BIT(IIO_CHAN_INFO_CALIBSCALE),		\
 	.event_spec = ade9000_current_events,				\
 	.num_event_specs = ARRAY_SIZE(ade9000_current_events),		\
-	.scan_index = num,						\
+	.scan_index = num * 2,						\
 	.indexed = 1,							\
 	.scan_type = {							\
 		.sign = 's',						\
@@ -464,7 +478,7 @@ static const struct iio_chan_spec_ext_info ade9000_ext_info[] = {
 			      BIT(IIO_CHAN_INFO_FREQUENCY),		\
 	.event_spec = ade9000_voltage_events,				\
 	.num_event_specs = ARRAY_SIZE(ade9000_voltage_events),		\
-	.scan_index = num + 1,	/* interleave with current channels */	\
+	.scan_index = num * 2 + 1, /* interleave with current channels */	\
 	.indexed = 1,							\
 	.scan_type = {							\
 		.sign = 's',						\
@@ -488,6 +502,7 @@ static const struct iio_chan_spec_ext_info ade9000_ext_info[] = {
 	.scan_index = -1						\
 }
 
+/* With swell/dip (sag) events - ADE9000 only */
 #define ADE9000_ALTVOLTAGE_RMS_CHANNEL(num) {				\
 	.type = IIO_ALTVOLTAGE,						\
 	.channel = num,							\
@@ -503,6 +518,20 @@ static const struct iio_chan_spec_ext_info ade9000_ext_info[] = {
 	.scan_index = -1						\
 }
 
+/* Without swell/dip events - parts lacking DIP_LVL/SWELL_LVL hardware */
+#define ADE9000_ALTVOLTAGE_RMS_CHANNEL_NO_EVENTS(num) {			\
+	.type = IIO_ALTVOLTAGE,						\
+	.channel = num,							\
+	.address = ADE9000_ADDR_ADJUST(ADE9000_REG_AVRMS, num),		\
+	.channel2 = IIO_MOD_RMS,					\
+	.modified = 1,							\
+	.indexed = 1,							\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |			\
+			      BIT(IIO_CHAN_INFO_SCALE) |		\
+			      BIT(IIO_CHAN_INFO_CALIBBIAS),		\
+	.scan_index = -1						\
+}
+
 #define ADE9000_POWER_ACTIVE_CHANNEL(num) {				\
 	.type = IIO_POWER,						\
 	.channel = num,							\
@@ -584,43 +613,82 @@ static const struct iio_chan_spec_ext_info ade9000_ext_info[] = {
 	.scan_index = -1						\
 }
 
-static const struct iio_chan_spec ade9000_channels[] = {
-	/* Phase A channels */
-	ADE9000_CURRENT_CHANNEL(ADE9000_PHASE_A_NR),
-	ADE9000_VOLTAGE_CHANNEL(ADE9000_PHASE_A_NR),
-	ADE9000_ALTCURRENT_RMS_CHANNEL(ADE9000_PHASE_A_NR),
-	ADE9000_ALTVOLTAGE_RMS_CHANNEL(ADE9000_PHASE_A_NR),
-	ADE9000_POWER_ACTIVE_CHANNEL(ADE9000_PHASE_A_NR),
-	ADE9000_POWER_REACTIVE_CHANNEL(ADE9000_PHASE_A_NR),
-	ADE9000_POWER_APPARENT_CHANNEL(ADE9000_PHASE_A_NR),
-	ADE9000_ENERGY_ACTIVE_CHANNEL(ADE9000_PHASE_A_NR, ADE9000_REG_AWATTHR_LO),
-	ADE9000_ENERGY_APPARENT_CHANNEL(ADE9000_PHASE_A_NR, ADE9000_REG_AVAHR_LO),
-	ADE9000_ENERGY_REACTIVE_CHANNEL(ADE9000_PHASE_A_NR, ADE9000_REG_AFVARHR_LO),
-	ADE9000_POWER_FACTOR_CHANNEL(ADE9000_PHASE_A_NR),
-	/* Phase B channels */
-	ADE9000_CURRENT_CHANNEL(ADE9000_PHASE_B_NR),
-	ADE9000_VOLTAGE_CHANNEL(ADE9000_PHASE_B_NR),
-	ADE9000_ALTCURRENT_RMS_CHANNEL(ADE9000_PHASE_B_NR),
-	ADE9000_ALTVOLTAGE_RMS_CHANNEL(ADE9000_PHASE_B_NR),
-	ADE9000_POWER_ACTIVE_CHANNEL(ADE9000_PHASE_B_NR),
-	ADE9000_POWER_REACTIVE_CHANNEL(ADE9000_PHASE_B_NR),
-	ADE9000_POWER_APPARENT_CHANNEL(ADE9000_PHASE_B_NR),
-	ADE9000_ENERGY_ACTIVE_CHANNEL(ADE9000_PHASE_B_NR, ADE9000_REG_BWATTHR_LO),
-	ADE9000_ENERGY_APPARENT_CHANNEL(ADE9000_PHASE_B_NR, ADE9000_REG_BVAHR_LO),
-	ADE9000_ENERGY_REACTIVE_CHANNEL(ADE9000_PHASE_B_NR, ADE9000_REG_BFVARHR_LO),
-	ADE9000_POWER_FACTOR_CHANNEL(ADE9000_PHASE_B_NR),
-	/* Phase C channels */
-	ADE9000_CURRENT_CHANNEL(ADE9000_PHASE_C_NR),
-	ADE9000_VOLTAGE_CHANNEL(ADE9000_PHASE_C_NR),
-	ADE9000_ALTCURRENT_RMS_CHANNEL(ADE9000_PHASE_C_NR),
-	ADE9000_ALTVOLTAGE_RMS_CHANNEL(ADE9000_PHASE_C_NR),
-	ADE9000_POWER_ACTIVE_CHANNEL(ADE9000_PHASE_C_NR),
-	ADE9000_POWER_REACTIVE_CHANNEL(ADE9000_PHASE_C_NR),
-	ADE9000_POWER_APPARENT_CHANNEL(ADE9000_PHASE_C_NR),
-	ADE9000_ENERGY_ACTIVE_CHANNEL(ADE9000_PHASE_C_NR, ADE9000_REG_CWATTHR_LO),
-	ADE9000_ENERGY_APPARENT_CHANNEL(ADE9000_PHASE_C_NR, ADE9000_REG_CVAHR_LO),
-	ADE9000_ENERGY_REACTIVE_CHANNEL(ADE9000_PHASE_C_NR, ADE9000_REG_CFVARHR_LO),
-	ADE9000_POWER_FACTOR_CHANNEL(ADE9000_PHASE_C_NR),
+/*
+ * Declare a full channel array. @altvoltage_rms picks the RMS voltage channel
+ * variant so parts without dip/swell hardware (e.g. ADE9078) omit those events.
+ */
+#define ADE9000_DECLARE_CHANNELS(_name, altvoltage_rms)			      \
+	static const struct iio_chan_spec _name[] = {			      \
+		/* Phase A channels */					      \
+		ADE9000_CURRENT_CHANNEL(ADE9000_PHASE_A_NR),		      \
+		ADE9000_VOLTAGE_CHANNEL(ADE9000_PHASE_A_NR),		      \
+		ADE9000_ALTCURRENT_RMS_CHANNEL(ADE9000_PHASE_A_NR),	      \
+		altvoltage_rms(ADE9000_PHASE_A_NR),			      \
+		ADE9000_POWER_ACTIVE_CHANNEL(ADE9000_PHASE_A_NR),	      \
+		ADE9000_POWER_REACTIVE_CHANNEL(ADE9000_PHASE_A_NR),	      \
+		ADE9000_POWER_APPARENT_CHANNEL(ADE9000_PHASE_A_NR),	      \
+		ADE9000_ENERGY_ACTIVE_CHANNEL(ADE9000_PHASE_A_NR,	      \
+					      ADE9000_REG_AWATTHR_LO),	      \
+		ADE9000_ENERGY_APPARENT_CHANNEL(ADE9000_PHASE_A_NR,	      \
+						ADE9000_REG_AVAHR_LO),	      \
+		ADE9000_ENERGY_REACTIVE_CHANNEL(ADE9000_PHASE_A_NR,	      \
+						ADE9000_REG_AFVARHR_LO),      \
+		ADE9000_POWER_FACTOR_CHANNEL(ADE9000_PHASE_A_NR),	      \
+		/* Phase B channels */					      \
+		ADE9000_CURRENT_CHANNEL(ADE9000_PHASE_B_NR),		      \
+		ADE9000_VOLTAGE_CHANNEL(ADE9000_PHASE_B_NR),		      \
+		ADE9000_ALTCURRENT_RMS_CHANNEL(ADE9000_PHASE_B_NR),	      \
+		altvoltage_rms(ADE9000_PHASE_B_NR),			      \
+		ADE9000_POWER_ACTIVE_CHANNEL(ADE9000_PHASE_B_NR),	      \
+		ADE9000_POWER_REACTIVE_CHANNEL(ADE9000_PHASE_B_NR),	      \
+		ADE9000_POWER_APPARENT_CHANNEL(ADE9000_PHASE_B_NR),	      \
+		ADE9000_ENERGY_ACTIVE_CHANNEL(ADE9000_PHASE_B_NR,	      \
+					      ADE9000_REG_BWATTHR_LO),	      \
+		ADE9000_ENERGY_APPARENT_CHANNEL(ADE9000_PHASE_B_NR,	      \
+						ADE9000_REG_BVAHR_LO),	      \
+		ADE9000_ENERGY_REACTIVE_CHANNEL(ADE9000_PHASE_B_NR,	      \
+						ADE9000_REG_BFVARHR_LO),      \
+		ADE9000_POWER_FACTOR_CHANNEL(ADE9000_PHASE_B_NR),	      \
+		/* Phase C channels */					      \
+		ADE9000_CURRENT_CHANNEL(ADE9000_PHASE_C_NR),		      \
+		ADE9000_VOLTAGE_CHANNEL(ADE9000_PHASE_C_NR),		      \
+		ADE9000_ALTCURRENT_RMS_CHANNEL(ADE9000_PHASE_C_NR),	      \
+		altvoltage_rms(ADE9000_PHASE_C_NR),			      \
+		ADE9000_POWER_ACTIVE_CHANNEL(ADE9000_PHASE_C_NR),	      \
+		ADE9000_POWER_REACTIVE_CHANNEL(ADE9000_PHASE_C_NR),	      \
+		ADE9000_POWER_APPARENT_CHANNEL(ADE9000_PHASE_C_NR),	      \
+		ADE9000_ENERGY_ACTIVE_CHANNEL(ADE9000_PHASE_C_NR,	      \
+					      ADE9000_REG_CWATTHR_LO),	      \
+		ADE9000_ENERGY_APPARENT_CHANNEL(ADE9000_PHASE_C_NR,	      \
+						ADE9000_REG_CVAHR_LO),	      \
+		ADE9000_ENERGY_REACTIVE_CHANNEL(ADE9000_PHASE_C_NR,	      \
+						ADE9000_REG_CFVARHR_LO),      \
+		ADE9000_POWER_FACTOR_CHANNEL(ADE9000_PHASE_C_NR),	      \
+	}
+
+ADE9000_DECLARE_CHANNELS(ade9000_channels, ADE9000_ALTVOLTAGE_RMS_CHANNEL);
+ADE9000_DECLARE_CHANNELS(ade9078_channels, ADE9000_ALTVOLTAGE_RMS_CHANNEL_NO_EVENTS);
+
+/*
+ * Full-scale codes referred from the respective datasheets. These are the
+ * digital codes produced when the ADC inputs are at full scale.
+ */
+static const struct ade9000_chip_info ade9000_chip_info = {
+	.name = "ade9000",
+	.channels = ade9000_channels,
+	.num_channels = ARRAY_SIZE(ade9000_channels),
+	.rms_full_scale_codes = 52866837,
+	.watt_full_scale_codes = 20694066,
+	.pcf_full_scale_codes = 74770000,
+};
+
+static const struct ade9000_chip_info ade9078_chip_info = {
+	.name = "ade9078",
+	.channels = ade9078_channels,
+	.num_channels = ARRAY_SIZE(ade9078_channels),
+	.rms_full_scale_codes = 52866837,
+	.watt_full_scale_codes = 20823646,
+	.pcf_full_scale_codes = 74680000,
 };
 
 static const struct reg_sequence ade9000_initialization_sequence[] = {
@@ -1064,7 +1132,7 @@ static int ade9000_read_raw(struct iio_dev *indio_dev,
 			case ADE9000_REG_CI_PCF:
 			case ADE9000_REG_CV_PCF:
 				*val = 1;
-				*val2 = ADE9000_PCF_FULL_SCALE_CODES;
+				*val2 = st->info->pcf_full_scale_codes;
 				return IIO_VAL_FRACTIONAL;
 			case ADE9000_REG_AIRMS:
 			case ADE9000_REG_AVRMS:
@@ -1073,14 +1141,14 @@ static int ade9000_read_raw(struct iio_dev *indio_dev,
 			case ADE9000_REG_CIRMS:
 			case ADE9000_REG_CVRMS:
 				*val = 1;
-				*val2 = ADE9000_RMS_FULL_SCALE_CODES;
+				*val2 = st->info->rms_full_scale_codes;
 				return IIO_VAL_FRACTIONAL;
 			default:
 				return -EINVAL;
 			}
 		case IIO_POWER:
 			*val = 1;
-			*val2 = ADE9000_WATT_FULL_SCALE_CODES;
+			*val2 = st->info->watt_full_scale_codes;
 			return IIO_VAL_FRACTIONAL;
 		default:
 			break;
@@ -1647,8 +1715,9 @@ static int ade9000_setup_clkout(struct device *dev, struct ade9000_state *st)
 		return 0;
 
 	/* CLKOUT passes through CLKIN with divider of 1 */
-	clkout_hw = devm_clk_hw_register_divider(dev, "clkout", __clk_get_name(st->clkin),
-						 CLK_SET_RATE_PARENT, NULL, 0, 1, 0, NULL);
+	clkout_hw = devm_clk_hw_register_fixed_factor(dev, "clkout",
+						      __clk_get_name(st->clkin),
+						      CLK_SET_RATE_PARENT, 1, 1);
 	if (IS_ERR(clkout_hw))
 		return dev_err_probe(dev, PTR_ERR(clkout_hw), "Failed to register clkout");
 
@@ -1692,6 +1761,10 @@ static int ade9000_probe(struct spi_device *spi)
 
 	st = iio_priv(indio_dev);
 
+	st->info = spi_get_device_match_data(spi);
+	if (!st->info)
+		return -ENODEV;
+
 	regmap = devm_regmap_init(dev, NULL, st, &ade9000_regmap_config);
 	if (IS_ERR(regmap))
 		return dev_err_probe(dev, PTR_ERR(regmap), "Unable to allocate ADE9000 regmap");
@@ -1705,18 +1778,6 @@ static int ade9000_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev);
-	if (ret)
-		return ret;
-
-	ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev);
-	if (ret)
-		return ret;
-
-	ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev);
-	if (ret)
-		return ret;
-
 	/* External CMOS clock input (optional - crystal can be used instead) */
 	st->clkin = devm_clk_get_optional_enabled(dev, NULL);
 	if (IS_ERR(st->clkin))
@@ -1726,7 +1787,7 @@ static int ade9000_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	indio_dev->name = "ade9000";
+	indio_dev->name = st->info->name;
 	indio_dev->info = &ade9000_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->setup_ops = &ade9000_buffer_ops;
@@ -1736,8 +1797,21 @@ static int ade9000_probe(struct spi_device *spi)
 		return dev_err_probe(&spi->dev, ret,
 				     "Failed to get and enable vdd regulator\n");
 
-	indio_dev->channels = ade9000_channels;
-	indio_dev->num_channels = ARRAY_SIZE(ade9000_channels);
+	/* Request IRQs only after powering the chip; the handlers do SPI access. */
+	ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev);
+	if (ret)
+		return ret;
+
+	ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev);
+	if (ret)
+		return ret;
+
+	ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev);
+	if (ret)
+		return ret;
+
+	indio_dev->channels = st->info->channels;
+	indio_dev->num_channels = st->info->num_channels;
 
 	ret = devm_iio_kfifo_buffer_setup(dev, indio_dev,
 					  &ade9000_buffer_ops);
@@ -1768,13 +1842,15 @@ static int ade9000_probe(struct spi_device *spi)
 };
 
 static const struct spi_device_id ade9000_id[] = {
-	{ .name = "ade9000" },
+	{ .name = "ade9000", .driver_data = (kernel_ulong_t)&ade9000_chip_info },
+	{ .name = "ade9078", .driver_data = (kernel_ulong_t)&ade9078_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ade9000_id);
 
 static const struct of_device_id ade9000_of_match[] = {
-	{ .compatible = "adi,ade9000" },
+	{ .compatible = "adi,ade9000", .data = &ade9000_chip_info },
+	{ .compatible = "adi,ade9078", .data = &ade9078_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ade9000_of_match);
diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c
index 589f03618ecf3a..8b7c45d51dbe8a 100644
--- a/drivers/iio/adc/adi-axi-adc.c
+++ b/drivers/iio/adc/adi-axi-adc.c
@@ -47,6 +47,9 @@
 #define    ADI_AXI_ADC_CTRL_DDR_EDGESEL_MASK	BIT(1)
 
 #define ADI_AXI_ADC_REG_CNTRL_3			0x004c
+/* Bit 8 is common to all AXI ADC cores; bits 7-0 are per-HDL-project. */
+#define    ADI_AXI_ADC_CNTRL_3_CRC_EN_MASK	BIT(8)
+
 #define   AXI_AD485X_CNTRL_3_OS_EN_MSK		BIT(2)
 #define   AXI_AD485X_CNTRL_3_PACKET_FORMAT_MSK	GENMASK(1, 0)
 #define   AXI_AD485X_PACKET_FORMAT_20BIT	0x0
@@ -596,6 +599,22 @@ static int axi_adc_reg_access(struct iio_backend *back, unsigned int reg,
 	return regmap_write(st->regmap, reg, writeval);
 }
 
+static int axi_adc_crc_enable(struct iio_backend *back)
+{
+	struct adi_axi_adc_state *st = iio_backend_get_priv(back);
+
+	return regmap_set_bits(st->regmap, ADI_AXI_ADC_REG_CNTRL_3,
+			       ADI_AXI_ADC_CNTRL_3_CRC_EN_MASK);
+}
+
+static int axi_adc_crc_disable(struct iio_backend *back)
+{
+	struct adi_axi_adc_state *st = iio_backend_get_priv(back);
+
+	return regmap_clear_bits(st->regmap, ADI_AXI_ADC_REG_CNTRL_3,
+				 ADI_AXI_ADC_CNTRL_3_CRC_EN_MASK);
+}
+
 static const struct regmap_config axi_adc_regmap_config = {
 	.val_bits = 32,
 	.reg_bits = 32,
@@ -644,6 +663,8 @@ static const struct iio_backend_ops adi_axi_adc_ops = {
 	.num_lanes_set = axi_adc_num_lanes_set,
 	.debugfs_reg_access = iio_backend_debugfs_ptr(axi_adc_reg_access),
 	.debugfs_print_chan_status = iio_backend_debugfs_ptr(axi_adc_debugfs_print_chan_status),
+	.crc_enable = axi_adc_crc_enable,
+	.crc_disable = axi_adc_crc_disable,
 };
 
 static const struct iio_backend_info adi_axi_adc_generic = {
@@ -714,6 +735,10 @@ static int adi_axi_adc_probe(struct platform_device *pdev)
 	if (!st)
 		return -ENOMEM;
 
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
diff --git a/drivers/iio/adc/aspeed_adc.c b/drivers/iio/adc/aspeed_adc.c
index 6169ce28261e12..452c30cf04f284 100644
--- a/drivers/iio/adc/aspeed_adc.c
+++ b/drivers/iio/adc/aspeed_adc.c
@@ -585,7 +585,9 @@ static int aspeed_adc_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(data->rst),
 				     "invalid or missing reset controller device tree entry");
 
-	reset_control_deassert(data->rst);
+	ret = reset_control_deassert(data->rst);
+	if (ret)
+		return ret;
 
 	ret = devm_add_action_or_reset(dev, aspeed_adc_reset_assert, data->rst);
 	if (ret)
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index e8a5285bb6d4f2..78081cbfde5e8c 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -2016,6 +2016,8 @@ static void at91_adc_dma_init(struct at91_adc_state *st)
 	unsigned int pages = DIV_ROUND_UP(AT91_HWFIFO_MAX_SIZE *
 					  sample_size * 2, PAGE_SIZE);
 
+	struct device *dma_dev;
+
 	if (st->dma_st.dma_chan)
 		return;
 
@@ -2026,7 +2028,8 @@ static void at91_adc_dma_init(struct at91_adc_state *st)
 		goto dma_exit;
 	}
 
-	st->dma_st.rx_buf = dma_alloc_coherent(st->dma_st.dma_chan->device->dev,
+	dma_dev = dmaengine_get_dma_device(st->dma_st.dma_chan);
+	st->dma_st.rx_buf = dma_alloc_coherent(dma_dev,
 					       pages * PAGE_SIZE,
 					       &st->dma_st.rx_dma_buf,
 					       GFP_KERNEL);
@@ -2054,7 +2057,7 @@ static void at91_adc_dma_init(struct at91_adc_state *st)
 	return;
 
 dma_free_area:
-	dma_free_coherent(st->dma_st.dma_chan->device->dev, pages * PAGE_SIZE,
+	dma_free_coherent(dma_dev, pages * PAGE_SIZE,
 			  st->dma_st.rx_buf, st->dma_st.rx_dma_buf);
 dma_chan_disable:
 	dma_release_channel(st->dma_st.dma_chan);
@@ -2078,7 +2081,7 @@ static void at91_adc_dma_disable(struct at91_adc_state *st)
 	/* wait for all transactions to be terminated first*/
 	dmaengine_terminate_sync(st->dma_st.dma_chan);
 
-	dma_free_coherent(st->dma_st.dma_chan->device->dev, pages * PAGE_SIZE,
+	dma_free_coherent(dmaengine_get_dma_device(st->dma_st.dma_chan), pages * PAGE_SIZE,
 			  st->dma_st.rx_buf, st->dma_st.rx_dma_buf);
 	dma_release_channel(st->dma_st.dma_chan);
 	st->dma_st.dma_chan = NULL;
@@ -2487,6 +2490,7 @@ static void at91_adc_remove(struct platform_device *pdev)
 
 	pm_runtime_disable(st->dev);
 	pm_runtime_set_suspended(st->dev);
+	pm_runtime_dont_use_autosuspend(st->dev);
 	clk_disable_unprepare(st->per_clk);
 
 	regulator_disable(st->vref);
diff --git a/drivers/iio/adc/axiado_saradc.c b/drivers/iio/adc/axiado_saradc.c
new file mode 100644
index 00000000000000..699ee31616fcdc
--- /dev/null
+++ b/drivers/iio/adc/axiado_saradc.c
@@ -0,0 +1,277 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2021-2026 Axiado Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/math.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include <linux/iio/iio.h>
+
+/* Register offsets */
+#define AX_SARADC_GLOBAL_CTRL_REG	0x0004
+#define AX_SARADC_MANUAL_CTRL_REG	0x0008
+#define AX_SARADC_DOUT_REG	0x001C
+
+/* GLOBAL_CTRL register fields */
+#define AX_SARADC_GLOBAL_CTRL_CH_EN_MASK	GENMASK(31, 16)
+#define AX_SARADC_GLOBAL_CTRL_SAMPLE_MASK	GENMASK(6, 5)
+#define AX_SARADC_GLOBAL_CTRL_MODE_MASK		GENMASK(4, 3)
+#define AX_SARADC_GLOBAL_CTRL_PD		BIT(2)
+#define AX_SARADC_GLOBAL_CTRL_ENABLE		BIT(0)
+
+/* GLOBAL_CTRL SAMPLE_MASK field value: 0 selects 16 samples. */
+#define AX_SARADC_GLOBAL_CTRL_SAMPLE_16	0
+
+/* GLOBAL_CTRL MODE_MASK field value: 1 selects manual mode. */
+#define AX_SARADC_GLOBAL_CTRL_MODE_MANUAL	1
+
+/* MANUAL_CTRL register fields */
+#define AX_SARADC_MANUAL_CTRL_ENABLE	BIT(0)
+#define AX_SARADC_MANUAL_CTRL_CH_SEL_MASK	GENMASK(4, 1)
+
+#define AX_RESOLUTION_BITS	10
+#define AX_SARADC_CONV_CYCLES	13
+#define AX_SARADC_CONV_DELAY_MARGIN_US	10
+
+struct axiado_saradc {
+	struct regmap *regmap;
+	struct mutex lock; /* Serializes ADC conversions. */
+	unsigned long clk_rate;
+	int vref_uV;
+};
+
+/*
+ * Registers contain transient control, status, and conversion data,
+ * so accesses must always go directly to hardware.
+ */
+static const struct regmap_config axiado_saradc_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = AX_SARADC_DOUT_REG,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+};
+
+static int axiado_saradc_conversion(struct axiado_saradc *info,
+				    struct iio_chan_spec const *chan, int *val)
+{
+	unsigned long usecs;
+	unsigned int regval;
+	int ret;
+
+	guard(mutex)(&info->lock);
+
+	/* Select the channel to be used and trigger conversion. */
+	ret = regmap_write(info->regmap, AX_SARADC_MANUAL_CTRL_REG,
+			   AX_SARADC_MANUAL_CTRL_ENABLE |
+			   FIELD_PREP(AX_SARADC_MANUAL_CTRL_CH_SEL_MASK, chan->channel));
+	if (ret)
+		return ret;
+
+	/* Hardware requires 13 conversion cycles at clk_rate. */
+	usecs = DIV_ROUND_UP(AX_SARADC_CONV_CYCLES * USEC_PER_SEC, info->clk_rate);
+	fsleep(usecs + AX_SARADC_CONV_DELAY_MARGIN_US);
+
+	ret = regmap_read(info->regmap, AX_SARADC_DOUT_REG, &regval);
+
+	/* Best effort to stop manual conversion. */
+	regmap_write(info->regmap, AX_SARADC_MANUAL_CTRL_REG, 0);
+
+	if (ret)
+		return ret;
+
+	*val = regval & GENMASK(AX_RESOLUTION_BITS - 1, 0);
+
+	return 0;
+}
+
+static int axiado_saradc_read_raw(struct iio_dev *indio_dev,
+				  struct iio_chan_spec const *chan,
+				  int *val, int *val2, long mask)
+{
+	struct axiado_saradc *info = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = axiado_saradc_conversion(info, chan, val);
+		if (ret)
+			return ret;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = info->vref_uV / (MICRO / MILLI);
+		*val2 = AX_RESOLUTION_BITS;
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info axiado_saradc_iio_info = {
+	.read_raw = axiado_saradc_read_raw,
+};
+
+struct axiado_saradc_soc_data {
+	const char *name;
+	unsigned int num_channels;
+};
+
+static const struct axiado_saradc_soc_data ax3000_saradc_data = {
+	.name = "ax3000_saradc",
+	.num_channels = 16,
+};
+
+static const struct axiado_saradc_soc_data ax3005_saradc_data = {
+	.name = "ax3005_saradc",
+	.num_channels = 8,
+};
+
+#define AX_SARADC_CH(_index)                                            \
+	{                                                               \
+		.type = IIO_VOLTAGE,                                    \
+		.indexed = 1,                                           \
+		.channel = (_index),                                    \
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),           \
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),   \
+		.datasheet_name = "adc" #_index,                        \
+	}
+
+static const struct iio_chan_spec axiado_saradc_iio_channels[] = {
+	AX_SARADC_CH(0),
+	AX_SARADC_CH(1),
+	AX_SARADC_CH(2),
+	AX_SARADC_CH(3),
+	AX_SARADC_CH(4),
+	AX_SARADC_CH(5),
+	AX_SARADC_CH(6),
+	AX_SARADC_CH(7),
+	AX_SARADC_CH(8),
+	AX_SARADC_CH(9),
+	AX_SARADC_CH(10),
+	AX_SARADC_CH(11),
+	AX_SARADC_CH(12),
+	AX_SARADC_CH(13),
+	AX_SARADC_CH(14),
+	AX_SARADC_CH(15),
+};
+
+static void axiado_saradc_disable(void *map)
+{
+	regmap_write(map, AX_SARADC_GLOBAL_CTRL_REG, AX_SARADC_GLOBAL_CTRL_PD);
+}
+
+static int axiado_saradc_probe(struct platform_device *pdev)
+{
+	const struct axiado_saradc_soc_data *soc_data;
+	struct device *dev = &pdev->dev;
+	struct axiado_saradc *info;
+	struct iio_dev *indio_dev;
+	void __iomem *regs;
+	struct regmap *map;
+	struct clk *clk;
+	u32 regval;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*info));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	info = iio_priv(indio_dev);
+
+	regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	map = devm_regmap_init_mmio(dev, regs, &axiado_saradc_regmap_config);
+	if (IS_ERR(map))
+		return PTR_ERR(map);
+	info->regmap = map;
+
+	clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	info->clk_rate = clk_get_rate(clk);
+	if (!info->clk_rate)
+		return dev_err_probe(dev, -EINVAL, "invalid clock rate\n");
+
+	ret = devm_regulator_get_enable_read_voltage(dev, "vref");
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to get vref voltage\n");
+
+	info->vref_uV = ret;
+
+	soc_data = device_get_match_data(dev);
+	if (!soc_data)
+		return dev_err_probe(dev, -ENODATA, "failed to get match data\n");
+
+	ret = devm_mutex_init(dev, &info->lock);
+	if (ret)
+		return ret;
+
+	regval = FIELD_PREP(AX_SARADC_GLOBAL_CTRL_CH_EN_MASK,
+			    GENMASK(soc_data->num_channels - 1, 0)) |
+		 FIELD_PREP(AX_SARADC_GLOBAL_CTRL_SAMPLE_MASK,
+			    AX_SARADC_GLOBAL_CTRL_SAMPLE_16) |
+		 FIELD_PREP(AX_SARADC_GLOBAL_CTRL_MODE_MASK,
+			    AX_SARADC_GLOBAL_CTRL_MODE_MANUAL) |
+		 AX_SARADC_GLOBAL_CTRL_ENABLE;
+
+	ret = regmap_write(map, AX_SARADC_GLOBAL_CTRL_REG, regval);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, axiado_saradc_disable, map);
+	if (ret)
+		return ret;
+
+	indio_dev->name = soc_data->name;
+	indio_dev->info = &axiado_saradc_iio_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = axiado_saradc_iio_channels;
+	indio_dev->num_channels = soc_data->num_channels;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id axiado_saradc_match[] = {
+	{
+		.compatible = "axiado,ax3000-saradc",
+		.data = &ax3000_saradc_data,
+	},
+	{
+		.compatible = "axiado,ax3005-saradc",
+		.data = &ax3005_saradc_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, axiado_saradc_match);
+
+static struct platform_driver axiado_saradc_driver = {
+	.driver = {
+		.name = "axiado-saradc",
+		.of_match_table = axiado_saradc_match,
+	},
+	.probe = axiado_saradc_probe,
+};
+module_platform_driver(axiado_saradc_driver);
+
+MODULE_AUTHOR("Axiado Corporation");
+MODULE_DESCRIPTION("Axiado SARADC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/adc/axp288_adc.c b/drivers/iio/adc/axp288_adc.c
index c8283279c47756..c98863d612ade8 100644
--- a/drivers/iio/adc/axp288_adc.c
+++ b/drivers/iio/adc/axp288_adc.c
@@ -207,6 +207,14 @@ static const struct dmi_system_id axp288_adc_ts_bias_override[] = {
 		},
 		.driver_data = (void *)(uintptr_t)AXP288_ADC_TS_BIAS_80UA,
 	},
+	{
+		/* Haier HV103H */
+		.matches = {
+		  DMI_MATCH(DMI_SYS_VENDOR, "Haier"),
+		  DMI_MATCH(DMI_PRODUCT_NAME, "HV103H"),
+		},
+		.driver_data = (void *)(uintptr_t)AXP288_ADC_TS_BIAS_80UA,
+	},
 	{ }
 };
 
diff --git a/drivers/iio/adc/bcm_iproc_adc.c b/drivers/iio/adc/bcm_iproc_adc.c
index cf4738b16e6293..ca81e0edd24d3e 100644
--- a/drivers/iio/adc/bcm_iproc_adc.c
+++ b/drivers/iio/adc/bcm_iproc_adc.c
@@ -3,14 +3,18 @@
  * Copyright 2016 Broadcom
  */
 
-#include <linux/module.h>
-#include <linux/io.h>
+#include <linux/array_size.h>
+#include <linux/bits.h>
 #include <linux/clk.h>
-#include <linux/mfd/syscon.h>
-#include <linux/regmap.h>
+#include <linux/completion.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
 
 #include <linux/iio/iio.h>
 
@@ -106,7 +110,6 @@ do { \
 
 struct iproc_adc_priv {
 	struct regmap *regmap;
-	struct clk *adc_clk;
 	struct mutex mutex;
 	int  irqno;
 	int chan_val;
@@ -448,6 +451,11 @@ static void iproc_adc_disable(struct iio_dev *indio_dev)
 	}
 }
 
+static void iproc_adc_disable_action(void *indio_dev)
+{
+	iproc_adc_disable(indio_dev);
+}
+
 static int iproc_adc_read_raw(struct iio_dev *indio_dev,
 			  struct iio_chan_spec const *chan,
 			  int *val,
@@ -506,35 +514,31 @@ static int iproc_adc_probe(struct platform_device *pdev)
 {
 	struct iproc_adc_priv *adc_priv;
 	struct iio_dev *indio_dev = NULL;
+	struct device *dev = &pdev->dev;
+	struct clk *adc_clk;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&pdev->dev,
-					sizeof(*adc_priv));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*adc_priv));
 	if (!indio_dev)
 		return -ENOMEM;
 
 	adc_priv = iio_priv(indio_dev);
-	platform_set_drvdata(pdev, indio_dev);
 
-	mutex_init(&adc_priv->mutex);
+	ret = devm_mutex_init(dev, &adc_priv->mutex);
+	if (ret)
+		return ret;
 
 	init_completion(&adc_priv->completion);
 
 	adc_priv->regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 			   "adc-syscon");
-	if (IS_ERR(adc_priv->regmap)) {
-		dev_err(&pdev->dev, "failed to get handle for tsc syscon\n");
-		ret = PTR_ERR(adc_priv->regmap);
-		return ret;
-	}
+	if (IS_ERR(adc_priv->regmap))
+		return dev_err_probe(dev, PTR_ERR(adc_priv->regmap),
+				     "failed to get handle for tsc syscon\n");
 
-	adc_priv->adc_clk = devm_clk_get(&pdev->dev, "tsc_clk");
-	if (IS_ERR(adc_priv->adc_clk)) {
-		dev_err(&pdev->dev,
-			"failed getting clock tsc_clk\n");
-		ret = PTR_ERR(adc_priv->adc_clk);
-		return ret;
-	}
+	adc_clk = devm_clk_get_enabled(dev, "tsc_clk");
+	if (IS_ERR(adc_clk))
+		return dev_err_probe(dev, PTR_ERR(adc_clk), "failed to enable clock\n");
 
 	adc_priv->irqno = platform_get_irq(pdev, 0);
 	if (adc_priv->irqno < 0)
@@ -542,32 +546,23 @@ static int iproc_adc_probe(struct platform_device *pdev)
 
 	ret = regmap_clear_bits(adc_priv->regmap, IPROC_REGCTL2,
 				IPROC_ADC_AUXIN_SCAN_ENA);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to write IPROC_REGCTL2 %d\n", ret);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to write IPROC_REGCTL2\n");
 
-	ret = devm_request_threaded_irq(&pdev->dev, adc_priv->irqno,
+	ret = devm_request_threaded_irq(dev, adc_priv->irqno,
 				iproc_adc_interrupt_handler,
 				iproc_adc_interrupt_thread,
 				IRQF_SHARED, "iproc-adc", indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "request_irq error %d\n", ret);
+	if (ret)
 		return ret;
-	}
 
-	ret = clk_prepare_enable(adc_priv->adc_clk);
-	if (ret) {
-		dev_err(&pdev->dev,
-			"clk_prepare_enable failed %d\n", ret);
+	ret = iproc_adc_enable(indio_dev);
+	if (ret)
 		return ret;
-	}
 
-	ret = iproc_adc_enable(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to enable adc %d\n", ret);
-		goto err_adc_enable;
-	}
+	ret = devm_add_action_or_reset(dev, iproc_adc_disable_action, indio_dev);
+	if (ret)
+		return ret;
 
 	indio_dev->name = "iproc-static-adc";
 	indio_dev->info = &iproc_adc_iio_info;
@@ -575,30 +570,7 @@ static int iproc_adc_probe(struct platform_device *pdev)
 	indio_dev->channels = iproc_adc_iio_channels;
 	indio_dev->num_channels = ARRAY_SIZE(iproc_adc_iio_channels);
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&pdev->dev, "iio_device_register failed:err %d\n", ret);
-		goto err_clk;
-	}
-
-	return 0;
-
-err_clk:
-	iproc_adc_disable(indio_dev);
-err_adc_enable:
-	clk_disable_unprepare(adc_priv->adc_clk);
-
-	return ret;
-}
-
-static void iproc_adc_remove(struct platform_device *pdev)
-{
-	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
-	struct iproc_adc_priv *adc_priv = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
-	iproc_adc_disable(indio_dev);
-	clk_disable_unprepare(adc_priv->adc_clk);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct of_device_id iproc_adc_of_match[] = {
@@ -609,7 +581,6 @@ MODULE_DEVICE_TABLE(of, iproc_adc_of_match);
 
 static struct platform_driver iproc_adc_driver = {
 	.probe = iproc_adc_probe,
-	.remove = iproc_adc_remove,
 	.driver = {
 		.name = "iproc-static-adc",
 		.of_match_table = iproc_adc_of_match,
diff --git a/drivers/iio/adc/ltc2497-core.c b/drivers/iio/adc/ltc2497-core.c
index 2dc5c704426949..a6e34da83b816f 100644
--- a/drivers/iio/adc/ltc2497-core.c
+++ b/drivers/iio/adc/ltc2497-core.c
@@ -9,9 +9,11 @@
 #include <linux/delay.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/driver.h>
+#include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/regulator/consumer.h>
+#include <linux/units.h>
 
 #include "ltc2497.h"
 
@@ -19,24 +21,54 @@
 #define LTC2497_DIFF			0
 #define LTC2497_SIGN			BIT(3)
 
-static int ltc2497core_wait_conv(struct ltc2497core_driverdata *ddata)
+/*
+ * Output-rate modes, indexed by ltc2497core_driverdata.speed_2x
+ * (0 = 1x, the power-on default; 1 = 2x, LTC2499 only).  The advertised
+ * sampling_frequency and the conversion-time budget are two views of the same
+ * mode, so they are kept in lock-step here.
+ */
+static const int ltc2497core_samp_freq_avail[] = {
+	6, 800000,	/* 1x: ~6.8 Hz  (1 / t_CONV_1 typ 146.9ms) */
+	13, 600000,	/* 2x: ~13.6 Hz (1 / t_CONV_2 typ  73.6ms) */
+};
+
+static const unsigned int ltc2497core_conv_time_ms_tbl[] = {
+	LTC2497_CONV_TIME_1X_MS,		/* 1x */
+	LTC2499_CONV_TIME_2X_MS,		/* 2x */
+};
+
+static unsigned int ltc2497core_conv_time_ms(struct ltc2497core_driverdata *ddata,
+					     u8 address)
+{
+	/*
+	 * SPD is ignored by the part during a temperature measurement: it
+	 * always converts at 1x, so budget the 1x time regardless of the
+	 * selected voltage-channel mode.
+	 */
+	if (address == LTC2497_TEMP_ADDR)
+		return ltc2497core_conv_time_ms_tbl[0];
+
+	return ltc2497core_conv_time_ms_tbl[ddata->speed_2x];
+}
+
+static int ltc2497core_wait_conv(struct ltc2497core_driverdata *ddata,
+				 unsigned int conv_time_ms)
 {
 	s64 time_elapsed;
 
 	time_elapsed = ktime_ms_delta(ktime_get(), ddata->time_prev);
 
-	if (time_elapsed < LTC2497_CONVERSION_TIME_MS) {
+	if (time_elapsed < conv_time_ms) {
 		/* delay if conversion time not passed
 		 * since last read or write
 		 */
-		if (msleep_interruptible(
-		    LTC2497_CONVERSION_TIME_MS - time_elapsed))
+		if (msleep_interruptible(conv_time_ms - time_elapsed))
 			return -ERESTARTSYS;
 
 		return 0;
 	}
 
-	if (time_elapsed - LTC2497_CONVERSION_TIME_MS <= 0) {
+	if (time_elapsed - conv_time_ms <= 0) {
 		/* We're in automatic mode -
 		 * so the last reading is still not outdated
 		 */
@@ -48,9 +80,18 @@ static int ltc2497core_wait_conv(struct ltc2497core_driverdata *ddata)
 
 static int ltc2497core_read(struct ltc2497core_driverdata *ddata, u8 address, int *val)
 {
+	unsigned int conv_time_ms = ltc2497core_conv_time_ms(ddata, address);
 	int ret;
 
-	ret = ltc2497core_wait_conv(ddata);
+	/*
+	 * Wait for the conversion currently in flight, whose duration was fixed
+	 * by the mode active when it was started (ddata->conv_time_prev).  This
+	 * can be longer than the freshly selected mode's time - e.g. a 1x
+	 * conversion is still running when the first 2x read arrives after a
+	 * sampling_frequency change - and reprogramming the device before it
+	 * finishes would be NACKed (-EIO).
+	 */
+	ret = ltc2497core_wait_conv(ddata, ddata->conv_time_prev);
 	if (ret < 0)
 		return ret;
 
@@ -60,7 +101,17 @@ static int ltc2497core_read(struct ltc2497core_driverdata *ddata, u8 address, in
 			return ret;
 		ddata->addr_prev = address;
 
-		if (msleep_interruptible(LTC2497_CONVERSION_TIME_MS))
+		/*
+		 * The reprogram above starts a conversion in the new mode.
+		 * Record its start time and duration before sleeping, so that if
+		 * msleep_interruptible() is interrupted the conversion state is
+		 * already consistent: the next retry then waits only the time
+		 * remaining from the real start instead of from a stale
+		 * time_prev, which would let it reprogram/read too early.
+		 */
+		ddata->time_prev = ktime_get();
+		ddata->conv_time_prev = conv_time_ms;
+		if (msleep_interruptible(conv_time_ms))
 			return -ERESTARTSYS;
 	}
 
@@ -69,6 +120,8 @@ static int ltc2497core_read(struct ltc2497core_driverdata *ddata, u8 address, in
 		return ret;
 
 	ddata->time_prev = ktime_get();
+	/* The read above auto-starts the next conversion in the current mode. */
+	ddata->conv_time_prev = conv_time_ms;
 
 	return ret;
 }
@@ -95,74 +148,209 @@ static int ltc2497core_read_raw(struct iio_dev *indio_dev,
 		if (ret < 0)
 			return ret;
 
-		*val = ret / 1000;
-		*val2 = ddata->chip_info->resolution + 1;
+		switch (chan->type) {
+		case IIO_TEMP:
+			/*
+			 * raw is normalised to 2^(resolution + 1), i.e.
+			 * raw = 2 * DATAOUT24, so the PTAT scale (datasheet
+			 * Vref / 1570 per Kelvin) doubles its denominator and,
+			 * in m°C, becomes Vref_uV / 3140000.
+			 */
+			*val = ret;
+			*val2 = 3140000;
+			return IIO_VAL_FRACTIONAL;
+		case IIO_VOLTAGE:
+			*val = ret / (MICRO / MILLI);
+			*val2 = ddata->chip_info->resolution + 1;
+			return IIO_VAL_FRACTIONAL_LOG2;
+		default:
+			return -EINVAL;
+		}
+
+	case IIO_CHAN_INFO_OFFSET:
+		switch (chan->type) {
+		case IIO_TEMP:
+			ret = regulator_get_voltage(ddata->ref);
+			if (ret < 0)
+				return ret;
+			if (ret == 0)
+				return -EINVAL;
+			/*
+			 * 0 °C == 273.15 K must map to raw + offset such that
+			 * (raw + offset) * scale == 0 m°C, i.e.
+			 *   offset = -273150 / scale
+			 *          = -273150 * 3140000 / Vref_uV
+			 * Computed in 64-bit to avoid overflow.
+			 */
+			*val = div_s64(ABSOLUTE_ZERO_MILLICELSIUS * 3140000LL, ret);
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		/*
+		 * Only advertised on the voltage channels of parts with a speed
+		 * mode; the sampling frequency is a property of the selected 1x/2x
+		 * mode, not of an individual conversion.
+		 */
+		mutex_lock(&ddata->lock);
+		*val = ltc2497core_samp_freq_avail[ddata->speed_2x * 2];
+		*val2 = ltc2497core_samp_freq_avail[ddata->speed_2x * 2 + 1];
+		mutex_unlock(&ddata->lock);
 
-		return IIO_VAL_FRACTIONAL_LOG2;
+		return IIO_VAL_INT_PLUS_MICRO;
 
 	default:
 		return -EINVAL;
 	}
 }
 
-#define LTC2497_CHAN(_chan, _addr, _ds_name) { \
+static int ltc2497core_read_avail(struct iio_dev *indio_dev,
+				  struct iio_chan_spec const *chan,
+				  const int **vals, int *type, int *length,
+				  long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*vals = ltc2497core_samp_freq_avail;
+		*type = IIO_VAL_INT_PLUS_MICRO;
+		*length = ARRAY_SIZE(ltc2497core_samp_freq_avail);
+		return IIO_AVAIL_LIST;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ltc2497core_write_raw(struct iio_dev *indio_dev,
+				 struct iio_chan_spec const *chan,
+				 int val, int val2, long mask)
+{
+	struct ltc2497core_driverdata *ddata = iio_priv(indio_dev);
+	unsigned int i;
+	bool speed_2x;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		/* Match the (val, val2) pair against the advertised rates. */
+		for (i = 0; i < ARRAY_SIZE(ltc2497core_samp_freq_avail); i += 2) {
+			if (val == ltc2497core_samp_freq_avail[i] &&
+			    val2 == ltc2497core_samp_freq_avail[i + 1])
+				break;
+		}
+		if (i == ARRAY_SIZE(ltc2497core_samp_freq_avail))
+			return -EINVAL;
+
+		speed_2x = i / 2;
+
+		mutex_lock(&ddata->lock);
+		ddata->speed_2x = speed_2x;
+		/*
+		 * The new speed only takes effect once the second command byte
+		 * is reprogrammed, so force the next read to reprogram rather
+		 * than reuse the value already latched for this address.
+		 * LTC2497_CONFIG_DEFAULT is not a valid channel/temperature
+		 * address, so it is a safe re-arm sentinel (as used at probe).
+		 *
+		 * A conversion started under the old speed may still be in
+		 * flight; its own duration (conv_time_prev), not the new mode's,
+		 * still gates the next reprogram, so the timing state is left
+		 * untouched here.
+		 */
+		ddata->addr_prev = LTC2497_CONFIG_DEFAULT;
+		mutex_unlock(&ddata->lock);
+
+		return 0;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+#define LTC2497_CHAN(_chan, _addr, _ds_name, _extra_mask) { \
 	.type = IIO_VOLTAGE, \
 	.indexed = 1, \
 	.channel = (_chan), \
 	.address = (_addr | (_chan / 2) | ((_chan & 1) ? LTC2497_SIGN : 0)), \
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
-	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | (_extra_mask), \
+	.info_mask_shared_by_type_available = (_extra_mask), \
 	.datasheet_name = (_ds_name), \
 }
 
-#define LTC2497_CHAN_DIFF(_chan, _addr) { \
+#define LTC2497_CHAN_DIFF(_chan, _addr, _extra_mask) { \
 	.type = IIO_VOLTAGE, \
 	.indexed = 1, \
 	.channel = (_chan) * 2 + ((_addr) & LTC2497_SIGN ? 1 : 0), \
 	.channel2 = (_chan) * 2 + ((_addr) & LTC2497_SIGN ? 0 : 1),\
 	.address = (_addr | _chan), \
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
-	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | (_extra_mask), \
+	.info_mask_shared_by_type_available = (_extra_mask), \
 	.differential = 1, \
 }
 
-static const struct iio_chan_spec ltc2497core_channel[] = {
-	LTC2497_CHAN(0, LTC2497_SGL, "CH0"),
-	LTC2497_CHAN(1, LTC2497_SGL, "CH1"),
-	LTC2497_CHAN(2, LTC2497_SGL, "CH2"),
-	LTC2497_CHAN(3, LTC2497_SGL, "CH3"),
-	LTC2497_CHAN(4, LTC2497_SGL, "CH4"),
-	LTC2497_CHAN(5, LTC2497_SGL, "CH5"),
-	LTC2497_CHAN(6, LTC2497_SGL, "CH6"),
-	LTC2497_CHAN(7, LTC2497_SGL, "CH7"),
-	LTC2497_CHAN(8, LTC2497_SGL, "CH8"),
-	LTC2497_CHAN(9, LTC2497_SGL, "CH9"),
-	LTC2497_CHAN(10, LTC2497_SGL, "CH10"),
-	LTC2497_CHAN(11, LTC2497_SGL, "CH11"),
-	LTC2497_CHAN(12, LTC2497_SGL, "CH12"),
-	LTC2497_CHAN(13, LTC2497_SGL, "CH13"),
-	LTC2497_CHAN(14, LTC2497_SGL, "CH14"),
-	LTC2497_CHAN(15, LTC2497_SGL, "CH15"),
-	LTC2497_CHAN_DIFF(0, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(1, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(2, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(3, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(4, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(5, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(6, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(7, LTC2497_DIFF),
-	LTC2497_CHAN_DIFF(0, LTC2497_DIFF | LTC2497_SIGN),
-	LTC2497_CHAN_DIFF(1, LTC2497_DIFF | LTC2497_SIGN),
-	LTC2497_CHAN_DIFF(2, LTC2497_DIFF | LTC2497_SIGN),
-	LTC2497_CHAN_DIFF(3, LTC2497_DIFF | LTC2497_SIGN),
-	LTC2497_CHAN_DIFF(4, LTC2497_DIFF | LTC2497_SIGN),
-	LTC2497_CHAN_DIFF(5, LTC2497_DIFF | LTC2497_SIGN),
-	LTC2497_CHAN_DIFF(6, LTC2497_DIFF | LTC2497_SIGN),
-	LTC2497_CHAN_DIFF(7, LTC2497_DIFF | LTC2497_SIGN),
-};
+/*
+ * SPD is ignored by the part during a temperature measurement, so the
+ * temperature channel carries no sampling-frequency attribute.
+ */
+#define LTC2497_TEMP_CHANNEL { \
+	.type = IIO_TEMP, \
+	.address = LTC2497_TEMP_ADDR, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
+			      BIT(IIO_CHAN_INFO_SCALE) | \
+			      BIT(IIO_CHAN_INFO_OFFSET), \
+}
+
+/*
+ * Parts with a speed mode advertise sampling_frequency on their voltage
+ * channels; the two tables are otherwise identical.
+ */
+#define LTC2497_DEFINE_CHANNELS(_name, _extra_mask)                     \
+static const struct iio_chan_spec _name[] = {                           \
+	LTC2497_CHAN(0, LTC2497_SGL, "CH0", _extra_mask),               \
+	LTC2497_CHAN(1, LTC2497_SGL, "CH1", _extra_mask),               \
+	LTC2497_CHAN(2, LTC2497_SGL, "CH2", _extra_mask),               \
+	LTC2497_CHAN(3, LTC2497_SGL, "CH3", _extra_mask),               \
+	LTC2497_CHAN(4, LTC2497_SGL, "CH4", _extra_mask),               \
+	LTC2497_CHAN(5, LTC2497_SGL, "CH5", _extra_mask),               \
+	LTC2497_CHAN(6, LTC2497_SGL, "CH6", _extra_mask),               \
+	LTC2497_CHAN(7, LTC2497_SGL, "CH7", _extra_mask),               \
+	LTC2497_CHAN(8, LTC2497_SGL, "CH8", _extra_mask),               \
+	LTC2497_CHAN(9, LTC2497_SGL, "CH9", _extra_mask),               \
+	LTC2497_CHAN(10, LTC2497_SGL, "CH10", _extra_mask),             \
+	LTC2497_CHAN(11, LTC2497_SGL, "CH11", _extra_mask),             \
+	LTC2497_CHAN(12, LTC2497_SGL, "CH12", _extra_mask),             \
+	LTC2497_CHAN(13, LTC2497_SGL, "CH13", _extra_mask),             \
+	LTC2497_CHAN(14, LTC2497_SGL, "CH14", _extra_mask),             \
+	LTC2497_CHAN(15, LTC2497_SGL, "CH15", _extra_mask),             \
+	LTC2497_CHAN_DIFF(0, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(1, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(2, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(3, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(4, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(5, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(6, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(7, LTC2497_DIFF, _extra_mask),                \
+	LTC2497_CHAN_DIFF(0, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_CHAN_DIFF(1, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_CHAN_DIFF(2, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_CHAN_DIFF(3, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_CHAN_DIFF(4, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_CHAN_DIFF(5, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_CHAN_DIFF(6, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_CHAN_DIFF(7, LTC2497_DIFF | LTC2497_SIGN, _extra_mask), \
+	LTC2497_TEMP_CHANNEL,                                           \
+}
+
+LTC2497_DEFINE_CHANNELS(ltc2497core_channel, 0);
+LTC2497_DEFINE_CHANNELS(ltc2497core_channel_samp_freq, BIT(IIO_CHAN_INFO_SAMP_FREQ));
 
 static const struct iio_info ltc2497core_info = {
 	.read_raw = ltc2497core_read_raw,
+	.read_avail = ltc2497core_read_avail,
+	.write_raw = ltc2497core_write_raw,
 };
 
 int ltc2497core_probe(struct device *dev, struct iio_dev *indio_dev)
@@ -182,8 +370,15 @@ int ltc2497core_probe(struct device *dev, struct iio_dev *indio_dev)
 
 	indio_dev->info = &ltc2497core_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = ltc2497core_channel;
-	indio_dev->num_channels = ARRAY_SIZE(ltc2497core_channel);
+	if (ddata->chip_info->has_speed_mode)
+		indio_dev->channels = ltc2497core_channel_samp_freq;
+	else
+		indio_dev->channels = ltc2497core_channel;
+	/* Only the ltc2499 has a temperature channel; it is the last entry. */
+	if (ddata->chip_info->has_temp)
+		indio_dev->num_channels = ARRAY_SIZE(ltc2497core_channel);
+	else
+		indio_dev->num_channels = ARRAY_SIZE(ltc2497core_channel) - 1;
 
 	ret = ddata->result_and_measure(ddata, LTC2497_CONFIG_DEFAULT, NULL);
 	if (ret < 0)
@@ -209,6 +404,8 @@ int ltc2497core_probe(struct device *dev, struct iio_dev *indio_dev)
 
 	ddata->addr_prev = LTC2497_CONFIG_DEFAULT;
 	ddata->time_prev = ktime_get();
+	/* Power-on default mode is 1x; a conversion is already in flight. */
+	ddata->conv_time_prev = LTC2497_CONV_TIME_1X_MS;
 
 	mutex_init(&ddata->lock);
 
diff --git a/drivers/iio/adc/ltc2497.c b/drivers/iio/adc/ltc2497.c
index c1668b5a351ec8..15458d4e21a108 100644
--- a/drivers/iio/adc/ltc2497.c
+++ b/drivers/iio/adc/ltc2497.c
@@ -84,6 +84,46 @@ static int ltc2497_result_and_measure(struct ltc2497core_driverdata *ddata,
 			return 0;
 	}
 
+	/*
+	 * Parts with a second config byte (LTC2499: internal PTAT sensor and/or
+	 * the 2x speed mode) latch their converter configuration from that byte
+	 * and only re-evaluate it when EN2 is set; a single byte, or a second
+	 * byte with EN2 = 0, means "keep previous".  A one-byte channel select
+	 * therefore cannot pull the device back out of temperature mode, so a
+	 * voltage read after a temperature read would keep returning the PTAT
+	 * result.  Always drive the second byte with EN2 set on these parts:
+	 *   - temperature read: IM = 1 (SPD is ignored by the part in
+	 *     temperature mode and is left 0 here);
+	 *   - voltage read: IM = 0 (external input), plus SPD when 2x is
+	 *     selected.
+	 * FA = FB = 0 keeps the power-on simultaneous 50/60Hz rejection, whose
+	 * worst-case conversion time the driver's wait already covers.
+	 *
+	 * The byte could be skipped while the latched configuration is already
+	 * the one wanted; it is sent on every conversion for simplicity.
+	 */
+	if (ddata->chip_info->has_temp || ddata->chip_info->has_speed_mode) {
+		u8 cmd[2];
+
+		if (address == LTC2497_TEMP_ADDR) {
+			cmd[0] = LTC2497_ENABLE | LTC2497_CONFIG_DEFAULT;
+			cmd[1] = LTC2499_EN2 | LTC2499_IM;
+		} else {
+			cmd[0] = LTC2497_ENABLE | address;
+			cmd[1] = LTC2499_EN2;
+			if (ddata->speed_2x)
+				cmd[1] |= LTC2499_SPD;
+		}
+
+		ret = i2c_master_send(st->client, cmd, sizeof(cmd));
+		if (ret < 0) {
+			dev_err(&st->client->dev, "i2c transfer failed: %pe\n",
+				ERR_PTR(ret));
+			return ret;
+		}
+		return 0;
+	}
+
 	ret = i2c_smbus_write_byte(st->client,
 				   LTC2497_ENABLE | address);
 	if (ret)
@@ -137,6 +177,8 @@ static const struct ltc2497_chip_info ltc2497_info[] = {
 	[TYPE_LTC2499] = {
 		.resolution = 24,
 		.name = "ltc2499",
+		.has_temp = true,
+		.has_speed_mode = true,
 	},
 };
 
diff --git a/drivers/iio/adc/ltc2497.h b/drivers/iio/adc/ltc2497.h
index 64e81c95a3dd05..cee9a2f50b822a 100644
--- a/drivers/iio/adc/ltc2497.h
+++ b/drivers/iio/adc/ltc2497.h
@@ -2,11 +2,35 @@
 
 #define LTC2497_ENABLE			0xA0
 #define LTC2497_CONFIG_DEFAULT		LTC2497_ENABLE
-#define LTC2497_CONVERSION_TIME_MS	150ULL
+
+/*
+ * Conversion-time bounds used to gate reads.  Each value is the datasheet
+ * t_CONV maximum, rounded UP to the next whole millisecond.
+ *
+ * The 2x mode (LTC2499_SPD, LTC2499 only) disables the offset auto-calibration
+ * to roughly double the output rate; adding the 2x wait time is what makes the
+ * SPD control actually faster.
+ */
+#define LTC2497_CONV_TIME_1X_MS		150ULL	/* ceil(t_CONV_1 simult. max 149.9) */
+#define LTC2499_CONV_TIME_2X_MS		76ULL	/* ceil(t_CONV_2 simult. max  75.1) */
+
+/*
+ * Sentinel passed as `address` to result_and_measure() to request a
+ * temperature conversion instead of a voltage channel.  Valid channel
+ * addresses fit in 5 bits (0x00–0x1F), so 0xFF is unambiguous.
+ */
+#define LTC2497_TEMP_ADDR		0xFF
+
+/* Second config-byte bits (LTC2499 / LTC2493 only) */
+#define LTC2499_EN2			BIT(7)	/* enable second config byte */
+#define LTC2499_IM			BIT(6)	/* 1 = measure internal temp sensor */
+#define LTC2499_SPD			BIT(3)	/* 1 = 2x output rate (offset cal off) */
 
 struct ltc2497_chip_info {
-	u32 resolution;
 	const char *name;
+	u32 resolution;
+	bool has_temp;
+	bool has_speed_mode;	/* SPD bit in the 2nd config byte (LTC2499/LTC2493) */
 };
 
 struct ltc2497core_driverdata {
@@ -16,6 +40,9 @@ struct ltc2497core_driverdata {
 	struct mutex lock;
 	const struct ltc2497_chip_info	*chip_info;
 	u8 addr_prev;
+	bool speed_2x;	/* SPD: false = 1x (default), true = 2x */
+	/* Conversion time (ms) of the conversion currently in flight. */
+	unsigned int conv_time_prev;
 	int (*result_and_measure)(struct ltc2497core_driverdata *ddata,
 				  u8 address, int *val);
 };
diff --git a/drivers/iio/adc/max11205.c b/drivers/iio/adc/max11205.c
index 63b6561dcf4753..53ac2f4f383e92 100644
--- a/drivers/iio/adc/max11205.c
+++ b/drivers/iio/adc/max11205.c
@@ -117,6 +117,8 @@ static int max11205_probe(struct spi_device *spi)
 	ad_sd_init(&st->sd, indio_dev, spi, &max11205_sigma_delta_info);
 
 	st->chip_info = spi_get_device_match_data(spi);
+	if (!st->chip_info)
+		return -ENODATA;
 
 	indio_dev->name = st->chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
diff --git a/drivers/iio/adc/max1363.c b/drivers/iio/adc/max1363.c
index 65a2d92bb1124c..629c53af6472ec 100644
--- a/drivers/iio/adc/max1363.c
+++ b/drivers/iio/adc/max1363.c
@@ -404,6 +404,14 @@ static int max1363_read_single_chan(struct iio_dev *indio_dev,
 
 		data = rxbuf[0];
 	}
+
+	/*
+	 * Differential channels are bipolar and the device returns the sample
+	 * in two's complement, so sign-extend it from the resolution bit.
+	 */
+	if (chan->differential)
+		data = sign_extend32(data, st->chip_info->bits - 1);
+
 	*val = data;
 
 	return 0;
diff --git a/drivers/iio/adc/max14001.c b/drivers/iio/adc/max14001.c
index 58adeac62dde4d..7694fbb6a9fd4f 100644
--- a/drivers/iio/adc/max14001.c
+++ b/drivers/iio/adc/max14001.c
@@ -366,6 +366,7 @@ static const struct spi_device_id max14001_id_table[] = {
 	{ .name = "max14002", .driver_data = (kernel_ulong_t)&max14002_chip_info },
 	{ }
 };
+MODULE_DEVICE_TABLE(spi, max14001_id_table);
 
 static const struct of_device_id max14001_of_match[] = {
 	{ .compatible = "adi,max14001", .data = &max14001_chip_info },
diff --git a/drivers/iio/adc/max40080.c b/drivers/iio/adc/max40080.c
new file mode 100644
index 00000000000000..c6946eb6ff5b78
--- /dev/null
+++ b/drivers/iio/adc/max40080.c
@@ -0,0 +1,574 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * MAX40080 Digital Current-Sense Amplifier driver
+ *
+ * Copyright 2026 Analog Devices, Inc.
+ *
+ * Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/MAX40080.pdf
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/i2c.h>
+#include <linux/iopoll.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/property.h>
+#include <linux/time.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include <asm/byteorder.h>
+
+#include <linux/iio/iio.h>
+
+#define MAX40080_REG_CFG		0x00
+#define  MAX40080_CFG_MODE_MSK		GENMASK(2, 0)
+#define  MAX40080_CFG_PEC_EN_MSK	BIT(5)
+#define  MAX40080_CFG_RANGE_MSK		BIT(6)
+#define  MAX40080_CFG_FILTER_MSK	GENMASK(14, 12)
+
+#define MAX40080_REG_FIFO_CFG		0x0A
+#define  MAX40080_FIFO_CFG_STORE_IV_MSK	GENMASK(1, 0)
+
+#define MAX40080_REG_IV			0x10
+/* Current is a 13-bit two's-complement value (magnitude + sign bit). */
+#define  MAX40080_IV_I_MSK		GENMASK(12, 0)
+#define  MAX40080_IV_I_SIGN_BIT		12
+#define  MAX40080_IV_V_MAG_MSK		GENMASK(27, 16)
+#define  MAX40080_IV_VALID_MSK		BIT(31)
+
+/* CFG.mode field values. */
+#define MAX40080_CFG_MODE_STDBY		0x00
+#define MAX40080_CFG_MODE_SINGLE	0x02
+
+/* CFG.range field values. */
+#define MAX40080_CFG_RANGE_50mV		0
+#define MAX40080_CFG_RANGE_10mV		1
+
+/* FIFO_CFG.store_iv field values. */
+#define MAX40080_FIFO_CFG_STORE_IV	0x02
+
+#define MAX40080_ADC_RES_BITS		12
+#define MAX40080_INTER_VREF_mV		1250
+#define MAX40080_V_BUFF_GAIN		30
+#define MAX40080_CSA_50mV_GAIN		25
+#define MAX40080_CSA_10mV_GAIN		125
+
+/*
+ * The RANGE field (CFG bit 6) selects one of two current-sense full-scale
+ * ranges (the MAX40080 supports exactly two: +/-50 mV and +/-10 mV). Indexed
+ * by the CFG.range field value.
+ */
+static const int max40080_csa_gain[] = {
+	[MAX40080_CFG_RANGE_50mV] = MAX40080_CSA_50mV_GAIN,
+	[MAX40080_CFG_RANGE_10mV] = MAX40080_CSA_10mV_GAIN,
+};
+
+struct max40080_state {
+	struct i2c_client *client;
+	/* Serializes read-modify-write access to the CFG register. */
+	struct mutex lock;
+	u32 shunt_resistor_uOhm;
+	/* Cached configuration: the selected RANGE index and oversampling ratio. */
+	unsigned int range;
+	int oversampling_ratio;
+	/*
+	 * Precomputed current scale (mA per code) for each RANGE setting, as
+	 * {integer, nano} pairs for IIO_VAL_INT_PLUS_NANO. The range is
+	 * selected by writing the corresponding scale.
+	 */
+	int current_scale[2][2];
+};
+
+static const int max40080_oversampling_avail[] = { 1, 8, 16, 32, 64, 128 };
+
+static int max40080_update_bits(struct max40080_state *st, u8 reg,
+				u16 mask, u16 val)
+{
+	int tmp;
+
+	tmp = i2c_smbus_read_word_data(st->client, reg);
+	if (tmp < 0)
+		return tmp;
+
+	tmp = (tmp & ~mask) | (val & mask);
+
+	return i2c_smbus_write_word_data(st->client, reg, tmp);
+}
+
+/*
+ * In single-measurement mode the device sits idle until it receives an SMBus
+ * Quick Command, then performs exactly one current and one voltage conversion
+ * and returns to idle. Triggering on demand this way (rather than running the
+ * FIFO continuously in active mode) means each read returns a fresh, coherent
+ * current/voltage pair instead of the oldest queued FIFO entry.
+ */
+static int max40080_trigger_measurement(struct max40080_state *st)
+{
+	return i2c_smbus_xfer(st->client->adapter, st->client->addr,
+			      st->client->flags, I2C_SMBUS_WRITE, 0,
+			      I2C_SMBUS_QUICK, NULL);
+}
+
+/*
+ * A single measurement holds the matched current/voltage pair in one 32-bit
+ * word (MAX40080_REG_IV). Reading all four bytes in one transaction returns
+ * both from the same conversion; reading the separate current (0x0C) and
+ * voltage (0x0E) registers would decorrelate the two channels.
+ *
+ * Unlike the word accesses used elsewhere, this is a plain I2C block read: the
+ * SMBus layer does not append or verify a PEC byte for it even when PEC is
+ * otherwise enabled for the device, so this transfer is not PEC protected.
+ */
+static int max40080_read_iv_once(struct max40080_state *st, u32 *iv)
+{
+	__le32 buf = 0;
+	int ret;
+
+	ret = i2c_smbus_read_i2c_block_data(st->client, MAX40080_REG_IV,
+					    sizeof(buf), (u8 *)&buf);
+	if (ret < 0)
+		return ret;
+
+	*iv = le32_to_cpu(buf);
+
+	return 0;
+}
+
+static int max40080_read_iv(struct max40080_state *st, u32 *iv)
+{
+	u32 tmp = 0;
+	int ret, io_ret;
+
+	guard(mutex)(&st->lock);
+
+	ret = max40080_trigger_measurement(st);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Wait for the conversion to complete by polling the FIFO valid bit
+	 * (or bail out on an I2C error). Polling the device's own status makes
+	 * this independent of the actual conversion time, which varies with the
+	 * oversampling ratio and the bus speed. The timeout is only a safety
+	 * ceiling: the worst case is the maximum 128x averaging on both the
+	 * current and voltage channels at the slowest 15 ksps base rate plus
+	 * the inter-channel switching time, i.e. roughly 20 ms; 50 ms leaves
+	 * ample margin.
+	 */
+	ret = read_poll_timeout(max40080_read_iv_once, io_ret,
+				io_ret || (tmp & MAX40080_IV_VALID_MSK),
+				500, 50 * USEC_PER_MSEC, false, st, &tmp);
+	if (io_ret)
+		return io_ret;
+
+	/*
+	 * Propagate the last-read value even on timeout so the caller can
+	 * inspect it for debugging.
+	 */
+	*iv = tmp;
+
+	return ret;
+}
+
+static int max40080_get_current(struct max40080_state *st, int *val)
+{
+	u32 iv;
+	int ret;
+
+	ret = max40080_read_iv(st, &iv);
+	if (ret)
+		return ret;
+
+	*val = sign_extend32(FIELD_GET(MAX40080_IV_I_MSK, iv),
+			     MAX40080_IV_I_SIGN_BIT);
+
+	return 0;
+}
+
+static int max40080_get_voltage(struct max40080_state *st, int *val)
+{
+	u32 iv;
+	int ret;
+
+	ret = max40080_read_iv(st, &iv);
+	if (ret)
+		return ret;
+
+	*val = FIELD_GET(MAX40080_IV_V_MAG_MSK, iv);
+
+	return 0;
+}
+
+static int max40080_set_range(struct max40080_state *st, unsigned int range)
+{
+	int ret;
+
+	ret = max40080_update_bits(st, MAX40080_REG_CFG, MAX40080_CFG_RANGE_MSK,
+				   FIELD_PREP(MAX40080_CFG_RANGE_MSK, range));
+	if (ret)
+		return ret;
+
+	WRITE_ONCE(st->range, range);
+
+	return 0;
+}
+
+/*
+ * Precompute the current scale (mA per code) for each RANGE setting as
+ * {integer, nano} pairs. The shunt drop for a full-scale code is
+ *   Vref[mV] / (BIT(ADC_RES_BITS) * gain)
+ * and current = Vshunt / Rshunt, so with Rshunt in micro-ohms the scale in
+ * mA/code is
+ *   Vref[mV] * NANO * MICRO / (BIT(ADC_RES_BITS) * gain * Rshunt[uOhm])
+ * expressed as an integer part plus a nano fractional part.
+ */
+static void max40080_calc_current_scale(struct max40080_state *st)
+{
+	u64 numerator, denominator;
+	u32 rem;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(max40080_csa_gain); i++) {
+		numerator = (u64)MAX40080_INTER_VREF_mV * NANO * MICRO;
+		denominator = BIT_ULL(MAX40080_ADC_RES_BITS) * max40080_csa_gain[i] *
+			      st->shunt_resistor_uOhm;
+		numerator = div64_u64(numerator, denominator);
+		st->current_scale[i][0] = div_u64_rem(numerator, NANO, &rem);
+		st->current_scale[i][1] = rem;
+	}
+}
+
+/*
+ * max40080_oversampling_avail[] is ordered so that its index is the FILTER
+ * field value (index 0 = no averaging, index 1 = 8x, ...). Return that index
+ * for an exact match, or -EINVAL for a value that is not on the list.
+ */
+static int max40080_oversampling_to_filter(int val)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(max40080_oversampling_avail); i++) {
+		if (max40080_oversampling_avail[i] == val)
+			return i;
+	}
+
+	return -EINVAL;
+}
+
+static int max40080_set_oversampling_ratio(struct max40080_state *st, int val)
+{
+	int filter;
+	int ret;
+
+	filter = max40080_oversampling_to_filter(val);
+	if (filter < 0)
+		return filter;
+
+	ret = max40080_update_bits(st, MAX40080_REG_CFG, MAX40080_CFG_FILTER_MSK,
+				   FIELD_PREP(MAX40080_CFG_FILTER_MSK, filter));
+	if (ret)
+		return ret;
+
+	WRITE_ONCE(st->oversampling_ratio, val);
+
+	return 0;
+}
+
+static int max40080_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
+{
+	struct max40080_state *st = iio_priv(indio_dev);
+	unsigned int range;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (chan->type == IIO_CURRENT) {
+			ret = max40080_get_current(st, val);
+			if (ret)
+				return ret;
+		} else if (chan->type == IIO_VOLTAGE) {
+			ret = max40080_get_voltage(st, val);
+			if (ret)
+				return ret;
+		}
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		if (chan->type == IIO_CURRENT) {
+			/*
+			 * The selectable current-sense range is exposed through
+			 * scale: each RANGE setting has its own precomputed
+			 * mA-per-code value. Userspace picks the range by
+			 * writing the matching scale.
+			 *
+			 * Use READ_ONCE to ensure the compiler reads st->range
+			 * exactly once, so val and val2 come from the same
+			 * setting even if a concurrent write changes st->range.
+			 */
+			range = READ_ONCE(st->range);
+			*val = st->current_scale[range][0];
+			*val2 = st->current_scale[range][1];
+			return IIO_VAL_INT_PLUS_NANO;
+		}
+		/* voltage[mV] = raw * Vref[mV] * buffer_gain / BIT(ADC_RES_BITS) */
+		*val = MAX40080_INTER_VREF_mV * MAX40080_V_BUFF_GAIN;
+		*val2 = MAX40080_ADC_RES_BITS;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*val = READ_ONCE(st->oversampling_ratio);
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int max40080_write_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long mask)
+{
+	struct max40080_state *st = iio_priv(indio_dev);
+
+	guard(mutex)(&st->lock);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		/* Only the current channel has a selectable range/scale. */
+		if (chan->type != IIO_CURRENT)
+			return -EINVAL;
+
+		for (unsigned int i = 0; i < ARRAY_SIZE(max40080_csa_gain); i++) {
+			if (val == st->current_scale[i][0] &&
+			    val2 == st->current_scale[i][1])
+				return max40080_set_range(st, i);
+		}
+
+		return -EINVAL;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		return max40080_set_oversampling_ratio(st, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int max40080_write_raw_get_fmt(struct iio_dev *indio_dev,
+				      struct iio_chan_spec const *chan,
+				      long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return IIO_VAL_INT;
+	}
+}
+
+static int max40080_read_avail(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       const int **vals, int *type, int *length,
+			       long info)
+{
+	struct max40080_state *st = iio_priv(indio_dev);
+
+	switch (info) {
+	case IIO_CHAN_INFO_SCALE:
+		if (chan->type != IIO_CURRENT)
+			return -EINVAL;
+
+		*vals = (int *)st->current_scale;
+		*length = ARRAY_SIZE(max40080_csa_gain) * 2;
+		*type = IIO_VAL_INT_PLUS_NANO;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*vals = max40080_oversampling_avail;
+		*length = ARRAY_SIZE(max40080_oversampling_avail);
+		*type = IIO_VAL_INT;
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int max40080_reg_access(struct iio_dev *indio_dev, unsigned int reg,
+			       unsigned int write_val, unsigned int *read_val)
+{
+	struct max40080_state *st = iio_priv(indio_dev);
+	int val;
+
+	if (read_val) {
+		val = i2c_smbus_read_word_data(st->client, reg);
+		if (val < 0)
+			return val;
+
+		*read_val = val;
+
+		return 0;
+	}
+
+	return i2c_smbus_write_word_data(st->client, reg, write_val);
+}
+
+static const struct iio_info max40080_info = {
+	.read_raw = max40080_read_raw,
+	.write_raw = max40080_write_raw,
+	.write_raw_get_fmt = max40080_write_raw_get_fmt,
+	.read_avail = max40080_read_avail,
+	.debugfs_reg_access = &max40080_reg_access,
+};
+
+static const struct iio_chan_spec max40080_channels[] = {
+	{
+		.type = IIO_CURRENT,
+		.indexed = 1,
+		.channel = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_shared_by_all_available =
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+	},
+	{
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.channel = 0,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_shared_by_all_available =
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+	},
+};
+
+/*
+ * Configure the device from the cached state. The device powers up in standby
+ * with PEC enabled (CFG POR = 0x0060), so PEC is kept enabled throughout.
+ */
+static int max40080_init(struct max40080_state *st)
+{
+	u16 fifo_cfg, cfg;
+	int ret, filter;
+
+	filter = max40080_oversampling_to_filter(st->oversampling_ratio);
+	if (filter < 0)
+		return filter;
+
+	/*
+	 * Put the device in standby before (re)configuring the FIFO: the FIFO
+	 * configuration register can only be written while the device is not
+	 * converting.
+	 */
+	cfg = FIELD_PREP(MAX40080_CFG_MODE_MSK, MAX40080_CFG_MODE_STDBY) |
+	      FIELD_PREP(MAX40080_CFG_PEC_EN_MSK, 1);
+	ret = i2c_smbus_write_word_data(st->client, MAX40080_REG_CFG, cfg);
+	if (ret)
+		return ret;
+
+	/* Store a matched current+voltage pair per conversion. */
+	fifo_cfg = FIELD_PREP(MAX40080_FIFO_CFG_STORE_IV_MSK, MAX40080_FIFO_CFG_STORE_IV);
+	ret = i2c_smbus_write_word_data(st->client, MAX40080_REG_FIFO_CFG,
+					fifo_cfg);
+	if (ret)
+		return ret;
+
+	/*
+	 * Use single-measurement mode: the device stays idle and converts once
+	 * per SMBus Quick Command (see max40080_trigger_measurement()), so each
+	 * read returns a fresh sample rather than a queued FIFO entry.
+	 */
+	cfg = FIELD_PREP(MAX40080_CFG_MODE_MSK, MAX40080_CFG_MODE_SINGLE) |
+	      FIELD_PREP(MAX40080_CFG_PEC_EN_MSK, 1) |
+	      FIELD_PREP(MAX40080_CFG_RANGE_MSK, st->range) |
+	      FIELD_PREP(MAX40080_CFG_FILTER_MSK, filter);
+
+	return i2c_smbus_write_word_data(st->client, MAX40080_REG_CFG, cfg);
+}
+
+static int max40080_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	const char *propname;
+	struct iio_dev *indio_dev;
+	struct max40080_state *st;
+	int ret;
+
+	/*
+	 * The device powers up with PEC enabled (CFG POR = 0x0060) and rejects
+	 * unprotected transactions, so PEC support is mandatory, along with
+	 * word access, the I2C block read used for the current/voltage pair,
+	 * and the Quick Command used to trigger a conversion.
+	 */
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_WORD_DATA |
+				     I2C_FUNC_SMBUS_I2C_BLOCK |
+				     I2C_FUNC_SMBUS_QUICK |
+				     I2C_FUNC_SMBUS_PEC))
+		return -EOPNOTSUPP;
+
+	client->flags |= I2C_CLIENT_PEC;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	st->client = client;
+
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
+	propname = "shunt-resistor-micro-ohms";
+	ret = device_property_read_u32(dev, propname, &st->shunt_resistor_uOhm);
+	if (ret)
+		return dev_err_probe(dev, ret, "can't read %s\n", propname);
+	if (!st->shunt_resistor_uOhm)
+		return dev_err_probe(dev, -EINVAL, "%s must be non-zero\n", propname);
+
+	max40080_calc_current_scale(st);
+
+	/* Defaults: 50 mV range, no averaging. */
+	st->range = MAX40080_CFG_RANGE_50mV;
+	st->oversampling_ratio = 1;
+
+	indio_dev->name = "max40080";
+	indio_dev->info = &max40080_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = max40080_channels;
+	indio_dev->num_channels = ARRAY_SIZE(max40080_channels);
+
+	ret = max40080_init(st);
+	if (ret)
+		return ret;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct i2c_device_id max40080_i2c_ids[] = {
+	{ .name = "max40080" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, max40080_i2c_ids);
+
+static const struct of_device_id max40080_of_match[] = {
+	{ .compatible = "adi,max40080" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, max40080_of_match);
+
+static struct i2c_driver max40080_driver = {
+	.driver = {
+		.name = "max40080",
+		.of_match_table = max40080_of_match,
+	},
+	.probe = max40080_probe,
+	.id_table = max40080_i2c_ids,
+};
+module_i2c_driver(max40080_driver);
+
+MODULE_AUTHOR("Ciprian Hegbeli <ciprian.hegbeli@analog.com>");
+MODULE_AUTHOR("Stefan Popa <stefan.popa@analog.com>");
+MODULE_DESCRIPTION("Analog Devices MAX40080 current-sense amplifier driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/adc/mcp3911.c b/drivers/iio/adc/mcp3911.c
index 797424f59f74b7..90167930f91322 100644
--- a/drivers/iio/adc/mcp3911.c
+++ b/drivers/iio/adc/mcp3911.c
@@ -721,6 +721,8 @@ static int mcp3911_probe(struct spi_device *spi)
 	adc = iio_priv(indio_dev);
 	adc->spi = spi;
 	adc->chip = spi_get_device_match_data(spi);
+	if (!adc->chip)
+		return -ENODATA;
 
 	ret = devm_regulator_get_enable_read_voltage(dev, "vref");
 	if (ret < 0 && ret != -ENODEV)
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index 000e39ca5c624e..2b003fd5afebaa 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -739,7 +739,7 @@ static int meson_sar_adc_clk_init(struct iio_dev *indio_dev,
 {
 	struct meson_sar_adc_priv *priv = iio_priv(indio_dev);
 	struct device *dev = indio_dev->dev.parent;
-	struct clk_init_data init;
+	struct clk_init_data init = { };
 	const char *clk_parents[1];
 
 	init.name = devm_kasprintf(dev, GFP_KERNEL, "%s#adc_div", dev_name(dev));
diff --git a/drivers/iio/adc/nct7201.c b/drivers/iio/adc/nct7201.c
index bea88a9b944029..cc4d758af12fdf 100644
--- a/drivers/iio/adc/nct7201.c
+++ b/drivers/iio/adc/nct7201.c
@@ -12,6 +12,7 @@
 #include <linux/dev_printk.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
 #include <linux/time.h>
diff --git a/drivers/iio/adc/nxp-sar-adc.c b/drivers/iio/adc/nxp-sar-adc.c
index 894d3211b28382..2f716a2f97b803 100644
--- a/drivers/iio/adc/nxp-sar-adc.c
+++ b/drivers/iio/adc/nxp-sar-adc.c
@@ -604,7 +604,7 @@ static void nxp_sar_adc_dma_cb(void *data)
 
 	dma_buf = &info->dma_buf;
 	dma_samples = (u32 *)dma_buf->buf;
-	dev_dma = info->dma_chan->device->dev;
+	dev_dma = dmaengine_get_dma_device(info->dma_chan);
 
 	/*
 	 * DMA in some corner cases might have already be charged for
diff --git a/drivers/iio/adc/pac1934.c b/drivers/iio/adc/pac1934.c
index 23055405a6e0cb..5e901ab25c8e8b 100644
--- a/drivers/iio/adc/pac1934.c
+++ b/drivers/iio/adc/pac1934.c
@@ -19,6 +19,7 @@
 #include <linux/i2c.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/kstrtox.h>
 #include <linux/unaligned.h>
 
 /*
@@ -494,11 +495,13 @@ static ssize_t pac1934_shunt_value_store(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct pac1934_chip_info *info = iio_priv(indio_dev);
 	struct iio_dev_attr *this_attr = to_iio_dev_attr(attr);
-	int sh_val;
+	unsigned int sh_val;
+	int ret;
 
-	if (kstrtouint(buf, 10, &sh_val)) {
+	ret = kstrtouint(buf, 10, &sh_val);
+	if (ret) {
 		dev_err(dev, "Shunt value is not valid\n");
-		return -EINVAL;
+		return ret;
 	}
 
 	scoped_guard(mutex, &info->lock)
@@ -1108,6 +1111,10 @@ static int pac1934_acpi_parse_channel_config(struct i2c_client *client,
 			devm_kmemdup(dev, rez->package.elements[i].string.pointer,
 				     (size_t)rez->package.elements[i].string.length + 1,
 				     GFP_KERNEL);
+		if (!info->labels[idx]) {
+			ACPI_FREE(rez);
+			return -ENOMEM;
+		}
 		info->labels[idx][rez->package.elements[i].string.length] = '\0';
 		info->shunts[idx] = rez->package.elements[i + 1].integer.value * 1000;
 		info->active_channels[idx] = (info->shunts[idx] != 0);
diff --git a/drivers/iio/adc/rockchip_saradc.c b/drivers/iio/adc/rockchip_saradc.c
index 0f0bf2906af038..5c45fae1189015 100644
--- a/drivers/iio/adc/rockchip_saradc.c
+++ b/drivers/iio/adc/rockchip_saradc.c
@@ -348,6 +348,19 @@ static const struct rockchip_saradc_data rk3588_saradc_data = {
 	.read = rockchip_saradc_read_v2,
 };
 
+static const struct iio_chan_spec rockchip_rv1106_saradc_iio_channels[] = {
+	SARADC_CHANNEL(0, "adc0", 10),
+	SARADC_CHANNEL(1, "adc1", 10),
+};
+
+static const struct rockchip_saradc_data rv1106_saradc_data = {
+	.channels = rockchip_rv1106_saradc_iio_channels,
+	.num_channels = ARRAY_SIZE(rockchip_rv1106_saradc_iio_channels),
+	.clk_rate = 1000000,
+	.start = rockchip_saradc_start_v2,
+	.read = rockchip_saradc_read_v2,
+};
+
 static const struct of_device_id rockchip_saradc_match[] = {
 	{
 		.compatible = "rockchip,saradc",
@@ -370,6 +383,9 @@ static const struct of_device_id rockchip_saradc_match[] = {
 	}, {
 		.compatible = "rockchip,rk3588-saradc",
 		.data = &rk3588_saradc_data,
+	}, {
+		.compatible = "rockchip,rv1106-saradc",
+		.data = &rv1106_saradc_data,
 	},
 	{ }
 };
diff --git a/drivers/iio/adc/rohm-bd79124.c b/drivers/iio/adc/rohm-bd79124.c
index ed542728896199..8091a138cca855 100644
--- a/drivers/iio/adc/rohm-bd79124.c
+++ b/drivers/iio/adc/rohm-bd79124.c
@@ -200,7 +200,7 @@ static int bd79124gpo_set_multiple(struct gpio_chip *gc, unsigned long *mask,
 	if (ret)
 		return ret;
 
-	if (all_gpos ^ *mask) {
+	if (*mask & ~all_gpos) {
 		dev_dbg(data->dev, "Invalid mux config. Can't set value.\n");
 
 		return -EINVAL;
@@ -381,6 +381,9 @@ static int bd79124_start_measurement(struct bd79124_data *data, int chan)
 
 	/* See if already started */
 	ret = regmap_read(data->map, BD79124_REG_AUTO_CHANNELS, &val);
+	if (ret)
+		return ret;
+
 	if (val & BIT(chan))
 		return 0;
 
@@ -420,11 +423,16 @@ static int bd79124_stop_measurement(struct bd79124_data *data, int chan)
 
 	/* See if already stopped */
 	ret = regmap_read(data->map, BD79124_REG_AUTO_CHANNELS, &enabled_chans);
+	if (ret)
+		return ret;
+
 	if (!(enabled_chans & BIT(chan)))
 		return 0;
 
 	ret = regmap_clear_bits(data->map, BD79124_REG_SEQ_CFG,
 				BD79124_MSK_SEQ_START);
+	if (ret)
+		return ret;
 
 	/* Clear the channel from the measured channels */
 	enabled_chans &= ~BIT(chan);
@@ -524,7 +532,7 @@ static int bd79124_enable_event(struct bd79124_data *data,
 		return ret;
 
 	if (dir == IIO_EV_DIR_RISING) {
-		limit = &data->alarm_f_limit[channel];
+		limit = &data->alarm_r_limit[channel];
 		reg = BD79124_GET_HIGH_LIMIT_REG(channel);
 	} else {
 		limit = &data->alarm_f_limit[channel];
@@ -919,13 +927,13 @@ static int bd79124_chan_init(struct bd79124_data *data, int channel)
 {
 	int ret;
 
-	ret = regmap_write(data->map, BD79124_GET_HIGH_LIMIT_REG(channel),
-			   BD79124_HIGH_LIMIT_MAX);
+	ret = bd79124_write_int_to_reg(data, BD79124_GET_HIGH_LIMIT_REG(channel),
+				       BD79124_HIGH_LIMIT_MAX);
 	if (ret)
 		return ret;
 
-	return regmap_write(data->map, BD79124_GET_LOW_LIMIT_REG(channel),
-			    BD79124_LOW_LIMIT_MIN);
+	return bd79124_write_int_to_reg(data, BD79124_GET_LOW_LIMIT_REG(channel),
+					BD79124_LOW_LIMIT_MIN);
 }
 
 static int bd79124_get_gpio_pins(const struct iio_chan_spec *cs, int num_channels)
diff --git a/drivers/iio/adc/rzg2l_adc.c b/drivers/iio/adc/rzg2l_adc.c
index 408fbf8c29cc97..3b95f47445a8b9 100644
--- a/drivers/iio/adc/rzg2l_adc.c
+++ b/drivers/iio/adc/rzg2l_adc.c
@@ -62,7 +62,7 @@
  * @adsmp_mask: ADC sampling period mask (see ADM3 register)
  * @adint_inten_mask: conversion end interrupt mask (see ADINT register)
  * @default_adcmp: default ADC cmp (see ADM3 register)
- * @num_channels: number of supported channels
+ * @max_channels: maximum number of channels
  * @adivc: specifies if ADVIC register is available
  */
 struct rzg2l_adc_hw_params {
@@ -70,7 +70,7 @@ struct rzg2l_adc_hw_params {
 	u16 adsmp_mask;
 	u16 adint_inten_mask;
 	u8 default_adcmp;
-	u8 num_channels;
+	u8 max_channels;
 	bool adivc;
 };
 
@@ -196,7 +196,7 @@ static int rzg2l_adc_conversion_setup(struct rzg2l_adc *adc, u8 ch)
 
 	/* Select analog input channel subjected to conversion. */
 	reg = rzg2l_adc_readl(adc, RZG2L_ADM(2));
-	reg &= ~GENMASK(hw_params->num_channels - 1, 0);
+	reg &= ~GENMASK(hw_params->max_channels - 1, 0);
 	reg |= BIT(ch);
 	rzg2l_adc_writel(adc, RZG2L_ADM(2), reg);
 
@@ -307,11 +307,11 @@ static irqreturn_t rzg2l_adc_isr(int irq, void *dev_id)
 		return IRQ_HANDLED;
 	}
 
-	intst = reg & GENMASK(hw_params->num_channels - 1, 0);
+	intst = reg & GENMASK(hw_params->max_channels - 1, 0);
 	if (!intst)
 		return IRQ_NONE;
 
-	for_each_set_bit(ch, &intst, hw_params->num_channels)
+	for_each_set_bit(ch, &intst, hw_params->max_channels)
 		adc->last_val[ch] = rzg2l_adc_readl(adc, RZG2L_ADCR(ch)) & RZG2L_ADCR_AD_MASK;
 
 	/* clear the channel interrupt */
@@ -341,12 +341,12 @@ static int rzg2l_adc_parse_properties(struct platform_device *pdev, struct rzg2l
 
 	num_channels = devm_iio_adc_device_alloc_chaninfo_se(&pdev->dev,
 						&rzg2l_adc_chan_template,
-						hw_params->num_channels - 1,
+						hw_params->max_channels - 1,
 						&chan_array);
 	if (num_channels < 0)
 		return num_channels;
 
-	if (num_channels > hw_params->num_channels)
+	if (num_channels > hw_params->max_channels)
 		return dev_err_probe(&pdev->dev, -EINVAL,
 				     "num of channel children out of range\n");
 
@@ -429,7 +429,7 @@ static int rzg2l_adc_probe(struct platform_device *pdev)
 	adc = iio_priv(indio_dev);
 
 	adc->hw_params = device_get_match_data(dev);
-	if (!adc->hw_params || adc->hw_params->num_channels > RZG2L_ADC_MAX_CHANNELS)
+	if (!adc->hw_params || adc->hw_params->max_channels > RZG2L_ADC_MAX_CHANNELS)
 		return -EINVAL;
 
 	ret = rzg2l_adc_parse_properties(pdev, adc);
@@ -484,7 +484,7 @@ static int rzg2l_adc_probe(struct platform_device *pdev)
 }
 
 static const struct rzg2l_adc_hw_params rzg2l_hw_params = {
-	.num_channels = 8,
+	.max_channels = 8,
 	.default_adcmp = 0xe,
 	.default_adsmp = { 0x578 },
 	.adsmp_mask = GENMASK(15, 0),
@@ -493,15 +493,24 @@ static const struct rzg2l_adc_hw_params rzg2l_hw_params = {
 };
 
 static const struct rzg2l_adc_hw_params rzg3s_hw_params = {
-	.num_channels = 9,
+	.max_channels = 9,
 	.default_adcmp = 0x1d,
 	.default_adsmp = { 0x7f, 0xff },
 	.adsmp_mask = GENMASK(7, 0),
 	.adint_inten_mask = GENMASK(11, 0),
 };
 
+static const struct rzg2l_adc_hw_params rzg3l_adc1_hw_params = {
+	.max_channels = 9,
+	.default_adcmp = 0x1d,
+	.default_adsmp = { 0x7f, 0xff },
+	.adsmp_mask = GENMASK(7, 0),
+	.adint_inten_mask = BIT(8),
+};
+
 static const struct of_device_id rzg2l_adc_match[] = {
 	{ .compatible = "renesas,r9a08g045-adc", .data = &rzg3s_hw_params },
+	{ .compatible = "renesas,r9a08g046-adc1", .data = &rzg3l_adc1_hw_params },
 	{ .compatible = "renesas,rzg2l-adc", .data = &rzg2l_hw_params },
 	{ }
 };
diff --git a/drivers/iio/adc/rzt2h_adc.c b/drivers/iio/adc/rzt2h_adc.c
index 4e0eb02d3d146e..8ba7b0fde8a19b 100644
--- a/drivers/iio/adc/rzt2h_adc.c
+++ b/drivers/iio/adc/rzt2h_adc.c
@@ -2,13 +2,19 @@
 
 #include <linux/bitfield.h>
 #include <linux/cleanup.h>
+#include <linux/clk.h>
 #include <linux/completion.h>
 #include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/iio/adc-helpers.h>
+#include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
+#include <linux/iio/kfifo_buf.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
+#include <linux/kthread.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -18,6 +24,7 @@
 #define RZT2H_ADCSR_ADIE_MASK		BIT(12)
 #define RZT2H_ADCSR_ADCS_MASK		GENMASK(14, 13)
 #define RZT2H_ADCSR_ADCS_SINGLE		0b00
+#define RZT2H_ADCSR_ADCS_CONTINUOUS	0b10
 #define RZT2H_ADCSR_ADST_MASK		BIT(15)
 
 #define RZT2H_ADANSA0_REG		0x04
@@ -25,24 +32,71 @@
 
 #define RZT2H_ADDR_REG(x)		(0x20 + 0x2 * (x))
 
+#define RZT2H_ADSSTRn(n)		(0xe0 + 0x1 * (n))
+
+/*
+ * Each A/D channel conversion takes a fixed base of 13 ADCLK cycles plus the
+ * sample time programmed in ADSSTRn, giving a conversion rate of
+ * ADCLK / (13 + ADSSTRn).
+ */
+#define RZT2H_ADC_CONV_CYCLES_BASE	13
+#define RZT2H_ADC_SST_MIN		0x7
+#define RZT2H_ADC_SST_MAX		0xff
+#define RZT2H_ADC_SST_DEFAULT		0xb
+
 #define RZT2H_ADCALCTL_REG		0x1f0
 #define RZT2H_ADCALCTL_CAL_MASK		BIT(0)
 #define RZT2H_ADCALCTL_CAL_RDY_MASK	BIT(1)
 #define RZT2H_ADCALCTL_CAL_ERR_MASK	BIT(2)
 
 #define RZT2H_ADC_MAX_CHANNELS		16
+#define RZT2H_ADC_CHANNEL_BYTES		sizeof(u16)
+#define RZT2H_ADC_DMA_PERIOD_SAMPLES	128
+#define RZT2H_ADC_DMA_PERIODS		64
+#define RZT2H_ADC_DMA_BUFFER_SAMPLES	(RZT2H_ADC_DMA_PERIODS * \
+					 RZT2H_ADC_DMA_PERIOD_SAMPLES)
+#define RZT2H_ADC_DMA_BUFFER_SIZE	(RZT2H_ADC_DMA_BUFFER_SAMPLES * \
+					 RZT2H_ADC_MAX_CHANNELS * \
+					 RZT2H_ADC_CHANNEL_BYTES)
+
+struct rzt2h_adc_dma {
+	struct dma_chan *chan;
+	u16 *buf;
+	dma_addr_t addr;
+
+	unsigned int period_index;
+	unsigned int period_bytes;
+	unsigned int first_chan;
+	unsigned int sample_chans;
+
+	u8 gather[RZT2H_ADC_MAX_CHANNELS];
+	unsigned int gather_len;
+
+	atomic_t pending_periods;
+
+	wait_queue_head_t wq;
+	struct task_struct *thread;
+};
 
 struct rzt2h_adc {
 	void __iomem *base;
 	struct device *dev;
 
+	phys_addr_t phys_base;
+	struct rzt2h_adc_dma dma;
 	struct completion completion;
 	/* lock to protect against multiple access to the device */
 	struct mutex lock;
 
 	const struct iio_chan_spec *channels;
 	unsigned int num_channels;
-	unsigned int max_channels;
+	u16 buf[RZT2H_ADC_MAX_CHANNELS];
+
+	unsigned long adclk_rate;
+	int samp_freq_avail[3];
+	u8 sst[RZT2H_ADC_MAX_CHANNELS];
+
+	int irq;
 };
 
 static void rzt2h_adc_start(struct rzt2h_adc *adc, unsigned int conversion_type)
@@ -72,6 +126,11 @@ static void rzt2h_adc_stop(struct rzt2h_adc *adc)
 	writew(reg, adc->base + RZT2H_ADCSR_REG);
 }
 
+static void rzt2h_adc_set_sst(struct rzt2h_adc *adc, unsigned int ch, u8 sst)
+{
+	writeb(sst, adc->base + RZT2H_ADSSTRn(ch));
+}
+
 static int rzt2h_adc_read_single(struct rzt2h_adc *adc, unsigned int ch, int *val)
 {
 	int ret;
@@ -87,13 +146,18 @@ static int rzt2h_adc_read_single(struct rzt2h_adc *adc, unsigned int ch, int *va
 	/* Enable a single channel */
 	writew(RZT2H_ADANSA0_CH_MASK(ch), adc->base + RZT2H_ADANSA0_REG);
 
+	rzt2h_adc_set_sst(adc, ch, adc->sst[ch]);
+
 	rzt2h_adc_start(adc, RZT2H_ADCSR_ADCS_SINGLE);
 
 	/*
-	 * Datasheet Page 2770, Table 41.1:
-	 * 0.32us per channel when sample-and-hold circuits are not in use.
+	 * Conversion can take up to ~4.3us at the maximum configurable ADSSTRn
+	 * (ADSSTRn + 13 cycles at 62.5 MHz), which rounds up to 1 jiffy. A bare
+	 * 1-jiffy timeout can expire almost immediately if it's armed right
+	 * before a tick, so add one more jiffy to guarantee the conversion time
+	 * actually elapses.
 	 */
-	ret = wait_for_completion_timeout(&adc->completion, usecs_to_jiffies(1));
+	ret = wait_for_completion_timeout(&adc->completion, usecs_to_jiffies(5) + 1);
 	if (!ret) {
 		ret = -ETIMEDOUT;
 		goto disable;
@@ -149,6 +213,281 @@ static int rzt2h_adc_calibrate(struct rzt2h_adc *adc)
 	return 0;
 }
 
+static void rzt2h_adc_push_period(struct iio_dev *indio_dev, u16 *period,
+				  dma_addr_t addr)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+	u16 *dst = adc->buf;
+	u16 *src = period;
+
+	dma_sync_single_for_cpu(adc->dev, addr, adc->dma.period_bytes,
+				DMA_FROM_DEVICE);
+
+	for (unsigned int sample = 0; sample < RZT2H_ADC_DMA_PERIOD_SAMPLES; sample++) {
+		for (unsigned int i = 0; i < adc->dma.gather_len; i++)
+			dst[i] = src[adc->dma.gather[i]];
+
+		src += adc->dma.sample_chans;
+
+		iio_push_to_buffers(indio_dev, adc->buf);
+	}
+}
+
+static void rzt2h_adc_advance_period_index(struct rzt2h_adc *adc, unsigned int i)
+{
+	adc->dma.period_index += i;
+	adc->dma.period_index %= RZT2H_ADC_DMA_PERIODS;
+}
+
+static void rzt2h_adc_dma_thread_loop(struct iio_dev *indio_dev)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+	int pending, drop;
+	dma_addr_t addr;
+	u16 *period;
+
+	pending = atomic_xchg(&adc->dma.pending_periods, 0);
+
+	if (pending >= RZT2H_ADC_DMA_PERIODS) {
+		/*
+		 * The consumer fell a full buffer behind and the oldest periods
+		 * have already been overwritten by the cyclic DMA transfer.
+		 * Drop them and jump straight to the oldest period that has
+		 * not been overwritten.
+		 */
+		drop = pending - RZT2H_ADC_DMA_PERIODS + 1;
+
+		rzt2h_adc_advance_period_index(adc, drop);
+		pending -= drop;
+	}
+
+	for (unsigned int i = 0; i < pending; i++) {
+		unsigned int backlog = atomic_read(&adc->dma.pending_periods) +
+				       pending - i;
+
+		/*
+		 * Bail if enough new periods have completed since reading the
+		 * pending_periods that the next period about to be read is at
+		 * risk of being overwritten.
+		 */
+		if (backlog >= RZT2H_ADC_DMA_PERIODS) {
+			rzt2h_adc_advance_period_index(adc, pending - i);
+			break;
+		}
+
+		period = adc->dma.buf + adc->dma.period_index *
+			 RZT2H_ADC_DMA_PERIOD_SAMPLES * adc->dma.sample_chans;
+		addr = adc->dma.addr + adc->dma.period_index *
+		       adc->dma.period_bytes;
+
+		rzt2h_adc_push_period(indio_dev, period, addr);
+		rzt2h_adc_advance_period_index(adc, 1);
+	}
+}
+
+static int rzt2h_adc_dma_thread(void *data)
+{
+	struct iio_dev *indio_dev = data;
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+
+	while (!kthread_should_stop()) {
+		wait_event_interruptible(adc->dma.wq,
+					 atomic_read(&adc->dma.pending_periods) ||
+					 kthread_should_stop());
+
+		if (kthread_should_stop())
+			break;
+
+		rzt2h_adc_dma_thread_loop(indio_dev);
+	}
+
+	return 0;
+}
+
+static void rzt2h_adc_dma_callback(void *data)
+{
+	struct iio_dev *indio_dev = data;
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+
+	atomic_inc(&adc->dma.pending_periods);
+	wake_up(&adc->dma.wq);
+}
+
+static void rzt2h_adc_dma_calc_layout(struct iio_dev *indio_dev)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+	unsigned int hi = 0, lo = RZT2H_ADC_MAX_CHANNELS - 1;
+	const struct iio_chan_spec *chan;
+	unsigned int sample_chans;
+	unsigned int first_chan;
+	unsigned int scan_index;
+	unsigned int swap;
+	unsigned int idx;
+
+	/* Find the lowest and highest enabled channel. */
+	iio_for_each_active_channel(indio_dev, scan_index) {
+		chan = &indio_dev->channels[scan_index];
+
+		lo = min_t(unsigned int, lo, chan->channel);
+		hi = max_t(unsigned int, hi, chan->channel);
+	}
+
+	/*
+	 * The DMA has no scatter-gather and transfers must have a power-of-two
+	 * width, so pick the smallest power-of-two-aligned block of channels
+	 * that covers all enabled channels.
+	 */
+	for (sample_chans = 1; sample_chans < RZT2H_ADC_MAX_CHANNELS; sample_chans <<= 1) {
+		first_chan = round_down(lo, sample_chans);
+
+		if (first_chan + sample_chans > hi)
+			break;
+	}
+
+	/*
+	 * Build a table to map each enabled channel to its position in the
+	 * transferred block, it will be used later to extract only the enabled
+	 * channels out of it.
+	 * The DMA moves data in 32-bit words, which swaps each pair of adjacent
+	 * 16-bit channels. Undo it.
+	 */
+	adc->dma.gather_len = 0;
+	swap = sample_chans > 1;
+	iio_for_each_active_channel(indio_dev, scan_index) {
+		chan = &indio_dev->channels[scan_index];
+		idx = chan->channel - first_chan;
+
+		adc->dma.gather[adc->dma.gather_len++] = idx ^ swap;
+	}
+
+	adc->dma.first_chan = first_chan;
+	adc->dma.sample_chans = sample_chans;
+	adc->dma.period_bytes = RZT2H_ADC_DMA_PERIOD_SAMPLES * sample_chans *
+				RZT2H_ADC_CHANNEL_BYTES;
+}
+
+static int rzt2h_adc_start_dma(struct iio_dev *indio_dev)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+	struct dma_async_tx_descriptor *desc;
+	struct dma_slave_config config;
+	unsigned int buffer_bytes;
+	dma_cookie_t cookie;
+	int ret;
+
+	rzt2h_adc_dma_calc_layout(indio_dev);
+
+	config = (struct dma_slave_config) {
+		.src_addr = adc->phys_base + RZT2H_ADDR_REG(adc->dma.first_chan),
+		.src_addr_width = adc->dma.sample_chans * RZT2H_ADC_CHANNEL_BYTES,
+	};
+
+	buffer_bytes = RZT2H_ADC_DMA_PERIODS * adc->dma.period_bytes;
+
+	ret = dmaengine_slave_config(adc->dma.chan, &config);
+	if (ret)
+		return ret;
+
+	desc = dmaengine_prep_dma_cyclic(adc->dma.chan, adc->dma.addr,
+					 buffer_bytes, adc->dma.period_bytes,
+					 DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT);
+	if (!desc)
+		return -EBUSY;
+
+	desc->callback = rzt2h_adc_dma_callback;
+	desc->callback_param = indio_dev;
+
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret) {
+		dmaengine_terminate_sync(adc->dma.chan);
+		return ret;
+	}
+
+	adc->dma.thread = kthread_run(rzt2h_adc_dma_thread, indio_dev,
+				      "rzt2h-adc-dma");
+	if (IS_ERR(adc->dma.thread)) {
+		dmaengine_terminate_sync(adc->dma.chan);
+		return PTR_ERR(adc->dma.thread);
+	}
+
+	disable_irq(adc->irq);
+
+	dma_async_issue_pending(adc->dma.chan);
+
+	return 0;
+}
+
+static int rzt2h_adc_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+	const struct iio_chan_spec *chan;
+	struct device *dev = adc->dev;
+	unsigned int scan_index;
+	u16 val = 0;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return ret;
+
+	iio_for_each_active_channel(indio_dev, scan_index) {
+		chan = &indio_dev->channels[scan_index];
+		val |= RZT2H_ADANSA0_CH_MASK(chan->channel);
+		rzt2h_adc_set_sst(adc, chan->channel, adc->sst[chan->channel]);
+	}
+
+	writew(val, adc->base + RZT2H_ADANSA0_REG);
+
+	adc->dma.period_index = 0;
+	atomic_set(&adc->dma.pending_periods, 0);
+
+	ret = rzt2h_adc_start_dma(indio_dev);
+	if (ret) {
+		pm_runtime_put_autosuspend(dev);
+		return ret;
+	}
+
+	rzt2h_adc_start(adc, RZT2H_ADCSR_ADCS_CONTINUOUS);
+
+	return 0;
+}
+
+static int rzt2h_adc_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+	struct device *dev = adc->dev;
+
+	rzt2h_adc_stop(adc);
+
+	dmaengine_terminate_sync(adc->dma.chan);
+
+	kthread_stop(adc->dma.thread);
+
+	enable_irq(adc->irq);
+
+	pm_runtime_put_autosuspend(dev);
+
+	return 0;
+}
+
+static int rzt2h_adc_sst_to_freq(struct rzt2h_adc *adc, u8 sst)
+{
+	return adc->adclk_rate / (RZT2H_ADC_CONV_CYCLES_BASE + sst);
+}
+
+static u8 rzt2h_adc_freq_to_sst(struct rzt2h_adc *adc, int freq)
+{
+	unsigned int cycles;
+
+	cycles = DIV_ROUND_CLOSEST(adc->adclk_rate, freq);
+	cycles = clamp_val(cycles,
+			   RZT2H_ADC_CONV_CYCLES_BASE + RZT2H_ADC_SST_MIN,
+			   RZT2H_ADC_CONV_CYCLES_BASE + RZT2H_ADC_SST_MAX);
+
+	return cycles - RZT2H_ADC_CONV_CYCLES_BASE;
+}
+
 static int rzt2h_adc_read_raw(struct iio_dev *indio_dev,
 			      struct iio_chan_spec const *chan,
 			      int *val, int *val2, long mask)
@@ -156,19 +495,76 @@ static int rzt2h_adc_read_raw(struct iio_dev *indio_dev,
 	struct rzt2h_adc *adc = iio_priv(indio_dev);
 
 	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
+	case IIO_CHAN_INFO_RAW: {
+		IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+		if (IIO_DEV_ACQUIRE_FAILED(claim))
+			return -EBUSY;
+
 		return rzt2h_adc_read_single(adc, chan->channel, val);
+	}
 	case IIO_CHAN_INFO_SCALE:
 		*val = 1800;
 		*val2 = 12;
 		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = rzt2h_adc_sst_to_freq(adc, adc->sst[chan->channel]);
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int rzt2h_adc_write_raw(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan,
+			       int val, int val2, long mask)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		if (val <= 0)
+			return -EINVAL;
+
+		IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+		if (IIO_DEV_ACQUIRE_FAILED(claim))
+			return -EBUSY;
+
+		adc->sst[chan->channel] = rzt2h_adc_freq_to_sst(adc, val);
+
+		return 0;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int rzt2h_adc_read_avail(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				const int **vals, int *type, int *length,
+				long mask)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*vals = adc->samp_freq_avail;
+		*type = IIO_VAL_INT;
+		*length = ARRAY_SIZE(adc->samp_freq_avail);
+		return IIO_AVAIL_RANGE;
 	default:
 		return -EINVAL;
 	}
 }
 
+static const struct iio_buffer_setup_ops rzt2h_adc_buffer_setup_ops = {
+	.postenable = rzt2h_adc_buffer_postenable,
+	.predisable = rzt2h_adc_buffer_predisable,
+};
+
 static const struct iio_info rzt2h_adc_iio_info = {
 	.read_raw = rzt2h_adc_read_raw,
+	.write_raw = rzt2h_adc_write_raw,
+	.read_avail = rzt2h_adc_read_avail,
 };
 
 static irqreturn_t rzt2h_adc_isr(int irq, void *private)
@@ -183,14 +579,21 @@ static irqreturn_t rzt2h_adc_isr(int irq, void *private)
 static const struct iio_chan_spec rzt2h_adc_chan_template = {
 	.indexed = 1,
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-			      BIT(IIO_CHAN_INFO_SCALE),
+			      BIT(IIO_CHAN_INFO_SCALE) |
+			      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+	.info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ),
 	.type = IIO_VOLTAGE,
+	.scan_type = {
+		.sign = 'u',
+		.realbits = 12,
+		.storagebits = 16,
+		.endianness = IIO_CPU,
+	},
 };
 
 static int rzt2h_adc_parse_properties(struct rzt2h_adc *adc)
 {
 	struct iio_chan_spec *chan_array;
-	unsigned int i;
 	int ret;
 
 	ret = devm_iio_adc_device_alloc_chaninfo_se(adc->dev,
@@ -203,19 +606,64 @@ static int rzt2h_adc_parse_properties(struct rzt2h_adc *adc)
 	adc->num_channels = ret;
 	adc->channels = chan_array;
 
-	for (i = 0; i < adc->num_channels; i++)
-		if (chan_array[i].channel + 1 > adc->max_channels)
-			adc->max_channels = chan_array[i].channel + 1;
+	for (unsigned int i = 0; i < adc->num_channels; i++) {
+		adc->sst[chan_array[i].channel] = RZT2H_ADC_SST_DEFAULT;
+
+		chan_array[i].scan_index = i;
+	}
 
 	return 0;
 }
 
+static void rzt2h_adc_free_dma_buf(void *p)
+{
+	struct rzt2h_adc *adc = p;
+
+	dma_free_noncoherent(adc->dev, RZT2H_ADC_DMA_BUFFER_SIZE,
+			     adc->dma.buf, adc->dma.addr, DMA_FROM_DEVICE);
+}
+
+static int rzt2h_adc_setup_dma(struct iio_dev *indio_dev)
+{
+	struct rzt2h_adc *adc = iio_priv(indio_dev);
+	struct device *dev = adc->dev;
+	int ret;
+
+	adc->dma.chan = devm_dma_request_chan(dev, "rx");
+	if (IS_ERR(adc->dma.chan)) {
+		ret = PTR_ERR(adc->dma.chan);
+		if (ret != -ENODEV)
+			return dev_err_probe(dev, ret, "DMA channel request failed\n");
+
+		adc->dma.chan = NULL;
+		return 0;
+	}
+
+	adc->dma.buf = dma_alloc_noncoherent(dev, RZT2H_ADC_DMA_BUFFER_SIZE,
+					     &adc->dma.addr, DMA_FROM_DEVICE,
+					     GFP_KERNEL);
+	if (!adc->dma.buf)
+		return -ENOMEM;
+
+	dma_sync_single_for_device(dev, adc->dma.addr, RZT2H_ADC_DMA_BUFFER_SIZE,
+				   DMA_FROM_DEVICE);
+
+	ret = devm_add_action_or_reset(dev, rzt2h_adc_free_dma_buf, adc);
+	if (ret)
+		return ret;
+
+	return devm_iio_kfifo_buffer_setup_ext(dev, indio_dev,
+					       &rzt2h_adc_buffer_setup_ops, NULL);
+}
+
 static int rzt2h_adc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct iio_dev *indio_dev;
 	struct rzt2h_adc *adc;
-	int ret, irq;
+	struct resource *res;
+	struct clk *adclk;
+	int ret;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*adc));
 	if (!indio_dev)
@@ -224,6 +672,7 @@ static int rzt2h_adc_probe(struct platform_device *pdev)
 	adc = iio_priv(indio_dev);
 	adc->dev = dev;
 	init_completion(&adc->completion);
+	init_waitqueue_head(&adc->dma.wq);
 
 	ret = devm_mutex_init(dev, &adc->lock);
 	if (ret)
@@ -235,21 +684,35 @@ static int rzt2h_adc_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	adc->base = devm_platform_ioremap_resource(pdev, 0);
+	adc->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(adc->base))
 		return PTR_ERR(adc->base);
 
+	adc->phys_base = res->start;
+
+	adclk = devm_clk_get(dev, "adclk");
+	if (IS_ERR(adclk))
+		return dev_err_probe(dev, PTR_ERR(adclk), "failed to get adclk\n");
+
+	adc->adclk_rate = clk_get_rate(adclk);
+	if (!adc->adclk_rate)
+		return dev_err_probe(dev, -EINVAL, "invalid adclk rate\n");
+
+	adc->samp_freq_avail[0] = rzt2h_adc_sst_to_freq(adc, RZT2H_ADC_SST_MAX);
+	adc->samp_freq_avail[1] = 1;
+	adc->samp_freq_avail[2] = rzt2h_adc_sst_to_freq(adc, RZT2H_ADC_SST_MIN);
+
 	pm_runtime_set_autosuspend_delay(dev, 300);
 	pm_runtime_use_autosuspend(dev);
 	ret = devm_pm_runtime_enable(dev);
 	if (ret)
 		return ret;
 
-	irq = platform_get_irq_byname(pdev, "adi");
-	if (irq < 0)
-		return irq;
+	adc->irq = platform_get_irq_byname(pdev, "adi");
+	if (adc->irq < 0)
+		return adc->irq;
 
-	ret = devm_request_irq(dev, irq, rzt2h_adc_isr, 0, dev_name(dev), adc);
+	ret = devm_request_irq(dev, adc->irq, rzt2h_adc_isr, 0, dev_name(dev), adc);
 	if (ret)
 		return ret;
 
@@ -259,6 +722,10 @@ static int rzt2h_adc_probe(struct platform_device *pdev)
 	indio_dev->channels = adc->channels;
 	indio_dev->num_channels = adc->num_channels;
 
+	ret = rzt2h_adc_setup_dma(indio_dev);
+	if (ret)
+		return ret;
+
 	return devm_iio_device_register(dev, indio_dev);
 }
 
diff --git a/drivers/iio/adc/sophgo-cv1800b-adc.c b/drivers/iio/adc/sophgo-cv1800b-adc.c
index bdc3e1326a9afd..c6eed09acfcd13 100644
--- a/drivers/iio/adc/sophgo-cv1800b-adc.c
+++ b/drivers/iio/adc/sophgo-cv1800b-adc.c
@@ -182,6 +182,8 @@ static int cv1800b_adc_probe(struct platform_device *pdev)
 		return PTR_ERR(saradc->regs);
 
 	saradc->irq = platform_get_irq_optional(pdev, 0);
+	if (saradc->irq < 0 && saradc->irq != -ENXIO)
+		return saradc->irq;
 	if (saradc->irq > 0) {
 		init_completion(&saradc->completion);
 		ret = devm_request_irq(dev, saradc->irq,
diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
index 5c6c06b269be6e..600f975f1d7b53 100644
--- a/drivers/iio/adc/stm32-adc.c
+++ b/drivers/iio/adc/stm32-adc.c
@@ -1608,11 +1608,16 @@ static int stm32_adc_read_raw(struct iio_dev *indio_dev,
 			ret = stm32_adc_single_conv(indio_dev, chan, val);
 		else
 			ret = -EINVAL;
+		iio_device_release_direct(indio_dev);
+		if (ret < 0)
+			return ret;
 
-		if (mask == IIO_CHAN_INFO_PROCESSED)
+		if (mask == IIO_CHAN_INFO_PROCESSED) {
+			if (*val == 0)
+				return -EINVAL;
 			*val = STM32_ADC_VREFINT_VOLTAGE * adc->vrefint.vrefint_cal / *val;
+		}
 
-		iio_device_release_direct(indio_dev);
 		return ret;
 
 	case IIO_CHAN_INFO_SCALE:
@@ -2263,33 +2268,37 @@ static int stm32_adc_populate_int_ch(struct iio_dev *indio_dev, const char *ch_n
 
 	for (i = 0; i < STM32_ADC_INT_CH_NB; i++) {
 		if (!strncmp(stm32_adc_ic[i].name, ch_name, STM32_ADC_CH_SZ)) {
+			bool na;
+
 			/* Check internal channel availability */
 			switch (i) {
 			case STM32_ADC_INT_CH_VDDCORE:
-				if (!adc->cfg->regs->or_vddcore.reg)
-					dev_warn(&indio_dev->dev,
-						 "%s channel not available\n", ch_name);
+				na = !adc->cfg->regs->or_vddcore.reg;
 				break;
 			case STM32_ADC_INT_CH_VDDCPU:
-				if (!adc->cfg->regs->or_vddcpu.reg)
-					dev_warn(&indio_dev->dev,
-						 "%s channel not available\n", ch_name);
+				na = !adc->cfg->regs->or_vddcpu.reg;
 				break;
 			case STM32_ADC_INT_CH_VDDQ_DDR:
-				if (!adc->cfg->regs->or_vddq_ddr.reg)
-					dev_warn(&indio_dev->dev,
-						 "%s channel not available\n", ch_name);
+				na = !adc->cfg->regs->or_vddq_ddr.reg;
 				break;
 			case STM32_ADC_INT_CH_VREFINT:
-				if (!adc->cfg->regs->ccr_vref.reg)
-					dev_warn(&indio_dev->dev,
-						 "%s channel not available\n", ch_name);
+				na = !adc->cfg->regs->ccr_vref.reg;
 				break;
 			case STM32_ADC_INT_CH_VBAT:
-				if (!adc->cfg->regs->ccr_vbat.reg)
-					dev_warn(&indio_dev->dev,
-						 "%s channel not available\n", ch_name);
+				na = !adc->cfg->regs->ccr_vbat.reg;
 				break;
+			default:
+				return -EINVAL;
+			}
+
+			if (na) {
+				/*
+				 * Channel label matches an internal STM32 ADC channel.
+				 * Warn about it, as there's normally no restriction on the
+				 * name but that's not among available internal channels.
+				 */
+				dev_warn(&indio_dev->dev, "no %s internal channel\n", ch_name);
+				return 0;
 			}
 
 			if (stm32_adc_ic[i].idx != STM32_ADC_INT_CH_VREFINT) {
@@ -2456,6 +2465,7 @@ static int stm32_adc_dma_request(struct device *dev, struct iio_dev *indio_dev)
 {
 	struct stm32_adc *adc = iio_priv(indio_dev);
 	struct dma_slave_config config = { };
+	struct device *dma_dev;
 	int ret;
 
 	adc->dma_chan = dma_request_chan(dev, "rx");
@@ -2470,7 +2480,8 @@ static int stm32_adc_dma_request(struct device *dev, struct iio_dev *indio_dev)
 		return 0;
 	}
 
-	adc->rx_buf = dma_alloc_coherent(adc->dma_chan->device->dev,
+	dma_dev = dmaengine_get_dma_device(adc->dma_chan);
+	adc->rx_buf = dma_alloc_coherent(dma_dev,
 					 STM32_DMA_BUFFER_SIZE,
 					 &adc->rx_dma_buf, GFP_KERNEL);
 	if (!adc->rx_buf) {
@@ -2490,7 +2501,7 @@ static int stm32_adc_dma_request(struct device *dev, struct iio_dev *indio_dev)
 	return 0;
 
 err_free:
-	dma_free_coherent(adc->dma_chan->device->dev, STM32_DMA_BUFFER_SIZE,
+	dma_free_coherent(dma_dev, STM32_DMA_BUFFER_SIZE,
 			  adc->rx_buf, adc->rx_dma_buf);
 err_release:
 	dma_release_channel(adc->dma_chan);
@@ -2617,7 +2628,7 @@ static int stm32_adc_probe(struct platform_device *pdev)
 
 err_dma_disable:
 	if (adc->dma_chan) {
-		dma_free_coherent(adc->dma_chan->device->dev,
+		dma_free_coherent(dmaengine_get_dma_device(adc->dma_chan),
 				  STM32_DMA_BUFFER_SIZE,
 				  adc->rx_buf, adc->rx_dma_buf);
 		dma_release_channel(adc->dma_chan);
@@ -2640,7 +2651,7 @@ static void stm32_adc_remove(struct platform_device *pdev)
 	pm_runtime_put_noidle(&pdev->dev);
 	iio_triggered_buffer_cleanup(indio_dev);
 	if (adc->dma_chan) {
-		dma_free_coherent(adc->dma_chan->device->dev,
+		dma_free_coherent(dmaengine_get_dma_device(adc->dma_chan),
 				  STM32_DMA_BUFFER_SIZE,
 				  adc->rx_buf, adc->rx_dma_buf);
 		dma_release_channel(adc->dma_chan);
@@ -2725,7 +2736,7 @@ static const struct stm32_adc_cfg stm32h7_adc_cfg = {
 	.set_ovs = stm32h7_adc_set_ovs,
 };
 
-static const unsigned int stm32_adc_min_ts_mp1[] = { 100, 100, 100, 4300, 9800 };
+static const unsigned int stm32_adc_min_ts_mp1[] = { 100, 0, 0, 4300, 9800 };
 static_assert(ARRAY_SIZE(stm32_adc_min_ts_mp1) == STM32_ADC_INT_CH_NB);
 
 static const struct stm32_adc_cfg stm32mp1_adc_cfg = {
@@ -2747,7 +2758,7 @@ static const struct stm32_adc_cfg stm32mp1_adc_cfg = {
 	.set_ovs = stm32h7_adc_set_ovs,
 };
 
-static const unsigned int stm32_adc_min_ts_mp13[] = { 100, 0, 0, 4300, 9800 };
+static const unsigned int stm32_adc_min_ts_mp13[] = { 1000, 1000, 1000, 4300, 9800 };
 static_assert(ARRAY_SIZE(stm32_adc_min_ts_mp13) == STM32_ADC_INT_CH_NB);
 
 static const struct stm32_adc_cfg stm32mp13_adc_cfg = {
diff --git a/drivers/iio/adc/stm32-dfsdm-adc.c b/drivers/iio/adc/stm32-dfsdm-adc.c
index 00f05e167afc93..674767da3ac379 100644
--- a/drivers/iio/adc/stm32-dfsdm-adc.c
+++ b/drivers/iio/adc/stm32-dfsdm-adc.c
@@ -1484,7 +1484,7 @@ static void stm32_dfsdm_dma_release(struct iio_dev *indio_dev)
 	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
 
 	if (adc->dma_chan) {
-		dma_free_coherent(adc->dma_chan->device->dev,
+		dma_free_coherent(dmaengine_get_dma_device(adc->dma_chan),
 				  DFSDM_DMA_BUFFER_SIZE,
 				  adc->rx_buf, adc->dma_buf);
 		dma_release_channel(adc->dma_chan);
@@ -1504,7 +1504,7 @@ static int stm32_dfsdm_dma_request(struct device *dev,
 		return ret;
 	}
 
-	adc->rx_buf = dma_alloc_coherent(adc->dma_chan->device->dev,
+	adc->rx_buf = dma_alloc_coherent(dmaengine_get_dma_device(adc->dma_chan),
 					 DFSDM_DMA_BUFFER_SIZE,
 					 &adc->dma_buf, GFP_KERNEL);
 	if (!adc->rx_buf) {
diff --git a/drivers/iio/adc/sun4i-gpadc-iio.c b/drivers/iio/adc/sun4i-gpadc-iio.c
index 203459ca99079f..ac5ba76b979e6d 100644
--- a/drivers/iio/adc/sun4i-gpadc-iio.c
+++ b/drivers/iio/adc/sun4i-gpadc-iio.c
@@ -638,10 +638,10 @@ static int sun4i_gpadc_probe(struct platform_device *pdev)
 		 * thermal because no thermal DT node is found.
 		 */
 		if (IS_ERR(info->tzd) && PTR_ERR(info->tzd) != -ENODEV) {
-			dev_err(&pdev->dev,
-				"could not register thermal sensor: %ld\n",
-				PTR_ERR(info->tzd));
-			return PTR_ERR(info->tzd);
+			ret = dev_err_probe(&pdev->dev,
+					    PTR_ERR(info->tzd),
+					    "could not register thermal sensor\n");
+			goto err_map;
 		}
 	}
 
@@ -657,7 +657,6 @@ static int sun4i_gpadc_probe(struct platform_device *pdev)
 	if (!info->no_irq && IS_ENABLED(CONFIG_THERMAL_OF))
 		iio_map_array_unregister(indio_dev);
 
-	pm_runtime_put(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
 	return ret;
@@ -668,7 +667,6 @@ static void sun4i_gpadc_remove(struct platform_device *pdev)
 	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
 	struct sun4i_gpadc_iio *info = iio_priv(indio_dev);
 
-	pm_runtime_put(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
 	if (!IS_ENABLED(CONFIG_THERMAL_OF))
diff --git a/drivers/iio/adc/ti-adc128s052.c b/drivers/iio/adc/ti-adc128s052.c
index 1899813c1aee34..f9a995ff403e0d 100644
--- a/drivers/iio/adc/ti-adc128s052.c
+++ b/drivers/iio/adc/ti-adc128s052.c
@@ -195,12 +195,14 @@ static int adc128_probe(struct spi_device *spi)
 	adc = iio_priv(indio_dev);
 	adc->spi = spi;
 
-	indio_dev->name = spi_get_device_id(spi)->name;
+	config = spi_get_device_match_data(spi);
+	if (!config)
+		return -ENODATA;
+
+	indio_dev->name = spi->modalias;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->info = &adc128_info;
 
-	config = spi_get_device_match_data(spi);
-
 	indio_dev->channels = config->channels;
 	indio_dev->num_channels = config->num_channels;
 
diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c
index 0fbfa4e499aa82..0f9244107c6ddb 100644
--- a/drivers/iio/adc/ti-ads1015.c
+++ b/drivers/iio/adc/ti-ads1015.c
@@ -233,6 +233,7 @@ static const struct iio_event_spec ads1015_events[] = {
 struct ads1015_channel_data {
 	unsigned int pga;
 	unsigned int data_rate;
+	const char *label;
 };
 
 struct ads1015_thresh_data {
@@ -586,6 +587,15 @@ static int ads1015_read_raw(struct iio_dev *indio_dev,
 	}
 }
 
+static int ads1015_read_label(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan, char *label)
+{
+	struct ads1015_data *data = iio_priv(indio_dev);
+
+	return sysfs_emit(label, "%s\n",
+			  data->channel_data[chan->address].label);
+}
+
 static int ads1015_write_raw(struct iio_dev *indio_dev,
 			     struct iio_chan_spec const *chan, int val,
 			     int val2, long mask)
@@ -843,6 +853,7 @@ static const struct iio_buffer_setup_ops ads1015_buffer_setup_ops = {
 static const struct iio_info ads1015_info = {
 	.read_avail	= ads1015_read_avail,
 	.read_raw	= ads1015_read_raw,
+	.read_label	= ads1015_read_label,
 	.write_raw	= ads1015_write_raw,
 	.read_event_value = ads1015_read_event,
 	.write_event_value = ads1015_write_event,
@@ -853,6 +864,7 @@ static const struct iio_info ads1015_info = {
 static const struct iio_info tla2024_info = {
 	.read_avail	= ads1015_read_avail,
 	.read_raw	= ads1015_read_raw,
+	.read_label	= ads1015_read_label,
 	.write_raw	= ads1015_write_raw,
 };
 
@@ -861,7 +873,9 @@ static int ads1015_client_get_channels_config(struct i2c_client *client)
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ads1015_data *data = iio_priv(indio_dev);
 	struct device *dev = &client->dev;
+	const char *label;
 	int i = -1;
+	int ret;
 
 	device_for_each_child_node_scoped(dev, node) {
 		u32 pval;
@@ -897,6 +911,15 @@ static int ads1015_client_get_channels_config(struct i2c_client *client)
 			}
 		}
 
+		if (fwnode_property_present(node, "label")) {
+			ret = fwnode_property_read_string(node, "label", &label);
+			if (ret) {
+				dev_err(dev, "invalid label on %pfw\n", node);
+				return ret;
+			}
+			data->channel_data[channel].label = label;
+		}
+
 		data->channel_data[channel].pga = pga;
 		data->channel_data[channel].data_rate = data_rate;
 
@@ -985,6 +1008,13 @@ static int ads1015_probe(struct i2c_client *client)
 	/* we need to keep this ABI the same as used by hwmon ADS1015 driver */
 	ads1015_get_channels_config(client);
 
+	/* Channels without a label property fall back to the datasheet name */
+	for (i = 0; i < ADS1015_CHANNELS; i++) {
+		if (!data->channel_data[i].label)
+			data->channel_data[i].label =
+				indio_dev->channels[i].datasheet_name;
+	}
+
 	data->regmap = devm_regmap_init_i2c(client, chip->has_comparator ?
 					    &ads1015_regmap_config :
 					    &tla2024_regmap_config);
diff --git a/drivers/iio/adc/ti-ads1018.c b/drivers/iio/adc/ti-ads1018.c
index d9ec4a8d7ce541..8f2d03eee77b6f 100644
--- a/drivers/iio/adc/ti-ads1018.c
+++ b/drivers/iio/adc/ti-ads1018.c
@@ -624,12 +624,16 @@ static int ads1018_trigger_setup(struct iio_dev *indio_dev)
 
 static int ads1018_spi_probe(struct spi_device *spi)
 {
-	const struct ads1018_chip_info *info = spi_get_device_match_data(spi);
+	const struct ads1018_chip_info *info;
 	struct device *dev = &spi->dev;
 	struct iio_dev *indio_dev;
 	struct ads1018 *ads1018;
 	int ret;
 
+	info = spi_get_device_match_data(spi);
+	if (!info)
+		return -ENODATA;
+
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*ads1018));
 	if (!indio_dev)
 		return -ENOMEM;
diff --git a/drivers/iio/adc/ti-ads1100.c b/drivers/iio/adc/ti-ads1100.c
index 9fe8d54cce83cb..9957c4d813fb11 100644
--- a/drivers/iio/adc/ti-ads1100.c
+++ b/drivers/iio/adc/ti-ads1100.c
@@ -5,7 +5,7 @@
  * Copyright (c) 2023, Topic Embedded Products
  *
  * Datasheet: https://www.ti.com/lit/gpn/ads1100
- * IIO driver for ADS1100 and ADS1000 ADC 16-bit I2C
+ * IIO driver for ADS1100 and similar single channel ADC 16-bit I2C
  */
 
 #include <linux/bitfield.h>
@@ -15,10 +15,12 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/i2c.h>
+#include <linux/iopoll.h>
 #include <linux/mutex.h>
 #include <linux/property.h>
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
+#include <linux/time.h>
 #include <linux/units.h>
 
 #include <linux/iio/iio.h>
@@ -39,17 +41,44 @@
 #define	ADS1100_SINGLESHOT	ADS1100_CFG_SC
 
 #define ADS1100_SLEEP_DELAY_MS	2000
+#define ADS1110_INTERNAL_REF_mV 2048
 
 static const int ads1100_data_rate[] = { 128, 32, 16, 8 };
+static const int ads1110_data_rate[] = { 240, 60, 30, 15 };
 static const int ads1100_data_rate_bits[] = { 12, 14, 15, 16 };
 
+/* Timeout based on the minimum sample rate of 8 SPS (7500ms) */
+#define ADS1100_MAX_DRDY_TIMEOUT_US	(7500 * USEC_PER_MSEC)
+
+struct ads1100_config {
+	const char *name;
+	const int *available_data_rate_Hz;
+	const int data_rate_count;
+	bool has_internal_vref_only;
+};
+
+static const struct ads1100_config ads1100_config = {
+	.name = "ads1100",
+	.available_data_rate_Hz = ads1100_data_rate,
+	.data_rate_count = ARRAY_SIZE(ads1100_data_rate),
+	.has_internal_vref_only = false,
+};
+
+static const struct ads1100_config ads1110_config = {
+	.name = "ads1110",
+	.available_data_rate_Hz = ads1110_data_rate,
+	.data_rate_count = ARRAY_SIZE(ads1110_data_rate),
+	.has_internal_vref_only = true,
+};
+
 struct ads1100_data {
 	struct i2c_client *client;
 	struct regulator *reg_vdd;
 	struct mutex lock;
 	int scale_avail[2 * 4]; /* 4 gain settings */
+	const struct ads1100_config *chip_info;
 	u8 config;
-	bool supports_data_rate; /* Only the ADS1100 can select the rate */
+	bool supports_data_rate;
 };
 
 static const struct iio_chan_spec ads1100_channel = {
@@ -85,6 +114,20 @@ static int ads1100_set_config_bits(struct ads1100_data *data, u8 mask, u8 value)
 	return 0;
 };
 
+static int ads1100_get_vref_millivolts(struct ads1100_data *data)
+{
+	int voltage_uV;
+
+	if (data->chip_info->has_internal_vref_only)
+		return ADS1110_INTERNAL_REF_mV;
+
+	voltage_uV = regulator_get_voltage(data->reg_vdd);
+	if (voltage_uV < 0)
+		return voltage_uV;
+
+	return voltage_uV / (MICRO / MILLI);
+}
+
 static int ads1100_data_bits(struct ads1100_data *data)
 {
 	return ads1100_data_rate_bits[FIELD_GET(ADS1100_DR_MASK, data->config)];
@@ -123,10 +166,86 @@ static int ads1100_get_adc_result(struct ads1100_data *data, int chan, int *val)
 	return 0;
 }
 
+static int ads1100_conversion_busy(struct ads1100_data *data)
+{
+	u8 buffer[3];
+	int ret;
+
+	ret = i2c_master_recv(data->client, (char *)&buffer, sizeof(buffer));
+	if (ret < 0) {
+		dev_err(&data->client->dev, "I2C read fail: %d\n", ret);
+		return ret;
+	}
+
+	return FIELD_GET(ADS1100_CFG_ST_BSY, buffer[2]);
+}
+
+static int ads1100_wait_single_conversion(struct ads1100_data *data)
+{
+	int data_rate_index = FIELD_GET(ADS1100_DR_MASK, data->config);
+	int data_rate_Hz = data->chip_info->available_data_rate_Hz[data_rate_index];
+	unsigned long poll_us = DIV_ROUND_CLOSEST(USEC_PER_SEC, data_rate_Hz) / 4;
+	int busy;
+	int ret;
+
+	ret = readx_poll_timeout(ads1100_conversion_busy, data,
+				 busy, busy <= 0,
+				 poll_us, ADS1100_MAX_DRDY_TIMEOUT_US);
+	if (busy < 0)
+		return busy;
+
+	return ret;
+}
+
+static int ads1100_start_single_conversion(struct ads1100_data *data)
+{
+	u8 config = data->config | ADS1100_CFG_SC;
+	int ret;
+
+	ret = i2c_master_send(data->client, &config, sizeof(config));
+	if (ret < 0) {
+		dev_err(&data->client->dev, "I2C write fail: %d\n", ret);
+		return ret;
+	}
+	/* Need to wait because of change from continuous to single mode */
+	ret = ads1100_wait_single_conversion(data);
+	if (ret)
+		return ret;
+
+	config |= ADS1100_CFG_ST_BSY;
+
+	ret = i2c_master_send(data->client, &config, sizeof(config));
+	if (ret < 0) {
+		dev_err(&data->client->dev, "I2C write fail: %d\n", ret);
+		return ret;
+	}
+
+	/* No need to cache it, it's status bit */
+	data->config = config & ~ADS1100_CFG_ST_BSY;
+
+	return 0;
+}
+
+static int ads1100_poll_data_ready(struct ads1100_data *data)
+{
+	int ret;
+
+	ret = ads1100_start_single_conversion(data);
+	if (ret)
+		return ret;
+
+	ret = ads1100_wait_single_conversion(data);
+	if (ret)
+		return ret;
+
+	return ads1100_set_config_bits(data, ADS1100_CFG_SC, ADS1100_CONTINUOUS);
+}
+
 static int ads1100_set_scale(struct ads1100_data *data, int val, int val2)
 {
 	int microvolts;
 	int gain;
+	int ret;
 
 	/* With Vdd between 2.7 and 5V, the scale is always below 1 */
 	if (val)
@@ -135,7 +254,12 @@ static int ads1100_set_scale(struct ads1100_data *data, int val, int val2)
 	if (!val2)
 		return -EINVAL;
 
-	microvolts = regulator_get_voltage(data->reg_vdd);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(&data->client->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	microvolts = ads1100_get_vref_millivolts(data) * (MICRO / MILLI);
 	/*
 	 * val2 is in 'micro' units, n = val2 / 1000000
 	 * result must be millivolts, d = microvolts / 1000
@@ -149,32 +273,40 @@ static int ads1100_set_scale(struct ads1100_data *data, int val, int val2)
 
 	ads1100_set_config_bits(data, ADS1100_PGA_MASK, ffs(gain) - 1);
 
-	return 0;
+	return ads1100_poll_data_ready(data);
 }
 
 static int ads1100_set_data_rate(struct ads1100_data *data, int chan, int rate)
 {
 	unsigned int i;
 	unsigned int size;
+	int ret;
 
-	size = data->supports_data_rate ? ARRAY_SIZE(ads1100_data_rate) : 1;
+	size = data->supports_data_rate ? data->chip_info->data_rate_count : 1;
 	for (i = 0; i < size; i++) {
-		if (ads1100_data_rate[i] == rate)
-			return ads1100_set_config_bits(data, ADS1100_DR_MASK,
-						       FIELD_PREP(ADS1100_DR_MASK, i));
+		if (data->chip_info->available_data_rate_Hz[i] == rate)
+			break;
 	}
 
-	return -EINVAL;
-}
+	if (i == size)
+		return -EINVAL;
 
-static int ads1100_get_vdd_millivolts(struct ads1100_data *data)
-{
-	return regulator_get_voltage(data->reg_vdd) / (MICRO / MILLI);
+	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(&data->client->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = ads1100_set_config_bits(data, ADS1100_DR_MASK,
+				      FIELD_PREP(ADS1100_DR_MASK, i));
+	if (ret)
+		return ret;
+
+	return ads1100_poll_data_ready(data);
 }
 
 static void ads1100_calc_scale_avail(struct ads1100_data *data)
 {
-	int millivolts = ads1100_get_vdd_millivolts(data);
+	int millivolts = ads1100_get_vref_millivolts(data);
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(data->scale_avail) / 2; i++) {
@@ -196,9 +328,9 @@ static int ads1100_read_avail(struct iio_dev *indio_dev,
 	switch (mask) {
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		*type = IIO_VAL_INT;
-		*vals = ads1100_data_rate;
+		*vals = data->chip_info->available_data_rate_Hz;
 		if (data->supports_data_rate)
-			*length = ARRAY_SIZE(ads1100_data_rate);
+			*length = data->chip_info->data_rate_count;
 		else
 			*length = 1;
 		return IIO_AVAIL_LIST;
@@ -218,6 +350,7 @@ static int ads1100_read_raw(struct iio_dev *indio_dev,
 {
 	int ret;
 	struct ads1100_data *data = iio_priv(indio_dev);
+	int data_rate_index;
 
 	guard(mutex)(&data->lock);
 	switch (mask) {
@@ -233,12 +366,12 @@ static int ads1100_read_raw(struct iio_dev *indio_dev,
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
 		/* full-scale is the supply voltage in millivolts */
-		*val = ads1100_get_vdd_millivolts(data);
+		*val = ads1100_get_vref_millivolts(data);
 		*val2 = 15 + FIELD_GET(ADS1100_PGA_MASK, data->config);
 		return IIO_VAL_FRACTIONAL_LOG2;
 	case IIO_CHAN_INFO_SAMP_FREQ:
-		*val = ads1100_data_rate[FIELD_GET(ADS1100_DR_MASK,
-						   data->config)];
+		data_rate_index = FIELD_GET(ADS1100_DR_MASK, data->config);
+		*val = data->chip_info->available_data_rate_Hz[data_rate_index];
 		return IIO_VAL_INT;
 	default:
 		return -EINVAL;
@@ -318,7 +451,12 @@ static int ads1100_probe(struct i2c_client *client)
 	data->client = client;
 	mutex_init(&data->lock);
 
-	indio_dev->name = "ads1100";
+	data->chip_info = i2c_get_match_data(client);
+	if (!data->chip_info)
+		return dev_err_probe(dev, -ENODATA,
+				     "Can't get device data from firmware\n");
+
+	indio_dev->name = data->chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = &ads1100_channel;
 	indio_dev->num_channels = 1;
@@ -400,16 +538,18 @@ static DEFINE_RUNTIME_DEV_PM_OPS(ads1100_pm_ops,
 				 NULL);
 
 static const struct i2c_device_id ads1100_id[] = {
-	{ .name = "ads1100" },
-	{ .name = "ads1000" },
+	{ .name = "ads1000", .driver_data = (kernel_ulong_t)&ads1100_config },
+	{ .name = "ads1100", .driver_data = (kernel_ulong_t)&ads1100_config },
+	{ .name = "ads1110", .driver_data = (kernel_ulong_t)&ads1110_config },
 	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ads1100_id);
 
 static const struct of_device_id ads1100_of_match[] = {
-	{.compatible = "ti,ads1100" },
-	{.compatible = "ti,ads1000" },
+	{ .compatible = "ti,ads1000", .data = &ads1100_config },
+	{ .compatible = "ti,ads1100", .data = &ads1100_config },
+	{ .compatible = "ti,ads1110", .data = &ads1110_config },
 	{ }
 };
 
diff --git a/drivers/iio/adc/ti-ads112c04.c b/drivers/iio/adc/ti-ads112c04.c
new file mode 100644
index 00000000000000..acf7209db76122
--- /dev/null
+++ b/drivers/iio/adc/ti-ads112c04.c
@@ -0,0 +1,524 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Texas Instruments ADS112C04 16-bit I2C ADC driver
+ *
+ * Copyright (c) 2026 Kyle Hsieh <kylehsieh1995@gmail.com>
+ *
+ * Datasheet: https://www.ti.com/lit/ds/symlink/ads112c04.pdf
+ * Based on TI Reference Code and standard Linux IIO framework.
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include <linux/iio/iio.h>
+
+#define ADS112C04_CMD_RESET		0x06
+#define ADS112C04_CMD_START_SYNC	0x08
+#define ADS112C04_CMD_RDATA		0x10
+#define ADS112C04_CMD_RREG(reg)		(0x20 | ((reg) << 2))
+#define ADS112C04_CMD_WREG(reg)		(0x40 | ((reg) << 2))
+
+#define ADS112C04_REG_CONFIG0			0x00
+#define   ADS112C04_CONF0_MUX			  GENMASK(7, 4)
+#define     ADS112C04_CONF0_MUX_AIN0_AIN1	    0
+#define     ADS112C04_CONF0_MUX_AIN_SINGLE_BASE	    8
+#define   ADS112C04_CONF0_GAIN			  GENMASK(3, 1)
+#define     ADS112C04_CONF0_GAIN_X1		    0
+#define   ADS112C04_CONF0_PGA_BYPASS		  BIT(0)
+
+#define ADS112C04_REG_CONFIG1			0x01
+#define   ADS112C04_CONF1_DR			  GENMASK(7, 5)
+#define     ADS112C04_CONF1_DR_20SPS		    0
+#define   ADS112C04_CONF1_MODE			  BIT(4)
+#define     ADS112C04_CONF1_MODE_NORMAL		    0
+#define   ADS112C04_CONF1_CM			  BIT(3)
+#define     ADS112C04_CONF1_CM_SINGLE_SHOT	    0
+#define   ADS112C04_CONF1_VREF			  GENMASK(2, 1)
+#define     ADS112C04_CONF1_VREF_INTERNAL	    0
+#define     ADS112C04_CONF1_VREF_EXTERNAL	    1
+#define     ADS112C04_CONF1_VREF_AVDD		    2
+#define   ADS112C04_CONF1_TS			  BIT(0)
+#define     ADS112C04_CONF1_TS_DISABLED		    0
+
+#define ADS112C04_REG_CONFIG2			0x02
+#define   ADS112C04_CONF2_DRDY			  BIT(7)
+
+#define ADS112C04_INT_REF_mV			2048
+
+#define ADS112C04_MAX_CHANNELS		12
+
+#define ADS112C04_RESOLUTION_BITS	16
+
+enum {
+	ADS112C04_VREF_SOURCE_INTERNAL,
+	ADS112C04_VREF_SOURCE_EXTERNAL,
+	ADS112C04_VREF_SOURCE_AVDD,
+};
+
+static const char * const ads112c04_vref_names[] = {
+	[ADS112C04_VREF_SOURCE_INTERNAL] = "internal",
+	[ADS112C04_VREF_SOURCE_EXTERNAL] = "external",
+	[ADS112C04_VREF_SOURCE_AVDD]     = "avdd",
+};
+
+static const u8 ads112c04_vref_reg_val[] = {
+	[ADS112C04_VREF_SOURCE_INTERNAL] = ADS112C04_CONF1_VREF_INTERNAL,
+	[ADS112C04_VREF_SOURCE_EXTERNAL] = ADS112C04_CONF1_VREF_EXTERNAL,
+	[ADS112C04_VREF_SOURCE_AVDD]     = ADS112C04_CONF1_VREF_AVDD,
+};
+
+/* Indexed by [AINP][AINN], -1 means the combination is not available. */
+static const s8 ads112c04_diff_mux[4][4] = {
+	{ -1,  0,  1,  2 },
+	{  3, -1,  4,  5 },
+	{ -1, -1, -1,  6 },
+	{ -1, -1,  7, -1 },
+};
+
+struct ads112c04_state {
+	struct i2c_client *client;
+	/* Protects concurrent ADC reads and device configuration */
+	struct mutex lock;
+	struct completion completion;
+	u32 avdd_mV;
+	u32 ext_ref_mV;
+	u8 *vref_source;	/* one entry per channel, indexed by scan order */
+	u8 config0;
+	u8 config1;
+};
+
+static int ads112c04_write_cmd(struct i2c_client *client, u8 cmd)
+{
+	return i2c_smbus_write_byte(client, cmd);
+}
+
+static int ads112c04_read_reg(struct i2c_client *client, u8 reg, u8 *val)
+{
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(client, ADS112C04_CMD_RREG(reg));
+	if (ret < 0)
+		return ret;
+
+	*val = ret;
+
+	return 0;
+}
+
+static int ads112c04_write_reg(struct i2c_client *client, u8 reg, u8 val)
+{
+	return i2c_smbus_write_byte_data(client, ADS112C04_CMD_WREG(reg), val);
+}
+
+static int ads112c04_wait_for_data(struct ads112c04_state *st)
+{
+	int ret, err;
+	u8 val;
+
+	if (st->client->irq > 0) {
+		/* Timeout is 100ms (slowest data rate is 20 SPS) */
+		if (!wait_for_completion_timeout(&st->completion, msecs_to_jiffies(100)))
+			return -ETIMEDOUT;
+
+		return 0;
+	}
+
+	ret = read_poll_timeout(ads112c04_read_reg, err,
+				err < 0 || (val & ADS112C04_CONF2_DRDY),
+				1 * USEC_PER_MSEC, 100 * USEC_PER_MSEC, false,
+				st->client, ADS112C04_REG_CONFIG2, &val);
+	if (err < 0)
+		return err;
+
+	return ret;
+}
+
+static int ads112c04_read_data(struct ads112c04_state *st, int *val)
+{
+	int ret;
+
+	ret = i2c_smbus_read_word_swapped(st->client, ADS112C04_CMD_RDATA);
+	if (ret < 0)
+		return ret;
+
+	*val = sign_extend32(ret, ADS112C04_RESOLUTION_BITS - 1);
+
+	return 0;
+}
+
+static int ads112c04_get_adc_result(struct iio_dev *indio_dev,
+				    struct iio_chan_spec const *chan,
+				    int *val)
+{
+	struct ads112c04_state *st = iio_priv(indio_dev);
+	unsigned int idx = chan - indio_dev->channels;
+	u8 new_config0, new_config1;
+	int ret;
+
+	new_config0 = st->config0;
+	FIELD_MODIFY(ADS112C04_CONF0_MUX, &new_config0, chan->address);
+
+	if (st->config0 != new_config0) {
+		ret = ads112c04_write_reg(st->client, ADS112C04_REG_CONFIG0,
+					  new_config0);
+		if (ret < 0)
+			return ret;
+
+		st->config0 = new_config0;
+	}
+
+	new_config1 = st->config1;
+	FIELD_MODIFY(ADS112C04_CONF1_VREF, &new_config1,
+		     ads112c04_vref_reg_val[st->vref_source[idx]]);
+
+	if (st->config1 != new_config1) {
+		ret = ads112c04_write_reg(st->client, ADS112C04_REG_CONFIG1,
+					  new_config1);
+		if (ret < 0)
+			return ret;
+
+		st->config1 = new_config1;
+	}
+
+	reinit_completion(&st->completion);
+
+	ret = ads112c04_write_cmd(st->client, ADS112C04_CMD_START_SYNC);
+	if (ret < 0)
+		return ret;
+
+	ret = ads112c04_wait_for_data(st);
+	if (ret < 0)
+		return ret;
+
+	return ads112c04_read_data(st, val);
+}
+
+static int ads112c04_read_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int *val, int *val2, long mask)
+{
+	struct ads112c04_state *st = iio_priv(indio_dev);
+	unsigned int idx = chan - indio_dev->channels;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		mutex_lock(&st->lock);
+		ret = ads112c04_get_adc_result(indio_dev, chan, val);
+		mutex_unlock(&st->lock);
+
+		if (ret < 0)
+			return ret;
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SCALE:
+		switch (st->vref_source[idx]) {
+		case ADS112C04_VREF_SOURCE_EXTERNAL:
+			*val = st->ext_ref_mV;
+			break;
+		case ADS112C04_VREF_SOURCE_AVDD:
+			*val = st->avdd_mV;
+			break;
+		default:
+			*val = ADS112C04_INT_REF_mV;
+			break;
+		}
+		*val2 = ADS112C04_RESOLUTION_BITS - 1;
+		return IIO_VAL_FRACTIONAL_LOG2;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static irqreturn_t ads112c04_irq_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct ads112c04_state *st = iio_priv(indio_dev);
+
+	complete(&st->completion);
+
+	return IRQ_HANDLED;
+}
+
+static const struct iio_info ads112c04_info = {
+	.read_raw = ads112c04_read_raw,
+};
+
+static int ads112c04_parse_vref_source(struct fwnode_handle *child)
+{
+	if (!fwnode_property_present(child, "reference-sources"))
+		return ADS112C04_VREF_SOURCE_INTERNAL;
+
+	return fwnode_property_match_property_string(child, "reference-sources",
+						    ads112c04_vref_names,
+						    ARRAY_SIZE(ads112c04_vref_names));
+}
+
+static int ads112c04_parse_channels(struct iio_dev *indio_dev,
+				    bool *need_avdd_ref, bool *need_ext_ref)
+{
+	struct device *dev = indio_dev->dev.parent;
+	struct ads112c04_state *st = iio_priv(indio_dev);
+	struct iio_chan_spec *channels;
+	u32 num_channels, pair[2], channel;
+	unsigned int i;
+	int ret;
+
+	num_channels = device_get_named_child_node_count(dev, "channel");
+	if (!num_channels)
+		return dev_err_probe(dev, -EINVAL, "no channel subnodes found\n");
+
+	if (num_channels > ADS112C04_MAX_CHANNELS)
+		return dev_err_probe(dev, -EINVAL,
+				     "num of channel nodes exceeds %d\n",
+				     ADS112C04_MAX_CHANNELS);
+
+	channels = devm_kcalloc(dev, num_channels, sizeof(*channels), GFP_KERNEL);
+	if (!channels)
+		return -ENOMEM;
+
+	st->vref_source = devm_kcalloc(dev, num_channels,
+				       sizeof(*st->vref_source), GFP_KERNEL);
+	if (!st->vref_source)
+		return -ENOMEM;
+
+	i = 0;
+	device_for_each_named_child_node_scoped(dev, child, "channel") {
+		const char *s_chan = "single-channel", *d_chan = "diff-channels";
+		struct iio_chan_spec *spec = &channels[i];
+
+		if (fwnode_property_present(child, "excitation-channels"))
+			return dev_err_probe(dev, -EOPNOTSUPP,
+					     "excitation-channels is not supported yet\n");
+
+		ret = ads112c04_parse_vref_source(child);
+		if (ret < 0)
+			return dev_err_probe(dev, ret,
+					     "invalid reference-sources value\n");
+
+		st->vref_source[i] = ret;
+
+		*need_ext_ref |= st->vref_source[i] == ADS112C04_VREF_SOURCE_EXTERNAL;
+		*need_avdd_ref |= st->vref_source[i] == ADS112C04_VREF_SOURCE_AVDD;
+
+		/*
+		 * REVISIT: when ti,refp-refn-resistor-ohms is implemented, a
+		 * channel using an external resistor reference is effectively
+		 * a resistance measurement and should use IIO_RESISTANCE.
+		 */
+		spec->type = IIO_VOLTAGE;
+		spec->indexed = 1;
+		spec->info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE);
+
+		if (fwnode_property_present(child, s_chan)) {
+			ret = fwnode_property_read_u32(child, s_chan, &channel);
+			if (ret)
+				return dev_err_probe(dev, ret,
+						     "failed to read %s property\n", s_chan);
+
+			if (channel > 3)
+				return dev_err_probe(dev, -EINVAL, "%s must be 0-3\n", s_chan);
+
+			spec->channel = channel;
+			spec->address = ADS112C04_CONF0_MUX_AIN_SINGLE_BASE + channel;
+		} else if (fwnode_property_present(child, d_chan)) {
+			ret = fwnode_property_read_u32_array(child, d_chan, pair, ARRAY_SIZE(pair));
+			if (ret)
+				return dev_err_probe(dev, ret,
+						     "failed to read %s property\n", d_chan);
+
+			if (pair[0] > 3 || pair[1] > 3)
+				return dev_err_probe(dev, -EINVAL, "%s must be 0-3\n", d_chan);
+
+			spec->channel = pair[0];
+			spec->channel2 = pair[1];
+			spec->differential = 1;
+
+			if (ads112c04_diff_mux[pair[0]][pair[1]] < 0)
+				return dev_err_probe(dev, -EINVAL,
+						     "invalid %s combination\n", d_chan);
+
+			spec->address = ads112c04_diff_mux[pair[0]][pair[1]];
+		} else {
+			return dev_err_probe(dev, -EINVAL,
+					     "channel node must have %s or %s\n", s_chan, d_chan);
+		}
+
+		i++;
+	}
+
+	indio_dev->channels = channels;
+	indio_dev->num_channels = i;
+
+	return 0;
+}
+
+static int ads112c04_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct iio_dev *indio_dev;
+	struct ads112c04_state *st;
+	struct reset_control *reset;
+	bool need_avdd_ref = false, need_ext_ref = false;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	st->client = client;
+
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
+	init_completion(&st->completion);
+
+	indio_dev->name = "ads112c04";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->info = &ads112c04_info;
+
+	/* Forward compatibility checks for unimplemented DT properties */
+	if (device_property_present(dev, "refn-supply") ||
+	    device_property_present(dev, "ti,refp-refn-resistor-ohms"))
+		return dev_err_probe(dev, -EOPNOTSUPP,
+				     "refn-supply and external resistors are not supported yet\n");
+
+	ret = ads112c04_parse_channels(indio_dev, &need_avdd_ref, &need_ext_ref);
+	if (ret)
+		return ret;
+
+	if (need_avdd_ref) {
+		ret = devm_regulator_get_enable_read_voltage(dev, "avdd");
+		if (ret < 0)
+			return dev_err_probe(dev, ret, "failed to get avdd voltage\n");
+
+		st->avdd_mV = ret / (MICRO / MILLI);
+	} else {
+		ret = devm_regulator_get_enable(dev, "avdd");
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to get avdd regulator\n");
+	}
+
+	ret = devm_regulator_get_enable(dev, "dvdd");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get dvdd regulator\n");
+
+	if (device_property_present(dev, "refp-supply")) {
+		ret = devm_regulator_get_enable_read_voltage(dev, "refp");
+		if (ret < 0)
+			return dev_err_probe(dev, ret, "failed to get refp voltage\n");
+
+		st->ext_ref_mV = ret / (MICRO / MILLI);
+	}
+
+	if (need_ext_ref && !st->ext_ref_mV)
+		return dev_err_probe(dev, -EINVAL,
+				     "external reference measurements require refp-supply\n");
+
+	/* Datasheet: POR releases ~500us after supplies are stable */
+	fsleep(500);
+
+	reset = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(reset))
+		return dev_err_probe(dev, PTR_ERR(reset), "failed to get reset\n");
+
+	if (reset) {
+		/* Datasheet: tw(RSL), the RESET low pulse, is 250ns minimum */
+		fsleep(1);
+
+		ret = reset_control_deassert(reset);
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to deassert reset\n");
+	} else {
+		ret = ads112c04_write_cmd(client, ADS112C04_CMD_RESET);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* Datasheet: td(RSSTA) is 100ns minimum after the RESET rising edge */
+	fsleep(1);
+
+	/*
+	 * Initialize CONFIG0 with all fields explicit: gain of 1 with the PGA
+	 * bypassed, which allows full-scale single-ended measurements. The MUX
+	 * field is updated per channel before each conversion.
+	 */
+	st->config0 = FIELD_PREP(ADS112C04_CONF0_MUX,
+				 ADS112C04_CONF0_MUX_AIN0_AIN1) |
+		      FIELD_PREP(ADS112C04_CONF0_GAIN,
+				 ADS112C04_CONF0_GAIN_X1) |
+		      ADS112C04_CONF0_PGA_BYPASS;
+
+	ret = ads112c04_write_reg(client, ADS112C04_REG_CONFIG0, st->config0);
+	if (ret)
+		return ret;
+
+	st->config1 = FIELD_PREP(ADS112C04_CONF1_DR,
+				 ADS112C04_CONF1_DR_20SPS) |
+		      FIELD_PREP(ADS112C04_CONF1_MODE,
+				 ADS112C04_CONF1_MODE_NORMAL) |
+		      FIELD_PREP(ADS112C04_CONF1_CM,
+				 ADS112C04_CONF1_CM_SINGLE_SHOT) |
+		      FIELD_PREP(ADS112C04_CONF1_VREF,
+				 ADS112C04_CONF1_VREF_INTERNAL) |
+		      FIELD_PREP(ADS112C04_CONF1_TS,
+				 ADS112C04_CONF1_TS_DISABLED);
+
+	ret = ads112c04_write_reg(client, ADS112C04_REG_CONFIG1, st->config1);
+	if (ret)
+		return ret;
+
+	if (client->irq > 0) {
+		ret = devm_request_irq(dev, client->irq, ads112c04_irq_handler, 0,
+				       indio_dev->name, indio_dev);
+		if (ret)
+			return ret;
+	}
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct i2c_device_id ads112c04_id[] = {
+	{ .name = "ads112c04" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, ads112c04_id);
+
+static const struct of_device_id ads112c04_of_match[] = {
+	{ .compatible = "ti,ads112c04" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ads112c04_of_match);
+
+static struct i2c_driver ads112c04_driver = {
+	.driver = {
+		.name = "ads112c04",
+		.of_match_table = ads112c04_of_match,
+	},
+	.probe = ads112c04_probe,
+	.id_table = ads112c04_id,
+};
+module_i2c_driver(ads112c04_driver);
+
+MODULE_AUTHOR("Kyle Hsieh <kylehsieh1995@gmail.com>");
+MODULE_DESCRIPTION("Texas Instruments ADS112C04 ADC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/adc/ti-ads112c14.c b/drivers/iio/adc/ti-ads112c14.c
index 60eab4852ba280..3c877126b0bebf 100644
--- a/drivers/iio/adc/ti-ads112c14.c
+++ b/drivers/iio/adc/ti-ads112c14.c
@@ -9,7 +9,10 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/bitmap.h>
 #include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/completion.h>
 #include <linux/crc8.h>
 #include <linux/delay.h>
 #include <linux/dev_printk.h>
@@ -17,8 +20,10 @@
 #include <linux/i2c.h>
 #include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
+#include <linux/iio/trigger.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
+#include <linux/interrupt.h>
 #include <linux/math64.h>
 #include <linux/minmax.h>
 #include <linux/module.h>
@@ -72,7 +77,14 @@
 #define   ADS112C14_DEVICE_CFG_PWDN			BIT(7)
 #define   ADS112C14_DEVICE_CFG_STBY_MODE		BIT(6)
 #define   ADS112C14_DEVICE_CFG_BOCS			GENMASK(5, 4)
+#define     ADS112C14_DEVICE_CFG_BOCS_DISABLED		  0
+#define     ADS112C14_DEVICE_CFG_BOCS_200_nA		  1
+#define     ADS112C14_DEVICE_CFG_BOCS_1_uA		  2
+#define     ADS112C14_DEVICE_CFG_BOCS_10_uA		  3
+
 #define   ADS112C14_DEVICE_CFG_CLK_SEL			BIT(3)
+#define     ADS112C14_DEVICE_CFG_CLK_SEL_INTERNAL	  0
+#define     ADS112C14_DEVICE_CFG_CLK_SEL_EXTERNAL	  1
 #define   ADS112C14_DEVICE_CFG_CONV_MODE		BIT(2)
 #define     ADS112C14_DEVICE_CFG_CONV_MODE_CONTINUOUS	  0
 #define     ADS112C14_DEVICE_CFG_CONV_MODE_SINGLE_SHOT	  1
@@ -82,6 +94,14 @@
 #define   ADS112C14_DATA_RATE_CFG_DELAY			GENMASK(7, 4)
 #define   ADS112C14_DATA_RATE_CFG_GC_EN			BIT(3)
 #define   ADS112C14_DATA_RATE_CFG_FLTR_OSR		GENMASK(2, 0)
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_16		  0
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_32		  1
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_128	  2
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_256	  3
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_512	  4
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024	  5
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS	  6
+#define     ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS	  7
 
 #define ADS112C14_REG_MUX_CFG				0x07
 #define   ADS112C14_MUX_CFG_AINP			GENMASK(7, 4)
@@ -117,9 +137,15 @@
 #define   ADS112C14_GPIO_CFG_GPIO2_CFG			GENMASK(5, 4)
 #define   ADS112C14_GPIO_CFG_GPIO1_CFG			GENMASK(3, 2)
 #define   ADS112C14_GPIO_CFG_GPIO0_CFG			GENMASK(1, 0)
+#define     ADS112C14_GPIO_CFG_GPIO_CFG_DISABLED	  0
+#define     ADS112C14_GPIO_CFG_GPIO_CFG_INPUT		  1
+#define     ADS112C14_GPIO_CFG_GPIO_CFG_OUTPUT_PUSH_PULL  2
+#define     ADS112C14_GPIO_CFG_GPIO_CFG_OUTPUT_OPEN_DRAIN 3
 
 #define ADS112C14_REG_GPIO_DATA_OUTPUT			0x0C
 #define   ADS112C14_GPIO_DATA_OUTPUT_GPIO3_SRC		BIT(7)
+#define     ADS112C14_GPIO_DATA_OUTPUT_GPIO3_SRC_DAT_OUT  0
+#define     ADS112C14_GPIO_DATA_OUTPUT_GPIO3_SRC_DRDY	  1
 #define   ADS112C14_GPIO_DATA_OUTPUT_GPIO2_SRC		BIT(6)
 #define   ADS112C14_GPIO_DATA_OUTPUT_GPIO3_DAT_OUT	BIT(3)
 #define   ADS112C14_GPIO_DATA_OUTPUT_GPIO2_DAT_OUT	BIT(2)
@@ -163,6 +189,63 @@ static const u32 ads112c14_pga_gains_x10[] = {
 	200, 320, 500, 640, 1000, 1280, 2000, 2560,	/* 8 - 15 */
 };
 
+#define ADS112C14_INTERNAL_CLK_Hz (4096 * HZ_PER_KHZ)
+
+/* Index corresponds to first 2 ADS112C14_DATA_RATE_CFG_FLTR_OSR values. */
+static const int ads112c14_sinc4_osr_available[] = {
+	16, 32
+};
+
+/* Index corresponds to next 4 ADS112C14_DATA_RATE_CFG_FLTR_OSR values. */
+static const int ads112c14_sinc4_sinc1_osr_available[] = {
+	128, 256, 512, 1024
+};
+
+/* Index corresponds to ADS112C14_DEVICE_CFG_SPEED_MODE value. */
+static const int ads112c14_sinc4_sinc1_pf1_20sps_osr_available[] = {
+	1600, 12800, 25600, 51200
+};
+
+/* Index corresponds to ADS112C14_DEVICE_CFG_SPEED_MODE value. */
+static const int ads112c14_sinc4_sinc1_pf1_25sps_osr_available[] = {
+	1280, 10240, 20480, 40960
+};
+
+/* Index corresponds to ADS112C14_DEVICE_CFG_SPEED_MODE value. */
+static const int ads112c14_fmod_div[] = {
+	128, 16, 8, 4
+};
+
+#define ADS112C14_DELAY_MAX FIELD_MAX(ADS112C14_DATA_RATE_CFG_DELAY)
+
+/* Table 7-6 latency in t_MOD for OSR [16, 32, 128, 256, 512, 1024]. */
+static const int ads112c14_sinc_latency_tmod[][ARRAY_SIZE(ads112c14_fmod_div)] = {
+	{ 80, 88, 88, 104 },
+	{ 144, 152, 152, 168 },
+	{ 240, 248, 248, 264 },
+	{ 368, 376, 376, 392 },
+	{ 624, 632, 632, 648 },
+	{ 1136, 1144, 1144, 1160 },
+};
+
+/* Table 7-7 latency in t_MOD for output data rates [25SPS, 20SPS]. */
+static const int ads112c14_fir_latency_tmod[][ARRAY_SIZE(ads112c14_fmod_div)] = {
+	{ 1416, 10384, 20624, 41120 },
+	{ 1736, 12944, 25744, 51360 },
+};
+
+enum ads112c14_filter_type {
+	ADS112C14_FILTER_TYPE_SINC4,
+	ADS112C14_FILTER_TYPE_SINC4_SINC1,
+	ADS112C14_FILTER_TYPE_SINC4_SINC1_PF1,
+};
+
+static const char * const ads112c14_filter_type_names[] = {
+	[ADS112C14_FILTER_TYPE_SINC4] = "sinc4",
+	[ADS112C14_FILTER_TYPE_SINC4_SINC1] = "sinc4+sinc1",
+	[ADS112C14_FILTER_TYPE_SINC4_SINC1_PF1] = "sinc4+sinc1+pf1",
+};
+
 #define ADS112C14_I2C_CRC8_POLYNOMIAL 0x07
 DECLARE_CRC8_TABLE(ads112c14_crc8_table);
 
@@ -184,6 +267,8 @@ enum {
 	ADS112C14_SYS_MON_CHANNEL_SHORT,
 };
 
+static const struct iio_chan_spec_ext_info ads112c14_ext_info[];
+
 static const struct iio_chan_spec ads112c14_sys_mon_channels[] = {
 	{
 		.type = IIO_TEMP,
@@ -192,7 +277,12 @@ static const struct iio_chan_spec ads112c14_sys_mon_channels[] = {
 		.address = 2,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)
 				    | BIT(IIO_CHAN_INFO_SCALE)
-				    | BIT(IIO_CHAN_INFO_OFFSET),
+				    | BIT(IIO_CHAN_INFO_OFFSET)
+				    | BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.ext_info = ads112c14_ext_info,
 	},
 	{
 		.type = IIO_VOLTAGE,
@@ -200,7 +290,12 @@ static const struct iio_chan_spec ads112c14_sys_mon_channels[] = {
 		.channel = ADS112C14_SYS_MON_CHANNEL_EXT_REF,
 		.address = 3,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)
-				    | BIT(IIO_CHAN_INFO_SCALE),
+				    | BIT(IIO_CHAN_INFO_SCALE)
+				    | BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.ext_info = ads112c14_ext_info,
 	},
 	{
 		.type = IIO_VOLTAGE,
@@ -208,7 +303,12 @@ static const struct iio_chan_spec ads112c14_sys_mon_channels[] = {
 		.channel = ADS112C14_SYS_MON_CHANNEL_AVDD,
 		.address = 4,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)
-				    | BIT(IIO_CHAN_INFO_SCALE),
+				    | BIT(IIO_CHAN_INFO_SCALE)
+				    | BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.ext_info = ads112c14_ext_info,
 	},
 	{
 		.type = IIO_VOLTAGE,
@@ -216,7 +316,12 @@ static const struct iio_chan_spec ads112c14_sys_mon_channels[] = {
 		.channel = ADS112C14_SYS_MON_CHANNEL_DVDD,
 		.address = 5,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)
-				    | BIT(IIO_CHAN_INFO_SCALE),
+				    | BIT(IIO_CHAN_INFO_SCALE)
+				    | BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.ext_info = ads112c14_ext_info,
 	},
 	{
 		.type = IIO_VOLTAGE,
@@ -226,8 +331,13 @@ static const struct iio_chan_spec ads112c14_sys_mon_channels[] = {
 		.differential = 1,
 		.address = 1,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)
-				    | BIT(IIO_CHAN_INFO_SCALE),
-		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE),
+				    | BIT(IIO_CHAN_INFO_SCALE)
+				    | BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE)
+				    | BIT(IIO_CHAN_INFO_SAMP_FREQ)
+				    | BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.ext_info = ads112c14_ext_info,
 	},
 };
 
@@ -241,16 +351,28 @@ struct ads112c14_measurement {
 	u8 idac2_mux;
 	u8 iadc_count;
 	u8 gain_val;
+	u8 burnout;
 	bool global_chop;
 	bool bipolar;
 	int scale_available[ARRAY_SIZE(ads112c14_pga_gains_x10)][2];
 };
 
+struct ads112c14_channel_state {
+	u8 speed_mode;
+	u8 filter_osr;
+	u8 delay;
+};
+
 struct ads112c14_data {
 	const struct ads112c14_chip_info *chip_info;
 	struct regmap *regmap;
+	struct iio_trigger *drdy_trig;
 	/* Synchronizes access to register value fields. */
 	struct mutex lock;
+	long fclk_Hz;
+	int drdy_irq;
+	struct completion drdy_completion;
+	bool continuous_mode;
 	bool i2c_crc_enabled;
 	u32 avdd_uV;
 	u32 ext_ref_uV;
@@ -258,13 +380,36 @@ struct ads112c14_data {
 	bool refn_is_gnd;
 	u32 ext_ref_ohms;
 	struct ads112c14_measurement *measurements;
+	struct ads112c14_channel_state *channel_states;
 	u32 num_measurements;
 	u8 sys_mon_chan_short_gain_val;
 	int sys_mon_chan_short_scale_available[ARRAY_SIZE(ads112c14_pga_gains_x10)][2];
+	int sinc4_sample_rate_available[ARRAY_SIZE(ads112c14_sinc4_osr_available)][ARRAY_SIZE(ads112c14_fmod_div)][2];
+	int sinc4_sinc1_sample_rate_available[ARRAY_SIZE(ads112c14_sinc4_sinc1_osr_available)][ARRAY_SIZE(ads112c14_fmod_div)][2];
+	int sinc4_sinc1_pf1_sample_rate_available[2][2];
+	int sinc_settling_time_range_available[ARRAY_SIZE(ads112c14_sinc_latency_tmod)][ARRAY_SIZE(ads112c14_fmod_div)][3][2];
+	int fir_settling_time_range_available[ARRAY_SIZE(ads112c14_fir_latency_tmod)][ARRAY_SIZE(ads112c14_fmod_div)][3][2];
 	IIO_DECLARE_BUFFER_WITH_TS(__be32, scan, ADS112C14_MAX_MEASUREMENT_CHANNELS +
 						 ARRAY_SIZE(ads112c14_sys_mon_channels));
 };
 
+static irqreturn_t ads112c14_drdy_irq_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct ads112c14_data *data = iio_priv(indio_dev);
+
+	if (READ_ONCE(data->continuous_mode))
+		iio_trigger_poll(data->drdy_trig);
+	else
+		complete(&data->drdy_completion);
+
+	return IRQ_HANDLED;
+}
+
+static const struct iio_trigger_ops ads112c14_trigger_ops = {
+	.validate_device = iio_trigger_validate_own_device,
+};
+
 static bool ads112c14_writeable_reg(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
@@ -429,13 +574,250 @@ static const struct regmap_config ads112c14_regmap_config = {
 	.cache_type = REGCACHE_MAPLE,
 };
 
+static int ads112c14_get_osr(struct ads112c14_channel_state *channel_state)
+{
+	u8 i;
+
+	switch (channel_state->filter_osr) {
+	case ADS112C14_DATA_RATE_CFG_FLTR_OSR_16...ADS112C14_DATA_RATE_CFG_FLTR_OSR_32:
+		i = channel_state->filter_osr;
+		return ads112c14_sinc4_osr_available[i];
+	case ADS112C14_DATA_RATE_CFG_FLTR_OSR_128...ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024:
+		i = channel_state->filter_osr - ADS112C14_DATA_RATE_CFG_FLTR_OSR_128;
+		return ads112c14_sinc4_sinc1_osr_available[i];
+	case ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS:
+		i = channel_state->speed_mode;
+		return ads112c14_sinc4_sinc1_pf1_25sps_osr_available[i];
+	case ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS:
+		i = channel_state->speed_mode;
+		return ads112c14_sinc4_sinc1_pf1_20sps_osr_available[i];
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ads112c14_get_fmod_Hz(struct ads112c14_data *data,
+				 struct ads112c14_channel_state *channel_state)
+{
+	return data->fclk_Hz / ads112c14_fmod_div[channel_state->speed_mode];
+}
+
+static int ads112c14_delay_to_tmod(u8 delay)
+{
+	if (!delay)
+		return 0;
+
+	return BIT(delay - 1);
+}
+
+static int ads112c14_get_latency_tmod(struct ads112c14_channel_state *channel_state)
+{
+	u8 speed_mode = channel_state->speed_mode;
+	u8 filter_osr = channel_state->filter_osr;
+
+	if (speed_mode >= ARRAY_SIZE(ads112c14_fmod_div))
+		return -EINVAL;
+
+	if (filter_osr <= ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024)
+		return ads112c14_sinc_latency_tmod[filter_osr][speed_mode];
+
+	if (filter_osr == ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS)
+		return ads112c14_fir_latency_tmod[0][speed_mode];
+
+	if (filter_osr == ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS)
+		return ads112c14_fir_latency_tmod[1][speed_mode];
+
+	return -EINVAL;
+}
+
+static int ads112c14_get_settling_time_us(struct ads112c14_data *data,
+					  struct ads112c14_channel_state *channel_state,
+					  u8 delay, u32 *settling_time_us)
+{
+	int fmod_Hz, latency_tmod;
+	u64 total_tmod;
+
+	fmod_Hz = ads112c14_get_fmod_Hz(data, channel_state);
+	if (fmod_Hz <= 0)
+		return -EINVAL;
+
+	latency_tmod = ads112c14_get_latency_tmod(channel_state);
+	if (latency_tmod < 0)
+		return latency_tmod;
+
+	total_tmod = latency_tmod + ads112c14_delay_to_tmod(delay);
+	*settling_time_us = div64_u64(total_tmod * USEC_PER_SEC, fmod_Hz);
+
+	return 0;
+}
+
+static int ads112c14_find_delay_for_settling_time_us(struct ads112c14_data *data,
+						     struct ads112c14_channel_state *channel_state,
+						     s64 settling_time_us, u8 *delay)
+{
+	u64 delay_us, delay_tmod_needed;
+	u32 fixed_latency_us;
+	int ret, fmod_Hz;
+	u8 i;
+
+	ret = ads112c14_get_settling_time_us(data, channel_state, 0, &fixed_latency_us);
+	if (ret)
+		return ret;
+
+	if (settling_time_us <= fixed_latency_us) {
+		*delay = 0;
+		return 0;
+	}
+
+	fmod_Hz = ads112c14_get_fmod_Hz(data, channel_state);
+	if (fmod_Hz <= 0)
+		return -EINVAL;
+
+	delay_us = settling_time_us - fixed_latency_us;
+	delay_tmod_needed = DIV_ROUND_UP_ULL(delay_us * fmod_Hz, USEC_PER_SEC);
+
+	for (i = 1; i < ADS112C14_DELAY_MAX; i++) {
+		if (ads112c14_delay_to_tmod(i) >= delay_tmod_needed)
+			break;
+	}
+
+	*delay = i;
+
+	return 0;
+}
+
+static ssize_t ads112c14_read_settling_time(struct iio_dev *indio_dev,
+					    uintptr_t private,
+					    const struct iio_chan_spec *chan,
+					    char *buf)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	struct ads112c14_channel_state *channel_state;
+	u32 settling_time_us;
+	int vals[2];
+	int ret;
+
+	guard(mutex)(&data->lock);
+
+	channel_state = &data->channel_states[chan->scan_index];
+
+	ret = ads112c14_get_settling_time_us(data, channel_state,
+					     channel_state->delay,
+					     &settling_time_us);
+	if (ret)
+		return ret;
+
+	iio_val_s64_decompose(settling_time_us, &vals[0], &vals[1]);
+
+	return iio_format_value(buf, IIO_VAL_DECIMAL64_MICRO, ARRAY_SIZE(vals), vals);
+}
+
+static ssize_t ads112c14_write_settling_time(struct iio_dev *indio_dev,
+					     uintptr_t private,
+					     const struct iio_chan_spec *chan,
+					     const char *buf, size_t len)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	struct ads112c14_channel_state *channel_state;
+	s64 settling_time_us;
+	int integer;
+	int fract;
+	u8 delay;
+	int ret;
+
+	ret = iio_str_to_fixpoint(buf, 100000, &integer, &fract);
+	if (ret)
+		return ret;
+
+	settling_time_us = integer * MICRO + fract;
+	if (settling_time_us < 0)
+		return -EINVAL;
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	guard(mutex)(&data->lock);
+
+	channel_state = &data->channel_states[chan->scan_index];
+
+	ret = ads112c14_find_delay_for_settling_time_us(data, channel_state,
+							settling_time_us, &delay);
+	if (ret)
+		return ret;
+
+	channel_state->delay = delay;
+
+	return len;
+}
+
+static ssize_t ads112c14_read_settling_time_available(struct iio_dev *indio_dev,
+						      uintptr_t private,
+						      const struct iio_chan_spec *chan,
+						      char *buf)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	struct ads112c14_channel_state *channel_state;
+	u8 filter_osr, speed_mode;
+	const int (*range)[2];
+	size_t len = 0;
+	int i;
+
+	guard(mutex)(&data->lock);
+
+	channel_state = &data->channel_states[chan->scan_index];
+
+	filter_osr = channel_state->filter_osr;
+	speed_mode = channel_state->speed_mode;
+
+	if (speed_mode >= ARRAY_SIZE(ads112c14_fmod_div))
+		return -EINVAL;
+
+	if (filter_osr <= ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024)
+		range = data->sinc_settling_time_range_available[filter_osr][speed_mode];
+	else if (filter_osr == ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS)
+		range = data->fir_settling_time_range_available[0][speed_mode];
+	else if (filter_osr == ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS)
+		range = data->fir_settling_time_range_available[1][speed_mode];
+	else
+		return -EINVAL;
+
+	len += sysfs_emit_at(buf, len, "[");
+	for (i = 0; i < 3; i++) {
+		s64 range_val;
+		s32 int_val, rem;
+
+		range_val = iio_val_s64_compose(range[i][0], range[i][1]);
+		int_val = div_s64_rem(range_val, MICRO, &rem);
+
+		if (i)
+			len += sysfs_emit_at(buf, len, " ");
+
+		len += sysfs_emit_at(buf, len, "%d.%06d", int_val, rem);
+	}
+	len += sysfs_emit_at(buf, len, "]\n");
+
+	return len;
+}
+
 static int ads112c14_prepare_measurement_channel(struct ads112c14_data *data,
-						 const struct iio_chan_spec *chan)
+						 const struct iio_chan_spec *chan,
+						 bool en_burnout)
 {
 	struct ads112c14_measurement *measurement = &data->measurements[chan->scan_index];
+	struct ads112c14_channel_state *channel_state;
 	u32 refp_buf_en, refn_buf_en, ref_val, ref_sel;
 	int ret;
 
+	channel_state = &data->channel_states[chan->scan_index];
+
+	ret = regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+				 ADS112C14_DEVICE_CFG_SPEED_MODE,
+				 FIELD_PREP(ADS112C14_DEVICE_CFG_SPEED_MODE,
+					    channel_state->speed_mode));
+	if (ret)
+		return ret;
+
 	ret = regmap_update_bits(data->regmap, ADS112C14_REG_MUX_CFG,
 				 ADS112C14_MUX_CFG_AINP | ADS112C14_MUX_CFG_AINN,
 				 FIELD_PREP(ADS112C14_MUX_CFG_AINP, chan->channel) |
@@ -482,9 +864,16 @@ static int ads112c14_prepare_measurement_channel(struct ads112c14_data *data,
 		return ret;
 
 	ret = regmap_update_bits(data->regmap, ADS112C14_REG_DATA_RATE_CFG,
-				 ADS112C14_DATA_RATE_CFG_GC_EN,
+				 ADS112C14_DATA_RATE_CFG_DELAY |
+				 ADS112C14_DATA_RATE_CFG_GC_EN |
+				 ADS112C14_DATA_RATE_CFG_FLTR_OSR,
+				 FIELD_PREP(ADS112C14_DATA_RATE_CFG_DELAY,
+					    channel_state->delay) |
 				 FIELD_PREP(ADS112C14_DATA_RATE_CFG_GC_EN,
-					    measurement->global_chop));
+					    (measurement->global_chop &&
+					     !en_burnout) ? 1 : 0) |
+				 FIELD_PREP(ADS112C14_DATA_RATE_CFG_FLTR_OSR,
+					    channel_state->filter_osr));
 	if (ret)
 		return ret;
 
@@ -527,9 +916,12 @@ static int ads112c14_prepare_measurement_channel(struct ads112c14_data *data,
 static int ads112c14_prepare_sys_mon_channel(struct ads112c14_data *data,
 					     const struct iio_chan_spec *chan)
 {
+	struct ads112c14_channel_state *channel_state;
 	u32 gain_val;
 	int ret;
 
+	channel_state = &data->channel_states[chan->scan_index];
+
 	/*
 	 * NB: IDAC registers are left as-is in case they are generating current
 	 * needed for the external reference measurement.
@@ -556,6 +948,25 @@ static int ads112c14_prepare_sys_mon_channel(struct ads112c14_data *data,
 	if (ret)
 		return ret;
 
+	ret = regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+				 ADS112C14_DEVICE_CFG_SPEED_MODE,
+				 FIELD_PREP(ADS112C14_DEVICE_CFG_SPEED_MODE,
+					    channel_state->speed_mode));
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(data->regmap, ADS112C14_REG_DATA_RATE_CFG,
+				 ADS112C14_DATA_RATE_CFG_DELAY |
+				 ADS112C14_DATA_RATE_CFG_GC_EN |
+				 ADS112C14_DATA_RATE_CFG_FLTR_OSR,
+				 FIELD_PREP(ADS112C14_DATA_RATE_CFG_DELAY,
+					    channel_state->delay) |
+				 FIELD_PREP(ADS112C14_DATA_RATE_CFG_GC_EN, 0) |
+				 FIELD_PREP(ADS112C14_DATA_RATE_CFG_FLTR_OSR,
+					    channel_state->filter_osr));
+	if (ret)
+		return ret;
+
 	/*
 	 * REVISIT: if we implement regulator support for the REFOUT pin, we
 	 * might need to make this voltage match what is required by that. In
@@ -581,35 +992,108 @@ static int ads112c14_prepare_sys_mon_channel(struct ads112c14_data *data,
 	return 0;
 }
 
+static int ads112c14_prepare_channel(struct ads112c14_data *data,
+				     const struct iio_chan_spec *chan,
+				     bool en_burnout)
+{
+	if (chan->channel < ADS112C14_SYS_MON_CHANNEL_BASE)
+		return ads112c14_prepare_measurement_channel(data, chan, en_burnout);
+
+	return ads112c14_prepare_sys_mon_channel(data, chan);
+}
+
+static int ads112c14_scan_read(struct ads112c14_data *data, u8 *buf)
+{
+	struct i2c_client *client = to_i2c_client(regmap_get_device(data->regmap));
+	int ret;
+	u8 len;
+
+	len = BITS_TO_BYTES(data->chip_info->resolution_bits);
+	if (data->i2c_crc_enabled)
+		len += 1;
+
+	ret = i2c_smbus_read_i2c_block_data(client, ADS112C14_CMD_RDATA, len, buf);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int ads112c14_wait_for_conversion_irq(struct ads112c14_data *data,
+					     u32 settle_time_us)
+{
+	unsigned long remaining;
+	int ret;
+
+	reinit_completion(&data->drdy_completion);
+
+	ret = regmap_write(data->regmap, ADS112C14_REG_CONVERSION_CTRL,
+			   ADS112C14_CONVERSION_CTRL_START);
+	if (ret)
+		return ret;
+
+	/* Give it 1ms more than calculated settling time. */
+	remaining = wait_for_completion_timeout(&data->drdy_completion,
+						usecs_to_jiffies(settle_time_us +
+								 1 * USEC_PER_MSEC));
+
+	return remaining ? 0 : -ETIMEDOUT;
+}
+
+static int ads112c14_wait_for_conversion_poll(struct ads112c14_data *data,
+					      u32 settle_time_us)
+{
+	u32 reg_val;
+	int ret;
+
+	ret = regmap_write(data->regmap, ADS112C14_REG_CONVERSION_CTRL,
+			   ADS112C14_CONVERSION_CTRL_START);
+	if (ret)
+		return ret;
+
+	/* Give it 1ms more than calculated settling time. */
+	return regmap_read_poll_timeout(data->regmap,
+					ADS112C14_REG_STATUS_MSB, reg_val,
+					FIELD_GET(ADS112C14_STATUS_MSB_DRDY, reg_val),
+					1 * USEC_PER_MSEC,
+					settle_time_us + 1 * USEC_PER_MSEC);
+}
+
 static int ads112c14_single_conversion(struct ads112c14_data *data,
 				       const struct iio_chan_spec *chan,
-				       u8 *buf, bool for_scan)
+				       u8 *buf, bool en_burnout, bool for_scan)
 {
 	struct i2c_client *client = to_i2c_client(regmap_get_device(data->regmap));
-	u32 reg_val;
+	struct ads112c14_channel_state *channel_state;
+	u32 settle_time_us;
 	int ret;
 
 	guard(mutex)(&data->lock);
 
-	if (chan->channel < ADS112C14_SYS_MON_CHANNEL_BASE) {
-		ret = ads112c14_prepare_measurement_channel(data, chan);
-		if (ret)
-			return ret;
-	} else {
-		ret = ads112c14_prepare_sys_mon_channel(data, chan);
-		if (ret)
-			return ret;
-	}
+	ret = ads112c14_prepare_channel(data, chan, en_burnout);
+	if (ret)
+		return ret;
 
-	ret = regmap_write(data->regmap, ADS112C14_REG_CONVERSION_CTRL,
-			   ADS112C14_CONVERSION_CTRL_START);
+	channel_state = &data->channel_states[chan->scan_index];
+	ret = ads112c14_get_settling_time_us(data, channel_state,
+					     channel_state->delay,
+					     &settle_time_us);
 	if (ret)
 		return ret;
 
-	ret = regmap_read_poll_timeout(data->regmap,
-				       ADS112C14_REG_STATUS_MSB, reg_val,
-				       FIELD_GET(ADS112C14_STATUS_MSB_DRDY, reg_val),
-				       1 * USEC_PER_MSEC, 100 * USEC_PER_MSEC);
+	ret = regmap_test_bits(data->regmap, ADS112C14_REG_DATA_RATE_CFG,
+			       ADS112C14_DATA_RATE_CFG_GC_EN);
+	if (ret < 0)
+		return ret;
+
+	/* Input chopping doubles the settling time. */
+	if (ret)
+		settle_time_us *= 2;
+
+	if (data->drdy_irq)
+		ret = ads112c14_wait_for_conversion_irq(data, settle_time_us);
+	else
+		ret = ads112c14_wait_for_conversion_poll(data, settle_time_us);
 	if (ret)
 		return ret;
 
@@ -619,23 +1103,21 @@ static int ads112c14_single_conversion(struct ads112c14_data *data,
 	 * with CRC errors, but rather leave it to userspace to decide what to
 	 * do.
 	 */
-	if (for_scan) {
-		u8 len = BITS_TO_BYTES(data->chip_info->resolution_bits) +
-			 (data->i2c_crc_enabled ? 1 : 0);
-
-		ret = i2c_smbus_read_i2c_block_data(client, ADS112C14_CMD_RDATA,
-						    len, buf);
-		if (ret < 0)
-			return ret;
-
-		return 0;
-	}
+	if (for_scan)
+		return ads112c14_scan_read(data, buf);
 
 	return ads112c14_i2c_read_bytes(client, ADS112C14_CMD_RDATA, buf,
 					BITS_TO_BYTES(data->chip_info->resolution_bits),
 					data->i2c_crc_enabled);
 }
 
+static bool ads112c14_using_drdy_trigger(struct iio_dev *indio_dev)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+
+	return data->drdy_trig && indio_dev->trig == data->drdy_trig;
+}
+
 static int ads112c14_read_raw(struct iio_dev *indio_dev,
 			      struct iio_chan_spec const *chan,
 			      int *val, int *val2, long mask)
@@ -665,7 +1147,7 @@ static int ads112c14_read_raw(struct iio_dev *indio_dev,
 		if (IIO_DEV_ACQUIRE_FAILED(claim))
 			return -EBUSY;
 
-		ret = ads112c14_single_conversion(data, chan, buf, false);
+		ret = ads112c14_single_conversion(data, chan, buf, false, false);
 		if (ret)
 			return ret;
 
@@ -743,6 +1225,45 @@ static int ads112c14_read_raw(struct iio_dev *indio_dev,
 		 */
 		*val = div_s64((s64)(25 * 405 - 119500) * BIT(fsr_bits), vref_uV);
 		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		struct ads112c14_channel_state *channel_state;
+		const int (*available)[2];
+		u8 i, j;
+
+		guard(mutex)(&data->lock);
+
+		channel_state = &data->channel_states[chan->scan_index];
+
+		switch (channel_state->filter_osr) {
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_16...ADS112C14_DATA_RATE_CFG_FLTR_OSR_32:
+			j = channel_state->filter_osr;
+			available = data->sinc4_sample_rate_available[j];
+			i = channel_state->speed_mode;
+			break;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_128...ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024:
+			j = channel_state->filter_osr - ADS112C14_DATA_RATE_CFG_FLTR_OSR_128;
+			available = data->sinc4_sinc1_sample_rate_available[j];
+			i = channel_state->speed_mode;
+			break;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS:
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS:
+			available = data->sinc4_sinc1_pf1_sample_rate_available;
+			i = channel_state->filter_osr - ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		*val = available[i][0];
+		*val2 = available[i][1];
+		return IIO_VAL_INT_PLUS_MICRO;
+	}
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO: {
+		guard(mutex)(&data->lock);
+
+		*val = ads112c14_get_osr(&data->channel_states[chan->scan_index]);
+		return IIO_VAL_INT;
+	}
 	default:
 		return -EINVAL;
 	}
@@ -753,6 +1274,9 @@ static int ads112c14_read_avail(struct iio_dev *indio_dev,
 				int *type, int *length, long mask)
 {
 	struct ads112c14_data *data = iio_priv(indio_dev);
+	struct ads112c14_channel_state *channel_state;
+
+	channel_state = &data->channel_states[chan->scan_index];
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SCALE:
@@ -778,22 +1302,74 @@ static int ads112c14_read_avail(struct iio_dev *indio_dev,
 		}
 
 		return -EINVAL;
-	default:
-		return -EINVAL;
-	}
-}
 
-static int ads112c14_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan, int val,
-			       int val2, long mask)
-{
-	struct ads112c14_data *data = iio_priv(indio_dev);
-	const int (*scale_avail)[2];
-	u8 *gain_val;
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		guard(mutex)(&data->lock);
 
-	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
-	if (IIO_DEV_ACQUIRE_FAILED(claim))
-		return -EBUSY;
+		switch (channel_state->filter_osr) {
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_16...ADS112C14_DATA_RATE_CFG_FLTR_OSR_32:
+			*vals = (const int *)data->sinc4_sample_rate_available[channel_state->filter_osr];
+			*length = 2 * ARRAY_SIZE(data->sinc4_sample_rate_available[0]);
+			*type = IIO_VAL_INT_PLUS_MICRO;
+			return IIO_AVAIL_LIST;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_128...ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024:
+			*vals = (const int *)data->sinc4_sinc1_sample_rate_available[channel_state->filter_osr - ADS112C14_DATA_RATE_CFG_FLTR_OSR_128];
+			*length = 2 * ARRAY_SIZE(data->sinc4_sinc1_sample_rate_available[0]);
+			*type = IIO_VAL_INT_PLUS_MICRO;
+			return IIO_AVAIL_LIST;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS...ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS:
+			*vals = (const int *)data->sinc4_sinc1_pf1_sample_rate_available;
+			*length = 2 * ARRAY_SIZE(data->sinc4_sinc1_pf1_sample_rate_available);
+			*type = IIO_VAL_INT_PLUS_MICRO;
+			return IIO_AVAIL_LIST;
+		default:
+			return -EINVAL;
+		}
+	}
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO: {
+		guard(mutex)(&data->lock);
+
+		switch (channel_state->filter_osr) {
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_16...ADS112C14_DATA_RATE_CFG_FLTR_OSR_32:
+			*vals = ads112c14_sinc4_osr_available;
+			*length = ARRAY_SIZE(ads112c14_sinc4_osr_available);
+			*type = IIO_VAL_INT;
+			return IIO_AVAIL_LIST;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_128...ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024:
+			*vals = ads112c14_sinc4_sinc1_osr_available;
+			*length = ARRAY_SIZE(ads112c14_sinc4_sinc1_osr_available);
+			*type = IIO_VAL_INT;
+			return IIO_AVAIL_LIST;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS:
+			*vals = ads112c14_sinc4_sinc1_pf1_25sps_osr_available;
+			*length = ARRAY_SIZE(ads112c14_sinc4_sinc1_pf1_25sps_osr_available);
+			*type = IIO_VAL_INT;
+			return IIO_AVAIL_LIST;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS:
+			*vals = ads112c14_sinc4_sinc1_pf1_20sps_osr_available;
+			*length = ARRAY_SIZE(ads112c14_sinc4_sinc1_pf1_20sps_osr_available);
+			*type = IIO_VAL_INT;
+			return IIO_AVAIL_LIST;
+		default:
+			return -EINVAL;
+		}
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ads112c14_write_raw(struct iio_dev *indio_dev,
+			       struct iio_chan_spec const *chan, int val,
+			       int val2, long mask)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	const int (*scale_avail)[2];
+	u8 *gain_val;
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SCALE: {
@@ -822,6 +1398,99 @@ static int ads112c14_write_raw(struct iio_dev *indio_dev,
 
 		return -EINVAL;
 	}
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		struct ads112c14_channel_state *channel_state;
+		const int (*available)[2];
+
+		guard(mutex)(&data->lock);
+
+		channel_state = &data->channel_states[chan->scan_index];
+
+		switch (channel_state->filter_osr) {
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_16...ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024:
+			if (channel_state->filter_osr < ADS112C14_DATA_RATE_CFG_FLTR_OSR_128) {
+				u8 idx = channel_state->filter_osr;
+
+				available = data->sinc4_sample_rate_available[idx];
+			} else {
+				u8 idx = channel_state->filter_osr - ADS112C14_DATA_RATE_CFG_FLTR_OSR_128;
+
+				available = data->sinc4_sinc1_sample_rate_available[idx];
+			}
+
+			for (unsigned int i = 0; i < ARRAY_SIZE(ads112c14_fmod_div); i++) {
+				if (val == available[i][0] && val2 == available[i][1]) {
+					channel_state->speed_mode = i;
+					return 0;
+				}
+			}
+
+			return -EINVAL;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS:
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS: {
+			available = data->sinc4_sinc1_pf1_sample_rate_available;
+
+			for (unsigned int i = 0; i < ARRAY_SIZE(data->sinc4_sinc1_pf1_sample_rate_available); i++) {
+				if (val == available[i][0] && val2 == available[i][1]) {
+					channel_state->filter_osr = i + ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS;
+					return 0;
+				}
+			}
+
+			return -EINVAL;
+		}
+		default:
+			return -EINVAL;
+		}
+	}
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO: {
+		struct ads112c14_channel_state *channel_state;
+
+		guard(mutex)(&data->lock);
+
+		channel_state = &data->channel_states[chan->scan_index];
+
+		switch (channel_state->filter_osr) {
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_16...ADS112C14_DATA_RATE_CFG_FLTR_OSR_32:
+			for (unsigned int i = 0; i < ARRAY_SIZE(ads112c14_sinc4_osr_available); i++) {
+				if (val == ads112c14_sinc4_osr_available[i]) {
+					channel_state->filter_osr = i;
+					return 0;
+				}
+			}
+
+			return -EINVAL;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_128...ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024:
+			for (unsigned int i = 0; i < ARRAY_SIZE(ads112c14_sinc4_sinc1_osr_available); i++) {
+				if (val == ads112c14_sinc4_sinc1_osr_available[i]) {
+					channel_state->filter_osr = i + ADS112C14_DATA_RATE_CFG_FLTR_OSR_128;
+					return 0;
+				}
+			}
+
+			return -EINVAL;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS:
+			for (unsigned int i = 0; i < ARRAY_SIZE(ads112c14_sinc4_sinc1_pf1_25sps_osr_available); i++) {
+				if (val == ads112c14_sinc4_sinc1_pf1_25sps_osr_available[i]) {
+					channel_state->speed_mode = i;
+					return 0;
+				}
+			}
+
+			return -EINVAL;
+		case ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS:
+			for (unsigned int i = 0; i < ARRAY_SIZE(ads112c14_sinc4_sinc1_pf1_20sps_osr_available); i++) {
+				if (val == ads112c14_sinc4_sinc1_pf1_20sps_osr_available[i]) {
+					channel_state->speed_mode = i;
+					return 0;
+				}
+			}
+
+			return -EINVAL;
+		default:
+			return -EINVAL;
+		}
+	}
 	default:
 		return -EINVAL;
 	}
@@ -839,6 +1508,19 @@ static int ads112c14_write_raw_get_fmt(struct iio_dev *indio_dev,
 	}
 }
 
+static int ads112c14_update_scan_mode(struct iio_dev *indio_dev,
+				      const unsigned long *scan_mask)
+{
+	/* Only continuous mode is limited to a single channel. */
+	if (!ads112c14_using_drdy_trigger(indio_dev))
+		return 0;
+
+	if (!iio_validate_scan_mask_onehot(indio_dev, scan_mask))
+		return -EINVAL;
+
+	return 0;
+}
+
 static int ads112c14_debugfs_reg_access(struct iio_dev *indio_dev,
 					unsigned int reg,
 					unsigned int writeval,
@@ -893,6 +1575,19 @@ static int ads112c14_read_label(struct iio_dev *indio_dev,
 	return sysfs_emit(label, "%s\n", label_source);
 }
 
+static const struct iio_chan_spec *
+ads112c14_first_active_channel(struct iio_dev *indio_dev)
+{
+	unsigned int scan_mask_len = iio_get_masklength(indio_dev);
+	unsigned int i;
+
+	i = find_first_bit(indio_dev->active_scan_mask, scan_mask_len);
+	if (i == scan_mask_len)
+		return NULL;
+
+	return &indio_dev->channels[i];
+}
+
 static irqreturn_t ads112c14_trigger_handler(int irq, void *private)
 {
 	struct iio_poll_func *pf = private;
@@ -902,12 +1597,32 @@ static irqreturn_t ads112c14_trigger_handler(int irq, void *private)
 	u32 i;
 	int ret;
 
+	if (ads112c14_using_drdy_trigger(indio_dev)) {
+		const struct iio_chan_spec *chan;
+
+		chan = ads112c14_first_active_channel(indio_dev);
+		if (!chan)
+			goto out;
+
+		ret = ads112c14_scan_read(data, (u8 *)&data->scan[0]);
+		if (ret) {
+			dev_err_once(indio_dev->dev.parent,
+				     "failed to read channel %d: %pe; additional errors will be suppressed\n",
+				     chan->channel, ERR_PTR(ret));
+			goto out;
+		}
+
+		iio_push_to_buffers_with_ts(indio_dev, data->scan,
+					    sizeof(data->scan), pf->timestamp);
+		goto out;
+	}
+
 	iio_for_each_active_channel(indio_dev, i) {
 		const struct iio_chan_spec *chan = &indio_dev->channels[i];
 
 		ret = ads112c14_single_conversion(data, chan,
 						  (u8 *)&data->scan[offset++],
-						  true);
+						  false, true);
 		if (ret) {
 			dev_err_once(indio_dev->dev.parent,
 				     "failed to read channel %d: %pe; additional errors will be suppressed\n",
@@ -929,10 +1644,258 @@ static const struct iio_info ads112c14_info = {
 	.read_avail = ads112c14_read_avail,
 	.write_raw = ads112c14_write_raw,
 	.write_raw_get_fmt = ads112c14_write_raw_get_fmt,
+	.update_scan_mode = ads112c14_update_scan_mode,
 	.debugfs_reg_access = ads112c14_debugfs_reg_access,
 	.read_label = ads112c14_read_label,
 };
 
+static int ads112c14_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	const struct iio_chan_spec *chan;
+	int ret;
+
+	if (!ads112c14_using_drdy_trigger(indio_dev))
+		return 0;
+
+	chan = ads112c14_first_active_channel(indio_dev);
+	if (!chan)
+		return -EINVAL;
+
+	guard(mutex)(&data->lock);
+
+	ret = ads112c14_prepare_channel(data, chan, false);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+				 ADS112C14_DEVICE_CFG_CONV_MODE,
+				 FIELD_PREP(ADS112C14_DEVICE_CFG_CONV_MODE,
+					    ADS112C14_DEVICE_CFG_CONV_MODE_CONTINUOUS));
+	if (ret)
+		return ret;
+
+	WRITE_ONCE(data->continuous_mode, true);
+
+	ret = regmap_write(data->regmap, ADS112C14_REG_CONVERSION_CTRL,
+			   ADS112C14_CONVERSION_CTRL_START);
+	if (ret) {
+		WRITE_ONCE(data->continuous_mode, false);
+		regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+				   ADS112C14_DEVICE_CFG_CONV_MODE,
+				   FIELD_PREP(ADS112C14_DEVICE_CFG_CONV_MODE,
+					      ADS112C14_DEVICE_CFG_CONV_MODE_SINGLE_SHOT));
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ads112c14_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	int ret;
+
+	if (!ads112c14_using_drdy_trigger(indio_dev))
+		return 0;
+
+	guard(mutex)(&data->lock);
+
+	WRITE_ONCE(data->continuous_mode, false);
+
+	ret = regmap_write(data->regmap, ADS112C14_REG_CONVERSION_CTRL,
+			   ADS112C14_CONVERSION_CTRL_STOP);
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+				  ADS112C14_DEVICE_CFG_CONV_MODE,
+				  FIELD_PREP(ADS112C14_DEVICE_CFG_CONV_MODE,
+					     ADS112C14_DEVICE_CFG_CONV_MODE_SINGLE_SHOT));
+}
+
+static const struct iio_buffer_setup_ops ads112c14_buffer_setup_ops = {
+	.postenable = ads112c14_buffer_postenable,
+	.predisable = ads112c14_buffer_predisable,
+};
+
+static ssize_t ads112c14_read_burnout_raw(struct iio_dev *indio_dev,
+					  uintptr_t private,
+					  struct iio_chan_spec const *chan,
+					  char *buf)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	struct ads112c14_measurement *measurement;
+	int ret, ret2, val;
+	u8 raw_buf[3];
+
+	if (chan->channel >= ADS112C14_SYS_MON_CHANNEL_BASE)
+		return -EINVAL;
+
+	measurement = &data->measurements[chan->scan_index];
+	if (!measurement->burnout)
+		return -EINVAL;
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	ret = regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+				 ADS112C14_DEVICE_CFG_BOCS,
+				 FIELD_PREP(ADS112C14_DEVICE_CFG_BOCS,
+					    measurement->burnout));
+	if (ret)
+		return ret;
+
+	ret = ads112c14_single_conversion(data, chan, raw_buf, true, false);
+
+	/*
+	 * Important to always turn off burnout current even if the conversion
+	 * fails so that it does not affect subsequent measurements.
+	 */
+	ret2 = regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+				  ADS112C14_DEVICE_CFG_BOCS,
+				  FIELD_PREP(ADS112C14_DEVICE_CFG_BOCS,
+					     ADS112C14_DEVICE_CFG_BOCS_DISABLED));
+	if (ret < 0)
+		return ret;
+	if (ret2)
+		return ret2;
+
+	switch (data->chip_info->resolution_bits) {
+	case 16:
+		val = get_unaligned_be16(raw_buf);
+		break;
+	case 24:
+		val = get_unaligned_be24(raw_buf);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (measurement->bipolar)
+		val = sign_extend32(val, data->chip_info->resolution_bits - 1);
+
+	return sysfs_emit(buf, "%d\n", val);
+}
+
+static int ads112c14_get_filter_type_from_state(struct ads112c14_channel_state *channel_state)
+{
+	switch (channel_state->filter_osr) {
+	case ADS112C14_DATA_RATE_CFG_FLTR_OSR_16...ADS112C14_DATA_RATE_CFG_FLTR_OSR_32:
+		return ADS112C14_FILTER_TYPE_SINC4;
+	case ADS112C14_DATA_RATE_CFG_FLTR_OSR_128...ADS112C14_DATA_RATE_CFG_FLTR_OSR_1024:
+		return ADS112C14_FILTER_TYPE_SINC4_SINC1;
+	case ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS...ADS112C14_DATA_RATE_CFG_FLTR_OSR_20SPS:
+		return ADS112C14_FILTER_TYPE_SINC4_SINC1_PF1;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ads112c14_set_filter_type(struct iio_dev *indio_dev,
+				     struct iio_chan_spec const *chan,
+				     unsigned int val)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	struct ads112c14_channel_state *channel_state;
+	int ret;
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	guard(mutex)(&data->lock);
+
+	channel_state = &data->channel_states[chan->scan_index];
+
+	ret = ads112c14_get_filter_type_from_state(channel_state);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * channel_state->filter_osr affects multiple attributes, so don't modify
+	 * it if the filter type is already set to the requested value.
+	 */
+	if (ret == val)
+		return 0;
+
+	/* Otherwise, pick an arbitrary default for each type. */
+	switch (val) {
+	case ADS112C14_FILTER_TYPE_SINC4:
+		channel_state->filter_osr = ADS112C14_DATA_RATE_CFG_FLTR_OSR_16;
+		break;
+	case ADS112C14_FILTER_TYPE_SINC4_SINC1:
+		channel_state->filter_osr = ADS112C14_DATA_RATE_CFG_FLTR_OSR_128;
+		break;
+	case ADS112C14_FILTER_TYPE_SINC4_SINC1_PF1:
+		channel_state->filter_osr = ADS112C14_DATA_RATE_CFG_FLTR_OSR_25SPS;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ads112c14_get_filter_type(struct iio_dev *indio_dev,
+				     struct iio_chan_spec const *chan)
+{
+	struct ads112c14_data *data = iio_priv(indio_dev);
+	struct ads112c14_channel_state *channel_state;
+
+	guard(mutex)(&data->lock);
+
+	channel_state = &data->channel_states[chan->scan_index];
+
+	return ads112c14_get_filter_type_from_state(channel_state);
+}
+
+static const struct iio_enum ads112c14_filter_type_enum = {
+	.items = ads112c14_filter_type_names,
+	.num_items = ARRAY_SIZE(ads112c14_filter_type_names),
+	.set = ads112c14_set_filter_type,
+	.get = ads112c14_get_filter_type,
+};
+
+static const struct iio_chan_spec_ext_info ads112c14_ext_info[] = {
+	{
+		.name = "settlingtime",
+		.read = ads112c14_read_settling_time,
+		.write = ads112c14_write_settling_time,
+		.shared = IIO_SEPARATE,
+	},
+	{
+		.name = "settlingtime_available",
+		.read = ads112c14_read_settling_time_available,
+		.shared = IIO_SEPARATE,
+	},
+	IIO_ENUM("filter_type", IIO_SEPARATE, &ads112c14_filter_type_enum),
+	IIO_ENUM_AVAILABLE("filter_type", IIO_SEPARATE, &ads112c14_filter_type_enum),
+	{ }
+};
+
+static const struct iio_chan_spec_ext_info ads112c14_ext_info_burnout[] = {
+	{
+		.name = "burnoutraw",
+		.read = ads112c14_read_burnout_raw,
+	},
+	{
+		.name = "settlingtime",
+		.read = ads112c14_read_settling_time,
+		.write = ads112c14_write_settling_time,
+		.shared = IIO_SEPARATE,
+	},
+	{
+		.name = "settlingtime_available",
+		.read = ads112c14_read_settling_time_available,
+		.shared = IIO_SEPARATE,
+	},
+	IIO_ENUM("filter_type", IIO_SEPARATE, &ads112c14_filter_type_enum),
+	IIO_ENUM_AVAILABLE("filter_type", IIO_SEPARATE, &ads112c14_filter_type_enum),
+	{ }
+};
+
 static int ads112c14_populate_idac_mag(u32 current_nA, u8 *idac_mag)
 {
 	u32 current_uA = current_nA / (NANO / MICRO);
@@ -955,7 +1918,7 @@ static int ads112c14_parse_channels(struct iio_dev *indio_dev,
 	struct ads112c14_data *data = iio_priv(indio_dev);
 	struct device *dev = indio_dev->dev.parent;
 	struct iio_chan_spec *channels;
-	u32 num_child_nodes, i, pair[2];
+	u32 num_child_nodes, num_data_chans, i, pair[2];
 	int ret;
 
 	*need_avdd_ref = false;
@@ -968,8 +1931,15 @@ static int ads112c14_parse_channels(struct iio_dev *indio_dev,
 	if (!data->measurements)
 		return -ENOMEM;
 
-	channels = devm_kcalloc(dev, num_child_nodes +
-				ARRAY_SIZE(ads112c14_sys_mon_channels) + 1,
+	num_data_chans = num_child_nodes + ARRAY_SIZE(ads112c14_sys_mon_channels);
+
+	data->channel_states = devm_kcalloc(dev, num_data_chans,
+					    sizeof(*data->channel_states),
+					    GFP_KERNEL);
+	if (!data->channel_states)
+		return -ENOMEM;
+
+	channels = devm_kcalloc(dev, num_data_chans + 1,
 				sizeof(*channels), GFP_KERNEL);
 	if (!channels)
 		return -ENOMEM;
@@ -978,10 +1948,13 @@ static int ads112c14_parse_channels(struct iio_dev *indio_dev,
 	device_for_each_named_child_node_scoped(dev, child, "channel") {
 		struct ads112c14_measurement *measurement = &data->measurements[i];
 		struct iio_chan_spec *spec = &channels[i];
+		const char *propname;
 
 		spec->indexed = 1;
 		spec->scan_index = i;
+		spec->ext_info = ads112c14_ext_info;
 		measurement->gain_val = 1;
+		data->channel_states[i].filter_osr = ADS112C14_DATA_RATE_CFG_FLTR_OSR_16;
 
 		if (fwnode_property_present(child, "label")) {
 			ret = fwnode_property_read_string(child, "label", &measurement->label);
@@ -1101,6 +2074,35 @@ static int ads112c14_parse_channels(struct iio_dev *indio_dev,
 		measurement->global_chop = fwnode_property_read_bool(child,
 								     "input-chopping");
 
+		propname = "burn-out-current-nanoamp";
+		if (fwnode_property_present(child, propname)) {
+			u32 burnout_nA;
+
+			ret = fwnode_property_read_u32(child, propname, &burnout_nA);
+			if (ret)
+				return dev_err_probe(dev, ret,
+						     "failed to read %s property\n",
+						     propname);
+
+			switch (burnout_nA) {
+			case 200:
+				measurement->burnout = ADS112C14_DEVICE_CFG_BOCS_200_nA;
+				break;
+			case 1000:
+				measurement->burnout = ADS112C14_DEVICE_CFG_BOCS_1_uA;
+				break;
+			case 10000:
+				measurement->burnout = ADS112C14_DEVICE_CFG_BOCS_10_uA;
+				break;
+			default:
+				return dev_err_probe(dev, -EINVAL,
+						     "invalid %s value\n", propname);
+			}
+
+			if (measurement->burnout != ADS112C14_DEVICE_CFG_BOCS_DISABLED)
+				spec->ext_info = ads112c14_ext_info_burnout;
+		}
+
 		if (fwnode_property_present(child, "reference-sources")) {
 			ret = fwnode_property_match_property_string(child,
 				"reference-sources", ads112c14_vref_source_names,
@@ -1117,8 +2119,13 @@ static int ads112c14_parse_channels(struct iio_dev *indio_dev,
 		if (measurement->vref_source == ADS112C14_VREF_SOURCE_EXTERNAL)
 			*need_ext_ref = true;
 
-		spec->info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE);
-		spec->info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE);
+		spec->info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+					   BIT(IIO_CHAN_INFO_SCALE) |
+					   BIT(IIO_CHAN_INFO_SAMP_FREQ) |
+					   BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO);
+		spec->info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE) |
+						     BIT(IIO_CHAN_INFO_SAMP_FREQ) |
+						     BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO);
 
 		/*
 		 * If reference source is resistor rather than voltage supply,
@@ -1152,6 +2159,10 @@ static int ads112c14_parse_channels(struct iio_dev *indio_dev,
 
 	for (u32 j = 0; j < ARRAY_SIZE(ads112c14_sys_mon_channels); j++) {
 		struct iio_chan_spec *spec = &channels[i];
+		struct ads112c14_channel_state *channel_state;
+
+		channel_state = &data->channel_states[i];
+		channel_state->filter_osr = ADS112C14_DATA_RATE_CFG_FLTR_OSR_16;
 
 		/* Update the template that was already copied with dynamic values. */
 		spec->scan_index = i;
@@ -1190,6 +2201,94 @@ static void ads112c14_populate_scale_available(int (*scale_avail)[2],
 	}
 }
 
+static void ads112c14_populate_odr_tables(struct ads112c14_data *data)
+{
+	int *available;
+	u32 osr, fmod_Hz;
+	u64 odr_uHz;
+	u32 rem;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(ads112c14_sinc4_osr_available); i++) {
+		osr = ads112c14_sinc4_osr_available[i];
+
+		for (unsigned int j = 0; j < ARRAY_SIZE(ads112c14_fmod_div); j++) {
+			fmod_Hz = data->fclk_Hz / ads112c14_fmod_div[j];
+			odr_uHz = div_u64((u64)fmod_Hz * MICRO, osr);
+			available = data->sinc4_sample_rate_available[i][j];
+			available[0] = div_u64_rem(odr_uHz, MICRO, &rem);
+			available[1] = rem;
+		}
+	}
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(ads112c14_sinc4_sinc1_osr_available); i++) {
+		osr = ads112c14_sinc4_sinc1_osr_available[i];
+
+		for (unsigned int j = 0; j < ARRAY_SIZE(ads112c14_fmod_div); j++) {
+			fmod_Hz = data->fclk_Hz / ads112c14_fmod_div[j];
+			odr_uHz = div_u64((u64)fmod_Hz * MICRO, osr);
+			available = data->sinc4_sinc1_sample_rate_available[i][j];
+			available[0] = div_u64_rem(odr_uHz, MICRO, &rem);
+			available[1] = rem;
+		}
+	}
+
+	odr_uHz = div_u64((u64)25 * data->fclk_Hz * MICRO, ADS112C14_INTERNAL_CLK_Hz);
+	available = data->sinc4_sinc1_pf1_sample_rate_available[0];
+	available[0] = div_u64_rem(odr_uHz, MICRO, &rem);
+	available[1] = rem;
+
+	odr_uHz = div_u64((u64)20 * data->fclk_Hz * MICRO, ADS112C14_INTERNAL_CLK_Hz);
+	available = data->sinc4_sinc1_pf1_sample_rate_available[1];
+	available[0] = div_u64_rem(odr_uHz, MICRO, &rem);
+	available[1] = rem;
+}
+
+static void ads112c14_populate_settling_range_tables(struct ads112c14_data *data)
+{
+	s32 (*avail)[2];
+	u32 i, j;
+
+	for (i = 0; i < ARRAY_SIZE(ads112c14_sinc_latency_tmod); i++) {
+		for (j = 0; j < ARRAY_SIZE(ads112c14_fmod_div); j++) {
+			u64 fmod_Hz = data->fclk_Hz / ads112c14_fmod_div[j];
+			u64 start_tmod = ads112c14_sinc_latency_tmod[i][j];
+			u64 step_tmod = ads112c14_delay_to_tmod(1);
+			u64 stop_tmod = start_tmod + ads112c14_delay_to_tmod(ADS112C14_DELAY_MAX);
+			s64 start_us, step_us, stop_us;
+
+			start_us = DIV_ROUND_CLOSEST_ULL(start_tmod * USEC_PER_SEC, fmod_Hz);
+			step_us = DIV_ROUND_CLOSEST_ULL(step_tmod * USEC_PER_SEC, fmod_Hz);
+			stop_us = DIV_ROUND_CLOSEST_ULL(stop_tmod * USEC_PER_SEC, fmod_Hz);
+
+			avail = data->sinc_settling_time_range_available[i][j];
+
+			iio_val_s64_decompose(start_us, &avail[0][0], &avail[0][1]);
+			iio_val_s64_decompose(step_us, &avail[1][0], &avail[1][1]);
+			iio_val_s64_decompose(stop_us, &avail[2][0], &avail[2][1]);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ads112c14_fir_latency_tmod); i++) {
+		for (j = 0; j < ARRAY_SIZE(ads112c14_fmod_div); j++) {
+			u64 fmod_Hz = data->fclk_Hz / ads112c14_fmod_div[j];
+			u64 start_tmod = ads112c14_fir_latency_tmod[i][j];
+			u64 step_tmod = ads112c14_delay_to_tmod(1);
+			u64 stop_tmod = start_tmod + ads112c14_delay_to_tmod(ADS112C14_DELAY_MAX);
+			s64 start_us, step_us, stop_us;
+
+			start_us = DIV_ROUND_CLOSEST_ULL(start_tmod * USEC_PER_SEC, fmod_Hz);
+			step_us = DIV_ROUND_CLOSEST_ULL(step_tmod * USEC_PER_SEC, fmod_Hz);
+			stop_us = DIV_ROUND_CLOSEST_ULL(stop_tmod * USEC_PER_SEC, fmod_Hz);
+
+			avail = data->fir_settling_time_range_available[i][j];
+
+			iio_val_s64_decompose(start_us, &avail[0][0], &avail[0][1]);
+			iio_val_s64_decompose(step_us, &avail[1][0], &avail[1][1]);
+			iio_val_s64_decompose(stop_us, &avail[2][0], &avail[2][1]);
+		}
+	}
+}
+
 static void ads112c14_populate_tables(struct ads112c14_data *data)
 {
 	u32 full_scale, fsr_bits;
@@ -1227,6 +2326,8 @@ static void ads112c14_populate_tables(struct ads112c14_data *data)
 
 	ads112c14_populate_scale_available(data->sys_mon_chan_short_scale_available,
 					   full_scale, fsr_bits);
+	ads112c14_populate_odr_tables(data);
+	ads112c14_populate_settling_range_tables(data);
 }
 
 static int ads112c14_probe(struct i2c_client *client)
@@ -1235,6 +2336,7 @@ static int ads112c14_probe(struct i2c_client *client)
 	const struct ads112c14_chip_info *info;
 	struct iio_dev *indio_dev;
 	struct ads112c14_data *data;
+	struct clk *clk;
 	bool need_avdd_ref, need_ext_ref;
 	u32 refp_uV = 0;
 	u32 refn_uV = 0;
@@ -1327,6 +2429,18 @@ static int ads112c14_probe(struct i2c_client *client)
 		return dev_err_probe(dev, -EINVAL,
 				     "external reference measurements require either refp-supply or ti,refp-refn-resistor-ohms property\n");
 
+	clk = devm_clk_get_optional_enabled(dev, NULL);
+	if (IS_ERR(clk))
+		return dev_err_probe(dev, PTR_ERR(clk), "failed to get clk\n");
+
+	if (clk) {
+		data->fclk_Hz = clk_get_rate(clk);
+		if (!data->fclk_Hz)
+			return dev_err_probe(dev, -EINVAL, "clk rate is 0\n");
+	} else {
+		data->fclk_Hz = ADS112C14_INTERNAL_CLK_Hz;
+	}
+
 	/* It takes some time for the internal reference to stabilize. */
 	fsleep(10 * USEC_PER_MSEC);
 
@@ -1391,6 +2505,71 @@ static int ads112c14_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
+	if (fwnode_property_match_string(dev_fwnode(dev), "interrupt-names", "drdy") >= 0) {
+		data->drdy_irq = fwnode_irq_get_byname(dev_fwnode(dev), "drdy");
+		if (data->drdy_irq < 0)
+			return dev_err_probe(dev, data->drdy_irq,
+					     "failed to get drdy interrupt\n");
+
+		if (clk)
+			return dev_err_probe(dev, -EINVAL,
+					     "cannot use both DRDY and CLK - they share the same pin\n");
+
+		/*
+		 * REVISIT: would probably need to implement a pin controller in
+		 * order to support open drain option here.
+		 */
+		ret = regmap_update_bits(data->regmap, ADS112C14_REG_GPIO_CFG,
+					 ADS112C14_GPIO_CFG_GPIO3_CFG,
+					 FIELD_PREP(ADS112C14_GPIO_CFG_GPIO3_CFG,
+						    ADS112C14_GPIO_CFG_GPIO_CFG_OUTPUT_PUSH_PULL));
+		if (ret)
+			return ret;
+
+		ret = regmap_update_bits(data->regmap, ADS112C14_REG_GPIO_DATA_OUTPUT,
+					 ADS112C14_GPIO_DATA_OUTPUT_GPIO3_SRC,
+					 FIELD_PREP(ADS112C14_GPIO_DATA_OUTPUT_GPIO3_SRC,
+						    ADS112C14_GPIO_DATA_OUTPUT_GPIO3_SRC_DRDY));
+		if (ret)
+			return ret;
+
+		init_completion(&data->drdy_completion);
+
+		data->drdy_trig = devm_iio_trigger_alloc(dev, "%s-dev%d-drdy",
+							 info->name,
+							 iio_device_id(indio_dev));
+		if (!data->drdy_trig)
+			return -ENOMEM;
+
+		data->drdy_trig->ops = &ads112c14_trigger_ops;
+		iio_trigger_set_drvdata(data->drdy_trig, indio_dev);
+
+		ret = devm_iio_trigger_register(dev, data->drdy_trig);
+		if (ret)
+			return ret;
+
+		ret = devm_request_irq(dev, data->drdy_irq, ads112c14_drdy_irq_handler,
+				       0, dev_name(dev), indio_dev);
+		if (ret)
+			return ret;
+	}
+
+	if (clk) {
+		ret = regmap_update_bits(data->regmap, ADS112C14_REG_GPIO_CFG,
+					 ADS112C14_GPIO_CFG_GPIO3_CFG,
+					 FIELD_PREP(ADS112C14_GPIO_CFG_GPIO3_CFG,
+						    ADS112C14_GPIO_CFG_GPIO_CFG_INPUT));
+		if (ret)
+			return ret;
+
+		ret = regmap_update_bits(data->regmap, ADS112C14_REG_DEVICE_CFG,
+					 ADS112C14_DEVICE_CFG_CLK_SEL,
+					 FIELD_PREP(ADS112C14_DEVICE_CFG_CLK_SEL,
+						    ADS112C14_DEVICE_CFG_CLK_SEL_EXTERNAL));
+		if (ret)
+			return ret;
+	}
+
 	ads112c14_populate_tables(data);
 
 	indio_dev->name = info->name;
@@ -1399,7 +2578,8 @@ static int ads112c14_probe(struct i2c_client *client)
 
 	ret = devm_iio_triggered_buffer_setup(dev, indio_dev,
 					      iio_pollfunc_store_time,
-					      ads112c14_trigger_handler, NULL);
+					      ads112c14_trigger_handler,
+					      &ads112c14_buffer_setup_ops);
 	if (ret)
 		return ret;
 
diff --git a/drivers/iio/adc/ti-ads1298.c b/drivers/iio/adc/ti-ads1298.c
index 06c6353aa1b2e0..33cf1fbd994723 100644
--- a/drivers/iio/adc/ti-ads1298.c
+++ b/drivers/iio/adc/ti-ads1298.c
@@ -10,6 +10,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
 #include <linux/log2.h>
 #include <linux/math.h>
 #include <linux/module.h>
diff --git a/drivers/iio/adc/ti-ads131m02.c b/drivers/iio/adc/ti-ads131m02.c
index 2f8f75c8216b29..7ecfb27fd7ef09 100644
--- a/drivers/iio/adc/ti-ads131m02.c
+++ b/drivers/iio/adc/ti-ads131m02.c
@@ -897,6 +897,8 @@ static int ads131m_probe(struct spi_device *spi)
 	indio_dev->info = &ads131m_info;
 
 	config = spi_get_device_match_data(spi);
+	if (!config)
+		return -ENODATA;
 
 	priv->config = config;
 	indio_dev->name = config->name;
diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c
index d27c1f2885fd6a..0d5708e0b831d0 100644
--- a/drivers/iio/adc/ti-ads7950.c
+++ b/drivers/iio/adc/ti-ads7950.c
@@ -527,8 +527,10 @@ static int ti_ads7950_probe(struct spi_device *spi)
 	st->spi = spi;
 
 	info = spi_get_device_match_data(spi);
+	if (!info)
+		return -ENODATA;
 
-	indio_dev->name = spi_get_device_id(spi)->name;
+	indio_dev->name = spi->modalias;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = info->channels;
 	indio_dev->num_channels = info->num_channels;
diff --git a/drivers/iio/adc/ti_am335x_adc.c b/drivers/iio/adc/ti_am335x_adc.c
index 1516dd332f905c..e53c839e87ef3a 100644
--- a/drivers/iio/adc/ti_am335x_adc.c
+++ b/drivers/iio/adc/ti_am335x_adc.c
@@ -548,7 +548,7 @@ static int tiadc_request_dma(struct platform_device *pdev,
 	}
 
 	/* RX buffer */
-	dma->buf = dma_alloc_coherent(dma->chan->device->dev, DMA_BUFFER_SIZE,
+	dma->buf = dma_alloc_coherent(dmaengine_get_dma_device(dma->chan), DMA_BUFFER_SIZE,
 				      &dma->addr, GFP_KERNEL);
 	if (!dma->buf)
 		goto err;
@@ -688,7 +688,7 @@ static void tiadc_remove(struct platform_device *pdev)
 	u32 step_en;
 
 	if (dma->chan) {
-		dma_free_coherent(dma->chan->device->dev, DMA_BUFFER_SIZE,
+		dma_free_coherent(dmaengine_get_dma_device(dma->chan), DMA_BUFFER_SIZE,
 				  dma->buf, dma->addr);
 		dma_release_channel(dma->chan);
 	}
diff --git a/drivers/iio/adc/xilinx-xadc-core.c b/drivers/iio/adc/xilinx-xadc-core.c
index cab66bb8cc1cca..66202b3a3abbc1 100644
--- a/drivers/iio/adc/xilinx-xadc-core.c
+++ b/drivers/iio/adc/xilinx-xadc-core.c
@@ -1401,13 +1401,14 @@ static int xadc_probe(struct platform_device *pdev)
 	}
 
 	if (irq > 0) {
-		ret = devm_request_irq(dev, irq, xadc->ops->interrupt_handler,
-				       0, dev_name(dev), indio_dev);
+		/* Clear out any work queued by the interrupt */
+		ret = devm_add_action_or_reset(dev, xadc_cancel_delayed_work,
+					       &xadc->zynq_unmask_work);
 		if (ret)
 			return ret;
 
-		ret = devm_add_action_or_reset(dev, xadc_cancel_delayed_work,
-					       &xadc->zynq_unmask_work);
+		ret = devm_request_irq(dev, irq, xadc->ops->interrupt_handler,
+				       0, dev_name(dev), indio_dev);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/iio/amplifiers/ad8366.c b/drivers/iio/amplifiers/ad8366.c
index e12f8604f59f48..9adff120de1abd 100644
--- a/drivers/iio/amplifiers/ad8366.c
+++ b/drivers/iio/amplifiers/ad8366.c
@@ -326,6 +326,8 @@ static int ad8366_probe(struct spi_device *spi)
 
 	st->spi = spi;
 	st->info = spi_get_device_match_data(spi);
+	if (!st->info)
+		return -ENODATA;
 
 	enable_gpio = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH);
 	if (IS_ERR(enable_gpio))
diff --git a/drivers/iio/buffer/industrialio-buffer-dmaengine.c b/drivers/iio/buffer/industrialio-buffer-dmaengine.c
index 1bd82d3db5ea5c..b9f8b5b3aa6b65 100644
--- a/drivers/iio/buffer/industrialio-buffer-dmaengine.c
+++ b/drivers/iio/buffer/industrialio-buffer-dmaengine.c
@@ -88,9 +88,6 @@ static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue,
 	unsigned int i;
 	int nents;
 
-	max_size = min(block->size, dmaengine_buffer->max_size);
-	max_size = round_down(max_size, dmaengine_buffer->align);
-
 	if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN)
 		dma_dir = DMA_DEV_TO_MEM;
 	else
@@ -104,10 +101,13 @@ static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue,
 	if (block->sg_table) {
 		unsigned long flags;
 
-		sgl = block->sg_table->sgl;
-		nents = sg_nents_for_len(sgl, block->bytes_used);
-		if (nents < 0)
-			return nents;
+		/*
+		 * Only the first sgt->nents entries carry a valid
+		 * sg_dma_address()/sg_dma_len() pair as mapping the table may
+		 * have coalesced entries, in which case nents is smaller than
+		 * orig_nents.
+		 */
+		nents = block->sg_table->nents;
 
 		vecs = kmalloc_objs(*vecs, nents, GFP_ATOMIC);
 		if (!vecs)
@@ -115,7 +115,8 @@ static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue,
 
 		len_total = block->bytes_used;
 
-		for (i = 0; i < nents; i++) {
+		sgl = block->sg_table->sgl;
+		for (i = 0; i < nents && len_total; i++) {
 			vecs[i].addr = sg_dma_address(sgl);
 			vecs[i].len = min(sg_dma_len(sgl), len_total);
 			len_total -= vecs[i].len;
@@ -123,6 +124,8 @@ static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue,
 			sgl = sg_next(sgl);
 		}
 
+		nents = i;
+
 		if (block->cyclic)
 			flags = DMA_PREP_REPEAT;
 		else
@@ -251,6 +254,7 @@ static const struct iio_dev_attr *iio_dmaengine_buffer_attrs[] = {
  */
 static struct iio_buffer *iio_dmaengine_buffer_alloc(struct dma_chan *chan)
 {
+	struct device *dma_dev = dmaengine_get_dma_device(chan);
 	struct dmaengine_buffer *dmaengine_buffer;
 	unsigned int width, src_width, dest_width;
 	struct dma_slave_caps caps;
@@ -278,9 +282,9 @@ static struct iio_buffer *iio_dmaengine_buffer_alloc(struct dma_chan *chan)
 	INIT_LIST_HEAD(&dmaengine_buffer->active);
 	dmaengine_buffer->chan = chan;
 	dmaengine_buffer->align = width;
-	dmaengine_buffer->max_size = dma_get_max_seg_size(chan->device->dev);
+	dmaengine_buffer->max_size = dma_get_max_seg_size(dma_dev);
 
-	iio_dma_buffer_init(&dmaengine_buffer->queue, chan->device->dev,
+	iio_dma_buffer_init(&dmaengine_buffer->queue, dma_dev,
 			    &iio_dmaengine_default_ops);
 
 	dmaengine_buffer->queue.buffer.attrs = iio_dmaengine_buffer_attrs;
diff --git a/drivers/iio/cdc/ad7150.c b/drivers/iio/cdc/ad7150.c
index 2f35c6d2f9cee5..b36ac4e2dae800 100644
--- a/drivers/iio/cdc/ad7150.c
+++ b/drivers/iio/cdc/ad7150.c
@@ -636,11 +636,13 @@ static const struct i2c_device_id ad7150_id[] = {
 MODULE_DEVICE_TABLE(i2c, ad7150_id);
 
 static const struct of_device_id ad7150_of_match[] = {
-	{ "adi,ad7150" },
-	{ "adi,ad7151" },
-	{ "adi,ad7156" },
+	{ .compatible = "adi,ad7150" },
+	{ .compatible = "adi,ad7151" },
+	{ .compatible = "adi,ad7156" },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, ad7150_of_match);
+
 static struct i2c_driver ad7150_driver = {
 	.driver = {
 		.name = "ad7150",
diff --git a/drivers/iio/chemical/ens160_core.c b/drivers/iio/chemical/ens160_core.c
index 7c1ffda38b99c3..5e2b15463c99e2 100644
--- a/drivers/iio/chemical/ens160_core.c
+++ b/drivers/iio/chemical/ens160_core.c
@@ -283,12 +283,7 @@ static int ens160_set_trigger_state(struct iio_trigger *trig, bool state)
 				ENS160_REG_CONFIG_INTDAT |
 				ENS160_REG_CONFIG_INT_CFG;
 
-	if (state)
-		return regmap_set_bits(data->regmap, ENS160_REG_CONFIG,
-				       int_bits);
-	else
-		return regmap_clear_bits(data->regmap, ENS160_REG_CONFIG,
-					 int_bits);
+	return regmap_assign_bits(data->regmap, ENS160_REG_CONFIG, int_bits, state);
 }
 
 static const struct iio_trigger_ops ens160_trigger_ops = {
diff --git a/drivers/iio/chemical/sps30.c b/drivers/iio/chemical/sps30.c
index 8e15baa314230e..b47d08ab4d0f75 100644
--- a/drivers/iio/chemical/sps30.c
+++ b/drivers/iio/chemical/sps30.c
@@ -15,6 +15,7 @@
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
 #include <linux/kernel.h>
+#include <linux/kstrtox.h>
 #include <linux/module.h>
 
 #include "sps30.h"
@@ -193,7 +194,10 @@ static ssize_t start_cleaning_store(struct device *dev,
 	struct sps30_state *state = iio_priv(indio_dev);
 	int val, ret;
 
-	if (kstrtoint(buf, 0, &val) || val != 1)
+	ret = kstrtoint(buf, 0, &val);
+	if (ret)
+		return ret;
+	if (val != 1)
 		return -EINVAL;
 
 	guard(mutex)(&state->lock);
@@ -230,8 +234,9 @@ static ssize_t cleaning_period_store(struct device *dev, struct device_attribute
 	struct sps30_state *state = iio_priv(indio_dev);
 	int val, ret;
 
-	if (kstrtoint(buf, 0, &val))
-		return -EINVAL;
+	ret = kstrtoint(buf, 0, &val);
+	if (ret)
+		return ret;
 
 	if ((val < SPS30_AUTO_CLEANING_PERIOD_MIN) ||
 	    (val > SPS30_AUTO_CLEANING_PERIOD_MAX))
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
index b971f8b646be53..551b86236abcca 100644
--- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
@@ -311,7 +311,7 @@ MODULE_DEVICE_TABLE(platform, cros_ec_sensors_ids);
 static struct platform_driver cros_ec_sensors_platform_driver = {
 	.driver = {
 		.name	= "cros-ec-sensors",
-		.pm	= &cros_ec_sensors_pm_ops,
+		.pm	= pm_sleep_ptr(&cros_ec_sensors_pm_ops),
 	},
 	.probe		= cros_ec_sensors_probe,
 	.id_table	= cros_ec_sensors_ids,
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
index 5133755c2ea61c..d03e40c9158266 100644
--- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors_core.c
@@ -862,7 +862,7 @@ int cros_ec_sensors_core_write(struct cros_ec_sensors_core_state *st,
 }
 EXPORT_SYMBOL_GPL(cros_ec_sensors_core_write);
 
-static int __maybe_unused cros_ec_sensors_resume(struct device *dev)
+static int cros_ec_sensors_resume(struct device *dev)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(dev);
 	struct cros_ec_sensors_core_state *st = iio_priv(indio_dev);
@@ -879,8 +879,7 @@ static int __maybe_unused cros_ec_sensors_resume(struct device *dev)
 	return ret;
 }
 
-SIMPLE_DEV_PM_OPS(cros_ec_sensors_pm_ops, NULL, cros_ec_sensors_resume);
-EXPORT_SYMBOL_GPL(cros_ec_sensors_pm_ops);
+EXPORT_GPL_SIMPLE_DEV_PM_OPS(cros_ec_sensors_pm_ops, NULL, cros_ec_sensors_resume);
 
 MODULE_DESCRIPTION("ChromeOS EC sensor hub core functions");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c b/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c
index 88a82d1370c504..8c9e81a77e6887 100644
--- a/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c
+++ b/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c
@@ -167,11 +167,16 @@ void inv_sensors_timestamp_interrupt(struct inv_sensors_timestamp *ts,
 		valid = inv_validate_period(ts, period);
 	}
 
-	/* if interrupt interval is valid, sync with interrupt timestamp */
+	/*
+	 * If interrupt interval is valid, sync with interrupt timestamp.
+	 * Otherwise, use estimated value while ensuring interrupt timestamp
+	 * remains the maximum possible value.
+	 */
+	period = inv_align_timestamp_it(ts, sample_nb);
 	if (valid)
-		ts->period = inv_align_timestamp_it(ts, sample_nb);
+		ts->period = period;
 	else
-		ts->period = ts->mult * ts->chip_period.val;
+		ts->period = min(ts->mult * ts->chip_period.val, period);
 }
 EXPORT_SYMBOL_NS_GPL(inv_sensors_timestamp_interrupt, "IIO_INV_SENSORS_TIMESTAMP");
 
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index 3b3c77d8b3e1b6..e373885e7c7a13 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -16,6 +16,8 @@ config AD3530R
 	  - AD3530/AD3530R (8-channel)
 	  - AD3531/AD3531R (4-channel)
 	  - AD3532/AD3532R (16-channel)
+	  - AD3536R (16-channel, 12-bit)
+	  - AD5710R/AD5711R (8-channel configurable IDAC/VDAC)
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called ad3530r.
@@ -137,6 +139,22 @@ config AD5449
 	  To compile this driver as a module, choose M here: the
 	  module will be called ad5449.
 
+config AD5529R
+	tristate "Analog Devices AD5529R High Voltage DAC driver"
+	depends on SPI_MASTER
+	select REGMAP_SPI
+	help
+	  Say yes here to build support for Analog Devices AD5529R 16-Channel,
+	  12-Bit/16-Bit, 40V High Voltage Precision Digital to Analog Converter.
+
+	  The device features multiple output voltage ranges from -20V to +20V,
+	  built-in 4.096V voltage reference, and digital functions including
+	  toggle, dither, and ramp modes. Supports both 12-bit and 16-bit
+	  resolution variants.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ad5529r.
+
 config AD5592R_BASE
 	tristate
 
@@ -593,9 +611,13 @@ config MCP47A1
 	  will be called mcp47a1.
 
 config MCP47FEB02
-	tristate "MCP47F(E/V)B01/02/04/08/11/12/14/18/21/22/24/28 DAC driver"
+	tristate
+
+config MCP47FEB02_I2C
+	tristate "MCP47F(E/V)B01/02/04/08/11/12/14/18/21/22/24/28 I2C DAC driver"
 	depends on I2C
 	select REGMAP_I2C
+	select MCP47FEB02
 	help
 	  Say yes here if you want to build the driver for the Microchip:
 	  - 8-bit DAC:
@@ -611,7 +633,31 @@ config MCP47FEB02
 	  (DAC) with I2C interface.
 
 	  To compile this driver as a module, choose M here: the module
-	  will be called mcp47feb02.
+	  will be called mcp47feb02_i2c and you will also get
+	  mcp47feb02_core for the core module.
+
+config MCP47FEB02_SPI
+	tristate "MCP48F(E/V)B01/02/04/08/11/12/14/18/21/22/24/28 SPI DAC driver"
+	depends on SPI
+	select REGMAP_SPI
+	select MCP47FEB02
+	help
+	  Say yes here if you want to build the driver for the Microchip:
+	  - 8-bit DAC:
+	    MCP48FEB01, MCP48FEB02, MCP48FEB04, MCP48FEB08,
+	    MCP48FVB01, MCP48FVB02, MCP48FVB04, MCP48FVB08
+	  - 10-bit DAC:
+	    MCP48FEB11, MCP48FEB12, MCP48FEB14, MCP48FEB18,
+	    MCP48FVB11, MCP48FVB12, MCP48FVB14, MCP48FVB18
+	  - 12-bit DAC:
+	    MCP48FEB21, MCP48FEB22, MCP48FEB24, MCP48FEB28,
+	    MCP48FVB21, MCP48FVB22, MCP48FVB24, MCP48FVB28
+	  having 1 to 8 channels, 8/10/12-bit digital-to-analog converter
+	  (DAC) with SPI interface.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called mcp47feb02_spi and you will also get
+	  mcp47feb02_core for the core module.
 
 config MCP4821
 	tristate "MCP4801/02/11/12/21/22 DAC driver"
diff --git a/drivers/iio/dac/Makefile b/drivers/iio/dac/Makefile
index 992f8930f95cae..2cd0fe23db9e60 100644
--- a/drivers/iio/dac/Makefile
+++ b/drivers/iio/dac/Makefile
@@ -18,6 +18,7 @@ obj-$(CONFIG_AD5446) += ad5446.o
 obj-$(CONFIG_AD5446_SPI) += ad5446-spi.o
 obj-$(CONFIG_AD5446_I2C) += ad5446-i2c.o
 obj-$(CONFIG_AD5449) += ad5449.o
+obj-$(CONFIG_AD5529R) += ad5529r.o
 obj-$(CONFIG_AD5592R_BASE) += ad5592r-base.o
 obj-$(CONFIG_AD5592R) += ad5592r.o
 obj-$(CONFIG_AD5593R) += ad5593r.o
@@ -56,7 +57,9 @@ obj-$(CONFIG_MCF54415_DAC) += mcf54415_dac.o
 obj-$(CONFIG_MCP4725) += mcp4725.o
 obj-$(CONFIG_MCP4728) += mcp4728.o
 obj-$(CONFIG_MCP47A1) += mcp47a1.o
-obj-$(CONFIG_MCP47FEB02) += mcp47feb02.o
+obj-$(CONFIG_MCP47FEB02) += mcp47feb02-core.o
+obj-$(CONFIG_MCP47FEB02_I2C) += mcp47feb02-i2c.o
+obj-$(CONFIG_MCP47FEB02_SPI) += mcp47feb02-spi.o
 obj-$(CONFIG_MCP4821) += mcp4821.o
 obj-$(CONFIG_MCP4922) += mcp4922.o
 obj-$(CONFIG_STM32_DAC_CORE) += stm32-dac-core.o
diff --git a/drivers/iio/dac/ad3530r.c b/drivers/iio/dac/ad3530r.c
index 06b7f280f61bf2..9e043639801d2f 100644
--- a/drivers/iio/dac/ad3530r.c
+++ b/drivers/iio/dac/ad3530r.c
@@ -3,8 +3,11 @@
  * AD3530R/AD3530 8-channel, 16-bit Voltage Output DAC Driver
  * AD3531R/AD3531 4-channel, 16-bit Voltage Output DAC Driver
  * AD3532R/AD3532 16-channel, 16-bit Voltage Output DAC Driver
+ * AD3536R 16-channel, 12-bit Voltage Output DAC Driver
+ * AD5710R 8-channel, 16-bit Configurable IDAC/VDAC Driver
+ * AD5711R 8-channel, 12-bit Configurable IDAC/VDAC Driver
  *
- * Copyright 2025 Analog Devices Inc.
+ * Copyright 2025-2026 Analog Devices Inc.
  */
 
 #include <linux/array_size.h>
@@ -27,6 +30,8 @@
 #include <linux/types.h>
 #include <linux/units.h>
 
+#include <dt-bindings/iio/addac/adi,ad74413r.h>
+
 #define AD3530R_INTERFACE_CONFIG_A		0x00
 #define AD3530R_OUTPUT_OPERATING_MODE_0		0x20
 #define AD3530R_OUTPUT_OPERATING_MODE_1		0x21
@@ -39,6 +44,8 @@
 #define AD3531R_SW_LDAC_TRIG_A			0xDD
 #define AD3531R_INPUT_CH			0xE3
 
+#define AD5710R_CHN_VMODE_EN			0xFF
+
 /* AD3532R/AD3532 bank 0 registers (channels 0-7) */
 #define AD3532R_INTERFACE_CONFIG_A_0		0x1000
 #define AD3532R_OUTPUT_OPERATING_MODE_0		0x1020
@@ -61,14 +68,14 @@
 #define AD3530R_SLD_TRIG_A			BIT(7)
 #define AD3530R_OUTPUT_CONTROL_RANGE		BIT(2)
 #define AD3530R_REFERENCE_CONTROL_SEL		BIT(0)
-#define AD3530R_REG_VAL_MASK			GENMASK(15, 0)
 #define AD3530R_OP_MODE_CHAN_MSK(chan)		(GENMASK(1, 0) << 2 * (chan))
+#define AD5710R_CHN_VMODE_EN_BIT(chan)		BIT(chan)
 
 #define AD3530R_SW_RESET			(BIT(7) | BIT(0))
 #define AD3530R_INTERNAL_VREF_mV		2500
+#define AD5710R_INTERNAL_IREF_mA		50
 #define AD3530R_LDAC_PULSE_US			100
 
-#define AD3530R_DAC_MAX_VAL			GENMASK(15, 0)
 #define AD3530R_CH_PER_REG			4
 #define AD3530R_CH_PER_BANK			8
 #define AD3531R_MAX_CHANNELS			4
@@ -99,7 +106,9 @@ struct ad3530r_chip_info {
 	unsigned int num_channels;
 	unsigned int num_banks;
 	unsigned int num_op_mode_regs;
+	unsigned int resolution;
 	bool internal_ref_support;
+	bool configurable_ch_func;
 };
 
 struct ad3530r_state {
@@ -195,6 +204,23 @@ static const struct iio_enum ad3532r_powerdown_mode_enum = {
 	.set = ad3530r_set_powerdown_mode,
 };
 
+static ssize_t ad5710r_read_powerdown_mode(struct iio_dev *indio_dev,
+					   uintptr_t private,
+					   const struct iio_chan_spec *chan,
+					   char *buf)
+{
+	struct ad3530r_state *st = iio_priv(indio_dev);
+	int ret;
+
+	/* Voltage output pulls to 15kohm_to_gnd, current goes three_state. */
+	ret = regmap_test_bits(st->regmap, AD5710R_CHN_VMODE_EN,
+			       AD5710R_CHN_VMODE_EN_BIT(chan->channel));
+	if (ret < 0)
+		return ret;
+
+	return sysfs_emit(buf, "%s\n", ret ? "15kohm_to_gnd" : "three_state");
+}
+
 static ssize_t ad3530r_get_dac_powerdown(struct iio_dev *indio_dev,
 					 uintptr_t private,
 					 const struct iio_chan_spec *chan,
@@ -278,6 +304,61 @@ static ssize_t ad3532r_set_dac_powerdown(struct iio_dev *indio_dev,
 	return len;
 }
 
+static void ad5710r_get_op_mode_reg(unsigned int channel, unsigned int *reg,
+				    unsigned int *mask)
+{
+	unsigned int local_ch, reg_in_bank, ch_in_reg;
+
+	local_ch = channel % AD3530R_CH_PER_BANK;
+	reg_in_bank = local_ch / AD3530R_CH_PER_REG;
+	ch_in_reg = local_ch % AD3530R_CH_PER_REG;
+
+	*reg = AD3530R_OUTPUT_OPERATING_MODE_0 + reg_in_bank;
+	*mask = AD3530R_OP_MODE_CHAN_MSK(ch_in_reg);
+}
+
+static ssize_t ad5710r_get_dac_powerdown(struct iio_dev *indio_dev,
+					 uintptr_t private,
+					 const struct iio_chan_spec *chan,
+					 char *buf)
+{
+	struct ad3530r_state *st = iio_priv(indio_dev);
+	unsigned int reg, mask;
+	int ret;
+
+	ad5710r_get_op_mode_reg(chan->channel, &reg, &mask);
+
+	ret = regmap_test_bits(st->regmap, reg, field_prep(mask, 1));
+	if (ret < 0)
+		return ret;
+
+	return sysfs_emit(buf, "%d\n", ret);
+}
+
+static ssize_t ad5710r_set_dac_powerdown(struct iio_dev *indio_dev,
+					 uintptr_t private,
+					 const struct iio_chan_spec *chan,
+					 const char *buf, size_t len)
+{
+	struct ad3530r_state *st = iio_priv(indio_dev);
+	unsigned int reg, mask;
+	bool powerdown;
+	int ret;
+
+	ret = kstrtobool(buf, &powerdown);
+	if (ret)
+		return ret;
+
+	ad5710r_get_op_mode_reg(chan->channel, &reg, &mask);
+
+	ret = regmap_update_bits(st->regmap, reg, mask,
+				 field_prep(mask, powerdown));
+	if (ret)
+		return ret;
+
+	return len;
+}
+
 static int ad3530r_trigger_sw_ldac_reg(unsigned int channel)
 {
 	return AD3530R_SW_LDAC_TRIG_A;
@@ -310,7 +391,7 @@ static int ad3530r_dac_write(struct ad3530r_state *st, unsigned int chan,
 	int ret;
 
 	guard(mutex)(&st->lock);
-	st->buf = cpu_to_be16(val);
+	st->buf = cpu_to_be16(val << (16 - st->chip_info->resolution));
 
 	ret = regmap_bulk_write(st->regmap, st->chip_info->input_ch_reg(chan),
 				&st->buf, sizeof(st->buf));
@@ -340,12 +421,15 @@ static int ad3530r_read_raw(struct iio_dev *indio_dev,
 		if (ret)
 			return ret;
 
-		*val = FIELD_GET(AD3530R_REG_VAL_MASK, be16_to_cpu(st->buf));
+		*val = be16_to_cpu(st->buf) >> (16 - st->chip_info->resolution);
 
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
-		*val = st->vref_mV;
-		*val2 = 16;
+		if (chan->type == IIO_CURRENT)
+			*val = AD5710R_INTERNAL_IREF_mA;
+		else
+			*val = st->vref_mV;
+		*val2 = st->chip_info->resolution;
 
 		return IIO_VAL_FRACTIONAL_LOG2;
 	default:
@@ -361,7 +445,7 @@ static int ad3530r_write_raw(struct iio_dev *indio_dev,
 
 	switch (info) {
 	case IIO_CHAN_INFO_RAW:
-		if (val < 0 || val > AD3530R_DAC_MAX_VAL)
+		if (val < 0 || val > GENMASK(st->chip_info->resolution - 1, 0))
 			return -EINVAL;
 
 		return ad3530r_dac_write(st, chan->channel, val);
@@ -420,6 +504,21 @@ static const struct iio_chan_spec_ext_info ad3532r_ext_info[] = {
 	{ }
 };
 
+static const struct iio_chan_spec_ext_info ad5710r_ext_info[] = {
+	{
+		.name = "powerdown",
+		.shared = IIO_SEPARATE,
+		.read = ad5710r_get_dac_powerdown,
+		.write = ad5710r_set_dac_powerdown,
+	},
+	{
+		.name = "powerdown_mode",
+		.shared = IIO_SEPARATE,
+		.read = ad5710r_read_powerdown_mode,
+	},
+	{ }
+};
+
 #define AD3530R_CHAN(_chan, _ext_info)				\
 {								\
 	.type = IIO_VOLTAGE,					\
@@ -468,6 +567,17 @@ static const struct iio_chan_spec ad3532r_channels[] = {
 	AD3530R_CHAN(15, ad3532r_ext_info),
 };
 
+static const struct iio_chan_spec ad5710r_channels[] = {
+	AD3530R_CHAN(0, ad5710r_ext_info),
+	AD3530R_CHAN(1, ad5710r_ext_info),
+	AD3530R_CHAN(2, ad5710r_ext_info),
+	AD3530R_CHAN(3, ad5710r_ext_info),
+	AD3530R_CHAN(4, ad5710r_ext_info),
+	AD3530R_CHAN(5, ad5710r_ext_info),
+	AD3530R_CHAN(6, ad5710r_ext_info),
+	AD3530R_CHAN(7, ad5710r_ext_info),
+};
+
 static const unsigned int ad3530r_if_config[] = {
 	AD3530R_INTERFACE_CONFIG_A,
 };
@@ -523,8 +633,15 @@ static const struct regmap_config ad3532r_regmap_config = {
 	.max_register = AD3532R_MAX_REG_ADDR,
 };
 
+static const struct regmap_config ad5710r_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 8,
+	.max_register = AD5710R_CHN_VMODE_EN,
+};
+
 static const struct ad3530r_chip_info ad3530_chip = {
 	.name = "ad3530",
+	.resolution = 16,
 	.channels = ad3530r_channels,
 	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3530r_channels),
@@ -537,10 +654,12 @@ static const struct ad3530r_chip_info ad3530_chip = {
 	.num_banks = ARRAY_SIZE(ad3530r_if_config),
 	.num_op_mode_regs = ARRAY_SIZE(ad3530r_op_mode),
 	.internal_ref_support = false,
+	.configurable_ch_func = false,
 };
 
 static const struct ad3530r_chip_info ad3530r_chip = {
 	.name = "ad3530r",
+	.resolution = 16,
 	.channels = ad3530r_channels,
 	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3530r_channels),
@@ -553,10 +672,12 @@ static const struct ad3530r_chip_info ad3530r_chip = {
 	.num_banks = ARRAY_SIZE(ad3530r_if_config),
 	.num_op_mode_regs = ARRAY_SIZE(ad3530r_op_mode),
 	.internal_ref_support = true,
+	.configurable_ch_func = false,
 };
 
 static const struct ad3530r_chip_info ad3531_chip = {
 	.name = "ad3531",
+	.resolution = 16,
 	.channels = ad3531r_channels,
 	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3531r_channels),
@@ -569,10 +690,12 @@ static const struct ad3530r_chip_info ad3531_chip = {
 	.num_banks = ARRAY_SIZE(ad3530r_if_config),
 	.num_op_mode_regs = ARRAY_SIZE(ad3531r_op_mode),
 	.internal_ref_support = false,
+	.configurable_ch_func = false,
 };
 
 static const struct ad3530r_chip_info ad3531r_chip = {
 	.name = "ad3531r",
+	.resolution = 16,
 	.channels = ad3531r_channels,
 	.regmap_config = &ad3530r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3531r_channels),
@@ -585,10 +708,12 @@ static const struct ad3530r_chip_info ad3531r_chip = {
 	.num_banks = ARRAY_SIZE(ad3530r_if_config),
 	.num_op_mode_regs = ARRAY_SIZE(ad3531r_op_mode),
 	.internal_ref_support = true,
+	.configurable_ch_func = false,
 };
 
 static const struct ad3530r_chip_info ad3532_chip = {
 	.name = "ad3532",
+	.resolution = 16,
 	.channels = ad3532r_channels,
 	.regmap_config = &ad3532r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3532r_channels),
@@ -601,10 +726,12 @@ static const struct ad3530r_chip_info ad3532_chip = {
 	.num_banks = ARRAY_SIZE(ad3532r_if_config),
 	.num_op_mode_regs = ARRAY_SIZE(ad3532r_op_mode),
 	.internal_ref_support = false,
+	.configurable_ch_func = false,
 };
 
 static const struct ad3530r_chip_info ad3532r_chip = {
 	.name = "ad3532r",
+	.resolution = 16,
 	.channels = ad3532r_channels,
 	.regmap_config = &ad3532r_regmap_config,
 	.num_channels = ARRAY_SIZE(ad3532r_channels),
@@ -617,8 +744,141 @@ static const struct ad3530r_chip_info ad3532r_chip = {
 	.num_banks = ARRAY_SIZE(ad3532r_if_config),
 	.num_op_mode_regs = ARRAY_SIZE(ad3532r_op_mode),
 	.internal_ref_support = true,
+	.configurable_ch_func = false,
 };
 
+static const struct ad3530r_chip_info ad3536r_chip = {
+	.name = "ad3536r",
+	.resolution = 12,
+	.channels = ad3532r_channels,
+	.regmap_config = &ad3532r_regmap_config,
+	.num_channels = ARRAY_SIZE(ad3532r_channels),
+	.sw_ldac_trig_reg = ad3532r_trigger_sw_ldac_reg,
+	.input_ch_reg = ad3532r_input_ch_reg,
+	.interface_config_a = ad3532r_if_config,
+	.output_control = ad3532r_out_ctrl,
+	.reference_control = ad3532r_ref_ctrl,
+	.op_mode = ad3532r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3532r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3532r_op_mode),
+	.internal_ref_support = true,
+	.configurable_ch_func = false,
+};
+
+static const struct ad3530r_chip_info ad5710r_chip = {
+	.name = "ad5710r",
+	.resolution = 16,
+	.regmap_config = &ad5710r_regmap_config,
+	.num_channels = ARRAY_SIZE(ad5710r_channels),
+	.sw_ldac_trig_reg = ad3530r_trigger_sw_ldac_reg,
+	.input_ch_reg = ad3530r_input_ch_reg,
+	.interface_config_a = ad3530r_if_config,
+	.output_control = ad3530r_out_ctrl,
+	.reference_control = ad3530r_ref_ctrl,
+	.op_mode = ad3530r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3530r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3530r_op_mode),
+	.internal_ref_support = true,
+	.configurable_ch_func = true,
+};
+
+static const struct ad3530r_chip_info ad5711r_chip = {
+	.name = "ad5711r",
+	.resolution = 12,
+	.regmap_config = &ad5710r_regmap_config,
+	.num_channels = ARRAY_SIZE(ad5710r_channels),
+	.sw_ldac_trig_reg = ad3530r_trigger_sw_ldac_reg,
+	.input_ch_reg = ad3530r_input_ch_reg,
+	.interface_config_a = ad3530r_if_config,
+	.output_control = ad3530r_out_ctrl,
+	.reference_control = ad3530r_ref_ctrl,
+	.op_mode = ad3530r_op_mode,
+	.num_banks = ARRAY_SIZE(ad3530r_if_config),
+	.num_op_mode_regs = ARRAY_SIZE(ad3530r_op_mode),
+	.internal_ref_support = true,
+	.configurable_ch_func = true,
+};
+
+static int ad3530r_parse_channel_cfg(struct iio_dev *indio_dev)
+{
+	struct ad3530r_state *st = iio_priv(indio_dev);
+	struct device *dev = regmap_get_device(st->regmap);
+	struct iio_chan_spec *channels;
+	unsigned int num_chan, i;
+	int ret;
+
+	num_chan = device_get_child_node_count(dev);
+	if (!num_chan)
+		return dev_err_probe(dev, -ENOENT, "No channels configured\n");
+
+	channels = devm_kcalloc(dev, num_chan, sizeof(*channels), GFP_KERNEL);
+	if (!channels)
+		return -ENOMEM;
+
+	i = 0;
+	device_for_each_child_node_scoped(dev, child) {
+		unsigned int mode_reg, mode_mask, ch_func;
+		enum iio_chan_type chan_type;
+		u32 reg;
+
+		ret = fwnode_property_read_u32(child, "reg", &reg);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to read reg property of %pfwP\n",
+					     child);
+
+		if (reg >= st->chip_info->num_channels)
+			return dev_err_probe(dev, -ECHRNG,
+					     "reg out of range in %pfwP\n",
+					     child);
+
+		ret = fwnode_property_read_u32(child, "adi,ch-func", &ch_func);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Missing adi,ch-func property for %pfwP\n",
+					     child);
+
+		switch (ch_func) {
+		case CH_FUNC_VOLTAGE_OUTPUT:
+			ret = regmap_set_bits(st->regmap, AD5710R_CHN_VMODE_EN,
+					      AD5710R_CHN_VMODE_EN_BIT(reg));
+			if (ret)
+				return dev_err_probe(dev, ret,
+						     "Failed to set voltage output for %pfwP\n",
+						     child);
+
+			chan_type = IIO_VOLTAGE;
+			break;
+		case CH_FUNC_CURRENT_OUTPUT:
+			chan_type = IIO_CURRENT;
+			break;
+		default:
+			return dev_err_probe(dev, -EINVAL,
+					     "Invalid adi,ch-func %u for %pfwP\n",
+					     ch_func, child);
+		}
+
+		channels[i] = ad5710r_channels[reg];
+		channels[i].type = chan_type;
+		i++;
+
+		ad5710r_get_op_mode_reg(reg, &mode_reg, &mode_mask);
+
+		/* Enable the channel in normal operation mode */
+		ret = regmap_update_bits(st->regmap, mode_reg, mode_mask,
+					 field_prep(mode_mask, AD3530R_NORMAL_OP));
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to set normal operating mode for %pfwP\n",
+					     child);
+	}
+
+	indio_dev->channels = channels;
+	indio_dev->num_channels = num_chan;
+
+	return 0;
+}
+
 static int ad3530r_set_reg_bank_bits(const struct ad3530r_state *st,
 				     const unsigned int *regs,
 				     unsigned int num_regs,
@@ -651,8 +911,9 @@ static int ad3530r_write_reg_banks(const struct ad3530r_state *st,
 	return 0;
 }
 
-static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
+static int ad3530r_setup(struct iio_dev *indio_dev, int external_vref_uV)
 {
+	struct ad3530r_state *st = iio_priv(indio_dev);
 	const struct ad3530r_chip_info *chip_info = st->chip_info;
 	struct device *dev = regmap_get_device(st->regmap);
 	struct gpio_desc *reset_gpio;
@@ -702,19 +963,29 @@ static int ad3530r_setup(struct ad3530r_state *st, int external_vref_uV)
 		st->vref_mV = range_multiplier * AD3530R_INTERNAL_VREF_mV;
 	}
 
-	/* Set normal operating mode for all channels */
-	val = FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(0), AD3530R_NORMAL_OP) |
-	      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(1), AD3530R_NORMAL_OP) |
-	      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(2), AD3530R_NORMAL_OP) |
-	      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(3), AD3530R_NORMAL_OP);
+	if (chip_info->configurable_ch_func) {
+		/* Channels and their operating mode are configured from DT */
+		ret = ad3530r_parse_channel_cfg(indio_dev);
+		if (ret)
+			return ret;
+	} else {
+		/* Set normal operating mode for all channels */
+		val = FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(0), AD3530R_NORMAL_OP) |
+		      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(1), AD3530R_NORMAL_OP) |
+		      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(2), AD3530R_NORMAL_OP) |
+		      FIELD_PREP(AD3530R_OP_MODE_CHAN_MSK(3), AD3530R_NORMAL_OP);
+
+		ret = ad3530r_write_reg_banks(st, chip_info->op_mode,
+					      chip_info->num_op_mode_regs, val);
+		if (ret)
+			return ret;
 
-	ret = ad3530r_write_reg_banks(st, st->chip_info->op_mode,
-				      st->chip_info->num_op_mode_regs, val);
-	if (ret)
-		return ret;
+		for (unsigned int i = 0; i < chip_info->num_channels; i++)
+			st->chan[i].powerdown_mode = AD3530R_POWERDOWN_32K;
 
-	for (unsigned int i = 0; i < st->chip_info->num_channels; i++)
-		st->chan[i].powerdown_mode = AD3530R_POWERDOWN_32K;
+		indio_dev->channels = chip_info->channels;
+		indio_dev->num_channels = chip_info->num_channels;
+	}
 
 	st->ldac_gpio = devm_gpiod_get_optional(dev, "ldac", GPIOD_OUT_LOW);
 	if (IS_ERR(st->ldac_gpio))
@@ -772,15 +1043,13 @@ static int ad3530r_probe(struct spi_device *spi)
 	if (!st->chip_info->internal_ref_support && external_vref_uV == 0)
 		return -ENODEV;
 
-	ret = ad3530r_setup(st, external_vref_uV);
+	ret = ad3530r_setup(indio_dev, external_vref_uV);
 	if (ret)
 		return ret;
 
 	indio_dev->name = st->chip_info->name;
 	indio_dev->info = &ad3530r_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = st->chip_info->channels;
-	indio_dev->num_channels = st->chip_info->num_channels;
 
 	return devm_iio_device_register(&spi->dev, indio_dev);
 }
@@ -792,6 +1061,9 @@ static const struct spi_device_id ad3530r_id[] = {
 	{ .name = "ad3531r", .driver_data = (kernel_ulong_t)&ad3531r_chip },
 	{ .name = "ad3532", .driver_data = (kernel_ulong_t)&ad3532_chip },
 	{ .name = "ad3532r", .driver_data = (kernel_ulong_t)&ad3532r_chip },
+	{ .name = "ad3536r", .driver_data = (kernel_ulong_t)&ad3536r_chip },
+	{ .name = "ad5710r", .driver_data = (kernel_ulong_t)&ad5710r_chip },
+	{ .name = "ad5711r", .driver_data = (kernel_ulong_t)&ad5711r_chip },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad3530r_id);
@@ -803,6 +1075,9 @@ static const struct of_device_id ad3530r_of_match[] = {
 	{ .compatible = "adi,ad3531r", .data = &ad3531r_chip },
 	{ .compatible = "adi,ad3532", .data = &ad3532_chip },
 	{ .compatible = "adi,ad3532r", .data = &ad3532r_chip },
+	{ .compatible = "adi,ad3536r", .data = &ad3536r_chip },
+	{ .compatible = "adi,ad5710r", .data = &ad5710r_chip },
+	{ .compatible = "adi,ad5711r", .data = &ad5711r_chip },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ad3530r_of_match);
diff --git a/drivers/iio/dac/ad3552r.c b/drivers/iio/dac/ad3552r.c
index 16db94fef9d42c..039af514addeb7 100644
--- a/drivers/iio/dac/ad3552r.c
+++ b/drivers/iio/dac/ad3552r.c
@@ -696,6 +696,6 @@ static struct spi_driver ad3552r_driver = {
 module_spi_driver(ad3552r_driver);
 
 MODULE_AUTHOR("Mihail Chindris <mihail.chindris@analog.com>");
-MODULE_DESCRIPTION("Analog Device AD3552R DAC");
+MODULE_DESCRIPTION("Analog Devices AD3552R DAC");
 MODULE_LICENSE("GPL v2");
 MODULE_IMPORT_NS("IIO_AD3552R");
diff --git a/drivers/iio/dac/ad5529r.c b/drivers/iio/dac/ad5529r.c
new file mode 100644
index 00000000000000..8154b87ff831ea
--- /dev/null
+++ b/drivers/iio/dac/ad5529r.c
@@ -0,0 +1,546 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AD5529R Digital-to-Analog Converter Driver
+ * 16-Channel, 12/16-Bit, 40V High Voltage Precision DAC
+ *
+ * Copyright 2026 Analog Devices Inc.
+ * Author: Janani Sunil <janani.sunil@analog.com>
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/spi/spi.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include <linux/iio/iio.h>
+
+#define AD5529R_REG_INTERFACE_CONFIG_A		0x00
+#define   AD5529R_INTERFACE_CONFIG_A_SW_RESET	(BIT(7) | BIT(0))
+#define   AD5529R_INTERFACE_CONFIG_A_ADDR_ASCENSION	BIT(5)
+#define   AD5529R_INTERFACE_CONFIG_A_SDO_ENABLE	BIT(4)
+#define AD5529R_REG_DEVICE_CONFIG		0x02
+#define AD5529R_REG_CHIP_GRADE			0x06
+#define AD5529R_REG_SCRATCH_PAD			0x0A
+#define AD5529R_REG_SPI_REVISION		0x0B
+#define AD5529R_REG_VENDOR_H			0x0D
+#define AD5529R_REG_STREAM_MODE			0x0E
+#define AD5529R_REG_INTERFACE_STATUS_A		0x11
+#define AD5529R_REG_MULTI_DAC_CH_SEL		0x14
+#define AD5529R_REG_OUT_RANGE_BASE		0x3C
+#define AD5529R_REG_OUT_RANGE(ch)		(AD5529R_REG_OUT_RANGE_BASE + (ch) * 2)
+#define AD5529R_REG_DAC_INPUT_A_BASE		0x148
+#define AD5529R_REG_DAC_INPUT_A(ch)		(AD5529R_REG_DAC_INPUT_A_BASE + (ch) * 2)
+#define AD5529R_REG_DAC_DATA_READBACK_BASE	0x16A
+#define AD5529R_REG_TSENS_ALERT_FLAG		0x18C
+#define AD5529R_REG_TSENS_SHTD_FLAG		0x18E
+#define AD5529R_REG_FUNC_BUSY			0x1A0
+#define AD5529R_REG_REF_SEL			0x1A2
+#define   AD5529R_REF_SEL_INTERNAL_REF		BIT(0)
+#define AD5529R_REG_INIT_CRC_ERR_STAT		0x1A4
+#define AD5529R_REG_MULTI_DAC_HOTPATH_SW_LDAC	0x1A8
+
+#define AD5529R_MAX_REGISTER			0x232
+#define AD5529R_8BIT_REG_MAX			0x13
+#define AD5529R_MAX_CHANNELS			16
+#define AD5529R_SPI_READ_FLAG			0x80
+#define AD5529R_ADDR_SHIFT			12
+
+struct ad5529r_model_data {
+	const char *model_name;
+	unsigned int resolution;
+};
+
+#define AD5529R_DAC_CHANNEL(chan) ((struct iio_chan_spec) {		\
+	.type = IIO_VOLTAGE,						\
+	.indexed = 1,							\
+	.output = 1,							\
+	.channel = (chan),						\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |			\
+			      BIT(IIO_CHAN_INFO_SCALE) |		\
+			      BIT(IIO_CHAN_INFO_OFFSET),		\
+})
+
+static const char * const ad5529r_supply_names[] = {
+	"vdd-io",
+	"vdd-5v",
+	"vdd-hv0",
+	"vdd-hv1",
+	"vdd-hv2",
+	"vdd-hv3",
+};
+
+static const char * const ad5529r_vss_supply_names[] = {
+	"vss-hv0",
+	"vss-hv1",
+	"vss-hv2",
+	"vss-hv3",
+};
+
+static const struct ad5529r_model_data ad5529r_16bit_model_data = {
+	.model_name = "ad5529r-16",
+	.resolution = 16,
+};
+
+static const struct ad5529r_model_data ad5529r_12bit_model_data = {
+	.model_name = "ad5529r-12",
+	.resolution = 12,
+};
+
+enum ad5529r_output_range {
+	AD5529R_RANGE_0V_5V,
+	AD5529R_RANGE_0V_10V,
+	AD5529R_RANGE_0V_20V,
+	AD5529R_RANGE_0V_40V,
+	AD5529R_RANGE_M5V_5V,
+	AD5529R_RANGE_M10V_10V,
+	AD5529R_RANGE_M15V_15V,
+	AD5529R_RANGE_M20V_20V,
+};
+
+static const s32 ad5529r_output_ranges_mV[8][2] = {
+	[AD5529R_RANGE_0V_5V] = { 0, 5000 },
+	[AD5529R_RANGE_0V_10V] = { 0, 10000 },
+	[AD5529R_RANGE_0V_20V] = { 0, 20000 },
+	[AD5529R_RANGE_0V_40V] = { 0, 40000 },
+	[AD5529R_RANGE_M5V_5V] = { -5000, 5000 },
+	[AD5529R_RANGE_M10V_10V] = { -10000, 10000 },
+	[AD5529R_RANGE_M15V_15V] = { -15000, 15000 },
+	[AD5529R_RANGE_M20V_20V] = { -20000, 20000 },
+};
+
+struct ad5529r_state {
+	const struct ad5529r_model_data *model_data;
+	struct regmap *regmap_8bit;
+	struct regmap *regmap_16bit;
+	struct iio_chan_spec channels[AD5529R_MAX_CHANNELS];
+	unsigned int num_channels;
+	enum ad5529r_output_range output_range_idx[AD5529R_MAX_CHANNELS];
+};
+
+static const struct regmap_range ad5529r_8bit_readable_ranges[] = {
+	regmap_reg_range(AD5529R_REG_INTERFACE_CONFIG_A, AD5529R_REG_CHIP_GRADE),
+	regmap_reg_range(AD5529R_REG_SCRATCH_PAD, AD5529R_REG_VENDOR_H),
+	regmap_reg_range(AD5529R_REG_STREAM_MODE, AD5529R_REG_INTERFACE_STATUS_A),
+};
+
+static const struct regmap_range ad5529r_16bit_readable_ranges[] = {
+	regmap_reg_range(AD5529R_REG_MULTI_DAC_CH_SEL, AD5529R_REG_INIT_CRC_ERR_STAT),
+	regmap_reg_range(AD5529R_REG_MULTI_DAC_HOTPATH_SW_LDAC, AD5529R_MAX_REGISTER),
+};
+
+static const struct regmap_access_table ad5529r_8bit_readable_table = {
+	.yes_ranges = ad5529r_8bit_readable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(ad5529r_8bit_readable_ranges),
+};
+
+static const struct regmap_access_table ad5529r_16bit_readable_table = {
+	.yes_ranges = ad5529r_16bit_readable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(ad5529r_16bit_readable_ranges),
+};
+
+static const struct regmap_range ad5529r_8bit_read_only_ranges[] = {
+	regmap_reg_range(AD5529R_REG_DEVICE_CONFIG, AD5529R_REG_CHIP_GRADE),
+	regmap_reg_range(AD5529R_REG_SPI_REVISION, AD5529R_REG_VENDOR_H),
+};
+
+static const struct regmap_range ad5529r_16bit_read_only_ranges[] = {
+	regmap_reg_range(AD5529R_REG_DAC_DATA_READBACK_BASE,
+			 AD5529R_REG_DAC_DATA_READBACK_BASE +
+			 (AD5529R_MAX_CHANNELS - 1) * 2),
+	regmap_reg_range(AD5529R_REG_TSENS_ALERT_FLAG, AD5529R_REG_TSENS_SHTD_FLAG),
+	regmap_reg_range(AD5529R_REG_FUNC_BUSY, AD5529R_REG_FUNC_BUSY),
+	regmap_reg_range(AD5529R_REG_INIT_CRC_ERR_STAT, AD5529R_REG_INIT_CRC_ERR_STAT),
+};
+
+static const struct regmap_access_table ad5529r_8bit_writeable_table = {
+	.no_ranges = ad5529r_8bit_read_only_ranges,
+	.n_no_ranges = ARRAY_SIZE(ad5529r_8bit_read_only_ranges),
+};
+
+static const struct regmap_access_table ad5529r_16bit_writeable_table = {
+	.no_ranges = ad5529r_16bit_read_only_ranges,
+	.n_no_ranges = ARRAY_SIZE(ad5529r_16bit_read_only_ranges),
+};
+
+static struct regmap *ad5529r_get_regmap(struct ad5529r_state *st,
+					 unsigned int reg)
+{
+	if (reg <= AD5529R_8BIT_REG_MAX)
+		return st->regmap_8bit;
+
+	return st->regmap_16bit;
+}
+
+static int ad5529r_reset(struct ad5529r_state *st)
+{
+	struct regmap *map = st->regmap_8bit;
+	struct device *dev = regmap_get_device(map);
+	struct reset_control *rst;
+	int ret;
+
+	rst = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(rst))
+		return PTR_ERR(rst);
+
+	if (rst) {
+		ret = reset_control_assert(rst);
+		if (ret)
+			return ret;
+
+		/* Minimum reset low width (t_reset) is 20 ns per datasheet. */
+		ndelay(20);
+
+		ret = reset_control_deassert(rst);
+		if (ret)
+			return ret;
+	} else {
+		ret = regmap_write(map, AD5529R_REG_INTERFACE_CONFIG_A,
+				   AD5529R_INTERFACE_CONFIG_A_SW_RESET);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * Wait 10 ms for digital initialization to complete.
+	 * Per datasheet, Interface Status A register NOT_READY_ERR bit is
+	 * set if SPI transactions are attempted before digital initialization
+	 * completes.
+	 */
+	fsleep(10 * USEC_PER_MSEC);
+
+	return regmap_write(map, AD5529R_REG_INTERFACE_CONFIG_A,
+			    AD5529R_INTERFACE_CONFIG_A_SDO_ENABLE |
+			    AD5529R_INTERFACE_CONFIG_A_ADDR_ASCENSION);
+}
+
+static int ad5529r_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct ad5529r_state *st = iio_priv(indio_dev);
+	enum ad5529r_output_range range_idx;
+	unsigned int reg_addr, reg_val_h;
+	int ret, span_mV;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		/*
+		 * Read from DAC_INPUT_A register rather than DAC_DATA_READBACK.
+		 * The DAC operates in transparent mode and directly reflects
+		 * whatever value is written to the INPUT_A register.
+		 */
+		reg_addr = AD5529R_REG_DAC_INPUT_A(chan->channel);
+		ret = regmap_read(st->regmap_16bit, reg_addr, &reg_val_h);
+		if (ret)
+			return ret;
+
+		*val = reg_val_h;
+
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		range_idx = st->output_range_idx[chan->channel];
+
+		/*
+		 * The datasheet specifies a 4.096 V external reference,
+		 * matching the nominal output voltage of the internal
+		 * reference.
+		 */
+		span_mV = ad5529r_output_ranges_mV[range_idx][1] -
+			  ad5529r_output_ranges_mV[range_idx][0];
+		*val = span_mV;
+		*val2 = st->model_data->resolution;
+
+		return IIO_VAL_FRACTIONAL_LOG2;
+	case IIO_CHAN_INFO_OFFSET:
+		range_idx = st->output_range_idx[chan->channel];
+
+		if (ad5529r_output_ranges_mV[range_idx][0] < 0)
+			*val = -BIT(st->model_data->resolution - 1);
+		else
+			*val = 0;
+
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad5529r_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	struct ad5529r_state *st = iio_priv(indio_dev);
+	unsigned int reg_addr;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (!in_range(val, 0, BIT(st->model_data->resolution)))
+			return -EINVAL;
+
+		reg_addr = AD5529R_REG_DAC_INPUT_A(chan->channel);
+
+		return regmap_write(st->regmap_16bit, reg_addr, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad5529r_find_output_range(const s32 *vals)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(ad5529r_output_ranges_mV); i++) {
+		const s32 *range = ad5529r_output_ranges_mV[i];
+
+		if (vals[0] == range[0] * (MICRO / MILLI) &&
+		    vals[1] == range[1] * (MICRO / MILLI))
+			return i;
+	}
+
+	return -EINVAL;
+}
+
+static int ad5529r_parse_channel_ranges(struct device *dev,
+					struct ad5529r_state *st)
+{
+	unsigned long channel_mask = 0;
+
+	device_for_each_child_node_scoped(dev, child) {
+		int ret, range_idx;
+		u32 ch;
+
+		if (st->num_channels == ARRAY_SIZE(st->channels))
+			return dev_err_probe(dev, -ENOSPC, "Too many channels\n");
+
+		ret = fwnode_property_read_u32(child, "reg", &ch);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Missing reg property in channel node\n");
+
+		if (ch >= AD5529R_MAX_CHANNELS)
+			return dev_err_probe(dev, -ECHRNG,
+					     "Channel %u exceeds maximum %u\n",
+					     ch, AD5529R_MAX_CHANNELS - 1);
+
+		if (__test_and_set_bit(ch, &channel_mask))
+			return dev_err_probe(dev, -EEXIST,
+					     "Duplicate channel %u\n", ch);
+
+		if (fwnode_property_present(child, "output-range-microvolt")) {
+			s32 vals[2];
+
+			/*
+			 * DT stores cells as raw 32-bit values; signed endpoints are
+			 * encoded by dtc in two's-complement and then interpreted
+			 * here as s32.
+			 */
+			ret = fwnode_property_read_u32_array(child,
+							     "output-range-microvolt",
+							     (u32 *)vals, ARRAY_SIZE(vals));
+			if (ret < 0)
+				return dev_err_probe(dev, ret,
+						     "Failed to read range for ch %u\n",
+						     ch);
+
+			range_idx = ad5529r_find_output_range(vals);
+			if (range_idx < 0)
+				return dev_err_probe(dev, range_idx,
+						     "Invalid range [%d %d] for ch %u\n",
+						     vals[0], vals[1], ch);
+		} else {
+			range_idx = AD5529R_RANGE_0V_5V;
+		}
+
+		st->output_range_idx[ch] = range_idx;
+		ret = regmap_write(st->regmap_16bit,
+				   AD5529R_REG_OUT_RANGE(ch), range_idx);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to configure range for ch %u\n",
+					     ch);
+
+		st->channels[st->num_channels++] = AD5529R_DAC_CHANNEL(ch);
+	}
+
+	return 0;
+}
+
+static int ad5529r_reg_access(struct iio_dev *indio_dev,
+			      unsigned int reg,
+			      unsigned int writeval,
+			      unsigned int *readval)
+{
+	struct ad5529r_state *st = iio_priv(indio_dev);
+
+	if (readval)
+		return regmap_read(ad5529r_get_regmap(st, reg), reg, readval);
+
+	return regmap_write(ad5529r_get_regmap(st, reg), reg, writeval);
+}
+
+static const struct iio_info ad5529r_info = {
+	.read_raw = ad5529r_read_raw,
+	.write_raw = ad5529r_write_raw,
+	.debugfs_reg_access = ad5529r_reg_access,
+};
+
+static int ad5529r_probe(struct spi_device *spi)
+{
+	struct regmap_config regmap_16bit_cfg;
+	struct regmap_config regmap_8bit_cfg;
+	struct device *dev = &spi->dev;
+	struct iio_dev *indio_dev;
+	struct ad5529r_state *st;
+	bool external_vref;
+	u32 dev_addr;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+
+	st->model_data = spi_get_device_match_data(spi);
+	if (!st->model_data)
+		return dev_err_probe(dev, -ENODATA,
+				     "Failed to identify device variant\n");
+
+	if (device_property_present(dev, "spi-device-addr")) {
+		ret = device_property_read_u32(dev, "spi-device-addr", &dev_addr);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to read spi-device-addr\n");
+
+		if (dev_addr > 3)
+			return dev_err_probe(dev, -EADDRNOTAVAIL,
+					     "spi-device-addr %u out of range [0, 3]\n",
+					     dev_addr);
+	} else {
+		dev_addr = 0;
+	}
+
+	regmap_8bit_cfg = (struct regmap_config) {
+		.name = "ad5529r-8bit",
+		.reg_bits = 16,
+		.val_bits = 8,
+		.max_register = AD5529R_8BIT_REG_MAX,
+		.read_flag_mask = AD5529R_SPI_READ_FLAG,
+		.rd_table = &ad5529r_8bit_readable_table,
+		.wr_table = &ad5529r_8bit_writeable_table,
+		.reg_base = dev_addr << AD5529R_ADDR_SHIFT,
+	};
+	regmap_16bit_cfg = (struct regmap_config) {
+		.name = "ad5529r-16bit",
+		.reg_bits = 16,
+		.val_bits = 16,
+		.max_register = AD5529R_MAX_REGISTER,
+		.read_flag_mask = AD5529R_SPI_READ_FLAG,
+		.val_format_endian = REGMAP_ENDIAN_LITTLE,
+		.rd_table = &ad5529r_16bit_readable_table,
+		.wr_table = &ad5529r_16bit_writeable_table,
+		.reg_stride = 2,
+		.reg_base = dev_addr << AD5529R_ADDR_SHIFT,
+	};
+
+	ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(ad5529r_supply_names),
+					     ad5529r_supply_names);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to get and enable regulators\n");
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(ad5529r_vss_supply_names); i++) {
+		ret = devm_regulator_get_enable_optional(dev,
+							 ad5529r_vss_supply_names[i]);
+		if (ret && ret != -ENODEV)
+			return dev_err_probe(dev, ret,
+					     "Failed to get and enable %s regulator\n",
+					     ad5529r_vss_supply_names[i]);
+	}
+
+	ret = devm_regulator_get_enable_optional(dev, "vref");
+	if (ret == -ENODEV)
+		external_vref = false;
+	else if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to get and enable vref regulator\n");
+	else
+		external_vref = true;
+
+	/* Wait 10 ms after power-up before the first SPI transaction. */
+	fsleep(10 * USEC_PER_MSEC);
+
+	st->regmap_8bit = devm_regmap_init_spi(spi, &regmap_8bit_cfg);
+	if (IS_ERR(st->regmap_8bit))
+		return dev_err_probe(dev, PTR_ERR(st->regmap_8bit),
+				     "Failed to initialize 8-bit regmap\n");
+
+	st->regmap_16bit = devm_regmap_init_spi(spi, &regmap_16bit_cfg);
+	if (IS_ERR(st->regmap_16bit))
+		return dev_err_probe(dev, PTR_ERR(st->regmap_16bit),
+				     "Failed to initialize 16-bit regmap\n");
+
+	ret = ad5529r_reset(st);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to reset device\n");
+
+	ret = regmap_assign_bits(st->regmap_16bit, AD5529R_REG_REF_SEL,
+				 AD5529R_REF_SEL_INTERNAL_REF,
+				 !external_vref);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to configure reference\n");
+
+	ret = ad5529r_parse_channel_ranges(dev, st);
+	if (ret)
+		return ret;
+
+	indio_dev->name = st->model_data->model_name;
+	indio_dev->info = &ad5529r_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = st->channels;
+	indio_dev->num_channels = st->num_channels;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id ad5529r_of_match[] = {
+	{ .compatible = "adi,ad5529r-16", .data = &ad5529r_16bit_model_data },
+	{ .compatible = "adi,ad5529r-12", .data = &ad5529r_12bit_model_data },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ad5529r_of_match);
+
+static const struct spi_device_id ad5529r_id[] = {
+	{
+		.name = "ad5529r-16",
+		.driver_data = (kernel_ulong_t)&ad5529r_16bit_model_data,
+	},
+	{
+		.name = "ad5529r-12",
+		.driver_data = (kernel_ulong_t)&ad5529r_12bit_model_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, ad5529r_id);
+
+static struct spi_driver ad5529r_driver = {
+	.driver = {
+		.name = "ad5529r",
+		.of_match_table = ad5529r_of_match,
+	},
+	.probe = ad5529r_probe,
+	.id_table = ad5529r_id,
+};
+module_spi_driver(ad5529r_driver);
+
+MODULE_AUTHOR("Janani Sunil <janani.sunil@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD5529R 12/16-bit DAC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c
index 0828b99090990d..730289dd8ef698 100644
--- a/drivers/iio/dac/ad5755.c
+++ b/drivers/iio/dac/ad5755.c
@@ -819,6 +819,9 @@ static int ad5755_probe(struct spi_device *spi)
 	spi_set_drvdata(spi, indio_dev);
 
 	st->chip_info = spi_get_device_match_data(spi);
+	if (!st->chip_info)
+		return -ENODATA;
+
 	st->spi = spi;
 	st->pwr_down = 0xf;
 
diff --git a/drivers/iio/dac/ad5758.c b/drivers/iio/dac/ad5758.c
index 44efa373cd80a6..1e21ad2a050dbe 100644
--- a/drivers/iio/dac/ad5758.c
+++ b/drivers/iio/dac/ad5758.c
@@ -9,6 +9,7 @@
 #include <linux/bsearch.h>
 #include <linux/delay.h>
 #include <linux/kernel.h>
+#include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/property.h>
 #include <linux/spi/spi.h>
@@ -117,7 +118,7 @@ struct ad5758_state {
 	unsigned int dc_dc_ilim;
 	unsigned int slew_time;
 	bool pwr_down;
-	__be32 d32[3];
+	__be32 d32[3] __aligned(IIO_DMA_MINALIGN);
 };
 
 /*
@@ -540,7 +541,7 @@ static int ad5758_read_raw(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_OFFSET:
 		min = st->out_range.min;
 		max = st->out_range.max;
-		*val = ((min * (1 << 16)) / (max - min)) / 1000;
+		*val = div_s64((s64)min * (1 << 16), max - min);
 		return IIO_VAL_INT;
 	default:
 		return -EINVAL;
@@ -556,6 +557,9 @@ static int ad5758_write_raw(struct iio_dev *indio_dev,
 
 	switch (info) {
 	case IIO_CHAN_INFO_RAW:
+		if (val < 0 || val > U16_MAX)
+			return -EINVAL;
+
 		mutex_lock(&st->lock);
 		ret = ad5758_spi_reg_write(st, AD5758_DAC_INPUT, val);
 		mutex_unlock(&st->lock);
diff --git a/drivers/iio/dac/ad8460.c b/drivers/iio/dac/ad8460.c
index a2292810e4cff5..469294fd178399 100644
--- a/drivers/iio/dac/ad8460.c
+++ b/drivers/iio/dac/ad8460.c
@@ -304,7 +304,7 @@ static ssize_t ad8460_write_powerdown(struct iio_dev *indio_dev, uintptr_t priva
 	 * If powerdown is set, HVDAC is enabled and the HV driver is
 	 * enabled via HV_RESET in case it is in shutdown mode,
 	 * If powerdown is cleared, HVDAC is set to shutdown state
-	 * as well as the HV driver. Quiescent current decreases and ouput is
+	 * as well as the HV driver. Quiescent current decreases and output is
 	 * floating (high impedance).
 	 */
 
diff --git a/drivers/iio/dac/mcp47a1.c b/drivers/iio/dac/mcp47a1.c
index 0bf994aa0e4f58..3ed306f3060f81 100644
--- a/drivers/iio/dac/mcp47a1.c
+++ b/drivers/iio/dac/mcp47a1.c
@@ -26,7 +26,7 @@ struct mcp47a1_data {
 	int vref_mV;
 };
 
-static const int mcp47a1_raw_avail[] = { 0, 1, MCP47A1_MAX_STEPS - 1 };
+static const int mcp47a1_raw_avail[] = { 0, 1, MCP47A1_MAX_STEPS };
 
 static const struct iio_chan_spec mcp47a1_channel = {
 	.type = IIO_VOLTAGE,
@@ -46,7 +46,7 @@ static int mcp47a1_write(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		if (!in_range(val, 0, MCP47A1_MAX_STEPS))
+		if (!in_range(val, 0, MCP47A1_MAX_STEPS + 1))
 			return -EINVAL;
 
 		return i2c_smbus_write_byte_data(data->client, MCP47A1_CMD_CODE,
diff --git a/drivers/iio/dac/mcp47feb02-core.c b/drivers/iio/dac/mcp47feb02-core.c
new file mode 100644
index 00000000000000..88bffbab08541f
--- /dev/null
+++ b/drivers/iio/dac/mcp47feb02-core.c
@@ -0,0 +1,984 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * IIO driver for MCP47FEB02 Multi-Channel DAC with I2C and SPI interface
+ *
+ * Copyright (C) 2025-2026 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Ariana Lazar <ariana.lazar@microchip.com>
+ *
+ * Datasheet links:
+ * [MCP47FEBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005375A.pdf
+ * [MCP47FVBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005405A.pdf
+ * [MCP47FxBx4/8] https://ww1.microchip.com/downloads/aemDocuments/documents/MSLD/ProductDocuments/DataSheets/MCP47FXBX48-Data-Sheet-DS200006368A.pdf
+ */
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/kstrtox.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/time64.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include "mcp47feb02.h"
+
+/* Register addresses must be left shifted with 3 positions in order to append command mask */
+#define MCP47FEB02_DAC0_REG_ADDR			0x00
+#define MCP47FEB02_VREF_REG_ADDR			0x40
+#define MCP47FEB02_POWER_DOWN_REG_ADDR			0x48
+#define MCP47FEB02_DAC_CTRL_MASK			GENMASK(1, 0)
+
+#define MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR		0x50
+#define MCP47FEB02_GAIN_BIT_STATUS_EEWA_MASK		BIT(6)
+#define MCP47FEB02_GAIN_BITS_MASK			GENMASK(15, 8)
+
+#define MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR		0x58
+
+#define MCP47FEB02_NV_DAC0_REG_ADDR			0x80
+#define MCP47FEB02_NV_VREF_REG_ADDR			0xC0
+#define MCP47FEB02_NV_POWER_DOWN_REG_ADDR		0xC8
+#define MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR	0xD0
+#define MCP47FEB02_NV_I2C_SLAVE_ADDR_MASK		GENMASK(7, 0)
+
+/* Voltage reference, Power-Down control register and DAC Wiperlock status register fields */
+#define MCP47FEB02_VREF_PD_MASK(ch)			(GENMASK(1, 0) << (2 * (ch)))
+
+/* Gain Control and I2C Slave Address Register fields */
+#define DAC_GAIN_MASK(ch)				(BIT(0) << (8 + (ch)))
+
+#define REG_ADDR(reg)					((reg) << 3)
+#define NV_REG_ADDR(reg)				((NV_DAC_ADDR_OFFSET + (reg)) << 3)
+#define READFLAG_MASK					GENMASK(2, 1)
+
+#define MCP47FEB02_MAX_CH				8
+#define MCP47FEB02_MAX_SCALES_CH			3
+#define MCP47FEB02_DAC_WIPER_UNLOCKED			0
+#define MCP47FEB02_NORMAL_OPERATION			0
+#define MCP47FEB02_INTERNAL_BAND_GAP_uV			2440000
+#define NV_DAC_ADDR_OFFSET				0x10
+
+enum mcp47feb02_vref_mode {
+	MCP47FEB02_VREF_VDD = 0,
+	MCP47FEB02_INTERNAL_BAND_GAP = 1,
+	MCP47FEB02_EXTERNAL_VREF_UNBUFFERED = 2,
+	MCP47FEB02_EXTERNAL_VREF_BUFFERED = 3,
+};
+
+enum mcp47feb02_scale {
+	MCP47FEB02_SCALE_VDD = 0,
+	MCP47FEB02_SCALE_GAIN_X1 = 1,
+	MCP47FEB02_SCALE_GAIN_X2 = 2,
+};
+
+enum mcp47feb02_gain_bit_mode {
+	MCP47FEB02_GAIN_BIT_X1 = 0,
+	MCP47FEB02_GAIN_BIT_X2 = 1,
+};
+
+static const char * const mcp47feb02_powerdown_modes[] = {
+	"1kohm_to_gnd",
+	"100kohm_to_gnd",
+	"open_circuit",
+};
+
+/**
+ * struct mcp47feb02_channel_data - channel configuration
+ * @ref_mode: chosen voltage for reference
+ * @use_2x_gain: output driver gain control
+ * @powerdown: is false if the channel is in normal operation mode
+ * @powerdown_mode: selected power-down mode
+ * @dac_data: dac value
+ */
+struct mcp47feb02_channel_data {
+	u8 ref_mode;
+	bool use_2x_gain;
+	bool powerdown;
+	u8 powerdown_mode;
+	u16 dac_data;
+};
+
+/**
+ * struct mcp47feb02_data - chip configuration
+ * @chdata: options configured for each channel on the device
+ * @lock: prevents concurrent reads/writes to driver's state members
+ * @chip_features: pointer to features struct
+ * @scale_1: scales set on channels that are based on Vref1
+ * @scale: scales set on channels that are based on Vref/Vref0
+ * @active_channels_mask: enabled channels
+ * @regmap: regmap for directly accessing device register
+ * @labels: table with channels labels
+ * @phys_channels: physical channels on the device
+ * @vref1_buffered: Vref1 buffer is enabled
+ * @vref_buffered: Vref/Vref0 buffer is enabled
+ * @use_vref1: vref1-supply is defined
+ * @use_vref: vref-supply is defined
+ */
+struct mcp47feb02_data {
+	struct mcp47feb02_channel_data chdata[MCP47FEB02_MAX_CH];
+	struct mutex lock; /* prevents concurrent reads/writes to driver's state members */
+	const struct mcp47feb02_features *chip_features;
+	int scale_1[2 * MCP47FEB02_MAX_SCALES_CH];
+	int scale[2 * MCP47FEB02_MAX_SCALES_CH];
+	unsigned long active_channels_mask;
+	struct regmap *regmap;
+	const char *labels[MCP47FEB02_MAX_CH];
+	u16 phys_channels;
+	bool vref1_buffered;
+	bool vref_buffered;
+	bool use_vref1;
+	bool use_vref;
+};
+
+static const struct regmap_range mcp47feb02_readable_ranges[] = {
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
+};
+
+static const struct regmap_range mcp47feb02_writable_ranges[] = {
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
+};
+
+static const struct regmap_range mcp47feb02_volatile_ranges[] = {
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
+};
+
+static const struct regmap_access_table mcp47feb02_readable_table = {
+	.yes_ranges = mcp47feb02_readable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(mcp47feb02_readable_ranges),
+};
+
+static const struct regmap_access_table mcp47feb02_writable_table = {
+	.yes_ranges = mcp47feb02_writable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(mcp47feb02_writable_ranges),
+};
+
+static const struct regmap_access_table mcp47feb02_volatile_table = {
+	.yes_ranges = mcp47feb02_volatile_ranges,
+	.n_yes_ranges = ARRAY_SIZE(mcp47feb02_volatile_ranges),
+};
+
+const struct regmap_config mcp47feb02_regmap_config = {
+	.name = "mcp47feb02_regmap",
+	.reg_bits = 8,
+	.val_bits = 16,
+	.rd_table = &mcp47feb02_readable_table,
+	.wr_table = &mcp47feb02_writable_table,
+	.volatile_table = &mcp47feb02_volatile_table,
+	.max_register = MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR,
+	.read_flag_mask = READFLAG_MASK,
+	.cache_type = REGCACHE_MAPLE,
+	.val_format_endian = REGMAP_ENDIAN_BIG,
+};
+EXPORT_SYMBOL_NS_GPL(mcp47feb02_regmap_config, "IIO_MCP47FEB02");
+
+/* For devices that doesn't have nonvolatile memory */
+static const struct regmap_range mcp47fvb02_readable_ranges[] = {
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+};
+
+static const struct regmap_range mcp47fvb02_writable_ranges[] = {
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+};
+
+static const struct regmap_range mcp47fvb02_volatile_ranges[] = {
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
+};
+
+static const struct regmap_access_table mcp47fvb02_readable_table = {
+	.yes_ranges = mcp47fvb02_readable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(mcp47fvb02_readable_ranges),
+};
+
+static const struct regmap_access_table mcp47fvb02_writable_table = {
+	.yes_ranges = mcp47fvb02_writable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(mcp47fvb02_writable_ranges),
+};
+
+static const struct regmap_access_table mcp47fvb02_volatile_table = {
+	.yes_ranges = mcp47fvb02_volatile_ranges,
+	.n_yes_ranges = ARRAY_SIZE(mcp47fvb02_volatile_ranges),
+};
+
+const struct regmap_config mcp47fvb02_regmap_config = {
+	.name = "mcp47fvb02_regmap",
+	.reg_bits = 8,
+	.val_bits = 16,
+	.rd_table = &mcp47fvb02_readable_table,
+	.wr_table = &mcp47fvb02_writable_table,
+	.volatile_table = &mcp47fvb02_volatile_table,
+	.max_register = MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR,
+	.read_flag_mask = READFLAG_MASK,
+	.cache_type = REGCACHE_MAPLE,
+	.val_format_endian = REGMAP_ENDIAN_BIG,
+};
+EXPORT_SYMBOL_NS_GPL(mcp47fvb02_regmap_config, "IIO_MCP47FEB02");
+
+static int mcp47feb02_write_to_eeprom(struct mcp47feb02_data *data, unsigned int reg,
+				      unsigned int val)
+{
+	int eewa_val, ret;
+
+	/*
+	 * Wait until the currently occurring EEPROM Write Cycle is completed.
+	 * Only serial commands to the volatile memory are allowed.
+	 */
+	ret = regmap_read_poll_timeout(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR,
+				       eewa_val,
+				       !(eewa_val & MCP47FEB02_GAIN_BIT_STATUS_EEWA_MASK),
+				       1 * USEC_PER_MSEC, 150 * USEC_PER_MSEC);
+	if (ret)
+		return ret;
+
+	return regmap_write(data->regmap, reg, val);
+}
+
+static ssize_t store_eeprom_store(struct device *dev, struct device_attribute *attr,
+				  const char *buf, size_t len)
+{
+	struct mcp47feb02_data *data = iio_priv(dev_to_iio_dev(dev));
+	unsigned int i, val, val1, eewa_val;
+	bool state;
+	int ret;
+
+	ret = kstrtobool(buf, &state);
+	if (ret)
+		return ret;
+
+	if (!state)
+		return len;
+
+	guard(mutex)(&data->lock);
+
+	/*
+	 * Verify DAC Wiper and DAC Configuration are unlocked. If both are disabled,
+	 * writing to EEPROM is available.
+	 */
+	ret = regmap_read(data->regmap, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR, &val);
+	if (ret)
+		return ret;
+
+	if (val) {
+		dev_err(dev, "DAC Wiper and DAC Configuration not are unlocked.\n");
+		return -EINVAL;
+	}
+
+	for_each_set_bit(i, &data->active_channels_mask, data->phys_channels) {
+		ret = mcp47feb02_write_to_eeprom(data, NV_REG_ADDR(i),
+						 data->chdata[i].dac_data);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_read(data->regmap, MCP47FEB02_VREF_REG_ADDR, &val);
+	if (ret)
+		return ret;
+
+	ret = mcp47feb02_write_to_eeprom(data, MCP47FEB02_NV_VREF_REG_ADDR, val);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR, &val);
+	if (ret)
+		return ret;
+
+	ret = mcp47feb02_write_to_eeprom(data, MCP47FEB02_NV_POWER_DOWN_REG_ADDR, val);
+	if (ret)
+		return ret;
+
+	ret = regmap_read_poll_timeout(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR, eewa_val,
+				       !(eewa_val & MCP47FEB02_GAIN_BIT_STATUS_EEWA_MASK),
+				       1 * USEC_PER_MSEC, 150 * USEC_PER_MSEC);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(data->regmap, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR, &val);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR, &val1);
+	if (ret)
+		return ret;
+
+	ret = mcp47feb02_write_to_eeprom(data, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR,
+					 (val1 & MCP47FEB02_GAIN_BITS_MASK) |
+					 (val & MCP47FEB02_NV_I2C_SLAVE_ADDR_MASK));
+	if (ret)
+		return ret;
+
+	return len;
+}
+
+static IIO_DEVICE_ATTR_WO(store_eeprom, 0);
+
+static struct attribute *mcp47feb02_attributes[] = {
+	&iio_dev_attr_store_eeprom.dev_attr.attr,
+	NULL
+};
+
+static const struct attribute_group mcp47feb02_attribute_group = {
+	.attrs = mcp47feb02_attributes,
+};
+
+static int mcp47feb02_suspend(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+	int ret;
+	u8 ch;
+
+	guard(mutex)(&data->lock);
+
+	for_each_set_bit(ch, &data->active_channels_mask, data->phys_channels) {
+		u8 pd_mode;
+		u32 mask;
+
+		data->chdata[ch].powerdown = true;
+		pd_mode = data->chdata[ch].powerdown_mode + 1;
+		mask = MCP47FEB02_VREF_PD_MASK(ch);
+		ret = regmap_update_bits(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR,
+					 mask, field_prep(mask, pd_mode));
+		if (ret)
+			return ret;
+
+		ret = regmap_write(data->regmap, REG_ADDR(ch), data->chdata[ch].dac_data);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int mcp47feb02_resume(struct device *dev)
+{
+	struct iio_dev *indio_dev = dev_get_drvdata(dev);
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+	u8 ch;
+
+	guard(mutex)(&data->lock);
+
+	for_each_set_bit(ch, &data->active_channels_mask, data->phys_channels) {
+		u32 mask;
+		int ret;
+
+		data->chdata[ch].powerdown = false;
+
+		ret = regmap_write(data->regmap, REG_ADDR(ch), data->chdata[ch].dac_data);
+		if (ret)
+			return ret;
+
+		mask =  MCP47FEB02_VREF_PD_MASK(ch);
+		ret = regmap_update_bits(data->regmap, MCP47FEB02_VREF_REG_ADDR,
+					 mask, field_prep(mask, data->chdata[ch].ref_mode));
+		if (ret)
+			return ret;
+
+		mask = DAC_GAIN_MASK(ch);
+		ret = regmap_update_bits(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR,
+					 mask, field_prep(mask, data->chdata[ch].use_2x_gain));
+		if (ret)
+			return ret;
+
+		mask = MCP47FEB02_VREF_PD_MASK(ch);
+		ret = regmap_update_bits(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR,
+					 mask, field_prep(mask, MCP47FEB02_NORMAL_OPERATION));
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int mcp47feb02_get_powerdown_mode(struct iio_dev *indio_dev,
+					 const struct iio_chan_spec *chan)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+
+	return data->chdata[chan->address].powerdown_mode;
+}
+
+static int mcp47feb02_set_powerdown_mode(struct iio_dev *indio_dev, const struct iio_chan_spec *ch,
+					 unsigned int mode)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+
+	data->chdata[ch->address].powerdown_mode = mode;
+
+	return 0;
+}
+
+static ssize_t mcp47feb02_read_powerdown(struct iio_dev *indio_dev, uintptr_t private,
+					 const struct iio_chan_spec *ch, char *buf)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+
+	/* Print if channel is in a power-down mode or not */
+	return sysfs_emit(buf, "%d\n", data->chdata[ch->address].powerdown);
+}
+
+static ssize_t mcp47feb02_write_powerdown(struct iio_dev *indio_dev, uintptr_t private,
+					  const struct iio_chan_spec *ch, const char *buf,
+					  size_t len)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+	u32 reg = ch->address;
+	u8 tmp_pd_mode;
+	bool state;
+	u32 mask;
+	int ret;
+
+	guard(mutex)(&data->lock);
+
+	ret = kstrtobool(buf, &state);
+	if (ret)
+		return ret;
+
+	/*
+	 * Set the channel to the specified power-down mode. Exiting power-down mode
+	 * requires writing normal operation mode (0) to the channel-specific register bits.
+	 */
+	tmp_pd_mode = state ? (data->chdata[reg].powerdown_mode + 1) : MCP47FEB02_NORMAL_OPERATION;
+	mask = MCP47FEB02_VREF_PD_MASK(reg);
+	ret = regmap_update_bits(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR,
+				 mask, field_prep(mask, tmp_pd_mode));
+	if (ret)
+		return ret;
+
+	data->chdata[reg].powerdown = state;
+
+	return len;
+}
+
+EXPORT_NS_SIMPLE_DEV_PM_OPS(mcp47feb02_pm_ops, mcp47feb02_suspend,
+			    mcp47feb02_resume, IIO_MCP47FEB02);
+
+static const struct iio_enum mcp47febxx_powerdown_mode_enum = {
+	.items = mcp47feb02_powerdown_modes,
+	.num_items = ARRAY_SIZE(mcp47feb02_powerdown_modes),
+	.get = mcp47feb02_get_powerdown_mode,
+	.set = mcp47feb02_set_powerdown_mode,
+};
+
+static const struct iio_chan_spec_ext_info mcp47feb02_ext_info[] = {
+	{
+		.name = "powerdown",
+		.read = mcp47feb02_read_powerdown,
+		.write = mcp47feb02_write_powerdown,
+		.shared = IIO_SEPARATE,
+	},
+	IIO_ENUM("powerdown_mode", IIO_SEPARATE, &mcp47febxx_powerdown_mode_enum),
+	IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &mcp47febxx_powerdown_mode_enum),
+	{ }
+};
+
+static const struct iio_chan_spec mcp47febxx_ch_template = {
+	.type = IIO_VOLTAGE,
+	.output = 1,
+	.indexed = 1,
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+	.info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE),
+	.ext_info = mcp47feb02_ext_info,
+};
+
+static void mcp47feb02_init_scale(struct mcp47feb02_data *data, enum mcp47feb02_scale scale,
+				  int vref_uV, int scale_avail[])
+{
+	u32 value_micro, value_int;
+	u64 tmp;
+
+	/* vref_uV should not be negative */
+	tmp = (u64)vref_uV * MILLI >> data->chip_features->resolution;
+	value_int = div_u64_rem(tmp, MICRO, &value_micro);
+	scale_avail[scale * 2] = value_int;
+	scale_avail[scale * 2 + 1] = value_micro;
+}
+
+static int mcp47feb02_init_scales_avail(struct mcp47feb02_data *data, int vdd_uV,
+					int vref_uV, int vref1_uV)
+{
+	int tmp_vref;
+
+	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_uV, data->scale);
+
+	if (data->use_vref)
+		tmp_vref = vref_uV;
+	else
+		tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_uV;
+
+	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X1, tmp_vref, data->scale);
+	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X2, tmp_vref * 2, data->scale);
+
+	if (data->phys_channels >= 4) {
+		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_uV, data->scale_1);
+
+		if (data->use_vref1)
+			tmp_vref = vref1_uV;
+		else
+			tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_uV;
+
+		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X1,
+				      tmp_vref, data->scale_1);
+		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X2,
+				      tmp_vref * 2, data->scale_1);
+	}
+
+	return 0;
+}
+
+static int mcp47feb02_read_avail(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
+				 const int **vals, int *type, int *length, long info)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+
+	switch (info) {
+	case IIO_CHAN_INFO_SCALE:
+		switch (ch->type) {
+		case IIO_VOLTAGE:
+			if (data->phys_channels >= 4 && (ch->address % 2))
+				*vals = data->scale_1;
+			else
+				*vals = data->scale;
+
+			*length = 2 * MCP47FEB02_MAX_SCALES_CH;
+			*type = IIO_VAL_INT_PLUS_MICRO;
+			return IIO_AVAIL_LIST;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static void mcp47feb02_get_scale(int ch, struct mcp47feb02_data *data, int *val, int *val2)
+{
+	enum mcp47feb02_scale current_scale;
+
+	if (data->chdata[ch].ref_mode == MCP47FEB02_VREF_VDD)
+		current_scale = MCP47FEB02_SCALE_VDD;
+	else if (data->chdata[ch].use_2x_gain)
+		current_scale = MCP47FEB02_SCALE_GAIN_X2;
+	else
+		current_scale = MCP47FEB02_SCALE_GAIN_X1;
+
+	if (data->phys_channels >= 4 && (ch % 2)) {
+		*val = data->scale_1[current_scale * 2];
+		*val2 = data->scale_1[current_scale * 2 + 1];
+	} else {
+		*val = data->scale[current_scale * 2];
+		*val2 = data->scale[current_scale * 2 + 1];
+	}
+}
+
+static int mcp47feb02_check_scale(struct mcp47feb02_data *data, int val, int val2, int scale[])
+{
+	unsigned int i;
+
+	for (i = 0; i < MCP47FEB02_MAX_SCALES_CH; i++) {
+		if (scale[i * 2] == val && scale[i * 2 + 1] == val2)
+			return i;
+	}
+
+	return -EINVAL;
+}
+
+static int mcp47feb02_ch_scale(struct mcp47feb02_data *data, int ch, int scale)
+{
+	int tmp_val, ret;
+	u32 mask;
+
+	if (scale == MCP47FEB02_SCALE_VDD) {
+		tmp_val = MCP47FEB02_VREF_VDD;
+	} else if (data->phys_channels >= 4 && (ch % 2)) {
+		if (data->use_vref1) {
+			if (data->vref1_buffered)
+				tmp_val = MCP47FEB02_EXTERNAL_VREF_BUFFERED;
+			else
+				tmp_val = MCP47FEB02_EXTERNAL_VREF_UNBUFFERED;
+		} else {
+			tmp_val = MCP47FEB02_INTERNAL_BAND_GAP;
+		}
+	} else if (data->use_vref) {
+		if (data->vref_buffered)
+			tmp_val = MCP47FEB02_EXTERNAL_VREF_BUFFERED;
+		else
+			tmp_val = MCP47FEB02_EXTERNAL_VREF_UNBUFFERED;
+	} else {
+		tmp_val = MCP47FEB02_INTERNAL_BAND_GAP;
+	}
+
+	mask = MCP47FEB02_VREF_PD_MASK(ch);
+	ret = regmap_update_bits(data->regmap, MCP47FEB02_VREF_REG_ADDR,
+				 mask, field_prep(mask, tmp_val));
+	if (ret)
+		return ret;
+
+	data->chdata[ch].ref_mode = tmp_val;
+
+	return 0;
+}
+
+/*
+ * Setting the scale in order to choose between VDD and (Vref or Band Gap) from the user
+ * space. The VREF pin is either an input or an output, therefore the user cannot
+ * simultaneously connect an external voltage reference to the pin and select the
+ * internal Band Gap.
+ * When the DAC’s voltage reference is configured as the VREF pin, the pin is an input.
+ * When the DAC’s voltage reference is configured as the internal Band Gap,
+ * the VREF pin is an output.
+ * If Vref/Vref1 voltage is not available, then the internal Band Gap will be used
+ * to calculate the values for the scale.
+ */
+static int mcp47feb02_set_scale(struct mcp47feb02_data *data, int ch, int scale)
+{
+	int tmp_val, ret;
+	u32 mask;
+
+	ret = mcp47feb02_ch_scale(data, ch, scale);
+	if (ret)
+		return ret;
+
+	if (scale == MCP47FEB02_SCALE_GAIN_X2)
+		tmp_val = MCP47FEB02_GAIN_BIT_X2;
+	else
+		tmp_val = MCP47FEB02_GAIN_BIT_X1;
+
+	mask = DAC_GAIN_MASK(ch);
+	ret = regmap_update_bits(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR,
+				 mask, field_prep(mask, tmp_val));
+	if (ret)
+		return ret;
+
+	data->chdata[ch].use_2x_gain = tmp_val;
+
+	return 0;
+}
+
+static int mcp47feb02_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
+			       int *val, int *val2, long mask)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = regmap_read(data->regmap, REG_ADDR(ch->address), val);
+		if (ret)
+			return ret;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		mcp47feb02_get_scale(ch->address, data, val, val2);
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int mcp47feb02_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
+				int val, int val2, long mask)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+	int *tmp_scale, ret;
+
+	guard(mutex)(&data->lock);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = regmap_write(data->regmap, REG_ADDR(ch->address), val);
+		if (ret)
+			return ret;
+
+		data->chdata[ch->address].dac_data = val;
+		return 0;
+	case IIO_CHAN_INFO_SCALE:
+		if (data->phys_channels >= 4 && (ch->address % 2))
+			tmp_scale = data->scale_1;
+		else
+			tmp_scale = data->scale;
+
+		ret = mcp47feb02_check_scale(data, val, val2, tmp_scale);
+		if (ret < 0)
+			return ret;
+
+		return mcp47feb02_set_scale(data, ch->address, ret);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int mcp47feb02_read_label(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
+				 char *label)
+{
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+
+	if (data->labels[ch->address])
+		return sysfs_emit(label, "%s\n", data->labels[ch->address]);
+
+	return -ENOENT;
+}
+
+static const struct iio_info mcp47feb02_info = {
+	.read_raw = mcp47feb02_read_raw,
+	.write_raw = mcp47feb02_write_raw,
+	.read_label = mcp47feb02_read_label,
+	.read_avail = &mcp47feb02_read_avail,
+	.attrs = &mcp47feb02_attribute_group,
+};
+
+static const struct iio_info mcp47fvb02_info = {
+	.read_raw = mcp47feb02_read_raw,
+	.write_raw = mcp47feb02_write_raw,
+	.read_label = mcp47feb02_read_label,
+	.read_avail = &mcp47feb02_read_avail,
+};
+
+static int mcp47feb02_parse_fw(struct iio_dev *indio_dev,
+			       const struct mcp47feb02_features *chip_features)
+{
+	struct iio_chan_spec chanspec = mcp47febxx_ch_template;
+	struct mcp47feb02_data *data = iio_priv(indio_dev);
+	struct device *dev = regmap_get_device(data->regmap);
+	struct iio_chan_spec *channels;
+	u32 num_channels;
+	u8 chan_idx = 0;
+
+	num_channels = device_get_child_node_count(dev);
+	if (num_channels > chip_features->phys_channels)
+		return dev_err_probe(dev, -EINVAL, "More channels than the chip supports\n");
+
+	if (!num_channels)
+		return dev_err_probe(dev, -EINVAL, "No channel specified in the devicetree.\n");
+
+	channels = devm_kcalloc(dev, num_channels, sizeof(*channels), GFP_KERNEL);
+	if (!channels)
+		return -ENOMEM;
+
+	device_for_each_child_node_scoped(dev, child) {
+		u32 reg = 0;
+		int ret;
+
+		ret = fwnode_property_read_u32(child, "reg", &reg);
+		if (ret)
+			return dev_err_probe(dev, ret, "Invalid channel number\n");
+
+		if (reg >= chip_features->phys_channels)
+			return dev_err_probe(dev, -EINVAL,
+					     "The index of the channels does not match the chip\n");
+
+		set_bit(reg, &data->active_channels_mask);
+
+		if (fwnode_property_present(child, "label")) {
+			ret = fwnode_property_read_string(child, "label", &data->labels[reg]);
+			if (ret)
+				dev_warn_probe(dev, ret, "%pfw: invalid label\n", child);
+		}
+
+		chanspec.address = reg;
+		chanspec.channel = reg;
+		channels[chan_idx] = chanspec;
+		chan_idx++;
+	}
+
+	indio_dev->num_channels = num_channels;
+	indio_dev->channels = channels;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	data->phys_channels = chip_features->phys_channels;
+
+	data->vref_buffered = device_property_read_bool(dev, "microchip,vref-buffered");
+
+	if (chip_features->have_ext_vref1)
+		data->vref1_buffered = device_property_read_bool(dev, "microchip,vref1-buffered");
+
+	return 0;
+}
+
+static int mcp47feb02_init_ctrl_regs(struct mcp47feb02_data *data)
+{
+	unsigned int i, vref_ch, gain_ch, pd_ch;
+	int ret;
+
+	ret = regmap_read(data->regmap, MCP47FEB02_VREF_REG_ADDR, &vref_ch);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR, &gain_ch);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR, &pd_ch);
+	if (ret)
+		return ret;
+
+	for_each_set_bit(i, &data->active_channels_mask, data->phys_channels) {
+		struct device *dev = regmap_get_device(data->regmap);
+		unsigned int pd_tmp, dac_val;
+
+		ret = regmap_read(data->regmap, REG_ADDR(i), &dac_val);
+		if (ret)
+			return ret;
+		data->chdata[i].dac_data = dac_val;
+
+		data->chdata[i].ref_mode = field_get(MCP47FEB02_VREF_PD_MASK(i), vref_ch);
+		data->chdata[i].use_2x_gain = field_get(DAC_GAIN_MASK(i), gain_ch);
+
+		/*
+		 * Inform the user that the current voltage reference read from the volatile
+		 * register of the chip is different from the one specified in the device tree.
+		 * Considering that the user cannot have an external voltage reference connected
+		 * to the pin and select the internal Band Gap at the same time, in order to avoid
+		 * miscofiguring the reference voltage, the volatile register will not be written.
+		 * In order to overwrite the setting from volatile register with the one from the
+		 * device tree, the user needs to write the chosen scale.
+		 */
+		switch (data->chdata[i].ref_mode) {
+		case MCP47FEB02_INTERNAL_BAND_GAP:
+			if (data->phys_channels >= 4 && (i % 2) && data->use_vref1) {
+				dev_dbg(dev, "ch[%u]: was configured to use internal band gap", i);
+				dev_dbg(dev, "ch[%u]: reference voltage set to VREF1", i);
+				break;
+			}
+			if ((data->phys_channels < 4 || (data->phys_channels >= 4 && !(i % 2))) &&
+			    data->use_vref) {
+				dev_dbg(dev, "ch[%u]: was configured to use internal band gap", i);
+				dev_dbg(dev, "ch[%u]: reference voltage set to VREF", i);
+				break;
+			}
+			break;
+		case MCP47FEB02_EXTERNAL_VREF_UNBUFFERED:
+		case MCP47FEB02_EXTERNAL_VREF_BUFFERED:
+			if (data->phys_channels >= 4 && (i % 2) && !data->use_vref1) {
+				dev_dbg(dev, "ch[%u]: was configured to use VREF1", i);
+				dev_dbg(dev,
+					"ch[%u]: reference voltage set to internal band gap", i);
+				break;
+			}
+			if ((data->phys_channels < 4 || (data->phys_channels >= 4 && !(i % 2))) &&
+			    !data->use_vref) {
+				dev_dbg(dev, "ch[%u]: was configured to use VREF", i);
+				dev_dbg(dev,
+					"ch[%u]: reference voltage set to internal band gap", i);
+				break;
+			}
+			break;
+		}
+
+		pd_tmp = field_get(MCP47FEB02_VREF_PD_MASK(i), pd_ch);
+		data->chdata[i].powerdown_mode = pd_tmp ? (pd_tmp - 1) : pd_tmp;
+		data->chdata[i].powerdown = !!(data->chdata[i].powerdown_mode);
+	}
+
+	return 0;
+}
+
+static int mcp47feb02_init_ch_scales(struct mcp47feb02_data *data, int vdd_uV,
+				     int vref_uV, int vref1_uV)
+{
+	unsigned int i;
+
+	for_each_set_bit(i, &data->active_channels_mask, data->phys_channels) {
+		struct device *dev = regmap_get_device(data->regmap);
+		int ret;
+
+		ret = mcp47feb02_init_scales_avail(data, vdd_uV, vref_uV, vref1_uV);
+		if (ret)
+			return dev_err_probe(dev, ret, "failed to init scales for ch %u\n", i);
+	}
+
+	return 0;
+}
+
+int mcp47feb02_common_probe(const struct mcp47feb02_features *chip_features,
+			    struct regmap *regmap)
+{
+	struct device *dev = regmap_get_device(regmap);
+	struct mcp47feb02_data *data;
+	struct iio_dev *indio_dev;
+	int vref1_uV, vref_uV, vdd_uV, ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, indio_dev);
+
+	data = iio_priv(indio_dev);
+	data->chip_features = chip_features;
+	data->regmap = regmap;
+
+	if (chip_features->have_eeprom)
+		indio_dev->info = &mcp47feb02_info;
+	else
+		indio_dev->info = &mcp47fvb02_info;
+
+	indio_dev->name = chip_features->name;
+
+	ret = mcp47feb02_parse_fw(indio_dev, chip_features);
+	if (ret)
+		return dev_err_probe(dev, ret, "Error parsing firmware data\n");
+
+	ret = devm_mutex_init(dev, &data->lock);
+	if (ret)
+		return ret;
+
+	ret = devm_regulator_get_enable_read_voltage(dev, "vdd");
+	if (ret < 0)
+		return ret;
+
+	vdd_uV = ret;
+
+	if (device_property_present(dev, "vref-supply")) {
+		vref_uV = devm_regulator_get_enable_read_voltage(dev, "vref");
+		if (vref_uV < 0)
+			return vref_uV;
+
+		if (vref_uV == 0)
+			return dev_err_probe(dev, -EINVAL, "Vref is 0 uV.\n");
+
+		data->use_vref = true;
+	} else {
+		vref_uV = 0;
+		dev_dbg(dev, "Using internal band gap as voltage reference.\n");
+	}
+
+	if (chip_features->have_ext_vref1 &&
+	    device_property_present(dev, "vref1-supply")) {
+		vref1_uV = devm_regulator_get_enable_read_voltage(dev, "vref1");
+		if (vref1_uV < 0)
+			return vref1_uV;
+
+		if (vref1_uV == 0)
+			return dev_err_probe(dev, -EINVAL, "Vref1 is 0 uV.\n");
+
+		data->use_vref1 = true;
+	} else {
+		vref1_uV = 0;
+		dev_dbg(dev, "Using internal band gap as voltage reference 1.\n");
+	}
+
+	ret = mcp47feb02_init_ctrl_regs(data);
+	if (ret)
+		return dev_err_probe(dev, ret, "Error initialising vref register\n");
+
+	ret = mcp47feb02_init_ch_scales(data, vdd_uV, vref_uV, vref1_uV);
+	if (ret)
+		return ret;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+EXPORT_SYMBOL_NS(mcp47feb02_common_probe, "IIO_MCP47FEB02");
+
+MODULE_AUTHOR("Ariana Lazar <ariana.lazar@microchip.com>");
+MODULE_DESCRIPTION("IIO driver for MCP47FEB02/MCP48FEB02 Multi-Channel DAC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/dac/mcp47feb02-i2c.c b/drivers/iio/dac/mcp47feb02-i2c.c
new file mode 100644
index 00000000000000..7ec0cccad79bc9
--- /dev/null
+++ b/drivers/iio/dac/mcp47feb02-i2c.c
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * IIO driver for MCP47FEB02 Multi-Channel DAC with I2C interface
+ *
+ * Copyright (C) 2025-2026 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Ariana Lazar <ariana.lazar@microchip.com>
+ *
+ * Datasheet links for devices with I2C interface:
+ * [MCP47FEBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005375A.pdf
+ * [MCP47FVBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005405A.pdf
+ * [MCP47FxBx4/8] https://ww1.microchip.com/downloads/aemDocuments/documents/MSLD/ProductDocuments/DataSheets/MCP47FXBX48-Data-Sheet-DS200006368A.pdf
+ */
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/pm.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#include "mcp47feb02.h"
+
+/* Parts with EEPROM memory */
+MCP47FEB02_CHIP_INFO(mcp47feb01, 1, 8,  false, true);
+MCP47FEB02_CHIP_INFO(mcp47feb02, 2, 8,  false, true);
+MCP47FEB02_CHIP_INFO(mcp47feb04, 4, 8,  true,  true);
+MCP47FEB02_CHIP_INFO(mcp47feb08, 8, 8,  true,  true);
+MCP47FEB02_CHIP_INFO(mcp47feb11, 1, 10, false, true);
+MCP47FEB02_CHIP_INFO(mcp47feb12, 2, 10, false, true);
+MCP47FEB02_CHIP_INFO(mcp47feb14, 4, 10, true,  true);
+MCP47FEB02_CHIP_INFO(mcp47feb18, 8, 10, true,  true);
+MCP47FEB02_CHIP_INFO(mcp47feb21, 1, 12, false, true);
+MCP47FEB02_CHIP_INFO(mcp47feb22, 2, 12, false, true);
+MCP47FEB02_CHIP_INFO(mcp47feb24, 4, 12, true,  true);
+MCP47FEB02_CHIP_INFO(mcp47feb28, 8, 12, true,  true);
+
+/* Parts without EEPROM memory */
+MCP47FEB02_CHIP_INFO(mcp47fvb01, 1, 8,  false, false);
+MCP47FEB02_CHIP_INFO(mcp47fvb02, 2, 8,  false, false);
+MCP47FEB02_CHIP_INFO(mcp47fvb04, 4, 8,  true,  false);
+MCP47FEB02_CHIP_INFO(mcp47fvb08, 8, 8,  true,  false);
+MCP47FEB02_CHIP_INFO(mcp47fvb11, 1, 10, false, false);
+MCP47FEB02_CHIP_INFO(mcp47fvb12, 2, 10, false, false);
+MCP47FEB02_CHIP_INFO(mcp47fvb14, 4, 10, true,  false);
+MCP47FEB02_CHIP_INFO(mcp47fvb18, 8, 10, true,  false);
+MCP47FEB02_CHIP_INFO(mcp47fvb21, 1, 12, false, false);
+MCP47FEB02_CHIP_INFO(mcp47fvb22, 2, 12, false, false);
+MCP47FEB02_CHIP_INFO(mcp47fvb24, 4, 12, true,  false);
+MCP47FEB02_CHIP_INFO(mcp47fvb28, 8, 12, true,  false);
+
+static int mcp47feb02_i2c_probe(struct i2c_client *client)
+{
+	const struct mcp47feb02_features *chip_features;
+	struct device *dev = &client->dev;
+	struct regmap *regmap;
+
+	chip_features = i2c_get_match_data(client);
+	if (!chip_features)
+		return dev_err_probe(dev, -ENODATA, "No match data found\n");
+
+	if (chip_features->have_eeprom)
+		regmap = devm_regmap_init_i2c(client, &mcp47feb02_regmap_config);
+	else
+		regmap = devm_regmap_init_i2c(client, &mcp47fvb02_regmap_config);
+
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Error initializing I2C regmap\n");
+
+	return mcp47feb02_common_probe(chip_features, regmap);
+}
+
+static const struct i2c_device_id mcp47feb02_i2c_id[] = {
+	{ .name = "mcp47feb01", .driver_data = (kernel_ulong_t)&mcp47feb01_chip_features },
+	{ .name = "mcp47feb02", .driver_data = (kernel_ulong_t)&mcp47feb02_chip_features },
+	{ .name = "mcp47feb04", .driver_data = (kernel_ulong_t)&mcp47feb04_chip_features },
+	{ .name = "mcp47feb08", .driver_data = (kernel_ulong_t)&mcp47feb08_chip_features },
+	{ .name = "mcp47feb11", .driver_data = (kernel_ulong_t)&mcp47feb11_chip_features },
+	{ .name = "mcp47feb12", .driver_data = (kernel_ulong_t)&mcp47feb12_chip_features },
+	{ .name = "mcp47feb14", .driver_data = (kernel_ulong_t)&mcp47feb14_chip_features },
+	{ .name = "mcp47feb18", .driver_data = (kernel_ulong_t)&mcp47feb18_chip_features },
+	{ .name = "mcp47feb21", .driver_data = (kernel_ulong_t)&mcp47feb21_chip_features },
+	{ .name = "mcp47feb22", .driver_data = (kernel_ulong_t)&mcp47feb22_chip_features },
+	{ .name = "mcp47feb24", .driver_data = (kernel_ulong_t)&mcp47feb24_chip_features },
+	{ .name = "mcp47feb28", .driver_data = (kernel_ulong_t)&mcp47feb28_chip_features },
+	{ .name = "mcp47fvb01", .driver_data = (kernel_ulong_t)&mcp47fvb01_chip_features },
+	{ .name = "mcp47fvb02", .driver_data = (kernel_ulong_t)&mcp47fvb02_chip_features },
+	{ .name = "mcp47fvb04", .driver_data = (kernel_ulong_t)&mcp47fvb04_chip_features },
+	{ .name = "mcp47fvb08", .driver_data = (kernel_ulong_t)&mcp47fvb08_chip_features },
+	{ .name = "mcp47fvb11", .driver_data = (kernel_ulong_t)&mcp47fvb11_chip_features },
+	{ .name = "mcp47fvb12", .driver_data = (kernel_ulong_t)&mcp47fvb12_chip_features },
+	{ .name = "mcp47fvb14", .driver_data = (kernel_ulong_t)&mcp47fvb14_chip_features },
+	{ .name = "mcp47fvb18", .driver_data = (kernel_ulong_t)&mcp47fvb18_chip_features },
+	{ .name = "mcp47fvb21", .driver_data = (kernel_ulong_t)&mcp47fvb21_chip_features },
+	{ .name = "mcp47fvb22", .driver_data = (kernel_ulong_t)&mcp47fvb22_chip_features },
+	{ .name = "mcp47fvb24", .driver_data = (kernel_ulong_t)&mcp47fvb24_chip_features },
+	{ .name = "mcp47fvb28", .driver_data = (kernel_ulong_t)&mcp47fvb28_chip_features },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, mcp47feb02_i2c_id);
+
+static const struct of_device_id mcp47feb02_of_i2c_match[] = {
+	{ .compatible = "microchip,mcp47feb01", .data = &mcp47feb01_chip_features },
+	{ .compatible = "microchip,mcp47feb02", .data = &mcp47feb02_chip_features },
+	{ .compatible = "microchip,mcp47feb04", .data = &mcp47feb04_chip_features },
+	{ .compatible = "microchip,mcp47feb08", .data = &mcp47feb08_chip_features },
+	{ .compatible = "microchip,mcp47feb11", .data = &mcp47feb11_chip_features },
+	{ .compatible = "microchip,mcp47feb12", .data = &mcp47feb12_chip_features },
+	{ .compatible = "microchip,mcp47feb14", .data = &mcp47feb14_chip_features },
+	{ .compatible = "microchip,mcp47feb18", .data = &mcp47feb18_chip_features },
+	{ .compatible = "microchip,mcp47feb21", .data = &mcp47feb21_chip_features },
+	{ .compatible = "microchip,mcp47feb22", .data = &mcp47feb22_chip_features },
+	{ .compatible = "microchip,mcp47feb24", .data = &mcp47feb24_chip_features },
+	{ .compatible = "microchip,mcp47feb28", .data = &mcp47feb28_chip_features },
+	{ .compatible = "microchip,mcp47fvb01", .data = &mcp47fvb01_chip_features },
+	{ .compatible = "microchip,mcp47fvb02", .data = &mcp47fvb02_chip_features },
+	{ .compatible = "microchip,mcp47fvb04", .data = &mcp47fvb04_chip_features },
+	{ .compatible = "microchip,mcp47fvb08", .data = &mcp47fvb08_chip_features },
+	{ .compatible = "microchip,mcp47fvb11", .data = &mcp47fvb11_chip_features },
+	{ .compatible = "microchip,mcp47fvb12", .data = &mcp47fvb12_chip_features },
+	{ .compatible = "microchip,mcp47fvb14",	.data = &mcp47fvb14_chip_features },
+	{ .compatible = "microchip,mcp47fvb18", .data = &mcp47fvb18_chip_features },
+	{ .compatible = "microchip,mcp47fvb21", .data = &mcp47fvb21_chip_features },
+	{ .compatible = "microchip,mcp47fvb22", .data = &mcp47fvb22_chip_features },
+	{ .compatible = "microchip,mcp47fvb24", .data = &mcp47fvb24_chip_features },
+	{ .compatible = "microchip,mcp47fvb28", .data = &mcp47fvb28_chip_features },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mcp47feb02_of_i2c_match);
+
+static struct i2c_driver mcp47feb02_i2c_driver = {
+	.driver = {
+		.name = "mcp47feb02",
+		.of_match_table = mcp47feb02_of_i2c_match,
+		.pm = pm_sleep_ptr(&mcp47feb02_pm_ops),
+	},
+	.probe = mcp47feb02_i2c_probe,
+	.id_table = mcp47feb02_i2c_id,
+};
+module_i2c_driver(mcp47feb02_i2c_driver);
+
+MODULE_AUTHOR("Ariana Lazar <ariana.lazar@microchip.com>");
+MODULE_DESCRIPTION("IIO driver for MCP47FEB02 Multi-Channel DAC with I2C interface");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_MCP47FEB02");
diff --git a/drivers/iio/dac/mcp47feb02-spi.c b/drivers/iio/dac/mcp47feb02-spi.c
new file mode 100644
index 00000000000000..974f8fe1b86e93
--- /dev/null
+++ b/drivers/iio/dac/mcp47feb02-spi.c
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * IIO driver for MCP48FEB02 Multi-Channel DAC with SPI interface
+ *
+ * Copyright (C) 2025-2026 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Ariana Lazar <ariana.lazar@microchip.com>
+ *
+ * Datasheet links for devices with SPI interface:
+ * [MCP48FEBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005429B.pdf
+ * [MCP48FVBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005466A.pdf
+ * [MCP48FxBx4/8] https://ww1.microchip.com/downloads/aemDocuments/documents/MSLD/ProductDocuments/DataSheets/MCP48FXBX4-8-Family-Data-Sheet-DS20006362A.pdf
+ */
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/pm.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+#include <linux/types.h>
+
+#include "mcp47feb02.h"
+
+/* Parts with EEPROM memory */
+MCP47FEB02_CHIP_INFO(mcp48feb01, 1, 8,  false, true);
+MCP47FEB02_CHIP_INFO(mcp48feb02, 2, 8,  false, true);
+MCP47FEB02_CHIP_INFO(mcp48feb04, 4, 8,  true,  true);
+MCP47FEB02_CHIP_INFO(mcp48feb08, 8, 8,  true,  true);
+MCP47FEB02_CHIP_INFO(mcp48feb11, 1, 10, false, true);
+MCP47FEB02_CHIP_INFO(mcp48feb12, 2, 10, false, true);
+MCP47FEB02_CHIP_INFO(mcp48feb14, 4, 10, true,  true);
+MCP47FEB02_CHIP_INFO(mcp48feb18, 8, 10, true,  true);
+MCP47FEB02_CHIP_INFO(mcp48feb21, 1, 12, false, true);
+MCP47FEB02_CHIP_INFO(mcp48feb22, 2, 12, false, true);
+MCP47FEB02_CHIP_INFO(mcp48feb24, 4, 12, true,  true);
+MCP47FEB02_CHIP_INFO(mcp48feb28, 8, 12, true,  true);
+
+/* Parts without EEPROM memory */
+MCP47FEB02_CHIP_INFO(mcp48fvb01, 1, 8,  false, false);
+MCP47FEB02_CHIP_INFO(mcp48fvb02, 2, 8,  false, false);
+MCP47FEB02_CHIP_INFO(mcp48fvb04, 4, 8,  true,  false);
+MCP47FEB02_CHIP_INFO(mcp48fvb08, 8, 8,  true,  false);
+MCP47FEB02_CHIP_INFO(mcp48fvb11, 1, 10, false, false);
+MCP47FEB02_CHIP_INFO(mcp48fvb12, 2, 10, false, false);
+MCP47FEB02_CHIP_INFO(mcp48fvb14, 4, 10, true,  false);
+MCP47FEB02_CHIP_INFO(mcp48fvb18, 8, 10, true,  false);
+MCP47FEB02_CHIP_INFO(mcp48fvb21, 1, 12, false, false);
+MCP47FEB02_CHIP_INFO(mcp48fvb22, 2, 12, false, false);
+MCP47FEB02_CHIP_INFO(mcp48fvb24, 4, 12, true,  false);
+MCP47FEB02_CHIP_INFO(mcp48fvb28, 8, 12, true,  false);
+
+static int mcp47feb02_spi_probe(struct spi_device *spi)
+{
+	const struct mcp47feb02_features *chip_features;
+	struct device *dev = &spi->dev;
+	struct regmap *regmap;
+
+	chip_features = spi_get_device_match_data(spi);
+	if (!chip_features)
+		return dev_err_probe(dev, -ENODATA, "No match data found\n");
+
+	if (chip_features->have_eeprom)
+		regmap = devm_regmap_init_spi(spi, &mcp47feb02_regmap_config);
+	else
+		regmap = devm_regmap_init_spi(spi, &mcp47fvb02_regmap_config);
+
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Error initializing SPI regmap\n");
+
+	return mcp47feb02_common_probe(chip_features, regmap);
+}
+
+static const struct spi_device_id mcp47feb02_spi_id[] = {
+	{ .name = "mcp48feb01", .driver_data = (kernel_ulong_t)&mcp48feb01_chip_features },
+	{ .name = "mcp48feb02", .driver_data = (kernel_ulong_t)&mcp48feb02_chip_features },
+	{ .name = "mcp48feb04", .driver_data = (kernel_ulong_t)&mcp48feb04_chip_features },
+	{ .name = "mcp48feb08", .driver_data = (kernel_ulong_t)&mcp48feb08_chip_features },
+	{ .name = "mcp48feb11", .driver_data = (kernel_ulong_t)&mcp48feb11_chip_features },
+	{ .name = "mcp48feb12", .driver_data = (kernel_ulong_t)&mcp48feb12_chip_features },
+	{ .name = "mcp48feb14", .driver_data = (kernel_ulong_t)&mcp48feb14_chip_features },
+	{ .name = "mcp48feb18", .driver_data =  (kernel_ulong_t)&mcp48feb18_chip_features },
+	{ .name = "mcp48feb21", .driver_data = (kernel_ulong_t)&mcp48feb21_chip_features },
+	{ .name = "mcp48feb22", .driver_data = (kernel_ulong_t)&mcp48feb22_chip_features },
+	{ .name = "mcp48feb24", .driver_data = (kernel_ulong_t)&mcp48feb24_chip_features },
+	{ .name = "mcp48feb28", .driver_data = (kernel_ulong_t)&mcp48feb28_chip_features },
+	{ .name = "mcp48fvb01", .driver_data = (kernel_ulong_t)&mcp48fvb01_chip_features },
+	{ .name = "mcp48fvb02", .driver_data = (kernel_ulong_t)&mcp48fvb02_chip_features },
+	{ .name = "mcp48fvb04", .driver_data = (kernel_ulong_t)&mcp48fvb04_chip_features },
+	{ .name = "mcp48fvb08", .driver_data = (kernel_ulong_t)&mcp48fvb08_chip_features },
+	{ .name = "mcp48fvb11", .driver_data = (kernel_ulong_t)&mcp48fvb11_chip_features },
+	{ .name = "mcp48fvb12", .driver_data = (kernel_ulong_t)&mcp48fvb12_chip_features },
+	{ .name = "mcp48fvb14", .driver_data = (kernel_ulong_t)&mcp48fvb14_chip_features },
+	{ .name = "mcp48fvb18", .driver_data = (kernel_ulong_t)&mcp48fvb18_chip_features },
+	{ .name = "mcp48fvb21", .driver_data = (kernel_ulong_t)&mcp48fvb21_chip_features },
+	{ .name = "mcp48fvb22", .driver_data = (kernel_ulong_t)&mcp48fvb22_chip_features },
+	{ .name = "mcp48fvb24", .driver_data = (kernel_ulong_t)&mcp48fvb24_chip_features },
+	{ .name = "mcp48fvb28", .driver_data = (kernel_ulong_t)&mcp48fvb28_chip_features },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, mcp47feb02_spi_id);
+
+static const struct of_device_id mcp47feb02_of_spi_match[] = {
+	{ .compatible = "microchip,mcp48feb01", .data = &mcp48feb01_chip_features },
+	{ .compatible = "microchip,mcp48feb02", .data = &mcp48feb02_chip_features },
+	{ .compatible = "microchip,mcp48feb04", .data = &mcp48feb04_chip_features },
+	{ .compatible = "microchip,mcp48feb08", .data = &mcp48feb08_chip_features },
+	{ .compatible = "microchip,mcp48feb11", .data = &mcp48feb11_chip_features },
+	{ .compatible = "microchip,mcp48feb12", .data = &mcp48feb12_chip_features },
+	{ .compatible = "microchip,mcp48feb14", .data = &mcp48feb14_chip_features },
+	{ .compatible = "microchip,mcp48feb18", .data = &mcp48feb18_chip_features },
+	{ .compatible = "microchip,mcp48feb21", .data = &mcp48feb21_chip_features },
+	{ .compatible = "microchip,mcp48feb22", .data = &mcp48feb22_chip_features },
+	{ .compatible = "microchip,mcp48feb24", .data = &mcp48feb24_chip_features },
+	{ .compatible = "microchip,mcp48feb28", .data = &mcp48feb28_chip_features },
+	{ .compatible = "microchip,mcp48fvb01", .data = &mcp48fvb01_chip_features },
+	{ .compatible = "microchip,mcp48fvb02", .data = &mcp48fvb02_chip_features },
+	{ .compatible = "microchip,mcp48fvb04", .data = &mcp48fvb04_chip_features },
+	{ .compatible = "microchip,mcp48fvb08", .data = &mcp48fvb08_chip_features },
+	{ .compatible = "microchip,mcp48fvb11", .data = &mcp48fvb11_chip_features },
+	{ .compatible = "microchip,mcp48fvb12", .data = &mcp48fvb12_chip_features },
+	{ .compatible = "microchip,mcp48fvb14",	.data = &mcp48fvb14_chip_features },
+	{ .compatible = "microchip,mcp48fvb18", .data = &mcp48fvb18_chip_features },
+	{ .compatible = "microchip,mcp48fvb21", .data = &mcp48fvb21_chip_features },
+	{ .compatible = "microchip,mcp48fvb22", .data = &mcp48fvb22_chip_features },
+	{ .compatible = "microchip,mcp48fvb24", .data = &mcp48fvb24_chip_features },
+	{ .compatible = "microchip,mcp48fvb28", .data = &mcp48fvb28_chip_features },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mcp47feb02_of_spi_match);
+
+static struct spi_driver mcp47feb02_spi_driver = {
+	.driver = {
+		.name = "mcp47feb02",
+		.of_match_table = mcp47feb02_of_spi_match,
+		.pm = pm_sleep_ptr(&mcp47feb02_pm_ops),
+	},
+	.probe = mcp47feb02_spi_probe,
+	.id_table = mcp47feb02_spi_id,
+};
+module_spi_driver(mcp47feb02_spi_driver);
+
+MODULE_AUTHOR("Ariana Lazar <ariana.lazar@microchip.com>");
+MODULE_DESCRIPTION("IIO driver for MCP48FEB02 Multi-Channel DAC with SPI interface");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_MCP47FEB02");
diff --git a/drivers/iio/dac/mcp47feb02.c b/drivers/iio/dac/mcp47feb02.c
deleted file mode 100644
index 8640b0ef4433da..00000000000000
--- a/drivers/iio/dac/mcp47feb02.c
+++ /dev/null
@@ -1,1249 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * IIO driver for MCP47FEB02 Multi-Channel DAC with I2C interface
- *
- * Copyright (C) 2025 Microchip Technology Inc. and its subsidiaries
- *
- * Author: Ariana Lazar <ariana.lazar@microchip.com>
- *
- * Datasheet links:
- * [MCP47FEBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005375A.pdf
- * [MCP47FVBxx] https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/DataSheets/20005405A.pdf
- * [MCP47FxBx4/8] https://ww1.microchip.com/downloads/aemDocuments/documents/MSLD/ProductDocuments/DataSheets/MCP47FXBX48-Data-Sheet-DS200006368A.pdf
- */
-#include <linux/array_size.h>
-#include <linux/bits.h>
-#include <linux/bitfield.h>
-#include <linux/delay.h>
-#include <linux/err.h>
-#include <linux/i2c.h>
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
-#include <linux/kstrtox.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/property.h>
-#include <linux/regmap.h>
-#include <linux/regulator/consumer.h>
-#include <linux/time64.h>
-#include <linux/types.h>
-#include <linux/units.h>
-
-/* Register addresses must be left shifted with 3 positions in order to append command mask */
-#define MCP47FEB02_DAC0_REG_ADDR			0x00
-#define MCP47FEB02_VREF_REG_ADDR			0x40
-#define MCP47FEB02_POWER_DOWN_REG_ADDR			0x48
-#define MCP47FEB02_DAC_CTRL_MASK			GENMASK(1, 0)
-
-#define MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR		0x50
-#define MCP47FEB02_GAIN_BIT_MASK			BIT(0)
-#define MCP47FEB02_GAIN_BIT_STATUS_EEWA_MASK		BIT(6)
-#define MCP47FEB02_GAIN_BITS_MASK			GENMASK(15, 8)
-
-#define MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR		0x58
-
-#define MCP47FEB02_NV_DAC0_REG_ADDR			0x80
-#define MCP47FEB02_NV_VREF_REG_ADDR			0xC0
-#define MCP47FEB02_NV_POWER_DOWN_REG_ADDR		0xC8
-#define MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR	0xD0
-#define MCP47FEB02_NV_I2C_SLAVE_ADDR_MASK		GENMASK(7, 0)
-
-/* Voltage reference, Power-Down control register and DAC Wiperlock status register fields */
-#define DAC_CTRL_MASK(ch)				(GENMASK(1, 0) << (2 * (ch)))
-#define DAC_CTRL_VAL(ch, val)				((val) << (2 * (ch)))
-
-/* Gain Control and I2C Slave Address Reguster fields */
-#define DAC_GAIN_MASK(ch)				(BIT(0) << (8 + (ch)))
-#define DAC_GAIN_VAL(ch, val)				((val) << (8 + (ch)))
-
-#define REG_ADDR(reg)					((reg) << 3)
-#define NV_REG_ADDR(reg)				((NV_DAC_ADDR_OFFSET + (reg)) << 3)
-#define READFLAG_MASK					GENMASK(2, 1)
-
-#define MCP47FEB02_MAX_CH				8
-#define MCP47FEB02_MAX_SCALES_CH			3
-#define MCP47FEB02_DAC_WIPER_UNLOCKED			0
-#define MCP47FEB02_NORMAL_OPERATION			0
-#define MCP47FEB02_INTERNAL_BAND_GAP_uV			2440000
-#define NV_DAC_ADDR_OFFSET				0x10
-
-enum mcp47feb02_vref_mode {
-	MCP47FEB02_VREF_VDD = 0,
-	MCP47FEB02_INTERNAL_BAND_GAP = 1,
-	MCP47FEB02_EXTERNAL_VREF_UNBUFFERED = 2,
-	MCP47FEB02_EXTERNAL_VREF_BUFFERED = 3,
-};
-
-enum mcp47feb02_scale {
-	MCP47FEB02_SCALE_VDD = 0,
-	MCP47FEB02_SCALE_GAIN_X1 = 1,
-	MCP47FEB02_SCALE_GAIN_X2 = 2,
-};
-
-enum mcp47feb02_gain_bit_mode {
-	MCP47FEB02_GAIN_BIT_X1 = 0,
-	MCP47FEB02_GAIN_BIT_X2 = 1,
-};
-
-static const char * const mcp47feb02_powerdown_modes[] = {
-	"1kohm_to_gnd",
-	"100kohm_to_gnd",
-	"open_circuit",
-};
-
-/**
- * struct mcp47feb02_features - chip specific data
- * @name: device name
- * @phys_channels: number of hardware channels
- * @resolution: DAC resolution
- * @have_ext_vref1: does the hardware have an the second external voltage reference?
- * @have_eeprom: does the hardware have an internal eeprom?
- */
-struct mcp47feb02_features {
-	const char *name;
-	unsigned int phys_channels;
-	unsigned int resolution;
-	bool have_ext_vref1;
-	bool have_eeprom;
-};
-
-static const struct mcp47feb02_features mcp47feb01_chip_features = {
-	.name = "mcp47feb01",
-	.phys_channels = 1,
-	.resolution = 8,
-	.have_ext_vref1 = false,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb02_chip_features = {
-	.name = "mcp47feb02",
-	.phys_channels = 2,
-	.resolution = 8,
-	.have_ext_vref1 = false,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb04_chip_features = {
-	.name = "mcp47feb04",
-	.phys_channels = 4,
-	.resolution = 8,
-	.have_ext_vref1 = true,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb08_chip_features = {
-	.name = "mcp47feb08",
-	.phys_channels = 8,
-	.resolution = 8,
-	.have_ext_vref1 = true,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb11_chip_features = {
-	.name = "mcp47feb11",
-	.phys_channels = 1,
-	.resolution = 10,
-	.have_ext_vref1 = false,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb12_chip_features = {
-	.name = "mcp47feb12",
-	.phys_channels = 2,
-	.resolution = 10,
-	.have_ext_vref1 = false,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb14_chip_features = {
-	.name = "mcp47feb14",
-	.phys_channels = 4,
-	.resolution = 10,
-	.have_ext_vref1 = true,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb18_chip_features = {
-	.name = "mcp47feb18",
-	.phys_channels = 8,
-	.resolution = 10,
-	.have_ext_vref1 = true,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb21_chip_features = {
-	.name = "mcp47feb21",
-	.phys_channels = 1,
-	.resolution = 12,
-	.have_ext_vref1 = false,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb22_chip_features = {
-	.name = "mcp47feb22",
-	.phys_channels = 2,
-	.resolution = 12,
-	.have_ext_vref1 = false,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb24_chip_features = {
-	.name = "mcp47feb24",
-	.phys_channels = 4,
-	.resolution = 12,
-	.have_ext_vref1 = true,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47feb28_chip_features = {
-	.name = "mcp47feb28",
-	.phys_channels = 8,
-	.resolution = 12,
-	.have_ext_vref1 = true,
-	.have_eeprom = true,
-};
-
-static const struct mcp47feb02_features mcp47fvb01_chip_features = {
-	.name = "mcp47fvb01",
-	.phys_channels = 1,
-	.resolution = 8,
-	.have_ext_vref1 = false,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb02_chip_features = {
-	.name = "mcp47fvb02",
-	.phys_channels = 2,
-	.resolution = 8,
-	.have_ext_vref1 = false,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb04_chip_features = {
-	.name = "mcp47fvb04",
-	.phys_channels = 4,
-	.resolution = 8,
-	.have_ext_vref1 = true,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb08_chip_features = {
-	.name = "mcp47fvb08",
-	.phys_channels = 8,
-	.resolution = 8,
-	.have_ext_vref1 = true,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb11_chip_features = {
-	.name = "mcp47fvb11",
-	.phys_channels = 1,
-	.resolution = 10,
-	.have_ext_vref1 = false,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb12_chip_features = {
-	.name = "mcp47fvb12",
-	.phys_channels = 2,
-	.resolution = 10,
-	.have_ext_vref1 = false,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb14_chip_features = {
-	.name = "mcp47fvb14",
-	.phys_channels = 4,
-	.resolution = 10,
-	.have_ext_vref1 = true,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb18_chip_features = {
-	.name = "mcp47fvb18",
-	.phys_channels = 8,
-	.resolution = 10,
-	.have_ext_vref1 = true,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb21_chip_features = {
-	.name = "mcp47fvb21",
-	.phys_channels = 1,
-	.resolution = 12,
-	.have_ext_vref1 = false,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb22_chip_features = {
-	.name = "mcp47fvb22",
-	.phys_channels = 2,
-	.resolution = 12,
-	.have_ext_vref1 = false,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb24_chip_features = {
-	.name = "mcp47fvb24",
-	.phys_channels = 4,
-	.resolution = 12,
-	.have_ext_vref1 = true,
-	.have_eeprom = false,
-};
-
-static const struct mcp47feb02_features mcp47fvb28_chip_features = {
-	.name = "mcp47fvb28",
-	.phys_channels = 8,
-	.resolution = 12,
-	.have_ext_vref1 = true,
-	.have_eeprom = false,
-};
-
-/**
- * struct mcp47feb02_channel_data - channel configuration
- * @ref_mode: chosen voltage for reference
- * @use_2x_gain: output driver gain control
- * @powerdown: is false if the channel is in normal operation mode
- * @powerdown_mode: selected power-down mode
- * @dac_data: dac value
- */
-struct mcp47feb02_channel_data {
-	u8 ref_mode;
-	bool use_2x_gain;
-	bool powerdown;
-	u8 powerdown_mode;
-	u16 dac_data;
-};
-
-/**
- * struct mcp47feb02_data - chip configuration
- * @chdata: options configured for each channel on the device
- * @lock: prevents concurrent reads/writes to driver's state members
- * @chip_features: pointer to features struct
- * @scale_1: scales set on channels that are based on Vref1
- * @scale: scales set on channels that are based on Vref/Vref0
- * @active_channels_mask: enabled channels
- * @regmap: regmap for directly accessing device register
- * @labels: table with channels labels
- * @phys_channels: physical channels on the device
- * @vref1_buffered: Vref1 buffer is enabled
- * @vref_buffered: Vref/Vref0 buffer is enabled
- * @use_vref1: vref1-supply is defined
- * @use_vref: vref-supply is defined
- */
-struct mcp47feb02_data {
-	struct mcp47feb02_channel_data chdata[MCP47FEB02_MAX_CH];
-	struct mutex lock; /* prevents concurrent reads/writes to driver's state members */
-	const struct mcp47feb02_features *chip_features;
-	int scale_1[2 * MCP47FEB02_MAX_SCALES_CH];
-	int scale[2 * MCP47FEB02_MAX_SCALES_CH];
-	unsigned long active_channels_mask;
-	struct regmap *regmap;
-	const char *labels[MCP47FEB02_MAX_CH];
-	u16 phys_channels;
-	bool vref1_buffered;
-	bool vref_buffered;
-	bool use_vref1;
-	bool use_vref;
-};
-
-static const struct regmap_range mcp47feb02_readable_ranges[] = {
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
-};
-
-static const struct regmap_range mcp47feb02_writable_ranges[] = {
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
-};
-
-static const struct regmap_range mcp47feb02_volatile_ranges[] = {
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-	regmap_reg_range(MCP47FEB02_NV_DAC0_REG_ADDR, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR),
-};
-
-static const struct regmap_access_table mcp47feb02_readable_table = {
-	.yes_ranges = mcp47feb02_readable_ranges,
-	.n_yes_ranges = ARRAY_SIZE(mcp47feb02_readable_ranges),
-};
-
-static const struct regmap_access_table mcp47feb02_writable_table = {
-	.yes_ranges = mcp47feb02_writable_ranges,
-	.n_yes_ranges = ARRAY_SIZE(mcp47feb02_writable_ranges),
-};
-
-static const struct regmap_access_table mcp47feb02_volatile_table = {
-	.yes_ranges = mcp47feb02_volatile_ranges,
-	.n_yes_ranges = ARRAY_SIZE(mcp47feb02_volatile_ranges),
-};
-
-static const struct regmap_config mcp47feb02_regmap_config = {
-	.name = "mcp47feb02_regmap",
-	.reg_bits = 8,
-	.val_bits = 16,
-	.rd_table = &mcp47feb02_readable_table,
-	.wr_table = &mcp47feb02_writable_table,
-	.volatile_table = &mcp47feb02_volatile_table,
-	.max_register = MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR,
-	.read_flag_mask = READFLAG_MASK,
-	.cache_type = REGCACHE_MAPLE,
-	.val_format_endian = REGMAP_ENDIAN_BIG,
-};
-
-/* For devices that doesn't have nonvolatile memory */
-static const struct regmap_range mcp47fvb02_readable_ranges[] = {
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-};
-
-static const struct regmap_range mcp47fvb02_writable_ranges[] = {
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-};
-
-static const struct regmap_range mcp47fvb02_volatile_ranges[] = {
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-	regmap_reg_range(MCP47FEB02_DAC0_REG_ADDR, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR),
-};
-
-static const struct regmap_access_table mcp47fvb02_readable_table = {
-	.yes_ranges = mcp47fvb02_readable_ranges,
-	.n_yes_ranges = ARRAY_SIZE(mcp47fvb02_readable_ranges),
-};
-
-static const struct regmap_access_table mcp47fvb02_writable_table = {
-	.yes_ranges = mcp47fvb02_writable_ranges,
-	.n_yes_ranges = ARRAY_SIZE(mcp47fvb02_writable_ranges),
-};
-
-static const struct regmap_access_table mcp47fvb02_volatile_table = {
-	.yes_ranges = mcp47fvb02_volatile_ranges,
-	.n_yes_ranges = ARRAY_SIZE(mcp47fvb02_volatile_ranges),
-};
-
-static const struct regmap_config mcp47fvb02_regmap_config = {
-	.name = "mcp47fvb02_regmap",
-	.reg_bits = 8,
-	.val_bits = 16,
-	.rd_table = &mcp47fvb02_readable_table,
-	.wr_table = &mcp47fvb02_writable_table,
-	.volatile_table = &mcp47fvb02_volatile_table,
-	.max_register = MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR,
-	.read_flag_mask = READFLAG_MASK,
-	.cache_type = REGCACHE_MAPLE,
-	.val_format_endian = REGMAP_ENDIAN_BIG,
-};
-
-static int mcp47feb02_write_to_eeprom(struct mcp47feb02_data *data, unsigned int reg,
-				      unsigned int val)
-{
-	int eewa_val, ret;
-
-	/*
-	 * Wait until the currently occurring EEPROM Write Cycle is completed.
-	 * Only serial commands to the volatile memory are allowed.
-	 */
-	guard(mutex)(&data->lock);
-
-	ret = regmap_read_poll_timeout(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR,
-				       eewa_val,
-				       !(eewa_val & MCP47FEB02_GAIN_BIT_STATUS_EEWA_MASK),
-				       USEC_PER_MSEC, USEC_PER_MSEC * 5);
-	if (ret)
-		return ret;
-
-	return regmap_write(data->regmap, reg, val);
-}
-
-static ssize_t store_eeprom_store(struct device *dev, struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct mcp47feb02_data *data = iio_priv(dev_to_iio_dev(dev));
-	unsigned int i, val, val1, eewa_val;
-	bool state;
-	int ret;
-
-	ret = kstrtobool(buf, &state);
-	if (ret)
-		return ret;
-
-	if (!state)
-		return 0;
-
-	/*
-	 * Verify DAC Wiper and DAC Configuration are unlocked. If both are disabled,
-	 * writing to EEPROM is available.
-	 */
-	ret = regmap_read(data->regmap, MCP47FEB02_WIPERLOCK_STATUS_REG_ADDR, &val);
-	if (ret)
-		return ret;
-
-	if (val) {
-		dev_err(dev, "DAC Wiper and DAC Configuration not are unlocked.\n");
-		return -EINVAL;
-	}
-
-	for_each_set_bit(i, &data->active_channels_mask, data->phys_channels) {
-		ret = mcp47feb02_write_to_eeprom(data, NV_REG_ADDR(i),
-						 data->chdata[i].dac_data);
-		if (ret)
-			return ret;
-	}
-
-	ret = regmap_read(data->regmap, MCP47FEB02_VREF_REG_ADDR, &val);
-	if (ret)
-		return ret;
-
-	ret = mcp47feb02_write_to_eeprom(data, MCP47FEB02_NV_VREF_REG_ADDR, val);
-	if (ret)
-		return ret;
-
-	ret = regmap_read(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR, &val);
-	if (ret)
-		return ret;
-
-	ret = mcp47feb02_write_to_eeprom(data, MCP47FEB02_NV_POWER_DOWN_REG_ADDR, val);
-	if (ret)
-		return ret;
-
-	ret = regmap_read_poll_timeout(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR, eewa_val,
-				       !(eewa_val & MCP47FEB02_GAIN_BIT_STATUS_EEWA_MASK),
-				       USEC_PER_MSEC, USEC_PER_MSEC * 5);
-	if (ret)
-		return ret;
-
-	ret = regmap_read(data->regmap, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR, &val);
-	if (ret)
-		return ret;
-
-	ret = regmap_read(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR, &val1);
-	if (ret)
-		return ret;
-
-	ret = mcp47feb02_write_to_eeprom(data, MCP47FEB02_NV_GAIN_CTRL_I2C_SLAVE_REG_ADDR,
-					 (val1 & MCP47FEB02_GAIN_BITS_MASK) |
-					 (val & MCP47FEB02_NV_I2C_SLAVE_ADDR_MASK));
-	if (ret)
-		return ret;
-
-	return len;
-}
-
-static IIO_DEVICE_ATTR_WO(store_eeprom, 0);
-
-static struct attribute *mcp47feb02_attributes[] = {
-	&iio_dev_attr_store_eeprom.dev_attr.attr,
-	NULL
-};
-
-static const struct attribute_group mcp47feb02_attribute_group = {
-	.attrs = mcp47feb02_attributes,
-};
-
-static int mcp47feb02_suspend(struct device *dev)
-{
-	struct iio_dev *indio_dev = dev_get_drvdata(dev);
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-	int ret;
-	u8 ch;
-
-	guard(mutex)(&data->lock);
-
-	for_each_set_bit(ch, &data->active_channels_mask, data->phys_channels) {
-		u8 pd_mode;
-
-		data->chdata[ch].powerdown = true;
-		pd_mode = data->chdata[ch].powerdown_mode + 1;
-		ret = regmap_update_bits(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR,
-					 DAC_CTRL_MASK(ch), DAC_CTRL_VAL(ch, pd_mode));
-		if (ret)
-			return ret;
-
-		ret = regmap_write(data->regmap, REG_ADDR(ch), data->chdata[ch].dac_data);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int mcp47feb02_resume(struct device *dev)
-{
-	struct iio_dev *indio_dev = dev_get_drvdata(dev);
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-	u8 ch;
-
-	guard(mutex)(&data->lock);
-
-	for_each_set_bit(ch, &data->active_channels_mask, data->phys_channels) {
-		u8 pd_mode;
-		int ret;
-
-		data->chdata[ch].powerdown = false;
-		pd_mode = data->chdata[ch].powerdown_mode + 1;
-
-		ret = regmap_write(data->regmap, REG_ADDR(ch), data->chdata[ch].dac_data);
-		if (ret)
-			return ret;
-
-		ret = regmap_update_bits(data->regmap, MCP47FEB02_VREF_REG_ADDR,
-					 DAC_CTRL_MASK(ch), DAC_CTRL_VAL(ch, pd_mode));
-		if (ret)
-			return ret;
-
-		ret = regmap_update_bits(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR,
-					 DAC_GAIN_MASK(ch),
-					 DAC_GAIN_VAL(ch, data->chdata[ch].use_2x_gain));
-		if (ret)
-			return ret;
-
-		ret = regmap_update_bits(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR,
-					 DAC_CTRL_MASK(ch),
-					 DAC_CTRL_VAL(ch, MCP47FEB02_NORMAL_OPERATION));
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int mcp47feb02_get_powerdown_mode(struct iio_dev *indio_dev,
-					 const struct iio_chan_spec *chan)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-
-	return data->chdata[chan->address].powerdown_mode;
-}
-
-static int mcp47feb02_set_powerdown_mode(struct iio_dev *indio_dev, const struct iio_chan_spec *ch,
-					 unsigned int mode)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-
-	data->chdata[ch->address].powerdown_mode = mode;
-
-	return 0;
-}
-
-static ssize_t mcp47feb02_read_powerdown(struct iio_dev *indio_dev, uintptr_t private,
-					 const struct iio_chan_spec *ch, char *buf)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-
-	/* Print if channel is in a power-down mode or not */
-	return sysfs_emit(buf, "%d\n", data->chdata[ch->address].powerdown);
-}
-
-static ssize_t mcp47feb02_write_powerdown(struct iio_dev *indio_dev, uintptr_t private,
-					  const struct iio_chan_spec *ch, const char *buf,
-					  size_t len)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-	u32 reg = ch->address;
-	u8 tmp_pd_mode;
-	bool state;
-	int ret;
-
-	guard(mutex)(&data->lock);
-
-	ret = kstrtobool(buf, &state);
-	if (ret)
-		return ret;
-
-	/*
-	 * Set the channel to the specified power-down mode. Exiting power-down mode
-	 * requires writing normal operation mode (0) to the channel-specific register bits.
-	 */
-	tmp_pd_mode = state ? (data->chdata[reg].powerdown_mode + 1) : MCP47FEB02_NORMAL_OPERATION;
-	ret = regmap_update_bits(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR,
-				 DAC_CTRL_MASK(reg), DAC_CTRL_VAL(reg, tmp_pd_mode));
-	if (ret)
-		return ret;
-
-	data->chdata[reg].powerdown = state;
-
-	return len;
-}
-
-static DEFINE_SIMPLE_DEV_PM_OPS(mcp47feb02_pm_ops, mcp47feb02_suspend, mcp47feb02_resume);
-
-static const struct iio_enum mcp47febxx_powerdown_mode_enum = {
-	.items = mcp47feb02_powerdown_modes,
-	.num_items = ARRAY_SIZE(mcp47feb02_powerdown_modes),
-	.get = mcp47feb02_get_powerdown_mode,
-	.set = mcp47feb02_set_powerdown_mode,
-};
-
-static const struct iio_chan_spec_ext_info mcp47feb02_ext_info[] = {
-	{
-		.name = "powerdown",
-		.read = mcp47feb02_read_powerdown,
-		.write = mcp47feb02_write_powerdown,
-		.shared = IIO_SEPARATE,
-	},
-	IIO_ENUM("powerdown_mode", IIO_SEPARATE, &mcp47febxx_powerdown_mode_enum),
-	IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE, &mcp47febxx_powerdown_mode_enum),
-	{ }
-};
-
-static const struct iio_chan_spec mcp47febxx_ch_template = {
-	.type = IIO_VOLTAGE,
-	.output = 1,
-	.indexed = 1,
-	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
-	.info_mask_separate_available = BIT(IIO_CHAN_INFO_SCALE),
-	.ext_info = mcp47feb02_ext_info,
-};
-
-static void mcp47feb02_init_scale(struct mcp47feb02_data *data, enum mcp47feb02_scale scale,
-				  int vref_uV, int scale_avail[])
-{
-	u32 value_micro, value_int;
-	u64 tmp;
-
-	/* vref_uV should not be negative */
-	tmp = (u64)vref_uV * MILLI >> data->chip_features->resolution;
-	value_int = div_u64_rem(tmp, MICRO, &value_micro);
-	scale_avail[scale * 2] = value_int;
-	scale_avail[scale * 2 + 1] = value_micro;
-}
-
-static int mcp47feb02_init_scales_avail(struct mcp47feb02_data *data, int vdd_uV,
-					int vref_uV, int vref1_uV)
-{
-	int tmp_vref;
-
-	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_uV, data->scale);
-
-	if (data->use_vref)
-		tmp_vref = vref_uV;
-	else
-		tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_uV;
-
-	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X1, tmp_vref, data->scale);
-	mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X2, tmp_vref * 2, data->scale);
-
-	if (data->phys_channels >= 4) {
-		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_VDD, vdd_uV, data->scale_1);
-
-		if (data->use_vref1)
-			tmp_vref = vref1_uV;
-		else
-			tmp_vref = MCP47FEB02_INTERNAL_BAND_GAP_uV;
-
-		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X1,
-				      tmp_vref, data->scale_1);
-		mcp47feb02_init_scale(data, MCP47FEB02_SCALE_GAIN_X2,
-				      tmp_vref * 2, data->scale_1);
-	}
-
-	return 0;
-}
-
-static int mcp47feb02_read_avail(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
-				 const int **vals, int *type, int *length, long info)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-
-	switch (info) {
-	case IIO_CHAN_INFO_SCALE:
-		switch (ch->type) {
-		case IIO_VOLTAGE:
-			if (data->phys_channels >= 4 && (ch->address % 2))
-				*vals = data->scale_1;
-			else
-				*vals = data->scale;
-
-			*length = 2 * MCP47FEB02_MAX_SCALES_CH;
-			*type = IIO_VAL_INT_PLUS_MICRO;
-			return IIO_AVAIL_LIST;
-		default:
-			return -EINVAL;
-		}
-	default:
-		return -EINVAL;
-	}
-}
-
-static void mcp47feb02_get_scale(int ch, struct mcp47feb02_data *data, int *val, int *val2)
-{
-	enum mcp47feb02_scale current_scale;
-
-	if (data->chdata[ch].ref_mode == MCP47FEB02_VREF_VDD)
-		current_scale = MCP47FEB02_SCALE_VDD;
-	else if (data->chdata[ch].use_2x_gain)
-		current_scale = MCP47FEB02_SCALE_GAIN_X2;
-	else
-		current_scale = MCP47FEB02_SCALE_GAIN_X1;
-
-	if (data->phys_channels >= 4 && (ch % 2)) {
-		*val = data->scale_1[current_scale * 2];
-		*val2 = data->scale_1[current_scale * 2 + 1];
-	} else {
-		*val = data->scale[current_scale * 2];
-		*val2 = data->scale[current_scale * 2 + 1];
-	}
-}
-
-static int mcp47feb02_check_scale(struct mcp47feb02_data *data, int val, int val2, int scale[])
-{
-	unsigned int i;
-
-	for (i = 0; i < MCP47FEB02_MAX_SCALES_CH; i++) {
-		if (scale[i * 2] == val && scale[i * 2 + 1] == val2)
-			return i;
-	}
-
-	return -EINVAL;
-}
-
-static int mcp47feb02_ch_scale(struct mcp47feb02_data *data, int ch, int scale)
-{
-	int tmp_val, ret;
-
-	if (scale == MCP47FEB02_SCALE_VDD) {
-		tmp_val = MCP47FEB02_VREF_VDD;
-	} else if (data->phys_channels >= 4 && (ch % 2)) {
-		if (data->use_vref1) {
-			if (data->vref1_buffered)
-				tmp_val = MCP47FEB02_EXTERNAL_VREF_BUFFERED;
-			else
-				tmp_val = MCP47FEB02_EXTERNAL_VREF_UNBUFFERED;
-		} else {
-			tmp_val = MCP47FEB02_INTERNAL_BAND_GAP;
-		}
-	} else if (data->use_vref) {
-		if (data->vref_buffered)
-			tmp_val = MCP47FEB02_EXTERNAL_VREF_BUFFERED;
-		else
-			tmp_val = MCP47FEB02_EXTERNAL_VREF_UNBUFFERED;
-	} else {
-		tmp_val = MCP47FEB02_INTERNAL_BAND_GAP;
-	}
-
-	ret = regmap_update_bits(data->regmap, MCP47FEB02_VREF_REG_ADDR,
-				 DAC_CTRL_MASK(ch), DAC_CTRL_VAL(ch, tmp_val));
-	if (ret)
-		return ret;
-
-	data->chdata[ch].ref_mode = tmp_val;
-
-	return 0;
-}
-
-/*
- * Setting the scale in order to choose between VDD and (Vref or Band Gap) from the user
- * space. The VREF pin is either an input or an output, therefore the user cannot
- * simultaneously connect an external voltage reference to the pin and select the
- * internal Band Gap.
- * When the DAC’s voltage reference is configured as the VREF pin, the pin is an input.
- * When the DAC’s voltage reference is configured as the internal Band Gap,
- * the VREF pin is an output.
- * If Vref/Vref1 voltage is not available, then the internal Band Gap will be used
- * to calculate the values for the scale.
- */
-static int mcp47feb02_set_scale(struct mcp47feb02_data *data, int ch, int scale)
-{
-	int tmp_val, ret;
-
-	ret = mcp47feb02_ch_scale(data, ch, scale);
-	if (ret)
-		return ret;
-
-	if (scale == MCP47FEB02_SCALE_GAIN_X2)
-		tmp_val = MCP47FEB02_GAIN_BIT_X2;
-	else
-		tmp_val = MCP47FEB02_GAIN_BIT_X1;
-
-	ret = regmap_update_bits(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR,
-				 DAC_GAIN_MASK(ch), DAC_GAIN_VAL(ch, tmp_val));
-	if (ret)
-		return ret;
-
-	data->chdata[ch].use_2x_gain = tmp_val;
-
-	return 0;
-}
-
-static int mcp47feb02_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
-			       int *val, int *val2, long mask)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-	int ret;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		ret = regmap_read(data->regmap, REG_ADDR(ch->address), val);
-		if (ret)
-			return ret;
-		return IIO_VAL_INT;
-	case IIO_CHAN_INFO_SCALE:
-		mcp47feb02_get_scale(ch->address, data, val, val2);
-		return IIO_VAL_INT_PLUS_MICRO;
-	default:
-		return -EINVAL;
-	}
-}
-
-static int mcp47feb02_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
-				int val, int val2, long mask)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-	int *tmp_scale, ret;
-
-	guard(mutex)(&data->lock);
-
-	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		ret = regmap_write(data->regmap, REG_ADDR(ch->address), val);
-		if (ret)
-			return ret;
-
-		data->chdata[ch->address].dac_data = val;
-		return 0;
-	case IIO_CHAN_INFO_SCALE:
-		if (data->phys_channels >= 4 && (ch->address % 2))
-			tmp_scale = data->scale_1;
-		else
-			tmp_scale = data->scale;
-
-		ret = mcp47feb02_check_scale(data, val, val2, tmp_scale);
-		if (ret < 0)
-			return ret;
-
-		return mcp47feb02_set_scale(data, ch->address, ret);
-	default:
-		return -EINVAL;
-	}
-}
-
-static int mcp47feb02_read_label(struct iio_dev *indio_dev, struct iio_chan_spec const *ch,
-				 char *label)
-{
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-
-	return sysfs_emit(label, "%s\n", data->labels[ch->address]);
-}
-
-static const struct iio_info mcp47feb02_info = {
-	.read_raw = mcp47feb02_read_raw,
-	.write_raw = mcp47feb02_write_raw,
-	.read_label = mcp47feb02_read_label,
-	.read_avail = &mcp47feb02_read_avail,
-	.attrs = &mcp47feb02_attribute_group,
-};
-
-static const struct iio_info mcp47fvb02_info = {
-	.read_raw = mcp47feb02_read_raw,
-	.write_raw = mcp47feb02_write_raw,
-	.read_label = mcp47feb02_read_label,
-	.read_avail = &mcp47feb02_read_avail,
-};
-
-static int mcp47feb02_parse_fw(struct iio_dev *indio_dev,
-			       const struct mcp47feb02_features *chip_features)
-{
-	struct iio_chan_spec chanspec = mcp47febxx_ch_template;
-	struct mcp47feb02_data *data = iio_priv(indio_dev);
-	struct device *dev = regmap_get_device(data->regmap);
-	struct iio_chan_spec *channels;
-	u32 num_channels;
-	u8 chan_idx = 0;
-
-	num_channels = device_get_child_node_count(dev);
-	if (num_channels > chip_features->phys_channels)
-		return dev_err_probe(dev, -EINVAL, "More channels than the chip supports\n");
-
-	if (!num_channels)
-		return dev_err_probe(dev, -EINVAL, "No channel specified in the devicetree.\n");
-
-	channels = devm_kcalloc(dev, num_channels, sizeof(*channels), GFP_KERNEL);
-	if (!channels)
-		return -ENOMEM;
-
-	device_for_each_child_node_scoped(dev, child) {
-		u32 reg = 0;
-		int ret;
-
-		ret = fwnode_property_read_u32(child, "reg", &reg);
-		if (ret)
-			return dev_err_probe(dev, ret, "Invalid channel number\n");
-
-		if (reg >= chip_features->phys_channels)
-			return dev_err_probe(dev, -EINVAL,
-					     "The index of the channels does not match the chip\n");
-
-		set_bit(reg, &data->active_channels_mask);
-
-		ret = fwnode_property_read_string(child, "label", &data->labels[reg]);
-		if (ret)
-			return dev_err_probe(dev, ret, "%pfw: invalid label\n",
-					     fwnode_get_name(child));
-
-		chanspec.address = reg;
-		chanspec.channel = reg;
-		channels[chan_idx] = chanspec;
-		chan_idx++;
-	}
-
-	indio_dev->num_channels = num_channels;
-	indio_dev->channels = channels;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-	data->phys_channels = chip_features->phys_channels;
-
-	data->vref_buffered = device_property_read_bool(dev, "microchip,vref-buffered");
-
-	if (chip_features->have_ext_vref1)
-		data->vref1_buffered = device_property_read_bool(dev, "microchip,vref1-buffered");
-
-	return 0;
-}
-
-static int mcp47feb02_init_ctrl_regs(struct mcp47feb02_data *data)
-{
-	unsigned int i, vref_ch, gain_ch, pd_ch;
-	int ret;
-
-	ret = regmap_read(data->regmap, MCP47FEB02_VREF_REG_ADDR, &vref_ch);
-	if (ret)
-		return ret;
-
-	ret = regmap_read(data->regmap, MCP47FEB02_GAIN_CTRL_STATUS_REG_ADDR, &gain_ch);
-	if (ret)
-		return ret;
-
-	ret = regmap_read(data->regmap, MCP47FEB02_POWER_DOWN_REG_ADDR, &pd_ch);
-	if (ret)
-		return ret;
-
-	gain_ch = gain_ch & MCP47FEB02_GAIN_BITS_MASK;
-	for_each_set_bit(i, &data->active_channels_mask, data->phys_channels) {
-		struct device *dev = regmap_get_device(data->regmap);
-		unsigned int pd_tmp;
-
-		data->chdata[i].ref_mode = (vref_ch >> (2 * i)) & MCP47FEB02_DAC_CTRL_MASK;
-		data->chdata[i].use_2x_gain = (gain_ch >> i)  & MCP47FEB02_GAIN_BIT_MASK;
-
-		/*
-		 * Inform the user that the current voltage reference read from the volatile
-		 * register of the chip is different from the one specified in the device tree.
-		 * Considering that the user cannot have an external voltage reference connected
-		 * to the pin and select the internal Band Gap at the same time, in order to avoid
-		 * miscofiguring the reference voltage, the volatile register will not be written.
-		 * In order to overwrite the setting from volatile register with the one from the
-		 * device tree, the user needs to write the chosen scale.
-		 */
-		switch (data->chdata[i].ref_mode) {
-		case MCP47FEB02_INTERNAL_BAND_GAP:
-			if (data->phys_channels >= 4 && (i % 2) && data->use_vref1) {
-				dev_dbg(dev, "ch[%u]: was configured to use internal band gap", i);
-				dev_dbg(dev, "ch[%u]: reference voltage set to VREF1", i);
-				break;
-			}
-			if ((data->phys_channels < 4 || (data->phys_channels >= 4 && !(i % 2))) &&
-			    data->use_vref) {
-				dev_dbg(dev, "ch[%u]: was configured to use internal band gap", i);
-				dev_dbg(dev, "ch[%u]: reference voltage set to VREF", i);
-				break;
-			}
-			break;
-		case MCP47FEB02_EXTERNAL_VREF_UNBUFFERED:
-		case MCP47FEB02_EXTERNAL_VREF_BUFFERED:
-			if (data->phys_channels >= 4 && (i % 2) && !data->use_vref1) {
-				dev_dbg(dev, "ch[%u]: was configured to use VREF1", i);
-				dev_dbg(dev,
-					"ch[%u]: reference voltage set to internal band gap", i);
-				break;
-			}
-			if ((data->phys_channels < 4 || (data->phys_channels >= 4 && !(i % 2))) &&
-			    !data->use_vref) {
-				dev_dbg(dev, "ch[%u]: was configured to use VREF", i);
-				dev_dbg(dev,
-					"ch[%u]: reference voltage set to internal band gap", i);
-				break;
-			}
-			break;
-		}
-
-		pd_tmp = (pd_ch >> (2 * i)) & MCP47FEB02_DAC_CTRL_MASK;
-		data->chdata[i].powerdown_mode = pd_tmp ? (pd_tmp - 1) : pd_tmp;
-		data->chdata[i].powerdown = !!(data->chdata[i].powerdown_mode);
-	}
-
-	return 0;
-}
-
-static int mcp47feb02_init_ch_scales(struct mcp47feb02_data *data, int vdd_uV,
-				     int vref_uV, int vref1_uV)
-{
-	unsigned int i;
-
-	for_each_set_bit(i, &data->active_channels_mask, data->phys_channels) {
-		struct device *dev = regmap_get_device(data->regmap);
-		int ret;
-
-		ret = mcp47feb02_init_scales_avail(data, vdd_uV, vref_uV, vref1_uV);
-		if (ret)
-			return dev_err_probe(dev, ret, "failed to init scales for ch %u\n", i);
-	}
-
-	return 0;
-}
-
-static int mcp47feb02_probe(struct i2c_client *client)
-{
-	const struct mcp47feb02_features *chip_features;
-	struct device *dev = &client->dev;
-	struct mcp47feb02_data *data;
-	struct iio_dev *indio_dev;
-	int vref1_uV, vref_uV, vdd_uV, ret;
-
-	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
-	if (!indio_dev)
-		return -ENOMEM;
-
-	data = iio_priv(indio_dev);
-	chip_features = i2c_get_match_data(client);
-	if (!chip_features)
-		return -EINVAL;
-
-	data->chip_features = chip_features;
-
-	if (chip_features->have_eeprom) {
-		data->regmap = devm_regmap_init_i2c(client, &mcp47feb02_regmap_config);
-		indio_dev->info = &mcp47feb02_info;
-	} else {
-		data->regmap = devm_regmap_init_i2c(client, &mcp47fvb02_regmap_config);
-		indio_dev->info = &mcp47fvb02_info;
-	}
-	if (IS_ERR(data->regmap))
-		return dev_err_probe(dev, PTR_ERR(data->regmap), "Error initializing i2c regmap\n");
-
-	indio_dev->name = chip_features->name;
-
-	ret = mcp47feb02_parse_fw(indio_dev, chip_features);
-	if (ret)
-		return dev_err_probe(dev, ret, "Error parsing firmware data\n");
-
-	ret = devm_mutex_init(dev, &data->lock);
-	if (ret)
-		return ret;
-
-	ret = devm_regulator_get_enable_read_voltage(dev, "vdd");
-	if (ret < 0)
-		return ret;
-
-	vdd_uV = ret;
-
-	if (device_property_present(dev, "vref-supply")) {
-		vref_uV = devm_regulator_get_enable_read_voltage(dev, "vref");
-		if (vref_uV < 0)
-			return vref_uV;
-
-		if (vref_uV == 0)
-			return dev_err_probe(dev, -EINVAL, "Vref is 0 uV.\n");
-
-		data->use_vref = true;
-	} else {
-		vref_uV = 0;
-		dev_dbg(dev, "Using internal band gap as voltage reference.\n");
-	}
-
-	if (chip_features->have_ext_vref1 &&
-	    device_property_present(dev, "vref1-supply")) {
-		vref1_uV = devm_regulator_get_enable_read_voltage(dev, "vref1");
-		if (vref1_uV < 0)
-			return vref1_uV;
-
-		if (vref1_uV == 0)
-			return dev_err_probe(dev, -EINVAL, "Vref1 is 0 uV.\n");
-
-		data->use_vref1 = true;
-	} else {
-		vref1_uV = 0;
-		dev_dbg(dev, "Using internal band gap as voltage reference 1.\n");
-	}
-
-	ret = mcp47feb02_init_ctrl_regs(data);
-	if (ret)
-		return dev_err_probe(dev, ret, "Error initialising vref register\n");
-
-	ret = mcp47feb02_init_ch_scales(data, vdd_uV, vref_uV, vref1_uV);
-	if (ret)
-		return ret;
-
-	return devm_iio_device_register(dev, indio_dev);
-}
-
-static const struct i2c_device_id mcp47feb02_id[] = {
-	{ .name = "mcp47feb01", .driver_data = (kernel_ulong_t)&mcp47feb01_chip_features },
-	{ .name = "mcp47feb02", .driver_data = (kernel_ulong_t)&mcp47feb02_chip_features },
-	{ .name = "mcp47feb04", .driver_data = (kernel_ulong_t)&mcp47feb04_chip_features },
-	{ .name = "mcp47feb08", .driver_data = (kernel_ulong_t)&mcp47feb08_chip_features },
-	{ .name = "mcp47feb11", .driver_data = (kernel_ulong_t)&mcp47feb11_chip_features },
-	{ .name = "mcp47feb12", .driver_data = (kernel_ulong_t)&mcp47feb12_chip_features },
-	{ .name = "mcp47feb14", .driver_data = (kernel_ulong_t)&mcp47feb14_chip_features },
-	{ .name = "mcp47feb18", .driver_data = (kernel_ulong_t)&mcp47feb18_chip_features },
-	{ .name = "mcp47feb21", .driver_data = (kernel_ulong_t)&mcp47feb21_chip_features },
-	{ .name = "mcp47feb22", .driver_data = (kernel_ulong_t)&mcp47feb22_chip_features },
-	{ .name = "mcp47feb24", .driver_data = (kernel_ulong_t)&mcp47feb24_chip_features },
-	{ .name = "mcp47feb28", .driver_data = (kernel_ulong_t)&mcp47feb28_chip_features },
-	{ .name = "mcp47fvb01", .driver_data = (kernel_ulong_t)&mcp47fvb01_chip_features },
-	{ .name = "mcp47fvb02", .driver_data = (kernel_ulong_t)&mcp47fvb02_chip_features },
-	{ .name = "mcp47fvb04", .driver_data = (kernel_ulong_t)&mcp47fvb04_chip_features },
-	{ .name = "mcp47fvb08", .driver_data = (kernel_ulong_t)&mcp47fvb08_chip_features },
-	{ .name = "mcp47fvb11", .driver_data = (kernel_ulong_t)&mcp47fvb11_chip_features },
-	{ .name = "mcp47fvb12", .driver_data = (kernel_ulong_t)&mcp47fvb12_chip_features },
-	{ .name = "mcp47fvb14", .driver_data = (kernel_ulong_t)&mcp47fvb14_chip_features },
-	{ .name = "mcp47fvb18", .driver_data = (kernel_ulong_t)&mcp47fvb18_chip_features },
-	{ .name = "mcp47fvb21", .driver_data = (kernel_ulong_t)&mcp47fvb21_chip_features },
-	{ .name = "mcp47fvb22", .driver_data = (kernel_ulong_t)&mcp47fvb22_chip_features },
-	{ .name = "mcp47fvb24", .driver_data = (kernel_ulong_t)&mcp47fvb24_chip_features },
-	{ .name = "mcp47fvb28", .driver_data = (kernel_ulong_t)&mcp47fvb28_chip_features },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, mcp47feb02_id);
-
-static const struct of_device_id mcp47feb02_of_match[] = {
-	{ .compatible = "microchip,mcp47feb01", .data = &mcp47feb01_chip_features },
-	{ .compatible = "microchip,mcp47feb02", .data = &mcp47feb02_chip_features },
-	{ .compatible = "microchip,mcp47feb04", .data = &mcp47feb04_chip_features },
-	{ .compatible = "microchip,mcp47feb08", .data = &mcp47feb08_chip_features },
-	{ .compatible = "microchip,mcp47feb11", .data = &mcp47feb11_chip_features },
-	{ .compatible = "microchip,mcp47feb12", .data = &mcp47feb12_chip_features },
-	{ .compatible = "microchip,mcp47feb14", .data = &mcp47feb14_chip_features },
-	{ .compatible = "microchip,mcp47feb18", .data = &mcp47feb18_chip_features },
-	{ .compatible = "microchip,mcp47feb21", .data = &mcp47feb21_chip_features },
-	{ .compatible = "microchip,mcp47feb22", .data = &mcp47feb22_chip_features },
-	{ .compatible = "microchip,mcp47feb24", .data = &mcp47feb24_chip_features },
-	{ .compatible = "microchip,mcp47feb28", .data = &mcp47feb28_chip_features },
-	{ .compatible = "microchip,mcp47fvb01", .data = &mcp47fvb01_chip_features },
-	{ .compatible = "microchip,mcp47fvb02", .data = &mcp47fvb02_chip_features },
-	{ .compatible = "microchip,mcp47fvb04", .data = &mcp47fvb04_chip_features },
-	{ .compatible = "microchip,mcp47fvb08", .data = &mcp47fvb08_chip_features },
-	{ .compatible = "microchip,mcp47fvb11", .data = &mcp47fvb11_chip_features },
-	{ .compatible = "microchip,mcp47fvb12", .data = &mcp47fvb12_chip_features },
-	{ .compatible = "microchip,mcp47fvb14",	.data = &mcp47fvb14_chip_features },
-	{ .compatible = "microchip,mcp47fvb18", .data = &mcp47fvb18_chip_features },
-	{ .compatible = "microchip,mcp47fvb21", .data = &mcp47fvb21_chip_features },
-	{ .compatible = "microchip,mcp47fvb22", .data = &mcp47fvb22_chip_features },
-	{ .compatible = "microchip,mcp47fvb24", .data = &mcp47fvb24_chip_features },
-	{ .compatible = "microchip,mcp47fvb28", .data = &mcp47fvb28_chip_features },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, mcp47feb02_of_match);
-
-static struct i2c_driver mcp47feb02_driver = {
-	.driver = {
-		.name	= "mcp47feb02",
-		.of_match_table = mcp47feb02_of_match,
-		.pm	= pm_sleep_ptr(&mcp47feb02_pm_ops),
-	},
-	.probe		= mcp47feb02_probe,
-	.id_table	= mcp47feb02_id,
-};
-module_i2c_driver(mcp47feb02_driver);
-
-MODULE_AUTHOR("Ariana Lazar <ariana.lazar@microchip.com>");
-MODULE_DESCRIPTION("IIO driver for MCP47FEB02 Multi-Channel DAC with I2C interface");
-MODULE_LICENSE("GPL");
diff --git a/drivers/iio/dac/mcp47feb02.h b/drivers/iio/dac/mcp47feb02.h
new file mode 100644
index 00000000000000..37fa4497c1c1af
--- /dev/null
+++ b/drivers/iio/dac/mcp47feb02.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef __DRIVERS_IIO_DAC_MCP47FEB02_H__
+#define __DRIVERS_IIO_DAC_MCP47FEB02_H__
+
+#include <linux/pm.h>
+#include <linux/types.h>
+
+#define MCP47FEB02_CHIP_INFO(_name, _channels, _res, _vref1, _eeprom) \
+static const struct mcp47feb02_features _name##_chip_features = { \
+	.name = #_name, \
+	.phys_channels = _channels, \
+	.resolution = _res, \
+	.have_ext_vref1 = _vref1, \
+	.have_eeprom = _eeprom, \
+}
+
+/**
+ * struct mcp47feb02_features - chip specific data
+ * @name: device name
+ * @phys_channels: number of hardware channels
+ * @resolution: DAC resolution
+ * @have_ext_vref1: does the hardware have an the second external voltage reference?
+ * @have_eeprom: does the hardware have an internal eeprom?
+ */
+struct mcp47feb02_features {
+	const char *name;
+	unsigned int phys_channels;
+	unsigned int resolution;
+	bool have_ext_vref1;
+	bool have_eeprom;
+};
+
+extern const struct regmap_config mcp47feb02_regmap_config;
+extern const struct regmap_config mcp47fvb02_regmap_config;
+
+int mcp47feb02_common_probe(const struct mcp47feb02_features *chip_features,
+			    struct regmap *regmap);
+
+extern const struct dev_pm_ops mcp47feb02_pm_ops;
+
+#endif /* __DRIVERS_IIO_DAC_MCP47FEB02_H__ */
diff --git a/drivers/iio/dac/mcp4821.c b/drivers/iio/dac/mcp4821.c
index 8d8c00cd60e985..59defb1b5e6216 100644
--- a/drivers/iio/dac/mcp4821.c
+++ b/drivers/iio/dac/mcp4821.c
@@ -255,9 +255,13 @@ static const struct iio_info mcp4821_info = {
 
 static int mcp4821_probe(struct spi_device *spi)
 {
+	const struct mcp4821_chip_info *info;
 	struct iio_dev *indio_dev;
 	struct mcp4821_state *state;
-	const struct mcp4821_chip_info *info;
+
+	info = spi_get_device_match_data(spi);
+	if (!info)
+		return -ENODATA;
 
 	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*state));
 	if (indio_dev == NULL)
@@ -268,7 +272,6 @@ static int mcp4821_probe(struct spi_device *spi)
 
 	/* default gain is 2x */
 	state->gain = 2;
-	info = spi_get_device_match_data(spi);
 	indio_dev->name = info->name;
 	indio_dev->info = &mcp4821_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
diff --git a/drivers/iio/dac/rohm-bd79703.c b/drivers/iio/dac/rohm-bd79703.c
index 6e2e7787f8a472..127ccc2c4a2293 100644
--- a/drivers/iio/dac/rohm-bd79703.c
+++ b/drivers/iio/dac/rohm-bd79703.c
@@ -73,6 +73,9 @@ static int bd79703_write_raw(struct iio_dev *idev,
 {
 	struct bd79703_data *data = iio_priv(idev);
 
+	if (mask != IIO_CHAN_INFO_RAW)
+		return -EINVAL;
+
 	if (val < 0 || val >= 1 << BD79703_DAC_BITS)
 		return -EINVAL;
 
diff --git a/drivers/iio/dummy/iio_simple_dummy.c b/drivers/iio/dummy/iio_simple_dummy.c
index 19fcdbbc11c63e..28ae1e7b593d54 100644
--- a/drivers/iio/dummy/iio_simple_dummy.c
+++ b/drivers/iio/dummy/iio_simple_dummy.c
@@ -116,7 +116,7 @@ static const struct iio_chan_spec iio_dummy_channels[] = {
 		BIT(IIO_CHAN_INFO_OFFSET) |
 		/*
 		 * in_voltage0_scale
-		 * Multipler for userspace to apply post offset
+		 * Multiplier for userspace to apply post offset
 		 * when converting to standard units (microvolts)
 		 */
 		BIT(IIO_CHAN_INFO_SCALE),
diff --git a/drivers/iio/frequency/Kconfig b/drivers/iio/frequency/Kconfig
index 90c6304c4bcda9..994d199080ebf1 100644
--- a/drivers/iio/frequency/Kconfig
+++ b/drivers/iio/frequency/Kconfig
@@ -23,6 +23,27 @@ config AD9523
 
 endmenu
 
+menu "Direct Digital Synthesis"
+
+config AD9910
+	tristate "Analog Devices AD9910 Direct Digital Synthesizer"
+	depends on SPI
+	depends on GPIOLIB
+	select CRC32
+	select FW_LOADER
+	select FW_UPLOAD
+	help
+	  Say yes here to build support for Analog Devices AD9910
+	  1 GSPS, 14-Bit DDS with integrated DAC.
+
+	  Supports single tone mode with 8 configurable profiles
+	  and digital ramp generation.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ad9910.
+
+endmenu
+
 #
 # Phase-Locked Loop (PLL) frequency synthesizers
 #
diff --git a/drivers/iio/frequency/Makefile b/drivers/iio/frequency/Makefile
index 53b4d01414d8d2..e3b1d36ed620c1 100644
--- a/drivers/iio/frequency/Makefile
+++ b/drivers/iio/frequency/Makefile
@@ -5,6 +5,7 @@
 
 # When adding new entries keep the list in alphabetical order
 obj-$(CONFIG_AD9523) += ad9523.o
+obj-$(CONFIG_AD9910) += ad9910.o
 obj-$(CONFIG_ADF41513) += adf41513.o
 obj-$(CONFIG_ADF4350) += adf4350.o
 obj-$(CONFIG_ADF4371) += adf4371.o
diff --git a/drivers/iio/frequency/ad9910.c b/drivers/iio/frequency/ad9910.c
new file mode 100644
index 00000000000000..04a4e184b873b7
--- /dev/null
+++ b/drivers/iio/frequency/ad9910.c
@@ -0,0 +1,2347 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AD9910 SPI DDS (Direct Digital Synthesizer) driver
+ *
+ * Copyright 2026 Analog Devices Inc.
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/crc32.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device-id/of.h>
+#include <linux/device-id/spi.h>
+#include <linux/device/devres.h>
+#include <linux/err.h>
+#include <linux/firmware.h>
+#include <linux/gpio/consumer.h>
+#include <linux/log2.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/seq_file.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include <linux/units.h>
+#include <linux/unaligned.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+/* Register addresses */
+#define AD9910_REG_CFR1			0x00
+#define AD9910_REG_CFR2			0x01
+#define AD9910_REG_CFR3			0x02
+#define AD9910_REG_AUX_DAC		0x03
+#define AD9910_REG_IO_UPDATE_RATE	0x04
+#define AD9910_REG_FTW			0x07
+#define AD9910_REG_POW			0x08
+#define AD9910_REG_ASF			0x09
+#define AD9910_REG_MULTICHIP_SYNC	0x0A
+#define AD9910_REG_DRG_LIMIT		0x0B
+#define AD9910_REG_DRG_STEP		0x0C
+#define AD9910_REG_DRG_RATE		0x0D
+#define AD9910_REG_PROFILE0		0x0E
+#define AD9910_REG_PROFILE1		0x0F
+#define AD9910_REG_PROFILE2		0x10
+#define AD9910_REG_PROFILE3		0x11
+#define AD9910_REG_PROFILE4		0x12
+#define AD9910_REG_PROFILE5		0x13
+#define AD9910_REG_PROFILE6		0x14
+#define AD9910_REG_PROFILE7		0x15
+#define AD9910_REG_RAM			0x16
+
+#define AD9910_REG_NUM_CACHED		0x16
+#define AD9910_REG_PROFILE(x)		(AD9910_REG_PROFILE0 + (x))
+
+/* CFR1 bit definitions */
+#define AD9910_CFR1_RAM_ENABLE_MSK		BIT(31)
+#define AD9910_CFR1_RAM_PLAYBACK_DEST_MSK	GENMASK(30, 29)
+#define AD9910_CFR1_OSK_MANUAL_EXT_CTL_MSK	BIT(23)
+#define AD9910_CFR1_INV_SINC_EN_MSK		BIT(22)
+#define AD9910_CFR1_INT_PROFILE_CTL_MSK		GENMASK(20, 17)
+#define AD9910_CFR1_SELECT_SINE_MSK		BIT(16)
+#define AD9910_CFR1_LOAD_LRR_IO_UPDATE_MSK	BIT(15)
+#define AD9910_CFR1_AUTOCLR_DIG_RAMP_ACCUM_MSK	BIT(14)
+#define AD9910_CFR1_AUTOCLR_PHASE_ACCUM_MSK	BIT(13)
+#define AD9910_CFR1_CLEAR_DIG_RAMP_ACCUM_MSK	BIT(12)
+#define AD9910_CFR1_CLEAR_PHASE_ACCUM_MSK	BIT(11)
+#define AD9910_CFR1_LOAD_ARR_IO_UPDATE_MSK	BIT(10)
+#define AD9910_CFR1_OSK_ENABLE_MSK		BIT(9)
+#define AD9910_CFR1_SELECT_AUTO_OSK_MSK		BIT(8)
+#define AD9910_CFR1_DIGITAL_POWER_DOWN_MSK	BIT(7)
+#define AD9910_CFR1_DAC_POWER_DOWN_MSK		BIT(6)
+#define AD9910_CFR1_REFCLK_INPUT_POWER_DOWN_MSK	BIT(5)
+#define AD9910_CFR1_AUX_DAC_POWER_DOWN_MSK	BIT(4)
+#define AD9910_CFR1_EXT_POWER_DOWN_CTL_MSK	BIT(3)
+#define AD9910_CFR1_SDIO_INPUT_ONLY_MSK		BIT(1)
+#define AD9910_CFR1_LSB_FIRST_MSK		BIT(0)
+
+/* DIGITAL and REFCLK_INPUT powerdown bits are effective without I/O update */
+#define AD9910_CFR1_SW0_POWER_DOWN_MSK	(AD9910_CFR1_DIGITAL_POWER_DOWN_MSK | \
+					 AD9910_CFR1_REFCLK_INPUT_POWER_DOWN_MSK)
+/* DAC and AUX_DAC powerdown require I/O update */
+#define AD9910_CFR1_SW1_POWER_DOWN_MSK	(AD9910_CFR1_DAC_POWER_DOWN_MSK | \
+					 AD9910_CFR1_AUX_DAC_POWER_DOWN_MSK)
+
+#define AD9910_CFR1_SW_POWER_DOWN_MSK	(AD9910_CFR1_SW0_POWER_DOWN_MSK | \
+					 AD9910_CFR1_SW1_POWER_DOWN_MSK)
+
+/* CFR2 bit definitions */
+#define AD9910_CFR2_AMP_SCALE_SINGLE_TONE_MSK	BIT(24)
+#define AD9910_CFR2_INTERNAL_IO_UPDATE_MSK	BIT(23)
+#define AD9910_CFR2_SYNC_CLK_EN_MSK		BIT(22)
+#define AD9910_CFR2_DRG_DEST_MSK		GENMASK(21, 20)
+#define AD9910_CFR2_DRG_ENABLE_MSK		BIT(19)
+#define AD9910_CFR2_DRG_NO_DWELL_HIGH_MSK	BIT(18)
+#define AD9910_CFR2_DRG_NO_DWELL_LOW_MSK	BIT(17)
+#define AD9910_CFR2_DRG_NO_DWELL_MSK		GENMASK(18, 17)
+#define AD9910_CFR2_READ_EFFECTIVE_FTW_MSK	BIT(16)
+#define AD9910_CFR2_IO_UPDATE_RATE_CTL_MSK	GENMASK(15, 14)
+#define AD9910_CFR2_PDCLK_ENABLE_MSK		BIT(11)
+#define AD9910_CFR2_PDCLK_INVERT_MSK		BIT(10)
+#define AD9910_CFR2_TXENABLE_INVERT_MSK		BIT(9)
+#define AD9910_CFR2_MATCHED_LATENCY_EN_MSK	BIT(7)
+#define AD9910_CFR2_DATA_ASM_HOLD_LAST_MSK	BIT(6)
+#define AD9910_CFR2_SYNC_TIMING_VAL_DISABLE_MSK	BIT(5)
+#define AD9910_CFR2_PARALLEL_DATA_PORT_EN_MSK	BIT(4)
+#define AD9910_CFR2_FM_GAIN_MSK			GENMASK(3, 0)
+
+/* CFR3 bit definitions */
+#define AD9910_CFR3_OPEN_MSK			(BIT(27) | GENMASK(18, 16))
+#define AD9910_CFR3_DRV0_MSK			GENMASK(29, 28)
+#define AD9910_CFR3_VCO_SEL_MSK			GENMASK(26, 24)
+#define AD9910_CFR3_ICP_MSK			GENMASK(21, 19)
+#define AD9910_CFR3_REFCLK_DIV_BYPASS_MSK	BIT(15)
+#define AD9910_CFR3_REFCLK_DIV_RESETB_MSK	BIT(14)
+#define AD9910_CFR3_PFD_RESET_MSK		BIT(10)
+#define AD9910_CFR3_PLL_EN_MSK			BIT(8)
+#define AD9910_CFR3_N_MSK			GENMASK(7, 1)
+
+/* Auxiliary DAC Control Register Bits */
+#define AD9910_AUX_DAC_FSC_MSK			GENMASK(7, 0)
+
+/* POW Register Bits */
+#define AD9910_POW_PP_LSB_MSK			GENMASK(7, 0)
+
+/* ASF Register Bits */
+#define AD9910_ASF_RAMP_RATE_MSK		GENMASK(31, 16)
+#define AD9910_ASF_SCALE_FACTOR_MSK		GENMASK(15, 2)
+#define AD9910_ASF_SCALE_FACTOR_PP_LSB_MSK	GENMASK(7, 2)
+#define AD9910_ASF_STEP_SIZE_MSK		GENMASK(1, 0)
+
+/* Multichip Sync Register Bits */
+#define AD9910_MC_SYNC_VALIDATION_DELAY_MSK	GENMASK(31, 28)
+#define AD9910_MC_SYNC_RECEIVER_ENABLE_MSK	BIT(27)
+#define AD9910_MC_SYNC_GENERATOR_ENABLE_MSK	BIT(26)
+#define AD9910_MC_SYNC_GENERATOR_POLARITY_MSK	BIT(25)
+#define AD9910_MC_SYNC_STATE_PRESET_MSK		GENMASK(23, 18)
+#define AD9910_MC_SYNC_OUTPUT_DELAY_MSK		GENMASK(15, 11)
+#define AD9910_MC_SYNC_INPUT_DELAY_MSK		GENMASK(7, 3)
+
+/* Digital Ramp Limit Register */
+#define AD9910_DRG_LIMIT_UPPER_MSK		GENMASK_ULL(63, 32)
+#define AD9910_DRG_LIMIT_LOWER_MSK		GENMASK_ULL(31, 0)
+
+/* Digital Ramp Step Register */
+#define AD9910_DRG_STEP_DEC_MSK			GENMASK_ULL(63, 32)
+#define AD9910_DRG_STEP_INC_MSK			GENMASK_ULL(31, 0)
+
+/* Digital Ramp Rate Register */
+#define AD9910_DRG_RATE_DEC_MSK			GENMASK(31, 16)
+#define AD9910_DRG_RATE_INC_MSK			GENMASK(15, 0)
+
+/* Profile Register Format (Single Tone Mode) */
+#define AD9910_PROFILE_ST_ASF_MSK		GENMASK_ULL(61, 48)
+#define AD9910_PROFILE_ST_POW_MSK		GENMASK_ULL(47, 32)
+#define AD9910_PROFILE_ST_FTW_MSK		GENMASK_ULL(31, 0)
+
+/* Profile Register Format (RAM Mode) */
+#define AD9910_PROFILE_RAM_OPEN_MSK		GENMASK_ULL(61, 56)
+#define AD9910_PROFILE_RAM_STEP_RATE_MSK	GENMASK_ULL(55, 40)
+#define AD9910_PROFILE_RAM_END_ADDR_MSK		GENMASK_ULL(39, 30)
+#define AD9910_PROFILE_RAM_START_ADDR_MSK	GENMASK_ULL(23, 14)
+#define AD9910_PROFILE_RAM_NO_DWELL_HIGH_MSK	BIT_ULL(5)
+#define AD9910_PROFILE_RAM_ZERO_CROSSING_MSK	BIT_ULL(3)
+#define AD9910_PROFILE_RAM_MODE_CONTROL_MSK	GENMASK_ULL(2, 0)
+
+/* Device constants */
+#define AD9910_PI_NANORAD		3141592653UL
+#define AD9910_PI_PICORAD		3141592653590ULL
+
+#define AD9910_MAX_SYSCLK_HZ		(1000 * HZ_PER_MHZ)
+#define AD9910_MAX_PHASE_MICRORAD	(AD9910_PI_NANORAD / 500)
+
+#define AD9910_ASF_MAX			FIELD_MAX(AD9910_PROFILE_ST_ASF_MSK)
+#define AD9910_ASF_PP_LSB_MAX		FIELD_MAX(AD9910_ASF_SCALE_FACTOR_PP_LSB_MSK)
+#define AD9910_POW_MAX			FIELD_MAX(AD9910_PROFILE_ST_POW_MSK)
+#define AD9910_POW_PP_LSB_MAX		FIELD_MAX(AD9910_POW_PP_LSB_MSK)
+#define AD9910_STEP_RATE_MAX		FIELD_MAX(AD9910_DRG_RATE_DEC_MSK)
+#define AD9910_NUM_PROFILES		8
+
+#define AD9910_RAM_FW_MAGIC		0x00AD9910
+#define AD9910_RAM_FW_V1		0x0001
+#define AD9910_RAM_SIZE_MAX_WORDS	1024
+#define AD9910_RAM_WORD_SIZE		sizeof(u32)
+#define AD9910_RAM_SIZE_MAX_BYTES	(AD9910_RAM_SIZE_MAX_WORDS * AD9910_RAM_WORD_SIZE)
+#define AD9910_RAM_ADDR_MAX		(AD9910_RAM_SIZE_MAX_WORDS - 1)
+
+#define AD9910_RAM_ENABLED(st)		\
+	FIELD_GET(AD9910_CFR1_RAM_ENABLE_MSK, (st)->reg[AD9910_REG_CFR1].val32)
+
+#define AD9910_DEST_FREQUENCY		0
+#define AD9910_DEST_PHASE		1
+#define AD9910_DEST_AMPLITUDE		2
+#define AD9910_DEST_POLAR		3
+
+/* PLL constants */
+#define AD9910_PLL_MIN_N		12
+#define AD9910_PLL_MAX_N		127
+
+#define AD9910_PLL_IN_MIN_FREQ_HZ	(3307 * HZ_PER_KHZ)
+#define AD9910_PLL_IN_MAX_FREQ_HZ	(60 * HZ_PER_MHZ)
+
+#define AD9910_PLL_OUT_MIN_FREQ_HZ	(420 * HZ_PER_MHZ)
+#define AD9910_PLL_OUT_MAX_FREQ_HZ	AD9910_MAX_SYSCLK_HZ
+
+#define AD9910_VCO0_RANGE_AUTO_MAX_HZ	(457 * HZ_PER_MHZ)
+#define AD9910_VCO1_RANGE_AUTO_MAX_HZ	(530 * HZ_PER_MHZ)
+#define AD9910_VCO2_RANGE_AUTO_MAX_HZ	(632 * HZ_PER_MHZ)
+#define AD9910_VCO3_RANGE_AUTO_MAX_HZ	(775 * HZ_PER_MHZ)
+#define AD9910_VCO4_RANGE_AUTO_MAX_HZ	(897 * HZ_PER_MHZ)
+#define AD9910_VCO_RANGE_NUM		6
+
+#define AD9910_ICP_MIN_uA		212
+#define AD9910_ICP_MAX_uA		387
+#define AD9910_ICP_STEP_uA		25
+
+#define AD9910_DAC_IOUT_MAX_uA		31590
+#define AD9910_DAC_IOUT_DEFAULT_uA	20070
+#define AD9910_DAC_IOUT_MIN_uA		8640
+
+/* altcurrent ABI is in mA */
+#define AD9910_NANO_MILLIAMP_PER_MICROAMP	1000000UL
+#define AD9910_PICO_MILLIAMP_PER_MICROAMP	1000000000UL
+
+#define AD9910_REFDIV2_MIN_FREQ_HZ	(25 * HZ_PER_MHZ)
+#define AD9910_REFDIV2_MAX_FREQ_HZ	(1900 * HZ_PER_MHZ)
+
+#define AD9910_WAKEUP_DELAY_us		1000	/* 1ms: datasheet Table 1 */
+#define AD9910_RESET_DELAY_us		1	/* 5 sysclk cycles: < 1us */
+
+#define AD9910_SPI_DATA_IDX		1
+#define AD9910_SPI_DATA_LEN_MAX		AD9910_RAM_SIZE_MAX_BYTES
+#define AD9910_SPI_MESSAGE_LEN_MAX	(AD9910_SPI_DATA_IDX + AD9910_SPI_DATA_LEN_MAX)
+#define AD9910_SPI_READ_MSK		BIT(7)
+#define AD9910_SPI_ADDR_MSK		GENMASK(4, 0)
+
+/**
+ * enum ad9910_channel - AD9910 channel identifiers in priority order
+ *
+ * @AD9910_CHANNEL_PHY: Physical output channel
+ * @AD9910_CHANNEL_PROFILE_0: Profile 0 output channel
+ * @AD9910_CHANNEL_PROFILE_1: Profile 1 output channel
+ * @AD9910_CHANNEL_PROFILE_2: Profile 2 output channel
+ * @AD9910_CHANNEL_PROFILE_3: Profile 3 output channel
+ * @AD9910_CHANNEL_PROFILE_4: Profile 4 output channel
+ * @AD9910_CHANNEL_PROFILE_5: Profile 5 output channel
+ * @AD9910_CHANNEL_PROFILE_6: Profile 6 output channel
+ * @AD9910_CHANNEL_PROFILE_7: Profile 7 output channel
+ * @AD9910_CHANNEL_PARALLEL: Parallel Data output channel
+ * @AD9910_CHANNEL_PARALLEL_POLAR: Parallel Polar Data output channel
+ * @AD9910_CHANNEL_DRG: Digital Ramp Generator output channel
+ * @AD9910_CHANNEL_DRG_RAMP_UP: DRG ramp up channel
+ * @AD9910_CHANNEL_DRG_RAMP_DOWN: DRG ramp down channel
+ * @AD9910_CHANNEL_RAM: RAM control output channel
+ * @AD9910_CHANNEL_OSK: Output Shift Keying output channel
+ */
+enum ad9910_channel {
+	AD9910_CHANNEL_PHY = 100,
+	AD9910_CHANNEL_PROFILE_0 = 110,
+	AD9910_CHANNEL_PROFILE_1 = 111,
+	AD9910_CHANNEL_PROFILE_2 = 112,
+	AD9910_CHANNEL_PROFILE_3 = 113,
+	AD9910_CHANNEL_PROFILE_4 = 114,
+	AD9910_CHANNEL_PROFILE_5 = 115,
+	AD9910_CHANNEL_PROFILE_6 = 116,
+	AD9910_CHANNEL_PROFILE_7 = 117,
+	AD9910_CHANNEL_PARALLEL = 120,
+	AD9910_CHANNEL_PARALLEL_POLAR = 121,
+	AD9910_CHANNEL_DRG = 130,
+	AD9910_CHANNEL_DRG_RAMP_UP = 131,
+	AD9910_CHANNEL_DRG_RAMP_DOWN = 132,
+	AD9910_CHANNEL_RAM = 140,
+	AD9910_CHANNEL_OSK = 150,
+};
+
+/**
+ * struct ad9910_ram_fw - AD9910 RAM firmware format
+ * @magic:	Magic number for RAM firmware validation
+ * @version:	Firmware version number
+ * @wcount:	Number of RAM words to be written
+ * @crc:	CRC32 checksum of the RAM data for integrity verification
+ * @cfr1:	Value of CFR1 register to be configured (not all fields are
+ *		used, but this is included here for convenience)
+ * @profiles:	Array of RAM profile configurations
+ * @words:	Array of RAM words to be written. Data pattern should be set in
+ *		reverse order and wcount specifies the number of words in this
+ *		array
+ */
+struct ad9910_ram_fw {
+	__be32 magic;
+	__be16 version;
+	__be16 wcount;
+	__be32 crc;
+	__be32 cfr1;
+	__be64 profiles[AD9910_NUM_PROFILES];
+	__be32 words[] __counted_by_be(wcount);
+} __packed;
+
+enum {
+	AD9910_CHAN_IDX_PHY,
+	AD9910_CHAN_IDX_PROFILE_0,
+	AD9910_CHAN_IDX_PROFILE_1,
+	AD9910_CHAN_IDX_PROFILE_2,
+	AD9910_CHAN_IDX_PROFILE_3,
+	AD9910_CHAN_IDX_PROFILE_4,
+	AD9910_CHAN_IDX_PROFILE_5,
+	AD9910_CHAN_IDX_PROFILE_6,
+	AD9910_CHAN_IDX_PROFILE_7,
+	AD9910_CHAN_IDX_PARALLEL_AMP,
+	AD9910_CHAN_IDX_PARALLEL_PHASE,
+	AD9910_CHAN_IDX_PARALLEL_FREQ,
+	AD9910_CHAN_IDX_PARALLEL_POLAR_AMP,
+	AD9910_CHAN_IDX_PARALLEL_POLAR_PHASE,
+	AD9910_CHAN_IDX_DRG_FREQ,
+	AD9910_CHAN_IDX_DRG_PHASE,
+	AD9910_CHAN_IDX_DRG_AMP,
+	AD9910_CHAN_IDX_DRG_FREQ_RAMP_UP,
+	AD9910_CHAN_IDX_DRG_FREQ_RAMP_DOWN,
+	AD9910_CHAN_IDX_DRG_PHASE_RAMP_UP,
+	AD9910_CHAN_IDX_DRG_PHASE_RAMP_DOWN,
+	AD9910_CHAN_IDX_DRG_AMP_RAMP_UP,
+	AD9910_CHAN_IDX_DRG_AMP_RAMP_DOWN,
+	AD9910_CHAN_IDX_RAM,
+	AD9910_CHAN_IDX_OSK,
+};
+
+enum {
+	AD9910_POWERDOWN,
+	AD9910_DWELL_EN,
+	AD9910_ROC,
+	AD9910_ROC_AVAIL,
+};
+
+struct ad9910_data {
+	u32 sysclk_freq_hz;
+	u32 output_current_uA;
+
+	u16 pll_charge_pump_current;
+	u8 refclk_out_drv;
+	bool pll_enabled;
+};
+
+union ad9910_reg {
+	u64 val64;
+	u32 val32;
+	u16 val16;
+};
+
+struct ad9910_state {
+	struct spi_device *spi;
+	struct clk *refclk;
+	struct fw_upload *ram_fwu;
+
+	struct gpio_desc *gpio_pwdown;
+	struct gpio_desc *gpio_update;
+	struct gpio_descs *gpio_profile;
+
+	/* cached registers */
+	union ad9910_reg reg[AD9910_REG_NUM_CACHED];
+
+	/*
+	 * alternate profile registers used to store RAM profile settings when
+	 * RAM mode is disabled and Single Tone profile settings when RAM mode
+	 * is enabled.
+	 */
+	u64 reg_profile[AD9910_NUM_PROFILES];
+
+	/* Lock for accessing device registers and state variables */
+	struct mutex lock;
+
+	struct ad9910_data data;
+	u8 profile;
+
+	bool ram_fwu_cancel;
+	char ram_fwu_name[20];
+
+	/*
+	 * RAM loading requires a reasonable amount of bytes, at the same time
+	 * DMA capable SPI drivers requires the transfer buffers to live in
+	 * their own cache lines.
+	 */
+	u8 tx_buf[AD9910_SPI_MESSAGE_LEN_MAX] __aligned(IIO_DMA_MINALIGN);
+};
+
+/**
+ * ad9910_rational_scale() - Perform scaling of input given a reference.
+ * @input: The input value to be scaled.
+ * @scale: The numerator of the scaling factor.
+ * @reference: The denominator of the scaling factor.
+ *
+ * Closest rounding with mul_u64_add_u64_div_u64
+ *
+ * Return: The scaled value.
+ */
+static u64 ad9910_rational_scale(u64 input, u64 scale, u64 reference)
+{
+	return mul_u64_add_u64_div_u64(input, scale, reference >> 1, reference);
+}
+
+static inline u64 ad9910_ram_profile_val(struct ad9910_state *st)
+{
+	if (AD9910_RAM_ENABLED(st))
+		return st->reg[AD9910_REG_PROFILE(st->profile)].val64;
+	else
+		return st->reg_profile[st->profile];
+}
+
+static inline u64 ad9910_st_profile_val(struct ad9910_state *st, u8 profile)
+{
+	if (AD9910_RAM_ENABLED(st))
+		return st->reg_profile[profile];
+	else
+		return st->reg[AD9910_REG_PROFILE(profile)].val64;
+}
+
+static int ad9910_io_update(struct ad9910_state *st)
+{
+	if (st->gpio_update) {
+		gpiod_set_value_cansleep(st->gpio_update, 1);
+		fsleep(1);
+		gpiod_set_value_cansleep(st->gpio_update, 0);
+	}
+
+	return 0;
+}
+
+static int ad9910_spi_read(struct ad9910_state *st, u8 reg, void *data,
+			   size_t len)
+{
+	u8 inst = AD9910_SPI_READ_MSK | FIELD_PREP(AD9910_SPI_ADDR_MSK, reg);
+
+	return spi_write_then_read(st->spi, &inst, sizeof(inst), data, len);
+}
+
+static int ad9910_spi_write(struct ad9910_state *st, u8 reg, size_t len,
+			    bool update)
+{
+	int ret;
+
+	st->tx_buf[0] = FIELD_PREP(AD9910_SPI_ADDR_MSK, reg);
+	ret = spi_write(st->spi, st->tx_buf, AD9910_SPI_DATA_IDX + len);
+	if (ret)
+		return ret;
+
+	if (update)
+		return ad9910_io_update(st);
+
+	return 0;
+}
+
+#define AD9910_REG_READ_FN(nb)						\
+static int ad9910_reg##nb##_read(struct ad9910_state *st, u8 reg,	\
+				 u##nb * data)				\
+{									\
+	__be##nb be_data;						\
+	int ret;							\
+									\
+	ret = ad9910_spi_read(st, reg, &be_data, sizeof(be_data));	\
+	if (ret)							\
+		return ret;						\
+									\
+	*data = be##nb##_to_cpu(be_data);				\
+	return ret;							\
+}
+
+AD9910_REG_READ_FN(16)
+AD9910_REG_READ_FN(32)
+AD9910_REG_READ_FN(64)
+
+#define AD9910_REG_WRITE_FN(nb)						\
+static int ad9910_reg##nb##_write(struct ad9910_state *st, u8 reg,	\
+				  u##nb data, bool update)		\
+{									\
+	int ret;							\
+									\
+	put_unaligned_be##nb(data, &st->tx_buf[AD9910_SPI_DATA_IDX]);	\
+	ret = ad9910_spi_write(st, reg, sizeof(data), update);		\
+	if (ret)							\
+		return ret;						\
+									\
+	st->reg[reg].val##nb = data;					\
+	return ret;							\
+}
+
+AD9910_REG_WRITE_FN(16)
+AD9910_REG_WRITE_FN(32)
+AD9910_REG_WRITE_FN(64)
+
+#define AD9910_REG_UPDATE_FN(nb)					\
+static int ad9910_reg##nb##_update(struct ad9910_state *st,		\
+				   u8 reg, u##nb mask,			\
+				   u##nb data, bool update)		\
+{									\
+	u##nb reg_val = (st->reg[reg].val##nb & ~mask) | (data & mask);	\
+									\
+	if (reg_val == st->reg[reg].val##nb && !update)			\
+		return 0;						\
+									\
+	return ad9910_reg##nb##_write(st, reg, reg_val, update);	\
+}
+
+AD9910_REG_UPDATE_FN(16)
+AD9910_REG_UPDATE_FN(32)
+AD9910_REG_UPDATE_FN(64)
+
+static int ad9910_set_dac_current(struct ad9910_state *st, u32 val_uA,
+				  bool update)
+{
+	u32 code;
+
+	/* FSC = (86.4 / Rset) * (1 + CODE/96) where Rset = 10k ohms */
+	val_uA = clamp(val_uA, AD9910_DAC_IOUT_MIN_uA, AD9910_DAC_IOUT_MAX_uA);
+	code = DIV_ROUND_CLOSEST(val_uA - AD9910_DAC_IOUT_MIN_uA, 90);
+	st->data.output_current_uA = AD9910_DAC_IOUT_MIN_uA + code * 90;
+
+	return ad9910_reg32_write(st, AD9910_REG_AUX_DAC, code, update);
+}
+
+static int ad9910_set_sysclk_freq(struct ad9910_state *st, u32 freq_hz,
+				  bool update)
+{
+	struct device *dev = &st->spi->dev;
+	unsigned long refclk_freq_hz;
+	u32 sysclk_freq_hz;
+	u32 tmp32, vco_sel;
+	int ret;
+
+	if (!freq_hz || freq_hz > AD9910_MAX_SYSCLK_HZ)
+		return -EINVAL;
+
+	refclk_freq_hz = clk_get_rate(st->refclk);
+	if (st->data.pll_enabled) {
+		if (refclk_freq_hz < AD9910_PLL_IN_MIN_FREQ_HZ ||
+		    refclk_freq_hz > AD9910_PLL_IN_MAX_FREQ_HZ) {
+			dev_err(dev,
+				"REF_CLK freq %lu Hz is out of PLL input range\n",
+				refclk_freq_hz);
+			return -ERANGE;
+		}
+
+		tmp32 = DIV_ROUND_CLOSEST(freq_hz, refclk_freq_hz);
+		tmp32 = clamp(tmp32, DIV_ROUND_UP(AD9910_PLL_OUT_MIN_FREQ_HZ, refclk_freq_hz),
+			      AD9910_PLL_OUT_MAX_FREQ_HZ / refclk_freq_hz);
+		tmp32 = clamp(tmp32, AD9910_PLL_MIN_N, AD9910_PLL_MAX_N);
+		sysclk_freq_hz = refclk_freq_hz * tmp32;
+
+		if (sysclk_freq_hz <= AD9910_VCO0_RANGE_AUTO_MAX_HZ)
+			vco_sel = 0;
+		else if (sysclk_freq_hz <= AD9910_VCO1_RANGE_AUTO_MAX_HZ)
+			vco_sel = 1;
+		else if (sysclk_freq_hz <= AD9910_VCO2_RANGE_AUTO_MAX_HZ)
+			vco_sel = 2;
+		else if (sysclk_freq_hz <= AD9910_VCO3_RANGE_AUTO_MAX_HZ)
+			vco_sel = 3;
+		else if (sysclk_freq_hz <= AD9910_VCO4_RANGE_AUTO_MAX_HZ)
+			vco_sel = 4;
+		else
+			vco_sel = 5;
+
+		ret = ad9910_reg32_update(st, AD9910_REG_CFR3,
+					  AD9910_CFR3_N_MSK | AD9910_CFR3_VCO_SEL_MSK,
+					  FIELD_PREP(AD9910_CFR3_N_MSK, tmp32) |
+					  FIELD_PREP(AD9910_CFR3_VCO_SEL_MSK, vco_sel),
+					  update);
+		if (ret)
+			return ret;
+	} else {
+		if (refclk_freq_hz < AD9910_REFDIV2_MIN_FREQ_HZ ||
+		    refclk_freq_hz > AD9910_REFDIV2_MAX_FREQ_HZ) {
+			dev_err(dev,
+				"REF_CLK freq %lu Hz is out of divider range\n",
+				refclk_freq_hz);
+			return -ERANGE;
+		}
+
+		tmp32 = DIV_ROUND_CLOSEST(refclk_freq_hz, freq_hz);
+		tmp32 = clamp(tmp32, 1U, 2U);
+		sysclk_freq_hz = refclk_freq_hz / tmp32;
+		tmp32 = AD9910_CFR3_VCO_SEL_MSK |
+			FIELD_PREP(AD9910_CFR3_REFCLK_DIV_BYPASS_MSK, tmp32 % 2);
+		ret = ad9910_reg32_update(st, AD9910_REG_CFR3,
+					  AD9910_CFR3_VCO_SEL_MSK |
+					  AD9910_CFR3_REFCLK_DIV_BYPASS_MSK,
+					  tmp32, update);
+		if (ret)
+			return ret;
+	}
+
+	st->data.sysclk_freq_hz = sysclk_freq_hz;
+
+	return 0;
+}
+
+static int ad9910_profile_set(struct ad9910_state *st, u8 profile)
+{
+	DECLARE_BITMAP(values, BITS_PER_TYPE(profile));
+
+	st->profile = profile;
+	values[0] = profile;
+	gpiod_multi_set_value_cansleep(st->gpio_profile, values);
+
+	return 0;
+}
+
+static bool ad9910_sw_powerdown_get(struct ad9910_state *st)
+{
+	return FIELD_GET(AD9910_CFR1_SW_POWER_DOWN_MSK,
+			 st->reg[AD9910_REG_CFR1].val32) ? true : false;
+}
+
+static int ad9910_sw_powerdown_set(struct ad9910_state *st, bool enable)
+{
+	int ret;
+
+	if (ad9910_sw_powerdown_get(st) == enable)
+		return 0;
+
+	/*
+	 * When powering down, the DAC and AUX_DAC (SW1) must be powered down
+	 * first, as they require an I/O update to take effect. The opposite is
+	 * true when powering up, the DAC and AUX_DAC must be powered up last,
+	 * i.e., after the DIGITAL and REFCLK_INPUT (SW0) power up.
+	 */
+	if (enable) {
+		ret = ad9910_reg32_update(st, AD9910_REG_CFR1,
+					  AD9910_CFR1_SW1_POWER_DOWN_MSK,
+					  AD9910_CFR1_SW1_POWER_DOWN_MSK,
+					  true);
+		if (ret)
+			return ret;
+
+		return ad9910_reg32_update(st, AD9910_REG_CFR1,
+					   AD9910_CFR1_SW0_POWER_DOWN_MSK,
+					   AD9910_CFR1_SW0_POWER_DOWN_MSK,
+					   false);
+	}
+
+	ret = ad9910_reg32_update(st, AD9910_REG_CFR1,
+				  AD9910_CFR1_SW0_POWER_DOWN_MSK, 0, false);
+	if (ret)
+		return ret;
+
+	return ad9910_reg32_update(st, AD9910_REG_CFR1,
+				   AD9910_CFR1_SW1_POWER_DOWN_MSK, 0, true);
+}
+
+static ssize_t ad9910_ext_info_read(struct iio_dev *indio_dev,
+				    uintptr_t private,
+				    const struct iio_chan_spec *chan,
+				    char *buf)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	int val;
+
+	guard(mutex)(&st->lock);
+
+	switch (private) {
+	case AD9910_POWERDOWN:
+		val = ad9910_sw_powerdown_get(st);
+		break;
+	case AD9910_DWELL_EN:
+		if (chan->channel == AD9910_CHANNEL_DRG_RAMP_UP)
+			val = FIELD_GET(AD9910_CFR2_DRG_NO_DWELL_HIGH_MSK,
+					st->reg[AD9910_REG_CFR2].val32) ? 0 : 1;
+		else
+			val = FIELD_GET(AD9910_CFR2_DRG_NO_DWELL_LOW_MSK,
+					st->reg[AD9910_REG_CFR2].val32) ? 0 : 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return iio_format_value(buf, IIO_VAL_INT, 1, &val);
+}
+
+static ssize_t ad9910_ext_info_write(struct iio_dev *indio_dev,
+				     uintptr_t private,
+				     const struct iio_chan_spec *chan,
+				     const char *buf, size_t len)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	u32 val32;
+	int ret;
+
+	ret = kstrtou32(buf, 10, &val32);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&st->lock);
+
+	switch (private) {
+	case AD9910_POWERDOWN:
+		ret = ad9910_sw_powerdown_set(st, val32 ? true : false);
+		if (ret)
+			return ret;
+		break;
+	case AD9910_DWELL_EN:
+		if (chan->channel == AD9910_CHANNEL_DRG_RAMP_UP) {
+			val32 = val32 ? 0 : AD9910_CFR2_DRG_NO_DWELL_HIGH_MSK;
+			ret = ad9910_reg32_update(st, AD9910_REG_CFR2,
+						  AD9910_CFR2_DRG_NO_DWELL_HIGH_MSK,
+						  val32, true);
+			if (ret)
+				return ret;
+		} else {
+			val32 = val32 ? 0 : AD9910_CFR2_DRG_NO_DWELL_LOW_MSK;
+			ret = ad9910_reg32_update(st, AD9910_REG_CFR2,
+						  AD9910_CFR2_DRG_NO_DWELL_LOW_MSK,
+						  val32, true);
+			if (ret)
+				return ret;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return len;
+}
+
+static ssize_t ad9910_drg_roc_read(struct iio_dev *indio_dev, uintptr_t private,
+				   const struct iio_chan_spec *chan,
+				   char *buf)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	u64 roc64;
+	u32 rate;
+
+	guard(mutex)(&st->lock);
+
+	switch (chan->channel) {
+	case AD9910_CHANNEL_DRG_RAMP_UP:
+		roc64 = FIELD_GET(AD9910_DRG_STEP_INC_MSK,
+				  st->reg[AD9910_REG_DRG_STEP].val64);
+		rate = FIELD_GET(AD9910_DRG_RATE_INC_MSK,
+				 st->reg[AD9910_REG_DRG_RATE].val32);
+		break;
+	case AD9910_CHANNEL_DRG_RAMP_DOWN:
+		roc64 = FIELD_GET(AD9910_DRG_STEP_DEC_MSK,
+				  st->reg[AD9910_REG_DRG_STEP].val64);
+		rate = FIELD_GET(AD9910_DRG_RATE_DEC_MSK,
+				 st->reg[AD9910_REG_DRG_RATE].val32);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (!rate)
+		return -ERANGE;
+
+	roc64 *= st->data.sysclk_freq_hz;
+	return sysfs_emit(buf, "%llu\n", div_u64(roc64, 4 * rate));
+}
+
+static ssize_t ad9910_drg_roc_write(struct iio_dev *indio_dev,
+				    uintptr_t private,
+				    const struct iio_chan_spec *chan,
+				    const char *buf, size_t len)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	u64 tmp64;
+	u32 rate;
+	int ret;
+
+	ret = kstrtou64(buf, 10, &tmp64);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&st->lock);
+
+	switch (chan->channel) {
+	case AD9910_CHANNEL_DRG_RAMP_UP:
+		rate = FIELD_GET(AD9910_DRG_RATE_INC_MSK,
+				 st->reg[AD9910_REG_DRG_RATE].val32);
+		if (!rate)
+			return -ERANGE;
+
+		tmp64 = ad9910_rational_scale(tmp64, 4 * rate, st->data.sysclk_freq_hz);
+		tmp64 = min_t(u64, tmp64, U32_MAX);
+
+		ret = ad9910_reg64_update(st, AD9910_REG_DRG_STEP,
+					  AD9910_DRG_STEP_INC_MSK,
+					  FIELD_PREP(AD9910_DRG_STEP_INC_MSK, tmp64),
+					  true);
+		if (ret)
+			return ret;
+		break;
+	case AD9910_CHANNEL_DRG_RAMP_DOWN:
+		rate = FIELD_GET(AD9910_DRG_RATE_DEC_MSK,
+				 st->reg[AD9910_REG_DRG_RATE].val32);
+		if (!rate)
+			return -ERANGE;
+
+		tmp64 = ad9910_rational_scale(tmp64, 4 * rate, st->data.sysclk_freq_hz);
+		tmp64 = min_t(u64, tmp64, U32_MAX);
+
+		ret = ad9910_reg64_update(st, AD9910_REG_DRG_STEP,
+					  AD9910_DRG_STEP_DEC_MSK,
+					  FIELD_PREP(AD9910_DRG_STEP_DEC_MSK, tmp64),
+					  true);
+		if (ret)
+			return ret;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return len;
+}
+
+static const u32 ad9910_osk_raw_step[] = {
+	0,		/* no step: manual mode (NOT pin controlled) */
+	1,		/* step size 1: automatic mode (pin controlled) */
+	2,		/* step size 2: automatic mode (pin controlled) */
+	4,		/* step size 4: automatic mode (pin controlled) */
+	8,		/* step size 8: automatic mode (pin controlled) */
+	GENMASK(13, 0),	/* max step: manual mode (pin controlled) */
+};
+
+static ssize_t ad9910_osk_attrs_read(struct iio_dev *indio_dev,
+				     uintptr_t private,
+				     const struct iio_chan_spec *chan,
+				     char *buf)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	bool auto_en, pinctrl_en;
+	u32 rate, step;
+	u64 roc64;
+
+	guard(mutex)(&st->lock);
+
+	rate = FIELD_GET(AD9910_ASF_RAMP_RATE_MSK, st->reg[AD9910_REG_ASF].val32);
+	if (!rate)
+		return -ERANGE;
+
+	switch (private) {
+	case AD9910_ROC:
+		auto_en = FIELD_GET(AD9910_CFR1_SELECT_AUTO_OSK_MSK,
+				    st->reg[AD9910_REG_CFR1].val32);
+		pinctrl_en = FIELD_GET(AD9910_CFR1_OSK_MANUAL_EXT_CTL_MSK,
+				       st->reg[AD9910_REG_CFR1].val32);
+		if (auto_en) {
+			step = FIELD_GET(AD9910_ASF_STEP_SIZE_MSK,
+					 st->reg[AD9910_REG_ASF].val32);
+			step = ad9910_osk_raw_step[step + 1];
+		} else if (pinctrl_en) {
+			step = ad9910_osk_raw_step[ARRAY_SIZE(ad9910_osk_raw_step) - 1];
+		} else {
+			step = ad9910_osk_raw_step[0];
+		}
+
+		roc64 = div_u64((u64)step * st->data.sysclk_freq_hz, 4 * rate);
+		return sysfs_emit(buf, "%llu\n", roc64);
+	case AD9910_ROC_AVAIL: {
+		ssize_t len = 0;
+
+		for (unsigned int i = 0; i < ARRAY_SIZE(ad9910_osk_raw_step); i++) {
+			roc64 = div_u64((u64)ad9910_osk_raw_step[i] * st->data.sysclk_freq_hz,
+					4 * rate);
+			len += sysfs_emit_at(buf, len, "%llu ", roc64);
+		}
+
+		buf[len - 1] = '\n'; /* replace last space with a newline */
+		return len;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static ssize_t ad9910_osk_attrs_write(struct iio_dev *indio_dev,
+				      uintptr_t private,
+				      const struct iio_chan_spec *chan,
+				      const char *buf, size_t len)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	u32 idx, reg_val, rate;
+	u64 step;
+	int ret;
+
+	ret = kstrtou64(buf, 10, &step);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&st->lock);
+
+	rate = FIELD_GET(AD9910_ASF_RAMP_RATE_MSK, st->reg[AD9910_REG_ASF].val32);
+	if (!rate)
+		return -ERANGE;
+
+	switch (private) {
+	case AD9910_ROC:
+		step = ad9910_rational_scale(step, 4 * rate,
+					     st->data.sysclk_freq_hz);
+		step = min(step, AD9910_ASF_MAX);
+		idx = find_closest(step, ad9910_osk_raw_step,
+				   ARRAY_SIZE(ad9910_osk_raw_step));
+		if (idx == ARRAY_SIZE(ad9910_osk_raw_step) - 1) {
+			/* manual mode: pin-controlled */
+			reg_val = AD9910_CFR1_OSK_MANUAL_EXT_CTL_MSK;
+		} else if (idx == 0) {
+			/* manual mode, NOT pin-controlled */
+			reg_val = 0;
+		} else {
+			/* auto mode: pin-controlled */
+			reg_val = FIELD_PREP(AD9910_ASF_STEP_SIZE_MSK, idx - 1);
+			ret = ad9910_reg32_update(st, AD9910_REG_ASF,
+						  AD9910_ASF_STEP_SIZE_MSK,
+						  reg_val, false);
+			if (ret)
+				return ret;
+
+			reg_val = AD9910_CFR1_SELECT_AUTO_OSK_MSK;
+		}
+
+		ret = ad9910_reg32_update(st, AD9910_REG_CFR1,
+					  AD9910_CFR1_SELECT_AUTO_OSK_MSK |
+					  AD9910_CFR1_OSK_MANUAL_EXT_CTL_MSK,
+					  reg_val, true);
+		if (ret)
+			return ret;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return len;
+}
+
+static const struct iio_chan_spec_ext_info ad9910_phy_ext_info[] = {
+	{
+		.name = "powerdown",
+		.read = ad9910_ext_info_read,
+		.write = ad9910_ext_info_write,
+		.private = AD9910_POWERDOWN,
+		.shared = IIO_SEPARATE,
+	},
+	{ }
+};
+
+static const struct iio_chan_spec_ext_info ad9910_drg_ramp_ext_info[] = {
+	{
+		.name = "dwell_en",
+		.read = ad9910_ext_info_read,
+		.write = ad9910_ext_info_write,
+		.private = AD9910_DWELL_EN,
+		.shared = IIO_SEPARATE,
+	},
+	{
+		.name = "raw_roc",
+		.read = ad9910_drg_roc_read,
+		.write = ad9910_drg_roc_write,
+		.private = AD9910_ROC,
+		.shared = IIO_SEPARATE,
+	},
+	{ }
+};
+
+static const struct iio_chan_spec_ext_info ad9910_osk_ext_info[] = {
+	{
+		.name = "raw_roc",
+		.read = ad9910_osk_attrs_read,
+		.write = ad9910_osk_attrs_write,
+		.private = AD9910_ROC,
+		.shared = IIO_SEPARATE,
+	},
+	{
+		.name = "raw_roc_available",
+		.read = ad9910_osk_attrs_read,
+		.private = AD9910_ROC_AVAIL,
+		.shared = IIO_SEPARATE,
+	},
+	{ }
+};
+
+#define AD9910_PROFILE_CHAN(idx) {				\
+	.type = IIO_ALTCURRENT,					\
+	.indexed = 1,						\
+	.output = 1,						\
+	.channel = AD9910_CHANNEL_PROFILE_ ## idx,		\
+	.address = AD9910_CHAN_IDX_PROFILE_ ## idx,		\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_ENABLE) |	\
+			      BIT(IIO_CHAN_INFO_FREQUENCY) |	\
+			      BIT(IIO_CHAN_INFO_PHASE) |	\
+			      BIT(IIO_CHAN_INFO_RAW),		\
+	.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],	\
+}
+
+static const struct iio_chan_spec ad9910_channels[] = {
+	[AD9910_CHAN_IDX_PHY] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_PHY,
+		.address = AD9910_CHAN_IDX_PHY,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+		.ext_info = ad9910_phy_ext_info,
+	},
+	[AD9910_CHAN_IDX_PROFILE_0] = AD9910_PROFILE_CHAN(0),
+	[AD9910_CHAN_IDX_PROFILE_1] = AD9910_PROFILE_CHAN(1),
+	[AD9910_CHAN_IDX_PROFILE_2] = AD9910_PROFILE_CHAN(2),
+	[AD9910_CHAN_IDX_PROFILE_3] = AD9910_PROFILE_CHAN(3),
+	[AD9910_CHAN_IDX_PROFILE_4] = AD9910_PROFILE_CHAN(4),
+	[AD9910_CHAN_IDX_PROFILE_5] = AD9910_PROFILE_CHAN(5),
+	[AD9910_CHAN_IDX_PROFILE_6] = AD9910_PROFILE_CHAN(6),
+	[AD9910_CHAN_IDX_PROFILE_7] = AD9910_PROFILE_CHAN(7),
+	[AD9910_CHAN_IDX_PARALLEL_AMP] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_PARALLEL,
+		.address = AD9910_CHAN_IDX_PARALLEL_AMP,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_PARALLEL_PHASE] = {
+		.type = IIO_PHASE,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_PARALLEL,
+		.address = AD9910_CHAN_IDX_PARALLEL_PHASE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_SCALE),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_PARALLEL_FREQ] = {
+		.type = IIO_FREQUENCY,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_PARALLEL,
+		.address = AD9910_CHAN_IDX_PARALLEL_FREQ,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_PARALLEL_POLAR_AMP] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_PARALLEL_POLAR,
+		.address = AD9910_CHAN_IDX_PARALLEL_POLAR_AMP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_PARALLEL_POLAR_PHASE] = {
+		.type = IIO_PHASE,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_PARALLEL_POLAR,
+		.address = AD9910_CHAN_IDX_PARALLEL_POLAR_PHASE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_DRG_FREQ] = {
+		.type = IIO_FREQUENCY,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG,
+		.address = AD9910_CHAN_IDX_DRG_FREQ,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_ENABLE) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_DRG_PHASE] = {
+		.type = IIO_PHASE,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG,
+		.address = AD9910_CHAN_IDX_DRG_PHASE,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_ENABLE) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_DRG_AMP] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG,
+		.address = AD9910_CHAN_IDX_DRG_AMP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_ENABLE) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_DRG_FREQ_RAMP_UP] = {
+		.type = IIO_FREQUENCY,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG_RAMP_UP,
+		.address = AD9910_CHAN_IDX_DRG_FREQ_RAMP_UP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.ext_info = ad9910_drg_ramp_ext_info,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_DRG_FREQ],
+	},
+	[AD9910_CHAN_IDX_DRG_FREQ_RAMP_DOWN] = {
+		.type = IIO_FREQUENCY,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG_RAMP_DOWN,
+		.address = AD9910_CHAN_IDX_DRG_FREQ_RAMP_DOWN,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.ext_info = ad9910_drg_ramp_ext_info,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_DRG_FREQ],
+	},
+	[AD9910_CHAN_IDX_DRG_PHASE_RAMP_UP] = {
+		.type = IIO_PHASE,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG_RAMP_UP,
+		.address = AD9910_CHAN_IDX_DRG_PHASE_RAMP_UP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.ext_info = ad9910_drg_ramp_ext_info,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_DRG_PHASE],
+	},
+	[AD9910_CHAN_IDX_DRG_PHASE_RAMP_DOWN] = {
+		.type = IIO_PHASE,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG_RAMP_DOWN,
+		.address = AD9910_CHAN_IDX_DRG_PHASE_RAMP_DOWN,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.ext_info = ad9910_drg_ramp_ext_info,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_DRG_PHASE],
+	},
+	[AD9910_CHAN_IDX_DRG_AMP_RAMP_UP] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG_RAMP_UP,
+		.address = AD9910_CHAN_IDX_DRG_AMP_RAMP_UP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.ext_info = ad9910_drg_ramp_ext_info,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_DRG_AMP],
+	},
+	[AD9910_CHAN_IDX_DRG_AMP_RAMP_DOWN] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_DRG_RAMP_DOWN,
+		.address = AD9910_CHAN_IDX_DRG_AMP_RAMP_DOWN,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.ext_info = ad9910_drg_ramp_ext_info,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_DRG_AMP],
+	},
+	[AD9910_CHAN_IDX_RAM] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_RAM,
+		.address = AD9910_CHAN_IDX_RAM,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_ENABLE) |
+				      BIT(IIO_CHAN_INFO_FREQUENCY) |
+				      BIT(IIO_CHAN_INFO_PHASE) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+	[AD9910_CHAN_IDX_OSK] = {
+		.type = IIO_ALTCURRENT,
+		.indexed = 1,
+		.output = 1,
+		.channel = AD9910_CHANNEL_OSK,
+		.address = AD9910_CHAN_IDX_OSK,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_ENABLE) |
+				      BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.ext_info = ad9910_osk_ext_info,
+		.parent = &ad9910_channels[AD9910_CHAN_IDX_PHY],
+	},
+};
+
+static int ad9910_read_raw(struct iio_dev *indio_dev,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2, long info)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	u64 tmp64;
+	u32 tmp32;
+
+	guard(mutex)(&st->lock);
+
+	switch (info) {
+	case IIO_CHAN_INFO_ENABLE:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			if (ad9910_sw_powerdown_get(st)) {
+				*val = 0;
+			} else {
+				tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+				*val = (tmp32 == st->profile);
+			}
+			break;
+		case AD9910_CHANNEL_DRG:
+			tmp32 = FIELD_GET(AD9910_CFR2_DRG_DEST_MSK,
+					  st->reg[AD9910_REG_CFR2].val32);
+			if (tmp32 == (chan->address - AD9910_CHAN_IDX_DRG_FREQ))
+				*val = FIELD_GET(AD9910_CFR2_DRG_ENABLE_MSK,
+						 st->reg[AD9910_REG_CFR2].val32);
+			else
+				*val = 0;
+			break;
+		case AD9910_CHANNEL_RAM:
+			*val = FIELD_GET(AD9910_CFR1_RAM_ENABLE_MSK,
+					 st->reg[AD9910_REG_CFR1].val32);
+			break;
+		case AD9910_CHANNEL_OSK:
+			*val = FIELD_GET(AD9910_CFR1_OSK_ENABLE_MSK,
+					 st->reg[AD9910_REG_CFR1].val32);
+			break;
+		default:
+			return -EINVAL;
+		}
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_FREQUENCY:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+			tmp64 = FIELD_GET(AD9910_PROFILE_ST_FTW_MSK,
+					  ad9910_st_profile_val(st, tmp32));
+			break;
+		case AD9910_CHANNEL_RAM:
+			tmp64 = st->reg[AD9910_REG_FTW].val32;
+			break;
+		default:
+			return -EINVAL;
+		}
+		tmp64 *= st->data.sysclk_freq_hz;
+		*val = tmp64 >> 32;
+		*val2 = ((tmp64 & GENMASK_ULL(31, 0)) * MICRO) >> 32;
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_PHASE:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+			tmp64 = FIELD_GET(AD9910_PROFILE_ST_POW_MSK,
+					  ad9910_st_profile_val(st, tmp32));
+			break;
+		case AD9910_CHANNEL_RAM:
+			tmp64 = st->reg[AD9910_REG_POW].val16;
+			break;
+		default:
+			return -EINVAL;
+		}
+		tmp32 = (tmp64 * AD9910_MAX_PHASE_MICRORAD) >> 16;
+		*val = tmp32 / MICRO;
+		*val2 = tmp32 % MICRO;
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_RAW:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+			*val = FIELD_GET(AD9910_PROFILE_ST_ASF_MSK,
+					 ad9910_st_profile_val(st, tmp32));
+			return IIO_VAL_INT;
+		case AD9910_CHANNEL_DRG_RAMP_UP:
+			tmp64 = FIELD_GET(AD9910_DRG_LIMIT_UPPER_MSK,
+					  st->reg[AD9910_REG_DRG_LIMIT].val64);
+			iio_val_s64_decompose(tmp64, val, val2);
+			return IIO_VAL_INT_64;
+		case AD9910_CHANNEL_DRG_RAMP_DOWN:
+			tmp64 = FIELD_GET(AD9910_DRG_LIMIT_LOWER_MSK,
+					  st->reg[AD9910_REG_DRG_LIMIT].val64);
+			iio_val_s64_decompose(tmp64, val, val2);
+			return IIO_VAL_INT_64;
+		case AD9910_CHANNEL_OSK:
+			*val = FIELD_GET(AD9910_ASF_SCALE_FACTOR_MSK,
+					 st->reg[AD9910_REG_ASF].val32);
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PHY:
+			*val = st->data.sysclk_freq_hz;
+			return IIO_VAL_INT;
+		case AD9910_CHANNEL_DRG_RAMP_UP:
+			tmp32 = FIELD_GET(AD9910_DRG_RATE_INC_MSK,
+					  st->reg[AD9910_REG_DRG_RATE].val32);
+			break;
+		case AD9910_CHANNEL_DRG_RAMP_DOWN:
+			tmp32 = FIELD_GET(AD9910_DRG_RATE_DEC_MSK,
+					  st->reg[AD9910_REG_DRG_RATE].val32);
+			break;
+		case AD9910_CHANNEL_RAM:
+			tmp32 = FIELD_GET(AD9910_PROFILE_RAM_STEP_RATE_MSK,
+					  ad9910_ram_profile_val(st));
+			break;
+		case AD9910_CHANNEL_OSK:
+			tmp32 = FIELD_GET(AD9910_ASF_RAMP_RATE_MSK,
+					  st->reg[AD9910_REG_ASF].val32);
+			break;
+		default:
+			return -EINVAL;
+		}
+		if (!tmp32)
+			return -ERANGE;
+		tmp32 *= 4;
+		*val = st->data.sysclk_freq_hz / tmp32;
+		*val2 = div_u64((u64)(st->data.sysclk_freq_hz % tmp32) * MICRO, tmp32);
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->address) {
+		case AD9910_CHAN_IDX_PHY:
+			tmp64 = (u64)st->data.output_current_uA *
+				AD9910_NANO_MILLIAMP_PER_MICROAMP;
+			*val = 0;
+			*val2 = tmp64 >> 14;
+			return IIO_VAL_INT_PLUS_NANO;
+		case AD9910_CHAN_IDX_PARALLEL_PHASE:
+			*val = 0;
+			*val2 = AD9910_PI_NANORAD >> 15;
+			return IIO_VAL_INT_PLUS_NANO;
+		case AD9910_CHAN_IDX_PARALLEL_FREQ:
+			tmp32 = FIELD_GET(AD9910_CFR2_FM_GAIN_MSK,
+					  st->reg[AD9910_REG_CFR2].val32);
+			tmp64 = (u64)st->data.sysclk_freq_hz << tmp32;
+			tmp64 = ad9910_rational_scale(tmp64, NANO, BIT_ULL(32));
+			*val = div_s64_rem(tmp64, NANO, val2);
+			return IIO_VAL_INT_PLUS_NANO;
+		case AD9910_CHAN_IDX_PARALLEL_POLAR_AMP:
+			tmp64 = (u64)st->data.output_current_uA *
+				AD9910_NANO_MILLIAMP_PER_MICROAMP;
+			*val = 0;
+			*val2 = tmp64 >> 8;
+			return IIO_VAL_INT_PLUS_NANO;
+		case AD9910_CHAN_IDX_PARALLEL_POLAR_PHASE:
+			*val = 0;
+			*val2 = AD9910_PI_NANORAD >> 7;
+			return IIO_VAL_INT_PLUS_NANO;
+		case AD9910_CHAN_IDX_DRG_FREQ:
+			tmp64 = ad9910_rational_scale(st->data.sysclk_freq_hz,
+						      PICO, BIT_ULL(32));
+			iio_val_s64_decompose(tmp64, val, val2);
+			return IIO_VAL_DECIMAL64_PICO;
+		case AD9910_CHAN_IDX_DRG_PHASE:
+			tmp64 = DIV_U64_ROUND_CLOSEST(AD9910_PI_PICORAD, BIT(31));
+			iio_val_s64_decompose(tmp64, val, val2);
+			return IIO_VAL_DECIMAL64_PICO;
+		case AD9910_CHAN_IDX_DRG_AMP:
+			tmp64 = (u64)st->data.output_current_uA *
+				AD9910_PICO_MILLIAMP_PER_MICROAMP;
+			tmp64 = DIV_U64_ROUND_CLOSEST(tmp64 >> 1, BIT(31));
+			iio_val_s64_decompose(tmp64, val, val2);
+			return IIO_VAL_DECIMAL64_PICO;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_OFFSET:
+		switch (chan->address) {
+		case AD9910_CHAN_IDX_PARALLEL_FREQ:
+			tmp64 = (u64)st->reg[AD9910_REG_FTW].val32 * MICRO;
+			tmp64 >>= FIELD_GET(AD9910_CFR2_FM_GAIN_MSK,
+					    st->reg[AD9910_REG_CFR2].val32);
+			iio_val_s64_decompose(tmp64, val, val2);
+			return IIO_VAL_DECIMAL64_MICRO;
+		case AD9910_CHAN_IDX_PARALLEL_POLAR_AMP:
+			tmp32 = FIELD_GET(AD9910_ASF_SCALE_FACTOR_PP_LSB_MSK,
+					  st->reg[AD9910_REG_ASF].val32);
+			iio_val_s64_decompose(MICRO * tmp32 >> 6, val, val2);
+			return IIO_VAL_DECIMAL64_MICRO;
+		case AD9910_CHAN_IDX_PARALLEL_POLAR_PHASE:
+			tmp32 = FIELD_GET(AD9910_POW_PP_LSB_MSK,
+					  st->reg[AD9910_REG_POW].val16);
+			iio_val_s64_decompose(MICRO * tmp32 >> 8, val, val2);
+			return IIO_VAL_DECIMAL64_MICRO;
+		default:
+			return -EINVAL;
+		}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad9910_write_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int val, int val2, long info)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	u64 tmp64;
+	u32 tmp32;
+	int ret, i;
+
+	guard(mutex)(&st->lock);
+
+	switch (info) {
+	case IIO_CHAN_INFO_ENABLE:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+			if (!val) {
+				if (tmp32 != st->profile)
+					return 0; /* nothing to do */
+
+				return ad9910_sw_powerdown_set(st, true);
+			}
+
+			ret = ad9910_sw_powerdown_set(st, false);
+			if (ret)
+				return ret;
+
+			return ad9910_profile_set(st, tmp32);
+		case AD9910_CHANNEL_DRG:
+			tmp32 = chan->address - AD9910_CHAN_IDX_DRG_FREQ;
+			if (val) {
+				tmp32 = AD9910_CFR2_DRG_ENABLE_MSK |
+					FIELD_PREP(AD9910_CFR2_DRG_DEST_MSK, tmp32);
+			} else {
+				if (tmp32 != FIELD_GET(AD9910_CFR2_DRG_DEST_MSK,
+						       st->reg[AD9910_REG_CFR2].val32))
+					return 0; /* nothing to do */
+				tmp32 = 0;
+			}
+
+			return ad9910_reg32_update(st, AD9910_REG_CFR2,
+						   AD9910_CFR2_DRG_DEST_MSK |
+						   AD9910_CFR2_DRG_ENABLE_MSK,
+						   tmp32, true);
+		case AD9910_CHANNEL_RAM:
+			if (AD9910_RAM_ENABLED(st) == !!val)
+				return 0;
+
+			/* swap profile configs */
+			for (i = 0; i < AD9910_NUM_PROFILES; i++) {
+				tmp64 = st->reg[AD9910_REG_PROFILE(i)].val64;
+				ret = ad9910_reg64_write(st,
+							 AD9910_REG_PROFILE(i),
+							 st->reg_profile[i],
+							 false);
+				if (ret)
+					break;
+				st->reg_profile[i] = tmp64;
+			}
+
+			if (ret) {
+				/*
+				 * After the write failure, profiles 0..i-1 were
+				 * already swapped in SW, but Hw registers are
+				 * still pending an IO update, so swap them back
+				 * in SW to keep the state consistent.
+				 */
+				while (i--) {
+					tmp64 = st->reg[AD9910_REG_PROFILE(i)].val64;
+					st->reg[AD9910_REG_PROFILE(i)].val64 = st->reg_profile[i];
+					st->reg_profile[i] = tmp64;
+				}
+				return ret;
+			}
+
+			tmp32 = FIELD_PREP(AD9910_CFR1_RAM_ENABLE_MSK, !!val);
+			return ad9910_reg32_update(st, AD9910_REG_CFR1,
+						   AD9910_CFR1_RAM_ENABLE_MSK,
+						   tmp32, true);
+		case AD9910_CHANNEL_OSK:
+			tmp32 = FIELD_PREP(AD9910_CFR1_OSK_ENABLE_MSK, !!val);
+			return ad9910_reg32_update(st, AD9910_REG_CFR1,
+						   AD9910_CFR1_OSK_ENABLE_MSK,
+						   tmp32, true);
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_FREQUENCY:
+		if (val < 0 || val2 < 0 || val >= st->data.sysclk_freq_hz / 2)
+			return -EINVAL;
+
+		tmp64 = ad9910_rational_scale((u64)val * MICRO + val2, BIT_ULL(32),
+					      (u64)MICRO * st->data.sysclk_freq_hz);
+		tmp64 = min_t(u64, tmp64, U32_MAX);
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+			if (AD9910_RAM_ENABLED(st)) {
+				FIELD_MODIFY(AD9910_PROFILE_ST_FTW_MSK,
+					     &st->reg_profile[tmp32], tmp64);
+				return 0;
+			}
+			tmp64 = FIELD_PREP(AD9910_PROFILE_ST_FTW_MSK, tmp64);
+			return ad9910_reg64_update(st, AD9910_REG_PROFILE(tmp32),
+						   AD9910_PROFILE_ST_FTW_MSK,
+						   tmp64, true);
+		case AD9910_CHANNEL_RAM:
+			return ad9910_reg32_write(st, AD9910_REG_FTW, tmp64, true);
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_PHASE:
+		if (val < 0 || val2 < 0)
+			return -EINVAL;
+
+		tmp64 = (u64)val * MICRO + val2;
+		if (tmp64 >= AD9910_MAX_PHASE_MICRORAD)
+			return -EINVAL;
+
+		tmp64 = DIV_U64_ROUND_CLOSEST(tmp64 << 16, AD9910_MAX_PHASE_MICRORAD);
+		tmp64 = min(tmp64, AD9910_POW_MAX);
+
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+
+			if (AD9910_RAM_ENABLED(st)) {
+				FIELD_MODIFY(AD9910_PROFILE_ST_POW_MSK,
+					     &st->reg_profile[tmp32], tmp64);
+				return 0;
+			}
+
+			tmp64 = FIELD_PREP(AD9910_PROFILE_ST_POW_MSK, tmp64);
+			return ad9910_reg64_update(st, AD9910_REG_PROFILE(tmp32),
+						   AD9910_PROFILE_ST_POW_MSK,
+						   tmp64, true);
+		case AD9910_CHANNEL_RAM:
+			return ad9910_reg16_write(st, AD9910_REG_POW, tmp64, true);
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_RAW:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+			if (val < 0)
+				return -EINVAL;
+
+			tmp32 = chan->channel - AD9910_CHANNEL_PROFILE_0;
+			tmp64 = min_t(u64, val, AD9910_ASF_MAX);
+
+			if (AD9910_RAM_ENABLED(st)) {
+				FIELD_MODIFY(AD9910_PROFILE_ST_ASF_MSK,
+					     &st->reg_profile[tmp32], tmp64);
+				return 0;
+			}
+
+			tmp64 = FIELD_PREP(AD9910_PROFILE_ST_ASF_MSK, tmp64);
+			return ad9910_reg64_update(st, AD9910_REG_PROFILE(tmp32),
+						   AD9910_PROFILE_ST_ASF_MSK,
+						   tmp64, true);
+		case AD9910_CHANNEL_DRG_RAMP_UP: {
+			s64 val64 = iio_val_s64_compose(val, val2);
+
+			if (val64 < 0)
+				return -EINVAL;
+
+			tmp64 = min_t(u64, val64, U32_MAX);
+			tmp64 = FIELD_PREP(AD9910_DRG_LIMIT_UPPER_MSK, tmp64);
+			return ad9910_reg64_update(st, AD9910_REG_DRG_LIMIT,
+						   AD9910_DRG_LIMIT_UPPER_MSK,
+						   tmp64, true);
+		}
+		case AD9910_CHANNEL_DRG_RAMP_DOWN: {
+			s64 val64 = iio_val_s64_compose(val, val2);
+
+			if (val64 < 0)
+				return -EINVAL;
+
+			tmp64 = min_t(u64, val64, U32_MAX);
+			tmp64 = FIELD_PREP(AD9910_DRG_LIMIT_LOWER_MSK, tmp64);
+			return ad9910_reg64_update(st, AD9910_REG_DRG_LIMIT,
+						   AD9910_DRG_LIMIT_LOWER_MSK,
+						   tmp64, true);
+		}
+		case AD9910_CHANNEL_OSK:
+			if (val < 0)
+				return -EINVAL;
+
+			tmp32 = min_t(u32, val, AD9910_ASF_MAX);
+			tmp32 = FIELD_PREP(AD9910_ASF_SCALE_FACTOR_MSK, tmp32);
+			return ad9910_reg32_update(st, AD9910_REG_ASF,
+						   AD9910_ASF_SCALE_FACTOR_MSK,
+						   tmp32, true);
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (chan->channel == AD9910_CHANNEL_PHY)
+			return ad9910_set_sysclk_freq(st, val, true);
+
+		if (val < 0 || val2 < 0 || val > st->data.sysclk_freq_hz / 4)
+			return -EINVAL;
+
+		tmp64 = ((u64)val * MICRO + val2) * 4;
+		if (!tmp64)
+			return -EINVAL;
+
+		tmp64 = DIV64_U64_ROUND_CLOSEST((u64)st->data.sysclk_freq_hz * MICRO, tmp64);
+		tmp32 = clamp(tmp64, 1U, AD9910_STEP_RATE_MAX);
+
+		switch (chan->channel) {
+		case AD9910_CHANNEL_DRG_RAMP_UP:
+			tmp32 = FIELD_PREP(AD9910_DRG_RATE_INC_MSK, tmp32);
+			return ad9910_reg32_update(st, AD9910_REG_DRG_RATE,
+						   AD9910_DRG_RATE_INC_MSK,
+						   tmp32, true);
+		case AD9910_CHANNEL_DRG_RAMP_DOWN:
+			tmp32 = FIELD_PREP(AD9910_DRG_RATE_DEC_MSK, tmp32);
+			return ad9910_reg32_update(st, AD9910_REG_DRG_RATE,
+						   AD9910_DRG_RATE_DEC_MSK,
+						   tmp32, true);
+		case AD9910_CHANNEL_RAM:
+			if (!AD9910_RAM_ENABLED(st)) {
+				FIELD_MODIFY(AD9910_PROFILE_RAM_STEP_RATE_MSK,
+					     &st->reg_profile[st->profile], tmp32);
+				return 0;
+			}
+
+			tmp64 = FIELD_PREP(AD9910_PROFILE_RAM_STEP_RATE_MSK, tmp32);
+			return ad9910_reg64_update(st, AD9910_REG_PROFILE(st->profile),
+						   AD9910_PROFILE_RAM_STEP_RATE_MSK,
+						   tmp64, true);
+		case AD9910_CHANNEL_OSK:
+			return ad9910_reg32_update(st, AD9910_REG_ASF,
+						   AD9910_ASF_RAMP_RATE_MSK,
+						   FIELD_PREP(AD9910_ASF_RAMP_RATE_MSK, tmp32),
+						   true);
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->address) {
+		case AD9910_CHAN_IDX_PHY:
+			if (val != 0 || val2 < 0)
+				return -EINVAL;
+
+			tmp32 = DIV_U64_ROUND_CLOSEST((u64)val2 << 14,
+						      AD9910_NANO_MILLIAMP_PER_MICROAMP);
+			return ad9910_set_dac_current(st, tmp32, true);
+		case AD9910_CHAN_IDX_PARALLEL_FREQ:
+			if (val < 0 || val2 < 0)
+				return -EINVAL;
+
+			tmp64 = ad9910_rational_scale((u64)val * NANO + val2, BIT_ULL(32),
+						      (u64)st->data.sysclk_freq_hz * NANO);
+			tmp64 = roundup_pow_of_two(clamp(tmp64, 1ULL, BIT_ULL(15)));
+			tmp32 = FIELD_PREP(AD9910_CFR2_FM_GAIN_MSK, ilog2(tmp64));
+			return ad9910_reg32_update(st, AD9910_REG_CFR2,
+						   AD9910_CFR2_FM_GAIN_MSK,
+						   tmp32, true);
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_OFFSET: {
+		s64 val64 = iio_val_s64_compose(val, val2);
+
+		if (val64 < 0)
+			return -EINVAL;
+
+		tmp64 = val64;
+		switch (chan->address) {
+		case AD9910_CHAN_IDX_PARALLEL_FREQ:
+			tmp32 = BIT(FIELD_GET(AD9910_CFR2_FM_GAIN_MSK,
+					      st->reg[AD9910_REG_CFR2].val32));
+			tmp64 = ad9910_rational_scale(tmp64, tmp32, MICRO);
+			tmp64 = min_t(u64, tmp64, U32_MAX);
+			return ad9910_reg32_write(st, AD9910_REG_FTW, tmp64, true);
+		case AD9910_CHAN_IDX_PARALLEL_POLAR_AMP:
+			tmp64 = ad9910_rational_scale(tmp64, BIT(6), MICRO);
+			tmp64 = min_t(u64, tmp64, AD9910_ASF_PP_LSB_MAX);
+			tmp32 = FIELD_PREP(AD9910_ASF_SCALE_FACTOR_PP_LSB_MSK, tmp64);
+			return ad9910_reg32_update(st, AD9910_REG_ASF,
+						   AD9910_ASF_SCALE_FACTOR_PP_LSB_MSK,
+						   tmp32, true);
+		case AD9910_CHAN_IDX_PARALLEL_POLAR_PHASE:
+			tmp64 = ad9910_rational_scale(tmp64, BIT(8), MICRO);
+			tmp64 = min_t(u64, tmp64, AD9910_POW_PP_LSB_MAX);
+			tmp32 = FIELD_PREP(AD9910_POW_PP_LSB_MSK, tmp64);
+			return ad9910_reg16_update(st, AD9910_REG_POW,
+						   AD9910_POW_PP_LSB_MSK,
+						   tmp32, true);
+		default:
+			return -EINVAL;
+		}
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad9910_write_raw_get_fmt(struct iio_dev *indio_dev,
+				    struct iio_chan_spec const *chan,
+				    long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_ENABLE:
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_FREQUENCY:
+	case IIO_CHAN_INFO_PHASE:
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_RAW:
+		switch (chan->channel) {
+		case AD9910_CHANNEL_PROFILE_0 ... AD9910_CHANNEL_PROFILE_7:
+		case AD9910_CHANNEL_OSK:
+			return IIO_VAL_INT;
+		case AD9910_CHANNEL_DRG_RAMP_UP:
+		case AD9910_CHANNEL_DRG_RAMP_DOWN:
+			return IIO_VAL_INT_64;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (chan->channel == AD9910_CHANNEL_PHY)
+			return IIO_VAL_INT;
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_SCALE:
+		return IIO_VAL_INT_PLUS_NANO;
+	case IIO_CHAN_INFO_OFFSET:
+		return IIO_VAL_DECIMAL64_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad9910_debugfs_reg_access(struct iio_dev *indio_dev,
+				     unsigned int reg, u64 writeval,
+				     u64 *readval)
+{
+	struct ad9910_state *st = iio_priv(indio_dev);
+	union ad9910_reg tmp;
+	int ret;
+
+	if (reg >= AD9910_REG_RAM)
+		return -EINVAL;
+
+	guard(mutex)(&st->lock);
+
+	switch (reg) {
+	case AD9910_REG_DRG_LIMIT:
+	case AD9910_REG_DRG_STEP:
+	case AD9910_REG_PROFILE0 ... AD9910_REG_PROFILE7:
+		if (!readval)
+			return ad9910_reg64_write(st, reg, writeval, true);
+
+		ret = ad9910_reg64_read(st, reg, &tmp.val64);
+		if (ret)
+			return ret;
+		*readval = tmp.val64;
+		return 0;
+	case AD9910_REG_POW:
+		if (!readval)
+			return ad9910_reg16_write(st, reg, writeval, true);
+
+		ret = ad9910_reg16_read(st, reg, &tmp.val16);
+		if (ret)
+			return ret;
+		*readval = tmp.val16;
+		return 0;
+	default:
+		if (!readval)
+			return ad9910_reg32_write(st, reg, writeval, true);
+
+		ret = ad9910_reg32_read(st, reg, &tmp.val32);
+		if (ret)
+			return ret;
+		*readval = tmp.val32;
+		return 0;
+	}
+}
+
+static const char * const ad9910_channel_str[] = {
+	[AD9910_CHAN_IDX_PHY] = "phy",
+	[AD9910_CHAN_IDX_PROFILE_0] = "profile0",
+	[AD9910_CHAN_IDX_PROFILE_1] = "profile1",
+	[AD9910_CHAN_IDX_PROFILE_2] = "profile2",
+	[AD9910_CHAN_IDX_PROFILE_3] = "profile3",
+	[AD9910_CHAN_IDX_PROFILE_4] = "profile4",
+	[AD9910_CHAN_IDX_PROFILE_5] = "profile5",
+	[AD9910_CHAN_IDX_PROFILE_6] = "profile6",
+	[AD9910_CHAN_IDX_PROFILE_7] = "profile7",
+	[AD9910_CHAN_IDX_PARALLEL_AMP] = "parallel_amplitude",
+	[AD9910_CHAN_IDX_PARALLEL_PHASE] = "parallel_phase",
+	[AD9910_CHAN_IDX_PARALLEL_FREQ] = "parallel_frequency",
+	[AD9910_CHAN_IDX_PARALLEL_POLAR_AMP] = "parallel_polar_amplitude",
+	[AD9910_CHAN_IDX_PARALLEL_POLAR_PHASE] = "parallel_polar_phase",
+	[AD9910_CHAN_IDX_DRG_FREQ] = "drg_frequency",
+	[AD9910_CHAN_IDX_DRG_PHASE] = "drg_phase",
+	[AD9910_CHAN_IDX_DRG_AMP] = "drg_amplitude",
+	[AD9910_CHAN_IDX_DRG_FREQ_RAMP_UP] = "drg_rising_frequency",
+	[AD9910_CHAN_IDX_DRG_FREQ_RAMP_DOWN] = "drg_falling_frequency",
+	[AD9910_CHAN_IDX_DRG_PHASE_RAMP_UP] = "drg_rising_phase",
+	[AD9910_CHAN_IDX_DRG_PHASE_RAMP_DOWN] = "drg_falling_phase",
+	[AD9910_CHAN_IDX_DRG_AMP_RAMP_UP] = "drg_rising_amplitude",
+	[AD9910_CHAN_IDX_DRG_AMP_RAMP_DOWN] = "drg_falling_amplitude",
+	[AD9910_CHAN_IDX_RAM] = "ram",
+	[AD9910_CHAN_IDX_OSK] = "osk",
+};
+
+static int ad9910_read_label(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     char *label)
+{
+	return sysfs_emit(label, "%s\n", ad9910_channel_str[chan->address]);
+}
+
+static enum fw_upload_err ad9910_ram_fwu_prepare(struct fw_upload *fw_upload,
+						 const u8 *data, u32 size)
+{
+	struct ad9910_state *st = fw_upload->dd_handle;
+	const struct ad9910_ram_fw *fw_data = (const struct ad9910_ram_fw *)data;
+	size_t wcount, bcount;
+
+	scoped_guard(mutex, &st->lock) {
+		/* cancel requests before prepare() are ignored */
+		st->ram_fwu_cancel = false;
+	}
+
+	if (size < sizeof(struct ad9910_ram_fw))
+		return FW_UPLOAD_ERR_INVALID_SIZE;
+
+	if (get_unaligned_be32(&fw_data->magic) != AD9910_RAM_FW_MAGIC)
+		return FW_UPLOAD_ERR_FW_INVALID;
+
+	if (get_unaligned_be16(&fw_data->version) != AD9910_RAM_FW_V1)
+		return FW_UPLOAD_ERR_FW_INVALID;
+
+	wcount = get_unaligned_be16(&fw_data->wcount);
+	bcount = size - sizeof(struct ad9910_ram_fw);
+	if (wcount > AD9910_RAM_SIZE_MAX_WORDS ||
+	    bcount != (wcount * AD9910_RAM_WORD_SIZE))
+		return FW_UPLOAD_ERR_INVALID_SIZE;
+
+	bcount += sizeof(fw_data->cfr1) + sizeof(fw_data->profiles);
+	if (crc32(0, &fw_data->cfr1, bcount) != get_unaligned_be32(&fw_data->crc))
+		return FW_UPLOAD_ERR_FW_INVALID;
+
+	return FW_UPLOAD_ERR_NONE;
+}
+
+static enum fw_upload_err ad9910_ram_fwu_write(struct fw_upload *fw_upload,
+					       const u8 *data, u32 offset,
+					       u32 size, u32 *written)
+{
+	const struct ad9910_ram_fw *fw_data = (const struct ad9910_ram_fw *)data;
+	struct ad9910_state *st = fw_upload->dd_handle;
+	int ret, ret2, idx, wcount;
+	u64 tmp64, backup;
+
+	if (offset != 0)
+		return FW_UPLOAD_ERR_INVALID_SIZE;
+
+	guard(mutex)(&st->lock);
+
+	if (st->ram_fwu_cancel)
+		return FW_UPLOAD_ERR_CANCELED;
+
+	if (AD9910_RAM_ENABLED(st))
+		return FW_UPLOAD_ERR_HW_ERROR;
+
+	for (idx = 0; idx < AD9910_NUM_PROFILES; idx++)
+		st->reg_profile[idx] = get_unaligned_be64(&fw_data->profiles[idx]) |
+				       AD9910_PROFILE_RAM_OPEN_MSK;
+
+	ret = ad9910_reg32_update(st, AD9910_REG_CFR1,
+				  AD9910_CFR1_RAM_PLAYBACK_DEST_MSK |
+				  AD9910_CFR1_INT_PROFILE_CTL_MSK,
+				  get_unaligned_be32(&fw_data->cfr1), true);
+	if (ret)
+		return FW_UPLOAD_ERR_RW_ERROR;
+
+	wcount = get_unaligned_be16(&fw_data->wcount);
+	if (!wcount) {
+		*written = size;
+		return FW_UPLOAD_ERR_NONE; /* nothing else to write */
+	}
+
+	ret = ad9910_profile_set(st, st->profile);
+	if (ret)
+		return FW_UPLOAD_ERR_HW_ERROR;
+
+	/* backup profile register and update it with required address range */
+	backup = st->reg[AD9910_REG_PROFILE(st->profile)].val64;
+	tmp64 = AD9910_PROFILE_RAM_STEP_RATE_MSK |
+		FIELD_PREP(AD9910_PROFILE_RAM_START_ADDR_MSK, 0) |
+		FIELD_PREP(AD9910_PROFILE_RAM_END_ADDR_MSK, wcount - 1);
+	ret = ad9910_reg64_write(st, AD9910_REG_PROFILE(st->profile), tmp64, true);
+	if (ret)
+		return FW_UPLOAD_ERR_RW_ERROR;
+
+	/* copy data words into spi buffer and transfer */
+	memcpy(&st->tx_buf[1], fw_data->words, wcount * AD9910_RAM_WORD_SIZE);
+	ret = ad9910_spi_write(st, AD9910_REG_RAM,
+			       wcount * AD9910_RAM_WORD_SIZE, false);
+
+	/* restore active single tone profile regardless */
+	st->reg[AD9910_REG_PROFILE(st->profile)].val64 = backup;
+	ret2 = ad9910_reg64_write(st, AD9910_REG_PROFILE(st->profile), backup, true);
+	if (ret || ret2)
+		return FW_UPLOAD_ERR_RW_ERROR;
+
+	*written = size;
+
+	return FW_UPLOAD_ERR_NONE;
+}
+
+static enum fw_upload_err ad9910_ram_fwu_poll_complete(struct fw_upload *fw_upload)
+{
+	return FW_UPLOAD_ERR_NONE;
+}
+
+static void ad9910_ram_fwu_cancel(struct fw_upload *fw_upload)
+{
+	struct ad9910_state *st = fw_upload->dd_handle;
+
+	guard(mutex)(&st->lock);
+	st->ram_fwu_cancel = true;
+}
+
+static void ad9910_ram_fwu_unregister(void *data)
+{
+	firmware_upload_unregister(data);
+}
+
+static const struct fw_upload_ops ad9910_ram_fwu_ops = {
+	.prepare = ad9910_ram_fwu_prepare,
+	.write = ad9910_ram_fwu_write,
+	.poll_complete = ad9910_ram_fwu_poll_complete,
+	.cancel = ad9910_ram_fwu_cancel,
+};
+
+static const struct iio_info ad9910_info = {
+	.read_raw = ad9910_read_raw,
+	.write_raw = ad9910_write_raw,
+	.write_raw_get_fmt = ad9910_write_raw_get_fmt,
+	.read_label = ad9910_read_label,
+	.debugfs_reg64_access = &ad9910_debugfs_reg_access,
+};
+
+static int ad9910_cfg_sysclk(struct ad9910_state *st, bool update)
+{
+	u32 cfr3 = AD9910_CFR3_OPEN_MSK;
+	u32 tmp32;
+
+	cfr3 |= FIELD_PREP(AD9910_CFR3_DRV0_MSK, st->data.refclk_out_drv);
+
+	if (st->data.pll_enabled) {
+		tmp32 = st->data.pll_charge_pump_current - AD9910_ICP_MIN_uA;
+		tmp32 = DIV_ROUND_CLOSEST(tmp32, AD9910_ICP_STEP_uA);
+		cfr3 |= FIELD_PREP(AD9910_CFR3_ICP_MSK, tmp32) |
+			AD9910_CFR3_PLL_EN_MSK;
+	} else {
+		cfr3 |= AD9910_CFR3_ICP_MSK |
+			AD9910_CFR3_REFCLK_DIV_RESETB_MSK |
+			AD9910_CFR3_PFD_RESET_MSK;
+	}
+	st->reg[AD9910_REG_CFR3].val32 = cfr3;
+
+	return ad9910_set_sysclk_freq(st, AD9910_MAX_SYSCLK_HZ, update);
+}
+
+static int ad9910_parse_fw(struct ad9910_state *st)
+{
+	static const char * const refclk_out_drv0[] = {
+		"disabled", "low", "medium", "high",
+	};
+	struct device *dev = &st->spi->dev;
+	const char *prop;
+	u32 tmp;
+	int ret;
+
+	st->data.pll_enabled = device_property_read_bool(dev, "adi,pll-enable");
+	if (st->data.pll_enabled) {
+		prop = "adi,charge-pump-current-microamp";
+		if (device_property_present(dev, prop)) {
+			ret = device_property_read_u32(dev, prop, &tmp);
+			if (ret)
+				return dev_err_probe(dev, ret, "property read: %s\n", prop);
+
+			if (tmp < AD9910_ICP_MIN_uA || tmp > AD9910_ICP_MAX_uA)
+				return dev_err_probe(dev, -ERANGE,
+						     "invalid charge pump current %u\n", tmp);
+		} else {
+			tmp = AD9910_ICP_MIN_uA;
+		}
+		st->data.pll_charge_pump_current = tmp;
+
+		prop = "adi,refclk-out-drive-strength";
+		if (device_property_present(dev, prop)) {
+			ret = device_property_match_property_string(dev, prop,
+								    refclk_out_drv0,
+								    ARRAY_SIZE(refclk_out_drv0));
+			if (ret < 0)
+				return dev_err_probe(dev, ret, "property read: %s\n", prop);
+
+			st->data.refclk_out_drv = ret;
+		}
+	}
+
+	return 0;
+}
+
+static void ad9910_sw_powerdown_action(void *data)
+{
+	ad9910_sw_powerdown_set(data, true);
+}
+
+static void ad9910_hw_powerdown_action(void *data)
+{
+	struct ad9910_state *st = data;
+
+	gpiod_set_value_cansleep(st->gpio_pwdown, 1);
+}
+
+static int ad9910_setup(struct device *dev, struct ad9910_state *st,
+			struct reset_control *dev_rst)
+{
+	int ret;
+
+	ret = reset_control_assert(dev_rst);
+	if (ret)
+		return ret;
+
+	fsleep(AD9910_RESET_DELAY_us);
+
+	ret = reset_control_deassert(dev_rst);
+	if (ret)
+		return ret;
+
+	ret = ad9910_reg32_write(st, AD9910_REG_CFR1,
+				 (st->spi->mode & SPI_3WIRE ? 0 :
+				 AD9910_CFR1_SDIO_INPUT_ONLY_MSK), false);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, ad9910_sw_powerdown_action, st);
+	if (ret)
+		return ret;
+
+	ret = ad9910_reg32_write(st, AD9910_REG_CFR2,
+				 AD9910_CFR2_AMP_SCALE_SINGLE_TONE_MSK |
+				 AD9910_CFR2_SYNC_TIMING_VAL_DISABLE_MSK |
+				 AD9910_CFR2_DRG_NO_DWELL_MSK |
+				 AD9910_CFR2_DATA_ASM_HOLD_LAST_MSK |
+				 AD9910_CFR2_SYNC_CLK_EN_MSK |
+				 AD9910_CFR2_PDCLK_ENABLE_MSK, false);
+	if (ret)
+		return ret;
+
+	ret = ad9910_cfg_sysclk(st, false);
+	if (ret)
+		return ret;
+
+	ret = ad9910_set_dac_current(st, AD9910_DAC_IOUT_DEFAULT_uA, false);
+	if (ret)
+		return ret;
+
+	/* configure step rate with default values */
+	ret = ad9910_reg32_write(st, AD9910_REG_ASF,
+				 FIELD_PREP(AD9910_ASF_RAMP_RATE_MSK, 1),
+				 false);
+	if (ret)
+		return ret;
+
+	ret = ad9910_reg32_write(st, AD9910_REG_DRG_RATE,
+				 FIELD_PREP(AD9910_DRG_RATE_DEC_MSK, 1) |
+				 FIELD_PREP(AD9910_DRG_RATE_INC_MSK, 1),
+				 false);
+	if (ret)
+		return ret;
+
+	for (unsigned int i = 0; i < AD9910_NUM_PROFILES; i++) {
+		st->reg_profile[i] = AD9910_PROFILE_RAM_OPEN_MSK;
+		st->reg_profile[i] |= FIELD_PREP(AD9910_PROFILE_RAM_STEP_RATE_MSK, 1);
+		st->reg_profile[i] |= FIELD_PREP(AD9910_PROFILE_RAM_END_ADDR_MSK,
+						 AD9910_RAM_ADDR_MAX);
+	}
+
+	return ad9910_io_update(st);
+}
+
+static inline const char *ad9910_frequency_source_get(struct ad9910_state *st)
+{
+	bool ram_en, mode_en;
+
+	guard(mutex)(&st->lock);
+
+	/* RAM enabled and data destination is frequency */
+	ram_en = AD9910_RAM_ENABLED(st);
+	if (ram_en && AD9910_DEST_FREQUENCY ==
+		      FIELD_GET(AD9910_CFR1_RAM_PLAYBACK_DEST_MSK,
+				st->reg[AD9910_REG_CFR1].val32))
+		return ad9910_channel_str[AD9910_CHAN_IDX_RAM];
+
+	/* DRG enabled and data destination is frequency */
+	mode_en = FIELD_GET(AD9910_CFR2_DRG_ENABLE_MSK,
+			    st->reg[AD9910_REG_CFR2].val32);
+	if (mode_en && AD9910_DEST_FREQUENCY ==
+		       FIELD_GET(AD9910_CFR2_DRG_DEST_MSK,
+				 st->reg[AD9910_REG_CFR2].val32))
+		return ad9910_channel_str[AD9910_CHAN_IDX_DRG_FREQ];
+
+	/* FTW: RAM enabled and data destination is phase, amplitude, or polar */
+	if (ram_en)
+		return ad9910_channel_str[AD9910_CHAN_IDX_RAM];
+
+	/* single tone profiles */
+	return ad9910_channel_str[AD9910_CHAN_IDX_PROFILE_0 + st->profile];
+}
+
+static int ad9910_frequency_source_show(struct seq_file *s, void *ignored)
+{
+	seq_printf(s, "%s\n", ad9910_frequency_source_get(s->private));
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(ad9910_frequency_source);
+
+static inline const char *ad9910_phase_source_get(struct ad9910_state *st)
+{
+	bool ram_en, mode_en;
+	u32 destination;
+
+	guard(mutex)(&st->lock);
+
+	/* RAM enabled and data destination is phase or polar  */
+	ram_en = AD9910_RAM_ENABLED(st);
+	if (ram_en) {
+		destination = FIELD_GET(AD9910_CFR1_RAM_PLAYBACK_DEST_MSK,
+					st->reg[AD9910_REG_CFR1].val32);
+		if (destination == AD9910_DEST_PHASE ||
+		    destination == AD9910_DEST_POLAR)
+			return ad9910_channel_str[AD9910_CHAN_IDX_RAM];
+	}
+
+	/* DRG enabled and data destination is phase */
+	mode_en = FIELD_GET(AD9910_CFR2_DRG_ENABLE_MSK,
+			    st->reg[AD9910_REG_CFR2].val32);
+	if (mode_en && AD9910_DEST_PHASE ==
+		       FIELD_GET(AD9910_CFR2_DRG_DEST_MSK,
+				 st->reg[AD9910_REG_CFR2].val32))
+		return ad9910_channel_str[AD9910_CHAN_IDX_DRG_PHASE];
+
+	/* POW: RAM enabled and data destination is frequency or amplitude */
+	if (ram_en)
+		return ad9910_channel_str[AD9910_CHAN_IDX_RAM];
+
+	/* single tone profiles */
+	return ad9910_channel_str[AD9910_CHAN_IDX_PROFILE_0 + st->profile];
+}
+
+static int ad9910_phase_source_show(struct seq_file *s, void *ignored)
+{
+	seq_printf(s, "%s\n", ad9910_phase_source_get(s->private));
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(ad9910_phase_source);
+
+static inline const char *ad9910_amplitude_source_get(struct ad9910_state *st)
+{
+	bool ram_en, mode_en;
+	u32 destination;
+
+	guard(mutex)(&st->lock);
+
+	/* OSK enabled */
+	mode_en = FIELD_GET(AD9910_CFR1_OSK_ENABLE_MSK,
+			    st->reg[AD9910_REG_CFR1].val32);
+	if (mode_en)
+		return ad9910_channel_str[AD9910_CHAN_IDX_OSK];
+
+	/* RAM enabled and data destination is amplitude or polar */
+	ram_en = AD9910_RAM_ENABLED(st);
+	if (ram_en) {
+		destination = FIELD_GET(AD9910_CFR1_RAM_PLAYBACK_DEST_MSK,
+					st->reg[AD9910_REG_CFR1].val32);
+		if (destination == AD9910_DEST_AMPLITUDE ||
+		    destination == AD9910_DEST_POLAR)
+			return ad9910_channel_str[AD9910_CHAN_IDX_RAM];
+	}
+
+	/* DRG enabled and data destination is amplitude */
+	mode_en = FIELD_GET(AD9910_CFR2_DRG_ENABLE_MSK,
+			    st->reg[AD9910_REG_CFR2].val32);
+	if (mode_en && AD9910_DEST_AMPLITUDE ==
+		       FIELD_GET(AD9910_CFR2_DRG_DEST_MSK,
+				 st->reg[AD9910_REG_CFR2].val32))
+		return ad9910_channel_str[AD9910_CHAN_IDX_DRG_AMP];
+
+	/* only way to control amplitude at this point is through OSK */
+	if (ram_en)
+		return ad9910_channel_str[AD9910_CHAN_IDX_OSK];
+
+	/* single tone profiles */
+	return ad9910_channel_str[AD9910_CHAN_IDX_PROFILE_0 + st->profile];
+}
+
+static int ad9910_amplitude_source_show(struct seq_file *s, void *ignored)
+{
+	seq_printf(s, "%s\n", ad9910_amplitude_source_get(s->private));
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(ad9910_amplitude_source);
+
+static inline void ad9910_debugfs_init(struct ad9910_state *st,
+				       struct iio_dev *indio_dev)
+{
+	struct dentry *d = iio_get_debugfs_dentry(indio_dev);
+
+	debugfs_create_file("frequency_source", 0400, d, st,
+			    &ad9910_frequency_source_fops);
+	debugfs_create_file("phase_source", 0400, d, st,
+			    &ad9910_phase_source_fops);
+	debugfs_create_file("amplitude_source", 0400, d, st,
+			    &ad9910_amplitude_source_fops);
+}
+
+static int ad9910_probe(struct spi_device *spi)
+{
+	static const char * const supplies[] = {
+		"dvdd-io33", "avdd33", "dvdd18", "avdd18",
+	};
+	struct device *dev = &spi->dev;
+	struct reset_control *dev_rst;
+	struct gpio_desc *io_rst_gpio;
+	struct iio_dev *indio_dev;
+	struct ad9910_state *st;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	st->spi = spi;
+
+	indio_dev->name = "ad9910";
+	indio_dev->info = &ad9910_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = ad9910_channels;
+	indio_dev->num_channels = ARRAY_SIZE(ad9910_channels);
+
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
+	ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(supplies), supplies);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get regulators\n");
+
+	st->refclk = devm_clk_get_enabled(dev, "ref_clk");
+	if (IS_ERR(st->refclk))
+		return dev_err_probe(dev, PTR_ERR(st->refclk),
+				     "Failed to get reference clock\n");
+
+	dev_rst = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(dev_rst))
+		return dev_err_probe(dev, PTR_ERR(dev_rst),
+				     "failed to get device reset control\n");
+
+	/*
+	 * The IO RESET pin is not used in this driver, as we assume that all
+	 * SPI transfers are complete, but if it is wired up, we need to make
+	 * sure it is not floating. We can use either a reset controller or a
+	 * GPIO for this.
+	 */
+	io_rst_gpio = devm_gpiod_get_optional(dev, "io-reset", GPIOD_OUT_LOW);
+	if (IS_ERR(io_rst_gpio))
+		return dev_err_probe(dev, PTR_ERR(io_rst_gpio),
+				     "failed to get io reset gpio\n");
+
+	st->gpio_update = devm_gpiod_get_optional(dev, "update", GPIOD_OUT_LOW);
+	if (IS_ERR(st->gpio_update))
+		return dev_err_probe(dev, PTR_ERR(st->gpio_update),
+				     "failed to get update gpio\n");
+
+	st->gpio_profile = devm_gpiod_get_array_optional(dev, "profile",
+							 GPIOD_OUT_LOW);
+	if (IS_ERR(st->gpio_profile))
+		return dev_err_probe(dev, PTR_ERR(st->gpio_profile),
+				     "failed to get profile gpios\n");
+
+	if (st->gpio_profile && st->gpio_profile->ndescs != 3)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid number of profile gpios\n");
+
+	st->gpio_pwdown = devm_gpiod_get_optional(dev, "powerdown",
+						  GPIOD_OUT_LOW);
+	if (IS_ERR(st->gpio_pwdown))
+		return dev_err_probe(dev, PTR_ERR(st->gpio_pwdown),
+				     "failed to get powerdown gpio\n");
+
+	ret = devm_add_action_or_reset(dev, ad9910_hw_powerdown_action, st);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to add hw powerdown action\n");
+
+	fsleep(AD9910_WAKEUP_DELAY_us);
+
+	ret = ad9910_parse_fw(st);
+	if (ret)
+		return ret;
+
+	ret = ad9910_setup(dev, st, dev_rst);
+	if (ret)
+		return dev_err_probe(dev, ret, "device setup failed\n");
+
+	snprintf(st->ram_fwu_name, sizeof(st->ram_fwu_name), "%s:ram",
+		 dev_name(&indio_dev->dev));
+	st->ram_fwu = firmware_upload_register(THIS_MODULE, dev, st->ram_fwu_name,
+					       &ad9910_ram_fwu_ops, st);
+	if (IS_ERR(st->ram_fwu))
+		return dev_err_probe(dev, PTR_ERR(st->ram_fwu),
+				     "failed to register ram upload ops\n");
+
+	ret = devm_add_action_or_reset(dev, ad9910_ram_fwu_unregister, st->ram_fwu);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to add ram upload unregister action\n");
+
+	ret = devm_iio_device_register(dev, indio_dev);
+	if (ret)
+		return ret;
+
+	ad9910_debugfs_init(st, indio_dev);
+
+	return 0;
+}
+
+static const struct spi_device_id ad9910_id[] = {
+	{ .name = "ad9910" },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, ad9910_id);
+
+static const struct of_device_id ad9910_of_match[] = {
+	{ .compatible = "adi,ad9910" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ad9910_of_match);
+
+static struct spi_driver ad9910_driver = {
+	.driver = {
+		.name = "ad9910",
+		.of_match_table = ad9910_of_match,
+	},
+	.probe = ad9910_probe,
+	.id_table = ad9910_id,
+};
+module_spi_driver(ad9910_driver);
+
+MODULE_AUTHOR("Rodrigo Alencar <rodrigo.alencar@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD9910 DDS driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/frequency/adf4350.c b/drivers/iio/frequency/adf4350.c
index 315317d6eec412..ec45c0f65fe3cd 100644
--- a/drivers/iio/frequency/adf4350.c
+++ b/drivers/iio/frequency/adf4350.c
@@ -461,7 +461,7 @@ static const struct clk_ops adf4350_clk_ops = {
 static int adf4350_clk_register(struct adf4350_state *st)
 {
 	struct spi_device *spi = st->spi;
-	struct clk_init_data init;
+	struct clk_init_data init = { };
 	struct clk *clk;
 	const char *parent_name;
 	int ret;
diff --git a/drivers/iio/frequency/adf4377.c b/drivers/iio/frequency/adf4377.c
index 4dd19a9aa9943c..52266f83877ac3 100644
--- a/drivers/iio/frequency/adf4377.c
+++ b/drivers/iio/frequency/adf4377.c
@@ -988,10 +988,10 @@ static const struct clk_ops adf4377_clk_ops = {
 
 static int adf4377_clk_register(struct adf4377_state *st)
 {
+	struct clk_parent_data parent_data = { };
 	struct spi_device *spi = st->spi;
 	struct device *dev = &spi->dev;
-	struct clk_init_data init;
-	struct clk_parent_data parent_data;
+	struct clk_init_data init = { };
 	int ret;
 
 	if (!device_property_present(dev, "#clock-cells"))
@@ -1060,6 +1060,9 @@ static int adf4377_probe(struct spi_device *spi)
 	st->regmap = regmap;
 	st->spi = spi;
 	st->chip_info = spi_get_device_match_data(spi);
+	if (!st->chip_info)
+		return -ENODATA;
+
 	mutex_init(&st->lock);
 
 	ret = adf4377_properties_parse(st);
diff --git a/drivers/iio/frequency/admv1013.c b/drivers/iio/frequency/admv1013.c
index c9baaab574e3f3..078ea3e5aa0301 100644
--- a/drivers/iio/frequency/admv1013.c
+++ b/drivers/iio/frequency/admv1013.c
@@ -195,7 +195,7 @@ static int admv1013_read_raw(struct iio_dev *indio_dev,
 
 	switch (info) {
 	case IIO_CHAN_INFO_CALIBBIAS:
-		switch (chan->channel) {
+		switch (chan->channel2) {
 		case IIO_MOD_I:
 			addr = ADMV1013_REG_OFFSET_ADJUST_I;
 			break;
@@ -601,11 +601,14 @@ static int admv1013_probe(struct spi_device *spi)
 				     "failed to get the LO input clock\n");
 
 	st->nb.notifier_call = admv1013_freq_change;
-	ret = devm_clk_notifier_register(dev, st->clkin, &st->nb);
+
+	ret = devm_mutex_init(dev, &st->lock);
 	if (ret)
 		return ret;
 
-	mutex_init(&st->lock);
+	ret = devm_clk_notifier_register(dev, st->clkin, &st->nb);
+	if (ret)
+		return ret;
 
 	ret = admv1013_init(st, vcm_uv);
 	if (ret)
diff --git a/drivers/iio/gyro/adis16136.c b/drivers/iio/gyro/adis16136.c
index ff335f81e9ae98..b687588ada03bd 100644
--- a/drivers/iio/gyro/adis16136.c
+++ b/drivers/iio/gyro/adis16136.c
@@ -145,7 +145,7 @@ static int adis16136_debugfs_init(struct iio_dev *indio_dev)
 	struct adis16136 *adis16136 = iio_priv(indio_dev);
 	struct dentry *d = iio_get_debugfs_dentry(indio_dev);
 
-	debugfs_create_file_unsafe("serial_number", 0400,
+	debugfs_create_file("serial_number", 0400,
 		d, adis16136, &adis16136_serial_fops);
 	debugfs_create_file_unsafe("product_id", 0400,
 		d, adis16136, &adis16136_product_id_fops);
diff --git a/drivers/iio/gyro/adxrs290.c b/drivers/iio/gyro/adxrs290.c
index 563de27243969c..b61cc81122a66a 100644
--- a/drivers/iio/gyro/adxrs290.c
+++ b/drivers/iio/gyro/adxrs290.c
@@ -502,8 +502,8 @@ static irqreturn_t adxrs290_trigger_handler(int irq, void *p)
 		if (ret < 0)
 			break;
 
-		iio_push_to_buffers_with_timestamp(indio_dev, &st->buffer,
-						   pf->timestamp);
+		iio_push_to_buffers_with_ts(indio_dev, &st->buffer,
+					    sizeof(st->buffer), pf->timestamp);
 	} while (0);
 
 	iio_trigger_notify_done(indio_dev->trig);
diff --git a/drivers/iio/gyro/bmg160_core.c b/drivers/iio/gyro/bmg160_core.c
index d611341a0e2ad8..b007f46d3fd624 100644
--- a/drivers/iio/gyro/bmg160_core.c
+++ b/drivers/iio/gyro/bmg160_core.c
@@ -897,8 +897,8 @@ static irqreturn_t bmg160_trigger_handler(int irq, void *p)
 	if (ret < 0)
 		goto err;
 
-	iio_push_to_buffers_with_timestamp(indio_dev, &data->scan,
-					   pf->timestamp);
+	iio_push_to_buffers_with_ts(indio_dev, &data->scan,
+				    sizeof(data->scan), pf->timestamp);
 err:
 	iio_trigger_notify_done(indio_dev->trig);
 
diff --git a/drivers/iio/gyro/itg3200_buffer.c b/drivers/iio/gyro/itg3200_buffer.c
index 87efa2c74ca4e5..426d04d496b4f0 100644
--- a/drivers/iio/gyro/itg3200_buffer.c
+++ b/drivers/iio/gyro/itg3200_buffer.c
@@ -59,7 +59,8 @@ static irqreturn_t itg3200_trigger_handler(int irq, void *p)
 	if (ret < 0)
 		goto error_ret;
 
-	iio_push_to_buffers_with_timestamp(indio_dev, &scan, pf->timestamp);
+	iio_push_to_buffers_with_ts(indio_dev, &scan, sizeof(scan),
+				    pf->timestamp);
 
 error_ret:
 	iio_trigger_notify_done(indio_dev->trig);
diff --git a/drivers/iio/health/max30102.c b/drivers/iio/health/max30102.c
index c37316c86f140a..aee96167f01e4e 100644
--- a/drivers/iio/health/max30102.c
+++ b/drivers/iio/health/max30102.c
@@ -290,9 +290,15 @@ static irqreturn_t max30102_interrupt_handler(int irq, void *private)
 {
 	struct iio_dev *indio_dev = private;
 	struct max30102_data *data = iio_priv(indio_dev);
-	unsigned int measurements = bitmap_weight(indio_dev->active_scan_mask,
-						  iio_get_masklength(indio_dev));
-	int ret, cnt = 0;
+	unsigned int measurements;
+	int ret, cnt;
+
+	cnt = max30102_fifo_count(data);
+	if (cnt <= 0)
+		return IRQ_HANDLED;
+
+	measurements = bitmap_weight(indio_dev->active_scan_mask,
+				     iio_get_masklength(indio_dev));
 
 	mutex_lock(&data->lock);
 
diff --git a/drivers/iio/humidity/Kconfig b/drivers/iio/humidity/Kconfig
index 54f11f000b6ffc..13e267943f67a7 100644
--- a/drivers/iio/humidity/Kconfig
+++ b/drivers/iio/humidity/Kconfig
@@ -5,13 +5,15 @@
 menu "Humidity sensors"
 
 config AM2315
-	tristate "Aosong AM2315 relative humidity and temperature sensor"
+	tristate "Aosong AM2315 and similar relative humidity and temperature sensor"
 	depends on I2C
 	select IIO_BUFFER
 	select IIO_TRIGGERED_BUFFER
 	help
-	  If you say yes here you get support for the Aosong AM2315
-	  relative humidity and ambient temperature sensor.
+	  If you say yes here you get support for the Aosong relative
+	  humidity and ambient temperature sensors:
+	  - AM2315
+	  - AM2320
 
 	  This driver can also be built as a module. If so, the module will
 	  be called am2315.
diff --git a/drivers/iio/humidity/am2315.c b/drivers/iio/humidity/am2315.c
index f29baa251f9f78..5c7582f3a4f94f 100644
--- a/drivers/iio/humidity/am2315.c
+++ b/drivers/iio/humidity/am2315.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Aosong AM2315 relative humidity and temperature
+ * Aosong AM2315 and similar relative humidity and temperature
  *
  * Copyright (c) 2016, Intel Corporation.
  *
@@ -27,7 +27,17 @@
 #define AM2315_TEMP_OFFSET			4
 #define AM2315_ALL_CHANNEL_MASK			GENMASK(1, 0)
 
-#define AM2315_DRIVER_NAME			"am2315"
+struct am2315_chip_info {
+	const char *name;
+};
+
+static const struct am2315_chip_info am2315_chip_info = {
+	.name = "am2315",
+};
+
+static const struct am2315_chip_info am2320_chip_info = {
+	.name = "am2320",
+};
 
 struct am2315_data {
 	struct i2c_client *client;
@@ -220,9 +230,14 @@ static const struct iio_info am2315_info = {
 static int am2315_probe(struct i2c_client *client)
 {
 	int ret;
+	const struct am2315_chip_info *chip_info;
 	struct iio_dev *indio_dev;
 	struct am2315_data *data;
 
+	chip_info = i2c_get_match_data(client);
+	if (!chip_info)
+		return -ENODATA;
+
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (!indio_dev)
 		return -ENOMEM;
@@ -233,7 +248,7 @@ static int am2315_probe(struct i2c_client *client)
 	mutex_init(&data->lock);
 
 	indio_dev->info = &am2315_info;
-	indio_dev->name = AM2315_DRIVER_NAME;
+	indio_dev->name = chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = am2315_channels;
 	indio_dev->num_channels = ARRAY_SIZE(am2315_channels);
@@ -249,8 +264,16 @@ static int am2315_probe(struct i2c_client *client)
 	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
+static const struct of_device_id am2315_of_match[] = {
+	{ .compatible = "aosong,am2315", .data = &am2315_chip_info },
+	{ .compatible = "aosong,am2320", .data = &am2320_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, am2315_of_match);
+
 static const struct i2c_device_id am2315_i2c_id[] = {
-	{ .name = "am2315" },
+	{ .name = "am2315", .driver_data = (kernel_ulong_t)&am2315_chip_info },
+	{ .name = "am2320", .driver_data = (kernel_ulong_t)&am2320_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, am2315_i2c_id);
@@ -258,6 +281,7 @@ MODULE_DEVICE_TABLE(i2c, am2315_i2c_id);
 static struct i2c_driver am2315_driver = {
 	.driver = {
 		.name = "am2315",
+		.of_match_table = am2315_of_match,
 	},
 	.probe =        am2315_probe,
 	.id_table =         am2315_i2c_id,
@@ -266,5 +290,5 @@ static struct i2c_driver am2315_driver = {
 module_i2c_driver(am2315_driver);
 
 MODULE_AUTHOR("Tiberiu Breana <tiberiu.a.breana@intel.com>");
-MODULE_DESCRIPTION("Aosong AM2315 relative humidity and temperature");
+MODULE_DESCRIPTION("Aosong AM2315 and similar relative humidity and temperature");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/humidity/ens210.c b/drivers/iio/humidity/ens210.c
index 81276195152b12..4bdd37e7227b9a 100644
--- a/drivers/iio/humidity/ens210.c
+++ b/drivers/iio/humidity/ens210.c
@@ -202,8 +202,8 @@ static int ens210_probe(struct i2c_client *client)
 	int ret;
 
 	if (!i2c_check_functionality(client->adapter,
-				     I2C_FUNC_SMBUS_WRITE_BYTE_DATA |
-				     I2C_FUNC_SMBUS_WRITE_BYTE |
+				     I2C_FUNC_SMBUS_BYTE_DATA |
+				     I2C_FUNC_SMBUS_READ_WORD_DATA |
 				     I2C_FUNC_SMBUS_READ_I2C_BLOCK)) {
 		return dev_err_probe(&client->dev, -EOPNOTSUPP,
 			"adapter does not support some i2c transactions\n");
diff --git a/drivers/iio/humidity/hts221_core.c b/drivers/iio/humidity/hts221_core.c
index bfeb0a60d3afe8..47a10884d8120c 100644
--- a/drivers/iio/humidity/hts221_core.c
+++ b/drivers/iio/humidity/hts221_core.c
@@ -10,7 +10,6 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/device.h>
-#include <linux/iio/sysfs.h>
 #include <linux/delay.h>
 #include <linux/pm.h>
 #include <linux/regmap.h>
@@ -49,7 +48,7 @@ struct hts221_odr {
 struct hts221_avg {
 	u8 addr;
 	u8 mask;
-	u16 avg_avl[HTS221_AVG_DEPTH];
+	int avg_avl[HTS221_AVG_DEPTH];
 };
 
 static const struct hts221_odr hts221_odr_table[] = {
@@ -58,6 +57,8 @@ static const struct hts221_odr hts221_odr_table[] = {
 	{ 13, 0x03 },	/* 12.5Hz */
 };
 
+static const int hts221_odr_avail[] = { 1, 7, 13 };
+
 static const struct hts221_avg hts221_avg_list[] = {
 	{
 		.addr = 0x10,
@@ -97,7 +98,11 @@ static const struct iio_chan_spec hts221_channels[] = {
 				      BIT(IIO_CHAN_INFO_OFFSET) |
 				      BIT(IIO_CHAN_INFO_SCALE) |
 				      BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
 		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.info_mask_shared_by_all_available =
+				BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.scan_index = 0,
 		.scan_type = {
 			.sign = 's',
@@ -113,7 +118,11 @@ static const struct iio_chan_spec hts221_channels[] = {
 				      BIT(IIO_CHAN_INFO_OFFSET) |
 				      BIT(IIO_CHAN_INFO_SCALE) |
 				      BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
+		.info_mask_separate_available =
+				BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),
 		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),
+		.info_mask_shared_by_all_available =
+				BIT(IIO_CHAN_INFO_SAMP_FREQ),
 		.scan_index = 1,
 		.scan_type = {
 			.sign = 's',
@@ -127,19 +136,15 @@ static const struct iio_chan_spec hts221_channels[] = {
 
 static int hts221_check_whoami(struct hts221_hw *hw)
 {
+	struct device *dev = hw->dev;
 	int err, data;
 
 	err = regmap_read(hw->regmap, HTS221_REG_WHOAMI_ADDR, &data);
-	if (err < 0) {
-		dev_err(hw->dev, "failed to read whoami register\n");
-		return err;
-	}
+	if (err < 0)
+		return dev_err_probe(dev, err, "failed to read whoami register\n");
 
-	if (data != HTS221_REG_WHOAMI_VAL) {
-		dev_err(hw->dev, "wrong whoami {%02x vs %02x}\n",
-			data, HTS221_REG_WHOAMI_VAL);
-		return -ENODEV;
-	}
+	if (data != HTS221_REG_WHOAMI_VAL)
+		dev_info(dev, "unexpected whoami 0x%02x, continuing\n", data);
 
 	return 0;
 }
@@ -192,53 +197,35 @@ static int hts221_update_avg(struct hts221_hw *hw,
 	return 0;
 }
 
-static ssize_t hts221_sysfs_sampling_freq(struct device *dev,
-					  struct device_attribute *attr,
-					  char *buf)
-{
-	int i;
-	ssize_t len = 0;
-
-	for (i = 0; i < ARRAY_SIZE(hts221_odr_table); i++)
-		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
-				 hts221_odr_table[i].hz);
-	buf[len - 1] = '\n';
-
-	return len;
-}
-
-static ssize_t
-hts221_sysfs_rh_oversampling_avail(struct device *dev,
-				   struct device_attribute *attr,
-				   char *buf)
-{
-	const struct hts221_avg *avg = &hts221_avg_list[HTS221_SENSOR_H];
-	ssize_t len = 0;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(avg->avg_avl); i++)
-		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
-				 avg->avg_avl[i]);
-	buf[len - 1] = '\n';
-
-	return len;
-}
-
-static ssize_t
-hts221_sysfs_temp_oversampling_avail(struct device *dev,
-				     struct device_attribute *attr,
-				     char *buf)
+static int hts221_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type, int *length,
+			     long mask)
 {
-	const struct hts221_avg *avg = &hts221_avg_list[HTS221_SENSOR_T];
-	ssize_t len = 0;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(avg->avg_avl); i++)
-		len += scnprintf(buf + len, PAGE_SIZE - len, "%d ",
-				 avg->avg_avl[i]);
-	buf[len - 1] = '\n';
-
-	return len;
+	switch (mask) {
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		switch (chan->type) {
+		case IIO_HUMIDITYRELATIVE:
+			*vals = hts221_avg_list[HTS221_SENSOR_H].avg_avl;
+			*length = ARRAY_SIZE(hts221_avg_list[HTS221_SENSOR_H].avg_avl);
+			break;
+		case IIO_TEMP:
+			*vals = hts221_avg_list[HTS221_SENSOR_T].avg_avl;
+			*length = ARRAY_SIZE(hts221_avg_list[HTS221_SENSOR_T].avg_avl);
+			break;
+		default:
+			return -EINVAL;
+		}
+		*type = IIO_VAL_INT;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*vals = hts221_odr_avail;
+		*type = IIO_VAL_INT;
+		*length = ARRAY_SIZE(hts221_odr_avail);
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
 }
 
 int hts221_set_enable(struct hts221_hw *hw, bool enable)
@@ -258,6 +245,7 @@ int hts221_set_enable(struct hts221_hw *hw, bool enable)
 
 static int hts221_parse_temp_caldata(struct hts221_hw *hw)
 {
+	struct device *dev = hw->dev;
 	int err, *slope, *b_gen, cal0, cal1;
 	s16 cal_x0, cal_x1, cal_y0, cal_y1;
 	__le16 val;
@@ -288,10 +276,20 @@ static int hts221_parse_temp_caldata(struct hts221_hw *hw)
 		return err;
 	cal_x1 = le16_to_cpu(val);
 
+	if (cal_x1 == cal_x0)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid temperature calibration points (x0 %d, x1 %d)\n",
+				     cal_x0, cal_x1);
+
 	slope = &hw->sensors[HTS221_SENSOR_T].slope;
 	b_gen = &hw->sensors[HTS221_SENSOR_T].b_gen;
 
 	*slope = ((cal_y1 - cal_y0) * 8000) / (cal_x1 - cal_x0);
+	if (!*slope)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid temperature calibration slope (y0 %d, y1 %d)\n",
+				     cal_y0, cal_y1);
+
 	*b_gen = (((s32)cal_x1 * cal_y0 - (s32)cal_x0 * cal_y1) * 1000) /
 		 (cal_x1 - cal_x0);
 	*b_gen *= 8;
@@ -301,6 +299,7 @@ static int hts221_parse_temp_caldata(struct hts221_hw *hw)
 
 static int hts221_parse_rh_caldata(struct hts221_hw *hw)
 {
+	struct device *dev = hw->dev;
 	int err, *slope, *b_gen, data;
 	s16 cal_x0, cal_x1, cal_y0, cal_y1;
 	__le16 val;
@@ -327,10 +326,20 @@ static int hts221_parse_rh_caldata(struct hts221_hw *hw)
 		return err;
 	cal_x1 = le16_to_cpu(val);
 
+	if (cal_x1 == cal_x0)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid rh calibration points (x0 %d, x1 %d)\n",
+				     cal_x0, cal_x1);
+
 	slope = &hw->sensors[HTS221_SENSOR_H].slope;
 	b_gen = &hw->sensors[HTS221_SENSOR_H].b_gen;
 
 	*slope = ((cal_y1 - cal_y0) * 8000) / (cal_x1 - cal_x0);
+	if (!*slope)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid rh calibration slope (y0 %d, y1 %d)\n",
+				     cal_y0, cal_y1);
+
 	*b_gen = (((s32)cal_x1 * cal_y0 - (s32)cal_x0 * cal_y1) * 1000) /
 		 (cal_x1 - cal_x0);
 	*b_gen *= 8;
@@ -521,27 +530,10 @@ static int hts221_validate_trigger(struct iio_dev *iio_dev,
 	return hw->trig == trig ? 0 : -EINVAL;
 }
 
-static IIO_DEVICE_ATTR(in_humidity_oversampling_ratio_available, S_IRUGO,
-		       hts221_sysfs_rh_oversampling_avail, NULL, 0);
-static IIO_DEVICE_ATTR(in_temp_oversampling_ratio_available, S_IRUGO,
-		       hts221_sysfs_temp_oversampling_avail, NULL, 0);
-static IIO_DEV_ATTR_SAMP_FREQ_AVAIL(hts221_sysfs_sampling_freq);
-
-static struct attribute *hts221_attributes[] = {
-	&iio_dev_attr_sampling_frequency_available.dev_attr.attr,
-	&iio_dev_attr_in_humidity_oversampling_ratio_available.dev_attr.attr,
-	&iio_dev_attr_in_temp_oversampling_ratio_available.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group hts221_attribute_group = {
-	.attrs = hts221_attributes,
-};
-
 static const struct iio_info hts221_info = {
-	.attrs = &hts221_attribute_group,
 	.read_raw = hts221_read_raw,
 	.write_raw = hts221_write_raw,
+	.read_avail = hts221_read_avail,
 	.validate_trigger = hts221_validate_trigger,
 };
 
@@ -608,33 +600,23 @@ int hts221_probe(struct device *dev, int irq, const char *name,
 
 	/* configure humidity sensor */
 	err = hts221_parse_rh_caldata(hw);
-	if (err < 0) {
-		dev_err(hw->dev, "failed to get rh calibration data\n");
-		return err;
-	}
+	if (err < 0)
+		return dev_err_probe(dev, err, "failed to get rh calibration data\n");
 
 	data = hts221_avg_list[HTS221_SENSOR_H].avg_avl[3];
 	err = hts221_update_avg(hw, HTS221_SENSOR_H, data);
-	if (err < 0) {
-		dev_err(hw->dev, "failed to set rh oversampling ratio\n");
-		return err;
-	}
+	if (err < 0)
+		return dev_err_probe(dev, err, "failed to set rh oversampling ratio\n");
 
 	/* configure temperature sensor */
 	err = hts221_parse_temp_caldata(hw);
-	if (err < 0) {
-		dev_err(hw->dev,
-			"failed to get temperature calibration data\n");
-		return err;
-	}
+	if (err < 0)
+		return dev_err_probe(dev, err, "failed to get temperature calibration data\n");
 
 	data = hts221_avg_list[HTS221_SENSOR_T].avg_avl[3];
 	err = hts221_update_avg(hw, HTS221_SENSOR_T, data);
-	if (err < 0) {
-		dev_err(hw->dev,
-			"failed to set temperature oversampling ratio\n");
-		return err;
-	}
+	if (err < 0)
+		return dev_err_probe(dev, err, "failed to set temperature oversampling ratio\n");
 
 	if (hw->irq > 0) {
 		err = hts221_allocate_buffers(iio_dev);
diff --git a/drivers/iio/humidity/hts221_i2c.c b/drivers/iio/humidity/hts221_i2c.c
index 40276abc5d2e76..e6dc9532cfbd1c 100644
--- a/drivers/iio/humidity/hts221_i2c.c
+++ b/drivers/iio/humidity/hts221_i2c.c
@@ -26,17 +26,14 @@ static const struct regmap_config hts221_i2c_regmap_config = {
 
 static int hts221_i2c_probe(struct i2c_client *client)
 {
+	struct device *dev = &client->dev;
 	struct regmap *regmap;
 
 	regmap = devm_regmap_init_i2c(client, &hts221_i2c_regmap_config);
-	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to register i2c regmap %ld\n",
-			PTR_ERR(regmap));
-		return PTR_ERR(regmap);
-	}
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Failed to register i2c regmap\n");
 
-	return hts221_probe(&client->dev, client->irq,
-			    client->name, regmap);
+	return hts221_probe(dev, client->irq, client->name, regmap);
 }
 
 static const struct acpi_device_id hts221_acpi_match[] = {
diff --git a/drivers/iio/humidity/hts221_spi.c b/drivers/iio/humidity/hts221_spi.c
index f962842cc71df5..f2c0b1a81a8f66 100644
--- a/drivers/iio/humidity/hts221_spi.c
+++ b/drivers/iio/humidity/hts221_spi.c
@@ -27,17 +27,14 @@ static const struct regmap_config hts221_spi_regmap_config = {
 
 static int hts221_spi_probe(struct spi_device *spi)
 {
+	struct device *dev = &spi->dev;
 	struct regmap *regmap;
 
 	regmap = devm_regmap_init_spi(spi, &hts221_spi_regmap_config);
-	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Failed to register spi regmap %ld\n",
-			PTR_ERR(regmap));
-		return PTR_ERR(regmap);
-	}
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Failed to register spi regmap\n");
 
-	return hts221_probe(&spi->dev, spi->irq,
-			    spi->modalias, regmap);
+	return hts221_probe(dev, spi->irq, spi->modalias, regmap);
 }
 
 static const struct of_device_id hts221_spi_of_match[] = {
diff --git a/drivers/iio/iio_core.h b/drivers/iio/iio_core.h
index b7d5f4f0fada79..59fa1bd9924b9f 100644
--- a/drivers/iio/iio_core.h
+++ b/drivers/iio/iio_core.h
@@ -109,4 +109,11 @@ void iio_device_wakeup_eventset(struct iio_dev *indio_dev);
 struct iio_event_interface;
 bool iio_event_enabled(const struct iio_event_interface *ev_int);
 
+#if IS_ENABLED(CONFIG_KUNIT)
+ssize_t __iio_chan_prefix_emit(struct device *dev,
+			       const struct iio_chan_spec *chan,
+			       enum iio_shared_by shared_by,
+			       char *buf, size_t len);
+#endif
+
 #endif
diff --git a/drivers/iio/imu/adis16400.c b/drivers/iio/imu/adis16400.c
index 4842346f9f0c0e..79f2974d37771b 100644
--- a/drivers/iio/imu/adis16400.c
+++ b/drivers/iio/imu/adis16400.c
@@ -280,7 +280,7 @@ static void adis16400_debugfs_init(struct iio_dev *indio_dev)
 		return;
 
 	if (st->variant->flags & ADIS16400_HAS_SERIAL_NUMBER)
-		debugfs_create_file_unsafe("serial_number", 0400,
+		debugfs_create_file("serial_number", 0400,
 				d, st, &adis16400_serial_number_fops);
 	if (st->variant->flags & ADIS16400_HAS_PROD_ID)
 		debugfs_create_file_unsafe("product_id", 0400,
@@ -420,7 +420,10 @@ static int adis16400_initial_setup(struct iio_dev *indio_dev)
 	else
 		st->adis.spi->max_speed_hz = ADIS16400_SPI_FAST;
 	st->adis.spi->mode = SPI_MODE_3;
-	spi_setup(st->adis.spi);
+
+	ret = spi_setup(st->adis.spi);
+	if (ret)
+		return ret;
 
 	ret = __adis_initial_startup(&st->adis);
 	if (ret)
@@ -1150,6 +1153,9 @@ static int adis16400_probe(struct spi_device *spi)
 
 	/* setup the industrialio driver allocated elements */
 	st->variant = spi_get_device_match_data(spi);
+	if (!st->variant)
+		return -ENODATA;
+
 	indio_dev->name = spi_get_device_id(spi)->name;
 	indio_dev->channels = st->variant->channels;
 	indio_dev->num_channels = st->variant->num_channels;
diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c
index 5187566c18763c..d51786874e74c4 100644
--- a/drivers/iio/imu/adis16480.c
+++ b/drivers/iio/imu/adis16480.c
@@ -309,9 +309,9 @@ static void adis16480_debugfs_init(struct iio_dev *indio_dev)
 	if (!IS_ENABLED(CONFIG_DEBUG_FS))
 		return;
 
-	debugfs_create_file_unsafe("firmware_revision", 0400,
+	debugfs_create_file("firmware_revision", 0400,
 		d, adis16480, &adis16480_firmware_revision_fops);
-	debugfs_create_file_unsafe("firmware_date", 0400,
+	debugfs_create_file("firmware_date", 0400,
 		d, adis16480, &adis16480_firmware_date_fops);
 	debugfs_create_file_unsafe("serial_number", 0400,
 		d, adis16480, &adis16480_serial_number_fops);
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600.h b/drivers/iio/imu/inv_icm42600/inv_icm42600.h
index b55d993f026438..e56943931b6a01 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600.h
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600.h
@@ -179,6 +179,7 @@ struct inv_icm42600_state {
 	struct {
 		s64 gyro;
 		s64 accel;
+		bool accel_force_odr;
 	} timestamp;
 	struct inv_icm42600_apex apex;
 	struct inv_icm42600_fifo fifo;
@@ -356,8 +357,8 @@ struct inv_icm42600_sensor_state {
 		cpu_to_le16((_wm) & GENMASK(11, 0))
 /* FIFO is 2048 bytes, let 12 samples for reading latency */
 #define INV_ICM42600_FIFO_WATERMARK_MAX			(2048 - 12 * 16)
-/* INV_ICM42600_FIFO_WATERMARK_MAX / 8 = 232 */
-#define INV_ICM42600_FIFO_WATERMARK_MAX_SAMPLES		232
+/* INV_ICM42600_FIFO_WATERMARK_MAX / 16 = 116 */
+#define INV_ICM42600_FIFO_WATERMARK_MAX_SAMPLES		116
 
 #define INV_ICM42600_REG_INT_CONFIG1			0x0064
 #define INV_ICM42600_INT_CONFIG1_TPULSE_DURATION	BIT(6)
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
index 4b0e3cd8a50674..cc8c074777c9da 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
@@ -254,15 +254,23 @@ static int inv_icm42600_accel_update_scan_mode(struct iio_dev *indio_dev,
 		fifo_en |= INV_ICM42600_SENSOR_ACCEL;
 	}
 
+	/*
+	 * Sleep maximum stabilization time before enabling data in FIFO.
+	 * We need to release the driver lock to not block gyro data processing.
+	 * There is no possible race here since we are under IIO mutex locked.
+	 */
+	sleep = max(sleep_accel, sleep_temp);
+	if (sleep) {
+		mutex_unlock(&st->lock);
+		msleep(sleep);
+		mutex_lock(&st->lock);
+	}
+
 	/* update data FIFO write */
 	ret = inv_icm42600_buffer_set_fifo_en(st, fifo_en | st->fifo.en);
 
 out_unlock:
 	mutex_unlock(&st->lock);
-	/* sleep maximum required time */
-	sleep = max(sleep_accel, sleep_temp);
-	if (sleep)
-		msleep(sleep);
 	return ret;
 }
 
@@ -1227,14 +1235,16 @@ int inv_icm42600_accel_parse_fifo(struct iio_dev *indio_dev)
 		if (size <= 0)
 			return size;
 
-		/* skip packet if no accel data or data is invalid */
-		if (accel == NULL || !inv_icm42600_fifo_is_data_valid(accel))
-			continue;
-
 		/* update odr */
-		if (odr & INV_ICM42600_SENSOR_ACCEL)
+		if (odr & INV_ICM42600_SENSOR_ACCEL) {
 			inv_sensors_timestamp_apply_odr(ts, st->fifo.period,
 							st->fifo.nb.total, no);
+			st->timestamp.accel_force_odr = false;
+		}
+
+		/* skip packet if no accel data or data is invalid */
+		if (accel == NULL || !inv_icm42600_fifo_is_data_valid(accel))
+			continue;
 
 		memcpy(&buffer.accel, accel, sizeof(buffer.accel));
 		/* convert 8 bits FIFO temperature in high resolution format */
@@ -1243,5 +1253,10 @@ int inv_icm42600_accel_parse_fifo(struct iio_dev *indio_dev)
 		iio_push_to_buffers_with_timestamp(indio_dev, &buffer, ts_val);
 	}
 
+	if (st->timestamp.accel_force_odr) {
+		inv_sensors_timestamp_apply_odr(ts, 0, 0, 0);
+		st->timestamp.accel_force_odr = false;
+	}
+
 	return 0;
 }
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
index 998d312f7bde39..2bf9fc29d7d296 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
@@ -5,6 +5,7 @@
 
 #include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/gcd.h>
 #include <linux/kernel.h>
 #include <linux/minmax.h>
 #include <linux/mutex.h>
@@ -131,13 +132,16 @@ int inv_icm42600_buffer_set_fifo_en(struct inv_icm42600_state *st,
 		INV_ICM42600_FIFO_CONFIG1_GYRO_EN |
 		INV_ICM42600_FIFO_CONFIG1_ACCEL_EN;
 
-	val = 0;
-	if (fifo_en & INV_ICM42600_SENSOR_GYRO)
-		val |= INV_ICM42600_FIFO_CONFIG1_GYRO_EN;
-	if (fifo_en & INV_ICM42600_SENSOR_ACCEL)
-		val |= INV_ICM42600_FIFO_CONFIG1_ACCEL_EN;
-	if (fifo_en & INV_ICM42600_SENSOR_TEMP)
-		val |= INV_ICM42600_FIFO_CONFIG1_TEMP_EN;
+	/*
+	 * Always enable/disable all bits to ensure we can flawlessly add
+	 * accel/gyro data in the FIFO while it is running.
+	 */
+	if (fifo_en)
+		val = INV_ICM42600_FIFO_CONFIG1_TEMP_EN |
+		      INV_ICM42600_FIFO_CONFIG1_GYRO_EN |
+		      INV_ICM42600_FIFO_CONFIG1_ACCEL_EN;
+	else
+		val = 0;
 
 	ret = regmap_update_bits(st->map, INV_ICM42600_REG_FIFO_CONFIG1, mask, val);
 	if (ret)
@@ -149,19 +153,6 @@ int inv_icm42600_buffer_set_fifo_en(struct inv_icm42600_state *st,
 	return 0;
 }
 
-static size_t inv_icm42600_get_packet_size(unsigned int fifo_en)
-{
-	size_t packet_size;
-
-	if ((fifo_en & INV_ICM42600_SENSOR_GYRO) &&
-	    (fifo_en & INV_ICM42600_SENSOR_ACCEL))
-		packet_size = INV_ICM42600_FIFO_2SENSORS_PACKET_SIZE;
-	else
-		packet_size = INV_ICM42600_FIFO_1SENSOR_PACKET_SIZE;
-
-	return packet_size;
-}
-
 static unsigned int inv_icm42600_wm_truncate(unsigned int watermark,
 					     size_t packet_size)
 {
@@ -185,15 +176,14 @@ static unsigned int inv_icm42600_wm_truncate(unsigned int watermark,
  *
  * FIFO watermark threshold is computed based on the required watermark values
  * set for gyro and accel sensors. Since watermark is all about acceptable data
- * latency, use the smallest setting between the 2. It means choosing the
- * smallest latency but this is not as simple as choosing the smallest watermark
- * value. Latency depends on watermark and ODR. It requires several steps:
- * 1) compute gyro and accel latencies and choose the smallest value.
- * 2) adapt the chosen latency so that it is a multiple of both gyro and accel
- *    ones. Otherwise it is possible that you don't meet a requirement. (for
- *    example with gyro @100Hz wm 4 and accel @100Hz with wm 6, choosing the
- *    value of 4 will not meet accel latency requirement because 6 is not a
- *    multiple of 4. You need to use the value 2.)
+ * latency, we should need to use the smallest latency value. But it is not as
+ * simple as choosing the smallest watermark value. Latency depends on watermark
+ * and ODR and IIO buffer watermark adds another requirement. The required steps:
+ * 1) compute gyro and accel periods and latencies
+ * 2) Use the smallest period and the GCD of the latencies. GCD is required
+ *    because of the IIO buffer watermark that will prevent send of data if not
+ *    crossed. Thus accel and gyro watermarks must be a multiple of the watermark
+ *    value. Computing the GCD gives us the biggest value that meets this criteria.
  * 3) Since all periods are multiple of each others, watermark is computed by
  *    dividing this computed latency by the smallest period, which corresponds
  *    to the FIFO frequency. Beware that this is only true because we are not
@@ -203,13 +193,13 @@ int inv_icm42600_buffer_update_watermark(struct inv_icm42600_state *st)
 {
 	size_t packet_size, wm_size;
 	unsigned int wm_gyro, wm_accel, watermark;
-	u32 period_gyro, period_accel;
+	u32 period_gyro, period_accel, period;
 	u32 latency_gyro, latency_accel, latency;
 	bool restore;
 	__le16 raw_wm;
 	int ret;
 
-	packet_size = inv_icm42600_get_packet_size(st->fifo.en);
+	packet_size = INV_ICM42600_FIFO_2SENSORS_PACKET_SIZE;
 
 	/* compute sensors latency, depending on sensor watermark and odr */
 	wm_gyro = inv_icm42600_wm_truncate(st->fifo.watermark.gyro, packet_size);
@@ -231,22 +221,17 @@ int inv_icm42600_buffer_update_watermark(struct inv_icm42600_state *st)
 		watermark = wm_gyro;
 		st->fifo.watermark.eff_gyro = wm_gyro;
 	} else {
-		/* compute the smallest latency that is a multiple of both */
-		if (latency_gyro <= latency_accel)
-			latency = latency_gyro - (latency_accel % latency_gyro);
-		else
-			latency = latency_accel - (latency_gyro % latency_accel);
-		/* all this works because periods are multiple of each others */
-		watermark = latency / min(period_gyro, period_accel);
-		if (watermark < 1)
-			watermark = 1;
-		/* update effective watermark */
-		st->fifo.watermark.eff_gyro = latency / period_gyro;
-		if (st->fifo.watermark.eff_gyro < 1)
-			st->fifo.watermark.eff_gyro = 1;
-		st->fifo.watermark.eff_accel = latency / period_accel;
-		if (st->fifo.watermark.eff_accel < 1)
-			st->fifo.watermark.eff_accel = 1;
+		/*
+		 * In case of both accel and gyro enabled, we need to use the
+		 * shortest period and the gcd of the latencies. Gcd is required
+		 * because of the IIO buffer watermark that will prevent data
+		 * sending if we are not crossing the watermark level.
+		 */
+		period = min(period_gyro, period_accel);
+		latency = gcd(latency_gyro, latency_accel);
+		watermark = max(latency / period, 1);
+		st->fifo.watermark.eff_gyro = max(latency / period_gyro, 1);
+		st->fifo.watermark.eff_accel = max(latency / period_accel, 1);
 	}
 
 	/* compute watermark value in bytes */
@@ -399,6 +384,8 @@ static int inv_icm42600_buffer_postdisable(struct iio_dev *indio_dev)
 	mutex_lock(&st->lock);
 
 	inv_sensors_timestamp_apply_odr(ts, 0, 0, 0);
+	if (sensor == INV_ICM42600_SENSOR_ACCEL)
+		st->timestamp.accel_force_odr = false;
 
 	ret = inv_icm42600_buffer_set_fifo_en(st, st->fifo.en & ~sensor);
 	if (ret)
@@ -492,24 +479,21 @@ int inv_icm42600_buffer_fifo_read(struct inv_icm42600_state *st,
 	st->fifo.nb.accel = 0;
 	st->fifo.nb.total = 0;
 
-	/* compute maximum FIFO read size (watermark for max = 0 interrupt case) */
-	if (max == 0)
-		max = st->fifo.watermark.value;
-	max_count = max * inv_icm42600_get_packet_size(st->fifo.en);
-
-	/* read FIFO count value */
-	raw_fifo_count = (__be16 *)st->buffer;
-	ret = regmap_bulk_read(st->map, INV_ICM42600_REG_FIFO_COUNT,
-			       raw_fifo_count, sizeof(*raw_fifo_count));
-	if (ret)
-		return ret;
-	st->fifo.count = be16_to_cpup(raw_fifo_count);
-
-	/* check and clamp FIFO count value */
-	if (st->fifo.count == 0)
-		return 0;
-	if (st->fifo.count > max_count)
-		st->fifo.count = max_count;
+	/* read watermark samples for interrupt case (max = 0) or read FIFO count */
+	if (max == 0) {
+		st->fifo.count = st->fifo.watermark.value *
+				 INV_ICM42600_FIFO_2SENSORS_PACKET_SIZE;
+	} else {
+		raw_fifo_count = (__be16 *)st->buffer;
+		ret = regmap_bulk_read(st->map, INV_ICM42600_REG_FIFO_COUNT,
+					raw_fifo_count, sizeof(*raw_fifo_count));
+		if (ret)
+			return ret;
+		max_count = max * INV_ICM42600_FIFO_2SENSORS_PACKET_SIZE;
+		st->fifo.count = min(be16_to_cpup(raw_fifo_count), max_count);
+		if (st->fifo.count == 0)
+			return 0;
+	}
 
 	/* read all FIFO data in internal buffer */
 	ret = regmap_noinc_read(st->map, INV_ICM42600_REG_FIFO_DATA,
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c
index dc97d8a274e363..cd2e53416f5b1d 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_core.c
@@ -8,6 +8,7 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/kernel.h>
+#include <linux/limits.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/pm_runtime.h>
@@ -324,6 +325,15 @@ int inv_icm42600_set_accel_conf(struct inv_icm42600_state *st,
 		break;
 	}
 
+	/*
+	 * If we change mode from low-power to low-noise because of odr,
+	 * we need to force odr change.
+	 */
+	if (oldconf->mode == INV_ICM42600_SENSOR_MODE_LOW_POWER &&
+	    conf->mode == INV_ICM42600_SENSOR_MODE_LOW_NOISE &&
+	    conf->odr != oldconf->odr)
+		st->timestamp.accel_force_odr = true;
+
 	/* set ACCEL_CONFIG0 register (accel fullscale & odr) */
 	if (conf->fs != oldconf->fs || conf->odr != oldconf->odr) {
 		val = INV_ICM42600_ACCEL_CONFIG0_FS(conf->fs) |
@@ -509,10 +519,20 @@ static int inv_icm42600_setup(struct inv_icm42600_state *st,
 	ret = regmap_read(st->map, INV_ICM42600_REG_WHOAMI, &val);
 	if (ret)
 		return ret;
+
+	/*
+	 * SPI interface has no ack mechanism.
+	 * 0xFF or 0x00 whoami means no response from the device.
+	 */
+	if (val == U8_MAX || val == 0)
+		return dev_err_probe(dev, -ENODEV,
+				     "invalid whoami %#04x expected %#04x (%s)\n",
+				     val, hw->whoami, hw->name);
+
 	if (val != hw->whoami) {
-		dev_err(dev, "invalid whoami %#02x expected %#02x (%s)\n",
-			val, hw->whoami, hw->name);
-		return -ENODEV;
+		dev_info(dev,
+			 "device id %#04x is not the %#04x associated with the FW-specified device (%s), probably using a valid fallback compatible\n",
+			 val, hw->whoami, hw->name);
 	}
 	st->name = hw->name;
 
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
index 253bf571439d91..4e20d61d897be8 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
@@ -126,15 +126,23 @@ static int inv_icm42600_gyro_update_scan_mode(struct iio_dev *indio_dev,
 		fifo_en |= INV_ICM42600_SENSOR_GYRO;
 	}
 
+	/*
+	 * Sleep maximum stabilization time before enabling data in FIFO.
+	 * We need to release the driver lock to not block accel data processing.
+	 * There is no possible race here since we are under IIO mutex locked.
+	 */
+	sleep = max(sleep_gyro, sleep_temp);
+	if (sleep) {
+		mutex_unlock(&st->lock);
+		msleep(sleep);
+		mutex_lock(&st->lock);
+	}
+
 	/* update data FIFO write */
 	ret = inv_icm42600_buffer_set_fifo_en(st, fifo_en | st->fifo.en);
 
 out_unlock:
 	mutex_unlock(&st->lock);
-	/* sleep maximum required time */
-	sleep = max(sleep_gyro, sleep_temp);
-	if (sleep)
-		msleep(sleep);
 	return ret;
 }
 
@@ -808,15 +816,15 @@ int inv_icm42600_gyro_parse_fifo(struct iio_dev *indio_dev)
 		if (size <= 0)
 			return size;
 
-		/* skip packet if no gyro data or data is invalid */
-		if (gyro == NULL || !inv_icm42600_fifo_is_data_valid(gyro))
-			continue;
-
 		/* update odr */
 		if (odr & INV_ICM42600_SENSOR_GYRO)
 			inv_sensors_timestamp_apply_odr(ts, st->fifo.period,
 							st->fifo.nb.total, no);
 
+		/* skip packet if no gyro data or data is invalid */
+		if (gyro == NULL || !inv_icm42600_fifo_is_data_valid(gyro))
+			continue;
+
 		memcpy(&buffer.gyro, gyro, sizeof(buffer.gyro));
 		/* convert 8 bits FIFO temperature in high resolution format */
 		buffer.temp = temp ? (*temp * 64) : 0;
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
index 28552d2db91d7d..e44ed6dcde0293 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_i2c.c
@@ -77,10 +77,10 @@ static const struct i2c_device_id inv_icm42600_id[] = {
 	{ .name = "icm42600", .driver_data = INV_CHIP_ICM42600 },
 	{ .name = "icm42602", .driver_data = INV_CHIP_ICM42602 },
 	{ .name = "icm42605", .driver_data = INV_CHIP_ICM42605 },
-	{ .name = "icm42686", .driver_data = INV_CHIP_ICM42686 },
 	{ .name = "icm42622", .driver_data = INV_CHIP_ICM42622 },
-	{ .name = "icm42688", .driver_data = INV_CHIP_ICM42688 },
 	{ .name = "icm42631", .driver_data = INV_CHIP_ICM42631 },
+	{ .name = "icm42686", .driver_data = INV_CHIP_ICM42686 },
+	{ .name = "icm42688", .driver_data = INV_CHIP_ICM42688 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, inv_icm42600_id);
@@ -95,18 +95,18 @@ static const struct of_device_id inv_icm42600_of_matches[] = {
 	}, {
 		.compatible = "invensense,icm42605",
 		.data = (void *)INV_CHIP_ICM42605,
-	}, {
-		.compatible = "invensense,icm42686",
-		.data = (void *)INV_CHIP_ICM42686,
 	}, {
 		.compatible = "invensense,icm42622",
 		.data = (void *)INV_CHIP_ICM42622,
-	}, {
-		.compatible = "invensense,icm42688",
-		.data = (void *)INV_CHIP_ICM42688,
 	}, {
 		.compatible = "invensense,icm42631",
 		.data = (void *)INV_CHIP_ICM42631,
+	}, {
+		.compatible = "invensense,icm42686",
+		.data = (void *)INV_CHIP_ICM42686,
+	}, {
+		.compatible = "invensense,icm42688",
+		.data = (void *)INV_CHIP_ICM42688,
 	},
 	{ }
 };
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c
index faf743bc644449..3e2f4eb42e9736 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_spi.c
@@ -74,10 +74,10 @@ static const struct spi_device_id inv_icm42600_id[] = {
 	{ .name = "icm42600", .driver_data = INV_CHIP_ICM42600 },
 	{ .name = "icm42602", .driver_data = INV_CHIP_ICM42602 },
 	{ .name = "icm42605", .driver_data = INV_CHIP_ICM42605 },
-	{ .name = "icm42686", .driver_data = INV_CHIP_ICM42686 },
 	{ .name = "icm42622", .driver_data = INV_CHIP_ICM42622 },
-	{ .name = "icm42688", .driver_data = INV_CHIP_ICM42688 },
 	{ .name = "icm42631", .driver_data = INV_CHIP_ICM42631 },
+	{ .name = "icm42686", .driver_data = INV_CHIP_ICM42686 },
+	{ .name = "icm42688", .driver_data = INV_CHIP_ICM42688 },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, inv_icm42600_id);
@@ -92,18 +92,18 @@ static const struct of_device_id inv_icm42600_of_matches[] = {
 	}, {
 		.compatible = "invensense,icm42605",
 		.data = (void *)INV_CHIP_ICM42605,
-	}, {
-		.compatible = "invensense,icm42686",
-		.data = (void *)INV_CHIP_ICM42686,
 	}, {
 		.compatible = "invensense,icm42622",
 		.data = (void *)INV_CHIP_ICM42622,
-	}, {
-		.compatible = "invensense,icm42688",
-		.data = (void *)INV_CHIP_ICM42688,
 	}, {
 		.compatible = "invensense,icm42631",
 		.data = (void *)INV_CHIP_ICM42631,
+	}, {
+		.compatible = "invensense,icm42686",
+		.data = (void *)INV_CHIP_ICM42686,
+	}, {
+		.compatible = "invensense,icm42688",
+		.data = (void *)INV_CHIP_ICM42688,
 	},
 	{ }
 };
diff --git a/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c b/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c
index 190e998f7b8ef2..f4ef75da22c762 100644
--- a/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c
+++ b/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c
@@ -537,9 +537,8 @@ static int inv_icm42607_enable_vddio_reg(struct inv_icm42607_state *st)
 	return 0;
 }
 
-static void inv_icm42607_sensors_off(void *_data)
+static int inv_icm42607_sensors_off(struct inv_icm42607_state *st)
 {
-	struct inv_icm42607_state *st = _data;
 	const struct device *dev = regmap_get_device(st->map);
 	int ret;
 
@@ -552,6 +551,13 @@ static void inv_icm42607_sensors_off(void *_data)
 					 st->conf.accel.mode);
 	if (ret)
 		dev_err(dev, "Unable to turn off sensors\n");
+
+	return ret;
+}
+
+static void inv_icm42607_sensors_off_action(void *data)
+{
+	inv_icm42607_sensors_off(data);
 }
 
 static void inv_icm42607_disable_vddio_reg(void *_data)
@@ -619,7 +625,7 @@ int inv_icm42607_core_probe(struct regmap *regmap,
 	 * Ensure if sensors get turned on at some point, they're turned off
 	 * as part of teardown.
 	 */
-	ret = devm_add_action_or_reset(dev, inv_icm42607_sensors_off, st);
+	ret = devm_add_action_or_reset(dev, inv_icm42607_sensors_off_action, st);
 	if (ret)
 		return ret;
 
@@ -658,9 +664,8 @@ static int inv_icm42607_suspend(struct device *dev)
 	return 0;
 }
 
-static int inv_icm42607_resume(struct device *dev)
+static int inv_icm42607_resume_core(struct inv_icm42607_state *st)
 {
-	struct inv_icm42607_state *st = dev_get_drvdata(dev);
 	int ret;
 
 	ret = inv_icm42607_enable_vddio_reg(st);
@@ -669,9 +674,25 @@ static int inv_icm42607_resume(struct device *dev)
 
 	/* Sync the regcache again after regulator shutdown. */
 	regcache_mark_dirty(st->map);
-	ret = regcache_sync(st->map);
-	if (ret)
+
+	return regcache_sync(st->map);
+}
+
+static int inv_icm42607_resume(struct device *dev)
+{
+	struct inv_icm42607_state *st = dev_get_drvdata(dev);
+	int ret;
+
+	ret = inv_icm42607_resume_core(st);
+	if (ret) {
+		int rc;
+
+		rc = pm_runtime_force_resume(dev);
+		if (rc)
+			dev_warn(dev, "Failed to restore runtime PM state: %d\n", rc);
+
 		return ret;
+	}
 
 	return pm_runtime_force_resume(dev);
 }
@@ -688,8 +709,7 @@ static int inv_icm42607_runtime_suspend(struct device *dev)
 	 * however the tradeoff is that an unused sensor won't be
 	 * turned off until the entire chip is no longer in use.
 	 */
-	inv_icm42607_sensors_off(st);
-	return 0;
+	return inv_icm42607_sensors_off(st);
 }
 
 EXPORT_NS_GPL_DEV_PM_OPS(inv_icm42607_pm_ops, IIO_ICM42607) = {
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
index 5796896d54cd86..38c3f3b398d4f8 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_core.c
@@ -13,6 +13,7 @@
 #include <linux/irq.h>
 #include <linux/interrupt.h>
 #include <linux/acpi.h>
+#include <linux/kstrtox.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
 #include <linux/math64.h>
@@ -1303,8 +1304,9 @@ inv_mpu6050_fifo_rate_store(struct device *dev, struct device_attribute *attr,
 	struct inv_mpu6050_state *st = iio_priv(indio_dev);
 	struct device *pdev = regmap_get_device(st->map);
 
-	if (kstrtoint(buf, 10, &fifo_rate))
-		return -EINVAL;
+	result = kstrtoint(buf, 10, &fifo_rate);
+	if (result)
+		return result;
 	if (fifo_rate < INV_MPU6050_MIN_FIFO_RATE ||
 	    fifo_rate > INV_MPU6050_MAX_FIFO_RATE)
 		return -EINVAL;
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c
index d6a1eeb151cabe..cbc47fa5b1012b 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_shub.c
@@ -766,8 +766,8 @@ st_lsm6dsx_shub_alloc_iiodev(struct st_lsm6dsx_hw *hw,
 			IIO_CHAN_SOFT_TIMESTAMP(3),
 		};
 
-		ext_channels = devm_kzalloc(hw->dev, sizeof(magn_channels),
-					    GFP_KERNEL);
+		ext_channels = devm_kcalloc(hw->dev, ARRAY_SIZE(magn_channels),
+					    sizeof(*ext_channels), GFP_KERNEL);
 		if (!ext_channels)
 			return NULL;
 
diff --git a/drivers/iio/industrialio-backend.c b/drivers/iio/industrialio-backend.c
index f7a4be8ec32078..fbf5c39d76551e 100644
--- a/drivers/iio/industrialio-backend.c
+++ b/drivers/iio/industrialio-backend.c
@@ -886,6 +886,39 @@ int iio_backend_num_lanes_set(struct iio_backend *back, unsigned int num_lanes)
 }
 EXPORT_SYMBOL_NS_GPL(iio_backend_num_lanes_set, "IIO_BACKEND");
 
+/**
+ * iio_backend_crc_enable - Enable CRC handling
+ * @back: Backend device
+ *
+ * On input data paths, enable backend handling of CRC data generated by the
+ * frontend. On output data paths, enable the backend to generate CRC data for
+ * the frontend to verify.
+ *
+ * RETURNS:
+ * 0 on success, negative error number on failure.
+ */
+int iio_backend_crc_enable(struct iio_backend *back)
+{
+	return iio_backend_op_call(back, crc_enable);
+}
+EXPORT_SYMBOL_NS_GPL(iio_backend_crc_enable, "IIO_BACKEND");
+
+/**
+ * iio_backend_crc_disable - Disable CRC handling
+ * @back: Backend device
+ *
+ * On input data paths, disable backend handling of CRC data generated by the
+ * frontend. On output data paths, disable CRC generation by the backend.
+ *
+ * RETURNS:
+ * 0 on success, negative error number on failure.
+ */
+int iio_backend_crc_disable(struct iio_backend *back)
+{
+	return iio_backend_op_call(back, crc_disable);
+}
+EXPORT_SYMBOL_NS_GPL(iio_backend_crc_disable, "IIO_BACKEND");
+
 /**
  * iio_backend_ddr_enable - Enable interface DDR (Double Data Rate) mode
  * @back: Backend device
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index 2c9ec93dff475e..4294101810fd71 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -1377,6 +1377,10 @@ EXPORT_SYMBOL_GPL(iio_update_buffers);
 
 void iio_disable_all_buffers(struct iio_dev *indio_dev)
 {
+	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
+
+	guard(mutex)(&iio_dev_opaque->mlock);
+
 	iio_disable_buffers(indio_dev);
 	iio_buffer_deactivate_all(indio_dev);
 }
@@ -2462,7 +2466,8 @@ int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev,
 
 		bb = devm_krealloc(&indio_dev->dev,
 				   iio_dev_opaque->bounce_buffer,
-				   indio_dev->scan_bytes, GFP_KERNEL);
+				   indio_dev->scan_bytes,
+				   GFP_KERNEL | __GFP_ZERO);
 		if (!bb)
 			return -ENOMEM;
 		iio_dev_opaque->bounce_buffer = bb;
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index 767a7794624a50..d73f2541375fd8 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -9,12 +9,15 @@
 
 #define pr_fmt(fmt) "iio-core: " fmt
 
+#include <kunit/visibility.h>
+
 #include <linux/anon_inodes.h>
 #include <linux/cdev.h>
 #include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/err.h>
+#include <linux/export.h>
 #include <linux/fs.h>
 #include <linux/idr.h>
 #include <linux/kdev_t.h>
@@ -25,6 +28,7 @@
 #include <linux/poll.h>
 #include <linux/property.h>
 #include <linux/sched.h>
+#include <linux/seq_buf.h>
 #include <linux/slab.h>
 #include <linux/wait.h>
 
@@ -100,6 +104,7 @@ static const char * const iio_chan_type_name_spec[] = {
 	[IIO_ALTCURRENT] = "altcurrent",
 	[IIO_COVERAGE] = "coverage",
 	[IIO_VOLUMEFLOW] = "volumeflow",
+	[IIO_FREQUENCY] = "frequency",
 };
 
 static const char * const iio_modifier_names[] = {
@@ -199,6 +204,63 @@ static const char * const iio_chan_info_postfix[] = {
 	[IIO_CHAN_INFO_CONVDELAY] = "convdelay",
 	[IIO_CHAN_INFO_POWERFACTOR] = "powerfactor",
 };
+
+VISIBLE_IF_KUNIT
+ssize_t __iio_chan_prefix_emit(struct device *dev,
+			       const struct iio_chan_spec *chan,
+			       enum iio_shared_by shared_by,
+			       char *buf, size_t len)
+{
+	const char *type = iio_chan_type_name_spec[chan->type];
+	const char *dir = iio_direction[chan->output];
+	struct seq_buf s;
+
+	seq_buf_init(&s, buf, len);
+
+	switch (shared_by) {
+	case IIO_SHARED_BY_ALL:
+		break;
+	case IIO_SHARED_BY_DIR:
+		seq_buf_printf(&s, "%s", dir);
+		break;
+	case IIO_SHARED_BY_TYPE:
+		seq_buf_printf(&s, "%s_%s", dir, type);
+		if (chan->differential)
+			seq_buf_printf(&s, "-%s", type);
+		break;
+	case IIO_SEPARATE:
+		/* Validate differential channel settings */
+		if (chan->differential) {
+			if (!chan->indexed) {
+				dev_err(dev, "Differential channels must be indexed\n");
+				return -EINVAL;
+			}
+			if (chan->modified) {
+				dev_err(dev, "Differential channels can not have modifier\n");
+				return -EINVAL;
+			}
+		}
+
+		seq_buf_printf(&s, "%s_%s", dir, type);
+
+		if (chan->indexed)
+			seq_buf_printf(&s, "%d", chan->channel);
+
+		if (chan->differential)
+			seq_buf_printf(&s, "-%s%d", type, chan->channel2);
+
+		if (chan->modified)
+			seq_buf_printf(&s, "_%s", iio_modifier_names[chan->channel2]);
+
+		if (chan->extend_name)
+			seq_buf_printf(&s, "_%s", chan->extend_name);
+		break;
+	}
+
+	return seq_buf_has_overflowed(&s) ? -EOVERFLOW : s.len;
+}
+EXPORT_SYMBOL_IF_KUNIT(__iio_chan_prefix_emit);
+
 /**
  * iio_device_id() - query the unique ID for the device
  * @indio_dev:		Device structure whose ID is being queried
@@ -341,7 +403,8 @@ s64 iio_get_time_ns(const struct iio_dev *indio_dev)
 	case CLOCK_TAI:
 		return ktime_get_clocktai_ns();
 	default:
-		BUG();
+		WARN_ON_ONCE(1);
+		return 0;
 	}
 }
 EXPORT_SYMBOL(iio_get_time_ns);
@@ -388,6 +451,7 @@ static ssize_t iio_debugfs_read_reg(struct file *file, char __user *userbuf,
 	struct iio_dev *indio_dev = file->private_data;
 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
 	unsigned int val = 0;
+	u64 val64 = 0;
 	int ret;
 
 	if (*ppos > 0)
@@ -395,9 +459,17 @@ static ssize_t iio_debugfs_read_reg(struct file *file, char __user *userbuf,
 					       iio_dev_opaque->read_buf,
 					       iio_dev_opaque->read_buf_len);
 
-	ret = indio_dev->info->debugfs_reg_access(indio_dev,
-						  iio_dev_opaque->cached_reg_addr,
-						  0, &val);
+	if (indio_dev->info->debugfs_reg64_access) {
+		ret = indio_dev->info->debugfs_reg64_access(indio_dev,
+							    iio_dev_opaque->cached_reg_addr,
+							    0, &val64);
+	} else {
+		ret = indio_dev->info->debugfs_reg_access(indio_dev,
+							  iio_dev_opaque->cached_reg_addr,
+							  0, &val);
+		val64 = val;
+	}
+
 	if (ret) {
 		dev_err(indio_dev->dev.parent, "%s: read failed\n", __func__);
 		return ret;
@@ -405,7 +477,7 @@ static ssize_t iio_debugfs_read_reg(struct file *file, char __user *userbuf,
 
 	iio_dev_opaque->read_buf_len = snprintf(iio_dev_opaque->read_buf,
 						sizeof(iio_dev_opaque->read_buf),
-						"0x%X\n", val);
+						"0x%llX\n", val64);
 
 	return simple_read_from_buffer(userbuf, count, ppos,
 				       iio_dev_opaque->read_buf,
@@ -417,8 +489,9 @@ static ssize_t iio_debugfs_write_reg(struct file *file,
 {
 	struct iio_dev *indio_dev = file->private_data;
 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
-	unsigned int reg, val;
+	unsigned int reg;
 	char buf[80];
+	u64 val64;
 	int ret;
 
 	if (*ppos != 0 || count >= sizeof(buf))
@@ -431,7 +504,7 @@ static ssize_t iio_debugfs_write_reg(struct file *file,
 
 	buf[ret] = '\0';
 
-	ret = sscanf(buf, "%i %i", &reg, &val);
+	ret = sscanf(buf, "%i %lli", &reg, &val64);
 
 	switch (ret) {
 	case 1:
@@ -439,8 +512,12 @@ static ssize_t iio_debugfs_write_reg(struct file *file,
 		break;
 	case 2:
 		iio_dev_opaque->cached_reg_addr = reg;
-		ret = indio_dev->info->debugfs_reg_access(indio_dev, reg,
-							  val, NULL);
+		if (indio_dev->info->debugfs_reg64_access)
+			ret = indio_dev->info->debugfs_reg64_access(indio_dev, reg,
+								    val64, NULL);
+		else
+			ret = indio_dev->info->debugfs_reg_access(indio_dev, reg,
+								  val64, NULL);
 		if (ret) {
 			dev_err(indio_dev->dev.parent, "%s: write failed\n",
 				__func__);
@@ -471,7 +548,8 @@ static void iio_device_register_debugfs(struct iio_dev *indio_dev)
 {
 	struct iio_dev_opaque *iio_dev_opaque;
 
-	if (indio_dev->info->debugfs_reg_access == NULL)
+	if (!indio_dev->info->debugfs_reg_access &&
+	    !indio_dev->info->debugfs_reg64_access)
 		return;
 
 	if (!iio_debugfs_dentry)
@@ -781,6 +859,20 @@ static ssize_t iio_read_channel_label(struct device *dev,
 					 to_iio_dev_attr(attr)->c, buf);
 }
 
+static ssize_t iio_read_channel_parent(struct device *dev,
+				       struct device_attribute *attr,
+				       char *buf)
+{
+	const struct iio_chan_spec *chan = to_iio_dev_attr(attr)->c->parent;
+	ssize_t len;
+
+	len = __iio_chan_prefix_emit(dev, chan, IIO_SEPARATE, buf, PAGE_SIZE - 1);
+	if (len < 0)
+		return len;
+
+	return len + sysfs_emit_at(buf, len, "\n");
+}
+
 static ssize_t iio_read_channel_info(struct device *dev,
 				     struct device_attribute *attr,
 				     char *buf)
@@ -1075,7 +1167,8 @@ static ssize_t iio_write_channel_info(struct device *dev,
 }
 
 static
-int __iio_device_attr_init(struct device_attribute *dev_attr,
+int __iio_device_attr_init(struct device *dev,
+			   struct device_attribute *dev_attr,
 			   const char *postfix,
 			   struct iio_chan_spec const *chan,
 			   ssize_t (*readfunc)(struct device *dev,
@@ -1087,105 +1180,21 @@ int __iio_device_attr_init(struct device_attribute *dev_attr,
 						size_t len),
 			   enum iio_shared_by shared_by)
 {
-	int ret = 0;
-	char *name = NULL;
-	char *full_postfix;
+	char prefix[NAME_MAX + 1];
+	ssize_t ret;
 
 	sysfs_attr_init(&dev_attr->attr);
 
-	/* Build up postfix of <extend_name>_<modifier>_postfix */
-	if (chan->modified && (shared_by == IIO_SEPARATE)) {
-		if (chan->extend_name)
-			full_postfix = kasprintf(GFP_KERNEL, "%s_%s_%s",
-						 iio_modifier_names[chan->channel2],
-						 chan->extend_name,
-						 postfix);
-		else
-			full_postfix = kasprintf(GFP_KERNEL, "%s_%s",
-						 iio_modifier_names[chan->channel2],
-						 postfix);
-	} else {
-		if (chan->extend_name == NULL || shared_by != IIO_SEPARATE)
-			full_postfix = kstrdup(postfix, GFP_KERNEL);
-		else
-			full_postfix = kasprintf(GFP_KERNEL,
-						 "%s_%s",
-						 chan->extend_name,
-						 postfix);
-	}
-	if (full_postfix == NULL)
-		return -ENOMEM;
-
-	if (chan->differential) { /* Differential can not have modifier */
-		switch (shared_by) {
-		case IIO_SHARED_BY_ALL:
-			name = kasprintf(GFP_KERNEL, "%s", full_postfix);
-			break;
-		case IIO_SHARED_BY_DIR:
-			name = kasprintf(GFP_KERNEL, "%s_%s",
-						iio_direction[chan->output],
-						full_postfix);
-			break;
-		case IIO_SHARED_BY_TYPE:
-			name = kasprintf(GFP_KERNEL, "%s_%s-%s_%s",
-					    iio_direction[chan->output],
-					    iio_chan_type_name_spec[chan->type],
-					    iio_chan_type_name_spec[chan->type],
-					    full_postfix);
-			break;
-		case IIO_SEPARATE:
-			if (!chan->indexed) {
-				WARN(1, "Differential channels must be indexed\n");
-				ret = -EINVAL;
-				goto error_free_full_postfix;
-			}
-			name = kasprintf(GFP_KERNEL,
-					    "%s_%s%d-%s%d_%s",
-					    iio_direction[chan->output],
-					    iio_chan_type_name_spec[chan->type],
-					    chan->channel,
-					    iio_chan_type_name_spec[chan->type],
-					    chan->channel2,
-					    full_postfix);
-			break;
-		}
-	} else { /* Single ended */
-		switch (shared_by) {
-		case IIO_SHARED_BY_ALL:
-			name = kasprintf(GFP_KERNEL, "%s", full_postfix);
-			break;
-		case IIO_SHARED_BY_DIR:
-			name = kasprintf(GFP_KERNEL, "%s_%s",
-						iio_direction[chan->output],
-						full_postfix);
-			break;
-		case IIO_SHARED_BY_TYPE:
-			name = kasprintf(GFP_KERNEL, "%s_%s_%s",
-					    iio_direction[chan->output],
-					    iio_chan_type_name_spec[chan->type],
-					    full_postfix);
-			break;
+	ret = __iio_chan_prefix_emit(dev, chan, shared_by, prefix, sizeof(prefix));
+	if (ret < 0)
+		return ret;
 
-		case IIO_SEPARATE:
-			if (chan->indexed)
-				name = kasprintf(GFP_KERNEL, "%s_%s%d_%s",
-						    iio_direction[chan->output],
-						    iio_chan_type_name_spec[chan->type],
-						    chan->channel,
-						    full_postfix);
-			else
-				name = kasprintf(GFP_KERNEL, "%s_%s_%s",
-						    iio_direction[chan->output],
-						    iio_chan_type_name_spec[chan->type],
-						    full_postfix);
-			break;
-		}
-	}
-	if (name == NULL) {
-		ret = -ENOMEM;
-		goto error_free_full_postfix;
-	}
-	dev_attr->attr.name = name;
+	if (ret)
+		dev_attr->attr.name = kasprintf(GFP_KERNEL, "%s_%s", prefix, postfix);
+	else
+		dev_attr->attr.name = kstrdup(postfix, GFP_KERNEL);
+	if (!dev_attr->attr.name)
+		return -ENOMEM;
 
 	if (readfunc) {
 		dev_attr->attr.mode |= 0444;
@@ -1197,10 +1206,7 @@ int __iio_device_attr_init(struct device_attribute *dev_attr,
 		dev_attr->store = writefunc;
 	}
 
-error_free_full_postfix:
-	kfree(full_postfix);
-
-	return ret;
+	return 0;
 }
 
 static void __iio_device_attr_deinit(struct device_attribute *dev_attr)
@@ -1229,7 +1235,7 @@ int __iio_add_chan_devattr(const char *postfix,
 	iio_attr = kzalloc_obj(*iio_attr);
 	if (iio_attr == NULL)
 		return -ENOMEM;
-	ret = __iio_device_attr_init(&iio_attr->dev_attr,
+	ret = __iio_device_attr_init(dev, &iio_attr->dev_attr,
 				     postfix, chan,
 				     readfunc, writefunc, shared_by);
 	if (ret)
@@ -1281,6 +1287,25 @@ static int iio_device_add_channel_label(struct iio_dev *indio_dev,
 	return 1;
 }
 
+static int iio_device_add_channel_parent_attr(struct iio_dev *indio_dev,
+					      struct iio_chan_spec const *chan)
+{
+	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
+	int ret;
+
+	if (!chan->parent)
+		return 0;
+
+	ret = __iio_add_chan_devattr("parent", chan,
+				     &iio_read_channel_parent, NULL,
+				     0, IIO_SEPARATE, &indio_dev->dev, NULL,
+				     &iio_dev_opaque->channel_attr_list);
+	if (ret < 0)
+		return ret;
+
+	return 1;
+}
+
 static int iio_device_add_info_mask_type(struct iio_dev *indio_dev,
 					 struct iio_chan_spec const *chan,
 					 enum iio_shared_by shared_by,
@@ -1419,6 +1444,11 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev,
 		return ret;
 	attrcount += ret;
 
+	ret = iio_device_add_channel_parent_attr(indio_dev, chan);
+	if (ret < 0)
+		return ret;
+	attrcount += ret;
+
 	if (chan->ext_info) {
 		unsigned int i = 0;
 
@@ -1518,7 +1548,8 @@ static ssize_t current_timestamp_clock_show(struct device *dev,
 	case CLOCK_TAI:
 		break;
 	default:
-		BUG();
+		WARN_ON_ONCE(1);
+		return -EINVAL;
 	}
 
 	return sysfs_emit(buf, "%s\n", clock_names[clk]);
diff --git a/drivers/iio/industrialio-gts-helper.c b/drivers/iio/industrialio-gts-helper.c
index 4f52dc373abf7a..af6ed37fb34a8b 100644
--- a/drivers/iio/industrialio-gts-helper.c
+++ b/drivers/iio/industrialio-gts-helper.c
@@ -5,7 +5,7 @@
  */
 
 #include <linux/device.h>
-#include <linux/errno.h>
+#include <linux/err.h>
 #include <linux/export.h>
 #include <linux/minmax.h>
 #include <linux/module.h>
diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c
index 17781c12bc858e..76926fee0f84d9 100644
--- a/drivers/iio/industrialio-trigger.c
+++ b/drivers/iio/industrialio-trigger.c
@@ -509,6 +509,8 @@ static void iio_trig_release(struct device *device)
 	struct iio_trigger *trig = to_iio_trigger(device);
 	int i;
 
+	cancel_work_sync(&trig->reenable_work);
+
 	if (trig->subirq_base) {
 		for (i = 0; i < CONFIG_IIO_CONSUMERS_PER_TRIGGER; i++) {
 			irq_modify_status(trig->subirq_base + i,
diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig
index f23bbce12c72c7..462fff1ddef207 100644
--- a/drivers/iio/light/Kconfig
+++ b/drivers/iio/light/Kconfig
@@ -741,6 +741,20 @@ config VEML6030
 	  To compile this driver as a module, choose M here: the
 	  module will be called veml6030.
 
+config VEML6031X00
+	tristate "VEML6031X00 ambient light sensor series"
+	select REGMAP_I2C
+	select IIO_GTS_HELPER
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	depends on I2C
+	help
+	  Say Y here if you want to build a driver for the Vishay VEML6031X00
+	  ambient light sensor series.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called veml6031x00.
+
 config VEML6040
 	tristate "VEML6040 RGBW light sensor"
 	select REGMAP_I2C
diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile
index 64e354c49ed8d7..6672ed01a8193b 100644
--- a/drivers/iio/light/Makefile
+++ b/drivers/iio/light/Makefile
@@ -68,6 +68,7 @@ obj-$(CONFIG_VCNL4035)		+= vcnl4035.o
 obj-$(CONFIG_VEML3235)		+= veml3235.o
 obj-$(CONFIG_VEML3328)		+= veml3328.o
 obj-$(CONFIG_VEML6030)		+= veml6030.o
+obj-$(CONFIG_VEML6031X00)	+= veml6031x00.o
 obj-$(CONFIG_VEML6040)		+= veml6040.o
 obj-$(CONFIG_VEML6046X00)	+= veml6046x00.o
 obj-$(CONFIG_VEML6070)		+= veml6070.o
diff --git a/drivers/iio/light/apds9306.c b/drivers/iio/light/apds9306.c
index f681b02e492254..7dd41136734f55 100644
--- a/drivers/iio/light/apds9306.c
+++ b/drivers/iio/light/apds9306.c
@@ -61,7 +61,7 @@
 #define APDS9306_NUM_REPEAT_RATES	7
 #define APDS9306_INT_SRC_CLEAR	0
 #define APDS9306_INT_SRC_ALS	1
-#define APDS9306_SAMP_FREQ_10HZ	0
+#define APDS9306_SAMP_FREQ_10HZ	2
 
 /**
  * struct part_id_gts_multiplier - Part no. and corresponding gts multiplier
diff --git a/drivers/iio/light/apds9999.c b/drivers/iio/light/apds9999.c
index 43fa9992c9c2c5..62620c4a2fbce2 100644
--- a/drivers/iio/light/apds9999.c
+++ b/drivers/iio/light/apds9999.c
@@ -100,10 +100,6 @@ static int apds9999_init(struct apds9999_data *data)
 	u8 regval;
 	int ret;
 
-	ret = devm_add_action_or_reset(dev, apds9999_standby, client);
-	if (ret)
-		return ret;
-
 	guard(mutex)(&data->lock);
 
 	regval = FIELD_PREP(APDS9999_LS_RES_MASK, APDS9999_RES_18BIT) |
@@ -121,8 +117,12 @@ static int apds9999_init(struct apds9999_data *data)
 		return ret;
 	data->als_gain_idx = APDS9999_GAIN_3X;
 
-	return i2c_smbus_write_byte_data(client, APDS9999_REG_MAIN_CTRL,
-					 APDS9999_MAIN_CTRL_LS_EN);
+	ret = i2c_smbus_write_byte_data(client, APDS9999_REG_MAIN_CTRL,
+					APDS9999_MAIN_CTRL_LS_EN);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, apds9999_standby, client);
 }
 
 static int apds9999_read_channel(struct apds9999_data *data, u8 reg,
diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c
index 7ab565b1fb9f34..b1c92de7922af5 100644
--- a/drivers/iio/light/cros_ec_light_prox.c
+++ b/drivers/iio/light/cros_ec_light_prox.c
@@ -249,7 +249,7 @@ MODULE_DEVICE_TABLE(platform, cros_ec_light_prox_ids);
 static struct platform_driver cros_ec_light_prox_platform_driver = {
 	.driver = {
 		.name	= "cros-ec-light-prox",
-		.pm	= &cros_ec_sensors_pm_ops,
+		.pm	= pm_sleep_ptr(&cros_ec_sensors_pm_ops),
 	},
 	.probe		= cros_ec_light_prox_probe,
 	.id_table	= cros_ec_light_prox_ids,
diff --git a/drivers/iio/light/gp2ap020a00f.c b/drivers/iio/light/gp2ap020a00f.c
index 63591b7ecb892d..b435dc5e8706f0 100644
--- a/drivers/iio/light/gp2ap020a00f.c
+++ b/drivers/iio/light/gp2ap020a00f.c
@@ -1497,6 +1497,7 @@ static int gp2ap020a00f_probe(struct i2c_client *client)
 	iio_trigger_unregister(data->trig);
 error_free_irq:
 	free_irq(client->irq, indio_dev);
+	irq_work_sync(&data->work);
 error_uninit_buffer:
 	iio_triggered_buffer_cleanup(indio_dev);
 error_regulator_disable:
@@ -1519,6 +1520,7 @@ static void gp2ap020a00f_remove(struct i2c_client *client)
 	iio_device_unregister(indio_dev);
 	iio_trigger_unregister(data->trig);
 	free_irq(client->irq, indio_dev);
+	irq_work_sync(&data->work);
 	iio_triggered_buffer_cleanup(indio_dev);
 	regulator_disable(data->vled_reg);
 }
diff --git a/drivers/iio/light/iqs621-als.c b/drivers/iio/light/iqs621-als.c
index cd5843e3e2c338..f9d215ef1970a5 100644
--- a/drivers/iio/light/iqs621-als.c
+++ b/drivers/iio/light/iqs621-als.c
@@ -179,19 +179,6 @@ static int iqs621_als_notifier(struct notifier_block *notifier,
 	return NOTIFY_OK;
 }
 
-static void iqs621_als_notifier_unregister(void *context)
-{
-	struct iqs621_als_private *iqs621_als = context;
-	struct iio_dev *indio_dev = iqs621_als->indio_dev;
-	int ret;
-
-	ret = blocking_notifier_chain_unregister(&iqs621_als->iqs62x->nh,
-						 &iqs621_als->notifier);
-	if (ret)
-		dev_err(indio_dev->dev.parent,
-			"Failed to unregister notifier: %d\n", ret);
-}
-
 static int iqs621_als_read_raw(struct iio_dev *indio_dev,
 			       struct iio_chan_spec const *chan,
 			       int *val, int *val2, long mask)
@@ -563,19 +550,14 @@ static int iqs621_als_probe(struct platform_device *pdev)
 	mutex_init(&iqs621_als->lock);
 
 	iqs621_als->notifier.notifier_call = iqs621_als_notifier;
-	ret = blocking_notifier_chain_register(&iqs621_als->iqs62x->nh,
-					       &iqs621_als->notifier);
+	ret = devm_blocking_notifier_chain_register(&pdev->dev,
+						    &iqs621_als->iqs62x->nh,
+						    &iqs621_als->notifier);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret);
 		return ret;
 	}
 
-	ret = devm_add_action_or_reset(&pdev->dev,
-				       iqs621_als_notifier_unregister,
-				       iqs621_als);
-	if (ret)
-		return ret;
-
 	return devm_iio_device_register(&pdev->dev, indio_dev);
 }
 
diff --git a/drivers/iio/light/isl29018.c b/drivers/iio/light/isl29018.c
index 759cb71ed1c58d..7f7a563df4d94c 100644
--- a/drivers/iio/light/isl29018.c
+++ b/drivers/iio/light/isl29018.c
@@ -10,6 +10,7 @@
 
 #include <linux/i2c.h>
 #include <linux/err.h>
+#include <linux/kstrtox.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/delay.h>
@@ -339,9 +340,11 @@ static ssize_t proximity_on_chip_ambient_infrared_suppression_store
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct isl29018_chip *chip = iio_priv(indio_dev);
 	int val;
+	int ret;
 
-	if (kstrtoint(buf, 10, &val))
-		return -EINVAL;
+	ret = kstrtoint(buf, 10, &val);
+	if (ret)
+		return ret;
 	if (!(val == 0 || val == 1))
 		return -EINVAL;
 
diff --git a/drivers/iio/light/isl29028.c b/drivers/iio/light/isl29028.c
index 33deb1726689ef..e481ac908fc137 100644
--- a/drivers/iio/light/isl29028.c
+++ b/drivers/iio/light/isl29028.c
@@ -342,8 +342,9 @@ static int isl29028_write_raw(struct iio_dev *indio_dev,
 	struct device *dev = regmap_get_device(chip->regmap);
 	int ret;
 
-	ret = pm_runtime_resume_and_get(dev);
-	if (ret < 0)
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
 		return ret;
 
 	mutex_lock(&chip->lock);
@@ -392,14 +393,7 @@ static int isl29028_write_raw(struct iio_dev *indio_dev,
 
 	mutex_unlock(&chip->lock);
 
-	if (ret < 0)
-		return ret;
-
-	ret = pm_runtime_put_autosuspend(dev);
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	return ret;
 }
 
 static int isl29028_read_raw(struct iio_dev *indio_dev,
@@ -408,10 +402,11 @@ static int isl29028_read_raw(struct iio_dev *indio_dev,
 {
 	struct isl29028_chip *chip = iio_priv(indio_dev);
 	struct device *dev = regmap_get_device(chip->regmap);
-	int ret, pm_ret;
+	int ret;
 
-	ret = pm_runtime_resume_and_get(dev);
-	if (ret < 0)
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
 		return ret;
 
 	mutex_lock(&chip->lock);
@@ -461,18 +456,6 @@ static int isl29028_read_raw(struct iio_dev *indio_dev,
 
 	mutex_unlock(&chip->lock);
 
-	if (ret < 0)
-		return ret;
-
-	/**
-	 * Preserve the ret variable if the call to
-	 * pm_runtime_put_autosuspend() is successful so the reading
-	 * (if applicable) is returned to user space.
-	 */
-	pm_ret = pm_runtime_put_autosuspend(dev);
-	if (pm_ret < 0)
-		return pm_ret;
-
 	return ret;
 }
 
diff --git a/drivers/iio/light/lm3533-als.c b/drivers/iio/light/lm3533-als.c
index 99f0b903018cf8..218cb93b4cf99c 100644
--- a/drivers/iio/light/lm3533-als.c
+++ b/drivers/iio/light/lm3533-als.c
@@ -13,18 +13,22 @@
 #include <linux/io.h>
 #include <linux/iio/events.h>
 #include <linux/iio/iio.h>
+#include <linux/kstrtox.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/mfd/core.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
+#include <linux/units.h>
 
 #include <linux/mfd/lm3533.h>
 
 
-#define LM3533_ALS_RESISTOR_MIN			1
-#define LM3533_ALS_RESISTOR_MAX			127
+#define LM3533_ALS_RESISTOR_MIN_OHMS		1575
+#define LM3533_ALS_RESISTOR_MAX_OHMS		200000
 #define LM3533_ALS_CHANNEL_CURRENT_MAX		2
 #define LM3533_ALS_THRESH_MAX			3
 #define LM3533_ALS_ZONE_MAX			4
@@ -48,7 +52,7 @@
 
 
 struct lm3533_als {
-	struct lm3533 *lm3533;
+	struct regmap *regmap;
 	struct platform_device *pdev;
 
 	unsigned long flags;
@@ -56,6 +60,9 @@ struct lm3533_als {
 
 	atomic_t zone;
 	struct mutex thresh_mutex;
+
+	u32 r_select;
+	bool pwm_mode;
 };
 
 
@@ -64,7 +71,7 @@ static int lm3533_als_get_adc(struct iio_dev *indio_dev, bool average,
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
 	u8 reg;
-	u8 val;
+	u32 val;
 	int ret;
 
 	if (average)
@@ -72,7 +79,7 @@ static int lm3533_als_get_adc(struct iio_dev *indio_dev, bool average,
 	else
 		reg = LM3533_REG_ALS_READ_ADC_RAW;
 
-	ret = lm3533_read(als->lm3533, reg, &val);
+	ret = regmap_read(als->regmap, reg, &val);
 	if (ret) {
 		dev_err(&indio_dev->dev, "failed to read adc\n");
 		return ret;
@@ -86,10 +93,10 @@ static int lm3533_als_get_adc(struct iio_dev *indio_dev, bool average,
 static int _lm3533_als_get_zone(struct iio_dev *indio_dev, u8 *zone)
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
-	u8 val;
+	u32 val;
 	int ret;
 
-	ret = lm3533_read(als->lm3533, LM3533_REG_ALS_ZONE_INFO, &val);
+	ret = regmap_read(als->regmap, LM3533_REG_ALS_ZONE_INFO, &val);
 	if (ret) {
 		dev_err(&indio_dev->dev, "failed to read zone\n");
 		return ret;
@@ -127,7 +134,7 @@ static inline u8 lm3533_als_get_target_reg(unsigned channel, unsigned zone)
 }
 
 static int lm3533_als_get_target(struct iio_dev *indio_dev, unsigned channel,
-							unsigned zone, u8 *val)
+				 unsigned zone, u32 *val)
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
 	u8 reg;
@@ -140,7 +147,7 @@ static int lm3533_als_get_target(struct iio_dev *indio_dev, unsigned channel,
 		return -EINVAL;
 
 	reg = lm3533_als_get_target_reg(channel, zone);
-	ret = lm3533_read(als->lm3533, reg, val);
+	ret = regmap_read(als->regmap, reg, val);
 	if (ret)
 		dev_err(&indio_dev->dev, "failed to get target current\n");
 
@@ -161,7 +168,7 @@ static int lm3533_als_set_target(struct iio_dev *indio_dev, unsigned channel,
 		return -EINVAL;
 
 	reg = lm3533_als_get_target_reg(channel, zone);
-	ret = lm3533_write(als->lm3533, reg, val);
+	ret = regmap_write(als->regmap, reg, val);
 	if (ret)
 		dev_err(&indio_dev->dev, "failed to set target current\n");
 
@@ -171,8 +178,8 @@ static int lm3533_als_set_target(struct iio_dev *indio_dev, unsigned channel,
 static int lm3533_als_get_current(struct iio_dev *indio_dev, unsigned channel,
 								int *val)
 {
+	u32 target;
 	u8 zone;
-	u8 target;
 	int ret;
 
 	ret = lm3533_als_get_zone(indio_dev, &zone);
@@ -271,16 +278,10 @@ static irqreturn_t lm3533_als_isr(int irq, void *dev_id)
 static int lm3533_als_set_int_mode(struct iio_dev *indio_dev, int enable)
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
-	u8 mask = LM3533_ALS_INT_ENABLE_MASK;
-	u8 val;
 	int ret;
 
-	if (enable)
-		val = mask;
-	else
-		val = 0;
-
-	ret = lm3533_update(als->lm3533, LM3533_REG_ALS_ZONE_INFO, val, mask);
+	ret = regmap_assign_bits(als->regmap, LM3533_REG_ALS_ZONE_INFO,
+				 LM3533_ALS_INT_ENABLE_MASK, enable);
 	if (ret) {
 		dev_err(&indio_dev->dev, "failed to set int mode %d\n",
 								enable);
@@ -293,17 +294,16 @@ static int lm3533_als_set_int_mode(struct iio_dev *indio_dev, int enable)
 static int lm3533_als_get_int_mode(struct iio_dev *indio_dev, int *enable)
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
-	u8 mask = LM3533_ALS_INT_ENABLE_MASK;
-	u8 val;
 	int ret;
 
-	ret = lm3533_read(als->lm3533, LM3533_REG_ALS_ZONE_INFO, &val);
-	if (ret) {
+	ret = regmap_test_bits(als->regmap, LM3533_REG_ALS_ZONE_INFO,
+			       LM3533_ALS_INT_ENABLE_MASK);
+	if (ret < 0) {
 		dev_err(&indio_dev->dev, "failed to get int mode\n");
 		return ret;
 	}
 
-	*enable = !!(val & mask);
+	*enable = ret;
 
 	return 0;
 }
@@ -316,7 +316,7 @@ static inline u8 lm3533_als_get_threshold_reg(unsigned nr, bool raising)
 }
 
 static int lm3533_als_get_threshold(struct iio_dev *indio_dev, unsigned nr,
-							bool raising, u8 *val)
+				    bool raising, u32 *val)
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
 	u8 reg;
@@ -326,7 +326,7 @@ static int lm3533_als_get_threshold(struct iio_dev *indio_dev, unsigned nr,
 		return -EINVAL;
 
 	reg = lm3533_als_get_threshold_reg(nr, raising);
-	ret = lm3533_read(als->lm3533, reg, val);
+	ret = regmap_read(als->regmap, reg, val);
 	if (ret)
 		dev_err(&indio_dev->dev, "failed to get threshold\n");
 
@@ -337,8 +337,8 @@ static int lm3533_als_set_threshold(struct iio_dev *indio_dev, unsigned nr,
 							bool raising, u8 val)
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
-	u8 val2;
 	u8 reg, reg2;
+	u32 val2;
 	int ret;
 
 	if (nr > LM3533_ALS_THRESH_MAX)
@@ -348,7 +348,7 @@ static int lm3533_als_set_threshold(struct iio_dev *indio_dev, unsigned nr,
 	reg2 = lm3533_als_get_threshold_reg(nr, !raising);
 
 	mutex_lock(&als->thresh_mutex);
-	ret = lm3533_read(als->lm3533, reg2, &val2);
+	ret = regmap_read(als->regmap, reg2, &val2);
 	if (ret) {
 		dev_err(&indio_dev->dev, "failed to get threshold\n");
 		goto out;
@@ -363,7 +363,7 @@ static int lm3533_als_set_threshold(struct iio_dev *indio_dev, unsigned nr,
 		goto out;
 	}
 
-	ret = lm3533_write(als->lm3533, reg, val);
+	ret = regmap_write(als->regmap, reg, val);
 	if (ret) {
 		dev_err(&indio_dev->dev, "failed to set threshold\n");
 		goto out;
@@ -375,11 +375,11 @@ static int lm3533_als_set_threshold(struct iio_dev *indio_dev, unsigned nr,
 }
 
 static int lm3533_als_get_hysteresis(struct iio_dev *indio_dev, unsigned nr,
-								u8 *val)
+				     u32 *val)
 {
 	struct lm3533_als *als = iio_priv(indio_dev);
-	u8 falling;
-	u8 raising;
+	u32 falling;
+	u32 raising;
 	int ret;
 
 	if (nr > LM3533_ALS_THRESH_MAX)
@@ -409,7 +409,7 @@ static ssize_t show_thresh_either_en(struct device *dev,
 	int enable;
 	int ret;
 
-	if (als->irq) {
+	if (als->irq > 0) {
 		ret = lm3533_als_get_int_mode(indio_dev, &enable);
 		if (ret)
 			return ret;
@@ -431,11 +431,12 @@ static ssize_t store_thresh_either_en(struct device *dev,
 	u8 zone;
 	int ret;
 
-	if (!als->irq)
+	if (als->irq <= 0)
 		return -EBUSY;
 
-	if (kstrtoul(buf, 0, &enable))
-		return -EINVAL;
+	ret = kstrtoul(buf, 0, &enable);
+	if (ret)
+		return ret;
 
 	int_enabled = test_bit(LM3533_ALS_FLAG_INT_ENABLED, &als->flags);
 
@@ -503,7 +504,7 @@ static ssize_t show_als_attr(struct device *dev,
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct lm3533_als_attribute *als_attr = to_lm3533_als_attr(attr);
-	u8 val;
+	u32 val;
 	int ret;
 
 	switch (als_attr->type) {
@@ -542,8 +543,9 @@ static ssize_t store_als_attr(struct device *dev,
 	u8 val;
 	int ret;
 
-	if (kstrtou8(buf, 0, &val))
-		return -EINVAL;
+	ret = kstrtou8(buf, 0, &val);
+	if (ret)
+		return ret;
 
 	switch (als_attr->type) {
 	case LM3533_ATTR_TYPE_TARGET:
@@ -714,71 +716,51 @@ static const struct attribute_group lm3533_als_attribute_group = {
 	.attrs = lm3533_als_attributes
 };
 
-static int lm3533_als_set_input_mode(struct lm3533_als *als, bool pwm_mode)
+static int lm3533_als_setup(struct lm3533_als *als)
 {
-	u8 mask = LM3533_ALS_INPUT_MODE_MASK;
-	u8 val;
+	struct device *dev = &als->pdev->dev;
 	int ret;
 
-	if (pwm_mode)
-		val = mask;	/* pwm input */
-	else
-		val = 0;	/* analog input */
-
-	ret = lm3533_update(als->lm3533, LM3533_REG_ALS_CONF, val, mask);
-	if (ret) {
-		dev_err(&als->pdev->dev, "failed to set input mode %d\n",
-								pwm_mode);
-		return ret;
-	}
-
-	return 0;
-}
+	als->pwm_mode = device_property_read_bool(dev, "ti,pwm-mode");
 
-static int lm3533_als_set_resistor(struct lm3533_als *als, u8 val)
-{
-	int ret;
+	ret = regmap_assign_bits(als->regmap, LM3533_REG_ALS_CONF,
+				 LM3533_ALS_INPUT_MODE_MASK, als->pwm_mode);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to set input mode %d\n",
+				     als->pwm_mode);
 
-	if (val < LM3533_ALS_RESISTOR_MIN || val > LM3533_ALS_RESISTOR_MAX) {
-		dev_err(&als->pdev->dev, "invalid resistor value\n");
-		return -EINVAL;
-	}
+	/* ALS input is always high impedance in PWM-mode. */
+	if (als->pwm_mode)
+		return 0;
 
-	ret = lm3533_write(als->lm3533, LM3533_REG_ALS_RESISTOR_SELECT, val);
-	if (ret) {
-		dev_err(&als->pdev->dev, "failed to set resistor\n");
-		return ret;
-	}
+	ret = device_property_read_u32(dev, "ti,resistor-ohms",
+				       &als->r_select);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to ger resistor value\n");
 
-	return 0;
-}
+	if (als->r_select < LM3533_ALS_RESISTOR_MIN_OHMS ||
+	    als->r_select > LM3533_ALS_RESISTOR_MAX_OHMS)
+		return dev_err_probe(dev, -EINVAL,
+				     "invalid resistor value\n");
 
-static int lm3533_als_setup(struct lm3533_als *als,
-			    const struct lm3533_als_platform_data *pdata)
-{
-	int ret;
+	als->r_select = DIV_ROUND_UP(2 * MICRO, 10 * als->r_select);
 
-	ret = lm3533_als_set_input_mode(als, pdata->pwm_mode);
+	ret = regmap_write(als->regmap, LM3533_REG_ALS_RESISTOR_SELECT,
+			   als->r_select);
 	if (ret)
-		return ret;
-
-	/* ALS input is always high impedance in PWM-mode. */
-	if (!pdata->pwm_mode) {
-		ret = lm3533_als_set_resistor(als, pdata->r_select);
-		if (ret)
-			return ret;
-	}
+		return dev_err_probe(dev, ret, "failed to set resistor\n");
 
 	return 0;
 }
 
 static int lm3533_als_setup_irq(struct lm3533_als *als, void *dev)
 {
-	u8 mask = LM3533_ALS_INT_ENABLE_MASK;
 	int ret;
 
 	/* Make sure interrupts are disabled. */
-	ret = lm3533_update(als->lm3533, LM3533_REG_ALS_ZONE_INFO, 0, mask);
+	ret = regmap_clear_bits(als->regmap, LM3533_REG_ALS_ZONE_INFO,
+				LM3533_ALS_INT_ENABLE_MASK);
 	if (ret) {
 		dev_err(&als->pdev->dev, "failed to disable interrupts\n");
 		return ret;
@@ -798,10 +780,10 @@ static int lm3533_als_setup_irq(struct lm3533_als *als, void *dev)
 
 static int lm3533_als_enable(struct lm3533_als *als)
 {
-	u8 mask = LM3533_ALS_ENABLE_MASK;
 	int ret;
 
-	ret = lm3533_update(als->lm3533, LM3533_REG_ALS_CONF, mask, mask);
+	ret = regmap_set_bits(als->regmap, LM3533_REG_ALS_CONF,
+			      LM3533_ALS_ENABLE_MASK);
 	if (ret)
 		dev_err(&als->pdev->dev, "failed to enable ALS\n");
 
@@ -810,10 +792,10 @@ static int lm3533_als_enable(struct lm3533_als *als)
 
 static int lm3533_als_disable(struct lm3533_als *als)
 {
-	u8 mask = LM3533_ALS_ENABLE_MASK;
 	int ret;
 
-	ret = lm3533_update(als->lm3533, LM3533_REG_ALS_CONF, 0, mask);
+	ret = regmap_clear_bits(als->regmap, LM3533_REG_ALS_CONF,
+				LM3533_ALS_ENABLE_MASK);
 	if (ret)
 		dev_err(&als->pdev->dev, "failed to disable ALS\n");
 
@@ -828,7 +810,6 @@ static const struct iio_info lm3533_als_info = {
 
 static int lm3533_als_probe(struct platform_device *pdev)
 {
-	const struct lm3533_als_platform_data *pdata;
 	struct lm3533 *lm3533;
 	struct lm3533_als *als;
 	struct iio_dev *indio_dev;
@@ -838,12 +819,6 @@ static int lm3533_als_probe(struct platform_device *pdev)
 	if (!lm3533)
 		return -EINVAL;
 
-	pdata = dev_get_platdata(&pdev->dev);
-	if (!pdata) {
-		dev_err(&pdev->dev, "no platform data\n");
-		return -EINVAL;
-	}
-
 	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*als));
 	if (!indio_dev)
 		return -ENOMEM;
@@ -852,25 +827,27 @@ static int lm3533_als_probe(struct platform_device *pdev)
 	indio_dev->channels = lm3533_als_channels;
 	indio_dev->num_channels = ARRAY_SIZE(lm3533_als_channels);
 	indio_dev->name = dev_name(&pdev->dev);
-	iio_device_set_parent(indio_dev, pdev->dev.parent);
 	indio_dev->modes = INDIO_DIRECT_MODE;
 
 	als = iio_priv(indio_dev);
-	als->lm3533 = lm3533;
+	als->regmap = lm3533->regmap;
 	als->pdev = pdev;
-	als->irq = lm3533->irq;
+	als->irq = platform_get_irq_optional(pdev, 0);
+	if (als->irq == -EPROBE_DEFER)
+		return -EPROBE_DEFER;
+
 	atomic_set(&als->zone, 0);
 	mutex_init(&als->thresh_mutex);
 
 	platform_set_drvdata(pdev, indio_dev);
 
-	if (als->irq) {
+	if (als->irq > 0) {
 		ret = lm3533_als_setup_irq(als, indio_dev);
 		if (ret)
 			return ret;
 	}
 
-	ret = lm3533_als_setup(als, pdata);
+	ret = lm3533_als_setup(als);
 	if (ret)
 		goto err_free_irq;
 
@@ -889,7 +866,7 @@ static int lm3533_als_probe(struct platform_device *pdev)
 err_disable:
 	lm3533_als_disable(als);
 err_free_irq:
-	if (als->irq)
+	if (als->irq > 0)
 		free_irq(als->irq, indio_dev);
 
 	return ret;
@@ -903,13 +880,20 @@ static void lm3533_als_remove(struct platform_device *pdev)
 	lm3533_als_set_int_mode(indio_dev, false);
 	iio_device_unregister(indio_dev);
 	lm3533_als_disable(als);
-	if (als->irq)
+	if (als->irq > 0)
 		free_irq(als->irq, indio_dev);
 }
 
+static const struct of_device_id lm3533_als_match_table[] = {
+	{ .compatible = "ti,lm3533-als" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lm3533_als_match_table);
+
 static struct platform_driver lm3533_als_driver = {
 	.driver	= {
 		.name	= "lm3533-als",
+		.of_match_table = lm3533_als_match_table,
 	},
 	.probe		= lm3533_als_probe,
 	.remove		= lm3533_als_remove,
diff --git a/drivers/iio/light/ltr390.c b/drivers/iio/light/ltr390.c
index bc031f2c314190..f1bb066248e6c3 100644
--- a/drivers/iio/light/ltr390.c
+++ b/drivers/iio/light/ltr390.c
@@ -10,12 +10,6 @@
  *
  * Datasheet:
  *   https://optoelectronics.liteon.com/upload/download/DS86-2015-0004/LTR-390UV_Final_%20DS_V1%201.pdf
- *
- * TODO:
- *   - Support for configurable gain and resolution
- *   - Sensor suspend/resume support
- *   - Add support for reading the ALS
- *   - Interrupt support
  */
 
 #include <linux/bitfield.h>
diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c
index fa0d16890c8bfe..302be2fc314c56 100644
--- a/drivers/iio/light/ltr501.c
+++ b/drivers/iio/light/ltr501.c
@@ -9,6 +9,7 @@
  * TODO: IR LED characteristics
  */
 
+#include <linux/array_size.h>
 #include <linux/module.h>
 #include <linux/i2c.h>
 #include <linux/err.h>
@@ -90,10 +91,11 @@ struct ltr501_samp_table {
 #define LTR501_RESERVED_GAIN -1
 
 enum {
-	ltr501 = 0,
-	ltr559,
 	ltr301,
 	ltr303,
+	ltr329,
+	ltr501,
+	ltr559,
 };
 
 struct ltr501_gain {
@@ -140,6 +142,7 @@ struct ltr501_chip_info {
 	u8 als_mode_active;
 	u8 als_gain_mask;
 	u8 als_gain_shift;
+	bool no_irq_support;
 	struct iio_chan_spec const *channels;
 	const int no_channels;
 	const struct iio_info *info;
@@ -178,6 +181,11 @@ static const struct ltr501_samp_table ltr501_ps_samp_table[] = {
 			{500000, 2000000}
 };
 
+static bool ltr501_has_irq_support(const struct ltr501_chip_info *info)
+{
+	return !info->no_irq_support;
+}
+
 static int ltr501_match_samp_freq(const struct ltr501_samp_table *tab,
 					   int len, int val, int val2)
 {
@@ -478,7 +486,7 @@ static int ltr501_write_intr_prst(struct ltr501_data *data,
 		if (ret < 0)
 			return ret;
 
-		/* period should be atleast equal to sampling period */
+		/* period should be at least equal to sampling period */
 		if (period < samp_period)
 			return -EINVAL;
 
@@ -498,7 +506,7 @@ static int ltr501_write_intr_prst(struct ltr501_data *data,
 		if (ret < 0)
 			return ret;
 
-		/* period should be atleast equal to rate */
+		/* period should be at least equal to rate */
 		if (period < samp_period)
 			return -EINVAL;
 
@@ -821,6 +829,9 @@ static int __ltr501_write_raw(struct iio_dev *indio_dev,
 			if (ret < 0)
 				return ret;
 
+			if (!ltr501_has_irq_support(info))
+				return ret;
+
 			/* update persistence count when changing frequency */
 			ret = ltr501_write_intr_prst(data, chan->type,
 						     0, data->als_period);
@@ -840,6 +851,9 @@ static int __ltr501_write_raw(struct iio_dev *indio_dev,
 			if (ret < 0)
 				return ret;
 
+			if (!ltr501_has_irq_support(info))
+				return ret;
+
 			/* update persistence count when changing frequency */
 			ret = ltr501_write_intr_prst(data, chan->type,
 						     0, data->ps_period);
@@ -1205,6 +1219,42 @@ static const struct iio_info ltr301_info = {
 };
 
 static const struct ltr501_chip_info ltr501_chip_info_tbl[] = {
+	[ltr301] = {
+		.partid = 0x08,
+		.als_gain = ltr501_als_gain_tbl,
+		.als_gain_tbl_size = ARRAY_SIZE(ltr501_als_gain_tbl),
+		.als_mode_active = BIT(0) | BIT(1),
+		.als_gain_mask = BIT(3),
+		.als_gain_shift = 3,
+		.info = &ltr301_info,
+		.info_no_irq = &ltr301_info_no_irq,
+		.channels = ltr301_channels,
+		.no_channels = ARRAY_SIZE(ltr301_channels),
+	},
+	[ltr303] = {
+		.partid = 0x0A,
+		.als_gain = ltr559_als_gain_tbl,
+		.als_gain_tbl_size = ARRAY_SIZE(ltr559_als_gain_tbl),
+		.als_mode_active = BIT(0),
+		.als_gain_mask = BIT(2) | BIT(3) | BIT(4),
+		.als_gain_shift = 2,
+		.info = &ltr301_info,
+		.info_no_irq = &ltr301_info_no_irq,
+		.channels = ltr301_channels,
+		.no_channels = ARRAY_SIZE(ltr301_channels),
+	},
+	[ltr329] = {
+		.partid = 0x0A,
+		.als_gain = ltr559_als_gain_tbl,
+		.als_gain_tbl_size = ARRAY_SIZE(ltr559_als_gain_tbl),
+		.als_mode_active = BIT(0),
+		.als_gain_mask = BIT(2) | BIT(3) | BIT(4),
+		.als_gain_shift = 2,
+		.no_irq_support = true,
+		.info_no_irq = &ltr301_info_no_irq,
+		.channels = ltr301_channels,
+		.no_channels = ARRAY_SIZE(ltr301_channels),
+	},
 	[ltr501] = {
 		.partid = 0x08,
 		.als_gain = ltr501_als_gain_tbl,
@@ -1233,30 +1283,6 @@ static const struct ltr501_chip_info ltr501_chip_info_tbl[] = {
 		.channels = ltr501_channels,
 		.no_channels = ARRAY_SIZE(ltr501_channels),
 	},
-	[ltr301] = {
-		.partid = 0x08,
-		.als_gain = ltr501_als_gain_tbl,
-		.als_gain_tbl_size = ARRAY_SIZE(ltr501_als_gain_tbl),
-		.als_mode_active = BIT(0) | BIT(1),
-		.als_gain_mask = BIT(3),
-		.als_gain_shift = 3,
-		.info = &ltr301_info,
-		.info_no_irq = &ltr301_info_no_irq,
-		.channels = ltr301_channels,
-		.no_channels = ARRAY_SIZE(ltr301_channels),
-	},
-	[ltr303] = {
-		.partid = 0x0A,
-		.als_gain = ltr559_als_gain_tbl,
-		.als_gain_tbl_size = ARRAY_SIZE(ltr559_als_gain_tbl),
-		.als_mode_active = BIT(0),
-		.als_gain_mask = BIT(2) | BIT(3) | BIT(4),
-		.als_gain_shift = 2,
-		.info = &ltr301_info,
-		.info_no_irq = &ltr301_info_no_irq,
-		.channels = ltr301_channels,
-		.no_channels = ARRAY_SIZE(ltr301_channels),
-	},
 };
 
 static int ltr501_write_contr(struct ltr501_data *data, u8 als_val, u8 ps_val)
@@ -1369,13 +1395,15 @@ static int ltr501_init(struct ltr501_data *data)
 
 	data->ps_contr = status | LTR501_CONTR_ACTIVE;
 
-	ret = ltr501_read_intr_prst(data, IIO_INTENSITY, &data->als_period);
-	if (ret < 0)
-		return ret;
+	if (ltr501_has_irq_support(data->chip_info)) {
+		ret = ltr501_read_intr_prst(data, IIO_INTENSITY, &data->als_period);
+		if (ret < 0)
+			return ret;
 
-	ret = ltr501_read_intr_prst(data, IIO_PROXIMITY, &data->ps_period);
-	if (ret < 0)
-		return ret;
+		ret = ltr501_read_intr_prst(data, IIO_PROXIMITY, &data->ps_period);
+		if (ret < 0)
+			return ret;
+	}
 
 	return ltr501_write_contr(data, data->als_contr, data->ps_contr);
 }
@@ -1530,6 +1558,11 @@ static int ltr501_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
+	if (!ltr501_has_irq_support(data->chip_info) && client->irq > 0) {
+		client->irq = 0;
+		dev_warn(&client->dev, "chip doesn't support IRQ");
+	}
+
 	if (client->irq > 0) {
 		ret = devm_request_threaded_irq(&client->dev, client->irq,
 						NULL, ltr501_interrupt_handler,
@@ -1538,7 +1571,7 @@ static int ltr501_probe(struct i2c_client *client)
 						"ltr501_thresh_event",
 						indio_dev);
 		if (ret)
-			return ret;
+			goto powerdown_on_error;
 	} else {
 		indio_dev->info = data->chip_info->info_no_irq;
 	}
@@ -1597,19 +1630,21 @@ static const struct acpi_device_id ltr_acpi_match[] = {
 MODULE_DEVICE_TABLE(acpi, ltr_acpi_match);
 
 static const struct i2c_device_id ltr501_id[] = {
-	{ .name = "ltr501", .driver_data = ltr501 },
-	{ .name = "ltr559", .driver_data = ltr559 },
 	{ .name = "ltr301", .driver_data = ltr301 },
 	{ .name = "ltr303", .driver_data = ltr303 },
+	{ .name = "ltr329", .driver_data = ltr329 },
+	{ .name = "ltr501", .driver_data = ltr501 },
+	{ .name = "ltr559", .driver_data = ltr559 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltr501_id);
 
 static const struct of_device_id ltr501_of_match[] = {
-	{ .compatible = "liteon,ltr501", },
-	{ .compatible = "liteon,ltr559", },
 	{ .compatible = "liteon,ltr301", },
 	{ .compatible = "liteon,ltr303", },
+	{ .compatible = "liteon,ltr329", },
+	{ .compatible = "liteon,ltr501", },
+	{ .compatible = "liteon,ltr559", },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ltr501_of_match);
diff --git a/drivers/iio/light/rohm-bu27034.c b/drivers/iio/light/rohm-bu27034.c
index 28d111ac8c0a7b..cd69e3177e2b8a 100644
--- a/drivers/iio/light/rohm-bu27034.c
+++ b/drivers/iio/light/rohm-bu27034.c
@@ -10,6 +10,7 @@
 #include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/i2c.h>
+#include <linux/kthread.h>
 #include <linux/module.h>
 #include <linux/property.h>
 #include <linux/regmap.h>
@@ -137,6 +138,7 @@ static const struct iio_gain_sel_pair bu27034_gains[] = {
 #define BU27034_MEAS_MODE_200MS		2
 #define BU27034_MEAS_MODE_400MS		4
 
+#define BU27034_INT_TIME_US_MIN (55 * USEC_PER_MSEC)
 static const struct iio_itime_sel_mul bu27034_itimes[] = {
 	GAIN_SCALE_ITIME_US(400000, BU27034_MEAS_MODE_400MS, 8),
 	GAIN_SCALE_ITIME_US(200000, BU27034_MEAS_MODE_200MS, 4),
@@ -296,7 +298,7 @@ static int bu27034_get_gain(struct bu27034_data *data, int chan, int *gain)
 	return 0;
 }
 
-static int bu27034_get_int_time(struct bu27034_data *data)
+static int bu27034_get_int_time(struct bu27034_data *data, int *itime)
 {
 	int ret, sel;
 
@@ -304,24 +306,30 @@ static int bu27034_get_int_time(struct bu27034_data *data)
 	if (ret)
 		return ret;
 
-	return iio_gts_find_int_time_by_sel(&data->gts,
-					    sel & BU27034_MASK_MEAS_MODE);
+	ret = iio_gts_find_int_time_by_sel(&data->gts,
+					   sel & BU27034_MASK_MEAS_MODE);
+	if (ret < 0)
+		return ret;
+
+	*itime = ret;
+
+	return 0;
 }
 
 static int _bu27034_get_scale(struct bu27034_data *data, int channel, int *val,
 			      int *val2)
 {
-	int gain, ret;
+	int gain, itime, ret;
 
 	ret = bu27034_get_gain(data, channel, &gain);
 	if (ret)
 		return ret;
 
-	ret = bu27034_get_int_time(data);
-	if (ret < 0)
+	ret = bu27034_get_int_time(data, &itime);
+	if (ret)
 		return ret;
 
-	return iio_gts_get_scale(&data->gts, gain, ret, val, val2);
+	return iio_gts_get_scale(&data->gts, gain, itime, val, val2);
 }
 
 static int bu27034_get_scale(struct bu27034_data *data, int channel, int *val,
@@ -397,12 +405,10 @@ static int bu27034_try_set_int_time(struct bu27034_data *data, int time_us)
 	int ret, int_time_old, i;
 
 	guard(mutex)(&data->mutex);
-	ret = bu27034_get_int_time(data);
-	if (ret < 0)
+	ret = bu27034_get_int_time(data, &int_time_old);
+	if (ret)
 		return ret;
 
-	int_time_old = ret;
-
 	if (!iio_gts_valid_time(&data->gts, time_us)) {
 		dev_err(data->dev, "Unsupported integration time %u\n",
 			time_us);
@@ -415,7 +421,7 @@ static int bu27034_try_set_int_time(struct bu27034_data *data, int time_us)
 	for (i = 0; i < numg; i++) {
 		ret = bu27034_get_gain(data, gains[i].chan, &gains[i].old_gain);
 		if (ret)
-			return 0;
+			return ret;
 
 		ret = iio_gts_find_new_gain_by_old_gain_time(&data->gts,
 							     gains[i].old_gain,
@@ -841,7 +847,7 @@ static int bu27034_meas_set(struct bu27034_data *data, bool en)
 static int bu27034_get_single_result(struct bu27034_data *data, int chan,
 				     int *val)
 {
-	int ret;
+	int ret, itime;
 
 	if (chan < BU27034_CHAN_DATA0 || chan > BU27034_CHAN_DATA1)
 		return -EINVAL;
@@ -850,11 +856,11 @@ static int bu27034_get_single_result(struct bu27034_data *data, int chan,
 	if (ret)
 		return ret;
 
-	ret = bu27034_get_int_time(data);
-	if (ret < 0)
+	ret = bu27034_get_int_time(data, &itime);
+	if (ret)
 		return ret;
 
-	msleep(ret / 1000);
+	msleep(itime / 1000);
 
 	return bu27034_read_result(data, chan, val);
 }
@@ -904,12 +910,10 @@ static int bu27034_calc_mlux(struct bu27034_data *data, __le16 *res, int *val)
 	if (ret)
 		return ret;
 
-	ret = bu27034_get_int_time(data);
-	if (ret < 0)
+	ret = bu27034_get_int_time(data, &meastime);
+	if (ret)
 		return ret;
 
-	meastime = ret;
-
 	d1_d0_ratio_scaled = (unsigned int)ch1 * (unsigned int)gain0 * 100;
 	helper64 = (u64)ch1 * (u64)gain0 * 100LLU;
 
@@ -970,9 +974,9 @@ static int bu27034_read_raw(struct iio_dev *idev,
 	switch (mask) {
 	case IIO_CHAN_INFO_INT_TIME:
 		*val = 0;
-		*val2 = bu27034_get_int_time(data);
-		if (*val2 < 0)
-			return *val2;
+		ret = bu27034_get_int_time(data, val2);
+		if (ret)
+			return ret;
 
 		return IIO_VAL_INT_PLUS_MICRO;
 
@@ -1157,11 +1161,20 @@ static int bu27034_buffer_thread(void *arg)
 {
 	struct iio_dev *idev = arg;
 	struct bu27034_data *data;
-	int wait_ms;
+	int wait_ms, ret;
 
 	data = iio_priv(idev);
 
-	wait_ms = bu27034_get_int_time(data);
+	/*
+	 * If reading the integration time fails, default to the minimum so we
+	 * don't lose samples. This may waste CPU cycles, but as a hardening
+	 * against theoretical, once-in-a-blue-moon error, this should be Ok.
+	 */
+	wait_ms = BU27034_INT_TIME_US_MIN;
+	ret = bu27034_get_int_time(data, &wait_ms);
+	if (ret)
+		dev_warn(data->dev, "Failed to get integration time\n");
+
 	wait_ms /= 1000;
 
 	wait_ms -= BU27034_MEAS_WAIT_PREMATURE_MS;
diff --git a/drivers/iio/light/stk3310.c b/drivers/iio/light/stk3310.c
index 7c8a1d2b2ed0bf..0a16e326e93901 100644
--- a/drivers/iio/light/stk3310.c
+++ b/drivers/iio/light/stk3310.c
@@ -40,7 +40,13 @@
 #define STK3310_REG_PS_DATA_LSB			0x12
 #define STK3310_REG_ALS_DATA_MSB		0x13
 #define STK3310_REG_ALS_DATA_LSB		0x14
+#define STK36C61_REG_RED_DATA_MSB		0x15
+#define STK36C61_REG_GREEN_DATA_MSB		0x17
+#define STK36C61_REG_BLUE_DATA_MSB		0x19
+#define STK36C61_REG_CLEAR_DATA_MSB		0x1B
+#define STK36C61_REG_CLEAR_DATA_LSB		0x1C
 #define STK3310_REG_ID				0x3E
+#define STK36C61_REG_GAINCTRL			0x4E
 #define STK3310_MAX_REG				0x80
 
 #define STK3310_STATE_EN_PS			BIT(0)
@@ -54,6 +60,7 @@
 #define STK3311S34_CHIP_ID_VAL			0x1E
 #define STK3311X_CHIP_ID_VAL			0x12
 #define STK3335_CHIP_ID_VAL			0x51
+#define STK36C61_CHIP_ID_VAL			0x95
 #define STK3310_PSINT_EN			0x01
 #define STK3310_PS_MAX_VAL			0xFFFF
 
@@ -83,6 +90,8 @@ static const struct reg_field stk3310_reg_field_als_gain =
 				REG_FIELD(STK3310_REG_ALSCTRL, 4, 5);
 static const struct reg_field stk3310_reg_field_ps_gain =
 				REG_FIELD(STK3310_REG_PSCTRL, 4, 5);
+static const struct reg_field stk3310_reg_field_clear_gain =
+				REG_FIELD(STK36C61_REG_GAINCTRL, 4, 5);
 static const struct reg_field stk3310_reg_field_als_it =
 				REG_FIELD(STK3310_REG_ALSCTRL, 0, 3);
 static const struct reg_field stk3310_reg_field_ps_it =
@@ -102,6 +111,7 @@ static const u8 stk3310_chip_ids[] = {
 	STK3311X_CHIP_ID_VAL,
 	STK3311_CHIP_ID_VAL,
 	STK3335_CHIP_ID_VAL,
+	STK36C61_CHIP_ID_VAL,
 };
 
 /* Estimate maximum proximity values with regard to measurement scale. */
@@ -132,6 +142,7 @@ struct stk3310_data {
 	struct regmap_field *reg_state;
 	struct regmap_field *reg_als_gain;
 	struct regmap_field *reg_ps_gain;
+	struct regmap_field *reg_clear_gain;
 	struct regmap_field *reg_als_it;
 	struct regmap_field *reg_ps_it;
 	struct regmap_field *reg_int_ps;
@@ -181,24 +192,78 @@ static const struct iio_chan_spec_ext_info stk3310_ext_info[] = {
 	{ }
 };
 
+#define STK3310_LIGHT_CHANNEL {				\
+	.type = IIO_LIGHT,				\
+	.address = STK3310_REG_ALS_DATA_MSB,		\
+	.info_mask_separate =				\
+		BIT(IIO_CHAN_INFO_RAW) |		\
+		BIT(IIO_CHAN_INFO_SCALE) |		\
+		BIT(IIO_CHAN_INFO_INT_TIME),		\
+}
+
+#define STK3310_PROXIMITY_CHANNEL {			\
+	.type = IIO_PROXIMITY,				\
+	.address = STK3310_REG_PS_DATA_MSB,		\
+	.info_mask_separate =				\
+		BIT(IIO_CHAN_INFO_RAW) |		\
+		BIT(IIO_CHAN_INFO_SCALE) |		\
+		BIT(IIO_CHAN_INFO_INT_TIME),		\
+	.event_spec = stk3310_events,			\
+	.num_event_specs = ARRAY_SIZE(stk3310_events),	\
+	.ext_info = stk3310_ext_info,			\
+}
+
+#define STK36C61_INTENSITY_CHANNEL(_mod, _reg) {	\
+	.type = IIO_INTENSITY,				\
+	.address = _reg,				\
+	.modified = 1,					\
+	.channel2 = IIO_MOD_LIGHT_##_mod,		\
+	.info_mask_separate =				\
+		BIT(IIO_CHAN_INFO_RAW) |		\
+		BIT(IIO_CHAN_INFO_SCALE),		\
+	.info_mask_shared_by_type =			\
+		BIT(IIO_CHAN_INFO_INT_TIME),		\
+	.info_mask_shared_by_type_available =		\
+		BIT(IIO_CHAN_INFO_SCALE) |		\
+		BIT(IIO_CHAN_INFO_INT_TIME),		\
+}
+
 static const struct iio_chan_spec stk3310_channels[] = {
-	{
-		.type = IIO_LIGHT,
-		.info_mask_separate =
-			BIT(IIO_CHAN_INFO_RAW) |
-			BIT(IIO_CHAN_INFO_SCALE) |
-			BIT(IIO_CHAN_INFO_INT_TIME),
-	},
-	{
-		.type = IIO_PROXIMITY,
-		.info_mask_separate =
-			BIT(IIO_CHAN_INFO_RAW) |
-			BIT(IIO_CHAN_INFO_SCALE) |
-			BIT(IIO_CHAN_INFO_INT_TIME),
-		.event_spec = stk3310_events,
-		.num_event_specs = ARRAY_SIZE(stk3310_events),
-		.ext_info = stk3310_ext_info,
-	}
+	STK3310_LIGHT_CHANNEL,
+	STK3310_PROXIMITY_CHANNEL,
+};
+
+static const struct iio_chan_spec stk36c61_channels[] = {
+	STK3310_LIGHT_CHANNEL,
+	STK3310_PROXIMITY_CHANNEL,
+	STK36C61_INTENSITY_CHANNEL(RED, STK36C61_REG_RED_DATA_MSB),
+	STK36C61_INTENSITY_CHANNEL(GREEN, STK36C61_REG_GREEN_DATA_MSB),
+	STK36C61_INTENSITY_CHANNEL(BLUE, STK36C61_REG_BLUE_DATA_MSB),
+	STK36C61_INTENSITY_CHANNEL(CLEAR, STK36C61_REG_CLEAR_DATA_MSB),
+};
+
+/**
+ * struct stk3310_chip_info - chip-specific data
+ * @name: device name reported to the IIO core
+ * @channels: channel specification
+ * @num_channels: number of channels
+ */
+struct stk3310_chip_info {
+	const char *name;
+	const struct iio_chan_spec *channels __counted_by_ptr(num_channels);
+	unsigned int num_channels;
+};
+
+static const struct stk3310_chip_info stk3310_chip_info = {
+	.name = STK3310_DRIVER_NAME,
+	.channels = stk3310_channels,
+	.num_channels = ARRAY_SIZE(stk3310_channels),
+};
+
+static const struct stk3310_chip_info stk36c61_chip_info = {
+	.name = "stk36c61",
+	.channels = stk36c61_channels,
+	.num_channels = ARRAY_SIZE(stk36c61_channels),
 };
 
 static IIO_CONST_ATTR(in_illuminance_scale_available, STK3310_SCALE_AVAILABLE);
@@ -370,25 +435,21 @@ static int stk3310_read_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan,
 			    int *val, int *val2, long mask)
 {
-	u8 reg;
 	__be16 buf;
 	int ret;
 	unsigned int index;
 	struct stk3310_data *data = iio_priv(indio_dev);
 	struct i2c_client *client = data->client;
+	struct regmap *map = data->regmap;
 
-	if (chan->type != IIO_LIGHT && chan->type != IIO_PROXIMITY)
+	if (chan->type != IIO_LIGHT && chan->type != IIO_PROXIMITY &&
+	    chan->type != IIO_INTENSITY)
 		return -EINVAL;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		if (chan->type == IIO_LIGHT)
-			reg = STK3310_REG_ALS_DATA_MSB;
-		else
-			reg = STK3310_REG_PS_DATA_MSB;
-
 		mutex_lock(&data->lock);
-		ret = regmap_bulk_read(data->regmap, reg, &buf, sizeof(buf));
+		ret = regmap_bulk_read(map, chan->address, &buf, sizeof(buf));
 		if (ret < 0) {
 			dev_err(&client->dev, "register read failed\n");
 			mutex_unlock(&data->lock);
@@ -398,10 +459,10 @@ static int stk3310_read_raw(struct iio_dev *indio_dev,
 		mutex_unlock(&data->lock);
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_INT_TIME:
-		if (chan->type == IIO_LIGHT)
-			ret = regmap_field_read(data->reg_als_it, &index);
-		else
+		if (chan->type == IIO_PROXIMITY)
 			ret = regmap_field_read(data->reg_ps_it, &index);
+		else
+			ret = regmap_field_read(data->reg_als_it, &index);
 		if (ret < 0)
 			return ret;
 
@@ -409,10 +470,12 @@ static int stk3310_read_raw(struct iio_dev *indio_dev,
 		*val2 = stk3310_it_table[index][1];
 		return IIO_VAL_INT_PLUS_MICRO;
 	case IIO_CHAN_INFO_SCALE:
-		if (chan->type == IIO_LIGHT)
-			ret = regmap_field_read(data->reg_als_gain, &index);
-		else
+		if (chan->type == IIO_PROXIMITY)
 			ret = regmap_field_read(data->reg_ps_gain, &index);
+		else if (chan->channel2 == IIO_MOD_LIGHT_CLEAR)
+			ret = regmap_field_read(data->reg_clear_gain, &index);
+		else
+			ret = regmap_field_read(data->reg_als_gain, &index);
 		if (ret < 0)
 			return ret;
 
@@ -432,7 +495,8 @@ static int stk3310_write_raw(struct iio_dev *indio_dev,
 	int index;
 	struct stk3310_data *data = iio_priv(indio_dev);
 
-	if (chan->type != IIO_LIGHT && chan->type != IIO_PROXIMITY)
+	if (chan->type != IIO_LIGHT && chan->type != IIO_PROXIMITY &&
+	    chan->type != IIO_INTENSITY)
 		return -EINVAL;
 
 	switch (mask) {
@@ -443,10 +507,10 @@ static int stk3310_write_raw(struct iio_dev *indio_dev,
 		if (index < 0)
 			return -EINVAL;
 		mutex_lock(&data->lock);
-		if (chan->type == IIO_LIGHT)
-			ret = regmap_field_write(data->reg_als_it, index);
-		else
+		if (chan->type == IIO_PROXIMITY)
 			ret = regmap_field_write(data->reg_ps_it, index);
+		else
+			ret = regmap_field_write(data->reg_als_it, index);
 		if (ret < 0)
 			dev_err(&data->client->dev,
 				"sensor configuration failed\n");
@@ -460,10 +524,12 @@ static int stk3310_write_raw(struct iio_dev *indio_dev,
 		if (index < 0)
 			return -EINVAL;
 		mutex_lock(&data->lock);
-		if (chan->type == IIO_LIGHT)
-			ret = regmap_field_write(data->reg_als_gain, index);
-		else
+		if (chan->type == IIO_PROXIMITY)
 			ret = regmap_field_write(data->reg_ps_gain, index);
+		else if (chan->channel2 == IIO_MOD_LIGHT_CLEAR)
+			ret = regmap_field_write(data->reg_clear_gain, index);
+		else
+			ret = regmap_field_write(data->reg_als_gain, index);
 		if (ret < 0)
 			dev_err(&data->client->dev,
 				"sensor configuration failed\n");
@@ -474,9 +540,31 @@ static int stk3310_write_raw(struct iio_dev *indio_dev,
 	return -EINVAL;
 }
 
+static int stk3310_read_avail(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      const int **vals, int *type, int *length,
+			      long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		*vals = (const int *)stk3310_scale_table;
+		*length = 2 * ARRAY_SIZE(stk3310_scale_table);
+		*type = IIO_VAL_INT_PLUS_MICRO;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_INT_TIME:
+		*vals = (const int *)stk3310_it_table;
+		*length = 2 * ARRAY_SIZE(stk3310_it_table);
+		*type = IIO_VAL_INT_PLUS_MICRO;
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
 static const struct iio_info stk3310_info = {
 	.read_raw		= stk3310_read_raw,
 	.write_raw		= stk3310_write_raw,
+	.read_avail		= stk3310_read_avail,
 	.attrs			= &stk3310_attribute_group,
 	.read_event_value	= stk3310_read_event,
 	.write_event_value	= stk3310_write_event,
@@ -548,6 +636,7 @@ static bool stk3310_is_volatile_reg(struct device *dev, unsigned int reg)
 	switch (reg) {
 	case STK3310_REG_ALS_DATA_MSB:
 	case STK3310_REG_ALS_DATA_LSB:
+	case STK36C61_REG_RED_DATA_MSB ... STK36C61_REG_CLEAR_DATA_LSB:
 	case STK3310_REG_PS_DATA_LSB:
 	case STK3310_REG_PS_DATA_MSB:
 	case STK3310_REG_FLAG:
@@ -582,6 +671,7 @@ static int stk3310_regmap_init(struct stk3310_data *data)
 	STK3310_REGFIELD(state);
 	STK3310_REGFIELD(als_gain);
 	STK3310_REGFIELD(ps_gain);
+	STK3310_REGFIELD(clear_gain);
 	STK3310_REGFIELD(als_it);
 	STK3310_REGFIELD(ps_it);
 	STK3310_REGFIELD(int_ps);
@@ -635,10 +725,16 @@ static irqreturn_t stk3310_irq_event_handler(int irq, void *private)
 
 static int stk3310_probe(struct i2c_client *client)
 {
+	const struct stk3310_chip_info *chip_info;
+	struct device *dev = &client->dev;
 	int ret;
 	struct iio_dev *indio_dev;
 	struct stk3310_data *data;
 
+	chip_info = i2c_get_match_data(client);
+	if (!chip_info)
+		return dev_err_probe(dev, -ENODATA, "missing driver data\n");
+
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (!indio_dev)
 		return -ENOMEM;
@@ -657,10 +753,10 @@ static int stk3310_probe(struct i2c_client *client)
 		return ret;
 
 	indio_dev->info = &stk3310_info;
-	indio_dev->name = STK3310_DRIVER_NAME;
+	indio_dev->name = chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->channels = stk3310_channels;
-	indio_dev->num_channels = ARRAY_SIZE(stk3310_channels);
+	indio_dev->channels = chip_info->channels;
+	indio_dev->num_channels = chip_info->num_channels;
 
 	ret = stk3310_init(indio_dev);
 	if (ret < 0)
@@ -766,28 +862,29 @@ static DEFINE_SIMPLE_DEV_PM_OPS(stk3310_pm_ops, stk3310_suspend,
 				stk3310_resume);
 
 static const struct i2c_device_id stk3310_i2c_id[] = {
-	{ .name = "STK3013" },
-	{ .name = "STK3310" },
-	{ .name = "STK3311" },
-	{ .name = "STK3335" },
+	{ .name = "stk3013", .driver_data = (kernel_ulong_t)&stk3310_chip_info },
+	{ .name = "stk3310", .driver_data = (kernel_ulong_t)&stk3310_chip_info },
+	{ .name = "stk3311", .driver_data = (kernel_ulong_t)&stk3310_chip_info },
+	{ .name = "stk3335", .driver_data = (kernel_ulong_t)&stk3310_chip_info },
+	{ .name = "stk36c61", .driver_data = (kernel_ulong_t)&stk36c61_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, stk3310_i2c_id);
 
 static const struct acpi_device_id stk3310_acpi_id[] = {
-	{"STK3013", 0},
-	{"STK3310", 0},
-	{"STK3311", 0},
+	{ .id = "STK3013", .driver_data = (kernel_ulong_t)&stk3310_chip_info },
+	{ .id = "STK3310", .driver_data = (kernel_ulong_t)&stk3310_chip_info },
+	{ .id = "STK3311", .driver_data = (kernel_ulong_t)&stk3310_chip_info },
 	{ }
 };
-
 MODULE_DEVICE_TABLE(acpi, stk3310_acpi_id);
 
 static const struct of_device_id stk3310_of_match[] = {
-	{ .compatible = "sensortek,stk3013", },
-	{ .compatible = "sensortek,stk3310", },
-	{ .compatible = "sensortek,stk3311", },
-	{ .compatible = "sensortek,stk3335", },
+	{ .compatible = "sensortek,stk3013", .data = &stk3310_chip_info },
+	{ .compatible = "sensortek,stk3310", .data = &stk3310_chip_info },
+	{ .compatible = "sensortek,stk3311", .data = &stk3310_chip_info },
+	{ .compatible = "sensortek,stk3335", .data = &stk3310_chip_info },
+	{ .compatible = "sensortek,stk36c61", .data = &stk36c61_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, stk3310_of_match);
diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c
index 53fd423aa7aef6..2d041b7530e65b 100644
--- a/drivers/iio/light/tsl2583.c
+++ b/drivers/iio/light/tsl2583.c
@@ -7,10 +7,11 @@
  * Copyright (c) 2016-2017 Brian Masney <masneyb@onstation.org>
  */
 
-#include <linux/kernel.h>
 #include <linux/i2c.h>
 #include <linux/errno.h>
 #include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/kstrtox.h>
 #include <linux/string.h>
 #include <linux/mutex.h>
 #include <linux/unistd.h>
@@ -483,9 +484,12 @@ static ssize_t in_illuminance_input_target_store(struct device *dev,
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct tsl2583_chip *chip = iio_priv(indio_dev);
-	int value;
+	int value, ret;
 
-	if (kstrtoint(buf, 0, &value) || !value)
+	ret = kstrtoint(buf, 0, &value);
+	if (ret)
+		return ret;
+	if (!value)
 		return -EINVAL;
 
 	mutex_lock(&chip->als_mutex);
@@ -503,7 +507,10 @@ static ssize_t in_illuminance_calibrate_store(struct device *dev,
 	struct tsl2583_chip *chip = iio_priv(indio_dev);
 	int value, ret;
 
-	if (kstrtoint(buf, 0, &value) || value != 1)
+	ret = kstrtoint(buf, 0, &value);
+	if (ret)
+		return ret;
+	if (value != 1)
 		return -EINVAL;
 
 	mutex_lock(&chip->als_mutex);
diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c
index 4486a1d9d84dc9..c8a7afc65c1fcd 100644
--- a/drivers/iio/light/tsl2772.c
+++ b/drivers/iio/light/tsl2772.c
@@ -13,6 +13,7 @@
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
+#include <linux/kstrtox.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/property.h>
@@ -951,8 +952,9 @@ static ssize_t in_illuminance0_target_input_store(struct device *dev,
 	u16 value;
 	int ret;
 
-	if (kstrtou16(buf, 0, &value))
-		return -EINVAL;
+	ret = kstrtou16(buf, 0, &value);
+	if (ret)
+		return ret;
 
 	chip->settings.als_cal_target = value;
 	ret = tsl2772_invoke_change(indio_dev);
@@ -970,7 +972,10 @@ static ssize_t in_illuminance0_calibrate_store(struct device *dev,
 	bool value;
 	int ret;
 
-	if (kstrtobool(buf, &value) || !value)
+	ret = kstrtobool(buf, &value);
+	if (ret)
+		return ret;
+	if (!value)
 		return -EINVAL;
 
 	ret = tsl2772_als_calibrate(indio_dev);
@@ -1061,7 +1066,10 @@ static ssize_t in_proximity0_calibrate_store(struct device *dev,
 	bool value;
 	int ret;
 
-	if (kstrtobool(buf, &value) || !value)
+	ret = kstrtobool(buf, &value);
+	if (ret)
+		return ret;
+	if (!value)
 		return -EINVAL;
 
 	ret = tsl2772_prox_cal(indio_dev);
diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c
index 336468d59ea7fe..911f8d1c74d42b 100644
--- a/drivers/iio/light/vcnl4000.c
+++ b/drivers/iio/light/vcnl4000.c
@@ -26,6 +26,7 @@
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
+#include <linux/sysfs.h>
 #include <linux/units.h>
 
 #include <linux/iio/buffer.h>
@@ -1504,7 +1505,7 @@ static ssize_t vcnl4000_read_near_level(struct iio_dev *indio_dev,
 {
 	struct vcnl4000_data *data = iio_priv(indio_dev);
 
-	return sprintf(buf, "%u\n", data->near_level);
+	return sysfs_emit(buf, "%u\n", data->near_level);
 }
 
 static irqreturn_t vcnl4010_irq_thread(int irq, void *p)
diff --git a/drivers/iio/light/veml3328.c b/drivers/iio/light/veml3328.c
index 7ff1753925c475..df36c48204be56 100644
--- a/drivers/iio/light/veml3328.c
+++ b/drivers/iio/light/veml3328.c
@@ -32,6 +32,11 @@
 #define VEML3328_REG_DATA_B		0x07
 #define VEML3328_REG_DATA_IR		0x08
 
+#define VEML3328_CONT_IT_50MSECS	0
+#define VEML3328_CONT_IT_100MSECS	1
+#define VEML3328_CONT_IT_200MSECS	2
+#define VEML3328_CONT_IT_400MSECS	3
+
 #define VEML3328_CONF_IT_MASK		GENMASK(5, 4)
 #define VEML3328_CONF_GAIN_MASK		GENMASK(11, 10)
 
@@ -89,21 +94,40 @@ static const struct iio_chan_spec veml3328_channels[] = {
  * Precomputed scale values (micro units).
  * Formula for calculation: 0.384 * (50000 / IT_us) * (1 / Gain)
  * Gain indexes: 0 (x0.5), 1 (x1), 2 (x2), 3 (x4)
- * IT indexes: 0 (50ms), 1 (100ms), 2 (200ms), 3 (400ms)
  */
-static const int veml3328_scale_vals[4][8] = {
-	{ 0, 768000, 0, 384000, 0, 192000, 0, 96000 },
-	{ 0, 384000, 0, 192000, 0, 96000,  0, 48000 },
-	{ 0, 192000, 0, 96000,  0, 48000,  0, 24000 },
-	{ 0, 96000,  0, 48000,  0, 24000,  0, 12000 },
+static const int veml3328_scale_vals[4][4][2] = {
+	[VEML3328_CONT_IT_50MSECS] = {
+		{ 0, 768000 },
+		{ 0, 384000 },
+		{ 0, 192000 },
+		{ 0, 96000 },
+	},
+	[VEML3328_CONT_IT_100MSECS] = {
+		{ 0, 384000 },
+		{ 0, 192000 },
+		{ 0, 96000 },
+		{ 0, 48000 },
+	},
+	[VEML3328_CONT_IT_200MSECS] = {
+		{ 0, 192000 },
+		{ 0, 96000 },
+		{ 0, 48000 },
+		{ 0, 24000 },
+	},
+	[VEML3328_CONT_IT_400MSECS] = {
+		{ 0, 96000 },
+		{ 0, 48000 },
+		{ 0, 24000 },
+		{ 0, 12000 },
+	},
 };
 
 /* integration times in microseconds */
 static const int veml3328_it_times[][2] = {
-	{ 0, 50 * USEC_PER_MSEC },
-	{ 0, 100 * USEC_PER_MSEC },
-	{ 0, 200 * USEC_PER_MSEC },
-	{ 0, 400 * USEC_PER_MSEC },
+	[VEML3328_CONT_IT_50MSECS] = { 0, 50 * USEC_PER_MSEC },
+	[VEML3328_CONT_IT_100MSECS] = { 0, 100 * USEC_PER_MSEC },
+	[VEML3328_CONT_IT_200MSECS] = { 0, 200 * USEC_PER_MSEC },
+	[VEML3328_CONT_IT_400MSECS] = { 0, 400 * USEC_PER_MSEC },
 };
 
 static int veml3328_power_down(struct veml3328_data *data)
@@ -184,9 +208,8 @@ static int veml3328_read_raw(struct iio_dev *indio_dev,
 		if (it_inx >= ARRAY_SIZE(veml3328_it_times) || gain_inx >= 4)
 			return -EINVAL;
 
-		/* Stride by 2 through the flattened array to match (val, val2) */
-		*val = veml3328_scale_vals[it_inx][gain_inx * 2];
-		*val2 = veml3328_scale_vals[it_inx][gain_inx * 2 + 1];
+		*val = veml3328_scale_vals[it_inx][gain_inx][0];
+		*val2 = veml3328_scale_vals[it_inx][gain_inx][1];
 
 		return IIO_VAL_INT_PLUS_MICRO;
 
@@ -282,8 +305,8 @@ static int veml3328_write_raw(struct iio_dev *indio_dev,
 			return -EINVAL;
 
 		for (i = 0; i < 4; i++) {
-			if (val == veml3328_scale_vals[it_inx][i * 2] &&
-			    val2 == veml3328_scale_vals[it_inx][i * 2 + 1])
+			if (val == veml3328_scale_vals[it_inx][i][0] &&
+			    val2 == veml3328_scale_vals[it_inx][i][1])
 				break;
 		}
 
diff --git a/drivers/iio/light/veml6031x00.c b/drivers/iio/light/veml6031x00.c
new file mode 100644
index 00000000000000..f6308498e42264
--- /dev/null
+++ b/drivers/iio/light/veml6031x00.c
@@ -0,0 +1,1387 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * VEML6031X00 Ambient Light Sensor
+ *
+ * Copyright (c) 2026, Javier Carrasco <javier.carrasco.cruz@gmail.com>
+ */
+
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/limits.h>
+#include <linux/lockdep.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/types.h>
+
+#include <asm/byteorder.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/events.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/iio-gts-helper.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
+
+/* Device registers */
+#define VEML6031X00_REG_CONF0       0x00
+#define VEML6031X00_REG_CONF1       0x01
+#define VEML6031X00_REG_WH_L        0x04
+#define VEML6031X00_REG_WH_H        0x05
+#define VEML6031X00_REG_WL_L        0x06
+#define VEML6031X00_REG_WL_H        0x07
+#define VEML6031X00_REG_ALS_L       0x10
+#define VEML6031X00_REG_ALS_H       0x11
+#define VEML6031X00_REG_IR_L        0x12
+#define VEML6031X00_REG_IR_H        0x13
+#define VEML6031X00_REG_ID_L        0x14
+#define VEML6031X00_REG_ID_H        0x15
+#define VEML6031X00_REG_INT         0x17
+
+/* Bit masks for specific functionality */
+#define VEML6031X00_CONF0_SD        BIT(0)
+#define VEML6031X00_CONF0_AF_TRIG   BIT(2)
+#define VEML6031X00_CONF0_AF        BIT(3)
+#define VEML6031X00_CONF1_ALS_CAL   BIT(0)
+#define VEML6031X00_CONF1_IR_SD     BIT(7)
+#define VEML6031X00_INT_TH_H        BIT(1)
+#define VEML6031X00_INT_TH_L        BIT(2)
+#define VEML6031X00_INT_DRDY        BIT(3)
+#define VEML6031X00_INT_MASK \
+	(VEML6031X00_INT_TH_L | VEML6031X00_INT_TH_H | VEML6031X00_INT_DRDY)
+
+#define VEML6031X00_GAIN_SEL(pd_div4, gain) (((pd_div4) << 2) | (gain))
+
+enum veml6031x00_scan {
+	VEML6031X00_SCAN_ALS,
+	VEML6031X00_SCAN_IR,
+	VEML6031X00_SCAN_TIMESTAMP,
+};
+
+struct veml6031x00_rf {
+	struct regmap_field *gain;
+	struct regmap_field *int_en;
+	struct regmap_field *it;
+	struct regmap_field *pd_div4;
+	struct regmap_field *pers;
+};
+
+struct veml6031x00_chip {
+	const char *name;
+	const unsigned int part_id;
+};
+
+struct veml6031x00_data {
+	struct iio_gts gts;
+	struct regmap *regmap;
+	struct iio_trigger *trig;
+	struct veml6031x00_rf rf;
+	const struct veml6031x00_chip *chip;
+	/*
+	 * Serialize access to scale register fields scattered across multiple
+	 * registers (rf.gain, rf.pd_div4, rf.it) to read and write them as a
+	 * consistent set.
+	 */
+	struct mutex scale_lock;
+	/*
+	 * Serialize access to irq enable/disable by events and trigger
+	 * (shared line).
+	 */
+	struct mutex irq_lock;
+	unsigned int interrupt_users;
+	bool ev_en;
+	bool trig_en;
+};
+
+static const struct iio_itime_sel_mul veml6031x00_it_sel[] = {
+	GAIN_SCALE_ITIME_US(3125, 0, 1),
+	GAIN_SCALE_ITIME_US(6250, 1, 2),
+	GAIN_SCALE_ITIME_US(12500, 2, 4),
+	GAIN_SCALE_ITIME_US(25000, 3, 8),
+	GAIN_SCALE_ITIME_US(50000, 4, 16),
+	GAIN_SCALE_ITIME_US(100000, 5, 32),
+	GAIN_SCALE_ITIME_US(200000, 6, 64),
+	GAIN_SCALE_ITIME_US(400000, 7, 128),
+};
+
+/*
+ * The gain selector encodes (PD_D4 << 2) | GAIN to identify each gain setting.
+ * Gains are multiplied by 8 to work with integers. The values in the iio-gts
+ * tables don't need corrections because the maximum value of the scale refers
+ * to GAIN = x1, and the rest of the values are obtained from the resulting
+ * linear function.
+ * TODO: add support for GAIN_0_165 and GAIN_0_660
+ */
+#define VEML6031X00_SEL_GAIN_0_125 0x07
+#define VEML6031X00_SEL_GAIN_0_250 0x04
+#define VEML6031X00_SEL_GAIN_0_500 0x03
+#define VEML6031X00_SEL_GAIN_1_000 0x00
+#define VEML6031X00_SEL_GAIN_2_000 0x01
+static const struct iio_gain_sel_pair veml6031x00_gain_sel[] = {
+	GAIN_SCALE_GAIN(1, VEML6031X00_SEL_GAIN_0_125),
+	GAIN_SCALE_GAIN(2, VEML6031X00_SEL_GAIN_0_250),
+	GAIN_SCALE_GAIN(4, VEML6031X00_SEL_GAIN_0_500),
+	GAIN_SCALE_GAIN(8, VEML6031X00_SEL_GAIN_1_000),
+	GAIN_SCALE_GAIN(16, VEML6031X00_SEL_GAIN_2_000),
+};
+
+static IIO_CONST_ATTR(in_illuminance_thresh_either_period_available, "1 2 4 8");
+
+static struct attribute *veml6031x00_event_attributes[] = {
+	&iio_const_attr_in_illuminance_thresh_either_period_available.dev_attr.attr,
+	NULL
+};
+
+static const struct attribute_group veml6031x00_event_attr_group = {
+	.attrs = veml6031x00_event_attributes,
+};
+
+/*
+ * The shutdown bits (SD and ALS_IR_SD) are in different registers, and both
+ * must be updated when changing the device power state.
+ */
+static int veml6031x00_set_power(struct veml6031x00_data *data, bool state)
+{
+	int ret;
+
+	ret = regmap_assign_bits(data->regmap, VEML6031X00_REG_CONF0,
+				 VEML6031X00_CONF0_SD, !state);
+	if (ret)
+		return ret;
+
+	return regmap_assign_bits(data->regmap, VEML6031X00_REG_CONF1,
+				  VEML6031X00_CONF1_IR_SD, !state);
+}
+
+static void veml6031x00_als_shutdown_action(void *data)
+{
+	struct veml6031x00_data *priv = data;
+	struct device *dev = regmap_get_device(priv->regmap);
+	int ret;
+
+	ret = veml6031x00_set_power(priv, false);
+	if (ret)
+		dev_warn(dev, "Failed to shut down device: %d\n", ret);
+}
+
+static const struct iio_event_spec veml6031x00_event_spec[] = {
+	{
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_FALLING,
+		.mask_separate = BIT(IIO_EV_INFO_VALUE),
+	}, {
+		.type = IIO_EV_TYPE_THRESH,
+		.dir = IIO_EV_DIR_EITHER,
+		.mask_separate = BIT(IIO_EV_INFO_PERIOD) |
+				 BIT(IIO_EV_INFO_ENABLE),
+	},
+};
+
+static const struct iio_chan_spec veml6031x00_channels[] = {
+	{
+		.type = IIO_LIGHT,
+		.address = VEML6031X00_REG_ALS_L,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_INT_TIME) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_INT_TIME) |
+						BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = VEML6031X00_SCAN_ALS,
+		.scan_type = {
+			.format = IIO_SCAN_FORMAT_UNSIGNED_INT,
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
+	},
+	{
+		.type = IIO_INTENSITY,
+		.address = VEML6031X00_REG_IR_L,
+		.modified = 1,
+		.channel2 = IIO_MOD_LIGHT_IR,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.scan_index = VEML6031X00_SCAN_IR,
+		.scan_type = {
+			.format = IIO_SCAN_FORMAT_UNSIGNED_INT,
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(VEML6031X00_SCAN_TIMESTAMP),
+};
+
+static const struct iio_chan_spec veml6031x00_channels_irq[] = {
+	{
+		.type = IIO_LIGHT,
+		.address = VEML6031X00_REG_ALS_L,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_INT_TIME) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_separate_available = BIT(IIO_CHAN_INFO_INT_TIME) |
+						BIT(IIO_CHAN_INFO_SCALE),
+		.event_spec = veml6031x00_event_spec,
+		.num_event_specs = ARRAY_SIZE(veml6031x00_event_spec),
+		.scan_index = VEML6031X00_SCAN_ALS,
+		.scan_type = {
+			.format = IIO_SCAN_FORMAT_UNSIGNED_INT,
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
+	},
+	{
+		.type = IIO_INTENSITY,
+		.address = VEML6031X00_REG_IR_L,
+		.modified = 1,
+		.channel2 = IIO_MOD_LIGHT_IR,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
+		.scan_index = VEML6031X00_SCAN_IR,
+		.scan_type = {
+			.format = IIO_SCAN_FORMAT_UNSIGNED_INT,
+			.realbits = 16,
+			.storagebits = 16,
+			.endianness = IIO_LE,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(VEML6031X00_SCAN_TIMESTAMP),
+};
+
+static const struct regmap_range veml6031x00_readable_ranges[] = {
+	regmap_reg_range(VEML6031X00_REG_CONF0, VEML6031X00_REG_CONF1),
+	regmap_reg_range(VEML6031X00_REG_WH_L, VEML6031X00_REG_WL_H),
+	regmap_reg_range(VEML6031X00_REG_ALS_L, VEML6031X00_REG_ID_H),
+	regmap_reg_range(VEML6031X00_REG_INT, VEML6031X00_REG_INT),
+};
+
+static const struct regmap_access_table veml6031x00_readable_table = {
+	.yes_ranges = veml6031x00_readable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(veml6031x00_readable_ranges),
+};
+
+static const struct regmap_range veml6031x00_writable_ranges[] = {
+	regmap_reg_range(VEML6031X00_REG_CONF0, VEML6031X00_REG_CONF1),
+	regmap_reg_range(VEML6031X00_REG_WH_L, VEML6031X00_REG_WL_H),
+};
+
+static const struct regmap_access_table veml6031x00_writable_table = {
+	.yes_ranges = veml6031x00_writable_ranges,
+	.n_yes_ranges = ARRAY_SIZE(veml6031x00_writable_ranges),
+};
+
+static const struct regmap_range veml6031x00_volatile_ranges[] = {
+	/* AF_TRIG in CONF0 is volatile */
+	regmap_reg_range(VEML6031X00_REG_CONF0, VEML6031X00_REG_CONF0),
+	regmap_reg_range(VEML6031X00_REG_ALS_L, VEML6031X00_REG_IR_H),
+	regmap_reg_range(VEML6031X00_REG_INT, VEML6031X00_REG_INT),
+};
+
+static const struct regmap_access_table veml6031x00_volatile_table = {
+	.yes_ranges = veml6031x00_volatile_ranges,
+	.n_yes_ranges = ARRAY_SIZE(veml6031x00_volatile_ranges),
+};
+
+static const struct regmap_range veml6031x00_precious_ranges[] = {
+	regmap_reg_range(VEML6031X00_REG_INT, VEML6031X00_REG_INT),
+};
+
+static const struct regmap_access_table veml6031x00_precious_table = {
+	.yes_ranges = veml6031x00_precious_ranges,
+	.n_yes_ranges = ARRAY_SIZE(veml6031x00_precious_ranges),
+};
+
+static const struct regmap_config veml6031x00_regmap_config = {
+	.name = "veml6031x00_regmap",
+	.reg_bits = 8,
+	.val_bits = 8,
+	.rd_table = &veml6031x00_readable_table,
+	.wr_table = &veml6031x00_writable_table,
+	.volatile_table = &veml6031x00_volatile_table,
+	.precious_table = &veml6031x00_precious_table,
+	.max_register = VEML6031X00_REG_INT,
+	.cache_type = REGCACHE_MAPLE,
+};
+
+static const struct reg_field veml6031x00_rf_int_en =
+	REG_FIELD(VEML6031X00_REG_CONF0, 1, 1);
+
+static const struct reg_field veml6031x00_rf_it =
+	REG_FIELD(VEML6031X00_REG_CONF0, 4, 6);
+
+static const struct reg_field veml6031x00_rf_pers =
+	REG_FIELD(VEML6031X00_REG_CONF1, 1, 2);
+
+static const struct reg_field veml6031x00_rf_gain =
+	REG_FIELD(VEML6031X00_REG_CONF1, 3, 4);
+
+static const struct reg_field veml6031x00_rf_pd_div4 =
+	REG_FIELD(VEML6031X00_REG_CONF1, 6, 6);
+
+static int veml6031x00_regfield_init(struct veml6031x00_data *data)
+{
+	struct regmap *map = data->regmap;
+	struct device *dev = regmap_get_device(map);
+	struct regmap_field *rm_field;
+	struct veml6031x00_rf *rf = &data->rf;
+
+	rm_field = devm_regmap_field_alloc(dev, map, veml6031x00_rf_gain);
+	if (IS_ERR(rm_field))
+		return PTR_ERR(rm_field);
+	rf->gain = rm_field;
+
+	rm_field = devm_regmap_field_alloc(dev, map, veml6031x00_rf_it);
+	if (IS_ERR(rm_field))
+		return PTR_ERR(rm_field);
+	rf->it = rm_field;
+
+	rm_field = devm_regmap_field_alloc(dev, map, veml6031x00_rf_int_en);
+	if (IS_ERR(rm_field))
+		return PTR_ERR(rm_field);
+	rf->int_en = rm_field;
+
+	rm_field = devm_regmap_field_alloc(dev, map, veml6031x00_rf_pd_div4);
+	if (IS_ERR(rm_field))
+		return PTR_ERR(rm_field);
+	rf->pd_div4 = rm_field;
+
+	rm_field = devm_regmap_field_alloc(dev, map, veml6031x00_rf_pers);
+	if (IS_ERR(rm_field))
+		return PTR_ERR(rm_field);
+	rf->pers = rm_field;
+
+	return 0;
+}
+
+static int __veml6031x00_get_it(struct veml6031x00_data *data, int *val2)
+{
+	unsigned int it_idx;
+	int ret;
+
+	ret = regmap_field_read(data->rf.it, &it_idx);
+	if (ret)
+		return ret;
+
+	ret = iio_gts_find_int_time_by_sel(&data->gts, it_idx);
+	if (ret < 0)
+		return ret;
+
+	*val2 = ret;
+
+	return IIO_VAL_INT_PLUS_MICRO;
+}
+
+static int veml6031x00_get_it(struct veml6031x00_data *data, int *val2)
+{
+	guard(mutex)(&data->scale_lock);
+
+	return __veml6031x00_get_it(data, val2);
+}
+
+static int veml6031x00_write_gain(struct veml6031x00_data *data, unsigned int gain)
+{
+	int ret;
+
+	ret = regmap_field_write(data->rf.pd_div4, gain >> 2);
+	if (ret)
+		return ret;
+
+	return regmap_field_write(data->rf.gain, gain & 0x03);
+}
+
+static int veml6031x00_set_it(struct iio_dev *iio, int val, int val2)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct device *dev = regmap_get_device(data->regmap);
+	int ret, gain_sel, new_gain, prev_gain, prev_it;
+	unsigned int gain_reg, it_idx, pd_div4;
+	struct iio_gts *gts = &data->gts;
+	bool gain_in_range;
+
+	if (val || !iio_gts_valid_time(gts, val2))
+		return -EINVAL;
+
+	guard(mutex)(&data->scale_lock);
+
+	ret = regmap_field_read(data->rf.it, &it_idx);
+	if (ret)
+		return ret;
+
+	ret = regmap_field_read(data->rf.gain, &gain_reg);
+	if (ret)
+		return ret;
+
+	ret = regmap_field_read(data->rf.pd_div4, &pd_div4);
+	if (ret)
+		return ret;
+
+	prev_it = iio_gts_find_int_time_by_sel(gts, it_idx);
+	if (prev_it < 0)
+		return prev_it;
+
+	if (prev_it == val2)
+		return 0;
+
+	gain_sel = VEML6031X00_GAIN_SEL(pd_div4, gain_reg);
+	prev_gain = iio_gts_find_gain_by_sel(gts, gain_sel);
+	if (prev_gain < 0)
+		return prev_gain;
+
+	ret = iio_gts_find_new_gain_by_gain_time_min(gts, prev_gain, prev_it, val2,
+						     &new_gain, &gain_in_range);
+	if (ret)
+		return ret;
+
+	if (!gain_in_range)
+		dev_dbg(dev, "Optimal gain out of range\n");
+
+	ret = iio_gts_find_sel_by_int_time(gts, val2);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_field_write(data->rf.it, ret);
+	if (ret)
+		return ret;
+
+	gain_sel = iio_gts_find_sel_by_gain(gts, new_gain);
+	if (gain_sel < 0)
+		return gain_sel;
+
+	return veml6031x00_write_gain(data, gain_sel);
+}
+
+static int veml6031x00_read_period(struct iio_dev *iio, int *val)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_field_read(data->rf.pers, &regval);
+	if (ret)
+		return ret;
+
+	*val = BIT(regval);
+
+	return IIO_VAL_INT;
+}
+
+static int veml6031x00_write_period(struct iio_dev *iio, int val)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+
+	if (val < 0 || val > 8 || !is_power_of_2(val))
+		return -EINVAL;
+
+	return regmap_field_write(data->rf.pers, ilog2(val));
+}
+
+static int veml6031x00_set_scale(struct iio_dev *iio, int val, int val2)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct iio_gts *gts = &data->gts;
+	int gain_sel, it_sel, ret;
+
+	ret = iio_gts_find_gain_time_sel_for_scale(gts, val, val2, &gain_sel, &it_sel);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&data->scale_lock);
+
+	ret = regmap_field_write(data->rf.it, it_sel);
+	if (ret)
+		return ret;
+
+	return veml6031x00_write_gain(data, gain_sel);
+}
+
+static int veml6031x00_get_scale(struct veml6031x00_data *data, int *val, int *val2)
+{
+	unsigned int gain_reg, pd_div4, it_reg;
+	int gain, it, ret, sel;
+
+	scoped_guard(mutex, &data->scale_lock) {
+		ret = regmap_field_read(data->rf.gain, &gain_reg);
+		if (ret)
+			return ret;
+
+		ret = regmap_field_read(data->rf.pd_div4, &pd_div4);
+		if (ret)
+			return ret;
+
+		sel = VEML6031X00_GAIN_SEL(pd_div4, gain_reg);
+		gain = iio_gts_find_gain_by_sel(&data->gts, sel);
+		if (gain < 0)
+			return gain;
+
+		ret = regmap_field_read(data->rf.it, &it_reg);
+		if (ret)
+			return ret;
+	}
+
+	it = iio_gts_find_int_time_by_sel(&data->gts, it_reg);
+	if (it < 0)
+		return it;
+
+	ret = iio_gts_get_scale(&data->gts, gain, it, val, val2);
+	if (ret)
+		return ret;
+
+	return IIO_VAL_INT_PLUS_NANO;
+}
+
+static int veml6031x00_read_th(struct iio_dev *iio, enum iio_event_direction dir,
+			       int *val, int *val2)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	__le16 regval;
+	int ret;
+
+	if (dir == IIO_EV_DIR_RISING)
+		ret = regmap_bulk_read(data->regmap, VEML6031X00_REG_WH_L,
+				       &regval, sizeof(regval));
+	else
+		ret = regmap_bulk_read(data->regmap, VEML6031X00_REG_WL_L,
+				       &regval, sizeof(regval));
+	if (ret)
+		return ret;
+
+	*val = le16_to_cpu(regval);
+
+	return IIO_VAL_INT;
+}
+
+static int veml6031x00_write_th(struct iio_dev *iio, enum iio_event_direction dir,
+				int val, int val2)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct device *dev = regmap_get_device(data->regmap);
+	__le16 regval;
+	int ret;
+
+	if (val < 0 || val > U16_MAX || val2)
+		return -EINVAL;
+
+	regval = cpu_to_le16(val);
+
+	if (dir == IIO_EV_DIR_RISING) {
+		ret = regmap_bulk_write(data->regmap, VEML6031X00_REG_WH_L,
+					&regval, sizeof(regval));
+		if (ret)
+			dev_dbg(dev, "Failed to set high threshold %d\n", ret);
+	} else {
+		ret = regmap_bulk_write(data->regmap, VEML6031X00_REG_WL_L,
+					&regval, sizeof(regval));
+		if (ret)
+			dev_dbg(dev, "Failed to set low threshold %d\n", ret);
+	}
+
+	return ret;
+}
+
+static int veml6031x00_single_read(struct iio_dev *iio, enum iio_chan_type type,
+				   int *val)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	unsigned int addr;
+	int it_usec, ret;
+	__le16 regval;
+
+	switch (type) {
+	case IIO_LIGHT:
+		addr = VEML6031X00_REG_ALS_L;
+		break;
+	case IIO_INTENSITY:
+		addr = VEML6031X00_REG_IR_L;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(iio, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	guard(mutex)(&data->scale_lock);
+
+	PM_RUNTIME_ACQUIRE_AUTOSUSPEND(regmap_get_device(data->regmap), pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret)
+		return ret;
+
+	ret = __veml6031x00_get_it(data, &it_usec);
+	if (ret < 0)
+		return ret;
+
+	/* integration time + 10% to ensure completion */
+	fsleep(it_usec + (it_usec / 10));
+
+	ret = regmap_bulk_read(data->regmap, addr, &regval, sizeof(regval));
+	if (ret)
+		return ret;
+
+	*val = le16_to_cpu(regval);
+
+	return IIO_VAL_INT;
+}
+
+static int veml6031x00_read_raw(struct iio_dev *iio,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		return veml6031x00_single_read(iio, chan->type, val);
+	case IIO_CHAN_INFO_INT_TIME:
+		*val = 0;
+		return veml6031x00_get_it(data, val2);
+	case IIO_CHAN_INFO_SCALE:
+		return veml6031x00_get_scale(data, val, val2);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int veml6031x00_read_avail(struct iio_dev *iio,
+				  struct iio_chan_spec const *chan,
+				  const int **vals, int *type, int *length,
+				  long mask)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_INT_TIME:
+		return iio_gts_avail_times(&data->gts, vals, type, length);
+	case IIO_CHAN_INFO_SCALE:
+		return iio_gts_all_avail_scales(&data->gts, vals, type, length);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int veml6031x00_write_raw(struct iio_dev *iio,
+				 struct iio_chan_spec const *chan,
+				 int val, int val2, long mask)
+{
+	IIO_DEV_ACQUIRE_DIRECT_MODE(iio, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_INT_TIME:
+		return veml6031x00_set_it(iio, val, val2);
+	case IIO_CHAN_INFO_SCALE:
+		return veml6031x00_set_scale(iio, val, val2);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int veml6031x00_write_raw_get_fmt(struct iio_dev *indio_dev,
+					 struct iio_chan_spec const *chan,
+					 long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		return IIO_VAL_INT_PLUS_NANO;
+	case IIO_CHAN_INFO_INT_TIME:
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int veml6031x00_set_interrupt(struct veml6031x00_data *data, bool state)
+	__must_hold(&data->irq_lock)
+{
+	int ret;
+
+	lockdep_assert_held(&data->irq_lock);
+
+	if (state) {
+		data->interrupt_users++;
+		if (data->interrupt_users > 1)
+			return 0;
+	} else {
+		if (!data->interrupt_users)
+			return 0;
+
+		data->interrupt_users--;
+		if (data->interrupt_users > 0)
+			return 0;
+	}
+
+	ret = regmap_field_write(data->rf.int_en, state);
+	if (ret) {
+		if (state)
+			data->interrupt_users--;
+		else
+			data->interrupt_users++;
+	}
+
+	return ret;
+}
+
+static int veml6031x00_read_event_val(struct iio_dev *iio,
+				      const struct iio_chan_spec *chan,
+				      enum iio_event_type type,
+				      enum iio_event_direction dir,
+				      enum iio_event_info info,
+				      int *val, int *val2)
+{
+	switch (type) {
+	case IIO_EV_TYPE_THRESH:
+		if (dir == IIO_EV_DIR_EITHER && info == IIO_EV_INFO_PERIOD)
+			return veml6031x00_read_period(iio, val);
+
+		return veml6031x00_read_th(iio, dir, val, val2);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int veml6031x00_write_event_val(struct iio_dev *iio,
+				       const struct iio_chan_spec *chan,
+				       enum iio_event_type type,
+				       enum iio_event_direction dir,
+				       enum iio_event_info info,
+				       int val, int val2)
+{
+	switch (info) {
+	case IIO_EV_INFO_VALUE:
+		return veml6031x00_write_th(iio, dir, val, val2);
+	case IIO_EV_INFO_PERIOD:
+		return veml6031x00_write_period(iio, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int veml6031x00_read_event_config(struct iio_dev *iio,
+					 const struct iio_chan_spec *chan,
+					 enum iio_event_type type,
+					 enum iio_event_direction dir)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+
+	guard(mutex)(&data->irq_lock);
+
+	return data->ev_en;
+}
+
+static int veml6031x00_event_enable(struct veml6031x00_data *data)
+{
+	struct device *dev = regmap_get_device(data->regmap);
+	int ret;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return ret;
+
+	ret = veml6031x00_set_interrupt(data, true);
+	if (ret) {
+		pm_runtime_put_autosuspend(dev);
+		return ret;
+	}
+
+	data->ev_en = true;
+
+	return 0;
+}
+
+static int veml6031x00_event_disable(struct veml6031x00_data *data)
+{
+	struct device *dev = regmap_get_device(data->regmap);
+	int ret;
+
+	ret = veml6031x00_set_interrupt(data, false);
+	if (ret)
+		return ret;
+
+	data->ev_en = false;
+	pm_runtime_put_autosuspend(dev);
+
+	return 0;
+}
+
+static int veml6031x00_write_event_config(struct iio_dev *iio,
+					  const struct iio_chan_spec *chan,
+					  enum iio_event_type type,
+					  enum iio_event_direction dir,
+					  bool state)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+
+	guard(mutex)(&data->irq_lock);
+
+	/* avoid multiple increments/decrements from one source */
+	if (state == data->ev_en)
+		return 0;
+
+	if (state)
+		return veml6031x00_event_enable(data);
+
+	return veml6031x00_event_disable(data);
+}
+
+static void veml6031x00_disable_event_action(void *arg)
+{
+	struct veml6031x00_data *data = arg;
+	struct device *dev = regmap_get_device(data->regmap);
+	int ret;
+
+	guard(mutex)(&data->irq_lock);
+
+	if (!data->ev_en)
+		return;
+
+	ret = veml6031x00_set_interrupt(data, false);
+	if (ret)
+		dev_err(dev, "Failed to disable events: %d\n", ret);
+	else
+		data->ev_en = false;
+
+	pm_runtime_put_sync(dev);
+}
+
+static const struct iio_info veml6031x00_info = {
+	.read_raw = veml6031x00_read_raw,
+	.read_avail = veml6031x00_read_avail,
+	.write_raw = veml6031x00_write_raw,
+	.write_raw_get_fmt = veml6031x00_write_raw_get_fmt,
+	.read_event_value = veml6031x00_read_event_val,
+	.write_event_value = veml6031x00_write_event_val,
+	.read_event_config = veml6031x00_read_event_config,
+	.write_event_config = veml6031x00_write_event_config,
+	.event_attrs = &veml6031x00_event_attr_group,
+};
+
+static const struct iio_info veml6031x00_info_no_irq = {
+	.read_raw = veml6031x00_read_raw,
+	.read_avail = veml6031x00_read_avail,
+	.write_raw = veml6031x00_write_raw,
+	.write_raw_get_fmt = veml6031x00_write_raw_get_fmt,
+};
+
+static irqreturn_t veml6031x00_irq(int irq, void *private)
+{
+	struct iio_dev *iio = private;
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct device *dev = regmap_get_device(data->regmap);
+	s64 timestamp;
+	unsigned int regval;
+	int ret;
+	bool trigger_poll;
+
+	ret = pm_runtime_get_if_active(dev);
+	if (ret <= 0)
+		return IRQ_NONE;
+
+	scoped_guard(mutex, &data->irq_lock) {
+		ret = regmap_read(data->regmap, VEML6031X00_REG_INT, &regval);
+		if (ret) {
+			dev_dbg(dev, "Failed to read interrupt register %d\n", ret);
+			pm_runtime_put(dev);
+			return IRQ_NONE;
+		}
+
+		if (!(regval & VEML6031X00_INT_MASK)) {
+			pm_runtime_put(dev);
+			return IRQ_NONE;
+		}
+
+		timestamp = iio_get_time_ns(iio);
+
+		if (data->ev_en &&
+		    (regval & (VEML6031X00_INT_TH_H | VEML6031X00_INT_TH_L))) {
+			if (regval & VEML6031X00_INT_TH_H)
+				iio_push_event(iio,
+					       IIO_UNMOD_EVENT_CODE(IIO_LIGHT, 0,
+								    IIO_EV_TYPE_THRESH,
+								    IIO_EV_DIR_RISING),
+					       timestamp);
+			if (regval & VEML6031X00_INT_TH_L)
+				iio_push_event(iio,
+					       IIO_UNMOD_EVENT_CODE(IIO_LIGHT, 0,
+								    IIO_EV_TYPE_THRESH,
+								    IIO_EV_DIR_FALLING),
+					       timestamp);
+		}
+
+		trigger_poll = data->trig_en && (regval & VEML6031X00_INT_DRDY);
+	}
+
+	/*
+	 * iio_trigger_poll_nested() must be called with irq_lock released:
+	 * it runs trig_handler() synchronously in this thread, which calls
+	 * reenable() on completion, and that callback also takes irq_lock.
+	 * iio_pollfunc_store_time() is not called for our own trigger, so the
+	 * timestamp is stored here.
+	 */
+	if (trigger_poll) {
+		iio->pollfunc->timestamp = timestamp;
+		iio_trigger_poll_nested(data->trig);
+	}
+
+	pm_runtime_put(dev);
+
+	return IRQ_HANDLED;
+}
+
+static int veml6031x00_buffer_preenable(struct iio_dev *iio)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct device *dev = regmap_get_device(data->regmap);
+	int ret, it_usec;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return ret;
+
+	ret = veml6031x00_get_it(data, &it_usec);
+	if (ret < 0) {
+		pm_runtime_put_autosuspend(dev);
+		return ret;
+	}
+
+	/*
+	 * Wait one integration period + 10% margin so the first triggered
+	 * read does not race with the sensor completing its first conversion
+	 * after power-on.
+	 */
+	fsleep(it_usec + (it_usec / 10));
+
+	return 0;
+}
+
+static int veml6031x00_buffer_postdisable(struct iio_dev *iio)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+
+	pm_runtime_put_autosuspend(regmap_get_device(data->regmap));
+
+	return 0;
+}
+
+static int __veml6031x00_set_trigger_state(struct veml6031x00_data *data,
+					   bool state)
+	__must_hold(&data->irq_lock)
+{
+	int ret;
+
+	lockdep_assert_held(&data->irq_lock);
+
+	ret = veml6031x00_set_interrupt(data, state);
+	if (ret)
+		return ret;
+
+	/* The AF bit must be updated before updating AF_TRIG */
+	ret = regmap_assign_bits(data->regmap, VEML6031X00_REG_CONF0,
+				 VEML6031X00_CONF0_AF, state);
+	if (ret)
+		goto disable_trigger;
+
+	ret = regmap_assign_bits(data->regmap, VEML6031X00_REG_CONF0,
+				 VEML6031X00_CONF0_AF_TRIG, state);
+	if (ret) {
+		regmap_assign_bits(data->regmap, VEML6031X00_REG_CONF0,
+				   VEML6031X00_CONF0_AF, false);
+		goto disable_trigger;
+	}
+
+	data->trig_en = state;
+
+	return 0;
+
+disable_trigger:
+	data->trig_en = false;
+	if (state)
+		veml6031x00_set_interrupt(data, false);
+	return ret;
+}
+
+static int veml6031x00_set_trigger_state(struct iio_trigger *trig, bool state)
+{
+	struct iio_dev *iio = iio_trigger_get_drvdata(trig);
+	struct veml6031x00_data *data = iio_priv(iio);
+
+	guard(mutex)(&data->irq_lock);
+
+	if (state == data->trig_en)
+		return 0;
+
+	return __veml6031x00_set_trigger_state(data, state);
+}
+
+static const struct iio_buffer_setup_ops veml6031x00_buffer_setup_ops = {
+	.preenable = veml6031x00_buffer_preenable,
+	.postdisable = veml6031x00_buffer_postdisable,
+};
+
+static void veml6031x00_trigger_reenable(struct iio_trigger *trig)
+{
+	struct iio_dev *iio = iio_trigger_get_drvdata(trig);
+	struct veml6031x00_data *data = iio_priv(iio);
+	int ret;
+
+	guard(mutex)(&data->irq_lock);
+
+	if (!data->trig_en)
+		return;
+
+	ret = regmap_assign_bits(data->regmap, VEML6031X00_REG_CONF0,
+				 VEML6031X00_CONF0_AF_TRIG, true);
+	if (ret)
+		dev_err(regmap_get_device(data->regmap),
+			"Failed to reenable trigger: %d\n", ret);
+}
+
+static const struct iio_trigger_ops veml6031x00_trigger_ops = {
+	.validate_device = iio_trigger_validate_own_device,
+	.set_trigger_state = veml6031x00_set_trigger_state,
+	.reenable = veml6031x00_trigger_reenable,
+};
+
+static irqreturn_t veml6031x00_trig_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *iio = pf->indio_dev;
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct regmap *map = data->regmap;
+	IIO_DECLARE_BUFFER_WITH_TS(__le16, scan, 2) = { };
+	unsigned int i;
+	int ch, ret;
+
+	if (test_bit(VEML6031X00_SCAN_ALS, iio->active_scan_mask) &&
+	    test_bit(VEML6031X00_SCAN_IR, iio->active_scan_mask)) {
+		ret = regmap_bulk_read(map, VEML6031X00_REG_ALS_L,
+				       scan, 2 * sizeof(*scan));
+		if (ret)
+			goto done;
+	} else {
+		i = 0;
+		iio_for_each_active_channel(iio, ch) {
+			ret = regmap_bulk_read(map, iio->channels[ch].address,
+					       &scan[i++], sizeof(*scan));
+			if (ret)
+				goto done;
+		}
+	}
+
+	iio_push_to_buffers_with_ts(iio, scan, sizeof(scan), pf->timestamp);
+
+done:
+	iio_trigger_notify_done(iio->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int veml6031x00_validate_part_id(struct veml6031x00_data *data)
+{
+	struct regmap *map = data->regmap;
+	struct device *dev = regmap_get_device(map);
+	unsigned int part_id;
+	int ret;
+	__le16 regval;
+
+	ret = regmap_bulk_read(map, VEML6031X00_REG_ID_L, &regval, sizeof(regval));
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to read ID\n");
+
+	part_id = le16_to_cpu(regval);
+	if (part_id != data->chip->part_id)
+		dev_info(dev, "Unknown or unexpected ID %04x\n", part_id);
+
+	return 0;
+}
+
+static int veml6031x00_hw_init(struct veml6031x00_data *data)
+{
+	struct regmap *map = data->regmap;
+	struct device *dev = regmap_get_device(map);
+	unsigned int val;
+	__le16 regval;
+	int ret;
+
+	ret = regmap_assign_bits(data->regmap, VEML6031X00_REG_CONF0,
+				 VEML6031X00_CONF0_AF | VEML6031X00_CONF0_AF_TRIG, 0);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to disable trigger\n");
+
+	ret = regmap_set_bits(data->regmap, VEML6031X00_REG_CONF1,
+			      VEML6031X00_CONF1_ALS_CAL);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable internal calibration\n");
+
+	regval = 0;
+	ret = regmap_bulk_write(map, VEML6031X00_REG_WL_L, &regval, sizeof(regval));
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to set low threshold\n");
+
+	regval = cpu_to_le16(U16_MAX);
+	ret = regmap_bulk_write(map, VEML6031X00_REG_WH_L, &regval, sizeof(regval));
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to set high threshold\n");
+
+	ret = regmap_field_write(data->rf.int_en, 0);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(map, VEML6031X00_REG_INT, &val);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to clear interrupts\n");
+
+	return 0;
+}
+
+static int veml6031x00_setup_trigger(struct iio_dev *iio)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct device *dev = regmap_get_device(data->regmap);
+
+	data->trig = devm_iio_trigger_alloc(dev, "%s-drdy%d",
+					    iio->name, iio_device_id(iio));
+	if (!data->trig)
+		return -ENOMEM;
+
+	data->trig->ops = &veml6031x00_trigger_ops;
+	iio_trigger_set_drvdata(data->trig, iio);
+
+	return devm_iio_trigger_register(dev, data->trig);
+}
+
+static int veml6031x00_init_iiodev(struct i2c_client *i2c, struct iio_dev *iio)
+{
+	struct veml6031x00_data *data = iio_priv(iio);
+	struct device *dev = regmap_get_device(data->regmap);
+	int ret;
+
+	iio->name = data->chip->name;
+	iio->modes = INDIO_DIRECT_MODE;
+
+	if (!i2c->irq) {
+		iio->channels = veml6031x00_channels;
+		iio->num_channels = ARRAY_SIZE(veml6031x00_channels);
+		iio->info = &veml6031x00_info_no_irq;
+
+		return 0;
+	}
+
+	iio->channels = veml6031x00_channels_irq;
+	iio->num_channels = ARRAY_SIZE(veml6031x00_channels_irq);
+	iio->info = &veml6031x00_info;
+
+	ret = veml6031x00_setup_trigger(iio);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, veml6031x00_disable_event_action, data);
+}
+
+static int veml6031x00_probe(struct i2c_client *i2c)
+{
+	struct device *dev = &i2c->dev;
+	struct veml6031x00_data *data;
+	struct iio_dev *iio;
+	int ret;
+
+	iio = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!iio)
+		return -ENOMEM;
+
+	data = iio_priv(iio);
+	i2c_set_clientdata(i2c, iio);
+
+	data->chip = i2c_get_match_data(i2c);
+	if (!data->chip)
+		return dev_err_probe(dev, -ENODATA, "Failed to get chip data\n");
+
+	ret = devm_mutex_init(dev, &data->scale_lock);
+	if (ret)
+		return ret;
+
+	ret = devm_mutex_init(dev, &data->irq_lock);
+	if (ret)
+		return ret;
+
+	data->regmap = devm_regmap_init_i2c(i2c, &veml6031x00_regmap_config);
+	if (IS_ERR(data->regmap))
+		return dev_err_probe(dev, PTR_ERR(data->regmap),
+				     "Failed to set regmap\n");
+
+	ret = veml6031x00_regfield_init(data);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to init regfield\n");
+
+	ret = devm_regulator_get_enable(dev, "vdd");
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable regulator\n");
+
+	/* The device starts in power down mode by default */
+	ret = veml6031x00_set_power(data, true);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to power on the device\n");
+
+	ret = devm_add_action_or_reset(dev, veml6031x00_als_shutdown_action, data);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to add shutdown action\n");
+
+	pm_runtime_set_autosuspend_delay(dev, 2000);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_set_active_enabled(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable runtime PM\n");
+
+	ret = veml6031x00_validate_part_id(data);
+	if (ret)
+		return ret;
+
+	ret = veml6031x00_hw_init(data);
+	if (ret)
+		return ret;
+
+	/* Max resolution = 6.9632 lx/cnt for gain = 0.125 and IT = 3.125ms */
+	ret = devm_iio_init_iio_gts(dev, 6, 963200000,
+				    veml6031x00_gain_sel,
+				    ARRAY_SIZE(veml6031x00_gain_sel),
+				    veml6031x00_it_sel,
+				    ARRAY_SIZE(veml6031x00_it_sel),
+				    &data->gts);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to init IIO GTS\n");
+
+	ret = veml6031x00_init_iiodev(i2c, iio);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to initialize IIO device\n");
+
+	ret = devm_iio_triggered_buffer_setup(dev, iio,
+					      iio_pollfunc_store_time,
+					      veml6031x00_trig_handler,
+					      &veml6031x00_buffer_setup_ops);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register triggered buffer\n");
+
+	if (i2c->irq) {
+		ret = devm_request_threaded_irq(dev, i2c->irq, NULL, veml6031x00_irq,
+						IRQF_ONESHOT, iio->name, iio);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to request IRQ\n");
+	}
+
+	ret = devm_iio_device_register(dev, iio);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register IIO device\n");
+
+	return 0;
+}
+
+static int veml6031x00_runtime_suspend(struct device *dev)
+{
+	struct veml6031x00_data *data = iio_priv(dev_get_drvdata(dev));
+
+	return veml6031x00_set_power(data, false);
+}
+
+static int veml6031x00_runtime_resume(struct device *dev)
+{
+	struct veml6031x00_data *data = iio_priv(dev_get_drvdata(dev));
+
+	return veml6031x00_set_power(data, true);
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(veml6031x00_pm_ops,
+				 veml6031x00_runtime_suspend,
+				 veml6031x00_runtime_resume,
+				 NULL);
+
+static const struct veml6031x00_chip veml6031x00_chip = {
+	.name = "veml6031x00",
+	.part_id = 0x0001,
+};
+
+static const struct veml6031x00_chip veml6031x01_chip = {
+	.name = "veml6031x01",
+	.part_id = 0x0001,
+};
+
+static const struct veml6031x00_chip veml60311x00_chip = {
+	.name = "veml60311x00",
+	.part_id = 0x1001,
+};
+
+static const struct veml6031x00_chip veml60311x01_chip = {
+	.name = "veml60311x01",
+	.part_id = 0x1001,
+};
+
+static const struct of_device_id veml6031x00_of_match[] = {
+	{
+		.compatible = "vishay,veml6031x00",
+		.data = &veml6031x00_chip,
+	},
+	{
+		.compatible = "vishay,veml6031x01",
+		.data = &veml6031x01_chip,
+	},
+	{
+		.compatible = "vishay,veml60311x00",
+		.data = &veml60311x00_chip,
+	},
+	{
+		.compatible = "vishay,veml60311x01",
+		.data = &veml60311x01_chip,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, veml6031x00_of_match);
+
+static const struct i2c_device_id veml6031x00_id[] = {
+	{
+		.name = "veml6031x00",
+		.driver_data = (kernel_ulong_t)&veml6031x00_chip,
+	},
+	{
+		.name = "veml6031x01",
+		.driver_data = (kernel_ulong_t)&veml6031x01_chip,
+	},
+	{
+		.name = "veml60311x00",
+		.driver_data = (kernel_ulong_t)&veml60311x00_chip,
+	},
+	{
+		.name = "veml60311x01",
+		.driver_data = (kernel_ulong_t)&veml60311x01_chip,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, veml6031x00_id);
+
+static struct i2c_driver veml6031x00_driver = {
+	.driver = {
+		.name = "veml6031x00",
+		.of_match_table = veml6031x00_of_match,
+		.pm = pm_ptr(&veml6031x00_pm_ops),
+	},
+	.probe = veml6031x00_probe,
+	.id_table = veml6031x00_id,
+};
+module_i2c_driver(veml6031x00_driver);
+
+MODULE_AUTHOR("Javier Carrasco <javier.carrasco.cruz@gmail.com>");
+MODULE_DESCRIPTION("VEML6031X00 Ambient Light Sensor");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_GTS_HELPER");
diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c
index 8b0c07f82602ba..cc5be1fa91eac8 100644
--- a/drivers/iio/magnetometer/ak8975.c
+++ b/drivers/iio/magnetometer/ak8975.c
@@ -44,8 +44,7 @@
 #define AK8975_REG_INFO			0x01
 
 #define AK8975_REG_ST1			0x02
-#define AK8975_REG_ST1_DRDY_SHIFT	0
-#define AK8975_REG_ST1_DRDY_MASK	(1 << AK8975_REG_ST1_DRDY_SHIFT)
+#define AK8975_REG_ST1_DRDY_MASK	BIT(0)
 
 #define AK8975_REG_HXL			0x03
 #define AK8975_REG_HXH			0x04
@@ -54,15 +53,12 @@
 #define AK8975_REG_HZL			0x07
 #define AK8975_REG_HZH			0x08
 #define AK8975_REG_ST2			0x09
-#define AK8975_REG_ST2_DERR_SHIFT	2
-#define AK8975_REG_ST2_DERR_MASK	(1 << AK8975_REG_ST2_DERR_SHIFT)
+#define AK8975_REG_ST2_DERR_MASK	BIT(2)
 
-#define AK8975_REG_ST2_HOFL_SHIFT	3
-#define AK8975_REG_ST2_HOFL_MASK	(1 << AK8975_REG_ST2_HOFL_SHIFT)
+#define AK8975_REG_ST2_HOFL_MASK	BIT(3)
 
 #define AK8975_REG_CNTL			0x0A
-#define AK8975_REG_CNTL_MODE_SHIFT	0
-#define AK8975_REG_CNTL_MODE_MASK	(0xF << AK8975_REG_CNTL_MODE_SHIFT)
+#define AK8975_REG_CNTL_MODE_MASK	GENMASK(3, 0)
 #define AK8975_REG_CNTL_MODE_POWER_DOWN	0x00
 #define AK8975_REG_CNTL_MODE_ONCE	0x01
 #define AK8975_REG_CNTL_MODE_SELF_TEST	0x08
@@ -94,8 +90,7 @@
 
 #define AK09912_REG_ST1			0x10
 
-#define AK09912_REG_ST1_DRDY_SHIFT	0
-#define AK09912_REG_ST1_DRDY_MASK	(1 << AK09912_REG_ST1_DRDY_SHIFT)
+#define AK09912_REG_ST1_DRDY_MASK	BIT(0)
 
 #define AK09912_REG_HXL			0x11
 #define AK09912_REG_HXH			0x12
@@ -106,8 +101,7 @@
 #define AK09912_REG_TMPS		0x17
 
 #define AK09912_REG_ST2			0x18
-#define AK09912_REG_ST2_HOFL_SHIFT	3
-#define AK09912_REG_ST2_HOFL_MASK	(1 << AK09912_REG_ST2_HOFL_SHIFT)
+#define AK09912_REG_ST2_HOFL_MASK	BIT(3)
 
 #define AK09912_REG_CNTL1		0x30
 
@@ -116,8 +110,7 @@
 #define AK09912_REG_CNTL_MODE_ONCE	0x01
 #define AK09912_REG_CNTL_MODE_SELF_TEST	0x10
 #define AK09912_REG_CNTL_MODE_FUSE_ROM	0x1F
-#define AK09912_REG_CNTL2_MODE_SHIFT	0
-#define AK09912_REG_CNTL2_MODE_MASK	(0x1F << AK09912_REG_CNTL2_MODE_SHIFT)
+#define AK09912_REG_CNTL2_MODE_MASK	GENMASK(4, 0)
 
 #define AK09912_REG_CNTL3		0x32
 
@@ -237,6 +230,13 @@ enum ak_ctrl_mode {
 	MODE_END,
 };
 
+enum ak_chan_index {
+	AK8975_CHAN_X,
+	AK8975_CHAN_Y,
+	AK8975_CHAN_Z,
+	AK8975_CHAN_TS,
+};
+
 struct ak_def {
 	enum asahi_compass_chipset type;
 	long (*raw_to_gauss)(u16 data);
@@ -418,6 +418,7 @@ struct ak8975_data {
 	wait_queue_head_t	data_ready_queue;
 	unsigned long		flags;
 	u8			cntl_cache;
+	bool			powered_on;
 	struct iio_mount_matrix orientation;
 	struct regulator	*vdd;
 	struct regulator	*vid;
@@ -430,20 +431,19 @@ struct ak8975_data {
 };
 
 /* Enable attached power regulator if any. */
-static int ak8975_power_on(const struct ak8975_data *data)
+static int ak8975_power_on(struct ak8975_data *data)
 {
+	struct device *dev = &data->client->dev;
 	int ret;
 
 	ret = regulator_enable(data->vdd);
 	if (ret) {
-		dev_warn(&data->client->dev,
-			 "Failed to enable specified Vdd supply\n");
+		dev_warn(dev, "Failed to enable specified Vdd supply\n");
 		return ret;
 	}
 	ret = regulator_enable(data->vid);
 	if (ret) {
-		dev_warn(&data->client->dev,
-			 "Failed to enable specified Vid supply\n");
+		dev_warn(dev, "Failed to enable specified Vid supply\n");
 		regulator_disable(data->vdd);
 		return ret;
 	}
@@ -457,16 +457,23 @@ static int ak8975_power_on(const struct ak8975_data *data)
 	 */
 	fsleep(500);
 
+	data->powered_on = true;
+
 	return 0;
 }
 
 /* Disable attached power regulator if any. */
-static void ak8975_power_off(const struct ak8975_data *data)
+static void ak8975_power_off(struct ak8975_data *data)
 {
+	if (!data->powered_on)
+		return;
+
 	gpiod_set_value_cansleep(data->reset_gpiod, 1);
 
 	regulator_disable(data->vid);
 	regulator_disable(data->vdd);
+
+	data->powered_on = false;
 }
 
 /*
@@ -494,14 +501,10 @@ static int ak8975_who_i_am(const struct ak8975_data *data,
 							AK09912_REG_WIA1,
 							sizeof(wia_val),
 							wia_val);
-	if (ret < 0) {
-		dev_err(&client->dev, "Error reading WIA\n");
-		return ret;
-	}
-	if (ret != sizeof(wia_val)) {
-		dev_err(&client->dev, "Error reading WIA\n");
-		return -EIO;
-	}
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret, "Error reading WIA\n");
+	if (ret != sizeof(wia_val))
+		return dev_err_probe(&client->dev, -EIO, "Error reading WIA\n");
 
 	if (wia_val[0] != AK8975_DEVICE_ID)
 		return -ENODEV;
@@ -577,6 +580,7 @@ static irqreturn_t ak8975_irq_handler(int irq, void *data)
 static int ak8975_setup_irq(struct ak8975_data *data)
 {
 	struct i2c_client *client = data->client;
+	struct device *dev = &client->dev;
 	int irq;
 	int ret;
 
@@ -587,9 +591,8 @@ static int ak8975_setup_irq(struct ak8975_data *data)
 	else
 		irq = gpiod_to_irq(data->eoc_gpiod);
 
-	ret = devm_request_irq(&client->dev, irq, ak8975_irq_handler,
-			       IRQF_TRIGGER_RISING,
-			       dev_name(&client->dev), data);
+	ret = devm_request_irq(dev, irq, ak8975_irq_handler, IRQF_TRIGGER_RISING,
+			       dev_name(dev), data);
 	if (ret)
 		return ret;
 
@@ -605,43 +608,33 @@ static int ak8975_setup_irq(struct ak8975_data *data)
 static int ak8975_setup(struct ak8975_data *data)
 {
 	struct i2c_client *client = data->client;
+	struct device *dev = &client->dev;
 	int ret;
 
 	/* Write the fused rom access mode. */
 	ret = ak8975_set_mode(data, FUSE_ROM);
-	if (ret < 0) {
-		dev_err(&client->dev, "Error in setting fuse access mode\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Error in setting fuse access mode\n");
 
 	/* Get asa data and store in the device data. */
 	ret = i2c_smbus_read_i2c_block_data_or_emulated(client,
 							data->def->ctrl_regs[ASA_BASE],
 							sizeof(data->asa),
 							data->asa);
-	if (ret < 0) {
-		dev_err(&client->dev, "Not able to read asa data\n");
-		return ret;
-	}
-	if (ret != sizeof(data->asa)) {
-		dev_err(&client->dev, "Error reading asa data\n");
-		return -EIO;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Not able to read asa data\n");
+	if (ret != sizeof(data->asa))
+		return dev_err_probe(dev, -EIO, "Error reading asa data\n");
 
 	/* After reading fuse ROM data set power-down mode */
 	ret = ak8975_set_mode(data, POWER_DOWN);
-	if (ret < 0) {
-		dev_err(&client->dev, "Error in setting power-down mode\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Error in setting power-down mode\n");
 
 	if (data->eoc_gpiod || client->irq > 0) {
 		ret = ak8975_setup_irq(data);
-		if (ret < 0) {
-			dev_err(&client->dev,
-				"Error setting data ready interrupt\n");
-			return ret;
-		}
+		if (ret < 0)
+			return dev_err_probe(dev, ret, "Error setting data ready interrupt\n");
 	}
 
 	data->raw_to_gauss[0] = data->def->raw_to_gauss(data->asa[0]);
@@ -745,10 +738,11 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 	struct ak8975_data *data = iio_priv(indio_dev);
 	const struct i2c_client *client = data->client;
 	const struct ak_def *def = data->def;
+	struct device *dev = &data->client->dev;
 	__le16 rval;
 	int ret;
 
-	pm_runtime_get_sync(&data->client->dev);
+	pm_runtime_get_sync(dev);
 
 	mutex_lock(&data->lock);
 
@@ -770,20 +764,20 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 	/* Read out ST2 for release lock on measurement data. */
 	ret = i2c_smbus_read_byte_data(client, data->def->ctrl_regs[ST2]);
 	if (ret < 0) {
-		dev_err(&client->dev, "Error in reading ST2\n");
+		dev_err(dev, "Error in reading ST2\n");
 		goto exit;
 	}
 
 	if (ret & (data->def->ctrl_masks[ST2_DERR] |
 		   data->def->ctrl_masks[ST2_HOFL])) {
-		dev_err(&client->dev, "ST2 status error 0x%x\n", ret);
+		dev_err(dev, "ST2 status error 0x%x\n", ret);
 		ret = -EINVAL;
 		goto exit;
 	}
 
 	mutex_unlock(&data->lock);
 
-	pm_runtime_put_autosuspend(&data->client->dev);
+	pm_runtime_put_autosuspend(dev);
 
 	/* Swap bytes and convert to valid range. */
 	*val = clamp_t(s16, le16_to_cpu(rval), -def->range, def->range);
@@ -792,8 +786,8 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 
 exit:
 	mutex_unlock(&data->lock);
-	pm_runtime_put_autosuspend(&data->client->dev);
-	dev_err(&client->dev, "Error in reading axis\n");
+	pm_runtime_put_autosuspend(dev);
+	dev_err(dev, "Error in reading axis\n");
 	return ret;
 }
 
@@ -848,11 +842,16 @@ static const struct iio_chan_spec_ext_info ak8975_ext_info[] = {
 	}
 
 static const struct iio_chan_spec ak8975_channels[] = {
-	AK8975_CHANNEL(X, 0), AK8975_CHANNEL(Y, 1), AK8975_CHANNEL(Z, 2),
-	IIO_CHAN_SOFT_TIMESTAMP(3),
+	AK8975_CHANNEL(X, AK8975_CHAN_X),
+	AK8975_CHANNEL(Y, AK8975_CHAN_Y),
+	AK8975_CHANNEL(Z, AK8975_CHAN_Z),
+	IIO_CHAN_SOFT_TIMESTAMP(AK8975_CHAN_TS),
 };
 
-static const unsigned long ak8975_scan_masks[] = { 0x7, 0 };
+static const unsigned long ak8975_scan_masks[] = {
+	BIT(AK8975_CHAN_X) | BIT(AK8975_CHAN_Y) | BIT(AK8975_CHAN_Z),
+	0
+};
 
 static const struct iio_info ak8975_info = {
 	.read_raw = &ak8975_read_raw,
@@ -934,9 +933,23 @@ static const struct iio_buffer_setup_ops ak8975_buffer_setup_ops = {
 	.preenable = ak8975_buffer_preenable,
 	.postdisable = ak8975_buffer_postdisable,
 };
+
+static void devm_ak8975_power_off(void *data)
+{
+	struct ak8975_data *ak = data;
+
+	if (!ak->powered_on)
+		return;
+
+	/* Soft-stop the chip before hard-stopping the regulators */
+	ak8975_set_mode(data, POWER_DOWN);
+	ak8975_power_off(data);
+}
+
 static int ak8975_probe(struct i2c_client *client)
 {
 	const struct i2c_device_id *id = i2c_client_get_device_id(client);
+	struct device *dev = &client->dev;
 	struct ak8975_data *data;
 	struct iio_dev *indio_dev;
 	struct gpio_desc *eoc_gpiod;
@@ -949,7 +962,7 @@ static int ak8975_probe(struct i2c_client *client)
 	 * We may not have a GPIO based IRQ to scan, that is fine, we will
 	 * poll if so.
 	 */
-	eoc_gpiod = devm_gpiod_get_optional(&client->dev, NULL, GPIOD_IN);
+	eoc_gpiod = devm_gpiod_get_optional(dev, NULL, GPIOD_IN);
 	if (IS_ERR(eoc_gpiod))
 		return PTR_ERR(eoc_gpiod);
 	gpiod_set_consumer_name(eoc_gpiod, "ak_8975");
@@ -959,13 +972,12 @@ static int ak8975_probe(struct i2c_client *client)
 	 * deassert reset on ak8975_power_on() and assert reset on
 	 * ak8975_power_off().
 	 */
-	reset_gpiod = devm_gpiod_get_optional(&client->dev,
-					      "reset", GPIOD_OUT_HIGH);
+	reset_gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(reset_gpiod))
 		return PTR_ERR(reset_gpiod);
 
 	/* Register with IIO */
-	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
 	if (indio_dev == NULL)
 		return -ENOMEM;
 
@@ -977,7 +989,7 @@ static int ak8975_probe(struct i2c_client *client)
 	data->reset_gpiod = reset_gpiod;
 	data->eoc_irq = 0;
 
-	ret = iio_read_mount_matrix(&client->dev, &data->orientation);
+	ret = iio_read_mount_matrix(dev, &data->orientation);
 	if (ret)
 		return ret;
 
@@ -987,16 +999,16 @@ static int ak8975_probe(struct i2c_client *client)
 		return -ENODEV;
 
 	/* If enumerated via firmware node, fix the ABI */
-	if (dev_fwnode(&client->dev))
-		name = dev_name(&client->dev);
+	if (dev_fwnode(dev))
+		name = dev_name(dev);
 	else
 		name = id->name;
 
 	/* Fetch the regulators */
-	data->vdd = devm_regulator_get(&client->dev, "vdd");
+	data->vdd = devm_regulator_get(dev, "vdd");
 	if (IS_ERR(data->vdd))
 		return PTR_ERR(data->vdd);
-	data->vid = devm_regulator_get(&client->dev, "vid");
+	data->vid = devm_regulator_get(dev, "vid");
 	if (IS_ERR(data->vid))
 		return PTR_ERR(data->vid);
 
@@ -1004,21 +1016,25 @@ static int ak8975_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
+	ret = devm_add_action_or_reset(dev, devm_ak8975_power_off, data);
+	if (ret)
+		return ret;
+
 	ret = ak8975_who_i_am(data, data->def->type);
-	if (ret) {
-		dev_err(&client->dev, "Unexpected device\n");
-		goto power_off;
-	}
-	dev_dbg(&client->dev, "Asahi compass chip %s\n", name);
+	if (ret)
+		return dev_err_probe(dev, ret, "Unexpected device\n");
+
+	dev_dbg(dev, "Asahi compass chip %s\n", name);
 
 	/* Perform some basic start-of-day setup of the device. */
 	ret = ak8975_setup(data);
-	if (ret) {
-		dev_err(&client->dev, "%s initialization fails\n", name);
-		goto power_off;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "%s initialization fails\n", name);
+
+	ret = devm_mutex_init(dev, &data->lock);
+	if (ret)
+		return ret;
 
-	mutex_init(&data->lock);
 	indio_dev->channels = ak8975_channels;
 	indio_dev->num_channels = ARRAY_SIZE(ak8975_channels);
 	indio_dev->info = &ak8975_info;
@@ -1026,52 +1042,29 @@ static int ak8975_probe(struct i2c_client *client)
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->name = name;
 
-	ret = iio_triggered_buffer_setup(indio_dev, NULL, ak8975_handle_trigger,
-					 &ak8975_buffer_setup_ops);
-	if (ret) {
-		dev_err(&client->dev, "triggered buffer setup failed\n");
-		goto power_off;
-	}
+	ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL,
+					      ak8975_handle_trigger,
+					      &ak8975_buffer_setup_ops);
+	if (ret)
+		return dev_err_probe(dev, ret, "triggered buffer setup failed\n");
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&client->dev, "device register failed\n");
-		goto cleanup_buffer;
-	}
+	pm_runtime_set_active(dev);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
+
+	ret = devm_iio_device_register(dev, indio_dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "device register failed\n");
 
-	/* Enable runtime PM */
-	pm_runtime_get_noresume(&client->dev);
-	pm_runtime_set_active(&client->dev);
-	pm_runtime_enable(&client->dev);
 	/*
 	 * The device comes online in 500us, so add two orders of magnitude
 	 * of delay before autosuspending: 50 ms.
 	 */
-	pm_runtime_set_autosuspend_delay(&client->dev, 50);
-	pm_runtime_use_autosuspend(&client->dev);
-	pm_runtime_put(&client->dev);
+	pm_runtime_set_autosuspend_delay(dev, 50);
+	pm_runtime_use_autosuspend(dev);
 
 	return 0;
-
-cleanup_buffer:
-	iio_triggered_buffer_cleanup(indio_dev);
-power_off:
-	ak8975_power_off(data);
-	return ret;
-}
-
-static void ak8975_remove(struct i2c_client *client)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(client);
-	struct ak8975_data *data = iio_priv(indio_dev);
-
-	pm_runtime_get_sync(&client->dev);
-	pm_runtime_put_noidle(&client->dev);
-	pm_runtime_disable(&client->dev);
-	iio_device_unregister(indio_dev);
-	iio_triggered_buffer_cleanup(indio_dev);
-	ak8975_set_mode(data, POWER_DOWN);
-	ak8975_power_off(data);
 }
 
 static int ak8975_runtime_suspend(struct device *dev)
@@ -1084,7 +1077,7 @@ static int ak8975_runtime_suspend(struct device *dev)
 	/* Set the device in power down if it wasn't already */
 	ret = ak8975_set_mode(data, POWER_DOWN);
 	if (ret < 0) {
-		dev_err(&client->dev, "Error in setting power-down mode\n");
+		dev_err(dev, "Error in setting power-down mode\n");
 		return ret;
 	}
 	/* Next cut the regulators */
@@ -1108,7 +1101,7 @@ static int ak8975_runtime_resume(struct device *dev)
 	 */
 	ret = ak8975_set_mode(data, POWER_DOWN);
 	if (ret < 0) {
-		dev_err(&client->dev, "Error in setting power-down mode\n");
+		dev_err(dev, "Error in setting power-down mode\n");
 		return ret;
 	}
 
@@ -1165,7 +1158,6 @@ static struct i2c_driver ak8975_driver = {
 		.acpi_match_table = ak_acpi_match,
 	},
 	.probe		= ak8975_probe,
-	.remove		= ak8975_remove,
 	.id_table	= ak8975_id,
 };
 module_i2c_driver(ak8975_driver);
diff --git a/drivers/iio/position/iqs624-pos.c b/drivers/iio/position/iqs624-pos.c
index 8239239c6ee272..d2aa2df7145708 100644
--- a/drivers/iio/position/iqs624-pos.c
+++ b/drivers/iio/position/iqs624-pos.c
@@ -96,19 +96,6 @@ static int iqs624_pos_notifier(struct notifier_block *notifier,
 	return ret;
 }
 
-static void iqs624_pos_notifier_unregister(void *context)
-{
-	struct iqs624_pos_private *iqs624_pos = context;
-	struct iio_dev *indio_dev = iqs624_pos->indio_dev;
-	int ret;
-
-	ret = blocking_notifier_chain_unregister(&iqs624_pos->iqs62x->nh,
-						 &iqs624_pos->notifier);
-	if (ret)
-		dev_err(indio_dev->dev.parent,
-			"Failed to unregister notifier: %d\n", ret);
-}
-
 static int iqs624_pos_angle_get(struct iqs62x_core *iqs62x, unsigned int *val)
 {
 	int ret;
@@ -255,19 +242,14 @@ static int iqs624_pos_probe(struct platform_device *pdev)
 	mutex_init(&iqs624_pos->lock);
 
 	iqs624_pos->notifier.notifier_call = iqs624_pos_notifier;
-	ret = blocking_notifier_chain_register(&iqs624_pos->iqs62x->nh,
-					       &iqs624_pos->notifier);
+	ret = devm_blocking_notifier_chain_register(&pdev->dev,
+						    &iqs624_pos->iqs62x->nh,
+						    &iqs624_pos->notifier);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret);
 		return ret;
 	}
 
-	ret = devm_add_action_or_reset(&pdev->dev,
-				       iqs624_pos_notifier_unregister,
-				       iqs624_pos);
-	if (ret)
-		return ret;
-
 	return devm_iio_device_register(&pdev->dev, indio_dev);
 }
 
diff --git a/drivers/iio/potentiometer/x9250.c b/drivers/iio/potentiometer/x9250.c
index 974e4447fc2ccb..675b78677baa4f 100644
--- a/drivers/iio/potentiometer/x9250.c
+++ b/drivers/iio/potentiometer/x9250.c
@@ -155,10 +155,15 @@ static const char *const x9250_regulator_names[] = {
 
 static int x9250_probe(struct spi_device *spi)
 {
+	const struct x9250_cfg *cfg;
 	struct iio_dev *indio_dev;
 	struct x9250 *x9250;
 	int ret;
 
+	cfg = spi_get_device_match_data(spi);
+	if (!cfg)
+		return -ENODATA;
+
 	ret = devm_regulator_bulk_get_enable(&spi->dev, ARRAY_SIZE(x9250_regulator_names),
 					     x9250_regulator_names);
 	if (ret)
@@ -176,7 +181,7 @@ static int x9250_probe(struct spi_device *spi)
 
 	x9250 = iio_priv(indio_dev);
 	x9250->spi = spi;
-	x9250->cfg = spi_get_device_match_data(spi);
+	x9250->cfg = cfg;
 	x9250->wp_gpio = devm_gpiod_get_optional(&spi->dev, "wp", GPIOD_OUT_LOW);
 	if (IS_ERR(x9250->wp_gpio))
 		return dev_err_probe(&spi->dev, PTR_ERR(x9250->wp_gpio),
diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig
index 838a8340c4c02a..cef8b90b9ae78d 100644
--- a/drivers/iio/pressure/Kconfig
+++ b/drivers/iio/pressure/Kconfig
@@ -112,6 +112,8 @@ config DPS310
 	tristate "Infineon DPS310 pressure and temperature sensor"
 	depends on I2C
 	select REGMAP_I2C
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	help
 	  Support for the Infineon DPS310 digital barometric pressure sensor.
 	  It can be accessed over I2C bus.
diff --git a/drivers/iio/pressure/Makefile b/drivers/iio/pressure/Makefile
index bc0d11a20acc41..4b1a05f7a0bdc6 100644
--- a/drivers/iio/pressure/Makefile
+++ b/drivers/iio/pressure/Makefile
@@ -3,6 +3,8 @@
 # Makefile for industrial I/O pressure drivers
 #
 
+CONTEXT_ANALYSIS_dps310.o := y
+
 # When adding new entries keep the list in alphabetical order
 obj-$(CONFIG_ABP060MG) += abp060mg.o
 obj-$(CONFIG_ABP2030PA) += abp2030pa.o
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index 1066792b908738..efdefe6ec747d4 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -46,6 +46,7 @@
 #include <linux/random.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
+#include <linux/string_choices.h>
 #include <linux/types.h>
 
 #include <linux/iio/buffer.h>
@@ -839,7 +840,7 @@ static int bmp280_write_sampling_frequency(struct bmp280_data *data,
 					   int val, int val2)
 {
 	const int (*avail)[2] = data->chip_info->sampling_freq_avail;
-	const int n = data->chip_info->num_sampling_freq_avail;
+	const int n = data->chip_info->num_sampling_freq_avail / 2;
 	int ret, prev;
 	int i;
 
diff --git a/drivers/iio/pressure/bmp280-spi.c b/drivers/iio/pressure/bmp280-spi.c
index a02a621b80150d..95d9635f47111f 100644
--- a/drivers/iio/pressure/bmp280-spi.c
+++ b/drivers/iio/pressure/bmp280-spi.c
@@ -83,6 +83,8 @@ static int bmp280_spi_probe(struct spi_device *spi)
 	struct regmap *regmap;
 
 	chip_info = spi_get_device_match_data(spi);
+	if (!chip_info)
+		return -ENODATA;
 
 	if (chip_info->spi_read_extra_byte)
 		bmp_regmap_bus = &bmp380_regmap_bus;
diff --git a/drivers/iio/pressure/bmp280.h b/drivers/iio/pressure/bmp280.h
index 25bb9c743a05a1..d19fb33e3a1573 100644
--- a/drivers/iio/pressure/bmp280.h
+++ b/drivers/iio/pressure/bmp280.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <linux/bitops.h>
 #include <linux/device.h>
+#include <linux/interrupt.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 
diff --git a/drivers/iio/pressure/cros_ec_baro.c b/drivers/iio/pressure/cros_ec_baro.c
index 6a567b6075d948..e4bc4df48d01b4 100644
--- a/drivers/iio/pressure/cros_ec_baro.c
+++ b/drivers/iio/pressure/cros_ec_baro.c
@@ -192,7 +192,7 @@ MODULE_DEVICE_TABLE(platform, cros_ec_baro_ids);
 static struct platform_driver cros_ec_baro_platform_driver = {
 	.driver = {
 		.name	= "cros-ec-baro",
-		.pm	= &cros_ec_sensors_pm_ops,
+		.pm	= pm_sleep_ptr(&cros_ec_sensors_pm_ops),
 	},
 	.probe		= cros_ec_baro_probe,
 	.id_table	= cros_ec_baro_ids,
diff --git a/drivers/iio/pressure/dps310.c b/drivers/iio/pressure/dps310.c
index 45bdb8c7670f06..85df58ac18090b 100644
--- a/drivers/iio/pressure/dps310.c
+++ b/drivers/iio/pressure/dps310.c
@@ -14,14 +14,19 @@
  *  - Optionally support the FIFO
  */
 
+#include <linux/cleanup.h>
 #include <linux/i2c.h>
 #include <linux/limits.h>
 #include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
+#include <linux/unaligned.h>
 
+#include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 
 #define DPS310_DEV_NAME		"dps310"
 
@@ -50,9 +55,9 @@
 #define DPS310_CFG_REG		0x09
 #define  DPS310_INT_HL		BIT(7)
 #define  DPS310_TMP_SHIFT_EN	BIT(3)
-#define  DPS310_PRS_SHIFT_EN	BIT(4)
-#define  DPS310_FIFO_EN		BIT(5)
-#define  DPS310_SPI_EN		BIT(6)
+#define  DPS310_PRS_SHIFT_EN	BIT(2)
+#define  DPS310_FIFO_EN		BIT(1)
+#define  DPS310_SPI_EN		BIT(0)
 #define DPS310_RESET		0x0c
 #define  DPS310_RESET_MAGIC	0x09
 #define DPS310_COEF_BASE	0x10
@@ -92,19 +97,51 @@ struct dps310_data {
 	bool timeout_recovery_failed;
 };
 
+enum dps310_scan_index {
+	DPS310_SCAN_TEMP,
+	DPS310_SCAN_PRESSURE,
+	DPS310_SCAN_TIMESTAMP,
+};
+
 static const struct iio_chan_spec dps310_channels[] = {
 	{
 		.type = IIO_TEMP,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO) |
 			BIT(IIO_CHAN_INFO_SAMP_FREQ) |
 			BIT(IIO_CHAN_INFO_PROCESSED),
+		.scan_index = DPS310_SCAN_TEMP,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 32,
+			.storagebits = 32,
+			.endianness = IIO_CPU,
+		},
 	},
 	{
 		.type = IIO_PRESSURE,
+		/*
+		 * _raw here is already compensated (section 4.9.1, which needs
+		 * a temperature too) and in Pa; _scale converts to the kPa the
+		 * ABI wants. _processed predates buffers and has to stay.
+		 *
+		 * Do not copy this into other drivers. A _raw that is not the
+		 * raw register value is only tolerable because the alternative
+		 * is losing resolution or breaking existing _processed users.
+		 */
 		.info_mask_separate = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO) |
 			BIT(IIO_CHAN_INFO_SAMP_FREQ) |
+			BIT(IIO_CHAN_INFO_RAW) |
+			BIT(IIO_CHAN_INFO_SCALE) |
 			BIT(IIO_CHAN_INFO_PROCESSED),
+		.scan_index = DPS310_SCAN_PRESSURE,
+		.scan_type = {
+			.sign = 's',
+			.realbits = 32,
+			.storagebits = 32,
+			.endianness = IIO_CPU,
+		},
 	},
+	IIO_CHAN_SOFT_TIMESTAMP(DPS310_SCAN_TIMESTAMP),
 };
 
 /* To be called after checking the COEF_RDY bit in MEAS_CFG */
@@ -286,8 +323,8 @@ static int dps310_get_temp_precision(struct dps310_data *data, int *val)
 	return 0;
 }
 
-/* Called with lock held */
 static int dps310_set_pres_precision(struct dps310_data *data, int val)
+	__must_hold(&data->lock)
 {
 	int rc;
 	u8 shift_en;
@@ -305,8 +342,8 @@ static int dps310_set_pres_precision(struct dps310_data *data, int val)
 				  DPS310_PRS_PRC_BITS, ilog2(val));
 }
 
-/* Called with lock held */
 static int dps310_set_temp_precision(struct dps310_data *data, int val)
+	__must_hold(&data->lock)
 {
 	int rc;
 	u8 shift_en;
@@ -324,8 +361,8 @@ static int dps310_set_temp_precision(struct dps310_data *data, int val)
 				  DPS310_TMP_PRC_BITS, ilog2(val));
 }
 
-/* Called with lock held */
 static int dps310_set_pres_samp_freq(struct dps310_data *data, int freq)
+	__must_hold(&data->lock)
 {
 	u8 val;
 
@@ -338,8 +375,8 @@ static int dps310_set_pres_samp_freq(struct dps310_data *data, int freq)
 				  DPS310_PRS_RATE_BITS, val);
 }
 
-/* Called with lock held */
 static int dps310_set_temp_samp_freq(struct dps310_data *data, int freq)
+	__must_hold(&data->lock)
 {
 	u8 val;
 
@@ -438,6 +475,7 @@ static int dps310_ready_status(struct dps310_data *data, int ready_bit, int time
 }
 
 static int dps310_ready(struct dps310_data *data, int ready_bit, int timeout)
+	__must_hold(&data->lock)
 {
 	int rc;
 
@@ -463,82 +501,83 @@ static int dps310_ready(struct dps310_data *data, int ready_bit, int timeout)
 	return 0;
 }
 
-static int dps310_read_pres_raw(struct dps310_data *data)
+static int dps310_read_pres_raw_locked(struct dps310_data *data)
+	__must_hold(&data->lock)
 {
 	int rc;
 	int rate;
 	int timeout;
-	s32 raw;
 	u8 val[3];
 
-	if (mutex_lock_interruptible(&data->lock))
-		return -EINTR;
-
 	rc = dps310_get_pres_samp_freq(data, &rate);
 	if (rc)
-		goto done;
+		return rc;
 
 	timeout = DPS310_POLL_TIMEOUT_US(rate);
 
 	/* Poll for sensor readiness; base the timeout upon the sample rate. */
 	rc = dps310_ready(data, DPS310_PRS_RDY, timeout);
 	if (rc)
-		goto done;
+		return rc;
 
 	rc = regmap_bulk_read(data->regmap, DPS310_PRS_BASE, val, sizeof(val));
 	if (rc < 0)
-		goto done;
+		return rc;
 
-	raw = (val[0] << 16) | (val[1] << 8) | val[2];
-	data->pressure_raw = sign_extend32(raw, 23);
+	data->pressure_raw = sign_extend32(get_unaligned_be24(val), 23);
 
-done:
-	mutex_unlock(&data->lock);
-	return rc;
+	return 0;
 }
 
-/* Called with lock held */
 static int dps310_read_temp_ready(struct dps310_data *data)
+	__must_hold(&data->lock)
 {
 	int rc;
 	u8 val[3];
-	s32 raw;
 
 	rc = regmap_bulk_read(data->regmap, DPS310_TMP_BASE, val, sizeof(val));
 	if (rc < 0)
 		return rc;
 
-	raw = (val[0] << 16) | (val[1] << 8) | val[2];
-	data->temp_raw = sign_extend32(raw, 23);
+	data->temp_raw = sign_extend32(get_unaligned_be24(val), 23);
 
 	return 0;
 }
 
-static int dps310_read_temp_raw(struct dps310_data *data)
+static int dps310_read_temp_raw_locked(struct dps310_data *data)
+	__must_hold(&data->lock)
 {
 	int rc;
 	int rate;
 	int timeout;
 
-	if (mutex_lock_interruptible(&data->lock))
-		return -EINTR;
-
 	rc = dps310_get_temp_samp_freq(data, &rate);
 	if (rc)
-		goto done;
+		return rc;
 
 	timeout = DPS310_POLL_TIMEOUT_US(rate);
 
 	/* Poll for sensor readiness; base the timeout upon the sample rate. */
 	rc = dps310_ready(data, DPS310_TMP_RDY, timeout);
 	if (rc)
-		goto done;
+		return rc;
 
-	rc = dps310_read_temp_ready(data);
+	return dps310_read_temp_ready(data);
+}
 
-done:
-	mutex_unlock(&data->lock);
-	return rc;
+/* Best effort: on error the previous temperature stands */
+static void dps310_refresh_temp_locked(struct dps310_data *data)
+	__must_hold(&data->lock)
+{
+	int rc;
+	int t_ready;
+
+	rc = regmap_read(data->regmap, DPS310_MEAS_CFG, &t_ready);
+	if (rc)
+		return;
+
+	if (t_ready & DPS310_TMP_RDY)
+		dps310_read_temp_ready(data);
 }
 
 static bool dps310_is_writeable_reg(struct device *dev, unsigned int reg)
@@ -580,59 +619,52 @@ static int dps310_write_raw(struct iio_dev *iio,
 			    struct iio_chan_spec const *chan, int val,
 			    int val2, long mask)
 {
-	int rc;
 	struct dps310_data *data = iio_priv(iio);
 
-	if (mutex_lock_interruptible(&data->lock))
+	/* Reconfiguring mid-capture would change the values being captured */
+	IIO_DEV_ACQUIRE_DIRECT_MODE(iio, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	ACQUIRE(mutex_intr, lock)(&data->lock);
+	if (ACQUIRE_ERR(mutex_intr, &lock))
 		return -EINTR;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SAMP_FREQ:
 		switch (chan->type) {
 		case IIO_PRESSURE:
-			rc = dps310_set_pres_samp_freq(data, val);
-			break;
+			return dps310_set_pres_samp_freq(data, val);
 
 		case IIO_TEMP:
-			rc = dps310_set_temp_samp_freq(data, val);
-			break;
+			return dps310_set_temp_samp_freq(data, val);
 
 		default:
-			rc = -EINVAL;
-			break;
+			return -EINVAL;
 		}
-		break;
 
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
 		switch (chan->type) {
 		case IIO_PRESSURE:
-			rc = dps310_set_pres_precision(data, val);
-			break;
+			return dps310_set_pres_precision(data, val);
 
 		case IIO_TEMP:
-			rc = dps310_set_temp_precision(data, val);
-			break;
+			return dps310_set_temp_precision(data, val);
 
 		default:
-			rc = -EINVAL;
-			break;
+			return -EINVAL;
 		}
-		break;
 
 	default:
-		rc = -EINVAL;
-		break;
+		return -EINVAL;
 	}
-
-	mutex_unlock(&data->lock);
-	return rc;
 }
 
 static int dps310_calculate_pressure(struct dps310_data *data, int *val)
+	__must_hold(&data->lock)
 {
 	int i;
 	int rc;
-	int t_ready;
 	int kpi;
 	int kti;
 	s64 rem = 0ULL;
@@ -656,15 +688,6 @@ static int dps310_calculate_pressure(struct dps310_data *data, int *val)
 	kp = (s64)kpi;
 	kt = (s64)kti;
 
-	/* Refresh temp if it's ready, otherwise just use the latest value */
-	if (mutex_trylock(&data->lock)) {
-		rc = regmap_read(data->regmap, DPS310_MEAS_CFG, &t_ready);
-		if (rc >= 0 && t_ready & DPS310_TMP_RDY)
-			dps310_read_temp_ready(data);
-
-		mutex_unlock(&data->lock);
-	}
-
 	p = (s64)data->pressure_raw;
 	t = (s64)data->temp_raw;
 
@@ -710,6 +733,23 @@ static int dps310_calculate_pressure(struct dps310_data *data, int *val)
 	return 0;
 }
 
+static int dps310_read_pressure_value(struct dps310_data *data, int *val)
+{
+	int rc;
+
+	ACQUIRE(mutex_intr, lock)(&data->lock);
+	if (ACQUIRE_ERR(mutex_intr, &lock))
+		return -EINTR;
+
+	rc = dps310_read_pres_raw_locked(data);
+	if (rc)
+		return rc;
+
+	dps310_refresh_temp_locked(data);
+
+	return dps310_calculate_pressure(data, val);
+}
+
 static int dps310_read_pressure(struct dps310_data *data, int *val, int *val2,
 				long mask)
 {
@@ -723,18 +763,27 @@ static int dps310_read_pressure(struct dps310_data *data, int *val, int *val2,
 
 		return IIO_VAL_INT;
 
-	case IIO_CHAN_INFO_PROCESSED:
-		rc = dps310_read_pres_raw(data);
+	case IIO_CHAN_INFO_RAW:
+		rc = dps310_read_pressure_value(data, val);
 		if (rc)
 			return rc;
 
-		rc = dps310_calculate_pressure(data, val);
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_PROCESSED:
+		rc = dps310_read_pressure_value(data, val);
 		if (rc)
 			return rc;
 
 		*val2 = 1000; /* Convert Pa to KPa per IIO ABI */
 		return IIO_VAL_FRACTIONAL;
 
+	case IIO_CHAN_INFO_SCALE:
+		/* The raw value is in Pa, the ABI wants kPa */
+		*val = 1;
+		*val2 = 1000;
+		return IIO_VAL_FRACTIONAL;
+
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
 		rc = dps310_get_pres_precision(data, val);
 		if (rc)
@@ -747,6 +796,7 @@ static int dps310_read_pressure(struct dps310_data *data, int *val, int *val2,
 }
 
 static int dps310_calculate_temp(struct dps310_data *data, int *val)
+	__must_hold(&data->lock)
 {
 	s64 c0;
 	s64 t;
@@ -768,6 +818,21 @@ static int dps310_calculate_temp(struct dps310_data *data, int *val)
 	return 0;
 }
 
+static int dps310_read_temp_value(struct dps310_data *data, int *val)
+{
+	int rc;
+
+	ACQUIRE(mutex_intr, lock)(&data->lock);
+	if (ACQUIRE_ERR(mutex_intr, &lock))
+		return -EINTR;
+
+	rc = dps310_read_temp_raw_locked(data);
+	if (rc)
+		return rc;
+
+	return dps310_calculate_temp(data, val);
+}
+
 static int dps310_read_temp(struct dps310_data *data, int *val, int *val2,
 			    long mask)
 {
@@ -782,11 +847,7 @@ static int dps310_read_temp(struct dps310_data *data, int *val, int *val2,
 		return IIO_VAL_INT;
 
 	case IIO_CHAN_INFO_PROCESSED:
-		rc = dps310_read_temp_raw(data);
-		if (rc)
-			return rc;
-
-		rc = dps310_calculate_temp(data, val);
+		rc = dps310_read_temp_value(data, val);
 		if (rc)
 			return rc;
 
@@ -804,12 +865,10 @@ static int dps310_read_temp(struct dps310_data *data, int *val, int *val2,
 	}
 }
 
-static int dps310_read_raw(struct iio_dev *iio,
-			   struct iio_chan_spec const *chan,
-			   int *val, int *val2, long mask)
+static int dps310_read_channel(struct dps310_data *data,
+			       struct iio_chan_spec const *chan,
+			       int *val, int *val2, long mask)
 {
-	struct dps310_data *data = iio_priv(iio);
-
 	switch (chan->type) {
 	case IIO_PRESSURE:
 		return dps310_read_pressure(data, val, val2, mask);
@@ -822,6 +881,87 @@ static int dps310_read_raw(struct iio_dev *iio,
 	}
 }
 
+static int dps310_read_raw(struct iio_dev *iio,
+			   struct iio_chan_spec const *chan,
+			   int *val, int *val2, long mask)
+{
+	struct dps310_data *data = iio_priv(iio);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+	case IIO_CHAN_INFO_PROCESSED: {
+		/* This consumes the measurement the capture path reads */
+		IIO_DEV_ACQUIRE_DIRECT_MODE(iio, claim);
+		if (IIO_DEV_ACQUIRE_FAILED(claim))
+			return -EBUSY;
+
+		return dps310_read_channel(data, chan, val, val2, mask);
+	}
+	default:
+		return dps310_read_channel(data, chan, val, val2, mask);
+	}
+}
+
+static int dps310_fill_channels(struct dps310_data *data,
+				const unsigned long *scan_mask,
+				s32 channels[2])
+	__must_hold(&data->lock)
+{
+	unsigned int i;
+	int rc;
+
+	/* Compensation needs a temperature, so it is sampled either way */
+	rc = dps310_read_temp_raw_locked(data);
+	if (rc)
+		return rc;
+
+	i = 0;
+	if (test_bit(DPS310_SCAN_TEMP, scan_mask)) {
+		/* Millidegrees Celsius */
+		rc = dps310_calculate_temp(data, &channels[i++]);
+		if (rc)
+			return rc;
+	}
+
+	if (test_bit(DPS310_SCAN_PRESSURE, scan_mask)) {
+		rc = dps310_read_pres_raw_locked(data);
+		if (rc)
+			return rc;
+
+		/* Pascals, see the channel definition */
+		rc = dps310_calculate_pressure(data, &channels[i++]);
+		if (rc)
+			return rc;
+	}
+
+	return 0;
+}
+
+static irqreturn_t dps310_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *iio = pf->indio_dev;
+	struct dps310_data *data = iio_priv(iio);
+	struct {
+		s32 channels[2];
+		aligned_s64 timestamp;
+	} scan = { };
+	int rc;
+
+	mutex_lock(&data->lock);
+	rc = dps310_fill_channels(data, iio->active_scan_mask, scan.channels);
+	mutex_unlock(&data->lock);
+	if (rc)
+		goto err;
+
+	iio_push_to_buffers_with_ts(iio, &scan, sizeof(scan), iio_get_time_ns(iio));
+
+err:
+	iio_trigger_notify_done(iio->trig);
+
+	return IRQ_HANDLED;
+}
+
 static void dps310_reset(void *action_data)
 {
 	struct dps310_data *data = action_data;
@@ -845,11 +985,12 @@ static const struct iio_info dps310_info = {
 
 static int dps310_probe(struct i2c_client *client)
 {
+	struct device *dev = &client->dev;
 	struct dps310_data *data;
 	struct iio_dev *iio;
 	int rc;
 
-	iio = devm_iio_device_alloc(&client->dev,  sizeof(*data));
+	iio = devm_iio_device_alloc(dev, sizeof(*data));
 	if (!iio)
 		return -ENOMEM;
 
@@ -868,7 +1009,7 @@ static int dps310_probe(struct i2c_client *client)
 		return PTR_ERR(data->regmap);
 
 	/* Register to run the device reset when the device is removed */
-	rc = devm_add_action_or_reset(&client->dev, dps310_reset, data);
+	rc = devm_add_action_or_reset(dev, dps310_reset, data);
 	if (rc)
 		return rc;
 
@@ -876,7 +1017,17 @@ static int dps310_probe(struct i2c_client *client)
 	if (rc)
 		return rc;
 
-	rc = devm_iio_device_register(&client->dev, iio);
+	/*
+	 * The device measures continuously in background mode, so a capture is
+	 * just a read of the latest results. The trigger is not aligned with
+	 * the measurements, so the timestamp is taken in the handler.
+	 */
+	rc = devm_iio_triggered_buffer_setup(dev, iio, NULL,
+					     dps310_trigger_handler, NULL);
+	if (rc)
+		return rc;
+
+	rc = devm_iio_device_register(dev, iio);
 	if (rc)
 		return rc;
 
diff --git a/drivers/iio/pressure/rohm-bm1390.c b/drivers/iio/pressure/rohm-bm1390.c
index 57941fb4a53593..7a6a16cffb66a6 100644
--- a/drivers/iio/pressure/rohm-bm1390.c
+++ b/drivers/iio/pressure/rohm-bm1390.c
@@ -267,7 +267,7 @@ static int bm1390_read_data(struct bm1390_data *data,
 	guard(mutex)(&data->mutex);
 	/*
 	 * We use 'continuous mode' even for raw read because according to the
-	 * data-sheet an one-shot mode can't be used with IIR filter.
+	 * data-sheet a one-shot mode can't be used with IIR filter.
 	 */
 	ret = bm1390_meas_set(data, BM1390_MEAS_MODE_CONTINUOUS);
 	if (ret)
@@ -289,7 +289,7 @@ static int bm1390_read_data(struct bm1390_data *data,
 	if (warn)
 		dev_warn(data->dev, "Failed to stop measurement (%d)\n", warn);
 
-	return 0;
+	return ret;
 }
 
 static int bm1390_read_raw(struct iio_dev *idev,
@@ -479,6 +479,7 @@ static const struct iio_info bm1390_info = {
 
 static int bm1390_chip_init(struct bm1390_data *data)
 {
+	u8 regval;
 	int ret;
 
 	ret = regmap_write_bits(data->regmap, BM1390_REG_POWER,
@@ -512,8 +513,9 @@ static int bm1390_chip_init(struct bm1390_data *data)
 	 * Default to use IIR filter in "middle" mode. Also the AVE_NUM must
 	 * be fixed when IIR is in use.
 	 */
+	regval = FIELD_PREP(BM1390_MASK_AVE_NUM, BM1390_IIR_AVE_NUM);
 	ret = regmap_update_bits(data->regmap, BM1390_REG_MODE_CTRL,
-				 BM1390_MASK_AVE_NUM, BM1390_IIR_AVE_NUM);
+				 BM1390_MASK_AVE_NUM, regval);
 	if (ret)
 		return ret;
 
diff --git a/drivers/iio/proximity/aw96103.c b/drivers/iio/proximity/aw96103.c
index 8fbb755dcae040..e3ab55170b6c2a 100644
--- a/drivers/iio/proximity/aw96103.c
+++ b/drivers/iio/proximity/aw96103.c
@@ -24,6 +24,7 @@
 #define AW96103_BIN_VALID_DATA_OFFSET		64
 #define AW96103_BIN_DATA_LEN_OFFSET		16
 #define AW96103_BIN_DATA_REG_NUM_SIZE		4
+#define AW96103_BIN_REG_SIZE			6
 #define AW96103_BIN_CHIP_TYPE_SIZE		8
 #define AW96103_BIN_CHIP_TYPE_OFFSET		24
 
@@ -229,14 +230,27 @@ static const struct aw_chip_info aw_chip_info_tbl[] = {
 	},
 };
 
-static void aw96103_parsing_bin_file(struct aw_bin *bin)
+static int aw96103_parsing_bin_file(struct aw_bin *bin)
 {
+	u32 data_len;
+
+	if (bin->len < AW96103_BIN_VALID_DATA_OFFSET)
+		return -EINVAL;
+
+	data_len = get_unaligned_le32(bin->data + AW96103_BIN_DATA_LEN_OFFSET);
+	if (data_len < AW96103_BIN_DATA_REG_NUM_SIZE)
+		return -EINVAL;
+
+	bin->valid_data_len = data_len - AW96103_BIN_DATA_REG_NUM_SIZE;
+	if (bin->valid_data_len > bin->len - AW96103_BIN_VALID_DATA_OFFSET ||
+	    bin->valid_data_len % AW96103_BIN_REG_SIZE)
+		return -EINVAL;
+
 	bin->valid_data_addr = AW96103_BIN_VALID_DATA_OFFSET;
-	bin->valid_data_len =
-		*(unsigned int *)(bin->data + AW96103_BIN_DATA_LEN_OFFSET) -
-		AW96103_BIN_DATA_REG_NUM_SIZE;
 	memcpy(bin->chip_type, bin->data + AW96103_BIN_CHIP_TYPE_OFFSET,
 	       AW96103_BIN_CHIP_TYPE_SIZE);
+
+	return 0;
 }
 
 static const struct regmap_config aw96103_regmap_confg = {
@@ -500,7 +514,7 @@ static int aw96103_bin_valid_loaded(struct aw96103 *aw96103,
 	int ret;
 
 	for (i = 0; i < aw_bin_data_s->valid_data_len;
-	     i += 6, start_addr += 6) {
+	     i += AW96103_BIN_REG_SIZE, start_addr += AW96103_BIN_REG_SIZE) {
 		reg_addr = get_unaligned_le16(aw_bin_data_s->data + start_addr);
 		reg_data = get_unaligned_le32(aw_bin_data_s->data +
 					      start_addr + 2);
@@ -550,6 +564,8 @@ static int aw96103_para_loaded(struct aw96103 *aw96103)
 static int aw96103_cfg_all_loaded(const struct firmware *cont,
 				  struct aw96103 *aw96103)
 {
+	int ret;
+
 	if (!cont)
 		return -EINVAL;
 
@@ -561,7 +577,9 @@ static int aw96103_cfg_all_loaded(const struct firmware *cont,
 	aw_bin->len = cont->size;
 	memcpy(aw_bin->data, cont->data, cont->size);
 	release_firmware(cont);
-	aw96103_parsing_bin_file(aw_bin);
+	ret = aw96103_parsing_bin_file(aw_bin);
+	if (ret)
+		return ret;
 
 	return aw96103_bin_valid_loaded(aw96103, aw_bin);
 }
@@ -669,7 +687,7 @@ static irqreturn_t aw96103_irq(int irq, void *data)
 				       iio_get_time_ns(indio_dev));
 			break;
 		default:
-			return IRQ_HANDLED;
+			continue;
 		}
 		aw96103->channels_arr[i].old_irq_status = curr_status;
 	}
diff --git a/drivers/iio/proximity/isl29501.c b/drivers/iio/proximity/isl29501.c
index 95fb7238f67887..a98a9975323c60 100644
--- a/drivers/iio/proximity/isl29501.c
+++ b/drivers/iio/proximity/isl29501.c
@@ -226,7 +226,7 @@ static int isl29501_register_read(struct isl29501_private *isl29501,
 	return ret;
 }
 
-static u32 isl29501_register_write(struct isl29501_private *isl29501,
+static int isl29501_register_write(struct isl29501_private *isl29501,
 				   enum isl29501_register_name name,
 				   u32 value)
 {
diff --git a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
index 400477b4c740e3..8a9ee21f2bf6aa 100644
--- a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
+++ b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
@@ -294,12 +294,14 @@ static int lidar_probe(struct i2c_client *client)
 
 	ret = pm_runtime_set_active(&client->dev);
 	if (ret)
-		goto error_unreg_buffer;
+		goto error_unreg_dev;
 	pm_runtime_enable(&client->dev);
 	pm_runtime_idle(&client->dev);
 
 	return 0;
 
+error_unreg_dev:
+	iio_device_unregister(indio_dev);
 error_unreg_buffer:
 	iio_triggered_buffer_cleanup(indio_dev);
 
diff --git a/drivers/iio/proximity/sx9324.c b/drivers/iio/proximity/sx9324.c
index 13b4ef2896e35c..255a36976ac41f 100644
--- a/drivers/iio/proximity/sx9324.c
+++ b/drivers/iio/proximity/sx9324.c
@@ -222,7 +222,7 @@ static const struct iio_chan_spec_ext_info sx9324_channel_ext_info[] = {
 	.scan_index = idx,					 \
 	.scan_type = {						 \
 		.sign = 's',					 \
-		.realbits = 12,					 \
+		.realbits = 16,					 \
 		.storagebits = 16,				 \
 		.endianness = IIO_BE,				 \
 	},							 \
diff --git a/drivers/iio/proximity/vcnl3020.c b/drivers/iio/proximity/vcnl3020.c
index 6b0d7a9038d90e..768870127ff874 100644
--- a/drivers/iio/proximity/vcnl3020.c
+++ b/drivers/iio/proximity/vcnl3020.c
@@ -584,16 +584,26 @@ static irqreturn_t vcnl3020_handle_irq_thread(int irq, void *p)
 		return IRQ_HANDLED;
 	}
 
-	if (!(isr & VCNL_ICR_THRES_EN))
+	if (!(isr & (VCNL_INT_TH_HI | VCNL_INT_TH_LOW)))
 		return IRQ_NONE;
 
-	iio_push_event(indio_dev,
-		       IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 1,
-				            IIO_EV_TYPE_THRESH,
-				            IIO_EV_DIR_RISING),
-		       iio_get_time_ns(indio_dev));
+	if (isr & VCNL_INT_TH_HI) {
+		iio_push_event(indio_dev,
+			       IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0,
+						    IIO_EV_TYPE_THRESH,
+						    IIO_EV_DIR_RISING),
+			       iio_get_time_ns(indio_dev));
+	}
+
+	if (isr & VCNL_INT_TH_LOW) {
+		iio_push_event(indio_dev,
+			       IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0,
+						    IIO_EV_TYPE_THRESH,
+						    IIO_EV_DIR_FALLING),
+			       iio_get_time_ns(indio_dev));
+	}
 
-	rc = regmap_write(data->regmap, VCNL_ISR, isr & VCNL_ICR_THRES_EN);
+	rc = regmap_write(data->regmap, VCNL_ISR, isr & (VCNL_INT_TH_HI | VCNL_INT_TH_LOW));
 	if (rc)
 		dev_err(data->dev, "Error (%d) writing in reg (0x%x)\n",
 			rc, VCNL_ISR);
diff --git a/drivers/iio/proximity/vl53l0x-i2c.c b/drivers/iio/proximity/vl53l0x-i2c.c
index 9fe14ceb8be793..173e69753df7db 100644
--- a/drivers/iio/proximity/vl53l0x-i2c.c
+++ b/drivers/iio/proximity/vl53l0x-i2c.c
@@ -219,7 +219,10 @@ static int vl53l0x_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
+		if (!iio_device_claim_direct(indio_dev))
+			return -EBUSY;
 		ret = vl53l0x_read_proximity(data, chan, val);
+		iio_device_release_direct(indio_dev);
 		if (ret < 0)
 			return ret;
 
diff --git a/drivers/iio/temperature/iqs620at-temp.c b/drivers/iio/temperature/iqs620at-temp.c
index e2f878d57af791..3a3b1f25658505 100644
--- a/drivers/iio/temperature/iqs620at-temp.c
+++ b/drivers/iio/temperature/iqs620at-temp.c
@@ -5,13 +5,18 @@
  * Copyright (C) 2019 Jeff LaBundy <jeff@labundy.com>
  */
 
+#include <linux/array_size.h>
+#include <linux/bits.h>
 #include <linux/device.h>
-#include <linux/iio/iio.h>
-#include <linux/kernel.h>
 #include <linux/mfd/iqs62x.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/types.h>
+
+#include <asm/byteorder.h>
+
+#include <linux/iio/iio.h>
 
 #define IQS620_TEMP_UI_OUT			0x1A
 
diff --git a/drivers/iio/temperature/ltc2983.c b/drivers/iio/temperature/ltc2983.c
index 6efb5252a77348..6655453c9abf94 100644
--- a/drivers/iio/temperature/ltc2983.c
+++ b/drivers/iio/temperature/ltc2983.c
@@ -5,23 +5,34 @@
  *
  * Copyright 2019 Analog Devices Inc.
  */
+
+#include <linux/array_size.h>
 #include <linux/bitfield.h>
+#include <linux/bitops.h>
 #include <linux/completion.h>
+#include <linux/container_of.h>
+#include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
-#include <linux/errno.h>
-#include <linux/kernel.h>
-#include <linux/iio/iio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/list.h>
+#include <linux/math64.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
 #include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
+#include <linux/slab.h>
 #include <linux/spi/spi.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
 
 #include <asm/byteorder.h>
-#include <linux/unaligned.h>
+
+#include <linux/iio/iio.h>
 
 /* register map */
 #define LTC2983_STATUS_REG			0x0000
diff --git a/drivers/iio/temperature/mlx90632.c b/drivers/iio/temperature/mlx90632.c
index 2fde48f337e2cb..74ae36f74e8b90 100644
--- a/drivers/iio/temperature/mlx90632.c
+++ b/drivers/iio/temperature/mlx90632.c
@@ -6,21 +6,25 @@
  *
  * Driver for the Melexis MLX90632 I2C 16-bit IR thermopile sensor
  */
+
+#include <linux/array_size.h>
 #include <linux/bitfield.h>
+#include <linux/bits.h>
 #include <linux/delay.h>
+#include <linux/dev_printk.h>
 #include <linux/device.h>
 #include <linux/err.h>
-#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/iopoll.h>
 #include <linux/jiffies.h>
-#include <linux/kernel.h>
 #include <linux/limits.h>
-#include <linux/module.h>
 #include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
+#include <linux/types.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
diff --git a/drivers/iio/temperature/mlx90635.c b/drivers/iio/temperature/mlx90635.c
index 0f31879a4f64f0..48c760c980f3dc 100644
--- a/drivers/iio/temperature/mlx90635.c
+++ b/drivers/iio/temperature/mlx90635.c
@@ -6,21 +6,24 @@
  *
  * Driver for the Melexis MLX90635 I2C 16-bit IR thermopile sensor
  */
+
+#include <linux/array_size.h>
 #include <linux/bitfield.h>
+#include <linux/bits.h>
 #include <linux/delay.h>
+#include <linux/dev_printk.h>
 #include <linux/device.h>
 #include <linux/err.h>
-#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/iopoll.h>
 #include <linux/jiffies.h>
-#include <linux/kernel.h>
-#include <linux/limits.h>
-#include <linux/module.h>
 #include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
+#include <linux/types.h>
 
 #include <linux/iio/iio.h>
 
diff --git a/drivers/iio/temperature/tmp117.c b/drivers/iio/temperature/tmp117.c
index 74cb8d62bef3cb..2d0e79416434be 100644
--- a/drivers/iio/temperature/tmp117.c
+++ b/drivers/iio/temperature/tmp117.c
@@ -10,16 +10,17 @@
  */
 
 #include <linux/array_size.h>
+#include <linux/bitops.h>
 #include <linux/delay.h>
+#include <linux/dev_printk.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
-#include <linux/module.h>
-#include <linux/bitops.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
 #include <linux/limits.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
 #include <linux/property.h>
 #include <linux/regulator/consumer.h>
+#include <linux/types.h>
 
 #include <linux/iio/iio.h>
 
@@ -97,15 +98,20 @@ static int tmp117_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec
 {
 	struct tmp117_data *data = iio_priv(indio_dev);
 	s16 off;
+	int ret;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_CALIBBIAS:
 		off = clamp_t(int, val, S16_MIN, S16_MAX);
 		if (off == data->calibbias)
 			return 0;
+
+		ret = i2c_smbus_write_word_swapped(data->client, TMP117_REG_TEMP_OFFSET, off);
+		if (ret)
+			return ret;
+
 		data->calibbias = off;
-		return i2c_smbus_write_word_swapped(data->client,
-						TMP117_REG_TEMP_OFFSET, off);
+		return 0;
 
 	default:
 		return -EINVAL;
diff --git a/drivers/iio/temperature/tsys01.c b/drivers/iio/temperature/tsys01.c
index 28b3ce022ce7b2..b19b4a3ef03806 100644
--- a/drivers/iio/temperature/tsys01.c
+++ b/drivers/iio/temperature/tsys01.c
@@ -8,14 +8,19 @@
  *  http://www.meas-spec.com/downloads/TSYS01_Digital_Temperature_Sensor.pdf
  */
 
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
+#include <linux/array_size.h>
+#include <linux/bits.h>
 #include <linux/device.h>
-#include <linux/mutex.h>
+#include <linux/i2c.h>
+#include <linux/math64.h>
 #include <linux/module.h>
-#include <linux/init.h>
-#include <linux/kernel.h>
-#include <linux/stat.h>
+#include <linux/mutex.h>
+#include <linux/sprintf.h>
+#include <linux/types.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
 #include "../common/ms_sensors/ms_sensors_i2c.h"
 
 /* TSYS01 Commands */
diff --git a/drivers/iio/temperature/tsys02d.c b/drivers/iio/temperature/tsys02d.c
index 3ef72347456ec6..2c67faebd90c14 100644
--- a/drivers/iio/temperature/tsys02d.c
+++ b/drivers/iio/temperature/tsys02d.c
@@ -10,11 +10,17 @@
  *  http://www.meas-spec.com/downloads/Digital_Sensor_TSYS02D.pdf
  */
 
-#include <linux/init.h>
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/dev_printk.h>
 #include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/stat.h>
+#include <linux/i2c.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/stat.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
diff --git a/drivers/iio/test/Kconfig b/drivers/iio/test/Kconfig
index 4fc17dd0dcd770..04930b4fdd5d5d 100644
--- a/drivers/iio/test/Kconfig
+++ b/drivers/iio/test/Kconfig
@@ -4,6 +4,19 @@
 #
 
 # Keep in alphabetical order
+config IIO_CHANNEL_PREFIX_KUNIT_TEST
+	tristate "Test IIO channel prefix" if !KUNIT_ALL_TESTS
+	depends on KUNIT && IIO
+	default KUNIT_ALL_TESTS
+	help
+	  Build unit tests for __iio_chan_prefix_emit(), the helper that
+	  builds IIO sysfs attribute name prefixes from an iio_chan_spec.
+
+	  For more information on KUnit and unit tests in general, please refer
+	  to the KUnit documentation in Documentation/dev-tools/kunit/.
+
+	  If unsure, say N.
+
 config IIO_GTS_KUNIT_TEST
 	tristate "Test IIO gain-time-scale helpers" if !KUNIT_ALL_TESTS
 	depends on KUNIT
diff --git a/drivers/iio/test/Makefile b/drivers/iio/test/Makefile
index 0c846bc21acda5..ba08afa7f3f22b 100644
--- a/drivers/iio/test/Makefile
+++ b/drivers/iio/test/Makefile
@@ -4,6 +4,7 @@
 #
 
 # Keep in alphabetical order
+obj-$(CONFIG_IIO_CHANNEL_PREFIX_KUNIT_TEST) += iio-test-channel-prefix.o
 obj-$(CONFIG_IIO_RESCALE_KUNIT_TEST) += iio-test-rescale.o
 obj-$(CONFIG_IIO_FORMAT_KUNIT_TEST) += iio-test-format.o
 obj-$(CONFIG_IIO_GTS_KUNIT_TEST) += iio-test-gts.o
diff --git a/drivers/iio/test/iio-test-channel-prefix.c b/drivers/iio/test/iio-test-channel-prefix.c
new file mode 100644
index 00000000000000..44c838fcd8d621
--- /dev/null
+++ b/drivers/iio/test/iio-test-channel-prefix.c
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Unit tests for IIO channel prefix generation.
+ */
+
+#include <kunit/test.h>
+#include <kunit/visibility.h>
+
+#include <linux/iio/iio.h>
+#include <linux/limits.h>
+#include <linux/string.h>
+
+#include "../iio_core.h"
+
+#define PREFIX_BUF_SIZE		(NAME_MAX + 1)
+
+#define EXPECT_PREFIX(_test, _buf, _ret, _expected) do {			\
+		struct kunit *__test = (_test);					\
+		const char *__expected = (_expected);				\
+										\
+		KUNIT_EXPECT_EQ(__test, (_ret), (ssize_t)strlen(__expected));	\
+		KUNIT_EXPECT_STREQ(__test, (_buf), __expected);			\
+	} while (0)
+
+static char *iio_test_prefix_alloc(struct kunit *test)
+{
+	char *buf = kunit_kzalloc(test, PREFIX_BUF_SIZE, GFP_KERNEL);
+
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+	return buf;
+}
+
+static void iio_test_prefix_shared_by_all(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+	};
+	const struct iio_chan_spec chan_noisy = {
+		.type = IIO_ACCEL,
+		.output = 1,
+		.indexed = 1,
+		.modified = 1,
+		.channel = 7,
+		.channel2 = IIO_MOD_Z,
+		.extend_name = "supply",
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SHARED_BY_ALL,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "");
+
+	ret = __iio_chan_prefix_emit(NULL, &chan_noisy, IIO_SHARED_BY_ALL,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "");
+}
+
+static void iio_test_prefix_shared_by_dir(struct kunit *test)
+{
+	const struct iio_chan_spec chan_in = {
+		.type = IIO_VOLTAGE,
+		.output = 0,
+	};
+	const struct iio_chan_spec chan_out = {
+		.type = IIO_VOLTAGE,
+		.output = 1,
+	};
+	const struct iio_chan_spec chan_in_noisy = {
+		.type = IIO_ACCEL,
+		.indexed = 1,
+		.modified = 1,
+		.channel = 5,
+		.channel2 = IIO_MOD_Y,
+		.extend_name = "supply",
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan_in, IIO_SHARED_BY_DIR,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in");
+
+	ret = __iio_chan_prefix_emit(NULL, &chan_out, IIO_SHARED_BY_DIR,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "out");
+
+	ret = __iio_chan_prefix_emit(NULL, &chan_in_noisy, IIO_SHARED_BY_DIR,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in");
+}
+
+static void iio_test_prefix_shared_by_type(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+	};
+	const struct iio_chan_spec chan_diff = {
+		.type = IIO_VOLTAGE,
+		.differential = 1,
+	};
+	const struct iio_chan_spec chan_noisy = {
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.modified = 1,
+		.channel = 4,
+		.channel2 = IIO_MOD_X,
+		.extend_name = "supply",
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SHARED_BY_TYPE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_voltage");
+
+	ret = __iio_chan_prefix_emit(NULL, &chan_diff, IIO_SHARED_BY_TYPE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_voltage-voltage");
+
+	ret = __iio_chan_prefix_emit(NULL, &chan_noisy, IIO_SHARED_BY_TYPE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_voltage");
+}
+
+static void iio_test_prefix_separate_simple(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_TEMP,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_temp");
+}
+
+static void iio_test_prefix_separate_indexed(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.channel = 3,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_voltage3");
+}
+
+static void iio_test_prefix_separate_indexed_diff(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.differential = 1,
+		.channel = 0,
+		.channel2 = 1,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_voltage0-voltage1");
+}
+
+static void iio_test_prefix_separate_modified(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_ACCEL,
+		.modified = 1,
+		.channel2 = IIO_MOD_X,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_accel_x");
+}
+
+static void iio_test_prefix_separate_indexed_modified(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_ACCEL,
+		.indexed = 1,
+		.modified = 1,
+		.channel = 2,
+		.channel2 = IIO_MOD_Y,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_accel2_y");
+}
+
+static void iio_test_prefix_separate_extend_name(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.channel = 2,
+		.extend_name = "supply",
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "in_voltage2_supply");
+}
+
+static void iio_test_prefix_output_separate(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+		.output = 1,
+		.indexed = 1,
+		.channel = 0,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	EXPECT_PREFIX(test, buf, ret, "out_voltage0");
+}
+
+static void iio_test_prefix_diff_unindexed_fails(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+		.differential = 1,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+static void iio_test_prefix_diff_modified_fails(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+		.indexed = 1,
+		.differential = 1,
+		.modified = 1,
+		.channel = 0,
+		.channel2 = 1,
+	};
+	char *buf = iio_test_prefix_alloc(test);
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SEPARATE,
+				     buf, PREFIX_BUF_SIZE);
+	KUNIT_EXPECT_EQ(test, ret, -EINVAL);
+}
+
+static void iio_test_prefix_overflow(struct kunit *test)
+{
+	const struct iio_chan_spec chan = {
+		.type = IIO_VOLTAGE,
+	};
+	char small[4];
+	ssize_t ret;
+
+	ret = __iio_chan_prefix_emit(NULL, &chan, IIO_SHARED_BY_TYPE,
+				     small, sizeof(small));
+	KUNIT_EXPECT_EQ(test, ret, -EOVERFLOW);
+}
+
+static struct kunit_case iio_chan_prefix_test_cases[] = {
+	KUNIT_CASE(iio_test_prefix_shared_by_all),
+	KUNIT_CASE(iio_test_prefix_shared_by_dir),
+	KUNIT_CASE(iio_test_prefix_shared_by_type),
+	KUNIT_CASE(iio_test_prefix_separate_simple),
+	KUNIT_CASE(iio_test_prefix_separate_indexed),
+	KUNIT_CASE(iio_test_prefix_separate_indexed_diff),
+	KUNIT_CASE(iio_test_prefix_separate_modified),
+	KUNIT_CASE(iio_test_prefix_separate_indexed_modified),
+	KUNIT_CASE(iio_test_prefix_separate_extend_name),
+	KUNIT_CASE(iio_test_prefix_output_separate),
+	KUNIT_CASE(iio_test_prefix_diff_unindexed_fails),
+	KUNIT_CASE(iio_test_prefix_diff_modified_fails),
+	KUNIT_CASE(iio_test_prefix_overflow),
+	{ }
+};
+
+static struct kunit_suite iio_channel_prefix_test_suite = {
+	.name = "iio-channel-prefix",
+	.test_cases = iio_chan_prefix_test_cases,
+};
+
+kunit_test_suite(iio_channel_prefix_test_suite);
+
+MODULE_AUTHOR("Rodrigo Alencar <rodrigo.alencar@analog.com>");
+MODULE_DESCRIPTION("Test IIO channel prefix generation");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
diff --git a/drivers/iio/test/iio-test-rescale.c b/drivers/iio/test/iio-test-rescale.c
index bcd4a607be192b..de4a09f432c034 100644
--- a/drivers/iio/test/iio-test-rescale.c
+++ b/drivers/iio/test/iio-test-rescale.c
@@ -587,7 +587,7 @@ KUNIT_ARRAY_PARAM(iio_rescale_offset, offset_cases, case_to_desc);
  * @str: The string to parse
  * @nano: The number as an integer
  *
- * Returns 0 on success, or a negative error code if the string cound not be
+ * Returns 0 on success, or a negative error code if the string could not be
  * parsed.
  */
 static int iio_str_to_nano(const char *str, s64 *nano)
@@ -617,7 +617,7 @@ static int iio_str_to_nano(const char *str, s64 *nano)
  * @real_str: The real value as a string
  * @exp_str: The expected value as a string
  *
- * Returns a negative error code if the strings cound not be parsed, or the
+ * Returns a negative error code if the strings could not be parsed, or the
  * relative error in parts-per-million.
  */
 static int iio_test_relative_error_ppm(const char *real_str, const char *exp_str)
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 73170b15fc3d2d..337a49d1acf729 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -4823,6 +4823,29 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
 }
 EXPORT_SYMBOL(rdma_reject);
 
+static int cma_force_disconnect(struct rdma_id_private *id_priv)
+{
+	struct cma_work *work;
+
+	work = kzalloc_obj(*work);
+	if (!work)
+		return -ENOMEM;
+
+	INIT_WORK(&work->work, cma_work_handler);
+	work->old_state = RDMA_CM_CONNECT;
+	work->new_state = RDMA_CM_DISCONNECT;
+	work->event.event = RDMA_CM_EVENT_DISCONNECTED;
+	work->event.status = -ECONNABORTED;
+
+	cma_id_get(id_priv);
+	work->id = id_priv;
+
+	trace_cm_force_disconnect(id_priv);
+
+	queue_work(cma_wq, &work->work);
+	return 0;
+}
+
 int rdma_disconnect(struct rdma_cm_id *id)
 {
 	struct rdma_id_private *id_priv;
@@ -4838,12 +4861,15 @@ int rdma_disconnect(struct rdma_cm_id *id)
 			goto out;
 		/* Initiate or respond to a disconnect. */
 		trace_cm_disconnect(id_priv);
-		if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
-			if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
-				trace_cm_sent_drep(id_priv);
-		} else {
+		if (!ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
 			trace_cm_sent_dreq(id_priv);
+			goto out;
+		}
+		if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) {
+			trace_cm_sent_drep(id_priv);
+			goto out;
 		}
+		ret = cma_force_disconnect(id_priv);
 	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
 		ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
 	} else
diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h
index 3456d5f3aa47ea..3d05d8e80856a6 100644
--- a/drivers/infiniband/core/cma_trace.h
+++ b/drivers/infiniband/core/cma_trace.h
@@ -61,6 +61,7 @@ DEFINE_CMA_FSM_EVENT(send_sidr_rep);
 DEFINE_CMA_FSM_EVENT(disconnect);
 DEFINE_CMA_FSM_EVENT(sent_drep);
 DEFINE_CMA_FSM_EVENT(sent_dreq);
+DEFINE_CMA_FSM_EVENT(force_disconnect);
 DEFINE_CMA_FSM_EVENT(id_destroy);
 
 TRACE_EVENT(cm_id_attach,
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 7a3ed5ecac0012..29af9fe860e27f 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -893,7 +893,7 @@ static int setup_port_data(struct ib_device *device)
  * ib_port_immutable_read() - Read rdma port's immutable data
  * @dev: IB device
  * @port: port number whose immutable data to read. It starts with index 1 and
- *        valid upto including rdma_end_port().
+ *        valid up to including rdma_end_port().
  */
 const struct ib_port_immutable*
 ib_port_immutable_read(struct ib_device *dev, unsigned int port)
@@ -3067,7 +3067,7 @@ int ib_add_sub_device(struct ib_device *parent,
 	sub->parent = parent;
 
 	mutex_lock(&parent->subdev_lock);
-	list_add_tail(&parent->subdev_list_head, &sub->subdev_list);
+	list_add_tail(&sub->subdev_list, &parent->subdev_list_head);
 	mutex_unlock(&parent->subdev_lock);
 
 	return ret;
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 0b7246ec559e60..4b05867dad895a 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -936,7 +936,7 @@ static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv,
 /*
  * CM_ID <-- IDLE
  *
- * If in the ESTBLISHED or CLOSING states, the QP will have have been
+ * If in the ESTBLISHED or CLOSING states, the QP will have been
  * moved by the provider to the ERR state. Disassociate the CM_ID from
  * the QP,  move to IDLE, and remove the 'connected' reference.
  *
diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c
index 00fe241737ffee..88aac911e8ee1c 100644
--- a/drivers/infiniband/core/lag.c
+++ b/drivers/infiniband/core/lag.c
@@ -59,7 +59,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev,
 		ip6h->version = 6;
 		ip6h->nexthdr = IPPROTO_UDP;
 		memcpy(&ip6h->flow_lbl, &ah_attr->grh.flow_label,
-		       sizeof(*ip6h->flow_lbl));
+		       sizeof(ip6h->flow_lbl));
 		memcpy(&ip6h->saddr, ah_attr->grh.sgid_attr->gid.raw,
 		       sizeof(struct in6_addr));
 		memcpy(&ip6h->daddr, ah_attr->grh.dgid.raw,
diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c
index 17c4c52a19e4c4..f05e9ce93c3353 100644
--- a/drivers/infiniband/core/mad_rmpp.c
+++ b/drivers/infiniband/core/mad_rmpp.c
@@ -245,12 +245,15 @@ static void recv_timeout_handler(struct work_struct *work)
 	struct ib_mad_recv_wc *rmpp_wc;
 	unsigned long flags;
 
-	spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
+	spin_lock_irqsave(&rmpp_recv->lock, flags);
 	if (rmpp_recv->state != RMPP_STATE_ACTIVE) {
-		spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
+		spin_unlock_irqrestore(&rmpp_recv->lock, flags);
 		return;
 	}
 	rmpp_recv->state = RMPP_STATE_TIMEOUT;
+	spin_unlock_irqrestore(&rmpp_recv->lock, flags);
+
+	spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
 	list_del(&rmpp_recv->list);
 	spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
 
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index bea7df3dd8f32d..f549fd928c1c65 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -189,7 +189,10 @@ static void release_group(struct mcast_group *group)
 
 	spin_lock_irqsave(&port->lock, flags);
 	if (atomic_dec_and_test(&group->refcount)) {
-		rb_erase(&group->node, &port->table);
+
+		if (!RB_EMPTY_NODE(&group->node))
+			rb_erase(&group->node, &port->table);
+
 		spin_unlock_irqrestore(&port->lock, flags);
 		kfree(group);
 		deref_port(port);
@@ -533,6 +536,8 @@ static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
 		group->rec = *rec;
 		if (mgids_changed) {
 			rb_erase(&group->node, &group->port->table);
+
+			RB_CLEAR_NODE(&group->node);
 			is_mgid0 = !memcmp(&mgid0, &group->rec.mgid,
 					   sizeof(mgid0));
 			mcast_insert(group->port, group, is_mgid0);
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 4e8fbee3474547..849a5db54e1cb0 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -700,7 +700,8 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin,
 
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
 		return -EMSGSIZE;
-	if (!rdma_is_kernel_res(res) &&
+
+	if (cq->uobject && cq->uobject->uevent.uobject.context &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 			cq->uobject->uevent.uobject.context->res.id))
 		return -EMSGSIZE;
@@ -790,7 +791,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id))
 		goto err;
 
-	if (!rdma_is_kernel_res(res) &&
+	if (pd->uobject && pd->uobject->context &&
 	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
 			pd->uobject->context->res.id))
 		goto err;
@@ -1895,8 +1896,10 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
 		mutex_lock(&nldev_dellink_mutex);
 		err = device->link_ops->dellink(device);
 		mutex_unlock(&nldev_dellink_mutex);
-		if (err)
+		if (err) {
+			ib_device_put(device);
 			return err;
+		}
 	}
 
 	ib_unregister_device_and_put(device);
diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index f89a81dad72faf..a8b838fdc7b5cf 100644
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -250,8 +250,7 @@ void rdma_restrack_add(struct rdma_restrack_entry *res)
 		ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL);
 		if (ret)
 			res->id = 0;
-
-		if (qp->qp_type >= IB_QPT_DRIVER)
+		else if (qp->qp_type >= IB_QPT_DRIVER)
 			xa_set_mark(&rt->xa, res->id, RESTRACK_DD);
 	} else if (res->type == RDMA_RESTRACK_COUNTER) {
 		/* Special case to ensure that cntn points to right counter */
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 88110b9661f57a..b1ab4f0359b018 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -55,7 +55,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
 
 	if (dirty)
 		ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt,
-					   DMA_BIDIRECTIONAL, umem->dma_attrs);
+					   umem->dma_dir, umem->dma_attrs);
 
 	for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) {
 		unpin_user_page_range_dirty_lock(sg_page(sg),
@@ -159,9 +159,15 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
 }
 EXPORT_SYMBOL(ib_umem_find_best_pgsz);
 
+static inline enum dma_data_direction ib_access_dma_dir(int access)
+{
+	return ib_access_writable(access) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+}
+
 static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 					unsigned long addr, size_t size,
-					int access)
+					int access,
+					enum dma_data_direction dir)
 {
 	struct ib_umem *umem;
 	struct page **page_list;
@@ -202,6 +208,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 	 */
 	umem->iova = addr;
 	umem->writable   = ib_access_writable(access);
+	umem->dma_dir    = dir;
 	umem->owning_mm = mm = current->mm;
 	umem->dma_attrs = DMA_ATTR_REQUIRE_COHERENT;
 	if (access & IB_ACCESS_RELAXED_ORDERING)
@@ -261,7 +268,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 	}
 
 	ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt,
-				       DMA_BIDIRECTIONAL, umem->dma_attrs);
+				       dir, umem->dma_attrs);
 	if (ret)
 		goto umem_release;
 	goto out;
@@ -279,17 +286,16 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
 	return ret ? ERR_PTR(ret) : umem;
 }
 
-/**
- * ib_umem_get_desc - Pin a umem from a buffer descriptor.
- * @device: IB device.
- * @desc:   buffer descriptor (VA or DMABUF).
- * @access: IB access flags.
+/*
+ * __ib_umem_get_desc_dir - core implementation for ib_umem_get_desc().
  *
- * Return: caller-owned umem on success, ERR_PTR(...) on error.
+ * @dir applies to VA buffers only; dmabuf direction is managed by the
+ * dmabuf subsystem and this argument is ignored for that type.
  */
-struct ib_umem *ib_umem_get_desc(struct ib_device *device,
-				 const struct ib_uverbs_buffer_desc *desc,
-				 int access)
+static struct ib_umem *
+__ib_umem_get_desc_dir(struct ib_device *device,
+		       const struct ib_uverbs_buffer_desc *desc,
+		       int access, enum dma_data_direction dir)
 {
 	struct ib_umem_dmabuf *umem_dmabuf;
 
@@ -310,11 +316,27 @@ struct ib_umem *ib_umem_get_desc(struct ib_device *device,
 		return &umem_dmabuf->umem;
 	case IB_UVERBS_BUFFER_TYPE_VA:
 		return __ib_umem_get_va(device, desc->addr, desc->length,
-					access);
+					access, dir);
 	default:
 		return ERR_PTR(-EINVAL);
 	}
 }
+
+/**
+ * ib_umem_get_desc - Pin a umem from a buffer descriptor.
+ * @device: IB device.
+ * @desc:   buffer descriptor (VA or DMABUF).
+ * @access: IB access flags.
+ *
+ * Return: caller-owned umem on success, ERR_PTR(...) on error.
+ */
+struct ib_umem *ib_umem_get_desc(struct ib_device *device,
+				 const struct ib_uverbs_buffer_desc *desc,
+				 int access)
+{
+	return __ib_umem_get_desc_dir(device, desc, access,
+				      ib_access_dma_dir(access));
+}
 EXPORT_SYMBOL(ib_umem_get_desc);
 
 /*
@@ -376,11 +398,12 @@ static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs,
 static struct ib_umem *
 ib_umem_get_desc_check(struct ib_device *device,
 		       const struct ib_uverbs_buffer_desc *desc,
-		       size_t min_size, int access)
+		       size_t min_size, int access,
+		       enum dma_data_direction dir)
 {
 	struct ib_umem *umem;
 
-	umem = ib_umem_get_desc(device, desc, access);
+	umem = __ib_umem_get_desc_dir(device, desc, access, dir);
 	if (IS_ERR(umem))
 		return umem;
 	if (umem->length < min_size) {
@@ -401,7 +424,7 @@ static struct ib_umem *
 ib_umem_get_from_attrs(struct ib_device *device,
 		       const struct uverbs_attr_bundle *attrs,
 		       u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler,
-		       size_t size, int access)
+		       size_t size, int access, enum dma_data_direction dir)
 {
 	struct ib_uverbs_buffer_desc desc = {};
 	int ret;
@@ -411,7 +434,7 @@ ib_umem_get_from_attrs(struct ib_device *device,
 		return NULL;
 	if (ret)
 		return ERR_PTR(ret);
-	return ib_umem_get_desc_check(device, &desc, size, access);
+	return ib_umem_get_desc_check(device, &desc, size, access, dir);
 }
 
 /*
@@ -431,7 +454,8 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device,
 			     const struct uverbs_attr_bundle *attrs,
 			     u16 attr_id,
 			     ib_umem_buf_desc_filler_t legacy_filler,
-			     u64 addr, size_t size, int access)
+			     u64 addr, size_t size, int access,
+			     enum dma_data_direction dir)
 {
 	struct ib_uverbs_buffer_desc desc = {};
 	int ret;
@@ -445,7 +469,7 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device,
 		};
 	else if (ret)
 		return ERR_PTR(ret);
-	return ib_umem_get_desc_check(device, &desc, size, access);
+	return ib_umem_get_desc_check(device, &desc, size, access, dir);
 }
 
 /**
@@ -464,7 +488,7 @@ struct ib_umem *ib_umem_get_attr(struct ib_device *device,
 				 u16 attr_id, size_t size, int access)
 {
 	return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size,
-				      access);
+				      access, ib_access_dma_dir(access));
 }
 EXPORT_SYMBOL(ib_umem_get_attr);
 
@@ -503,7 +527,8 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
 				       int access)
 {
 	return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr,
-					    size, access);
+					    size, access,
+					    ib_access_dma_dir(access));
 }
 EXPORT_SYMBOL(ib_umem_get_attr_or_va);
 
@@ -569,6 +594,9 @@ static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *att
  * must arrange its own backing (typically an in-kernel allocation)
  * when no source is available.
  *
+ * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it
+ * and the CPU only reads.
+ *
  * Return: caller-owned umem on success; NULL when no source supplied
  * a buffer; ERR_PTR(...) on error.
  */
@@ -579,7 +607,7 @@ struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device,
 	return ib_umem_get_from_attrs(device, attrs,
 				      UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
 				      uverbs_create_cq_get_buffer_desc,
-				      size, access);
+				      size, access, DMA_FROM_DEVICE);
 }
 EXPORT_SYMBOL(ib_umem_get_cq_buf);
 
@@ -595,6 +623,9 @@ EXPORT_SYMBOL(ib_umem_get_cq_buf);
  * Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the
  * UMEM attribute nor the legacy CQ buffer attributes are supplied.
  *
+ * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it
+ * and the CPU only reads.
+ *
  * See ib_umem_get_attr_or_va() for the note on @size's dual role and
  * the migration path for drivers that would distinguish a user-supplied
  * length from a driver-computed minimum.
@@ -608,7 +639,7 @@ struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device,
 	return ib_umem_get_from_attrs_or_va(device, attrs,
 					    UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
 					    uverbs_create_cq_get_buffer_desc,
-					    addr, size, access);
+					    addr, size, access, DMA_FROM_DEVICE);
 }
 EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va);
 
diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c
index ce0a7de0040567..db65032e80b4a4 100644
--- a/drivers/infiniband/core/uverbs_std_types_device.c
+++ b/drivers/infiniband/core/uverbs_std_types_device.c
@@ -272,6 +272,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)(
 		return ret;
 	ret = ib_init_ucontext(attrs);
 	if (ret) {
+		rdma_restrack_put(&attrs->context->res);
 		kfree(attrs->context);
 		attrs->context = NULL;
 		return ret;
diff --git a/drivers/infiniband/core/uverbs_std_types_wq.c b/drivers/infiniband/core/uverbs_std_types_wq.c
index 7ded8339346f31..c23f455231bf0f 100644
--- a/drivers/infiniband/core/uverbs_std_types_wq.c
+++ b/drivers/infiniband/core/uverbs_std_types_wq.c
@@ -140,7 +140,7 @@ DECLARE_UVERBS_NAMED_METHOD(
 	UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_CQ_HANDLE,
 			UVERBS_OBJECT_CQ,
 			UVERBS_ACCESS_READ,
-			UA_OPTIONAL),
+			UA_MANDATORY),
 	UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_WQ_EVENT_FD,
 		       UVERBS_OBJECT_ASYNC_EVENT,
 		       UVERBS_ACCESS_READ,
diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c
index d150099b99d238..b67db3790f03ca 100644
--- a/drivers/infiniband/core/uverbs_uapi.c
+++ b/drivers/infiniband/core/uverbs_uapi.c
@@ -729,7 +729,7 @@ void uverbs_disassociate_api(struct uverbs_api *uapi)
 			 *
 			 * release_cleanup is the exception because
 			 * uverbs_uobject_fd_release() needs it. In this case
-			 * the module reference held by the fops will guarentee
+			 * the module reference held by the fops will guarantee
 			 * the type_class remains valid too.
 			 */
 			if (type_attrs &&
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index c43e25d3726673..6d5825114c1973 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -2362,7 +2362,7 @@ EXPORT_SYMBOL(ib_dereg_mr_user);
  * @max_num_sg:    maximum sg entries available for registration.
  *
  * Notes:
- * Memory registeration page/sg lists must not exceed max_num_sg.
+ * Memory registration page/sg lists must not exceed max_num_sg.
  * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
  * max_num_sg * used_page_size.
  *
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index a43e678151d3bf..da471c63a3ba11 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -216,6 +216,10 @@ struct bnxt_re_dev {
 	unsigned long			event_bitmap;
 	struct bnxt_qplib_cc_param	cc_param;
 	struct workqueue_struct		*dcb_wq;
+	/* Protects dcb_wq against bnxt_re_uninit_dcb_wq() destroying it
+	 * concurrently with bnxt_re_async_notifier() queuing work on it.
+	 */
+	spinlock_t			dcb_lock;
 	struct dentry                   *cc_config;
 	struct bnxt_re_dbg_cc_config_params *cc_config_params;
 	struct dentry			*cq_coal_cfg;
diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c
index 651cf9d0e0c7c9..575b97030587e6 100644
--- a/drivers/infiniband/hw/bnxt_re/hw_counters.c
+++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c
@@ -413,7 +413,8 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
 	}
 
 done:
-	return bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
+	return bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+					 rdev->is_virtfn) ?
 		BNXT_RE_NUM_EXT_COUNTERS : BNXT_RE_NUM_STD_COUNTERS;
 }
 
@@ -423,7 +424,8 @@ struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev,
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 	int num_counters = 0;
 
-	if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
+	if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+				     rdev->is_virtfn))
 		num_counters = BNXT_RE_NUM_EXT_COUNTERS;
 	else
 		num_counters = BNXT_RE_NUM_STD_COUNTERS;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index ccd2702db78b55..ac15372376a236 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -165,10 +165,11 @@ static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,
 		qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
 }
 
-static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
+static u32 bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 			     struct bnxt_qplib_sge *sg_list, int num)
 {
-	int i, total = 0;
+	u32 total = 0;
+	int i;
 
 	for (i = 0; i < num; i++) {
 		sg_list[i].addr = ib_sg_list[i].addr;
@@ -2235,6 +2236,14 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
 		goto exit;
 	}
 
+	if (srq_init_attr->attr.max_sge > dev_attr->max_srq_sges) {
+		ibdev_err(&rdev->ibdev,
+			  "Create SRQ failed - max_sge %d exceeds supported %d",
+			  srq_init_attr->attr.max_sge, dev_attr->max_srq_sges);
+		rc = -EINVAL;
+		goto exit;
+	}
+
 	if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
 		rc = -EOPNOTSUPP;
 		goto exit;
@@ -2388,6 +2397,7 @@ int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
 	spin_lock_irqsave(&srq->lock, flags);
 	while (wr) {
 		/* Transcribe each ib_recv_wr to qplib_swqe */
+		wqe.flags = 0;
 		wqe.num_sge = wr->num_sge;
 		bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
 		wqe.wr_id = wr->wr_id;
@@ -2903,7 +2913,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
 	qp->send_psn &= BTH_PSN_MASK;
 	qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
 	/* DETH */
-	/* Use the priviledged Q_Key for QP1 */
+	/* Use the privileged Q_Key for QP1 */
 	qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
 	qp->qp1_hdr.deth.source_qpn = IB_QP1;
 
@@ -3168,8 +3178,9 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
 				wr->sg_list[i].addr;
 		sge_len = wr->sg_list[i].length;
 
-		if ((sge_len + wqe->inline_len) >
-		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
+		if (sge_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH ||
+		    ((sge_len + wqe->inline_len) >
+		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH)) {
 			ibdev_err(&rdev->ibdev,
 				  "Inline data size requested > supported value");
 			return -EINVAL;
@@ -3185,17 +3196,22 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
 
 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
 				   const struct ib_send_wr *wr,
-				   struct bnxt_qplib_swqe *wqe)
+				   struct bnxt_qplib_swqe *wqe,
+				   u32 *payload_sz)
 {
-	int payload_sz = 0;
+	int rc;
 
-	if (wr->send_flags & IB_SEND_INLINE)
-		payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
-	else
-		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
-					       wqe->num_sge);
+	if (wr->send_flags & IB_SEND_INLINE) {
+		rc = bnxt_re_copy_inline_data(rdev, wr, wqe);
+		if (rc < 0)
+			return rc;
+		*payload_sz = rc;
+	} else {
+		*payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
+						wqe->num_sge);
+	}
 
-	return payload_sz;
+	return 0;
 }
 
 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
@@ -3218,7 +3234,8 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
 				       struct bnxt_re_qp *qp,
 				       const struct ib_send_wr *wr)
 {
-	int rc = 0, payload_sz = 0;
+	int rc = 0;
+	u32 payload_sz = 0;
 	unsigned long flags;
 
 	spin_lock_irqsave(&qp->sq_lock, flags);
@@ -3234,11 +3251,9 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
 			goto bad;
 		}
 
-		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
-		if (payload_sz < 0) {
-			rc = -EINVAL;
+		rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz);
+		if (rc)
 			goto bad;
-		}
 		wqe.wr_id = wr->wr_id;
 
 		wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
@@ -3279,7 +3294,8 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
 {
 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 	struct bnxt_qplib_swqe wqe;
-	int rc = 0, payload_sz = 0;
+	int rc = 0;
+	u32 payload_sz = 0;
 	unsigned long flags;
 
 	spin_lock_irqsave(&qp->sq_lock, flags);
@@ -3296,11 +3312,9 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
 			goto bad;
 		}
 
-		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
-		if (payload_sz < 0) {
-			rc = -EINVAL;
+		rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz);
+		if (rc)
 			goto bad;
-		}
 		wqe.wr_id = wr->wr_id;
 
 		switch (wr->opcode) {
@@ -3749,12 +3763,13 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe,
 	if (rc)
 		goto fail;
 
-	cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va,
-					 entries * sizeof(struct cq_base),
-					 IB_ACCESS_LOCAL_WRITE);
+	cq->resize_umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, NULL,
+						   req.cq_va,
+						   entries * sizeof(struct cq_base),
+						   IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(cq->resize_umem)) {
 		rc = PTR_ERR(cq->resize_umem);
-		ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n",
+		ibdev_err(&rdev->ibdev, "%s: ib_umem_get_cq_buf_or_va failed! rc = %pe\n",
 			  __func__, cq->resize_umem);
 		cq->resize_umem = NULL;
 		goto fail;
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 17654a9e23fede..fa0e1323a32662 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -367,9 +367,15 @@ static int bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
 
 static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)
 {
-	if (!rdev->dcb_wq)
-		return;
-	destroy_workqueue(rdev->dcb_wq);
+	struct workqueue_struct *dcb_wq;
+
+	spin_lock_bh(&rdev->dcb_lock);
+	dcb_wq = rdev->dcb_wq;
+	rdev->dcb_wq = NULL;
+	spin_unlock_bh(&rdev->dcb_lock);
+
+	if (dcb_wq)
+		destroy_workqueue(dcb_wq);
 }
 
 static void bnxt_re_dcb_wq_task(struct work_struct *work)
@@ -424,14 +430,23 @@ static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *c
 
 	switch (event_id) {
 	case ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE:
+		spin_lock(&rdev->dcb_lock);
+		if (!rdev->dcb_wq) {
+			spin_unlock(&rdev->dcb_lock);
+			break;
+		}
+
 		dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC);
-		if (!dcb_work)
+		if (!dcb_work) {
+			spin_unlock(&rdev->dcb_lock);
 			break;
+		}
 
 		dcb_work->rdev = rdev;
 		memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl));
 		INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task);
 		queue_work(rdev->dcb_wq, &dcb_work->work);
+		spin_unlock(&rdev->dcb_lock);
 		break;
 	default:
 		break;
@@ -1448,6 +1463,7 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct auxiliary_device *adev,
 	INIT_LIST_HEAD(&rdev->qp_list);
 	mutex_init(&rdev->qp_lock);
 	mutex_init(&rdev->pacing.dbq_lock);
+	spin_lock_init(&rdev->dcb_lock);
 	atomic_set(&rdev->stats.res.qp_count, 0);
 	atomic_set(&rdev->stats.res.cq_count, 0);
 	atomic_set(&rdev->stats.res.srq_count, 0);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 756f8b5f042a94..7ff587ce912690 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -45,6 +45,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/if_vlan.h>
 #include <linux/vmalloc.h>
+#include <linux/bitops.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/iter.h>
 
@@ -668,9 +669,9 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
 {
 	u32 bytes;
 
-	bytes = max >> 3;
+	bytes = BITS_TO_LONGS(max) * sizeof(unsigned long);
 	if (!bytes)
-		bytes = 1;
+		bytes = sizeof(unsigned long);
 	pdt->tbl = kmalloc(bytes, GFP_KERNEL);
 	if (!pdt->tbl)
 		return -ENOMEM;
@@ -848,9 +849,9 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
 	if (!dpit->app_tbl)
 		return -ENOMEM;
 
-	bytes = dpit->max >> 3;
+	bytes = BITS_TO_LONGS(dpit->max) * sizeof(unsigned long);
 	if (!bytes)
-		bytes = 1;
+		bytes = sizeof(unsigned long);
 
 	dpit->tbl = kmalloc(bytes, GFP_KERNEL);
 	if (!dpit->tbl) {
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index ec9eb52a8ebf75..9aaa2b5204b8e6 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -160,7 +160,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
 
 	attr->max_srq = le16_to_cpu(sb->max_srq);
 	attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
-	attr->max_srq_sges = sb->max_srq_sge;
+	attr->max_srq_sges = min_t(u32, sb->max_srq_sge, BNXT_STATIC_MAX_SGE);
 	attr->max_pkey = 1;
 	attr->max_inline_data = attr->max_qp_sges * sizeof(struct sq_sge);
 	if (!bnxt_qplib_is_chip_gen_p7(rcfw->res->cctx))
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index b3b45c49077d5f..a6f520f49d5928 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -4006,8 +4006,10 @@ static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb,
 	int ret;
 
 	req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL);
-	if (!req_skb)
+	if (!req_skb) {
+		kfree_skb(skb);
 		return;
+	}
 	req = __skb_put_zero(req_skb, sizeof(*req));
 	req->op_compl = htonl(WR_OP_V(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL_F);
 	req->len16_pkd = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)));
@@ -4156,6 +4158,7 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)
 		pdev = ip_dev_find(&init_net, iph->daddr);
 		if (!pdev) {
 			pr_err("%s - failed to find device!\n", __func__);
+			neigh_release(neigh);
 			goto free_dst;
 		}
 		e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh,
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index d1517f2560b981..2a7be0a1f168ce 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -1182,7 +1182,7 @@ void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx)
 	struct c4iw_cq *rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
 	unsigned long flag;
 
-	/* locking heirarchy: cq lock first, then qp lock. */
+	/* locking hierarchy: cq lock first, then qp lock. */
 	spin_lock_irqsave(&rchp->lock, flag);
 	spin_lock(&qhp->lock);
 
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index ab830764e3b4bd..30a38e633bfddf 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -358,7 +358,8 @@ struct efa_admin_reg_mr_cmd {
 	 * 5:0 : phys_page_size_shift - page size is (1 <<
 	 *    phys_page_size_shift). Page size is used for
 	 *    building the Virtual to Physical address mapping
-	 * 6 : reserved - MBZ
+	 * 6 : relaxed_ordering_enable - Enable PCIe relaxed
+	 *    ordering for this memory region
 	 * 7 : mem_addr_phy_mode_en - Enable bit for physical
 	 *    memory registration (no translation), can be used
 	 *    only by privileged clients. If set, PBL must
@@ -1203,6 +1204,7 @@ struct efa_admin_modify_event_counter_resp {
 
 /* reg_mr_cmd */
 #define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK      GENMASK(5, 0)
+#define EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE_MASK   BIT(6)
 #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK      BIT(7)
 #define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK        BIT(0)
 #define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK       BIT(1)
@@ -1231,7 +1233,6 @@ struct efa_admin_modify_event_counter_resp {
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK  BIT(3)
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4)
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_EVENT_COUNTERS_MASK BIT(5)
-#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_SHIFT 10
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_MASK BIT(10)
 
 /* feature_queue_attr_desc_2 */
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index 1b00f16b8ea808..07bfcb093df854 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -236,8 +236,15 @@ int efa_com_register_mr(struct efa_com_dev *edev,
 	mr_cmd.mr_length = params->mr_length_in_bytes;
 	EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT,
 		params->page_shift);
+	if (params->relaxed_ordering)
+		EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE, 1);
 	mr_cmd.iova = params->iova;
-	mr_cmd.permissions = params->permissions;
+	if (params->permissions.local_write)
+		EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1);
+	if (params->permissions.remote_write)
+		EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE, 1);
+	if (params->permissions.remote_read)
+		EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE, 1);
 
 	if (params->inline_pbl) {
 		memcpy(mr_cmd.pbl.inline_pbl_array,
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index bca722f021eaf9..524407f36d4386 100644
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -204,8 +204,12 @@ struct efa_com_reg_mr_params {
 	 * address mapping
 	 */
 	u8 page_shift;
-	/* see permissions field of struct efa_admin_reg_mr_cmd */
-	u8 permissions;
+	struct {
+		u8 local_write : 1;
+		u8 remote_write : 1;
+		u8 remote_read : 1;
+	} permissions;
+	u8 relaxed_ordering;
 	u8 inline_pbl;
 	u8 indirect;
 };
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 2d28d68efe7765..9e4435401a4c71 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -1676,7 +1676,10 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,
 	params.pd = to_epd(ibpd)->pdn;
 	params.iova = virt_addr;
 	params.mr_length_in_bytes = length;
-	params.permissions = access_flags;
+	params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE);
+	params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE);
+	params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ);
+	params.relaxed_ordering = !!(access_flags & IB_ACCESS_RELAXED_ORDERING);
 
 	pg_sz = ib_umem_find_best_pgsz(mr->umem,
 				       dev->dev_attr.page_size_cap,
diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c
index 1f456327e63cb2..daec74f0aa6df7 100644
--- a/drivers/infiniband/hw/erdma/erdma_cq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cq.c
@@ -8,8 +8,8 @@
 
 static void *get_next_valid_cqe(struct erdma_cq *cq)
 {
-	__be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
-				      cq->depth, CQE_SHIFT);
+	__be32 *cqe = erdma_kmem_get_entry(
+		&cq->kern_cq.qbuf_mem, cq->kern_cq.ci, cq->depth, CQE_SHIFT);
 	u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
 			      be32_to_cpu(READ_ONCE(*cqe)));
 
@@ -161,8 +161,8 @@ static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc)
 	if (qtype == ERDMA_CQE_QTYPE_SQ) {
 		id_table = kern_qp->swr_tbl;
 		depth = qp->attrs.sq_size;
-		wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
-					  qp->attrs.sq_size, SQEBB_SHIFT);
+		wqe_hdr = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+					       qp->attrs.sq_size, SQEBB_SHIFT);
 		kern_qp->sq_ci =
 			FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) +
 			wqe_idx + 1;
@@ -242,15 +242,15 @@ void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn)
 
 	while (ncqe > 0) {
 		cur_cq_ci = prev_cq_ci + ncqe - 1;
-		cqe = get_queue_entry(cq->kern_cq.qbuf, cur_cq_ci, cq->depth,
-				      CQE_SHIFT);
+		cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, cur_cq_ci,
+					   cq->depth, CQE_SHIFT);
 
 		if (be32_to_cpu(cqe->qpn) == qpn) {
 			++nqp_cqe;
 		} else if (nqp_cqe) {
-			dst_cqe = get_queue_entry(cq->kern_cq.qbuf,
-						  cur_cq_ci + nqp_cqe,
-						  cq->depth, CQE_SHIFT);
+			dst_cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem,
+						       cur_cq_ci + nqp_cqe,
+						       cq->depth, CQE_SHIFT);
 			owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
 					  be32_to_cpu(dst_cqe->hdr));
 			cqe->hdr = cpu_to_be32(
diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index 445182c6bc5d37..62dc22388eeace 100644
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -50,7 +50,7 @@ static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
 	struct net_device *netdev;
 	int ret = -EPROBE_DEFER;
 
-	/* Already binded to a net_device, so we skip. */
+	/* Already bound to a net_device, so we skip. */
 	if (dev->netdev)
 		return 0;
 
diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c
index e002343832f74d..fc335b674746b1 100644
--- a/drivers/infiniband/hw/erdma/erdma_qp.c
+++ b/drivers/infiniband/hw/erdma/erdma_qp.c
@@ -259,8 +259,8 @@ static void erdma_reset_qp(struct erdma_qp *qp)
 	qp->kern_qp.rq_ci = 0;
 	memset(qp->kern_qp.swr_tbl, 0, qp->attrs.sq_size * sizeof(u64));
 	memset(qp->kern_qp.rwr_tbl, 0, qp->attrs.rq_size * sizeof(u64));
-	memset(qp->kern_qp.sq_buf, 0, qp->attrs.sq_size << SQEBB_SHIFT);
-	memset(qp->kern_qp.rq_buf, 0, qp->attrs.rq_size << RQE_SHIFT);
+	erdma_kmem_clear(&qp->kern_qp.sq_mem);
+	erdma_kmem_clear(&qp->kern_qp.rq_mem);
 	erdma_remove_cqes_of_qp(&qp->scq->ibcq, QP_ID(qp));
 	if (qp->rcq != qp->scq)
 		erdma_remove_cqes_of_qp(&qp->rcq->ibcq, QP_ID(qp));
@@ -332,8 +332,8 @@ static int fill_inline_data(struct erdma_qp *qp,
 
 	wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 	sgl_offset &= (SQEBB_SIZE - 1);
-	data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size,
-			       SQEBB_SHIFT);
+	data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+				    qp->attrs.sq_size, SQEBB_SHIFT);
 
 	while (i < send_wr->num_sge) {
 		bytes += send_wr->sg_list[i].length;
@@ -356,8 +356,9 @@ static int fill_inline_data(struct erdma_qp *qp,
 			wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 			sgl_offset &= (SQEBB_SIZE - 1);
 
-			data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
-					       qp->attrs.sq_size, SQEBB_SHIFT);
+			data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
+						    wqe_idx, qp->attrs.sq_size,
+						    SQEBB_SHIFT);
 			if (!remain_size)
 				break;
 		}
@@ -385,8 +386,8 @@ static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,
 	while (i < send_wr->num_sge) {
 		wqe_idx += (sgl_offset >> SQEBB_SHIFT);
 		sgl_offset &= (SQEBB_SIZE - 1);
-		sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
-				      qp->attrs.sq_size, SQEBB_SHIFT);
+		sgl = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
+					   qp->attrs.sq_size, SQEBB_SHIFT);
 
 		bytes += send_wr->sg_list[i].length;
 		memcpy(sgl + sgl_offset, &send_wr->sg_list[i],
@@ -463,8 +464,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 	    send_wr->opcode != IB_WR_SEND_WITH_IMM)
 		return -EINVAL;
 
-	entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size,
-				SQEBB_SHIFT);
+	entry = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx,
+				     qp->attrs.sq_size, SQEBB_SHIFT);
 
 	/* Clear the SQE header section. */
 	*entry = 0;
@@ -524,8 +525,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 		read_sqe->sink_to_h =
 			cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr));
 
-		sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
-				      qp->attrs.sq_size, SQEBB_SHIFT);
+		sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
+					   qp->attrs.sq_size, SQEBB_SHIFT);
 		sge->addr = cpu_to_le64(rdma_wr->remote_addr);
 		sge->key = cpu_to_le32(rdma_wr->rkey);
 		sge->length = cpu_to_le32(send_wr->sg_list[0].length);
@@ -569,8 +570,9 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 		if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) {
 			attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
 			/* Copy SGLs to SQE content to accelerate */
-			memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
-					       qp->attrs.sq_size, SQEBB_SHIFT),
+			memcpy(erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
+						    idx + 1, qp->attrs.sq_size,
+						    SQEBB_SHIFT),
 			       mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents));
 			wqe_size = sizeof(struct erdma_reg_mr_sqe) +
 				   MTT_SIZE(mr->mem.mtt_nents);
@@ -605,8 +607,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
 				cpu_to_le64(atomic_wr(send_wr)->compare_add);
 		}
 
-		sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
-				      qp->attrs.sq_size, SQEBB_SHIFT);
+		sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
+					   qp->attrs.sq_size, SQEBB_SHIFT);
 		sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
 		sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
 		sge++;
@@ -702,8 +704,8 @@ static int erdma_post_recv_one(struct erdma_qp *qp,
 			       const struct ib_recv_wr *recv_wr)
 {
 	struct erdma_rqe *rqe =
-		get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi,
-				qp->attrs.rq_size, RQE_SHIFT);
+		erdma_kmem_get_entry(&qp->kern_qp.rq_mem, qp->kern_qp.rq_pi,
+				     qp->attrs.rq_size, RQE_SHIFT);
 
 	rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1);
 	rqe->qpn = cpu_to_le32(QP_ID(qp));
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index f18b88bba2817e..dd29d55d412254 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -41,7 +41,7 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
 	struct erdma_dev *dev = to_edev(qp->ibqp.device);
 	struct erdma_pd *pd = to_epd(qp->ibqp.pd);
 	struct erdma_cmdq_create_qp_req req;
-	struct erdma_uqp *user_qp;
+	struct erdma_mem *sq_mem, *rq_mem;
 	u64 resp0, resp1;
 	int err;
 
@@ -63,67 +63,46 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
 				      ERDMA_QPT_UD);
 
 	if (rdma_is_kernel_res(&qp->ibqp.res)) {
-		u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
-
-		req.sq_cqn_mtt_cfg =
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-				   pgsz_range) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
-		req.rq_cqn_mtt_cfg =
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-				   pgsz_range) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
-
-		req.sq_mtt_cfg =
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
-				   ERDMA_MR_MTT_0LEVEL);
-		req.rq_mtt_cfg = req.sq_mtt_cfg;
-
-		req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
-		req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
+		sq_mem = &qp->kern_qp.sq_mem;
+		rq_mem = &qp->kern_qp.rq_mem;
 		req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
 		req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
 	} else {
-		user_qp = &qp->user_qp;
-		req.sq_cqn_mtt_cfg = FIELD_PREP(
-			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-			ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
+		sq_mem = &qp->user_qp.sq_mem;
+		rq_mem = &qp->user_qp.rq_mem;
+		req.sq_dbrec_dma = qp->user_qp.sq_dbrec_dma;
+		req.rq_dbrec_dma = qp->user_qp.rq_dbrec_dma;
+	}
+
+	req.sq_cqn_mtt_cfg =
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
+			   ilog2(sq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+	req.sq_cqn_mtt_cfg |=
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
+	req.rq_cqn_mtt_cfg =
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
+			   ilog2(rq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+	req.rq_cqn_mtt_cfg |=
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
+
+	req.sq_mtt_cfg =
+		sq_mem->page_offset |
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, sq_mem->mtt_nents);
+	req.rq_mtt_cfg =
+		rq_mem->page_offset |
+		FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, rq_mem->mtt_nents);
+
+	assemble_qbuf_mtt_for_cmd(sq_mem, &req.sq_mtt_cfg, &req.sq_buf_addr,
+				  req.sq_mtt_entry);
+	assemble_qbuf_mtt_for_cmd(rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr,
+				  req.rq_mtt_entry);
+	if (uctx && uctx->ext_db.enable) {
 		req.sq_cqn_mtt_cfg |=
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
-
-		req.rq_cqn_mtt_cfg = FIELD_PREP(
-			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
-			ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
-		req.rq_cqn_mtt_cfg |=
-			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
-
-		req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
-		req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
-					     user_qp->sq_mem.mtt_nents);
-
-		req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
-		req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
-					     user_qp->rq_mem.mtt_nents);
-
-		assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
-					  &req.sq_buf_addr, req.sq_mtt_entry);
-		assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
-					  &req.rq_buf_addr, req.rq_mtt_entry);
-
-		req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
-		req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
-
-		if (uctx->ext_db.enable) {
-			req.sq_cqn_mtt_cfg |=
-				FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
-			req.db_cfg =
-				FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
-					   uctx->ext_db.sdb_off) |
-				FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
-					   uctx->ext_db.rdb_off);
-		}
+			FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
+		req.db_cfg = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
+					uctx->ext_db.sdb_off) |
+			     FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
+					uctx->ext_db.rdb_off);
 	}
 
 	err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
@@ -194,7 +173,6 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
 	struct erdma_cmdq_create_cq_req req;
 	struct erdma_mem *mem;
-	u32 page_size;
 
 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
 				CMDQ_OPCODE_CREATE_CQ);
@@ -204,48 +182,34 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
 	req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
 
 	if (rdma_is_kernel_res(&cq->ibcq.res)) {
-		page_size = SZ_32M;
-		req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
-				       ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
-		req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
-		req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
-
-		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
-			    FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
-				       ERDMA_MR_MTT_0LEVEL);
-
-		req.first_page_offset = 0;
+		mem = &cq->kern_cq.qbuf_mem;
 		req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
 	} else {
 		mem = &cq->user_cq.qbuf_mem;
-		req.cfg0 |=
-			FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
-				   ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
-		if (mem->mtt_nents == 1) {
-			req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
-			req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
-			req.cfg1 |=
-				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
-					   ERDMA_MR_MTT_0LEVEL);
-		} else {
-			req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
-			req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
-			req.cfg1 |=
-				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
-					   ERDMA_MR_MTT_1LEVEL);
-		}
-		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
-				       mem->mtt_nents);
-
-		req.first_page_offset = mem->page_offset;
 		req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
+	}
 
-		if (uctx->ext_db.enable) {
-			req.cfg1 |= FIELD_PREP(
-				ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
-			req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
-					      uctx->ext_db.cdb_off);
-		}
+	req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
+			       ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
+	if (mem->mtt_nents == 1) {
+		req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
+		req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
+				       ERDMA_MR_MTT_0LEVEL);
+	} else {
+		req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
+		req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
+				       ERDMA_MR_MTT_1LEVEL);
+	}
+	req.cfg1 |=
+		FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, mem->mtt_nents);
+	req.first_page_offset = mem->page_offset;
+
+	if (uctx && uctx->ext_db.enable) {
+		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
+		req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
+				      uctx->ext_db.cdb_off);
 	}
 
 	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
@@ -505,96 +469,28 @@ static int erdma_qp_validate_attr(struct erdma_dev *dev,
 	return 0;
 }
 
-static void free_kernel_qp(struct erdma_qp *qp)
-{
-	struct erdma_dev *dev = qp->dev;
-
-	vfree(qp->kern_qp.swr_tbl);
-	vfree(qp->kern_qp.rwr_tbl);
-
-	if (qp->kern_qp.sq_buf)
-		dma_free_coherent(&dev->pdev->dev,
-				  qp->attrs.sq_size << SQEBB_SHIFT,
-				  qp->kern_qp.sq_buf,
-				  qp->kern_qp.sq_buf_dma_addr);
-
-	if (qp->kern_qp.sq_dbrec)
-		dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
-			      qp->kern_qp.sq_dbrec_dma);
-
-	if (qp->kern_qp.rq_buf)
-		dma_free_coherent(&dev->pdev->dev,
-				  qp->attrs.rq_size << RQE_SHIFT,
-				  qp->kern_qp.rq_buf,
-				  qp->kern_qp.rq_buf_dma_addr);
-
-	if (qp->kern_qp.rq_dbrec)
-		dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
-			      qp->kern_qp.rq_dbrec_dma);
-}
-
-static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
-			  struct ib_qp_init_attr *attrs)
+static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt)
 {
-	struct erdma_kqp *kqp = &qp->kern_qp;
-	int size;
-
-	if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
-		kqp->sig_all = 1;
-
-	kqp->sq_pi = 0;
-	kqp->sq_ci = 0;
-	kqp->rq_pi = 0;
-	kqp->rq_ci = 0;
-	kqp->hw_sq_db =
-		dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
-	kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
-
-	kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
-	kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
-	if (!kqp->swr_tbl || !kqp->rwr_tbl)
-		goto err_out;
-
-	size = qp->attrs.sq_size << SQEBB_SHIFT;
-	kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
-					 &kqp->sq_buf_dma_addr, GFP_KERNEL);
-	if (!kqp->sq_buf)
-		goto err_out;
-
-	kqp->sq_dbrec =
-		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
-	if (!kqp->sq_dbrec)
-		goto err_out;
-
-	size = qp->attrs.rq_size << RQE_SHIFT;
-	kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
-					 &kqp->rq_buf_dma_addr, GFP_KERNEL);
-	if (!kqp->rq_buf)
-		goto err_out;
-
-	kqp->rq_dbrec =
-		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
-	if (!kqp->rq_dbrec)
-		goto err_out;
-
-	return 0;
+	struct ib_block_iter biter;
+	u32 idx = 0;
 
-err_out:
-	free_kernel_qp(qp);
-	return -ENOMEM;
+	if (mem->type == ERDMA_UMEM) {
+		rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
+			mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
+	} else {
+		for (; idx < mem->page_cnt; idx++)
+			mtt->buf[idx] = mem->kmem.buf_list[idx].dma_addr;
+	}
 }
 
-static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_init_bottom_mtt(struct erdma_mem *mem)
 {
 	struct erdma_mtt *mtt = mem->mtt;
-	struct ib_block_iter biter;
-	u32 idx = 0;
 
 	while (mtt->low_level)
 		mtt = mtt->low_level;
 
-	rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
-		mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
+	erdma_init_mtt_leaf(mem, mtt);
 }
 
 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
@@ -826,54 +722,216 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
 	}
 }
 
-static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
-			   u64 start, u64 len, int access, u64 virt,
-			   unsigned long req_page_size, bool force_continuous)
+static void erdma_kmem_free(struct erdma_dev *dev, struct erdma_mem *mem)
 {
-	int ret = 0;
+	struct erdma_buf_list *buf_list = mem->kmem.buf_list;
+	u32 i;
 
-	mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
-	if (IS_ERR(mem->umem)) {
-		ret = PTR_ERR(mem->umem);
-		mem->umem = NULL;
-		return ret;
+	for (i = 0; i < mem->page_cnt; i++)
+		dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf,
+				  buf_list[i].dma_addr);
+
+	kfree(buf_list);
+}
+
+static int erdma_kmem_alloc(struct erdma_dev *dev, struct erdma_mem *mem,
+			    u32 page_cnt)
+{
+	struct erdma_buf_list *buf_list;
+	u32 i;
+
+	buf_list = kcalloc(page_cnt, sizeof(*buf_list), GFP_KERNEL);
+	if (!buf_list)
+		return -ENOMEM;
+
+	for (i = 0; i < page_cnt; i++) {
+		buf_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
+						     &buf_list[i].dma_addr,
+						     GFP_KERNEL);
+		if (!buf_list[i].buf)
+			goto err_free_pages;
+	}
+
+	mem->kmem.buf_list = buf_list;
+	return 0;
+
+err_free_pages:
+	while (i--)
+		dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf,
+				  buf_list[i].dma_addr);
+	kfree(buf_list);
+
+	return -ENOMEM;
+}
+
+static void erdma_mem_free(struct erdma_dev *dev, struct erdma_mem *mem)
+{
+	switch (mem->type) {
+	case ERDMA_UMEM:
+		ib_umem_release(mem->umem);
+		break;
+	case ERDMA_KMEM:
+		erdma_kmem_free(dev, mem);
+		break;
+	default:
+		break;
+	}
+}
+
+static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
+			  struct erdma_mem_init_attr *attr)
+{
+	struct erdma_mtt *mtt;
+	int ret;
+
+	mem->type = attr->type;
+
+	switch (mem->type) {
+	case ERDMA_UMEM:
+		if (attr->flags & ERDMA_MEM_FLAG_CQ_BUF)
+			mem->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
+							     attr->start,
+							     attr->len,
+							     attr->access);
+		else
+			mem->umem = ib_umem_get_va(&dev->ibdev, attr->start,
+						   attr->len, attr->access);
+		if (IS_ERR(mem->umem)) {
+			ret = PTR_ERR(mem->umem);
+			return ret;
+		}
+
+		mem->page_size = ib_umem_find_best_pgsz(
+			mem->umem, attr->req_page_size, attr->virt);
+		mem->mtt_nents =
+			ib_umem_num_dma_blocks(mem->umem, mem->page_size);
+		break;
+	case ERDMA_KMEM:
+		mem->page_size = PAGE_SIZE;
+		mem->mtt_nents = DIV_ROUND_UP(attr->len, PAGE_SIZE);
+		ret = erdma_kmem_alloc(dev, mem, mem->mtt_nents);
+		if (ret)
+			return ret;
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	mem->va = virt;
-	mem->len = len;
-	mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
-	mem->page_offset = start & (mem->page_size - 1);
-	mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
+	mem->va = attr->virt;
+	mem->len = attr->len;
+	mem->page_offset = attr->start & (mem->page_size - 1);
 	mem->page_cnt = mem->mtt_nents;
-	mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
-				    force_continuous);
-	if (IS_ERR(mem->mtt)) {
-		ret = PTR_ERR(mem->mtt);
-		goto error_ret;
+	mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
+			       !(attr->flags & ERDMA_MEM_FLAG_MR_BUF));
+	if (IS_ERR(mtt)) {
+		ret = PTR_ERR(mtt);
+		goto err_free_mem;
 	}
+	mem->mtt = mtt;
 
-	erdma_fill_bottom_mtt(dev, mem);
+	erdma_init_bottom_mtt(mem);
+	if (mtt->continuous)
+		dma_sync_single_for_device(&dev->pdev->dev, mtt->buf_dma,
+					   mtt->size, DMA_TO_DEVICE);
 
 	return 0;
 
-error_ret:
-	if (mem->umem) {
-		ib_umem_release(mem->umem);
-		mem->umem = NULL;
-	}
+err_free_mem:
+	erdma_mem_free(dev, mem);
 
 	return ret;
 }
 
-static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem)
 {
-	if (mem->mtt)
-		erdma_destroy_mtt(dev, mem->mtt);
+	erdma_destroy_mtt(dev, mem->mtt);
+	erdma_mem_free(dev, mem);
+}
 
-	if (mem->umem) {
-		ib_umem_release(mem->umem);
-		mem->umem = NULL;
+static void free_kernel_qp(struct erdma_qp *qp)
+{
+	struct erdma_dev *dev = qp->dev;
+
+	vfree(qp->kern_qp.swr_tbl);
+	vfree(qp->kern_qp.rwr_tbl);
+
+	erdma_mem_uninit(dev, &qp->kern_qp.sq_mem);
+	dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
+		      qp->kern_qp.sq_dbrec_dma);
+	erdma_mem_uninit(dev, &qp->kern_qp.rq_mem);
+	dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
+		      qp->kern_qp.rq_dbrec_dma);
+}
+
+static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
+			  struct ib_qp_init_attr *attrs)
+{
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_KMEM,
+	};
+	struct erdma_kqp *kqp = &qp->kern_qp;
+	int ret = -ENOMEM;
+	int size;
+
+	if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
+		kqp->sig_all = 1;
+
+	kqp->sq_pi = 0;
+	kqp->sq_ci = 0;
+	kqp->rq_pi = 0;
+	kqp->rq_ci = 0;
+	kqp->hw_sq_db =
+		dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
+	kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
+
+	kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
+	if (!kqp->swr_tbl)
+		return -ENOMEM;
+
+	kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
+	if (!kqp->rwr_tbl)
+		goto err_free_swr_tbl;
+
+	size = qp->attrs.sq_size << SQEBB_SHIFT;
+	attr.len = size;
+	ret = erdma_mem_init(dev, &kqp->sq_mem, &attr);
+	if (ret)
+		goto err_free_rwr_tbl;
+
+	kqp->sq_dbrec =
+		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
+	if (!kqp->sq_dbrec) {
+		ret = -ENOMEM;
+		goto err_free_sq_mem;
 	}
+
+	size = qp->attrs.rq_size << RQE_SHIFT;
+	attr.len = size;
+	ret = erdma_mem_init(dev, &kqp->rq_mem, &attr);
+	if (ret)
+		goto err_free_sq_dbrec;
+
+	kqp->rq_dbrec =
+		dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
+	if (!kqp->rq_dbrec) {
+		ret = -ENOMEM;
+		goto err_free_rq_mem;
+	}
+
+	return 0;
+
+err_free_rq_mem:
+	erdma_mem_uninit(dev, &kqp->rq_mem);
+err_free_sq_dbrec:
+	dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
+err_free_sq_mem:
+	erdma_mem_uninit(dev, &kqp->sq_mem);
+err_free_rwr_tbl:
+	vfree(kqp->rwr_tbl);
+err_free_swr_tbl:
+	vfree(kqp->swr_tbl);
+
+	return ret;
 }
 
 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
@@ -941,6 +999,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 			u64 va, u32 len, u64 dbrec_va)
 {
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_UMEM,
+	};
 	dma_addr_t dbrec_dma;
 	u32 rq_offset;
 	int ret;
@@ -949,45 +1010,47 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 		   qp->attrs.rq_size * RQE_SIZE))
 		return -EINVAL;
 
-	ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
-			      qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
-			      (SZ_1M - SZ_4K), true);
+	attr.start = va;
+	attr.virt = va;
+	attr.len = qp->attrs.sq_size << SQEBB_SHIFT;
+	attr.req_page_size = SZ_1M - SZ_4K;
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, &attr);
 	if (ret)
 		return ret;
 
 	rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
 	qp->user_qp.rq_offset = rq_offset;
 
-	ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
-			      qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
-			      (SZ_1M - SZ_4K), true);
+	attr.start = va + rq_offset;
+	attr.virt = va + rq_offset;
+	attr.len = qp->attrs.rq_size << RQE_SHIFT;
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, &attr);
 	if (ret)
-		goto put_sq_mtt;
+		goto uninit_sq_mem;
 
 	ret = erdma_map_user_dbrecords(uctx, dbrec_va,
-				       &qp->user_qp.user_dbr_page,
-				       &dbrec_dma);
+				       &qp->user_qp.user_dbr_page, &dbrec_dma);
 	if (ret)
-		goto put_rq_mtt;
+		goto uninit_rq_mem;
 
 	qp->user_qp.sq_dbrec_dma = dbrec_dma;
 	qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
 
 	return 0;
 
-put_rq_mtt:
-	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+uninit_rq_mem:
+	erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
 
-put_sq_mtt:
-	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
+uninit_sq_mem:
+	erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
 
 	return ret;
 }
 
 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
 {
-	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
-	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+	erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
+	erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
 	erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
 }
 
@@ -1234,8 +1297,11 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 				u64 virt, int access, struct ib_dmah *dmah,
 				struct ib_udata *udata)
 {
-	struct erdma_mr *mr = NULL;
 	struct erdma_dev *dev = to_edev(ibpd->device);
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_UMEM,
+	};
+	struct erdma_mr *mr = NULL;
 	u32 stag;
 	int ret;
 
@@ -1249,8 +1315,13 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
-			      SZ_2G - SZ_4K, false);
+	attr.start = start;
+	attr.virt = virt;
+	attr.len = len;
+	attr.req_page_size = SZ_2G - SZ_4K;
+	attr.access = access;
+	attr.flags = ERDMA_MEM_FLAG_MR_BUF;
+	ret = erdma_mem_init(dev, &mr->mem, &attr);
 	if (ret)
 		goto err_out_free;
 
@@ -1260,8 +1331,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 
 	mr->ibmr.lkey = mr->ibmr.rkey = stag;
 	mr->ibmr.pd = ibpd;
-	mr->mem.va = virt;
-	mr->mem.len = len;
 	mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
 	mr->valid = 1;
 	mr->type = ERDMA_MR_TYPE_NORMAL;
@@ -1277,7 +1346,7 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 		       mr->ibmr.lkey >> 8);
 
 err_out_put_mtt:
-	put_mtt_entries(dev, &mr->mem);
+	erdma_mem_uninit(dev, &mr->mem);
 
 err_out_free:
 	kfree(mr);
@@ -1314,7 +1383,8 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 
 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
 
-	put_mtt_entries(dev, &mr->mem);
+	if (mr->type != ERDMA_MR_TYPE_DMA)
+		erdma_mem_uninit(dev, &mr->mem);
 
 	kfree(mr);
 	return 0;
@@ -1349,13 +1419,12 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	wait_for_completion(&cq->free);
 
 	if (rdma_is_kernel_res(&cq->ibcq.res)) {
-		dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
-				  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+		erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
 		dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
 			      cq->kern_cq.dbrec_dma);
 	} else {
 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 	}
 
 	return 0;
@@ -1407,8 +1476,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 	if (rdma_is_kernel_res(&qp->ibqp.res)) {
 		free_kernel_qp(qp);
 	} else {
-		put_mtt_entries(dev, &qp->user_qp.sq_mem);
-		put_mtt_entries(dev, &qp->user_qp.rq_mem);
+		erdma_mem_uninit(dev, &qp->user_qp.sq_mem);
+		erdma_mem_uninit(dev, &qp->user_qp.rq_mem);
 		erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
 	}
 
@@ -1926,12 +1995,19 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 			      struct erdma_ureq_create_cq *ureq)
 {
-	int ret;
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_UMEM,
+	};
+	int ret;
 
-	ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
-			      ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
-			      true);
+	attr.start = ureq->qbuf_va;
+	attr.virt = ureq->qbuf_va;
+	attr.len = ureq->qbuf_len;
+	attr.req_page_size = SZ_64M - SZ_4K;
+	attr.access = IB_ACCESS_LOCAL_WRITE;
+	attr.flags = ERDMA_MEM_FLAG_CQ_BUF;
+	ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr);
 	if (ret)
 		return ret;
 
@@ -1939,7 +2015,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 				       &cq->user_cq.user_dbr_page,
 				       &cq->user_cq.dbrec_dma);
 	if (ret)
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 
 	return ret;
 }
@@ -1947,12 +2023,15 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 static int erdma_init_kernel_cq(struct erdma_cq *cq)
 {
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
+	struct erdma_mem_init_attr attr = {
+		.type = ERDMA_KMEM,
+		.len = cq->depth << CQE_SHIFT,
+	};
+	int ret;
 
-	cq->kern_cq.qbuf =
-		dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
-				   &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
-	if (!cq->kern_cq.qbuf)
-		return -ENOMEM;
+	ret = erdma_mem_init(dev, &cq->kern_cq.qbuf_mem, &attr);
+	if (ret)
+		return ret;
 
 	cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
 					    &cq->kern_cq.dbrec_dma);
@@ -1966,8 +2045,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq)
 	return 0;
 
 err_out:
-	dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
-			  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+	erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
 
 	return -ENOMEM;
 }
@@ -2032,10 +2110,9 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 err_free_res:
 	if (!rdma_is_kernel_res(&ibcq->res)) {
 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 	} else {
-		dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
-				  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
+		erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
 		dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
 			      cq->kern_cq.dbrec_dma);
 	}
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index c73cecf92f611a..0542e10a0c99bd 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -111,19 +111,71 @@ struct erdma_mtt {
 	struct erdma_mtt *low_level;
 };
 
+enum erdma_mem_type {
+	ERDMA_NO_MEM = 0,
+	ERDMA_UMEM = 1,
+	ERDMA_KMEM = 2,
+};
+
+struct erdma_buf_list {
+	void *buf;
+	dma_addr_t dma_addr;
+};
+
+struct erdma_kmem {
+	struct erdma_buf_list *buf_list;
+};
+
+enum erdma_mem_flags {
+	ERDMA_MEM_FLAG_MR_BUF = (1 << 0),
+	ERDMA_MEM_FLAG_CQ_BUF = (1 << 1),
+};
+
+struct erdma_mem_init_attr {
+	enum erdma_mem_type type;
+	u64 start;
+	u64 virt;
+	u64 len;
+	unsigned long req_page_size;
+	int access;
+	u32 flags;
+};
+
 struct erdma_mem {
-	struct ib_umem *umem;
-	struct erdma_mtt *mtt;
+	enum erdma_mem_type type;
+	union {
+		struct ib_umem *umem;
+		struct erdma_kmem kmem;
+	};
 
 	u32 page_size;
 	u32 page_offset;
 	u32 page_cnt;
 	u32 mtt_nents;
 
+	struct erdma_mtt *mtt;
+
 	u64 va;
 	u64 len;
 };
 
+static inline void *erdma_kmem_get_entry(struct erdma_mem *mem, u32 idx,
+					 u32 depth, u32 shift)
+{
+	u32 offset = (idx & (depth - 1)) << shift;
+
+	return (u8 *)mem->kmem.buf_list[offset >> PAGE_SHIFT].buf +
+	       offset_in_page(offset);
+}
+
+static inline void erdma_kmem_clear(struct erdma_mem *mem)
+{
+	u32 i;
+
+	for (i = 0; i < mem->page_cnt; i++)
+		memset(mem->kmem.buf_list[i].buf, 0, PAGE_SIZE);
+}
+
 struct erdma_mr {
 	struct ib_mr ibmr;
 	struct erdma_mem mem;
@@ -183,11 +235,8 @@ struct erdma_kqp {
 	void __iomem *hw_sq_db;
 	void __iomem *hw_rq_db;
 
-	void *sq_buf;
-	dma_addr_t sq_buf_dma_addr;
-
-	void *rq_buf;
-	dma_addr_t rq_buf_dma_addr;
+	struct erdma_mem sq_mem;
+	struct erdma_mem rq_mem;
 
 	void *sq_dbrec;
 	void *rq_dbrec;
@@ -320,8 +369,7 @@ struct erdma_qp {
 };
 
 struct erdma_kcq_info {
-	void *qbuf;
-	dma_addr_t qbuf_dma_addr;
+	struct erdma_mem qbuf_mem;
 	u32 ci;
 	u32 cmdsn;
 	u32 notify_cnt;
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c
index 2b20907c9c1248..d4bf75dcd1025d 100644
--- a/drivers/infiniband/hw/hfi1/affinity.c
+++ b/drivers/infiniband/hw/hfi1/affinity.c
@@ -386,7 +386,7 @@ static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
 
 /*
  * This function creates the table for looking up CPUs for completion vectors.
- * num_comp_vectors needs to have been initilized before calling this function.
+ * num_comp_vectors needs to have been initialized before calling this function.
  */
 static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
 					  struct hfi1_affinity_node *entry)
diff --git a/drivers/infiniband/hw/hfi1/fault.c b/drivers/infiniband/hw/hfi1/fault.c
index 4ab72ef03ba11b..941a0b96590b60 100644
--- a/drivers/infiniband/hw/hfi1/fault.c
+++ b/drivers/infiniband/hw/hfi1/fault.c
@@ -216,7 +216,6 @@ int hfi1_fault_init_debugfs(struct hfi1_ibdev *ibd)
 	ibd->fault->attr.interval = 1;
 	ibd->fault->attr.require_end = ULONG_MAX;
 	ibd->fault->attr.stacktrace_depth = 32;
-	ibd->fault->attr.dname = NULL;
 	ibd->fault->attr.verbose = 0;
 	ibd->fault->enable = false;
 	ibd->fault->opcode = false;
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index 1a36f995c4f6db..0fc2ba5f1f9113 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -70,7 +70,6 @@ static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg);
 static int ctxt_reset(struct hfi1_ctxtdata *uctxt);
 static int manage_rcvq(struct hfi1_ctxtdata *uctxt, u16 subctxt,
 		       unsigned long arg);
-static vm_fault_t vma_fault(struct vm_fault *vmf);
 static long hfi1_file_ioctl(struct file *fp, unsigned int cmd,
 			    unsigned long arg);
 
@@ -85,10 +84,6 @@ static const struct file_operations hfi1_file_ops = {
 	.llseek = noop_llseek,
 };
 
-static const struct vm_operations_struct vm_ops = {
-	.fault = vma_fault,
-};
-
 /*
  * Types of memories mapped into user processes' space
  */
@@ -304,13 +299,13 @@ static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 	return reqs;
 }
 
-static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 vmf,
+static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 is_vmalloc,
 			     u64 memaddr, void *memvirt, dma_addr_t memdma,
 			     ssize_t memlen, struct vm_area_struct *vma)
 {
 	hfi1_cdbg(PROC,
-		  "%u:%u type:%u io/vf/dma:%d/%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx",
-		  ctxt, subctxt, type, mapio, vmf, !!memdma,
+		  "%u:%u type:%u io/vmalloc/dma:%d/%d/%d, addr:0x%llx, len:%lu(%lu), flags:0x%lx",
+		  ctxt, subctxt, type, mapio, is_vmalloc, !!memdma,
 		  memaddr ?: (u64)memvirt, memlen,
 		  vma->vm_end - vma->vm_start, vma->vm_flags);
 }
@@ -325,7 +320,8 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		memaddr = 0;
 	void *memvirt = NULL;
 	dma_addr_t memdma = 0;
-	u8 subctxt, mapio = 0, vmf = 0, type;
+	u8 subctxt, mapio = 0, type;
+	u8 is_vmalloc = 0;
 	size_t memdmalen = 0;
 	ssize_t memlen = 0;
 	int ret = 0;
@@ -348,7 +344,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 	/*
 	 * vm_pgoff is used as a buffer selector cookie.  Always mmap from
 	 * the beginning.
-	 */ 
+	 */
 	vma->vm_pgoff = 0;
 	flags = vma->vm_flags;
 
@@ -367,7 +363,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		 */
 		memlen = PAGE_ALIGN(uctxt->sc->credits * PIO_BLOCK_SIZE);
 		flags &= ~VM_MAYREAD;
-		flags |= VM_DONTCOPY | VM_DONTEXPAND;
+		flags |= VM_DONTCOPY;
 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 		mapio = 1;
 		break;
@@ -411,6 +407,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		memlen = rcvhdrq_size(uctxt);
 		memvirt = uctxt->rcvhdrq;
 		memdma = uctxt->rcvhdrq_dma;
+		flags |= VM_DONTEXPAND;
 		break;
 	case RCV_EGRBUF: {
 		unsigned long vm_start_save;
@@ -432,7 +429,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 			ret = -EPERM;
 			goto done;
 		}
-		vm_flags_clear(vma, VM_MAYWRITE);
+		vm_flags_mod(vma, VM_DONTEXPAND, VM_MAYWRITE);
 		/*
 		 * Mmap multiple separate allocations into a single vma.  From
 		 * here, dma_mmap_coherent() calls dma_direct_mmap(), which
@@ -448,7 +445,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 			memvirt = uctxt->egrbufs.buffers[i].addr;
 			memdma = uctxt->egrbufs.buffers[i].dma;
 			vma->vm_end += memlen;
-			mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr,
+			mmap_cdbg(ctxt, subctxt, type, mapio, is_vmalloc, memaddr,
 				  memvirt, memdma, memlen, vma);
 			ret = dma_mmap_coherent(&dd->pcidev->dev, vma,
 						memvirt, memdma, memlen);
@@ -477,7 +474,7 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		 * user registers.
 		 */
 		memlen = PAGE_SIZE;
-		flags |= VM_DONTCOPY | VM_DONTEXPAND;
+		flags |= VM_DONTCOPY;
 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 		mapio = 1;
 		break;
@@ -486,15 +483,10 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		 * Use the page where this context's flags are. User level
 		 * knows where it's own bitmap is within the page.
 		 */
-		memaddr = (unsigned long)
-			(dd->events + uctxt_offset(uctxt)) & PAGE_MASK;
+		memvirt = dd->events + uctxt_offset(uctxt);
+		memvirt = (void *)(((uintptr_t)memvirt) & PAGE_MASK);
 		memlen = PAGE_SIZE;
-		/*
-		 * v3.7 removes VM_RESERVED but the effect is kept by
-		 * using VM_IO.
-		 */
-		flags |= VM_IO | VM_DONTEXPAND;
-		vmf = 1;
+		is_vmalloc = 1;
 		break;
 	case STATUS:
 		if (flags & VM_WRITE) {
@@ -503,7 +495,6 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		}
 		memaddr = kvirt_to_phys((void *)dd->status);
 		memlen = PAGE_SIZE;
-		flags |= VM_IO | VM_DONTEXPAND;
 		break;
 	case RTAIL:
 		if (!HFI1_CAP_IS_USET(DMA_RTAIL)) {
@@ -522,25 +513,23 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 		memvirt = (void *)hfi1_rcvhdrtail_kvaddr(uctxt);
 		memdma = uctxt->rcvhdrqtailaddr_dma;
 		flags &= ~VM_MAYWRITE;
+		flags |= VM_DONTEXPAND;
 		break;
 	case SUBCTXT_UREGS:
-		memaddr = (u64)uctxt->subctxt_uregbase;
+		memvirt = uctxt->subctxt_uregbase;
 		memlen = PAGE_SIZE;
-		flags |= VM_IO | VM_DONTEXPAND;
-		vmf = 1;
+		is_vmalloc = 1;
 		break;
 	case SUBCTXT_RCV_HDRQ:
-		memaddr = (u64)uctxt->subctxt_rcvhdr_base;
+		memvirt = uctxt->subctxt_rcvhdr_base;
 		memlen = rcvhdrq_size(uctxt) * uctxt->subctxt_cnt;
-		flags |= VM_IO | VM_DONTEXPAND;
-		vmf = 1;
+		is_vmalloc = 1;
 		break;
 	case SUBCTXT_EGRBUF:
-		memaddr = (u64)uctxt->subctxt_rcvegrbuf;
+		memvirt = uctxt->subctxt_rcvegrbuf;
 		memlen = uctxt->egrbufs.size * uctxt->subctxt_cnt;
-		flags |= VM_IO | VM_DONTEXPAND;
 		flags &= ~VM_MAYWRITE;
-		vmf = 1;
+		is_vmalloc = 1;
 		break;
 	case SDMA_COMP: {
 		struct hfi1_user_sdma_comp_q *cq = fd->cq;
@@ -549,10 +538,9 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 			ret = -EFAULT;
 			goto done;
 		}
-		memaddr = (u64)cq->comps;
+		memvirt = cq->comps;
 		memlen = PAGE_ALIGN(sizeof(*cq->comps) * cq->nentries);
-		flags |= VM_IO | VM_DONTEXPAND;
-		vmf = 1;
+		is_vmalloc = 1;
 		break;
 	}
 	default:
@@ -569,12 +557,10 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 	}
 
 	vm_flags_reset(vma, flags);
-	mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr, memvirt, memdma, 
+	mmap_cdbg(ctxt, subctxt, type, mapio, is_vmalloc, memaddr, memvirt, memdma,
 		  memlen, vma);
-	if (vmf) {
-		vma->vm_pgoff = PFN_DOWN(memaddr);
-		vma->vm_ops = &vm_ops;
-		ret = 0;
+	if (is_vmalloc) {
+		ret = remap_vmalloc_range(vma, memvirt, 0);
 	} else if (memdma) {
 		ret = dma_mmap_coherent(&dd->pcidev->dev, vma,
 					memvirt, memdma,
@@ -599,24 +585,6 @@ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma)
 	return ret;
 }
 
-/*
- * Local (non-chip) user memory is not mapped right away but as it is
- * accessed by the user-level code.
- */
-static vm_fault_t vma_fault(struct vm_fault *vmf)
-{
-	struct page *page;
-
-	page = vmalloc_to_page((void *)(vmf->pgoff << PAGE_SHIFT));
-	if (!page)
-		return VM_FAULT_SIGBUS;
-
-	get_page(page);
-	vmf->page = page;
-
-	return 0;
-}
-
 static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt)
 {
 	struct hfi1_ctxtdata *uctxt;
diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c
index 3c228aeaaf81b0..a69de0a9123832 100644
--- a/drivers/infiniband/hw/hfi1/firmware.c
+++ b/drivers/infiniband/hw/hfi1/firmware.c
@@ -77,7 +77,7 @@ struct css_header {
 /* size of platform configuration partition */
 #define MAX_PLATFORM_CONFIG_FILE_SIZE 4096
 
-/* size of file of plaform configuration encoded in format version 4 */
+/* size of file of platform configuration encoded in format version 4 */
 #define PLATFORM_CONFIG_FORMAT_4_FILE_SIZE 528
 
 /* the file itself */
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index 1154b8cc713ce3..106bed4932c856 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -53,17 +53,9 @@ int hfi1_pcie_init(struct pci_dev *pdev)
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (ret) {
-		/*
-		 * If the 64 bit setup fails, try 32 bit.  Some systems
-		 * do not setup 64 bit maps on systems with 2GB or less
-		 * memory installed.
-		 */
-		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (ret) {
-			dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n",
-				ERR_PTR(ret));
-			goto bail;
-		}
+		dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n",
+			ERR_PTR(ret));
+		goto bail;
 	}
 
 	pci_set_master(pdev);
diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c
index f2b12ae13a5842..8d8272705fb72e 100644
--- a/drivers/infiniband/hw/hns/hns_roce_bond.c
+++ b/drivers/infiniband/hw/hns/hns_roce_bond.c
@@ -81,7 +81,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp,
 {
 	struct net_device *active_dev;
 	struct net_device *old_dev;
-	int i, ret = 0;
+	int i = 0, ret = 0;
 
 	if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
 		rcu_read_lock();
@@ -89,7 +89,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp,
 			bond_option_active_slave_get_rcu(netdev_priv(bond_grp->upper_dev));
 		rcu_read_unlock();
 	} else {
-		for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) {
+		for (; i < ROCE_BOND_FUNC_MAX; i++) {
 			active_dev = bond_grp->bond_func_info[i].net_dev;
 			if (active_dev &&
 			    ib_get_curr_port_state(active_dev) == IB_PORT_ACTIVE)
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 1dd0efb5620de1..7dfaa01cda670b 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -261,10 +261,11 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
 	buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size;
 	buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num;
 	buf_attr.region_count = 1;
+	buf_attr.user_access = IB_ACCESS_LOCAL_WRITE;
 
 	ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
 				  hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT,
-				  udata, addr);
+				  udata, addr, true);
 	if (ret)
 		ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret);
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c
index 103b8c9ca9695f..4ef0b882ea2f0a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c
+++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c
@@ -338,6 +338,60 @@ static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev,
 	}
 }
 
+static int gsi_sl_debugfs_show(struct seq_file *file, void *offset)
+{
+	struct hns_roce_dev *hr_dev = file->private;
+
+	seq_printf(file, "%u\n", hr_dev->gsi_sl);
+
+	return 0;
+}
+
+static ssize_t gsi_sl_debugfs_store(char *buf, size_t count, void *data)
+{
+	struct hns_roce_dev *hr_dev = data;
+	struct net_device *netdev;
+	int ret;
+	u8 val;
+
+	netdev = ib_device_get_netdev(&hr_dev->ib_dev, 1);
+	if (!netdev)
+		return -ENODEV;
+
+	if (ib_get_curr_port_state(netdev) == IB_PORT_ACTIVE) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+
+	ret = kstrtou8(buf, 0, &val);
+	if (ret)
+		goto out;
+
+	if (!check_sl_valid(hr_dev, val)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	hr_dev->gsi_sl = val;
+
+out:
+	dev_put(netdev);
+	return ret ? : count;
+}
+
+static void create_gsi_sl_debugfs(struct hns_roce_dev *hr_dev,
+				  struct dentry *parent)
+{
+	struct hns_debugfs_seqfile *seqfile = &hr_dev->dbgfs.gsi_sl;
+
+	seqfile->read = gsi_sl_debugfs_show;
+	seqfile->write = gsi_sl_debugfs_store;
+	seqfile->data = hr_dev;
+
+	debugfs_create_file("gsi_sl", 0600, parent, seqfile,
+			    &hns_debugfs_seqfile_fops);
+}
+
 /* debugfs for device */
 void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
 {
@@ -348,6 +402,7 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
 
 	create_sw_stat_debugfs(hr_dev, dbgfs->root);
 	create_cc_param_debugfs(hr_dev, dbgfs->root);
+	create_gsi_sl_debugfs(hr_dev, dbgfs->root);
 }
 
 void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h
index 116b1e8b66772a..286704711785c2 100644
--- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h
+++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h
@@ -47,6 +47,7 @@ struct hns_roce_dev_debugfs {
 	struct dentry *root;
 	struct hns_sw_stat_debugfs sw_stat_root;
 	struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM];
+	struct hns_debugfs_seqfile gsi_sl;
 };
 
 struct hns_roce_dev;
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index eb7b1865e4c798..5890ee69247c26 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -654,6 +654,7 @@ struct hns_roce_qp {
 	u8			priority;
 	spinlock_t flush_lock;
 	struct hns_roce_dip *dip;
+	bool ud_sl_set;
 };
 
 struct hns_roce_ib_iboe {
@@ -737,7 +738,7 @@ enum hns_roce_scc_algo {
 struct hns_roce_caps {
 	u64		fw_ver;
 	u8		num_ports;
-	int		gid_table_len[HNS_ROCE_MAX_PORTS];
+	u32		gid_table_len[HNS_ROCE_MAX_PORTS];
 	int		pkey_table_len[HNS_ROCE_MAX_PORTS];
 	int		local_ca_ack_delay;
 	int		num_uars;
@@ -1047,6 +1048,7 @@ struct hns_roce_dev {
 	struct hns_roce_dev_debugfs dbgfs;
 	atomic64_t *dfx_cnt;
 	struct hns_roce_scc_param *scc_param;
+	u8 gsi_sl;
 };
 
 enum hns_roce_trace_type {
@@ -1230,7 +1232,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 			struct hns_roce_buf_attr *buf_attr,
 			unsigned int page_shift, struct ib_udata *udata,
-			unsigned long user_addr);
+			unsigned long user_addr, bool is_cq);
 void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
 			  struct hns_roce_mtr *mtr);
 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 27cc7df55ee70d..a8f319a57f25ac 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -431,7 +431,8 @@ static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 	return 0;
 }
 
-static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
+static int fill_ud_av(struct hns_roce_qp *qp,
+		      struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 		      struct hns_roce_ah *ah)
 {
 	struct ib_device *ib_dev = ah->ibah.device;
@@ -441,7 +442,13 @@ static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit);
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass);
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel);
-	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl);
+	if (!qp->ud_sl_set || qp->ibqp.qp_type == IB_QPT_GSI) {
+		qp->sl = qp->ibqp.qp_type == IB_QPT_GSI ?
+				hr_dev->gsi_sl : ah->av.sl;
+		qp->ud_sl_set = true;
+	}
+
+	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, qp->sl);
 
 	ud_sq_wqe->sgid_index = ah->av.gid_index;
 
@@ -491,12 +498,10 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 			  qp->qkey : ud_wr(wr)->remote_qkey);
 	hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn);
 
-	ret = fill_ud_av(ud_sq_wqe, ah);
+	ret = fill_ud_av(qp, ud_sq_wqe, ah);
 	if (ret)
 		return ret;
 
-	qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
-
 	set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
 
 	/*
@@ -2416,8 +2421,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev)
 					 caps->gmv_bt_num *
 					 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz));
 
-		caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /
-							  caps->gmv_entry_sz);
+		caps->gmv_entry_num = caps->gid_table_len[0];
 	} else {
 		u32 func_num = max_t(u32, 1, hr_dev->func_num);
 
@@ -3824,7 +3828,8 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
 	 * Now backwards through the CQ, removing CQ entries
 	 * that match our QP by overwriting them with next entries.
 	 */
-	while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
+	while (prod_index != hr_cq->cons_index) {
+		--prod_index;
 		cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
 		if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {
 			if (srq && hr_reg_read(cqe, CQE_S_R)) {
@@ -4353,18 +4358,39 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
 static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
 			       dma_addr_t base_addr)
 {
+	u32 obj_num_per_bt = HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz;
+	u32 chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + PAGE_SHIFT);
+	u32 bt_num_per_chunk = chunk_size / HNS_HW_PAGE_SIZE;
+	u32 first = obj / obj_num_per_bt;
+	u32 last = min(first + bt_num_per_chunk, hr_dev->caps.gmv_bt_num);
 	struct hns_roce_cmq_desc desc;
-	struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
-	u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz);
-	u64 addr = to_hr_hw_page_addr(base_addr);
+	struct hns_roce_cmq_req *req;
+	u64 addr;
+	int ret;
+	u32 i;
 
-	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
+	/* The GMV BT entry of hardware covers a fixed 4K region, so a buffer
+	 * chunk larger than 4K must be registered to hardware with one BT
+	 * entry per 4K block, otherwise the GMV entries beyond the first
+	 * 4K of the chunk are unreachable.
+	 */
+	for (i = first; i < last; i++) {
+		hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT,
+					      false);
+		req = (struct hns_roce_cmq_req *)desc.data;
 
-	hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
-	hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
-	hr_reg_write(req, CFG_GMV_BT_IDX, idx);
+		addr = to_hr_hw_page_addr(base_addr +
+					  (u64)(i - first) * HNS_HW_PAGE_SIZE);
+		hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
+		hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
+		hr_reg_write(req, CFG_GMV_BT_IDX, i);
 
-	return hns_roce_cmq_send(hr_dev, &desc, 1);
+		ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
@@ -5327,7 +5353,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp,
 	hr_reg_clear(qpc_mask, QPC_VLAN_ID);
 
 	if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
-		ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
+		ibdev_err(ibdev, "sgid_index(%u) too large. max is %u\n",
 			  grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
 		return -EINVAL;
 	}
@@ -5603,6 +5629,7 @@ static void v2_set_flushed_fields(struct ib_qp *ibqp,
 	hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head);
 	hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);
 	hr_qp->state = IB_QPS_ERR;
+	hr_qp->ud_sl_set = false;
 	spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
 
 	if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */
@@ -6913,7 +6940,7 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 
 	err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
 				  hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,
-				  0);
+				  0, false);
 	if (err)
 		dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err);
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index 7d41ae897458a0..4799f667eedafb 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -112,7 +112,7 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
 
 	err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,
 				  hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT,
-				  udata, start);
+				  udata, start, false);
 	if (err) {
 		ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err);
 		return err;
@@ -586,7 +586,8 @@ static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
 
 static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 			  struct hns_roce_buf_attr *buf_attr,
-			  struct ib_udata *udata, unsigned long user_addr)
+			  struct ib_udata *udata, unsigned long user_addr,
+			  bool is_cq)
 {
 	struct ib_device *ibdev = &hr_dev->ib_dev;
 	size_t total_size;
@@ -595,8 +596,14 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 
 	if (udata) {
 		mtr->kmem = NULL;
-		mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
-					   buf_attr->user_access);
+		if (is_cq)
+			mtr->umem = ib_umem_get_cq_buf_or_va(ibdev, NULL,
+							     user_addr,
+							     total_size,
+							     buf_attr->user_access);
+		else
+			mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
+						   buf_attr->user_access);
 		if (IS_ERR(mtr->umem)) {
 			ibdev_err(ibdev, "failed to get umem, ret = %pe.\n",
 				  mtr->umem);
@@ -1035,11 +1042,13 @@ static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
  * @ba_page_shift: page shift for multi-hop base address table
  * @udata: user space context, if it's NULL, means kernel space
  * @user_addr: userspace virtual address to start at
+ * @is_cq: true when @mtr backs a CQ buffer, which the device writes
+ *         completion entries into
  */
 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 			struct hns_roce_buf_attr *buf_attr,
 			unsigned int ba_page_shift, struct ib_udata *udata,
-			unsigned long user_addr)
+			unsigned long user_addr, bool is_cq)
 {
 	struct ib_device *ibdev = &hr_dev->ib_dev;
 	int ret;
@@ -1052,7 +1061,8 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
 		mtr->umem = NULL;
 		mtr->kmem = NULL;
 	} else {
-		ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);
+		ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr,
+				     is_cq);
 		if (ret) {
 			ibdev_err(ibdev,
 				  "failed to alloc mtr bufs, ret = %d.\n", ret);
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index e333a8c4acb52c..c6c0dfdbdbc303 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -808,7 +808,7 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 	}
 	ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
 				  PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
-				  udata, addr);
+				  udata, addr, false);
 	if (ret) {
 		ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
 		goto err_inline;
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index 4a54394f96bef4..b5e9ac9cfd591c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -180,7 +180,7 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 
 	ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,
 				  hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT,
-				  udata, addr);
+				  udata, addr, false);
 	if (ret) {
 		ibdev_err(ibdev,
 			  "failed to alloc SRQ idx mtr, ret = %d.\n", ret);
@@ -235,7 +235,7 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev,
 
 	ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,
 				  hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT,
-				  udata, addr);
+				  udata, addr, false);
 	if (ret)
 		ibdev_err(ibdev,
 			  "failed to alloc SRQ buf mtr, ret = %d.\n", ret);
diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c
index a70ef59a8064e6..2b68f377e1fa3e 100644
--- a/drivers/infiniband/hw/ionic/ionic_controlpath.c
+++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c
@@ -110,8 +110,9 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
 		if (rc)
 			goto err_qdesc;
 
-		cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr,
-					  req_cq->size, IB_ACCESS_LOCAL_WRITE);
+		cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
+						    req_cq->addr, req_cq->size,
+						    IB_ACCESS_LOCAL_WRITE);
 		if (IS_ERR(cq->umem)) {
 			rc = PTR_ERR(cq->umem);
 			goto err_qdesc;
@@ -418,6 +419,8 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
 
 	resp.udma_count = dev->lif_cfg.udma_count;
 	resp.expdb_mask = dev->lif_cfg.expdb_mask;
+	resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit;
+	resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS;
 
 	if (dev->lif_cfg.sq_expdb)
 		resp.expdb_qtypes |= IONIC_EXPDB_SQ;
@@ -1388,7 +1391,8 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
 			       struct ionic_qp *qp,
 			       struct ionic_tbl_buf *sq_buf,
 			       struct ionic_tbl_buf *rq_buf,
-			       struct ib_qp_init_attr *attr)
+			       struct ib_qp_init_attr *attr,
+			       u32 ionic_flags)
 {
 	const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject);
 	const u32 flags = to_ionic_qp_flags(0, 0,
@@ -1404,7 +1408,12 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
 			.len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN),
 			.cmd.create_qp = {
 				.pd_id = cpu_to_le32(pd->pdid),
-				.priv_flags = cpu_to_be32(flags),
+				/* User-supplied ionic_flags are passed through to
+				 * firmware, which validates and rejects any
+				 * unsupported or unauthorized bits.
+				 */
+				.priv_flags = cpu_to_be32(flags |
+						(ionic_flags & IONIC_QP_USER_FLAGS_MASK)),
 				.type_state = to_ionic_qp_type(attr->qp_type),
 				.dbid_flags = cpu_to_le16(dbid),
 				.id_ver = cpu_to_le32(qp->qpid),
@@ -2221,6 +2230,12 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		rc = ib_copy_validate_udata_in(udata, req, rsvd);
 		if (rc)
 			return rc;
+
+		if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK)
+			return -EINVAL;
+
+		if (req.rsvd_pad)
+			return -EINVAL;
 	} else {
 		req.sq_spec = IONIC_SPEC_HIGH;
 		req.rq_spec = IONIC_SPEC_HIGH;
@@ -2301,7 +2316,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 	rc = ionic_create_qp_cmd(dev, pd,
 				 to_ionic_vcq_cq(attr->send_cq, qp->udma_idx),
 				 to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx),
-				 qp, &sq_buf, &rq_buf, attr);
+				 qp, &sq_buf, &rq_buf, attr, req.ionic_flags);
 	if (rc)
 		goto err_cmd;
 
@@ -2336,7 +2351,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 						    &resp.sq_cmb_offset);
 			if (!qp->mmap_sq_cmb) {
 				rc = -ENOMEM;
-				goto err_mmap_sq;
+				goto err_qp_cmd;
 			}
 
 			resp.sq_cmb = qp->sq_cmb;
@@ -2373,7 +2388,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 						    &resp.rq_cmb_offset);
 			if (!qp->mmap_rq_cmb) {
 				rc = -ENOMEM;
-				goto err_mmap_rq;
+				goto err_qp_cmd;
 			}
 
 			resp.rq_cmb = qp->rq_cmb;
@@ -2381,7 +2396,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 
 		rc = ib_respond_udata(udata, resp);
 		if (rc)
-			goto err_resp;
+			goto err_qp_cmd;
 	}
 
 	ionic_pgtbl_unbuf(dev, &rq_buf);
@@ -2399,7 +2414,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		else
 			rc = xa_err(entry);
 
-		goto err_resp;
+		goto err_qp_cmd;
 	}
 
 	if (qp->has_sq) {
@@ -2430,13 +2445,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 
 	return 0;
 
-err_resp:
-	if (udata && (qp->rq_cmb & IONIC_CMB_ENABLE))
-		rdma_user_mmap_entry_remove(qp->mmap_rq_cmb);
-err_mmap_rq:
-	if (udata && (qp->sq_cmb & IONIC_CMB_ENABLE))
-		rdma_user_mmap_entry_remove(qp->mmap_sq_cmb);
-err_mmap_sq:
+err_qp_cmd:
 	ionic_destroy_qp_cmd(dev, qp->qpid);
 err_cmd:
 	ionic_pgtbl_unbuf(dev, &rq_buf);
diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h
index ee23062a1762e4..d0a5bce85ed81c 100644
--- a/drivers/infiniband/hw/ionic/ionic_fw.h
+++ b/drivers/infiniband/hw/ionic/ionic_fw.h
@@ -105,6 +105,8 @@ enum ionic_qp_flags {
 	IONIC_QPF_SQ_CMB		= BIT(13),
 	IONIC_QPF_RQ_CMB		= BIT(14),
 	IONIC_QPF_PRIVILEGED		= BIT(15),
+
+	IONIC_QP_USER_FLAGS_MASK	= GENMASK(31, 16),
 };
 
 static inline int from_ionic_qp_flags(int flags)
diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c
index cba7809ec3d92b..fa79732c2f7d39 100644
--- a/drivers/infiniband/hw/ionic/ionic_ibdev.c
+++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c
@@ -320,8 +320,8 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev)
 
 	ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg);
 
-	xa_init_flags(&dev->qp_tbl, GFP_ATOMIC);
-	xa_init_flags(&dev->cq_tbl, GFP_ATOMIC);
+	xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ);
 
 	ionic_init_resids(dev);
 
diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
index 8e48c00d4959ae..1e37bd09490f50 100644
--- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
+++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c
@@ -86,6 +86,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg)
 	cfg->udma_count = 2;
 
 	cfg->max_stride = ident->rdma.max_stride;
+	cfg->rcq_sign_bit = ident->rdma.rcq_sign_bit;
 	cfg->expdb_mask = ionic_get_expdb(lif);
 
 	cfg->sq_expdb =
diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
index 2b29e646c193b8..d7835ac27896b8 100644
--- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
+++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h
@@ -57,6 +57,7 @@ struct ionic_lif_cfg {
 	bool sq_expdb;
 	bool rq_expdb;
 	u8 expdb_mask;
+	u8 rcq_sign_bit;
 };
 
 void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg);
diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
index 335ae3c82e1733..0db73fd7d01b97 100644
--- a/drivers/infiniband/hw/irdma/ctrl.c
+++ b/drivers/infiniband/hw/irdma/ctrl.c
@@ -1689,11 +1689,9 @@ static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev,
  * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
  * @qp: sc qp struct
  * @info: fast mr info
- * @post_sq: flag for cqp db to ring
  */
 int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
-			      struct irdma_fast_reg_stag_info *info,
-			      bool post_sq)
+			      struct irdma_fast_reg_stag_info *info)
 {
 	u64 temp, hdr;
 	__le64 *wqe;
@@ -1723,9 +1721,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
 		  info->wr_id, wqe_idx,
 		  &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
 
-	temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
-		(uintptr_t)info->va : info->fbo;
-	set_64bit_val(wqe, 0, temp);
+	set_64bit_val(wqe, 0, info->va);
 
 	temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,
 			 info->first_pm_pbl_index >> 16);
@@ -1742,7 +1738,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
 	      FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |
 	      FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |
 	      FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |
-	      FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |
+	      IRDMAQPSQ_VABASEDTO |
 	      FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
 	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
 	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
@@ -1755,8 +1751,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
 	print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8,
 			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);
 
-	if (post_sq)
-		irdma_uk_qp_post_wr(&qp->qp_uk);
+	irdma_uk_qp_post_wr(&qp->qp_uk);
 
 	return 0;
 }
@@ -2625,11 +2620,9 @@ static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
  * @qp: sc qp
  * @info: dlush information
  * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
  */
 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
-			   struct irdma_qp_flush_info *info, u64 scratch,
-			   bool post_sq)
+			   struct irdma_qp_flush_info *info, u64 scratch)
 {
 	u64 temp = 0;
 	__le64 *wqe;
@@ -2697,8 +2690,7 @@ int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
 
 	print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8,
 			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
-	if (post_sq)
-		irdma_sc_cqp_post_sq(cqp);
+	irdma_sc_cqp_post_sq(cqp);
 
 	return 0;
 }
@@ -6114,8 +6106,7 @@ static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,
 	case IRDMA_OP_QP_FLUSH_WQES:
 		status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,
 						&pcmdinfo->in.u.qp_flush_wqes.info,
-						pcmdinfo->in.u.qp_flush_wqes.scratch,
-						pcmdinfo->post_sq);
+						pcmdinfo->in.u.qp_flush_wqes.scratch);
 		break;
 	case IRDMA_OP_GEN_AE:
 		status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index c345cc6542566b..e28f5f4876cd4b 100644
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -1252,12 +1252,21 @@ static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
 	}
 
 	msix_vec->ceq_id = ceq_id;
-	if (rf->sc_dev.privileged)
+	if (rf->sc_dev.privileged) {
 		rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id,
 						  msix_vec->idx, true);
-	else
+	} else {
 		status = irdma_vchnl_req_ceq_vec_map(&rf->sc_dev, ceq_id,
 						     msix_vec->idx);
+		if (!status)
+			return 0;
+
+		if (rf->msix_shared && !ceq_id)
+			irdma_destroy_irq(rf, msix_vec, rf);
+		else
+			irdma_destroy_irq(rf, msix_vec, iwceq);
+	}
+
 	return status;
 }
 
@@ -2742,7 +2751,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
 	hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
 	memcpy(hw_info, info, sizeof(*hw_info));
 	cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
-	cqp_info->post_sq = 1;
 	cqp_info->in.u.qp_flush_wqes.qp = qp;
 	cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
 	status = irdma_handle_cqp_op(rf, cqp_request);
@@ -2789,7 +2797,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
 				hw_info = &new_req->info.in.u.qp_flush_wqes.info;
 				memcpy(hw_info, info, sizeof(*hw_info));
 				cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
-				cqp_info->post_sq = 1;
 				cqp_info->in.u.qp_flush_wqes.qp = qp;
 				cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)new_req;
 
diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c
index 4b451d8482a480..f405ef761f8434 100644
--- a/drivers/infiniband/hw/irdma/icrdma_if.c
+++ b/drivers/infiniband/hw/irdma/icrdma_if.c
@@ -318,14 +318,15 @@ static void icrdma_remove(struct auxiliary_device *aux_dev)
 		container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
 	struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info;
 	struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);
-	u8 rdma_ver = iwdev->rf->rdma_ver;
+	struct irdma_pci_f *rf = iwdev->rf;
+	u8 rdma_ver = rf->rdma_ver;
 
 	ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false);
 	irdma_ib_unregister_device(iwdev);
-	icrdma_deinit_interrupts(iwdev->rf, cdev_info);
-	mutex_destroy(&iwdev->rf->ah_tbl_lock);
+	icrdma_deinit_interrupts(rf, cdev_info);
+	mutex_destroy(&rf->ah_tbl_lock);
 
-	kfree(iwdev->rf);
+	kfree(rf);
 
 	pr_debug("INIT: Gen[%d] func[%d] device remove success\n",
 		 rdma_ver, PCI_FUNC(cdev_info->pdev->devfn));
diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c
index 95957d52883dd0..106915b51e7cb4 100644
--- a/drivers/infiniband/hw/irdma/main.c
+++ b/drivers/infiniband/hw/irdma/main.c
@@ -80,6 +80,8 @@ static int ig3rdma_vport_probe(struct auxiliary_device *aux_dev,
 		return -ENOMEM;
 	}
 	iwdev = ib_alloc_device(irdma_device, ibdev);
+	if (!iwdev)
+		return -ENOMEM;
 	/* Fill iwdev info */
 	iwdev->is_vport = true;
 	iwdev->rf = rf;
diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c
index 28dfad7f940c24..0c2c5e96a05a4e 100644
--- a/drivers/infiniband/hw/irdma/pble.c
+++ b/drivers/infiniband/hw/irdma/pble.c
@@ -330,7 +330,7 @@ static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
 	if (root->addr)
 		irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo);
 
-	kfree(lvl2->leafmem.va);
+	kvfree(lvl2->leafmem.va);
 	lvl2->leaf = NULL;
 }
 
@@ -358,7 +358,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 	lvl2->leaf_cnt = total;
 
 	lvl2->leafmem.size = (sizeof(*leaf) * total);
-	lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
+	lvl2->leafmem.va = kvzalloc(lvl2->leafmem.size, GFP_KERNEL);
 	if (!lvl2->leafmem.va)
 		return -ENOMEM;
 
@@ -367,7 +367,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 	ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo,
 				       total << 3, &root->addr, &fpm_addr);
 	if (ret_code) {
-		kfree(lvl2->leafmem.va);
+		kvfree(lvl2->leafmem.va);
 		lvl2->leaf = NULL;
 		return -ENOMEM;
 	}
@@ -482,7 +482,7 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 
 		status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
 		/* if level1_only, only go through it once */
-		if (!status || lvl)
+		if (!status || lvl == PBLE_LEVEL_1)
 			break;
 	}
 
diff --git a/drivers/infiniband/hw/irdma/protos.h b/drivers/infiniband/hw/irdma/protos.h
index 324cfbf21764ba..561a0287b94526 100644
--- a/drivers/infiniband/hw/irdma/protos.h
+++ b/drivers/infiniband/hw/irdma/protos.h
@@ -19,8 +19,7 @@ void irdma_sc_rt_init(struct irdma_sc_dev *dev);
 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
 __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);
 int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
-			      struct irdma_fast_reg_stag_info *info,
-			      bool post_sq);
+			      struct irdma_fast_reg_stag_info *info);
 /* HMC/FPM functions */
 int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id);
 /* stats misc */
diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h
index 5557d9338796f9..8b4d2c0b283f4b 100644
--- a/drivers/infiniband/hw/irdma/type.h
+++ b/drivers/infiniband/hw/irdma/type.h
@@ -1173,13 +1173,11 @@ struct irdma_reg_ns_stag_info {
 struct irdma_fast_reg_stag_info {
 	u64 wr_id;
 	u64 reg_addr_pa;
-	u64 fbo;
-	void *va;
+	u64 va;
 	u64 total_len;
 	u32 page_size;
 	u32 chunk_size;
 	u32 first_pm_pbl_index;
-	enum irdma_addressing_type addr_type;
 	irdma_stag_index stag_idx;
 	u16 access_rights;
 	u32 pd_id;
@@ -1383,8 +1381,7 @@ int irdma_sc_qp_create(struct irdma_sc_qp *qp,
 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
 			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
-			   struct irdma_qp_flush_info *info, u64 scratch,
-			   bool post_sq);
+			   struct irdma_qp_flush_info *info, u64 scratch);
 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
 int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
 		       struct irdma_modify_qp_info *info, u64 scratch,
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 5d71300da0ff22..22a8511fffea94 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -4276,16 +4276,15 @@ static int irdma_post_send(struct ib_qp *ibqp,
 			stag_info.stag_idx = reg_wr(ib_wr)->key >> 8;
 			stag_info.page_size = reg_wr(ib_wr)->mr->page_size;
 			stag_info.wr_id = ib_wr->wr_id;
-			stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED;
-			stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
+			stag_info.va = iwmr->ibmr.iova;
 			stag_info.total_len = iwmr->ibmr.length;
 			stag_info.reg_addr_pa = *palloc->level1.addr;
 			stag_info.first_pm_pbl_index = palloc->level1.idx;
 			stag_info.local_fence = true;
 			if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
 				stag_info.chunk_size = 1;
-			err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,
-							true);
+			err = irdma_sc_mr_fast_register(&iwqp->sc_qp,
+							&stag_info);
 			break;
 		}
 		default:
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index d4e5e3f912689a..843e2c64b32bc1 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -5,6 +5,58 @@
 
 #include "mana_ib.h"
 
+static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error)
+{
+	switch (vendor_error) {
+	case VENDOR_ERR_OK:
+		return IB_WC_SUCCESS;
+	case VENDOR_ERR_RX_PKT_LEN:
+	case VENDOR_ERR_RX_MSG_LEN_OVFL:
+		return IB_WC_LOC_LEN_ERR;
+	case VENDOR_ERR_TX_GDMA_CORRUPTED_WQE:
+	case VENDOR_ERR_TX_PCIE_WQE:
+	case VENDOR_ERR_TX_PCIE_MSG:
+	case VENDOR_ERR_RX_MALFORMED_WQE:
+	case VENDOR_ERR_TX_GDMA_INVALID_STATE:
+	case VENDOR_ERR_TX_MISBEHAVING_CLIENT:
+	case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE:
+	case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD:
+	case VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED:
+	case VENDOR_ERR_TX_RDMA_WQE_LEN_ERR:
+	case VENDOR_ERR_TX_RDMA_MTU_ERR:
+		return IB_WC_LOC_QP_OP_ERR;
+	case VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION:
+	case VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE:
+	case VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR:
+	case VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION:
+	case VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE:
+	case VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR:
+	case VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE:
+	case VENDOR_ERR_RX_ATB_SGE_MISSCONFIG:
+		return IB_WC_LOC_PROT_ERR;
+	case VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT:
+	case VENDOR_ERR_RX_GFID:
+		return IB_WC_LOC_ACCESS_ERR;
+	case VENDOR_ERR_RX_MISBEHAVING_CLIENT:
+	case VENDOR_ERR_RX_CLIENT_ID:
+	case VENDOR_ERR_RX_PCIE:
+	case VENDOR_ERR_RX_NO_AVAIL_WQE:
+	case VENDOR_ERR_RX_ATB_WQE_MISCONFIG:
+	case VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT:
+	case VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE:
+	case VENDOR_ERR_TX_RDMA_INVALID_STATE:
+	case VENDOR_ERR_TX_RDMA_INVALID_NPT:
+	case VENDOR_ERR_TX_RDMA_INVALID_SGID:
+	case VENDOR_ERR_TX_RDMA_VFID_MISMATCH:
+		return IB_WC_FATAL_ERR;
+	case VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE:
+	case VENDOR_ERR_SW_FLUSHED:
+		return IB_WC_WR_FLUSH_ERR;
+	default:
+		return IB_WC_GENERAL_ERR;
+	}
+}
+
 int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct uverbs_attr_bundle *attrs)
 {
@@ -40,7 +92,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 		cq->cqe = attr->cqe;
 		err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
-					   &cq->queue);
+					   &cq->queue, true);
 		if (err) {
 			ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
 			return err;
@@ -64,6 +116,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		doorbell = mdev->gdma_dev->doorbell;
 	}
 
+	ibcq->cqe = cq->cqe;
+	cq->poll_credit = (cq->cqe << (GDMA_CQE_OWNER_BITS - 1)) - 1;
+
 	if (is_rnic_cq) {
 		err = mana_ib_gd_create_cq(mdev, cq, doorbell);
 		if (err) {
@@ -86,8 +141,8 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	}
 
 	spin_lock_init(&cq->cq_lock);
-	INIT_LIST_HEAD(&cq->list_send_qp);
-	INIT_LIST_HEAD(&cq->list_recv_qp);
+	INIT_LIST_HEAD(&cq->send_err_qp_list);
+	INIT_LIST_HEAD(&cq->recv_err_qp_list);
 
 	return 0;
 
@@ -174,171 +229,290 @@ void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
 	gc->cq_table[cq->queue.id] = NULL;
 }
 
+static inline bool gdma_cq_idx_produced(struct gdma_queue *gdma_cq, uint32_t idx)
+{
+	struct gdma_mem_info *gmi = &gdma_cq->mem_info;
+	u32 num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE;
+	u32 expected_bits = (idx / num_cqe) & GDMA_CQE_OWNER_MASK;
+	u32 offset = (idx % num_cqe) * GDMA_CQE_SIZE;
+	struct gdma_cqe *cqe;
+
+	if (gmi->nr_pages)
+		cqe = gmi->pages_va[offset / PAGE_SIZE] +
+		      (offset & (PAGE_SIZE - 1));
+	else
+		cqe = gdma_cq->queue_mem_ptr + offset;
+
+	return cqe->cqe_info.owner_bits == expected_bits;
+}
+
+static inline void mana_ib_cq_doorbell(struct mana_ib_cq *cq, uint8_t arm)
+{
+	struct mana_ib_dev *mdev = container_of(cq->ibcq.device, struct mana_ib_dev, ib_dev);
+	struct gdma_queue *gdma_cq = cq->queue.kmem;
+	u32 num_cqe, max_credit, idx;
+
+	num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE;
+	max_credit = num_cqe << (GDMA_CQE_OWNER_BITS - 1);
+	idx = gdma_cq->head;
+
+	if (cq->poll_credit >= max_credit) {
+		if (gdma_cq_idx_produced(gdma_cq, idx + cq->poll_credit - max_credit))
+			cq->poll_credit++;
+		else
+			return;
+	} else {
+		/* Set index of already polled CQE for unarm */
+		cq->poll_credit = max_credit - (arm ? 0 : 1);
+	}
+
+	idx += (cq->poll_credit - max_credit);
+	idx %= (num_cqe << GDMA_CQE_OWNER_BITS);
+
+	mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), gdma_cq, idx, arm, 0);
+}
+
 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 {
 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 	struct gdma_queue *gdma_cq = cq->queue.kmem;
+	unsigned long irq_flags;
 
 	if (!gdma_cq)
 		return -EINVAL;
 
-	mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);
+	spin_lock_irqsave(&cq->cq_lock, irq_flags);
+	mana_ib_cq_doorbell(cq, SET_ARM_BIT);
+	spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
+
 	return 0;
 }
 
-static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
-{
-	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
-	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
-	struct ud_sq_shadow_wqe *shadow_wqe;
+struct mana_cq_poll {
+	struct ib_wc *wc;
+	int budget;
+	int produced;
+};
 
-	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
-	if (!shadow_wqe)
-		return;
+static struct ib_wc *mana_fill_wc(struct mana_ib_qp *qp,
+				  struct mana_cq_poll *poll,
+				  const struct shadow_wqe_header *wqe,
+				  enum ib_wc_opcode opcode, u32 vendor_error)
+{
+	struct ib_wc *wc = &poll->wc[poll->produced++];
 
-	shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
+	memset(wc, 0, sizeof(*wc));
+	wc->wr_id = wqe->wr_id;
+	wc->status = vendor_error_to_wc_error(vendor_error);
+	wc->opcode = opcode;
+	wc->vendor_err = vendor_error;
+	wc->qp = &qp->ibqp;
 
-	wq->tail += shadow_wqe->header.posted_wqe_size;
-	shadow_queue_advance_next_to_complete(&qp->shadow_sq);
+	return wc;
 }
 
-static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
+static void mana_complete_send(struct mana_ib_qp *qp,
+			       struct mana_cq_poll *poll, u32 vendor_error)
 {
-	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
-	struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
-	struct ud_rq_shadow_wqe *shadow_wqe;
+	struct shadow_queue *shadow = &qp->shadow_sq;
+	struct shadow_wqe_header *wqe = shadow_queue_get_next_to_consume(shadow);
+	struct gdma_queue *queue;
 
-	shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
-	if (!shadow_wqe)
+	if (!wqe)
 		return;
 
-	shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
-	shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
-	shadow_wqe->header.error_code = IB_WC_SUCCESS;
+	if (vendor_error || !(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
+		mana_fill_wc(qp, poll, wqe, wqe->send_opcode, vendor_error);
 
-	wq->tail += shadow_wqe->header.posted_wqe_size;
-	shadow_queue_advance_next_to_complete(&qp->shadow_rq);
+	queue = mana_qp_get_sq(qp)->kmem;
+	queue->tail += wqe->wqe_size_in_bu;
+	shadow_queue_advance_consumer(shadow);
 }
 
-static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
+static void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct mana_rdma_cqe *rdma_cqe,
+			     struct mana_cq_poll *poll)
 {
-	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
+	u32 offset = rdma_cqe->ud_send.tx_wqe_offset & MANA_WQE_OFFSET_MASK;
+	struct shadow_queue *shadow = &qp->shadow_sq;
+	struct shadow_wqe_header *wqe;
+	u64 idx = shadow->cons_idx;
+	u32 to_complete = 0;
+	u64 prod_idx;
+
+	/* Pair with posting's release of the initialized shadow entries. */
+	prod_idx = smp_load_acquire(&shadow->prod_idx);
+	/* Find the target before retiring any entries: the CQE may be stale. */
+	for (; idx != prod_idx; idx++) {
+		wqe = shadow_queue_get_element(shadow, idx);
+		to_complete++;
+		if (wqe->wqe_offset_or_psn == offset)
+			break;
+		if (!(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
+			return;
+	}
+	if (idx == prod_idx)
+		return;
 
-	if (!qp)
+	for (; to_complete; to_complete--)
+		mana_complete_send(qp, poll, VENDOR_ERR_OK);
+}
+
+static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe,
+			  struct mana_cq_poll *poll)
+{
+	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
+	u32 offset = rdma_cqe->ud_recv.rx_wqe_offset / GDMA_WQE_BU_SIZE;
+	struct mana_ib_queue *rq = mana_qp_get_rq(qp);
+	struct gdma_queue *wq = rq->kmem;
+	struct shadow_wqe_header *wqe;
+	struct ib_wc *wc;
+
+	wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq);
+	if (!wqe || wqe->wqe_offset_or_psn != (offset & MANA_WQE_OFFSET_MASK))
 		return;
 
-	if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) {
-		if (cqe->is_sq)
-			handle_ud_sq_cqe(qp, cqe);
-		else
-			handle_ud_rq_cqe(qp, cqe);
+	wc = mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_OK);
+	switch (rdma_cqe->cqe_type) {
+	case CQE_TYPE_UD_SEND_IMM:
+		wc->ex.imm_data = cpu_to_be32(rdma_cqe->ud_recv.imm_data);
+		wc->wc_flags |= IB_WC_WITH_IMM;
+		fallthrough;
+	case CQE_TYPE_UD_SEND:
+		wc->byte_len = rdma_cqe->ud_recv.msg_len;
+		wc->src_qp = rdma_cqe->ud_recv.src_qpn;
+		wc->wc_flags |= IB_WC_GRH;
+		break;
+	default:
+		break;
 	}
 
-	mana_put_qp_ref(qp);
+	wq->tail += wqe->wqe_size_in_bu;
+	shadow_queue_advance_consumer(&qp->shadow_rq);
 }
 
-static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
-				       const struct shadow_wqe_header *shadow_wqe)
+static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev,
+			    struct mana_cq_poll *poll)
 {
-	const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
+	struct gdma_comp *cqe = &cq->pending_cqe;
+	struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
+	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
 
-	wc->wr_id = shadow_wqe->wr_id;
-	wc->status = shadow_wqe->error_code;
-	wc->opcode = shadow_wqe->opcode;
-	wc->vendor_err = shadow_wqe->error_code;
-	wc->wc_flags = 0;
-	wc->qp = &qp->ibqp;
-	wc->pkey_index = 0;
+	if (!qp)
+		return true;
 
-	if (shadow_wqe->opcode == IB_WC_RECV) {
-		wc->byte_len = ud_wqe->byte_len;
-		wc->src_qp = ud_wqe->src_qpn;
-		wc->wc_flags |= IB_WC_GRH;
+	switch (rdma_cqe->cqe_type) {
+	case CQE_TYPE_UD_SEND:
+		if (cqe->is_sq) {
+			handle_ud_sq_cqe(qp, rdma_cqe, poll);
+			break;
+		}
+		fallthrough;
+	case CQE_TYPE_UD_SEND_IMM:
+		handle_rq_cqe(qp, cqe, poll);
+		break;
+	default:
+		ibdev_warn_ratelimited(qp->ibqp.device, "Unexpected CQE type %u\n",
+				       rdma_cqe->cqe_type);
+		break;
 	}
+	mana_put_qp_ref(qp);
+	return true;
 }
 
-static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)
+static void mana_flush_completions(struct mana_ib_cq *cq, struct mana_cq_poll *poll)
 {
-	struct shadow_wqe_header *shadow_wqe;
+	struct shadow_wqe_header *wqe;
 	struct mana_ib_qp *qp;
-	int wc_index = 0;
 
-	/* process send shadow queue completions  */
-	list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
-		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
-				!= NULL) {
-			if (wc_index >= nwc)
-				goto out;
+	if (poll->produced >= poll->budget)
+		return;
 
-			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
-			shadow_queue_advance_consumer(&qp->shadow_sq);
-			wc_index++;
-		}
+	list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) {
+		while (poll->produced < poll->budget &&
+		       shadow_queue_get_next_to_consume(&qp->shadow_sq))
+			mana_complete_send(qp, poll, VENDOR_ERR_SW_FLUSHED);
+		if (poll->produced == poll->budget)
+			return;
 	}
 
-	/* process recv shadow queue completions */
-	list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
-		while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
-				!= NULL) {
-			if (wc_index >= nwc)
-				goto out;
-
-			fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
+	list_for_each_entry(qp, &cq->recv_err_qp_list, recv_err_node) {
+		while (poll->produced < poll->budget &&
+		       (wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))) {
+			mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_SW_FLUSHED);
 			shadow_queue_advance_consumer(&qp->shadow_rq);
-			wc_index++;
 		}
+		if (poll->produced == poll->budget)
+			return;
 	}
-
-out:
-	return wc_index;
 }
 
-void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
+static void mana_drain_gsi_sq(struct mana_ib_qp *qp)
 {
-	struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false);
-	struct ud_sq_shadow_wqe *shadow_wqe;
-	struct mana_ib_cq *cq;
+	struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
 	unsigned long flags;
 
-	if (!qp)
-		return;
-
-	cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
-
 	spin_lock_irqsave(&cq->cq_lock, flags);
-	while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq))
-			!= NULL) {
-		shadow_wqe->header.error_code = IB_WC_GENERAL_ERR;
-		shadow_queue_advance_next_to_complete(&qp->shadow_sq);
-	}
+	if (list_empty(&qp->send_err_node))
+		list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
 	spin_unlock_irqrestore(&cq->cq_lock, flags);
 
 	if (cq->ibcq.comp_handler)
 		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+}
 
-	mana_put_qp_ref(qp);
+void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
+{
+	struct mana_ib_qp *qp;
+	u32 port;
+
+	/* One GSI QP per port, indexed in the QP table by (port << 24 | MANA_GSI_QPN) */
+	for (port = 1; port <= mdev->ib_dev.phys_port_cnt; port++) {
+		qp = mana_get_qp_ref(mdev, (port << 24) | MANA_GSI_QPN, false);
+		if (!qp)
+			continue;
+
+		mana_drain_gsi_sq(qp);
+		mana_put_qp_ref(qp);
+	}
 }
 
 int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 {
 	struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
 	struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev);
+	struct mana_cq_poll poll = { .wc = wc, .budget = num_entries, .produced = 0 };
 	struct gdma_queue *queue = cq->queue.kmem;
-	struct gdma_comp gdma_cqe;
 	unsigned long flags;
-	int num_polled = 0;
-	int comp_read, i;
+	bool consumed;
+	int comp_read;
+
+	if (!queue)
+		return -EINVAL;
 
 	spin_lock_irqsave(&cq->cq_lock, flags);
-	for (i = 0; i < num_entries; i++) {
-		comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1);
-		if (comp_read < 1)
-			break;
-		mana_handle_cqe(mdev, &gdma_cqe);
+	while (poll.produced < poll.budget) {
+		if (!cq->has_pending_cqe) {
+			comp_read = mana_gd_poll_cq(queue, &cq->pending_cqe, 1);
+			if (comp_read < 0) {
+				if (!poll.produced)
+					poll.produced = comp_read;
+				goto out;
+			}
+			if (!comp_read)
+				break;
+
+			cq->poll_credit--;
+			if (!cq->poll_credit)
+				mana_ib_cq_doorbell(cq, 0);
+		}
+
+		consumed = mana_handle_cqe(cq, mdev, &poll);
+		cq->has_pending_cqe = !consumed;
 	}
 
-	num_polled = mana_process_completions(cq, num_entries, wc);
+	mana_flush_completions(cq, &poll);
+out:
 	spin_unlock_irqrestore(&cq->cq_lock, flags);
 
-	return num_polled;
+	return poll.produced;
 }
diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index a8d19586a78a7f..62c7803ca6c60b 100644
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -103,8 +103,7 @@ static int mana_ib_netdev_event(struct notifier_block *this,
 				ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
 
 				/* mana_get_primary_netdev() returns ndev with refcount held */
-				if (ndev)
-					netdev_put(ndev, &dev->dev_tracker);
+				netdev_put(ndev, &dev->dev_tracker);
 
 				return NOTIFY_OK;
 			default:
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index e4414d208d4aca..858bbc325eb29f 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -247,6 +247,9 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
 
 	ucontext->doorbell = doorbell_page;
 	ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT;
+	ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT;
+	if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT)
+		ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT;
 	ret = ib_respond_udata(udata, ucmd_resp);
 	if (ret)
 		return ret;
@@ -292,7 +295,7 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_qu
 }
 
 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
-			 struct mana_ib_queue *queue)
+			 struct mana_ib_queue *queue, bool is_cq)
 {
 	struct ib_umem *umem;
 	int err;
@@ -301,7 +304,12 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
 	queue->id = INVALID_QUEUE_ID;
 	queue->gdma_region = GDMA_INVALID_DMA_REGION;
 
-	umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
+	if (is_cq)
+		umem = ib_umem_get_cq_buf_or_va(&mdev->ib_dev, NULL, addr,
+						size, IB_ACCESS_LOCAL_WRITE);
+	else
+		umem = ib_umem_get_va(&mdev->ib_dev, addr, size,
+				      IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(umem)) {
 		ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem);
 		return PTR_ERR(umem);
@@ -592,7 +600,11 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 
 	if (mana_ib_is_rnic(dev)) {
-		if (port_num == 1) {
+		bool port_supports_cm = (port_num == 1 ||
+			(dev->adapter_caps.feature_flags &
+			 MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT));
+
+		if (port_supports_cm) {
 			immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
 			immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 		} else {
@@ -670,10 +682,14 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
 	ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width);
 	props->pkey_tbl_len = 1;
 	if (mana_ib_is_rnic(dev)) {
+		bool port_supports_cm = (port == 1 ||
+			(dev->adapter_caps.feature_flags &
+			 MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT));
+
 		props->gid_tbl_len = 16;
 		props->ip_gids = true;
 		props->max_msg_sz = SZ_16M;
-		if (port == 1)
+		if (port_supports_cm)
 			props->port_cap_flags = IB_PORT_CM_SUP;
 	}
 
@@ -1059,6 +1075,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	req.max_recv_sge = attr->cap.max_recv_sge;
 	req.flags = flags;
 
+	if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE)
+		req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu;
+
 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
 	if (err)
 		return err;
@@ -1134,6 +1153,7 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	struct gdma_context *gc = mdev_to_gc(mdev);
 	struct mana_rnic_create_udqp_resp resp = {};
 	struct mana_rnic_create_udqp_req req = {};
+	struct net_device *ndev;
 	int err, i;
 
 	mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp));
@@ -1150,6 +1170,21 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
 	req.max_send_sge = attr->cap.max_send_sge;
 	req.max_recv_sge = attr->cap.max_recv_sge;
 	req.qp_type = type;
+
+	/* For GSI QPs, pass the vNIC MAC so the SoC can associate the QP with
+	 * the correct port, transition to INIT, and allocate a per-port QPN.
+	 * MAC must be in reversed byte order.
+	 */
+	if (type == IB_QPT_GSI &&
+	    (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)) {
+		ndev = mana_ib_get_netdev(&mdev->ib_dev, attr->port_num);
+		if (ndev) {
+			copy_in_reverse(req.mac, ndev->dev_addr, ETH_ALEN);
+			req.flags = MANA_UD_QP_FLAG_CREATE_IN_INIT;
+			req.hdr.req.msg_version = GDMA_MESSAGE_V2;
+		}
+	}
+
 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
 	if (err)
 		return err;
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index f698666967764d..9173ca64d2e1ec 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -24,8 +24,12 @@
 /* MANA doesn't have any limit for MR size */
 #define MANA_IB_MAX_MR_SIZE	U64_MAX
 
-/* Send queue ID mask */
-#define MANA_SENDQ_MASK	BIT(31)
+/*
+ * Send queue ID mask. Queue IDs are 2-bit aligned (see MANA_QID_SUBTYPE_MASK),
+ * so bit 0 is always free to tag send queues in the lookup table. This keeps
+ * the whole top byte available to index per-port GSI QPs by (port << 24).
+ */
+#define MANA_SENDQ_MASK	BIT(0)
 /* Queue ID encodes type in the lower 2 bits */
 #define MANA_QID_SUBTYPE_MASK 0x3
 
@@ -162,17 +166,20 @@ struct mana_ib_cq {
 	struct mana_ib_queue queue;
 	/* protects CQ polling */
 	spinlock_t cq_lock;
-	struct list_head list_send_qp;
-	struct list_head list_recv_qp;
+	struct list_head send_err_qp_list;
+	struct list_head recv_err_qp_list;
+	struct gdma_comp pending_cqe;
+	bool has_pending_cqe;
 	int cqe;
 	u32 comp_vector;
+	u32 poll_credit;
 	mana_handle_t  cq_handle;
 };
 
 enum mana_rc_queue_type {
 	MANA_RC_SEND_QUEUE_REQUESTER = 0,
 	MANA_RC_SEND_QUEUE_RESPONDER,
-	MANA_RC_SEND_QUEUE_FMR,
+	MANA_RC_SEND_QUEUE_MMQ,
 	MANA_RC_RECV_QUEUE_REQUESTER,
 	MANA_RC_RECV_QUEUE_RESPONDER,
 	MANA_RC_QUEUE_TYPE_MAX,
@@ -180,6 +187,7 @@ enum mana_rc_queue_type {
 
 struct mana_ib_rc_qp {
 	struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
+	u32 wqe_size_in_bu;
 };
 
 enum mana_uc_queue_type {
@@ -201,7 +209,6 @@ enum mana_ud_queue_type {
 
 struct mana_ib_ud_qp {
 	struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX];
-	u32 sq_psn;
 };
 
 struct mana_ib_qp {
@@ -217,9 +224,14 @@ struct mana_ib_qp {
 
 	/* The port on the IB device, starting with 1 */
 	u32 port;
+	u32 sq_psn;
+	bool sq_sig_all;
+
+	/* Serializes QP modification and error-list transitions. */
+	struct mutex modify_lock;
 
-	struct list_head cq_send_list;
-	struct list_head cq_recv_list;
+	struct list_head send_err_node;
+	struct list_head recv_err_node;
 	struct shadow_queue shadow_rq;
 	struct shadow_queue shadow_sq;
 
@@ -262,6 +274,9 @@ enum mana_ib_adapter_features {
 	MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
 	MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
 	MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
+	MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
+	MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT = BIT(14),
+	MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15),
 };
 
 struct mana_ib_query_adapter_caps_resp {
@@ -372,7 +387,9 @@ struct mana_rnic_destroy_cq_resp {
 }; /* HW Data */
 
 enum mana_rnic_create_rc_flags {
-	MANA_RC_FLAG_NO_FMR = 2,
+	MANA_RC_FLAG_NO_MMQ = BIT(1),
+	MANA_RC_FLAG_FIXED_SIZE_WQE = BIT(3),
+	MANA_RC_FLAG_MSN_IN_WQE = BIT(4),
 };
 
 struct mana_rnic_create_qp_req {
@@ -389,7 +406,8 @@ struct mana_rnic_create_qp_req {
 	u32 max_recv_wr;
 	u32 max_send_sge;
 	u32 max_recv_sge;
-	u32 reserved;
+	u8 wqe_size_in_bu;
+	u8 reserved[3];
 }; /* HW Data */
 
 struct mana_rnic_create_qp_resp {
@@ -445,8 +463,14 @@ struct mana_rnic_create_udqp_req {
 	u32 max_recv_wr;
 	u32 max_send_sge;
 	u32 max_recv_sge;
+	u8 mac[ETH_ALEN];     /* V2: port MAC for multi-port GSI */
+	u16 flags;            /* V2: MANA_UD_QP_FLAG_* */
 }; /* HW Data */
 
+enum mana_ud_qp_flags {
+	MANA_UD_QP_FLAG_CREATE_IN_INIT = BIT(0),
+};
+
 struct mana_rnic_create_udqp_resp {
 	struct gdma_resp_hdr hdr;
 	mana_handle_t qp_handle;
@@ -508,7 +532,16 @@ struct mana_rnic_set_qp_state_resp {
 
 enum WQE_OPCODE_TYPES {
 	WQE_TYPE_UD_SEND = 0,
+	WQE_TYPE_RC_SEND = 2,
+	WQE_TYPE_RC_SEND_IMM = 3,
+	WQE_TYPE_RC_SEND_INV = 4,
+	WQE_TYPE_WRITE = 5,
+	WQE_TYPE_WRITE_IMM = 6,
+	WQE_TYPE_READ = 7,
 	WQE_TYPE_UD_RECV = 8,
+	WQE_TYPE_RC_RECV = 9,
+	WQE_TYPE_REG_MR = 10,
+	WQE_TYPE_LOCAL_INV = 12,
 }; /* HW DATA */
 
 struct rdma_send_oob {
@@ -527,7 +560,83 @@ struct rdma_send_oob {
 			u32 reserved1;
 			u32 reserved2;
 		} ud_send;
+		union {
+			u32 immediate;
+			u32 invalidate_key;
+		} rc_send;
+		struct {
+			u32 address_hi;
+			u32 address_low;
+			u32 rkey;
+			u32 dma_len;
+		} rdma;
+		struct {
+			u32 mkey;
+		} mm;
 	};
+	union {
+		u32 immediate_ext;
+		struct {
+			u16 rsn;
+			u16 reserved;
+		} read;
+	};
+	u32 fsn : 24;
+	u32 reserved2   : 8;
+}; /* HW DATA */
+
+struct rdma_recv_oob {
+	u32 psn_start   : 24;
+	u32 reserved1   : 8;
+	u32 msn         : 24;
+	u32 reserved2   : 8;
+}; /* HW DATA */
+
+enum mana_ib_error_code {
+	VENDOR_ERR_OK					= 0x0,
+	VENDOR_ERR_RX_PKT_LEN                           = 0x05,
+	VENDOR_ERR_RX_MSG_LEN_OVFL                      = 0x102,
+	VENDOR_ERR_RX_MISBEHAVING_CLIENT                = 0x108,
+	VENDOR_ERR_RX_MALFORMED_WQE                     = 0x109,
+	VENDOR_ERR_RX_CLIENT_ID                         = 0x10a,
+	VENDOR_ERR_RX_GFID                              = 0x10b,
+	VENDOR_ERR_RX_PCIE                              = 0x10c,
+	VENDOR_ERR_RX_NO_AVAIL_WQE                      = 0x111,
+	VENDOR_ERR_RX_ATB_SGE_MISSCONFIG                = 0x143,
+	VENDOR_ERR_RX_ATB_WQE_MISCONFIG                 = 0x145,
+	VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT                = 0x183,
+	VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT                = 0x185,
+	VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE                = 0x1c3,
+	VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE                = 0x1c5,
+	VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE              = 0x1c7,
+	VENDOR_ERR_TX_GDMA_CORRUPTED_WQE                = 0x201,
+	VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION          = 0x202,
+	VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE                = 0x203,
+	VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR                = 0x204,
+	VENDOR_ERR_TX_PCIE_WQE                          = 0x205,
+	VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION          = 0x206,
+	VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE                = 0x207,
+	VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR                = 0x208,
+	VENDOR_ERR_TX_PCIE_MSG                          = 0x209,
+	VENDOR_ERR_TX_GDMA_INVALID_STATE                = 0x20a,
+	VENDOR_ERR_TX_MISBEHAVING_CLIENT                = 0x20b,
+	VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE           = 0x210,
+	VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD          = 0x211,
+	VENDOR_ERR_TX_RDMA_INVALID_STATE                = 0x212,
+	VENDOR_ERR_TX_RDMA_INVALID_NPT                  = 0x213,
+	VENDOR_ERR_TX_RDMA_INVALID_SGID                 = 0x214,
+	VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED              = 0x215,
+	VENDOR_ERR_TX_RDMA_WQE_LEN_ERR                  = 0x216,
+	VENDOR_ERR_TX_RDMA_MTU_ERR                      = 0x217,
+	VENDOR_ERR_TX_RDMA_VFID_MISMATCH                = 0x218,
+	VENDOR_ERR_HW_MAX                               = 0x3ff,
+	/* SW vendor errors */
+	VENDOR_ERR_SW_FLUSHED				= 0xfff,
+};
+
+enum mana_ib_cqe_type {
+	CQE_TYPE_UD_SEND = 1,
+	CQE_TYPE_UD_SEND_IMM = 2,
 }; /* HW DATA */
 
 struct mana_rdma_cqe {
@@ -538,8 +647,7 @@ struct mana_rdma_cqe {
 		};
 		struct {
 			u32 cqe_type		: 8;
-			u32 vendor_error	: 9;
-			u32 reserved1		: 15;
+			u32 reserved1		: 24;
 			u32 sge_offset		: 5;
 			u32 tx_wqe_offset	: 27;
 		} ud_send;
@@ -714,7 +822,7 @@ int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
 int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type,
 				struct mana_ib_queue *queue);
 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
-			 struct mana_ib_queue *queue);
+			 struct mana_ib_queue *queue, bool is_cq);
 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
 
 struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 8b7be1255c0d04..15c5d6cefd6288 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -340,7 +340,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
 		  ucmd.sq_buf_addr, ucmd.port);
 
-	err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq);
+	err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq,
+				   false);
 	if (err) {
 		ibdev_dbg(&mdev->ib_dev,
 			  "Failed to create queue for create qp-raw, err %d\n", err);
@@ -422,6 +423,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
 	return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);
 }
 
+static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size)
+{
+	return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size));
+}
+
 static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)
 {
 	u32 queue_size;
@@ -509,8 +515,20 @@ static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
 	if (err)
 		goto err_remove_sq;
 
+	/* GSI QPs are additionally indexed by (port << 24 | MANA_GSI_QPN) so the
+	 * per-port GSI SQ drain can find each of them by iterating ports.
+	 */
+	if (qp->ibqp.qp_type == IB_QPT_GSI) {
+		err = mana_table_store_qp_qid(mdev, qp,
+					      (qp->port << 24) | MANA_GSI_QPN, false);
+		if (err)
+			goto err_remove_rq;
+	}
+
 	return 0;
 
+err_remove_rq:
+	mana_table_remove_qp_qid(mdev, rq->id, false);
 err_remove_sq:
 	mana_table_remove_qp_qid(mdev, sq->id, true);
 	mana_table_drain_qp_ref(qp);
@@ -529,6 +547,8 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp
 
 	mana_table_remove_qp_qid(mdev, sq->id, true);
 	mana_table_remove_qp_qid(mdev, rq->id, false);
+	if (qp->ibqp.qp_type == IB_QPT_GSI)
+		mana_table_remove_qp_qid(mdev, (qp->port << 24) | MANA_GSI_QPN, false);
 	mana_table_drain_qp_ref(qp);
 }
 
@@ -549,27 +569,42 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);
 	doorbell = mana_ucontext->doorbell;
-	flags = MANA_RC_FLAG_NO_FMR;
-	err = ib_copy_validate_udata_in(udata, ucmd, queue_size);
+	flags = MANA_RC_FLAG_NO_MMQ;
+	err = ib_copy_validate_udata_in_cm(udata, ucmd, queue_size,
+					   MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE);
 	if (err)
 		return err;
 
 	for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-		/* skip FMR for user-level RC QPs */
-		if (i == MANA_RC_SEND_QUEUE_FMR) {
-			qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
-			qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+		if (i == MANA_RC_SEND_QUEUE_MMQ) {
+			if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) {
+				flags &= ~MANA_RC_FLAG_NO_MMQ;
+				err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size,
+							   &qp->rc_qp.queues[i], false);
+				if (err)
+					goto destroy_queues;
+			} else {
+				qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
+				qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+			}
 			continue;
 		}
 		err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
-					   &qp->rc_qp.queues[i]);
-		if (err) {
-			ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err);
+					   &qp->rc_qp.queues[i], false);
+		if (err)
 			goto destroy_queues;
-		}
 		j++;
 	}
 
+	if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) {
+		u32 wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge,
+						      INLINE_OOB_EXTRA_LARGE_SIZE);
+		flags |= MANA_RC_FLAG_FIXED_SIZE_WQE;
+		if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT)
+			flags |= MANA_RC_FLAG_MSN_IN_WQE;
+		qp->rc_qp.wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE;
+	}
+
 	err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create rc qp  %d\n", err);
@@ -580,8 +615,10 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	if (udata) {
 		for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
-			if (i == MANA_RC_SEND_QUEUE_FMR)
+			if (i == MANA_RC_SEND_QUEUE_MMQ) {
+				resp.mmq_id = qp->rc_qp.queues[i].id;
 				continue;
+			}
 			resp.queue_id[j] = qp->rc_qp.queues[i].id;
 			j++;
 		}
@@ -627,7 +664,7 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 
 	for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; ++i) {
 		err = mana_ib_create_queue(mdev, ucmd.queue_buf[i], ucmd.queue_size[i],
-					   &qp->uc_qp.queues[i]);
+					   &qp->uc_qp.queues[i], false);
 		if (err)
 			goto destroy_queues;
 	}
@@ -658,33 +695,23 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	return err;
 }
 
-static void mana_add_qp_to_cqs(struct mana_ib_qp *qp)
+static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp, bool reset)
 {
 	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
 	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
 	unsigned long flags;
 
 	spin_lock_irqsave(&send_cq->cq_lock, flags);
-	list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
+	list_del_init(&qp->send_err_node);
+	/* Keep shadow reset serialized with hardware polling and SW flushing. */
+	if (reset)
+		reset_shadow_queue(&qp->shadow_sq);
 	spin_unlock_irqrestore(&send_cq->cq_lock, flags);
 
 	spin_lock_irqsave(&recv_cq->cq_lock, flags);
-	list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
-	spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
-}
-
-static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp)
-{
-	struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
-	struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
-	unsigned long flags;
-
-	spin_lock_irqsave(&send_cq->cq_lock, flags);
-	list_del(&qp->cq_send_list);
-	spin_unlock_irqrestore(&send_cq->cq_lock, flags);
-
-	spin_lock_irqsave(&recv_cq->cq_lock, flags);
-	list_del(&qp->cq_recv_list);
+	list_del_init(&qp->recv_err_node);
+	if (reset)
+		reset_shadow_queue(&qp->shadow_rq);
 	spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
 }
 
@@ -712,13 +739,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	doorbell = mdev->gdma_dev->doorbell;
 
 	err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr,
-				  sizeof(struct ud_rq_shadow_wqe));
+				  sizeof(struct shadow_wqe_header));
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err);
 		goto destroy_queues;
 	}
 	err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr,
-				  sizeof(struct ud_sq_shadow_wqe));
+				  sizeof(struct shadow_wqe_header));
 	if (err) {
 		ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err);
 		goto destroy_shadow_queues;
@@ -729,7 +756,8 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 		ibdev_err(&mdev->ib_dev, "Failed to create ud qp  %d\n", err);
 		goto destroy_shadow_queues;
 	}
-	qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
+	qp->ibqp.qp_num = (qp->ibqp.qp_type == IB_QPT_GSI) ?
+			  MANA_GSI_QPN : qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
 	qp->port = attr->port_num;
 
 	for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
@@ -739,8 +767,6 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	if (err)
 		goto destroy_qp;
 
-	mana_add_qp_to_cqs(qp);
-
 	return 0;
 
 destroy_qp:
@@ -757,6 +783,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		      struct ib_udata *udata)
 {
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	qp->sq_sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR;
+	mutex_init(&qp->modify_lock);
+	INIT_LIST_HEAD(&qp->send_err_node);
+	INIT_LIST_HEAD(&qp->recv_err_node);
+
 	switch (attr->qp_type) {
 	case IB_QPT_RAW_PACKET:
 		/* When rwq_ind_tbl is used, it's for creating WQs for RSS */
@@ -854,19 +887,56 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	return 0;
 }
 
+static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+				    int attr_mask, struct ib_udata *udata)
+{
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	if (udata)
+		return;
+
+	if (attr_mask & IB_QP_STATE) {
+		switch (attr->qp_state) {
+		case IB_QPS_RESET:
+			mana_remove_qp_from_cqs(qp, true);
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (attr_mask & IB_QP_SQ_PSN)
+		qp->sq_psn = attr->sq_psn;
+}
+
 int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		      int attr_mask, struct ib_udata *udata)
 {
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+	int ret;
+
+	mutex_lock(&qp->modify_lock);
+
 	switch (ibqp->qp_type) {
 	case IB_QPT_RC:
 	case IB_QPT_UC:
 	case IB_QPT_UD:
 	case IB_QPT_GSI:
-		return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
+		ret = mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
+		if (ret)
+			goto out_unlock;
+		break;
 	default:
-		ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type);
-		return -EOPNOTSUPP;
+		ret = -EOPNOTSUPP;
+		goto out_unlock;
 	}
+
+	mana_ib_modify_qp_state(ibqp, attr, attr_mask, udata);
+
+out_unlock:
+	mutex_unlock(&qp->modify_lock);
+
+	return ret;
 }
 
 static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
@@ -1003,8 +1073,8 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
 	if (err)
 		return err;
 
-	mana_remove_qp_from_cqs(qp);
 	mana_table_remove_qp(mdev, qp);
+	mana_remove_qp_from_cqs(qp, false);
 
 	destroy_shadow_queue(&qp->shadow_rq);
 	destroy_shadow_queue(&qp->shadow_sq);
diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h
index a4b3818f9c3971..b5d6bb28bd4527 100644
--- a/drivers/infiniband/hw/mana/shadow_queue.h
+++ b/drivers/infiniband/hw/mana/shadow_queue.h
@@ -6,21 +6,19 @@
 #ifndef _MANA_SHADOW_QUEUE_H_
 #define _MANA_SHADOW_QUEUE_H_
 
-struct shadow_wqe_header {
-	u16 opcode;
-	u16 error_code;
-	u32 posted_wqe_size;
-	u64 wr_id;
-};
+#include <linux/build_bug.h>
 
-struct ud_rq_shadow_wqe {
-	struct shadow_wqe_header header;
-	u32 byte_len;
-	u32 src_qpn;
-};
+#define MANA_WQ_FENCE_WC		BIT(0)
+#define MANA_WQ_NO_SIGNAL_WC		BIT(1)
+#define MANA_WQE_OFFSET_MASK	GENMASK(23, 0)
 
-struct ud_sq_shadow_wqe {
-	struct shadow_wqe_header header;
+struct shadow_wqe_header {
+	u64 wr_id;
+	u64 wqe_offset_or_psn : 24;
+	u64 wqe_size_in_bu : 8;
+	u64 fsn : 24;
+	u64 send_opcode : 4;
+	u64 flags : 2;
 };
 
 struct shadow_queue {
@@ -28,8 +26,6 @@ struct shadow_queue {
 	u64 prod_idx;
 	/* Unmasked consumer index, Incremented on cq polling */
 	u64 cons_idx;
-	/* Unmasked index of next-to-complete (from HW) shadow WQE */
-	u64 next_to_complete_idx;
 	/* queue size in wqes */
 	u32 length;
 	/* distance between elements in bytes */
@@ -50,6 +46,12 @@ static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t lengt
 	return 0;
 }
 
+static inline void reset_shadow_queue(struct shadow_queue *queue)
+{
+	queue->prod_idx = 0;
+	queue->cons_idx = 0;
+}
+
 static inline void destroy_shadow_queue(struct shadow_queue *queue)
 {
 	kvfree(queue->buffer);
@@ -57,12 +59,14 @@ static inline void destroy_shadow_queue(struct shadow_queue *queue)
 
 static inline bool shadow_queue_full(struct shadow_queue *queue)
 {
-	return (queue->prod_idx - queue->cons_idx) >= queue->length;
+	/* Do not reuse an entry until the poller has finished reading it. */
+	return (queue->prod_idx - smp_load_acquire(&queue->cons_idx)) >= queue->length;
 }
 
 static inline bool shadow_queue_empty(struct shadow_queue *queue)
 {
-	return queue->prod_idx == queue->cons_idx;
+	/* Pair with posting's release of the initialized shadow WQE. */
+	return smp_load_acquire(&queue->prod_idx) == queue->cons_idx;
 }
 
 static inline void *
@@ -82,34 +86,23 @@ shadow_queue_producer_entry(struct shadow_queue *queue)
 static inline void *
 shadow_queue_get_next_to_consume(const struct shadow_queue *queue)
 {
-	if (queue->cons_idx == queue->next_to_complete_idx)
+	/* The producer publishes the WQE before advancing prod_idx. */
+	if (queue->cons_idx == smp_load_acquire(&queue->prod_idx))
 		return NULL;
 
 	return shadow_queue_get_element(queue, queue->cons_idx);
 }
 
-static inline void *
-shadow_queue_get_next_to_complete(struct shadow_queue *queue)
-{
-	if (queue->next_to_complete_idx == queue->prod_idx)
-		return NULL;
-
-	return shadow_queue_get_element(queue, queue->next_to_complete_idx);
-}
-
 static inline void shadow_queue_advance_producer(struct shadow_queue *queue)
 {
-	queue->prod_idx++;
+	/* Publish all WQE fields to CQ polling on another CPU. */
+	smp_store_release(&queue->prod_idx, queue->prod_idx + 1);
 }
 
 static inline void shadow_queue_advance_consumer(struct shadow_queue *queue)
 {
-	queue->cons_idx++;
-}
-
-static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue)
-{
-	queue->next_to_complete_idx++;
+	/* Finish WC generation and queue-tail updates before allowing reuse. */
+	smp_store_release(&queue->cons_idx, queue->cons_idx + 1);
 }
 
 #endif
diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c
index 6b066d605dcb14..4042f26632c0fc 100644
--- a/drivers/infiniband/hw/mana/wq.c
+++ b/drivers/infiniband/hw/mana/wq.c
@@ -25,7 +25,8 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
 
 	ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);
 
-	err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue);
+	err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue,
+				   false);
 	if (err) {
 		ibdev_dbg(&mdev->ib_dev,
 			  "Failed to create queue for create wq, %d\n", err);
diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c
index 36a1d506f08f65..250d673f58f207 100644
--- a/drivers/infiniband/hw/mana/wr.c
+++ b/drivers/infiniband/hw/mana/wr.c
@@ -7,29 +7,30 @@
 
 #define MAX_WR_SGL_NUM (2)
 
-static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)
+static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)
 {
-	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
-	struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
+	struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0};
+	struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp);
 	struct gdma_posted_wqe_info wqe_info = {0};
-	struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM];
+	struct gdma_queue *queue = ib_rq->kmem;
 	struct gdma_wqe_request wqe_req = {0};
-	struct ud_rq_shadow_wqe *shadow_wqe;
-	int err, i;
+	struct shadow_wqe_header *shadow_wqe;
+	int err;
 
 	if (shadow_queue_full(&qp->shadow_rq))
 		return -EINVAL;
 
-	if (wr->num_sge > MAX_WR_SGL_NUM)
+	if (wr->num_sge > MAX_RX_WQE_SGL_ENTRIES)
 		return -EINVAL;
 
-	for (i = 0; i < wr->num_sge; ++i) {
-		gdma_sgl[i].address = wr->sg_list[i].addr;
-		gdma_sgl[i].mem_key = wr->sg_list[i].lkey;
-		gdma_sgl[i].size = wr->sg_list[i].length;
-	}
 	wqe_req.num_sge = wr->num_sge;
-	wqe_req.sgl = gdma_sgl;
+	wqe_req.ib_sgl = wr->sg_list;
+	wqe_req.flags = GDMA_WR_IB_SGL;
+
+	if (wr->num_sge == 0) {
+		wqe_req.ib_sgl = &mana_ib_dummy_sge;
+		wqe_req.num_sge = 1;
+	}
 
 	err = mana_gd_post_work_request(queue, &wqe_req, &wqe_info);
 	if (err)
@@ -37,42 +38,55 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *
 
 	shadow_wqe = shadow_queue_producer_entry(&qp->shadow_rq);
 	memset(shadow_wqe, 0, sizeof(*shadow_wqe));
-	shadow_wqe->header.opcode = IB_WC_RECV;
-	shadow_wqe->header.wr_id = wr->wr_id;
-	shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wr_id = wr->wr_id;
+	shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset;
 	shadow_queue_advance_producer(&qp->shadow_rq);
 
-	mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
 	return 0;
 }
 
-int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
-		      const struct ib_recv_wr **bad_wr)
+static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr,
+				const struct ib_recv_wr **bad_wr)
 {
-	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+	struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp);
+	struct gdma_queue *rq = ib_rq->kmem;
+	bool ring_rq = false;
 	int err = 0;
 
 	for (; wr; wr = wr->next) {
-		switch (ibqp->qp_type) {
-		case IB_QPT_UD:
-		case IB_QPT_GSI:
-			err = mana_ib_post_recv_ud(qp, wr);
-			if (unlikely(err)) {
-				*bad_wr = wr;
-				return err;
-			}
+		err = mana_ib_post_rq(qp, wr);
+		if (unlikely(err)) {
+			*bad_wr = wr;
 			break;
-		default:
-			ibdev_dbg(ibqp->device, "Posting recv wr on qp type %u is not supported\n",
-				  ibqp->qp_type);
-			return -EINVAL;
 		}
+		ring_rq = true;
 	}
 
+	if (ring_rq)
+		mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), rq);
+
 	return err;
 }
 
-static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
+int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
+		      const struct ib_recv_wr **bad_wr)
+{
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	switch (ibqp->qp_type) {
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return mana_ib_post_recv_ud(qp, wr, bad_wr);
+	default:
+		/* Unsupported QP type */
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+}
+
+static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
 {
 	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
 	struct mana_ib_ah *ah = container_of(wr->ah, struct mana_ib_ah, ibah);
@@ -82,14 +96,12 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
 	struct gdma_posted_wqe_info wqe_info = {0};
 	struct gdma_wqe_request wqe_req = {0};
 	struct rdma_send_oob send_oob = {0};
-	struct ud_sq_shadow_wqe *shadow_wqe;
+	struct shadow_wqe_header *shadow_wqe;
+	bool signaled = qp->sq_sig_all || (wr->wr.send_flags & IB_SEND_SIGNALED);
 	int err, i;
 
-	if (!ndev) {
-		ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n",
-			  qp->port, qp->ibqp.qp_num);
+	if (!ndev)
 		return -EINVAL;
-	}
 
 	if (wr->wr.opcode != IB_WR_SEND)
 		return -EINVAL;
@@ -111,16 +123,16 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
 
 	wqe_req.num_sge = wr->wr.num_sge + 1;
 	wqe_req.sgl = gdma_sgl;
-	wqe_req.inline_oob_size = sizeof(struct rdma_send_oob);
+	wqe_req.inline_oob_size = INLINE_OOB_LARGE_SIZE;
 	wqe_req.inline_oob_data = &send_oob;
 	wqe_req.flags = GDMA_WR_OOB_IN_SGL;
 	wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev->mtu));
 
 	send_oob.wqe_type = WQE_TYPE_UD_SEND;
 	send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE);
-	send_oob.signaled = !!(wr->wr.send_flags & IB_SEND_SIGNALED);
+	send_oob.signaled = signaled;
 	send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
-	send_oob.psn = qp->ud_qp.sq_psn;
+	send_oob.psn = qp->sq_psn;
 	send_oob.ssn_or_rqpn = wr->remote_qpn;
 	send_oob.ud_send.remote_qkey =
 		qp->ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr->remote_qkey;
@@ -129,40 +141,64 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
 	if (err)
 		return err;
 
-	qp->ud_qp.sq_psn++;
+	qp->sq_psn++;
 	shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq);
 	memset(shadow_wqe, 0, sizeof(*shadow_wqe));
-	shadow_wqe->header.opcode = IB_WC_SEND;
-	shadow_wqe->header.wr_id = wr->wr.wr_id;
-	shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wr_id = wr->wr.wr_id;
+	shadow_wqe->flags = signaled ? 0 : MANA_WQ_NO_SIGNAL_WC;
+	shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
+	shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset;
 	shadow_queue_advance_producer(&qp->shadow_sq);
 
-	mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
 	return 0;
 }
 
-int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
-		      const struct ib_send_wr **bad_wr)
+static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *wr,
+				const struct ib_send_wr **bad_wr)
 {
+	struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+	struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
+	struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
+	unsigned long flags;
+	bool ring_sq = false;
+	bool flush_send;
 	int err = 0;
-	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
 
 	for (; wr; wr = wr->next) {
-		switch (ibqp->qp_type) {
-		case IB_QPT_UD:
-		case IB_QPT_GSI:
-			err = mana_ib_post_send_ud(qp, ud_wr(wr));
-			if (unlikely(err)) {
-				*bad_wr = wr;
-				return err;
-			}
+		err = mana_ib_post_send_ud_one(qp, ud_wr(wr));
+		if (unlikely(err)) {
+			*bad_wr = wr;
 			break;
-		default:
-			ibdev_dbg(ibqp->device, "Posting send wr on qp type %u is not supported\n",
-				  ibqp->qp_type);
-			return -EINVAL;
 		}
+		ring_sq = true;
+	}
+
+	if (ring_sq) {
+		mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
+
+		spin_lock_irqsave(&cq->cq_lock, flags);
+		flush_send = !list_empty(&qp->send_err_node);
+		spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+		if (flush_send && cq->ibcq.comp_handler)
+			cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 	}
 
 	return err;
 }
+
+int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
+		      const struct ib_send_wr **bad_wr)
+{
+	struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+
+	switch (ibqp->qp_type) {
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+		return mana_ib_post_send_ud(qp, wr, bad_wr);
+	default:
+		/* Unsupported QP type */
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+}
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index 8879124697420c..81dc47e57352d6 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -185,9 +185,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
 			goto err_umem;
 		}
 	} else {
-		cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
-					  entries * cqe_size,
-					  IB_ACCESS_LOCAL_WRITE);
+		cq->umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
+						    ucmd.buf_addr,
+						    entries * cqe_size,
+						    IB_ACCESS_LOCAL_WRITE);
 		if (IS_ERR(cq->umem)) {
 			err = PTR_ERR(cq->umem);
 			goto err_cq;
@@ -354,9 +355,10 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
 	if (!cq->resize_buf)
 		return -ENOMEM;
 
-	cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
-					 entries * cqe_size,
-					 IB_ACCESS_LOCAL_WRITE);
+	cq->resize_umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
+						   ucmd.buf_addr,
+						   entries * cqe_size,
+						   IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(cq->resize_umem)) {
 		err = PTR_ERR(cq->resize_umem);
 		goto err_buf;
@@ -989,7 +991,8 @@ void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
 	 * Now sweep backwards through the CQ, removing CQ entries
 	 * that match our QP by copying older entries on top of them.
 	 */
-	while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
+	while (prod_index != cq->mcq.cons_index) {
+		--prod_index;
 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 		cqe += cqe_inc;
 
diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c
index dc7c0daaaba5b1..bba178ffe1b27d 100644
--- a/drivers/infiniband/hw/mlx4/mcg.c
+++ b/drivers/infiniband/hw/mlx4/mcg.c
@@ -433,7 +433,7 @@ static u16 cmp_rec(struct ib_sa_mcmember_data *src,
 }
 
 /* release group, return 1 if this was last release and group is destroyed
- * timout work is canceled sync */
+ * timeout work is canceled sync */
 static int release_group(struct mcast_group *group, int from_timeout_handler)
 {
 	struct mlx4_ib_demux_ctx *ctx = group->demux;
@@ -694,7 +694,7 @@ static void mlx4_ib_mcg_work_handler(struct work_struct *work)
 			cur_join_state = group->rec.scope_join_state & 0xf;
 
 			if (method == IB_MGMT_METHOD_GET_RESP) {
-				/* successfull join */
+				/* successful join */
 				if (!cur_join_state && resp_join_state)
 					--rc;
 			} else if (!resp_join_state)
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 49b4bf148a4a01..072b77a2c8a6cf 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -1169,7 +1169,8 @@ void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq)
 	/* Now sweep backwards through the CQ, removing CQ entries
 	 * that match our QP by copying older entries on top of them.
 	 */
-	while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
+	while (prod_index != cq->mcq.cons_index) {
+		--prod_index;
 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 		cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
 		if (is_equal_rsn(cqe64, rsn)) {
@@ -1245,9 +1246,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 	if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1)
 		return -EINVAL;
 
-	umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
-			      (size_t)ucmd.cqe_size * entries,
-			      IB_ACCESS_LOCAL_WRITE);
+	umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, ucmd.buf_addr,
+					(size_t)ucmd.cqe_size * entries,
+					IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(umem)) {
 		err = PTR_ERR(umem);
 		return err;
diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c
index d57484245c3823..93a56ee2c4ef49 100644
--- a/drivers/infiniband/hw/mlx5/data_direct.c
+++ b/drivers/infiniband/hw/mlx5/data_direct.c
@@ -65,13 +65,9 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (err) {
-		dev_warn(&pdev->dev,
-			 "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err);
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err);
-			return err;
-		}
+		dev_err(&pdev->dev,
+			"Can't set 64-bit PCI DMA mask, err=%d\n", err);
+		return err;
 	}
 
 	dma_set_max_seg_size(&pdev->dev, SZ_2G);
diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c
index b155baee0017c3..e63fdb4bc3e3fa 100644
--- a/drivers/infiniband/hw/mlx5/fs.c
+++ b/drivers/infiniband/hw/mlx5/fs.c
@@ -17,6 +17,7 @@
 #include <linux/mlx5/fs.h>
 #include <linux/mlx5/fs_helpers.h>
 #include <linux/mlx5/eswitch.h>
+#include <linux/overflow.h>
 #include <net/inet_ecn.h>
 #include "mlx5_ib.h"
 #include "counters.h"
@@ -1647,36 +1648,32 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de
 							  struct ib_flow_attr *flow_attr,
 							  struct mlx5_flow_destination *dst)
 {
+	DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_wc, flows, 1);
+	DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_uc, flows, 1);
+
 	struct mlx5_ib_flow_handler *handler_ucast = NULL;
 	struct mlx5_ib_flow_handler *handler = NULL;
 
-	static struct {
-		struct ib_flow_spec_eth eth_flow;
-		struct ib_flow_attr	flow_attr;
-	} leftovers_wc = { .flow_attr = { .num_of_specs = 1,
-					  .size = sizeof(leftovers_wc) },
-			   .eth_flow = {
-				   .type = IB_FLOW_SPEC_ETH,
-				   .size = sizeof(struct ib_flow_spec_eth),
-				   .mask = { .dst_mac = { 0x1 } },
-				   .val = { .dst_mac = { 0x1 } } } };
-
-	static struct {
-		struct ib_flow_spec_eth eth_flow;
-		struct ib_flow_attr	flow_attr;
-	} leftovers_uc = { .flow_attr = { .num_of_specs = 1,
-					  .size = sizeof(leftovers_uc) },
-			   .eth_flow = {
-				   .type = IB_FLOW_SPEC_ETH,
-				   .size = sizeof(struct ib_flow_spec_eth),
-				   .mask = { .dst_mac = { 0x1 } },
-				   .val = { .dst_mac = {} } } };
-
-	handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst);
+	flow_attr_wc->num_of_specs = 1;
+	flow_attr_wc->size = sizeof(struct ib_flow_attr) +
+			      sizeof(struct ib_flow_spec_eth);
+	flow_attr_wc->flows[0].eth.type = IB_FLOW_SPEC_ETH;
+	flow_attr_wc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth);
+	flow_attr_wc->flows[0].eth.mask.dst_mac[0] = 0x1;
+	flow_attr_wc->flows[0].eth.val.dst_mac[0] = 0x1;
+
+	flow_attr_uc->num_of_specs = 1;
+	flow_attr_uc->size = sizeof(struct ib_flow_attr) +
+			      sizeof(struct ib_flow_spec_eth);
+	flow_attr_uc->flows[0].eth.type = IB_FLOW_SPEC_ETH;
+	flow_attr_uc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth);
+	flow_attr_uc->flows[0].eth.mask.dst_mac[0] = 0x1;
+
+	handler = create_flow_rule(dev, ft_prio, flow_attr_wc, dst);
 	if (!IS_ERR(handler) &&
 	    flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
 		handler_ucast = create_flow_rule(dev, ft_prio,
-						 &leftovers_uc.flow_attr, dst);
+						 flow_attr_uc, dst);
 		if (IS_ERR(handler_ucast)) {
 			mlx5_del_flow_rules(handler->rule);
 			ft_prio->refcount--;
@@ -3056,8 +3053,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
 	return 0;
 
 destroy_res:
-	--ft_prio->anchor.rule_goto_table_ref;
-	mlx5_steering_anchor_destroy_res(ft_prio);
+	if (!--ft_prio->anchor.rule_goto_table_ref)
+		mlx5_steering_anchor_destroy_res(ft_prio);
 put_flow_table:
 	put_flow_table(dev, ft_prio, true);
 free_obj:
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index a457647edacdee..4a0b4dcf2830cf 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -3370,7 +3370,7 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_ib_resources *devr = &dev->devr;
 	struct ib_srq_init_attr attr;
-	struct ib_srq *s0, *s1;
+	struct ib_srq *s0 = NULL, *s1;
 	int ret = 0;
 
 	/*
@@ -3388,19 +3388,27 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
 	if (ret)
 		goto unlock;
 
-	memset(&attr, 0, sizeof(attr));
-	attr.attr.max_sge = 1;
-	attr.attr.max_wr = 1;
-	attr.srq_type = IB_SRQT_XRC;
-	attr.ext.cq = devr->c0;
-
-	s0 = ib_create_srq(devr->p0, &attr);
-	if (IS_ERR(s0)) {
-		ret = PTR_ERR(s0);
-		mlx5_ib_err(dev,
-			    "Couldn't create SRQ 0 for res init, err=%pe\n",
-			    s0);
-		goto unlock;
+	/*
+	 * s0 is an XRC-type placeholder SRQ used as the default XRQN for
+	 * XRC QPs.  Skip it when XRC is absent; all devr->s0 accesses in
+	 * qp.c are inside XRC QP paths that the verbs layer blocks before
+	 * reaching this driver when xrc=0, so NULL is safe.
+	 */
+	if (MLX5_CAP_GEN(dev->mdev, xrc)) {
+		memset(&attr, 0, sizeof(attr));
+		attr.attr.max_sge = 1;
+		attr.attr.max_wr = 1;
+		attr.srq_type = IB_SRQT_XRC;
+		attr.ext.cq = devr->c0;
+
+		s0 = ib_create_srq(devr->p0, &attr);
+		if (IS_ERR(s0)) {
+			ret = PTR_ERR(s0);
+			mlx5_ib_err(dev,
+				    "Couldn't create SRQ 0 for res init, err=%pe\n",
+				    s0);
+			goto unlock;
+		}
 	}
 
 	memset(&attr, 0, sizeof(attr));
@@ -3414,7 +3422,8 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
 		mlx5_ib_err(dev,
 			    "Couldn't create SRQ 1 for res init, err=%pe\n",
 			    s1);
-		ib_destroy_srq(s0);
+		if (s0)
+			ib_destroy_srq(s0);
 		goto unlock;
 	}
 
@@ -3431,8 +3440,11 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
 	struct mlx5_ib_resources *devr = &dev->devr;
 	int ret;
 
+	mutex_init(&devr->cq_lock);
+	mutex_init(&devr->srq_lock);
+
 	if (!MLX5_CAP_GEN(dev->mdev, xrc))
-		return -EOPNOTSUPP;
+		return 0;
 
 	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
 	if (ret)
@@ -3444,9 +3456,6 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
 		return ret;
 	}
 
-	mutex_init(&devr->cq_lock);
-	mutex_init(&devr->srq_lock);
-
 	return 0;
 }
 
@@ -3457,10 +3466,13 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
 	/* After s0/s1 init, they are not unset during the device lifetime. */
 	if (devr->s1) {
 		ib_destroy_srq(devr->s1);
-		ib_destroy_srq(devr->s0);
+		if (devr->s0)
+			ib_destroy_srq(devr->s0);
+	}
+	if (MLX5_CAP_GEN(dev->mdev, xrc)) {
+		mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
+		mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
 	}
-	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
-	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
 	/* After p0/c0 init, they are not unset during the device lifetime. */
 	if (devr->c0) {
 		ib_destroy_cq(devr->c0);
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index b8618610737aa0..8dcbb0e5ccd7c4 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -300,7 +300,7 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni,
 	/*
 	 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that
 	 * while we are doing the invalidation, no page fault will attempt to
-	 * overwrite the same MTTs.  Concurent invalidations might race us,
+	 * overwrite the same MTTs.  Concurrent invalidations might race us,
 	 * but they will write 0s as well, so no difference in the end result.
 	 */
 	for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 0d4f8b109ad2a2..627dd17804c985 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -1624,6 +1624,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		if (err)
 			return err;
 
+		sq->base.container_mibqp = qp;
+		sq->base.mqp.event = mlx5_ib_qp_event;
 		err = create_raw_packet_qp_sq(dev, udata, attrs, sq, in, pd,
 					      to_mcq(init_attr->send_cq));
 		if (err)
@@ -1635,9 +1637,6 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			resp->sqn = sq->base.mqp.qpn;
 			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
 		}
-
-		sq->base.container_mibqp = qp;
-		sq->base.mqp.event = mlx5_ib_qp_event;
 	}
 
 	if (qp->rq.wqe_cnt) {
@@ -1647,6 +1646,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		}
 
 		rq->base.container_mibqp = qp;
+		rq->base.mqp.event = mlx5_ib_qp_event;
 
 		if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
 			rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
@@ -2092,13 +2092,13 @@ static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	}
 
 	base = &qp->trans_qp.base;
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
 	kvfree(in);
 	if (err)
 		return err;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
 	if (MLX5_CAP_GEN(mdev, ece_support))
 		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
 
@@ -2236,14 +2236,14 @@ static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
 	}
 
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
 
 	kvfree(in);
 	if (err)
 		goto err_create;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
 	if (MLX5_CAP_GEN(mdev, ece_support))
 		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
 
@@ -2432,6 +2432,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
 	}
 
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
 	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
@@ -2446,8 +2448,6 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	if (err)
 		goto err_create;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
 	if (MLX5_CAP_GEN(mdev, ece_support))
 		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
 
@@ -2577,14 +2577,13 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	    MLX5_CAP_GEN(mdev, go_back_n))
 		MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
 
+	base->container_mibqp = qp;
+	base->mqp.event = mlx5_ib_qp_event;
 	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
 	kvfree(in);
 	if (err)
 		goto err_create;
 
-	base->container_mibqp = qp;
-	base->mqp.event = mlx5_ib_qp_event;
-
 	get_cqs(qp->type, attr->send_cq, attr->recv_cq,
 		&send_cq, &recv_cq);
 	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
@@ -3186,7 +3185,7 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 out:
 	if (err) {
-		mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
+		mlx5_ib_err(dev, "Create QP type %d failed, err = %d\n", qp->type, err);
 		return err;
 	}
 
@@ -5344,11 +5343,14 @@ static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
 			break;
 		default:
 			mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
-			return;
+			goto out;
 		}
 
 		rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
 	}
+
+out:
+	mlx5_core_res_put(&core_qp->common);
 }
 
 static int set_delay_drop(struct mlx5_ib_dev *dev)
@@ -5616,6 +5618,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 		rwq = kzalloc_obj(*rwq);
 		if (!rwq)
 			return ERR_PTR(-ENOMEM);
+		rwq->core_qp.event = mlx5_ib_wq_event;
 		err = prepare_user_rq(pd, init_attr, udata, rwq);
 		if (err)
 			goto err;
@@ -5639,7 +5642,6 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 			goto err_copy;
 	}
 
-	rwq->core_qp.event = mlx5_ib_wq_event;
 	rwq->ibwq.event_handler = init_attr->event_handler;
 	return &rwq->ibwq;
 
diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c
index a7a4f9420271a2..ce2ef8828c0957 100644
--- a/drivers/infiniband/hw/mlx5/qpc.c
+++ b/drivers/infiniband/hw/mlx5/qpc.c
@@ -128,7 +128,7 @@ static int rsc_event_notifier(struct notifier_block *nb,
 	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 		rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
-		rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
+		rsn |= ((u32)eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
 		break;
 	default:
 		return NOTIFY_DONE;
@@ -146,6 +146,8 @@ static int rsc_event_notifier(struct notifier_block *nb,
 	case MLX5_RES_RQ:
 	case MLX5_RES_SQ:
 		qp = (struct mlx5_core_qp *)common;
+		if (WARN_ON_ONCE(!qp->event))
+			goto out;
 		qp->event(qp, event_type);
 		/* Need to put resource in event handler */
 		return NOTIFY_OK;
@@ -165,6 +167,10 @@ static int create_resource_common(struct mlx5_ib_dev *dev,
 	int err;
 
 	qp->common.res = rsc_type;
+	refcount_set(&qp->common.refcount, 1);
+	init_completion(&qp->common.free);
+	qp->pid = current->pid;
+
 	spin_lock_irq(&table->lock);
 	err = radix_tree_insert(&table->tree,
 				qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN),
@@ -173,10 +179,6 @@ static int create_resource_common(struct mlx5_ib_dev *dev,
 	if (err)
 		return err;
 
-	refcount_set(&qp->common.refcount, 1);
-	init_completion(&qp->common.free);
-	qp->pid = current->pid;
-
 	return 0;
 }
 
@@ -258,6 +260,14 @@ int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
 
 	qp->uid = MLX5_GET(create_qp_in, in, uid);
 	qp->qpn = MLX5_GET(create_qp_out, out, qpn);
+	/* Set ibqp.qp_num before create_resource_common() inserts the QP into
+	 * the radix tree and makes it visible to EQE processing.  The
+	 * assignment in create_qp() happens only after this function returns,
+	 * leaving a window where an arriving EQE would observe qp_num == 0.
+	 * create_qp() still overrides this with 0/1 for QP0/QP1, and sets it
+	 * independently for DCT (which does not go through this function).
+	 */
+	to_mibqp(qp)->ibqp.qp_num = qp->qpn;
 
 	err = create_resource_common(dev, qp, MLX5_RES_QP);
 	if (err)
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index a973c1b7515f7b..6a4a937d97ce98 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -284,6 +284,7 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,
 
 	in.pd = to_mpd(ib_srq->pd)->pdn;
 	in.db_record = srq->db.dma;
+	srq->msrq.event = mlx5_ib_srq_event;
 	err = mlx5_cmd_create_srq(dev, &srq->msrq, &in);
 	kvfree(in.pas);
 	if (err) {
@@ -292,8 +293,6 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,
 	}
 
 	mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn);
-
-	srq->msrq.event = mlx5_ib_srq_event;
 	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
 
 	if (udata) {
diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c
index fca9e1d9d5e9bd..bb2c60380fe216 100644
--- a/drivers/infiniband/hw/mlx5/wr.c
+++ b/drivers/infiniband/hw/mlx5/wr.c
@@ -927,6 +927,7 @@ static noinline_for_stack int handle_reg_mr_integrity(struct mlx5_ib_dev *dev,
 	qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
 
 out:
+	mr->pi_mr = pi_mr;
 	return err;
 }
 
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 26c3408dcacaea..39d4ff15849e7f 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -300,7 +300,8 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn,
 	 * Now sweep backwards through the CQ, removing CQ entries
 	 * that match our QP by copying older entries on top of them.
 	 */
-	while ((int) --prod_index - (int) cq->cons_index >= 0) {
+	while (prod_index != cq->cons_index) {
+		--prod_index;
 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
 		if (cqe->my_qpn == cpu_to_be32(qpn)) {
 			if (srq && is_recv_cqe(cqe))
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index cfe3d19b73b312..97c2e18e67adda 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -1392,9 +1392,9 @@ int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	qp = get_ocrdma_qp(ibqp);
 	dev = get_ocrdma_dev(ibqp->device);
 
-	/* syncronize with multiple context trying to change, retrive qps */
+	/* synchronize with multiple context trying to change, retrieve qps */
 	mutex_lock(&dev->dev_lock);
-	/* syncronize with wqe, rqe posting and cqe processing contexts */
+	/* synchronize with wqe, rqe posting and cqe processing contexts */
 	spin_lock_irqsave(&qp->q_lock, flags);
 	old_qps = get_ibqp_state(qp->state);
 	if (attr_mask & IB_QP_STATE)
@@ -1599,7 +1599,7 @@ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
 	 */
 
 	cur_getp = cq->getp;
-	/* find upto when do we reap the cq. */
+	/* find up to when do we reap the cq. */
 	stop_getp = cur_getp;
 	do {
 		if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 012a0ab98d6be1..acab3cb98caadc 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -775,16 +775,23 @@ static inline int qedr_init_user_queue(struct ib_udata *udata,
 				       struct qedr_userq *q, u64 buf_addr,
 				       size_t buf_len, bool requires_db_rec,
 				       int access,
-				       int alloc_and_init)
+				       int alloc_and_init, bool is_cq)
 {
 	u32 fw_pages;
 	int rc;
 
 	q->buf_addr = buf_addr;
 	q->buf_len = buf_len;
-	q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, access);
+	if (is_cq)
+		q->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
+						   q->buf_addr, q->buf_len,
+						   access);
+	else
+		q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len,
+					 access);
 	if (IS_ERR(q->umem)) {
-		DP_ERR(dev, "create user queue: failed ib_umem_get_va, got %ld\n",
+		DP_ERR(dev, "create user queue: failed %s, got %ld\n",
+		       is_cq ? "ib_umem_get_cq_buf_or_va" : "ib_umem_get_va",
 		       PTR_ERR(q->umem));
 		return PTR_ERR(q->umem);
 	}
@@ -946,7 +953,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 		rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 					  ureq.len, true, IB_ACCESS_LOCAL_WRITE,
-					  1);
+					  1, true);
 		if (rc)
 			goto err0;
 
@@ -1075,7 +1082,7 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 	/* We don't want the IRQ handler to handle a non-existing CQ so we
 	 * wait until all CNQ interrupts, if any, are received. This will always
 	 * happen and will always happen very fast. If not, then a serious error
-	 * has occured. That is why we can use a long delay.
+	 * has occurred. That is why we can use a long delay.
 	 * We spin for a short time so we don’t lose time on context switching
 	 * in case all the completions are handled in that span. Otherwise
 	 * we sleep for a while and check again. Since the CNQ may be
@@ -1438,7 +1445,7 @@ static int qedr_init_srq_user_params(struct ib_udata *udata,
 	int rc;
 
 	rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
-				  ureq->srq_len, false, access, 1);
+				  ureq->srq_len, false, access, 1, false);
 	if (rc)
 		return rc;
 
@@ -1831,7 +1838,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
 	if (qedr_qp_has_sq(qp)) {
 		/* SQ - read access only (0) */
 		rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
-					  ureq.sq_len, true, 0, alloc_and_init);
+					  ureq.sq_len, true, 0, alloc_and_init,
+					  false);
 		if (rc)
 			return rc;
 	}
@@ -1839,7 +1847,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
 	if (qedr_qp_has_rq(qp)) {
 		/* RQ - read access only (0) */
 		rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
-					  ureq.rq_len, true, 0, alloc_and_init);
+					  ureq.rq_len, true, 0, alloc_and_init,
+					  false);
 		if (rc) {
 			ib_umem_release(qp->usq.umem);
 			qp->usq.umem = NULL;
diff --git a/drivers/infiniband/hw/usnic/usnic_abi.h b/drivers/infiniband/hw/usnic/usnic_abi.h
index 86a82a4da0aa91..a684bc54f3d479 100644
--- a/drivers/infiniband/hw/usnic/usnic_abi.h
+++ b/drivers/infiniband/hw/usnic/usnic_abi.h
@@ -72,7 +72,7 @@ struct usnic_ib_create_qp_resp {
 	u64				bar_bus_addr;
 	u32				bar_len;
 /*
- * WQ, RQ, CQ are explicitly specified bc exposing a generic resources inteface
+ * WQ, RQ, CQ are explicitly specified bc exposing a generic resources interface
  * expands the scope of ABI to many files.
  */
 	u32				wq_cnt;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index 1a1647d0e345f4..ebba3027cf485b 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -181,7 +181,7 @@ find_free_vf_and_create_qp_grp(struct ib_qp *qp,
 	}
 
 	if (usnic_ib_share_vf) {
-		/* Try to find resouces on a used vf which is in pd */
+		/* Try to find resources on a used vf which is in pd */
 		dev_list = usnic_uiom_get_dev_list(pd->umem_pd);
 		if (IS_ERR(dev_list))
 			return PTR_ERR(dev_list);
diff --git a/drivers/infiniband/hw/usnic/usnic_transport.h b/drivers/infiniband/hw/usnic/usnic_transport.h
index 9a7a2d9755c021..c963c92218d365 100644
--- a/drivers/infiniband/hw/usnic/usnic_transport.h
+++ b/drivers/infiniband/hw/usnic/usnic_transport.h
@@ -50,7 +50,7 @@ int usnic_transport_sock_to_str(char *buf, int buf_sz,
 u16 usnic_transport_rsrv_port(enum usnic_transport_type type, u16 port_num);
 void usnic_transport_unrsrv_port(enum usnic_transport_type type, u16 port_num);
 /*
- * Do a fget on the socket refered to by sock_fd and returns the socket.
+ * Do a fget on the socket referred to by sock_fd and returns the socket.
  * Socket will not be destroyed before usnic_transport_put_socket has
  * been called.
  */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
index e939cd5ce40b09..53cc49f2b8c9fe 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
@@ -270,7 +270,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 				/* set qp->sq.wqe_cnt, shift, buf_size.. */
 				qp->rumem = ib_umem_get_va(ibqp->device,
 							   ucmd.rbuf_addr,
-							   ucmd.rbuf_size, 0);
+							   ucmd.rbuf_size,
+							   IB_ACCESS_LOCAL_WRITE);
 				if (IS_ERR(qp->rumem)) {
 					ret = PTR_ERR(qp->rumem);
 					goto err_qp;
@@ -282,7 +283,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 			}
 
 			qp->sumem = ib_umem_get_va(ibqp->device, ucmd.sbuf_addr,
-						   ucmd.sbuf_size, 0);
+						   ucmd.sbuf_size,
+						   IB_ACCESS_LOCAL_WRITE);
 			if (IS_ERR(qp->sumem)) {
 				if (!is_srq)
 					ib_umem_release(qp->rumem);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
index 345ec486a223ae..3252c2eb405a1b 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
@@ -146,7 +146,8 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
 	if (ret)
 		goto err_srq;
 
-	srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0);
+	srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size,
+				   IB_ACCESS_LOCAL_WRITE);
 	if (IS_ERR(srq->umem)) {
 		ret = PTR_ERR(srq->umem);
 		goto err_srq;
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index c40cce69e9457a..d8ee485c49a6b4 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -814,7 +814,7 @@ int rvt_alloc_rq(struct rvt_rq *rq, u32 size, int node,
  *
  * This function is called from both rvt_create_qp() and
  * rvt_reset_qp().   The difference is that the reset
- * patch the necessary locks to protect against concurent
+ * patch the necessary locks to protect against concurrent
  * access.
  */
 static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index d022aa56c5bfd5..8122b46ed4ab19 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -286,7 +286,7 @@ int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 }
 
 /**
- * rvt_destroy_srq - destory an srq
+ * rvt_destroy_srq - destroy an srq
  * @ibsrq: srq object to destroy
  * @udata: user data for libibverbs.so
  */
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 64d636bf80fd2c..7c3cc48e845c45 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -92,6 +92,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem);
 /* rxe_net.c */
 struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 				int paylen, struct rxe_pkt_info *pkt);
+void rxe_put_skb(struct sk_buff *skb);
 int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt,
 		struct sk_buff *skb);
 int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 71d9ea47728909..615da4bb9a38c6 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -796,6 +796,12 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 		return -EINVAL;
 	}
 
+	/* an MR with no umem is never an ODP MR */
+	if (unlikely(access & IB_ACCESS_ON_DEMAND)) {
+		rxe_dbg_mr(mr, "access = 0x%x requests ODP\n", access);
+		return -EINVAL;
+	}
+
 	mr->access = access;
 	mr->lkey = key;
 	mr->rkey = key;
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 53daaf4c1eb2e0..9f1c7f21d88d9f 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -429,6 +429,29 @@ int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt,
 	return err;
 }
 
+/*
+ * skb->dev may be rewritten on the TX path (VLAN/bond/tunnel, etc.).
+ * The netdev we held in rxe_init_packet() is kept in destructor_arg so
+ * that put always matches hold. skb->cb cannot be used: it is already
+ * occupied by struct rxe_pkt_info and rewritten by IP control blocks.
+ */
+static void rxe_skb_set_held_ndev(struct sk_buff *skb, struct net_device *ndev)
+{
+	dev_hold(ndev);
+	skb->dev = ndev;
+	skb_shinfo(skb)->destructor_arg = ndev;
+}
+
+static void rxe_skb_put_held_ndev(struct sk_buff *skb)
+{
+	struct net_device *ndev = skb_shinfo(skb)->destructor_arg;
+
+	if (ndev) {
+		skb_shinfo(skb)->destructor_arg = NULL;
+		dev_put(ndev);
+	}
+}
+
 static void rxe_skb_tx_dtor(struct sk_buff *skb)
 {
 	struct rxe_qp *qp = skb->sk->sk_user_data;
@@ -441,6 +464,18 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb)
 
 	rxe_put(qp);
 	sock_put(skb->sk);
+	rxe_skb_put_held_ndev(skb);
+}
+
+/*
+ * Free an skb that still holds a netdev reference from rxe_init_packet()
+ * and does not yet have rxe_skb_tx_dtor() installed. Once the TX
+ * destructor is set, callers must use kfree_skb() instead.
+ */
+void rxe_put_skb(struct sk_buff *skb)
+{
+	rxe_skb_put_held_ndev(skb);
+	kfree_skb(skb);
 }
 
 static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt)
@@ -529,7 +564,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	goto done;
 
 drop:
-	kfree_skb(skb);
+	rxe_put_skb(skb);
 	err = 0;
 done:
 	return err;
@@ -574,8 +609,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 
 	skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(ndev));
 
-	/* FIXME: hold reference to this netdev until life of this skb. */
-	skb->dev	= ndev;
+	rxe_skb_set_held_ndev(skb, ndev);
 	rcu_read_unlock();
 
 	if (av->network_type == RXE_NETWORK_TYPE_IPV4)
@@ -638,9 +672,8 @@ static void rxe_sock_put(struct sock *sk,
 	if (refcount_read(&sk->sk_refcnt) > SK_REF_FOR_TUNNEL) {
 		__sock_put(sk);
 	} else {
+		set_sk(net, NULL);
 		rxe_release_udp_tunnel(sk);
-		sk = NULL;
-		set_sk(net, sk);
 	}
 }
 
@@ -710,6 +743,28 @@ void rxe_set_port_state(struct rxe_dev *rxe)
 	dev_put(ndev);
 }
 
+/*
+ * Move all QPs to the error state so pending send/recv work is drained and
+ * in-flight TX skbs (which hold a netdev reference) can be released. Called
+ * from the netdev notifier so unregister cannot stall on held skbs.
+ */
+static void rxe_flush_qps(struct rxe_dev *rxe)
+{
+	struct rxe_pool_elem *elem;
+	struct rxe_qp *qp;
+	unsigned long index;
+
+	rcu_read_lock();
+	xa_for_each(&rxe->qp_pool.xa, index, elem) {
+		if (!elem || !kref_get_unless_zero(&elem->ref_cnt))
+			continue;
+		qp = elem->obj;
+		rxe_qp_error(qp);
+		rxe_put(qp);
+	}
+	rcu_read_unlock();
+}
+
 static int rxe_notify(struct notifier_block *not_blk,
 		      unsigned long event,
 		      void *arg)
@@ -721,7 +776,12 @@ static int rxe_notify(struct notifier_block *not_blk,
 		return NOTIFY_OK;
 
 	switch (event) {
+	case NETDEV_GOING_DOWN:
+		/* Start draining TX queues before the netdev disappears. */
+		rxe_flush_qps(rxe);
+		break;
 	case NETDEV_UNREGISTER:
+		rxe_flush_qps(rxe);
 		ib_unregister_device_queued(&rxe->ib_dev);
 		rxe_net_del(&rxe->ib_dev);
 		break;
@@ -735,7 +795,6 @@ static int rxe_notify(struct notifier_block *not_blk,
 			rxe_counter_inc(rxe, RXE_CNT_LINK_DOWNED);
 		break;
 	case NETDEV_REBOOT:
-	case NETDEV_GOING_DOWN:
 	case NETDEV_CHANGEADDR:
 	case NETDEV_CHANGENAME:
 	case NETDEV_FEAT_CHANGE:
diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c
index ab21b620e94ca4..c375d3efd99927 100644
--- a/drivers/infiniband/sw/rxe/rxe_odp.c
+++ b/drivers/infiniband/sw/rxe/rxe_odp.c
@@ -469,7 +469,7 @@ static int rxe_ib_prefetch_sg_list(struct ib_pd *ibpd,
 		struct rxe_mr *mr;
 		struct ib_umem_odp *umem_odp;
 
-		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE,
+		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND,
 			       sg_list[i].lkey, RXE_LOOKUP_LOCAL);
 
 		if (!mr) {
@@ -535,7 +535,7 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd,
 
 	for (i = 0; i < num_sge; ++i) {
 		/* Takes a reference, which will be released in the queued work */
-		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE,
+		mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND,
 			       sg_list[i].lkey, RXE_LOOKUP_LOCAL);
 		if (!mr) {
 			mr = ERR_PTR(-EINVAL);
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 24f5c044363f77..53f7c1e7fe55b3 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -811,7 +811,7 @@ int rxe_requester(struct rxe_qp *qp)
 			wqe->status = IB_WC_LOC_PROT_ERR;
 		else
 			wqe->status = IB_WC_LOC_QP_OP_ERR;
-		kfree_skb(skb);
+		rxe_put_skb(skb);
 		if (ah)
 			rxe_put(ah);
 		goto err;
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 02b16e2b49b8f9..ba3b2d23b36b71 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -288,6 +288,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
 	}
 	size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge);
 	memcpy(&qp->resp.srq_wqe, wqe, size);
+	qp->resp.srq_wqe.wqe.dma.num_sge = num_sge;
 
 	qp->resp.wqe = &qp->resp.srq_wqe.wqe;
 	queue_advance_consumer(q, QUEUE_TYPE_FROM_CLIENT);
@@ -328,6 +329,7 @@ static enum resp_states rxe_get_recv_wqe(struct rxe_qp *qp)
 	}
 	size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge);
 	memcpy(&qp->resp.srq_wqe, wqe, size);
+	qp->resp.srq_wqe.wqe.dma.num_sge = num_sge;
 
 	qp->resp.wqe = &qp->resp.srq_wqe.wqe;
 	return RESPST_CHK_LENGTH;
@@ -866,7 +868,7 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,
 
 	err = rxe_prepare(&qp->pri_av, ack, skb);
 	if (err) {
-		kfree_skb(skb);
+		rxe_put_skb(skb);
 		return NULL;
 	}
 
@@ -994,7 +996,7 @@ static enum resp_states read_reply(struct rxe_qp *qp,
 	err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt),
 			  payload, RXE_FROM_MR_OBJ);
 	if (err) {
-		kfree_skb(skb);
+		rxe_put_skb(skb);
 		state = RESPST_ERR_RKEY_VIOLATION;
 		goto err_out;
 	}
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 3864284522ebc1..46d0810ea2a7a1 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -1337,6 +1337,12 @@ static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags,
 		return ERR_PTR(-EOPNOTSUPP);
 	}
 
+	if ((flags & IB_MR_REREG_ACCESS) &&
+	    ((access ^ mr->access) & IB_ACCESS_ON_DEMAND)) {
+		rxe_err_mr(mr, "cannot change IB_ACCESS_ON_DEMAND\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
 	if (flags & IB_MR_REREG_PD) {
 		rxe_put(old_pd);
 		rxe_get(pd);
diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c
index 9735b75ac933a4..c08796e4262aef 100644
--- a/drivers/infiniband/sw/siw/siw_main.c
+++ b/drivers/infiniband/sw/siw/siw_main.c
@@ -217,8 +217,11 @@ static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id)
 		/*
 		 * siw_qp_id2obj() increments object reference count
 		 */
+		struct ib_qp *base_qp = &qp->base_qp;
+
 		siw_qp_put(qp);
-		return &qp->base_qp;
+
+		return base_qp;
 	}
 	return NULL;
 }
diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c
index f7dd32c6e5ba4d..56bd009879c228 100644
--- a/drivers/infiniband/sw/siw/siw_qp_tx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_tx.c
@@ -336,7 +336,6 @@ static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset,
 		if (size + offset <= PAGE_SIZE)
 			msg.msg_flags &= ~MSG_MORE;
 
-		tcp_rate_check_app_limited(sk);
 		if (!sendpage_ok(page[i]))
 			msg.msg_flags &= ~MSG_SPLICE_PAGES;
 		bvec_set_page(&bvec, page[i], bytes, offset);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 6c14246befb1f9..dfee72c8e46a01 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1378,7 +1378,8 @@ struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)
 	return neigh;
 }
 
-static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
+static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv,
+			       int gc_interval)
 {
 	struct ipoib_neigh_table *ntbl = &priv->ntbl;
 	struct ipoib_neigh_hash *htbl;
@@ -1397,7 +1398,7 @@ static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
 		goto out_unlock;
 
 	/* neigh is obsolete if it was idle for two GC periods */
-	dt = 2 * arp_tbl.gc_interval;
+	dt = 2 * gc_interval;
 	neigh_obsolete = jiffies - dt;
 
 	for (i = 0; i < htbl->size; i++) {
@@ -1433,11 +1434,16 @@ static void ipoib_reap_neigh(struct work_struct *work)
 {
 	struct ipoib_dev_priv *priv =
 		container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);
+	struct net_device *dev = priv->dev;
+	struct net *net = dev_net(dev);
+	struct neigh_table *tbl;
+	int gc_interval;
 
-	__ipoib_reap_neigh(priv);
+	tbl = arp_table(net);
+	gc_interval = tbl->gc_interval;
+	__ipoib_reap_neigh(priv, gc_interval);
 
-	queue_delayed_work(priv->wq, &priv->neigh_reap_task,
-			   arp_tbl.gc_interval);
+	queue_delayed_work(priv->wq, &priv->neigh_reap_task, gc_interval);
 }
 
 
@@ -1590,8 +1596,11 @@ void ipoib_neigh_free(struct ipoib_neigh *neigh)
 static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
 {
 	struct ipoib_neigh_table *ntbl = &priv->ntbl;
-	struct ipoib_neigh_hash *htbl;
 	struct ipoib_neigh __rcu **buckets;
+	struct net_device *dev = priv->dev;
+	struct net *net = dev_net(dev);
+	struct ipoib_neigh_hash *htbl;
+	struct neigh_table *tbl;
 	u32 size;
 
 	clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
@@ -1599,7 +1608,9 @@ static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
 	htbl = kzalloc_obj(*htbl);
 	if (!htbl)
 		return -ENOMEM;
-	size = roundup_pow_of_two(arp_tbl.gc_thresh3);
+
+	tbl = arp_table(net);
+	size = roundup_pow_of_two(tbl->gc_thresh3);
 	buckets = kvzalloc_objs(*buckets, size);
 	if (!buckets) {
 		kfree(htbl);
@@ -1614,7 +1625,7 @@ static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
 
 	/* start garbage collection */
 	queue_delayed_work(priv->wq, &priv->neigh_reap_task,
-			   arp_tbl.gc_interval);
+			   tbl->gc_interval);
 
 	return 0;
 }
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
index ba3b45152907ef..697cad0d063d89 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
@@ -140,7 +140,7 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv,
 		goto out_early;
 	}
 
-	/* RTNL childs don't need proprietary sysfs entries */
+	/* RTNL children don't need proprietary sysfs entries */
 	if (type == IPOIB_LEGACY_CHILD) {
 		if (ipoib_cm_add_mode_attr(ndev))
 			goto sysfs_failed;
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 7df44168578034..31e3a8ab6c4818 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -498,7 +498,7 @@ static int iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
  * iscsi_iser_conn_start() - start iscsi-iser connection
  * @cls_conn: iscsi class connection
  *
- * Notes: Here iser intialize (or re-initialize) stop_completion as
+ * Notes: Here iser initialize (or re-initialize) stop_completion as
  *        from this point iscsi must call conn_stop in session/connection
  *        teardown so iser transport must wait for it.
  */
@@ -593,7 +593,7 @@ static inline unsigned int iser_dif_prot_caps(int prot_caps)
  * @qdepth:         session command queue depth
  * @initial_cmdsn:  initiator command sequnce number
  *
- * Allocates and adds a scsi host, expose DIF supprot if
+ * Allocates and adds a scsi host, expose DIF support if
  * exists, and sets up an iscsi session.
  */
 static struct iscsi_cls_session *
@@ -866,7 +866,7 @@ static int iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
  * @ep:    iscsi endpoint handle
  *
  * This routine is not blocked by iser and RDMA termination process
- * completion as we queue a deffered work for iser/RDMA destruction
+ * completion as we queue a deferred work for iser/RDMA destruction
  * and cleanup or actually call it immediately in case we didn't pass
  * iscsi conn bind/start stage, thus it is safe.
  */
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 55fe68e5b8375e..ab2a27d23478a9 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -347,7 +347,7 @@ void iser_release_work(struct work_struct *work)
 
 	/* Wait for conn_stop to complete */
 	wait_for_completion(&iser_conn->stop_completion);
-	/* Wait for IB resouces cleanup to complete */
+	/* Wait for IB resources cleanup to complete */
 	wait_for_completion(&iser_conn->ib_completion);
 
 	mutex_lock(&iser_conn->state_mutex);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index e69db43370ff93..efa8d7b7844c97 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -2583,7 +2583,7 @@ static void
 isert_put_unsol_pending_cmds(struct iscsit_conn *conn)
 {
 	struct iscsit_cmd *cmd, *tmp;
-	static LIST_HEAD(drop_cmd_list);
+	LIST_HEAD(drop_cmd_list);
 
 	spin_lock_bh(&conn->cmd_lock);
 	list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index eac38b57b00d7f..3f34cf2b7294eb 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -70,19 +70,24 @@ __rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type)
 {
 	size_t max_depth = clt->queue_depth;
 	struct rtrs_permit *permit;
-	int bit;
+	unsigned long bit = 0;
 
 	/*
-	 * Adapted from null_blk get_tag(). Callers from different cpus may
-	 * grab the same bit, since find_first_zero_bit is not atomic.
-	 * But then the test_and_set_bit_lock will fail for all the
-	 * callers but one, so that they will loop again.
-	 * This way an explicit spinlock is not required.
+	 * Callers from different CPUs may grab the same bit, since the bitmap
+	 * scan is not atomic. But then the test_and_set_bit_lock() will fail
+	 * for all the callers but one, so that they loop again. This way an
+	 * explicit spinlock is not required. find_next_zero_bit() resumes
+	 * from the last position so that a lost race does not rescan the
+	 * already-set low bits; if it reaches the end, wrap to the beginning
+	 * to exhaust the map and still find a permit freed below the cursor.
 	 */
 	do {
-		bit = find_first_zero_bit(clt->permits_map, max_depth);
-		if (bit >= max_depth)
-			return NULL;
+		bit = find_next_zero_bit(clt->permits_map, max_depth, bit);
+		if (bit >= max_depth) {
+			bit = find_first_zero_bit(clt->permits_map, max_depth);
+			if (bit >= max_depth)
+				return NULL;
+		}
 	} while (test_and_set_bit_lock(bit, clt->permits_map));
 
 	permit = get_permit(clt, bit);
@@ -1853,6 +1858,10 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con,
 	}
 	if (con->c.cid == 0) {
 		queue_depth = le16_to_cpu(msg->queue_depth);
+		if (!queue_depth) {
+			rtrs_err(clt, "Invalid queue depth %u\n", queue_depth);
+			return -ECONNRESET;
+		}
 
 		if (clt_path->queue_depth > 0 && queue_depth != clt_path->queue_depth) {
 			rtrs_err(clt, "Error: queue depth changed\n");
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 7197d95f2216b0..43c3ff1ad8e7ed 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -1888,7 +1888,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	 * We divide up our send queue size into half SEND WRs to send the
 	 * completions, and half R/W contexts to actually do the RDMA
 	 * READ/WRITE transfers.  Note that we need to allocate CQ slots for
-	 * both both, as RDMA contexts will also post completions for the
+	 * both, as RDMA contexts will also post completions for the
 	 * RDMA READ case.
 	 */
 	qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index 4c46b301eea186..506c11b843593b 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -350,7 +350,7 @@ struct srpt_nexus {
  * struct srpt_port_attrib - attributes for SRPT port
  * @srp_max_rdma_size: Maximum size of SRP RDMA transfers for new connections.
  * @srp_max_rsp_size: Maximum size of SRP response messages in bytes.
- * @srp_sq_size: Shared receive queue (SRQ) size.
+ * @srp_sq_size: Send queue size.
  * @use_srq: Whether or not to use SRQ.
  */
 struct srpt_port_attrib {
diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c
index 3a718d6000063e..8bfaaa45e0b951 100644
--- a/drivers/input/evdev.c
+++ b/drivers/input/evdev.c
@@ -1229,6 +1229,8 @@ static long evdev_do_ioctl(struct file *file, unsigned int cmd,
 
 			t = _IOC_NR(cmd) & ABS_MAX;
 
+			memset(&abs, 0, sizeof(abs));
+
 			if (copy_from_user(&abs, p, min_t(size_t,
 					size, sizeof(struct input_absinfo))))
 				return -EFAULT;
diff --git a/drivers/input/input-compat.c b/drivers/input/input-compat.c
index a5043193ead85a..8860a073029427 100644
--- a/drivers/input/input-compat.c
+++ b/drivers/input/input-compat.c
@@ -76,6 +76,8 @@ int input_ff_effect_from_user(const char __user *buffer, size_t size,
 		 */
 		compat_effect = (struct ff_effect_compat *)effect;
 
+		memset(effect, 0, sizeof(*effect));
+
 		if (copy_from_user(compat_effect, buffer,
 				   sizeof(struct ff_effect_compat)))
 			return -EFAULT;
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index 2da0b7f1722aae..6406ceab988ade 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -215,7 +215,7 @@ static const struct xpad_device {
 	{ 0x0e6f, 0x0139, "Afterglow Prismatic Wired Controller", 0, XTYPE_XBOXONE },
 	{ 0x0e6f, 0x013a, "PDP Xbox One Controller", 0, XTYPE_XBOXONE },
 	{ 0x0e6f, 0x0146, "Rock Candy Wired Controller for Xbox One", 0, XTYPE_XBOXONE },
-	{ 0x0e6f, 0x0147, "PDP Marvel Xbox One Controller", 0, XTYPE_XBOXONE },
+	{ 0x0e6f, 0x0147, "PDP Marvel Xbox 360 Controller", 0, XTYPE_XBOX360 },
 	{ 0x0e6f, 0x015c, "PDP Xbox One Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE },
 	{ 0x0e6f, 0x015d, "PDP Mirror's Edge Official Wired Controller for Xbox One", 0, XTYPE_XBOXONE },
 	{ 0x0e6f, 0x0161, "PDP Xbox One Controller", 0, XTYPE_XBOXONE },
@@ -227,6 +227,7 @@ static const struct xpad_device {
 	{ 0x0e6f, 0x0213, "Afterglow Gamepad for Xbox 360", 0, XTYPE_XBOX360 },
 	{ 0x0e6f, 0x021f, "Rock Candy Gamepad for Xbox 360", 0, XTYPE_XBOX360 },
 	{ 0x0e6f, 0x0246, "Rock Candy Gamepad for Xbox One 2015", 0, XTYPE_XBOXONE },
+	{ 0x0e6f, 0x024c, "PDP Victrix Pro BFG Wired Controller for Xbox", 0, XTYPE_XBOXONE },
 	{ 0x0e6f, 0x02a0, "PDP Xbox One Controller", 0, XTYPE_XBOXONE },
 	{ 0x0e6f, 0x02a1, "PDP Xbox One Controller", 0, XTYPE_XBOXONE },
 	{ 0x0e6f, 0x02a2, "PDP Wired Controller for Xbox One - Crimson Red", 0, XTYPE_XBOXONE },
@@ -292,6 +293,12 @@ static const struct xpad_device {
 	{ 0x1689, 0xfd00, "Razer Onza Tournament Edition", 0, XTYPE_XBOX360 },
 	{ 0x1689, 0xfd01, "Razer Onza Classic Edition", 0, XTYPE_XBOX360 },
 	{ 0x1689, 0xfe00, "Razer Sabertooth", 0, XTYPE_XBOX360 },
+	{ 0x16d0, 0x1103, "Azeron Cyro", 0, XTYPE_XBOX360 },
+	{ 0x16d0, 0x113c, "Azeron Cyborg", 0, XTYPE_XBOX360 },
+	{ 0x16d0, 0x1192, "Azeron Classic/Compact", 0, XTYPE_XBOX360 },
+	{ 0x16d0, 0x1212, "Azeron Cyro Lefty", 0, XTYPE_XBOX360 },
+	{ 0x16d0, 0x12f7, "Azeron Cyborg II", 0, XTYPE_XBOX360 },
+	{ 0x16d0, 0x13ea, "Azeron Keyzen", 0, XTYPE_XBOX360 },
 	{ 0x17ef, 0x6182, "Lenovo Legion Controller for Windows", 0, XTYPE_XBOX360 },
 	{ 0x1949, 0x041a, "Amazon Game Controller", 0, XTYPE_XBOX360 },
 	{ 0x1a86, 0xe310, "Legion Go S", 0, XTYPE_XBOX360 },
@@ -534,6 +541,7 @@ static const struct usb_device_id xpad_table[] = {
 	XPAD_XBOX360_VENDOR(0x15e4),		/* Numark Xbox 360 controllers */
 	XPAD_XBOX360_VENDOR(0x162e),		/* Joytech Xbox 360 controllers */
 	XPAD_XBOX360_VENDOR(0x1689),		/* Razer Onza */
+	XPAD_XBOX360_VENDOR(0x16d0),		/* Azeron controllers */
 	XPAD_XBOX360_VENDOR(0x17ef),		/* Lenovo */
 	XPAD_XBOX360_VENDOR(0x1949),		/* Amazon controllers */
 	XPAD_XBOX360_VENDOR(0x1a86),		/* Nanjing Qinheng Microelectronics (WCH) */
diff --git a/drivers/input/keyboard/adp5588-keys.c b/drivers/input/keyboard/adp5588-keys.c
index 40371f5bd9bac0..4f0ddff5babae6 100644
--- a/drivers/input/keyboard/adp5588-keys.c
+++ b/drivers/input/keyboard/adp5588-keys.c
@@ -446,12 +446,6 @@ static int adp5588_gpio_add(struct adp5588_kpad *kpad)
 
 	mutex_init(&kpad->gpio_lock);
 
-	error = devm_gpiochip_add_data(dev, &kpad->gc, kpad);
-	if (error) {
-		dev_err(dev, "gpiochip_add failed: %d\n", error);
-		return error;
-	}
-
 	for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) {
 		kpad->dat_out[i] = adp5588_read(kpad->client,
 						GPIO_DAT_OUT1 + i);
@@ -459,6 +453,12 @@ static int adp5588_gpio_add(struct adp5588_kpad *kpad)
 		kpad->pull_dis[i] = adp5588_read(kpad->client, GPIO_PULL1 + i);
 	}
 
+	error = devm_gpiochip_add_data(dev, &kpad->gc, kpad);
+	if (error) {
+		dev_err(dev, "gpiochip_add failed: %d\n", error);
+		return error;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c
index b9ad2381f885de..93650c98206d6b 100644
--- a/drivers/input/keyboard/atkbd.c
+++ b/drivers/input/keyboard/atkbd.c
@@ -1946,6 +1946,14 @@ static const struct dmi_system_id atkbd_dmi_quirk_table[] __initconst = {
 		},
 		.callback = atkbd_deactivate_fixup,
 	},
+	{
+		/* Xiaomi Redmi Book Pro 16 2026 (TM2425) */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "XIAOMI"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "REDMI Book Pro 16 2026"),
+		},
+		.callback = atkbd_deactivate_fixup,
+	},
 	{ }
 };
 
diff --git a/drivers/input/keyboard/samsung-keypad.c b/drivers/input/keyboard/samsung-keypad.c
index deffa98749c0c3..f56c5249401de8 100644
--- a/drivers/input/keyboard/samsung-keypad.c
+++ b/drivers/input/keyboard/samsung-keypad.c
@@ -12,6 +12,7 @@
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/input.h>
+#include <linux/input/matrix_keypad.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
@@ -21,7 +22,9 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/sched.h>
-#include <linux/input/samsung-keypad.h>
+
+#define SAMSUNG_MAX_ROWS			8
+#define SAMSUNG_MAX_COLS			8
 
 #define SAMSUNG_KEYIFCON			0x00
 #define SAMSUNG_KEYIFSTSCLR			0x04
@@ -221,145 +224,92 @@ static void samsung_keypad_close(struct input_dev *input_dev)
 	samsung_keypad_stop(keypad);
 }
 
-#ifdef CONFIG_OF
-static struct samsung_keypad_platdata *
-samsung_keypad_parse_dt(struct device *dev)
+static int samsung_parse_verbose_keymap(struct samsung_keypad *keypad)
 {
-	struct samsung_keypad_platdata *pdata;
-	struct matrix_keymap_data *keymap_data;
-	uint32_t *keymap, num_rows = 0, num_cols = 0;
-	struct device_node *np = dev->of_node, *key_np;
-	unsigned int key_count;
-
-	if (!np) {
-		dev_err(dev, "missing device tree data\n");
-		return ERR_PTR(-EINVAL);
-	}
-
-	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata) {
-		dev_err(dev, "could not allocate memory for platform data\n");
-		return ERR_PTR(-ENOMEM);
-	}
+	struct matrix_keymap_data keymap_data = { 0 };
+	struct device *dev = &keypad->pdev->dev;
+	struct fwnode_handle *child;
+	unsigned int key_count = device_get_child_node_count(dev);
+	unsigned int i;
+
+	u32 *keymap __free(kfree) = kcalloc(key_count, sizeof(*keymap), GFP_KERNEL);
+	if (!keymap)
+		return -ENOMEM;
 
-	of_property_read_u32(np, "samsung,keypad-num-rows", &num_rows);
-	of_property_read_u32(np, "samsung,keypad-num-columns", &num_cols);
-	if (!num_rows || !num_cols) {
-		dev_err(dev, "number of keypad rows/columns not specified\n");
-		return ERR_PTR(-EINVAL);
-	}
-	pdata->rows = num_rows;
-	pdata->cols = num_cols;
+	keymap_data.keymap_size = key_count;
+	keymap_data.keymap = keymap;
 
-	keymap_data = devm_kzalloc(dev, sizeof(*keymap_data), GFP_KERNEL);
-	if (!keymap_data) {
-		dev_err(dev, "could not allocate memory for keymap data\n");
-		return ERR_PTR(-ENOMEM);
-	}
-	pdata->keymap_data = keymap_data;
-
-	key_count = of_get_child_count(np);
-	keymap_data->keymap_size = key_count;
-	keymap = devm_kcalloc(dev, key_count, sizeof(uint32_t), GFP_KERNEL);
-	if (!keymap) {
-		dev_err(dev, "could not allocate memory for keymap\n");
-		return ERR_PTR(-ENOMEM);
-	}
-	keymap_data->keymap = keymap;
-
-	for_each_child_of_node(np, key_np) {
+	i = 0;
+	device_for_each_child_node(dev, child) {
 		u32 row, col, key_code;
-		of_property_read_u32(key_np, "keypad,row", &row);
-		of_property_read_u32(key_np, "keypad,column", &col);
-		of_property_read_u32(key_np, "linux,code", &key_code);
-		*keymap++ = KEY(row, col, key_code);
-	}
 
-	pdata->no_autorepeat = of_property_read_bool(np, "linux,input-no-autorepeat");
-
-	pdata->wakeup = of_property_read_bool(np, "wakeup-source") ||
-			/* legacy name */
-			of_property_read_bool(np, "linux,input-wakeup");
+		fwnode_property_read_u32(child, "keypad,row", &row);
+		fwnode_property_read_u32(child, "keypad,column", &col);
+		fwnode_property_read_u32(child, "linux,code", &key_code);
 
+		keymap[i++] = KEY(row, col, key_code);
+	}
 
-	return pdata;
+	return matrix_keypad_build_keymap(&keymap_data, NULL,
+					  keypad->rows, keypad->cols,
+					  keypad->keycodes,
+					  keypad->input_dev);
 }
-#else
-static struct samsung_keypad_platdata *
-samsung_keypad_parse_dt(struct device *dev)
+
+static int samsung_keypad_parse_keymap(struct samsung_keypad *keypad)
 {
-	dev_err(dev, "no platform data defined\n");
+	struct device *dev = &keypad->pdev->dev;
+
+	if (device_property_present(dev, "linux,keymap"))
+		return matrix_keypad_build_keymap(NULL, NULL,
+						  keypad->rows, keypad->cols,
+						  keypad->keycodes,
+						  keypad->input_dev);
 
-	return ERR_PTR(-EINVAL);
+	return samsung_parse_verbose_keymap(keypad);
 }
-#endif
 
 static int samsung_keypad_probe(struct platform_device *pdev)
 {
-	const struct samsung_keypad_platdata *pdata;
-	const struct matrix_keymap_data *keymap_data;
 	const struct platform_device_id *id;
+	struct device *dev = &pdev->dev;
 	struct samsung_keypad *keypad;
 	struct resource *res;
 	struct input_dev *input_dev;
 	unsigned int row_shift;
+	u32 num_rows = 0, num_cols = 0;
+	bool wakeup;
 	int error;
 
-	pdata = dev_get_platdata(&pdev->dev);
-	if (!pdata) {
-		pdata = samsung_keypad_parse_dt(&pdev->dev);
-		if (IS_ERR(pdata))
-			return PTR_ERR(pdata);
-	}
-
-	keymap_data = pdata->keymap_data;
-	if (!keymap_data) {
-		dev_err(&pdev->dev, "no keymap data defined\n");
-		return -EINVAL;
+	if (device_property_present(dev, "linux,keymap")) {
+		error = matrix_keypad_parse_properties(dev, &num_rows, &num_cols);
+		if (error)
+			return error;
+	} else {
+		device_property_read_u32(dev, "samsung,keypad-num-rows", &num_rows);
+		device_property_read_u32(dev, "samsung,keypad-num-columns", &num_cols);
+		if (!num_rows || !num_cols) {
+			dev_err(dev, "number of keypad rows/columns not specified\n");
+			return -EINVAL;
+		}
 	}
 
-	if (!pdata->rows || pdata->rows > SAMSUNG_MAX_ROWS)
+	if (num_rows > SAMSUNG_MAX_ROWS || num_cols > SAMSUNG_MAX_COLS)
 		return -EINVAL;
 
-	if (!pdata->cols || pdata->cols > SAMSUNG_MAX_COLS)
-		return -EINVAL;
-
-	/* initialize the gpio */
-	if (pdata->cfg_gpio)
-		pdata->cfg_gpio(pdata->rows, pdata->cols);
-
-	row_shift = get_count_order(pdata->cols);
+	row_shift = get_count_order(num_cols);
 
 	keypad = devm_kzalloc(&pdev->dev,
 			      struct_size(keypad, keycodes,
-					  pdata->rows << row_shift),
+					  num_rows << row_shift),
 			      GFP_KERNEL);
 	if (!keypad)
 		return -ENOMEM;
 
-	input_dev = devm_input_allocate_device(&pdev->dev);
-	if (!input_dev)
-		return -ENOMEM;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
-	keypad->base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
-	if (!keypad->base)
-		return -EBUSY;
-
-	keypad->clk = devm_clk_get_prepared(&pdev->dev, "keypad");
-	if (IS_ERR(keypad->clk)) {
-		dev_err(&pdev->dev, "failed to get keypad clk\n");
-		return PTR_ERR(keypad->clk);
-	}
-
-	keypad->input_dev = input_dev;
 	keypad->pdev = pdev;
 	keypad->row_shift = row_shift;
-	keypad->rows = pdata->rows;
-	keypad->cols = pdata->cols;
+	keypad->rows = num_rows;
+	keypad->cols = num_cols;
 	keypad->stopped = true;
 	init_waitqueue_head(&keypad->wait);
 
@@ -375,26 +325,45 @@ static int samsung_keypad_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	input_dev = devm_input_allocate_device(&pdev->dev);
+	if (!input_dev)
+		return -ENOMEM;
+
+	keypad->input_dev = input_dev;
+
 	input_dev->name = pdev->name;
 	input_dev->id.bustype = BUS_HOST;
 
 	input_dev->open = samsung_keypad_open;
 	input_dev->close = samsung_keypad_close;
 
-	error = matrix_keypad_build_keymap(keymap_data, NULL,
-					   pdata->rows, pdata->cols,
-					   keypad->keycodes, input_dev);
+	error = samsung_keypad_parse_keymap(keypad);
 	if (error) {
 		dev_err(&pdev->dev, "failed to build keymap\n");
 		return error;
 	}
 
 	input_set_capability(input_dev, EV_MSC, MSC_SCAN);
-	if (!pdata->no_autorepeat)
+
+	if (!device_property_read_bool(&pdev->dev, "linux,input-no-autorepeat"))
 		__set_bit(EV_REP, input_dev->evbit);
 
 	input_set_drvdata(input_dev, keypad);
 
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+
+	keypad->base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+	if (!keypad->base)
+		return -EBUSY;
+
+	keypad->clk = devm_clk_get_prepared(&pdev->dev, "keypad");
+	if (IS_ERR(keypad->clk)) {
+		dev_err(&pdev->dev, "failed to get keypad clk\n");
+		return PTR_ERR(keypad->clk);
+	}
+
 	keypad->irq = platform_get_irq(pdev, 0);
 	if (keypad->irq < 0) {
 		error = keypad->irq;
@@ -410,7 +379,11 @@ static int samsung_keypad_probe(struct platform_device *pdev)
 		return error;
 	}
 
-	device_init_wakeup(&pdev->dev, pdata->wakeup);
+	wakeup = device_property_read_bool(dev, "wakeup-source") ||
+		 /* legacy name */
+		 device_property_read_bool(dev, "linux,input-wakeup");
+	device_init_wakeup(&pdev->dev, wakeup);
+
 	platform_set_drvdata(pdev, keypad);
 
 	error = devm_pm_runtime_enable(&pdev->dev);
@@ -421,11 +394,6 @@ static int samsung_keypad_probe(struct platform_device *pdev)
 	if (error)
 		return error;
 
-	if (pdev->dev.of_node) {
-		devm_kfree(&pdev->dev, (void *)pdata->keymap_data->keymap);
-		devm_kfree(&pdev->dev, (void *)pdata->keymap_data);
-		devm_kfree(&pdev->dev, (void *)pdata);
-	}
 	return 0;
 }
 
diff --git a/drivers/input/keyboard/snvs_pwrkey.c b/drivers/input/keyboard/snvs_pwrkey.c
index 9d51c7ff1ebb05..72d43d8deb78f1 100644
--- a/drivers/input/keyboard/snvs_pwrkey.c
+++ b/drivers/input/keyboard/snvs_pwrkey.c
@@ -179,7 +179,7 @@ static int imx_snvs_pwrkey_probe(struct platform_device *pdev)
 	clk = devm_clk_get_optional_enabled(dev, NULL);
 	if (IS_ERR(clk))
 		return dev_err_probe(dev, PTR_ERR(clk),
-				     "Failed to get snvs clock (%pe)\n", clk);
+				     "Failed to get snvs clock\n");
 
 	pdata->wakeup = of_property_read_bool(np, "wakeup-source");
 
@@ -236,14 +236,13 @@ static int imx_snvs_pwrkey_probe(struct platform_device *pdev)
 	pdata->input = input;
 	platform_set_drvdata(pdev, pdata);
 
-	error = devm_request_irq(dev, pdata->irq,
-				 imx_snvs_pwrkey_interrupt,
+	error = devm_request_irq(dev, pdata->irq, imx_snvs_pwrkey_interrupt,
 				 0, pdev->name, pdev);
 	if (error)
-		return dev_err_probe(dev, error, "interrupt not available.\n");
+		return error;
 
 	error = input_register_device(input);
-	if (error < 0)
+	if (error)
 		return dev_err_probe(dev, error, "failed to register input device\n");
 
 	device_init_wakeup(dev, pdata->wakeup);
diff --git a/drivers/input/keyboard/st-keyscan.c b/drivers/input/keyboard/st-keyscan.c
index e53ef4c670e4b8..8e9bac339dc478 100644
--- a/drivers/input/keyboard/st-keyscan.c
+++ b/drivers/input/keyboard/st-keyscan.c
@@ -173,10 +173,9 @@ static int keyscan_probe(struct platform_device *pdev)
 		return PTR_ERR(keypad_data->base);
 
 	keypad_data->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(keypad_data->clk)) {
-		dev_err(&pdev->dev, "cannot get clock\n");
-		return PTR_ERR(keypad_data->clk);
-	}
+	if (IS_ERR(keypad_data->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(keypad_data->clk),
+				     "cannot get clock\n");
 
 	error = clk_enable(keypad_data->clk);
 	if (error) {
@@ -188,14 +187,12 @@ static int keyscan_probe(struct platform_device *pdev)
 
 	keypad_data->irq = platform_get_irq(pdev, 0);
 	if (keypad_data->irq < 0)
-		return -EINVAL;
+		return keypad_data->irq;
 
 	error = devm_request_irq(&pdev->dev, keypad_data->irq, keyscan_isr, 0,
 				 pdev->name, keypad_data);
-	if (error) {
-		dev_err(&pdev->dev, "failed to request IRQ\n");
+	if (error)
 		return error;
-	}
 
 	error = input_register_device(input_dev);
 	if (error) {
diff --git a/drivers/input/matrix-keymap.c b/drivers/input/matrix-keymap.c
index 3bea3575a0a941..e4227e96aa4e7d 100644
--- a/drivers/input/matrix-keymap.c
+++ b/drivers/input/matrix-keymap.c
@@ -35,6 +35,13 @@ static bool matrix_keypad_map_key(struct input_dev *input_dev,
 		return false;
 	}
 
+	if (code > KEY_MAX) {
+		dev_err(input_dev->dev.parent,
+			"%s: invalid keycode %u in keymap entry 0x%x\n",
+			__func__, code, key);
+		return false;
+	}
+
 	keymap[MATRIX_SCAN_CODE(row, col, row_shift)] = code;
 	__set_bit(code, input_dev->keybit);
 
diff --git a/drivers/input/misc/aw86927.c b/drivers/input/misc/aw86927.c
index f53b8f004cb369..1f73f063690e46 100644
--- a/drivers/input/misc/aw86927.c
+++ b/drivers/input/misc/aw86927.c
@@ -12,6 +12,7 @@
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/input.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
diff --git a/drivers/input/misc/soc_button_array.c b/drivers/input/misc/soc_button_array.c
index a6c98420512357..064243677fcffd 100644
--- a/drivers/input/misc/soc_button_array.c
+++ b/drivers/input/misc/soc_button_array.c
@@ -15,6 +15,7 @@
 #include <linux/dmi.h>
 #include <linux/gpio/consumer.h>
 #include <linux/gpio_keys.h>
+#include <linux/platform_data/x86/soc.h>
 #include <linux/platform_device.h>
 
 static bool use_low_level_irq;
@@ -159,7 +160,7 @@ soc_button_device_create(struct platform_device *pdev,
 	struct gpio_keys_platform_data *gpio_keys_pdata;
 	const struct dmi_system_id *dmi_id;
 	int invalid_acpi_index = -1;
-	int error, gpio, irq;
+	int error, gpio, irq = 0;
 	int n_buttons = 0;
 
 	for (info = button_info; info->name; info++)
@@ -190,8 +191,9 @@ soc_button_device_create(struct platform_device *pdev,
 		error = soc_button_lookup_gpio(&pdev->dev, info->acpi_index, &gpio, &irq);
 		if (error || irq < 0) {
 			/*
-			 * Skip GPIO if not present. Note we deliberately
-			 * ignore -EPROBE_DEFER errors here. On some devices
+			 * Propagate -EPROBE_DEFER, skip button on other errors.
+			 *
+			 * -EPROBE_DEFER is ignored on Bay & Cherry Trail. Here
 			 * Intel is using so called virtual GPIOs which are not
 			 * GPIOs at all but some way for AML code to check some
 			 * random status bits without need a custom opregion.
@@ -200,6 +202,12 @@ soc_button_device_create(struct platform_device *pdev,
 			 * we do not have a driver for these so they will never
 			 * show up, therefore we ignore -EPROBE_DEFER.
 			 */
+			if ((error == -EPROBE_DEFER || irq == -EPROBE_DEFER) &&
+			    !(soc_intel_is_byt() || soc_intel_is_cht())) {
+				error = -EPROBE_DEFER;
+				goto err_free_mem;
+			}
+
 			continue;
 		}
 
@@ -368,7 +376,7 @@ static struct soc_button_info *soc_button_get_button_info(struct device *dev)
 		}
 	}
 
-	if (!btns_desc) {
+	if (!btns_desc || !btns_desc->package.count) {
 		dev_err(dev, "ACPI Button Descriptors not found\n");
 		button_info = ERR_PTR(-ENODEV);
 		goto out;
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index 2170bbe4c5897f..a8ce89d41faa1a 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -1838,6 +1838,14 @@ static int synaptics_setup_intertouch(struct psmouse *psmouse,
 
 			return -ENXIO;
 		}
+
+		/* Disable intertouch on known-broken board revisions */
+		if (info->board_id == 2722) {
+			psmouse_info(psmouse,
+				     "Disabling intertouch for board id %u\n",
+				     info->board_id);
+			return -ENXIO;
+		}
 	}
 
 	psmouse_info(psmouse, "Trying to set up SMBus access\n");
diff --git a/drivers/input/rmi4/rmi_driver.c b/drivers/input/rmi4/rmi_driver.c
index 5d49a9021c7d05..a349dfd17519c5 100644
--- a/drivers/input/rmi4/rmi_driver.c
+++ b/drivers/input/rmi4/rmi_driver.c
@@ -991,6 +991,15 @@ int rmi_driver_suspend(struct rmi_device *rmi_dev, bool enable_wake)
 {
 	int retval;
 
+	/*
+	 * Transport driver will try to suspend RMI device even if physical
+	 * driver did not bind to the RMI device, because transport device
+	 * (I2C, SPI) is fully registered and operational. Exit early if
+	 * there is no driver data attached to the RMI device.
+	 */
+	if (!dev_get_drvdata(&rmi_dev->dev))
+		return 0;
+
 	retval = rmi_suspend_functions(rmi_dev);
 	if (retval)
 		dev_warn(&rmi_dev->dev, "Failed to suspend functions: %d\n",
@@ -1005,6 +1014,10 @@ int rmi_driver_resume(struct rmi_device *rmi_dev, bool clear_wake)
 {
 	int retval;
 
+	/* Skip if not fully bound to RMI driver */
+	if (!dev_get_drvdata(&rmi_dev->dev))
+		return 0;
+
 	rmi_enable_irq(rmi_dev, clear_wake);
 
 	retval = rmi_resume_functions(rmi_dev);
diff --git a/drivers/input/rmi4/rmi_smbus.c b/drivers/input/rmi4/rmi_smbus.c
index 3160714a514a55..8b61c238229768 100644
--- a/drivers/input/rmi4/rmi_smbus.c
+++ b/drivers/input/rmi4/rmi_smbus.c
@@ -140,7 +140,7 @@ static int rmi_smb_write_block(struct rmi_transport_dev *xport, u16 rmiaddr,
 	u8 commandcode;
 	struct rmi_smb_xport *rmi_smb =
 		container_of(xport, struct rmi_smb_xport, xport);
-	int cur_len = (int)len;
+	size_t cur_len = len;
 
 	mutex_lock(&rmi_smb->page_mutex);
 
@@ -148,7 +148,7 @@ static int rmi_smb_write_block(struct rmi_transport_dev *xport, u16 rmiaddr,
 		/*
 		 * break into 32 bytes chunks to write get command code
 		 */
-		int block_len = min_t(int, len, SMB_MAX_COUNT);
+		int block_len = min_t(size_t, cur_len, SMB_MAX_COUNT);
 
 		retval = rmi_smb_get_command_code(xport, rmiaddr, block_len,
 						  false, &commandcode);
@@ -161,9 +161,9 @@ static int rmi_smb_write_block(struct rmi_transport_dev *xport, u16 rmiaddr,
 			goto exit;
 
 		/* prepare to write next block of bytes */
-		cur_len -= SMB_MAX_COUNT;
-		databuff += SMB_MAX_COUNT;
-		rmiaddr += SMB_MAX_COUNT;
+		cur_len -= block_len;
+		databuff += block_len;
+		rmiaddr += block_len;
 	}
 exit:
 	mutex_unlock(&rmi_smb->page_mutex);
diff --git a/drivers/input/serio/gscps2.c b/drivers/input/serio/gscps2.c
index 43453ec533b22f..985539f1d0886a 100644
--- a/drivers/input/serio/gscps2.c
+++ b/drivers/input/serio/gscps2.c
@@ -1,6 +1,4 @@
 /*
- * drivers/input/serio/gscps2.c
- *
  * Copyright (c) 2004-2006 Helge Deller <deller@gmx.de>
  * Copyright (c) 2002 Laurent Canet <canetl@esiee.fr>
  * Copyright (c) 2002 Thibaut Varene <varenet@parisc-linux.org>
@@ -28,6 +26,7 @@
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/property.h>
+#include <linux/rcupdate.h>
 #include <linux/serio.h>
 
 #include <asm/irq.h>
@@ -37,16 +36,10 @@ MODULE_AUTHOR("Laurent Canet <canetl@esiee.fr>, Thibaut Varene <varenet@parisc-l
 MODULE_DESCRIPTION("HP GSC PS2 port driver");
 MODULE_LICENSE("GPL");
 
-#define PFX "gscps2.c: "
-
 /*
  * Driver constants
  */
 
-/* various constants */
-#define ENABLE			1
-#define DISABLE			0
-
 #define GSC_DINO_OFFSET		0x0800	/* offset for DINO controller versus LASI one */
 
 /* PS/2 IO port offsets */
@@ -77,23 +70,21 @@ MODULE_LICENSE("GPL");
 #define GSC_ID_KEYBOARD		0	/* device ID values */
 #define GSC_ID_MOUSE		1
 
-#ifndef CONFIG_SERIO_GSCPS2_RDI_KEYCODES
-# define CONFLICT(x, y) x
-#else
-# define CONFLICT(x, y) y
-#endif
+#define RDI_CONFLICT(x, y) \
+	((x) * !IS_ENABLED(CONFIG_SERIO_GSCPS2_RDI_KEYCODES) + \
+	 (y) * IS_ENABLED(CONFIG_SERIO_GSCPS2_RDI_KEYCODES))
 
 /*
  * Sadly RDI (Tadpole) decided to ship a different keyboard layout
  * than HP for their PS/2 laptop keyboard which leads to conflicting
  * keycodes between a normal HP PS/2 keyboard and a RDI PrecisionBook.
- *                       HP:            RDI:
+ *                           HP:                RDI:
  */
-#define C_07	CONFLICT(KEY_F12,	KEY_F1)
-#define C_11	CONFLICT(KEY_LEFTALT,	KEY_LEFTCTRL)
-#define C_14	CONFLICT(KEY_LEFTCTRL,	KEY_CAPSLOCK)
-#define C_58	CONFLICT(KEY_CAPSLOCK,	KEY_RIGHTCTRL)
-#define C_61	CONFLICT(KEY_102ND,	KEY_LEFT)
+#define C_07	RDI_CONFLICT(KEY_F12,		KEY_F1)
+#define C_11	RDI_CONFLICT(KEY_LEFTALT,	KEY_LEFTCTRL)
+#define C_14	RDI_CONFLICT(KEY_LEFTCTRL,	KEY_CAPSLOCK)
+#define C_58	RDI_CONFLICT(KEY_CAPSLOCK,	KEY_RIGHTCTRL)
+#define C_61	RDI_CONFLICT(KEY_102ND,		KEY_LEFT)
 
 /*
  * Special keycode value recognized by atkbd (ATKBD_KEY_NULL) to silently
@@ -188,8 +179,6 @@ static const struct software_node gscps2_keyboard_node = {
 	.properties = gscps2_props,
 };
 
-static irqreturn_t gscps2_interrupt(int irq, void *dev);
-
 #define BUFFER_SIZE 0x0f
 
 /* GSC PS/2 port device struct */
@@ -197,33 +186,35 @@ struct gscps2port {
 	struct list_head node;
 	struct parisc_device *padev;
 	struct serio *port;
-	spinlock_t lock;
+	spinlock_t lock; /* serializes HW access */
 	char __iomem *addr;
 	u8 act, append; /* position in buffer[] */
 	struct {
 		u8 data;
 		u8 str;
-	} buffer[BUFFER_SIZE+1];
+	} buffer[BUFFER_SIZE + 1];
 	int id;
 };
 
+static DEFINE_SPINLOCK(ps2port_list_lock);
+static LIST_HEAD(ps2port_list);
+
 /*
  * Various HW level routines
  */
 
-#define gscps2_readb_input(x)		readb((x)+GSC_RCVDATA)
-#define gscps2_readb_control(x)		readb((x)+GSC_CONTROL)
-#define gscps2_readb_status(x)		readb((x)+GSC_STATUS)
-#define gscps2_writeb_control(x, y)	writeb((x), (y)+GSC_CONTROL)
-
+#define gscps2_readb_input(x)		readb((x) + GSC_RCVDATA)
+#define gscps2_readb_control(x)		readb((x) + GSC_CONTROL)
+#define gscps2_readb_status(x)		readb((x) + GSC_STATUS)
+#define gscps2_writeb_control(x, y)	writeb((x), (y) + GSC_CONTROL)
 
 /*
  * wait_TBE() - wait for Transmit Buffer Empty
  */
-
 static int wait_TBE(char __iomem *addr)
 {
 	int timeout = 25000; /* device is expected to react within 250 msec */
+
 	while (gscps2_readb_status(addr) & GSC_STAT_TBNE) {
 		if (!--timeout)
 			return 0;	/* This should not happen */
@@ -232,102 +223,39 @@ static int wait_TBE(char __iomem *addr)
 	return 1;
 }
 
-
 /*
  * gscps2_flush() - flush the receive buffer
  */
-
 static void gscps2_flush(struct gscps2port *ps2port)
 {
+	lockdep_assert_held(&ps2port->lock);
+
 	while (gscps2_readb_status(ps2port->addr) & GSC_STAT_RBNE)
 		gscps2_readb_input(ps2port->addr);
-	ps2port->act = ps2port->append = 0;
+	ps2port->act = 0;
+	ps2port->append = 0;
 }
 
-/*
- * gscps2_writeb_output() - write a byte to the port
- *
- * returns 1 on success, 0 on error
- */
-
-static inline int gscps2_writeb_output(struct gscps2port *ps2port, u8 data)
+static bool gscps2_read_data(struct gscps2port *ps2port)
 {
-	char __iomem *addr = ps2port->addr;
-
-	if (!wait_TBE(addr)) {
-		printk(KERN_DEBUG PFX "timeout - could not write byte %#x\n", data);
-		return 0;
-	}
-
-	while (gscps2_readb_status(addr) & GSC_STAT_RBNE)
-		/* wait */;
-
-	scoped_guard(spinlock_irqsave, &ps2port->lock)
-		writeb(data, addr+GSC_XMTDATA);
-
-	/* this is ugly, but due to timing of the port it seems to be necessary. */
-	mdelay(6);
-
-	/* make sure any received data is returned as fast as possible */
-	/* this is important e.g. when we set the LEDs on the keyboard */
-	gscps2_interrupt(0, NULL);
-
-	return 1;
-}
-
-
-/*
- * gscps2_enable() - enables or disables the port
- */
-
-static void gscps2_enable(struct gscps2port *ps2port, int enable)
-{
-	u8 data;
-
-	/* now enable/disable the port */
-	scoped_guard(spinlock_irqsave, &ps2port->lock) {
-		gscps2_flush(ps2port);
-		data = gscps2_readb_control(ps2port->addr);
-		if (enable)
-			data |= GSC_CTRL_ENBL;
-		else
-			data &= ~GSC_CTRL_ENBL;
-		gscps2_writeb_control(data, ps2port->addr);
-	}
-
-	wait_TBE(ps2port->addr);
-	gscps2_flush(ps2port);
-}
-
-/*
- * gscps2_reset() - resets the PS/2 port
- */
+	bool read_any = false;
+	u8 status;
 
-static void gscps2_reset(struct gscps2port *ps2port)
-{
-	/* reset the interface */
 	guard(spinlock_irqsave)(&ps2port->lock);
-	gscps2_flush(ps2port);
-	writeb(0xff, ps2port->addr + GSC_RESET);
-	gscps2_flush(ps2port);
-}
-
-static LIST_HEAD(ps2port_list);
-
-static void gscps2_read_data(struct gscps2port *ps2port)
-{
-	u8 status;
 
 	do {
 		status = gscps2_readb_status(ps2port->addr);
 		if (!(status & GSC_STAT_RBNE))
 			break;
 
+		read_any = true;
 		ps2port->buffer[ps2port->append].str = status;
 		ps2port->buffer[ps2port->append].data =
 				gscps2_readb_input(ps2port->addr);
 		ps2port->append = (ps2port->append + 1) & BUFFER_SIZE;
 	} while (true);
+
+	return read_any;
 }
 
 static bool gscps2_report_data(struct gscps2port *ps2port)
@@ -335,7 +263,7 @@ static bool gscps2_report_data(struct gscps2port *ps2port)
 	unsigned int rxflags;
 	u8 data, status;
 
-	while (ps2port->act != ps2port->append) {
+	while (true) {
 		/*
 		 * Did new data arrived while we read existing data ?
 		 * If yes, exit now and let the new irq handler start
@@ -344,19 +272,24 @@ static bool gscps2_report_data(struct gscps2port *ps2port)
 		if (gscps2_readb_status(ps2port->addr) & GSC_STAT_CMPINTR)
 			return true;
 
-		status = ps2port->buffer[ps2port->act].str;
-		data   = ps2port->buffer[ps2port->act].data;
+		scoped_guard(spinlock_irqsave, &ps2port->lock) {
+			if (ps2port->act == ps2port->append)
+				return false;
 
-		ps2port->act = (ps2port->act + 1) & BUFFER_SIZE;
-		rxflags = ((status & GSC_STAT_TERR) ? SERIO_TIMEOUT : 0 ) |
-			  ((status & GSC_STAT_PERR) ? SERIO_PARITY  : 0 );
+			status = ps2port->buffer[ps2port->act].str;
+			data   = ps2port->buffer[ps2port->act].data;
+			ps2port->act = (ps2port->act + 1) & BUFFER_SIZE;
+		}
+
+		rxflags = ((status & GSC_STAT_TERR) ? SERIO_TIMEOUT : 0) |
+			  ((status & GSC_STAT_PERR) ? SERIO_PARITY  : 0);
 
 		serio_interrupt(ps2port->port, data, rxflags);
 	}
-
-	return false;
 }
 
+static DEFINE_SPINLOCK(gscps2_interrupt_lock);
+
 /**
  * gscps2_interrupt() - Interruption service routine
  * @irq: interrupt number which triggered (unused)
@@ -370,39 +303,106 @@ static bool gscps2_report_data(struct gscps2port *ps2port)
  * the data as fast as possible and handle the reporting to the upper layer
  * later.
  */
-
 static irqreturn_t gscps2_interrupt(int irq, void *dev)
 {
 	struct gscps2port *ps2port;
+	bool handled = false;
+	bool more_data;
 
-	list_for_each_entry(ps2port, &ps2port_list, node) {
-		guard(spinlock_irqsave)(&ps2port->lock);
+	ACQUIRE(spinlock_irqsave_try, lock)(&gscps2_interrupt_lock);
+	if (ACQUIRE_ERR(spinlock_irqsave_try, &lock))
+		return IRQ_NONE;
 
-		gscps2_read_data(ps2port);
-	} /* list_for_each_entry */
+	guard(rcu)();
 
-	/* all data was read from the ports - now report the data to upper layer */
-	list_for_each_entry(ps2port, &ps2port_list, node) {
-		if (gscps2_report_data(ps2port)) {
-			/* More data ready - break early to restart interrupt */
-			break;
+	do {
+		more_data = false;
+
+		list_for_each_entry_rcu(ps2port, &ps2port_list, node) {
+			if (gscps2_read_data(ps2port))
+				handled = true;
 		}
+
+		/* all data was read from the ports - now report the data to upper layer */
+		list_for_each_entry_rcu(ps2port, &ps2port_list, node) {
+			if (gscps2_report_data(ps2port)) {
+				/* More data ready - restart loop to read new data */
+				more_data = true;
+				break;
+			}
+		}
+	} while (more_data);
+
+	return IRQ_RETVAL(handled);
+}
+
+/*
+ * gscps2_writeb_output() - write a byte to the port
+ *
+ * returns 1 on success, 0 on error
+ */
+static inline int gscps2_writeb_output(struct gscps2port *ps2port, u8 data)
+{
+	char __iomem *addr = ps2port->addr;
+
+	if (!wait_TBE(addr)) {
+		dev_dbg(&ps2port->padev->dev, "timeout - could not write byte %#x\n", data);
+		return 0;
+	}
+
+	while (gscps2_readb_status(addr) & GSC_STAT_RBNE)
+		cpu_relax();
+
+	scoped_guard(spinlock_irqsave, &ps2port->lock)
+		writeb(data, addr + GSC_XMTDATA);
+
+	return 1;
+}
+
+/*
+ * gscps2_enable() - enables or disables the port
+ */
+static void gscps2_enable(struct gscps2port *ps2port, bool enable)
+{
+	u8 data;
+
+	/* now enable/disable the port */
+	scoped_guard(spinlock_irqsave, &ps2port->lock) {
+		gscps2_flush(ps2port);
+		data = gscps2_readb_control(ps2port->addr);
+		if (enable)
+			data |= GSC_CTRL_ENBL;
+		else
+			data &= ~GSC_CTRL_ENBL;
+		gscps2_writeb_control(data, ps2port->addr);
 	}
 
-	return IRQ_HANDLED;
+	wait_TBE(ps2port->addr);
+	scoped_guard(spinlock_irqsave, &ps2port->lock)
+		gscps2_flush(ps2port);
 }
 
+/*
+ * gscps2_reset() - resets the PS/2 port
+ */
+static void gscps2_reset(struct gscps2port *ps2port)
+{
+	/* reset the interface */
+	guard(spinlock_irqsave)(&ps2port->lock);
+	gscps2_flush(ps2port);
+	writeb(0xff, ps2port->addr + GSC_RESET);
+	gscps2_flush(ps2port);
+}
 
 /*
  * gscps2_write() - send a byte out through the aux interface.
  */
-
 static int gscps2_write(struct serio *port, unsigned char data)
 {
 	struct gscps2port *ps2port = port->port_data;
 
 	if (!gscps2_writeb_output(ps2port, data)) {
-		printk(KERN_DEBUG PFX "sending byte %#x failed.\n", data);
+		dev_dbg(&ps2port->padev->dev, "sending byte %#x failed.\n", data);
 		return -1;
 	}
 	return 0;
@@ -412,15 +412,17 @@ static int gscps2_write(struct serio *port, unsigned char data)
  * gscps2_open() is called when a port is opened by the higher layer.
  * It resets and enables the port.
  */
-
 static int gscps2_open(struct serio *port)
 {
 	struct gscps2port *ps2port = port->port_data;
 
 	gscps2_reset(ps2port);
 
+	scoped_guard(spinlock_irqsave, &ps2port_list_lock)
+		list_add_tail_rcu(&ps2port->node, &ps2port_list);
+
 	/* enable it */
-	gscps2_enable(ps2port, ENABLE);
+	gscps2_enable(ps2port, true);
 
 	gscps2_interrupt(0, NULL);
 
@@ -430,11 +432,16 @@ static int gscps2_open(struct serio *port)
 /*
  * gscps2_close() disables the port
  */
-
 static void gscps2_close(struct serio *port)
 {
 	struct gscps2port *ps2port = port->port_data;
-	gscps2_enable(ps2port, DISABLE);
+
+	gscps2_enable(ps2port, false);
+
+	scoped_guard(spinlock_irqsave, &ps2port_list_lock)
+		list_del_rcu(&ps2port->node);
+
+	synchronize_rcu();
 }
 
 /**
@@ -443,7 +450,6 @@ static void gscps2_close(struct serio *port)
  *
  * @return: success/error report
  */
-
 static int __init gscps2_probe(struct parisc_device *dev)
 {
 	struct gscps2port *ps2port;
@@ -469,6 +475,7 @@ static int __init gscps2_probe(struct parisc_device *dev)
 
 	ps2port->port = serio;
 	ps2port->padev = dev;
+	INIT_LIST_HEAD(&ps2port->node);
 	ps2port->addr = ioremap(hpa, GSC_STATUS + 4);
 	if (!ps2port->addr) {
 		ret = -ENOMEM;
@@ -494,8 +501,8 @@ static int __init gscps2_probe(struct parisc_device *dev)
 		goto fail_miserably;
 
 	if (ps2port->id != GSC_ID_KEYBOARD && ps2port->id != GSC_ID_MOUSE) {
-		printk(KERN_WARNING PFX "Unsupported PS/2 port at 0x%08lx (id=%d) ignored\n",
-				hpa, ps2port->id);
+		dev_warn(&dev->dev, "Unsupported PS/2 port at 0x%08lx (id=%d) ignored\n",
+			 hpa, ps2port->id);
 		ret = -ENODEV;
 		goto fail;
 	}
@@ -524,8 +531,6 @@ static int __init gscps2_probe(struct parisc_device *dev)
 
 	serio_register_port(ps2port->port);
 
-	list_add_tail(&ps2port->node, &ps2port_list);
-
 	return 0;
 
 fail:
@@ -552,7 +557,6 @@ static int __init gscps2_probe(struct parisc_device *dev)
  *
  * @return: success/error report
  */
-
 static void __exit gscps2_remove(struct parisc_device *dev)
 {
 	struct gscps2port *ps2port = dev_get_drvdata(&dev->dev);
@@ -562,17 +566,15 @@ static void __exit gscps2_remove(struct parisc_device *dev)
 
 	serio_unregister_port(ps2port->port);
 	free_irq(dev->irq, ps2port);
-	gscps2_flush(ps2port);
-	list_del(&ps2port->node);
+	scoped_guard(spinlock_irqsave, &ps2port->lock)
+		gscps2_flush(ps2port);
 	iounmap(ps2port->addr);
 #if 0
 	release_mem_region(dev->hpa, GSC_STATUS + 4);
 #endif
-	dev_set_drvdata(&dev->dev, NULL);
 	kfree(ps2port);
 }
 
-
 static const struct parisc_device_id gscps2_device_tbl[] __initconst = {
 	{ HPHW_FIO, HVERSION_REV_ANY_ID, HVERSION_ANY_ID, 0x00084 }, /* LASI PS/2 */
 #ifdef DINO_TESTED
@@ -610,6 +612,5 @@ static void __exit gscps2_exit(void)
 	software_node_unregister(&gscps2_keyboard_node);
 }
 
-
 module_init(gscps2_init);
 module_exit(gscps2_exit);
diff --git a/drivers/input/serio/hp_sdc.c b/drivers/input/serio/hp_sdc.c
index 1461ef319f92a8..ecf5347b63ab74 100644
--- a/drivers/input/serio/hp_sdc.c
+++ b/drivers/input/serio/hp_sdc.c
@@ -981,7 +981,7 @@ static void hp_sdc_exit(void)
 	free_irq(hp_sdc.irq, &hp_sdc);
 	write_unlock_irq(&hp_sdc.lock);
 
-	timer_delete_sync(&hp_sdc.kicker);
+	timer_shutdown_sync(&hp_sdc.kicker);
 
 	tasklet_kill(&hp_sdc.task);
 
diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h
index 9ecb0eed48c4bb..54eb3687000a59 100644
--- a/drivers/input/serio/i8042-acpipnpio.h
+++ b/drivers/input/serio/i8042-acpipnpio.h
@@ -259,6 +259,13 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = {
 		},
 		.driver_data = (void *)(SERIO_QUIRK_RESET_ALWAYS)
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire AG15-42P"),
+		},
+		.driver_data = (void *)(SERIO_QUIRK_RESET_ALWAYS)
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
diff --git a/drivers/input/touchscreen/cyttsp5.c b/drivers/input/touchscreen/cyttsp5.c
index 9266c07314beee..e878a02dc9b75b 100644
--- a/drivers/input/touchscreen/cyttsp5.c
+++ b/drivers/input/touchscreen/cyttsp5.c
@@ -710,6 +710,7 @@ static irqreturn_t cyttsp5_handle_irq(int irq, void *handle)
 		size = 2;
 	} else {
 		report_id = ts->input_buf[2];
+		size = min(size, CY_MAX_INPUT);
 	}
 
 	switch (report_id) {
diff --git a/drivers/input/touchscreen/eeti_ts.c b/drivers/input/touchscreen/eeti_ts.c
index b12602bc368efc..b870a939508bb2 100644
--- a/drivers/input/touchscreen/eeti_ts.c
+++ b/drivers/input/touchscreen/eeti_ts.c
@@ -276,6 +276,7 @@ static const struct of_device_id of_eeti_ts_match[] = {
 	{ .compatible = "eeti,exc3000-i2c", },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, of_eeti_ts_match);
 #endif
 
 static struct i2c_driver eeti_ts_driver = {
diff --git a/drivers/input/touchscreen/goodix_berlin_core.c b/drivers/input/touchscreen/goodix_berlin_core.c
index b0938a4f3fec30..2311b3e091ef8a 100644
--- a/drivers/input/touchscreen/goodix_berlin_core.c
+++ b/drivers/input/touchscreen/goodix_berlin_core.c
@@ -29,6 +29,7 @@
 #include <linux/input.h>
 #include <linux/input/mt.h>
 #include <linux/input/touchscreen.h>
+#include <linux/interrupt.h>
 #include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
diff --git a/drivers/input/touchscreen/hynitron_cstxxx.c b/drivers/input/touchscreen/hynitron_cstxxx.c
index 49fb35c830dc15..3a54a3e6a3658c 100644
--- a/drivers/input/touchscreen/hynitron_cstxxx.c
+++ b/drivers/input/touchscreen/hynitron_cstxxx.c
@@ -16,6 +16,7 @@
 #include <linux/err.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
 #include <linux/input/touchscreen.h>
diff --git a/drivers/input/touchscreen/imagis.c b/drivers/input/touchscreen/imagis.c
index 7bbb00beec3be4..5ca19ab7a73c30 100644
--- a/drivers/input/touchscreen/imagis.c
+++ b/drivers/input/touchscreen/imagis.c
@@ -4,6 +4,7 @@
 #include <linux/bits.h>
 #include <linux/delay.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
 #include <linux/input/touchscreen.h>
diff --git a/drivers/input/touchscreen/tsc2007_core.c b/drivers/input/touchscreen/tsc2007_core.c
index e4d7da0f4434e8..e2f49b37e18c0d 100644
--- a/drivers/input/touchscreen/tsc2007_core.c
+++ b/drivers/input/touchscreen/tsc2007_core.c
@@ -221,7 +221,6 @@ static int tsc2007_get_pendown_state_gpio(struct device *dev)
 static int tsc2007_probe_properties(struct device *dev, struct tsc2007 *ts)
 {
 	u32 val32;
-	u64 val64;
 
 	if (!device_property_read_u32(dev, "ti,max-rt", &val32))
 		ts->max_rt = val32;
@@ -237,8 +236,8 @@ static int tsc2007_probe_properties(struct device *dev, struct tsc2007 *ts)
 	if (!device_property_read_u32(dev, "ti,fuzzz", &val32))
 		ts->fuzzz = val32;
 
-	if (!device_property_read_u64(dev, "ti,poll-period", &val64))
-		ts->poll_period = msecs_to_jiffies(val64);
+	if (!device_property_read_u32(dev, "ti,poll-period", &val32))
+		ts->poll_period = msecs_to_jiffies(val32);
 	else
 		ts->poll_period = msecs_to_jiffies(1);
 
diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c
index 4aa991a5410183..802f07e806d5ae 100644
--- a/drivers/interconnect/core.c
+++ b/drivers/interconnect/core.c
@@ -423,6 +423,17 @@ static void devm_icc_release(struct device *dev, void *res)
 	icc_put(*(struct icc_path **)res);
 }
 
+/**
+ * devm_of_icc_get() - get a path handle from a DT node based on name
+ * @dev: device pointer for the consumer device
+ * @name: interconnect path name
+ *
+ * This function is the resource managed version of of_icc_get(). The path
+ * is released with icc_put() automatically when @dev is unbound.
+ *
+ * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
+ * when the API is disabled or the "interconnects" DT property is missing.
+ */
 struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
 {
 	struct icc_path **ptr, *path;
@@ -786,12 +797,33 @@ static int __icc_enable(struct icc_path *path, bool enable)
 			  path->reqs[0].peak_bw);
 }
 
+/**
+ * icc_enable() - enable a path
+ * @path: interconnect path
+ *
+ * Mark all requests on the path as enabled and reapply the bandwidth that
+ * was last set with icc_set_bw().
+ * Passing a NULL @path is a no-op and returns 0.
+ *
+ * Return: 0 on success, or an appropriate error code otherwise.
+ */
 int icc_enable(struct icc_path *path)
 {
 	return __icc_enable(path, true);
 }
 EXPORT_SYMBOL_GPL(icc_enable);
 
+/**
+ * icc_disable() - disable a path
+ * @path: interconnect path
+ *
+ * Mark all requests on the path as disabled, so that they no longer count
+ * towards the aggregated bandwidth, and reapply the constraints. The
+ * requested bandwidth is kept and restored by icc_enable().
+ * Passing a NULL @path is a no-op and returns 0.
+ *
+ * Return: 0 on success, or an appropriate error code otherwise.
+ */
 int icc_disable(struct icc_path *path)
 {
 	return __icc_enable(path, false);
diff --git a/drivers/interconnect/debugfs-client.c b/drivers/interconnect/debugfs-client.c
index 91f86d9237a688..238d282303ffad 100644
--- a/drivers/interconnect/debugfs-client.c
+++ b/drivers/interconnect/debugfs-client.c
@@ -5,6 +5,7 @@
 #include <linux/debugfs.h>
 #include <linux/interconnect.h>
 #include <linux/platform_device.h>
+#include <linux/slab.h>
 
 #include "internal.h"
 
@@ -36,6 +37,59 @@ struct debugfs_path {
 	struct list_head list;
 };
 
+static ssize_t icc_node_read(struct file *file, char __user *user_buf,
+			     size_t count, loff_t *ppos)
+{
+	char **node = file->private_data;
+	char *copy;
+	size_t len;
+	ssize_t ret;
+
+	scoped_guard(mutex, &debugfs_lock) {
+		copy = kstrdup(*node ?: "", GFP_KERNEL);
+	}
+	if (!copy)
+		return -ENOMEM;
+
+	len = strlen(copy);
+	copy[len++] = '\n';
+	ret = simple_read_from_buffer(user_buf, count, ppos, copy, len);
+	kfree(copy);
+	return ret;
+}
+
+static ssize_t icc_node_write(struct file *file, const char __user *user_buf,
+			      size_t count, loff_t *ppos)
+{
+	char **node = file->private_data;
+	char *old, *new;
+
+	if (*ppos)
+		return -EINVAL;
+	if (count + 1 > PAGE_SIZE)
+		return -E2BIG;
+
+	new = memdup_user_nul(user_buf, count);
+	if (IS_ERR(new))
+		return PTR_ERR(new);
+	strim(new);
+
+	scoped_guard(mutex, &debugfs_lock) {
+		old = *node;
+		*node = new;
+	}
+
+	kfree(old);
+	return count;
+}
+
+static const struct file_operations icc_node_fops = {
+	.open = simple_open,
+	.read = icc_node_read,
+	.write = icc_node_write,
+	.llseek = default_llseek,
+};
+
 static struct icc_path *get_path(const char *src, const char *dst)
 {
 	struct debugfs_path *path;
@@ -54,26 +108,19 @@ static int icc_get_set(void *data, u64 val)
 	char *src, *dst;
 	int ret = 0;
 
-	mutex_lock(&debugfs_lock);
-
-	rcu_read_lock();
-	src = rcu_dereference(src_node);
-	dst = rcu_dereference(dst_node);
+	guard(mutex)(&debugfs_lock);
 
 	/*
 	 * If we've already looked up a path, then use the existing one instead
 	 * of calling icc_get() again. This allows for updating previous BW
 	 * votes when "get" is written to multiple times for multiple paths.
 	 */
-	cur_path = get_path(src, dst);
-	if (cur_path) {
-		rcu_read_unlock();
+	cur_path = get_path(src_node, dst_node);
+	if (cur_path)
 		goto out;
-	}
 
-	src = kstrdup(src, GFP_ATOMIC);
-	dst = kstrdup(dst, GFP_ATOMIC);
-	rcu_read_unlock();
+	src = kstrdup(src_node, GFP_KERNEL);
+	dst = kstrdup(dst_node, GFP_KERNEL);
 
 	if (!src || !dst) {
 		ret = -ENOMEM;
@@ -105,7 +152,6 @@ static int icc_get_set(void *data, u64 val)
 	kfree(src);
 	kfree(dst);
 out:
-	mutex_unlock(&debugfs_lock);
 	return ret;
 }
 
@@ -115,7 +161,7 @@ static int icc_commit_set(void *data, u64 val)
 {
 	int ret;
 
-	mutex_lock(&debugfs_lock);
+	guard(mutex)(&debugfs_lock);
 
 	if (!cur_path) {
 		ret = -EINVAL;
@@ -130,7 +176,6 @@ static int icc_commit_set(void *data, u64 val)
 	icc_set_tag(cur_path, tag);
 	ret = icc_set_bw(cur_path, avg_bw, peak_bw);
 out:
-	mutex_unlock(&debugfs_lock);
 	return ret;
 }
 
@@ -162,8 +207,10 @@ int icc_debugfs_client_init(struct dentry *icc_dir)
 
 	client_dir = debugfs_create_dir("test_client", icc_dir);
 
-	debugfs_create_str("src_node", 0600, client_dir, &src_node);
-	debugfs_create_str("dst_node", 0600, client_dir, &dst_node);
+	debugfs_create_file("src_node", 0600, client_dir, &src_node,
+			    &icc_node_fops);
+	debugfs_create_file("dst_node", 0600, client_dir, &dst_node,
+			    &icc_node_fops);
 	debugfs_create_file("get", 0200, client_dir, NULL, &icc_get_fops);
 	debugfs_create_u32("avg_bw", 0600, client_dir, &avg_bw);
 	debugfs_create_u32("peak_bw", 0600, client_dir, &peak_bw);
diff --git a/drivers/interconnect/mediatek/Makefile b/drivers/interconnect/mediatek/Makefile
index 6bd656668f5d8b..64170ab163ff7b 100644
--- a/drivers/interconnect/mediatek/Makefile
+++ b/drivers/interconnect/mediatek/Makefile
@@ -3,4 +3,4 @@
 obj-$(CONFIG_INTERCONNECT_MTK_DVFSRC_EMI) += icc-emi.o
 obj-$(CONFIG_INTERCONNECT_MTK_MT8183) += mt8183.o
 obj-$(CONFIG_INTERCONNECT_MTK_MT8195) += mt8195.o
-obj-$(CONFIG_INTERCONNECT_MTK_MT8195) += mt8196.o
+obj-$(CONFIG_INTERCONNECT_MTK_MT8196) += mt8196.o
diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig
index b3187acf90b8d3..7f3a977e8f10bd 100644
--- a/drivers/interconnect/qcom/Kconfig
+++ b/drivers/interconnect/qcom/Kconfig
@@ -2,6 +2,7 @@
 config INTERCONNECT_QCOM
 	tristate "Qualcomm Network-on-Chip interconnect drivers"
 	depends on ARCH_QCOM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Support for Qualcomm's Network-on-Chip interconnect hardware.
 
@@ -14,6 +15,7 @@ config INTERCONNECT_QCOM_ELIZA
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on Eliza-based
 	  platforms.
@@ -24,6 +26,7 @@ config INTERCONNECT_QCOM_GLYMUR
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on glymur-based
 	  platforms.
@@ -31,8 +34,10 @@ config INTERCONNECT_QCOM_GLYMUR
 config INTERCONNECT_QCOM_HAWI
 	tristate "Qualcomm HAWI interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on hawi-based
 	  platforms.
@@ -43,13 +48,28 @@ config INTERCONNECT_QCOM_KAANAPALI
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on kaanapali-based
 	  platforms.
 
+config INTERCONNECT_QCOM_KUNO
+	tristate "Qualcomm Kuno interconnect driver"
+	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM || COMPILE_TEST
+	select INTERCONNECT_QCOM_RPMH
+	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
+	help
+	  This is a driver for the Qualcomm Network-on-Chip on kuno-based
+	  platforms. The interconnect provider collects and aggregates the
+	  consumer bandwidth requests to satisfy constraints placed on the
+	  Network-on-Chip performance states.
+
 config INTERCONNECT_QCOM_MAILI
 	tristate "Qualcomm MAILI interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	default ARCH_QCOM
@@ -72,6 +92,7 @@ config INTERCONNECT_QCOM_MSM8916
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM if ARM64
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on msm8916-based
 	  platforms.
@@ -100,6 +121,7 @@ config INTERCONNECT_QCOM_MSM8953
 	depends on QCOM_SMD_RPM
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on msm8953-based
 	  platforms.
@@ -110,6 +132,7 @@ config INTERCONNECT_QCOM_MSM8974
 	depends on QCOM_SMD_RPM
 	depends on ARM || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	 This is a driver for the Qualcomm Network-on-Chip on msm8974-based
 	 platforms.
@@ -130,6 +153,7 @@ config INTERCONNECT_QCOM_MSM8996
 	depends on QCOM_SMD_RPM
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on msm8996-based
 	  platforms.
@@ -137,8 +161,10 @@ config INTERCONNECT_QCOM_MSM8996
 config INTERCONNECT_QCOM_NORD
 	tristate "Qualcomm Nord interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on Nord-based
 	  platforms. The topology consists of several NoCs controlled by
@@ -149,6 +175,7 @@ config INTERCONNECT_QCOM_OSM_L3
 	tristate "Qualcomm OSM L3 interconnect driver"
 	depends on INTERCONNECT_QCOM || COMPILE_TEST
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  Say y here to support the Operating State Manager (OSM) interconnect
 	  driver which controls the scaling of L3 caches on Qualcomm SoCs.
@@ -159,6 +186,7 @@ config INTERCONNECT_QCOM_QCM2290
 	depends on QCOM_SMD_RPM
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on qcm2290-based
 	  platforms.
@@ -169,6 +197,7 @@ config INTERCONNECT_QCOM_QCS404
 	depends on QCOM_SMD_RPM
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on qcs404-based
 	  platforms.
@@ -179,6 +208,7 @@ config INTERCONNECT_QCOM_QCS615
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on qcs615-based
 	  platforms.
@@ -189,6 +219,7 @@ config INTERCONNECT_QCOM_QCS8300
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Technologies, Inc. Network-on-Chip
 	  on QCS8300-based platforms. The interconnect provider collects and
@@ -201,6 +232,7 @@ config INTERCONNECT_QCOM_QDU1000
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on QDU1000-based
 	  and QRU1000-based platforms.
@@ -225,6 +257,7 @@ config INTERCONNECT_QCOM_SA8775P
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sa8775p-based
 	  platforms.
@@ -235,6 +268,7 @@ config INTERCONNECT_QCOM_SAR2130P
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on SAR2130P-based
 	  platforms.
@@ -245,6 +279,7 @@ config INTERCONNECT_QCOM_SC7180
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sc7180-based
 	  platforms.
@@ -255,6 +290,7 @@ config INTERCONNECT_QCOM_SC7280
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sc7280-based
 	  platforms.
@@ -265,6 +301,7 @@ config INTERCONNECT_QCOM_SC8180X
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sc8180x-based
 	  platforms.
@@ -275,6 +312,7 @@ config INTERCONNECT_QCOM_SC8280XP
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on SC8280XP-based
 	  platforms.
@@ -285,6 +323,7 @@ config INTERCONNECT_QCOM_SDM660
 	depends on ARM64 || COMPILE_TEST
 	depends on QCOM_SMD_RPM
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sdm660-based
 	  platforms.
@@ -295,6 +334,7 @@ config INTERCONNECT_QCOM_SDM670
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sdm670-based
 	  platforms.
@@ -305,6 +345,7 @@ config INTERCONNECT_QCOM_SDM845
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sdm845-based
 	  platforms.
@@ -315,6 +356,7 @@ config INTERCONNECT_QCOM_SDX55
 	depends on ARM || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sdx55-based
 	  platforms.
@@ -335,6 +377,7 @@ config INTERCONNECT_QCOM_SDX75
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sdx75-based
 	  platforms.
@@ -343,6 +386,8 @@ config INTERCONNECT_QCOM_SHIKRA
 	tristate "Qualcomm SHIKRA interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on shikra-based
@@ -354,6 +399,7 @@ config INTERCONNECT_QCOM_SM6115
 	depends on QCOM_SMD_RPM
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sm6115-based
 	  platforms.
@@ -364,6 +410,7 @@ config INTERCONNECT_QCOM_SM6350
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sm6350-based
 	  platforms.
@@ -374,6 +421,7 @@ config INTERCONNECT_QCOM_SM7150
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sm7150-based
 	  platforms.
@@ -384,6 +432,7 @@ config INTERCONNECT_QCOM_MILOS
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on Milos-based
 	  platforms.
@@ -394,6 +443,7 @@ config INTERCONNECT_QCOM_SM8150
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sm8150-based
 	  platforms.
@@ -404,6 +454,7 @@ config INTERCONNECT_QCOM_SM8250
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sm8250-based
 	  platforms.
@@ -414,6 +465,7 @@ config INTERCONNECT_QCOM_SM8350
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on SM8350-based
 	  platforms.
@@ -424,6 +476,7 @@ config INTERCONNECT_QCOM_SM8450
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on SM8450-based
 	  platforms.
@@ -434,6 +487,7 @@ config INTERCONNECT_QCOM_SM8550
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on SM8550-based
 	  platforms.
@@ -444,6 +498,7 @@ config INTERCONNECT_QCOM_SM8650
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on SM8650-based
 	  platforms.
@@ -454,6 +509,7 @@ config INTERCONNECT_QCOM_SM8750
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on SM8750-based
 	  platforms.
@@ -464,6 +520,7 @@ config INTERCONNECT_QCOM_X1E80100
 	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
+	default ARCH_QCOM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on X1E80100-based
 	  platforms.
diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile
index 5f139a370bbea5..a71478aaf591f5 100644
--- a/drivers/interconnect/qcom/Makefile
+++ b/drivers/interconnect/qcom/Makefile
@@ -8,6 +8,7 @@ qnoc-eliza-objs				:= eliza.o
 qnoc-glymur-objs			:= glymur.o
 qnoc-hawi-objs				:= hawi.o
 qnoc-kaanapali-objs			:= kaanapali.o
+qnoc-kuno-objs				:= kuno.o
 qnoc-maili-objs				:= maili.o
 qnoc-milos-objs				:= milos.o
 qnoc-msm8909-objs			:= msm8909.o
@@ -57,6 +58,7 @@ obj-$(CONFIG_INTERCONNECT_QCOM_ELIZA) += qnoc-eliza.o
 obj-$(CONFIG_INTERCONNECT_QCOM_GLYMUR) += qnoc-glymur.o
 obj-$(CONFIG_INTERCONNECT_QCOM_HAWI) += qnoc-hawi.o
 obj-$(CONFIG_INTERCONNECT_QCOM_KAANAPALI) += qnoc-kaanapali.o
+obj-$(CONFIG_INTERCONNECT_QCOM_KUNO) += qnoc-kuno.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MAILI) += qnoc-maili.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MILOS) += qnoc-milos.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MSM8909) += qnoc-msm8909.o
diff --git a/drivers/interconnect/qcom/bcm-voter.c b/drivers/interconnect/qcom/bcm-voter.c
index a2d437a05a11fa..0880a287b0c0aa 100644
--- a/drivers/interconnect/qcom/bcm-voter.c
+++ b/drivers/interconnect/qcom/bcm-voter.c
@@ -3,7 +3,6 @@
  * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
  */
 
-#include <asm/div64.h>
 #include <linux/interconnect-provider.h>
 #include <linux/list_sort.h>
 #include <linux/module.h>
@@ -14,6 +13,7 @@
 #include <soc/qcom/tcs.h>
 
 #include "bcm-voter.h"
+#include "icc-common.h"
 #include "icc-rpmh.h"
 
 static LIST_HEAD(bcm_voters);
@@ -47,17 +47,6 @@ static int cmp_vcd(void *priv, const struct list_head *a, const struct list_head
 	return bcm_a->aux_data.vcd - bcm_b->aux_data.vcd;
 }
 
-static u64 bcm_div(u64 num, u32 base)
-{
-	/* Ensure that small votes aren't lost. */
-	if (num && num < base)
-		return 1;
-
-	do_div(num, base);
-
-	return num;
-}
-
 /* BCMs with enable_mask use one-hot-encoding for on/off signaling */
 static void bcm_aggregate_mask(struct qcom_icc_bcm *bcm)
 {
@@ -99,20 +88,20 @@ static void bcm_aggregate(struct qcom_icc_bcm *bcm)
 	for (bucket = 0; bucket < QCOM_ICC_NUM_BUCKETS; bucket++) {
 		for (i = 0; i < bcm->num_nodes; i++) {
 			node = bcm->nodes[i];
-			temp = bcm_div(node->sum_avg[bucket] * bcm->aux_data.width,
-				       node->buswidth * node->channels);
+			temp = qcom_bw_div(node->sum_avg[bucket] * le16_to_cpu(bcm->aux_data.width),
+					   node->buswidth * node->channels);
 			agg_avg[bucket] = max(agg_avg[bucket], temp);
 
-			temp = bcm_div(node->max_peak[bucket] * bcm->aux_data.width,
-				       node->buswidth);
+			temp = qcom_bw_div(node->max_peak[bucket] * le16_to_cpu(bcm->aux_data.width),
+					   node->buswidth);
 			agg_peak[bucket] = max(agg_peak[bucket], temp);
 		}
 
 		temp = agg_avg[bucket] * bcm->vote_scale;
-		bcm->vote_x[bucket] = bcm_div(temp, bcm->aux_data.unit);
+		bcm->vote_x[bucket] = qcom_bw_div(temp, le32_to_cpu(bcm->aux_data.unit));
 
 		temp = agg_peak[bucket] * bcm->vote_scale;
-		bcm->vote_y[bucket] = bcm_div(temp, bcm->aux_data.unit);
+		bcm->vote_y[bucket] = qcom_bw_div(temp, le32_to_cpu(bcm->aux_data.unit));
 	}
 
 	if (bcm->keepalive && bcm->vote_x[QCOM_ICC_BUCKET_AMC] == 0 &&
@@ -261,6 +250,40 @@ void qcom_icc_bcm_voter_add(struct bcm_voter *voter, struct qcom_icc_bcm *bcm)
 }
 EXPORT_SYMBOL_GPL(qcom_icc_bcm_voter_add);
 
+/**
+ * qcom_icc_bcm_get_bw - get current bcm vote
+ * @voter: voter used to query bcm
+ * @bcm: bcm to get current vote from
+ */
+void qcom_icc_bcm_get_bw(struct bcm_voter *voter,
+			 struct qcom_icc_bcm *bcm)
+{
+	struct tcs_cmd cmd = { .addr = bcm->addr };
+	int ret, i;
+	u64 x, y;
+
+	mutex_lock(&voter->lock);
+
+	ret = rpmh_read(voter->dev, &cmd);
+	if (ret) {
+		pr_err("Error sending AMC RPMH requests (%d)\n", ret);
+		goto out;
+	}
+
+	x = FIELD_GET(BCM_TCS_CMD_VOTE_X_MASK, cmd.data);
+	y = FIELD_GET(BCM_TCS_CMD_VOTE_Y_MASK, cmd.data);
+
+	/* For boot-up, fill the AMC vote in all buckets */
+	for (i = 0; i < QCOM_ICC_NUM_BUCKETS; i++) {
+		bcm->vote_x[i] = x;
+		bcm->vote_y[i] = y;
+	}
+
+out:
+	mutex_unlock(&voter->lock);
+}
+EXPORT_SYMBOL_GPL(qcom_icc_bcm_get_bw);
+
 /**
  * qcom_icc_bcm_voter_commit - generates and commits tcs cmds based on bcms
  * @voter: voter that needs flushing
diff --git a/drivers/interconnect/qcom/bcm-voter.h b/drivers/interconnect/qcom/bcm-voter.h
index b4d36e349f3c0d..fc75d457dcc7bc 100644
--- a/drivers/interconnect/qcom/bcm-voter.h
+++ b/drivers/interconnect/qcom/bcm-voter.h
@@ -13,6 +13,7 @@
 #include "icc-rpmh.h"
 
 struct bcm_voter *of_bcm_voter_get(struct device *dev, const char *name);
+void qcom_icc_bcm_get_bw(struct bcm_voter *voter, struct qcom_icc_bcm *bcm);
 void qcom_icc_bcm_voter_add(struct bcm_voter *voter, struct qcom_icc_bcm *bcm);
 int qcom_icc_bcm_voter_commit(struct bcm_voter *voter);
 
diff --git a/drivers/interconnect/qcom/eliza.c b/drivers/interconnect/qcom/eliza.c
index 891e4e6e8ba8fd..ad03adef79ba76 100644
--- a/drivers/interconnect/qcom/eliza.c
+++ b/drivers/interconnect/qcom/eliza.c
@@ -451,7 +451,7 @@ static struct qcom_icc_node qns_gem_noc_cnoc = {
 	.link_nodes = { &qnm_gemnoc_cnoc },
 };
 
-static struct qcom_icc_qosbox alm_gpu_tcu_qos = {
+static const struct qcom_icc_qosbox alm_gpu_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x155000 },
 	.prio = 1,
@@ -468,7 +468,7 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox alm_sys_tcu_qos = {
+static const struct qcom_icc_qosbox alm_sys_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x157000 },
 	.prio = 6,
@@ -494,7 +494,7 @@ static struct qcom_icc_node chm_apps = {
 			&qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_gpu_qos = {
+static const struct qcom_icc_qosbox qnm_gpu_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x31000, 0xb1000 },
 	.prio = 0,
@@ -511,7 +511,7 @@ static struct qcom_icc_node qnm_gpu = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_lpass_gemnoc_qos = {
+static const struct qcom_icc_qosbox qnm_lpass_gemnoc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x159000 },
 	.prio = 0,
@@ -538,7 +538,7 @@ static struct qcom_icc_node qnm_mdsp = {
 			&qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x33000, 0xb3000 },
 	.prio = 0,
@@ -555,7 +555,7 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x35000, 0xb5000 },
 	.prio = 0,
@@ -572,7 +572,7 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_nsp_gemnoc_qos = {
+static const struct qcom_icc_qosbox qnm_nsp_gemnoc_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x37000, 0xb7000 },
 	.prio = 0,
@@ -590,7 +590,7 @@ static struct qcom_icc_node qnm_nsp_gemnoc = {
 			&qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_pcie_qos = {
+static const struct qcom_icc_qosbox qnm_pcie_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x15b000 },
 	.prio = 2,
@@ -688,7 +688,7 @@ static struct qcom_icc_node qnm_lpiaon_noc = {
 	.link_nodes = { &qns_lpass_ag_noc_gemnoc },
 };
 
-static struct qcom_icc_qosbox qnm_camnoc_nrt_icp_sf_qos = {
+static const struct qcom_icc_qosbox qnm_camnoc_nrt_icp_sf_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x25000 },
 	.prio = 4,
diff --git a/drivers/interconnect/qcom/icc-common.h b/drivers/interconnect/qcom/icc-common.h
index 21c39b1639486b..af4b5a6f30b71b 100644
--- a/drivers/interconnect/qcom/icc-common.h
+++ b/drivers/interconnect/qcom/icc-common.h
@@ -7,6 +7,15 @@
 #define __DRIVERS_INTERCONNECT_QCOM_ICC_COMMON_H__
 
 #include <linux/interconnect-provider.h>
+#include <linux/math64.h>
+
+static inline u64 qcom_bw_div(u64 num, u32 base)
+{
+	if (num && num < base)
+		return 1;
+
+	return div_u64(num, base);
+}
 
 struct icc_node_data *qcom_icc_xlate_extended(const struct of_phandle_args *spec,
 					      void *data);
diff --git a/drivers/interconnect/qcom/icc-rpm.c b/drivers/interconnect/qcom/icc-rpm.c
index ca4c8a944755ed..6f325789a96ea8 100644
--- a/drivers/interconnect/qcom/icc-rpm.c
+++ b/drivers/interconnect/qcom/icc-rpm.c
@@ -58,15 +58,32 @@ static int qcom_icc_set_qnoc_qos(struct icc_node *src)
 	struct qcom_icc_qos *qos = &qn->qos;
 	int rc;
 
+	rc = regmap_update_bits(qp->regmap,
+				qp->qos_offset + QNOC_QOS_MCTL_LOWn_ADDR(qos->qos_port),
+				QNOC_QOS_MCTL_DFLT_PRIO_MASK,
+				qos->areq_prio << QNOC_QOS_MCTL_DFLT_PRIO_SHIFT);
+	if (rc)
+		return rc;
+
 	rc = regmap_update_bits(qp->regmap,
 			qp->qos_offset + QNOC_QOS_MCTL_LOWn_ADDR(qos->qos_port),
-			QNOC_QOS_MCTL_DFLT_PRIO_MASK,
-			qos->areq_prio << QNOC_QOS_MCTL_DFLT_PRIO_SHIFT);
+			QNOC_QOS_MCTL_URGFWD_EN_MASK,
+			!!qos->urg_fwd_en << QNOC_QOS_MCTL_URGFWD_EN_SHIFT);
+	if (rc)
+		return rc;
+
+	if (!qos->aux_qos_port)
+		return 0;
+
+	rc = regmap_update_bits(qp->regmap,
+				qp->qos_offset + QNOC_QOS_MCTL_LOWn_ADDR(qos->aux_qos_port),
+				QNOC_QOS_MCTL_DFLT_PRIO_MASK,
+				qos->areq_prio << QNOC_QOS_MCTL_DFLT_PRIO_SHIFT);
 	if (rc)
 		return rc;
 
 	return regmap_update_bits(qp->regmap,
-			qp->qos_offset + QNOC_QOS_MCTL_LOWn_ADDR(qos->qos_port),
+			qp->qos_offset + QNOC_QOS_MCTL_LOWn_ADDR(qos->aux_qos_port),
 			QNOC_QOS_MCTL_URGFWD_EN_MASK,
 			!!qos->urg_fwd_en << QNOC_QOS_MCTL_URGFWD_EN_SHIFT);
 }
@@ -75,8 +92,9 @@ static int qcom_icc_bimc_set_qos_health(struct qcom_icc_provider *qp,
 					struct qcom_icc_qos *qos,
 					int regnum)
 {
-	u32 val;
 	u32 mask;
+	u32 val;
+	int ret;
 
 	val = qos->prio_level;
 	mask = M_BKE_HEALTH_CFG_PRIOLVL_MASK;
@@ -90,8 +108,17 @@ static int qcom_icc_bimc_set_qos_health(struct qcom_icc_provider *qp,
 		mask |= M_BKE_HEALTH_CFG_LIMITCMDS_MASK;
 	}
 
+	ret = regmap_update_bits(qp->regmap,
+				 qp->qos_offset + M_BKE_HEALTH_CFG_ADDR(regnum, qos->qos_port),
+				 mask, val);
+	if (ret)
+		return ret;
+
+	if (!qos->aux_qos_port)
+		return 0;
+
 	return regmap_update_bits(qp->regmap,
-				  qp->qos_offset + M_BKE_HEALTH_CFG_ADDR(regnum, qos->qos_port),
+				  qp->qos_offset + M_BKE_HEALTH_CFG_ADDR(regnum, qos->aux_qos_port),
 				  mask, val);
 }
 
@@ -102,7 +129,7 @@ static int qcom_icc_set_bimc_qos(struct icc_node *src)
 	struct icc_provider *provider;
 	u32 mode = NOC_QOS_MODE_BYPASS;
 	u32 val = 0;
-	int i, rc = 0;
+	int i, rc;
 
 	qn = src->data;
 	provider = src->provider;
@@ -116,8 +143,7 @@ static int qcom_icc_set_bimc_qos(struct icc_node *src)
 	 */
 	if (mode != NOC_QOS_MODE_BYPASS) {
 		for (i = 3; i >= 0; i--) {
-			rc = qcom_icc_bimc_set_qos_health(qp,
-							  &qn->qos, i);
+			rc = qcom_icc_bimc_set_qos_health(qp, &qn->qos, i);
 			if (rc)
 				return rc;
 		}
@@ -126,8 +152,17 @@ static int qcom_icc_set_bimc_qos(struct icc_node *src)
 		val = 1;
 	}
 
+	rc = regmap_update_bits(qp->regmap,
+				qp->qos_offset + M_BKE_EN_ADDR(qn->qos.qos_port),
+				M_BKE_EN_EN_BMASK, val);
+	if (rc)
+		return rc;
+
+	if (!qn->qos.aux_qos_port)
+		return 0;
+
 	return regmap_update_bits(qp->regmap,
-				  qp->qos_offset + M_BKE_EN_ADDR(qn->qos.qos_port),
+				  qp->qos_offset + M_BKE_EN_ADDR(qn->qos.aux_qos_port),
 				  M_BKE_EN_EN_BMASK, val);
 }
 
@@ -145,8 +180,24 @@ static int qcom_icc_noc_set_qos_priority(struct qcom_icc_provider *qp,
 	if (rc)
 		return rc;
 
+	rc = regmap_update_bits(qp->regmap,
+				qp->qos_offset + NOC_QOS_PRIORITYn_ADDR(qos->qos_port),
+				NOC_QOS_PRIORITY_P0_MASK, qos->prio_level);
+	if (rc)
+		return rc;
+
+	if (!qos->aux_qos_port)
+		return 0;
+
+	val = qos->areq_prio << NOC_QOS_PRIORITY_P1_SHIFT;
+	rc = regmap_update_bits(qp->regmap,
+				qp->qos_offset + NOC_QOS_PRIORITYn_ADDR(qos->aux_qos_port),
+				NOC_QOS_PRIORITY_P1_MASK, val);
+	if (rc)
+		return rc;
+
 	return regmap_update_bits(qp->regmap,
-				  qp->qos_offset + NOC_QOS_PRIORITYn_ADDR(qos->qos_port),
+				  qp->qos_offset + NOC_QOS_PRIORITYn_ADDR(qos->aux_qos_port),
 				  NOC_QOS_PRIORITY_P0_MASK, qos->prio_level);
 }
 
@@ -182,8 +233,17 @@ static int qcom_icc_set_noc_qos(struct icc_node *src)
 		/* How did we get here? */
 	}
 
+	rc = regmap_update_bits(qp->regmap,
+				qp->qos_offset + NOC_QOS_MODEn_ADDR(qn->qos.qos_port),
+				NOC_QOS_MODEn_MASK, mode);
+	if (rc)
+		return rc;
+
+	if (!qn->qos.aux_qos_port)
+		return 0;
+
 	return regmap_update_bits(qp->regmap,
-				  qp->qos_offset + NOC_QOS_MODEn_ADDR(qn->qos.qos_port),
+				  qp->qos_offset + NOC_QOS_MODEn_ADDR(qn->qos.aux_qos_port),
 				  NOC_QOS_MODEn_MASK, mode);
 }
 
@@ -298,25 +358,25 @@ static u64 qcom_icc_calc_rate(struct qcom_icc_provider *qp, struct qcom_icc_node
 	u64 agg_avg_rate, agg_peak_rate, agg_rate;
 
 	if (qn->channels)
-		agg_avg_rate = div_u64(qn->sum_avg[ctx], qn->channels);
+		agg_avg_rate = qcom_bw_div(qn->sum_avg[ctx], qn->channels);
 	else
 		agg_avg_rate = qn->sum_avg[ctx];
 
 	if (qn->ab_coeff) {
 		agg_avg_rate = agg_avg_rate * qn->ab_coeff;
-		agg_avg_rate = div_u64(agg_avg_rate, 100);
+		agg_avg_rate = qcom_bw_div(agg_avg_rate, 100);
 	}
 
 	if (qn->ib_coeff) {
 		agg_peak_rate = qn->max_peak[ctx] * 100;
-		agg_peak_rate = div_u64(agg_peak_rate, qn->ib_coeff);
+		agg_peak_rate = qcom_bw_div(agg_peak_rate, qn->ib_coeff);
 	} else {
 		agg_peak_rate = qn->max_peak[ctx];
 	}
 
 	agg_rate = max_t(u64, agg_avg_rate, agg_peak_rate);
 
-	return div_u64(agg_rate, qn->buswidth);
+	return qcom_bw_div(agg_rate, qn->buswidth);
 }
 
 /**
@@ -402,7 +462,7 @@ static int qcom_icc_set(struct icc_node *src, struct icc_node *dst)
 		if (ret)
 			return ret;
 
-		/* Cache the rate after we've successfully commited it to RPM */
+		/* Cache the rate after we've successfully committed it to RPM */
 		qp->bus_clk_rate[QCOM_SMD_RPM_ACTIVE_STATE] = active_rate;
 	}
 
@@ -412,7 +472,7 @@ static int qcom_icc_set(struct icc_node *src, struct icc_node *dst)
 		if (ret)
 			return ret;
 
-		/* Cache the rate after we've successfully commited it to RPM */
+		/* Cache the rate after we've successfully committed it to RPM */
 		qp->bus_clk_rate[QCOM_SMD_RPM_SLEEP_STATE] = sleep_rate;
 	}
 
@@ -583,8 +643,15 @@ int qnoc_probe(struct platform_device *pdev)
 		node->data = qnodes[i];
 		icc_node_add(node, provider);
 
-		for (j = 0; j < qnodes[i]->num_links; j++)
-			icc_link_create(node, qnodes[i]->links[j]);
+		for (j = 0; j < qnodes[i]->num_links; j++) {
+			ret = icc_link_create(node, qnodes[i]->links[j]);
+			if (ret) {
+				icc_nodes_remove(provider);
+				clk_bulk_disable_unprepare(qp->num_intf_clks,
+							   qp->intf_clks);
+				goto err_disable_unprepare_clk;
+			}
+		}
 
 		/* Set QoS registers (we only need to do it once, generally) */
 		if (qnodes[i]->qos.ap_owned &&
diff --git a/drivers/interconnect/qcom/icc-rpm.h b/drivers/interconnect/qcom/icc-rpm.h
index cdd5f444b1520b..d2c5f45eeedf4c 100644
--- a/drivers/interconnect/qcom/icc-rpm.h
+++ b/drivers/interconnect/qcom/icc-rpm.h
@@ -30,7 +30,7 @@ enum qcom_icc_type {
  * @resource_type: RPM resource type of the clock resource
  * @clock_id: index of the clock resource of a specific resource type
  * @branch: whether the resource represents a branch clock
-*/
+ */
 struct rpm_clk_resource {
 	u32 resource_type;
 	u32 clock_id;
@@ -77,6 +77,7 @@ struct qcom_icc_provider {
  * @ap_owned: indicates if the node is owned by the AP or by the RPM
  * @qos_mode: default qos mode for this node
  * @qos_port: qos port number for finding qos registers of this node
+ * @aux_qos_port: optional QoS port in addition to @qos_port. Zero means "none".
  * @urg_fwd_en: enable urgent forwarding
  */
 struct qcom_icc_qos {
@@ -86,6 +87,7 @@ struct qcom_icc_qos {
 	bool ap_owned;
 	int qos_mode;
 	int qos_port;
+	u16 aux_qos_port;
 	bool urg_fwd_en;
 };
 
@@ -167,7 +169,7 @@ int qnoc_probe(struct platform_device *pdev);
 void qnoc_remove(struct platform_device *pdev);
 
 bool qcom_icc_rpm_smd_available(void);
-int qcom_icc_rpm_smd_send(int ctx, int rsc_type, int id, u32 val);
+int qcom_icc_rpm_smd_send(int ctx, int rsc_type, int id, u64 val);
 int qcom_icc_rpm_set_bus_rate(const struct rpm_clk_resource *clk, int ctx, u32 rate);
 
 #endif
diff --git a/drivers/interconnect/qcom/icc-rpmh.c b/drivers/interconnect/qcom/icc-rpmh.c
index 3b445acefece7f..4a2a746a4e59db 100644
--- a/drivers/interconnect/qcom/icc-rpmh.c
+++ b/drivers/interconnect/qcom/icc-rpmh.c
@@ -136,6 +136,61 @@ int qcom_icc_set(struct icc_node *src, struct icc_node *dst)
 }
 EXPORT_SYMBOL_GPL(qcom_icc_set);
 
+static int qcom_icc_get_bw(struct icc_node *node, u32 *avg, u32 *peak)
+{
+	struct qcom_icc_node *qn = node->data;
+	u32 avg_max = 0;
+	u32 peak_max = 0;
+	u64 x, y;
+	int i;
+
+	if (!qn->num_bcms) {
+		*avg = INT_MAX;
+		*peak = INT_MAX;
+
+		return 0;
+	}
+
+	for (i = 0; i < qn->num_bcms; ++i) {
+		struct qcom_icc_bcm *bcm = qn->bcms[i];
+
+		/* Use AMC vote for boot-up */
+		x = bcm->vote_x[QCOM_ICC_BUCKET_AMC];
+		y = bcm->vote_y[QCOM_ICC_BUCKET_AMC];
+
+		/* Consider enable mask and convert to INT_MAX */
+		if (bcm->enable_mask) {
+			if (x & bcm->enable_mask)
+				avg_max = INT_MAX;
+			if (y & bcm->enable_mask)
+				peak_max = INT_MAX;
+		} else {
+			if (x) {
+				x *= le32_to_cpu(bcm->aux_data.unit);
+				do_div(x, bcm->vote_scale);
+				x *= qn->buswidth * qn->channels;
+				do_div(x, le16_to_cpu(bcm->aux_data.width));
+
+				avg_max = max(avg_max, x);
+			}
+
+			if (y) {
+				y *= le32_to_cpu(bcm->aux_data.unit);
+				do_div(y, bcm->vote_scale);
+				y *= qn->buswidth;
+				do_div(y, le16_to_cpu(bcm->aux_data.width));
+
+				peak_max = max(peak_max, y);
+			}
+		}
+	}
+
+	*avg = avg_max;
+	*peak = peak_max;
+
+	return 0;
+}
+
 /**
  * qcom_icc_bcm_init - populates bcm aux data and connect qnodes
  * @bcm: bcm to be initialized
@@ -173,10 +228,7 @@ int qcom_icc_bcm_init(struct qcom_icc_bcm *bcm, struct device *dev)
 		return -EINVAL;
 	}
 
-	bcm->aux_data.unit = le32_to_cpu(data->unit);
-	bcm->aux_data.width = le16_to_cpu(data->width);
-	bcm->aux_data.vcd = data->vcd;
-	bcm->aux_data.reserved = data->reserved;
+	bcm->aux_data = *data;
 	INIT_LIST_HEAD(&bcm->list);
 	INIT_LIST_HEAD(&bcm->ws_list);
 
@@ -255,6 +307,13 @@ int qcom_icc_rpmh_probe(struct platform_device *pdev)
 	provider = &qp->provider;
 	provider->dev = dev;
 	provider->set = qcom_icc_set;
+	/*
+	 * Programming the QoS registers below requires the NoC to be clocked,
+	 * which during probe is only guaranteed by the INT_MAX initial votes
+	 * the interconnect core applies when get_bw() is absent.
+	 */
+	if (!desc->config)
+		provider->get_bw = qcom_icc_get_bw;
 	provider->pre_aggregate = qcom_icc_pre_aggregate;
 	provider->aggregate = qcom_icc_aggregate;
 	provider->xlate_extended = qcom_icc_xlate_extended;
@@ -272,8 +331,10 @@ int qcom_icc_rpmh_probe(struct platform_device *pdev)
 	if (IS_ERR(qp->voter))
 		return PTR_ERR(qp->voter);
 
-	for (i = 0; i < qp->num_bcms; i++)
+	for (i = 0; i < qp->num_bcms; i++) {
 		qcom_icc_bcm_init(qp->bcms[i], dev);
+		qcom_icc_bcm_get_bw(qp->voter, qp->bcms[i]);
+	}
 
 	for (i = 0; i < num_nodes; i++) {
 		qn = qnodes[i];
diff --git a/drivers/interconnect/qcom/kuno.c b/drivers/interconnect/qcom/kuno.c
new file mode 100644
index 00000000000000..546622cf8ceaa2
--- /dev/null
+++ b/drivers/interconnect/qcom/kuno.c
@@ -0,0 +1,988 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/device.h>
+#include <linux/interconnect.h>
+#include <linux/interconnect-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/interconnect/qcom,kuno.h>
+
+#include "bcm-voter.h"
+#include "icc-rpmh.h"
+
+static struct qcom_icc_node qup0_core_slave = {
+	.name = "qup0_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_ahb2_phy = {
+	.name = "qhs_ahb2_phy",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_aoss = {
+	.name = "qhs_aoss",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_apss = {
+	.name = "qhs_apss",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_audio = {
+	.name = "qhs_audio",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_clk_ctl = {
+	.name = "qhs_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_cpr_cx = {
+	.name = "qhs_cpr_cx",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_cpr_mxa = {
+	.name = "qhs_cpr_mxa",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_cpr_mxc = {
+	.name = "qhs_cpr_mxc",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_crypto_cfg = {
+	.name = "qhs_crypto_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_emac0_cfg = {
+	.name = "qhs_emac0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_imem_cfg = {
+	.name = "qhs_imem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_ipa = {
+	.name = "qhs_ipa",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_ipc_router = {
+	.name = "qhs_ipc_router",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_mss_cfg = {
+	.name = "qhs_mss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_pcie0_cfg = {
+	.name = "qhs_pcie0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_pcie_rscc = {
+	.name = "qhs_pcie_rscc",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_pdm = {
+	.name = "qhs_pdm",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_pmu_wrapper_cfg = {
+	.name = "qhs_pmu_wrapper_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_prng = {
+	.name = "qhs_prng",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_qdss_cfg = {
+	.name = "qhs_qdss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_qpic = {
+	.name = "qhs_qpic",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_qup0 = {
+	.name = "qhs_qup0",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_sdc4 = {
+	.name = "qhs_sdc4",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_spmi_vgi_coex = {
+	.name = "qhs_spmi_vgi_coex",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_tcsr = {
+	.name = "qhs_tcsr",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_tlmm = {
+	.name = "qhs_tlmm",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_tme_cfg = {
+	.name = "qhs_tme_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_usb3 = {
+	.name = "qhs_usb3",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_vsense_ctrl_cfg = {
+	.name = "qhs_vsense_ctrl_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qss_anoc_throttle_cfg = {
+	.name = "qss_anoc_throttle_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qxs_boot_imem = {
+	.name = "qxs_boot_imem",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qxs_imem = {
+	.name = "qxs_imem",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node xs_qdss_stm = {
+	.name = "xs_qdss_stm",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node xs_sys_tcu_cfg = {
+	.name = "xs_sys_tcu_cfg",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_lagg = {
+	.name = "qhs_lagg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qhs_mccc_master = {
+	.name = "qhs_mccc_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node ebi = {
+	.name = "ebi",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node srvc_memnoc = {
+	.name = "srvc_memnoc",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node xs_pcie_0 = {
+	.name = "xs_pcie_0",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 0,
+};
+
+static struct qcom_icc_node qup0_core_master = {
+	.name = "qup0_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_node qnm_cnoc = {
+	.name = "qnm_cnoc",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 2,
+	.link_nodes = { &qhs_lagg, &qhs_mccc_master },
+};
+
+static struct qcom_icc_node llcc_mc = {
+	.name = "llcc_mc",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ebi },
+};
+
+static struct qcom_icc_node qnm_memnoc_cfg = {
+	.name = "qnm_memnoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &srvc_memnoc },
+};
+
+static struct qcom_icc_node qnm_memnoc_pcie = {
+	.name = "qnm_memnoc_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &xs_pcie_0 },
+};
+
+static struct qcom_icc_node xm_ipa2pcie = {
+	.name = "xm_ipa2pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &xs_pcie_0 },
+};
+
+static struct qcom_icc_node qns_ddrss_cfg = {
+	.name = "qns_ddrss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qnm_cnoc },
+};
+
+static struct qcom_icc_node qns_llcc = {
+	.name = "qns_llcc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &llcc_mc },
+};
+
+static struct qcom_icc_node qns_pcie = {
+	.name = "qns_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_memnoc_pcie },
+};
+
+static struct qcom_icc_node qhm_pcie_rscc = {
+	.name = "qhm_pcie_rscc",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 34,
+	.link_nodes = { &qhs_ahb2_phy, &qhs_aoss,
+			&qhs_apss, &qhs_audio,
+			&qhs_clk_ctl, &qhs_cpr_cx,
+			&qhs_cpr_mxa, &qhs_cpr_mxc,
+			&qhs_crypto_cfg, &qhs_emac0_cfg,
+			&qhs_imem_cfg, &qhs_ipa,
+			&qhs_ipc_router, &qhs_mss_cfg,
+			&qhs_pcie0_cfg, &qhs_pdm,
+			&qhs_pmu_wrapper_cfg, &qhs_prng,
+			&qhs_qdss_cfg, &qhs_qpic,
+			&qhs_qup0, &qhs_sdc4,
+			&qhs_spmi_vgi_coex, &qhs_tcsr,
+			&qhs_tlmm, &qhs_tme_cfg,
+			&qhs_usb3, &qhs_vsense_ctrl_cfg,
+			&qns_ddrss_cfg, &qss_anoc_throttle_cfg,
+			&qxs_boot_imem, &qxs_imem,
+			&xs_qdss_stm, &xs_sys_tcu_cfg },
+};
+
+static struct qcom_icc_node qnm_memnoc_cnoc = {
+	.name = "qnm_memnoc_cnoc",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 35,
+	.link_nodes = { &qhs_ahb2_phy, &qhs_aoss,
+			&qhs_apss, &qhs_audio,
+			&qhs_clk_ctl, &qhs_cpr_cx,
+			&qhs_cpr_mxa, &qhs_cpr_mxc,
+			&qhs_crypto_cfg, &qhs_emac0_cfg,
+			&qhs_imem_cfg, &qhs_ipa,
+			&qhs_ipc_router, &qhs_mss_cfg,
+			&qhs_pcie0_cfg, &qhs_pcie_rscc,
+			&qhs_pdm, &qhs_pmu_wrapper_cfg,
+			&qhs_prng, &qhs_qdss_cfg,
+			&qhs_qpic, &qhs_qup0,
+			&qhs_sdc4, &qhs_spmi_vgi_coex,
+			&qhs_tcsr, &qhs_tlmm,
+			&qhs_tme_cfg, &qhs_usb3,
+			&qhs_vsense_ctrl_cfg, &qns_ddrss_cfg,
+			&qss_anoc_throttle_cfg, &qxs_boot_imem,
+			&qxs_imem, &xs_qdss_stm,
+			&xs_sys_tcu_cfg },
+};
+
+static struct qcom_icc_node qns_memnoc_cnoc = {
+	.name = "qns_memnoc_cnoc",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_memnoc_cnoc },
+};
+
+static struct qcom_icc_node alm_sys_tcu = {
+	.name = "alm_sys_tcu",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &qns_llcc, &qns_memnoc_cnoc },
+};
+
+static struct qcom_icc_node qnm_mdsp = {
+	.name = "qnm_mdsp",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 3,
+	.link_nodes = { &qns_llcc, &qns_memnoc_cnoc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_pcie = {
+	.name = "qnm_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &qns_llcc, &qns_memnoc_cnoc },
+};
+
+static struct qcom_icc_node qnm_snoc_sf = {
+	.name = "qnm_snoc_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &qns_llcc, &qns_memnoc_cnoc },
+};
+
+static struct qcom_icc_node xm_apps0 = {
+	.name = "xm_apps0",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 3,
+	.link_nodes = { &qns_llcc, &qns_memnoc_cnoc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qns_pcie_memnoc = {
+	.name = "qns_pcie_memnoc",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_pcie },
+};
+
+static struct qcom_icc_node qns_memnoc_sf = {
+	.name = "qns_memnoc_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_snoc_sf },
+};
+
+static struct qcom_icc_node xm_pcie3_0 = {
+	.name = "xm_pcie3_0",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_memnoc },
+};
+
+static struct qcom_icc_node qhm_cpucp = {
+	.name = "qhm_cpucp",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qns_memnoc_sf },
+};
+
+static struct qcom_icc_node qnm_aggre_noc = {
+	.name = "qnm_aggre_noc",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &qns_memnoc_sf, &xs_pcie_0 },
+};
+
+static struct qcom_icc_node qnm_cnoc_datapath = {
+	.name = "qnm_cnoc_datapath",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 2,
+	.link_nodes = { &qns_memnoc_sf, &xs_pcie_0 },
+};
+
+static struct qcom_icc_node qxm_mss_nav_ce = {
+	.name = "qxm_mss_nav_ce",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &qns_memnoc_sf, &xs_pcie_0 },
+};
+
+static struct qcom_icc_node qxm_tme = {
+	.name = "qxm_tme",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &qns_memnoc_sf, &xs_pcie_0 },
+};
+
+static struct qcom_icc_node qns_a1noc = {
+	.name = "qns_a1noc",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre_noc },
+};
+
+static struct qcom_icc_node qns_snoc_datapath = {
+	.name = "qns_snoc_datapath",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qnm_cnoc_datapath },
+};
+
+static struct qcom_icc_node qhm_audio = {
+	.name = "qhm_audio",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node qhm_qdss_bam = {
+	.name = "qhm_qdss_bam",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node qhm_qpic = {
+	.name = "qhm_qpic",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node qhm_qup0 = {
+	.name = "qhm_qup0",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node qxm_crypto = {
+	.name = "qxm_crypto",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node qxm_ipa = {
+	.name = "qxm_ipa",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node xm_emac_0 = {
+	.name = "xm_emac_0",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node xm_qdss_etr0 = {
+	.name = "xm_qdss_etr0",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node xm_qdss_etr1 = {
+	.name = "xm_qdss_etr1",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node xm_sdc4 = {
+	.name = "xm_sdc4",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node xm_usb3 = {
+	.name = "xm_usb3",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc },
+};
+
+static struct qcom_icc_node xm_qdss_dap = {
+	.name = "xm_qdss_dap",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 36,
+	.link_nodes = { &qhs_ahb2_phy, &qhs_aoss,
+			&qhs_apss, &qhs_audio,
+			&qhs_clk_ctl, &qhs_cpr_cx,
+			&qhs_cpr_mxa, &qhs_cpr_mxc,
+			&qhs_crypto_cfg, &qhs_emac0_cfg,
+			&qhs_imem_cfg, &qhs_ipa,
+			&qhs_ipc_router, &qhs_mss_cfg,
+			&qhs_pcie0_cfg, &qhs_pcie_rscc,
+			&qhs_pdm, &qhs_pmu_wrapper_cfg,
+			&qhs_prng, &qhs_qdss_cfg,
+			&qhs_qpic, &qhs_qup0,
+			&qhs_sdc4, &qhs_spmi_vgi_coex,
+			&qhs_tcsr, &qhs_tlmm,
+			&qhs_tme_cfg, &qhs_usb3,
+			&qhs_vsense_ctrl_cfg, &qns_ddrss_cfg,
+			&qns_snoc_datapath, &qss_anoc_throttle_cfg,
+			&qxs_boot_imem, &qxs_imem,
+			&xs_qdss_stm, &xs_sys_tcu_cfg },
+};
+
+static struct qcom_icc_bcm bcm_acv = {
+	.name = "ACV",
+	.enable_mask = 0x8,
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_ce0 = {
+	.name = "CE0",
+	.num_nodes = 1,
+	.nodes = { &qxm_crypto },
+};
+
+static struct qcom_icc_bcm bcm_cn0 = {
+	.name = "CN0",
+	.keepalive = true,
+	.enable_mask = 0x1,
+	.num_nodes = 39,
+	.nodes = { &qhm_pcie_rscc, &qnm_memnoc_cnoc,
+		   &xm_qdss_dap, &qhs_ahb2_phy,
+		   &qhs_aoss, &qhs_apss,
+		   &qhs_audio, &qhs_clk_ctl,
+		   &qhs_cpr_cx, &qhs_cpr_mxa,
+		   &qhs_cpr_mxc, &qhs_crypto_cfg,
+		   &qhs_emac0_cfg, &qhs_imem_cfg,
+		   &qhs_ipa, &qhs_ipc_router,
+		   &qhs_mss_cfg, &qhs_pcie0_cfg,
+		   &qhs_pcie_rscc, &qhs_pdm,
+		   &qhs_pmu_wrapper_cfg, &qhs_prng,
+		   &qhs_qdss_cfg, &qhs_qpic,
+		   &qhs_qup0, &qhs_sdc4,
+		   &qhs_spmi_vgi_coex, &qhs_tcsr,
+		   &qhs_tlmm, &qhs_tme_cfg,
+		   &qhs_usb3, &qhs_vsense_ctrl_cfg,
+		   &qns_ddrss_cfg, &qns_snoc_datapath,
+		   &qss_anoc_throttle_cfg, &qxs_boot_imem,
+		   &qxs_imem, &xs_qdss_stm,
+		   &xs_sys_tcu_cfg },
+};
+
+static struct qcom_icc_bcm bcm_mc0 = {
+	.name = "MC0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_qup0 = {
+	.name = "QUP0",
+	.vote_scale = 1,
+	.num_nodes = 1,
+	.nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_sh0 = {
+	.name = "SH0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_bcm bcm_sh1 = {
+	.name = "SH1",
+	.enable_mask = 0x1,
+	.num_nodes = 2,
+	.nodes = { &qns_memnoc_cnoc, &qns_pcie },
+};
+
+static struct qcom_icc_bcm bcm_sn0 = {
+	.name = "SN0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_memnoc_sf },
+};
+
+static struct qcom_icc_bcm bcm_sn1 = {
+	.name = "SN1",
+	.enable_mask = 0x1,
+	.num_nodes = 1,
+	.nodes = { &xm_pcie3_0 },
+};
+
+static struct qcom_icc_bcm bcm_sn2 = {
+	.name = "SN2",
+	.num_nodes = 2,
+	.nodes = { &qns_a1noc, &qnm_aggre_noc },
+};
+
+static struct qcom_icc_bcm bcm_sn4 = {
+	.name = "SN4",
+	.num_nodes = 1,
+	.nodes = { &qxm_mss_nav_ce },
+};
+
+static struct qcom_icc_bcm bcm_sn7 = {
+	.name = "SN7",
+	.num_nodes = 1,
+	.nodes = { &xs_pcie_0 },
+};
+
+static struct qcom_icc_bcm * const aggre_noc_bcms[] = {
+	&bcm_ce0,
+};
+
+static struct qcom_icc_node * const aggre_noc_nodes[] = {
+	[MASTER_AUDIO] = &qhm_audio,
+	[MASTER_QDSS_BAM] = &qhm_qdss_bam,
+	[MASTER_QPIC] = &qhm_qpic,
+	[MASTER_QUP_0] = &qhm_qup0,
+	[MASTER_CRYPTO] = &qxm_crypto,
+	[MASTER_IPA] = &qxm_ipa,
+	[MASTER_EMAC] = &xm_emac_0,
+	[MASTER_QDSS_ETR] = &xm_qdss_etr0,
+	[MASTER_QDSS_ETR_1] = &xm_qdss_etr1,
+	[MASTER_SDCC_4] = &xm_sdc4,
+	[MASTER_USB3_0] = &xm_usb3,
+	[SLAVE_A1NOC_CFG] = &qns_a1noc,
+};
+
+static const struct qcom_icc_desc kuno_aggre_noc = {
+	.nodes = aggre_noc_nodes,
+	.num_nodes = ARRAY_SIZE(aggre_noc_nodes),
+	.bcms = aggre_noc_bcms,
+	.num_bcms = ARRAY_SIZE(aggre_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const clk_virt_bcms[] = {
+	&bcm_qup0,
+};
+
+static struct qcom_icc_node * const clk_virt_nodes[] = {
+	[MASTER_QUP_CORE_0] = &qup0_core_master,
+	[SLAVE_QUP_CORE_0] = &qup0_core_slave,
+};
+
+static const struct qcom_icc_desc kuno_clk_virt = {
+	.nodes = clk_virt_nodes,
+	.num_nodes = ARRAY_SIZE(clk_virt_nodes),
+	.bcms = clk_virt_bcms,
+	.num_bcms = ARRAY_SIZE(clk_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const cnoc_main_bcms[] = {
+	&bcm_cn0,
+};
+
+static struct qcom_icc_node * const cnoc_main_nodes[] = {
+	[MASTER_PCIE_RSCC] = &qhm_pcie_rscc,
+	[MASTER_GEM_NOC_CNOC] = &qnm_memnoc_cnoc,
+	[MASTER_QDSS_DAP] = &xm_qdss_dap,
+	[SLAVE_AHB2PHY] = &qhs_ahb2_phy,
+	[SLAVE_AOSS] = &qhs_aoss,
+	[SLAVE_APPSS] = &qhs_apss,
+	[SLAVE_AUDIO] = &qhs_audio,
+	[SLAVE_CLK_CTL] = &qhs_clk_ctl,
+	[SLAVE_RBCPR_CX_CFG] = &qhs_cpr_cx,
+	[SLAVE_RBCPR_MXA_CFG] = &qhs_cpr_mxa,
+	[SLAVE_RBCPR_MXC_CFG] = &qhs_cpr_mxc,
+	[SLAVE_CRYPTO_0_CFG] = &qhs_crypto_cfg,
+	[SLAVE_EMAC_CFG] = &qhs_emac0_cfg,
+	[SLAVE_IMEM_CFG] = &qhs_imem_cfg,
+	[SLAVE_IPA_CFG] = &qhs_ipa,
+	[SLAVE_IPC_ROUTER_CFG] = &qhs_ipc_router,
+	[SLAVE_CNOC_MSS] = &qhs_mss_cfg,
+	[SLAVE_PCIE_0_CFG] = &qhs_pcie0_cfg,
+	[SLAVE_PCIE_RSC_CFG] = &qhs_pcie_rscc,
+	[SLAVE_PDM] = &qhs_pdm,
+	[SLAVE_PMU_WRAPPER_CFG] = &qhs_pmu_wrapper_cfg,
+	[SLAVE_PRNG] = &qhs_prng,
+	[SLAVE_QDSS_CFG] = &qhs_qdss_cfg,
+	[SLAVE_QPIC] = &qhs_qpic,
+	[SLAVE_QUP_0] = &qhs_qup0,
+	[SLAVE_SDCC_4] = &qhs_sdc4,
+	[SLAVE_SPMI_VGI_COEX] = &qhs_spmi_vgi_coex,
+	[SLAVE_TCSR] = &qhs_tcsr,
+	[SLAVE_TLMM] = &qhs_tlmm,
+	[SLAVE_TME_CFG] = &qhs_tme_cfg,
+	[SLAVE_USB3] = &qhs_usb3,
+	[SLAVE_VSENSE_CTRL_CFG] = &qhs_vsense_ctrl_cfg,
+	[SLAVE_DDRSS_CFG] = &qns_ddrss_cfg,
+	[SLAVE_SNOC_CNOC] = &qns_snoc_datapath,
+	[SLAVE_ANOC_THROTTLE_CFG] = &qss_anoc_throttle_cfg,
+	[SLAVE_BOOT_IMEM] = &qxs_boot_imem,
+	[SLAVE_IMEM] = &qxs_imem,
+	[SLAVE_QDSS_STM] = &xs_qdss_stm,
+	[SLAVE_TCU] = &xs_sys_tcu_cfg,
+};
+
+static const struct qcom_icc_desc kuno_cnoc_main = {
+	.nodes = cnoc_main_nodes,
+	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
+	.bcms = cnoc_main_bcms,
+	.num_bcms = ARRAY_SIZE(cnoc_main_bcms),
+};
+
+static struct qcom_icc_node * const dc_noc_nodes[] = {
+	[MASTER_CNOC_DC_NOC] = &qnm_cnoc,
+	[SLAVE_LAGG_CFG] = &qhs_lagg,
+	[SLAVE_MCCC_MASTER] = &qhs_mccc_master,
+};
+
+static const struct qcom_icc_desc kuno_dc_noc = {
+	.nodes = dc_noc_nodes,
+	.num_nodes = ARRAY_SIZE(dc_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const mc_virt_bcms[] = {
+	&bcm_acv,
+	&bcm_mc0,
+};
+
+static struct qcom_icc_node * const mc_virt_nodes[] = {
+	[MASTER_LLCC] = &llcc_mc,
+	[SLAVE_EBI1] = &ebi,
+};
+
+static const struct qcom_icc_desc kuno_mc_virt = {
+	.nodes = mc_virt_nodes,
+	.num_nodes = ARRAY_SIZE(mc_virt_nodes),
+	.bcms = mc_virt_bcms,
+	.num_bcms = ARRAY_SIZE(mc_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const mem_noc_bcms[] = {
+	&bcm_sh0,
+	&bcm_sh1,
+};
+
+static struct qcom_icc_node * const mem_noc_nodes[] = {
+	[MASTER_SYS_TCU] = &alm_sys_tcu,
+	[MASTER_MSS_PROC] = &qnm_mdsp,
+	[MASTER_GEM_NOC_CFG] = &qnm_memnoc_cfg,
+	[MASTER_ANOC_PCIE_GEM_NOC] = &qnm_pcie,
+	[MASTER_SNOC_SF_MEM_NOC] = &qnm_snoc_sf,
+	[MASTER_APPSS_PROC] = &xm_apps0,
+	[SLAVE_LLCC] = &qns_llcc,
+	[SLAVE_GEM_NOC_CNOC] = &qns_memnoc_cnoc,
+	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
+	[SLAVE_SERVICE_GEM_NOC] = &srvc_memnoc,
+};
+
+static const struct qcom_icc_desc kuno_mem_noc = {
+	.nodes = mem_noc_nodes,
+	.num_nodes = ARRAY_SIZE(mem_noc_nodes),
+	.bcms = mem_noc_bcms,
+	.num_bcms = ARRAY_SIZE(mem_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const pcie_anoc_bcms[] = {
+	&bcm_sn1,
+};
+
+static struct qcom_icc_node * const pcie_anoc_nodes[] = {
+	[MASTER_PCIE_0] = &xm_pcie3_0,
+	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_memnoc,
+};
+
+static const struct qcom_icc_desc kuno_pcie_anoc = {
+	.nodes = pcie_anoc_nodes,
+	.num_nodes = ARRAY_SIZE(pcie_anoc_nodes),
+	.bcms = pcie_anoc_bcms,
+	.num_bcms = ARRAY_SIZE(pcie_anoc_bcms),
+};
+
+static struct qcom_icc_bcm * const system_noc_bcms[] = {
+	&bcm_sn0,
+	&bcm_sn2,
+	&bcm_sn4,
+	&bcm_sn7,
+};
+
+static struct qcom_icc_node * const system_noc_nodes[] = {
+	[MASTER_CPUCP] = &qhm_cpucp,
+	[MASTER_ANOC_SNOC] = &qnm_aggre_noc,
+	[MASTER_CNOC_SNOC] = &qnm_cnoc_datapath,
+	[MASTER_GEM_NOC_PCIE_SNOC] = &qnm_memnoc_pcie,
+	[MASTER_MSS_NAV] = &qxm_mss_nav_ce,
+	[MASTER_TME] = &qxm_tme,
+	[MASTER_IPA_PCIE] = &xm_ipa2pcie,
+	[SLAVE_SNOC_MEM_NOC_SF] = &qns_memnoc_sf,
+	[SLAVE_PCIE_0] = &xs_pcie_0,
+};
+
+static const struct qcom_icc_desc kuno_system_noc = {
+	.nodes = system_noc_nodes,
+	.num_nodes = ARRAY_SIZE(system_noc_nodes),
+	.bcms = system_noc_bcms,
+	.num_bcms = ARRAY_SIZE(system_noc_bcms),
+};
+
+static const struct of_device_id qnoc_of_match[] = {
+	{ .compatible = "qcom,kuno-aggre-noc", .data = &kuno_aggre_noc },
+	{ .compatible = "qcom,kuno-clk-virt", .data = &kuno_clk_virt },
+	{ .compatible = "qcom,kuno-cnoc-main", .data = &kuno_cnoc_main },
+	{ .compatible = "qcom,kuno-dc-noc", .data = &kuno_dc_noc },
+	{ .compatible = "qcom,kuno-mc-virt", .data = &kuno_mc_virt },
+	{ .compatible = "qcom,kuno-mem-noc", .data = &kuno_mem_noc },
+	{ .compatible = "qcom,kuno-pcie-anoc", .data = &kuno_pcie_anoc },
+	{ .compatible = "qcom,kuno-system-noc", .data = &kuno_system_noc },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qnoc_of_match);
+
+static struct platform_driver qnoc_driver = {
+	.probe = qcom_icc_rpmh_probe,
+	.remove = qcom_icc_rpmh_remove,
+	.driver = {
+		.name = "qnoc-kuno",
+		.of_match_table = qnoc_of_match,
+		.sync_state = icc_sync_state,
+	},
+};
+
+static int __init qnoc_driver_init(void)
+{
+	return platform_driver_register(&qnoc_driver);
+}
+core_initcall(qnoc_driver_init);
+
+static void __exit qnoc_driver_exit(void)
+{
+	platform_driver_unregister(&qnoc_driver);
+}
+module_exit(qnoc_driver_exit);
+
+MODULE_DESCRIPTION("Kuno NoC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/milos.c b/drivers/interconnect/qcom/milos.c
index d010b106728a37..8b5b5ee53e3b2f 100644
--- a/drivers/interconnect/qcom/milos.c
+++ b/drivers/interconnect/qcom/milos.c
@@ -137,7 +137,7 @@ static struct qcom_icc_node srvc_pcie_aggre_noc;
 static struct qcom_icc_node qns_gemnoc_gc;
 static struct qcom_icc_node qns_gemnoc_sf;
 
-static struct qcom_icc_qosbox qhm_qup1_qos = {
+static const struct qcom_icc_qosbox qhm_qup1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc000 },
 	.prio = 2,
@@ -154,7 +154,7 @@ static struct qcom_icc_node qhm_qup1 = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_ufs_mem_qos = {
+static const struct qcom_icc_qosbox xm_ufs_mem_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf200 },
 	.prio = 2,
@@ -171,7 +171,7 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_usb3_0_qos = {
+static const struct qcom_icc_qosbox xm_usb3_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x10000 },
 	.prio = 2,
@@ -188,7 +188,7 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox qhm_qdss_bam_qos = {
+static const struct qcom_icc_qosbox qhm_qdss_bam_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x14000 },
 	.prio = 2,
@@ -205,7 +205,7 @@ static struct qcom_icc_node qhm_qdss_bam = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qhm_qspi_qos = {
+static const struct qcom_icc_qosbox qhm_qspi_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x12000 },
 	.prio = 2,
@@ -222,7 +222,7 @@ static struct qcom_icc_node qhm_qspi = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qhm_qup0_qos = {
+static const struct qcom_icc_qosbox qhm_qup0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x13000 },
 	.prio = 2,
@@ -239,7 +239,7 @@ static struct qcom_icc_node qhm_qup0 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qxm_crypto_qos = {
+static const struct qcom_icc_qosbox qxm_crypto_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x15000 },
 	.prio = 2,
@@ -256,7 +256,7 @@ static struct qcom_icc_node qxm_crypto = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qxm_ipa_qos = {
+static const struct qcom_icc_qosbox qxm_ipa_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x16000 },
 	.prio = 2,
@@ -273,7 +273,7 @@ static struct qcom_icc_node qxm_ipa = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_qdss_etr_0_qos = {
+static const struct qcom_icc_qosbox xm_qdss_etr_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x17000 },
 	.prio = 2,
@@ -290,7 +290,7 @@ static struct qcom_icc_node xm_qdss_etr_0 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_qdss_etr_1_qos = {
+static const struct qcom_icc_qosbox xm_qdss_etr_1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x18000 },
 	.prio = 2,
@@ -307,7 +307,7 @@ static struct qcom_icc_node xm_qdss_etr_1 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_sdc1_qos = {
+static const struct qcom_icc_qosbox xm_sdc1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x1a000 },
 	.prio = 2,
@@ -324,7 +324,7 @@ static struct qcom_icc_node xm_sdc1 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_sdc2_qos = {
+static const struct qcom_icc_qosbox xm_sdc2_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x19000 },
 	.prio = 2,
@@ -404,7 +404,7 @@ static struct qcom_icc_node qnm_gemnoc_pcie = {
 	.link_nodes = { &xs_pcie_0, &xs_pcie_1 },
 };
 
-static struct qcom_icc_qosbox alm_gpu_tcu_qos = {
+static const struct qcom_icc_qosbox alm_gpu_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf1000 },
 	.prio = 1,
@@ -421,7 +421,7 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox alm_sys_tcu_qos = {
+static const struct qcom_icc_qosbox alm_sys_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf3000 },
 	.prio = 6,
@@ -447,7 +447,7 @@ static struct qcom_icc_node chm_apps = {
 		   &qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_gpu_qos = {
+static const struct qcom_icc_qosbox qnm_gpu_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x31000, 0x71000 },
 	.prio = 0,
@@ -464,7 +464,7 @@ static struct qcom_icc_node qnm_gpu = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_lpass_gemnoc_qos = {
+static const struct qcom_icc_qosbox qnm_lpass_gemnoc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf5000 },
 	.prio = 0,
@@ -491,7 +491,7 @@ static struct qcom_icc_node qnm_mdsp = {
 		   &qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x33000, 0x73000 },
 	.prio = 0,
@@ -508,7 +508,7 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x35000, 0x75000 },
 	.prio = 0,
@@ -525,7 +525,7 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_nsp_gemnoc_qos = {
+static const struct qcom_icc_qosbox qnm_nsp_gemnoc_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x37000, 0x77000 },
 	.prio = 0,
@@ -543,7 +543,7 @@ static struct qcom_icc_node qnm_nsp_gemnoc = {
 		   &qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_pcie_qos = {
+static const struct qcom_icc_qosbox qnm_pcie_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf7000 },
 	.prio = 2,
@@ -560,7 +560,7 @@ static struct qcom_icc_node qnm_pcie = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_snoc_gc_qos = {
+static const struct qcom_icc_qosbox qnm_snoc_gc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf9000 },
 	.prio = 0,
@@ -577,7 +577,7 @@ static struct qcom_icc_node qnm_snoc_gc = {
 	.link_nodes = { &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_snoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_snoc_sf_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xfb000 },
 	.prio = 0,
@@ -620,7 +620,7 @@ static struct qcom_icc_node llcc_mc = {
 	.link_nodes = { &ebi },
 };
 
-static struct qcom_icc_qosbox qnm_camnoc_hf_qos = {
+static const struct qcom_icc_qosbox qnm_camnoc_hf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0xa8000, 0xa9000 },
 	.prio = 0,
@@ -637,7 +637,7 @@ static struct qcom_icc_node qnm_camnoc_hf = {
 	.link_nodes = { &qns_mem_noc_hf },
 };
 
-static struct qcom_icc_qosbox qnm_camnoc_icp_qos = {
+static const struct qcom_icc_qosbox qnm_camnoc_icp_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x2a000 },
 	.prio = 5,
@@ -654,7 +654,7 @@ static struct qcom_icc_node qnm_camnoc_icp = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_camnoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_camnoc_sf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x2b000, 0x2c000 },
 	.prio = 0,
@@ -671,7 +671,7 @@ static struct qcom_icc_node qnm_camnoc_sf = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_mdp_qos = {
+static const struct qcom_icc_qosbox qnm_mdp_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xad000 },
 	.prio = 0,
@@ -688,7 +688,7 @@ static struct qcom_icc_node qnm_mdp = {
 	.link_nodes = { &qns_mem_noc_hf },
 };
 
-static struct qcom_icc_qosbox qnm_video_qos = {
+static const struct qcom_icc_qosbox qnm_video_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x30000 },
 	.prio = 0,
@@ -737,7 +737,7 @@ static struct qcom_icc_node qsm_pcie_anoc_cfg = {
 	.link_nodes = { &srvc_pcie_aggre_noc },
 };
 
-static struct qcom_icc_qosbox xm_pcie3_0_qos = {
+static const struct qcom_icc_qosbox xm_pcie3_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xb000 },
 	.prio = 2,
@@ -754,7 +754,7 @@ static struct qcom_icc_node xm_pcie3_0 = {
 	.link_nodes = { &qns_pcie_mem_noc },
 };
 
-static struct qcom_icc_qosbox xm_pcie3_1_qos = {
+static const struct qcom_icc_qosbox xm_pcie3_1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc000 },
 	.prio = 2,
@@ -787,7 +787,7 @@ static struct qcom_icc_node qnm_aggre2_noc = {
 	.link_nodes = { &qns_gemnoc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_apss_noc_qos = {
+static const struct qcom_icc_qosbox qnm_apss_noc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x1c000 },
 	.prio = 2,
@@ -804,7 +804,7 @@ static struct qcom_icc_node qnm_apss_noc = {
 	.link_nodes = { &qns_gemnoc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_cnoc_data_qos = {
+static const struct qcom_icc_qosbox qnm_cnoc_data_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x1d000 },
 	.prio = 2,
@@ -821,7 +821,7 @@ static struct qcom_icc_node qnm_cnoc_data = {
 	.link_nodes = { &qns_gemnoc_sf },
 };
 
-static struct qcom_icc_qosbox qxm_pimem_qos = {
+static const struct qcom_icc_qosbox qxm_pimem_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x1e000 },
 	.prio = 2,
@@ -838,7 +838,7 @@ static struct qcom_icc_node qxm_pimem = {
 	.link_nodes = { &qns_gemnoc_gc },
 };
 
-static struct qcom_icc_qosbox xm_gic_qos = {
+static const struct qcom_icc_qosbox xm_gic_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x1f000 },
 	.prio = 2,
diff --git a/drivers/interconnect/qcom/msm8976.c b/drivers/interconnect/qcom/msm8976.c
index c219dcfd43b6c7..11100264ef691a 100644
--- a/drivers/interconnect/qcom/msm8976.c
+++ b/drivers/interconnect/qcom/msm8976.c
@@ -454,7 +454,8 @@ static struct qcom_icc_node mas_oxili = {
 	.qos.qos_mode = NOC_QOS_MODE_BYPASS,
 	.qos.areq_prio = 0,
 	.qos.prio_level = 0,
-	.qos.qos_port = 16, /* [16, 17] */
+	.qos.qos_port = 16,
+	.qos.aux_qos_port = 17,
 	.num_links = ARRAY_SIZE(mas_oxili_links),
 	.links = mas_oxili_links,
 };
diff --git a/drivers/interconnect/qcom/msm8996.c b/drivers/interconnect/qcom/msm8996.c
index 882f9d2b6d4e18..7fc808cc597575 100644
--- a/drivers/interconnect/qcom/msm8996.c
+++ b/drivers/interconnect/qcom/msm8996.c
@@ -501,6 +501,7 @@ static struct qcom_icc_node mas_venus = {
 	.qos.areq_prio = 0,
 	.qos.prio_level = 0,
 	.qos.qos_port = 3,
+	.qos.aux_qos_port = 4,
 	.num_links = ARRAY_SIZE(mas_mnoc_bimc_common_links),
 	.links = mas_mnoc_bimc_common_links
 };
diff --git a/drivers/interconnect/qcom/sm6350.c b/drivers/interconnect/qcom/sm6350.c
index c098c608b836eb..8680d242ccfab9 100644
--- a/drivers/interconnect/qcom/sm6350.c
+++ b/drivers/interconnect/qcom/sm6350.c
@@ -149,7 +149,7 @@ static struct qcom_icc_node qhm_a1noc_cfg = {
 	.link_nodes = { &srvc_aggre1_noc },
 };
 
-static struct qcom_icc_qosbox qhm_qup_0_qos = {
+static const struct qcom_icc_qosbox qhm_qup_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xa000 },
 	.prio = 2,
@@ -165,7 +165,7 @@ static struct qcom_icc_node qhm_qup_0 = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_emmc_qos = {
+static const struct qcom_icc_qosbox xm_emmc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x7000 },
 	.prio = 2,
@@ -181,7 +181,7 @@ static struct qcom_icc_node xm_emmc = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_ufs_mem_qos = {
+static const struct qcom_icc_qosbox xm_ufs_mem_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x8000 },
 	.prio = 4,
@@ -205,7 +205,7 @@ static struct qcom_icc_node qhm_a2noc_cfg = {
 	.link_nodes = { &srvc_aggre2_noc },
 };
 
-static struct qcom_icc_qosbox qhm_qdss_bam_qos = {
+static const struct qcom_icc_qosbox qhm_qdss_bam_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xb000 },
 	.prio = 2,
@@ -221,7 +221,7 @@ static struct qcom_icc_node qhm_qdss_bam = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qhm_qup_1_qos = {
+static const struct qcom_icc_qosbox qhm_qup_1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x9000 },
 	.prio = 2,
@@ -236,7 +236,7 @@ static struct qcom_icc_node qhm_qup_1 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qxm_crypto_qos = {
+static const struct qcom_icc_qosbox qxm_crypto_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x6000 },
 	.prio = 2,
@@ -252,7 +252,7 @@ static struct qcom_icc_node qxm_crypto = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qxm_ipa_qos = {
+static const struct qcom_icc_qosbox qxm_ipa_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x7000 },
 	.prio = 2,
@@ -268,7 +268,7 @@ static struct qcom_icc_node qxm_ipa = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_qdss_etr_qos = {
+static const struct qcom_icc_qosbox xm_qdss_etr_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc000 },
 	.prio = 2,
@@ -284,7 +284,7 @@ static struct qcom_icc_node xm_qdss_etr = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_sdc2_qos = {
+static const struct qcom_icc_qosbox xm_sdc2_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x18000 },
 	.prio = 2,
@@ -300,7 +300,7 @@ static struct qcom_icc_node xm_sdc2 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_usb3_0_qos = {
+static const struct qcom_icc_qosbox xm_usb3_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xd000 },
 	.prio = 2,
@@ -356,7 +356,7 @@ static struct qcom_icc_node qup1_core_master = {
 	.link_nodes = { &qup1_core_slave },
 };
 
-static struct qcom_icc_qosbox qnm_npu_qos = {
+static const struct qcom_icc_qosbox qnm_npu_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0xf000, 0x11000 },
 	.prio = 0,
@@ -372,7 +372,7 @@ static struct qcom_icc_node qnm_npu = {
 	.link_nodes = { &qns_cdsp_gemnoc },
 };
 
-static struct qcom_icc_qosbox qxm_npu_dsp_qos = {
+static const struct qcom_icc_qosbox qxm_npu_dsp_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x13000 },
 	.prio = 0,
@@ -495,7 +495,7 @@ static struct qcom_icc_node qhm_cnoc_dc_noc = {
 			&qhs_gemnoc },
 };
 
-static struct qcom_icc_qosbox acm_apps_qos = {
+static const struct qcom_icc_qosbox acm_apps_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x2f100, 0x2f000 },
 	.prio = 0,
@@ -512,7 +512,7 @@ static struct qcom_icc_node acm_apps = {
 			&qns_gem_noc_snoc },
 };
 
-static struct qcom_icc_qosbox acm_sys_tcu_qos = {
+static const struct qcom_icc_qosbox acm_sys_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x35000 },
 	.prio = 6,
@@ -539,7 +539,7 @@ static struct qcom_icc_node qhm_gemnoc_cfg = {
 			&qhs_mdsp_ms_mpu_cfg },
 };
 
-static struct qcom_icc_qosbox qnm_cmpnoc_qos = {
+static const struct qcom_icc_qosbox qnm_cmpnoc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x2e000 },
 	.prio = 0,
@@ -556,7 +556,7 @@ static struct qcom_icc_node qnm_cmpnoc = {
 			&qns_gem_noc_snoc },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x30000 },
 	.prio = 0,
@@ -573,7 +573,7 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 			&qns_gem_noc_snoc },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x34000 },
 	.prio = 0,
@@ -590,7 +590,7 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 			&qns_gem_noc_snoc },
 };
 
-static struct qcom_icc_qosbox qnm_snoc_gc_qos = {
+static const struct qcom_icc_qosbox qnm_snoc_gc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x32000 },
 	.prio = 0,
@@ -606,7 +606,7 @@ static struct qcom_icc_node qnm_snoc_gc = {
 	.link_nodes = { &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_snoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_snoc_sf_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x31000 },
 	.prio = 0,
@@ -622,7 +622,7 @@ static struct qcom_icc_node qnm_snoc_sf = {
 	.link_nodes = { &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qxm_gpu_qos = {
+static const struct qcom_icc_qosbox qxm_gpu_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x33000, 0x33080 },
 	.prio = 0,
@@ -655,7 +655,7 @@ static struct qcom_icc_node qhm_mnoc_cfg = {
 	.link_nodes = { &srvc_mnoc },
 };
 
-static struct qcom_icc_qosbox qnm_video0_qos = {
+static const struct qcom_icc_qosbox qnm_video0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf000 },
 	.prio = 2,
@@ -671,7 +671,7 @@ static struct qcom_icc_node qnm_video0 = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_video_cvp_qos = {
+static const struct qcom_icc_qosbox qnm_video_cvp_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xe000 },
 	.prio = 5,
@@ -687,7 +687,7 @@ static struct qcom_icc_node qnm_video_cvp = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qxm_camnoc_hf_qos = {
+static const struct qcom_icc_qosbox qxm_camnoc_hf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0xa000, 0xb000 },
 	.prio = 3,
@@ -703,7 +703,7 @@ static struct qcom_icc_node qxm_camnoc_hf = {
 	.link_nodes = { &qns_mem_noc_hf },
 };
 
-static struct qcom_icc_qosbox qxm_camnoc_icp_qos = {
+static const struct qcom_icc_qosbox qxm_camnoc_icp_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xd000 },
 	.prio = 5,
@@ -719,7 +719,7 @@ static struct qcom_icc_node qxm_camnoc_icp = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qxm_camnoc_sf_qos = {
+static const struct qcom_icc_qosbox qxm_camnoc_sf_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x9000 },
 	.prio = 3,
@@ -735,7 +735,7 @@ static struct qcom_icc_node qxm_camnoc_sf = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qxm_mdp0_qos = {
+static const struct qcom_icc_qosbox qxm_mdp0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc000 },
 	.prio = 3,
@@ -822,7 +822,7 @@ static struct qcom_icc_node qnm_gemnoc = {
 			&xs_qdss_stm },
 };
 
-static struct qcom_icc_qosbox qxm_pimem_qos = {
+static const struct qcom_icc_qosbox qxm_pimem_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xd000 },
 	.prio = 2,
@@ -839,7 +839,7 @@ static struct qcom_icc_node qxm_pimem = {
 			&qxs_imem },
 };
 
-static struct qcom_icc_qosbox xm_gic_qos = {
+static const struct qcom_icc_qosbox xm_gic_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xb000 },
 	.prio = 3,
diff --git a/drivers/interconnect/qcom/sm8650.c b/drivers/interconnect/qcom/sm8650.c
index cf3ae734d4c357..1967475b9132a5 100644
--- a/drivers/interconnect/qcom/sm8650.c
+++ b/drivers/interconnect/qcom/sm8650.c
@@ -154,7 +154,7 @@ static const struct regmap_config icc_regmap_config = {
 	.fast_io = true,
 };
 
-static struct qcom_icc_qosbox qhm_qspi_qos = {
+static const struct qcom_icc_qosbox qhm_qspi_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc000 },
 	.prio = 2,
@@ -171,7 +171,7 @@ static struct qcom_icc_node qhm_qspi = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox qhm_qup1_qos = {
+static const struct qcom_icc_qosbox qhm_qup1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xd000 },
 	.prio = 2,
@@ -196,7 +196,7 @@ static struct qcom_icc_node qxm_qup02 = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_sdc4_qos = {
+static const struct qcom_icc_qosbox xm_sdc4_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xe000 },
 	.prio = 2,
@@ -213,7 +213,7 @@ static struct qcom_icc_node xm_sdc4 = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_ufs_mem_qos = {
+static const struct qcom_icc_qosbox xm_ufs_mem_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xf000 },
 	.prio = 2,
@@ -230,7 +230,7 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_usb3_0_qos = {
+static const struct qcom_icc_qosbox xm_usb3_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x10000 },
 	.prio = 2,
@@ -247,7 +247,7 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.link_nodes = { &qns_a1noc_snoc },
 };
 
-static struct qcom_icc_qosbox qhm_qdss_bam_qos = {
+static const struct qcom_icc_qosbox qhm_qdss_bam_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x12000 },
 	.prio = 2,
@@ -264,7 +264,7 @@ static struct qcom_icc_node qhm_qdss_bam = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qhm_qup2_qos = {
+static const struct qcom_icc_qosbox qhm_qup2_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x13000 },
 	.prio = 2,
@@ -281,7 +281,7 @@ static struct qcom_icc_node qhm_qup2 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qxm_crypto_qos = {
+static const struct qcom_icc_qosbox qxm_crypto_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x15000 },
 	.prio = 2,
@@ -298,7 +298,7 @@ static struct qcom_icc_node qxm_crypto = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox qxm_ipa_qos = {
+static const struct qcom_icc_qosbox qxm_ipa_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x16000 },
 	.prio = 2,
@@ -323,7 +323,7 @@ static struct qcom_icc_node qxm_sp = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_qdss_etr_0_qos = {
+static const struct qcom_icc_qosbox xm_qdss_etr_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x17000 },
 	.prio = 2,
@@ -340,7 +340,7 @@ static struct qcom_icc_node xm_qdss_etr_0 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_qdss_etr_1_qos = {
+static const struct qcom_icc_qosbox xm_qdss_etr_1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x18000 },
 	.prio = 2,
@@ -357,7 +357,7 @@ static struct qcom_icc_node xm_qdss_etr_1 = {
 	.link_nodes = { &qns_a2noc_snoc },
 };
 
-static struct qcom_icc_qosbox xm_sdc2_qos = {
+static const struct qcom_icc_qosbox xm_sdc2_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x19000 },
 	.prio = 2,
@@ -448,7 +448,7 @@ static struct qcom_icc_node qnm_gemnoc_pcie = {
 	.link_nodes = { &xs_pcie_0, &xs_pcie_1 },
 };
 
-static struct qcom_icc_qosbox alm_gpu_tcu_qos = {
+static const struct qcom_icc_qosbox alm_gpu_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xbf000 },
 	.prio = 1,
@@ -465,7 +465,7 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox alm_sys_tcu_qos = {
+static const struct qcom_icc_qosbox alm_sys_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc1000 },
 	.prio = 6,
@@ -482,7 +482,7 @@ static struct qcom_icc_node alm_sys_tcu = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox alm_ubwc_p_tcu_qos = {
+static const struct qcom_icc_qosbox alm_ubwc_p_tcu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc5000 },
 	.prio = 1,
@@ -508,7 +508,7 @@ static struct qcom_icc_node chm_apps = {
 			&qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_gpu_qos = {
+static const struct qcom_icc_qosbox qnm_gpu_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x31000, 0x71000 },
 	.prio = 0,
@@ -525,7 +525,7 @@ static struct qcom_icc_node qnm_gpu = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_lpass_gemnoc_qos = {
+static const struct qcom_icc_qosbox qnm_lpass_gemnoc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xb5000 },
 	.prio = 0,
@@ -552,7 +552,7 @@ static struct qcom_icc_node qnm_mdsp = {
 			&qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_hf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x33000, 0x73000 },
 	.prio = 0,
@@ -569,7 +569,7 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_mnoc_sf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x35000, 0x75000 },
 	.prio = 0,
@@ -586,7 +586,7 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_nsp_gemnoc_qos = {
+static const struct qcom_icc_qosbox qnm_nsp_gemnoc_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x37000, 0x77000 },
 	.prio = 0,
@@ -604,7 +604,7 @@ static struct qcom_icc_node qnm_nsp_gemnoc = {
 			&qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_pcie_qos = {
+static const struct qcom_icc_qosbox qnm_pcie_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xb7000 },
 	.prio = 2,
@@ -621,7 +621,7 @@ static struct qcom_icc_node qnm_pcie = {
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
 
-static struct qcom_icc_qosbox qnm_snoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_snoc_sf_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xbb000 },
 	.prio = 0,
@@ -639,7 +639,7 @@ static struct qcom_icc_node qnm_snoc_sf = {
 			&qns_pcie },
 };
 
-static struct qcom_icc_qosbox qnm_ubwc_p_qos = {
+static const struct qcom_icc_qosbox qnm_ubwc_p_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc3000 },
 	.prio = 1,
@@ -656,7 +656,7 @@ static struct qcom_icc_node qnm_ubwc_p = {
 	.link_nodes = { &qns_llcc },
 };
 
-static struct qcom_icc_qosbox xm_gic_qos = {
+static const struct qcom_icc_qosbox xm_gic_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xb9000 },
 	.prio = 4,
@@ -705,7 +705,7 @@ static struct qcom_icc_node llcc_mc = {
 	.link_nodes = { &ebi },
 };
 
-static struct qcom_icc_qosbox qnm_camnoc_hf_qos = {
+static const struct qcom_icc_qosbox qnm_camnoc_hf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x28000, 0x29000 },
 	.prio = 0,
@@ -722,7 +722,7 @@ static struct qcom_icc_node qnm_camnoc_hf = {
 	.link_nodes = { &qns_mem_noc_hf },
 };
 
-static struct qcom_icc_qosbox qnm_camnoc_icp_qos = {
+static const struct qcom_icc_qosbox qnm_camnoc_icp_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x2a000 },
 	.prio = 4,
@@ -739,7 +739,7 @@ static struct qcom_icc_node qnm_camnoc_icp = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_camnoc_sf_qos = {
+static const struct qcom_icc_qosbox qnm_camnoc_sf_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x2b000, 0x2c000 },
 	.prio = 0,
@@ -756,7 +756,7 @@ static struct qcom_icc_node qnm_camnoc_sf = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_mdp_qos = {
+static const struct qcom_icc_qosbox qnm_mdp_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x2d000, 0x2e000 },
 	.prio = 0,
@@ -781,7 +781,7 @@ static struct qcom_icc_node qnm_vapss_hcp = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_video_qos = {
+static const struct qcom_icc_qosbox qnm_video_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x30000, 0x31000 },
 	.prio = 0,
@@ -798,7 +798,7 @@ static struct qcom_icc_node qnm_video = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_video_cv_cpu_qos = {
+static const struct qcom_icc_qosbox qnm_video_cv_cpu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x32000 },
 	.prio = 4,
@@ -815,7 +815,7 @@ static struct qcom_icc_node qnm_video_cv_cpu = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_video_cvp_qos = {
+static const struct qcom_icc_qosbox qnm_video_cvp_qos = {
 	.num_ports = 2,
 	.port_offsets = { 0x33000, 0x34000 },
 	.prio = 0,
@@ -832,7 +832,7 @@ static struct qcom_icc_node qnm_video_cvp = {
 	.link_nodes = { &qns_mem_noc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_video_v_cpu_qos = {
+static const struct qcom_icc_qosbox qnm_video_v_cpu_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x35000 },
 	.prio = 4,
@@ -873,7 +873,7 @@ static struct qcom_icc_node qsm_pcie_anoc_cfg = {
 	.link_nodes = { &srvc_pcie_aggre_noc },
 };
 
-static struct qcom_icc_qosbox xm_pcie3_0_qos = {
+static const struct qcom_icc_qosbox xm_pcie3_0_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xb000 },
 	.prio = 3,
@@ -890,7 +890,7 @@ static struct qcom_icc_node xm_pcie3_0 = {
 	.link_nodes = { &qns_pcie_mem_noc },
 };
 
-static struct qcom_icc_qosbox xm_pcie3_1_qos = {
+static const struct qcom_icc_qosbox xm_pcie3_1_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0xc000 },
 	.prio = 2,
@@ -923,7 +923,7 @@ static struct qcom_icc_node qnm_aggre2_noc = {
 	.link_nodes = { &qns_gemnoc_sf },
 };
 
-static struct qcom_icc_qosbox qnm_apss_noc_qos = {
+static const struct qcom_icc_qosbox qnm_apss_noc_qos = {
 	.num_ports = 1,
 	.port_offsets = { 0x1c000 },
 	.prio = 2,
diff --git a/drivers/interconnect/qcom/smd-rpm.c b/drivers/interconnect/qcom/smd-rpm.c
index dbc7ae50b02b79..6ac9aef942ad2c 100644
--- a/drivers/interconnect/qcom/smd-rpm.c
+++ b/drivers/interconnect/qcom/smd-rpm.c
@@ -20,7 +20,7 @@ static struct qcom_smd_rpm *icc_smd_rpm;
 struct icc_rpm_smd_req {
 	__le32 key;
 	__le32 nbytes;
-	__le32 value;
+	__le64 value;
 };
 
 bool qcom_icc_rpm_smd_available(void)
@@ -29,12 +29,12 @@ bool qcom_icc_rpm_smd_available(void)
 }
 EXPORT_SYMBOL_GPL(qcom_icc_rpm_smd_available);
 
-int qcom_icc_rpm_smd_send(int ctx, int rsc_type, int id, u32 val)
+int qcom_icc_rpm_smd_send(int ctx, int rsc_type, int id, u64 val)
 {
 	struct icc_rpm_smd_req req = {
 		.key = cpu_to_le32(RPM_KEY_BW),
-		.nbytes = cpu_to_le32(sizeof(u32)),
-		.value = cpu_to_le32(val),
+		.nbytes = cpu_to_le32(sizeof(u64)),
+		.value = cpu_to_le64(val),
 	};
 
 	return qcom_rpm_smd_write(icc_smd_rpm, ctx, rsc_type, id, &req,
diff --git a/drivers/interconnect/qcom/x1e80100.c b/drivers/interconnect/qcom/x1e80100.c
index 8075e0ff205973..2ba2823c7860e9 100644
--- a/drivers/interconnect/qcom/x1e80100.c
+++ b/drivers/interconnect/qcom/x1e80100.c
@@ -173,13 +173,6 @@ static struct qcom_icc_node qhm_qspi = {
 	.name = "qhm_qspi",
 	.channels = 1,
 	.buswidth = 4,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xb000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -188,13 +181,6 @@ static struct qcom_icc_node qhm_qup1 = {
 	.name = "qhm_qup1",
 	.channels = 1,
 	.buswidth = 4,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xc000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -203,13 +189,6 @@ static struct qcom_icc_node xm_sdc4 = {
 	.name = "xm_sdc4",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xd000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -218,13 +197,6 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.name = "xm_ufs_mem",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xe000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -233,13 +205,6 @@ static struct qcom_icc_node qhm_qup0 = {
 	.name = "qhm_qup0",
 	.channels = 1,
 	.buswidth = 4,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x16000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -248,13 +213,6 @@ static struct qcom_icc_node qhm_qup2 = {
 	.name = "qhm_qup2",
 	.channels = 1,
 	.buswidth = 4,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x11000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -263,13 +221,6 @@ static struct qcom_icc_node qxm_crypto = {
 	.name = "qxm_crypto",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x12000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -286,13 +237,6 @@ static struct qcom_icc_node xm_qdss_etr_0 = {
 	.name = "xm_qdss_etr_0",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x13000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -301,13 +245,6 @@ static struct qcom_icc_node xm_qdss_etr_1 = {
 	.name = "xm_qdss_etr_1",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x14000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -316,13 +253,6 @@ static struct qcom_icc_node xm_sdc2 = {
 	.name = "xm_sdc2",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x15000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -407,13 +337,6 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.name = "alm_gpu_tcu",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x271000 },
-		.prio = 1,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -422,13 +345,6 @@ static struct qcom_icc_node alm_pcie_tcu = {
 	.name = "alm_pcie_tcu",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x27d000 },
-		.prio = 3,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -437,13 +353,6 @@ static struct qcom_icc_node alm_sys_tcu = {
 	.name = "alm_sys_tcu",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x273000 },
-		.prio = 6,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -461,13 +370,6 @@ static struct qcom_icc_node qnm_gpu = {
 	.name = "qnm_gpu",
 	.channels = 4,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 4,
-		.port_offsets = { 0x51000, 0x58000, 0xd1000, 0xd8000 },
-		.prio = 0,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -476,13 +378,6 @@ static struct qcom_icc_node qnm_lpass = {
 	.name = "qnm_lpass",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x275000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -492,13 +387,6 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.name = "qnm_mnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 2,
-		.port_offsets = { 0x53000, 0xd3000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -507,13 +395,6 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.name = "qnm_mnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 2,
-		.port_offsets = { 0x55000, 0xd5000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -522,13 +403,6 @@ static struct qcom_icc_node qnm_nsp_noc = {
 	.name = "qnm_nsp_noc",
 	.channels = 2,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 2,
-		.port_offsets = { 0x57000, 0xd7000 },
-		.prio = 0,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -538,13 +412,6 @@ static struct qcom_icc_node qnm_pcie = {
 	.name = "qnm_pcie",
 	.channels = 1,
 	.buswidth = 64,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x277000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -553,13 +420,6 @@ static struct qcom_icc_node qnm_snoc_sf = {
 	.name = "qnm_snoc_sf",
 	.channels = 1,
 	.buswidth = 64,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x27b000 },
-		.prio = 2,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -569,13 +429,6 @@ static struct qcom_icc_node xm_gic = {
 	.name = "xm_gic",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x27f000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -616,13 +469,6 @@ static struct qcom_icc_node qnm_av1_enc = {
 	.name = "qnm_av1_enc",
 	.channels = 1,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x2f000 },
-		.prio = 4,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -631,13 +477,6 @@ static struct qcom_icc_node qnm_camnoc_hf = {
 	.name = "qnm_camnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 2,
-		.port_offsets = { 0x28000, 0x29000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -646,13 +485,6 @@ static struct qcom_icc_node qnm_camnoc_icp = {
 	.name = "qnm_camnoc_icp",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x2a000 },
-		.prio = 4,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -661,13 +493,6 @@ static struct qcom_icc_node qnm_camnoc_sf = {
 	.name = "qnm_camnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 2,
-		.port_offsets = { 0x2b000, 0x2c000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -676,13 +501,6 @@ static struct qcom_icc_node qnm_eva = {
 	.name = "qnm_eva",
 	.channels = 1,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x33000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -691,13 +509,6 @@ static struct qcom_icc_node qnm_mdp = {
 	.name = "qnm_mdp",
 	.channels = 2,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 2,
-		.port_offsets = { 0x2d000, 0x2e000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -706,13 +517,6 @@ static struct qcom_icc_node qnm_video = {
 	.name = "qnm_video",
 	.channels = 2,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 2,
-		.port_offsets = { 0x30000, 0x31000 },
-		.prio = 0,
-		.urg_fwd = 1,
-		.prio_fwd_disable = 0,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -721,13 +525,6 @@ static struct qcom_icc_node qnm_video_cv_cpu = {
 	.name = "qnm_video_cv_cpu",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x32000 },
-		.prio = 4,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -736,13 +533,6 @@ static struct qcom_icc_node qnm_video_v_cpu = {
 	.name = "qnm_video_v_cpu",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x34000 },
-		.prio = 4,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -783,13 +573,6 @@ static struct qcom_icc_node xm_pcie_3 = {
 	.name = "xm_pcie_3",
 	.channels = 1,
 	.buswidth = 64,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x7000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -798,13 +581,6 @@ static struct qcom_icc_node xm_pcie_4 = {
 	.name = "xm_pcie_4",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x8000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -813,13 +589,6 @@ static struct qcom_icc_node xm_pcie_5 = {
 	.name = "xm_pcie_5",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x9000 },
-		.prio = 3,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -828,13 +597,6 @@ static struct qcom_icc_node xm_pcie_0 = {
 	.name = "xm_pcie_0",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x9000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -843,13 +605,6 @@ static struct qcom_icc_node xm_pcie_1 = {
 	.name = "xm_pcie_1",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xa000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -858,13 +613,6 @@ static struct qcom_icc_node xm_pcie_2 = {
 	.name = "xm_pcie_2",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xb000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -873,13 +621,6 @@ static struct qcom_icc_node xm_pcie_6a = {
 	.name = "xm_pcie_6a",
 	.channels = 1,
 	.buswidth = 32,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xc000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -888,13 +629,6 @@ static struct qcom_icc_node xm_pcie_6b = {
 	.name = "xm_pcie_6b",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xd000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -919,13 +653,6 @@ static struct qcom_icc_node qnm_gic = {
 	.name = "qnm_gic",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x1c000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_sf },
 };
@@ -958,13 +685,6 @@ static struct qcom_icc_node xm_usb2_0 = {
 	.name = "xm_usb2_0",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x6000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_north_snoc },
 };
@@ -973,13 +693,6 @@ static struct qcom_icc_node xm_usb3_mp = {
 	.name = "xm_usb3_mp",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0x7000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_north_snoc },
 };
@@ -988,13 +701,6 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.name = "xm_usb3_0",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xa000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1003,13 +709,6 @@ static struct qcom_icc_node xm_usb3_1 = {
 	.name = "xm_usb3_1",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xb000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1018,13 +717,6 @@ static struct qcom_icc_node xm_usb3_2 = {
 	.name = "xm_usb3_2",
 	.channels = 1,
 	.buswidth = 8,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xc000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1033,13 +725,6 @@ static struct qcom_icc_node xm_usb4_0 = {
 	.name = "xm_usb4_0",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xd000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1048,13 +733,6 @@ static struct qcom_icc_node xm_usb4_1 = {
 	.name = "xm_usb4_1",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xe000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1063,13 +741,6 @@ static struct qcom_icc_node xm_usb4_2 = {
 	.name = "xm_usb4_2",
 	.channels = 1,
 	.buswidth = 16,
-	.qosbox = &(const struct qcom_icc_qosbox) {
-		.num_ports = 1,
-		.port_offsets = { 0xf000 },
-		.prio = 2,
-		.urg_fwd = 0,
-		.prio_fwd_disable = 1,
-	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1795,21 +1466,11 @@ static struct qcom_icc_node * const aggre1_noc_nodes[] = {
 	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
 };
 
-static const struct regmap_config x1e80100_aggre1_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x14400,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_aggre1_noc = {
-	.config = &x1e80100_aggre1_noc_regmap_config,
 	.nodes = aggre1_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
 	.bcms = aggre1_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
-	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const aggre2_noc_bcms[] = {
@@ -1827,16 +1488,7 @@ static struct qcom_icc_node * const aggre2_noc_nodes[] = {
 	[SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc,
 };
 
-static const struct regmap_config x1e80100_aggre2_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x1c400,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_aggre2_noc = {
-	.config = &x1e80100_aggre2_noc_regmap_config,
 	.nodes = aggre2_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre2_noc_nodes),
 	.bcms = aggre2_noc_bcms,
@@ -1921,16 +1573,7 @@ static struct qcom_icc_node * const cnoc_cfg_nodes[] = {
 	[SLAVE_TCU] = &xs_sys_tcu_cfg,
 };
 
-static const struct regmap_config x1e80100_cnoc_cfg_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x6600,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_cnoc_cfg = {
-	.config = &x1e80100_cnoc_cfg_regmap_config,
 	.nodes = cnoc_cfg_nodes,
 	.num_nodes = ARRAY_SIZE(cnoc_cfg_nodes),
 	.bcms = cnoc_cfg_bcms,
@@ -1960,16 +1603,7 @@ static struct qcom_icc_node * const cnoc_main_nodes[] = {
 	[SLAVE_PCIE_6B] = &xs_pcie_6b,
 };
 
-static const struct regmap_config x1e80100_cnoc_main_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x14400,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_cnoc_main = {
-	.config = &x1e80100_cnoc_main_regmap_config,
 	.nodes = cnoc_main_nodes,
 	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
 	.bcms = cnoc_main_bcms,
@@ -1999,16 +1633,7 @@ static struct qcom_icc_node * const gem_noc_nodes[] = {
 	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
 };
 
-static const struct regmap_config x1e80100_gem_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x311200,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_gem_noc = {
-	.config = &x1e80100_gem_noc_regmap_config,
 	.nodes = gem_noc_nodes,
 	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
 	.bcms = gem_noc_bcms,
@@ -2023,16 +1648,7 @@ static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
 	[SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc,
 };
 
-static const struct regmap_config x1e80100_lpass_ag_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0xe080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_lpass_ag_noc = {
-	.config = &x1e80100_lpass_ag_noc_regmap_config,
 	.nodes = lpass_ag_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
 	.bcms = lpass_ag_noc_bcms,
@@ -2048,16 +1664,7 @@ static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = {
 	[SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc,
 };
 
-static const struct regmap_config x1e80100_lpass_lpiaon_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x19080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_lpass_lpiaon_noc = {
-	.config = &x1e80100_lpass_lpiaon_noc_regmap_config,
 	.nodes = lpass_lpiaon_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes),
 	.bcms = lpass_lpiaon_noc_bcms,
@@ -2072,16 +1679,7 @@ static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = {
 	[SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc,
 };
 
-static const struct regmap_config x1e80100_lpass_lpicx_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x3a200,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_lpass_lpicx_noc = {
-	.config = &x1e80100_lpass_lpicx_noc_regmap_config,
 	.nodes = lpass_lpicx_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes),
 	.bcms = lpass_lpicx_noc_bcms,
@@ -2126,16 +1724,7 @@ static struct qcom_icc_node * const mmss_noc_nodes[] = {
 	[SLAVE_SERVICE_MNOC] = &srvc_mnoc,
 };
 
-static const struct regmap_config x1e80100_mmss_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x5b800,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_mmss_noc = {
-	.config = &x1e80100_mmss_noc_regmap_config,
 	.nodes = mmss_noc_nodes,
 	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
 	.bcms = mmss_noc_bcms,
@@ -2151,16 +1740,7 @@ static struct qcom_icc_node * const nsp_noc_nodes[] = {
 	[SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc,
 };
 
-static const struct regmap_config x1e80100_nsp_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0xe080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_nsp_noc = {
-	.config = &x1e80100_nsp_noc_regmap_config,
 	.nodes = nsp_noc_nodes,
 	.num_nodes = ARRAY_SIZE(nsp_noc_nodes),
 	.bcms = nsp_noc_bcms,
@@ -2177,16 +1757,7 @@ static struct qcom_icc_node * const pcie_center_anoc_nodes[] = {
 	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_mem_noc,
 };
 
-static const struct regmap_config x1e80100_pcie_center_anoc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x7000,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_pcie_center_anoc = {
-	.config = &x1e80100_pcie_center_anoc_regmap_config,
 	.nodes = pcie_center_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_center_anoc_nodes),
 	.bcms = pcie_center_anoc_bcms,
@@ -2203,16 +1774,7 @@ static struct qcom_icc_node * const pcie_north_anoc_nodes[] = {
 	[SLAVE_PCIE_NORTH] = &qns_pcie_north_gem_noc,
 };
 
-static const struct regmap_config x1e80100_pcie_north_anoc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x9080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_pcie_north_anoc = {
-	.config = &x1e80100_pcie_north_anoc_regmap_config,
 	.nodes = pcie_north_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_north_anoc_nodes),
 	.bcms = pcie_north_anoc_bcms,
@@ -2231,16 +1793,7 @@ static struct qcom_icc_node * const pcie_south_anoc_nodes[] = {
 	[SLAVE_PCIE_SOUTH] = &qns_pcie_south_gem_noc,
 };
 
-static const struct regmap_config x1e80100_pcie_south_anoc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0xd080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_pcie_south_anoc = {
-	.config = &x1e80100_pcie_south_anoc_regmap_config,
 	.nodes = pcie_south_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_south_anoc_nodes),
 	.bcms = pcie_south_anoc_bcms,
@@ -2262,16 +1815,7 @@ static struct qcom_icc_node * const system_noc_nodes[] = {
 	[SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf,
 };
 
-static const struct regmap_config x1e80100_system_noc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x1c080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_system_noc = {
-	.config = &x1e80100_system_noc_regmap_config,
 	.nodes = system_noc_nodes,
 	.num_nodes = ARRAY_SIZE(system_noc_nodes),
 	.bcms = system_noc_bcms,
@@ -2287,16 +1831,7 @@ static struct qcom_icc_node * const usb_center_anoc_nodes[] = {
 	[SLAVE_USB_NOC_SNOC] = &qns_aggre_usb_snoc,
 };
 
-static const struct regmap_config x1e80100_usb_center_anoc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x8800,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_usb_center_anoc = {
-	.config = &x1e80100_usb_center_anoc_regmap_config,
 	.nodes = usb_center_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_center_anoc_nodes),
 	.bcms = usb_center_anoc_bcms,
@@ -2312,21 +1847,11 @@ static struct qcom_icc_node * const usb_north_anoc_nodes[] = {
 	[SLAVE_AGGRE_USB_NORTH] = &qns_aggre_usb_north_snoc,
 };
 
-static const struct regmap_config x1e80100_usb_north_anoc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0x7080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_usb_north_anoc = {
-	.config = &x1e80100_usb_north_anoc_regmap_config,
 	.nodes = usb_north_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_north_anoc_nodes),
 	.bcms = usb_north_anoc_bcms,
 	.num_bcms = ARRAY_SIZE(usb_north_anoc_bcms),
-	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const usb_south_anoc_bcms[] = {
@@ -2342,21 +1867,11 @@ static struct qcom_icc_node * const usb_south_anoc_nodes[] = {
 	[SLAVE_AGGRE_USB_SOUTH] = &qns_aggre_usb_south_snoc,
 };
 
-static const struct regmap_config x1e80100_usb_south_anoc_regmap_config = {
-	.reg_bits = 32,
-	.reg_stride = 4,
-	.val_bits = 32,
-	.max_register = 0xf080,
-	.fast_io = true,
-};
-
 static const struct qcom_icc_desc x1e80100_usb_south_anoc = {
-	.config = &x1e80100_usb_south_anoc_regmap_config,
 	.nodes = usb_south_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_south_anoc_nodes),
 	.bcms = usb_south_anoc_bcms,
 	.num_bcms = ARRAY_SIZE(usb_south_anoc_bcms),
-	.qos_requires_clocks = true,
 };
 
 static const struct of_device_id qnoc_of_match[] = {
diff --git a/drivers/interconnect/samsung/exynos.c b/drivers/interconnect/samsung/exynos.c
index 8e8f56186a3669..9a4789fadac2ea 100644
--- a/drivers/interconnect/samsung/exynos.c
+++ b/drivers/interconnect/samsung/exynos.c
@@ -98,6 +98,7 @@ static void exynos_generic_icc_remove(struct platform_device *pdev)
 	struct exynos_icc_priv *priv = platform_get_drvdata(pdev);
 
 	icc_provider_deregister(&priv->provider);
+	dev_pm_qos_remove_request(&priv->qos_req);
 	icc_nodes_remove(&priv->provider);
 }
 
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 6e07bd69467a3d..1fde782608fdf1 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -303,7 +303,7 @@ config SPAPR_TCE_IOMMU
 
 config APPLE_DART
 	tristate "Apple DART IOMMU Support"
-	depends on ARCH_APPLE || COMPILE_TEST
+	depends on ARCH_APPLE || (COMPILE_TEST && 64BIT)
 	depends on !GENERIC_ATOMIC64	# for IOMMU_IO_PGTABLE_DART
 	select IOMMU_API
 	select IOMMU_IO_PGTABLE_DART
@@ -386,6 +386,21 @@ config VSI_IOMMU
 	  Say Y here if you want to use this IOMMU in front of these
 	  hardware blocks.
 
+config BCM2712_IOMMU
+	bool "BCM2712 IOMMU driver"
+	depends on (ARCH_BCM2835 && ARM64) || COMPILE_TEST
+	select IOMMU_API
+	select GENERIC_PT
+	select IOMMU_PT
+	select IOMMU_PT_BCM2712
+	help
+	  Support for IOMMU on BCM2712 SoC. This IOMMU can be used by the
+	  display controller and various multimedia devices to perform
+	  efficient memory management.
+
+	  Say Y here if you want to use this IOMMU in front of these
+	  hardware blocks.
+
 config IOMMU_DEBUG_PAGEALLOC
 	bool "Debug IOMMU mappings against page allocations"
 	depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 2f05725eaab185..29a26d2a3af12e 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -37,4 +37,5 @@ obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o
 obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
 obj-$(CONFIG_APPLE_DART) += apple-dart.o
 obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o
+obj-$(CONFIG_BCM2712_IOMMU) += bcm2712-iommu.o bcm2712-iommu-cache.o
 obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 71113e86085947..93845b3c3fbf72 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -48,6 +48,9 @@ extern u8 amd_iommu_hpt_vasize;
 extern unsigned long amd_iommu_pgsize_bitmap;
 extern bool amd_iommu_hatdis;
 
+int amd_iommu_perfopt_clear(struct amd_iommu *iommu);
+int amd_iommu_perfopt_restore(struct amd_iommu *iommu);
+
 /* Protection domain ops */
 void amd_iommu_init_identity_domain(void);
 struct protection_domain *protection_domain_alloc(void);
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 8241ef92251923..1c1624b7c9a368 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -53,6 +53,7 @@
 #define MMIO_MSI_ADDR_LO_OFFSET	0x015C
 #define MMIO_MSI_ADDR_HI_OFFSET	0x0160
 #define MMIO_MSI_DATA_OFFSET	0x0164
+#define MMIO_PERF_OPT_OFFSET	0x016C
 #define MMIO_INTCAPXT_EVT_OFFSET	0x0170
 #define MMIO_INTCAPXT_PPR_OFFSET	0x0178
 #define MMIO_INTCAPXT_GALOG_OFFSET	0x0180
@@ -87,6 +88,8 @@
 #define FEATURE_GLX		GENMASK_ULL(15, 14)
 #define FEATURE_GAM_VAPIC	BIT_ULL(21)
 #define FEATURE_PASMAX		GENMASK_ULL(36, 32)
+#define FEATURE_PERF_OPT	BIT_ULL(45)
+#define PERF_OPT_EN		BIT(13)
 #define FEATURE_GIOSUP		BIT_ULL(48)
 #define FEATURE_HASUP		BIT_ULL(49)
 #define FEATURE_EPHSUP		BIT_ULL(50)
@@ -654,6 +657,9 @@ struct amd_iommu {
 	/* Extended features 2 */
 	u64 features2;
 
+	/* Devices requesting PerfOpt; the shared PERF_OPT_EN bit is on while >0. Protected by @lock. */
+	int perfopt_refcount;
+
 	/* PCI device id of the IOMMU device */
 	u16 devid;
 
@@ -815,6 +821,7 @@ struct iommu_dev_data {
 	u8 ppr          :1;		  /* Enable device PPR support */
 	bool use_vapic;			  /* Enable device to use vapic mode */
 	bool defer_attach;
+	bool perfopt;
 
 	struct ratelimit_state rs;        /* Ratelimit IOPF messages */
 };
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 8a410d4aa3704b..8ec8a6fccaaf3c 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -163,7 +163,6 @@ int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
 
-static bool amd_iommu_detected;
 static bool amd_iommu_disabled __initdata;
 static bool amd_iommu_force_enable __initdata;
 static bool amd_iommu_irtcachedis;
@@ -916,13 +915,13 @@ static void free_ga_log(struct amd_iommu *iommu)
 }
 
 #ifdef CONFIG_IRQ_REMAP
-static int iommu_ga_log_enable(struct amd_iommu *iommu)
+static bool iommu_ga_log_enable(struct amd_iommu *iommu)
 {
 	u32 status, i;
 	u64 entry;
 
 	if (!iommu->ga_log)
-		return -EINVAL;
+		return false;
 
 	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
@@ -946,35 +945,31 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu)
 	}
 
 	if (WARN_ON(i >= MMIO_STATUS_TIMEOUT))
-		return -EINVAL;
+		return false;
 
-	return 0;
+	return true;
 }
+#endif /* CONFIG_IRQ_REMAP */
 
-static int iommu_init_ga_log(struct amd_iommu *iommu)
+static int alloc_ga_log(struct amd_iommu *iommu)
 {
+#ifdef CONFIG_IRQ_REMAP
 	int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
 
-	if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)))
-		return -EINVAL;
-
-	if (iommu->ga_log && iommu->ga_log_tail)
-		return 0;
-
 	iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE);
 	if (!iommu->ga_log)
-		goto err_out;
+		return -ENOMEM;
 
 	iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8);
-	if (!iommu->ga_log_tail)
-		goto err_out;
+	if (!iommu->ga_log_tail) {
+		iommu_free_pages(iommu->ga_log);
+		iommu->ga_log = NULL;
+		return -ENOMEM;
+	}
+#endif
 
 	return 0;
-err_out:
-	free_ga_log(iommu);
-	return -EINVAL;
 }
-#endif /* CONFIG_IRQ_REMAP */
 
 static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
 {
@@ -1948,6 +1943,9 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
 	if (!iommu->mmio_base)
 		return -ENOMEM;
 
+	if (amd_iommu_perfopt_clear(iommu))
+		pr_err("IOMMU%d: failed to clear PerfOpt\n", iommu->index);
+
 	return init_iommu_from_acpi(iommu, h);
 }
 
@@ -2152,6 +2150,27 @@ static void __init late_iommu_features_init(struct amd_iommu *iommu)
 	}
 }
 
+/* Must be called after SNP support check is complete (iommu_snp_enable()) */
+static bool check_vapic_support(void)
+{
+	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
+		return false;
+
+	if (!check_feature(FEATURE_GAM_VAPIC)) {
+		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
+		return false;
+	}
+
+	if (amd_iommu_snp_en &&
+	    !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
+		pr_warn("Force to disable Virtual APIC due to SNP\n");
+		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
+		return false;
+	}
+
+	return true;
+}
+
 static int __init iommu_init_pci(struct amd_iommu *iommu)
 {
 	int cap_ptr = iommu->cap_ptr;
@@ -2196,6 +2215,12 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
 	if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))
 		return -ENOMEM;
 
+	if (check_vapic_support()) {
+		ret = alloc_ga_log(iommu);
+		if (ret)
+			return ret;
+	}
+
 	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
 		pr_info("Using strict mode due to virtualization\n");
 		iommu_set_dma_strict();
@@ -3010,22 +3035,9 @@ static void enable_iommus_vapic(void)
 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
 		return;
 
-	if (!check_feature(FEATURE_GAM_VAPIC)) {
-		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
-		return;
-	}
-
-	if (amd_iommu_snp_en &&
-	    !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
-		pr_warn("Force to disable Virtual APIC due to SNP\n");
-		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
-		return;
-	}
-
 	/* Enabling GAM and SNPAVIC support */
 	for_each_iommu(iommu) {
-		if (iommu_init_ga_log(iommu) ||
-		    iommu_ga_log_enable(iommu))
+		if (!iommu_ga_log_enable(iommu))
 			return;
 
 		iommu_feature_enable(iommu, CONTROL_GAM_EN);
@@ -3038,10 +3050,46 @@ static void enable_iommus_vapic(void)
 #endif
 }
 
+static int clear_perfopt_all(void)
+{
+	struct amd_iommu *iommu;
+	int err, ret = 0;
+
+	for_each_iommu(iommu) {
+		err = amd_iommu_perfopt_clear(iommu);
+		if (err)
+			ret = err;
+	}
+
+	return ret;
+}
+
+static int restore_perfopt_all(void)
+{
+	struct amd_iommu *iommu;
+	int err, ret = 0;
+
+	for_each_iommu(iommu) {
+		err = amd_iommu_perfopt_restore(iommu);
+		if (err)
+			ret = err;
+	}
+
+	return ret;
+}
+
 static void disable_iommus(void)
 {
 	struct amd_iommu *iommu;
 
+	/*
+	 * PerfOpt is an optional performance bit, so a failure to clear it must
+	 * not skip the mandatory disable below. This also runs from the void
+	 * amd_iommu_disable() shutdown/kexec path, which cannot report an error.
+	 */
+	if (clear_perfopt_all())
+		pr_err("Failed to clear PerfOpt while disabling IOMMUs\n");
+
 	for_each_iommu(iommu)
 		iommu_disable(iommu);
 
@@ -3067,6 +3115,10 @@ static void amd_iommu_resume(void *data)
 	for_each_iommu(iommu)
 		early_enable_iommu(iommu);
 
+	/* early_enable_iommu() cleared PERF_OPT_EN; re-assert it from the refcount. */
+	if (restore_perfopt_all())
+		pr_err("Failed to restore PerfOpt after IOMMU resume\n");
+
 	iommu_enable_event_buffer();
 	amd_iommu_enable_interrupts();
 }
@@ -3228,9 +3280,6 @@ static int __init early_amd_iommu_init(void)
 	acpi_status status;
 	u8 efr_hats, max_vasize;
 
-	if (!amd_iommu_detected)
-		return -ENODEV;
-
 	status = acpi_get_table("IVRS", 0, &ivrs_base);
 	if (status == AE_NOT_FOUND)
 		return -ENODEV;
@@ -3309,7 +3358,7 @@ static int __init early_amd_iommu_init(void)
 	}
 
 	/* Disable any previously enabled IOMMUs */
-	if (!is_kdump_kernel() || amd_iommu_disabled)
+	if (!is_kdump_kernel())
 		disable_iommus();
 
 	if (amd_iommu_irq_remap)
@@ -3404,12 +3453,6 @@ static __init void iommu_snp_enable(void)
 	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
 		return;
 
-	/* SNP support required IOMMU to be ON */
-	if (no_iommu) {
-		pr_warn("SNP: IOMMU disabled, SNP cannot be supported.\n");
-		goto disable_snp;
-	}
-
 	amd_iommu_snp_mode0_sup = check_feature2(FEATURE_SNP_PAGE_MODE0_SUP);
 	/*
 	 * If SNP page mode 0 is not enabled, then SNP support requires that IOMMU
@@ -3476,6 +3519,8 @@ static void amd_iommu_apply_erratum_snp(void)
 #endif
 }
 
+static bool amd_iommu_sme_check(void);
+
 /****************************************************************************
  *
  * AMD IOMMU Initialization State Machine
@@ -3488,7 +3533,13 @@ static int __init state_next(void)
 
 	switch (init_state) {
 	case IOMMU_START_STATE:
-		if (!detect_ivrs()) {
+		if (no_iommu || amd_iommu_disabled) {
+			init_state	= IOMMU_CMDLINE_DISABLED;
+			ret		= -EINVAL;
+		} else if (!amd_iommu_sme_check()) {
+			init_state	= IOMMU_INIT_ERROR;
+			ret		= -EINVAL;
+		} else if (!detect_ivrs()) {
 			init_state	= IOMMU_NOT_FOUND;
 			ret		= -ENODEV;
 		} else {
@@ -3496,13 +3547,8 @@ static int __init state_next(void)
 		}
 		break;
 	case IOMMU_IVRS_DETECTED:
-		if (amd_iommu_disabled) {
-			init_state = IOMMU_CMDLINE_DISABLED;
-			ret = -EINVAL;
-		} else {
-			ret = early_amd_iommu_init();
-			init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
-		}
+		ret = early_amd_iommu_init();
+		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
 		break;
 	case IOMMU_ACPI_FINISHED:
 		early_enable_iommus();
@@ -3574,6 +3620,19 @@ static int __init iommu_go_to_state(enum iommu_init_state state)
 {
 	int ret = -EINVAL;
 
+	/*
+	 * Some essential housekeeping work is done by amd_iommu_detect().
+	 * Skipping calling it implies that the platform (e.g., Xen hypervisor)
+	 * has taken over the hardware. Progressing the state machine in this
+	 * case is worthless and fragile.
+	 *
+	 * There are several paths requesting later states, so disallow implicit
+	 * START_STATE => IVRS_DETECTED transition to prevent these paths from
+	 * accidentally progressing the state machine.
+	 */
+	if (init_state == IOMMU_START_STATE && state != IOMMU_IVRS_DETECTED)
+		goto out;
+
 	while (init_state != state) {
 		if (init_state == IOMMU_NOT_FOUND         ||
 		    init_state == IOMMU_INIT_ERROR        ||
@@ -3582,6 +3641,7 @@ static int __init iommu_go_to_state(enum iommu_init_state state)
 		ret = state_next();
 	}
 
+out:
 	/*
 	 * SNP platform initilazation requires IOMMUs to be fully configured.
 	 * If the SNP support on IOMMUs has NOT been checked, simply mark SNP
@@ -3701,17 +3761,10 @@ void __init amd_iommu_detect(void)
 {
 	int ret;
 
-	if (no_iommu || (iommu_detected && !gart_iommu_aperture))
-		goto disable_snp;
-
-	if (!amd_iommu_sme_check())
-		goto disable_snp;
-
 	ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
 	if (ret)
 		goto disable_snp;
 
-	amd_iommu_detected = true;
 	iommu_detected = 1;
 	x86_init.iommu.iommu_init = amd_iommu_init;
 	return;
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 56262f6b1f7009..c9b28e5e582eaa 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -573,10 +573,17 @@ static inline int pdev_enable_cap_ats(struct pci_dev *pdev)
 	if (amd_iommu_iotlb_sup &&
 	    (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) {
 		ret = pci_enable_ats(pdev, PAGE_SHIFT);
-		if (!ret) {
-			dev_data->ats_enabled = 1;
-			dev_data->ats_qdep    = pci_ats_queue_depth(pdev);
-		}
+
+		/*
+		 * pci_enable_ats() should not fail here because earlier
+		 * checks have already verified support & config.
+		 */
+		if (WARN_ON(ret))
+			return ret;
+
+		dev_data->ats_enabled = 1;
+		dev_data->ats_qdep    = pci_ats_queue_depth(pdev);
+		ret = 0;
 	}
 
 	return ret;
@@ -675,7 +682,8 @@ static void pdev_disable_caps(struct pci_dev *pdev)
  * This function checks if the driver got a valid device from the caller to
  * avoid dereferencing invalid pointers.
  */
-static bool check_device(struct device *dev)
+static bool lookup_device(struct device *dev,
+				struct amd_iommu **iommu_out, u16 *devid_out)
 {
 	struct amd_iommu_pci_seg *pci_seg;
 	struct amd_iommu *iommu;
@@ -690,7 +698,7 @@ static bool check_device(struct device *dev)
 	devid = PCI_SBDF_TO_DEVID(sbdf);
 
 	iommu = rlookup_amd_iommu(dev);
-	if (!iommu)
+	if (!iommu || !iommu->iommu.ops)
 		return false;
 
 	/* Out of our scope? */
@@ -698,65 +706,35 @@ static bool check_device(struct device *dev)
 	if (devid > pci_seg->last_bdf)
 		return false;
 
+	*iommu_out = iommu;
+	*devid_out = devid;
 	return true;
 }
 
-static int iommu_init_device(struct amd_iommu *iommu, struct device *dev)
+static struct iommu_dev_data *iommu_init_device(struct amd_iommu *iommu,
+						struct device *dev, u16 devid)
 {
 	struct iommu_dev_data *dev_data;
-	int devid, sbdf;
 
-	if (dev_iommu_priv_get(dev))
-		return 0;
-
-	sbdf = get_device_sbdf_id(dev);
-	if (sbdf < 0)
-		return sbdf;
-
-	devid = PCI_SBDF_TO_DEVID(sbdf);
 	dev_data = find_dev_data(iommu, devid);
 	if (!dev_data)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	dev_data->dev = dev;
 
 	/*
-	 * The dev_iommu_priv_set() needes to be called before setup_aliases.
+	 * The dev_iommu_priv_set() needs to be called before setup_aliases.
 	 * Otherwise, subsequent call to dev_iommu_priv_get() will fail.
 	 */
 	dev_iommu_priv_set(dev, dev_data);
 	setup_aliases(iommu, dev);
 
-	/*
-	 * By default we use passthrough mode for IOMMUv2 capable device.
-	 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to
-	 * invalid address), we ignore the capability for the device so
-	 * it'll be forced to go into translation mode.
-	 */
-	if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
-	    dev_is_pci(dev) && amd_iommu_gt_ppr_supported()) {
-		dev_data->flags = pdev_get_caps(to_pci_dev(dev));
-	}
+	/* Wait for DTE updates to go through */
+	iommu_completion_wait(iommu);
 
-	return 0;
+	return dev_data;
 }
 
-static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev)
-{
-	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
-	struct dev_table_entry *dev_table = get_dev_table(iommu);
-	int devid, sbdf;
-
-	sbdf = get_device_sbdf_id(dev);
-	if (sbdf < 0)
-		return;
-
-	devid = PCI_SBDF_TO_DEVID(sbdf);
-	pci_seg->rlookup_table[devid] = NULL;
-	memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));
-
-	setup_aliases(iommu, dev);
-}
 
 
 /****************************************************************************
@@ -2392,6 +2370,9 @@ static int attach_device(struct device *dev,
 	if (ret)
 		goto out;
 
+	if (dev_data->perfopt)
+		goto skip_caps;
+
 	/* Setup GCR3 table */
 	if (pdom_is_sva_capable(domain)) {
 		ret = init_gcr3_table(dev_data, domain);
@@ -2416,6 +2397,7 @@ static int attach_device(struct device *dev,
 		pdev_enable_cap_ats(pdev);
 	}
 
+skip_caps:
 	/* Update data structures */
 	dev_data->domain = domain;
 	spin_lock_irqsave(&domain->lock, flags);
@@ -2484,73 +2466,289 @@ static void detach_device(struct device *dev)
 	mutex_unlock(&dev_data->mutex);
 }
 
+static int iommu_init_device_caps(struct iommu_dev_data *dev_data,
+				  struct device *dev,
+				  struct amd_iommu *iommu)
+{
+	int ret;
+
+	if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
+		dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K;
+	else
+		dev_data->max_irqs = MAX_IRQS_PER_TABLE_512;
+
+	amd_iommu_set_pci_msi_domain(dev, iommu);
+
+	if (!dev_is_pci(dev))
+		return 0;
+
+	/*
+	 * By default we use passthrough mode for IOMMUv2 capable device.
+	 * But if amd_iommu=force_isolation is set (e.g. to debug DMA to
+	 * invalid address), we ignore the capability for the device so
+	 * it'll be forced to go into translation mode.
+	 */
+	if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
+	    amd_iommu_gt_ppr_supported()) {
+		dev_data->flags = pdev_get_caps(to_pci_dev(dev));
+	}
+
+	/*
+	 * If IOMMU and device supports PASID then it will contain max
+	 * supported PASIDs, else it will be zero.
+	 */
+	if (amd_iommu_pasid_supported() &&
+	    pdev_pasid_supported(dev_data)) {
+		dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,
+					     pci_max_pasids(to_pci_dev(dev)));
+	}
+
+	if (pci_ats_supported(to_pci_dev(dev))) {
+		ret = pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/* Program the per-IOMMU PerfOpt enable bit. Caller must hold iommu->lock. */
+static int __perfopt_write(struct amd_iommu *iommu, bool enable)
+{
+	u32 old, val, readback;
+
+	if (!(readq(iommu->mmio_base + MMIO_EXT_FEATURES) & FEATURE_PERF_OPT))
+		return enable ? -ENODEV : 0;
+
+	old = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
+	if (old == U32_MAX)
+		return -EIO;
+
+	val = enable ? old | PERF_OPT_EN : old & ~PERF_OPT_EN;
+	if (val != old)
+		writel(val, iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
+	readback = readl(iommu->mmio_base + MMIO_PERF_OPT_OFFSET);
+	if (readback == U32_MAX ||
+	    (readback & PERF_OPT_EN) != (val & PERF_OPT_EN))
+		return -EIO;
+	return 0;
+}
+
+/*
+ * PERF_OPT_EN is a single bit shared by every device behind @iommu, so it is
+ * reference counted: armed on the first requesting device, cleared on the last.
+ */
+static int perfopt_get(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	if (iommu->perfopt_refcount == 0) {
+		ret = __perfopt_write(iommu, true);
+		if (ret)
+			goto out;
+	}
+	iommu->perfopt_refcount++;
+out:
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+static int perfopt_put(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	if (iommu->perfopt_refcount > 0 && --iommu->perfopt_refcount == 0)
+		ret = __perfopt_write(iommu, false);
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+/*
+ * Force PERF_OPT_EN off without touching the refcount (used on init, shutdown,
+ * and suspend). The count is preserved so amd_iommu_perfopt_restore() can
+ * re-arm on resume.
+ */
+int amd_iommu_perfopt_clear(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	ret = __perfopt_write(iommu, false);
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+/*
+ * Re-assert PERF_OPT_EN from the refcount after the hardware was reprogrammed on
+ * resume, so devices armed before suspend keep the optimization without each
+ * consumer driver re-arming.
+ */
+int amd_iommu_perfopt_restore(struct amd_iommu *iommu)
+{
+	unsigned long flags;
+	int ret;
+
+	if (!iommu->mmio_base)
+		return 0;
+
+	raw_spin_lock_irqsave(&iommu->lock, flags);
+	ret = __perfopt_write(iommu, iommu->perfopt_refcount > 0);
+	raw_spin_unlock_irqrestore(&iommu->lock, flags);
+	return ret;
+}
+
+int amd_iommu_enable_perfopt(struct pci_dev *pdev)
+{
+	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
+	struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
+	struct protection_domain *domain;
+	int ret;
+
+	if (!iommu || !dev_data)
+		return -ENODEV;
+
+	if (!(iommu->features & FEATURE_PERF_OPT))
+		return -ENODEV;
+
+	domain = dev_data->domain;
+	if (!domain)
+		return -ENODEV;
+
+	/* Already armed for this device (e.g. re-entry on resume). */
+	if (dev_data->perfopt)
+		return 0;
+
+	/*
+	 * The bit is only architecturally valid while the device is untranslated:
+	 * identity domain with ATS/PRI/PASID off. The identity domain is
+	 * SVA-capable so attach_device() enabled ATS/PRI/PASID and built a GCR3
+	 * table. Re-home the device onto the same identity domain with
+	 * perfopt set, so the attach_device() skip_caps path leaves
+	 * ATS/PRI/PASID off and no GCR3 table. This follows the detach/attach
+	 * pattern used by amd_iommu_attach_device().
+	 *
+	 * Locking: this and amd_iommu_disable_perfopt() run only from the
+	 * consumer driver's bind/unbind path. group->mutex is not exposed to
+	 * drivers, but a device bound to its native driver cannot have its domain
+	 * changed concurrently by the core (VFIO ownership is mutually exclusive;
+	 * sysfs domain changes require an unused group), so the detach/attach pair
+	 * is serialized without it.
+	 */
+	dev_data->perfopt = true;
+	detach_device(&pdev->dev);
+	ret = attach_device(&pdev->dev, domain);
+	if (ret)
+		goto err_restore;
+
+	ret = perfopt_get(iommu);
+	if (ret)
+		goto err_rearm;
+
+	dev_info_once(&pdev->dev, "PerfOpt armed on IOMMU%d\n", iommu->index);
+	return 0;
+
+err_rearm:
+	detach_device(&pdev->dev);
+err_restore:
+	dev_data->perfopt = false;
+	if (attach_device(&pdev->dev, domain))
+		pci_err(pdev, "failed to restore state after PerfOpt setup; device left detached\n");
+	dev_err_once(&pdev->dev, "PerfOpt failed to arm on IOMMU%d (%d)\n",
+		     iommu->index, ret);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(amd_iommu_enable_perfopt);
+
+void amd_iommu_disable_perfopt(struct pci_dev *pdev)
+{
+	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);
+	struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev);
+	struct protection_domain *domain;
+
+	if (!iommu || !dev_data || !dev_data->perfopt || !dev_data->domain)
+		return;
+
+	if (WARN_ON(perfopt_put(iommu)))
+		pci_err(pdev, "failed to clear PerfOpt\n");
+
+	/*
+	 * Restore ATS/PRI/PASID (and thus SVA) by re-homing the device onto its
+	 * identity domain with the flag cleared, so a later bind without PerfOpt
+	 * sees a normally-capable device. See the locking note in
+	 * amd_iommu_enable_perfopt().
+	 */
+	domain = dev_data->domain;
+	dev_data->perfopt = false;
+	detach_device(&pdev->dev);
+	if (attach_device(&pdev->dev, domain))
+		pci_err(pdev, "failed to restore caps after PerfOpt disable\n");
+}
+EXPORT_SYMBOL_GPL(amd_iommu_disable_perfopt);
 static struct iommu_device *amd_iommu_probe_device(struct device *dev)
 {
 	struct iommu_device *iommu_dev;
 	struct amd_iommu *iommu;
 	struct iommu_dev_data *dev_data;
+	u16 devid;
 	int ret;
 
-	if (!check_device(dev))
-		return ERR_PTR(-ENODEV);
-
-	iommu = rlookup_amd_iommu(dev);
-	if (!iommu)
-		return ERR_PTR(-ENODEV);
-
-	/* Not registered yet? */
-	if (!iommu->iommu.ops)
+	if (!lookup_device(dev, &iommu, &devid))
 		return ERR_PTR(-ENODEV);
 
 	if (dev_iommu_priv_get(dev))
 		return &iommu->iommu;
 
-	ret = iommu_init_device(iommu, dev);
-	if (ret) {
+	dev_data = iommu_init_device(iommu, dev, devid);
+	if (IS_ERR(dev_data)) {
 		dev_err(dev, "Failed to initialize - trying to proceed anyway\n");
-		iommu_dev = ERR_PTR(ret);
-		iommu_ignore_device(iommu, dev);
-		goto out_err;
+		return ERR_CAST(dev_data);
 	}
 
-	amd_iommu_set_pci_msi_domain(dev, iommu);
+	ret = iommu_init_device_caps(dev_data, dev, iommu);
+	if (ret)
+		return ERR_PTR(ret);
+
 	iommu_dev = &iommu->iommu;
 
 	/*
-	 * If IOMMU and device supports PASID then it will contain max
-	 * supported PASIDs, else it will be zero.
+	 * When DMA translation is unavailable return error so the iommu core
+	 * won't attempt domain attach for this device, while preserving its
+	 * rlookup entry for interrupt remapping.
 	 */
-	dev_data = dev_iommu_priv_get(dev);
-	if (amd_iommu_pasid_supported() && dev_is_pci(dev) &&
-	    pdev_pasid_supported(dev_data)) {
-		dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,
-					     pci_max_pasids(to_pci_dev(dev)));
-	}
-
 	if (amd_iommu_pgtable == PD_MODE_NONE) {
 		pr_warn_once("%s: DMA translation not supported by iommu.\n",
 			     __func__);
-		iommu_dev = ERR_PTR(-ENODEV);
-		goto out_err;
+		return ERR_PTR(-ENODEV);
 	}
 
-	iommu_completion_wait(iommu);
-
-	if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
-		dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K;
-	else
-		dev_data->max_irqs = MAX_IRQS_PER_TABLE_512;
-
-	if (dev_is_pci(dev))
-		pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
-
-out_err:
 	return iommu_dev;
 }
 
 static void amd_iommu_release_device(struct device *dev)
 {
 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
+
+	if (dev_data->perfopt) {
+		if (WARN_ON(perfopt_put(iommu)))
+			dev_err(dev, "IOMMU%d: failed to clear PerfOpt on release\n",
+				iommu->index);
+		dev_data->perfopt = false;
+	}
 
 	WARN_ON(dev_data->domain);
 
@@ -2925,6 +3123,19 @@ static int blocked_domain_attach_device(struct iommu_domain *domain,
 					struct iommu_domain *old)
 {
 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
+
+	/*
+	 * blocked_domain is also the .release_domain, so this is the normal
+	 * teardown path: drop the reference and clear the flag here too, and
+	 * don't fail teardown if the WARN-guarded write doesn't stick.
+	 */
+	if (dev_data->perfopt) {
+		if (WARN_ON(perfopt_put(iommu)))
+			dev_err(dev, "IOMMU%d: failed to clear PerfOpt for blocked domain\n",
+				iommu->index);
+		dev_data->perfopt = false;
+	}
 
 	if (dev_data->domain)
 		detach_device(dev);
@@ -2993,6 +3204,9 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct device *dev,
 	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev);
 	int ret;
 
+	if (dev_data->perfopt && !pdom_is_in_pt_mode(domain))
+		return -EBUSY;
+
 	/*
 	 * Skip attach device to domain if new domain is same as
 	 * devices current domain
@@ -3162,6 +3376,26 @@ static bool amd_iommu_is_attach_deferred(struct device *dev)
 	return dev_data->defer_attach;
 }
 
+static bool quirks_force_identity_mapping(struct pci_dev *pdev)
+{
+	int class = pdev->class >> 8;
+
+	/* AMD GPU vendor ID */
+	if (pdev->vendor != PCI_VENDOR_ID_ATI)
+		return false;
+
+	/* GPU class */
+	if (class != PCI_CLASS_DISPLAY_VGA &&
+	    class != PCI_CLASS_DISPLAY_OTHER)
+		return false;
+
+	if (pci_upstream_bridge(pdev)->vendor == PCI_VENDOR_ID_ATI)
+		return false;
+
+	/* It is the GPU in an APU, force identity domain */
+	return true;
+}
+
 static int amd_iommu_def_domain_type(struct device *dev)
 {
 	struct iommu_dev_data *dev_data;
@@ -3170,20 +3404,23 @@ static int amd_iommu_def_domain_type(struct device *dev)
 	if (!dev_data)
 		return 0;
 
+	if (!dev_is_pci(dev))
+		return 0;
+
 	/* Always use DMA domain for untrusted device */
-	if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted)
+	if (to_pci_dev(dev)->untrusted)
 		return IOMMU_DOMAIN_DMA;
 
-	/*
-	 * Do not identity map IOMMUv2 capable devices when:
-	 *  - memory encryption is active, because some of those devices
-	 *    (AMD GPUs) don't have the encryption bit in their DMA-mask
-	 *    and require remapping.
-	 *  - SNP is enabled, because it prohibits DTE[Mode]=0.
-	 */
-	if (pdev_pasid_supported(dev_data) &&
-	    !cc_platform_has(CC_ATTR_MEM_ENCRYPT) &&
-	    !amd_iommu_snp_en) {
+	/* Apply device specific quirks */
+	if (quirks_force_identity_mapping(to_pci_dev(dev))) {
+		/*
+		 * When memory encryption is active, some of these devices
+		 * don't have the encryption bit in their DMA-mask and
+		 * require remapping.
+		 */
+		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+			return 0;
+
 		return IOMMU_DOMAIN_IDENTITY;
 	}
 
@@ -3902,8 +4139,11 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
 out_free_data:
 	for (i--; i >= 0; i--) {
 		irq_data = irq_domain_get_irq_data(domain, virq + i);
-		if (irq_data)
-			kfree(irq_data->chip_data);
+		if (irq_data && irq_data->chip_data) {
+			data = irq_data->chip_data;
+			kfree(data->entry);
+			kfree(data);
+		}
 	}
 	for (i = 0; i < nr_irqs; i++)
 		free_irte(iommu, devid, index + i);
diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c
index b160fb464c5fb2..ebf4547d32b41d 100644
--- a/drivers/iommu/apple-dart.c
+++ b/drivers/iommu/apple-dart.c
@@ -21,6 +21,7 @@
 #include <linux/io-pgtable.h>
 #include <linux/iommu.h>
 #include <linux/iopoll.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
@@ -267,6 +268,7 @@ struct apple_dart_domain {
 	struct io_pgtable_ops *pgtbl_ops;
 
 	bool finalized;
+	u64 mask;
 	struct mutex init_lock;
 	struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 
@@ -285,6 +287,13 @@ struct apple_dart_master_cfg {
 	/* Intersection of DART capabilitles */
 	u32 supports_bypass : 1;
 
+	/*
+	 * DMA aperture start and end to be overridden by "iommus"' phandle
+	 * args. By default determined by DART's ias but may be outside of it.
+	 */
+	dma_addr_t dma_min;
+	dma_addr_t dma_max;
+
 	struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 };
 
@@ -537,7 +546,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
 	if (!ops)
 		return 0;
 
-	return ops->iova_to_phys(ops, iova);
+	return ops->iova_to_phys(ops, iova & dart_domain->mask);
 }
 
 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
@@ -551,8 +560,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
 	if (!ops)
 		return -ENODEV;
 
-	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
-			      mapped);
+	return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize,
+			      pgcount, prot, gfp, mapped);
 }
 
 static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
@@ -563,7 +572,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
 	struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 
-	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
+	return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount,
+				gather);
 }
 
 static void
@@ -590,6 +600,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 {
 	struct apple_dart *dart = cfg->stream_maps[0].dart;
 	struct io_pgtable_cfg pgtbl_cfg;
+	u32 ias = min_t(u32, dart->ias, fls64(cfg->dma_max));
 	int ret = 0;
 	int i, j;
 
@@ -610,7 +621,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 
 	pgtbl_cfg = (struct io_pgtable_cfg){
 		.pgsize_bitmap = dart->pgsize,
-		.ias = dart->ias,
+		.ias = ias,
 		.oas = dart->oas,
 		.coherent_walk = 1,
 		.iommu_dev = dart->dev,
@@ -623,10 +634,10 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 		goto done;
 	}
 
+	dart_domain->mask = DMA_BIT_MASK(pgtbl_cfg.ias);
 	dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
-	dart_domain->domain.geometry.aperture_start = 0;
-	dart_domain->domain.geometry.aperture_end =
-		(dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias);
+	dart_domain->domain.geometry.aperture_start = cfg->dma_min;
+	dart_domain->domain.geometry.aperture_end = cfg->dma_max;
 	dart_domain->domain.geometry.force_aperture = true;
 
 	dart_domain->finalized = true;
@@ -803,20 +814,65 @@ static int apple_dart_of_xlate(struct device *dev,
 	struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
 	struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
 	struct apple_dart *cfg_dart;
+	dma_addr_t dma_max = DMA_BIT_MASK(dart->ias);
+	dma_addr_t dma_min = 0;
 	int i, sid;
 
 	put_device(&iommu_pdev->dev);
 
-	if (args->args_count != 1)
+	if (args->args_count != 1 && args->args_count != 5)
 		return -EINVAL;
+
 	sid = args->args[0];
 
+	if (args->args_count == 5) {
+		dma_addr_t length = ((dma_addr_t)args->args[3] << 32) | args->args[4];
+
+		if (!length)
+			return -EINVAL;
+
+		dma_min = ((dma_addr_t)args->args[1] << 32) | args->args[2];
+
+		if (!IS_ALIGNED(dma_min, dart->pgsize) ||
+		    !IS_ALIGNED(length, dart->pgsize)) {
+			dev_err(dev, "Unaligned DMA window %pad, %pad (0x%x)\n",
+				&dma_min, &length, dart->pgsize);
+			return -EINVAL;
+		}
+		if (check_add_overflow(dma_min, length - 1, &dma_max)) {
+			dev_err(dev, "DMA window length (%pad) overflows range for start %pad\n",
+				&length, &dma_min);
+			return -EINVAL;
+		}
+
+		/*
+		 * Ensure that the DMA window does not exceed the DART's ias.
+		 */
+		if ((dma_min ^ dma_max) & ~DMA_BIT_MASK(dart->ias)) {
+			dev_err(dev, "Invalid DMA window for ias=%d\n",
+				dart->ias);
+			return -EINVAL;
+		}
+	}
+
 	if (!cfg) {
 		cfg = kzalloc_obj(*cfg);
 		if (!cfg)
 			return -ENOMEM;
 		/* Will be ANDed with DART capabilities */
 		cfg->supports_bypass = true;
+		/* Will be merged with other DARTs to the common range. */
+		cfg->dma_min = dma_min;
+		cfg->dma_max = dma_max;
+	} else {
+		if (dma_min >= cfg->dma_max || cfg->dma_min >= dma_max) {
+			dev_err(dev, "non-overlapping DMA windows: %pad..%pad, %pad..%pad\n",
+				&dma_min, &dma_max,
+				&cfg->dma_min, &cfg->dma_max);
+			return -EINVAL;
+		}
+		cfg->dma_min = max(dma_min, cfg->dma_min);
+		cfg->dma_max = min(dma_max, cfg->dma_max);
 	}
 	dev_iommu_priv_set(dev, cfg);
 
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
index 25982bdbcbd9ab..ab1078a97d801e 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c
@@ -322,6 +322,106 @@ struct arm_vsmmu_invalidation_cmd {
 	};
 };
 
+/* Reject the range field values that the spec defines as Reserved */
+static int arm_vsmmu_validate_range(struct arm_smmu_device *smmu, u64 data[2])
+{
+	bool range = !!(data[0] & (CMDQ_TLBI_0_NUM | CMDQ_TLBI_0_SCALE));
+	u8 ttl = FIELD_GET(CMDQ_TLBI_1_TTL, data[1]);
+	u8 tg = FIELD_GET(CMDQ_TLBI_1_TG, data[1]);
+
+	/* NUM, SCALE and TTL are RES0 when TG == 0 */
+	if (!tg)
+		return (range || ttl) ? -EIO : 0;
+	/* TTL == 0b01 with a 16KB TG requires SMMU_IDR5.DS */
+	if (tg == 2 && ttl == 1 && !(smmu->features & ARM_SMMU_FEAT_DS))
+		return -EIO;
+	/* NUM == 0, SCALE == 0 with TTL == 0 is a reserved combination */
+	if (!range && !ttl)
+		return -EIO;
+	return 0;
+}
+
+static int arm_vsmmu_validate_user_cmd(struct arm_vsmmu *vsmmu, u64 data[2])
+{
+	struct arm_smmu_device *smmu = vsmmu->smmu;
+	u64 allowed[2] = { CMDQ_0_OP };
+
+	/* Collect the fields userspace is allowed to set for each opcode */
+	switch (data[0] & CMDQ_0_OP) {
+	case CMDQ_OP_TLBI_NSNH_ALL:
+		break;
+	case CMDQ_OP_TLBI_NH_VA:
+		/* An SMMU with 8-bit ASIDs treats the upper 8 bits as RES0 */
+		allowed[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID,
+					 GENMASK(smmu->asid_bits - 1, 0));
+		fallthrough;
+	case CMDQ_OP_TLBI_NH_VAA:
+		/* NUM/SCALE/TG/TTL are range fields gated on FEAT_RANGE_INV */
+		if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
+			if (arm_vsmmu_validate_range(smmu, data))
+				return -EIO;
+			allowed[0] |= CMDQ_TLBI_0_NUM | CMDQ_TLBI_0_SCALE;
+			allowed[1] |= CMDQ_TLBI_1_TG | CMDQ_TLBI_1_TTL;
+			/* SCALE bit 25 (values above 31) is RES0 without DS */
+			if (!(smmu->features & ARM_SMMU_FEAT_DS))
+				allowed[0] &= ~FIELD_PREP(CMDQ_TLBI_0_SCALE,
+							  BIT(5));
+		}
+		allowed[0] |= CMDQ_TLBI_0_VMID;
+		allowed[1] |= CMDQ_TLBI_1_LEAF | CMDQ_TLBI_1_VA_MASK;
+		break;
+	case CMDQ_OP_TLBI_NH_ASID:
+		/* An SMMU with 8-bit ASIDs treats the upper 8 bits as RES0 */
+		allowed[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID,
+					 GENMASK(smmu->asid_bits - 1, 0));
+		fallthrough;
+	case CMDQ_OP_TLBI_NH_ALL:
+		allowed[0] |= CMDQ_TLBI_0_VMID;
+		break;
+	case CMDQ_OP_ATC_INV:
+		/* ATC_INV is illegal unless the SMMU implements ATS */
+		if (!(smmu->features & ARM_SMMU_FEAT_ATS))
+			return -EIO;
+		/* A Size above 52 (invalidate-all) may raise a CERROR_ILL */
+		if (FIELD_GET(CMDQ_ATC_1_SIZE, data[1]) > ATC_INV_SIZE_ALL)
+			return -EIO;
+		/*
+		 * SSV/SSID/Global need substream support. SSID and Global are
+		 * IGNORED (not RES0) when SSV == 0, so they need no SSV check.
+		 */
+		if (smmu->ssid_bits)
+			allowed[0] |= CMDQ_0_SSV | CMDQ_ATC_0_SSID |
+				      CMDQ_ATC_0_GLOBAL;
+		allowed[0] |= CMDQ_ATC_0_SID;
+		allowed[1] |= CMDQ_ATC_1_SIZE | CMDQ_ATC_1_ADDR_MASK;
+		break;
+	case CMDQ_OP_CFGI_CD:
+		/* No SSV for CFGI_CD; SSID requires substream support */
+		if (smmu->ssid_bits)
+			allowed[0] |= CMDQ_CFGI_0_SSID;
+		allowed[1] |= CMDQ_CFGI_1_LEAF;
+		fallthrough;
+	case CMDQ_OP_CFGI_CD_ALL:
+		allowed[0] |= CMDQ_CFGI_0_SID;
+		break;
+	default:
+		return -EIO;
+	}
+
+	/*
+	 * Reject any other bit, e.g. a RES0 bit or a Secure bit, before the
+	 * command reaches the trusted main cmdq, so a guest cannot wedge the
+	 * shared queue for every device with a CERROR_ILL.
+	 *
+	 * By contrast, an out-of-range address or ID value does not need a
+	 * check: the spec defines it as CONSTRAINED UNPREDICTABLE, which is
+	 * scoped to the guest itself and does not raise a CERROR_ILL.
+	 */
+	if ((data[0] & ~allowed[0]) || (data[1] & ~allowed[1]))
+		return -EIO;
+	return 0;
+}
+
 /*
  * Convert, in place, the raw invalidation command into an internal format that
  * can be passed to arm_smmu_cmdq_issue_cmdlist(). Internally commands are
@@ -332,33 +432,40 @@ struct arm_vsmmu_invalidation_cmd {
 static int arm_vsmmu_convert_user_cmd(struct arm_vsmmu *vsmmu,
 				      struct arm_vsmmu_invalidation_cmd *cmd)
 {
+	u64 *data = cmd->cmd.data;
+	int ret;
+
 	/* Commands are le64 stored in u64 */
-	cmd->cmd.data[0] = le64_to_cpu(cmd->ucmd.cmd[0]);
-	cmd->cmd.data[1] = le64_to_cpu(cmd->ucmd.cmd[1]);
+	data[0] = le64_to_cpu(cmd->ucmd.cmd[0]);
+	data[1] = le64_to_cpu(cmd->ucmd.cmd[1]);
 
-	switch (cmd->cmd.data[0] & CMDQ_0_OP) {
+	ret = arm_vsmmu_validate_user_cmd(vsmmu, data);
+	if (ret)
+		return ret;
+
+	switch (data[0] & CMDQ_0_OP) {
 	case CMDQ_OP_TLBI_NSNH_ALL:
 		/* Convert to NH_ALL */
-		cmd->cmd.data[0] = CMDQ_OP_TLBI_NH_ALL |
-			      FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
-		cmd->cmd.data[1] = 0;
+		data[0] = CMDQ_OP_TLBI_NH_ALL |
+			  FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
+		data[1] = 0;
 		break;
 	case CMDQ_OP_TLBI_NH_VA:
 	case CMDQ_OP_TLBI_NH_VAA:
 	case CMDQ_OP_TLBI_NH_ALL:
 	case CMDQ_OP_TLBI_NH_ASID:
-		cmd->cmd.data[0] &= ~CMDQ_TLBI_0_VMID;
-		cmd->cmd.data[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
+		data[0] &= ~CMDQ_TLBI_0_VMID;
+		data[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vsmmu->vmid);
 		break;
 	case CMDQ_OP_ATC_INV:
 	case CMDQ_OP_CFGI_CD:
 	case CMDQ_OP_CFGI_CD_ALL: {
-		u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, cmd->cmd.data[0]);
+		u32 sid, vsid = FIELD_GET(CMDQ_CFGI_0_SID, data[0]);
 
 		if (arm_vsmmu_vsid_to_sid(vsmmu, vsid, &sid))
 			return -EIO;
-		cmd->cmd.data[0] &= ~CMDQ_CFGI_0_SID;
-		cmd->cmd.data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
+		data[0] &= ~CMDQ_CFGI_0_SID;
+		data[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid);
 		break;
 	}
 	default:
@@ -371,49 +478,57 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu,
 			       struct iommu_user_data_array *array)
 {
 	struct arm_vsmmu *vsmmu = container_of(viommu, struct arm_vsmmu, core);
+	struct arm_vsmmu_invalidation_cmd cmds[CMDQ_BATCH_ENTRIES - 1];
 	struct arm_smmu_device *smmu = vsmmu->smmu;
-	struct arm_vsmmu_invalidation_cmd *last;
-	struct arm_vsmmu_invalidation_cmd *cmds;
-	struct arm_vsmmu_invalidation_cmd *cur;
-	struct arm_vsmmu_invalidation_cmd *end;
+	struct iommu_user_data_array batch = {
+		.type = array->type,
+		.uptr = array->uptr,
+		.entry_len = array->entry_len,
+	};
+	u32 processed = 0;
 	int ret;
-
-	cmds = kzalloc_objs(*cmds, array->entry_num);
-	if (!cmds)
-		return -ENOMEM;
-	cur = cmds;
-	end = cmds + array->entry_num;
+	u32 i;
 
 	static_assert(sizeof(*cmds) == 2 * sizeof(u64));
+
+	if (array->type != IOMMU_VIOMMU_INVALIDATE_DATA_ARM_SMMUV3) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* A zero-length array only probes the type, validated above */
+	if (!array->entry_num)
+		return 0;
+
+	/*
+	 * The core re-invokes this op for the remaining requests, so copy one
+	 * cmdq batch worth of commands into a fixed on-stack buffer rather than
+	 * allocating for the whole array.
+	 */
+	batch.entry_num = min_t(u32, array->entry_num, ARRAY_SIZE(cmds));
 	ret = iommu_copy_struct_from_full_user_array(
-		cmds, sizeof(*cmds), array,
+		cmds, sizeof(*cmds), &batch,
 		IOMMU_VIOMMU_INVALIDATE_DATA_ARM_SMMUV3);
 	if (ret)
 		goto out;
 
-	last = cmds;
-	while (cur != end) {
-		ret = arm_vsmmu_convert_user_cmd(vsmmu, cur);
+	/*
+	 * Convert the whole batch. Sending an illegal command is a VMM bug, so
+	 * a single one fails the entire batch, issuing nothing.
+	 */
+	for (i = 0; i < batch.entry_num; i++) {
+		ret = arm_vsmmu_convert_user_cmd(vsmmu, &cmds[i]);
 		if (ret)
 			goto out;
-
-		/* FIXME work in blocks of CMDQ_BATCH_ENTRIES and copy each block? */
-		cur++;
-		if (cur != end && (cur - last) != CMDQ_BATCH_ENTRIES - 1)
-			continue;
-
-		/* FIXME always uses the main cmdq rather than trying to group by type */
-		ret = __arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &last->cmd,
-						    cur - last, true);
-		if (ret) {
-			cur--;
-			goto out;
-		}
-		last = cur;
 	}
+
+	/* FIXME always uses the main cmdq rather than trying to group by type */
+	ret = __arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, &cmds->cmd,
+					    batch.entry_num, true);
+	if (!ret)
+		processed = batch.entry_num;
 out:
-	array->entry_num = cur - cmds;
-	kfree(cmds);
+	array->entry_num = processed;
 	return ret;
 }
 
diff --git a/drivers/iommu/bcm2712-iommu-cache.c b/drivers/iommu/bcm2712-iommu-cache.c
new file mode 100644
index 00000000000000..8ed436943b8304
--- /dev/null
+++ b/drivers/iommu/bcm2712-iommu-cache.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * IOMMU driver for BCM2712 TLB cache
+ *
+ * Copyright (c) 2023 Raspberry Pi Ltd.
+ * Copyright (c) 2026 Daniel Drake
+ *
+ * The BCM2712 IOMMUC is a centralized TLB which accelerates address translation
+ * across the SoC's IOMMU devices. If an address mapping is not found in the
+ * IOMMU's local TLB cache, then this IOMMUC is consulted. The IOMMUC must be
+ * explicitly invalidated when modifying or unmapping IOMMU page tables.
+ */
+
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/err.h>
+#include <linux/iopoll.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+#include "bcm2712-iommu-cache.h"
+
+struct bcm2712_iommu_cache {
+	struct device *dev;
+	spinlock_t hw_lock;
+	void __iomem *reg_base;
+};
+
+#define MMUC_CONTROL_ENABLE   BIT(0)
+#define MMUC_CONTROL_FLUSH    BIT(1)
+#define MMUC_CONTROL_FLUSHING BIT(2)
+
+void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache)
+{
+	u32 val;
+	int ret;
+
+	scoped_guard(spinlock_irqsave, &cache->hw_lock) {
+		writel(MMUC_CONTROL_ENABLE | MMUC_CONTROL_FLUSH,
+		       cache->reg_base);
+
+		ret = readl_poll_timeout_atomic(cache->reg_base, val,
+						!(val & MMUC_CONTROL_FLUSHING),
+						0, 50);
+	}
+
+	if (ret)
+		dev_err_ratelimited(cache->dev, "cache flush timed out\n");
+}
+
+static int bcm2712_iommu_cache_probe(struct platform_device *pdev)
+{
+	struct bcm2712_iommu_cache *cache;
+
+	cache = devm_kzalloc(&pdev->dev, sizeof(*cache), GFP_KERNEL);
+	if (!cache)
+		return -ENOMEM;
+
+	cache->dev = &pdev->dev;
+	spin_lock_init(&cache->hw_lock);
+
+	cache->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(cache->reg_base))
+		return PTR_ERR(cache->reg_base);
+
+	platform_set_drvdata(pdev, cache);
+	return 0;
+}
+
+static const struct of_device_id bcm2712_iommu_cache_of_match[] = {
+	{ .compatible = "brcm,bcm2712-iommuc" },
+	{ /* sentinel */ },
+};
+
+static struct platform_driver bcm2712_iommu_cache_driver = {
+	.probe = bcm2712_iommu_cache_probe,
+	.driver = {
+		.name = "bcm2712-iommu-cache",
+		.of_match_table = bcm2712_iommu_cache_of_match,
+		.suppress_bind_attrs = true,
+	},
+};
+builtin_platform_driver(bcm2712_iommu_cache_driver);
diff --git a/drivers/iommu/bcm2712-iommu-cache.h b/drivers/iommu/bcm2712-iommu-cache.h
new file mode 100644
index 00000000000000..d2f7851831ccb5
--- /dev/null
+++ b/drivers/iommu/bcm2712-iommu-cache.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _BCM2712_IOMMU_CACHE_H
+#define _BCM2712_IOMMU_CACHE_H
+
+struct bcm2712_iommu_cache;
+
+void bcm2712_iommu_cache_flush(struct bcm2712_iommu_cache *cache);
+
+#endif
diff --git a/drivers/iommu/bcm2712-iommu.c b/drivers/iommu/bcm2712-iommu.c
new file mode 100644
index 00000000000000..301855475c88bf
--- /dev/null
+++ b/drivers/iommu/bcm2712-iommu.c
@@ -0,0 +1,550 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * IOMMU driver for Broadcom BCM2712
+ *
+ * Copyright (c) 2023-2025 Raspberry Pi Ltd.
+ * Copyright (c) 2026 Daniel Drake
+ *
+ * Each BCM2712 IOMMU has multiple devices hardwired into it, whose
+ * DMA transactions all route through the IOMMU. There is no stream ID tagging
+ * or any other kind of segmentation to differentiate between requests from
+ * different devices. It is also not possible to toggle a specific device
+ * between iommu-mapped and bypass modes.
+ *
+ * The hardware supports the simultaneous configuration of a bypass window
+ * (where memory accesses operate in identity mode) and a translation aperture
+ * (supporting page mapping). While combining these two would allow for
+ * GART-like operation, this driver's paging domain implementation configures
+ * only the translation aperture (starting at IOVA 0) offering a degree of
+ * memory protection.
+ *
+ * The page table format is a two-level format handled by generic_pt/bcm2712.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/cleanup.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/iommu.h>
+#include <linux/iopoll.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/sizes.h>
+#include <linux/generic_pt/iommu.h>
+
+#include "bcm2712-iommu-cache.h"
+#include "dma-iommu.h"
+#include "iommu-pages.h"
+
+/* BCM2712 IOMMU is organized around 4Kbyte pages */
+#define IOMMU_PAGE_SHIFT       12
+#define IOMMU_PAGE_SIZE        (1ul << IOMMU_PAGE_SHIFT)
+/* A PTE is 4 bytes */
+#define PTE_SIZE_SHIFT         2
+/* L1/L2 table sizing (IOMMU hardware pages): 1024 entries per page */
+#define PTES_PER_IOPG_SHIFT    (IOMMU_PAGE_SHIFT - PTE_SIZE_SHIFT)
+
+#define MMMU_CTRL_OFFSET                       0x00
+#define MMMU_CTRL_CAP_EXCEEDED                 BIT(27)
+#define MMMU_CTRL_CAP_EXCEEDED_ABORT_EN        BIT(26)
+#define MMMU_CTRL_CAP_EXCEEDED_INT_EN          BIT(25)
+#define MMMU_CTRL_CAP_EXCEEDED_EXCEPTION_EN    BIT(24)
+#define MMMU_CTRL_PT_INVALID                   BIT(20)
+#define MMMU_CTRL_PT_INVALID_ABORT_EN          BIT(19)
+#define MMMU_CTRL_PT_INVALID_INT_EN            BIT(18)
+#define MMMU_CTRL_PT_INVALID_EXCEPTION_EN      BIT(17)
+#define MMMU_CTRL_PT_INVALID_EN                BIT(16)
+#define MMMU_CTRL_WRITE_VIOLATION              BIT(12)
+#define MMMU_CTRL_WRITE_VIOLATION_ABORT_EN     BIT(11)
+#define MMMU_CTRL_WRITE_VIOLATION_INT_EN       BIT(10)
+#define MMMU_CTRL_WRITE_VIOLATION_EXCEPTION_EN BIT(9)
+#define MMMU_CTRL_BYPASS                       BIT(8)
+#define MMMU_CTRL_TLB_CLEARING                 BIT(7)
+#define MMMU_CTRL_STATS_CLEAR                  BIT(3)
+#define MMMU_CTRL_TLB_CLEAR                    BIT(2)
+#define MMMU_CTRL_STATS_ENABLE                 BIT(1)
+#define MMMU_CTRL_ENABLE                       BIT(0)
+
+#define MMMU_CTRL_OPERATING_FLAGS (\
+	MMMU_CTRL_CAP_EXCEEDED_ABORT_EN    | \
+	MMMU_CTRL_PT_INVALID_ABORT_EN      | \
+	MMMU_CTRL_PT_INVALID_EN            | \
+	MMMU_CTRL_WRITE_VIOLATION_ABORT_EN | \
+	MMMU_CTRL_STATS_ENABLE             | \
+	MMMU_CTRL_ENABLE)
+
+#define MMMU_PT_PA_BASE_OFFSET                 0x04
+
+#define MMMU_ADDR_CAP_OFFSET                   0x14
+#define MMMU_ADDR_CAP_ENABLE                   BIT(31)
+#define ADDR_CAP_SHIFT                         ilog2(SZ_256M)
+
+#define MMMU_SHOOT_DOWN_OFFSET                 0x18
+#define MMMU_SHOOT_DOWN_SHOOTING               BIT(31)
+#define MMMU_SHOOT_DOWN_SHOOT                  BIT(30)
+
+#define MMMU_BYPASS_START_OFFSET               0x1c
+#define MMMU_BYPASS_START_ENABLE               BIT(31)
+
+#define MMMU_BYPASS_END_OFFSET                 0x20
+#define MMMU_BYPASS_END_ENABLE                 BIT(31)
+
+#define MMMU_MISC_OFFSET                       0x24
+#define MMMU_MISC_SINGLE_TABLE                 BIT(31)
+
+#define MMMU_ILLEGAL_ADR_OFFSET                0x30
+#define MMMU_ILLEGAL_ADR_ENABLE                BIT(31)
+
+#define MMMU_DEBUG_INFO_OFFSET                 0x38
+#define MMMU_DEBUG_INFO_VERSION_MASK           0x0000000Fu
+#define MMMU_DEBUG_INFO_VA_WIDTH_MASK          0x000000F0u
+#define MMMU_DEBUG_INFO_PA_WIDTH_MASK          0x00000F00u
+#define MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK     0x000FF000u
+#define MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK   0x0FF00000u
+#define MMMU_DEBUG_INFO_BYPASS_4M              BIT(28)
+#define MMMU_DEBUG_INFO_BYPASS                 BIT(29)
+
+struct bcm2712_iommu {
+	struct device *dev;
+	struct iommu_device iommu;
+	struct bcm2712_iommu_domain *domain;
+	struct bcm2712_iommu_cache *cache;
+	void __iomem *reg_base;
+	spinlock_t hw_lock;
+	size_t bigpage_size;
+	size_t superpage_size;
+};
+
+struct bcm2712_iommu_domain {
+	union {
+		struct iommu_domain base;
+		struct pt_iommu_bcm2712 pt;
+	};
+	struct bcm2712_iommu *mmu;
+	void *default_page;
+};
+
+static inline void bcm2712_iommu_writel(struct bcm2712_iommu *mmu,
+					unsigned int offset, u32 val)
+{
+	writel(val, mmu->reg_base + offset);
+}
+
+static inline u32 bcm2712_iommu_readl(struct bcm2712_iommu *mmu,
+				      unsigned int offset)
+{
+	return readl(mmu->reg_base + offset);
+}
+
+static struct bcm2712_iommu_domain *
+to_bcm2712_domain(struct iommu_domain *domain)
+{
+	return container_of(domain, struct bcm2712_iommu_domain, base);
+}
+
+static void bcm2712_iommu_init(struct bcm2712_iommu *mmu)
+{
+	unsigned int bigpage_width, superpage_width;
+	u32 u = bcm2712_iommu_readl(mmu, MMMU_DEBUG_INFO_OFFSET);
+	u32 pa_width = FIELD_GET(MMMU_DEBUG_INFO_PA_WIDTH_MASK, u);
+
+	dev_dbg(mmu->dev, "DEBUG_INFO = 0x%08x\n", u);
+	WARN_ON(FIELD_GET(MMMU_DEBUG_INFO_VERSION_MASK, u) < 4 ||
+		FIELD_GET(MMMU_DEBUG_INFO_VA_WIDTH_MASK, u) < 6 ||
+		pa_width < 6 || !(u & MMMU_DEBUG_INFO_BYPASS));
+
+	dma_set_mask_and_coherent(mmu->dev, DMA_BIT_MASK(pa_width + 30u));
+
+	bigpage_width = FIELD_GET(MMMU_DEBUG_INFO_BIGPAGE_WIDTH_MASK, u);
+	if (bigpage_width)
+		mmu->bigpage_size = IOMMU_PAGE_SIZE << bigpage_width;
+
+	superpage_width = FIELD_GET(MMMU_DEBUG_INFO_SUPERPAGE_WIDTH_MASK, u);
+	if (superpage_width)
+		mmu->superpage_size = IOMMU_PAGE_SIZE << superpage_width;
+
+	/* Disable MMU and clear sticky flags */
+	bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET,
+			     MMMU_CTRL_CAP_EXCEEDED | MMMU_CTRL_PT_INVALID |
+			     MMMU_CTRL_WRITE_VIOLATION |
+			     MMMU_CTRL_STATS_CLEAR);
+
+	/* Put MMU into 2-level mode */
+	bcm2712_iommu_writel(mmu, MMMU_MISC_OFFSET,
+			     bcm2712_iommu_readl(mmu, MMMU_MISC_OFFSET) &
+				     ~MMMU_MISC_SINGLE_TABLE);
+}
+
+static int bcm2712_iommu_identity_attach(struct iommu_domain *identity_domain,
+					 struct device *dev,
+					 struct iommu_domain *old)
+{
+	struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
+
+	guard(spinlock_irqsave)(&mmu->hw_lock);
+	bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
+	mmu->domain = NULL;
+
+	return 0;
+}
+
+static struct iommu_domain bcm2712_identity_domain = {
+	.type = IOMMU_DOMAIN_IDENTITY,
+	.ops = &(const struct iommu_domain_ops) {
+		.attach_dev = bcm2712_iommu_identity_attach,
+	},
+};
+
+static int bcm2712_iommu_clear_and_enable(struct bcm2712_iommu *mmu)
+	__must_hold(&mmu->hw_lock)
+{
+	u32 val;
+
+	bcm2712_iommu_cache_flush(mmu->cache);
+	bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET,
+			     MMMU_CTRL_OPERATING_FLAGS | MMMU_CTRL_TLB_CLEAR);
+	return readl_poll_timeout_atomic(mmu->reg_base + MMMU_CTRL_OFFSET, val,
+					 !(val & MMMU_CTRL_TLB_CLEARING), 0,
+					 50);
+}
+
+
+static int bcm2712_iommu_attach_dev(struct iommu_domain *domain,
+				    struct device *dev,
+				    struct iommu_domain *old)
+{
+	struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
+	struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain);
+	struct pt_iommu_bcm2712_hw_info info;
+	u32 default_page_pfn;
+	int ret = 0;
+
+	/* all attached devices must belong to the same IOMMU instance */
+	if (mydomain->mmu != mmu)
+		return -EINVAL;
+
+	scoped_guard(spinlock_irqsave, &mmu->hw_lock) {
+		if (mmu->domain == mydomain)
+			break;
+
+		/* Configure translation aperture */
+		bcm2712_iommu_writel(mmu, MMMU_ADDR_CAP_OFFSET,
+				     MMMU_ADDR_CAP_ENABLE |
+					     (domain->geometry.aperture_end >>
+					      ADDR_CAP_SHIFT));
+
+		/*
+		 * When the IOMMU handles a request, it adds the
+		 * PT_PA_BASE_OFFSET to (IOVA>>32) to calculate the PFN of the
+		 * corresponding L1 directory page.
+		 * IOVA bits [31:22] are then used to fetch the L1 descriptor
+		 * within (which in turn points to the L2 table).
+		 * This clever logic would allow for a L1 table larger than 4kb
+		 * (and hence a larger aperture).
+		 */
+		pt_iommu_bcm2712_hw_info(&mydomain->pt, &info);
+		bcm2712_iommu_writel(mmu, MMMU_PT_PA_BASE_OFFSET,
+				     info.pt_base >> IOMMU_PAGE_SHIFT);
+
+		/* Disable bypass window */
+		bcm2712_iommu_writel(mmu, MMMU_BYPASS_START_OFFSET, 0);
+		bcm2712_iommu_writel(mmu, MMMU_BYPASS_END_OFFSET, 0);
+
+		/*
+		 * A default (error) page is used to catch illegal reads/writes.
+		 */
+		default_page_pfn = virt_to_phys(mydomain->default_page) >>
+				   IOMMU_PAGE_SHIFT;
+		bcm2712_iommu_writel(mmu, MMMU_ILLEGAL_ADR_OFFSET,
+				     MMMU_ILLEGAL_ADR_ENABLE |
+					     default_page_pfn);
+
+		ret = bcm2712_iommu_clear_and_enable(mmu);
+		if (ret) {
+			bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
+			mmu->domain = NULL;
+			break;
+		}
+
+		mmu->domain = mydomain;
+	}
+
+	if (ret)
+		dev_err_ratelimited(mmu->dev,
+				    "TLB clear timed out during attach\n");
+
+	return ret;
+}
+
+static int bcm2712_iommu_shootdown_range(struct bcm2712_iommu *mmu,
+					 unsigned long iova, size_t size)
+	__must_hold(&mmu->hw_lock)
+{
+	unsigned long start_pfn = iova >> IOMMU_PAGE_SHIFT;
+	unsigned long end_pfn = (iova + size - 1) >> IOMMU_PAGE_SHIFT;
+	unsigned long pfn;
+	u32 val;
+	int ret;
+
+	/* Shootdown register deals with 4 pages at a time */
+	for (pfn = round_down(start_pfn, 4); pfn <= end_pfn; pfn += 4) {
+		bcm2712_iommu_writel(mmu, MMMU_SHOOT_DOWN_OFFSET,
+				     MMMU_SHOOT_DOWN_SHOOT | pfn);
+		ret = readl_poll_timeout_atomic(
+			mmu->reg_base + MMMU_SHOOT_DOWN_OFFSET, val,
+			!(val & MMMU_SHOOT_DOWN_SHOOTING), 0, 50);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int bcm2712_iommu_sync_range(struct iommu_domain *domain,
+				    unsigned long iova, size_t size)
+{
+	struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain);
+	struct bcm2712_iommu *mmu = mydomain->mmu;
+	int ret = 0;
+
+	if (!size)
+		return 0;
+
+	scoped_guard(spinlock_irqsave, &mmu->hw_lock) {
+		if (mmu->domain != mydomain)
+			return 0;
+
+		/* If invalidating more than 1MB, just do a full TLB clear */
+		if (size >= SZ_1M) {
+			ret = bcm2712_iommu_clear_and_enable(mmu);
+		} else {
+			bcm2712_iommu_cache_flush(mmu->cache);
+			ret = bcm2712_iommu_shootdown_range(mmu, iova, size);
+		}
+	}
+
+	if (ret)
+		dev_err_ratelimited(mmu->dev,
+				    "TLB sync timed out (size=%#zx)\n", size);
+
+	return ret;
+}
+
+static void bcm2712_iommu_sync(struct iommu_domain *domain,
+			       struct iommu_iotlb_gather *gather)
+{
+	bcm2712_iommu_sync_range(domain, gather->start,
+				 gather->end - gather->start + 1);
+	iommu_put_pages_list(&gather->freelist);
+}
+
+static int bcm2712_iommu_sync_map(struct iommu_domain *domain,
+				  unsigned long iova, size_t size)
+{
+	return bcm2712_iommu_sync_range(domain, iova, size);
+}
+
+static void bcm2712_iommu_sync_all(struct iommu_domain *domain)
+{
+	size_t aperture_size = domain->geometry.aperture_end -
+			       domain->geometry.aperture_start + 1;
+
+	bcm2712_iommu_sync_range(domain, domain->geometry.aperture_start,
+				 aperture_size);
+}
+
+static void bcm2712_iommu_domain_free(struct iommu_domain *domain)
+{
+	struct bcm2712_iommu_domain *mydomain = to_bcm2712_domain(domain);
+	struct bcm2712_iommu *mmu = mydomain->mmu;
+
+	scoped_guard(spinlock_irqsave, &mmu->hw_lock) {
+		if (mmu->domain == mydomain) {
+			bcm2712_iommu_writel(mmu, MMMU_CTRL_OFFSET, 0);
+			mmu->domain = NULL;
+		}
+	}
+
+	pt_iommu_deinit(&mydomain->pt.iommu);
+	if (mydomain->default_page)
+		iommu_free_pages(mydomain->default_page);
+	kfree(mydomain);
+}
+
+static const struct iommu_domain_ops bcm2712_paging_domain_ops = {
+	IOMMU_PT_DOMAIN_OPS(bcm2712),
+	.attach_dev	 = bcm2712_iommu_attach_dev,
+	.iotlb_sync      = bcm2712_iommu_sync,
+	.iotlb_sync_map  = bcm2712_iommu_sync_map,
+	.flush_iotlb_all = bcm2712_iommu_sync_all,
+	.free		 = bcm2712_iommu_domain_free,
+};
+
+static struct iommu_domain *bcm2712_iommu_domain_alloc(struct device *dev)
+{
+	struct bcm2712_iommu *mmu = dev_iommu_priv_get(dev);
+	struct bcm2712_iommu_domain *domain;
+	struct pt_iommu_bcm2712_cfg cfg = {};
+	int ret;
+
+	domain = kzalloc_obj(*domain);
+	if (!domain)
+		return NULL;
+
+	domain->mmu = mmu;
+	domain->pt.iommu.iommu_device = mmu->dev;
+	cfg.common.features = BIT(PT_FEAT_DMA_INCOHERENT);
+
+	/* Bigpage and superpage sizes are typically 64K and 1M, but may vary */
+	if (mmu->bigpage_size)
+		cfg.bigpage_lg2 = ilog2(mmu->bigpage_size);
+	if (mmu->superpage_size)
+		cfg.superpage_lg2 = ilog2(mmu->superpage_size);
+
+	/* 2-level format: 10-bit L1 + 10-bit L2 + 12-bit page offset */
+	cfg.common.hw_max_vasz_lg2 =
+		(2 * PTES_PER_IOPG_SHIFT) + IOMMU_PAGE_SHIFT;
+
+	/* PTEs encode a 25-bit output address PFN */
+	cfg.common.hw_max_oasz_lg2 = 25 + IOMMU_PAGE_SHIFT;
+
+	ret = pt_iommu_bcm2712_init(&domain->pt, &cfg, GFP_KERNEL);
+	if (ret)
+		goto err;
+
+	/* Set up a default (error) page used to catch illegal reads/writes */
+	domain->default_page = iommu_alloc_pages_sz(GFP_KERNEL, PAGE_SIZE);
+	if (!domain->default_page)
+		goto err;
+
+	domain->base.ops = &bcm2712_paging_domain_ops;
+	return &domain->base;
+
+err:
+	bcm2712_iommu_domain_free(&domain->base);
+	return NULL;
+}
+
+static struct bcm2712_iommu *
+bcm2712_iommu_get_by_fwnode(struct fwnode_handle *fwnode)
+{
+	struct device *dev __free(put_device) =
+		bus_find_device_by_fwnode(&platform_bus_type, fwnode);
+
+	return dev ? dev_get_drvdata(dev) : NULL;
+}
+
+static struct iommu_device *bcm2712_iommu_probe_device(struct device *dev)
+{
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct bcm2712_iommu *mmu;
+
+	if (!fwspec || !fwspec->iommu_fwnode)
+		return ERR_PTR(-ENODEV);
+
+	mmu = bcm2712_iommu_get_by_fwnode(fwspec->iommu_fwnode);
+	if (!mmu)
+		return ERR_PTR(-ENODEV);
+
+	dev_iommu_priv_set(dev, mmu);
+
+	return &mmu->iommu;
+}
+
+static int bcm2712_iommu_of_xlate(struct device *dev,
+				  const struct of_phandle_args *args)
+{
+	return iommu_fwspec_add_ids(dev, args->args, 0);
+}
+
+static const struct iommu_ops bcm2712_iommu_ops = {
+	.identity_domain     = &bcm2712_identity_domain,
+	.domain_alloc_paging = bcm2712_iommu_domain_alloc,
+	.probe_device        = bcm2712_iommu_probe_device,
+	.device_group        = generic_single_device_group,
+	.get_resv_regions    = iommu_dma_get_resv_regions,
+	.of_xlate            = bcm2712_iommu_of_xlate,
+};
+
+static const struct of_device_id bcm2712_iommu_of_match[] = {
+	{ .compatible = "brcm,bcm2712-iommu" },
+	{ /* sentinel */ }
+};
+
+static int bcm2712_iommu_init_cache(struct bcm2712_iommu *mmu,
+				    struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	struct device_node *cache_np __free(device_node) =
+		of_parse_phandle(dev->of_node, "brcm,iommu-cache", 0);
+	if (!cache_np)
+		return dev_err_probe(dev, -ENOENT,
+				     "missing brcm,iommu-cache property\n");
+
+	struct platform_device *cache_pdev __free(platform_device_put) =
+		of_find_device_by_node(cache_np);
+	if (!cache_pdev)
+		return dev_err_probe(dev, -EPROBE_DEFER,
+				     "waiting for cache device\n");
+
+	mmu->cache = platform_get_drvdata(cache_pdev);
+	if (!mmu->cache)
+		return dev_err_probe(dev, -EPROBE_DEFER,
+				     "waiting for cache driver probe\n");
+
+	return 0;
+}
+
+static int bcm2712_iommu_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct bcm2712_iommu *mmu;
+	int ret;
+
+	mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);
+	if (!mmu)
+		return -ENOMEM;
+
+	mmu->dev = dev;
+	spin_lock_init(&mmu->hw_lock);
+
+	mmu->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(mmu->reg_base))
+		return PTR_ERR(mmu->reg_base);
+
+	ret = bcm2712_iommu_init_cache(mmu, pdev);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, mmu);
+	bcm2712_iommu_init(mmu);
+
+	ret = iommu_device_sysfs_add(&mmu->iommu, dev, NULL, "%s",
+				     dev_name(dev));
+	if (ret)
+		return ret;
+
+	ret = iommu_device_register(&mmu->iommu, &bcm2712_iommu_ops, dev);
+	if (ret) {
+		iommu_device_sysfs_remove(&mmu->iommu);
+		return ret;
+	}
+
+	return 0;
+}
+
+static struct platform_driver bcm2712_iommu_driver = {
+	.driver	= {
+		.name		     = "bcm2712-iommu",
+		.of_match_table	     = bcm2712_iommu_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe	= bcm2712_iommu_probe,
+};
+builtin_platform_driver(bcm2712_iommu_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Daniel Drake <dan@reactivated.net>");
+MODULE_DESCRIPTION("Broadcom BCM2712 IOMMU driver");
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 58c624513cd430..26b9c2d71cd309 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1228,9 +1228,9 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 	 * If both the physical buffer start address and size are page aligned,
 	 * we don't need to use a bounce page.
 	 */
-	if (dev_use_swiotlb(dev, size, dir) &&
+	if (!(attrs & DMA_ATTR_MMIO) && dev_use_swiotlb(dev, size, dir) &&
 	    iova_unaligned(iovad, phys, size)) {
-		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
+		if (attrs & DMA_ATTR_REQUIRE_COHERENT)
 			return DMA_MAPPING_ERROR;
 
 		phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs);
@@ -1628,10 +1628,12 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size,
 	void *cpu_addr;
 
 	page = dma_alloc_contiguous(dev, alloc_size, gfp);
-	if (!page)
-		page = alloc_pages_node(node, gfp, get_order(alloc_size));
-	if (!page)
-		return NULL;
+	if (!page) {
+		cpu_addr = alloc_pages_exact_nid(node, alloc_size, gfp);
+		if (!cpu_addr)
+			return NULL;
+		page = virt_to_page(cpu_addr);
+	}
 
 	if (!coherent || PageHighMem(page)) {
 		pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
@@ -1762,7 +1764,7 @@ unsigned long iommu_dma_get_merge_boundary(struct device *dev)
 	return (1UL << __ffs(domain->pgsize_bitmap)) - 1;
 }
 
-size_t iommu_dma_opt_mapping_size(void)
+size_t iommu_dma_max_opt_mapping_size(void)
 {
 	return iova_rcache_range();
 }
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 874d05f4b39694..0bca3662fdc281 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -8,6 +8,7 @@
 #define DEBUG
 #endif
 
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
@@ -143,6 +144,10 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
 #define CTRL_VM_FAULT_MODE_STALL	BIT(3)
 #define CAPA0_CAPA1_EXIST		BIT(11)
 #define CAPA1_VCR_ENABLED		BIT(14)
+#define CAPA1_NO_BLOCK_MODE		BIT(15)
+#define FAULT_INFO_AXID			GENMASK(15, 0)
+#define FAULT_INFO_AXLEN		GENMASK(19, 16)
+#define FAULT_INFO_WRITE		BIT(20)
 
 /* common registers */
 #define REG_MMU_CTRL		0x000
@@ -193,6 +198,7 @@ struct sysmmu_fault {
 	sysmmu_iova_t addr;	/* IOVA address that caused fault */
 	const char *name;	/* human readable fault name */
 	unsigned int type;	/* fault type for report_iommu_fault() */
+	u32 info;		/* raw transaction info (v7+ only) */
 };
 
 struct sysmmu_v1_fault_info {
@@ -306,6 +312,7 @@ struct sysmmu_drvdata {
 
 	/* v7 fields */
 	bool has_vcr;			/* virtual machine control register */
+	bool no_block;			/* BLOCK mode not implemented */
 };
 
 #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg)
@@ -358,7 +365,8 @@ static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data,
 
 	fault->addr = readl(SYSMMU_REG(data, fault_va));
 	fault->name = sysmmu_v7_fault_names[itype % 4];
-	fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
+	fault->type = (info & FAULT_INFO_WRITE) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
+	fault->info = info;
 
 	return 0;
 }
@@ -511,6 +519,7 @@ static void __sysmmu_get_vcr(struct sysmmu_drvdata *data)
 	u32 capa1 = readl(data->sfrbase + REG_V7_CAPA1);
 
 	data->has_vcr = capa1 & CAPA1_VCR_ENABLED;
+	data->no_block = capa1 & CAPA1_NO_BLOCK_MODE;
 }
 
 static void __sysmmu_get_version(struct sysmmu_drvdata *data)
@@ -555,6 +564,12 @@ static void show_fault_information(struct sysmmu_drvdata *data,
 		dev_name(data->master),
 		fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE",
 		fault->name, fault->addr);
+	/* AxID identifies the issuing port inside the master */
+	if (data->variant->fault_info)
+		dev_err(data->sysmmu, "transaction info %#010x: AxID %#lx, AxLEN %lu\n",
+			fault->info,
+			FIELD_GET(FAULT_INFO_AXID, fault->info),
+			FIELD_GET(FAULT_INFO_AXLEN, fault->info));
 	dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable);
 	ent = section_entry(phys_to_virt(data->pgtable), fault->addr);
 	dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent);
@@ -652,7 +667,12 @@ static void __sysmmu_enable(struct sysmmu_drvdata *data)
 	__sysmmu_enable_clocks(data);
 
 	spin_lock_irqsave(&data->lock, flags);
-	writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
+	/*
+	 * On no-block hardware CTRL_BLOCK acts as a plain enable; keep the
+	 * MMU disabled until it is fully programmed.
+	 */
+	if (!data->no_block)
+		writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
 	__sysmmu_init_config(data);
 	__sysmmu_set_ptbase(data, data->pgtable);
 	__sysmmu_enable_vid(data);
@@ -677,7 +697,14 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
 	spin_lock_irqsave(&data->lock, flags);
 	if (data->active && data->version >= MAKE_MMU_VER(3, 3)) {
 		clk_enable(data->clk_master);
-		if (sysmmu_block(data)) {
+		/*
+		 * No-block hardware accepts invalidation writes while the
+		 * MMU is running; it is also v7+, so flush-all covers the
+		 * FLPD cache.
+		 */
+		if (data->no_block) {
+			__sysmmu_tlb_invalidate(data);
+		} else if (sysmmu_block(data)) {
 			if (data->version >= MAKE_MMU_VER(5, 0))
 				__sysmmu_tlb_invalidate(data);
 			else
@@ -713,7 +740,9 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
 		if (MMU_MAJ_VER(data->version) == 2)
 			num_inv = min_t(unsigned int, size / SPAGE_SIZE, 64);
 
-		if (sysmmu_block(data)) {
+		if (data->no_block) {
+			__sysmmu_tlb_invalidate_entry(data, iova, num_inv);
+		} else if (sysmmu_block(data)) {
 			__sysmmu_tlb_invalidate_entry(data, iova, num_inv);
 			sysmmu_unblock(data);
 		}
diff --git a/drivers/iommu/generic_pt/.kunitconfig b/drivers/iommu/generic_pt/.kunitconfig
index 0bb98fe581fe9c..a3da74328f12fc 100644
--- a/drivers/iommu/generic_pt/.kunitconfig
+++ b/drivers/iommu/generic_pt/.kunitconfig
@@ -7,6 +7,7 @@ CONFIG_IOMMU_PT_AMDV1=y
 CONFIG_IOMMU_PT_VTDSS=y
 CONFIG_IOMMU_PT_RISCV64=y
 CONFIG_IOMMU_PT_X86_64=y
+CONFIG_IOMMU_PT_BCM2712=y
 CONFIG_IOMMU_PT_KUNIT_TEST=y
 
 CONFIG_IOMMUFD=y
diff --git a/drivers/iommu/generic_pt/Kconfig b/drivers/iommu/generic_pt/Kconfig
index f4ed1add58b749..2a81dac960ac02 100644
--- a/drivers/iommu/generic_pt/Kconfig
+++ b/drivers/iommu/generic_pt/Kconfig
@@ -72,6 +72,15 @@ config IOMMU_PT_X86_64
 
 	  Selected automatically by an IOMMU driver that uses this format.
 
+config IOMMU_PT_BCM2712
+	tristate "IOMMU page table for Broadcom BCM2712"
+	help
+	  iommu_domain implementation for the Broadcom BCM2712 IOMMU found on
+	  Raspberry Pi 5. It supports 4K page sizes over a 2-level page table
+	  format.
+
+	  Selected automatically by an IOMMU driver that uses this format.
+
 config IOMMU_PT_KUNIT_TEST
 	tristate "IOMMU Page Table KUnit Test" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -79,6 +88,7 @@ config IOMMU_PT_KUNIT_TEST
 	depends on IOMMU_PT_RISCV64 || !IOMMU_PT_RISCV64
 	depends on IOMMU_PT_X86_64 || !IOMMU_PT_X86_64
 	depends on IOMMU_PT_VTDSS || !IOMMU_PT_VTDSS
+	depends on IOMMU_PT_BCM2712 || !IOMMU_PT_BCM2712
 	default KUNIT_ALL_TESTS
 	help
 	  Enable kunit tests for GENERIC_PT and IOMMU_PT that covers all the
diff --git a/drivers/iommu/generic_pt/fmt/Makefile b/drivers/iommu/generic_pt/fmt/Makefile
index ea024d582594ee..82ca3c36823d04 100644
--- a/drivers/iommu/generic_pt/fmt/Makefile
+++ b/drivers/iommu/generic_pt/fmt/Makefile
@@ -9,6 +9,8 @@ iommu_pt_fmt-$(CONFIG_IOMMU_PT_RISCV64) += riscv64
 
 iommu_pt_fmt-$(CONFIG_IOMMU_PT_X86_64) += x86_64
 
+iommu_pt_fmt-$(CONFIG_IOMMU_PT_BCM2712) += bcm2712
+
 IOMMU_PT_KUNIT_TEST :=
 define create_format
 obj-$(2) += iommu_$(1).o
diff --git a/drivers/iommu/generic_pt/fmt/bcm2712.h b/drivers/iommu/generic_pt/fmt/bcm2712.h
new file mode 100644
index 00000000000000..2c5b034441c14b
--- /dev/null
+++ b/drivers/iommu/generic_pt/fmt/bcm2712.h
@@ -0,0 +1,288 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2026 Daniel Drake
+ *
+ * BCM2712 IOMMU simple two level page table
+ */
+#ifndef __GENERIC_PT_FMT_BCM2712_H
+#define __GENERIC_PT_FMT_BCM2712_H
+
+#include "defs_bcm2712.h"
+#include "../pt_defs.h"
+
+#include <linux/bitfield.h>
+#include <linux/container_of.h>
+#include <linux/log2.h>
+#include <linux/sizes.h>
+
+enum {
+	/* Hardware provides a two-level page table */
+	PT_MAX_TOP_LEVEL = 1,
+
+	/* Hardware page size is strictly 4kb */
+	PT_GRANULE_LG2SZ = ilog2(SZ_4K),
+	PT_TABLEMEM_LG2SZ = PT_GRANULE_LG2SZ,
+
+	/* Table entries and leaf entries are 32 bits */
+	PT_ITEM_WORD_SIZE = sizeof(pt_bcm2712_entry_t),
+
+	/* Leaf entries encode a 25-bit PFN */
+	PT_MAX_OUTPUT_ADDRESS_LG2 = 25 + PT_GRANULE_LG2SZ,
+
+	/*
+	 * For simplicity, only manage mappings within an address space of
+	 * exactly 4GB. This is because our Level 1 directory page will be 4kb
+	 * in the smallest case, permitting 1024 entries pointing at Level 0
+	 * pages, each permitting 4mb of mapped memory.
+	 */
+	PT_MAX_VA_ADDRESS_LG2 = ilog2(SZ_4G),
+
+	/* Level 1 base address is programmed as a 32-bit PFN */
+	PT_TOP_PHYS_MASK = GENMASK_ULL(31 + PT_GRANULE_LG2SZ, PT_GRANULE_LG2SZ),
+};
+
+/* PTE bits */
+enum {
+	BCM2712PT_VALID = BIT(28),
+	BCM2712PT_WRITE = BIT(29),
+	BCM2712PT_PAGESIZE = GENMASK(31, 30),
+	BCM2712PT_PFN = GENMASK(24, 0),
+};
+
+#define common_to_bcm2712pt(common_ptr) \
+	container_of_const(common_ptr, struct pt_bcm2712, common)
+#define to_bcm2712pt(pts) common_to_bcm2712pt((pts)->range->common)
+
+static inline pt_oaddr_t bcm2712pt_table_pa(const struct pt_state *pts)
+{
+	return oalog2_mul(FIELD_GET(BCM2712PT_PFN, pts->entry),
+			  PT_GRANULE_LG2SZ);
+}
+#define pt_table_pa bcm2712pt_table_pa
+#define pt_item_oa bcm2712pt_table_pa
+
+static inline bool bcm2712pt_can_have_leaf(const struct pt_state *pts)
+{
+	return true;
+}
+#define pt_can_have_leaf bcm2712pt_can_have_leaf
+
+/* 4MB pages are installed at level 1, everything else at level 0 */
+static inline unsigned int bcm2712pt_pgsz_lg2_to_level(struct pt_common *common,
+						       unsigned int pgsize_lg2)
+{
+	return pgsize_lg2 == ilog2(SZ_4M) ? 1 : 0;
+}
+#define pt_pgsz_lg2_to_level bcm2712pt_pgsz_lg2_to_level
+
+static inline pt_vaddr_t bcm2712pt_possible_sizes(const struct pt_state *pts)
+{
+	struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common);
+
+	if (pts->level == 1)
+		return SZ_4M;
+
+	return SZ_4K |
+	       (table->bigpage_lg2 ? BIT_ULL(table->bigpage_lg2) : 0) |
+	       (table->superpage_lg2 ? BIT_ULL(table->superpage_lg2) : 0);
+}
+#define pt_possible_sizes bcm2712pt_possible_sizes
+
+static inline unsigned int
+bcm2712pt_entry_num_contig_lg2(const struct pt_state *pts)
+{
+	struct pt_bcm2712 *table = common_to_bcm2712pt(pts->range->common);
+	u32 pgsz = FIELD_GET(BCM2712PT_PAGESIZE, pts->entry);
+
+	if (pts->level != 0 || !(pts->entry & BCM2712PT_VALID))
+		return 0;
+
+	/*
+	 * Superpage/bigpage contiguous mapping (really just hinting) is handled
+	 * via PAGESIZE bits on each leaf entry.
+	 */
+	if (pgsz == 2 && table->superpage_lg2)
+		return table->superpage_lg2 - PT_GRANULE_LG2SZ;
+	else if (pgsz == 1 && table->bigpage_lg2)
+		return table->bigpage_lg2 - PT_GRANULE_LG2SZ;
+	return 0;
+}
+#define pt_entry_num_contig_lg2 bcm2712pt_entry_num_contig_lg2
+
+static inline unsigned int bcm2712pt_num_items_lg2(const struct pt_state *pts)
+{
+	return PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE);
+}
+#define pt_num_items_lg2 bcm2712pt_num_items_lg2
+
+static inline enum pt_entry_type bcm2712pt_load_entry_raw(struct pt_state *pts)
+{
+	const pt_bcm2712_entry_t *tablep =
+		pt_cur_table(pts, pt_bcm2712_entry_t);
+
+	pts->entry = READ_ONCE(tablep[pts->index]);
+	if (!(pts->entry & BCM2712PT_VALID))
+		return PT_ENTRY_EMPTY;
+
+	if (pts->level == 1) {
+		if (FIELD_GET(BCM2712PT_PAGESIZE, pts->entry) == 3)
+			return PT_ENTRY_OA;
+		return PT_ENTRY_TABLE;
+	}
+
+	return PT_ENTRY_OA;
+}
+#define pt_load_entry_raw bcm2712pt_load_entry_raw
+
+static inline void
+bcm2712pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa,
+			     unsigned int oasz_lg2,
+			     const struct pt_write_attrs *attrs)
+{
+	pt_bcm2712_entry_t *tablep = pt_cur_table(pts, pt_bcm2712_entry_t);
+	pt_bcm2712_entry_t entry;
+
+	if (!pt_check_install_leaf_args(pts, oa, oasz_lg2))
+		return;
+
+	entry = BCM2712PT_VALID | attrs->descriptor_bits |
+		FIELD_PREP(BCM2712PT_PFN, oalog2_div(oa, PT_GRANULE_LG2SZ));
+
+	if (pts->level == 1) {
+		/* Level 1 hugepage (4MB) */
+		entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 3);
+		WRITE_ONCE(tablep[pts->index], entry);
+		pts->entry = entry;
+		return;
+	}
+
+	if (oasz_lg2 == PT_GRANULE_LG2SZ) {
+		WRITE_ONCE(tablep[pts->index], entry);
+		pts->entry = entry;
+	} else {
+		struct pt_bcm2712 *table =
+			common_to_bcm2712pt(pts->range->common);
+		u32 *end;
+
+		tablep += pts->index;
+		end = tablep + log2_to_int(oasz_lg2 - PT_GRANULE_LG2SZ);
+
+		/*
+		 * Leaf entries can contain hints indicating bigpage/superpage
+		 * contiguous mappings to permit TLB optimization
+		 */
+		if (oasz_lg2 == table->superpage_lg2)
+			entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 2);
+		else if (oasz_lg2 == table->bigpage_lg2)
+			entry |= FIELD_PREP(BCM2712PT_PAGESIZE, 1);
+
+		pts->entry = entry;
+		for (; tablep != end; tablep++, entry++)
+			WRITE_ONCE(*tablep, entry);
+	}
+}
+#define pt_install_leaf_entry bcm2712pt_install_leaf_entry
+
+static inline bool bcm2712pt_install_table(struct pt_state *pts,
+					   pt_oaddr_t table_pa,
+					   const struct pt_write_attrs *attrs)
+{
+	pt_bcm2712_entry_t entry =
+		BCM2712PT_VALID |
+		FIELD_PREP(BCM2712PT_PFN,
+			   oalog2_div(table_pa, PT_GRANULE_LG2SZ));
+
+	return pt_table_install32(pts, entry);
+}
+#define pt_install_table bcm2712pt_install_table
+
+static inline void bcm2712pt_attr_from_entry(const struct pt_state *pts,
+					     struct pt_write_attrs *attrs)
+{
+	attrs->descriptor_bits = pts->entry & BCM2712PT_WRITE;
+}
+#define pt_attr_from_entry bcm2712pt_attr_from_entry
+
+static inline unsigned int bcm2712pt_max_sw_bit(struct pt_common *common)
+{
+	return 2;
+}
+#define pt_max_sw_bit bcm2712pt_max_sw_bit
+
+static inline u32 bcm2712pt_sw_bit(unsigned int bitnr)
+{
+	if (__builtin_constant_p(bitnr) && bitnr > 2)
+		BUILD_BUG();
+	if (PT_WARN_ON(bitnr > 2))
+		return 0;
+
+	/* Bits 27, 26 and 25 are believed spare */
+	return BIT(27 - bitnr);
+}
+#define pt_sw_bit bcm2712pt_sw_bit
+
+/* --- iommu */
+#include <linux/generic_pt/iommu.h>
+#include <linux/iommu.h>
+
+#define pt_iommu_table pt_iommu_bcm2712
+
+/* The common struct is in the per-format common struct */
+static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table)
+{
+	return &container_of(iommu_table, struct pt_iommu_table, iommu)
+			->bcm2712pt.common;
+}
+
+static inline struct pt_iommu *iommu_from_common(struct pt_common *common)
+{
+	return &container_of(common, struct pt_iommu_table, bcm2712pt.common)
+			->iommu;
+}
+
+static inline int bcm2712_pt_iommu_set_prot(struct pt_common *common,
+					    struct pt_write_attrs *attrs,
+					    unsigned int iommu_prot)
+{
+	attrs->descriptor_bits = 0;
+	if (iommu_prot & IOMMU_WRITE)
+		attrs->descriptor_bits |= BCM2712PT_WRITE;
+	return 0;
+}
+#define pt_iommu_set_prot bcm2712_pt_iommu_set_prot
+
+static inline int bcm2712_pt_fmt_init(struct pt_iommu_table *fmt_table,
+				      const struct pt_iommu_bcm2712_cfg *cfg)
+{
+	fmt_table->bcm2712pt.bigpage_lg2 = cfg->bigpage_lg2;
+	fmt_table->bcm2712pt.superpage_lg2 = cfg->superpage_lg2;
+
+	pt_top_set_level(&fmt_table->bcm2712pt.common, PT_MAX_TOP_LEVEL);
+	return 0;
+}
+#define pt_iommu_fmt_init bcm2712_pt_fmt_init
+
+static inline void
+bcm2712pt_iommu_fmt_hw_info(struct pt_iommu_bcm2712 *table,
+			    const struct pt_range *top_range,
+			    struct pt_iommu_bcm2712_hw_info *info)
+{
+	info->pt_base = virt_to_phys(top_range->top_table);
+	PT_WARN_ON(info->pt_base & ~PT_TOP_PHYS_MASK);
+}
+#define pt_iommu_fmt_hw_info bcm2712pt_iommu_fmt_hw_info
+
+#if defined(GENERIC_PT_KUNIT)
+static const struct pt_iommu_bcm2712_cfg bcm2712_kunit_fmt_cfgs[] = {
+	[0] = {
+		.common.hw_max_vasz_lg2 = 32,
+		.common.hw_max_oasz_lg2 = 37,
+		.bigpage_lg2 = 16,
+		.superpage_lg2 = 20,
+	},
+};
+#define kunit_fmt_cfgs bcm2712_kunit_fmt_cfgs
+enum { KUNIT_FMT_FEATURES = 0 };
+#endif
+
+#endif
diff --git a/drivers/iommu/generic_pt/fmt/defs_bcm2712.h b/drivers/iommu/generic_pt/fmt/defs_bcm2712.h
new file mode 100644
index 00000000000000..f1d983be171650
--- /dev/null
+++ b/drivers/iommu/generic_pt/fmt/defs_bcm2712.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __GENERIC_PT_FMT_DEFS_BCM2712_H
+#define __GENERIC_PT_FMT_DEFS_BCM2712_H
+
+#include <linux/generic_pt/common.h>
+#include <linux/types.h>
+
+typedef u32 pt_bcm2712_entry_t;
+typedef u64 pt_vaddr_t;
+typedef u64 pt_oaddr_t;
+
+struct bcm2712pt_write_attrs {
+	pt_bcm2712_entry_t descriptor_bits;
+	gfp_t gfp;
+};
+#define pt_write_attrs bcm2712pt_write_attrs
+
+#endif
diff --git a/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c b/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c
new file mode 100644
index 00000000000000..43fbc3cf3adcbd
--- /dev/null
+++ b/drivers/iommu/generic_pt/fmt/iommu_bcm2712.c
@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#define PT_FMT bcm2712
+#define PT_SUPPORTED_FEATURES BIT(PT_FEAT_DMA_INCOHERENT)
+
+#include <linux/generic_pt/iommu.h>
+#include "iommu_template.h"
diff --git a/drivers/iommu/generic_pt/kunit_iommu_pt.h b/drivers/iommu/generic_pt/kunit_iommu_pt.h
index ece1c9b8c55d38..9fc71db74234d6 100644
--- a/drivers/iommu/generic_pt/kunit_iommu_pt.h
+++ b/drivers/iommu/generic_pt/kunit_iommu_pt.h
@@ -403,7 +403,7 @@ static void test_pgsize_boundary(struct kunit *test)
 	    priv->smallest_pgsz != SZ_4K)
 		kunit_skip(test, "Format does not have the required range");
 
-	do_map(test, 0xfef80000, 0x208b95d000, 0xfef9ffff - 0xfef80000 + 1);
+	do_map(test, 0xfef80000, 0x8b95d000, 0xfef9ffff - 0xfef80000 + 1);
 }
 
 /* See https://lore.kernel.org/r/20250826143816.38686-1-eugkoira@amazon.com */
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index ba675b08cd2089..6377ba97a9ea6b 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -1344,6 +1344,49 @@ static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)
 	       (unsigned long long)desc->qw1);
 }
 
+static int qi_drain_remaining_descs(struct intel_iommu *iommu, int head,
+				    int wait_index)
+{
+	struct q_inval *qi = iommu->qi;
+	int shift = qi_shift(iommu);
+	struct qi_desc wait_desc = {};
+	struct qi_desc nop_desc = {};
+	cycles_t start;
+
+	nop_desc.qw0 = QI_IWD_FENCE | QI_IWD_TYPE;
+
+	wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) | QI_IWD_PRQ_DRAIN |
+			QI_IWD_STATUS_WRITE | QI_IWD_FENCE | QI_IWD_TYPE;
+	wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
+
+	while (head != wait_index) {
+		memcpy(qi->desc + (head << shift), &nop_desc, 1 << shift);
+		head = (head + 1) % QI_LENGTH;
+	}
+
+	WRITE_ONCE(qi->desc_status[wait_index], QI_IN_USE);
+	memcpy(qi->desc + (wait_index << shift), &wait_desc, 1 << shift);
+
+	/*
+	 * Order the descriptor rewrites before the writel() that restarts
+	 * the fetch engine.
+	 */
+	wmb();
+
+	writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
+
+	start = get_cycles();
+	while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) {
+		if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start)) {
+			pr_err("Timeout draining invalidation queue after IQE\n");
+			return -ETIMEDOUT;
+		}
+		cpu_relax();
+	}
+
+	return 0;
+}
+
 static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
 {
 	u32 fault;
@@ -1366,18 +1409,25 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
 	 * is cleared.
 	 */
 	if (fault & DMA_FSTS_IQE) {
+		int head_idx, ret;
+
 		head = readl(iommu->reg + DMAR_IQH_REG);
-		if ((head >> shift) == index) {
-			struct qi_desc *desc = qi->desc + head;
-
-			/*
-			 * desc->qw2 and desc->qw3 are either reserved or
-			 * used by software as private data. We won't print
-			 * out these two qw's for security consideration.
-			 */
-			memcpy(desc, qi->desc + (wait_index << shift),
-			       1 << shift);
-			writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
+		head_idx = (head >> shift) % QI_LENGTH;
+
+		/*
+		 * The faulting descriptor can be anywhere within the current
+		 * submission's range [index, wait_index]. Since the queue is
+		 * circular, this submission may wrap around QI_LENGTH
+		 * (index > wait_index in that case), so check both the
+		 * non-wrapped and wrapped cases of the range.
+		 */
+		if (index <= wait_index ?
+		    (head_idx >= index && head_idx <= wait_index) :
+		    (head_idx >= index || head_idx <= wait_index)) {
+			ret = qi_drain_remaining_descs(iommu, head_idx, wait_index);
+			if (ret)
+				return ret;
+
 			pr_info("Invalidation Queue Error (IQE) cleared\n");
 			return -EINVAL;
 		}
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 2e3b3ab216f8f5..e3ec7f3b78261d 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1452,12 +1452,103 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu)
 	}
 }
 
+/*
+ * Reserve a domain ID inherited from the previous kernel so that it is not
+ * handed out again while the copied translation structures are still live.
+ *
+ * Returns 0 when the ID is reserved, was already reserved, or cannot be
+ * re-assigned, and a negative errno for a genuine allocation failure.
+ */
+static int reserve_domain_id(struct intel_iommu *iommu, int did)
+{
+	int ret;
+
+	if (did < 0 || did >= iommu->max_domain_id)
+		return 0;
+
+	ret = ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);
+	/*
+	 * Devices sharing a domain share its ID, so the same ID is seen in
+	 * more than one context entry; -ENOSPC merely reports that it is
+	 * already reserved.  On success the allocated ID is returned, which
+	 * is not an error either.
+	 */
+	if (ret == -ENOSPC || ret >= 0)
+		return 0;
+
+	return ret;
+}
+
+/*
+ * Reserve the domain IDs used by a scalable mode context entry copied from
+ * the previous kernel.
+ */
+static int copy_pasid_table_dids(struct intel_iommu *iommu, struct context_entry *ce)
+{
+	struct pasid_dir_entry *dir;
+	unsigned long dir_size;
+	phys_addr_t dir_phys;
+	int ret = 0;
+	int i, j;
+
+	dir_phys = ce->lo & VTD_PAGE_MASK;
+	if (!dir_phys)
+		return 0;
+
+	dir_size = get_pasid_dir_size(ce);
+	dir = memremap(dir_phys, dir_size * sizeof(*dir), MEMREMAP_WB);
+	if (!dir)
+		return -ENOMEM;
+
+	for (i = 0; i < dir_size; i++) {
+		struct pasid_entry *table;
+		phys_addr_t table_phys;
+
+		if (!pasid_pde_is_present(&dir[i]))
+			continue;
+
+		/*
+		 * Do not use get_pasid_table_from_pde(); that returns a
+		 * phys_to_virt() pointer, which is not valid for memory
+		 * owned by the previous kernel.
+		 */
+		table_phys = READ_ONCE(dir[i].val) & PDE_PFN_MASK;
+		if (!table_phys)
+			continue;
+
+		/* A PASID table is one page: PASID_TBL_ENTRIES * 64 bytes. */
+		table = memremap(table_phys, PAGE_SIZE, MEMREMAP_WB);
+		if (!table) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		for (j = 0; j < PASID_TBL_ENTRIES; j++) {
+			if (!pasid_pte_is_present(&table[j]))
+				continue;
+
+			ret = reserve_domain_id(iommu, pasid_get_domain_id(&table[j]));
+			if (ret) {
+				memunmap(table);
+				goto out;
+			}
+		}
+
+		memunmap(table);
+	}
+
+out:
+	memunmap(dir);
+
+	return ret;
+}
+
 static int copy_context_table(struct intel_iommu *iommu,
 			      struct root_entry *old_re,
 			      struct context_entry **tbl,
 			      int bus, bool ext)
 {
-	int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did;
+	int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0;
 	struct context_entry *new_ce = NULL, ce;
 	struct context_entry *old_ce = NULL;
 	struct root_entry re;
@@ -1519,10 +1610,20 @@ static int copy_context_table(struct intel_iommu *iommu,
 
 		if (!context_present(&ce))
 			continue;
-
-		did = context_domain_id(&ce);
-		if (did >= 0 && did < iommu->max_domain_id)
-			ida_alloc_range(&iommu->domain_ida, did, did, GFP_KERNEL);
+		/*
+		 * The context entry only holds a domain ID in legacy mode.
+		 * In scalable mode the IDs are in the PASID table entries.
+		 */
+		if (ext)
+			ret = copy_pasid_table_dids(iommu, &ce);
+		else
+			ret = reserve_domain_id(iommu, context_domain_id(&ce));
+		if (ret) {
+			/* Not yet published through @tbl, so free it here. */
+			iommu_free_pages(new_ce);
+			new_ce = NULL;
+			goto out_unmap;
+		}
 
 		set_context_copied(iommu, bus, devfn);
 		new_ce[idx] = ce;
@@ -1591,7 +1692,7 @@ static int copy_translation_tables(struct intel_iommu *iommu)
 		if (ret) {
 			pr_err("%s: Failed to copy context table for bus %d\n",
 				iommu->name, bus);
-			continue;
+			goto err_free_ctxt_tbls;
 		}
 	}
 
@@ -1623,11 +1724,27 @@ static int copy_translation_tables(struct intel_iommu *iommu)
 	memunmap(old_rt);
 	return 0;
 
+err_free_ctxt_tbls:
+	/*
+	 * None of these tables have been linked into iommu->root_entry yet,
+	 * so they are unreachable and must be freed here.
+	 */
+	for (bus = 0; bus < ctxt_table_entries; bus++)
+		iommu_free_pages(ctxt_tbls[bus]);
+	kfree(ctxt_tbls);
 out_unmap:
 	memunmap(old_rt);
 err_free_bitmap:
 	bitmap_free(iommu->copied_tables);
 	iommu->copied_tables = NULL;
+
+	/*
+	 * Only reservations taken from the old context entries can be in the
+	 * ida at this point; no domain has been allocated on this IOMMU yet.
+	 * ida_destroy() empties it and leaves it ready for reuse.
+	 */
+	ida_destroy(&iommu->domain_ida);
+
 	return ret;
 }
 
@@ -2782,9 +2899,7 @@ static int blocking_domain_attach_dev(struct iommu_domain *domain,
 				      struct device *dev,
 				      struct iommu_domain *old)
 {
-	struct device_domain_info *info = dev_iommu_priv_get(dev);
-
-	iopf_for_domain_remove(info->domain ? &info->domain->domain : NULL, dev);
+	iopf_for_domain_remove(old, dev);
 	device_block_translation(dev);
 	return 0;
 }
@@ -2911,10 +3026,10 @@ static unsigned int compute_vasz_lg2_ss(struct intel_iommu *iommu,
 		*top_level = 4;
 		return min(57, mgaw);
 	} else if (mgaw > 39 && sagaw >= BIT(2)) {
-		*top_level = 3 + ffs(sagaw >> 3);
+		*top_level = 2 + ffs(sagaw >> 2);
 		return min(48, mgaw);
 	} else if (mgaw > 30 && sagaw >= BIT(1)) {
-		*top_level = 2 + ffs(sagaw >> 2);
+		*top_level = 1 + ffs(sagaw >> 1);
 		return min(39, mgaw);
 	}
 	return 0;
@@ -3870,6 +3985,14 @@ static int identity_domain_attach_dev(struct iommu_domain *domain,
 	if (dev_is_real_dma_subdevice(dev))
 		return 0;
 
+	if (sm_supported(iommu))
+		ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
+	else
+		ret = device_setup_pass_through(dev);
+
+	if (ret)
+		return ret;
+
 	/*
 	 * The identity domain has no iopf_handler, so no IOPF reference is
 	 * taken for it.  The reference held by the old domain must still be
@@ -3877,16 +4000,9 @@ static int identity_domain_attach_dev(struct iommu_domain *domain,
 	 * not affect the IOPF reference count.
 	 */
 	iopf_for_domain_remove(old, dev);
+	info->domain_attached = true;
 
-	if (sm_supported(iommu))
-		ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
-	else
-		ret = device_setup_pass_through(dev);
-
-	if (!ret)
-		info->domain_attached = true;
-
-	return ret;
+	return 0;
 }
 
 static int identity_domain_set_dev_pasid(struct iommu_domain *domain,
diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h
index 23dbe6c24439fe..5b3d234ae26599 100644
--- a/drivers/iommu/intel/iommu.h
+++ b/drivers/iommu/intel/iommu.h
@@ -1105,12 +1105,20 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
 					   unsigned int size_order,
 					   struct qi_desc *desc)
 {
-	unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
-
 	desc->qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
 		QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
 		QI_DEV_IOTLB_PFSID(pfsid);
 
+	/*
+	 * The widest range the descriptor can express is a full flush, encoded
+	 * by making bit 63 the least significant zero bit of ADDR, that is
+	 * @size_order == 64 - VTD_PAGE_SHIFT.  Bound @size_order there so that
+	 * a caller passing something larger cannot produce an out-of-range
+	 * shift below.
+	 */
+	if (size_order > 64 - VTD_PAGE_SHIFT)
+		size_order = 64 - VTD_PAGE_SHIFT;
+
 	/*
 	 * If S bit is 0, we only flush a single page. If S bit is set,
 	 * The least significant zero bit indicates the invalidation address
@@ -1120,7 +1128,8 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
 	 * Max Invs Pending (MIP) is set to 0 for now until we have DIT in
 	 * ECAP.
 	 */
-	if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
+	if (size_order < 64 - VTD_PAGE_SHIFT &&
+	    !IS_ALIGNED(addr, BIT_ULL(VTD_PAGE_SHIFT + size_order)))
 		pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
 				    addr, size_order);
 
@@ -1136,7 +1145,7 @@ static inline void qi_desc_dev_iotlb_pasid(u16 sid, u16 pfsid, u32 pasid,
 		desc->qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
 					VTD_PAGE_SHIFT);
 		/* Clear size_order bit to indicate size */
-		desc->qw1 &= ~mask;
+		desc->qw1 &= ~BIT_ULL(VTD_PAGE_SHIFT + size_order - 1);
 		/* Set the S bit to indicate flushing more than 1 page */
 		desc->qw1 |= QI_DEV_EIOTLB_SIZE;
 	}
diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c
index 2b979bec56cefe..df72724242bfd5 100644
--- a/drivers/iommu/intel/nested.c
+++ b/drivers/iommu/intel/nested.c
@@ -59,6 +59,8 @@ static int intel_nested_attach_dev(struct iommu_domain *domain,
 	if (ret)
 		goto disable_iopf;
 
+	iopf_for_domain_remove(old, dev);
+
 	info->domain = dmar_domain;
 	info->domain_attached = true;
 	spin_lock_irqsave(&dmar_domain->lock, flags);
@@ -93,7 +95,7 @@ static int intel_nested_cache_invalidate_user(struct iommu_domain *domain,
 {
 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
 	struct iommu_hwpt_vtd_s1_invalidate inv_entry;
-	u32 index, processed = 0;
+	u32 processed = 0;
 	int ret = 0;
 
 	if (array->type != IOMMU_HWPT_INVALIDATE_DATA_VTD_S1) {
@@ -101,31 +103,37 @@ static int intel_nested_cache_invalidate_user(struct iommu_domain *domain,
 		goto out;
 	}
 
-	for (index = 0; index < array->entry_num; index++) {
-		ret = iommu_copy_struct_from_user_array(&inv_entry, array,
-							IOMMU_HWPT_INVALIDATE_DATA_VTD_S1,
-							index, __reserved);
-		if (ret)
-			break;
-
-		if ((inv_entry.flags & ~IOMMU_VTD_INV_FLAGS_LEAF) ||
-		    inv_entry.__reserved) {
-			ret = -EOPNOTSUPP;
-			break;
-		}
-
-		if (!IS_ALIGNED(inv_entry.addr, VTD_PAGE_SIZE) ||
-		    ((inv_entry.npages == U64_MAX) && inv_entry.addr)) {
-			ret = -EINVAL;
-			break;
-		}
-
-		cache_tag_flush_range(dmar_domain, inv_entry.addr,
-				      inv_entry.addr + nrpages_to_size(inv_entry.npages) - 1,
-				      inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF);
-		processed++;
+	/*
+	 * The core re-invokes this op for the remaining requests, so handle one
+	 * request per call. A zero-length array only probes the type, validated
+	 * above.
+	 */
+	if (!array->entry_num)
+		return 0;
+
+	ret = iommu_copy_struct_from_user_array(
+		&inv_entry, array, IOMMU_HWPT_INVALIDATE_DATA_VTD_S1, 0,
+		__reserved);
+	if (ret)
+		goto out;
+
+	if ((inv_entry.flags & ~IOMMU_VTD_INV_FLAGS_LEAF) ||
+	    inv_entry.__reserved) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+
+	if (!IS_ALIGNED(inv_entry.addr, VTD_PAGE_SIZE) ||
+	    (inv_entry.npages == U64_MAX && inv_entry.addr)) {
+		ret = -EINVAL;
+		goto out;
 	}
 
+	cache_tag_flush_range(dmar_domain, inv_entry.addr,
+			      inv_entry.addr +
+				      nrpages_to_size(inv_entry.npages) - 1,
+			      inv_entry.flags & IOMMU_VTD_INV_FLAGS_LEAF);
+	processed = 1;
 out:
 	array->entry_num = processed;
 	return ret;
diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c
index 9742cb593577df..9827cb8043635c 100644
--- a/drivers/iommu/iommu-sva.c
+++ b/drivers/iommu/iommu-sva.c
@@ -11,7 +11,6 @@
 #include "iommu-priv.h"
 
 static DEFINE_MUTEX(iommu_sva_lock);
-static bool iommu_sva_present;
 static LIST_HEAD(iommu_sva_mms);
 static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
 						   struct mm_struct *mm);
@@ -137,11 +136,9 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm
 		goto out_free_domain;
 	domain->users = 1;
 
-	if (list_empty(&iommu_mm->sva_domains)) {
-		if (list_empty(&iommu_sva_mms))
-			iommu_sva_present = true;
+	if (list_empty(&iommu_mm->sva_domains))
 		list_add(&iommu_mm->mm_list_elm, &iommu_sva_mms);
-	}
+
 	list_add(&domain->next, &iommu_mm->sva_domains);
 out:
 	refcount_set(&handle->users, 1);
@@ -182,11 +179,8 @@ void iommu_sva_unbind_device(struct iommu_sva *handle)
 	iommu_detach_device_pasid(domain, dev, iommu_mm->pasid);
 	if (--domain->users == 0) {
 		list_del(&domain->next);
-		if (list_empty(&iommu_mm->sva_domains)) {
+		if (list_empty(&iommu_mm->sva_domains))
 			list_del(&iommu_mm->mm_list_elm);
-			if (list_empty(&iommu_sva_mms))
-				iommu_sva_present = false;
-		}
 
 		iommu_domain_free(domain);
 	}
@@ -334,8 +328,6 @@ void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end)
 	struct iommu_mm_data *iommu_mm;
 
 	guard(mutex)(&iommu_sva_lock);
-	if (!iommu_sva_present)
-		return;
 
 	list_for_each_entry(iommu_mm, &iommu_sva_mms, mm_list_elm)
 		mmu_notifier_arch_invalidate_secondary_tlbs(iommu_mm->mm, start, end);
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index cd1bca7ede9af6..b486b8bbd1fcdc 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -4243,6 +4243,10 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
 	if (!group)
 		return 0;
 
+	ret = iommu_deferred_attach(dev, iommu_get_domain_for_dev(dev));
+	if (ret)
+		return ret;
+
 	mutex_lock(&group->mutex);
 	/* An IDENTITY domain must pass through */
 	if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c
index ef6e119c2a752b..ae62db50885c9d 100644
--- a/drivers/iommu/iommufd/hw_pagetable.c
+++ b/drivers/iommu/iommufd/hw_pagetable.c
@@ -514,6 +514,8 @@ int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd)
 		.entry_len = cmd->entry_len,
 		.entry_num = cmd->entry_num,
 	};
+	struct iommufd_hw_pagetable *hwpt = NULL;
+	struct iommufd_viommu *viommu = NULL;
 	struct iommufd_object *pt_obj;
 	u32 done_num = 0;
 	int rc;
@@ -540,31 +542,34 @@ int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd)
 		goto out;
 	}
 	if (pt_obj->type == IOMMUFD_OBJ_HWPT_NESTED) {
-		struct iommufd_hw_pagetable *hwpt =
-			container_of(pt_obj, struct iommufd_hw_pagetable, obj);
-
+		hwpt = container_of(pt_obj, struct iommufd_hw_pagetable, obj);
 		if (!hwpt->domain->ops ||
 		    !hwpt->domain->ops->cache_invalidate_user) {
 			rc = -EOPNOTSUPP;
 			goto out_put_pt;
 		}
-		rc = hwpt->domain->ops->cache_invalidate_user(hwpt->domain,
-							      &data_array);
 	} else if (pt_obj->type == IOMMUFD_OBJ_VIOMMU) {
-		struct iommufd_viommu *viommu =
-			container_of(pt_obj, struct iommufd_viommu, obj);
-
+		viommu = container_of(pt_obj, struct iommufd_viommu, obj);
 		if (!viommu->ops || !viommu->ops->cache_invalidate) {
 			rc = -EOPNOTSUPP;
 			goto out_put_pt;
 		}
-		rc = viommu->ops->cache_invalidate(viommu, &data_array);
 	} else {
 		rc = -EINVAL;
 		goto out_put_pt;
 	}
 
-	done_num = data_array.entry_num;
+	do {
+		if (viommu)
+			rc = viommu->ops->cache_invalidate(viommu, &data_array);
+		else
+			rc = hwpt->domain->ops->cache_invalidate_user(
+				hwpt->domain, &data_array);
+
+		done_num += data_array.entry_num;
+		data_array.uptr += data_array.entry_num * data_array.entry_len;
+		data_array.entry_num = cmd->entry_num - done_num;
+	} while (!rc && done_num != cmd->entry_num);
 
 out_put_pt:
 	iommufd_put_object(ucmd->ictx, pt_obj);
diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c
index ee706f18f7e917..25387969ab7536 100644
--- a/drivers/iommu/iommufd/selftest.c
+++ b/drivers/iommu/iommufd/selftest.c
@@ -633,70 +633,63 @@ mock_viommu_alloc_domain_nested(struct iommufd_viommu *viommu, u32 flags,
 static int mock_viommu_cache_invalidate(struct iommufd_viommu *viommu,
 					struct iommu_user_data_array *array)
 {
-	struct iommu_viommu_invalidate_selftest *cmds;
-	struct iommu_viommu_invalidate_selftest *cur;
-	struct iommu_viommu_invalidate_selftest *end;
-	int rc;
+	struct iommu_viommu_invalidate_selftest cmd;
+	struct mock_dev *mdev;
+	struct device *dev;
+	u32 processed = 0;
+	int rc = 0;
+	int i;
 
-	/* A zero-length array is allowed to validate the array type */
-	if (array->entry_num == 0 &&
-	    array->type == IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST) {
-		array->entry_num = 0;
-		return 0;
+	if (array->type != IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST) {
+		rc = -EINVAL;
+		goto out;
 	}
 
-	cmds = kzalloc_objs(*cmds, array->entry_num);
-	if (!cmds)
-		return -ENOMEM;
-	cur = cmds;
-	end = cmds + array->entry_num;
+	/*
+	 * The core re-invokes this op for the remaining requests, so handle one
+	 * request per call. A zero-length array only probes the type, validated
+	 * above.
+	 */
+	if (!array->entry_num)
+		return 0;
 
-	static_assert(sizeof(*cmds) == 3 * sizeof(u32));
-	rc = iommu_copy_struct_from_full_user_array(
-		cmds, sizeof(*cmds), array,
-		IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST);
+	rc = iommu_copy_struct_from_user_array(
+		&cmd, array, IOMMU_VIOMMU_INVALIDATE_DATA_SELFTEST, 0,
+		cache_id);
 	if (rc)
 		goto out;
 
-	while (cur != end) {
-		struct mock_dev *mdev;
-		struct device *dev;
-		int i;
-
-		if (cur->flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) {
-			rc = -EOPNOTSUPP;
-			goto out;
-		}
-
-		if (cur->cache_id > MOCK_DEV_CACHE_ID_MAX) {
-			rc = -EINVAL;
-			goto out;
-		}
+	if (cmd.flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) {
+		rc = -EOPNOTSUPP;
+		goto out;
+	}
 
-		xa_lock(&viommu->vdevs);
-		dev = iommufd_viommu_find_dev(viommu,
-					      (unsigned long)cur->vdev_id);
-		if (!dev) {
-			xa_unlock(&viommu->vdevs);
-			rc = -EINVAL;
-			goto out;
-		}
-		mdev = container_of(dev, struct mock_dev, dev);
+	if (cmd.cache_id > MOCK_DEV_CACHE_ID_MAX) {
+		rc = -EINVAL;
+		goto out;
+	}
 
-		if (cur->flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) {
-			/* Invalidate all cache entries and ignore cache_id */
-			for (i = 0; i < MOCK_DEV_CACHE_NUM; i++)
-				mdev->cache[i] = 0;
-		} else {
-			mdev->cache[cur->cache_id] = 0;
-		}
+	xa_lock(&viommu->vdevs);
+	dev = iommufd_viommu_find_dev(viommu, (unsigned long)cmd.vdev_id);
+	if (!dev) {
 		xa_unlock(&viommu->vdevs);
-
-		cur++;
+		rc = -EINVAL;
+		goto out;
 	}
+	mdev = container_of(dev, struct mock_dev, dev);
+
+	if (cmd.flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) {
+		/* Invalidate all cache entries and ignore cache_id */
+		for (i = 0; i < MOCK_DEV_CACHE_NUM; i++)
+			mdev->cache[i] = 0;
+	} else {
+		mdev->cache[cmd.cache_id] = 0;
+	}
+	xa_unlock(&viommu->vdevs);
+
+	processed = 1;
 out:
-	array->entry_num = cur - cmds;
-	kfree(cmds);
+	array->entry_num = processed;
 	return rc;
 }
 
@@ -877,42 +870,46 @@ mock_domain_cache_invalidate_user(struct iommu_domain *domain,
 	struct mock_iommu_domain_nested *mock_nested = to_mock_nested(domain);
 	struct iommu_hwpt_invalidate_selftest inv;
 	u32 processed = 0;
-	int i = 0, j;
 	int rc = 0;
+	int i;
 
 	if (array->type != IOMMU_HWPT_INVALIDATE_DATA_SELFTEST) {
 		rc = -EINVAL;
 		goto out;
 	}
 
-	for ( ; i < array->entry_num; i++) {
-		rc = iommu_copy_struct_from_user_array(&inv, array,
-						       IOMMU_HWPT_INVALIDATE_DATA_SELFTEST,
-						       i, iotlb_id);
-		if (rc)
-			break;
+	/*
+	 * The core re-invokes this op for the remaining requests, so handle one
+	 * request per call. A zero-length array only probes the type, validated
+	 * above.
+	 */
+	if (!array->entry_num)
+		return 0;
 
-		if (inv.flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) {
-			rc = -EOPNOTSUPP;
-			break;
-		}
+	rc = iommu_copy_struct_from_user_array(
+		&inv, array, IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, 0, iotlb_id);
+	if (rc)
+		goto out;
 
-		if (inv.iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX) {
-			rc = -EINVAL;
-			break;
-		}
+	if (inv.flags & ~IOMMU_TEST_INVALIDATE_FLAG_ALL) {
+		rc = -EOPNOTSUPP;
+		goto out;
+	}
 
-		if (inv.flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) {
-			/* Invalidate all mock iotlb entries and ignore iotlb_id */
-			for (j = 0; j < MOCK_NESTED_DOMAIN_IOTLB_NUM; j++)
-				mock_nested->iotlb[j] = 0;
-		} else {
-			mock_nested->iotlb[inv.iotlb_id] = 0;
-		}
+	if (inv.iotlb_id > MOCK_NESTED_DOMAIN_IOTLB_ID_MAX) {
+		rc = -EINVAL;
+		goto out;
+	}
 
-		processed++;
+	if (inv.flags & IOMMU_TEST_INVALIDATE_FLAG_ALL) {
+		/* Invalidate all mock iotlb entries and ignore iotlb_id */
+		for (i = 0; i < MOCK_NESTED_DOMAIN_IOTLB_NUM; i++)
+			mock_nested->iotlb[i] = 0;
+	} else {
+		mock_nested->iotlb[inv.iotlb_id] = 0;
 	}
 
+	processed = 1;
 out:
 	array->entry_num = processed;
 	return rc;
@@ -2056,11 +2053,9 @@ static int iommufd_test_dmabuf_get(struct iommufd_ucmd *ucmd,
 	}
 
 	rc = dma_buf_fd(dmabuf, open_flags);
-	if (rc < 0) {
+	if (rc < 0)
 		dma_buf_put(dmabuf);
-		return rc;
-	}
-	return 0;
+	return rc;
 
 err_free:
 	kfree(priv->memory);
diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index fe8e6d0f8a23b2..fbdca69c436e12 100644
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -16,6 +16,7 @@
 #include <linux/acpi_rimt.h>
 #include <linux/compiler.h>
 #include <linux/crash_dump.h>
+#include <linux/dma-map-ops.h>
 #include <linux/init.h>
 #include <linux/iommu.h>
 #include <linux/iopoll.h>
@@ -1444,6 +1445,16 @@ static struct iommu_group *riscv_iommu_device_group(struct device *dev)
 	return generic_device_group(dev);
 }
 
+static bool riscv_iommu_capable(struct device *dev, enum iommu_cap cap)
+{
+	switch (cap) {
+	case IOMMU_CAP_CACHE_COHERENCY:
+		return dev_is_dma_coherent(dev);
+	default:
+		return false;
+	}
+}
+
 static int riscv_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
 {
 	return iommu_fwspec_add_ids(dev, args->args, 1);
@@ -1505,6 +1516,7 @@ static void riscv_iommu_release_device(struct device *dev)
 
 static const struct iommu_ops riscv_iommu_ops = {
 	.of_xlate = riscv_iommu_of_xlate,
+	.capable = riscv_iommu_capable,
 	.identity_domain = &riscv_iommu_identity_domain,
 	.blocked_domain = &riscv_iommu_blocking_domain,
 	.release_domain = &riscv_iommu_blocking_domain,
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 20b77fbc51ee03..a8f8c423b7957b 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -522,17 +522,19 @@ config GOLDFISH_PIC
 
 config QCOM_PDC
 	tristate "Qualcomm PDC"
-	depends on ARCH_QCOM
+	depends on ARCH_QCOM || COMPILE_TEST
 	select IRQ_DOMAIN_HIERARCHY
+	default ARCH_QCOM
 	help
 	  Power Domain Controller driver to manage and configure wakeup
 	  IRQs for Qualcomm Technologies Inc (QTI) mobile chips.
 
 config QCOM_MPM
 	tristate "Qualcomm MPM"
-	depends on ARCH_QCOM
+	depends on ARCH_QCOM || COMPILE_TEST
 	depends on MAILBOX
 	select IRQ_DOMAIN_HIERARCHY
+	default ARCH_QCOM if ARM64
 	help
 	  MSM Power Manager driver to manage and configure wakeup
 	  IRQs for Qualcomm Technologies Inc (QTI) mobile chips.
diff --git a/drivers/irqchip/irq-aclint-sswi.c b/drivers/irqchip/irq-aclint-sswi.c
index ca06efd86fa1a7..5e010fc401f7bc 100644
--- a/drivers/irqchip/irq-aclint-sswi.c
+++ b/drivers/irqchip/irq-aclint-sswi.c
@@ -138,7 +138,7 @@ static int __init aclint_sswi_probe(struct fwnode_handle *fwnode)
 	}
 
 	/* Register SSWI irq and handler */
-	virq = ipi_mux_create(BITS_PER_BYTE, aclint_sswi_ipi_send);
+	virq = ipi_mux_create(IPI_MAX, aclint_sswi_ipi_send);
 	if (virq <= 0) {
 		pr_err("unable to create muxed IPIs\n");
 		irq_dispose_mapping(sswi_ipi_virq);
@@ -152,7 +152,7 @@ static int __init aclint_sswi_probe(struct fwnode_handle *fwnode)
 			  aclint_sswi_starting_cpu,
 			  aclint_sswi_dying_cpu);
 
-	riscv_ipi_set_virq_range(virq, BITS_PER_BYTE);
+	riscv_ipi_set_virq_range(virq, IPI_MAX);
 
 	return 0;
 }
diff --git a/drivers/irqchip/irq-al-fic.c b/drivers/irqchip/irq-al-fic.c
index d10ac9b63c9924..760317db9dba70 100644
--- a/drivers/irqchip/irq-al-fic.c
+++ b/drivers/irqchip/irq-al-fic.c
@@ -180,7 +180,7 @@ static int al_fic_register(struct device_node *node,
  * @name: name of the fic
  * @parent_irq: interrupt of parent
  *
- * This API will configure the fic hardware to to work in wire mode.
+ * This API will configure the fic hardware to work in wire mode.
  * In wire mode, fic hardware is generating a wire ("wired") interrupt.
  * Interrupt can be generated based on positive edge or level - configuration is
  * to be determined based on connected hardware to this fic.
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index e9807af235373a..b6509b495d6d89 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -2248,7 +2248,8 @@ static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
 
 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
 {
-	WARN_ON(free_lpi_range(base, nr_ids));
+	if (free_lpi_range(base, nr_ids))
+		pr_err_ratelimited("ITS: failed to free LPI range %u:%u\n", base, nr_ids);
 	bitmap_free(bitmap);
 }
 
@@ -4895,6 +4896,9 @@ static bool __maybe_unused its_enable_quirk_hip09_162100801(void *data)
 }
 
 static const char * const dma_32bit_impaired_platforms[] = {
+#ifdef CONFIG_ALTERA_ERRATUM_AGILEX5_2_1_23
+	"intel,socfpga-agilex5",
+#endif
 #ifdef CONFIG_RENESAS_ERRATUM_GEN4GICITS1
 	"renesas,r8a779f0",
 	"renesas,r8a779g0",
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 6e1fa5b247fc46..19a15c2ea61b11 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -687,7 +687,7 @@ static void gic_eoimode1_eoi_irq(struct irq_data *d)
 {
 	/*
 	 * No need to deactivate an LPI, or an interrupt that
-	 * is is getting forwarded to a vcpu.
+	 * is getting forwarded to a vcpu.
 	 */
 	if (irqd_to_hwirq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
 		return;
@@ -2276,7 +2276,6 @@ static struct
 	bool single_redist;
 	int enabled_rdists;
 	u32 maint_irq;
-	int maint_irq_mode;
 	phys_addr_t vcpu_base;
 } acpi_data __initdata;
 
@@ -2454,21 +2453,19 @@ static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *hea
 {
 	struct acpi_madt_generic_interrupt *gicc =
 		(struct acpi_madt_generic_interrupt *)header;
-	int maint_irq_mode;
 	static int first_madt = true;
 
 	if (!(gicc->flags &
 	      (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
 		return 0;
 
-	maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
-		ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
+	if (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE)
+		pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n");
 
 	if (first_madt) {
 		first_madt = false;
 
 		acpi_data.maint_irq = gicc->vgic_interrupt;
-		acpi_data.maint_irq_mode = maint_irq_mode;
 		acpi_data.vcpu_base = gicc->gicv_base_address;
 
 		return 0;
@@ -2478,7 +2475,6 @@ static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *hea
 	 * The maintenance interrupt and GICV should be the same for every CPU
 	 */
 	if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
-	    (acpi_data.maint_irq_mode != maint_irq_mode) ||
 	    (acpi_data.vcpu_base != gicc->gicv_base_address))
 		return -EINVAL;
 
@@ -2511,7 +2507,7 @@ static void __init gic_acpi_setup_kvm_info(void)
 	gic_v3_kvm_info.type = GIC_V3;
 
 	irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
-				acpi_data.maint_irq_mode,
+				ACPI_LEVEL_SENSITIVE,
 				ACPI_ACTIVE_HIGH);
 	if (irq <= 0)
 		return;
diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c
index 4486645a4b6f28..abf92b1c619d68 100644
--- a/drivers/irqchip/irq-gic-v5-irs.c
+++ b/drivers/irqchip/irq-gic-v5-irs.c
@@ -21,8 +21,6 @@
  */
 #define LPI_ID_BITS_LINEAR		12
 
-#define IRS_FLAGS_NON_COHERENT		BIT(0)
-
 static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data);
 static LIST_HEAD(irs_nodes);
 
@@ -50,6 +48,11 @@ static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
 	writeq_relaxed(val, irs_data->irs_base + reg_offset);
 }
 
+struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void)
+{
+	return per_cpu(per_cpu_irs_data, 0);
+}
+
 /*
  * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
  * provides the memory barriers (through MMIO accessors)
@@ -266,24 +269,24 @@ int gicv5_irs_iste_alloc(const u32 lpi)
  * itself is not supported) again serves to make it easier to find physically
  * contiguous blocks of memory.
  */
-static unsigned int gicv5_irs_l2_sz(u32 idr2)
+unsigned int gicv5_irs_l2_sz(u32 l2sz)
 {
 	switch (PAGE_SIZE) {
 	case SZ_64K:
-		if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
+		if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(l2sz))
 			return GICV5_IRS_IST_CFGR_L2SZ_64K;
 		fallthrough;
 	case SZ_4K:
-		if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
+		if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz))
 			return GICV5_IRS_IST_CFGR_L2SZ_4K;
 		fallthrough;
 	case SZ_16K:
-		if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
+		if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(l2sz))
 			return GICV5_IRS_IST_CFGR_L2SZ_16K;
 		break;
 	}
 
-	if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
+	if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz))
 		return GICV5_IRS_IST_CFGR_L2SZ_4K;
 
 	return GICV5_IRS_IST_CFGR_L2SZ_64K;
@@ -331,7 +334,7 @@ static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
 	lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
 
 	if (two_levels)
-		l2sz = gicv5_irs_l2_sz(idr2);
+		l2sz = gicv5_irs_l2_sz(FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, idr2));
 
 	istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
 
diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c
index 5f2551cf077d78..8c0303af71e7da 100644
--- a/drivers/irqchip/irq-gic-v5.c
+++ b/drivers/irqchip/irq-gic-v5.c
@@ -1122,8 +1122,10 @@ static void gicv5_set_cpuif_idbits(void)
 #ifdef CONFIG_KVM
 static struct gic_kvm_info gic_v5_kvm_info __initdata;
 
-static void __init gic_of_setup_kvm_info(struct device_node *node)
+static int __init gic_setup_kvm_info(unsigned int maint_irq)
 {
+	struct gicv5_irs_chip_data *irs_data = gicv5_irs_get_chip_data();
+
 	/*
 	 * If we don't have native GICv5 virtualisation support, then
 	 * we also don't have FEAT_GCIE_LEGACY - the architecture
@@ -1131,25 +1133,120 @@ static void __init gic_of_setup_kvm_info(struct device_node *node)
 	 */
 	if (!gicv5_global_data.virt_capable) {
 		pr_info("GIC implementation is not virtualization capable\n");
-		return;
+		return -ENODEV;
 	}
 
+	if (WARN_ON(!irs_data))
+		return -ENODEV;
+
 	gic_v5_kvm_info.type = GIC_V5;
+	gic_v5_kvm_info.gicv5_irs.base = irs_data->irs_base;
+	gic_v5_kvm_info.gicv5_irs.non_coherent = !!(irs_data->flags & IRS_FLAGS_NON_COHERENT);
 
 	/* GIC Virtual CPU interface maintenance interrupt */
 	gic_v5_kvm_info.no_maint_irq_mask = false;
-	gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
-	if (!gic_v5_kvm_info.maint_irq) {
-		pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n");
+	gic_v5_kvm_info.maint_irq = maint_irq;
+
+	/*
+	 * We require an MI if we have legacy support, but don't, otherwise.
+	 * Given that there's an existing flag to convey that an MI isn't
+	 * needed, we (ab)use it to tell KVM that the MI isn't needed if we
+	 * don't support legacy.
+	 *
+	 * The check for ARM64_HAS_GICV5_LEGACY explicitly doesn't use
+	 * cpus_have_final_cap() here as we run too early.
+	 */
+	if (!cpus_have_cap(ARM64_HAS_GICV5_LEGACY) && !gic_v5_kvm_info.maint_irq)
+		gic_v5_kvm_info.no_maint_irq_mask = true;
+
+	vgic_set_kvm_info(&gic_v5_kvm_info);
+	return 0;
+}
+
+static void __init gic_of_setup_kvm_info(struct device_node *node)
+{
+	unsigned int maint_irq;
+
+	/* GIC Virtual CPU interface maintenance interrupt */
+	maint_irq = irq_of_parse_and_map(node, 0);
+	if (gic_setup_kvm_info(maint_irq))
+		irq_dispose_mapping(maint_irq);
+}
+
+#ifdef CONFIG_ACPI
+struct gicv5_acpi_kvm_info {
+	u32 maint_irq;
+	bool valid;
+};
+
+static struct gicv5_acpi_kvm_info acpi_v5_kvm_info __initdata;
+
+static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
+						const unsigned long end)
+{
+	struct acpi_madt_generic_interrupt *gicc =
+		(struct acpi_madt_generic_interrupt *)header;
+
+	if (!(gicc->flags &
+	      (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
+		return 0;
+
+	if (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE)
+		pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n");
+
+	if (!acpi_v5_kvm_info.valid) {
+		acpi_v5_kvm_info.valid = true;
+		acpi_v5_kvm_info.maint_irq = gicc->vgic_interrupt;
+		return 0;
+	}
+
+	if (acpi_v5_kvm_info.maint_irq != gicc->vgic_interrupt)
+		pr_warn_once(FW_BUG "MI not the same on all CPUs\n");
+
+	return 0;
+}
+
+static bool __init gic_acpi_collect_virt_info(void)
+{
+	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
+			      gic_acpi_parse_virt_madt_gicc, 0);
+
+	return acpi_v5_kvm_info.valid;
+}
+
+static void __init gic_acpi_setup_kvm_info(void)
+{
+	int irq = 0;
+
+	if (!gic_acpi_collect_virt_info()) {
+		pr_warn("Unable to get hardware information used for virtualization\n");
 		return;
 	}
 
-	vgic_set_kvm_info(&gic_v5_kvm_info);
+	if (acpi_v5_kvm_info.maint_irq) {
+		irq = acpi_register_gsi(NULL, acpi_v5_kvm_info.maint_irq,
+					ACPI_LEVEL_SENSITIVE,
+					ACPI_ACTIVE_HIGH);
+		if (irq <= 0) {
+			pr_warn("Failed to register GSI for GICv5 maintenance IRQ\n");
+			irq = 0;
+		}
+	}
+
+	if (gic_setup_kvm_info(irq) && irq)
+		acpi_unregister_gsi(acpi_v5_kvm_info.maint_irq);
 }
+#endif // CONFIG_ACPI
 #else
 static inline void __init gic_of_setup_kvm_info(struct device_node *node)
 {
 }
+
+#ifdef CONFIG_ACPI
+static inline void __init gic_acpi_setup_kvm_info(void)
+{
+}
+#endif
 #endif // CONFIG_KVM
 
 static int __init gicv5_init_common(struct fwnode_handle *parent_domain)
@@ -1168,21 +1265,18 @@ static int __init gicv5_init_common(struct fwnode_handle *parent_domain)
 	if (ret)
 		goto out_int;
 
-	ret = set_handle_irq(gicv5_handle_irq);
+	ret = gicv5_irs_enable();
 	if (ret)
 		goto out_int;
 
-	ret = gicv5_irs_enable();
-	if (ret)
-		goto out_handle;
+	if (set_handle_irq(gicv5_handle_irq))
+		panic("GICv5: unable to install root IRQ handler\n");
 
 	gicv5_smp_init();
 
 	gicv5_irs_its_probe();
 	return 0;
 
-out_handle:
-	set_handle_irq(NULL);
 out_int:
 	gicv5_cpu_disable_interrupts();
 	gicv5_free_domains();
@@ -1259,6 +1353,8 @@ static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsig
 	acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id,
 			   gic_v5_get_gsi_handle);
 
+	gic_acpi_setup_kvm_info();
+
 	return 0;
 
 out_irs:
diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c
index f6bc29f515fb79..b2926a3ddaf153 100644
--- a/drivers/irqchip/irq-gic.c
+++ b/drivers/irqchip/irq-gic.c
@@ -1527,7 +1527,6 @@ static struct
 {
 	phys_addr_t cpu_phys_base;
 	u32 maint_irq;
-	int maint_irq_mode;
 	phys_addr_t vctrl_base;
 	phys_addr_t vcpu_base;
 } acpi_data __initdata;
@@ -1553,10 +1552,11 @@ gic_acpi_parse_madt_cpu(union acpi_subtable_headers *header,
 	if (cpu_base_assigned && gic_cpu_base != acpi_data.cpu_phys_base)
 		return -EINVAL;
 
+	if (processor->flags & ACPI_MADT_VGIC_IRQ_MODE)
+		pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n");
+
 	acpi_data.cpu_phys_base = gic_cpu_base;
 	acpi_data.maint_irq = processor->vgic_interrupt;
-	acpi_data.maint_irq_mode = (processor->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
-				    ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
 	acpi_data.vctrl_base = processor->gich_base_address;
 	acpi_data.vcpu_base = processor->gicv_base_address;
 
@@ -1616,7 +1616,7 @@ static void __init gic_acpi_setup_kvm_info(void)
 	vcpu_res->end = vcpu_res->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
 
 	irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
-				acpi_data.maint_irq_mode,
+				ACPI_LEVEL_SENSITIVE,
 				ACPI_ACTIVE_HIGH);
 	if (irq <= 0)
 		return;
diff --git a/drivers/irqchip/irq-lan966x-oic.c b/drivers/irqchip/irq-lan966x-oic.c
index 8af08d0e418230..5122f3f1b353c5 100644
--- a/drivers/irqchip/irq-lan966x-oic.c
+++ b/drivers/irqchip/irq-lan966x-oic.c
@@ -220,7 +220,6 @@ static int lan966x_oic_probe(struct platform_device *pdev)
 	};
 	struct irq_domain_info d_info = {
 		.fwnode		= of_fwnode_handle(pdev->dev.of_node),
-		.domain_flags	= IRQ_DOMAIN_FLAG_DESTROY_GC,
 		.size		= LAN966X_OIC_NR_IRQ,
 		.hwirq_max	= LAN966X_OIC_NR_IRQ,
 		.ops		= &irq_generic_chip_ops,
diff --git a/drivers/irqchip/irq-mtk-cirq.c b/drivers/irqchip/irq-mtk-cirq.c
index 914d1d639fe303..d30c34ce0f5685 100644
--- a/drivers/irqchip/irq-mtk-cirq.c
+++ b/drivers/irqchip/irq-mtk-cirq.c
@@ -247,7 +247,7 @@ static int mtk_cirq_suspend(void *data)
 		writel_relaxed(mask, reg);
 	}
 
-	/* set edge_only mode, record edge-triggerd interrupts */
+	/* set edge_only mode, record edge-triggered interrupts */
 	/* enable cirq */
 	reg = mtk_cirq_reg(cirq_data, CIRQ_CONTROL);
 	value = readl_relaxed(reg);
diff --git a/drivers/irqchip/irq-pruss-intc.c b/drivers/irqchip/irq-pruss-intc.c
index 81078d56f38dc5..5a3e9e5bccbead 100644
--- a/drivers/irqchip/irq-pruss-intc.c
+++ b/drivers/irqchip/irq-pruss-intc.c
@@ -181,7 +181,7 @@ static void pruss_intc_map(struct pruss_intc *intc, unsigned long hwirq)
 	u8 ch, host, reg_idx;
 	u32 val;
 
-	mutex_lock(&intc->lock);
+	guard(mutex)(&intc->lock);
 
 	intc->event_channel[hwirq].ref_count++;
 
@@ -206,8 +206,6 @@ static void pruss_intc_map(struct pruss_intc *intc, unsigned long hwirq)
 
 	dev_dbg(dev, "mapped system_event = %lu channel = %d host = %d",
 		hwirq, ch, host);
-
-	mutex_unlock(&intc->lock);
 }
 
 /**
@@ -224,7 +222,7 @@ static void pruss_intc_unmap(struct pruss_intc *intc, unsigned long hwirq)
 	u8 ch, host, reg_idx;
 	u32 val;
 
-	mutex_lock(&intc->lock);
+	guard(mutex)(&intc->lock);
 
 	ch = intc->event_channel[hwirq].value;
 	host = intc->channel_host[ch].value;
@@ -251,8 +249,6 @@ static void pruss_intc_unmap(struct pruss_intc *intc, unsigned long hwirq)
 
 	dev_dbg(intc->dev, "unmapped system_event = %lu channel = %d host = %d\n",
 		hwirq, ch, host);
-
-	mutex_unlock(&intc->lock);
 }
 
 static void pruss_intc_init(struct pruss_intc *intc)
@@ -376,17 +372,15 @@ static int pruss_intc_validate_mapping(struct pruss_intc *intc, int event,
 				       int channel, int host)
 {
 	struct device *dev = intc->dev;
-	int ret = 0;
 
-	mutex_lock(&intc->lock);
+	guard(mutex)(&intc->lock);
 
 	/* check if sysevent already assigned */
 	if (intc->event_channel[event].ref_count > 0 &&
 	    intc->event_channel[event].value != channel) {
 		dev_err(dev, "event %d (req. ch %d) already assigned to channel %d\n",
 			event, channel, intc->event_channel[event].value);
-		ret = -EBUSY;
-		goto unlock;
+		return -EBUSY;
 	}
 
 	/* check if channel already assigned */
@@ -394,16 +388,13 @@ static int pruss_intc_validate_mapping(struct pruss_intc *intc, int event,
 	    intc->channel_host[channel].value != host) {
 		dev_err(dev, "channel %d (req. host %d) already assigned to host %d\n",
 			channel, host, intc->channel_host[channel].value);
-		ret = -EBUSY;
-		goto unlock;
+		return -EBUSY;
 	}
 
 	intc->event_channel[event].value = channel;
 	intc->channel_host[channel].value = host;
 
-unlock:
-	mutex_unlock(&intc->lock);
-	return ret;
+	return 0;
 }
 
 static int
@@ -516,24 +507,21 @@ static const char * const irq_names[MAX_NUM_HOST_IRQS] = {
 
 static int pruss_intc_probe(struct platform_device *pdev)
 {
-	const struct pruss_intc_match_data *data;
 	struct device *dev = &pdev->dev;
 	struct pruss_intc *intc;
 	struct pruss_host_irq_data *host_data;
 	int i, irq, ret;
 	u8 max_system_events, irqs_reserved = 0;
 
-	data = of_device_get_match_data(dev);
-	if (!data)
-		return -ENODEV;
-
-	max_system_events = data->num_system_events;
-
 	intc = devm_kzalloc(dev, sizeof(*intc), GFP_KERNEL);
 	if (!intc)
 		return -ENOMEM;
 
-	intc->soc_config = data;
+	intc->soc_config = of_device_get_match_data(dev);
+	if (!intc->soc_config)
+		return -ENODEV;
+	max_system_events = intc->soc_config->num_system_events;
+
 	intc->dev = dev;
 	platform_set_drvdata(pdev, intc);
 
@@ -553,7 +541,9 @@ static int pruss_intc_probe(struct platform_device *pdev)
 
 	pruss_intc_init(intc);
 
-	mutex_init(&intc->lock);
+	ret = devm_mutex_init(dev, &intc->lock);
+	if (ret)
+		return ret;
 
 	intc->domain = irq_domain_create_linear(dev_fwnode(dev), max_system_events,
 						&pruss_intc_irq_domain_ops, intc);
diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c
index 12efd241ce8800..823f5f2ecb3ddd 100644
--- a/drivers/irqchip/irq-riscv-imsic-early.c
+++ b/drivers/irqchip/irq-riscv-imsic-early.c
@@ -67,12 +67,12 @@ static int __init imsic_ipi_domain_init(void)
 		return 0;
 
 	/* Create IMSIC IPI multiplexing */
-	virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send);
+	virq = ipi_mux_create(IPI_MAX, imsic_ipi_send);
 	if (virq <= 0)
 		return virq < 0 ? virq : -ENOMEM;
 
 	/* Set vIRQ range */
-	riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI);
+	riscv_ipi_set_virq_range(virq, IPI_MAX);
 
 	/* Announce that IMSIC is providing IPIs */
 	pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID);
diff --git a/drivers/irqchip/irq-riscv-imsic-state.c b/drivers/irqchip/irq-riscv-imsic-state.c
index b8d1bbbf42f730..9505ddbd9eec95 100644
--- a/drivers/irqchip/irq-riscv-imsic-state.c
+++ b/drivers/irqchip/irq-riscv-imsic-state.c
@@ -7,6 +7,7 @@
 #define pr_fmt(fmt) "riscv-imsic: " fmt
 #include <linux/acpi.h>
 #include <linux/cpu.h>
+#include <linux/bits.h>
 #include <linux/bitmap.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
@@ -769,9 +770,9 @@ static int __init imsic_parse_fwnode(struct fwnode_handle *fwnode,
 		return -EINVAL;
 	}
 	global->base_addr = res.start;
-	global->base_addr &= ~(BIT(global->guest_index_bits +
-				   global->hart_index_bits +
-				   IMSIC_MMIO_PAGE_SHIFT) - 1);
+	global->base_addr &= ~GENMASK(global->guest_index_bits +
+				       global->hart_index_bits +
+				       IMSIC_MMIO_PAGE_SHIFT - 1, 0);
 	global->base_addr &= ~((BIT(global->group_index_bits) - 1) <<
 			       global->group_index_shift);
 
@@ -850,9 +851,9 @@ int __init imsic_setup_state(struct fwnode_handle *fwnode, void *opaque)
 		}
 
 		base_addr = mmios[i].start;
-		base_addr &= ~(BIT(global->guest_index_bits +
-				   global->hart_index_bits +
-				   IMSIC_MMIO_PAGE_SHIFT) - 1);
+		base_addr &= ~GENMASK(global->guest_index_bits +
+				      global->hart_index_bits +
+				      IMSIC_MMIO_PAGE_SHIFT - 1, 0);
 		base_addr &= ~((BIT(global->group_index_bits) - 1) <<
 			       global->group_index_shift);
 		if (base_addr != global->base_addr) {
diff --git a/drivers/irqchip/irq-riscv-imsic-state.h b/drivers/irqchip/irq-riscv-imsic-state.h
index c42ee180b3050c..878cc192ccecf0 100644
--- a/drivers/irqchip/irq-riscv-imsic-state.h
+++ b/drivers/irqchip/irq-riscv-imsic-state.h
@@ -13,7 +13,6 @@
 #include <linux/timer.h>
 
 #define IMSIC_IPI_ID				1
-#define IMSIC_NR_IPI				8
 
 struct imsic_vector {
 	/* Fixed details of the vector */
diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c
index 5b0dac1048140e..a7cddadddf40ff 100644
--- a/drivers/irqchip/irq-sifive-plic.c
+++ b/drivers/irqchip/irq-sifive-plic.c
@@ -452,7 +452,7 @@ static irq_hw_number_t cp100_get_hwirq(struct plic_handler *handler, void __iome
 		return 0;
 
 	/*
-	 * Interrupts delievered to hardware still become pending, but only
+	 * Interrupts delivered to hardware still become pending, but only
 	 * interrupts that are both pending and enabled can be claimed.
 	 * Clearing the enable bit for all interrupts but the first pending
 	 * one avoids a hardware bug that occurs during read from the claim
diff --git a/drivers/irqchip/irq-vic.c b/drivers/irqchip/irq-vic.c
index e38104c5064e6b..607e3284f70098 100644
--- a/drivers/irqchip/irq-vic.c
+++ b/drivers/irqchip/irq-vic.c
@@ -478,7 +478,7 @@ static void __init __vic_init(void __iomem *base, int parent_irq, int irq_start,
 /**
  * vic_init() - initialise a vectored interrupt controller
  * @base: iomem base address
- * @irq_start: starting interrupt number, must be muliple of 32
+ * @irq_start: starting interrupt number, must be multiple of 32
  * @vic_sources: bitmask of interrupt sources to allow
  * @resume_sources: bitmask of interrupt sources to allow for resume
  */
diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c
index ce6d80c7f17a3a..29025a212ece36 100644
--- a/drivers/irqchip/qcom-pdc.c
+++ b/drivers/irqchip/qcom-pdc.c
@@ -715,7 +715,10 @@ static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *pare
 		}
 
 		pdc->x1e_quirk = true;
+	}
 
+	if (of_device_is_compatible(node, "qcom,x1e80100-pdc") ||
+	    of_device_is_compatible(node, "qcom,x1p42100-pdc")) {
 		if (!qcom_scm_is_available())
 			return -EPROBE_DEFER;
 
diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig
index d0d3182236be4c..7ec2c8d7854261 100644
--- a/drivers/leds/Kconfig
+++ b/drivers/leds/Kconfig
@@ -1042,6 +1042,18 @@ config LEDS_ACER_A500
 	  This option enables support for the Power Button LED of
 	  Acer Iconia Tab A500.
 
+config LEDS_LTC3208
+	tristate "LED Driver for Analog Devices LTC3208"
+	depends on LEDS_CLASS && I2C
+	select REGMAP_I2C
+	help
+	  Say Y to enable the LTC3208 LED driver.
+	  This enables the LED device LTC3208, a 7-channel, 17-current source
+	  multidisplay high-current LED driver, configured via I2C.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called ltc3208.
+
 source "drivers/leds/blink/Kconfig"
 
 comment "Flash and Torch LED drivers"
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index a68244bd50fb1a..4d4b089156e24a 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -61,6 +61,7 @@ obj-$(CONFIG_LEDS_LP8788)		+= leds-lp8788.o
 obj-$(CONFIG_LEDS_LP8860)		+= leds-lp8860.o
 obj-$(CONFIG_LEDS_LP8864)		+= leds-lp8864.o
 obj-$(CONFIG_LEDS_LT3593)		+= leds-lt3593.o
+obj-$(CONFIG_LEDS_LTC3208)		+= leds-ltc3208.o
 obj-$(CONFIG_LEDS_LTC3220)		+= leds-ltc3220.o
 obj-$(CONFIG_LEDS_MAX5970)		+= leds-max5970.o
 obj-$(CONFIG_LEDS_MAX77650)		+= leds-max77650.o
diff --git a/drivers/leds/flash/leds-aat1290.c b/drivers/leds/flash/leds-aat1290.c
index 49251cfd33509e..4bde0b07fe7854 100644
--- a/drivers/leds/flash/leds-aat1290.c
+++ b/drivers/leds/flash/leds-aat1290.c
@@ -311,8 +311,9 @@ static int init_mm_current_scale(struct aat1290_led *led,
 	int i, max_mm_current =
 			AAT1290_MAX_MM_CURRENT(cfg->max_flash_current);
 
-	led->mm_current_scale = devm_kzalloc(&led->pdev->dev,
-					sizeof(max_mm_current_percent),
+	led->mm_current_scale = devm_kcalloc(&led->pdev->dev,
+					ARRAY_SIZE(max_mm_current_percent),
+					sizeof(*led->mm_current_scale),
 					GFP_KERNEL);
 	if (!led->mm_current_scale)
 		return -ENOMEM;
diff --git a/drivers/leds/flash/leds-ktd2692.c b/drivers/leds/flash/leds-ktd2692.c
index 22fbfccd487354..43d4d55503ab50 100644
--- a/drivers/leds/flash/leds-ktd2692.c
+++ b/drivers/leds/flash/leds-ktd2692.c
@@ -69,6 +69,8 @@ static const struct expresswire_timing ktd2692_timing = {
 };
 
 struct ktd2692_context {
+	struct device *dev;
+
 	/* Common ExpressWire properties (ctrl GPIO and timing) */
 	struct expresswire_common_props props;
 
@@ -196,13 +198,12 @@ static void ktd2692_setup(struct ktd2692_context *led)
 
 static void regulator_disable_action(void *_data)
 {
-	struct device *dev = _data;
-	struct ktd2692_context *led = dev_get_drvdata(dev);
+	struct ktd2692_context *led = _data;
 	int ret;
 
 	ret = regulator_disable(led->regulator);
 	if (ret)
-		dev_err(dev, "Failed to disable supply: %d\n", ret);
+		dev_err(led->dev, "Failed to disable supply: %d\n", ret);
 }
 
 static int ktd2692_parse_dt(struct ktd2692_context *led, struct device *dev,
@@ -233,7 +234,7 @@ static int ktd2692_parse_dt(struct ktd2692_context *led, struct device *dev,
 			dev_err(dev, "Failed to enable supply: %d\n", ret);
 		} else {
 			ret = devm_add_action_or_reset(dev,
-						regulator_disable_action, dev);
+						regulator_disable_action, led);
 			if (ret)
 				return ret;
 		}
@@ -290,6 +291,7 @@ static int ktd2692_probe(struct platform_device *pdev)
 	if (!led)
 		return -ENOMEM;
 
+	led->dev = &pdev->dev;
 	fled_cdev = &led->fled_cdev;
 	led_cdev = &fled_cdev->led_cdev;
 	led->props.timing = ktd2692_timing;
diff --git a/drivers/leds/led-class-flash.c b/drivers/leds/led-class-flash.c
index 165035a8826ca7..181dfd49c6ada9 100644
--- a/drivers/leds/led-class-flash.c
+++ b/drivers/leds/led-class-flash.c
@@ -187,26 +187,23 @@ static ssize_t flash_fault_show(struct device *dev,
 {
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
 	struct led_classdev_flash *fled_cdev = lcdev_to_flcdev(led_cdev);
-	u32 fault, mask = 0x1;
-	char *pbuf = buf;
-	int i, ret, buf_len;
+	u32 fault = 0, mask = 0x1;
+	int i, ret;
 
 	ret = led_get_flash_fault(fled_cdev, &fault);
 	if (ret < 0)
 		return -EINVAL;
 
-	*buf = '\0';
-
-	for (i = 0; i < LED_NUM_FLASH_FAULTS; ++i) {
-		if (fault & mask) {
-			buf_len = sprintf(pbuf, "%s ",
-					  led_flash_fault_names[i]);
-			pbuf += buf_len;
-		}
+	for (ret = 0, i = 0; i < LED_NUM_FLASH_FAULTS; ++i) {
+		if (fault & mask)
+			ret += sysfs_emit_at(buf, ret, "%s%s",
+					     (ret ? " " : ""),
+					     led_flash_fault_names[i]);
 		mask <<= 1;
 	}
 
-	return strlen(strcat(buf, "\n"));
+	ret += sysfs_emit_at(buf, ret, "\n");
+	return ret;
 }
 static DEVICE_ATTR_RO(flash_fault);
 
diff --git a/drivers/leds/led-class.c b/drivers/leds/led-class.c
index a51b0ed5388644..00be3cb96c00f2 100644
--- a/drivers/leds/led-class.c
+++ b/drivers/leds/led-class.c
@@ -27,29 +27,19 @@ static LIST_HEAD(leds_lookup_list);
 
 static struct workqueue_struct *leds_wq;
 
-static bool led_trigger_is_hw_controlled(struct led_classdev *led_cdev)
-{
-#ifdef CONFIG_LEDS_TRIGGERS
-	guard(rwsem_read)(&led_cdev->trigger_lock);
-	return led_cdev->trigger && led_cdev->trigger->trigger_type;
-#else
-	return false;
-#endif
-}
-
 static ssize_t brightness_show(struct device *dev,
 		struct device_attribute *attr, char *buf)
 {
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
 	unsigned int brightness;
 
-	if (led_trigger_is_hw_controlled(led_cdev))
-		return -ENODATA;
+	scoped_guard(mutex, &led_cdev->led_access) {
+		if (led_trigger_is_hw_controlled(led_cdev))
+			return -ENODATA;
 
-	mutex_lock(&led_cdev->led_access);
-	led_update_brightness(led_cdev);
-	brightness = led_cdev->brightness;
-	mutex_unlock(&led_cdev->led_access);
+		led_update_brightness(led_cdev);
+		brightness = led_cdev->brightness;
+	}
 
 	return sysfs_emit(buf, "%u\n", brightness);
 }
@@ -74,6 +64,9 @@ static ssize_t brightness_store(struct device *dev,
 
 	if (state == LED_OFF)
 		led_trigger_remove(led_cdev);
+	else
+		led_trigger_remove_hw_control(led_cdev);
+
 	led_set_brightness(led_cdev, state);
 
 	ret = size;
@@ -97,17 +90,6 @@ static ssize_t max_brightness_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(max_brightness);
 
-#ifdef CONFIG_LEDS_TRIGGERS
-static const BIN_ATTR(trigger, 0644, led_trigger_read, led_trigger_write, 0);
-static const struct bin_attribute *const led_trigger_bin_attrs[] = {
-	&bin_attr_trigger,
-	NULL,
-};
-static const struct attribute_group led_trigger_group = {
-	.bin_attrs = led_trigger_bin_attrs,
-};
-#endif
-
 static struct attribute *led_class_attrs[] = {
 	&dev_attr_brightness.attr,
 	&dev_attr_max_brightness.attr,
@@ -577,6 +559,11 @@ int led_classdev_register_ext(struct device *parent,
 #endif
 #ifdef CONFIG_LEDS_BRIGHTNESS_HW_CHANGED
 	led_cdev->brightness_hw_changed = -1;
+#endif
+#ifdef CONFIG_LEDS_TRIGGERS_HW_CHANGED
+	if (led_cdev->flags & LED_TRIG_HW_CHANGED)
+		INIT_WORK(&led_cdev->trigger_hw_changed_work,
+			  led_trigger_hw_control_changed_worker);
 #endif
 	if (!led_cdev->max_brightness)
 		led_cdev->max_brightness = LED_FULL;
@@ -616,6 +603,11 @@ void led_classdev_unregister(struct led_classdev *led_cdev)
 	if (IS_ERR_OR_NULL(led_cdev->dev))
 		return;
 
+#ifdef CONFIG_LEDS_TRIGGERS_HW_CHANGED
+	if (led_cdev->flags & LED_TRIG_HW_CHANGED)
+		disable_work_sync(&led_cdev->trigger_hw_changed_work);
+#endif
+
 #ifdef CONFIG_LEDS_TRIGGERS
 	down_write(&led_cdev->trigger_lock);
 	if (led_cdev->trigger)
diff --git a/drivers/leds/led-core.c b/drivers/leds/led-core.c
index 073c547068cc31..80e281dce19528 100644
--- a/drivers/leds/led-core.c
+++ b/drivers/leds/led-core.c
@@ -267,10 +267,7 @@ void led_blink_set_oneshot(struct led_classdev *led_cdev,
 	set_bit(LED_BLINK_ONESHOT, &led_cdev->work_flags);
 	clear_bit(LED_BLINK_ONESHOT_STOP, &led_cdev->work_flags);
 
-	if (invert)
-		set_bit(LED_BLINK_INVERT, &led_cdev->work_flags);
-	else
-		clear_bit(LED_BLINK_INVERT, &led_cdev->work_flags);
+	assign_bit(LED_BLINK_INVERT, &led_cdev->work_flags, invert);
 
 	led_blink_setup(led_cdev, delay_on, delay_off);
 }
diff --git a/drivers/leds/led-triggers.c b/drivers/leds/led-triggers.c
index b1223218bda113..f16603b5ff44f0 100644
--- a/drivers/leds/led-triggers.c
+++ b/drivers/leds/led-triggers.c
@@ -7,9 +7,13 @@
  * Author: Richard Purdie <rpurdie@openedhand.com>
  */
 
+#include <linux/bug.h>
+#include <linux/cleanup.h>
+#include <linux/compiler.h>
 #include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
+#include <linux/lockdep.h>
 #include <linux/spinlock.h>
 #include <linux/device.h>
 #include <linux/timer.h>
@@ -33,9 +37,38 @@ trigger_relevant(struct led_classdev *led_cdev, struct led_trigger *trig)
 	return !trig->trigger_type || trig->trigger_type == led_cdev->trigger_type;
 }
 
-ssize_t led_trigger_write(struct file *filp, struct kobject *kobj,
-			  const struct bin_attribute *bin_attr, char *buf,
-			  loff_t pos, size_t count)
+static bool __led_trigger_is_hw_controlled(struct led_classdev *led_cdev)
+{
+	lockdep_assert_held(&led_cdev->trigger_lock);
+
+	if (!led_cdev->trigger)
+		return false;
+
+	if (!led_cdev->hw_control_trigger ||
+	    strcmp(led_cdev->hw_control_trigger, led_cdev->trigger->name))
+		return false;
+
+	if (led_cdev->trigger->hw_offloaded)
+		return led_cdev->trigger->hw_offloaded(led_cdev);
+
+	dev_warn_once(led_cdev->dev,
+		      "Hardware control trigger %s doesn't provide offloaded state\n",
+		      led_cdev->trigger->name);
+
+	/* Otherwise assume private triggers are always offloaded. */
+	return led_cdev->trigger->trigger_type;
+}
+
+bool led_trigger_is_hw_controlled(struct led_classdev *led_cdev)
+{
+	guard(rwsem_read)(&led_cdev->trigger_lock);
+	return __led_trigger_is_hw_controlled(led_cdev);
+}
+EXPORT_SYMBOL_GPL(led_trigger_is_hw_controlled);
+
+static ssize_t trigger_write(struct file *filp, struct kobject *kobj,
+			     const struct bin_attribute *bin_attr, char *buf,
+			     loff_t pos, size_t count)
 {
 	struct device *dev = kobj_to_dev(kobj);
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
@@ -78,7 +111,6 @@ ssize_t led_trigger_write(struct file *filp, struct kobject *kobj,
 	mutex_unlock(&led_cdev->led_access);
 	return ret;
 }
-EXPORT_SYMBOL_GPL(led_trigger_write);
 
 __printf(3, 4)
 static int led_trigger_snprintf(char *buf, ssize_t size, const char *fmt, ...)
@@ -130,9 +162,9 @@ static int led_trigger_format(char *buf, size_t size,
  * attribute, which is not limited by length. This is _not_ good design, do not
  * copy it.
  */
-ssize_t led_trigger_read(struct file *filp, struct kobject *kobj,
-			const struct bin_attribute *attr, char *buf,
-			loff_t pos, size_t count)
+static ssize_t trigger_read(struct file *filp, struct kobject *kobj,
+			    const struct bin_attribute *attr, char *buf,
+			    loff_t pos, size_t count)
 {
 	struct device *dev = kobj_to_dev(kobj);
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
@@ -160,10 +192,53 @@ ssize_t led_trigger_read(struct file *filp, struct kobject *kobj,
 
 	return len;
 }
-EXPORT_SYMBOL_GPL(led_trigger_read);
+static const BIN_ATTR_RW(trigger, 0);
+
+static const struct bin_attribute *const led_trigger_bin_attrs[] = {
+	&bin_attr_trigger,
+	NULL
+};
+
+static ssize_t trigger_may_offload_to_hw_show(struct device *dev,
+					      const struct device_attribute *attr, char *buf)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	bool offloaded = led_trigger_is_hw_controlled(led_cdev);
+
+	return sysfs_emit(buf, "%s%s%s\n",
+			  offloaded ? "[" : "",
+			  led_cdev->hw_control_trigger,
+			  offloaded ? "]" : "");
+}
+static const DEVICE_ATTR_RO(trigger_may_offload_to_hw);
+
+static const struct attribute *const led_trigger_attrs[] = {
+	&dev_attr_trigger_may_offload_to_hw.attr,
+	NULL
+};
+
+static umode_t led_trigger_is_visible(struct kobject *kobj,
+				      const struct attribute *attr, int idx)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+
+	if (attr == &dev_attr_trigger_may_offload_to_hw.attr)
+		return led_cdev->hw_control_trigger ? attr->mode : 0;
+
+	return attr->mode;
+}
+
+const struct attribute_group led_trigger_group = {
+	.bin_attrs = led_trigger_bin_attrs,
+	.attrs_const = led_trigger_attrs,
+	.is_visible_const = led_trigger_is_visible,
+};
+EXPORT_SYMBOL_GPL(led_trigger_group);
 
 /* Caller must ensure led_cdev->trigger_lock held */
-int led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trig)
+static int __led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trig,
+			     bool hw_triggered)
 {
 	char *event = NULL;
 	char *envp[2];
@@ -194,7 +269,21 @@ int led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trig)
 		led_cdev->trigger_data = NULL;
 		led_cdev->activated = false;
 		led_cdev->flags &= ~LED_INIT_DEFAULT_TRIGGER;
-		led_set_brightness(led_cdev, LED_OFF);
+
+		/*
+		 * Hardware may have selected a new brightness level during its
+		 * hardware control transition, so only reset brightness if we
+		 * are switching to another trigger or if the switching is not
+		 * hardware triggered.
+		 *
+		 * Note that this does not apply to the error path, as running
+		 * into the error path implies a none => private trigger
+		 * transition. This hints that the LED driver and its private
+		 * trigger must have some fundamental bugs, so the error path
+		 * always turns off the LED to reset it to a certain state.
+		 */
+		if (trig || !hw_triggered)
+			led_set_brightness(led_cdev, LED_OFF);
 	}
 	if (trig) {
 		spin_lock(&trig->leddev_list_lock);
@@ -258,6 +347,11 @@ int led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trig)
 
 	return ret;
 }
+
+int led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trig)
+{
+	return __led_trigger_set(led_cdev, trig, false);
+}
 EXPORT_SYMBOL_GPL(led_trigger_set);
 
 void led_trigger_remove(struct led_classdev *led_cdev)
@@ -268,6 +362,15 @@ void led_trigger_remove(struct led_classdev *led_cdev)
 }
 EXPORT_SYMBOL_GPL(led_trigger_remove);
 
+void led_trigger_remove_hw_control(struct led_classdev *led_cdev)
+{
+	guard(rwsem_write)(&led_cdev->trigger_lock);
+
+	if (__led_trigger_is_hw_controlled(led_cdev))
+		led_trigger_set(led_cdev, NULL);
+}
+EXPORT_SYMBOL_GPL(led_trigger_remove_hw_control);
+
 static bool led_match_default_trigger(struct led_classdev *led_cdev,
 				      struct led_trigger *trig)
 {
@@ -403,6 +506,60 @@ int devm_led_trigger_register(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_led_trigger_register);
 
+#ifdef CONFIG_LEDS_TRIGGERS_HW_CHANGED
+
+static void led_trigger_do_hw_control_transition(struct led_classdev *led_cdev, bool activate,
+						 struct led_trigger *hc_trig)
+{
+	if (activate && !led_cdev->trigger) /* "none" => private trigger. */
+		__led_trigger_set(led_cdev, hc_trig, true);
+	else if (!activate && led_cdev->trigger == hc_trig) /* private trigger => "none". */
+		__led_trigger_set(led_cdev, NULL, true);
+
+	/* Already in the desired state, or another trigger is active, ignore. */
+}
+
+void led_trigger_hw_control_changed_worker(struct work_struct *work)
+{
+	struct led_classdev *led_cdev =
+		container_of(work, struct led_classdev, trigger_hw_changed_work);
+	bool activate = READ_ONCE(led_cdev->trigger_hw_changed);
+
+	scoped_guard(rwsem_read, &triggers_list_lock) {
+		struct led_trigger *trig;
+
+		list_for_each_entry(trig, &trigger_list, next_trig) {
+			if (trig->trigger_type == led_cdev->trigger_type &&
+			    !strcmp(trig->name, led_cdev->hw_control_trigger)) {
+				guard(rwsem_write)(&led_cdev->trigger_lock);
+
+				led_trigger_do_hw_control_transition(led_cdev, activate, trig);
+				return;
+			}
+		}
+	}
+
+	dev_warn(led_cdev->dev,
+		 "Private trigger %s is not registered, can't toggle hardware control\n",
+		 led_cdev->hw_control_trigger);
+}
+EXPORT_SYMBOL_GPL(led_trigger_hw_control_changed_worker);
+
+void led_trigger_notify_hw_control_changed(struct led_classdev *led_cdev, bool activate)
+{
+	/* Restricted to private triggers. */
+	if (WARN_ON(!(led_cdev->flags & LED_TRIG_HW_CHANGED) ||
+		    !led_cdev->hw_control_trigger || !led_cdev->trigger_type))
+		return;
+
+	WRITE_ONCE(led_cdev->trigger_hw_changed, activate);
+
+	schedule_work(&led_cdev->trigger_hw_changed_work);
+}
+EXPORT_SYMBOL_GPL(led_trigger_notify_hw_control_changed);
+
+#endif /* CONFIG_LEDS_TRIGGERS_HW_CHANGED */
+
 /* Simple LED Trigger Interface */
 
 void led_trigger_event(struct led_trigger *trig,
diff --git a/drivers/leds/leds-cros_ec.c b/drivers/leds/leds-cros_ec.c
index 1844d0cd5f5284..0e93aa28a8b16d 100644
--- a/drivers/leds/leds-cros_ec.c
+++ b/drivers/leds/leds-cros_ec.c
@@ -85,12 +85,18 @@ static int cros_ec_led_trigger_activate(struct led_classdev *led_cdev)
 	return cros_ec_led_send_cmd(priv->cros_ec, &arg);
 }
 
+static bool cros_ec_led_trigger_hw_offloaded(struct led_classdev *led_cdev)
+{
+	return true;
+}
+
 static struct led_hw_trigger_type cros_ec_led_trigger_type;
 
 static struct led_trigger cros_ec_led_trigger = {
 	.name = "chromeos-auto",
 	.trigger_type = &cros_ec_led_trigger_type,
 	.activate = cros_ec_led_trigger_activate,
+	.hw_offloaded = cros_ec_led_trigger_hw_offloaded,
 };
 
 static int cros_ec_led_brightness_set_blocking(struct led_classdev *led_cdev,
diff --git a/drivers/leds/leds-is31fl32xx.c b/drivers/leds/leds-is31fl32xx.c
index 6c8d6b83326004..285085d274538d 100644
--- a/drivers/leds/leds-is31fl32xx.c
+++ b/drivers/leds/leds-is31fl32xx.c
@@ -378,23 +378,23 @@ static int is31fl32xx_init_regs(struct is31fl32xx_priv *priv)
 }
 
 static int is31fl32xx_parse_child_dt(const struct device *dev,
-				     const struct device_node *child,
+				     const struct fwnode_handle *child,
 				     struct is31fl32xx_led_data *led_data)
 {
 	struct led_classdev *cdev = &led_data->cdev;
 	int ret = 0;
 	u32 reg;
 
-	ret = of_property_read_u32(child, "reg", &reg);
+	ret = fwnode_property_read_u32(child, "reg", &reg);
 	if (ret || reg < 1 || reg > led_data->priv->cdef->channels) {
 		dev_err(dev,
-			"Child node %pOF does not have a valid reg property\n",
+			"Child node %pfwP does not have a valid reg property\n",
 			child);
 		return -EINVAL;
 	}
 	led_data->channel = reg;
 
-	of_property_read_u32(child, "led-max-microamp", &led_data->max_microamp);
+	fwnode_property_read_u32(child, "led-max-microamp", &led_data->max_microamp);
 
 	cdev->brightness_set_blocking = is31fl32xx_brightness_set;
 
@@ -422,7 +422,7 @@ static int is31fl32xx_parse_dt(struct device *dev,
 	int ret = 0;
 
 	if ((cdef->output_frequency_setting_reg != IS31FL32XX_REG_NONE) &&
-	    of_property_read_bool(dev_of_node(dev), "issi,22khz-pwm")) {
+	    device_property_read_bool(dev, "issi,22khz-pwm")) {
 
 		ret = is31fl32xx_write(priv, cdef->output_frequency_setting_reg,
 				       IS31FL32XX_PWM_FREQUENCY_22KHZ);
@@ -433,7 +433,7 @@ static int is31fl32xx_parse_dt(struct device *dev,
 		}
 	}
 
-	for_each_available_child_of_node_scoped(dev_of_node(dev), child) {
+	device_for_each_child_node_scoped(dev, child) {
 		struct led_init_data init_data = {};
 		struct is31fl32xx_led_data *led_data =
 			&priv->leds[priv->num_leds];
@@ -451,17 +451,17 @@ static int is31fl32xx_parse_dt(struct device *dev,
 							  led_data->channel);
 		if (other_led_data) {
 			dev_err(dev,
-				"Node %pOF 'reg' conflicts with another LED\n",
+				"Node %pfwP 'reg' conflicts with another LED\n",
 				child);
 			return -EINVAL;
 		}
 
-		init_data.fwnode = of_fwnode_handle(child);
+		init_data.fwnode = child;
 
 		ret = devm_led_classdev_register_ext(dev, &led_data->cdev,
 						     &init_data);
 		if (ret) {
-			dev_err(dev, "Failed to register LED for %pOF: %d\n",
+			dev_err(dev, "Failed to register LED for %pfwP: %d\n",
 				child, ret);
 			return ret;
 		}
@@ -576,7 +576,7 @@ static int is31fl32xx_probe(struct i2c_client *client)
 
 	cdef = device_get_match_data(dev);
 
-	count = of_get_available_child_count(dev_of_node(dev));
+	count = device_get_child_node_count(dev);
 	if (!count)
 		return -EINVAL;
 
diff --git a/drivers/leds/leds-lm3533.c b/drivers/leds/leds-lm3533.c
index 45795f2a10426a..7259097bd6a71a 100644
--- a/drivers/leds/leds-lm3533.c
+++ b/drivers/leds/leds-lm3533.c
@@ -12,6 +12,8 @@
 #include <linux/mfd/core.h>
 #include <linux/mutex.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
 #include <linux/slab.h>
 
 #include <linux/mfd/lm3533.h>
@@ -41,13 +43,18 @@
 
 
 struct lm3533_led {
-	struct lm3533 *lm3533;
+	struct regmap *regmap;
 	struct lm3533_ctrlbank cb;
 	struct led_classdev cdev;
 	int id;
 
 	struct mutex mutex;
 	unsigned long flags;
+
+	u32 max_current;
+	u32 pwm;
+
+	bool have_als;
 };
 
 
@@ -80,7 +87,6 @@ static inline u8 lm3533_led_get_pattern_reg(struct lm3533_led *led,
 static int lm3533_led_pattern_enable(struct lm3533_led *led, int enable)
 {
 	u8 mask;
-	u8 val;
 	int pattern;
 	int state;
 	int ret = 0;
@@ -96,12 +102,8 @@ static int lm3533_led_pattern_enable(struct lm3533_led *led, int enable)
 	pattern = lm3533_led_get_pattern(led);
 	mask = 1 << (2 * pattern);
 
-	if (enable)
-		val = mask;
-	else
-		val = 0;
-
-	ret = lm3533_update(led->lm3533, LM3533_REG_PATTERN_ENABLE, val, mask);
+	ret = regmap_assign_bits(led->regmap,
+				 LM3533_REG_PATTERN_ENABLE, mask, enable);
 	if (ret) {
 		dev_err(led->cdev.dev, "failed to enable pattern %d (%d)\n",
 							pattern, enable);
@@ -131,7 +133,7 @@ static int lm3533_led_set(struct led_classdev *cdev,
 static enum led_brightness lm3533_led_get(struct led_classdev *cdev)
 {
 	struct lm3533_led *led = to_lm3533_led(cdev);
-	u8 val;
+	u32 val;
 	int ret;
 
 	ret = lm3533_ctrlbank_get_brightness(&led->cb, &val);
@@ -259,7 +261,7 @@ static u8 lm3533_led_delay_set(struct lm3533_led *led, u8 base,
 	dev_dbg(led->cdev.dev, "%s - %lu: %u (0x%02x)\n", __func__,
 							*delay, t, val);
 	reg = lm3533_led_get_pattern_reg(led, base);
-	ret = lm3533_write(led->lm3533, reg, val);
+	ret = regmap_write(led->regmap, reg, val);
 	if (ret)
 		dev_err(led->cdev.dev, "failed to set delay (%02x)\n", reg);
 
@@ -337,10 +339,10 @@ static ssize_t show_risefalltime(struct device *dev,
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
 	ssize_t ret;
 	u8 reg;
-	u8 val;
+	u32 val;
 
 	reg = lm3533_led_get_pattern_reg(led, base);
-	ret = lm3533_read(led->lm3533, reg, &val);
+	ret = regmap_read(led->regmap, reg, &val);
 	if (ret)
 		return ret;
 
@@ -375,7 +377,7 @@ static ssize_t store_risefalltime(struct device *dev,
 		return -EINVAL;
 
 	reg = lm3533_led_get_pattern_reg(led, base);
-	ret = lm3533_write(led->lm3533, reg, val);
+	ret = regmap_write(led->regmap, reg, val);
 	if (ret)
 		return ret;
 
@@ -405,11 +407,11 @@ static ssize_t show_als_channel(struct device *dev,
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
 	unsigned channel;
 	u8 reg;
-	u8 val;
+	u32 val;
 	int ret;
 
 	reg = lm3533_led_get_lv_reg(led, LM3533_REG_CTRLBANK_BCONF_BASE);
-	ret = lm3533_read(led->lm3533, reg, &val);
+	ret = regmap_read(led->regmap, reg, &val);
 	if (ret)
 		return ret;
 
@@ -441,7 +443,7 @@ static ssize_t store_als_channel(struct device *dev,
 	mask = LM3533_REG_CTRLBANK_BCONF_ALS_CHANNEL_MASK;
 	val = channel - 1;
 
-	ret = lm3533_update(led->lm3533, reg, val, mask);
+	ret = regmap_update_bits(led->regmap, reg, mask, val);
 	if (ret)
 		return ret;
 
@@ -455,11 +457,11 @@ static ssize_t show_als_en(struct device *dev,
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
 	bool enable;
 	u8 reg;
-	u8 val;
+	u32 val;
 	int ret;
 
 	reg = lm3533_led_get_lv_reg(led, LM3533_REG_CTRLBANK_BCONF_BASE);
-	ret = lm3533_read(led->lm3533, reg, &val);
+	ret = regmap_read(led->regmap, reg, &val);
 	if (ret)
 		return ret;
 
@@ -476,22 +478,15 @@ static ssize_t store_als_en(struct device *dev,
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
 	unsigned enable;
 	u8 reg;
-	u8 mask;
-	u8 val;
 	int ret;
 
 	if (kstrtouint(buf, 0, &enable))
 		return -EINVAL;
 
 	reg = lm3533_led_get_lv_reg(led, LM3533_REG_CTRLBANK_BCONF_BASE);
-	mask = LM3533_REG_CTRLBANK_BCONF_ALS_EN_MASK;
 
-	if (enable)
-		val = mask;
-	else
-		val = 0;
-
-	ret = lm3533_update(led->lm3533, reg, val, mask);
+	ret = regmap_assign_bits(led->regmap, reg,
+				 LM3533_REG_CTRLBANK_BCONF_ALS_EN_MASK, enable);
 	if (ret)
 		return ret;
 
@@ -504,12 +499,12 @@ static ssize_t show_linear(struct device *dev,
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
 	u8 reg;
-	u8 val;
+	u32 val;
 	int linear;
 	int ret;
 
 	reg = lm3533_led_get_lv_reg(led, LM3533_REG_CTRLBANK_BCONF_BASE);
-	ret = lm3533_read(led->lm3533, reg, &val);
+	ret = regmap_read(led->regmap, reg, &val);
 	if (ret)
 		return ret;
 
@@ -529,22 +524,15 @@ static ssize_t store_linear(struct device *dev,
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
 	unsigned long linear;
 	u8 reg;
-	u8 mask;
-	u8 val;
 	int ret;
 
 	if (kstrtoul(buf, 0, &linear))
 		return -EINVAL;
 
 	reg = lm3533_led_get_lv_reg(led, LM3533_REG_CTRLBANK_BCONF_BASE);
-	mask = LM3533_REG_CTRLBANK_BCONF_MAPPING_MASK;
-
-	if (linear)
-		val = mask;
-	else
-		val = 0;
 
-	ret = lm3533_update(led->lm3533, reg, val, mask);
+	ret = regmap_assign_bits(led->regmap, reg,
+				 LM3533_REG_CTRLBANK_BCONF_MAPPING_MASK, linear);
 	if (ret)
 		return ret;
 
@@ -557,7 +545,7 @@ static ssize_t show_pwm(struct device *dev,
 {
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
-	u8 val;
+	u32 val;
 	int ret;
 
 	ret = lm3533_ctrlbank_get_pwm(&led->cb, &val);
@@ -573,10 +561,10 @@ static ssize_t store_pwm(struct device *dev,
 {
 	struct led_classdev *led_cdev = dev_get_drvdata(dev);
 	struct lm3533_led *led = to_lm3533_led(led_cdev);
-	u8 val;
+	u32 val;
 	int ret;
 
-	if (kstrtou8(buf, 0, &val))
+	if (kstrtou32(buf, 0, &val))
 		return -EINVAL;
 
 	ret = lm3533_ctrlbank_set_pwm(&led->cb, val);
@@ -615,7 +603,7 @@ static umode_t lm3533_led_attr_is_visible(struct kobject *kobj,
 
 	if (attr == &dev_attr_als_channel.attr ||
 					attr == &dev_attr_als_en.attr) {
-		if (!led->lm3533->have_als)
+		if (!led->have_als)
 			mode = 0;
 	}
 
@@ -632,22 +620,20 @@ static const struct attribute_group *lm3533_led_attribute_groups[] = {
 	NULL
 };
 
-static int lm3533_led_setup(struct lm3533_led *led,
-					struct lm3533_led_platform_data *pdata)
+static int lm3533_led_setup(struct lm3533_led *led)
 {
 	int ret;
 
-	ret = lm3533_ctrlbank_set_max_current(&led->cb, pdata->max_current);
+	ret = lm3533_ctrlbank_set_max_current(&led->cb, led->max_current);
 	if (ret)
 		return ret;
 
-	return lm3533_ctrlbank_set_pwm(&led->cb, pdata->pwm);
+	return lm3533_ctrlbank_set_pwm(&led->cb, led->pwm);
 }
 
 static int lm3533_led_probe(struct platform_device *pdev)
 {
 	struct lm3533 *lm3533;
-	struct lm3533_led_platform_data *pdata;
 	struct lm3533_led *led;
 	int ret;
 
@@ -657,12 +643,6 @@ static int lm3533_led_probe(struct platform_device *pdev)
 	if (!lm3533)
 		return -EINVAL;
 
-	pdata = dev_get_platdata(&pdev->dev);
-	if (!pdata) {
-		dev_err(&pdev->dev, "no platform data\n");
-		return -EINVAL;
-	}
-
 	if (pdev->id < 0 || pdev->id >= LM3533_LVCTRLBANK_COUNT) {
 		dev_err(&pdev->dev, "illegal LED id %d\n", pdev->id);
 		return -EINVAL;
@@ -672,9 +652,9 @@ static int lm3533_led_probe(struct platform_device *pdev)
 	if (!led)
 		return -ENOMEM;
 
-	led->lm3533 = lm3533;
-	led->cdev.name = pdata->name;
-	led->cdev.default_trigger = pdata->default_trigger;
+	led->regmap = lm3533->regmap;
+	led->have_als = lm3533->have_als;
+
 	led->cdev.brightness_set_blocking = lm3533_led_set;
 	led->cdev.brightness_get = lm3533_led_get;
 	led->cdev.blink_set = lm3533_led_blink_set;
@@ -682,19 +662,28 @@ static int lm3533_led_probe(struct platform_device *pdev)
 	led->cdev.groups = lm3533_led_attribute_groups;
 	led->id = pdev->id;
 
+	led->cdev.name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-%d",
+					pdev->name, led->id);
+	if (!led->cdev.name)
+		return -ENOMEM;
+
+	led->cdev.default_trigger = "none";
+	device_property_read_string(&pdev->dev, "linux,default-trigger",
+				    &led->cdev.default_trigger);
+
 	mutex_init(&led->mutex);
 
 	/* The class framework makes a callback to get brightness during
 	 * registration so use parent device (for error reporting) until
 	 * registered.
 	 */
-	led->cb.lm3533 = lm3533;
+	led->cb.regmap = lm3533->regmap;
 	led->cb.id = lm3533_led_get_ctrlbank_id(led);
 	led->cb.dev = lm3533->dev;
 
 	platform_set_drvdata(pdev, led);
 
-	ret = led_classdev_register(pdev->dev.parent, &led->cdev);
+	ret = led_classdev_register(&pdev->dev, &led->cdev);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register LED %d\n", pdev->id);
 		return ret;
@@ -702,7 +691,14 @@ static int lm3533_led_probe(struct platform_device *pdev)
 
 	led->cb.dev = led->cdev.dev;
 
-	ret = lm3533_led_setup(led, pdata);
+	device_property_read_u32(&pdev->dev, "led-max-microamp",
+				 &led->max_current);
+	led->max_current = clamp(led->max_current, LM3533_MAX_CURRENT_MIN,
+				 LM3533_MAX_CURRENT_MAX);
+
+	device_property_read_u32(&pdev->dev, "ti,pwm-config-mask", &led->pwm);
+
+	ret = lm3533_led_setup(led);
 	if (ret)
 		goto err_deregister;
 
@@ -739,9 +735,16 @@ static void lm3533_led_shutdown(struct platform_device *pdev)
 	lm3533_led_set(&led->cdev, LED_OFF);		/* disable blink */
 }
 
+static const struct of_device_id lm3533_led_match_table[] = {
+	{ .compatible = "ti,lm3533-leds" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lm3533_led_match_table);
+
 static struct platform_driver lm3533_led_driver = {
 	.driver = {
 		.name = "lm3533-leds",
+		.of_match_table = lm3533_led_match_table,
 	},
 	.probe		= lm3533_led_probe,
 	.remove		= lm3533_led_remove,
diff --git a/drivers/leds/leds-lm3692x.c b/drivers/leds/leds-lm3692x.c
index 8d2678dc9e4fdf..9c58d54c57830a 100644
--- a/drivers/leds/leds-lm3692x.c
+++ b/drivers/leds/leds-lm3692x.c
@@ -276,9 +276,11 @@ static int lm3692x_leds_enable(struct lm3692x_led *led)
 
 	ret = regmap_update_bits(led->regmap, LM3692X_EN, LM3692X_ENABLE_MASK,
 				 enable_state | LM3692X_DEVICE_EN);
+	if (ret)
+		goto out;
 
 	led->enabled = true;
-	return ret;
+	return 0;
 out:
 	dev_err(&led->client->dev, "Fail writing initialization values\n");
 
diff --git a/drivers/leds/leds-lp8860.c b/drivers/leds/leds-lp8860.c
index 69f064781f699f..f6e4227de903a6 100644
--- a/drivers/leds/leds-lp8860.c
+++ b/drivers/leds/leds-lp8860.c
@@ -7,6 +7,7 @@
  * Author: Dan Murphy <dmurphy@ti.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/leds.h>
@@ -274,7 +275,6 @@ static int lp8860_probe(struct i2c_client *client)
 	int ret;
 	struct lp8860_led *led;
 	struct device_node *np = dev_of_node(&client->dev);
-	struct device_node *child_node;
 	struct led_init_data init_data = {};
 	struct gpio_desc *enable_gpio;
 
@@ -282,7 +282,8 @@ static int lp8860_probe(struct i2c_client *client)
 	if (!led)
 		return -ENOMEM;
 
-	child_node = of_get_next_available_child(np, NULL);
+	struct device_node *child_node __free(device_node) =
+		of_get_next_available_child(np, NULL);
 	if (!child_node)
 		return -EINVAL;
 
diff --git a/drivers/leds/leds-ltc3208.c b/drivers/leds/leds-ltc3208.c
new file mode 100644
index 00000000000000..64091f29f18a05
--- /dev/null
+++ b/drivers/leds/leds-ltc3208.c
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * LED driver for Analog Devices LTC3208 Multi-Display Driver
+ *
+ * Copyright 2026 Analog Devices Inc.
+ *
+ * Author: Jan Carlo Roleda <jancarlo.roleda@analog.com>
+ *
+ */
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+/* Registers */
+#define LTC3208_REG_A_GRNRED_CURRENT	0x1	/* Green and Red current DAC */
+#define LTC3208_REG_B_AUXBLU_CURRENT	0x2	/* AUX and Blue current DAC */
+#define LTC3208_REG_C_MAIN_CURRENT	0x3	/* Main current DAC */
+#define LTC3208_REG_D_SUB_CURRENT	0x4	/* Sub current DAC */
+#define LTC3208_REG_E_AUX_SELECT	0x5	/* AUX DAC Select */
+#define  LTC3208_AUX1_MASK		GENMASK(1, 0)
+#define  LTC3208_AUX2_MASK		GENMASK(3, 2)
+#define  LTC3208_AUX3_MASK		GENMASK(5, 4)
+#define  LTC3208_AUX4_MASK		GENMASK(7, 6)
+#define LTC3208_REG_F_CAM_CURRENT	0x6	/* CAM (High and Low) current DAC */
+#define LTC3208_REG_G_OPT		0x7	/* Device Options */
+#define  LTC3208_OPT_CPO_MASK		GENMASK(7, 6)
+#define  LTC3208_OPT_DIS_RGBDROP	BIT(3)
+#define  LTC3208_OPT_DIS_CAMHILO	BIT(2)
+#define  LTC3208_OPT_EN_RGBS		BIT(1)
+
+#define LTC3208_MAX_BRIGHTNESS_4BIT	0xF
+#define LTC3208_MAX_BRIGHTNESS_8BIT	0xFF
+
+#define LTC3208_NUM_AUX_LEDS		4	/* Number of configurable aux channels */
+#define LTC3208_NUM_REGS		7	/* Number of available registers to access */
+
+enum ltc3208_aux_channel {
+	LTC3208_AUX_CHAN_AUX = 0,
+	LTC3208_AUX_CHAN_MAIN,
+	LTC3208_AUX_CHAN_SUB,
+	LTC3208_AUX_CHAN_CAM
+};
+
+enum ltc3208_channel {
+	LTC3208_CHAN_MAIN = 0,
+	LTC3208_CHAN_SUB,
+	LTC3208_CHAN_AUX,
+	LTC3208_CHAN_CAML,
+	LTC3208_CHAN_CAMH,
+	LTC3208_CHAN_RED,
+	LTC3208_CHAN_BLUE,
+	LTC3208_CHAN_GREEN,
+	LTC3208_CHAN_N_COUNT,
+};
+
+static const char *const ltc3208_dt_aux_channels[] = {
+	"adi,aux1-channel",
+	"adi,aux2-channel",
+	"adi,aux3-channel",
+	"adi,aux4-channel"
+};
+
+static const char *const ltc3208_aux_opt[] = { "aux", "main", "sub", "cam" };
+
+struct ltc3208_led {
+	struct led_classdev cdev;
+	struct regmap_field *rfield;
+};
+
+static const struct reg_default ltc3208_reg_defaults[] = {
+	{ LTC3208_REG_A_GRNRED_CURRENT, 0 },
+	{ LTC3208_REG_B_AUXBLU_CURRENT, 0 },
+	{ LTC3208_REG_C_MAIN_CURRENT, 0 },
+	{ LTC3208_REG_D_SUB_CURRENT, 0 },
+	{ LTC3208_REG_E_AUX_SELECT, 0 },
+	{ LTC3208_REG_F_CAM_CURRENT, 0 },
+	{ LTC3208_REG_G_OPT, 0 }
+};
+
+static const struct regmap_config ltc3208_regmap_cfg = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = LTC3208_REG_G_OPT,
+	.cache_type = REGCACHE_FLAT_S,
+	.reg_defaults = ltc3208_reg_defaults,
+	.num_reg_defaults = LTC3208_NUM_REGS,
+};
+
+static const struct reg_field ltc3208_led_reg_field[LTC3208_CHAN_N_COUNT] = {
+	[LTC3208_CHAN_MAIN]  = REG_FIELD(LTC3208_REG_C_MAIN_CURRENT, 0, 7),
+	[LTC3208_CHAN_SUB]   = REG_FIELD(LTC3208_REG_D_SUB_CURRENT, 0, 7),
+	[LTC3208_CHAN_BLUE]  = REG_FIELD(LTC3208_REG_B_AUXBLU_CURRENT, 0, 3),
+	[LTC3208_CHAN_AUX]   = REG_FIELD(LTC3208_REG_B_AUXBLU_CURRENT, 4, 7),
+	[LTC3208_CHAN_CAML]  = REG_FIELD(LTC3208_REG_F_CAM_CURRENT, 0, 3),
+	[LTC3208_CHAN_CAMH]  = REG_FIELD(LTC3208_REG_F_CAM_CURRENT, 4, 7),
+	[LTC3208_CHAN_RED]   = REG_FIELD(LTC3208_REG_A_GRNRED_CURRENT, 0, 3),
+	[LTC3208_CHAN_GREEN] = REG_FIELD(LTC3208_REG_A_GRNRED_CURRENT, 4, 7),
+};
+
+static int ltc3208_led_set_brightness(struct led_classdev *led_cdev, enum led_brightness brightness)
+{
+	struct ltc3208_led *led = container_of(led_cdev, struct ltc3208_led, cdev);
+	u8 current_level = brightness;
+
+	return regmap_field_write(led->rfield, current_level);
+}
+
+static int ltc3208_probe(struct i2c_client *client)
+{
+	enum ltc3208_aux_channel aux_channels[LTC3208_NUM_AUX_LEDS];
+	DECLARE_BITMAP(registered_chans, LTC3208_CHAN_N_COUNT) = { };
+	struct regmap *regmap;
+	bool disable_rgb_aux4_dropout_signal;
+	bool disable_camhl_pin;
+	bool set_sub_control_pin;
+	int ret;
+	u8 dev_options;
+
+	regmap = devm_regmap_init_i2c(client, &ltc3208_regmap_cfg);
+	if (IS_ERR(regmap))
+		return dev_err_probe(&client->dev, PTR_ERR(regmap), "Failed to initialize regmap");
+
+	regcache_mark_dirty(regmap);
+	ret = regcache_sync(regmap);
+	if (ret)
+		return dev_err_probe(&client->dev, ret, "Failed to sync register cache");
+
+	disable_camhl_pin = device_property_read_bool(&client->dev, "adi,disable-camhl-pin");
+
+	set_sub_control_pin = device_property_read_bool(&client->dev, "adi,cfg-enrgbs-pin");
+
+	disable_rgb_aux4_dropout_signal =
+		device_property_read_bool(&client->dev, "adi,disable-rgb-aux4-dropout");
+
+	dev_options = FIELD_PREP(LTC3208_OPT_EN_RGBS, set_sub_control_pin) |
+		      FIELD_PREP(LTC3208_OPT_DIS_CAMHILO, disable_camhl_pin) |
+		      FIELD_PREP(LTC3208_OPT_CPO_MASK, 0) |
+		      FIELD_PREP(LTC3208_OPT_DIS_RGBDROP, disable_rgb_aux4_dropout_signal);
+
+	ret = regmap_write(regmap, LTC3208_REG_G_OPT, dev_options);
+	if (ret)
+		return dev_err_probe(&client->dev, ret, "Failed to set device options register");
+
+	/* Initialize AUX channel configurations */
+	for (int i = 0; i < LTC3208_NUM_AUX_LEDS; i++) {
+		/* Default value (AUX) if property is not present in DT */
+		if (!device_property_present(&client->dev, ltc3208_dt_aux_channels[i])) {
+			aux_channels[i] = LTC3208_AUX_CHAN_AUX;
+			continue;
+		}
+
+		ret = device_property_match_property_string(&client->dev,
+							    ltc3208_dt_aux_channels[i],
+							    ltc3208_aux_opt, LTC3208_NUM_AUX_LEDS);
+		if (ret < 0)
+			return dev_err_probe(&client->dev, ret, "Error reading AUX Channel %d", i);
+
+		aux_channels[i] = ret;
+	}
+
+	dev_options = FIELD_PREP(LTC3208_AUX1_MASK, aux_channels[0]) |
+		      FIELD_PREP(LTC3208_AUX2_MASK, aux_channels[1]) |
+		      FIELD_PREP(LTC3208_AUX3_MASK, aux_channels[2]) |
+		      FIELD_PREP(LTC3208_AUX4_MASK, aux_channels[3]);
+
+	ret = regmap_write(regmap, LTC3208_REG_E_AUX_SELECT, dev_options);
+	if (ret)
+		return dev_err_probe(&client->dev, ret, "Error writing to aux channel register");
+
+	device_for_each_child_node_scoped(&client->dev, child) {
+		struct ltc3208_led *led;
+		struct led_init_data init_data = { };
+		u32 chan;
+
+		ret = fwnode_property_read_u32(child, "reg", &chan);
+		if (ret)
+			return dev_err_probe(&client->dev, ret,
+					     "Failed to get register value of LED");
+		if (chan >= LTC3208_CHAN_N_COUNT)
+			return dev_err_probe(&client->dev, -EINVAL, "%u is an invalid LED ID",
+					     chan);
+		if (test_and_set_bit(chan, registered_chans))
+			return dev_err_probe(&client->dev, -EINVAL, "%u is already registered",
+					     chan);
+
+		led = devm_kzalloc(&client->dev, sizeof(*led), GFP_KERNEL);
+		if (!led)
+			return -ENOMEM;
+
+		led->rfield = devm_regmap_field_alloc(&client->dev, regmap,
+						      ltc3208_led_reg_field[chan]);
+		if (IS_ERR(led->rfield))
+			return dev_err_probe(&client->dev, PTR_ERR(led->rfield),
+					     "Cannot allocate regmap field");
+		led->cdev.brightness_set_blocking = ltc3208_led_set_brightness;
+		led->cdev.max_brightness = LTC3208_MAX_BRIGHTNESS_4BIT;
+
+		if (chan == LTC3208_CHAN_MAIN || chan == LTC3208_CHAN_SUB)
+			led->cdev.max_brightness = LTC3208_MAX_BRIGHTNESS_8BIT;
+
+		init_data.fwnode = child;
+
+		ret = devm_led_classdev_register_ext(&client->dev, &led->cdev, &init_data);
+		if (ret)
+			return dev_err_probe(&client->dev, ret, "Failed to register LED %u", chan);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id ltc3208_match_table[] = {
+	{ .compatible = "adi,ltc3208" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ltc3208_match_table);
+
+static struct i2c_driver ltc3208_driver = {
+	.driver = {
+		.name = "ltc3208",
+		.of_match_table = ltc3208_match_table,
+	},
+	.probe = ltc3208_probe,
+};
+module_i2c_driver(ltc3208_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Jan Carlo Roleda <jancarlo.roleda@analog.com>");
+MODULE_DESCRIPTION("LTC3208 LED Driver");
diff --git a/drivers/leds/leds-max77705.c b/drivers/leds/leds-max77705.c
index 1e2054c1bf80bf..4fd803c9598909 100644
--- a/drivers/leds/leds-max77705.c
+++ b/drivers/leds/leds-max77705.c
@@ -160,7 +160,6 @@ static int max77705_add_led(struct device *dev, struct regmap *regmap, struct fw
 	struct max77705_led *led;
 	struct led_classdev *cdev;
 	struct mc_subled *info;
-	struct fwnode_handle *child;
 	struct led_init_data init_data = {};
 
 	led = devm_kzalloc(dev, sizeof(*led), GFP_KERNEL);
@@ -191,7 +190,7 @@ static int max77705_add_led(struct device *dev, struct regmap *regmap, struct fw
 		cdev->brightness_set_blocking = max77705_led_brightness_set_multi;
 		cdev->blink_set = max77705_rgb_blink;
 
-		fwnode_for_each_child_node(np, child) {
+		fwnode_for_each_child_node_scoped(np, child) {
 			ret = max77705_parse_subled(dev, child, &info[i]);
 			if (ret < 0)
 				return ret;
diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c
index b2e081c8f13938..1133fa348e2464 100644
--- a/drivers/leds/leds-pca9532.c
+++ b/drivers/leds/leds-pca9532.c
@@ -298,12 +298,15 @@ static enum pca9532_state pca9532_getled(struct pca9532_led *led)
 	struct i2c_client *client = led->client;
 	struct pca9532_data *data = i2c_get_clientdata(client);
 	u8 maxleds = data->chip_info->num_leds;
-	char reg;
-	enum pca9532_state ret;
+	int reg;
+	enum pca9532_state ret = PCA9532_OFF;
 
 	mutex_lock(&data->update_lock);
 	reg = i2c_smbus_read_byte_data(client, LED_REG(maxleds, led->id));
-	ret = (reg & LED_MASK(led->id)) >> LED_SHIFT(led->id);
+	if (reg >= 0)
+		ret = (reg & LED_MASK(led->id)) >> LED_SHIFT(led->id);
+	else
+		dev_warn(&client->dev, "failed to read LED register: %d\n", reg);
 	mutex_unlock(&data->update_lock);
 	return ret;
 }
diff --git a/drivers/leds/leds-pca963x.c b/drivers/leds/leds-pca963x.c
index 605cf7497e4c3f..5363afbe93eaf9 100644
--- a/drivers/leds/leds-pca963x.c
+++ b/drivers/leds/leds-pca963x.c
@@ -208,10 +208,7 @@ static void pca963x_track_power_state(struct pca963x_led *led, unsigned int led_
 {
 	unsigned long *leds_on = &led->chip->leds_on;
 
-	if (brightness)
-		set_bit(led_num, leds_on);
-	else
-		clear_bit(led_num, leds_on);
+	assign_bit(led_num, leds_on, brightness);
 }
 
 static int pca963x_sync_power_state(struct pca963x_led *led, unsigned long cached_leds)
diff --git a/drivers/leds/leds-ss4200.c b/drivers/leds/leds-ss4200.c
index f24ca75c7cb183..c782339b13309d 100644
--- a/drivers/leds/leds-ss4200.c
+++ b/drivers/leds/leds-ss4200.c
@@ -339,7 +339,7 @@ static void ich7_lpc_cleanup(struct device *dev)
 
 /*
  * The OS has determined that the LPC of the Intel ICH7 Southbridge is present
- * so we can retrive the required operational information and prepare the GPIO.
+ * so we can retrieve the required operational information and prepare the GPIO.
  */
 static struct pci_dev *nas_gpio_pci_dev;
 static int ich7_lpc_probe(struct pci_dev *dev,
diff --git a/drivers/leds/leds-st1202.c b/drivers/leds/leds-st1202.c
index 168df5ecf27b7c..8c2e1d58ba0c80 100644
--- a/drivers/leds/leds-st1202.c
+++ b/drivers/leds/leds-st1202.c
@@ -12,9 +12,11 @@
 #include <linux/i2c.h>
 #include <linux/leds.h>
 #include <linux/module.h>
+#include <linux/property.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 
+#define ST1202_BLINK_DEFAULT_DELAY         500
 #define ST1202_CHAN_DISABLE_ALL            0x00
 #define ST1202_CHAN_ENABLE_HIGH            0x03
 #define ST1202_CHAN_ENABLE_LOW             0x02
@@ -23,7 +25,8 @@
 #define ST1202_CONFIG_REG_PATS             BIT(7)
 /* PATSR: Pattern sequence runs (self-clear when sequence is finished) */
 #define ST1202_CONFIG_REG_PATSR            BIT(6)
-#define ST1202_CONFIG_REG_SHFT             BIT(3)
+/* SHFT: Phase-shift delay enable */
+#define ST1202_CONFIG_REG_PHASE_SHIFT      BIT(3)
 #define ST1202_DEV_ENABLE                  0x01
 #define ST1202_DEV_ENABLE_ON               BIT(0)
 #define ST1202_DEV_ENABLE_RESET            BIT(7)
@@ -57,7 +60,7 @@ static struct st1202_led *cdev_to_st1202_led(struct led_classdev *cdev)
 	return container_of(cdev, struct st1202_led, led_cdev);
 }
 
-static int st1202_read_reg(struct st1202_chip *chip, int reg, uint8_t *val)
+static int st1202_read_reg(struct st1202_chip *chip, int reg, u8 *val)
 {
 	struct device *dev = &chip->client->dev;
 	int ret;
@@ -68,11 +71,11 @@ static int st1202_read_reg(struct st1202_chip *chip, int reg, uint8_t *val)
 		return ret;
 	}
 
-	*val = (uint8_t)ret;
+	*val = (u8)ret;
 	return 0;
 }
 
-static int st1202_write_reg(struct st1202_chip *chip, int reg, uint8_t val)
+static int st1202_write_reg(struct st1202_chip *chip, int reg, u8 val)
 {
 	struct device *dev = &chip->client->dev;
 	int ret;
@@ -84,11 +87,27 @@ static int st1202_write_reg(struct st1202_chip *chip, int reg, uint8_t val)
 	return ret;
 }
 
-static uint8_t st1202_prescalar_to_miliseconds(unsigned int value)
+static u8 st1202_milliseconds_to_prescaler(unsigned int value)
 {
 	return value / ST1202_MILLIS_PATTERN_DUR_MIN;
 }
 
+/* The channel current register is 8 bits wide, whatever max-brightness says */
+static u8 st1202_iled_max(struct led_classdev *led_cdev)
+{
+	return min_t(unsigned int, led_cdev->max_brightness, U8_MAX);
+}
+
+static u16 st1202_brightness_to_pwm(int brightness, unsigned int max_brightness)
+{
+	if (!max_brightness)
+		return 0;
+
+	brightness = clamp_t(int, brightness, LED_OFF, max_brightness);
+
+	return brightness * ST1202_PATTERN_PWM_FULL / max_brightness;
+}
+
 static int st1202_pwm_pattern_write(struct st1202_chip *chip, int led_num,
 				int pattern, unsigned int value)
 {
@@ -125,7 +144,7 @@ static int st1202_duration_pattern_write(struct st1202_chip *chip, int pattern,
 					unsigned int value)
 {
 	return st1202_write_reg(chip, (ST1202_PATTERN_DUR + pattern),
-				st1202_prescalar_to_miliseconds(value));
+				st1202_milliseconds_to_prescaler(value));
 }
 
 static int __st1202_channel_set(struct st1202_chip *chip, int led_num, bool active)
@@ -160,27 +179,6 @@ static int __st1202_channel_set(struct st1202_chip *chip, int led_num, bool acti
 	return 0;
 }
 
-static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active)
-{
-	guard(mutex)(&chip->lock);
-
-	return __st1202_channel_set(chip, led_num, active);
-}
-
-static void st1202_brightness_set(struct led_classdev *led_cdev,
-				enum led_brightness value)
-{
-	struct st1202_led *led = cdev_to_st1202_led(led_cdev);
-	struct st1202_chip *chip = led->chip;
-
-	guard(mutex)(&chip->lock);
-
-	for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++)
-		st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL);
-	st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value);
-	__st1202_channel_set(chip, led->led_num, !!value);
-}
-
 static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev)
 {
 	struct st1202_led *led = cdev_to_st1202_led(led_cdev);
@@ -197,8 +195,30 @@ static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev)
 static int st1202_led_set(struct led_classdev *ldev, enum led_brightness value)
 {
 	struct st1202_led *led = cdev_to_st1202_led(ldev);
+	struct st1202_chip *chip = led->chip;
+	int ret;
+
+	guard(mutex)(&chip->lock);
 
-	return st1202_channel_set(led->chip, led->led_num, !!value);
+	/*
+	 * The output of a channel is ILED x Pattern_PWM / 4095. Setting every
+	 * PWM slot to full scale makes it equal ILED whatever the state of the
+	 * sequencer, so the brightness takes effect without stopping the
+	 * sequencer, which is global and would disturb the other channels.
+	 */
+	for (int pattern = 0; pattern < ST1202_MAX_PATTERNS; pattern++) {
+		ret = st1202_pwm_pattern_write(chip, led->led_num, pattern,
+						ST1202_PATTERN_PWM_FULL);
+		if (ret)
+			return ret;
+	}
+
+	ret = st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num,
+				min_t(unsigned int, value, st1202_iled_max(ldev)));
+	if (ret)
+		return ret;
+
+	return __st1202_channel_set(chip, led->led_num, !!value);
 }
 
 static int st1202_led_pattern_clear(struct led_classdev *ldev)
@@ -209,16 +229,25 @@ static int st1202_led_pattern_clear(struct led_classdev *ldev)
 
 	guard(mutex)(&chip->lock);
 
-	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT);
+	ret = st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, LED_OFF);
+	if (ret != 0)
+		return ret;
+
+	ret = __st1202_channel_set(chip, led->led_num, false);
+	if (ret != 0)
+		return ret;
+
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_PHASE_SHIFT);
 	if (ret != 0)
 		return ret;
 
-	for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) {
-		ret = st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL);
+	for (int pattern = 0; pattern < ST1202_MAX_PATTERNS; pattern++) {
+		ret = st1202_pwm_pattern_write(chip, led->led_num, pattern,
+						ST1202_PATTERN_PWM_FULL);
 		if (ret != 0)
 			return ret;
 
-		ret = st1202_write_reg(chip, ST1202_PATTERN_DUR + patt, 0);
+		ret = st1202_write_reg(chip, ST1202_PATTERN_DUR + pattern, 0);
 		if (ret != 0)
 			return ret;
 	}
@@ -227,34 +256,45 @@ static int st1202_led_pattern_clear(struct led_classdev *ldev)
 }
 
 static int st1202_led_pattern_set(struct led_classdev *ldev,
-				struct led_pattern *pattern,
+				struct led_pattern *patterns,
 				u32 len, int repeat)
 {
 	struct st1202_led *led = cdev_to_st1202_led(ldev);
 	struct st1202_chip *chip = led->chip;
+	unsigned int max_brightness;
 	int ret;
 
+	max_brightness = st1202_iled_max(ldev);
+
 	if (len > ST1202_MAX_PATTERNS)
 		return -EINVAL;
 
-	for (int patt = 0; patt < len; patt++) {
-		if (pattern[patt].delta_t < ST1202_MILLIS_PATTERN_DUR_MIN ||
-				pattern[patt].delta_t > ST1202_MILLIS_PATTERN_DUR_MAX)
+	for (int pattern = 0; pattern < len; pattern++) {
+		if (patterns[pattern].delta_t < ST1202_MILLIS_PATTERN_DUR_MIN ||
+				patterns[pattern].delta_t > ST1202_MILLIS_PATTERN_DUR_MAX)
 			return -EINVAL;
 	}
 
 	guard(mutex)(&chip->lock);
 
-	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT);
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_PHASE_SHIFT);
 	if (ret != 0)
 		return ret;
 
-	for (int patt = 0; patt < len; patt++) {
-		ret = st1202_pwm_pattern_write(chip, led->led_num, patt, pattern[patt].brightness);
+	for (int pattern = 0; pattern < len; pattern++) {
+		ret = st1202_pwm_pattern_write(chip, led->led_num, pattern,
+					st1202_brightness_to_pwm(patterns[pattern].brightness,
+								max_brightness));
+		if (ret != 0)
+			return ret;
+
+		ret = st1202_duration_pattern_write(chip, pattern, patterns[pattern].delta_t);
 		if (ret != 0)
 			return ret;
+	}
 
-		ret = st1202_duration_pattern_write(chip, patt, pattern[patt].delta_t);
+	for (int pattern = len; pattern < ST1202_MAX_PATTERNS; pattern++) {
+		ret = st1202_pwm_pattern_write(chip, led->led_num, pattern, LED_OFF);
 		if (ret != 0)
 			return ret;
 	}
@@ -263,18 +303,117 @@ static int st1202_led_pattern_set(struct led_classdev *ldev,
 	if (ret != 0)
 		return ret;
 
+	ret = st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, max_brightness);
+	if (ret != 0)
+		return ret;
+
 	ret = __st1202_channel_set(chip, led->led_num, true);
 	if (ret != 0)
 		return ret;
 
-	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, (ST1202_CONFIG_REG_PATSR |
-							ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_SHFT));
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG,
+				ST1202_CONFIG_REG_PATSR | ST1202_CONFIG_REG_PATS |
+				ST1202_CONFIG_REG_PHASE_SHIFT);
 	if (ret != 0)
 		return ret;
 
 	return 0;
 }
 
+static int st1202_blink_set(struct led_classdev *led_cdev,
+			unsigned long *delay_on, unsigned long *delay_off)
+{
+	struct st1202_led *led = cdev_to_st1202_led(led_cdev);
+	struct st1202_chip *chip = led->chip;
+	unsigned long on, off;
+	int ret;
+
+	if (!*delay_on)
+		*delay_on = ST1202_BLINK_DEFAULT_DELAY;
+	if (!*delay_off)
+		*delay_off = ST1202_BLINK_DEFAULT_DELAY;
+
+	on = *delay_on;
+	off = *delay_off;
+
+	on = clamp_val(on, ST1202_MILLIS_PATTERN_DUR_MIN, ST1202_MILLIS_PATTERN_DUR_MAX);
+	off = clamp_val(off, ST1202_MILLIS_PATTERN_DUR_MIN, ST1202_MILLIS_PATTERN_DUR_MAX);
+	on = roundup(on, ST1202_MILLIS_PATTERN_DUR_MIN);
+	off = roundup(off, ST1202_MILLIS_PATTERN_DUR_MIN);
+
+	guard(mutex)(&chip->lock);
+
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_PHASE_SHIFT);
+	if (ret)
+		return ret;
+
+	/* Zero out PWM for all other active channels to prevent them from blinking */
+	for (int chan = 0; chan < ST1202_MAX_LEDS; chan++) {
+		if (!chip->leds[chan].is_active || chan == led->led_num)
+			continue;
+
+		ret = st1202_pwm_pattern_write(chip, chan, 0, LED_OFF);
+		if (ret)
+			return ret;
+
+		ret = st1202_pwm_pattern_write(chip, chan, 1, LED_OFF);
+		if (ret)
+			return ret;
+	}
+
+	ret = st1202_pwm_pattern_write(chip, led->led_num, 0, ST1202_PATTERN_PWM_FULL);
+	if (ret)
+		return ret;
+
+	for (int pattern = 1; pattern < ST1202_MAX_PATTERNS; pattern++) {
+		ret = st1202_pwm_pattern_write(chip, led->led_num, pattern, LED_OFF);
+		if (ret)
+			return ret;
+	}
+
+	ret = st1202_duration_pattern_write(chip, 0, on);
+	if (ret)
+		return ret;
+
+	ret = st1202_duration_pattern_write(chip, 1, off);
+	if (ret)
+		return ret;
+
+	for (int pattern = 2; pattern < ST1202_MAX_PATTERNS; pattern++) {
+		ret = st1202_write_reg(chip, ST1202_PATTERN_DUR + pattern, 0);
+		if (ret)
+			return ret;
+	}
+
+	ret = st1202_write_reg(chip, ST1202_PATTERN_REP, U8_MAX);
+	if (ret)
+		return ret;
+
+	ret = st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, st1202_iled_max(led_cdev));
+	if (ret)
+		return ret;
+
+	ret = __st1202_channel_set(chip, led->led_num, true);
+	if (ret)
+		return ret;
+
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG,
+				ST1202_CONFIG_REG_PATSR | ST1202_CONFIG_REG_PATS |
+				ST1202_CONFIG_REG_PHASE_SHIFT);
+	if (ret)
+		return ret;
+
+	*delay_on = on;
+	*delay_off = off;
+
+	return 0;
+}
+
+static void st1202_fwnode_put(void *data)
+{
+	fwnode_handle_put(data);
+}
+
 static int st1202_dt_init(struct st1202_chip *chip)
 {
 	struct device *dev = &chip->client->dev;
@@ -294,14 +433,18 @@ static int st1202_dt_init(struct st1202_chip *chip)
 
 		led = &chip->leds[reg];
 		led->is_active = true;
-		led->fwnode = of_fwnode_handle(child);
+		led->fwnode = fwnode_handle_get(of_fwnode_handle(child));
+
+		err = devm_add_action_or_reset(dev, st1202_fwnode_put, led->fwnode);
+		if (err)
+			return err;
 
 		led->led_cdev.max_brightness = U8_MAX;
 		led->led_cdev.brightness_set_blocking = st1202_led_set;
 		led->led_cdev.pattern_set = st1202_led_pattern_set;
 		led->led_cdev.pattern_clear = st1202_led_pattern_clear;
 		led->led_cdev.default_trigger = "pattern";
-		led->led_cdev.brightness_set = st1202_brightness_set;
+		led->led_cdev.blink_set = st1202_blink_set;
 		led->led_cdev.brightness_get = st1202_brightness_get;
 	}
 
@@ -330,7 +473,7 @@ static int st1202_setup(struct st1202_chip *chip)
 		return ret;
 
 	/* Enable phase-shift delay feature */
-	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT);
+	ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_PHASE_SHIFT);
 	if (ret < 0)
 		return ret;
 
@@ -342,7 +485,7 @@ static int st1202_setup(struct st1202_chip *chip)
 	/* Duration of initialization */
 	usleep_range(6500, 10000);
 
-	/* Deactivate all LEDS (channels) and activate only the ones found in Device Tree */
+	/* Deactivate all LEDs (channels); each is enabled when it is lit */
 	ret = st1202_write_reg(chip, ST1202_CHAN_ENABLE_LOW, ST1202_CHAN_DISABLE_ALL);
 	if (ret < 0)
 		return ret;
@@ -389,11 +532,6 @@ static int st1202_probe(struct i2c_client *client)
 		if (!led->is_active)
 			continue;
 
-		ret = st1202_channel_set(led->chip, led->led_num, true);
-		if (ret < 0)
-			return dev_err_probe(&client->dev, ret,
-					"Failed to activate LED channel\n");
-
 		ret = st1202_led_pattern_clear(&led->led_cdev);
 		if (ret < 0)
 			return dev_err_probe(&client->dev, ret,
diff --git a/drivers/leds/leds-syscon.c b/drivers/leds/leds-syscon.c
index d633ad519d0cdc..741850f35f1927 100644
--- a/drivers/leds/leds-syscon.c
+++ b/drivers/leds/leds-syscon.c
@@ -56,7 +56,6 @@ static int syscon_led_probe(struct platform_device *pdev)
 {
 	struct led_init_data init_data = {};
 	struct device *dev = &pdev->dev;
-	struct device_node *np = dev_of_node(dev);
 	struct device *parent;
 	struct regmap *map;
 	struct syscon_led *sled;
@@ -81,13 +80,13 @@ static int syscon_led_probe(struct platform_device *pdev)
 
 	sled->map = map;
 
-	if (of_property_read_u32(np, "reg", &sled->offset) &&
-	    of_property_read_u32(np, "offset", &sled->offset))
+	if (device_property_read_u32(dev, "reg", &sled->offset) &&
+	    device_property_read_u32(dev, "offset", &sled->offset))
 		return -EINVAL;
-	if (of_property_read_u32(np, "mask", &sled->mask))
+	if (device_property_read_u32(dev, "mask", &sled->mask))
 		return -EINVAL;
 
-	init_data.fwnode = of_fwnode_handle(np);
+	init_data.fwnode = dev_fwnode(dev);
 
 	state = led_init_default_state_get(init_data.fwnode);
 	switch (state) {
diff --git a/drivers/leds/leds-turris-omnia.c b/drivers/leds/leds-turris-omnia.c
index ed6a47bbb44f9c..ab10cc77c47f48 100644
--- a/drivers/leds/leds-turris-omnia.c
+++ b/drivers/leds/leds-turris-omnia.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/led-class-multicolor.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
@@ -195,10 +196,16 @@ static void omnia_hwtrig_deactivate(struct led_classdev *cdev)
 			err);
 }
 
+static bool omnia_hwtrig_hw_offloaded(struct led_classdev *cdev)
+{
+	return true;
+}
+
 static struct led_trigger omnia_hw_trigger = {
 	.name		= "omnia-mcu",
 	.activate	= omnia_hwtrig_activate,
 	.deactivate	= omnia_hwtrig_deactivate,
+	.hw_offloaded	= omnia_hwtrig_hw_offloaded,
 	.trigger_type	= &omnia_hw_trigger_type,
 };
 
@@ -251,6 +258,7 @@ static int omnia_led_register(struct i2c_client *client, struct omnia_led *led,
 	 * by LED class from the linux,default-trigger property.
 	 */
 	cdev->default_trigger = omnia_hw_trigger.name;
+	cdev->hw_control_trigger = omnia_hw_trigger.name;
 
 	/* Put the LED into software mode */
 	ret = omnia_cmd_write_u8(client, OMNIA_CMD_LED_MODE, OMNIA_CMD_LED_MODE_LED(led->reg) |
diff --git a/drivers/leds/leds.h b/drivers/leds/leds.h
index bee46651e068f7..6d00e6f4412606 100644
--- a/drivers/leds/leds.h
+++ b/drivers/leds/leds.h
@@ -21,14 +21,10 @@ void led_init_core(struct led_classdev *led_cdev);
 void led_stop_software_blink(struct led_classdev *led_cdev);
 void led_set_brightness_nopm(struct led_classdev *led_cdev, unsigned int value);
 void led_set_brightness_nosleep(struct led_classdev *led_cdev, unsigned int value);
-ssize_t led_trigger_read(struct file *filp, struct kobject *kobj,
-			const struct bin_attribute *attr, char *buf,
-			loff_t pos, size_t count);
-ssize_t led_trigger_write(struct file *filp, struct kobject *kobj,
-			const struct bin_attribute *bin_attr, char *buf,
-			loff_t pos, size_t count);
+void led_trigger_hw_control_changed_worker(struct work_struct *work);
 
 extern struct rw_semaphore leds_list_lock;
 extern struct list_head leds_list;
+extern const struct attribute_group led_trigger_group;
 
 #endif	/* __LEDS_H_INCLUDED */
diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c
index d7d6518de30f2b..24ffe079f22153 100644
--- a/drivers/leds/rgb/leds-qcom-lpg.c
+++ b/drivers/leds/rgb/leds-qcom-lpg.c
@@ -11,6 +11,7 @@
 #include <linux/nvmem-consumer.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/pwm.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
@@ -1342,7 +1343,7 @@ static int lpg_add_pwm(struct lpg *lpg)
 	return ret;
 }
 
-static int lpg_parse_channel(struct lpg *lpg, struct device_node *np,
+static int lpg_parse_channel(struct lpg *lpg, struct fwnode_handle *fw,
 			     struct lpg_channel **channel)
 {
 	struct lpg_channel *chan;
@@ -1350,17 +1351,18 @@ static int lpg_parse_channel(struct lpg *lpg, struct device_node *np,
 	u32 reg;
 	int ret;
 
-	ret = of_property_read_u32(np, "reg", &reg);
+	ret = fwnode_property_read_u32(fw, "reg", &reg);
 	if (ret || !reg || reg > lpg->num_channels)
-		return dev_err_probe(lpg->dev, -EINVAL, "invalid \"reg\" of %pOFn\n", np);
+		return dev_err_probe(lpg->dev, -EINVAL,
+				     "invalid \"reg\" of %s\n", fwnode_get_name(fw));
 
 	chan = &lpg->channels[reg - 1];
 	chan->in_use = true;
 
-	ret = of_property_read_u32(np, "color", &color);
+	ret = fwnode_property_read_u32(fw, "color", &color);
 	if (ret < 0 && ret != -EINVAL)
 		return dev_err_probe(lpg->dev, ret,
-				     "failed to parse \"color\" of %pOF\n", np);
+				     "failed to parse \"color\" of %s\n", fwnode_get_name(fw));
 
 	chan->color = color;
 
@@ -1369,25 +1371,26 @@ static int lpg_parse_channel(struct lpg *lpg, struct device_node *np,
 	return 0;
 }
 
-static int lpg_add_led(struct lpg *lpg, struct device_node *np)
+static int lpg_add_led(struct lpg *lpg, struct fwnode_handle *fw)
 {
 	struct led_init_data init_data = {};
 	struct led_classdev *cdev;
 	struct mc_subled *info;
 	struct lpg_led *led;
+	const char *trigger;
 	const char *state;
 	int num_channels;
 	u32 color = 0;
 	int ret;
 	int i;
 
-	ret = of_property_read_u32(np, "color", &color);
+	ret = fwnode_property_read_u32(fw, "color", &color);
 	if (ret < 0 && ret != -EINVAL)
 		return dev_err_probe(lpg->dev, ret,
-			      "failed to parse \"color\" of %pOF\n", np);
+			      "failed to parse \"color\" of %s\n", fwnode_get_name(fw));
 
 	if (color == LED_COLOR_ID_RGB || color == LED_COLOR_ID_MULTI)
-		num_channels = of_get_available_child_count(np);
+		num_channels = fwnode_get_child_node_count(fw);
 	else
 		num_channels = 1;
 
@@ -1403,7 +1406,7 @@ static int lpg_add_led(struct lpg *lpg, struct device_node *np)
 		if (!info)
 			return -ENOMEM;
 		i = 0;
-		for_each_available_child_of_node_scoped(np, child) {
+		fwnode_for_each_available_child_node_scoped(fw, child) {
 			ret = lpg_parse_channel(lpg, child, &led->channels[i]);
 			if (ret < 0)
 				return ret;
@@ -1414,7 +1417,7 @@ static int lpg_add_led(struct lpg *lpg, struct device_node *np)
 		}
 
 		led->mcdev.subled_info = info;
-		led->mcdev.num_colors = num_channels;
+		led->mcdev.num_colors = i;
 
 		cdev = &led->mcdev.led_cdev;
 		cdev->brightness_set_blocking = lpg_brightness_mc_set;
@@ -1426,7 +1429,7 @@ static int lpg_add_led(struct lpg *lpg, struct device_node *np)
 			cdev->pattern_clear = lpg_pattern_mc_clear;
 		}
 	} else {
-		ret = lpg_parse_channel(lpg, np, &led->channels[0]);
+		ret = lpg_parse_channel(lpg, fw, &led->channels[0]);
 		if (ret < 0)
 			return ret;
 
@@ -1441,14 +1444,15 @@ static int lpg_add_led(struct lpg *lpg, struct device_node *np)
 		}
 	}
 
-	cdev->default_trigger = of_get_property(np, "linux,default-trigger", NULL);
+	if (!fwnode_property_read_string(fw, "linux,default-trigger", &trigger))
+		cdev->default_trigger = trigger;
 
 	if (lpg->lpg_chan_sdam)
 		cdev->max_brightness = PPG_MAX_LED_BRIGHTNESS;
 	else
 		cdev->max_brightness = LPG_RESOLUTION_9BIT - 1;
 
-	if (!of_property_read_string(np, "default-state", &state) &&
+	if (!fwnode_property_read_string(fw, "default-state", &state) &&
 	    !strcmp(state, "on"))
 		cdev->brightness = cdev->max_brightness;
 	else
@@ -1456,7 +1460,7 @@ static int lpg_add_led(struct lpg *lpg, struct device_node *np)
 
 	cdev->brightness_set_blocking(cdev, cdev->brightness);
 
-	init_data.fwnode = of_fwnode_handle(np);
+	init_data.fwnode = fw;
 
 	if (color == LED_COLOR_ID_RGB || color == LED_COLOR_ID_MULTI)
 		ret = devm_led_classdev_multicolor_register_ext(lpg->dev, &led->mcdev, &init_data);
@@ -1638,8 +1642,8 @@ static int lpg_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	for_each_available_child_of_node_scoped(pdev->dev.of_node, np) {
-		ret = lpg_add_led(lpg, np);
+	device_for_each_child_node_scoped(&pdev->dev, child) {
+		ret = lpg_add_led(lpg, child);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig
index c11282a74b5ac3..a11d04ce4ab2c8 100644
--- a/drivers/leds/trigger/Kconfig
+++ b/drivers/leds/trigger/Kconfig
@@ -9,6 +9,16 @@ menuconfig LEDS_TRIGGERS
 
 if LEDS_TRIGGERS
 
+config LEDS_TRIGGERS_HW_CHANGED
+	bool "LED hardware-initiated trigger transition support"
+	default LEDS_BRIGHTNESS_HW_CHANGED
+	help
+	  This option enables support for hardware initiated hardware control
+	  transitions, where the LED hardware autonomously switches between
+	  "none" (i.e., no trigger) and its private trigger.
+
+	  See Documentation/leds/leds-class.rst for details.
+
 config LEDS_TRIGGER_TIMER
 	tristate "LED Timer Trigger"
 	help
diff --git a/drivers/leds/trigger/ledtrig-netdev.c b/drivers/leds/trigger/ledtrig-netdev.c
index 5b4e92c14dbb41..5b0132484594c7 100644
--- a/drivers/leds/trigger/ledtrig-netdev.c
+++ b/drivers/leds/trigger/ledtrig-netdev.c
@@ -414,10 +414,7 @@ static ssize_t netdev_led_attr_store(struct device *dev, const char *buf,
 		return -EINVAL;
 	}
 
-	if (state)
-		set_bit(bit, &mode);
-	else
-		clear_bit(bit, &mode);
+	assign_bit(bit, &mode, state);
 
 	if (test_bit(TRIGGER_NETDEV_LINK, &mode) &&
 	    (test_bit(TRIGGER_NETDEV_LINK_10, &mode) ||
@@ -798,10 +795,18 @@ static void netdev_trig_deactivate(struct led_classdev *led_cdev)
 	kfree(trigger_data);
 }
 
+static bool netdev_trig_hw_offloaded(struct led_classdev *led_cdev)
+{
+	struct led_netdev_data *trigger_data = led_get_trigger_data(led_cdev);
+
+	return trigger_data->hw_control;
+}
+
 static struct led_trigger netdev_led_trigger = {
 	.name = "netdev",
 	.activate = netdev_trig_activate,
 	.deactivate = netdev_trig_deactivate,
+	.hw_offloaded = netdev_trig_hw_offloaded,
 	.groups = netdev_trig_groups,
 };
 
diff --git a/drivers/macintosh/windfarm_core.c b/drivers/macintosh/windfarm_core.c
index 5307b1e34261d3..e66de11c69a384 100644
--- a/drivers/macintosh/windfarm_core.c
+++ b/drivers/macintosh/windfarm_core.c
@@ -34,6 +34,7 @@
 #include <linux/platform_device.h>
 #include <linux/mutex.h>
 #include <linux/freezer.h>
+#include <linux/umh.h>
 
 #include "windfarm.h"
 
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index df27c7d066d23d..9a90e81817c73a 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -610,6 +610,20 @@ config DM_VERITY_VERIFY_ROOTHASH_SIG_PLATFORM_KEYRING
 
 	  If unsure, say N.
 
+config DM_VERITY_VERIFY_ROOTHASH_SIG_FORCE
+	bool "Require dm-verity root hash signature verification"
+	depends on DM_VERITY_VERIFY_ROOTHASH_SIG
+	help
+	  Reject dm-verity devices that are created without a valid root hash
+	  signature.  Without this, whether a signature is required is decided
+	  at boot time by the dm_verity.require_signatures parameter which
+	  defaults to off.
+
+	  Enabling this makes that parameter default to on and it can then no
+	  longer be turned off.
+
+	  If unsure, say N.
+
 config DM_VERITY_FEC
 	bool "Verity forward error correction support"
 	depends on DM_VERITY
diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c
index acd9b179fcb3f2..179b76173255b7 100644
--- a/drivers/md/dm-cache-metadata.c
+++ b/drivers/md/dm-cache-metadata.c
@@ -49,7 +49,7 @@ enum superblock_flag_bits {
 enum mapping_bits {
 	/*
 	 * A valid mapping.  Because we're using an array we clear this
-	 * flag for an non existant mapping.
+	 * flag for an non existent mapping.
 	 */
 	M_VALID = 1,
 
diff --git a/drivers/md/dm-clone-metadata.c b/drivers/md/dm-clone-metadata.c
index e50a5b5a4aeaaa..75ea39d18c12a9 100644
--- a/drivers/md/dm-clone-metadata.c
+++ b/drivers/md/dm-clone-metadata.c
@@ -300,9 +300,6 @@ static void __prepare_superblock(struct dm_clone_metadata *cmd,
 {
 	sb->flags = cpu_to_le32(0UL);
 
-	/* FIXME: UUID is currently unused */
-	memset(sb->uuid, 0, sizeof(sb->uuid));
-
 	sb->magic = cpu_to_le64(SUPERBLOCK_MAGIC);
 	sb->version = cpu_to_le32(DM_CLONE_MAX_METADATA_VERSION);
 
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 9e170de50ad323..8e838530faaba3 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -513,7 +513,7 @@ static void crypt_iv_lmk_wipe(struct crypt_config *cc)
 	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
 
 	if (lmk->seed)
-		memset(lmk->seed, 0, LMK_SEED_SIZE);
+		memzero_explicit(lmk->seed, LMK_SEED_SIZE);
 }
 
 static void crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
@@ -630,8 +630,8 @@ static void crypt_iv_tcw_wipe(struct crypt_config *cc)
 {
 	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
 
-	memset(tcw->iv_seed, 0, cc->iv_size);
-	memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
+	memzero_explicit(tcw->iv_seed, cc->iv_size);
+	memzero_explicit(tcw->whitening, TCW_WHITENING_SIZE);
 }
 
 static void crypt_iv_tcw_whitening(struct crypt_config *cc,
@@ -2365,7 +2365,7 @@ static unsigned int crypt_authenckey_size(struct crypt_config *cc)
 /*
  * If AEAD is composed like authenc(hmac(sha256),xts(aes)),
  * the key must be for some reason in special format.
- * This funcion converts cc->key to this special format.
+ * This function converts cc->key to this special format.
  */
 static void crypt_copy_authenckey(char *p, const void *key,
 				  unsigned int enckeylen, unsigned int authkeylen)
@@ -2595,7 +2595,7 @@ static int get_key_size(char **key_string)
 static int crypt_set_key(struct crypt_config *cc, char *key)
 {
 	int r = -EINVAL;
-	int key_string_len = strlen(key);
+	size_t key_string_len = strlen(key);
 
 	/* Hyphen (which gives a key_size of zero) means there is no key. */
 	if (!cc->key_size && strcmp(key, "-"))
@@ -2614,6 +2614,9 @@ static int crypt_set_key(struct crypt_config *cc, char *key)
 	kfree_sensitive(cc->key_string);
 	cc->key_string = NULL;
 
+	if (cc->key_size && key_string_len != cc->key_size * 2)
+		goto out;
+
 	/* Decode key from its hex representation. */
 	if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0)
 		goto out;
@@ -2624,7 +2627,7 @@ static int crypt_set_key(struct crypt_config *cc, char *key)
 
 out:
 	/* Hex key string not needed after here, so wipe it. */
-	memset(key, '0', key_string_len);
+	memzero_explicit(key, key_string_len);
 
 	return r;
 }
@@ -2643,7 +2646,7 @@ static int crypt_wipe_key(struct crypt_config *cc)
 	kfree_sensitive(cc->key_string);
 	cc->key_string = NULL;
 	r = crypt_setkey(cc);
-	memset(&cc->key, 0, cc->key_size * sizeof(u8));
+	memzero_explicit(cc->key, cc->key_size);
 
 	return r;
 }
@@ -3064,7 +3067,7 @@ static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
 
 	/* wipe the kernel key payload copy */
 	if (cc->key_string)
-		memset(cc->key, 0, cc->key_size * sizeof(u8));
+		memzero_explicit(cc->key, cc->key_size);
 
 	return ret;
 }
@@ -3643,7 +3646,7 @@ static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv,
 			/* The key size may not be changed. */
 			key_size = get_key_size(&argv[2]);
 			if (key_size < 0 || cc->key_size != key_size) {
-				memset(argv[2], '0', strlen(argv[2]));
+				memzero_explicit(argv[2], strlen(argv[2]));
 				return -EINVAL;
 			}
 
@@ -3654,7 +3657,7 @@ static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv,
 				ret = cc->iv_gen_ops->init(cc);
 			/* wipe the kernel key payload copy */
 			if (cc->key_string)
-				memset(cc->key, 0, cc->key_size * sizeof(u8));
+				memzero_explicit(cc->key, cc->key_size);
 			return ret;
 		}
 		if (argc == 2 && !strcasecmp(argv[1], "wipe"))
diff --git a/drivers/md/dm-delay.c b/drivers/md/dm-delay.c
index 4864671c2222ff..20c97e92d8061a 100644
--- a/drivers/md/dm-delay.c
+++ b/drivers/md/dm-delay.c
@@ -301,6 +301,7 @@ static int delay_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	}
 
 	ti->num_flush_bios = 1;
+	ti->flush_supported = dc->flush.delay != 0;
 	ti->num_discard_bios = 1;
 	ti->accounts_remapped_io = true;
 	ti->per_io_data_size = sizeof(struct dm_delay_info);
@@ -451,7 +452,7 @@ static int delay_iterate_devices(struct dm_target *ti,
 
 static struct target_type delay_target = {
 	.name	     = "delay",
-	.version     = {1, 5, 0},
+	.version     = {1, 6, 0},
 	.features    = DM_TARGET_PASSES_INTEGRITY | DM_TARGET_ZONED_HM,
 	.module      = THIS_MODULE,
 	.ctr	     = delay_ctr,
diff --git a/drivers/md/dm-exception-store.h b/drivers/md/dm-exception-store.h
index 061b4d3108132c..6924d673cf4593 100644
--- a/drivers/md/dm-exception-store.h
+++ b/drivers/md/dm-exception-store.h
@@ -135,7 +135,7 @@ struct dm_dev *dm_snap_origin(struct dm_snapshot *snap);
 struct dm_dev *dm_snap_cow(struct dm_snapshot *snap);
 
 /*
- * Funtions to manipulate consecutive chunks
+ * Functions to manipulate consecutive chunks
  */
 #define DM_CHUNK_CONSECUTIVE_BITS 8
 #define DM_CHUNK_NUMBER_BITS 56
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 7cb7bb6233b645..ae9318f5f21b7c 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -1604,10 +1604,7 @@ static void pg_init_done(void *data, int errors)
 		goto out;
 
 	if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
-		if (delay_retry)
-			set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
-		else
-			clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
+		assign_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags, delay_retry);
 
 		if (__pg_init_all_paths(m))
 			goto out;
diff --git a/drivers/md/dm-pcache/backing_dev.c b/drivers/md/dm-pcache/backing_dev.c
index 7165fc0364bbc1..23002beb192aee 100644
--- a/drivers/md/dm-pcache/backing_dev.c
+++ b/drivers/md/dm-pcache/backing_dev.c
@@ -68,7 +68,7 @@ void backing_dev_stop(struct dm_pcache *pcache)
 	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
 
 	/*
-	 * There should not be any new request comming, just wait
+	 * There should not be any new request coming, just wait
 	 * inflight requests done.
 	 */
 	wait_event(backing_dev->inflight_wq,
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 8f5a5e1342a956..ac019a0f5a929e 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -1784,7 +1784,7 @@ static int rs_check_takeover(struct raid_set *rs)
 
 		/* raid10* -> raid0 */
 		if (mddev->new_level == 0) {
-			/* Can takeover raid10_near with raid disks divisable by data copies! */
+			/* Can takeover raid10_near with raid disks divisible by data copies! */
 			if (near_copies > 1 &&
 			    !(mddev->raid_disks % near_copies)) {
 				mddev->raid_disks /= near_copies;
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index da2a5e2002ecaf..36516338ca6ff3 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -1393,7 +1393,7 @@ static void mirror_resume(struct dm_target *ti)
  * we have encountered.
  *    A => Alive - No failures
  *    D => Dead - A write failure occurred leaving mirror out-of-sync
- *    S => Sync - A sychronization failure occurred, mirror out-of-sync
+ *    S => Sync - A synchronization failure occurred, mirror out-of-sync
  *    R => Read - A read failure occurred, mirror data unaffected
  *
  * Returns: <char>
diff --git a/drivers/md/dm-unstripe.c b/drivers/md/dm-unstripe.c
index bfcbe6bfa71a83..770a9958def0aa 100644
--- a/drivers/md/dm-unstripe.c
+++ b/drivers/md/dm-unstripe.c
@@ -33,7 +33,7 @@ static void cleanup_unstripe(struct unstripe_c *uc, struct dm_target *ti)
 }
 
 /*
- * Contruct an unstriped mapping.
+ * Construct an unstriped mapping.
  * <number of stripes> <chunk size> <stripe #> <dev_path> <offset>
  */
 static int unstripe_ctr(struct dm_target *ti, unsigned int argc, char **argv)
diff --git a/drivers/md/dm-vdo/block-map.c b/drivers/md/dm-vdo/block-map.c
index 5ffc360540ed1e..258ad5521f51f9 100644
--- a/drivers/md/dm-vdo/block-map.c
+++ b/drivers/md/dm-vdo/block-map.c
@@ -240,7 +240,7 @@ static int __must_check allocate_cache_components(struct vdo_page_cache *cache)
  * assert_on_cache_thread() - Assert that a function has been called on the VDO page cache's
  *                            thread.
  * @cache: The page cache.
- * @function_name: The funtion name to report if the assertion fails.
+ * @function_name: The function name to report if the assertion fails.
  */
 static inline void assert_on_cache_thread(struct vdo_page_cache *cache,
 					  const char *function_name)
diff --git a/drivers/md/dm-vdo/dm-vdo-target.c b/drivers/md/dm-vdo/dm-vdo-target.c
index 1d8375cc3c3ed1..043549b1fc2ee5 100644
--- a/drivers/md/dm-vdo/dm-vdo-target.c
+++ b/drivers/md/dm-vdo/dm-vdo-target.c
@@ -443,7 +443,7 @@ static int __must_check parse_slab_size(const char *slab_str, block_count_t *sla
 
 	result = kstrtoull(slab_str, 10, &value);
 	if (result) {
-		vdo_log_error("optional parameter error: invalid slab size, must be a postive integer");
+		vdo_log_error("optional parameter error: invalid slab size, must be a positive integer");
 		return -EINVAL;
 	}
 
diff --git a/drivers/md/dm-vdo/indexer/chapter-index.c b/drivers/md/dm-vdo/indexer/chapter-index.c
index bb3b0ab5d50dc5..9fb9e59238d4cb 100644
--- a/drivers/md/dm-vdo/indexer/chapter-index.c
+++ b/drivers/md/dm-vdo/indexer/chapter-index.c
@@ -284,10 +284,17 @@ int uds_search_chapter_index_page(struct delta_index_page *index_page,
 	if (result != UDS_SUCCESS)
 		return result;
 
-	if (was_entry_found(&entry, address))
-		*record_page_ptr = uds_get_delta_entry_value(&entry);
-	else
+	if (!was_entry_found(&entry, address)) {
 		*record_page_ptr = NO_CHAPTER_INDEX_ENTRY;
+		return UDS_SUCCESS;
+	}
 
-	return UDS_SUCCESS;
+	*record_page_ptr = uds_get_delta_entry_value(&entry);
+	result = VDO_ASSERT(*record_page_ptr < geometry->record_pages_per_chapter,
+			    "0 <= %d < %u", *record_page_ptr,
+			    geometry->record_pages_per_chapter);
+	if (result != VDO_SUCCESS)
+		*record_page_ptr = NO_CHAPTER_INDEX_ENTRY;
+
+	return result;
 }
diff --git a/drivers/md/dm-vdo/indexer/delta-index.c b/drivers/md/dm-vdo/indexer/delta-index.c
index b288749067dee8..257612b81a9d9f 100644
--- a/drivers/md/dm-vdo/indexer/delta-index.c
+++ b/drivers/md/dm-vdo/indexer/delta-index.c
@@ -800,7 +800,7 @@ static void compute_new_list_offsets(struct delta_zone *delta_zone, u32 growing_
 		(delta_zone->size * BITS_PER_BYTE - delta_lists[tail_guard_index].size);
 }
 
-static void rebalance_lists(struct delta_zone *delta_zone)
+static int rebalance_lists(struct delta_zone *delta_zone)
 {
 	struct delta_list *delta_lists;
 	u32 i;
@@ -811,9 +811,17 @@ static void rebalance_lists(struct delta_zone *delta_zone)
 	for (i = 0; i <= delta_zone->list_count + 1; i++)
 		used_space += get_delta_list_byte_size(&delta_lists[i]);
 
+	if (delta_zone->size < used_space) {
+		return vdo_log_warning_strerror(UDS_CORRUPT_DATA,
+						"delta zone lists overflow zone size %zu",
+						delta_zone->size);
+	}
+
 	compute_new_list_offsets(delta_zone, 0, 0, used_space);
 	for (i = 1; i <= delta_zone->list_count + 1; i++)
 		delta_lists[i].start = delta_zone->new_offsets[i];
+
+	return UDS_SUCCESS;
 }
 
 /* Start restoring a delta index from multiple input streams. */
@@ -941,8 +949,11 @@ int uds_start_restoring_delta_index(struct delta_index *delta_index,
 	}
 
 	/* Prepare each zone to start receiving the delta list data. */
-	for (z = 0; z < delta_index->zone_count; z++)
-		rebalance_lists(&delta_index->delta_zones[z]);
+	for (z = 0; z < delta_index->zone_count; z++) {
+		result = rebalance_lists(&delta_index->delta_zones[z]);
+		if (result != UDS_SUCCESS)
+			return result;
+	}
 
 	return UDS_SUCCESS;
 }
diff --git a/drivers/md/dm-vdo/indexer/volume.c b/drivers/md/dm-vdo/indexer/volume.c
index 78c3729f1f65ee..38e51fa1cbd057 100644
--- a/drivers/md/dm-vdo/indexer/volume.c
+++ b/drivers/md/dm-vdo/indexer/volume.c
@@ -891,12 +891,6 @@ int uds_search_cached_record_page(struct volume *volume, struct uds_request *req
 	if (record_page_number == NO_CHAPTER_INDEX_ENTRY)
 		return UDS_SUCCESS;
 
-	result = VDO_ASSERT(record_page_number < geometry->record_pages_per_chapter,
-			    "0 <= %d < %u", record_page_number,
-			    geometry->record_pages_per_chapter);
-	if (result != VDO_SUCCESS)
-		return result;
-
 	page_number = geometry->index_pages_per_chapter + record_page_number;
 
 	physical_page = map_to_physical_page(&volume->geometry, chapter, page_number);
diff --git a/drivers/md/dm-verity-verify-sig.c b/drivers/md/dm-verity-verify-sig.c
index b2b55c41e2cb55..aadcf5e4a47c83 100644
--- a/drivers/md/dm-verity-verify-sig.c
+++ b/drivers/md/dm-verity-verify-sig.c
@@ -21,8 +21,8 @@ static bool dm_verity_keyring_unsealed __ro_after_init;
 module_param_named(keyring_unsealed, dm_verity_keyring_unsealed, bool, 0444);
 MODULE_PARM_DESC(keyring_unsealed, "Leave the dm-verity keyring unsealed");
 
-static bool require_signatures;
-module_param(require_signatures, bool, 0444);
+static bool require_signatures = IS_ENABLED(CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG_FORCE);
+module_param(require_signatures, bool_enable_only, 0444);
 MODULE_PARM_DESC(require_signatures,
 		"Verify the roothash of dm-verity hash tree");
 
diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index 978646f71f996b..68e3fb300fc143 100644
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -81,15 +81,9 @@ struct wc_memory_superblock {
 struct wc_entry {
 	struct rb_node rb_node;
 	struct list_head lru;
+	u32 age; // jiffies
 	unsigned short wc_list_contiguous;
-#if BITS_PER_LONG == 64
-	bool write_in_progress : 1;
-	unsigned long index : 47;
-#else
 	bool write_in_progress;
-	unsigned long index;
-#endif
-	unsigned long age;
 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
 	uint64_t original_sector;
 	uint64_t seq_count;
@@ -389,20 +383,28 @@ static struct wc_memory_superblock *sb(struct dm_writecache *wc)
 	return wc->memory_map;
 }
 
+static inline unsigned long wc_entry_index(const struct wc_entry *base, const struct wc_entry *entry)
+{
+	return entry - base;
+}
+
 static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
 {
-	return &sb(wc)->entries[e->index];
+	const unsigned long index = wc_entry_index(wc->entries, e);
+	return &sb(wc)->entries[index];
 }
 
 static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
 {
-	return (char *)wc->block_start + (e->index << wc->block_size_bits);
+	const unsigned long index = wc_entry_index(wc->entries, e);
+	return (char *)wc->block_start + (index << wc->block_size_bits);
 }
 
 static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
 {
+	const unsigned long index = wc_entry_index(wc->entries, e);
 	return wc->start_sector + wc->metadata_sectors +
-		((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));
+		((sector_t)index << (wc->block_size_bits - SECTOR_SHIFT));
 }
 
 static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
@@ -668,7 +670,7 @@ static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *i
 	rb_link_node(&ins->rb_node, parent, node);
 	rb_insert_color(&ins->rb_node, &wc->tree);
 	list_add(&ins->lru, &wc->lru);
-	ins->age = jiffies;
+	ins->age = (u32)jiffies;
 }
 
 static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
@@ -969,7 +971,6 @@ static int writecache_alloc_entries(struct dm_writecache *wc)
 	for (b = 0; b < wc->n_blocks; b++) {
 		struct wc_entry *e = &wc->entries[b];
 
-		e->index = b;
 		e->write_in_progress = false;
 		cond_resched();
 	}
@@ -1986,7 +1987,7 @@ static void writecache_writeback(struct work_struct *work)
 	while (!list_empty(&wc->lru) &&
 	       (wc->writeback_all ||
 		wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark ||
-		(jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age >=
+		((u32)(jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age) >=
 		 wc->max_age - wc->max_age / MAX_AGE_DIV))) {
 
 		n_walked++;
@@ -2107,7 +2108,6 @@ static int calculate_memory_size(uint64_t device_size, unsigned int block_size,
 				 size_t *n_blocks_p, size_t *n_metadata_blocks_p)
 {
 	uint64_t n_blocks, offset;
-	struct wc_entry e;
 
 	n_blocks = device_size;
 	do_div(n_blocks, block_size + sizeof(struct wc_memory_entry));
@@ -2126,11 +2126,6 @@ static int calculate_memory_size(uint64_t device_size, unsigned int block_size,
 		n_blocks--;
 	}
 
-	/* check if the bit field overflows */
-	e.index = n_blocks;
-	if (e.index != n_blocks)
-		return -EFBIG;
-
 	if (n_blocks_p)
 		*n_blocks_p = n_blocks;
 	if (n_metadata_blocks_p)
@@ -2444,13 +2439,15 @@ static int writecache_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 			wc->autocommit_time_set = true;
 		} else if (!strcasecmp(string, "max_age") && opt_params >= 1) {
 			unsigned int max_age_msecs;
+			unsigned long max_age_jiffies;
 
 			string = dm_shift_arg(&as), opt_params--;
 			if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1)
 				goto invalid_optional;
-			if (max_age_msecs > 86400000)
+			max_age_jiffies = msecs_to_jiffies(max_age_msecs);
+			if (max_age_jiffies >= min(MAX_JIFFY_OFFSET, (7 << 28))) // 7/8ths of 1 << 31
 				goto invalid_optional;
-			wc->max_age = msecs_to_jiffies(max_age_msecs);
+			wc->max_age = max_age_jiffies;
 			wc->max_age_set = true;
 			wc->max_age_value = max_age_msecs;
 		} else if (!strcasecmp(string, "cleaner")) {
diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c
index f29acf64429a75..35916c6e490e4e 100644
--- a/drivers/md/dm-zone.c
+++ b/drivers/md/dm-zone.c
@@ -447,10 +447,8 @@ void dm_finalize_zone_settings(struct dm_table *t, struct queue_limits *lim)
 	struct mapped_device *md = t->md;
 
 	if (lim->features & BLK_FEAT_ZONED) {
-		if (dm_table_supports_zone_append(t))
-			clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
-		else
-			set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
+		assign_bit(DMF_EMULATE_ZONE_APPEND, &md->flags,
+			   !dm_table_supports_zone_append(t));
 	} else {
 		clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
 		md->disk->nr_zones = 0;
diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c
index b8325cb09a371f..8adf6ddca3d605 100644
--- a/drivers/md/md-bitmap.c
+++ b/drivers/md/md-bitmap.c
@@ -516,7 +516,8 @@ static void end_bitmap_write(struct bio *bio)
 
 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
 {
-	struct buffer_head *bh = page_buffers(page);
+	struct folio *folio = page_folio(page);
+	struct buffer_head *bh = folio_buffers(folio);
 
 	while (bh && bh->b_blocknr) {
 		atomic_inc(&bitmap->pending_writes);
@@ -533,19 +534,15 @@ static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
 
 static void free_buffers(struct page *page)
 {
-	struct buffer_head *bh;
-
-	if (!PagePrivate(page))
-		return;
+	struct folio *folio = page_folio(page);
+	struct buffer_head *bh = folio_detach_private(folio);
 
-	bh = page_buffers(page);
 	while (bh) {
 		struct buffer_head *next = bh->b_this_page;
 		free_buffer_head(bh);
 		bh = next;
 	}
-	detach_page_private(page);
-	put_page(page);
+	folio_put(folio);
 }
 
 /* read a page from a file.
@@ -560,6 +557,7 @@ static int read_file_page(struct file *file, unsigned long index,
 {
 	int ret = 0;
 	struct inode *inode = file_inode(file);
+	struct folio *folio = page_folio(page);
 	struct buffer_head *bh;
 	sector_t block, blk_cur;
 	unsigned long blocksize = i_blocksize(inode);
@@ -567,12 +565,12 @@ static int read_file_page(struct file *file, unsigned long index,
 	pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
 		 (unsigned long long)index << PAGE_SHIFT);
 
-	bh = alloc_page_buffers(page, blocksize);
+	bh = folio_alloc_buffers(folio, blocksize, GFP_NOFS | __GFP_ACCOUNT);
 	if (!bh) {
 		ret = -ENOMEM;
 		goto out;
 	}
-	attach_page_private(page, bh);
+	folio_attach_private(folio, bh);
 	blk_cur = index << (PAGE_SHIFT - inode->i_blkbits);
 	while (bh) {
 		block = blk_cur;
diff --git a/drivers/md/persistent-data/dm-bitset.c b/drivers/md/persistent-data/dm-bitset.c
index 00c0a3f186b718..6680f05c25ec86 100644
--- a/drivers/md/persistent-data/dm-bitset.c
+++ b/drivers/md/persistent-data/dm-bitset.c
@@ -59,10 +59,7 @@ static int pack_bits(uint32_t index, void *value, void *context)
 		if (r)
 			return r;
 
-		if (bv)
-			set_bit(bit, (unsigned long *) &word);
-		else
-			clear_bit(bit, (unsigned long *) &word);
+		assign_bit(bit, (unsigned long *)&word, bv);
 	}
 
 	*((__le64 *) value) = cpu_to_le64(word);
diff --git a/drivers/md/persistent-data/dm-btree-remove.c b/drivers/md/persistent-data/dm-btree-remove.c
index aeec5b9a1dd5c5..2357ff9b53f3ae 100644
--- a/drivers/md/persistent-data/dm-btree-remove.c
+++ b/drivers/md/persistent-data/dm-btree-remove.c
@@ -31,7 +31,7 @@
  * least MIN_ENTRIES + 1.  We do this in the following ways:
  *
  * [A] No siblings => this can only happen if the node is the root, in which
- *     case we copy the childs contents over the root.
+ *     case we copy the children contents over the root.
  *
  * [B] No left sibling
  *     ==> rebalance(node, right sibling)
diff --git a/drivers/media/cec/platform/meson/ao-cec-g12a.c b/drivers/media/cec/platform/meson/ao-cec-g12a.c
index b175be3f2bf4de..55bd0d416460f8 100644
--- a/drivers/media/cec/platform/meson/ao-cec-g12a.c
+++ b/drivers/media/cec/platform/meson/ao-cec-g12a.c
@@ -334,7 +334,7 @@ static int meson_ao_cec_g12a_setup_clk(struct meson_ao_cec_g12a_device *ao_cec)
 {
 	struct meson_ao_cec_g12a_dualdiv_clk *dualdiv_clk;
 	struct device *dev = &ao_cec->pdev->dev;
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	const char *parent_name;
 	struct clk *clk;
 	char *name;
diff --git a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
index 3c6ce6f3d93e72..1f8ed7ff31e523 100644
--- a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
+++ b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c
@@ -847,7 +847,9 @@ static irqreturn_t extron_interrupt(struct serio *serio, unsigned char data,
 			return IRQ_HANDLED;
 		memcpy(extron->data, extron->buf, extron->idx);
 		extron->len = extron->idx;
-		extron->data[extron->len] = 0;
+		/* Keep fixed-offset response tests from using stale tail bytes. */
+		memset(extron->data + extron->len, 0,
+		       min_t(size_t, 10, sizeof(extron->data) - extron->len));
 		if (debug)
 			dev_info(extron->dev, "received %s\n", extron->data);
 		extron->idx = 0;
diff --git a/drivers/media/common/cypress_firmware.c b/drivers/media/common/cypress_firmware.c
index 66274fdf524381..d0f66ed01c4b1e 100644
--- a/drivers/media/common/cypress_firmware.c
+++ b/drivers/media/common/cypress_firmware.c
@@ -59,6 +59,8 @@ static int cypress_get_hexline(const struct firmware *fw,
 	if (hx->type == 0x04) {
 		/* b[4] and b[5] are the Extended linear address record data
 		 * field */
+		if (hx->len != 2)
+			return -EINVAL;
 		hx->addr |= (b[4] << 24) | (b[5] << 16);
 	}
 
diff --git a/drivers/media/common/saa7146/saa7146_core.c b/drivers/media/common/saa7146/saa7146_core.c
index c297d019f2b3f8..b705c4bcaffa91 100644
--- a/drivers/media/common/saa7146/saa7146_core.c
+++ b/drivers/media/common/saa7146/saa7146_core.c
@@ -252,6 +252,15 @@ int saa7146_pgtable_build_single(struct pci_dev *pci, struct saa7146_pgtable *pt
 
 	ptr = pt->cpu;
 	for_each_sg_dma_page(list, &dma_iter, sglen, 0) {
+		/*
+		 * The page table is exactly PAGE_SIZE large, i.e. it holds
+		 * PAGE_SIZE / sizeof(__le32) entries. Buffers needing more
+		 * pages would overflow it.
+		 */
+		if (nr_pages >= PAGE_SIZE / sizeof(__le32)) {
+			pr_err("page table too small\n");
+			return -EIO;
+		}
 		*ptr++ = cpu_to_le32(sg_page_iter_dma_address(&dma_iter));
 		nr_pages++;
 	}
@@ -340,6 +349,8 @@ static int saa7146_init_one(struct pci_dev *pci, const struct pci_device_id *ent
 		goto out;
 	}
 
+	spin_lock_init(&dev->int_slock);
+
 	/* create a nice device name */
 	sprintf(dev->name, "saa7146 (%d)", saa7146_num);
 
@@ -425,7 +436,6 @@ static int saa7146_init_one(struct pci_dev *pci, const struct pci_device_id *ent
 	dev->ext = ext;
 
 	mutex_init(&dev->v4l2_lock);
-	spin_lock_init(&dev->int_slock);
 	spin_lock_init(&dev->slock);
 
 	mutex_init(&dev->i2c_lock);
diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/media/common/saa7146/saa7146_video.c
index 733e18001d0d41..c895c90b137855 100644
--- a/drivers/media/common/saa7146/saa7146_video.c
+++ b/drivers/media/common/saa7146/saa7146_video.c
@@ -410,6 +410,17 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_forma
 		f->fmt.pix.bytesperline = calc_bpl;
 
 	f->fmt.pix.sizeimage = f->fmt.pix.bytesperline * f->fmt.pix.height;
+
+	/*
+	 * The DMA page tables hold one entry per page and are exactly
+	 * PAGE_SIZE large. Reject formats whose buffer would need more
+	 * entries than the tables can hold.
+	 */
+	if (f->fmt.pix.sizeimage > PAGE_SIZE / sizeof(__le32) * PAGE_SIZE) {
+		DEB_D("sizeimage %d too large\n", f->fmt.pix.sizeimage);
+		return -EINVAL;
+	}
+
 	DEB_D("w:%d, h:%d, bytesperline:%d, sizeimage:%d\n",
 	      f->fmt.pix.width, f->fmt.pix.height,
 	      f->fmt.pix.bytesperline, f->fmt.pix.sizeimage);
diff --git a/drivers/media/common/videobuf2/frame_vector.c b/drivers/media/common/videobuf2/frame_vector.c
index 41f289c75cbb6e..de3b303237debc 100644
--- a/drivers/media/common/videobuf2/frame_vector.c
+++ b/drivers/media/common/videobuf2/frame_vector.c
@@ -5,7 +5,6 @@
 #include <linux/mm.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
-#include <linux/pagemap.h>
 #include <linux/sched.h>
 
 #include <media/frame_vector.h>
diff --git a/drivers/media/dvb-frontends/drxd_map_firm.h b/drivers/media/dvb-frontends/drxd_map_firm.h
index bdcc63576df101..a6585e53d87970 100644
--- a/drivers/media/dvb-frontends/drxd_map_firm.h
+++ b/drivers/media/dvb-frontends/drxd_map_firm.h
@@ -10,8 +10,8 @@
 
 /*
  * Note: originally, this file contained 12000+ lines of data
- * Probably a few lines for every firwmare assembler instruction. However,
- * only a few defines were actually used. So, removed all uneeded lines.
+ * Probably a few lines for every firmware assembler instruction. However,
+ * only a few defines were actually used. So, removed all unneeded lines.
  * If ever needed, the other lines can be easily obtained via git history.
  */
 
diff --git a/drivers/media/dvb-frontends/stv0900_core.c b/drivers/media/dvb-frontends/stv0900_core.c
index d15c55de2723ed..0ca6b6d812731a 100644
--- a/drivers/media/dvb-frontends/stv0900_core.c
+++ b/drivers/media/dvb-frontends/stv0900_core.c
@@ -1773,6 +1773,8 @@ static int stv0900_recv_slave_reply(struct dvb_frontend *fe,
 
 	if (stv0900_get_bits(intp, RX_END)) {
 		reply->msg_len = stv0900_get_bits(intp, FIFO_BYTENBR);
+		if (reply->msg_len > sizeof(reply->msg))
+			reply->msg_len = sizeof(reply->msg);
 
 		for (i = 0; i < reply->msg_len; i++)
 			reply->msg[i] = stv0900_read_reg(intp, DISRXDATA);
diff --git a/drivers/media/dvb-frontends/stv090x.c b/drivers/media/dvb-frontends/stv090x.c
index 932bbed5497aa6..63be9ee104abcc 100644
--- a/drivers/media/dvb-frontends/stv090x.c
+++ b/drivers/media/dvb-frontends/stv090x.c
@@ -3902,6 +3902,8 @@ static int stv090x_recv_slave_reply(struct dvb_frontend *fe, struct dvb_diseqc_s
 
 	if (rx_end) {
 		reply->msg_len = STV090x_GETFIELD_Px(reg, FIFO_BYTENBR_FIELD);
+		if (reply->msg_len > sizeof(reply->msg))
+			reply->msg_len = sizeof(reply->msg);
 		for (i = 0; i < reply->msg_len; i++)
 			reply->msg[i] = STV090x_READ_DEMOD(state, DISRXDATA);
 	}
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 5c52007f9cbeb7..9b28e91b8a67be 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -137,6 +137,17 @@ config VIDEO_HI847
           To compile this driver as a module, choose M here: the
           module will be called hi847.
 
+config VIDEO_HM1246
+	tristate "Himax HM1246 sensor support"
+	depends on OF
+	select V4L2_CCI_I2C
+	help
+	  This is a Video4Linux2 sensor driver for the Himax
+	  HM1246 camera.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called hm1246.
+
 config VIDEO_IMX111
 	tristate "Sony IMX111 sensor support"
 	select V4L2_CCI_I2C
@@ -396,6 +407,16 @@ config VIDEO_OG0VE1B
 	  To compile this driver as a module, choose M here: the
 	  module will be called og0ve1b.
 
+config VIDEO_OS02G10
+	tristate "OmniVision OS02G10 sensor support"
+	select V4L2_CCI_I2C
+	help
+	  This is a Video4Linux2 sensor driver for Omnivision
+	  OS02G10 camera sensor.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called os02g10.
+
 config VIDEO_OS05B10
         tristate "OmniVision OS05B10 sensor support"
         select V4L2_CCI_I2C
@@ -989,14 +1010,12 @@ endmenu
 # V4L2 I2C drivers that aren't related with Camera support
 #
 
-comment "audio, video and radio I2C drivers auto-selected by 'Autoselect ancillary drivers'"
-	depends on MEDIA_HIDE_ANCILLARY_SUBDRV
+comment "audio, video and radio I2C drivers"
 #
 # Encoder / Decoder module configuration
 #
 
 menu "Audio decoders, processors and mixers"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 config VIDEO_CS3308
 	tristate "Cirrus Logic CS3308 audio ADC"
@@ -1159,7 +1178,6 @@ config VIDEO_WM8775
 endmenu
 
 menu "RDS decoders"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 config VIDEO_SAA6588
 	tristate "SAA6588 Radio Chip RDS decoder support"
@@ -1176,7 +1194,6 @@ config VIDEO_SAA6588
 endmenu
 
 menu "Video decoders"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 config VIDEO_ADV7180
 	tristate "Analog Devices ADV7180 decoder"
@@ -1301,6 +1318,25 @@ config VIDEO_ISL7998X
 	  Support for Intersil ISL7998x analog to MIPI-CSI2 or
 	  BT.656 decoder.
 
+config VIDEO_IT6625
+	tristate "IT6625 HDMI to MIPI CSI bridge"
+	depends on VIDEO_DEV && I2C
+	depends on OF
+	select CEC_CORE
+	select HDMI
+	select MEDIA_CONTROLLER
+	select REGMAP_I2C
+	select V4L2_FWNODE
+	select VIDEO_V4L2_SUBDEV_API
+	help
+	  V4L2 subdevice driver for the ITE IT6625/IT6626 HDMI to MIPI
+	  CSI-2 bridge chips. IT6625 accepts an HDMI 2.0 input and
+	  IT6626 an HDMI 2.1 input, converting it to a MIPI CSI-2
+	  output. The driver also exposes an HDMI CEC adapter.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called it6625.
+
 config VIDEO_LT6911UXE
 	tristate "Lontium LT6911UXE decoder"
 	depends on ACPI && VIDEO_DEV && I2C
@@ -1517,7 +1553,6 @@ source "drivers/media/i2c/cx25840/Kconfig"
 endmenu
 
 menu "Video encoders"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 config VIDEO_ADV7170
 	tristate "Analog Devices ADV7170 video encoder"
@@ -1616,7 +1651,6 @@ config VIDEO_THS8200
 endmenu
 
 menu "Video improvement chips"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 config VIDEO_UPD64031A
 	tristate "NEC Electronics uPD64031A Ghost Reduction"
@@ -1645,7 +1679,6 @@ config VIDEO_UPD64083
 endmenu
 
 menu "Audio/Video compression chips"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 config VIDEO_SAA6752HS
 	tristate "Philips SAA6752HS MPEG-2 Audio/Video Encoder"
@@ -1661,7 +1694,6 @@ config VIDEO_SAA6752HS
 endmenu
 
 menu "SDR tuner chips"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 config SDR_MAX2175
 	tristate "Maxim 2175 RF to Bits tuner"
@@ -1678,7 +1710,6 @@ config SDR_MAX2175
 endmenu
 
 menu "Miscellaneous helper chips"
-	visible if !MEDIA_HIDE_ANCILLARY_SUBDRV
 
 source "drivers/media/i2c/cvs/Kconfig"
 
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index d04bd5724552e8..fd1cb25718c045 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -45,6 +45,7 @@ obj-$(CONFIG_VIDEO_GC2145) += gc2145.o
 obj-$(CONFIG_VIDEO_HI556) += hi556.o
 obj-$(CONFIG_VIDEO_HI846) += hi846.o
 obj-$(CONFIG_VIDEO_HI847) += hi847.o
+obj-$(CONFIG_VIDEO_HM1246) += hm1246.o
 obj-$(CONFIG_VIDEO_I2C) += video-i2c.o
 obj-$(CONFIG_VIDEO_IMX111) += imx111.o
 obj-$(CONFIG_VIDEO_IMX208) += imx208.o
@@ -65,6 +66,7 @@ obj-$(CONFIG_VIDEO_IMX678) += imx678.o
 obj-$(CONFIG_VIDEO_IMX471) += imx471.o
 obj-$(CONFIG_VIDEO_IR_I2C) += ir-kbd-i2c.o
 obj-$(CONFIG_VIDEO_ISL7998X) += isl7998x.o
+obj-$(CONFIG_VIDEO_IT6625) += it6625.o
 obj-$(CONFIG_VIDEO_KS0127) += ks0127.o
 obj-$(CONFIG_VIDEO_LM3560) += lm3560.o
 obj-$(CONFIG_VIDEO_LM3646) += lm3646.o
@@ -86,6 +88,7 @@ obj-$(CONFIG_VIDEO_MT9V032) += mt9v032.o
 obj-$(CONFIG_VIDEO_MT9V111) += mt9v111.o
 obj-$(CONFIG_VIDEO_OG01A1B) += og01a1b.o
 obj-$(CONFIG_VIDEO_OG0VE1B) += og0ve1b.o
+obj-$(CONFIG_VIDEO_OS02G10) += os02g10.o
 obj-$(CONFIG_VIDEO_OS05B10) += os05b10.o
 obj-$(CONFIG_VIDEO_OV01A10) += ov01a10.o
 obj-$(CONFIG_VIDEO_OV02A10) += ov02a10.o
diff --git a/drivers/media/i2c/adv7170.c b/drivers/media/i2c/adv7170.c
index 812998729207be..9a7edd6583c138 100644
--- a/drivers/media/i2c/adv7170.c
+++ b/drivers/media/i2c/adv7170.c
@@ -284,6 +284,7 @@ static int adv7170_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int adv7170_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/adv7175.c b/drivers/media/i2c/adv7175.c
index f1caab8e2abd95..b77737e23a4759 100644
--- a/drivers/media/i2c/adv7175.c
+++ b/drivers/media/i2c/adv7175.c
@@ -209,7 +209,7 @@ static int adv7175_s_std_output(struct v4l2_subdev *sd, v4l2_std_id std)
 		/* This is an attempt to convert
 		 * SECAM->PAL (typically it does not work
 		 * due to genlock: when decoder is in SECAM
-		 * and encoder in in PAL the subcarrier can
+		 * and encoder in PAL the subcarrier can
 		 * not be synchronized with horizontal
 		 * quency) */
 		adv7175_write_block(sd, init_pal, sizeof(init_pal));
@@ -322,6 +322,7 @@ static int adv7175_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int adv7175_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c
index a1c7f68225b45f..02197f81375857 100644
--- a/drivers/media/i2c/adv7180.c
+++ b/drivers/media/i2c/adv7180.c
@@ -770,6 +770,7 @@ static int adv7180_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int adv7180_set_pad_format(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_format *format)
 {
@@ -807,7 +808,7 @@ static int adv7180_init_state(struct v4l2_subdev *sd,
 		: V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
 
-	return adv7180_set_pad_format(sd, sd_state, &fmt);
+	return adv7180_set_pad_format(sd, NULL, sd_state, &fmt);
 }
 
 static int adv7180_get_mbus_config(struct v4l2_subdev *sd,
@@ -1454,6 +1455,7 @@ static int adv7180_probe(struct i2c_client *client)
 	if (state == NULL)
 		return -ENOMEM;
 
+	mutex_init(&state->mutex);
 	state->client = client;
 	state->field = V4L2_FIELD_ALTERNATE;
 	state->chip_info = i2c_get_match_data(client);
@@ -1497,7 +1499,6 @@ static int adv7180_probe(struct i2c_client *client)
 	}
 
 	state->irq = client->irq;
-	mutex_init(&state->mutex);
 	state->curr_norm = V4L2_STD_NTSC;
 
 	state->input = 0;
diff --git a/drivers/media/i2c/adv7183.c b/drivers/media/i2c/adv7183.c
index a04a1a205fe0d6..9e4cbebe6e5aac 100644
--- a/drivers/media/i2c/adv7183.c
+++ b/drivers/media/i2c/adv7183.c
@@ -420,6 +420,7 @@ static int adv7183_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int adv7183_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
@@ -598,7 +599,7 @@ static int adv7183_probe(struct i2c_client *client)
 	adv7183_s_std(sd, decoder->std);
 	fmt.format.width = 720;
 	fmt.format.height = 576;
-	adv7183_set_fmt(sd, NULL, &fmt);
+	adv7183_set_fmt(sd, NULL, NULL, &fmt);
 
 	/* initialize the hardware to the default control values */
 	ret = v4l2_ctrl_handler_setup(hdl);
diff --git a/drivers/media/i2c/adv7343.c b/drivers/media/i2c/adv7343.c
index 9b91d7073d3c8e..2bf17cd633c766 100644
--- a/drivers/media/i2c/adv7343.c
+++ b/drivers/media/i2c/adv7343.c
@@ -1,18 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * adv7343 - ADV7343 Video Encoder Driver
  *
  * The encoder hardware does not support SECAM.
  *
  * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed .as is. WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/media/i2c/adv7393.c b/drivers/media/i2c/adv7393.c
index 6f948ba02f8646..b00783c141a98d 100644
--- a/drivers/media/i2c/adv7393.c
+++ b/drivers/media/i2c/adv7393.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * adv7393 - ADV7393 Video Encoder Driver
  *
@@ -9,15 +10,6 @@
  * Based on ADV7343 driver,
  *
  * Copyright (C) 2009 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed .as is. WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/media/i2c/adv748x/adv748x-afe.c b/drivers/media/i2c/adv748x/adv748x-afe.c
index 678199196b84f4..28b755df30cf81 100644
--- a/drivers/media/i2c/adv748x/adv748x-afe.c
+++ b/drivers/media/i2c/adv748x/adv748x-afe.c
@@ -349,6 +349,7 @@ static int adv748x_afe_get_format(struct v4l2_subdev *sd,
 }
 
 static int adv748x_afe_set_format(struct v4l2_subdev *sd,
+				      const struct v4l2_subdev_client_info *ci,
 				      struct v4l2_subdev_state *sd_state,
 				      struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/media/i2c/adv748x/adv748x-core.c b/drivers/media/i2c/adv748x/adv748x-core.c
index 3eb6d5e8f08265..70594d0018048f 100644
--- a/drivers/media/i2c/adv748x/adv748x-core.c
+++ b/drivers/media/i2c/adv748x/adv748x-core.c
@@ -675,9 +675,6 @@ static int adv748x_parse_dt(struct adv748x_state *state)
 			continue;
 		}
 
-		of_node_get(ep_np);
-		state->endpoints[ep.port] = ep_np;
-
 		/*
 		 * At least one input endpoint and one output endpoint shall
 		 * be defined.
@@ -689,8 +686,13 @@ static int adv748x_parse_dt(struct adv748x_state *state)
 
 		/* Store number of CSI-2 lanes used for TXA and TXB. */
 		ret = adv748x_parse_csi2_lanes(state, ep.port, ep_np);
-		if (ret)
+		if (ret) {
+			of_node_put(ep_np);
 			return ret;
+		}
+
+		of_node_get(ep_np);
+		state->endpoints[ep.port] = ep_np;
 	}
 
 	return in_found && out_found ? 0 : -ENODEV;
@@ -739,7 +741,7 @@ static int adv748x_probe(struct i2c_client *client)
 	ret = adv748x_parse_dt(state);
 	if (ret) {
 		adv_err(state, "Failed to parse device tree");
-		goto err_free_mutex;
+		goto err_cleanup_dt;
 	}
 
 	/* Configure IO Regmap region */
@@ -809,7 +811,6 @@ static int adv748x_probe(struct i2c_client *client)
 	adv748x_unregister_clients(state);
 err_cleanup_dt:
 	adv748x_dt_cleanup(state);
-err_free_mutex:
 	mutex_destroy(&state->mutex);
 
 	return ret;
diff --git a/drivers/media/i2c/adv748x/adv748x-csi2.c b/drivers/media/i2c/adv748x/adv748x-csi2.c
index ebe7da8ebed773..1d7ab685d2294f 100644
--- a/drivers/media/i2c/adv748x/adv748x-csi2.c
+++ b/drivers/media/i2c/adv748x/adv748x-csi2.c
@@ -226,6 +226,7 @@ static bool adv748x_csi2_is_fmt_supported(struct adv748x_csi2 *tx, u32 code)
 }
 
 static int adv748x_csi2_set_format(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/media/i2c/adv748x/adv748x-hdmi.c b/drivers/media/i2c/adv748x/adv748x-hdmi.c
index b154dea29ba2d4..8d5ebb36996106 100644
--- a/drivers/media/i2c/adv748x/adv748x-hdmi.c
+++ b/drivers/media/i2c/adv748x/adv748x-hdmi.c
@@ -440,6 +440,7 @@ static int adv748x_hdmi_get_format(struct v4l2_subdev *sd,
 }
 
 static int adv748x_hdmi_set_format(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/media/i2c/adv7511-v4l2.c b/drivers/media/i2c/adv7511-v4l2.c
index 860cff50c522d0..27e0e2c31dbd1d 100644
--- a/drivers/media/i2c/adv7511-v4l2.c
+++ b/drivers/media/i2c/adv7511-v4l2.c
@@ -1273,6 +1273,7 @@ static int adv7511_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int adv7511_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c
index ac9c69ce438f1d..2de743d8ca2171 100644
--- a/drivers/media/i2c/adv7604.c
+++ b/drivers/media/i2c/adv7604.c
@@ -1943,6 +1943,7 @@ static int adv76xx_get_format(struct v4l2_subdev *sd,
 }
 
 static int adv76xx_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -1963,6 +1964,7 @@ static int adv76xx_get_selection(struct v4l2_subdev *sd,
 }
 
 static int adv76xx_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *format)
 {
@@ -2449,7 +2451,7 @@ static int adv76xx_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid)
 	cec_s_phys_addr(state->cec_adap, parent_pa, false);
 
 	/* enable hotplug after 143 ms */
-	schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 7);
+	schedule_delayed_work(&state->delayed_work_enable_hotplug, V4L2_SET_EDID_HPD_LOW_JIFFIES);
 	return 0;
 }
 
@@ -2641,8 +2643,9 @@ static int adv76xx_log_status(struct v4l2_subdev *sd)
 					"(16-235)" : "(0-255)",
 				(reg_io_0x02 & 0x08) ? "enabled" : "disabled");
 	}
+	ret = cp_read(sd, info->cp_csc) >> 4;
 	v4l2_info(sd, "Color space conversion: %s\n",
-			csc_coeff_sel_rb[cp_read(sd, info->cp_csc) >> 4]);
+			ret < 0 ? "" : csc_coeff_sel_rb[ret]);
 
 	if (!is_digital_input(sd))
 		return 0;
diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c
index 3cfae89ce944a2..df6601d6d2b664 100644
--- a/drivers/media/i2c/adv7842.c
+++ b/drivers/media/i2c/adv7842.c
@@ -747,8 +747,9 @@ static int edid_write_vga_segment(struct v4l2_subdev *sd)
 		return -EIO;
 	}
 
-	/* enable hotplug after 200 ms */
-	schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 5);
+	/* enable hotplug after 143 ms */
+	schedule_delayed_work(&state->delayed_work_enable_hotplug,
+			      V4L2_SET_EDID_HPD_LOW_JIFFIES);
 
 	return 0;
 }
@@ -830,8 +831,9 @@ static int edid_write_hdmi_segment(struct v4l2_subdev *sd, u8 port)
 	}
 	cec_s_phys_addr(state->cec_adap, parent_pa, false);
 
-	/* enable hotplug after 200 ms */
-	schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 5);
+	/* enable hotplug after 143 ms */
+	schedule_delayed_work(&state->delayed_work_enable_hotplug,
+			      V4L2_SET_EDID_HPD_LOW_JIFFIES);
 
 	return 0;
 }
@@ -2110,6 +2112,7 @@ static int adv7842_get_format(struct v4l2_subdev *sd,
 }
 
 static int adv7842_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ak881x.c b/drivers/media/i2c/ak881x.c
index cea46f01997da1..3449d95ee8a0dc 100644
--- a/drivers/media/i2c/ak881x.c
+++ b/drivers/media/i2c/ak881x.c
@@ -122,6 +122,7 @@ static int ak881x_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int ak881x_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -216,7 +217,6 @@ static const struct v4l2_subdev_video_ops ak881x_subdev_video_ops = {
 static const struct v4l2_subdev_pad_ops ak881x_subdev_pad_ops = {
 	.enum_mbus_code = ak881x_enum_mbus_code,
 	.get_selection	= ak881x_get_selection,
-	.set_fmt	= ak881x_fill_fmt,
 	.get_fmt	= ak881x_fill_fmt,
 };
 
diff --git a/drivers/media/i2c/alvium-csi2.c b/drivers/media/i2c/alvium-csi2.c
index f51f9b98775922..d9a73565457f31 100644
--- a/drivers/media/i2c/alvium-csi2.c
+++ b/drivers/media/i2c/alvium-csi2.c
@@ -1887,6 +1887,7 @@ static int alvium_init_state(struct v4l2_subdev *sd,
 }
 
 static int alvium_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
@@ -1922,6 +1923,7 @@ static int alvium_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int alvium_set_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1957,6 +1959,7 @@ static int alvium_set_selection(struct v4l2_subdev *sd,
 }
 
 static int alvium_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -2523,6 +2526,7 @@ static void alvium_remove(struct i2c_client *client)
 	 * make sure to turn power off manually.
 	 */
 	pm_runtime_disable(dev);
+	pm_runtime_put_noidle(dev);
 	if (!pm_runtime_status_suspended(dev))
 		alvium_set_power(alvium, false);
 	pm_runtime_set_suspended(dev);
diff --git a/drivers/media/i2c/ar0521.c b/drivers/media/i2c/ar0521.c
index ed324c2d87aa25..3ddbf10ef8a4e5 100644
--- a/drivers/media/i2c/ar0521.c
+++ b/drivers/media/i2c/ar0521.c
@@ -457,6 +457,7 @@ static int ar0521_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int ar0521_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
@@ -1142,7 +1143,6 @@ static int ar0521_probe(struct i2c_client *client)
 
 disable:
 	v4l2_async_unregister_subdev(&sensor->sd);
-	media_entity_cleanup(&sensor->sd.entity);
 free_ctrls:
 	v4l2_ctrl_handler_free(&sensor->ctrls.handler);
 entity_cleanup:
diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c
index 8e25f970fd12c3..1cc4350856fdf9 100644
--- a/drivers/media/i2c/ccs/ccs-core.c
+++ b/drivers/media/i2c/ccs/ccs-core.c
@@ -2189,6 +2189,7 @@ static void ccs_propagate(struct v4l2_subdev *subdev,
 }
 
 static int ccs_set_format_source(struct v4l2_subdev *subdev,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -2242,6 +2243,7 @@ static int ccs_set_format_source(struct v4l2_subdev *subdev,
 }
 
 static int ccs_set_format(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -2252,7 +2254,7 @@ static int ccs_set_format(struct v4l2_subdev *subdev,
 	if (fmt->pad == ssd->source_pad) {
 		int rval;
 
-		rval = ccs_set_format_source(subdev, sd_state, fmt);
+		rval = ccs_set_format_source(subdev, ci, sd_state, fmt);
 
 		return rval;
 	}
@@ -2467,6 +2469,7 @@ static void ccs_set_compose_scaler(struct v4l2_subdev *subdev,
 }
 /* We're only called on source pads. This function sets scaling. */
 static int ccs_set_compose(struct v4l2_subdev *subdev,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_selection *sel)
 {
@@ -2536,6 +2539,7 @@ static int ccs_sel_supported(struct v4l2_subdev *subdev,
 }
 
 static int ccs_set_crop(struct v4l2_subdev *subdev,
+			const struct v4l2_subdev_client_info *ci,
 			struct v4l2_subdev_state *sd_state,
 			struct v4l2_subdev_selection *sel)
 {
@@ -2587,6 +2591,7 @@ static void ccs_get_native_size(struct ccs_subdev *ssd, struct v4l2_rect *r)
 }
 
 static int ccs_get_selection(struct v4l2_subdev *subdev,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
@@ -2633,6 +2638,7 @@ static int ccs_get_selection(struct v4l2_subdev *subdev,
 }
 
 static int ccs_set_selection(struct v4l2_subdev *subdev,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
@@ -2655,10 +2661,10 @@ static int ccs_set_selection(struct v4l2_subdev *subdev,
 
 	switch (sel->target) {
 	case V4L2_SEL_TGT_CROP:
-		ret = ccs_set_crop(subdev, sd_state, sel);
+		ret = ccs_set_crop(subdev, ci, sd_state, sel);
 		break;
 	case V4L2_SEL_TGT_COMPOSE:
-		ret = ccs_set_compose(subdev, sd_state, sel);
+		ret = ccs_set_compose(subdev, ci, sd_state, sel);
 		break;
 	default:
 		ret = -EINVAL;
diff --git a/drivers/media/i2c/ccs/ccs-data.h b/drivers/media/i2c/ccs/ccs-data.h
index 638df69804ec19..522466b8192902 100644
--- a/drivers/media/i2c/ccs/ccs-data.h
+++ b/drivers/media/i2c/ccs/ccs-data.h
@@ -160,7 +160,7 @@ struct ccs_pdaf_pix_loc_pixel_desc_group {
  * @main_offset_y: Start Y coordinate of PDAF pixel blocks
  * @global_pdaf_type: PDAF pattern type
  * @block_width: Width of a block in pixels
- * @block_height: Heigth of a block in pixels
+ * @block_height: Height of a block in pixels
  * @num_block_desc_groups: Number of block descriptor groups
  * @block_desc_groups: Block descriptor groups
  * @num_pixel_desc_grups: Number of pixel descriptor groups
@@ -197,7 +197,7 @@ struct ccs_pdaf_pix_loc {
  * @module_rules: Rules for the module
  * @sensor_pdaf: PDAF data for the sensor
  * @module_pdaf: PDAF data for the module
- * @license_length: Lenght of the license data
+ * @license_length: Length of the license data
  * @license: License data
  * @end: Whether or not there's an end block
  * @backing: Raw data, pointed to from elsewhere so keep it around
diff --git a/drivers/media/i2c/cvs/Kconfig b/drivers/media/i2c/cvs/Kconfig
index 4309d20dd72632..f482f1ccc38a8a 100644
--- a/drivers/media/i2c/cvs/Kconfig
+++ b/drivers/media/i2c/cvs/Kconfig
@@ -4,6 +4,7 @@ config VIDEO_INTEL_CVS
 	tristate "Intel CVS CSI-2 bridge support"
 	depends on I2C && ACPI && VIDEO_DEV
 	depends on IPU_BRIDGE || !IPU_BRIDGE
+	default VIDEO_INTEL_IPU6
 	select MEDIA_CONTROLLER
 	select VIDEO_V4L2_SUBDEV_API
 	select V4L2_FWNODE
diff --git a/drivers/media/i2c/cvs/core.c b/drivers/media/i2c/cvs/core.c
index d4a3b9c3bab1e5..d9c51e1c0fb5df 100644
--- a/drivers/media/i2c/cvs/core.c
+++ b/drivers/media/i2c/cvs/core.c
@@ -21,7 +21,6 @@
 #include <linux/workqueue.h>
 
 #include <media/ipu-bridge.h>
-#include <media/ipu6-pci-table.h>
 
 #include "icvs.h"
 
@@ -70,6 +69,13 @@ static const struct icvs_device_quirk cvs_quirk_table[] = {
 			  ICVS_NO_CAPS |
 			  ICVS_NO_FW_UPDATE
 	},	/* Lattice NX33 */
+	{ 0x2ac1, 0x20d1, ICVS_NO_MIPI_CONFIG |
+			  ICVS_NO_CAPS |
+			  ICVS_NO_FW_UPDATE
+	},	/*
+		 * The Lattice device was found on a Dell laptop
+		 * XPS 14 (Dell 14 Premium) DA14250
+		 */
 	{ 0x06CB, 0x0701, ICVS_SKIP_FW_RESET |
 			  ICVS_HOST_SENSOR_PWR_CTRL |
 			  ICVS_HOST_PRIV_CTRL |
@@ -656,14 +662,12 @@ static int cvs_configure_dev_caps(struct icvs *ctx)
  */
 static int cvs_core_probe(struct device *dev, struct i2c_client *i2c)
 {
-	struct pci_dev *ipu = NULL;
+	struct pci_dev *ipu;
 	struct icvs *ctx;
 	int ret;
 
 	/* Locate IPU device */
-	for (unsigned int i = 0; !ipu && ipu6_pci_tbl[i].vendor; i++)
-		ipu = pci_get_device(ipu6_pci_tbl[i].vendor,
-				     ipu6_pci_tbl[i].device, NULL);
+	ipu = ipu_bridge_get_ipu6();
 	for (unsigned int i = 0; !ipu && icvs_ipu7_tbl[i].vendor; i++)
 		ipu = pci_get_device(icvs_ipu7_tbl[i].vendor,
 				     icvs_ipu7_tbl[i].device, NULL);
@@ -725,8 +729,6 @@ static int cvs_core_probe(struct device *dev, struct i2c_client *i2c)
 	}
 
 	if (ctx->res == ICVS_FULLCAP) {
-		struct gpio_desc *wake;
-
 		ctx->rst = devm_gpiod_get(dev, "rst", GPIOD_OUT_HIGH);
 		if (IS_ERR(ctx->rst)) {
 			ret = dev_err_probe(dev, PTR_ERR(ctx->rst),
@@ -734,14 +736,12 @@ static int cvs_core_probe(struct device *dev, struct i2c_client *i2c)
 			goto err_put_ipu;
 		}
 
-		wake = devm_gpiod_get(dev, "wake", GPIOD_IN);
-		if (IS_ERR(wake)) {
-			ret = dev_err_probe(dev, PTR_ERR(wake),
-					    "failed to get wake GPIO\n");
-			goto err_put_ipu;
-		}
-
-		ctx->irq = gpiod_to_irq(wake);
+		/*
+		 * Do not request the line: another device's _CRS may list
+		 * the same pin, and its driver would then fail with -EBUSY.
+		 */
+		ctx->irq = acpi_dev_gpio_irq_get_by(ACPI_COMPANION(dev),
+						    "wake", 0);
 		if (ctx->irq < 0) {
 			ret = dev_err_probe(dev, ctx->irq,
 					    "failed to get wake IRQ\n");
diff --git a/drivers/media/i2c/cvs/v4l2.c b/drivers/media/i2c/cvs/v4l2.c
index 9fadca7a3beed2..776fe946016852 100644
--- a/drivers/media/i2c/cvs/v4l2.c
+++ b/drivers/media/i2c/cvs/v4l2.c
@@ -13,6 +13,7 @@
 #include <media/v4l2-async.h>
 #include <media/v4l2-common.h>
 #include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-fwnode.h>
 #include <media/v4l2-mc.h>
@@ -71,19 +72,6 @@ static int csi_set_link_cfg(struct icvs *ctx, u64 link_freq)
  * Streaming
  */
 
-/**
- * cvs_csi_enable_streams - Start streaming through the bridge
- * @sd: Sub-device pointer
- * @state: Active state
- * @pad: Pad identifier (must be ICVS_CSI_PAD_SOURCE)
- * @streams_mask: Streams to enable (bit 0 supported)
- *
- * Runtime-resumes the bridge (triggering cvs_runtime_resume() to claim CSI-2
- * link ownership), fetches the link frequency, programs the MIPI configuration,
- * and forwards the enable request downstream.
- *
- * Return: 0 on success or negative errno.
- */
 static int cvs_csi_enable_streams(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *state,
 				  u32 pad, u64 streams_mask)
@@ -129,19 +117,6 @@ static int cvs_csi_enable_streams(struct v4l2_subdev *sd,
 	return ret;
 }
 
-/**
- * cvs_csi_disable_streams - Stop streaming through the bridge
- * @sd: Sub-device pointer
- * @state: Active state
- * @pad: Pad identifier (must be ICVS_CSI_PAD_SOURCE)
- * @streams_mask: Streams to disable (bit 0 supported)
- *
- * Disables the remote sensor stream then drops the PM reference acquired
- * during enable. After the autosuspend delay, cvs_runtime_suspend() will
- * return CSI-2 link ownership to CVS firmware.
- *
- * Return: 0 on success or negative errno.
- */
 static int cvs_csi_disable_streams(struct v4l2_subdev *sd,
 				   struct v4l2_subdev_state *state,
 				   u32 pad, u64 streams_mask)
@@ -167,15 +142,6 @@ static int cvs_csi_disable_streams(struct v4l2_subdev *sd,
 /*
  * Pad operations / formats
  */
-/**
- * cvs_csi_init_state - Initialize pad formats in subdev state
- * @sd: Sub-device
- * @state: State container
- *
- * Sets all pad formats to a minimal 1x1 default.
- *
- * Return: 0.
- */
 static int cvs_csi_init_state(struct v4l2_subdev *sd,
 			      struct v4l2_subdev_state *state)
 {
@@ -186,18 +152,8 @@ static int cvs_csi_init_state(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * cvs_csi_set_fmt - Negotiate pad format
- * @sd: Sub-device
- * @state: State
- * @format: Desired / returned format
- *
- * Mirrors sink format onto source pad. Accepts many media bus codes, falling
- * back to Y8 if unsupported. Normalizes field setting.
- *
- * Return: 0.
- */
 static int cvs_csi_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *state,
 			   struct v4l2_subdev_format *format)
 {
@@ -319,17 +275,6 @@ static int cvs_csi_set_fmt(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * cvs_csi_get_mbus_config - Provide current CSI-2 bus configuration
- * @sd: Sub-device
- * @pad: Pad index
- * @cfg: Returned bus config
- *
- * Fills lane ordering and number of lanes; retrieves link frequency from
- * remote entity.
- *
- * Return: 0 on success or negative errno.
- */
 static int cvs_csi_get_mbus_config(struct v4l2_subdev *sd, unsigned int pad,
 				   struct v4l2_mbus_config *cfg)
 {
@@ -384,23 +329,12 @@ static const struct media_entity_operations cvs_csi_entity_ops = {
 /*
  * Async notifier
  */
-/**
- * cvs_csi_notify_bound - Remote sensor bound callback
- * @notifier: Async notifier
- * @sd: Remote subdev
- * @asc: Async match connection
- *
- * Locates the source pad of the remote sensor and creates a media link to
- * the CVS bridge sink pad enabling it by default.
- *
- * Return: 0 on success or negative errno.
- */
 static int cvs_csi_notify_bound(struct v4l2_async_notifier *notifier,
 				struct v4l2_subdev *sd,
 				struct v4l2_async_connection *asc)
 {
 	struct icvs *ctx = notifier_to_csi(notifier);
-	int pad;
+	int pad, ret;
 
 	pad = media_entity_get_fwnode_pad(&sd->entity, asc->match.fwnode,
 					  MEDIA_PAD_FL_SOURCE);
@@ -409,17 +343,15 @@ static int cvs_csi_notify_bound(struct v4l2_async_notifier *notifier,
 
 	ctx->remote = &sd->entity.pads[pad];
 
-	return media_create_pad_link(&sd->entity, pad, &ctx->subdev.entity,
-				     ICVS_CSI_PAD_SINK, MEDIA_LNK_FL_ENABLED |
-				     MEDIA_LNK_FL_IMMUTABLE);
+	ret = media_create_pad_link(&sd->entity, pad, &ctx->subdev.entity,
+				    ICVS_CSI_PAD_SINK, MEDIA_LNK_FL_ENABLED |
+				    MEDIA_LNK_FL_IMMUTABLE);
+	if (ret)
+		return ret;
+
+	return v4l2_device_register_subdev_nodes(sd->v4l2_dev);
 }
 
-/**
- * cvs_csi_notify_unbind - Remote sensor unbind callback
- * @notifier: Notifier
- * @sd: Remote subdev
- * @asc: Connection
- */
 static void cvs_csi_notify_unbind(struct v4l2_async_notifier *notifier,
 				  struct v4l2_subdev *sd,
 				  struct v4l2_async_connection *asc)
@@ -437,14 +369,6 @@ static const struct v4l2_async_notifier_operations cvs_csi_notify_ops = {
 /*
  * Controls
  */
-/**
- * cvs_csi_init_controls - Initialize V4L2 controls
- * @ctx: CVS context
- *
- * Currently sets up a read-only privacy control placeholder.
- *
- * Return: 0 on success or negative errno.
- */
 static int cvs_csi_init_controls(struct icvs *ctx)
 {
 	struct v4l2_ctrl *privacy_ctrl;
diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c
index 8b7dd43ed208fb..e90d71fe9959bc 100644
--- a/drivers/media/i2c/cx25840/cx25840-core.c
+++ b/drivers/media/i2c/cx25840/cx25840-core.c
@@ -1771,6 +1771,7 @@ static int cx25840_s_ctrl(struct v4l2_ctrl *ctrl)
 /* ----------------------------------------------------------------------- */
 
 static int cx25840_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *format)
 {
@@ -2289,29 +2290,27 @@ static int cx25840_init(struct v4l2_subdev *sd, u32 val)
 {
 	struct cx25840_state *state = to_state(sd);
 
+	if (!is_cx2584x(state))
+		return -EOPNOTSUPP;
+
 	state->generic_mode = true;
 
-	if (is_cx2584x(state)) {
-		/* set datasheet video output defaults */
-		state->vid_config = CX25840_VCONFIG_FMT_BT656 |
-				    CX25840_VCONFIG_RES_8BIT |
-				    CX25840_VCONFIG_VBIRAW_DISABLED |
-				    CX25840_VCONFIG_ANCDATA_ENABLED |
-				    CX25840_VCONFIG_TASKBIT_ONE |
-				    CX25840_VCONFIG_ACTIVE_HORIZONTAL |
-				    CX25840_VCONFIG_VALID_NORMAL |
-				    CX25840_VCONFIG_HRESETW_NORMAL |
-				    CX25840_VCONFIG_CLKGATE_NONE |
-				    CX25840_VCONFIG_DCMODE_DWORDS |
-				    CX25840_VCONFIG_IDID0S_NORMAL |
-				    CX25840_VCONFIG_VIPCLAMP_DISABLED;
-
-		/* add additional settings */
-		cx25840_vconfig_add(state, val);
-	} else {
-		/* TODO: generic mode needs to be developed for other chips */
-		WARN_ON(1);
-	}
+	/* set datasheet video output defaults */
+	state->vid_config = CX25840_VCONFIG_FMT_BT656 |
+		CX25840_VCONFIG_RES_8BIT |
+		CX25840_VCONFIG_VBIRAW_DISABLED |
+		CX25840_VCONFIG_ANCDATA_ENABLED |
+		CX25840_VCONFIG_TASKBIT_ONE |
+		CX25840_VCONFIG_ACTIVE_HORIZONTAL |
+		CX25840_VCONFIG_VALID_NORMAL |
+		CX25840_VCONFIG_HRESETW_NORMAL |
+		CX25840_VCONFIG_CLKGATE_NONE |
+		CX25840_VCONFIG_DCMODE_DWORDS |
+		CX25840_VCONFIG_IDID0S_NORMAL |
+		CX25840_VCONFIG_VIPCLAMP_DISABLED;
+
+	/* add additional settings */
+	cx25840_vconfig_add(state, val);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/ds90ub913.c b/drivers/media/i2c/ds90ub913.c
index 6abb5e324ae190..ca0b07d7109beb 100644
--- a/drivers/media/i2c/ds90ub913.c
+++ b/drivers/media/i2c/ds90ub913.c
@@ -373,6 +373,7 @@ static int ub913_set_routing(struct v4l2_subdev *sd,
 }
 
 static int ub913_set_fmt(struct v4l2_subdev *sd,
+			 const struct v4l2_subdev_client_info *ci,
 			 struct v4l2_subdev_state *state,
 			 struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ds90ub953.c b/drivers/media/i2c/ds90ub953.c
index d4228e1134ff88..e6ae9cadefa280 100644
--- a/drivers/media/i2c/ds90ub953.c
+++ b/drivers/media/i2c/ds90ub953.c
@@ -426,6 +426,7 @@ static int ub953_set_routing(struct v4l2_subdev *sd,
 
 
 static int ub953_set_fmt(struct v4l2_subdev *sd,
+			 const struct v4l2_subdev_client_info *ci,
 			 struct v4l2_subdev_state *state,
 			 struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ds90ub960.c b/drivers/media/i2c/ds90ub960.c
index 15a9797b47ac9c..7d7ba6d349fc41 100644
--- a/drivers/media/i2c/ds90ub960.c
+++ b/drivers/media/i2c/ds90ub960.c
@@ -4045,6 +4045,7 @@ static int ub960_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
 }
 
 static int ub960_set_fmt(struct v4l2_subdev *sd,
+			 const struct v4l2_subdev_client_info *ci,
 			 struct v4l2_subdev_state *state,
 			 struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/et8ek8/et8ek8_driver.c b/drivers/media/i2c/et8ek8/et8ek8_driver.c
index 738e2801016a6b..e8a7b7d87bb177 100644
--- a/drivers/media/i2c/et8ek8/et8ek8_driver.c
+++ b/drivers/media/i2c/et8ek8/et8ek8_driver.c
@@ -1013,6 +1013,7 @@ static int et8ek8_get_pad_format(struct v4l2_subdev *subdev,
 }
 
 static int et8ek8_set_pad_format(struct v4l2_subdev *subdev,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/gc0308.c b/drivers/media/i2c/gc0308.c
index 15900d5414cf24..893dfe2e1c3e13 100644
--- a/drivers/media/i2c/gc0308.c
+++ b/drivers/media/i2c/gc0308.c
@@ -1043,6 +1043,7 @@ static void gc0308_update_pad_format(const struct gc0308_frame_size *mode,
 }
 
 static int gc0308_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/gc0310.c b/drivers/media/i2c/gc0310.c
index 754e82ad50ae40..5e941291dbf7ad 100644
--- a/drivers/media/i2c/gc0310.c
+++ b/drivers/media/i2c/gc0310.c
@@ -365,6 +365,7 @@ static void gc0310_fill_format(struct v4l2_mbus_framefmt *fmt)
 }
 
 static int gc0310_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -537,7 +538,6 @@ static const struct v4l2_subdev_pad_ops gc0310_pad_ops = {
 	.enum_mbus_code = gc0310_enum_mbus_code,
 	.enum_frame_size = gc0310_enum_frame_size,
 	.get_fmt = v4l2_subdev_get_fmt,
-	.set_fmt = v4l2_subdev_get_fmt, /* Only 1 fixed mode supported */
 	.get_selection = gc0310_get_selection,
 	.set_selection = gc0310_get_selection,
 	.enable_streams = gc0310_enable_streams,
diff --git a/drivers/media/i2c/gc05a2.c b/drivers/media/i2c/gc05a2.c
index 7cf7cde1f93647..5b95d888164965 100644
--- a/drivers/media/i2c/gc05a2.c
+++ b/drivers/media/i2c/gc05a2.c
@@ -730,6 +730,7 @@ static void gc05a2_update_pad_format(struct gc05a2 *gc08a3,
 }
 
 static int gc05a2_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *fmt)
 {
@@ -761,6 +762,7 @@ static int gc05a2_set_format(struct v4l2_subdev *sd,
 }
 
 static int gc05a2_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -795,7 +797,7 @@ static int gc05a2_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	gc05a2_set_format(sd, state, &fmt);
+	gc05a2_set_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/gc08a3.c b/drivers/media/i2c/gc08a3.c
index 4144aad8f2dac9..c1ae05ed1f2ade 100644
--- a/drivers/media/i2c/gc08a3.c
+++ b/drivers/media/i2c/gc08a3.c
@@ -705,6 +705,7 @@ static void gc08a3_update_pad_format(struct gc08a3 *gc08a3,
 }
 
 static int gc08a3_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *fmt)
 {
@@ -737,6 +738,7 @@ static int gc08a3_set_format(struct v4l2_subdev *sd,
 }
 
 static int gc08a3_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -771,7 +773,7 @@ static int gc08a3_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	gc08a3_set_format(sd, state, &fmt);
+	gc08a3_set_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/gc2145.c b/drivers/media/i2c/gc2145.c
index b215963a2648b7..661a59641f1940 100644
--- a/drivers/media/i2c/gc2145.c
+++ b/drivers/media/i2c/gc2145.c
@@ -709,6 +709,7 @@ static int gc2145_init_state(struct v4l2_subdev *sd,
 }
 
 static int gc2145_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -774,6 +775,7 @@ static int gc2145_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int gc2145_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/hi556.c b/drivers/media/i2c/hi556.c
index de573cee44510e..4c13adf4a9002f 100644
--- a/drivers/media/i2c/hi556.c
+++ b/drivers/media/i2c/hi556.c
@@ -960,6 +960,7 @@ __hi556_get_pad_crop(struct hi556 *hi556,
 }
 
 static int hi556_get_selection(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_selection *sel)
 {
@@ -1071,6 +1072,7 @@ static int hi556_set_stream(struct v4l2_subdev *sd, int enable)
 }
 
 static int hi556_set_format(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/hi846.c b/drivers/media/i2c/hi846.c
index a3f77b8434ca70..025f8bcd17bc72 100644
--- a/drivers/media/i2c/hi846.c
+++ b/drivers/media/i2c/hi846.c
@@ -1688,6 +1688,7 @@ static int __maybe_unused hi846_resume(struct device *dev)
 }
 
 static int hi846_set_format(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *format)
 {
@@ -1840,6 +1841,7 @@ static int hi846_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int hi846_get_selection(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/hi847.c b/drivers/media/i2c/hi847.c
index def01aa07b2f50..41b8cac6a5e3cd 100644
--- a/drivers/media/i2c/hi847.c
+++ b/drivers/media/i2c/hi847.c
@@ -2187,7 +2187,7 @@ struct hi847 {
 	/* Current mode */
 	const struct hi847_mode *cur_mode;
 
-	/* To serialize asynchronus callbacks */
+	/* To serialize asynchronous callbacks */
 	struct mutex mutex;
 };
 
@@ -2639,6 +2639,7 @@ static int hi847_set_stream(struct v4l2_subdev *sd, int enable)
 }
 
 static int hi847_set_format(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/hm1246.c b/drivers/media/i2c/hm1246.c
new file mode 100644
index 00000000000000..be1c3d75657ebd
--- /dev/null
+++ b/drivers/media/i2c/hm1246.c
@@ -0,0 +1,1287 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Himax HM1246 image sensor
+ *
+ * Copyright 2026 Matthias Fend <matthias.fend@emfend.at>
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/limits.h>
+#include <linux/math.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/types.h>
+#include <linux/units.h>
+#include <media/media-entity.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-cci.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-subdev.h>
+
+/* Status registers */
+#define HM1246_MODEL_ID_REG		 CCI_REG16(0x0000)
+
+/* General setup registers */
+#define HM1246_MODE_SELECT_REG		 CCI_REG8(0x0100)
+#define HM1246_MODE_SELECT_STANDBY	 0x00
+#define HM1246_MODE_SELECT_STREAM	 0x01
+#define HM1246_MODE_SELECT_STOP		 0x02
+#define HM1246_IMAGE_ORIENTATION_REG	 CCI_REG8(0x0101)
+#define HM1246_IMAGE_ORIENTATION_VFLIP	 BIT(1)
+#define HM1246_IMAGE_ORIENTATION_HFLIP	 BIT(0)
+#define HM1246_CMU_UPDATE_REG		 CCI_REG8(0x0104)
+
+/* Output setup registers */
+#define HM1246_COARSE_INTG_REG		 CCI_REG16(0x0202)
+#define HM1246_ANALOG_GLOBAL_GAIN_REG	 CCI_REG8(0x0205)
+
+/* Clock setup registers */
+#define HM1246_PLL1CFG_REG		 CCI_REG8(0x0303)
+#define HM1246_PLL1CFG_MULTIPLIER(x)	 (((x) & 0xff) << 0)
+#define HM1246_PLL2CFG_REG		 CCI_REG8(0x0305)
+#define HM1246_PLL2CFG_PRE_DIV(x)	 (((x) & 0x1f) << 1)
+#define HM1246_PLL2CFG_MULTIPLIER(x)	 (((x) & 0x01) << 0)
+#define HM1246_PLL3CFG_REG		 CCI_REG8(0x0307)
+#define HM1246_PLL3CFG_POST_DIV(x)	 (((x) & 0x3) << 6)
+#define HM1246_PLL3CFG_SYSCLK_DIV(x)	 (((x) & 0x3) << 4)
+#define HM1246_PLL3CFG_PCLK_DIV(x)	 (((x) & 0x7) << 0)
+
+/* Frame timing registers */
+#define HM1246_FRAME_LENGTH_LINES_REG	 CCI_REG16(0x0340)
+#define HM1246_LINE_LENGTH_PCK_REG	 CCI_REG16(0x0342)
+
+/* Image size registers */
+#define HM1246_X_ADDR_START_REG		 CCI_REG16(0x0344)
+#define HM1246_Y_ADDR_START_REG		 CCI_REG16(0x0346)
+#define HM1246_X_ADDR_END_REG		 CCI_REG16(0x0348)
+#define HM1246_Y_ADDR_END_REG		 CCI_REG16(0x034a)
+#define HM1246_X_LA_START_REG		 CCI_REG16(0x0351)
+#define HM1246_X_LA_END_REG		 CCI_REG16(0x0353)
+#define HM1246_Y_LA_START_REG		 CCI_REG16(0x0355)
+#define HM1246_Y_LA_END_REG		 CCI_REG16(0x0357)
+
+/* Test pattern registers */
+#define HM1246_TEST_PATTERN_MODE_REG	 CCI_REG8(0x0601)
+#define HM1246_TEST_PATTERN_MODE_MODE(x) (((x) & 0xf) << 4)
+#define HM1246_TEST_PATTERN_MODE_ENABLE	 BIT(0)
+#define HM1246_TEST_DATA_BLUE_REG	 CCI_REG16(0x0602)
+#define HM1246_TEST_DATA_GB_REG		 CCI_REG16(0x0604)
+#define HM1246_TEST_DATA_RED_REG	 CCI_REG16(0x0606)
+#define HM1246_TEST_DATA_GR_REG		 CCI_REG16(0x0608)
+
+/* SBC registers */
+#define HM1246_SBC_BOOT_REF2_REG	 CCI_REG8(0x2001)
+#define HM1246_SBC_BOOT_REF2_PLL_LOCK	 BIT(4)
+#define HM1246_SBC_CTRL_REG		 CCI_REG8(0x2003)
+#define HM1246_SBC_CTRL_PLL_EN		 BIT(0)
+
+/* System registers */
+#define HM1246_OUTPUT_PRT_CTRL_REG	 CCI_REG8(0x2f02)
+#define HM1246_POLARITY_CTRL_REG	 CCI_REG8(0x2f20)
+#define HM1246_POLARITY_CTRL_HSYNC	 BIT(7)
+#define HM1246_POLARITY_CTRL_VSYNC	 BIT(6)
+#define HM1246_PCLK_CTRL_REG		 CCI_REG8(0x2f24)
+#define HM1246_PCLK_CTRL_POL		 BIT(3)
+
+/* Digital window control & parameter registers */
+#define HM1246_DWIN_XOFFSET_REG		 CCI_REG16(0xd5e4)
+#define HM1246_DWIN_XSIZE_REG		 CCI_REG16(0xd5e6)
+#define HM1246_DWIN_YOFFSET_REG		 CCI_REG16(0xd5e8)
+#define HM1246_DWIN_YSIZE_REG		 CCI_REG16(0xd5ea)
+
+#define HM1246_MODEL_ID			 0x1245
+
+#define HM1246_NATIVE_WIDTH		 1296
+#define HM1246_NATIVE_HEIGHT		 976
+
+#define HM1246_VTS_MAX			 65535
+
+#define HM1246_COARSE_INTG_MARGIN	 2
+#define HM1246_COARSE_INTG_MIN		 4
+#define HM1246_COARSE_INTG_STEP		 1
+
+#define HM1246_ANALOG_GLOBAL_GAIN_MIN	 0x00
+#define HM1246_ANALOG_GLOBAL_GAIN_MAX	 0xe8
+#define HM1246_ANALOG_GLOBAL_GAIN_STEP	 0x01
+
+#define HM1246_XCLK_MIN			 (6 * HZ_PER_MHZ)
+#define HM1246_XCLK_MAX			 (27 * HZ_PER_MHZ)
+
+#define HM1246_PCLK_MIN			 (8 * HZ_PER_MHZ)
+#define HM1246_PCLK_MAX			 (96 * HZ_PER_MHZ)
+
+#define HM1246_PLL_VCO_MIN		 (360 * HZ_PER_MHZ)
+#define HM1246_PLL_VCO_MAX		 (680 * HZ_PER_MHZ)
+
+#define HM1246_PLL_INCLK_MIN		 (1000 * HZ_PER_KHZ)
+#define HM1246_PLL_INCLK_MAX		 (2500 * HZ_PER_KHZ)
+
+#define HM1246_PLL_MULTI_L_MIN		 1
+#define HM1246_PLL_MULTI_L_MAX		 256
+
+#define HM1246_PLL_MULTI_H_MIN		 2
+#define HM1246_PLL_MULTI_H_MAX		 3
+
+#define HM1246_PLL_MULTI_MIN \
+	(HM1246_PLL_MULTI_H_MIN * HM1246_PLL_MULTI_L_MIN)
+#define HM1246_PLL_MULTI_MAX \
+	(HM1246_PLL_MULTI_H_MAX * HM1246_PLL_MULTI_L_MAX)
+
+static const char *const hm1246_test_pattern_menu[] = {
+	"Disabled",
+	"Checkerboard",
+	"Ramp",
+	"Moving ones",
+	"Blending color bars",
+	"Color bars",
+	"Solid white",
+	"Solid black",
+	"Solid red",
+	"Solid green",
+	"Solid blue",
+};
+
+static const char *const hm1246_supply_names[] = {
+	"avdd",
+	"iovdd",
+	"dvdd",
+};
+
+struct hm1246 {
+	struct v4l2_subdev sd;
+	struct media_pad pad;
+	struct device *dev;
+
+	struct regulator_bulk_data supplies[ARRAY_SIZE(hm1246_supply_names)];
+	struct clk *xclk;
+	unsigned long xclk_freq;
+	struct reset_control *reset;
+	unsigned int mbus_flags;
+	s64 link_frequency;
+
+	struct v4l2_ctrl_handler ctrls;
+	struct v4l2_ctrl *exposure_ctrl;
+	struct v4l2_ctrl *hflip_ctrl;
+	struct v4l2_ctrl *vflip_ctrl;
+
+	struct regmap *regmap;
+
+	bool identified;
+};
+
+static const struct cci_reg_sequence mode_1296x976_raw[] = {
+	{ HM1246_X_LA_START_REG, 60 },
+	{ HM1246_X_LA_END_REG, 1355 },
+	{ HM1246_Y_LA_START_REG, 0 },
+	{ HM1246_Y_LA_END_REG, 975 },
+	{ HM1246_OUTPUT_PRT_CTRL_REG, 0x20 },
+	{ CCI_REG8(0x300a), 0x01 },
+	{ CCI_REG8(0x300b), 0x00 },
+	{ CCI_REG8(0x50f5), 0x01 },
+	{ CCI_REG8(0x50dd), 0x00 },
+	{ CCI_REG8(0x50a1), 0x02 },
+	{ CCI_REG8(0x50aa), 0x1c },
+	{ CCI_REG8(0x50ac), 0xdd },
+	{ CCI_REG8(0x50ad), 0x08 },
+	{ CCI_REG8(0x50ab), 0x04 },
+	{ CCI_REG8(0x50a0), 0x40 },
+	{ CCI_REG8(0x50a2), 0x12 },
+	{ CCI_REG8(0x50ae), 0x30 },
+	{ CCI_REG8(0x50b3), 0x04 },
+	{ CCI_REG8(0x5204), 0x40 },
+	{ CCI_REG8(0x5208), 0x55 },
+	{ CCI_REG8(0x520b), 0x05 },
+	{ CCI_REG8(0x520d), 0x40 },
+	{ CCI_REG8(0x5214), 0x18 },
+	{ CCI_REG8(0x5215), 0x0f },
+	{ CCI_REG8(0x5217), 0x01 },
+	{ CCI_REG8(0x5218), 0x07 },
+	{ CCI_REG8(0x5219), 0x01 },
+	{ CCI_REG8(0x521a), 0x50 },
+	{ CCI_REG8(0x521b), 0x24 },
+	{ CCI_REG8(0x5232), 0x01 },
+	{ CCI_REG8(0x5220), 0x11 },
+	{ CCI_REG8(0x5227), 0x01 },
+	{ CCI_REG8(0x5106), 0xc1 },
+	{ CCI_REG8(0x5115), 0xc0 },
+	{ CCI_REG8(0x5116), 0xc1 },
+	{ CCI_REG8(0x5138), 0x40 },
+	{ CCI_REG8(0x5139), 0x60 },
+	{ CCI_REG8(0x513a), 0x80 },
+	{ CCI_REG8(0x513b), 0xa0 },
+	{ CCI_REG8(0x513c), 0xa1 },
+	{ CCI_REG8(0x513d), 0xa2 },
+	{ CCI_REG8(0x513e), 0xa3 },
+	{ CCI_REG8(0x5140), 0x40 },
+	{ CCI_REG8(0x5141), 0x60 },
+	{ CCI_REG8(0x5142), 0x80 },
+	{ CCI_REG8(0x5143), 0x81 },
+	{ CCI_REG8(0x5144), 0x82 },
+	{ CCI_REG8(0x5145), 0x83 },
+	{ CCI_REG8(0x5146), 0x93 },
+	{ CCI_REG8(0x51c1), 0xc3 },
+	{ CCI_REG8(0x51c5), 0xc3 },
+	{ CCI_REG8(0x51c9), 0xc3 },
+	{ CCI_REG8(0x51cd), 0xc2 },
+	{ CCI_REG8(0x51d1), 0xc1 },
+	{ CCI_REG8(0x51d5), 0xc1 },
+	{ CCI_REG8(0x51d9), 0x81 },
+	{ CCI_REG8(0x51dd), 0x81 },
+	{ CCI_REG8(0x51c2), 0x49 },
+	{ CCI_REG8(0x51c6), 0x49 },
+	{ CCI_REG8(0x51ca), 0x49 },
+	{ CCI_REG8(0x51ce), 0x49 },
+	{ CCI_REG8(0x51d2), 0x49 },
+	{ CCI_REG8(0x51d6), 0x59 },
+	{ CCI_REG8(0x51da), 0x59 },
+	{ CCI_REG8(0x51de), 0x59 },
+	{ CCI_REG8(0x51c3), 0x20 },
+	{ CCI_REG8(0x51c7), 0x38 },
+	{ CCI_REG8(0x51cb), 0x21 },
+	{ CCI_REG8(0x51cf), 0x11 },
+	{ CCI_REG8(0x51d3), 0x11 },
+	{ CCI_REG8(0x51d7), 0x13 },
+	{ CCI_REG8(0x51db), 0x13 },
+	{ CCI_REG8(0x51df), 0x13 },
+	{ CCI_REG8(0x51e0), 0x03 },
+	{ CCI_REG8(0x51e2), 0x03 },
+	{ CCI_REG8(0x51f0), 0x42 },
+	{ CCI_REG8(0x51f1), 0x40 },
+	{ CCI_REG8(0x51f2), 0x4a },
+	{ CCI_REG8(0x51f3), 0x48 },
+	{ CCI_REG8(0x5015), 0x73 },
+	{ CCI_REG8(0x504a), 0x04 },
+	{ CCI_REG8(0x5044), 0x07 },
+	{ CCI_REG8(0x5040), 0x03 },
+	{ CCI_REG8(0x5135), 0xc4 },
+	{ CCI_REG8(0x5136), 0xc5 },
+	{ CCI_REG8(0x5166), 0xc4 },
+	{ CCI_REG8(0x5196), 0xc4 },
+	{ CCI_REG8(0x51c0), 0x10 },
+	{ CCI_REG8(0x51c4), 0x10 },
+	{ CCI_REG8(0x51c8), 0xa0 },
+	{ CCI_REG8(0x51cc), 0xa0 },
+	{ CCI_REG8(0x51d0), 0xa1 },
+	{ CCI_REG8(0x51d4), 0xa5 },
+	{ CCI_REG8(0x51d8), 0xa5 },
+	{ CCI_REG8(0x51dc), 0xa5 },
+	{ CCI_REG8(0x5200), 0xe4 },
+	{ CCI_REG8(0x5209), 0x04 },
+	{ CCI_REG8(0x301b), 0x01 },
+	{ CCI_REG8(0x3130), 0x01 },
+	{ CCI_REG8(0x5013), 0x07 },
+	{ CCI_REG8(0x5016), 0x01 },
+	{ CCI_REG8(0x501d), 0x50 },
+	{ CCI_REG8(0x0350), 0xfe },
+	{ CCI_REG8(0x2f03), 0x15 },
+	{ CCI_REG8(0xd380), 0x00 },
+	{ CCI_REG8(0x3047), 0x7f },
+	{ CCI_REG8(0x304d), 0x34 },
+	{ CCI_REG8(0x3041), 0x4b },
+	{ CCI_REG8(0x3042), 0x2d },
+	{ CCI_REG8(0x3056), 0x64 },
+	{ CCI_REG8(0x3059), 0x1e },
+	{ CCI_REG8(0x305e), 0x10 },
+	{ CCI_REG8(0x305f), 0x10 },
+	{ CCI_REG8(0x306d), 0x10 },
+	{ CCI_REG8(0x306e), 0x0c },
+	{ CCI_REG8(0x3064), 0x50 },
+	{ CCI_REG8(0x3067), 0x78 },
+	{ CCI_REG8(0x3068), 0x4b },
+	{ CCI_REG8(0x306a), 0x78 },
+	{ CCI_REG8(0x306b), 0x4b },
+	{ CCI_REG8(0xd442), 0x3d },
+	{ CCI_REG8(0xd443), 0x06 },
+	{ CCI_REG8(0xd440), 0x63 },
+	{ CCI_REG8(0xd446), 0xb0 },
+	{ CCI_REG8(0xd447), 0x60 },
+	{ CCI_REG8(0xd448), 0x48 },
+	{ CCI_REG8(0xd449), 0x30 },
+	{ CCI_REG8(0xd44a), 0x18 },
+	{ CCI_REG8(0xd360), 0x03 },
+	{ CCI_REG8(0x30ac), 0x10 },
+	{ CCI_REG8(0x30ad), 0x10 },
+	{ CCI_REG8(0x30ae), 0x10 },
+	{ CCI_REG8(0x3040), 0x0b },
+	{ CCI_REG8(0x2002), 0x00 },
+	{ CCI_REG8(0x2000), 0x08 },
+};
+
+struct hm1246_reg_list {
+	u32 num_of_regs;
+	const struct cci_reg_sequence *regs;
+};
+
+struct hm1246_mode {
+	u32 codes[4];
+	u32 clocks_per_pixel;
+	struct v4l2_rect rect;
+	u32 hts;
+	u32 vts_min;
+	const struct hm1246_reg_list reg_list;
+};
+
+#define FLIP_FORMAT_INDEX(v, h) ((v ? 2 : 0) | (h ? 1 : 0))
+
+/* Get the format code of the mode considering current flip setting. */
+static u32 hm1246_get_format_code(struct hm1246 *hm1246,
+				  const struct hm1246_mode *hm1246_mode)
+{
+	return hm1246_mode->codes[FLIP_FORMAT_INDEX(hm1246->vflip_ctrl->val,
+						    hm1246->hflip_ctrl->val)];
+}
+
+static const struct hm1246_mode hm1246_modes[] = {
+	{
+		.codes = {
+			[FLIP_FORMAT_INDEX(0, 0)] = MEDIA_BUS_FMT_SBGGR10_1X10,
+			[FLIP_FORMAT_INDEX(0, 1)] = MEDIA_BUS_FMT_SGBRG10_1X10,
+			[FLIP_FORMAT_INDEX(1, 0)] = MEDIA_BUS_FMT_SGRBG10_1X10,
+			[FLIP_FORMAT_INDEX(1, 1)] = MEDIA_BUS_FMT_SRGGB10_1X10,
+		},
+		.clocks_per_pixel = 1,
+		.rect.top = 0,
+		.rect.left = 0,
+		.rect.width = 1296,
+		.rect.height = 976,
+		.hts = 1420,
+		.vts_min = 990,
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(mode_1296x976_raw),
+			.regs = mode_1296x976_raw,
+		},
+	},
+};
+
+static inline struct hm1246 *to_hm1246(struct v4l2_subdev *sd)
+{
+	return container_of_const(sd, struct hm1246, sd);
+}
+
+static const struct hm1246_mode *
+hm1246_find_mode_by_mbus_code(struct hm1246 *hm1246, u32 code)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(hm1246_modes); i++) {
+		if (code == hm1246_get_format_code(hm1246, &hm1246_modes[i]))
+			return &hm1246_modes[i];
+	}
+
+	return NULL;
+}
+
+static int hm1246_power_on(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct hm1246 *hm1246 = to_hm1246(sd);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(hm1246_supply_names),
+				    hm1246->supplies);
+	if (ret) {
+		dev_err(hm1246->dev, "failed to enable regulators\n");
+		return ret;
+	}
+
+	ret = clk_prepare_enable(hm1246->xclk);
+	if (ret) {
+		regulator_bulk_disable(ARRAY_SIZE(hm1246_supply_names),
+				       hm1246->supplies);
+		dev_err(hm1246->dev, "failed to enable clock\n");
+		return ret;
+	}
+
+	reset_control_deassert(hm1246->reset);
+
+	/*
+	 * XSHUTDOWN to crystal clock oscillation (tcrystal):  650us (typical)
+	 * Sample bootstrap pin (tsample):                    2000us (maximum)
+	 * Built in self test (tbist):                        3000us (maximum)
+	 */
+	fsleep(6 * USEC_PER_MSEC);
+
+	return 0;
+}
+
+static int hm1246_power_off(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct hm1246 *hm1246 = to_hm1246(sd);
+
+	reset_control_assert(hm1246->reset);
+
+	clk_disable_unprepare(hm1246->xclk);
+
+	regulator_bulk_disable(ARRAY_SIZE(hm1246_supply_names),
+			       hm1246->supplies);
+
+	return 0;
+}
+
+static int hm1246_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct hm1246 *hm1246 = to_hm1246(sd);
+
+	if (code->index >= ARRAY_SIZE(hm1246_modes))
+		return -EINVAL;
+
+	code->code = hm1246_get_format_code(hm1246, &hm1246_modes[code->index]);
+
+	return 0;
+}
+
+static int hm1246_enum_frame_size(struct v4l2_subdev *subdev,
+				  struct v4l2_subdev_state *sd_state,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct hm1246 *hm1246 = to_hm1246(subdev);
+	const struct hm1246_mode *mode;
+
+	if (fse->index > 0)
+		return -EINVAL;
+
+	mode = hm1246_find_mode_by_mbus_code(hm1246, fse->code);
+	if (!mode)
+		return -EINVAL;
+
+	fse->min_width = mode->rect.width;
+	fse->max_width = mode->rect.width;
+	fse->min_height = mode->rect.height;
+	fse->max_height = mode->rect.height;
+
+	return 0;
+}
+
+static void hm1246_update_pad_format(struct hm1246 *hm1246,
+				     const struct hm1246_mode *hm1246_mode,
+				     struct v4l2_mbus_framefmt *fmt)
+{
+	fmt->width = hm1246_mode->rect.width;
+	fmt->height = hm1246_mode->rect.height;
+	fmt->code = hm1246_get_format_code(hm1246, hm1246_mode);
+	fmt->field = V4L2_FIELD_NONE;
+	fmt->colorspace = V4L2_COLORSPACE_RAW;
+	fmt->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(fmt->colorspace);
+	fmt->quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	fmt->xfer_func = V4L2_XFER_FUNC_NONE;
+}
+
+static int hm1246_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
+			     struct v4l2_subdev_state *state,
+			     struct v4l2_subdev_format *fmt)
+{
+	struct hm1246 *hm1246 = to_hm1246(sd);
+	struct v4l2_mbus_framefmt *mbus_fmt;
+	struct v4l2_rect *crop;
+	const struct hm1246_mode *mode;
+
+	mode = hm1246_find_mode_by_mbus_code(hm1246, fmt->format.code);
+	if (!mode)
+		mode = &hm1246_modes[0];
+
+	crop = v4l2_subdev_state_get_crop(state, 0);
+	*crop = mode->rect;
+
+	hm1246_update_pad_format(hm1246, mode, &fmt->format);
+	mbus_fmt = v4l2_subdev_state_get_format(state, 0);
+	*mbus_fmt = fmt->format;
+
+	return 0;
+}
+
+static int hm1246_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
+				struct v4l2_subdev_state *state,
+				struct v4l2_subdev_selection *sel)
+{
+	const struct v4l2_mbus_framefmt *format;
+	const struct hm1246_mode *mode;
+
+	format = v4l2_subdev_state_get_format(state, 0);
+	mode = v4l2_find_nearest_size(hm1246_modes, ARRAY_SIZE(hm1246_modes),
+				      rect.width, rect.height, format->width,
+				      format->height);
+
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP:
+		sel->r = *v4l2_subdev_state_get_crop(state, 0);
+		return 0;
+
+	case V4L2_SEL_TGT_NATIVE_SIZE:
+		sel->r.top = 0;
+		sel->r.left = 0;
+		sel->r.width = HM1246_NATIVE_WIDTH;
+		sel->r.height = HM1246_NATIVE_HEIGHT;
+		return 0;
+
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+		sel->r = mode->rect;
+		return 0;
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int hm1246_init_state(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_state *state)
+{
+	struct hm1246 *hm1246 = to_hm1246(sd);
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+		.pad = 0,
+		.format = {
+			.code = hm1246_get_format_code(hm1246,
+						       &hm1246_modes[0]),
+			.width = hm1246_modes[0].rect.width,
+			.height = hm1246_modes[0].rect.height,
+		},
+	};
+
+	hm1246_set_format(sd, NULL, state, &fmt);
+
+	return 0;
+}
+
+static int hm1246_calc_pll(u32 xclk, u32 link_freq, u32 clocks_per_pixel,
+			   u8 *pll1, u8 *pll2, u8 *pll3)
+{
+	static const u8 pclk_div_table[] = { 4, 5, 6, 7, 8, 12, 14, 16 };
+	static const u8 sysclk_div_table[] = { 1, 2, 3, 4 };
+	static const u8 post_div_table[] = { 1, 2, 4, 8 };
+	static const int sysclk_pclk_ratio = 3; /* Recommended value */
+	u32 pclk, vco_out;
+	int pclk_div_index, sysclk_div_index, post_div_index;
+	bool sysclk_pclk_ratio_found = false;
+
+	if (link_freq < HM1246_PCLK_MIN || link_freq > HM1246_PCLK_MAX)
+		return -EINVAL;
+
+	/*
+	 * In raw mode (1 pixel per clock) the pixel clock is internally
+	 * divided by two.
+	 */
+	pclk = 2 * link_freq / clocks_per_pixel;
+
+	/* Find suitable PCLK and SYSCLK dividers. */
+	for (pclk_div_index = 0; pclk_div_index < ARRAY_SIZE(pclk_div_table);
+	     pclk_div_index++) {
+		for (sysclk_div_index = 0;
+		     sysclk_div_index < ARRAY_SIZE(sysclk_div_table);
+		     sysclk_div_index++) {
+			if (sysclk_div_table[sysclk_div_index] *
+				    sysclk_pclk_ratio ==
+			    pclk_div_table[pclk_div_index]) {
+				sysclk_pclk_ratio_found = true;
+				break;
+			}
+		}
+		if (sysclk_pclk_ratio_found)
+			break;
+	}
+
+	if (!sysclk_pclk_ratio_found)
+		return -EINVAL;
+
+	/* Determine an appropriate post divider. */
+	for (post_div_index = 0; post_div_index < ARRAY_SIZE(post_div_table);
+	     post_div_index++) {
+		vco_out = pclk * pclk_div_table[pclk_div_index] *
+			  post_div_table[post_div_index];
+
+		if (vco_out >= HM1246_PLL_VCO_MIN &&
+		    vco_out <= HM1246_PLL_VCO_MAX)
+			break;
+	}
+	if (post_div_index >= ARRAY_SIZE(post_div_table))
+		return -EINVAL;
+
+	/* Find pre-divider and multiplier values. */
+	for (u32 div = DIV_ROUND_UP(xclk, HM1246_PLL_INCLK_MAX);
+	     div <= xclk / HM1246_PLL_INCLK_MIN; div++) {
+		u32 multi, multi_h, multi_l, vco;
+
+		multi = DIV_ROUND_CLOSEST_ULL((u64)vco_out * div, xclk);
+		if (multi < HM1246_PLL_MULTI_MIN ||
+		    multi > HM1246_PLL_MULTI_MAX)
+			continue;
+
+		multi_h = multi / (HM1246_PLL_MULTI_H_MIN *
+				   HM1246_PLL_MULTI_L_MAX) +
+			  2;
+		multi_l = multi / multi_h;
+		vco = div_u64((u64)xclk * multi_h * multi_l, div);
+
+		if (vco != vco_out)
+			continue;
+
+		if (pll1 && pll2 && pll3) {
+			*pll1 = HM1246_PLL1CFG_MULTIPLIER(multi_l - 1);
+			*pll2 = HM1246_PLL2CFG_PRE_DIV(div - 1) |
+				HM1246_PLL2CFG_MULTIPLIER(multi_h - 2);
+			*pll3 = HM1246_PLL3CFG_POST_DIV(post_div_index) |
+				HM1246_PLL3CFG_SYSCLK_DIV(sysclk_div_index) |
+				HM1246_PLL3CFG_PCLK_DIV(pclk_div_index);
+		}
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int hm1246_cci_write_pll(struct hm1246 *hm1246, u8 pll1, u8 pll2,
+				u8 pll3)
+{
+	const struct cci_reg_sequence pll_regs[] = {
+		{ HM1246_PLL1CFG_REG, pll1 },
+		{ HM1246_PLL2CFG_REG, pll2 },
+		{ HM1246_PLL3CFG_REG, pll3 },
+		{ HM1246_SBC_CTRL_REG, HM1246_SBC_CTRL_PLL_EN },
+	};
+
+	return cci_multi_reg_write(hm1246->regmap, pll_regs,
+				   ARRAY_SIZE(pll_regs), NULL);
+}
+
+static int hm1246_pll_check_locked(struct hm1246 *hm1246)
+{
+	u64 boot_ref2;
+	int ret;
+
+	ret = cci_read(hm1246->regmap, HM1246_SBC_BOOT_REF2_REG, &boot_ref2,
+		       NULL);
+	if (ret)
+		return ret;
+
+	return (boot_ref2 & HM1246_SBC_BOOT_REF2_PLL_LOCK) ? 0 : -EIO;
+}
+
+static int hm1246_setup_pll(struct hm1246 *hm1246,
+			    const struct hm1246_mode *mode)
+{
+	u8 pll1, pll2, pll3;
+	int ret;
+
+	ret = hm1246_calc_pll(hm1246->xclk_freq, hm1246->link_frequency,
+			      mode->clocks_per_pixel, &pll1, &pll2, &pll3);
+	if (ret)
+		return ret;
+
+	ret = hm1246_cci_write_pll(hm1246, pll1, pll2, pll3);
+	if (ret)
+		return ret;
+
+	/* PLL lock time (tpll): 100us (typical) */
+	fsleep(200);
+
+	return hm1246_pll_check_locked(hm1246);
+}
+
+static int hm1246_cci_write_test_pattern(struct hm1246 *hm1246, u8 mode,
+					 u16 r, u16 g, u16 b)
+{
+	const struct cci_reg_sequence tpg_enable_regs[] = {
+		{ HM1246_TEST_DATA_RED_REG, r },
+		{ HM1246_TEST_DATA_GR_REG, g },
+		{ HM1246_TEST_DATA_GB_REG, g },
+		{ HM1246_TEST_DATA_BLUE_REG, b },
+		{ HM1246_TEST_PATTERN_MODE_REG, mode },
+	};
+
+	return cci_multi_reg_write(hm1246->regmap, tpg_enable_regs,
+				   ARRAY_SIZE(tpg_enable_regs), NULL);
+}
+
+static int hm1246_test_pattern(struct hm1246 *hm1246, u32 index)
+{
+	static const u16 RGBMIN = 0, RGBMAX = 0x3ff;
+	static const struct tp {
+		int pattern;
+		u16 r, g, b;
+	} tps[] = {
+		/* Disabled */
+		[0] = { .pattern = 0, .r = RGBMIN, .g = RGBMIN, .b = RGBMIN },
+		/* Checkboard pattern */
+		[1] = { .pattern = 0, .r = RGBMIN, .g = RGBMIN, .b = RGBMIN },
+		/* Ramp */
+		[2] = { .pattern = 1, .r = RGBMIN, .g = RGBMIN, .b = RGBMIN },
+		/* Moving ones */
+		[3] = { .pattern = 2, .r = RGBMIN, .g = RGBMIN, .b = RGBMIN },
+		/* Blending color bars */
+		[4] = { .pattern = 3, .r = RGBMIN, .g = RGBMIN, .b = RGBMIN },
+		/* Color bars */
+		[5] = { .pattern = 4, .r = RGBMIN, .g = RGBMIN, .b = RGBMIN },
+		/* Solid white */
+		[6] = { .pattern = 15, .r = RGBMAX, .g = RGBMAX, .b = RGBMAX },
+		/* Solid black */
+		[7] = { .pattern = 15, .r = RGBMIN, .g = RGBMIN, .b = RGBMIN },
+		/* Solid red */
+		[8] = { .pattern = 15, .r = RGBMAX, .g = RGBMIN, .b = RGBMIN },
+		/* Solid green */
+		[9] = { .pattern = 15, .r = RGBMIN, .g = RGBMAX, .b = RGBMIN },
+		/* Solid blue */
+		[10] = { .pattern = 15, .r = RGBMIN, .g = RGBMIN, .b = RGBMAX },
+	};
+	u8 mode;
+
+	if (index >= ARRAY_SIZE(tps))
+		return -EINVAL;
+
+	mode = HM1246_TEST_PATTERN_MODE_MODE(tps[index].pattern);
+	if (index)
+		mode |= HM1246_TEST_PATTERN_MODE_ENABLE;
+
+	return hm1246_cci_write_test_pattern(hm1246, mode, tps[index].r,
+					     tps[index].g, tps[index].b);
+}
+
+static int hm1246_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct hm1246 *hm1246 =
+		container_of_const(ctrl->handler, struct hm1246, ctrls);
+	struct v4l2_subdev_state *state;
+	const struct v4l2_mbus_framefmt *format;
+	u32 val;
+	bool needs_cmu_update = true;
+	int ret;
+
+	state = v4l2_subdev_get_locked_active_state(&hm1246->sd);
+	format = v4l2_subdev_state_get_format(state, 0);
+
+	if (ctrl->id == V4L2_CID_VBLANK) {
+		s64 exposure_max;
+
+		exposure_max =
+			format->height + ctrl->val - HM1246_COARSE_INTG_MARGIN;
+		ret = __v4l2_ctrl_modify_range(hm1246->exposure_ctrl,
+					       hm1246->exposure_ctrl->minimum,
+					       exposure_max,
+					       hm1246->exposure_ctrl->step,
+					       exposure_max);
+
+		if (ret) {
+			dev_err(hm1246->dev, "exposure ctrl range update failed\n");
+			return ret;
+		}
+	}
+
+	if (!pm_runtime_get_if_active(hm1246->dev))
+		return 0;
+
+	ret = 0;
+	switch (ctrl->id) {
+	case V4L2_CID_EXPOSURE:
+		cci_write(hm1246->regmap, HM1246_COARSE_INTG_REG, ctrl->val,
+			  &ret);
+		break;
+
+	case V4L2_CID_ANALOGUE_GAIN:
+		cci_write(hm1246->regmap, HM1246_ANALOG_GLOBAL_GAIN_REG,
+			  ctrl->val, &ret);
+		break;
+
+	case V4L2_CID_VBLANK:
+		val = format->height + ctrl->val;
+		cci_write(hm1246->regmap, HM1246_FRAME_LENGTH_LINES_REG, val,
+			  &ret);
+		break;
+
+	case V4L2_CID_HFLIP:
+	case V4L2_CID_VFLIP:
+		val = 0;
+		if (hm1246->hflip_ctrl->val)
+			val |= HM1246_IMAGE_ORIENTATION_HFLIP;
+		if (hm1246->vflip_ctrl->val)
+			val |= HM1246_IMAGE_ORIENTATION_VFLIP;
+
+		cci_write(hm1246->regmap, HM1246_IMAGE_ORIENTATION_REG, val,
+			  &ret);
+		break;
+
+	case V4L2_CID_TEST_PATTERN:
+		ret = hm1246_test_pattern(hm1246, ctrl->val);
+		needs_cmu_update = false;
+		break;
+
+	default:
+		ret = -EINVAL;
+		needs_cmu_update = false;
+		break;
+	}
+
+	if (needs_cmu_update)
+		cci_write(hm1246->regmap, HM1246_CMU_UPDATE_REG, 0, &ret);
+
+	pm_runtime_put(hm1246->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops hm1246_ctrl_ops = {
+	.s_ctrl = hm1246_set_ctrl,
+};
+
+static int hm1246_identify_module(struct hm1246 *hm1246)
+{
+	u64 model_id;
+	int ret;
+
+	if (hm1246->identified)
+		return 0;
+
+	ret = cci_read(hm1246->regmap, HM1246_MODEL_ID_REG, &model_id, NULL);
+	if (ret)
+		return ret;
+
+	if (model_id != HM1246_MODEL_ID) {
+		dev_err(hm1246->dev, "model id mismatch: 0x%llx!=0x%x\n",
+			model_id, HM1246_MODEL_ID);
+		return -ENXIO;
+	}
+
+	hm1246->identified = true;
+
+	return 0;
+}
+
+static int hm1246_setup_moderegs(struct hm1246 *hm1246,
+				 const struct hm1246_mode *mode)
+{
+	const struct hm1246_reg_list *reg_list = &mode->reg_list;
+	const struct cci_reg_sequence modeaw[] = {
+		{ HM1246_X_ADDR_START_REG, mode->rect.left },
+		{ HM1246_Y_ADDR_START_REG, mode->rect.top },
+		{ HM1246_X_ADDR_END_REG, mode->rect.width - 1 },
+		{ HM1246_Y_ADDR_END_REG, mode->rect.height - 1 },
+		{ HM1246_DWIN_XOFFSET_REG, mode->rect.left },
+		{ HM1246_DWIN_YOFFSET_REG, mode->rect.top },
+		{ HM1246_DWIN_XSIZE_REG, mode->rect.width },
+		{ HM1246_DWIN_YSIZE_REG, mode->rect.height },
+		{ HM1246_LINE_LENGTH_PCK_REG, mode->hts },
+	};
+	int ret = 0;
+
+	cci_multi_reg_write(hm1246->regmap, modeaw, ARRAY_SIZE(modeaw), &ret);
+	cci_multi_reg_write(hm1246->regmap, reg_list->regs,
+			    reg_list->num_of_regs, &ret);
+
+	return ret;
+}
+
+static int hm1246_setup_bus(struct hm1246 *hm1246)
+{
+	u64 polarity_ctrl = 0, pclk_ctrl = 0;
+	int ret = 0;
+
+	if (hm1246->mbus_flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
+		polarity_ctrl |= HM1246_POLARITY_CTRL_HSYNC;
+
+	if (hm1246->mbus_flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
+		polarity_ctrl |= HM1246_POLARITY_CTRL_VSYNC;
+
+	cci_write(hm1246->regmap, HM1246_POLARITY_CTRL_REG, polarity_ctrl,
+		  &ret);
+
+	/*
+	 * If the clock output polarity flag PCLK_CTRL[3] is set (high), the
+	 * data lines change state on the falling edge of PCLK and should
+	 * therefore be sampled on the rising edge.
+	 * This is different than described in the data sheet.
+	 */
+	if (hm1246->mbus_flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
+		pclk_ctrl |= HM1246_PCLK_CTRL_POL;
+
+	cci_write(hm1246->regmap, HM1246_PCLK_CTRL_REG, pclk_ctrl, &ret);
+
+	return ret;
+}
+
+static int hm1246_enable_streams(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state, u32 pad,
+				 u64 streams_mask)
+{
+	struct hm1246 *hm1246 = to_hm1246(sd);
+	const struct v4l2_mbus_framefmt *format;
+	const struct hm1246_mode *mode;
+	int ret;
+
+	format = v4l2_subdev_state_get_format(state, 0);
+	mode = v4l2_find_nearest_size(hm1246_modes, ARRAY_SIZE(hm1246_modes),
+				      rect.width, rect.height, format->width,
+				      format->height);
+
+	ret = pm_runtime_resume_and_get(hm1246->dev);
+	if (ret)
+		return ret;
+
+	ret = hm1246_identify_module(hm1246);
+	if (ret)
+		goto err_rpm_put;
+
+	ret = hm1246_setup_pll(hm1246, mode);
+	if (ret) {
+		dev_err(hm1246->dev, "failed to setup PLL\n");
+		goto err_rpm_put;
+	}
+
+	ret = hm1246_setup_moderegs(hm1246, mode);
+	if (ret)
+		goto err_rpm_put;
+
+	ret = hm1246_setup_bus(hm1246);
+	if (ret)
+		goto err_rpm_put;
+
+	ret = __v4l2_ctrl_handler_setup(&hm1246->ctrls);
+	if (ret) {
+		dev_err(hm1246->dev, "failed to setup v4l2 controls\n");
+		goto err_rpm_put;
+	}
+
+	ret = cci_write(hm1246->regmap, HM1246_MODE_SELECT_REG,
+			HM1246_MODE_SELECT_STREAM, NULL);
+	if (ret)
+		goto err_rpm_put;
+
+	/*
+	 * Since mirroring may change the actual pixel format, it must not be
+	 * changed during streaming.
+	 */
+	__v4l2_ctrl_grab(hm1246->vflip_ctrl, true);
+	__v4l2_ctrl_grab(hm1246->hflip_ctrl, true);
+
+	return 0;
+
+err_rpm_put:
+	pm_runtime_put_autosuspend(hm1246->dev);
+
+	return ret;
+}
+
+static int hm1246_disable_streams(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *state, u32 pad,
+				  u64 streams_mask)
+{
+	struct hm1246 *hm1246 = to_hm1246(sd);
+	int ret;
+
+	ret = cci_write(hm1246->regmap, HM1246_MODE_SELECT_REG,
+			HM1246_MODE_SELECT_STANDBY, NULL);
+
+	__v4l2_ctrl_grab(hm1246->vflip_ctrl, false);
+	__v4l2_ctrl_grab(hm1246->hflip_ctrl, false);
+
+	pm_runtime_put_autosuspend(hm1246->dev);
+
+	return ret;
+}
+
+static const struct v4l2_subdev_video_ops hm1246_video_ops = {
+	.s_stream = v4l2_subdev_s_stream_helper,
+};
+
+static const struct v4l2_subdev_pad_ops hm1246_subdev_pad_ops = {
+	.enum_mbus_code = hm1246_enum_mbus_code,
+	.enum_frame_size = hm1246_enum_frame_size,
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = hm1246_set_format,
+	.get_selection = hm1246_get_selection,
+	.enable_streams = hm1246_enable_streams,
+	.disable_streams = hm1246_disable_streams,
+};
+
+static const struct v4l2_subdev_ops hm1246_subdev_ops = {
+	.video = &hm1246_video_ops,
+	.pad = &hm1246_subdev_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops hm1246_internal_ops = {
+	.init_state = hm1246_init_state,
+};
+
+static int hm1246_get_regulators(struct device *dev, struct hm1246 *hm1246)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(hm1246_supply_names); i++)
+		hm1246->supplies[i].supply = hm1246_supply_names[i];
+
+	return devm_regulator_bulk_get(dev, ARRAY_SIZE(hm1246_supply_names),
+				       hm1246->supplies);
+}
+
+static int hm1246_parse_fwnode(struct hm1246 *hm1246)
+{
+	struct fwnode_handle *endpoint;
+	struct v4l2_fwnode_endpoint bus_cfg = {
+		.bus_type = V4L2_MBUS_PARALLEL,
+	};
+	int ret;
+
+	endpoint = fwnode_graph_get_endpoint_by_id(dev_fwnode(hm1246->dev),
+						   0, 0,
+						   FWNODE_GRAPH_ENDPOINT_NEXT);
+
+	ret = v4l2_fwnode_endpoint_alloc_parse(endpoint, &bus_cfg);
+	fwnode_handle_put(endpoint);
+	if (ret)
+		return dev_err_probe(hm1246->dev, ret,
+				     "parsing endpoint node failed\n");
+
+	hm1246->mbus_flags = bus_cfg.bus.parallel.flags;
+
+	if (bus_cfg.nr_of_link_frequencies > 0)
+		hm1246->link_frequency = bus_cfg.link_frequencies[0];
+
+	v4l2_fwnode_endpoint_free(&bus_cfg);
+
+	if (!hm1246->link_frequency)
+		return dev_err_probe(hm1246->dev, -EINVAL,
+				     "one link frequency expected\n");
+
+	return 0;
+}
+
+static int hm1246_init_controls(struct hm1246 *hm1246)
+{
+	const struct hm1246_mode *mode = &hm1246_modes[0];
+	struct v4l2_fwnode_device_properties props;
+	struct v4l2_ctrl_handler *ctrl_hdlr = &hm1246->ctrls;
+	struct v4l2_ctrl *ctrl;
+	s64 pixel_rate, exposure_max, vblank_min, hblank;
+	int ret;
+
+	ret = v4l2_fwnode_device_parse(hm1246->dev, &props);
+	if (ret)
+		return ret;
+
+	v4l2_ctrl_handler_init(ctrl_hdlr, 11);
+
+	hm1246->hflip_ctrl = v4l2_ctrl_new_std(ctrl_hdlr, &hm1246_ctrl_ops,
+					       V4L2_CID_HFLIP, 0, 1, 1, 0);
+	if (hm1246->hflip_ctrl)
+		hm1246->hflip_ctrl->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT;
+
+	hm1246->vflip_ctrl = v4l2_ctrl_new_std(ctrl_hdlr, &hm1246_ctrl_ops,
+					       V4L2_CID_VFLIP, 0, 1, 1, 0);
+	if (hm1246->vflip_ctrl)
+		hm1246->vflip_ctrl->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT;
+
+	v4l2_ctrl_cluster(2, &hm1246->hflip_ctrl);
+
+	ctrl = v4l2_ctrl_new_int_menu(ctrl_hdlr, &hm1246_ctrl_ops,
+				      V4L2_CID_LINK_FREQ,
+				      0, 0,
+				      &hm1246->link_frequency);
+	if (ctrl)
+		ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	pixel_rate = div_u64(hm1246->link_frequency, mode->clocks_per_pixel);
+	ctrl = v4l2_ctrl_new_std(ctrl_hdlr, &hm1246_ctrl_ops,
+				 V4L2_CID_PIXEL_RATE,
+				 pixel_rate, pixel_rate, 1,
+				 pixel_rate);
+
+	vblank_min = mode->vts_min - mode->rect.height;
+	ctrl = v4l2_ctrl_new_std(ctrl_hdlr, &hm1246_ctrl_ops,
+				 V4L2_CID_VBLANK, vblank_min,
+				 HM1246_VTS_MAX - mode->rect.height,
+				 1, vblank_min);
+
+	hblank = mode->hts - mode->rect.width;
+	ctrl = v4l2_ctrl_new_std(ctrl_hdlr, &hm1246_ctrl_ops,
+				 V4L2_CID_HBLANK, hblank, hblank,
+				 1, hblank);
+	if (ctrl)
+		ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &hm1246_ctrl_ops, V4L2_CID_ANALOGUE_GAIN,
+			  HM1246_ANALOG_GLOBAL_GAIN_MIN,
+			  HM1246_ANALOG_GLOBAL_GAIN_MAX,
+			  HM1246_ANALOG_GLOBAL_GAIN_STEP,
+			  HM1246_ANALOG_GLOBAL_GAIN_MIN);
+
+	exposure_max = mode->vts_min - HM1246_COARSE_INTG_MARGIN;
+	hm1246->exposure_ctrl = v4l2_ctrl_new_std(ctrl_hdlr, &hm1246_ctrl_ops,
+						  V4L2_CID_EXPOSURE,
+						  HM1246_COARSE_INTG_MIN,
+						  exposure_max,
+						  HM1246_COARSE_INTG_STEP,
+						  exposure_max);
+
+	v4l2_ctrl_new_std_menu_items(ctrl_hdlr, &hm1246_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(hm1246_test_pattern_menu) - 1,
+				     0, 0, hm1246_test_pattern_menu);
+
+	v4l2_ctrl_new_fwnode_properties(ctrl_hdlr, &hm1246_ctrl_ops, &props);
+
+	if (ctrl_hdlr->error) {
+		v4l2_ctrl_handler_free(ctrl_hdlr);
+		return ctrl_hdlr->error;
+	}
+
+	hm1246->sd.ctrl_handler = ctrl_hdlr;
+
+	return 0;
+}
+
+static int hm1246_probe(struct i2c_client *client)
+{
+	struct hm1246 *hm1246;
+	int ret;
+
+	hm1246 = devm_kzalloc(&client->dev, sizeof(*hm1246), GFP_KERNEL);
+	if (!hm1246)
+		return -ENOMEM;
+
+	hm1246->dev = &client->dev;
+
+	ret = hm1246_parse_fwnode(hm1246);
+	if (ret)
+		return ret;
+
+	hm1246->regmap = devm_cci_regmap_init_i2c(client, 16);
+	if (IS_ERR(hm1246->regmap))
+		return dev_err_probe(hm1246->dev, PTR_ERR(hm1246->regmap),
+				     "failed to init CCI\n");
+
+	hm1246->xclk = devm_v4l2_sensor_clk_get(hm1246->dev, NULL);
+	if (IS_ERR(hm1246->xclk))
+		return dev_err_probe(hm1246->dev, PTR_ERR(hm1246->xclk),
+				     "failed to get xclk\n");
+
+	hm1246->xclk_freq = clk_get_rate(hm1246->xclk);
+	if (hm1246->xclk_freq < HM1246_XCLK_MIN ||
+	    hm1246->xclk_freq > HM1246_XCLK_MAX)
+		return dev_err_probe(hm1246->dev, -EINVAL,
+				     "xclk frequency out of range: %luHz\n",
+				     hm1246->xclk_freq);
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(hm1246_modes); i++) {
+		ret = hm1246_calc_pll(hm1246->xclk_freq, hm1246->link_frequency,
+				      hm1246_modes[i].clocks_per_pixel,
+				      NULL, NULL, NULL);
+		if (ret)
+			return dev_err_probe(hm1246->dev, ret,
+					     "no PLL setup for %lld Hz\n",
+					     hm1246->link_frequency);
+	}
+
+	ret = hm1246_get_regulators(hm1246->dev, hm1246);
+	if (ret)
+		return dev_err_probe(hm1246->dev, ret,
+				     "failed to get regulators\n");
+
+	hm1246->reset = devm_reset_control_get_optional(hm1246->dev, NULL);
+	if (IS_ERR(hm1246->reset))
+		return dev_err_probe(hm1246->dev, PTR_ERR(hm1246->reset),
+				     "failed to get reset\n");
+	reset_control_assert(hm1246->reset);
+
+	v4l2_i2c_subdev_init(&hm1246->sd, client, &hm1246_subdev_ops);
+	hm1246->sd.internal_ops = &hm1246_internal_ops;
+
+	ret = hm1246_init_controls(hm1246);
+	if (ret)
+		return dev_err_probe(hm1246->dev, ret,
+				     "failed to init controls\n");
+
+	hm1246->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	hm1246->pad.flags = MEDIA_PAD_FL_SOURCE;
+	hm1246->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+
+	ret = media_entity_pads_init(&hm1246->sd.entity, 1, &hm1246->pad);
+	if (ret) {
+		dev_err_probe(hm1246->dev, ret, "failed to init media pads\n");
+		goto err_v4l2_ctrl_handler_free;
+	}
+
+	hm1246->sd.state_lock = hm1246->ctrls.lock;
+	ret = v4l2_subdev_init_finalize(&hm1246->sd);
+	if (ret) {
+		dev_err_probe(hm1246->dev, ret, "failed to init v4l2 subdev\n");
+		goto err_media_entity_cleanup;
+	}
+
+	pm_runtime_enable(hm1246->dev);
+	pm_runtime_set_autosuspend_delay(hm1246->dev, 1000);
+	pm_runtime_use_autosuspend(hm1246->dev);
+
+	ret = v4l2_async_register_subdev_sensor(&hm1246->sd);
+	if (ret) {
+		dev_err_probe(hm1246->dev, ret,
+			      "failed to register v4l2 subdev\n");
+		goto err_subdev_cleanup;
+	}
+
+	return 0;
+
+err_subdev_cleanup:
+	v4l2_subdev_cleanup(&hm1246->sd);
+	pm_runtime_disable(hm1246->dev);
+	pm_runtime_set_suspended(hm1246->dev);
+
+err_media_entity_cleanup:
+	media_entity_cleanup(&hm1246->sd.entity);
+
+err_v4l2_ctrl_handler_free:
+	v4l2_ctrl_handler_free(&hm1246->ctrls);
+
+	return ret;
+}
+
+static void hm1246_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct hm1246 *hm1246 = to_hm1246(sd);
+
+	v4l2_async_unregister_subdev(&hm1246->sd);
+	v4l2_subdev_cleanup(sd);
+	media_entity_cleanup(&hm1246->sd.entity);
+	v4l2_ctrl_handler_free(&hm1246->ctrls);
+
+	pm_runtime_disable(&client->dev);
+	if (!pm_runtime_status_suspended(&client->dev)) {
+		hm1246_power_off(hm1246->dev);
+		pm_runtime_set_suspended(&client->dev);
+	}
+}
+
+static const struct of_device_id hm1246_of_match[] = {
+	{ .compatible = "himax,hm1246" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, hm1246_of_match);
+
+static DEFINE_RUNTIME_DEV_PM_OPS(hm1246_pm_ops,
+				 hm1246_power_off, hm1246_power_on, NULL);
+
+static struct i2c_driver hm1246_i2c_driver = {
+	.driver = {
+		.of_match_table = hm1246_of_match,
+		.pm = pm_ptr(&hm1246_pm_ops),
+		.name = "hm1246",
+	},
+	.probe = hm1246_probe,
+	.remove = hm1246_remove,
+};
+module_i2c_driver(hm1246_i2c_driver);
+
+MODULE_DESCRIPTION("Himax HM1246 camera driver");
+MODULE_AUTHOR("Matthias Fend <matthias.fend@emfend.at>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/imx111.c b/drivers/media/i2c/imx111.c
index 8eb919788ef794..5b860fa052b125 100644
--- a/drivers/media/i2c/imx111.c
+++ b/drivers/media/i2c/imx111.c
@@ -1122,6 +1122,7 @@ static int imx111_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int imx111_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/imx208.c b/drivers/media/i2c/imx208.c
index d5350bb46f14a5..3a899b41a9d295 100644
--- a/drivers/media/i2c/imx208.c
+++ b/drivers/media/i2c/imx208.c
@@ -574,6 +574,7 @@ static int imx208_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int imx208_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/imx214.c b/drivers/media/i2c/imx214.c
index d4945b192776bc..96833f2bf287f7 100644
--- a/drivers/media/i2c/imx214.c
+++ b/drivers/media/i2c/imx214.c
@@ -662,6 +662,7 @@ static const struct v4l2_subdev_core_ops imx214_core_ops = {
 };
 
 static int imx214_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *format)
 {
@@ -717,6 +718,7 @@ static int imx214_set_format(struct v4l2_subdev *sd,
 }
 
 static int imx214_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -754,7 +756,7 @@ static int imx214_entity_init_state(struct v4l2_subdev *subdev,
 	fmt.format.width = imx214_modes[0].width;
 	fmt.format.height = imx214_modes[0].height;
 
-	imx214_set_format(subdev, sd_state, &fmt);
+	imx214_set_format(subdev, NULL, sd_state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c
index 9571f3622d2d4a..1580562093e82b 100644
--- a/drivers/media/i2c/imx219.c
+++ b/drivers/media/i2c/imx219.c
@@ -78,6 +78,7 @@
 #define IMX219_LLP_MIN			0x0d78
 #define IMX219_BINNED_LLP_MIN		0x0de8
 #define IMX219_LLP_MAX			0x7ff0
+#define IMX219_LLP_STEP			8
 
 #define IMX219_REG_X_ADD_STA_A		CCI_REG16(0x0164)
 #define IMX219_REG_X_ADD_END_A		CCI_REG16(0x0166)
@@ -340,13 +341,13 @@ static const struct imx219_mode supported_modes[] = {
 		/* 2x2 binned 60fps mode */
 		.width = 1640,
 		.height = 1232,
-		.fll_def = 1707,
+		.fll_def = 1706,
 	},
 	{
 		/* 640x480 60fps mode */
 		.width = 640,
 		.height = 480,
-		.fll_def = 1707,
+		.fll_def = 1706,
 	},
 };
 
@@ -466,14 +467,14 @@ static int imx219_set_ctrl(struct v4l2_ctrl *ctrl)
 		int exposure_max, exposure_def;
 
 		/* Update max exposure while meeting expected vblanking */
-		exposure_max = format->height + ctrl->val - IMX219_EXPOSURE_OFFSET;
+		exposure_max = format->height + ctrl->val -
+			IMX219_EXPOSURE_OFFSET * rate_factor;
 		exposure_def = (exposure_max < IMX219_EXPOSURE_DEFAULT) ?
 				exposure_max : IMX219_EXPOSURE_DEFAULT;
 		ret = __v4l2_ctrl_modify_range(imx219->exposure,
 					       imx219->exposure->minimum,
 					       exposure_max,
-					       imx219->exposure->step,
-					       exposure_def);
+					       rate_factor, exposure_def);
 		if (ret)
 			return ret;
 
@@ -593,7 +594,8 @@ static int imx219_init_controls(struct imx219 *imx219)
 	imx219->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &imx219_ctrl_ops,
 					   V4L2_CID_HBLANK,
 					   IMX219_LLP_MIN - mode->width,
-					   IMX219_LLP_MAX - mode->width, 1,
+					   IMX219_LLP_MAX - mode->width,
+					   IMX219_LLP_STEP,
 					   IMX219_LLP_MIN - mode->width);
 	exposure_max = mode->fll_def - IMX219_EXPOSURE_OFFSET;
 	exposure_def = (exposure_max < IMX219_EXPOSURE_DEFAULT) ?
@@ -844,6 +846,7 @@ static int imx219_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int imx219_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -851,7 +854,7 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 	const struct imx219_mode *mode;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *crop;
-	u8 bin_h, bin_v, binning;
+	u8 bin_h, bin_v, bin_hv;
 	int ret;
 
 	format = v4l2_subdev_state_get_format(state, 0);
@@ -884,15 +887,16 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 	bin_v = min(IMX219_ACTIVE_AREA_HEIGHT / format->height, 2U);
 
 	/* Ensure bin_h and bin_v are same to avoid 1:2 or 2:1 stretching */
-	binning = min(bin_h, bin_v);
+	bin_hv = min(bin_h, bin_v);
 
 	crop = v4l2_subdev_state_get_crop(state, 0);
-	crop->width = format->width * binning;
-	crop->height = format->height * binning;
+	crop->width = format->width * bin_hv;
+	crop->height = format->height * bin_hv;
 	crop->left = (IMX219_NATIVE_WIDTH - crop->width) / 2;
 	crop->top = (IMX219_NATIVE_HEIGHT - crop->height) / 2;
 
 	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		int rate_factor = imx219_get_rate_factor(state);
 		int exposure_max;
 		int exposure_def;
 		int llp_min;
@@ -900,7 +904,8 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 
 		/* Update limits and set FPS to default */
 		ret = __v4l2_ctrl_modify_range(imx219->vblank, IMX219_VBLANK_MIN,
-					       IMX219_FLL_MAX - mode->height, 1,
+					       IMX219_FLL_MAX - mode->height,
+					       rate_factor,
 					       mode->fll_def - mode->height);
 		if (ret)
 			return ret;
@@ -911,14 +916,14 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 			return ret;
 
 		/* Update max exposure while meeting expected vblanking */
-		exposure_max = mode->fll_def - IMX219_EXPOSURE_OFFSET;
+		exposure_max = mode->fll_def -
+			IMX219_EXPOSURE_OFFSET * rate_factor;
 		exposure_def = (exposure_max < IMX219_EXPOSURE_DEFAULT) ?
 				exposure_max : IMX219_EXPOSURE_DEFAULT;
 		ret = __v4l2_ctrl_modify_range(imx219->exposure,
 					       imx219->exposure->minimum,
 					       exposure_max,
-					       imx219->exposure->step,
-					       exposure_def);
+					       rate_factor, exposure_def);
 		if (ret)
 			return ret;
 
@@ -933,7 +938,8 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 				  IMX219_BINNED_LLP_MIN : IMX219_LLP_MIN;
 		ret = __v4l2_ctrl_modify_range(imx219->hblank,
 					       llp_min - mode->width,
-					       IMX219_LLP_MAX - mode->width, 1,
+					       IMX219_LLP_MAX - mode->width,
+					       IMX219_LLP_STEP,
 					       llp_min - mode->width);
 		if (ret)
 			return ret;
@@ -943,8 +949,7 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 			return ret;
 
 		/* Scale the pixel rate based on the mode specific factor */
-		pixel_rate = imx219_get_pixel_rate(imx219) *
-			     imx219_get_rate_factor(state);
+		pixel_rate = imx219_get_pixel_rate(imx219) * rate_factor;
 		ret = __v4l2_ctrl_modify_range(imx219->pixel_rate, pixel_rate,
 					       pixel_rate, 1, pixel_rate);
 		if (ret)
@@ -955,6 +960,7 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 }
 
 static int imx219_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -997,7 +1003,7 @@ static int imx219_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	return imx219_set_pad_format(sd, state, &fmt);
+	return imx219_set_pad_format(sd, NULL, state, &fmt);
 }
 
 static const struct v4l2_subdev_video_ops imx219_video_ops = {
diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c
index bc9ee449a87c52..065c33380f60c8 100644
--- a/drivers/media/i2c/imx258.c
+++ b/drivers/media/i2c/imx258.c
@@ -913,6 +913,7 @@ static int imx258_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int imx258_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -988,6 +989,7 @@ __imx258_get_pad_crop(struct imx258 *imx258,
 }
 
 static int imx258_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c
index 76da647f9cf969..3245afc8ae5b87 100644
--- a/drivers/media/i2c/imx274.c
+++ b/drivers/media/i2c/imx274.c
@@ -1059,6 +1059,7 @@ static int imx274_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int imx274_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
@@ -1092,6 +1093,7 @@ static int imx274_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int imx274_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1208,6 +1210,7 @@ static int imx274_set_selection_crop(struct stimx274 *imx274,
 }
 
 static int imx274_set_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/imx283.c b/drivers/media/i2c/imx283.c
index 38ea1902f2c257..e3cd27ca238ee0 100644
--- a/drivers/media/i2c/imx283.c
+++ b/drivers/media/i2c/imx283.c
@@ -958,6 +958,7 @@ static void imx283_set_framing_limits(struct imx283 *imx283,
 }
 
 static int imx283_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -1260,6 +1261,7 @@ static int imx283_identify_module(struct imx283 *imx283)
 }
 
 static int imx283_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/imx290.c b/drivers/media/i2c/imx290.c
index 21cbc81cb2edcd..d562a9aa455c37 100644
--- a/drivers/media/i2c/imx290.c
+++ b/drivers/media/i2c/imx290.c
@@ -1149,6 +1149,7 @@ static int imx290_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int imx290_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -1187,6 +1188,7 @@ static int imx290_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int imx290_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1246,7 +1248,7 @@ static int imx290_entity_init_state(struct v4l2_subdev *subdev,
 		},
 	};
 
-	imx290_set_fmt(subdev, sd_state, &fmt);
+	imx290_set_fmt(subdev, NULL, sd_state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/imx296.c b/drivers/media/i2c/imx296.c
index 69636db11a2beb..74bb295799fddc 100644
--- a/drivers/media/i2c/imx296.c
+++ b/drivers/media/i2c/imx296.c
@@ -675,6 +675,7 @@ static int imx296_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int imx296_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *fmt)
 {
@@ -726,6 +727,7 @@ static int imx296_set_format(struct v4l2_subdev *sd,
 }
 
 static int imx296_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -751,6 +753,7 @@ static int imx296_get_selection(struct v4l2_subdev *sd,
 }
 
 static int imx296_set_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -812,8 +815,8 @@ static int imx296_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	imx296_set_selection(sd, state, &sel);
-	imx296_set_format(sd, state, &format);
+	imx296_set_selection(sd, NULL, state, &sel);
+	imx296_set_format(sd, NULL, state, &format);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/imx319.c b/drivers/media/i2c/imx319.c
index 953310ef30460f..281d39ed4194c1 100644
--- a/drivers/media/i2c/imx319.c
+++ b/drivers/media/i2c/imx319.c
@@ -2030,6 +2030,7 @@ static int imx319_get_pad_format(struct v4l2_subdev *sd,
 
 static int
 imx319_set_pad_format(struct v4l2_subdev *sd,
+		      const struct v4l2_subdev_client_info *ci,
 		      struct v4l2_subdev_state *sd_state,
 		      struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/imx334.c b/drivers/media/i2c/imx334.c
index 553a16b84f4d26..da19d5fc19f540 100644
--- a/drivers/media/i2c/imx334.c
+++ b/drivers/media/i2c/imx334.c
@@ -109,6 +109,7 @@
 /* CSI2 HW configuration */
 #define IMX334_LINK_FREQ_891M		891000000
 #define IMX334_LINK_FREQ_445M		445500000
+#define IMX334_LINK_FREQ_222M		222750000
 #define IMX334_NUM_DATA_LANES		4
 
 #define IMX334_REG_MIN			0x00
@@ -154,7 +155,6 @@ struct imx334_reg_list {
  * @vblank_min: Minimal vertical blanking in lines
  * @vblank_max: Maximum vertical blanking in lines
  * @pclk: Sensor pixel clock
- * @link_freq_idx: Link frequency index
  * @reg_list: Register list for sensor mode
  */
 struct imx334_mode {
@@ -165,7 +165,28 @@ struct imx334_mode {
 	u32 vblank_min;
 	u32 vblank_max;
 	u64 pclk;
-	u32 link_freq_idx;
+	struct imx334_reg_list reg_list;
+};
+
+/**
+ * struct imx334_clk_params - imx334 sensor clock parameters
+ * @data_rate_per_lane: Data rate per lane in bits per second
+ * @link_freq: Link frequency in Hz
+ * @width_max: Maximum image width in pixels
+ * @height_max: Maximum image height in pixels
+ * @width_min: Minimum image width in pixels
+ * @height_min: Minimum image height in pixels
+ * @default_mode: Pointer to the default sensor mode
+ * @reg_list: Register list for clock configuration
+ */
+struct imx334_clk_params {
+	u32 data_rate_per_lane;
+	u32 link_freq;
+	u32 width_max;
+	u32 height_max;
+	u32 width_min;
+	u32 height_min;
+	const struct imx334_mode *default_mode;
 	struct imx334_reg_list reg_list;
 };
 
@@ -216,6 +237,7 @@ struct imx334 {
 static const s64 link_freq[] = {
 	IMX334_LINK_FREQ_891M,
 	IMX334_LINK_FREQ_445M,
+	IMX334_LINK_FREQ_222M,
 };
 
 /* Sensor common mode registers values */
@@ -233,13 +255,6 @@ static const struct cci_reg_sequence common_mode_regs[] = {
 	{ IMX334_REG_UNREAD_PARAM6,	0x0008 },
 	{ IMX334_REG_XVS_XHS_OUTSEL,	0x20 },
 	{ IMX334_REG_XVS_XHS_DRV,	0x0f },
-	{ IMX334_REG_BCWAIT_TIME,	0x3b },
-	{ IMX334_REG_CPWAIT_TIME,	0x2a },
-	{ IMX334_REG_INCKSEL1,		0x0129 },
-	{ IMX334_REG_INCKSEL2,		0x06 },
-	{ IMX334_REG_INCKSEL3,		0xa0 },
-	{ IMX334_REG_INCKSEL4,		0x7e },
-	{ IMX334_REG_SYS_MODE,		0x02 },
 	{ IMX334_REG_HADD_VADD,		0x00 },
 	{ IMX334_REG_VALID_EXPAND,	0x03 },
 	{ IMX334_REG_TCYCLE,		0x00 },
@@ -397,6 +412,39 @@ static const struct cci_reg_sequence mode_3840x2160_regs[] = {
 	{ IMX334_REG_TPLX,		0x005f },
 };
 
+/* Data rate 1782Mbps per lane and 891Mhz link frequency */
+static const struct cci_reg_sequence link_freq_891m_regs[] = {
+	{ IMX334_REG_BCWAIT_TIME,	0x3b },
+	{ IMX334_REG_CPWAIT_TIME,	0x2a },
+	{ IMX334_REG_INCKSEL1,		0x0129 },
+	{ IMX334_REG_INCKSEL2,		0x02 },
+	{ IMX334_REG_INCKSEL3,		0xa0 },
+	{ IMX334_REG_INCKSEL4,		0x7e },
+	{ IMX334_REG_SYS_MODE,		0x00 },
+};
+
+/* Data rate 891Mbps per lane and 445Mhz link frequency */
+static const struct cci_reg_sequence link_freq_445m_regs[] = {
+	{ IMX334_REG_BCWAIT_TIME,	0x3b },
+	{ IMX334_REG_CPWAIT_TIME,	0x2a },
+	{ IMX334_REG_INCKSEL1,		0x0129 },
+	{ IMX334_REG_INCKSEL2,		0x06 },
+	{ IMX334_REG_INCKSEL3,		0xa0 },
+	{ IMX334_REG_INCKSEL4,		0x7e },
+	{ IMX334_REG_SYS_MODE,		0x02 },
+};
+
+/* Data rate 445Mbps per lane and 222Mhz link frequency */
+static const struct cci_reg_sequence link_freq_222m_regs[] = {
+	{ IMX334_REG_BCWAIT_TIME,	0x3b },
+	{ IMX334_REG_CPWAIT_TIME,	0x2a },
+	{ IMX334_REG_INCKSEL1,		0x0129 },
+	{ IMX334_REG_INCKSEL2,		0x0a },
+	{ IMX334_REG_INCKSEL3,		0xa0 },
+	{ IMX334_REG_INCKSEL4,		0x7e },
+	{ IMX334_REG_SYS_MODE,		0x02 },
+};
+
 static const char * const imx334_test_pattern_menu[] = {
 	"Disabled",
 	"Vertical Color Bars",
@@ -442,7 +490,6 @@ static const struct imx334_mode supported_modes[] = {
 		.vblank_min = 90,
 		.vblank_max = 132840,
 		.pclk = 594000000,
-		.link_freq_idx = 0,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_3840x2160_regs),
 			.regs = mode_3840x2160_regs,
@@ -455,7 +502,6 @@ static const struct imx334_mode supported_modes[] = {
 		.vblank_min = 45,
 		.vblank_max = 132840,
 		.pclk = 297000000,
-		.link_freq_idx = 1,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1920x1080_regs),
 			.regs = mode_1920x1080_regs,
@@ -468,7 +514,6 @@ static const struct imx334_mode supported_modes[] = {
 		.vblank_min = 45,
 		.vblank_max = 132840,
 		.pclk = 297000000,
-		.link_freq_idx = 1,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_1280x720_regs),
 			.regs = mode_1280x720_regs,
@@ -481,7 +526,6 @@ static const struct imx334_mode supported_modes[] = {
 		.vblank_min = 45,
 		.vblank_max = 132840,
 		.pclk = 297000000,
-		.link_freq_idx = 1,
 		.reg_list = {
 			.num_of_regs = ARRAY_SIZE(mode_640x480_regs),
 			.regs = mode_640x480_regs,
@@ -489,6 +533,46 @@ static const struct imx334_mode supported_modes[] = {
 	},
 };
 
+static const struct imx334_clk_params imx334_clk_params[] = {
+	{
+		.data_rate_per_lane = 1782000000,
+		.link_freq = IMX334_LINK_FREQ_891M,
+		.width_max = 3840,
+		.height_max = 2160,
+		.width_min = 3840,
+		.height_min = 2160,
+		.default_mode = &supported_modes[0], /* 3840x2160 */
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(link_freq_891m_regs),
+			.regs = link_freq_891m_regs,
+		},
+	}, {
+		.data_rate_per_lane = 891000000,
+		.link_freq = IMX334_LINK_FREQ_445M,
+		.width_max = 1920,
+		.height_max = 1080,
+		.width_min = 640,
+		.height_min = 480,
+		.default_mode = &supported_modes[1], /* 1920x1080 */
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(link_freq_445m_regs),
+			.regs = link_freq_445m_regs,
+		},
+	}, {
+		.data_rate_per_lane = 445500000,
+		.link_freq = IMX334_LINK_FREQ_222M,
+		.width_max = 1920,
+		.height_max = 1080,
+		.width_min = 640,
+		.height_min = 480,
+		.default_mode = &supported_modes[1], /* 1920x1080 */
+		.reg_list = {
+			.num_of_regs = ARRAY_SIZE(link_freq_222m_regs),
+			.regs = link_freq_222m_regs,
+		},
+	}
+};
+
 /**
  * to_imx334() - imv334 V4L2 sub-device to imx334 device.
  * @subdev: pointer to imx334 V4L2 sub-device
@@ -512,10 +596,6 @@ static int imx334_update_controls(struct imx334 *imx334,
 {
 	int ret;
 
-	ret = __v4l2_ctrl_s_ctrl(imx334->link_freq_ctrl, mode->link_freq_idx);
-	if (ret)
-		return ret;
-
 	ret = __v4l2_ctrl_modify_range(imx334->pclk_ctrl, mode->pclk,
 				       mode->pclk, 1, mode->pclk);
 	if (ret)
@@ -741,11 +821,13 @@ static int imx334_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int imx334_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
 	struct imx334 *imx334 = to_imx334(sd);
 	const struct imx334_mode *mode;
+	const struct imx334_clk_params *clk_params;
 	int ret = 0;
 
 	mode = v4l2_find_nearest_size(supported_modes,
@@ -753,6 +835,11 @@ static int imx334_set_pad_format(struct v4l2_subdev *sd,
 				      width, height,
 				      fmt->format.width, fmt->format.height);
 
+	clk_params = &imx334_clk_params[imx334->link_freq_ctrl->val];
+	if (mode->width > clk_params->width_max || mode->height > clk_params->height_max ||
+	    mode->width < clk_params->width_min || mode->height < clk_params->height_min)
+		mode = clk_params->default_mode;
+
 	imx334_fill_pad_format(imx334, mode, fmt);
 	fmt->format.code = imx334_get_format_code(imx334, fmt->format.code);
 
@@ -786,7 +873,7 @@ static int imx334_init_state(struct v4l2_subdev *sd,
 				 ~(imx334->link_freq_bitmap),
 				 __ffs(imx334->link_freq_bitmap));
 
-	return imx334_set_pad_format(sd, sd_state, &fmt);
+	return imx334_set_pad_format(sd, NULL, sd_state, &fmt);
 }
 
 static int imx334_set_framefmt(struct imx334 *imx334)
@@ -824,6 +911,15 @@ static int imx334_enable_streams(struct v4l2_subdev *sd,
 		goto err_rpm_put;
 	}
 
+	/* Write sensor link freq registers */
+	reg_list = &imx334_clk_params[imx334->link_freq_ctrl->val].reg_list;
+	ret = cci_multi_reg_write(imx334->cci, reg_list->regs,
+				  reg_list->num_of_regs, NULL);
+	if (ret) {
+		dev_err(imx334->dev, "fail to write initial registers\n");
+		goto err_rpm_put;
+	}
+
 	/* Write sensor mode registers */
 	reg_list = &imx334->cur_mode->reg_list;
 	ret = cci_multi_reg_write(imx334->cci, reg_list->regs,
@@ -1096,9 +1192,6 @@ static int imx334_init_controls(struct imx334 *imx334)
 							__ffs(imx334->link_freq_bitmap),
 							link_freq);
 
-	if (imx334->link_freq_ctrl)
-		imx334->link_freq_ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY;
-
 	imx334->hblank_ctrl = v4l2_ctrl_new_std(ctrl_hdlr,
 						&imx334_ctrl_ops,
 						V4L2_CID_HBLANK,
diff --git a/drivers/media/i2c/imx335.c b/drivers/media/i2c/imx335.c
index 1f777a1a819255..8f29d9f2da83fa 100644
--- a/drivers/media/i2c/imx335.c
+++ b/drivers/media/i2c/imx335.c
@@ -844,6 +844,7 @@ static void imx335_fill_pad_format(struct imx335 *imx335,
 }
 
 static int imx335_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -901,10 +902,11 @@ static int imx335_init_state(struct v4l2_subdev *sd,
 				 ~(imx335->link_freq_bitmap),
 				 __ffs(imx335->link_freq_bitmap));
 
-	return imx335_set_pad_format(sd, sd_state, &fmt);
+	return imx335_set_pad_format(sd, NULL, sd_state, &fmt);
 }
 
 static int imx335_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c
index 8eb8588cb71bb0..45e3002bd93cad 100644
--- a/drivers/media/i2c/imx355.c
+++ b/drivers/media/i2c/imx355.c
@@ -714,6 +714,7 @@ static void imx355_update_pad_format(struct imx355 *imx355,
 
 static int
 imx355_set_pad_format(struct v4l2_subdev *sd,
+		      const struct v4l2_subdev_client_info *ci,
 		      struct v4l2_subdev_state *sd_state,
 		      struct v4l2_subdev_format *fmt)
 {
@@ -765,6 +766,7 @@ imx355_set_pad_format(struct v4l2_subdev *sd,
 }
 
 static int imx355_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -796,7 +798,7 @@ static int imx355_entity_init_state(struct v4l2_subdev *subdev,
 	fmt.format.width = supported_modes[0].width;
 	fmt.format.height = supported_modes[0].height;
 
-	imx355_set_pad_format(subdev, sd_state, &fmt);
+	imx355_set_pad_format(subdev, NULL, sd_state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c
index d2760e9e6194e4..4628e2eda231ad 100644
--- a/drivers/media/i2c/imx412.c
+++ b/drivers/media/i2c/imx412.c
@@ -588,6 +588,7 @@ static int imx412_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int imx412_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -621,7 +622,7 @@ static int imx412_init_state(struct v4l2_subdev *sd,
 	fmt.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE;
 	imx412_fill_pad_format(imx412, &supported_mode, &fmt);
 
-	return imx412_set_pad_format(sd, sd_state, &fmt);
+	return imx412_set_pad_format(sd, NULL, sd_state, &fmt);
 }
 
 /**
diff --git a/drivers/media/i2c/imx415.c b/drivers/media/i2c/imx415.c
index c3b22b238ea0bd..2571718cffff30 100644
--- a/drivers/media/i2c/imx415.c
+++ b/drivers/media/i2c/imx415.c
@@ -1021,6 +1021,7 @@ static int imx415_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int imx415_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *fmt)
 {
@@ -1042,6 +1043,7 @@ static int imx415_set_format(struct v4l2_subdev *sd,
 }
 
 static int imx415_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1070,7 +1072,7 @@ static int imx415_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	imx415_set_format(sd, state, &format);
+	imx415_set_format(sd, NULL, state, &format);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/imx471.c b/drivers/media/i2c/imx471.c
index 4053aed8434048..03173197f33276 100644
--- a/drivers/media/i2c/imx471.c
+++ b/drivers/media/i2c/imx471.c
@@ -41,7 +41,7 @@
 /* Analog gain control */
 #define IMX471_REG_ANALOG_GAIN			CCI_REG16(0x0204)
 #define IMX471_ANA_GAIN_MIN			0
-#define IMX471_ANA_GAIN_MAX			800
+#define IMX471_ANA_GAIN_MAX			960
 #define IMX471_ANA_GAIN_STEP			1
 #define IMX471_ANA_GAIN_DEFAULT			0
 
@@ -416,6 +416,7 @@ static void imx471_update_pad_format(struct imx471 *sensor,
 }
 
 static int imx471_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -455,6 +456,7 @@ static int imx471_set_pad_format(struct v4l2_subdev *sd,
 }
 
 static int imx471_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -494,7 +496,7 @@ static int imx471_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	return imx471_set_pad_format(sd, sd_state, &fmt);
+	return imx471_set_pad_format(sd, NULL, sd_state, &fmt);
 }
 
 static int imx471_identify_module(struct imx471 *sensor)
diff --git a/drivers/media/i2c/imx678.c b/drivers/media/i2c/imx678.c
index 0efbf43d2fe6c1..8413fd44d65754 100644
--- a/drivers/media/i2c/imx678.c
+++ b/drivers/media/i2c/imx678.c
@@ -849,6 +849,7 @@ static int imx678_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int imx678_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1130,7 +1131,6 @@ static const struct v4l2_subdev_video_ops imx678_video_ops = {
 static const struct v4l2_subdev_pad_ops imx678_pad_ops = {
 	.enum_mbus_code = imx678_enum_mbus_code,
 	.get_fmt = v4l2_subdev_get_fmt,
-	.set_fmt = v4l2_subdev_get_fmt,
 	.get_selection = imx678_get_selection,
 	.enum_frame_size = imx678_enum_frame_size,
 	.enable_streams = imx678_enable_streams,
diff --git a/drivers/media/i2c/isl7998x.c b/drivers/media/i2c/isl7998x.c
index a77538d2343c1b..adb49eb6264272 100644
--- a/drivers/media/i2c/isl7998x.c
+++ b/drivers/media/i2c/isl7998x.c
@@ -1027,6 +1027,7 @@ static int isl7998x_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int isl7998x_set_fmt(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *format)
 {
@@ -1460,8 +1461,7 @@ static int isl7998x_probe(struct i2c_client *client)
 	int nr_inputs;
 	int ret;
 
-	ret = i2c_check_functionality(adapter, I2C_FUNC_SMBUS_WORD_DATA);
-	if (!ret) {
+	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_WORD_DATA)) {
 		dev_warn(&adapter->dev,
 			 "I2C-Adapter doesn't support I2C_FUNC_SMBUS_WORD\n");
 		return -EIO;
diff --git a/drivers/media/i2c/it6625.c b/drivers/media/i2c/it6625.c
new file mode 100644
index 00000000000000..25f9e61a7b03ff
--- /dev/null
+++ b/drivers/media/i2c/it6625.c
@@ -0,0 +1,2391 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * it6625 - ite HDMI to MIPI bridge
+ */
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/hdmi.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_graph.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/timer.h>
+#include <linux/v4l2-dv-timings.h>
+#include <linux/videodev2.h>
+#include <linux/workqueue.h>
+
+#include <media/cec.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dv-timings.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fwnode.h>
+#include <uapi/linux/it6625.h>
+
+static int debug = 3;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "debug level (0-3)");
+
+#define REG_CHIP_ID_0 0x00
+#define REG_CHIP_ID_1 0x01
+#define REG_FW_VER_MAJOR 0x03
+#define REG_FW_VER_MINOR 0x04
+#define REG_PROTOCOL_VERSION 0x05
+#define B_PVER_MINOR  BIT(0)
+#define B_PVER_MAJOR  BIT(4)
+
+#define REG_CMD_SET 0x10
+#define CMD_SET_CEC_LA 0xC0
+#define CMD_SET_CEC_ENABLE 0xC1
+
+#define REG_EDID_START 0x20
+#define REG_CEC_RX_DATA 0x20
+#define REG_CEC_TX_DATA 0x30
+
+#define REG_H_ACTIVE_1 0x50
+#define REG_H_ACTIVE_0 0x51
+#define REG_V_ACTIVE_1 0x52
+#define REG_V_ACTIVE_0 0x53
+#define REG_H_TOTAL_1 0x54
+#define REG_H_TOTAL_0 0x55
+#define REG_V_TOTAL_1 0x56
+#define REG_V_TOTAL_0 0x57
+#define REG_VID_PCLK 0x58
+#define REG_H_FP_1 0x5C
+#define REG_H_FP_0 0x5D
+#define REG_H_SW_1 0x5E
+#define REG_H_SW_0 0x5F
+#define REG_H_BP_1 0x60
+#define REG_H_BP_0 0x61
+#define REG_V_FP_1 0x62
+#define REG_V_FP_0 0x63
+#define REG_V_SW_1 0x64
+#define REG_V_SW_0 0x65
+#define REG_V_BP_1 0x66
+#define REG_V_BP_0 0x67
+#define REG_VID_INFO 0x68
+#define B_INTERLACE  BIT(0)
+#define B_HSWPOL  BIT(1)
+#define B_VSWPOL  BIT(2)
+#define B_PIXEL_REP  BIT(4)
+
+#define REG_VIC 0x69
+#define REG_HDMI_VIDEO_INFO 0x6A
+#define B_COLOR_MODE  BIT(0)
+#define B_COLOR_DEPTH  BIT(4)
+#define REG_HDMI_AUDIO_INFO1 0x6B
+#define B_AUD_FS  BIT(0)
+
+#define REG_HDMI_AUDIO_INFO2 0x6C
+#define B_AUD_CH  BIT(0)
+#define B_AUD_WL  BIT(4)
+
+#define REG_HDMI_AUDIO_INFO3 0x6D
+#define B_AUD_TYPE  BIT(0)
+#define B_AUD_MS  BIT(2)
+#define B_AUD_3D  BIT(3)
+
+#define REG_AUDIO_FMT 0x6E
+#define B_I2S_WL  BIT(0)
+#define B_I2S_ALN BIT(2)
+#define B_I2S_DLY BIT(3)
+#define B_I2S_LR  BIT(4)
+#define B_I2S_SFT BIT(5)
+#define B_AUD_OUT_IF  BIT(6)
+
+#define REG_IF_LATCH_HB 0x6F
+#define REG_IF_DATA 0x70
+#define REG_EMP_DATA 0xA0
+#define REG_AVI_DATA 0xAE
+
+#define REG_TX_STATUS 0xE0
+
+#define REG_RX_STATUS 0xE2
+#define B_RX_5V BIT(0)
+#define B_RX_HPD BIT(1)
+#define B_RX_STABLE BIT(2)
+#define B_RX_HDMI BIT(3)
+#define B_RX_AVMUTE BIT(4)
+#define B_RX_AUD_ON BIT(5)
+
+#define REG_RX_HDCP_STS 0xE3
+#define B_HDCP1_AUTH_START BIT(0)
+#define B_HDCP2_AUTH_START BIT(1)
+#define B_HDCP_AUTH_DONE BIT(2)
+#define B_HDCP_ENC BIT(3)
+
+#define REG_CEC_STATUS 0xE4
+#define B_CEC_TX_DONE BIT(0)
+#define B_CEC_TX_NACK BIT(1)
+
+#define REG_CEC_RX_DATA_LEN 0xE5
+#define REG_CEC_TX_DATA_LEN 0xE6
+
+#define REG_SYS_MIPI_INT 0xEB
+#define B_MIPI_OUTPUT_ENABLE BIT(0)
+#define B_MIPI_VIDEO_UNSTABLE BIT(1)
+
+#define REG_RX_INT_STATUS1 0xEC
+#define B_HDMI_5V_CHG   BIT(0)
+#define B_HDMI_VID_CHG  BIT(1)
+#define B_HDMI_AUD_CHG  BIT(2)
+#define B_HDMI_CP_CHG   BIT(3)
+#define B_HDMI_IF_LATCH BIT(4)
+#define B_HDMI_NO_IF_LATCH  BIT(5)
+#define B_HDMI_EMP      BIT(6)
+#define B_HDMI_NO_EMP   BIT(7)
+
+#define REG_RX_INT_STATUS2 0xED
+#define B_HDMI_AVI      BIT(0)
+#define B_HDMI_NO_AVI   BIT(1)
+#define B_HDMI_VSIF     BIT(2)
+#define B_HDMI_NO_VSIF  BIT(3)
+#define B_HDMI_AVMUTE_CHG  BIT(4)
+
+#define REG_CHIP_CONTROL 0xF0
+#define B_HDMI_RESET  BIT(5)
+#define B_FW_START  BIT(6)
+
+#define REG_MIPI_CFG 0xF1
+#define M_MIPI_LANE  0x03
+#define B_MIPI_SPLIT  BIT(2)
+#define B_MIPI_SPLIT_CFG  BIT(3)
+#define B_MIPI_DPHY  BIT(4)
+#define B_MIPI_USE_DSI  BIT(5)
+#define B_MIPI_CONTINU_CLK  BIT(6)
+
+#define REG_MIPI_DATA_TYPE 0xF2
+#define CSI_RGB444          0x20
+#define CSI_RGB555          0x21
+#define CSI_RGB565          0x22
+#define CSI_RGB666          0x23
+#define CSI_RGB888          0x24
+#define CSI_YUV420_8b_L     0x1A
+#define CSI_YUV420_8b       0x1C
+#define CSI_YUV420_10b      0x1D
+#define CSI_YUV422_8b       0x1E
+#define CSI_YUV422_10b      0x1F
+#define CSI_RGB_10b         0x30
+#define CSI_RGB_12b         0x31
+#define CSI_YUV422_12b      0x32
+#define CSI_YUV420_10b_L    0x33
+#define CSI_YUV420_12b      0x34
+#define CSI_YUV444_8b       0x35
+#define CSI_YUV444_10b      0x36
+#define CSI_YUV444_12b      0x37
+
+#define REG_MIPI_CONTROL 0xF3
+#define B_MIPI_OUTPUT    BIT(0)
+
+#define REG_RX_CFG 0xF4
+#define B_MANUAL_HPD    BIT(0)
+#define B_HPD_HIGH    BIT(1)
+#define B_HPD_TOGGLE    BIT(3)
+
+#define REG_CSC_CFG 0xF5
+#define B_DYNAMIC_RANGE BIT(0)
+
+#define REG_MISC_CFG 0xF6
+#define B_BAUD_RATE BIT(0)
+#define B_DEBUG_MSG BIT(1)
+
+#define REG_HPD_DELAY 0xF7
+#define B_DELAY_COUNT BIT(0)
+#define B_DELAY_UNIT BIT(7)
+
+#define REG_INFO_BANK_SEL 0xFD
+#define CTL_BANK_EDID_READ 1
+#define CTL_BANK_EDID_WRITE 5
+
+#define REG_HOST_CTRL_INT 0xFE
+#define B_CMD_SET BIT(4)
+#define B_CEC_SEND_DATA BIT(5)
+#define B_CONFIG_UPDATE BIT(6)
+#define B_IF_BANK BIT(7)
+
+#define REG_MCU_INTERRUPT 0xFF
+#define B_SYS_INT_ACTIVE BIT(0)
+#define B_CEC_RX_RECEIVED BIT(1)
+#define B_CEC_TX_UPDATE BIT(2)
+
+#define EDID_NUM_BLOCKS_MAX 4
+#define EDID_BLOCK_SIZE 128
+
+#define I2C_MAX_XFER_SIZE  8
+#define POLL_INTERVAL_CEC_MS	10
+#define POLL_INTERVAL_MS	40
+
+#define AUD32K   0x03
+#define AUD44K   0x00
+#define AUD48K   0x02
+#define AUD64K   0x0B
+#define AUD88K   0x08
+#define AUD96K   0x0A
+#define AUD128K  0x2B
+#define AUD176K  0x0C
+#define AUD192K  0x0E
+#define AUD256K  0x1B
+#define AUD352K  0x0D
+#define AUD384K  0x05
+#define AUD512K  0x3B
+#define AUD705K  0x2D
+#define AUD768K  0x09
+#define AUD1024K 0x35
+#define AUD1411K 0x1D
+#define AUD1536K 0x15
+
+enum it6625_chip_type {
+	IT6625_CHIP = 0,
+	IT6626_CHIP = 1,
+};
+
+struct it6625 {
+	struct device *dev;
+	struct i2c_client *i2c_client;
+	struct regmap *it6625_regmap;
+	enum it6625_chip_type chip_type;
+
+	/* protects concurrent access to the chip's registers and state */
+	struct mutex it6625_lock;
+	/* serializes the complete VIDIOC_S_EDID sequence against itself */
+	struct mutex edid_lock;
+	/* serializes a full InfoFrame debugfs read transaction against itself */
+	struct mutex if_read_lock;
+	/*
+	 * protects if_active/if_type/if_snapshot/if_snapshot_err/
+	 * if_snapshot_done and the REG_MCU_INTERRUPT/REG_RX_INT_STATUS1/2/
+	 * REG_IF_LATCH_HB/REG_IF_DATA register group between the interrupt
+	 * path and the InfoFrame debugfs callback. Never held across
+	 * wait_for_completion_timeout(). Nests outside it6625_lock.
+	 */
+	struct mutex if_state_lock;
+
+	struct v4l2_subdev sd;
+	struct v4l2_mbus_config_mipi_csi2 bus;
+	struct video_device *vdev;
+	struct media_pad pad;
+	struct v4l2_ctrl_handler hdl;
+
+	/* controls */
+	struct v4l2_ctrl *ctrl_5v_detect;
+	struct v4l2_ctrl *ctrl_audio_sampling_rate;
+	struct v4l2_ctrl *ctrl_audio_present;
+	struct v4l2_ctrl *ctrl_link_freq;
+
+	struct delayed_work hpd_delayed_work;
+
+	struct timer_list timer;
+	struct work_struct polling_work;
+
+	struct v4l2_dv_timings timings;
+
+	u8 csi_lanes;
+	u8 port_num;
+	enum v4l2_mbus_type bus_type;
+	u8 csi_format;
+	u32 mbus_fmt_code;
+	/* number of EDID blocks currently loaded, protected by edid_lock */
+	u8 edid_blocks;
+
+	struct gpio_desc *reset_gpio;
+
+	struct cec_adapter *cec_adap;
+
+	struct dentry *debugfs_dir;
+
+	struct v4l2_debugfs_if *infoframes;
+	struct completion if_latched;
+	/* true while an InfoFrame debugfs request is outstanding */
+	bool if_active;
+	/* HDMI packet-type byte currently armed in REG_IF_LATCH_HB */
+	u32 if_type;
+	/* driver-private copy of REG_IF_DATA, captured at the genuine latch */
+	u8 if_snapshot[31];
+	/* it6625_read_bytes() result for the if_snapshot capture */
+	int if_snapshot_err;
+	/* set just before complete(), to disambiguate timeout vs. capture */
+	bool if_snapshot_done;
+};
+
+/*
+ * Index 0: D-PHY (4-lane). Index 1: C-PHY (3-trio) -- the confirmed
+ * hardware max C-PHY capability, tested single-port/three-trio.
+ */
+static const s64 it6625_link_freq[] = {
+	445500000,
+	2500000000LL,
+};
+
+/*
+ * Shared cap for every topology except the reference exception below:
+ * IT6625 (no C-PHY support at all), IT6626 running D-PHY, and IT6626
+ * running C-PHY with fewer than three trios. This is conservative
+ * scoping, not a claim that one-/two-trio C-PHY can't also support a
+ * higher rate -- they're simply unvalidated.
+ */
+static const struct v4l2_dv_timings_cap it6625_timings_cap = {
+	.type = V4L2_DV_BT_656_1120,
+	/* keep this initialization for compatibility with GCC < 4.4.6 */
+	.reserved = { 0 },
+
+	V4L2_INIT_BT_TIMINGS(640, 3840, 480, 2160, 27000000, 300000000,
+			     V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_DMT |
+			     V4L2_DV_BT_STD_GTF | V4L2_DV_BT_STD_CVT,
+			     V4L2_DV_BT_CAP_PROGRESSIVE | V4L2_DV_BT_CAP_INTERLACED |
+			     V4L2_DV_BT_CAP_REDUCED_BLANKING | V4L2_DV_BT_CAP_CUSTOM)
+};
+
+/*
+ * IT6626 C-PHY, three trios: raised pixel-clock ceiling (594 MHz vs
+ * the shared 300 MHz cap) for this topology's higher C-PHY capability.
+ */
+static const struct v4l2_dv_timings_cap it6626_cphy_3trio_timings_cap = {
+	.type = V4L2_DV_BT_656_1120,
+	.reserved = { 0 },
+
+	V4L2_INIT_BT_TIMINGS(640, 3840, 480, 2160, 27000000, 594000000,
+			     V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_DMT |
+			     V4L2_DV_BT_STD_GTF | V4L2_DV_BT_STD_CVT,
+			     V4L2_DV_BT_CAP_PROGRESSIVE | V4L2_DV_BT_CAP_INTERLACED |
+			     V4L2_DV_BT_CAP_REDUCED_BLANKING | V4L2_DV_BT_CAP_CUSTOM)
+};
+
+static const struct v4l2_dv_timings_cap *
+it6625_get_timings_cap(struct it6625 *it6625)
+{
+	if (it6625->chip_type == IT6626_CHIP &&
+	    it6625->bus_type == V4L2_MBUS_CSI2_CPHY &&
+	    it6625->csi_lanes == 3)
+		return &it6626_cphy_3trio_timings_cap;
+
+	return &it6625_timings_cap;
+}
+
+static const struct it6625_format_info {
+	u8 csi_format;
+	u32 mbus_fmt_code;
+} it6625_formats[] = {
+	{ CSI_YUV422_8b, MEDIA_BUS_FMT_UYVY8_1X16 },
+	{ CSI_RGB888, MEDIA_BUS_FMT_RGB888_1X24 },
+	{ CSI_YUV444_8b, MEDIA_BUS_FMT_YUV8_1X24 },
+};
+
+static inline int it6625_csi_format_idx(u8 csi_format)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(it6625_formats); i++) {
+		if (it6625_formats[i].csi_format == csi_format)
+			return i;
+	}
+
+	return -EINVAL;
+}
+
+static inline int it6625_csi_mbus_code_idx(u32 mbus_fmt_code)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(it6625_formats); i++) {
+		if (it6625_formats[i].mbus_fmt_code == mbus_fmt_code)
+			return i;
+	}
+
+	return -EINVAL;
+}
+
+static inline struct it6625 *sd_to_6625(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct it6625, sd);
+}
+
+static const struct regmap_config it6625_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = 0xff,
+	.cache_type = REGCACHE_NONE,
+	.max_raw_read = I2C_MAX_XFER_SIZE,
+	.max_raw_write = I2C_MAX_XFER_SIZE,
+};
+
+static int it6625_regmap_i2c_init(struct i2c_client *client,
+				  struct it6625 *it6625)
+{
+	it6625->i2c_client = client;
+	it6625->dev = &client->dev;
+
+	it6625->it6625_regmap = devm_regmap_init_i2c(it6625->i2c_client,
+						     &it6625_regmap_config);
+	if (IS_ERR(it6625->it6625_regmap))
+		return PTR_ERR(it6625->it6625_regmap);
+
+	return 0;
+}
+
+static int it6625_read_byte(struct it6625 *it6625, u8 reg)
+{
+	unsigned int val;
+	int err;
+	struct device *dev = it6625->dev;
+
+	err = regmap_read(it6625->it6625_regmap, reg, &val);
+	if (err < 0) {
+		dev_err(dev, "reg[0x%x] read failed err: %d", reg, err);
+		return err;
+	}
+
+	return val;
+}
+
+static int it6625_write_byte(struct it6625 *it6625, u8 reg, u8 val)
+{
+	int err;
+	struct device *dev = it6625->dev;
+
+	err = regmap_write(it6625->it6625_regmap, reg, val);
+	if (err < 0) {
+		dev_err(dev, "reg[0x%x] write failed err: %d", reg, err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int it6625_set_bits(struct it6625 *it6625, u8 reg, u8 mask, u8 val)
+{
+	int err;
+	struct device *dev = it6625->dev;
+
+	err = regmap_update_bits(it6625->it6625_regmap, reg, mask, val);
+	if (err < 0) {
+		dev_err(dev, "reg[0x%x] set bits failed err: %d", reg, err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int it6625_read_bytes(struct it6625 *it6625, u8 reg, u8 *buf, int len)
+{
+	int err;
+	struct device *dev = it6625->dev;
+
+	err = regmap_bulk_read(it6625->it6625_regmap, reg, buf, len);
+	if (err < 0) {
+		dev_err(dev, "reg[0x%x] read failed err: %d", reg, err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int it6625_write_bytes(struct it6625 *it6625, u8 reg, u8 *buf, int len)
+{
+	int err;
+	struct device *dev = it6625->dev;
+
+	err = regmap_bulk_write(it6625->it6625_regmap, reg, buf, len);
+	if (err < 0) {
+		dev_err(dev, "reg[0x%x] write failed err: %d", reg, err);
+		return err;
+	}
+
+	return 0;
+}
+
+static int it6625_wait_for_status(struct it6625 *it6625, u8 reg, u8 val,
+				  int timeout_ms)
+{
+	struct device *dev = it6625->dev;
+	int status;
+	int rval;
+	int sleep_ms = 10;
+	int timeout_round_ms = DIV_ROUND_UP(timeout_ms, sleep_ms) * sleep_ms;
+
+	status = read_poll_timeout(it6625_read_byte, rval, rval == val,
+				   sleep_ms * 1000,
+				   timeout_round_ms * 1000,
+				   false, it6625, reg);
+
+	dev_info(dev, "%s status = %d %d", __func__, status, (int)rval);
+	if (status < 0) {
+		dev_err(dev, "%s err status = %d", __func__, status);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void it6625_write_command(struct it6625 *it6625, u8 *cmds, int cmd_len)
+{
+	it6625_write_bytes(it6625, REG_CMD_SET, cmds, cmd_len);
+	it6625_write_byte(it6625, REG_HOST_CTRL_INT, B_CMD_SET);
+	it6625_wait_for_status(it6625, REG_HOST_CTRL_INT, 0x00, 25);
+}
+
+static int it6625_update_config(struct it6625 *it6625)
+{
+	int err;
+
+	err = it6625_set_bits(it6625, REG_HOST_CTRL_INT,
+			      B_CONFIG_UPDATE, B_CONFIG_UPDATE);
+	if (err < 0)
+		return err;
+
+	return it6625_wait_for_status(it6625, REG_HOST_CTRL_INT, 0x00, 25);
+}
+
+static int it6625_set_bank(struct it6625 *it6625, u8 bank)
+{
+	int err;
+
+	err = it6625_write_byte(it6625, REG_INFO_BANK_SEL, bank);
+	if (err < 0)
+		return err;
+
+	err = it6625_write_byte(it6625, REG_HOST_CTRL_INT, B_IF_BANK);
+	if (err < 0)
+		return err;
+
+	return it6625_wait_for_status(it6625, REG_HOST_CTRL_INT, 0x00, 25);
+}
+
+static inline bool is_hdmi(struct it6625 *it6625)
+{
+	int val;
+
+	val = it6625_read_byte(it6625, REG_RX_STATUS);
+	return (val < 0) ? false : (val & B_RX_HDMI);
+}
+
+static inline bool hdmi_5v_power_present(struct it6625 *it6625)
+{
+	int val;
+
+	val = it6625_read_byte(it6625, REG_RX_STATUS);
+	return (val < 0) ? false : (val & B_RX_5V);
+}
+
+static inline bool no_signal(struct it6625 *it6625)
+{
+	int val;
+
+	val = it6625_read_byte(it6625, REG_RX_STATUS);
+	return (val < 0) ? true : !(val & B_RX_STABLE);
+}
+
+static inline bool audio_present(struct it6625 *it6625)
+{
+	int val;
+
+	val = it6625_read_byte(it6625, REG_RX_STATUS);
+	return (val < 0) ? false : (val & B_RX_AUD_ON);
+}
+
+static int get_audio_sampling_rate(struct it6625 *it6625)
+{
+	int fs_id;
+	int i, freq = 0;
+	const struct fs_id_map {
+		u8 fs_id;
+		u32 freq;
+	} s_fsid_map[] = {
+		{ AUD32K,    32000 },
+		{ AUD44K,    44100 },
+		{ AUD48K,    48000 },
+		{ AUD64K,    64000 },
+		{ AUD88K,    88200 },
+		{ AUD96K,    96000 },
+
+		{ AUD128K,  128000 },
+		{ AUD176K,  176400 },
+		{ AUD192K,  192000 },
+		{ AUD256K,  256000 },
+		{ AUD352K,  352800 },
+		{ AUD384K,  384000 },
+
+		{ AUD512K,  512000 },
+		{ AUD705K,  705600 },
+		{ AUD768K,  768000 },
+		{ AUD1024K, 1024000 },
+		{ AUD1411K, 1411200 },
+		{ AUD1536K, 1536000 },
+	};
+
+	if (no_signal(it6625) || !audio_present(it6625))
+		return 0;
+
+	guard(mutex)(&it6625->it6625_lock);
+
+	fs_id = it6625_read_byte(it6625, REG_HDMI_AUDIO_INFO1);
+	if (fs_id < 0)
+		return 0;
+
+	for (i = 0; i < ARRAY_SIZE(s_fsid_map); i++) {
+		if (s_fsid_map[i].fs_id == fs_id) {
+			freq = s_fsid_map[i].freq;
+			break;
+		}
+	}
+
+	return freq;
+}
+
+static u64 it6625_get_pclk(struct it6625 *it6625)
+{
+	u32 pclk;
+	u8 ck[4];
+	int ret;
+
+	ret = it6625_read_bytes(it6625, REG_VID_PCLK, ck, 4);
+	if (ret < 0) {
+		dev_err(it6625->dev, "failed to read pixel clock");
+		return 0;
+	}
+
+	pclk = ck[0];
+	pclk <<= 8;
+	pclk |= ck[1];
+	pclk <<= 8;
+	pclk |= ck[2];
+	pclk <<= 8;
+	pclk |= ck[3];
+
+	v4l2_dbg(1, debug, &it6625->sd, "%s: pclk=%u (%08x)",
+		 __func__, pclk, pclk);
+
+	return (u64)pclk * 1000;
+}
+
+static int it6625_read_edid(struct it6625 *it6625, u8 *edid, int start_block,
+			    int num_blocks)
+{
+	int i, bank_ctrl, err = 0;
+	struct device *dev = it6625->dev;
+
+	if (!edid) {
+		dev_err(dev, "edid buffer is NULL");
+		return -EINVAL;
+	}
+
+	if (start_block < 0 || num_blocks <= 0 ||
+	    start_block > EDID_NUM_BLOCKS_MAX ||
+	    num_blocks > EDID_NUM_BLOCKS_MAX ||
+	    start_block + num_blocks > EDID_NUM_BLOCKS_MAX) {
+		dev_err(dev,
+			"invalid block range: start_block=%d, num_blocks=%d",
+			start_block, num_blocks);
+		return -EINVAL;
+	}
+
+	guard(mutex)(&it6625->it6625_lock);
+	for (i = 0; i < num_blocks; i++) {
+		bank_ctrl = CTL_BANK_EDID_READ + start_block + i;
+		err = it6625_set_bank(it6625, bank_ctrl);
+		if (err < 0)
+			break;
+
+		err = it6625_read_bytes(it6625, REG_EDID_START,
+					edid + (i * 128), 128);
+		if (err < 0)
+			break;
+	}
+
+	it6625_set_bank(it6625, 0);
+
+	return err < 0 ? err : num_blocks;
+}
+
+static int it6625_write_edid(struct it6625 *it6625, u8 *edid, int start_block,
+			     int num_blocks)
+{
+	int i, bank_ctrl, err = 0;
+	struct device *dev = it6625->dev;
+
+	if (start_block < 0 || num_blocks <= 0 ||
+	    start_block > EDID_NUM_BLOCKS_MAX ||
+	    num_blocks > EDID_NUM_BLOCKS_MAX ||
+	    start_block + num_blocks > EDID_NUM_BLOCKS_MAX) {
+		dev_err(dev,
+			"invalid block range: start_block=%d, num_blocks=%d",
+			start_block, num_blocks);
+		return -EINVAL;
+	}
+
+	guard(mutex)(&it6625->it6625_lock);
+	for (i = 0; i < num_blocks; i++) {
+		bank_ctrl = CTL_BANK_EDID_WRITE + start_block + i;
+		err = it6625_set_bank(it6625, bank_ctrl);
+		if (err < 0)
+			break;
+
+		err = it6625_write_bytes(it6625, REG_EDID_START,
+					 edid + (i * 128), 128);
+		if (err < 0)
+			break;
+
+		err = it6625_update_config(it6625);
+		if (err < 0)
+			break;
+	}
+
+	it6625_set_bank(it6625, 0);
+
+	return err < 0 ? err : num_blocks;
+}
+
+static void it6625_enable_auto_hpd(struct it6625 *it6625)
+{
+	dev_dbg(it6625->dev, "%s: auto HPD", __func__);
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_set_bits(it6625, REG_RX_CFG, 0x03, 0x00);
+	it6625_update_config(it6625);
+}
+
+static void it6625_disable_hpd(struct it6625 *it6625)
+{
+	cancel_delayed_work_sync(&it6625->hpd_delayed_work);
+
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_set_bits(it6625, REG_RX_CFG, 0x03, 0x01);
+	it6625_update_config(it6625);
+}
+
+static void it6625_enable_hpd(struct it6625 *it6625)
+{
+	schedule_delayed_work(&it6625->hpd_delayed_work, V4L2_SET_EDID_HPD_LOW_JIFFIES);
+}
+
+static void it6625_hpd_delayed_work(struct work_struct *work)
+{
+	struct it6625 *it6625 = container_of(work,
+			struct it6625, hpd_delayed_work.work);
+	int val = 0;
+
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_set_bits(it6625, REG_RX_CFG, 0x03, val);
+	it6625_update_config(it6625);
+}
+
+static int it6625_get_detected_timings(struct it6625 *it6625,
+				       struct v4l2_dv_timings *timings)
+{
+	struct v4l2_bt_timings *bt = &timings->bt;
+	int val;
+	unsigned int width, height;
+	u8 buffer[4];
+	u8 buffer2[12];
+
+	if (no_signal(it6625)) {
+		dev_err(it6625->dev, "no signal detected");
+		return -ENOLINK;
+	}
+
+	guard(mutex)(&it6625->it6625_lock);
+
+	memset(timings, 0, sizeof(struct v4l2_dv_timings));
+	timings->type = V4L2_DV_BT_656_1120;
+	val = it6625_read_byte(it6625, REG_VID_INFO);
+	if (val < 0) {
+		dev_err(it6625->dev, "failed to read video info");
+		return -EIO;
+	}
+
+	bt->interlaced = val & B_INTERLACE ?
+			 V4L2_DV_INTERLACED : V4L2_DV_PROGRESSIVE;
+
+	if (it6625_read_bytes(it6625, REG_H_ACTIVE_1, buffer, 4) < 0)
+		return -EIO;
+
+	width = ((buffer[0] & 0xff) << 8) + buffer[1];
+	height = ((buffer[2] & 0xff) << 8) + buffer[3];
+
+	bt->width = width;
+	bt->height = height;
+
+	if (it6625_read_bytes(it6625, REG_H_FP_1, buffer2, 12) < 0)
+		return -EIO;
+
+	bt->hfrontporch = ((buffer2[0] & 0xff) << 8) + buffer2[1];
+	bt->hsync = ((buffer2[2] & 0xff) << 8) + buffer2[3];
+	bt->hbackporch = ((buffer2[4] & 0xff) << 8) + buffer2[5];
+	bt->vfrontporch = ((buffer2[6] & 0xff) << 8) + buffer2[7];
+	bt->vsync = ((buffer2[8] & 0xff) << 8) + buffer2[9];
+	bt->vbackporch = ((buffer2[10] & 0xff) << 8) + buffer2[11];
+
+	bt->pixelclock = it6625_get_pclk(it6625);
+	if (bt->interlaced == V4L2_DV_INTERLACED) {
+		bt->height *= 2;
+		bt->il_vsync = bt->vsync + 1;
+	}
+
+	return 0;
+}
+
+static void it6625_show_avi_infoframe(struct it6625 *it6625)
+{
+	struct device *dev = it6625->dev;
+	union hdmi_infoframe frame;
+	u8 buffer[HDMI_INFOFRAME_SIZE(AVI)];
+	u8 ver, len;
+	int ret;
+
+	if (!is_hdmi(it6625)) {
+		dev_err(dev, "not HDMI signal, skip AVI infoframe log");
+		return;
+	}
+
+	ret = it6625_read_bytes(it6625, REG_AVI_DATA, buffer + 1,
+				HDMI_INFOFRAME_SIZE(AVI) - 1);
+	if (ret < 0) {
+		dev_err(dev, "failed to read AVI infoframe data");
+		return;
+	}
+
+	len = buffer[1];
+	ver = buffer[2];
+
+	buffer[0] = HDMI_INFOFRAME_TYPE_AVI;
+	buffer[1] = ver;
+	buffer[2] = len;
+
+	ret = hdmi_infoframe_unpack(&frame, buffer, sizeof(buffer));
+	if (ret < 0) {
+		dev_err(dev, "unpack of AVI infoframe failed");
+		return;
+	}
+
+	hdmi_infoframe_log(KERN_INFO, dev, &frame);
+}
+
+static int it6625_s_ctrl_detect_hdmi_5v(struct v4l2_subdev *sd)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	return v4l2_ctrl_s_ctrl(it6625->ctrl_5v_detect,
+				hdmi_5v_power_present(it6625));
+}
+
+static int it6625_s_ctrl_audio_sampling_rate(struct v4l2_subdev *sd)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	return v4l2_ctrl_s_ctrl(it6625->ctrl_audio_sampling_rate,
+				get_audio_sampling_rate(it6625));
+}
+
+static int it6625_s_ctrl_audio_present(struct v4l2_subdev *sd)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	return v4l2_ctrl_s_ctrl(it6625->ctrl_audio_present,
+				audio_present(it6625));
+}
+
+static void it6625_v4l2_sd_ctrl_update(struct v4l2_subdev *sd)
+{
+	it6625_s_ctrl_detect_hdmi_5v(sd);
+	it6625_s_ctrl_audio_sampling_rate(sd);
+	it6625_s_ctrl_audio_present(sd);
+}
+
+static void it6625_enable_stream_locked(struct it6625 *it6625, bool enable)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+	int val;
+
+	lockdep_assert_held(&it6625->it6625_lock);
+
+	v4l2_dbg(3, debug, sd, "%s: %sable",
+		 __func__, enable ? "en" : "dis");
+
+	val = enable ? B_MIPI_OUTPUT : 0;
+	it6625_set_bits(it6625, REG_MIPI_CONTROL, B_MIPI_OUTPUT, val);
+	it6625_update_config(it6625);
+}
+
+static void it6625_enable_stream(struct it6625 *it6625, bool enable)
+{
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_enable_stream_locked(it6625, enable);
+}
+
+static void it6625_set_mipi_config_locked(struct it6625 *it6625, u32 cfg_val)
+{
+	u8 mipi_data_type;
+
+	lockdep_assert_held(&it6625->it6625_lock);
+
+	dev_dbg(it6625->dev, "mipi_data_type = 0x%x", cfg_val);
+
+	mipi_data_type = cfg_val & 0xFF;
+	it6625_write_byte(it6625, REG_MIPI_DATA_TYPE, mipi_data_type);
+	it6625_update_config(it6625);
+}
+
+static inline unsigned int fps_from_bt_timings(const struct v4l2_bt_timings *t)
+{
+	if (!V4L2_DV_BT_FRAME_HEIGHT(t) || !V4L2_DV_BT_FRAME_WIDTH(t))
+		return 0;
+
+	return DIV_ROUND_CLOSEST((unsigned int)t->pixelclock,
+				 V4L2_DV_BT_FRAME_HEIGHT(t) *
+				 V4L2_DV_BT_FRAME_WIDTH(t));
+}
+
+static void it6625_initial_setup(struct it6625 *it6625)
+{
+	int val = 0;
+
+	guard(mutex)(&it6625->it6625_lock);
+
+	/*
+	 * REG_MIPI_CFG[0:2] lane count field: 1 lane -> 0, 2 lanes -> 1,
+	 * 3 lanes (C-PHY only) -> 3, 4 lanes (D-PHY only) -> 3.
+	 */
+	switch (it6625->csi_lanes) {
+	case 1:
+		val = FIELD_PREP(M_MIPI_LANE, 0);
+		break;
+	case 2:
+		val = FIELD_PREP(M_MIPI_LANE, 1);
+		break;
+	default:
+		val = FIELD_PREP(M_MIPI_LANE, 3);
+		break;
+	}
+
+	if (it6625->bus_type == V4L2_MBUS_CSI2_DPHY)
+		val |= FIELD_PREP(B_MIPI_DPHY, 1);
+
+	if (it6625->port_num == 2)
+		val |= FIELD_PREP(B_MIPI_SPLIT, 1);
+
+	it6625_write_byte(it6625, REG_MIPI_CFG, val);
+	it6625_write_byte(it6625, REG_MIPI_DATA_TYPE, it6625->csi_format);
+	it6625_write_byte(it6625, REG_MIPI_CONTROL, 0x00);
+	it6625_write_byte(it6625, REG_RX_CFG, 0x00);
+
+	it6625_set_bits(it6625, REG_HOST_CTRL_INT, B_CONFIG_UPDATE, B_CONFIG_UPDATE);
+	it6625_wait_for_status(it6625, REG_HOST_CTRL_INT, 0x00, 25);
+}
+
+static int it6625_cec_adap_enable(struct cec_adapter *adap, bool enable)
+{
+	struct it6625 *it6625 = adap->priv;
+	u8 cmds[2];
+
+	cmds[0] = CMD_SET_CEC_ENABLE;
+	cmds[1] = enable ? 1 : 0;
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_write_command(it6625, cmds, sizeof(cmds));
+
+	return 0;
+}
+
+static void it6625_cec_reset_la(struct it6625 *it6625, bool keep_enabled)
+{
+	u8 cmds[2];
+
+	if (keep_enabled) {
+		cmds[0] = CMD_SET_CEC_LA;
+		cmds[1] = CEC_LOG_ADDR_UNREGISTERED;
+	} else {
+		cmds[0] = CMD_SET_CEC_ENABLE;
+		cmds[1] = 0;
+	}
+
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_write_command(it6625, cmds, sizeof(cmds));
+}
+
+static int it6625_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr)
+{
+	struct it6625 *it6625 = adap->priv;
+	u8 cmds[2] = {CMD_SET_CEC_LA, log_addr};
+
+	dev_dbg(it6625->dev, "%s: la=%d", __func__, log_addr);
+
+	if (log_addr == CEC_LOG_ADDR_INVALID) {
+		it6625_cec_reset_la(it6625, adap->is_enabled);
+		return 0;
+	}
+
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_write_command(it6625, cmds, sizeof(cmds));
+
+	return 0;
+}
+
+static int it6625_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
+				    u32 signal_free_time, struct cec_msg *msg)
+{
+	struct it6625 *it6625 = adap->priv;
+
+	guard(mutex)(&it6625->it6625_lock);
+	it6625_write_bytes(it6625, REG_CEC_TX_DATA, msg->msg, msg->len);
+	it6625_write_byte(it6625, REG_CEC_TX_DATA_LEN, msg->len);
+	it6625_set_bits(it6625, REG_HOST_CTRL_INT, B_CEC_SEND_DATA, B_CEC_SEND_DATA);
+	it6625_wait_for_status(it6625, REG_HOST_CTRL_INT, 0x00, 25);
+
+	return 0;
+}
+
+static const struct cec_adap_ops it6625_cec_adap_ops = {
+	.adap_enable = it6625_cec_adap_enable,
+	.adap_log_addr = it6625_cec_adap_log_addr,
+	.adap_transmit = it6625_cec_adap_transmit,
+};
+
+static void it6625_cec_handler(struct it6625 *it6625, u8 intstatus)
+{
+	struct cec_msg rxmsg = {};
+	int val = 0;
+
+	if (intstatus & B_CEC_RX_RECEIVED) {
+		scoped_guard(mutex, &it6625->it6625_lock) {
+			val = it6625_read_byte(it6625, REG_CEC_RX_DATA_LEN);
+			if (val > 0 && val <= CEC_MAX_MSG_SIZE)
+				it6625_read_bytes(it6625, REG_CEC_RX_DATA, &rxmsg.msg[0], val);
+			it6625_write_byte(it6625, REG_CEC_RX_DATA_LEN, 0);
+		}
+
+		if (val > 0 && val <= CEC_MAX_MSG_SIZE) {
+			rxmsg.len = val;
+			cec_received_msg(it6625->cec_adap, &rxmsg);
+		} else {
+			dev_err(it6625->dev, "invalid CEC RX length %d", val);
+		}
+	}
+
+	if (intstatus & B_CEC_TX_UPDATE) {
+		val = it6625_read_byte(it6625, REG_CEC_STATUS);
+		if (val < 0) {
+			dev_err(it6625->dev, "read CEC status failed");
+			return;
+		}
+
+		if (val & BIT(0)) {
+			cec_transmit_attempt_done(it6625->cec_adap,
+						  CEC_TX_STATUS_OK);
+		} else if (val & BIT(1)) {
+			cec_transmit_attempt_done(it6625->cec_adap,
+						  CEC_TX_STATUS_NACK);
+		} else {
+			dev_info(it6625->dev, "unknown CEC status %02X", val);
+			cec_transmit_attempt_done(it6625->cec_adap,
+						  CEC_TX_STATUS_NACK);
+		}
+	}
+}
+
+static void it6625_irq_format_change(struct it6625 *it6625)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+	struct v4l2_dv_timings timings;
+	const struct v4l2_event it6625_ev_fmt = {
+		.type = V4L2_EVENT_SOURCE_CHANGE,
+		.u.src_change.changes = V4L2_EVENT_SRC_CH_RESOLUTION,
+	};
+	int ret;
+
+	if (no_signal(it6625)) {
+		if (sd->devnode)
+			v4l2_subdev_notify_event(sd, &it6625_ev_fmt);
+		return;
+	}
+
+	ret = it6625_get_detected_timings(it6625, &timings);
+	if (ret < 0) {
+		v4l2_dbg(1, debug, sd, "Failed to get detected timings");
+		return;
+	}
+
+	if (sd->devnode)
+		v4l2_subdev_notify_event(sd, &it6625_ev_fmt);
+}
+
+static void it6625_irq_hdmi_audio_change(struct it6625 *it6625)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+
+	it6625_s_ctrl_audio_sampling_rate(sd);
+	it6625_s_ctrl_audio_present(sd);
+}
+
+static void it6625_get_timings(struct it6625 *it6625,
+			       struct v4l2_dv_timings *timings)
+{
+	guard(mutex)(&it6625->it6625_lock);
+	*timings = it6625->timings;
+}
+
+static void it6625_clear_timings(struct it6625 *it6625)
+{
+	guard(mutex)(&it6625->it6625_lock);
+	memset(&it6625->timings, 0, sizeof(it6625->timings));
+}
+
+static void it6625_irq_hdmi_5v_change(struct it6625 *it6625)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+
+	it6625_clear_timings(it6625);
+	it6625_v4l2_sd_ctrl_update(sd);
+}
+
+static void it6625_irq_hdcp_change(struct it6625 *it6625)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+	u8 cp_sts;
+
+	cp_sts = it6625_read_byte(it6625, REG_RX_HDCP_STS);
+	v4l2_info(sd, "HDCP change to %02X", cp_sts);
+}
+
+static void it6625_irq_infoframe_latch(struct it6625 *it6625)
+{
+	if (!it6625->if_active)
+		return;
+
+	it6625->if_active = false;
+
+	scoped_guard(mutex, &it6625->it6625_lock) {
+		it6625->if_snapshot_err =
+			it6625_read_bytes(it6625, REG_IF_DATA,
+					  it6625->if_snapshot, 31);
+	}
+	it6625->if_snapshot_done = true;
+	complete(&it6625->if_latched);
+}
+
+static void it6625_irq_emata_packet_latch(struct it6625 *it6625)
+{
+	u8 extend_packet[14];
+
+	it6625_read_bytes(it6625, REG_EMP_DATA, extend_packet, 13);
+}
+
+static void it6625_irq_avmute_change(struct it6625 *it6625)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+	int val;
+	u8 avmute;
+
+	val = it6625_read_byte(it6625, REG_RX_STATUS);
+	if (val < 0)
+		return;
+
+	avmute = (val & B_RX_AVMUTE) ? 1 : 0;
+	v4l2_info(sd, "AVMute change to %d", avmute);
+}
+
+static void it6625_dispatch_int_status1(struct it6625 *it6625, int int_sts1)
+{
+	if (int_sts1 & B_HDMI_5V_CHG)
+		it6625_irq_hdmi_5v_change(it6625);
+
+	if (int_sts1 & B_HDMI_VID_CHG)
+		it6625_irq_format_change(it6625);
+
+	if (int_sts1 & B_HDMI_AUD_CHG)
+		it6625_irq_hdmi_audio_change(it6625);
+
+	if (int_sts1 & B_HDMI_CP_CHG)
+		it6625_irq_hdcp_change(it6625);
+
+	if (int_sts1 & B_HDMI_IF_LATCH)
+		it6625_irq_infoframe_latch(it6625);
+
+	if (int_sts1 & B_HDMI_EMP)
+		it6625_irq_emata_packet_latch(it6625);
+}
+
+static void it6625_dispatch_int_status2(struct it6625 *it6625, int int_sts2)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+
+	if (int_sts2 & B_HDMI_AVI)
+		it6625_show_avi_infoframe(it6625);
+
+	if (int_sts2 & B_HDMI_NO_AVI)
+		v4l2_info(sd, "AVI-info stopped");
+
+	if (int_sts2 & B_HDMI_AVMUTE_CHG)
+		it6625_irq_avmute_change(it6625);
+}
+
+/* caller must already hold if_state_lock */
+static int it6625_drain_rx_int_status(struct it6625 *it6625)
+{
+	int val1, val2, err;
+
+	val1 = it6625_read_byte(it6625, REG_RX_INT_STATUS1);
+	val2 = it6625_read_byte(it6625, REG_RX_INT_STATUS2);
+
+	err = 0;
+	if (val1 < 0)
+		err = val1;
+	else if (val2 < 0)
+		err = val2;
+
+	if (val1 >= 0) {
+		int werr = it6625_write_byte(it6625, REG_RX_INT_STATUS1, 0x00);
+
+		if (werr && !err)
+			err = werr;
+		it6625_dispatch_int_status1(it6625, val1);
+	}
+
+	if (val2 >= 0) {
+		int werr = it6625_write_byte(it6625, REG_RX_INT_STATUS2, 0x00);
+
+		if (werr && !err)
+			err = werr;
+		it6625_dispatch_int_status2(it6625, val2);
+	}
+
+	return err;
+}
+
+/* caller must already hold if_state_lock */
+static int it6625_drain_interrupts(struct it6625 *it6625, bool *had_event)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+	int val, err;
+
+	val = it6625_read_byte(it6625, REG_MCU_INTERRUPT);
+	if (val < 0) {
+		if (had_event)
+			*had_event = false;
+		return val;
+	}
+	if (had_event)
+		*had_event = val > 0;
+	if (val == 0)
+		return 0;
+
+	err = it6625_write_byte(it6625, REG_MCU_INTERRUPT, val);
+
+	v4l2_dbg(1, debug, sd, "%s: INT = 0x%02X", __func__, val);
+
+	if (val & (B_CEC_RX_RECEIVED | B_CEC_TX_UPDATE) && it6625->cec_adap)
+		it6625_cec_handler(it6625, val & (B_CEC_RX_RECEIVED | B_CEC_TX_UPDATE));
+
+	if (val & B_SYS_INT_ACTIVE) {
+		int child_err = it6625_drain_rx_int_status(it6625);
+
+		if (child_err && !err)
+			err = child_err;
+	}
+
+	return err;
+}
+
+static bool it6625_interrupt_handler(struct it6625 *it6625)
+{
+	bool had_event = false;
+
+	guard(mutex)(&it6625->if_state_lock);
+	it6625_drain_interrupts(it6625, &had_event);
+	return had_event;
+}
+
+static irqreturn_t it6625_irq_handler(int unused, void *data)
+{
+	struct it6625 *it6625 = data;
+
+	return it6625_interrupt_handler(it6625) ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static void it6625_irq_poll_timer(struct timer_list *t)
+{
+	struct it6625 *it6625 = timer_container_of(it6625, t, timer);
+	unsigned int msecs;
+
+	schedule_work(&it6625->polling_work);
+	/*
+	 * If CEC is present, then we need to poll more frequently,
+	 * otherwise we will miss CEC messages.
+	 */
+	msecs = it6625->cec_adap ? POLL_INTERVAL_CEC_MS : POLL_INTERVAL_MS;
+	mod_timer(&it6625->timer, jiffies + msecs_to_jiffies(msecs));
+}
+
+static void it6625_polling_work(struct work_struct *work)
+{
+	struct it6625 *it6625 = container_of(work, struct it6625,
+					polling_work);
+
+	it6625_interrupt_handler(it6625);
+}
+
+static const char *it6625_csi_format_name(u8 csi_format)
+{
+	switch (csi_format) {
+	case CSI_YUV422_8b:
+		return "YUV422 8bit";
+	case CSI_RGB888:
+		return "RGB888 8bit";
+	case CSI_YUV444_8b:
+		return "YUV444 8bit";
+	default:
+		return "unknown";
+	}
+}
+
+static int it6625_log_status(struct v4l2_subdev *sd)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	struct v4l2_dv_timings timings, configured_timings;
+	struct v4l2_bt_timings bt;
+	u8 csi_format;
+
+	if (it6625_get_detected_timings(it6625, &timings))
+		v4l2_info(sd, "No video detected");
+	else
+		v4l2_print_dv_timings(sd->name, "Detected format: ", &timings,
+				      true);
+
+	it6625_get_timings(it6625, &configured_timings);
+	v4l2_print_dv_timings(sd->name, "Configured format: ",
+			      &configured_timings, true);
+
+	/* snapshot together so the reported pair was actually configured together */
+	scoped_guard(mutex, &it6625->it6625_lock) {
+		csi_format = it6625->csi_format;
+		bt = it6625->timings.bt;
+	}
+
+	v4l2_info(sd, "CSI format: %s @ %uHz",
+		  it6625_csi_format_name(csi_format),
+		  fps_from_bt_timings(&bt));
+
+	it6625_show_avi_infoframe(it6625);
+
+	return 0;
+}
+
+static int it6625_isr(struct v4l2_subdev *sd, u32 status, bool *handled)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	schedule_work(&it6625->polling_work);
+	*handled = true;
+
+	return 0;
+}
+
+static int it6625_subscribe_event(struct v4l2_subdev *sd, struct v4l2_fh *fh,
+				  struct v4l2_event_subscription *sub)
+{
+	switch (sub->type) {
+	case V4L2_EVENT_SOURCE_CHANGE:
+		return v4l2_src_change_event_subdev_subscribe(sd, fh, sub);
+	case V4L2_EVENT_CTRL:
+		return v4l2_ctrl_subdev_subscribe_event(sd, fh, sub);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int it6625_g_input_status(struct v4l2_subdev *sd, u32 *status)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	bool val;
+
+	val = no_signal(it6625);
+	*status = 0;
+	*status |= val ? V4L2_IN_ST_NO_SIGNAL : 0;
+	*status |= val ? V4L2_IN_ST_NO_SYNC : 0;
+
+	v4l2_dbg(1, debug, sd, "%s: status = 0x%x", __func__, *status);
+
+	return 0;
+}
+
+static int
+it6625_update_timings_if_changed(struct it6625 *it6625,
+				 const struct v4l2_dv_timings *timings)
+{
+	int ret;
+
+	guard(mutex)(&it6625->it6625_lock);
+	if (v4l2_match_dv_timings(&it6625->timings, timings, 0, false)) {
+		ret = 0;
+	} else if (!v4l2_valid_dv_timings(timings, it6625_get_timings_cap(it6625),
+					  NULL, NULL)) {
+		ret = -ERANGE;
+	} else {
+		it6625->timings = *timings;
+		ret = 1;
+	}
+
+	return ret;
+}
+
+static int it6625_enum_dv_timings(struct v4l2_subdev *sd,
+				  struct v4l2_enum_dv_timings *timings)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	if (timings->pad != 0)
+		return -EINVAL;
+
+	return v4l2_enum_dv_timings_cap(timings,
+					it6625_get_timings_cap(it6625), NULL, NULL);
+}
+
+static int it6625_dv_timings_cap(struct v4l2_subdev *sd,
+				 struct v4l2_dv_timings_cap *cap)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	if (cap->pad != 0)
+		return -EINVAL;
+
+	*cap = *it6625_get_timings_cap(it6625);
+
+	return 0;
+}
+
+static int it6625_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	it6625_enable_stream(it6625, enable);
+	return 0;
+}
+
+static int it6625_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	dev_dbg(sd->dev, "%s: index=%d", __func__, code->index);
+
+	if (code->index >= ARRAY_SIZE(it6625_formats))
+		return -EINVAL;
+
+	dev_dbg(sd->dev, "%s: code=0x%08x", __func__,
+		it6625_formats[code->index].mbus_fmt_code);
+	code->code = it6625_formats[code->index].mbus_fmt_code;
+
+	return 0;
+}
+
+static int it6625_get_mbus_config(struct v4l2_subdev *sd,
+				  unsigned int pad,
+				  struct v4l2_mbus_config *cfg)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	if (pad != 0)
+		return -EINVAL;
+
+	cfg->type = it6625->bus_type;
+	cfg->bus.mipi_csi2.flags = 0;
+	cfg->bus.mipi_csi2.num_data_lanes = it6625->csi_lanes;
+
+	return 0;
+}
+
+static int it6625_pad_s_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
+				   struct v4l2_dv_timings *timings)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	int ret;
+
+	if (pad != 0)
+		return -EINVAL;
+
+	if (!timings)
+		return -EINVAL;
+
+	if (debug)
+		v4l2_print_dv_timings(sd->name, __func__, timings, false);
+
+	ret = it6625_update_timings_if_changed(it6625, timings);
+	if (ret == -ERANGE) {
+		v4l2_dbg(1, debug, sd, "%s: timings out of range", __func__);
+		return ret;
+	}
+
+	if (ret == 0)
+		v4l2_dbg(1, debug, sd, "%s: no change", __func__);
+
+	return 0;
+}
+
+static int it6625_pad_g_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
+				   struct v4l2_dv_timings *timings)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	if (pad != 0)
+		return -EINVAL;
+
+	it6625_get_timings(it6625, timings);
+
+	return 0;
+}
+
+static int it6625_pad_query_dv_timings(struct v4l2_subdev *sd,
+				       unsigned int pad,
+				       struct v4l2_dv_timings *timings)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	int ret;
+
+	if (pad != 0)
+		return -EINVAL;
+
+	ret = it6625_get_detected_timings(it6625, timings);
+	if (ret)
+		return ret;
+
+	if (debug)
+		v4l2_print_dv_timings(sd->name, __func__, timings, false);
+
+	if (!v4l2_valid_dv_timings(timings, it6625_get_timings_cap(it6625), NULL, NULL)) {
+		v4l2_dbg(1, debug, sd, "%s: timings out of range", __func__);
+		return -ERANGE;
+	}
+
+	return 0;
+}
+
+static inline u32 format_to_colorspace(u8 csi_format)
+{
+	switch (csi_format) {
+	case CSI_RGB444:
+	case CSI_RGB555:
+	case CSI_RGB565:
+	case CSI_RGB666:
+	case CSI_RGB888:
+	case CSI_RGB_10b:
+	case CSI_RGB_12b:
+		return V4L2_COLORSPACE_SRGB;
+	case CSI_YUV420_8b_L:
+	case CSI_YUV420_8b:
+	case CSI_YUV420_10b:
+	case CSI_YUV422_8b:
+	case CSI_YUV422_10b:
+	case CSI_YUV422_12b:
+	case CSI_YUV420_10b_L:
+	case CSI_YUV420_12b:
+	case CSI_YUV444_8b:
+	case CSI_YUV444_10b:
+	case CSI_YUV444_12b:
+		return V4L2_COLORSPACE_REC709;
+	default:
+		return 0;
+	}
+}
+
+static int it6625_get_fmt(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_state *sd_state,
+			  struct v4l2_subdev_format *format)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	struct v4l2_dv_timings timings;
+
+	if (format->pad != 0)
+		return -EINVAL;
+
+	it6625_get_timings(it6625, &timings);
+	format->format.width = timings.bt.width;
+	format->format.height = timings.bt.height;
+	format->format.field = timings.bt.interlaced == V4L2_DV_INTERLACED ?
+			       V4L2_FIELD_INTERLACED : V4L2_FIELD_NONE;
+
+	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
+		struct v4l2_mbus_framefmt *fmt;
+
+		fmt = v4l2_subdev_state_get_format(sd_state, format->pad);
+		format->format.code = fmt->code;
+		format->format.colorspace = fmt->colorspace;
+	} else {
+		scoped_guard(mutex, &it6625->it6625_lock) {
+			format->format.colorspace =
+				format_to_colorspace(it6625->csi_format);
+			format->format.code = it6625->mbus_fmt_code;
+		}
+	}
+
+	return 0;
+}
+
+static int it6625_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
+			  struct v4l2_subdev_state *sd_state,
+			  struct v4l2_subdev_format *format)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	int ret;
+	u32 mbus_fmt_code = format->format.code;
+
+	ret = it6625_get_fmt(sd, sd_state, format);
+	format->format.code = mbus_fmt_code;
+
+	if (ret)
+		return ret;
+
+	ret = it6625_csi_mbus_code_idx(mbus_fmt_code);
+
+	if (ret < 0) {
+		v4l2_dbg(1, debug, sd,
+			 "%s: unsupported format code 0x%x, falling back to default",
+			 __func__, mbus_fmt_code);
+		ret = 0;
+		mbus_fmt_code = it6625_formats[ret].mbus_fmt_code;
+		format->format.code = mbus_fmt_code;
+	}
+
+	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
+		struct v4l2_mbus_framefmt *fmt;
+
+		fmt = v4l2_subdev_state_get_format(sd_state, format->pad);
+		fmt->code = format->format.code;
+		fmt->colorspace = format_to_colorspace(it6625_formats[ret].csi_format);
+		format->format.colorspace = fmt->colorspace;
+		v4l2_dbg(1, debug, sd, "%s: try format code = 0x%x",
+			 __func__, format->format.code);
+		return 0;
+	}
+
+	scoped_guard(mutex, &it6625->it6625_lock) {
+		it6625->csi_format = it6625_formats[ret].csi_format;
+		it6625->mbus_fmt_code = format->format.code;
+		it6625_enable_stream_locked(it6625, false);
+		it6625_set_mipi_config_locked(it6625, it6625->csi_format);
+	}
+
+	format->format.colorspace = format_to_colorspace(it6625_formats[ret].csi_format);
+
+	return 0;
+}
+
+static int it6625_g_edid(struct v4l2_subdev *sd,
+			 struct v4l2_subdev_edid *edid)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	int err;
+
+	if (edid->pad != 0)
+		return -EINVAL;
+
+	memset(edid->reserved, 0, sizeof(edid->reserved));
+
+	guard(mutex)(&it6625->edid_lock);
+
+	if (edid->start_block == 0 && edid->blocks == 0) {
+		edid->blocks = it6625->edid_blocks;
+		return 0;
+	}
+
+	if (it6625->edid_blocks == 0)
+		return -ENODATA;
+
+	if (edid->start_block >= it6625->edid_blocks || edid->blocks == 0)
+		return -EINVAL;
+
+	if (edid->blocks > it6625->edid_blocks - edid->start_block)
+		edid->blocks = it6625->edid_blocks - edid->start_block;
+
+	err = it6625_read_edid(it6625, edid->edid, edid->start_block,
+			       edid->blocks);
+
+	return (err < 0) ? err : 0;
+}
+
+static int it6625_s_edid(struct v4l2_subdev *sd,
+			 struct v4l2_subdev_edid *edid)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	int err;
+	u16 parent_pa = CEC_PHYS_ADDR_INVALID;
+
+	if (edid->pad != 0) {
+		v4l2_err(sd, "invalid pad %d", edid->pad);
+		return -EINVAL;
+	}
+
+	memset(edid->reserved, 0, sizeof(edid->reserved));
+
+	if (edid->start_block != 0) {
+		v4l2_err(sd, "start_block must be 0 for set edid");
+		return -EINVAL;
+	}
+
+	if (edid->blocks > EDID_NUM_BLOCKS_MAX) {
+		v4l2_err(sd, "too many edid blocks: %d", edid->blocks);
+		edid->blocks = EDID_NUM_BLOCKS_MAX;
+		return -E2BIG;
+	}
+
+	if (edid->blocks != 0) {
+		u16 pa = v4l2_get_edid_phys_addr(edid->edid,
+						 edid->blocks * 128, NULL);
+		err = v4l2_phys_addr_validate(pa, &parent_pa, NULL);
+		if (err) {
+			v4l2_err(sd, "invalid CEC physical address in EDID");
+			return err;
+		}
+	}
+
+	guard(mutex)(&it6625->edid_lock);
+
+	it6625_disable_hpd(it6625);
+	cec_phys_addr_invalidate(it6625->cec_adap);
+	it6625->edid_blocks = 0;
+
+	if (edid->blocks == 0)
+		return 0;
+
+	err = it6625_write_edid(it6625, edid->edid,
+				edid->start_block, edid->blocks);
+	if (err < 0) {
+		v4l2_err(sd, "write edid failed");
+		return err;
+	}
+
+	it6625->edid_blocks = edid->blocks;
+	cec_s_phys_addr(it6625->cec_adap, parent_pa, false);
+
+	if (hdmi_5v_power_present(it6625)) {
+		it6625_enable_hpd(it6625);
+		it6625_s_ctrl_detect_hdmi_5v(sd);
+	} else {
+		it6625_enable_auto_hpd(it6625);
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops it6625_core_ops = {
+	.log_status = it6625_log_status,
+	.interrupt_service_routine = it6625_isr,
+	.subscribe_event = it6625_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static const struct v4l2_subdev_video_ops it6625_video_ops = {
+	.g_input_status = it6625_g_input_status,
+	.s_stream = it6625_s_stream,
+};
+
+static const struct v4l2_subdev_pad_ops it6625_pad_ops = {
+	.enum_mbus_code = it6625_enum_mbus_code,
+	.set_fmt = it6625_set_fmt,
+	.get_fmt = it6625_get_fmt,
+	.get_edid = it6625_g_edid,
+	.set_edid = it6625_s_edid,
+	.enum_dv_timings = it6625_enum_dv_timings,
+	.dv_timings_cap = it6625_dv_timings_cap,
+	.get_mbus_config = it6625_get_mbus_config,
+	.s_dv_timings = it6625_pad_s_dv_timings,
+	.g_dv_timings = it6625_pad_g_dv_timings,
+	.query_dv_timings = it6625_pad_query_dv_timings,
+};
+
+static const struct v4l2_subdev_ops it6625_ops = {
+	.core = &it6625_core_ops,
+	.video = &it6625_video_ops,
+	.pad = &it6625_pad_ops,
+};
+
+static int it6625_init_state(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_state *sd_state)
+{
+	struct v4l2_mbus_framefmt *fmt = v4l2_subdev_state_get_format(sd_state, 0);
+
+	fmt->code = it6625_formats[0].mbus_fmt_code;
+	fmt->colorspace = format_to_colorspace(it6625_formats[0].csi_format);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_internal_ops it6625_internal_ops = {
+	.init_state = it6625_init_state,
+};
+
+static const struct v4l2_ctrl_config it6625_ctrl_audio_sampling_rate = {
+	.id = V4L2_CID_IT6625_AUDIO_SAMPLING_RATE,
+	.name = "Audio Sampling Rate",
+	.type = V4L2_CTRL_TYPE_INTEGER,
+	.min = 0,
+	.max = 1536000,
+	.step = 1,
+	.def = 0,
+	.flags = V4L2_CTRL_FLAG_READ_ONLY,
+};
+
+static const struct v4l2_ctrl_config it6625_ctrl_audio_present = {
+	.id = V4L2_CID_IT6625_AUDIO_PRESENT,
+	.name = "Audio Present",
+	.type = V4L2_CTRL_TYPE_BOOLEAN,
+	.min = 0,
+	.max = 1,
+	.step = 1,
+	.def = 0,
+	.flags = V4L2_CTRL_FLAG_READ_ONLY,
+};
+
+static int it6625_v4l2_init_controls(struct v4l2_subdev *sd)
+{
+	struct it6625 *it6625 = sd_to_6625(sd);
+	struct v4l2_ctrl_handler *hdl = &it6625->hdl;
+
+	v4l2_ctrl_handler_init(hdl, 4);
+	it6625->ctrl_5v_detect =
+		v4l2_ctrl_new_std(hdl, NULL, V4L2_CID_DV_RX_POWER_PRESENT,
+				  0, 1, 0, 0);
+
+	it6625->ctrl_audio_sampling_rate =
+		v4l2_ctrl_new_custom(hdl,
+				     &it6625_ctrl_audio_sampling_rate,
+				     NULL);
+	it6625->ctrl_audio_present =
+		v4l2_ctrl_new_custom(hdl, &it6625_ctrl_audio_present, NULL);
+	it6625->ctrl_link_freq =
+		v4l2_ctrl_new_int_menu(hdl, NULL, V4L2_CID_LINK_FREQ,
+				       ARRAY_SIZE(it6625_link_freq) - 1,
+				       it6625->bus_type == V4L2_MBUS_CSI2_CPHY ? 1 : 0,
+				       it6625_link_freq);
+	if (hdl->error) {
+		v4l2_err(sd, "Failed to initialize controls");
+		v4l2_ctrl_handler_free(hdl);
+		return hdl->error;
+	}
+
+	sd->ctrl_handler = hdl;
+
+	return 0;
+}
+
+static void it6625_regdump_print(struct seq_file *s, const u8 *reg_buf)
+{
+	int i;
+
+	seq_puts(s, "     0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F\n");
+
+	for (i = 0; i < 256; i++) {
+		if (i % 16 == 0)
+			seq_printf(s, "[%02X] ", i & 0xF0);
+		seq_printf(s, "0x%02X ", reg_buf[i]);
+		if (i % 16 == 15)
+			seq_putc(s, '\n');
+	}
+}
+
+static int it6625_mipi_reg_show(struct seq_file *s, void *data)
+{
+	struct it6625 *it6625 = s->private;
+	u8 reg_buf[256];
+	int ret;
+
+	scoped_guard(mutex, &it6625->it6625_lock)
+		ret = it6625_read_bytes(it6625, 0x00, reg_buf, sizeof(reg_buf));
+	if (ret < 0)
+		return ret;
+	it6625_regdump_print(s, reg_buf);
+
+	return 0;
+}
+
+static int it6625_mipi_reg_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, it6625_mipi_reg_show, inode->i_private);
+}
+
+static ssize_t it6625_mipi_reg_write(struct file *file,
+				     const char __user *user_buf,
+				     size_t count, loff_t *ppos)
+{
+	struct it6625 *it6625 = file_inode(file)->i_private;
+	char buf[32] = {};
+	unsigned int addr, val;
+	ssize_t len;
+	loff_t pos = 0;
+
+	len = simple_write_to_buffer(buf, sizeof(buf) - 1, &pos, user_buf, count);
+	if (len < 0)
+		return len;
+	buf[len] = '\0';
+
+	if (sscanf(buf, "%X %X", &addr, &val) != 2)
+		return -EINVAL;
+
+	scoped_guard(mutex, &it6625->it6625_lock)
+		it6625_write_byte(it6625, addr, val);
+
+	return count;
+}
+
+static const struct file_operations it6625_mipi_reg_fops = {
+	.owner = THIS_MODULE,
+	.open = it6625_mipi_reg_open,
+	.read = seq_read,
+	.write = it6625_mipi_reg_write,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+/*
+ * Maps a V4L2_DEBUGFS_IF_* framework flag to the actual HDMI packet-type
+ * header byte REG_IF_LATCH_HB expects to arm that InfoFrame type.
+ */
+static int it6625_if_packet_type(u32 type)
+{
+	switch (type) {
+	case V4L2_DEBUGFS_IF_AVI:
+		return 0x82;
+	case V4L2_DEBUGFS_IF_AUDIO:
+		return 0x84;
+	case V4L2_DEBUGFS_IF_SPD:
+		return 0x83;
+	case V4L2_DEBUGFS_IF_HDMI:
+		return 0x81;
+	default:
+		return -EINVAL;
+	}
+}
+
+static ssize_t it6625_debugfs_if_read(u32 type, void *priv, struct file *filp,
+				      char __user *ubuf, size_t count,
+				      loff_t *ppos)
+{
+	struct v4l2_subdev *sd = priv;
+	struct it6625 *it6625 = sd_to_6625(sd);
+	u8 buf[32] = {};
+	int packet_type;
+	int err, err_reset;
+	bool captured;
+	int len;
+
+	packet_type = it6625_if_packet_type(type);
+	if (packet_type < 0)
+		return 0;
+
+	guard(mutex)(&it6625->if_read_lock);
+
+	scoped_guard(mutex, &it6625->if_state_lock) {
+		scoped_guard(mutex, &it6625->it6625_lock)
+			err = it6625_write_byte(it6625, REG_IF_LATCH_HB, 0);
+		if (err)
+			return err;
+
+		err = it6625_drain_interrupts(it6625, NULL);
+		if (err)
+			return err;
+
+		reinit_completion(&it6625->if_latched);
+
+		it6625->if_active = true;
+		it6625->if_type = packet_type;
+		it6625->if_snapshot_done = false;
+		it6625->if_snapshot_err = 0;
+
+		scoped_guard(mutex, &it6625->it6625_lock)
+			err = it6625_write_byte(it6625, REG_IF_LATCH_HB,
+						packet_type);
+		if (err) {
+			scoped_guard(mutex, &it6625->it6625_lock)
+				it6625_write_byte(it6625, REG_IF_LATCH_HB, 0);
+			it6625->if_active = false;
+			return err;
+		}
+	}
+
+	wait_for_completion_timeout(&it6625->if_latched, msecs_to_jiffies(100));
+
+	scoped_guard(mutex, &it6625->if_state_lock) {
+		captured = it6625->if_snapshot_done;
+		it6625->if_active = false;
+
+		scoped_guard(mutex, &it6625->it6625_lock)
+			err_reset = it6625_write_byte(it6625, REG_IF_LATCH_HB, 0);
+
+		if (!captured) {
+			if (err_reset)
+				return err_reset;
+			return 0;
+		}
+
+		if (!it6625->if_snapshot_err) {
+			buf[0] = packet_type;
+			memcpy(&buf[1], it6625->if_snapshot, 31);
+		}
+
+		err = it6625->if_snapshot_err ? it6625->if_snapshot_err : err_reset;
+		if (err)
+			return err;
+	}
+
+	len = buf[2] ? buf[2] + 4 : -ENOENT;
+	if (len > (int)sizeof(buf))
+		len = -ENOENT;
+	if (len < 0)
+		return 0;
+	return simple_read_from_buffer(ubuf, count, ppos, buf, len);
+}
+
+static void it6625_debugfs_init(struct it6625 *it6625, struct i2c_client *client)
+{
+	it6625->debugfs_dir = debugfs_create_dir(dev_name(&client->dev), NULL);
+
+	debugfs_create_file("mipi_reg", 0600, it6625->debugfs_dir, it6625,
+			    &it6625_mipi_reg_fops);
+
+	it6625->infoframes = v4l2_debugfs_if_alloc(it6625->debugfs_dir,
+						   V4L2_DEBUGFS_IF_AVI | V4L2_DEBUGFS_IF_AUDIO |
+						   V4L2_DEBUGFS_IF_SPD | V4L2_DEBUGFS_IF_HDMI,
+						   &it6625->sd, it6625_debugfs_if_read);
+}
+
+static void it6625_init_data(struct it6625 *it6625)
+{
+	static struct v4l2_dv_timings default_timing =
+			V4L2_DV_BT_CEA_1920X1080P60;
+
+	it6625->csi_lanes = 4;
+	it6625->port_num = 1;
+	it6625->bus_type = V4L2_MBUS_CSI2_DPHY;
+	it6625->csi_format = it6625_formats[0].csi_format;
+	it6625->mbus_fmt_code = it6625_formats[0].mbus_fmt_code;
+	it6625->timings = default_timing;
+	/* firmware ships with a verified 2-block default EDID in EDID RAM */
+	it6625->edid_blocks = 2;
+}
+
+static int it6625_parse_endpoint(struct it6625 *it6625)
+{
+	struct device *dev = it6625->dev;
+	/*
+	 * Pre-setting bus_type here makes v4l2_fwnode_endpoint_alloc_parse()
+	 * treat it as a hard requirement and reject any endpoint whose DT
+	 * bus-type disagrees, so this must stay V4L2_MBUS_UNKNOWN to let it
+	 * autodetect C-PHY vs D-PHY from the endpoint itself.
+	 */
+	struct v4l2_fwnode_endpoint endpoint = { .bus_type = V4L2_MBUS_UNKNOWN };
+	struct device_node *ep = NULL;
+	unsigned int max_lanes;
+	unsigned int port;
+	int ret;
+
+	/*
+	 * port@0 and port@1 are the two CSI-2 output ports MIPI0/MIPI1
+	 * (port@2 is the HDMI input). This chip series can drive both
+	 * simultaneously in split or mirror mode, so port_num counts how
+	 * many of MIPI0/MIPI1 have an endpoint wired up. This driver only
+	 * wires up a single source pad, so lane/bus-type configuration is
+	 * parsed from whichever of the two is found first.
+	 */
+	it6625->port_num = 0;
+	for (port = 0; port < 2; port++) {
+		struct device_node *port_ep =
+			of_graph_get_endpoint_by_regs(dev->of_node, port, -1);
+
+		if (!port_ep)
+			continue;
+
+		it6625->port_num++;
+		if (!ep)
+			ep = port_ep;
+		else
+			of_node_put(port_ep);
+	}
+
+	if (!ep) {
+		it6625->port_num = 1;
+		dev_dbg(dev, "no CSI-2 endpoint node found, using default %u CSI lanes",
+			it6625->csi_lanes);
+		return 0;
+	}
+
+	ret = v4l2_fwnode_endpoint_alloc_parse(of_fwnode_handle(ep), &endpoint);
+	of_node_put(ep);
+	if (ret) {
+		dev_err(dev, "failed to parse endpoint: %d", ret);
+		return ret;
+	}
+
+	if (endpoint.bus_type != V4L2_MBUS_CSI2_DPHY &&
+	    endpoint.bus_type != V4L2_MBUS_CSI2_CPHY) {
+		dev_err(dev, "unsupported bus type %d, expected CSI-2 D-PHY or C-PHY",
+			endpoint.bus_type);
+		v4l2_fwnode_endpoint_free(&endpoint);
+		return -EINVAL;
+	}
+
+	if (endpoint.bus_type == V4L2_MBUS_CSI2_CPHY &&
+	    it6625->chip_type != IT6626_CHIP) {
+		dev_err(dev, "IT6625 does not support C-PHY, only IT6626 does");
+		v4l2_fwnode_endpoint_free(&endpoint);
+		return -EINVAL;
+	}
+
+	max_lanes = (endpoint.bus_type == V4L2_MBUS_CSI2_CPHY) ? 3 : 4;
+
+	if (endpoint.bus.mipi_csi2.num_data_lanes == 0 ||
+	    endpoint.bus.mipi_csi2.num_data_lanes > max_lanes) {
+		dev_err(dev,
+			"invalid number of CSI data lanes: %u (max %u for this bus type)",
+			endpoint.bus.mipi_csi2.num_data_lanes, max_lanes);
+		v4l2_fwnode_endpoint_free(&endpoint);
+		return -EINVAL;
+	}
+
+	it6625->csi_lanes = endpoint.bus.mipi_csi2.num_data_lanes;
+	it6625->bus_type = endpoint.bus_type;
+	v4l2_fwnode_endpoint_free(&endpoint);
+
+	return 0;
+}
+
+static int it6625_parse_dt(struct it6625 *it6625)
+{
+	return it6625_parse_endpoint(it6625);
+}
+
+static int it6625_init_v4l2_subdev(struct it6625 *it6625)
+{
+	struct v4l2_subdev *sd = &it6625->sd;
+	int err;
+
+	sd->dev = it6625->dev;
+
+	v4l2_i2c_subdev_init(sd, it6625->i2c_client, &it6625_ops);
+	sd->internal_ops = &it6625_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
+	if (it6625_v4l2_init_controls(sd)) {
+		dev_err(it6625->dev, "Failed to initialize v4l2 controls");
+		return -ENOMEM;
+	}
+
+	it6625->pad.flags = MEDIA_PAD_FL_SOURCE;
+	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
+	err = media_entity_pads_init(&sd->entity, 1, &it6625->pad);
+	if (err < 0) {
+		dev_err(it6625->dev, "%s %d err=%d", __func__, __LINE__, err);
+		v4l2_ctrl_handler_free(sd->ctrl_handler);
+		return err;
+	}
+
+	return 0;
+}
+
+static int it6625_check_device(struct it6625 *it6625)
+{
+	static const u8 chip_ids[][2] = {
+		{ 0x66, 0x25 },
+		{ 0x66, 0x26 },
+	};
+	int chip_id0, chip_id1;
+
+	chip_id0 = it6625_read_byte(it6625, REG_CHIP_ID_0);
+	chip_id1 = it6625_read_byte(it6625, REG_CHIP_ID_1);
+	if (chip_id0 != chip_ids[it6625->chip_type][0] ||
+	    chip_id1 != chip_ids[it6625->chip_type][1]) {
+		dev_err(it6625->dev,
+			"chip ID mismatch: got 0x%02x%02x, expected 0x%02x%02x",
+			chip_id0, chip_id1,
+			chip_ids[it6625->chip_type][0],
+			chip_ids[it6625->chip_type][1]);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int it6625_probe(struct i2c_client *client)
+{
+	struct it6625 *it6625;
+	struct v4l2_subdev *sd;
+	int err;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA))
+		return -EIO;
+
+	it6625 = devm_kzalloc(&client->dev, sizeof(struct it6625), GFP_KERNEL);
+	if (!it6625)
+		return -ENOMEM;
+
+	it6625->chip_type = (uintptr_t)i2c_get_match_data(client);
+
+	it6625->reset_gpio = devm_gpiod_get_optional(&client->dev, "reset",
+						     GPIOD_OUT_HIGH);
+	if (IS_ERR(it6625->reset_gpio))
+		return PTR_ERR(it6625->reset_gpio);
+
+	if (it6625->reset_gpio) {
+		usleep_range(1000, 2000);
+		gpiod_set_value_cansleep(it6625->reset_gpio, 0);
+		usleep_range(10000, 11000);
+	}
+
+	err = it6625_regmap_i2c_init(client, it6625);
+	if (err)
+		return err;
+
+	err = it6625_check_device(it6625);
+	if (err)
+		return err;
+
+	it6625_init_data(it6625);
+
+	err = it6625_parse_dt(it6625);
+	if (err)
+		return err;
+
+	mutex_init(&it6625->it6625_lock);
+	mutex_init(&it6625->edid_lock);
+	mutex_init(&it6625->if_read_lock);
+	mutex_init(&it6625->if_state_lock);
+	init_completion(&it6625->if_latched);
+	INIT_DELAYED_WORK(&it6625->hpd_delayed_work, it6625_hpd_delayed_work);
+	INIT_WORK(&it6625->polling_work, it6625_polling_work);
+
+	if (client->irq) {
+		err = devm_request_threaded_irq(&client->dev, client->irq,
+						NULL, it6625_irq_handler,
+						IRQF_ONESHOT |
+						IRQF_NO_AUTOEN,
+						"it6625", it6625);
+		if (err)
+			goto err_clean_work_queues;
+	} else {
+		dev_info(it6625->dev, "no IRQ, falling back to polling");
+		timer_setup(&it6625->timer, it6625_irq_poll_timer, 0);
+	}
+
+	sd = &it6625->sd;
+	err = it6625_init_v4l2_subdev(it6625);
+	if (err)
+		goto err_clean_work_queues;
+
+	err = v4l2_ctrl_handler_setup(sd->ctrl_handler);
+	if (err)
+		goto err_clean_hdl;
+
+	it6625->cec_adap = cec_allocate_adapter(&it6625_cec_adap_ops,
+						it6625, dev_name(it6625->dev),
+						CEC_CAP_DEFAULTS |
+						CEC_CAP_MONITOR_ALL |
+						CEC_CAP_PHYS_ADDR,
+						1);
+	if (IS_ERR(it6625->cec_adap)) {
+		err = PTR_ERR(it6625->cec_adap);
+		dev_err(it6625->dev, "%s %d", __func__, __LINE__);
+		goto err_clean_hdl;
+	}
+
+	err = cec_register_adapter(it6625->cec_adap, &client->dev);
+	if (err < 0) {
+		dev_err(it6625->dev, "%s: failed to register the cec device", __func__);
+		cec_delete_adapter(it6625->cec_adap);
+		it6625->cec_adap = NULL;
+		goto err_clean_hdl;
+	}
+
+	it6625_debugfs_init(it6625, client);
+
+	it6625_initial_setup(it6625);
+	it6625_v4l2_sd_ctrl_update(sd);
+
+	err = v4l2_async_register_subdev(sd);
+	if (err < 0) {
+		dev_err(it6625->dev, "%s %d err=%d", __func__, __LINE__, err);
+		goto err_clean_debugfs;
+	}
+
+	if (client->irq)
+		enable_irq(client->irq);
+	else
+		mod_timer(&it6625->timer, jiffies + msecs_to_jiffies(POLL_INTERVAL_MS));
+
+	return 0;
+
+err_clean_debugfs:
+	v4l2_debugfs_if_free(it6625->infoframes);
+	debugfs_remove_recursive(it6625->debugfs_dir);
+	cec_unregister_adapter(it6625->cec_adap);
+err_clean_hdl:
+	media_entity_cleanup(&sd->entity);
+	v4l2_ctrl_handler_free(&it6625->hdl);
+
+err_clean_work_queues:
+	if (!client->irq)
+		timer_shutdown_sync(&it6625->timer);
+	cancel_work_sync(&it6625->polling_work);
+	cancel_delayed_work_sync(&it6625->hpd_delayed_work);
+	mutex_destroy(&it6625->it6625_lock);
+	mutex_destroy(&it6625->edid_lock);
+	mutex_destroy(&it6625->if_read_lock);
+	mutex_destroy(&it6625->if_state_lock);
+	return err;
+}
+
+static void it6625_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct it6625 *it6625 = sd_to_6625(sd);
+
+	v4l2_debugfs_if_free(it6625->infoframes);
+
+	if (client->irq)
+		disable_irq(client->irq);
+	else
+		timer_shutdown_sync(&it6625->timer);
+
+	cancel_work_sync(&it6625->polling_work);
+	cancel_delayed_work_sync(&it6625->hpd_delayed_work);
+
+	v4l2_async_unregister_subdev(sd);
+	v4l2_device_unregister_subdev(sd);
+
+	debugfs_remove_recursive(it6625->debugfs_dir);
+	cec_unregister_adapter(it6625->cec_adap);
+	mutex_destroy(&it6625->it6625_lock);
+	mutex_destroy(&it6625->edid_lock);
+	mutex_destroy(&it6625->if_read_lock);
+	mutex_destroy(&it6625->if_state_lock);
+	media_entity_cleanup(&sd->entity);
+	v4l2_ctrl_handler_free(&it6625->hdl);
+}
+
+static const struct i2c_device_id it6625_id[] = {
+	{ .name = "it6625", .driver_data = IT6625_CHIP },
+	{ .name = "it6626", .driver_data = IT6626_CHIP },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, it6625_id);
+
+static const struct of_device_id it6625_of_match[] = {
+	{ .compatible = "ite,it6625", .data = (void *)IT6625_CHIP },
+	{ .compatible = "ite,it6626", .data = (void *)IT6626_CHIP },
+	{},
+};
+MODULE_DEVICE_TABLE(of, it6625_of_match);
+
+static struct i2c_driver it6625_driver = {
+	.driver = {
+		.name = "it6625",
+		.of_match_table = it6625_of_match,
+	},
+	.probe = it6625_probe,
+	.remove = it6625_remove,
+	.id_table = it6625_id,
+};
+module_i2c_driver(it6625_driver);
+
+MODULE_DESCRIPTION("iTE it6625/it6626 HDMI to MIPI CSI bridge driver");
+MODULE_AUTHOR("Hermes Wu <Hermes.wu@ite.com.tw>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/lt6911uxe.c b/drivers/media/i2c/lt6911uxe.c
index bdefdd157e69ca..c9da174dbfa6c9 100644
--- a/drivers/media/i2c/lt6911uxe.c
+++ b/drivers/media/i2c/lt6911uxe.c
@@ -384,6 +384,7 @@ static int lt6911uxe_disable_streams(struct v4l2_subdev *sd,
 }
 
 static int lt6911uxe_set_format(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_format *fmt)
 {
@@ -439,7 +440,7 @@ static int lt6911uxe_init_state(struct v4l2_subdev *sd,
 		: V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
 
-	return lt6911uxe_set_format(sd, sd_state, &fmt);
+	return lt6911uxe_set_format(sd, NULL, sd_state, &fmt);
 }
 
 static const struct v4l2_subdev_video_ops lt6911uxe_video_ops = {
@@ -562,7 +563,7 @@ static irqreturn_t lt6911uxe_threaded_irq_fn(int irq, void *dev_id)
 	 * As a HDMI to CSI2 bridge, it needs to update the format in time
 	 * when the HDMI source changes.
 	 */
-	lt6911uxe_set_format(sd, state, &fmt);
+	lt6911uxe_set_format(sd, NULL, state, &fmt);
 	v4l2_subdev_unlock_state(state);
 
 	return IRQ_HANDLED;
diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c
index 79eab9045e245b..050ad9405c5f67 100644
--- a/drivers/media/i2c/max9286.c
+++ b/drivers/media/i2c/max9286.c
@@ -910,6 +910,7 @@ static int max9286_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int max9286_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *state,
 			   struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/max96714.c b/drivers/media/i2c/max96714.c
index e3e625e6f11a81..0c2ff62a77ea23 100644
--- a/drivers/media/i2c/max96714.c
+++ b/drivers/media/i2c/max96714.c
@@ -327,6 +327,7 @@ static int max96714_disable_streams(struct v4l2_subdev *sd,
 }
 
 static int max96714_set_fmt(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *state,
 			    struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/max96717.c b/drivers/media/i2c/max96717.c
index 72f021b1a7b9fc..ff1df1d84049c6 100644
--- a/drivers/media/i2c/max96717.c
+++ b/drivers/media/i2c/max96717.c
@@ -414,6 +414,7 @@ static int max96717_set_routing(struct v4l2_subdev *sd,
 }
 
 static int max96717_set_fmt(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *state,
 			    struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ml86v7667.c b/drivers/media/i2c/ml86v7667.c
index 48b7d589df3180..eda41a80d20cf1 100644
--- a/drivers/media/i2c/ml86v7667.c
+++ b/drivers/media/i2c/ml86v7667.c
@@ -298,7 +298,6 @@ static const struct v4l2_subdev_video_ops ml86v7667_subdev_video_ops = {
 static const struct v4l2_subdev_pad_ops ml86v7667_subdev_pad_ops = {
 	.enum_mbus_code = ml86v7667_enum_mbus_code,
 	.get_fmt = ml86v7667_fill_fmt,
-	.set_fmt = ml86v7667_fill_fmt,
 	.get_mbus_config = ml86v7667_get_mbus_config,
 };
 
diff --git a/drivers/media/i2c/mt9m001.c b/drivers/media/i2c/mt9m001.c
index 0ade967b357bc4..fe46ea65550a9d 100644
--- a/drivers/media/i2c/mt9m001.c
+++ b/drivers/media/i2c/mt9m001.c
@@ -248,6 +248,7 @@ static int mt9m001_s_stream(struct v4l2_subdev *sd, int enable)
 }
 
 static int mt9m001_set_selection(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_selection *sel)
 {
@@ -289,6 +290,7 @@ static int mt9m001_set_selection(struct v4l2_subdev *sd,
 }
 
 static int mt9m001_get_selection(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_selection *sel)
 {
@@ -343,6 +345,8 @@ static int mt9m001_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int mt9m001_s_fmt(struct v4l2_subdev *sd,
+			 const struct v4l2_subdev_client_info *ci,
+			 struct v4l2_subdev_state *state,
 			 const struct mt9m001_datafmt *fmt,
 			 struct v4l2_mbus_framefmt *mf)
 {
@@ -359,7 +363,7 @@ static int mt9m001_s_fmt(struct v4l2_subdev *sd,
 	int ret;
 
 	/* No support for scaling so far, just crop. TODO: use skipping */
-	ret = mt9m001_set_selection(sd, NULL, &sel);
+	ret = mt9m001_set_selection(sd, ci, state, &sel);
 	if (!ret) {
 		mf->width	= mt9m001->rect.width;
 		mf->height	= mt9m001->rect.height;
@@ -371,6 +375,7 @@ static int mt9m001_s_fmt(struct v4l2_subdev *sd,
 }
 
 static int mt9m001_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
@@ -404,7 +409,7 @@ static int mt9m001_set_fmt(struct v4l2_subdev *sd,
 	mf->xfer_func	= V4L2_XFER_FUNC_DEFAULT;
 
 	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE)
-		return mt9m001_s_fmt(sd, fmt, mf);
+		return mt9m001_s_fmt(sd, ci, sd_state, fmt, mf);
 	*v4l2_subdev_state_get_format(sd_state, 0) = *mf;
 	return 0;
 }
diff --git a/drivers/media/i2c/mt9m111.c b/drivers/media/i2c/mt9m111.c
index 4e748080b79868..f911c1cb43e1e0 100644
--- a/drivers/media/i2c/mt9m111.c
+++ b/drivers/media/i2c/mt9m111.c
@@ -446,6 +446,7 @@ static int mt9m111_reset(struct mt9m111 *mt9m111)
 }
 
 static int mt9m111_set_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -490,6 +491,7 @@ static int mt9m111_set_selection(struct v4l2_subdev *sd,
 }
 
 static int mt9m111_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -617,6 +619,7 @@ static int mt9m111_set_pixfmt(struct mt9m111 *mt9m111,
 }
 
 static int mt9m111_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/mt9m114.c b/drivers/media/i2c/mt9m114.c
index 848ea06e70abbd..c6c950d5c8f3d6 100644
--- a/drivers/media/i2c/mt9m114.c
+++ b/drivers/media/i2c/mt9m114.c
@@ -1254,6 +1254,7 @@ static int mt9m114_pa_enum_framesizes(struct v4l2_subdev *sd,
 }
 
 static int mt9m114_pa_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *state,
 			      struct v4l2_subdev_format *fmt)
 {
@@ -1281,6 +1282,7 @@ static int mt9m114_pa_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int mt9m114_pa_get_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -1304,6 +1306,7 @@ static int mt9m114_pa_get_selection(struct v4l2_subdev *sd,
 }
 
 static int mt9m114_pa_set_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -1877,6 +1880,7 @@ static void mt9m114_ifp_update_sel_and_src_fmt(struct v4l2_subdev_state *state)
 }
 
 static int mt9m114_ifp_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *state,
 			       struct v4l2_subdev_format *fmt)
 {
@@ -1923,6 +1927,7 @@ static int mt9m114_ifp_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int mt9m114_ifp_get_selection(struct v4l2_subdev *sd,
+				     const struct v4l2_subdev_client_info *ci,
 				     struct v4l2_subdev_state *state,
 				     struct v4l2_subdev_selection *sel)
 {
@@ -1982,6 +1987,7 @@ static int mt9m114_ifp_get_selection(struct v4l2_subdev *sd,
 }
 
 static int mt9m114_ifp_set_selection(struct v4l2_subdev *sd,
+				     const struct v4l2_subdev_client_info *ci,
 				     struct v4l2_subdev_state *state,
 				     struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c
index 2b09e8315c8ee1..5b0a84c66043e8 100644
--- a/drivers/media/i2c/mt9p031.c
+++ b/drivers/media/i2c/mt9p031.c
@@ -580,6 +580,7 @@ static int mt9p031_get_format(struct v4l2_subdev *subdev,
 }
 
 static int mt9p031_set_format(struct v4l2_subdev *subdev,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *format)
 {
@@ -618,6 +619,7 @@ static int mt9p031_set_format(struct v4l2_subdev *subdev,
 }
 
 static int mt9p031_get_selection(struct v4l2_subdev *subdev,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -642,6 +644,7 @@ static int mt9p031_get_selection(struct v4l2_subdev *subdev,
 }
 
 static int mt9p031_set_selection(struct v4l2_subdev *subdev,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/mt9t112.c b/drivers/media/i2c/mt9t112.c
index b3a6c6d766321e..7a47756b93560a 100644
--- a/drivers/media/i2c/mt9t112.c
+++ b/drivers/media/i2c/mt9t112.c
@@ -872,6 +872,7 @@ static int mt9t112_set_params(struct mt9t112_priv *priv,
 }
 
 static int mt9t112_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -897,6 +898,7 @@ static int mt9t112_get_selection(struct v4l2_subdev *sd,
 }
 
 static int mt9t112_set_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -953,6 +955,7 @@ static int mt9t112_s_fmt(struct v4l2_subdev *sd,
 }
 
 static int mt9t112_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/mt9v011.c b/drivers/media/i2c/mt9v011.c
index ff55a8ff32b4f8..6ea31a4a64633e 100644
--- a/drivers/media/i2c/mt9v011.c
+++ b/drivers/media/i2c/mt9v011.c
@@ -336,6 +336,7 @@ static int mt9v011_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int mt9v011_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/mt9v032.c b/drivers/media/i2c/mt9v032.c
index 5113826534d762..29823378f0c71f 100644
--- a/drivers/media/i2c/mt9v032.c
+++ b/drivers/media/i2c/mt9v032.c
@@ -502,6 +502,7 @@ static unsigned int mt9v032_calc_ratio(unsigned int input, unsigned int output)
 }
 
 static int mt9v032_set_format(struct v4l2_subdev *subdev,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *format)
 {
@@ -546,6 +547,7 @@ static int mt9v032_set_format(struct v4l2_subdev *subdev,
 }
 
 static int mt9v032_get_selection(struct v4l2_subdev *subdev,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -560,6 +562,7 @@ static int mt9v032_get_selection(struct v4l2_subdev *subdev,
 }
 
 static int mt9v032_set_selection(struct v4l2_subdev *subdev,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/mt9v111.c b/drivers/media/i2c/mt9v111.c
index 64a758c95ab7b7..6e9b3e190bcaa1 100644
--- a/drivers/media/i2c/mt9v111.c
+++ b/drivers/media/i2c/mt9v111.c
@@ -886,6 +886,7 @@ static int mt9v111_get_format(struct v4l2_subdev *subdev,
 }
 
 static int mt9v111_set_format(struct v4l2_subdev *subdev,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/og01a1b.c b/drivers/media/i2c/og01a1b.c
index 1675f0460969ac..b207b1d853a400 100644
--- a/drivers/media/i2c/og01a1b.c
+++ b/drivers/media/i2c/og01a1b.c
@@ -677,6 +677,7 @@ static int og01a1b_disable_streams(struct v4l2_subdev *sd,
 }
 
 static int og01a1b_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
@@ -763,7 +764,7 @@ static int og01a1b_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	og01a1b_set_format(sd, state, &fmt);
+	og01a1b_set_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
@@ -956,6 +957,11 @@ static void og01a1b_remove(struct i2c_client *client)
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(og01a1b->dev);
+
+	if (!pm_runtime_status_suspended(og01a1b->dev)) {
+		og01a1b_power_off(og01a1b->dev);
+		pm_runtime_set_suspended(og01a1b->dev);
+	}
 }
 
 static int og01a1b_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/og0ve1b.c b/drivers/media/i2c/og0ve1b.c
index 84a28cdcade10f..84682389d98924 100644
--- a/drivers/media/i2c/og0ve1b.c
+++ b/drivers/media/i2c/og0ve1b.c
@@ -481,6 +481,7 @@ static int og0ve1b_disable_streams(struct v4l2_subdev *sd,
 }
 
 static int og0ve1b_set_pad_format(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *state,
 				  struct v4l2_subdev_format *fmt)
 {
@@ -544,7 +545,7 @@ static int og0ve1b_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	og0ve1b_set_pad_format(sd, state, &fmt);
+	og0ve1b_set_pad_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/os02g10.c b/drivers/media/i2c/os02g10.c
new file mode 100644
index 00000000000000..141b7cb11fc5fc
--- /dev/null
+++ b/drivers/media/i2c/os02g10.c
@@ -0,0 +1,934 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * V4L2 Support for the OS02G10
+ *
+ * Copyright (C) 2026 Silicon Signals Pvt. Ltd.
+ *
+ */
+
+#include <linux/array_size.h>
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/container_of.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/units.h>
+#include <linux/types.h>
+#include <linux/time.h>
+
+#include <media/v4l2-cci.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-mediabus.h>
+
+#define OS02G10_XCLK_FREQ_24MHZ			(24 * HZ_PER_MHZ)
+
+/* Page 0 */
+#define OS02G10_REG_CHIPID			CCI_REG24(0x002)
+#define OS02G10_CHIPID				0x560247
+
+#define OS02G10_REG_PLL_DIV_CTRL		CCI_REG8(0x030)
+#define OS02G10_REG_PLL_DCTL_BIAS_CTRL		CCI_REG8(0x035)
+#define OS02G10_REG_GATE_EN_CTRL		CCI_REG8(0x038)
+#define OS02G10_REG_DPLL_NC			CCI_REG8(0x041)
+#define OS02G10_REG_MP_PHASE_CTRL		CCI_REG8(0x044)
+
+/* Page 1 */
+#define OS02G10_REG_FRAME_SYNC			CCI_REG8(0x101)
+
+#define OS02G10_REG_LONG_EXPOSURE		CCI_REG16(0x103)
+#define OS02G10_EXPOSURE_MIN			4
+#define OS02G10_EXPOSURE_STEP			1
+#define OS02G10_EXPOSURE_MARGIN			9
+
+#define OS02G10_REG_HBLANK			CCI_REG16(0x109)
+
+#define OS02G10_REG_FRAME_TEST_CTRL		CCI_REG8(0x10d)
+#define OS02G10_FRAME_EXP_SEPERATE_EN		BIT(4)
+#define OS02G10_TEST_PATTERN_ENABLE		BIT(0)
+
+#define OS02G10_REG_FRAME_LENGTH		CCI_REG16(0x10e)
+#define OS02G10_FRAME_LENGTH_MAX		(BIT(16) - 1)
+
+#define OS02G10_REG_ANALOG_GAIN			CCI_REG8(0x124)
+#define OS02G10_ANALOG_GAIN_MIN			16
+#define OS02G10_ANALOG_GAIN_MAX			248
+#define OS02G10_ANALOG_GAIN_STEP		1
+#define OS02G10_ANALOG_GAIN_DEFAULT		16
+
+#define OS02G10_REG_DIGITAL_GAIN_H		CCI_REG8(0x137)
+#define OS02G10_REG_DIGITAL_GAIN_L		CCI_REG8(0x139)
+#define OS02G10_DIGITAL_GAIN_MIN		64
+#define OS02G10_DIGITAL_GAIN_MAX		2048
+#define OS02G10_DIGITAL_GAIN_STEP		64
+#define OS02G10_DIGITAL_GAIN_DEFAULT		64
+
+#define OS02G10_REG_ULP_PWD_DUMMY_CTRL		CCI_REG8(0x13c)
+
+#define OS02G10_REG_FLIP_MIRROR			CCI_REG8(0x13f)
+#define OS02G10_FLIP				BIT(1)
+#define OS02G10_MIRROR				BIT(0)
+
+#define OS02G10_REG_DC_LEVEL_LIMIT_EN		CCI_REG8(0x146)
+#define OS02G10_REG_DC_LEVEL_LIMIT_L		CCI_REG8(0x147)
+#define OS02G10_REG_BLC_DATA_LIMIT_L		CCI_REG8(0x148)
+#define OS02G10_REG_DC_BLC_LIMIT_H		CCI_REG8(0x149)
+
+#define OS02G10_REG_H_SIZE_MIPI			CCI_REG16(0x18e)
+#define OS02G10_REG_V_SIZE_MIPI			CCI_REG16(0x190)
+
+#define OS02G10_REG_HS_LP_CTRL			CCI_REG8(0x192)
+#define OS02G10_REG_HS_LEVEL			CCI_REG8(0x19d)
+#define OS02G10_REG_HS_DRV			CCI_REG8(0x19e)
+
+#define OS02G10_REG_MIPI_TX_SPEED_CTRL		CCI_REG8(0x1a1)
+
+#define OS02G10_REG_STREAM_CTRL			CCI_REG8(0x1b1)
+#define OS02G10_STREAM_CTRL_ON			0x03
+#define OS02G10_STREAM_CTRL_OFF			0x00
+
+#define OS02G10_REG_GB_SUBOFFSET		CCI_REG8(0x1f0)
+#define OS02G10_REG_BLUE_SUBOFFSET		CCI_REG8(0x1f1)
+#define OS02G10_REG_RED_SUBOFFSET		CCI_REG8(0x1f2)
+#define OS02G10_REG_GR_SUBOFFSET		CCI_REG8(0x1f3)
+
+#define OS02G10_REG_ABL_TRIGGER			CCI_REG8(0x1fa)
+#define OS02G10_REG_ABL				CCI_REG8(0x1fb)
+
+/* Page 2 */
+#define OS02G10_REG_SIF_CTRL			CCI_REG8(0x25e)
+#define OS02G10_ORIENTATION_BAYER_FIX		0x32
+
+#define OS02G10_REG_V_START			CCI_REG16(0x2a0)
+#define OS02G10_REG_V_SIZE			CCI_REG16(0x2a2)
+#define OS02G10_REG_H_START			CCI_REG16(0x2a4)
+#define OS02G10_REG_H_SIZE			CCI_REG16(0x2a6)
+
+#define OS02G10_LINK_FREQ_720MHZ		(720 * HZ_PER_MHZ)
+#define OS02G10_DATA_LANES			2
+
+/* OS02G10 native and active pixel array size */
+static const struct v4l2_rect os02g10_native_area = {
+	.top = 0,
+	.left = 0,
+	.width = 1928,
+	.height = 1088,
+};
+
+static const struct v4l2_rect os02g10_active_area = {
+	.top = 4,
+	.left = 4,
+	.width = 1920,
+	.height = 1080,
+};
+
+static const char * const os02g10_supply_name[] = {
+	"avdd",		/* Analog power */
+	"dovdd",	/* Digital I/O power */
+	"dvdd",		/* Digital core power */
+};
+
+struct os02g10 {
+	struct device *dev;
+	struct regmap *cci;
+	struct v4l2_subdev sd;
+	struct media_pad pad;
+	struct clk *xclk;
+	struct gpio_desc *reset_gpio;
+	struct regulator_bulk_data supplies[ARRAY_SIZE(os02g10_supply_name)];
+
+	/* V4L2 Controls */
+	struct v4l2_ctrl_handler handler;
+	struct v4l2_ctrl *vblank;
+	struct v4l2_ctrl *exposure;
+	struct v4l2_ctrl *vflip;
+	struct v4l2_ctrl *hflip;
+};
+
+struct os02g10_mode {
+	u32 width;
+	u32 height;
+	u32 vts_def;
+	u32 exp_def;
+	u32 x_start;
+	u32 y_start;
+};
+
+static const struct cci_reg_sequence os02g10_common_regs[] = {
+	{ OS02G10_REG_PLL_DIV_CTRL,		0x0a},
+	{ OS02G10_REG_PLL_DCTL_BIAS_CTRL,	0x04},
+	{ OS02G10_REG_GATE_EN_CTRL,		0x11},
+	{ OS02G10_REG_DPLL_NC,			0x06},
+	{ OS02G10_REG_MP_PHASE_CTRL,		0x20},
+	{ CCI_REG8(0x119),			0x50},
+	{ CCI_REG8(0x11a),			0x0c},
+	{ CCI_REG8(0x11b),			0x0d},
+	{ CCI_REG8(0x11c),			0x00},
+	{ CCI_REG8(0x11d),			0x75},
+	{ CCI_REG8(0x11e),			0x52},
+	{ CCI_REG8(0x122),			0x14},
+	{ CCI_REG8(0x125),			0x44},
+	{ CCI_REG8(0x126),			0x0f},
+	{ OS02G10_REG_ULP_PWD_DUMMY_CTRL,	0xca},
+	{ CCI_REG8(0x13d),			0x4a},
+	{ CCI_REG8(0x140),			0x0f},
+	{ CCI_REG8(0x143),			0x38},
+	{ OS02G10_REG_DC_LEVEL_LIMIT_EN,	0x01},
+	{ OS02G10_REG_DC_LEVEL_LIMIT_L,		0x00},
+	{ OS02G10_REG_DC_BLC_LIMIT_H,		0x32},
+	{ CCI_REG8(0x150),			0x01},
+	{ CCI_REG8(0x151),			0x28},
+	{ CCI_REG8(0x152),			0x20},
+	{ CCI_REG8(0x153),			0x03},
+	{ CCI_REG8(0x157),			0x16},
+	{ CCI_REG8(0x159),			0x01},
+	{ CCI_REG8(0x15a),			0x01},
+	{ CCI_REG8(0x15d),			0x04},
+	{ CCI_REG8(0x16a),			0x04},
+	{ CCI_REG8(0x16b),			0x03},
+	{ CCI_REG8(0x16e),			0x28},
+	{ CCI_REG8(0x171),			0xc2},
+	{ CCI_REG8(0x172),			0x04},
+	{ CCI_REG8(0x173),			0x38},
+	{ CCI_REG8(0x174),			0x04},
+	{ CCI_REG8(0x179),			0x00},
+	{ CCI_REG8(0x17a),			0xb2},
+	{ CCI_REG8(0x17b),			0x10},
+	{ OS02G10_REG_HS_LP_CTRL,		0x02},
+	{ OS02G10_REG_HS_LEVEL,			0x03},
+	{ OS02G10_REG_HS_DRV,			0x55},
+	{ CCI_REG8(0x1b8),			0x70},
+	{ CCI_REG8(0x1b9),			0x70},
+	{ CCI_REG8(0x1ba),			0x70},
+	{ CCI_REG8(0x1bb),			0x70},
+	{ CCI_REG8(0x1bc),			0x00},
+	{ CCI_REG8(0x1c4),			0x6d},
+	{ CCI_REG8(0x1c5),			0x6d},
+	{ CCI_REG8(0x1c6),			0x6d},
+	{ CCI_REG8(0x1c7),			0x6d},
+	{ CCI_REG8(0x1cc),			0x11},
+	{ CCI_REG8(0x1cd),			0xe0},
+	{ CCI_REG8(0x1d0),			0x1b},
+	{ CCI_REG8(0x1d2),			0x76},
+	{ CCI_REG8(0x1d3),			0x68},
+	{ CCI_REG8(0x1d4),			0x68},
+	{ CCI_REG8(0x1d5),			0x73},
+	{ CCI_REG8(0x1d6),			0x73},
+	{ CCI_REG8(0x1e8),			0x55},
+	{ OS02G10_REG_GB_SUBOFFSET,		0x40},
+	{ OS02G10_REG_BLUE_SUBOFFSET,		0x40},
+	{ OS02G10_REG_RED_SUBOFFSET,		0x40},
+	{ OS02G10_REG_GR_SUBOFFSET,		0x40},
+	{ OS02G10_REG_ABL_TRIGGER,		0x1c},
+	{ OS02G10_REG_ABL,			0x33},
+	{ CCI_REG8(0x1fc),			0x80},
+	{ CCI_REG8(0x1fe),			0x80},
+	{ CCI_REG8(0x303),			0x67},
+	{ CCI_REG8(0x300),			0x59},
+	{ CCI_REG8(0x304),			0x11},
+	{ CCI_REG8(0x305),			0x04},
+	{ CCI_REG8(0x306),			0x0c},
+	{ CCI_REG8(0x307),			0x08},
+	{ CCI_REG8(0x308),			0x08},
+	{ CCI_REG8(0x309),			0x4f},
+	{ CCI_REG8(0x30b),			0x08},
+	{ CCI_REG8(0x30d),			0x26},
+	{ CCI_REG8(0x30f),			0x00},
+	{ CCI_REG8(0x234),			0xfe},
+	{ OS02G10_REG_MIPI_TX_SPEED_CTRL,	0x05},
+};
+
+static const struct os02g10_mode supported_modes[] = {
+	{
+		.width = 1920,
+		.height = 1080,
+		.vts_def = 1246,
+		.exp_def = 1100,
+		.x_start = 2,
+		.y_start = 6,
+	},
+};
+
+static const s64 link_freq_menu_items[] = {
+	OS02G10_LINK_FREQ_720MHZ,
+};
+
+static const char * const os02g10_test_pattern_menu[] = {
+	"Disabled",
+	"Colorbar",
+};
+
+static inline struct os02g10 *to_os02g10(struct v4l2_subdev *sd)
+{
+	return container_of_const(sd, struct os02g10, sd);
+}
+
+static u32 os02g10_get_format_code(struct os02g10 *os02g10)
+{
+	static const u32 codes[2][2] = {
+		{ MEDIA_BUS_FMT_SBGGR10_1X10, MEDIA_BUS_FMT_SGBRG10_1X10, },
+		{ MEDIA_BUS_FMT_SGRBG10_1X10, MEDIA_BUS_FMT_SRGGB10_1X10, },
+	};
+
+	return codes[os02g10->vflip->val][os02g10->hflip->val];
+}
+
+static int os02g10_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct os02g10 *os02g10 = container_of_const(ctrl->handler,
+						     struct os02g10, handler);
+	struct v4l2_subdev_state *state;
+	struct v4l2_mbus_framefmt *fmt;
+	int ret = 0;
+
+	state = v4l2_subdev_get_locked_active_state(&os02g10->sd);
+	fmt = v4l2_subdev_state_get_format(state, 0);
+
+	if (ctrl->id == V4L2_CID_VBLANK) {
+		/* Honour the VBLANK limits when setting exposure */
+		s64 max = fmt->height + ctrl->val - OS02G10_EXPOSURE_MARGIN;
+
+		ret = __v4l2_ctrl_modify_range(os02g10->exposure,
+					       os02g10->exposure->minimum, max,
+					       os02g10->exposure->step,
+					       os02g10->exposure->default_value);
+		if (ret)
+			return ret;
+	}
+
+	if (pm_runtime_get_if_active(os02g10->dev) == 0)
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_EXPOSURE:
+		cci_write(os02g10->cci, OS02G10_REG_LONG_EXPOSURE,
+			  ctrl->val, &ret);
+		break;
+	case V4L2_CID_ANALOGUE_GAIN:
+		cci_write(os02g10->cci, OS02G10_REG_ANALOG_GAIN,
+			  ctrl->val, &ret);
+		break;
+	case V4L2_CID_DIGITAL_GAIN:
+		cci_write(os02g10->cci, OS02G10_REG_DIGITAL_GAIN_L,
+			  (ctrl->val & 0xff), &ret);
+		cci_write(os02g10->cci, OS02G10_REG_DIGITAL_GAIN_H,
+			  ((ctrl->val >> 8) & 0x7), &ret);
+		break;
+	case V4L2_CID_VBLANK: {
+		u64 vts = ctrl->val + fmt->height;
+
+		cci_update_bits(os02g10->cci, OS02G10_REG_FRAME_TEST_CTRL,
+				OS02G10_FRAME_EXP_SEPERATE_EN,
+				OS02G10_FRAME_EXP_SEPERATE_EN, &ret);
+		cci_write(os02g10->cci, OS02G10_REG_FRAME_LENGTH, vts, &ret);
+		break;
+	}
+	case V4L2_CID_HFLIP:
+	case V4L2_CID_VFLIP:
+		cci_write(os02g10->cci, OS02G10_REG_FLIP_MIRROR,
+			  os02g10->hflip->val | os02g10->vflip->val << 1, &ret);
+		cci_write(os02g10->cci, OS02G10_REG_SIF_CTRL,
+			  OS02G10_ORIENTATION_BAYER_FIX, &ret);
+		break;
+	case V4L2_CID_TEST_PATTERN:
+		cci_update_bits(os02g10->cci,
+				OS02G10_REG_FRAME_TEST_CTRL,
+				OS02G10_TEST_PATTERN_ENABLE,
+				ctrl->val ? OS02G10_TEST_PATTERN_ENABLE : 0,
+				&ret);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	cci_write(os02g10->cci, OS02G10_REG_FRAME_SYNC, 0x01, &ret);
+
+	pm_runtime_put(os02g10->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops os02g10_ctrl_ops = {
+	.s_ctrl = os02g10_set_ctrl,
+};
+
+static int os02g10_init_controls(struct os02g10 *os02g10)
+{
+	const struct os02g10_mode *mode = &supported_modes[0];
+	struct v4l2_fwnode_device_properties props;
+	u64 vblank_def, exp_max, pixel_rate;
+	struct v4l2_ctrl_handler *ctrl_hdlr;
+	struct v4l2_ctrl *link_freq;
+	int ret;
+
+	ret = v4l2_fwnode_device_parse(os02g10->dev, &props);
+	if (ret)
+		return ret;
+
+	ctrl_hdlr = &os02g10->handler;
+	v4l2_ctrl_handler_init(ctrl_hdlr, 11);
+
+	/* pixel_rate = link_freq * 2 * nr_of_lanes / bits_per_sample */
+	pixel_rate = div_u64(OS02G10_LINK_FREQ_720MHZ * 2 * OS02G10_DATA_LANES, 10);
+	v4l2_ctrl_new_std(ctrl_hdlr, &os02g10_ctrl_ops, V4L2_CID_PIXEL_RATE, 0,
+			  pixel_rate, 1, pixel_rate);
+
+	link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr, &os02g10_ctrl_ops,
+					   V4L2_CID_LINK_FREQ,
+					   ARRAY_SIZE(link_freq_menu_items) - 1,
+					   0, link_freq_menu_items);
+
+	vblank_def = mode->vts_def - mode->height;
+	os02g10->vblank = v4l2_ctrl_new_std(ctrl_hdlr, &os02g10_ctrl_ops,
+					    V4L2_CID_VBLANK, vblank_def,
+					    OS02G10_FRAME_LENGTH_MAX - mode->height,
+					    1, vblank_def);
+
+	exp_max = mode->vts_def - OS02G10_EXPOSURE_MARGIN;
+	os02g10->exposure =
+		v4l2_ctrl_new_std(ctrl_hdlr, &os02g10_ctrl_ops,
+				  V4L2_CID_EXPOSURE,
+				  OS02G10_EXPOSURE_MIN, exp_max,
+				  OS02G10_EXPOSURE_STEP, mode->exp_def);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &os02g10_ctrl_ops,
+			  V4L2_CID_ANALOGUE_GAIN, OS02G10_ANALOG_GAIN_MIN,
+			  OS02G10_ANALOG_GAIN_MAX, OS02G10_ANALOG_GAIN_STEP,
+			  OS02G10_ANALOG_GAIN_DEFAULT);
+
+	v4l2_ctrl_new_std(ctrl_hdlr, &os02g10_ctrl_ops,
+			  V4L2_CID_DIGITAL_GAIN, OS02G10_DIGITAL_GAIN_MIN,
+			  OS02G10_DIGITAL_GAIN_MAX, OS02G10_DIGITAL_GAIN_STEP,
+			  OS02G10_DIGITAL_GAIN_DEFAULT);
+
+	os02g10->hflip = v4l2_ctrl_new_std(ctrl_hdlr, &os02g10_ctrl_ops,
+					   V4L2_CID_HFLIP, 0, 1, 1, 0);
+
+	os02g10->vflip = v4l2_ctrl_new_std(ctrl_hdlr, &os02g10_ctrl_ops,
+					   V4L2_CID_VFLIP, 0, 1, 1, 0);
+
+	v4l2_ctrl_new_std_menu_items(ctrl_hdlr, &os02g10_ctrl_ops,
+				     V4L2_CID_TEST_PATTERN,
+				     ARRAY_SIZE(os02g10_test_pattern_menu) - 1,
+				     0, 0, os02g10_test_pattern_menu);
+
+	ret = v4l2_ctrl_new_fwnode_properties(ctrl_hdlr,
+					      &os02g10_ctrl_ops, &props);
+	if (ret)
+		goto err_handler_free;
+
+	if (ctrl_hdlr->error) {
+		ret = ctrl_hdlr->error;
+		goto err_handler_free;
+	}
+
+	link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+	os02g10->hflip->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT;
+	os02g10->vflip->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT;
+
+	os02g10->sd.ctrl_handler = ctrl_hdlr;
+
+	return 0;
+
+err_handler_free:
+	v4l2_ctrl_handler_free(ctrl_hdlr);
+
+	return ret;
+}
+
+static int os02g10_set_framefmt(struct os02g10 *os02g10,
+				struct v4l2_subdev_state *state)
+{
+	const struct v4l2_mbus_framefmt *format;
+	const struct os02g10_mode *mode;
+	int ret = 0;
+
+	format = v4l2_subdev_state_get_format(state, 0);
+	mode = v4l2_find_nearest_size(supported_modes,
+				      ARRAY_SIZE(supported_modes), width,
+				      height, format->width, format->height);
+
+	cci_write(os02g10->cci, OS02G10_REG_V_START, mode->y_start, &ret);
+	cci_write(os02g10->cci, OS02G10_REG_V_SIZE, mode->height, &ret);
+	cci_write(os02g10->cci, OS02G10_REG_V_SIZE_MIPI, mode->height, &ret);
+	cci_write(os02g10->cci, OS02G10_REG_H_START, mode->x_start, &ret);
+	cci_write(os02g10->cci, OS02G10_REG_H_SIZE, mode->width, &ret);
+	cci_write(os02g10->cci, OS02G10_REG_H_SIZE_MIPI, mode->width, &ret);
+
+	return ret;
+}
+
+static int os02g10_enable_streams(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *state, u32 pad,
+				  u64 streams_mask)
+{
+	struct os02g10 *os02g10 = to_os02g10(sd);
+	int ret;
+
+	ret = pm_runtime_resume_and_get(os02g10->dev);
+	if (ret < 0)
+		return ret;
+
+	ret = cci_multi_reg_write(os02g10->cci, os02g10_common_regs,
+				  ARRAY_SIZE(os02g10_common_regs), NULL);
+	if (ret) {
+		dev_err(os02g10->dev, "failed to write common registers\n");
+		goto err_rpm_put;
+	}
+
+	ret = os02g10_set_framefmt(os02g10, state);
+	if (ret) {
+		dev_err(os02g10->dev, "failed to set frame foramt\n");
+		goto err_rpm_put;
+	}
+
+	/* Apply customized values from user */
+	ret = __v4l2_ctrl_handler_setup(os02g10->sd.ctrl_handler);
+	if (ret)
+		goto err_rpm_put;
+
+	ret = cci_write(os02g10->cci, OS02G10_REG_STREAM_CTRL,
+			OS02G10_STREAM_CTRL_ON, NULL);
+	if (ret)
+		goto err_rpm_put;
+
+	/* vflip and hflip cannot change during streaming */
+	__v4l2_ctrl_grab(os02g10->vflip, true);
+	__v4l2_ctrl_grab(os02g10->hflip, true);
+
+	return 0;
+
+err_rpm_put:
+	pm_runtime_put(os02g10->dev);
+	return ret;
+}
+
+static int os02g10_disable_streams(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_state *state, u32 pad,
+				   u64 streams_mask)
+{
+	struct os02g10 *os02g10 = to_os02g10(sd);
+	int ret;
+
+	ret = cci_write(os02g10->cci, OS02G10_REG_STREAM_CTRL,
+			OS02G10_STREAM_CTRL_OFF, NULL);
+	if (ret)
+		dev_err(os02g10->dev, "Failed to stop stream\n");
+
+	__v4l2_ctrl_grab(os02g10->vflip, false);
+	__v4l2_ctrl_grab(os02g10->hflip, false);
+
+	pm_runtime_put(os02g10->dev);
+
+	return ret;
+}
+
+static int os02g10_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_selection *sel)
+{
+	switch (sel->target) {
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_NATIVE_SIZE:
+		sel->r = os02g10_native_area;
+		return 0;
+	case V4L2_SEL_TGT_CROP:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sel->r = os02g10_active_area;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int os02g10_enum_mbus_code(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *sd_state,
+				  struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct os02g10 *os02g10 = to_os02g10(sd);
+
+	if (code->index)
+		return -EINVAL;
+
+	code->code = os02g10_get_format_code(os02g10);
+
+	return 0;
+}
+
+static int os02g10_enum_frame_size(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_state *sd_state,
+				   struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct os02g10 *os02g10 = to_os02g10(sd);
+
+	if (fse->index >= ARRAY_SIZE(supported_modes))
+		return -EINVAL;
+
+	if (fse->code != os02g10_get_format_code(os02g10))
+		return -EINVAL;
+
+	fse->min_width = supported_modes[fse->index].width;
+	fse->max_width = fse->min_width;
+	fse->min_height = supported_modes[fse->index].height;
+	fse->max_height = fse->min_height;
+
+	return 0;
+}
+
+static int os02g10_set_pad_format(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
+				  struct v4l2_subdev_state *sd_state,
+				  struct v4l2_subdev_format *fmt)
+{
+	struct os02g10 *os02g10 = to_os02g10(sd);
+	struct v4l2_mbus_framefmt *format;
+	const struct os02g10_mode *mode;
+
+	format = v4l2_subdev_state_get_format(sd_state, 0);
+
+	mode = v4l2_find_nearest_size(supported_modes,
+				      ARRAY_SIZE(supported_modes),
+				      width, height,
+				      fmt->format.width, fmt->format.height);
+
+	fmt->format.code = os02g10_get_format_code(os02g10);
+	fmt->format.width = mode->width;
+	fmt->format.height = mode->height;
+	fmt->format.field = V4L2_FIELD_NONE;
+	fmt->format.colorspace = V4L2_COLORSPACE_RAW;
+	fmt->format.quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	fmt->format.xfer_func = V4L2_XFER_FUNC_NONE;
+
+	*format = fmt->format;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		u32 vblank_def = mode->vts_def - mode->height;
+
+		return __v4l2_ctrl_modify_range(os02g10->vblank, vblank_def,
+						OS02G10_FRAME_LENGTH_MAX -
+						mode->height, 1, vblank_def);
+	}
+
+	return 0;
+}
+
+static int os02g10_init_state(struct v4l2_subdev *sd,
+			      struct v4l2_subdev_state *state)
+{
+	struct v4l2_subdev_format fmt = {
+		.which = V4L2_SUBDEV_FORMAT_TRY,
+		.format = {
+			.width = supported_modes[0].width,
+			.height = supported_modes[0].height,
+		},
+	};
+
+	return os02g10_set_pad_format(sd, NULL, state, &fmt);
+}
+
+static const struct v4l2_subdev_video_ops os02g10_video_ops = {
+	.s_stream = v4l2_subdev_s_stream_helper,
+};
+
+static const struct v4l2_subdev_pad_ops os02g10_pad_ops = {
+	.enum_mbus_code = os02g10_enum_mbus_code,
+	.enum_frame_size = os02g10_enum_frame_size,
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = os02g10_set_pad_format,
+	.get_selection = os02g10_get_selection,
+	.enable_streams = os02g10_enable_streams,
+	.disable_streams = os02g10_disable_streams,
+};
+
+static const struct v4l2_subdev_ops os02g10_subdev_ops = {
+	.video = &os02g10_video_ops,
+	.pad = &os02g10_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops os02g10_internal_ops = {
+	.init_state = os02g10_init_state,
+};
+
+static int os02g10_power_on(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct os02g10 *os02g10 = to_os02g10(sd);
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(os02g10_supply_name),
+				    os02g10->supplies);
+	if (ret) {
+		dev_err(os02g10->dev, "failed to enable regulators\n");
+		return ret;
+	}
+
+	/* Wait for T3/T4 timing requirements after supplies become stable */
+	fsleep(5 * USEC_PER_MSEC);
+
+	ret = clk_prepare_enable(os02g10->xclk);
+	if (ret)
+		goto err_regulator_off;
+
+	gpiod_set_value_cansleep(os02g10->reset_gpio, 0);
+
+	/* T5: delay from sensor power up stable to SCCB initialization */
+	fsleep(5 * USEC_PER_MSEC);
+
+	return 0;
+
+err_regulator_off:
+	regulator_bulk_disable(ARRAY_SIZE(os02g10_supply_name), os02g10->supplies);
+
+	return ret;
+}
+
+static int os02g10_power_off(struct device *dev)
+{
+	struct v4l2_subdev *sd = dev_get_drvdata(dev);
+	struct os02g10 *os02g10 = to_os02g10(sd);
+
+	clk_disable_unprepare(os02g10->xclk);
+	gpiod_set_value_cansleep(os02g10->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(os02g10_supply_name), os02g10->supplies);
+
+	return 0;
+}
+
+static int os02g10_identify_module(struct os02g10 *os02g10)
+{
+	u64 chip_id;
+	int ret;
+
+	ret = cci_read(os02g10->cci, OS02G10_REG_CHIPID, &chip_id, NULL);
+	if (ret)
+		return dev_err_probe(os02g10->dev, ret,
+				     "failed to read chip id %x\n",
+				     OS02G10_CHIPID);
+
+	if (chip_id != OS02G10_CHIPID)
+		return dev_err_probe(os02g10->dev, -EIO,
+				     "chip id mismatch: %x!=%llx\n",
+				     OS02G10_CHIPID, chip_id);
+
+	return 0;
+}
+
+static int os02g10_parse_endpoint(struct os02g10 *os02g10)
+{
+	struct v4l2_fwnode_endpoint bus_cfg = {
+		.bus_type = V4L2_MBUS_CSI2_DPHY,
+	};
+	unsigned long link_freq_bitmap;
+	struct fwnode_handle *ep;
+	int ret;
+
+	ep = fwnode_graph_get_next_endpoint(dev_fwnode(os02g10->dev), NULL);
+	ret = v4l2_fwnode_endpoint_alloc_parse(ep, &bus_cfg);
+	fwnode_handle_put(ep);
+	if (ret)
+		return ret;
+
+	ret = v4l2_link_freq_to_bitmap(os02g10->dev, bus_cfg.link_frequencies,
+				       bus_cfg.nr_of_link_frequencies,
+				       link_freq_menu_items,
+				       ARRAY_SIZE(link_freq_menu_items),
+				       &link_freq_bitmap);
+
+	v4l2_fwnode_endpoint_free(&bus_cfg);
+
+	return ret;
+};
+
+static const struct regmap_range_cfg os02g10_ranges[] = {
+	{
+		.range_min      = 0x0000,
+		.range_max      = 0x03ff,
+		.selector_reg   = 0xfd,
+		.selector_mask  = 0x03,
+		.selector_shift = 0,
+		.window_start   = 0x00,
+		.window_len     = 0x100,
+	},
+};
+
+static const struct regmap_config os02g10_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.reg_format_endian = REGMAP_ENDIAN_BIG,
+	.max_register = 0x3ff,
+	.ranges = os02g10_ranges,
+	.num_ranges = ARRAY_SIZE(os02g10_ranges),
+	.disable_locking = true,
+};
+
+static int os02g10_probe(struct i2c_client *client)
+{
+	struct os02g10 *os02g10;
+	unsigned int xclk_freq;
+	int ret;
+
+	os02g10 = devm_kzalloc(&client->dev, sizeof(*os02g10), GFP_KERNEL);
+	if (!os02g10)
+		return -ENOMEM;
+
+	os02g10->dev = &client->dev;
+
+	v4l2_i2c_subdev_init(&os02g10->sd, client, &os02g10_subdev_ops);
+	os02g10->sd.internal_ops = &os02g10_internal_ops;
+
+	/*
+	 * This is not using devm_cci_regmap_init_i2c(), because the driver
+	 * makes use of regmap's pagination feature. The chosen settings are
+	 * compatible with the CCI helpers.
+	 */
+	os02g10->cci = devm_regmap_init_i2c(client, &os02g10_regmap_config);
+	if (IS_ERR(os02g10->cci))
+		return dev_err_probe(os02g10->dev, PTR_ERR(os02g10->cci),
+				     "failed to initialize CCI\n");
+
+	ret = os02g10_parse_endpoint(os02g10);
+	if (ret)
+		return dev_err_probe(os02g10->dev, ret,
+				     "failed to parse endpoint configuration\n");
+
+	/* Get system clock (xvclk) */
+	os02g10->xclk = devm_v4l2_sensor_clk_get(os02g10->dev, NULL);
+	if (IS_ERR(os02g10->xclk))
+		return dev_err_probe(os02g10->dev, PTR_ERR(os02g10->xclk),
+				     "failed to get xclk\n");
+
+	xclk_freq = clk_get_rate(os02g10->xclk);
+	if (xclk_freq != OS02G10_XCLK_FREQ_24MHZ)
+		return dev_err_probe(os02g10->dev, -EINVAL,
+				     "xclk frequency not supported: %u Hz\n",
+				     xclk_freq);
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(os02g10_supply_name); i++)
+		os02g10->supplies[i].supply = os02g10_supply_name[i];
+
+	ret = devm_regulator_bulk_get(os02g10->dev,
+				      ARRAY_SIZE(os02g10_supply_name),
+				      os02g10->supplies);
+	if (ret)
+		return dev_err_probe(os02g10->dev, ret,
+				     "failed to get regulators\n");
+
+	os02g10->reset_gpio = devm_gpiod_get_optional(os02g10->dev,
+						      "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(os02g10->reset_gpio))
+		return dev_err_probe(os02g10->dev, PTR_ERR(os02g10->reset_gpio),
+				     "failed to get reset GPIO\n");
+
+	ret = os02g10_power_on(os02g10->dev);
+	if (ret)
+		return ret;
+
+	ret = os02g10_identify_module(os02g10);
+	if (ret)
+		goto error_power_off;
+
+	ret = os02g10_init_controls(os02g10);
+	if (ret)
+		goto error_power_off;
+
+	/* Initialize subdev */
+	os02g10->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	os02g10->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+	os02g10->pad.flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&os02g10->sd.entity, 1, &os02g10->pad);
+	if (ret) {
+		dev_err_probe(os02g10->dev, ret, "failed to init entity pads\n");
+		goto error_handler_free;
+	}
+
+	os02g10->sd.state_lock = os02g10->handler.lock;
+	ret = v4l2_subdev_init_finalize(&os02g10->sd);
+	if (ret) {
+		dev_err_probe(os02g10->dev, ret, "subdev init error\n");
+		goto error_media_entity;
+	}
+
+	pm_runtime_set_active(os02g10->dev);
+	pm_runtime_enable(os02g10->dev);
+
+	ret = v4l2_async_register_subdev_sensor(&os02g10->sd);
+	if (ret) {
+		dev_err_probe(os02g10->dev, ret,
+			      "failed to register os02g10 sub-device\n");
+		goto error_subdev_cleanup;
+	}
+
+	pm_runtime_idle(os02g10->dev);
+
+	return 0;
+
+error_subdev_cleanup:
+	v4l2_subdev_cleanup(&os02g10->sd);
+	pm_runtime_disable(os02g10->dev);
+	pm_runtime_set_suspended(os02g10->dev);
+
+error_media_entity:
+	media_entity_cleanup(&os02g10->sd.entity);
+
+error_handler_free:
+	v4l2_ctrl_handler_free(os02g10->sd.ctrl_handler);
+
+error_power_off:
+	os02g10_power_off(os02g10->dev);
+
+	return ret;
+}
+
+static void os02g10_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct os02g10 *os02g10 = to_os02g10(sd);
+
+	v4l2_async_unregister_subdev(sd);
+	v4l2_subdev_cleanup(&os02g10->sd);
+	media_entity_cleanup(&sd->entity);
+	v4l2_ctrl_handler_free(os02g10->sd.ctrl_handler);
+
+	pm_runtime_disable(&client->dev);
+	if (!pm_runtime_status_suspended(&client->dev)) {
+		os02g10_power_off(&client->dev);
+		pm_runtime_set_suspended(&client->dev);
+	}
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(os02g10_pm_ops,
+				 os02g10_power_off, os02g10_power_on, NULL);
+
+static const struct of_device_id os02g10_id[] = {
+	{ .compatible = "ovti,os02g10" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, os02g10_id);
+
+static struct i2c_driver os02g10_driver = {
+	.driver = {
+		.name = "os02g10",
+		.pm = pm_ptr(&os02g10_pm_ops),
+		.of_match_table = os02g10_id,
+	},
+	.probe = os02g10_probe,
+	.remove = os02g10_remove,
+};
+module_i2c_driver(os02g10_driver);
+
+MODULE_DESCRIPTION("OS02G10 Camera Sensor Driver");
+MODULE_AUTHOR("Tarang Raval <tarang.raval@siliconsignals.io>");
+MODULE_AUTHOR("Elgin Perumbilly <elgin.perumbilly@siliconsignals.io>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/os05b10.c b/drivers/media/i2c/os05b10.c
index e0453c988e4a09..a46cfa513e9885 100644
--- a/drivers/media/i2c/os05b10.c
+++ b/drivers/media/i2c/os05b10.c
@@ -595,6 +595,7 @@ static int os05b10_set_framing_limits(struct os05b10 *os05b10,
 }
 
 static int os05b10_set_pad_format(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_format *fmt)
 {
@@ -624,6 +625,7 @@ static int os05b10_set_pad_format(struct v4l2_subdev *sd,
 }
 
 static int os05b10_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov01a10.c b/drivers/media/i2c/ov01a10.c
index 8a29e5b4b6ba06..b9a7dacd603ebd 100644
--- a/drivers/media/i2c/ov01a10.c
+++ b/drivers/media/i2c/ov01a10.c
@@ -634,6 +634,7 @@ static void ov01a10_update_blank_ctrls(struct ov01a10 *ov01a10,
 }
 
 static int ov01a10_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
@@ -715,6 +716,7 @@ static int ov01a10_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int ov01a10_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -747,6 +749,7 @@ static int ov01a10_get_selection(struct v4l2_subdev *sd,
 }
 
 static int ov01a10_set_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov02a10.c b/drivers/media/i2c/ov02a10.c
index 0150e4d296af63..375cd52d61b914 100644
--- a/drivers/media/i2c/ov02a10.c
+++ b/drivers/media/i2c/ov02a10.c
@@ -296,6 +296,7 @@ static void ov02a10_fill_fmt(const struct ov02a10_mode *mode,
 }
 
 static int ov02a10_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
@@ -523,7 +524,7 @@ static int ov02a10_init_state(struct v4l2_subdev *sd,
 		}
 	};
 
-	ov02a10_set_fmt(sd, sd_state, &fmt);
+	ov02a10_set_fmt(sd, NULL, sd_state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/ov02c10.c b/drivers/media/i2c/ov02c10.c
index cf93d36032e147..d622f5dcac60a6 100644
--- a/drivers/media/i2c/ov02c10.c
+++ b/drivers/media/i2c/ov02c10.c
@@ -701,6 +701,7 @@ static int ov02c10_power_on(struct device *dev)
 }
 
 static int ov02c10_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov02e10.c b/drivers/media/i2c/ov02e10.c
index 4a64cba99991f3..5c1e7013f55935 100644
--- a/drivers/media/i2c/ov02e10.c
+++ b/drivers/media/i2c/ov02e10.c
@@ -590,6 +590,7 @@ static int ov02e10_power_on(struct device *dev)
 }
 
 static int ov02e10_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov08d10.c b/drivers/media/i2c/ov08d10.c
index 9adef5446a61f3..0710d2243cc05c 100644
--- a/drivers/media/i2c/ov08d10.c
+++ b/drivers/media/i2c/ov08d10.c
@@ -1178,6 +1178,7 @@ static int ov08d10_set_stream(struct v4l2_subdev *sd, int enable)
 }
 
 static int ov08d10_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov08x40.c b/drivers/media/i2c/ov08x40.c
index 5eaf454f4763b6..6de0c17e633d82 100644
--- a/drivers/media/i2c/ov08x40.c
+++ b/drivers/media/i2c/ov08x40.c
@@ -1844,6 +1844,7 @@ static int ov08x40_get_pad_format(struct v4l2_subdev *sd,
 
 static int
 ov08x40_set_pad_format(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *sd_state,
 		       struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov13858.c b/drivers/media/i2c/ov13858.c
index 09d58e8b1c7fbd..de2b79a9a0e369 100644
--- a/drivers/media/i2c/ov13858.c
+++ b/drivers/media/i2c/ov13858.c
@@ -1345,6 +1345,7 @@ static int ov13858_get_pad_format(struct v4l2_subdev *sd,
 
 static int
 ov13858_set_pad_format(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *sd_state,
 		       struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov13b10.c b/drivers/media/i2c/ov13b10.c
index b0d34141a13a93..242a9583412671 100644
--- a/drivers/media/i2c/ov13b10.c
+++ b/drivers/media/i2c/ov13b10.c
@@ -1118,6 +1118,7 @@ static int ov13b10_get_pad_format(struct v4l2_subdev *sd,
 
 static int
 ov13b10_set_pad_format(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *sd_state,
 		       struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov2640.c b/drivers/media/i2c/ov2640.c
index 50feb608b92b64..3dbefac6d305f0 100644
--- a/drivers/media/i2c/ov2640.c
+++ b/drivers/media/i2c/ov2640.c
@@ -938,6 +938,7 @@ static int ov2640_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov2640_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
@@ -1028,6 +1029,7 @@ static int ov2640_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int ov2640_get_selection(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c
index 7d8c7c3465a4cf..5bc30e7b8325b0 100644
--- a/drivers/media/i2c/ov2659.c
+++ b/drivers/media/i2c/ov2659.c
@@ -1080,6 +1080,7 @@ static void __ov2659_try_frame_size(struct v4l2_mbus_framefmt *mf,
 }
 
 static int ov2659_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov2680.c b/drivers/media/i2c/ov2680.c
index 5f1938c7a94492..147862a813c578 100644
--- a/drivers/media/i2c/ov2680.c
+++ b/drivers/media/i2c/ov2680.c
@@ -639,6 +639,7 @@ static int ov2680_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov2680_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
@@ -703,6 +704,7 @@ static int ov2680_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov2680_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -733,6 +735,7 @@ static int ov2680_get_selection(struct v4l2_subdev *sd,
 }
 
 static int ov2680_set_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov2685.c b/drivers/media/i2c/ov2685.c
index 4911a4eea126a8..162859a140ebfd 100644
--- a/drivers/media/i2c/ov2685.c
+++ b/drivers/media/i2c/ov2685.c
@@ -340,6 +340,7 @@ static void ov2685_fill_fmt(const struct ov2685_mode *mode,
 }
 
 static int ov2685_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -413,6 +414,7 @@ __ov2685_get_pad_crop(struct ov2685 *ov2685,
 }
 
 static int ov2685_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov2732.c b/drivers/media/i2c/ov2732.c
index 40035320fec631..57dd0dd3fd85e3 100644
--- a/drivers/media/i2c/ov2732.c
+++ b/drivers/media/i2c/ov2732.c
@@ -281,6 +281,7 @@ static int ov2732_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int ov2732_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -317,6 +318,7 @@ static int ov2732_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov2732_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -442,7 +444,7 @@ static int ov2732_init_state(struct v4l2_subdev *sd,
 		}
 	};
 
-	return ov2732_set_fmt(sd, sd_state, &fmt);
+	return ov2732_set_fmt(sd, NULL, sd_state, &fmt);
 }
 
 static const struct v4l2_subdev_internal_ops ov2732_internal_ops = {
diff --git a/drivers/media/i2c/ov2735.c b/drivers/media/i2c/ov2735.c
index dcb1add1fd9fc8..0361904c484185 100644
--- a/drivers/media/i2c/ov2735.c
+++ b/drivers/media/i2c/ov2735.c
@@ -674,6 +674,7 @@ static int ov2735_disable_streams(struct v4l2_subdev *sd,
 }
 
 static int ov2735_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -742,6 +743,7 @@ static int ov2735_set_framing_limits(struct ov2735 *ov2735,
 }
 
 static int ov2735_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -792,7 +794,7 @@ static int ov2735_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	ov2735_set_pad_format(sd, state, &fmt);
+	ov2735_set_pad_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
@@ -1081,6 +1083,7 @@ static void ov2735_remove(struct i2c_client *client)
 	v4l2_subdev_cleanup(&ov2735->sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(ov2735->sd.ctrl_handler);
+	ov2735_power_off(ov2735->dev);
 }
 
 static DEFINE_RUNTIME_DEV_PM_OPS(ov2735_pm_ops,
diff --git a/drivers/media/i2c/ov2740.c b/drivers/media/i2c/ov2740.c
index 39003c1632ad15..536325bb7b92c5 100644
--- a/drivers/media/i2c/ov2740.c
+++ b/drivers/media/i2c/ov2740.c
@@ -1023,6 +1023,7 @@ static int ov2740_set_stream(struct v4l2_subdev *sd, int enable)
 }
 
 static int ov2740_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov4689.c b/drivers/media/i2c/ov4689.c
index a59d25b09b5b69..466b6f74d982aa 100644
--- a/drivers/media/i2c/ov4689.c
+++ b/drivers/media/i2c/ov4689.c
@@ -330,6 +330,7 @@ static void ov4689_fill_fmt(const struct ov4689_mode *mode,
 }
 
 static int ov4689_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -385,6 +386,7 @@ static int ov4689_enable_test_pattern(struct ov4689 *ov4689, u32 pattern)
 }
 
 static int ov4689_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
index 8deb5f5501fafd..22d08d0a90cf9f 100644
--- a/drivers/media/i2c/ov5640.c
+++ b/drivers/media/i2c/ov5640.c
@@ -2947,6 +2947,7 @@ static int ov5640_update_pixel_rate(struct ov5640_dev *sensor)
 }
 
 static int ov5640_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
@@ -2993,6 +2994,7 @@ static int ov5640_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov5640_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov5645.c b/drivers/media/i2c/ov5645.c
index c772ef6e51d200..a8d740ed4dc980 100644
--- a/drivers/media/i2c/ov5645.c
+++ b/drivers/media/i2c/ov5645.c
@@ -848,6 +848,7 @@ static int ov5645_enum_frame_size(struct v4l2_subdev *subdev,
 }
 
 static int ov5645_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *format)
 {
@@ -905,12 +906,13 @@ static int ov5645_init_state(struct v4l2_subdev *subdev,
 		},
 	};
 
-	ov5645_set_format(subdev, sd_state, &fmt);
+	ov5645_set_format(subdev, NULL, sd_state, &fmt);
 
 	return 0;
 }
 
 static int ov5645_get_selection(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c
index 3facf92b3841e6..e1120ca688283f 100644
--- a/drivers/media/i2c/ov5647.c
+++ b/drivers/media/i2c/ov5647.c
@@ -787,6 +787,7 @@ static int ov5647_get_pad_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov5647_set_pad_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *format)
 {
@@ -837,6 +838,7 @@ static int ov5647_set_pad_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov5647_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1271,6 +1273,10 @@ static void ov5647_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&sensor->ctrls);
 	v4l2_device_unregister_subdev(sd);
 	pm_runtime_disable(&client->dev);
+	if (!pm_runtime_status_suspended(&client->dev)) {
+		ov5647_power_off(&client->dev);
+		pm_runtime_set_suspended(&client->dev);
+	}
 }
 
 static const struct dev_pm_ops ov5647_pm_ops = {
diff --git a/drivers/media/i2c/ov5648.c b/drivers/media/i2c/ov5648.c
index f0b839cd65f11a..6a1efa12c38d99 100644
--- a/drivers/media/i2c/ov5648.c
+++ b/drivers/media/i2c/ov5648.c
@@ -2145,13 +2145,13 @@ static int ov5648_s_stream(struct v4l2_subdev *subdev, int enable)
 
 	mutex_lock(&sensor->mutex);
 	ret = ov5648_sw_standby(sensor, !enable);
+	if (!ret)
+		state->streaming = !!enable;
 	mutex_unlock(&sensor->mutex);
 
 	if (ret)
 		return ret;
 
-	state->streaming = !!enable;
-
 	if (!enable)
 		pm_runtime_put(sensor->dev);
 
@@ -2215,6 +2215,7 @@ static int ov5648_get_fmt(struct v4l2_subdev *subdev,
 }
 
 static int ov5648_set_fmt(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ov5670.c b/drivers/media/i2c/ov5670.c
index 01fa892de0794c..940f21fe0dd35a 100644
--- a/drivers/media/i2c/ov5670.c
+++ b/drivers/media/i2c/ov5670.c
@@ -2285,6 +2285,7 @@ static int ov5670_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int ov5670_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -2553,6 +2554,7 @@ __ov5670_get_pad_crop(struct ov5670 *sensor, struct v4l2_subdev_state *state,
 }
 
 static int ov5670_get_selection(struct v4l2_subdev *subdev,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov5675.c b/drivers/media/i2c/ov5675.c
index 1c31b2a57eeac9..c5eb07576129b0 100644
--- a/drivers/media/i2c/ov5675.c
+++ b/drivers/media/i2c/ov5675.c
@@ -1015,6 +1015,7 @@ static int ov5675_power_on(struct device *dev)
 }
 
 static int ov5675_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *fmt)
 {
@@ -1074,6 +1075,7 @@ static int ov5675_get_format(struct v4l2_subdev *sd,
 }
 
 static int ov5675_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov5693.c b/drivers/media/i2c/ov5693.c
index 4cc796bbee9263..56599f9ca74ba2 100644
--- a/drivers/media/i2c/ov5693.c
+++ b/drivers/media/i2c/ov5693.c
@@ -35,6 +35,11 @@
 #define OV5693_STOP_STREAMING			0x00
 #define OV5693_SW_RESET				0x01
 
+/* MIPI transmitter control */
+#define OV5693_MIPI_CTRL00_REG			CCI_REG8(0x4800)
+/* Gate the clock lane when there is no packet to transmit */
+#define OV5693_MIPI_CTRL00_CLOCK_LANE_GATE	BIT(5)
+
 #define OV5693_REG_CHIP_ID			CCI_REG16(0x300a)
 /* Yes, this is right. The datasheet for the OV5693 gives its ID as 0x5690 */
 #define OV5693_CHIP_ID				0x5690
@@ -144,6 +149,9 @@ struct ov5693_device {
 	struct regulator_bulk_data supplies[OV5693_NUM_SUPPLIES];
 	struct clk *xvclk;
 
+	/* Gate the MIPI clock lane when idle (CSI-2 non-continuous clock) */
+	bool clock_ncont;
+
 	struct ov5693_mode {
 		struct v4l2_rect crop;
 		struct v4l2_mbus_framefmt format;
@@ -611,6 +619,19 @@ static int ov5693_enable_streaming(struct ov5693_device *ov5693, bool enable)
 {
 	int ret = 0;
 
+	/*
+	 * Gate the MIPI clock lane while idle if the CSI-2 link is configured
+	 * for a non-continuous clock. Only that bit is touched, and only in
+	 * that case, so the register keeps whatever the platform left in it
+	 * and the clock stays free-running as before everywhere else. It
+	 * needs no counterpart at stream off: the link is down by then, and
+	 * the register returns to its default when the sensor is powered off.
+	 */
+	if (enable && ov5693->clock_ncont)
+		cci_update_bits(ov5693->regmap, OV5693_MIPI_CTRL00_REG,
+				OV5693_MIPI_CTRL00_CLOCK_LANE_GATE,
+				OV5693_MIPI_CTRL00_CLOCK_LANE_GATE, &ret);
+
 	cci_write(ov5693->regmap, OV5693_SW_STREAM_REG,
 		  enable ? OV5693_START_STREAMING : OV5693_STOP_STREAMING,
 		  &ret);
@@ -807,6 +828,7 @@ static int ov5693_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov5693_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *state,
 			  struct v4l2_subdev_format *format)
 {
@@ -882,6 +904,7 @@ static int ov5693_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov5693_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -915,6 +938,7 @@ static int ov5693_get_selection(struct v4l2_subdev *sd,
 }
 
 static int ov5693_set_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1259,6 +1283,9 @@ static int ov5693_check_hwcfg(struct ov5693_device *ov5693)
 		goto out_free_bus_cfg;
 	}
 
+	ov5693->clock_ncont = bus_cfg.bus.mipi_csi2.flags &
+			      V4L2_MBUS_CSI2_NONCONTINUOUS_CLOCK;
+
 out_free_bus_cfg:
 	v4l2_fwnode_endpoint_free(&bus_cfg);
 
@@ -1396,6 +1423,7 @@ static const struct dev_pm_ops ov5693_pm_ops = {
 
 static const struct acpi_device_id ov5693_acpi_match[] = {
 	{"INT33BE"},
+	{"OVTI5693"},
 	{},
 };
 MODULE_DEVICE_TABLE(acpi, ov5693_acpi_match);
diff --git a/drivers/media/i2c/ov5695.c b/drivers/media/i2c/ov5695.c
index 5bb6ce7b32375f..5b23f04ab9f875 100644
--- a/drivers/media/i2c/ov5695.c
+++ b/drivers/media/i2c/ov5695.c
@@ -805,6 +805,7 @@ ov5695_find_best_fit(struct v4l2_subdev_format *fmt)
 }
 
 static int ov5695_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov6211.c b/drivers/media/i2c/ov6211.c
index 034d5d57d67e87..2ad31b2c5249c9 100644
--- a/drivers/media/i2c/ov6211.c
+++ b/drivers/media/i2c/ov6211.c
@@ -459,6 +459,7 @@ static int ov6211_disable_streams(struct v4l2_subdev *sd,
 }
 
 static int ov6211_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -522,7 +523,7 @@ static int ov6211_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	ov6211_set_pad_format(sd, state, &fmt);
+	ov6211_set_pad_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/ov64a40.c b/drivers/media/i2c/ov64a40.c
index ed59b4818c55e2..b369c002a25a56 100644
--- a/drivers/media/i2c/ov64a40.c
+++ b/drivers/media/i2c/ov64a40.c
@@ -3106,6 +3106,7 @@ static int ov64a40_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int ov64a40_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -3137,6 +3138,7 @@ static int ov64a40_get_selection(struct v4l2_subdev *sd,
 }
 
 static int ov64a40_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *state,
 			      struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov7251.c b/drivers/media/i2c/ov7251.c
index 311c61d9e25d94..fce6344b512030 100644
--- a/drivers/media/i2c/ov7251.c
+++ b/drivers/media/i2c/ov7251.c
@@ -1213,6 +1213,7 @@ ov7251_find_mode_by_ival(struct ov7251 *ov7251, struct v4l2_fract *timeperframe)
 }
 
 static int ov7251_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *format)
 {
@@ -1295,12 +1296,13 @@ static int ov7251_init_state(struct v4l2_subdev *subdev,
 		}
 	};
 
-	ov7251_set_format(subdev, sd_state, &fmt);
+	ov7251_set_format(subdev, NULL, sd_state, &fmt);
 
 	return 0;
 }
 
 static int ov7251_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c
index 4d040e9feeac72..5a553d50f3a367 100644
--- a/drivers/media/i2c/ov7670.c
+++ b/drivers/media/i2c/ov7670.c
@@ -1097,6 +1097,7 @@ static int ov7670_apply_fmt(struct v4l2_subdev *sd)
  * Set a format.
  */
 static int ov7670_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ov772x.c b/drivers/media/i2c/ov772x.c
index be3ba284ee0ba2..cb3f3f8138375a 100644
--- a/drivers/media/i2c/ov772x.c
+++ b/drivers/media/i2c/ov772x.c
@@ -122,7 +122,7 @@
 #define LC_COEFB    0x4B /* Lens B channel compensation coefficient */
 #define LC_COEFR    0x4C /* Lens R channel compensation coefficient */
 
-#define FIXGAIN     0x4D /* Analog fix gain amplifer */
+#define FIXGAIN     0x4D /* Analog fix gain amplifier */
 #define AREF0       0x4E /* Sensor reference control */
 #define AREF1       0x4F /* Sensor reference current control */
 #define AREF2       0x50 /* Analog reference control */
@@ -1171,6 +1171,7 @@ static int ov772x_set_params(struct ov772x_priv *priv,
 }
 
 static int ov772x_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1212,6 +1213,7 @@ static int ov772x_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov772x_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c
index b4e14171556f9a..c8a74ce39841d5 100644
--- a/drivers/media/i2c/ov7740.c
+++ b/drivers/media/i2c/ov7740.c
@@ -766,6 +766,7 @@ static int ov7740_try_fmt_internal(struct v4l2_subdev *sd,
 }
 
 static int ov7740_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ov8856.c b/drivers/media/i2c/ov8856.c
index 8bedb47cd7cf6a..42c4b9c4e380f3 100644
--- a/drivers/media/i2c/ov8856.c
+++ b/drivers/media/i2c/ov8856.c
@@ -2082,9 +2082,6 @@ static int ov8856_power_on(struct device *dev)
 	struct ov8856 *ov8856 = to_ov8856(sd);
 	int ret;
 
-	if (is_acpi_node(dev_fwnode(dev)))
-		return 0;
-
 	ret = clk_prepare_enable(ov8856->xvclk);
 	if (ret < 0) {
 		dev_err(dev, "failed to enable xvclk\n");
@@ -2120,9 +2117,6 @@ static int ov8856_power_off(struct device *dev)
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
 	struct ov8856 *ov8856 = to_ov8856(sd);
 
-	if (is_acpi_node(dev_fwnode(dev)))
-		return 0;
-
 	gpiod_set_value_cansleep(ov8856->reset_gpio, 1);
 	regulator_bulk_disable(ARRAY_SIZE(ov8856_supply_names),
 			       ov8856->supplies);
@@ -2132,6 +2126,7 @@ static int ov8856_power_off(struct device *dev)
 }
 
 static int ov8856_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *fmt)
 {
@@ -2293,21 +2288,18 @@ static int ov8856_get_hwcfg(struct ov8856 *ov8856)
 		dev_warn(dev, "external clock rate %u is unsupported",
 			 xvclk_rate);
 
-	if (!is_acpi_node(fwnode)) {
-		ov8856->reset_gpio = devm_gpiod_get_optional(dev, "reset",
-							     GPIOD_OUT_LOW);
-		if (IS_ERR(ov8856->reset_gpio))
-			return PTR_ERR(ov8856->reset_gpio);
+	ov8856->reset_gpio = devm_gpiod_get_optional(dev, "reset",
+						     GPIOD_OUT_LOW);
+	if (IS_ERR(ov8856->reset_gpio))
+		return PTR_ERR(ov8856->reset_gpio);
 
-		for (i = 0; i < ARRAY_SIZE(ov8856_supply_names); i++)
-			ov8856->supplies[i].supply = ov8856_supply_names[i];
+	for (i = 0; i < ARRAY_SIZE(ov8856_supply_names); i++)
+		ov8856->supplies[i].supply = ov8856_supply_names[i];
 
-		ret = devm_regulator_bulk_get(dev,
-					      ARRAY_SIZE(ov8856_supply_names),
-					      ov8856->supplies);
-		if (ret)
-			return ret;
-	}
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ov8856_supply_names),
+				      ov8856->supplies);
+	if (ret)
+		return ret;
 
 	ep = fwnode_graph_get_next_endpoint(fwnode, NULL);
 	if (!ep)
@@ -2327,7 +2319,8 @@ static int ov8856_get_hwcfg(struct ov8856 *ov8856)
 		goto check_hwcfg_error;
 	}
 
-	dev_dbg(dev, "Using %u data lanes\n", ov8856->cur_mode->data_lanes);
+	dev_dbg(dev, "Using %u data lanes\n",
+		bus_cfg.bus.mipi_csi2.num_data_lanes);
 
 	if (bus_cfg.bus.mipi_csi2.num_data_lanes == 2)
 		ov8856->priv_lane = &lane_cfg_2;
diff --git a/drivers/media/i2c/ov8858.c b/drivers/media/i2c/ov8858.c
index 3f45f7fab83350..0bfc4350a8c94f 100644
--- a/drivers/media/i2c/ov8858.c
+++ b/drivers/media/i2c/ov8858.c
@@ -1409,6 +1409,7 @@ static const struct v4l2_subdev_video_ops ov8858_video_ops = {
  */
 
 static int ov8858_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -1486,7 +1487,7 @@ static int ov8858_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	ov8858_set_fmt(sd, sd_state, &fmt);
+	ov8858_set_fmt(sd, NULL, sd_state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/ov8865.c b/drivers/media/i2c/ov8865.c
index 289a34a3eeb17d..5a59cca8eeae5f 100644
--- a/drivers/media/i2c/ov8865.c
+++ b/drivers/media/i2c/ov8865.c
@@ -2702,6 +2702,7 @@ static int ov8865_get_fmt(struct v4l2_subdev *subdev,
 }
 
 static int ov8865_set_fmt(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
@@ -2810,6 +2811,7 @@ __ov8865_get_pad_crop(struct ov8865_sensor *sensor,
 }
 
 static int ov8865_get_selection(struct v4l2_subdev *subdev,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov9282.c b/drivers/media/i2c/ov9282.c
index 5d301660a87d8b..3204e8d6080976 100644
--- a/drivers/media/i2c/ov9282.c
+++ b/drivers/media/i2c/ov9282.c
@@ -780,6 +780,7 @@ static int ov9282_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int ov9282_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -826,7 +827,7 @@ static int ov9282_init_state(struct v4l2_subdev *sd,
 	ov9282_fill_pad_format(ov9282, &supported_modes[DEFAULT_MODE],
 			       ov9282->code, &fmt);
 
-	return ov9282_set_pad_format(sd, sd_state, &fmt);
+	return ov9282_set_pad_format(sd, NULL, sd_state, &fmt);
 }
 
 static const struct v4l2_rect *
@@ -845,6 +846,7 @@ __ov9282_get_pad_crop(struct ov9282 *ov9282,
 }
 
 static int ov9282_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov9640.c b/drivers/media/i2c/ov9640.c
index 122f411044ce50..d24b3be1e0b693 100644
--- a/drivers/media/i2c/ov9640.c
+++ b/drivers/media/i2c/ov9640.c
@@ -519,6 +519,7 @@ static int ov9640_s_fmt(struct v4l2_subdev *sd,
 }
 
 static int ov9640_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
@@ -563,6 +564,7 @@ static int ov9640_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int ov9640_get_selection(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c
index 5c85db8a4a3836..b596cfb2d7f350 100644
--- a/drivers/media/i2c/ov9650.c
+++ b/drivers/media/i2c/ov9650.c
@@ -1226,6 +1226,7 @@ static void __ov965x_try_frame_size(struct v4l2_mbus_framefmt *mf,
 }
 
 static int ov965x_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/ov9734.c b/drivers/media/i2c/ov9734.c
index 0eaf33807fc924..8f39609ad1b647 100644
--- a/drivers/media/i2c/ov9734.c
+++ b/drivers/media/i2c/ov9734.c
@@ -681,6 +681,7 @@ static int ov9734_set_stream(struct v4l2_subdev *sd, int enable)
 }
 
 static int ov9734_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/rdacm20.c b/drivers/media/i2c/rdacm20.c
index 52e8e2620b4d47..0179508626b5b8 100644
--- a/drivers/media/i2c/rdacm20.c
+++ b/drivers/media/i2c/rdacm20.c
@@ -442,7 +442,6 @@ static const struct v4l2_subdev_video_ops rdacm20_video_ops = {
 static const struct v4l2_subdev_pad_ops rdacm20_subdev_pad_ops = {
 	.enum_mbus_code = rdacm20_enum_mbus_code,
 	.get_fmt	= rdacm20_get_fmt,
-	.set_fmt	= rdacm20_get_fmt,
 };
 
 static const struct v4l2_subdev_ops rdacm20_subdev_ops = {
diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c
index ece8a410e7cedd..68d2b9d83c3c84 100644
--- a/drivers/media/i2c/rdacm21.c
+++ b/drivers/media/i2c/rdacm21.c
@@ -322,7 +322,6 @@ static const struct v4l2_subdev_video_ops rdacm21_video_ops = {
 static const struct v4l2_subdev_pad_ops rdacm21_subdev_pad_ops = {
 	.enum_mbus_code = rdacm21_enum_mbus_code,
 	.get_fmt	= rdacm21_get_fmt,
-	.set_fmt	= rdacm21_get_fmt,
 };
 
 static const struct v4l2_subdev_ops rdacm21_subdev_ops = {
diff --git a/drivers/media/i2c/rj54n1cb0c.c b/drivers/media/i2c/rj54n1cb0c.c
index 23352d71a108d5..211f890a9cbd21 100644
--- a/drivers/media/i2c/rj54n1cb0c.c
+++ b/drivers/media/i2c/rj54n1cb0c.c
@@ -541,6 +541,7 @@ static int rj54n1_sensor_scale(struct v4l2_subdev *sd, s32 *in_w, s32 *in_h,
 			       s32 *out_w, s32 *out_h);
 
 static int rj54n1_set_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -578,6 +579,7 @@ static int rj54n1_set_selection(struct v4l2_subdev *sd,
 }
 
 static int rj54n1_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -973,6 +975,7 @@ static int rj54n1_reg_init(struct i2c_client *client)
 }
 
 static int rj54n1_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/s5c73m3/s5c73m3-core.c b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
index 551387cea521e5..680add1da244b4 100644
--- a/drivers/media/i2c/s5c73m3/s5c73m3-core.c
+++ b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
@@ -1069,6 +1069,7 @@ static int s5c73m3_oif_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int s5c73m3_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
@@ -1108,6 +1109,7 @@ static int s5c73m3_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int s5c73m3_oif_set_fmt(struct v4l2_subdev *sd,
+			 const struct v4l2_subdev_client_info *ci,
 			 struct v4l2_subdev_state *sd_state,
 			 struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/s5k3m5.c b/drivers/media/i2c/s5k3m5.c
index c591b580d2e7e0..bdf5bea786850f 100644
--- a/drivers/media/i2c/s5k3m5.c
+++ b/drivers/media/i2c/s5k3m5.c
@@ -958,6 +958,7 @@ static void s5k3m5_update_pad_format(struct s5k3m5 *s5k3m5,
 }
 
 static int s5k3m5_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -1038,6 +1039,7 @@ static int s5k3m5_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int s5k3m5_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1075,7 +1077,7 @@ static int s5k3m5_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	s5k3m5_set_pad_format(sd, state, &fmt);
+	s5k3m5_set_pad_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/s5k5baf.c b/drivers/media/i2c/s5k5baf.c
index 378d273055eec4..b13c4754ec3c40 100644
--- a/drivers/media/i2c/s5k5baf.c
+++ b/drivers/media/i2c/s5k5baf.c
@@ -1307,6 +1307,7 @@ static int s5k5baf_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int s5k5baf_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
@@ -1370,6 +1371,7 @@ static int s5k5baf_is_bound_target(u32 target)
 }
 
 static int s5k5baf_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -1463,6 +1465,7 @@ static bool s5k5baf_cmp_rect(const struct v4l2_rect *r1,
 }
 
 static int s5k5baf_set_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/i2c/s5k6a3.c b/drivers/media/i2c/s5k6a3.c
index ba6477e88da36f..071b31728a0053 100644
--- a/drivers/media/i2c/s5k6a3.c
+++ b/drivers/media/i2c/s5k6a3.c
@@ -131,6 +131,7 @@ static struct v4l2_mbus_framefmt *__s5k6a3_get_format(
 }
 
 static int s5k6a3_set_fmt(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/s5kjn1.c b/drivers/media/i2c/s5kjn1.c
index a707cb740556c2..80d713ba3a1ee3 100644
--- a/drivers/media/i2c/s5kjn1.c
+++ b/drivers/media/i2c/s5kjn1.c
@@ -985,6 +985,7 @@ static void s5kjn1_update_pad_format(struct s5kjn1 *s5kjn1,
 }
 
 static int s5kjn1_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -1065,6 +1066,7 @@ static int s5kjn1_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int s5kjn1_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1102,7 +1104,7 @@ static int s5kjn1_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	s5kjn1_set_pad_format(sd, state, &fmt);
+	s5kjn1_set_pad_format(sd, NULL, state, &fmt);
 
 	return 0;
 }
diff --git a/drivers/media/i2c/saa6752hs.c b/drivers/media/i2c/saa6752hs.c
index c6bf0b0902e843..6e2a3e4a63f781 100644
--- a/drivers/media/i2c/saa6752hs.c
+++ b/drivers/media/i2c/saa6752hs.c
@@ -563,6 +563,7 @@ static int saa6752hs_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int saa6752hs_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/saa7115.c b/drivers/media/i2c/saa7115.c
index d20695238b4805..88b159418c7ce4 100644
--- a/drivers/media/i2c/saa7115.c
+++ b/drivers/media/i2c/saa7115.c
@@ -1159,6 +1159,7 @@ static int saa711x_s_sliced_fmt(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_f
 }
 
 static int saa711x_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/saa717x.c b/drivers/media/i2c/saa717x.c
index 0536ceb5465067..367539667948a7 100644
--- a/drivers/media/i2c/saa717x.c
+++ b/drivers/media/i2c/saa717x.c
@@ -980,6 +980,7 @@ static int saa717x_s_register(struct v4l2_subdev *sd, const struct v4l2_dbg_regi
 #endif
 
 static int saa717x_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/st-mipid02.c b/drivers/media/i2c/st-mipid02.c
index 4675181af5fb00..f9eabd9632d3ec 100644
--- a/drivers/media/i2c/st-mipid02.c
+++ b/drivers/media/i2c/st-mipid02.c
@@ -597,6 +597,7 @@ static int mipid02_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int mipid02_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/t4ka3.c b/drivers/media/i2c/t4ka3.c
index a5a68e3fbec22c..6506225683bb07 100644
--- a/drivers/media/i2c/t4ka3.c
+++ b/drivers/media/i2c/t4ka3.c
@@ -363,6 +363,7 @@ static void t4ka3_get_vblank_limits(struct t4ka3_data *sensor,
 }
 
 static int t4ka3_set_pad_format(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_format *format)
 {
@@ -626,6 +627,7 @@ static int t4ka3_disable_stream(struct v4l2_subdev *sd,
 }
 
 static int t4ka3_get_selection(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *state,
 			       struct v4l2_subdev_selection *sel)
 {
@@ -651,6 +653,7 @@ static int t4ka3_get_selection(struct v4l2_subdev *sd,
 }
 
 static int t4ka3_set_selection(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *state,
 			       struct v4l2_subdev_selection *sel)
 {
@@ -1034,7 +1037,6 @@ static int t4ka3_probe(struct i2c_client *client)
 
 err_pm_disable:
 	pm_runtime_disable(&client->dev);
-	pm_runtime_put_noidle(&client->dev);
 	t4ka3_pm_suspend(&client->dev);
 
 	return ret;
diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c
index fbd38bbfee0385..d9cd0563d50b62 100644
--- a/drivers/media/i2c/tc358743.c
+++ b/drivers/media/i2c/tc358743.c
@@ -453,7 +453,7 @@ static void tc358743_enable_edid(struct v4l2_subdev *sd)
 
 	/* Enable hotplug after 143 ms. DDC access to EDID is also enabled when
 	 * hotplug is enabled. See register DDC_CTL */
-	schedule_delayed_work(&state->delayed_work_enable_hotplug, HZ / 7);
+	schedule_delayed_work(&state->delayed_work_enable_hotplug, V4L2_SET_EDID_HPD_LOW_JIFFIES);
 
 	tc358743_enable_interrupts(sd, true);
 	tc358743_s_ctrl_detect_tx_5v(sd);
@@ -1373,7 +1373,8 @@ static int tc358743_log_status(struct v4l2_subdev *sd)
 	static const char * const input_color_space[] = {
 		"RGB", "YCbCr 601", "opRGB", "YCbCr 709", "NA (4)",
 		"xvYCC 601", "NA(6)", "xvYCC 709", "NA(8)", "sYCC601",
-		"NA(10)", "NA(11)", "NA(12)", "opYCC 601"};
+		"NA(10)", "NA(11)", "NA(12)", "opYCC 601", "NA(14)",
+		"NA(15)"};
 
 	v4l2_info(sd, "-----Chip status-----\n");
 	v4l2_info(sd, "Chip ID: 0x%02x\n",
@@ -1816,6 +1817,7 @@ static int tc358743_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int tc358743_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/tc358746.c b/drivers/media/i2c/tc358746.c
index 86d9ba3ea4e545..d55539c35a268e 100644
--- a/drivers/media/i2c/tc358746.c
+++ b/drivers/media/i2c/tc358746.c
@@ -873,6 +873,7 @@ static int tc358746_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int tc358746_set_fmt(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c
index afa1d6f34c9c56..30dfd8a7b42bc8 100644
--- a/drivers/media/i2c/tda1997x.c
+++ b/drivers/media/i2c/tda1997x.c
@@ -590,7 +590,7 @@ static void tda1997x_enable_edid(struct v4l2_subdev *sd)
 	v4l2_dbg(1, debug, sd, "%s\n", __func__);
 
 	/* Enable hotplug after 143ms */
-	schedule_delayed_work(&state->delayed_work_enable_hpd, HZ / 7);
+	schedule_delayed_work(&state->delayed_work_enable_hpd, V4L2_SET_EDID_HPD_LOW_JIFFIES);
 }
 
 /* -----------------------------------------------------------------------------
@@ -1798,6 +1798,7 @@ static int tda1997x_get_format(struct v4l2_subdev *sd,
 }
 
 static int tda1997x_set_format(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/thp7312.c b/drivers/media/i2c/thp7312.c
index 775cfba188d81d..f1c7f149c06c35 100644
--- a/drivers/media/i2c/thp7312.c
+++ b/drivers/media/i2c/thp7312.c
@@ -729,6 +729,7 @@ static int thp7312_enum_frame_interval(struct v4l2_subdev *sd,
 }
 
 static int thp7312_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/ths7303.c b/drivers/media/i2c/ths7303.c
index fa5b9c884c1d73..e04da5519a3964 100644
--- a/drivers/media/i2c/ths7303.c
+++ b/drivers/media/i2c/ths7303.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * ths7303/53- THS7303/53 Video Amplifier driver
  *
@@ -10,15 +11,6 @@
  *     Hans Verkuil <hverkuil@kernel.org>
  *     Lad, Prabhakar <prabhakar.lad@ti.com>
  *     Martin Bugge <marbugge@cisco.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed .as is. WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/i2c.h>
diff --git a/drivers/media/i2c/ths8200.c b/drivers/media/i2c/ths8200.c
index 808ef16ec3b3f4..00535958faf183 100644
--- a/drivers/media/i2c/ths8200.c
+++ b/drivers/media/i2c/ths8200.c
@@ -1,20 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  * ths8200 - Texas Instruments THS8200 video encoder driver
  *
  * Copyright 2013 Cisco Systems, Inc. and/or its affiliates.
- *
- * This program is free software; you may redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed .as is. WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/i2c.h>
diff --git a/drivers/media/i2c/ths8200_regs.h b/drivers/media/i2c/ths8200_regs.h
index 6bc9fd1111dbb6..6a96b6a9440615 100644
--- a/drivers/media/i2c/ths8200_regs.h
+++ b/drivers/media/i2c/ths8200_regs.h
@@ -1,20 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * ths8200 - Texas Instruments THS8200 video encoder driver
  *
  * Copyright 2013 Cisco Systems, Inc. and/or its affiliates.
- *
- * This program is free software; you may redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed .as is. WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #ifndef THS8200_REGS_H
diff --git a/drivers/media/i2c/tvp514x.c b/drivers/media/i2c/tvp514x.c
index 99ace2acdb3507..1e927896a4b3da 100644
--- a/drivers/media/i2c/tvp514x.c
+++ b/drivers/media/i2c/tvp514x.c
@@ -879,6 +879,7 @@ static int tvp514x_get_pad_format(struct v4l2_subdev *sd,
 }
 
 static int tvp514x_set_pad_format(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c
index 9c204f38935dd3..91064f06f8c1b3 100644
--- a/drivers/media/i2c/tvp5150.c
+++ b/drivers/media/i2c/tvp5150.c
@@ -1104,6 +1104,7 @@ static void tvp5150_set_hw_selection(struct v4l2_subdev *sd,
 }
 
 static int tvp5150_set_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -1156,6 +1157,7 @@ static int tvp5150_set_selection(struct v4l2_subdev *sd,
 }
 
 static int tvp5150_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -1724,7 +1726,6 @@ static const struct v4l2_subdev_vbi_ops tvp5150_vbi_ops = {
 static const struct v4l2_subdev_pad_ops tvp5150_pad_ops = {
 	.enum_mbus_code = tvp5150_enum_mbus_code,
 	.enum_frame_size = tvp5150_enum_frame_size,
-	.set_fmt = tvp5150_fill_fmt,
 	.get_fmt = tvp5150_fill_fmt,
 	.get_selection = tvp5150_get_selection,
 	.set_selection = tvp5150_set_selection,
diff --git a/drivers/media/i2c/tvp7002.c b/drivers/media/i2c/tvp7002.c
index 3979ccde5a954f..ada28cb8af7cfb 100644
--- a/drivers/media/i2c/tvp7002.c
+++ b/drivers/media/i2c/tvp7002.c
@@ -854,6 +854,7 @@ tvp7002_get_pad_format(struct v4l2_subdev *sd,
  */
 static int
 tvp7002_set_pad_format(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *sd_state,
 		       struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/tw9900.c b/drivers/media/i2c/tw9900.c
index 617fcf7f0b4539..60c102aba75035 100644
--- a/drivers/media/i2c/tw9900.c
+++ b/drivers/media/i2c/tw9900.c
@@ -198,6 +198,7 @@ static int tw9900_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int tw9900_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/i2c/tw9910.c b/drivers/media/i2c/tw9910.c
index 872207e688bf6c..731c97a40c82fb 100644
--- a/drivers/media/i2c/tw9910.c
+++ b/drivers/media/i2c/tw9910.c
@@ -715,6 +715,7 @@ static int tw9910_set_frame(struct v4l2_subdev *sd, u32 *width, u32 *height)
 }
 
 static int tw9910_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -792,6 +793,7 @@ static int tw9910_s_fmt(struct v4l2_subdev *sd,
 }
 
 static int tw9910_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/i2c/vd55g1.c b/drivers/media/i2c/vd55g1.c
index 6f458f611f6345..22b1497e885182 100644
--- a/drivers/media/i2c/vd55g1.c
+++ b/drivers/media/i2c/vd55g1.c
@@ -1255,6 +1255,7 @@ static int vd55g1_patch(struct vd55g1 *sensor)
 }
 
 static int vd55g1_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1332,6 +1333,7 @@ static int vd55g1_new_format_change_controls(struct vd55g1 *sensor,
 }
 
 static int vd55g1_set_pad_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *sd_fmt)
 {
@@ -1402,7 +1404,7 @@ static int vd55g1_init_state(struct v4l2_subdev *sd,
 	fmt.format.width = vd55g1_supported_modes[VD55G1_MODE_IDX_DEF].width;
 	fmt.format.height = vd55g1_supported_modes[VD55G1_MODE_IDX_DEF].height;
 
-	return vd55g1_set_pad_fmt(sd, sd_state, &fmt);
+	return vd55g1_set_pad_fmt(sd, NULL, sd_state, &fmt);
 }
 
 static int vd55g1_enum_frame_size(struct v4l2_subdev *sd,
diff --git a/drivers/media/i2c/vd56g3.c b/drivers/media/i2c/vd56g3.c
index 157acea9e2867a..ca0dd1b2407274 100644
--- a/drivers/media/i2c/vd56g3.c
+++ b/drivers/media/i2c/vd56g3.c
@@ -823,6 +823,7 @@ static void vd56g3_update_img_pad_format(struct vd56g3 *sensor,
 }
 
 static int vd56g3_set_pad_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *sd_fmt)
 {
@@ -858,6 +859,7 @@ static int vd56g3_set_pad_fmt(struct v4l2_subdev *sd,
 }
 
 static int vd56g3_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1038,7 +1040,7 @@ static int vd56g3_init_state(struct v4l2_subdev *sd,
 		},
 	};
 
-	return vd56g3_set_pad_fmt(sd, sd_state, &fmt);
+	return vd56g3_set_pad_fmt(sd, NULL, sd_state, &fmt);
 }
 
 static const struct v4l2_subdev_video_ops vd56g3_video_ops = {
diff --git a/drivers/media/i2c/vgxy61.c b/drivers/media/i2c/vgxy61.c
index 3fb2166c81ef01..e7819691723dcf 100644
--- a/drivers/media/i2c/vgxy61.c
+++ b/drivers/media/i2c/vgxy61.c
@@ -652,6 +652,7 @@ static int vgxy61_try_fmt_internal(struct v4l2_subdev *sd,
 }
 
 static int vgxy61_get_selection(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
 {
@@ -1197,6 +1198,7 @@ static int vgxy61_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
 }
 
 static int vgxy61_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
@@ -1260,7 +1262,7 @@ static int vgxy61_init_state(struct v4l2_subdev *sd,
 	vgxy61_fill_framefmt(sensor, sensor->current_mode, &fmt.format,
 			     VGXY61_MEDIA_BUS_FMT_DEF);
 
-	return vgxy61_set_fmt(sd, sd_state, &fmt);
+	return vgxy61_set_fmt(sd, NULL, sd_state, &fmt);
 }
 
 static int vgxy61_s_ctrl(struct v4l2_ctrl *ctrl)
diff --git a/drivers/media/i2c/wm8739.c b/drivers/media/i2c/wm8739.c
index db62bafb447c4c..9eaf48f1c3a2c1 100644
--- a/drivers/media/i2c/wm8739.c
+++ b/drivers/media/i2c/wm8739.c
@@ -217,7 +217,7 @@ static int wm8739_probe(struct i2c_client *client)
 
 	/* reset */
 	wm8739_write(sd, R15, 0x00);
-	/* filter setting, high path, offet clear */
+	/* filter setting, high path, offset clear */
 	wm8739_write(sd, R5, 0x000);
 	/* ADC, OSC, Power Off mode Disable */
 	wm8739_write(sd, R6, 0x000);
diff --git a/drivers/media/mc/mc-request.c b/drivers/media/mc/mc-request.c
index 13e77648807c29..e1387f039780b7 100644
--- a/drivers/media/mc/mc-request.c
+++ b/drivers/media/mc/mc-request.c
@@ -316,15 +316,15 @@ int media_request_alloc(struct media_device *mdev, int *alloc_fd)
 	FD_PREPARE(fdf, O_CLOEXEC,
 		   anon_inode_getfile("request", &request_fops, NULL,
 				      O_CLOEXEC));
-	if (fdf.err) {
-		ret = fdf.err;
+	if (fdf->fd < 0) {
+		ret = fdf->fd;
 		goto err_free_req;
 	}
 
-	fd_prepare_file(fdf)->private_data = req;
+	fdf->file->private_data = req;
 
 	snprintf(req->debug_str, sizeof(req->debug_str), "%u:%d",
-		 atomic_inc_return(&mdev->request_id), fd_prepare_fd(fdf));
+		 atomic_inc_return(&mdev->request_id), fdf->fd);
 	atomic_inc(&mdev->num_requests);
 	dev_dbg(mdev->dev, "request: allocated %s\n", req->debug_str);
 
diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
index c631b8bbd38638..8b9ca59f9434f8 100644
--- a/drivers/media/pci/bt8xx/bttv-driver.c
+++ b/drivers/media/pci/bt8xx/bttv-driver.c
@@ -3504,7 +3504,7 @@ static void bttv_remove(struct pci_dev *pci_dev)
 	return;
 }
 
-static int __maybe_unused bttv_suspend(struct device *dev)
+static int bttv_suspend(struct device *dev)
 {
 	struct v4l2_device *v4l2_dev = dev_get_drvdata(dev);
 	struct bttv *btv = to_bttv(v4l2_dev);
@@ -3535,7 +3535,7 @@ static int __maybe_unused bttv_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused bttv_resume(struct device *dev)
+static int bttv_resume(struct device *dev)
 {
 	struct v4l2_device *v4l2_dev = dev_get_drvdata(dev);
 	struct bttv *btv = to_bttv(v4l2_dev);
@@ -3573,7 +3573,7 @@ static const struct pci_device_id bttv_pci_tbl[] = {
 
 MODULE_DEVICE_TABLE(pci, bttv_pci_tbl);
 
-static SIMPLE_DEV_PM_OPS(bttv_pm_ops,
+static DEFINE_SIMPLE_DEV_PM_OPS(bttv_pm_ops,
 			 bttv_suspend,
 			 bttv_resume);
 
@@ -3582,7 +3582,7 @@ static struct pci_driver bttv_pci_driver = {
 	.id_table  = bttv_pci_tbl,
 	.probe     = bttv_probe,
 	.remove    = bttv_remove,
-	.driver.pm = &bttv_pm_ops,
+	.driver.pm = pm_sleep_ptr(&bttv_pm_ops),
 };
 
 static int __init bttv_init_module(void)
diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c
index 7b1ca1238c8d9d..ca96cbe4f194e0 100644
--- a/drivers/media/pci/cobalt/cobalt-driver.c
+++ b/drivers/media/pci/cobalt/cobalt-driver.c
@@ -524,8 +524,8 @@ static int cobalt_subdevs_init(struct cobalt *cobalt)
 				&cobalt_edid);
 		if (err)
 			return err;
-		err = v4l2_subdev_call(s[i].sd, pad, set_fmt, NULL,
-				&sd_fmt);
+		err = v4l2_subdev_call(s[i].sd, pad, set_fmt, NULL, NULL,
+				       &sd_fmt);
 		if (err)
 			return err;
 		/* Reset channel video module */
@@ -608,8 +608,8 @@ static int cobalt_subdevs_hsma_init(struct cobalt *cobalt)
 
 		if (err)
 			return err;
-		err = v4l2_subdev_call(s->sd, pad, set_fmt, NULL,
-				&sd_fmt);
+		err = v4l2_subdev_call(s->sd, pad, set_fmt, NULL, NULL,
+				       &sd_fmt);
 		if (err)
 			return err;
 		cobalt->have_hsma_rx = true;
diff --git a/drivers/media/pci/cobalt/cobalt-v4l2.c b/drivers/media/pci/cobalt/cobalt-v4l2.c
index 51fd9576c6c2a7..ad89e62a274760 100644
--- a/drivers/media/pci/cobalt/cobalt-v4l2.c
+++ b/drivers/media/pci/cobalt/cobalt-v4l2.c
@@ -172,7 +172,7 @@ static void cobalt_enable_output(struct cobalt_stream *s)
 		sd_fmt.format.code = MEDIA_BUS_FMT_RGB888_1X24;
 		break;
 	}
-	v4l2_subdev_call(s->sd, pad, set_fmt, NULL, &sd_fmt);
+	v4l2_subdev_call(s->sd, pad, set_fmt, NULL, NULL, &sd_fmt);
 
 	iowrite32(0, &vo->control);
 	/* 1080p60 */
@@ -223,14 +223,14 @@ static void cobalt_enable_input(struct cobalt_stream *s)
 		iowrite32(M00235_CONTROL_BITMAP_ENABLE_MSK |
 			  (1 << M00235_CONTROL_BITMAP_PACK_FORMAT_OFST),
 			  &packer->control);
-		v4l2_subdev_call(s->sd, pad, set_fmt, NULL,
+		v4l2_subdev_call(s->sd, pad, set_fmt, NULL, NULL,
 				 &sd_fmt_yuyv);
 		break;
 	case V4L2_PIX_FMT_RGB24:
 		iowrite32(M00235_CONTROL_BITMAP_ENABLE_MSK |
 			  (2 << M00235_CONTROL_BITMAP_PACK_FORMAT_OFST),
 			  &packer->control);
-		v4l2_subdev_call(s->sd, pad, set_fmt, NULL,
+		v4l2_subdev_call(s->sd, pad, set_fmt, NULL, NULL,
 				 &sd_fmt_rgb);
 		break;
 	case V4L2_PIX_FMT_BGR32:
@@ -238,7 +238,7 @@ static void cobalt_enable_input(struct cobalt_stream *s)
 			  M00235_CONTROL_BITMAP_ENDIAN_FORMAT_MSK |
 			  (3 << M00235_CONTROL_BITMAP_PACK_FORMAT_OFST),
 			  &packer->control);
-		v4l2_subdev_call(s->sd, pad, set_fmt, NULL,
+		v4l2_subdev_call(s->sd, pad, set_fmt, NULL, NULL,
 				 &sd_fmt_rgb);
 		break;
 	}
@@ -938,7 +938,7 @@ static int cobalt_s_fmt_vid_out(struct file *file, void *priv,
 	s->ycbcr_enc = pix->ycbcr_enc;
 	s->quantization = pix->quantization;
 	v4l2_fill_mbus_format(&sd_fmt.format, pix, code);
-	v4l2_subdev_call(s->sd, pad, set_fmt, NULL, &sd_fmt);
+	v4l2_subdev_call(s->sd, pad, set_fmt, NULL, NULL, &sd_fmt);
 	return 0;
 }
 
diff --git a/drivers/media/pci/cx18/cx18-av-core.c b/drivers/media/pci/cx18/cx18-av-core.c
index 4fb19d26ee29e8..f5961853eeaef9 100644
--- a/drivers/media/pci/cx18/cx18-av-core.c
+++ b/drivers/media/pci/cx18/cx18-av-core.c
@@ -930,6 +930,7 @@ static int cx18_av_s_ctrl(struct v4l2_ctrl *ctrl)
 }
 
 static int cx18_av_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/pci/cx18/cx18-controls.c b/drivers/media/pci/cx18/cx18-controls.c
index 78eadad8b6e821..41f388ba22ab7f 100644
--- a/drivers/media/pci/cx18/cx18-controls.c
+++ b/drivers/media/pci/cx18/cx18-controls.c
@@ -85,7 +85,7 @@ static int cx18_s_video_encoding(struct cx2341x_handler *cxhdl, u32 val)
 	fmt->width = cxhdl->width / (is_mpeg1 ? 2 : 1);
 	fmt->height = cxhdl->height;
 	fmt->code = MEDIA_BUS_FMT_FIXED;
-	v4l2_subdev_call(cx->sd_av, pad, set_fmt, NULL, &format);
+	v4l2_subdev_call(cx->sd_av, pad, set_fmt, NULL, NULL, &format);
 	return 0;
 }
 
diff --git a/drivers/media/pci/cx18/cx18-driver.c b/drivers/media/pci/cx18/cx18-driver.c
index 214fac7af61e53..ac90ce6cd0445f 100644
--- a/drivers/media/pci/cx18/cx18-driver.c
+++ b/drivers/media/pci/cx18/cx18-driver.c
@@ -805,12 +805,12 @@ static int cx18_setup_pci(struct cx18 *cx, struct pci_dev *pci_dev,
 	}
 	if (dma_set_mask(&pci_dev->dev, DMA_BIT_MASK(32))) {
 		CX18_ERR("No suitable DMA available, card %d\n", cx->instance);
-		return -EIO;
+		goto err_disable_device;
 	}
 	if (!request_mem_region(cx->base_addr, CX18_MEM_SIZE, "cx18 encoder")) {
 		CX18_ERR("Cannot request encoder memory region, card %d\n",
 			 cx->instance);
-		return -EIO;
+		goto err_disable_device;
 	}
 
 	/* Enable bus mastering and memory mapped IO for the CX23418 */
@@ -834,6 +834,10 @@ static int cx18_setup_pci(struct cx18 *cx, struct pci_dev *pci_dev,
 		   cx->pci_dev->irq, pci_latency, (u64)cx->base_addr);
 
 	return 0;
+
+err_disable_device:
+	pci_disable_device(pci_dev);
+	return -EIO;
 }
 
 static void cx18_init_subdevs(struct cx18 *cx)
@@ -1120,6 +1124,7 @@ static int cx18_probe(struct pci_dev *pci_dev,
 	cx18_iounmap(cx);
 free_mem:
 	release_mem_region(cx->base_addr, CX18_MEM_SIZE);
+	pci_disable_device(pci_dev);
 free_workqueues:
 	destroy_workqueue(cx->in_work_queue);
 err:
diff --git a/drivers/media/pci/cx18/cx18-driver.h b/drivers/media/pci/cx18/cx18-driver.h
index ef38903709d0ab..51f8a5f5c5f131 100644
--- a/drivers/media/pci/cx18/cx18-driver.h
+++ b/drivers/media/pci/cx18/cx18-driver.h
@@ -24,7 +24,6 @@
 #include <linux/i2c-algo-bit.h>
 #include <linux/list.h>
 #include <linux/unistd.h>
-#include <linux/pagemap.h>
 #include <linux/workqueue.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
diff --git a/drivers/media/pci/cx18/cx18-ioctl.c b/drivers/media/pci/cx18/cx18-ioctl.c
index 0d676a57e24ec9..6f8d7f70001642 100644
--- a/drivers/media/pci/cx18/cx18-ioctl.c
+++ b/drivers/media/pci/cx18/cx18-ioctl.c
@@ -150,7 +150,7 @@ static int cx18_s_fmt_vid_cap(struct file *file, void *fh,
 	format.format.width = cx->cxhdl.width = w;
 	format.format.height = cx->cxhdl.height = h;
 	format.format.code = MEDIA_BUS_FMT_FIXED;
-	v4l2_subdev_call(cx->sd_av, pad, set_fmt, NULL, &format);
+	v4l2_subdev_call(cx->sd_av, pad, set_fmt, NULL, NULL, &format);
 	return cx18_g_fmt_vid_cap(file, fh, fmt);
 }
 
diff --git a/drivers/media/pci/cx18/cx18-queue.c b/drivers/media/pci/cx18/cx18-queue.c
index 04d6828f025945..a973367d4cdece 100644
--- a/drivers/media/pci/cx18/cx18-queue.c
+++ b/drivers/media/pci/cx18/cx18-queue.c
@@ -114,7 +114,10 @@ static inline void cx18_mdl_update_bufs_for_cpu(struct cx18_stream *s,
 	if (list_is_singular(&mdl->buf_list)) {
 		buf = list_first_entry(&mdl->buf_list, struct cx18_buffer,
 				       list);
-		buf->bytesused = mdl->bytesused;
+		if (mdl->bytesused > s->buf_size)
+			buf->bytesused = s->buf_size;
+		else
+			buf->bytesused = mdl->bytesused;
 		buf->readpos = 0;
 		cx18_buf_sync_for_cpu(s, buf);
 	} else {
diff --git a/drivers/media/pci/cx18/cx23418.h b/drivers/media/pci/cx18/cx23418.h
index 22486f39bcda75..4ee45be757cb60 100644
--- a/drivers/media/pci/cx18/cx23418.h
+++ b/drivers/media/pci/cx18/cx23418.h
@@ -24,7 +24,7 @@
 #define CX18_CREATE_TASK			(MGR_CMD_MASK | 0x0001)
 
 /* Description: This command destroys an instance of a task
-   IN[0] - Task handle. Hanlde of the task to destroy
+   IN[0] - Task handle. Handle of the task to destroy
    ReturnCode - One of the ERR_SYS_... */
 #define CX18_DESTROY_TASK			(MGR_CMD_MASK | 0x0002)
 
diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c
index 5fb26285e4af71..7498091176b7a0 100644
--- a/drivers/media/pci/cx23885/cx23885-core.c
+++ b/drivers/media/pci/cx23885/cx23885-core.c
@@ -2246,6 +2246,10 @@ static void cx23885_finidev(struct pci_dev *pci_dev)
 	/* unregister stuff */
 	free_irq(pci_dev->irq, dev);
 
+	cancel_work_sync(&dev->cx25840_work);
+	cancel_work_sync(&dev->ir_rx_work);
+	cancel_work_sync(&dev->ir_tx_work);
+
 	pci_disable_device(pci_dev);
 
 	cx23885_dev_unregister(dev);
diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c
index f240ccda40ed2b..348abde0ba26ac 100644
--- a/drivers/media/pci/cx23885/cx23885-dvb.c
+++ b/drivers/media/pci/cx23885/cx23885-dvb.c
@@ -1158,8 +1158,12 @@ static int dvb_register_ci_mac(struct cx23885_tsport *port)
 		info.platform_data = &sp2_config;
 		request_module(info.type);
 		client_ci = i2c_new_client_device(&i2c_bus->i2c_adap, &info);
-		if (!i2c_client_has_driver(client_ci))
+		if (IS_ERR(client_ci))
+			return PTR_ERR(client_ci);
+		if (!client_ci->dev.driver) {
+			i2c_unregister_device(client_ci);
 			return -ENODEV;
+		}
 		if (!try_module_get(client_ci->dev.driver->owner)) {
 			i2c_unregister_device(client_ci);
 			return -ENODEV;
@@ -1202,7 +1206,7 @@ static int dvb_register(struct cx23885_tsport *port)
 	int (*p_set_voltage)(struct dvb_frontend *fe,
 			     enum fe_sec_voltage voltage) = NULL;
 	int mfe_shared = 0; /* bus not shared by default */
-	int ret;
+	int ret = -EINVAL;
 
 	/* Get the first frontend */
 	fe0 = vb2_dvb_get_frontend(&port->frontends, 1);
@@ -2586,8 +2590,10 @@ static int dvb_register(struct cx23885_tsport *port)
 		goto frontend_detach;
 
 	ret = dvb_register_ci_mac(port);
-	if (ret)
+	if (ret) {
+		vb2_dvb_unregister_bus(&port->frontends);
 		goto frontend_detach;
+	}
 
 	return 0;
 
@@ -2618,7 +2624,7 @@ static int dvb_register(struct cx23885_tsport *port)
 
 	port->gate_ctrl = NULL;
 	vb2_dvb_dealloc_frontends(&port->frontends);
-	return -EINVAL;
+	return ret;
 }
 
 int cx23885_dvb_register(struct cx23885_tsport *port)
diff --git a/drivers/media/pci/cx23885/cx23885-video.c b/drivers/media/pci/cx23885/cx23885-video.c
index 14d219fd1d8a0a..94ce00154e1fdb 100644
--- a/drivers/media/pci/cx23885/cx23885-video.c
+++ b/drivers/media/pci/cx23885/cx23885-video.c
@@ -134,7 +134,7 @@ int cx23885_set_tvnorm(struct cx23885_dev *dev, v4l2_std_id norm)
 	format.format.width = dev->width;
 	format.format.height = dev->height;
 	format.format.field = dev->field;
-	call_all(dev, pad, set_fmt, NULL, &format);
+	call_all(dev, pad, set_fmt, NULL, NULL, &format);
 
 	return 0;
 }
@@ -619,7 +619,7 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
 	dprintk(2, "%s() width=%d height=%d field=%d\n", __func__,
 		dev->width, dev->height, dev->field);
 	v4l2_fill_mbus_format(&format.format, &f->fmt.pix, MEDIA_BUS_FMT_FIXED);
-	call_all(dev, pad, set_fmt, NULL, &format);
+	call_all(dev, pad, set_fmt, NULL, NULL, &format);
 	v4l2_fill_pix_format(&f->fmt.pix, &format.format);
 	/* set_fmt overwrites f->fmt.pix.field, restore it */
 	f->fmt.pix.field = dev->field;
diff --git a/drivers/media/pci/cx88/cx88-input.c b/drivers/media/pci/cx88/cx88-input.c
index 5d9ce4f9af011a..c96d289e5ab783 100644
--- a/drivers/media/pci/cx88/cx88-input.c
+++ b/drivers/media/pci/cx88/cx88-input.c
@@ -259,7 +259,7 @@ static void cx88_ir_close(struct rc_dev *rc)
 int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 {
 	struct cx88_IR *ir;
-	struct rc_dev *dev;
+	struct rc_dev *dev = NULL;
 	char *ir_codes = NULL;
 	u64 rc_proto = RC_PROTO_BIT_OTHER;
 	int err = -ENOMEM;
@@ -268,11 +268,8 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 				 */
 
 	ir = kzalloc_obj(*ir);
-	dev = rc_allocate_device(RC_DRIVER_IR_RAW);
-	if (!ir || !dev)
-		goto err_out_free;
-
-	ir->dev = dev;
+	if (!ir)
+		return -ENOMEM;
 
 	/* detect & configure */
 	switch (core->boardnr) {
@@ -439,6 +436,13 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 		goto err_out_free;
 	}
 
+	dev = rc_allocate_device(ir->sampling ?
+				 RC_DRIVER_IR_RAW : RC_DRIVER_SCANCODE);
+	if (!dev)
+		goto err_out_free;
+
+	ir->dev = dev;
+
 	/*
 	 * The usage of mask_keycode were very convenient, due to several
 	 * reasons. Among others, the scancode tables were using the scancode
@@ -477,12 +481,10 @@ int cx88_ir_init(struct cx88_core *core, struct pci_dev *pci)
 	dev->close = cx88_ir_close;
 	dev->scancode_mask = hardware_mask;
 
-	if (ir->sampling) {
+	if (ir->sampling)
 		dev->timeout = MS_TO_US(10); /* 10 ms */
-	} else {
-		dev->driver_type = RC_DRIVER_SCANCODE;
+	else
 		dev->allowed_protocols = rc_proto;
-	}
 
 	ir->core = core;
 	core->ir = ir;
diff --git a/drivers/media/pci/cx88/cx88-mpeg.c b/drivers/media/pci/cx88/cx88-mpeg.c
index 0c07ad33579914..cd2366768f4a32 100644
--- a/drivers/media/pci/cx88/cx88-mpeg.c
+++ b/drivers/media/pci/cx88/cx88-mpeg.c
@@ -392,7 +392,8 @@ static int cx8802_init_common(struct cx8802_dev *dev)
 	err = dma_set_mask(&dev->pci->dev, DMA_BIT_MASK(32));
 	if (err) {
 		pr_err("Oops: no 32bit PCI DMA ???\n");
-		return -EIO;
+		err = -EIO;
+		goto fail_disable_device;
 	}
 
 	dev->pci_rev = dev->pci->revision;
@@ -413,13 +414,17 @@ static int cx8802_init_common(struct cx8802_dev *dev)
 			  IRQF_SHARED, dev->core->name, dev);
 	if (err < 0) {
 		pr_err("can't get IRQ %d\n", dev->pci->irq);
-		return err;
+		goto fail_disable_device;
 	}
 	cx_set(MO_PCI_INTMSK, core->pci_irqmask);
 
 	/* everything worked */
 	pci_set_drvdata(dev->pci, dev);
 	return 0;
+
+fail_disable_device:
+	pci_disable_device(dev->pci);
+	return err;
 }
 
 static void cx8802_fini_common(struct cx8802_dev *dev)
diff --git a/drivers/media/pci/cx88/cx88-video.c b/drivers/media/pci/cx88/cx88-video.c
index eaa46a2f92e747..f8f2843495f3c5 100644
--- a/drivers/media/pci/cx88/cx88-video.c
+++ b/drivers/media/pci/cx88/cx88-video.c
@@ -1549,7 +1549,7 @@ static void cx8800_finidev(struct pci_dev *pci_dev)
 	kfree(dev);
 }
 
-static int __maybe_unused cx8800_suspend(struct device *dev_d)
+static int cx8800_suspend(struct device *dev_d)
 {
 	struct cx8800_dev *dev = dev_get_drvdata(dev_d);
 	struct cx88_core *core = dev->core;
@@ -1576,7 +1576,7 @@ static int __maybe_unused cx8800_suspend(struct device *dev_d)
 	return 0;
 }
 
-static int __maybe_unused cx8800_resume(struct device *dev_d)
+static int cx8800_resume(struct device *dev_d)
 {
 	struct cx8800_dev *dev = dev_get_drvdata(dev_d);
 	struct cx88_core *core = dev->core;
@@ -1617,14 +1617,14 @@ static const struct pci_device_id cx8800_pci_tbl[] = {
 };
 MODULE_DEVICE_TABLE(pci, cx8800_pci_tbl);
 
-static SIMPLE_DEV_PM_OPS(cx8800_pm_ops, cx8800_suspend, cx8800_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(cx8800_pm_ops, cx8800_suspend, cx8800_resume);
 
 static struct pci_driver cx8800_pci_driver = {
 	.name      = "cx8800",
 	.id_table  = cx8800_pci_tbl,
 	.probe     = cx8800_initdev,
 	.remove    = cx8800_finidev,
-	.driver.pm = &cx8800_pm_ops,
+	.driver.pm = pm_sleep_ptr(&cx8800_pm_ops),
 };
 
 module_pci_driver(cx8800_pci_driver);
diff --git a/drivers/media/pci/hws/hws.h b/drivers/media/pci/hws/hws.h
index 8fbe1fe27844dd..d87d52674b6921 100644
--- a/drivers/media/pci/hws/hws.h
+++ b/drivers/media/pci/hws/hws.h
@@ -33,8 +33,8 @@ struct hws_pix_state {
 	u32 sizeimage;		/* full frame */
 	enum v4l2_field field;	/* V4L2_FIELD_NONE or INTERLACED */
 	enum v4l2_colorspace colorspace;	/* e.g., REC709 */
-	enum v4l2_ycbcr_encoding ycbcr_enc;	/* V4L2_YCBCR_ENC_DEFAULT */
-	enum v4l2_quantization quantization;	/* V4L2_QUANTIZATION_LIM_RANGE */
+	enum v4l2_ycbcr_encoding ycbcr_enc;	/* V4L2_YCBCR_ENC_601 */
+	enum v4l2_quantization quantization;	/* V4L2_QUANTIZATION_FULL_RANGE */
 	enum v4l2_xfer_func xfer_func;	/* V4L2_XFER_FUNC_DEFAULT */
 	bool interlaced;	/* cached hardware state */
 	u32 half_size;		/* hardware half-frame size */
diff --git a/drivers/media/pci/hws/hws_pci.c b/drivers/media/pci/hws/hws_pci.c
index 30bb7d34465b8f..f06e60dc2ee618 100644
--- a/drivers/media/pci/hws/hws_pci.c
+++ b/drivers/media/pci/hws/hws_pci.c
@@ -33,8 +33,8 @@ static unsigned long long hws_elapsed_us(u64 start_ns)
 }
 
 /* register layout inside HWS_REG_DEVICE_INFO */
-#define DEVINFO_VER GENMASK(7, 0)
-#define DEVINFO_SUBVER GENMASK(15, 8)
+#define DEVINFO_VER GENMASK(15, 8)
+#define DEVINFO_SUBVER GENMASK(23, 16)
 #define DEVINFO_YV12 GENMASK(31, 28)
 #define DEVINFO_HWKEY GENMASK(27, 24)
 #define DEVINFO_PORTID GENMASK(25, 24) /* low 2 bits of HW-key */
@@ -441,7 +441,7 @@ static int hws_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id)
 	hws_init_video_sys(hws, false);
 
 	/* 5) Init channels (video state, locks, vb2, ctrls) */
-	for (i = 0; i < hws->max_channels; i++) {
+	for (i = 0; i < hws->cur_max_video_ch; i++) {
 		ret = hws_video_init_channel(hws, i);
 		if (ret) {
 			dev_err(&pdev->dev, "video channel init failed (ch=%d)\n", i);
diff --git a/drivers/media/pci/hws/hws_reg.h b/drivers/media/pci/hws/hws_reg.h
index e4fb4af4443403..ba1d22f4376c15 100644
--- a/drivers/media/pci/hws/hws_reg.h
+++ b/drivers/media/pci/hws/hws_reg.h
@@ -70,7 +70,8 @@
 #define HWS_CTL_IRQ_ENABLE_BIT BIT(0)   /* Global interrupt enable bit */
 /*  Write 0x00 to fully reset decoder,
  *  set bit 31=1 to "start run",
- *  low byte=0x13 selects YUYV/BT.709/etc,
+ *  low byte=0x13 is the vendor baseline capture-mode value; its individual
+ *  bit meanings are not documented,
  *  in ReadChipId() we also write 0x00 and 0x10 here for chip-ID sequencing.
  */
 
@@ -121,9 +122,10 @@
 #define HWS_REG_DEVICE_INFO   (CVBS_IN_BASE +  88 * PCIE_BARADDROFSIZE)
 /*
  * Reading this 32-bit word returns:
- *   bits 7:0   = "device version"
- *   bits 15:8  = "device sub-version"
- *   bits 23:24 = "HW key / port ID" etc.
+ *   bits 7:0   = unused by the baseline driver
+ *   bits 15:8  = device version
+ *   bits 23:16 = device sub-version
+ *   bits 27:24 = HW key (port ID in bits 25:24)
  *   bits 31:28 = "support YV12" flags
  */
 
diff --git a/drivers/media/pci/hws/hws_v4l2_ioctl.c b/drivers/media/pci/hws/hws_v4l2_ioctl.c
index 0303e311ee4a8e..ce396b7225d220 100644
--- a/drivers/media/pci/hws/hws_v4l2_ioctl.c
+++ b/drivers/media/pci/hws/hws_v4l2_ioctl.c
@@ -511,8 +511,9 @@ static inline void hws_set_colorimetry_state(struct hws_pix_state *p)
 {
 	bool sd = p->height <= 576;
 
+	/* Captured samples use full-range BT.601 Y'CbCr at every resolution. */
 	p->colorspace   = sd ? V4L2_COLORSPACE_SMPTE170M : V4L2_COLORSPACE_REC709;
-	p->ycbcr_enc    = V4L2_YCBCR_ENC_DEFAULT;
+	p->ycbcr_enc    = V4L2_YCBCR_ENC_601;
 	p->quantization = V4L2_QUANTIZATION_FULL_RANGE;
 	p->xfer_func    = V4L2_XFER_FUNC_DEFAULT;
 }
@@ -736,7 +737,7 @@ static inline void hws_set_colorimetry_fmt(struct v4l2_pix_format *p)
 	bool sd = p->height <= 576;
 
 	p->colorspace   = sd ? V4L2_COLORSPACE_SMPTE170M : V4L2_COLORSPACE_REC709;
-	p->ycbcr_enc    = V4L2_YCBCR_ENC_DEFAULT;
+	p->ycbcr_enc    = V4L2_YCBCR_ENC_601;
 	p->quantization = V4L2_QUANTIZATION_FULL_RANGE;
 	p->xfer_func    = V4L2_XFER_FUNC_DEFAULT;
 }
diff --git a/drivers/media/pci/hws/hws_video.c b/drivers/media/pci/hws/hws_video.c
index 18e4bc6901d308..dbe0fc2a66b575 100644
--- a/drivers/media/pci/hws/hws_video.c
+++ b/drivers/media/pci/hws/hws_video.c
@@ -327,7 +327,7 @@ int hws_video_init_channel(struct hws_pcie_dev *pdev, int ch)
 	vid->pix.sizeimage = vid->pix.bytesperline * vid->pix.height;
 	vid->pix.field = V4L2_FIELD_NONE;
 	vid->pix.colorspace = V4L2_COLORSPACE_REC709;
-	vid->pix.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	vid->pix.ycbcr_enc = V4L2_YCBCR_ENC_601;
 	vid->pix.quantization = V4L2_QUANTIZATION_FULL_RANGE;
 	vid->pix.xfer_func = V4L2_XFER_FUNC_DEFAULT;
 	vid->pix.interlaced = false;
@@ -895,6 +895,8 @@ static void hws_video_apply_mode_change(struct hws_pcie_dev *pdx,
 	v->pix.width = w;
 	v->pix.height = h;
 	v->pix.interlaced = interlaced;
+	v->pix.colorspace = h <= 576 ? V4L2_COLORSPACE_SMPTE170M :
+					 V4L2_COLORSPACE_REC709;
 	hws_set_current_dv_timings(v, w, h, interlaced);
 	v->current_fps = fps;
 
@@ -1292,6 +1294,8 @@ static void hws_stop_streaming(struct vb2_queue *q)
 	WRITE_ONCE(v->stop_requested, true);
 
 	hws_enable_video_capture(v->parent, v->channel_index, false);
+	if (hws->irq >= 0)
+		synchronize_irq(hws->irq);
 
 	/* 2) Collect in-flight + queued under the IRQ lock */
 	spin_lock_irqsave(&v->irq_lock, flags);
diff --git a/drivers/media/pci/intel/ipu-bridge.c b/drivers/media/pci/intel/ipu-bridge.c
index bd64c0400c0db2..e476e269a50a4d 100644
--- a/drivers/media/pci/intel/ipu-bridge.c
+++ b/drivers/media/pci/intel/ipu-bridge.c
@@ -8,6 +8,7 @@
 #include <linux/dmi.h>
 #include <linux/i2c.h>
 #include <linux/mei_cl_bus.h>
+#include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/property.h>
@@ -15,6 +16,7 @@
 #include <linux/workqueue.h>
 
 #include <media/ipu-bridge.h>
+#include <media/ipu6-pci-table.h>
 #include <media/v4l2-fwnode.h>
 
 #define ADEV_DEV(adev) ACPI_PTR(&((adev)->dev))
@@ -48,6 +50,14 @@
  *
  * Please keep the list sorted by ACPI HID.
  */
+/* IPU6 variants whose CSI-2 receiver needs the ov5693 clock lane gated */
+static const u16 ipu6_ov5693_ncont_clk[] = {
+	PCI_DEVICE_ID_INTEL_IPU6,		/* Tiger Lake */
+	PCI_DEVICE_ID_INTEL_IPU6EP_ADLP,	/* Alder Lake-P */
+	PCI_DEVICE_ID_INTEL_IPU6EP_ADLN,	/* Alder Lake-N */
+	0
+};
+
 static const struct ipu_sensor_config ipu_supported_sensors[] = {
 	/* Himax HM1092 */
 	IPU_SENSOR_CONFIG("HIMX1092", 2, 180000000, 180480000),
@@ -60,6 +70,9 @@ static const struct ipu_sensor_config ipu_supported_sensors[] = {
 	/* GalaxyCore GC0310 */
 	IPU_SENSOR_CONFIG("INT0310", 1, 55692000),
 	/* Omnivision OV5693 */
+	IPU_SENSOR_CONFIG_MATCH_FL("INT33BE", ipu6_ov5693_ncont_clk,
+				   IPU_BR_FL_CSI2_CLK_NONCONTINUOUS,
+				   1, 419200000),
 	IPU_SENSOR_CONFIG("INT33BE", 1, 419200000),
 	/* Onsemi MT9M114 */
 	IPU_SENSOR_CONFIG("INT33F0", 1, 384000000),
@@ -75,19 +88,22 @@ static const struct ipu_sensor_config ipu_supported_sensors[] = {
 	IPU_SENSOR_CONFIG("INT3537", 1, 437000000),
 	/* Lontium lt6911uxe */
 	IPU_SENSOR_CONFIG("INTC10C5", 0),
-	/* Omnivision OV01A10 / OV01A1S */
+	/* Omnivision OV01A10 / OV01A1B / OV01A1S */
 	IPU_SENSOR_CONFIG("OVTI01A0", 1, 400000000),
+	IPU_SENSOR_CONFIG("OVTI01AB", 1, 400000000),
 	IPU_SENSOR_CONFIG("OVTI01AS", 1, 400000000),
 	/* Omnivision OV02C10 */
 	IPU_SENSOR_CONFIG("OVTI02C1", 1, 400000000),
 	/* Omnivision OV02E10 */
 	IPU_SENSOR_CONFIG("OVTI02E1", 1, 360000000),
 	/* Omnivision ov05c10 */
-	IPU_SENSOR_CONFIG("OVTI05C1", 1, 480000000),
+	IPU_SENSOR_CONFIG("OVTI05C1", 2, 480000000, 900000000),
 	/* Omnivision OV08A10 */
 	IPU_SENSOR_CONFIG("OVTI08A1", 1, 500000000),
 	/* Omnivision OV08x40 */
 	IPU_SENSOR_CONFIG("OVTI08F4", 3, 400000000, 749000000, 800000000),
+	/* Omnivision OV13858 */
+	IPU_SENSOR_CONFIG("OVTID858", 2, 540000000, 270000000),
 	/* Omnivision OV13B10 */
 	IPU_SENSOR_CONFIG("OVTI13B1", 1, 560000000),
 	IPU_SENSOR_CONFIG("OVTIDB10", 1, 560000000),
@@ -95,6 +111,11 @@ static const struct ipu_sensor_config ipu_supported_sensors[] = {
 	IPU_SENSOR_CONFIG("OVTI2680", 1, 331200000),
 	/* Omnivision OV5675 */
 	IPU_SENSOR_CONFIG("OVTI5675", 1, 450000000),
+	/* Omnivision OV5693 */
+	IPU_SENSOR_CONFIG_MATCH_FL("OVTI5693", ipu6_ov5693_ncont_clk,
+				   IPU_BR_FL_CSI2_CLK_NONCONTINUOUS,
+				   1, 419200000),
+	IPU_SENSOR_CONFIG("OVTI5693", 1, 419200000),
 	/* Omnivision OV8856 */
 	IPU_SENSOR_CONFIG("OVTI8856", 3, 180000000, 360000000, 720000000),
 	/* Sony IMX471 */
@@ -124,6 +145,13 @@ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = {
 		},
 		.driver_data = "OVTI02C1",
 	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 14 (Dell 14 Premium) DA14250"),
+		},
+		.driver_data = "OVTI02C1",
+	},
 	{
 		.matches = {
 			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
@@ -145,6 +173,13 @@ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = {
 		},
 		.driver_data = "OVTI02C1",
 	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Dell Pro 14 Premium PA14260"),
+		},
+		.driver_data = "OVTI08F4",
+	},
 	/*
 	 * The first four characters of DMI_BOARD_NAME identify the Lenovo
 	 * machine type/model. For example, a DMI_BOARD_NAME starting with
@@ -184,6 +219,14 @@ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = {
 		},
 		.driver_data = "SONY471A",
 	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Microsoft Corporation"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME,
+					"Surface Pro for Business 11th Edition with Intel"),
+		},
+		.driver_data = "OVTID858",
+	},
 	{
 		/* Samsung Galaxy Book5 Pro 360 */
 		.matches = {
@@ -192,6 +235,14 @@ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = {
 		},
 		.driver_data = "OVTI02E1",
 	},
+	{
+		/* Samsung Galaxy Book3 Ultra */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "960XFH"),
+		},
+		.driver_data = "OVTI02C1",
+	},
 	{} /* Terminating entry */
 };
 
@@ -326,8 +377,8 @@ static int ipu_bridge_check_ivsc_dev(struct ipu_sensor *sensor,
 	if (adev) {
 		csi_dev = ipu_bridge_get_ivsc_csi_dev(adev);
 		if (!csi_dev) {
-			acpi_dev_put(adev);
 			dev_err(ADEV_DEV(adev), "Failed to find MEI or CVS CSI dev\n");
+			acpi_dev_put(adev);
 			return -ENODEV;
 		}
 
@@ -436,6 +487,7 @@ static enum v4l2_fwnode_orientation ipu_bridge_parse_orientation(struct acpi_dev
 
 int ipu_bridge_parse_ssdb(struct acpi_device *adev, struct ipu_sensor *sensor)
 {
+	acpi_handle handle = acpi_device_handle(ACPI_PTR(adev));
 	struct ipu_sensor_ssdb ssdb = {};
 	int ret;
 
@@ -459,8 +511,15 @@ int ipu_bridge_parse_ssdb(struct acpi_device *adev, struct ipu_sensor *sensor)
 	sensor->rotation = ipu_bridge_parse_rotation(adev, &ssdb);
 	sensor->orientation = ipu_bridge_parse_orientation(adev);
 
-	if (ssdb.vcmtype)
+	acpi_handle_debug(handle,
+			  "CSI-2 port %u, lanes %u, mclkspeed %u, rotation %u (SSDB %u), orientation %u\n",
+			  sensor->link, sensor->lanes, sensor->mclkspeed,
+			  sensor->rotation, ssdb.degree, sensor->orientation);
+
+	if (ssdb.vcmtype) {
 		sensor->vcm_type = ipu_vcm_types[ssdb.vcmtype - 1];
+		acpi_handle_debug(handle, "VCM %s\n", sensor->vcm_type);
+	}
 
 	return 0;
 }
@@ -473,6 +532,7 @@ static void ipu_bridge_create_fwnode_properties(
 {
 	struct ipu_property_names *names = &sensor->prop_names;
 	struct software_node *nodes = sensor->swnodes;
+	unsigned int i = 0;
 
 	sensor->prop_names = prop_names;
 
@@ -530,21 +590,25 @@ static void ipu_bridge_create_fwnode_properties(
 			PROPERTY_ENTRY_REF_ARRAY("lens-focus", sensor->vcm_ref);
 	}
 
-	sensor->ep_properties[0] = PROPERTY_ENTRY_U32(
-					sensor->prop_names.bus_type,
-					V4L2_FWNODE_BUS_TYPE_CSI2_DPHY);
-	sensor->ep_properties[1] = PROPERTY_ENTRY_U32_ARRAY_LEN(
-					sensor->prop_names.data_lanes,
-					bridge->data_lanes, sensor->lanes);
-	sensor->ep_properties[2] = PROPERTY_ENTRY_REF_ARRAY(
-					sensor->prop_names.remote_endpoint,
-					sensor->local_ref);
+	sensor->ep_properties[IPU_BRIDGE_NEXT_PROPERTY(i, EP_BUS_TYPE)] =
+		PROPERTY_ENTRY_U32(names->bus_type,
+				   V4L2_FWNODE_BUS_TYPE_CSI2_DPHY);
+	sensor->ep_properties[IPU_BRIDGE_NEXT_PROPERTY(i, EP_DATA_LANES)] =
+		PROPERTY_ENTRY_U32_ARRAY_LEN(names->data_lanes,
+					     bridge->data_lanes, sensor->lanes);
+	sensor->ep_properties[IPU_BRIDGE_NEXT_PROPERTY(i, EP_REMOTE_EP)] =
+		PROPERTY_ENTRY_REF_ARRAY(names->remote_endpoint,
+					 sensor->local_ref);
 
 	if (cfg->nr_link_freqs > 0)
-		sensor->ep_properties[3] = PROPERTY_ENTRY_U64_ARRAY_LEN(
-			sensor->prop_names.link_frequencies,
-			cfg->link_freqs,
-			cfg->nr_link_freqs);
+		sensor->ep_properties[IPU_BRIDGE_NEXT_PROPERTY(i, EP_LINK_FREQUENCIES)] =
+			PROPERTY_ENTRY_U64_ARRAY_LEN(names->link_frequencies,
+						     cfg->link_freqs,
+						     cfg->nr_link_freqs);
+
+	if (cfg->flags & IPU_BR_FL_CSI2_CLK_NONCONTINUOUS)
+		sensor->ep_properties[IPU_BRIDGE_NEXT_PROPERTY(i, EP_CLOCK_NONCONTINUOUS)] =
+			PROPERTY_ENTRY_BOOL("clock-noncontinuous");
 
 	sensor->ipu_properties[0] = PROPERTY_ENTRY_U32_ARRAY_LEN(
 					sensor->prop_names.data_lanes,
@@ -856,8 +920,8 @@ static int ipu_bridge_connect_sensor(const struct ipu_sensor_config *cfg,
 		if (ret)
 			goto err_free_swnodes;
 
-		dev_info(bridge->dev, "Found supported sensor %s\n",
-			 acpi_dev_name(adev));
+		dev_info(bridge->dev, "Found supported sensor %s (%pfw)\n",
+			 acpi_dev_name(adev), primary);
 
 		bridge->n_sensors++;
 	}
@@ -874,8 +938,28 @@ static int ipu_bridge_connect_sensor(const struct ipu_sensor_config *cfg,
 	return ret;
 }
 
+/*
+ * Whether a sensor config applies to the IPU this bridge sits on. A config
+ * listing PCI product IDs only applies to those IPUs.
+ */
+static bool ipu_bridge_config_matches(const struct ipu_sensor_config *cfg,
+				      struct ipu_bridge *bridge)
+{
+	const u16 *id;
+
+	if (!cfg->pci_ids)
+		return true;
+
+	for (id = cfg->pci_ids; *id; id++)
+		if (*id == bridge->pci_id)
+			return true;
+
+	return false;
+}
+
 static int ipu_bridge_connect_sensors(struct ipu_bridge *bridge)
 {
+	const char *done_hid = NULL;
 	unsigned int i;
 	int ret;
 
@@ -883,9 +967,22 @@ static int ipu_bridge_connect_sensors(struct ipu_bridge *bridge)
 		const struct ipu_sensor_config *cfg =
 			&ipu_supported_sensors[i];
 
+		/*
+		 * Entries for one HID are adjacent, IPU-specific ones first,
+		 * so the generic entry is skipped once a specific one has
+		 * matched and the sensor is not connected twice.
+		 */
+		if (done_hid && !strcmp(cfg->hid, done_hid))
+			continue;
+
+		if (!ipu_bridge_config_matches(cfg, bridge))
+			continue;
+
 		ret = ipu_bridge_connect_sensor(cfg, bridge);
 		if (ret)
 			goto err_unregister_sensors;
+
+		done_hid = cfg->hid;
 	}
 
 	return 0;
@@ -942,6 +1039,18 @@ static int ipu_bridge_check_fwnode_graph(struct fwnode_handle *fwnode)
 	return ipu_bridge_check_fwnode_graph(fwnode->secondary);
 }
 
+struct pci_dev *ipu_bridge_get_ipu6(void)
+{
+	struct pci_dev *ipu = NULL;
+
+	for (unsigned int i = 0; !ipu && ipu6_pci_tbl[i].vendor; i++)
+		ipu = pci_get_device(ipu6_pci_tbl[i].vendor,
+				     ipu6_pci_tbl[i].device, NULL);
+
+	return ipu;
+}
+EXPORT_SYMBOL_NS_GPL(ipu_bridge_get_ipu6, "INTEL_IPU_BRIDGE");
+
 static DEFINE_MUTEX(ipu_bridge_mutex);
 
 int ipu_bridge_init(struct device *dev,
@@ -969,6 +1078,7 @@ int ipu_bridge_init(struct device *dev,
 		sizeof(bridge->ipu_node_name));
 	bridge->ipu_hid_node.name = bridge->ipu_node_name;
 	bridge->dev = dev;
+	bridge->pci_id = dev_is_pci(dev) ? to_pci_dev(dev)->device : 0;
 	bridge->parse_sensor_fwnode = parse_sensor_fwnode;
 
 	ret = software_node_register(&bridge->ipu_hid_node);
diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2.c b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
index eb1824ee86fdde..4d2b231291f562 100644
--- a/drivers/media/pci/intel/ipu3/ipu3-cio2.c
+++ b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
@@ -1229,6 +1229,7 @@ static int cio2_subdev_init_state(struct v4l2_subdev *sd,
 }
 
 static int cio2_subdev_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/pci/intel/ipu6/Kconfig b/drivers/media/pci/intel/ipu6/Kconfig
index 1129e2beb4bec7..a1d036a78b571d 100644
--- a/drivers/media/pci/intel/ipu6/Kconfig
+++ b/drivers/media/pci/intel/ipu6/Kconfig
@@ -16,3 +16,12 @@ config VIDEO_INTEL_IPU6
 
 	  To compile this driver, say Y here! It contains 2 modules -
 	  intel_ipu6 and intel_ipu6_isys.
+
+config VIDEO_INTEL_IPU6_IPU7
+	bool "Support IPU 7 and IPU 7.5 in ipu6 driver by default"
+	depends on VIDEO_INTEL_IPU6
+	default y
+	help
+	  Support Intel IPU 7 and IPU 7.5 in ipu6 driver by default, instead of
+	  the ipu7 staging driver. This behaviour can be changed at runtime with
+	  module parameters in the respective drivers.
diff --git a/drivers/media/pci/intel/ipu6/Makefile b/drivers/media/pci/intel/ipu6/Makefile
index a821b0a1567f49..676bc978b97219 100644
--- a/drivers/media/pci/intel/ipu6/Makefile
+++ b/drivers/media/pci/intel/ipu6/Makefile
@@ -4,20 +4,26 @@ intel-ipu6-y			:= ipu6.o \
 				ipu6-bus.o \
 				ipu6-dma.o \
 				ipu6-mmu.o \
+				ipu6-mmu-hw.o \
+				ipu7-mmu-hw.o \
 				ipu6-buttress.o \
 				ipu6-cpd.o \
-				ipu6-fw-com.o
+				ipu6-fw-com.o \
+				ipu7-fw-com.o \
+				ipu7-boot.o
 
 obj-$(CONFIG_VIDEO_INTEL_IPU6)	+= intel-ipu6.o
 
 intel-ipu6-isys-y		:= ipu6-isys.o \
 				ipu6-isys-csi2.o \
 				ipu6-fw-isys.o \
+				ipu7-fw-isys.o \
 				ipu6-isys-video.o \
 				ipu6-isys-queue.o \
 				ipu6-isys-subdev.o \
 				ipu6-isys-mcd-phy.o \
 				ipu6-isys-jsl-phy.o \
-				ipu6-isys-dwc-phy.o
+				ipu6-isys-dwc-phy.o \
+				ipu7-isys-csi-phy.o
 
 obj-$(CONFIG_VIDEO_INTEL_IPU6)	+= intel-ipu6-isys.o
diff --git a/drivers/media/pci/intel/ipu6/ipu6-bus.h b/drivers/media/pci/intel/ipu6/ipu6-bus.h
index a08c5468d536b7..d2f93eb03deaa9 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-bus.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-bus.h
@@ -26,17 +26,18 @@ struct ipu6_bus_device {
 	struct ipu6_mmu *mmu;
 	struct ipu6_device *isp;
 	const struct ipu6_buttress_ctrl *ctrl;
-	const struct firmware *fw;
 	struct sg_table fw_sgt;
 	u64 *pkg_dir;
 	dma_addr_t pkg_dir_dma_addr;
 	unsigned int pkg_dir_size;
+	u32 fw_entry;
 };
 
 struct ipu6_auxdrv_data {
 	irqreturn_t (*isr)(struct ipu6_bus_device *adev);
 	irqreturn_t (*isr_threaded)(struct ipu6_bus_device *adev);
 	bool wake_isr_thread;
+	const struct ipu6_fw_isys_ops *fw_ops;
 };
 
 #define to_ipu6_bus_device(_dev) \
@@ -45,6 +46,9 @@ struct ipu6_auxdrv_data {
 	container_of(_auxdev, struct ipu6_bus_device, auxdev)
 #define ipu6_bus_get_drvdata(adev) dev_get_drvdata(&(adev)->auxdev.dev)
 
+extern const struct ipu6_fw_isys_ops ipu6_fw_isys_ops;
+extern const struct ipu6_fw_isys_ops ipu7_fw_isys_ops;
+
 struct ipu6_bus_device *
 ipu6_bus_initialize_device(struct pci_dev *pdev, struct device *parent,
 			   void *pdata, const struct ipu6_buttress_ctrl *ctrl,
diff --git a/drivers/media/pci/intel/ipu6/ipu6-buttress.c b/drivers/media/pci/intel/ipu6/ipu6-buttress.c
index e0ecb4c8b0815d..105de1744dffb8 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-buttress.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-buttress.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2013--2024 Intel Corporation
  */
 
+#include <asm/cpu_device_id.h>
 #include <linux/bitfield.h>
 #include <linux/bits.h>
 #include <linux/completion.h>
@@ -55,16 +56,11 @@
 
 #define BUTTRESS_MAX_CONSECUTIVE_IRQS	100
 
-static const u32 ipu6_adev_irq_mask[2] = {
-	BUTTRESS_ISR_IS_IRQ,
-	BUTTRESS_ISR_PS_IRQ
-};
-
-int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
-			    struct ipu6_buttress_ipc *ipc)
+int ipu6_buttress_ipc_reset(struct ipu6_device *isp)
 {
 	unsigned int retries = BUTTRESS_IPC_RESET_RETRY;
 	struct ipu6_buttress *b = &isp->buttress;
+	const struct ipu6_buttress_registers *regs = b->regs;
 	u32 val = 0, csr_in_clr;
 
 	if (!isp->secure_mode) {
@@ -75,11 +71,11 @@ int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
 	mutex_lock(&b->ipc_mutex);
 
 	/* Clear-by-1 CSR (all bits), corresponding internal states. */
-	val = readl(isp->base + ipc->csr_in);
-	writel(val, isp->base + ipc->csr_in);
+	val = readl(isp->base + regs->csr_in);
+	writel(val, isp->base + regs->csr_in);
 
 	/* Set peer CSR bit IPC_PEER_COMP_ACTIONS_RST_PHASE1 */
-	writel(ENTRY, isp->base + ipc->csr_out);
+	writel(ENTRY, isp->base + regs->csr_out);
 	/*
 	 * Clear-by-1 all CSR bits EXCEPT following
 	 * bits:
@@ -94,7 +90,7 @@ int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
 
 	do {
 		usleep_range(400, 500);
-		val = readl(isp->base + ipc->csr_in);
+		val = readl(isp->base + regs->csr_in);
 		switch (val) {
 		case ENTRY | EXIT:
 		case ENTRY | EXIT | QUERY:
@@ -105,8 +101,8 @@ int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
 			 * 2) Set peer CSR bit
 			 * IPC_PEER_QUERIED_IP_COMP_ACTIONS_RST_PHASE.
 			 */
-			writel(ENTRY | EXIT, isp->base + ipc->csr_in);
-			writel(QUERY, isp->base + ipc->csr_out);
+			writel(ENTRY | EXIT, isp->base + regs->csr_in);
+			writel(QUERY, isp->base + regs->csr_out);
 			break;
 		case ENTRY:
 		case ENTRY | QUERY:
@@ -117,8 +113,8 @@ int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
 			 * 2) Set peer CSR bit
 			 * IPC_PEER_COMP_ACTIONS_RST_PHASE1.
 			 */
-			writel(ENTRY | QUERY, isp->base + ipc->csr_in);
-			writel(ENTRY, isp->base + ipc->csr_out);
+			writel(ENTRY | QUERY, isp->base + regs->csr_in);
+			writel(ENTRY, isp->base + regs->csr_out);
 			break;
 		case EXIT:
 		case EXIT | QUERY:
@@ -135,17 +131,17 @@ int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
 			 * 3) Set peer CSR bit
 			 * IPC_PEER_COMP_ACTIONS_RST_PHASE2.
 			 */
-			writel(EXIT, isp->base + ipc->csr_in);
-			writel(0, isp->base + ipc->db0_in);
-			writel(csr_in_clr, isp->base + ipc->csr_in);
-			writel(EXIT, isp->base + ipc->csr_out);
+			writel(EXIT, isp->base + regs->csr_in);
+			writel(0, isp->base + regs->db0_in);
+			writel(csr_in_clr, isp->base + regs->csr_in);
+			writel(EXIT, isp->base + regs->csr_out);
 
 			/*
 			 * Read csr_in again to make sure if RST_PHASE2 is done.
 			 * If csr_in is QUERY, it should be handled again.
 			 */
 			usleep_range(200, 300);
-			val = readl(isp->base + ipc->csr_in);
+			val = readl(isp->base + regs->csr_in);
 			if (val & QUERY) {
 				dev_dbg(&isp->pdev->dev,
 					"RST_PHASE2 retry csr_in = %x\n", val);
@@ -160,8 +156,8 @@ int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
 			 * 2) Set peer CSR bit
 			 * IPC_PEER_COMP_ACTIONS_RST_PHASE1
 			 */
-			writel(QUERY, isp->base + ipc->csr_in);
-			writel(ENTRY, isp->base + ipc->csr_out);
+			writel(QUERY, isp->base + regs->csr_in);
+			writel(ENTRY, isp->base + regs->csr_out);
 			break;
 		default:
 			dev_dbg_ratelimited(&isp->pdev->dev,
@@ -176,42 +172,42 @@ int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
 	return -ETIMEDOUT;
 }
 
-static void ipu6_buttress_ipc_validity_close(struct ipu6_device *isp,
-					     struct ipu6_buttress_ipc *ipc)
+static void ipu6_buttress_ipc_validity_close(struct ipu6_device *isp)
 {
 	writel(BUTTRESS_IU2CSECSR_IPC_PEER_DEASSERTED_REG_VALID_REQ,
-	       isp->base + ipc->csr_out);
+	       isp->base + isp->buttress.regs->csr_out);
 }
 
 static int
-ipu6_buttress_ipc_validity_open(struct ipu6_device *isp,
-				struct ipu6_buttress_ipc *ipc)
+ipu6_buttress_ipc_validity_open(struct ipu6_device *isp)
 {
 	unsigned int mask = BUTTRESS_IU2CSECSR_IPC_PEER_ACKED_REG_VALID;
+	const struct ipu6_buttress_registers *regs = isp->buttress.regs;
 	void __iomem *addr;
 	int ret;
 	u32 val;
 
 	writel(BUTTRESS_IU2CSECSR_IPC_PEER_ASSERTED_REG_VALID_REQ,
-	       isp->base + ipc->csr_out);
+	       isp->base + regs->csr_out);
 
-	addr = isp->base + ipc->csr_in;
+	addr = isp->base + regs->csr_in;
 	ret = readl_poll_timeout(addr, val, val & mask, 200,
 				 BUTTRESS_IPC_VALIDITY_TIMEOUT_US);
 	if (ret) {
 		dev_err(&isp->pdev->dev, "CSE validity timeout 0x%x\n", val);
-		ipu6_buttress_ipc_validity_close(isp, ipc);
+		ipu6_buttress_ipc_validity_close(isp);
 	}
 
 	return ret;
 }
 
-static void ipu6_buttress_ipc_recv(struct ipu6_device *isp,
-				   struct ipu6_buttress_ipc *ipc, u32 *ipc_msg)
+static void ipu6_buttress_ipc_recv(struct ipu6_device *isp, u32 *ipc_msg)
 {
+	const struct ipu6_buttress_registers *regs = isp->buttress.regs;
+
 	if (ipc_msg)
-		*ipc_msg = readl(isp->base + ipc->data0_in);
-	writel(0, isp->base + ipc->db0_in);
+		*ipc_msg = readl(isp->base + regs->data0_in);
+	writel(0, isp->base + regs->db0_in);
 }
 
 static int ipu6_buttress_ipc_send_bulk(struct ipu6_device *isp,
@@ -221,14 +217,15 @@ static int ipu6_buttress_ipc_send_bulk(struct ipu6_device *isp,
 	unsigned long tx_timeout_jiffies, rx_timeout_jiffies;
 	unsigned int i, retry = BUTTRESS_IPC_CMD_SEND_RETRY;
 	struct ipu6_buttress *b = &isp->buttress;
-	struct ipu6_buttress_ipc *ipc = &b->cse;
+	struct ipu6_buttress_ipc *ipc = &b->ipc;
+	const struct ipu6_buttress_registers *regs = b->regs;
 	u32 val;
 	int ret;
 	int tout;
 
 	mutex_lock(&b->ipc_mutex);
 
-	ret = ipu6_buttress_ipc_validity_open(isp, ipc);
+	ret = ipu6_buttress_ipc_validity_open(isp);
 	if (ret) {
 		dev_err(&isp->pdev->dev, "IPC validity open failed\n");
 		goto out;
@@ -244,9 +241,9 @@ static int ipu6_buttress_ipc_send_bulk(struct ipu6_device *isp,
 
 		dev_dbg(&isp->pdev->dev, "bulk IPC command: 0x%x\n",
 			msgs[i].cmd);
-		writel(msgs[i].cmd, isp->base + ipc->data0_out);
+		writel(msgs[i].cmd, isp->base + regs->data0_out);
 		val = BUTTRESS_IU2CSEDB0_BUSY | msgs[i].cmd_size;
-		writel(val, isp->base + ipc->db0_out);
+		writel(val, isp->base + regs->db0_out);
 
 		tout = wait_for_completion_timeout(&ipc->send_complete,
 						   tx_timeout_jiffies);
@@ -258,7 +255,7 @@ static int ipu6_buttress_ipc_send_bulk(struct ipu6_device *isp,
 			}
 
 			/* Try again if CSE is not responding on first try */
-			writel(0, isp->base + ipc->db0_out);
+			writel(0, isp->base + regs->db0_out);
 			i--;
 			continue;
 		}
@@ -276,8 +273,8 @@ static int ipu6_buttress_ipc_send_bulk(struct ipu6_device *isp,
 			goto out;
 		}
 
-		if (ipc->nack_mask &&
-		    (ipc->recv_data & ipc->nack_mask) == ipc->nack) {
+		if ((ipc->recv_data & BUTTRESS_CSE2IUDATA0_IPC_NACK_MASK) ==
+		     BUTTRESS_CSE2IUDATA0_IPC_NACK) {
 			dev_err(&isp->pdev->dev,
 				"IPC NACK for cmd 0x%x\n", msgs[i].cmd);
 			ret = -EIO;
@@ -296,7 +293,7 @@ static int ipu6_buttress_ipc_send_bulk(struct ipu6_device *isp,
 	dev_dbg(&isp->pdev->dev, "bulk IPC commands done\n");
 
 out:
-	ipu6_buttress_ipc_validity_close(isp, ipc);
+	ipu6_buttress_ipc_validity_close(isp);
 	mutex_unlock(&b->ipc_mutex);
 	return ret;
 }
@@ -337,7 +334,8 @@ irqreturn_t ipu6_buttress_isr(int irq, void *isp_ptr)
 	struct ipu6_device *isp = isp_ptr;
 	struct ipu6_bus_device *adev[] = { isp->isys, isp->psys };
 	struct ipu6_buttress *b = &isp->buttress;
-	u32 reg_irq_sts = BUTTRESS_REG_ISR_STATUS;
+	const struct ipu6_buttress_registers *regs = b->regs;
+	const u32 adev_irq_mask[] = { regs->irq_is, regs->irq_ps };
 	irqreturn_t ret = IRQ_NONE;
 	u32 disable_irqs = 0;
 	u32 irq_status;
@@ -348,7 +346,14 @@ irqreturn_t ipu6_buttress_isr(int irq, void *isp_ptr)
 	if (!active)
 		return IRQ_NONE;
 
-	irq_status = readl(isp->base + reg_irq_sts);
+	if (IS_IPU7(isp)) {
+		u32 pb_irq;
+
+		pb_irq = readl(isp->pb_base + IPU7_PB_INTERRUPT_STATUS);
+		writel(pb_irq, isp->pb_base + IPU7_PB_INTERRUPT_STATUS);
+	}
+
+	irq_status = readl(isp->base + regs->irq_status);
 	if (irq_status == 0 || WARN_ON_ONCE(irq_status == 0xffffffffu)) {
 		if (active > 0)
 			pm_runtime_put_noidle(&isp->pdev->dev);
@@ -356,39 +361,40 @@ irqreturn_t ipu6_buttress_isr(int irq, void *isp_ptr)
 	}
 
 	do {
-		writel(irq_status, isp->base + BUTTRESS_REG_ISR_CLEAR);
+		writel(irq_status, isp->base + regs->irq_clear);
 
-		for (i = 0; i < ARRAY_SIZE(ipu6_adev_irq_mask); i++) {
+		for (i = 0; i < ARRAY_SIZE(adev_irq_mask); i++) {
 			irqreturn_t r = ipu6_buttress_call_isr(adev[i]);
 
-			if (!(irq_status & ipu6_adev_irq_mask[i]))
+			if (!(irq_status & adev_irq_mask[i]))
 				continue;
 
 			if (r == IRQ_WAKE_THREAD) {
 				ret = IRQ_WAKE_THREAD;
-				disable_irqs |= ipu6_adev_irq_mask[i];
+				disable_irqs |= adev_irq_mask[i];
 			} else if (ret == IRQ_NONE && r == IRQ_HANDLED) {
 				ret = IRQ_HANDLED;
 			}
 		}
 
-		if ((irq_status & BUTTRESS_EVENT) && ret == IRQ_NONE)
+		if ((irq_status & regs->irq_events) && ret == IRQ_NONE)
 			ret = IRQ_HANDLED;
 
-		if (irq_status & BUTTRESS_ISR_IPC_FROM_CSE_IS_WAITING) {
+		if (irq_status & regs->irq_cse_ipc) {
 			dev_dbg(&isp->pdev->dev,
 				"BUTTRESS_ISR_IPC_FROM_CSE_IS_WAITING\n");
-			ipu6_buttress_ipc_recv(isp, &b->cse, &b->cse.recv_data);
-			complete(&b->cse.recv_complete);
+
+			ipu6_buttress_ipc_recv(isp, &b->ipc.recv_data);
+			complete(&b->ipc.recv_complete);
 		}
 
-		if (irq_status & BUTTRESS_ISR_IPC_EXEC_DONE_BY_CSE) {
+		if (irq_status & regs->irq_exec_done) {
 			dev_dbg(&isp->pdev->dev,
 				"BUTTRESS_ISR_IPC_EXEC_DONE_BY_CSE\n");
-			complete(&b->cse.send_complete);
+			complete(&b->ipc.send_complete);
 		}
 
-		if (irq_status & BUTTRESS_ISR_SAI_VIOLATION &&
+		if (irq_status & regs->irq_sai &&
 		    ipu6_buttress_get_secure_mode(isp))
 			dev_err(&isp->pdev->dev,
 				"BUTTRESS_ISR_SAI_VIOLATION\n");
@@ -407,12 +413,12 @@ irqreturn_t ipu6_buttress_isr(int irq, void *isp_ptr)
 			break;
 		}
 
-		irq_status = readl(isp->base + reg_irq_sts);
+		irq_status = readl(isp->base + regs->irq_status);
 	} while (irq_status);
 
 	if (disable_irqs)
-		writel(BUTTRESS_IRQS & ~disable_irqs,
-		       isp->base + BUTTRESS_REG_ISR_ENABLE);
+		writel(regs->irq_all & ~disable_irqs,
+		       isp->base + regs->irq_enable);
 
 	if (active > 0)
 		pm_runtime_put(&isp->pdev->dev);
@@ -423,12 +429,13 @@ irqreturn_t ipu6_buttress_isr(int irq, void *isp_ptr)
 irqreturn_t ipu6_buttress_isr_threaded(int irq, void *isp_ptr)
 {
 	struct ipu6_device *isp = isp_ptr;
+	const struct ipu6_buttress_registers *regs = isp->buttress.regs;
 	struct ipu6_bus_device *adev[] = { isp->isys, isp->psys };
 	const struct ipu6_auxdrv_data *drv_data = NULL;
 	irqreturn_t ret = IRQ_NONE;
 	unsigned int i;
 
-	for (i = 0; i < ARRAY_SIZE(ipu6_adev_irq_mask) && adev[i]; i++) {
+	for (i = 0; i < ARRAY_SIZE(adev) && adev[i]; i++) {
 		drv_data = adev[i]->auxdrv_data;
 		if (!drv_data)
 			continue;
@@ -438,46 +445,179 @@ irqreturn_t ipu6_buttress_isr_threaded(int irq, void *isp_ptr)
 			ret = IRQ_HANDLED;
 	}
 
-	writel(BUTTRESS_IRQS, isp->base + BUTTRESS_REG_ISR_ENABLE);
+	writel(regs->irq_all, isp->base + regs->irq_enable);
 
 	return ret;
 }
 
-int ipu6_buttress_power(struct device *dev,
-			const struct ipu6_buttress_ctrl *ctrl, bool on)
+static int ipu7_isys_d2d_power(struct ipu6_device *isp, bool on)
+{
+	u32 target = on ? IPU7_BUTTRESS_D2D_PWR_ACK : 0U;
+	u32 val;
+	int ret;
+
+	val = readl(isp->base + IPU7_BUTTRESS_REG_D2D_CTL);
+	if ((val & IPU7_BUTTRESS_D2D_PWR_ACK) == target)
+		return 0;
+
+	if (on)
+		val |= IPU7_BUTTRESS_D2D_PWR_EN;
+	else
+		val &= ~IPU7_BUTTRESS_D2D_PWR_EN;
+	writel(val, isp->base + IPU7_BUTTRESS_REG_D2D_CTL);
+
+	ret = readl_poll_timeout(isp->base + IPU7_BUTTRESS_REG_D2D_CTL, val,
+				 (val & IPU7_BUTTRESS_D2D_PWR_ACK) == target,
+				 100, BUTTRESS_POWER_TIMEOUT_US);
+	if (ret)
+		dev_err(&isp->pdev->dev, "D2D power %s timeout: 0x%x\n",
+			on ? "up" : "down", val);
+
+	return ret;
+}
+
+static void ipu7_nde_control(struct ipu6_device *isp, bool on)
+{
+	u32 val;
+
+	val = FIELD_PREP(IPU7_NDE_VAL_MASK,
+			 on ? IPU7_NDE_VAL_ACTIVE : IPU7_NDE_VAL_DEFAULT) |
+	      FIELD_PREP(IPU7_NDE_SCALE_MASK,
+			 on ? IPU7_NDE_SCALE_ACTIVE : IPU7_NDE_SCALE_DEFAULT) |
+	      FIELD_PREP(IPU7_NDE_VALID_MASK,
+			 on ? IPU7_NDE_VALID_ACTIVE : IPU7_NDE_VALID_DEFAULT) |
+	      FIELD_PREP(IPU7_NDE_RESVEC_MASK, IPU7_NDE_RESVEC);
+	writel(val, isp->base + IPU7_BUTTRESS_REG_NDE_CONTROL);
+}
+
+static int __ipu7_power_on(struct device *dev,
+			   const struct ipu6_buttress_ctrl *ctrl)
+{
+	struct ipu6_device *isp = to_ipu6_bus_device(dev)->isp;
+	bool is_isys = ctrl->subsys_id == IPU_ISYS;
+	u32 pwr_sts, val, ovrd_clk, slp, own_clk_ack;
+	int ret;
+
+	val = ctrl->ratio | (IPU7_FREQ_CTL_CDYN << IPU7_FREQ_CTL_CDYN_SHIFT);
+	pwr_sts = ctrl->pwr_sts_on << ctrl->pwr_sts_shift;
+
+	if (is_isys) {
+		ret = ipu7_isys_d2d_power(isp, true);
+		if (ret)
+			return ret;
+
+		ipu7_nde_control(isp, true);
+	}
+
+	/* Request clock ownership. */
+	ovrd_clk = is_isys ? IPU7_BUTTRESS_OVERRIDE_IS_CLK :
+			     IPU7_BUTTRESS_OVERRIDE_PS_CLK;
+
+	slp = readl(isp->base + IPU7_BUTTRESS_REG_SLEEP_LEVEL_CFG);
+	writel(slp | ovrd_clk, isp->base + IPU7_BUTTRESS_REG_SLEEP_LEVEL_CFG);
+
+	own_clk_ack = is_isys ? IPU7_BUTTRESS_OWN_ACK_IS_CLK :
+				IPU7_BUTTRESS_OWN_ACK_PS_CLK;
+	ret = readl_poll_timeout(isp->base + IPU7_BUTTRESS_REG_SLEEP_LEVEL_STS,
+				 slp, (slp & own_clk_ack),
+				 100, BUTTRESS_POWER_TIMEOUT_US);
+	if (ret)
+		dev_warn(&isp->pdev->dev, "clock ownership timeout: 0x%x\n",
+			 slp);
+
+	writel(val, isp->base + ctrl->freq_ctl);
+
+	ret = readl_poll_timeout(isp->base + isp->buttress.regs->pwr_status,
+				 val, (val & ctrl->pwr_sts_mask) == pwr_sts,
+				 100, BUTTRESS_POWER_TIMEOUT_US);
+	if (ret) {
+		dev_err(&isp->pdev->dev,
+			"Change power status timeout with 0x%x\n", val);
+		return ret;
+	}
+
+	slp = readl(isp->base + IPU7_BUTTRESS_REG_SLEEP_LEVEL_CFG);
+	writel(slp & ~ovrd_clk, isp->base + IPU7_BUTTRESS_REG_SLEEP_LEVEL_CFG);
+
+	return 0;
+}
+
+static int __ipu7_power_off(struct device *dev,
+			    const struct ipu6_buttress_ctrl *ctrl)
 {
 	struct ipu6_device *isp = to_ipu6_bus_device(dev)->isp;
 	u32 pwr_sts, val;
 	int ret;
 
-	if (!ctrl)
-		return 0;
+	writel(0x8U, isp->base + ctrl->freq_ctl);
 
-	mutex_lock(&isp->buttress.power_mutex);
+	pwr_sts = ctrl->pwr_sts_off << ctrl->pwr_sts_shift;
+	ret = readl_poll_timeout(isp->base + isp->buttress.regs->pwr_status,
+				 val, (val & ctrl->pwr_sts_mask) == pwr_sts,
+				 100, BUTTRESS_POWER_TIMEOUT_US);
+	if (ret) {
+		dev_err(&isp->pdev->dev,
+			"Change power status timeout with 0x%x\n", val);
+		return ret;
+	}
+
+	if (ctrl->subsys_id == IPU_ISYS) {
+		ipu7_isys_d2d_power(isp, false);
+		ipu7_nde_control(isp, false);
+	}
+
+	return 0;
+}
+
+static int __ipu6_power(struct device *dev,
+			const struct ipu6_buttress_ctrl *ctrl, bool on)
+{
+	struct ipu6_device *isp = to_ipu6_bus_device(dev)->isp;
+	u32 pwr_sts, val;
+	int ret;
 
 	if (!on) {
 		val = 0;
 		pwr_sts = ctrl->pwr_sts_off << ctrl->pwr_sts_shift;
 	} else {
 		val = BUTTRESS_FREQ_CTL_START |
-			FIELD_PREP(BUTTRESS_FREQ_CTL_RATIO_MASK,
-				   ctrl->ratio) |
-			FIELD_PREP(BUTTRESS_FREQ_CTL_QOS_FLOOR_MASK,
-				   ctrl->qos_floor) |
-			BUTTRESS_FREQ_CTL_ICCMAX_LEVEL;
+		      FIELD_PREP(BUTTRESS_FREQ_CTL_RATIO_MASK, ctrl->ratio) |
+		      FIELD_PREP(BUTTRESS_FREQ_CTL_QOS_FLOOR_MASK,
+				 ctrl->qos_floor) |
+		      BUTTRESS_FREQ_CTL_ICCMAX_LEVEL;
 
 		pwr_sts = ctrl->pwr_sts_on << ctrl->pwr_sts_shift;
 	}
 
 	writel(val, isp->base + ctrl->freq_ctl);
 
-	ret = readl_poll_timeout(isp->base + BUTTRESS_REG_PWR_STATE,
+	ret = readl_poll_timeout(isp->base + isp->buttress.regs->pwr_status,
 				 val, (val & ctrl->pwr_sts_mask) == pwr_sts,
 				 100, BUTTRESS_POWER_TIMEOUT_US);
 	if (ret)
 		dev_err(&isp->pdev->dev,
 			"Change power status timeout with 0x%x\n", val);
 
+	return ret;
+}
+
+int ipu6_buttress_power(struct device *dev,
+			const struct ipu6_buttress_ctrl *ctrl, bool on)
+{
+	struct ipu6_device *isp = to_ipu6_bus_device(dev)->isp;
+	int ret;
+
+	if (!ctrl)
+		return 0;
+
+	mutex_lock(&isp->buttress.power_mutex);
+
+	if (IS_IPU7(isp))
+		ret = on ? __ipu7_power_on(dev, ctrl) :
+			   __ipu7_power_off(dev, ctrl);
+	else
+		ret = __ipu6_power(dev, ctrl, on);
+
 	mutex_unlock(&isp->buttress.power_mutex);
 
 	return ret;
@@ -487,7 +627,7 @@ bool ipu6_buttress_get_secure_mode(struct ipu6_device *isp)
 {
 	u32 val;
 
-	val = readl(isp->base + BUTTRESS_REG_SECURITY_CTL);
+	val = readl(isp->base + isp->buttress.regs->security_ctl);
 
 	return val & BUTTRESS_SECURITY_CTL_FW_SECURE_MODE;
 }
@@ -499,7 +639,7 @@ bool ipu6_buttress_auth_done(struct ipu6_device *isp)
 	if (!isp->secure_mode)
 		return true;
 
-	val = readl(isp->base + BUTTRESS_REG_SECURITY_CTL);
+	val = readl(isp->base + isp->buttress.regs->security_ctl);
 	val = FIELD_GET(BUTTRESS_SECURITY_CTL_FW_SETUP_MASK, val);
 
 	return val == BUTTRESS_SECURITY_CTL_AUTH_DONE;
@@ -517,10 +657,10 @@ int ipu6_buttress_reset_authentication(struct ipu6_device *isp)
 	}
 
 	writel(BUTTRESS_FW_RESET_CTL_START, isp->base +
-	       BUTTRESS_REG_FW_RESET_CTL);
+	       isp->buttress.regs->fw_reset_ctl);
 
-	ret = readl_poll_timeout(isp->base + BUTTRESS_REG_FW_RESET_CTL, val,
-				 val & BUTTRESS_FW_RESET_CTL_DONE, 500,
+	ret = readl_poll_timeout(isp->base + isp->buttress.regs->fw_reset_ctl,
+				 val, val & BUTTRESS_FW_RESET_CTL_DONE, 500,
 				 BUTTRESS_CSE_FWRESET_TIMEOUT_US);
 	if (ret) {
 		dev_err(&isp->pdev->dev,
@@ -529,62 +669,63 @@ int ipu6_buttress_reset_authentication(struct ipu6_device *isp)
 	}
 
 	dev_dbg(&isp->pdev->dev, "FW reset for authentication done\n");
-	writel(0, isp->base + BUTTRESS_REG_FW_RESET_CTL);
+	writel(0, isp->base + isp->buttress.regs->fw_reset_ctl);
+
 	/* leave some time for HW restore */
 	usleep_range(800, 1000);
 
 	return 0;
 }
 
-int ipu6_buttress_map_fw_image(struct ipu6_bus_device *sys,
-			       const struct firmware *fw, struct sg_table *sgt)
+int ipu6_map_fw_region(struct ipu6_bus_device *sys, const void *data,
+		       size_t size, enum dma_data_direction dir,
+		       unsigned long attrs)
 {
-	bool is_vmalloc = is_vmalloc_addr(fw->data);
+	bool is_vmalloc = is_vmalloc_addr(data);
 	struct pci_dev *pdev = sys->isp->pdev;
+	struct sg_table *sgt = &sys->fw_sgt;
 	struct page **pages;
-	const void *addr;
 	unsigned long n_pages;
 	unsigned int i;
 	int ret;
 
-	if (!is_vmalloc && !virt_addr_valid(fw->data))
+	if (!is_vmalloc && !virt_addr_valid(data))
 		return -EDOM;
 
-	n_pages = PFN_UP(fw->size);
+	n_pages = PFN_UP(size);
 
 	pages = kmalloc_objs(*pages, n_pages);
 	if (!pages)
 		return -ENOMEM;
 
-	addr = fw->data;
 	for (i = 0; i < n_pages; i++) {
 		struct page *p = is_vmalloc ?
-			vmalloc_to_page(addr) : virt_to_page(addr);
+			vmalloc_to_page(data) : virt_to_page(data);
 
 		if (!p) {
 			ret = -ENOMEM;
 			goto out;
 		}
 		pages[i] = p;
-		addr += PAGE_SIZE;
+		data += PAGE_SIZE;
 	}
 
-	ret = sg_alloc_table_from_pages(sgt, pages, n_pages, 0, fw->size,
+	ret = sg_alloc_table_from_pages(sgt, pages, n_pages, 0, size,
 					GFP_KERNEL);
 	if (ret) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
-	ret = dma_map_sgtable(&pdev->dev, sgt, DMA_TO_DEVICE, 0);
+	ret = dma_map_sgtable(&pdev->dev, sgt, dir, 0);
 	if (ret) {
 		sg_free_table(sgt);
 		goto out;
 	}
 
-	ret = ipu6_dma_map_sgtable(sys, sgt, DMA_TO_DEVICE, 0);
+	ret = ipu6_dma_map_sgtable(sys, sgt, dir, attrs);
 	if (ret) {
-		dma_unmap_sgtable(&pdev->dev, sgt, DMA_TO_DEVICE, 0);
+		dma_unmap_sgtable(&pdev->dev, sgt, dir, 0);
 		sg_free_table(sgt);
 		goto out;
 	}
@@ -596,18 +737,18 @@ int ipu6_buttress_map_fw_image(struct ipu6_bus_device *sys,
 
 	return ret;
 }
-EXPORT_SYMBOL_NS_GPL(ipu6_buttress_map_fw_image, "INTEL_IPU6");
+EXPORT_SYMBOL_NS_GPL(ipu6_map_fw_region, "INTEL_IPU6");
 
-void ipu6_buttress_unmap_fw_image(struct ipu6_bus_device *sys,
-				  struct sg_table *sgt)
+void ipu6_unmap_fw_region(struct ipu6_bus_device *sys,
+			  enum dma_data_direction dir)
 {
 	struct pci_dev *pdev = sys->isp->pdev;
 
-	ipu6_dma_unmap_sgtable(sys, sgt, DMA_TO_DEVICE, 0);
-	dma_unmap_sgtable(&pdev->dev, sgt, DMA_TO_DEVICE, 0);
-	sg_free_table(sgt);
+	ipu6_dma_unmap_sgtable(sys, &sys->fw_sgt, dir, 0);
+	dma_unmap_sgtable(&pdev->dev, &sys->fw_sgt, dir, 0);
+	sg_free_table(&sys->fw_sgt);
 }
-EXPORT_SYMBOL_NS_GPL(ipu6_buttress_unmap_fw_image, "INTEL_IPU6");
+EXPORT_SYMBOL_NS_GPL(ipu6_unmap_fw_region, "INTEL_IPU6");
 
 int ipu6_buttress_authenticate(struct ipu6_device *isp)
 {
@@ -634,11 +775,18 @@ int ipu6_buttress_authenticate(struct ipu6_device *isp)
 	 * Write address of FIT table to FW_SOURCE register
 	 * Let's use fw address. I.e. not using FIT table yet
 	 */
-	data = lower_32_bits(isp->psys->pkg_dir_dma_addr);
-	writel(data, isp->base + BUTTRESS_REG_FW_SOURCE_BASE_LO);
+	if (IS_IPU7(isp)) {
+		writel(isp->cpd_fw->size,
+		       isp->base + IPU7_BUTTRESS_REG_FW_SOURCE_SIZE);
+		writel(sg_dma_address(isp->psys->fw_sgt.sgl),
+		       isp->base + IPU7_BUTTRESS_REG_FW_SOURCE_BASE);
+	} else {
+		data = lower_32_bits(isp->psys->pkg_dir_dma_addr);
+		writel(data, isp->base + BUTTRESS_REG_FW_SOURCE_BASE_LO);
 
-	data = upper_32_bits(isp->psys->pkg_dir_dma_addr);
-	writel(data, isp->base + BUTTRESS_REG_FW_SOURCE_BASE_HI);
+		data = upper_32_bits(isp->psys->pkg_dir_dma_addr);
+		writel(data, isp->base + BUTTRESS_REG_FW_SOURCE_BASE_HI);
+	}
 
 	/*
 	 * Write boot_load into IU2CSEDATA0
@@ -659,7 +807,7 @@ int ipu6_buttress_authenticate(struct ipu6_device *isp)
 	mask = BUTTRESS_SECURITY_CTL_FW_SETUP_MASK;
 	done = BUTTRESS_SECURITY_CTL_FW_SETUP_DONE;
 	fail = BUTTRESS_SECURITY_CTL_AUTH_FAILED;
-	ret = readl_poll_timeout(isp->base + BUTTRESS_REG_SECURITY_CTL, data,
+	ret = readl_poll_timeout(isp->base + b->regs->security_ctl, data,
 				 ((data & mask) == done ||
 				  (data & mask) == fail), 500,
 				 BUTTRESS_CSE_BOOTLOAD_TIMEOUT_US);
@@ -674,8 +822,11 @@ int ipu6_buttress_authenticate(struct ipu6_device *isp)
 		goto out_unlock;
 	}
 
-	ret = readl_poll_timeout(psys_pdata->base + BOOTLOADER_STATUS_OFFSET,
-				 data, data == BOOTLOADER_MAGIC_KEY, 500,
+	void __iomem *base = IS_IPU7(isp) ?
+			     isp->base + IPU7_BUTTRESS_REG_FW_BOOT_PARAMS7 :
+			     psys_pdata->base + BOOTLOADER_STATUS_OFFSET;
+
+	ret = readl_poll_timeout(base, data, data == BOOTLOADER_MAGIC_KEY, 500,
 				 BUTTRESS_CSE_BOOTLOAD_TIMEOUT_US);
 	if (ret) {
 		dev_err(&isp->pdev->dev, "Unexpected magic number 0x%x\n",
@@ -699,7 +850,7 @@ int ipu6_buttress_authenticate(struct ipu6_device *isp)
 	}
 
 	done = BUTTRESS_SECURITY_CTL_AUTH_DONE;
-	ret = readl_poll_timeout(isp->base + BUTTRESS_REG_SECURITY_CTL, data,
+	ret = readl_poll_timeout(isp->base + b->regs->security_ctl, data,
 				 ((data & mask) == done ||
 				  (data & mask) == fail), 500,
 				 BUTTRESS_CSE_AUTHENTICATE_TIMEOUT_US);
@@ -724,15 +875,16 @@ int ipu6_buttress_authenticate(struct ipu6_device *isp)
 
 static int ipu6_buttress_send_tsc_request(struct ipu6_device *isp)
 {
+	const struct ipu6_buttress_registers *regs = isp->buttress.regs;
 	u32 val, mask, done;
 	int ret;
 
 	mask = BUTTRESS_PWR_STATE_HH_STATUS_MASK;
 
 	writel(BUTTRESS_FABRIC_CMD_START_TSC_SYNC,
-	       isp->base + BUTTRESS_REG_FABRIC_CMD);
+	       isp->base + regs->fabric_cmd);
 
-	val = readl(isp->base + BUTTRESS_REG_PWR_STATE);
+	val = readl(isp->base + regs->pwr_status);
 	val = FIELD_GET(mask, val);
 	if (val == BUTTRESS_PWR_STATE_HH_STATE_ERR) {
 		dev_err(&isp->pdev->dev, "Start tsc sync failed\n");
@@ -740,8 +892,8 @@ static int ipu6_buttress_send_tsc_request(struct ipu6_device *isp)
 	}
 
 	done = BUTTRESS_PWR_STATE_HH_STATE_DONE;
-	ret = readl_poll_timeout(isp->base + BUTTRESS_REG_PWR_STATE, val,
-				 FIELD_GET(mask, val) == done, 500,
+	ret = readl_poll_timeout(isp->base + regs->pwr_status,
+				 val, FIELD_GET(mask, val) == done, 500,
 				 BUTTRESS_TSC_SYNC_TIMEOUT_US);
 	if (ret)
 		dev_err(&isp->pdev->dev, "Start tsc sync timeout\n");
@@ -749,11 +901,30 @@ static int ipu6_buttress_send_tsc_request(struct ipu6_device *isp)
 	return ret;
 }
 
-int ipu6_buttress_start_tsc_sync(struct ipu6_device *isp)
+static int __ipu7p5_start_tsc_sync(struct ipu6_device *isp)
 {
-	unsigned int i;
+	u32 val;
+
+	val = readl(isp->base + IPU7_BUTTRESS_REG_TSC_CTL);
+	val |= IPU7_BUTTRESS_SEL_PB_TIMESTAMP;
+	writel(val, isp->base + IPU7_BUTTRESS_REG_TSC_CTL);
+
+	for (unsigned int i = 0; i < BUTTRESS_TSC_SYNC_RESET_TRIAL_MAX; i++) {
+		val = readl(isp->base + IPU7_BUTTRESS_REG_PB_TIMESTAMP_VALID);
+		if (val == 1)
+			return 0;
+
+		usleep_range(40, 50);
+	}
+
+	dev_err(&isp->pdev->dev, "TSC sync failed (timeout)\n");
 
-	for (i = 0; i < BUTTRESS_TSC_SYNC_RESET_TRIAL_MAX; i++) {
+	return -ETIMEDOUT;
+}
+
+static int __ipu6_start_tsc_sync(struct ipu6_device *isp)
+{
+	for (unsigned int i = 0; i < BUTTRESS_TSC_SYNC_RESET_TRIAL_MAX; i++) {
 		u32 val;
 		int ret;
 
@@ -761,28 +932,39 @@ int ipu6_buttress_start_tsc_sync(struct ipu6_device *isp)
 		if (ret != -ETIMEDOUT)
 			return ret;
 
-		val = readl(isp->base + BUTTRESS_REG_TSW_CTL);
+		u32 tsw_ctl = isp->buttress.regs->tsw_ctl;
+
+		val = readl(isp->base + tsw_ctl);
 		val = val | BUTTRESS_TSW_CTL_SOFT_RESET;
-		writel(val, isp->base + BUTTRESS_REG_TSW_CTL);
+		writel(val, isp->base + tsw_ctl);
 		val = val & ~BUTTRESS_TSW_CTL_SOFT_RESET;
-		writel(val, isp->base + BUTTRESS_REG_TSW_CTL);
+		writel(val, isp->base + tsw_ctl);
 	}
 
 	dev_err(&isp->pdev->dev, "TSC sync failed (timeout)\n");
 
 	return -ETIMEDOUT;
 }
+
+int ipu6_buttress_start_tsc_sync(struct ipu6_device *isp)
+{
+	if (IS_IPU7P5(isp))
+		return __ipu7p5_start_tsc_sync(isp);
+
+	return __ipu6_start_tsc_sync(isp);
+}
 EXPORT_SYMBOL_NS_GPL(ipu6_buttress_start_tsc_sync, "INTEL_IPU6");
 
 void ipu6_buttress_tsc_read(struct ipu6_device *isp, u64 *val)
 {
+	void __iomem *tsc = isp->base + isp->buttress.regs->tsc_lo;
 	u32 tsc_hi_1, tsc_hi_2, tsc_lo;
 	unsigned long flags;
 
 	local_irq_save(flags);
-	tsc_hi_1 = readl(isp->base + BUTTRESS_REG_TSC_HI);
-	tsc_lo = readl(isp->base + BUTTRESS_REG_TSC_LO);
-	tsc_hi_2 = readl(isp->base + BUTTRESS_REG_TSC_HI);
+	tsc_hi_1 = readl(tsc + BUTTRESS_TSC_HI_OFFSET);
+	tsc_lo = readl(tsc);
+	tsc_hi_2 = readl(tsc + BUTTRESS_TSC_HI_OFFSET);
 	if (tsc_hi_1 == tsc_hi_2) {
 		*val = (u64)tsc_hi_1 << 32 | tsc_lo;
 	} else {
@@ -811,13 +993,77 @@ u64 ipu6_buttress_tsc_ticks_to_ns(u64 ticks, const struct ipu6_device *isp)
 }
 EXPORT_SYMBOL_NS_GPL(ipu6_buttress_tsc_ticks_to_ns, "INTEL_IPU6");
 
+/* trigger uc control to wakeup fw */
+void ipu7_buttress_wakeup_isys(const struct ipu6_device *isp)
+{
+	u32 val;
+
+	val = readl(isp->base + IPU7_BUTTRESS_REG_ISYS_UCX_CTRL_STATUS);
+	val |= IPU7_UCX_CTL_WAKEUP;
+	writel(val, isp->base + IPU7_BUTTRESS_REG_ISYS_UCX_CTRL_STATUS);
+}
+EXPORT_SYMBOL_NS_GPL(ipu7_buttress_wakeup_isys, "INTEL_IPU6");
+
+u32 ipu7_buttress_get_isys_freq(struct ipu6_device *isp)
+{
+	u32 val;
+
+	val = readl(isp->base + IPU7_BUTTRESS_REG_IS_WORKPOINT_REQ);
+	val &= IPU7_BUTTRESS_IS_FREQ_CTL_RATIO_MASK;
+
+	return val * 50 / 3;
+}
+EXPORT_SYMBOL_NS_GPL(ipu7_buttress_get_isys_freq, "INTEL_IPU6");
+
+static const struct x86_cpu_id ipu7_misc_cfg_exclusion[] = {
+	X86_MATCH_VFM_STEPS(INTEL_PANTHERLAKE_L, 0x1, 0x1, 0),
+	{},
+};
+
+#define IPU7_WRXREQOP_OVRD_VAL_MASK  GENMASK(22, 19)
+
+static void ipu7_buttress_setup(struct ipu6_device *isp)
+{
+	struct ipu6_buttress *b = &isp->buttress;
+	u32 val;
+
+	/* program PB BAR */
+	writel(0, isp->pb_base + IPU7_GLOBAL_INTERRUPT_MASK);
+	val = readl(isp->pb_base + IPU7_BAR2_MISC_CONFIG);
+
+	val |= 0x100U;
+	if (!IS_IPU7_MTL(isp) && !x86_match_cpu(ipu7_misc_cfg_exclusion))
+		val |= FIELD_PREP(IPU7_WRXREQOP_OVRD_VAL_MASK, 0xf) | BIT(18);
+
+	writel(val, isp->pb_base + IPU7_BAR2_MISC_CONFIG);
+
+	if (IS_IPU7P5(isp)) {
+		writel(BIT(14), isp->pb_base + IPU7_TLBID_HASH_ENABLE_63_32);
+		writel(BIT(9), isp->pb_base + IPU7_TLBID_HASH_ENABLE_95_64);
+	} else {
+		writel(BIT(22), isp->pb_base + IPU7_TLBID_HASH_ENABLE_63_32);
+		writel(BIT(1), isp->pb_base + IPU7_TLBID_HASH_ENABLE_127_96);
+	}
+
+	writel(b->regs->irq_all, isp->base + b->regs->irq_clear);
+	writel(b->regs->irq_all, isp->base + IPU7_BUTTRESS_REG_IRQ_MASK);
+	writel(b->regs->irq_all, isp->base + b->regs->irq_enable);
+
+	/* LNL SW workaround for PS PD hang when PS sub-domain during PD */
+	writel(IPU7_BUTTRESS_CG_CTRL_PS_FSM_CG, isp->base + IPU7_BUTTRESS_REG_CG_CTRL_BITS);
+}
+
 void ipu6_buttress_restore(struct ipu6_device *isp)
 {
 	struct ipu6_buttress *b = &isp->buttress;
 
-	writel(BUTTRESS_IRQS, isp->base + BUTTRESS_REG_ISR_CLEAR);
-	writel(BUTTRESS_IRQS, isp->base + BUTTRESS_REG_ISR_ENABLE);
-	writel(b->wdt_cached_value, isp->base + BUTTRESS_REG_WDT);
+	if (IS_IPU7(isp)) {
+		ipu7_buttress_setup(isp);
+	} else {
+		writel(b->regs->irq_all, isp->base + b->regs->irq_clear);
+		writel(b->regs->irq_all, isp->base + b->regs->irq_enable);
+	}
+	writel(b->wdt_cached_value, isp->base + b->regs->wdt);
 }
 
 int ipu6_buttress_init(struct ipu6_device *isp)
@@ -830,54 +1076,52 @@ int ipu6_buttress_init(struct ipu6_device *isp)
 	mutex_init(&b->auth_mutex);
 	mutex_init(&b->cons_mutex);
 	mutex_init(&b->ipc_mutex);
-	init_completion(&b->cse.send_complete);
-	init_completion(&b->cse.recv_complete);
-
-	b->cse.nack = BUTTRESS_CSE2IUDATA0_IPC_NACK;
-	b->cse.nack_mask = BUTTRESS_CSE2IUDATA0_IPC_NACK_MASK;
-	b->cse.csr_in = BUTTRESS_REG_CSE2IUCSR;
-	b->cse.csr_out = BUTTRESS_REG_IU2CSECSR;
-	b->cse.db0_in = BUTTRESS_REG_CSE2IUDB0;
-	b->cse.db0_out = BUTTRESS_REG_IU2CSEDB0;
-	b->cse.data0_in = BUTTRESS_REG_CSE2IUDATA0;
-	b->cse.data0_out = BUTTRESS_REG_IU2CSEDATA0;
+	init_completion(&b->ipc.send_complete);
+	init_completion(&b->ipc.recv_complete);
 
 	INIT_LIST_HEAD(&b->constraints);
-
 	isp->secure_mode = ipu6_buttress_get_secure_mode(isp);
-	dev_dbg(&isp->pdev->dev, "IPU6 in %s mode touch 0x%x mask 0x%x\n",
-		isp->secure_mode ? "secure" : "non-secure",
-		readl(isp->base + BUTTRESS_REG_SECURITY_TOUCH),
-		readl(isp->base + BUTTRESS_REG_CAMERA_MASK));
-
-	b->wdt_cached_value = readl(isp->base + BUTTRESS_REG_WDT);
-	writel(BUTTRESS_IRQS, isp->base + BUTTRESS_REG_ISR_CLEAR);
-	writel(BUTTRESS_IRQS, isp->base + BUTTRESS_REG_ISR_ENABLE);
-
-	/* get ref_clk frequency by reading the indication in btrs control */
-	val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL);
-	val = FIELD_GET(BUTTRESS_REG_BTRS_CTRL_REF_CLK_IND, val);
-
-	switch (val) {
-	case 0x0:
-		b->ref_clk = 240;
-		break;
-	case 0x1:
-		b->ref_clk = 192;
-		break;
-	case 0x2:
+
+	dev_dbg(&isp->pdev->dev, "IPU in %s mode\n",
+		isp->secure_mode ? "secure" : "non-secure");
+
+	if (IS_IPU7(isp)) {
+		ipu7_buttress_setup(isp);
 		b->ref_clk = 384;
-		break;
-	default:
-		dev_warn(&isp->pdev->dev,
-			 "Unsupported ref clock, use 19.2Mhz by default.\n");
-		b->ref_clk = 192;
-		break;
+	} else {
+		dev_dbg(&isp->pdev->dev, "IPU6 touch 0x%x mask 0x%x\n",
+			readl(isp->base + BUTTRESS_REG_SECURITY_TOUCH),
+			readl(isp->base + BUTTRESS_REG_CAMERA_MASK));
+
+		writel(b->regs->irq_all, isp->base + b->regs->irq_clear);
+		writel(b->regs->irq_all, isp->base + b->regs->irq_enable);
+
+		/* get ref_clk frequency by reading the indication in btrs control */
+		val = readl(isp->base + b->regs->btrs_ctrl);
+		val = FIELD_GET(BUTTRESS_REG_BTRS_CTRL_REF_CLK_IND, val);
+
+		switch (val) {
+		case 0x0:
+			b->ref_clk = 240;
+			break;
+		case 0x1:
+			b->ref_clk = 192;
+			break;
+		case 0x2:
+			b->ref_clk = 384;
+			break;
+		default:
+			dev_warn(&isp->pdev->dev,
+				 "Unsupported ref clock, use 19.2Mhz by default.\n");
+			b->ref_clk = 192;
+			break;
+		}
 	}
+	b->wdt_cached_value = readl(isp->base + b->regs->wdt);
 
 	/* Retry couple of times in case of CSE initialization is delayed */
 	do {
-		ret = ipu6_buttress_ipc_reset(isp, &b->cse);
+		ret = ipu6_buttress_ipc_reset(isp);
 		if (ret) {
 			dev_warn(&isp->pdev->dev,
 				 "IPC reset protocol failed, retrying\n");
@@ -901,7 +1145,7 @@ void ipu6_buttress_exit(struct ipu6_device *isp)
 {
 	struct ipu6_buttress *b = &isp->buttress;
 
-	writel(0, isp->base + BUTTRESS_REG_ISR_ENABLE);
+	writel(0, isp->base + b->regs->irq_enable);
 
 	mutex_destroy(&b->power_mutex);
 	mutex_destroy(&b->auth_mutex);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-buttress.h b/drivers/media/pci/intel/ipu6/ipu6-buttress.h
index 51e5ad48db8294..79d8f58936860a 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-buttress.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-buttress.h
@@ -26,25 +26,50 @@ struct ipu6_buttress_ctrl {
 	u32 freq_ctl, pwr_sts_shift, pwr_sts_mask, pwr_sts_on, pwr_sts_off;
 	unsigned int ratio;
 	unsigned int qos_floor;
+	int subsys_id;
 };
 
 struct ipu6_buttress_ipc {
 	struct completion send_complete;
 	struct completion recv_complete;
-	u32 nack;
-	u32 nack_mask;
 	u32 recv_data;
-	u32 csr_out;
+};
+
+struct ipu6_buttress_registers {
+	/* Registers */
+	u32 irq_status;
+	u32 irq_clear;
+	u32 irq_enable;
+	u32 pwr_status;
+	u32 security_ctl;
+	u32 fw_reset_ctl;
+	u32 fabric_cmd;
+	u32 tsw_ctl;
+	u32 tsc_lo;
+	u32 wdt;
+	u32 btrs_ctrl;
 	u32 csr_in;
+	u32 csr_out;
 	u32 db0_in;
 	u32 db0_out;
-	u32 data0_out;
 	u32 data0_in;
+	u32 data0_out;
+	u32 sku_id;
+
+	/* Bitmasks */
+	u32 irq_is;
+	u32 irq_ps;
+	u32 irq_all;
+	u32 irq_events;
+	u32 irq_cse_ipc;
+	u32 irq_exec_done;
+	u32 irq_sai;
 };
 
 struct ipu6_buttress {
 	struct mutex power_mutex, auth_mutex, cons_mutex, ipc_mutex;
-	struct ipu6_buttress_ipc cse;
+	struct ipu6_buttress_ipc ipc;
+	const struct ipu6_buttress_registers *regs;
 	struct list_head constraints;
 	u32 wdt_cached_value;
 	bool force_suspend;
@@ -58,13 +83,12 @@ struct ipu6_ipc_buttress_bulk_msg {
 	u8 cmd_size;
 };
 
-int ipu6_buttress_ipc_reset(struct ipu6_device *isp,
-			    struct ipu6_buttress_ipc *ipc);
-int ipu6_buttress_map_fw_image(struct ipu6_bus_device *sys,
-			       const struct firmware *fw,
-			       struct sg_table *sgt);
-void ipu6_buttress_unmap_fw_image(struct ipu6_bus_device *sys,
-				  struct sg_table *sgt);
+int ipu6_buttress_ipc_reset(struct ipu6_device *isp);
+int ipu6_map_fw_region(struct ipu6_bus_device *sys, const void *data,
+		       size_t size, enum dma_data_direction dir,
+		       unsigned long attrs);
+void ipu6_unmap_fw_region(struct ipu6_bus_device *sys,
+			  enum dma_data_direction dir);
 int ipu6_buttress_power(struct device *dev,
 			const struct ipu6_buttress_ctrl *ctrl, bool on);
 bool ipu6_buttress_get_secure_mode(struct ipu6_device *isp);
@@ -82,4 +106,6 @@ void ipu6_buttress_exit(struct ipu6_device *isp);
 void ipu6_buttress_csi_port_config(struct ipu6_device *isp,
 				   u32 legacy, u32 combo);
 void ipu6_buttress_restore(struct ipu6_device *isp);
+void ipu7_buttress_wakeup_isys(const struct ipu6_device *isp);
+u32 ipu7_buttress_get_isys_freq(struct ipu6_device *isp);
 #endif /* IPU6_BUTTRESS_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu6-cpd.c b/drivers/media/pci/intel/ipu6/ipu6-cpd.c
index b7013f6524ec6e..1fadb0f2f6c1ac 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-cpd.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-cpd.c
@@ -16,6 +16,7 @@
 #include "ipu6-bus.h"
 #include "ipu6-cpd.h"
 #include "ipu6-dma.h"
+#include "ipu7-mmu-hw.h"
 
 /* 15 entries + header*/
 #define MAX_PKG_DIR_ENT_CNT		16
@@ -62,6 +63,20 @@ static inline const struct ipu6_cpd_ent *ipu6_cpd_get_entry(const void *cpd,
 #define ipu6_cpd_get_metadata(cpd) ipu6_cpd_get_entry(cpd, METADATA_IDX)
 #define ipu6_cpd_get_moduledata(cpd) ipu6_cpd_get_entry(cpd, MODULEDATA_IDX)
 
+#define IPU7_CPD_BINARY_START_IDX	1U
+#define IPU7_CPD_METADATA_START_IDX	2U
+#define IPU7_CPD_BINARY_NUM		2U /* ISYS + PSYS */
+#define IPU7_CPD_METADATA_ATTR		0xa
+#define IPU7_CPD_METADATA_IPL		0x1c
+/*
+ * Entries include:
+ * 1 manifest entry.
+ * 1 metadata entry for each sub system(ISYS and PSYS).
+ * 1 binary entry for each sub system(ISYS and PSYS).
+ */
+#define IPU7_CPD_ENTRY_NUM	(IPU7_CPD_BINARY_NUM * 2U + 1U)
+#define IPU7_MAX_MANIFEST_SIZE	(SZ_4K * sizeof(u32))
+
 static const struct ipu6_cpd_metadata_cmpnt_hdr *
 ipu6_cpd_metadata_get_cmpnt(struct ipu6_device *isp, const void *metadata,
 			    unsigned int metadata_size, u8 idx)
@@ -220,8 +235,9 @@ EXPORT_SYMBOL_NS_GPL(ipu6_cpd_create_pkg_dir, "INTEL_IPU6");
 
 void ipu6_cpd_free_pkg_dir(struct ipu6_bus_device *adev)
 {
-	ipu6_dma_free(adev, adev->pkg_dir_size, adev->pkg_dir,
-		      adev->pkg_dir_dma_addr, 0);
+	if (adev->pkg_dir)
+		ipu6_dma_free(adev, adev->pkg_dir_size, adev->pkg_dir,
+			      adev->pkg_dir_dma_addr, 0);
 }
 EXPORT_SYMBOL_NS_GPL(ipu6_cpd_free_pkg_dir, "INTEL_IPU6");
 
@@ -266,7 +282,6 @@ static int ipu6_cpd_validate_moduledata(struct ipu6_device *isp,
 					u32 moduledata_size)
 {
 	const struct ipu6_cpd_module_data_hdr *mod_hdr = moduledata;
-	int ret;
 
 	/* Ensure moduledata hdr is within moduledata */
 	if (moduledata_size < sizeof(*mod_hdr) ||
@@ -276,15 +291,9 @@ static int ipu6_cpd_validate_moduledata(struct ipu6_device *isp,
 	}
 
 	dev_dbg(&isp->pdev->dev, "FW version: %x\n", mod_hdr->fw_pkg_date);
-	ret = ipu6_cpd_validate_cpd(isp, moduledata + mod_hdr->hdr_len,
-				    moduledata_size - mod_hdr->hdr_len,
-				    moduledata_size);
-	if (ret) {
-		dev_err(&isp->pdev->dev, "Invalid CPD in moduledata\n");
-		return ret;
-	}
-
-	return 0;
+	return ipu6_cpd_validate_cpd(isp, moduledata + mod_hdr->hdr_len,
+				     moduledata_size - mod_hdr->hdr_len,
+				     moduledata_size);
 }
 
 static int ipu6_cpd_validate_metadata(struct ipu6_device *isp,
@@ -316,47 +325,163 @@ static int ipu6_cpd_validate_metadata(struct ipu6_device *isp,
 	return 0;
 }
 
-int ipu6_cpd_validate_cpd_file(struct ipu6_device *isp, const void *cpd_file,
-			       unsigned long cpd_file_size)
+static struct ipu7_cpd_metadata *ipu7_cpd_get_metadata(const void *cpd, int idx)
 {
-	const struct ipu6_cpd_hdr *hdr = cpd_file;
-	const struct ipu6_cpd_ent *ent;
-	int ret;
+	const struct ipu6_cpd_ent *cpd_ent =
+		ipu6_cpd_get_entry(cpd, IPU7_CPD_METADATA_START_IDX + idx * 2);
 
-	ret = ipu6_cpd_validate_cpd(isp, cpd_file, cpd_file_size,
-				    cpd_file_size);
-	if (ret) {
-		dev_err(&isp->pdev->dev, "Invalid CPD in file\n");
-		return ret;
+	return (struct ipu7_cpd_metadata *)((u8 *)cpd + cpd_ent->offset);
+}
+
+static int ipu7_cpd_validate_metadata(struct ipu6_device *isp,
+				      const void *cpd, int idx)
+{
+	const struct ipu6_cpd_ent *cpd_ent =
+		ipu6_cpd_get_entry(cpd, IPU7_CPD_METADATA_START_IDX + idx * 2);
+	const struct ipu6_cpd_ent *bin_ent =
+		ipu6_cpd_get_entry(cpd, IPU7_CPD_BINARY_START_IDX + idx * 2);
+	const struct ipu7_cpd_metadata *metadata =
+		ipu7_cpd_get_metadata(cpd, idx);
+	struct device *dev = &isp->pdev->dev;
+	u32 offset;
+
+	/* Sanity check for metadata size */
+	if (cpd_ent->len != sizeof(struct ipu7_cpd_metadata)) {
+		dev_err(dev, "Invalid metadata size\n");
+		return -EINVAL;
 	}
 
-	/* Check for CPD file marker */
-	if (hdr->hdr_mark != CPD_HDR_MARK) {
-		dev_err(&isp->pdev->dev, "Invalid CPD header\n");
+	/* Validate type and length of metadata sections */
+	if (metadata->attr.hdr.type != IPU7_CPD_METADATA_ATTR) {
+		dev_err(dev, "Invalid metadata attr type (%d)\n",
+			metadata->attr.hdr.type);
+		return -EINVAL;
+	}
+	if (metadata->attr.hdr.len != sizeof(struct ipu7_cpd_metadata_attr)) {
+		dev_err(dev, "Invalid metadata attr size (%d)\n",
+			metadata->attr.hdr.len);
+		return -EINVAL;
+	}
+	if (metadata->ipl.hdr.type != IPU7_CPD_METADATA_IPL) {
+		dev_err(dev, "Invalid metadata ipl type (%d)\n",
+			metadata->ipl.hdr.type);
+		return -EINVAL;
+	}
+	if (metadata->ipl.hdr.len != sizeof(struct ipu7_cpd_metadata_ipl)) {
+		dev_err(dev, "Invalid metadata ipl size (%d)\n",
+			metadata->ipl.hdr.len);
+		return -EINVAL;
+	}
+
+	offset = metadata->ipl.param[0];
+	if (offset > IPU7_FW_CODE_REGION_SIZE ||
+	    bin_ent->len > IPU7_FW_CODE_REGION_SIZE - offset) {
+		dev_err(dev, "Incorrect binary size %u and offset %u\n",
+			bin_ent->len, offset);
 		return -EINVAL;
 	}
 
-	/* Sanity check for manifest size */
+	return 0;
+}
+
+static int __ipu7_validate_cpd_file(struct ipu6_device *isp, const void *cpd_file,
+				    unsigned long cpd_file_size)
+{
+	const struct ipu6_cpd_ent *ent;
+	const struct ipu7_cpd_hdr *hdr = cpd_file;
+	unsigned int i;
+
 	ent = ipu6_cpd_get_manifest(cpd_file);
-	if (ent->len > MAX_MANIFEST_SIZE) {
+	if (ent->len > IPU7_MAX_MANIFEST_SIZE) {
 		dev_err(&isp->pdev->dev, "Invalid CPD manifest size\n");
 		return -EINVAL;
 	}
+	/* Sanity check for CPD entry header */
+	if (hdr->ent_cnt != IPU7_CPD_ENTRY_NUM) {
+		dev_err(&isp->pdev->dev, "Invalid CPD entry number %d\n",
+			hdr->ent_cnt);
+		return -EINVAL;
+	}
 
 	/* Validate metadata */
+	for (i = 0; i < IPU7_CPD_BINARY_NUM; i++) {
+		int ret = ipu7_cpd_validate_metadata(isp, cpd_file, i);
+
+		if (ret) {
+			dev_err(&isp->pdev->dev, "Invalid metadata(%d)\n", i);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+static int __ipu6_validate_cpd_file(struct ipu6_device *isp, const void *cpd_file,
+				    unsigned long cpd_file_size)
+{
+	const struct ipu6_cpd_ent *ent;
+	int ret;
+
+	ent = ipu6_cpd_get_manifest(cpd_file);
+	if (ent->len > MAX_MANIFEST_SIZE) {
+		dev_err(&isp->pdev->dev, "Invalid CPD manifest size\n");
+		return -EINVAL;
+	}
+
 	ent = ipu6_cpd_get_metadata(cpd_file);
 	ret = ipu6_cpd_validate_metadata(isp, cpd_file + ent->offset, ent->len);
-	if (ret) {
-		dev_err(&isp->pdev->dev, "Invalid CPD metadata\n");
+	if (ret)
 		return ret;
-	}
 
-	/* Validate moduledata */
 	ent = ipu6_cpd_get_moduledata(cpd_file);
-	ret = ipu6_cpd_validate_moduledata(isp, cpd_file + ent->offset,
-					   ent->len);
+	return ipu6_cpd_validate_moduledata(isp, cpd_file + ent->offset,
+					    ent->len);
+}
+
+int ipu6_cpd_validate_cpd_file(struct ipu6_device *isp, const void *cpd_file,
+			       unsigned long cpd_file_size)
+{
+	const struct ipu6_cpd_hdr *hdr = cpd_file;
+	int ret;
+
+	ret = ipu6_cpd_validate_cpd(isp, cpd_file, cpd_file_size,
+				    cpd_file_size);
 	if (ret)
-		dev_err(&isp->pdev->dev, "Invalid CPD moduledata\n");
+		return ret;
+
+	/* Check for CPD file marker */
+	if (hdr->hdr_mark != CPD_HDR_MARK) {
+		dev_err(&isp->pdev->dev, "Invalid CPD header\n");
+		return -EINVAL;
+	}
+
+	if (IS_IPU7(isp))
+		return __ipu7_validate_cpd_file(isp, cpd_file, cpd_file_size);
+
+	return __ipu6_validate_cpd_file(isp, cpd_file, cpd_file_size);
+}
+
+int ipu6_ipu7_cpd_copy_binary(const void *cpd, const char *name, void *dst,
+			      u32 *entry)
+{
+	unsigned int i;
+
+	for (i = 0; i < IPU7_CPD_BINARY_NUM; i++) {
+		const struct ipu7_cpd_metadata *metadata;
+		u8 idx = IPU7_CPD_BINARY_START_IDX + i * 2U;
+		const struct ipu6_cpd_ent *ent =
+			ipu6_cpd_get_entry(cpd, idx);
+
+		if (strncmp(ent->name, name, sizeof(ent->name)))
+			continue;
+
+		metadata = ipu7_cpd_get_metadata(cpd, i);
+		memcpy(dst + metadata->ipl.param[0], cpd + ent->offset,
+		       ent->len);
+		*entry = metadata->ipl.param[2];
+
+		return 0;
+	}
 
-	return ret;
+	return -ENOENT;
 }
+EXPORT_SYMBOL_NS_GPL(ipu6_ipu7_cpd_copy_binary, "INTEL_IPU6");
diff --git a/drivers/media/pci/intel/ipu6/ipu6-cpd.h b/drivers/media/pci/intel/ipu6/ipu6-cpd.h
index e0e4fdeca902fc..c3a4859d8e7945 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-cpd.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-cpd.h
@@ -98,8 +98,51 @@ struct ipu6_cpd_client_pkg_hdr {
 	u32 prog_bin_size;
 } __packed;
 
+/* IPU7 */
+
+struct ipu7_cpd_hdr {
+	u32 hdr_mark;
+	u32 ent_cnt;
+	u8 hdr_ver;
+	u8 ent_ver;
+	u8 hdr_len;
+	u8 rsvd;
+	u8 partition_name[4];
+	u32 crc32;
+} __packed;
+
+struct ipu7_cpd_metadata_hdr {
+	u32 type;
+	u32 len;
+} __packed;
+
+struct ipu7_cpd_metadata_attr {
+	struct ipu7_cpd_metadata_hdr hdr;
+	u8 compression_type;
+	u8 encryption_type;
+	u8 rsvd[2];
+	u32 uncompressed_size;
+	u32 compressed_size;
+	u32 module_id;
+	u8 hash[48];
+} __packed;
+
+struct ipu7_cpd_metadata_ipl {
+	struct ipu7_cpd_metadata_hdr hdr;
+	u32 param[4];
+	u8 rsvd[8];
+} __packed;
+
+struct ipu7_cpd_metadata {
+	struct ipu7_cpd_metadata_attr attr;
+	struct ipu7_cpd_metadata_ipl ipl;
+} __packed;
+
 int ipu6_cpd_create_pkg_dir(struct ipu6_bus_device *adev, const void *src);
 void ipu6_cpd_free_pkg_dir(struct ipu6_bus_device *adev);
 int ipu6_cpd_validate_cpd_file(struct ipu6_device *isp, const void *cpd_file,
 			       unsigned long cpd_file_size);
+int ipu6_ipu7_cpd_copy_binary(const void *cpd, const char *name, void *dst,
+			      u32 *entry);
+
 #endif /* IPU6_CPD_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu6-dma.c b/drivers/media/pci/intel/ipu6/ipu6-dma.c
index fdcdb15b073c75..90f530a9377944 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-dma.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-dma.c
@@ -286,7 +286,7 @@ void ipu6_dma_free(struct ipu6_bus_device *sys, size_t size, void *vaddr,
 
 	__free_buffer(pages, size, attrs);
 
-	mmu->tlb_invalidate(mmu);
+	mmu->ops->tlb_invalidate(mmu);
 
 	__free_iova(&mmu->dmap->iovad, iova);
 
@@ -366,11 +366,22 @@ void ipu6_dma_unmap_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
 	ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
 		       PFN_PHYS(iova_size(iova)));
 
-	mmu->tlb_invalidate(mmu);
+	mmu->ops->tlb_invalidate(mmu);
 	__free_iova(&mmu->dmap->iovad, iova);
 }
 EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sg, "INTEL_IPU6");
 
+static struct iova *ipu7_get_fw_code_region(struct ipu6_bus_device *sys)
+{
+	struct ipu6_mmu *mmu = sys->mmu;
+	unsigned long lo, hi;
+
+	lo = iova_pfn(&mmu->dmap->iovad, IPU7_FW_CODE_REGION_START);
+	hi = iova_pfn(&mmu->dmap->iovad, IPU7_FW_CODE_REGION_END) - 1U;
+
+	return reserve_iova(&mmu->dmap->iovad, lo, hi);
+}
+
 int ipu6_dma_map_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
 		    int nents, enum dma_data_direction dir,
 		    unsigned long attrs)
@@ -397,10 +408,13 @@ int ipu6_dma_map_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
 	dev_dbg(dev, "dmamap trying to map %d ents %zu pages\n",
 		nents, npages);
 
-	iova = alloc_iova(&mmu->dmap->iovad, npages,
-			  PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
+	if (attrs & DMA_ATTR_RESERVE_REGION)
+		iova = ipu7_get_fw_code_region(sys);
+	else
+		iova = alloc_iova(&mmu->dmap->iovad, npages,
+				  PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
 	if (!iova)
-		return 0;
+		return -ENOMEM;
 
 	dev_dbg(dev, "dmamap: iova low pfn %lu, high pfn %lu\n", iova->pfn_lo,
 		iova->pfn_hi);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-dma.h b/drivers/media/pci/intel/ipu6/ipu6-dma.h
index ae9b9a5df57f69..e5e0e786042349 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-dma.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-dma.h
@@ -10,6 +10,8 @@
 
 #include "ipu6-bus.h"
 
+#define DMA_ATTR_RESERVE_REGION		BIT(31)
+
 struct ipu6_mmu_info;
 
 struct ipu6_dma_mapping {
diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
index 62ed92ff1d3039..f4f1cf7c86d20f 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.c
@@ -7,6 +7,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/io.h>
+#include <linux/pm_runtime.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
@@ -15,6 +16,7 @@
 #include "ipu6-isys.h"
 #include "ipu6-platform-isys-csi2-reg.h"
 #include "ipu6-platform-regs.h"
+#include "ipu7-fw-isys.h"
 
 static const char send_msg_types[N_IPU6_FW_ISYS_SEND_TYPE][32] = {
 	"STREAM_OPEN",
@@ -32,7 +34,7 @@ static int handle_proxy_response(struct ipu6_isys *isys, unsigned int req_id)
 	struct ipu6_fw_isys_proxy_resp_info_abi *resp;
 	int ret;
 
-	resp = ipu6_recv_get_token(isys->fwcom, IPU6_BASE_PROXY_RECV_QUEUES);
+	resp = ipu6_recv_get_token(isys->fwctx, IPU6_BASE_PROXY_RECV_QUEUES);
 	if (!resp)
 		return 1;
 
@@ -42,7 +44,7 @@ static int handle_proxy_response(struct ipu6_isys *isys, unsigned int req_id)
 
 	ret = req_id == resp->request_id ? 0 : -EIO;
 
-	ipu6_recv_put_token(isys->fwcom, IPU6_BASE_PROXY_RECV_QUEUES);
+	ipu6_recv_put_token(isys->fwctx, IPU6_BASE_PROXY_RECV_QUEUES);
 
 	return ret;
 }
@@ -52,7 +54,7 @@ int ipu6_fw_isys_send_proxy_token(struct ipu6_isys *isys,
 				  unsigned int index,
 				  unsigned int offset, u32 value)
 {
-	struct ipu6_fw_com_context *ctx = isys->fwcom;
+	struct ipu6_fw_com_context *ctx = isys->fwctx;
 	struct device *dev = &isys->adev->auxdev.dev;
 	struct ipu6_fw_proxy_send_queue_token *token;
 	unsigned int timeout = 1000;
@@ -90,13 +92,13 @@ int ipu6_fw_isys_send_proxy_token(struct ipu6_isys *isys,
 	return ret;
 }
 
-int ipu6_fw_isys_complex_cmd(struct ipu6_isys *isys,
-			     const unsigned int stream_handle,
-			     void *cpu_mapped_buf,
-			     dma_addr_t dma_mapped_buf,
-			     size_t size, u16 send_type)
+static int ipu6_fw_isys_complex_cmd(struct ipu6_isys *isys,
+				    const unsigned int stream_handle,
+				    void *cpu_mapped_buf,
+				    dma_addr_t dma_mapped_buf,
+				    size_t size, u16 send_type)
 {
-	struct ipu6_fw_com_context *ctx = isys->fwcom;
+	struct ipu6_fw_com_context *ctx = isys->fwctx;
 	struct device *dev = &isys->adev->auxdev.dev;
 	struct ipu6_fw_send_queue_token *token;
 
@@ -126,19 +128,11 @@ int ipu6_fw_isys_complex_cmd(struct ipu6_isys *isys,
 	return 0;
 }
 
-int ipu6_fw_isys_simple_cmd(struct ipu6_isys *isys,
-			    const unsigned int stream_handle, u16 send_type)
-{
-	return ipu6_fw_isys_complex_cmd(isys, stream_handle, NULL, 0, 0,
-					send_type);
-}
-
-int ipu6_fw_isys_close(struct ipu6_isys *isys)
+static int ipu6_fw_isys_close(struct ipu6_isys *isys)
 {
 	struct device *dev = &isys->adev->auxdev.dev;
 	int retry = IPU6_ISYS_CLOSE_RETRY;
-	unsigned long flags;
-	void *fwcom;
+	void *fwctx;
 	int ret;
 
 	/*
@@ -147,40 +141,36 @@ int ipu6_fw_isys_close(struct ipu6_isys *isys)
 	 * to SP icache.
 	 * spinlock to wait the interrupt handler to be finished
 	 */
-	spin_lock_irqsave(&isys->power_lock, flags);
-	ret = ipu6_fw_com_close(isys->fwcom);
-	fwcom = isys->fwcom;
-	isys->fwcom = NULL;
-	spin_unlock_irqrestore(&isys->power_lock, flags);
+	ret = ipu6_fw_com_close(isys->fwctx);
+	fwctx = isys->fwctx;
+	isys->fwctx = NULL;
 	if (ret)
 		dev_err(dev, "Device close failure: %d\n", ret);
 
 	/* release probably fails if the close failed. Let's try still */
 	do {
 		usleep_range(400, 500);
-		ret = ipu6_fw_com_release(fwcom, 0);
+		ret = ipu6_fw_com_release(fwctx, 0);
 		retry--;
 	} while (ret && retry);
 
 	if (ret) {
 		dev_err(dev, "Device release time out %d\n", ret);
-		spin_lock_irqsave(&isys->power_lock, flags);
-		isys->fwcom = fwcom;
-		spin_unlock_irqrestore(&isys->power_lock, flags);
+		isys->fwctx = fwctx;
 	}
 
 	return ret;
 }
 
-void ipu6_fw_isys_cleanup(struct ipu6_isys *isys)
+static void ipu6_fw_isys_cleanup(struct ipu6_isys *isys)
 {
 	int ret;
 
-	ret = ipu6_fw_com_release(isys->fwcom, 1);
+	ret = ipu6_fw_com_release(isys->fwctx, 1);
 	if (ret < 0)
 		dev_warn(&isys->adev->auxdev.dev,
 			 "Device busy, fw_com release failed.");
-	isys->fwcom = NULL;
+	isys->fwctx = NULL;
 }
 
 static void start_sp(struct ipu6_bus_device *adev)
@@ -212,7 +202,7 @@ static int query_sp(struct ipu6_bus_device *adev)
 }
 
 static int ipu6_isys_fwcom_cfg_init(struct ipu6_isys *isys,
-				    struct ipu6_fw_com_cfg *fwcom,
+				    struct ipu6_fw_com_cfg *fwcom_cfg,
 				    unsigned int num_streams)
 {
 	unsigned int max_send_queues, max_sram_blocks, max_devq_size;
@@ -258,14 +248,16 @@ static int ipu6_isys_fwcom_cfg_init(struct ipu6_isys *isys,
 	if (!output_queue_cfg)
 		return -ENOMEM;
 
-	fwcom->input = input_queue_cfg;
-	fwcom->output = output_queue_cfg;
+	fwcom_cfg->input = input_queue_cfg;
+	fwcom_cfg->output = output_queue_cfg;
 
-	fwcom->num_input_queues = isys_fw_cfg->num_send_queues[type_proxy] +
+	fwcom_cfg->num_input_queues =
+		isys_fw_cfg->num_send_queues[type_proxy] +
 		isys_fw_cfg->num_send_queues[type_dev] +
 		isys_fw_cfg->num_send_queues[type_msg];
 
-	fwcom->num_output_queues = isys_fw_cfg->num_recv_queues[type_proxy] +
+	fwcom_cfg->num_output_queues =
+		isys_fw_cfg->num_recv_queues[type_proxy] +
 		isys_fw_cfg->num_recv_queues[type_dev] +
 		isys_fw_cfg->num_recv_queues[type_msg];
 
@@ -280,7 +272,7 @@ static int ipu6_isys_fwcom_cfg_init(struct ipu6_isys *isys,
 			isys_fw_cfg->buffer_partition.num_gda_pages[i] = 0;
 	}
 
-	/* FW assumes proxy interface at fwcom queue 0 */
+	/* FW assumes proxy interface at fwcom_cfg queue 0 */
 	for (i = 0; i < isys_fw_cfg->num_send_queues[type_proxy]; i++) {
 		input_queue_cfg[i].token_size =
 			sizeof(struct ipu6_fw_proxy_send_queue_token);
@@ -314,34 +306,34 @@ static int ipu6_isys_fwcom_cfg_init(struct ipu6_isys *isys,
 			IPU6_ISYS_SIZE_RECV_QUEUE;
 	}
 
-	fwcom->dmem_addr = isys->pdata->ipdata->hw_variant.dmem_offset;
-	fwcom->specific_addr = isys_fw_cfg;
-	fwcom->specific_size = sizeof(*isys_fw_cfg);
+	fwcom_cfg->dmem_addr = isys->pdata->ipdata->hw_variant.dmem_offset;
+	fwcom_cfg->specific_addr = isys_fw_cfg;
+	fwcom_cfg->specific_size = sizeof(*isys_fw_cfg);
 
 	return 0;
 }
 
-int ipu6_fw_isys_init(struct ipu6_isys *isys, unsigned int num_streams)
+static int ipu6_fw_isys_init(struct ipu6_isys *isys, unsigned int num_streams)
 {
 	struct device *dev = &isys->adev->auxdev.dev;
 	int retry = IPU6_ISYS_OPEN_RETRY;
-	struct ipu6_fw_com_cfg fwcom = {
+	struct ipu6_fw_com_cfg fwcom_cfg = {
 		.cell_start = start_sp,
 		.cell_ready = query_sp,
 		.buttress_boot_offset = SYSCOM_BUTTRESS_FW_PARAMS_ISYS_OFFSET,
 	};
 	int ret;
 
-	ipu6_isys_fwcom_cfg_init(isys, &fwcom, num_streams);
+	ipu6_isys_fwcom_cfg_init(isys, &fwcom_cfg, num_streams);
 
-	isys->fwcom = ipu6_fw_com_prepare(&fwcom, isys->adev,
+	isys->fwctx = ipu6_fw_com_prepare(&fwcom_cfg, isys->adev,
 					  isys->pdata->base);
-	if (!isys->fwcom) {
+	if (!isys->fwctx) {
 		dev_err(dev, "isys fw com prepare failed\n");
 		return -EIO;
 	}
 
-	ret = ipu6_fw_com_open(isys->fwcom);
+	ret = ipu6_fw_com_open(isys->fwctx);
 	if (ret) {
 		dev_err(dev, "isys fw com open failed %d\n", ret);
 		return ret;
@@ -349,7 +341,7 @@ int ipu6_fw_isys_init(struct ipu6_isys *isys, unsigned int num_streams)
 
 	do {
 		usleep_range(400, 500);
-		if (ipu6_fw_com_ready(isys->fwcom))
+		if (ipu6_fw_com_ready(isys->fwctx))
 			break;
 		retry--;
 	} while (retry > 0);
@@ -363,20 +355,21 @@ int ipu6_fw_isys_init(struct ipu6_isys *isys, unsigned int num_streams)
 	return ret;
 }
 
-struct ipu6_fw_isys_resp_info_abi *
-ipu6_fw_isys_get_resp(void *context, unsigned int queue)
+static struct ipu6_fw_isys_resp_info_abi *
+ipu6_fw_isys_get_resp(struct ipu6_isys *isys)
 {
-	return ipu6_recv_get_token(context, queue);
+	return ipu6_recv_get_token(isys->fwctx, IPU6_BASE_MSG_RECV_QUEUES);
 }
 
-void ipu6_fw_isys_put_resp(void *context, unsigned int queue)
+static void ipu6_fw_isys_put_resp(struct ipu6_isys *isys)
 {
-	ipu6_recv_put_token(context, queue);
+	ipu6_recv_put_token(isys->fwctx, IPU6_BASE_MSG_RECV_QUEUES);
 }
 
-void ipu6_fw_isys_dump_stream_cfg(struct device *dev,
-				  struct ipu6_fw_isys_stream_cfg_data_abi *cfg)
+static void ipu6_fw_isys_dump_stream_cfg(struct device *dev,
+					 struct isys_fw_msgs *msg)
 {
+	struct ipu6_fw_isys_stream_cfg_data_abi *cfg = &msg->ipu6.stream;
 	unsigned int i;
 
 	dev_dbg(dev, "-----------------------------------------------------\n");
@@ -451,13 +444,15 @@ void ipu6_fw_isys_dump_stream_cfg(struct device *dev,
 	dev_dbg(dev, "-----------------------------------------------------\n");
 }
 
-void
-ipu6_fw_isys_dump_frame_buff_set(struct device *dev,
-				 struct ipu6_fw_isys_frame_buff_set_abi *buf,
-				 unsigned int outputs)
+static void
+ipu6_fw_isys_dump_frame_buf_set(struct device *dev, struct isys_fw_msgs *msg,
+				unsigned int outputs)
 {
+	struct ipu6_fw_isys_frame_buff_set_abi *buf;
 	unsigned int i;
 
+	buf = &msg->ipu6.frame;
+
 	dev_dbg(dev, "-----------------------------------------------------\n");
 	dev_dbg(dev, "IPU6_FW_ISYS_FRAME_BUFF_SET\n");
 
@@ -485,3 +480,478 @@ ipu6_fw_isys_dump_frame_buff_set(struct device *dev,
 
 	dev_dbg(dev, "-----------------------------------------------------\n");
 }
+
+struct fwmsg {
+	int type;
+	char *msg;
+	bool valid_ts;
+};
+
+static const struct fwmsg fw_msg[] = {
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_OPEN_DONE, "STREAM_OPEN_DONE", 0},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_CLOSE_ACK, "STREAM_CLOSE_ACK", 0},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_START_ACK, "STREAM_START_ACK", 0},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK,
+	 "STREAM_START_AND_CAPTURE_ACK", 0},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_STOP_ACK, "STREAM_STOP_ACK", 0},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_FLUSH_ACK, "STREAM_FLUSH_ACK", 0},
+	{IPU6_FW_ISYS_RESP_TYPE_PIN_DATA_READY, "PIN_DATA_READY", 1},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_ACK, "STREAM_CAPTURE_ACK", 0},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE,
+	 "STREAM_START_AND_CAPTURE_DONE", 1},
+	{IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_DONE, "STREAM_CAPTURE_DONE", 1},
+	{IPU6_FW_ISYS_RESP_TYPE_FRAME_SOF, "FRAME_SOF", 1},
+	{IPU6_FW_ISYS_RESP_TYPE_FRAME_EOF, "FRAME_EOF", 1},
+	{IPU6_FW_ISYS_RESP_TYPE_STATS_DATA_READY, "STATS_READY", 1},
+	{-1, "UNKNOWN MESSAGE", 0}
+};
+
+static u32 resp_type_to_index(int type)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(fw_msg); i++)
+		if (fw_msg[i].type == type)
+			return i;
+
+	return  ARRAY_SIZE(fw_msg) - 1;
+}
+
+int ipu6_isys_isr_one(struct ipu6_bus_device *adev)
+{
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	struct ipu6_fw_isys_resp_info_abi *resp;
+	struct ipu6_isys_stream *stream;
+	struct ipu6_isys_csi2 *csi2 = NULL;
+	struct isys_fw_msgs *isys_fw_msg = NULL;
+	u32 index;
+	u64 ts;
+
+	resp = ipu6_fw_isys_get_resp(isys);
+	if (!resp)
+		return 1;
+
+	ts = (u64)resp->timestamp[1] << 32 | resp->timestamp[0];
+
+	index = resp_type_to_index(resp->type);
+	dev_dbg(&adev->auxdev.dev,
+		"FW resp %02d %s, stream %u, ts 0x%16.16llx, pin %d\n",
+		resp->type, fw_msg[index].msg, resp->stream_handle,
+		fw_msg[index].valid_ts ? ts : 0, resp->pin_id);
+
+	if (resp->error_info.error == IPU6_FW_ISYS_ERROR_STREAM_IN_SUSPENSION)
+		/* Suspension is kind of special case: not enough buffers */
+		dev_dbg(&adev->auxdev.dev,
+			"FW error resp SUSPENSION, details %d\n",
+			resp->error_info.error_details);
+	else if (resp->error_info.error)
+		dev_dbg(&adev->auxdev.dev,
+			"FW error resp error %d, details %d\n",
+			resp->error_info.error, resp->error_info.error_details);
+
+	guard(spinlock_irqsave)(&isys->streams_lock);
+
+	if (resp->stream_handle >= IPU6_ISYS_MAX_STREAMS) {
+		dev_err(&adev->auxdev.dev, "bad stream handle %u\n",
+			resp->stream_handle);
+		goto leave;
+	}
+
+	stream = resp->stream_handle < IPU6_ISYS_MAX_STREAMS ?
+		isys->streams_by_handle[resp->stream_handle] : NULL;
+	if (!stream) {
+		dev_err(&adev->auxdev.dev, "stream of stream_handle %u is unused\n",
+			resp->stream_handle);
+		goto leave;
+	}
+	stream->error = resp->error_info.error;
+
+	csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
+
+	switch (resp->type) {
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_OPEN_DONE:
+		complete(&stream->stream_open_completion);
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_CLOSE_ACK:
+		complete(&stream->stream_close_completion);
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_START_ACK:
+		complete(&stream->stream_start_completion);
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK:
+		complete(&stream->stream_start_completion);
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_STOP_ACK:
+		complete(&stream->stream_stop_completion);
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_FLUSH_ACK:
+		complete(&stream->stream_stop_completion);
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_PIN_DATA_READY:
+		/*
+		 * firmware only release the capture message until software
+		 * get pin_data_ready event
+		 */
+		if (!resp->buf_id) {
+			dev_warn(&adev->auxdev.dev, "%d: Invalid buf ID\n",
+				 resp->stream_handle);
+			goto leave;
+		}
+
+		isys_fw_msg = container_of((void *)(uintptr_t)resp->buf_id,
+					   struct isys_fw_msgs, dummy);
+
+		ipu6_put_fw_msg_buf(ipu6_bus_get_drvdata(adev), isys_fw_msg);
+		if (resp->pin_id < IPU6_ISYS_OUTPUT_PINS &&
+		    stream->output_pins_queue[resp->pin_id])
+			ipu6_isys_queue_buf_ready(stream, resp);
+		else
+			dev_warn(&adev->auxdev.dev,
+				 "%d:No queue for pin id %d\n",
+				 resp->stream_handle, resp->pin_id);
+		if (csi2)
+			ipu6_isys_csi2_error(csi2);
+
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_ACK:
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE:
+	case IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_DONE:
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_FRAME_SOF:
+
+		ipu6_isys_csi2_sof_event_by_stream(stream);
+		stream->seq[stream->seq_index].sequence =
+			atomic_read(&stream->sequence) - 1;
+		stream->seq[stream->seq_index].timestamp = ts;
+		dev_dbg(&adev->auxdev.dev,
+			"sof: handle %d: (index %u), timestamp 0x%16.16llx\n",
+			resp->stream_handle,
+			stream->seq[stream->seq_index].sequence, ts);
+		stream->seq_index = (stream->seq_index + 1)
+			% IPU6_ISYS_MAX_PARALLEL_SOF;
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_FRAME_EOF:
+		ipu6_isys_csi2_eof_event_by_stream(stream);
+		dev_dbg(&adev->auxdev.dev,
+			"eof: handle %d: (index %u), timestamp 0x%16.16llx\n",
+			resp->stream_handle,
+			stream->seq[stream->seq_index].sequence, ts);
+		break;
+	case IPU6_FW_ISYS_RESP_TYPE_STATS_DATA_READY:
+		break;
+	default:
+		dev_err(&adev->auxdev.dev, "%d:unknown response type %u\n",
+			resp->stream_handle, resp->type);
+		break;
+	}
+
+leave:
+	ipu6_fw_isys_put_resp(isys);
+	return 0;
+}
+
+static void ipu6_isys_csi2_isr(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys_stream *stream;
+	unsigned int i;
+	u32 status;
+
+	ipu6_isys_register_errors(csi2);
+
+	status = readl(csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
+		       CSI_PORT_REG_BASE_IRQ_STATUS_OFFSET);
+
+	writel(status, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
+	       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
+
+	scoped_guard(spinlock, &csi2->isys->streams_lock) {
+		for (i = 0; i < NR_OF_CSI2_VC; i++) {
+			if (status & IPU_CSI_RX_IRQ_FS_VC(i)) {
+				stream = csi2->streams_by_vc[i];
+				if (!stream)
+					continue;
+
+				ipu6_isys_csi2_sof_event_by_stream(stream);
+			}
+
+			if (status & IPU_CSI_RX_IRQ_FE_VC(i)) {
+				stream = csi2->streams_by_vc[i];
+				if (!stream)
+					continue;
+
+				ipu6_isys_csi2_eof_event_by_stream(stream);
+			}
+		}
+	}
+}
+
+irqreturn_t ipu6_isys_isr(struct ipu6_bus_device *adev)
+{
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	void __iomem *base = isys->pdata->base;
+	u32 status_sw, status_csi;
+	u32 ctrl0_status, ctrl0_clear;
+	int pm_status;
+
+	pm_status = pm_runtime_get_if_active(&adev->auxdev.dev);
+	if (!pm_status)
+		return 0;
+
+	ctrl0_status = isys->pdata->ipdata->csi2.ctrl0_irq_status;
+	ctrl0_clear = isys->pdata->ipdata->csi2.ctrl0_irq_clear;
+
+	status_csi = readl(isys->pdata->base + ctrl0_status);
+	status_sw = readl(isys->pdata->base +
+			  IPU6_REG_ISYS_UNISPART_IRQ_STATUS);
+
+	writel(ISYS_UNISPART_IRQS & ~IPU6_ISYS_UNISPART_IRQ_SW,
+	       base + IPU6_REG_ISYS_UNISPART_IRQ_MASK);
+
+	do {
+		writel(status_csi, isys->pdata->base + ctrl0_clear);
+
+		writel(status_sw, isys->pdata->base +
+		       IPU6_REG_ISYS_UNISPART_IRQ_CLEAR);
+
+		if (isys->isr_csi2_bits & status_csi) {
+			unsigned int i;
+
+			for (i = 0; i < isys->pdata->ipdata->csi2.nports; i++) {
+				/* irq from not enabled port */
+				if (!isys->csi2[i].base)
+					continue;
+				if (status_csi & IPU6_ISYS_UNISPART_IRQ_CSI2(i))
+					ipu6_isys_csi2_isr(&isys->csi2[i]);
+			}
+		}
+
+		writel(0, base + IPU6_REG_ISYS_UNISPART_SW_IRQ_REG);
+
+		if (!ipu6_isys_isr_one(adev))
+			status_sw = IPU6_ISYS_UNISPART_IRQ_SW;
+		else
+			status_sw = 0;
+
+		status_csi = readl(isys->pdata->base + ctrl0_status);
+		status_sw |= readl(isys->pdata->base +
+				   IPU6_REG_ISYS_UNISPART_IRQ_STATUS);
+	} while ((status_csi & isys->isr_csi2_bits) ||
+		 (status_sw & IPU6_ISYS_UNISPART_IRQ_SW));
+
+	writel(ISYS_UNISPART_IRQS, base + IPU6_REG_ISYS_UNISPART_IRQ_MASK);
+
+	if (pm_status > 0)
+		pm_runtime_put(&adev->auxdev.dev);
+
+	return IRQ_HANDLED;
+}
+
+static int ipu6_isys_fw_pin_cfg(struct ipu6_isys_video *av,
+				struct ipu6_isys_stream *stream,
+				struct media_pad *src_pad,
+				struct v4l2_mbus_frame_desc_entry *entry,
+				void *__cfg)
+{
+	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(src_pad->entity);
+	struct v4l2_subdev_state *state = v4l2_subdev_get_locked_active_state(sd);
+	struct ipu6_fw_isys_stream_cfg_data_abi *cfg = __cfg;
+	struct ipu6_fw_isys_input_pin_info_abi *input_pin;
+	struct ipu6_fw_isys_output_pin_info_abi *output_pin;
+	struct ipu6_isys_queue *aq = &av->aq;
+	struct v4l2_mbus_framefmt *fmt;
+	const struct ipu6_isys_pixelformat *pfmt =
+		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
+	struct v4l2_rect v4l2_crop;
+	struct ipu6_isys *isys = av->isys;
+	int input_pins = cfg->nof_input_pins++;
+	int output_pins;
+	u32 src_stream;
+
+	src_stream = __ipu6_isys_get_src_stream_by_src_pad(state, src_pad->index);
+	fmt = v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
+	v4l2_crop = *v4l2_subdev_state_get_crop(state, src_pad->index, src_stream);
+
+	input_pin = &cfg->input_pins[input_pins];
+	input_pin->input_res.width = fmt->width;
+	input_pin->input_res.height = fmt->height;
+	input_pin->dt = entry->bus.csi2.dt;
+	input_pin->bits_per_pix = pfmt->bpp_packed;
+	input_pin->mapped_dt = 0x40; /* invalid mipi data type */
+	input_pin->mipi_decompression = 0;
+	input_pin->capture_mode = IPU6_FW_ISYS_CAPTURE_MODE_REGULAR;
+	input_pin->mipi_store_mode = pfmt->bpp == pfmt->bpp_packed ?
+		IPU6_FW_ISYS_MIPI_STORE_MODE_DISCARD_LONG_HEADER :
+		IPU6_FW_ISYS_MIPI_STORE_MODE_NORMAL;
+	input_pin->crop_first_and_last_lines = v4l2_crop.top & 1;
+
+	output_pins = cfg->nof_output_pins++;
+	aq->fw_output = output_pins;
+	stream->output_pins_queue[output_pins] = aq;
+
+	output_pin = &cfg->output_pins[output_pins];
+	output_pin->input_pin_id = input_pins;
+	output_pin->output_res.width = ipu6_isys_get_frame_width(av);
+	output_pin->output_res.height = ipu6_isys_get_frame_height(av);
+
+	output_pin->stride = ipu6_isys_get_bytes_per_line(av);
+	if (pfmt->bpp != pfmt->bpp_packed)
+		output_pin->pt = IPU6_FW_ISYS_PIN_TYPE_RAW_SOC;
+	else
+		output_pin->pt = IPU6_FW_ISYS_PIN_TYPE_MIPI;
+	output_pin->ft = pfmt->css_pixelformat;
+	output_pin->send_irq = 1;
+	memset(output_pin->ts_offsets, 0, sizeof(output_pin->ts_offsets));
+	output_pin->s2m_pixel_soc_pixel_remapping =
+		S2M_PIXEL_SOC_PIXEL_REMAPPING_FLAG_NO_REMAPPING;
+	output_pin->csi_be_soc_pixel_remapping =
+		CSI_BE_SOC_PIXEL_REMAPPING_FLAG_NO_REMAPPING;
+
+	output_pin->snoopable = true;
+	output_pin->error_handling_enable = false;
+	output_pin->sensor_type = isys->sensor_type++;
+	if (isys->sensor_type > isys->pdata->ipdata->sensor_type_end)
+		isys->sensor_type = isys->pdata->ipdata->sensor_type_start;
+
+	return 0;
+}
+
+static int ipu6_fw_isys_prepare_stream_cfg(struct ipu6_isys_stream *stream,
+					   struct v4l2_mbus_frame_desc *desc,
+					   struct isys_fw_msgs *msg)
+{
+	struct ipu6_fw_isys_stream_cfg_data_abi *stream_cfg;
+	struct device *dev = &stream->isys->adev->auxdev.dev;
+	int ret;
+
+	stream_cfg = &msg->ipu6.stream;
+	stream_cfg->src = stream->asd->source;
+	stream_cfg->vc = stream->vc;
+	stream_cfg->isl_use = 0;
+	stream_cfg->sensor_type = IPU6_FW_ISYS_SENSOR_MODE_NORMAL;
+
+	ret = ipu6_isys_fw_pins_prepare(stream, desc, ipu6_isys_fw_pin_cfg,
+					stream_cfg);
+	if (ret)
+		return ret;
+
+	ipu6_fw_isys_dump_stream_cfg(dev, msg);
+
+	stream->nr_output_pins = stream_cfg->nof_output_pins;
+
+	return 0;
+}
+
+static int ipu6_fw_isys_stream_open(struct ipu6_isys *isys,
+				    const unsigned int stream_handle,
+				    struct isys_fw_msgs *msg)
+{
+	return ipu6_fw_isys_complex_cmd(isys, stream_handle,
+				       &msg->ipu6.stream, msg->dma_addr,
+				       sizeof(msg->ipu6.stream),
+				       IPU6_FW_ISYS_SEND_TYPE_STREAM_OPEN);
+}
+
+static int ipu6_fw_isys_stream_close(struct ipu6_isys *isys,
+				     const unsigned int stream_handle)
+{
+	return ipu6_fw_isys_complex_cmd(isys, stream_handle, NULL, 0, 0,
+					IPU6_FW_ISYS_SEND_TYPE_STREAM_CLOSE);
+}
+
+static int ipu6_fw_isys_stream_flush(struct ipu6_isys *isys,
+				     const unsigned int stream_handle)
+{
+	return ipu6_fw_isys_complex_cmd(isys, stream_handle, NULL, 0, 0,
+					IPU6_FW_ISYS_SEND_TYPE_STREAM_FLUSH);
+}
+
+static int ipu6_fw_isys_stream_start(struct ipu6_isys *isys,
+				     const unsigned int stream_handle,
+				     struct isys_fw_msgs *msg, bool capture)
+{
+	u16 cmd_type;
+
+	if (capture)
+		cmd_type = IPU6_FW_ISYS_SEND_TYPE_STREAM_START_AND_CAPTURE;
+	else
+		cmd_type = IPU6_FW_ISYS_SEND_TYPE_STREAM_START;
+
+	return ipu6_fw_isys_complex_cmd(isys, stream_handle,
+					&msg->ipu6.stream, msg->dma_addr,
+					sizeof(msg->ipu6.stream), cmd_type);
+}
+
+static int ipu6_fw_isys_stream_capture(struct ipu6_isys *isys,
+				       const unsigned int stream_handle,
+				       struct isys_fw_msgs *msg)
+{
+	return ipu6_fw_isys_complex_cmd(isys, stream_handle,
+					&msg->ipu6.stream, msg->dma_addr,
+					sizeof(msg->ipu6.stream),
+					IPU6_FW_ISYS_SEND_TYPE_STREAM_CAPTURE);
+}
+
+static void
+ipu6_isys_buf_to_fw_frame_buf_pin(struct vb2_buffer *vb,
+				  struct ipu6_fw_isys_frame_buff_set_abi *set)
+{
+	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vvb = to_vb2_v4l2_buffer(vb);
+	struct ipu6_isys_video_buffer *ivb =
+		vb2_buffer_to_ipu6_isys_video_buffer(vvb);
+
+	set->output_pins[aq->fw_output].addr = ivb->dma_addr;
+	set->output_pins[aq->fw_output].out_buf_id = vb->index + 1;
+}
+
+/*
+ * Convert a buffer list to a isys fw ABI framebuffer set. The
+ * buffer list is not modified.
+ */
+#define IPU6_ISYS_FRAME_NUM_THRESHOLD  (30)
+static void ipu6_fw_isys_prepare_buf_set(struct isys_fw_msgs *msg,
+					 struct ipu6_isys_stream *stream,
+					 struct ipu6_isys_buffer_list *bl)
+{
+	struct ipu6_fw_isys_frame_buff_set_abi *set = &msg->ipu6.frame;
+	struct ipu6_isys_buffer *ib;
+
+	WARN_ON(!bl->nbufs);
+
+	set->send_irq_sof = 1;
+	set->send_resp_sof = 1;
+	set->send_irq_eof = 0;
+	set->send_resp_eof = 0;
+
+	set->send_irq_capture_ack = 1;
+	set->send_irq_capture_done = 0;
+
+	set->send_resp_capture_ack = 1;
+	set->send_resp_capture_done = 1;
+	if (atomic_read(&stream->sequence) >= IPU6_ISYS_FRAME_NUM_THRESHOLD) {
+		set->send_resp_capture_ack = 0;
+		set->send_resp_capture_done = 0;
+	}
+
+	list_for_each_entry(ib, &bl->head, head) {
+		struct vb2_buffer *vb = ipu6_isys_buffer_to_vb2_buffer(ib);
+
+		ipu6_isys_buf_to_fw_frame_buf_pin(vb, set);
+	}
+}
+
+const struct ipu6_fw_isys_ops ipu6_fw_isys_ops = {
+	.init = ipu6_fw_isys_init,
+	.close = ipu6_fw_isys_close,
+	.cleanup = ipu6_fw_isys_cleanup,
+	.prepare_stream_cfg = ipu6_fw_isys_prepare_stream_cfg,
+	.prepare_buf_set = ipu6_fw_isys_prepare_buf_set,
+	.stream_open = ipu6_fw_isys_stream_open,
+	.stream_start = ipu6_fw_isys_stream_start,
+	.stream_capture = ipu6_fw_isys_stream_capture,
+	.stream_flush = ipu6_fw_isys_stream_flush,
+	.stream_close = ipu6_fw_isys_stream_close,
+	.dump_stream_cfg = ipu6_fw_isys_dump_stream_cfg,
+	.dump_frame_buf_set = ipu6_fw_isys_dump_frame_buf_set,
+};
diff --git a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.h b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.h
index b60f02076d8a51..2a679d995961d9 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-fw-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-fw-isys.h
@@ -8,6 +8,10 @@
 
 struct device;
 struct ipu6_isys;
+struct ipu6_isys_video;
+struct isys_fw_msgs;
+struct ipu6_isys_stream;
+struct ipu6_isys_buffer_list;
 
 /* Max number of Input/Output Pins */
 #define IPU6_MAX_IPINS 4
@@ -511,21 +515,6 @@ struct ipu6_fw_isys_proxy_resp_info_abi {
 	struct ipu6_fw_isys_proxy_error_info_abi error_info;
 };
 
-/**
- * struct ipu6_fw_proxy_write_queue_token - ISYS proxy write queue token
- * @request_id: update id for the specific proxy write request
- * @region_index: Region id for the proxy write request
- * @offset: Offset of the write request according to the base address
- *	    of the region
- * @value: Value that is requested to be written with the proxy write request
- */
-struct ipu6_fw_proxy_write_queue_token {
-	u32 request_id;
-	u32 region_index;
-	u32 offset;
-	u32 value;
-};
-
 /**
  * struct ipu6_fw_resp_queue_token - ISYS response queue token
  * @resp_info: response info
@@ -570,27 +559,10 @@ struct ipu6_fw_proxy_send_queue_token {
 	u32 value;
 };
 
-void
-ipu6_fw_isys_dump_stream_cfg(struct device *dev,
-			     struct ipu6_fw_isys_stream_cfg_data_abi *cfg);
-void
-ipu6_fw_isys_dump_frame_buff_set(struct device *dev,
-				 struct ipu6_fw_isys_frame_buff_set_abi *buf,
-				 unsigned int outputs);
-int ipu6_fw_isys_init(struct ipu6_isys *isys, unsigned int num_streams);
-int ipu6_fw_isys_close(struct ipu6_isys *isys);
-int ipu6_fw_isys_simple_cmd(struct ipu6_isys *isys,
-			    const unsigned int stream_handle, u16 send_type);
-int ipu6_fw_isys_complex_cmd(struct ipu6_isys *isys,
-			     const unsigned int stream_handle,
-			     void *cpu_mapped_buf, dma_addr_t dma_mapped_buf,
-			     size_t size, u16 send_type);
-int ipu6_fw_isys_send_proxy_token(struct ipu6_isys *isys,
-				  unsigned int req_id,
-				  unsigned int index,
-				  unsigned int offset, u32 value);
-void ipu6_fw_isys_cleanup(struct ipu6_isys *isys);
-struct ipu6_fw_isys_resp_info_abi *
-ipu6_fw_isys_get_resp(void *context, unsigned int queue);
-void ipu6_fw_isys_put_resp(void *context, unsigned int queue);
+int ipu6_fw_isys_send_proxy_token(struct ipu6_isys *isys, unsigned int req_id,
+				  unsigned int index, unsigned int offset,
+				  u32 value);
+int ipu6_isys_isr_one(struct ipu6_bus_device *adev);
+irqreturn_t ipu6_isys_isr(struct ipu6_bus_device *adev);
+
 #endif
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
index 7e539a0c6c92f1..0db6971d8d3e8b 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.c
@@ -11,9 +11,12 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/minmax.h>
+#include <linux/pm_runtime.h>
 #include <linux/sprintf.h>
+#include <linux/string_choices.h>
 
 #include <media/media-entity.h>
+#include <media/mipi-csi2.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
@@ -23,7 +26,9 @@
 #include "ipu6-isys.h"
 #include "ipu6-isys-csi2.h"
 #include "ipu6-isys-subdev.h"
+#include "ipu6-isys-video.h"
 #include "ipu6-platform-isys-csi2-reg.h"
+#include "ipu7-isys-csi2-regs.h"
 
 static const u32 csi2_supported_codes[] = {
 	MEDIA_BUS_FMT_RGB565_1X16,
@@ -228,56 +233,124 @@ void ipu6_isys_csi2_error(struct ipu6_isys_csi2 *csi2)
 	}
 }
 
-static int ipu6_isys_csi2_set_stream(struct v4l2_subdev *sd,
-				     const struct ipu6_isys_csi2_timing *timing,
-				     unsigned int nlanes, int enable)
+static void ipu6_isys_csi2_setup_watermark(struct ipu6_isys_csi2 *csi2,
+					   struct v4l2_subdev_state *csi2_state,
+					   struct v4l2_subdev *remote_sd)
+{
+	struct device *dev = &csi2->isys->adev->auxdev.dev;
+	struct v4l2_control hb = { .id = V4L2_CID_HBLANK, .value = 0 };
+	struct v4l2_subdev_route *route;
+	u32 max_stream_data_rate = 0, hblank = 0;
+	s64 link_freq;
+	int ret;
+
+	if (IS_IPU7(csi2->asd.isys->adev->isp))
+		return;
+
+	ret = v4l2_g_ctrl(remote_sd->ctrl_handler, &hb);
+	if (!ret)
+		hblank = max(0, hb.value);
+
+	link_freq = ipu6_isys_csi2_get_link_freq(csi2);
+	if (link_freq <= 0) {
+		csi2->watermark.force_iwake_disable = true;
+		dev_warn(dev, "unexpected link_freq %lld (source %s)\n",
+			 link_freq, remote_sd->entity.name);
+		ipu6_isys_update_watermark_setting(csi2->isys);
+		return;
+	}
+
+	for_each_active_route(&csi2_state->routing, route) {
+		struct v4l2_mbus_framefmt *fmt;
+
+		fmt = v4l2_subdev_state_get_format(csi2_state, CSI2_PAD_SINK,
+						   route->sink_stream);
+		if (WARN_ON(!fmt))
+			continue;
+
+		u32 bpp = ipu6_isys_mbus_code_to_bpp(fmt->code);
+		u64 pixel_rate = mul_u64_u32_div(link_freq, csi2->nlanes * 2,
+						 bpp);
+		u32 pixels_per_line = fmt->width + hblank;
+		u64 line_time_ns = div_u64(pixels_per_line * NSEC_PER_SEC,
+					   pixel_rate);
+		u32 bytes_per_line = fmt->width * bpp / 8;
+		u32 pages_per_line =
+			DIV_ROUND_UP(bytes_per_line,
+				     csi2->isys->pdata->ipdata->sram_gran_size);
+		u32 pb_bytes_per_line =
+			pages_per_line << csi2->isys->pdata->ipdata->sram_gran_shift;
+		u64 stream_data_rate =
+			div64_u64(pb_bytes_per_line * 1000, line_time_ns);
+
+		dev_dbg(dev, "stream %u:%u -> %u:%u data rate %lld\n",
+			route->sink_pad, route->sink_stream, route->source_pad,
+			route->source_stream, stream_data_rate);
+
+		max_stream_data_rate = max(max_stream_data_rate,
+					   stream_data_rate);
+	}
+
+	csi2->watermark.stream_data_rate = max_stream_data_rate;
+
+	ipu6_isys_update_watermark_setting(csi2->isys);
+}
+
+static void ipu6_isys_csi2_clear_watermark(struct ipu6_isys_csi2 *csi2)
+{
+	if (IS_IPU7(csi2->asd.isys->adev->isp))
+		return;
+
+	csi2->watermark.force_iwake_disable = false;
+	csi2->watermark.stream_data_rate = 0;
+	ipu6_isys_update_watermark_setting(csi2->isys);
+}
+
+static void ipu6_isys_csi2_stream_disable(struct ipu6_isys_csi2 *csi2)
 {
-	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(sd);
-	struct ipu6_isys_csi2 *csi2 = to_ipu6_isys_csi2(asd);
 	struct ipu6_isys *isys = csi2->isys;
-	struct device *dev = &isys->adev->auxdev.dev;
 	struct ipu6_isys_csi2_config cfg;
-	unsigned int nports;
-	int ret = 0;
-	u32 mask = 0;
-	u32 i;
-
-	dev_dbg(dev, "stream %s CSI2-%u with %u lanes\n", enable ? "on" : "off",
-		csi2->port, nlanes);
+	u32 mask = isys->pdata->ipdata->csi2.irq_mask;
 
 	cfg.port = csi2->port;
-	cfg.nlanes = nlanes;
+	cfg.nlanes = csi2->nlanes;
 
-	mask = isys->pdata->ipdata->csi2.irq_mask;
-	nports = isys->pdata->ipdata->csi2.nports;
+	writel(0, csi2->base + CSI_REG_CSI_FE_ENABLE);
+	writel(0, csi2->base + CSI_REG_PPI2CSI_ENABLE);
+	writel(0, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI +
+	       CSI_PORT_REG_BASE_IRQ_ENABLE_OFFSET);
+	writel(mask, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI +
+	       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
+	writel(0, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
+	       CSI_PORT_REG_BASE_IRQ_ENABLE_OFFSET);
+	writel(0xffffffff, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
+	       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
 
-	if (!enable) {
-		writel(0, csi2->base + CSI_REG_CSI_FE_ENABLE);
-		writel(0, csi2->base + CSI_REG_PPI2CSI_ENABLE);
-
-		writel(0,
-		       csi2->base + CSI_PORT_REG_BASE_IRQ_CSI +
-		       CSI_PORT_REG_BASE_IRQ_ENABLE_OFFSET);
-		writel(mask,
-		       csi2->base + CSI_PORT_REG_BASE_IRQ_CSI +
-		       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
-		writel(0,
-		       csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
-		       CSI_PORT_REG_BASE_IRQ_ENABLE_OFFSET);
-		writel(0xffffffff,
-		       csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
-		       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
-
-		isys->phy_set_power(isys, &cfg, timing, false);
-
-		writel(0, isys->pdata->base + CSI_REG_HUB_FW_ACCESS_PORT
-		       (isys->pdata->ipdata->csi2.fw_access_port_ofs,
-			csi2->port));
-		writel(0, isys->pdata->base +
-		       CSI_REG_HUB_DRV_ACCESS_PORT(csi2->port));
+	isys->phy_set_power(isys, &cfg, NULL, false);
 
+	writel(0, isys->pdata->base + CSI_REG_HUB_FW_ACCESS_PORT
+	       (isys->pdata->ipdata->csi2.fw_access_port_ofs, csi2->port));
+	writel(0, isys->pdata->base + CSI_REG_HUB_DRV_ACCESS_PORT(csi2->port));
+}
+
+static int ipu6_isys_csi2_stream_enable(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	struct ipu6_isys_csi2_timing timing = { };
+	struct ipu6_isys_csi2_config cfg;
+	unsigned int nports;
+	u32 mask;
+	int ret;
+
+	cfg.port = csi2->port;
+	cfg.nlanes = csi2->nlanes;
+
+	ret = ipu6_isys_csi2_calc_timing(csi2, &timing, CSI2_ACCINV);
+	if (ret)
 		return ret;
-	}
+
+	mask = isys->pdata->ipdata->csi2.irq_mask;
+	nports = isys->pdata->ipdata->csi2.nports;
 
 	/* reset port reset */
 	writel(0x1, csi2->base + CSI_REG_PORT_GPREG_SRST);
@@ -285,7 +358,7 @@ static int ipu6_isys_csi2_set_stream(struct v4l2_subdev *sd,
 	writel(0x0, csi2->base + CSI_REG_PORT_GPREG_SRST);
 
 	/* enable port clock */
-	for (i = 0; i < nports; i++) {
+	for (unsigned int i = 0; i < nports; i++) {
 		writel(1, isys->pdata->base + CSI_REG_HUB_DRV_ACCESS_PORT(i));
 		writel(1, isys->pdata->base + CSI_REG_HUB_FW_ACCESS_PORT
 		       (isys->pdata->ipdata->csi2.fw_access_port_ofs, i));
@@ -329,18 +402,217 @@ static int ipu6_isys_csi2_set_stream(struct v4l2_subdev *sd,
 	writel(CSI_SENSOR_INPUT, csi2->base + CSI_REG_CSI_FE_MUX_CTRL);
 	writel(CSI_CNTR_SENSOR_LINE_ID | CSI_CNTR_SENSOR_FRAME_ID,
 	       csi2->base + CSI_REG_CSI_FE_SYNC_CNTR_SEL);
-	writel(FIELD_PREP(PPI_INTF_CONFIG_NOF_ENABLED_DLANES_MASK, nlanes - 1),
+	writel(FIELD_PREP(PPI_INTF_CONFIG_NOF_ENABLED_DLANES_MASK,
+			  csi2->nlanes - 1),
 	       csi2->base + CSI_REG_PPI2CSI_CONFIG_PPI_INTF);
 
 	writel(1, csi2->base + CSI_REG_PPI2CSI_ENABLE);
 	writel(1, csi2->base + CSI_REG_CSI_FE_ENABLE);
 
-	ret = isys->phy_set_power(isys, &cfg, timing, true);
+	return isys->phy_set_power(isys, &cfg, &timing, true);
+}
+
+static void ipu7_csi2_irq_enable(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	unsigned int offset, mask;
+
+	/* enable CSI2 legacy error irq */
+	offset = IPU7_IS_IO_CSI2_ERR_LEGACY_IRQ_CTL_BASE(csi2->port);
+	mask = IPU7_CSI_RX_ERROR_IRQ_MASK;
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_CLEAR);
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_MASK);
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_ENABLE);
+
+	/* enable CSI2 legacy sync irq */
+	offset = IPU7_IS_IO_CSI2_SYNC_LEGACY_IRQ_CTL_BASE(csi2->port);
+	mask = IPU7_CSI_RX_SYNC_IRQ_MASK;
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_CLEAR);
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_MASK);
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_ENABLE);
+
+	if (!IS_IPU7_MTL(isys->adev->isp)) {
+		mask = IPU7P5_CSI_RX_SYNC_FE_IRQ_MASK;
+		writel(mask, csi2->base + offset + IPU7_IRQ1_CTL_CLEAR);
+		writel(mask, csi2->base + offset + IPU7_IRQ1_CTL_MASK);
+		writel(mask, csi2->base + offset + IPU7_IRQ1_CTL_ENABLE);
+	}
+}
+
+static void ipu7_csi2_irq_disable(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	unsigned int offset, mask;
+
+	/* disable CSI2 legacy error irq */
+	offset = IPU7_IS_IO_CSI2_ERR_LEGACY_IRQ_CTL_BASE(csi2->port);
+	mask = IPU7_CSI_RX_ERROR_IRQ_MASK;
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_CLEAR);
+	writel(0, csi2->base + offset + IPU7_IRQ_CTL_MASK);
+	writel(0, csi2->base + offset + IPU7_IRQ_CTL_ENABLE);
+
+	/* disable CSI2 legacy sync irq */
+	offset = IPU7_IS_IO_CSI2_SYNC_LEGACY_IRQ_CTL_BASE(csi2->port);
+	mask = IPU7_CSI_RX_SYNC_IRQ_MASK;
+	writel(mask, csi2->base + offset + IPU7_IRQ_CTL_CLEAR);
+	writel(0, csi2->base + offset + IPU7_IRQ_CTL_MASK);
+	writel(0, csi2->base + offset + IPU7_IRQ_CTL_ENABLE);
+
+	if (!IS_IPU7_MTL(isys->adev->isp)) {
+		writel(mask, csi2->base + offset + IPU7_IRQ1_CTL_CLEAR);
+		writel(0, csi2->base + offset + IPU7_IRQ1_CTL_MASK);
+		writel(0, csi2->base + offset + IPU7_IRQ1_CTL_ENABLE);
+	}
+}
+
+static void ipu7_isys_csi2_stream_disable(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	void __iomem *base = isys->pdata->base;
+	struct ipu6_isys_csi2_config cfg;
+
+	cfg.port = csi2->port;
+	cfg.nlanes = csi2->nlanes;
+
+	isys->phy_set_power(isys, &cfg, NULL, false);
+
+	writel(0x4,
+	       base + IPU7_IS_IO_GPREGS_BASE + IPU7_CLK_DIV_FACTOR_APB_CLK);
+	ipu7_csi2_irq_disable(csi2);
+}
+
+static int ipu7_isys_csi2_stream_enable(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	void __iomem *base = isys->pdata->base;
+	struct ipu6_isys_csi2_config cfg;
+	unsigned int offset = IPU7_IS_IO_GPREGS_BASE;
+	int ret;
+
+	writel(0x2, base + offset + IPU7_CLK_DIV_FACTOR_APB_CLK);
+	if (csi2->port == 0U && csi2->nlanes == 4U &&
+	    !IS_IPU7_MTL(isys->adev->isp))
+		writel(0x1, base + offset + IPU7_CSI_PORTAB_AGGREGATION);
+
+	/* input is coming from CSI receiver (sensor) */
+	offset = IPU7_IS_IO_CSI2_ADPL_PORT_BASE(csi2->port);
+	writel(CSI_SENSOR_INPUT, base + offset + IPU7_CSI2_ADPL_INPUT_MODE);
+	writel(1, base + offset + IPU7_CSI2_ADPL_CSI_RX_ERR_IRQ_CLEAR_EN);
+
+	cfg.port = csi2->port;
+	cfg.nlanes = csi2->nlanes;
+
+	ret = isys->phy_set_power(isys, &cfg, NULL, true);
 	if (ret)
-		dev_err(dev, "csi-%d phy power up failed %d\n", csi2->port,
-			ret);
+		return ret;
 
-	return ret;
+	ipu7_csi2_irq_enable(csi2);
+
+	return 0;
+}
+
+static int ipu6_isys_csi2_streaming_change(struct ipu6_isys_subdev *asd,
+					   struct v4l2_subdev_state *state,
+					   u32 pad,
+					   struct v4l2_mbus_frame_desc *desc,
+					   u8 *vc, bool enable)
+{
+	struct v4l2_mbus_frame_desc_entry *this_entry = NULL;
+	struct v4l2_subdev_route *route, *this_route = NULL;
+	u32 streams_enabled = 0, nodes_streaming = 0;
+
+	for_each_active_route(&state->routing, this_route)
+		if (pad == this_route->source_pad)
+			break;
+	if (!this_route) {
+		dev_dbg(asd->sd.dev, "no route found for pad %u\n", pad);
+		return -EINVAL;
+	}
+
+	for (unsigned int i = 0; i < desc->num_entries; i++) {
+		if (desc->entry[i].stream == this_route->sink_stream) {
+			this_entry = &desc->entry[i];
+			break;
+		}
+	}
+	if (!this_entry) {
+		dev_dbg(asd->sd.dev,
+			"no frame descriptor entry found for stream %u\n",
+			this_route->sink_stream);
+		return -EINVAL;
+	}
+
+	for_each_active_route(&state->routing, route) {
+		struct v4l2_mbus_frame_desc_entry *entry = NULL;
+
+		for (unsigned int i = 0; i < desc->num_entries; i++) {
+			if (desc->entry[i].stream == route->sink_stream) {
+				entry = &desc->entry[i];
+				break;
+			}
+		}
+
+		if (!entry) {
+			dev_dbg(asd->sd.dev, "cannot find stream %u from frame descriptor\n",
+				route->sink_stream);
+			return -EINVAL;
+		}
+
+		if (entry->bus.csi2.vc != this_entry->bus.csi2.vc)
+			continue;
+
+		struct media_pad *video_pad =
+			media_pad_remote_pad_first(&asd->sd.entity.pads[route->source_pad]);
+		if (!video_pad)
+			return -EINVAL;
+
+		struct ipu6_isys_video *av =
+			container_of_const(video_pad, struct ipu6_isys_video,
+					   pad);
+
+		streams_enabled++;
+		if (av->streaming || (enable && pad == route->source_pad))
+			nodes_streaming++;
+	}
+
+	if (vc)
+		*vc = this_entry->bus.csi2.vc;
+
+	if (streams_enabled == nodes_streaming) {
+		dev_dbg(asd->sd.dev,
+			"changing streaming state to %s on \"%s\"\n",
+			str_enabled_disabled(enable), asd->sd.entity.name);
+		return 1;
+	}
+
+	dev_dbg(asd->sd.dev, "not setting streaming %s %s (%u/%u)\n",
+		str_enabled_disabled(enable), enable ? "yet" : "anymore",
+		nodes_streaming, streams_enabled);
+
+	return 0;
+}
+
+static int ipu6_isys_get_frame_desc(struct v4l2_subdev *remote_sd,
+				    unsigned int pad,
+				    struct v4l2_mbus_frame_desc *desc)
+{
+	struct v4l2_subdev_format fmt = { .which = V4L2_SUBDEV_FORMAT_ACTIVE };
+	int ret;
+
+	ret = v4l2_subdev_call(remote_sd, pad, get_frame_desc, pad, desc);
+	if (!ret || ret != -ENOIOCTLCMD)
+		return ret;
+
+	ret = v4l2_subdev_call_state_active(remote_sd, pad, get_fmt, &fmt);
+	if (ret)
+		return ret;
+
+	desc->type = V4L2_MBUS_FRAME_DESC_TYPE_CSI2;
+	desc->num_entries = 1;
+	desc->entry[0].pixelcode = fmt.format.code;
+	desc->entry[0].bus.csi2.dt = ipu6_isys_mbus_code_to_mipi(fmt.format.code);
+
+	return 0;
 }
 
 static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
@@ -349,60 +621,174 @@ static int ipu6_isys_csi2_enable_streams(struct v4l2_subdev *sd,
 {
 	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(sd);
 	struct ipu6_isys_csi2 *csi2 = to_ipu6_isys_csi2(asd);
-	struct ipu6_isys_csi2_timing timing = { };
-	struct v4l2_subdev *remote_sd;
-	struct media_pad *remote_pad;
+	struct ipu6_device *isp = asd->isys->adev->isp;
+	struct media_pad *remote_pad =
+		media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]),
+		*vdev_pad = media_pad_remote_pad_unique(&sd->entity.pads[pad]);
+	struct ipu6_isys_video *av =
+		container_of_const(vdev_pad, struct ipu6_isys_video, pad);
+	struct v4l2_subdev *remote_sd =
+		media_entity_to_v4l2_subdev(remote_pad->entity);
+	struct v4l2_mbus_frame_desc desc = { 0 };
+	struct ipu6_isys_stream *stream;
+	struct ipu6_isys_buffer_list bl;
 	u64 sink_streams;
 	int ret;
+	u8 vc;
 
-	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
-	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
+	lockdep_assert_held(&csi2->isys->stream_mutex);
+
+	ret = ipu6_isys_get_frame_desc(remote_sd, remote_pad->index, &desc);
+	if (ret)
+		return ret;
+
+	list_add(&av->csi2_entry, &csi2->av_head);
 
 	sink_streams =
 		v4l2_subdev_state_xlate_streams(state, pad, CSI2_PAD_SINK,
 						&streams_mask);
+	csi2->stream_ids |= sink_streams;
 
-	ret = ipu6_isys_csi2_calc_timing(csi2, &timing, CSI2_ACCINV);
-	if (ret)
+	ret = ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, &vc,
+					      true);
+	if (!ret)
 		return ret;
+	if (ret < 0)
+		goto err_av_del;
+
+	ret = pm_runtime_resume_and_get(sd->dev);
+	if (ret < 0)
+		goto err_del_av;
 
-	ret = ipu6_isys_csi2_set_stream(sd, &timing, csi2->nlanes, true);
+	ipu6_isys_csi2_setup_watermark(csi2, state, remote_sd);
+
+	stream = ipu6_isys_alloc_stream_firmware(csi2, state, &desc, vc);
+	if (IS_ERR(stream)) {
+		ret = PTR_ERR(stream);
+		dev_err(sd->dev, "allocating firmware stream failed\n");
+		goto err_clear_watermark;
+	}
+
+	ret = ipu6_isys_buffer_list_get(stream, &bl);
+	if (ret < 0) {
+		dev_warn(sd->dev, "no buffer available, DRIVER BUG?\n");
+		goto err_free_stream_firmware;
+	}
+
+	ret = ipu6_isys_start_stream_firmware(stream, &bl, &desc);
 	if (ret)
-		return ret;
+		goto err_requeue_buffers;
 
-	ret = v4l2_subdev_enable_streams(remote_sd, remote_pad->index,
-					 sink_streams);
-	if (ret) {
-		ipu6_isys_csi2_set_stream(sd, NULL, 0, false);
-		return ret;
+	if (!csi2->streaming_vc) {
+		ret = IS_IPU7(isp) ? ipu7_isys_csi2_stream_enable(csi2) :
+				     ipu6_isys_csi2_stream_enable(csi2);
+		if (ret)
+			goto err_stop_stream_firmware;
 	}
 
+	ret = v4l2_subdev_enable_streams(remote_sd, remote_pad->index,
+					 csi2->stream_ids);
+	if (ret)
+		goto err_stop_stream_csi2;
+
+	csi2->streaming_vc |= BIT(vc);
+
 	return 0;
+
+err_stop_stream_csi2:
+	if (IS_IPU7(isp))
+		ipu7_isys_csi2_stream_disable(csi2);
+	else
+		ipu6_isys_csi2_stream_disable(csi2);
+
+err_requeue_buffers:
+	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_INCOMING, 0);
+
+err_stop_stream_firmware:
+	ipu6_isys_stop_stream_firmware(stream);
+	ipu6_isys_close_stream_firmware(stream);
+
+err_free_stream_firmware:
+	ipu6_isys_free_stream_firmware(stream);
+
+err_clear_watermark:
+	ipu6_isys_csi2_clear_watermark(csi2);
+	pm_runtime_put(sd->dev);
+
+err_del_av:
+	ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, NULL, false);
+	csi2->stream_ids &= ~sink_streams;
+err_av_del:
+	list_del(&av->csi2_entry);
+
+	return ret;
 }
 
 static int ipu6_isys_csi2_disable_streams(struct v4l2_subdev *sd,
 					  struct v4l2_subdev_state *state,
 					  u32 pad, u64 streams_mask)
 {
-	struct v4l2_subdev *remote_sd;
-	struct media_pad *remote_pad;
+	struct media_pad *remote_pad =
+		media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]),
+		*vdev_pad = media_pad_remote_pad_unique(&sd->entity.pads[pad]);
+	struct ipu6_isys_video *av =
+		container_of_const(vdev_pad, struct ipu6_isys_video, pad);
+	struct v4l2_subdev *remote_sd =
+		media_entity_to_v4l2_subdev(remote_pad->entity);
+	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(sd);
+	struct ipu6_isys_csi2 *csi2 = to_ipu6_isys_csi2(asd);
+	struct ipu6_device *isp = asd->isys->adev->isp;
+	struct v4l2_mbus_frame_desc desc = { 0 };
 	u64 sink_streams;
+	int ret;
+	u8 vc;
+
+	lockdep_assert_held(&csi2->isys->stream_mutex);
+
+	ret = ipu6_isys_get_frame_desc(remote_sd, remote_pad->index, &desc);
+	if (ret)
+		return ret;
 
 	sink_streams =
 		v4l2_subdev_state_xlate_streams(state, pad, CSI2_PAD_SINK,
 						&streams_mask);
 
-	remote_pad = media_pad_remote_pad_first(&sd->entity.pads[CSI2_PAD_SINK]);
-	remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity);
+	csi2->stream_ids &= ~sink_streams;
+
+	ret = ipu6_isys_csi2_streaming_change(asd, state, pad, &desc, &vc,
+					      false);
+	if (ret <= 0)
+		goto out_del_csi2_entry;
+
+	csi2->streaming_vc &= ~BIT(vc);
+
+	struct ipu6_isys_stream *stream =
+		ipu6_isys_find_stream_firmware(csi2, vc);
+	ipu6_isys_stop_stream_firmware(stream);
 
-	ipu6_isys_csi2_set_stream(sd, NULL, 0, false);
+	if IS_IPU7(isp)
+		ipu7_isys_csi2_stream_disable(csi2);
+	else
+		ipu6_isys_csi2_stream_disable(csi2);
 
-	v4l2_subdev_disable_streams(remote_sd, remote_pad->index, sink_streams);
+	v4l2_subdev_disable_streams(remote_sd, remote_pad->index,
+				    csi2->stream_ids | sink_streams);
+
+	ipu6_isys_close_stream_firmware(stream);
+	ipu6_isys_free_stream_firmware(stream);
+
+	ipu6_isys_csi2_clear_watermark(csi2);
+
+	pm_runtime_put(sd->dev);
+
+out_del_csi2_entry:
+	list_del(&av->csi2_entry);
 
 	return 0;
 }
 
 static int ipu6_isys_csi2_set_sel(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *state,
 				  struct v4l2_subdev_selection *sel)
 {
@@ -454,6 +840,7 @@ static int ipu6_isys_csi2_set_sel(struct v4l2_subdev *sd,
 }
 
 static int ipu6_isys_csi2_get_sel(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *state,
 				  struct v4l2_subdev_selection *sel)
 {
@@ -542,6 +929,8 @@ int ipu6_isys_csi2_init(struct ipu6_isys_csi2 *csi2,
 	if (ret)
 		goto fail;
 
+	INIT_LIST_HEAD(&csi2->av_head);
+	INIT_LIST_HEAD(&csi2->streams);
 	csi2->asd.source = IPU6_FW_ISYS_STREAM_SRC_CSI2_PORT0 + index;
 	csi2->asd.supported_codes = csi2_supported_codes;
 	snprintf(csi2->asd.sd.name, sizeof(csi2->asd.sd.name),
@@ -592,56 +981,3 @@ void ipu6_isys_csi2_eof_event_by_stream(struct ipu6_isys_stream *stream)
 	dev_dbg(dev, "eof_event::csi2-%i sequence: %i\n",
 		csi2->port, frame_sequence);
 }
-
-int ipu6_isys_csi2_get_remote_desc(u32 source_stream,
-				   struct ipu6_isys_csi2 *csi2,
-				   struct media_entity *source_entity,
-				   struct v4l2_mbus_frame_desc_entry *entry)
-{
-	struct v4l2_mbus_frame_desc_entry *desc_entry = NULL;
-	struct device *dev = &csi2->isys->adev->auxdev.dev;
-	struct v4l2_mbus_frame_desc desc;
-	struct v4l2_subdev *source;
-	struct media_pad *pad;
-	unsigned int i;
-	int ret;
-
-	source = media_entity_to_v4l2_subdev(source_entity);
-	if (!source)
-		return -EPIPE;
-
-	pad = media_pad_remote_pad_first(&csi2->asd.pad[CSI2_PAD_SINK]);
-	if (!pad)
-		return -EPIPE;
-
-	ret = v4l2_subdev_call(source, pad, get_frame_desc, pad->index, &desc);
-	if (ret)
-		return ret;
-
-	if (desc.type != V4L2_MBUS_FRAME_DESC_TYPE_CSI2) {
-		dev_err(dev, "Unsupported frame descriptor type\n");
-		return -EINVAL;
-	}
-
-	for (i = 0; i < desc.num_entries; i++) {
-		if (source_stream == desc.entry[i].stream) {
-			desc_entry = &desc.entry[i];
-			break;
-		}
-	}
-
-	if (!desc_entry) {
-		dev_err(dev, "Failed to find stream %u from remote subdev\n",
-			source_stream);
-		return -EINVAL;
-	}
-
-	if (desc_entry->bus.csi2.vc >= NR_OF_CSI2_VC) {
-		dev_err(dev, "invalid vc %d\n", desc_entry->bus.csi2.vc);
-		return -EINVAL;
-	}
-
-	*entry = *desc_entry;
-
-	return 0;
-}
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
index ce8eed91065c73..1620ac16f90dee 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-csi2.h
@@ -16,6 +16,9 @@ struct ipu6_isys_video;
 struct ipu6_isys;
 struct ipu6_isys_stream;
 
+#define PHY_MODE_DPHY		0
+#define PHY_MODE_CPHY		1
+
 #define NR_OF_CSI2_VC		16
 #define INVALID_VC_ID		-1
 #define NR_OF_CSI2_SINK_PADS	1
@@ -38,11 +41,22 @@ struct ipu6_isys_csi2 {
 	struct ipu6_isys_subdev asd;
 	struct ipu6_isys *isys;
 	struct ipu6_isys_video av[NR_OF_CSI2_SRC_PADS];
+	struct list_head av_head;
+	struct list_head streams;
+	struct ipu6_isys_stream *streams_by_vc[NR_OF_CSI2_VC];
 
 	void __iomem *base;
 	u32 receiver_errors;
 	unsigned int nlanes;
 	unsigned int port;
+	u32 legacy_irq_mask;
+	unsigned int phy_mode;
+	struct {
+		u32 stream_data_rate;
+		bool force_iwake_disable;
+	} watermark;
+	u32 streaming_vc;
+	u64 stream_ids;
 };
 
 struct ipu6_isys_csi2_timing {
@@ -70,9 +84,5 @@ void ipu6_isys_csi2_sof_event_by_stream(struct ipu6_isys_stream *stream);
 void ipu6_isys_csi2_eof_event_by_stream(struct ipu6_isys_stream *stream);
 void ipu6_isys_register_errors(struct ipu6_isys_csi2 *csi2);
 void ipu6_isys_csi2_error(struct ipu6_isys_csi2 *csi2);
-int ipu6_isys_csi2_get_remote_desc(u32 source_stream,
-				   struct ipu6_isys_csi2 *csi2,
-				   struct media_entity *source_entity,
-				   struct v4l2_mbus_frame_desc_entry *entry);
 
 #endif /* IPU6_ISYS_CSI2_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
index fabaed63df0cea..2608e3b91acbec 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.c
@@ -21,6 +21,7 @@
 #include "ipu6-dma.h"
 #include "ipu6-fw-isys.h"
 #include "ipu6-isys.h"
+#include "ipu6-isys-queue.h"
 #include "ipu6-isys-video.h"
 
 static int ipu6_isys_buf_init(struct vb2_buffer *vb)
@@ -191,8 +192,8 @@ static void flush_firmware_streamon_fail(struct ipu6_isys_stream *stream)
  * that contains one entry from each video buffer queue. If a buffer can't be
  * obtained from every queue, the buffers are returned back to the queue.
  */
-static int buffer_list_get(struct ipu6_isys_stream *stream,
-			   struct ipu6_isys_buffer_list *bl)
+int ipu6_isys_buffer_list_get(struct ipu6_isys_stream *stream,
+			      struct ipu6_isys_buffer_list *bl)
 {
 	struct device *dev = &stream->isys->adev->auxdev.dev;
 	struct ipu6_isys_queue *aq;
@@ -233,106 +234,53 @@ static int buffer_list_get(struct ipu6_isys_stream *stream,
 	return 0;
 }
 
-static void
-ipu6_isys_buf_to_fw_frame_buf_pin(struct vb2_buffer *vb,
-				  struct ipu6_fw_isys_frame_buff_set_abi *set)
-{
-	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(vb->vb2_queue);
-	struct vb2_v4l2_buffer *vvb = to_vb2_v4l2_buffer(vb);
-	struct ipu6_isys_video_buffer *ivb =
-		vb2_buffer_to_ipu6_isys_video_buffer(vvb);
-
-	set->output_pins[aq->fw_output].addr = ivb->dma_addr;
-	set->output_pins[aq->fw_output].out_buf_id = vb->index + 1;
-}
-
-/*
- * Convert a buffer list to a isys fw ABI framebuffer set. The
- * buffer list is not modified.
- */
-#define IPU6_ISYS_FRAME_NUM_THRESHOLD  (30)
-void
-ipu6_isys_buf_to_fw_frame_buf(struct ipu6_fw_isys_frame_buff_set_abi *set,
-			      struct ipu6_isys_stream *stream,
-			      struct ipu6_isys_buffer_list *bl)
-{
-	struct ipu6_isys_buffer *ib;
-
-	WARN_ON(!bl->nbufs);
-
-	set->send_irq_sof = 1;
-	set->send_resp_sof = 1;
-	set->send_irq_eof = 0;
-	set->send_resp_eof = 0;
-
-	if (stream->streaming)
-		set->send_irq_capture_ack = 0;
-	else
-		set->send_irq_capture_ack = 1;
-	set->send_irq_capture_done = 0;
-
-	set->send_resp_capture_ack = 1;
-	set->send_resp_capture_done = 1;
-	if (atomic_read(&stream->sequence) >= IPU6_ISYS_FRAME_NUM_THRESHOLD) {
-		set->send_resp_capture_ack = 0;
-		set->send_resp_capture_done = 0;
-	}
-
-	list_for_each_entry(ib, &bl->head, head) {
-		struct vb2_buffer *vb = ipu6_isys_buffer_to_vb2_buffer(ib);
-
-		ipu6_isys_buf_to_fw_frame_buf_pin(vb, set);
-	}
-}
-
 /* Start streaming for real. The buffer list must be available. */
-static int ipu6_isys_stream_start(struct ipu6_isys_video *av,
-				  struct ipu6_isys_buffer_list *bl)
+static int ipu6_isys_stream_start(struct ipu6_isys_video *av)
 {
-	struct ipu6_isys_stream *stream = av->stream;
-	struct device *dev = &stream->isys->adev->auxdev.dev;
-	struct ipu6_isys_buffer_list __bl;
+	struct ipu6_bus_device *adev = av->isys->adev;
+	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
+	struct device *dev = &adev->auxdev.dev;
+	struct ipu6_isys_buffer_list bl;
+	struct isys_fw_msgs *msg;
 	int ret;
 
-	guard(mutex)(&stream->isys->stream_mutex);
-	ret = ipu6_isys_video_set_streaming(av, 1, bl);
+	guard(mutex)(&av->isys->stream_mutex);
+	ret = ipu6_isys_video_set_streaming(av, 1);
 	if (ret)
-		goto out_requeue;
+		return ret;
 
-	stream->streaming = 1;
+	if (!av->stream || !(BIT(av->stream->vc) & av->csi2->streaming_vc))
+		return 0;
+
+	struct ipu6_isys_stream *stream = av->stream;
 
-	bl = &__bl;
+	guard(mutex)(&stream->mutex);
 
 	do {
-		struct ipu6_fw_isys_frame_buff_set_abi *buf = NULL;
-		struct isys_fw_msgs *msg;
-		u16 send_type = IPU6_FW_ISYS_SEND_TYPE_STREAM_CAPTURE;
-
-		ret = buffer_list_get(stream, bl);
+		ret = ipu6_isys_buffer_list_get(stream, &bl);
 		if (ret < 0)
-			break;
+			return 0;
 
 		msg = ipu6_get_fw_msg_buf(stream);
-		if (!msg)
-			return -ENOMEM;
-
-		buf = &msg->fw_msg.frame;
-		ipu6_isys_buf_to_fw_frame_buf(buf, stream, bl);
-		ipu6_fw_isys_dump_frame_buff_set(dev, buf,
-						 stream->nr_output_pins);
-		ipu6_isys_buffer_list_queue(bl, IPU6_ISYS_BUFFER_LIST_FL_ACTIVE,
-					    0);
-		ret = ipu6_fw_isys_complex_cmd(stream->isys,
-					       stream->stream_handle, buf,
-					       msg->dma_addr, sizeof(*buf),
-					       send_type);
-	} while (!WARN_ON(ret));
+		if (WARN_ON(!msg))
+			goto out_requeue;
+
+		fw_ops->prepare_buf_set(msg, stream, &bl);
+		fw_ops->dump_frame_buf_set(dev, msg, stream->nr_output_pins);
+		ipu6_isys_buffer_list_queue(&bl,
+					    IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
+		ret = fw_ops->stream_capture(stream->isys,
+					     stream->stream_handle, msg);
+		if (WARN_ON(ret))
+			break;
+	} while (true);
 
-	return 0;
+	/* Error handling begins here. */
+	ipu6_put_fw_msg_buf(stream->isys, msg);
 
 out_requeue:
-	if (bl && bl->nbufs)
-		ipu6_isys_buffer_list_queue(bl,
+	if (bl.nbufs)
+		ipu6_isys_buffer_list_queue(&bl,
 					    IPU6_ISYS_BUFFER_LIST_FL_INCOMING,
 					    VB2_BUF_STATE_QUEUED);
 	flush_firmware_streamon_fail(stream);
@@ -344,13 +292,15 @@ static void buf_queue(struct vb2_buffer *vb)
 {
 	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(vb->vb2_queue);
 	struct ipu6_isys_video *av = ipu6_isys_queue_to_video(aq);
+	struct ipu6_bus_device *adev = av->isys->adev;
+	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
 	struct vb2_v4l2_buffer *vvb = to_vb2_v4l2_buffer(vb);
 	struct ipu6_isys_video_buffer *ivb =
 		vb2_buffer_to_ipu6_isys_video_buffer(vvb);
 	struct ipu6_isys_buffer *ib = &ivb->ib;
-	struct device *dev = &av->isys->adev->auxdev.dev;
-	struct ipu6_fw_isys_frame_buff_set_abi *buf = NULL;
+	struct device *dev = &adev->auxdev.dev;
 	struct ipu6_isys_stream *stream = av->stream;
+	struct ipu6_isys_csi2 *csi2;
 	struct ipu6_isys_buffer_list bl;
 	struct isys_fw_msgs *msg;
 	unsigned long flags;
@@ -374,7 +324,8 @@ static void buf_queue(struct vb2_buffer *vb)
 
 	mutex_lock(&stream->mutex);
 
-	if (stream->nr_streaming != stream->nr_queues) {
+	csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
+	if (!(BIT(stream->vc) & csi2->streaming_vc)) {
 		dev_dbg(dev, "not streaming yet, adding to incoming\n");
 		goto out;
 	}
@@ -384,7 +335,7 @@ static void buf_queue(struct vb2_buffer *vb)
 	 * (above). Let's see whether all queues in the pipeline would
 	 * have a buffer.
 	 */
-	ret = buffer_list_get(stream, &bl);
+	ret = ipu6_isys_buffer_list_get(stream, &bl);
 	if (ret < 0) {
 		dev_dbg(dev, "No buffers available\n");
 		goto out;
@@ -396,9 +347,8 @@ static void buf_queue(struct vb2_buffer *vb)
 		goto out;
 	}
 
-	buf = &msg->fw_msg.frame;
-	ipu6_isys_buf_to_fw_frame_buf(buf, stream, &bl);
-	ipu6_fw_isys_dump_frame_buff_set(dev, buf, stream->nr_output_pins);
+	fw_ops->prepare_buf_set(msg, stream, &bl);
+	fw_ops->dump_frame_buf_set(dev, msg, stream->nr_output_pins);
 
 	/*
 	 * We must queue the buffers in the buffer list to the
@@ -408,9 +358,7 @@ static void buf_queue(struct vb2_buffer *vb)
 	 */
 	ipu6_isys_buffer_list_queue(&bl, IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
 
-	ret = ipu6_fw_isys_complex_cmd(stream->isys, stream->stream_handle,
-				       buf, msg->dma_addr, sizeof(*buf),
-				       IPU6_FW_ISYS_SEND_TYPE_STREAM_CAPTURE);
+	ret = fw_ops->stream_capture(stream->isys, stream->stream_handle, msg);
 	if (ret < 0)
 		dev_err(dev, "send stream capture failed\n");
 
@@ -525,7 +473,6 @@ static void return_buffers(struct ipu6_isys_queue *aq,
 static void ipu6_isys_stream_cleanup(struct ipu6_isys_video *av)
 {
 	video_device_pipeline_stop(&av->vdev);
-	ipu6_isys_put_stream(av->stream);
 	av->stream = NULL;
 }
 
@@ -536,10 +483,8 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 	struct device *dev = &av->isys->adev->auxdev.dev;
 	const struct ipu6_isys_pixelformat *pfmt =
 		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	struct ipu6_isys_buffer_list __bl, *bl = NULL;
-	struct ipu6_isys_stream *stream;
 	struct media_pad *source_pad, *remote_pad;
-	int nr_queues, ret;
+	int ret;
 
 	dev_dbg(dev, "stream: %s: width %u, height %u, css pixelformat %u\n",
 		av->vdev.name, ipu6_isys_get_frame_width(av),
@@ -559,11 +504,9 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 		goto out_return_buffers;
 	}
 
-	ret = ipu6_isys_setup_video(av, remote_pad, source_pad, &nr_queues);
-	if (ret < 0) {
-		dev_dbg(dev, "failed to setup video\n");
+	ret = video_device_pipeline_alloc_start(&av->vdev);
+	if (ret < 0)
 		goto out_return_buffers;
-	}
 
 	ret = ipu6_isys_link_fmt_validate(aq);
 	if (ret) {
@@ -573,55 +516,12 @@ static int start_streaming(struct vb2_queue *q, unsigned int count)
 		goto out_pipeline_stop;
 	}
 
-	ret = ipu6_isys_fw_open(av->isys);
+	ret = ipu6_isys_stream_start(av);
 	if (ret)
 		goto out_pipeline_stop;
 
-	stream = av->stream;
-	mutex_lock(&stream->mutex);
-	if (!stream->nr_streaming) {
-		ret = ipu6_isys_video_prepare_stream(av, source_pad->entity,
-						     nr_queues);
-		if (ret)
-			goto out_fw_close;
-	}
-
-	stream->nr_streaming++;
-	dev_dbg(dev, "queue %u of %u\n", stream->nr_streaming,
-		stream->nr_queues);
-
-	list_add(&aq->node, &stream->queues);
-	ipu6_isys_configure_stream_watermark(av, source_pad->entity);
-	ipu6_isys_update_stream_watermark(av, true);
-
-	if (stream->nr_streaming != stream->nr_queues)
-		goto out;
-
-	bl = &__bl;
-	ret = buffer_list_get(stream, bl);
-	if (ret < 0) {
-		dev_warn(dev, "no buffer available, DRIVER BUG?\n");
-		goto out;
-	}
-
-	ret = ipu6_isys_stream_start(av, bl);
-	if (ret)
-		goto out_stream_start;
-
-out:
-	mutex_unlock(&stream->mutex);
-
 	return 0;
 
-out_stream_start:
-	ipu6_isys_update_stream_watermark(av, false);
-	list_del(&aq->node);
-	stream->nr_streaming--;
-
-out_fw_close:
-	mutex_unlock(&stream->mutex);
-	ipu6_isys_fw_close(av->isys);
-
 out_pipeline_stop:
 	ipu6_isys_stream_cleanup(av);
 
@@ -635,27 +535,14 @@ static void stop_streaming(struct vb2_queue *q)
 {
 	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(q);
 	struct ipu6_isys_video *av = ipu6_isys_queue_to_video(aq);
-	struct ipu6_isys_stream *stream = av->stream;
-
-	mutex_lock(&stream->mutex);
-
-	ipu6_isys_update_stream_watermark(av, false);
 
 	mutex_lock(&av->isys->stream_mutex);
-	if (stream->nr_streaming == stream->nr_queues && stream->streaming)
-		ipu6_isys_video_set_streaming(av, 0, NULL);
-	list_del(&aq->node);
+	ipu6_isys_video_set_streaming(av, 0);
 	mutex_unlock(&av->isys->stream_mutex);
 
-	stream->nr_streaming--;
-	stream->streaming = 0;
-	mutex_unlock(&stream->mutex);
-
 	ipu6_isys_stream_cleanup(av);
 
 	return_buffers(aq, VB2_BUF_STATE_ERROR);
-
-	ipu6_isys_fw_close(av->isys);
 }
 
 static unsigned int
@@ -741,7 +628,7 @@ static void ipu6_isys_queue_buf_done(struct ipu6_isys_buffer *ib)
 	}
 }
 
-static void
+void
 ipu6_stream_buf_ready(struct ipu6_isys_stream *stream, u8 pin_id, u32 pin_addr,
 		      u64 time, bool error_check)
 {
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h
index dec1fed44dd245..0e0886f5915006 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-queue.h
@@ -60,11 +60,11 @@ struct ipu6_isys_buffer_list {
 void ipu6_isys_buffer_list_queue(struct ipu6_isys_buffer_list *bl,
 				 unsigned long op_flags,
 				 enum vb2_buffer_state state);
-void
-ipu6_isys_buf_to_fw_frame_buf(struct ipu6_fw_isys_frame_buff_set_abi *set,
-			      struct ipu6_isys_stream *stream,
+int ipu6_isys_buffer_list_get(struct ipu6_isys_stream *stream,
 			      struct ipu6_isys_buffer_list *bl);
 void ipu6_isys_queue_buf_ready(struct ipu6_isys_stream *stream,
 			       struct ipu6_fw_isys_resp_info_abi *info);
 int ipu6_isys_queue_init(struct ipu6_isys_queue *aq);
+void ipu6_stream_buf_ready(struct ipu6_isys_stream *stream, u8 pin_id,
+			   u32 pin_addr, u64 time, bool error_check);
 #endif /* IPU6_ISYS_QUEUE_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
index dbd6f76a066d23..60070842ee68fd 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.c
@@ -160,6 +160,7 @@ u32 ipu6_isys_convert_bayer_order(u32 code, int x, int y)
 }
 
 int ipu6_isys_subdev_set_fmt(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *format)
 {
@@ -265,17 +266,13 @@ static int subdev_set_routing(struct v4l2_subdev *sd,
 	return v4l2_subdev_set_routing_with_fmt(sd, state, routing, &format);
 }
 
-u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad)
+u32 __ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev_state *state,
+					  u32 pad)
 {
-	struct v4l2_subdev_state *state;
 	struct v4l2_subdev_route *routes;
 	unsigned int i;
 	u32 source_stream = 0;
 
-	state = v4l2_subdev_lock_and_get_active_state(sd);
-	if (!state)
-		return 0;
-
 	routes = state->routing.routes;
 	for (i = 0; i < state->routing.num_routes; i++) {
 		if (routes[i].source_pad == pad) {
@@ -284,6 +281,20 @@ u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad)
 		}
 	}
 
+	return source_stream;
+}
+
+u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad)
+{
+	struct v4l2_subdev_state *state;
+	u32 source_stream = 0;
+
+	state = v4l2_subdev_lock_and_get_active_state(sd);
+	if (!state)
+		return 0;
+
+	source_stream = __ipu6_isys_get_src_stream_by_src_pad(state, pad);
+
 	v4l2_subdev_unlock_state(state);
 
 	return source_stream;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h
index 35069099c3642f..b892d96992af06 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-subdev.h
@@ -31,12 +31,15 @@ bool ipu6_isys_is_bayer_format(u32 code);
 u32 ipu6_isys_convert_bayer_order(u32 code, int x, int y);
 
 int ipu6_isys_subdev_set_fmt(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *fmt);
 int ipu6_isys_subdev_enum_mbus_code(struct v4l2_subdev *sd,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_mbus_code_enum
 				    *code);
+u32 __ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev_state *state,
+					  u32 pad);
 u32 ipu6_isys_get_src_stream_by_src_pad(struct v4l2_subdev *sd, u32 pad);
 int ipu6_isys_subdev_set_routing(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *state,
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
index 3ac48d2076da9f..129016e57446ba 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.c
@@ -432,185 +432,110 @@ static int link_validate(struct media_link *link)
 	return ret;
 }
 
-static void get_stream_opened(struct ipu6_isys_video *av)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&av->isys->streams_lock, flags);
-	av->isys->stream_opened++;
-	spin_unlock_irqrestore(&av->isys->streams_lock, flags);
-}
+int ipu6_isys_fw_pins_prepare(struct ipu6_isys_stream *stream,
+			      struct v4l2_mbus_frame_desc *desc,
+			      int (*fw_pin_cfg)(struct ipu6_isys_video *av,
+						struct ipu6_isys_stream *stream,
+						struct media_pad *src_pad,
+						struct v4l2_mbus_frame_desc_entry *entry,
+						void *__cfg), void *stream_cfg)
+{
+	struct v4l2_subdev_state *csi2_state =
+		v4l2_subdev_get_locked_active_state(&stream->asd->sd);
+	struct device *dev = &stream->isys->adev->auxdev.dev;
+	struct ipu6_isys_queue *aq;
 
-static void put_stream_opened(struct ipu6_isys_video *av)
-{
-	unsigned long flags;
+	list_for_each_entry(aq, &stream->queues, node) {
+		struct ipu6_isys_video *__av = ipu6_isys_queue_to_video(aq);
+		struct media_pad *remote_pad =
+			media_pad_remote_pad_first(&__av->pad);
+		u64 source_streams = 1;
+		unsigned int sink_stream =
+			__ffs(v4l2_subdev_state_xlate_streams(csi2_state,
+							      remote_pad->index,
+							      CSI2_PAD_SINK,
+							      &source_streams));
+		struct v4l2_mbus_frame_desc_entry *entry = NULL;
+		int ret;
+
+		for (unsigned int i = 0; i < desc->num_entries; i++) {
+			if (desc->entry[i].stream == sink_stream) {
+				entry = &desc->entry[i];
+				break;
+			}
+		}
 
-	spin_lock_irqsave(&av->isys->streams_lock, flags);
-	av->isys->stream_opened--;
-	spin_unlock_irqrestore(&av->isys->streams_lock, flags);
-}
+		if (!entry) {
+			dev_err(dev, "cannot find frame desc entry for sink stream %u\n",
+				sink_stream);
+			return -EINVAL;
+		}
 
-static int ipu6_isys_fw_pin_cfg(struct ipu6_isys_video *av,
-				struct ipu6_fw_isys_stream_cfg_data_abi *cfg)
-{
-	struct media_pad *src_pad = media_pad_remote_pad_first(&av->pad);
-	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(src_pad->entity);
-	struct v4l2_subdev_state *state = v4l2_subdev_get_locked_active_state(sd);
-	struct ipu6_fw_isys_input_pin_info_abi *input_pin;
-	struct ipu6_fw_isys_output_pin_info_abi *output_pin;
-	struct ipu6_isys_stream *stream = av->stream;
-	struct ipu6_isys_queue *aq = &av->aq;
-	struct v4l2_mbus_framefmt fmt;
-	const struct ipu6_isys_pixelformat *pfmt =
-		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	struct v4l2_rect v4l2_crop;
-	struct ipu6_isys *isys = av->isys;
-	int input_pins = cfg->nof_input_pins++;
-	int output_pins;
-	u32 src_stream;
-
-	src_stream = ipu6_isys_get_src_stream_by_src_pad(sd, src_pad->index);
-	fmt = *v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
-	v4l2_crop = *v4l2_subdev_state_get_crop(state, src_pad->index, src_stream);
-
-	input_pin = &cfg->input_pins[input_pins];
-	input_pin->input_res.width = fmt.width;
-	input_pin->input_res.height = fmt.height;
-	input_pin->dt = av->dt;
-	input_pin->bits_per_pix = pfmt->bpp_packed;
-	input_pin->mapped_dt = 0x40; /* invalid mipi data type */
-	input_pin->mipi_decompression = 0;
-	input_pin->capture_mode = IPU6_FW_ISYS_CAPTURE_MODE_REGULAR;
-	input_pin->mipi_store_mode = pfmt->bpp == pfmt->bpp_packed ?
-		IPU6_FW_ISYS_MIPI_STORE_MODE_DISCARD_LONG_HEADER :
-		IPU6_FW_ISYS_MIPI_STORE_MODE_NORMAL;
-	input_pin->crop_first_and_last_lines = v4l2_crop.top & 1;
-
-	output_pins = cfg->nof_output_pins++;
-	aq->fw_output = output_pins;
-	stream->output_pins_queue[output_pins] = aq;
-
-	output_pin = &cfg->output_pins[output_pins];
-	output_pin->input_pin_id = input_pins;
-	output_pin->output_res.width = ipu6_isys_get_frame_width(av);
-	output_pin->output_res.height = ipu6_isys_get_frame_height(av);
-
-	output_pin->stride = ipu6_isys_get_bytes_per_line(av);
-	if (pfmt->bpp != pfmt->bpp_packed)
-		output_pin->pt = IPU6_FW_ISYS_PIN_TYPE_RAW_SOC;
-	else
-		output_pin->pt = IPU6_FW_ISYS_PIN_TYPE_MIPI;
-	output_pin->ft = pfmt->css_pixelformat;
-	output_pin->send_irq = 1;
-	memset(output_pin->ts_offsets, 0, sizeof(output_pin->ts_offsets));
-	output_pin->s2m_pixel_soc_pixel_remapping =
-		S2M_PIXEL_SOC_PIXEL_REMAPPING_FLAG_NO_REMAPPING;
-	output_pin->csi_be_soc_pixel_remapping =
-		CSI_BE_SOC_PIXEL_REMAPPING_FLAG_NO_REMAPPING;
-
-	output_pin->snoopable = true;
-	output_pin->error_handling_enable = false;
-	output_pin->sensor_type = isys->sensor_type++;
-	if (isys->sensor_type > isys->pdata->ipdata->sensor_type_end)
-		isys->sensor_type = isys->pdata->ipdata->sensor_type_start;
+		ret = fw_pin_cfg(__av, stream, remote_pad, entry, stream_cfg);
+		if (ret < 0)
+			return ret;
+	}
 
 	return 0;
 }
 
-static int start_stream_firmware(struct ipu6_isys_video *av,
-				 struct ipu6_isys_buffer_list *bl)
+int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
+				    struct ipu6_isys_buffer_list *bl,
+				    struct v4l2_mbus_frame_desc *desc)
 {
-	struct ipu6_fw_isys_stream_cfg_data_abi *stream_cfg;
-	struct ipu6_fw_isys_frame_buff_set_abi *buf = NULL;
-	struct ipu6_isys_stream *stream = av->stream;
-	struct device *dev = &av->isys->adev->auxdev.dev;
+	struct ipu6_bus_device *adev = stream->asd->isys->adev;
+	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
+	struct device *dev = &adev->auxdev.dev;
 	struct isys_fw_msgs *msg = NULL;
-	struct ipu6_isys_queue *aq;
 	int ret, retout, tout;
-	u16 send_type;
+	bool capture = bl ? true : false;
 
 	msg = ipu6_get_fw_msg_buf(stream);
 	if (!msg)
 		return -ENOMEM;
 
-	stream_cfg = &msg->fw_msg.stream;
-	stream_cfg->src = stream->stream_source;
-	stream_cfg->vc = stream->vc;
-	stream_cfg->isl_use = 0;
-	stream_cfg->sensor_type = IPU6_FW_ISYS_SENSOR_MODE_NORMAL;
-
-	list_for_each_entry(aq, &stream->queues, node) {
-		struct ipu6_isys_video *__av = ipu6_isys_queue_to_video(aq);
-
-		ret = ipu6_isys_fw_pin_cfg(__av, stream_cfg);
-		if (ret < 0) {
-			ipu6_put_fw_msg_buf(av->isys, (uintptr_t)stream_cfg);
-			return ret;
-		}
+	ret = fw_ops->prepare_stream_cfg(stream, desc, msg);
+	if (ret < 0) {
+		ipu6_put_fw_msg_buf(stream->isys, msg);
+		return ret;
 	}
 
-	ipu6_fw_isys_dump_stream_cfg(dev, stream_cfg);
-
-	stream->nr_output_pins = stream_cfg->nof_output_pins;
-
 	reinit_completion(&stream->stream_open_completion);
 
-	ret = ipu6_fw_isys_complex_cmd(av->isys, stream->stream_handle,
-				       stream_cfg, msg->dma_addr,
-				       sizeof(*stream_cfg),
-				       IPU6_FW_ISYS_SEND_TYPE_STREAM_OPEN);
+	ret = fw_ops->stream_open(stream->isys, stream->stream_handle, msg);
 	if (ret < 0) {
 		dev_err(dev, "can't open stream (%d)\n", ret);
-		ipu6_put_fw_msg_buf(av->isys, (uintptr_t)stream_cfg);
+		ipu6_put_fw_msg_buf(stream->isys, msg);
 		return ret;
 	}
 
-	get_stream_opened(av);
-
 	tout = wait_for_completion_timeout(&stream->stream_open_completion,
 					   IPU6_FW_CALL_TIMEOUT_JIFFIES);
 
-	ipu6_put_fw_msg_buf(av->isys, (uintptr_t)stream_cfg);
+	ipu6_put_fw_msg_buf(stream->isys, msg);
 
 	if (!tout) {
 		dev_err(dev, "stream open time out\n");
-		ret = -ETIMEDOUT;
-		goto out_put_stream_opened;
+		return -ETIMEDOUT;
 	}
 	if (stream->error) {
 		dev_err(dev, "stream open error: %d\n", stream->error);
-		ret = -EIO;
-		goto out_put_stream_opened;
+		return -EIO;
 	}
 	dev_dbg(dev, "start stream: open complete\n");
 
-	if (bl) {
-		msg = ipu6_get_fw_msg_buf(stream);
-		if (!msg) {
-			ret = -ENOMEM;
-			goto out_put_stream_opened;
-		}
-		buf = &msg->fw_msg.frame;
-		ipu6_isys_buf_to_fw_frame_buf(buf, stream, bl);
-		ipu6_isys_buffer_list_queue(bl,
-					    IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
+	msg = ipu6_get_fw_msg_buf(stream);
+	if (!msg) {
+		return -ENOMEM;
 	}
 
-	reinit_completion(&stream->stream_start_completion);
+	fw_ops->prepare_buf_set(msg, stream, bl);
+	ipu6_isys_buffer_list_queue(bl, IPU6_ISYS_BUFFER_LIST_FL_ACTIVE, 0);
 
-	if (bl) {
-		send_type = IPU6_FW_ISYS_SEND_TYPE_STREAM_START_AND_CAPTURE;
-		ipu6_fw_isys_dump_frame_buff_set(dev, buf,
-						 stream_cfg->nof_output_pins);
-		ret = ipu6_fw_isys_complex_cmd(av->isys, stream->stream_handle,
-					       buf, msg->dma_addr,
-					       sizeof(*buf), send_type);
-	} else {
-		send_type = IPU6_FW_ISYS_SEND_TYPE_STREAM_START;
-		ret = ipu6_fw_isys_simple_cmd(av->isys, stream->stream_handle,
-					      send_type);
-	}
+	reinit_completion(&stream->stream_start_completion);
 
+	ret = fw_ops->stream_start(stream->isys, stream->stream_handle, msg,
+				   capture);
 	if (ret < 0) {
 		dev_err(dev, "can't start streaming (%d)\n", ret);
 		goto out_stream_close;
@@ -635,12 +560,10 @@ static int start_stream_firmware(struct ipu6_isys_video *av,
 out_stream_close:
 	reinit_completion(&stream->stream_close_completion);
 
-	retout = ipu6_fw_isys_simple_cmd(av->isys,
-					 stream->stream_handle,
-					 IPU6_FW_ISYS_SEND_TYPE_STREAM_CLOSE);
+	retout = fw_ops->stream_close(stream->isys, stream->stream_handle);
 	if (retout < 0) {
 		dev_dbg(dev, "can't close stream (%d)\n", retout);
-		goto out_put_stream_opened;
+		return retout;
 	}
 
 	tout = wait_for_completion_timeout(&stream->stream_close_completion,
@@ -652,23 +575,19 @@ static int start_stream_firmware(struct ipu6_isys_video *av,
 	else
 		dev_dbg(dev, "stream close complete\n");
 
-out_put_stream_opened:
-	put_stream_opened(av);
-
 	return ret;
 }
 
-static void stop_streaming_firmware(struct ipu6_isys_video *av)
+void ipu6_isys_stop_stream_firmware(struct ipu6_isys_stream *stream)
 {
-	struct device *dev = &av->isys->adev->auxdev.dev;
-	struct ipu6_isys_stream *stream = av->stream;
+	struct ipu6_bus_device *adev = stream->asd->isys->adev;
+	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
+	struct device *dev = &adev->auxdev.dev;
 	int ret, tout;
 
 	reinit_completion(&stream->stream_stop_completion);
 
-	ret = ipu6_fw_isys_simple_cmd(av->isys, stream->stream_handle,
-				      IPU6_FW_ISYS_SEND_TYPE_STREAM_FLUSH);
-
+	ret = fw_ops->stream_flush(stream->isys, stream->stream_handle);
 	if (ret < 0) {
 		dev_err(dev, "can't stop stream (%d)\n", ret);
 		return;
@@ -684,16 +603,17 @@ static void stop_streaming_firmware(struct ipu6_isys_video *av)
 		dev_dbg(dev, "stop stream: complete\n");
 }
 
-static void close_streaming_firmware(struct ipu6_isys_video *av)
+void ipu6_isys_close_stream_firmware(struct ipu6_isys_stream *stream)
 {
-	struct ipu6_isys_stream *stream = av->stream;
-	struct device *dev = &av->isys->adev->auxdev.dev;
+	struct ipu6_bus_device *adev = stream->asd->isys->adev;
+	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
+	struct device *dev = &adev->auxdev.dev;
+	struct ipu6_isys_csi2 *csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
 	int ret, tout;
 
 	reinit_completion(&stream->stream_close_completion);
 
-	ret = ipu6_fw_isys_simple_cmd(av->isys, stream->stream_handle,
-				      IPU6_FW_ISYS_SEND_TYPE_STREAM_CLOSE);
+	ret = fw_ops->stream_close(stream->isys, stream->stream_handle);
 	if (ret < 0) {
 		dev_err(dev, "can't close stream (%d)\n", ret);
 		return;
@@ -708,349 +628,153 @@ static void close_streaming_firmware(struct ipu6_isys_video *av)
 	else
 		dev_dbg(dev, "close stream: complete\n");
 
-	put_stream_opened(av);
+	scoped_guard(spinlock_irqsave, &stream->isys->streams_lock) {
+		stream->isys->streams_by_handle[stream->stream_handle] = NULL;
+		csi2->streams_by_vc[stream->vc] = NULL;
+	}
 }
 
-int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
-				   struct media_entity *source_entity,
-				   int nr_queues)
+struct ipu6_isys_stream *
+ipu6_isys_find_stream_firmware(struct ipu6_isys_csi2 *csi2, u8 vc)
 {
-	struct ipu6_isys_stream *stream = av->stream;
-	struct ipu6_isys_csi2 *csi2;
+	struct ipu6_isys_stream *stream;
 
-	if (WARN_ON(stream->nr_streaming))
-		return -EINVAL;
+	list_for_each_entry(stream, &csi2->streams, csi2_entry)
+		if (stream->vc == vc)
+			return stream;
 
-	stream->nr_queues = nr_queues;
-	atomic_set(&stream->sequence, 0);
-
-	stream->seq_index = 0;
-	memset(stream->seq, 0, sizeof(stream->seq));
-
-	if (WARN_ON(!list_empty(&stream->queues)))
-		return -EINVAL;
-
-	stream->stream_source = stream->asd->source;
-	csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
-	csi2->receiver_errors = 0;
-
-	dev_dbg(&av->isys->adev->auxdev.dev,
-		"prepare stream: external entity %s\n",
-		source_entity->name);
-
-	return 0;
+	return NULL;
 }
 
-void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
-					  struct media_entity *source)
+void ipu6_isys_free_stream_firmware(struct ipu6_isys_stream *stream)
 {
-	struct ipu6_isys *isys = av->isys;
-	struct ipu6_isys_csi2 *csi2 = NULL;
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-	struct device *dev = &isys->adev->auxdev.dev;
-	struct v4l2_mbus_framefmt format;
-	struct v4l2_subdev *esd;
-	struct v4l2_control hb = { .id = V4L2_CID_HBLANK, .value = 0 };
-	unsigned int bpp, lanes;
-	s64 link_freq = 0;
-	u64 pixel_rate = 0;
-	int ret;
-
-	esd = media_entity_to_v4l2_subdev(source);
-
-	av->watermark.width = ipu6_isys_get_frame_width(av);
-	av->watermark.height = ipu6_isys_get_frame_height(av);
-	av->watermark.sram_gran_shift = isys->pdata->ipdata->sram_gran_shift;
-	av->watermark.sram_gran_size = isys->pdata->ipdata->sram_gran_size;
-
-	ret = v4l2_g_ctrl(esd->ctrl_handler, &hb);
-	if (!ret && hb.value >= 0)
-		av->watermark.hblank = hb.value;
-	else
-		av->watermark.hblank = 0;
-
-	csi2 = ipu6_isys_subdev_to_csi2(av->stream->asd);
-	link_freq = ipu6_isys_csi2_get_link_freq(csi2);
-	if (link_freq > 0) {
-		struct v4l2_subdev_state *state =
-			v4l2_subdev_lock_and_get_active_state(&csi2->asd.sd);
-
-		lanes = csi2->nlanes;
-		format = *v4l2_subdev_state_get_format(state, 0,
-						       av->source_stream);
-		bpp = ipu6_isys_mbus_code_to_bpp(format.code);
-		pixel_rate = mul_u64_u32_div(link_freq, lanes * 2, bpp);
+	struct ipu6_isys_csi2 *csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
+	struct ipu6_isys_queue *aq, *aq_safe;
 
-		v4l2_subdev_unlock_state(state);
-	}
-
-	av->watermark.pixel_rate = pixel_rate;
+	list_for_each_entry_safe(aq, aq_safe, &stream->queues, node) {
+		struct ipu6_isys_video *av =
+			container_of_const(aq, struct ipu6_isys_video, aq);
 
-	if (!pixel_rate) {
-		mutex_lock(&iwake_watermark->mutex);
-		iwake_watermark->force_iwake_disable = true;
-		mutex_unlock(&iwake_watermark->mutex);
-		dev_warn(dev, "unexpected pixel_rate from %s, disable iwake.\n",
-			 source->name);
+		list_del(&aq->node);
+		av->stream = NULL;
 	}
-}
 
-static void calculate_stream_datarate(struct ipu6_isys_video *av)
-{
-	struct video_stream_watermark *watermark = &av->watermark;
-	const struct ipu6_isys_pixelformat *pfmt =
-		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	u32 pages_per_line, pb_bytes_per_line, pixels_per_line, bytes_per_line;
-	u64 line_time_ns, stream_data_rate;
-	u16 shift, size;
-
-	shift = watermark->sram_gran_shift;
-	size = watermark->sram_gran_size;
-
-	pixels_per_line = watermark->width + watermark->hblank;
-	line_time_ns =  div_u64(pixels_per_line * NSEC_PER_SEC,
-				watermark->pixel_rate);
-	bytes_per_line = watermark->width * pfmt->bpp / 8;
-	pages_per_line = DIV_ROUND_UP(bytes_per_line, size);
-	pb_bytes_per_line = pages_per_line << shift;
-	stream_data_rate = div64_u64(pb_bytes_per_line * 1000, line_time_ns);
-
-	watermark->stream_data_rate = stream_data_rate;
+	list_del(&stream->csi2_entry);
+	ida_free(&csi2->isys->streams, stream->stream_handle);
+	kfree(stream);
 }
 
-void ipu6_isys_update_stream_watermark(struct ipu6_isys_video *av, bool state)
-{
-	struct isys_iwake_watermark *iwake_watermark =
-		&av->isys->iwake_watermark;
-
-	if (!av->watermark.pixel_rate)
-		return;
+struct ipu6_isys_stream *
+ipu6_isys_alloc_stream_firmware(struct ipu6_isys_csi2 *csi2,
+				struct v4l2_subdev_state *csi2_state,
+				struct v4l2_mbus_frame_desc *desc,
+				u8 vc)
+{
+	struct device *dev = &csi2->isys->adev->auxdev.dev;
+	struct ipu6_isys_stream *stream;
+	struct v4l2_subdev_route *route;
+	int ret;
 
-	if (state) {
-		calculate_stream_datarate(av);
-		mutex_lock(&iwake_watermark->mutex);
-		list_add(&av->watermark.stream_node,
-			 &iwake_watermark->video_list);
-		mutex_unlock(&iwake_watermark->mutex);
-	} else {
-		av->watermark.stream_data_rate = 0;
-		mutex_lock(&iwake_watermark->mutex);
-		list_del(&av->watermark.stream_node);
-		mutex_unlock(&iwake_watermark->mutex);
-	}
+	stream = kzalloc_obj(*stream);
+	if (!stream)
+		return ERR_PTR(-ENOMEM);
 
-	update_watermark_setting(av->isys);
-}
+	ret = ida_alloc_max(&csi2->isys->streams, IPU6_ISYS_MAX_STREAMS - 1,
+			    GFP_KERNEL);
+	if (ret < 0)
+		goto err_free_stream;
 
-void ipu6_isys_put_stream(struct ipu6_isys_stream *stream)
-{
-	struct device *dev;
-	unsigned int i;
-	unsigned long flags;
+	stream->stream_handle = ret;
+	mutex_init(&stream->mutex);
+	init_completion(&stream->stream_open_completion);
+	init_completion(&stream->stream_close_completion);
+	init_completion(&stream->stream_start_completion);
+	init_completion(&stream->stream_stop_completion);
+	INIT_LIST_HEAD(&stream->queues);
+	stream->isys = csi2->asd.isys;
+	stream->asd = &csi2->asd;
+	stream->vc = vc;
 
-	if (!stream) {
-		pr_err("ipu6-isys: no available stream\n");
-		return;
+	scoped_guard(spinlock_irqsave, &stream->isys->streams_lock) {
+		stream->isys->streams_by_handle[stream->stream_handle] = stream;
+		csi2->streams_by_vc[stream->vc] = stream;
 	}
 
-	dev = &stream->isys->adev->auxdev.dev;
-
-	spin_lock_irqsave(&stream->isys->streams_lock, flags);
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		if (&stream->isys->streams[i] == stream) {
-			if (stream->isys->streams_ref_count[i] > 0)
-				stream->isys->streams_ref_count[i]--;
-			else
-				dev_warn(dev, "invalid stream %d\n", i);
-
-			break;
-		}
-	}
-	spin_unlock_irqrestore(&stream->isys->streams_lock, flags);
-}
+	list_add(&stream->csi2_entry, &csi2->streams);
 
-static struct ipu6_isys_stream *
-ipu6_isys_get_stream(struct ipu6_isys_video *av, struct ipu6_isys_subdev *asd)
-{
-	struct ipu6_isys_stream *stream = NULL;
-	struct ipu6_isys *isys = av->isys;
-	unsigned long flags;
-	unsigned int i;
-	u8 vc = av->vc;
+	for_each_active_route(&csi2_state->routing, route) {
+		struct media_pad *vdev_pad =
+			media_pad_remote_pad_first(&csi2->asd.pad[route->source_pad]);
+		struct v4l2_mbus_frame_desc_entry *entry = NULL;
 
-	if (!isys)
-		return NULL;
+		for (unsigned int i = 0; i < desc->num_entries; i++) {
+			if (desc->entry[i].stream != route->sink_stream)
+				continue;
 
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		if (isys->streams_ref_count[i] && isys->streams[i].vc == vc &&
-		    isys->streams[i].asd == asd) {
-			isys->streams_ref_count[i]++;
-			stream = &isys->streams[i];
+			entry = &desc->entry[i];
 			break;
 		}
-	}
 
-	if (!stream) {
-		for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-			if (!isys->streams_ref_count[i]) {
-				isys->streams_ref_count[i]++;
-				stream = &isys->streams[i];
-				stream->vc = vc;
-				stream->asd = asd;
-				break;
-			}
+		if (!entry) {
+			dev_dbg(dev, "cannot find stream %u in frame desc\n",
+				route->sink_stream);
+			ret = -EINVAL;
+			goto err_ida_free;
 		}
-	}
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
-
-	return stream;
-}
 
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_handle(struct ipu6_isys *isys, u8 stream_handle)
-{
-	unsigned long flags;
-	struct ipu6_isys_stream *stream = NULL;
-
-	if (!isys)
-		return NULL;
-
-	if (stream_handle >= IPU6_ISYS_MAX_STREAMS) {
-		dev_err(&isys->adev->auxdev.dev,
-			"stream_handle %d is invalid\n", stream_handle);
-		return NULL;
-	}
-
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	if (isys->streams_ref_count[stream_handle] > 0) {
-		isys->streams_ref_count[stream_handle]++;
-		stream = &isys->streams[stream_handle];
-	}
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
-
-	return stream;
-}
-
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_source(struct ipu6_isys *isys, int source, u8 vc)
-{
-	struct ipu6_isys_stream *stream = NULL;
-	unsigned long flags;
-	unsigned int i;
-
-	if (!isys)
-		return NULL;
+		if (entry->bus.csi2.vc != vc)
+			continue;
 
-	if (source < 0) {
-		dev_err(&isys->adev->auxdev.dev,
-			"query stream with invalid port number\n");
-		return NULL;
-	}
+		struct ipu6_isys_video *av =
+			container_of_const(vdev_pad, struct ipu6_isys_video,
+					   pad);
 
-	spin_lock_irqsave(&isys->streams_lock, flags);
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		if (!isys->streams_ref_count[i])
-			continue;
+		list_add(&av->aq.node, &stream->queues);
 
-		if (isys->streams[i].stream_source == source &&
-		    isys->streams[i].vc == vc) {
-			stream = &isys->streams[i];
-			isys->streams_ref_count[i]++;
-			break;
-		}
+		stream->nr_output_pins++;
+		av->stream = stream;
 	}
-	spin_unlock_irqrestore(&isys->streams_lock, flags);
 
 	return stream;
-}
-
-static u64 get_stream_mask_by_pipeline(struct ipu6_isys_video *__av)
-{
-	struct media_pipeline *pipeline =
-		media_entity_pipeline(&__av->vdev.entity);
-	unsigned int i;
-	u64 stream_mask = 0;
 
-	for (i = 0; i < NR_OF_CSI2_SRC_PADS; i++) {
-		struct ipu6_isys_video *av = &__av->csi2->av[i];
+err_ida_free:
+	list_del(&stream->csi2_entry);
+	ida_free(&csi2->isys->streams, stream->stream_handle);
 
-		if (pipeline == media_entity_pipeline(&av->vdev.entity))
-			stream_mask |= BIT_ULL(av->source_stream);
-	}
+err_free_stream:
+	kfree(stream);
 
-	return stream_mask;
+	return ERR_PTR(ret);
 }
 
-int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state,
-				  struct ipu6_isys_buffer_list *bl)
+int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state)
 {
-	struct v4l2_subdev_krouting *routing;
-	struct ipu6_isys_stream *stream = av->stream;
-	struct v4l2_subdev_state *subdev_state;
 	struct device *dev = &av->isys->adev->auxdev.dev;
 	struct v4l2_subdev *sd;
 	struct media_pad *r_pad;
-	u32 sink_pad, sink_stream;
-	u64 r_stream;
-	u64 stream_mask = 0;
 	int ret = 0;
 
-	dev_dbg(dev, "set stream: %d\n", state);
-
-	sd = &stream->asd->sd;
+	sd = &av->csi2->asd.sd;
 	r_pad = media_pad_remote_pad_first(&av->pad);
-	r_stream = ipu6_isys_get_src_stream_by_src_pad(sd, r_pad->index);
-
-	subdev_state = v4l2_subdev_lock_and_get_active_state(sd);
-	routing = &subdev_state->routing;
-	ret = v4l2_subdev_routing_find_opposite_end(routing, r_pad->index,
-						    r_stream, &sink_pad,
-						    &sink_stream);
-	v4l2_subdev_unlock_state(subdev_state);
-	if (ret)
-		return ret;
 
-	stream_mask = get_stream_mask_by_pipeline(av);
 	if (!state) {
-		stop_streaming_firmware(av);
-
 		/* stop sub-device which connects with video */
-		dev_dbg(dev, "stream off entity %s pad:%d mask:0x%llx\n",
-			sd->name, r_pad->index, stream_mask);
-		ret = v4l2_subdev_disable_streams(sd, r_pad->index,
-						  stream_mask);
+		dev_dbg(dev, "stream off %s pad:%d\n", sd->name, r_pad->index);
+		ret = v4l2_subdev_disable_streams(sd, r_pad->index, 1);
 		if (ret)
 			dev_err(dev, "stream off %s failed with %d\n", sd->name,
 				ret);
-
-		close_streaming_firmware(av);
 	} else {
-		ret = start_stream_firmware(av, bl);
-		if (ret) {
-			dev_err(dev, "start stream of firmware failed\n");
-			return ret;
-		}
-
 		/* start sub-device which connects with video */
-		dev_dbg(dev, "stream on %s pad %d mask 0x%llx\n", sd->name,
-			r_pad->index, stream_mask);
-		ret = v4l2_subdev_enable_streams(sd, r_pad->index, stream_mask);
-		if (ret) {
+		dev_dbg(dev, "stream on %s pad %d\n", sd->name, r_pad->index);
+		ret = v4l2_subdev_enable_streams(sd, r_pad->index, 1);
+		if (ret)
 			dev_err(dev, "stream on %s failed with %d\n", sd->name,
 				ret);
-			goto out_media_entity_stop_streaming_firmware;
-		}
 	}
 
 	av->streaming = state;
 
-	return 0;
-
-out_media_entity_stop_streaming_firmware:
-	stop_streaming_firmware(av);
-	close_streaming_firmware(av);
-
 	return ret;
 }
 
@@ -1089,148 +813,6 @@ static const struct v4l2_file_operations isys_fops = {
 	.release = vb2_fop_release,
 };
 
-int ipu6_isys_fw_open(struct ipu6_isys *isys)
-{
-	struct ipu6_bus_device *adev = isys->adev;
-	const struct ipu6_isys_internal_pdata *ipdata = isys->pdata->ipdata;
-	int ret;
-
-	ret = pm_runtime_resume_and_get(&adev->auxdev.dev);
-	if (ret < 0)
-		return ret;
-
-	mutex_lock(&isys->mutex);
-
-	if (isys->ref_count++)
-		goto unlock;
-
-	ipu6_configure_spc(adev->isp, &ipdata->hw_variant,
-			   IPU6_CPD_PKG_DIR_ISYS_SERVER_IDX, isys->pdata->base,
-			   adev->pkg_dir, adev->pkg_dir_dma_addr);
-
-	/*
-	 * Buffers could have been left to wrong queue at last closure.
-	 * Move them now back to empty buffer queue.
-	 */
-	ipu6_cleanup_fw_msg_bufs(isys);
-
-	if (isys->fwcom) {
-		/*
-		 * Something went wrong in previous shutdown. As we are now
-		 * restarting isys we can safely delete old context.
-		 */
-		dev_warn(&adev->auxdev.dev, "clearing old context\n");
-		ipu6_fw_isys_cleanup(isys);
-	}
-
-	ret = ipu6_fw_isys_init(isys, ipdata->num_parallel_streams);
-	if (ret < 0)
-		goto out;
-
-unlock:
-	mutex_unlock(&isys->mutex);
-
-	return 0;
-
-out:
-	isys->ref_count--;
-	mutex_unlock(&isys->mutex);
-	pm_runtime_put(&adev->auxdev.dev);
-
-	return ret;
-}
-
-void ipu6_isys_fw_close(struct ipu6_isys *isys)
-{
-	mutex_lock(&isys->mutex);
-
-	isys->ref_count--;
-	if (!isys->ref_count) {
-		ipu6_fw_isys_close(isys);
-		if (isys->fwcom) {
-			isys->need_reset = true;
-			dev_warn(&isys->adev->auxdev.dev,
-				 "failed to close fw isys\n");
-		}
-	}
-
-	mutex_unlock(&isys->mutex);
-
-	if (isys->need_reset)
-		pm_runtime_put_sync(&isys->adev->auxdev.dev);
-	else
-		pm_runtime_put(&isys->adev->auxdev.dev);
-}
-
-int ipu6_isys_setup_video(struct ipu6_isys_video *av,
-			  struct media_pad *remote_pad,
-			  struct media_pad *source_pad, int *nr_queues)
-{
-	const struct ipu6_isys_pixelformat *pfmt =
-		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
-	struct device *dev = &av->isys->adev->auxdev.dev;
-	struct v4l2_mbus_frame_desc_entry entry;
-	struct v4l2_subdev_route *route = NULL;
-	struct v4l2_subdev_route *r;
-	struct v4l2_subdev_state *state;
-	struct v4l2_subdev *remote_sd =
-		media_entity_to_v4l2_subdev(remote_pad->entity);
-	struct ipu6_isys_subdev *asd = to_ipu6_isys_subdev(remote_sd);
-	int ret = -EINVAL;
-
-	*nr_queues = 0;
-
-	/* Find the root */
-	state = v4l2_subdev_lock_and_get_active_state(remote_sd);
-	for_each_active_route(&state->routing, r) {
-		(*nr_queues)++;
-
-		if (r->source_pad == remote_pad->index)
-			route = r;
-	}
-
-	if (!route) {
-		v4l2_subdev_unlock_state(state);
-		dev_dbg(dev, "Failed to find route\n");
-		return -ENODEV;
-	}
-	av->source_stream = route->sink_stream;
-	v4l2_subdev_unlock_state(state);
-
-	ret = ipu6_isys_csi2_get_remote_desc(av->source_stream,
-					     to_ipu6_isys_csi2(asd),
-					     source_pad->entity, &entry);
-	if (ret == -ENOIOCTLCMD) {
-		av->vc = 0;
-		av->dt = ipu6_isys_mbus_code_to_mipi(pfmt->code);
-	} else if (!ret) {
-		dev_dbg(dev, "Framedesc: stream %u, len %u, vc %u, dt %#x\n",
-			entry.stream, entry.length, entry.bus.csi2.vc,
-			entry.bus.csi2.dt);
-
-		av->vc = entry.bus.csi2.vc;
-		av->dt = entry.bus.csi2.dt;
-	} else {
-		dev_err(dev, "failed to get remote frame desc\n");
-		return ret;
-	}
-
-	ret = video_device_pipeline_alloc_start(&av->vdev);
-	if (ret < 0) {
-		dev_dbg(dev, "media pipeline start failed\n");
-		return ret;
-	}
-
-	av->stream = ipu6_isys_get_stream(av, asd);
-	if (!av->stream) {
-		video_device_pipeline_stop(&av->vdev);
-		dev_err(dev, "no available stream for firmware\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
 /*
  * Do everything that's needed to initialise things related to video
  * buffer queue, video node, and the related media entity. The caller
@@ -1262,7 +844,7 @@ int ipu6_isys_video_init(struct ipu6_isys_video *av)
 
 	ret = ipu6_isys_queue_init(&av->aq);
 	if (ret)
-		goto out_free_watermark;
+		goto out_mutex_destroy;
 
 	av->pad.flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT;
 	ret = media_entity_pads_init(&av->vdev.entity, 1, &av->pad);
@@ -1299,7 +881,7 @@ int ipu6_isys_video_init(struct ipu6_isys_video *av)
 out_vb2_queue_release:
 	vb2_queue_release(&av->aq.vbq);
 
-out_free_watermark:
+out_mutex_destroy:
 	mutex_destroy(&av->mutex);
 
 	return ret;
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
index 2ff53315d7b937..2821b9b1b943b6 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-video.h
@@ -22,6 +22,7 @@ struct file;
 struct ipu6_isys;
 struct ipu6_isys_csi2;
 struct ipu6_isys_subdev;
+struct v4l2_mbus_frame_desc;
 
 struct ipu6_isys_pixelformat {
 	u32 pixelformat;
@@ -44,17 +45,15 @@ struct sequence_info {
 struct ipu6_isys_stream {
 	struct mutex mutex;
 	atomic_t sequence;
+	atomic_t buf_id;
 	unsigned int seq_index;
 	struct sequence_info seq[IPU6_ISYS_MAX_PARALLEL_SOF];
-	int stream_source;
 	int stream_handle;
 	unsigned int nr_output_pins;
 	struct ipu6_isys_subdev *asd;
-
-	int nr_queues;	/* Number of capture queues */
-	int nr_streaming;
-	int streaming;	/* Has streaming been really started? */
 	struct list_head queues;
+	struct list_head csi2_entry;
+
 	struct completion stream_open_completion;
 	struct completion stream_close_completion;
 	struct completion stream_start_completion;
@@ -66,20 +65,9 @@ struct ipu6_isys_stream {
 	u8 vc;
 };
 
-struct video_stream_watermark {
-	u32 width;
-	u32 height;
-	u32 hblank;
-	u32 frame_rate;
-	u64 pixel_rate;
-	u64 stream_data_rate;
-	u16 sram_gran_shift;
-	u16 sram_gran_size;
-	struct list_head stream_node;
-};
-
 struct ipu6_isys_video {
 	struct ipu6_isys_queue aq;
+	struct list_head csi2_entry;
 	/* Serialise access to other fields in the struct. */
 	struct mutex mutex;
 	struct media_pad pad;
@@ -90,10 +78,7 @@ struct ipu6_isys_video {
 	struct ipu6_isys_csi2 *csi2;
 	struct ipu6_isys_stream *stream;
 	unsigned int streaming;
-	struct video_stream_watermark watermark;
 	u32 source_stream;
-	u8 vc;
-	u8 dt;
 };
 
 #define ipu6_isys_queue_to_video(__aq) \
@@ -104,27 +89,34 @@ extern const struct ipu6_isys_pixelformat ipu6_isys_pfmts_packed[];
 
 const struct ipu6_isys_pixelformat *
 ipu6_isys_get_isys_format(u32 pixelformat, u32 code);
-int ipu6_isys_video_prepare_stream(struct ipu6_isys_video *av,
-				   struct media_entity *source_entity,
-				   int nr_queues);
-int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state,
-				  struct ipu6_isys_buffer_list *bl);
+int ipu6_isys_fw_pins_prepare(struct ipu6_isys_stream *stream,
+			      struct v4l2_mbus_frame_desc *desc,
+			      int (*fw_pin_cfg)(struct ipu6_isys_video *av,
+						struct ipu6_isys_stream *stream,
+						struct media_pad *src_pad,
+						struct v4l2_mbus_frame_desc_entry *entry,
+						void *__cfg), void *stream_cfg);
+int ipu6_isys_start_stream_firmware(struct ipu6_isys_stream *stream,
+				    struct ipu6_isys_buffer_list *bl,
+				    struct v4l2_mbus_frame_desc *desc);
+void ipu6_isys_stop_stream_firmware(struct ipu6_isys_stream *stream);
+void ipu6_isys_close_stream_firmware(struct ipu6_isys_stream *stream);
+struct ipu6_isys_stream *
+ipu6_isys_find_stream_firmware(struct ipu6_isys_csi2 *csi2, u8 vc);
+void ipu6_isys_free_stream_firmware(struct ipu6_isys_stream *stream);
+struct ipu6_isys_stream *
+ipu6_isys_alloc_stream_firmware(struct ipu6_isys_csi2 *csi2,
+				struct v4l2_subdev_state *state,
+				struct v4l2_mbus_frame_desc *desc,
+				u8 vc);
+int ipu6_isys_video_set_streaming(struct ipu6_isys_video *av, int state);
 int ipu6_isys_fw_open(struct ipu6_isys *isys);
 void ipu6_isys_fw_close(struct ipu6_isys *isys);
 int ipu6_isys_setup_video(struct ipu6_isys_video *av,
 			  struct media_pad *remote_pad,
-			  struct media_pad *source_pad, int *nr_queues);
+			  struct media_pad *source_pad);
 int ipu6_isys_video_init(struct ipu6_isys_video *av);
 void ipu6_isys_video_cleanup(struct ipu6_isys_video *av);
-void ipu6_isys_put_stream(struct ipu6_isys_stream *stream);
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_handle(struct ipu6_isys *isys, u8 stream_handle);
-struct ipu6_isys_stream *
-ipu6_isys_query_stream_by_source(struct ipu6_isys *isys, int source, u8 vc);
-
-void ipu6_isys_configure_stream_watermark(struct ipu6_isys_video *av,
-					  struct media_entity *source);
-void ipu6_isys_update_stream_watermark(struct ipu6_isys_video *av, bool state);
 
 u32 ipu6_isys_get_format(struct ipu6_isys_video *av);
 u32 ipu6_isys_get_data_size(struct ipu6_isys_video *av);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c
index 24db2763de5443..60f5f9ea291060 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c
@@ -41,6 +41,9 @@
 #include "ipu6-platform-buttress-regs.h"
 #include "ipu6-platform-isys-csi2-reg.h"
 #include "ipu6-platform-regs.h"
+#include "ipu7-isys-csi-phy.h"
+#include "ipu7-isys-csi2-regs.h"
+#include "ipu7-platform-regs.h"
 
 #define IPU6_BUTTRESS_FABIC_CONTROL		0x68
 #define GDA_ENABLE_IWAKE_INDEX			2
@@ -98,9 +101,28 @@ enum ltr_did_type {
 	LTR_TYPE_MAX
 };
 
-#define ISYS_PM_QOS_VALUE	300
+struct ltr_did {
+	union {
+		u32 value;
+		struct {
+			u8 val0;
+			u8 val1;
+			u8 val2;
+			u8 val3;
+		} bits;
+	} lut_ltr;
+	union {
+		u32 value;
+		struct {
+			u8 th0;
+			u8 th1;
+			u8 th2;
+			u8 th3;
+		} bits;
+	} lut_fill_time;
+};
 
-static int isys_isr_one(struct ipu6_bus_device *adev);
+#define ISYS_PM_QOS_VALUE	300
 
 static int
 isys_complete_ext_device_registration(struct ipu6_isys *isys,
@@ -132,6 +154,9 @@ isys_complete_ext_device_registration(struct ipu6_isys *isys,
 	}
 
 	isys->csi2[csi2->port].nlanes = csi2->nlanes;
+	isys->csi2[csi2->port].phy_mode =
+		csi2->bus_type == V4L2_MBUS_CSI2_DPHY ?
+		PHY_MODE_DPHY : PHY_MODE_CPHY;
 
 	return 0;
 
@@ -141,23 +166,6 @@ isys_complete_ext_device_registration(struct ipu6_isys *isys,
 	return ret;
 }
 
-static void isys_stream_init(struct ipu6_isys *isys)
-{
-	u32 i;
-
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++) {
-		mutex_init(&isys->streams[i].mutex);
-		init_completion(&isys->streams[i].stream_open_completion);
-		init_completion(&isys->streams[i].stream_close_completion);
-		init_completion(&isys->streams[i].stream_start_completion);
-		init_completion(&isys->streams[i].stream_stop_completion);
-		INIT_LIST_HEAD(&isys->streams[i].queues);
-		isys->streams[i].isys = isys;
-		isys->streams[i].stream_handle = i;
-		isys->streams[i].vc = INVALID_VC_ID;
-	}
-}
-
 static void isys_csi2_unregister_subdevices(struct ipu6_isys *isys)
 {
 	const struct ipu6_isys_internal_csi2_pdata *csi2 =
@@ -176,13 +184,23 @@ static int isys_csi2_register_subdevices(struct ipu6_isys *isys)
 	int ret;
 
 	for (i = 0; i < csi2_pdata->nports; i++) {
-		ret = ipu6_isys_csi2_init(&isys->csi2[i], isys,
-					  isys->pdata->base +
-					  CSI_REG_PORT_BASE(i), i);
+		void __iomem *base = isys->pdata->base;
+
+		if (IS_IPU7(isys->adev->isp)) {
+			u32 mask = IS_IPU7_MTL(isys->adev->isp) ?
+					IPU7_CSI_LEGACY_IRQ_MASK(i) :
+					IPU7P5_CSI_LEGACY_IRQ_MASK(i);
+
+			isys->isr_csi2_bits |= mask;
+			isys->csi2[i].legacy_irq_mask = mask;
+		} else {
+			base += CSI_REG_PORT_BASE(i);
+			isys->isr_csi2_bits |= IPU6_ISYS_UNISPART_IRQ_CSI2(i);
+		}
+
+		ret = ipu6_isys_csi2_init(&isys->csi2[i], isys, base, i);
 		if (ret)
 			goto fail;
-
-		isys->isr_csi2_bits |= IPU6_ISYS_UNISPART_IRQ_CSI2(i);
 	}
 
 	return 0;
@@ -269,7 +287,36 @@ static int isys_register_video_devices(struct ipu6_isys *isys)
 	return ret;
 }
 
-static void isys_setup_hw(struct ipu6_isys *isys)
+static void ipu7_isys_setup_hw(struct ipu6_isys *isys)
+{
+	u32 offset, mask;
+	void __iomem *base = isys->pdata->base;
+
+	offset = IPU7_IS_IO_GPREGS_BASE;
+
+	writel(0x0, base + offset + IPU7_CLK_EN_TXCLKESC);
+	/* Update if ISYS freq updated (0: 400/1, 1:400/2, 63:400/64) */
+	writel(0x0, base + offset + IPU7_CLK_DIV_FACTOR_IS_CLK);
+	/* correct the initial printf configuration */
+	writel(0x200, base + IPU7_IS_UC_CTRL_BASE + IPU7_REG_PRINTF_AXI_CNTL);
+
+	offset = IPU7_IS_UC_CTRL_BASE;
+	mask = IPU7_IS_UC_TO_SW_IRQ_MASK;
+
+	writel(mask, base + offset + IPU7_TO_SW_IRQ_CNTL_CLEAR);
+	writel(mask, base + offset + IPU7_TO_SW_IRQ_CNTL_MASK_N);
+	writel(mask, base + offset + IPU7_TO_SW_IRQ_CNTL_ENABLE);
+
+	offset = IPU7_IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE;
+	mask = IPU7_CSI_RX_LEGACY_IRQ_MASK;
+
+	writel(mask, base + offset + IPU7_IRQ_CTL_EDGE);
+	writel(mask, base + offset + IPU7_IRQ_CTL_CLEAR);
+	writel(mask, base + offset + IPU7_IRQ_CTL_MASK);
+	writel(mask, base + offset + IPU7_IRQ_CTL_ENABLE);
+}
+
+static void ipu6_isys_setup_hw(struct ipu6_isys *isys)
 {
 	void __iomem *base = isys->pdata->base;
 	const u8 *thd = isys->pdata->ipdata->hw_variant.cdc_fifo_threshold;
@@ -304,118 +351,6 @@ static void isys_setup_hw(struct ipu6_isys *isys)
 		writel(thd[i], base + IPU6_REG_ISYS_CDC_THRESHOLD(i));
 }
 
-static void ipu6_isys_csi2_isr(struct ipu6_isys_csi2 *csi2)
-{
-	struct ipu6_isys_stream *stream;
-	unsigned int i;
-	u32 status;
-	int source;
-
-	ipu6_isys_register_errors(csi2);
-
-	status = readl(csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
-		       CSI_PORT_REG_BASE_IRQ_STATUS_OFFSET);
-
-	writel(status, csi2->base + CSI_PORT_REG_BASE_IRQ_CSI_SYNC +
-	       CSI_PORT_REG_BASE_IRQ_CLEAR_OFFSET);
-
-	source = csi2->asd.source;
-	for (i = 0; i < NR_OF_CSI2_VC; i++) {
-		if (status & IPU_CSI_RX_IRQ_FS_VC(i)) {
-			stream = ipu6_isys_query_stream_by_source(csi2->isys,
-								  source, i);
-			if (stream) {
-				ipu6_isys_csi2_sof_event_by_stream(stream);
-				ipu6_isys_put_stream(stream);
-			}
-		}
-
-		if (status & IPU_CSI_RX_IRQ_FE_VC(i)) {
-			stream = ipu6_isys_query_stream_by_source(csi2->isys,
-								  source, i);
-			if (stream) {
-				ipu6_isys_csi2_eof_event_by_stream(stream);
-				ipu6_isys_put_stream(stream);
-			}
-		}
-	}
-}
-
-static irqreturn_t isys_isr(struct ipu6_bus_device *adev)
-{
-	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
-	void __iomem *base = isys->pdata->base;
-	u32 status_sw, status_csi;
-	u32 ctrl0_status, ctrl0_clear;
-
-	spin_lock(&isys->power_lock);
-	if (!isys->power) {
-		spin_unlock(&isys->power_lock);
-		return IRQ_NONE;
-	}
-
-	ctrl0_status = isys->pdata->ipdata->csi2.ctrl0_irq_status;
-	ctrl0_clear = isys->pdata->ipdata->csi2.ctrl0_irq_clear;
-
-	status_csi = readl(isys->pdata->base + ctrl0_status);
-	status_sw = readl(isys->pdata->base +
-			  IPU6_REG_ISYS_UNISPART_IRQ_STATUS);
-
-	writel(ISYS_UNISPART_IRQS & ~IPU6_ISYS_UNISPART_IRQ_SW,
-	       base + IPU6_REG_ISYS_UNISPART_IRQ_MASK);
-
-	do {
-		writel(status_csi, isys->pdata->base + ctrl0_clear);
-
-		writel(status_sw, isys->pdata->base +
-		       IPU6_REG_ISYS_UNISPART_IRQ_CLEAR);
-
-		if (isys->isr_csi2_bits & status_csi) {
-			unsigned int i;
-
-			for (i = 0; i < isys->pdata->ipdata->csi2.nports; i++) {
-				/* irq from not enabled port */
-				if (!isys->csi2[i].base)
-					continue;
-				if (status_csi & IPU6_ISYS_UNISPART_IRQ_CSI2(i))
-					ipu6_isys_csi2_isr(&isys->csi2[i]);
-			}
-		}
-
-		writel(0, base + IPU6_REG_ISYS_UNISPART_SW_IRQ_REG);
-
-		if (!isys_isr_one(adev))
-			status_sw = IPU6_ISYS_UNISPART_IRQ_SW;
-		else
-			status_sw = 0;
-
-		status_csi = readl(isys->pdata->base + ctrl0_status);
-		status_sw |= readl(isys->pdata->base +
-				   IPU6_REG_ISYS_UNISPART_IRQ_STATUS);
-	} while ((status_csi & isys->isr_csi2_bits) ||
-		 (status_sw & IPU6_ISYS_UNISPART_IRQ_SW));
-
-	writel(ISYS_UNISPART_IRQS, base + IPU6_REG_ISYS_UNISPART_IRQ_MASK);
-
-	spin_unlock(&isys->power_lock);
-
-	return IRQ_HANDLED;
-}
-
-static void get_lut_ltrdid(struct ipu6_isys *isys, struct ltr_did *pltr_did)
-{
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-	struct ltr_did ltrdid_default;
-
-	ltrdid_default.lut_ltr.value = LTR_DEFAULT_VALUE;
-	ltrdid_default.lut_fill_time.value = FILL_TIME_DEFAULT_VALUE;
-
-	if (iwake_watermark->ltrdid.lut_ltr.value)
-		*pltr_did = iwake_watermark->ltrdid;
-	else
-		*pltr_did = ltrdid_default;
-}
-
 static int set_iwake_register(struct ipu6_isys *isys, u32 index, u32 value)
 {
 	struct device *dev = &isys->adev->auxdev.dev;
@@ -511,69 +446,57 @@ static void set_iwake_ltrdid(struct ipu6_isys *isys, u16 ltr, u16 did,
  */
 static void enable_iwake(struct ipu6_isys *isys, bool enable)
 {
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
 	int ret;
 
-	mutex_lock(&iwake_watermark->mutex);
-
-	if (iwake_watermark->iwake_enabled == enable) {
-		mutex_unlock(&iwake_watermark->mutex);
+	if (isys->iwake_watermark_enabled == enable)
 		return;
-	}
 
 	ret = set_iwake_register(isys, GDA_ENABLE_IWAKE_INDEX, enable);
 	if (!ret)
-		iwake_watermark->iwake_enabled = enable;
-
-	mutex_unlock(&iwake_watermark->mutex);
+		isys->iwake_watermark_enabled = enable;
 }
 
-void update_watermark_setting(struct ipu6_isys *isys)
+void ipu6_isys_update_watermark_setting(struct ipu6_isys *isys)
 {
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
 	u32 iwake_threshold, iwake_critical_threshold, page_num;
 	struct device *dev = &isys->adev->auxdev.dev;
 	u32 calc_fill_time_us = 0, ltr = 0, did = 0;
-	struct video_stream_watermark *p_watermark;
 	enum ltr_did_type ltr_did_type;
-	struct list_head *stream_node;
 	u64 isys_pb_datarate_mbs = 0;
 	u32 mem_open_threshold = 0;
 	struct ltr_did ltrdid;
 	u64 threshold_bytes;
 	u32 max_sram_size;
 	u32 shift;
+	bool force_iwake_disable = false;
+
+	lockdep_assert_held(&isys->stream_mutex);
 
 	shift = isys->pdata->ipdata->sram_gran_shift;
 	max_sram_size = isys->pdata->ipdata->max_sram_size;
 
-	mutex_lock(&iwake_watermark->mutex);
-	if (iwake_watermark->force_iwake_disable) {
+	for (unsigned int i = 0; i < isys->pdata->ipdata->csi2.nports; i++) {
+		isys_pb_datarate_mbs +=
+			isys->csi2[i].watermark.stream_data_rate;
+		force_iwake_disable |=
+			isys->csi2[i].watermark.force_iwake_disable;
+	}
+
+	if (force_iwake_disable) {
+		dev_dbg(dev, "watermark: forcing iwake disabled\n");
 		set_iwake_ltrdid(isys, 0, 0, LTR_IWAKE_OFF);
 		set_iwake_register(isys, GDA_IRQ_CRITICAL_THRESHOLD_INDEX,
 				   CRITICAL_THRESHOLD_IWAKE_DISABLE);
-		goto unlock_exit;
-	}
-
-	if (list_empty(&iwake_watermark->video_list)) {
-		isys_pb_datarate_mbs = 0;
-	} else {
-		list_for_each(stream_node, &iwake_watermark->video_list) {
-			p_watermark = list_entry(stream_node,
-						 struct video_stream_watermark,
-						 stream_node);
-			isys_pb_datarate_mbs += p_watermark->stream_data_rate;
-		}
+		return;
 	}
-	mutex_unlock(&iwake_watermark->mutex);
 
 	if (!isys_pb_datarate_mbs) {
+		dev_dbg(dev, "watermark: disabled iwake\n");
 		enable_iwake(isys, false);
 		set_iwake_ltrdid(isys, 0, 0, LTR_IWAKE_OFF);
-		mutex_lock(&iwake_watermark->mutex);
 		set_iwake_register(isys, GDA_IRQ_CRITICAL_THRESHOLD_INDEX,
 				   CRITICAL_THRESHOLD_IWAKE_DISABLE);
-		goto unlock_exit;
+		return;
 	}
 
 	enable_iwake(isys, true);
@@ -584,7 +507,8 @@ void update_watermark_setting(struct ipu6_isys *isys)
 		did = calc_fill_time_us * DEFAULT_DID_RATIO / 100;
 		ltr_did_type = LTR_ENHANNCE_IWAKE;
 	} else {
-		get_lut_ltrdid(isys, &ltrdid);
+		ltrdid.lut_ltr.value = LTR_DEFAULT_VALUE;
+		ltrdid.lut_fill_time.value = FILL_TIME_DEFAULT_VALUE;
 
 		if (calc_fill_time_us <= ltrdid.lut_fill_time.bits.th0)
 			ltr = 0;
@@ -608,7 +532,6 @@ void update_watermark_setting(struct ipu6_isys *isys)
 	iwake_threshold = max_t(u32, 1, threshold_bytes >> shift);
 	iwake_threshold = min_t(u32, iwake_threshold, max_sram_size);
 
-	mutex_lock(&iwake_watermark->mutex);
 	if (isys->pdata->ipdata->enhanced_iwake) {
 		set_iwake_register(isys, GDA_IWAKE_THRESHOLD_INDEX,
 				   DEFAULT_IWAKE_THRESHOLD);
@@ -630,7 +553,7 @@ void update_watermark_setting(struct ipu6_isys *isys)
 	iwake_critical_threshold = iwake_threshold +
 		(IS_PIXEL_BUFFER_PAGES - iwake_threshold) / 2;
 
-	dev_dbg(dev, "threshold: %u critical: %u\n", iwake_threshold,
+	dev_dbg(dev, "watermark: threshold: %u critical: %u\n", iwake_threshold,
 		iwake_critical_threshold);
 
 	set_iwake_register(isys, GDA_IRQ_CRITICAL_THRESHOLD_INDEX,
@@ -640,32 +563,6 @@ void update_watermark_setting(struct ipu6_isys *isys)
 	       isys->adev->isp->base + REG_PKGC_PMON_CFG);
 	writel(VAL_PKGC_PMON_CFG_START,
 	       isys->adev->isp->base + REG_PKGC_PMON_CFG);
-unlock_exit:
-	mutex_unlock(&iwake_watermark->mutex);
-}
-
-static void isys_iwake_watermark_init(struct ipu6_isys *isys)
-{
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-
-	INIT_LIST_HEAD(&iwake_watermark->video_list);
-	mutex_init(&iwake_watermark->mutex);
-
-	iwake_watermark->ltrdid.lut_ltr.value = 0;
-	iwake_watermark->isys = isys;
-	iwake_watermark->iwake_enabled = false;
-	iwake_watermark->force_iwake_disable = false;
-}
-
-static void isys_iwake_watermark_cleanup(struct ipu6_isys *isys)
-{
-	struct isys_iwake_watermark *iwake_watermark = &isys->iwake_watermark;
-
-	mutex_lock(&iwake_watermark->mutex);
-	list_del(&iwake_watermark->video_list);
-	mutex_unlock(&iwake_watermark->mutex);
-
-	mutex_destroy(&iwake_watermark->mutex);
 }
 
 /* The .bound() notifier callback when a match is found */
@@ -736,6 +633,10 @@ static int isys_notifier_init(struct ipu6_isys *isys)
 			continue;
 
 		ret = v4l2_fwnode_endpoint_parse(ep, &vep);
+		if (ret && IS_IPU7(isp)) {
+			vep.bus_type = V4L2_MBUS_CSI2_CPHY;
+			ret = v4l2_fwnode_endpoint_parse(ep, &vep);
+		}
 		if (ret) {
 			dev_err(dev, "fwnode endpoint parse failed: %d\n", ret);
 			goto err_parse;
@@ -751,6 +652,7 @@ static int isys_notifier_init(struct ipu6_isys *isys)
 
 		s_asd->csi2.port = vep.base.port;
 		s_asd->csi2.nlanes = vep.bus.mipi_csi2.num_data_lanes;
+		s_asd->csi2.bus_type = vep.bus_type;
 
 		dev_dbg(dev, "remote endpoint port %d with %d lanes added\n",
 			s_asd->csi2.port, s_asd->csi2.nlanes);
@@ -789,7 +691,7 @@ static int isys_register_devices(struct ipu6_isys *isys)
 
 	isys->media_dev.dev = dev;
 	media_device_pci_init(&isys->media_dev,
-			      pdev, IPU6_MEDIA_DEV_MODEL_NAME);
+			      pdev, isys->adev->isp->model_name);
 
 	strscpy(isys->v4l2_dev.name, isys->media_dev.model,
 		sizeof(isys->v4l2_dev.name));
@@ -853,9 +755,10 @@ static void isys_unregister_devices(struct ipu6_isys *isys)
 static int isys_runtime_pm_resume(struct device *dev)
 {
 	struct ipu6_bus_device *adev = to_ipu6_bus_device(dev);
+	const struct ipu6_fw_isys_ops *fw_ops = adev->auxdrv_data->fw_ops;
 	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	const struct ipu6_isys_internal_pdata *ipdata = isys->pdata->ipdata;
 	struct ipu6_device *isp = adev->isp;
-	unsigned long flags;
 	int ret;
 
 	ret = ipu6_mmu_hw_init(adev->mmu);
@@ -866,49 +769,82 @@ static int isys_runtime_pm_resume(struct device *dev)
 
 	ret = ipu6_buttress_start_tsc_sync(isp);
 	if (ret)
-		return ret;
+		goto err_mmu_hw_cleanup;
 
-	spin_lock_irqsave(&isys->power_lock, flags);
-	isys->power = 1;
-	spin_unlock_irqrestore(&isys->power_lock, flags);
+	if (IS_IPU7(isp)) {
+		ipu7_isys_setup_hw(isys);
+	} else {
+		ipu6_isys_setup_hw(isys);
+		set_iwake_ltrdid(isys, 0, 0, LTR_ISYS_ON);
+	}
 
-	isys_setup_hw(isys);
+	ipu6_configure_spc(adev->isp, &ipdata->hw_variant,
+			   IPU6_CPD_PKG_DIR_ISYS_SERVER_IDX, isys->pdata->base,
+			   adev->pkg_dir, adev->pkg_dir_dma_addr);
 
-	set_iwake_ltrdid(isys, 0, 0, LTR_ISYS_ON);
+	/*
+	 * Buffers could have been left to wrong queue at last closure.
+	 * Move them now back to empty buffer queue.
+	 */
+	ipu6_cleanup_fw_msg_bufs(isys);
 
-	return 0;
+	if (isys->fwctx) {
+		/*
+		 * Something went wrong in previous shutdown. As we are now
+		 * restarting isys we can safely delete old context.
+		 */
+		dev_warn(&adev->auxdev.dev, "clearing old context\n");
+		fw_ops->cleanup(isys);
+	}
+
+	ret = fw_ops->init(isys, ipdata->num_parallel_streams);
+	if (!ret)
+		return 0;
+
+	isys->phy_termcal_val = 0;
+	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
+
+	if (!IS_IPU7(isp))
+		set_iwake_ltrdid(isys, 0, 0, LTR_ISYS_OFF);
+
+err_mmu_hw_cleanup:
+	ipu6_mmu_hw_cleanup(adev->mmu);
+
+	return ret;
 }
 
 static int isys_runtime_pm_suspend(struct device *dev)
 {
 	struct ipu6_bus_device *adev = to_ipu6_bus_device(dev);
 	struct ipu6_isys *isys = dev_get_drvdata(dev);
-	unsigned long flags;
-
-	spin_lock_irqsave(&isys->power_lock, flags);
-	isys->power = 0;
-	spin_unlock_irqrestore(&isys->power_lock, flags);
+	struct ipu6_device *isp = adev->isp;
+	int ret = 0;
 
-	mutex_lock(&isys->mutex);
-	isys->need_reset = false;
-	mutex_unlock(&isys->mutex);
+	isys->adev->auxdrv_data->fw_ops->close(isys);
+	if (isys->fwctx) {
+		dev_warn(&isys->adev->auxdev.dev, "failed to close fw isys\n");
+		ret = -EIO;
+	}
 
 	isys->phy_termcal_val = 0;
 	cpu_latency_qos_update_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 
-	set_iwake_ltrdid(isys, 0, 0, LTR_ISYS_OFF);
+	if (!IS_IPU7(isp))
+		set_iwake_ltrdid(isys, 0, 0, LTR_ISYS_OFF);
 
 	ipu6_mmu_hw_cleanup(adev->mmu);
 
-	return 0;
+	return ret;
 }
 
 static int isys_suspend(struct device *dev)
 {
 	struct ipu6_isys *isys = dev_get_drvdata(dev);
 
+	guard(mutex)(&isys->stream_mutex);
+
 	/* If stream is open, refuse to suspend */
-	if (isys->stream_opened)
+	if (!ida_is_empty(&isys->streams))
 		return -EBUSY;
 
 	return 0;
@@ -1003,7 +939,7 @@ struct isys_fw_msgs *ipu6_get_fw_msg_buf(struct ipu6_isys_stream *stream)
 	msg = list_last_entry(&isys->framebuflist, struct isys_fw_msgs, head);
 	list_move(&msg->head, &isys->framebuflist_fw);
 	spin_unlock_irqrestore(&isys->listlock, flags);
-	memset(&msg->fw_msg, 0, sizeof(msg->fw_msg));
+	memset(&msg->ipu6, 0, sizeof(msg->ipu6));
 
 	return msg;
 }
@@ -1019,30 +955,28 @@ void ipu6_cleanup_fw_msg_bufs(struct ipu6_isys *isys)
 	spin_unlock_irqrestore(&isys->listlock, flags);
 }
 
-void ipu6_put_fw_msg_buf(struct ipu6_isys *isys, uintptr_t data)
+void ipu6_put_fw_msg_buf(struct ipu6_isys *isys, struct isys_fw_msgs *msg)
 {
-	struct isys_fw_msgs *msg;
 	unsigned long flags;
-	void *ptr = (void *)data;
 
-	if (!ptr)
+	if (!msg)
 		return;
 
 	spin_lock_irqsave(&isys->listlock, flags);
-	msg = container_of(ptr, struct isys_fw_msgs, fw_msg.dummy);
 	list_move(&msg->head, &isys->framebuflist);
 	spin_unlock_irqrestore(&isys->listlock, flags);
 }
 
+static const struct ipu6_auxdrv_data ipu6_isys_auxdrv_data;
+static const struct ipu6_auxdrv_data ipu7_isys_auxdrv_data;
+
 static int isys_probe(struct auxiliary_device *auxdev,
 		      const struct auxiliary_device_id *auxdev_id)
 {
 	const struct ipu6_isys_internal_csi2_pdata *csi2_pdata;
 	struct ipu6_bus_device *adev = auxdev_to_adev(auxdev);
 	struct ipu6_device *isp = adev->isp;
-	const struct firmware *fw;
 	struct ipu6_isys *isys;
-	unsigned int i;
 	int ret;
 
 	if (!isp->bus_ready_to_probe)
@@ -1052,8 +986,8 @@ static int isys_probe(struct auxiliary_device *auxdev,
 	if (!isys)
 		return -ENOMEM;
 
-	adev->auxdrv_data =
-		(const struct ipu6_auxdrv_data *)auxdev_id->driver_data;
+	adev->auxdrv_data = IS_IPU7(isp) ? &ipu7_isys_auxdrv_data :
+					   &ipu6_isys_auxdrv_data;
 	adev->auxdrv = to_auxiliary_drv(auxdev->dev.driver);
 	isys->adev = adev;
 	isys->pdata = adev->pdata;
@@ -1068,8 +1002,6 @@ static int isys_probe(struct auxiliary_device *auxdev,
 	isys->sensor_type = isys->pdata->ipdata->sensor_type_start;
 
 	spin_lock_init(&isys->streams_lock);
-	spin_lock_init(&isys->power_lock);
-	isys->power = 0;
 	isys->phy_termcal_val = 0;
 
 	mutex_init(&isys->mutex);
@@ -1083,18 +1015,7 @@ static int isys_probe(struct auxiliary_device *auxdev,
 
 	dev_set_drvdata(&auxdev->dev, isys);
 
-	isys_stream_init(isys);
-
-	if (!isp->secure_mode) {
-		fw = isp->cpd_fw;
-		ret = ipu6_buttress_map_fw_image(adev, fw, &adev->fw_sgt);
-		if (ret)
-			goto release_firmware;
-
-		ret = ipu6_cpd_create_pkg_dir(adev, isp->cpd_fw->data);
-		if (ret)
-			goto remove_shared_buffer;
-	}
+	ida_init(&isys->streams);
 
 	cpu_latency_qos_add_request(&isys->pm_qos, PM_QOS_DEFAULT_VALUE);
 
@@ -1102,11 +1023,11 @@ static int isys_probe(struct auxiliary_device *auxdev,
 	if (ret < 0)
 		goto out_remove_pkg_dir_shared_buffer;
 
-	isys_iwake_watermark_init(isys);
-
-	if (is_ipu6se(adev->isp->hw_ver))
+	if (IS_IPU7(adev->isp))
+		isys->phy_set_power = ipu7_isys_csi_phy_set_power;
+	else if (IS_IPU6SE(adev->isp))
 		isys->phy_set_power = ipu6_isys_jsl_phy_set_power;
-	else if (is_ipu6ep_mtl(adev->isp->hw_ver))
+	else if (IS_IPU6EP_MTL(adev->isp))
 		isys->phy_set_power = ipu6_isys_dwc_phy_set_power;
 	else
 		isys->phy_set_power = ipu6_isys_mcd_phy_set_power;
@@ -1121,17 +1042,6 @@ static int isys_probe(struct auxiliary_device *auxdev,
 	free_fw_msg_bufs(isys);
 out_remove_pkg_dir_shared_buffer:
 	cpu_latency_qos_remove_request(&isys->pm_qos);
-	if (!isp->secure_mode)
-		ipu6_cpd_free_pkg_dir(adev);
-remove_shared_buffer:
-	if (!isp->secure_mode)
-		ipu6_buttress_unmap_fw_image(adev, &adev->fw_sgt);
-release_firmware:
-	if (!isp->secure_mode)
-		release_firmware(adev->fw);
-
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++)
-		mutex_destroy(&isys->streams[i].mutex);
 
 	mutex_destroy(&isys->mutex);
 	mutex_destroy(&isys->stream_mutex);
@@ -1141,10 +1051,9 @@ static int isys_probe(struct auxiliary_device *auxdev,
 
 static void isys_remove(struct auxiliary_device *auxdev)
 {
-	struct ipu6_bus_device *adev = auxdev_to_adev(auxdev);
 	struct ipu6_isys *isys = dev_get_drvdata(&auxdev->dev);
-	struct ipu6_device *isp = adev->isp;
-	unsigned int i;
+
+	ida_destroy(&isys->streams);
 
 	free_fw_msg_bufs(isys);
 
@@ -1153,191 +1062,27 @@ static void isys_remove(struct auxiliary_device *auxdev)
 
 	cpu_latency_qos_remove_request(&isys->pm_qos);
 
-	if (!isp->secure_mode) {
-		ipu6_cpd_free_pkg_dir(adev);
-		ipu6_buttress_unmap_fw_image(adev, &adev->fw_sgt);
-		release_firmware(adev->fw);
-	}
-
-	for (i = 0; i < IPU6_ISYS_MAX_STREAMS; i++)
-		mutex_destroy(&isys->streams[i].mutex);
-
-	isys_iwake_watermark_cleanup(isys);
 	mutex_destroy(&isys->stream_mutex);
 	mutex_destroy(&isys->mutex);
 }
 
-struct fwmsg {
-	int type;
-	char *msg;
-	bool valid_ts;
-};
-
-static const struct fwmsg fw_msg[] = {
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_OPEN_DONE, "STREAM_OPEN_DONE", 0},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_CLOSE_ACK, "STREAM_CLOSE_ACK", 0},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_START_ACK, "STREAM_START_ACK", 0},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK,
-	 "STREAM_START_AND_CAPTURE_ACK", 0},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_STOP_ACK, "STREAM_STOP_ACK", 0},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_FLUSH_ACK, "STREAM_FLUSH_ACK", 0},
-	{IPU6_FW_ISYS_RESP_TYPE_PIN_DATA_READY, "PIN_DATA_READY", 1},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_ACK, "STREAM_CAPTURE_ACK", 0},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE,
-	 "STREAM_START_AND_CAPTURE_DONE", 1},
-	{IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_DONE, "STREAM_CAPTURE_DONE", 1},
-	{IPU6_FW_ISYS_RESP_TYPE_FRAME_SOF, "FRAME_SOF", 1},
-	{IPU6_FW_ISYS_RESP_TYPE_FRAME_EOF, "FRAME_EOF", 1},
-	{IPU6_FW_ISYS_RESP_TYPE_STATS_DATA_READY, "STATS_READY", 1},
-	{-1, "UNKNOWN MESSAGE", 0}
+static const struct ipu6_auxdrv_data ipu6_isys_auxdrv_data = {
+	.isr = ipu6_isys_isr,
+	.isr_threaded = NULL,
+	.wake_isr_thread = false,
+	.fw_ops = &ipu6_fw_isys_ops,
 };
 
-static u32 resp_type_to_index(int type)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(fw_msg); i++)
-		if (fw_msg[i].type == type)
-			return i;
-
-	return  ARRAY_SIZE(fw_msg) - 1;
-}
-
-static int isys_isr_one(struct ipu6_bus_device *adev)
-{
-	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
-	struct ipu6_fw_isys_resp_info_abi *resp;
-	struct ipu6_isys_stream *stream;
-	struct ipu6_isys_csi2 *csi2 = NULL;
-	u32 index;
-	u64 ts;
-
-	if (!isys->fwcom)
-		return 1;
-
-	resp = ipu6_fw_isys_get_resp(isys->fwcom, IPU6_BASE_MSG_RECV_QUEUES);
-	if (!resp)
-		return 1;
-
-	ts = (u64)resp->timestamp[1] << 32 | resp->timestamp[0];
-
-	index = resp_type_to_index(resp->type);
-	dev_dbg(&adev->auxdev.dev,
-		"FW resp %02d %s, stream %u, ts 0x%16.16llx, pin %d\n",
-		resp->type, fw_msg[index].msg, resp->stream_handle,
-		fw_msg[index].valid_ts ? ts : 0, resp->pin_id);
-
-	if (resp->error_info.error == IPU6_FW_ISYS_ERROR_STREAM_IN_SUSPENSION)
-		/* Suspension is kind of special case: not enough buffers */
-		dev_dbg(&adev->auxdev.dev,
-			"FW error resp SUSPENSION, details %d\n",
-			resp->error_info.error_details);
-	else if (resp->error_info.error)
-		dev_dbg(&adev->auxdev.dev,
-			"FW error resp error %d, details %d\n",
-			resp->error_info.error, resp->error_info.error_details);
-
-	if (resp->stream_handle >= IPU6_ISYS_MAX_STREAMS) {
-		dev_err(&adev->auxdev.dev, "bad stream handle %u\n",
-			resp->stream_handle);
-		goto leave;
-	}
-
-	stream = ipu6_isys_query_stream_by_handle(isys, resp->stream_handle);
-	if (!stream) {
-		dev_err(&adev->auxdev.dev, "stream of stream_handle %u is unused\n",
-			resp->stream_handle);
-		goto leave;
-	}
-	stream->error = resp->error_info.error;
-
-	csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
-
-	switch (resp->type) {
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_OPEN_DONE:
-		complete(&stream->stream_open_completion);
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_CLOSE_ACK:
-		complete(&stream->stream_close_completion);
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_START_ACK:
-		complete(&stream->stream_start_completion);
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK:
-		complete(&stream->stream_start_completion);
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_STOP_ACK:
-		complete(&stream->stream_stop_completion);
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_FLUSH_ACK:
-		complete(&stream->stream_stop_completion);
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_PIN_DATA_READY:
-		/*
-		 * firmware only release the capture msg until software
-		 * get pin_data_ready event
-		 */
-		ipu6_put_fw_msg_buf(ipu6_bus_get_drvdata(adev), resp->buf_id);
-		if (resp->pin_id < IPU6_ISYS_OUTPUT_PINS &&
-		    stream->output_pins_queue[resp->pin_id])
-			ipu6_isys_queue_buf_ready(stream, resp);
-		else
-			dev_warn(&adev->auxdev.dev,
-				 "%d:No queue for pin id %d\n",
-				 resp->stream_handle, resp->pin_id);
-		if (csi2)
-			ipu6_isys_csi2_error(csi2);
-
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_ACK:
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE:
-	case IPU6_FW_ISYS_RESP_TYPE_STREAM_CAPTURE_DONE:
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_FRAME_SOF:
-
-		ipu6_isys_csi2_sof_event_by_stream(stream);
-		stream->seq[stream->seq_index].sequence =
-			atomic_read(&stream->sequence) - 1;
-		stream->seq[stream->seq_index].timestamp = ts;
-		dev_dbg(&adev->auxdev.dev,
-			"sof: handle %d: (index %u), timestamp 0x%16.16llx\n",
-			resp->stream_handle,
-			stream->seq[stream->seq_index].sequence, ts);
-		stream->seq_index = (stream->seq_index + 1)
-			% IPU6_ISYS_MAX_PARALLEL_SOF;
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_FRAME_EOF:
-		ipu6_isys_csi2_eof_event_by_stream(stream);
-		dev_dbg(&adev->auxdev.dev,
-			"eof: handle %d: (index %u), timestamp 0x%16.16llx\n",
-			resp->stream_handle,
-			stream->seq[stream->seq_index].sequence, ts);
-		break;
-	case IPU6_FW_ISYS_RESP_TYPE_STATS_DATA_READY:
-		break;
-	default:
-		dev_err(&adev->auxdev.dev, "%d:unknown response type %u\n",
-			resp->stream_handle, resp->type);
-		break;
-	}
-
-	ipu6_isys_put_stream(stream);
-leave:
-	ipu6_fw_isys_put_resp(isys->fwcom, IPU6_BASE_MSG_RECV_QUEUES);
-	return 0;
-}
-
-static const struct ipu6_auxdrv_data ipu6_isys_auxdrv_data = {
-	.isr = isys_isr,
+static const struct ipu6_auxdrv_data ipu7_isys_auxdrv_data = {
+	.isr = ipu7_isys_isr,
 	.isr_threaded = NULL,
 	.wake_isr_thread = false,
+	.fw_ops = &ipu7_fw_isys_ops,
 };
 
 static const struct auxiliary_device_id ipu6_isys_id_table[] = {
 	{
 		.name = "intel_ipu6.isys",
-		.driver_data = (kernel_ulong_t)&ipu6_isys_auxdrv_data,
 	},
 	{ }
 };
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.h b/drivers/media/pci/intel/ipu6/ipu6-isys.h
index 7fb8cb820912b9..2af20f56a9653b 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys.h
@@ -4,6 +4,7 @@
 #ifndef IPU6_ISYS_H
 #define IPU6_ISYS_H
 
+#include <linux/idr.h>
 #include <linux/irqreturn.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
@@ -14,9 +15,11 @@
 #include <media/media-device.h>
 #include <media/v4l2-async.h>
 #include <media/v4l2-device.h>
+#include <media/v4l2-mediabus.h>
 
 #include "ipu6.h"
 #include "ipu6-fw-isys.h"
+#include "ipu7-fw-isys.h"
 #include "ipu6-isys-csi2.h"
 #include "ipu6-isys-video.h"
 
@@ -60,41 +63,10 @@ struct ipu6_bus_device;
 #define IPU6EP_MTL_LTR_VALUE			1023
 #define IPU6EP_MTL_MIN_MEMOPEN_TH		0xc
 
-struct ltr_did {
-	union {
-		u32 value;
-		struct {
-			u8 val0;
-			u8 val1;
-			u8 val2;
-			u8 val3;
-		} bits;
-	} lut_ltr;
-	union {
-		u32 value;
-		struct {
-			u8 th0;
-			u8 th1;
-			u8 th2;
-			u8 th3;
-		} bits;
-	} lut_fill_time;
-};
-
-struct isys_iwake_watermark {
-	bool iwake_enabled;
-	bool force_iwake_disable;
-	u32 iwake_threshold;
-	u64 isys_pixelbuffer_datarate;
-	struct ltr_did ltrdid;
-	struct mutex mutex; /* protect whole struct */
-	struct ipu6_isys *isys;
-	struct list_head video_list;
-};
-
 struct ipu6_isys_csi2_config {
 	u32 nlanes;
 	u32 port;
+	enum v4l2_mbus_type bus_type;
 };
 
 struct sensor_async_sd {
@@ -108,17 +80,12 @@ struct sensor_async_sd {
  * @media_dev: Media device
  * @v4l2_dev: V4L2 device
  * @adev: ISYS bus device
- * @power: Is ISYS powered on or not?
- * @isr_bits: Which bits does the ISR handle?
- * @power_lock: Serialise access to power (power state in general)
  * @csi2_rx_ctrl_cached: cached shared value between all CSI2 receivers
  * @streams_lock: serialise access to streams
  * @streams: streams per firmware stream ID
- * @fwcom: fw communication layer private pointer
- *         or optional external library private pointer
+ * @fwctx: fw communication layer context pointer
  * @phy_termcal_val: the termination calibration value, only used for DWC PHY
  * @need_reset: Isys requires d0i0->i3 transition
- * @ref_count: total number of callers fw open
  * @mutex: serialise access isys video open/release related operations
  * @stream_mutex: serialise stream start and stop, queueing requests
  * @pdata: platform data pointer
@@ -129,20 +96,17 @@ struct ipu6_isys {
 	struct v4l2_device v4l2_dev;
 	struct ipu6_bus_device *adev;
 
-	int power;
-	spinlock_t power_lock;
 	u32 isr_csi2_bits;
 	u32 csi2_rx_ctrl_cached;
 	spinlock_t streams_lock;
-	struct ipu6_isys_stream streams[IPU6_ISYS_MAX_STREAMS];
-	int streams_ref_count[IPU6_ISYS_MAX_STREAMS];
-	void *fwcom;
+	struct ipu6_isys_stream *streams_by_handle[IPU6_ISYS_MAX_STREAMS];
+	void *fwctx;
 	u32 phy_termcal_val;
+	u32 phy_rext_cal;
 	bool need_reset;
 	bool icache_prefetch;
 	bool csi2_cse_ipc_not_supported;
-	unsigned int ref_count;
-	unsigned int stream_opened;
+	bool iwake_watermark_enabled;
 	unsigned int sensor_type;
 
 	struct mutex mutex;
@@ -162,26 +126,65 @@ struct ipu6_isys {
 	struct list_head framebuflist;
 	struct list_head framebuflist_fw;
 	struct v4l2_async_notifier notifier;
-	struct isys_iwake_watermark iwake_watermark;
+	struct ida streams;
 };
 
 struct isys_fw_msgs {
 	union {
 		u64 dummy;
-		struct ipu6_fw_isys_frame_buff_set_abi frame;
-		struct ipu6_fw_isys_stream_cfg_data_abi stream;
-	} fw_msg;
+		union {
+			struct ipu6_fw_isys_frame_buff_set_abi frame;
+			struct ipu6_fw_isys_stream_cfg_data_abi stream;
+		} ipu6;
+		union {
+			struct ipu7_fw_isys_frame_buff_set frame;
+			struct ipu7_fw_isys_stream_cfg stream;
+		} ipu7;
+	};
 	struct list_head head;
 	dma_addr_t dma_addr;
 };
 
+struct ipu6_fw_isys_ops {
+	int (*init)(struct ipu6_isys *isys, unsigned int num_streams);
+	int (*close)(struct ipu6_isys *isys);
+	int (*send_cmd)(struct ipu6_isys *isys,
+			const unsigned int stream_handle,
+			void *cpu_mapped_buf,
+			dma_addr_t dma_mapped_buf,
+			size_t size, u16 send_type);
+	void (*cleanup)(struct ipu6_isys *isys);
+	int (*prepare_stream_cfg)(struct ipu6_isys_stream *stream,
+				  struct v4l2_mbus_frame_desc *desc,
+				  struct isys_fw_msgs *msg);
+	void (*prepare_buf_set)(struct isys_fw_msgs *msg,
+				struct ipu6_isys_stream *stream,
+				struct ipu6_isys_buffer_list *bl);
+	int (*stream_open)(struct ipu6_isys *isys,
+			   const unsigned int stream_handle,
+			   struct isys_fw_msgs *msg);
+	int (*stream_start)(struct ipu6_isys *isys,
+			    const unsigned int stream_handle,
+			    struct isys_fw_msgs *msg, bool capture);
+	int (*stream_capture)(struct ipu6_isys *isys,
+			      const unsigned int stream_handle,
+			      struct isys_fw_msgs *msg);
+	int (*stream_flush)(struct ipu6_isys *isys,
+			    const unsigned int stream_handle);
+	int (*stream_close)(struct ipu6_isys *isys,
+			    const unsigned int stream_handle);
+	void (*dump_stream_cfg)(struct device *dev, struct isys_fw_msgs *msg);
+	void (*dump_frame_buf_set)(struct device *dev, struct isys_fw_msgs *msg,
+				   unsigned int outputs);
+};
+
 struct isys_fw_msgs *ipu6_get_fw_msg_buf(struct ipu6_isys_stream *stream);
-void ipu6_put_fw_msg_buf(struct ipu6_isys *isys, uintptr_t data);
+void ipu6_put_fw_msg_buf(struct ipu6_isys *isys, struct isys_fw_msgs *msg);
 void ipu6_cleanup_fw_msg_bufs(struct ipu6_isys *isys);
 
 extern const struct v4l2_ioctl_ops ipu6_isys_ioctl_ops;
 
-void update_watermark_setting(struct ipu6_isys *isys);
+void ipu6_isys_update_watermark_setting(struct ipu6_isys *isys);
 
 int ipu6_isys_mcd_phy_set_power(struct ipu6_isys *isys,
 				struct ipu6_isys_csi2_config *cfg,
diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c b/drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c
new file mode 100644
index 00000000000000..2b395fbd596917
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu6-mmu-hw.c
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Intel Corporation
+ */
+#include <asm/barrier.h>
+
+#include <linux/bits.h>
+#include <linux/gfp.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+#include "ipu6.h"
+#include "ipu6-dma.h"
+#include "ipu6-mmu.h"
+#include "ipu6-platform-regs.h"
+
+#define ISP_PAGE_SHIFT		12
+#define ISP_PAGE_SIZE		BIT(ISP_PAGE_SHIFT)
+#define ISP_PAGE_MASK		(~(ISP_PAGE_SIZE - 1))
+
+#define ISP_L1PT_SHIFT		22
+#define ISP_L1PT_MASK		(~((1U << ISP_L1PT_SHIFT) - 1))
+
+#define ISP_L2PT_SHIFT		12
+#define ISP_L2PT_MASK		(~(ISP_L1PT_MASK | (~(ISP_PAGE_MASK))))
+
+#define ISP_L1PT_PTES           1024
+#define ISP_L2PT_PTES           1024
+
+#define ISP_PADDR_SHIFT		12
+
+#define REG_TLB_INVALIDATE	0x0000
+
+#define REG_L1_PHYS		0x0004	/* 27-bit pfn */
+#define REG_INFO		0x0008
+
+#define TBL_PHYS_ADDR(a)	((phys_addr_t)(a) << ISP_PADDR_SHIFT)
+
+static struct ipu6_mmu_hw ipu6_isys_mmu_hwdata[] = {
+	{
+		.offset = IPU6_ISYS_IOMMU0_OFFSET,
+		.info_bits = IPU6_INFO_REQUEST_DESTINATION_IOSF,
+		.nr_l1streams = 16,
+		.l1_block_sz = {
+			3, 8, 2, 2, 2, 2, 2, 2, 1, 1,
+			1, 1, 1, 1, 1, 1
+		},
+		.nr_l2streams = 16,
+		.l2_block_sz = {
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2
+		},
+		.insert_read_before_invalidate = false,
+		.l1_stream_id_reg_offset =
+		IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+		.l2_stream_id_reg_offset =
+		IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+	},
+	{
+		.offset = IPU6_ISYS_IOMMU1_OFFSET,
+		.info_bits = 0,
+		.nr_l1streams = 16,
+		.l1_block_sz = {
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 1, 1, 4
+		},
+		.nr_l2streams = 16,
+		.l2_block_sz = {
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2
+		},
+		.insert_read_before_invalidate = false,
+		.l1_stream_id_reg_offset =
+		IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+		.l2_stream_id_reg_offset =
+		IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+	},
+	{
+		.offset = IPU6_ISYS_IOMMUI_OFFSET,
+		.info_bits = 0,
+		.nr_l1streams = 0,
+		.nr_l2streams = 0,
+		.insert_read_before_invalidate = false,
+	},
+};
+
+static struct ipu6_mmu_hw ipu6_psys_mmu_hwdata[] = {
+	{
+		.offset = IPU6_PSYS_IOMMU0_OFFSET,
+		.info_bits =
+		IPU6_INFO_REQUEST_DESTINATION_IOSF,
+		.nr_l1streams = 16,
+		.l1_block_sz = {
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2
+		},
+		.nr_l2streams = 16,
+		.l2_block_sz = {
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2
+		},
+		.insert_read_before_invalidate = false,
+		.l1_stream_id_reg_offset =
+		IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+		.l2_stream_id_reg_offset =
+		IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+	},
+	{
+		.offset = IPU6_PSYS_IOMMU1_OFFSET,
+		.info_bits = 0,
+		.nr_l1streams = 32,
+		.l1_block_sz = {
+			1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 10,
+			5, 4, 14, 6, 4, 14, 6, 4, 8,
+			4, 2, 1, 1, 1, 1, 14
+		},
+		.nr_l2streams = 32,
+		.l2_block_sz = {
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2
+		},
+		.insert_read_before_invalidate = false,
+		.l1_stream_id_reg_offset =
+		IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+		.l2_stream_id_reg_offset =
+		IPU6_PSYS_MMU1W_L2_STREAM_ID_REG_OFFSET,
+	},
+	{
+		.offset = IPU6_PSYS_IOMMU1R_OFFSET,
+		.info_bits = 0,
+		.nr_l1streams = 16,
+		.l1_block_sz = {
+			1, 4, 4, 4, 4, 16, 8, 4, 32,
+			16, 16, 2, 2, 2, 1, 12
+		},
+		.nr_l2streams = 16,
+		.l2_block_sz = {
+			2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+			2, 2, 2, 2, 2, 2
+		},
+		.insert_read_before_invalidate = false,
+		.l1_stream_id_reg_offset =
+		IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
+		.l2_stream_id_reg_offset =
+		IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
+	},
+	{
+		.offset = IPU6_PSYS_IOMMUI_OFFSET,
+		.info_bits = 0,
+		.nr_l1streams = 0,
+		.nr_l2streams = 0,
+		.insert_read_before_invalidate = false,
+	},
+};
+
+struct ipu6_mmu_hwdata {
+	struct ipu6_mmu_hw *hwdata;
+	unsigned int nr_mmus;
+};
+
+static const struct ipu6_mmu_hwdata ipu6_mmu_hwdata_lookup[IPU_SUBSYS_NUM] = {
+	[IPU_PSYS] = {
+		.hwdata = ipu6_psys_mmu_hwdata,
+		.nr_mmus = ARRAY_SIZE(ipu6_psys_mmu_hwdata),
+	},
+	[IPU_ISYS] = {
+		.hwdata = ipu6_isys_mmu_hwdata,
+		.nr_mmus = ARRAY_SIZE(ipu6_isys_mmu_hwdata),
+	},
+};
+
+static void __ipu6_tlb_invalidate(struct ipu6_mmu *mmu)
+{
+	struct ipu6_mmu_hw *mmu_hw = mmu->ipu6_mmu_hw;
+	unsigned long flags;
+	unsigned int i;
+
+	spin_lock_irqsave(&mmu->ready_lock, flags);
+	if (!mmu->ready) {
+		spin_unlock_irqrestore(&mmu->ready_lock, flags);
+		return;
+	}
+
+	for (i = 0; i < mmu->nr_mmus; i++) {
+		/*
+		 * To avoid the HW bug induced dead lock in some of the IPU6
+		 * MMUs on successive invalidate calls, we need to first do a
+		 * read to the page table base before writing the invalidate
+		 * register. MMUs which need to implement this WA, will have
+		 * the insert_read_before_invalidate flags set as true.
+		 * Disregard the return value of the read.
+		 */
+		if (mmu_hw[i].insert_read_before_invalidate)
+			readl(mmu_hw[i].base + REG_L1_PHYS);
+
+		writel(0xffffffff, mmu_hw[i].base + REG_TLB_INVALIDATE);
+		/*
+		 * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
+		 * When the actual MMIO write reaches the IPU6 TLB Invalidate
+		 * register, wmb() will force the TLB invalidate out if the CPU
+		 * attempts to update the IOMMU page table (or sooner).
+		 */
+		wmb();
+	}
+	spin_unlock_irqrestore(&mmu->ready_lock, flags);
+}
+
+static int __ipu6_mmu_hw_init(struct ipu6_mmu *mmu)
+{
+	struct ipu6_mmu_info *mmu_info = mmu->dmap->mmu_info;
+	struct ipu6_mmu_hw *mmu_hw = mmu->ipu6_mmu_hw;
+
+	/* Initialise the each MMU HW block */
+	for (unsigned int i = 0; i < mmu->nr_mmus; i++) {
+		unsigned int j;
+		u16 block_addr;
+
+		/* Write page table address per MMU */
+		writel((phys_addr_t)mmu_info->l1_pt_dma,
+		       mmu_hw[i].base + REG_L1_PHYS);
+
+		/* Set info bits per MMU */
+		writel(mmu_hw[i].info_bits, mmu_hw[i].base + REG_INFO);
+
+		/* Configure MMU TLB stream configuration for L1 */
+		for (j = 0, block_addr = 0; j < mmu_hw[i].nr_l1streams;
+		     block_addr += mmu_hw[i].l1_block_sz[j], j++) {
+			if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) {
+				dev_err(mmu->dev, "invalid L1 configuration\n");
+				return -EINVAL;
+			}
+
+			/* Write block start address for each streams */
+			writel(block_addr, mmu_hw[i].base +
+			       mmu_hw[i].l1_stream_id_reg_offset + 4 * j);
+		}
+
+		/* Configure MMU TLB stream configuration for L2 */
+		for (j = 0, block_addr = 0; j < mmu_hw[i].nr_l2streams;
+		     block_addr += mmu_hw[i].l2_block_sz[j], j++) {
+			if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) {
+				dev_err(mmu->dev, "invalid L2 configuration\n");
+				return -EINVAL;
+			}
+
+			writel(block_addr, mmu_hw[i].base +
+			       mmu_hw[i].l2_stream_id_reg_offset + 4 * j);
+		}
+	}
+
+	return 0;
+}
+
+static int __ipu6_mmu_init_hw_data(struct ipu6_mmu *mmu, struct device *dev,
+				   void __iomem *base)
+{
+	const struct ipu6_mmu_hwdata *lookup;
+	struct ipu6_mmu_hw *mmu_hw, *src;
+	unsigned int i, nr_mmus;
+
+	if (mmu->mmid >= IPU_SUBSYS_NUM)
+		return -EINVAL;
+
+	lookup = &ipu6_mmu_hwdata_lookup[mmu->mmid];
+	src = lookup->hwdata;
+	nr_mmus = lookup->nr_mmus;
+
+	mmu_hw = devm_kcalloc(dev, nr_mmus, sizeof(*mmu_hw), GFP_KERNEL);
+	if (!mmu_hw)
+		return -ENOMEM;
+
+	for (i = 0; i < nr_mmus; i++) {
+		if (src[i].nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS ||
+		    src[i].nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS)
+			return -EINVAL;
+
+		mmu_hw[i] = src[i];
+		mmu_hw[i].base = base + src[i].offset;
+	}
+
+	mmu->nr_mmus = nr_mmus;
+	mmu->ipu6_mmu_hw = mmu_hw;
+
+	return 0;
+}
+
+const struct ipu6_mmu_hw_ops ipu6_mmu_ops = {
+	.init_hw_data = __ipu6_mmu_init_hw_data,
+	.hw_init = __ipu6_mmu_hw_init,
+	.tlb_invalidate = __ipu6_tlb_invalidate,
+};
diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu.c b/drivers/media/pci/intel/ipu6/ipu6-mmu.c
index 0e6dca36b6ba65..243d438786ba9e 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-mmu.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-mmu.c
@@ -5,7 +5,6 @@
 #include <asm/barrier.h>
 
 #include <linux/align.h>
-#include <linux/atomic.h>
 #include <linux/bitops.h>
 #include <linux/bits.h>
 #include <linux/bug.h>
@@ -13,10 +12,8 @@
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/gfp.h>
-#include <linux/io.h>
 #include <linux/iova.h>
 #include <linux/math.h>
-#include <linux/minmax.h>
 #include <linux/mm.h>
 #include <linux/pfn.h>
 #include <linux/slab.h>
@@ -51,42 +48,6 @@
 
 #define TBL_PHYS_ADDR(a)	((phys_addr_t)(a) << ISP_PADDR_SHIFT)
 
-static void tlb_invalidate(struct ipu6_mmu *mmu)
-{
-	unsigned long flags;
-	unsigned int i;
-
-	spin_lock_irqsave(&mmu->ready_lock, flags);
-	if (!mmu->ready) {
-		spin_unlock_irqrestore(&mmu->ready_lock, flags);
-		return;
-	}
-
-	for (i = 0; i < mmu->nr_mmus; i++) {
-		/*
-		 * To avoid the HW bug induced dead lock in some of the IPU6
-		 * MMUs on successive invalidate calls, we need to first do a
-		 * read to the page table base before writing the invalidate
-		 * register. MMUs which need to implement this WA, will have
-		 * the insert_read_before_invalidate flags set as true.
-		 * Disregard the return value of the read.
-		 */
-		if (mmu->mmu_hw[i].insert_read_before_invalidate)
-			readl(mmu->mmu_hw[i].base + REG_L1_PHYS);
-
-		writel(0xffffffff, mmu->mmu_hw[i].base +
-		       REG_TLB_INVALIDATE);
-		/*
-		 * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
-		 * When the actual MMIO write reaches the IPU6 TLB Invalidate
-		 * register, wmb() will force the TLB invalidate out if the CPU
-		 * attempts to update the IOMMU page table (or sooner).
-		 */
-		wmb();
-	}
-	spin_unlock_irqrestore(&mmu->ready_lock, flags);
-}
-
 #ifdef DEBUG
 static void page_table_dump(struct ipu6_mmu_info *mmu_info)
 {
@@ -472,55 +433,14 @@ static int allocate_trash_buffer(struct ipu6_mmu *mmu)
 
 int ipu6_mmu_hw_init(struct ipu6_mmu *mmu)
 {
-	struct ipu6_mmu_info *mmu_info;
 	unsigned long flags;
-	unsigned int i;
-
-	mmu_info = mmu->dmap->mmu_info;
-
-	/* Initialise the each MMU HW block */
-	for (i = 0; i < mmu->nr_mmus; i++) {
-		struct ipu6_mmu_hw *mmu_hw = &mmu->mmu_hw[i];
-		unsigned int j;
-		u16 block_addr;
-
-		/* Write page table address per MMU */
-		writel((phys_addr_t)mmu_info->l1_pt_dma,
-		       mmu->mmu_hw[i].base + REG_L1_PHYS);
-
-		/* Set info bits per MMU */
-		writel(mmu->mmu_hw[i].info_bits,
-		       mmu->mmu_hw[i].base + REG_INFO);
-
-		/* Configure MMU TLB stream configuration for L1 */
-		for (j = 0, block_addr = 0; j < mmu_hw->nr_l1streams;
-		     block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) {
-			if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) {
-				dev_err(mmu->dev, "invalid L1 configuration\n");
-				return -EINVAL;
-			}
-
-			/* Write block start address for each streams */
-			writel(block_addr, mmu_hw->base +
-			       mmu_hw->l1_stream_id_reg_offset + 4 * j);
-		}
-
-		/* Configure MMU TLB stream configuration for L2 */
-		for (j = 0, block_addr = 0; j < mmu_hw->nr_l2streams;
-		     block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) {
-			if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) {
-				dev_err(mmu->dev, "invalid L2 configuration\n");
-				return -EINVAL;
-			}
+	int ret;
 
-			writel(block_addr, mmu_hw->base +
-			       mmu_hw->l2_stream_id_reg_offset + 4 * j);
-		}
-	}
+	ret = mmu->ops->hw_init(mmu);
+	if (ret)
+		return ret;
 
 	if (!mmu->trash_page) {
-		int ret;
-
 		mmu->trash_page = alloc_page(GFP_KERNEL);
 		if (!mmu->trash_page) {
 			dev_err(mmu->dev, "insufficient memory for trash buffer\n");
@@ -553,7 +473,12 @@ static struct ipu6_mmu_info *ipu6_mmu_alloc(struct ipu6_device *isp)
 	if (!mmu_info)
 		return NULL;
 
-	mmu_info->aperture_start = 0;
+	if (IS_IPU7(isp))
+		mmu_info->aperture_start = isp->secure_mode ?
+			IPU7_FW_CODE_REGION_END : IPU7_FW_CODE_REGION_START;
+	else
+		mmu_info->aperture_start = 0;
+
 	mmu_info->aperture_end =
 		(dma_addr_t)DMA_BIT_MASK(isp->secure_mode ?
 					 IPU6_MMU_ADDR_BITS :
@@ -612,6 +537,7 @@ EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_cleanup, "INTEL_IPU6");
 static struct ipu6_dma_mapping *alloc_dma_mapping(struct ipu6_device *isp)
 {
 	struct ipu6_dma_mapping *dmap;
+	unsigned long base_pfn;
 
 	dmap = kzalloc_obj(*dmap);
 	if (!dmap)
@@ -623,7 +549,9 @@ static struct ipu6_dma_mapping *alloc_dma_mapping(struct ipu6_device *isp)
 		return NULL;
 	}
 
-	init_iova_domain(&dmap->iovad, SZ_4K, 1);
+	base_pfn = max_t(unsigned long, 1,
+			 PFN_DOWN(dmap->mmu_info->aperture_start));
+	init_iova_domain(&dmap->iovad, SZ_4K, base_pfn);
 	dmap->mmu_info->dmap = dmap;
 
 	dev_dbg(&isp->pdev->dev, "alloc mapping\n");
@@ -746,45 +674,26 @@ static void ipu6_mmu_destroy(struct ipu6_mmu *mmu)
 }
 
 struct ipu6_mmu *ipu6_mmu_init(struct device *dev,
-			       void __iomem *base, int mmid,
-			       const struct ipu6_hw_variants *hw)
+			       void __iomem *base, int mmid)
 {
 	struct ipu6_device *isp = pci_get_drvdata(to_pci_dev(dev));
-	struct ipu6_mmu_pdata *pdata;
 	struct ipu6_mmu *mmu;
-	unsigned int i;
-
-	if (hw->nr_mmus > IPU6_MMU_MAX_DEVICES)
-		return ERR_PTR(-EINVAL);
-
-	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata)
-		return ERR_PTR(-ENOMEM);
-
-	for (i = 0; i < hw->nr_mmus; i++) {
-		struct ipu6_mmu_hw *pdata_mmu = &pdata->mmu_hw[i];
-		const struct ipu6_mmu_hw *src_mmu = &hw->mmu_hw[i];
-
-		if (src_mmu->nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS ||
-		    src_mmu->nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS)
-			return ERR_PTR(-EINVAL);
-
-		*pdata_mmu = *src_mmu;
-		pdata_mmu->base = base + src_mmu->offset;
-	}
+	int ret;
 
 	mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);
 	if (!mmu)
 		return ERR_PTR(-ENOMEM);
 
+	mmu->ops = IS_IPU7(isp) ? &ipu7_mmu_ops : &ipu6_mmu_ops;
 	mmu->mmid = mmid;
-	mmu->mmu_hw = pdata->mmu_hw;
-	mmu->nr_mmus = hw->nr_mmus;
-	mmu->tlb_invalidate = tlb_invalidate;
 	mmu->ready = false;
 	INIT_LIST_HEAD(&mmu->vma_list);
 	spin_lock_init(&mmu->ready_lock);
 
+	ret = mmu->ops->init_hw_data(mmu, dev, base);
+	if (ret)
+		return ERR_PTR(ret);
+
 	mmu->dmap = alloc_dma_mapping(isp);
 	if (!mmu->dmap) {
 		dev_err(dev, "can't alloc dma mapping\n");
diff --git a/drivers/media/pci/intel/ipu6/ipu6-mmu.h b/drivers/media/pci/intel/ipu6/ipu6-mmu.h
index 8e40b4a66d7dd7..44880478d242cf 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-mmu.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-mmu.h
@@ -1,19 +1,17 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (C) 2013--2024 Intel Corporation */
+/* Copyright (C) 2013--2026 Intel Corporation */
 
 #ifndef IPU6_MMU_H
 #define IPU6_MMU_H
 
-#define ISYS_MMID 1
-#define PSYS_MMID 0
-
 #include <linux/list.h>
 #include <linux/spinlock_types.h>
 #include <linux/types.h>
 
+#include "ipu7-mmu-hw.h"
+
 struct device;
 struct page;
-struct ipu6_hw_variants;
 
 struct ipu6_mmu_info {
 	struct device *dev;
@@ -35,10 +33,150 @@ struct ipu6_mmu_info {
 	struct ipu6_dma_mapping *dmap;
 };
 
+/*
+ * MMU Invalidation HW bug workaround by ZLW mechanism
+ *
+ * Old IPU6 MMUV2 has a bug in the invalidation mechanism which might result in
+ * wrong translation or replication of the translation. This will cause data
+ * corruption. So we cannot directly use the MMU V2 invalidation registers
+ * to invalidate the MMU. Instead, whenever an invalidate is called, we need to
+ * clear the TLB by evicting all the valid translations by filling it with trash
+ * buffer (which is guaranteed not to be used by any other processes). ZLW is
+ * used to fill the L1 and L2 caches with the trash buffer translations. ZLW
+ * or Zero length write, is pre-fetch mechanism to pre-fetch the pages in
+ * advance to the L1 and L2 caches without triggering any memory operations.
+ *
+ * In MMU V2, L1 -> 16 streams and 64 blocks, maximum 16 blocks per stream
+ * One L1 block has 16 entries, hence points to 16 * 4K pages
+ * L2 -> 16 streams and 32 blocks. 2 blocks per streams
+ * One L2 block maps to 1024 L1 entries, hence points to 4MB address range
+ * 2 blocks per L2 stream means, 1 stream points to 8MB range
+ *
+ * As we need to clear the caches and 8MB being the biggest cache size, we need
+ * to have trash buffer which points to 8MB address range. As these trash
+ * buffers are not used for any memory transactions, we need only the least
+ * amount of physical memory. So we reserve 8MB IOVA address range but only
+ * one page is reserved from physical memory. Each of this 8MB IOVA address
+ * range is then mapped to the same physical memory page.
+ */
+/* One L2 entry maps 1024 L1 entries and one L1 entry per page */
+#define IPU6_MMUV2_L2_RANGE		(1024 * PAGE_SIZE)
+/* Max L2 blocks per stream */
+#define IPU6_MMUV2_MAX_L2_BLOCKS	2
+/* Max L1 blocks per stream */
+#define IPU6_MMUV2_MAX_L1_BLOCKS	16
+#define IPU6_MMUV2_TRASH_RANGE	(IPU6_MMUV2_L2_RANGE * IPU6_MMUV2_MAX_L2_BLOCKS)
+/* Entries per L1 block */
+#define MMUV2_ENTRIES_PER_L1_BLOCK	16
+#define MMUV2_TRASH_L1_BLOCK_OFFSET	(MMUV2_ENTRIES_PER_L1_BLOCK * PAGE_SIZE)
+#define MMUV2_TRASH_L2_BLOCK_OFFSET	IPU6_MMUV2_L2_RANGE
+
+/*
+ * In some of the IPU6 MMUs, there is provision to configure L1 and L2 page
+ * table caches. Both these L1 and L2 caches are divided into multiple sections
+ * called streams. There is maximum 16 streams for both caches. Each of these
+ * sections are subdivided into multiple blocks. When nr_l1streams = 0 and
+ * nr_l2streams = 0, means the MMU is of type MMU_V1 and do not support
+ * L1/L2 page table caches.
+ *
+ * L1 stream per block sizes are configurable and varies per usecase.
+ * L2 has constant block sizes - 2 blocks per stream.
+ *
+ * MMU1 support pre-fetching of the pages to have less cache lookup misses. To
+ * enable the pre-fetching, MMU1 AT (Address Translator) device registers
+ * need to be configured.
+ *
+ * There are four types of memory accesses which requires ZLW configuration.
+ * ZLW(Zero Length Write) is a mechanism to enable VT-d pre-fetching on IOMMU.
+ *
+ * 1. Sequential Access or 1D mode
+ *	Set ZLW_EN -> 1
+ *	set ZLW_PAGE_CROSS_1D -> 1
+ *	Set ZLW_N to "N" pages so that ZLW will be inserte N pages ahead where
+ *		  N is pre-defined and hardcoded in the platform data
+ *	Set ZLW_2D -> 0
+ *
+ * 2. ZLW 2D mode
+ *	Set ZLW_EN -> 1
+ *	set ZLW_PAGE_CROSS_1D -> 1,
+ *	Set ZLW_N -> 0
+ *	Set ZLW_2D -> 1
+ *
+ * 3. ZLW Enable (no 1D or 2D mode)
+ *	Set ZLW_EN -> 1
+ *	set ZLW_PAGE_CROSS_1D -> 0,
+ *	Set ZLW_N -> 0
+ *	Set ZLW_2D -> 0
+ *
+ * 4. ZLW disable
+ *	Set ZLW_EN -> 0
+ *	set ZLW_PAGE_CROSS_1D -> 0,
+ *	Set ZLW_N -> 0
+ *	Set ZLW_2D -> 0
+ *
+ * To configure the ZLW for the above memory access, four registers are
+ * available. Hence to track these four settings, we have the following entries
+ * in the struct ipu6_mmu_hw. Each of these entries are per stream and
+ * available only for the L1 streams.
+ *
+ * a. l1_zlw_en -> To track zlw enabled per stream (ZLW_EN)
+ * b. l1_zlw_1d_mode -> Track 1D mode per stream. ZLW inserted at page boundary
+ * c. l1_ins_zlw_ahead_pages -> to track how advance the ZLW need to be inserted
+ *			Insert ZLW request N pages ahead address.
+ * d. l1_zlw_2d_mode -> To track 2D mode per stream (ZLW_2D)
+ *
+ *
+ * Currently L1/L2 streams, blocks, AT ZLW configurations etc. are pre-defined
+ * as per the usecase specific calculations. Any change to this pre-defined
+ * table has to happen in sync with IPU6 FW.
+ */
+
+struct ipu6_mmu_hw {
+	union {
+		unsigned long offset;
+		void __iomem *base;
+	};
+	u32 info_bits;
+	u8 nr_l1streams;
+	/*
+	 * L1 has variable blocks per stream - total of 64 blocks and maximum of
+	 * 16 blocks per stream. Configurable by using the block start address
+	 * per stream. Block start address is calculated from the block size
+	 */
+	u8 l1_block_sz[IPU6_MMU_MAX_TLB_L1_STREAMS];
+	/* Is ZLW is enabled in each stream */
+	bool l1_zlw_en[IPU6_MMU_MAX_TLB_L1_STREAMS];
+	bool l1_zlw_1d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
+	u8 l1_ins_zlw_ahead_pages[IPU6_MMU_MAX_TLB_L1_STREAMS];
+	bool l1_zlw_2d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
+
+	u32 l1_stream_id_reg_offset;
+	u32 l2_stream_id_reg_offset;
+
+	u8 nr_l2streams;
+	/*
+	 * L2 has fixed 2 blocks per stream. Block address is calculated
+	 * from the block size
+	 */
+	u8 l2_block_sz[IPU6_MMU_MAX_TLB_L2_STREAMS];
+	/* flag to track if WA is needed for successive invalidate HW bug */
+	bool insert_read_before_invalidate;
+};
+
+struct ipu6_mmu_hw_ops {
+	int (*init_hw_data)(struct ipu6_mmu *mmu, struct device *dev,
+			    void __iomem *base);
+	int (*hw_init)(struct ipu6_mmu *mmu);
+	void (*tlb_invalidate)(struct ipu6_mmu *mmu);
+};
+
 struct ipu6_mmu {
 	struct list_head node;
 
-	struct ipu6_mmu_hw *mmu_hw;
+	union {
+		struct ipu6_mmu_hw *ipu6_mmu_hw;
+		struct ipu7_mmu_hw *ipu7_mmu_hw;
+	};
 	unsigned int nr_mmus;
 	unsigned int mmid;
 
@@ -55,12 +193,14 @@ struct ipu6_mmu {
 	bool ready;
 	spinlock_t ready_lock;	/* Serialize access to bool ready */
 
-	void (*tlb_invalidate)(struct ipu6_mmu *mmu);
+	const struct ipu6_mmu_hw_ops *ops;
 };
 
+extern const struct ipu6_mmu_hw_ops ipu6_mmu_ops;
+extern const struct ipu6_mmu_hw_ops ipu7_mmu_ops;
+
 struct ipu6_mmu *ipu6_mmu_init(struct device *dev,
-			       void __iomem *base, int mmid,
-			       const struct ipu6_hw_variants *hw);
+			       void __iomem *base, int mmid);
 void ipu6_mmu_cleanup(struct ipu6_mmu *mmu);
 int ipu6_mmu_hw_init(struct ipu6_mmu *mmu);
 void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu);
diff --git a/drivers/media/pci/intel/ipu6/ipu6-platform-buttress-regs.h b/drivers/media/pci/intel/ipu6/ipu6-platform-buttress-regs.h
index efd65e494c1649..57e661cc2177f0 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-platform-buttress-regs.h
+++ b/drivers/media/pci/intel/ipu6/ipu6-platform-buttress-regs.h
@@ -212,7 +212,7 @@ enum {
 #define BUTTRESS_TSW_CTL_SOFT_RESET	BIT(8)
 
 #define BUTTRESS_REG_TSC_LO	0x164
-#define BUTTRESS_REG_TSC_HI	0x168
+#define BUTTRESS_TSC_HI_OFFSET	4
 
 #define BUTTRESS_IRQS		(BUTTRESS_ISR_IPC_FROM_CSE_IS_WAITING | \
 				 BUTTRESS_ISR_IPC_EXEC_DONE_BY_CSE |    \
@@ -221,4 +221,113 @@ enum {
 #define BUTTRESS_EVENT		 (BUTTRESS_ISR_IPC_FROM_CSE_IS_WAITING | \
 				  BUTTRESS_ISR_IPC_EXEC_DONE_BY_CSE |    \
 				  BUTTRESS_ISR_SAI_VIOLATION)
+
+/* IPU7 */
+
+/* IPU7 buttress register offsets */
+#define IPU7_BUTTRESS_REG_SKU			0x3030
+#define IPU7_BUTTRESS_REG_IRQ_STATUS		0x2000
+#define IPU7_BUTTRESS_REG_IRQ_ENABLE		0x2008
+#define IPU7_BUTTRESS_REG_IRQ_CLEAR		0x200c
+#define IPU7_BUTTRESS_REG_IRQ_MASK		0x2010
+#define IPU7_BUTTRESS_REG_TSC_CMD		0x2014
+#define IPU7_BUTTRESS_REG_TSC_CTL		0x2018
+#define IPU7_BUTTRESS_REG_TSC_LO		0x201c
+#define IPU7_BUTTRESS_REG_PB_TIMESTAMP_LO	0x2030
+#define IPU7_BUTTRESS_REG_PB_TIMESTAMP_VALID	0x2038
+#define IPU7_BUTTRESS_REG_IS_WORKPOINT_REQ	0x2104
+#define IPU7_BUTTRESS_REG_PS_WORKPOINT_REQ	0x2100
+#define IPU7_BUTTRESS_REG_IDLE_WDT		0x218c
+#define IPU7_BUTTRESS_REG_ISYS_UCX_CTRL_STATUS	0x2200
+#define IPU7_BUTTRESS_REG_ISYS_UCX_START_ADDR	0x2204
+#define IPU7_BUTTRESS_REG_PSYS_UCX_CTRL_STATUS	0x2208
+#define IPU7_BUTTRESS_REG_PSYS_UCX_START_ADDR	0x220c
+#define IPU7_BUTTRESS_REG_SECURITY_CTL		0x2318
+#define IPU7_BUTTRESS_REG_FW_RESET_CTL		0x2334
+#define IPU7_BUTTRESS_REG_FW_SOURCE_SIZE	0x2338
+#define IPU7_BUTTRESS_REG_FW_SOURCE_BASE	0x233c
+#define IPU7_BUTTRESS_REG_CSE2IUDB0		0x2500
+#define IPU7_BUTTRESS_REG_CSE2IUDATA0		0x2504
+#define IPU7_BUTTRESS_REG_CSE2IUCSR		0x2508
+#define IPU7_BUTTRESS_REG_IU2CSEDB0		0x250c
+#define IPU7_BUTTRESS_REG_IU2CSEDATA0		0x2510
+#define IPU7_BUTTRESS_REG_IU2CSECSR		0x2514
+#define IPU7_BUTTRESS_REG_CG_CTRL_BITS		0x3014
+#define IPU7_BUTTRESS_REG_FW_BOOT_PARAMS0	0x4000
+#define IPU7_BUTTRESS_REG_FW_BOOT_PARAMS7	0x401c
+#define IPU7_BUTTRESS_REG_PWR_STATUS		0x2114
+
+#define IPU7_BUTTRESS_IRQ_IPC_EXEC_DONE_BY_CSE		BIT(0)
+#define IPU7_BUTTRESS_IRQ_IPC_FROM_CSE_IS_WAITING	BIT(1)
+#define IPU7_BUTTRESS_IRQ_CSE_CSR_SET			BIT(2)
+#define IPU7_BUTTRESS_IRQ_SAI_VIOLATION			BIT(4)
+#define IPU7_BUTTRESS_IRQ_IS_IRQ			BIT(30)
+#define IPU7_BUTTRESS_IRQ_PS_IRQ			BIT(31)
+
+#define IPU7_BUTTRESS_IRQS	(IPU7_BUTTRESS_IRQ_IS_IRQ |		\
+				 IPU7_BUTTRESS_IRQ_PS_IRQ |		\
+				 IPU7_BUTTRESS_IRQ_IPC_FROM_CSE_IS_WAITING | \
+				 IPU7_BUTTRESS_IRQ_CSE_CSR_SET |	\
+				 IPU7_BUTTRESS_IRQ_IPC_EXEC_DONE_BY_CSE)
+
+/* LNL SW workaround for PS PD hang */
+#define IPU7_BUTTRESS_CG_CTRL_PS_FSM_CG		BIT(3)
+
+/* P-unit Bridge (PB) BAR registers */
+#define IPU7_PB_INTERRUPT_STATUS		0x0
+
+/* Frequency control encoding */
+#define IPU7_FREQ_CTL_CDYN			0x80
+#define IPU7_FREQ_CTL_CDYN_SHIFT		8
+#define IPU7_BUTTRESS_IS_FREQ_CTL_RATIO_MASK	GENMASK(7, 0)
+#define IPU7_IS_FREQ_CTL_DEFAULT_RATIO		0x1b
+#define IPU7_PS_FREQ_CTL_DEFAULT_RATIO		0x14
+
+/* D2D power control */
+#define IPU7_BUTTRESS_REG_D2D_CTL		0x21d4
+#define IPU7_BUTTRESS_D2D_PWR_EN		BIT(0)
+#define IPU7_BUTTRESS_D2D_PWR_ACK		BIT(4)
+
+#define IPU7_BUTTRESS_PWR_STATE_IS_PWR_SHIFT	0
+#define IPU7_BUTTRESS_PWR_STATE_IS_PWR_MASK	(0x3U << 0)
+#define IPU7_BUTTRESS_PWR_STATE_PS_PWR_SHIFT	4
+#define IPU7_BUTTRESS_PWR_STATE_PS_PWR_MASK	(0x3U << 4)
+
+/* NDE */
+#define IPU7_BUTTRESS_REG_NDE_CONTROL		0x21a4
+#define IPU7_NDE_VAL_MASK			GENMASK(9, 0)
+#define IPU7_NDE_SCALE_MASK			GENMASK(12, 10)
+#define IPU7_NDE_VALID_MASK			BIT(13)
+#define IPU7_NDE_RESVEC_MASK			GENMASK(19, 16)
+#define IPU7_NDE_VAL_ACTIVE			48
+#define IPU7_NDE_SCALE_ACTIVE			2
+#define IPU7_NDE_VALID_ACTIVE			1
+#define IPU7_NDE_VAL_DEFAULT			1023
+#define IPU7_NDE_SCALE_DEFAULT			2
+#define IPU7_NDE_VALID_DEFAULT			0
+#define IPU7_NDE_RESVEC				0xe
+
+/* IS UCX control */
+#define IPU7_UCX_CTL_RESET			BIT(0)
+#define IPU7_UCX_CTL_RUN			BIT(1)
+#define IPU7_UCX_CTL_WAKEUP			BIT(2)
+
+#define IPU7_BUTTRESS_REG_SLEEP_LEVEL_CFG	0x21b0
+#define IPU7_BUTTRESS_REG_SLEEP_LEVEL_STS	0x21b4
+#define IPU7_BUTTRESS_OVERRIDE_IS_CLK		BIT(1)
+#define IPU7_BUTTRESS_OWN_ACK_IS_CLK		BIT(9)
+#define IPU7_BUTTRESS_OVERRIDE_PS_CLK		BIT(2)
+#define IPU7_BUTTRESS_OWN_ACK_PS_CLK		BIT(10)
+
+/* PB registers */
+#define IPU7_GLOBAL_INTERRUPT_MASK		0x8
+#define IPU7_TLBID_HASH_ENABLE_31_0		0x30
+#define IPU7_TLBID_HASH_ENABLE_63_32		0x34
+#define IPU7_TLBID_HASH_ENABLE_95_64		0x38
+#define IPU7_TLBID_HASH_ENABLE_127_96		0x3c
+#define IPU7_BAR2_MISC_CONFIG			0x64
+
+/* IPU7.5 */
+#define IPU7_BUTTRESS_SEL_PB_TIMESTAMP		BIT(9)
+
 #endif /* IPU6_PLATFORM_BUTTRESS_REGS_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c
index 5449a2006bcce7..43d951735f72e9 100644
--- a/drivers/media/pci/intel/ipu6/ipu6.c
+++ b/drivers/media/pci/intel/ipu6/ipu6.c
@@ -19,6 +19,7 @@
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <linux/types.h>
+#include <linux/vmalloc.h>
 
 #include <media/ipu-bridge.h>
 #include <media/ipu6-pci-table.h>
@@ -27,13 +28,20 @@
 #include "ipu6-bus.h"
 #include "ipu6-buttress.h"
 #include "ipu6-cpd.h"
+#include "ipu6-dma.h"
 #include "ipu6-isys.h"
 #include "ipu6-mmu.h"
 #include "ipu6-platform-buttress-regs.h"
 #include "ipu6-platform-isys-csi2-reg.h"
 #include "ipu6-platform-regs.h"
+#include "ipu7-isys-csi2-regs.h"
 
 #define IPU6_PCI_BAR		0
+#define IPU7_PCI_PBBAR		4
+
+static int force_no_probe_ipu7 = !IS_BUILTIN(CONFIG_VIDEO_INTEL_IPU6_IPU7);
+module_param(force_no_probe_ipu7, int, 0644);
+MODULE_PARM_DESC(force_no_probe_ipu7, "Don't probe ipu7 and ipu7.5 devices");
 
 struct ipu6_cell_program {
 	u32 magic_number;
@@ -73,54 +81,6 @@ struct ipu6_cell_program {
 static struct ipu6_isys_internal_pdata isys_ipdata = {
 	.hw_variant = {
 		.offset = IPU6_UNIFIED_OFFSET,
-		.nr_mmus = 3,
-		.mmu_hw = {
-			{
-				.offset = IPU6_ISYS_IOMMU0_OFFSET,
-				.info_bits = IPU6_INFO_REQUEST_DESTINATION_IOSF,
-				.nr_l1streams = 16,
-				.l1_block_sz = {
-					3, 8, 2, 2, 2, 2, 2, 2, 1, 1,
-					1, 1, 1, 1, 1, 1
-				},
-				.nr_l2streams = 16,
-				.l2_block_sz = {
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2
-				},
-				.insert_read_before_invalidate = false,
-				.l1_stream_id_reg_offset =
-				IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
-				.l2_stream_id_reg_offset =
-				IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
-			},
-			{
-				.offset = IPU6_ISYS_IOMMU1_OFFSET,
-				.info_bits = 0,
-				.nr_l1streams = 16,
-				.l1_block_sz = {
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 1, 1, 4
-				},
-				.nr_l2streams = 16,
-				.l2_block_sz = {
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2
-				},
-				.insert_read_before_invalidate = false,
-				.l1_stream_id_reg_offset =
-				IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
-				.l2_stream_id_reg_offset =
-				IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
-			},
-			{
-				.offset = IPU6_ISYS_IOMMUI_OFFSET,
-				.info_bits = 0,
-				.nr_l1streams = 0,
-				.nr_l2streams = 0,
-				.insert_read_before_invalidate = false,
-			},
-		},
 		.cdc_fifos = 3,
 		.cdc_fifo_threshold = {6, 8, 2},
 		.dmem_offset = IPU6_ISYS_DMEM_OFFSET,
@@ -132,83 +92,12 @@ static struct ipu6_isys_internal_pdata isys_ipdata = {
 static struct ipu6_psys_internal_pdata psys_ipdata = {
 	.hw_variant = {
 		.offset = IPU6_UNIFIED_OFFSET,
-		.nr_mmus = 4,
-		.mmu_hw = {
-			{
-				.offset = IPU6_PSYS_IOMMU0_OFFSET,
-				.info_bits =
-				IPU6_INFO_REQUEST_DESTINATION_IOSF,
-				.nr_l1streams = 16,
-				.l1_block_sz = {
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2
-				},
-				.nr_l2streams = 16,
-				.l2_block_sz = {
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2
-				},
-				.insert_read_before_invalidate = false,
-				.l1_stream_id_reg_offset =
-				IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
-				.l2_stream_id_reg_offset =
-				IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
-			},
-			{
-				.offset = IPU6_PSYS_IOMMU1_OFFSET,
-				.info_bits = 0,
-				.nr_l1streams = 32,
-				.l1_block_sz = {
-					1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 10,
-					5, 4, 14, 6, 4, 14, 6, 4, 8,
-					4, 2, 1, 1, 1, 1, 14
-				},
-				.nr_l2streams = 32,
-				.l2_block_sz = {
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2
-				},
-				.insert_read_before_invalidate = false,
-				.l1_stream_id_reg_offset =
-				IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
-				.l2_stream_id_reg_offset =
-				IPU6_PSYS_MMU1W_L2_STREAM_ID_REG_OFFSET,
-			},
-			{
-				.offset = IPU6_PSYS_IOMMU1R_OFFSET,
-				.info_bits = 0,
-				.nr_l1streams = 16,
-				.l1_block_sz = {
-					1, 4, 4, 4, 4, 16, 8, 4, 32,
-					16, 16, 2, 2, 2, 1, 12
-				},
-				.nr_l2streams = 16,
-				.l2_block_sz = {
-					2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-					2, 2, 2, 2, 2, 2
-				},
-				.insert_read_before_invalidate = false,
-				.l1_stream_id_reg_offset =
-				IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
-				.l2_stream_id_reg_offset =
-				IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
-			},
-			{
-				.offset = IPU6_PSYS_IOMMUI_OFFSET,
-				.info_bits = 0,
-				.nr_l1streams = 0,
-				.nr_l2streams = 0,
-				.insert_read_before_invalidate = false,
-			},
-		},
 		.dmem_offset = IPU6_PSYS_DMEM_OFFSET,
 	},
 };
 
-static const struct ipu6_buttress_ctrl isys_buttress_ctrl = {
+static const struct ipu6_buttress_ctrl ipu6_isys_buttress_ctrl = {
+	.subsys_id = IPU_ISYS,
 	.ratio = IPU6_IS_FREQ_CTL_DEFAULT_RATIO,
 	.qos_floor = IPU6_IS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO,
 	.freq_ctl = IPU6_BUTTRESS_REG_IS_FREQ_CTL,
@@ -218,7 +107,8 @@ static const struct ipu6_buttress_ctrl isys_buttress_ctrl = {
 	.pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
 };
 
-static const struct ipu6_buttress_ctrl psys_buttress_ctrl = {
+static const struct ipu6_buttress_ctrl ipu6_psys_buttress_ctrl = {
+	.subsys_id = IPU_PSYS,
 	.ratio = IPU6_PS_FREQ_CTL_DEFAULT_RATIO,
 	.qos_floor = IPU6_PS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO,
 	.freq_ctl = IPU6_BUTTRESS_REG_PS_FREQ_CTL,
@@ -228,6 +118,119 @@ static const struct ipu6_buttress_ctrl psys_buttress_ctrl = {
 	.pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
 };
 
+static const struct ipu6_buttress_ctrl ipu7_isys_buttress_ctrl = {
+	.subsys_id = IPU_ISYS,
+	.ratio = IPU7_IS_FREQ_CTL_DEFAULT_RATIO,
+	.qos_floor = 0,
+	.freq_ctl = IPU7_BUTTRESS_REG_IS_WORKPOINT_REQ,
+	.pwr_sts_shift = IPU7_BUTTRESS_PWR_STATE_IS_PWR_SHIFT,
+	.pwr_sts_mask = IPU7_BUTTRESS_PWR_STATE_IS_PWR_MASK,
+	.pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE,
+	.pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
+};
+
+static const struct ipu6_buttress_ctrl ipu7_psys_buttress_ctrl = {
+	.subsys_id = IPU_PSYS,
+	.ratio = IPU7_PS_FREQ_CTL_DEFAULT_RATIO,
+	.qos_floor = 0,
+	.freq_ctl = IPU7_BUTTRESS_REG_PS_WORKPOINT_REQ,
+	.pwr_sts_shift = IPU7_BUTTRESS_PWR_STATE_PS_PWR_SHIFT,
+	.pwr_sts_mask = IPU7_BUTTRESS_PWR_STATE_PS_PWR_MASK,
+	.pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE,
+	.pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
+};
+
+static const struct ipu6_buttress_registers ipu6_buttress_regs = {
+	/* Registers */
+	.irq_status	= BUTTRESS_REG_ISR_STATUS,
+	.irq_clear	= BUTTRESS_REG_ISR_CLEAR,
+	.irq_enable	= BUTTRESS_REG_ISR_ENABLE,
+	.pwr_status	= BUTTRESS_REG_PWR_STATE,
+	.security_ctl	= BUTTRESS_REG_SECURITY_CTL,
+	.fw_reset_ctl	= BUTTRESS_REG_FW_RESET_CTL,
+	.fabric_cmd	= BUTTRESS_REG_FABRIC_CMD,
+	.tsw_ctl	= BUTTRESS_REG_TSW_CTL,
+	.tsc_lo		= BUTTRESS_REG_TSC_LO,
+	.wdt		= BUTTRESS_REG_WDT,
+	.btrs_ctrl	= BUTTRESS_REG_BTRS_CTRL,
+	.csr_in		= BUTTRESS_REG_CSE2IUCSR,
+	.csr_out	= BUTTRESS_REG_IU2CSECSR,
+	.db0_in		= BUTTRESS_REG_CSE2IUDB0,
+	.db0_out	= BUTTRESS_REG_IU2CSEDB0,
+	.data0_in	= BUTTRESS_REG_CSE2IUDATA0,
+	.data0_out	= BUTTRESS_REG_IU2CSEDATA0,
+	.sku_id		= BUTTRESS_REG_SKU,
+
+	/* Bitmasks */
+	.irq_is		= BUTTRESS_ISR_IS_IRQ,
+	.irq_ps		= BUTTRESS_ISR_PS_IRQ,
+	.irq_all	= BUTTRESS_IRQS,
+	.irq_events	= BUTTRESS_EVENT,
+	.irq_cse_ipc	= BUTTRESS_ISR_IPC_FROM_CSE_IS_WAITING,
+	.irq_exec_done	= BUTTRESS_ISR_IPC_EXEC_DONE_BY_CSE,
+	.irq_sai	= BUTTRESS_ISR_SAI_VIOLATION,
+};
+
+static const struct ipu6_buttress_registers ipu7_buttress_regs = {
+	/* Registers */
+	.irq_status	= IPU7_BUTTRESS_REG_IRQ_STATUS,
+	.irq_clear	= IPU7_BUTTRESS_REG_IRQ_CLEAR,
+	.irq_enable	= IPU7_BUTTRESS_REG_IRQ_ENABLE,
+	.pwr_status	= IPU7_BUTTRESS_REG_PWR_STATUS,
+	.security_ctl	= IPU7_BUTTRESS_REG_SECURITY_CTL,
+	.fw_reset_ctl	= IPU7_BUTTRESS_REG_FW_RESET_CTL,
+	.fabric_cmd	= IPU7_BUTTRESS_REG_TSC_CMD,
+	.tsw_ctl	= IPU7_BUTTRESS_REG_TSC_CTL,
+	.tsc_lo		= IPU7_BUTTRESS_REG_TSC_LO,
+	.wdt		= IPU7_BUTTRESS_REG_IDLE_WDT,
+	.csr_in		= IPU7_BUTTRESS_REG_CSE2IUCSR,
+	.csr_out	= IPU7_BUTTRESS_REG_IU2CSECSR,
+	.db0_in		= IPU7_BUTTRESS_REG_CSE2IUDB0,
+	.db0_out	= IPU7_BUTTRESS_REG_IU2CSEDB0,
+	.data0_in	= IPU7_BUTTRESS_REG_CSE2IUDATA0,
+	.data0_out	= IPU7_BUTTRESS_REG_IU2CSEDATA0,
+	.sku_id		= IPU7_BUTTRESS_REG_SKU,
+
+	/* Bitmasks */
+	.irq_is		= IPU7_BUTTRESS_IRQ_IS_IRQ,
+	.irq_ps		= IPU7_BUTTRESS_IRQ_PS_IRQ,
+	.irq_all	= IPU7_BUTTRESS_IRQS,
+	.irq_events	= IPU7_BUTTRESS_IRQS,
+	.irq_cse_ipc	= IPU7_BUTTRESS_IRQ_IPC_FROM_CSE_IS_WAITING,
+	.irq_exec_done	= IPU7_BUTTRESS_IRQ_IPC_EXEC_DONE_BY_CSE,
+	.irq_sai	= IPU7_BUTTRESS_IRQ_SAI_VIOLATION,
+};
+
+static const struct ipu6_buttress_registers ipu7p5_buttress_regs = {
+	/* Registers */
+	.irq_status	= IPU7_BUTTRESS_REG_IRQ_STATUS,
+	.irq_clear	= IPU7_BUTTRESS_REG_IRQ_CLEAR,
+	.irq_enable	= IPU7_BUTTRESS_REG_IRQ_ENABLE,
+	.pwr_status	= IPU7_BUTTRESS_REG_PWR_STATUS,
+	.security_ctl	= IPU7_BUTTRESS_REG_SECURITY_CTL,
+	.fw_reset_ctl	= IPU7_BUTTRESS_REG_FW_RESET_CTL,
+	.fabric_cmd	= IPU7_BUTTRESS_REG_TSC_CMD,
+	.tsw_ctl	= IPU7_BUTTRESS_REG_TSC_CTL,
+	.tsc_lo		= IPU7_BUTTRESS_REG_PB_TIMESTAMP_LO,
+	.wdt		= IPU7_BUTTRESS_REG_IDLE_WDT,
+	.csr_in		= IPU7_BUTTRESS_REG_CSE2IUCSR,
+	.csr_out	= IPU7_BUTTRESS_REG_IU2CSECSR,
+	.db0_in		= IPU7_BUTTRESS_REG_CSE2IUDB0,
+	.db0_out	= IPU7_BUTTRESS_REG_IU2CSEDB0,
+	.data0_in	= IPU7_BUTTRESS_REG_CSE2IUDATA0,
+	.data0_out	= IPU7_BUTTRESS_REG_IU2CSEDATA0,
+	.sku_id		= IPU7_BUTTRESS_REG_SKU,
+
+	/* Bitmasks */
+	.irq_is		= IPU7_BUTTRESS_IRQ_IS_IRQ,
+	.irq_ps		= IPU7_BUTTRESS_IRQ_PS_IRQ,
+	.irq_all	= IPU7_BUTTRESS_IRQS,
+	.irq_events	= IPU7_BUTTRESS_IRQS,
+	.irq_cse_ipc	= IPU7_BUTTRESS_IRQ_IPC_FROM_CSE_IS_WAITING,
+	.irq_exec_done	= IPU7_BUTTRESS_IRQ_IPC_EXEC_DONE_BY_CSE,
+	.irq_sai	= IPU7_BUTTRESS_IRQ_SAI_VIOLATION,
+};
+
 static void
 ipu6_pkg_dir_configure_spc(struct ipu6_device *isp,
 			   const struct ipu6_hw_variants *hw_variant,
@@ -267,10 +270,16 @@ void ipu6_configure_spc(struct ipu6_device *isp,
 			int pkg_dir_idx, void __iomem *base, u64 *pkg_dir,
 			dma_addr_t pkg_dir_dma_addr)
 {
-	void __iomem *dmem_base = base + hw_variant->dmem_offset;
-	void __iomem *spc_regs_base = base + hw_variant->spc_offset;
+	void __iomem *dmem_base;
+	void __iomem *spc_regs_base;
 	u32 val;
 
+	if (IS_IPU7(isp))
+		return;
+
+	dmem_base = base + hw_variant->dmem_offset;
+	spc_regs_base = base + hw_variant->spc_offset;
+
 	val = readl(spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL);
 	val |= IPU6_PSYS_SPC_STATUS_CTRL_ICACHE_INVALIDATE;
 	writel(val, spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL);
@@ -290,8 +299,6 @@ EXPORT_SYMBOL_NS_GPL(ipu6_configure_spc, "INTEL_IPU6");
 
 static void ipu6_internal_pdata_init(struct ipu6_device *isp)
 {
-	u8 hw_ver = isp->hw_ver;
-
 	isys_ipdata.num_parallel_streams = IPU6_ISYS_NUM_STREAMS;
 	isys_ipdata.sram_gran_shift = IPU6_SRAM_GRANULARITY_SHIFT;
 	isys_ipdata.sram_gran_size = IPU6_SRAM_GRANULARITY_SIZE;
@@ -314,19 +321,19 @@ static void ipu6_internal_pdata_init(struct ipu6_device *isp)
 		IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_STATUS;
 	isys_ipdata.csi2.ctrl0_irq_lnp =
 		IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_LEVEL_NOT_PULSE;
-	isys_ipdata.enhanced_iwake = is_ipu6ep_mtl(hw_ver) || is_ipu6ep(hw_ver);
+	isys_ipdata.enhanced_iwake = IS_IPU6EP_MTL(isp) || IS_IPU6EP(isp);
 	psys_ipdata.hw_variant.spc_offset = IPU6_PSYS_SPC_OFFSET;
 	isys_ipdata.csi2.fw_access_port_ofs = CSI_REG_HUB_FW_ACCESS_PORT_OFS;
 
-	if (is_ipu6ep(hw_ver)) {
+	if (IS_IPU6EP(isp)) {
 		isys_ipdata.ltr = IPU6EP_LTR_VALUE;
 		isys_ipdata.memopen_threshold = IPU6EP_MIN_MEMOPEN_TH;
 	}
 
-	if (is_ipu6_tgl(hw_ver))
+	if (IS_IPU6_TGL(isp))
 		isys_ipdata.csi2.nports = IPU6_TGL_ISYS_CSI2_NPORTS;
 
-	if (is_ipu6ep_mtl(hw_ver)) {
+	if (IS_IPU6EP_MTL(isp)) {
 		isys_ipdata.csi2.nports = IPU6EP_MTL_ISYS_CSI2_NPORTS;
 
 		isys_ipdata.csi2.ctrl0_irq_edge =
@@ -347,7 +354,7 @@ static void ipu6_internal_pdata_init(struct ipu6_device *isp)
 		isys_ipdata.memopen_threshold = IPU6EP_MTL_MIN_MEMOPEN_TH;
 	}
 
-	if (is_ipu6se(hw_ver)) {
+	if (IS_IPU6SE(isp)) {
 		isys_ipdata.csi2.nports = IPU6SE_ISYS_CSI2_NPORTS;
 		isys_ipdata.csi2.irq_mask = IPU6SE_CSI_RX_ERROR_IRQ_MASK;
 		isys_ipdata.num_parallel_streams = IPU6SE_ISYS_NUM_STREAMS;
@@ -363,15 +370,21 @@ static void ipu6_internal_pdata_init(struct ipu6_device *isp)
 		isys_ipdata.max_devq_size = IPU6SE_DEV_SEND_QUEUE_SIZE;
 		psys_ipdata.hw_variant.spc_offset = IPU6SE_PSYS_SPC_OFFSET;
 	}
+
+	if (IS_IPU7(isp)) {
+		isys_ipdata.csi2.gpreg = IPU7_IS_IO_CSI2_GPREGS_BASE;
+		isys_ipdata.csi2.nports = 4;
+	}
 }
 
 static struct ipu6_bus_device *
 ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
-	       struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
+	       const struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
 	       const struct ipu6_isys_internal_pdata *ipdata)
 {
 	struct device *dev = &pdev->dev;
 	struct ipu6_bus_device *isys_adev;
+	struct ipu6_buttress_ctrl *devm_ctrl;
 	struct ipu6_isys_pdata *pdata;
 	int ret;
 
@@ -381,6 +394,10 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
 		return ERR_PTR(ret);
 	}
 
+	devm_ctrl = devm_kmemdup(dev, ctrl, sizeof(*ctrl), GFP_KERNEL);
+	if (!devm_ctrl)
+		return ERR_PTR(-ENOMEM);
+
 	pdata = kzalloc_obj(*pdata);
 	if (!pdata)
 		return ERR_PTR(-ENOMEM);
@@ -388,7 +405,7 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
 	pdata->base = base;
 	pdata->ipdata = ipdata;
 
-	isys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl,
+	isys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, devm_ctrl,
 					       IPU6_ISYS_NAME);
 	if (IS_ERR(isys_adev)) {
 		kfree(pdata);
@@ -396,8 +413,7 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
 				"ipu6_bus_initialize_device isys failed\n");
 	}
 
-	isys_adev->mmu = ipu6_mmu_init(dev, base, ISYS_MMID,
-				       &ipdata->hw_variant);
+	isys_adev->mmu = ipu6_mmu_init(dev, base, IPU_ISYS);
 	if (IS_ERR(isys_adev->mmu)) {
 		put_device(&isys_adev->auxdev.dev);
 		return dev_err_cast_probe(dev, isys_adev->mmu,
@@ -415,13 +431,19 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
 
 static struct ipu6_bus_device *
 ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
-	       struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
+	       const struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
 	       const struct ipu6_psys_internal_pdata *ipdata)
 {
+	struct device *dev = &pdev->dev;
 	struct ipu6_bus_device *psys_adev;
+	struct ipu6_buttress_ctrl *devm_ctrl;
 	struct ipu6_psys_pdata *pdata;
 	int ret;
 
+	devm_ctrl = devm_kmemdup(dev, ctrl, sizeof(*ctrl), GFP_KERNEL);
+	if (!devm_ctrl)
+		return ERR_PTR(-ENOMEM);
+
 	pdata = kzalloc_obj(*pdata);
 	if (!pdata)
 		return ERR_PTR(-ENOMEM);
@@ -429,7 +451,7 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
 	pdata->base = base;
 	pdata->ipdata = ipdata;
 
-	psys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl,
+	psys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, devm_ctrl,
 					       IPU6_PSYS_NAME);
 	if (IS_ERR(psys_adev)) {
 		kfree(pdata);
@@ -437,8 +459,7 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
 				"ipu6_bus_initialize_device psys failed\n");
 	}
 
-	psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, PSYS_MMID,
-				       &ipdata->hw_variant);
+	psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, IPU_PSYS);
 	if (IS_ERR(psys_adev->mmu)) {
 		put_device(&psys_adev->auxdev.dev);
 		return dev_err_cast_probe(&pdev->dev, psys_adev->mmu,
@@ -454,12 +475,13 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
 	return psys_adev;
 }
 
-static int ipu6_pci_config_setup(struct pci_dev *dev, u8 hw_ver)
+static int ipu6_pci_config_setup(struct pci_dev *dev)
 {
+	struct ipu6_device *isp = pci_get_drvdata(dev);
 	int ret;
 
 	/* No PCI msi capability for IPU6EP */
-	if (is_ipu6ep(hw_ver) || is_ipu6ep_mtl(hw_ver)) {
+	if (IS_IPU6EP(isp) || IS_IPU6EP_MTL(isp)) {
 		/* likely do nothing as msi not enabled by default */
 		pci_disable_msi(dev);
 		return 0;
@@ -474,7 +496,12 @@ static int ipu6_pci_config_setup(struct pci_dev *dev, u8 hw_ver)
 
 static void ipu6_configure_vc_mechanism(struct ipu6_device *isp)
 {
-	u32 val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL);
+	u32 val;
+
+	if (IS_IPU7(isp))
+		return;
+
+	val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL);
 
 	if (IPU6_BTRS_ARB_STALL_MODE_VC0 == IPU6_BTRS_ARB_MODE_TYPE_STALL)
 		val |= BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC0;
@@ -489,70 +516,193 @@ static void ipu6_configure_vc_mechanism(struct ipu6_device *isp)
 	writel(val, isp->base + BUTTRESS_REG_BTRS_CTRL);
 }
 
+static int __ipu6_map_fw_by_sys(struct ipu6_device *isp, struct ipu6_bus_device *adev)
+{
+	int ret;
+
+	ret = ipu6_map_fw_region(adev, isp->cpd_fw->data, isp->cpd_fw->size,
+				 DMA_TO_DEVICE, 0);
+	if (ret) {
+		dev_err_probe(&isp->pdev->dev, ret,
+			      "Firmware mapping failed\n");
+		return ret;
+	}
+
+	ret = ipu6_cpd_create_pkg_dir(adev, isp->cpd_fw->data);
+	if (ret) {
+		dev_err_probe(&isp->pdev->dev, ret,
+			      "failed to create pkg dir\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ipu6_map_fw(struct ipu6_device *isp)
+{
+	int ret;
+
+	ret = __ipu6_map_fw_by_sys(isp, isp->psys);
+	if (ret)
+		return ret;
+
+	if (!isp->secure_mode)
+		return __ipu6_map_fw_by_sys(isp, isp->isys);
+
+	return 0;
+}
+
+static int __ipu7_map_fw_non_secure(struct ipu6_device *isp)
+{
+	int ret;
+
+	/*
+	 * Allocate and map memory for running the firmware. Not
+	 * required in secure mode, in which firmware runs in IMR.
+	 */
+	isp->fw_code_region = vmalloc(IPU7_FW_CODE_REGION_SIZE);
+	if (!isp->fw_code_region)
+		return -ENOMEM;
+
+	ret = ipu6_ipu7_cpd_copy_binary(isp->cpd_fw->data, "isys",
+					isp->fw_code_region,
+					&isp->isys->fw_entry);
+	if (ret)
+		return ret;
+
+	ret = ipu6_map_fw_region(isp->isys, isp->fw_code_region,
+				 IPU7_FW_CODE_REGION_SIZE, DMA_BIDIRECTIONAL,
+				 DMA_ATTR_RESERVE_REGION);
+	if (ret)
+		return ret;
+
+	ret = ipu6_ipu7_cpd_copy_binary(isp->cpd_fw->data, "psys",
+					isp->fw_code_region,
+					&isp->psys->fw_entry);
+	if (ret)
+		return ret;
+
+	return ipu6_map_fw_region(isp->psys, isp->fw_code_region,
+				  IPU7_FW_CODE_REGION_SIZE, DMA_BIDIRECTIONAL,
+				  DMA_ATTR_RESERVE_REGION);
+}
+
+static int ipu7_map_fw(struct ipu6_device *isp)
+{
+	int ret;
+
+	ret = isp->secure_mode ?
+		ipu6_map_fw_region(isp->psys, isp->cpd_fw->data,
+				   isp->cpd_fw->size, DMA_BIDIRECTIONAL, 0) :
+		__ipu7_map_fw_non_secure(isp);
+	if (ret) {
+		dev_err_probe(&isp->pdev->dev, ret,
+			      "Failed to init ipu7 firmware region\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
-	struct ipu6_buttress_ctrl *isys_ctrl = NULL, *psys_ctrl = NULL;
+	const struct ipu6_buttress_ctrl *isys_ctrl, *psys_ctrl;
 	struct device *dev = &pdev->dev;
 	void __iomem *isys_base = NULL;
 	void __iomem *psys_base = NULL;
 	struct ipu6_device *isp;
 	phys_addr_t phys;
 	u32 val, version, sku_id;
+	unsigned long dir;
 	int ret;
 
+	if ((id->device == PCI_DEVICE_ID_INTEL_IPU7 ||
+	     id->device == PCI_DEVICE_ID_INTEL_IPU7P5) && force_no_probe_ipu7)
+		return -ENODEV;
+
 	isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL);
 	if (!isp)
 		return -ENOMEM;
 
-	isp->pdev = pdev;
-	INIT_LIST_HEAD(&isp->devices);
-
-	ret = pcim_enable_device(pdev);
-	if (ret)
-		return dev_err_probe(dev, ret, "Enable PCI device failed\n");
-
-	phys = pci_resource_start(pdev, IPU6_PCI_BAR);
-	dev_dbg(dev, "IPU6 PCI bar[%u] = %pa\n", IPU6_PCI_BAR, &phys);
-
-	isp->base = pcim_iomap_region(pdev, IPU6_PCI_BAR, IPU6_NAME);
-	if (IS_ERR(isp->base))
-		return dev_err_probe(dev, PTR_ERR(isp->base),
-				     "Failed to I/O mem remapping\n");
-
-	pci_set_drvdata(pdev, isp);
-	pci_set_master(pdev);
-
 	isp->cpd_metadata_cmpnt_size = sizeof(struct ipu6_cpd_metadata_cmpnt);
+	isp->buttress.regs = &ipu6_buttress_regs;
+	isp->model_name = IPU6_MEDIA_DEV_MODEL_NAME;
+	isys_ctrl = &ipu6_isys_buttress_ctrl;
+	psys_ctrl = &ipu6_psys_buttress_ctrl;
+
 	switch (id->device) {
 	case PCI_DEVICE_ID_INTEL_IPU6:
-		isp->hw_ver = IPU6_VER_6;
+		isp->hw_ver = IPU_VERSION_6;
 		isp->cpd_fw_name = IPU6_FIRMWARE_NAME;
 		break;
 	case PCI_DEVICE_ID_INTEL_IPU6SE:
-		isp->hw_ver = IPU6_VER_6SE;
+		isp->hw_ver = IPU_VERSION_6SE;
 		isp->cpd_fw_name = IPU6SE_FIRMWARE_NAME;
 		isp->cpd_metadata_cmpnt_size =
 			sizeof(struct ipu6se_cpd_metadata_cmpnt);
 		break;
 	case PCI_DEVICE_ID_INTEL_IPU6EP_ADLP:
 	case PCI_DEVICE_ID_INTEL_IPU6EP_RPLP:
-		isp->hw_ver = IPU6_VER_6EP;
+		isp->hw_ver = IPU_VERSION_6EP;
 		isp->cpd_fw_name = IPU6EP_FIRMWARE_NAME;
 		break;
 	case PCI_DEVICE_ID_INTEL_IPU6EP_ADLN:
-		isp->hw_ver = IPU6_VER_6EP;
+		isp->hw_ver = IPU_VERSION_6EP;
 		isp->cpd_fw_name = IPU6EPADLN_FIRMWARE_NAME;
 		break;
 	case PCI_DEVICE_ID_INTEL_IPU6EP_MTL:
-		isp->hw_ver = IPU6_VER_6EP_MTL;
+		isp->hw_ver = IPU_VERSION_6EP_MTL;
 		isp->cpd_fw_name = IPU6EPMTL_FIRMWARE_NAME;
 		break;
+	case PCI_DEVICE_ID_INTEL_IPU7:
+		isp->hw_ver = IPU_VERSION_7;
+		isp->cpd_fw_name = IPU7_FIRMWARE_NAME;
+		isp->model_name = IPU7_MEDIA_DEV_MODEL_NAME;
+		isp->buttress.regs = &ipu7_buttress_regs;
+		isys_ctrl = &ipu7_isys_buttress_ctrl;
+		psys_ctrl = &ipu7_psys_buttress_ctrl;
+		break;
+	case PCI_DEVICE_ID_INTEL_IPU7P5:
+		isp->hw_ver = IPU_VERSION_7P5;
+		isp->cpd_fw_name = IPU7P5_FIRMWARE_NAME;
+		isp->model_name = IPU7P5_MEDIA_DEV_MODEL_NAME;
+		isp->buttress.regs = &ipu7p5_buttress_regs;
+		isys_ctrl = &ipu7_isys_buttress_ctrl;
+		psys_ctrl = &ipu7_psys_buttress_ctrl;
+		break;
 	default:
 		return dev_err_probe(dev, -ENODEV,
 				     "Unsupported IPU6 device %x\n",
 				     id->device);
 	}
 
+	isp->pdev = pdev;
+	INIT_LIST_HEAD(&isp->devices);
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Enable PCI device failed\n");
+
+	phys = pci_resource_start(pdev, IPU6_PCI_BAR);
+	dev_dbg(dev, "IPU6 PCI bar[%u] = %pa\n", IPU6_PCI_BAR, &phys);
+
+	isp->base = pcim_iomap_region(pdev, IPU6_PCI_BAR, IPU6_NAME);
+	if (IS_ERR(isp->base))
+		return dev_err_probe(dev, PTR_ERR(isp->base),
+				     "Failed to I/O mem remapping\n");
+
+	if (IS_IPU7(isp)) {
+		isp->pb_base = pcim_iomap_region(pdev, IPU7_PCI_PBBAR,
+						 IPU6_NAME);
+		if (IS_ERR(isp->pb_base))
+			return dev_err_probe(dev, PTR_ERR(isp->pb_base),
+					     "I/O remapping PB BAR %u failed\n",
+					     IPU7_PCI_PBBAR);
+	}
+
+	pci_set_drvdata(pdev, isp);
+	pci_set_master(pdev);
+
 	ipu6_internal_pdata_init(isp);
 
 	isys_base = isp->base + isys_ipdata.hw_variant.offset;
@@ -564,7 +714,7 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	dma_set_max_seg_size(dev, UINT_MAX);
 
-	ret = ipu6_pci_config_setup(pdev, isp->hw_ver);
+	ret = ipu6_pci_config_setup(pdev);
 	if (ret)
 		return ret;
 
@@ -588,13 +738,6 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto out_ipu6_bus_del_devices;
 	}
 
-	isys_ctrl = devm_kmemdup(dev, &isys_buttress_ctrl,
-				 sizeof(isys_buttress_ctrl), GFP_KERNEL);
-	if (!isys_ctrl) {
-		ret = -ENOMEM;
-		goto out_ipu6_bus_del_devices;
-	}
-
 	isp->isys = ipu6_isys_init(pdev, dev, isys_ctrl, isys_base,
 				   &isys_ipdata);
 	if (IS_ERR(isp->isys)) {
@@ -602,13 +745,6 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto out_ipu6_bus_del_devices;
 	}
 
-	psys_ctrl = devm_kmemdup(dev, &psys_buttress_ctrl,
-				 sizeof(psys_buttress_ctrl), GFP_KERNEL);
-	if (!psys_ctrl) {
-		ret = -ENOMEM;
-		goto out_ipu6_bus_del_devices;
-	}
-
 	isp->psys = ipu6_psys_init(pdev, &isp->isys->auxdev.dev, psys_ctrl,
 				   psys_base, &psys_ipdata);
 	if (IS_ERR(isp->psys)) {
@@ -627,19 +763,9 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto out_ipu6_rpm_put;
 	}
 
-	ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw,
-					 &isp->psys->fw_sgt);
-	if (ret) {
-		dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n");
-		goto out_ipu6_rpm_put;
-	}
-
-	ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data);
-	if (ret) {
-		dev_err_probe(&isp->pdev->dev, ret,
-			      "failed to create pkg dir\n");
+	ret = IS_IPU7(isp) ? ipu7_map_fw(isp) : ipu6_map_fw(isp);
+	if (ret)
 		goto out_ipu6_rpm_put;
-	}
 
 	ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr,
 					ipu6_buttress_isr_threaded,
@@ -662,7 +788,7 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	/* Configure the arbitration mechanisms for VC requests */
 	ipu6_configure_vc_mechanism(isp);
 
-	val = readl(isp->base + BUTTRESS_REG_SKU);
+	val = readl(isp->base + isp->buttress.regs->sku_id);
 	sku_id = FIELD_GET(GENMASK(6, 4), val);
 	version = FIELD_GET(GENMASK(3, 0), val);
 	dev_info(dev, "IPU%u-v%u[%x] hardware version %d\n", version, sku_id,
@@ -680,10 +806,21 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 out_ipu6_rpm_put:
 	pm_runtime_put_sync(&isp->psys->auxdev.dev);
 out_ipu6_bus_del_devices:
+	dir = IS_IPU7(isp) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 	if (!IS_ERR_OR_NULL(isp->psys)) {
 		ipu6_cpd_free_pkg_dir(isp->psys);
-		ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);
+		if (isp->psys->fw_sgt.nents)
+			ipu6_unmap_fw_region(isp->psys, dir);
+	}
+	if (!IS_ERR_OR_NULL(isp->isys)) {
+		ipu6_cpd_free_pkg_dir(isp->isys);
+		if (isp->isys->fw_sgt.nents)
+			ipu6_unmap_fw_region(isp->isys, dir);
 	}
+
+	vfree(isp->fw_code_region);
+	isp->fw_code_region = NULL;
+
 	if (!IS_ERR_OR_NULL(isp->psys) && !IS_ERR_OR_NULL(isp->psys->mmu))
 		ipu6_mmu_cleanup(isp->psys->mmu);
 	if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu))
@@ -699,13 +836,23 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 static void ipu6_pci_remove(struct pci_dev *pdev)
 {
 	struct ipu6_device *isp = pci_get_drvdata(pdev);
-	struct ipu6_mmu *isys_mmu = isp->isys->mmu;
-	struct ipu6_mmu *psys_mmu = isp->psys->mmu;
+	unsigned long dir;
 
 	devm_free_irq(&pdev->dev, pdev->irq, isp);
+
+	dir = IS_IPU7(isp) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 	ipu6_cpd_free_pkg_dir(isp->psys);
+	ipu6_unmap_fw_region(isp->psys, dir);
+
+	if (isp->isys) {
+		ipu6_cpd_free_pkg_dir(isp->isys);
+		if (isp->isys->fw_sgt.nents)
+			ipu6_unmap_fw_region(isp->isys, dir);
+	}
+
+	vfree(isp->fw_code_region);
+	isp->fw_code_region = NULL;
 
-	ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);
 	ipu6_buttress_exit(isp);
 
 	ipu6_bus_del_devices(pdev);
@@ -715,8 +862,10 @@ static void ipu6_pci_remove(struct pci_dev *pdev)
 
 	release_firmware(isp->cpd_fw);
 
-	ipu6_mmu_cleanup(psys_mmu);
-	ipu6_mmu_cleanup(isys_mmu);
+	if (isp->psys)
+		ipu6_mmu_cleanup(isp->psys->mmu);
+	if (isp->isys)
+		ipu6_mmu_cleanup(isp->isys->mmu);
 }
 
 static void ipu6_pci_reset_prepare(struct pci_dev *pdev)
@@ -754,7 +903,6 @@ static int ipu6_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct ipu6_device *isp = pci_get_drvdata(pdev);
-	struct ipu6_buttress *b = &isp->buttress;
 	int ret;
 
 	/* Configure the arbitration mechanisms for VC requests */
@@ -766,7 +914,7 @@ static int ipu6_resume(struct device *dev)
 
 	ipu6_buttress_restore(isp);
 
-	ret = ipu6_buttress_ipc_reset(isp, &b->cse);
+	ret = ipu6_buttress_ipc_reset(isp);
 	if (ret)
 		dev_err(&isp->pdev->dev, "IPC reset protocol failed!\n");
 
@@ -795,10 +943,8 @@ static int ipu6_runtime_resume(struct device *dev)
 	ipu6_buttress_restore(isp);
 
 	if (isp->need_ipc_reset) {
-		struct ipu6_buttress *b = &isp->buttress;
-
 		isp->need_ipc_reset = false;
-		ret = ipu6_buttress_ipc_reset(isp, &b->cse);
+		ret = ipu6_buttress_ipc_reset(isp);
 		if (ret)
 			dev_err(&isp->pdev->dev, "IPC reset protocol failed\n");
 	}
diff --git a/drivers/media/pci/intel/ipu6/ipu6.h b/drivers/media/pci/intel/ipu6/ipu6.h
index 92e3c3414c917d..72c5063d5198f8 100644
--- a/drivers/media/pci/intel/ipu6/ipu6.h
+++ b/drivers/media/pci/intel/ipu6/ipu6.h
@@ -16,46 +16,32 @@ struct ipu6_bus_device;
 
 #define IPU6_NAME			"intel-ipu6"
 #define IPU6_MEDIA_DEV_MODEL_NAME	"ipu6"
+#define IPU7_MEDIA_DEV_MODEL_NAME	"ipu7"
+#define IPU7P5_MEDIA_DEV_MODEL_NAME	"ipu7.5"
 
 #define IPU6SE_FIRMWARE_NAME		"intel/ipu/ipu6se_fw.bin"
 #define IPU6EP_FIRMWARE_NAME		"intel/ipu/ipu6ep_fw.bin"
 #define IPU6_FIRMWARE_NAME		"intel/ipu/ipu6_fw.bin"
 #define IPU6EPMTL_FIRMWARE_NAME		"intel/ipu/ipu6epmtl_fw.bin"
 #define IPU6EPADLN_FIRMWARE_NAME	"intel/ipu/ipu6epadln_fw.bin"
-
-enum ipu6_version {
-	IPU6_VER_INVALID = 0,
-	IPU6_VER_6 = 1,
-	IPU6_VER_6SE = 3,
-	IPU6_VER_6EP = 5,
-	IPU6_VER_6EP_MTL = 6,
-};
-
-/*
- * IPU6 - TGL
- * IPU6SE - JSL
- * IPU6EP - ADL/RPL
- * IPU6EP_MTL - MTL
- */
-static inline bool is_ipu6se(u8 hw_ver)
-{
-	return hw_ver == IPU6_VER_6SE;
-}
-
-static inline bool is_ipu6ep(u8 hw_ver)
-{
-	return hw_ver == IPU6_VER_6EP;
-}
-
-static inline bool is_ipu6ep_mtl(u8 hw_ver)
-{
-	return hw_ver == IPU6_VER_6EP_MTL;
-}
-
-static inline bool is_ipu6_tgl(u8 hw_ver)
-{
-	return hw_ver == IPU6_VER_6;
-}
+#define IPU7_FIRMWARE_NAME		"intel/ipu/ipu7_fw.bin"
+#define IPU7P5_FIRMWARE_NAME		"intel/ipu/ipu7ptl_fw.bin"
+
+#define IPU_VERSION_6		BIT(0) /* TGL */
+#define IPU_VERSION_6SE		BIT(1) /* JSL */
+#define IPU_VERSION_6EP		BIT(2) /* ADL/RPL */
+#define IPU_VERSION_6EP_MTL	BIT(3) /* MTL */
+#define IPU_VERSION_7		BIT(4) /* LNL */
+#define IPU_VERSION_7P5		BIT(5) /* PTL */
+
+#define IS_IPU6_TGL(isp)	((isp)->hw_ver & IPU_VERSION_6)
+#define IS_IPU6SE(isp)		((isp)->hw_ver & IPU_VERSION_6SE)
+#define IS_IPU6EP(isp)		((isp)->hw_ver & IPU_VERSION_6EP)
+#define IS_IPU6EP_MTL(isp)	((isp)->hw_ver & IPU_VERSION_6EP_MTL)
+#define IS_IPU7(isp)		((isp)->hw_ver & \
+				 (IPU_VERSION_7 | IPU_VERSION_7P5))
+#define IS_IPU7_MTL(isp)	((isp)->hw_ver & IPU_VERSION_7)
+#define IS_IPU7P5(isp)		((isp)->hw_ver & IPU_VERSION_7P5)
 
 /*
  * ISYS DMA can overshoot. For higher resolutions over allocation is one line
@@ -80,14 +66,21 @@ struct ipu6_device {
 	const struct firmware *cpd_fw;
 	const char *cpd_fw_name;
 	u32 cpd_metadata_cmpnt_size;
+	const char *model_name;
 
 	void __iomem *base;
+	void __iomem *pb_base;
 	bool need_ipc_reset;
 	bool secure_mode;
 	u8 hw_ver;
 	bool bus_ready_to_probe;
+	u32 *fw_code_region;
 };
 
+#define IPU_PSYS	0
+#define IPU_ISYS	1
+#define IPU_SUBSYS_NUM	2
+
 #define IPU6_ISYS_NAME "isys"
 #define IPU6_PSYS_NAME "psys"
 
@@ -134,141 +127,6 @@ struct ipu6_device {
 #define IPU6_BTRS_ARB_STALL_MODE_VC1	\
 			IPU6_BTRS_ARB_MODE_TYPE_REARB
 
-/*
- * MMU Invalidation HW bug workaround by ZLW mechanism
- *
- * Old IPU6 MMUV2 has a bug in the invalidation mechanism which might result in
- * wrong translation or replication of the translation. This will cause data
- * corruption. So we cannot directly use the MMU V2 invalidation registers
- * to invalidate the MMU. Instead, whenever an invalidate is called, we need to
- * clear the TLB by evicting all the valid translations by filling it with trash
- * buffer (which is guaranteed not to be used by any other processes). ZLW is
- * used to fill the L1 and L2 caches with the trash buffer translations. ZLW
- * or Zero length write, is pre-fetch mechanism to pre-fetch the pages in
- * advance to the L1 and L2 caches without triggering any memory operations.
- *
- * In MMU V2, L1 -> 16 streams and 64 blocks, maximum 16 blocks per stream
- * One L1 block has 16 entries, hence points to 16 * 4K pages
- * L2 -> 16 streams and 32 blocks. 2 blocks per streams
- * One L2 block maps to 1024 L1 entries, hence points to 4MB address range
- * 2 blocks per L2 stream means, 1 stream points to 8MB range
- *
- * As we need to clear the caches and 8MB being the biggest cache size, we need
- * to have trash buffer which points to 8MB address range. As these trash
- * buffers are not used for any memory transactions, we need only the least
- * amount of physical memory. So we reserve 8MB IOVA address range but only
- * one page is reserved from physical memory. Each of this 8MB IOVA address
- * range is then mapped to the same physical memory page.
- */
-/* One L2 entry maps 1024 L1 entries and one L1 entry per page */
-#define IPU6_MMUV2_L2_RANGE		(1024 * PAGE_SIZE)
-/* Max L2 blocks per stream */
-#define IPU6_MMUV2_MAX_L2_BLOCKS	2
-/* Max L1 blocks per stream */
-#define IPU6_MMUV2_MAX_L1_BLOCKS	16
-#define IPU6_MMUV2_TRASH_RANGE	(IPU6_MMUV2_L2_RANGE * IPU6_MMUV2_MAX_L2_BLOCKS)
-/* Entries per L1 block */
-#define MMUV2_ENTRIES_PER_L1_BLOCK	16
-#define MMUV2_TRASH_L1_BLOCK_OFFSET	(MMUV2_ENTRIES_PER_L1_BLOCK * PAGE_SIZE)
-#define MMUV2_TRASH_L2_BLOCK_OFFSET	IPU6_MMUV2_L2_RANGE
-
-/*
- * In some of the IPU6 MMUs, there is provision to configure L1 and L2 page
- * table caches. Both these L1 and L2 caches are divided into multiple sections
- * called streams. There is maximum 16 streams for both caches. Each of these
- * sections are subdivided into multiple blocks. When nr_l1streams = 0 and
- * nr_l2streams = 0, means the MMU is of type MMU_V1 and do not support
- * L1/L2 page table caches.
- *
- * L1 stream per block sizes are configurable and varies per usecase.
- * L2 has constant block sizes - 2 blocks per stream.
- *
- * MMU1 support pre-fetching of the pages to have less cache lookup misses. To
- * enable the pre-fetching, MMU1 AT (Address Translator) device registers
- * need to be configured.
- *
- * There are four types of memory accesses which requires ZLW configuration.
- * ZLW(Zero Length Write) is a mechanism to enable VT-d pre-fetching on IOMMU.
- *
- * 1. Sequential Access or 1D mode
- *	Set ZLW_EN -> 1
- *	set ZLW_PAGE_CROSS_1D -> 1
- *	Set ZLW_N to "N" pages so that ZLW will be inserte N pages ahead where
- *		  N is pre-defined and hardcoded in the platform data
- *	Set ZLW_2D -> 0
- *
- * 2. ZLW 2D mode
- *	Set ZLW_EN -> 1
- *	set ZLW_PAGE_CROSS_1D -> 1,
- *	Set ZLW_N -> 0
- *	Set ZLW_2D -> 1
- *
- * 3. ZLW Enable (no 1D or 2D mode)
- *	Set ZLW_EN -> 1
- *	set ZLW_PAGE_CROSS_1D -> 0,
- *	Set ZLW_N -> 0
- *	Set ZLW_2D -> 0
- *
- * 4. ZLW disable
- *	Set ZLW_EN -> 0
- *	set ZLW_PAGE_CROSS_1D -> 0,
- *	Set ZLW_N -> 0
- *	Set ZLW_2D -> 0
- *
- * To configure the ZLW for the above memory access, four registers are
- * available. Hence to track these four settings, we have the following entries
- * in the struct ipu6_mmu_hw. Each of these entries are per stream and
- * available only for the L1 streams.
- *
- * a. l1_zlw_en -> To track zlw enabled per stream (ZLW_EN)
- * b. l1_zlw_1d_mode -> Track 1D mode per stream. ZLW inserted at page boundary
- * c. l1_ins_zlw_ahead_pages -> to track how advance the ZLW need to be inserted
- *			Insert ZLW request N pages ahead address.
- * d. l1_zlw_2d_mode -> To track 2D mode per stream (ZLW_2D)
- *
- *
- * Currently L1/L2 streams, blocks, AT ZLW configurations etc. are pre-defined
- * as per the usecase specific calculations. Any change to this pre-defined
- * table has to happen in sync with IPU6 FW.
- */
-struct ipu6_mmu_hw {
-	union {
-		unsigned long offset;
-		void __iomem *base;
-	};
-	u32 info_bits;
-	u8 nr_l1streams;
-	/*
-	 * L1 has variable blocks per stream - total of 64 blocks and maximum of
-	 * 16 blocks per stream. Configurable by using the block start address
-	 * per stream. Block start address is calculated from the block size
-	 */
-	u8 l1_block_sz[IPU6_MMU_MAX_TLB_L1_STREAMS];
-	/* Is ZLW is enabled in each stream */
-	bool l1_zlw_en[IPU6_MMU_MAX_TLB_L1_STREAMS];
-	bool l1_zlw_1d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
-	u8 l1_ins_zlw_ahead_pages[IPU6_MMU_MAX_TLB_L1_STREAMS];
-	bool l1_zlw_2d_mode[IPU6_MMU_MAX_TLB_L1_STREAMS];
-
-	u32 l1_stream_id_reg_offset;
-	u32 l2_stream_id_reg_offset;
-
-	u8 nr_l2streams;
-	/*
-	 * L2 has fixed 2 blocks per stream. Block address is calculated
-	 * from the block size
-	 */
-	u8 l2_block_sz[IPU6_MMU_MAX_TLB_L2_STREAMS];
-	/* flag to track if WA is needed for successive invalidate HW bug */
-	bool insert_read_before_invalidate;
-};
-
-struct ipu6_mmu_pdata {
-	u32 nr_mmus;
-	struct ipu6_mmu_hw mmu_hw[IPU6_MMU_MAX_DEVICES];
-	int mmid;
-};
-
 struct ipu6_isys_csi2_pdata {
 	void __iomem *base;
 };
@@ -283,6 +141,8 @@ struct ipu6_isys_internal_csi2_pdata {
 	u32 ctrl0_irq_lnp;
 	u32 ctrl0_irq_status;
 	u32 fw_access_port_ofs;
+	/* IPU7-specific field */
+	u32 gpreg;
 };
 
 struct ipu6_isys_internal_tpg_pdata {
@@ -293,8 +153,6 @@ struct ipu6_isys_internal_tpg_pdata {
 
 struct ipu6_hw_variants {
 	unsigned long offset;
-	u32 nr_mmus;
-	struct ipu6_mmu_hw mmu_hw[IPU6_MMU_MAX_DEVICES];
 	u8 cdc_fifos;
 	u8 cdc_fifo_threshold[IPU6_MAX_VC_IOSF_PORTS];
 	u32 dmem_offset;
diff --git a/drivers/media/pci/intel/ipu6/ipu7-boot.c b/drivers/media/pci/intel/ipu6/ipu7-boot.c
new file mode 100644
index 00000000000000..952c73efd23708
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-boot.c
@@ -0,0 +1,405 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 - 2026 Intel Corporation
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/iopoll.h>
+#include <linux/types.h>
+
+#include "ipu6.h"
+#include "ipu6-bus.h"
+#include "ipu6-buttress.h"
+#include "ipu6-dma.h"
+#include "ipu6-isys.h"
+#include "ipu6-platform-buttress-regs.h"
+#include "ipu7-boot.h"
+#include "ipu7-platform-regs.h"
+
+#define IPU7_FW_START_STOP_TIMEOUT		2000
+#define IPU7_BOOT_CELL_RESET_TIMEOUT		(2 * USEC_PER_SEC)
+#define IPU7_BOOT_STATE_CRITICAL(s)		(((s) & 0xffff0000U) == 0xdead0000U)
+#define IPU7_BOOT_STATE_READY(s)		((s) == 0x57a7e100U)
+#define IPU7_BOOT_STATE_INACTIVE(s)		((s) == 0x57a7e300U)
+#define IPU7_BUTTRESS_REG_FW_BOOT_PARAMS0				0x4000
+#define IPU7_BUTTRESS_FW_BOOT_PARAMS_ENTRY(i) \
+	(IPU7_BUTTRESS_REG_FW_BOOT_PARAMS0 + ((i) * 4U))
+
+struct boot_regs {
+	u32 base;
+	u32 dmem_address;
+	u32 status_ctrl_reg;
+	u32 fw_start_address_reg;
+	u32 fw_code_base_reg;
+};
+
+enum ipu7_boot_reg_id {
+	IPU7_FW_BOOT_CONFIG_ID = 0,
+	IPU7_FW_BOOT_STATE_ID = 1,
+	IPU7_FW_BOOT_SYSCOM_QUEUE_INDICES_BASE_ID = 2,
+	IPU7_FW_BOOT_UNTRUSTED_ADDR_MIN_ID = 3,
+	IPU7_FW_BOOT_MESSAGING_VERSION_ID = 4,
+	IPU7_FW_BOOT_ID_MAX,
+};
+
+enum ipu7_boot_state {
+	IPU7_FW_BOOT_STATE_UNINIT = 0x57a7e000U,
+	IPU7_FW_BOOT_STATE_READY = 0x57a7e100U,
+	IPU7_FW_BOOT_STATE_SHUTDOWN_CMD = 0x57a7f001U,
+	IPU7_FW_BOOT_STATE_INACTIVE = 0x57a7e300U,
+};
+
+static const struct boot_regs boot_regs[IPU_SUBSYS_NUM] = {
+	[IPU_ISYS] = {
+		.dmem_address = IPU7_ISYS_DMEM_OFFSET,
+		.status_ctrl_reg = IPU7_BUTTRESS_REG_ISYS_UCX_CTRL_STATUS,
+		.fw_start_address_reg = IPU7_BUTTRESS_REG_ISYS_UCX_START_ADDR,
+		.fw_code_base_reg = IPU7_IS_UC_CTRL_BASE
+	},
+	[IPU_PSYS] = {
+		.dmem_address = IPU7_PSYS_DMEM_OFFSET,
+		.status_ctrl_reg = IPU7_BUTTRESS_REG_PSYS_UCX_CTRL_STATUS,
+		.fw_start_address_reg = IPU7_BUTTRESS_REG_PSYS_UCX_START_ADDR,
+		.fw_code_base_reg = IPU7_PS_UC_CTRL_BASE
+	}
+};
+
+static u32 get_fw_boot_reg_addr(const struct ipu6_bus_device *adev,
+				enum ipu7_boot_reg_id reg)
+{
+	u32 base = (adev->ctrl->subsys_id == IPU_ISYS) ?
+				0U : (u32)IPU7_FW_BOOT_ID_MAX;
+
+	return IPU7_BUTTRESS_FW_BOOT_PARAMS_ENTRY(base + (u32)reg);
+}
+
+static void write_fw_boot_param(const struct ipu6_bus_device *adev,
+				enum ipu7_boot_reg_id reg,
+				u32 val)
+{
+	void __iomem *base = adev->isp->base;
+
+	dev_dbg(&adev->auxdev.dev,
+		"write boot param reg: %d addr: %x val: 0x%x\n",
+		reg, get_fw_boot_reg_addr(adev, reg), val);
+	writel(val, base + get_fw_boot_reg_addr(adev, reg));
+}
+
+static u32 read_fw_boot_param(const struct ipu6_bus_device *adev,
+			      enum ipu7_boot_reg_id reg)
+{
+	void __iomem *base = adev->isp->base;
+
+	return readl(base + get_fw_boot_reg_addr(adev, reg));
+}
+
+static int ipu7_boot_cell_reset(const struct ipu6_bus_device *adev)
+{
+	const struct device *dev = &adev->auxdev.dev;
+	const struct boot_regs *regs = &boot_regs[adev->ctrl->subsys_id];
+	u32 ucx_ctrl_status = regs->status_ctrl_reg;
+	u32 timeout = IPU7_BOOT_CELL_RESET_TIMEOUT;
+	void __iomem *base = adev->isp->base;
+	u32 val, val2;
+	int ret;
+
+	val = readl(base + ucx_ctrl_status);
+	val |= IPU7_UCX_CTL_RESET;
+	val &= ~IPU7_UCX_CTL_RUN;
+
+	writel(val, base + ucx_ctrl_status);
+
+	ret = readl_poll_timeout(base + ucx_ctrl_status, val2,
+				 (val2 & 0x3U) == (val & 0x3U), 100, timeout);
+	if (ret) {
+		dev_err(dev, "cell enter reset timeout. status: 0x%x\n", val2);
+		return -ETIMEDOUT;
+	}
+
+	val = readl(base + ucx_ctrl_status);
+	val &= ~(IPU7_UCX_CTL_RESET | IPU7_UCX_CTL_RUN);
+	writel(val, base + ucx_ctrl_status);
+
+	ret = readl_poll_timeout(base + ucx_ctrl_status, val2,
+				 (val2 & 0x3U) == (val & 0x3U), 100, timeout);
+	if (ret) {
+		dev_err(dev, "cell exit reset timeout. status: 0x%x\n", val2);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void ipu7_boot_cell_start(const struct ipu6_bus_device *adev)
+{
+	const struct boot_regs *regs = &boot_regs[adev->ctrl->subsys_id];
+	void __iomem *base = adev->isp->base;
+	u32 val;
+
+	val = readl(base + regs->status_ctrl_reg);
+	WARN_ON(val & (IPU7_UCX_CTL_RESET | IPU7_UCX_CTL_RUN));
+
+	val &= ~IPU7_UCX_CTL_RESET;
+	val |= IPU7_UCX_CTL_RUN;
+	writel(val, base + regs->status_ctrl_reg);
+}
+
+static void ipu7_boot_cell_stop(const struct ipu6_bus_device *adev)
+{
+	const struct boot_regs *regs = &boot_regs[adev->ctrl->subsys_id];
+	void __iomem *base = adev->isp->base;
+	u32 val;
+
+	val = readl(base + regs->status_ctrl_reg);
+	val &= ~IPU7_UCX_CTL_RUN;
+	writel(val, base + regs->status_ctrl_reg);
+
+	/* Wait for uC transactions complete */
+	usleep_range(10, 20);
+
+	val = readl(base + regs->status_ctrl_reg);
+	val |= IPU7_UCX_CTL_RESET;
+	writel(val, base + regs->status_ctrl_reg);
+}
+
+static int ipu7_boot_cell_init(const struct ipu6_bus_device *adev)
+{
+	const struct boot_regs *regs = &boot_regs[adev->ctrl->subsys_id];
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	struct ipu7_fw_com_context *fwctx = isys->fwctx;
+	void __iomem *base = adev->isp->base;
+
+	writel(fwctx->fw_entry, base + regs->fw_start_address_reg);
+
+	return ipu7_boot_cell_reset(adev);
+}
+
+static void init_cfg_versions(struct ipu7_boot_abi_cfg *boot_cfg, u32 length, u8 major)
+{
+	boot_cfg->length = length;
+	boot_cfg->config_version.major = 1U;
+	boot_cfg->config_version.minor = 0U;
+	boot_cfg->config_version.subminor = 0U;
+	boot_cfg->config_version.patch = 0U;
+
+	boot_cfg->client_version_support.num_versions = 1U;
+	boot_cfg->client_version_support.versions[0].major = major;
+	boot_cfg->client_version_support.versions[0].minor = 0U;
+	boot_cfg->client_version_support.versions[0].subminor = 0U;
+	boot_cfg->client_version_support.versions[0].patch = 0U;
+}
+
+int ipu6_ipu7_init_boot_config(struct ipu6_bus_device *adev,
+			       struct ipu7_fw_com_queue_config *qconfigs,
+			       int num_queues, u32 uc_freq,
+			       dma_addr_t subsys_config, u8 major)
+{
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	struct ipu7_fw_com_context *fwctx = isys->fwctx;
+	struct ipu7_boot_abi_cfg *boot_config;
+	struct ipu7_fw_com_queue_params_config *cfgs;
+	struct device *dev = &adev->auxdev.dev;
+	u32 total_queue_size_aligned = 0;
+	dma_addr_t queue_mem_dma_ptr;
+	void *queue_mem_ptr;
+	unsigned int i;
+
+	/* Allocate boot config. */
+	fwctx->boot_config_size =
+		sizeof(*cfgs) * num_queues + sizeof(*boot_config);
+	fwctx->boot_config = ipu6_dma_alloc(adev, fwctx->boot_config_size,
+					    &fwctx->boot_config_dma_addr,
+					    GFP_KERNEL, 0);
+	if (!fwctx->boot_config) {
+		dev_err(dev, "Failed to allocate boot config.\n");
+		return -ENOMEM;
+	}
+
+	boot_config = fwctx->boot_config;
+	memset(boot_config, 0, sizeof(*boot_config));
+	init_cfg_versions(boot_config, fwctx->boot_config_size, major);
+	boot_config->subsys_config = subsys_config;
+
+	boot_config->uc_tile_frequency = uc_freq;
+	boot_config->uc_tile_frequency_units = 0;
+	boot_config->fw_com_config.max_output_queues =
+		fwctx->num_output_queues;
+	boot_config->fw_com_config.max_input_queues =
+		fwctx->num_input_queues;
+
+	ipu6_dma_sync_single(adev, fwctx->boot_config_dma_addr,
+			     fwctx->boot_config_size);
+
+	for (i = 0; i < num_queues; i++) {
+		u32 queue_size = qconfigs[i].max_capacity *
+			qconfigs[i].token_size_in_bytes;
+
+		queue_size = ALIGN(queue_size, 64U);
+		total_queue_size_aligned += queue_size;
+		qconfigs[i].queue_size = queue_size;
+	}
+
+	/* Allocate queue memory */
+	fwctx->queue_mem = ipu6_dma_alloc(adev, total_queue_size_aligned,
+					  &fwctx->queue_mem_dma_addr,
+					  GFP_KERNEL, 0);
+	if (!fwctx->queue_mem) {
+		dev_err(dev, "Failed to allocate queue memory.\n");
+		return -ENOMEM;
+	}
+	fwctx->queue_mem_size = total_queue_size_aligned;
+
+	cfgs = ipu7_fw_com_get_queue_config(&boot_config->fw_com_config);
+	queue_mem_ptr = fwctx->queue_mem;
+	queue_mem_dma_ptr = fwctx->queue_mem_dma_addr;
+	for (i = 0; i < num_queues; i++) {
+		cfgs[i].token_array_mem = queue_mem_dma_ptr;
+		cfgs[i].max_capacity = qconfigs[i].max_capacity;
+		cfgs[i].token_size_in_bytes = qconfigs[i].token_size_in_bytes;
+		qconfigs[i].token_array_mem = queue_mem_ptr;
+		queue_mem_dma_ptr += qconfigs[i].queue_size;
+		queue_mem_ptr += qconfigs[i].queue_size;
+	}
+
+	ipu6_dma_sync_single(adev, fwctx->queue_mem_dma_addr,
+			     total_queue_size_aligned);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(ipu6_ipu7_init_boot_config, "INTEL_IPU6");
+
+void ipu6_ipu7_release_boot_config(struct ipu6_bus_device *adev)
+{
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	struct ipu7_fw_com_context *fwctx;
+
+	if (!isys || !isys->fwctx)
+		return;
+
+	fwctx = isys->fwctx;
+
+	if (fwctx->queue_mem) {
+		ipu6_dma_free(adev, fwctx->queue_mem_size,
+			      fwctx->queue_mem,
+			      fwctx->queue_mem_dma_addr, 0);
+		fwctx->queue_mem = NULL;
+		fwctx->queue_mem_dma_addr = 0;
+	}
+
+	if (fwctx->boot_config) {
+		ipu6_dma_free(adev, fwctx->boot_config_size,
+			      fwctx->boot_config,
+			      fwctx->boot_config_dma_addr, 0);
+		fwctx->boot_config = NULL;
+		fwctx->boot_config_dma_addr = 0;
+	}
+}
+EXPORT_SYMBOL_NS_GPL(ipu6_ipu7_release_boot_config, "INTEL_IPU6");
+
+int ipu6_ipu7_boot_start_fw(const struct ipu6_bus_device *adev)
+{
+	const struct device *dev = &adev->auxdev.dev;
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	struct ipu7_fw_com_context *fwctx = isys->fwctx;
+	u32 timeout = IPU7_FW_START_STOP_TIMEOUT;
+	void __iomem *base = adev->isp->base;
+	u32 boot_state, last_boot_state;
+	u32 indices_addr, msg_ver, id;
+	int ret;
+
+	ret = ipu7_boot_cell_init(adev);
+	if (ret)
+		return ret;
+
+	/* store "uninit" state to boot state reg */
+	write_fw_boot_param(adev, IPU7_FW_BOOT_STATE_ID,
+			    IPU7_FW_BOOT_STATE_UNINIT);
+	/* Set registers to zero, recommended for diagnostics. */
+	write_fw_boot_param(adev,
+			    IPU7_FW_BOOT_SYSCOM_QUEUE_INDICES_BASE_ID, 0);
+	write_fw_boot_param(adev, IPU7_FW_BOOT_MESSAGING_VERSION_ID, 0);
+	/* store firmware configuration address */
+	write_fw_boot_param(adev, IPU7_FW_BOOT_CONFIG_ID,
+			    fwctx->boot_config_dma_addr);
+
+	ipu7_boot_cell_start(adev);
+
+	last_boot_state = IPU7_FW_BOOT_STATE_UNINIT;
+	while (timeout--) {
+		boot_state = read_fw_boot_param(adev,
+						IPU7_FW_BOOT_STATE_ID);
+		if (boot_state != last_boot_state) {
+			dev_dbg(dev, "boot state changed from 0x%x to 0x%x\n",
+				last_boot_state, boot_state);
+			last_boot_state = boot_state;
+		}
+		if (IPU7_BOOT_STATE_CRITICAL(boot_state) ||
+		    IPU7_BOOT_STATE_READY(boot_state))
+			break;
+		usleep_range(1000, 1200);
+	}
+
+	if (IPU7_BOOT_STATE_CRITICAL(boot_state)) {
+		dev_err(dev, "critical boot state error 0x%x\n", boot_state);
+		return -EINVAL;
+	} else if (!IPU7_BOOT_STATE_READY(boot_state)) {
+		dev_err(dev, "fw boot timeout. state: 0x%x\n", boot_state);
+		return -ETIMEDOUT;
+	}
+	dev_dbg(dev, "fw boot done.\n");
+
+	id = IPU7_FW_BOOT_SYSCOM_QUEUE_INDICES_BASE_ID;
+	indices_addr = read_fw_boot_param(adev, id);
+	fwctx->queue_indices = base + indices_addr;
+	dev_dbg(dev, "fw queue indices offset is 0x%x\n", indices_addr);
+
+	msg_ver = read_fw_boot_param(adev,
+				     IPU7_FW_BOOT_MESSAGING_VERSION_ID);
+	dev_dbg(dev, "ipu message version is 0x%08x\n", msg_ver);
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(ipu6_ipu7_boot_start_fw, "INTEL_IPU6");
+
+int ipu6_ipu7_boot_stop_fw(const struct ipu6_bus_device *adev)
+{
+	const struct device *dev = &adev->auxdev.dev;
+	u32 timeout = IPU7_FW_START_STOP_TIMEOUT;
+	u32 boot_state;
+
+	boot_state = read_fw_boot_param(adev, IPU7_FW_BOOT_STATE_ID);
+	if (IPU7_BOOT_STATE_CRITICAL(boot_state) ||
+	    !IPU7_BOOT_STATE_READY(boot_state)) {
+		dev_err(dev, "fw not ready for shutdown, state 0x%x\n",
+			boot_state);
+		return -EBUSY;
+	}
+
+	/* Issue shutdown to start shutdown process */
+	dev_dbg(dev, "stopping fw...\n");
+	write_fw_boot_param(adev, IPU7_FW_BOOT_STATE_ID,
+			    IPU7_FW_BOOT_STATE_SHUTDOWN_CMD);
+	while (timeout--) {
+		boot_state = read_fw_boot_param(adev,
+						IPU7_FW_BOOT_STATE_ID);
+		if (IPU7_BOOT_STATE_CRITICAL(boot_state) ||
+		    IPU7_BOOT_STATE_INACTIVE(boot_state))
+			break;
+		usleep_range(1000, 1200);
+	}
+
+	if (IPU7_BOOT_STATE_CRITICAL(boot_state)) {
+		dev_err(dev, "critical boot state error 0x%x\n", boot_state);
+		return -EINVAL;
+	} else if (!IPU7_BOOT_STATE_INACTIVE(boot_state)) {
+		dev_err(dev, "stop fw timeout. state: 0x%x\n", boot_state);
+		return -ETIMEDOUT;
+	}
+
+	ipu7_boot_cell_stop(adev);
+	dev_dbg(dev, "stop fw done.\n");
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(ipu6_ipu7_boot_stop_fw, "INTEL_IPU6");
diff --git a/drivers/media/pci/intel/ipu6/ipu7-boot.h b/drivers/media/pci/intel/ipu6/ipu7-boot.h
new file mode 100644
index 00000000000000..f2172b53bed525
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-boot.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2026 Intel Corporation */
+
+#ifndef IPU7_BOOT_H
+#define IPU7_BOOT_H
+
+#include "ipu7-fw-com.h"
+
+#define IPU7_BOOT_MSG_VER_MAX_ENTRIES	3U
+
+struct ipu7_boot_abi_version {
+	u8 patch;
+	u8 subminor;
+	u8 minor;
+	u8 major;
+};
+
+struct ipu7_boot_abi_msg_versions {
+	u8 num_versions;
+	u8 reserved[3];
+	struct ipu7_boot_abi_version versions[IPU7_BOOT_MSG_VER_MAX_ENTRIES];
+};
+
+struct ipu7_boot_abi_cfg {
+	u32 length;
+	struct ipu7_boot_abi_version config_version;
+	struct ipu7_boot_abi_msg_versions client_version_support;
+	u32 pkg_dir;
+	u32 subsys_config;
+	u32 uc_tile_frequency;
+	u16 checksum;
+	u8 uc_tile_frequency_units;
+	u8 padding[1];
+	u32 reserved[58];
+	struct ipu7_fw_com_config fw_com_config;
+} __packed;
+
+int ipu6_ipu7_init_boot_config(struct ipu6_bus_device *adev,
+			       struct ipu7_fw_com_queue_config *qconfigs,
+			       int num_queues, u32 uc_freq,
+			       dma_addr_t subsys_config, u8 major);
+void ipu6_ipu7_release_boot_config(struct ipu6_bus_device *adev);
+int ipu6_ipu7_boot_start_fw(const struct ipu6_bus_device *adev);
+int ipu6_ipu7_boot_stop_fw(const struct ipu6_bus_device *adev);
+
+#endif
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-com.c b/drivers/media/pci/intel/ipu6/ipu7-fw-com.c
new file mode 100644
index 00000000000000..7dd1e683aa9232
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-com.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Intel Corporation
+ */
+
+#include <linux/io.h>
+
+#include "ipu7-fw-com.h"
+
+static void __iomem *ipu7_fw_com_get_indices(struct ipu7_fw_com_context *ctx,
+					     u32 q)
+{
+	return ctx->queue_indices + (q * sizeof(struct ipu7_fw_com_queue_indices));
+}
+
+void *ipu7_fw_com_get_token(struct ipu7_fw_com_context *ctx, int q)
+{
+	struct ipu7_fw_com_queue_config *queue_params = &ctx->queue_configs[q];
+	void __iomem *queue_indices = ipu7_fw_com_get_indices(ctx, q);
+	u32 write_index = readl(queue_indices +
+				offsetof(struct ipu7_fw_com_queue_indices,
+					 write_index));
+	u32 read_index = readl(queue_indices +
+			       offsetof(struct ipu7_fw_com_queue_indices,
+					read_index));
+	void *token = NULL;
+
+	if (q < ctx->num_output_queues) {
+		/* Output queue */
+		bool empty = (write_index == read_index);
+
+		if (!empty)
+			token = queue_params->token_array_mem +
+				read_index *
+				queue_params->token_size_in_bytes;
+	} else {
+		/* Input queue */
+		bool full = (read_index == ((write_index + 1U) %
+					    (u32)queue_params->max_capacity));
+
+		if (!full)
+			token = queue_params->token_array_mem +
+				write_index * queue_params->token_size_in_bytes;
+	}
+	return token;
+}
+EXPORT_SYMBOL_NS_GPL(ipu7_fw_com_get_token, "INTEL_IPU6");
+
+void ipu7_fw_com_put_token(struct ipu7_fw_com_context *ctx, int q)
+{
+	struct ipu7_fw_com_queue_config *queue_params = &ctx->queue_configs[q];
+	void __iomem *queue_indices = ipu7_fw_com_get_indices(ctx, q);
+	u32 offset, index;
+
+	if (q < ctx->num_output_queues)
+		/* Output queue */
+		offset = offsetof(struct ipu7_fw_com_queue_indices, read_index);
+
+	else
+		/* Input queue */
+		offset = offsetof(struct ipu7_fw_com_queue_indices, write_index);
+
+	index = readl(queue_indices + offset);
+	writel((index + 1U) % queue_params->max_capacity,
+	       queue_indices + offset);
+}
+EXPORT_SYMBOL_NS_GPL(ipu7_fw_com_put_token, "INTEL_IPU6");
+
+struct ipu7_fw_com_queue_params_config *
+ipu7_fw_com_get_queue_config(struct ipu7_fw_com_config *config)
+{
+	return (struct ipu7_fw_com_queue_params_config *)(&config[1]);
+}
+EXPORT_SYMBOL_NS_GPL(ipu7_fw_com_get_queue_config, "INTEL_IPU6");
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-com.h b/drivers/media/pci/intel/ipu6/ipu7-fw-com.h
new file mode 100644
index 00000000000000..097eaab99547e4
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-com.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2026 Intel Corporation */
+
+#ifndef IPU7_FW_COM_H
+#define IPU7_FW_COM_H
+
+#include <linux/types.h>
+
+struct ipu7_fw_com_queue_config {
+	void *token_array_mem;
+	u32 queue_size;
+	u16 token_size_in_bytes;
+	u16 max_capacity;
+};
+
+struct ipu7_fw_com_context {
+	u16 num_input_queues;
+	u16 num_output_queues;
+	struct ipu7_fw_com_queue_config *queue_configs;
+	void __iomem *queue_indices;
+	dma_addr_t queue_mem_dma_addr;
+	void *queue_mem;
+	u32 queue_mem_size;
+	struct ipu7_boot_abi_cfg *boot_config;
+	dma_addr_t boot_config_dma_addr;
+	u32 boot_config_size;
+	u32 fw_entry;
+	struct ipu7_insys_config *fw_config;
+	dma_addr_t fw_config_dma_addr;
+};
+
+struct ipu7_fw_com_queue_params_config {
+	u32 token_array_mem;
+	u16 token_size_in_bytes;
+	u16 max_capacity;
+};
+
+struct ipu7_fw_com_config {
+	u16 max_output_queues;
+	u16 max_input_queues;
+};
+
+struct ipu7_fw_com_queue_indices {
+	u32 read_index;
+	u32 write_index;
+};
+
+void ipu7_fw_com_put_token(struct ipu7_fw_com_context *ctx, int q);
+void *ipu7_fw_com_get_token(struct ipu7_fw_com_context *ctx, int q);
+struct ipu7_fw_com_queue_params_config *
+ipu7_fw_com_get_queue_config(struct ipu7_fw_com_config *config);
+
+#endif /* IPU7_FW_COM_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
new file mode 100644
index 00000000000000..0876cc54faa7aa
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.c
@@ -0,0 +1,796 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Intel Corporation
+ */
+
+#include <linux/cleanup.h>
+#include <linux/cacheflush.h>
+#include <linux/pm_runtime.h>
+
+#include "ipu6-bus.h"
+#include "ipu6-dma.h"
+#include "ipu6-isys.h"
+#include "ipu6-platform-regs.h"
+#include "ipu7-boot.h"
+#include "ipu7-fw-com.h"
+#include "ipu7-fw-isys.h"
+#include "ipu7-isys-csi2-regs.h"
+#include "ipu7-platform-regs.h"
+
+static void ipu7_fw_isys_cleanup(struct ipu6_isys *isys)
+{
+	struct ipu6_bus_device *adev = isys->adev;
+	struct ipu7_fw_com_context *fwctx = isys->fwctx;
+
+	if (!fwctx)
+		return;
+
+	ipu6_ipu7_release_boot_config(adev);
+
+	if (fwctx->fw_config) {
+		ipu6_dma_free(adev, sizeof(*fwctx->fw_config), fwctx->fw_config,
+			      fwctx->fw_config_dma_addr, 0);
+		fwctx->fw_config = NULL;
+		fwctx->fw_config_dma_addr = 0;
+	}
+
+	isys->fwctx = NULL;
+}
+
+static int ipu7_fw_isys_open(struct ipu6_isys *isys)
+{
+	return ipu6_ipu7_boot_start_fw(isys->adev);
+}
+
+static int ipu7_fw_isys_close(struct ipu6_isys *isys)
+{
+	int ret;
+
+	ret = ipu6_ipu7_boot_stop_fw(isys->adev);
+	if (ret)
+		return ret;
+
+	ipu7_fw_isys_cleanup(isys);
+
+	return ret;
+}
+
+static int ipu7_fw_isys_init(struct ipu6_isys *isys, unsigned int num_streams)
+{
+	struct ipu7_fw_com_queue_config *queue_configs;
+	struct ipu6_bus_device *adev = isys->adev;
+	struct device *dev = &adev->auxdev.dev;
+	struct ipu7_insys_config *fw_config;
+	struct ipu7_fw_com_context *fwctx;
+	dma_addr_t fw_config_dma_addr;
+	unsigned int num_queues;
+	u32 freq;
+	int ret;
+
+	/* Allocate and init firmware context. */
+	fwctx = devm_kzalloc(dev, sizeof(struct ipu7_fw_com_context),
+			     GFP_KERNEL);
+	if (!fwctx)
+		return -ENOMEM;
+
+	fwctx->num_input_queues = IPU7_INSYS_MAX_INPUT_QUEUES;
+	fwctx->num_output_queues = IPU7_INSYS_MAX_OUTPUT_QUEUES;
+	num_queues = fwctx->num_input_queues + fwctx->num_output_queues;
+
+	queue_configs = devm_kcalloc(dev, num_queues, sizeof(*queue_configs),
+				     GFP_KERNEL);
+	if (!queue_configs) {
+		ipu7_fw_isys_cleanup(isys);
+		return -ENOMEM;
+	}
+	fwctx->fw_entry = adev->fw_entry;
+	fwctx->queue_configs = queue_configs;
+	queue_configs[IPU7_INSYS_OUTPUT_MSG_QUEUE].max_capacity =
+		IPU7_ISYS_SIZE_RECV_QUEUE;
+	queue_configs[IPU7_INSYS_OUTPUT_MSG_QUEUE].token_size_in_bytes =
+		sizeof(struct ipu7_insys_resp);
+	queue_configs[IPU7_INSYS_OUTPUT_LOG_QUEUE].max_capacity =
+		IPU7_ISYS_SIZE_LOG_QUEUE;
+	queue_configs[IPU7_INSYS_OUTPUT_LOG_QUEUE].token_size_in_bytes =
+		sizeof(struct ipu7_insys_resp);
+	queue_configs[IPU7_INSYS_OUTPUT_RESERVED_QUEUE].max_capacity = 0;
+	queue_configs[IPU7_INSYS_OUTPUT_RESERVED_QUEUE].token_size_in_bytes = 0;
+
+	queue_configs[IPU7_INSYS_INPUT_DEV_QUEUE].max_capacity =
+		IPU7_ISYS_MAX_STREAMS;
+	queue_configs[IPU7_INSYS_INPUT_DEV_QUEUE].token_size_in_bytes =
+		sizeof(struct ipu7_insys_send_queue_token);
+
+	for (unsigned int i = IPU7_INSYS_INPUT_MSG_QUEUE; i < num_queues; i++) {
+		queue_configs[i].max_capacity = IPU7_ISYS_SIZE_SEND_QUEUE;
+		queue_configs[i].token_size_in_bytes =
+			sizeof(struct ipu7_insys_send_queue_token);
+	}
+
+	/* Allocate ISYS subsys config. */
+	fw_config = ipu6_dma_alloc(adev, sizeof(*fw_config),
+				   &fw_config_dma_addr, GFP_KERNEL, 0);
+	if (!fw_config) {
+		dev_err(dev, "Failed to allocate isys subsys config.\n");
+		ipu7_fw_isys_cleanup(isys);
+		return -ENOMEM;
+	}
+	fwctx->fw_config = fw_config;
+	fwctx->fw_config_dma_addr = fw_config_dma_addr;
+	memset(fw_config, 0, sizeof(*fw_config));
+	fw_config->logger_config.use_source_severity = 0;
+	fw_config->logger_config.use_channels_enable_bitmask = 1;
+	fw_config->logger_config.channels_enable_bitmask =
+				IPU7_LOGGER_CFG_CHANNEL_ENABLE_SYSCOM;
+	fw_config->logger_config.hw_printf_buffer_base_addr = 0U;
+	fw_config->logger_config.hw_printf_buffer_size_bytes = 0U;
+	fw_config->wdt_config.wdt_timer1_us = 0;
+	fw_config->wdt_config.wdt_timer2_us = 0;
+	freq = ipu7_buttress_get_isys_freq(adev->isp);
+
+	ipu6_dma_sync_single(adev, fw_config_dma_addr,
+			     sizeof(struct ipu7_insys_config));
+
+	isys->fwctx = fwctx;
+
+	ret = ipu6_ipu7_init_boot_config(adev, queue_configs, num_queues,
+					 freq, fw_config_dma_addr, 1U);
+	if (ret) {
+		ipu7_fw_isys_cleanup(isys);
+		return ret;
+	}
+
+	ret = ipu7_fw_isys_open(isys);
+	if (ret)
+		ipu7_fw_isys_cleanup(isys);
+
+	return ret;
+}
+
+static struct ipu7_insys_resp *ipu7_fw_isys_get_resp(struct ipu6_isys *isys)
+{
+	return ipu7_fw_com_get_token(isys->fwctx, IPU7_INSYS_OUTPUT_MSG_QUEUE);
+}
+
+static void ipu7_fw_isys_put_resp(struct ipu6_isys *isys)
+{
+	ipu7_fw_com_put_token(isys->fwctx, IPU7_INSYS_OUTPUT_MSG_QUEUE);
+}
+
+static int ipu7_isys_fw_pin_cfg(struct ipu6_isys_video *av,
+				struct ipu6_isys_stream *stream,
+				struct media_pad *src_pad,
+				struct v4l2_mbus_frame_desc_entry *entry,
+				void *__cfg)
+{
+	struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(src_pad->entity);
+	struct v4l2_subdev_state *state = v4l2_subdev_get_locked_active_state(sd);
+	struct ipu7_fw_isys_stream_cfg *cfg = __cfg;
+	struct ipu7_fw_isys_input_pin *input_pin;
+	struct ipu7_fw_isys_output_pin *output_pin;
+	struct ipu6_isys_queue *aq = &av->aq;
+	struct v4l2_mbus_framefmt *fmt;
+	const struct ipu6_isys_pixelformat *pfmt =
+		ipu6_isys_get_isys_format(ipu6_isys_get_format(av), 0);
+	int input_pins = cfg->nof_input_pins++;
+	int output_pins;
+	u32 src_stream;
+
+	src_stream = __ipu6_isys_get_src_stream_by_src_pad(state, src_pad->index);
+	fmt = v4l2_subdev_state_get_format(state, src_pad->index, src_stream);
+
+	input_pin = &cfg->input_pins[input_pins];
+	input_pin->input_res.width = fmt->width;
+	input_pin->input_res.height = fmt->height;
+	input_pin->dt = entry->bus.csi2.dt;
+	input_pin->disable_mipi_unpacking = 0;
+	if (pfmt->bpp == pfmt->bpp_packed && pfmt->bpp % BITS_PER_BYTE)
+		input_pin->disable_mipi_unpacking = 1;
+	input_pin->mapped_dt = IPU7_N_INSYS_MIPI_DATA_TYPE;
+	input_pin->dt_rename_mode = IPU7_INSYS_MIPI_DT_NO_RENAME;
+	input_pin->sync_msg_map =
+		IPU7_INSYS_STREAM_SYNC_MSG_SEND_RESP_SOF |
+		IPU7_INSYS_STREAM_SYNC_MSG_SEND_RESP_SOF_DISCARDED |
+		IPU7_INSYS_STREAM_SYNC_MSG_SEND_IRQ_SOF |
+		IPU7_INSYS_STREAM_SYNC_MSG_SEND_IRQ_SOF_DISCARDED;
+
+	output_pins = cfg->nof_output_pins++;
+	aq->fw_output = output_pins;
+	av->stream->output_pins_queue[output_pins] = aq;
+
+	output_pin = &cfg->output_pins[output_pins];
+	memset(output_pin, 0, sizeof(*output_pin));
+	output_pin->link.buffer_lines = 0;
+	output_pin->link.foreign_key = IPU7_MSG_LINK_FOREIGN_KEY_NONE;
+	output_pin->link.pbk_id = IPU7_MSG_LINK_PBK_ID_DONT_CARE;
+	output_pin->link.pbk_slot_id = IPU7_MSG_LINK_PBK_SLOT_ID_DONT_CARE;
+	output_pin->link.dest = 0; /* IPU_INSYS_OUTPUT_LINK_DEST_MEM */
+	output_pin->link.use_sw_managed = 1;
+	output_pin->crop.line_top = 0;
+	output_pin->crop.line_bottom = 0;
+	output_pin->dpcm.enable = 0;
+	output_pin->ft = pfmt->css_pixelformat;
+	output_pin->stride = ipu6_isys_get_bytes_per_line(av);
+	output_pin->send_irq = 1;
+	output_pin->input_pin_id = input_pins;
+
+	return 0;
+}
+
+static int
+ipu7_fw_isys_send_cmd(struct ipu6_isys *isys, const unsigned int stream_handle,
+		      void *cpu_mapped_buf, dma_addr_t dma_mapped_buf,
+		      size_t size, u16 send_type)
+{
+	struct ipu7_fw_com_context *ctx = isys->fwctx;
+	/*struct device *dev = &isys->adev->auxdev.dev;*/
+	struct ipu7_insys_send_queue_token *token;
+
+	if (send_type >= N_IPU7_INSYS_SEND_TYPE)
+		return -EINVAL;
+
+	if (cpu_mapped_buf)
+		clflush_cache_range(cpu_mapped_buf, size);
+
+	token = ipu7_fw_com_get_token(ctx, stream_handle +
+				      IPU7_INSYS_INPUT_MSG_QUEUE);
+	if (!token)
+		return -EBUSY;
+
+	token->addr = dma_mapped_buf;
+	token->buf_handle = (unsigned long)cpu_mapped_buf;
+	token->send_type = send_type;
+	token->stream_id = stream_handle;
+	token->flag = IPU7_INSYS_SEND_QUEUE_TOKEN_FLAG_NONE;
+
+	ipu7_fw_com_put_token(ctx, stream_handle + IPU7_INSYS_INPUT_MSG_QUEUE);
+	/* now wakeup FW */
+	ipu7_buttress_wakeup_isys(isys->adev->isp);
+
+	return 0;
+}
+
+static void ipu7_fw_isys_dump_stream_cfg(struct device *dev,
+					 struct isys_fw_msgs *msg)
+{
+	struct ipu7_fw_isys_stream_cfg *cfg = &msg->ipu7.stream;
+	unsigned int i;
+
+	dev_dbg(dev, "---------------------------\n");
+	dev_dbg(dev, "IPU_FW_ISYS_STREAM_CFG_DATA\n");
+
+	dev_dbg(dev, ".port id %d\n", cfg->port_id);
+	dev_dbg(dev, ".vc %d\n", cfg->vc);
+	dev_dbg(dev, ".nof_input_pins = %d\n", cfg->nof_input_pins);
+	dev_dbg(dev, ".nof_output_pins = %d\n", cfg->nof_output_pins);
+	dev_dbg(dev, ".stream_msg_map = 0x%x\n", cfg->stream_msg_map);
+
+	for (i = 0; i < cfg->nof_input_pins; i++) {
+		dev_dbg(dev, ".input_pin[%d]:\n", i);
+		dev_dbg(dev, "\t.dt = 0x%0x\n",
+			cfg->input_pins[i].dt);
+		dev_dbg(dev, "\t.disable_mipi_unpacking = %d\n",
+			cfg->input_pins[i].disable_mipi_unpacking);
+		dev_dbg(dev, "\t.dt_rename_mode = %d\n",
+			cfg->input_pins[i].dt_rename_mode);
+		dev_dbg(dev, "\t.mapped_dt = 0x%0x\n",
+			cfg->input_pins[i].mapped_dt);
+		dev_dbg(dev, "\t.input_res = %d x %d\n",
+			cfg->input_pins[i].input_res.width,
+			cfg->input_pins[i].input_res.height);
+		dev_dbg(dev, "\t.sync_msg_map = 0x%x\n",
+			cfg->input_pins[i].sync_msg_map);
+	}
+
+	for (i = 0; i < cfg->nof_output_pins; i++) {
+		dev_dbg(dev, ".output_pin[%d]:\n", i);
+		dev_dbg(dev, "\t.input_pin_id = %d\n",
+			cfg->output_pins[i].input_pin_id);
+		dev_dbg(dev, "\t.stride = %d\n", cfg->output_pins[i].stride);
+		dev_dbg(dev, "\t.send_irq = %d\n",
+			cfg->output_pins[i].send_irq);
+		dev_dbg(dev, "\t.ft = %d\n", cfg->output_pins[i].ft);
+
+		dev_dbg(dev, "\t.link.buffer_lines = %d\n",
+			cfg->output_pins[i].link.buffer_lines);
+		dev_dbg(dev, "\t.link.foreign_key = %d\n",
+			cfg->output_pins[i].link.foreign_key);
+		dev_dbg(dev, "\t.link.granularity_pointer_update = %d\n",
+			cfg->output_pins[i].link.granularity_pointer_update);
+		dev_dbg(dev, "\t.link.msg_link_streaming_mode = %d\n",
+			cfg->output_pins[i].link.msg_link_streaming_mode);
+		dev_dbg(dev, "\t.link.pbk_id = %d\n",
+			cfg->output_pins[i].link.pbk_id);
+		dev_dbg(dev, "\t.link.pbk_slot_id = %d\n",
+			cfg->output_pins[i].link.pbk_slot_id);
+		dev_dbg(dev, "\t.link.dest = %d\n",
+			cfg->output_pins[i].link.dest);
+		dev_dbg(dev, "\t.link.use_sw_managed = %d\n",
+			cfg->output_pins[i].link.use_sw_managed);
+		dev_dbg(dev, "\t.link.is_snoop = %d\n",
+			cfg->output_pins[i].link.is_snoop);
+
+		dev_dbg(dev, "\t.crop.line_top = %d\n",
+			cfg->output_pins[i].crop.line_top);
+		dev_dbg(dev, "\t.crop.line_bottom = %d\n",
+			cfg->output_pins[i].crop.line_bottom);
+
+		dev_dbg(dev, "\t.dpcm_enable = %d\n",
+			cfg->output_pins[i].dpcm.enable);
+		dev_dbg(dev, "\t.dpcm.type = %d\n",
+			cfg->output_pins[i].dpcm.type);
+		dev_dbg(dev, "\t.dpcm.predictor = %d\n",
+			cfg->output_pins[i].dpcm.predictor);
+	}
+	dev_dbg(dev, "---------------------------\n");
+}
+
+static void ipu7_fw_isys_dump_frame_buf_set(struct device *dev,
+					    struct isys_fw_msgs *msg,
+					    unsigned int outputs)
+{
+	struct ipu7_fw_isys_frame_buff_set *buf = &msg->ipu7.frame;
+
+	dev_dbg(dev, "--------------------------\n");
+	dev_dbg(dev, "IPU_ISYS_BUFF_SET\n");
+	dev_dbg(dev, ".capture_msg_map = %d\n", buf->capture_msg_map);
+	dev_dbg(dev, ".frame_id = %d\n", buf->frame_id);
+	dev_dbg(dev, ".skip_frame = %d\n", buf->skip_frame);
+
+	for (unsigned int i = 0; i < outputs; i++) {
+		dev_dbg(dev, ".output_pin[%d]:\n", i);
+		dev_dbg(dev, "\t.user_token = %llx\n",
+			buf->output_pins[i].user_token);
+		dev_dbg(dev, "\t.addr = 0x%x\n", buf->output_pins[i].addr);
+	}
+	dev_dbg(dev, "---------------------------\n");
+}
+
+static int ipu7_fw_isys_prepare_stream_cfg(struct ipu6_isys_stream *stream,
+					   struct v4l2_mbus_frame_desc *desc,
+					   struct isys_fw_msgs *msg)
+{
+	struct ipu7_fw_isys_stream_cfg *cfg = &msg->ipu7.stream;
+	struct device *dev = &stream->isys->adev->auxdev.dev;
+	int ret;
+
+	memset(cfg, 0, sizeof(*cfg));
+	cfg->port_id = stream->asd->source;
+	cfg->vc = stream->vc;
+	cfg->stream_msg_map = IPU7_INSYS_STREAM_ENABLE_MSG_SEND_RESP |
+			      IPU7_INSYS_STREAM_ENABLE_MSG_SEND_IRQ;
+
+	ret = ipu6_isys_fw_pins_prepare(stream, desc, ipu7_isys_fw_pin_cfg,
+					cfg);
+	if (ret)
+		return ret;
+
+	stream->nr_output_pins = cfg->nof_output_pins;
+
+	ipu7_fw_isys_dump_stream_cfg(dev, msg);
+
+	return 0;
+}
+
+static void
+ipu7_isys_buf_to_fw_frame_buf_pin(struct vb2_buffer *vb,
+				  struct ipu7_fw_isys_frame_buff_set *set)
+{
+	struct ipu6_isys_queue *aq = vb2_queue_to_isys_queue(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vvb = to_vb2_v4l2_buffer(vb);
+	struct ipu6_isys_video_buffer *ivb =
+		vb2_buffer_to_ipu6_isys_video_buffer(vvb);
+
+	set->output_pins[aq->fw_output].addr = ivb->dma_addr;
+	set->output_pins[aq->fw_output].user_token = (u64)(uintptr_t)set;
+}
+
+static void
+ipu7_fw_isys_prepare_buf_set(struct isys_fw_msgs *msg,
+			     struct ipu6_isys_stream *stream,
+			     struct ipu6_isys_buffer_list *bl)
+{
+	struct ipu7_fw_isys_frame_buff_set *set = &msg->ipu7.frame;
+	struct ipu6_isys_buffer *ib;
+
+	WARN_ON(!bl->nbufs);
+
+	memset(set, 0, sizeof(*set));
+	set->capture_msg_map = IPU7_INSYS_FRAME_ENABLE_MSG_SEND_RESP |
+			       IPU7_INSYS_FRAME_ENABLE_MSG_SEND_IRQ;
+	set->frame_id = atomic_fetch_inc(&stream->buf_id) % 256;
+
+	list_for_each_entry(ib, &bl->head, head) {
+		struct vb2_buffer *vb = ipu6_isys_buffer_to_vb2_buffer(ib);
+
+		ipu7_isys_buf_to_fw_frame_buf_pin(vb, set);
+	}
+
+	dev_dbg(&stream->isys->adev->auxdev.dev,
+		"ipu7 frame_buf: cap_map=0x%x fid=%u opin[0].addr=0x%x token=0x%llx\n",
+		set->capture_msg_map, set->frame_id,
+		set->output_pins[0].addr, set->output_pins[0].user_token);
+}
+
+static int ipu7_fw_isys_stream_open(struct ipu6_isys *isys,
+				    const unsigned int stream_handle,
+				    struct isys_fw_msgs *msg)
+{
+	return ipu7_fw_isys_send_cmd(isys, stream_handle, &msg->ipu7.stream,
+				     msg->dma_addr, sizeof(msg->ipu7.stream),
+				     IPU7_INSYS_SEND_TYPE_STREAM_OPEN);
+}
+
+static int ipu7_fw_isys_stream_close(struct ipu6_isys *isys,
+				     const unsigned int stream_handle)
+{
+	return ipu7_fw_isys_send_cmd(isys, stream_handle, NULL, 0, 0,
+				     IPU7_INSYS_SEND_TYPE_STREAM_CLOSE);
+}
+
+static int ipu7_fw_isys_stream_flush(struct ipu6_isys *isys,
+				     const unsigned int stream_handle)
+{
+	return ipu7_fw_isys_send_cmd(isys, stream_handle, NULL, 0, 0,
+				     IPU7_INSYS_SEND_TYPE_STREAM_FLUSH);
+}
+
+static int ipu7_fw_isys_stream_start(struct ipu6_isys *isys,
+				     const unsigned int stream_handle,
+				     struct isys_fw_msgs *msg, bool capture)
+{
+	return ipu7_fw_isys_send_cmd(isys, stream_handle, &msg->ipu7.frame,
+				     msg->dma_addr, sizeof(msg->ipu7.frame),
+				     IPU7_INSYS_SEND_TYPE_STREAM_START_AND_CAPTURE);
+}
+
+static int ipu7_fw_isys_stream_capture(struct ipu6_isys *isys,
+				       const unsigned int stream_handle,
+				       struct isys_fw_msgs *msg)
+{
+	return ipu7_fw_isys_send_cmd(isys, stream_handle, &msg->ipu7.frame,
+				     msg->dma_addr, sizeof(msg->ipu7.frame),
+				     IPU7_INSYS_SEND_TYPE_STREAM_CAPTURE);
+}
+
+const struct ipu6_fw_isys_ops ipu7_fw_isys_ops = {
+	.init = ipu7_fw_isys_init,
+	.close = ipu7_fw_isys_close,
+	.send_cmd = ipu7_fw_isys_send_cmd,
+	.cleanup = ipu7_fw_isys_cleanup,
+	.prepare_stream_cfg = ipu7_fw_isys_prepare_stream_cfg,
+	.prepare_buf_set = ipu7_fw_isys_prepare_buf_set,
+	.stream_open = ipu7_fw_isys_stream_open,
+	.stream_start = ipu7_fw_isys_stream_start,
+	.stream_capture = ipu7_fw_isys_stream_capture,
+	.stream_flush = ipu7_fw_isys_stream_flush,
+	.stream_close = ipu7_fw_isys_stream_close,
+	.dump_stream_cfg = ipu7_fw_isys_dump_stream_cfg,
+	.dump_frame_buf_set = ipu7_fw_isys_dump_frame_buf_set,
+};
+
+static const struct ipu7_csi2_error {
+	const char *error_string;
+	bool is_info_only;
+} dphy_rx_errors[] = {
+	{ "Error handler FIFO full", false },
+	{ "Reserved Short Packet encoding detected", true },
+	{ "Reserved Long Packet encoding detected", true },
+	{ "Received packet is too short", false},
+	{ "Received packet is too long", false},
+	{ "Short packet discarded due to errors", false },
+	{ "Long packet discarded due to errors", false },
+	{ "CSI Combo Rx interrupt", false },
+	{ "IDI CDC FIFO overflow(remaining bits are reserved as 0)", false },
+	{ "Received NULL packet", true },
+	{ "Received blanking packet", true },
+	{ "Tie to 0", true },
+};
+
+static void ipu7_isys_register_errors(struct ipu6_isys_csi2 *csi2)
+{
+	u32 offset = IPU7_IS_IO_CSI2_ERR_LEGACY_IRQ_CTL_BASE(csi2->port);
+	u32 status = readl(csi2->base + offset + IPU7_IRQ_CTL_STATUS);
+	u32 mask = IPU7_CSI_RX_ERROR_IRQ_MASK;
+
+	if (!status)
+		return;
+
+	dev_dbg(&csi2->isys->adev->auxdev.dev, "csi2-%u error status 0x%08x\n",
+		csi2->port, status);
+
+	writel(status & mask, csi2->base + offset + IPU7_IRQ_CTL_CLEAR);
+	csi2->receiver_errors |= status & mask;
+}
+
+static void ipu7_isys_csi2_error(struct ipu6_isys_csi2 *csi2)
+{
+	u32 status;
+
+	/* Register errors once more in case of error interrupts are disabled */
+	ipu7_isys_register_errors(csi2);
+	status = csi2->receiver_errors;
+	csi2->receiver_errors = 0;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(dphy_rx_errors); i++) {
+		if (status & BIT(i))
+			dev_err_ratelimited(&csi2->isys->adev->auxdev.dev,
+					    "csi2-%i error: %s\n",
+					    csi2->port,
+					    dphy_rx_errors[i].error_string);
+	}
+}
+
+static const struct resp_to_msg {
+	enum ipu7_insys_resp_type type;
+	const char *msg;
+} is_fw_msg[] = {
+	{ IPU7_INSYS_RESP_TYPE_STREAM_OPEN_DONE, "STREAM_OPEN_DONE" },
+	{ IPU7_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK,
+	 "STREAM_START_AND_CAPTURE_ACK" },
+	{ IPU7_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK, "STREAM_CAPTURE_ACK" },
+	{ IPU7_INSYS_RESP_TYPE_STREAM_ABORT_ACK, "STREAM_ABORT_ACK" },
+	{ IPU7_INSYS_RESP_TYPE_STREAM_FLUSH_ACK, "STREAM_FLUSH_ACK" },
+	{ IPU7_INSYS_RESP_TYPE_STREAM_CLOSE_ACK, "STREAM_CLOSE_ACK" },
+	{ IPU7_INSYS_RESP_TYPE_PIN_DATA_READY, "PIN_DATA_READY" },
+	{ IPU7_INSYS_RESP_TYPE_FRAME_SOF, "FRAME_SOF" },
+	{ IPU7_INSYS_RESP_TYPE_FRAME_EOF, "FRAME_EOF" },
+	{ IPU7_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE,
+	 "STREAM_START_AND_CAPTURE_DONE" },
+	{ IPU7_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE, "STREAM_CAPTURE_DONE" },
+	{ N_IPU7_INSYS_RESP_TYPE, "N_IPU7_INSYS_RESP_TYPE" },
+};
+
+static int ipu7_isys_isr_one(struct ipu6_bus_device *adev)
+{
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	struct ipu6_isys_stream *stream = NULL;
+	struct device *dev = &adev->auxdev.dev;
+	struct ipu6_isys_csi2 *csi2 = NULL;
+	struct ipu7_fw_isys_msg_err err_info;
+	struct isys_fw_msgs *isys_fw_msg;
+	struct ipu7_insys_resp *resp;
+	unsigned long flags;
+	u64 ts;
+
+	resp = ipu7_fw_isys_get_resp(isys);
+	if (!resp)
+		return 1;
+
+	if (resp->type >= N_IPU7_INSYS_RESP_TYPE) {
+		dev_err(dev, "Unknown response type %u stream %u\n",
+			resp->type, resp->stream_id);
+		ipu7_fw_isys_put_resp(isys);
+		return 1;
+	}
+
+	err_info = resp->error_info;
+	ts = ((u64)resp->timestamp[1] << 32) | resp->timestamp[0];
+
+	if (err_info.err_group == INSYS_MSG_ERR_GROUP_CAPTURE &&
+	    err_info.err_code == INSYS_MSG_ERR_CAPTURE_SYNC_FRAME_DROP) {
+		/* receive a sp w/o command, firmware drop it */
+		dev_dbg(dev, "FRAME DROP: %02u %s stream %u\n",
+			resp->type, is_fw_msg[resp->type].msg,
+			resp->stream_id);
+		dev_dbg(dev, "\tpin %u buf_id %llx frame %u\n",
+			resp->pin_id, resp->buf_id, resp->frame_id);
+		dev_dbg(dev, "\terror group %u code %u details [%u %u]\n",
+			err_info.err_group, err_info.err_code,
+			err_info.err_detail[0], err_info.err_detail[1]);
+	} else if (err_info.err_code) {
+		dev_err(dev, "%02u %s stream %u pin %u buf_id %llx frame %u\n",
+			resp->type, is_fw_msg[resp->type].msg, resp->stream_id,
+			resp->pin_id, resp->buf_id, resp->frame_id);
+		dev_err(dev, "\terror group %u code %u details [%u %u]\n",
+			err_info.err_group, err_info.err_code,
+			err_info.err_detail[0], err_info.err_detail[1]);
+	} else {
+		dev_dbg(dev, "%02u %s stream %u pin %u buf_id %llx frame %u\n",
+			resp->type, is_fw_msg[resp->type].msg, resp->stream_id,
+			resp->pin_id, resp->buf_id, resp->frame_id);
+		dev_dbg(dev, "\tts %llu\n", ts);
+	}
+
+	if (resp->stream_id >= IPU7_ISYS_MAX_STREAMS) {
+		dev_err(dev, "bad stream handle %u\n",
+			resp->stream_id);
+		goto leave_nounlock;
+	}
+
+	spin_lock_irqsave(&isys->streams_lock, flags);
+
+	stream = resp->stream_id < IPU6_ISYS_MAX_STREAMS ?
+		isys->streams_by_handle[resp->stream_id] : NULL;
+	if (!stream) {
+		dev_err(dev, "stream of stream_handle %u is unused\n",
+			resp->stream_id);
+		goto leave;
+	}
+
+	stream->error = err_info.err_code;
+
+	if (stream->asd)
+		csi2 = ipu6_isys_subdev_to_csi2(stream->asd);
+
+	switch (resp->type) {
+	case IPU7_INSYS_RESP_TYPE_STREAM_OPEN_DONE:
+		complete(&stream->stream_open_completion);
+		break;
+	case IPU7_INSYS_RESP_TYPE_STREAM_CLOSE_ACK:
+		complete(&stream->stream_close_completion);
+		break;
+	case IPU7_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK:
+		complete(&stream->stream_start_completion);
+		break;
+	case IPU7_INSYS_RESP_TYPE_STREAM_ABORT_ACK:
+		complete(&stream->stream_stop_completion);
+		break;
+	case IPU7_INSYS_RESP_TYPE_STREAM_FLUSH_ACK:
+		complete(&stream->stream_stop_completion);
+		break;
+	case IPU7_INSYS_RESP_TYPE_PIN_DATA_READY:
+		/*
+		 * firmware only release the capture msg until software
+		 * get pin_data_ready event
+		 */
+		isys_fw_msg = container_of((void *)(uintptr_t)resp->buf_id,
+					   struct isys_fw_msgs, dummy);
+
+		ipu6_put_fw_msg_buf(ipu6_bus_get_drvdata(adev), isys_fw_msg);
+		if (resp->pin_id < IPU6_ISYS_OUTPUT_PINS)
+			ipu6_stream_buf_ready(stream, resp->pin_id,
+					      resp->pin.addr, ts, 0);
+		else
+			dev_err(dev, "No handler for pin %u ready\n",
+				resp->pin_id);
+		if (csi2)
+			ipu7_isys_csi2_error(csi2);
+
+		break;
+	case IPU7_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK:
+		break;
+	case IPU7_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE:
+	case IPU7_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE:
+		break;
+	case IPU7_INSYS_RESP_TYPE_FRAME_SOF:
+		if (csi2)
+			ipu6_isys_csi2_sof_event_by_stream(stream);
+
+		stream->seq[stream->seq_index].sequence =
+			atomic_read(&stream->sequence) - 1U;
+		stream->seq[stream->seq_index].timestamp = ts;
+		dev_dbg(dev,
+			"SOF: stream %u frame %u (index %u), ts 0x%16.16llx\n",
+			resp->stream_id, resp->frame_id,
+			stream->seq[stream->seq_index].sequence, ts);
+		stream->seq_index = (stream->seq_index + 1U)
+			% IPU6_ISYS_MAX_PARALLEL_SOF;
+		break;
+	case IPU7_INSYS_RESP_TYPE_FRAME_EOF:
+		if (csi2)
+			ipu6_isys_csi2_eof_event_by_stream(stream);
+
+		dev_dbg(dev, "eof: stream %d(index %u) ts 0x%16.16llx\n",
+			resp->stream_id,
+			stream->seq[stream->seq_index].sequence, ts);
+		break;
+	default:
+		dev_err(dev, "Unknown response type %u stream %u\n",
+			resp->type, resp->stream_id);
+		break;
+	}
+
+leave:
+	spin_unlock_irqrestore(&isys->streams_lock, flags);
+
+leave_nounlock:
+	ipu7_fw_isys_put_resp(isys);
+
+	return 0;
+}
+
+#define IPU7_NR_OF_CSI2_VC		16U
+static void ipu7_isys_csi2_isr(struct ipu6_isys_csi2 *csi2)
+{
+	struct device *dev = &csi2->isys->adev->auxdev.dev;
+	struct ipu6_device *isp = csi2->isys->adev->isp;
+	struct ipu6_isys_stream *s;
+	u32 sync, offset;
+	u32 fe = 0;
+	u8 vc;
+
+	ipu7_isys_register_errors(csi2);
+
+	offset = IPU7_IS_IO_CSI2_SYNC_LEGACY_IRQ_CTL_BASE(csi2->port);
+	sync = readl(csi2->base + offset + IPU7_IRQ_CTL_STATUS);
+	writel(sync, csi2->base + offset + IPU7_IRQ_CTL_CLEAR);
+	dev_dbg(dev, "csi2-%u sync status 0x%08x\n", csi2->port, sync);
+
+	if (!IS_IPU7_MTL(isp)) {
+		fe = readl(csi2->base + offset + IPU7_IRQ1_CTL_STATUS);
+		writel(fe, csi2->base + offset + IPU7_IRQ1_CTL_CLEAR);
+		dev_dbg(dev, "csi2-%u FE status 0x%08x\n", csi2->port, fe);
+	}
+
+	for (vc = 0; vc < IPU7_NR_OF_CSI2_VC && (sync || fe); vc++) {
+		s = csi2->streams_by_vc[vc];
+		if (!s)
+			continue;
+
+		if (!IS_IPU7_MTL(isp)) {
+			if (sync & IPU7P5_CSI_RX_SYNC_FS_VC & (1U << vc))
+				ipu6_isys_csi2_sof_event_by_stream(s);
+
+			if (fe & IPU7P5_CSI_RX_SYNC_FE_VC & (1U << vc))
+				ipu6_isys_csi2_eof_event_by_stream(s);
+		} else {
+			if (sync & IPU7_CSI_RX_SYNC_FS_VC & (1U << (vc * 2)))
+				ipu6_isys_csi2_sof_event_by_stream(s);
+
+			if (sync & IPU7_CSI_RX_SYNC_FE_VC & (2U << (vc * 2)))
+				ipu6_isys_csi2_eof_event_by_stream(s);
+		}
+	}
+}
+
+static void ipu7_dispatch_csi2_isr(struct ipu6_isys *isys, u32 status)
+{
+	for (unsigned int i = 0; i < isys->pdata->ipdata->csi2.nports; i++) {
+		if (!isys->csi2[i].base)
+			continue;
+		if (status & isys->csi2[i].legacy_irq_mask)
+			ipu7_isys_csi2_isr(&isys->csi2[i]);
+	}
+}
+
+irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev)
+{
+	struct ipu6_isys *isys = ipu6_bus_get_drvdata(adev);
+	void __iomem *base = isys->pdata->base;
+	u32 status_sw, status_csi;
+	u32 csi_offset, sw_offset;
+	int pm_status;
+
+	pm_status = pm_runtime_get_if_active(&adev->auxdev.dev);
+	if (!pm_status)
+		return 0;
+
+	csi_offset = IPU7_IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE;
+	sw_offset = IPU7_IS_UC_CTRL_BASE;
+
+	status_csi = readl(base + csi_offset + IPU7_IRQ_CTL_STATUS);
+	status_sw = readl(base + sw_offset + IPU7_TO_SW_IRQ_CNTL_STATUS);
+
+	if (!status_csi && !status_sw) {
+		if (pm_status > 0)
+			pm_runtime_put(&adev->auxdev.dev);
+		return IRQ_NONE;
+	}
+
+	do {
+		writel(status_sw, base + sw_offset + IPU7_TO_SW_IRQ_CNTL_CLEAR);
+		writel(status_csi, base + csi_offset + IPU7_IRQ_CTL_CLEAR);
+
+		if (isys->isr_csi2_bits & status_csi)
+			ipu7_dispatch_csi2_isr(isys, status_csi);
+
+		if (!ipu7_isys_isr_one(adev))
+			status_sw = IPU7_TO_SW_IRQ_FW;
+		else
+			status_sw = 0;
+
+		status_csi = readl(base + csi_offset + IPU7_IRQ_CTL_STATUS);
+		status_sw |= readl(base + sw_offset +
+				   IPU7_TO_SW_IRQ_CNTL_STATUS);
+	} while ((status_csi & isys->isr_csi2_bits) ||
+		 (status_sw & IPU7_TO_SW_IRQ_FW));
+
+	writel(IPU7_IS_UC_TO_SW_IRQ_MASK,
+	       base + sw_offset + IPU7_TO_SW_IRQ_CNTL_MASK_N);
+
+	if (pm_status > 0)
+		pm_runtime_put(&adev->auxdev.dev);
+
+	return IRQ_HANDLED;
+}
diff --git a/drivers/media/pci/intel/ipu6/ipu7-fw-isys.h b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.h
new file mode 100644
index 00000000000000..d5289f6add8cd1
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-fw-isys.h
@@ -0,0 +1,296 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2026 Intel Corporation */
+
+#ifndef IPU7_FW_ISYS_H
+#define IPU7_FW_ISYS_H
+
+#define IPU7_FWLOG_MAX_LOGGER_SOURCES		(64U)
+#define IPU7_INSYS_MAX_OUTPUT_QUEUES		3U
+#define IPU7_INSYS_STREAM_ID_MAX		16U
+#define IPU7_INSYS_MAX_INPUT_QUEUES		(IPU7_INSYS_STREAM_ID_MAX + 1U)
+#define IPU7_INSYS_OUTPUT_MSG_QUEUE		0U
+#define IPU7_INSYS_OUTPUT_LOG_QUEUE		1U
+#define IPU7_INSYS_OUTPUT_RESERVED_QUEUE	2U
+#define IPU7_INSYS_INPUT_DEV_QUEUE		3U
+
+#define IPU7_INSYS_INPUT_FIRST_QUEUE		3U
+#define IPU7_INSYS_INPUT_MSG_QUEUE		4U
+#define IPU7_INSYS_INPUT_MSG_MAX_QUEUE		16U
+
+#define IPU7_MSG_ERR_MAX_DETAILS		4U
+#define IPU7_ISYS_SIZE_RECV_QUEUE		40U
+#define IPU7_ISYS_SIZE_LOG_QUEUE		256U
+#define IPU7_ISYS_SIZE_SEND_QUEUE		40U
+#define IPU7_ISYS_NUM_RECV_QUEUE		1U
+#define IPU7_INSYS_SEND_QUEUE_TOKEN_FLAG_NONE	0U
+
+#define IPU7_LOGGER_CFG_CHANNEL_ENABLE_SYSCOM	BIT(1)
+
+#define IPU7_ISYS_MAX_STREAMS	16U
+#define IPU7_MAX_OPINS		4
+#define IPU7_MAX_IPINS		4
+
+#define IPU7_N_INSYS_MIPI_DATA_TYPE	0x40
+
+#define IPU7_MSG_LINK_FOREIGN_KEY_NONE		(65535U)
+#define IPU7_MSG_LINK_PBK_ID_DONT_CARE		(255U)
+#define IPU7_MSG_LINK_PBK_SLOT_ID_DONT_CARE	(255U)
+
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_RESP_SOF		BIT(0)
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_RESP_EOF		BIT(1)
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_IRQ_SOF			BIT(2)
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_IRQ_EOF			BIT(3)
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_RESP_SOF_DISCARDED	BIT(4)
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_RESP_EOF_DISCARDED	BIT(5)
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_IRQ_SOF_DISCARDED	BIT(6)
+#define IPU7_INSYS_STREAM_SYNC_MSG_SEND_IRQ_EOF_DISCARDED	BIT(7)
+
+#define IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_OPEN_DONE	BIT(0)
+#define IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_OPEN_DONE		BIT(1)
+#define IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_START_ACK	BIT(2)
+#define IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_START_ACK		BIT(3)
+#define IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_CLOSE_ACK	BIT(4)
+#define IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_CLOSE_ACK		BIT(5)
+#define IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_FLUSH_ACK	BIT(6)
+#define IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_FLUSH_ACK		BIT(7)
+#define IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_ABORT_ACK	BIT(8)
+#define IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_ABORT_ACK		BIT(9)
+
+#define IPU7_INSYS_STREAM_ENABLE_MSG_SEND_RESP ( \
+	IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_OPEN_DONE | \
+	IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_START_ACK | \
+	IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_CLOSE_ACK | \
+	IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_FLUSH_ACK | \
+	IPU7_INSYS_STREAM_MSG_SEND_RESP_STREAM_ABORT_ACK)
+#define IPU7_INSYS_STREAM_ENABLE_MSG_SEND_IRQ ( \
+	IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_OPEN_DONE | \
+	IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_START_ACK | \
+	IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_CLOSE_ACK | \
+	IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_FLUSH_ACK | \
+	IPU7_INSYS_STREAM_MSG_SEND_IRQ_STREAM_ABORT_ACK)
+
+#define IPU7_INSYS_FRAME_MSG_SEND_RESP_CAPTURE_ACK		BIT(0)
+#define IPU7_INSYS_FRAME_MSG_SEND_IRQ_CAPTURE_ACK		BIT(1)
+#define IPU7_INSYS_FRAME_MSG_SEND_RESP_CAPTURE_DONE		BIT(2)
+#define IPU7_INSYS_FRAME_MSG_SEND_IRQ_CAPTURE_DONE		BIT(3)
+#define IPU7_INSYS_FRAME_MSG_SEND_RESP_PIN_DATA_READY		BIT(4)
+#define IPU7_INSYS_FRAME_MSG_SEND_IRQ_PIN_DATA_READY		BIT(5)
+
+#define IPU7_INSYS_FRAME_ENABLE_MSG_SEND_RESP ( \
+	IPU7_INSYS_FRAME_MSG_SEND_RESP_CAPTURE_ACK | \
+	IPU7_INSYS_FRAME_MSG_SEND_RESP_CAPTURE_DONE | \
+	IPU7_INSYS_FRAME_MSG_SEND_RESP_PIN_DATA_READY)
+#define IPU7_INSYS_FRAME_ENABLE_MSG_SEND_IRQ ( \
+	IPU7_INSYS_FRAME_MSG_SEND_IRQ_CAPTURE_ACK | \
+	IPU7_INSYS_FRAME_MSG_SEND_IRQ_CAPTURE_DONE | \
+	IPU7_INSYS_FRAME_MSG_SEND_IRQ_PIN_DATA_READY)
+
+enum ipu7_insys_send_type {
+	IPU7_INSYS_SEND_TYPE_STREAM_OPEN = 0,
+	IPU7_INSYS_SEND_TYPE_STREAM_START_AND_CAPTURE = 1,
+	IPU7_INSYS_SEND_TYPE_STREAM_CAPTURE = 2,
+	IPU7_INSYS_SEND_TYPE_STREAM_ABORT = 3,
+	IPU7_INSYS_SEND_TYPE_STREAM_FLUSH = 4,
+	IPU7_INSYS_SEND_TYPE_STREAM_CLOSE = 5,
+	N_IPU7_INSYS_SEND_TYPE
+};
+
+enum ipu7_insys_resp_type {
+	IPU7_INSYS_RESP_TYPE_STREAM_OPEN_DONE = 0,
+	IPU7_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_ACK = 1,
+	IPU7_INSYS_RESP_TYPE_STREAM_CAPTURE_ACK = 2,
+	IPU7_INSYS_RESP_TYPE_STREAM_ABORT_ACK = 3,
+	IPU7_INSYS_RESP_TYPE_STREAM_FLUSH_ACK = 4,
+	IPU7_INSYS_RESP_TYPE_STREAM_CLOSE_ACK = 5,
+	IPU7_INSYS_RESP_TYPE_PIN_DATA_READY = 6,
+	IPU7_INSYS_RESP_TYPE_FRAME_SOF = 7,
+	IPU7_INSYS_RESP_TYPE_FRAME_EOF = 8,
+	IPU7_INSYS_RESP_TYPE_STREAM_START_AND_CAPTURE_DONE = 9,
+	IPU7_INSYS_RESP_TYPE_STREAM_CAPTURE_DONE = 10,
+	IPU7_INSYS_RESP_TYPE_PWM_IRQ = 11,
+	N_IPU7_INSYS_RESP_TYPE
+};
+
+enum ipu7_insys_mipi_dt_rename_mode {
+	IPU7_INSYS_MIPI_DT_NO_RENAME = 0,
+	IPU7_INSYS_MIPI_DT_RENAMED_MODE = 1,
+	N_IPU7_INSYS_MIPI_DT_MODE
+};
+
+enum insys_msg_err_capture {
+	INSYS_MSG_ERR_CAPTURE_OK = 0,
+	INSYS_MSG_ERR_CAPTURE_STREAM_ID = 1,
+	INSYS_MSG_ERR_CAPTURE_PAYLOAD_PTR = 2,
+	INSYS_MSG_ERR_CAPTURE_MEM_SLOT = 3,
+	INSYS_MSG_ERR_CAPTURE_STREAMING_MODE = 4,
+	INSYS_MSG_ERR_CAPTURE_AVAILABLE_CMD_SLOT = 5,
+	INSYS_MSG_ERR_CAPTURE_CONSUMED_CMD_SLOT = 6,
+	INSYS_MSG_ERR_CAPTURE_CMD_SLOT_PAYLOAD_PTR = 7,
+	INSYS_MSG_ERR_CAPTURE_CMD_PREPARE = 8,
+	INSYS_MSG_ERR_CAPTURE_OUTPUT_PIN = 9,
+	INSYS_MSG_ERR_CAPTURE_SYNC_FRAME_DROP = 10,
+	INSYS_MSG_ERR_CAPTURE_FRAME_MESSAGES_MAP = 11,
+	INSYS_MSG_ERR_CAPTURE_TIMEOUT = 12,
+	INSYS_MSG_ERR_CAPTURE_INVALID_STREAM_STATE = 13,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_MULTIBIT_PH_ERROR_DETECTED = 14,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_PAYLOAD_CRC_ERROR = 15,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_INPUT_DATA_LOSS_ELASTIC_FIFO_OVFL  = 16,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_PIXEL_BUFFER_OVERFLOW = 17,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_BAD_FRAME_DIM = 18,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_PHY_SYNC_ERR = 19,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_SECURE_TOUCH = 20,
+	INSYS_MSG_ERR_CAPTURE_HW_ERR_MASTER_SLAVE_SYNC_ERR = 21,
+	INSYS_MSG_ERR_CAPTURE_FRAME_SKIP_ERR = 22,
+	INSYS_MSG_ERR_CAPTURE_FE_INPUT_FIFO_OVERFLOW_ERR = 23,
+	INSYS_MSG_ERR_CAPTURE_CMD_SUBMIT_TO_HW = 24,
+	INSYS_MSG_ERR_CAPTURE_N
+};
+
+enum insys_msg_err_groups {
+	INSYS_MSG_ERR_GROUP_RESERVED = 0,
+	INSYS_MSG_ERR_GROUP_GENERAL = 1,
+	INSYS_MSG_ERR_GROUP_STREAM = 2,
+	INSYS_MSG_ERR_GROUP_CAPTURE = 3,
+	INSYS_MSG_ERR_GROUP_N,
+};
+
+struct ipu7_fw_isys_logger_config {
+	u8 use_source_severity;
+	u8 source_severity[IPU7_FWLOG_MAX_LOGGER_SOURCES];
+	u8 use_channels_enable_bitmask;
+	u8 channels_enable_bitmask;
+	u8 padding[1];
+	u32 hw_printf_buffer_base_addr;
+	u32 hw_printf_buffer_size_bytes;
+};
+
+struct ipu7_wdt_abi {
+	u32 wdt_timer1_us;
+	u32 wdt_timer2_us;
+};
+
+struct ipu7_insys_config {
+	u32 timeout_val_ms;
+	struct ipu7_fw_isys_logger_config logger_config;
+	struct ipu7_wdt_abi wdt_config;
+};
+
+struct ipu7_insys_capture_output_pin_payload {
+	u64 user_token;
+	u32 addr;
+	u8 pad[4];
+};
+
+struct ipu7_fw_isys_msg_err {
+	u32 err_group;
+	u32 err_code;
+	u32 err_detail[IPU7_MSG_ERR_MAX_DETAILS];
+};
+
+struct ipu7_insys_resp {
+	u64 buf_id;
+	struct ipu7_insys_capture_output_pin_payload pin;
+	struct ipu7_fw_isys_msg_err error_info;
+	u32 timestamp[2];
+	u8 type;
+	u8 msg_link_streaming_mode;
+	u8 stream_id;
+	u8 pin_id;
+	u8 frame_id;
+	u8 skip_frame;
+	u8 pad[2];
+};
+
+struct ipu7_insys_resp_queue_token {
+	struct ipu7_insys_resp resp_info;
+};
+
+struct ipu7_insys_send_queue_token {
+	u64 buf_handle;
+	u32 addr;
+	u16 stream_id;
+	u8 send_type;
+	u8 flag;
+};
+
+struct ipu7_fw_isys_output_link {
+	u32 buffer_lines;
+	u16 foreign_key;
+	u16 granularity_pointer_update;
+	u8 msg_link_streaming_mode;
+	u8 pbk_id;
+	u8 pbk_slot_id;
+	u8 dest;
+	u8 use_sw_managed;
+	u8 is_snoop;
+	u8 pad[2];
+} __packed;
+
+struct ipu7_fw_isys_output_cropping {
+	u16 line_top;
+	u16 line_bottom;
+} __packed;
+
+struct ipu7_fw_isys_output_dpcm {
+	u8 enable;
+	u8 type;
+	u8 predictor;
+	u8 pad;
+} __packed;
+
+struct ipu7_fw_isys_output_pin {
+	struct ipu7_fw_isys_output_link link;
+	struct ipu7_fw_isys_output_cropping crop;
+	struct ipu7_fw_isys_output_dpcm dpcm;
+	u32 stride;
+	u16 ft;
+	u8 send_irq;
+	u8 input_pin_id;
+	u8 early_ack_en;
+	u8 pad[3];
+} __packed;
+
+struct ipu7_fw_isys_resolution {
+	u32 width;
+	u32 height;
+} __packed;
+
+struct ipu7_fw_isys_input_pin {
+	struct ipu7_fw_isys_resolution input_res;
+	u16 sync_msg_map;
+	u8 dt;
+	u8 disable_mipi_unpacking;
+	u8 dt_rename_mode;
+	u8 mapped_dt;
+	u8 pad[2];
+} __packed;
+
+struct ipu7_fw_isys_stream_cfg {
+	struct ipu7_fw_isys_input_pin input_pins[IPU7_MAX_IPINS];
+	struct ipu7_fw_isys_output_pin output_pins[IPU7_MAX_OPINS];
+	u16 stream_msg_map;
+	u8 port_id;
+	u8 vc;
+	u8 nof_input_pins;
+	u8 nof_output_pins;
+	u8 pad[2];
+} __packed;
+
+struct ipu7_fw_isys_capture_output_pin {
+	u64 user_token;
+	u32 addr;
+	u8 pad[4];
+} __packed;
+
+struct ipu7_fw_isys_frame_buff_set {
+	struct ipu7_fw_isys_capture_output_pin output_pins[IPU7_MAX_OPINS];
+	u8 capture_msg_map;
+	u8 frame_id;
+	u8 skip_frame;
+	u8 pad[5];
+} __packed;
+
+struct ipu6_fw_isys_ops *ipu7_fw_isys_get_ops(void);
+irqreturn_t ipu7_isys_isr(struct ipu6_bus_device *adev);
+
+#endif
diff --git a/drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.c b/drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.c
new file mode 100644
index 00000000000000..10273c687faaa0
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.c
@@ -0,0 +1,1072 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2013 - 2026 Intel Corporation
+ */
+
+#include <linux/bitmap.h>
+#include <linux/bug.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+#include <media/mipi-csi2.h>
+#include <media/v4l2-device.h>
+
+#include "ipu6.h"
+#include "ipu6-bus.h"
+#include "ipu6-isys.h"
+#include "ipu6-isys-csi2.h"
+#include "ipu7-isys-csi2-regs.h"
+#include "ipu7-isys-csi-phy.h"
+
+#define PORT_A		0U
+#define PORT_B		1U
+#define PORT_C		2U
+#define PORT_D		3U
+
+#define N_DATA_IDS	8U
+static DECLARE_BITMAP(data_ids, N_DATA_IDS);
+
+struct ddlcal_counter_ref_s {
+	u16 min_mbps;
+	u16 max_mbps;
+
+	u16 ddlcal_counter_ref;
+};
+
+struct ddlcal_params {
+	u16 min_mbps;
+	u16 max_mbps;
+	u16 oa_lanex_hsrx_cdphy_sel_fast;
+	u16 ddlcal_max_phase;
+	u16 phase_bound;
+	u16 ddlcal_dll_fbk;
+	u16 ddlcal_ddl_coarse_bank;
+	u16 fjump_deskew;
+	u16 min_eye_opening_deskew;
+};
+
+struct i_thssettle_params {
+	u16 min_mbps;
+	u16 max_mbps;
+	u16 i_thssettle;
+};
+
+/* lane2 for 4l3t, lane1 for 2l2t */
+struct oa_lane_clk_div_params {
+	u16 min_mbps;
+	u16 max_mbps;
+	u16 oa_lane_hsrx_hs_clk_div;
+};
+
+struct cdr_fbk_cap_prog_params {
+	u16 min_mbps;
+	u16 max_mbps;
+	u16 val;
+};
+
+static const struct ddlcal_counter_ref_s table0[] = {
+	{ 1500, 1999, 118 },
+	{ 2000, 2499, 157 },
+	{ 2500, 3499, 196 },
+	{ 3500, 4499, 274 },
+	{ 4500, 4500, 352 },
+	{ }
+};
+
+static const struct ddlcal_params table1[] = {
+	{ 1500, 1587, 0, 143, 167, 17, 3, 4, 29 },
+	{ 1588, 1687, 0, 135, 167, 15, 3, 4, 27 },
+	{ 1688, 1799, 0, 127, 135, 15, 2, 4, 26 },
+	{ 1800, 1928, 0, 119, 135, 13, 2, 3, 24 },
+	{ 1929, 2076, 0, 111, 135, 13, 2, 3, 23 },
+	{ 2077, 2249, 0, 103, 135, 11, 2, 3, 21 },
+	{ 2250, 2454, 0, 95, 103, 11, 1, 3, 19 },
+	{ 2455, 2699, 0, 87, 103, 9, 1, 3, 18 },
+	{ 2700, 2999, 0, 79, 103, 9, 1, 2, 16 },
+	{ 3000, 3229, 0, 71, 71, 7, 1, 2, 15 },
+	{ 3230, 3599, 1, 87, 103, 9, 1, 3, 18 },
+	{ 3600, 3999, 1, 79, 103, 9, 1, 2, 16 },
+	{ 4000, 4499, 1, 71, 103, 7, 1, 2, 15 },
+	{ 4500, 4500, 1, 63, 71, 7, 0, 2, 13 },
+	{ }
+};
+
+static const struct i_thssettle_params table2[] = {
+	{ 80, 124, 24 },
+	{ 125, 249, 20 },
+	{ 250, 499, 16 },
+	{ 500, 749, 14 },
+	{ 750, 1499, 13 },
+	{ 1500, 4500, 12 },
+	{ }
+};
+
+static const struct oa_lane_clk_div_params table6[] = {
+	{ 80, 159, 0x1 },
+	{ 160, 319, 0x2 },
+	{ 320, 639, 0x3 },
+	{ 640, 1279, 0x4 },
+	{ 1280, 2560, 0x5 },
+	{ 2561, 4500, 0x6 },
+	{ }
+};
+
+static const struct cdr_fbk_cap_prog_params table7[] = {
+	{ 80, 919, 0 },
+	{ 920, 1029, 1 },
+	{ 1030, 1169, 2 },
+	{ 1170, 1349, 3 },
+	{ 1350, 1589, 4 },
+	{ 1590, 1949, 5 },
+	{ 1950, 2499, 6 },
+	{ 2500, 3500, 7 },
+	{ }
+};
+
+static void dwc_phy_write(struct ipu6_isys *isys, u32 id, u32 addr, u16 data)
+{
+	void __iomem *isys_base = isys->pdata->base;
+	void __iomem *base = isys_base + IPU7_IS_IO_CDPHY_BASE(id);
+
+	dev_dbg(&isys->adev->auxdev.dev, "phy write: reg 0x%lx = data 0x%04x",
+		(unsigned long)(base + addr - isys_base), data);
+	writew(data, base + addr);
+}
+
+static u16 dwc_phy_read(struct ipu6_isys *isys, u32 id, u32 addr)
+{
+	void __iomem *isys_base = isys->pdata->base;
+	void __iomem *base = isys_base + IPU7_IS_IO_CDPHY_BASE(id);
+	u16 data;
+
+	data = readw(base + addr);
+	dev_dbg(&isys->adev->auxdev.dev, "phy read: reg 0x%lx = data 0x%04x",
+		(unsigned long)(base + addr - isys_base), data);
+
+	return data;
+}
+
+static void dwc_csi_write(struct ipu6_isys *isys, u32 id, u32 addr, u32 data)
+{
+	void __iomem *isys_base = isys->pdata->base;
+	void __iomem *base = isys_base + IPU7_IS_IO_CSI2_HOST_BASE(id);
+	struct device *dev = &isys->adev->auxdev.dev;
+
+	dev_dbg(dev, "csi write: reg 0x%lx = data 0x%08x",
+		(unsigned long)(base + addr - isys_base), data);
+	writel(data, base + addr);
+	dev_dbg(dev, "csi read: reg 0x%lx = data 0x%08x",
+		(unsigned long)(base + addr - isys_base),
+		readl(base + addr));
+}
+
+static void gpreg_write(struct ipu6_isys *isys, u32 id, u32 addr, u32 data)
+{
+	void __iomem *isys_base = isys->pdata->base;
+	u32 gpreg = isys->pdata->ipdata->csi2.gpreg;
+	void __iomem *base = isys_base + gpreg + 0x1000 * id;
+	struct device *dev = &isys->adev->auxdev.dev;
+
+	dev_dbg(dev, "gpreg write: reg 0x%lx = data 0x%08x",
+		(unsigned long)(base + addr - isys_base), data);
+	writel(data, base + addr);
+	dev_dbg(dev, "gpreg read: reg 0x%lx = data 0x%08x",
+		(unsigned long)(base + addr - isys_base),
+		readl(base + addr));
+}
+
+static u32 dwc_csi_read(struct ipu6_isys *isys, u32 id, u32 addr)
+{
+	void __iomem *isys_base = isys->pdata->base;
+	void __iomem *base = isys_base + IPU7_IS_IO_CSI2_HOST_BASE(id);
+	u32 data;
+
+	data = readl(base + addr);
+	dev_dbg(&isys->adev->auxdev.dev, "csi read: reg 0x%lx = data 0x%x",
+		(unsigned long)(base + addr - isys_base), data);
+
+	return data;
+}
+
+static void dwc_phy_write_mask(struct ipu6_isys *isys, u32 id, u32 addr,
+			       u16 val, u8 lo, u8 hi)
+{
+	u32 temp, mask;
+
+	WARN_ON(lo > hi);
+	WARN_ON(hi > 15);
+
+	mask = ((~0U - (1U << lo) + 1U)) & (~0U >> (31 - hi));
+	temp = dwc_phy_read(isys, id, addr);
+	temp &= ~mask;
+	temp |= (val << lo) & mask;
+	dwc_phy_write(isys, id, addr, temp);
+}
+
+static void dwc_csi_write_mask(struct ipu6_isys *isys, u32 id, u32 addr,
+			       u32 val, u8 hi, u8 lo)
+{
+	u32 temp, mask;
+
+	WARN_ON(lo > hi);
+
+	mask = ((~0U - (1U << lo) + 1U)) & (~0U >> (31 - hi));
+	temp = dwc_csi_read(isys, id, addr);
+	temp &= ~mask;
+	temp |= (val << lo) & mask;
+	dwc_csi_write(isys, id, addr, temp);
+}
+
+static void ipu7_isys_csi_ctrl_cfg(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	struct device *dev = &isys->adev->auxdev.dev;
+	u32 id, lanes, phy_mode;
+	u32 val;
+
+	id = csi2->port;
+	lanes = csi2->nlanes;
+	phy_mode = csi2->phy_mode;
+	dev_dbg(dev, "csi-%d controller init with %u lanes, phy mode %u",
+		id, lanes, phy_mode);
+
+	val = dwc_csi_read(isys, id, IPU7_VERSION);
+	dev_dbg(dev, "csi-%d controller version = 0x%x", id, val);
+
+	/* num of active data lanes */
+	dwc_csi_write(isys, id, IPU7_N_LANES, lanes - 1);
+	dwc_csi_write(isys, id, IPU7_CDPHY_MODE, phy_mode);
+	dwc_csi_write(isys, id, IPU7_VC_EXTENSION, 0);
+
+	/* only mask PHY_FATAL and PKT_FATAL interrupts */
+	dwc_csi_write(isys, id, IPU7_INT_MSK_PHY_FATAL, 0xff);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_PKT_FATAL, 0x3);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_PHY, 0x0);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_LINE, 0x0);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_BNDRY_FRAME_FATAL, 0x0);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_SEQ_FRAME_FATAL, 0x0);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_CRC_FRAME_FATAL, 0x0);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_PLD_CRC_FATAL, 0x0);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_DATA_ID, 0x0);
+	dwc_csi_write(isys, id, IPU7_INT_MSK_ECC_CORRECTED, 0x0);
+}
+
+static void ipu7_isys_csi_phy_reset(struct ipu6_isys *isys, u32 id)
+{
+	dwc_csi_write(isys, id, IPU7_PHY_SHUTDOWNZ, 0);
+	dwc_csi_write(isys, id, IPU7_DPHY_RSTZ, 0);
+	dwc_csi_write(isys, id, IPU7_CSI2_RESETN, 0);
+	gpreg_write(isys, id, IPU7_PHY_RESET, 0);
+	gpreg_write(isys, id, IPU7_PHY_SHUTDOWN, 0);
+}
+
+/* 8 Data ID monitors, each Data ID is composed by pair of VC and data type */
+static int __dids_config(struct ipu6_isys_csi2 *csi2, u32 id, u8 vc, u8 dt)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	u32 reg, n;
+	u8 lo, hi;
+	int ret;
+
+	dev_dbg(&isys->adev->auxdev.dev,
+		"config CSI-%u with vc:%u dt:0x%02x\n", id, vc, dt);
+
+	dwc_csi_write(isys, id, IPU7_VC_EXTENSION, 0x0);
+	n = find_first_zero_bit(data_ids, N_DATA_IDS);
+	if (n == N_DATA_IDS)
+		return -ENOSPC;
+
+	ret = test_and_set_bit(n, data_ids);
+	if (ret)
+		return -EBUSY;
+
+	reg = n < 4 ? IPU7_DATA_IDS_VC_1 : IPU7_DATA_IDS_VC_2;
+	lo = (n % 4) * 8;
+	hi = lo + 4;
+	dwc_csi_write_mask(isys, id, reg, vc & GENMASK(4, 0), hi, lo);
+
+	reg = n < 4 ? IPU7_DATA_IDS_1 : IPU7_DATA_IDS_2;
+	lo = (n % 4) * 8;
+	hi = lo + 5;
+	dwc_csi_write_mask(isys, id, reg, dt & GENMASK(5, 0), hi, lo);
+
+	return 0;
+}
+
+static int ipu7_isys_csi_ctrl_dids_config(struct ipu6_isys_csi2 *csi2, u32 id)
+{
+	struct v4l2_mbus_frame_desc_entry *desc_entry = NULL;
+	struct device *dev = &csi2->isys->adev->auxdev.dev;
+	struct v4l2_mbus_frame_desc desc;
+	struct v4l2_subdev *ext_sd;
+	struct media_pad *pad;
+	int ret;
+
+	pad = media_entity_remote_source_pad_unique(&csi2->asd.sd.entity);
+	if (IS_ERR(pad)) {
+		dev_warn(dev, "can't get remote source pad of %s (%pe)\n",
+			 csi2->asd.sd.name, pad);
+		return PTR_ERR(pad);
+	}
+
+	ext_sd = media_entity_to_v4l2_subdev(pad->entity);
+	if (WARN(!ext_sd, "Failed to get subdev for entity %s\n",
+		 pad->entity->name))
+		return -ENODEV;
+
+	ret = v4l2_subdev_call(ext_sd, pad, get_frame_desc, pad->index, &desc);
+	if (ret)
+		return ret;
+
+	if (desc.type != V4L2_MBUS_FRAME_DESC_TYPE_CSI2) {
+		dev_warn(dev, "Unsupported frame descriptor type\n");
+		return -EINVAL;
+	}
+
+	for (unsigned int i = 0; i < desc.num_entries; i++) {
+		desc_entry = &desc.entry[i];
+		if (desc_entry->bus.csi2.vc < NR_OF_CSI2_VC) {
+			ret = __dids_config(csi2, id, desc_entry->bus.csi2.vc,
+					    desc_entry->bus.csi2.dt);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+#define CDPHY_TIMEOUT 5000000U
+static int ipu7_isys_phy_ready(struct ipu6_isys *isys, u32 id)
+{
+	void __iomem *isys_base = isys->pdata->base;
+	u32 gpreg_offset = isys->pdata->ipdata->csi2.gpreg;
+	void __iomem *gpreg = isys_base + gpreg_offset + 0x1000 * id;
+	struct device *dev = &isys->adev->auxdev.dev;
+	u32 phy_ready;
+	u32 reg, rext;
+	int ret;
+
+	dev_dbg(dev, "waiting phy ready...\n");
+	ret = readl_poll_timeout(gpreg + IPU7_PHY_READY, phy_ready,
+				 phy_ready & BIT(0) && phy_ready != ~0U,
+				 100, CDPHY_TIMEOUT);
+	dev_dbg(dev, "phy %u ready = 0x%08x\n",
+		id, readl(gpreg + IPU7_PHY_READY));
+	dev_dbg(dev, "csi %u IPU7_PHY_RX = 0x%08x\n", id,
+		dwc_csi_read(isys, id, IPU7_PHY_RX));
+	dev_dbg(dev, "csi %u IPU7_PHY_STOPSTATE = 0x%08x\n", id,
+		dwc_csi_read(isys, id, IPU7_PHY_STOPSTATE));
+	dev_dbg(dev, "csi %u IPU7_PHY_CAL = 0x%08x\n", id,
+		dwc_csi_read(isys, id, IPU7_PHY_CAL));
+	for (unsigned int i = 0; i < 4U; i++) {
+		reg = IPU7_CORE_DIG_DLANE_0_R_HS_RX_0 + (i * 0x400U);
+		dev_dbg(dev, "phy %u DLANE%u skewcal = 0x%04x\n",
+			id, i, dwc_phy_read(isys, id, reg));
+	}
+	dev_dbg(dev, "phy %u DDLCAL = 0x%04x\n", id,
+		dwc_phy_read(isys, id,
+			     IPU7_PPI_CALIBCTRL_R_COMMON_CALIBCTRL_2_5));
+	dev_dbg(dev, "phy %u TERMCAL = 0x%04x\n", id,
+		dwc_phy_read(isys, id, IPU7_PPI_R_TERMCAL_DEBUG_0));
+	dev_dbg(dev, "phy %u LPDCOCAL = 0x%04x\n", id,
+		dwc_phy_read(isys, id, IPU7_PPI_R_LPDCOCAL_DEBUG_RB));
+	dev_dbg(dev, "phy %u HSDCOCAL = 0x%04x\n", id,
+		dwc_phy_read(isys, id, IPU7_PPI_R_HSDCOCAL_DEBUG_RB));
+	dev_dbg(dev, "phy %u LPDCOCAL_VT = 0x%04x\n", id,
+		dwc_phy_read(isys, id, IPU7_PPI_R_LPDCOCAL_DEBUG_VT));
+
+	if (!ret) {
+		if (id) {
+			dev_dbg(dev, "ignore phy %u rext\n", id);
+			return 0;
+		}
+
+		rext = dwc_phy_read(isys, id,
+				    IPU7_CORE_DIG_IOCTRL_R_AFE_CB_CTRL_2_15) &
+					0xfU;
+		dev_dbg(dev, "phy %u rext value = %u\n", id, rext);
+		isys->phy_rext_cal = (rext ? rext : 5);
+
+		return 0;
+	}
+
+	dev_err(dev, "wait phy ready timeout!\n");
+
+	return ret;
+}
+
+static int lookup_table1(u64 mbps)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(table1); i++) {
+		if (mbps >= table1[i].min_mbps && mbps <= table1[i].max_mbps)
+			return i;
+	}
+
+	return -ENXIO;
+}
+
+static const u16 deskew_fine_mem[] = {
+	0x0404, 0x040c, 0x0414, 0x041c,
+	0x0423, 0x0429, 0x0430, 0x043a,
+	0x0445, 0x044a, 0x0450, 0x045a,
+	0x0465, 0x0469, 0x0472, 0x047a,
+	0x0485, 0x0489, 0x0490, 0x049a,
+	0x04a4, 0x04ac, 0x04b4, 0x04bc,
+	0x04c4, 0x04cc, 0x04d4, 0x04dc,
+	0x04e4, 0x04ec, 0x04f4, 0x04fc,
+	0x0504, 0x050c, 0x0514, 0x051c,
+	0x0523, 0x0529, 0x0530, 0x053a,
+	0x0545, 0x054a, 0x0550, 0x055a,
+	0x0565, 0x0569, 0x0572, 0x057a,
+	0x0585, 0x0589, 0x0590, 0x059a,
+	0x05a4, 0x05ac, 0x05b4, 0x05bc,
+	0x05c4, 0x05cc, 0x05d4, 0x05dc,
+	0x05e4, 0x05ec, 0x05f4, 0x05fc,
+	0x0604, 0x060c, 0x0614, 0x061c,
+	0x0623, 0x0629, 0x0632, 0x063a,
+	0x0645, 0x064a, 0x0650, 0x065a,
+	0x0665, 0x0669, 0x0672, 0x067a,
+	0x0685, 0x0689, 0x0690, 0x069a,
+	0x06a4, 0x06ac, 0x06b4, 0x06bc,
+	0x06c4, 0x06cc, 0x06d4, 0x06dc,
+	0x06e4, 0x06ec, 0x06f4, 0x06fc,
+	0x0704, 0x070c, 0x0714, 0x071c,
+	0x0723, 0x072a, 0x0730, 0x073a,
+	0x0745, 0x074a, 0x0750, 0x075a,
+	0x0765, 0x0769, 0x0772, 0x077a,
+	0x0785, 0x0789, 0x0790, 0x079a,
+	0x07a4, 0x07ac, 0x07b4, 0x07bc,
+	0x07c4, 0x07cc, 0x07d4, 0x07dc,
+	0x07e4, 0x07ec, 0x07f4, 0x07fc,
+};
+
+static void ipu7_isys_dphy_config(struct ipu6_isys *isys, u8 id, u8 lanes,
+				  bool aggregation, u64 mbps)
+{
+	struct ipu6_device *isp = isys->adev->isp;
+	u16 hsrxval0 = 0;
+	u16 hsrxval1 = 0;
+	u16 hsrxval2 = 0;
+	int index;
+	u16 reg;
+	u16 val;
+	u32 i;
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_RW_COMMON_7, 0, 0, 9);
+	if (mbps > 1500)
+		dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_7,
+				   40, 0, 7);
+	else
+		dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_7,
+				   104, 0, 7);
+
+	dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_8,
+			   80, 0, 7);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_0, 191, 0, 9);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_7, 34, 7, 12);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_1, 38, 8, 15);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_2, 4, 12, 15);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_2, 2, 10, 11);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_2, 1, 8, 8);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_2, 38, 0, 7);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_2, 1, 9, 9);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_4, 10, 0, 9);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_6, 20, 0, 9);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_7, 19, 0, 6);
+
+	for (i = 0; i < ARRAY_SIZE(table0); i++) {
+		if (mbps >= table0[i].min_mbps && mbps <= table0[i].max_mbps) {
+			dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_3,
+					   table0[i].ddlcal_counter_ref,
+					   0, 9);
+			break;
+		}
+	}
+
+	index = lookup_table1(mbps);
+	if (index >= 0) {
+		dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_1,
+				   table1[index].phase_bound, 0, 7);
+		dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_5,
+				   table1[index].ddlcal_dll_fbk, 4, 9);
+		dwc_phy_write_mask(isys, id, IPU7_PPI_RW_DDLCAL_CFG_5,
+				   table1[index].ddlcal_ddl_coarse_bank, 0, 3);
+
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_8;
+		val = table1[index].oa_lanex_hsrx_cdphy_sel_fast;
+		for (i = 0; i < lanes + 1; i++)
+			dwc_phy_write_mask(isys, id, reg + (i * 0x400), val,
+					   12, 12);
+	}
+
+	reg = IPU7_CORE_DIG_DLANE_0_RW_LP_0;
+	for (i = 0; i < lanes; i++)
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 6, 8, 11);
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_2,
+			   0, 0, 0);
+	if (!IS_IPU7_MTL(isp) || id == PORT_B || id == PORT_C) {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_2,
+				   1, 0, 0);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_2,
+				   0, 0, 0);
+	} else {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_2,
+				   0, 0, 0);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_2,
+				   1, 0, 0);
+	}
+
+	if (lanes == 4 && IS_IPU7_MTL(isp)) {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_2,
+				   0, 0, 0);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_2,
+				   0, 0, 0);
+	}
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_RW_COMMON_6, 1, 0, 2);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_RW_COMMON_6, 1, 3, 5);
+
+	reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_12;
+	val = (mbps > 1500) ? 0 : 1;
+	for (i = 0; i < lanes + 1; i++) {
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), val, 1, 1);
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), !val, 3, 3);
+	}
+
+	reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_13;
+	val = (mbps > 1500) ? 0 : 1;
+	for (i = 0; i < lanes + 1; i++) {
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), val, 1, 1);
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), val, 3, 3);
+	}
+
+	if (!IS_IPU7_MTL(isp) || id == PORT_B || id == PORT_C)
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_9;
+	else
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_9;
+
+	for (i = 0; i < ARRAY_SIZE(table6); i++) {
+		if (mbps >= table6[i].min_mbps && mbps <= table6[i].max_mbps) {
+			dwc_phy_write_mask(isys, id, reg,
+					   table6[i].oa_lane_hsrx_hs_clk_div,
+					   5, 7);
+			break;
+		}
+	}
+
+	if (aggregation) {
+		dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_RW_COMMON_0, 1,
+				   1, 1);
+
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_15;
+		dwc_phy_write_mask(isys, id, reg, 3, 3, 4);
+
+		val = (id == PORT_A) ? 3 : 0;
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_15;
+		dwc_phy_write_mask(isys, id, reg, val, 3, 4);
+
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_15;
+		dwc_phy_write_mask(isys, id, reg, 3, 3, 4);
+	}
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_0,
+			   28, 0, 7);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_7,
+			   6, 0, 7);
+
+	reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_0;
+	for (i = 0; i < ARRAY_SIZE(table2); i++) {
+		if (mbps >= table2[i].min_mbps && mbps <= table2[i].max_mbps) {
+			u8 j;
+
+			for (j = 0; j < lanes; j++)
+				dwc_phy_write_mask(isys, id, reg + (j * 0x400),
+						   table2[i].i_thssettle,
+						   8, 15);
+			break;
+		}
+	}
+
+	/* deskew */
+	for (i = 0; i < lanes; i++) {
+		reg = IPU7_CORE_DIG_DLANE_0_RW_CFG_1;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400),
+				   ((mbps > 1500) ? 0x1 : 0x2), 2, 3);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_2;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400),
+				   ((mbps > 2500) ? 0 : 1), 15, 15);
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 1, 13, 13);
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 7, 9, 12);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_LP_0;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 1, 12, 15);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_LP_2;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 0, 0, 0);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_1;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 16, 0, 7);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_3;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 2, 0, 2);
+		index = lookup_table1(mbps);
+		if (index >= 0) {
+			val = table1[index].fjump_deskew;
+			dwc_phy_write_mask(isys, id, reg + (i * 0x400), val,
+					   3, 8);
+		}
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_4;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 150, 0, 15);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_5;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 0, 0, 7);
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 1, 8, 15);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_6;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 2, 0, 7);
+		index = lookup_table1(mbps);
+		if (index >= 0) {
+			val = table1[index].min_eye_opening_deskew;
+			dwc_phy_write_mask(isys, id, reg + (i * 0x400), val,
+					   8, 15);
+		}
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_7;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 0, 13, 13);
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 0, 15, 15);
+
+		reg = IPU7_CORE_DIG_DLANE_0_RW_HS_RX_9;
+		index = lookup_table1(mbps);
+		if (index >= 0) {
+			val = table1[index].ddlcal_max_phase;
+			dwc_phy_write_mask(isys, id, reg + (i * 0x400),
+					   val, 0, 7);
+		}
+	}
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_DLANE_CLK_RW_LP_0,
+			   1, 12, 15);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_DLANE_CLK_RW_LP_2, 0, 0, 0);
+
+	for (i = 0; i < ARRAY_SIZE(deskew_fine_mem); i++)
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_COMMON_RW_DESKEW_FINE_MEM,
+				   deskew_fine_mem[i], 0, 15);
+
+	if (mbps > 1500) {
+		hsrxval0 = 4;
+		hsrxval2 = 3;
+	}
+
+	if (mbps > 2500)
+		hsrxval1 = 2;
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9,
+			   hsrxval0, 0, 2);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_9,
+			   hsrxval0, 0, 2);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_9,
+			   hsrxval0, 0, 2);
+	if (lanes == 4 && IS_IPU7_MTL(isp)) {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_9,
+				   hsrxval0, 0, 2);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_9,
+				   hsrxval0, 0, 2);
+	}
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9,
+			   hsrxval1, 3, 4);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_9,
+			   hsrxval1, 3, 4);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_9,
+			   hsrxval1, 3, 4);
+	if (lanes == 4 && IS_IPU7_MTL(isp)) {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_9,
+				   hsrxval1, 3, 4);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_9,
+				   hsrxval1, 3, 4);
+	}
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_15,
+			   hsrxval2, 0, 2);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_15,
+			   hsrxval2, 0, 2);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_15,
+			   hsrxval2, 0, 2);
+	if (lanes == 4 && IS_IPU7_MTL(isp)) {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_15,
+				   hsrxval2, 0, 2);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_15,
+				   hsrxval2, 0, 2);
+	}
+
+	/* force and override rext */
+	if (isys->phy_rext_cal && id) {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_8,
+				   isys->phy_rext_cal, 0, 3);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7,
+				   1, 11, 11);
+	}
+}
+
+static void ipu7_isys_cphy_config(struct ipu6_isys *isys, u8 id, u8 lanes,
+				  bool aggregation, u64 mbps)
+{
+	struct ipu6_device *isp = isys->adev->isp;
+	u8 trios = 2;
+	u16 coarse_target;
+	u16 deass_thresh;
+	u16 delay_thresh;
+	u16 reset_thresh;
+	u16 cap_prog = 6U;
+	u16 reg;
+	u16 val;
+	u32 i;
+	u64 r64;
+	u32 r;
+
+	if (IS_IPU7P5(isp))
+		val = 0x15;
+	else
+		val = 0x155;
+
+	if (IS_IPU7_MTL(isp))
+		trios = 3;
+
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_RW_COMMON_7, val, 0, 9);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_7,
+			   104, 0, 7);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_8,
+			   16, 0, 7);
+
+	reg = IPU7_CORE_DIG_CLANE_0_RW_LP_0;
+	for (i = 0; i < trios; i++)
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 6, 8, 11);
+
+	val = (mbps > 900U) ? 1U : 0U;
+	for (i = 0; i < trios; i++) {
+		reg = IPU7_CORE_DIG_CLANE_0_RW_HS_RX_0;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 1, 0, 0);
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), val, 1, 1);
+
+		reg = IPU7_CORE_DIG_CLANE_0_RW_HS_RX_1;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 38, 0, 15);
+
+		reg = IPU7_CORE_DIG_CLANE_0_RW_HS_RX_5;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 38, 0, 15);
+
+		reg = IPU7_CORE_DIG_CLANE_0_RW_HS_RX_6;
+		dwc_phy_write_mask(isys, id, reg + (i * 0x400), 10, 0, 15);
+	}
+
+	/*
+	 * Below 900Msps, always use the same value.
+	 * The formula is suitable for data rate 80-3500Msps.
+	 * Timebase (us) = 1, DIV = 32, TDDL (UI) = 0.5
+	 */
+	if (mbps >= 80U)
+		coarse_target = DIV_ROUND_UP_ULL(mbps, 16) - 1;
+	else
+		coarse_target = 56;
+
+	for (i = 0; i < trios; i++) {
+		reg = IPU7_CORE_DIG_CLANE_0_RW_HS_RX_2 + i * 0x400;
+		dwc_phy_write_mask(isys, id, reg, coarse_target, 0, 15);
+	}
+
+	dwc_phy_write_mask(isys, id,
+			   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_2, 1, 0, 0);
+	dwc_phy_write_mask(isys, id,
+			   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_2, 0, 0, 0);
+	dwc_phy_write_mask(isys, id,
+			   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_2, 1, 0, 0);
+
+	if (!IS_IPU7P5(isp) && lanes == 4) {
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_2,
+				   1, 0, 0);
+		dwc_phy_write_mask(isys, id,
+				   IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_2,
+				   0, 0, 0);
+	}
+
+	for (i = 0; i < trios; i++) {
+		reg = IPU7_CORE_DIG_RW_TRIO0_0 + i * 0x400;
+		dwc_phy_write_mask(isys, id, reg, 1, 6, 8);
+		dwc_phy_write_mask(isys, id, reg, 1, 3, 5);
+		dwc_phy_write_mask(isys, id, reg, 2, 0, 2);
+	}
+
+	deass_thresh = (u16)div64_u64_rem(7 * 1000 * 6, mbps * 5U, &r64) + 1;
+	if (r64 != 0)
+		deass_thresh++;
+
+	reg = IPU7_CORE_DIG_RW_TRIO0_2;
+	for (i = 0; i < trios; i++)
+		dwc_phy_write_mask(isys, id, reg + 0x400 * i,
+				   deass_thresh, 0, 7);
+
+	delay_thresh = div64_u64((224U - (9U * 7U)) * 1000U, 5U * mbps) - 7u;
+
+	if (delay_thresh < 1)
+		delay_thresh = 1;
+
+	reg = IPU7_CORE_DIG_RW_TRIO0_1;
+	for (i = 0; i < trios; i++)
+		dwc_phy_write_mask(isys, id, reg + 0x400 * i,
+				   delay_thresh, 0, 15);
+
+	reset_thresh = (u16)div_u64_rem(2U * 5U * mbps, 7U * 1000U, &r);
+	if (!r)
+		reset_thresh--;
+
+	if (reset_thresh < 1)
+		reset_thresh = 1;
+
+	reg = IPU7_CORE_DIG_RW_TRIO0_0;
+	for (i = 0; i < trios; i++)
+		dwc_phy_write_mask(isys, id, reg + 0x400 * i,
+				   reset_thresh, 9, 11);
+
+	/* Tuning ITMINRX to 2 for CPHY */
+	reg = IPU7_CORE_DIG_CLANE_0_RW_LP_0;
+	for (i = 0; i < trios; i++)
+		dwc_phy_write_mask(isys, id, reg + 0x400 * i, 2, 12, 15);
+
+	reg = IPU7_CORE_DIG_CLANE_0_RW_LP_2;
+	for (i = 0; i < trios; i++)
+		dwc_phy_write_mask(isys, id, reg + 0x400 * i, 0, 0, 0);
+
+	reg = IPU7_CORE_DIG_CLANE_0_RW_HS_RX_0;
+	for (i = 0; i < trios; i++)
+		dwc_phy_write_mask(isys, id, reg + 0x400 * i, 12, 2, 6);
+
+	for (i = 0; i < ARRAY_SIZE(table7); i++) {
+		if (mbps >= table7[i].min_mbps && mbps <= table7[i].max_mbps) {
+			cap_prog = table7[i].val;
+			break;
+		}
+	}
+
+	for (i = 0; i < (lanes + 1); i++) {
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9 + 0x400 * i;
+		dwc_phy_write_mask(isys, id, reg, 4U, 0, 2);
+		/* Set GMODE to 2 when CPHY >= 1.5Gsps */
+		if (mbps >= 1500)
+			dwc_phy_write_mask(isys, id, reg, 2U, 3, 4);
+		else
+			dwc_phy_write_mask(isys, id, reg, 0U, 3, 4);
+
+		reg = IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_7 + 0x400 * i;
+		dwc_phy_write_mask(isys, id, reg, cap_prog, 10, 12);
+	}
+}
+
+static int ipu7_isys_phy_config(struct ipu6_isys *isys, u8 id, u8 lanes,
+				bool aggregation)
+{
+	struct device *dev = &isys->adev->auxdev.dev;
+	u32 phy_mode;
+	s64 link_freq;
+	u64 mbps;
+
+	if (aggregation)
+		link_freq = ipu6_isys_csi2_get_link_freq(&isys->csi2[0]);
+	else
+		link_freq = ipu6_isys_csi2_get_link_freq(&isys->csi2[id]);
+
+	if (link_freq < 0) {
+		dev_err(dev, "get link freq failed (%lld)\n", link_freq);
+		return link_freq;
+	}
+
+	mbps = div_u64(link_freq, 500000);
+	dev_dbg(dev, "config phy %u with lanes %u aggregation %d mbps %lld\n",
+		id, lanes, aggregation, mbps);
+
+	dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_10,
+			   48, 0, 7);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_2,
+			   1, 12, 13);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_0,
+			   63, 2, 7);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_STARTUP_1_1,
+			   563, 0, 11);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_2,
+			   5, 0, 7);
+	/* bypass the RCAL state (bit6) */
+	if (aggregation && id != PORT_A)
+		dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_2,
+				   0x45, 0, 7);
+
+	dwc_phy_write_mask(isys, id, IPU7_PPI_STARTUP_RW_COMMON_DPHY_6,
+			   39, 0, 7);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_CALIBCTRL_RW_COMMON_BG_0,
+			   500, 0, 8);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_TERMCAL_CFG_0, 38, 0, 6);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_OFFSETCAL_CFG_0, 7, 0, 4);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_TIMEBASE, 153, 0, 9);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_NREF, 800, 0, 10);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_NREF_RANGE,
+			   27, 0, 4);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_TWAIT_CONFIG,
+			   47, 0, 8);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_TWAIT_CONFIG,
+			   127, 9, 15);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_VT_CONFIG, 47, 7, 15);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_VT_CONFIG, 27, 2, 6);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_VT_CONFIG, 3, 0, 1);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_LPDCOCAL_COARSE_CFG, 1, 0, 1);
+	dwc_phy_write_mask(isys, id, IPU7_PPI_RW_COMMON_CFG, 3, 0, 1);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_0,
+			   0, 10, 10);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_1,
+			   1, 10, 10);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_1,
+			   0, 15, 15);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_3,
+			   3, 8, 9);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_0,
+			   0, 15, 15);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_6,
+			   7, 12, 14);
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7,
+			   0, 8, 10);
+	/* resistance tuning: 1 for 45ohm, 0 for 50ohm */
+	dwc_phy_write_mask(isys, id, IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_5,
+			   1, 8, 8);
+
+	if (aggregation)
+		phy_mode = isys->csi2[0].phy_mode;
+	else
+		phy_mode = isys->csi2[id].phy_mode;
+
+	if (phy_mode == PHY_MODE_DPHY) {
+		ipu7_isys_dphy_config(isys, id, lanes, aggregation, mbps);
+	} else if (phy_mode == PHY_MODE_CPHY) {
+		ipu7_isys_cphy_config(isys, id, lanes, aggregation, mbps);
+	} else {
+		dev_err(dev, "unsupported phy mode %d!\n",
+			isys->csi2[id].phy_mode);
+	}
+
+	return 0;
+}
+
+static int ipu7_isys_csi_phy_powerup(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+	u32 lanes = csi2->nlanes;
+	bool aggregation = false;
+	u32 id = csi2->port;
+	int ret;
+
+	/* lanes remapping for aggregation (port AB) mode */
+	if (!IS_IPU7_MTL(isys->adev->isp) && lanes > 2 && id == PORT_A) {
+		aggregation = true;
+		lanes = 2;
+	}
+
+	ipu7_isys_csi_phy_reset(isys, id);
+	gpreg_write(isys, id, IPU7_PHY_CLK_LANE_CONTROL, 0x1);
+	gpreg_write(isys, id, IPU7_PHY_CLK_LANE_FORCE_CONTROL, 0x2);
+	gpreg_write(isys, id, IPU7_PHY_LANE_CONTROL_EN, (1U << lanes) - 1U);
+	gpreg_write(isys, id, IPU7_PHY_LANE_FORCE_CONTROL, 0xf);
+	gpreg_write(isys, id, IPU7_PHY_MODE, csi2->phy_mode);
+
+	/* config PORT_B if aggregation mode */
+	if (aggregation) {
+		ipu7_isys_csi_phy_reset(isys, PORT_B);
+		gpreg_write(isys, PORT_B, IPU7_PHY_CLK_LANE_CONTROL, 0x0);
+		gpreg_write(isys, PORT_B, IPU7_PHY_LANE_CONTROL_EN, 0x3);
+		gpreg_write(isys, PORT_B, IPU7_PHY_CLK_LANE_FORCE_CONTROL, 0x2);
+		gpreg_write(isys, PORT_B, IPU7_PHY_LANE_FORCE_CONTROL, 0xf);
+		gpreg_write(isys, PORT_B, IPU7_PHY_MODE, csi2->phy_mode);
+	}
+
+	ipu7_isys_csi_ctrl_cfg(csi2);
+	ipu7_isys_csi_ctrl_dids_config(csi2, id);
+
+	ret = ipu7_isys_phy_config(isys, id, lanes, aggregation);
+	if (ret < 0)
+		return ret;
+
+	gpreg_write(isys, id, IPU7_PHY_RESET, 1);
+	gpreg_write(isys, id, IPU7_PHY_SHUTDOWN, 1);
+	dwc_csi_write(isys, id, IPU7_DPHY_RSTZ, 1);
+	dwc_csi_write(isys, id, IPU7_PHY_SHUTDOWNZ, 1);
+	dwc_csi_write(isys, id, IPU7_CSI2_RESETN, 1);
+
+	ret = ipu7_isys_phy_ready(isys, id);
+	if (ret < 0)
+		return ret;
+
+	gpreg_write(isys, id, IPU7_PHY_LANE_FORCE_CONTROL, 0);
+	gpreg_write(isys, id, IPU7_PHY_CLK_LANE_FORCE_CONTROL, 0);
+
+	/* config PORT_B if aggregation mode */
+	if (aggregation) {
+		ret = ipu7_isys_phy_config(isys, PORT_B, 2, aggregation);
+		if (ret < 0)
+			return ret;
+
+		gpreg_write(isys, PORT_B, IPU7_PHY_RESET, 1);
+		gpreg_write(isys, PORT_B, IPU7_PHY_SHUTDOWN, 1);
+		dwc_csi_write(isys, PORT_B, IPU7_DPHY_RSTZ, 1);
+		dwc_csi_write(isys, PORT_B, IPU7_PHY_SHUTDOWNZ, 1);
+		dwc_csi_write(isys, PORT_B, IPU7_CSI2_RESETN, 1);
+		ret = ipu7_isys_phy_ready(isys, PORT_B);
+		if (ret < 0)
+			return ret;
+
+		gpreg_write(isys, PORT_B, IPU7_PHY_LANE_FORCE_CONTROL, 0);
+		gpreg_write(isys, PORT_B, IPU7_PHY_CLK_LANE_FORCE_CONTROL, 0);
+	}
+
+	return 0;
+}
+
+static void ipu7_isys_csi_phy_powerdown(struct ipu6_isys_csi2 *csi2)
+{
+	struct ipu6_isys *isys = csi2->isys;
+
+	ipu7_isys_csi_phy_reset(isys, csi2->port);
+	if (!IS_IPU7_MTL(isys->adev->isp) &&
+	    csi2->nlanes > 2U && csi2->port == PORT_A)
+		ipu7_isys_csi_phy_reset(isys, PORT_B);
+}
+
+int ipu7_isys_csi_phy_set_power(struct ipu6_isys *isys,
+				struct ipu6_isys_csi2_config *cfg,
+				const struct ipu6_isys_csi2_timing *timing,
+				bool on)
+{
+	struct ipu6_isys_csi2 *csi2 = &isys->csi2[cfg->port];
+
+	if (on)
+		return ipu7_isys_csi_phy_powerup(csi2);
+
+	ipu7_isys_csi_phy_powerdown(csi2);
+
+	return 0;
+}
diff --git a/drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.h b/drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.h
new file mode 100644
index 00000000000000..849fe888db070b
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-isys-csi-phy.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2026 Intel Corporation */
+
+#ifndef IPU7_ISYS_CSI_PHY_H
+#define IPU7_ISYS_CSI_PHY_H
+
+struct ipu6_isys;
+struct ipu6_isys_csi2_config;
+struct ipu6_isys_csi2_timing;
+
+int ipu7_isys_csi_phy_set_power(struct ipu6_isys *isys,
+				struct ipu6_isys_csi2_config *cfg,
+				const struct ipu6_isys_csi2_timing *timing,
+				bool on);
+
+#endif /* IPU7_ISYS_CSI_PHY_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu7-isys-csi2-regs.h b/drivers/media/pci/intel/ipu6/ipu7-isys-csi2-regs.h
new file mode 100644
index 00000000000000..3859401d480231
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-isys-csi2-regs.h
@@ -0,0 +1,1189 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 - 2026 Intel Corporation
+ */
+
+#ifndef IPU7_ISYS_CSI2_REG_H
+#define IPU7_ISYS_CSI2_REG_H
+
+/* IS main regs base */
+#define IPU7_IS_MAIN_BASE			0x240000
+#define IPU7_IS_MAIN_S2B_BASE			(IPU7_IS_MAIN_BASE + 0x22000)
+#define IPU7_IS_MAIN_B2O_BASE			(IPU7_IS_MAIN_BASE + 0x26000)
+#define IPU7_IS_MAIN_ISD_M0_BASE		(IPU7_IS_MAIN_BASE + 0x2b000)
+#define IPU7_IS_MAIN_ISD_M1_BASE		(IPU7_IS_MAIN_BASE + 0x2b100)
+#define IPU7_IS_MAIN_ISD_INT_BASE		(IPU7_IS_MAIN_BASE + 0x2b200)
+#define IPU7_IS_MAIN_GDA_BASE			(IPU7_IS_MAIN_BASE + 0x32000)
+#define IPU7_IS_MAIN_GPREGS_MAIN_BASE		(IPU7_IS_MAIN_BASE + 0x32500)
+#define IPU7_IS_MAIN_IRQ_CTRL_BASE		(IPU7_IS_MAIN_BASE + 0x32700)
+#define IPU7_IS_MAIN_PWM_CTRL_BASE		(IPU7_IS_MAIN_BASE + 0x32b00)
+
+#define IPU7_S2B_IRQ_COMMON_0_CTL_STATUS	(IPU7_IS_MAIN_S2B_BASE + 0x1c)
+#define IPU7_S2B_IRQ_COMMON_0_CTL_CLEAR		(IPU7_IS_MAIN_S2B_BASE + 0x20)
+#define IPU7_S2B_IRQ_COMMON_0_CTL_ENABLE	(IPU7_IS_MAIN_S2B_BASE + 0x24)
+#define IPU7_S2B_IID_IRQ_CTL_STATUS(iid)	(IPU7_IS_MAIN_S2B_BASE + 0x94 + \
+						 0x100 * (iid))
+
+#define IPU7_B2O_IRQ_COMMON_0_CTL_STATUS	(IPU7_IS_MAIN_B2O_BASE + 0x30)
+#define IPU7_B2O_IRQ_COMMON_0_CTL_CLEAR		(IPU7_IS_MAIN_B2O_BASE + 0x34)
+#define IPU7_B2O_IRQ_COMMON_0_CTL_ENABLE	(IPU7_IS_MAIN_B2O_BASE + 0x38)
+#define IPU7_B2O_IID_IRQ_CTL_STATUS(oid)	(IPU7_IS_MAIN_B2O_BASE + 0x3dc + \
+						 0x200 * (oid))
+
+#define IPU7_ISD_M0_IRQ_CTL_STATUS		(IPU7_IS_MAIN_ISD_M0_BASE + 0x1c)
+#define IPU7_ISD_M0_IRQ_CTL_CLEAR		(IPU7_IS_MAIN_ISD_M0_BASE + 0x20)
+#define IPU7_ISD_M0_IRQ_CTL_ENABLE		(IPU7_IS_MAIN_ISD_M0_BASE + 0x24)
+
+#define IPU7_ISD_M1_IRQ_CTL_STATUS		(IPU7_IS_MAIN_ISD_M1_BASE + 0x1c)
+#define IPU7_ISD_M1_IRQ_CTL_CLEAR		(IPU7_IS_MAIN_ISD_M1_BASE + 0x20)
+#define IPU7_ISD_M1_IRQ_CTL_ENABLE		(IPU7_IS_MAIN_ISD_M1_BASE + 0x24)
+
+#define IPU7_ISD_INT_IRQ_CTL_STATUS		(IPU7_IS_MAIN_ISD_INT_BASE + 0x1c)
+#define IPU7_ISD_INT_IRQ_CTL_CLEAR		(IPU7_IS_MAIN_ISD_INT_BASE + 0x20)
+#define IPU7_ISD_INT_IRQ_CTL_ENABLE		(IPU7_IS_MAIN_ISD_INT_BASE + 0x24)
+
+#define IPU7_GDA_IRQ_CTL_STATUS			(IPU7_IS_MAIN_GDA_BASE + 0x1c)
+#define IPU7_GDA_IRQ_CTL_CLEAR			(IPU7_IS_MAIN_GDA_BASE + 0x20)
+#define IPU7_GDA_IRQ_CTL_ENABLE			(IPU7_IS_MAIN_GDA_BASE + 0x24)
+
+#define IPU7_IS_MAIN_IRQ_CTL_EDGE		IPU7_IS_MAIN_IRQ_CTRL_BASE
+#define IPU7_IS_MAIN_IRQ_CTL_MASK		(IPU7_IS_MAIN_IRQ_CTRL_BASE + 0x4)
+#define IPU7_IS_MAIN_IRQ_CTL_STATUS		(IPU7_IS_MAIN_IRQ_CTRL_BASE + 0x8)
+#define IPU7_IS_MAIN_IRQ_CTL_CLEAR		(IPU7_IS_MAIN_IRQ_CTRL_BASE + 0xc)
+#define IPU7_IS_MAIN_IRQ_CTL_ENABLE		(IPU7_IS_MAIN_IRQ_CTRL_BASE + 0x10)
+#define IPU7_IS_MAIN_IRQ_CTL_LEVEL_NOT_PULSE	(IPU7_IS_MAIN_IRQ_CTRL_BASE + 0x14)
+
+/* IS IO regs base */
+#define IPU7_IS_PHY_NUM				4U
+#define IPU7_IS_IO_BASE				0x280000
+
+/* dwc csi cdphy registers */
+#define IPU7_IS_IO_CDPHY_BASE(i)			(IPU7_IS_IO_BASE + 0x10000 * (i))
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_0		0x1800
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_1		0x1802
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_2		0x1804
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_3		0x1806
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_4		0x1808
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_5		0x180a
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_6		0x180c
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_7		0x180e
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_8		0x1810
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_9		0x1812
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_A		0x1814
+#define IPU7_PPI_STARTUP_RW_COMMON_DPHY_10		0x1820
+#define IPU7_PPI_STARTUP_RW_COMMON_STARTUP_1_1		0x1822
+#define IPU7_PPI_STARTUP_RW_COMMON_STARTUP_1_2		0x1824
+#define IPU7_PPI_CALIBCTRL_RW_COMMON_CALIBCTRL_2_0	0x1840
+#define IPU7_PPI_CALIBCTRL_R_COMMON_CALIBCTRL_2_1	0x1842
+#define IPU7_PPI_CALIBCTRL_R_COMMON_CALIBCTRL_2_2	0x1844
+#define IPU7_PPI_CALIBCTRL_R_COMMON_CALIBCTRL_2_3	0x1846
+#define IPU7_PPI_CALIBCTRL_R_COMMON_CALIBCTRL_2_4	0x1848
+#define IPU7_PPI_CALIBCTRL_R_COMMON_CALIBCTRL_2_5	0x184a
+#define IPU7_PPI_CALIBCTRL_RW_COMMON_BG_0		0x184c
+#define IPU7_PPI_CALIBCTRL_RW_COMMON_CALIBCTRL_2_7	0x184e
+#define IPU7_PPI_CALIBCTRL_RW_ADC_CFG_0			0x1850
+#define IPU7_PPI_CALIBCTRL_RW_ADC_CFG_1			0x1852
+#define IPU7_PPI_CALIBCTRL_R_ADC_DEBUG			0x1854
+#define IPU7_PPI_RW_LPDCOCAL_TOP_OVERRIDE		0x1c00
+#define IPU7_PPI_RW_LPDCOCAL_TIMEBASE			0x1c02
+#define IPU7_PPI_RW_LPDCOCAL_NREF			0x1c04
+#define IPU7_PPI_RW_LPDCOCAL_NREF_RANGE			0x1c06
+#define IPU7_PPI_RW_LPDCOCAL_NREF_TRIGGER_MAN		0x1c08
+#define IPU7_PPI_RW_LPDCOCAL_TWAIT_CONFIG		0x1c0a
+#define IPU7_PPI_RW_LPDCOCAL_VT_CONFIG			0x1c0c
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_RB			0x1c0e
+#define IPU7_PPI_RW_LPDCOCAL_COARSE_CFG			0x1c10
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_COARSE_RB		0x1c12
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_COARSE_MEAS_0_RB	0x1c14
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_COARSE_MEAS_1_RB	0x1c16
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_COARSE_FWORD_RB	0x1c18
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_MEASURE_CURR_ERROR	0x1c1a
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_MEASURE_LAST_ERROR	0x1c1c
+#define IPU7_PPI_R_LPDCOCAL_DEBUG_VT			0x1c1e
+#define IPU7_PPI_RW_LB_TIMEBASE_CONFIG			0x1c20
+#define IPU7_PPI_RW_LB_STARTCMU_CONFIG			0x1c22
+#define IPU7_PPI_R_LBPULSE_COUNTER_RB			0x1c24
+#define IPU7_PPI_R_LB_START_CMU_RB			0x1c26
+#define IPU7_PPI_RW_LB_DPHY_BURST_START			0x1c28
+#define IPU7_PPI_RW_LB_CPHY_BURST_START			0x1c2a
+#define IPU7_PPI_RW_DDLCAL_CFG_0			0x1c40
+#define IPU7_PPI_RW_DDLCAL_CFG_1			0x1c42
+#define IPU7_PPI_RW_DDLCAL_CFG_2			0x1c44
+#define IPU7_PPI_RW_DDLCAL_CFG_3			0x1c46
+#define IPU7_PPI_RW_DDLCAL_CFG_4			0x1c48
+#define IPU7_PPI_RW_DDLCAL_CFG_5			0x1c4a
+#define IPU7_PPI_RW_DDLCAL_CFG_6			0x1c4c
+#define IPU7_PPI_RW_DDLCAL_CFG_7			0x1c4e
+#define IPU7_PPI_R_DDLCAL_DEBUG_0			0x1c50
+#define IPU7_PPI_R_DDLCAL_DEBUG_1			0x1c52
+#define IPU7_PPI_RW_PARITY_TEST				0x1c60
+#define IPU7_PPI_RW_STARTUP_OVR_0			0x1c62
+#define IPU7_PPI_RW_STARTUP_STATE_OVR_1			0x1c64
+#define IPU7_PPI_RW_DTB_SELECTOR			0x1c66
+#define IPU7_PPI_RW_DPHY_CLK_SPARE			0x1c6a
+#define IPU7_PPI_RW_COMMON_CFG				0x1c6c
+#define IPU7_PPI_RW_TERMCAL_CFG_0			0x1c80
+#define IPU7_PPI_R_TERMCAL_DEBUG_0			0x1c82
+#define IPU7_PPI_RW_TERMCAL_CTRL_0			0x1c84
+#define IPU7_PPI_RW_OFFSETCAL_CFG_0			0x1ca0
+#define IPU7_PPI_R_OFFSETCAL_DEBUG_LANE0		0x1ca2
+#define IPU7_PPI_R_OFFSETCAL_DEBUG_LANE1		0x1ca4
+#define IPU7_PPI_R_OFFSETCAL_DEBUG_LANE2		0x1ca6
+#define IPU7_PPI_R_OFFSETCAL_DEBUG_LANE3		0x1ca8
+#define IPU7_PPI_R_OFFSETCAL_DEBUG_LANE4		0x1caa
+#define IPU7_PPI_RW_HSDCOCAL_CFG_O			0x1d00
+#define IPU7_PPI_RW_HSDCOCAL_CFG_1			0x1d02
+#define IPU7_PPI_RW_HSDCOCAL_CFG_2			0x1d04
+#define IPU7_PPI_RW_HSDCOCAL_CFG_3			0x1d06
+#define IPU7_PPI_RW_HSDCOCAL_CFG_4			0x1d08
+#define IPU7_PPI_RW_HSDCOCAL_CFG_5			0x1d0a
+#define IPU7_PPI_RW_HSDCOCAL_CFG_6			0x1d0c
+#define IPU7_PPI_RW_HSDCOCAL_CFG_7			0x1d0e
+#define IPU7_PPI_RW_HSDCOCAL_CFG_8			0x1d10
+#define IPU7_PPI_R_HSDCOCAL_DEBUG_RB			0x1d12
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_0	0x2000
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_1	0x2002
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_2	0x2004
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_3	0x2006
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_4	0x2008
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_5	0x200a
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_6	0x200c
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_7	0x200e
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE0_OVR_0_8	0x2010
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE0_OVR_0_9	0x2012
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE0_OVR_0_10	0x2014
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE0_OVR_0_11	0x2016
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE0_OVR_0_12	0x2018
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE0_OVR_0_13	0x201a
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE0_OVR_0_14	0x201c
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE0_OVR_0_15	0x201e
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_0	0x2020
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_1	0x2022
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_2	0x2024
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_3	0x2026
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_4	0x2028
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_5	0x202a
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_6	0x202c
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_7	0x202e
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_8	0x2030
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_1_9	0x2032
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_1_10	0x2034
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_1_11	0x2036
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_1_12	0x2038
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_1_13	0x203a
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_1_14	0x203c
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_1_15	0x203e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_0	0x2040
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_1	0x2042
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_2	0x2044
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_3	0x2046
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_4	0x2048
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_5	0x204a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_6	0x204c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_7	0x204e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_8	0x2050
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_9	0x2052
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_10	0x2054
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_11	0x2056
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_12	0x2058
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_13	0x205a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_14	0x205c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_2_15	0x205e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_0	0x2060
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_1	0x2062
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_2	0x2064
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_3	0x2066
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_4	0x2068
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_5	0x206a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_6	0x206c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE0_CTRL_3_7	0x206e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_8	0x2070
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_9	0x2072
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_10	0x2074
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_11	0x2076
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_12	0x2078
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_13	0x207a
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_14	0x207c
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_3_15	0x207e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_4_0	0x2080
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_4_1	0x2082
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_4_2	0x2084
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_4_3	0x2086
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE0_CTRL_4_4	0x2088
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_5_0	0x20a0
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_5_1	0x20a2
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_5_2	0x20a4
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE0_OVR_5_3	0x20a6
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE0_OVR_5_4	0x20a8
+#define IPU7_CORE_DIG_RW_TRIO0_0			0x2100
+#define IPU7_CORE_DIG_RW_TRIO0_1			0x2102
+#define IPU7_CORE_DIG_RW_TRIO0_2			0x2104
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_0	0x2400
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_1	0x2402
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_2	0x2404
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_3	0x2406
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_4	0x2408
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_5	0x240a
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_6	0x240c
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_7	0x240e
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE1_OVR_0_8	0x2410
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE1_OVR_0_9	0x2412
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE1_OVR_0_10	0x2414
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE1_OVR_0_11	0x2416
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE1_OVR_0_12	0x2418
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE1_OVR_0_13	0x241a
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE1_OVR_0_14	0x241c
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE1_OVR_0_15	0x241e
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_0	0x2420
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_1	0x2422
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_2	0x2424
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_3	0x2426
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_4	0x2428
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_5	0x242a
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_6	0x242c
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_7	0x242e
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_8	0x2430
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_1_9	0x2432
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_1_10	0x2434
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_1_11	0x2436
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_1_12	0x2438
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_1_13	0x243a
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_1_14	0x243c
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_1_15	0x243e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_0	0x2440
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_1	0x2442
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_2	0x2444
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_3	0x2446
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_4	0x2448
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_5	0x244a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_6	0x244c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_7	0x244e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_8	0x2450
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_9	0x2452
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_10	0x2454
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_11	0x2456
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_12	0x2458
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_13	0x245a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_14	0x245c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_2_15	0x245e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_0	0x2460
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_1	0x2462
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_2	0x2464
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_3	0x2466
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_4	0x2468
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_5	0x246a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_6	0x246c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE1_CTRL_3_7	0x246e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_8	0x2470
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_9	0x2472
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_10	0x2474
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_11	0x2476
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_12	0x2478
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_13	0x247a
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_14	0x247c
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_3_15	0x247e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_4_0	0x2480
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_4_1	0x2482
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_4_2	0x2484
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_4_3	0x2486
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE1_CTRL_4_4	0x2488
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_5_0	0x24a0
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_5_1	0x24a2
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_5_2	0x24a4
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE1_OVR_5_3	0x24a6
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE1_OVR_5_4	0x24a8
+#define IPU7_CORE_DIG_RW_TRIO1_0			0x2500
+#define IPU7_CORE_DIG_RW_TRIO1_1			0x2502
+#define IPU7_CORE_DIG_RW_TRIO1_2			0x2504
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_0	0x2800
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_1	0x2802
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_2	0x2804
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_3	0x2806
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_4	0x2808
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_5	0x280a
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_6	0x280c
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_7	0x280e
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE2_OVR_0_8	0x2810
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE2_OVR_0_9	0x2812
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE2_OVR_0_10	0x2814
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE2_OVR_0_11	0x2816
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE2_OVR_0_12	0x2818
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE2_OVR_0_13	0x281a
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE2_OVR_0_14	0x281c
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE2_OVR_0_15	0x281e
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_0	0x2820
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_1	0x2822
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_2	0x2824
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_3	0x2826
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_4	0x2828
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_5	0x282a
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_6	0x282c
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_7	0x282e
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_8	0x2830
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_1_9	0x2832
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_1_10	0x2834
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_1_11	0x2836
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_1_12	0x2838
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_1_13	0x283a
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_1_14	0x283c
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_1_15	0x283e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_0	0x2840
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_1	0x2842
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_2	0x2844
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_3	0x2846
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_4	0x2848
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_5	0x284a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_6	0x284c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_7	0x284e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_8	0x2850
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_9	0x2852
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_10	0x2854
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_11	0x2856
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_12	0x2858
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_13	0x285a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_14	0x285c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_2_15	0x285e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_0	0x2860
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_1	0x2862
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_2	0x2864
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_3	0x2866
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_4	0x2868
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_5	0x286a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_6	0x286c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE2_CTRL_3_7	0x286e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_8	0x2870
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_9	0x2872
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_10	0x2874
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_11	0x2876
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_12	0x2878
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_13	0x287a
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_14	0x287c
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_3_15	0x287e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_4_0	0x2880
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_4_1	0x2882
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_4_2	0x2884
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_4_3	0x2886
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE2_CTRL_4_4	0x2888
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_5_0	0x28a0
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_5_1	0x28a2
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_5_2	0x28a4
+#define IPU7_CORE_DIG_IOCTRL_RW_CPHY_PPI_LANE2_OVR_5_3	0x28a6
+#define IPU7_CORE_DIG_IOCTRL_R_CPHY_PPI_LANE2_OVR_5_4	0x28a8
+#define IPU7_CORE_DIG_RW_TRIO2_0			0x2900
+#define IPU7_CORE_DIG_RW_TRIO2_1			0x2902
+#define IPU7_CORE_DIG_RW_TRIO2_2			0x2904
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_0	0x2c00
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_1	0x2c02
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_2	0x2c04
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_3	0x2c06
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_4	0x2c08
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_5	0x2c0a
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_6	0x2c0c
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_7	0x2c0e
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_LANE3_OVR_0_8	0x2c10
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE3_OVR_0_9	0x2c12
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE3_OVR_0_10	0x2c14
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE3_OVR_0_11	0x2c16
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE3_OVR_0_12	0x2c18
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE3_OVR_0_13	0x2c1a
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE3_OVR_0_14	0x2c1c
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_LANE3_OVR_0_15	0x2c1e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_0	0x2c40
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_1	0x2c42
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_2	0x2c44
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_3	0x2c46
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_4	0x2c48
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_5	0x2c4a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_6	0x2c4c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_7	0x2c4e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_8	0x2c50
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_9	0x2c52
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_10	0x2c54
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_11	0x2c56
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_12	0x2c58
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_13	0x2c5a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_14	0x2c5c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_2_15	0x2c5e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_0	0x2c60
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_1	0x2c62
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_2	0x2c64
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_3	0x2c66
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_4	0x2c68
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_5	0x2c6a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_6	0x2c6c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE3_CTRL_3_7	0x2c6e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_8	0x2c70
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_9	0x2c72
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_10	0x2c74
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_11	0x2c76
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_12	0x2c78
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_13	0x2c7a
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_14	0x2c7c
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_3_15	0x2c7e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_4_0	0x2c80
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_4_1	0x2c82
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_4_2	0x2c84
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_4_3	0x2c86
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE3_CTRL_4_4	0x2c88
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_0	0x3040
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_1	0x3042
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_2	0x3044
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_3	0x3046
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_4	0x3048
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_5	0x304a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_6	0x304c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_7	0x304e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_8	0x3050
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_9	0x3052
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_10	0x3054
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_11	0x3056
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_12	0x3058
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_13	0x305a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_14	0x305c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_2_15	0x305e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_0	0x3060
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_1	0x3062
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_2	0x3064
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_3	0x3066
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_4	0x3068
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_5	0x306a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_6	0x306c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_LANE4_CTRL_3_7	0x306e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_8	0x3070
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_9	0x3072
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_10	0x3074
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_11	0x3076
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_12	0x3078
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_13	0x307a
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_14	0x307c
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_3_15	0x307e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_4_0	0x3080
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_4_1	0x3082
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_4_2	0x3084
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_4_3	0x3086
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_LANE4_CTRL_4_4	0x3088
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_CLK_OVR_0_0	0x3400
+#define IPU7_CORE_DIG_IOCTRL_RW_DPHY_PPI_CLK_OVR_0_1	0x3402
+#define IPU7_CORE_DIG_IOCTRL_R_DPHY_PPI_CLK_OVR_0_2	0x3404
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_0	0x3800
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_1	0x3802
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_2	0x3804
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_3	0x3806
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_4	0x3808
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_5	0x380a
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_6	0x380c
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_7	0x380e
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_8	0x3810
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_9	0x3812
+#define IPU7_CORE_DIG_IOCTRL_RW_COMMON_PPI_OVR_0_10	0x3814
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_0_11	0x3816
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_0_12	0x3818
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_0_13	0x381a
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_0_14	0x381c
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_0_15	0x381e
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_1_0	0x3820
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_1_1	0x3822
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_1_2	0x3824
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_1_3	0x3826
+#define IPU7_CORE_DIG_IOCTRL_R_COMMON_PPI_OVR_1_4	0x3828
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_0		0x3840
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_1		0x3842
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_2		0x3844
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_3		0x3846
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_4		0x3848
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_5		0x384a
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_6		0x384c
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_7		0x384e
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_8		0x3850
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_9		0x3852
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_10	0x3854
+#define IPU7_CORE_DIG_IOCTRL_RW_AFE_CB_CTRL_2_11	0x3856
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_CB_CTRL_2_12		0x3858
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_CB_CTRL_2_13		0x385a
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_CB_CTRL_2_14		0x385c
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_CB_CTRL_2_15		0x385e
+#define IPU7_CORE_DIG_IOCTRL_R_AFE_CB_CTRL_3_0		0x3860
+#define IPU7_CORE_DIG_RW_COMMON_0			0x3880
+#define IPU7_CORE_DIG_RW_COMMON_1			0x3882
+#define IPU7_CORE_DIG_RW_COMMON_2			0x3884
+#define IPU7_CORE_DIG_RW_COMMON_3			0x3886
+#define IPU7_CORE_DIG_RW_COMMON_4			0x3888
+#define IPU7_CORE_DIG_RW_COMMON_5			0x388a
+#define IPU7_CORE_DIG_RW_COMMON_6			0x388c
+#define IPU7_CORE_DIG_RW_COMMON_7			0x388e
+#define IPU7_CORE_DIG_RW_COMMON_8			0x3890
+#define IPU7_CORE_DIG_RW_COMMON_9			0x3892
+#define IPU7_CORE_DIG_RW_COMMON_10			0x3894
+#define IPU7_CORE_DIG_RW_COMMON_11			0x3896
+#define IPU7_CORE_DIG_RW_COMMON_12			0x3898
+#define IPU7_CORE_DIG_RW_COMMON_13			0x389a
+#define IPU7_CORE_DIG_RW_COMMON_14			0x389c
+#define IPU7_CORE_DIG_RW_COMMON_15			0x389e
+#define IPU7_CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_0	0x39e0
+#define IPU7_CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_1	0x39e2
+#define IPU7_CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_2	0x39e4
+#define IPU7_CORE_DIG_ANACTRL_RW_COMMON_ANACTRL_3	0x39e6
+#define IPU7_CORE_DIG_COMMON_RW_DESKEW_FINE_MEM		0x3fe0
+#define IPU7_CORE_DIG_COMMON_R_DESKEW_FINE_MEM		0x3fe2
+#define IPU7_PPI_RW_DPHY_LANE0_LBERT_0			0x4000
+#define IPU7_PPI_RW_DPHY_LANE0_LBERT_1			0x4002
+#define IPU7_PPI_R_DPHY_LANE0_LBERT_0			0x4004
+#define IPU7_PPI_R_DPHY_LANE0_LBERT_1			0x4006
+#define IPU7_PPI_RW_DPHY_LANE0_SPARE			0x4008
+#define IPU7_PPI_RW_DPHY_LANE1_LBERT_0			0x4400
+#define IPU7_PPI_RW_DPHY_LANE1_LBERT_1			0x4402
+#define IPU7_PPI_R_DPHY_LANE1_LBERT_0			0x4404
+#define IPU7_PPI_R_DPHY_LANE1_LBERT_1			0x4406
+#define IPU7_PPI_RW_DPHY_LANE1_SPARE			0x4408
+#define IPU7_PPI_RW_DPHY_LANE2_LBERT_0			0x4800
+#define IPU7_PPI_RW_DPHY_LANE2_LBERT_1			0x4802
+#define IPU7_PPI_R_DPHY_LANE2_LBERT_0			0x4804
+#define IPU7_PPI_R_DPHY_LANE2_LBERT_1			0x4806
+#define IPU7_PPI_RW_DPHY_LANE2_SPARE			0x4808
+#define IPU7_PPI_RW_DPHY_LANE3_LBERT_0			0x4c00
+#define IPU7_PPI_RW_DPHY_LANE3_LBERT_1			0x4c02
+#define IPU7_PPI_R_DPHY_LANE3_LBERT_0			0x4c04
+#define IPU7_PPI_R_DPHY_LANE3_LBERT_1			0x4c06
+#define IPU7_PPI_RW_DPHY_LANE3_SPARE			0x4c08
+#define IPU7_CORE_DIG_DLANE_0_RW_CFG_0			0x6000
+#define IPU7_CORE_DIG_DLANE_0_RW_CFG_1			0x6002
+#define IPU7_CORE_DIG_DLANE_0_RW_CFG_2			0x6004
+#define IPU7_CORE_DIG_DLANE_0_RW_LP_0			0x6080
+#define IPU7_CORE_DIG_DLANE_0_RW_LP_1			0x6082
+#define IPU7_CORE_DIG_DLANE_0_RW_LP_2			0x6084
+#define IPU7_CORE_DIG_DLANE_0_R_LP_0			0x60a0
+#define IPU7_CORE_DIG_DLANE_0_R_LP_1			0x60a2
+#define IPU7_CORE_DIG_DLANE_0_R_HS_TX_0			0x60e0
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_0		0x6100
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_1		0x6102
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_2		0x6104
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_3		0x6106
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_4		0x6108
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_5		0x610a
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_6		0x610c
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_7		0x610e
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_8		0x6110
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_RX_9		0x6112
+#define IPU7_CORE_DIG_DLANE_0_R_HS_RX_0			0x6120
+#define IPU7_CORE_DIG_DLANE_0_R_HS_RX_1			0x6122
+#define IPU7_CORE_DIG_DLANE_0_R_HS_RX_2			0x6124
+#define IPU7_CORE_DIG_DLANE_0_R_HS_RX_3			0x6126
+#define IPU7_CORE_DIG_DLANE_0_R_HS_RX_4			0x6128
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_0		0x6200
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_1		0x6202
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_2		0x6204
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_3		0x6206
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_4		0x6208
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_5		0x620a
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_6		0x620c
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_7		0x620e
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_8		0x6210
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_9		0x6212
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_10		0x6214
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_11		0x6216
+#define IPU7_CORE_DIG_DLANE_0_RW_HS_TX_12		0x6218
+#define IPU7_CORE_DIG_DLANE_1_RW_CFG_0			0x6400
+#define IPU7_CORE_DIG_DLANE_1_RW_CFG_1			0x6402
+#define IPU7_CORE_DIG_DLANE_1_RW_CFG_2			0x6404
+#define IPU7_CORE_DIG_DLANE_1_RW_LP_0			0x6480
+#define IPU7_CORE_DIG_DLANE_1_RW_LP_1			0x6482
+#define IPU7_CORE_DIG_DLANE_1_RW_LP_2			0x6484
+#define IPU7_CORE_DIG_DLANE_1_R_LP_0			0x64a0
+#define IPU7_CORE_DIG_DLANE_1_R_LP_1			0x64a2
+#define IPU7_CORE_DIG_DLANE_1_R_HS_TX_0			0x64e0
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_0		0x6500
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_1		0x6502
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_2		0x6504
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_3		0x6506
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_4		0x6508
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_5		0x650a
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_6		0x650c
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_7		0x650e
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_8		0x6510
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_RX_9		0x6512
+#define IPU7_CORE_DIG_DLANE_1_R_HS_RX_0			0x6520
+#define IPU7_CORE_DIG_DLANE_1_R_HS_RX_1			0x6522
+#define IPU7_CORE_DIG_DLANE_1_R_HS_RX_2			0x6524
+#define IPU7_CORE_DIG_DLANE_1_R_HS_RX_3			0x6526
+#define IPU7_CORE_DIG_DLANE_1_R_HS_RX_4			0x6528
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_0		0x6600
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_1		0x6602
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_2		0x6604
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_3		0x6606
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_4		0x6608
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_5		0x660a
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_6		0x660c
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_7		0x660e
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_8		0x6610
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_9		0x6612
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_10		0x6614
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_11		0x6616
+#define IPU7_CORE_DIG_DLANE_1_RW_HS_TX_12		0x6618
+#define IPU7_CORE_DIG_DLANE_2_RW_CFG_0			0x6800
+#define IPU7_CORE_DIG_DLANE_2_RW_CFG_1			0x6802
+#define IPU7_CORE_DIG_DLANE_2_RW_CFG_2			0x6804
+#define IPU7_CORE_DIG_DLANE_2_RW_LP_0			0x6880
+#define IPU7_CORE_DIG_DLANE_2_RW_LP_1			0x6882
+#define IPU7_CORE_DIG_DLANE_2_RW_LP_2			0x6884
+#define IPU7_CORE_DIG_DLANE_2_R_LP_0			0x68a0
+#define IPU7_CORE_DIG_DLANE_2_R_LP_1			0x68a2
+#define IPU7_CORE_DIG_DLANE_2_R_HS_TX_0			0x68e0
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_0		0x6900
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_1		0x6902
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_2		0x6904
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_3		0x6906
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_4		0x6908
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_5		0x690a
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_6		0x690c
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_7		0x690e
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_8		0x6910
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_RX_9		0x6912
+#define IPU7_CORE_DIG_DLANE_2_R_HS_RX_0			0x6920
+#define IPU7_CORE_DIG_DLANE_2_R_HS_RX_1			0x6922
+#define IPU7_CORE_DIG_DLANE_2_R_HS_RX_2			0x6924
+#define IPU7_CORE_DIG_DLANE_2_R_HS_RX_3			0x6926
+#define IPU7_CORE_DIG_DLANE_2_R_HS_RX_4			0x6928
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_0		0x6a00
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_1		0x6a02
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_2		0x6a04
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_3		0x6a06
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_4		0x6a08
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_5		0x6a0a
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_6		0x6a0c
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_7		0x6a0e
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_8		0x6a10
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_9		0x6a12
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_10		0x6a14
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_11		0x6a16
+#define IPU7_CORE_DIG_DLANE_2_RW_HS_TX_12		0x6a18
+#define IPU7_CORE_DIG_DLANE_3_RW_CFG_0			0x6c00
+#define IPU7_CORE_DIG_DLANE_3_RW_CFG_1			0x6c02
+#define IPU7_CORE_DIG_DLANE_3_RW_CFG_2			0x6c04
+#define IPU7_CORE_DIG_DLANE_3_RW_LP_0			0x6c80
+#define IPU7_CORE_DIG_DLANE_3_RW_LP_1			0x6c82
+#define IPU7_CORE_DIG_DLANE_3_RW_LP_2			0x6c84
+#define IPU7_CORE_DIG_DLANE_3_R_LP_0			0x6ca0
+#define IPU7_CORE_DIG_DLANE_3_R_LP_1			0x6ca2
+#define IPU7_CORE_DIG_DLANE_3_R_HS_TX_0			0x6ce0
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_0		0x6d00
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_1		0x6d02
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_2		0x6d04
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_3		0x6d06
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_4		0x6d08
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_5		0x6d0a
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_6		0x6d0c
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_7		0x6d0e
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_8		0x6d10
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_RX_9		0x6d12
+#define IPU7_CORE_DIG_DLANE_3_R_HS_RX_0			0x6d20
+#define IPU7_CORE_DIG_DLANE_3_R_HS_RX_1			0x6d22
+#define IPU7_CORE_DIG_DLANE_3_R_HS_RX_2			0x6d24
+#define IPU7_CORE_DIG_DLANE_3_R_HS_RX_3			0x6d26
+#define IPU7_CORE_DIG_DLANE_3_R_HS_RX_4			0x6d28
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_0		0x6e00
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_1		0x6e02
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_2		0x6e04
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_3		0x6e06
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_4		0x6e08
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_5		0x6e0a
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_6		0x6e0c
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_7		0x6e0e
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_8		0x6e10
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_9		0x6e12
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_10		0x6e14
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_11		0x6e16
+#define IPU7_CORE_DIG_DLANE_3_RW_HS_TX_12		0x6e18
+#define IPU7_CORE_DIG_DLANE_CLK_RW_CFG_0		0x7000
+#define IPU7_CORE_DIG_DLANE_CLK_RW_CFG_1		0x7002
+#define IPU7_CORE_DIG_DLANE_CLK_RW_CFG_2		0x7004
+#define IPU7_CORE_DIG_DLANE_CLK_RW_LP_0			0x7080
+#define IPU7_CORE_DIG_DLANE_CLK_RW_LP_1			0x7082
+#define IPU7_CORE_DIG_DLANE_CLK_RW_LP_2			0x7084
+#define IPU7_CORE_DIG_DLANE_CLK_R_LP_0			0x70a0
+#define IPU7_CORE_DIG_DLANE_CLK_R_LP_1			0x70a2
+#define IPU7_CORE_DIG_DLANE_CLK_R_HS_TX_0		0x70e0
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_0		0x7100
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_1		0x7102
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_2		0x7104
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_3		0x7106
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_4		0x7108
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_5		0x710a
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_6		0x710c
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_7		0x710e
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_8		0x7110
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_RX_9		0x7112
+#define IPU7_CORE_DIG_DLANE_CLK_R_HS_RX_0		0x7120
+#define IPU7_CORE_DIG_DLANE_CLK_R_HS_RX_1		0x7122
+#define IPU7_CORE_DIG_DLANE_CLK_R_HS_RX_2		0x7124
+#define IPU7_CORE_DIG_DLANE_CLK_R_HS_RX_3		0x7126
+#define IPU7_CORE_DIG_DLANE_CLK_R_HS_RX_4		0x7128
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_0		0x7200
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_1		0x7202
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_2		0x7204
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_3		0x7206
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_4		0x7208
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_5		0x720a
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_6		0x720c
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_7		0x720e
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_8		0x7210
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_9		0x7212
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_10		0x7214
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_11		0x7216
+#define IPU7_CORE_DIG_DLANE_CLK_RW_HS_TX_12		0x7218
+#define IPU7_PPI_RW_CPHY_TRIO0_LBERT_0			0x8000
+#define IPU7_PPI_RW_CPHY_TRIO0_LBERT_1			0x8002
+#define IPU7_PPI_R_CPHY_TRIO0_LBERT_0			0x8004
+#define IPU7_PPI_R_CPHY_TRIO0_LBERT_1			0x8006
+#define IPU7_PPI_RW_CPHY_TRIO0_SPARE			0x8008
+#define IPU7_PPI_RW_CPHY_TRIO1_LBERT_0			0x8400
+#define IPU7_PPI_RW_CPHY_TRIO1_LBERT_1			0x8402
+#define IPU7_PPI_R_CPHY_TRIO1_LBERT_0			0x8404
+#define IPU7_PPI_R_CPHY_TRIO1_LBERT_1			0x8406
+#define IPU7_PPI_RW_CPHY_TRIO1_SPARE			0x8408
+#define IPU7_PPI_RW_CPHY_TRIO2_LBERT_0			0x8800
+#define IPU7_PPI_RW_CPHY_TRIO2_LBERT_1			0x8802
+#define IPU7_PPI_R_CPHY_TRIO2_LBERT_0			0x8804
+#define IPU7_PPI_R_CPHY_TRIO2_LBERT_1			0x8806
+#define IPU7_PPI_RW_CPHY_TRIO2_SPARE			0x8808
+#define IPU7_CORE_DIG_CLANE_0_RW_CFG_0			0xa000
+#define IPU7_CORE_DIG_CLANE_0_RW_CFG_2			0xa004
+#define IPU7_CORE_DIG_CLANE_0_RW_LP_0			0xa080
+#define IPU7_CORE_DIG_CLANE_0_RW_LP_1			0xa082
+#define IPU7_CORE_DIG_CLANE_0_RW_LP_2			0xa084
+#define IPU7_CORE_DIG_CLANE_0_R_LP_0			0xa0a0
+#define IPU7_CORE_DIG_CLANE_0_R_LP_1			0xa0a2
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_RX_0		0xa100
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_RX_1		0xa102
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_RX_2		0xa104
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_RX_3		0xa106
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_RX_4		0xa108
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_RX_5		0xa10a
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_RX_6		0xa10c
+#define IPU7_CORE_DIG_CLANE_0_R_RX_0			0xa120
+#define IPU7_CORE_DIG_CLANE_0_R_RX_1			0xa122
+#define IPU7_CORE_DIG_CLANE_0_R_TX_0			0xa124
+#define IPU7_CORE_DIG_CLANE_0_R_RX_2			0xa126
+#define IPU7_CORE_DIG_CLANE_0_R_RX_3			0xa128
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_0		0xa200
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_1		0xa202
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_2		0xa204
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_3		0xa206
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_4		0xa208
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_5		0xa20a
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_6		0xa20c
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_7		0xa20e
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_8		0xa210
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_9		0xa212
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_10		0xa214
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_11		0xa216
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_12		0xa218
+#define IPU7_CORE_DIG_CLANE_0_RW_HS_TX_13		0xa21a
+#define IPU7_CORE_DIG_CLANE_1_RW_CFG_0			0xa400
+#define IPU7_CORE_DIG_CLANE_1_RW_CFG_2			0xa404
+#define IPU7_CORE_DIG_CLANE_1_RW_LP_0			0xa480
+#define IPU7_CORE_DIG_CLANE_1_RW_LP_1			0xa482
+#define IPU7_CORE_DIG_CLANE_1_RW_LP_2			0xa484
+#define IPU7_CORE_DIG_CLANE_1_R_LP_0			0xa4a0
+#define IPU7_CORE_DIG_CLANE_1_R_LP_1			0xa4a2
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_RX_0		0xa500
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_RX_1		0xa502
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_RX_2		0xa504
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_RX_3		0xa506
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_RX_4		0xa508
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_RX_5		0xa50a
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_RX_6		0xa50c
+#define IPU7_CORE_DIG_CLANE_1_R_RX_0			0xa520
+#define IPU7_CORE_DIG_CLANE_1_R_RX_1			0xa522
+#define IPU7_CORE_DIG_CLANE_1_R_TX_0			0xa524
+#define IPU7_CORE_DIG_CLANE_1_R_RX_2			0xa526
+#define IPU7_CORE_DIG_CLANE_1_R_RX_3			0xa528
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_0		0xa600
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_1		0xa602
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_2		0xa604
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_3		0xa606
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_4		0xa608
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_5		0xa60a
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_6		0xa60c
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_7		0xa60e
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_8		0xa610
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_9		0xa612
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_10		0xa614
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_11		0xa616
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_12		0xa618
+#define IPU7_CORE_DIG_CLANE_1_RW_HS_TX_13		0xa61a
+#define IPU7_CORE_DIG_CLANE_2_RW_CFG_0			0xa800
+#define IPU7_CORE_DIG_CLANE_2_RW_CFG_2			0xa804
+#define IPU7_CORE_DIG_CLANE_2_RW_LP_0			0xa880
+#define IPU7_CORE_DIG_CLANE_2_RW_LP_1			0xa882
+#define IPU7_CORE_DIG_CLANE_2_RW_LP_2			0xa884
+#define IPU7_CORE_DIG_CLANE_2_R_LP_0			0xa8a0
+#define IPU7_CORE_DIG_CLANE_2_R_LP_1			0xa8a2
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_RX_0		0xa900
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_RX_1		0xa902
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_RX_2		0xa904
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_RX_3		0xa906
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_RX_4		0xa908
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_RX_5		0xa90a
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_RX_6		0xa90c
+#define IPU7_CORE_DIG_CLANE_2_R_RX_0			0xa920
+#define IPU7_CORE_DIG_CLANE_2_R_RX_1			0xa922
+#define IPU7_CORE_DIG_CLANE_2_R_TX_0			0xa924
+#define IPU7_CORE_DIG_CLANE_2_R_RX_2			0xa926
+#define IPU7_CORE_DIG_CLANE_2_R_RX_3			0xa928
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_0		0xaa00
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_1		0xaa02
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_2		0xaa04
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_3		0xaa06
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_4		0xaa08
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_5		0xaa0a
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_6		0xaa0c
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_7		0xaa0e
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_8		0xaa10
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_9		0xaa12
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_10		0xaa14
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_11		0xaa16
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_12		0xaa18
+#define IPU7_CORE_DIG_CLANE_2_RW_HS_TX_13		0xaa1a
+
+/* dwc csi host controller registers */
+#define IPU7_IS_IO_CSI2_HOST_BASE(i)		(IPU7_IS_IO_BASE + 0x40000 + \
+						 0x2000 * (i))
+#define IPU7_VERSION				0x0
+#define IPU7_N_LANES				0x4
+#define IPU7_CSI2_RESETN			0x8
+#define IPU7_INT_ST_MAIN			0xc
+#define IPU7_DATA_IDS_1				0x10
+#define IPU7_DATA_IDS_2				0x14
+#define IPU7_CDPHY_MODE				0x1c
+#define IPU7_DATA_IDS_VC_1			0x30
+#define IPU7_DATA_IDS_VC_2			0x34
+#define IPU7_PHY_SHUTDOWNZ			0x40
+#define IPU7_DPHY_RSTZ				0x44
+#define IPU7_PHY_RX				0x48
+#define IPU7_PHY_STOPSTATE			0x4c
+#define IPU7_PHY_TEST_CTRL0			0x50
+#define IPU7_PHY_TEST_CTRL1			0x54
+#define IPU7_PPI_PG_PATTERN_VRES		0x60
+#define IPU7_PPI_PG_PATTERN_HRES		0x64
+#define IPU7_PPI_PG_CONFIG			0x68
+#define IPU7_PPI_PG_ENABLE			0x6c
+#define IPU7_PPI_PG_STATUS			0x70
+#define IPU7_VC_EXTENSION			0xc8
+#define IPU7_PHY_CAL				0xcc
+#define IPU7_INT_ST_PHY_FATAL			0xe0
+#define IPU7_INT_MSK_PHY_FATAL			0xe4
+#define IPU7_INT_FORCE_PHY_FATAL		0xe8
+#define IPU7_INT_ST_PKT_FATAL			0xf0
+#define IPU7_INT_MSK_PKT_FATAL			0xf4
+#define IPU7_INT_FORCE_PKT_FATAL		0xf8
+#define IPU7_INT_ST_PHY				0x110
+#define IPU7_INT_MSK_PHY			0x114
+#define IPU7_INT_FORCE_PHY			0x118
+#define IPU7_INT_ST_LINE			0x130
+#define IPU7_INT_MSK_LINE			0x134
+#define IPU7_INT_FORCE_LINE			0x138
+#define IPU7_INT_ST_BNDRY_FRAME_FATAL		0x280
+#define IPU7_INT_MSK_BNDRY_FRAME_FATAL		0x284
+#define IPU7_INT_FORCE_BNDRY_FRAME_FATAL	0x288
+#define IPU7_INT_ST_SEQ_FRAME_FATAL		0x290
+#define IPU7_INT_MSK_SEQ_FRAME_FATAL		0x294
+#define IPU7_INT_FORCE_SEQ_FRAME_FATAL		0x298
+#define IPU7_INT_ST_CRC_FRAME_FATAL		0x2a0
+#define IPU7_INT_MSK_CRC_FRAME_FATAL		0x2a4
+#define IPU7_INT_FORCE_CRC_FRAME_FATAL		0x2a8
+#define IPU7_INT_ST_PLD_CRC_FATAL		0x2b0
+#define IPU7_INT_MSK_PLD_CRC_FATAL		0x2b4
+#define IPU7_INT_FORCE_PLD_CRC_FATAL		0x2b8
+#define IPU7_INT_ST_DATA_ID			0x2c0
+#define IPU7_INT_MSK_DATA_ID			0x2c4
+#define IPU7_INT_FORCE_DATA_ID			0x2c8
+#define IPU7_INT_ST_ECC_CORRECTED		0x2d0
+#define IPU7_INT_MSK_ECC_CORRECTED		0x2d4
+#define IPU7_INT_FORCE_ECC_CORRECTED		0x2d8
+#define IPU7_SCRAMBLING				0x300
+#define IPU7_SCRAMBLING_SEED1			0x304
+#define IPU7_SCRAMBLING_SEED2			0x308
+#define IPU7_SCRAMBLING_SEED3			0x30c
+#define IPU7_SCRAMBLING_SEED4			0x310
+#define IPU7_SCRAMBLING				0x300
+
+#define IPU7_IS_IO_CSI2_ADPL_PORT_BASE(i)		(IPU7_IS_IO_BASE + 0x40800 + \
+							 0x2000 * (i))
+#define IPU7_CSI2_ADPL_INPUT_MODE			0x0
+#define IPU7_CSI2_ADPL_CSI_RX_ERR_IRQ_CLEAR_EN		0x4
+#define IPU7_CSI2_ADPL_CSI_RX_ERR_IRQ_CLEAR_ADDR	0x8
+#define IPU7_CSI2_ADPL_CSI_RX_ERR_IRQ_STATUS		0xc
+#define IPU7_CSI2_ADPL_IRQ_CTL_COMMON_STATUS		0xa4
+#define IPU7_CSI2_ADPL_IRQ_CTL_COMMON_CLEAR		0xa8
+#define IPU7_CSI2_ADPL_IRQ_CTL_COMMON_ENABLE		0xac
+#define IPU7_CSI2_ADPL_IRQ_CTL_FS_STATUS		0xbc
+#define IPU7_CSI2_ADPL_IRQ_CTL_FS_CLEAR			0xc0
+#define IPU7_CSI2_ADPL_IRQ_CTL_FS_ENABLE		0xc4
+#define IPU7_CSI2_ADPL_IRQ_CTL_FE_STATUS		0xc8
+#define IPU7_CSI2_ADPL_IRQ_CTL_FE_CLEAR			0xcc
+#define IPU7_CSI2_ADPL_IRQ_CTL_FE_ENABLE		0xd0
+
+/* software control the legacy csi irq */
+#define IPU7_IS_IO_CSI2_ERR_LEGACY_IRQ_CTL_BASE(i)	(IPU7_IS_IO_BASE + \
+							 0x40c00 + 0x2000 * (i))
+#define IPU7_IS_IO_CSI2_SYNC_LEGACY_IRQ_CTL_BASE(i)	(IPU7_IS_IO_BASE + \
+							 0x40d00 + 0x2000 * (i))
+#define IPU7_IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE		(IPU7_IS_IO_BASE + \
+							 0x49000)
+#define IPU7_IS_IO_CSI2_IRQ_CTRL_BASE			(IPU7_IS_IO_BASE + 0x4e100)
+
+#define IPU7_IRQ_CTL_EDGE				0x0
+#define IPU7_IRQ_CTL_MASK				0x4
+#define IPU7_IRQ_CTL_STATUS				0x8
+#define IPU7_IRQ_CTL_CLEAR				0xc
+#define IPU7_IRQ_CTL_ENABLE				0x10
+/* FE irq for PTL */
+#define IPU7_IRQ1_CTL_MASK				0x14
+#define IPU7_IRQ1_CTL_STATUS				0x18
+#define IPU7_IRQ1_CTL_CLEAR				0x1c
+#define IPU7_IRQ1_CTL_ENABLE				0x20
+
+/* software to set the clock gate to use the port or mgc */
+#define IPU7_IS_IO_GPREGS_BASE			(IPU7_IS_IO_BASE + 0x49200)
+#define IPU7_SRST_PORT_ARB			0x0
+#define IPU7_SRST_MGC				0x4
+#define IPU7_SRST_WIDTH_CONV			0x8
+#define IPU7_SRST_CSI_IRQ			0xc
+#define IPU7_SRST_CSI_LEGACY_IRQ		0x10
+#define IPU7_CLK_EN_TXCLKESC			0x14
+#define IPU7_CLK_DIV_FACTOR_IS_CLK		0x18
+#define IPU7_CLK_DIV_FACTOR_APB_CLK		0x1c
+#define IPU7_CSI_PORT_CLK_GATE			0x20
+#define IPU7_CSI_PORTAB_AGGREGATION		0x24
+#define IPU7_MGC_CLK_GATE			0x2c
+#define IPU7_CG_CTRL_BITS			0x3c
+#define IPU7_SPARE_RW				0xf8
+#define IPU7_SPARE_RO				0xfc
+
+#define IPU7_IS_IO_CSI2_MPF_PORT_BASE(i)	(IPU7_IS_IO_BASE + 0x53000 + \
+						 0x1000 * (i))
+#define IPU7_MPF_16_IRQ_CNTRL_STATUS		0x238
+#define IPU7_MPF_16_IRQ_CNTRL_CLEAR		0x23c
+#define IPU7_MPF_16_IRQ_CNTRL_ENABLE		0x240
+
+/* software config the phy */
+#define IPU7_IS_IO_CSI2_GPREGS_BASE		(IPU7_IS_IO_BASE + 0x53400)
+#define IPU8_IS_IO_CSI2_GPREGS_BASE		(IPU7_IS_IO_BASE + 0x40e00)
+#define IPU7_CSI_ADAPT_LAYER_SRST		0x0
+#define IPU7_MPF_SRST_RST			0x4
+#define IPU7_CSI_ERR_IRQ_CTRL_SRST		0x8
+#define IPU7_CSI_SYNC_RC_SRST			0xc
+#define IPU7_CSI_CG_CTRL_BITS			0x10
+#define IPU7_SOC_CSI2HOST_SELECT		0x14
+#define IPU7_PHY_RESET				0x18
+#define IPU7_PHY_SHUTDOWN			0x1c
+#define IPU7_PHY_MODE				0x20
+#define IPU7_PHY_READY				0x24
+#define IPU7_PHY_CLK_LANE_FORCE_CONTROL		0x28
+#define IPU7_PHY_CLK_LANE_CONTROL		0x2c
+#define IPU7_PHY_CLK_LANE_STATUS		0x30
+#define IPU7_PHY_LANE_RX_ESC_REQ		0x34
+#define IPU7_PHY_LANE_RX_ESC_DATA		0x38
+#define IPU7_PHY_LANE_TURNDISABLE		0x3c
+#define IPU7_PHY_LANE_DIRECTION			0x40
+#define IPU7_PHY_LANE_FORCE_CONTROL		0x44
+#define IPU7_PHY_LANE_CONTROL_EN		0x48
+#define IPU7_PHY_LANE_CONTROL_DATAWIDTH		0x4c
+#define IPU7_PHY_LANE_STATUS			0x50
+#define IPU7_PHY_LANE_ERR			0x54
+#define IPU7_PHY_LANE_RXALP			0x58
+#define IPU7_PHY_LANE_RXALP_NIBBLE		0x5c
+#define IPU7_PHY_PARITY_ERROR			0x60
+#define IPU7_PHY_DEBUG_REGS_CLK_GATE_EN		0x64
+#define IPU7_SPARE_RW				0xf8
+#define IPU7_SPARE_RO				0xfc
+
+/* software not touch */
+#define IPU7_PORT_ARB_BASE			(IPU7_IS_IO_BASE + 0x4e000)
+#define IPU7_PORT_ARB_IRQ_CTL_STATUS		0x4
+#define IPU7_PORT_ARB_IRQ_CTL_CLEAR		0x8
+#define IPU7_PORT_ARB_IRQ_CTL_ENABLE		0xc
+
+#define IPU7_MGC_PPC				4U
+#define IPU7_MGC_DTYPE_RAW(i)			(((i) - 8) / 2)
+#define IPU7_IS_IO_MGC_BASE			(IPU7_IS_IO_BASE + 0x48000)
+#define IPU7_MGC_KICK				0x0
+#define IPU7_MGC_ASYNC_STOP			0x4
+#define IPU7_MGC_PORT_OFFSET			0x100
+#define IPU7_MGC_CSI_PORT_MAP(i)		(0x8 + (i) * 0x4)
+#define IPU7_MGC_MG_PORT(i)			(IPU7_IS_IO_MGC_BASE + \
+						 (i) * IPU7_MGC_PORT_OFFSET)
+/* per mgc instance */
+#define IPU7_MGC_MG_CSI_ADAPT_LAYER_TYPE	0x28
+#define IPU7_MGC_MG_MODE			0x2c
+#define IPU7_MGC_MG_INIT_COUNTER		0x30
+#define IPU7_MGC_MG_MIPI_VC			0x34
+#define IPU7_MGC_MG_MIPI_DTYPES			0x38
+#define IPU7_MGC_MG_MULTI_DTYPES_MODE		0x3c
+#define IPU7_MGC_MG_NOF_FRAMES			0x40
+#define IPU7_MGC_MG_FRAME_DIM			0x44
+#define IPU7_MGC_MG_HBLANK			0x48
+#define IPU7_MGC_MG_VBLANK			0x4c
+#define IPU7_MGC_MG_TPG_MODE			0x50
+#define IPU7_MGC_MG_TPG_R0			0x54
+#define IPU7_MGC_MG_TPG_G0			0x58
+#define IPU7_MGC_MG_TPG_B0			0x5c
+#define IPU7_MGC_MG_TPG_R1			0x60
+#define IPU7_MGC_MG_TPG_G1			0x64
+#define IPU7_MGC_MG_TPG_B1			0x68
+#define IPU7_MGC_MG_TPG_FACTORS			0x6c
+#define IPU7_MGC_MG_TPG_MASKS			0x70
+#define IPU7_MGC_MG_TPG_XY_MASK			0x74
+#define IPU7_MGC_MG_TPG_TILE_DIM		0x78
+#define IPU7_MGC_MG_PRBS_LFSR_INIT_0		0x7c
+#define IPU7_MGC_MG_PRBS_LFSR_INIT_1		0x80
+#define IPU7_MGC_MG_SYNC_STOP_POINT		0x84
+#define IPU7_MGC_MG_SYNC_STOP_POINT_LOC		0x88
+#define IPU7_MGC_MG_ERR_INJECT			0x8c
+#define IPU7_MGC_MG_ERR_LOCATION		0x90
+#define IPU7_MGC_MG_DTO_SPEED_CTRL_EN		0x94
+#define IPU7_MGC_MG_DTO_SPEED_CTRL_INCR_VAL	0x98
+#define IPU7_MGC_MG_HOR_LOC_STTS		0x9c
+#define IPU7_MGC_MG_VER_LOC_STTS		0xa0
+#define IPU7_MGC_MG_FRAME_NUM_STTS		0xa4
+#define IPU7_MGC_MG_BUSY_STTS			0xa8
+#define IPU7_MGC_MG_STOPPED_STTS		0xac
+/* tile width and height in pixels for Chess board and Color palette */
+#define IPU7_MGC_TPG_TILE_WIDTH			64U
+#define IPU7_MGC_TPG_TILE_HEIGHT		64U
+
+#define IPU7_CSI_PORT_A_ADDR_OFFSET		0x0
+#define IPU7_CSI_PORT_B_ADDR_OFFSET		0x0
+#define IPU7_CSI_PORT_C_ADDR_OFFSET		0x0
+#define IPU7_CSI_PORT_D_ADDR_OFFSET		0x0
+
+/*
+ * 0 - CSI RX Port 0 interrupt;
+ * 1 - MPF Port 0 interrupt;
+ * 2 - CSI RX Port 1 interrupt;
+ * 3 - MPF Port 1 interrupt;
+ * 4 - CSI RX Port 2 interrupt;
+ * 5 - MPF Port 2 interrupt;
+ * 6 - CSI RX Port 3 interrupt;
+ * 7 - MPF Port 3 interrupt;
+ * 8 - Port ARB FIFO 0 overflow;
+ * 9 - Port ARB FIFO 1 overflow;
+ * 10 - Port ARB FIFO 2 overflow;
+ * 11 - Port ARB FIFO 3 overflow;
+ * 12 - isys_cfgnoc_err_probe_intl;
+ * 13-15 - reserved
+ */
+#define IPU7_CSI_IS_IO_IRQ_MASK		0xffff
+
+/* Adapter layer irq */
+#define IPU7_CSI_ADPL_IRQ_MASK		0xffff
+
+/* sw irq from legacy irq control
+ * legacy irq status
+ * IPU7
+ * 0 - CSI Port 0 error interrupt
+ * 1 - CSI Port 0 sync interrupt
+ * 2 - CSI Port 1 error interrupt
+ * 3 - CSI Port 1 sync interrupt
+ * 4 - CSI Port 2 error interrupt
+ * 5 - CSI Port 2 sync interrupt
+ * 6 - CSI Port 3 error interrupt
+ * 7 - CSI Port 3 sync interrupt
+ * IPU7P5
+ * 0 - CSI Port 0 error interrupt
+ * 1 - CSI Port 0 fs interrupt
+ * 2 - CSI Port 0 fe interrupt
+ * 3 - CSI Port 1 error interrupt
+ * 4 - CSI Port 1 fs interrupt
+ * 5 - CSI Port 1 fe interrupt
+ * 6 - CSI Port 2 error interrupt
+ * 7 - CSI Port 2 fs interrupt
+ * 8 - CSI Port 2 fe interrupt
+ */
+#define IPU7_CSI_RX_LEGACY_IRQ_MASK	0x1ff
+
+/* legacy error status per port
+ * 0 - Error handler FIFO full;
+ * 1 - Reserved Short Packet encoding detected;
+ * 2 - Reserved Long Packet encoding detected;
+ * 3 - Received packet is too short (fewer data words than specified in header);
+ * 4 - Received packet is too long (more data words than specified in header);
+ * 5 - Short packet discarded due to errors;
+ * 6 - Long packet discarded due to errors;
+ * 7 - CSI Combo Rx interrupt;
+ * 8 - IDI CDC FIFO overflow; remaining bits are reserved and tied to 0;
+ */
+#define IPU7_CSI_RX_ERROR_IRQ_MASK	0xfff
+
+/*
+ * 0 - VC0 frame start received
+ * 1 - VC0 frame end received
+ * 2 - VC1 frame start received
+ * 3 - VC1 frame end received
+ * 4 - VC2 frame start received
+ * 5 - VC2 frame end received
+ * 6 - VC3 frame start received
+ * 7 - VC3 frame end received
+ * 8 - VC4 frame start received
+ * 9 - VC4 frame end received
+ * 10 - VC5 frame start received
+ * 11 - VC5 frame end received
+ * 12 - VC6 frame start received
+ * 13 - VC6 frame end received
+ * 14 - VC7 frame start received
+ * 15 - VC7 frame end received
+ * 16 - VC8 frame start received
+ * 17 - VC8 frame end received
+ * 18 - VC9 frame start received
+ * 19 - VC9 frame end received
+ * 20 - VC10 frame start received
+ * 21 - VC10 frame end received
+ * 22 - VC11 frame start received
+ * 23 - VC11 frame end received
+ * 24 - VC12 frame start received
+ * 25 - VC12 frame end received
+ * 26 - VC13 frame start received
+ * 27 - VC13 frame end received
+ * 28 - VC14 frame start received
+ * 29 - VC14 frame end received
+ * 30 - VC15 frame start received
+ * 31 - VC15 frame end received
+ */
+
+#define IPU7_CSI_RX_SYNC_IRQ_MASK		0x0
+#define IPU7P5_CSI_RX_SYNC_FE_IRQ_MASK		0x0
+
+#define IPU7_CSI_RX_NUM_ERRORS_IN_IRQ		12U
+#define IPU7_CSI_RX_NUM_SYNC_IN_IRQ		32U
+
+enum IPU7_MGC_CSI_ADPL_TYPE {
+	IPU7_MGC_MAPPED_2_LANES = 0,
+	IPU7_MGC_MAPPED_4_LANES = 1,
+};
+
+enum IPU7_CSI2HOST_SELECTION {
+	IPU7_CSI2HOST_SEL_SOC = 0,
+	IPU7_CSI2HOST_SEL_CSI2HOST = 1,
+};
+
+#define IPU7_CSI_LEGACY_IRQ_MASK(port)		(0x3 << ((port) *  2))
+#define IPU7P5_CSI_LEGACY_IRQ_MASK(port)	(0x7 << ((port) *  3))
+
+#define IPU7_ISYS_LEGACY_IRQ_CSI2(port)		(0x3 << (port))
+#define IPU7P5_ISYS_LEGACY_IRQ_CSI2(port)	(0x7 << (port))
+
+/* ---------------------------------------------------------------- */
+#define IPU7_CSI_REG_BASE				0x220000
+#define IPU7_CSI_REG_BASE_PORT(id)			((id) * 0x1000)
+
+/* CSI Port General Purpose Registers */
+#define IPU7_CSI_REG_PORT_GPREG_SRST			0x0
+#define IPU7_CSI_REG_PORT_GPREG_CSI2_SLV_REG_SRST	0x4
+#define IPU7_CSI_REG_PORT_GPREG_CSI2_PORT_CONTROL	0x8
+
+#define IPU7_CSI_RX_SYNC_FS_VC				0x55555555
+#define IPU7_CSI_RX_SYNC_FE_VC				0xaaaaaaaa
+#define IPU7P5_CSI_RX_SYNC_FS_VC			0xffff
+#define IPU7P5_CSI_RX_SYNC_FE_VC			0xffff
+
+#endif /* IPU7_ISYS_CSI2_REG_H */
diff --git a/drivers/media/pci/intel/ipu6/ipu7-mmu-hw.c b/drivers/media/pci/intel/ipu6/ipu7-mmu-hw.c
new file mode 100644
index 00000000000000..c9b015e89ee6d5
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-mmu-hw.c
@@ -0,0 +1,858 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Intel Corporation
+ */
+
+#include <linux/types.h>
+#include <linux/iopoll.h>
+
+#include "ipu6.h"
+#include "ipu6-dma.h"
+#include "ipu6-mmu.h"
+
+static struct ipu7_mmu_hw ipu7_isys_mmu_hwdata[] = {
+	{
+		.offset = IPU7_IS_MMU_FW_RD_OFFSET,
+		.zlx_offset = IPU7_IS_ZLX_UC_RD_OFFSET,
+		.uao_offset = IPU7_IS_UAO_UC_RD_OFFSET,
+		.info_bits = 0x20006701,
+		.refill = 0x00002726,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_IS_MMU_FW_RD_L1_BLOCKNR_REG,
+		.l2_block = IPU7_IS_MMU_FW_RD_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_IS_MMU_FW_RD_STREAM_NUM,
+		.nr_l2streams = IPU7_IS_MMU_FW_RD_STREAM_NUM,
+		.l1_block_sz = { 0x0, 0x8, 0xa },
+		.l2_block_sz = { 0x0, 0x2, 0x4 },
+		.zlx_nr = IPU7_IS_ZLX_UC_RD_NUM,
+		.zlx_axi_pool = { 0x00000f30 },
+		.zlx_en = { 0, 0, 0, 0 },
+		.zlx_conf = { 0x0, 0x0, 0x0, 0x0 },
+		.uao_p_num = IPU7_IS_UAO_UC_RD_PLANENUM,
+		.uao_p2tlb = { 0x61, 0x64, 0x65 },
+	},
+	{
+		.offset = IPU7_IS_MMU_FW_WR_OFFSET,
+		.zlx_offset = IPU7_IS_ZLX_UC_WR_OFFSET,
+		.uao_offset = IPU7_IS_UAO_UC_WR_OFFSET,
+		.info_bits = 0x20006801,
+		.refill = 0x00002524,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_IS_MMU_FW_WR_L1_BLOCKNR_REG,
+		.l2_block = IPU7_IS_MMU_FW_WR_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_IS_MMU_FW_WR_STREAM_NUM,
+		.nr_l2streams = IPU7_IS_MMU_FW_WR_STREAM_NUM,
+		.l1_block_sz = { 0x0, 0x8, 0xa },
+		.l2_block_sz = { 0x0, 0x2, 0x4	},
+		.zlx_nr = IPU7_IS_ZLX_UC_WR_NUM,
+		.zlx_axi_pool = { 0x00000f20 },
+		.zlx_en = { 0, 1, 1, 0 },
+		.zlx_conf = { 0x0, 0x00010101, 0x00010101 },
+		.uao_p_num = IPU7_IS_UAO_UC_WR_PLANENUM,
+		.uao_p2tlb = { 0x61, 0x62, 0x63 },
+	},
+	{
+		.offset = IPU7_IS_MMU_M0_OFFSET,
+		.zlx_offset = IPU7_IS_ZLX_M0_OFFSET,
+		.uao_offset = IPU7_IS_UAO_M0_WR_OFFSET,
+		.info_bits = 0x20006601,
+		.refill = 0x00002120,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_IS_MMU_M0_L1_BLOCKNR_REG,
+		.l2_block = IPU7_IS_MMU_M0_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_IS_MMU_M0_STREAM_NUM,
+		.nr_l2streams = IPU7_IS_MMU_M0_STREAM_NUM,
+		.l1_block_sz = { 0x0, 0x3, 0x6, 0x8, 0xa, 0xc, 0xe, 0x10 },
+		.l2_block_sz = { 0x0, 0x2, 0x4, 0x6, 0x8, 0xa, 0xc, 0xe },
+		.zlx_nr = IPU7_IS_ZLX_M0_NUM,
+		.zlx_axi_pool = { 0x00000f10 },
+		.zlx_en = { 1, 1, 1, 1, 1, 1, 1, 1 },
+		.zlx_conf = {
+			0x00010103,
+			0x00010103,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+		},
+		.uao_p_num = IPU7_IS_UAO_M0_WR_PLANENUM,
+		.uao_p2tlb = {
+			0x00000049,
+			0x0000004a,
+			0x0000004b,
+			0x0000004c,
+			0x0000004d,
+			0x0000004e,
+			0x0000004f,
+			0x00000050,
+		},
+	},
+	{
+		.offset = IPU7_IS_MMU_M1_OFFSET,
+		.zlx_offset = IPU7_IS_ZLX_M1_OFFSET,
+		.uao_offset = IPU7_IS_UAO_M1_WR_OFFSET,
+		.info_bits = 0x20006901,
+		.refill = 0x00002322,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_IS_MMU_M1_L1_BLOCKNR_REG,
+		.l2_block = IPU7_IS_MMU_M1_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_IS_MMU_M1_STREAM_NUM,
+		.nr_l2streams = IPU7_IS_MMU_M1_STREAM_NUM,
+		.l1_block_sz = {
+			0x0, 0x3, 0x6, 0x9, 0xc,
+			0xe, 0x10, 0x12, 0x14, 0x16,
+			0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22,
+		},
+		.l2_block_sz = {
+			0x0, 0x2, 0x4, 0x6, 0x8,
+			0xa, 0xc, 0xe, 0x10, 0x12,
+			0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e,
+		},
+		.zlx_nr = IPU7_IS_ZLX_M1_NUM,
+		.zlx_axi_pool = { 0x00000f20 },
+		.zlx_en = { 1, 1, 1, 1, 1, 1, 1, 1,
+			    1, 1, 1, 1, 1, 1, 1, 1,
+		},
+		.zlx_conf = {
+			0x00010103,
+			0x00010103,
+			0x00010103,
+			0x00010103,
+			0x00010103,
+			0x00010103,
+			0x00010103,
+			0x00010103,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+		},
+		.uao_p_num = IPU7_IS_UAO_M1_WR_PLANENUM,
+		.uao_p2tlb = {
+			0x00000051,
+			0x00000052,
+			0x00000053,
+			0x00000054,
+			0x00000055,
+			0x00000056,
+			0x00000057,
+			0x00000058,
+			0x00000059,
+			0x0000005a,
+			0x0000005b,
+			0x0000005c,
+			0x0000005d,
+			0x0000005e,
+			0x0000005f,
+			0x00000060,
+		},
+	},
+};
+
+static struct ipu7_mmu_hw ipu7_psys_mmu_hwdata[] = {
+	{
+		.name = "PS_FW_RD",
+		.offset = IPU7_PS_MMU_FW_RD_OFFSET,
+		.zlx_offset = IPU7_PS_ZLX_FW_RD_OFFSET,
+		.uao_offset = IPU7_PS_UAO_FW_RD_OFFSET,
+		.info_bits = 0x20004801,
+		.refill = 0x00002726,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_PS_MMU_FW_RD_L1_BLOCKNR_REG,
+		.l2_block = IPU7_PS_MMU_FW_RD_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_PS_MMU_FW_RD_STREAM_NUM,
+		.nr_l2streams = IPU7_PS_MMU_FW_RD_STREAM_NUM,
+		.l1_block_sz = {
+			0, 0x8, 0xa, 0xc, 0xd,
+			0xf, 0x11, 0x12, 0x13, 0x14,
+			0x16, 0x18, 0x19, 0x1a, 0x1a,
+			0x1a, 0x1a, 0x1a, 0x1a, 0x1a,
+		},
+		.l2_block_sz = {
+			0x0, 0x2, 0x4, 0x6, 0x8,
+			0xa, 0xc, 0xe, 0x10, 0x12,
+			0x14, 0x16, 0x18, 0x1a, 0x1c,
+			0x1e, 0x20, 0x22, 0x24, 0x26,
+		},
+		.zlx_nr = IPU7_PS_ZLX_FW_RD_NUM,
+		.zlx_axi_pool = { 0x00000f30 },
+		.zlx_en = {
+			0, 0, 0, 0, 0, 0, 0, 0,
+			0, 0, 0, 0, 0, 0, 0, 0,
+		},
+		.zlx_conf = { 0x0 },
+		.uao_p_num = IPU7_PS_UAO_FW_RD_PLANENUM,
+		.uao_p2tlb = {
+			0x00000036,
+			0x0000003d,
+			0x0000003e,
+			0x00000039,
+			0x0000003f,
+			0x00000040,
+			0x00000041,
+			0x0000003a,
+			0x0000003b,
+			0x00000042,
+			0x00000043,
+			0x00000044,
+			0x0000003c,
+		},
+	},
+	{
+		.offset = IPU7_PS_MMU_FW_WR_OFFSET,
+		.zlx_offset = IPU7_PS_ZLX_FW_WR_OFFSET,
+		.uao_offset = IPU7_PS_UAO_FW_WR_OFFSET,
+		.info_bits = 0x20004601,
+		.refill = 0x00002322,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_PS_MMU_FW_WR_L1_BLOCKNR_REG,
+		.l2_block = IPU7_PS_MMU_FW_WR_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_PS_MMU_FW_WR_STREAM_NUM,
+		.nr_l2streams = IPU7_PS_MMU_FW_WR_STREAM_NUM,
+		.l1_block_sz = {
+			0, 0x8, 0xa, 0xc, 0xd,
+			0xe, 0xf, 0x10, 0x10, 0x10,
+		},
+		.l2_block_sz = {
+			0x0, 0x2, 0x4, 0x6, 0x8,
+			0xa, 0xc, 0xe, 0x10, 0x12,
+		},
+		.zlx_nr = IPU7_PS_ZLX_FW_WR_NUM,
+		.zlx_axi_pool = { 0x00000f20 },
+		.zlx_en = { 0, 1, 1, 0, 0, 0, 0, 0, 0, 0 },
+		.zlx_conf = { 0x0, 0x00010101, 0x00010101 },
+		.uao_p_num = IPU7_PS_UAO_FW_WR_PLANENUM,
+		.uao_p2tlb = { 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c },
+	},
+	{
+		.offset = IPU7_PS_MMU_SRT_RD_OFFSET,
+		.zlx_offset = IPU7_PS_ZLX_DATA_RD_OFFSET,
+		.uao_offset = IPU7_PS_UAO_SRT_RD_OFFSET,
+		.info_bits = 0x20004701,
+		.refill = 0x00002120,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_PS_MMU_SRT_RD_L1_BLOCKNR_REG,
+		.l2_block = IPU7_PS_MMU_SRT_RD_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_PS_MMU_SRT_RD_STREAM_NUM,
+		.nr_l2streams = IPU7_PS_MMU_SRT_RD_STREAM_NUM,
+		.l1_block_sz = {
+			0x0, 0x4, 0x6, 0x8, 0xb,
+			0xd, 0xf, 0x11, 0x13, 0x15,
+			0x17, 0x23, 0x2b, 0x37, 0x3f,
+			0x41, 0x43, 0x44, 0x45, 0x46,
+			0x47, 0x48, 0x49, 0x4a, 0x4b,
+			0x4c, 0x4d, 0x4e, 0x4f, 0x50,
+			0x51, 0x52, 0x53, 0x55, 0x57,
+			0x59, 0x5b, 0x5d, 0x5f, 0x61,
+		},
+		.l2_block_sz = {
+			0x0, 0x2, 0x4, 0x6, 0x8,
+			0xa, 0xc, 0xe, 0x10, 0x12,
+			0x14, 0x16, 0x18, 0x1a, 0x1c,
+			0x1e, 0x20, 0x22, 0x24, 0x26,
+			0x28, 0x2a, 0x2c, 0x2e, 0x30,
+			0x32, 0x34, 0x36, 0x38, 0x3a,
+			0x3c, 0x3e, 0x40, 0x42, 0x44,
+			0x46, 0x48, 0x4a, 0x4c, 0x4e,
+		},
+		.zlx_nr = IPU7_PS_ZLX_DATA_RD_NUM,
+		.zlx_axi_pool = { 0x00000f30 },
+		.zlx_en = {
+			1, 1, 1, 1, 1, 1, 1, 1,
+			1, 1, 1, 1, 1, 1, 1, 1,
+			0, 0, 0, 0, 0, 0, 0, 0,
+			0, 0, 0, 0, 0, 0, 0, 0,
+		},
+		.zlx_conf = {
+			0x00030303,
+			0x00010101,
+			0x00010101,
+			0x00030202,
+			0x00010101,
+			0x00010101,
+			0x00010101,
+			0x00030800,
+			0x00030500,
+			0x00020101,
+			0x00042000,
+			0x00031000,
+			0x00042000,
+			0x00031000,
+			0x00020400,
+			0x00010101,
+		},
+		.uao_p_num = IPU7_PS_UAO_SRT_RD_PLANENUM,
+		.uao_p2tlb = {
+			0x00000022,
+			0x00000023,
+			0x00000024,
+			0x00000025,
+			0x00000026,
+			0x00000027,
+			0x00000028,
+			0x00000029,
+			0x0000002a,
+			0x0000002b,
+			0x0000002c,
+			0x0000002d,
+			0x0000002e,
+			0x0000002f,
+			0x00000030,
+			0x00000031,
+			0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+			0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+			0x0000001e,
+			0x0000001f,
+			0x00000020,
+			0x00000021,
+			0x00000032,
+			0x00000033,
+			0x00000034,
+			0x00000035,
+		},
+	},
+	{
+		.offset = IPU7_PS_MMU_SRT_WR_OFFSET,
+		.zlx_offset = IPU7_PS_ZLX_DATA_WR_OFFSET,
+		.uao_offset = IPU7_PS_UAO_SRT_WR_OFFSET,
+		.info_bits = 0x20004501,
+		.refill = 0x00002120,
+		.collapse_en_bitmap = 0x0,
+		.l1_block = IPU7_PS_MMU_SRT_WR_L1_BLOCKNR_REG,
+		.l2_block = IPU7_PS_MMU_SRT_WR_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7_PS_MMU_SRT_WR_STREAM_NUM,
+		.nr_l2streams = IPU7_PS_MMU_SRT_WR_STREAM_NUM,
+		.l1_block_sz = {
+			0x0, 0x2, 0x6, 0xa, 0xc,
+			0xe, 0x10, 0x12, 0x14, 0x16,
+			0x18, 0x1a, 0x1c, 0x1e, 0x20,
+			0x22, 0x24, 0x26, 0x32, 0x3a,
+			0x3c, 0x3e, 0x4a, 0x52, 0x58,
+			0x64, 0x6c, 0x72, 0x7e, 0x86,
+			0x8c, 0x8d, 0x8e, 0x8f, 0x90,
+			0x91, 0x92, 0x94, 0x96, 0x98,
+		},
+		.l2_block_sz = {
+			0x0, 0x2, 0x4, 0x6, 0x8,
+			0xa, 0xc, 0xe, 0x10, 0x12,
+			0x14, 0x16, 0x18, 0x1a, 0x1c,
+			0x1e, 0x20, 0x22, 0x24, 0x26,
+			0x28, 0x2a, 0x2c, 0x2e, 0x30,
+			0x32, 0x34, 0x36, 0x38, 0x3a,
+			0x3c, 0x3e, 0x40, 0x42, 0x44,
+			0x46, 0x48, 0x4a, 0x4c, 0x4e,
+		},
+		.zlx_nr = IPU7_PS_ZLX_DATA_WR_NUM,
+		.zlx_axi_pool = { 0x00000f50 },
+		.zlx_en = {
+			1, 1, 1, 1, 1, 1, 1, 1,
+			0, 0, 1, 1, 1, 1, 1, 1,
+			1, 1, 1, 1, 1, 1, 1, 1,
+			1, 1, 1, 1, 1, 1, 0, 0,
+		},
+		.zlx_conf = {
+			0x00010102,
+			0x00030103,
+			0x00030103,
+			0x00010101,
+			0x00010101,
+			0x00030101,
+			0x00010101,
+			0x38010101,
+			0x0,
+			0x0,
+			0x38010101,
+			0x38010101,
+			0x38010101,
+			0x38010101,
+			0x38010101,
+			0x38010101,
+			0x00010101,
+			0x00042000,
+			0x00031000,
+			0x00010101,
+			0x00010101,
+			0x00042000,
+			0x00031000,
+			0x00031000,
+			0x00042000,
+			0x00031000,
+			0x00031000,
+			0x00042000,
+			0x00031000,
+			0x00031000,
+			0x0,
+			0x0,
+		},
+		.uao_p_num = IPU7_PS_UAO_SRT_WR_PLANENUM,
+		.uao_p2tlb = {
+			0x00000000,
+			0x00000001,
+			0x00000002,
+			0x00000003,
+			0x00000004,
+			0x00000005,
+			0x00000006,
+			0x00000007,
+			0x00000008,
+			0x00000009,
+			0x0000000a,
+			0x0000000b,
+			0x0000000c,
+			0x0000000d,
+			0x0000000e,
+			0x0000000f,
+			0x00000010,
+			0x00000011,
+			0x00000012,
+			0x00000013,
+			0x00000014,
+			0x00000015,
+			0x00000016,
+			0x00000017,
+			0x00000018,
+			0x00000019,
+			0x0000001a,
+			0x0000001b,
+			0x0000001c,
+			0x0000001d,
+			0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+			0x0000001e,
+			0x0000001f,
+			0x00000020,
+			0x00000021,
+		},
+	},
+};
+
+static struct ipu7_mmu_hw ipu7p5_isys_mmu_hwdata[] = {
+	{
+		.name = "IS_FW_RD",
+		.offset = IPU7P5_IS_MMU_FW_RD_OFFSET,
+		.zlx_offset = IPU7P5_IS_ZLX_UC_RD_OFFSET,
+		.uao_offset = IPU7P5_IS_UAO_UC_RD_OFFSET,
+		.info_bits = 0x20005101,
+		.refill = 0x00002726,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_IS_MMU_FW_RD_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_IS_MMU_FW_RD_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_IS_MMU_FW_RD_STREAM_NUM,
+		.nr_l2streams = IPU7P5_IS_MMU_FW_RD_STREAM_NUM,
+		.l1_block_sz = { 0x0, 0x8, 0xa },
+		.l2_block_sz = { 0x0, 0x2, 0x4 },
+		.zlx_nr = IPU7P5_IS_ZLX_UC_RD_NUM,
+		.zlx_axi_pool = { 0x00000f30 },
+		.zlx_en = { 0, 1, 0, 0 },
+		.zlx_conf = { 0x0 },
+		.uao_p_num = IPU7P5_IS_UAO_UC_RD_PLANENUM,
+		.uao_p2tlb = { 0x49, 0x4c, 0x4d, 0x0 },
+	},
+	{
+		.name = "IS_FW_WR",
+		.offset = IPU7P5_IS_MMU_FW_WR_OFFSET,
+		.zlx_offset = IPU7P5_IS_ZLX_UC_WR_OFFSET,
+		.uao_offset = IPU7P5_IS_UAO_UC_WR_OFFSET,
+		.info_bits = 0x20005001,
+		.refill = 0x00002524,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_IS_MMU_FW_WR_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_IS_MMU_FW_WR_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_IS_MMU_FW_WR_STREAM_NUM,
+		.nr_l2streams = IPU7P5_IS_MMU_FW_WR_STREAM_NUM,
+		.l1_block_sz = { 0x0, 0x8, 0xa },
+		.l2_block_sz = { 0x0, 0x2, 0x4 },
+		.zlx_nr = IPU7P5_IS_ZLX_UC_WR_NUM,
+		.zlx_axi_pool = { 0x00000f20 },
+		.zlx_en = { 0, 1, 1, 0 },
+		.zlx_conf = { 0x0, 0x00010101, 0x00010101, 0x0 },
+		.uao_p_num = IPU7P5_IS_UAO_UC_WR_PLANENUM,
+		.uao_p2tlb = { 0x49, 0x4a, 0x4b, 0x0 },
+	},
+	{
+		.name = "IS_DATA_WR_ISOC",
+		.offset = IPU7P5_IS_MMU_M0_OFFSET,
+		.zlx_offset = IPU7P5_IS_ZLX_M0_OFFSET,
+		.uao_offset = IPU7P5_IS_UAO_M0_WR_OFFSET,
+		.info_bits = 0x20004e01,
+		.refill = 0x00002120,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_IS_MMU_M0_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_IS_MMU_M0_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_IS_MMU_M0_STREAM_NUM,
+		.nr_l2streams = IPU7P5_IS_MMU_M0_STREAM_NUM,
+		.l1_block_sz = { 0x0, 0x2, 0x4, 0x6, 0x8, 0xa, 0xc, 0xe, 0x10,
+				 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x01e },
+		.l2_block_sz = { 0x0, 0x2, 0x4, 0x6, 0x8, 0xa, 0xc, 0xe, 0x10,
+				 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e },
+		.zlx_nr = IPU7P5_IS_ZLX_M0_NUM,
+		.zlx_axi_pool = { 0x00000f10 },
+		.zlx_en = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
+		.zlx_conf = { 0x00010103, 0x00010103, 0x00010103, 0x00010103,
+			      0x00010103, 0x00010103, 0x00010103, 0x00010103,
+			      0x00010103, 0x00010103, 0x00010103, 0x00010103,
+			      0x00010103, 0x00010103, 0x00010103, 0x00010103 },
+		.uao_p_num = IPU7P5_IS_UAO_M0_WR_PLANENUM,
+		.uao_p2tlb = { 0x00000041, 0x00000042, 0x00000043, 0x00000044,
+			       0x00000041, 0x00000042, 0x00000043, 0x00000044,
+			       0x00000041, 0x00000042, 0x00000043, 0x00000044,
+			       0x00000041, 0x00000042, 0x00000043, 0x00000044 },
+	},
+	{
+		.name = "IS_DATA_WR_SNOOP",
+		.offset = IPU7P5_IS_MMU_M1_OFFSET,
+		.zlx_offset = IPU7P5_IS_ZLX_M1_OFFSET,
+		.uao_offset = IPU7P5_IS_UAO_M1_WR_OFFSET,
+		.info_bits = 0x20004f01,
+		.refill = 0x00002322,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_IS_MMU_M1_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_IS_MMU_M1_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_IS_MMU_M1_STREAM_NUM,
+		.nr_l2streams = IPU7P5_IS_MMU_M1_STREAM_NUM,
+		.l1_block_sz = { 0x0, 0x2, 0x4, 0x6, 0x8, 0xa, 0xc, 0xe, 0x10,
+				 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e },
+		.l2_block_sz = { 0x0, 0x2, 0x4, 0x6, 0x8, 0xa, 0xc, 0xe, 0x10,
+				 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e },
+		.zlx_nr = IPU7P5_IS_ZLX_M1_NUM,
+		.zlx_axi_pool = { 0x00000f20 },
+		.zlx_en = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
+		.zlx_conf = { 0x00010103, 0x00010103, 0x00010103, 0x00010103,
+			      0x00010103, 0x00010103, 0x00010103, 0x00010103,
+			      0x00010103, 0x00010103, 0x00010103, 0x00010103,
+			      0x00010103, 0x00010103, 0x00010103, 0x00010103 },
+		.uao_p_num = IPU7P5_IS_UAO_M1_WR_PLANENUM,
+		.uao_p2tlb = { 0x00000045, 0x00000046, 0x00000047, 0x00000048,
+			       0x00000045, 0x00000046, 0x00000047, 0x00000048,
+			       0x00000045, 0x00000046, 0x00000047, 0x00000048,
+			       0x00000045, 0x00000046, 0x00000047, 0x00000048 },
+	},
+};
+
+static struct ipu7_mmu_hw ipu7p5_psys_mmu_hwdata[] = {
+	{
+		.name = "PS_FW_RD",
+		.offset = IPU7P5_PS_MMU_FW_RD_OFFSET,
+		.zlx_offset = IPU7P5_PS_ZLX_FW_RD_OFFSET,
+		.uao_offset = IPU7P5_PS_UAO_FW_RD_OFFSET,
+		.info_bits = 0x20004001,
+		.refill = 0x00002726,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_PS_MMU_FW_RD_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_PS_MMU_FW_RD_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_PS_MMU_FW_RD_STREAM_NUM,
+		.nr_l2streams = IPU7P5_PS_MMU_FW_RD_STREAM_NUM,
+		.l1_block_sz = { 0x00, 0x08, 0x0a, 0x0c, 0x0d, 0x0f, 0x11,
+				 0x12, 0x13, 0x14, 0x16, 0x18, 0x19, 0x1a,
+				 0x1a, 0x1a },
+		.l2_block_sz = { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c,
+				 0x0e, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a,
+				 0x1c, 0x1e },
+		.zlx_nr = IPU7P5_PS_ZLX_FW_RD_NUM,
+		.zlx_axi_pool = { 0x00000f30 },
+		.zlx_en = { 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0 },
+		.zlx_conf = { 0x00000000, 0x00010101, 0x00000000, 0x00000000,
+			      0x00010101, 0x00010101, 0x00000000, 0x00000000,
+			      0x00000000, 0x00010101, 0x00010101, 0x00000000,
+			      0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+		.uao_p_num = IPU7P5_PS_UAO_FW_RD_PLANENUM,
+		.uao_p2tlb = { 0x2e, 0x35, 0x36, 0x31, 0x37, 0x38, 0x39, 0x32,
+			       0x33, 0x3a, 0x3b, 0x3c, 0x34, 0x00, 0x00, 0x00 },
+	},
+	{
+		.name = "PS_FW_WR",
+		.offset = IPU7P5_PS_MMU_FW_WR_OFFSET,
+		.zlx_offset = IPU7P5_PS_ZLX_FW_WR_OFFSET,
+		.uao_offset = IPU7P5_PS_UAO_FW_WR_OFFSET,
+		.info_bits = 0x20003e01,
+		.refill = 0x00002322,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_PS_MMU_FW_WR_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_PS_MMU_FW_WR_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_PS_MMU_FW_WR_STREAM_NUM,
+		.nr_l2streams = IPU7P5_PS_MMU_FW_WR_STREAM_NUM,
+		.l1_block_sz = { 0x00, 0x08, 0x0a, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
+				 0x10, 0x10 },
+		.l2_block_sz = { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e,
+				 0x10, 0x12 },
+		.zlx_nr = IPU7P5_PS_ZLX_FW_WR_NUM,
+		.zlx_axi_pool = { 0x00000f20 },
+		.zlx_en = { 0, 1, 1, 0, 0, 0, 0, 0, 0, 0 },
+		.zlx_conf = { 0x00000000, 0x00010101, 0x00010101, 0x00000000,
+			      0x00000000, 0x00000000, 0x00000000, 0x00000000,
+			      0x00000000, 0x00000000 },
+		.uao_p_num = IPU7P5_PS_UAO_FW_WR_PLANENUM,
+		.uao_p2tlb = { 0x0000002e, 0x0000002f, 0x00000030, 0x00000031,
+			       0x00000032, 0x00000033, 0x00000034, 0x00000000,
+			       0x00000000, 0x00000000 },
+	},
+	{
+		.name = "PS_DATA_RD",
+		.offset = IPU7P5_PS_MMU_SRT_RD_OFFSET,
+		.zlx_offset = IPU7P5_PS_ZLX_DATA_RD_OFFSET,
+		.uao_offset = IPU7P5_PS_UAO_SRT_RD_OFFSET,
+		.info_bits = 0x20003f01,
+		.refill = 0x00002524,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_PS_MMU_SRT_RD_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_PS_MMU_SRT_RD_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_PS_MMU_SRT_RD_STREAM_NUM,
+		.nr_l2streams = IPU7P5_PS_MMU_SRT_RD_STREAM_NUM,
+		.l1_block_sz = { 0x00, 0x04, 0x06, 0x08, 0x0b, 0x0d, 0x0f, 0x13,
+				 0x17, 0x19, 0x1b, 0x1d, 0x1f, 0x2b, 0x33, 0x3f,
+				 0x47, 0x49, 0x4b, 0x4c, 0x4d, 0x4e },
+		.l2_block_sz = { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e,
+				 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e,
+				 0x20, 0x22, 0x24, 0x26, 0x28, 0x2a },
+		.zlx_nr = IPU7P5_PS_ZLX_DATA_RD_NUM,
+		.zlx_axi_pool = { 0x00000f30 },
+		.zlx_en = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+			    1, 1, 0, 0, 0, 0 },
+		.zlx_conf = { 0x00030303, 0x00010101, 0x00010101, 0x00030202,
+			      0x00010101, 0x00010101, 0x00030303, 0x00030303,
+			      0x00010101, 0x00030800, 0x00030500, 0x00020101,
+			      0x00042000, 0x00031000, 0x00042000, 0x00031000,
+			      0x00020400, 0x00010101, 0x00000000, 0x00000000,
+			      0x00000000, 0x00000000 },
+		.uao_p_num = IPU7P5_PS_UAO_SRT_RD_PLANENUM,
+		.uao_p2tlb = { 0x0000001c, 0x0000001d, 0x0000001e, 0x0000001f,
+			       0x00000020, 0x00000021, 0x00000022, 0x00000023,
+			       0x00000024, 0x00000025, 0x00000026, 0x00000027,
+			       0x00000028, 0x00000029, 0x0000002a, 0x0000002b,
+			       0x0000002c, 0x0000002d, 0x00000000, 0x00000000,
+			       0x00000000, 0x00000000 },
+	},
+	{
+		.name = "PS_DATA_WR",
+		.offset = IPU7P5_PS_MMU_SRT_WR_OFFSET,
+		.zlx_offset = IPU7P5_PS_ZLX_DATA_WR_OFFSET,
+		.uao_offset = IPU7P5_PS_UAO_SRT_WR_OFFSET,
+		.info_bits = 0x20003d01,
+		.refill = 0x00002120,
+		.collapse_en_bitmap = 0x1,
+		.at_sp_arb_cfg = 0x1,
+		.l1_block = IPU7P5_PS_MMU_SRT_WR_L1_BLOCKNR_REG,
+		.l2_block = IPU7P5_PS_MMU_SRT_WR_L2_BLOCKNR_REG,
+		.nr_l1streams = IPU7P5_PS_MMU_SRT_WR_STREAM_NUM,
+		.nr_l2streams = IPU7P5_PS_MMU_SRT_WR_STREAM_NUM,
+		.l1_block_sz = { 0x00, 0x02, 0x06, 0x0a, 0x0c, 0x0e, 0x10, 0x12,
+				 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22,
+				 0x24, 0x28, 0x2a, 0x36, 0x3e, 0x40, 0x42, 0x4e,
+				 0x56, 0x5c, 0x68, 0x70, 0x76, 0x77, 0x78, 0x79 },
+		.l2_block_sz = { 0x00, 0x02, 0x06, 0x0a, 0x0c, 0x0e, 0x10, 0x12,
+				 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22,
+				 0x24, 0x28, 0x2a, 0x36, 0x3e, 0x40, 0x42, 0x4e,
+				 0x56, 0x5c, 0x68, 0x70, 0x76, 0x77, 0x78, 0x79 },
+		.zlx_nr = IPU7P5_PS_ZLX_DATA_WR_NUM,
+		.zlx_axi_pool = { 0x00000f50 },
+		.zlx_en = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1,
+			    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 },
+		.zlx_conf = { 0x00010102, 0x00030103, 0x00030103, 0x00010101,
+			      0x00010101, 0x00030101, 0x00010101, 0x38010101,
+			      0x00000000, 0x00000000, 0x38010101, 0x38010101,
+			      0x38010101, 0x38010101, 0x38010101, 0x38010101,
+			      0x00030303, 0x00010101, 0x00042000, 0x00031000,
+			      0x00010101, 0x00010101, 0x00042000, 0x00031000,
+			      0x00031000, 0x00042000, 0x00031000, 0x00031000,
+			      0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+		.uao_p_num = IPU7P5_PS_UAO_SRT_WR_PLANENUM,
+		.uao_p2tlb = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+			       0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+			       0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+			       0x18, 0x19, 0x1a, 0x1b, 0x00, 0x00, 0x00, 0x00 },
+	},
+};
+
+static const struct ipu7_mmu_hwdata ipu7_mmu_hwdata_lookup[IPU_SUBSYS_NUM] = {
+	[IPU_PSYS] = {
+		.hwdata = ipu7_psys_mmu_hwdata,
+		.nr_mmus = ARRAY_SIZE(ipu7_psys_mmu_hwdata),
+	},
+	[IPU_ISYS] = {
+		.hwdata = ipu7_isys_mmu_hwdata,
+		.nr_mmus = ARRAY_SIZE(ipu7_isys_mmu_hwdata),
+	},
+};
+
+static const struct ipu7_mmu_hwdata ipu7p5_mmu_hwdata_lookup[IPU_SUBSYS_NUM] = {
+	[IPU_PSYS] = {
+		.hwdata = ipu7p5_psys_mmu_hwdata,
+		.nr_mmus = ARRAY_SIZE(ipu7p5_psys_mmu_hwdata),
+	},
+	[IPU_ISYS] = {
+		.hwdata = ipu7p5_isys_mmu_hwdata,
+		.nr_mmus = ARRAY_SIZE(ipu7p5_isys_mmu_hwdata),
+	},
+};
+
+static void __ipu7_tlb_invalidate(struct ipu6_mmu *mmu)
+{
+	struct ipu7_mmu_hw *mmu_hw = mmu->ipu7_mmu_hw;
+	unsigned long flags;
+	unsigned int i;
+	int ret;
+	u32 val;
+
+	spin_lock_irqsave(&mmu->ready_lock, flags);
+	if (!mmu->ready) {
+		spin_unlock_irqrestore(&mmu->ready_lock, flags);
+		return;
+	}
+
+	for (i = 0; i < mmu->nr_mmus; i++) {
+		writel(0xffffffffU, mmu_hw[i].base +
+		       IPU7_MMU_REG_INVALIDATE_0);
+
+		/* Need check with HW, use l1streams or l2streams */
+		if (mmu_hw[i].nr_l2streams > 32)
+			writel(0xffffffffU, mmu_hw[i].base +
+			       IPU7_MMU_REG_INVALIDATE_1);
+
+		/*
+		 * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
+		 * When the actual MMIO write reaches the IPU TLB Invalidate
+		 * register, wmb() will force the TLB invalidate out if the CPU
+		 * attempts to update the IOMMU page table (or sooner).
+		 */
+		wmb();
+
+		/* wait invalidation done */
+		ret = readl_poll_timeout_atomic(mmu_hw[i].base +
+						IPU7_MMU_REG_INVALIDATION_STATUS,
+						val, !(val & 0x1U), 500,
+						IPU7_MMU_TLB_INVALIDATE_TIMEOUT);
+		if (ret)
+			dev_err(mmu->dev, "MMU[%u] TLB invalidate failed\n", i);
+	}
+
+	spin_unlock_irqrestore(&mmu->ready_lock, flags);
+}
+
+static int __ipu7_mmu_hw_init(struct ipu6_mmu *mmu)
+{
+	struct ipu6_mmu_info *mmu_info;
+	struct ipu7_mmu_hw *mmu_hw = mmu->ipu7_mmu_hw;
+	unsigned int i, j;
+
+	mmu_info = mmu->dmap->mmu_info;
+	for (i = 0; i < mmu->nr_mmus; i++) {
+		/* Write page table address per MMU */
+		writel((phys_addr_t)mmu_info->l1_pt_dma,
+		       mmu_hw[i].base + IPU7_MMU_REG_PAGE_TABLE_BASE_ADDR);
+
+		/* Set info bits and axi_refill per MMU */
+		writel(mmu_hw[i].info_bits,
+		       mmu_hw[i].base + IPU7_MMU_REG_USER_INFO_BITS);
+		writel(mmu_hw[i].refill, mmu_hw[i].base + IPU7_MMU_REG_AXI_REFILL_IF_ID);
+		writel(mmu_hw[i].collapse_en_bitmap,
+		       mmu_hw[i].base + IPU7_MMU_REG_COLLAPSE_ENABLE_BITMAP);
+
+		if (mmu_hw[i].at_sp_arb_cfg)
+			writel(mmu_hw[i].at_sp_arb_cfg,
+			       mmu_hw[i].base + IPU7_MMU_REG_AT_SP_ARB_CFG);
+
+		/* Default irq configuration */
+		writel(0x3ff, mmu_hw[i].base + IPU7_MMU_REG_IRQ_MASK);
+		writel(0x3ff, mmu_hw[i].base + IPU7_MMU_REG_IRQ_ENABLE);
+
+		/* Configure MMU TLB stream configuration for L1/L2 */
+		for (j = 0; j < mmu_hw[i].nr_l1streams; j++) {
+			writel(mmu_hw[i].l1_block_sz[j], mmu_hw[i].base +
+			       mmu_hw[i].l1_block + 4U * j);
+		}
+
+		for (j = 0; j < mmu_hw[i].nr_l2streams; j++) {
+			writel(mmu_hw[i].l2_block_sz[j], mmu_hw[i].base +
+			       mmu_hw[i].l2_block + 4U * j);
+		}
+
+		for (j = 0; j < mmu_hw[i].uao_p_num; j++) {
+			if (!mmu_hw[i].uao_p2tlb[j])
+				continue;
+			writel(mmu_hw[i].uao_p2tlb[j], mmu_hw[i].uao_base + 4U * j);
+		}
+	}
+
+	for (i = 0; i < mmu->nr_mmus; i++) {
+		for (j = 0; j < IPU7_ZLX_POOL_NUM; j++) {
+			if (!mmu_hw[i].zlx_axi_pool[j])
+				continue;
+			writel(mmu_hw[i].zlx_axi_pool[j],
+			       mmu_hw[i].zlx_base + IPU7_ZLX_REG_AXI_POOL + j * 0x4U);
+		}
+
+		for (j = 0; j < mmu_hw[i].zlx_nr; j++) {
+			if (!mmu_hw[i].zlx_conf[j])
+				continue;
+
+			writel(mmu_hw[i].zlx_conf[j],
+			       mmu_hw[i].zlx_base + IPU7_ZLX_REG_CONF + j * 0x8U);
+		}
+
+		for (j = 0; j < mmu_hw[i].zlx_nr; j++) {
+			if (!mmu_hw[i].zlx_en[j])
+				continue;
+
+			writel(mmu_hw[i].zlx_en[j],
+			       mmu_hw[i].zlx_base + IPU7_ZLX_REG_EN + j * 0x8U);
+		}
+	}
+
+	return 0;
+}
+
+static int __ipu7_mmu_init_hw_data(struct ipu6_mmu *mmu, struct device *dev,
+				   void __iomem *base)
+{
+	struct ipu6_device *isp = pci_get_drvdata(to_pci_dev(dev));
+	const struct ipu7_mmu_hwdata *lookup;
+	struct ipu7_mmu_hw *mmu_hw, *src;
+	unsigned int i, nr_mmus;
+
+	if (mmu->mmid >= IPU_SUBSYS_NUM)
+		return -EINVAL;
+
+	lookup = IS_IPU7P5(isp) ? ipu7p5_mmu_hwdata_lookup :
+				  ipu7_mmu_hwdata_lookup;
+
+	src = lookup[mmu->mmid].hwdata;
+	nr_mmus = lookup[mmu->mmid].nr_mmus;
+
+	mmu_hw = devm_kcalloc(dev, nr_mmus, sizeof(*mmu_hw), GFP_KERNEL);
+	if (!mmu_hw)
+		return -ENOMEM;
+
+	for (i = 0; i < nr_mmus; i++) {
+		if (src[i].nr_l1streams > IPU7_MMU_MAX_TLB_L1_STREAMS ||
+		    src[i].nr_l2streams > IPU7_MMU_MAX_TLB_L2_STREAMS)
+			return -EINVAL;
+
+		mmu_hw[i] = src[i];
+		mmu_hw[i].base = base + src[i].offset;
+		mmu_hw[i].zlx_base = base + src[i].zlx_offset;
+		mmu_hw[i].uao_base = base + src[i].uao_offset;
+	}
+
+	mmu->nr_mmus = nr_mmus;
+	mmu->ipu7_mmu_hw = mmu_hw;
+
+	return 0;
+}
+
+const struct ipu6_mmu_hw_ops ipu7_mmu_ops = {
+	.init_hw_data = __ipu7_mmu_init_hw_data,
+	.hw_init = __ipu7_mmu_hw_init,
+	.tlb_invalidate = __ipu7_tlb_invalidate,
+};
diff --git a/drivers/media/pci/intel/ipu6/ipu7-mmu-hw.h b/drivers/media/pci/intel/ipu6/ipu7-mmu-hw.h
new file mode 100644
index 00000000000000..ba31ed31b245c9
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-mmu-hw.h
@@ -0,0 +1,254 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2026 Intel Corporation */
+
+#ifndef IPU7_MMU_HW_H
+#define IPU7_MMU_HW_H
+
+#define IPU7_MMU_REG_IRQ_MASK			0x2c
+#define IPU7_MMU_REG_IRQ_ENABLE			0x30
+#define IPU7_MMU_REG_PAGE_TABLE_BASE_ADDR	0x08
+#define IPU7_MMU_REG_USER_INFO_BITS		0x0c
+#define IPU7_MMU_REG_AXI_REFILL_IF_ID		0x10
+#define IPU7_MMU_REG_COLLAPSE_ENABLE_BITMAP	0x18
+#define IPU7_MMU_REG_AT_SP_ARB_CFG		0x20
+
+#define IPU7_ZLX_REG_AXI_POOL			0x0
+#define IPU7_ZLX_REG_EN				0x20
+#define IPU7_ZLX_REG_CONF			0x24
+#define IPU7_ZLX_POOL_NUM			8U
+
+#define IPU7_MMU_MAX_TLB_L1_STREAMS		40U
+#define IPU7_MMU_MAX_TLB_L2_STREAMS		40U
+#define IPU7_UAO_PLANE_MAX_NUM			64U
+#define IPU7_ZLX_MAX_NUM			32U
+
+#define IPU7_MMU_REG_INVALIDATE_0		0x00
+#define IPU7_MMU_REG_INVALIDATE_1		0x04
+#define IPU7_MMU_REG_INVALIDATION_STATUS	0x24
+#define IPU7_MMU_TLB_INVALIDATE_TIMEOUT		2000
+
+#define IPU7_FW_CODE_REGION_SIZE	0x1000000 /* 16MB */
+#define IPU7_FW_CODE_REGION_START	0x4000000 /* 64MB */
+#define IPU7_FW_CODE_REGION_END		(IPU7_FW_CODE_REGION_START +	\
+					 IPU7_FW_CODE_REGION_SIZE) /* 80MB */
+
+#define IPU7_IS_MMU_FW_RD_OFFSET		0x274000
+#define IPU7_IS_MMU_FW_RD_STREAM_NUM		3
+#define IPU7_IS_MMU_FW_RD_L1_BLOCKNR_REG	0x54
+#define IPU7_IS_MMU_FW_RD_L2_BLOCKNR_REG	0x60
+
+#define IPU7_IS_MMU_FW_WR_OFFSET		0x275000
+#define IPU7_IS_MMU_FW_WR_STREAM_NUM		3
+#define IPU7_IS_MMU_FW_WR_L1_BLOCKNR_REG	0x54
+#define IPU7_IS_MMU_FW_WR_L2_BLOCKNR_REG	0x60
+
+#define IPU7_IS_MMU_M0_OFFSET			0x276000
+#define IPU7_IS_MMU_M0_STREAM_NUM		8
+#define IPU7_IS_MMU_M0_L1_BLOCKNR_REG		0x54
+#define IPU7_IS_MMU_M0_L2_BLOCKNR_REG		0x74
+
+#define IPU7_IS_MMU_M1_OFFSET			0x277000
+#define IPU7_IS_MMU_M1_STREAM_NUM		16
+#define IPU7_IS_MMU_M1_L1_BLOCKNR_REG		0x54
+#define IPU7_IS_MMU_M1_L2_BLOCKNR_REG		0x94
+
+#define IPU7_PS_MMU_FW_RD_OFFSET		0x148000
+#define IPU7_PS_MMU_FW_RD_STREAM_NUM		20
+#define IPU7_PS_MMU_FW_RD_L1_BLOCKNR_REG	0x54
+#define IPU7_PS_MMU_FW_RD_L2_BLOCKNR_REG	0xa4
+
+#define IPU7_PS_MMU_FW_WR_OFFSET		0x149000
+#define IPU7_PS_MMU_FW_WR_STREAM_NUM		10
+#define IPU7_PS_MMU_FW_WR_L1_BLOCKNR_REG	0x54
+#define IPU7_PS_MMU_FW_WR_L2_BLOCKNR_REG	0x7c
+
+#define IPU7_PS_MMU_SRT_RD_OFFSET		0x14a000
+#define IPU7_PS_MMU_SRT_RD_STREAM_NUM		40
+#define IPU7_PS_MMU_SRT_RD_L1_BLOCKNR_REG	0x54
+#define IPU7_PS_MMU_SRT_RD_L2_BLOCKNR_REG	0xf4
+
+#define IPU7_PS_MMU_SRT_WR_OFFSET		0x14b000
+#define IPU7_PS_MMU_SRT_WR_STREAM_NUM		40
+#define IPU7_PS_MMU_SRT_WR_L1_BLOCKNR_REG	0x54
+#define IPU7_PS_MMU_SRT_WR_L2_BLOCKNR_REG	0xf4
+
+#define IPU7_IS_UAO_UC_RD_OFFSET		0x27c000
+#define IPU7_IS_UAO_UC_RD_PLANENUM		4
+
+#define IPU7_IS_UAO_UC_WR_OFFSET		0x27d000
+#define IPU7_IS_UAO_UC_WR_PLANENUM		4
+
+#define IPU7_IS_UAO_M0_WR_OFFSET		0x27e000
+#define IPU7_IS_UAO_M0_WR_PLANENUM		8
+
+#define IPU7_IS_UAO_M1_WR_OFFSET		0x27f000
+#define IPU7_IS_UAO_M1_WR_PLANENUM		16
+
+#define IPU7_PS_UAO_FW_RD_OFFSET		0x156000
+#define IPU7_PS_UAO_FW_RD_PLANENUM		20
+
+#define IPU7_PS_UAO_FW_WR_OFFSET		0x157000
+#define IPU7_PS_UAO_FW_WR_PLANENUM		16
+
+#define IPU7_PS_UAO_SRT_RD_OFFSET		0x154000
+#define IPU7_PS_UAO_SRT_RD_PLANENUM		40
+
+#define IPU7_PS_UAO_SRT_WR_OFFSET		0x155000
+#define IPU7_PS_UAO_SRT_WR_PLANENUM		40
+
+#define IPU7_IS_ZLX_UC_RD_OFFSET		0x278000
+#define IPU7_IS_ZLX_UC_WR_OFFSET		0x279000
+#define IPU7_IS_ZLX_M0_OFFSET			0x27a000
+#define IPU7_IS_ZLX_M1_OFFSET			0x27b000
+#define IPU7_IS_ZLX_UC_RD_NUM			4
+#define IPU7_IS_ZLX_UC_WR_NUM			4
+#define IPU7_IS_ZLX_M0_NUM			8
+#define IPU7_IS_ZLX_M1_NUM			16
+
+#define IPU7_PS_ZLX_DATA_RD_OFFSET		0x14e000
+#define IPU7_PS_ZLX_DATA_WR_OFFSET		0x14f000
+#define IPU7_PS_ZLX_FW_RD_OFFSET		0x150000
+#define IPU7_PS_ZLX_FW_WR_OFFSET		0x151000
+#define IPU7_PS_ZLX_DATA_RD_NUM			32
+#define IPU7_PS_ZLX_DATA_WR_NUM			32
+#define IPU7_PS_ZLX_FW_RD_NUM			16
+#define IPU7_PS_ZLX_FW_WR_NUM			10
+
+/* IPU7P5 */
+/* IS MMU Cmd RD */
+#define IPU7P5_IS_MMU_FW_RD_OFFSET		0x274000
+#define IPU7P5_IS_MMU_FW_RD_STREAM_NUM		3
+#define IPU7P5_IS_MMU_FW_RD_L1_BLOCKNR_REG	0x54
+#define IPU7P5_IS_MMU_FW_RD_L2_BLOCKNR_REG	0x60
+
+/* IS MMU Cmd WR */
+#define IPU7P5_IS_MMU_FW_WR_OFFSET		0x275000
+#define IPU7P5_IS_MMU_FW_WR_STREAM_NUM		3
+#define IPU7P5_IS_MMU_FW_WR_L1_BLOCKNR_REG	0x54
+#define IPU7P5_IS_MMU_FW_WR_L2_BLOCKNR_REG	0x60
+
+/* IS MMU Data WR Snoop */
+#define IPU7P5_IS_MMU_M0_OFFSET			0x276000
+#define IPU7P5_IS_MMU_M0_STREAM_NUM		16
+#define IPU7P5_IS_MMU_M0_L1_BLOCKNR_REG		0x54
+#define IPU7P5_IS_MMU_M0_L2_BLOCKNR_REG		0x94
+
+/* IS MMU Data WR ISOC */
+#define IPU7P5_IS_MMU_M1_OFFSET			0x277000
+#define IPU7P5_IS_MMU_M1_STREAM_NUM		16
+#define IPU7P5_IS_MMU_M1_L1_BLOCKNR_REG		0x54
+#define IPU7P5_IS_MMU_M1_L2_BLOCKNR_REG		0x94
+
+/* PS MMU FW RD */
+#define IPU7P5_PS_MMU_FW_RD_OFFSET		0x148000
+#define IPU7P5_PS_MMU_FW_RD_STREAM_NUM		16
+#define IPU7P5_PS_MMU_FW_RD_L1_BLOCKNR_REG	0x54
+#define IPU7P5_PS_MMU_FW_RD_L2_BLOCKNR_REG	0x94
+
+/* PS MMU FW WR */
+#define IPU7P5_PS_MMU_FW_WR_OFFSET		0x149000
+#define IPU7P5_PS_MMU_FW_WR_STREAM_NUM		10
+#define IPU7P5_PS_MMU_FW_WR_L1_BLOCKNR_REG	0x54
+#define IPU7P5_PS_MMU_FW_WR_L2_BLOCKNR_REG	0x7c
+
+/* PS MMU FW Data RD VC0 */
+#define IPU7P5_PS_MMU_SRT_RD_OFFSET		0x14a000
+#define IPU7P5_PS_MMU_SRT_RD_STREAM_NUM		22
+#define IPU7P5_PS_MMU_SRT_RD_L1_BLOCKNR_REG	0x54
+#define IPU7P5_PS_MMU_SRT_RD_L2_BLOCKNR_REG	0xac
+
+/* PS MMU FW Data WR VC0 */
+#define IPU7P5_PS_MMU_SRT_WR_OFFSET		0x14b000
+#define IPU7P5_PS_MMU_SRT_WR_STREAM_NUM		32
+#define IPU7P5_PS_MMU_SRT_WR_L1_BLOCKNR_REG	0x54
+#define IPU7P5_PS_MMU_SRT_WR_L2_BLOCKNR_REG	0xd4
+
+/* IS UAO UC RD */
+#define IPU7P5_IS_UAO_UC_RD_OFFSET		0x27c000
+#define IPU7P5_IS_UAO_UC_RD_PLANENUM		4
+
+/* IS UAO UC WR */
+#define IPU7P5_IS_UAO_UC_WR_OFFSET		0x27d000
+#define IPU7P5_IS_UAO_UC_WR_PLANENUM		4
+
+/* IS UAO M0 WR */
+#define IPU7P5_IS_UAO_M0_WR_OFFSET		0x27e000
+#define IPU7P5_IS_UAO_M0_WR_PLANENUM		16
+
+/* IS UAO M1 WR */
+#define IPU7P5_IS_UAO_M1_WR_OFFSET		0x27f000
+#define IPU7P5_IS_UAO_M1_WR_PLANENUM		16
+
+/* PS UAO FW RD */
+#define IPU7P5_PS_UAO_FW_RD_OFFSET		0x156000
+#define IPU7P5_PS_UAO_FW_RD_PLANENUM		16
+
+/* PS UAO FW WR */
+#define IPU7P5_PS_UAO_FW_WR_OFFSET		0x157000
+#define IPU7P5_PS_UAO_FW_WR_PLANENUM		10
+
+/* PS UAO SRT RD */
+#define IPU7P5_PS_UAO_SRT_RD_OFFSET		0x154000
+#define IPU7P5_PS_UAO_SRT_RD_PLANENUM		22
+
+/* PS UAO SRT WR */
+#define IPU7P5_PS_UAO_SRT_WR_OFFSET		0x155000
+#define IPU7P5_PS_UAO_SRT_WR_PLANENUM		32
+
+#define IPU7P5_IS_ZLX_UC_RD_OFFSET		0x278000
+#define IPU7P5_IS_ZLX_UC_WR_OFFSET		0x279000
+#define IPU7P5_IS_ZLX_M0_OFFSET			0x27a000
+#define IPU7P5_IS_ZLX_M1_OFFSET			0x27b000
+#define IPU7P5_IS_ZLX_UC_RD_NUM			4
+#define IPU7P5_IS_ZLX_UC_WR_NUM			4
+#define IPU7P5_IS_ZLX_M0_NUM			16
+#define IPU7P5_IS_ZLX_M1_NUM			16
+
+#define IPU7P5_PS_ZLX_DATA_RD_OFFSET		0x14e000
+#define IPU7P5_PS_ZLX_DATA_WR_OFFSET		0x14f000
+#define IPU7P5_PS_ZLX_FW_RD_OFFSET		0x150000
+#define IPU7P5_PS_ZLX_FW_WR_OFFSET		0x151000
+#define IPU7P5_PS_ZLX_DATA_RD_NUM		22
+#define IPU7P5_PS_ZLX_DATA_WR_NUM		32
+#define IPU7P5_PS_ZLX_FW_RD_NUM			16
+#define IPU7P5_PS_ZLX_FW_WR_NUM			10
+
+struct ipu7_mmu_hw {
+	char name[32];
+
+	void __iomem *base;
+	void __iomem *zlx_base;
+	void __iomem *uao_base;
+
+	u32 offset;
+	u32 zlx_offset;
+	u32 uao_offset;
+
+	u32 info_bits;
+	u32 refill;
+	u32 collapse_en_bitmap;
+	u32 at_sp_arb_cfg;
+
+	u32 l1_block;
+	u32 l2_block;
+
+	u8 nr_l1streams;
+	u8 nr_l2streams;
+	u32 l1_block_sz[IPU7_MMU_MAX_TLB_L1_STREAMS];
+	u32 l2_block_sz[IPU7_MMU_MAX_TLB_L2_STREAMS];
+
+	u8 zlx_nr;
+	u32 zlx_axi_pool[IPU7_ZLX_POOL_NUM];
+	u32 zlx_en[IPU7_ZLX_MAX_NUM];
+	u32 zlx_conf[IPU7_ZLX_MAX_NUM];
+
+	u32 uao_p_num;
+	u32 uao_p2tlb[IPU7_UAO_PLANE_MAX_NUM];
+};
+
+struct ipu7_mmu_hwdata {
+	struct ipu7_mmu_hw *hwdata;
+	unsigned int nr_mmus;
+};
+
+#endif
diff --git a/drivers/media/pci/intel/ipu6/ipu7-platform-regs.h b/drivers/media/pci/intel/ipu6/ipu7-platform-regs.h
new file mode 100644
index 00000000000000..39cd5c0890e5f1
--- /dev/null
+++ b/drivers/media/pci/intel/ipu6/ipu7-platform-regs.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2026 Intel Corporation */
+
+#ifndef IPU7_PLATFORM_REGS_H
+#define IPU7_PLATFORM_REGS_H
+
+#define IPU7_IS_UC_CTRL_BASE                   0x230000
+#define IPU7_ISYS_DMEM_OFFSET                  0x200000
+#define IPU7_PS_UC_CTRL_BASE                   0x130000
+#define IPU7_PSYS_DMEM_OFFSET                  0x100000
+
+#define IPU7_IS_IO_BASE				0x280000
+#define IPU7_IS_IO_CSI2_GPREGS_BASE		(IPU7_IS_IO_BASE + 0x53400)
+
+#define IPU7_IS_IO_CSI2_LEGACY_IRQ_CTRL_BASE	(IPU7_IS_IO_BASE + 0x49000)
+#define IPU7_IRQ_CTL_EDGE			0x0
+#define IPU7_IRQ_CTL_MASK			0x4
+#define IPU7_IRQ_CTL_STATUS			0x8
+#define IPU7_IRQ_CTL_CLEAR			0xc
+#define IPU7_IRQ_CTL_ENABLE			0x10
+#define IPU7_CSI_RX_LEGACY_IRQ_MASK		0x1ff
+
+#define IPU7_TO_SW_IRQ_CNTL_EDGE		0x4000
+#define IPU7_TO_SW_IRQ_CNTL_MASK_N		0x4004
+#define IPU7_TO_SW_IRQ_CNTL_STATUS		0x4008
+#define IPU7_TO_SW_IRQ_CNTL_CLEAR		0x400c
+#define IPU7_TO_SW_IRQ_CNTL_ENABLE		0x4010
+#define IPU7_IS_UC_TO_SW_IRQ_MASK		0xf
+#define IPU7_TO_SW_IRQ_FW			BIT(0)
+#define IPU7_REG_PRINTF_AXI_CNTL		0x301c
+
+#endif
diff --git a/drivers/media/pci/intel/ivsc/mei_ace.c b/drivers/media/pci/intel/ivsc/mei_ace.c
index b306a320b70ffa..bb57656fc85a39 100644
--- a/drivers/media/pci/intel/ivsc/mei_ace.c
+++ b/drivers/media/pci/intel/ivsc/mei_ace.c
@@ -414,13 +414,13 @@ static int mei_ace_setup_dev_link(struct mei_ace *ace)
 	/* setup link between mei_ace and mei_csi */
 	ace->csi_link = device_link_add(csi_dev, dev, DL_FLAG_PM_RUNTIME |
 					DL_FLAG_RPM_ACTIVE | DL_FLAG_STATELESS);
-	put_device(csi_dev);
 	if (!ace->csi_link) {
 		ret = -EINVAL;
 		dev_err(dev, "failed to link to %s\n", dev_name(csi_dev));
-		goto err;
+		goto err_put;
 	}
 
+	put_device(csi_dev);
 	ace->csi_dev = csi_dev;
 
 	return 0;
diff --git a/drivers/media/pci/intel/ivsc/mei_csi.c b/drivers/media/pci/intel/ivsc/mei_csi.c
index c2917e1563452d..005b1414a252c3 100644
--- a/drivers/media/pci/intel/ivsc/mei_csi.c
+++ b/drivers/media/pci/intel/ivsc/mei_csi.c
@@ -27,7 +27,6 @@
 #include <linux/workqueue.h>
 
 #include <media/ipu-bridge.h>
-#include <media/ipu6-pci-table.h>
 #include <media/v4l2-async.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-fwnode.h>
@@ -338,6 +337,7 @@ static int mei_csi_init_state(struct v4l2_subdev *sd,
 }
 
 static int mei_csi_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *format)
 {
@@ -644,13 +644,9 @@ static int mei_csi_probe(struct mei_cl_device *cldev,
 	struct device *dev = &cldev->dev;
 	struct pci_dev *ipu;
 	struct mei_csi *csi;
-	unsigned int i;
 	int ret;
 
-	for (i = 0, ipu = NULL; !ipu && ipu6_pci_tbl[i].vendor; i++)
-		ipu = pci_get_device(ipu6_pci_tbl[i].vendor,
-				     ipu6_pci_tbl[i].device, NULL);
-
+	ipu = ipu_bridge_get_ipu6();
 	if (!ipu)
 		return -ENODEV;
 
diff --git a/drivers/media/pci/ivtv/ivtv-controls.c b/drivers/media/pci/ivtv/ivtv-controls.c
index f087a12c4ebd2b..816e634de1eb28 100644
--- a/drivers/media/pci/ivtv/ivtv-controls.c
+++ b/drivers/media/pci/ivtv/ivtv-controls.c
@@ -60,7 +60,7 @@ static int ivtv_s_video_encoding(struct cx2341x_handler *cxhdl, u32 val)
 	format.format.width = cxhdl->width / (is_mpeg1 ? 2 : 1);
 	format.format.height = cxhdl->height;
 	format.format.code = MEDIA_BUS_FMT_FIXED;
-	v4l2_subdev_call(itv->sd_video, pad, set_fmt, NULL, &format);
+	v4l2_subdev_call(itv->sd_video, pad, set_fmt, NULL, NULL, &format);
 	return 0;
 }
 
diff --git a/drivers/media/pci/ivtv/ivtv-driver.h b/drivers/media/pci/ivtv/ivtv-driver.h
index f1f18911332e84..000e8beecc7ca8 100644
--- a/drivers/media/pci/ivtv/ivtv-driver.h
+++ b/drivers/media/pci/ivtv/ivtv-driver.h
@@ -30,13 +30,13 @@
 #include <linux/sched/signal.h>
 #include <linux/fs.h>
 #include <linux/pci.h>
+#include <linux/highmem.h>
 #include <linux/interrupt.h>
 #include <linux/spinlock.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <linux/list.h>
 #include <linux/unistd.h>
-#include <linux/pagemap.h>
 #include <linux/scatterlist.h>
 #include <linux/kthread.h>
 #include <linux/mutex.h>
diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c
index fc95f0bf48d557..aba55fcc1e2201 100644
--- a/drivers/media/pci/ivtv/ivtv-ioctl.c
+++ b/drivers/media/pci/ivtv/ivtv-ioctl.c
@@ -587,7 +587,7 @@ static int ivtv_s_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f
 	format.format.width = fmt->fmt.pix.width;
 	format.format.height = h;
 	format.format.code = MEDIA_BUS_FMT_FIXED;
-	v4l2_subdev_call(itv->sd_video, pad, set_fmt, NULL, &format);
+	v4l2_subdev_call(itv->sd_video, pad, set_fmt, NULL, NULL, &format);
 	return ivtv_g_fmt_vid_cap(file, fh, fmt);
 }
 
diff --git a/drivers/media/pci/saa7134/saa7134-core.c b/drivers/media/pci/saa7134/saa7134-core.c
index 2f5b258d682b21..507ad23776e0a2 100644
--- a/drivers/media/pci/saa7134/saa7134-core.c
+++ b/drivers/media/pci/saa7134/saa7134-core.c
@@ -1368,7 +1368,7 @@ static int __maybe_unused saa7134_buffer_requeue(struct saa7134_dev *dev,
 	return 0;
 }
 
-static int __maybe_unused saa7134_suspend(struct device *dev_d)
+static int saa7134_suspend(struct device *dev_d)
 {
 	struct pci_dev *pci_dev = to_pci_dev(dev_d);
 	struct v4l2_device *v4l2_dev = pci_get_drvdata(pci_dev);
@@ -1400,7 +1400,7 @@ static int __maybe_unused saa7134_suspend(struct device *dev_d)
 	return 0;
 }
 
-static int __maybe_unused saa7134_resume(struct device *dev_d)
+static int saa7134_resume(struct device *dev_d)
 {
 	struct v4l2_device *v4l2_dev = dev_get_drvdata(dev_d);
 	struct saa7134_dev *dev = container_of(v4l2_dev, struct saa7134_dev, v4l2_dev);
@@ -1484,14 +1484,14 @@ EXPORT_SYMBOL(saa7134_ts_unregister);
 
 /* ----------------------------------------------------------- */
 
-static SIMPLE_DEV_PM_OPS(saa7134_pm_ops, saa7134_suspend, saa7134_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(saa7134_pm_ops, saa7134_suspend, saa7134_resume);
 
 static struct pci_driver saa7134_pci_driver = {
 	.name     = "saa7134",
 	.id_table = saa7134_pci_tbl,
 	.probe    = saa7134_initdev,
 	.remove   = saa7134_finidev,
-	.driver.pm = &saa7134_pm_ops,
+	.driver.pm = pm_sleep_ptr(&saa7134_pm_ops),
 };
 
 static int __init saa7134_init(void)
diff --git a/drivers/media/pci/saa7134/saa7134-empress.c b/drivers/media/pci/saa7134/saa7134-empress.c
index 8c4f70e4177d18..d04a68bb05d86c 100644
--- a/drivers/media/pci/saa7134/saa7134-empress.c
+++ b/drivers/media/pci/saa7134/saa7134-empress.c
@@ -122,7 +122,7 @@ static int empress_s_fmt_vid_cap(struct file *file, void *priv,
 	};
 
 	v4l2_fill_mbus_format(&format.format, &f->fmt.pix, MEDIA_BUS_FMT_FIXED);
-	saa_call_all(dev, pad, set_fmt, NULL, &format);
+	saa_call_all(dev, pad, set_fmt, NULL, NULL, &format);
 	v4l2_fill_pix_format(&f->fmt.pix, &format.format);
 
 	f->fmt.pix.pixelformat  = V4L2_PIX_FMT_MPEG;
@@ -145,7 +145,7 @@ static int empress_try_fmt_vid_cap(struct file *file, void *priv,
 	};
 
 	v4l2_fill_mbus_format(&format.format, &f->fmt.pix, MEDIA_BUS_FMT_FIXED);
-	saa_call_all(dev, pad, set_fmt, &pad_state, &format);
+	saa_call_all(dev, pad, set_fmt, NULL, &pad_state, &format);
 	v4l2_fill_pix_format(&f->fmt.pix, &format.format);
 
 	f->fmt.pix.pixelformat  = V4L2_PIX_FMT_MPEG;
diff --git a/drivers/media/pci/saa7134/saa7134-input.c b/drivers/media/pci/saa7134/saa7134-input.c
index 7f6680de315645..b2a2372d149a38 100644
--- a/drivers/media/pci/saa7134/saa7134-input.c
+++ b/drivers/media/pci/saa7134/saa7134-input.c
@@ -769,7 +769,8 @@ int saa7134_input_init1(struct saa7134_dev *dev)
 	}
 
 	ir = kzalloc_obj(*ir);
-	rc = rc_allocate_device(RC_DRIVER_SCANCODE);
+	rc = rc_allocate_device(raw_decode ?
+				RC_DRIVER_IR_RAW : RC_DRIVER_SCANCODE);
 	if (!ir || !rc) {
 		err = -ENOMEM;
 		goto err_out_free;
@@ -792,10 +793,8 @@ int saa7134_input_init1(struct saa7134_dev *dev)
 	rc->priv = dev;
 	rc->open = saa7134_ir_open;
 	rc->close = saa7134_ir_close;
-	if (raw_decode) {
-		rc->driver_type = RC_DRIVER_IR_RAW;
+	if (raw_decode)
 		rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
-	}
 
 	rc->device_name = saa7134_boards[dev->board].name;
 	rc->input_phys = ir->phys;
diff --git a/drivers/media/pci/saa7146/mxb.c b/drivers/media/pci/saa7146/mxb.c
index d759e8a87e247f..129d41c1862b9e 100644
--- a/drivers/media/pci/saa7146/mxb.c
+++ b/drivers/media/pci/saa7146/mxb.c
@@ -37,7 +37,7 @@
 /* global variable */
 static int mxb_num;
 
-/* initial frequence the tuner will be tuned to.
+/* initial frequency the tuner will be tuned to.
    in verden (lower saxony, germany) 4148 is a
    channel called "phoenix" */
 static int freq = 4148;
@@ -429,7 +429,7 @@ static int mxb_init_done(struct saa7146_dev* dev)
 	/* some stuff is done via direct write to the registers */
 
 	/* this is ugly, but because of the fact that this is completely
-	   hardware dependend, it should be done directly... */
+	   hardware dependent, it should be done directly... */
 	saa7146_write(dev, DD1_STREAM_B,	0x00000000);
 	saa7146_write(dev, DD1_INIT,		0x02000200);
 	saa7146_write(dev, MC2, (MASK_09 | MASK_25 | MASK_10 | MASK_26));
diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c
index ac5eb6b923e213..debcfaaf36dcb5 100644
--- a/drivers/media/pci/saa7164/saa7164-core.c
+++ b/drivers/media/pci/saa7164/saa7164-core.c
@@ -742,35 +742,6 @@ u32 saa7164_getcurrentfirmwareversion(struct saa7164_dev *dev)
 	return reg;
 }
 
-/* TODO: Debugging func, remove */
-void saa7164_dumpregs(struct saa7164_dev *dev, u32 addr)
-{
-	int i;
-
-	dprintk(1, "--------------------> 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f\n");
-
-	for (i = 0; i < 0x100; i += 16)
-		dprintk(1, "region0[0x%08x] = %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
-			i,
-			(u8)saa7164_readb(addr + i + 0),
-			(u8)saa7164_readb(addr + i + 1),
-			(u8)saa7164_readb(addr + i + 2),
-			(u8)saa7164_readb(addr + i + 3),
-			(u8)saa7164_readb(addr + i + 4),
-			(u8)saa7164_readb(addr + i + 5),
-			(u8)saa7164_readb(addr + i + 6),
-			(u8)saa7164_readb(addr + i + 7),
-			(u8)saa7164_readb(addr + i + 8),
-			(u8)saa7164_readb(addr + i + 9),
-			(u8)saa7164_readb(addr + i + 10),
-			(u8)saa7164_readb(addr + i + 11),
-			(u8)saa7164_readb(addr + i + 12),
-			(u8)saa7164_readb(addr + i + 13),
-			(u8)saa7164_readb(addr + i + 14),
-			(u8)saa7164_readb(addr + i + 15)
-			);
-}
-
 static void saa7164_dump_hwdesc(struct saa7164_dev *dev)
 {
 	dprintk(1, "@0x%p hwdesc sizeof(struct tmComResHWDescr) = %d bytes\n",
@@ -856,7 +827,7 @@ static void saa7164_get_descriptors(struct saa7164_dev *dev)
 		saa7164_dump_hwdesc(dev);
 
 	if (dev->intfdesc.bLength != sizeof(struct tmComResInterfaceDescr)) {
-		printk(KERN_ERR "struct struct tmComResInterfaceDescr is mangled\n");
+		printk(KERN_ERR "struct tmComResInterfaceDescr is mangled\n");
 		printk(KERN_ERR "Need %x got %d\n", dev->intfdesc.bLength,
 			(u32)sizeof(struct tmComResInterfaceDescr));
 	} else
@@ -1344,7 +1315,6 @@ static int saa7164_initdev(struct pci_dev *pci_dev,
 		}
 
 		saa7164_get_descriptors(dev);
-		saa7164_dumpregs(dev, 0);
 		saa7164_getcurrentfirmwareversion(dev);
 		saa7164_getfirmwarestatus(dev);
 		err = saa7164_bus_setup(dev);
diff --git a/drivers/media/pci/saa7164/saa7164.h b/drivers/media/pci/saa7164/saa7164.h
index 94e987e7b5e5d4..53ef9c3343baaa 100644
--- a/drivers/media/pci/saa7164/saa7164.h
+++ b/drivers/media/pci/saa7164/saa7164.h
@@ -490,7 +490,6 @@ extern unsigned int vbi_buffers;
 
 /* ----------------------------------------------------------- */
 /* saa7164-core.c                                              */
-void saa7164_dumpregs(struct saa7164_dev *dev, u32 addr);
 void saa7164_getfirmwarestatus(struct saa7164_dev *dev);
 u32 saa7164_getcurrentfirmwareversion(struct saa7164_dev *dev);
 void saa7164_histogram_update(struct saa7164_histogram *hg, u32 val);
diff --git a/drivers/media/pci/tw68/tw68-core.c b/drivers/media/pci/tw68/tw68-core.c
index 509d7ddec15059..0909b2d9fcdd12 100644
--- a/drivers/media/pci/tw68/tw68-core.c
+++ b/drivers/media/pci/tw68/tw68-core.c
@@ -228,7 +228,7 @@ static int tw68_initdev(struct pci_dev *pci_dev,
 
 	/* pci init */
 	dev->pci = pci_dev;
-	if (pci_enable_device(pci_dev)) {
+	if (pcim_enable_device(pci_dev)) {
 		err = -EIO;
 		goto fail1;
 	}
@@ -359,7 +359,7 @@ static void tw68_finidev(struct pci_dev *pci_dev)
 	v4l2_device_unregister(&dev->v4l2_dev);
 }
 
-static int __maybe_unused tw68_suspend(struct device *dev_d)
+static int tw68_suspend(struct device *dev_d)
 {
 	struct pci_dev *pci_dev = to_pci_dev(dev_d);
 	struct v4l2_device *v4l2_dev = pci_get_drvdata(pci_dev);
@@ -377,7 +377,7 @@ static int __maybe_unused tw68_suspend(struct device *dev_d)
 	return 0;
 }
 
-static int __maybe_unused tw68_resume(struct device *dev_d)
+static int tw68_resume(struct device *dev_d)
 {
 	struct v4l2_device *v4l2_dev = dev_get_drvdata(dev_d);
 	struct tw68_dev *dev = container_of(v4l2_dev,
@@ -405,14 +405,14 @@ static int __maybe_unused tw68_resume(struct device *dev_d)
 
 /* ----------------------------------------------------------- */
 
-static SIMPLE_DEV_PM_OPS(tw68_pm_ops, tw68_suspend, tw68_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(tw68_pm_ops, tw68_suspend, tw68_resume);
 
 static struct pci_driver tw68_pci_driver = {
 	.name	   = "tw68",
 	.id_table  = tw68_pci_tbl,
 	.probe	   = tw68_initdev,
 	.remove	   = tw68_finidev,
-	.driver.pm = &tw68_pm_ops,
+	.driver.pm = pm_sleep_ptr(&tw68_pm_ops),
 };
 
 module_pci_driver(tw68_pci_driver);
diff --git a/drivers/media/platform/amd/isp4/isp4_subdev.c b/drivers/media/platform/amd/isp4/isp4_subdev.c
index 6716ab9c128a01..a108d90ace2dbe 100644
--- a/drivers/media/platform/amd/isp4/isp4_subdev.c
+++ b/drivers/media/platform/amd/isp4/isp4_subdev.c
@@ -906,6 +906,7 @@ static const struct v4l2_subdev_video_ops isp4sd_video_ops = {
 };
 
 static int isp4sd_set_pad_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/amd/isp4/isp4_video.c b/drivers/media/platform/amd/isp4/isp4_video.c
index 0cebb39f98e1da..856a2a0b4a1253 100644
--- a/drivers/media/platform/amd/isp4/isp4_video.c
+++ b/drivers/media/platform/amd/isp4/isp4_video.c
@@ -240,7 +240,7 @@ static int isp4vid_set_fmt_2_isp(struct v4l2_subdev *sdev,
 	fmt.pad = ISP4VID_PAD_VIDEO_OUTPUT;
 	fmt.format.width = pix_fmt->width;
 	fmt.format.height = pix_fmt->height;
-	return v4l2_subdev_call(sdev, pad, set_fmt, NULL, &fmt);
+	return v4l2_subdev_call(sdev, pad, set_fmt, NULL, NULL, &fmt);
 }
 
 static int isp4vid_s_fmt_vid_cap(struct file *file, void *priv,
diff --git a/drivers/media/platform/amlogic/c3/isp/c3-isp-core.c b/drivers/media/platform/amlogic/c3/isp/c3-isp-core.c
index ff6413fff889f1..553c808f8f3b81 100644
--- a/drivers/media/platform/amlogic/c3/isp/c3-isp-core.c
+++ b/drivers/media/platform/amlogic/c3/isp/c3-isp-core.c
@@ -461,6 +461,7 @@ static void c3_isp_core_set_source_fmt(struct v4l2_subdev_state *state,
 }
 
 static int c3_isp_core_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *state,
 			       struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/amlogic/c3/isp/c3-isp-resizer.c b/drivers/media/platform/amlogic/c3/isp/c3-isp-resizer.c
index 453a889e0b27ba..1f9c16eb08423a 100644
--- a/drivers/media/platform/amlogic/c3/isp/c3-isp-resizer.c
+++ b/drivers/media/platform/amlogic/c3/isp/c3-isp-resizer.c
@@ -621,6 +621,7 @@ static void c3_isp_rsz_set_source_fmt(struct v4l2_subdev_state *state,
 }
 
 static int c3_isp_rsz_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *state,
 			      struct v4l2_subdev_format *format)
 {
@@ -633,6 +634,7 @@ static int c3_isp_rsz_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int c3_isp_rsz_get_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -674,6 +676,7 @@ static int c3_isp_rsz_get_selection(struct v4l2_subdev *sd,
 }
 
 static int c3_isp_rsz_set_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/amlogic/c3/mipi-adapter/c3-mipi-adap.c b/drivers/media/platform/amlogic/c3/mipi-adapter/c3-mipi-adap.c
index 4bd98fb9c7e9b0..6f75112723b542 100644
--- a/drivers/media/platform/amlogic/c3/mipi-adapter/c3-mipi-adap.c
+++ b/drivers/media/platform/amlogic/c3/mipi-adapter/c3-mipi-adap.c
@@ -521,6 +521,7 @@ static int c3_mipi_adap_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int c3_mipi_adap_set_fmt(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/amlogic/c3/mipi-csi2/c3-mipi-csi2.c b/drivers/media/platform/amlogic/c3/mipi-csi2/c3-mipi-csi2.c
index b9e4ef3fc308b1..0c399665f7faf0 100644
--- a/drivers/media/platform/amlogic/c3/mipi-csi2/c3-mipi-csi2.c
+++ b/drivers/media/platform/amlogic/c3/mipi-csi2/c3-mipi-csi2.c
@@ -491,6 +491,7 @@ static int c3_mipi_csi_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int c3_mipi_csi_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *state,
 			       struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-isp.c b/drivers/media/platform/arm/mali-c55/mali-c55-isp.c
index e128adf6ee37e4..c8464dec9a2162 100644
--- a/drivers/media/platform/arm/mali-c55/mali-c55-isp.c
+++ b/drivers/media/platform/arm/mali-c55/mali-c55-isp.c
@@ -203,6 +203,7 @@ static int mali_c55_isp_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_isp_set_fmt(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_format *format)
 {
@@ -265,6 +266,7 @@ static int mali_c55_isp_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_isp_get_selection(struct v4l2_subdev *sd,
+				      const struct v4l2_subdev_client_info *ci,
 				      struct v4l2_subdev_state *state,
 				      struct v4l2_subdev_selection *sel)
 {
@@ -278,6 +280,7 @@ static int mali_c55_isp_get_selection(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_isp_set_selection(struct v4l2_subdev *sd,
+				      const struct v4l2_subdev_client_info *ci,
 				      struct v4l2_subdev_state *state,
 				      struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c b/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c
index 6706939b4a9020..774352c8a9397c 100644
--- a/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c
+++ b/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c
@@ -715,6 +715,7 @@ static int mali_c55_rsz_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_rsz_set_sink_fmt(struct v4l2_subdev *sd,
+				     const struct v4l2_subdev_client_info *ci,
 				     struct v4l2_subdev_state *state,
 				     struct v4l2_subdev_format *format)
 {
@@ -773,6 +774,7 @@ static int mali_c55_rsz_set_sink_fmt(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_rsz_set_source_fmt(struct v4l2_subdev *sd,
+				       const struct v4l2_subdev_client_info *ci,
 				       struct v4l2_subdev_state *state,
 				       struct v4l2_subdev_format *format)
 {
@@ -812,6 +814,7 @@ static int mali_c55_rsz_set_source_fmt(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_rsz_set_fmt(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_format *format)
 {
@@ -826,12 +829,13 @@ static int mali_c55_rsz_set_fmt(struct v4l2_subdev *sd,
 
 	if (format->pad == MALI_C55_RSZ_SINK_PAD ||
 	    format->pad == MALI_C55_RSZ_SINK_BYPASS_PAD)
-		return mali_c55_rsz_set_sink_fmt(sd, state, format);
+		return mali_c55_rsz_set_sink_fmt(sd, ci, state, format);
 
-	return mali_c55_rsz_set_source_fmt(sd, state, format);
+	return mali_c55_rsz_set_source_fmt(sd, ci, state, format);
 }
 
 static int mali_c55_rsz_get_selection(struct v4l2_subdev *sd,
+				      const struct v4l2_subdev_client_info *ci,
 				      struct v4l2_subdev_state *state,
 				      struct v4l2_subdev_selection *sel)
 {
@@ -850,6 +854,7 @@ static int mali_c55_rsz_get_selection(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_rsz_set_crop(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -908,6 +913,7 @@ static int mali_c55_rsz_set_crop(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_rsz_set_compose(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -956,6 +962,7 @@ static int mali_c55_rsz_set_compose(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_rsz_set_selection(struct v4l2_subdev *sd,
+				      const struct v4l2_subdev_client_info *ci,
 				      struct v4l2_subdev_state *state,
 				      struct v4l2_subdev_selection *sel)
 {
@@ -963,10 +970,10 @@ static int mali_c55_rsz_set_selection(struct v4l2_subdev *sd,
 		return -EINVAL;
 
 	if (sel->target == V4L2_SEL_TGT_CROP)
-		return mali_c55_rsz_set_crop(sd, state, sel);
+		return mali_c55_rsz_set_crop(sd, ci, state, sel);
 
 	if (sel->target == V4L2_SEL_TGT_COMPOSE)
-		return mali_c55_rsz_set_compose(sd, state, sel);
+		return mali_c55_rsz_set_compose(sd, ci, state, sel);
 
 	return -EINVAL;
 }
diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c b/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c
index 894f4cf377afc5..cc390a3fc25326 100644
--- a/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c
+++ b/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c
@@ -180,6 +180,7 @@ static int mali_c55_tpg_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int mali_c55_tpg_set_fmt(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/aspeed/aspeed-video.c b/drivers/media/platform/aspeed/aspeed-video.c
index a292275f6b7b9e..7ba9314fdbd2a6 100644
--- a/drivers/media/platform/aspeed/aspeed-video.c
+++ b/drivers/media/platform/aspeed/aspeed-video.c
@@ -2267,19 +2267,19 @@ static int aspeed_video_init(struct aspeed_video *video)
 	if (rc)
 		goto err_unprepare_eclk;
 
-	of_reserved_mem_device_init(dev);
+	devm_of_reserved_mem_device_init(dev);
 
 	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 	if (rc) {
 		dev_err(dev, "Failed to set DMA mask\n");
-		goto err_release_reserved_mem;
+		goto err_unprepare_vclk;
 	}
 
 	if (!aspeed_video_alloc_buf(video, &video->jpeg,
 				    VE_JPEG_HEADER_SIZE)) {
 		dev_err(dev, "Failed to allocate DMA for JPEG header\n");
 		rc = -ENOMEM;
-		goto err_release_reserved_mem;
+		goto err_unprepare_vclk;
 	}
 	dev_info(video->dev, "alloc mem size(%d) at %pad for jpeg header\n",
 		 VE_JPEG_HEADER_SIZE, &video->jpeg.dma);
@@ -2288,8 +2288,7 @@ static int aspeed_video_init(struct aspeed_video *video)
 
 	return 0;
 
-err_release_reserved_mem:
-	of_reserved_mem_device_release(dev);
+err_unprepare_vclk:
 	clk_unprepare(video->vclk);
 err_unprepare_eclk:
 	clk_unprepare(video->eclk);
@@ -2343,7 +2342,6 @@ static int aspeed_video_probe(struct platform_device *pdev)
 	rc = aspeed_video_setup_video(video);
 	if (rc) {
 		aspeed_video_free_buf(video, &video->jpeg);
-		of_reserved_mem_device_release(&pdev->dev);
 		clk_unprepare(video->vclk);
 		clk_unprepare(video->eclk);
 		return rc;
@@ -2374,8 +2372,6 @@ static void aspeed_video_remove(struct platform_device *pdev)
 	v4l2_device_unregister(v4l2_dev);
 
 	aspeed_video_free_buf(video, &video->jpeg);
-
-	of_reserved_mem_device_release(dev);
 }
 
 static struct platform_driver aspeed_video_driver = {
diff --git a/drivers/media/platform/atmel/atmel-isi.c b/drivers/media/platform/atmel/atmel-isi.c
index a05a744cbb7569..fa33d11f736fcd 100644
--- a/drivers/media/platform/atmel/atmel-isi.c
+++ b/drivers/media/platform/atmel/atmel-isi.c
@@ -609,7 +609,7 @@ static int isi_try_fmt(struct atmel_isi *isi, struct v4l2_format *f,
 
 	isi_try_fse(isi, isi_fmt, &pad_state);
 
-	ret = v4l2_subdev_call(isi->entity.subdev, pad, set_fmt,
+	ret = v4l2_subdev_call(isi->entity.subdev, pad, set_fmt, NULL,
 			       &pad_state, &format);
 	if (ret < 0)
 		return ret;
@@ -641,7 +641,7 @@ static int isi_set_fmt(struct atmel_isi *isi, struct v4l2_format *f)
 	v4l2_fill_mbus_format(&format.format, &f->fmt.pix,
 			      current_fmt->mbus_code);
 	ret = v4l2_subdev_call(isi->entity.subdev, pad,
-			       set_fmt, NULL, &format);
+			       set_fmt, NULL, NULL, &format);
 	if (ret < 0)
 		return ret;
 
@@ -1121,10 +1121,13 @@ static void isi_graph_notify_unbind(struct v4l2_async_notifier *notifier,
 {
 	struct atmel_isi *isi = notifier_to_isi(notifier);
 
+	if (!video_is_registered(isi->vdev))
+		return;
+
 	dev_dbg(isi->dev, "Removing %s\n", video_device_node_name(isi->vdev));
 
-	/* Checks internally if vdev have been init or not */
 	video_unregister_device(isi->vdev);
+	isi->vdev = NULL;
 }
 
 static int isi_graph_notify_bound(struct v4l2_async_notifier *notifier,
@@ -1323,6 +1326,8 @@ static void atmel_isi_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	v4l2_async_nf_unregister(&isi->notifier);
 	v4l2_async_nf_cleanup(&isi->notifier);
+	if (isi->vdev)
+		video_device_release(isi->vdev);
 	v4l2_device_unregister(&isi->v4l2_dev);
 }
 
diff --git a/drivers/media/platform/broadcom/bcm2835-unicam.c b/drivers/media/platform/broadcom/bcm2835-unicam.c
index 14bb916dd7b1b1..e7c708cdd7f8bf 100644
--- a/drivers/media/platform/broadcom/bcm2835-unicam.c
+++ b/drivers/media/platform/broadcom/bcm2835-unicam.c
@@ -1327,6 +1327,7 @@ static int unicam_subdev_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int unicam_subdev_set_format(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/cadence/cdns-csi2rx.c b/drivers/media/platform/cadence/cdns-csi2rx.c
index 1ff2d8f78d5b4c..de7cce76397ab0 100644
--- a/drivers/media/platform/cadence/cdns-csi2rx.c
+++ b/drivers/media/platform/cadence/cdns-csi2rx.c
@@ -166,6 +166,10 @@ static const struct csi2rx_fmt formats[] = {
 	{ .code	= MEDIA_BUS_FMT_SGBRG10_1X10, .bpp = 10, .max_pixels = 2, },
 	{ .code	= MEDIA_BUS_FMT_SGRBG10_1X10, .bpp = 10, .max_pixels = 2, },
 	{ .code	= MEDIA_BUS_FMT_SRGGB10_1X10, .bpp = 10, .max_pixels = 2, },
+	{ .code	= MEDIA_BUS_FMT_SBGGR12_1X12, .bpp = 12, .max_pixels = 2, },
+	{ .code	= MEDIA_BUS_FMT_SGBRG12_1X12, .bpp = 12, .max_pixels = 2, },
+	{ .code	= MEDIA_BUS_FMT_SGRBG12_1X12, .bpp = 12, .max_pixels = 2, },
+	{ .code	= MEDIA_BUS_FMT_SRGGB12_1X12, .bpp = 12, .max_pixels = 2, },
 	{ .code	= MEDIA_BUS_FMT_RGB565_1X16,  .bpp = 16, .max_pixels = 1, },
 	{ .code	= MEDIA_BUS_FMT_RGB888_1X24,  .bpp = 24, .max_pixels = 1, },
 	{ .code	= MEDIA_BUS_FMT_BGR888_1X24,  .bpp = 24, .max_pixels = 1, },
@@ -622,6 +626,7 @@ static int csi2rx_set_routing(struct v4l2_subdev *subdev,
 }
 
 static int csi2rx_set_fmt(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *state,
 			  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/cadence/cdns-csi2tx.c b/drivers/media/platform/cadence/cdns-csi2tx.c
index 629b0fa838a27a..07277866dca788 100644
--- a/drivers/media/platform/cadence/cdns-csi2tx.c
+++ b/drivers/media/platform/cadence/cdns-csi2tx.c
@@ -201,6 +201,7 @@ static int csi2tx_get_pad_format(struct v4l2_subdev *subdev,
 }
 
 static int csi2tx_set_pad_format(struct v4l2_subdev *subdev,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/intel/pxa_camera.c b/drivers/media/platform/intel/pxa_camera.c
index 2269d852ab0e0f..9e1fdd963b5599 100644
--- a/drivers/media/platform/intel/pxa_camera.c
+++ b/drivers/media/platform/intel/pxa_camera.c
@@ -1819,7 +1819,7 @@ static int pxac_vidioc_try_fmt_vid_cap(struct file *filp, void *priv,
 			      pixfmt == V4L2_PIX_FMT_YUV422P ? 4 : 0);
 
 	v4l2_fill_mbus_format(mf, pix, xlate->code);
-	ret = sensor_call(pcdev, pad, set_fmt, &pad_state, &format);
+	ret = sensor_call(pcdev, pad, set_fmt, NULL, &pad_state, &format);
 	if (ret < 0)
 		return ret;
 
@@ -1882,7 +1882,7 @@ static int pxac_vidioc_s_fmt_vid_cap(struct file *filp, void *priv,
 	xlate = pxa_mbus_xlate_by_fourcc(pcdev->user_formats,
 					 pix->pixelformat);
 	v4l2_fill_mbus_format(&format.format, pix, xlate->code);
-	ret = sensor_call(pcdev, pad, set_fmt, NULL, &format);
+	ret = sensor_call(pcdev, pad, set_fmt, NULL, NULL, &format);
 	if (ret < 0) {
 		dev_warn(pcdev_to_dev(pcdev),
 			 "Failed to configure for format %x\n",
@@ -2091,7 +2091,7 @@ static int pxa_camera_sensor_bound(struct v4l2_async_notifier *notifier,
 	if (err)
 		goto out;
 
-	err = sensor_call(pcdev, pad, set_fmt, NULL, &format);
+	err = sensor_call(pcdev, pad, set_fmt, NULL, NULL, &format);
 	if (err)
 		goto out_sensor_poweroff;
 
diff --git a/drivers/media/platform/m2m-deinterlace.c b/drivers/media/platform/m2m-deinterlace.c
index 1d0e70eaea3d01..9dcc4bd6cbdd58 100644
--- a/drivers/media/platform/m2m-deinterlace.c
+++ b/drivers/media/platform/m2m-deinterlace.c
@@ -822,7 +822,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq,
 	q_data[V4L2_M2M_DST].width = 640;
 	q_data[V4L2_M2M_DST].height = 480;
 	q_data[V4L2_M2M_DST].sizeimage = (640 * 480 * 3) / 2;
-	q_data[V4L2_M2M_SRC].field = V4L2_FIELD_INTERLACED_TB;
+	q_data[V4L2_M2M_DST].field = V4L2_FIELD_INTERLACED_TB;
 
 	return vb2_queue_init(dst_vq);
 }
diff --git a/drivers/media/platform/marvell/Kconfig b/drivers/media/platform/marvell/Kconfig
index d31f4730f2a38b..ab58175e433c33 100644
--- a/drivers/media/platform/marvell/Kconfig
+++ b/drivers/media/platform/marvell/Kconfig
@@ -22,6 +22,7 @@ config VIDEO_MMP_CAMERA
 	depends on V4L_PLATFORM_DRIVERS
 	depends on I2C && VIDEO_DEV
 	depends on ARCH_MMP || COMPILE_TEST
+	depends on GPIOLIB || COMPILE_TEST
 	depends on COMMON_CLK
 	select VIDEO_OV7670 if MEDIA_SUBDRV_AUTOSELECT && VIDEO_CAMERA_SENSOR
 	select I2C_GPIO
diff --git a/drivers/media/platform/marvell/cafe-driver.c b/drivers/media/platform/marvell/cafe-driver.c
index 22034df6cba964..13d8faaabb502f 100644
--- a/drivers/media/platform/marvell/cafe-driver.c
+++ b/drivers/media/platform/marvell/cafe-driver.c
@@ -127,7 +127,7 @@ struct cafe_camera {
  * Debugging and related.
  */
 #define cam_err(cam, fmt, arg...) \
-	dev_err(&(cam)->pdev->dev, fmt, ##arg);
+	dev_err(&(cam)->pdev->dev, fmt, ##arg)
 #define cam_warn(cam, fmt, arg...) \
 	dev_warn(&(cam)->pdev->dev, fmt, ##arg);
 
diff --git a/drivers/media/platform/marvell/mcam-core.c b/drivers/media/platform/marvell/mcam-core.c
index b8360d37000a7d..34809d075a42c7 100644
--- a/drivers/media/platform/marvell/mcam-core.c
+++ b/drivers/media/platform/marvell/mcam-core.c
@@ -224,11 +224,11 @@ static void mcam_buffer_done(struct mcam_camera *cam, int frame,
  * Debugging and related.
  */
 #define cam_err(cam, fmt, arg...) \
-	dev_err((cam)->dev, fmt, ##arg);
+	dev_err((cam)->dev, fmt, ##arg)
 #define cam_warn(cam, fmt, arg...) \
-	dev_warn((cam)->dev, fmt, ##arg);
+	dev_warn((cam)->dev, fmt, ##arg)
 #define cam_dbg(cam, fmt, arg...) \
-	dev_dbg((cam)->dev, fmt, ##arg);
+	dev_dbg((cam)->dev, fmt, ##arg)
 
 
 /*
@@ -406,6 +406,8 @@ static void mcam_free_dma_bufs(struct mcam_camera *cam)
 {
 	int i;
 
+	cancel_work_sync(&cam->s_bh_work);
+
 	for (i = 0; i < cam->nbufs; i++) {
 		dma_free_coherent(cam->dev, cam->dma_buf_size,
 				cam->dma_bufs[i], cam->dma_handles[i]);
@@ -1022,7 +1024,7 @@ static int mcam_cam_configure(struct mcam_camera *cam)
 	v4l2_fill_mbus_format(&format.format, &cam->pix_format, cam->mbus_code);
 	ret = sensor_call(cam, core, init, 0);
 	if (ret == 0)
-		ret = sensor_call(cam, pad, set_fmt, NULL, &format);
+		ret = sensor_call(cam, pad, set_fmt, NULL, NULL, &format);
 	/*
 	 * OV7670 does weird things if flip is set *before* format...
 	 */
@@ -1306,7 +1308,6 @@ static int mcam_setup_vb2(struct mcam_camera *cam)
 		break;
 	case B_vmalloc:
 #ifdef MCAM_MODE_VMALLOC
-		INIT_WORK(&cam->s_bh_work, mcam_frame_work);
 		vq->ops = &mcam_vb2_ops;
 		vq->mem_ops = &vb2_vmalloc_memops;
 		cam->dma_setup = mcam_ctlr_dma_vmalloc;
@@ -1362,7 +1363,7 @@ static int mcam_vidioc_try_fmt_vid_cap(struct file *filp, void *priv,
 	f = mcam_find_format(pix->pixelformat);
 	pix->pixelformat = f->pixelformat;
 	v4l2_fill_mbus_format(&format.format, pix, f->mbus_code);
-	ret = sensor_call(cam, pad, set_fmt, &pad_state, &format);
+	ret = sensor_call(cam, pad, set_fmt, NULL, &pad_state, &format);
 	v4l2_fill_pix_format(pix, &format.format);
 	pix->bytesperline = pix->width * f->bpp;
 	switch (f->pixelformat) {
@@ -1864,6 +1865,12 @@ int mccic_register(struct mcam_camera *cam)
 		goto out;
 	}
 
+#ifdef MCAM_MODE_VMALLOC
+	/* Init before sensor bind: armed by IRQ, cancelled on probe-error paths. */
+	if (cam->buffer_mode == B_vmalloc)
+		INIT_WORK(&cam->s_bh_work, mcam_frame_work);
+#endif
+
 	mutex_init(&cam->s_mutex);
 	cam->state = S_NOTREADY;
 	mcam_set_config_needed(cam, 1);
@@ -1922,10 +1929,15 @@ void mccic_shutdown(struct mcam_camera *cam)
 	 * take it down again will wedge the machine, which is frowned
 	 * upon.
 	 */
+	mutex_lock(&cam->s_mutex);
 	if (!list_empty(&cam->vdev.fh_list)) {
 		cam_warn(cam, "Removing a device with users!\n");
+		/* Stop so the IRQ can't re-arm s_bh_work after the buffers are freed. */
+		if (cam->state == S_STREAMING)
+			mcam_ctlr_stop_dma(cam);
 		sensor_call(cam, core, s_power, 0);
 	}
+	mutex_unlock(&cam->s_mutex);
 	if (cam->buffer_mode == B_vmalloc)
 		mcam_free_dma_bufs(cam);
 	v4l2_ctrl_handler_free(&cam->ctrl_handler);
diff --git a/drivers/media/platform/marvell/mmp-driver.c b/drivers/media/platform/marvell/mmp-driver.c
index d3da7ebb4a2be6..4c31b62751a12a 100644
--- a/drivers/media/platform/marvell/mmp-driver.c
+++ b/drivers/media/platform/marvell/mmp-driver.c
@@ -25,6 +25,7 @@
 #include <linux/list.h>
 #include <linux/pm.h>
 #include <linux/clk.h>
+#include <linux/clk-provider.h>
 
 #include "mcam-core.h"
 
@@ -269,8 +270,8 @@ static int mmpcam_probe(struct platform_device *pdev)
 	/*
 	 * Add OF clock provider.
 	 */
-	ret = of_clk_add_provider(pdev->dev.of_node, of_clk_src_simple_get,
-								mcam->mclk);
+	ret = devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_simple_get,
+					  &mcam->mclk_hw);
 	if (ret) {
 		dev_err(&pdev->dev, "can't add DT clock provider\n");
 		goto out;
diff --git a/drivers/media/platform/mediatek/mdp/mtk_mdp_ipi.h b/drivers/media/platform/mediatek/mdp/mtk_mdp_ipi.h
index b810c96695c831..9744e00b006633 100644
--- a/drivers/media/platform/mediatek/mdp/mtk_mdp_ipi.h
+++ b/drivers/media/platform/mediatek/mdp/mtk_mdp_ipi.h
@@ -56,7 +56,7 @@ struct mdp_ipi_comm {
  * @ipi_id        : IPI_MDP
  * @ap_inst       : AP mtk_mdp_vpu address
  * @vpu_inst_addr : VPU MDP instance address
- * @status        : VPU exeuction result
+ * @status        : VPU execution result
  */
 struct mdp_ipi_comm_ack {
 	uint32_t msg_id;
diff --git a/drivers/media/platform/mediatek/vcodec/decoder/vdec_ipi_msg.h b/drivers/media/platform/mediatek/vcodec/decoder/vdec_ipi_msg.h
index 47070be2a99148..c2c2a3a63dfd30 100644
--- a/drivers/media/platform/mediatek/vcodec/decoder/vdec_ipi_msg.h
+++ b/drivers/media/platform/mediatek/vcodec/decoder/vdec_ipi_msg.h
@@ -53,7 +53,7 @@ struct vdec_ap_ipi_cmd {
 /**
  * struct vdec_vpu_ipi_ack - generic VPU to AP ipi command format
  * @msg_id	: vdec_ipi_msgid
- * @status	: VPU exeuction result
+ * @status	: VPU execution result
  * @ap_inst_addr	: AP video decoder instance address
  */
 struct vdec_vpu_ipi_ack {
@@ -98,7 +98,7 @@ struct vdec_ap_ipi_dec_start {
 /**
  * struct vdec_vpu_ipi_init_ack - for VPU_IPIMSG_DEC_INIT_ACK
  * @msg_id	: VPU_IPIMSG_DEC_INIT_ACK
- * @status	: VPU exeuction result
+ * @status	: VPU execution result
  * @ap_inst_addr	: AP vcodec_vpu_inst instance address
  * @vpu_inst_addr	: VPU decoder instance address
  * @vdec_abi_version:	ABI version of the firmware. Kernel can use it to
diff --git a/drivers/media/platform/microchip/microchip-csi2dc.c b/drivers/media/platform/microchip/microchip-csi2dc.c
index e69292f3b2a909..0368f3e80c0c44 100644
--- a/drivers/media/platform/microchip/microchip-csi2dc.c
+++ b/drivers/media/platform/microchip/microchip-csi2dc.c
@@ -244,6 +244,7 @@ static int csi2dc_get_fmt(struct v4l2_subdev *csi2dc_sd,
 }
 
 static int csi2dc_set_fmt(struct v4l2_subdev *csi2dc_sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *req_fmt)
 {
@@ -735,6 +736,7 @@ static int csi2dc_probe(struct platform_device *pdev)
 	return 0;
 
 csi2dc_probe_cleanup_notifier:
+	v4l2_async_nf_unregister(&csi2dc->notifier);
 	v4l2_async_nf_cleanup(&csi2dc->notifier);
 csi2dc_probe_cleanup_entity:
 	media_entity_cleanup(&csi2dc->csi2dc_sd.entity);
diff --git a/drivers/media/platform/microchip/microchip-isc-base.c b/drivers/media/platform/microchip/microchip-isc-base.c
index a7cdc743fda71d..963d5fd53b589b 100644
--- a/drivers/media/platform/microchip/microchip-isc-base.c
+++ b/drivers/media/platform/microchip/microchip-isc-base.c
@@ -8,6 +8,7 @@
  * Author: Eugen Hristev <eugen.hristev@microchip.com>
  *
  */
+#include <linux/bitfield.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/math64.h>
@@ -62,17 +63,17 @@ static inline void isc_update_awb_ctrls(struct isc_device *isc)
 	/* In here we set our actual hw pipeline config */
 
 	regmap_write(isc->regmap, ISC_WB_O_RGR,
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_R])) |
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_GR]) << 16));
+		     FIELD_PREP(ISC_WB_O_LO, ctrls->offset[ISC_HIS_CFG_MODE_R]) |
+		     FIELD_PREP(ISC_WB_O_HI, ctrls->offset[ISC_HIS_CFG_MODE_GR]));
 	regmap_write(isc->regmap, ISC_WB_O_BGB,
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_B])) |
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_GB]) << 16));
+		     FIELD_PREP(ISC_WB_O_LO, ctrls->offset[ISC_HIS_CFG_MODE_B]) |
+		     FIELD_PREP(ISC_WB_O_HI, ctrls->offset[ISC_HIS_CFG_MODE_GB]));
 	regmap_write(isc->regmap, ISC_WB_G_RGR,
-		     ctrls->gain[ISC_HIS_CFG_MODE_R] |
-		     (ctrls->gain[ISC_HIS_CFG_MODE_GR] << 16));
+		     FIELD_PREP(ISC_WB_G_LO, ctrls->gain[ISC_HIS_CFG_MODE_R]) |
+		     FIELD_PREP(ISC_WB_G_HI, ctrls->gain[ISC_HIS_CFG_MODE_GR]));
 	regmap_write(isc->regmap, ISC_WB_G_BGB,
-		     ctrls->gain[ISC_HIS_CFG_MODE_B] |
-		     (ctrls->gain[ISC_HIS_CFG_MODE_GB] << 16));
+		     FIELD_PREP(ISC_WB_G_LO, ctrls->gain[ISC_HIS_CFG_MODE_B]) |
+		     FIELD_PREP(ISC_WB_G_HI, ctrls->gain[ISC_HIS_CFG_MODE_GB]));
 }
 
 static inline void isc_reset_awb_ctrls(struct isc_device *isc)
@@ -289,8 +290,10 @@ static int isc_configure(struct isc_device *isc)
 	struct regmap *regmap = isc->regmap;
 	u32 pfe_cfg0, dcfg, mask, pipeline;
 	struct isc_subdev_entity *subdev = isc->current_subdev;
+	int ret;
 
-	pfe_cfg0 = isc->config.sd_format->pfe_cfg0_bps;
+	pfe_cfg0 = FIELD_PREP(ISC_PFE_CFG0_BPS_MASK,
+			      isc->config.sd_format->pfe_cfg0_bps);
 	pipeline = isc->config.bits_pipeline;
 
 	dcfg = isc->config.dcfg_imode | isc->dcfg;
@@ -321,7 +324,15 @@ static int isc_configure(struct isc_device *isc)
 		isc_set_histogram(isc, false);
 
 	/* Update profile */
-	return isc_update_profile(isc);
+	ret = isc_update_profile(isc);
+	if (ret) {
+		/* flush the histogram work before the clocks are gated */
+		isc_set_histogram(isc, false);
+		synchronize_irq(isc->irq);
+		cancel_work_sync(&isc->awb_work);
+	}
+
+	return ret;
 }
 
 static int isc_prepare_streaming(struct vb2_queue *vq)
@@ -425,6 +436,13 @@ static void isc_stop_streaming(struct vb2_queue *vq)
 	/* Disable DMA interrupt */
 	regmap_write(isc->regmap, ISC_INTDIS, ISC_INT_DDONE);
 
+	isc_set_histogram(isc, false);
+
+	/* let a running IRQ handler finish before the clock is disabled */
+	synchronize_irq(isc->irq);
+
+	cancel_work_sync(&isc->awb_work);
+
 	pm_runtime_put_sync(isc->dev);
 
 	/* Disable stream on the sub device */
@@ -1416,7 +1434,7 @@ static void isc_awb_work(struct work_struct *w)
 	/* streaming is not active anymore */
 	if (isc->stop) {
 		mutex_unlock(&isc->awb_mutex);
-		return;
+		goto out_pm_put;
 	}
 
 	isc_update_profile(isc);
@@ -1427,6 +1445,7 @@ static void isc_awb_work(struct work_struct *w)
 	if (ctrls->awb)
 		regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_HISREQ);
 
+out_pm_put:
 	pm_runtime_put_sync(isc->dev);
 }
 
@@ -1495,20 +1514,24 @@ static int isc_s_awb_ctrl(struct v4l2_ctrl *ctrl)
 		if (ctrl->cluster[ISC_CTRL_GB_OFF]->is_new)
 			ctrls->offset[ISC_HIS_CFG_MODE_GB] = isc->gb_off_ctrl->val;
 
-		isc_update_awb_ctrls(isc);
-
 		mutex_lock(&isc->awb_mutex);
-		if (vb2_is_streaming(&isc->vb2_vidq)) {
+		if (vb2_is_streaming(&isc->vb2_vidq) && !isc->stop) {
+			unsigned long flags;
+
 			/*
-			 * If we are streaming, we can update profile to
-			 * have the new settings in place.
+			 * awb_lock keeps the DMA done IRQ from latching a
+			 * partially written WB pipeline.
 			 */
+			spin_lock_irqsave(&isc->awb_lock, flags);
+			isc_update_awb_ctrls(isc);
+			spin_unlock_irqrestore(&isc->awb_lock, flags);
+
 			isc_update_profile(isc);
 		} else {
 			/*
-			 * The auto cluster will activate automatically this
-			 * control. This has to be deactivated when not
-			 * streaming.
+			 * Not streaming: keep the cached values for the next
+			 * stream start and deactivate the cluster-activated
+			 * do_white_balance button.
 			 */
 			v4l2_ctrl_activate(isc->do_wb_ctrl, false);
 		}
@@ -1703,7 +1726,6 @@ static void isc_async_unbind(struct v4l2_async_notifier *notifier,
 {
 	struct isc_device *isc = container_of(notifier->v4l2_dev,
 					      struct isc_device, v4l2_dev);
-	mutex_destroy(&isc->awb_mutex);
 	cancel_work_sync(&isc->awb_work);
 	video_unregister_device(&isc->video_dev);
 	v4l2_ctrl_handler_free(&isc->ctrls.handler);
@@ -1767,8 +1789,6 @@ static int isc_async_complete(struct v4l2_async_notifier *notifier)
 
 	isc->current_subdev = container_of(notifier,
 					   struct isc_subdev_entity, notifier);
-	mutex_init(&isc->lock);
-	mutex_init(&isc->awb_mutex);
 
 	init_completion(&isc->comp);
 
@@ -1787,7 +1807,7 @@ static int isc_async_complete(struct v4l2_async_notifier *notifier)
 	ret = vb2_queue_init(q);
 	if (ret < 0) {
 		dev_err(isc->dev, "vb2_queue_init() failed: %d\n", ret);
-		goto isc_async_complete_err;
+		return ret;
 	}
 
 	/* Init video dma queues */
@@ -1798,13 +1818,13 @@ static int isc_async_complete(struct v4l2_async_notifier *notifier)
 	ret = isc_set_default_fmt(isc);
 	if (ret) {
 		dev_err(isc->dev, "Could not set default format\n");
-		goto isc_async_complete_err;
+		return ret;
 	}
 
 	ret = isc_ctrl_init(isc);
 	if (ret) {
 		dev_err(isc->dev, "Init isc ctrols failed: %d\n", ret);
-		goto isc_async_complete_err;
+		return ret;
 	}
 
 	/* Register video device */
@@ -1824,7 +1844,7 @@ static int isc_async_complete(struct v4l2_async_notifier *notifier)
 	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
 	if (ret < 0) {
 		dev_err(isc->dev, "video_register_device failed: %d\n", ret);
-		goto isc_async_complete_err;
+		return ret;
 	}
 
 	ret = isc_scaler_link(isc);
@@ -1839,10 +1859,6 @@ static int isc_async_complete(struct v4l2_async_notifier *notifier)
 
 isc_async_complete_unregister_device:
 	video_unregister_device(vdev);
-
-isc_async_complete_err:
-	mutex_destroy(&isc->awb_mutex);
-	mutex_destroy(&isc->lock);
 	return ret;
 }
 
@@ -1860,6 +1876,12 @@ void microchip_isc_subdev_cleanup(struct isc_device *isc)
 	list_for_each_entry(subdev_entity, &isc->subdev_entities, list) {
 		v4l2_async_nf_unregister(&subdev_entity->notifier);
 		v4l2_async_nf_cleanup(&subdev_entity->notifier);
+		/*
+		 * Release the endpoint reference taken while parsing. It is
+		 * NULL for entities the bind loop already consumed, so this
+		 * only drops the ones left over on an early exit.
+		 */
+		of_node_put(subdev_entity->epn);
 	}
 
 	INIT_LIST_HEAD(&isc->subdev_entities);
diff --git a/drivers/media/platform/microchip/microchip-isc-clk.c b/drivers/media/platform/microchip/microchip-isc-clk.c
index 24358d804e75d7..66dc522a619081 100644
--- a/drivers/media/platform/microchip/microchip-isc-clk.c
+++ b/drivers/media/platform/microchip/microchip-isc-clk.c
@@ -98,15 +98,14 @@ static int isc_clk_is_enabled(struct clk_hw *hw)
 {
 	struct isc_clk *isc_clk = to_isc_clk(hw);
 	u32 status;
-	int ret;
 
-	ret = pm_runtime_resume_and_get(isc_clk->dev);
-	if (ret < 0)
+	/* Runs in atomic context, so must not sleep to resume the ISC. */
+	if (pm_runtime_get_if_active(isc_clk->dev) <= 0)
 		return 0;
 
 	regmap_read(isc_clk->regmap, ISC_CLKSR, &status);
 
-	pm_runtime_put_sync(isc_clk->dev);
+	pm_runtime_put(isc_clk->dev);
 
 	return status & ISC_CLK(isc_clk->id) ? 1 : 0;
 }
diff --git a/drivers/media/platform/microchip/microchip-isc-regs.h b/drivers/media/platform/microchip/microchip-isc-regs.h
index e77e1d9a1db87a..fe145b142b826d 100644
--- a/drivers/media/platform/microchip/microchip-isc-regs.h
+++ b/drivers/media/platform/microchip/microchip-isc-regs.h
@@ -31,11 +31,11 @@
 #define ISC_PFE_CFG0_MODE_PROGRESSIVE   (0x0 << 4)
 #define ISC_PFE_CFG0_MODE_MASK          GENMASK(6, 4)
 
-#define ISC_PFE_CFG0_BPS_EIGHT  (0x4 << 28)
-#define ISC_PFG_CFG0_BPS_NINE   (0x3 << 28)
-#define ISC_PFG_CFG0_BPS_TEN    (0x2 << 28)
-#define ISC_PFG_CFG0_BPS_ELEVEN (0x1 << 28)
-#define ISC_PFG_CFG0_BPS_TWELVE (0x0 << 28)
+#define ISC_PFE_CFG0_BPS_EIGHT  0x4
+#define ISC_PFE_CFG0_BPS_NINE   0x3
+#define ISC_PFE_CFG0_BPS_TEN    0x2
+#define ISC_PFE_CFG0_BPS_ELEVEN 0x1
+#define ISC_PFE_CFG0_BPS_TWELVE 0x0
 #define ISC_PFE_CFG0_BPS_MASK   GENMASK(30, 28)
 
 #define ISC_PFE_CFG0_COLEN	BIT(12)
@@ -149,6 +149,12 @@
 /* ISC White Balance Gain for B, GB Register */
 #define ISC_WB_G_BGB	0x0000006c
 
+/* Each WB offset/gain register packs two 13-bit fields, low and high */
+#define ISC_WB_O_LO	GENMASK(12, 0)	/* R or B  offset [12:0]  */
+#define ISC_WB_O_HI	GENMASK(28, 16)	/* GR or GB offset [28:16] */
+#define ISC_WB_G_LO	GENMASK(12, 0)	/* R or B  gain [12:0]  */
+#define ISC_WB_G_HI	GENMASK(28, 16)	/* GR or GB gain [28:16] */
+
 /* ISC Color Filter Array Control Register */
 #define ISC_CFA_CTRL    0x00000070
 
diff --git a/drivers/media/platform/microchip/microchip-isc-scaler.c b/drivers/media/platform/microchip/microchip-isc-scaler.c
index e83463543e2100..192532f8727163 100644
--- a/drivers/media/platform/microchip/microchip-isc-scaler.c
+++ b/drivers/media/platform/microchip/microchip-isc-scaler.c
@@ -46,6 +46,7 @@ static int isc_scaler_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int isc_scaler_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *req_fmt)
 {
@@ -124,6 +125,7 @@ static int isc_scaler_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int isc_scaler_g_sel(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/microchip/microchip-isc.h b/drivers/media/platform/microchip/microchip-isc.h
index ad4e98a1dd8fca..d7bcd74efff923 100644
--- a/drivers/media/platform/microchip/microchip-isc.h
+++ b/drivers/media/platform/microchip/microchip-isc.h
@@ -62,7 +62,9 @@ struct isc_subdev_entity {
  * @mbus_code:		V4L2 media bus format code.
  * @cfa_baycfg:		If this format is RAW BAYER, indicate the type of bayer.
 			this is either BGBG, RGRG, etc.
- * @pfe_cfg0_bps:	Number of hardware data lines connected to the ISC
+ * @pfe_cfg0_bps:	ISC_PFE_CFG0 BPS field value (e.g. ISC_PFE_CFG0_BPS_EIGHT),
+			written into PFE_CFG0 with FIELD_PREP(ISC_PFE_CFG0_BPS_MASK)
+			at configure time.
  * @raw:		If the format is raw bayer.
  */
 
@@ -287,6 +289,7 @@ struct isc_device {
 	u32			dcfg;
 
 	struct device		*dev;
+	int			irq;
 	struct v4l2_device	v4l2_dev;
 	struct video_device	video_dev;
 
diff --git a/drivers/media/platform/microchip/microchip-sama5d2-isc.c b/drivers/media/platform/microchip/microchip-sama5d2-isc.c
index 66d3d7891991e9..5e41eee45dbd7b 100644
--- a/drivers/media/platform/microchip/microchip-sama5d2-isc.c
+++ b/drivers/media/platform/microchip/microchip-sama5d2-isc.c
@@ -30,6 +30,7 @@
 #include <linux/interrupt.h>
 #include <linux/math64.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
@@ -146,49 +147,49 @@ static struct isc_format sama5d2_formats_list[] = {
 	{
 		.fourcc		= V4L2_PIX_FMT_SBGGR10,
 		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
+		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGBRG10,
 		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGRBG10,
 		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SRGGB10,
 		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SBGGR12,
 		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGBRG12,
 		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGRBG12,
 		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SRGGB12,
 		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
 	},
 	{
@@ -209,7 +210,7 @@ static struct isc_format sama5d2_formats_list[] = {
 	{
 		.fourcc		= V4L2_PIX_FMT_Y10,
 		.mbus_code	= MEDIA_BUS_FMT_Y10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 	},
 
 };
@@ -356,28 +357,30 @@ static int isc_parse_dt(struct device *dev, struct isc_device *isc)
 	struct device_node *epn;
 	struct isc_subdev_entity *subdev_entity;
 	unsigned int flags;
+	int ret;
 
 	INIT_LIST_HEAD(&isc->subdev_entities);
 
 	for_each_endpoint_of_node(np, epn) {
 		struct v4l2_fwnode_endpoint v4l2_epn = { .bus_type = 0 };
-		int ret;
 
 		ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(epn),
 						 &v4l2_epn);
 		if (ret) {
-			of_node_put(epn);
 			dev_err(dev, "Could not parse the endpoint\n");
-			return -EINVAL;
+			of_node_put(epn);
+			ret = -EINVAL;
+			goto err_cleanup;
 		}
 
 		subdev_entity = devm_kzalloc(dev, sizeof(*subdev_entity),
 					     GFP_KERNEL);
 		if (!subdev_entity) {
 			of_node_put(epn);
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto err_cleanup;
 		}
-		subdev_entity->epn = epn;
+		subdev_entity->epn = of_node_get(epn);
 
 		flags = v4l2_epn.bus.parallel.flags;
 
@@ -398,6 +401,11 @@ static int isc_parse_dt(struct device *dev, struct isc_device *isc)
 	}
 
 	return 0;
+
+err_cleanup:
+	list_for_each_entry(subdev_entity, &isc->subdev_entities, list)
+		of_node_put(subdev_entity->epn);
+	return ret;
 }
 
 static int microchip_isc_probe(struct platform_device *pdev)
@@ -417,6 +425,14 @@ static int microchip_isc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, isc);
 	isc->dev = dev;
 
+	ret = devm_mutex_init(dev, &isc->lock);
+	if (ret)
+		return ret;
+
+	ret = devm_mutex_init(dev, &isc->awb_mutex);
+	if (ret)
+		return ret;
+
 	io_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(io_base))
 		return PTR_ERR(io_base);
@@ -432,6 +448,8 @@ static int microchip_isc_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
+	isc->irq = irq;
+
 	ret = devm_request_irq(dev, irq, microchip_isc_interrupt, 0,
 			       "microchip-sama5d2-isc", isc);
 	if (ret < 0) {
diff --git a/drivers/media/platform/microchip/microchip-sama7g5-isc.c b/drivers/media/platform/microchip/microchip-sama7g5-isc.c
index 7383341ec51d6e..9e78ea07b013a7 100644
--- a/drivers/media/platform/microchip/microchip-sama7g5-isc.c
+++ b/drivers/media/platform/microchip/microchip-sama7g5-isc.c
@@ -33,6 +33,7 @@
 #include <linux/interrupt.h>
 #include <linux/math64.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
@@ -155,49 +156,49 @@ static struct isc_format sama7g5_formats_list[] = {
 	{
 		.fourcc		= V4L2_PIX_FMT_SBGGR10,
 		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
+		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGBRG10,
 		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGRBG10,
 		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SRGGB10,
 		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SBGGR12,
 		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGBRG12,
 		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGRBG12,
 		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SRGGB12,
 		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
 	},
 	{
@@ -223,7 +224,7 @@ static struct isc_format sama7g5_formats_list[] = {
 	{
 		.fourcc		= V4L2_PIX_FMT_Y10,
 		.mbus_code	= MEDIA_BUS_FMT_Y10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
+		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_TEN,
 	},
 };
 
@@ -340,6 +341,7 @@ static int xisc_parse_dt(struct device *dev, struct isc_device *isc)
 	struct isc_subdev_entity *subdev_entity;
 	unsigned int flags;
 	bool mipi_mode;
+	int ret;
 
 	INIT_LIST_HEAD(&isc->subdev_entities);
 
@@ -347,23 +349,24 @@ static int xisc_parse_dt(struct device *dev, struct isc_device *isc)
 
 	for_each_endpoint_of_node(np, epn) {
 		struct v4l2_fwnode_endpoint v4l2_epn = { .bus_type = 0 };
-		int ret;
 
 		ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(epn),
 						 &v4l2_epn);
 		if (ret) {
-			of_node_put(epn);
 			dev_err(dev, "Could not parse the endpoint\n");
-			return -EINVAL;
+			of_node_put(epn);
+			ret = -EINVAL;
+			goto err_cleanup;
 		}
 
 		subdev_entity = devm_kzalloc(dev, sizeof(*subdev_entity),
 					     GFP_KERNEL);
 		if (!subdev_entity) {
 			of_node_put(epn);
-			return -ENOMEM;
+			ret = -ENOMEM;
+			goto err_cleanup;
 		}
-		subdev_entity->epn = epn;
+		subdev_entity->epn = of_node_get(epn);
 
 		flags = v4l2_epn.bus.parallel.flags;
 
@@ -387,6 +390,11 @@ static int xisc_parse_dt(struct device *dev, struct isc_device *isc)
 	}
 
 	return 0;
+
+err_cleanup:
+	list_for_each_entry(subdev_entity, &isc->subdev_entities, list)
+		of_node_put(subdev_entity->epn);
+	return ret;
 }
 
 static int microchip_xisc_probe(struct platform_device *pdev)
@@ -406,6 +414,14 @@ static int microchip_xisc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, isc);
 	isc->dev = dev;
 
+	ret = devm_mutex_init(dev, &isc->lock);
+	if (ret)
+		return ret;
+
+	ret = devm_mutex_init(dev, &isc->awb_mutex);
+	if (ret)
+		return ret;
+
 	io_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(io_base))
 		return PTR_ERR(io_base);
@@ -421,6 +437,8 @@ static int microchip_xisc_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
+	isc->irq = irq;
+
 	ret = devm_request_irq(dev, irq, microchip_isc_interrupt, 0,
 			       "microchip-sama7g5-xisc", isc);
 	if (ret < 0) {
diff --git a/drivers/media/platform/nuvoton/npcm-video.c b/drivers/media/platform/nuvoton/npcm-video.c
index 52505af35c0874..a6737c65d9b289 100644
--- a/drivers/media/platform/nuvoton/npcm-video.c
+++ b/drivers/media/platform/nuvoton/npcm-video.c
@@ -120,6 +120,7 @@ struct npcm_video {
 
 	struct list_head buffers;
 	struct mutex buffer_lock; /* buffer list lock */
+	int irq;
 	unsigned long flags;
 	unsigned int sequence;
 
@@ -1486,6 +1487,7 @@ static int npcm_video_start_streaming(struct vb2_queue *q, unsigned int count)
 	}
 
 	set_bit(VIDEO_STREAMING, &video->flags);
+	enable_irq(video->irq);
 	return 0;
 }
 
@@ -1494,6 +1496,7 @@ static void npcm_video_stop_streaming(struct vb2_queue *q)
 	struct npcm_video *video = vb2_get_drv_priv(q);
 	struct regmap *vcd = video->vcd_regmap;
 
+	disable_irq(video->irq);
 	clear_bit(VIDEO_STREAMING, &video->flags);
 	regmap_write(vcd, VCD_INTE, 0);
 	regmap_write(vcd, VCD_STAT, VCD_STAT_CLEAR);
@@ -1707,25 +1710,24 @@ static int npcm_video_init(struct npcm_video *video)
 		dev_err(dev, "Failed to find VCD IRQ\n");
 		return -ENODEV;
 	}
+	video->irq = irq;
 
 	rc = devm_request_threaded_irq(dev, irq, NULL, npcm_video_irq,
-				       IRQF_ONESHOT, DEVICE_NAME, video);
+				       IRQF_ONESHOT | IRQF_NO_AUTOEN, DEVICE_NAME, video);
 	if (rc < 0) {
 		dev_err(dev, "Failed to request IRQ %d\n", irq);
 		return rc;
 	}
 
-	of_reserved_mem_device_init(dev);
+	devm_of_reserved_mem_device_init(dev);
 	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 	if (rc) {
 		dev_err(dev, "Failed to set DMA mask\n");
-		of_reserved_mem_device_release(dev);
 		return rc;
 	}
 
 	rc = npcm_video_ece_init(video);
 	if (rc) {
-		of_reserved_mem_device_release(dev);
 		dev_err(dev, "Failed to initialize ECE\n");
 		return rc;
 	}
@@ -1789,13 +1791,11 @@ static int npcm_video_probe(struct platform_device *pdev)
 
 	rc = npcm_video_setup_video(video);
 	if (rc)
-		goto err_release_mem;
+		goto err_free;
 
 	dev_info(video->dev, "NPCM video driver probed\n");
 	return 0;
 
-err_release_mem:
-	of_reserved_mem_device_release(&pdev->dev);
 err_free:
 	kfree(video);
 	return rc;
@@ -1807,14 +1807,12 @@ static void npcm_video_remove(struct platform_device *pdev)
 	struct v4l2_device *v4l2_dev = dev_get_drvdata(dev);
 	struct npcm_video *video = to_npcm_video(v4l2_dev);
 
-	video_unregister_device(&video->vdev);
-	vb2_queue_release(&video->queue);
+	vb2_video_unregister_device(&video->vdev);
 	v4l2_ctrl_handler_free(&video->ctrl_handler);
 	v4l2_device_unregister(v4l2_dev);
 	if (video->ece.enable)
 		npcm_video_ece_stop(video);
 	kfree(video);
-	of_reserved_mem_device_release(dev);
 }
 
 static const struct of_device_id npcm_video_match[] = {
diff --git a/drivers/media/platform/nxp/Kconfig b/drivers/media/platform/nxp/Kconfig
index 40e3436669e213..5ff263fe7c6f2c 100644
--- a/drivers/media/platform/nxp/Kconfig
+++ b/drivers/media/platform/nxp/Kconfig
@@ -28,6 +28,22 @@ config VIDEO_IMX8MQ_MIPI_CSI2
 	  Video4Linux2 driver for the MIPI CSI-2 receiver found on the i.MX8MQ
 	  SoC.
 
+config VIDEO_IMX95_CSI_FORMATTER
+	tristate "NXP i.MX95 CSI Pixel Formatter driver"
+	depends on ARCH_MXC || COMPILE_TEST
+	depends on PM_CLK
+	depends on VIDEO_DEV
+	select MEDIA_CONTROLLER
+	select MFD_SYSCON
+	select V4L2_FWNODE
+	select VIDEO_V4L2_SUBDEV_API
+	help
+	  This driver provides support for the CSI Pixel Formatter found on
+	  i.MX95 series SoCs. This module unpacks the pixels received from the
+	  CSI-2 interface and reformats them to meet pixel link requirements.
+
+	  Say Y here to enable CSI Pixel Formatter module for i.MX95 SoC.
+
 config VIDEO_IMX_MIPI_CSIS
 	tristate "NXP MIPI CSI-2 CSIS receiver found on i.MX7 and i.MX8 models"
 	depends on ARCH_MXC || COMPILE_TEST
diff --git a/drivers/media/platform/nxp/Makefile b/drivers/media/platform/nxp/Makefile
index 4d90eb71365259..6410115d870e81 100644
--- a/drivers/media/platform/nxp/Makefile
+++ b/drivers/media/platform/nxp/Makefile
@@ -6,6 +6,7 @@ obj-y += imx8-isi/
 
 obj-$(CONFIG_VIDEO_IMX7_CSI) += imx7-media-csi.o
 obj-$(CONFIG_VIDEO_IMX8MQ_MIPI_CSI2) += imx8mq-mipi-csi2.o
+obj-$(CONFIG_VIDEO_IMX95_CSI_FORMATTER) += imx95-csi-formatter.o
 obj-$(CONFIG_VIDEO_IMX_MIPI_CSIS) += imx-mipi-csis.o
 obj-$(CONFIG_VIDEO_IMX_PXP) += imx-pxp.o
 obj-$(CONFIG_VIDEO_MX2_EMMAPRP) += mx2_emmaprp.o
diff --git a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
index 725e941528848e..e170859354352e 100644
--- a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
+++ b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
@@ -1590,15 +1590,15 @@ static void mxc_jpeg_device_run(void *priv)
 	dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
 	if (!src_buf || !dst_buf) {
 		dev_err(dev, "Null src or dst buf\n");
-		goto end;
+		goto job_finish;
 	}
 
 	q_data_cap = mxc_jpeg_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
 	if (!q_data_cap)
-		goto end;
+		goto buf_finish;
 	q_data_out = mxc_jpeg_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
 	if (!q_data_out)
-		goto end;
+		goto buf_finish;
 	src_buf->sequence = q_data_out->sequence++;
 	dst_buf->sequence = q_data_cap->sequence++;
 
@@ -1636,11 +1636,11 @@ static void mxc_jpeg_device_run(void *priv)
 	ctx->slot = mxc_get_free_slot(&jpeg->slot_data);
 	if (ctx->slot < 0) {
 		dev_err(dev, "No more free slots\n");
-		goto end;
+		goto buf_finish;
 	}
 	if (!mxc_jpeg_alloc_slot_data(jpeg)) {
 		dev_err(dev, "Cannot allocate slot data\n");
-		goto end;
+		goto buf_finish;
 	}
 
 	mxc_jpeg_enable_slot(reg, ctx->slot);
@@ -1660,8 +1660,17 @@ static void mxc_jpeg_device_run(void *priv)
 		mxc_jpeg_dec_mode_go(dev, reg);
 	}
 	schedule_delayed_work(&ctx->task_timer, msecs_to_jiffies(hw_timeout));
-end:
+
+	spin_unlock_irqrestore(&ctx->mxc_jpeg->hw_lock, flags);
+	return;
+buf_finish:
+	v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+	v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+	v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
+	v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_ERROR);
+job_finish:
 	spin_unlock_irqrestore(&ctx->mxc_jpeg->hw_lock, flags);
+	v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
 }
 
 static int mxc_jpeg_decoder_cmd(struct file *file, void *priv,
@@ -1798,6 +1807,8 @@ static void mxc_jpeg_stop_streaming(struct vb2_queue *q)
 
 	dev_dbg(ctx->mxc_jpeg->dev, "Stop streaming ctx=%p", ctx);
 
+	cancel_delayed_work_sync(&ctx->task_timer);
+
 	/* Release all active buffers */
 	for (;;) {
 		if (V4L2_TYPE_IS_OUTPUT(q->type))
diff --git a/drivers/media/platform/nxp/imx-mipi-csis.c b/drivers/media/platform/nxp/imx-mipi-csis.c
index 5f9691d76434a3..1b9012dbae3946 100644
--- a/drivers/media/platform/nxp/imx-mipi-csis.c
+++ b/drivers/media/platform/nxp/imx-mipi-csis.c
@@ -1104,6 +1104,7 @@ static int mipi_csis_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int mipi_csis_set_fmt(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *sdformat)
 {
@@ -1225,7 +1226,7 @@ static int mipi_csis_init_state(struct v4l2_subdev *sd,
 		V4L2_MAP_QUANTIZATION_DEFAULT(false, fmt.format.colorspace,
 					      fmt.format.ycbcr_enc);
 
-	return mipi_csis_set_fmt(sd, state, &fmt);
+	return mipi_csis_set_fmt(sd, NULL, state, &fmt);
 }
 
 static int mipi_csis_log_status(struct v4l2_subdev *sd)
diff --git a/drivers/media/platform/nxp/imx7-media-csi.c b/drivers/media/platform/nxp/imx7-media-csi.c
index 7ddc7ba06e3d4e..f06b03967aefaa 100644
--- a/drivers/media/platform/nxp/imx7-media-csi.c
+++ b/drivers/media/platform/nxp/imx7-media-csi.c
@@ -466,7 +466,7 @@ static int imx7_csi_alloc_dma_buf(struct imx7_csi *csi,
 
 	buf->len = PAGE_ALIGN(size);
 	buf->virt = dma_alloc_coherent(csi->dev, buf->len, &buf->dma_addr,
-				       GFP_DMA | GFP_KERNEL);
+				       GFP_KERNEL);
 	if (!buf->virt)
 		return -ENOMEM;
 
@@ -1890,6 +1890,7 @@ static void imx7_csi_try_fmt(struct v4l2_subdev *sd,
 }
 
 static int imx7_csi_set_fmt(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
index 7bb1335f211069..8d50e5fb3b1348 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
@@ -260,6 +260,7 @@ static int mxc_isi_crossbar_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int mxc_isi_crossbar_set_fmt(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
index 934f7b35625879..a12e7445073c0e 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
@@ -524,6 +524,7 @@ static int mxc_isi_pipe_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int mxc_isi_pipe_set_fmt(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_format *fmt)
 {
@@ -619,6 +620,7 @@ static int mxc_isi_pipe_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int mxc_isi_pipe_get_selection(struct v4l2_subdev *sd,
+				      const struct v4l2_subdev_client_info *ci,
 				      struct v4l2_subdev_state *state,
 				      struct v4l2_subdev_selection *sel)
 {
@@ -678,6 +680,7 @@ static int mxc_isi_pipe_get_selection(struct v4l2_subdev *sd,
 }
 
 static int mxc_isi_pipe_set_selection(struct v4l2_subdev *sd,
+				      const struct v4l2_subdev_client_info *ci,
 				      struct v4l2_subdev_state *state,
 				      struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
index f45c2aae59ce90..fc907d357149ee 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
@@ -773,7 +773,7 @@ static int mxc_isi_video_alloc_discard_buffers(struct mxc_isi_video *video)
 
 		buf->size = PAGE_ALIGN(video->pix.plane_fmt[i].sizeimage);
 		buf->addr = dma_alloc_coherent(video->pipe->isi->dev, buf->size,
-					       &buf->dma, GFP_DMA | GFP_KERNEL);
+					       &buf->dma, GFP_KERNEL);
 		if (!buf->addr) {
 			mxc_isi_video_free_discard_buffers(video);
 			return -ENOMEM;
diff --git a/drivers/media/platform/nxp/imx8mq-mipi-csi2.c b/drivers/media/platform/nxp/imx8mq-mipi-csi2.c
index 04ebed8a0493ec..17756b7fe1dded 100644
--- a/drivers/media/platform/nxp/imx8mq-mipi-csi2.c
+++ b/drivers/media/platform/nxp/imx8mq-mipi-csi2.c
@@ -597,6 +597,7 @@ static int imx8mq_mipi_csi_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int imx8mq_mipi_csi_set_fmt(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/media/platform/nxp/imx95-csi-formatter.c b/drivers/media/platform/nxp/imx95-csi-formatter.c
new file mode 100644
index 00000000000000..37dfcefbf0d12a
--- /dev/null
+++ b/drivers/media/platform/nxp/imx95-csi-formatter.c
@@ -0,0 +1,759 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2025 NXP
+ */
+
+#include <linux/bits.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_clock.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+
+#include <media/mipi-csi2.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-mc.h>
+#include <media/v4l2-subdev.h>
+
+/* CSI Pixel Formatter registers map */
+
+#define CSI_VC_INTERLACED_LINE_CNT(vc)		(0x00 + (vc) * 0x04)
+#define INTERLACED_ODD_LINE_CNT_SET(x)		FIELD_PREP(GENMASK(13, 0), (x))
+#define INTERLACED_EVEN_LINE_CNT_SET(x)		FIELD_PREP(GENMASK(29, 16), (x))
+
+#define CSI_VC_INTERLACED_CTRL			0x20
+
+#define CSI_VC_INTERLACED_ERR			0x24
+#define CSI_VC_ERR_MASK				GENMASK(7, 0)
+#define CSI_VC_ERR(vc)				BIT((vc))
+
+#define CSI_VC_YUV420_FIRST_LINE_EVEN		0x28
+#define YUV420_FIRST_LINE_EVEN(vc)		BIT((vc))
+
+#define CSI_RAW32_CTRL				0x30
+#define CSI_VC_RAW32_MODE(vc)			BIT((vc))
+#define CSI_VC_RAW32_SWAP_MODE(vc)		BIT((vc) + 8)
+
+#define CSI_STREAM_FENCING_CTRL			0x34
+#define CSI_VC_STREAM_FENCING(vc)		BIT((vc))
+#define CSI_VC_STREAM_FENCING_RST(vc)		BIT((vc) + 8)
+
+#define CSI_STREAM_FENCING_STS			0x38
+#define CSI_STREAM_FENCING_STS_MASK		GENMASK(7, 0)
+
+#define CSI_VC_NON_PIXEL_DATA_TYPE(vc)		(0x40 + (vc) * 0x04)
+
+#define CSI_VC_PIXEL_DATA_CTRL(vc)		(0x60 + (vc) * 0x04)
+#define NEW_VC(vc)				FIELD_PREP(GENMASK(3, 1), vc)
+#define REROUTE_VC_ENABLE			BIT(0)
+
+#define CSI_VC_ROUTE_PIXEL_DATA_TYPE(vc)	(0x80 + (vc) * 0x04)
+
+#define CSI_VC_NON_PIXEL_DATA_CTRL(vc)		(0xa0 + (vc) * 0x04)
+
+#define CSI_VC_PIXEL_DATA_TYPE(vc)		(0xc0 + (vc) * 0x04)
+
+#define CSI_VC_PIXEL_DATA_TYPE_ERR(vc)		(0xe0 + (vc) * 0x04)
+
+#define CSI_FORMATTER_PAD_SINK			0
+#define CSI_FORMATTER_PAD_SOURCE		1
+#define CSI_FORMATTER_PAD_NUM			2
+
+#define CSI_FORMATTER_VC_NUM			8 /* Number of virtual channels */
+
+struct csi_formatter_pix_format {
+	u32 code;
+	u32 data_type;
+};
+
+struct csi_formatter {
+	struct device *dev;
+	struct regmap *regs;
+	u32 reg_offset;
+
+	struct v4l2_subdev sd;
+	struct v4l2_async_notifier notifier;
+	struct media_pad pads[CSI_FORMATTER_PAD_NUM];
+
+	struct v4l2_subdev *remote_sd;
+	u32 remote_pad;
+
+	u64 enabled_streams;
+};
+
+struct csi_formatter_dt_index {
+	u8 dtype;
+	u8 index;
+};
+
+/*
+ * The index corresponds to the bit index in the register that enables
+ * the data type of pixel data transported by the Formatter.
+ */
+static const struct csi_formatter_dt_index csi_formatter_dt_to_index_map[] = {
+	{ .dtype = MIPI_CSI2_DT_YUV420_8B,        .index = 0 },
+	{ .dtype = MIPI_CSI2_DT_YUV420_8B_LEGACY, .index = 2 },
+	{ .dtype = MIPI_CSI2_DT_YUV422_8B,        .index = 6 },
+	{ .dtype = MIPI_CSI2_DT_RGB444,		  .index = 8 },
+	{ .dtype = MIPI_CSI2_DT_RGB555,           .index = 9 },
+	{ .dtype = MIPI_CSI2_DT_RGB565,           .index = 10 },
+	{ .dtype = MIPI_CSI2_DT_RGB666,           .index = 11 },
+	{ .dtype = MIPI_CSI2_DT_RGB888,           .index = 12 },
+	{ .dtype = MIPI_CSI2_DT_RAW6,             .index = 16 },
+	{ .dtype = MIPI_CSI2_DT_RAW7,             .index = 17 },
+	{ .dtype = MIPI_CSI2_DT_RAW8,             .index = 18 },
+	{ .dtype = MIPI_CSI2_DT_RAW10,            .index = 19 },
+	{ .dtype = MIPI_CSI2_DT_RAW12,            .index = 20 },
+	{ .dtype = MIPI_CSI2_DT_RAW14,            .index = 21 },
+	{ .dtype = MIPI_CSI2_DT_RAW16,            .index = 22 },
+};
+
+static const struct csi_formatter_pix_format csi_formatter_formats[] = {
+	/* YUV formats */
+	{ MEDIA_BUS_FMT_UYVY8_1X16,	MIPI_CSI2_DT_YUV422_8B },
+	/* RGB formats */
+	{ MEDIA_BUS_FMT_RGB565_1X16,	MIPI_CSI2_DT_RGB565 },
+	{ MEDIA_BUS_FMT_RGB888_1X24,	MIPI_CSI2_DT_RGB888 },
+	/* RAW (Bayer and greyscale) formats */
+	{ MEDIA_BUS_FMT_SBGGR8_1X8,	MIPI_CSI2_DT_RAW8 },
+	{ MEDIA_BUS_FMT_SGBRG8_1X8,	MIPI_CSI2_DT_RAW8 },
+	{ MEDIA_BUS_FMT_SGRBG8_1X8,	MIPI_CSI2_DT_RAW8 },
+	{ MEDIA_BUS_FMT_SRGGB8_1X8,	MIPI_CSI2_DT_RAW8 },
+	{ MEDIA_BUS_FMT_Y8_1X8,		MIPI_CSI2_DT_RAW8 },
+	{ MEDIA_BUS_FMT_SBGGR10_1X10,	MIPI_CSI2_DT_RAW10 },
+	{ MEDIA_BUS_FMT_SGBRG10_1X10,	MIPI_CSI2_DT_RAW10 },
+	{ MEDIA_BUS_FMT_SGRBG10_1X10,	MIPI_CSI2_DT_RAW10 },
+	{ MEDIA_BUS_FMT_SRGGB10_1X10,	MIPI_CSI2_DT_RAW10 },
+	{ MEDIA_BUS_FMT_Y10_1X10,	MIPI_CSI2_DT_RAW10 },
+	{ MEDIA_BUS_FMT_SBGGR12_1X12,	MIPI_CSI2_DT_RAW12 },
+	{ MEDIA_BUS_FMT_SGBRG12_1X12,	MIPI_CSI2_DT_RAW12 },
+	{ MEDIA_BUS_FMT_SGRBG12_1X12,	MIPI_CSI2_DT_RAW12 },
+	{ MEDIA_BUS_FMT_SRGGB12_1X12,	MIPI_CSI2_DT_RAW12 },
+	{ MEDIA_BUS_FMT_Y12_1X12,	MIPI_CSI2_DT_RAW12 },
+	{ MEDIA_BUS_FMT_SBGGR14_1X14,	MIPI_CSI2_DT_RAW14 },
+	{ MEDIA_BUS_FMT_SGBRG14_1X14,	MIPI_CSI2_DT_RAW14 },
+	{ MEDIA_BUS_FMT_SGRBG14_1X14,	MIPI_CSI2_DT_RAW14 },
+	{ MEDIA_BUS_FMT_SRGGB14_1X14,	MIPI_CSI2_DT_RAW14 },
+	{ MEDIA_BUS_FMT_SBGGR16_1X16,	MIPI_CSI2_DT_RAW16 },
+	{ MEDIA_BUS_FMT_SGBRG16_1X16,	MIPI_CSI2_DT_RAW16 },
+	{ MEDIA_BUS_FMT_SGRBG16_1X16,	MIPI_CSI2_DT_RAW16 },
+	{ MEDIA_BUS_FMT_SRGGB16_1X16,	MIPI_CSI2_DT_RAW16 },
+};
+
+static const struct v4l2_mbus_framefmt formatter_default_fmt = {
+	.code = MEDIA_BUS_FMT_UYVY8_1X16,
+	.width = 1920U,
+	.height = 1080U,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = V4L2_COLORSPACE_SMPTE170M,
+	.xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(V4L2_COLORSPACE_SMPTE170M),
+	.ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(V4L2_COLORSPACE_SMPTE170M),
+	.quantization = V4L2_QUANTIZATION_LIM_RANGE,
+};
+
+static const struct csi_formatter_pix_format *csi_formatter_find_format(u32 code)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(csi_formatter_formats); i++) {
+		if (code == csi_formatter_formats[i].code)
+			return &csi_formatter_formats[i];
+	}
+
+	return NULL;
+}
+
+/* -----------------------------------------------------------------------------
+ * V4L2 subdev operations
+ */
+
+static inline struct csi_formatter *sd_to_formatter(struct v4l2_subdev *sdev)
+{
+	return container_of(sdev, struct csi_formatter, sd);
+}
+
+static int __csi_formatter_subdev_set_routing(struct v4l2_subdev *sd,
+					      struct v4l2_subdev_state *state,
+					      struct v4l2_subdev_krouting *routing)
+{
+	int ret;
+
+	ret = v4l2_subdev_routing_validate(sd, routing,
+					   V4L2_SUBDEV_ROUTING_ONLY_1_TO_1);
+	if (ret)
+		return ret;
+
+	return v4l2_subdev_set_routing_with_fmt(sd, state, routing,
+						&formatter_default_fmt);
+}
+
+static int csi_formatter_subdev_init_state(struct v4l2_subdev *sd,
+					   struct v4l2_subdev_state *sd_state)
+{
+	struct v4l2_subdev_route routes[] = {
+		{
+			.sink_pad = CSI_FORMATTER_PAD_SINK,
+			.sink_stream = 0,
+			.source_pad = CSI_FORMATTER_PAD_SOURCE,
+			.source_stream = 0,
+			.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
+		},
+	};
+
+	struct v4l2_subdev_krouting routing = {
+		.num_routes = ARRAY_SIZE(routes),
+		.routes = routes,
+	};
+
+	return __csi_formatter_subdev_set_routing(sd, sd_state, &routing);
+}
+
+static int csi_formatter_subdev_enum_mbus_code(struct v4l2_subdev *sd,
+					       struct v4l2_subdev_state *sd_state,
+					       struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->pad == CSI_FORMATTER_PAD_SOURCE) {
+		struct v4l2_mbus_framefmt *fmt;
+
+		if (code->index > 0)
+			return -EINVAL;
+
+		fmt = v4l2_subdev_state_get_format(sd_state, code->pad,
+						   code->stream);
+		code->code = fmt->code;
+		return 0;
+	}
+
+	if (code->index >= ARRAY_SIZE(csi_formatter_formats))
+		return -EINVAL;
+
+	code->code = csi_formatter_formats[code->index].code;
+
+	return 0;
+}
+
+static int csi_formatter_subdev_set_fmt(struct v4l2_subdev *sd,
+					const struct v4l2_subdev_client_info *ci,
+					struct v4l2_subdev_state *sd_state,
+					struct v4l2_subdev_format *format)
+{
+	struct v4l2_mbus_framefmt *fmt;
+
+	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE &&
+	    media_entity_is_streaming(&sd->entity))
+		return -EBUSY;
+
+	if (format->pad == CSI_FORMATTER_PAD_SOURCE)
+		return v4l2_subdev_get_fmt(sd, sd_state, format);
+
+	if (!csi_formatter_find_format(format->format.code))
+		format->format.code = csi_formatter_formats[0].code;
+
+	v4l_bound_align_image(&format->format.width, 1, 0xffff, 2,
+			      &format->format.height, 1, 0xffff, 0, 0);
+
+	/* TODO: Add interlaced format support */
+	format->format.field = V4L2_FIELD_NONE;
+
+	fmt = v4l2_subdev_state_get_format(sd_state, format->pad,
+					   format->stream);
+	*fmt = format->format;
+
+	/* Propagate the format from sink stream to source stream */
+	fmt = v4l2_subdev_state_get_opposite_stream_format(sd_state, format->pad,
+							   format->stream);
+	*fmt = format->format;
+
+	return 0;
+}
+
+static int csi_formatter_subdev_set_routing(struct v4l2_subdev *sd,
+					    struct v4l2_subdev_state *state,
+					    enum v4l2_subdev_format_whence which,
+					    struct v4l2_subdev_krouting *routing)
+{
+	if (which == V4L2_SUBDEV_FORMAT_ACTIVE &&
+	    media_entity_is_streaming(&sd->entity))
+		return -EBUSY;
+
+	return __csi_formatter_subdev_set_routing(sd, state, routing);
+}
+
+static u8 csi_formatter_get_index_by_dt(struct csi_formatter *formatter,
+					u8 data_type)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(csi_formatter_dt_to_index_map); ++i) {
+		const struct csi_formatter_dt_index *entry =
+			&csi_formatter_dt_to_index_map[i];
+
+		if (data_type == entry->dtype)
+			return entry->index;
+	}
+
+	dev_WARN(formatter->dev, "Unsupported data type 0x%x, using default\n",
+		 data_type);
+
+	return csi_formatter_dt_to_index_map[0].index;
+}
+
+static int csi_formatter_get_vc(struct csi_formatter *formatter,
+				struct v4l2_mbus_frame_desc *fd,
+				unsigned int stream)
+{
+	struct v4l2_mbus_frame_desc_entry *entry = NULL;
+	u8 vc;
+
+	for (unsigned int i = 0; i < fd->num_entries; ++i) {
+		if (fd->entry[i].stream == stream) {
+			entry = &fd->entry[i];
+			break;
+		}
+	}
+
+	if (!entry) {
+		dev_err(formatter->dev,
+			"No frame desc entry for stream %u\n", stream);
+		return -EPIPE;
+	}
+
+	vc = entry->bus.csi2.vc;
+
+	if (vc >= CSI_FORMATTER_VC_NUM) {
+		dev_err(formatter->dev, "Invalid virtual channel %u\n", vc);
+		return -EPIPE;
+	}
+
+	return vc;
+}
+
+static int csi_formatter_get_frame_desc(struct csi_formatter *formatter,
+					struct v4l2_mbus_frame_desc *fd)
+{
+	int ret;
+
+	ret = v4l2_subdev_call(formatter->remote_sd, pad, get_frame_desc,
+			       formatter->remote_pad, fd);
+	if (ret < 0 && ret != -ENOIOCTLCMD) {
+		dev_err(formatter->dev, "Failed to get frame desc: %d\n", ret);
+		return ret;
+	}
+
+	/*
+	 * If the source doesn't implement .get_frame_desc(), assume a single
+	 * stream on VC 0. fd is zero-initialized, only set the fields that have
+	 * a non-zero value.
+	 */
+	if (ret == -ENOIOCTLCMD) {
+		fd->type = V4L2_MBUS_FRAME_DESC_TYPE_CSI2;
+		fd->num_entries = 1;
+	}
+
+	return 0;
+}
+
+static void csi_formatter_stop_stream(struct csi_formatter *formatter,
+				      struct v4l2_subdev_state *state,
+				      u64 stream_mask)
+{
+	const struct csi_formatter_pix_format *pix_fmt;
+	struct v4l2_mbus_frame_desc fd = {};
+	struct v4l2_subdev_route *route;
+	struct v4l2_mbus_framefmt *fmt;
+	unsigned int reg;
+	unsigned int mask;
+	int vc;
+	int ret;
+
+	ret = csi_formatter_get_frame_desc(formatter, &fd);
+	if (ret)
+		return;
+
+	for_each_active_route(&state->routing, route) {
+		if (!(stream_mask & BIT_ULL(route->source_stream)))
+			continue;
+
+		vc = csi_formatter_get_vc(formatter, &fd, route->sink_stream);
+		if (vc < 0)
+			continue;
+
+		fmt = v4l2_subdev_state_get_format(state, route->sink_pad,
+						   route->sink_stream);
+
+		pix_fmt = csi_formatter_find_format(fmt->code);
+		if (WARN_ON(!pix_fmt))
+			continue;
+
+		reg = CSI_VC_PIXEL_DATA_TYPE(vc) + formatter->reg_offset;
+		mask = BIT(csi_formatter_get_index_by_dt(formatter,
+							 pix_fmt->data_type));
+
+		/* Clear the data type bit to disable this VC */
+		regmap_clear_bits(formatter->regs, reg, mask);
+	}
+}
+
+static int csi_formatter_start_stream(struct csi_formatter *formatter,
+				      struct v4l2_subdev_state *state,
+				      u64 stream_mask)
+{
+	const struct csi_formatter_pix_format *pix_fmt;
+	struct v4l2_subdev_route *route;
+	struct v4l2_mbus_framefmt *fmt;
+	struct v4l2_mbus_frame_desc fd = {};
+	u64 configured_streams = 0;
+	unsigned int reg;
+	unsigned int mask;
+	int vc;
+	int ret;
+
+	ret = csi_formatter_get_frame_desc(formatter, &fd);
+	if (ret)
+		return ret;
+
+	for_each_active_route(&state->routing, route) {
+		if (!(stream_mask & BIT_ULL(route->source_stream)))
+			continue;
+
+		vc = csi_formatter_get_vc(formatter, &fd, route->sink_stream);
+		if (vc < 0) {
+			ret = vc;
+			goto err_cleanup;
+		}
+
+		fmt = v4l2_subdev_state_get_format(state, route->sink_pad,
+						   route->sink_stream);
+
+		pix_fmt = csi_formatter_find_format(fmt->code);
+		if (WARN_ON(!pix_fmt)) {
+			ret = -EINVAL;
+			goto err_cleanup;
+		}
+
+		reg = CSI_VC_PIXEL_DATA_TYPE(vc) + formatter->reg_offset;
+		mask = BIT(csi_formatter_get_index_by_dt(formatter,
+							 pix_fmt->data_type));
+
+		/* Set the data type bit to enable this VC */
+		regmap_set_bits(formatter->regs, reg, mask);
+
+		configured_streams |= BIT_ULL(route->source_stream);
+	}
+
+	return 0;
+
+err_cleanup:
+	csi_formatter_stop_stream(formatter, state, configured_streams);
+	return ret;
+}
+
+static int csi_formatter_subdev_enable_streams(struct v4l2_subdev *sd,
+					       struct v4l2_subdev_state *state,
+					       u32 pad, u64 streams_mask)
+{
+	struct csi_formatter *formatter = sd_to_formatter(sd);
+	struct device *dev = formatter->dev;
+	u64 sink_streams;
+	int ret;
+
+	sink_streams = v4l2_subdev_state_xlate_streams(state,
+						       CSI_FORMATTER_PAD_SOURCE,
+						       CSI_FORMATTER_PAD_SINK,
+						       &streams_mask);
+	if (!sink_streams || !streams_mask)
+		return -EINVAL;
+
+	if (!formatter->enabled_streams) {
+		ret = pm_runtime_resume_and_get(formatter->dev);
+		if (ret < 0) {
+			dev_err(dev, "Failed to resume runtime PM: %d\n", ret);
+			return ret;
+		}
+	}
+
+	ret = csi_formatter_start_stream(formatter, state, streams_mask);
+	if (ret)
+		goto err_runtime_put;
+
+	ret = v4l2_subdev_enable_streams(formatter->remote_sd,
+					 formatter->remote_pad,
+					 sink_streams);
+	if (ret)
+		goto err_stop_stream;
+
+	formatter->enabled_streams |= streams_mask;
+
+	return 0;
+
+err_stop_stream:
+	csi_formatter_stop_stream(formatter, state, streams_mask);
+err_runtime_put:
+	if (!formatter->enabled_streams)
+		pm_runtime_put_autosuspend(formatter->dev);
+	return ret;
+}
+
+static int csi_formatter_subdev_disable_streams(struct v4l2_subdev *sd,
+						struct v4l2_subdev_state *state,
+						u32 pad, u64 streams_mask)
+{
+	struct csi_formatter *formatter = sd_to_formatter(sd);
+	u64 sink_streams;
+	int ret;
+
+	sink_streams = v4l2_subdev_state_xlate_streams(state,
+						       CSI_FORMATTER_PAD_SOURCE,
+						       CSI_FORMATTER_PAD_SINK,
+						       &streams_mask);
+	if (!sink_streams || !streams_mask)
+		return -EINVAL;
+
+	ret = v4l2_subdev_disable_streams(formatter->remote_sd, formatter->remote_pad,
+					  sink_streams);
+	if (ret)
+		dev_err(formatter->dev, "Failed to disable streams: %d\n", ret);
+
+	csi_formatter_stop_stream(formatter, state, streams_mask);
+
+	formatter->enabled_streams &= ~streams_mask;
+
+	if (!formatter->enabled_streams)
+		pm_runtime_put_autosuspend(formatter->dev);
+
+	return ret;
+}
+
+static const struct v4l2_subdev_pad_ops formatter_subdev_pad_ops = {
+	.enum_mbus_code		= csi_formatter_subdev_enum_mbus_code,
+	.get_fmt		= v4l2_subdev_get_fmt,
+	.set_fmt		= csi_formatter_subdev_set_fmt,
+	.get_frame_desc		= v4l2_subdev_get_frame_desc_passthrough,
+	.set_routing		= csi_formatter_subdev_set_routing,
+	.enable_streams		= csi_formatter_subdev_enable_streams,
+	.disable_streams	= csi_formatter_subdev_disable_streams,
+};
+
+static const struct v4l2_subdev_ops formatter_subdev_ops = {
+	.pad = &formatter_subdev_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops formatter_internal_ops = {
+	.init_state = csi_formatter_subdev_init_state,
+};
+
+/* -----------------------------------------------------------------------------
+ * Media entity operations
+ */
+
+static const struct media_entity_operations formatter_entity_ops = {
+	.link_validate	= v4l2_subdev_link_validate,
+	.get_fwnode_pad = v4l2_subdev_get_fwnode_pad_1_to_1,
+};
+
+static int csi_formatter_subdev_init(struct csi_formatter *formatter)
+{
+	struct v4l2_subdev *sd = &formatter->sd;
+	int ret;
+
+	v4l2_subdev_init(sd, &formatter_subdev_ops);
+
+	strscpy(sd->name, dev_name(formatter->dev), sizeof(sd->name));
+	sd->internal_ops = &formatter_internal_ops;
+
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
+		     V4L2_SUBDEV_FL_STREAMS;
+	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
+	sd->entity.ops = &formatter_entity_ops;
+	sd->dev = formatter->dev;
+
+	formatter->pads[CSI_FORMATTER_PAD_SINK].flags = MEDIA_PAD_FL_SINK
+						      | MEDIA_PAD_FL_MUST_CONNECT;
+	formatter->pads[CSI_FORMATTER_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&sd->entity, CSI_FORMATTER_PAD_NUM,
+				     formatter->pads);
+	if (ret) {
+		dev_err(formatter->dev, "Failed to init pads\n");
+		return ret;
+	}
+
+	ret = v4l2_subdev_init_finalize(sd);
+	if (ret)
+		media_entity_cleanup(&sd->entity);
+
+	return ret;
+}
+
+static inline struct csi_formatter *
+notifier_to_csi_formatter(struct v4l2_async_notifier *n)
+{
+	return container_of(n, struct csi_formatter, notifier);
+}
+
+static int csi_formatter_notify_bound(struct v4l2_async_notifier *notifier,
+				      struct v4l2_subdev *sd,
+				      struct v4l2_async_connection *asc)
+{
+	const unsigned int link_flags = MEDIA_LNK_FL_IMMUTABLE
+				      | MEDIA_LNK_FL_ENABLED;
+	struct csi_formatter *formatter = notifier_to_csi_formatter(notifier);
+	struct v4l2_subdev *sdev = &formatter->sd;
+	struct media_pad *sink = &sdev->entity.pads[CSI_FORMATTER_PAD_SINK];
+	int ret;
+
+	formatter->remote_sd = sd;
+
+	ret = v4l2_create_fwnode_links_to_pad(sd, sink, link_flags);
+	if (ret < 0)
+		return ret;
+
+	formatter->remote_pad = media_pad_remote_pad_first(sink)->index;
+
+	return 0;
+}
+
+static const struct v4l2_async_notifier_operations formatter_notify_ops = {
+	.bound = csi_formatter_notify_bound,
+};
+
+static int csi_formatter_async_register(struct csi_formatter *formatter)
+{
+	struct device *dev = formatter->dev;
+	struct v4l2_async_connection *asc;
+	int ret;
+
+	struct fwnode_handle *ep __free(fwnode_handle) =
+		fwnode_graph_get_endpoint_by_id(dev_fwnode(dev), 0, 0,
+						FWNODE_GRAPH_ENDPOINT_NEXT);
+	if (!ep)
+		return -ENOTCONN;
+
+	v4l2_async_subdev_nf_init(&formatter->notifier, &formatter->sd);
+
+	asc = v4l2_async_nf_add_fwnode_remote(&formatter->notifier, ep,
+					      struct v4l2_async_connection);
+	if (IS_ERR(asc)) {
+		ret = PTR_ERR(asc);
+		goto err_cleanup_notifier;
+	}
+
+	formatter->notifier.ops = &formatter_notify_ops;
+
+	ret = v4l2_async_nf_register(&formatter->notifier);
+	if (ret)
+		goto err_cleanup_notifier;
+
+	ret = v4l2_async_register_subdev(&formatter->sd);
+	if (ret)
+		goto err_unregister_notifier;
+
+	return 0;
+
+err_unregister_notifier:
+	v4l2_async_nf_unregister(&formatter->notifier);
+err_cleanup_notifier:
+	v4l2_async_nf_cleanup(&formatter->notifier);
+	return ret;
+}
+
+static void csi_formatter_async_unregister(struct csi_formatter *formatter)
+{
+	v4l2_async_unregister_subdev(&formatter->sd);
+	v4l2_async_nf_unregister(&formatter->notifier);
+	v4l2_async_nf_cleanup(&formatter->notifier);
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(csi_formatter_pm_ops,
+				 pm_clk_suspend, pm_clk_resume, NULL);
+
+static int csi_formatter_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct csi_formatter *formatter;
+	int ret;
+
+	formatter = devm_kzalloc(dev, sizeof(*formatter), GFP_KERNEL);
+	if (!formatter)
+		return -ENOMEM;
+
+	formatter->dev = dev;
+
+	formatter->regs = syscon_node_to_regmap(dev->parent->of_node);
+	if (IS_ERR(formatter->regs))
+		return dev_err_probe(dev, PTR_ERR(formatter->regs),
+				     "Failed to get csi formatter regmap\n");
+
+	ret = of_property_read_u32(dev->of_node, "reg", &formatter->reg_offset);
+	if (ret < 0)
+		return dev_err_probe(dev, ret,
+				     "Failed to get csi formatter reg property\n");
+
+	ret = devm_pm_clk_create(dev);
+	if (ret)
+		return ret;
+
+	ret = of_pm_clk_add_clks(dev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to add clocks\n");
+
+	ret = csi_formatter_subdev_init(formatter);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to initialize formatter subdev\n");
+
+	platform_set_drvdata(pdev, formatter);
+
+	/* Enable runtime PM with autosuspend. */
+	pm_runtime_set_autosuspend_delay(dev, 1000);
+	pm_runtime_use_autosuspend(dev);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		goto err_pm_disable;
+
+	ret = csi_formatter_async_register(formatter);
+	if (ret < 0) {
+		dev_err_probe(dev, ret, "Failed to register async subdevice\n");
+		goto err_pm_disable;
+	}
+
+	return 0;
+
+err_pm_disable:
+	pm_runtime_dont_use_autosuspend(dev);
+	v4l2_subdev_cleanup(&formatter->sd);
+	media_entity_cleanup(&formatter->sd.entity);
+	return ret;
+}
+
+static void csi_formatter_remove(struct platform_device *pdev)
+{
+	struct csi_formatter *formatter = platform_get_drvdata(pdev);
+
+	csi_formatter_async_unregister(formatter);
+
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
+	pm_runtime_suspend(&pdev->dev);
+
+	v4l2_subdev_cleanup(&formatter->sd);
+	media_entity_cleanup(&formatter->sd.entity);
+}
+
+static const struct of_device_id csi_formatter_of_match[] = {
+	{ .compatible = "fsl,imx95-csi-formatter" },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, csi_formatter_of_match);
+
+static struct platform_driver csi_formatter_device_driver = {
+	.driver = {
+		.name           = "csi-pixel-formatter",
+		.of_match_table = csi_formatter_of_match,
+		.pm             = pm_ptr(&csi_formatter_pm_ops),
+	},
+	.probe  = csi_formatter_probe,
+	.remove = csi_formatter_remove,
+};
+
+module_platform_driver(csi_formatter_device_driver);
+
+MODULE_AUTHOR("NXP Semiconductor, Inc.");
+MODULE_DESCRIPTION("NXP i.MX95 CSI Pixel Formatter driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/qcom/camss/camss-csid.c b/drivers/media/platform/qcom/camss/camss-csid.c
index 48459b46a981bc..a00791b00feb79 100644
--- a/drivers/media/platform/qcom/camss/camss-csid.c
+++ b/drivers/media/platform/qcom/camss/camss-csid.c
@@ -987,6 +987,7 @@ static int csid_get_format(struct v4l2_subdev *sd,
  * Return -EINVAL or zero on success
  */
 static int csid_set_format(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
@@ -1036,7 +1037,7 @@ static int csid_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 		}
 	};
 
-	return csid_set_format(sd, fh ? fh->state : NULL, &format);
+	return csid_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 }
 
 /*
diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.c b/drivers/media/platform/qcom/camss/camss-csiphy.c
index 539ac4888b608b..0dd50f3879d9cc 100644
--- a/drivers/media/platform/qcom/camss/camss-csiphy.c
+++ b/drivers/media/platform/qcom/camss/camss-csiphy.c
@@ -503,6 +503,7 @@ static int csiphy_get_format(struct v4l2_subdev *sd,
  * Return -EINVAL or zero on success
  */
 static int csiphy_set_format(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_format *fmt)
 {
@@ -555,7 +556,7 @@ static int csiphy_init_formats(struct v4l2_subdev *sd,
 		}
 	};
 
-	return csiphy_set_format(sd, fh ? fh->state : NULL, &format);
+	return csiphy_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 }
 
 static bool __printf(2, 3)
diff --git a/drivers/media/platform/qcom/camss/camss-ispif.c b/drivers/media/platform/qcom/camss/camss-ispif.c
index 20ccd7b1f11fb7..f266f8532ecc73 100644
--- a/drivers/media/platform/qcom/camss/camss-ispif.c
+++ b/drivers/media/platform/qcom/camss/camss-ispif.c
@@ -1037,6 +1037,7 @@ static int ispif_get_format(struct v4l2_subdev *sd,
  * Return -EINVAL or zero on success
  */
 static int ispif_set_format(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *fmt)
 {
@@ -1085,7 +1086,7 @@ static int ispif_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 		}
 	};
 
-	return ispif_set_format(sd, fh ? fh->state : NULL, &format);
+	return ispif_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 }
 
 /*
diff --git a/drivers/media/platform/qcom/camss/camss-tpg.c b/drivers/media/platform/qcom/camss/camss-tpg.c
index c5b75132add44b..5764a60fa34cba 100644
--- a/drivers/media/platform/qcom/camss/camss-tpg.c
+++ b/drivers/media/platform/qcom/camss/camss-tpg.c
@@ -292,6 +292,7 @@ static int tpg_get_format(struct v4l2_subdev *sd,
 }
 
 static int tpg_set_format(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -322,7 +323,7 @@ static int tpg_init_formats(struct v4l2_subdev *sd,
 		}
 	};
 
-	return tpg_set_format(sd, fh ? fh->state : NULL, &format);
+	return tpg_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 }
 
 static int tpg_s_ctrl(struct v4l2_ctrl *ctrl)
diff --git a/drivers/media/platform/qcom/camss/camss-vfe.c b/drivers/media/platform/qcom/camss/camss-vfe.c
index 319d1915898847..c14d97a131f69b 100644
--- a/drivers/media/platform/qcom/camss/camss-vfe.c
+++ b/drivers/media/platform/qcom/camss/camss-vfe.c
@@ -1562,6 +1562,7 @@ static int vfe_get_format(struct v4l2_subdev *sd,
 }
 
 static int vfe_set_selection(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel);
 
@@ -1574,6 +1575,7 @@ static int vfe_set_selection(struct v4l2_subdev *sd,
  * Return -EINVAL or zero on success
  */
 static int vfe_set_format(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -1608,7 +1610,7 @@ static int vfe_set_format(struct v4l2_subdev *sd,
 		sel.target = V4L2_SEL_TGT_COMPOSE;
 		sel.r.width = fmt->format.width;
 		sel.r.height = fmt->format.height;
-		ret = vfe_set_selection(sd, sd_state, &sel);
+		ret = vfe_set_selection(sd, ci, sd_state, &sel);
 		if (ret < 0)
 			return ret;
 	}
@@ -1625,6 +1627,7 @@ static int vfe_set_format(struct v4l2_subdev *sd,
  * Return -EINVAL or zero on success
  */
 static int vfe_get_selection(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
@@ -1695,6 +1698,7 @@ static int vfe_get_selection(struct v4l2_subdev *sd,
  * Return -EINVAL or zero on success
  */
 static int vfe_set_selection(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
@@ -1721,7 +1725,7 @@ static int vfe_set_selection(struct v4l2_subdev *sd,
 		crop.pad = MSM_VFE_PAD_SRC;
 		crop.target = V4L2_SEL_TGT_CROP;
 		crop.r = *rect;
-		ret = vfe_set_selection(sd, sd_state, &crop);
+		ret = vfe_set_selection(sd, ci, sd_state, &crop);
 	} else if (sel->target == V4L2_SEL_TGT_CROP &&
 		sel->pad == MSM_VFE_PAD_SRC) {
 		struct v4l2_subdev_format fmt = { 0 };
@@ -1742,7 +1746,7 @@ static int vfe_set_selection(struct v4l2_subdev *sd,
 
 		fmt.format.width = rect->width;
 		fmt.format.height = rect->height;
-		ret = vfe_set_format(sd, sd_state, &fmt);
+		ret = vfe_set_format(sd, ci, sd_state, &fmt);
 	} else {
 		ret = -EINVAL;
 	}
@@ -1772,7 +1776,7 @@ static int vfe_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 		}
 	};
 
-	return vfe_set_format(sd, fh ? fh->state : NULL, &format);
+	return vfe_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 }
 
 /*
diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c
index 2123f6388e3d7e..23f3cc30a15a5e 100644
--- a/drivers/media/platform/qcom/camss/camss.c
+++ b/drivers/media/platform/qcom/camss/camss.c
@@ -4793,30 +4793,23 @@ static int camss_parse_endpoint_node(struct device *dev,
 static int camss_parse_ports(struct camss *camss)
 {
 	struct device *dev = camss->dev;
-	struct fwnode_handle *fwnode = dev_fwnode(dev), *ep;
+	struct fwnode_handle *fwnode = dev_fwnode(dev);
 	int ret;
 
-	fwnode_graph_for_each_endpoint(fwnode, ep) {
+	fwnode_graph_for_each_endpoint_scoped(fwnode, ep) {
 		struct camss_async_subdev *csd;
 
 		csd = v4l2_async_nf_add_fwnode_remote(&camss->notifier, ep,
 						      typeof(*csd));
-		if (IS_ERR(csd)) {
-			ret = PTR_ERR(csd);
-			goto err_cleanup;
-		}
+		if (IS_ERR(csd))
+			return PTR_ERR(csd);
 
 		ret = camss_parse_endpoint_node(dev, ep, csd);
 		if (ret < 0)
-			goto err_cleanup;
+			return ret;
 	}
 
 	return 0;
-
-err_cleanup:
-	fwnode_handle_put(ep);
-
-	return ret;
 }
 
 /*
diff --git a/drivers/media/platform/raspberrypi/rp1-cfe/csi2.c b/drivers/media/platform/raspberrypi/rp1-cfe/csi2.c
index 104908afbf4187..66d0dde6d84018 100644
--- a/drivers/media/platform/raspberrypi/rp1-cfe/csi2.c
+++ b/drivers/media/platform/raspberrypi/rp1-cfe/csi2.c
@@ -404,6 +404,7 @@ static int csi2_init_state(struct v4l2_subdev *sd,
 }
 
 static int csi2_pad_set_fmt(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *state,
 			    struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/raspberrypi/rp1-cfe/pisp-fe.c b/drivers/media/platform/raspberrypi/rp1-cfe/pisp-fe.c
index 05762b1be2bc75..ea21fb1f60cf9f 100644
--- a/drivers/media/platform/raspberrypi/rp1-cfe/pisp-fe.c
+++ b/drivers/media/platform/raspberrypi/rp1-cfe/pisp-fe.c
@@ -429,6 +429,7 @@ static int pisp_fe_init_state(struct v4l2_subdev *sd,
 }
 
 static int pisp_fe_pad_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *state,
 			       struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/renesas/rcar-csi2.c b/drivers/media/platform/renesas/rcar-csi2.c
index 6635f578217553..4851fe649d8aae 100644
--- a/drivers/media/platform/renesas/rcar-csi2.c
+++ b/drivers/media/platform/renesas/rcar-csi2.c
@@ -702,6 +702,17 @@ static const struct rcar_csi2_format rcar_csi2_formats[] = {
 	},
 };
 
+static const struct v4l2_mbus_framefmt rcar_csi2_default_fmt = {
+	.width		= 1920,
+	.height		= 1080,
+	.code		= MEDIA_BUS_FMT_RGB888_1X24,
+	.colorspace	= V4L2_COLORSPACE_SRGB,
+	.field		= V4L2_FIELD_NONE,
+	.ycbcr_enc	= V4L2_YCBCR_ENC_DEFAULT,
+	.quantization	= V4L2_QUANTIZATION_DEFAULT,
+	.xfer_func	= V4L2_XFER_FUNC_DEFAULT,
+};
+
 static const struct rcar_csi2_format *rcsi2_code_to_fmt(unsigned int code)
 {
 	unsigned int i;
@@ -773,7 +784,7 @@ struct rcar_csi2 {
 
 	int channel_vc[4];
 
-	int stream_count;
+	u64 enabled_sink_streams_mask;
 
 	bool cphy;
 	unsigned short lanes;
@@ -1023,17 +1034,24 @@ static int rcsi2_calc_mbps(struct rcar_csi2 *priv,
 	 */
 	freq = v4l2_get_link_freq(remote_pad, 0, 0);
 	if (freq < 0) {
+		const struct v4l2_subdev_route *route;
 		const struct rcar_csi2_format *format;
 		const struct v4l2_mbus_framefmt *fmt;
 		unsigned int lanes;
 		unsigned int bpp;
 		int ret;
 
+		if (state->routing.num_routes != 1)
+			return -EINVAL;
+
 		ret = rcsi2_get_active_lanes(priv, &lanes);
 		if (ret)
 			return ret;
 
-		fmt = v4l2_subdev_state_get_format(state, RCAR_CSI2_SINK);
+		route = &state->routing.routes[0];
+
+		fmt = v4l2_subdev_state_get_format(state, route->sink_pad,
+						   route->sink_stream);
 		if (!fmt)
 			return -EINVAL;
 
@@ -1062,52 +1080,93 @@ static int rcsi2_calc_mbps(struct rcar_csi2 *priv,
 static int rcsi2_start_receiver_gen3(struct rcar_csi2 *priv,
 				     struct v4l2_subdev_state *state)
 {
-	const struct rcar_csi2_format *format;
-	u32 phycnt, vcdt = 0, vcdt2 = 0, fld = 0;
-	const struct v4l2_mbus_framefmt *fmt;
+	u32 phycnt, vcdt = 0, vcdt2 = 0;
+	u32 fld = FLD_DET_SEL(1);
+	struct v4l2_mbus_frame_desc source_fd;
+	struct v4l2_subdev_route *route;
 	unsigned int lanes;
-	unsigned int i;
 	int mbps, ret;
+	u8 ch = 0;
+
+	ret = v4l2_subdev_call(priv->remote, pad, get_frame_desc,
+			       priv->remote_pad, &source_fd);
+	if (ret && ret != -ENOIOCTLCMD)
+		return ret;
 
-	/* Use the format on the sink pad to compute the receiver config. */
-	fmt = v4l2_subdev_state_get_format(state, RCAR_CSI2_SINK);
+	if (ret == -ENOIOCTLCMD) {
+		/* Create a fallback source_fd */
+		struct v4l2_mbus_frame_desc *fd = &source_fd;
+		const struct v4l2_subdev_route *route;
+		const struct rcar_csi2_format *format;
+		struct v4l2_mbus_framefmt *fmt;
 
-	dev_dbg(priv->dev, "Input size (%ux%u%c)\n",
-		fmt->width, fmt->height,
-		fmt->field == V4L2_FIELD_NONE ? 'p' : 'i');
+		if (state->routing.num_routes != 1)
+			return -EINVAL;
 
-	/* Code is validated in set_fmt. */
-	format = rcsi2_code_to_fmt(fmt->code);
-	if (!format)
-		return -EINVAL;
+		route = &state->routing.routes[0];
 
-	/*
-	 * Enable all supported CSI-2 channels with virtual channel and
-	 * data type matching.
-	 *
-	 * NOTE: It's not possible to get individual datatype for each
-	 *       source virtual channel. Once this is possible in V4L2
-	 *       it should be used here.
-	 */
-	for (i = 0; i < priv->info->num_channels; i++) {
+		fmt = v4l2_subdev_state_get_format(state, route->sink_pad,
+						   route->sink_stream);
+		if (!fmt)
+			return -EINVAL;
+
+		format = rcsi2_code_to_fmt(fmt->code);
+		if (!format)
+			return -EINVAL;
+
+		memset(fd, 0, sizeof(*fd));
+
+		fd->num_entries = 1;
+		fd->type = V4L2_MBUS_FRAME_DESC_TYPE_CSI2;
+		fd->entry[0].stream = 0;
+		fd->entry[0].pixelcode = fmt->code;
+		fd->entry[0].bus.csi2.vc = 0;
+		fd->entry[0].bus.csi2.dt = format->datatype;
+	}
+
+	for_each_active_route(&state->routing, route) {
+		const struct v4l2_mbus_frame_desc_entry *source_entry = NULL;
+		const struct v4l2_mbus_framefmt *fmt;
+		unsigned int i;
 		u32 vcdt_part;
 
-		if (priv->channel_vc[i] < 0)
-			continue;
+		for (i = 0; i < source_fd.num_entries; i++) {
+			if (source_fd.entry[i].stream == route->sink_stream) {
+				source_entry = &source_fd.entry[i];
+				break;
+			}
+		}
+
+		if (!source_entry) {
+			dev_err(priv->dev,
+				"Failed to find stream from source frame desc\n");
+			return -EPIPE;
+		}
 
-		vcdt_part = VCDT_SEL_VC(priv->channel_vc[i]) | VCDT_VCDTN_EN |
-			VCDT_SEL_DTN_ON | VCDT_SEL_DT(format->datatype);
+		vcdt_part = VCDT_SEL_VC(source_entry->bus.csi2.vc) |
+			    VCDT_VCDTN_EN | VCDT_SEL_DTN_ON |
+			    VCDT_SEL_DT(source_entry->bus.csi2.dt);
 
 		/* Store in correct reg and offset. */
-		if (i < 2)
-			vcdt |= vcdt_part << ((i % 2) * 16);
+		if (ch < 2)
+			vcdt |= vcdt_part << ((ch % 2) * 16);
 		else
-			vcdt2 |= vcdt_part << ((i % 2) * 16);
-	}
+			vcdt2 |= vcdt_part << ((ch % 2) * 16);
+
+		fmt = v4l2_subdev_state_get_format(state, RCAR_CSI2_SINK,
+						   route->sink_stream);
+		if (!fmt)
+			return -EINVAL;
 
-	if (fmt->field == V4L2_FIELD_ALTERNATE)
-		fld = FLD_DET_SEL(1) | FLD_FLD_EN(3) | FLD_FLD_EN(2) |
-		      FLD_FLD_EN(1) | FLD_FLD_EN(0);
+		dev_dbg(priv->dev, "Input size (%ux%u%c)\n",
+			fmt->width, fmt->height,
+			fmt->field == V4L2_FIELD_NONE ? 'p' : 'i');
+
+		if (fmt->field == V4L2_FIELD_ALTERNATE)
+			fld |= FLD_FLD_EN(ch);
+
+		ch++;
+	}
 
 	/*
 	 * Get the number of active data lanes inspecting the remote mbus
@@ -1822,20 +1881,12 @@ static int rcsi2_start(struct rcar_csi2 *priv, struct v4l2_subdev_state *state)
 		return ret;
 	}
 
-	ret = v4l2_subdev_enable_streams(priv->remote, priv->remote_pad,
-					 BIT_ULL(0));
-	if (ret) {
-		rcsi2_enter_standby(priv);
-		return ret;
-	}
-
 	return 0;
 }
 
 static void rcsi2_stop(struct rcar_csi2 *priv)
 {
 	rcsi2_enter_standby(priv);
-	v4l2_subdev_disable_streams(priv->remote, priv->remote_pad, BIT_ULL(0));
 }
 
 static int rcsi2_enable_streams(struct v4l2_subdev *sd,
@@ -1843,21 +1894,32 @@ static int rcsi2_enable_streams(struct v4l2_subdev *sd,
 				u64 source_streams_mask)
 {
 	struct rcar_csi2 *priv = sd_to_csi2(sd);
-	int ret = 0;
-
-	if (source_streams_mask != 1)
-		return -EINVAL;
+	u64 sink_streams;
+	int ret;
 
 	if (!priv->remote)
 		return -ENODEV;
 
-	if (priv->stream_count == 0) {
+	if (!priv->enabled_sink_streams_mask) {
 		ret = rcsi2_start(priv, state);
 		if (ret)
 			return ret;
 	}
 
-	priv->stream_count += 1;
+	sink_streams = v4l2_subdev_state_xlate_streams(state,
+						       source_pad,
+						       RCAR_CSI2_SINK,
+						       &source_streams_mask);
+
+	ret = v4l2_subdev_enable_streams(priv->remote, priv->remote_pad,
+					 sink_streams);
+	if (ret) {
+		if (!priv->enabled_sink_streams_mask)
+			rcsi2_stop(priv);
+		return ret;
+	}
+
+	priv->enabled_sink_streams_mask |= sink_streams;
 
 	return ret;
 }
@@ -1867,28 +1929,36 @@ static int rcsi2_disable_streams(struct v4l2_subdev *sd,
 				 u32 source_pad, u64 source_streams_mask)
 {
 	struct rcar_csi2 *priv = sd_to_csi2(sd);
-	int ret = 0;
-
-	if (source_streams_mask != 1)
-		return -EINVAL;
+	u64 sink_streams;
+	int ret;
 
 	if (!priv->remote)
 		return -ENODEV;
 
-	if (priv->stream_count == 1)
+	sink_streams = v4l2_subdev_state_xlate_streams(state,
+						       source_pad,
+						       RCAR_CSI2_SINK,
+						       &source_streams_mask);
+
+	if (priv->enabled_sink_streams_mask == sink_streams)
 		rcsi2_stop(priv);
 
-	priv->stream_count -= 1;
+	ret = v4l2_subdev_disable_streams(priv->remote, priv->remote_pad,
+					  sink_streams);
+	if (ret)
+		return ret;
 
-	return ret;
+	priv->enabled_sink_streams_mask &= ~sink_streams;
+
+	return 0;
 }
 
 static int rcsi2_set_pad_format(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_format *format)
 {
-	struct rcar_csi2 *priv = sd_to_csi2(sd);
-	unsigned int num_pads = rcsi2_num_pads(priv);
+	struct v4l2_mbus_framefmt *fmt;
 
 	if (format->pad > RCAR_CSI2_SINK)
 		return v4l2_subdev_get_fmt(sd, state, format);
@@ -1896,21 +1966,136 @@ static int rcsi2_set_pad_format(struct v4l2_subdev *sd,
 	if (!rcsi2_code_to_fmt(format->format.code))
 		format->format.code = rcar_csi2_formats[0].code;
 
-	*v4l2_subdev_state_get_format(state, format->pad) = format->format;
+	/* Set sink format. */
+	fmt = v4l2_subdev_state_get_format(state, format->pad, format->stream);
+	if (!fmt)
+		return -EINVAL;
+
+	*fmt = format->format;
+
+	/* Propagate the format to the source pad. */
+	fmt = v4l2_subdev_state_get_opposite_stream_format(state, format->pad,
+							   format->stream);
+	if (!fmt)
+		return -EINVAL;
 
-	/* Propagate the format to the source pads. */
-	for (unsigned int i = RCAR_CSI2_SOURCE_VC0; i < num_pads; i++)
-		*v4l2_subdev_state_get_format(state, i) = format->format;
+	*fmt = format->format;
 
 	return 0;
 }
 
+static int rcsi2_set_routing(struct v4l2_subdev *sd,
+			     struct v4l2_subdev_state *state,
+			     enum v4l2_subdev_format_whence which,
+			     struct v4l2_subdev_krouting *routing)
+{
+	struct rcar_csi2 *priv = sd_to_csi2(sd);
+	int ret;
+
+	if (priv->info->use_isp) {
+		ret = v4l2_subdev_routing_validate(sd, routing,
+						   V4L2_SUBDEV_ROUTING_ONLY_1_TO_1);
+	} else {
+		ret = v4l2_subdev_routing_validate(sd, routing,
+						   V4L2_SUBDEV_ROUTING_ONLY_1_TO_1 |
+						   V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING);
+	}
+
+	if (ret)
+		return ret;
+
+	ret = v4l2_subdev_set_routing_with_fmt(sd, state, routing,
+					       &rcar_csi2_default_fmt);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int rcsi2_get_frame_desc_fallback(struct v4l2_subdev *sd,
+					 unsigned int pad,
+					 struct v4l2_mbus_frame_desc *fd)
+{
+	struct v4l2_subdev_route *route;
+	const struct rcar_csi2_format *format;
+	struct v4l2_subdev_state *state;
+	struct v4l2_mbus_framefmt *fmt;
+	int ret = 0;
+
+	state = v4l2_subdev_lock_and_get_active_state(sd);
+
+	if (state->routing.num_routes != 1) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	route = &state->routing.routes[0];
+
+	if (route->source_pad != pad) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	fmt = v4l2_subdev_state_get_format(state, route->sink_pad,
+					   route->sink_stream);
+	if (!fmt) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	format = rcsi2_code_to_fmt(fmt->code);
+	if (!format) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	fd->num_entries = 1;
+	fd->type = V4L2_MBUS_FRAME_DESC_TYPE_CSI2;
+	fd->entry[0].stream = route->source_stream;
+	fd->entry[0].pixelcode = fmt->code;
+	fd->entry[0].bus.csi2.vc = 0;
+	fd->entry[0].bus.csi2.dt = format->datatype;
+
+out:
+	v4l2_subdev_unlock_state(state);
+
+	return ret;
+}
+
+static int rcsi2_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
+				struct v4l2_mbus_frame_desc *fd)
+{
+	struct rcar_csi2 *priv = sd_to_csi2(sd);
+	int ret;
+
+	/*
+	 * Providing a frame descriptor on the source pad only makes sense on
+	 * Gen4. On Gen3 the CSI-2 IP is used to demux the CSI streams to each
+	 * VIN instance, while on Gen4 this job is done by the Channel Selector
+	 * of the ISP, and that needs the frame descriptors. So WARN if someone
+	 * calls this on Gen3, as it indicates a driver bug.
+	 */
+	if (WARN_ON(!priv->info->use_isp))
+		return -ENOTTY;
+
+	if (WARN_ON(pad != RCAR_CSI2_SOURCE_VC0))
+		return -EINVAL;
+
+	ret = v4l2_subdev_get_frame_desc_passthrough(sd, pad, fd);
+	if (ret == -ENOIOCTLCMD)
+		ret = rcsi2_get_frame_desc_fallback(sd, pad, fd);
+	return ret;
+}
+
 static const struct v4l2_subdev_pad_ops rcar_csi2_pad_ops = {
 	.enable_streams = rcsi2_enable_streams,
 	.disable_streams = rcsi2_disable_streams,
 
 	.set_fmt = rcsi2_set_pad_format,
 	.get_fmt = v4l2_subdev_get_fmt,
+
+	.set_routing = rcsi2_set_routing,
+	.get_frame_desc = rcsi2_get_frame_desc,
 };
 
 static const struct v4l2_subdev_ops rcar_csi2_subdev_ops = {
@@ -1920,24 +2105,23 @@ static const struct v4l2_subdev_ops rcar_csi2_subdev_ops = {
 static int rcsi2_init_state(struct v4l2_subdev *sd,
 			    struct v4l2_subdev_state *state)
 {
-	struct rcar_csi2 *priv = sd_to_csi2(sd);
-	unsigned int num_pads = rcsi2_num_pads(priv);
-
-	static const struct v4l2_mbus_framefmt rcar_csi2_default_fmt = {
-		.width		= 1920,
-		.height		= 1080,
-		.code		= MEDIA_BUS_FMT_RGB888_1X24,
-		.colorspace	= V4L2_COLORSPACE_SRGB,
-		.field		= V4L2_FIELD_NONE,
-		.ycbcr_enc	= V4L2_YCBCR_ENC_DEFAULT,
-		.quantization	= V4L2_QUANTIZATION_DEFAULT,
-		.xfer_func	= V4L2_XFER_FUNC_DEFAULT,
+	static struct v4l2_subdev_route routes[] = {
+		{
+			.sink_pad = RCAR_CSI2_SINK,
+			.sink_stream = 0,
+			.source_pad = RCAR_CSI2_SOURCE_VC0,
+			.source_stream = 0,
+			.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
+		},
 	};
 
-	for (unsigned int i = RCAR_CSI2_SINK; i < num_pads; i++)
-		*v4l2_subdev_state_get_format(state, i) = rcar_csi2_default_fmt;
+	static const struct v4l2_subdev_krouting routing = {
+		.num_routes = ARRAY_SIZE(routes),
+		.routes = routes,
+	};
 
-	return 0;
+	return v4l2_subdev_set_routing_with_fmt(sd, state, &routing,
+						&rcar_csi2_default_fmt);
 }
 
 static const struct v4l2_subdev_internal_ops rcar_csi2_internal_ops = {
@@ -1971,14 +2155,44 @@ static irqreturn_t rcsi2_irq_thread(int irq, void *data)
 {
 	struct v4l2_subdev_state *state;
 	struct rcar_csi2 *priv = data;
+	int ret;
 
 	state = v4l2_subdev_lock_and_get_active_state(&priv->subdev);
 
+	if (!priv->enabled_sink_streams_mask)
+		goto out;
+
 	rcsi2_stop(priv);
+
+	ret = v4l2_subdev_disable_streams(priv->remote, priv->remote_pad,
+					  priv->enabled_sink_streams_mask);
+	if (ret) {
+		dev_warn(priv->dev,
+			 "Error recovery: failed to disable streams: %d\n",
+			 ret);
+		goto out;
+	}
+
 	usleep_range(1000, 2000);
-	if (rcsi2_start(priv, state))
-		dev_warn(priv->dev, "Failed to restart CSI-2 receiver\n");
 
+	ret = rcsi2_start(priv, state);
+	if (ret) {
+		dev_warn(priv->dev,
+			 "Error recovery: failed to start CSI-2 receiver: %d\n",
+			 ret);
+		goto out;
+	}
+
+	ret = v4l2_subdev_enable_streams(priv->remote, priv->remote_pad,
+					 priv->enabled_sink_streams_mask);
+	if (ret) {
+		dev_warn(priv->dev,
+			 "Error recovery: failed to start streams: %d\n",
+			 ret);
+		goto out;
+	}
+
+out:
 	v4l2_subdev_unlock_state(state);
 
 	return IRQ_HANDLED;
@@ -2569,8 +2783,6 @@ static int rcsi2_probe(struct platform_device *pdev)
 
 	priv->dev = &pdev->dev;
 
-	priv->stream_count = 0;
-
 	ret = rcsi2_probe_resources(priv, pdev);
 	if (ret) {
 		dev_err(priv->dev, "Failed to get resources\n");
@@ -2590,7 +2802,8 @@ static int rcsi2_probe(struct platform_device *pdev)
 	v4l2_set_subdevdata(&priv->subdev, &pdev->dev);
 	snprintf(priv->subdev.name, sizeof(priv->subdev.name), "%s %s",
 		 KBUILD_MODNAME, dev_name(&pdev->dev));
-	priv->subdev.flags = V4L2_SUBDEV_FL_HAS_DEVNODE;
+	priv->subdev.flags = V4L2_SUBDEV_FL_HAS_DEVNODE |
+			     V4L2_SUBDEV_FL_STREAMS;
 
 	priv->subdev.entity.function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
 	priv->subdev.entity.ops = &rcar_csi2_entity_ops;
diff --git a/drivers/media/platform/renesas/rcar-isp/csisp.c b/drivers/media/platform/renesas/rcar-isp/csisp.c
index 53ce47020d1704..635eeacb3b19c7 100644
--- a/drivers/media/platform/renesas/rcar-isp/csisp.c
+++ b/drivers/media/platform/renesas/rcar-isp/csisp.c
@@ -41,6 +41,9 @@
 #define ISPCS_DT_CODE03_EN0				BIT(7)
 #define ISPCS_DT_CODE03_DT0(dt)				((dt) & 0x3f)
 
+/* ISP has 12 channels, of which channels 4 to 11 are connected to VINs */
+#define ISPCS_NUM_CHANNELS	12
+
 struct rcar_isp_format {
 	u32 code;
 	unsigned int datatype;
@@ -123,6 +126,17 @@ static const struct rcar_isp_format rcar_isp_formats[] = {
 	},
 };
 
+static const struct v4l2_mbus_framefmt risp_default_fmt = {
+	.width = 1920,
+	.height = 1080,
+	.code = MEDIA_BUS_FMT_RGB888_1X24,
+	.colorspace = V4L2_COLORSPACE_SRGB,
+	.field = V4L2_FIELD_NONE,
+	.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT,
+	.quantization = V4L2_QUANTIZATION_DEFAULT,
+	.xfer_func = V4L2_XFER_FUNC_DEFAULT,
+};
+
 static const struct rcar_isp_format *risp_code_to_fmt(unsigned int code)
 {
 	unsigned int i;
@@ -214,24 +228,82 @@ static void risp_power_off(struct rcar_isp *isp)
 	pm_runtime_put(isp->dev);
 }
 
-static int risp_start(struct rcar_isp *isp, struct v4l2_subdev_state *state)
+static int risp_configure_routing(struct rcar_isp *isp,
+				  struct v4l2_subdev_state *state)
 {
-	const struct v4l2_mbus_framefmt *fmt;
-	const struct rcar_isp_format *format;
-	unsigned int vc;
-	u32 sel_csi = 0;
+	struct v4l2_mbus_frame_desc source_fd;
+	struct v4l2_subdev_route *route;
 	int ret;
 
-	fmt = v4l2_subdev_state_get_format(state, RCAR_ISP_SINK);
-	if (!fmt)
-		return -EINVAL;
+	ret = v4l2_subdev_call(isp->remote, pad, get_frame_desc,
+			       isp->remote_pad, &source_fd);
+	if (ret)
+		return ret;
 
-	format = risp_code_to_fmt(fmt->code);
-	if (!format) {
-		dev_err(isp->dev, "Unsupported bus format\n");
-		return -EINVAL;
+	/* Clear the channel registers */
+	for (unsigned int ch = 0; ch < ISPCS_NUM_CHANNELS; ++ch) {
+		risp_write_cs(isp, ISPCS_FILTER_ID_CH_REG(ch), 0);
+		risp_write_cs(isp, ISPCS_DT_CODE03_CH_REG(ch), 0);
 	}
 
+	for_each_active_route(&state->routing, route) {
+		struct v4l2_mbus_frame_desc_entry *source_entry = NULL;
+		const struct rcar_isp_format *format;
+		const struct v4l2_mbus_framefmt *fmt;
+		unsigned int i;
+		u8 vc, dt, ch;
+		u32 v;
+
+		for (i = 0; i < source_fd.num_entries; i++) {
+			if (source_fd.entry[i].stream == route->sink_stream) {
+				source_entry = &source_fd.entry[i];
+				break;
+			}
+		}
+
+		if (!source_entry) {
+			dev_err(isp->dev,
+				"Failed to find source frame desc entry for stream\n");
+			return -EPIPE;
+		}
+
+		vc = source_entry->bus.csi2.vc;
+		dt = source_entry->bus.csi2.dt;
+		/* Channels 4 - 11 go to VIN */
+		ch = route->source_pad - 1 + 4;
+
+		fmt = v4l2_subdev_state_get_format(state, route->sink_pad,
+						   route->sink_stream);
+		if (!fmt)
+			return -EINVAL;
+
+		format = risp_code_to_fmt(fmt->code);
+		if (!format) {
+			dev_err(isp->dev, "Unsupported bus format\n");
+			return -EINVAL;
+		}
+
+		/* VC Filtering */
+		risp_write_cs(isp, ISPCS_FILTER_ID_CH_REG(ch), BIT(vc));
+
+		/* DT Filtering */
+		risp_write_cs(isp, ISPCS_DT_CODE03_CH_REG(ch),
+			      ISPCS_DT_CODE03_EN0 | ISPCS_DT_CODE03_DT0(dt));
+
+		/* Proc mode */
+		v = risp_read_cs(isp, ISPPROCMODE_DT_REG(dt));
+		v |= ISPPROCMODE_DT_PROC_MODE_VCn(vc, format->procmode);
+		risp_write_cs(isp, ISPPROCMODE_DT_REG(dt), v);
+	}
+
+	return 0;
+}
+
+static int risp_start(struct rcar_isp *isp, struct v4l2_subdev_state *state)
+{
+	u32 sel_csi = 0;
+	int ret;
+
 	ret = risp_power_on(isp);
 	if (ret) {
 		dev_err(isp->dev, "Failed to power on ISP\n");
@@ -245,41 +317,18 @@ static int risp_start(struct rcar_isp *isp, struct v4l2_subdev_state *state)
 	risp_write_cs(isp, ISPINPUTSEL0_REG,
 		      risp_read_cs(isp, ISPINPUTSEL0_REG) | sel_csi);
 
-	/* Configure Channel Selector. */
-	for (vc = 0; vc < 4; vc++) {
-		u8 ch = vc + 4;
-		u8 dt = format->datatype;
-
-		risp_write_cs(isp, ISPCS_FILTER_ID_CH_REG(ch), BIT(vc));
-		risp_write_cs(isp, ISPCS_DT_CODE03_CH_REG(ch),
-			      ISPCS_DT_CODE03_EN3 | ISPCS_DT_CODE03_DT3(dt) |
-			      ISPCS_DT_CODE03_EN2 | ISPCS_DT_CODE03_DT2(dt) |
-			      ISPCS_DT_CODE03_EN1 | ISPCS_DT_CODE03_DT1(dt) |
-			      ISPCS_DT_CODE03_EN0 | ISPCS_DT_CODE03_DT0(dt));
-	}
-
-	/* Setup processing method. */
-	risp_write_cs(isp, ISPPROCMODE_DT_REG(format->datatype),
-		      ISPPROCMODE_DT_PROC_MODE_VCn(3, format->procmode) |
-		      ISPPROCMODE_DT_PROC_MODE_VCn(2, format->procmode) |
-		      ISPPROCMODE_DT_PROC_MODE_VCn(1, format->procmode) |
-		      ISPPROCMODE_DT_PROC_MODE_VCn(0, format->procmode));
+	ret = risp_configure_routing(isp, state);
+	if (ret)
+		return ret;
 
 	/* Start ISP. */
 	risp_write_cs(isp, ISPSTART_REG, ISPSTART_START);
 
-	ret = v4l2_subdev_enable_streams(isp->remote, isp->remote_pad,
-					 BIT_ULL(0));
-	if (ret)
-		risp_power_off(isp);
-
-	return ret;
+	return 0;
 }
 
 static void risp_stop(struct rcar_isp *isp)
 {
-	v4l2_subdev_disable_streams(isp->remote, isp->remote_pad, BIT_ULL(0));
-
 	/* Stop ISP. */
 	risp_write_cs(isp, ISPSTART_REG, ISPSTART_STOP);
 
@@ -291,7 +340,8 @@ static int risp_enable_streams(struct v4l2_subdev *sd,
 			       u64 source_streams_mask)
 {
 	struct rcar_isp *isp = sd_to_isp(sd);
-	int ret = 0;
+	u64 sink_streams;
+	int ret;
 
 	if (source_streams_mask != 1)
 		return -EINVAL;
@@ -305,9 +355,22 @@ static int risp_enable_streams(struct v4l2_subdev *sd,
 			return ret;
 	}
 
+	sink_streams = v4l2_subdev_state_xlate_streams(state,
+						       source_pad,
+						       RCAR_ISP_SINK,
+						       &source_streams_mask);
+
+	ret = v4l2_subdev_enable_streams(isp->remote, isp->remote_pad,
+					 sink_streams);
+	if (ret) {
+		if (isp->stream_count == 0)
+			risp_stop(isp);
+		return ret;
+	}
+
 	isp->stream_count += 1;
 
-	return ret;
+	return 0;
 }
 
 static int risp_disable_streams(struct v4l2_subdev *sd,
@@ -315,6 +378,8 @@ static int risp_disable_streams(struct v4l2_subdev *sd,
 				u64 source_streams_mask)
 {
 	struct rcar_isp *isp = sd_to_isp(sd);
+	u64 sink_streams;
+	int ret;
 
 	if (source_streams_mask != 1)
 		return -EINVAL;
@@ -322,6 +387,15 @@ static int risp_disable_streams(struct v4l2_subdev *sd,
 	if (!isp->remote)
 		return -ENODEV;
 
+	sink_streams = v4l2_subdev_state_xlate_streams(state,
+						       source_pad,
+						       RCAR_ISP_SINK,
+						       &source_streams_mask);
+
+	ret = v4l2_subdev_disable_streams(isp->remote, isp->remote_pad, sink_streams);
+	if (ret)
+		return ret;
+
 	if (isp->stream_count == 1)
 		risp_stop(isp);
 
@@ -331,10 +405,11 @@ static int risp_disable_streams(struct v4l2_subdev *sd,
 }
 
 static int risp_set_pad_format(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *state,
 			       struct v4l2_subdev_format *format)
 {
-	struct v4l2_mbus_framefmt *framefmt;
+	struct v4l2_mbus_framefmt *fmt;
 
 	if (format->pad > RCAR_ISP_SINK)
 		return v4l2_subdev_get_fmt(sd, state, format);
@@ -342,10 +417,41 @@ static int risp_set_pad_format(struct v4l2_subdev *sd,
 	if (!risp_code_to_fmt(format->format.code))
 		format->format.code = rcar_isp_formats[0].code;
 
-	for (unsigned int i = 0; i < RCAR_ISP_NUM_PADS; i++) {
-		framefmt = v4l2_subdev_state_get_format(state, i);
-		*framefmt = format->format;
-	}
+	/* Set sink format. */
+	fmt = v4l2_subdev_state_get_format(state, format->pad, format->stream);
+	if (!fmt)
+		return -EINVAL;
+
+	*fmt = format->format;
+
+	/* Propagate the format to the source pad. */
+	fmt = v4l2_subdev_state_get_opposite_stream_format(state, format->pad,
+							   format->stream);
+	if (!fmt)
+		return -EINVAL;
+
+	*fmt = format->format;
+
+	return 0;
+}
+
+static int risp_set_routing(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_state *state,
+			    enum v4l2_subdev_format_whence which,
+			    struct v4l2_subdev_krouting *routing)
+{
+	int ret;
+
+	ret = v4l2_subdev_routing_validate(sd, routing,
+					   V4L2_SUBDEV_ROUTING_ONLY_1_TO_1 |
+					   V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING);
+	if (ret)
+		return ret;
+
+	ret = v4l2_subdev_set_routing_with_fmt(sd, state, routing,
+					       &risp_default_fmt);
+	if (ret)
+		return ret;
 
 	return 0;
 }
@@ -356,12 +462,39 @@ static const struct v4l2_subdev_pad_ops risp_pad_ops = {
 	.set_fmt = risp_set_pad_format,
 	.get_fmt = v4l2_subdev_get_fmt,
 	.link_validate = v4l2_subdev_link_validate_default,
+	.set_routing = risp_set_routing,
 };
 
 static const struct v4l2_subdev_ops rcar_isp_subdev_ops = {
 	.pad	= &risp_pad_ops,
 };
 
+static int risp_init_state(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_state *state)
+{
+	static struct v4l2_subdev_route routes[] = {
+		{
+			.sink_pad = RCAR_ISP_SINK,
+			.sink_stream = 0,
+			.source_pad = RCAR_ISP_PORT0,
+			.source_stream = 0,
+			.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
+		},
+	};
+
+	static const struct v4l2_subdev_krouting routing = {
+		.num_routes = ARRAY_SIZE(routes),
+		.routes = routes,
+	};
+
+	return v4l2_subdev_set_routing_with_fmt(sd, state, &routing,
+						&risp_default_fmt);
+}
+
+static const struct v4l2_subdev_internal_ops risp_internal_ops = {
+	.init_state = risp_init_state,
+};
+
 /* -----------------------------------------------------------------------------
  * Async handling and registration of subdevices and links
  */
@@ -534,11 +667,12 @@ static int risp_probe(struct platform_device *pdev)
 
 	isp->subdev.owner = THIS_MODULE;
 	isp->subdev.dev = &pdev->dev;
+	isp->subdev.internal_ops = &risp_internal_ops;
 	v4l2_subdev_init(&isp->subdev, &rcar_isp_subdev_ops);
 	v4l2_set_subdevdata(&isp->subdev, &pdev->dev);
 	snprintf(isp->subdev.name, sizeof(isp->subdev.name), "%s %s",
 		 KBUILD_MODNAME, dev_name(&pdev->dev));
-	isp->subdev.flags = V4L2_SUBDEV_FL_HAS_DEVNODE;
+	isp->subdev.flags = V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_STREAMS;
 
 	isp->subdev.entity.function = MEDIA_ENT_F_VID_MUX;
 	isp->subdev.entity.ops = &risp_entity_ops;
diff --git a/drivers/media/platform/renesas/rcar-vin/rcar-core.c b/drivers/media/platform/renesas/rcar-vin/rcar-core.c
index c8d564aa1eba87..2fcea715101cef 100644
--- a/drivers/media/platform/renesas/rcar-vin/rcar-core.c
+++ b/drivers/media/platform/renesas/rcar-vin/rcar-core.c
@@ -673,23 +673,26 @@ static int rvin_csi2_create_link(struct rvin_group *group, unsigned int id,
 	struct media_entity *source = &group->remotes[route->csi].subdev->entity;
 	struct media_entity *sink = &group->vin[id]->vdev.entity;
 	struct media_pad *sink_pad = &sink->pads[0];
+	struct media_pad *source_pad;
+	unsigned int source_idx;
 	unsigned int channel;
-	int ret;
 
-	for (channel = 0; channel < 4; channel++) {
-		unsigned int source_idx = rvin_group_csi_channel_to_pad(channel);
-		struct media_pad *source_pad = &source->pads[source_idx];
+	/*
+	 * The channels from CSI-2 blocks and the VIN groups have a set of
+	 * hardcoded routing options to choose from. We only support the routing
+	 * where all VINs in a group are connected to the same CSI-2 block,
+	 * and the Nth VIN in the group is connected to the Nth CSI-2 channel.
+	 */
 
-		/* Skip if link already exists. */
-		if (media_entity_find_link(source_pad, sink_pad))
-			continue;
+	channel = id % 4;
+	source_idx = rvin_group_csi_channel_to_pad(channel);
+	source_pad = &source->pads[source_idx];
 
-		ret = media_create_pad_link(source, source_idx, sink, 0, 0);
-		if (ret)
-			return ret;
-	}
+	/* Skip if link already exists. */
+	if (media_entity_find_link(source_pad, sink_pad))
+		return 0;
 
-	return 0;
+	return media_create_pad_link(source, source_idx, sink, 0, 0);
 }
 
 static int rvin_parallel_setup_links(struct rvin_group *group)
diff --git a/drivers/media/platform/renesas/rcar-vin/rcar-dma.c b/drivers/media/platform/renesas/rcar-vin/rcar-dma.c
index 73cda0e2d45a94..5e515d16c16463 100644
--- a/drivers/media/platform/renesas/rcar-vin/rcar-dma.c
+++ b/drivers/media/platform/renesas/rcar-vin/rcar-dma.c
@@ -678,7 +678,7 @@ void rvin_crop_scale_comp(struct rvin_dev *vin)
 
 	/*
 	 * VNIS_REG has four lowest bits always 0, i.e. the stride has to be
-	 * aligned to 16 bytes. This is done in rvin_format_bytesperline().
+	 * aligned to 16 pixels. This is done in rvin_format_bytesperline().
 	 */
 
 	fmt = rvin_format_from_pixel(vin, vin->format.pixelformat);
diff --git a/drivers/media/platform/renesas/rcar_drif.c b/drivers/media/platform/renesas/rcar_drif.c
index 0844934f7aa68b..bd67e8e16f537e 100644
--- a/drivers/media/platform/renesas/rcar_drif.c
+++ b/drivers/media/platform/renesas/rcar_drif.c
@@ -464,7 +464,7 @@ rcar_drif_get_fbuf(struct rcar_drif_sdr *sdr)
 					rcar_drif_frame_buf, list);
 	if (!fbuf) {
 		/*
-		 * App is late in enqueing buffers. Samples lost & there will
+		 * App is late in enqueuing buffers. Samples lost & there will
 		 * be a gap in sequence number when app recovers
 		 */
 		rdrif_dbg(sdr, "\napp late: prod %u\n", sdr->produced);
diff --git a/drivers/media/platform/renesas/renesas-ceu.c b/drivers/media/platform/renesas/renesas-ceu.c
index 65f7659a9e02b9..47b9e475330153 100644
--- a/drivers/media/platform/renesas/renesas-ceu.c
+++ b/drivers/media/platform/renesas/renesas-ceu.c
@@ -841,12 +841,13 @@ static int __ceu_try_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt
 	 * time.
 	 */
 	sd_format.format.code = mbus_code;
-	ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, &pad_state, &sd_format);
+	ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, NULL, &pad_state,
+			       &sd_format);
 	if (ret) {
 		if (ret == -EINVAL) {
 			/* fallback */
 			sd_format.format.code = mbus_code_old;
-			ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt,
+			ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, NULL,
 					       &pad_state, &sd_format);
 		}
 
@@ -900,7 +901,7 @@ static int ceu_set_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt)
 
 	format.format.code = mbus_code;
 	v4l2_fill_mbus_format_mplane(&format.format, &v4l2_fmt->fmt.pix_mp);
-	ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, NULL, &format);
+	ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, NULL, NULL, &format);
 	if (ret)
 		return ret;
 
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c
index 798ef29162624e..8c7f46ccce5f77 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-core.c
@@ -360,7 +360,7 @@ static const struct rzg2l_cru_info rzg3e_cru_info = {
 	.max_width = 4095,
 	.max_height = 4095,
 	.image_conv = ICnIPMC_C0,
-	.has_stride = true,
+	.stride_align = 128,
 	.regs = rzg3e_cru_regs,
 	.irq_handler = rzg3e_cru_irq,
 	.enable_interrupts = rzg3e_cru_enable_interrupts,
@@ -405,6 +405,7 @@ static const struct rzg2l_cru_info rzg2l_cru_info = {
 	.max_width = 2800,
 	.max_height = 4095,
 	.image_conv = ICnMC,
+	.stride_align = 1,
 	.regs = rzg2l_cru_regs,
 	.irq_handler = rzg2l_cru_irq,
 	.enable_interrupts = rzg2l_cru_enable_interrupts,
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
index b426bc7898bf72..2c192d370dcb2d 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
@@ -75,7 +75,7 @@ struct rzg2l_cru_info {
 	unsigned int max_height;
 	u16 image_conv;
 	const u16 *regs;
-	bool has_stride;
+	u8 stride_align;
 	irqreturn_t (*irq_handler)(int irq, void *data);
 	void (*enable_interrupts)(struct rzg2l_cru_dev *cru);
 	void (*disable_interrupts)(struct rzg2l_cru_dev *cru);
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c
index 6dc4b53607b4c6..c17283cba3a622 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-csi2.c
@@ -633,6 +633,7 @@ static int rzg2l_csi2_post_streamoff(struct v4l2_subdev *sd)
 }
 
 static int rzg2l_csi2_set_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_format *fmt)
 {
@@ -687,7 +688,7 @@ static int rzg2l_csi2_init_state(struct v4l2_subdev *sd,
 	fmt.format.quantization = V4L2_QUANTIZATION_DEFAULT;
 	fmt.format.xfer_func = V4L2_XFER_FUNC_DEFAULT;
 
-	return rzg2l_csi2_set_format(sd, sd_state, &fmt);
+	return rzg2l_csi2_set_format(sd, NULL, sd_state, &fmt);
 }
 
 static int rzg2l_csi2_enum_mbus_code(struct v4l2_subdev *sd,
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-ip.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-ip.c
index 5f2c87858bfe6d..21dd0162fd6ee1 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-ip.c
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-ip.c
@@ -200,6 +200,7 @@ static int rzg2l_cru_ip_s_stream(struct v4l2_subdev *sd, int enable)
 }
 
 static int rzg2l_cru_ip_set_format(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_format *fmt)
 {
@@ -300,7 +301,7 @@ static int rzg2l_cru_ip_init_state(struct v4l2_subdev *sd,
 	fmt.format.quantization = V4L2_QUANTIZATION_DEFAULT;
 	fmt.format.xfer_func = V4L2_XFER_FUNC_DEFAULT;
 
-	return rzg2l_cru_ip_set_format(sd, sd_state, &fmt);
+	return rzg2l_cru_ip_set_format(sd, NULL, sd_state, &fmt);
 }
 
 static const struct v4l2_subdev_video_ops rzg2l_cru_ip_video_ops = {
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c
index 91eda503424836..a7b6dce665705a 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c
@@ -32,7 +32,6 @@
 #define RZG2L_CRU_DEFAULT_COLORSPACE	V4L2_COLORSPACE_SRGB
 
 #define RZG2L_CRU_STRIDE_MAX		32640
-#define RZG2L_CRU_STRIDE_ALIGN		128
 
 struct rzg2l_cru_buffer {
 	struct vb2_v4l2_buffer vb;
@@ -277,11 +276,11 @@ static void rzg2l_cru_initialize_axi(struct rzg2l_cru_dev *cru)
 		rzg2l_cru_fill_hw_slot(cru, cru->num_buf - 1);
 	}
 
-	if (info->has_stride) {
+	if (info->stride_align > 1) {
 		u32 stride = cru->format.bytesperline;
 		u32 amnis;
 
-		stride /= RZG2L_CRU_STRIDE_ALIGN;
+		stride /= info->stride_align;
 		amnis = rzg2l_cru_read(cru, AMnIS) & ~AMnIS_IS_MASK;
 		rzg2l_cru_write(cru, AMnIS, amnis | AMnIS_IS(stride));
 	}
@@ -849,12 +848,8 @@ static void rzg2l_cru_format_align(struct rzg2l_cru_dev *cru,
 	v4l_bound_align_image(&pix->width, 320, info->max_width, 1,
 			      &pix->height, 240, info->max_height, 2, 0);
 
-	v4l2_fill_pixfmt(pix, pix->pixelformat, pix->width, pix->height);
-
-	if (info->has_stride) {
-		pix->bytesperline = ALIGN(pix->bytesperline, RZG2L_CRU_STRIDE_ALIGN);
-		pix->sizeimage = pix->bytesperline * pix->height;
-	}
+	v4l2_fill_pixfmt_aligned(pix, pix->pixelformat, pix->width, pix->height,
+				 info->stride_align);
 
 	dev_dbg(cru->dev, "Format %ux%u bpl: %u size: %u\n",
 		pix->width, pix->height, pix->bytesperline, pix->sizeimage);
diff --git a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-subdev.c b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-subdev.c
index b1659544eaa0dc..04e168b9526f22 100644
--- a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-subdev.c
+++ b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-subdev.c
@@ -157,6 +157,7 @@ static int rzv2h_ivc_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int rzv2h_ivc_set_fmt(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/renesas/sh_vou.c b/drivers/media/platform/renesas/sh_vou.c
index 4ad7ae188d5b25..6d4ec80edd8f0d 100644
--- a/drivers/media/platform/renesas/sh_vou.c
+++ b/drivers/media/platform/renesas/sh_vou.c
@@ -714,7 +714,7 @@ static int sh_vou_set_fmt_vid_out(struct sh_vou_device *vou_dev,
 	mbfmt->width = geo.output.width;
 	mbfmt->height = geo.output.height;
 	ret = v4l2_device_call_until_err(&vou_dev->v4l2_dev, 0, pad,
-					 set_fmt, NULL, &format);
+					 set_fmt, NULL, NULL, &format);
 	/* Must be implemented, so, don't check for -ENOIOCTLCMD */
 	if (ret < 0)
 		return ret;
@@ -974,11 +974,11 @@ static int sh_vou_s_selection(struct file *file, void *fh,
 	 * final encoder configuration.
 	 */
 	v4l2_device_call_until_err(&vou_dev->v4l2_dev, 0, pad,
-				   set_selection, NULL, &sd_sel);
+				   set_selection, NULL, NULL, &sd_sel);
 	format.format.width = geo.output.width;
 	format.format.height = geo.output.height;
 	ret = v4l2_device_call_until_err(&vou_dev->v4l2_dev, 0, pad,
-					 set_fmt, NULL, &format);
+					 set_fmt, NULL, NULL, &format);
 	/* Must be implemented, so, don't check for -ENOIOCTLCMD */
 	if (ret < 0)
 		return ret;
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_brx.c b/drivers/media/platform/renesas/vsp1/vsp1_brx.c
index 360a4250294736..a70b82e22e69cc 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_brx.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_brx.c
@@ -124,6 +124,7 @@ static void brx_try_format(struct vsp1_brx *brx,
 }
 
 static int brx_set_format(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
@@ -165,6 +166,7 @@ static int brx_set_format(struct v4l2_subdev *subdev,
 }
 
 static int brx_get_selection(struct v4l2_subdev *subdev,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
@@ -200,6 +202,7 @@ static int brx_get_selection(struct v4l2_subdev *subdev,
 }
 
 static int brx_set_selection(struct v4l2_subdev *subdev,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_dl.c b/drivers/media/platform/renesas/vsp1/vsp1_dl.c
index 6430f2ec8b327a..8257759421f3c3 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_dl.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_dl.c
@@ -649,7 +649,7 @@ static void __vsp1_dl_list_put(struct vsp1_dl_list *dl)
 	dl->post_cmd = NULL;
 
 	/*
-	 * body0 is reused as as an optimisation as presently every display list
+	 * body0 is reused as an optimisation as presently every display list
 	 * has at least one body, thus we reinitialise the entries list.
 	 */
 	dl->body0->num_entries = 0;
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drm.c b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
index 9cd5c025d2be84..adb89e69938903 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
@@ -137,7 +137,8 @@ static int vsp1_du_insert_uif(struct vsp1_device *vsp1,
 
 	format.pad = UIF_PAD_SINK;
 
-	ret = v4l2_subdev_call(&uif->subdev, pad, set_fmt, NULL, &format);
+	ret = v4l2_subdev_call(&uif->subdev, pad, set_fmt, NULL, NULL,
+			       &format);
 	if (ret < 0)
 		return ret;
 
@@ -184,7 +185,7 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 	format.format.ycbcr_enc = input->ycbcr_enc;
 	format.format.quantization = input->quantization;
 
-	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
@@ -199,6 +200,7 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 	sel.r = input->crop;
 
 	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_selection, NULL,
+			       NULL,
 			       &sel);
 	if (ret < 0)
 		return ret;
@@ -226,7 +228,7 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 
 	format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
 
-	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&rpf->entity.subdev, pad, set_fmt, NULL, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
@@ -240,7 +242,7 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 	/* BRx sink, propagate the format from the RPF source. */
 	format.pad = brx_input;
 
-	ret = v4l2_subdev_call(&pipe->brx->subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->brx->subdev, pad, set_fmt, NULL, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
@@ -254,6 +256,7 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 	sel.r = vsp1->drm->inputs[rpf->entity.index].compose;
 
 	ret = v4l2_subdev_call(&pipe->brx->subdev, pad, set_selection, NULL,
+			       NULL,
 			       &sel);
 	if (ret < 0)
 		return ret;
@@ -387,7 +390,7 @@ static int vsp1_du_pipeline_setup_brx(struct vsp1_device *vsp1,
 	format.format.height = drm_pipe->height;
 	format.format.field = V4L2_FIELD_NONE;
 
-	ret = v4l2_subdev_call(&brx->subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&brx->subdev, pad, set_fmt, NULL, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
@@ -538,7 +541,8 @@ static int vsp1_du_pipeline_setup_output(struct vsp1_device *vsp1,
 	format.format.code = MEDIA_BUS_FMT_ARGB8888_1X32;
 	format.format.field = V4L2_FIELD_NONE;
 
-	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->output->entity.subdev, pad, set_fmt,
+			       NULL, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
@@ -558,7 +562,7 @@ static int vsp1_du_pipeline_setup_output(struct vsp1_device *vsp1,
 		format.format.code, pipe->output->entity.index);
 
 	format.pad = LIF_PAD_SINK;
-	ret = v4l2_subdev_call(&pipe->lif->subdev, pad, set_fmt, NULL,
+	ret = v4l2_subdev_call(&pipe->lif->subdev, pad, set_fmt, NULL, NULL,
 			       &format);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_entity.c b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
index 26b21559878d1a..f7359d34794758 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
@@ -300,6 +300,7 @@ int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev,
  * entity's limits, and propagates the sink pad format to the source pad.
  */
 int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt)
 {
@@ -380,7 +381,8 @@ static int vsp1_entity_init_state(struct v4l2_subdev *subdev,
 			       : V4L2_SUBDEV_FORMAT_ACTIVE,
 		};
 
-		v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &format);
+		v4l2_subdev_call(subdev, pad, set_fmt, NULL, sd_state,
+				 &format);
 	}
 
 	return 0;
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_entity.h b/drivers/media/platform/renesas/vsp1/vsp1_entity.h
index c0c1fe7d3e406b..677a1efc104a63 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_entity.h
+++ b/drivers/media/platform/renesas/vsp1/vsp1_entity.h
@@ -188,6 +188,7 @@ int vsp1_subdev_get_pad_format(struct v4l2_subdev *subdev,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt);
 int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt);
 int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev,
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_histo.c b/drivers/media/platform/renesas/vsp1/vsp1_histo.c
index 97dbfb93abe97e..bd9efb42fd1fcf 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_histo.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_histo.c
@@ -185,6 +185,7 @@ static int histo_enum_frame_size(struct v4l2_subdev *subdev,
 }
 
 static int histo_get_selection(struct v4l2_subdev *subdev,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_selection *sel)
 {
@@ -306,6 +307,7 @@ static int histo_set_compose(struct v4l2_subdev *subdev,
 }
 
 static int histo_set_selection(struct v4l2_subdev *subdev,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_selection *sel)
 {
@@ -330,6 +332,7 @@ static int histo_set_selection(struct v4l2_subdev *subdev,
 }
 
 static int histo_set_format(struct v4l2_subdev *subdev,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_hsit.c b/drivers/media/platform/renesas/vsp1/vsp1_hsit.c
index df069c228243a7..d734b5775af656 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_hsit.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_hsit.c
@@ -109,6 +109,7 @@ static int hsit_enum_frame_size(struct v4l2_subdev *subdev,
 }
 
 static int hsit_set_format(struct v4l2_subdev *subdev,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
index ced01870acd6ef..ff5894265b5426 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
@@ -110,6 +110,7 @@ static int vsp1_rwpf_enum_frame_size(struct v4l2_subdev *subdev,
 }
 
 static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_format *fmt)
 {
@@ -214,6 +215,7 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
 }
 
 static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_selection *sel)
 {
@@ -255,6 +257,7 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
 }
 
 static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_sru.c b/drivers/media/platform/renesas/vsp1/vsp1_sru.c
index 3fd9fde5c7242a..5e1cf60be31122 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_sru.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_sru.c
@@ -210,6 +210,7 @@ static void sru_try_format(struct vsp1_sru *sru,
 }
 
 static int sru_set_format(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_uds.c b/drivers/media/platform/renesas/vsp1/vsp1_uds.c
index 9f7bb112929ed2..d129964ec14ddb 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_uds.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_uds.c
@@ -193,6 +193,7 @@ static void uds_try_format(struct vsp1_uds *uds,
 }
 
 static int uds_set_format(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_uif.c b/drivers/media/platform/renesas/vsp1/vsp1_uif.c
index 52dbfe58a70d22..16ddcd594134a5 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_uif.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_uif.c
@@ -54,6 +54,7 @@ static const unsigned int uif_codes[] = {
 };
 
 static int uif_get_selection(struct v4l2_subdev *subdev,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
@@ -92,6 +93,7 @@ static int uif_get_selection(struct v4l2_subdev *subdev,
 }
 
 static int uif_set_selection(struct v4l2_subdev *subdev,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_vspx.c b/drivers/media/platform/renesas/vsp1/vsp1_vspx.c
index 1673479be0ffef..5c39bacfb13f31 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_vspx.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_vspx.c
@@ -120,7 +120,8 @@ static int vsp1_vspx_rwpf_set_subdev_fmt(struct vsp1_device *vsp1,
 	format.format.field = V4L2_FIELD_NONE;
 	format.format.code = rwpf->fmtinfo->mbus;
 
-	return v4l2_subdev_call(&ent->subdev, pad, set_fmt, NULL, &format);
+	return v4l2_subdev_call(&ent->subdev, pad, set_fmt, NULL, NULL,
+				&format);
 }
 
 /* Configure the RPF->IIF->WPF pipeline for ConfigDMA or RAW image transfer. */
diff --git a/drivers/media/platform/rockchip/rkcif/rkcif-interface.c b/drivers/media/platform/rockchip/rkcif/rkcif-interface.c
index 414a9980cf2e55..2cba42d7c5d9c5 100644
--- a/drivers/media/platform/rockchip/rkcif/rkcif-interface.c
+++ b/drivers/media/platform/rockchip/rkcif/rkcif-interface.c
@@ -25,6 +25,7 @@ static const struct media_entity_operations rkcif_interface_media_ops = {
 };
 
 static int rkcif_interface_set_fmt(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_format *format)
 {
@@ -77,6 +78,7 @@ static int rkcif_interface_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int rkcif_interface_get_sel(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_selection *sel)
 {
@@ -115,6 +117,7 @@ static int rkcif_interface_get_sel(struct v4l2_subdev *sd,
 }
 
 static int rkcif_interface_set_sel(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-csi.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-csi.c
index ddc6182f3e4bda..9fa8a0407c90ec 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-csi.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-csi.c
@@ -304,6 +304,7 @@ static int rkisp1_csi_init_state(struct v4l2_subdev *sd,
 }
 
 static int rkisp1_csi_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
index 1791c02a40ae18..f90e01301943c7 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-dev.c
@@ -187,7 +187,6 @@ static int rkisp1_subdev_notifier_register(struct rkisp1_device *rkisp1)
 {
 	struct v4l2_async_notifier *ntf = &rkisp1->notifier;
 	struct fwnode_handle *fwnode = dev_fwnode(rkisp1->dev);
-	struct fwnode_handle *ep;
 	unsigned int index = 0;
 	int ret = 0;
 
@@ -195,7 +194,7 @@ static int rkisp1_subdev_notifier_register(struct rkisp1_device *rkisp1)
 
 	ntf->ops = &rkisp1_subdev_notifier_ops;
 
-	fwnode_graph_for_each_endpoint(fwnode, ep) {
+	fwnode_graph_for_each_endpoint_scoped(fwnode, ep) {
 		struct fwnode_handle *port;
 		struct v4l2_fwnode_endpoint vep = { };
 		struct rkisp1_sensor_async *rk_asd;
@@ -286,7 +285,6 @@ static int rkisp1_subdev_notifier_register(struct rkisp1_device *rkisp1)
 	}
 
 	if (ret) {
-		fwnode_handle_put(ep);
 		v4l2_async_nf_cleanup(ntf);
 		return ret;
 	}
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c
index 2311672cedb1b6..4043a75ca3f0c5 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c
@@ -817,6 +817,7 @@ static void rkisp1_isp_set_sink_fmt(struct rkisp1_isp *isp,
 }
 
 static int rkisp1_isp_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
@@ -834,6 +835,7 @@ static int rkisp1_isp_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int rkisp1_isp_get_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -872,6 +874,7 @@ static int rkisp1_isp_get_selection(struct v4l2_subdev *sd,
 }
 
 static int rkisp1_isp_set_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c
index 8e6b753d3081ac..5926917434af44 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c
@@ -543,6 +543,7 @@ static void rkisp1_rsz_set_sink_fmt(struct rkisp1_resizer *rsz,
 }
 
 static int rkisp1_rsz_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
@@ -558,6 +559,7 @@ static int rkisp1_rsz_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int rkisp1_rsz_get_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -591,6 +593,7 @@ static int rkisp1_rsz_get_selection(struct v4l2_subdev *sd,
 }
 
 static int rkisp1_rsz_set_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-capture.c b/drivers/media/platform/samsung/exynos4-is/fimc-capture.c
index d85811f4b8c52b..8d7e7b190532bc 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-capture.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-capture.c
@@ -796,7 +796,8 @@ static int fimc_pipeline_try_format(struct fimc_ctx *ctx,
 			sd = media_entity_to_v4l2_subdev(me);
 
 			sfmt.pad = 0;
-			ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, &sfmt);
+			ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, NULL,
+					       &sfmt);
 			if (ret)
 				return ret;
 
@@ -804,7 +805,7 @@ static int fimc_pipeline_try_format(struct fimc_ctx *ctx,
 				sfmt.pad = me->num_pads - 1;
 				mf->code = tfmt->code;
 				ret = v4l2_subdev_call(sd, pad, set_fmt, NULL,
-									&sfmt);
+						       NULL, &sfmt);
 				if (ret)
 					return ret;
 			}
@@ -1509,6 +1510,7 @@ static int fimc_subdev_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int fimc_subdev_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt)
 {
@@ -1575,6 +1577,7 @@ static int fimc_subdev_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int fimc_subdev_get_selection(struct v4l2_subdev *sd,
+				     const struct v4l2_subdev_client_info *ci,
 				     struct v4l2_subdev_state *sd_state,
 				     struct v4l2_subdev_selection *sel)
 {
@@ -1631,6 +1634,7 @@ static int fimc_subdev_get_selection(struct v4l2_subdev *sd,
 }
 
 static int fimc_subdev_set_selection(struct v4l2_subdev *sd,
+				     const struct v4l2_subdev_client_info *ci,
 				     struct v4l2_subdev_state *sd_state,
 				     struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-isp.c b/drivers/media/platform/samsung/exynos4-is/fimc-isp.c
index 3c5d7bee2655b5..a0a9093081dde1 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-isp.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-isp.c
@@ -191,6 +191,7 @@ static void __isp_subdev_try_format(struct fimc_isp *isp,
 }
 
 static int fimc_isp_subdev_set_fmt(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-lite.c b/drivers/media/platform/samsung/exynos4-is/fimc-lite.c
index 8be20fd32d1c72..1b7bae15189b44 100644
--- a/drivers/media/platform/samsung/exynos4-is/fimc-lite.c
+++ b/drivers/media/platform/samsung/exynos4-is/fimc-lite.c
@@ -1052,6 +1052,7 @@ static int fimc_lite_subdev_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int fimc_lite_subdev_set_fmt(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_format *fmt)
 {
@@ -1113,6 +1114,7 @@ static int fimc_lite_subdev_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int fimc_lite_subdev_get_selection(struct v4l2_subdev *sd,
+					  const struct v4l2_subdev_client_info *ci,
 					  struct v4l2_subdev_state *sd_state,
 					  struct v4l2_subdev_selection *sel)
 {
@@ -1148,6 +1150,7 @@ static int fimc_lite_subdev_get_selection(struct v4l2_subdev *sd,
 }
 
 static int fimc_lite_subdev_set_selection(struct v4l2_subdev *sd,
+					  const struct v4l2_subdev_client_info *ci,
 					  struct v4l2_subdev_state *sd_state,
 					  struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/samsung/exynos4-is/mipi-csis.c b/drivers/media/platform/samsung/exynos4-is/mipi-csis.c
index 452880b5350cd1..da6e67a79c09b0 100644
--- a/drivers/media/platform/samsung/exynos4-is/mipi-csis.c
+++ b/drivers/media/platform/samsung/exynos4-is/mipi-csis.c
@@ -575,6 +575,7 @@ static struct v4l2_mbus_framefmt *__s5pcsis_get_format(
 }
 
 static int s5pcsis_set_fmt(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/samsung/s3c-camif/camif-capture.c b/drivers/media/platform/samsung/s3c-camif/camif-capture.c
index ed1a1d693293b3..98dfd394a1aa82 100644
--- a/drivers/media/platform/samsung/s3c-camif/camif-capture.c
+++ b/drivers/media/platform/samsung/s3c-camif/camif-capture.c
@@ -1275,6 +1275,7 @@ static void __camif_subdev_try_format(struct camif_dev *camif,
 }
 
 static int s3c_camif_subdev_set_fmt(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_format *fmt)
 {
@@ -1342,6 +1343,7 @@ static int s3c_camif_subdev_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int s3c_camif_subdev_get_selection(struct v4l2_subdev *sd,
+					  const struct v4l2_subdev_client_info *ci,
 					  struct v4l2_subdev_state *sd_state,
 					  struct v4l2_subdev_selection *sel)
 {
@@ -1429,6 +1431,7 @@ static void __camif_try_crop(struct camif_dev *camif, struct v4l2_rect *r)
 }
 
 static int s3c_camif_subdev_set_selection(struct v4l2_subdev *sd,
+					  const struct v4l2_subdev_client_info *ci,
 					  struct v4l2_subdev_state *sd_state,
 					  struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/platform/samsung/s3c-camif/camif-core.c b/drivers/media/platform/samsung/s3c-camif/camif-core.c
index bb06847f3a63e0..2ff71e375efed2 100644
--- a/drivers/media/platform/samsung/s3c-camif/camif-core.c
+++ b/drivers/media/platform/samsung/s3c-camif/camif-core.c
@@ -227,7 +227,7 @@ static int camif_register_sensor(struct camif_dev *camif)
 		return 0;
 
 	format.pad = CAMIF_SD_PAD_SINK;
-	v4l2_subdev_call(&camif->subdev, pad, set_fmt, NULL, &format);
+	v4l2_subdev_call(&camif->subdev, pad, set_fmt, NULL, NULL, &format);
 
 	v4l2_info(sd, "Initial format from sensor: %dx%d, %#x\n",
 		  format.format.width, format.format.height,
diff --git a/drivers/media/platform/st/stm32/stm32-csi.c b/drivers/media/platform/st/stm32/stm32-csi.c
index fd2b6dfbd44c57..9dda2b1c2d8c7e 100644
--- a/drivers/media/platform/st/stm32/stm32-csi.c
+++ b/drivers/media/platform/st/stm32/stm32-csi.c
@@ -9,6 +9,7 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
@@ -736,6 +737,7 @@ static int stm32_csi_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int stm32_csi_set_pad_format(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_format *format)
 {
@@ -825,7 +827,7 @@ static int stm32_csi_async_bound(struct v4l2_async_notifier *notifier,
 	int remote_pad;
 
 	remote_pad = media_entity_get_fwnode_pad(&s_subdev->entity,
-						 s_subdev->fwnode,
+						 asd->match.fwnode,
 						 MEDIA_PAD_FL_SOURCE);
 	if (remote_pad < 0) {
 		dev_err(csidev->dev, "Couldn't find output pad for subdev %s\n",
@@ -1059,6 +1061,7 @@ static int stm32_csi_probe(struct platform_device *pdev)
 	return 0;
 
 err_cleanup:
+	v4l2_async_nf_unregister(&csidev->notifier);
 	v4l2_async_nf_cleanup(&csidev->notifier);
 	return ret;
 }
@@ -1067,6 +1070,8 @@ static void stm32_csi_remove(struct platform_device *pdev)
 {
 	struct stm32_csi_dev *csidev = platform_get_drvdata(pdev);
 
+	v4l2_async_nf_unregister(&csidev->notifier);
+	v4l2_async_nf_cleanup(&csidev->notifier);
 	v4l2_async_unregister_subdev(&csidev->sd);
 
 	pm_runtime_disable(&pdev->dev);
diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c
index aa79fe60ff0c9c..3d774cb4907b3a 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmi.c
@@ -738,7 +738,7 @@ static int dcmi_pipeline_s_fmt(struct stm32_dcmi *dcmi,
 			format->format.width, format->format.height);
 
 		fmt.pad = pad->index;
-		ret = v4l2_subdev_call(subdev, pad, set_fmt, NULL, &fmt);
+		ret = v4l2_subdev_call(subdev, pad, set_fmt, NULL, NULL, &fmt);
 		if (ret < 0) {
 			dev_err(dcmi->dev, "%s: Failed to set format 0x%x %ux%u on \"%s\":%d pad (%d)\n",
 				__func__, format->format.code,
@@ -1022,6 +1022,27 @@ static void __find_outer_frame_size(struct stm32_dcmi *dcmi,
 	*framesize = *match;
 }
 
+static int dcmi_source_call_try_state_set_fmt(struct v4l2_subdev *source,
+					      struct v4l2_subdev_format *fmt)
+{
+	static struct lock_class_key lock_key;
+	const char *lock_name = KBUILD_BASENAME ":" __stringify(__LINE__)
+		":state->lock";
+	struct v4l2_subdev_state *source_state;
+	int ret;
+
+	source_state = __v4l2_subdev_state_alloc(source, lock_name, &lock_key);
+	if (IS_ERR(source_state))
+		return PTR_ERR(source_state);
+
+	v4l2_subdev_lock_state(source_state);
+	ret = v4l2_subdev_call(source, pad, set_fmt, NULL, source_state, fmt);
+	v4l2_subdev_unlock_state(source_state);
+	__v4l2_subdev_state_free(source_state);
+
+	return ret;
+}
+
 static int dcmi_try_fmt(struct stm32_dcmi *dcmi, struct v4l2_format *f,
 			const struct dcmi_format **sd_format,
 			struct dcmi_framesize *sd_framesize)
@@ -1063,8 +1084,8 @@ static int dcmi_try_fmt(struct stm32_dcmi *dcmi, struct v4l2_format *f,
 	}
 
 	v4l2_fill_mbus_format(&format.format, pix, sd_fmt->mbus_code);
-	ret = v4l2_subdev_call_state_try(dcmi->source, pad, set_fmt, &format);
-	if (ret < 0)
+	ret = dcmi_source_call_try_state_set_fmt(dcmi->source, &format);
+	if (ret)
 		return ret;
 
 	/* Update pix regarding to what sensor can do */
@@ -1224,7 +1245,7 @@ static int dcmi_set_sensor_format(struct stm32_dcmi *dcmi,
 	}
 
 	v4l2_fill_mbus_format(&format.format, pix, sd_fmt->mbus_code);
-	ret = v4l2_subdev_call_state_try(dcmi->source, pad, set_fmt, &format);
+	ret = dcmi_source_call_try_state_set_fmt(dcmi->source, &format);
 	if (ret < 0)
 		return ret;
 
@@ -1246,7 +1267,7 @@ static int dcmi_get_sensor_bounds(struct stm32_dcmi *dcmi,
 	/*
 	 * Get sensor bounds first
 	 */
-	ret = v4l2_subdev_call(dcmi->source, pad, get_selection,
+	ret = v4l2_subdev_call(dcmi->source, pad, get_selection, NULL,
 			       NULL, &bounds);
 	if (!ret)
 		*r = bounds.r;
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/Makefile b/drivers/media/platform/st/stm32/stm32-dcmipp/Makefile
index 159105fb40b88b..e35d45a0aca233 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/Makefile
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/Makefile
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-stm32-dcmipp-y := dcmipp-core.o dcmipp-common.o dcmipp-input.o dcmipp-byteproc.o dcmipp-bytecap.o
+stm32-dcmipp-y := dcmipp-core.o dcmipp-common.o dcmipp-input.o dcmipp-byteproc.o dcmipp-capture.o
+stm32-dcmipp-y += dcmipp-pixelcommon.o dcmipp-isp.o dcmipp-pixelproc.o
 
 obj-$(CONFIG_VIDEO_STM32_DCMIPP) += stm32-dcmipp.o
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c
deleted file mode 100644
index f0e809458489f6..00000000000000
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c
+++ /dev/null
@@ -1,968 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Driver for STM32 Digital Camera Memory Interface Pixel Processor
- *
- * Copyright (C) STMicroelectronics SA 2023
- * Authors: Hugues Fruchet <hugues.fruchet@foss.st.com>
- *          Alain Volmat <alain.volmat@foss.st.com>
- *          for STMicroelectronics.
- */
-
-#include <linux/iopoll.h>
-#include <linux/pm_runtime.h>
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-mc.h>
-#include <media/videobuf2-core.h>
-#include <media/videobuf2-dma-contig.h>
-
-#include "dcmipp-common.h"
-
-#define DCMIPP_PRSR		0x1f8
-#define DCMIPP_CMIER		0x3f0
-#define DCMIPP_CMIER_P0FRAMEIE	BIT(9)
-#define DCMIPP_CMIER_P0VSYNCIE	BIT(10)
-#define DCMIPP_CMIER_P0OVRIE	BIT(15)
-#define DCMIPP_CMIER_P0ALL	(DCMIPP_CMIER_P0VSYNCIE |\
-				 DCMIPP_CMIER_P0FRAMEIE |\
-				 DCMIPP_CMIER_P0OVRIE)
-#define DCMIPP_CMSR1		0x3f4
-#define DCMIPP_CMSR2		0x3f8
-#define DCMIPP_CMSR2_P0FRAMEF	BIT(9)
-#define DCMIPP_CMSR2_P0VSYNCF	BIT(10)
-#define DCMIPP_CMSR2_P0OVRF	BIT(15)
-#define DCMIPP_CMFCR		0x3fc
-#define DCMIPP_P0FSCR		0x404
-#define DCMIPP_P0FSCR_PIPEN	BIT(31)
-#define DCMIPP_P0FCTCR		0x500
-#define DCMIPP_P0FCTCR_CPTREQ	BIT(3)
-#define DCMIPP_P0DCCNTR		0x5b0
-#define DCMIPP_P0DCLMTR		0x5b4
-#define DCMIPP_P0DCLMTR_ENABLE	BIT(31)
-#define DCMIPP_P0DCLMTR_LIMIT_MASK	GENMASK(23, 0)
-#define DCMIPP_P0PPM0AR1	0x5c4
-#define DCMIPP_P0SR		0x5f8
-#define DCMIPP_P0SR_CPTACT	BIT(23)
-
-struct dcmipp_bytecap_pix_map {
-	unsigned int code;
-	u32 pixelformat;
-};
-
-#define PIXMAP_MBUS_PFMT(mbus, fmt)			\
-	{						\
-		.code = MEDIA_BUS_FMT_##mbus,		\
-		.pixelformat = V4L2_PIX_FMT_##fmt	\
-	}
-
-static const struct dcmipp_bytecap_pix_map dcmipp_bytecap_pix_map_list[] = {
-	PIXMAP_MBUS_PFMT(RGB565_2X8_LE, RGB565),
-	PIXMAP_MBUS_PFMT(RGB565_1X16, RGB565),
-	PIXMAP_MBUS_PFMT(RGB888_1X24, RGB24),
-	PIXMAP_MBUS_PFMT(YUYV8_2X8, YUYV),
-	PIXMAP_MBUS_PFMT(YUYV8_1X16, YUYV),
-	PIXMAP_MBUS_PFMT(YVYU8_2X8, YVYU),
-	PIXMAP_MBUS_PFMT(YVYU8_1X16, YVYU),
-	PIXMAP_MBUS_PFMT(UYVY8_2X8, UYVY),
-	PIXMAP_MBUS_PFMT(UYVY8_1X16, UYVY),
-	PIXMAP_MBUS_PFMT(VYUY8_2X8, VYUY),
-	PIXMAP_MBUS_PFMT(VYUY8_1X16, VYUY),
-	PIXMAP_MBUS_PFMT(Y8_1X8, GREY),
-	PIXMAP_MBUS_PFMT(Y10_1X10, Y10),
-	PIXMAP_MBUS_PFMT(Y12_1X12, Y12),
-	PIXMAP_MBUS_PFMT(Y14_1X14, Y14),
-	PIXMAP_MBUS_PFMT(SBGGR8_1X8, SBGGR8),
-	PIXMAP_MBUS_PFMT(SGBRG8_1X8, SGBRG8),
-	PIXMAP_MBUS_PFMT(SGRBG8_1X8, SGRBG8),
-	PIXMAP_MBUS_PFMT(SRGGB8_1X8, SRGGB8),
-	PIXMAP_MBUS_PFMT(SBGGR10_1X10, SBGGR10),
-	PIXMAP_MBUS_PFMT(SGBRG10_1X10, SGBRG10),
-	PIXMAP_MBUS_PFMT(SGRBG10_1X10, SGRBG10),
-	PIXMAP_MBUS_PFMT(SRGGB10_1X10, SRGGB10),
-	PIXMAP_MBUS_PFMT(SBGGR12_1X12, SBGGR12),
-	PIXMAP_MBUS_PFMT(SGBRG12_1X12, SGBRG12),
-	PIXMAP_MBUS_PFMT(SGRBG12_1X12, SGRBG12),
-	PIXMAP_MBUS_PFMT(SRGGB12_1X12, SRGGB12),
-	PIXMAP_MBUS_PFMT(SBGGR14_1X14, SBGGR14),
-	PIXMAP_MBUS_PFMT(SGBRG14_1X14, SGBRG14),
-	PIXMAP_MBUS_PFMT(SGRBG14_1X14, SGRBG14),
-	PIXMAP_MBUS_PFMT(SRGGB14_1X14, SRGGB14),
-	PIXMAP_MBUS_PFMT(JPEG_1X8, JPEG),
-};
-
-static const struct dcmipp_bytecap_pix_map *
-dcmipp_bytecap_pix_map_by_pixelformat(u32 pixelformat)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(dcmipp_bytecap_pix_map_list); i++) {
-		if (dcmipp_bytecap_pix_map_list[i].pixelformat == pixelformat)
-			return &dcmipp_bytecap_pix_map_list[i];
-	}
-
-	return NULL;
-}
-
-struct dcmipp_buf {
-	struct vb2_v4l2_buffer	vb;
-	bool			prepared;
-	dma_addr_t		addr;
-	size_t			size;
-	struct list_head	list;
-};
-
-enum dcmipp_state {
-	DCMIPP_STOPPED = 0,
-	DCMIPP_WAIT_FOR_BUFFER,
-	DCMIPP_RUNNING,
-};
-
-struct dcmipp_bytecap_device {
-	struct dcmipp_ent_device ved;
-	struct video_device vdev;
-	struct device *dev;
-	struct v4l2_pix_format format;
-	struct vb2_queue queue;
-	struct list_head buffers;
-	/*
-	 * Protects concurrent calls of buf queue / irq handler
-	 * and buffer handling related variables / lists
-	 */
-	spinlock_t irqlock;
-	/* mutex used as vdev and queue lock */
-	struct mutex lock;
-	u32 sequence;
-	struct media_pipeline pipe;
-	struct v4l2_subdev *s_subdev;
-	u32 s_subdev_pad_nb;
-
-	enum dcmipp_state state;
-
-	/*
-	 * DCMIPP driver is handling 2 buffers
-	 * active: buffer into which DCMIPP is currently writing into
-	 * next: buffer given to the DCMIPP and which will become
-	 *       automatically active on next VSYNC
-	 */
-	struct dcmipp_buf *active, *next;
-
-	void __iomem *regs;
-
-	u32 cmsr2;
-
-	struct {
-		u32 errors;
-		u32 limit;
-		u32 overrun;
-		u32 buffers;
-		u32 vsync;
-		u32 frame;
-		u32 it;
-		u32 underrun;
-		u32 nactive;
-	} count;
-};
-
-static const struct v4l2_pix_format fmt_default = {
-	.width = DCMIPP_FMT_WIDTH_DEFAULT,
-	.height = DCMIPP_FMT_HEIGHT_DEFAULT,
-	.pixelformat = V4L2_PIX_FMT_RGB565,
-	.field = V4L2_FIELD_NONE,
-	.bytesperline = DCMIPP_FMT_WIDTH_DEFAULT * 2,
-	.sizeimage = DCMIPP_FMT_WIDTH_DEFAULT * DCMIPP_FMT_HEIGHT_DEFAULT * 2,
-	.colorspace = DCMIPP_COLORSPACE_DEFAULT,
-	.ycbcr_enc = DCMIPP_YCBCR_ENC_DEFAULT,
-	.quantization = DCMIPP_QUANTIZATION_DEFAULT,
-	.xfer_func = DCMIPP_XFER_FUNC_DEFAULT,
-};
-
-static int dcmipp_bytecap_querycap(struct file *file, void *priv,
-				   struct v4l2_capability *cap)
-{
-	strscpy(cap->driver, DCMIPP_PDEV_NAME, sizeof(cap->driver));
-	strscpy(cap->card, KBUILD_MODNAME, sizeof(cap->card));
-
-	return 0;
-}
-
-static int dcmipp_bytecap_g_fmt_vid_cap(struct file *file, void *priv,
-					struct v4l2_format *f)
-{
-	struct dcmipp_bytecap_device *vcap = video_drvdata(file);
-
-	f->fmt.pix = vcap->format;
-
-	return 0;
-}
-
-static int dcmipp_bytecap_try_fmt_vid_cap(struct file *file, void *priv,
-					  struct v4l2_format *f)
-{
-	struct dcmipp_bytecap_device *vcap = video_drvdata(file);
-	struct v4l2_pix_format *format = &f->fmt.pix;
-	const struct dcmipp_bytecap_pix_map *vpix;
-	u32 in_w, in_h;
-
-	/* Don't accept a pixelformat that is not on the table */
-	vpix = dcmipp_bytecap_pix_map_by_pixelformat(format->pixelformat);
-	if (!vpix)
-		format->pixelformat = fmt_default.pixelformat;
-
-	/* Adjust width & height */
-	in_w = format->width;
-	in_h = format->height;
-	v4l_bound_align_image(&format->width, DCMIPP_FRAME_MIN_WIDTH,
-			      DCMIPP_FRAME_MAX_WIDTH, 0, &format->height,
-			      DCMIPP_FRAME_MIN_HEIGHT, DCMIPP_FRAME_MAX_HEIGHT,
-			      0, 0);
-	if (format->width != in_w || format->height != in_h)
-		dev_dbg(vcap->dev, "resolution updated: %dx%d -> %dx%d\n",
-			in_w, in_h, format->width, format->height);
-
-	if (format->pixelformat == V4L2_PIX_FMT_JPEG) {
-		format->bytesperline = format->width;
-		format->sizeimage = format->bytesperline * format->height;
-	} else {
-		v4l2_fill_pixfmt(format, format->pixelformat,
-				 format->width, format->height);
-	}
-
-	if (format->field == V4L2_FIELD_ANY)
-		format->field = fmt_default.field;
-
-	dcmipp_colorimetry_clamp(format);
-
-	return 0;
-}
-
-static int dcmipp_bytecap_s_fmt_vid_cap(struct file *file, void *priv,
-					struct v4l2_format *f)
-{
-	struct dcmipp_bytecap_device *vcap = video_drvdata(file);
-	int ret;
-
-	/* Do not change the format while stream is on */
-	if (vb2_is_busy(&vcap->queue))
-		return -EBUSY;
-
-	ret = dcmipp_bytecap_try_fmt_vid_cap(file, priv, f);
-	if (ret)
-		return ret;
-
-	dev_dbg(vcap->dev, "%s: format update: old:%ux%u (0x%p4cc, %u, %u, %u, %u) new:%ux%d (0x%p4cc, %u, %u, %u, %u)\n",
-		vcap->vdev.name,
-		/* old */
-		vcap->format.width, vcap->format.height,
-		&vcap->format.pixelformat, vcap->format.colorspace,
-		vcap->format.quantization, vcap->format.xfer_func,
-		vcap->format.ycbcr_enc,
-		/* new */
-		f->fmt.pix.width, f->fmt.pix.height,
-		&f->fmt.pix.pixelformat, f->fmt.pix.colorspace,
-		f->fmt.pix.quantization, f->fmt.pix.xfer_func,
-		f->fmt.pix.ycbcr_enc);
-
-	vcap->format = f->fmt.pix;
-
-	return 0;
-}
-
-static int dcmipp_bytecap_enum_fmt_vid_cap(struct file *file, void *priv,
-					   struct v4l2_fmtdesc *f)
-{
-	const struct dcmipp_bytecap_pix_map *vpix;
-	unsigned int index = f->index;
-	unsigned int i, prev_pixelformat = 0;
-
-	/*
-	 * List up all formats (or only ones matching f->mbus_code), taking
-	 * care of removing duplicated entries (due to support of both
-	 * parallel & csi 16 bits formats
-	 */
-	for (i = 0; i < ARRAY_SIZE(dcmipp_bytecap_pix_map_list); i++) {
-		vpix = &dcmipp_bytecap_pix_map_list[i];
-		/* Skip formats not matching requested mbus code */
-		if (f->mbus_code && vpix->code != f->mbus_code)
-			continue;
-
-		/* Skip duplicated pixelformat */
-		if (vpix->pixelformat == prev_pixelformat)
-			continue;
-
-		prev_pixelformat = vpix->pixelformat;
-
-		if (index == 0)
-			break;
-
-		index--;
-	}
-
-	if (i == ARRAY_SIZE(dcmipp_bytecap_pix_map_list))
-		return -EINVAL;
-
-	f->pixelformat = vpix->pixelformat;
-
-	return 0;
-}
-
-static int dcmipp_bytecap_enum_framesizes(struct file *file, void *fh,
-					  struct v4l2_frmsizeenum *fsize)
-{
-	const struct dcmipp_bytecap_pix_map *vpix;
-
-	if (fsize->index)
-		return -EINVAL;
-
-	/* Only accept code in the pix map table */
-	vpix = dcmipp_bytecap_pix_map_by_pixelformat(fsize->pixel_format);
-	if (!vpix)
-		return -EINVAL;
-
-	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
-	fsize->stepwise.min_width = DCMIPP_FRAME_MIN_WIDTH;
-	fsize->stepwise.max_width = DCMIPP_FRAME_MAX_WIDTH;
-	fsize->stepwise.min_height = DCMIPP_FRAME_MIN_HEIGHT;
-	fsize->stepwise.max_height = DCMIPP_FRAME_MAX_HEIGHT;
-	fsize->stepwise.step_width = 1;
-	fsize->stepwise.step_height = 1;
-
-	return 0;
-}
-
-static const struct v4l2_file_operations dcmipp_bytecap_fops = {
-	.owner		= THIS_MODULE,
-	.open		= v4l2_fh_open,
-	.release	= vb2_fop_release,
-	.read           = vb2_fop_read,
-	.poll		= vb2_fop_poll,
-	.unlocked_ioctl = video_ioctl2,
-	.mmap           = vb2_fop_mmap,
-};
-
-static const struct v4l2_ioctl_ops dcmipp_bytecap_ioctl_ops = {
-	.vidioc_querycap = dcmipp_bytecap_querycap,
-
-	.vidioc_g_fmt_vid_cap = dcmipp_bytecap_g_fmt_vid_cap,
-	.vidioc_s_fmt_vid_cap = dcmipp_bytecap_s_fmt_vid_cap,
-	.vidioc_try_fmt_vid_cap = dcmipp_bytecap_try_fmt_vid_cap,
-	.vidioc_enum_fmt_vid_cap = dcmipp_bytecap_enum_fmt_vid_cap,
-	.vidioc_enum_framesizes = dcmipp_bytecap_enum_framesizes,
-
-	.vidioc_reqbufs = vb2_ioctl_reqbufs,
-	.vidioc_create_bufs = vb2_ioctl_create_bufs,
-	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
-	.vidioc_querybuf = vb2_ioctl_querybuf,
-	.vidioc_qbuf = vb2_ioctl_qbuf,
-	.vidioc_dqbuf = vb2_ioctl_dqbuf,
-	.vidioc_expbuf = vb2_ioctl_expbuf,
-	.vidioc_streamon = vb2_ioctl_streamon,
-	.vidioc_streamoff = vb2_ioctl_streamoff,
-};
-
-static void dcmipp_start_capture(struct dcmipp_bytecap_device *vcap,
-				 struct dcmipp_buf *buf)
-{
-	/* Set buffer address */
-	reg_write(vcap, DCMIPP_P0PPM0AR1, buf->addr);
-
-	/* Set buffer size */
-	reg_write(vcap, DCMIPP_P0DCLMTR, DCMIPP_P0DCLMTR_ENABLE |
-		  ((buf->size / 4) & DCMIPP_P0DCLMTR_LIMIT_MASK));
-
-	/* Capture request */
-	reg_set(vcap, DCMIPP_P0FCTCR, DCMIPP_P0FCTCR_CPTREQ);
-}
-
-static void dcmipp_bytecap_all_buffers_done(struct dcmipp_bytecap_device *vcap,
-					    enum vb2_buffer_state state)
-{
-	struct dcmipp_buf *buf, *node;
-
-	list_for_each_entry_safe(buf, node, &vcap->buffers, list) {
-		list_del_init(&buf->list);
-		vb2_buffer_done(&buf->vb.vb2_buf, state);
-	}
-}
-
-static int dcmipp_bytecap_start_streaming(struct vb2_queue *vq,
-					  unsigned int count)
-{
-	struct dcmipp_bytecap_device *vcap = vb2_get_drv_priv(vq);
-	struct media_entity *entity = &vcap->vdev.entity;
-	struct dcmipp_buf *buf;
-	struct media_pad *pad;
-	int ret;
-
-	vcap->sequence = 0;
-	memset(&vcap->count, 0, sizeof(vcap->count));
-
-	/*
-	 * Get source subdev - since link is IMMUTABLE, pointer is cached
-	 * within the dcmipp_bytecap_device structure
-	 */
-	if (!vcap->s_subdev) {
-		pad = media_pad_remote_pad_first(&vcap->vdev.entity.pads[0]);
-		if (!pad || !is_media_entity_v4l2_subdev(pad->entity)) {
-			ret = -EINVAL;
-			goto err_buffer_done;
-		}
-		vcap->s_subdev = media_entity_to_v4l2_subdev(pad->entity);
-		vcap->s_subdev_pad_nb = pad->index;
-	}
-
-	ret = pm_runtime_resume_and_get(vcap->dev);
-	if (ret < 0) {
-		dev_err(vcap->dev, "%s: Failed to start streaming, cannot get sync (%d)\n",
-			__func__, ret);
-		goto err_buffer_done;
-	}
-
-	ret = media_pipeline_start(entity->pads, &vcap->pipe);
-	if (ret) {
-		dev_dbg(vcap->dev, "%s: Failed to start streaming, media pipeline start error (%d)\n",
-			__func__, ret);
-		goto err_pm_put;
-	}
-
-	ret = v4l2_subdev_enable_streams(vcap->s_subdev,
-					 vcap->s_subdev_pad_nb, BIT_ULL(0));
-	if (ret)
-		goto err_media_pipeline_stop;
-
-	spin_lock_irq(&vcap->irqlock);
-
-	/* Enable pipe at the end of programming */
-	reg_set(vcap, DCMIPP_P0FSCR, DCMIPP_P0FSCR_PIPEN);
-
-	/*
-	 * vb2 framework guarantee that we have at least 'min_queued_buffers'
-	 * buffers in the list at this moment
-	 */
-	vcap->next = list_first_entry(&vcap->buffers, typeof(*buf), list);
-	dev_dbg(vcap->dev, "Start with next [%d] %p phy=%pad\n",
-		vcap->next->vb.vb2_buf.index, vcap->next, &vcap->next->addr);
-
-	dcmipp_start_capture(vcap, vcap->next);
-
-	/* Enable interruptions */
-	reg_set(vcap, DCMIPP_CMIER, DCMIPP_CMIER_P0ALL);
-
-	vcap->state = DCMIPP_RUNNING;
-
-	spin_unlock_irq(&vcap->irqlock);
-
-	return 0;
-
-err_media_pipeline_stop:
-	media_pipeline_stop(entity->pads);
-err_pm_put:
-	pm_runtime_put(vcap->dev);
-err_buffer_done:
-	spin_lock_irq(&vcap->irqlock);
-	/*
-	 * Return all buffers to vb2 in QUEUED state.
-	 * This will give ownership back to userspace
-	 */
-	dcmipp_bytecap_all_buffers_done(vcap, VB2_BUF_STATE_QUEUED);
-	vcap->active = NULL;
-	spin_unlock_irq(&vcap->irqlock);
-
-	return ret;
-}
-
-static void dcmipp_dump_status(struct dcmipp_bytecap_device *vcap)
-{
-	struct device *dev = vcap->dev;
-
-	dev_dbg(dev, "[DCMIPP_PRSR]  =%#10.8x\n", reg_read(vcap, DCMIPP_PRSR));
-	dev_dbg(dev, "[DCMIPP_P0SR] =%#10.8x\n", reg_read(vcap, DCMIPP_P0SR));
-	dev_dbg(dev, "[DCMIPP_P0DCCNTR]=%#10.8x\n",
-		reg_read(vcap, DCMIPP_P0DCCNTR));
-	dev_dbg(dev, "[DCMIPP_CMSR1] =%#10.8x\n", reg_read(vcap, DCMIPP_CMSR1));
-	dev_dbg(dev, "[DCMIPP_CMSR2] =%#10.8x\n", reg_read(vcap, DCMIPP_CMSR2));
-}
-
-/*
- * Stop the stream engine. Any remaining buffers in the stream queue are
- * dequeued and passed on to the vb2 framework marked as STATE_ERROR.
- */
-static void dcmipp_bytecap_stop_streaming(struct vb2_queue *vq)
-{
-	struct dcmipp_bytecap_device *vcap = vb2_get_drv_priv(vq);
-	int ret;
-	u32 status;
-
-	ret = v4l2_subdev_disable_streams(vcap->s_subdev,
-					  vcap->s_subdev_pad_nb, BIT_ULL(0));
-	if (ret)
-		dev_warn(vcap->dev, "Failed to disable stream\n");
-
-	/* Stop the media pipeline */
-	media_pipeline_stop(vcap->vdev.entity.pads);
-
-	/* Disable interruptions */
-	reg_clear(vcap, DCMIPP_CMIER, DCMIPP_CMIER_P0ALL);
-
-	/* Stop capture */
-	reg_clear(vcap, DCMIPP_P0FCTCR, DCMIPP_P0FCTCR_CPTREQ);
-
-	/* Wait until CPTACT become 0 */
-	ret = readl_relaxed_poll_timeout(vcap->regs + DCMIPP_P0SR, status,
-					 !(status & DCMIPP_P0SR_CPTACT),
-					 20 * USEC_PER_MSEC,
-					 1000 * USEC_PER_MSEC);
-	if (ret)
-		dev_warn(vcap->dev, "Timeout when stopping\n");
-
-	/* Disable pipe */
-	reg_clear(vcap, DCMIPP_P0FSCR, DCMIPP_P0FSCR_PIPEN);
-
-	/* Clear any pending interrupts */
-	reg_write(vcap, DCMIPP_CMFCR, DCMIPP_CMIER_P0ALL);
-
-	spin_lock_irq(&vcap->irqlock);
-
-	/* Return all queued buffers to vb2 in ERROR state */
-	dcmipp_bytecap_all_buffers_done(vcap, VB2_BUF_STATE_ERROR);
-	INIT_LIST_HEAD(&vcap->buffers);
-
-	vcap->active = NULL;
-	vcap->state = DCMIPP_STOPPED;
-
-	spin_unlock_irq(&vcap->irqlock);
-
-	dcmipp_dump_status(vcap);
-
-	pm_runtime_put(vcap->dev);
-
-	if (vcap->count.errors)
-		dev_warn(vcap->dev, "Some errors found while streaming: errors=%d (overrun=%d, limit=%d, nactive=%d), underrun=%d, buffers=%d\n",
-			 vcap->count.errors, vcap->count.overrun,
-			 vcap->count.limit, vcap->count.nactive,
-			 vcap->count.underrun, vcap->count.buffers);
-}
-
-static int dcmipp_bytecap_buf_prepare(struct vb2_buffer *vb)
-{
-	struct dcmipp_bytecap_device *vcap =  vb2_get_drv_priv(vb->vb2_queue);
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct dcmipp_buf *buf = container_of(vbuf, struct dcmipp_buf, vb);
-	unsigned long size;
-
-	size = vcap->format.sizeimage;
-
-	if (vb2_plane_size(vb, 0) < size) {
-		dev_err(vcap->dev, "%s data will not fit into plane (%lu < %lu)\n",
-			__func__, vb2_plane_size(vb, 0), size);
-		return -EINVAL;
-	}
-
-	vb2_set_plane_payload(vb, 0, size);
-
-	if (!buf->prepared) {
-		/* Get memory addresses */
-		buf->addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
-		buf->size = vb2_plane_size(&buf->vb.vb2_buf, 0);
-		buf->prepared = true;
-
-		vb2_set_plane_payload(&buf->vb.vb2_buf, 0, buf->size);
-
-		dev_dbg(vcap->dev, "Setup [%d] phy=%pad size=%zu\n",
-			vb->index, &buf->addr, buf->size);
-	}
-
-	return 0;
-}
-
-static void dcmipp_bytecap_buf_queue(struct vb2_buffer *vb2_buf)
-{
-	struct dcmipp_bytecap_device *vcap =
-		vb2_get_drv_priv(vb2_buf->vb2_queue);
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb2_buf);
-	struct dcmipp_buf *buf = container_of(vbuf, struct dcmipp_buf, vb);
-
-	dev_dbg(vcap->dev, "Queue [%d] %p phy=%pad\n", buf->vb.vb2_buf.index,
-		buf, &buf->addr);
-
-	spin_lock_irq(&vcap->irqlock);
-	list_add_tail(&buf->list, &vcap->buffers);
-
-	if (vcap->state == DCMIPP_WAIT_FOR_BUFFER) {
-		vcap->next = buf;
-		dev_dbg(vcap->dev, "Restart with next [%d] %p phy=%pad\n",
-			buf->vb.vb2_buf.index, buf, &buf->addr);
-
-		dcmipp_start_capture(vcap, buf);
-
-		vcap->state = DCMIPP_RUNNING;
-	}
-
-	spin_unlock_irq(&vcap->irqlock);
-}
-
-static int dcmipp_bytecap_queue_setup(struct vb2_queue *vq,
-				      unsigned int *nbuffers,
-				      unsigned int *nplanes,
-				      unsigned int sizes[],
-				      struct device *alloc_devs[])
-{
-	struct dcmipp_bytecap_device *vcap = vb2_get_drv_priv(vq);
-	unsigned int size;
-
-	size = vcap->format.sizeimage;
-
-	/* Make sure the image size is large enough */
-	if (*nplanes)
-		return sizes[0] < vcap->format.sizeimage ? -EINVAL : 0;
-
-	*nplanes = 1;
-	sizes[0] = vcap->format.sizeimage;
-
-	dev_dbg(vcap->dev, "Setup queue, count=%d, size=%d\n",
-		*nbuffers, size);
-
-	return 0;
-}
-
-static int dcmipp_bytecap_buf_init(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct dcmipp_buf *buf = container_of(vbuf, struct dcmipp_buf, vb);
-
-	INIT_LIST_HEAD(&buf->list);
-
-	return 0;
-}
-
-static const struct vb2_ops dcmipp_bytecap_qops = {
-	.start_streaming	= dcmipp_bytecap_start_streaming,
-	.stop_streaming		= dcmipp_bytecap_stop_streaming,
-	.buf_init		= dcmipp_bytecap_buf_init,
-	.buf_prepare		= dcmipp_bytecap_buf_prepare,
-	.buf_queue		= dcmipp_bytecap_buf_queue,
-	.queue_setup		= dcmipp_bytecap_queue_setup,
-};
-
-static void dcmipp_bytecap_release(struct video_device *vdev)
-{
-	struct dcmipp_bytecap_device *vcap =
-		container_of(vdev, struct dcmipp_bytecap_device, vdev);
-
-	dcmipp_pads_cleanup(vcap->ved.pads);
-	mutex_destroy(&vcap->lock);
-
-	kfree(vcap);
-}
-
-void dcmipp_bytecap_ent_release(struct dcmipp_ent_device *ved)
-{
-	struct dcmipp_bytecap_device *vcap =
-		container_of(ved, struct dcmipp_bytecap_device, ved);
-
-	media_entity_cleanup(ved->ent);
-	vb2_video_unregister_device(&vcap->vdev);
-}
-
-static void dcmipp_buffer_done(struct dcmipp_bytecap_device *vcap,
-			       struct dcmipp_buf *buf,
-			       size_t bytesused,
-			       int err)
-{
-	struct vb2_v4l2_buffer *vbuf;
-
-	list_del_init(&buf->list);
-
-	vbuf = &buf->vb;
-
-	vbuf->sequence = vcap->sequence++;
-	vbuf->field = V4L2_FIELD_NONE;
-	vbuf->vb2_buf.timestamp = ktime_get_ns();
-	vb2_set_plane_payload(&vbuf->vb2_buf, 0, bytesused);
-	vb2_buffer_done(&vbuf->vb2_buf,
-			err ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE);
-	dev_dbg(vcap->dev, "Done  [%d] %p phy=%pad\n", buf->vb.vb2_buf.index,
-		buf, &buf->addr);
-	vcap->count.buffers++;
-}
-
-/* irqlock must be held */
-static void
-dcmipp_bytecap_set_next_frame_or_stop(struct dcmipp_bytecap_device *vcap)
-{
-	if (!vcap->next && list_is_singular(&vcap->buffers)) {
-		/*
-		 * If there is no available buffer (none or a single one in the
-		 * list while two are expected), stop the capture (effective
-		 * for next frame). On-going frame capture will continue until
-		 * FRAME END but no further capture will be done.
-		 */
-		reg_clear(vcap, DCMIPP_P0FCTCR, DCMIPP_P0FCTCR_CPTREQ);
-
-		dev_dbg(vcap->dev, "Capture restart is deferred to next buffer queueing\n");
-		vcap->next = NULL;
-		vcap->state = DCMIPP_WAIT_FOR_BUFFER;
-		return;
-	}
-
-	/* If we don't have buffer yet, pick the one after active */
-	if (!vcap->next)
-		vcap->next = list_next_entry(vcap->active, list);
-
-	/*
-	 * Set buffer address
-	 * This register is shadowed and will be taken into
-	 * account on next VSYNC (start of next frame)
-	 */
-	reg_write(vcap, DCMIPP_P0PPM0AR1, vcap->next->addr);
-	dev_dbg(vcap->dev, "Write [%d] %p phy=%pad\n",
-		vcap->next->vb.vb2_buf.index, vcap->next, &vcap->next->addr);
-}
-
-/* irqlock must be held */
-static void dcmipp_bytecap_process_frame(struct dcmipp_bytecap_device *vcap,
-					 size_t bytesused)
-{
-	int err = 0;
-	struct dcmipp_buf *buf = vcap->active;
-
-	if (!buf) {
-		vcap->count.nactive++;
-		vcap->count.errors++;
-		return;
-	}
-
-	if (bytesused > buf->size) {
-		dev_dbg(vcap->dev, "frame larger than expected (%zu > %zu)\n",
-			bytesused, buf->size);
-		/* Clip to buffer size and return buffer to V4L2 in error */
-		bytesused = buf->size;
-		vcap->count.limit++;
-		vcap->count.errors++;
-		err = -EOVERFLOW;
-	}
-
-	dcmipp_buffer_done(vcap, buf, bytesused, err);
-	vcap->active = NULL;
-}
-
-static irqreturn_t dcmipp_bytecap_irq_thread(int irq, void *arg)
-{
-	struct dcmipp_bytecap_device *vcap =
-			container_of(arg, struct dcmipp_bytecap_device, ved);
-	size_t bytesused = 0;
-
-	spin_lock_irq(&vcap->irqlock);
-
-	/*
-	 * If we have an overrun, a frame-end will probably not be generated,
-	 * in that case the active buffer will be recycled as next buffer by
-	 * the VSYNC handler
-	 */
-	if (vcap->cmsr2 & DCMIPP_CMSR2_P0OVRF) {
-		vcap->count.errors++;
-		vcap->count.overrun++;
-	}
-
-	if (vcap->cmsr2 & DCMIPP_CMSR2_P0FRAMEF) {
-		vcap->count.frame++;
-
-		/* Read captured buffer size */
-		bytesused = reg_read(vcap, DCMIPP_P0DCCNTR);
-		dcmipp_bytecap_process_frame(vcap, bytesused);
-	}
-
-	if (vcap->cmsr2 & DCMIPP_CMSR2_P0VSYNCF) {
-		vcap->count.vsync++;
-		if (vcap->state == DCMIPP_WAIT_FOR_BUFFER) {
-			vcap->count.underrun++;
-			goto out;
-		}
-
-		/*
-		 * On VSYNC, the previously set next buffer is going to become
-		 * active thanks to the shadowing mechanism of the DCMIPP. In
-		 * most of the cases, since a FRAMEEND has already come,
-		 * pointer next is NULL since active is reset during the
-		 * FRAMEEND handling. However, in case of framerate adjustment,
-		 * there are more VSYNC than FRAMEEND. Thus we recycle the
-		 * active (but not used) buffer and put it back into next.
-		 */
-		swap(vcap->active, vcap->next);
-		dcmipp_bytecap_set_next_frame_or_stop(vcap);
-	}
-
-out:
-	spin_unlock_irq(&vcap->irqlock);
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t dcmipp_bytecap_irq_callback(int irq, void *arg)
-{
-	struct dcmipp_bytecap_device *vcap =
-			container_of(arg, struct dcmipp_bytecap_device, ved);
-
-	/* Store interrupt status register */
-	vcap->cmsr2 = reg_read(vcap, DCMIPP_CMSR2) & DCMIPP_CMIER_P0ALL;
-	vcap->count.it++;
-
-	/* Clear interrupt */
-	reg_write(vcap, DCMIPP_CMFCR, vcap->cmsr2);
-
-	return IRQ_WAKE_THREAD;
-}
-
-static int dcmipp_bytecap_link_validate(struct media_link *link)
-{
-	struct media_entity *entity = link->sink->entity;
-	struct video_device *vd = media_entity_to_video_device(entity);
-	struct dcmipp_bytecap_device *vcap = container_of(vd,
-					struct dcmipp_bytecap_device, vdev);
-	struct v4l2_subdev *source_sd =
-		media_entity_to_v4l2_subdev(link->source->entity);
-	struct v4l2_subdev_format source_fmt = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-		.pad = link->source->index,
-	};
-	int ret, i;
-
-	ret = v4l2_subdev_call(source_sd, pad, get_fmt, NULL, &source_fmt);
-	if (ret < 0)
-		return 0;
-
-	if (source_fmt.format.width != vcap->format.width ||
-	    source_fmt.format.height != vcap->format.height) {
-		dev_err(vcap->dev, "Wrong width or height %ux%u (%ux%u expected)\n",
-			vcap->format.width, vcap->format.height,
-			source_fmt.format.width, source_fmt.format.height);
-		return -EINVAL;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(dcmipp_bytecap_pix_map_list); i++) {
-		if (dcmipp_bytecap_pix_map_list[i].pixelformat ==
-			vcap->format.pixelformat &&
-		    dcmipp_bytecap_pix_map_list[i].code ==
-			source_fmt.format.code)
-			break;
-	}
-
-	if (i == ARRAY_SIZE(dcmipp_bytecap_pix_map_list)) {
-		dev_err(vcap->dev, "mbus code 0x%x do not match capture device format (0x%x)\n",
-			vcap->format.pixelformat, source_fmt.format.code);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static const struct media_entity_operations dcmipp_bytecap_entity_ops = {
-	.link_validate = dcmipp_bytecap_link_validate,
-};
-
-struct dcmipp_ent_device *dcmipp_bytecap_ent_init(struct device *dev,
-						  const char *entity_name,
-						  struct v4l2_device *v4l2_dev,
-						  void __iomem *regs)
-{
-	struct dcmipp_bytecap_device *vcap;
-	struct video_device *vdev;
-	struct vb2_queue *q;
-	const unsigned long pad_flag = MEDIA_PAD_FL_SINK;
-	int ret = 0;
-
-	/* Allocate the dcmipp_bytecap_device struct */
-	vcap = kzalloc_obj(*vcap);
-	if (!vcap)
-		return ERR_PTR(-ENOMEM);
-
-	/* Allocate the pads */
-	vcap->ved.pads = dcmipp_pads_init(1, &pad_flag);
-	if (IS_ERR(vcap->ved.pads)) {
-		ret = PTR_ERR(vcap->ved.pads);
-		goto err_free_vcap;
-	}
-
-	/* Initialize the media entity */
-	vcap->vdev.entity.name = entity_name;
-	vcap->vdev.entity.function = MEDIA_ENT_F_IO_V4L;
-	vcap->vdev.entity.ops = &dcmipp_bytecap_entity_ops;
-	ret = media_entity_pads_init(&vcap->vdev.entity, 1, vcap->ved.pads);
-	if (ret)
-		goto err_clean_pads;
-
-	/* Initialize the lock */
-	mutex_init(&vcap->lock);
-
-	/* Initialize the vb2 queue */
-	q = &vcap->queue;
-	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	q->io_modes = VB2_MMAP | VB2_DMABUF;
-	q->lock = &vcap->lock;
-	q->drv_priv = vcap;
-	q->buf_struct_size = sizeof(struct dcmipp_buf);
-	q->ops = &dcmipp_bytecap_qops;
-	q->mem_ops = &vb2_dma_contig_memops;
-	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-	q->min_queued_buffers = 1;
-	q->dev = dev;
-
-	/* DCMIPP requires 16 bytes aligned buffers */
-	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
-	if (ret) {
-		dev_err(dev, "Failed to set DMA mask\n");
-		goto err_mutex_destroy;
-	}
-
-	ret = vb2_queue_init(q);
-	if (ret) {
-		dev_err(dev, "%s: vb2 queue init failed (err=%d)\n",
-			entity_name, ret);
-		goto err_clean_m_ent;
-	}
-
-	/* Initialize buffer list and its lock */
-	INIT_LIST_HEAD(&vcap->buffers);
-	spin_lock_init(&vcap->irqlock);
-
-	/* Set default frame format */
-	vcap->format = fmt_default;
-
-	/* Fill the dcmipp_ent_device struct */
-	vcap->ved.ent = &vcap->vdev.entity;
-	vcap->ved.handler = dcmipp_bytecap_irq_callback;
-	vcap->ved.thread_fn = dcmipp_bytecap_irq_thread;
-	vcap->dev = dev;
-	vcap->regs = regs;
-
-	/* Initialize the video_device struct */
-	vdev = &vcap->vdev;
-	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING |
-			    V4L2_CAP_IO_MC;
-	vdev->release = dcmipp_bytecap_release;
-	vdev->fops = &dcmipp_bytecap_fops;
-	vdev->ioctl_ops = &dcmipp_bytecap_ioctl_ops;
-	vdev->lock = &vcap->lock;
-	vdev->queue = q;
-	vdev->v4l2_dev = v4l2_dev;
-	strscpy(vdev->name, entity_name, sizeof(vdev->name));
-	video_set_drvdata(vdev, &vcap->ved);
-
-	/* Register the video_device with the v4l2 and the media framework */
-	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
-	if (ret) {
-		dev_err(dev, "%s: video register failed (err=%d)\n",
-			vcap->vdev.name, ret);
-		goto err_clean_m_ent;
-	}
-
-	return &vcap->ved;
-
-err_clean_m_ent:
-	media_entity_cleanup(&vcap->vdev.entity);
-err_mutex_destroy:
-	mutex_destroy(&vcap->lock);
-err_clean_pads:
-	dcmipp_pads_cleanup(vcap->ved.pads);
-err_free_vcap:
-	kfree(vcap);
-
-	return ERR_PTR(ret);
-}
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c
index 0c7aeb0888f66b..346cfddcf2292c 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-byteproc.c
@@ -25,6 +25,7 @@
 #define DCMIPP_P0SCSZR_HSIZE_SHIFT	0
 #define DCMIPP_P0SCSZR_VSIZE_SHIFT	16
 #define DCMIPP_P0PPCR	0x5c0
+#define DCMIPP_P0PPCR_SWAPYUV		BIT(0)
 #define DCMIPP_P0PPCR_BSM_1_2		0x1
 #define DCMIPP_P0PPCR_BSM_1_4		0x2
 #define DCMIPP_P0PPCR_BSM_2_4		0x3
@@ -264,6 +265,7 @@ dcmipp_byteproc_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int dcmipp_byteproc_set_fmt(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *sd_state,
 				   struct v4l2_subdev_format *fmt)
 {
@@ -298,6 +300,7 @@ static int dcmipp_byteproc_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int dcmipp_byteproc_get_selection(struct v4l2_subdev *sd,
+					 const struct v4l2_subdev_client_info *ci,
 					 struct v4l2_subdev_state *sd_state,
 					 struct v4l2_subdev_selection *s)
 {
@@ -351,6 +354,7 @@ static int dcmipp_byteproc_get_selection(struct v4l2_subdev *sd,
 }
 
 static int dcmipp_byteproc_set_selection(struct v4l2_subdev *sd,
+					 const struct v4l2_subdev_client_info *ci,
 					 struct v4l2_subdev_state *sd_state,
 					 struct v4l2_subdev_selection *s)
 {
@@ -428,9 +432,11 @@ static int dcmipp_byteproc_configure_scale_crop
 	if (!vpix)
 		return -EINVAL;
 
-	/* clear decimation/crop */
+	/* clear decimation/crop/swap yuv */
 	reg_clear(byteproc, DCMIPP_P0PPCR, DCMIPP_P0PPCR_BSM_MASK);
 	reg_clear(byteproc, DCMIPP_P0PPCR, DCMIPP_P0PPCR_LSM);
+	if (byteproc->ved.dcmipp->pipe_cfg->has_swapyuv)
+		reg_clear(byteproc, DCMIPP_P0PPCR, DCMIPP_P0PPCR_SWAPYUV);
 	reg_write(byteproc, DCMIPP_P0SCSTR, 0);
 	reg_write(byteproc, DCMIPP_P0SCSZR, 0);
 
@@ -451,6 +457,17 @@ static int dcmipp_byteproc_configure_scale_crop
 	if (vprediv == 2)
 		val |= DCMIPP_P0PPCR_LSM | DCMIPP_P0PPCR_OELS;
 
+	/*
+	 * Perform a SWAP YUV if input is parallel since in this mode
+	 * the DCMIPP will swap YUV by default
+	 */
+	if (byteproc->ved.dcmipp->pipe_cfg->has_swapyuv &&
+	    (sink_fmt->code == MEDIA_BUS_FMT_YUYV8_2X8 ||
+	     sink_fmt->code == MEDIA_BUS_FMT_YVYU8_2X8 ||
+	     sink_fmt->code == MEDIA_BUS_FMT_UYVY8_2X8 ||
+	     sink_fmt->code == MEDIA_BUS_FMT_VYUY8_2X8))
+		val |= DCMIPP_P0PPCR_SWAPYUV;
+
 	/* decimate using bytes and lines skipping */
 	if (val) {
 		reg_set(byteproc, DCMIPP_P0PPCR, val);
@@ -571,8 +588,8 @@ void dcmipp_byteproc_ent_release(struct dcmipp_ent_device *ved)
 }
 
 struct dcmipp_ent_device *
-dcmipp_byteproc_ent_init(struct device *dev, const char *entity_name,
-			 struct v4l2_device *v4l2_dev, void __iomem *regs)
+dcmipp_byteproc_ent_init(const char *entity_name,
+			 struct dcmipp_device *dcmipp)
 {
 	struct dcmipp_byteproc_device *byteproc;
 	const unsigned long pads_flag[] = {
@@ -585,11 +602,11 @@ dcmipp_byteproc_ent_init(struct device *dev, const char *entity_name,
 	if (!byteproc)
 		return ERR_PTR(-ENOMEM);
 
-	byteproc->regs = regs;
+	byteproc->regs = dcmipp->regs;
 
 	/* Initialize ved and sd */
 	ret = dcmipp_ent_sd_register(&byteproc->ved, &byteproc->sd,
-				     v4l2_dev, entity_name,
+				     &dcmipp->v4l2_dev, entity_name,
 				     MEDIA_ENT_F_PROC_VIDEO_SCALER,
 				     ARRAY_SIZE(pads_flag), pads_flag,
 				     &dcmipp_byteproc_int_ops,
@@ -600,7 +617,8 @@ dcmipp_byteproc_ent_init(struct device *dev, const char *entity_name,
 		return ERR_PTR(ret);
 	}
 
-	byteproc->dev = dev;
+	byteproc->ved.dcmipp = dcmipp;
+	byteproc->dev = dcmipp->dev;
 
 	return &byteproc->ved;
 }
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-capture.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-capture.c
new file mode 100644
index 00000000000000..fe7767f908b7cf
--- /dev/null
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-capture.c
@@ -0,0 +1,1258 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for STM32 Digital Camera Memory Interface Pixel Processor
+ *
+ * Copyright (C) STMicroelectronics SA 2023
+ * Authors: Hugues Fruchet <hugues.fruchet@foss.st.com>
+ *          Alain Volmat <alain.volmat@foss.st.com>
+ *          for STMicroelectronics.
+ */
+
+#include <linux/iopoll.h>
+#include <linux/pm_runtime.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "dcmipp-common.h"
+
+#define DCMIPP_PRSR		0x1f8
+#define DCMIPP_CMIER		0x3f0
+#define DCMIPP_CMIER_P0FRAMEIE	BIT(9)
+#define DCMIPP_CMIER_P0VSYNCIE	BIT(10)
+#define DCMIPP_CMIER_P0OVRIE	BIT(15)
+#define DCMIPP_CMIER_P0ALL	(DCMIPP_CMIER_P0VSYNCIE |\
+				 DCMIPP_CMIER_P0FRAMEIE |\
+				 DCMIPP_CMIER_P0OVRIE)
+#define DCMIPP_CMIER_P1FRAMEIE	BIT(17)
+#define DCMIPP_CMIER_P1VSYNCIE	BIT(18)
+#define DCMIPP_CMIER_P1OVRIE	BIT(23)
+#define DCMIPP_CMIER_P1ALL	(DCMIPP_CMIER_P1VSYNCIE |\
+				 DCMIPP_CMIER_P1FRAMEIE |\
+				 DCMIPP_CMIER_P1OVRIE)
+#define DCMIPP_CMIER_P2FRAMEIE	BIT(25)
+#define DCMIPP_CMIER_P2VSYNCIE	BIT(26)
+#define DCMIPP_CMIER_P2OVRIE	BIT(31)
+#define DCMIPP_CMIER_P2ALL	(DCMIPP_CMIER_P2VSYNCIE |\
+				 DCMIPP_CMIER_P2FRAMEIE |\
+				 DCMIPP_CMIER_P2OVRIE)
+#define DCMIPP_CMIER_PxALL(id)	(((id) == 0) ? DCMIPP_CMIER_P0ALL :	\
+				 (((id) == 1) ? DCMIPP_CMIER_P1ALL :	\
+						DCMIPP_CMIER_P2ALL))
+#define DCMIPP_CMSR1		0x3f4
+#define DCMIPP_CMSR2		0x3f8
+#define DCMIPP_CMSR2_P0FRAMEF	BIT(9)
+#define DCMIPP_CMSR2_P0VSYNCF	BIT(10)
+#define DCMIPP_CMSR2_P0OVRF	BIT(15)
+#define DCMIPP_CMSR2_P1FRAMEF	BIT(17)
+#define DCMIPP_CMSR2_P1VSYNCF	BIT(18)
+#define DCMIPP_CMSR2_P1OVRF	BIT(23)
+#define DCMIPP_CMSR2_P2FRAMEF	BIT(25)
+#define DCMIPP_CMSR2_P2VSYNCF	BIT(26)
+#define DCMIPP_CMSR2_P2OVRF	BIT(31)
+#define DCMIPP_CMSR2_PxFRAMEF(id)	(((id) == 0) ? DCMIPP_CMSR2_P0FRAMEF :\
+					 (((id) == 1) ? DCMIPP_CMSR2_P1FRAMEF :\
+						       DCMIPP_CMSR2_P2FRAMEF))
+#define DCMIPP_CMSR2_PxVSYNCF(id)	(((id) == 0) ? DCMIPP_CMSR2_P0VSYNCF :\
+					 (((id) == 1) ? DCMIPP_CMSR2_P1VSYNCF :\
+						       DCMIPP_CMSR2_P2VSYNCF))
+#define DCMIPP_CMSR2_PxOVRF(id)	(((id) == 0) ? DCMIPP_CMSR2_P0OVRF :\
+				 (((id) == 1) ? DCMIPP_CMSR2_P1OVRF :\
+					       DCMIPP_CMSR2_P2OVRF))
+#define DCMIPP_CMFCR		0x3fc
+#define DCMIPP_PxFSCR(id)	(0x404 + ((id) * 0x400))
+#define DCMIPP_PxFSCR_PIPEN	BIT(31)
+#define DCMIPP_PxFCTCR(id)	(0x500 + ((id) * 0x400))
+#define DCMIPP_PxFCTCR_CPTREQ	BIT(3)
+#define DCMIPP_P0DCCNTR		0x5b0
+#define DCMIPP_P0DCLMTR		0x5b4
+#define DCMIPP_P0DCLMTR_ENABLE	BIT(31)
+#define DCMIPP_P0DCLMTR_LIMIT_MASK	GENMASK(23, 0)
+
+#define DCMIPP_PxPPM0AR1(id)	(0x5c4 + ((id) * 0x400))
+#define DCMIPP_PxPPM0PR(id)	(0x9cc + (((id) - 1) * 0x400))
+#define DCMIPP_P1PPM1AR1	0x9d4
+#define DCMIPP_P1PPM1PR		0x9dc
+#define DCMIPP_P1PPM2AR1	0x9e4
+
+#define DCMIPP_PxSR(id)		(0x5f8 + ((id) * 0x400))
+#define DCMIPP_PxSR_CPTACT	BIT(23)
+
+#define DCMIPP_PxPPCR(id)	(0x9c0 + (((id) - 1) * 0x400))
+#define DCMIPP_PxPPCR_FORMAT_RGB888	0x0
+#define DCMIPP_PxPPCR_FORMAT_RGB565	0x1
+#define DCMIPP_PxPPCR_FORMAT_ARGB8888	0x2
+#define DCMIPP_PxPPCR_FORMAT_RGBA8888	0x3
+#define DCMIPP_PxPPCR_FORMAT_Y8		0x4
+#define DCMIPP_PxPPCR_FORMAT_YUV444	0x5
+#define DCMIPP_PxPPCR_FORMAT_YUYV	0x6
+#define DCMIPP_P1PPCR_FORMAT_NV61	0x7
+#define DCMIPP_P1PPCR_FORMAT_NV21	0x8
+#define DCMIPP_P1PPCR_FORMAT_YV12	0x9
+#define DCMIPP_PxPPCR_FORMAT_UYVY	0xa
+
+#define DCMIPP_PxPPCR_SWAPRB		BIT(4)
+
+struct dcmipp_capture_pix_map {
+	unsigned int code;
+	u32 pixelformat;
+	u32 plane_nb;
+	unsigned int ppcr_fmt;
+	unsigned int swap_uv;
+};
+
+#define PIXMAP_MBUS_PFMT(mbus, fmt)			\
+	{						\
+		.code = MEDIA_BUS_FMT_##mbus,		\
+		.pixelformat = V4L2_PIX_FMT_##fmt	\
+	}
+
+static const struct dcmipp_capture_pix_map dcmipp_capture_dump_pix_map_list[] = {
+	PIXMAP_MBUS_PFMT(RGB565_2X8_LE, RGB565),
+	PIXMAP_MBUS_PFMT(RGB565_1X16, RGB565),
+	PIXMAP_MBUS_PFMT(RGB888_1X24, RGB24),
+	PIXMAP_MBUS_PFMT(YUYV8_2X8, YUYV),
+	PIXMAP_MBUS_PFMT(YUYV8_1X16, YUYV),
+	PIXMAP_MBUS_PFMT(YVYU8_2X8, YVYU),
+	PIXMAP_MBUS_PFMT(YVYU8_1X16, YVYU),
+	PIXMAP_MBUS_PFMT(UYVY8_2X8, UYVY),
+	PIXMAP_MBUS_PFMT(UYVY8_1X16, UYVY),
+	PIXMAP_MBUS_PFMT(VYUY8_2X8, VYUY),
+	PIXMAP_MBUS_PFMT(VYUY8_1X16, VYUY),
+	PIXMAP_MBUS_PFMT(Y8_1X8, GREY),
+	PIXMAP_MBUS_PFMT(Y10_1X10, Y10),
+	PIXMAP_MBUS_PFMT(Y12_1X12, Y12),
+	PIXMAP_MBUS_PFMT(Y14_1X14, Y14),
+	PIXMAP_MBUS_PFMT(SBGGR8_1X8, SBGGR8),
+	PIXMAP_MBUS_PFMT(SGBRG8_1X8, SGBRG8),
+	PIXMAP_MBUS_PFMT(SGRBG8_1X8, SGRBG8),
+	PIXMAP_MBUS_PFMT(SRGGB8_1X8, SRGGB8),
+	PIXMAP_MBUS_PFMT(SBGGR10_1X10, SBGGR10),
+	PIXMAP_MBUS_PFMT(SGBRG10_1X10, SGBRG10),
+	PIXMAP_MBUS_PFMT(SGRBG10_1X10, SGRBG10),
+	PIXMAP_MBUS_PFMT(SRGGB10_1X10, SRGGB10),
+	PIXMAP_MBUS_PFMT(SBGGR12_1X12, SBGGR12),
+	PIXMAP_MBUS_PFMT(SGBRG12_1X12, SGBRG12),
+	PIXMAP_MBUS_PFMT(SGRBG12_1X12, SGRBG12),
+	PIXMAP_MBUS_PFMT(SRGGB12_1X12, SRGGB12),
+	PIXMAP_MBUS_PFMT(SBGGR14_1X14, SBGGR14),
+	PIXMAP_MBUS_PFMT(SGBRG14_1X14, SGBRG14),
+	PIXMAP_MBUS_PFMT(SGRBG14_1X14, SGRBG14),
+	PIXMAP_MBUS_PFMT(SRGGB14_1X14, SRGGB14),
+	PIXMAP_MBUS_PFMT(JPEG_1X8, JPEG),
+};
+
+#define PIXMAP_MBUS_PIXEL_PFMT(mbus, fmt, nb_plane, pp_code, swap)		\
+	{						\
+		.code = MEDIA_BUS_FMT_##mbus,		\
+		.pixelformat = V4L2_PIX_FMT_##fmt,	\
+		.plane_nb = nb_plane,			\
+		.ppcr_fmt = pp_code,			\
+		.swap_uv = swap,			\
+	}
+
+static const struct dcmipp_capture_pix_map dcmipp_capture_pixel_pix_map_list[] = {
+	/* Coplanar formats are supported on main & aux pipe */
+	PIXMAP_MBUS_PIXEL_PFMT(RGB888_1X24, RGB565, 1, DCMIPP_PxPPCR_FORMAT_RGB565, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, YUYV, 1, DCMIPP_PxPPCR_FORMAT_YUYV, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, YVYU, 1, DCMIPP_PxPPCR_FORMAT_YUYV, 1),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, UYVY, 1, DCMIPP_PxPPCR_FORMAT_UYVY, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, VYUY, 1, DCMIPP_PxPPCR_FORMAT_UYVY, 1),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, GREY, 1, DCMIPP_PxPPCR_FORMAT_Y8, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(RGB888_1X24, RGB24, 1, DCMIPP_PxPPCR_FORMAT_RGB888, 1),
+	PIXMAP_MBUS_PIXEL_PFMT(RGB888_1X24, BGR24, 1, DCMIPP_PxPPCR_FORMAT_RGB888, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(RGB888_1X24, ARGB32, 1, DCMIPP_PxPPCR_FORMAT_RGBA8888, 1),
+	PIXMAP_MBUS_PIXEL_PFMT(RGB888_1X24, ABGR32, 1, DCMIPP_PxPPCR_FORMAT_ARGB8888, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(RGB888_1X24, RGBA32, 1, DCMIPP_PxPPCR_FORMAT_ARGB8888, 1),
+	PIXMAP_MBUS_PIXEL_PFMT(RGB888_1X24, BGRA32, 1, DCMIPP_PxPPCR_FORMAT_RGBA8888, 0),
+
+	/* Semiplanar & planar formats (plane_nb > 1) are only supported on main pipe */
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, NV12, 2, DCMIPP_P1PPCR_FORMAT_NV21, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, NV21, 2, DCMIPP_P1PPCR_FORMAT_NV21, 1),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, NV16, 2, DCMIPP_P1PPCR_FORMAT_NV61, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, NV61, 2, DCMIPP_P1PPCR_FORMAT_NV61, 1),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, YUV420, 3, DCMIPP_P1PPCR_FORMAT_YV12, 0),
+	PIXMAP_MBUS_PIXEL_PFMT(YUV8_1X24, YVU420, 3, DCMIPP_P1PPCR_FORMAT_YV12, 1),
+};
+
+struct dcmipp_buf {
+	struct vb2_v4l2_buffer	vb;
+	bool			prepared;
+	dma_addr_t		addr;
+	size_t			size;
+	dma_addr_t		addrs[3];
+	u32			strides[3];
+	u64			sizes[3];
+	struct list_head	list;
+};
+
+struct dcmipp_capture_device {
+	struct dcmipp_ent_device ved;
+	struct video_device vdev;
+	struct device *dev;
+	struct v4l2_pix_format format;
+	struct vb2_queue queue;
+	struct list_head buffers;
+	/*
+	 * Protects concurrent calls of buf queue / irq handler
+	 * and buffer handling related variables / lists
+	 */
+	spinlock_t irqlock;
+	/* mutex used as vdev and queue lock */
+	struct mutex lock;
+	u32 sequence;
+	struct v4l2_subdev *s_subdev;
+	u32 s_subdev_pad_nb;
+
+	enum dcmipp_state state;
+
+	/*
+	 * DCMIPP driver is handling 2 buffers
+	 * active: buffer into which DCMIPP is currently writing into
+	 * next: buffer given to the DCMIPP and which will become
+	 *       automatically active on next VSYNC
+	 */
+	struct dcmipp_buf *active, *next;
+
+	void __iomem *regs;
+
+	int pipe_id;
+
+	const struct dcmipp_capture_pix_map *pix_map;
+	unsigned int pix_map_array_size;
+
+	u32 cmsr2;
+
+	struct {
+		u32 errors;
+		u32 limit;
+		u32 overrun;
+		u32 buffers;
+		u32 vsync;
+		u32 frame;
+		u32 it;
+		u32 underrun;
+		u32 nactive;
+	} count;
+};
+
+static const struct dcmipp_capture_pix_map *
+dcmipp_capture_pix_map_by_pixelformat(struct dcmipp_capture_device *vcap,
+				      u32 pixelformat)
+{
+	for (unsigned int i = 0; i < vcap->pix_map_array_size; i++) {
+		if (vcap->pix_map[i].pixelformat == pixelformat)
+			return &vcap->pix_map[i];
+	}
+
+	return NULL;
+}
+
+static bool dcmipp_capture_is_format_valid(struct dcmipp_capture_device *vcap,
+					   unsigned int pixelformat)
+{
+	const struct dcmipp_capture_pix_map *vpix =
+		dcmipp_capture_pix_map_by_pixelformat(vcap, pixelformat);
+
+	if (!vpix || (vpix->plane_nb > 1 && vcap->pipe_id != 1))
+		return false;
+
+	return true;
+}
+
+static const struct v4l2_pix_format fmt_default = {
+	.width = DCMIPP_FMT_WIDTH_DEFAULT,
+	.height = DCMIPP_FMT_HEIGHT_DEFAULT,
+	.pixelformat = V4L2_PIX_FMT_RGB565,
+	.field = V4L2_FIELD_NONE,
+	.bytesperline = DCMIPP_FMT_WIDTH_DEFAULT * 2,
+	.sizeimage = DCMIPP_FMT_WIDTH_DEFAULT * DCMIPP_FMT_HEIGHT_DEFAULT * 2,
+	.colorspace = DCMIPP_COLORSPACE_DEFAULT,
+	.ycbcr_enc = DCMIPP_YCBCR_ENC_DEFAULT,
+	.quantization = DCMIPP_QUANTIZATION_DEFAULT,
+	.xfer_func = DCMIPP_XFER_FUNC_DEFAULT,
+};
+
+static inline int hdw_pixel_alignment(u32 format)
+{
+	/* 16 bytes alignment required by hardware */
+	switch (format) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_NV21:
+	case V4L2_PIX_FMT_YUV420:
+	case V4L2_PIX_FMT_YVU420:
+	case V4L2_PIX_FMT_NV16:
+	case V4L2_PIX_FMT_NV61:
+	case V4L2_PIX_FMT_GREY:
+		return 4;/* 2^4 = 16 pixels = 16 bytes */
+	case V4L2_PIX_FMT_RGB565:
+	case V4L2_PIX_FMT_YUYV:
+	case V4L2_PIX_FMT_YVYU:
+	case V4L2_PIX_FMT_UYVY:
+	case V4L2_PIX_FMT_VYUY:
+		return 3;/* 2^3  = 8 pixels = 16 bytes */
+	case V4L2_PIX_FMT_RGB24:
+	case V4L2_PIX_FMT_BGR24:
+		return 4;/* 2^4 = 16 pixels = 48 bytes */
+	case V4L2_PIX_FMT_ARGB32:
+	case V4L2_PIX_FMT_ABGR32:
+	case V4L2_PIX_FMT_RGBA32:
+	case V4L2_PIX_FMT_BGRA32:
+		return 2;/* 2^2  = 4 pixels = 16 bytes */
+	default:
+		return 0;
+	}
+}
+
+static inline int frame_planes(dma_addr_t base_addr, dma_addr_t addrs[],
+			       u32 strides[], u64 sizes[],
+			       u32 width, u32 height, u32 format)
+{
+	const struct v4l2_format_info *info;
+
+	/* Only used by dump pipe hence addrs[0] is enough */
+	if (format == V4L2_PIX_FMT_JPEG) {
+		addrs[0] = base_addr;
+		return 0;
+	}
+
+	info = v4l2_format_info(format);
+	if (!info)
+		return -EINVAL;
+
+	/* Fill-in each plane information */
+	addrs[0] = base_addr;
+	strides[0] = width * info->bpp[0];
+	sizes[0] = strides[0] * height;
+
+	if (info->comp_planes > 1) {
+		addrs[1] = addrs[0] + sizes[0];
+		strides[1] = width * info->bpp[1] / info->hdiv;
+		sizes[1] = strides[1] * height / info->vdiv;
+	}
+
+	if (info->comp_planes > 2) {
+		addrs[2] = addrs[1] + sizes[1];
+		strides[2] = width * info->bpp[2] / info->hdiv;
+		sizes[2] = strides[2] * height / info->vdiv;
+	}
+
+	return 0;
+}
+
+static int dcmipp_capture_querycap(struct file *file, void *priv,
+				   struct v4l2_capability *cap)
+{
+	strscpy(cap->driver, DCMIPP_PDEV_NAME, sizeof(cap->driver));
+	strscpy(cap->card, KBUILD_MODNAME, sizeof(cap->card));
+
+	return 0;
+}
+
+static int dcmipp_capture_g_fmt_vid_cap(struct file *file, void *priv,
+					struct v4l2_format *f)
+{
+	struct dcmipp_capture_device *vcap = video_drvdata(file);
+
+	f->fmt.pix = vcap->format;
+
+	return 0;
+}
+
+static int dcmipp_capture_try_fmt_vid_cap(struct file *file, void *priv,
+					  struct v4l2_format *f)
+{
+	struct dcmipp_capture_device *vcap = video_drvdata(file);
+	struct v4l2_pix_format *format = &f->fmt.pix;
+	u32 in_w, in_h;
+
+	/* Don't accept a pixelformat that is not on the table */
+	if (!dcmipp_capture_is_format_valid(vcap, format->pixelformat))
+		format->pixelformat = fmt_default.pixelformat;
+
+	/* Adjust width & height */
+	in_w = format->width;
+	in_h = format->height;
+	format->width = clamp_t(u32, format->width, DCMIPP_FRAME_MIN_WIDTH,
+				vcap->pipe_id != 0 ?
+				DCMIPP_FRAME_MAX_WIDTH : DCMIPP_FRAME_MAX_WIDTH);
+	if (vcap->pipe_id != 0)
+		format->width = round_up(format->width,
+					 1 << hdw_pixel_alignment(format->pixelformat));
+	format->height = clamp_t(u32, format->height, DCMIPP_FRAME_MIN_HEIGHT,
+				 vcap->pipe_id != 0 ?
+				 DCMIPP_FRAME_MAX_HEIGHT : DCMIPP_FRAME_MAX_HEIGHT);
+	if (format->width != in_w || format->height != in_h)
+		dev_dbg(vcap->dev, "resolution updated: %dx%d -> %dx%d\n",
+			in_w, in_h, format->width, format->height);
+
+	if (format->pixelformat == V4L2_PIX_FMT_JPEG) {
+		format->bytesperline = format->width;
+		format->sizeimage = format->bytesperline * format->height;
+	} else {
+		v4l2_fill_pixfmt(format, format->pixelformat,
+				 format->width, format->height);
+	}
+
+	if (format->field == V4L2_FIELD_ANY)
+		format->field = fmt_default.field;
+
+	dcmipp_colorimetry_clamp(format);
+
+	return 0;
+}
+
+static int dcmipp_capture_s_fmt_vid_cap(struct file *file, void *priv,
+					struct v4l2_format *f)
+{
+	struct dcmipp_capture_device *vcap = video_drvdata(file);
+	int ret;
+
+	/* Do not change the format while stream is on */
+	if (vb2_is_busy(&vcap->queue))
+		return -EBUSY;
+
+	ret = dcmipp_capture_try_fmt_vid_cap(file, priv, f);
+	if (ret)
+		return ret;
+
+	dev_dbg(vcap->dev, "%s: format update: old:%ux%u (0x%p4cc, %u, %u, %u, %u) new:%ux%d (0x%p4cc, %u, %u, %u, %u)\n",
+		vcap->vdev.name,
+		/* old */
+		vcap->format.width, vcap->format.height,
+		&vcap->format.pixelformat, vcap->format.colorspace,
+		vcap->format.quantization, vcap->format.xfer_func,
+		vcap->format.ycbcr_enc,
+		/* new */
+		f->fmt.pix.width, f->fmt.pix.height,
+		&f->fmt.pix.pixelformat, f->fmt.pix.colorspace,
+		f->fmt.pix.quantization, f->fmt.pix.xfer_func,
+		f->fmt.pix.ycbcr_enc);
+
+	vcap->format = f->fmt.pix;
+
+	return 0;
+}
+
+static int dcmipp_capture_enum_fmt_vid_cap(struct file *file, void *priv,
+					   struct v4l2_fmtdesc *f)
+{
+	struct dcmipp_capture_device *vcap = video_drvdata(file);
+	unsigned int index = f->index;
+	unsigned int i, prev_pixelformat = 0;
+
+	/*
+	 * List up all formats (or only ones matching f->mbus_code), taking
+	 * care of removing duplicated entries (due to support of both
+	 * parallel & csi 16 bits formats
+	 */
+	for (i = 0; i < vcap->pix_map_array_size; i++) {
+		/* Only main pipe supports (Semi)-planar formats */
+		if (vcap->pipe_id != 1 && vcap->pix_map[i].plane_nb > 1)
+			continue;
+
+		/* Skip formats not matching requested mbus code */
+		if (f->mbus_code && vcap->pix_map[i].code != f->mbus_code)
+			continue;
+
+		/* Skip duplicated pixelformat */
+		if (vcap->pix_map[i].pixelformat == prev_pixelformat)
+			continue;
+
+		prev_pixelformat = vcap->pix_map[i].pixelformat;
+
+		if (index == 0)
+			break;
+
+		index--;
+	}
+
+	if (i == vcap->pix_map_array_size)
+		return -EINVAL;
+
+	f->pixelformat = vcap->pix_map[i].pixelformat;
+
+	return 0;
+}
+
+static int dcmipp_capture_enum_framesizes(struct file *file, void *fh,
+					  struct v4l2_frmsizeenum *fsize)
+{
+	struct dcmipp_capture_device *vcap = video_drvdata(file);
+
+
+	if (fsize->index)
+		return -EINVAL;
+
+	/* Only accept code in the pix map table */
+	if (!dcmipp_capture_is_format_valid(vcap, fsize->pixel_format))
+		return -EINVAL;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
+	fsize->stepwise.min_width = DCMIPP_FRAME_MIN_WIDTH;
+	fsize->stepwise.max_width = DCMIPP_FRAME_MAX_WIDTH;
+	fsize->stepwise.min_height = DCMIPP_FRAME_MIN_HEIGHT;
+	fsize->stepwise.max_height = DCMIPP_FRAME_MAX_HEIGHT;
+	fsize->stepwise.step_width = 1;
+	fsize->stepwise.step_height = 1;
+
+	return 0;
+}
+
+static const struct v4l2_file_operations dcmipp_capture_fops = {
+	.owner		= THIS_MODULE,
+	.open		= v4l2_fh_open,
+	.release	= vb2_fop_release,
+	.read           = vb2_fop_read,
+	.poll		= vb2_fop_poll,
+	.unlocked_ioctl = video_ioctl2,
+	.mmap           = vb2_fop_mmap,
+};
+
+static const struct v4l2_ioctl_ops dcmipp_capture_ioctl_ops = {
+	.vidioc_querycap = dcmipp_capture_querycap,
+
+	.vidioc_g_fmt_vid_cap = dcmipp_capture_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap = dcmipp_capture_s_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap = dcmipp_capture_try_fmt_vid_cap,
+	.vidioc_enum_fmt_vid_cap = dcmipp_capture_enum_fmt_vid_cap,
+	.vidioc_enum_framesizes = dcmipp_capture_enum_framesizes,
+
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+};
+
+static void dcmipp_start_capture(struct dcmipp_capture_device *vcap,
+				 struct dcmipp_buf *buf)
+{
+	/* Set buffer address */
+	reg_write(vcap, DCMIPP_PxPPM0AR1(vcap->pipe_id), buf->addrs[0]);
+
+	if (vcap->pipe_id == 0) {
+		/* Set buffer size */
+		reg_write(vcap, DCMIPP_P0DCLMTR, DCMIPP_P0DCLMTR_ENABLE |
+			  ((buf->size / 4) & DCMIPP_P0DCLMTR_LIMIT_MASK));
+	} else {
+		reg_write(vcap, DCMIPP_PxPPM0PR(vcap->pipe_id),
+			  buf->strides[0]);
+
+		if (buf->addrs[1]) {
+			reg_write(vcap, DCMIPP_P1PPM1AR1, buf->addrs[1]);
+			reg_write(vcap, DCMIPP_P1PPM1PR, buf->strides[1]);
+		}
+
+		if (buf->addrs[2])
+			reg_write(vcap, DCMIPP_P1PPM2AR1, buf->addrs[2]);
+	}
+
+	/* Capture request */
+	reg_set(vcap, DCMIPP_PxFCTCR(vcap->pipe_id), DCMIPP_PxFCTCR_CPTREQ);
+}
+
+static void dcmipp_capture_all_buffers_done(struct dcmipp_capture_device *vcap,
+					    enum vb2_buffer_state state)
+{
+	struct dcmipp_buf *buf, *node;
+
+	list_for_each_entry_safe(buf, node, &vcap->buffers, list) {
+		list_del_init(&buf->list);
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
+	}
+}
+
+static int dcmipp_capture_start_streaming(struct vb2_queue *vq,
+					  unsigned int count)
+{
+	struct dcmipp_capture_device *vcap = vb2_get_drv_priv(vq);
+	struct media_entity *entity = &vcap->vdev.entity;
+	struct dcmipp_buf *buf;
+	struct media_pad *pad;
+	int ret;
+
+	vcap->sequence = 0;
+	memset(&vcap->count, 0, sizeof(vcap->count));
+
+	/*
+	 * Get source subdev - since link is IMMUTABLE, pointer is cached
+	 * within the dcmipp_capture_device structure
+	 */
+	if (!vcap->s_subdev) {
+		pad = media_pad_remote_pad_first(&vcap->vdev.entity.pads[0]);
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity)) {
+			ret = -EINVAL;
+			goto err_buffer_done;
+		}
+		vcap->s_subdev = media_entity_to_v4l2_subdev(pad->entity);
+		vcap->s_subdev_pad_nb = pad->index;
+	}
+
+	ret = pm_runtime_resume_and_get(vcap->dev);
+	if (ret < 0) {
+		dev_err(vcap->dev, "%s: Failed to start streaming, cannot get sync (%d)\n",
+			__func__, ret);
+		goto err_buffer_done;
+	}
+
+	ret = media_pipeline_start(entity->pads, &vcap->ved.dcmipp->pipe);
+	if (ret) {
+		dev_dbg(vcap->dev, "%s: Failed to start streaming, media pipeline start error (%d)\n",
+			__func__, ret);
+		goto err_pm_put;
+	}
+
+	ret = v4l2_subdev_enable_streams(vcap->s_subdev,
+					 vcap->s_subdev_pad_nb, BIT_ULL(0));
+	if (ret)
+		goto err_media_pipeline_stop;
+
+	spin_lock_irq(&vcap->irqlock);
+
+	if (vcap->pipe_id != 0) {
+		const struct dcmipp_capture_pix_map *vpix =
+			dcmipp_capture_pix_map_by_pixelformat(vcap, vcap->format.pixelformat);
+		unsigned int ppcr = 0;
+
+		/*
+		 * Configure the Pixel Packer
+		 * vpix is guaranteed to be valid since pixelformat is validated
+		 * in dcmipp_pixelcap_s_fmt_vid_cap function before
+		 */
+		ppcr = vpix->ppcr_fmt;
+		if (vpix->swap_uv)
+			ppcr |= DCMIPP_PxPPCR_SWAPRB;
+
+		reg_write(vcap, DCMIPP_PxPPCR(vcap->pipe_id), ppcr);
+	}
+
+	/* Enable pipe at the end of programming */
+	reg_set(vcap, DCMIPP_PxFSCR(vcap->pipe_id), DCMIPP_PxFSCR_PIPEN);
+
+	/*
+	 * vb2 framework guarantee that we have at least 'min_queued_buffers'
+	 * buffers in the list at this moment
+	 */
+	vcap->next = list_first_entry(&vcap->buffers, typeof(*buf), list);
+	dev_dbg(vcap->dev, "Start with next [%d] %p phy=%pad\n",
+		vcap->next->vb.vb2_buf.index, vcap->next, &vcap->next->addr);
+
+	dcmipp_start_capture(vcap, vcap->next);
+
+	/* Enable interruptions */
+	spin_lock(&vcap->vdev.v4l2_dev->lock);
+	reg_set(vcap, DCMIPP_CMIER, DCMIPP_CMIER_PxALL(vcap->pipe_id));
+	spin_unlock(&vcap->vdev.v4l2_dev->lock);
+
+	vcap->state = DCMIPP_RUNNING;
+
+	spin_unlock_irq(&vcap->irqlock);
+
+	return 0;
+
+err_media_pipeline_stop:
+	media_pipeline_stop(entity->pads);
+err_pm_put:
+	pm_runtime_put(vcap->dev);
+err_buffer_done:
+	spin_lock_irq(&vcap->irqlock);
+	/*
+	 * Return all buffers to vb2 in QUEUED state.
+	 * This will give ownership back to userspace
+	 */
+	dcmipp_capture_all_buffers_done(vcap, VB2_BUF_STATE_QUEUED);
+	vcap->active = NULL;
+	spin_unlock_irq(&vcap->irqlock);
+
+	return ret;
+}
+
+static void dcmipp_dump_status(struct dcmipp_capture_device *vcap)
+{
+	struct device *dev = vcap->dev;
+
+	dev_dbg(dev, "[DCMIPP_PRSR]  =%#10.8x\n", reg_read(vcap, DCMIPP_PRSR));
+	dev_dbg(dev, "[DCMIPP_P0SR] =%#10.8x\n", reg_read(vcap, DCMIPP_PxSR(0)));
+	dev_dbg(dev, "[DCMIPP_P0DCCNTR]=%#10.8x\n",
+		reg_read(vcap, DCMIPP_P0DCCNTR));
+	dev_dbg(dev, "[DCMIPP_CMSR1] =%#10.8x\n", reg_read(vcap, DCMIPP_CMSR1));
+	dev_dbg(dev, "[DCMIPP_CMSR2] =%#10.8x\n", reg_read(vcap, DCMIPP_CMSR2));
+}
+
+/*
+ * Stop the stream engine. Any remaining buffers in the stream queue are
+ * dequeued and passed on to the vb2 framework marked as STATE_ERROR.
+ */
+static void dcmipp_capture_stop_streaming(struct vb2_queue *vq)
+{
+	struct dcmipp_capture_device *vcap = vb2_get_drv_priv(vq);
+	int ret;
+	u32 status;
+
+	ret = v4l2_subdev_disable_streams(vcap->s_subdev,
+					  vcap->s_subdev_pad_nb, BIT_ULL(0));
+	if (ret)
+		dev_warn(vcap->dev, "Failed to disable stream\n");
+
+	/* Stop the media pipeline */
+	media_pipeline_stop(vcap->vdev.entity.pads);
+
+	/* Disable interruptions */
+	spin_lock(&vcap->vdev.v4l2_dev->lock);
+	reg_clear(vcap, DCMIPP_CMIER, DCMIPP_CMIER_PxALL(vcap->pipe_id));
+	spin_unlock(&vcap->vdev.v4l2_dev->lock);
+
+	/* Stop capture */
+	reg_clear(vcap, DCMIPP_PxFCTCR(vcap->pipe_id), DCMIPP_PxFCTCR_CPTREQ);
+
+	/* Wait until CPTACT become 0 */
+	ret = readl_relaxed_poll_timeout(vcap->regs + DCMIPP_PxSR(vcap->pipe_id),
+					 status,
+					 !(status & DCMIPP_PxSR_CPTACT),
+					 20 * USEC_PER_MSEC,
+					 1000 * USEC_PER_MSEC);
+	if (ret)
+		dev_warn(vcap->dev, "Timeout when stopping\n");
+
+	/* Disable pipe */
+	reg_clear(vcap, DCMIPP_PxFSCR(vcap->pipe_id), DCMIPP_PxFSCR_PIPEN);
+
+	/* Clear any pending interrupts */
+	reg_write(vcap, DCMIPP_CMFCR, DCMIPP_CMIER_PxALL(vcap->pipe_id));
+
+	spin_lock_irq(&vcap->irqlock);
+
+	/* Return all queued buffers to vb2 in ERROR state */
+	dcmipp_capture_all_buffers_done(vcap, VB2_BUF_STATE_ERROR);
+	INIT_LIST_HEAD(&vcap->buffers);
+
+	vcap->active = NULL;
+	vcap->state = DCMIPP_STOPPED;
+
+	spin_unlock_irq(&vcap->irqlock);
+
+	if (vcap->pipe_id == 0)
+		dcmipp_dump_status(vcap);
+
+	pm_runtime_put(vcap->dev);
+
+	if (vcap->count.errors)
+		dev_warn(vcap->dev, "Some errors found while streaming: errors=%d (overrun=%d, limit=%d, nactive=%d), underrun=%d, buffers=%d\n",
+			 vcap->count.errors, vcap->count.overrun,
+			 vcap->count.limit, vcap->count.nactive,
+			 vcap->count.underrun, vcap->count.buffers);
+}
+
+static int dcmipp_capture_buf_prepare(struct vb2_buffer *vb)
+{
+	struct dcmipp_capture_device *vcap =  vb2_get_drv_priv(vb->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct dcmipp_buf *buf = container_of(vbuf, struct dcmipp_buf, vb);
+	struct v4l2_pix_format *format = &vcap->format;
+	unsigned long size;
+	int ret;
+
+	size = vcap->format.sizeimage;
+
+	if (vb2_plane_size(vb, 0) < size) {
+		dev_err(vcap->dev, "%s data will not fit into plane (%lu < %lu)\n",
+			__func__, vb2_plane_size(vb, 0), size);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(vb, 0, size);
+
+	if (!buf->prepared) {
+		/* Get memory addresses */
+		buf->addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
+		buf->size = vb2_plane_size(&buf->vb.vb2_buf, 0);
+
+		ret = frame_planes(buf->addr,
+				   buf->addrs, buf->strides, buf->sizes,
+				   format->width, format->height,
+				   format->pixelformat);
+		if (ret) {
+			dev_err(vcap->dev, "%s: Unsupported pixel format (%x)\n",
+				__func__, format->pixelformat);
+			return ret;
+		}
+
+		/* Check for 16 bytes alignment required by hardware */
+		WARN_ON(buf->addrs[0] & 15);
+		if (vcap->pipe_id != 0) {
+			WARN_ON(buf->strides[0] & 15);
+			WARN_ON(buf->addrs[1] & 15);
+			WARN_ON(buf->strides[1] & 15);
+			WARN_ON(buf->addrs[2] & 15);
+		}
+
+		buf->prepared = true;
+
+		vb2_set_plane_payload(&buf->vb.vb2_buf, 0, buf->size);
+
+		dev_dbg(vcap->dev, "Setup [%d] phy=%pad size=%zu\n",
+			vb->index, &buf->addr, buf->size);
+	}
+
+	return 0;
+}
+
+static void dcmipp_capture_buf_queue(struct vb2_buffer *vb2_buf)
+{
+	struct dcmipp_capture_device *vcap =
+		vb2_get_drv_priv(vb2_buf->vb2_queue);
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb2_buf);
+	struct dcmipp_buf *buf = container_of(vbuf, struct dcmipp_buf, vb);
+
+	dev_dbg(vcap->dev, "Queue [%d] %p phy=%pad\n", buf->vb.vb2_buf.index,
+		buf, &buf->addr);
+
+	spin_lock_irq(&vcap->irqlock);
+	list_add_tail(&buf->list, &vcap->buffers);
+
+	if (vcap->state == DCMIPP_WAIT_FOR_BUFFER) {
+		vcap->next = buf;
+		dev_dbg(vcap->dev, "Restart with next [%d] %p phy=%pad\n",
+			buf->vb.vb2_buf.index, buf, &buf->addr);
+
+		dcmipp_start_capture(vcap, buf);
+
+		vcap->state = DCMIPP_RUNNING;
+	}
+
+	spin_unlock_irq(&vcap->irqlock);
+}
+
+static int dcmipp_capture_queue_setup(struct vb2_queue *vq,
+				      unsigned int *nbuffers,
+				      unsigned int *nplanes,
+				      unsigned int sizes[],
+				      struct device *alloc_devs[])
+{
+	struct dcmipp_capture_device *vcap = vb2_get_drv_priv(vq);
+	unsigned int size;
+
+	size = vcap->format.sizeimage;
+
+	/* Make sure the image size is large enough */
+	if (*nplanes)
+		return sizes[0] < vcap->format.sizeimage ? -EINVAL : 0;
+
+	*nplanes = 1;
+	sizes[0] = vcap->format.sizeimage;
+
+	dev_dbg(vcap->dev, "Setup queue, count=%d, size=%d\n",
+		*nbuffers, size);
+
+	return 0;
+}
+
+static int dcmipp_capture_buf_init(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+	struct dcmipp_buf *buf = container_of(vbuf, struct dcmipp_buf, vb);
+
+	INIT_LIST_HEAD(&buf->list);
+
+	return 0;
+}
+
+static const struct vb2_ops dcmipp_capture_qops = {
+	.start_streaming	= dcmipp_capture_start_streaming,
+	.stop_streaming		= dcmipp_capture_stop_streaming,
+	.buf_init		= dcmipp_capture_buf_init,
+	.buf_prepare		= dcmipp_capture_buf_prepare,
+	.buf_queue		= dcmipp_capture_buf_queue,
+	.queue_setup		= dcmipp_capture_queue_setup,
+};
+
+static void dcmipp_capture_release(struct video_device *vdev)
+{
+	struct dcmipp_capture_device *vcap =
+		container_of(vdev, struct dcmipp_capture_device, vdev);
+
+	dcmipp_pads_cleanup(vcap->ved.pads);
+	mutex_destroy(&vcap->lock);
+
+	kfree(vcap);
+}
+
+void dcmipp_capture_ent_release(struct dcmipp_ent_device *ved)
+{
+	struct dcmipp_capture_device *vcap =
+		container_of(ved, struct dcmipp_capture_device, ved);
+
+	media_entity_cleanup(ved->ent);
+	vb2_video_unregister_device(&vcap->vdev);
+}
+
+static void dcmipp_buffer_done(struct dcmipp_capture_device *vcap,
+			       struct dcmipp_buf *buf,
+			       size_t bytesused,
+			       int err)
+{
+	struct vb2_v4l2_buffer *vbuf;
+
+	list_del_init(&buf->list);
+
+	vbuf = &buf->vb;
+
+	vbuf->sequence = vcap->sequence++;
+	vbuf->field = V4L2_FIELD_NONE;
+	vbuf->vb2_buf.timestamp = ktime_get_ns();
+	vb2_set_plane_payload(&vbuf->vb2_buf, 0, bytesused);
+	vb2_buffer_done(&vbuf->vb2_buf,
+			err ? VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE);
+	dev_dbg(vcap->dev, "Done  [%d] %p phy=%pad\n", buf->vb.vb2_buf.index,
+		buf, &buf->addr);
+	vcap->count.buffers++;
+}
+
+/* irqlock must be held */
+static void
+dcmipp_capture_set_next_frame_or_stop(struct dcmipp_capture_device *vcap)
+{
+	if (!vcap->next && list_is_singular(&vcap->buffers)) {
+		/*
+		 * If there is no available buffer (none or a single one in the
+		 * list while two are expected), stop the capture (effective
+		 * for next frame). On-going frame capture will continue until
+		 * FRAME END but no further capture will be done.
+		 */
+		reg_clear(vcap, DCMIPP_PxFCTCR(vcap->pipe_id), DCMIPP_PxFCTCR_CPTREQ);
+
+		dev_dbg(vcap->dev, "Capture restart is deferred to next buffer queueing\n");
+		vcap->next = NULL;
+		vcap->state = DCMIPP_WAIT_FOR_BUFFER;
+		return;
+	}
+
+	/* If we don't have buffer yet, pick the one after active */
+	if (!vcap->next)
+		vcap->next = list_next_entry(vcap->active, list);
+
+	/*
+	 * Set buffer address
+	 * This register is shadowed and will be taken into
+	 * account on next VSYNC (start of next frame)
+	 */
+	reg_write(vcap, DCMIPP_PxPPM0AR1(vcap->pipe_id), vcap->next->addrs[0]);
+	if (vcap->pipe_id == 1) {
+		if (vcap->next->addrs[1])
+			reg_write(vcap, DCMIPP_P1PPM1AR1, vcap->next->addrs[1]);
+		if (vcap->next->addrs[2])
+			reg_write(vcap, DCMIPP_P1PPM2AR1, vcap->next->addrs[2]);
+	}
+	dev_dbg(vcap->dev, "Write [%d] %p phy=%pad\n",
+		vcap->next->vb.vb2_buf.index, vcap->next, &vcap->next->addr);
+}
+
+/* irqlock must be held */
+static void dcmipp_capture_process_frame(struct dcmipp_capture_device *vcap,
+					 size_t bytesused)
+{
+	int err = 0;
+	struct dcmipp_buf *buf = vcap->active;
+
+	if (!buf) {
+		vcap->count.nactive++;
+		vcap->count.errors++;
+		return;
+	}
+
+	if (bytesused > buf->size) {
+		dev_dbg(vcap->dev, "frame larger than expected (%zu > %zu)\n",
+			bytesused, buf->size);
+		/* Clip to buffer size and return buffer to V4L2 in error */
+		bytesused = buf->size;
+		vcap->count.limit++;
+		vcap->count.errors++;
+		err = -EOVERFLOW;
+	}
+
+	dcmipp_buffer_done(vcap, buf, bytesused, err);
+	vcap->active = NULL;
+}
+
+static irqreturn_t dcmipp_capture_irq_thread(int irq, void *arg)
+{
+	struct dcmipp_capture_device *vcap =
+			container_of(arg, struct dcmipp_capture_device, ved);
+	u32 cmsr2_pxframef;
+	u32 cmsr2_pxvsyncf;
+	u32 cmsr2_pxovrf;
+	size_t bytesused = 0;
+
+	spin_lock_irq(&vcap->irqlock);
+
+	cmsr2_pxovrf = DCMIPP_CMSR2_PxOVRF(vcap->pipe_id);
+	cmsr2_pxvsyncf = DCMIPP_CMSR2_PxVSYNCF(vcap->pipe_id);
+	cmsr2_pxframef = DCMIPP_CMSR2_PxFRAMEF(vcap->pipe_id);
+
+	/*
+	 * If we have an overrun, a frame-end will probably not be generated,
+	 * in that case the active buffer will be recycled as next buffer by
+	 * the VSYNC handler
+	 */
+	if (vcap->cmsr2 & cmsr2_pxovrf) {
+		vcap->count.errors++;
+		vcap->count.overrun++;
+	}
+
+	if (vcap->cmsr2 & cmsr2_pxframef) {
+		vcap->count.frame++;
+
+		/* Read captured buffer size */
+		if (vcap->pipe_id == 0)
+			bytesused = reg_read(vcap, DCMIPP_P0DCCNTR);
+		else
+			bytesused = vcap->format.sizeimage;
+		dcmipp_capture_process_frame(vcap, bytesused);
+	}
+
+	if (vcap->cmsr2 & cmsr2_pxvsyncf) {
+		vcap->count.vsync++;
+		if (vcap->state == DCMIPP_WAIT_FOR_BUFFER) {
+			vcap->count.underrun++;
+			goto out;
+		}
+
+		/*
+		 * On VSYNC, the previously set next buffer is going to become
+		 * active thanks to the shadowing mechanism of the DCMIPP. In
+		 * most of the cases, since a FRAMEEND has already come,
+		 * pointer next is NULL since active is reset during the
+		 * FRAMEEND handling. However, in case of framerate adjustment,
+		 * there are more VSYNC than FRAMEEND. Thus we recycle the
+		 * active (but not used) buffer and put it back into next.
+		 */
+		swap(vcap->active, vcap->next);
+		dcmipp_capture_set_next_frame_or_stop(vcap);
+	}
+
+out:
+	spin_unlock_irq(&vcap->irqlock);
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t dcmipp_capture_irq_callback(int irq, void *arg)
+{
+	struct dcmipp_capture_device *vcap =
+			container_of(arg, struct dcmipp_capture_device, ved);
+	struct dcmipp_ent_device *ved = arg;
+
+	/* Store interrupt status register */
+	vcap->cmsr2 = ved->cmsr2 & DCMIPP_CMIER_PxALL(vcap->pipe_id);
+	if (!vcap->cmsr2)
+		return IRQ_HANDLED;
+	vcap->count.it++;
+
+	/* Clear interrupt */
+	reg_write(vcap, DCMIPP_CMFCR, vcap->cmsr2);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static int dcmipp_capture_link_validate(struct media_link *link)
+{
+	struct media_entity *entity = link->sink->entity;
+	struct video_device *vd = media_entity_to_video_device(entity);
+	struct dcmipp_capture_device *vcap = container_of(vd,
+					struct dcmipp_capture_device, vdev);
+	struct v4l2_subdev *source_sd =
+		media_entity_to_v4l2_subdev(link->source->entity);
+	struct v4l2_subdev_format source_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = link->source->index,
+	};
+	u32 width_aligned;
+	int ret, i;
+
+	ret = v4l2_subdev_call(source_sd, pad, get_fmt, NULL, &source_fmt);
+	if (ret < 0)
+		return 0;
+
+	width_aligned = source_fmt.format.width;
+
+	/*
+	 * On pixel pipes there can be alignment constraints.
+	 * Depending on the format & pixelpacker constraints, vcap width is
+	 * different from mbus width.  Compute expected vcap width based on
+	 * mbus width
+	 */
+	if (vcap->pipe_id != 0)
+		width_aligned = round_up(source_fmt.format.width,
+					 1 << hdw_pixel_alignment(vcap->format.pixelformat));
+
+	if (width_aligned != vcap->format.width ||
+	    source_fmt.format.height != vcap->format.height) {
+		dev_err(vcap->dev, "Wrong width or height %ux%u (%ux%u expected)\n",
+			vcap->format.width, vcap->format.height,
+			width_aligned, source_fmt.format.height);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < vcap->pix_map_array_size; i++) {
+		if (vcap->pix_map[i].pixelformat == vcap->format.pixelformat &&
+		    vcap->pix_map[i].code == source_fmt.format.code)
+			break;
+	}
+
+	if (i == vcap->pix_map_array_size) {
+		dev_err(vcap->dev, "mbus code 0x%x do not match capture device format (0x%x)\n",
+			vcap->format.pixelformat, source_fmt.format.code);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct media_entity_operations dcmipp_capture_entity_ops = {
+	.link_validate = dcmipp_capture_link_validate,
+};
+
+static int dcmipp_name_to_pipe_id(const char *name)
+{
+	if (strstr(name, "dump"))
+		return 0;
+	else if (strstr(name, "main"))
+		return 1;
+	else if (strstr(name, "aux"))
+		return 2;
+	else
+		return -EINVAL;
+}
+
+struct dcmipp_ent_device *dcmipp_capture_ent_init(const char *entity_name,
+						  struct dcmipp_device *dcmipp)
+{
+	struct dcmipp_capture_device *vcap;
+	struct device *dev = dcmipp->dev;
+	struct video_device *vdev;
+	struct vb2_queue *q;
+	const unsigned long pad_flag = MEDIA_PAD_FL_SINK;
+	int ret = 0;
+
+	/* Allocate the dcmipp_capture_device struct */
+	vcap = kzalloc_obj(*vcap);
+	if (!vcap)
+		return ERR_PTR(-ENOMEM);
+
+	/* Retrieve the pipe_id */
+	vcap->pipe_id = dcmipp_name_to_pipe_id(entity_name);
+	if (vcap->pipe_id < 0) {
+		ret = -EIO;
+		dev_err(dev, "failed to retrieve pipe_id\n");
+		goto err_free_vcap;
+	}
+
+	/* Initialize supported format table format */
+	if (vcap->pipe_id == 0) {
+		vcap->pix_map = dcmipp_capture_dump_pix_map_list;
+		vcap->pix_map_array_size = ARRAY_SIZE(dcmipp_capture_dump_pix_map_list);
+	} else {
+		vcap->pix_map = dcmipp_capture_pixel_pix_map_list;
+		vcap->pix_map_array_size = ARRAY_SIZE(dcmipp_capture_pixel_pix_map_list);
+	}
+
+	/* Allocate the pads */
+	vcap->ved.pads = dcmipp_pads_init(1, &pad_flag);
+	if (IS_ERR(vcap->ved.pads)) {
+		ret = PTR_ERR(vcap->ved.pads);
+		goto err_free_vcap;
+	}
+
+	vcap->ved.dcmipp = dcmipp;
+
+	/* Initialize the media entity */
+	vcap->vdev.entity.name = entity_name;
+	vcap->vdev.entity.function = MEDIA_ENT_F_IO_V4L;
+	vcap->vdev.entity.ops = &dcmipp_capture_entity_ops;
+	ret = media_entity_pads_init(&vcap->vdev.entity, 1, vcap->ved.pads);
+	if (ret)
+		goto err_clean_pads;
+
+	/* Initialize the lock */
+	mutex_init(&vcap->lock);
+
+	/* Initialize the vb2 queue */
+	q = &vcap->queue;
+	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	q->io_modes = VB2_MMAP | VB2_DMABUF;
+	q->lock = &vcap->lock;
+	q->drv_priv = vcap;
+	q->buf_struct_size = sizeof(struct dcmipp_buf);
+	q->ops = &dcmipp_capture_qops;
+	q->mem_ops = &vb2_dma_contig_memops;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->min_queued_buffers = 1;
+	q->dev = dev;
+
+	/* DCMIPP requires 16 bytes aligned buffers */
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(dev, "Failed to set DMA mask\n");
+		goto err_mutex_destroy;
+	}
+
+	ret = vb2_queue_init(q);
+	if (ret) {
+		dev_err(dev, "%s: vb2 queue init failed (err=%d)\n",
+			entity_name, ret);
+		goto err_clean_m_ent;
+	}
+
+	/* Initialize buffer list and its lock */
+	INIT_LIST_HEAD(&vcap->buffers);
+	spin_lock_init(&vcap->irqlock);
+
+	/* Set default frame format */
+	vcap->format = fmt_default;
+
+	/* Fill the dcmipp_ent_device struct */
+	vcap->ved.ent = &vcap->vdev.entity;
+	vcap->ved.handler = dcmipp_capture_irq_callback;
+	vcap->ved.thread_fn = dcmipp_capture_irq_thread;
+	vcap->dev = dev;
+	vcap->regs = dcmipp->regs;
+
+	/* Initialize the video_device struct */
+	vdev = &vcap->vdev;
+	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING |
+			    V4L2_CAP_IO_MC;
+	vdev->release = dcmipp_capture_release;
+	vdev->fops = &dcmipp_capture_fops;
+	vdev->ioctl_ops = &dcmipp_capture_ioctl_ops;
+	vdev->lock = &vcap->lock;
+	vdev->queue = q;
+	vdev->v4l2_dev = &dcmipp->v4l2_dev;
+	strscpy(vdev->name, entity_name, sizeof(vdev->name));
+	video_set_drvdata(vdev, &vcap->ved);
+
+	/* Register the video_device with the v4l2 and the media framework */
+	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+	if (ret) {
+		dev_err(dev, "%s: video register failed (err=%d)\n",
+			vcap->vdev.name, ret);
+		goto err_clean_m_ent;
+	}
+
+	return &vcap->ved;
+
+err_clean_m_ent:
+	media_entity_cleanup(&vcap->vdev.entity);
+err_mutex_destroy:
+	mutex_destroy(&vcap->lock);
+err_clean_pads:
+	dcmipp_pads_cleanup(vcap->ved.pads);
+err_free_vcap:
+	kfree(vcap);
+
+	return ERR_PTR(ret);
+}
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-common.h b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-common.h
index fe5f97233f5e8b..007912c3a7f8ed 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-common.h
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-common.h
@@ -58,16 +58,82 @@ do {									\
 		(fmt)->xfer_func = DCMIPP_XFER_FUNC_DEFAULT;		\
 } while (0)
 
+struct dcmipp_device {
+	/* The platform device */
+	struct platform_device		pdev;
+	struct device			*dev;
+
+	/* Hardware resources */
+	void __iomem			*regs;
+	struct clk			*mclk;
+	struct clk			*kclk;
+
+	/* The pipeline configuration */
+	const struct dcmipp_pipeline_config	*pipe_cfg;
+
+	/* The Associated media_device parent */
+	struct media_device		mdev;
+	struct media_pipeline		pipe;
+
+	/* Internal v4l2 parent device*/
+	struct v4l2_device		v4l2_dev;
+
+	/* Entities */
+	struct dcmipp_ent_device	**entity;
+
+	struct v4l2_async_notifier	notifier;
+};
+
+#define DCMIPP_ENT_LINK(src, srcpad, sink, sinkpad, link_flags) {	\
+	.src_ent = src,						\
+	.src_pad = srcpad,					\
+	.sink_ent = sink,					\
+	.sink_pad = sinkpad,					\
+	.flags = link_flags,					\
+}
+
+/* Structure which describes individual configuration for each entity */
+struct dcmipp_ent_config {
+	const char *name;
+	struct dcmipp_ent_device *(*init)
+		(const char *entity_name,
+		 struct dcmipp_device *dcmipp);
+	void (*release)(struct dcmipp_ent_device *ved);
+};
+
+/* Structure which describes links between entities */
+struct dcmipp_ent_link {
+	unsigned int src_ent;
+	u16 src_pad;
+	unsigned int sink_ent;
+	u16 sink_pad;
+	u32 flags;
+};
+
+/* Structure which describes the whole topology */
+struct dcmipp_pipeline_config {
+	const struct dcmipp_ent_config *ents;
+	size_t num_ents;
+	const struct dcmipp_ent_link *links;
+	size_t num_links;
+	u32 hw_revision;
+	bool has_csi2;
+	bool needs_mclk;
+	bool has_swapyuv;
+};
+
 /**
  * struct dcmipp_ent_device - core struct that represents a node in the topology
  *
  * @ent:		the pointer to struct media_entity for the node
+ * @dcmipp:		the pointer to the parent dcmipp_device
  * @pads:		the list of pads of the node
  * @bus:		struct v4l2_mbus_config_parallel describing input bus
  * @bus_type:		type of input bus (parallel or BT656)
  * @handler:		irq handler dedicated to the subdev
  * @handler_ret:	value returned by the irq handler
  * @thread_fn:		threaded irq handler
+ * @cmsr2:		dcmipp status reg value captured upon an interrupt
  *
  * The DCMIPP provides a single IRQ line and a IRQ status registers for all
  * subdevs, hence once the main irq handler (registered at probe time) is
@@ -84,6 +150,7 @@ do {									\
  */
 struct dcmipp_ent_device {
 	struct media_entity *ent;
+	struct dcmipp_device *dcmipp;
 	struct media_pad *pads;
 
 	/* Parallel input device */
@@ -92,6 +159,13 @@ struct dcmipp_ent_device {
 	irq_handler_t handler;
 	irqreturn_t handler_ret;
 	irq_handler_t thread_fn;
+	u32 cmsr2;
+};
+
+enum dcmipp_state {
+	DCMIPP_STOPPED = 0,
+	DCMIPP_WAIT_FOR_BUFFER,
+	DCMIPP_RUNNING,
 };
 
 /**
@@ -199,19 +273,22 @@ static inline void __reg_clear(struct device *dev, void __iomem *base, u32 reg,
 }
 
 /* DCMIPP subdev init / release entry points */
-struct dcmipp_ent_device *dcmipp_inp_ent_init(struct device *dev,
-					      const char *entity_name,
-					      struct v4l2_device *v4l2_dev,
-					      void __iomem *regs);
+struct dcmipp_ent_device *dcmipp_inp_ent_init(const char *entity_name,
+					      struct dcmipp_device *dcmipp);
 void dcmipp_inp_ent_release(struct dcmipp_ent_device *ved);
 struct dcmipp_ent_device *
-dcmipp_byteproc_ent_init(struct device *dev, const char *entity_name,
-			 struct v4l2_device *v4l2_dev, void __iomem *regs);
+dcmipp_byteproc_ent_init(const char *entity_name,
+			 struct dcmipp_device *dcmipp);
 void dcmipp_byteproc_ent_release(struct dcmipp_ent_device *ved);
-struct dcmipp_ent_device *dcmipp_bytecap_ent_init(struct device *dev,
-						  const char *entity_name,
-						  struct v4l2_device *v4l2_dev,
-						  void __iomem *regs);
-void dcmipp_bytecap_ent_release(struct dcmipp_ent_device *ved);
+struct dcmipp_ent_device *dcmipp_capture_ent_init(const char *entity_name,
+						  struct dcmipp_device *dcmipp);
+void dcmipp_capture_ent_release(struct dcmipp_ent_device *ved);
+struct dcmipp_ent_device *dcmipp_isp_ent_init(const char *entity_name,
+					      struct dcmipp_device *dcmipp);
+void dcmipp_isp_ent_release(struct dcmipp_ent_device *ved);
+struct dcmipp_ent_device *
+dcmipp_pixelproc_ent_init(const char *entity_name,
+			  struct dcmipp_device *dcmipp);
+void dcmipp_pixelproc_ent_release(struct dcmipp_ent_device *ved);
 
 #endif
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c
index 516334541b2c85..54e28b4a5b2c50 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-core.c
@@ -25,38 +25,7 @@
 
 #define DCMIPP_MDEV_MODEL_NAME "DCMIPP MDEV"
 
-#define DCMIPP_ENT_LINK(src, srcpad, sink, sinkpad, link_flags) {	\
-	.src_ent = src,						\
-	.src_pad = srcpad,					\
-	.sink_ent = sink,					\
-	.sink_pad = sinkpad,					\
-	.flags = link_flags,					\
-}
-
-struct dcmipp_device {
-	/* The platform device */
-	struct platform_device		pdev;
-	struct device			*dev;
-
-	/* Hardware resources */
-	void __iomem			*regs;
-	struct clk			*mclk;
-	struct clk			*kclk;
-
-	/* The pipeline configuration */
-	const struct dcmipp_pipeline_config	*pipe_cfg;
-
-	/* The Associated media_device parent */
-	struct media_device		mdev;
-
-	/* Internal v4l2 parent device*/
-	struct v4l2_device		v4l2_dev;
-
-	/* Entities */
-	struct dcmipp_ent_device	**entity;
-
-	struct v4l2_async_notifier	notifier;
-};
+#define DCMIPP_CMSR2	0x3f8
 
 static inline struct dcmipp_device *
 notifier_to_dcmipp(struct v4l2_async_notifier *n)
@@ -64,35 +33,6 @@ notifier_to_dcmipp(struct v4l2_async_notifier *n)
 	return container_of(n, struct dcmipp_device, notifier);
 }
 
-/* Structure which describes individual configuration for each entity */
-struct dcmipp_ent_config {
-	const char *name;
-	struct dcmipp_ent_device *(*init)
-		(struct device *dev, const char *entity_name,
-		 struct v4l2_device *v4l2_dev, void __iomem *regs);
-	void (*release)(struct dcmipp_ent_device *ved);
-};
-
-/* Structure which describes links between entities */
-struct dcmipp_ent_link {
-	unsigned int src_ent;
-	u16 src_pad;
-	unsigned int sink_ent;
-	u16 sink_pad;
-	u32 flags;
-};
-
-/* Structure which describes the whole topology */
-struct dcmipp_pipeline_config {
-	const struct dcmipp_ent_config *ents;
-	size_t num_ents;
-	const struct dcmipp_ent_link *links;
-	size_t num_links;
-	u32 hw_revision;
-	bool has_csi2;
-	bool needs_mclk;
-};
-
 /* --------------------------------------------------------------------------
  * Topology Configuration
  */
@@ -110,8 +50,8 @@ static const struct dcmipp_ent_config stm32mp13_ent_config[] = {
 	},
 	{
 		.name = "dcmipp_dump_capture",
-		.init = dcmipp_bytecap_ent_init,
-		.release = dcmipp_bytecap_ent_release,
+		.init = dcmipp_capture_ent_init,
+		.release = dcmipp_capture_ent_release,
 	},
 };
 
@@ -135,6 +75,11 @@ static const struct dcmipp_pipeline_config stm32mp13_pipe_cfg = {
 	.hw_revision	= DCMIPP_STM32MP13_VERR
 };
 
+#define	ID_MAIN_ISP 3
+#define	ID_MAIN_POSTPROC 4
+#define	ID_MAIN_CAPTURE	5
+#define	ID_AUX_POSTPROC 6
+#define	ID_AUX_CAPTURE 7
 static const struct dcmipp_ent_config stm32mp25_ent_config[] = {
 	{
 		.name = "dcmipp_input",
@@ -148,16 +93,49 @@ static const struct dcmipp_ent_config stm32mp25_ent_config[] = {
 	},
 	{
 		.name = "dcmipp_dump_capture",
-		.init = dcmipp_bytecap_ent_init,
-		.release = dcmipp_bytecap_ent_release,
+		.init = dcmipp_capture_ent_init,
+		.release = dcmipp_capture_ent_release,
+	},
+	{
+		.name = "dcmipp_main_isp",
+		.init = dcmipp_isp_ent_init,
+		.release = dcmipp_isp_ent_release,
+	},
+	{
+		.name = "dcmipp_main_postproc",
+		.init = dcmipp_pixelproc_ent_init,
+		.release = dcmipp_pixelproc_ent_release,
+	},
+	{
+		.name = "dcmipp_main_capture",
+		.init = dcmipp_capture_ent_init,
+		.release = dcmipp_capture_ent_release,
+	},
+	{
+		.name = "dcmipp_aux_postproc",
+		.init = dcmipp_pixelproc_ent_init,
+		.release = dcmipp_pixelproc_ent_release,
+	},
+	{
+		.name = "dcmipp_aux_capture",
+		.init = dcmipp_capture_ent_init,
+		.release = dcmipp_capture_ent_release,
 	},
 };
 
 static const struct dcmipp_ent_link stm32mp25_ent_links[] = {
-	DCMIPP_ENT_LINK(ID_INPUT, 1, ID_DUMP_BYTEPROC, 0,
-			MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE),
+	DCMIPP_ENT_LINK(ID_INPUT, 1, ID_DUMP_BYTEPROC, 0, MEDIA_LNK_FL_ENABLED),
 	DCMIPP_ENT_LINK(ID_DUMP_BYTEPROC, 1, ID_DUMP_CAPTURE,  0,
 			MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE),
+	DCMIPP_ENT_LINK(ID_INPUT,	2, ID_MAIN_ISP,  0, 0),
+	DCMIPP_ENT_LINK(ID_MAIN_ISP,	1, ID_MAIN_POSTPROC,  0,
+			MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE),
+	DCMIPP_ENT_LINK(ID_MAIN_ISP,	2, ID_AUX_POSTPROC,  0, 0),
+	DCMIPP_ENT_LINK(ID_MAIN_POSTPROC,	1, ID_MAIN_CAPTURE,  0,
+			MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE),
+	DCMIPP_ENT_LINK(ID_INPUT,	3, ID_AUX_POSTPROC,  0, 0),
+	DCMIPP_ENT_LINK(ID_AUX_POSTPROC,	1, ID_AUX_CAPTURE,  0,
+			MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE),
 };
 
 #define DCMIPP_STM32MP25_VERR  0x30
@@ -168,7 +146,8 @@ static const struct dcmipp_pipeline_config stm32mp25_pipe_cfg = {
 	.num_links	= ARRAY_SIZE(stm32mp25_ent_links),
 	.hw_revision    = DCMIPP_STM32MP25_VERR,
 	.has_csi2	= true,
-	.needs_mclk	= true
+	.needs_mclk	= true,
+	.has_swapyuv	= true
 };
 
 #define LINK_FLAG_TO_STR(f) ((f) == 0 ? "" :\
@@ -221,9 +200,7 @@ static int dcmipp_create_subdevs(struct dcmipp_device *dcmipp)
 
 		dev_dbg(dcmipp->dev, "add subdev %s\n", name);
 		dcmipp->entity[i] =
-			dcmipp->pipe_cfg->ents[i].init(dcmipp->dev, name,
-						       &dcmipp->v4l2_dev,
-						       dcmipp->regs);
+			dcmipp->pipe_cfg->ents[i].init(name, dcmipp);
 		if (IS_ERR(dcmipp->entity[i])) {
 			dev_err(dcmipp->dev, "failed to init subdev %s\n",
 				name);
@@ -278,10 +255,15 @@ static irqreturn_t dcmipp_irq_callback(int irq, void *arg)
 	struct dcmipp_ent_device *ved;
 	irqreturn_t ret = IRQ_HANDLED;
 	unsigned int i;
+	u32 cmsr2;
+
+	/* Centralized read of CMSR2 */
+	cmsr2 = reg_read(dcmipp, DCMIPP_CMSR2);
 
 	/* Call irq handler of each entities of pipeline */
 	for (i = 0; i < dcmipp->pipe_cfg->num_ents; i++) {
 		ved = dcmipp->entity[i];
+		ved->cmsr2 = cmsr2;
 		if (ved->handler)
 			ved->handler_ret = ved->handler(irq, ved);
 		else if (ved->thread_fn)
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-input.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-input.c
index 7d3f5857cbfe8c..e66a9456deb63b 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-input.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-input.c
@@ -43,12 +43,18 @@
 #define DCMIPP_CMCR_INSEL	BIT(0)
 
 #define DCMIPP_P0FSCR	0x404
-#define DCMIPP_P0FSCR_DTMODE_MASK	GENMASK(17, 16)
-#define DCMIPP_P0FSCR_DTMODE_SHIFT	16
-#define DCMIPP_P0FSCR_DTMODE_DTIDA	0x00
+#define DCMIPP_P1FSCR	0x804
+#define DCMIPP_P2FSCR	0xC04
+#define DCMIPP_PXFSCR_DTMODE_MASK	GENMASK(17, 16)
+#define DCMIPP_PXFSCR_DTMODE_SHIFT	16
+#define DCMIPP_PXFSCR_DTMODE_DTIDA	0x00
 #define DCMIPP_P0FSCR_DTMODE_ALLDT	0x03
-#define DCMIPP_P0FSCR_DTIDA_MASK	GENMASK(5, 0)
-#define DCMIPP_P0FSCR_DTIDA_SHIFT	0
+#define DCMIPP_PXFSCR_DTIDA_MASK	GENMASK(5, 0)
+#define DCMIPP_PXFSCR_DTIDA_SHIFT	0
+
+#define DCMIPP_PXFSCR(a) (((a) == 0) ? DCMIPP_P0FSCR :\
+			  ((a) == 1) ? DCMIPP_P1FSCR :\
+			   DCMIPP_P2FSCR)
 
 #define IS_SINK(pad) (!(pad))
 #define IS_SRC(pad)  ((pad))
@@ -80,15 +86,15 @@ static const struct dcmipp_inp_pix_map dcmipp_inp_pix_map_list[] = {
 	PIXMAP_SINK_SRC_PRCR_SWAP(RGB888_3X8, RGB888_3X8, RGB888, 0, MIPI_CSI2_DT_RGB888),
 	PIXMAP_SINK_SRC_PRCR_SWAP(RGB888_1X24, RGB888_1X24, RGB888, 0, MIPI_CSI2_DT_RGB888),
 	/* YUV422 */
-	PIXMAP_SINK_SRC_PRCR_SWAP(YUYV8_2X8, YUYV8_2X8, YUV422, 1, MIPI_CSI2_DT_YUV422_8B),
+	PIXMAP_SINK_SRC_PRCR_SWAP(YUYV8_2X8, YUYV8_2X8, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
 	PIXMAP_SINK_SRC_PRCR_SWAP(YUYV8_1X16, YUYV8_1X16, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
-	PIXMAP_SINK_SRC_PRCR_SWAP(YUYV8_2X8, UYVY8_2X8, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
-	PIXMAP_SINK_SRC_PRCR_SWAP(UYVY8_2X8, UYVY8_2X8, YUV422, 1, MIPI_CSI2_DT_YUV422_8B),
+	PIXMAP_SINK_SRC_PRCR_SWAP(YUYV8_2X8, UYVY8_2X8, YUV422, 1, MIPI_CSI2_DT_YUV422_8B),
+	PIXMAP_SINK_SRC_PRCR_SWAP(UYVY8_2X8, UYVY8_2X8, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
 	PIXMAP_SINK_SRC_PRCR_SWAP(UYVY8_1X16, UYVY8_1X16, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
-	PIXMAP_SINK_SRC_PRCR_SWAP(UYVY8_2X8, YUYV8_2X8, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
-	PIXMAP_SINK_SRC_PRCR_SWAP(YVYU8_2X8, YVYU8_2X8, YUV422, 1, MIPI_CSI2_DT_YUV422_8B),
+	PIXMAP_SINK_SRC_PRCR_SWAP(UYVY8_2X8, YUYV8_2X8, YUV422, 1, MIPI_CSI2_DT_YUV422_8B),
+	PIXMAP_SINK_SRC_PRCR_SWAP(YVYU8_2X8, YVYU8_2X8, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
 	PIXMAP_SINK_SRC_PRCR_SWAP(YVYU8_1X16, YVYU8_1X16, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
-	PIXMAP_SINK_SRC_PRCR_SWAP(VYUY8_2X8, VYUY8_2X8, YUV422, 1, MIPI_CSI2_DT_YUV422_8B),
+	PIXMAP_SINK_SRC_PRCR_SWAP(VYUY8_2X8, VYUY8_2X8, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
 	PIXMAP_SINK_SRC_PRCR_SWAP(VYUY8_1X16, VYUY8_1X16, YUV422, 0, MIPI_CSI2_DT_YUV422_8B),
 	/* GREY */
 	PIXMAP_SINK_SRC_PRCR_SWAP(Y8_1X8, Y8_1X8, G8, 0, MIPI_CSI2_DT_RAW8),
@@ -269,11 +275,13 @@ static void dcmipp_inp_adjust_fmt(struct dcmipp_inp_device *inp,
 }
 
 static int dcmipp_inp_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
 	struct dcmipp_inp_device *inp = v4l2_get_subdevdata(sd);
 	struct v4l2_mbus_framefmt *mf;
+	int pad;
 
 	if (v4l2_subdev_is_streaming(sd))
 		return -EBUSY;
@@ -298,9 +306,11 @@ static int dcmipp_inp_set_fmt(struct v4l2_subdev *sd,
 
 	/* When setting the sink format, report that format on the src pad */
 	if (IS_SINK(fmt->pad)) {
-		mf = v4l2_subdev_state_get_format(sd_state, 1);
-		*mf = fmt->format;
-		dcmipp_inp_adjust_fmt(inp, mf, 1);
+		for (pad = 1; pad < sd->entity.num_pads; pad++) {
+			mf = v4l2_subdev_state_get_format(sd_state, pad);
+			*mf = fmt->format;
+			dcmipp_inp_adjust_fmt(inp, mf, pad);
+		}
 	}
 
 	return 0;
@@ -356,8 +366,20 @@ static int dcmipp_inp_configure_parallel(struct dcmipp_inp_device *inp,
 	val |= vpix->prcr_format << DCMIPP_PRCR_FORMAT_SHIFT;
 
 	/* swap cycles */
-	if (vpix->prcr_swapcycles)
-		val |= DCMIPP_PRCR_SWAPCYCLES;
+	/*
+	 * Table dcmipp_inp_pix_map_list take into consideration that SWAPYUV
+	 * bit is available when dealing with 16bit YUV formats. If it is not
+	 * available (such as on stm32mp13), swapcycle setting should be
+	 * reversed
+	 */
+	if (!inp->ved.dcmipp->pipe_cfg->has_swapyuv &&
+	    (src_fmt->code == MEDIA_BUS_FMT_YUYV8_2X8 ||
+	     src_fmt->code == MEDIA_BUS_FMT_YVYU8_2X8 ||
+	     src_fmt->code == MEDIA_BUS_FMT_UYVY8_2X8 ||
+	     src_fmt->code == MEDIA_BUS_FMT_VYUY8_2X8))
+		val |= (!vpix->prcr_swapcycles ? DCMIPP_PRCR_SWAPCYCLES : 0);
+	else
+		val |= (vpix->prcr_swapcycles ? DCMIPP_PRCR_SWAPCYCLES : 0);
 
 	reg_write(inp, DCMIPP_PRCR, val);
 
@@ -371,7 +393,8 @@ static int dcmipp_inp_configure_parallel(struct dcmipp_inp_device *inp,
 }
 
 static int dcmipp_inp_configure_csi(struct dcmipp_inp_device *inp,
-				    struct v4l2_subdev_state *state)
+				    struct v4l2_subdev_state *state,
+				    u32 pad)
 {
 	const struct dcmipp_inp_pix_map *vpix;
 	struct v4l2_mbus_framefmt *sink_fmt;
@@ -379,7 +402,7 @@ static int dcmipp_inp_configure_csi(struct dcmipp_inp_device *inp,
 
 	/* Get format information */
 	sink_fmt = v4l2_subdev_state_get_format(state, 0);
-	src_fmt = v4l2_subdev_state_get_format(state, 1);
+	src_fmt = v4l2_subdev_state_get_format(state, pad);
 
 	vpix = dcmipp_inp_pix_map_by_code(sink_fmt->code, src_fmt->code);
 	if (!vpix) {
@@ -387,22 +410,28 @@ static int dcmipp_inp_configure_csi(struct dcmipp_inp_device *inp,
 		return -EINVAL;
 	}
 
-	/* Apply configuration on each input pipe */
-	reg_clear(inp, DCMIPP_P0FSCR,
-		  DCMIPP_P0FSCR_DTMODE_MASK | DCMIPP_P0FSCR_DTIDA_MASK);
+	/* Perform the configuration on the related pad/pipe */
+	reg_clear(inp, DCMIPP_PXFSCR(pad - 1),
+		  DCMIPP_PXFSCR_DTMODE_MASK | DCMIPP_PXFSCR_DTIDA_MASK);
 
 	/* In case of JPEG we don't know the DT so we allow all data */
 	/*
 	 * TODO - check instead dt == 0 for the time being to allow other
 	 * unknown data-type
 	 */
-	if (!vpix->dt)
-		reg_set(inp, DCMIPP_P0FSCR,
-			DCMIPP_P0FSCR_DTMODE_ALLDT << DCMIPP_P0FSCR_DTMODE_SHIFT);
-	else
+	if (!vpix->dt) {
+		if (pad != 1) {
+			dev_err(inp->dev, "JPEG only available on pipe 0\n");
+			return -EINVAL;
+		}
+		/* Only available on Pipe #0 */
 		reg_set(inp, DCMIPP_P0FSCR,
-			vpix->dt << DCMIPP_P0FSCR_DTIDA_SHIFT |
-			DCMIPP_P0FSCR_DTMODE_DTIDA);
+			DCMIPP_P0FSCR_DTMODE_ALLDT << DCMIPP_PXFSCR_DTMODE_SHIFT);
+	} else {
+		reg_set(inp, DCMIPP_PXFSCR(pad - 1),
+			vpix->dt << DCMIPP_PXFSCR_DTIDA_SHIFT |
+			DCMIPP_PXFSCR_DTMODE_DTIDA);
+	}
 
 	/* Select the DCMIPP CSI interface */
 	reg_write(inp, DCMIPP_CMCR, DCMIPP_CMCR_INSEL);
@@ -420,20 +449,24 @@ static int dcmipp_inp_enable_streams(struct v4l2_subdev *sd,
 	struct media_pad *s_pad;
 	int ret = 0;
 
-	/* Get source subdev */
-	s_pad = media_pad_remote_pad_first(&sd->entity.pads[0]);
-	if (!s_pad || !is_media_entity_v4l2_subdev(s_pad->entity))
-		return -EINVAL;
-	s_subdev = media_entity_to_v4l2_subdev(s_pad->entity);
-
 	if (inp->ved.bus_type == V4L2_MBUS_PARALLEL ||
 	    inp->ved.bus_type == V4L2_MBUS_BT656)
 		ret = dcmipp_inp_configure_parallel(inp, state);
 	else if (inp->ved.bus_type == V4L2_MBUS_CSI2_DPHY)
-		ret = dcmipp_inp_configure_csi(inp, state);
+		ret = dcmipp_inp_configure_csi(inp, state, pad);
 	if (ret)
 		return ret;
 
+	/* If there where no other pad enabled, then enable the source subdev */
+	if (sd->enabled_pads)
+		return 0;
+
+	/* Get source subdev */
+	s_pad = media_pad_remote_pad_first(&sd->entity.pads[0]);
+	if (!s_pad || !is_media_entity_v4l2_subdev(s_pad->entity))
+		return -EINVAL;
+	s_subdev = media_entity_to_v4l2_subdev(s_pad->entity);
+
 	ret = v4l2_subdev_enable_streams(s_subdev, s_pad->index, BIT_ULL(0));
 	if (ret < 0) {
 		dev_err(inp->dev,
@@ -454,6 +487,10 @@ static int dcmipp_inp_disable_streams(struct v4l2_subdev *sd,
 	struct media_pad *s_pad;
 	int ret;
 
+	/* Don't do anything if there are still other pads enabled */
+	if ((sd->enabled_pads & ~BIT(pad)))
+		return 0;
+
 	/* Get source subdev */
 	s_pad = media_pad_remote_pad_first(&sd->entity.pads[0]);
 	if (!s_pad || !is_media_entity_v4l2_subdev(s_pad->entity))
@@ -515,15 +552,16 @@ void dcmipp_inp_ent_release(struct dcmipp_ent_device *ved)
 	dcmipp_ent_sd_unregister(ved, &inp->sd);
 }
 
-struct dcmipp_ent_device *dcmipp_inp_ent_init(struct device *dev,
-					      const char *entity_name,
-					      struct v4l2_device *v4l2_dev,
-					      void __iomem *regs)
+struct dcmipp_ent_device *dcmipp_inp_ent_init(const char *entity_name,
+					      struct dcmipp_device *dcmipp)
 {
 	struct dcmipp_inp_device *inp;
 	const unsigned long pads_flag[] = {
 		MEDIA_PAD_FL_SINK, MEDIA_PAD_FL_SOURCE,
+		MEDIA_PAD_FL_SOURCE, MEDIA_PAD_FL_SOURCE,
 	};
+	struct device *dev = dcmipp->dev;
+	u16 num_pads = ARRAY_SIZE(pads_flag);
 	int ret;
 
 	/* Allocate the inp struct */
@@ -531,12 +569,17 @@ struct dcmipp_ent_device *dcmipp_inp_ent_init(struct device *dev,
 	if (!inp)
 		return ERR_PTR(-ENOMEM);
 
-	inp->regs = regs;
+	inp->regs = dcmipp->regs;
+	inp->ved.dcmipp = dcmipp;
+
+	/* For DCMIPP without CSI2, there is only a single pipe hence 2 pads */
+	if (!inp->ved.dcmipp->pipe_cfg->has_csi2)
+		num_pads = 2;
 
 	/* Initialize ved and sd */
-	ret = dcmipp_ent_sd_register(&inp->ved, &inp->sd, v4l2_dev,
+	ret = dcmipp_ent_sd_register(&inp->ved, &inp->sd, &dcmipp->v4l2_dev,
 				     entity_name, MEDIA_ENT_F_VID_IF_BRIDGE,
-				     ARRAY_SIZE(pads_flag), pads_flag,
+				     num_pads, pads_flag,
 				     &dcmipp_inp_int_ops, &dcmipp_inp_ops,
 				     NULL, NULL);
 	if (ret) {
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-isp.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-isp.c
new file mode 100644
index 00000000000000..088301c2054796
--- /dev/null
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-isp.c
@@ -0,0 +1,493 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for STM32 Digital Camera Memory Interface Pixel Processor
+ *
+ * Copyright (C) STMicroelectronics SA 2026
+ * Authors: Hugues Fruchet <hugues.fruchet@foss.st.com>
+ *          Alain Volmat <alain.volmat@foss.st.com>
+ *          for STMicroelectronics.
+ */
+
+#include <linux/v4l2-mediabus.h>
+#include <media/v4l2-rect.h>
+#include <media/v4l2-subdev.h>
+
+#include "dcmipp-common.h"
+#include "dcmipp-pixelcommon.h"
+
+#define DCMIPP_P1FSCR	0x804
+#define DCMIPP_P1FSCR_PIPEDIFF BIT(18)
+
+#define DCMIPP_P1SRCR	0x820
+#define DCMIPP_P1SRCR_LASTLINE_SHIFT	0
+#define DCMIPP_P1SRCR_FIRSTLINEDEL_SHIFT	12
+#define DCMIPP_P1SRCR_CROPEN		BIT(15)
+
+#define DCMIPP_P1DECR	0x830
+#define DCMIPP_P1DECR_ENABLE		BIT(0)
+#define DCMIPP_P1DECR_HDEC_SHIFT	1
+#define DCMIPP_P1DECR_VDEC_SHIFT	3
+
+#define DCMIPP_P1DMCR	0x870
+#define DCMIPP_P1DMCR_ENABLE		BIT(0)
+#define DCMIPP_P1DMCR_TYPE_SHIFT	1
+#define DCMIPP_P1DMCR_TYPE_MASK		GENMASK(2, 1)
+#define DCMIPP_P1DMCR_TYPE_RGGB		0x0
+#define DCMIPP_P1DMCR_TYPE_GRBG		0x1
+#define DCMIPP_P1DMCR_TYPE_GBRG		0x2
+#define DCMIPP_P1DMCR_TYPE_BGGR		0x3
+
+#define ISP_MEDIA_BUS_SINK_FMT_DEFAULT MEDIA_BUS_FMT_RGB565_1X16
+#define ISP_MEDIA_BUS_SRC_FMT_DEFAULT MEDIA_BUS_FMT_RGB888_1X24
+
+struct dcmipp_isp_device {
+	struct dcmipp_ent_device ved;
+	struct v4l2_subdev sd;
+	struct device *dev;
+
+	void __iomem *regs;
+};
+
+static const struct v4l2_mbus_framefmt fmt_default = {
+	.width = DCMIPP_FMT_WIDTH_DEFAULT,
+	.height = DCMIPP_FMT_HEIGHT_DEFAULT,
+	.code = ISP_MEDIA_BUS_SINK_FMT_DEFAULT,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = DCMIPP_COLORSPACE_DEFAULT,
+	.ycbcr_enc = DCMIPP_YCBCR_ENC_DEFAULT,
+	.quantization = DCMIPP_QUANTIZATION_DEFAULT,
+	.xfer_func = DCMIPP_XFER_FUNC_DEFAULT,
+};
+
+static inline unsigned int dcmipp_isp_set_compose(__u32 size, __u32 req)
+{
+	unsigned int i = 0;
+
+	if (req > size)
+		return size;
+
+	/* Maximum decimation factor is 8 */
+	while (size > req && i++ < 3)
+		size /= 2;
+
+	return size;
+}
+
+static void dcmipp_isp_adjust_fmt(struct v4l2_mbus_framefmt *fmt, u32 pad)
+{
+	/* Only accept code in the pix map table */
+	if (!dcmipp_pixelpipe_pix_map_by_code(fmt->code, DCMIPP_ISP, pad))
+		fmt->code = IS_SRC(pad) ? ISP_MEDIA_BUS_SRC_FMT_DEFAULT :
+					  ISP_MEDIA_BUS_SINK_FMT_DEFAULT;
+
+	fmt->width = clamp_t(u32, fmt->width, DCMIPP_FRAME_MIN_WIDTH,
+			     DCMIPP_FRAME_MAX_WIDTH) & ~1;
+	fmt->height = clamp_t(u32, fmt->height, DCMIPP_FRAME_MIN_HEIGHT,
+			      DCMIPP_FRAME_MAX_HEIGHT);
+
+	if (fmt->field == V4L2_FIELD_ANY || fmt->field == V4L2_FIELD_ALTERNATE)
+		fmt->field = V4L2_FIELD_NONE;
+
+	dcmipp_colorimetry_clamp(fmt);
+}
+
+static int dcmipp_isp_init_state(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state)
+{
+	for (unsigned int i = 0; i < sd->entity.num_pads; i++) {
+		struct v4l2_mbus_framefmt *mf;
+
+		mf = v4l2_subdev_state_get_format(state, i);
+		*mf = fmt_default;
+		mf->code = IS_SRC(i) ? ISP_MEDIA_BUS_SRC_FMT_DEFAULT :
+				       ISP_MEDIA_BUS_SINK_FMT_DEFAULT;
+
+		if (IS_SINK(i)) {
+			struct v4l2_rect r = {
+				.top = 0,
+				.left = 0,
+				.width = DCMIPP_FMT_WIDTH_DEFAULT,
+				.height = DCMIPP_FMT_HEIGHT_DEFAULT,
+			};
+
+			*v4l2_subdev_state_get_crop(state, i) = r;
+			*v4l2_subdev_state_get_compose(state, i) = r;
+		}
+	}
+
+	return 0;
+}
+
+static int dcmipp_isp_enum_mbus_code(struct v4l2_subdev *sd,
+				     struct v4l2_subdev_state *state,
+				     struct v4l2_subdev_mbus_code_enum *code)
+{
+	return dcmipp_pixelpipe_enum_mbus_code(DCMIPP_ISP, code);
+}
+
+static int dcmipp_isp_enum_frame_size(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_state *state,
+				      struct v4l2_subdev_frame_size_enum *fse)
+{
+	return dcmipp_pixelpipe_enum_frame_size(DCMIPP_ISP, fse);
+}
+
+static int dcmipp_isp_set_fmt(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
+			      struct v4l2_subdev_state *state,
+			      struct v4l2_subdev_format *fmt)
+{
+	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE &&
+	    v4l2_subdev_is_streaming(sd))
+		return -EBUSY;
+
+	dcmipp_isp_adjust_fmt(&fmt->format, fmt->pad);
+
+	if (IS_SINK(fmt->pad)) {
+		struct v4l2_mbus_framefmt *src_fmt =
+			v4l2_subdev_state_get_format(state, 1);
+		struct v4l2_rect r = {
+			.top = 0,
+			.left = 0,
+			.width = fmt->format.width,
+			.height = fmt->format.height,
+		};
+
+		/* Adjust SINK pad crop/compose */
+		*v4l2_subdev_state_get_crop(state, 0) = r;
+		*v4l2_subdev_state_get_compose(state, 0) = r;
+
+		/* Forward format to SRC pads */
+		*src_fmt = fmt->format;
+		src_fmt->code = dcmipp_pixelpipe_src_format(fmt->format.code);
+		*v4l2_subdev_state_get_format(state, 2) = *src_fmt;
+	} else {
+		struct v4l2_mbus_framefmt *sink_fmt =
+			v4l2_subdev_state_get_format(state, 0);
+		struct v4l2_rect *compose =
+			v4l2_subdev_state_get_compose(state, 0);
+
+		fmt->format = *sink_fmt;
+		fmt->format.code = dcmipp_pixelpipe_src_format(sink_fmt->code);
+		if (compose->width && compose->height) {
+			fmt->format.width = compose->width;
+			fmt->format.height = compose->height;
+		}
+		/* Set to the 2nd SRC pad */
+		*v4l2_subdev_state_get_format(state, fmt->pad == 1 ? 2 : 1) =
+			fmt->format;
+	}
+
+	/* Update the selected pad format */
+	*v4l2_subdev_state_get_format(state, fmt->pad) = fmt->format;
+
+	return 0;
+}
+
+static void dcmipp_isp_adjust_crop(struct v4l2_rect *r,
+				   const struct v4l2_mbus_framefmt *fmt)
+{
+	struct v4l2_rect crop_min = {
+		.width = fmt->width,
+		.height = DCMIPP_FRAME_MIN_HEIGHT,
+	};
+
+	/*
+	 * Crop is related here to the statistics removal, that is
+	 * firsts and/or lasts lines of the frame. Up to the first
+	 * 7 lines can be skipped
+	 */
+#define DCMIPP_ISP_MAX_TOP_CROP	7
+	v4l2_rect_set_min_size(r, &crop_min);
+	if (r->top > DCMIPP_ISP_MAX_TOP_CROP)
+		r->top = DCMIPP_ISP_MAX_TOP_CROP;
+	if ((r->height + r->top) > fmt->height)
+		r->height = fmt->height - r->top;
+}
+
+static int dcmipp_isp_set_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
+				    struct v4l2_subdev_state *state,
+				    struct v4l2_subdev_selection *s)
+{
+	struct dcmipp_isp_device *isp = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *sink_fmt, *src_fmt;
+	struct v4l2_rect *crop, *compose;
+	__u32 pad;
+
+	if (IS_SRC(s->pad))
+		return -EINVAL;
+
+	if (s->which == V4L2_SUBDEV_FORMAT_ACTIVE &&
+	    v4l2_subdev_is_streaming(sd))
+		return -EBUSY;
+
+	crop = v4l2_subdev_state_get_crop(state, s->pad);
+	compose = v4l2_subdev_state_get_compose(state, s->pad);
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_CROP:
+		sink_fmt = v4l2_subdev_state_get_format(state, s->pad);
+		dcmipp_isp_adjust_crop(&s->r, sink_fmt);
+
+		*crop = s->r;
+		*compose = s->r;
+
+		dev_dbg(isp->dev, "s_selection: crop (%d,%d)/%ux%u\n",
+			crop->left, crop->top, crop->width, crop->height);
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+		s->r.top = 0;
+		s->r.left = 0;
+		if (s->r.width < DCMIPP_FRAME_MIN_WIDTH)
+			s->r.width = DCMIPP_FRAME_MIN_WIDTH;
+		s->r.width = dcmipp_isp_set_compose(crop->width, s->r.width);
+		if (s->r.height < DCMIPP_FRAME_MIN_HEIGHT)
+			s->r.height = DCMIPP_FRAME_MIN_HEIGHT;
+		s->r.height = dcmipp_isp_set_compose(crop->height, s->r.height);
+		*compose = s->r;
+
+		dev_dbg(isp->dev, "s_selection: compose (%d,%d)/%ux%u\n",
+			compose->left, compose->top,
+			compose->width, compose->height);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Update the source pad size */
+	for (pad = 1; pad < sd->entity.num_pads; pad++) {
+		src_fmt = v4l2_subdev_state_get_format(state, pad);
+		src_fmt->width = s->r.width;
+		src_fmt->height = s->r.height;
+	}
+
+	return 0;
+}
+
+#define STM32_DCMIPP_IS_BAYER_VARIANT(code, variant)	\
+	((code) == MEDIA_BUS_FMT_S##variant##8_1X8 ||	\
+	 (code) == MEDIA_BUS_FMT_S##variant##10_1X10 ||	\
+	 (code) == MEDIA_BUS_FMT_S##variant##12_1X12 ||	\
+	 (code) == MEDIA_BUS_FMT_S##variant##14_1X14 ||	\
+	 (code) == MEDIA_BUS_FMT_S##variant##16_1X16)
+static void dcmipp_isp_config_demosaicing(struct dcmipp_isp_device *isp,
+					  struct v4l2_subdev_state *state)
+{
+	__u32 code = v4l2_subdev_state_get_format(state, 0)->code;
+	unsigned int val = 0;
+
+	/* Disable demosaicing */
+	reg_clear(isp, DCMIPP_P1DMCR,
+		  DCMIPP_P1DMCR_ENABLE | DCMIPP_P1DMCR_TYPE_MASK);
+
+	/* Only perform demosaicing if format is bayer */
+	if (code < MEDIA_BUS_FMT_SBGGR8_1X8 || code >= MEDIA_BUS_FMT_JPEG_1X8)
+		return;
+
+	dev_dbg(isp->dev, "Input is RawBayer, enable Demosaicing\n");
+
+	if (STM32_DCMIPP_IS_BAYER_VARIANT(code, BGGR))
+		val = DCMIPP_P1DMCR_TYPE_BGGR << DCMIPP_P1DMCR_TYPE_SHIFT;
+	else if (STM32_DCMIPP_IS_BAYER_VARIANT(code, GBRG))
+		val = DCMIPP_P1DMCR_TYPE_GBRG << DCMIPP_P1DMCR_TYPE_SHIFT;
+	else if (STM32_DCMIPP_IS_BAYER_VARIANT(code, GRBG))
+		val = DCMIPP_P1DMCR_TYPE_GRBG << DCMIPP_P1DMCR_TYPE_SHIFT;
+	else if (STM32_DCMIPP_IS_BAYER_VARIANT(code, RGGB))
+		val = DCMIPP_P1DMCR_TYPE_RGGB << DCMIPP_P1DMCR_TYPE_SHIFT;
+
+	val |= DCMIPP_P1DMCR_ENABLE;
+
+	reg_set(isp, DCMIPP_P1DMCR, val);
+}
+
+static bool dcmipp_isp_is_aux_output_enabled(struct dcmipp_isp_device *isp)
+{
+	struct media_link *link;
+
+	for_each_media_entity_data_link(isp->ved.ent, link) {
+		if (link->source != &isp->ved.pads[2])
+			continue;
+
+		if (!(link->flags & MEDIA_LNK_FL_ENABLED))
+			continue;
+
+		if (!strcmp(link->sink->entity->name, "dcmipp_aux_postproc"))
+			return true;
+	}
+
+	return false;
+}
+
+static void dcmipp_isp_config_decimation(struct dcmipp_isp_device *isp,
+					 struct v4l2_subdev_state *state)
+{
+	struct v4l2_rect *crop = v4l2_subdev_state_get_crop(state, 0);
+	struct v4l2_rect *compose = v4l2_subdev_state_get_compose(state, 0);
+	u32 decr;
+
+	decr = (fls(crop->width / compose->width) - 1) << DCMIPP_P1DECR_HDEC_SHIFT |
+	       (fls(crop->height / compose->height) - 1) << DCMIPP_P1DECR_VDEC_SHIFT;
+	if (decr)
+		decr |= DCMIPP_P1DECR_ENABLE;
+
+	reg_write(isp, DCMIPP_P1DECR, decr);
+}
+
+static int dcmipp_isp_enable_streams(struct v4l2_subdev *sd,
+				     struct v4l2_subdev_state *state,
+				     u32 pad, u64 streams_mask)
+{
+	struct dcmipp_isp_device *isp = v4l2_get_subdevdata(sd);
+	struct v4l2_rect *crop = v4l2_subdev_state_get_crop(state, 0);
+	struct v4l2_subdev *s_subdev;
+	struct media_pad *s_pad;
+	int ret;
+
+	/* Perform configuration only if no other pad is enabled */
+	if (sd->enabled_pads)
+		return 0;
+
+	/* Get source subdev */
+	s_pad = media_pad_remote_pad_first(&sd->entity.pads[0]);
+	if (!s_pad || !is_media_entity_v4l2_subdev(s_pad->entity))
+		return -EINVAL;
+	s_subdev = media_entity_to_v4l2_subdev(s_pad->entity);
+
+	/* Check if link between ISP & Pipe2 postproc is enabled */
+	if (dcmipp_isp_is_aux_output_enabled(isp))
+		reg_clear(isp, DCMIPP_P1FSCR, DCMIPP_P1FSCR_PIPEDIFF);
+	else
+		reg_set(isp, DCMIPP_P1FSCR, DCMIPP_P1FSCR_PIPEDIFF);
+
+	/* Configure Statistic Removal */
+	crop = v4l2_subdev_state_get_crop(state, 0);
+	reg_write(isp, DCMIPP_P1SRCR,
+		  ((crop->top << DCMIPP_P1SRCR_FIRSTLINEDEL_SHIFT) |
+		   (crop->height << DCMIPP_P1SRCR_LASTLINE_SHIFT) |
+		   DCMIPP_P1SRCR_CROPEN));
+
+	/* Configure Decimation */
+	dcmipp_isp_config_decimation(isp, state);
+
+	/* Configure Demosaicing */
+	dcmipp_isp_config_demosaicing(isp, state);
+
+	ret = v4l2_subdev_enable_streams(s_subdev, s_pad->index, BIT_ULL(0));
+	if (ret < 0) {
+		dev_err(isp->dev,
+			"failed to start source subdev streaming (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int dcmipp_isp_disable_streams(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_state *state,
+				      u32 pad, u64 streams_mask)
+{
+	struct dcmipp_isp_device *isp = v4l2_get_subdevdata(sd);
+	struct v4l2_subdev *s_subdev;
+	struct media_pad *s_pad;
+	int ret;
+
+	/* Don't do anything if there are still other pads enabled */
+	if ((sd->enabled_pads & ~BIT(pad)))
+		return 0;
+
+	/* Get source subdev */
+	s_pad = media_pad_remote_pad_first(&sd->entity.pads[0]);
+	if (!s_pad || !is_media_entity_v4l2_subdev(s_pad->entity))
+		return -EINVAL;
+	s_subdev = media_entity_to_v4l2_subdev(s_pad->entity);
+
+	/* Disable all blocks */
+	reg_write(isp, DCMIPP_P1SRCR, 0);
+	reg_write(isp, DCMIPP_P1DECR, 0);
+	reg_write(isp, DCMIPP_P1DMCR, 0);
+
+	ret = v4l2_subdev_disable_streams(s_subdev, s_pad->index, BIT_ULL(0));
+	if (ret < 0) {
+		dev_err(isp->dev,
+			"failed to disable source subdev streaming (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops dcmipp_isp_pad_ops = {
+	.enum_mbus_code		= dcmipp_isp_enum_mbus_code,
+	.enum_frame_size	= dcmipp_isp_enum_frame_size,
+	.get_fmt		= v4l2_subdev_get_fmt,
+	.set_fmt		= dcmipp_isp_set_fmt,
+	.get_selection		= dcmipp_pixelpipe_get_selection,
+	.set_selection		= dcmipp_isp_set_selection,
+	.enable_streams		= dcmipp_isp_enable_streams,
+	.disable_streams	= dcmipp_isp_disable_streams,
+};
+
+static const struct v4l2_subdev_video_ops dcmipp_isp_video_ops = {
+	.s_stream = v4l2_subdev_s_stream_helper,
+};
+
+static const struct v4l2_subdev_ops dcmipp_isp_ops = {
+	.pad = &dcmipp_isp_pad_ops,
+	.video = &dcmipp_isp_video_ops,
+};
+
+static void dcmipp_isp_release(struct v4l2_subdev *sd)
+{
+	struct dcmipp_isp_device *isp = v4l2_get_subdevdata(sd);
+
+	kfree(isp);
+}
+
+static const struct v4l2_subdev_internal_ops dcmipp_isp_int_ops = {
+	.init_state = dcmipp_isp_init_state,
+	.release = dcmipp_isp_release,
+};
+
+void dcmipp_isp_ent_release(struct dcmipp_ent_device *ved)
+{
+	struct dcmipp_isp_device *isp =
+			container_of(ved, struct dcmipp_isp_device, ved);
+
+	dcmipp_ent_sd_unregister(ved, &isp->sd);
+}
+
+struct dcmipp_ent_device *dcmipp_isp_ent_init(const char *entity_name,
+					      struct dcmipp_device *dcmipp)
+{
+	struct dcmipp_isp_device *isp;
+	const unsigned long pads_flag[] = {
+		MEDIA_PAD_FL_SINK, MEDIA_PAD_FL_SOURCE,
+		MEDIA_PAD_FL_SOURCE,
+	};
+	int ret;
+
+	/* Allocate the isp struct */
+	isp = kzalloc_obj(*isp);
+	if (!isp)
+		return ERR_PTR(-ENOMEM);
+
+	isp->regs = dcmipp->regs;
+
+	/* Initialize ved and sd */
+	ret = dcmipp_ent_sd_register(&isp->ved, &isp->sd,
+				     &dcmipp->v4l2_dev, entity_name,
+				     MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER,
+				     ARRAY_SIZE(pads_flag), pads_flag,
+				     &dcmipp_isp_int_ops, &dcmipp_isp_ops,
+				     NULL, NULL);
+	if (ret) {
+		kfree(isp);
+		return ERR_PTR(ret);
+	}
+
+	isp->ved.dcmipp = dcmipp;
+	isp->dev = dcmipp->dev;
+
+	return &isp->ved;
+}
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.c
new file mode 100644
index 00000000000000..d6f19db848d5df
--- /dev/null
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for STM32 Digital Camera Memory Interface Pixel Processor
+ *
+ * Copyright (C) STMicroelectronics SA 2026
+ * Authors: Hugues Fruchet <hugues.fruchet@foss.st.com>
+ *          Alain Volmat <alain.volmat@foss.st.com>
+ *          for STMicroelectronics.
+ */
+
+#include <linux/v4l2-mediabus.h>
+#include <media/v4l2-rect.h>
+#include <media/v4l2-subdev.h>
+
+#include "dcmipp-common.h"
+#include "dcmipp-pixelcommon.h"
+
+#define DCMIPP_ENT(id, pad) (1 << (2 * (id) + (pad)))
+#define DCMIPP_ISP_SINK			(DCMIPP_ENT(DCMIPP_ISP, 0))
+#define DCMIPP_ISP_SRC			(DCMIPP_ENT(DCMIPP_ISP, 1))
+#define DCMIPP_ISP_INOUT		(DCMIPP_ISP_SINK | DCMIPP_ISP_SRC)
+#define DCMIPP_MAIN_POSTPROC_SINK	(DCMIPP_ENT(DCMIPP_MAIN, 0))
+#define DCMIPP_MAIN_POSTPROC_SRC	(DCMIPP_ENT(DCMIPP_MAIN, 1))
+#define DCMIPP_MAIN_POSTPROC_INOUT					\
+	(DCMIPP_MAIN_POSTPROC_SINK | DCMIPP_MAIN_POSTPROC_SRC)
+#define DCMIPP_AUX_POSTPROC_SINK	(DCMIPP_ENT(DCMIPP_AUX, 0))
+#define DCMIPP_AUX_POSTPROC_SRC	(DCMIPP_ENT(DCMIPP_AUX, 1))
+#define DCMIPP_AUX_POSTPROC_INOUT					\
+	(DCMIPP_AUX_POSTPROC_SINK | DCMIPP_AUX_POSTPROC_SRC)
+#define DCMIPP_ALL_POSTPROC_SINK					\
+	(DCMIPP_MAIN_POSTPROC_SINK | DCMIPP_AUX_POSTPROC_SINK)
+#define DCMIPP_ALL_POSTPROC_INOUT					\
+	(DCMIPP_MAIN_POSTPROC_INOUT | DCMIPP_AUX_POSTPROC_INOUT)
+
+#define PIXMAP_MBUS(mbus, applicable_pipes)		\
+	{						\
+		.code = MEDIA_BUS_FMT_##mbus,		\
+		.pipes = applicable_pipes,		\
+	}
+static const struct dcmipp_pixelpipe_pix_map
+dcmipp_pixel_formats_list[] = {
+	/* RGB formats */
+	/* RGB565 / RGB888 */
+	PIXMAP_MBUS(RGB565_2X8_LE, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(RGB565_1X16, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(RGB888_3X8, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(RGB888_1X24, DCMIPP_ALL_POSTPROC_INOUT | DCMIPP_ISP_INOUT),
+	/* YUV formats */
+	PIXMAP_MBUS(YUYV8_2X8, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(UYVY8_1X16, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(YUV8_1X24, DCMIPP_ALL_POSTPROC_INOUT | DCMIPP_ISP_SRC),
+	/* GREY */
+	PIXMAP_MBUS(Y8_1X8, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(Y10_1X10, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(Y12_1X12, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(Y14_1X14, DCMIPP_AUX_POSTPROC_SINK | DCMIPP_ISP_SINK),
+	/* Raw Bayer */
+	/* Raw 8 */
+	PIXMAP_MBUS(SBGGR8_1X8, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGBRG8_1X8, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGRBG8_1X8, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SRGGB8_1X8, DCMIPP_ISP_SINK),
+	/* Raw 10 */
+	PIXMAP_MBUS(SBGGR10_1X10, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGBRG10_1X10, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGRBG10_1X10, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SRGGB10_1X10, DCMIPP_ISP_SINK),
+	/* Raw 12 */
+	PIXMAP_MBUS(SBGGR12_1X12, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGBRG12_1X12, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGRBG12_1X12, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SRGGB12_1X12, DCMIPP_ISP_SINK),
+	/* Raw 14 */
+	PIXMAP_MBUS(SBGGR14_1X14, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGBRG14_1X14, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SGRBG14_1X14, DCMIPP_ISP_SINK),
+	PIXMAP_MBUS(SRGGB14_1X14, DCMIPP_ISP_SINK),
+};
+
+const struct dcmipp_pixelpipe_pix_map *
+dcmipp_pixelpipe_pix_map_by_code(__u32 code, unsigned int id, unsigned int pad)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dcmipp_pixel_formats_list); i++) {
+		if (dcmipp_pixel_formats_list[i].code == code &&
+		    dcmipp_pixel_formats_list[i].pipes & DCMIPP_ENT(id, pad))
+			return &dcmipp_pixel_formats_list[i];
+	}
+
+	return NULL;
+}
+
+int dcmipp_pixelpipe_enum_mbus_code(unsigned int id,
+				    struct v4l2_subdev_mbus_code_enum *code)
+{
+	unsigned int index = code->index;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(dcmipp_pixel_formats_list); i++) {
+		if (!(dcmipp_pixel_formats_list[i].pipes &
+		      DCMIPP_ENT(id, code->pad)))
+			continue;
+
+		if (index == 0)
+			break;
+
+		index--;
+	}
+
+	if (i == ARRAY_SIZE(dcmipp_pixel_formats_list))
+		return -EINVAL;
+
+	code->code = dcmipp_pixel_formats_list[i].code;
+
+	return 0;
+}
+
+int dcmipp_pixelpipe_enum_frame_size(unsigned int id,
+				     struct v4l2_subdev_frame_size_enum *fse)
+{
+	const struct dcmipp_pixelpipe_pix_map *vpix;
+
+	if (fse->index)
+		return -EINVAL;
+
+	/* Only accept code in the pix map table */
+	vpix = dcmipp_pixelpipe_pix_map_by_code(fse->code, id, fse->pad);
+	if (!vpix)
+		return -EINVAL;
+
+	fse->min_width = DCMIPP_FRAME_MIN_WIDTH;
+	fse->max_width = DCMIPP_FRAME_MAX_WIDTH;
+	fse->min_height = DCMIPP_FRAME_MIN_HEIGHT;
+	fse->max_height = DCMIPP_FRAME_MAX_HEIGHT;
+
+	return 0;
+}
+
+int dcmipp_pixelpipe_get_selection(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
+				   struct v4l2_subdev_state *state,
+				   struct v4l2_subdev_selection *s)
+{
+	struct v4l2_mbus_framefmt *sink_fmt;
+
+	if (IS_SRC(s->pad))
+		return -EINVAL;
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_CROP:
+	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+	case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+		s->r = *v4l2_subdev_state_get_crop(state, s->pad);
+		break;
+	case V4L2_SEL_TGT_CROP_BOUNDS:
+	case V4L2_SEL_TGT_CROP_DEFAULT:
+		sink_fmt = v4l2_subdev_state_get_format(state, s->pad);
+		s->r.top = 0;
+		s->r.left = 0;
+		s->r.width = sink_fmt->width;
+		s->r.height = sink_fmt->height;
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+		s->r = *v4l2_subdev_state_get_compose(state, s->pad);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+__u32 dcmipp_pixelpipe_src_format(__u32 input_format)
+{
+	if (input_format >= MEDIA_BUS_FMT_Y8_1X8 &&
+	    input_format < MEDIA_BUS_FMT_SBGGR8_1X8)
+		return MEDIA_BUS_FMT_YUV8_1X24;
+
+	return MEDIA_BUS_FMT_RGB888_1X24;
+}
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.h b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.h
new file mode 100644
index 00000000000000..2d7c16c36c7db8
--- /dev/null
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelcommon.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Driver for STM32 Digital Camera Memory Interface Pixel Processor
+ *
+ * Copyright (C) STMicroelectronics SA 2026
+ * Authors: Hugues Fruchet <hugues.fruchet@foss.st.com>
+ *          Alain Volmat <alain.volmat@foss.st.com>
+ *          for STMicroelectronics.
+ */
+
+#ifndef _DCMIPP_PIXELCOMMON_H
+#define _DCMIPP_PIXELCOMMON_H
+
+#define IS_SINK(pad) (!(pad))
+#define IS_SRC(pad)  ((pad))
+
+#define DCMIPP_ISP		0
+#define DCMIPP_MAIN		1
+#define DCMIPP_AUX		2
+
+struct dcmipp_pixelpipe_pix_map {
+	__u32 code;
+	__u32 pipes;
+};
+
+const struct dcmipp_pixelpipe_pix_map *
+dcmipp_pixelpipe_pix_map_by_code(__u32 code, unsigned int id, unsigned int pad);
+
+int dcmipp_pixelpipe_enum_mbus_code(unsigned int id,
+				    struct v4l2_subdev_mbus_code_enum *code);
+
+int dcmipp_pixelpipe_enum_frame_size(unsigned int id,
+				     struct v4l2_subdev_frame_size_enum *fse);
+
+int dcmipp_pixelpipe_get_selection(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
+				   struct v4l2_subdev_state *state,
+				   struct v4l2_subdev_selection *s);
+
+__u32 dcmipp_pixelpipe_src_format(__u32 input_format);
+
+#endif
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelproc.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelproc.c
new file mode 100644
index 00000000000000..7d8e5f0863bf44
--- /dev/null
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-pixelproc.c
@@ -0,0 +1,942 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for STM32 Digital Camera Memory Interface Pixel Processor
+ *
+ * Copyright (C) STMicroelectronics SA 2026
+ * Authors: Hugues Fruchet <hugues.fruchet@foss.st.com>
+ *          Alain Volmat <alain.volmat@foss.st.com>
+ *          for STMicroelectronics.
+ */
+
+#include <linux/pm_runtime.h>
+#include <linux/v4l2-mediabus.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-rect.h>
+#include <media/v4l2-subdev.h>
+#include <uapi/linux/media/st/dcmipp_config.h>
+
+#include "dcmipp-common.h"
+#include "dcmipp-pixelcommon.h"
+
+#define DCMIPP_P1CRSTR	0x904
+#define DCMIPP_P2CRSTR	0xD04
+#define DCMIPP_PxCRSTR(id) (((id) == 1) ? DCMIPP_P1CRSTR :\
+			   DCMIPP_P2CRSTR)
+#define DCMIPP_PxCRSTR_HSTART_SHIFT	0
+#define DCMIPP_PxCRSTR_VSTART_SHIFT	16
+#define DCMIPP_P1CRSZR	0x908
+#define DCMIPP_P2CRSZR	0xD08
+#define DCMIPP_PxCRSZR(id) (((id) == 1) ? DCMIPP_P1CRSZR :\
+			   DCMIPP_P2CRSZR)
+#define DCMIPP_PxCRSZR_ENABLE		BIT(31)
+#define DCMIPP_PxCRSZR_HSIZE_SHIFT	0
+#define DCMIPP_PxCRSZR_VSIZE_SHIFT	16
+
+#define DCMIPP_P1DCCR	0x90C
+#define DCMIPP_P2DCCR	0xD0C
+#define DCMIPP_PxDCCR(id) (((id) == 1) ? DCMIPP_P1DCCR :\
+			   DCMIPP_P2DCCR)
+#define DCMIPP_PxDCCR_ENABLE		BIT(0)
+#define DCMIPP_PxDCCR_HDEC_SHIFT	1
+#define DCMIPP_PxDCCR_VDEC_SHIFT	3
+
+#define DCMIPP_P1DSCR	0x910
+#define DCMIPP_P2DSCR	0xD10
+#define DCMIPP_PxDSCR(id) (((id) == 1) ? DCMIPP_P1DSCR :\
+			   DCMIPP_P2DSCR)
+#define DCMIPP_PxDSCR_HDIV_SHIFT	0
+#define DCMIPP_PxDSCR_VDIV_SHIFT	16
+#define DCMIPP_PxDSCR_ENABLE		BIT(31)
+
+#define DCMIPP_P1DSRTIOR	0x914
+#define DCMIPP_P2DSRTIOR	0xD14
+#define DCMIPP_PxDSRTIOR(id) (((id) == 1) ? DCMIPP_P1DSRTIOR :\
+			   DCMIPP_P2DSRTIOR)
+#define DCMIPP_PxDSRTIOR_HRATIO_SHIFT	0
+#define DCMIPP_PxDSRTIOR_HRATIO_MASK	GENMASK(15, 0)
+#define DCMIPP_PxDSRTIOR_VRATIO_SHIFT	16
+#define DCMIPP_PxDSRTIOR_VRATIO_MASK	GENMASK(31, 16)
+
+#define DCMIPP_P1DSSZR	0x918
+#define DCMIPP_P2DSSZR	0xD18
+#define DCMIPP_PxDSSZR(id) (((id) == 1) ? DCMIPP_P1DSSZR :\
+			   DCMIPP_P2DSSZR)
+#define DCMIPP_PxDSSZR_HSIZE_SHIFT	0
+#define DCMIPP_PxDSSZR_HSIZE_MASK	GENMASK(11, 0)
+#define DCMIPP_PxDSSZR_VSIZE_SHIFT	16
+#define DCMIPP_PxDSSZR_VSIZE_MASK	GENMASK(27, 16)
+
+#define DCMIPP_P1GMCR	0x970
+#define DCMIPP_P2GMCR	0xD70
+#define DCMIPP_PxGMCR(id) (((id) == 1) ? DCMIPP_P1GMCR :\
+			   DCMIPP_P2GMCR)
+#define DCMIPP_PxGMCR_ENABLE		BIT(0)
+
+#define DCMIPP_P1YUVCR	0x980
+#define DCMIPP_P1YUVCR_ENABLE		BIT(0)
+#define DCMIPP_P1YUVCR_TYPE_RGB		BIT(1)
+#define DCMIPP_P1YUVCR_CLAMP		BIT(2)
+#define DCMIPP_P1YUVRR1	0x984
+#define DCMIPP_P1YUVRR2	0x988
+#define DCMIPP_P1YUVGR1	0x98C
+#define DCMIPP_P1YUVGR2	0x990
+#define DCMIPP_P1YUVBR1	0x994
+#define DCMIPP_P1YUVBR2	0x998
+
+#define PIXELPROC_MEDIA_BUS_FMT_DEFAULT MEDIA_BUS_FMT_RGB888_1X24
+
+/* Macro for negative coefficient, 11 bits coded */
+#define N11(val) (((val) ^ 0x7ff) + 1)
+/* Macro for added value, 10 bits coded */
+#define N10(val) (((val) ^ 0x3ff) + 1)
+
+/* Macro to convert row matrix to DCMIPP PxCCCyy register value */
+#define CCTBL(rr, rg, rb, ra, gr, gg, gb, ga, br, bg, bb, ba)		\
+	.conv_matrix = {						\
+		((rg) << 16 | (rr)), ((ra) << 16 | (rb)),		\
+		((gg) << 16 | (gr)), ((ga) << 16 | (gb)),		\
+		((bg) << 16 | (br)), ((ba) << 16 | (bb)) }
+
+struct dcmipp_colorconv_config {
+	unsigned int conv_matrix[6];
+	bool clamping;
+	bool clamping_as_rgb;
+};
+
+static const struct dcmipp_colorconv_config dcmipp_rgbfull_to_yuv601full = {
+	/*    R		G		B		Add */
+	CCTBL(131,	N11(110),	N11(21),	128,	/* Cr */
+	      77,	150,		29,		0,	/* Y */
+	      N11(44),	N11(87),	131,		128),	/* Cb */
+};
+
+static const struct dcmipp_colorconv_config dcmipp_rgbfull_to_yuv601lim = {
+	/*	R	G		B		Add */
+	CCTBL(112,	N11(94),	N11(18),	128,	/* Cr */
+	      66,	129,		25,		16,	/* Y */
+	      N11(38),	N11(74),	112,		128),	/* Cb */
+	.clamping = true,
+};
+
+static const struct dcmipp_colorconv_config dcmipp_rgbfull_to_yuv709full = {
+	/*    R		G		B		Add */
+	CCTBL(131,	N11(119),	N11(12),	128,	/* Cr */
+	      55,	183,		18,		0,	/* Y */
+	      N11(30),	N11(101),	131,		128),	/* Cb */
+};
+
+static const struct dcmipp_colorconv_config dcmipp_rgbfull_to_yuv709lim = {
+	/*    R		G		B		Add */
+	CCTBL(112,	N11(102),	N11(10),	128,	/* Cr */
+	      47,	157,		16,		16,	/* Y */
+	      N11(26),	N11(87),	112,		128),	/* Cb */
+	.clamping = true,
+};
+
+static const struct dcmipp_colorconv_config dcmipp_rgblim_to_yuv601lim = {
+	/*    R		G		B		Add */
+	CCTBL(131,	N11(110),	N11(21),	128,	/* Cr */
+	      77,	150,		29,		0,	/* Y */
+	      N11(44),	N11(87),	131,		128),	/* Cb */
+	.clamping = true,
+};
+
+static const struct dcmipp_colorconv_config dcmipp_rgblim_to_yuv709lim = {
+	/*    R		G		B		Add */
+	CCTBL(131,	N11(119),	N11(12),	128,	/* Cr */
+	      55,	183,		18,		0,	/* Y */
+	      N11(30),	N11(101),	131,		128),	/* Cb */
+	.clamping = true,
+};
+
+static const struct dcmipp_colorconv_config dcmipp_yuv601full_to_rgbfull = {
+	/*    Cr	Y	Cb		Add */
+	CCTBL(351,	256,	0,		N10(175),	/* R */
+	      N11(179),	256,	N11(86),	132,		/* G */
+	      0,	256,	443,		N10(222)),	/* B */
+};
+
+static const struct dcmipp_colorconv_config dcmipp_yuv601lim_to_rgbfull = {
+	/*    Cr	Y	Cb		Add */
+	CCTBL(409,	298,	0,		N10(223),	/* R */
+	      N11(208),	298,	N11(100),	135,		/* G */
+	      0,	298,	517,		N10(277)),	/* B */
+};
+
+static const struct dcmipp_colorconv_config dcmipp_yuv601lim_to_rgblim = {
+	/*    Cr	Y	Cb		Add */
+	CCTBL(351,	256,	0,		N10(175),	/* R */
+	      N11(179),	256,	N11(86),	132,		/* G */
+	      0,	256,	443,		N10(222)),	/* B */
+	.clamping = true,
+	.clamping_as_rgb = true,
+};
+
+static const struct dcmipp_colorconv_config dcmipp_yuv709full_to_rgbfull = {
+	/*    Cr	Y	Cb		Add */
+	CCTBL(394,	256,	0,		N10(197),	/* R */
+	      N11(118),	256,	N11(47),	82,		/* G */
+	      0,	256,	456,		N10(232)),	/* B */
+};
+
+static const struct dcmipp_colorconv_config dcmipp_yuv709lim_to_rgbfull = {
+	/*    Cr	Y	Cb		Add */
+	CCTBL(459,	298,	0,		N10(248),	/* R */
+	      N11(137),	298,	N11(55),	77,		/* G */
+	      0,	298,	541,		N10(289)),	/* B */
+};
+
+static const struct dcmipp_colorconv_config dcmipp_yuv709lim_to_rgblim = {
+	/*    Cr	Y	Cb		Add */
+	CCTBL(394,	256,	0,		N10(197),	/* R */
+	      N11(118),	256,	N11(47),	82,		/* G */
+	      0,	256,	465,		N10(232)),	/* B */
+	.clamping = true,
+	.clamping_as_rgb = true,
+};
+
+/* cconv_matrices[src_fmt][src_range][sink_fmt][sink_range] */
+static const struct dcmipp_colorconv_config *dcmipp_cconv_cfgs[3][2][3][2] = {
+	/* RGB */
+	{
+		/* RGB full range */
+		{
+			/* RGB full range => RGB */
+			{
+				NULL, NULL,
+			},
+			/* RGB full range => YUV601 */
+			{
+				&dcmipp_rgbfull_to_yuv601full,
+				&dcmipp_rgbfull_to_yuv601lim,
+			},
+			/* RGB full range => YUV709 */
+			{
+				&dcmipp_rgbfull_to_yuv709full,
+				&dcmipp_rgbfull_to_yuv709lim,
+			},
+		},
+		/* RGB limited range */
+		{
+			/* RGB limited range => RGB */
+			{
+				NULL, NULL,
+			},
+			/* RGB limited range => YUV601 */
+			{
+				NULL, &dcmipp_rgblim_to_yuv601lim,
+			},
+			/* RGB limited range => YUV709 */
+			{
+				NULL, &dcmipp_rgblim_to_yuv709lim,
+			},
+		},
+	},
+	/* YUV601 */
+	{
+		/* YUV601 full range */
+		{
+			/* YUV601 full range => RGB */
+			{
+				&dcmipp_yuv601full_to_rgbfull, NULL,
+			},
+			/* YUV601 full range => YUV601 */
+			{
+				NULL, NULL,
+			},
+			/* YUV601 full range => YUV709 */
+			{
+				NULL, NULL,
+			},
+		},
+		/* YUV601 limited range */
+		{
+			/* YUV601 limited range => RGB */
+			{
+				&dcmipp_yuv601lim_to_rgbfull,
+				&dcmipp_yuv601lim_to_rgblim,
+			},
+			/* YUV601 limited range => YUV601 */
+			{
+				NULL, NULL,
+			},
+			/* YUV601 limited range => YUV709 */
+			{
+				NULL, NULL,
+			},
+		},
+	},
+	/* YUV709 */
+	{
+		/* YUV709 full range */
+		{
+			/* YUV709 full range => RGB */
+			{
+				&dcmipp_yuv709full_to_rgbfull, NULL,
+			},
+			/* YUV709 full range => YUV601 */
+			{
+				NULL, NULL,
+			},
+			/* YUV709 full range => YUV709 */
+			{
+				NULL, NULL,
+			},
+		},
+		/* YUV709 limited range */
+		{
+			/* YUV709 limited range => RGB */
+			{
+				&dcmipp_yuv709lim_to_rgbfull,
+				&dcmipp_yuv709lim_to_rgblim,
+			},
+			/* YUV709 limited range => YUV601 */
+			{
+				NULL, NULL,
+			},
+			/* YUV709 limited range => YUV709 */
+			{
+				NULL, NULL,
+			},
+		},
+	},
+};
+
+enum dcmipp_cconv_fmt {
+	FMT_RGB = 0,
+	FMT_YUV601,
+	FMT_YUV709
+};
+
+static inline enum dcmipp_cconv_fmt to_cconv_fmt(struct v4l2_mbus_framefmt *fmt)
+{
+	/* YUV format codes are within the 0x2xxx */
+	if (fmt->code >= MEDIA_BUS_FMT_Y8_1X8 &&
+	    fmt->code < MEDIA_BUS_FMT_SBGGR8_1X8) {
+		if (fmt->ycbcr_enc == V4L2_YCBCR_ENC_709)
+			return FMT_YUV709;
+		else
+			return FMT_YUV601;
+	}
+
+	/* All other formats are referred as RGB, indeed, demosaicing bloc
+	 * generate RGB format
+	 */
+	return FMT_RGB;
+};
+
+#define FMT_STR(f) ({					\
+	typeof(f) __f = (f);				\
+	(__f) == FMT_RGB ? "RGB" :			\
+	(__f) == FMT_YUV601 ? "YUV601" :		\
+	(__f) == FMT_YUV709 ? "YUV709" : "?"; })
+
+enum dcmipp_cconv_range {
+	RANGE_FULL = 0,
+	RANGE_LIMITED,
+};
+
+static inline enum dcmipp_cconv_range
+to_cconv_range(struct v4l2_mbus_framefmt *fmt)
+{
+	if (fmt->quantization == V4L2_QUANTIZATION_FULL_RANGE)
+		return RANGE_FULL;
+
+	return RANGE_LIMITED;
+};
+
+#define RANGE_STR(range) ((range) == RANGE_FULL ? "full" : "limited")
+
+struct dcmipp_pixelproc_device {
+	struct dcmipp_ent_device ved;
+	struct v4l2_subdev sd;
+	struct device *dev;
+	bool streaming;
+
+	void __iomem *regs;
+	struct v4l2_ctrl_handler ctrls;
+
+	u32 pipe_id;
+};
+
+static const struct v4l2_mbus_framefmt fmt_default = {
+	.width = DCMIPP_FMT_WIDTH_DEFAULT,
+	.height = DCMIPP_FMT_HEIGHT_DEFAULT,
+	.code = PIXELPROC_MEDIA_BUS_FMT_DEFAULT,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = DCMIPP_COLORSPACE_DEFAULT,
+	.ycbcr_enc = DCMIPP_YCBCR_ENC_DEFAULT,
+	.quantization = DCMIPP_QUANTIZATION_DEFAULT,
+	.xfer_func = DCMIPP_XFER_FUNC_DEFAULT,
+};
+
+static const struct v4l2_rect min_rect = {
+	.width = DCMIPP_FRAME_MIN_WIDTH,
+	.height = DCMIPP_FRAME_MIN_HEIGHT,
+	.top = 0,
+	.left = 0,
+};
+
+/*
+ * Downscale is a combination of both decimation block (1/2/4/8)
+ * and downsize block (up to 8x) for a total of maximum downscale of 64
+ */
+#define DCMIPP_MAX_DECIMATION_RATIO	8
+#define DCMIPP_MAX_DOWNSIZE_RATIO	8
+#define DCMIPP_MAX_DOWNSCALE_RATIO	64
+
+/*
+ * Functions handling controls
+ */
+static int dcmipp_pixelproc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct dcmipp_pixelproc_device *pixelproc =
+		container_of(ctrl->handler,
+			     struct dcmipp_pixelproc_device, ctrls);
+
+	if (!pm_runtime_get_if_in_use(pixelproc->dev))
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_DCMIPP_PIXELPROC_GAMMA_CORRECTION_ENABLE:
+		reg_write(pixelproc, DCMIPP_PxGMCR(pixelproc->pipe_id),
+			  (ctrl->val ? DCMIPP_PxGMCR_ENABLE : 0));
+		break;
+	}
+
+	pm_runtime_put(pixelproc->dev);
+
+	return 0;
+};
+
+static const struct v4l2_ctrl_ops dcmipp_pixelproc_ctrl_ops = {
+	.s_ctrl = dcmipp_pixelproc_s_ctrl,
+};
+
+static const struct v4l2_ctrl_config dcmipp_pixelproc_ctrls[] = {
+	{
+		.ops		= &dcmipp_pixelproc_ctrl_ops,
+		.id		= V4L2_CID_DCMIPP_PIXELPROC_GAMMA_CORRECTION_ENABLE,
+		.type		= V4L2_CTRL_TYPE_BOOLEAN,
+		.name		= "Gamma Correction Enable",
+		.min = 0,
+		.max = 1,
+		.step = 1,
+		.def = 0,
+	}
+};
+
+static void dcmipp_pixelproc_adjust_crop(struct v4l2_rect *r,
+					 const struct v4l2_mbus_framefmt *fmt)
+{
+	struct v4l2_rect src_rect = {
+		.top = 0,
+		.left = 0,
+		.width = fmt->width,
+		.height = fmt->height,
+	};
+
+	/* Disallow rectangles smaller than the minimal one. */
+	v4l2_rect_set_min_size(r, &min_rect);
+	v4l2_rect_map_inside(r, &src_rect);
+}
+
+static void
+dcmipp_pixelproc_adjust_fmt(struct dcmipp_pixelproc_device *pixelproc,
+			    struct v4l2_mbus_framefmt *fmt, u32 pad)
+{
+	const struct dcmipp_pixelpipe_pix_map *vpix;
+
+	/* Only accept code in the pix map table */
+	vpix = dcmipp_pixelpipe_pix_map_by_code(fmt->code,
+						pixelproc->pipe_id == 1 ? DCMIPP_MAIN : DCMIPP_AUX,
+						pad);
+	if (!vpix)
+		fmt->code = PIXELPROC_MEDIA_BUS_FMT_DEFAULT;
+
+	fmt->width = clamp_t(u32, fmt->width, DCMIPP_FRAME_MIN_WIDTH,
+			     DCMIPP_FRAME_MAX_WIDTH);
+	fmt->height = clamp_t(u32, fmt->height, DCMIPP_FRAME_MIN_HEIGHT,
+			      DCMIPP_FRAME_MAX_HEIGHT);
+
+	if (fmt->field == V4L2_FIELD_ANY || fmt->field == V4L2_FIELD_ALTERNATE)
+		fmt->field = V4L2_FIELD_NONE;
+
+	dcmipp_colorimetry_clamp(fmt);
+}
+
+static int dcmipp_pixelproc_init_state(struct v4l2_subdev *sd,
+				       struct v4l2_subdev_state *state)
+{
+	unsigned int i;
+
+	for (i = 0; i < sd->entity.num_pads; i++) {
+		*v4l2_subdev_state_get_format(state, i) = fmt_default;
+
+		if (IS_SINK(i)) {
+			struct v4l2_rect r = {
+				.top = 0,
+				.left = 0,
+				.width = DCMIPP_FMT_WIDTH_DEFAULT,
+				.height = DCMIPP_FMT_HEIGHT_DEFAULT,
+			};
+			*v4l2_subdev_state_get_crop(state, i) = r;
+			*v4l2_subdev_state_get_compose(state, i) = r;
+		}
+	}
+
+	return 0;
+}
+
+static int
+dcmipp_pixelproc_enum_mbus_code(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *state,
+				struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct dcmipp_pixelproc_device *pixelproc = v4l2_get_subdevdata(sd);
+
+	return dcmipp_pixelpipe_enum_mbus_code(pixelproc->pipe_id == 1 ? DCMIPP_MAIN : DCMIPP_AUX,
+					       code);
+}
+
+static int
+dcmipp_pixelproc_enum_frame_size(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state,
+				 struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct dcmipp_pixelproc_device *pixelproc = v4l2_get_subdevdata(sd);
+
+	return dcmipp_pixelpipe_enum_frame_size(pixelproc->pipe_id == 1 ? DCMIPP_MAIN : DCMIPP_AUX,
+						fse);
+}
+
+static int dcmipp_pixelproc_set_fmt(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
+				    struct v4l2_subdev_state *state,
+				    struct v4l2_subdev_format *fmt)
+{
+	struct dcmipp_pixelproc_device *pixelproc = v4l2_get_subdevdata(sd);
+
+	if (v4l2_subdev_is_streaming(sd))
+		return -EBUSY;
+
+	dcmipp_pixelproc_adjust_fmt(pixelproc, &fmt->format, fmt->pad);
+
+	if (IS_SINK(fmt->pad)) {
+		struct v4l2_mbus_framefmt *src_fmt =
+			v4l2_subdev_state_get_format(state, 1);
+		struct v4l2_rect r = {
+			.top = 0,
+			.left = 0,
+			.width = fmt->format.width,
+			.height = fmt->format.height,
+		};
+
+		/* Adjust SINK pad crop/compose */
+		*v4l2_subdev_state_get_crop(state, 0) = r;
+		*v4l2_subdev_state_get_compose(state, 0) = r;
+
+		/* Forward format to SRC pad */
+		*src_fmt = fmt->format;
+		src_fmt->code = dcmipp_pixelpipe_src_format(fmt->format.code);
+	} else {
+		struct v4l2_rect *compose =
+			v4l2_subdev_state_get_compose(state, 0);
+
+		/* AUX (pipe_nb 2) cannot perform color conv */
+		if (pixelproc->pipe_id == 2) {
+			struct v4l2_mbus_framefmt *sink_fmt =
+				v4l2_subdev_state_get_format(state, 0);
+
+			fmt->format = *sink_fmt;
+			fmt->format.code =
+				dcmipp_pixelpipe_src_format(fmt->format.code);
+		}
+
+		fmt->format.width = compose->width;
+		fmt->format.height = compose->height;
+	}
+
+	/* Update the selected pad format */
+	*v4l2_subdev_state_get_format(state, fmt->pad) = fmt->format;
+
+	return 0;
+}
+
+static int dcmipp_pixelproc_set_selection(struct v4l2_subdev *sd,
+					  const struct v4l2_subdev_client_info *ci,
+					  struct v4l2_subdev_state *state,
+					  struct v4l2_subdev_selection *s)
+{
+	struct dcmipp_pixelproc_device *pixelproc = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *sink_fmt, *src_fmt;
+	struct v4l2_rect *crop, *compose;
+
+	if (IS_SRC(s->pad))
+		return -EINVAL;
+
+	if (v4l2_subdev_is_streaming(sd))
+		return -EBUSY;
+
+	crop = v4l2_subdev_state_get_crop(state, s->pad);
+	compose = v4l2_subdev_state_get_compose(state, s->pad);
+
+	switch (s->target) {
+	case V4L2_SEL_TGT_CROP:
+		sink_fmt = v4l2_subdev_state_get_format(state, s->pad);
+		dcmipp_pixelproc_adjust_crop(&s->r, sink_fmt);
+
+		*crop = s->r;
+		*compose = s->r;
+
+		dev_dbg(pixelproc->dev, "s_selection: crop (%d,%d)/%ux%u\n",
+			crop->left, crop->top, crop->width, crop->height);
+		break;
+	case V4L2_SEL_TGT_COMPOSE:
+		s->r.top = 0;
+		s->r.left = 0;
+		s->r.width = clamp_t(u32, s->r.width,
+				     crop->width / DCMIPP_MAX_DOWNSCALE_RATIO,
+				     crop->width);
+		s->r.height = clamp_t(u32, s->r.height,
+				      crop->height / DCMIPP_MAX_DOWNSCALE_RATIO,
+				      crop->height);
+		v4l2_rect_set_min_size(&s->r, &min_rect);
+		*compose = s->r;
+
+		dev_dbg(pixelproc->dev, "s_selection: compose (%d,%d)/%ux%u\n",
+			compose->left, compose->top,
+			compose->width, compose->height);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Update the source pad size */
+	src_fmt = v4l2_subdev_state_get_format(state, 1);
+	src_fmt->width = s->r.width;
+	src_fmt->height = s->r.height;
+
+	return 0;
+}
+
+static int
+dcmipp_pixelproc_colorconv_config(struct dcmipp_pixelproc_device *pixelproc,
+				  struct v4l2_mbus_framefmt *sink,
+				  struct v4l2_mbus_framefmt *src)
+{
+	const struct dcmipp_colorconv_config *cconv_cfg;
+	enum dcmipp_cconv_fmt sink_fmt = to_cconv_fmt(sink);
+	enum dcmipp_cconv_range sink_range = to_cconv_range(sink);
+	enum dcmipp_cconv_fmt src_fmt = to_cconv_fmt(src);
+	enum dcmipp_cconv_range src_range = to_cconv_range(src);
+	unsigned int val = 0;
+	int i;
+
+	/* Disable color conversion by default */
+	reg_write(pixelproc, DCMIPP_P1YUVCR, 0);
+
+	if (sink_fmt == src_fmt && sink_range == src_range)
+		return 0;
+
+	/* color conversion */
+	cconv_cfg = dcmipp_cconv_cfgs[sink_fmt][sink_range][src_fmt][src_range];
+	if (!cconv_cfg) {
+		dev_err(pixelproc->dev,
+			"Unsupported color conversion %s-%s => %s-%s\n",
+			FMT_STR(sink_fmt), RANGE_STR(sink_range),
+			FMT_STR(src_fmt), RANGE_STR(src_range));
+		return -EINVAL;
+	}
+
+	dev_dbg(pixelproc->dev, "color conversion %s-%s => %s-%s\n",
+		FMT_STR(sink_fmt), RANGE_STR(sink_range),
+		FMT_STR(src_fmt), RANGE_STR(src_range));
+
+	for (i = 0; i < 6; i++)
+		reg_write(pixelproc, DCMIPP_P1YUVRR1 + (4 * i),
+			  cconv_cfg->conv_matrix[i]);
+
+	if (cconv_cfg->clamping)
+		val |= DCMIPP_P1YUVCR_CLAMP;
+	if (cconv_cfg->clamping_as_rgb)
+		val |= DCMIPP_P1YUVCR_TYPE_RGB;
+	val |= DCMIPP_P1YUVCR_ENABLE;
+
+	reg_write(pixelproc, DCMIPP_P1YUVCR, val);
+
+	return 0;
+}
+
+#define DCMIPP_PIXELPROC_HVRATIO_CONS	8192
+#define DCMIPP_PIXELPROC_HVRATIO_MAX	65535
+#define DCMIPP_PIXELPROC_HVDIV_CONS	1024
+#define DCMIPP_PIXELPROC_HVDIV_MAX	1023
+static void
+dcmipp_pixelproc_set_crop_downscale(struct dcmipp_pixelproc_device *pixelproc,
+				    struct v4l2_rect *compose,
+				    struct v4l2_rect *crop)
+{
+	unsigned int hratio, vratio, hdiv, vdiv;
+	unsigned int hdec = 0, vdec = 0;
+	unsigned int h_post_dec = crop->width;
+	unsigned int v_post_dec = crop->height;
+
+	/* Configure cropping */
+	reg_write(pixelproc, DCMIPP_PxCRSTR(pixelproc->pipe_id),
+		  (crop->top << DCMIPP_PxCRSTR_VSTART_SHIFT) |
+		  (crop->left << DCMIPP_PxCRSTR_HSTART_SHIFT));
+	reg_write(pixelproc, DCMIPP_PxCRSZR(pixelproc->pipe_id),
+		  (crop->width << DCMIPP_PxCRSZR_HSIZE_SHIFT) |
+		  (crop->height << DCMIPP_PxCRSZR_VSIZE_SHIFT) |
+		  DCMIPP_PxCRSZR_ENABLE);
+
+	/* Compute decimation factors (HDEC/VDEC) */
+	while (compose->width * DCMIPP_MAX_DOWNSIZE_RATIO < h_post_dec) {
+		hdec++;
+		h_post_dec /= 2;
+	}
+	while (compose->height * DCMIPP_MAX_DOWNSIZE_RATIO < v_post_dec) {
+		vdec++;
+		v_post_dec /= 2;
+	}
+
+	/* Compute downsize factor */
+	hratio = h_post_dec * DCMIPP_PIXELPROC_HVRATIO_CONS /
+		 compose->width;
+	if (hratio > DCMIPP_PIXELPROC_HVRATIO_MAX)
+		hratio = DCMIPP_PIXELPROC_HVRATIO_MAX;
+	vratio = v_post_dec * DCMIPP_PIXELPROC_HVRATIO_CONS /
+		 compose->height;
+	if (vratio > DCMIPP_PIXELPROC_HVRATIO_MAX)
+		vratio = DCMIPP_PIXELPROC_HVRATIO_MAX;
+	hdiv = (DCMIPP_PIXELPROC_HVDIV_CONS * compose->width) /
+		h_post_dec;
+	if (hdiv > DCMIPP_PIXELPROC_HVDIV_MAX)
+		hdiv = DCMIPP_PIXELPROC_HVDIV_MAX;
+	vdiv = (DCMIPP_PIXELPROC_HVDIV_CONS * compose->height) /
+		v_post_dec;
+	if (vdiv > DCMIPP_PIXELPROC_HVDIV_MAX)
+		vdiv = DCMIPP_PIXELPROC_HVDIV_MAX;
+
+	dev_dbg(pixelproc->dev, "%s: decimation config: hdec: 0x%x, vdec: 0x%x\n",
+		pixelproc->sd.name,
+		hdec, vdec);
+	dev_dbg(pixelproc->dev, "%s: downsize config: hratio: 0x%x, vratio: 0x%x, hdiv: 0x%x, vdiv: 0x%x\n",
+		pixelproc->sd.name,
+		hratio, vratio,
+		hdiv, vdiv);
+
+	reg_clear(pixelproc, DCMIPP_PxDCCR(pixelproc->pipe_id),
+		  DCMIPP_PxDCCR_ENABLE);
+	if (hdec || vdec)
+		reg_write(pixelproc, DCMIPP_PxDCCR(pixelproc->pipe_id),
+			  (hdec << DCMIPP_PxDCCR_HDEC_SHIFT) |
+			  (vdec << DCMIPP_PxDCCR_VDEC_SHIFT) |
+			  DCMIPP_PxDCCR_ENABLE);
+
+	reg_clear(pixelproc, DCMIPP_PxDSCR(pixelproc->pipe_id),
+		  DCMIPP_PxDSCR_ENABLE);
+	reg_write(pixelproc, DCMIPP_PxDSRTIOR(pixelproc->pipe_id),
+		  (hratio << DCMIPP_PxDSRTIOR_HRATIO_SHIFT) |
+		  (vratio << DCMIPP_PxDSRTIOR_VRATIO_SHIFT));
+	reg_write(pixelproc, DCMIPP_PxDSSZR(pixelproc->pipe_id),
+		  (compose->width << DCMIPP_PxDSSZR_HSIZE_SHIFT) |
+		  (compose->height << DCMIPP_PxDSSZR_VSIZE_SHIFT));
+	reg_write(pixelproc, DCMIPP_PxDSCR(pixelproc->pipe_id),
+		  (hdiv << DCMIPP_PxDSCR_HDIV_SHIFT) |
+		  (vdiv << DCMIPP_PxDSCR_VDIV_SHIFT) |
+		  DCMIPP_PxDSCR_ENABLE);
+}
+
+static int dcmipp_pixelproc_enable_streams(struct v4l2_subdev *sd,
+					   struct v4l2_subdev_state *state,
+					   u32 pad, u64 streams_mask)
+{
+	struct dcmipp_pixelproc_device *pixelproc = v4l2_get_subdevdata(sd);
+	struct v4l2_subdev *s_subdev;
+	struct media_pad *s_pad;
+	int ret;
+
+	/* Get source subdev */
+	s_pad = media_pad_remote_pad_first(&sd->entity.pads[0]);
+	if (!s_pad || !is_media_entity_v4l2_subdev(s_pad->entity))
+		return -EINVAL;
+	s_subdev = media_entity_to_v4l2_subdev(s_pad->entity);
+
+	/* Configure crop/downscale */
+	dcmipp_pixelproc_set_crop_downscale(pixelproc,
+					    v4l2_subdev_state_get_compose(state, 0),
+					    v4l2_subdev_state_get_crop(state, 0));
+
+	/* Configure YUV Conversion (if applicable) */
+	if (pixelproc->pipe_id == 1) {
+		ret = dcmipp_pixelproc_colorconv_config(pixelproc,
+							v4l2_subdev_state_get_format(state, 0),
+							v4l2_subdev_state_get_format(state, 1));
+		if (ret)
+			return ret;
+	}
+
+	/* Apply customized values from user when stream starts. */
+	ret =  v4l2_ctrl_handler_setup(pixelproc->sd.ctrl_handler);
+	if (ret < 0) {
+		dev_err(pixelproc->dev,
+			"failed to start source subdev streaming (%d)\n", ret);
+		return ret;
+	}
+
+	ret = v4l2_subdev_enable_streams(s_subdev, s_pad->index, BIT_ULL(0));
+	if (ret < 0) {
+		dev_err(pixelproc->dev,
+			"failed to start source subdev streaming (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int dcmipp_pixelproc_disable_streams(struct v4l2_subdev *sd,
+					    struct v4l2_subdev_state *state,
+					    u32 pad, u64 streams_mask)
+{
+	struct dcmipp_pixelproc_device *pixelproc = v4l2_get_subdevdata(sd);
+	struct v4l2_subdev *s_subdev;
+	struct media_pad *s_pad;
+	int ret;
+
+	/* Get source subdev */
+	s_pad = media_pad_remote_pad_first(&sd->entity.pads[0]);
+	if (!s_pad || !is_media_entity_v4l2_subdev(s_pad->entity))
+		return -EINVAL;
+	s_subdev = media_entity_to_v4l2_subdev(s_pad->entity);
+
+	ret = v4l2_subdev_disable_streams(s_subdev, s_pad->index, BIT_ULL(0));
+	if (ret < 0)
+		dev_err(pixelproc->dev,
+			"failed to stop source subdev streaming (%d)\n",
+			ret);
+	return ret;
+}
+
+static const struct v4l2_subdev_pad_ops dcmipp_pixelproc_pad_ops = {
+	.enum_mbus_code		= dcmipp_pixelproc_enum_mbus_code,
+	.enum_frame_size	= dcmipp_pixelproc_enum_frame_size,
+	.get_fmt		= v4l2_subdev_get_fmt,
+	.set_fmt		= dcmipp_pixelproc_set_fmt,
+	.get_selection		= dcmipp_pixelpipe_get_selection,
+	.set_selection		= dcmipp_pixelproc_set_selection,
+	.enable_streams		= dcmipp_pixelproc_enable_streams,
+	.disable_streams	= dcmipp_pixelproc_disable_streams,
+};
+
+static const struct v4l2_subdev_core_ops dcmipp_pixelproc_core_ops = {
+	.subscribe_event = v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static const struct v4l2_subdev_video_ops dcmipp_pixelproc_video_ops = {
+	.s_stream = v4l2_subdev_s_stream_helper,
+};
+
+static const struct v4l2_subdev_ops dcmipp_pixelproc_ops = {
+	.core = &dcmipp_pixelproc_core_ops,
+	.pad = &dcmipp_pixelproc_pad_ops,
+	.video = &dcmipp_pixelproc_video_ops,
+};
+
+static void dcmipp_pixelproc_release(struct v4l2_subdev *sd)
+{
+	struct dcmipp_pixelproc_device *pixelproc = v4l2_get_subdevdata(sd);
+
+	v4l2_ctrl_handler_free(&pixelproc->ctrls);
+	kfree(pixelproc);
+}
+
+static const struct v4l2_subdev_internal_ops dcmipp_pixelproc_int_ops = {
+	.init_state = dcmipp_pixelproc_init_state,
+	.release = dcmipp_pixelproc_release,
+};
+
+void dcmipp_pixelproc_ent_release(struct dcmipp_ent_device *ved)
+{
+	struct dcmipp_pixelproc_device *pixelproc =
+			container_of(ved, struct dcmipp_pixelproc_device, ved);
+
+	dcmipp_ent_sd_unregister(ved, &pixelproc->sd);
+}
+
+static int dcmipp_name_to_pipe_id(const char *name)
+{
+	if (strstr(name, "main"))
+		return 1;
+	else if (strstr(name, "aux"))
+		return 2;
+	else
+		return -EINVAL;
+}
+
+struct dcmipp_ent_device *
+dcmipp_pixelproc_ent_init(const char *entity_name,
+			  struct dcmipp_device *dcmipp)
+{
+	struct dcmipp_pixelproc_device *pixelproc;
+	const unsigned long pads_flag[] = {
+		MEDIA_PAD_FL_SINK, MEDIA_PAD_FL_SOURCE,
+	};
+	int ret, i;
+
+	/* Allocate the pixelproc struct */
+	pixelproc = kzalloc_obj(*pixelproc);
+	if (!pixelproc)
+		return ERR_PTR(-ENOMEM);
+
+	pixelproc->regs = dcmipp->regs;
+	pixelproc->dev = dcmipp->dev;
+
+	/* Pipe identifier */
+	pixelproc->pipe_id = dcmipp_name_to_pipe_id(entity_name);
+	if (pixelproc->pipe_id != 1 && pixelproc->pipe_id != 2) {
+		dev_err(pixelproc->dev, "failed to retrieve pipe_id\n");
+		ret = -EIO;
+		goto err_kfree;
+	}
+
+	/* Initialize controls */
+	v4l2_ctrl_handler_init(&pixelproc->ctrls,
+			       ARRAY_SIZE(dcmipp_pixelproc_ctrls));
+
+	for (i = 0; i < ARRAY_SIZE(dcmipp_pixelproc_ctrls); i++)
+		v4l2_ctrl_new_custom(&pixelproc->ctrls,
+				     &dcmipp_pixelproc_ctrls[i], NULL);
+
+	pixelproc->sd.ctrl_handler = &pixelproc->ctrls;
+	if (pixelproc->ctrls.error) {
+		ret = pixelproc->ctrls.error;
+		dev_err(pixelproc->dev, "control initialization error %d\n", ret);
+		goto err_ctrl_handler_free;
+	}
+
+	/* Initialize ved and sd */
+	ret = dcmipp_ent_sd_register(&pixelproc->ved, &pixelproc->sd,
+				     &dcmipp->v4l2_dev, entity_name,
+				     MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER,
+				     ARRAY_SIZE(pads_flag), pads_flag,
+				     &dcmipp_pixelproc_int_ops,
+				     &dcmipp_pixelproc_ops,
+				     NULL, NULL);
+	if (ret)
+		goto err_ctrl_handler_free;
+
+	pixelproc->ved.dcmipp = dcmipp;
+
+	return &pixelproc->ved;
+
+err_ctrl_handler_free:
+	v4l2_ctrl_handler_free(&pixelproc->ctrls);
+err_kfree:
+	kfree(pixelproc);
+
+	return ERR_PTR(ret);
+}
diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c
index 744197b0fccb03..af35ece1d5cfaa 100644
--- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c
+++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c
@@ -295,6 +295,7 @@ static int sun4i_csi_subdev_get_fmt(struct v4l2_subdev *subdev,
 }
 
 static int sun4i_csi_subdev_set_fmt(struct v4l2_subdev *subdev,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.c
index d006d9dd017081..f9ee94fc1293b8 100644
--- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.c
+++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.c
@@ -13,26 +13,6 @@
 #include "sun6i_csi_bridge.h"
 #include "sun6i_csi_reg.h"
 
-/* Helpers */
-
-void sun6i_csi_bridge_dimensions(struct sun6i_csi_device *csi_dev,
-				 unsigned int *width, unsigned int *height)
-{
-	if (width)
-		*width = csi_dev->bridge.mbus_format.width;
-	if (height)
-		*height = csi_dev->bridge.mbus_format.height;
-}
-
-void sun6i_csi_bridge_format(struct sun6i_csi_device *csi_dev,
-			     u32 *mbus_code, u32 *field)
-{
-	if (mbus_code)
-		*mbus_code = csi_dev->bridge.mbus_format.code;
-	if (field)
-		*field = csi_dev->bridge.mbus_format.field;
-}
-
 /* Format */
 
 static const struct sun6i_csi_bridge_format sun6i_csi_bridge_formats[] = {
@@ -226,7 +206,8 @@ static void sun6i_csi_bridge_disable(struct sun6i_csi_device *csi_dev)
 }
 
 static void
-sun6i_csi_bridge_configure_parallel(struct sun6i_csi_device *csi_dev)
+sun6i_csi_bridge_configure_parallel(struct sun6i_csi_device *csi_dev,
+				    const struct v4l2_mbus_framefmt *mbus_format)
 {
 	struct device *dev = csi_dev->dev;
 	struct regmap *regmap = csi_dev->regmap;
@@ -234,11 +215,9 @@ sun6i_csi_bridge_configure_parallel(struct sun6i_csi_device *csi_dev)
 		&csi_dev->bridge.source_parallel.endpoint;
 	unsigned char bus_width = endpoint->bus.parallel.bus_width;
 	unsigned int flags = endpoint->bus.parallel.flags;
-	u32 field;
+	u32 field = mbus_format->field;
 	u32 value = SUN6I_CSI_IF_CFG_IF_CSI;
 
-	sun6i_csi_bridge_format(csi_dev, NULL, &field);
-
 	if (field == V4L2_FIELD_INTERLACED ||
 	    field == V4L2_FIELD_INTERLACED_TB ||
 	    field == V4L2_FIELD_INTERLACED_BT)
@@ -317,13 +296,12 @@ sun6i_csi_bridge_configure_parallel(struct sun6i_csi_device *csi_dev)
 }
 
 static void
-sun6i_csi_bridge_configure_mipi_csi2(struct sun6i_csi_device *csi_dev)
+sun6i_csi_bridge_configure_mipi_csi2(struct sun6i_csi_device *csi_dev,
+				     const struct v4l2_mbus_framefmt *mbus_format)
 {
 	struct regmap *regmap = csi_dev->regmap;
 	u32 value = SUN6I_CSI_IF_CFG_IF_MIPI;
-	u32 field;
-
-	sun6i_csi_bridge_format(csi_dev, NULL, &field);
+	u32 field = mbus_format->field;
 
 	if (field == V4L2_FIELD_INTERLACED ||
 	    field == V4L2_FIELD_INTERLACED_TB ||
@@ -335,19 +313,20 @@ sun6i_csi_bridge_configure_mipi_csi2(struct sun6i_csi_device *csi_dev)
 	regmap_write(regmap, SUN6I_CSI_IF_CFG_REG, value);
 }
 
-static void sun6i_csi_bridge_configure_format(struct sun6i_csi_device *csi_dev)
+static void
+sun6i_csi_bridge_configure_format(struct sun6i_csi_device *csi_dev,
+				  const struct v4l2_mbus_framefmt *mbus_format)
 {
 	struct regmap *regmap = csi_dev->regmap;
 	bool capture_streaming = csi_dev->capture.state.streaming;
 	const struct sun6i_csi_bridge_format *bridge_format;
 	const struct sun6i_csi_capture_format *capture_format;
-	u32 mbus_code, field, pixelformat;
+	u32 pixelformat;
+	u32 field = mbus_format->field;
 	u8 input_format, input_yuv_seq, output_format;
 	u32 value = 0;
 
-	sun6i_csi_bridge_format(csi_dev, &mbus_code, &field);
-
-	bridge_format = sun6i_csi_bridge_format_find(mbus_code);
+	bridge_format = sun6i_csi_bridge_format_find(mbus_format->code);
 	if (WARN_ON(!bridge_format))
 		return;
 
@@ -391,16 +370,17 @@ static void sun6i_csi_bridge_configure_format(struct sun6i_csi_device *csi_dev)
 }
 
 static void sun6i_csi_bridge_configure(struct sun6i_csi_device *csi_dev,
-				       struct sun6i_csi_bridge_source *source)
+				       struct sun6i_csi_bridge_source *source,
+				       const struct v4l2_mbus_framefmt *mbus_format)
 {
 	struct sun6i_csi_bridge *bridge = &csi_dev->bridge;
 
 	if (source == &bridge->source_parallel)
-		sun6i_csi_bridge_configure_parallel(csi_dev);
+		sun6i_csi_bridge_configure_parallel(csi_dev, mbus_format);
 	else
-		sun6i_csi_bridge_configure_mipi_csi2(csi_dev);
+		sun6i_csi_bridge_configure_mipi_csi2(csi_dev, mbus_format);
 
-	sun6i_csi_bridge_configure_format(csi_dev);
+	sun6i_csi_bridge_configure_format(csi_dev, mbus_format);
 }
 
 /* V4L2 Subdev */
@@ -415,6 +395,8 @@ static int sun6i_csi_bridge_s_stream(struct v4l2_subdev *subdev, int on)
 	struct sun6i_csi_bridge_source *source;
 	struct v4l2_subdev *source_subdev;
 	struct media_pad *remote_pad;
+	struct v4l2_subdev_state *state;
+	const struct v4l2_mbus_framefmt *mbus_format;
 	int ret;
 
 	/* Source */
@@ -433,6 +415,10 @@ static int sun6i_csi_bridge_s_stream(struct v4l2_subdev *subdev, int on)
 	else
 		source = &bridge->source_mipi_csi2;
 
+	/* Active State */
+
+	state = v4l2_subdev_lock_and_get_active_state(subdev);
+
 	if (!on) {
 		v4l2_subdev_call(source_subdev, video, s_stream, 0);
 		ret = 0;
@@ -443,7 +429,7 @@ static int sun6i_csi_bridge_s_stream(struct v4l2_subdev *subdev, int on)
 
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret < 0)
-		return ret;
+		goto unlock;
 
 	/* Clear */
 
@@ -451,7 +437,9 @@ static int sun6i_csi_bridge_s_stream(struct v4l2_subdev *subdev, int on)
 
 	/* Configure */
 
-	sun6i_csi_bridge_configure(csi_dev, source);
+	mbus_format = v4l2_subdev_state_get_format(state,
+						   SUN6I_CSI_BRIDGE_PAD_SINK);
+	sun6i_csi_bridge_configure(csi_dev, source, mbus_format);
 
 	if (capture_streaming)
 		sun6i_csi_capture_configure(csi_dev);
@@ -472,7 +460,8 @@ static int sun6i_csi_bridge_s_stream(struct v4l2_subdev *subdev, int on)
 	if (ret && ret != -ENOIOCTLCMD)
 		goto disable;
 
-	return 0;
+	ret = 0;
+	goto unlock;
 
 disable:
 	if (capture_streaming)
@@ -482,6 +471,8 @@ static int sun6i_csi_bridge_s_stream(struct v4l2_subdev *subdev, int on)
 
 	pm_runtime_put(dev);
 
+unlock:
+	v4l2_subdev_unlock_state(state);
 	return ret;
 }
 
@@ -504,21 +495,23 @@ sun6i_csi_bridge_mbus_format_prepare(struct v4l2_mbus_framefmt *mbus_format)
 static int sun6i_csi_bridge_init_state(struct v4l2_subdev *subdev,
 				       struct v4l2_subdev_state *state)
 {
-	struct sun6i_csi_device *csi_dev = v4l2_get_subdevdata(subdev);
-	unsigned int pad = SUN6I_CSI_BRIDGE_PAD_SINK;
-	struct v4l2_mbus_framefmt *mbus_format =
-		v4l2_subdev_state_get_format(state, pad);
-	struct mutex *lock = &csi_dev->bridge.lock;
+	unsigned int pad;
 
-	mutex_lock(lock);
+	/*
+	 * This subdev does not perform format conversion,
+	 * initialize both pads identically.
+	 */
+	for (pad = 0; pad < subdev->entity.num_pads; pad++) {
+		struct v4l2_mbus_framefmt *mbus_format;
 
-	mbus_format->code = sun6i_csi_bridge_formats[0].mbus_code;
-	mbus_format->width = 1280;
-	mbus_format->height = 720;
+		mbus_format = v4l2_subdev_state_get_format(state, pad);
 
-	sun6i_csi_bridge_mbus_format_prepare(mbus_format);
+		mbus_format->code = sun6i_csi_bridge_formats[0].mbus_code;
+		mbus_format->width = 1280;
+		mbus_format->height = 720;
 
-	mutex_unlock(lock);
+		sun6i_csi_bridge_mbus_format_prepare(mbus_format);
+	}
 
 	return 0;
 }
@@ -536,53 +529,33 @@ sun6i_csi_bridge_enum_mbus_code(struct v4l2_subdev *subdev,
 	return 0;
 }
 
-static int sun6i_csi_bridge_get_fmt(struct v4l2_subdev *subdev,
-				    struct v4l2_subdev_state *state,
-				    struct v4l2_subdev_format *format)
-{
-	struct sun6i_csi_device *csi_dev = v4l2_get_subdevdata(subdev);
-	struct v4l2_mbus_framefmt *mbus_format = &format->format;
-	struct mutex *lock = &csi_dev->bridge.lock;
-
-	mutex_lock(lock);
-
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
-		*mbus_format = *v4l2_subdev_state_get_format(state,
-							     format->pad);
-	else
-		*mbus_format = csi_dev->bridge.mbus_format;
-
-	mutex_unlock(lock);
-
-	return 0;
-}
-
 static int sun6i_csi_bridge_set_fmt(struct v4l2_subdev *subdev,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *state,
 				    struct v4l2_subdev_format *format)
 {
-	struct sun6i_csi_device *csi_dev = v4l2_get_subdevdata(subdev);
-	struct v4l2_mbus_framefmt *mbus_format = &format->format;
-	struct mutex *lock = &csi_dev->bridge.lock;
+	struct v4l2_mbus_framefmt *fmt;
 
-	mutex_lock(lock);
+	/* The format on the source pad always matches the sink pad. */
+	if (format->pad != SUN6I_CSI_BRIDGE_PAD_SINK)
+		return v4l2_subdev_get_fmt(subdev, state, format);
 
-	sun6i_csi_bridge_mbus_format_prepare(mbus_format);
+	sun6i_csi_bridge_mbus_format_prepare(&format->format);
 
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
-		*v4l2_subdev_state_get_format(state, format->pad) =
-			*mbus_format;
-	else
-		csi_dev->bridge.mbus_format = *mbus_format;
+	/* Set the format on the sink pad. */
+	fmt = v4l2_subdev_state_get_format(state, format->pad);
+	*fmt = format->format;
 
-	mutex_unlock(lock);
+	/* Propagate the format to the source pad. */
+	fmt = v4l2_subdev_state_get_format(state, SUN6I_CSI_BRIDGE_PAD_SOURCE);
+	*fmt = format->format;
 
 	return 0;
 }
 
 static const struct v4l2_subdev_pad_ops sun6i_csi_bridge_pad_ops = {
 	.enum_mbus_code	= sun6i_csi_bridge_enum_mbus_code,
-	.get_fmt	= sun6i_csi_bridge_get_fmt,
+	.get_fmt	= v4l2_subdev_get_fmt,
 	.set_fmt	= sun6i_csi_bridge_set_fmt,
 };
 
@@ -780,8 +753,6 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 	};
 	int ret;
 
-	mutex_init(&bridge->lock);
-
 	/* V4L2 Subdev */
 
 	v4l2_subdev_init(subdev, &sun6i_csi_bridge_subdev_ops);
@@ -809,6 +780,12 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 	if (ret < 0)
 		return ret;
 
+	/* V4L2 Subdev finalize */
+
+	ret = v4l2_subdev_init_finalize(subdev);
+	if (ret < 0)
+		goto error_media_entity;
+
 	/* V4L2 Subdev */
 
 	if (csi_dev->isp_available)
@@ -818,7 +795,7 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 
 	if (ret) {
 		dev_err(dev, "failed to register v4l2 subdev: %d\n", ret);
-		goto error_media_entity;
+		goto error_subdev_finalize;
 	}
 
 	/* V4L2 Async */
@@ -852,6 +829,9 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 	else
 		v4l2_device_unregister_subdev(subdev);
 
+error_subdev_finalize:
+	v4l2_subdev_cleanup(subdev);
+
 error_media_entity:
 	media_entity_cleanup(&subdev->entity);
 
@@ -868,5 +848,7 @@ void sun6i_csi_bridge_cleanup(struct sun6i_csi_device *csi_dev)
 
 	v4l2_device_unregister_subdev(subdev);
 
+	v4l2_subdev_cleanup(subdev);
+
 	media_entity_cleanup(&subdev->entity);
 }
diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.h b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.h
index 44653b38f72218..a5b0a6f064dda8 100644
--- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.h
+++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.h
@@ -42,20 +42,11 @@ struct sun6i_csi_bridge {
 	struct v4l2_subdev		subdev;
 	struct v4l2_async_notifier	notifier;
 	struct media_pad		pads[2];
-	struct v4l2_mbus_framefmt	mbus_format;
-	struct mutex			lock; /* Mbus format lock. */
 
 	struct sun6i_csi_bridge_source	source_parallel;
 	struct sun6i_csi_bridge_source	source_mipi_csi2;
 };
 
-/* Helpers */
-
-void sun6i_csi_bridge_dimensions(struct sun6i_csi_device *csi_dev,
-				 unsigned int *width, unsigned int *height);
-void sun6i_csi_bridge_format(struct sun6i_csi_device *csi_dev,
-			     u32 *mbus_code, u32 *field);
-
 /* Format */
 
 const struct sun6i_csi_bridge_format *
diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.c
index 65879f4802c00c..d90abba213098d 100644
--- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.c
+++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.c
@@ -888,14 +888,19 @@ static int sun6i_csi_capture_link_validate(struct media_link *link)
 		media_entity_to_video_device(link->sink->entity);
 	struct sun6i_csi_device *csi_dev = video_get_drvdata(video_dev);
 	struct v4l2_device *v4l2_dev = csi_dev->v4l2_dev;
+	struct v4l2_subdev *src_subdev =
+		media_entity_to_v4l2_subdev(link->source->entity);
 	const struct sun6i_csi_capture_format *capture_format;
 	const struct sun6i_csi_bridge_format *bridge_format;
 	unsigned int capture_width, capture_height;
-	unsigned int bridge_width, bridge_height;
 	const struct v4l2_format_info *format_info;
+	struct v4l2_subdev_format src_fmt = {
+		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
+		.pad = link->source->index
+	};
 	u32 pixelformat, capture_field;
-	u32 mbus_code, bridge_field;
 	bool match;
+	int ret;
 
 	sun6i_csi_capture_dimensions(csi_dev, &capture_width, &capture_height);
 
@@ -904,19 +909,22 @@ static int sun6i_csi_capture_link_validate(struct media_link *link)
 	if (WARN_ON(!capture_format))
 		return -EINVAL;
 
-	sun6i_csi_bridge_dimensions(csi_dev, &bridge_width, &bridge_height);
+	/* Resolve csi bridge format. */
+	ret = v4l2_subdev_call(src_subdev, pad, get_fmt, NULL, &src_fmt);
+	if (ret)
+		return ret;
 
-	sun6i_csi_bridge_format(csi_dev, &mbus_code, &bridge_field);
-	bridge_format = sun6i_csi_bridge_format_find(mbus_code);
+	bridge_format = sun6i_csi_bridge_format_find(src_fmt.format.code);
 	if (WARN_ON(!bridge_format))
 		return -EINVAL;
 
 	/* No cropping/scaling is supported. */
-	if (capture_width != bridge_width || capture_height != bridge_height) {
+	if (capture_width != src_fmt.format.width ||
+	    capture_height != src_fmt.format.height) {
 		v4l2_err(v4l2_dev,
 			 "invalid input/output dimensions: %ux%u/%ux%u\n",
-			 bridge_width, bridge_height, capture_width,
-			 capture_height);
+			 src_fmt.format.width, src_fmt.format.height,
+			 capture_width, capture_height);
 		return -EINVAL;
 	}
 
@@ -947,7 +955,8 @@ static int sun6i_csi_capture_link_validate(struct media_link *link)
 	/* With raw input mode, we need a 1:1 match between input and output. */
 	if (bridge_format->input_format == SUN6I_CSI_INPUT_FMT_RAW ||
 	    capture_format->input_format_raw) {
-		match = sun6i_csi_capture_format_match(pixelformat, mbus_code);
+		match = sun6i_csi_capture_format_match(pixelformat,
+						       src_fmt.format.code);
 		if (!match)
 			goto invalid;
 	}
diff --git a/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c b/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c
index b06cb73015cdd1..dcc8c7eadf9ac0 100644
--- a/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c
+++ b/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.c
@@ -95,12 +95,12 @@ static void sun6i_mipi_csi2_disable(struct sun6i_mipi_csi2_device *csi2_dev)
 			   SUN6I_MIPI_CSI2_CTL_EN, 0);
 }
 
-static void sun6i_mipi_csi2_configure(struct sun6i_mipi_csi2_device *csi2_dev)
+static void sun6i_mipi_csi2_configure(struct sun6i_mipi_csi2_device *csi2_dev,
+				      const struct v4l2_mbus_framefmt *mbus_format)
 {
 	struct regmap *regmap = csi2_dev->regmap;
 	unsigned int lanes_count =
 		csi2_dev->bridge.endpoint.bus.mipi_csi2.num_data_lanes;
-	struct v4l2_mbus_framefmt *mbus_format = &csi2_dev->bridge.mbus_format;
 	const struct sun6i_mipi_csi2_format *format;
 	struct device *dev = csi2_dev->dev;
 	u32 version = 0;
@@ -173,7 +173,8 @@ static int sun6i_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 	struct v4l2_subdev *source_subdev = csi2_dev->bridge.source_subdev;
 	union phy_configure_opts dphy_opts = { 0 };
 	struct phy_configure_opts_mipi_dphy *dphy_cfg = &dphy_opts.mipi_dphy;
-	struct v4l2_mbus_framefmt *mbus_format = &csi2_dev->bridge.mbus_format;
+	struct v4l2_subdev_state *state;
+	const struct v4l2_mbus_framefmt *mbus_format;
 	const struct sun6i_mipi_csi2_format *format;
 	struct phy *dphy = csi2_dev->dphy;
 	struct device *dev = csi2_dev->dev;
@@ -183,8 +184,12 @@ static int sun6i_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 	unsigned long pixel_rate;
 	int ret;
 
-	if (!source_subdev)
-		return -ENODEV;
+	state = v4l2_subdev_lock_and_get_active_state(subdev);
+
+	if (!source_subdev) {
+		ret = -ENODEV;
+		goto unlock;
+	}
 
 	if (!on) {
 		v4l2_subdev_call(source_subdev, video, s_stream, 0);
@@ -196,7 +201,7 @@ static int sun6i_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret < 0)
-		return ret;
+		goto unlock;
 
 	/* Sensor Pixel Rate */
 
@@ -222,6 +227,8 @@ static int sun6i_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 		goto error_pm;
 	}
 
+	mbus_format = v4l2_subdev_state_get_format(state,
+						   SUN6I_MIPI_CSI2_PAD_SINK);
 	format = sun6i_mipi_csi2_format_find(mbus_format->code);
 	if (WARN_ON(!format)) {
 		ret = -ENODEV;
@@ -260,7 +267,7 @@ static int sun6i_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 
 	/* Controller */
 
-	sun6i_mipi_csi2_configure(csi2_dev);
+	sun6i_mipi_csi2_configure(csi2_dev, mbus_format);
 	sun6i_mipi_csi2_enable(csi2_dev);
 
 	/* D-PHY */
@@ -277,7 +284,8 @@ static int sun6i_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 	if (ret && ret != -ENOIOCTLCMD)
 		goto disable;
 
-	return 0;
+	ret = 0;
+	goto unlock;
 
 disable:
 	phy_power_off(dphy);
@@ -286,6 +294,8 @@ static int sun6i_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 error_pm:
 	pm_runtime_put(dev);
 
+unlock:
+	v4l2_subdev_unlock_state(state);
 	return ret;
 }
 
@@ -308,21 +318,23 @@ sun6i_mipi_csi2_mbus_format_prepare(struct v4l2_mbus_framefmt *mbus_format)
 static int sun6i_mipi_csi2_init_state(struct v4l2_subdev *subdev,
 				      struct v4l2_subdev_state *state)
 {
-	struct sun6i_mipi_csi2_device *csi2_dev = v4l2_get_subdevdata(subdev);
-	unsigned int pad = SUN6I_MIPI_CSI2_PAD_SINK;
-	struct v4l2_mbus_framefmt *mbus_format =
-		v4l2_subdev_state_get_format(state, pad);
-	struct mutex *lock = &csi2_dev->bridge.lock;
+	unsigned int pad;
 
-	mutex_lock(lock);
+	/*
+	 * This subdev does not perform format conversion,
+	 * initialize both pads identically.
+	 */
+	for (pad = 0; pad < subdev->entity.num_pads; pad++) {
+		struct v4l2_mbus_framefmt *mbus_format;
 
-	mbus_format->code = sun6i_mipi_csi2_formats[0].mbus_code;
-	mbus_format->width = 640;
-	mbus_format->height = 480;
+		mbus_format = v4l2_subdev_state_get_format(state, pad);
 
-	sun6i_mipi_csi2_mbus_format_prepare(mbus_format);
+		mbus_format->code = sun6i_mipi_csi2_formats[0].mbus_code;
+		mbus_format->width = 640;
+		mbus_format->height = 480;
 
-	mutex_unlock(lock);
+		sun6i_mipi_csi2_mbus_format_prepare(mbus_format);
+	}
 
 	return 0;
 }
@@ -340,53 +352,33 @@ sun6i_mipi_csi2_enum_mbus_code(struct v4l2_subdev *subdev,
 	return 0;
 }
 
-static int sun6i_mipi_csi2_get_fmt(struct v4l2_subdev *subdev,
-				   struct v4l2_subdev_state *state,
-				   struct v4l2_subdev_format *format)
-{
-	struct sun6i_mipi_csi2_device *csi2_dev = v4l2_get_subdevdata(subdev);
-	struct v4l2_mbus_framefmt *mbus_format = &format->format;
-	struct mutex *lock = &csi2_dev->bridge.lock;
-
-	mutex_lock(lock);
-
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
-		*mbus_format = *v4l2_subdev_state_get_format(state,
-							     format->pad);
-	else
-		*mbus_format = csi2_dev->bridge.mbus_format;
-
-	mutex_unlock(lock);
-
-	return 0;
-}
-
 static int sun6i_mipi_csi2_set_fmt(struct v4l2_subdev *subdev,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_format *format)
 {
-	struct sun6i_mipi_csi2_device *csi2_dev = v4l2_get_subdevdata(subdev);
-	struct v4l2_mbus_framefmt *mbus_format = &format->format;
-	struct mutex *lock = &csi2_dev->bridge.lock;
+	struct v4l2_mbus_framefmt *fmt;
 
-	mutex_lock(lock);
+	/* The format on the source pad always matches the sink pad. */
+	if (format->pad != SUN6I_MIPI_CSI2_PAD_SINK)
+		return v4l2_subdev_get_fmt(subdev, state, format);
 
-	sun6i_mipi_csi2_mbus_format_prepare(mbus_format);
+	sun6i_mipi_csi2_mbus_format_prepare(&format->format);
 
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
-		*v4l2_subdev_state_get_format(state, format->pad) =
-			*mbus_format;
-	else
-		csi2_dev->bridge.mbus_format = *mbus_format;
+	/* Set the format on the sink pad. */
+	fmt = v4l2_subdev_state_get_format(state, format->pad);
+	*fmt = format->format;
 
-	mutex_unlock(lock);
+	/* Propagate the format to the source pad. */
+	fmt = v4l2_subdev_state_get_format(state, SUN6I_MIPI_CSI2_PAD_SOURCE);
+	*fmt = format->format;
 
 	return 0;
 }
 
 static const struct v4l2_subdev_pad_ops sun6i_mipi_csi2_pad_ops = {
 	.enum_mbus_code	= sun6i_mipi_csi2_enum_mbus_code,
-	.get_fmt	= sun6i_mipi_csi2_get_fmt,
+	.get_fmt	= v4l2_subdev_get_fmt,
 	.set_fmt	= sun6i_mipi_csi2_set_fmt,
 };
 
@@ -502,8 +494,6 @@ static int sun6i_mipi_csi2_bridge_setup(struct sun6i_mipi_csi2_device *csi2_dev)
 	bool notifier_registered = false;
 	int ret;
 
-	mutex_init(&bridge->lock);
-
 	/* V4L2 Subdev */
 
 	v4l2_subdev_init(subdev, &sun6i_mipi_csi2_subdev_ops);
@@ -532,6 +522,12 @@ static int sun6i_mipi_csi2_bridge_setup(struct sun6i_mipi_csi2_device *csi2_dev)
 	if (ret)
 		return ret;
 
+	/* V4L2 Subdev finalize */
+
+	ret = v4l2_subdev_init_finalize(subdev);
+	if (ret < 0)
+		goto error_media_entity_cleanup;
+
 	/* V4L2 Async */
 
 	v4l2_async_subdev_nf_init(notifier, subdev);
@@ -565,6 +561,9 @@ static int sun6i_mipi_csi2_bridge_setup(struct sun6i_mipi_csi2_device *csi2_dev)
 error_v4l2_notifier_cleanup:
 	v4l2_async_nf_cleanup(notifier);
 
+	v4l2_subdev_cleanup(subdev);
+
+error_media_entity_cleanup:
 	media_entity_cleanup(&subdev->entity);
 
 	return ret;
@@ -579,6 +578,7 @@ sun6i_mipi_csi2_bridge_cleanup(struct sun6i_mipi_csi2_device *csi2_dev)
 	v4l2_async_unregister_subdev(subdev);
 	v4l2_async_nf_unregister(notifier);
 	v4l2_async_nf_cleanup(notifier);
+	v4l2_subdev_cleanup(subdev);
 	media_entity_cleanup(&subdev->entity);
 }
 
diff --git a/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.h b/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.h
index 24b15e34b5e888..d72dfbd6a9939e 100644
--- a/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.h
+++ b/drivers/media/platform/sunxi/sun6i-mipi-csi2/sun6i_mipi_csi2.h
@@ -32,8 +32,6 @@ struct sun6i_mipi_csi2_bridge {
 	struct media_pad		pads[SUN6I_MIPI_CSI2_PAD_COUNT];
 	struct v4l2_fwnode_endpoint	endpoint;
 	struct v4l2_async_notifier	notifier;
-	struct v4l2_mbus_framefmt	mbus_format;
-	struct mutex			lock; /* Mbus format lock. */
 
 	struct v4l2_subdev		*source_subdev;
 };
diff --git a/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c b/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c
index dbc51daa4fe378..46604dcc8dcc46 100644
--- a/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c
+++ b/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.c
@@ -144,12 +144,12 @@ sun8i_a83t_mipi_csi2_disable(struct sun8i_a83t_mipi_csi2_device *csi2_dev)
 }
 
 static void
-sun8i_a83t_mipi_csi2_configure(struct sun8i_a83t_mipi_csi2_device *csi2_dev)
+sun8i_a83t_mipi_csi2_configure(struct sun8i_a83t_mipi_csi2_device *csi2_dev,
+			       const struct v4l2_mbus_framefmt *mbus_format)
 {
 	struct regmap *regmap = csi2_dev->regmap;
 	unsigned int lanes_count =
 		csi2_dev->bridge.endpoint.bus.mipi_csi2.num_data_lanes;
-	struct v4l2_mbus_framefmt *mbus_format = &csi2_dev->bridge.mbus_format;
 	const struct sun8i_a83t_mipi_csi2_format *format;
 	struct device *dev = csi2_dev->dev;
 	u32 version = 0;
@@ -205,7 +205,8 @@ static int sun8i_a83t_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 	struct v4l2_subdev *source_subdev = csi2_dev->bridge.source_subdev;
 	union phy_configure_opts dphy_opts = { 0 };
 	struct phy_configure_opts_mipi_dphy *dphy_cfg = &dphy_opts.mipi_dphy;
-	struct v4l2_mbus_framefmt *mbus_format = &csi2_dev->bridge.mbus_format;
+	struct v4l2_subdev_state *state;
+	const struct v4l2_mbus_framefmt *mbus_format;
 	const struct sun8i_a83t_mipi_csi2_format *format;
 	struct phy *dphy = csi2_dev->dphy;
 	struct device *dev = csi2_dev->dev;
@@ -215,8 +216,12 @@ static int sun8i_a83t_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 	unsigned long pixel_rate;
 	int ret;
 
-	if (!source_subdev)
-		return -ENODEV;
+	state = v4l2_subdev_lock_and_get_active_state(subdev);
+
+	if (!source_subdev) {
+		ret = -ENODEV;
+		goto unlock;
+	}
 
 	if (!on) {
 		v4l2_subdev_call(source_subdev, video, s_stream, 0);
@@ -228,7 +233,7 @@ static int sun8i_a83t_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret < 0)
-		return ret;
+		goto unlock;
 
 	/* Sensor pixel rate */
 
@@ -254,6 +259,9 @@ static int sun8i_a83t_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 		goto error_pm;
 	}
 
+	mbus_format =
+		v4l2_subdev_state_get_format(state,
+					     SUN8I_A83T_MIPI_CSI2_PAD_SINK);
 	format = sun8i_a83t_mipi_csi2_format_find(mbus_format->code);
 	if (WARN_ON(!format)) {
 		ret = -ENODEV;
@@ -292,7 +300,7 @@ static int sun8i_a83t_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 
 	/* Controller */
 
-	sun8i_a83t_mipi_csi2_configure(csi2_dev);
+	sun8i_a83t_mipi_csi2_configure(csi2_dev, mbus_format);
 	sun8i_a83t_mipi_csi2_enable(csi2_dev);
 
 	/* D-PHY */
@@ -309,7 +317,8 @@ static int sun8i_a83t_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 	if (ret && ret != -ENOIOCTLCMD)
 		goto disable;
 
-	return 0;
+	ret = 0;
+	goto unlock;
 
 disable:
 	phy_power_off(dphy);
@@ -318,6 +327,8 @@ static int sun8i_a83t_mipi_csi2_s_stream(struct v4l2_subdev *subdev, int on)
 error_pm:
 	pm_runtime_put(dev);
 
+unlock:
+	v4l2_subdev_unlock_state(state);
 	return ret;
 }
 
@@ -341,22 +352,23 @@ sun8i_a83t_mipi_csi2_mbus_format_prepare(struct v4l2_mbus_framefmt *mbus_format)
 static int sun8i_a83t_mipi_csi2_init_state(struct v4l2_subdev *subdev,
 					   struct v4l2_subdev_state *state)
 {
-	struct sun8i_a83t_mipi_csi2_device *csi2_dev =
-		v4l2_get_subdevdata(subdev);
-	unsigned int pad = SUN8I_A83T_MIPI_CSI2_PAD_SINK;
-	struct v4l2_mbus_framefmt *mbus_format =
-		v4l2_subdev_state_get_format(state, pad);
-	struct mutex *lock = &csi2_dev->bridge.lock;
+	unsigned int pad;
 
-	mutex_lock(lock);
+	/*
+	 * This subdev does not perform format conversion,
+	 * initialize both pads identically.
+	 */
+	for (pad = 0; pad < subdev->entity.num_pads; pad++) {
+		struct v4l2_mbus_framefmt *mbus_format;
 
-	mbus_format->code = sun8i_a83t_mipi_csi2_formats[0].mbus_code;
-	mbus_format->width = 640;
-	mbus_format->height = 480;
+		mbus_format = v4l2_subdev_state_get_format(state, pad);
 
-	sun8i_a83t_mipi_csi2_mbus_format_prepare(mbus_format);
+		mbus_format->code = sun8i_a83t_mipi_csi2_formats[0].mbus_code;
+		mbus_format->width = 640;
+		mbus_format->height = 480;
 
-	mutex_unlock(lock);
+		sun8i_a83t_mipi_csi2_mbus_format_prepare(mbus_format);
+	}
 
 	return 0;
 }
@@ -375,55 +387,34 @@ sun8i_a83t_mipi_csi2_enum_mbus_code(struct v4l2_subdev *subdev,
 	return 0;
 }
 
-static int sun8i_a83t_mipi_csi2_get_fmt(struct v4l2_subdev *subdev,
-					struct v4l2_subdev_state *state,
-					struct v4l2_subdev_format *format)
-{
-	struct sun8i_a83t_mipi_csi2_device *csi2_dev =
-		v4l2_get_subdevdata(subdev);
-	struct v4l2_mbus_framefmt *mbus_format = &format->format;
-	struct mutex *lock = &csi2_dev->bridge.lock;
-
-	mutex_lock(lock);
-
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
-		*mbus_format = *v4l2_subdev_state_get_format(state,
-							     format->pad);
-	else
-		*mbus_format = csi2_dev->bridge.mbus_format;
-
-	mutex_unlock(lock);
-
-	return 0;
-}
-
 static int sun8i_a83t_mipi_csi2_set_fmt(struct v4l2_subdev *subdev,
+					const struct v4l2_subdev_client_info *ci,
 					struct v4l2_subdev_state *state,
 					struct v4l2_subdev_format *format)
 {
-	struct sun8i_a83t_mipi_csi2_device *csi2_dev =
-		v4l2_get_subdevdata(subdev);
-	struct v4l2_mbus_framefmt *mbus_format = &format->format;
-	struct mutex *lock = &csi2_dev->bridge.lock;
+	struct v4l2_mbus_framefmt *fmt;
 
-	mutex_lock(lock);
+	/* The format on the source pad always matches the sink pad. */
+	if (format->pad != SUN8I_A83T_MIPI_CSI2_PAD_SINK)
+		return v4l2_subdev_get_fmt(subdev, state, format);
 
-	sun8i_a83t_mipi_csi2_mbus_format_prepare(mbus_format);
+	sun8i_a83t_mipi_csi2_mbus_format_prepare(&format->format);
 
-	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
-		*v4l2_subdev_state_get_format(state, format->pad) =
-			*mbus_format;
-	else
-		csi2_dev->bridge.mbus_format = *mbus_format;
+	/* Set the format on the sink pad. */
+	fmt = v4l2_subdev_state_get_format(state, format->pad);
+	*fmt = format->format;
 
-	mutex_unlock(lock);
+	/* Propagate the format to the source pad. */
+	fmt = v4l2_subdev_state_get_format(state,
+					   SUN8I_A83T_MIPI_CSI2_PAD_SOURCE);
+	*fmt = format->format;
 
 	return 0;
 }
 
 static const struct v4l2_subdev_pad_ops sun8i_a83t_mipi_csi2_pad_ops = {
 	.enum_mbus_code	= sun8i_a83t_mipi_csi2_enum_mbus_code,
-	.get_fmt	= sun8i_a83t_mipi_csi2_get_fmt,
+	.get_fmt	= v4l2_subdev_get_fmt,
 	.set_fmt	= sun8i_a83t_mipi_csi2_set_fmt,
 };
 
@@ -540,8 +531,6 @@ sun8i_a83t_mipi_csi2_bridge_setup(struct sun8i_a83t_mipi_csi2_device *csi2_dev)
 	bool notifier_registered = false;
 	int ret;
 
-	mutex_init(&bridge->lock);
-
 	/* V4L2 Subdev */
 
 	v4l2_subdev_init(subdev, &sun8i_a83t_mipi_csi2_subdev_ops);
@@ -570,6 +559,12 @@ sun8i_a83t_mipi_csi2_bridge_setup(struct sun8i_a83t_mipi_csi2_device *csi2_dev)
 	if (ret)
 		return ret;
 
+	/* V4L2 Subdev finalize */
+
+	ret = v4l2_subdev_init_finalize(subdev);
+	if (ret < 0)
+		goto error_media_entity_cleanup;
+
 	/* V4L2 Async */
 
 	v4l2_async_subdev_nf_init(notifier, subdev);
@@ -603,6 +598,9 @@ sun8i_a83t_mipi_csi2_bridge_setup(struct sun8i_a83t_mipi_csi2_device *csi2_dev)
 error_v4l2_notifier_cleanup:
 	v4l2_async_nf_cleanup(notifier);
 
+	v4l2_subdev_cleanup(subdev);
+
+error_media_entity_cleanup:
 	media_entity_cleanup(&subdev->entity);
 
 	return ret;
@@ -617,6 +615,7 @@ sun8i_a83t_mipi_csi2_bridge_cleanup(struct sun8i_a83t_mipi_csi2_device *csi2_dev
 	v4l2_async_unregister_subdev(subdev);
 	v4l2_async_nf_unregister(notifier);
 	v4l2_async_nf_cleanup(notifier);
+	v4l2_subdev_cleanup(subdev);
 	media_entity_cleanup(&subdev->entity);
 }
 
diff --git a/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.h b/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.h
index f1e64c53434c8e..819527bcd64dde 100644
--- a/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.h
+++ b/drivers/media/platform/sunxi/sun8i-a83t-mipi-csi2/sun8i_a83t_mipi_csi2.h
@@ -33,8 +33,6 @@ struct sun8i_a83t_mipi_csi2_bridge {
 	struct media_pad		pads[SUN8I_A83T_MIPI_CSI2_PAD_COUNT];
 	struct v4l2_fwnode_endpoint	endpoint;
 	struct v4l2_async_notifier	notifier;
-	struct v4l2_mbus_framefmt	mbus_format;
-	struct mutex			lock; /* Mbus format lock. */
 
 	struct v4l2_subdev		*source_subdev;
 };
diff --git a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
index 41e48365167e5f..b7247d597c9cc0 100644
--- a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
+++ b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
@@ -479,6 +479,7 @@ dw_mipi_csi2rx_enum_mbus_code(struct v4l2_subdev *sd,
 }
 
 static int dw_mipi_csi2rx_set_fmt(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *state,
 				  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
index 25f8ca0d6d9465..c62614077d972c 100644
--- a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
+++ b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
@@ -41,6 +41,8 @@
 #include <media/videobuf2-dma-contig.h>
 #include <media/videobuf2-v4l2.h>
 
+#include <sound/hdmi-codec.h>
+
 #include "snps_hdmirx.h"
 #include "snps_hdmirx_cec.h"
 
@@ -132,6 +134,13 @@ struct snps_hdmirx_dev {
 	struct delayed_work delayed_work_hotplug;
 	struct delayed_work delayed_work_res_change;
 	struct hdmirx_cec *cec;
+	struct platform_device *audio_pdev;
+	struct clk *audio_clk;
+	struct delayed_work audio_work;
+	u32 audio_clkrate;
+	u32 audio_fs;
+	int audio_pre_state;
+	bool audio_streaming;
 	struct mutex phy_rw_lock; /* to protect phy r/w configuration */
 	struct mutex stream_lock; /* to lock video stream capture */
 	struct mutex work_lock; /* to lock the critical section of hotplug event */
@@ -156,7 +165,6 @@ struct snps_hdmirx_dev {
 	int num_clks;
 	u32 edid_blocks_written;
 	u32 cur_fmt_fourcc;
-	u32 color_depth;
 	spinlock_t rst_lock; /* to lock register access */
 	u8 edid[EDID_NUM_BLOCKS_MAX * EDID_BLOCK_SIZE];
 };
@@ -380,6 +388,38 @@ static void hdmirx_toggle_polarity(struct snps_hdmirx_dev *hdmirx_dev)
 			   VPROC_HSYNC_POL_OVR_EN, 0);
 }
 
+static u32 hdmirx_get_colordepth(struct snps_hdmirx_dev *hdmirx_dev)
+{
+	struct v4l2_device *v4l2_dev = &hdmirx_dev->v4l2_dev;
+	u32 val, color_depth_reg, color_depth;
+
+	val = hdmirx_readl(hdmirx_dev, DMA_STATUS11);
+	color_depth_reg = (val & HDMIRX_COLOR_DEPTH_MASK) >> 3;
+
+	switch (color_depth_reg) {
+	case 0x4:
+		color_depth = 24;
+		break;
+	case 0x5:
+		color_depth = 30;
+		break;
+	case 0x6:
+		color_depth = 36;
+		break;
+	case 0x7:
+		color_depth = 48;
+		break;
+	default:
+		color_depth = 24;
+		break;
+	}
+
+	v4l2_dbg(1, debug, v4l2_dev, "%s: color_depth: %d, reg_val:%d\n",
+		 __func__, color_depth, color_depth_reg);
+
+	return color_depth;
+}
+
 /*
  * When querying DV timings during preview, if the DMA's timing is stable,
  * we retrieve the timings directly from the DMA. However, if the current
@@ -393,7 +433,7 @@ static int hdmirx_get_detected_timings(struct snps_hdmirx_dev *hdmirx_dev,
 	struct v4l2_bt_timings *bt = &timings->bt;
 	u32 val, tmdsqpclk_freq, pix_clk;
 	unsigned int num_retries = 0;
-	u32 field_type, deframer_st;
+	u32 field_type, deframer_st, color_depth;
 	u64 tmp_data, tmds_clk;
 	bool is_dvi_mode;
 	int ret;
@@ -414,10 +454,11 @@ static int hdmirx_get_detected_timings(struct snps_hdmirx_dev *hdmirx_dev,
 	deframer_st = hdmirx_readl(hdmirx_dev, DEFRAMER_STATUS);
 	is_dvi_mode = !(deframer_st & OPMODE_STS_MASK);
 
+	color_depth = hdmirx_get_colordepth(hdmirx_dev);
 	tmdsqpclk_freq = hdmirx_readl(hdmirx_dev, CMU_TMDSQPCLK_FREQ);
 	tmds_clk = tmdsqpclk_freq * 4 * 1000;
 	tmp_data = tmds_clk * 24;
-	do_div(tmp_data, hdmirx_dev->color_depth);
+	do_div(tmp_data, color_depth);
 	pix_clk = tmp_data;
 	bt->pixelclock = pix_clk;
 
@@ -429,7 +470,7 @@ static int hdmirx_get_detected_timings(struct snps_hdmirx_dev *hdmirx_dev,
 	v4l2_dbg(2, debug, v4l2_dev, "tmds_clk:%llu, pix_clk:%d\n", tmds_clk, pix_clk);
 	v4l2_dbg(1, debug, v4l2_dev, "interlace:%d, fmt:%d, color:%d, mode:%s\n",
 		 bt->interlaced, hdmirx_dev->pix_fmt,
-		 hdmirx_dev->color_depth,
+		 color_depth,
 		 is_dvi_mode ? "dvi" : "hdmi");
 	v4l2_dbg(2, debug, v4l2_dev, "deframer_st:%#x\n", deframer_st);
 
@@ -470,12 +511,12 @@ static int hdmirx_query_dv_timings(struct file *file, void *priv,
 	int ret;
 
 	if (port_no_link(hdmirx_dev)) {
-		v4l2_err(v4l2_dev, "%s: port has no link\n", __func__);
+		v4l2_dbg(1, debug, v4l2_dev, "%s: port has no link\n", __func__);
 		return -ENOLINK;
 	}
 
 	if (signal_not_lock(hdmirx_dev)) {
-		v4l2_err(v4l2_dev, "%s: signal is not locked\n", __func__);
+		v4l2_dbg(1, debug, v4l2_dev, "%s: signal is not locked\n", __func__);
 		return -ENOLCK;
 	}
 
@@ -506,9 +547,9 @@ static void hdmirx_hpd_ctrl(struct snps_hdmirx_dev *hdmirx_dev, bool en)
 	hdmirx_writel(hdmirx_dev, CORE_CONFIG,
 		      hdmirx_dev->hpd_trigger_level_high ? en : !en);
 
-	/* 100ms delay as per HDMI spec + extra 50ms to cover internal delay */
+	/* 100ms delay as per HDMI spec + extra 43ms to cover internal delay */
 	if (!en)
-		msleep(100 + 50);
+		msleep(jiffies_to_msecs(V4L2_SET_EDID_HPD_LOW_JIFFIES));
 }
 
 static void hdmirx_write_edid_data(struct snps_hdmirx_dev *hdmirx_dev,
@@ -988,36 +1029,6 @@ static void hdmirx_controller_init(struct snps_hdmirx_dev *hdmirx_dev)
 			   VS_REMAPFILTER_EN_QST | VS_FILTER_ORDER_QST(0x3));
 }
 
-static void hdmirx_get_colordepth(struct snps_hdmirx_dev *hdmirx_dev)
-{
-	struct v4l2_device *v4l2_dev = &hdmirx_dev->v4l2_dev;
-	u32 val, color_depth_reg;
-
-	val = hdmirx_readl(hdmirx_dev, DMA_STATUS11);
-	color_depth_reg = (val & HDMIRX_COLOR_DEPTH_MASK) >> 3;
-
-	switch (color_depth_reg) {
-	case 0x4:
-		hdmirx_dev->color_depth = 24;
-		break;
-	case 0x5:
-		hdmirx_dev->color_depth = 30;
-		break;
-	case 0x6:
-		hdmirx_dev->color_depth = 36;
-		break;
-	case 0x7:
-		hdmirx_dev->color_depth = 48;
-		break;
-	default:
-		hdmirx_dev->color_depth = 24;
-		break;
-	}
-
-	v4l2_dbg(1, debug, v4l2_dev, "%s: color_depth: %d, reg_val:%d\n",
-		 __func__, hdmirx_dev->color_depth, color_depth_reg);
-}
-
 static void hdmirx_get_pix_fmt(struct snps_hdmirx_dev *hdmirx_dev)
 {
 	struct v4l2_device *v4l2_dev = &hdmirx_dev->v4l2_dev;
@@ -1128,7 +1139,6 @@ static void hdmirx_format_change(struct snps_hdmirx_dev *hdmirx_dev)
 	};
 
 	hdmirx_get_pix_fmt(hdmirx_dev);
-	hdmirx_get_colordepth(hdmirx_dev);
 	hdmirx_get_avi_infoframe(hdmirx_dev);
 
 	v4l2_dbg(1, debug, v4l2_dev, "%s: queue res_chg_event\n", __func__);
@@ -1198,6 +1208,9 @@ static void hdmirx_submodule_init(struct snps_hdmirx_dev *hdmirx_dev)
 static int hdmirx_enum_input(struct file *file, void *priv,
 			     struct v4l2_input *input)
 {
+	struct hdmirx_stream *stream = video_drvdata(file);
+	struct snps_hdmirx_dev *hdmirx_dev = stream->hdmirx_dev;
+
 	if (input->index > 0)
 		return -EINVAL;
 
@@ -1206,6 +1219,12 @@ static int hdmirx_enum_input(struct file *file, void *priv,
 	strscpy(input->name, "HDMI IN", sizeof(input->name));
 	input->capabilities = V4L2_IN_CAP_DV_TIMINGS;
 
+	input->status = 0;
+	if (port_no_link(hdmirx_dev))
+		input->status |= V4L2_IN_ST_NO_POWER;
+	if (signal_not_lock(hdmirx_dev))
+		input->status |= V4L2_IN_ST_NO_SIGNAL;
+
 	return 0;
 }
 
@@ -2265,11 +2284,6 @@ static const struct hdmirx_cec_ops hdmirx_cec_ops = {
 	.read = hdmirx_readl,
 };
 
-static void devm_hdmirx_of_reserved_mem_device_release(void *dev)
-{
-	of_reserved_mem_device_release(dev);
-}
-
 static int hdmirx_parse_dt(struct snps_hdmirx_dev *hdmirx_dev)
 {
 	struct device *dev = hdmirx_dev->dev;
@@ -2279,6 +2293,13 @@ static int hdmirx_parse_dt(struct snps_hdmirx_dev *hdmirx_dev)
 	if (hdmirx_dev->num_clks < 1)
 		return -ENODEV;
 
+	for (int i = 0; i < hdmirx_dev->num_clks; i++) {
+		if (!strcmp(hdmirx_dev->clks[i].id, "audio")) {
+			hdmirx_dev->audio_clk = hdmirx_dev->clks[i].clk;
+			break;
+		}
+	}
+
 	hdmirx_dev->resets[HDMIRX_RST_A].id = "axi";
 	hdmirx_dev->resets[HDMIRX_RST_P].id = "apb";
 	hdmirx_dev->resets[HDMIRX_RST_REF].id = "ref";
@@ -2316,16 +2337,9 @@ static int hdmirx_parse_dt(struct snps_hdmirx_dev *hdmirx_dev)
 	if (!device_property_read_bool(dev, "hpd-is-active-low"))
 		hdmirx_dev->hpd_trigger_level_high = true;
 
-	ret = of_reserved_mem_device_init(dev);
-	if (ret) {
+	ret = devm_of_reserved_mem_device_init(dev);
+	if (ret)
 		dev_warn(dev, "no reserved memory for HDMIRX, use default CMA\n");
-	} else {
-		ret = devm_add_action_or_reset(dev,
-					       devm_hdmirx_of_reserved_mem_device_release,
-					       dev);
-		if (ret)
-			return ret;
-	}
 
 	return 0;
 }
@@ -2523,10 +2537,19 @@ static void hdmirx_enable_irq(struct device *dev)
 			   msecs_to_jiffies(110));
 }
 
+static void hdmirx_audio_setup(struct snps_hdmirx_dev *hdmirx_dev, u32 fs);
+
 static __maybe_unused int hdmirx_suspend(struct device *dev)
 {
 	struct snps_hdmirx_dev *hdmirx_dev = dev_get_drvdata(dev);
 
+	/*
+	 * Stop the audio worker before the controller clocks are gated;
+	 * the audio path is restored and the worker re-armed from
+	 * resume() while a capture stream is active.
+	 */
+	cancel_delayed_work_sync(&hdmirx_dev->audio_work);
+
 	hdmirx_disable_irq(dev);
 
 	/* TODO store CEC HW state */
@@ -2549,6 +2572,19 @@ static __maybe_unused int hdmirx_resume(struct device *dev)
 		hdmirx_hpd_ctrl(hdmirx_dev, true);
 	}
 
+	/*
+	 * hdmirx_enable() fully reset the controller, wiping the audio
+	 * configuration. If a capture stream is active across suspend,
+	 * re-program the audio path with the last known sample rate and
+	 * restart the worker; its rate change and FIFO error paths
+	 * resynchronize once the source delivers audio again.
+	 */
+	if (READ_ONCE(hdmirx_dev->audio_streaming)) {
+		hdmirx_audio_setup(hdmirx_dev, hdmirx_dev->audio_fs);
+		mod_delayed_work(system_unbound_wq, &hdmirx_dev->audio_work,
+				 msecs_to_jiffies(200));
+	}
+
 	/* TODO restore CEC HW state */
 	enable_irq(hdmirx_dev->cec->irq);
 
@@ -2595,10 +2631,8 @@ static int hdmirx_setup_irq(struct snps_hdmirx_dev *hdmirx_dev,
 	ret = devm_request_threaded_irq(dev, irq, NULL, hdmirx_dma_irq_handler,
 					IRQF_ONESHOT, "rk_hdmirx-dma",
 					hdmirx_dev);
-	if (ret) {
-		dev_err_probe(dev, ret, "failed to request dma irq\n");
+	if (ret)
 		return ret;
-	}
 
 	irq = gpiod_to_irq(hdmirx_dev->detect_5v_gpio);
 	if (irq < 0) {
@@ -2609,10 +2643,11 @@ static int hdmirx_setup_irq(struct snps_hdmirx_dev *hdmirx_dev,
 	irq_set_status_flags(irq, IRQ_NOAUTOEN);
 
 	hdmirx_dev->det_irq = irq;
-	ret = devm_request_irq(dev, irq, hdmirx_5v_det_irq_handler,
-			       IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING,
-			       "rk_hdmirx-5v", hdmirx_dev);
-	if (ret) {
+	ret = devm_request_any_context_irq(dev, irq, hdmirx_5v_det_irq_handler,
+					   IRQF_TRIGGER_FALLING |
+					   IRQF_TRIGGER_RISING,
+					   "rk_hdmirx-5v", hdmirx_dev);
+	if (ret < 0) {
 		dev_err_probe(dev, ret, "failed to request hdmirx-5v irq\n");
 		return ret;
 	}
@@ -2646,6 +2681,266 @@ static int hdmirx_register_cec(struct snps_hdmirx_dev *hdmirx_dev,
 	return 0;
 }
 
+#define HDMIRX_AUDIO_INIT_FIFO_STATE	128
+#define HDMIRX_AUDIO_INIT_STATE		(HDMIRX_AUDIO_INIT_FIFO_STATE * 4)
+
+static const int hdmirx_supported_fs[] = {
+	32000, 44100, 48000, 88200, 96000, 176400, 192000, 768000, -1
+};
+
+static int hdmirx_audio_closest_fs(int fs)
+{
+	int i = 0, fs_t = hdmirx_supported_fs[0];
+
+	while (fs_t > 0) {
+		if (abs(fs - fs_t) <= 2000)
+			return fs_t;
+		fs_t = hdmirx_supported_fs[++i];
+	}
+	return 0;
+}
+
+/* Recover the incoming audio sample rate from the ACR N/CTS + TMDS clock. */
+static u32 hdmirx_audio_fs(struct snps_hdmirx_dev *hdmirx_dev)
+{
+	u64 tmds_clk, fs_audio = 0;
+	u32 acr_cts, acr_n, tmdsqpclk_freq;
+	u32 acr_pb3_0, acr_pb7_4;
+
+	tmdsqpclk_freq = hdmirx_readl(hdmirx_dev, CMU_TMDSQPCLK_FREQ);
+	hdmirx_readl(hdmirx_dev, PKTDEC_ACR_PH2_1);
+	acr_pb3_0 = hdmirx_readl(hdmirx_dev, PKTDEC_ACR_PB3_0);
+	acr_pb7_4 = hdmirx_readl(hdmirx_dev, PKTDEC_ACR_PB7_4);
+	/*
+	 * The packet decoder stores the ACR subpacket bytes with packet byte
+	 * 0 in register bits [7:0], so byte-reverse each word to line the
+	 * bytes up: CTS is packet bytes 1-3 (PKTDEC_ACR_PB3_0) and N is
+	 * packet bytes 4-6 (PKTDEC_ACR_PB7_4), 20 bits each. readl()
+	 * already abstracts the bus endianness, so the reversal is
+	 * unconditional.
+	 */
+	acr_cts = swab32(acr_pb3_0) & 0xfffff;
+	acr_n = (swab32(acr_pb7_4) & 0x0fffff00) >> 8;
+	tmds_clk = tmdsqpclk_freq * 4 * 1000U;
+	if (acr_cts != 0) {
+		fs_audio = div_u64((tmds_clk * acr_n), acr_cts);
+		fs_audio /= 128;
+		fs_audio = hdmirx_audio_closest_fs(fs_audio);
+	}
+	return (u32)fs_audio;
+}
+
+/* Nudge the audio reference clock by +/- ppm to keep the FIFO balanced. */
+static void hdmirx_audio_clk_ppm_inc(struct snps_hdmirx_dev *hdmirx_dev, int ppm)
+{
+	int delta, inc;
+	long rate = hdmirx_dev->audio_clkrate;
+
+	if (ppm < 0) {
+		ppm = -ppm;
+		inc = -1;
+	} else {
+		inc = 1;
+	}
+	delta = (int)div64_u64((u64)rate * ppm + 500000, 1000000);
+	delta *= inc;
+	rate = hdmirx_dev->audio_clkrate + delta;
+	clk_set_rate(hdmirx_dev->audio_clk, rate);
+	hdmirx_dev->audio_clkrate = rate;
+}
+
+static int hdmirx_audio_clk_adjust(struct snps_hdmirx_dev *hdmirx_dev,
+				   int total_offset, int single_offset)
+{
+	int schedule_time = 500;
+	int ppm = 10;
+	u32 offset_abs = abs(total_offset);
+
+	if (offset_abs > 200) {
+		ppm += 200;
+		schedule_time -= 100;
+	}
+	if (offset_abs > 100) {
+		ppm += 200;
+		schedule_time -= 100;
+	}
+	if (offset_abs > 32) {
+		ppm += 20;
+		schedule_time -= 100;
+	}
+	if (offset_abs > 16)
+		ppm += 20;
+	if (total_offset > 16 && single_offset > 0)
+		hdmirx_audio_clk_ppm_inc(hdmirx_dev, ppm);
+	else if (total_offset < -16 && single_offset < 0)
+		hdmirx_audio_clk_ppm_inc(hdmirx_dev, -ppm);
+	return schedule_time;
+}
+
+static void hdmirx_audio_fifo_reinit(struct snps_hdmirx_dev *hdmirx_dev)
+{
+	hdmirx_writel(hdmirx_dev, AUDIO_FIFO_CONTROL, 1);
+	usleep_range(200, 210);
+	hdmirx_writel(hdmirx_dev, AUDIO_FIFO_CONTROL, 0);
+}
+
+/*
+ * Program the audio clock, FIFO thresholds and enables for the given
+ * sample rate. Shared by hw_params and system resume: the controller is
+ * fully reset on resume, so the whole configuration must be re-applied.
+ */
+static void hdmirx_audio_setup(struct snps_hdmirx_dev *hdmirx_dev, u32 fs)
+{
+	hdmirx_dev->audio_fs = fs;
+	hdmirx_dev->audio_clkrate = fs * 128;
+	clk_set_rate(hdmirx_dev->audio_clk, hdmirx_dev->audio_clkrate);
+
+	hdmirx_audio_fifo_reinit(hdmirx_dev);
+	hdmirx_writel(hdmirx_dev, AUDIO_FIFO_THR_PASS, HDMIRX_AUDIO_INIT_FIFO_STATE);
+	hdmirx_writel(hdmirx_dev, AUDIO_FIFO_THR,
+		      AFIFO_THR_LOW_QST(0x20) | AFIFO_THR_HIGH_QST(0x160));
+	hdmirx_writel(hdmirx_dev, AUDIO_FIFO_MUTE_THR,
+		      AFIFO_THR_MUTE_LOW_QST(0x8) | AFIFO_THR_MUTE_HIGH_QST(0x178));
+
+	hdmirx_update_bits(hdmirx_dev, AUDIO_PROC_CONFIG0, I2S_EN, I2S_EN);
+	hdmirx_update_bits(hdmirx_dev, GLOBAL_SWENABLE, AUDIO_ENABLE, AUDIO_ENABLE);
+
+	hdmirx_dev->audio_pre_state = 0;
+}
+
+/*
+ * Periodic worker that locks the local audio clock to the source by keeping
+ * the audio FIFO fill level close to its target, avoiding under/overflow.
+ */
+static void hdmirx_audio_work(struct work_struct *work)
+{
+	struct snps_hdmirx_dev *hdmirx_dev =
+		container_of(to_delayed_work(work), struct snps_hdmirx_dev, audio_work);
+	unsigned long delay = 200;
+	int cur, total, single;
+	u32 fifo, fs;
+
+	fs = hdmirx_audio_fs(hdmirx_dev);
+	fifo = hdmirx_readl(hdmirx_dev, AUDIO_FIFO_STATUS2);
+
+	if (fifo & (AFIFO_UNDERFLOW_ST | AFIFO_OVERFLOW_ST)) {
+		if (fs) {
+			clk_set_rate(hdmirx_dev->audio_clk, fs * 128);
+			hdmirx_dev->audio_clkrate = fs * 128;
+			hdmirx_dev->audio_fs = fs;
+		}
+		hdmirx_audio_fifo_reinit(hdmirx_dev);
+		hdmirx_dev->audio_pre_state = 0;
+		goto out;
+	}
+
+	cur = fifo & 0xffff;
+	total = cur - HDMIRX_AUDIO_INIT_STATE;
+	single = cur - hdmirx_dev->audio_pre_state;
+
+	if (fs && abs((int)fs - (int)hdmirx_dev->audio_fs) > 1000) {
+		clk_set_rate(hdmirx_dev->audio_clk, fs * 128);
+		hdmirx_dev->audio_clkrate = fs * 128;
+		hdmirx_dev->audio_fs = fs;
+		hdmirx_audio_fifo_reinit(hdmirx_dev);
+		hdmirx_dev->audio_pre_state = 0;
+		goto out;
+	}
+
+	if (cur != 0)
+		delay = hdmirx_audio_clk_adjust(hdmirx_dev, total, single);
+	hdmirx_dev->audio_pre_state = cur;
+out:
+	/* Only re-arm while streaming; avoids a self-reschedule race with
+	 * the cancel_delayed_work_sync() callers (hw_params and
+	 * audio_shutdown).
+	 */
+	if (READ_ONCE(hdmirx_dev->audio_streaming))
+		queue_delayed_work(system_unbound_wq, &hdmirx_dev->audio_work,
+				   msecs_to_jiffies(delay));
+}
+
+static int hdmirx_audio_hw_params(struct device *dev, void *data,
+				  struct hdmi_codec_daifmt *fmt,
+				  struct hdmi_codec_params *hparms)
+{
+	struct snps_hdmirx_dev *hdmirx_dev = dev_get_drvdata(dev);
+	u32 fs;
+
+	/* Only the I2S interface (DAI 0) is wired up so far. */
+	if (fmt->fmt == HDMI_SPDIF)
+		return -EOPNOTSUPP;
+
+	/*
+	 * Stop the worker before touching the shared audio state; it is
+	 * re-armed below once the new parameters are in place.
+	 */
+	WRITE_ONCE(hdmirx_dev->audio_streaming, false);
+	cancel_delayed_work_sync(&hdmirx_dev->audio_work);
+
+	fs = hdmirx_audio_fs(hdmirx_dev);
+	if (!fs)
+		fs = hparms ? hparms->sample_rate : 48000;
+	if (!fs)
+		fs = 48000;
+
+	hdmirx_audio_setup(hdmirx_dev, fs);
+
+	WRITE_ONCE(hdmirx_dev->audio_streaming, true);
+	mod_delayed_work(system_unbound_wq, &hdmirx_dev->audio_work,
+			 msecs_to_jiffies(200));
+
+	dev_dbg(dev, "audio hw_params: fs=%u\n", fs);
+	return 0;
+}
+
+static void hdmirx_audio_shutdown(struct device *dev, void *data)
+{
+	struct snps_hdmirx_dev *hdmirx_dev = dev_get_drvdata(dev);
+
+	WRITE_ONCE(hdmirx_dev->audio_streaming, false);
+	cancel_delayed_work_sync(&hdmirx_dev->audio_work);
+	hdmirx_update_bits(hdmirx_dev, GLOBAL_SWENABLE, AUDIO_ENABLE, 0);
+}
+
+static const struct hdmi_codec_ops hdmirx_audio_codec_ops = {
+	.hw_params = hdmirx_audio_hw_params,
+	.audio_shutdown = hdmirx_audio_shutdown,
+};
+
+static int hdmirx_register_audio_device(struct snps_hdmirx_dev *hdmirx_dev)
+{
+	struct hdmi_codec_pdata codec_data = {
+		.ops = &hdmirx_audio_codec_ops,
+		.i2s = 1,
+		.no_i2s_playback = 1,
+		.max_i2s_channels = 8,
+		/*
+		 * The controller also has an S/PDIF audio interface (DAI 1 in
+		 * the binding). Register it so DAI indexes match the binding,
+		 * but reject its use in hw_params() until it is wired up.
+		 */
+		.spdif = 1,
+		.no_spdif_playback = 1,
+		.data = hdmirx_dev,
+	};
+	struct platform_device *audio_pdev;
+
+	if (!hdmirx_dev->audio_clk)
+		return -ENODEV;
+
+	audio_pdev = platform_device_register_data(hdmirx_dev->dev,
+						   HDMI_CODEC_DRV_NAME,
+						   PLATFORM_DEVID_AUTO,
+						   &codec_data, sizeof(codec_data));
+	if (IS_ERR(audio_pdev))
+		return PTR_ERR(audio_pdev);
+
+	hdmirx_dev->audio_pdev = audio_pdev;
+
+	return 0;
+}
+
 static int hdmirx_probe(struct platform_device *pdev)
 {
 	struct snps_hdmirx_dev *hdmirx_dev;
@@ -2697,6 +2992,7 @@ static int hdmirx_probe(struct platform_device *pdev)
 			  hdmirx_delayed_work_hotplug);
 	INIT_DELAYED_WORK(&hdmirx_dev->delayed_work_res_change,
 			  hdmirx_delayed_work_res_change);
+	INIT_DELAYED_WORK(&hdmirx_dev->audio_work, hdmirx_audio_work);
 
 	hdmirx_dev->cur_fmt_fourcc = V4L2_PIX_FMT_BGR24;
 	hdmirx_dev->timings = cea640x480;
@@ -2765,6 +3061,10 @@ static int hdmirx_probe(struct platform_device *pdev)
 						       V4L2_DEBUGFS_IF_AVI, hdmirx_dev,
 						       hdmirx_debugfs_if_read);
 
+	ret = hdmirx_register_audio_device(hdmirx_dev);
+	if (ret)
+		dev_warn(dev, "failed to register HDMI audio codec: %d\n", ret);
+
 	return 0;
 
 err_unreg_video_dev:
@@ -2784,6 +3084,9 @@ static void hdmirx_remove(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct snps_hdmirx_dev *hdmirx_dev = dev_get_drvdata(dev);
 
+	if (hdmirx_dev->audio_pdev)
+		platform_device_unregister(hdmirx_dev->audio_pdev);
+
 	v4l2_debugfs_if_free(hdmirx_dev->infoframes);
 	debugfs_remove_recursive(hdmirx_dev->debugfs_dir);
 
diff --git a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.h b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.h
index 31b887e94e871e..a99f54fd1744c0 100644
--- a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.h
+++ b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.h
@@ -81,6 +81,7 @@
 #define DATAPATH_ENABLE				BIT(12)
 #define PKTFIFO_ENABLE				BIT(11)
 #define AVPUNIT_ENABLE				BIT(8)
+#define AUDIO_ENABLE				BIT(9)
 #define MAIN_ENABLE				BIT(0)
 #define GLOBAL_TIMER_REF_BASE			0x0028
 #define CORE_CONFIG				0x0050
@@ -177,20 +178,27 @@
 #define VPROC_FMT_OVR_VALUE(x)			UPDATE(x, 6, 4)
 #define VPROC_FMT_OVR_EN			BIT(0)
 
+#define AUDIO_FIFO_CONFIG			0x0460
 #define AFIFO_FILL_RESTART			BIT(0)
+#define AUDIO_FIFO_CONTROL			0x0464
 #define AFIFO_INIT_P				BIT(0)
+#define AUDIO_FIFO_THR_PASS			0x0468
+#define AUDIO_FIFO_THR				0x046c
 #define AFIFO_THR_LOW_QST_MASK			GENMASK(25, 16)
 #define AFIFO_THR_LOW_QST(x)			UPDATE(x, 25, 16)
 #define AFIFO_THR_HIGH_QST_MASK			GENMASK(9, 0)
 #define AFIFO_THR_HIGH_QST(x)			UPDATE(x, 9, 0)
+#define AUDIO_FIFO_MUTE_THR			0x0470
 #define AFIFO_THR_MUTE_LOW_QST_MASK		GENMASK(25, 16)
 #define AFIFO_THR_MUTE_LOW_QST(x)		UPDATE(x, 25, 16)
 #define AFIFO_THR_MUTE_HIGH_QST_MASK		GENMASK(9, 0)
 #define AFIFO_THR_MUTE_HIGH_QST(x)		UPDATE(x, 9, 0)
 
+#define AUDIO_FIFO_STATUS2			0x0478
 #define AFIFO_UNDERFLOW_ST			BIT(25)
 #define AFIFO_OVERFLOW_ST			BIT(24)
 
+#define AUDIO_PROC_CONFIG0			0x0480
 #define SPEAKER_ALLOC_OVR_EN			BIT(16)
 #define I2S_BPCUV_EN				BIT(4)
 #define SPDIF_EN				BIT(2)
diff --git a/drivers/media/platform/ti/Kconfig b/drivers/media/platform/ti/Kconfig
index 1a020b2bbb4f21..03df5617e99df9 100644
--- a/drivers/media/platform/ti/Kconfig
+++ b/drivers/media/platform/ti/Kconfig
@@ -30,17 +30,6 @@ config VIDEO_TI_CAL
 	  In TI Technical Reference Manual this module is referred as
 	  Camera Interface Subsystem (CAMSS).
 
-config VIDEO_TI_CAL_MC
-	bool "Media Controller centric mode by default"
-	depends on VIDEO_TI_CAL
-	default n
-	help
-	  Enables Media Controller centric mode by default.
-
-	  If set, CAL driver will start in Media Controller mode by
-	  default. Note that this behavior can be overridden via
-	  module parameter 'mc_api'.
-
 config VIDEO_TI_VIP
         tristate "TI Video Input Port"
         depends on VIDEO_DEV
diff --git a/drivers/media/platform/ti/am437x/am437x-vpfe.c b/drivers/media/platform/ti/am437x/am437x-vpfe.c
index 1ca559df7e5900..3cd5ee69d3c4f8 100644
--- a/drivers/media/platform/ti/am437x/am437x-vpfe.c
+++ b/drivers/media/platform/ti/am437x/am437x-vpfe.c
@@ -1314,7 +1314,7 @@ static int __subdev_set_format(struct vpfe_device *vpfe,
 
 	*mbus_fmt = *fmt;
 
-	ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, &sd_fmt);
+	ret = v4l2_subdev_call(sd, pad, set_fmt, NULL, NULL, &sd_fmt);
 	if (ret)
 		return ret;
 
diff --git a/drivers/media/platform/ti/cal/cal-camerarx.c b/drivers/media/platform/ti/cal/cal-camerarx.c
index 00a71dac0ff4d6..a978d27fe0295e 100644
--- a/drivers/media/platform/ti/cal/cal-camerarx.c
+++ b/drivers/media/platform/ti/cal/cal-camerarx.c
@@ -762,6 +762,7 @@ static int cal_camerarx_sd_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd,
+				   const struct v4l2_subdev_client_info *ci,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/ti/cal/cal-video.c b/drivers/media/platform/ti/cal/cal-video.c
index d40e24ab112783..1990cab7734470 100644
--- a/drivers/media/platform/ti/cal/cal-video.c
+++ b/drivers/media/platform/ti/cal/cal-video.c
@@ -123,6 +123,21 @@ static int __subdev_get_format(struct cal_ctx *ctx,
 	return 0;
 }
 
+static int cal_call_active_state_set_fmt(struct v4l2_subdev *source,
+					 struct v4l2_subdev_format *fmt)
+{
+	struct v4l2_subdev_state *source_state;
+	int ret;
+
+	source_state = v4l2_subdev_lock_and_get_active_state(source);
+
+	ret = v4l2_subdev_call(source, pad, set_fmt, NULL, source_state, fmt);
+	if (source_state)
+		v4l2_subdev_unlock_state(source_state);
+
+	return ret;
+}
+
 static int __subdev_set_format(struct cal_ctx *ctx,
 			       struct v4l2_mbus_framefmt *fmt)
 {
@@ -136,7 +151,7 @@ static int __subdev_set_format(struct cal_ctx *ctx,
 
 	*mbus_fmt = *fmt;
 
-	ret = v4l2_subdev_call_state_active(sd, pad, set_fmt, &sd_fmt);
+	ret = cal_call_active_state_set_fmt(sd, &sd_fmt);
 	if (ret)
 		return ret;
 
@@ -281,7 +296,7 @@ static int cal_legacy_s_fmt_vid_cap(struct file *file, void *priv,
 	ctx->v_fmt.fmt.pix.field = sd_fmt.format.field;
 	cal_calc_format_size(ctx, fmtinfo, &ctx->v_fmt);
 
-	v4l2_subdev_call_state_active(sd, pad, set_fmt, &sd_fmt);
+	cal_call_active_state_set_fmt(sd, &sd_fmt);
 
 	ctx->fmtinfo = fmtinfo;
 	*f = ctx->v_fmt;
diff --git a/drivers/media/platform/ti/cal/cal.c b/drivers/media/platform/ti/cal/cal.c
index b7e77b6b8950cc..5c82378d1eeedd 100644
--- a/drivers/media/platform/ti/cal/cal.c
+++ b/drivers/media/platform/ti/cal/cal.c
@@ -43,13 +43,7 @@ unsigned int cal_debug;
 module_param_named(debug, cal_debug, uint, 0644);
 MODULE_PARM_DESC(debug, "activates debug info");
 
-#ifdef CONFIG_VIDEO_TI_CAL_MC
-#define CAL_MC_API_DEFAULT 1
-#else
-#define CAL_MC_API_DEFAULT 0
-#endif
-
-bool cal_mc_api = CAL_MC_API_DEFAULT;
+bool cal_mc_api = 1;
 module_param_named(mc_api, cal_mc_api, bool, 0444);
 MODULE_PARM_DESC(mc_api, "activates the MC API");
 
@@ -1228,6 +1222,8 @@ static int cal_probe(struct platform_device *pdev)
 
 	/* Create contexts. */
 	if (!cal_mc_api) {
+		dev_warn(cal->dev, "The legacy non-MC API is deprecated\n");
+
 		for (i = 0; i < cal->data->num_csi2_phy; ++i) {
 			struct cal_ctx *ctx;
 
diff --git a/drivers/media/platform/ti/davinci/vpif_capture.c b/drivers/media/platform/ti/davinci/vpif_capture.c
index 91cb6223561a8e..397f3115af5b7d 100644
--- a/drivers/media/platform/ti/davinci/vpif_capture.c
+++ b/drivers/media/platform/ti/davinci/vpif_capture.c
@@ -1602,7 +1602,7 @@ vpif_capture_get_pdata(struct platform_device *pdev,
 static int vpif_probe(struct platform_device *pdev)
 {
 	struct vpif_subdev_info *subdevdata;
-	struct i2c_adapter *i2c_adap;
+	struct i2c_adapter *i2c_adap = NULL;
 	int subdev_count;
 	int res_idx = 0;
 	int i, err;
@@ -1692,9 +1692,12 @@ static int vpif_probe(struct platform_device *pdev)
 		}
 	}
 
+	i2c_put_adapter(i2c_adap);
+
 	return 0;
 
 probe_subdev_out:
+	i2c_put_adapter(i2c_adap);
 	v4l2_async_nf_cleanup(&vpif_obj.notifier);
 	/* free sub devices memory */
 	kfree(vpif_obj.sd);
diff --git a/drivers/media/platform/ti/davinci/vpif_display.c b/drivers/media/platform/ti/davinci/vpif_display.c
index 08877ae1513a8d..8bea1de6ccc82d 100644
--- a/drivers/media/platform/ti/davinci/vpif_display.c
+++ b/drivers/media/platform/ti/davinci/vpif_display.c
@@ -1289,9 +1289,12 @@ static int vpif_probe(struct platform_device *pdev)
 	if (err)
 		goto probe_subdev_out;
 
+	i2c_put_adapter(i2c_adap);
+
 	return 0;
 
 probe_subdev_out:
+	i2c_put_adapter(i2c_adap);
 	kfree(vpif_obj.sd);
 vpif_unregister:
 	v4l2_device_unregister(&vpif_obj.v4l2_dev);
diff --git a/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c b/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c
index 4769931b19306d..022ea3d616c9e1 100644
--- a/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c
+++ b/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c
@@ -225,6 +225,30 @@ static const struct ti_csi2rx_fmt ti_csi2rx_formats[] = {
 		.csi_dt			= MIPI_CSI2_DT_RAW10,
 		.bpp			= 16,
 		.size			= SHIM_DMACNTX_SIZE_16,
+	}, {
+		.fourcc			= V4L2_PIX_FMT_SBGGR12,
+		.code			= MEDIA_BUS_FMT_SBGGR12_1X12,
+		.csi_dt			= MIPI_CSI2_DT_RAW12,
+		.bpp			= 16,
+		.size			= SHIM_DMACNTX_SIZE_16,
+	}, {
+		.fourcc			= V4L2_PIX_FMT_SGBRG12,
+		.code			= MEDIA_BUS_FMT_SGBRG12_1X12,
+		.csi_dt			= MIPI_CSI2_DT_RAW12,
+		.bpp			= 16,
+		.size			= SHIM_DMACNTX_SIZE_16,
+	}, {
+		.fourcc			= V4L2_PIX_FMT_SGRBG12,
+		.code			= MEDIA_BUS_FMT_SGRBG12_1X12,
+		.csi_dt			= MIPI_CSI2_DT_RAW12,
+		.bpp			= 16,
+		.size			= SHIM_DMACNTX_SIZE_16,
+	}, {
+		.fourcc			= V4L2_PIX_FMT_SRGGB12,
+		.code			= MEDIA_BUS_FMT_SRGGB12_1X12,
+		.csi_dt			= MIPI_CSI2_DT_RAW12,
+		.bpp			= 16,
+		.size			= SHIM_DMACNTX_SIZE_16,
 	}, {
 		.fourcc			= V4L2_PIX_FMT_RGB565X,
 		.code			= MEDIA_BUS_FMT_RGB565_1X16,
@@ -1043,6 +1067,7 @@ static int ti_csi2rx_enum_mbus_code(struct v4l2_subdev *subdev,
 }
 
 static int ti_csi2rx_sd_set_fmt(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *state,
 				struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/platform/ti/omap3isp/ispccdc.c b/drivers/media/platform/ti/omap3isp/ispccdc.c
index 4708b630349337..f422d70ab6e462 100644
--- a/drivers/media/platform/ti/omap3isp/ispccdc.c
+++ b/drivers/media/platform/ti/omap3isp/ispccdc.c
@@ -2232,6 +2232,7 @@ static int ccdc_enum_frame_size(struct v4l2_subdev *sd,
  * Return 0 on success or a negative error code otherwise.
  */
 static int ccdc_get_selection(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_selection *sel)
 {
@@ -2276,6 +2277,7 @@ static int ccdc_get_selection(struct v4l2_subdev *sd,
  * Return 0 on success or a negative error code otherwise.
  */
 static int ccdc_set_selection(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_selection *sel)
 {
@@ -2346,6 +2348,7 @@ static int ccdc_get_format(struct v4l2_subdev *sd,
  * to the format type.
  */
 static int ccdc_set_format(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
@@ -2472,7 +2475,7 @@ static int ccdc_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 	format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
 	format.format.width = 4096;
 	format.format.height = 4096;
-	ccdc_set_format(sd, fh ? fh->state : NULL, &format);
+	ccdc_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 
 	return 0;
 }
diff --git a/drivers/media/platform/ti/omap3isp/ispccp2.c b/drivers/media/platform/ti/omap3isp/ispccp2.c
index d668111b44f448..c0da5cca9ddc2e 100644
--- a/drivers/media/platform/ti/omap3isp/ispccp2.c
+++ b/drivers/media/platform/ti/omap3isp/ispccp2.c
@@ -776,6 +776,7 @@ static int ccp2_get_format(struct v4l2_subdev *sd,
  * returns zero
  */
 static int ccp2_set_format(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
@@ -820,7 +821,7 @@ static int ccp2_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 	format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
 	format.format.width = 4096;
 	format.format.height = 4096;
-	ccp2_set_format(sd, fh ? fh->state : NULL, &format);
+	ccp2_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 
 	return 0;
 }
diff --git a/drivers/media/platform/ti/omap3isp/ispcsi2.c b/drivers/media/platform/ti/omap3isp/ispcsi2.c
index f227042b61b68e..8fc2948d654c63 100644
--- a/drivers/media/platform/ti/omap3isp/ispcsi2.c
+++ b/drivers/media/platform/ti/omap3isp/ispcsi2.c
@@ -994,6 +994,7 @@ static int csi2_get_format(struct v4l2_subdev *sd,
  * return -EINVAL or zero on success
  */
 static int csi2_set_format(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
@@ -1038,7 +1039,7 @@ static int csi2_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 	format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
 	format.format.width = 4096;
 	format.format.height = 4096;
-	csi2_set_format(sd, fh ? fh->state : NULL, &format);
+	csi2_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 
 	return 0;
 }
diff --git a/drivers/media/platform/ti/omap3isp/isppreview.c b/drivers/media/platform/ti/omap3isp/isppreview.c
index 3f3b5bd9cdc7ad..ac5025ce4315f8 100644
--- a/drivers/media/platform/ti/omap3isp/isppreview.c
+++ b/drivers/media/platform/ti/omap3isp/isppreview.c
@@ -1925,6 +1925,7 @@ static int preview_enum_frame_size(struct v4l2_subdev *sd,
  * Return 0 on success or a negative error code otherwise.
  */
 static int preview_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -1968,6 +1969,7 @@ static int preview_get_selection(struct v4l2_subdev *sd,
  * Return 0 on success or a negative error code otherwise.
  */
 static int preview_set_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -2035,6 +2037,7 @@ static int preview_get_format(struct v4l2_subdev *sd,
  * return -EINVAL or zero on success
  */
 static int preview_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
@@ -2090,7 +2093,7 @@ static int preview_init_formats(struct v4l2_subdev *sd,
 	format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10;
 	format.format.width = 4096;
 	format.format.height = 4096;
-	preview_set_format(sd, fh ? fh->state : NULL, &format);
+	preview_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 
 	return 0;
 }
diff --git a/drivers/media/platform/ti/omap3isp/ispresizer.c b/drivers/media/platform/ti/omap3isp/ispresizer.c
index ad0127f5b5cbda..874aa451a2d08e 100644
--- a/drivers/media/platform/ti/omap3isp/ispresizer.c
+++ b/drivers/media/platform/ti/omap3isp/ispresizer.c
@@ -1222,6 +1222,7 @@ static void resizer_try_crop(const struct v4l2_mbus_framefmt *sink,
  * Return 0 on success or a negative error code otherwise.
  */
 static int resizer_get_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -1275,6 +1276,7 @@ static int resizer_get_selection(struct v4l2_subdev *sd,
  * Return 0 on success or a negative error code otherwise.
  */
 static int resizer_set_selection(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_selection *sel)
 {
@@ -1500,6 +1502,7 @@ static int resizer_get_format(struct v4l2_subdev *sd,
  * return -EINVAL or zero on success
  */
 static int resizer_set_format(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *sd_state,
 			      struct v4l2_subdev_format *fmt)
 {
@@ -1577,7 +1580,7 @@ static int resizer_init_formats(struct v4l2_subdev *sd,
 	format.format.code = MEDIA_BUS_FMT_YUYV8_1X16;
 	format.format.width = 4096;
 	format.format.height = 4096;
-	resizer_set_format(sd, fh ? fh->state : NULL, &format);
+	resizer_set_format(sd, NULL, fh ? fh->state : NULL, &format);
 
 	return 0;
 }
diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c
index b946c8087c7780..3cb75880aea048 100644
--- a/drivers/media/platform/ti/omap3isp/ispvideo.c
+++ b/drivers/media/platform/ti/omap3isp/ispvideo.c
@@ -13,7 +13,6 @@
 #include <linux/clk.h>
 #include <linux/mm.h>
 #include <linux/module.h>
-#include <linux/pagemap.h>
 #include <linux/scatterlist.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
@@ -839,7 +838,7 @@ isp_video_get_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 	 * implemented.
 	 */
 	sdsel.pad = pad;
-	ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel);
+	ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, NULL, &sdsel);
 	if (!ret)
 		sel->r = sdsel.r;
 	if (ret != -ENOIOCTLCMD)
@@ -890,7 +889,7 @@ isp_video_set_selection(struct file *file, void *fh, struct v4l2_selection *sel)
 
 	sdsel.pad = pad;
 	mutex_lock(&video->mutex);
-	ret = v4l2_subdev_call(subdev, pad, set_selection, NULL, &sdsel);
+	ret = v4l2_subdev_call(subdev, pad, set_selection, NULL, NULL, &sdsel);
 	mutex_unlock(&video->mutex);
 	if (!ret)
 		sel->r = sdsel.r;
diff --git a/drivers/media/platform/ti/vpe/vip.c b/drivers/media/platform/ti/vpe/vip.c
index ccb688f4d7afd3..2bbe8eb3155591 100644
--- a/drivers/media/platform/ti/vpe/vip.c
+++ b/drivers/media/platform/ti/vpe/vip.c
@@ -1748,7 +1748,7 @@ static int vip_s_fmt_vid_cap(struct file *file, void *priv,
 
 	sfmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	sfmt.pad = 0;
-	ret = v4l2_subdev_call(port->subdev, pad, set_fmt, NULL, &sfmt);
+	ret = v4l2_subdev_call(port->subdev, pad, set_fmt, NULL, NULL, &sfmt);
 	if (ret) {
 		v4l2_dbg(1, debug, &dev->v4l2_dev, "set_fmt failed in subdev\n");
 		return ret;
@@ -2620,7 +2620,7 @@ static int vip_init_port(struct vip_port *port)
 		mbus_fmt->code = fmt->code;
 		sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 		sd_fmt.pad = 0;
-		ret = v4l2_subdev_call(port->subdev, pad, set_fmt,
+		ret = v4l2_subdev_call(port->subdev, pad, set_fmt, NULL,
 				       NULL, &sd_fmt);
 		if (ret)
 			v4l2_dbg(1, debug, &dev->v4l2_dev, "init_port set_fmt failed in subdev: (%d)\n",
diff --git a/drivers/media/platform/via/via-camera.c b/drivers/media/platform/via/via-camera.c
index 1b81acba7da0a8..6a6e10c6f649c5 100644
--- a/drivers/media/platform/via/via-camera.c
+++ b/drivers/media/platform/via/via-camera.c
@@ -249,7 +249,7 @@ static int viacam_configure_sensor(struct via_camera *cam)
 	v4l2_fill_mbus_format(&format.format, &cam->sensor_format, cam->mbus_code);
 	ret = sensor_call(cam, core, init, 0);
 	if (ret == 0)
-		ret = sensor_call(cam, pad, set_fmt, NULL, &format);
+		ret = sensor_call(cam, pad, set_fmt, NULL, NULL, &format);
 	/*
 	 * OV7670 does weird things if flip is set *before* format...
 	 */
@@ -842,7 +842,7 @@ static int viacam_do_try_fmt(struct via_camera *cam,
 	upix->pixelformat = f->pixelformat;
 	viacam_fmt_pre(upix, spix);
 	v4l2_fill_mbus_format(&format.format, spix, f->mbus_code);
-	ret = sensor_call(cam, pad, set_fmt, &pad_state, &format);
+	ret = sensor_call(cam, pad, set_fmt, NULL, &pad_state, &format);
 	v4l2_fill_pix_format(spix, &format.format);
 	viacam_fmt_post(upix, spix);
 	return ret;
diff --git a/drivers/media/platform/video-mux.c b/drivers/media/platform/video-mux.c
index cba34893258a85..b253b260b605d3 100644
--- a/drivers/media/platform/video-mux.c
+++ b/drivers/media/platform/video-mux.c
@@ -146,6 +146,7 @@ static const struct v4l2_subdev_video_ops video_mux_subdev_video_ops = {
 };
 
 static int video_mux_set_format(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/media/platform/xilinx/xilinx-csi2rxss.c b/drivers/media/platform/xilinx/xilinx-csi2rxss.c
index 146131b8f37e5a..2f6f3af4e492c5 100644
--- a/drivers/media/platform/xilinx/xilinx-csi2rxss.c
+++ b/drivers/media/platform/xilinx/xilinx-csi2rxss.c
@@ -694,6 +694,7 @@ static int xcsi2rxss_get_format(struct v4l2_subdev *sd,
 }
 
 static int xcsi2rxss_set_format(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/platform/xilinx/xilinx-tpg.c b/drivers/media/platform/xilinx/xilinx-tpg.c
index 7deec6e37edc19..bf39ddb9c2967b 100644
--- a/drivers/media/platform/xilinx/xilinx-tpg.c
+++ b/drivers/media/platform/xilinx/xilinx-tpg.c
@@ -278,6 +278,7 @@ static int xtpg_get_format(struct v4l2_subdev *subdev,
 }
 
 static int xtpg_set_format(struct v4l2_subdev *subdev,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/radio/si4713/radio-usb-si4713.c b/drivers/media/radio/si4713/radio-usb-si4713.c
index 6e6764143e21f3..62877980fc0472 100644
--- a/drivers/media/radio/si4713/radio-usb-si4713.c
+++ b/drivers/media/radio/si4713/radio-usb-si4713.c
@@ -192,7 +192,7 @@ static int si4713_send_startup_command(struct si4713_usb_device *radio)
 		}
 		if (time_is_before_jiffies(until_jiffies))
 			return -EIO;
-		msleep(3);
+		usleep_range(3000, 4000);
 	}
 
 	return retval;
@@ -339,7 +339,7 @@ static int si4713_i2c_read(struct si4713_usb_device *radio, char *data, int len)
 			data[0] = 0;
 			return 0;
 		}
-		msleep(3);
+		usleep_range(3000, 4000);
 	}
 }
 
diff --git a/drivers/media/radio/si4713/si4713.c b/drivers/media/radio/si4713/si4713.c
index 0c0354566b0a78..150d07f6b3196a 100644
--- a/drivers/media/radio/si4713/si4713.c
+++ b/drivers/media/radio/si4713/si4713.c
@@ -667,7 +667,7 @@ static int si4713_tx_tune_measure(struct si4713_device *sdev, u16 frequency,
 /*
  * si4713_tx_tune_status- Returns the status of the tx_tune_freq, tx_tune_mea or
  *			tx_tune_power commands. This command return the current
- *			frequency, output voltage in dBuV, the antenna tunning
+ *			frequency, output voltage in dBuV, the antenna tuning
  *			capacitance value and the received noise level. The
  *			command also clears the stcint interrupt bit when the
  *			first bit of its arguments is high.
diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c
index 2f7564f26445ce..14ab611e744577 100644
--- a/drivers/media/rc/bpf-lirc.c
+++ b/drivers/media/rc/bpf-lirc.c
@@ -148,12 +148,13 @@ static int lirc_bpf_attach(struct rc_dev *rcdev, struct bpf_prog *prog)
 	if (ret)
 		return ret;
 
-	raw = rcdev->raw;
-	if (!raw) {
+	if (!rcdev->registered) {
 		ret = -ENODEV;
 		goto unlock;
 	}
 
+	raw = rcdev->raw;
+
 	old_array = lirc_rcu_dereference(raw->progs);
 	if (old_array && bpf_prog_array_length(old_array) >= BPF_MAX_PROGS) {
 		ret = -E2BIG;
@@ -186,12 +187,13 @@ static int lirc_bpf_detach(struct rc_dev *rcdev, struct bpf_prog *prog)
 	if (ret)
 		return ret;
 
-	raw = rcdev->raw;
-	if (!raw) {
+	if (!rcdev->registered) {
 		ret = -ENODEV;
 		goto unlock;
 	}
 
+	raw = rcdev->raw;
+
 	old_array = lirc_rcu_dereference(raw->progs);
 	ret = bpf_prog_array_copy(old_array, prog, NULL, 0, &new_array);
 	/*
@@ -235,7 +237,8 @@ void lirc_bpf_free(struct rc_dev *rcdev)
 	struct bpf_prog_array_item *item;
 	struct bpf_prog_array *array;
 
-	array = lirc_rcu_dereference(rcdev->raw->progs);
+	array = rcu_replace_pointer(rcdev->raw->progs, NULL,
+				    lockdep_is_held(&ir_raw_handler_lock));
 	if (!array)
 		return;
 
@@ -316,6 +319,11 @@ int lirc_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
 	if (ret)
 		goto put;
 
+	if (!rcdev->registered) {
+		ret = -ENODEV;
+		goto unlock;
+	}
+
 	progs = lirc_rcu_dereference(rcdev->raw->progs);
 	cnt = progs ? bpf_prog_array_length(progs) : 0;
 
diff --git a/drivers/media/rc/ene_ir.c b/drivers/media/rc/ene_ir.c
index 6f7dccc965e7fd..f98d76277b62af 100644
--- a/drivers/media/rc/ene_ir.c
+++ b/drivers/media/rc/ene_ir.c
@@ -1103,15 +1103,15 @@ static void ene_remove(struct pnp_dev *pnp_dev)
 	unsigned long flags;
 
 	rc_unregister_device(dev->rdev);
-	timer_delete_sync(&dev->tx_sim_timer);
 	spin_lock_irqsave(&dev->hw_lock, flags);
 	ene_rx_disable(dev);
 	ene_rx_restore_hw_buffer(dev);
 	spin_unlock_irqrestore(&dev->hw_lock, flags);
 
-	rc_free_device(dev->rdev);
 	free_irq(dev->irq, dev);
+	timer_delete_sync(&dev->tx_sim_timer);
 	release_region(dev->hw_io, ENE_IO_SIZE);
+	rc_free_device(dev->rdev);
 	kfree(dev);
 }
 
diff --git a/drivers/media/rc/fintek-cir.c b/drivers/media/rc/fintek-cir.c
index c196ee923ecd34..d4368181fef662 100644
--- a/drivers/media/rc/fintek-cir.c
+++ b/drivers/media/rc/fintek-cir.c
@@ -140,17 +140,6 @@ static int fintek_hw_detect(struct fintek_dev *fintek)
 	ir_class = fintek_cir_reg_read(fintek, CIR_CR_CLASS);
 	fit_dbg("ir_class reg: 0x%02x", ir_class);
 
-	switch (ir_class) {
-	case CLASS_RX_2TX:
-	case CLASS_RX_1TX:
-		fintek->hw_tx_capable = true;
-		break;
-	case CLASS_RX_ONLY:
-	default:
-		fintek->hw_tx_capable = false;
-		break;
-	}
-
 	chip_major = fintek_cr_read(fintek, GCR_CHIP_ID_HI);
 	chip_minor = fintek_cr_read(fintek, GCR_CHIP_ID_LO);
 	chip  = chip_major << 8 | chip_minor;
@@ -169,7 +158,6 @@ static int fintek_hw_detect(struct fintek_dev *fintek)
 	spin_lock_irqsave(&fintek->fintek_lock, flags);
 	fintek->chip_major  = chip_major;
 	fintek->chip_minor  = chip_minor;
-	fintek->chip_vendor = vendor;
 
 	/*
 	 * Newer reviews of this chipset uses port 8 instead of 5
@@ -496,7 +484,6 @@ static int fintek_probe(struct pnp_dev *pdev, const struct pnp_device_id *dev_id
 	spin_lock_init(&fintek->fintek_lock);
 
 	pnp_set_drvdata(pdev, fintek);
-	fintek->pdev = pdev;
 
 	ret = fintek_hw_detect(fintek);
 	if (ret)
diff --git a/drivers/media/rc/fintek-cir.h b/drivers/media/rc/fintek-cir.h
index 20696359e9bad8..9ade9d7e7bfbd4 100644
--- a/drivers/media/rc/fintek-cir.h
+++ b/drivers/media/rc/fintek-cir.h
@@ -40,11 +40,9 @@ static int debug;
 			KBUILD_MODNAME ": " text "\n" , ## __VA_ARGS__)
 
 
-#define TX_BUF_LEN 256
 #define RX_BUF_LEN 32
 
 struct fintek_dev {
-	struct pnp_dev *pdev;
 	struct rc_dev *rdev;
 
 	spinlock_t fintek_lock;
@@ -53,14 +51,6 @@ struct fintek_dev {
 	u8 buf[RX_BUF_LEN];
 	unsigned int pkts;
 
-	struct {
-		spinlock_t lock;
-		u8 buf[TX_BUF_LEN];
-		unsigned int buf_count;
-		unsigned int cur_buf_num;
-		wait_queue_head_t queue;
-	} tx;
-
 	/* Config register index/data port pair */
 	u32 cr_ip;
 	u32 cr_dp;
@@ -73,17 +63,8 @@ struct fintek_dev {
 	/* hardware id */
 	u8 chip_major;
 	u8 chip_minor;
-	u16 chip_vendor;
 	u8 logical_dev_cir;
 
-	/* hardware features */
-	bool hw_learning_capable;
-	bool hw_tx_capable;
-
-	/* rx settings */
-	bool learning_enabled;
-	bool carrier_detect_enabled;
-
 	enum {
 		CMD_HEADER = 0,
 		SUBCMD,
@@ -92,9 +73,6 @@ struct fintek_dev {
 	} parser_state;
 
 	u8 cmd, rem;
-
-	/* carrier period = 1 / frequency */
-	u32 carrier;
 };
 
 /* buffer packet constants, largely identical to mceusb.c */
diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c
index 049a73b5f88296..2668461ed409d3 100644
--- a/drivers/media/rc/imon.c
+++ b/drivers/media/rc/imon.c
@@ -96,7 +96,6 @@ struct imon_context {
 	bool dev_present_intf1;		/* USB device presence, interface 1 */
 
 	struct mutex lock;		/* to lock this object */
-	wait_queue_head_t remove_ok;	/* For unexpected USB disconnects */
 
 	struct usb_endpoint_descriptor *rx_endpoint_intf0;
 	struct usb_endpoint_descriptor *rx_endpoint_intf1;
@@ -378,7 +377,7 @@ static const struct usb_device_id imon_usb_id_table[] = {
 	 * SoundGraph iMON PAD (IR & LCD)
 	 * SoundGraph iMON Knob (IR only)
 	 */
-	{ USB_DEVICE(0x15c2, 0xffdc),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0xffdc, 0),
 	  .driver_info = (unsigned long)&imon_default_table },
 
 	/*
@@ -387,61 +386,61 @@ static const struct usb_device_id imon_usb_id_table[] = {
 	 * Need user input to fill in details on unknown devices.
 	 */
 	/* SoundGraph iMON OEM Touch LCD (IR & 7" VGA LCD) */
-	{ USB_DEVICE(0x15c2, 0x0034),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0034, 0),
 	  .driver_info = (unsigned long)&imon_DH102 },
 	/* SoundGraph iMON OEM Touch LCD (IR & 4.3" VGA LCD) */
-	{ USB_DEVICE(0x15c2, 0x0035),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0035, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* SoundGraph iMON OEM VFD (IR & VFD) */
-	{ USB_DEVICE(0x15c2, 0x0036),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0036, 0),
 	  .driver_info = (unsigned long)&imon_OEM_VFD },
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x0037),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0037, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* SoundGraph iMON OEM LCD (IR & LCD) */
-	{ USB_DEVICE(0x15c2, 0x0038),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0038, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* SoundGraph iMON UltraBay (IR & LCD) */
-	{ USB_DEVICE(0x15c2, 0x0039),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0039, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x003a),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x003a, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x003b),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x003b, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* SoundGraph iMON OEM Inside (IR only) */
-	{ USB_DEVICE(0x15c2, 0x003c),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x003c, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x003d),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x003d, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x003e),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x003e, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x003f),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x003f, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x0040),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0040, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* SoundGraph iMON MINI (IR only) */
-	{ USB_DEVICE(0x15c2, 0x0041),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0041, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* Antec Veris Multimedia Station EZ External (IR only) */
-	{ USB_DEVICE(0x15c2, 0x0042),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0042, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* Antec Veris Multimedia Station Basic Internal (IR only) */
-	{ USB_DEVICE(0x15c2, 0x0043),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0043, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* Antec Veris Multimedia Station Elite (IR & VFD) */
-	{ USB_DEVICE(0x15c2, 0x0044),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0044, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* Antec Veris Multimedia Station Premiere (IR & LCD) */
-	{ USB_DEVICE(0x15c2, 0x0045),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0045, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	/* device specifics unknown */
-	{ USB_DEVICE(0x15c2, 0x0046),
+	{ USB_DEVICE_INTERFACE_NUMBER(0x15c2, 0x0046, 0),
 	  .driver_info = (unsigned long)&imon_default_table},
 	{}
 };
@@ -805,19 +804,16 @@ static ssize_t associate_remote_show(struct device *d,
 				     char *buf)
 {
 	struct imon_context *ictx = dev_get_drvdata(d);
+	int len;
 
 	if (!ictx)
 		return -ENODEV;
 
 	mutex_lock(&ictx->lock);
-	if (ictx->rf_isassociating)
-		strscpy(buf, "associating\n", PAGE_SIZE);
-	else
-		strscpy(buf, "closed\n", PAGE_SIZE);
-
+	len = sysfs_emit(buf, (ictx->rf_isassociating ? "associating\n" : "closed\n"));
 	dev_info(d, "Visit https://www.lirc.org/html/imon-24g.html for instructions on how to associate your iMON 2.4G DT/LT remote\n");
 	mutex_unlock(&ictx->lock);
-	return strlen(buf);
+	return len;
 }
 
 static ssize_t associate_remote_store(struct device *d,
@@ -2331,15 +2327,20 @@ static struct imon_context *imon_init_intf1(struct usb_interface *intf,
 	struct usb_host_interface *iface_desc;
 	int ret = -ENOMEM;
 
+	ret = usb_driver_claim_interface(&imon_driver, intf, ictx);
+	if (ret) {
+		dev_err(ictx->dev, "failed to claim second interface (%d)\n", ret);
+		return NULL;
+	}
+
 	rx_urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!rx_urb)
 		goto rx_urb_alloc_failed;
 
 	mutex_lock(&ictx->lock);
 
-	if (ictx->display_type == IMON_DISPLAY_TYPE_VGA) {
+	if (ictx->display_type == IMON_DISPLAY_TYPE_VGA)
 		timer_setup(&ictx->ttimer, imon_touch_display_timeout, 0);
-	}
 
 	ictx->usbdev_intf1 = interface_to_usbdev(intf);
 	ictx->rx_urb_intf1 = rx_urb;
@@ -2366,13 +2367,14 @@ static struct imon_context *imon_init_intf1(struct usb_interface *intf,
 	ret = usb_submit_urb(ictx->rx_urb_intf1, GFP_KERNEL);
 
 	if (ret) {
-		pr_err("usb_submit_urb failed for intf1 (%d)\n", ret);
+		dev_err(ictx->dev, "usb_submit_urb failed for intf1 (%d)\n", ret);
 		goto urb_submit_failed;
 	}
 
 	ictx->dev_present_intf1 = true;
 
 	mutex_unlock(&ictx->lock);
+
 	return ictx;
 
 urb_submit_failed:
@@ -2380,11 +2382,14 @@ static struct imon_context *imon_init_intf1(struct usb_interface *intf,
 		input_unregister_device(ictx->touch);
 touch_setup_failed:
 find_endpoint_failed:
+	if (ictx->display_type == IMON_DISPLAY_TYPE_VGA)
+		timer_delete_sync(&ictx->ttimer);
 	ictx->usbdev_intf1 = NULL;
 	mutex_unlock(&ictx->lock);
 	usb_free_urb(rx_urb);
 	ictx->rx_urb_intf1 = NULL;
 rx_urb_alloc_failed:
+	usb_driver_release_interface(&imon_driver, intf);
 	dev_err(ictx->dev, "unable to initialize intf1, err %d\n", ret);
 
 	return NULL;
@@ -2419,90 +2424,50 @@ static void imon_init_display(struct imon_context *ictx,
 static int imon_probe(struct usb_interface *interface,
 		      const struct usb_device_id *id)
 {
-	struct usb_device *usbdev = NULL;
-	struct usb_host_interface *iface_desc = NULL;
-	struct usb_interface *first_if;
+	struct usb_device *usbdev;
+	struct usb_interface *second_if;
 	struct device *dev = &interface->dev;
-	int ifnum, sysfs_err;
-	int ret = 0;
-	struct imon_context *ictx = NULL;
+	int sysfs_err;
+	struct imon_context *ictx;
 	u16 vendor, product;
 
-	usbdev     = interface_to_usbdev(interface);
-	iface_desc = interface->cur_altsetting;
-	ifnum      = iface_desc->desc.bInterfaceNumber;
-	vendor     = le16_to_cpu(usbdev->descriptor.idVendor);
-	product    = le16_to_cpu(usbdev->descriptor.idProduct);
-
-	dev_dbg(dev, "%s: found iMON device (%04x:%04x, intf%d)\n",
-		__func__, vendor, product, ifnum);
+	if (interface->cur_altsetting->desc.bInterfaceNumber != 0)
+		return -ENODEV;
 
-	first_if = usb_ifnum_to_if(usbdev, 0);
-	if (!first_if) {
-		ret = -ENODEV;
-		goto fail;
-	}
+	usbdev  = interface_to_usbdev(interface);
+	vendor  = le16_to_cpu(usbdev->descriptor.idVendor);
+	product = le16_to_cpu(usbdev->descriptor.idProduct);
 
-	if (first_if->dev.driver != interface->dev.driver) {
-		dev_err(&interface->dev, "inconsistent driver matching\n");
-		ret = -EINVAL;
-		goto fail;
-	}
+	dev_dbg(dev, "found iMON device (%04x:%04x)\n", vendor, product);
 
-	if (ifnum == 0) {
-		ictx = imon_init_intf0(interface, id);
-		if (!ictx) {
-			pr_err("failed to initialize context!\n");
-			ret = -ENODEV;
-			goto fail;
-		}
-		refcount_set(&ictx->users, 1);
+	ictx = imon_init_intf0(interface, id);
+	if (!ictx)
+		return -ENODEV;
 
-	} else {
-		/* this is the secondary interface on the device */
-		struct imon_context *first_if_ctx = usb_get_intfdata(first_if);
+	refcount_set(&ictx->users, 1);
 
-		/* fail early if first intf failed to register */
-		if (!first_if_ctx) {
-			ret = -ENODEV;
-			goto fail;
-		}
+	usb_set_intfdata(interface, ictx);
 
-		ictx = imon_init_intf1(interface, first_if_ctx);
-		if (!ictx) {
-			pr_err("failed to attach to context!\n");
-			ret = -ENODEV;
-			goto fail;
-		}
+	/* Newer devices export a second interface for display/touchscreen */
+	second_if = usb_ifnum_to_if(usbdev, 1);
+	if (second_if && imon_init_intf1(second_if, ictx))
 		refcount_inc(&ictx->users);
 
+	if (product == 0xffdc && ictx->rf_device) {
+		sysfs_err = sysfs_create_group(&interface->dev.kobj,
+					       &imon_rf_attr_group);
+		if (sysfs_err)
+			pr_err("Could not create RF sysfs entries(%d)\n",
+			       sysfs_err);
 	}
 
-	usb_set_intfdata(interface, ictx);
+	if (ictx->display_supported)
+		imon_init_display(ictx, interface);
 
-	if (ifnum == 0) {
-		if (product == 0xffdc && ictx->rf_device) {
-			sysfs_err = sysfs_create_group(&interface->dev.kobj,
-						       &imon_rf_attr_group);
-			if (sysfs_err)
-				pr_err("Could not create RF sysfs entries(%d)\n",
-				       sysfs_err);
-		}
-
-		if (ictx->display_supported)
-			imon_init_display(ictx, interface);
-	}
-
-	dev_info(dev, "iMON device (%04x:%04x, intf%d) on usb<%d:%d> initialized\n",
-		 vendor, product, ifnum,
-		 usbdev->bus->busnum, usbdev->devnum);
+	dev_info(dev, "iMON device (%04x:%04x) on usb<%d:%d> initialized\n",
+		 vendor, product, usbdev->bus->busnum, usbdev->devnum);
 
 	return 0;
-
-fail:
-	dev_err(dev, "unable to register, err %d\n", ret);
-
-	return ret;
 }
 
 /*
@@ -2510,11 +2475,15 @@ static int imon_probe(struct usb_interface *interface,
  */
 static void imon_disconnect(struct usb_interface *interface)
 {
+	struct usb_device *usbdev = interface_to_usbdev(interface);
+	struct usb_interface *other_if;
 	struct imon_context *ictx;
 	struct device *dev;
 	int ifnum;
 
 	ictx = usb_get_intfdata(interface);
+	if (!ictx)
+		return;
 
 	mutex_lock(&ictx->lock);
 	ictx->disconnected = true;
@@ -2550,6 +2519,7 @@ static void imon_disconnect(struct usb_interface *interface)
 			else if (ictx->display_type == IMON_DISPLAY_TYPE_VFD)
 				usb_deregister_dev(interface, &imon_vfd_class);
 		}
+		other_if = usb_ifnum_to_if(usbdev, 1);
 	} else {
 		ictx->dev_present_intf1 = false;
 		usb_kill_urb(ictx->rx_urb_intf1);
@@ -2557,13 +2527,16 @@ static void imon_disconnect(struct usb_interface *interface)
 			timer_delete_sync(&ictx->ttimer);
 			input_unregister_device(ictx->touch);
 		}
+		other_if = usb_ifnum_to_if(usbdev, 0);
 	}
 
+	if (other_if)
+		usb_driver_release_interface(&imon_driver, other_if);
+
 	if (refcount_dec_and_test(&ictx->users))
 		free_imon_context(ictx);
 
-	dev_dbg(dev, "%s: iMON device (intf%d) disconnected\n",
-		__func__, ifnum);
+	dev_dbg(dev, "iMON device (intf%d) disconnected\n", ifnum);
 }
 
 static int imon_suspend(struct usb_interface *intf, pm_message_t message)
diff --git a/drivers/media/rc/ir-hix5hd2.c b/drivers/media/rc/ir-hix5hd2.c
index 1b061e4a3dcfa9..aa3de4d57a585d 100644
--- a/drivers/media/rc/ir-hix5hd2.c
+++ b/drivers/media/rc/ir-hix5hd2.c
@@ -316,6 +316,9 @@ static int hix5hd2_ir_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto clkerr;
 
+	priv->rdev = rdev;
+	priv->dev = dev;
+
 	if (devm_request_irq(dev, priv->irq, hix5hd2_ir_rx_interrupt,
 			     0, pdev->name, priv) < 0) {
 		dev_err(dev, "IRQ %d register failed\n", priv->irq);
@@ -323,8 +326,6 @@ static int hix5hd2_ir_probe(struct platform_device *pdev)
 		goto regerr;
 	}
 
-	priv->rdev = rdev;
-	priv->dev = dev;
 	platform_set_drvdata(pdev, priv);
 
 	return ret;
diff --git a/drivers/media/rc/ir-mce_kbd-decoder.c b/drivers/media/rc/ir-mce_kbd-decoder.c
index bb2d7c37c26369..d1c9263511b092 100644
--- a/drivers/media/rc/ir-mce_kbd-decoder.c
+++ b/drivers/media/rc/ir-mce_kbd-decoder.c
@@ -219,6 +219,7 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 	struct mce_kbd_dec *data = &dev->raw->mce_kbd;
 	u32 scancode;
 	unsigned long delay;
+	unsigned long flags;
 	struct lirc_scancode lsc = {};
 
 	if (!is_timing_event(ev)) {
@@ -319,7 +320,7 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 			scancode = data->body & 0xffffff;
 			dev_dbg(&dev->dev, "keyboard data 0x%08x\n",
 				data->body);
-			spin_lock(&data->keylock);
+			spin_lock_irqsave(&data->keylock, flags);
 			if (scancode) {
 				delay = usecs_to_jiffies(dev->timeout) +
 					msecs_to_jiffies(100);
@@ -329,7 +330,7 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 			}
 			/* Pass data to keyboard buffer parser */
 			ir_mce_kbd_process_keyboard_data(dev, scancode);
-			spin_unlock(&data->keylock);
+			spin_unlock_irqrestore(&data->keylock, flags);
 			lsc.rc_proto = RC_PROTO_MCIR2_KBD;
 			break;
 		case MCIR2_MOUSE_NBITS:
diff --git a/drivers/media/rc/ir_toy.c b/drivers/media/rc/ir_toy.c
index ee645882dd5a6d..58b23d08a89726 100644
--- a/drivers/media/rc/ir_toy.c
+++ b/drivers/media/rc/ir_toy.c
@@ -55,7 +55,6 @@ enum state {
 
 struct irtoy {
 	struct device *dev;
-	struct usb_device *usbdev;
 
 	struct rc_dev *rc;
 	struct urb *urb_in, *urb_out;
@@ -373,7 +372,7 @@ static int irtoy_tx_carrier(struct rc_dev *rc, uint32_t carrier)
 	u8 buf[3];
 	int err;
 
-	if (carrier < 11800)
+	if (carrier < 11800 || carrier > 3000000)
 		return -EINVAL;
 
 	buf[0] = 0x06;
@@ -441,7 +440,6 @@ static int irtoy_probe(struct usb_interface *intf,
 			  irtoy_out_callback, irtoy);
 
 	irtoy->dev = &intf->dev;
-	irtoy->usbdev = usbdev;
 	irtoy->rc = rc;
 	irtoy->urb_out = urb;
 	irtoy->pulse = true;
diff --git a/drivers/media/rc/ite-cir.c b/drivers/media/rc/ite-cir.c
index 1fbafcd8219e5a..d0e98095d1b643 100644
--- a/drivers/media/rc/ite-cir.c
+++ b/drivers/media/rc/ite-cir.c
@@ -1359,7 +1359,6 @@ static int ite_probe(struct pnp_dev *pdev, const struct pnp_device_id
 
 	/* set driver data into the pnp device */
 	pnp_set_drvdata(pdev, itdev);
-	itdev->pdev = pdev;
 
 	/* initialize waitqueues for transmission */
 	init_waitqueue_head(&itdev->tx_queue);
diff --git a/drivers/media/rc/ite-cir.h b/drivers/media/rc/ite-cir.h
index 4b4294d77555d4..a8382aa9dc3548 100644
--- a/drivers/media/rc/ite-cir.h
+++ b/drivers/media/rc/ite-cir.h
@@ -76,7 +76,6 @@ struct ite_dev_params {
 
 /* ITE CIR device structure */
 struct ite_dev {
-	struct pnp_dev *pdev;
 	struct rc_dev *rdev;
 
 	/* sync data */
diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c
index 183f1939b9416d..f7b8f1f66d9bca 100644
--- a/drivers/media/rc/lirc_dev.c
+++ b/drivers/media/rc/lirc_dev.c
@@ -492,6 +492,8 @@ static long lirc_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 			ret = -ENOTTY;
 		else if (val <= 0)
 			ret = -EINVAL;
+		else if (fh->carrier_low && fh->carrier_low > val)
+			ret = -EINVAL;
 		else
 			ret = dev->s_rx_carrier_range(dev, fh->carrier_low,
 						      val);
@@ -775,6 +777,9 @@ void lirc_unregister(struct rc_dev *dev)
 	unsigned long flags;
 	struct lirc_fh *fh;
 
+	if (!device_is_registered(&dev->lirc_dev))
+		return;
+
 	dev_dbg(&dev->dev, "lirc_dev: driver %s unregistered from minor = %d\n",
 		dev->driver_name, MINOR(dev->lirc_dev.devt));
 
diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c
index 39ba7f6a2549ec..356a54f91d0540 100644
--- a/drivers/media/rc/mceusb.c
+++ b/drivers/media/rc/mceusb.c
@@ -484,7 +484,6 @@ struct mceusb_dev {
 	u8 cmd, rem;		/* Remaining IR data bytes in packet */
 
 	struct {
-		u32 connected:1;
 		u32 tx_mask_normal:1;
 		u32 microsoft_gen1:1;
 		u32 no_tx:1;
diff --git a/drivers/media/rc/meson-ir-tx.c b/drivers/media/rc/meson-ir-tx.c
index fded2c256f2a62..a0d07da09f67af 100644
--- a/drivers/media/rc/meson-ir-tx.c
+++ b/drivers/media/rc/meson-ir-tx.c
@@ -219,6 +219,9 @@ static int meson_irtx_set_carrier(struct rc_dev *rc, u32 carrier)
 	if (carrier == 0)
 		return -EINVAL;
 
+	if (!DIV_ROUND_CLOSEST(USEC_PER_SEC, carrier))
+		return -EINVAL;
+
 	ir->carrier = carrier;
 	meson_irtx_set_mod(ir);
 
@@ -288,11 +291,12 @@ static int meson_irtx_mod_clock_probe(struct meson_irtx *ir,
 	if (!np)
 		return -ENODEV;
 
-	clock = devm_clk_get(ir->dev, "xtal");
-	if (IS_ERR(clock) || clk_prepare_enable(clock))
-		return -ENODEV;
-
 	*clk_nr = IRB_MOD_XTAL3_CLK;
+
+	clock = devm_clk_get_enabled(ir->dev, "xtal");
+	if (IS_ERR(clock))
+		return PTR_ERR(clock);
+
 	ir->clk_rate = clk_get_rate(clock) / 3;
 
 	if (ir->clk_rate < IRB_MOD_1US_CLK_RATE) {
@@ -324,7 +328,7 @@ static int meson_irtx_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
-		return -ENODEV;
+		return irq;
 
 	ir->dev = dev;
 	ir->carrier = MIRTX_DEFAULT_CARRIER;
@@ -345,7 +349,7 @@ static int meson_irtx_probe(struct platform_device *pdev)
 	if (ret)
 		return dev_err_probe(dev, ret, "irq request failed\n");
 
-	rc = rc_allocate_device(RC_DRIVER_IR_RAW_TX);
+	rc = devm_rc_allocate_device(dev, RC_DRIVER_IR_RAW_TX);
 	if (!rc)
 		return -ENOMEM;
 
@@ -358,10 +362,8 @@ static int meson_irtx_probe(struct platform_device *pdev)
 	rc->s_tx_duty_cycle = meson_irtx_set_duty_cycle;
 
 	ret = devm_rc_register_device(dev, rc);
-	if (ret < 0) {
-		rc_free_device(rc);
+	if (ret < 0)
 		return dev_err_probe(dev, ret, "rc_dev registration failed\n");
-	}
 
 	return 0;
 }
diff --git a/drivers/media/rc/nuvoton-cir.h b/drivers/media/rc/nuvoton-cir.h
index ed7d93beaa280a..e66e3cfedeec4a 100644
--- a/drivers/media/rc/nuvoton-cir.h
+++ b/drivers/media/rc/nuvoton-cir.h
@@ -77,9 +77,6 @@ struct nvt_dev {
 	/* hardware id */
 	u8 chip_major;
 	u8 chip_minor;
-
-	/* carrier period = 1 / frequency */
-	u32 carrier;
 };
 
 /* buffer packet constants */
diff --git a/drivers/media/rc/rc-ir-raw.c b/drivers/media/rc/rc-ir-raw.c
index ba24c2f22d39f9..75dee4a483d46d 100644
--- a/drivers/media/rc/rc-ir-raw.c
+++ b/drivers/media/rc/rc-ir-raw.c
@@ -10,7 +10,8 @@
 #include <linux/sched.h>
 #include "rc-core-priv.h"
 
-/* Used to keep track of IR raw clients, protected by ir_raw_handler_lock */
+/* Used to keep track of IR raw clients, protected by ir_raw_client_lock */
+static DEFINE_MUTEX(ir_raw_client_lock);
 static LIST_HEAD(ir_raw_client_list);
 
 /* Used to handle IR raw handler extensions */
@@ -71,9 +72,6 @@ static int ir_raw_event_thread(void *data)
  */
 int ir_raw_event_store(struct rc_dev *dev, struct ir_raw_event *ev)
 {
-	if (!dev->raw)
-		return -EINVAL;
-
 	dev_dbg(&dev->dev, "sample: (%05dus %s)\n",
 		ev->duration, TO_STR(ev->pulse));
 
@@ -102,9 +100,6 @@ int ir_raw_event_store_edge(struct rc_dev *dev, bool pulse)
 	ktime_t			now;
 	struct ir_raw_event	ev = {};
 
-	if (!dev->raw)
-		return -EINVAL;
-
 	now = ktime_get();
 	ev.duration = ktime_to_us(ktime_sub(now, dev->raw->last_event));
 	ev.pulse = !pulse;
@@ -129,9 +124,6 @@ int ir_raw_event_store_with_timeout(struct rc_dev *dev, struct ir_raw_event *ev)
 	ktime_t		now;
 	int		rc = 0;
 
-	if (!dev->raw)
-		return -EINVAL;
-
 	now = ktime_get();
 
 	spin_lock(&dev->raw->edge_spinlock);
@@ -166,9 +158,6 @@ EXPORT_SYMBOL_GPL(ir_raw_event_store_with_timeout);
  */
 int ir_raw_event_store_with_filter(struct rc_dev *dev, struct ir_raw_event *ev)
 {
-	if (!dev->raw)
-		return -EINVAL;
-
 	/* Ignore spaces in idle mode */
 	if (dev->idle && !ev->pulse)
 		return 0;
@@ -200,9 +189,6 @@ EXPORT_SYMBOL_GPL(ir_raw_event_store_with_filter);
  */
 void ir_raw_event_set_idle(struct rc_dev *dev, bool idle)
 {
-	if (!dev->raw)
-		return;
-
 	dev_dbg(&dev->dev, "%s idle mode\n", idle ? "enter" : "leave");
 
 	if (idle) {
@@ -226,7 +212,7 @@ EXPORT_SYMBOL_GPL(ir_raw_event_set_idle);
  */
 void ir_raw_event_handle(struct rc_dev *dev)
 {
-	if (!dev->raw || !dev->raw->thread)
+	if (!dev->raw->thread)
 		return;
 
 	wake_up_process(dev->raw->thread);
@@ -288,13 +274,6 @@ static int change_protocol(struct rc_dev *dev, u64 *rc_proto)
 	return 0;
 }
 
-static void ir_raw_disable_protocols(struct rc_dev *dev, u64 protocols)
-{
-	mutex_lock(&dev->lock);
-	dev->enabled_protocols &= ~protocols;
-	mutex_unlock(&dev->lock);
-}
-
 /**
  * ir_raw_gen_manchester() - Encode data with Manchester (bi-phase) modulation.
  * @ev:		Pointer to pointer to next free event. *@ev is incremented for
@@ -612,9 +591,6 @@ EXPORT_SYMBOL(ir_raw_encode_carrier);
  */
 int ir_raw_event_prepare(struct rc_dev *dev)
 {
-	if (!dev)
-		return -EINVAL;
-
 	dev->raw = kzalloc_obj(*dev->raw);
 	if (!dev->raw)
 		return -ENOMEM;
@@ -633,50 +609,63 @@ int ir_raw_event_register(struct rc_dev *dev)
 {
 	struct task_struct *thread;
 
+	/* Holding dev->lock could result in a dead-lock */
+	lockdep_assert_not_held(&dev->lock);
+
 	thread = kthread_run(ir_raw_event_thread, dev->raw, "rc%u", dev->minor);
 	if (IS_ERR(thread))
 		return PTR_ERR(thread);
 
 	dev->raw->thread = thread;
 
-	mutex_lock(&ir_raw_handler_lock);
+	mutex_lock(&ir_raw_client_lock);
 	list_add_tail(&dev->raw->list, &ir_raw_client_list);
-	mutex_unlock(&ir_raw_handler_lock);
+	mutex_unlock(&ir_raw_client_lock);
 
 	return 0;
 }
 
 void ir_raw_event_free(struct rc_dev *dev)
 {
-	kfree(dev->raw);
-	dev->raw = NULL;
+	if (dev->raw) {
+		timer_delete_sync(&dev->raw->edge_handle);
+		mutex_lock(&ir_raw_handler_lock);
+		if (dev->raw->thread)
+			put_task_struct(dev->raw->thread);
+		lirc_bpf_free(dev);
+		mutex_unlock(&ir_raw_handler_lock);
+		kfree(dev->raw);
+		dev->raw = NULL;
+	}
 }
 
 void ir_raw_event_unregister(struct rc_dev *dev)
 {
 	struct ir_raw_handler *handler;
 
-	if (!dev || !dev->raw)
-		return;
-
+	/*
+	 * After ir_raw_event_unregister() is called, an sync
+	 * call to ir_raw_event_handle() can still arrive. This function
+	 * may call wake_up_process(dev->raw->thread). Ensure this memory
+	 * is not freed by kthread_stop().
+	 */
+	get_task_struct(dev->raw->thread);
 	kthread_stop(dev->raw->thread);
 	timer_delete_sync(&dev->raw->edge_handle);
 
-	mutex_lock(&ir_raw_handler_lock);
+	mutex_lock(&ir_raw_client_lock);
 	list_del(&dev->raw->list);
+
+	mutex_lock(&ir_raw_handler_lock);
 	list_for_each_entry(handler, &ir_raw_handler_list, list)
 		if (handler->raw_unregister &&
 		    (handler->protocols & dev->enabled_protocols))
 			handler->raw_unregister(dev);
 
 	lirc_bpf_free(dev);
-
-	/*
-	 * A user can be calling bpf(BPF_PROG_{QUERY|ATTACH|DETACH}), so
-	 * ensure that the raw member is null on unlock; this is how
-	 * "device gone" is checked.
-	 */
 	mutex_unlock(&ir_raw_handler_lock);
+
+	mutex_unlock(&ir_raw_client_lock);
 }
 
 /*
@@ -699,15 +688,24 @@ void ir_raw_handler_unregister(struct ir_raw_handler *ir_raw_handler)
 	struct ir_raw_event_ctrl *raw;
 	u64 protocols = ir_raw_handler->protocols;
 
+	mutex_lock(&ir_raw_client_lock);
+
 	mutex_lock(&ir_raw_handler_lock);
 	list_del(&ir_raw_handler->list);
+	atomic64_andnot(protocols, &available_protocols);
+	mutex_unlock(&ir_raw_handler_lock);
+
 	list_for_each_entry(raw, &ir_raw_client_list, list) {
+		mutex_lock(&raw->dev->lock);
+		mutex_lock(&ir_raw_handler_lock);
 		if (ir_raw_handler->raw_unregister &&
 		    (raw->dev->enabled_protocols & protocols))
 			ir_raw_handler->raw_unregister(raw->dev);
-		ir_raw_disable_protocols(raw->dev, protocols);
+		raw->dev->enabled_protocols &= ~protocols;
+		mutex_unlock(&ir_raw_handler_lock);
+		mutex_unlock(&raw->dev->lock);
 	}
-	atomic64_andnot(protocols, &available_protocols);
-	mutex_unlock(&ir_raw_handler_lock);
+
+	mutex_unlock(&ir_raw_client_lock);
 }
 EXPORT_SYMBOL(ir_raw_handler_unregister);
diff --git a/drivers/media/rc/rc-loopback.c b/drivers/media/rc/rc-loopback.c
index 53d0540717b360..9d56e77c9351fc 100644
--- a/drivers/media/rc/rc-loopback.c
+++ b/drivers/media/rc/rc-loopback.c
@@ -74,11 +74,6 @@ static int loop_set_rx_carrier_range(struct rc_dev *dev, u32 min, u32 max)
 {
 	struct loopback_dev *lodev = dev->priv;
 
-	if (min < 1 || min > max) {
-		dev_dbg(&dev->dev, "invalid rx carrier range %u to %u\n", min, max);
-		return -EINVAL;
-	}
-
 	dev_dbg(&dev->dev, "setting rx carrier range %u to %u\n", min, max);
 	lodev->rxcarriermin = min;
 	lodev->rxcarriermax = max;
diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c
index dda3479ea3addf..a867bf9e84166c 100644
--- a/drivers/media/rc/rc-main.c
+++ b/drivers/media/rc/rc-main.c
@@ -17,9 +17,8 @@
 #include <linux/module.h>
 #include "rc-core-priv.h"
 
-/* Sizes are in bytes, 256 bytes allows for 32 entries on x64 */
-#define IR_TAB_MIN_SIZE	256
-#define IR_TAB_MAX_SIZE	8192
+#define IR_TAB_MIN_ENTRIES	32
+#define IR_TAB_MAX_ENTRIES	1024
 
 static const struct {
 	const char *name;
@@ -105,7 +104,6 @@ static struct rc_map_list *seek_rc_map(const char *name)
 
 struct rc_map *rc_map_get(const char *name)
 {
-
 	struct rc_map_list *map;
 
 	map = seek_rc_map(name);
@@ -202,7 +200,7 @@ static int scancode_to_u64(const struct input_keymap_entry *ke, u64 *scancode)
  * ir_create_table() - initializes a scancode table
  * @dev:	the rc_dev device
  * @rc_map:	the rc_map to initialize
- * @name:	name to assign to the table
+ * @map_name:	name to assign to the table
  * @rc_proto:	ir type to assign to the new table
  * @size:	initial size of the table
  *
@@ -212,23 +210,33 @@ static int scancode_to_u64(const struct input_keymap_entry *ke, u64 *scancode)
  * return:	zero on success or a negative error code
  */
 static int ir_create_table(struct rc_dev *dev, struct rc_map *rc_map,
-			   const char *name, u64 rc_proto, size_t size)
+			   const char *map_name, u64 rc_proto, size_t size)
 {
-	rc_map->name = kstrdup(name, GFP_KERNEL);
-	if (!rc_map->name)
+	struct rc_map_table *scan;
+	unsigned int alloc;
+	char *name;
+
+	name = kstrdup(map_name, GFP_KERNEL);
+	if (!name)
 		return -ENOMEM;
-	rc_map->rc_proto = rc_proto;
-	rc_map->alloc = roundup_pow_of_two(size * sizeof(struct rc_map_table));
-	rc_map->size = rc_map->alloc / sizeof(struct rc_map_table);
-	rc_map->scan = kmalloc(rc_map->alloc, GFP_KERNEL);
-	if (!rc_map->scan) {
-		kfree(rc_map->name);
-		rc_map->name = NULL;
+
+	alloc = roundup_pow_of_two(size);
+	scan = kmalloc_objs(struct rc_map_table, alloc, GFP_KERNEL);
+	if (!scan) {
+		kfree(name);
 		return -ENOMEM;
 	}
 
-	dev_dbg(&dev->dev, "Allocated space for %u keycode entries (%u bytes)\n",
-		rc_map->size, rc_map->alloc);
+	scoped_guard(spinlock_irqsave, &rc_map->lock) {
+		rc_map->name = name;
+		rc_map->scan = scan;
+		rc_map->rc_proto = rc_proto;
+		rc_map->len = 0;
+		rc_map->size = alloc;
+	}
+
+	dev_dbg(&dev->dev, "Allocated space for %u keycode entries (%zu bytes)\n",
+		alloc, alloc * sizeof(struct rc_map_table));
 	return 0;
 }
 
@@ -236,16 +244,26 @@ static int ir_create_table(struct rc_dev *dev, struct rc_map *rc_map,
  * ir_free_table() - frees memory allocated by a scancode table
  * @rc_map:	the table whose mappings need to be freed
  *
- * This routine will free memory alloctaed for key mappings used by given
+ * This routine will free memory allocated for key mappings used by given
  * scancode table.
  */
 static void ir_free_table(struct rc_map *rc_map)
 {
-	rc_map->size = 0;
-	kfree(rc_map->name);
-	rc_map->name = NULL;
-	kfree(rc_map->scan);
-	rc_map->scan = NULL;
+	struct rc_map_table *scan;
+	const char *name;
+
+	scoped_guard(spinlock_irqsave, &rc_map->lock) {
+		name = rc_map->name;
+		scan = rc_map->scan;
+
+		rc_map->size = 0;
+		rc_map->len = 0;
+		rc_map->name = NULL;
+		rc_map->scan = NULL;
+	}
+
+	kfree(name);
+	kfree(scan);
 }
 
 /**
@@ -262,38 +280,38 @@ static void ir_free_table(struct rc_map *rc_map)
 static int ir_resize_table(struct rc_dev *dev, struct rc_map *rc_map,
 			   gfp_t gfp_flags)
 {
-	unsigned int oldalloc = rc_map->alloc;
-	unsigned int newalloc = oldalloc;
-	struct rc_map_table *oldscan = rc_map->scan;
+	unsigned int newsize = rc_map->size;
 	struct rc_map_table *newscan;
 
+	lockdep_assert_held(&rc_map->lock);
+
 	if (rc_map->size == rc_map->len) {
 		/* All entries in use -> grow keytable */
-		if (rc_map->alloc >= IR_TAB_MAX_SIZE)
+		if (newsize >= IR_TAB_MAX_ENTRIES)
 			return -ENOMEM;
 
-		newalloc *= 2;
-		dev_dbg(&dev->dev, "Growing table to %u bytes\n", newalloc);
+		newsize *= 2;
+
+		dev_dbg(&dev->dev, "Growing table to %u entries\n", newsize);
 	}
 
-	if ((rc_map->len * 3 < rc_map->size) && (oldalloc > IR_TAB_MIN_SIZE)) {
+	if (rc_map->len * 3 < rc_map->size && rc_map->size > IR_TAB_MIN_ENTRIES) {
 		/* Less than 1/3 of entries in use -> shrink keytable */
-		newalloc /= 2;
-		dev_dbg(&dev->dev, "Shrinking table to %u bytes\n", newalloc);
+		newsize /= 2;
+		dev_dbg(&dev->dev, "Shrinking table to %u entries\n", newsize);
 	}
 
-	if (newalloc == oldalloc)
+	if (newsize == rc_map->size)
 		return 0;
 
-	newscan = kmalloc(newalloc, gfp_flags);
+	newscan = krealloc_array(rc_map->scan, newsize,
+				 sizeof(struct rc_map_table), gfp_flags);
 	if (!newscan)
 		return -ENOMEM;
 
-	memcpy(newscan, rc_map->scan, rc_map->len * sizeof(struct rc_map_table));
 	rc_map->scan = newscan;
-	rc_map->alloc = newalloc;
-	rc_map->size = rc_map->alloc / sizeof(struct rc_map_table);
-	kfree(oldscan);
+	rc_map->size = newsize;
+
 	return 0;
 }
 
@@ -318,6 +336,8 @@ static unsigned int ir_update_mapping(struct rc_dev *dev,
 	int old_keycode = rc_map->scan[index].keycode;
 	int i;
 
+	lockdep_assert_held(&rc_map->lock);
+
 	/* Did the user wish to remove the mapping? */
 	if (new_keycode == KEY_RESERVED || new_keycode == KEY_UNKNOWN) {
 		dev_dbg(&dev->dev, "#%d: Deleting scan 0x%04llx\n",
@@ -371,7 +391,9 @@ static unsigned int ir_establish_scancode(struct rc_dev *dev,
 					  struct rc_map *rc_map,
 					  u64 scancode, bool resize)
 {
-	unsigned int i;
+	unsigned int i, lo, hi;
+
+	lockdep_assert_held(&rc_map->lock);
 
 	/*
 	 * Unfortunately, some hardware-based IR decoders don't provide
@@ -384,20 +406,26 @@ static unsigned int ir_establish_scancode(struct rc_dev *dev,
 	if (dev->scancode_mask)
 		scancode &= dev->scancode_mask;
 
-	/* First check if we already have a mapping for this ir command */
-	for (i = 0; i < rc_map->len; i++) {
+	/*
+	 * Binary search for an existing mapping for this ir command.
+	 */
+	lo = 0;
+	hi = rc_map->len;
+	while (lo < hi) {
+		i = lo + (hi - lo) / 2;
 		if (rc_map->scan[i].scancode == scancode)
 			return i;
-
-		/* Keytable is sorted from lowest to highest scancode */
-		if (rc_map->scan[i].scancode >= scancode)
-			break;
+		if (rc_map->scan[i].scancode < scancode)
+			lo = i + 1;
+		else
+			hi = i;
 	}
+	i = lo;
 
 	/* No previous mapping found, we might need to grow the table */
 	if (rc_map->size == rc_map->len) {
 		if (!resize || ir_resize_table(dev, rc_map, GFP_ATOMIC))
-			return -1U;
+			return UINT_MAX;
 	}
 
 	/* i is the proper index to insert our new keycode */
@@ -479,16 +507,18 @@ static int ir_setkeytable(struct rc_dev *dev, const struct rc_map *from)
 	if (rc)
 		return rc;
 
-	for (i = 0; i < from->size; i++) {
-		index = ir_establish_scancode(dev, rc_map,
-					      from->scan[i].scancode, false);
-		if (index >= rc_map->len) {
-			rc = -ENOMEM;
-			break;
-		}
+	scoped_guard(spinlock_irqsave, &rc_map->lock) {
+		for (i = 0; i < from->size; i++) {
+			index = ir_establish_scancode(dev, rc_map,
+						      from->scan[i].scancode, false);
+			if (index >= rc_map->len) {
+				rc = -ENOMEM;
+				break;
+			}
 
-		ir_update_mapping(dev, rc_map, index,
-				  from->scan[i].keycode);
+			ir_update_mapping(dev, rc_map, index,
+					  from->scan[i].keycode);
+		}
 	}
 
 	if (rc)
@@ -524,6 +554,8 @@ static unsigned int ir_lookup_by_scancode(const struct rc_map *rc_map,
 {
 	struct rc_map_table *res;
 
+	lockdep_assert_held(&rc_map->lock);
+
 	res = bsearch(&scancode, rc_map->scan, rc_map->len,
 		      sizeof(struct rc_map_table), rc_map_cmp);
 	if (!res)
@@ -1701,14 +1733,24 @@ static const struct device_type rc_dev_type = {
 struct rc_dev *rc_allocate_device(enum rc_driver_type type)
 {
 	struct rc_dev *dev;
+	int ret;
 
 	dev = kzalloc_obj(*dev);
 	if (!dev)
 		return NULL;
 
+	if (type == RC_DRIVER_IR_RAW) {
+		ret = ir_raw_event_prepare(dev);
+		if (ret < 0) {
+			kfree(dev);
+			return NULL;
+		}
+	}
+
 	if (type != RC_DRIVER_IR_RAW_TX) {
 		dev->input_dev = input_allocate_device();
 		if (!dev->input_dev) {
+			ir_raw_event_free(dev);
 			kfree(dev);
 			return NULL;
 		}
@@ -1724,6 +1766,7 @@ struct rc_dev *rc_allocate_device(enum rc_driver_type type)
 		spin_lock_init(&dev->rc_map.lock);
 		spin_lock_init(&dev->keylock);
 	}
+
 	mutex_init(&dev->lock);
 
 	dev->dev.type = &rc_dev_type;
@@ -1742,7 +1785,12 @@ void rc_free_device(struct rc_dev *dev)
 	if (!dev)
 		return;
 
-	input_free_device(dev->input_dev);
+	if (dev->input_dev) {
+		timer_delete_sync(&dev->timer_keyup);
+		timer_delete_sync(&dev->timer_repeat);
+	}
+
+	input_put_device(dev->input_dev);
 
 	put_device(&dev->dev);
 
@@ -1854,6 +1902,8 @@ static int rc_setup_rx_device(struct rc_dev *dev)
 	if (rc)
 		return rc;
 
+	input_get_device(dev->input_dev);
+
 	/*
 	 * Default delay of 250ms is too short for some protocols, especially
 	 * since the timeout is currently set to 250ms. Increase it to 500ms,
@@ -1880,10 +1930,8 @@ static void rc_free_rx_device(struct rc_dev *dev)
 	if (!dev)
 		return;
 
-	if (dev->input_dev) {
+	if (dev->input_dev)
 		input_unregister_device(dev->input_dev);
-		dev->input_dev = NULL;
-	}
 
 	ir_free_table(&dev->rc_map);
 }
@@ -1917,19 +1965,14 @@ int rc_register_device(struct rc_dev *dev)
 		dev->sysfs_groups[attr++] = &rc_dev_wakeup_filter_attr_grp;
 	dev->sysfs_groups[attr++] = NULL;
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW) {
-		rc = ir_raw_event_prepare(dev);
-		if (rc < 0)
-			goto out_minor;
-	}
-
 	if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
 		rc = rc_prepare_rx_device(dev);
 		if (rc)
 			goto out_raw;
 	}
 
-	dev->registered = true;
+	scoped_guard(mutex, &dev->lock)
+		dev->registered = true;
 
 	rc = device_add(&dev->dev);
 	if (rc)
@@ -1949,7 +1992,7 @@ int rc_register_device(struct rc_dev *dev)
 	if (dev->allowed_protocols != RC_PROTO_BIT_CEC) {
 		rc = lirc_register(dev);
 		if (rc < 0)
-			goto out_dev;
+			goto out_lirc;
 	}
 
 	if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
@@ -1972,15 +2015,20 @@ int rc_register_device(struct rc_dev *dev)
 out_rx:
 	rc_free_rx_device(dev);
 out_lirc:
-	if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
-		lirc_unregister(dev);
-out_dev:
+	scoped_guard(mutex, &dev->lock)
+		dev->registered = false;
+
+	lirc_unregister(dev);
 	device_del(&dev->dev);
+	/* registered already cleared above */
+	goto out_free_table;
 out_rx_free:
-	ir_free_table(&dev->rc_map);
+	scoped_guard(mutex, &dev->lock)
+		dev->registered = false;
+out_free_table:
+	if (dev->driver_type != RC_DRIVER_IR_RAW_TX)
+		ir_free_table(&dev->rc_map);
 out_raw:
-	ir_raw_event_free(dev);
-out_minor:
 	ida_free(&rc_ida, minor);
 	return rc;
 }
@@ -2018,18 +2066,18 @@ void rc_unregister_device(struct rc_dev *dev)
 	if (!dev)
 		return;
 
+	mutex_lock(&dev->lock);
+	dev->registered = false;
+	if (dev->users && dev->close)
+		dev->close(dev);
+	mutex_unlock(&dev->lock);
+
 	if (dev->driver_type == RC_DRIVER_IR_RAW)
 		ir_raw_event_unregister(dev);
 
 	timer_delete_sync(&dev->timer_keyup);
 	timer_delete_sync(&dev->timer_repeat);
 
-	mutex_lock(&dev->lock);
-	if (dev->users && dev->close)
-		dev->close(dev);
-	dev->registered = false;
-	mutex_unlock(&dev->lock);
-
 	rc_free_rx_device(dev);
 
 	/*
diff --git a/drivers/media/rc/redrat3.c b/drivers/media/rc/redrat3.c
index 3f828a564e192f..3e82b5e66477ee 100644
--- a/drivers/media/rc/redrat3.c
+++ b/drivers/media/rc/redrat3.c
@@ -358,8 +358,24 @@ static void redrat3_process_ir_data(struct redrat3_dev *rr3)
 
 	/* process each rr3 encoded byte into an int */
 	sig_size = be16_to_cpu(rr3->irdata.sig_size);
+
+	/*
+	 * Note we are not checking if we are reading beyond the end of the
+	 * packet which was sent, and reading stale data. If the device
+	 * sends a packet which is short then we get garbage IR, but no
+	 * out of bounds read.
+	 */
+	if (sig_size > RR3_MAX_SIG_SIZE) {
+		dev_err(dev, "length %u is incorrect\n", sig_size);
+		return;
+	}
+
 	for (i = 0; i < sig_size; i++) {
 		offset = rr3->irdata.sigdata[i];
+		if (offset >= RR3_DRIVER_MAXLENS) {
+			dev_err(dev, "offset %u is incorrect\n", offset);
+			return;
+		}
 		val = get_unaligned_be16(&rr3->irdata.lens[offset]);
 
 		/* we should always get pulse/space/pulse/space samples */
@@ -757,17 +773,9 @@ static int redrat3_transmit_ir(struct rc_dev *rcdev, unsigned *txbuf,
 	u8 curlencheck = 0;
 	unsigned i, sendbuf_len;
 
-	if (rr3->transmitting) {
-		dev_warn(dev, "%s: transmitter already in use\n", __func__);
-		return -EAGAIN;
-	}
-
 	if (count > RR3_MAX_SIG_SIZE - RR3_TX_TRAILER_LEN)
 		return -EINVAL;
 
-	/* rr3 will disable rc detector on transmit */
-	rr3->transmitting = true;
-
 	sample_lens = kzalloc_objs(*sample_lens, RR3_DRIVER_MAXLENS);
 	if (!sample_lens)
 		return -ENOMEM;
@@ -778,6 +786,9 @@ static int redrat3_transmit_ir(struct rc_dev *rcdev, unsigned *txbuf,
 		goto out;
 	}
 
+	/* rr3 will disable rc detector on transmit */
+	rr3->transmitting = true;
+
 	for (i = 0; i < count; i++) {
 		cur_sample_len = redrat3_us_to_len(txbuf[i]);
 		if (cur_sample_len > 0xffff) {
@@ -975,6 +986,7 @@ static int redrat3_dev_probe(struct usb_interface *intf,
 	struct device *dev = &intf->dev;
 	struct usb_host_interface *uhi;
 	struct redrat3_dev *rr3;
+	struct rc_dev *rc;
 	struct usb_endpoint_descriptor *ep;
 	struct usb_endpoint_descriptor *ep_narrow = NULL;
 	struct usb_endpoint_descriptor *ep_wide = NULL;
@@ -1119,9 +1131,12 @@ static int redrat3_dev_probe(struct usb_interface *intf,
 	return 0;
 
 led_free:
+	rc_unregister_device(rr3->rc);
 	led_classdev_unregister(&rr3->led);
 redrat_free:
+	rc = rr3->rc;
 	redrat3_delete(rr3, rr3->udev);
+	rc_free_device(rc);
 
 no_endpoints:
 	return retval;
@@ -1148,6 +1163,7 @@ static int redrat3_dev_suspend(struct usb_interface *intf, pm_message_t message)
 	usb_kill_urb(rr3->narrow_urb);
 	usb_kill_urb(rr3->wide_urb);
 	usb_kill_urb(rr3->flash_urb);
+	usb_kill_urb(rr3->learn_urb);
 	return 0;
 }
 
diff --git a/drivers/media/rc/serial_ir.c b/drivers/media/rc/serial_ir.c
index 992fff82b5245e..f6702b3df9a03c 100644
--- a/drivers/media/rc/serial_ir.c
+++ b/drivers/media/rc/serial_ir.c
@@ -798,7 +798,7 @@ static int __init serial_ir_init_module(void)
 
 static void __exit serial_ir_exit_module(void)
 {
-	timer_delete_sync(&serial_ir.timeout_timer);
+	timer_shutdown_sync(&serial_ir.timeout_timer);
 	serial_ir_exit();
 }
 
diff --git a/drivers/media/rc/streamzap.c b/drivers/media/rc/streamzap.c
index 307985d74fe8a2..41195ad82734f2 100644
--- a/drivers/media/rc/streamzap.c
+++ b/drivers/media/rc/streamzap.c
@@ -365,6 +365,7 @@ static int streamzap_probe(struct usb_interface *intf,
 
 	return 0;
 rc_submit_fail:
+	rc_unregister_device(sz->rdev);
 	rc_free_device(sz->rdev);
 	usb_set_intfdata(intf, NULL);
 rc_dev_fail:
diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c
index 28e840a7e5b8c3..af1ee08ffdbec8 100644
--- a/drivers/media/rc/sunxi-cir.c
+++ b/drivers/media/rc/sunxi-cir.c
@@ -374,8 +374,8 @@ static void sunxi_ir_remove(struct platform_device *pdev)
 	struct sunxi_ir *ir = platform_get_drvdata(pdev);
 
 	rc_unregister_device(ir->rc);
-	rc_free_device(ir->rc);
 	sunxi_ir_hw_exit(&pdev->dev);
+	rc_free_device(ir->rc);
 }
 
 static void sunxi_ir_shutdown(struct platform_device *pdev)
diff --git a/drivers/media/spi/Kconfig b/drivers/media/spi/Kconfig
index 4656afae5bb467..da3e225d420d72 100644
--- a/drivers/media/spi/Kconfig
+++ b/drivers/media/spi/Kconfig
@@ -1,15 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0-only
 if VIDEO_DEV && SPI
 
-comment "SPI I2C drivers auto-selected by 'Autoselect ancillary drivers'"
-	depends on MEDIA_HIDE_ANCILLARY_SUBDRV && SPI
-
 menu "Media SPI Adapters"
 
 config CXD2880_SPI_DRV
 	tristate "Sony CXD2880 SPI support"
 	depends on DVB_CORE && SPI
-	default m if !MEDIA_SUBDRV_AUTOSELECT
 	help
 	  Choose if you would like to have SPI interface support for Sony CXD2880.
 
diff --git a/drivers/media/test-drivers/vicodec/codec-fwht.h b/drivers/media/test-drivers/vicodec/codec-fwht.h
index 0eab24020e9e6e..4b4d3903108992 100644
--- a/drivers/media/test-drivers/vicodec/codec-fwht.h
+++ b/drivers/media/test-drivers/vicodec/codec-fwht.h
@@ -61,7 +61,7 @@
  * both luma and chroma components resolutions are rounded up to
  * a multiple of 8
  */
-#define vic_round_dim(dim, div) (round_up((dim) / (div), 8) * (div))
+#define vic_round_dim(dim, div) round_up(dim, 8 * (div))
 
 struct fwht_cframe_hdr {
 	u32 magic1;
diff --git a/drivers/media/test-drivers/vicodec/vicodec-core.c b/drivers/media/test-drivers/vicodec/vicodec-core.c
index ff9d50fb05fd13..7ea024b14f0e9d 100644
--- a/drivers/media/test-drivers/vicodec/vicodec-core.c
+++ b/drivers/media/test-drivers/vicodec/vicodec-core.c
@@ -890,6 +890,8 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 				  struct v4l2_format *f)
 {
 	struct vicodec_ctx *ctx = file2ctx(file);
+	struct vicodec_q_data *q_data_out =
+		get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
 	struct v4l2_pix_format_mplane *pix_mp;
 	struct v4l2_pix_format *pix;
 
@@ -900,6 +902,10 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 		pix = &f->fmt.pix;
 		pix->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
 				   find_fmt(f->fmt.pix.pixelformat)->id;
+		if (ctx->is_enc) {
+			pix->width = q_data_out->coded_width;
+			pix->height = q_data_out->coded_height;
+		}
 		pix->colorspace = ctx->state.colorspace;
 		pix->xfer_func = ctx->state.xfer_func;
 		pix->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -911,6 +917,10 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 		pix_mp = &f->fmt.pix_mp;
 		pix_mp->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
 				      find_fmt(pix_mp->pixelformat)->id;
+		if (ctx->is_enc) {
+			pix_mp->width = q_data_out->coded_width;
+			pix_mp->height = q_data_out->coded_height;
+		}
 		pix_mp->colorspace = ctx->state.colorspace;
 		pix_mp->xfer_func = ctx->state.xfer_func;
 		pix_mp->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -1595,9 +1605,9 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 	}
 	state->ref_stride = q_data->coded_width * info->luma_alpha_step;
 
-	state->ref_frame.buf = kvmalloc(total_planes_size, GFP_KERNEL);
+	state->ref_frame.buf = kvzalloc(total_planes_size, GFP_KERNEL);
 	state->ref_frame.luma = state->ref_frame.buf;
-	new_comp_frame = kvmalloc(ctx->comp_max_size, GFP_KERNEL);
+	new_comp_frame = kvzalloc(ctx->comp_max_size, GFP_KERNEL);
 
 	if (!state->ref_frame.luma || !new_comp_frame) {
 		kvfree(state->ref_frame.luma);
diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
index fd69b4ee16f4bd..4c98dfbd406f81 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
@@ -44,6 +44,10 @@
 #define LNB_LOW_FREQ		9750000		/* low IF frequency */
 #define LNB_HIGH_FREQ		10600000	/* transition frequency */
 
+/* The Ku-band range covered by such an LNBf, in kHz */
+#define LNB_KU_BAND_MIN_FREQ	10700000
+#define LNB_KU_BAND_MAX_FREQ	12750000
+
 static unsigned int drop_tslock_prob_on_low_snr;
 module_param(drop_tslock_prob_on_low_snr, uint, 0444);
 MODULE_PARM_DESC(drop_tslock_prob_on_low_snr,
@@ -367,6 +371,36 @@ static int vidtv_bridge_probe_demod(struct vidtv_dvb *dvb, u32 n)
 	return 0;
 }
 
+/*
+ * Reject frequencies the simulation could never tune into, as the module
+ * would otherwise load just fine and then never lock on anything, leaving
+ * no clue about what went wrong.
+ */
+static int vidtv_bridge_check_freqs(struct vidtv_dvb *dvb,
+				    const unsigned int *freqs,
+				    u32 array_sz,
+				    u32 min_freq,
+				    u32 max_freq,
+				    const char *name)
+{
+	u32 i;
+
+	for (i = 0; i < array_sz; i++) {
+		/* a zeroed entry means an unused slot */
+		if (!freqs[i])
+			continue;
+
+		if (freqs[i] < min_freq || freqs[i] > max_freq) {
+			dev_err(&dvb->pdev->dev,
+				"%s[%u]: %u is out of range (%u..%u)\n",
+				name, i, freqs[i], min_freq, max_freq);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 static int vidtv_bridge_probe_tuner(struct vidtv_dvb *dvb, u32 n)
 {
 	struct vidtv_tuner_config cfg = {
@@ -374,10 +408,45 @@ static int vidtv_bridge_probe_tuner(struct vidtv_dvb *dvb, u32 n)
 		.mock_power_up_delay_msec = mock_power_up_delay_msec,
 		.mock_tune_delay_msec     = mock_tune_delay_msec,
 	};
+	u32 min_freq = dvb->fe[n]->ops.info.frequency_min_hz;
+	u32 max_freq = dvb->fe[n]->ops.info.frequency_max_hz;
 	u32 freq;
+	int ret;
 	int i;
 
-	/* TODO: check if the frequencies are at a valid range */
+	/*
+	 * Terrestrial and cable frequencies are given in Hz and are used as
+	 * is, so they have to fit within the range the demod reports to the
+	 * DVB core: the core rejects a tuning request outside of it before
+	 * the tuner is ever asked about the frequency.
+	 */
+	ret = vidtv_bridge_check_freqs(dvb, vidtv_valid_dvb_t_freqs,
+				       ARRAY_SIZE(vidtv_valid_dvb_t_freqs),
+				       min_freq, max_freq,
+				       "vidtv_valid_dvb_t_freqs");
+	if (ret)
+		return ret;
+
+	ret = vidtv_bridge_check_freqs(dvb, vidtv_valid_dvb_c_freqs,
+				       ARRAY_SIZE(vidtv_valid_dvb_c_freqs),
+				       min_freq, max_freq,
+				       "vidtv_valid_dvb_c_freqs");
+	if (ret)
+		return ret;
+
+	/*
+	 * Satellite frequencies are given in kHz at Ku-band and are
+	 * downconverted below, so check them against the band the simulated
+	 * LNBf covers instead. Doing so also ensures that the frequencies
+	 * are above the LNBf local oscillators.
+	 */
+	ret = vidtv_bridge_check_freqs(dvb, vidtv_valid_dvb_s_freqs,
+				       ARRAY_SIZE(vidtv_valid_dvb_s_freqs),
+				       LNB_KU_BAND_MIN_FREQ,
+				       LNB_KU_BAND_MAX_FREQ,
+				       "vidtv_valid_dvb_s_freqs");
+	if (ret)
+		return ret;
 
 	memcpy(cfg.vidtv_valid_dvb_t_freqs,
 	       vidtv_valid_dvb_t_freqs,
@@ -474,6 +543,7 @@ static int vidtv_bridge_dvb_init(struct vidtv_dvb *dvb)
 fail_demod_probe:
 	for (i = i - 1; i >= 0; --i) {
 		dvb_unregister_frontend(dvb->fe[i]);
+		dvb_frontend_detach(dvb->fe[i]);
 fail_fe:
 		dvb_module_release(dvb->i2c_client_tuner[i]);
 fail_tuner_probe:
@@ -550,8 +620,11 @@ static void vidtv_bridge_remove(struct platform_device *pdev)
 
 	mutex_destroy(&dvb->feed_lock);
 
+	vidtv_stop_streaming(dvb);
+
 	for (i = 0; i < NUM_FE; ++i) {
 		dvb_unregister_frontend(dvb->fe[i]);
+		dvb_frontend_detach(dvb->fe[i]);
 		dvb_module_release(dvb->i2c_client_tuner[i]);
 		dvb_module_release(dvb->i2c_client_demod[i]);
 	}
diff --git a/drivers/media/test-drivers/vidtv/vidtv_demod.c b/drivers/media/test-drivers/vidtv/vidtv_demod.c
index 6e5fe402976bb7..3aa58600463864 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_demod.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_demod.c
@@ -343,13 +343,6 @@ static int vidtv_diseqc_send_burst(struct dvb_frontend *fe,
 	return 0;
 }
 
-static void vidtv_demod_release(struct dvb_frontend *fe)
-{
-	struct vidtv_demod_state *state = fe->demodulator_priv;
-
-	kfree(state);
-}
-
 static const struct dvb_frontend_ops vidtv_demod_ops = {
 	.delsys = {
 		SYS_DVBT,
@@ -390,8 +383,6 @@ static const struct dvb_frontend_ops vidtv_demod_ops = {
 			FE_CAN_HIERARCHY_AUTO,
 	},
 
-	.release = vidtv_demod_release,
-
 	.set_frontend = vidtv_demod_set_frontend,
 	.get_frontend = vidtv_demod_get_frontend,
 
diff --git a/drivers/media/test-drivers/vim2m.c b/drivers/media/test-drivers/vim2m.c
index bb2dd11eef0e4e..459fd4aedf3096 100644
--- a/drivers/media/test-drivers/vim2m.c
+++ b/drivers/media/test-drivers/vim2m.c
@@ -150,8 +150,8 @@ enum {
 	V4L2_M2M_DST = 1,
 };
 
-#define V4L2_CID_TRANS_TIME_MSEC	(V4L2_CID_USER_BASE + 0x1000)
-#define V4L2_CID_TRANS_NUM_BUFS		(V4L2_CID_USER_BASE + 0x1001)
+#define V4L2_CID_TRANS_TIME_MSEC	(V4L2_CID_USER_VIM2M_BASE + 0)
+#define V4L2_CID_TRANS_NUM_BUFS		(V4L2_CID_USER_VIM2M_BASE + 1)
 
 static struct vim2m_fmt *find_format(u32 fourcc)
 {
@@ -205,6 +205,7 @@ struct vim2m_ctx {
 	struct vim2m_dev	*dev;
 
 	struct v4l2_ctrl_handler hdl;
+	struct v4l2_ctrl	*trans_num_bufs_ctrl;
 
 	/* Processed buffers in this transaction */
 	u8			num_processed;
@@ -1258,9 +1259,27 @@ static int vim2m_start_streaming(struct vb2_queue *q, unsigned int count)
 		ctx->aborting = 0;
 
 	q_data->sequence = 0;
+	v4l2_ctrl_grab(ctx->trans_num_bufs_ctrl, true);
+
 	return 0;
 }
 
+static bool vim2m_other_queue_is_streaming(struct vim2m_ctx *ctx,
+					   struct vb2_queue *q)
+{
+	struct vb2_queue *other_vq;
+
+	if (!ctx->fh.m2m_ctx)
+		return false;
+
+	if (V4L2_TYPE_IS_OUTPUT(q->type))
+		other_vq = v4l2_m2m_get_dst_vq(ctx->fh.m2m_ctx);
+	else
+		other_vq = v4l2_m2m_get_src_vq(ctx->fh.m2m_ctx);
+
+	return vb2_is_streaming(other_vq);
+}
+
 static void vim2m_stop_streaming(struct vb2_queue *q)
 {
 	struct vim2m_ctx *ctx = vb2_get_drv_priv(q);
@@ -1274,11 +1293,14 @@ static void vim2m_stop_streaming(struct vb2_queue *q)
 		else
 			vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
 		if (!vbuf)
-			return;
+			break;
 		v4l2_ctrl_request_complete(vbuf->vb2_buf.req_obj.req,
 					   &ctx->hdl);
 		v4l2_m2m_buf_done(vbuf, VB2_BUF_STATE_ERROR);
 	}
+
+	if (!vim2m_other_queue_is_streaming(ctx, q))
+		v4l2_ctrl_grab(ctx->trans_num_bufs_ctrl, false);
 }
 
 static void vim2m_buf_request_complete(struct vb2_buffer *vb)
@@ -1380,7 +1402,8 @@ static int vim2m_open(struct file *file)
 
 	vim2m_ctrl_trans_time_msec.def = default_transtime;
 	v4l2_ctrl_new_custom(hdl, &vim2m_ctrl_trans_time_msec, NULL);
-	v4l2_ctrl_new_custom(hdl, &vim2m_ctrl_trans_num_bufs, NULL);
+	ctx->trans_num_bufs_ctrl =
+		v4l2_ctrl_new_custom(hdl, &vim2m_ctrl_trans_num_bufs, NULL);
 	if (hdl->error) {
 		rc = hdl->error;
 		v4l2_ctrl_handler_free(hdl);
@@ -1435,10 +1458,10 @@ static int vim2m_release(struct file *file)
 
 	v4l2_fh_del(&ctx->fh, file);
 	v4l2_fh_exit(&ctx->fh);
-	v4l2_ctrl_handler_free(&ctx->hdl);
 	mutex_lock(&dev->dev_mutex);
 	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
 	mutex_unlock(&dev->dev_mutex);
+	v4l2_ctrl_handler_free(&ctx->hdl);
 	kfree(ctx);
 
 	atomic_dec(&dev->num_inst);
diff --git a/drivers/media/test-drivers/vimc/vimc-debayer.c b/drivers/media/test-drivers/vimc/vimc-debayer.c
index 0c2e715a8a16c2..cfda447a5b76c0 100644
--- a/drivers/media/test-drivers/vimc/vimc-debayer.c
+++ b/drivers/media/test-drivers/vimc/vimc-debayer.c
@@ -240,6 +240,7 @@ static void vimc_debayer_adjust_sink_fmt(struct v4l2_mbus_framefmt *fmt)
 }
 
 static int vimc_debayer_set_fmt(struct v4l2_subdev *sd,
+				const struct v4l2_subdev_client_info *ci,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/test-drivers/vimc/vimc-scaler.c b/drivers/media/test-drivers/vimc/vimc-scaler.c
index e2037c67e42361..d8dc65d33f62b8 100644
--- a/drivers/media/test-drivers/vimc/vimc-scaler.c
+++ b/drivers/media/test-drivers/vimc/vimc-scaler.c
@@ -140,6 +140,7 @@ static int vimc_scaler_enum_frame_size(struct v4l2_subdev *sd,
 }
 
 static int vimc_scaler_set_fmt(struct v4l2_subdev *sd,
+			    const struct v4l2_subdev_client_info *ci,
 			    struct v4l2_subdev_state *sd_state,
 			    struct v4l2_subdev_format *format)
 {
@@ -204,6 +205,7 @@ static int vimc_scaler_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int vimc_scaler_get_selection(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_selection *sel)
 {
@@ -239,6 +241,7 @@ static void vimc_scaler_adjust_sink_crop(struct v4l2_rect *r,
 }
 
 static int vimc_scaler_set_selection(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/media/test-drivers/vimc/vimc-sensor.c b/drivers/media/test-drivers/vimc/vimc-sensor.c
index 83dcc9d61ee05b..f07f4237f6ada1 100644
--- a/drivers/media/test-drivers/vimc/vimc-sensor.c
+++ b/drivers/media/test-drivers/vimc/vimc-sensor.c
@@ -97,18 +97,14 @@ static void vimc_sensor_update_frame_timing(struct v4l2_subdev *sd,
 {
 	struct vimc_sensor_device *vsensor =
 		container_of(sd, struct vimc_sensor_device, sd);
-	u64 pixel_rate = vsensor->pixel_rate->val;
+	u32 pixel_rate = vsensor->pixel_rate->val;
 	u32 hts = width + vsensor->hblank->val;
 	u32 vts = height + vsensor->vblank->val;
 	u64 total_pixels = (u64)hts * vts;
 	u64 frame_interval_ns;
 
-	/* Sanity check, pixel rate is fixed and fits in 32 bits. */
-	if (WARN_ON(pixel_rate >= 0x100000000))
-		return;
-
 	frame_interval_ns = total_pixels * NSEC_PER_SEC;
-	do_div(frame_interval_ns, (u32)pixel_rate);
+	do_div(frame_interval_ns, pixel_rate);
 	vsensor->hw.fps_jiffies = nsecs_to_jiffies(frame_interval_ns);
 	if (vsensor->hw.fps_jiffies == 0)
 		vsensor->hw.fps_jiffies = 1;
@@ -154,6 +150,7 @@ static u32 vimc_calc_vblank(u32 width, u32 height,
 }
 
 static int vimc_sensor_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/media/test-drivers/vivid/vivid-cec.c b/drivers/media/test-drivers/vivid/vivid-cec.c
index 2d15fdd5d999e0..8d301578e2eb9f 100644
--- a/drivers/media/test-drivers/vivid/vivid-cec.c
+++ b/drivers/media/test-drivers/vivid/vivid-cec.c
@@ -23,7 +23,7 @@ struct xfer_on_bus {
 static bool find_dest_adap(struct vivid_dev *dev,
 			   struct cec_adapter *adap, u8 dest)
 {
-	unsigned int i, j;
+	unsigned int i;
 
 	if (dest >= 0xf)
 		return false;
@@ -33,13 +33,12 @@ static bool find_dest_adap(struct vivid_dev *dev,
 	    cec_has_log_addr(dev->cec_rx_adap, dest))
 		return true;
 
-	for (i = 0, j = 0; i < dev->num_inputs; i++) {
+	for (i = 0; i < dev->num_inputs; i++) {
 		unsigned int menu_idx =
 			dev->input_is_connected_to_output[i];
 
 		if (dev->input_type[i] != HDMI)
 			continue;
-		j++;
 		if (menu_idx < FIXED_MENU_ITEMS)
 			continue;
 
@@ -113,7 +112,7 @@ static void adjust_sfts(struct vivid_dev *dev)
 int vivid_cec_bus_thread(void *_dev)
 {
 	u32 last_sft;
-	unsigned int i, j;
+	unsigned int i;
 	unsigned int dest;
 	ktime_t start, end;
 	s64 delta_us, retry_us;
@@ -210,13 +209,12 @@ int vivid_cec_bus_thread(void *_dev)
 		if (first_status == CEC_TX_STATUS_OK) {
 			if (xfers_on_bus[first_idx].adap != dev->cec_rx_adap)
 				cec_received_msg(dev->cec_rx_adap, &first_msg);
-			for (i = 0, j = 0; i < dev->num_inputs; i++) {
+			for (i = 0; i < dev->num_inputs; i++) {
 				unsigned int menu_idx =
 					dev->input_is_connected_to_output[i];
 
 				if (dev->input_type[i] != HDMI)
 					continue;
-				j++;
 				if (menu_idx < FIXED_MENU_ITEMS)
 					continue;
 
diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c
index 62cfb5feb2cf53..c042d92db175dd 100644
--- a/drivers/media/test-drivers/vivid/vivid-core.c
+++ b/drivers/media/test-drivers/vivid/vivid-core.c
@@ -854,6 +854,7 @@ static void vivid_dev_release(struct v4l2_device *v4l2_dev)
 	struct vivid_dev *dev = container_of(v4l2_dev, struct vivid_dev, v4l2_dev);
 
 	cancel_work_sync(&dev->update_hdmi_ctrl_work);
+	cancel_work_sync(&dev->update_svid_ctrl_work);
 	vivid_free_controls(dev);
 	v4l2_device_unregister(&dev->v4l2_dev);
 #ifdef CONFIG_MEDIA_CONTROLLER
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
index e204490847095c..147af0f9b07771 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c
+++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
@@ -570,6 +570,7 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv,
 	const struct vivid_fmt *fmt;
 	unsigned bytesperline, max_bpl;
 	unsigned factor = 1;
+	unsigned int vdiv = 1;
 	unsigned w, h;
 	unsigned p;
 	bool user_set_csc = !!(mp->flags & V4L2_PIX_FMT_FLAG_SET_CSC);
@@ -622,6 +623,18 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv,
 		mp->height = r.height / factor;
 	}
 
+	/*
+	 * The chroma planes of vertically subsampled formats hold
+	 * height / vdownsampling lines. If the height is not a multiple of
+	 * the subsampling factor, then the buffer size calculations round
+	 * that number down while the test pattern generator rounds it up,
+	 * so the generator writes one line past the end of the buffer.
+	 * Round the height down to keep both in sync.
+	 */
+	for (p = 0; p < fmt->planes; p++)
+		vdiv = max(vdiv, fmt->vdownsampling[p]);
+	mp->height = rounddown(mp->height, vdiv);
+
 	/* This driver supports custom bytesperline values */
 
 	mp->num_planes = fmt->buffers;
diff --git a/drivers/media/tuners/tda18250.c b/drivers/media/tuners/tda18250.c
index bd7ed50fb6e856..e4b4106e91b632 100644
--- a/drivers/media/tuners/tda18250.c
+++ b/drivers/media/tuners/tda18250.c
@@ -804,8 +804,7 @@ static int tda18250_probe(struct i2c_client *client)
 
 	/* read the three chip ID registers */
 	regmap_bulk_read(dev->regmap, R00_ID1, &chip_id, 3);
-	dev_dbg(&client->dev, "chip_id=%02x:%02x:%02x",
-			chip_id[0], chip_id[1], chip_id[2]);
+	dev_dbg(&client->dev, "chip_id=%3phC", chip_id);
 
 	switch (chip_id[0]) {
 	case 0xc7:
diff --git a/drivers/media/usb/au0828/au0828-core.c b/drivers/media/usb/au0828/au0828-core.c
index 445cbeb7abaee6..c3826b3b8a2eb0 100644
--- a/drivers/media/usb/au0828/au0828-core.c
+++ b/drivers/media/usb/au0828/au0828-core.c
@@ -697,6 +697,10 @@ static int au0828_usb_probe(struct usb_interface *interface,
 	retval = au0828_v4l2_device_register(interface, dev);
 	if (retval) {
 		au0828_usb_v4l2_media_release(dev);
+#ifdef CONFIG_MEDIA_CONTROLLER
+		media_device_delete(dev->media_dev, KBUILD_MODNAME,
+				    THIS_MODULE);
+#endif
 		mutex_unlock(&dev->lock);
 		kfree(dev);
 		return retval;
diff --git a/drivers/media/usb/au0828/au0828-dvb.c b/drivers/media/usb/au0828/au0828-dvb.c
index 9c95b7ceaecdcc..601a12e122f1c2 100644
--- a/drivers/media/usb/au0828/au0828-dvb.c
+++ b/drivers/media/usb/au0828/au0828-dvb.c
@@ -163,9 +163,6 @@ static int stop_urb_transfer(struct au0828_dev *dev)
 
 	dprintk(2, "%s()\n", __func__);
 
-	if (!dev->urb_streaming)
-		return 0;
-
 	if (dev->bulk_timeout_running == 1) {
 		dev->bulk_timeout_running = 0;
 		timer_delete(&dev->bulk_timeout);
@@ -179,6 +176,7 @@ static int stop_urb_transfer(struct au0828_dev *dev)
 				kfree(dev->urbs[i]->transfer_buffer);
 
 			usb_free_urb(dev->urbs[i]);
+			dev->urbs[i] = NULL;
 		}
 	}
 
@@ -200,8 +198,10 @@ static int start_urb_transfer(struct au0828_dev *dev)
 	for (i = 0; i < URB_COUNT; i++) {
 
 		dev->urbs[i] = usb_alloc_urb(0, GFP_KERNEL);
-		if (!dev->urbs[i])
-			return -ENOMEM;
+		if (!dev->urbs[i]) {
+			ret = -ENOMEM;
+			goto err;
+		}
 
 		purb = dev->urbs[i];
 
@@ -217,7 +217,7 @@ static int start_urb_transfer(struct au0828_dev *dev)
 			ret = -ENOMEM;
 			pr_err("%s: failed big buffer allocation, err = %d\n",
 			       __func__, ret);
-			return ret;
+			goto err;
 		}
 
 		purb->status = -EINPROGRESS;
@@ -235,10 +235,9 @@ static int start_urb_transfer(struct au0828_dev *dev)
 	for (i = 0; i < URB_COUNT; i++) {
 		ret = usb_submit_urb(dev->urbs[i], GFP_ATOMIC);
 		if (ret != 0) {
-			stop_urb_transfer(dev);
 			pr_err("%s: failed urb submission, err = %d\n",
 			       __func__, ret);
-			return ret;
+			goto err;
 		}
 	}
 
@@ -249,6 +248,10 @@ static int start_urb_transfer(struct au0828_dev *dev)
 	dev->bulk_timeout_running = 1;
 
 	return 0;
+
+err:
+	stop_urb_transfer(dev);
+	return ret;
 }
 
 static void au0828_start_transport(struct au0828_dev *dev)
@@ -537,6 +540,7 @@ void au0828_dvb_unregister(struct au0828_dev *dev)
 	if (dvb->frontend == NULL)
 		return;
 
+	timer_shutdown_sync(&dev->bulk_timeout);
 	cancel_work_sync(&dev->restart_streaming);
 
 	dvb_net_release(&dvb->net);
diff --git a/drivers/media/usb/cx231xx/cx231xx-417.c b/drivers/media/usb/cx231xx/cx231xx-417.c
index c695a97e202b71..d0955ea1819582 100644
--- a/drivers/media/usb/cx231xx/cx231xx-417.c
+++ b/drivers/media/usb/cx231xx/cx231xx-417.c
@@ -1666,7 +1666,7 @@ static int cx231xx_s_video_encoding(struct cx2341x_handler *cxhdl, u32 val)
 	format.format.width = cxhdl->width / (is_mpeg1 ? 2 : 1);
 	format.format.height = cxhdl->height;
 	format.format.code = MEDIA_BUS_FMT_FIXED;
-	v4l2_subdev_call(dev->sd_cx25840, pad, set_fmt, NULL, &format);
+	v4l2_subdev_call(dev->sd_cx25840, pad, set_fmt, NULL, NULL, &format);
 	return 0;
 }
 
diff --git a/drivers/media/usb/cx231xx/cx231xx-audio.c b/drivers/media/usb/cx231xx/cx231xx-audio.c
index b24ceef497e4cc..af04fb104cc4e5 100644
--- a/drivers/media/usb/cx231xx/cx231xx-audio.c
+++ b/drivers/media/usb/cx231xx/cx231xx-audio.c
@@ -581,12 +581,19 @@ static int cx231xx_audio_init(struct cx231xx *dev)
 	dev_dbg(dev->dev,
 		"probing for cx231xx non standard usbaudio\n");
 
+	/*
+	 * Extension init errors are ignored by the cx231xx core, so fini()
+	 * must be safe even if initialization fails part way through.
+	 */
+	spin_lock_init(&adev->slock);
+	INIT_WORK(&dev->wq_trigger, audio_trigger);
+	atomic_set(&dev->stream_started, 0);
+
 	err = snd_card_new(dev->dev, index[devnr], "Cx231xx Audio",
 			   THIS_MODULE, 0, &card);
 	if (err < 0)
 		return err;
 
-	spin_lock_init(&adev->slock);
 	err = snd_pcm_new(card, "Cx231xx Audio", 0, 0, 1, &pcm);
 	if (err < 0)
 		goto err_free_card;
@@ -601,8 +608,6 @@ static int cx231xx_audio_init(struct cx231xx *dev)
 	strscpy(card->shortname, "Cx231xx Audio", sizeof(card->shortname));
 	strscpy(card->longname, "Conexant cx231xx Audio", sizeof(card->longname));
 
-	INIT_WORK(&dev->wq_trigger, audio_trigger);
-
 	err = snd_card_register(card);
 	if (err < 0)
 		goto err_free_card;
@@ -656,8 +661,10 @@ static int cx231xx_audio_init(struct cx231xx *dev)
 
 err_free_pkt_size:
 	kfree(adev->alt_max_pkt_size);
+	adev->alt_max_pkt_size = NULL;
 err_free_card:
 	snd_card_free(card);
+	adev->sndcard = NULL;
 
 	return err;
 }
@@ -674,6 +681,8 @@ static int cx231xx_audio_fini(struct cx231xx *dev)
 		return 0;
 	}
 
+	disable_work_sync(&dev->wq_trigger);
+
 	if (dev->adev.sndcard) {
 		snd_card_free_when_closed(dev->adev.sndcard);
 		kfree(dev->adev.alt_max_pkt_size);
diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c
index 69b24205bc567c..b0941cd2a2d858 100644
--- a/drivers/media/usb/cx231xx/cx231xx-cards.c
+++ b/drivers/media/usb/cx231xx/cx231xx-cards.c
@@ -1826,7 +1826,7 @@ static int cx231xx_usb_probe(struct usb_interface *interface,
 
 	retval = cx231xx_init_v4l2(dev, udev, interface, isoc_pipe);
 	if (retval)
-		goto err_init;
+		goto err_video_alt;
 
 	if (dev->current_pcb_config.ts1_source != 0xff) {
 		/* compute alternate max packet sizes for TS1 */
diff --git a/drivers/media/usb/cx231xx/cx231xx-video.c b/drivers/media/usb/cx231xx/cx231xx-video.c
index 70aa99fead2780..e8c9c74b4b8a8d 100644
--- a/drivers/media/usb/cx231xx/cx231xx-video.c
+++ b/drivers/media/usb/cx231xx/cx231xx-video.c
@@ -909,7 +909,7 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
 	dev->format = format_by_fourcc(f->fmt.pix.pixelformat);
 
 	v4l2_fill_mbus_format(&format.format, &f->fmt.pix, MEDIA_BUS_FMT_FIXED);
-	call_all(dev, pad, set_fmt, NULL, &format);
+	call_all(dev, pad, set_fmt, NULL, NULL, &format);
 	v4l2_fill_pix_format(&f->fmt.pix, &format.format);
 
 	return rc;
@@ -950,7 +950,7 @@ static int vidioc_s_std(struct file *file, void *priv, v4l2_std_id norm)
 	format.format.code = MEDIA_BUS_FMT_FIXED;
 	format.format.width = dev->width;
 	format.format.height = dev->height;
-	call_all(dev, pad, set_fmt, NULL, &format);
+	call_all(dev, pad, set_fmt, NULL, NULL, &format);
 
 	/* do mode control overrides */
 	cx231xx_do_mode_ctrl_overrides(dev);
diff --git a/drivers/media/usb/cx231xx/cx231xx.h b/drivers/media/usb/cx231xx/cx231xx.h
index 19f5036a78d7f4..f85d054fbb5cf2 100644
--- a/drivers/media/usb/cx231xx/cx231xx.h
+++ b/drivers/media/usb/cx231xx/cx231xx.h
@@ -463,7 +463,7 @@ struct cx231xx_i2c_xfer_data {
 	u8 direction;		/* 1 - IN, 0 - OUT */
 	u8 saddr_len;		/* sub address len */
 	u16 saddr_dat;		/* sub addr data */
-	u8 buf_size;		/* buffer size */
+	u16 buf_size;		/* buffer size */
 	u8 *p_buffer;		/* pointer to the buffer */
 };
 
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf-i2c.c b/drivers/media/usb/dvb-usb-v2/mxl111sf-i2c.c
index 100a1052dcbc2b..1a7135b03c3337 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf-i2c.c
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf-i2c.c
@@ -755,7 +755,7 @@ static int mxl111sf_i2c_hw_xfer_msg(struct mxl111sf_state *state,
 	buf[0] = USB_WRITE_I2C_CMD;
 	buf[1] = 0x00;
 
-	/* de-initilize I2C BUS */
+	/* de-initialize I2C BUS */
 	buf[5] = USB_END_I2C_CMD;
 	mxl111sf_i2c_send_data(state, 0, buf);
 
@@ -769,7 +769,7 @@ static int mxl111sf_i2c_hw_xfer_msg(struct mxl111sf_state *state,
 	buf[6] = 0x00;
 	buf[7] = 0x00;
 
-	/* de-initilize I2C BUS */
+	/* de-initialize I2C BUS */
 	buf[8] = USB_END_I2C_CMD;
 	mxl111sf_i2c_send_data(state, 0, buf);
 
diff --git a/drivers/media/usb/dvb-usb/cxusb-analog.c b/drivers/media/usb/dvb-usb/cxusb-analog.c
index 3bbee1fcbc8dc3..8c7d0888c7211c 100644
--- a/drivers/media/usb/dvb-usb/cxusb-analog.c
+++ b/drivers/media/usb/dvb-usb/cxusb-analog.c
@@ -1031,7 +1031,8 @@ static int cxusb_medion_try_s_fmt_vid_cap(struct file *file,
 	subfmt.format.field = field;
 	subfmt.format.colorspace = V4L2_COLORSPACE_SMPTE170M;
 
-	ret = v4l2_subdev_call(cxdev->cx25840, pad, set_fmt, NULL, &subfmt);
+	ret = v4l2_subdev_call(cxdev->cx25840, pad, set_fmt, NULL, NULL,
+			       &subfmt);
 	if (ret != 0)
 		return ret;
 
@@ -1513,7 +1514,8 @@ int cxusb_medion_analog_init(struct dvb_usb_device *dvbdev)
 	subfmt.format.field = V4L2_FIELD_SEQ_TB;
 	subfmt.format.colorspace = V4L2_COLORSPACE_SMPTE170M;
 
-	ret = v4l2_subdev_call(cxdev->cx25840, pad, set_fmt, NULL, &subfmt);
+	ret = v4l2_subdev_call(cxdev->cx25840, pad, set_fmt, NULL, NULL,
+			       &subfmt);
 	if (ret != 0)
 		dev_warn(&dvbdev->udev->dev,
 			 "cx25840 format set failed (%d)\n", ret);
diff --git a/drivers/media/usb/dvb-usb/dib0700_core.c b/drivers/media/usb/dvb-usb/dib0700_core.c
index 1caabb51ea47ef..5d2fa7037c0038 100644
--- a/drivers/media/usb/dvb-usb/dib0700_core.c
+++ b/drivers/media/usb/dvb-usb/dib0700_core.c
@@ -560,7 +560,7 @@ int dib0700_download_firmware(struct usb_device *udev, const struct firmware *fw
 	fw_version = (buf[8] << 24) | (buf[9] << 16) | (buf[10] << 8) | buf[11];
 
 	/* set the buffer size - DVB-USB is allocating URB buffers
-	 * only after the firwmare download was successful */
+	 * only after the firmware download was successful */
 	for (i = 0; i < dib0700_device_count; i++) {
 		for (adap_num = 0; adap_num < dib0700_devices[i].num_adapters;
 				adap_num++) {
diff --git a/drivers/media/usb/dvb-usb/dvb-usb-firmware.c b/drivers/media/usb/dvb-usb/dvb-usb-firmware.c
index 0fb3fa6100e4aa..675d9d1d4f4723 100644
--- a/drivers/media/usb/dvb-usb/dvb-usb-firmware.c
+++ b/drivers/media/usb/dvb-usb/dvb-usb-firmware.c
@@ -141,6 +141,8 @@ int dvb_usb_get_hexline(const struct firmware *fw, struct hexline *hx,
 
 	if (hx->type == 0x04) {
 		/* b[4] and b[5] are the Extended linear address record data field */
+		if (hx->len != 2)
+			return -EINVAL;
 		hx->addr |= (b[4] << 24) | (b[5] << 16);
 /*		hx->len -= 2;
 		data_offs += 2; */
diff --git a/drivers/media/usb/em28xx/em28xx-camera.c b/drivers/media/usb/em28xx/em28xx-camera.c
index b5f58dc6dd0fa4..bc55ad66658ce0 100644
--- a/drivers/media/usb/em28xx/em28xx-camera.c
+++ b/drivers/media/usb/em28xx/em28xx-camera.c
@@ -392,7 +392,7 @@ int em28xx_init_camera(struct em28xx *dev)
 		format.format.code = MEDIA_BUS_FMT_YUYV8_2X8;
 		format.format.width = 640;
 		format.format.height = 480;
-		v4l2_subdev_call(subdev, pad, set_fmt, NULL, &format);
+		v4l2_subdev_call(subdev, pad, set_fmt, NULL, NULL, &format);
 
 		/* NOTE: for UXGA=1600x1200 switch to 12MHz */
 		dev->board.xclk = EM28XX_XCLK_FREQUENCY_24MHZ;
diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c
index c418add65bb514..ba992b2055dce4 100644
--- a/drivers/media/usb/em28xx/em28xx-video.c
+++ b/drivers/media/usb/em28xx/em28xx-video.c
@@ -431,6 +431,7 @@ static void em2828X_decoder_set_std(struct em28xx *dev, v4l2_std_id norm)
 		} else if (INPUT(dev->ctl_input)->vmux == EM2828X_TELEVISION) {
 			em28xx_write_reg(dev, 0x7A00, 0x32);
 			em28xx_write_reg(dev, 0x7A03, 0x09);
+			em28xx_write_reg(dev, 0x7A07, 0x2f);
 			em28xx_write_reg(dev, 0x7A30, 0x2a);
 			em28xx_write_reg(dev, 0x7A80, 0x03);
 			em28xx_write_reg(dev, 0x7A20, 0x35);
@@ -1410,6 +1411,9 @@ static int em28xx_vb2_setup(struct em28xx *dev)
 	if (rc < 0)
 		return rc;
 
+	if (!em28xx_vbi_supported(dev))
+		return 0;
+
 	/* Setup Videobuf2 for VBI capture */
 	q = &v4l2->vb_vbiq;
 	q->type = V4L2_BUF_TYPE_VBI_CAPTURE;
@@ -2416,7 +2420,7 @@ static int em28xx_v4l2_fini(struct em28xx *dev)
 	if (video_is_registered(&v4l2->radio_dev)) {
 		dev_info(&dev->intf->dev, "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->radio_dev));
-		vb2_video_unregister_device(&v4l2->radio_dev);
+		video_unregister_device(&v4l2->radio_dev);
 	}
 	if (video_is_registered(&v4l2->vbi_dev)) {
 		dev_info(&dev->intf->dev, "V4L2 device %s deregistered\n",
@@ -3082,7 +3086,7 @@ static int em28xx_v4l2_init(struct em28xx *dev)
 		dev_info(&dev->intf->dev,
 			 "V4L2 device %s deregistered\n",
 			 video_device_node_name(&v4l2->radio_dev));
-		vb2_video_unregister_device(&v4l2->radio_dev);
+		video_unregister_device(&v4l2->radio_dev);
 	}
 	if (video_is_registered(&v4l2->vbi_dev)) {
 		dev_info(&dev->intf->dev,
diff --git a/drivers/media/usb/go7007/go7007-driver.c b/drivers/media/usb/go7007/go7007-driver.c
index 453ab5c3aa03a7..7db2440886b0c0 100644
--- a/drivers/media/usb/go7007/go7007-driver.c
+++ b/drivers/media/usb/go7007/go7007-driver.c
@@ -306,10 +306,9 @@ int go7007_register_encoder(struct go7007 *go, unsigned num_i2c_devs)
 	if (ret < 0)
 		goto err_free_controls;
 
-	if (go->board_info->flags & GO7007_BOARD_HAS_AUDIO) {
+	if ((go->board_info->flags & GO7007_BOARD_HAS_AUDIO) &&
+	    go7007_snd_init(go) == 0)
 		go->audio_enabled = 1;
-		go7007_snd_init(go);
-	}
 	return 0;
 
 err_free_controls:
diff --git a/drivers/media/usb/go7007/go7007-usb.c b/drivers/media/usb/go7007/go7007-usb.c
index c0cb92fa6ab941..dc5b8750a110b1 100644
--- a/drivers/media/usb/go7007/go7007-usb.c
+++ b/drivers/media/usb/go7007/go7007-usb.c
@@ -1029,10 +1029,16 @@ static const struct i2c_algorithm go7007_usb_algo = {
 	.functionality	= go7007_usb_functionality,
 };
 
+static const struct i2c_adapter_quirks go7007_usb_quirks = {
+	.max_write_len	= 12,
+	.max_read_len	= 15,
+};
+
 static struct i2c_adapter go7007_usb_adap_templ = {
 	.owner			= THIS_MODULE,
 	.name			= "WIS GO7007SB EZ-USB",
 	.algo			= &go7007_usb_algo,
+	.quirks			= &go7007_usb_quirks,
 };
 
 /********************* USB add/remove functions *********************/
@@ -1320,6 +1326,8 @@ static int go7007_usb_probe(struct usb_interface *intf,
 	return 0;
 
 allocfail:
+	if (go->i2c_adapter_online)
+		i2c_del_adapter(&go->i2c_adapter);
 	go7007_usb_release(go);
 	kfree(go);
 	return -ENOMEM;
diff --git a/drivers/media/usb/go7007/go7007-v4l2.c b/drivers/media/usb/go7007/go7007-v4l2.c
index 2087ffcb85a5a6..ab80247958de88 100644
--- a/drivers/media/usb/go7007/go7007-v4l2.c
+++ b/drivers/media/usb/go7007/go7007-v4l2.c
@@ -12,7 +12,6 @@
 #include <linux/unistd.h>
 #include <linux/time.h>
 #include <linux/vmalloc.h>
-#include <linux/pagemap.h>
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 #include <linux/uaccess.h>
@@ -252,7 +251,7 @@ static int set_capture_size(struct go7007 *go, struct v4l2_format *fmt, int try)
 		go->encoder_h_halve = 0;
 		go->encoder_v_halve = 0;
 		go->encoder_subsample = 0;
-		call_all(&go->v4l2_dev, pad, set_fmt, NULL, &format);
+		call_all(&go->v4l2_dev, pad, set_fmt, NULL, NULL, &format);
 	} else {
 		if (width <= sensor_width / 4) {
 			go->encoder_h_halve = 1;
diff --git a/drivers/media/usb/go7007/s2250-board.c b/drivers/media/usb/go7007/s2250-board.c
index d11f8e7236245d..a57ae661c02be2 100644
--- a/drivers/media/usb/go7007/s2250-board.c
+++ b/drivers/media/usb/go7007/s2250-board.c
@@ -410,6 +410,7 @@ static int s2250_s_ctrl(struct v4l2_ctrl *ctrl)
 }
 
 static int s2250_set_fmt(struct v4l2_subdev *sd,
+		const struct v4l2_subdev_client_info *ci,
 		struct v4l2_subdev_state *sd_state,
 		struct v4l2_subdev_format *format)
 {
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index 594d73e50b9fa4..d107b2c68173f1 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -19,7 +19,6 @@
 #include <linux/slab.h>
 #include <linux/mm.h>
 #include <linux/string.h>
-#include <linux/pagemap.h>
 #include <linux/io.h>
 #include <asm/page.h>
 #include <linux/uaccess.h>
@@ -1636,9 +1635,8 @@ void gspca_disconnect(struct usb_interface *intf)
 #endif
 
 	v4l2_device_disconnect(&gspca_dev->v4l2_dev);
-	video_unregister_device(&gspca_dev->vdev);
-
 	mutex_unlock(&gspca_dev->usb_lock);
+	vb2_video_unregister_device(&gspca_dev->vdev);
 
 	/* (this will call gspca_release() immediately or on last close) */
 	v4l2_device_put(&gspca_dev->v4l2_dev);
diff --git a/drivers/media/usb/gspca/ov519.c b/drivers/media/usb/gspca/ov519.c
index bffa94e76da5ed..57219a738c7369 100644
--- a/drivers/media/usb/gspca/ov519.c
+++ b/drivers/media/usb/gspca/ov519.c
@@ -13,7 +13,7 @@
  * Copyright (c) 1999-2006 Mark W. McClelland
  * Support for OV519, OV8610 Copyright (c) 2003 Joerg Heckenbach
  * Many improvements by Bret Wallach <bwallac1@san.rr.com>
- * Color fixes by by Orion Sky Lawlor <olawlor@acm.org> (2/26/2000)
+ * Color fixes by Orion Sky Lawlor <olawlor@acm.org> (2/26/2000)
  * OV7620 fixes by Charl P. Botha <cpbotha@ieee.org>
  * Changes by Claudio Matsuoka <claudio@conectiva.com>
  *
diff --git a/drivers/media/usb/gspca/w996Xcf.c b/drivers/media/usb/gspca/w996Xcf.c
index 79baa0c1a031f4..700c39fe307cb5 100644
--- a/drivers/media/usb/gspca/w996Xcf.c
+++ b/drivers/media/usb/gspca/w996Xcf.c
@@ -407,7 +407,7 @@ static void w9968cf_set_crop_window(struct sd *sd)
 
 	if (sd->sensor == SEN_OV7620) {
 		/*
-		 * Sigh, this is dependend on the clock / framerate changes
+		 * Sigh, this is dependent on the clock / framerate changes
 		 * made by the frequency control, sick.
 		 *
 		 * Note we cannot use v4l2_ctrl_g_ctrl here, as we get called
diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c
index a15829a60e8833..70fd95f3e97dda 100644
--- a/drivers/media/usb/hackrf/hackrf.c
+++ b/drivers/media/usb/hackrf/hackrf.c
@@ -665,7 +665,7 @@ static int hackrf_free_urbs(struct hackrf_dev *dev)
 
 static int hackrf_alloc_urbs(struct hackrf_dev *dev, bool rcv)
 {
-	int i, j;
+	int i;
 	unsigned int pipe;
 	usb_complete_t complete;
 
@@ -681,11 +681,8 @@ static int hackrf_alloc_urbs(struct hackrf_dev *dev, bool rcv)
 	for (i = 0; i < MAX_BULK_BUFS; i++) {
 		dev_dbg(dev->dev, "alloc urb=%d\n", i);
 		dev->urb_list[i] = usb_alloc_urb(0, GFP_KERNEL);
-		if (!dev->urb_list[i]) {
-			for (j = 0; j < i; j++)
-				usb_free_urb(dev->urb_list[j]);
+		if (!dev->urb_list[i])
 			return -ENOMEM;
-		}
 		usb_fill_bulk_urb(dev->urb_list[i],
 				dev->udev,
 				pipe,
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
index 3c270ef0075261..c749ef7e490a87 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
@@ -2924,7 +2924,7 @@ static void pvr2_subdev_update(struct pvr2_hdw *hdw)
 		format.format.code = MEDIA_BUS_FMT_FIXED;
 		pvr2_trace(PVR2_TRACE_CHIPS, "subdev v4l2 set_size(%dx%d)",
 			   format.format.width, format.format.height);
-		v4l2_device_call_all(&hdw->v4l2_dev, 0, pad, set_fmt,
+		v4l2_device_call_all(&hdw->v4l2_dev, 0, pad, set_fmt, NULL,
 				     NULL, &format);
 	}
 
@@ -3669,7 +3669,9 @@ static int pvr2_send_request_ex(struct pvr2_hdw *hdw,
 			pvr2_trace(
 				PVR2_TRACE_ERROR_LEGS,
 				"Invalid write control endpoint");
-			return -EINVAL;
+			hdw->ctl_write_pend_flag = 0;
+			status = -EINVAL;
+			goto done;
 		}
 		status = usb_submit_urb(hdw->ctl_write_urb,GFP_KERNEL);
 		if (status < 0) {
@@ -3699,7 +3701,13 @@ status);
 			pvr2_trace(
 				PVR2_TRACE_ERROR_LEGS,
 				"Invalid read control endpoint");
-			return -EINVAL;
+			hdw->ctl_read_pend_flag = 0;
+			status = -EINVAL;
+			if (hdw->ctl_write_pend_flag) {
+				usb_unlink_urb(hdw->ctl_write_urb);
+				wait_for_completion(&hdw->ctl_done);
+			}
+			goto done;
 		}
 		status = usb_submit_urb(hdw->ctl_read_urb,GFP_KERNEL);
 		if (status < 0) {
diff --git a/drivers/media/usb/usbtv/usbtv-core.c b/drivers/media/usb/usbtv/usbtv-core.c
index 4f10f6613bc4cc..89c3424e76852a 100644
--- a/drivers/media/usb/usbtv/usbtv-core.c
+++ b/drivers/media/usb/usbtv/usbtv-core.c
@@ -139,8 +139,6 @@ static void usbtv_disconnect(struct usb_interface *intf)
 	usbtv_audio_free(usbtv);
 	usbtv_video_free(usbtv);
 
-	usbtv->udev = NULL;
-
 	/* the usbtv structure will be deallocated when v4l2 will be
 	   done using it */
 	v4l2_device_put(&usbtv->v4l2_dev);
diff --git a/drivers/media/usb/usbtv/usbtv-video.c b/drivers/media/usb/usbtv/usbtv-video.c
index 92bc7a2509c36c..2a91af8d1e30cc 100644
--- a/drivers/media/usb/usbtv/usbtv-video.c
+++ b/drivers/media/usb/usbtv/usbtv-video.c
@@ -966,5 +966,9 @@ void usbtv_video_free(struct usbtv *usbtv)
 	vb2_video_unregister_device(&usbtv->vdev);
 	v4l2_device_disconnect(&usbtv->v4l2_dev);
 
+	mutex_lock(&usbtv->v4l2_lock);
+	usbtv->udev = NULL;
+	mutex_unlock(&usbtv->v4l2_lock);
+
 	v4l2_device_put(&usbtv->v4l2_dev);
 }
diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c
index 65db7340ad3847..c825820fd4819f 100644
--- a/drivers/media/v4l2-core/v4l2-common.c
+++ b/drivers/media/v4l2-core/v4l2-common.c
@@ -333,6 +333,7 @@ const struct v4l2_format_info *v4l2_format_info(u32 format)
 		{ .format = V4L2_PIX_FMT_YVU420,  .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
 		{ .format = V4L2_PIX_FMT_YUV422P, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_GREY,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_Y12,     .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
 
 		/* Tiled YUV formats */
 		{ .format = V4L2_PIX_FMT_NV12_4L4, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 2 },
@@ -537,16 +538,8 @@ int v4l2_fill_pixfmt_mp_aligned(struct v4l2_pix_format_mplane *pixfmt,
 }
 EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt_mp_aligned);
 
-int v4l2_fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt,
-			u32 pixelformat, u32 width, u32 height)
-{
-	return v4l2_fill_pixfmt_mp_aligned(pixfmt, pixelformat,
-					   width, height, 1);
-}
-EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt_mp);
-
-int v4l2_fill_pixfmt(struct v4l2_pix_format *pixfmt, u32 pixelformat,
-		     u32 width, u32 height)
+int v4l2_fill_pixfmt_aligned(struct v4l2_pix_format *pixfmt, u32 pixelformat,
+			     u32 width, u32 height, u8 stride_alignment)
 {
 	const struct v4l2_format_info *info;
 	int i;
@@ -562,15 +555,17 @@ int v4l2_fill_pixfmt(struct v4l2_pix_format *pixfmt, u32 pixelformat,
 	pixfmt->width = width;
 	pixfmt->height = height;
 	pixfmt->pixelformat = pixelformat;
-	pixfmt->bytesperline = v4l2_format_plane_stride(info, 0, width, 1);
+	pixfmt->bytesperline = v4l2_format_plane_stride(info, 0, width,
+							stride_alignment);
 	pixfmt->sizeimage = 0;
 
 	for (i = 0; i < info->comp_planes; i++)
 		pixfmt->sizeimage +=
-			v4l2_format_plane_size(info, i, width, height, 1);
+			v4l2_format_plane_size(info, i, width, height,
+					       stride_alignment);
 	return 0;
 }
-EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt);
+EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt_aligned);
 
 #ifdef CONFIG_MEDIA_CONTROLLER
 static s64 v4l2_get_link_freq_ctrl(struct v4l2_ctrl_handler *handler,
diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c
index 648b88c868bc6d..6e751a043a2106 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls-core.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c
@@ -115,6 +115,7 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 	struct v4l2_ctrl_fwht_params *p_fwht_params;
 	struct v4l2_ctrl_h264_scaling_matrix *p_h264_scaling_matrix;
 	struct v4l2_ctrl_av1_sequence *p_av1_sequence;
+	struct v4l2_ctrl_hevc_sps *p_hevc_sps;
 	void *p = ptr.p + idx * ctrl->elem_size;
 
 	if (ctrl->p_def.p_const)
@@ -188,6 +189,12 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 		 */
 		memset(p_h264_scaling_matrix, 16, sizeof(*p_h264_scaling_matrix));
 		break;
+	case V4L2_CTRL_TYPE_HEVC_SPS:
+		p_hevc_sps = p;
+
+		/* 4:2:0 */
+		p_hevc_sps->chroma_format_idc = 1;
+		break;
 	}
 }
 
@@ -2554,7 +2561,10 @@ void v4l2_ctrl_cluster(unsigned ncontrols, struct v4l2_ctrl **controls)
 	int i;
 
 	/* The first control is the master control and it must not be NULL */
-	if (WARN_ON(ncontrols == 0 || controls[0] == NULL))
+	if (WARN_ON(ncontrols == 0))
+		return;
+
+	if (!controls[0])
 		return;
 
 	for (i = 0; i < ncontrols; i++) {
@@ -2576,8 +2586,13 @@ void v4l2_ctrl_auto_cluster(unsigned ncontrols, struct v4l2_ctrl **controls,
 	u32 flag = 0;
 	int i;
 
+	if (WARN_ON(ncontrols <= 1))
+		return;
+
+	if (!master)
+		return;
+
 	v4l2_ctrl_cluster(ncontrols, controls);
-	WARN_ON(ncontrols <= 1);
 	WARN_ON(manual_val < master->minimum || manual_val > master->maximum);
 	WARN_ON(set_volatile && !has_op(master, g_volatile_ctrl));
 	master->is_auto = true;
diff --git a/drivers/media/v4l2-core/v4l2-mc.c b/drivers/media/v4l2-core/v4l2-mc.c
index 937d358697e19a..5d7fcd67dc42ed 100644
--- a/drivers/media/v4l2-core/v4l2-mc.c
+++ b/drivers/media/v4l2-core/v4l2-mc.c
@@ -324,12 +324,10 @@ EXPORT_SYMBOL_GPL(v4l_vb2q_enable_media_source);
 int v4l2_create_fwnode_links_to_pad(struct v4l2_subdev *src_sd,
 				    struct media_pad *sink, u32 flags)
 {
-	struct fwnode_handle *endpoint;
-
 	if (!(sink->flags & MEDIA_PAD_FL_SINK))
 		return -EINVAL;
 
-	fwnode_graph_for_each_endpoint(src_sd->fwnode, endpoint) {
+	fwnode_graph_for_each_endpoint_scoped(src_sd->fwnode, endpoint) {
 		struct fwnode_handle *remote_ep;
 		int src_idx, sink_idx, ret;
 		struct media_pad *src;
@@ -397,7 +395,6 @@ int v4l2_create_fwnode_links_to_pad(struct v4l2_subdev *src_sd,
 				src_sd->entity.name, src_idx,
 				sink->entity->name, sink_idx, ret);
 
-			fwnode_handle_put(endpoint);
 			return ret;
 		}
 	}
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index e9f81b9be9e24b..a07d77e584c311 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -244,20 +244,44 @@ static inline int check_format(struct v4l2_subdev *sd,
 	       check_state(sd, state, format->which, format->pad, format->stream);
 }
 
+#define do_subdev_call(sd, check, o, f, args...)	\
+	(!(sd)->ops->o->f ? -ENOIOCTLCMD : (check) ? :	\
+	 (sd)->ops->o->f(sd, ##args))
+
 static int call_get_fmt(struct v4l2_subdev *sd,
 			struct v4l2_subdev_state *state,
 			struct v4l2_subdev_format *format)
 {
-	return check_format(sd, state, format) ? :
-	       sd->ops->pad->get_fmt(sd, state, format);
+	return do_subdev_call(sd, check_format(sd, state, format), pad, get_fmt,
+			      state, format);
 }
 
 static int call_set_fmt(struct v4l2_subdev *sd,
+			const struct v4l2_subdev_client_info *ci,
 			struct v4l2_subdev_state *state,
 			struct v4l2_subdev_format *format)
 {
-	return check_format(sd, state, format) ? :
-	       sd->ops->pad->set_fmt(sd, state, format);
+	int ret;
+
+	if (!sd->ops->pad->set_fmt && !sd->ops->pad->get_fmt)
+		return -ENOIOCTLCMD;
+
+	ret = check_format(sd, state, format);
+	if (ret)
+		return ret;
+
+	if (sd->ops->pad->set_fmt)
+		return sd->ops->pad->set_fmt(sd, ci, state, format);
+
+	return sd->ops->pad->get_fmt(sd, state, format);
+}
+
+static int check_which_pad_state(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state, u32 which,
+				 u32 pad, u32 stream)
+{
+	return check_which(which) ? : check_pad(sd, pad) ? :
+		check_state(sd, state, which, pad, stream);
 }
 
 static int call_enum_mbus_code(struct v4l2_subdev *sd,
@@ -267,9 +291,9 @@ static int call_enum_mbus_code(struct v4l2_subdev *sd,
 	if (!code)
 		return -EINVAL;
 
-	return check_which(code->which) ? : check_pad(sd, code->pad) ? :
-	       check_state(sd, state, code->which, code->pad, code->stream) ? :
-	       sd->ops->pad->enum_mbus_code(sd, state, code);
+	return do_subdev_call(sd, check_which_pad_state(sd, state, code->which,
+							code->pad, code->stream),
+			      pad, enum_mbus_code, state, code);
 }
 
 static int call_enum_frame_size(struct v4l2_subdev *sd,
@@ -279,9 +303,9 @@ static int call_enum_frame_size(struct v4l2_subdev *sd,
 	if (!fse)
 		return -EINVAL;
 
-	return check_which(fse->which) ? : check_pad(sd, fse->pad) ? :
-	       check_state(sd, state, fse->which, fse->pad, fse->stream) ? :
-	       sd->ops->pad->enum_frame_size(sd, state, fse);
+	return do_subdev_call(sd, check_which_pad_state(sd, state, fse->which,
+							fse->pad, fse->stream),
+			      pad, enum_frame_size, state, fse);
 }
 
 static int call_enum_frame_interval(struct v4l2_subdev *sd,
@@ -291,9 +315,9 @@ static int call_enum_frame_interval(struct v4l2_subdev *sd,
 	if (!fie)
 		return -EINVAL;
 
-	return check_which(fie->which) ? : check_pad(sd, fie->pad) ? :
-	       check_state(sd, state, fie->which, fie->pad, fie->stream) ? :
-	       sd->ops->pad->enum_frame_interval(sd, state, fie);
+	return do_subdev_call(sd, check_which_pad_state(sd, state, fie->which,
+							fie->pad, fie->stream),
+			      pad, enum_frame_interval, state, fie);
 }
 
 static inline int check_selection(struct v4l2_subdev *sd,
@@ -308,19 +332,21 @@ static inline int check_selection(struct v4l2_subdev *sd,
 }
 
 static int call_get_selection(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *state,
 			      struct v4l2_subdev_selection *sel)
 {
-	return check_selection(sd, state, sel) ? :
-	       sd->ops->pad->get_selection(sd, state, sel);
+	return do_subdev_call(sd, check_selection(sd, state, sel),
+			      pad, get_selection, ci, state, sel);
 }
 
 static int call_set_selection(struct v4l2_subdev *sd,
+			      const struct v4l2_subdev_client_info *ci,
 			      struct v4l2_subdev_state *state,
 			      struct v4l2_subdev_selection *sel)
 {
-	return check_selection(sd, state, sel) ? :
-	       sd->ops->pad->set_selection(sd, state, sel);
+	return do_subdev_call(sd, check_selection(sd, state, sel),
+			      pad, set_selection, ci, state, sel);
 }
 
 static inline int check_frame_interval(struct v4l2_subdev *sd,
@@ -338,16 +364,16 @@ static int call_get_frame_interval(struct v4l2_subdev *sd,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_frame_interval *fi)
 {
-	return check_frame_interval(sd, state, fi) ? :
-	       sd->ops->pad->get_frame_interval(sd, state, fi);
+	return do_subdev_call(sd, check_frame_interval(sd, state, fi),
+			      pad, get_frame_interval, state, fi);
 }
 
 static int call_set_frame_interval(struct v4l2_subdev *sd,
 				   struct v4l2_subdev_state *state,
 				   struct v4l2_subdev_frame_interval *fi)
 {
-	return check_frame_interval(sd, state, fi) ? :
-	       sd->ops->pad->set_frame_interval(sd, state, fi);
+	return do_subdev_call(sd, check_frame_interval(sd, state, fi),
+			      pad, set_frame_interval, state, fi);
 }
 
 static int call_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
@@ -361,6 +387,9 @@ static int call_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
 		return -EOPNOTSUPP;
 #endif
 
+	if (!sd->ops->pad->get_frame_desc)
+		return -ENOIOCTLCMD;
+
 	memset(fd, 0, sizeof(*fd));
 
 	ret = sd->ops->pad->get_frame_desc(sd, pad, fd);
@@ -405,12 +434,12 @@ static inline int check_edid(struct v4l2_subdev *sd,
 
 static int call_get_edid(struct v4l2_subdev *sd, struct v4l2_subdev_edid *edid)
 {
-	return check_edid(sd, edid) ? : sd->ops->pad->get_edid(sd, edid);
+	return do_subdev_call(sd, check_edid(sd, edid), pad, get_edid, edid);
 }
 
 static int call_set_edid(struct v4l2_subdev *sd, struct v4l2_subdev_edid *edid)
 {
-	return check_edid(sd, edid) ? : sd->ops->pad->set_edid(sd, edid);
+	return do_subdev_call(sd, check_edid(sd, edid), pad, set_edid, edid);
 }
 
 static int call_s_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
@@ -419,8 +448,8 @@ static int call_s_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
 	if (!timings)
 		return -EINVAL;
 
-	return check_pad(sd, pad) ? :
-	       sd->ops->pad->s_dv_timings(sd, pad, timings);
+	return do_subdev_call(sd, check_pad(sd, pad),
+			      pad, s_dv_timings, pad, timings);
 }
 
 static int call_g_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
@@ -429,8 +458,8 @@ static int call_g_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
 	if (!timings)
 		return -EINVAL;
 
-	return check_pad(sd, pad) ? :
-	       sd->ops->pad->g_dv_timings(sd, pad, timings);
+	return do_subdev_call(sd,  check_pad(sd, pad),
+			      pad, g_dv_timings, pad, timings);
 }
 
 static int call_query_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
@@ -439,8 +468,8 @@ static int call_query_dv_timings(struct v4l2_subdev *sd, unsigned int pad,
 	if (!timings)
 		return -EINVAL;
 
-	return check_pad(sd, pad) ? :
-	       sd->ops->pad->query_dv_timings(sd, pad, timings);
+	return do_subdev_call(sd, check_pad(sd, pad),
+			      pad, query_dv_timings, pad, timings);
 }
 
 static int call_dv_timings_cap(struct v4l2_subdev *sd,
@@ -449,8 +478,8 @@ static int call_dv_timings_cap(struct v4l2_subdev *sd,
 	if (!cap)
 		return -EINVAL;
 
-	return check_pad(sd, cap->pad) ? :
-	       sd->ops->pad->dv_timings_cap(sd, cap);
+	return do_subdev_call(sd, check_pad(sd, cap->pad),
+			      pad, dv_timings_cap, cap);
 }
 
 static int call_enum_dv_timings(struct v4l2_subdev *sd,
@@ -459,8 +488,8 @@ static int call_enum_dv_timings(struct v4l2_subdev *sd,
 	if (!dvt)
 		return -EINVAL;
 
-	return check_pad(sd, dvt->pad) ? :
-	       sd->ops->pad->enum_dv_timings(sd, dvt);
+	return do_subdev_call(sd, check_pad(sd, dvt->pad),
+			      pad, enum_dv_timings, dvt);
 }
 
 static int call_get_mbus_config(struct v4l2_subdev *sd, unsigned int pad,
@@ -468,14 +497,17 @@ static int call_get_mbus_config(struct v4l2_subdev *sd, unsigned int pad,
 {
 	memset(config, 0, sizeof(*config));
 
-	return check_pad(sd, pad) ? :
-	       sd->ops->pad->get_mbus_config(sd, pad, config);
+	return do_subdev_call(sd, check_pad(sd, pad), pad, get_mbus_config,
+			      pad, config);
 }
 
 static int call_s_stream(struct v4l2_subdev *sd, int enable)
 {
 	int ret;
 
+	if (!sd->ops->video->s_stream)
+		return -ENOIOCTLCMD;
+
 	/*
 	 * The .s_stream() operation must never be called to start or stop an
 	 * already started or stopped subdev. Catch offenders but don't return
@@ -509,7 +541,7 @@ static int call_s_stream(struct v4l2_subdev *sd, int enable)
  * wrapper handles the case where the caller does not provide the called
  * subdev's state. This should be removed when all the callers are fixed.
  */
-#define DEFINE_STATE_WRAPPER(f, arg_type)                                  \
+#define DEFINE_STATE_WRAPPER(f, arg_type)				   \
 	static int call_##f##_state(struct v4l2_subdev *sd,                \
 				    struct v4l2_subdev_state *_state,      \
 				    arg_type *arg)                         \
@@ -523,10 +555,25 @@ static int call_s_stream(struct v4l2_subdev *sd, int enable)
 			v4l2_subdev_unlock_state(state);                   \
 		return ret;                                                \
 	}
+#define DEFINE_CI_STATE_WRAPPER(f, arg_type)                               \
+	static int call_##f##_state(struct v4l2_subdev *sd,		   \
+				    const struct v4l2_subdev_client_info *ci, \
+				    struct v4l2_subdev_state *_state,      \
+				    arg_type *arg) \
+	{                                                                  \
+		struct v4l2_subdev_state *state = _state;                  \
+		int ret;                                                   \
+		if (!_state)                                               \
+			state = v4l2_subdev_lock_and_get_active_state(sd); \
+		ret = call_##f(sd, ci, state, arg);			   \
+		if (!_state && state)                                      \
+			v4l2_subdev_unlock_state(state);                   \
+		return ret;                                                \
+	}
 
 #else /* CONFIG_MEDIA_CONTROLLER */
 
-#define DEFINE_STATE_WRAPPER(f, arg_type)                            \
+#define DEFINE_STATE_WRAPPER(f, arg_type)			     \
 	static int call_##f##_state(struct v4l2_subdev *sd,          \
 				    struct v4l2_subdev_state *state, \
 				    arg_type *arg)                   \
@@ -534,15 +581,24 @@ static int call_s_stream(struct v4l2_subdev *sd, int enable)
 		return call_##f(sd, state, arg);                     \
 	}
 
+#define DEFINE_CI_STATE_WRAPPER(f, arg_type)                            \
+	static int call_##f##_state(struct v4l2_subdev *sd,          \
+				    const struct v4l2_subdev_client_info *ci, \
+				    struct v4l2_subdev_state *state, \
+				    arg_type *arg)                   \
+	{                                                            \
+		return call_##f(sd, ci, state, arg);		     \
+	}
+
 #endif /* CONFIG_MEDIA_CONTROLLER */
 
 DEFINE_STATE_WRAPPER(get_fmt, struct v4l2_subdev_format);
-DEFINE_STATE_WRAPPER(set_fmt, struct v4l2_subdev_format);
+DEFINE_CI_STATE_WRAPPER(set_fmt, struct v4l2_subdev_format);
 DEFINE_STATE_WRAPPER(enum_mbus_code, struct v4l2_subdev_mbus_code_enum);
 DEFINE_STATE_WRAPPER(enum_frame_size, struct v4l2_subdev_frame_size_enum);
 DEFINE_STATE_WRAPPER(enum_frame_interval, struct v4l2_subdev_frame_interval_enum);
-DEFINE_STATE_WRAPPER(get_selection, struct v4l2_subdev_selection);
-DEFINE_STATE_WRAPPER(set_selection, struct v4l2_subdev_selection);
+DEFINE_CI_STATE_WRAPPER(get_selection, struct v4l2_subdev_selection);
+DEFINE_CI_STATE_WRAPPER(set_selection, struct v4l2_subdev_selection);
 
 static const struct v4l2_subdev_pad_ops v4l2_subdev_call_pad_wrappers = {
 	.get_fmt		= call_get_fmt_state,
@@ -611,7 +667,7 @@ subdev_ioctl_get_state(struct v4l2_subdev *sd, struct v4l2_subdev_fh *subdev_fh,
 	case VIDIOC_SUBDEV_S_FRAME_INTERVAL: {
 		struct v4l2_subdev_frame_interval *fi = arg;
 
-		if (!(subdev_fh->client_caps &
+		if (!(subdev_fh->ci.caps &
 		      V4L2_SUBDEV_CLIENT_CAP_INTERVAL_USES_WHICH))
 			fi->which = V4L2_SUBDEV_FORMAT_ACTIVE;
 
@@ -650,7 +706,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	struct v4l2_subdev_fh *subdev_fh = to_v4l2_subdev_fh(vfh);
 	bool ro_subdev = test_bit(V4L2_FL_SUBDEV_RO_DEVNODE, &vdev->flags);
 	bool streams_subdev = sd->flags & V4L2_SUBDEV_FL_STREAMS;
-	bool client_supports_streams = subdev_fh->client_caps &
+	bool client_supports_streams = subdev_fh->ci.caps &
 				       V4L2_SUBDEV_CLIENT_CAP_STREAMS;
 	int rval;
 
@@ -817,7 +873,8 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 
 		memset(format->reserved, 0, sizeof(format->reserved));
 		memset(format->format.reserved, 0, sizeof(format->format.reserved));
-		return v4l2_subdev_call(sd, pad, set_fmt, state, format);
+		return v4l2_subdev_call(sd, pad, set_fmt, &subdev_fh->ci, state,
+					format);
 	}
 
 	case VIDIOC_SUBDEV_G_CROP: {
@@ -834,8 +891,8 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		sel.stream = crop->stream;
 		sel.target = V4L2_SEL_TGT_CROP;
 
-		rval = v4l2_subdev_call(
-			sd, pad, get_selection, state, &sel);
+		rval = v4l2_subdev_call(sd, pad, get_selection, &subdev_fh->ci,
+					state, &sel);
 
 		crop->rect = sel.r;
 
@@ -860,8 +917,8 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		sel.target = V4L2_SEL_TGT_CROP;
 		sel.r = crop->rect;
 
-		rval = v4l2_subdev_call(
-			sd, pad, set_selection, state, &sel);
+		rval = v4l2_subdev_call(sd, pad, set_selection, &subdev_fh->ci,
+					state, &sel);
 
 		crop->rect = sel.r;
 
@@ -931,8 +988,8 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 			sel->stream = 0;
 
 		memset(sel->reserved, 0, sizeof(sel->reserved));
-		return v4l2_subdev_call(
-			sd, pad, get_selection, state, sel);
+		return v4l2_subdev_call(sd, pad, get_selection, &subdev_fh->ci,
+					state, sel);
 	}
 
 	case VIDIOC_SUBDEV_S_SELECTION: {
@@ -945,8 +1002,8 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 			sel->stream = 0;
 
 		memset(sel->reserved, 0, sizeof(sel->reserved));
-		return v4l2_subdev_call(
-			sd, pad, set_selection, state, sel);
+		return v4l2_subdev_call(sd, pad, set_selection, &subdev_fh->ci,
+					state, sel);
 	}
 
 	case VIDIOC_G_EDID: {
@@ -1117,7 +1174,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 	case VIDIOC_SUBDEV_G_CLIENT_CAP: {
 		struct v4l2_subdev_client_capability *client_cap = arg;
 
-		client_cap->capabilities = subdev_fh->client_caps;
+		client_cap->capabilities = subdev_fh->ci.caps;
 
 		return 0;
 	}
@@ -1137,7 +1194,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		client_cap->capabilities &= (V4L2_SUBDEV_CLIENT_CAP_STREAMS |
 					     V4L2_SUBDEV_CLIENT_CAP_INTERVAL_USES_WHICH);
 
-		subdev_fh->client_caps = client_cap->capabilities;
+		subdev_fh->ci.caps = client_cap->capabilities;
 
 		return 0;
 	}
diff --git a/drivers/media/v4l2-core/v4l2-vp9.c b/drivers/media/v4l2-core/v4l2-vp9.c
index 859589f1fd35f5..e965ffbd9b8aa2 100644
--- a/drivers/media/v4l2-core/v4l2-vp9.c
+++ b/drivers/media/v4l2-core/v4l2-vp9.c
@@ -1582,25 +1582,25 @@ static inline u8 noncoef_merge_prob(u8 pre_prob, u32 ct0, u32 ct1)
  *	merge_prob(p[9], c[9], [10])
  */
 
-static inline void merge_probs_variant_a(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
+static noinline_for_stack void merge_probs_variant_a(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
 {
 	p[1] = merge_prob(p[1], c[0], c[1] + c[2], count_sat, update_factor);
 	p[2] = merge_prob(p[2], c[1], c[2], count_sat, update_factor);
 }
 
-static inline void merge_probs_variant_b(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
+static noinline_for_stack void merge_probs_variant_b(u8 *p, const u32 *c, u16 count_sat, u32 update_factor)
 {
 	p[0] = merge_prob(p[0], c[0], c[1], count_sat, update_factor);
 }
 
-static inline void merge_probs_variant_c(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_c(u8 *p, const u32 *c)
 {
 	p[0] = noncoef_merge_prob(p[0], c[2], c[1] + c[0] + c[3]);
 	p[1] = noncoef_merge_prob(p[1], c[0], c[1] + c[3]);
 	p[2] = noncoef_merge_prob(p[2], c[1], c[3]);
 }
 
-static void merge_probs_variant_d(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_d(u8 *p, const u32 *c)
 {
 	u32 sum = 0, s2;
 
@@ -1624,20 +1624,20 @@ static void merge_probs_variant_d(u8 *p, const u32 *c)
 	p[8] = noncoef_merge_prob(p[8], c[6], c[7]);
 }
 
-static inline void merge_probs_variant_e(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_e(u8 *p, const u32 *c)
 {
 	p[0] = noncoef_merge_prob(p[0], c[0], c[1] + c[2] + c[3]);
 	p[1] = noncoef_merge_prob(p[1], c[1], c[2] + c[3]);
 	p[2] = noncoef_merge_prob(p[2], c[2], c[3]);
 }
 
-static inline void merge_probs_variant_f(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_f(u8 *p, const u32 *c)
 {
 	p[0] = noncoef_merge_prob(p[0], c[0], c[1] + c[2]);
 	p[1] = noncoef_merge_prob(p[1], c[1], c[2]);
 }
 
-static void merge_probs_variant_g(u8 *p, const u32 *c)
+static noinline_for_stack void merge_probs_variant_g(u8 *p, const u32 *c)
 {
 	u32 sum;
 
@@ -1659,12 +1659,12 @@ static void merge_probs_variant_g(u8 *p, const u32 *c)
 }
 
 /* 8.4.3 Coefficient probability adaptation process */
-static inline void adapt_probs_variant_a_coef(u8 *p, const u32 *c, u32 update_factor)
+static noinline_for_stack void adapt_probs_variant_a_coef(u8 *p, const u32 *c, u32 update_factor)
 {
 	merge_probs_variant_a(p, c, 24, update_factor);
 }
 
-static inline void adapt_probs_variant_b_coef(u8 *p, const u32 *c, u32 update_factor)
+static noinline_for_stack void adapt_probs_variant_b_coef(u8 *p, const u32 *c, u32 update_factor)
 {
 	merge_probs_variant_b(p, c, 24, update_factor);
 }
@@ -1724,33 +1724,33 @@ static inline void adapt_probs_variant_b(u8 *p, const u32 *c)
 	merge_probs_variant_b(p, c, 20, 128);
 }
 
-static inline void adapt_probs_variant_c(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_c(u8 *p, const u32 *c)
 {
 	merge_probs_variant_c(p, c);
 }
 
-static inline void adapt_probs_variant_d(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_d(u8 *p, const u32 *c)
 {
 	merge_probs_variant_d(p, c);
 }
 
-static inline void adapt_probs_variant_e(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_e(u8 *p, const u32 *c)
 {
 	merge_probs_variant_e(p, c);
 }
 
-static inline void adapt_probs_variant_f(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_f(u8 *p, const u32 *c)
 {
 	merge_probs_variant_f(p, c);
 }
 
-static inline void adapt_probs_variant_g(u8 *p, const u32 *c)
+static noinline_for_stack void adapt_probs_variant_g(u8 *p, const u32 *c)
 {
 	merge_probs_variant_g(p, c);
 }
 
 /* 8.4.4 Non coefficient probability adaptation process, adapt_prob() */
-static inline u8 adapt_prob(u8 prob, const u32 counts[2])
+static noinline_for_stack u8 adapt_prob(u8 prob, const u32 counts[2])
 {
 	return noncoef_merge_prob(prob, counts[0], counts[1]);
 }
diff --git a/drivers/memory/brcmstb_dpfe.c b/drivers/memory/brcmstb_dpfe.c
index 08d9e05b1b3383..04ae3cfb449e45 100644
--- a/drivers/memory/brcmstb_dpfe.c
+++ b/drivers/memory/brcmstb_dpfe.c
@@ -182,6 +182,8 @@ struct brcmstb_dpfe_priv {
 	void __iomem *regs;
 	void __iomem *dmem;
 	void __iomem *imem;
+	resource_size_t regs_size;
+	resource_size_t dmem_size;
 	struct device *dev;
 	const struct dpfe_api *dpfe_api;
 	struct mutex lock;
@@ -401,9 +403,14 @@ static void __iomem *get_msg_ptr(struct brcmstb_dpfe_priv *priv, u32 response,
 	 */
 	switch (msg_type) {
 	case 1:
+		if (DCPU_MSG_RAM_START + offset + DRAM_VENDOR_ERROR +
+		    sizeof(u32) > priv->regs_size)
+			goto bad_offset;
 		ptr = priv->regs + DCPU_MSG_RAM_START + offset;
 		break;
 	case 0:
+		if (offset + DRAM_VENDOR_ERROR + sizeof(u32) > priv->dmem_size)
+			goto bad_offset;
 		ptr = priv->dmem + offset;
 		break;
 	default:
@@ -415,6 +422,12 @@ static void __iomem *get_msg_ptr(struct brcmstb_dpfe_priv *priv, u32 response,
 	}
 
 	return ptr;
+
+bad_offset:
+	dev_err(priv->dev, "DCPU returned out-of-range offset %#x\n", offset);
+	if (buf && size)
+		*size = sprintf(buf, "ERROR: DCPU offset out of range\n");
+	return NULL;
 }
 
 static void __finalize_command(struct brcmstb_dpfe_priv *priv)
@@ -490,6 +503,8 @@ static int __send_command(struct brcmstb_dpfe_priv *priv, unsigned int cmd,
 		for (i = 0; i < MSG_FIELD_MAX; i++)
 			result[i] = readl_relaxed(regs + DCPU_MSG_RAM(i));
 		chksum_idx = result[MSG_ARG_COUNT] + MSG_ARG_COUNT + 1;
+		if (chksum_idx >= MSG_FIELD_MAX)
+			ret = -EINVAL;
 	}
 
 	/* Tell DCPU we are done */
@@ -518,9 +533,15 @@ static int __verify_firmware(struct init_data *init,
 			     const struct firmware *fw)
 {
 	const struct dpfe_firmware_header *header = (void *)fw->data;
-	unsigned int dmem_size, imem_size, total_size;
+	unsigned int dmem_size, imem_size;
 	bool is_big_endian = false;
 	const u32 *chksum_ptr;
+	size_t payload_size;
+
+	if (fw->size < sizeof(*header) + sizeof(*chksum_ptr))
+		return ERR_INVALID_SIZE;
+
+	payload_size = fw->size - sizeof(*header) - sizeof(*chksum_ptr);
 
 	if (header->magic == DPFE_BE_MAGIC)
 		is_big_endian = true;
@@ -539,13 +560,8 @@ static int __verify_firmware(struct init_data *init,
 	if ((dmem_size % sizeof(u32)) != 0 || (imem_size % sizeof(u32)) != 0)
 		return ERR_INVALID_SIZE;
 
-	/*
-	 * The header + the data section + the instruction section + the
-	 * checksum must be equal to the total firmware size.
-	 */
-	total_size = dmem_size + imem_size + sizeof(*header) +
-		sizeof(*chksum_ptr);
-	if (total_size != fw->size)
+	/* The data and instruction sections must fill the payload exactly. */
+	if (dmem_size > payload_size || imem_size != payload_size - dmem_size)
 		return ERR_INVALID_SIZE;
 
 	/* The checksum comes at the very end. */
@@ -858,6 +874,7 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct brcmstb_dpfe_priv *priv;
+	struct resource *res;
 	int ret;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
@@ -869,17 +886,21 @@ static int brcmstb_dpfe_probe(struct platform_device *pdev)
 	mutex_init(&priv->lock);
 	platform_set_drvdata(pdev, priv);
 
-	priv->regs = devm_platform_ioremap_resource_byname(pdev, "dpfe-cpu");
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dpfe-cpu");
+	priv->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(priv->regs)) {
 		dev_err(dev, "couldn't map DCPU registers\n");
 		return -ENODEV;
 	}
+	priv->regs_size = resource_size(res);
 
-	priv->dmem = devm_platform_ioremap_resource_byname(pdev, "dpfe-dmem");
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dpfe-dmem");
+	priv->dmem = devm_ioremap_resource(dev, res);
 	if (IS_ERR(priv->dmem)) {
 		dev_err(dev, "Couldn't map DCPU data memory\n");
 		return -ENOENT;
 	}
+	priv->dmem_size = resource_size(res);
 
 	priv->imem = devm_platform_ioremap_resource_byname(pdev, "dpfe-imem");
 	if (IS_ERR(priv->imem)) {
diff --git a/drivers/memory/emif.c b/drivers/memory/emif.c
index 2fadad0666b1f0..886c2ca0dc19d4 100644
--- a/drivers/memory/emif.c
+++ b/drivers/memory/emif.c
@@ -922,6 +922,7 @@ static struct emif_data *of_get_memory_device_details(
 	if (!emif || !pd || !dev_info) {
 		dev_err(dev, "%s: Out of memory!!\n",
 			__func__);
+		of_node_put(np_ddr);
 		goto error;
 	}
 
@@ -946,6 +947,7 @@ static struct emif_data *of_get_memory_device_details(
 			pd->device_info->io_width, pd->phy_type, pd->ip_rev,
 			emif->dev)) {
 		dev_err(dev, "%s: invalid device data!!\n", __func__);
+		of_node_put(np_ddr);
 		goto error;
 	}
 	/*
@@ -1062,7 +1064,8 @@ static struct emif_data *get_device_details(
 	}
 
 	if (pd->min_tck) {
-		temp = devm_kzalloc(dev, sizeof(*pd->min_tck), GFP_KERNEL);
+		temp = devm_kzalloc(dev, sizeof(struct lpddr2_min_tck),
+				    GFP_KERNEL);
 		if (temp) {
 			memcpy(temp, pd->min_tck, sizeof(*pd->min_tck));
 			pd->min_tck = temp;
@@ -1140,6 +1143,7 @@ static void emif_remove(struct platform_device *pdev)
 	struct emif_data *emif = platform_get_drvdata(pdev);
 
 	emif_debugfs_exit(emif);
+	of_node_put(emif->np_ddr);
 }
 
 static void emif_shutdown(struct platform_device *pdev)
diff --git a/drivers/memory/emif.h b/drivers/memory/emif.h
index 55aeb36a5bf24c..5dfb09adb0960c 100644
--- a/drivers/memory/emif.h
+++ b/drivers/memory/emif.h
@@ -558,7 +558,7 @@
 #define EMIF_SRAM_AM33_REG_LAYOUT			0x00000000
 #define EMIF_SRAM_AM43_REG_LAYOUT			0x00000001
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 /*
  * Structure containing shadow of important registers in EMIF
  * The calculation function fills in this structure to be later used for
@@ -603,5 +603,5 @@ void ti_emif_enter_sr(void);
 void ti_emif_exit_sr(void);
 void ti_emif_abort_sr(void);
 
-#endif /* __ASSEMBLY__ */
+#endif /* __ASSEMBLER__ */
 #endif /* __EMIF_H */
diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c
index aaeba8ab211e97..8c4606cbd73d91 100644
--- a/drivers/memory/mtk-smi.c
+++ b/drivers/memory/mtk-smi.c
@@ -401,6 +401,30 @@ static const u8 mtk_smi_larb_mt8188_ostd[][SMI_LARB_PORT_NR_MAX] = {
 	[25] = {0x01},
 };
 
+static const u8 mtk_smi_larb_mt8189_ostd[][SMI_LARB_PORT_NR_MAX] = {
+	[0] = {0x8, 0x20, 0x20, 0x20, 0x20, 0x20, 0x10,},
+	[1] = {0x8, 0x20, 0x20, 0x20, 0x20, 0x20, 0x10,},
+	[2] = {0x7, 0x7, 0x4, 0x4, 0x1, 0x1, 0x2, 0x2, 0x7, 0x7,},
+	[4] = {0x2f, 0x1e, 0x9, 0x1, 0x1, 0x1, 0x1, 0x2, 0x2, 0x5, 0x1, 0x17,},
+	[7] = {0x20, 0x2, 0x1, 0x1, 0x1, 0x4, 0x2, 0x1, 0x1, 0x2, 0x3, 0x2,
+	       0xa, 0xf, 0x4, 0x6, 0x5, 0x1,},
+	[9] = {0x6, 0x3, 0xc, 0x6, 0x1, 0x4, 0x3, 0x1, 0x2, 0x4, 0x5, 0x2,
+	       0x4, 0x2, 0x3, 0xb, 0x1, 0x4, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
+	       0x1, 0x1,},
+	[11] = {0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1,
+		0x1, 0x1, 0x1, 0xb, 0x1, 0x4, 0x6, 0x5, 0x6, 0x1, 0x5, 0x2,
+		0x9, 0x5,},
+	[13] = {0x2, 0x8, 0x8, 0x8, 0x4, 0x4, 0x4, 0x4, 0x4, 0xe, 0x4, 0x1,
+		0x6, 0x6, 0x2,},
+	[14] = {0x1, 0x1, 0x1, 0x20, 0xe, 0x4, 0x8, 0x8, 0x6, 0x4,},
+	[16] = {0x1e, 0xc, 0x2, 0x8, 0xe, 0x2, 0x1e, 0x10, 0x4, 0x2, 0x2, 0x2,
+		0x2, 0x2, 0x4, 0x2, 0x4,},
+	[17] = {0x1e, 0xc, 0x2, 0x8, 0xe, 0x2, 0x1e, 0x10, 0x4, 0x2, 0x2, 0x2,
+		0x2, 0x2, 0x4, 0x2, 0x4,},
+	[19] = {0x2, 0x1, 0x3, 0x1,},
+	[20] = {0x7, 0x7, 0x3, 0x3, 0x1, 0x1,},
+};
+
 static const u8 mtk_smi_larb_mt8192_ostd[][SMI_LARB_PORT_NR_MAX] = {
 	[0] = {0x2, 0x2, 0x28, 0xa, 0xc, 0x28,},
 	[1] = {0x2, 0x2, 0x18, 0x18, 0x18, 0xa, 0xc, 0x28,},
@@ -533,6 +557,13 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt8188 = {
 	.ostd		            = mtk_smi_larb_mt8188_ostd,
 };
 
+static const struct mtk_smi_larb_gen mtk_smi_larb_mt8189 = {
+	.config_port                = mtk_smi_larb_config_port_gen2_general,
+	.flags_general	            = MTK_SMI_FLAG_THRT_UPDATE | MTK_SMI_FLAG_SW_FLAG |
+				      MTK_SMI_FLAG_SLEEP_CTL | MTK_SMI_FLAG_CFG_PORT_SEC_CTL,
+	.ostd		            = mtk_smi_larb_mt8189_ostd,
+};
+
 static const struct mtk_smi_larb_gen mtk_smi_larb_mt8192 = {
 	.config_port                = mtk_smi_larb_config_port_gen2_general,
 	.ostd			    = mtk_smi_larb_mt8192_ostd,
@@ -556,6 +587,7 @@ static const struct of_device_id mtk_smi_larb_of_ids[] = {
 	{.compatible = "mediatek,mt8183-smi-larb", .data = &mtk_smi_larb_mt8183},
 	{.compatible = "mediatek,mt8186-smi-larb", .data = &mtk_smi_larb_mt8186},
 	{.compatible = "mediatek,mt8188-smi-larb", .data = &mtk_smi_larb_mt8188},
+	{.compatible = "mediatek,mt8189-smi-larb", .data = &mtk_smi_larb_mt8189},
 	{.compatible = "mediatek,mt8192-smi-larb", .data = &mtk_smi_larb_mt8192},
 	{.compatible = "mediatek,mt8195-smi-larb", .data = &mtk_smi_larb_mt8195},
 	{}
@@ -808,6 +840,17 @@ static const struct mtk_smi_common_plat mtk_smi_common_mt8188_vpp = {
 	.init     = mtk_smi_common_mt8195_init,
 };
 
+static const struct mtk_smi_common_plat mtk_smi_common_mt8189 = {
+	.type     = MTK_SMI_GEN2,
+	.bus_sel  = F_MMU1_LARB(1) | F_MMU1_LARB(3) | F_MMU1_LARB(5) |
+		    F_MMU1_LARB(7),
+};
+
+static const struct mtk_smi_common_plat mtk_smi_sub_common_mt8189 = {
+	.type     = MTK_SMI_GEN2_SUB_COMM,
+	.has_gals = true,
+};
+
 static const struct mtk_smi_common_plat mtk_smi_common_mt8192 = {
 	.type     = MTK_SMI_GEN2,
 	.has_gals = true,
@@ -852,6 +895,8 @@ static const struct of_device_id mtk_smi_common_of_ids[] = {
 	{.compatible = "mediatek,mt8186-smi-common", .data = &mtk_smi_common_mt8186},
 	{.compatible = "mediatek,mt8188-smi-common-vdo", .data = &mtk_smi_common_mt8188_vdo},
 	{.compatible = "mediatek,mt8188-smi-common-vpp", .data = &mtk_smi_common_mt8188_vpp},
+	{.compatible = "mediatek,mt8189-smi-common",     .data = &mtk_smi_common_mt8189},
+	{.compatible = "mediatek,mt8189-smi-sub-common", .data = &mtk_smi_sub_common_mt8189},
 	{.compatible = "mediatek,mt8192-smi-common", .data = &mtk_smi_common_mt8192},
 	{.compatible = "mediatek,mt8195-smi-common-vdo", .data = &mtk_smi_common_mt8195_vdo},
 	{.compatible = "mediatek,mt8195-smi-common-vpp", .data = &mtk_smi_common_mt8195_vpp},
diff --git a/drivers/memory/renesas-rpc-if.c b/drivers/memory/renesas-rpc-if.c
index 3755956ae90658..ec901a5a99aaee 100644
--- a/drivers/memory/renesas-rpc-if.c
+++ b/drivers/memory/renesas-rpc-if.c
@@ -34,6 +34,7 @@ static const struct regmap_access_table rpcif_volatile_table = {
 
 static const struct regmap_range xspi_volatile_ranges[] = {
 	regmap_reg_range(XSPI_CDD0BUF0, XSPI_CDD0BUF0),
+	regmap_reg_range(XSPI_CDD1BUF0, XSPI_CDD1BUF0),
 };
 
 static const struct regmap_access_table xspi_volatile_table = {
@@ -311,6 +312,14 @@ static int xspi_hw_init_impl(struct rpcif_priv *xspi, bool hyperflash)
 	regmap_update_bits(xspi->regmap, XSPI_INTE, XSPI_INTE_CMDCMPE,
 			   XSPI_INTE_CMDCMPE);
 
+	regmap_update_bits(xspi->regmap, XSPI_BMCTL0,
+			   XSPI_BMCTL0_CS0ACC(0xff), XSPI_BMCTL0_CS0ACC(0x03));
+
+	regmap_update_bits(xspi->regmap, XSPI_BMCFG,
+			   XSPI_BMCFG_WRMD | XSPI_BMCFG_MWRCOMB |
+			   XSPI_BMCFG_MWRSIZE(0xff) | XSPI_BMCFG_PREEN,
+			   0 | XSPI_BMCFG_MWRCOMB | XSPI_BMCFG_MWRSIZE(0x0f) |
+			   XSPI_BMCFG_PREEN);
 	return 0;
 }
 
@@ -627,18 +636,14 @@ static int xspi_manual_xfer_impl(struct rpcif_priv *xspi)
 	u32 pos = 0, max = 8;
 	int ret = 0;
 
-	regmap_update_bits(xspi->regmap, XSPI_CDCTL0, XSPI_CDCTL0_TRNUM(0x3),
+	/* Clear transaction number and request */
+	regmap_update_bits(xspi->regmap, XSPI_CDCTL0,
+			   XSPI_CDCTL0_TRNUM(0x3) | XSPI_CDCTL0_TRREQ,
 			   XSPI_CDCTL0_TRNUM(0));
 
-	regmap_update_bits(xspi->regmap, XSPI_CDCTL0, XSPI_CDCTL0_TRREQ, 0);
-
 	regmap_write(xspi->regmap, XSPI_CDTBUF0,
-		     XSPI_CDTBUF_CMDSIZE(0x1) | XSPI_CDTBUF_CMD_FIELD(xspi->command));
-
-	regmap_write(xspi->regmap, XSPI_CDABUF0, 0);
-
-	regmap_update_bits(xspi->regmap, XSPI_CDTBUF0, XSPI_CDTBUF_ADDSIZE(0x7),
-			   XSPI_CDTBUF_ADDSIZE(xspi->addr_nbytes));
+		     XSPI_CDTBUF_CMDSIZE(0x1) | XSPI_CDTBUF_CMD_FIELD(xspi->command) |
+		     XSPI_CDTBUF_ADDSIZE(xspi->addr_nbytes));
 
 	regmap_write(xspi->regmap, XSPI_CDABUF0, xspi->smadr);
 
@@ -651,17 +656,12 @@ static int xspi_manual_xfer_impl(struct rpcif_priv *xspi)
 			u32 bytes_left = xspi->xferlen - pos;
 			u32 nbytes, data[2], *p = data;
 
-			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0,
-					   XSPI_CDTBUF_TRTYPE, XSPI_CDTBUF_TRTYPE);
-
 			nbytes = bytes_left >= max ? max : bytes_left;
 
 			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0,
-					   XSPI_CDTBUF_DATASIZE(0xf),
-					   XSPI_CDTBUF_DATASIZE(nbytes));
-
-			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0,
+					   XSPI_CDTBUF_TRTYPE | XSPI_CDTBUF_DATASIZE(0xf) |
 					   XSPI_CDTBUF_ADDSIZE(0x7),
+					   XSPI_CDTBUF_TRTYPE | XSPI_CDTBUF_DATASIZE(nbytes) |
 					   XSPI_CDTBUF_ADDSIZE(xspi->addr_nbytes));
 
 			memcpy(data, xspi->buffer + pos, nbytes);
@@ -693,30 +693,33 @@ static int xspi_manual_xfer_impl(struct rpcif_priv *xspi)
 		while (pos < xspi->xferlen) {
 			u32 bytes_left = xspi->xferlen - pos;
 			u32 nbytes, data[2], *p = data;
-
-			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0,
-					   XSPI_CDTBUF_TRTYPE,
-					   ~(u32)XSPI_CDTBUF_TRTYPE);
+			u32 cdtbuf0_mask, cdtbuf0_val;
 
 			/* nbytes can be up to 8 bytes */
 			nbytes = bytes_left >= max ? max : bytes_left;
 
-			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0,
-					   XSPI_CDTBUF_DATASIZE(0xf),
-					   XSPI_CDTBUF_DATASIZE(nbytes));
+			/* clear TRTYPE */
+			cdtbuf0_mask = XSPI_CDTBUF_TRTYPE;
+			cdtbuf0_val = 0;
 
-			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0,
-					   XSPI_CDTBUF_ADDSIZE(0x7),
-					   XSPI_CDTBUF_ADDSIZE(xspi->addr_nbytes));
+			/* program DATASIZE */
+			cdtbuf0_mask |= XSPI_CDTBUF_DATASIZE(0xf);
+			cdtbuf0_val |= XSPI_CDTBUF_DATASIZE(nbytes);
+
+			/* program ADDSIZE */
+			cdtbuf0_mask |= XSPI_CDTBUF_ADDSIZE(0x7);
+			cdtbuf0_val |= XSPI_CDTBUF_ADDSIZE(xspi->addr_nbytes);
+
+			/* program LATE */
+			cdtbuf0_mask |= XSPI_CDTBUF_LATE(0x1f);
+			cdtbuf0_val |= XSPI_CDTBUF_LATE(xspi->dummy);
+
+			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0, cdtbuf0_mask, cdtbuf0_val);
 
 			if (xspi->addr_nbytes)
 				regmap_write(xspi->regmap, XSPI_CDABUF0,
 					     xspi->smadr + pos);
 
-			regmap_update_bits(xspi->regmap, XSPI_CDTBUF0,
-					   XSPI_CDTBUF_LATE(0x1f),
-					   XSPI_CDTBUF_LATE(xspi->dummy));
-
 			regmap_update_bits(xspi->regmap, XSPI_CDCTL0,
 					   XSPI_CDCTL0_TRREQ, XSPI_CDCTL0_TRREQ);
 
@@ -773,7 +776,7 @@ int rpcif_manual_xfer(struct device *dev)
 
 	ret = rpc->info->impl->manual_xfer(rpc);
 
-	pm_runtime_put(dev);
+	pm_runtime_put_autosuspend(dev);
 
 	return ret;
 }
@@ -868,15 +871,6 @@ static size_t xspi_dirmap_read_impl(struct rpcif_priv *xspi, u64 offs,
 			   XSPI_CMCFG1_RDCMD_UPPER_BYTE(xspi->command) |
 			   XSPI_CMCFG1_RDLATE(xspi->dummy));
 
-	regmap_update_bits(xspi->regmap, XSPI_BMCTL0, XSPI_BMCTL0_CS0ACC(0xff),
-			   XSPI_BMCTL0_CS0ACC(0x01));
-
-	regmap_update_bits(xspi->regmap, XSPI_BMCFG,
-			   XSPI_BMCFG_WRMD | XSPI_BMCFG_MWRCOMB |
-			   XSPI_BMCFG_MWRSIZE(0xff) | XSPI_BMCFG_PREEN,
-			   0 | XSPI_BMCFG_MWRCOMB | XSPI_BMCFG_MWRSIZE(0x0f) |
-			   XSPI_BMCFG_PREEN);
-
 	regmap_update_bits(xspi->regmap, XSPI_LIOCFGCS0, XSPI_LIOCFG_PRTMD(0x3ff),
 			   XSPI_LIOCFG_PRTMD(xspi->proto));
 
@@ -897,7 +891,7 @@ ssize_t rpcif_dirmap_read(struct device *dev, u64 offs, size_t len, void *buf)
 
 	read = rpc->info->impl->dirmap_read(rpc, offs, len, buf);
 
-	pm_runtime_put(dev);
+	pm_runtime_put_autosuspend(dev);
 
 	return read;
 }
@@ -944,15 +938,6 @@ ssize_t xspi_dirmap_write(struct device *dev, u64 offs, size_t len, const void *
 			   XSPI_CMCFG2_WRCMD_UPPER(xspi->command) |
 			   XSPI_CMCFG2_WRLATE(xspi->dummy));
 
-	regmap_update_bits(xspi->regmap, XSPI_BMCTL0,
-			   XSPI_BMCTL0_CS0ACC(0xff), XSPI_BMCTL0_CS0ACC(0x03));
-
-	regmap_update_bits(xspi->regmap, XSPI_BMCFG,
-			   XSPI_BMCFG_WRMD | XSPI_BMCFG_MWRCOMB |
-			   XSPI_BMCFG_MWRSIZE(0xff) | XSPI_BMCFG_PREEN,
-			   0 | XSPI_BMCFG_MWRCOMB | XSPI_BMCFG_MWRSIZE(0x0f) |
-			   XSPI_BMCFG_PREEN);
-
 	regmap_update_bits(xspi->regmap, XSPI_LIOCFGCS0, XSPI_LIOCFG_PRTMD(0x3ff),
 			   XSPI_LIOCFG_PRTMD(xspi->proto));
 
@@ -963,7 +948,7 @@ ssize_t xspi_dirmap_write(struct device *dev, u64 offs, size_t len, const void *
 		regmap_update_bits(xspi->regmap, XSPI_BMCTL1,
 				   XSPI_BMCTL1_MWRPUSH, XSPI_BMCTL1_MWRPUSH);
 
-	pm_runtime_put(dev);
+	pm_runtime_put_autosuspend(dev);
 
 	return writebytes;
 }
diff --git a/drivers/memory/samsung/exynos5422-dmc.c b/drivers/memory/samsung/exynos5422-dmc.c
index 788d49c688b1d9..98c31955258315 100644
--- a/drivers/memory/samsung/exynos5422-dmc.c
+++ b/drivers/memory/samsung/exynos5422-dmc.c
@@ -1297,8 +1297,15 @@ static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
 	if (ret)
 		return ret;
 
-	clk_prepare_enable(dmc->fout_bpll);
-	clk_prepare_enable(dmc->mout_bpll);
+	ret = clk_prepare_enable(dmc->fout_bpll);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(dmc->mout_bpll);
+	if (ret) {
+		clk_disable_unprepare(dmc->fout_bpll);
+		return ret;
+	}
 
 	/*
 	 * Some bootloaders do not set clock routes correctly.
diff --git a/drivers/memory/stm32_omm.c b/drivers/memory/stm32_omm.c
index 84eb23c824f5b1..3599683ab04ac1 100644
--- a/drivers/memory/stm32_omm.c
+++ b/drivers/memory/stm32_omm.c
@@ -274,6 +274,8 @@ static int stm32_omm_configure(struct device *dev)
 	writel_relaxed(omm->cr, omm->io_base + OMM_CR);
 
 	ret = stm32_omm_set_amcr(dev, true);
+	if (ret && (mux & CR_MUXEN))
+		stm32_omm_toggle_child_clock(dev, false);
 
 error:
 	pm_runtime_put_sync_suspend(dev);
diff --git a/drivers/memory/tegra/tegra264.c b/drivers/memory/tegra/tegra264.c
index 790349295dcc10..57e6395861ba9c 100644
--- a/drivers/memory/tegra/tegra264.c
+++ b/drivers/memory/tegra/tegra264.c
@@ -15,6 +15,47 @@
 #include "mc.h"
 #include "tegra264-bwmgr.h"
 
+/*
+ * Description of a single MCF error type: where to read the status and the
+ * faulting address, how to extract the high address bits, which bits encode
+ * the access direction and security state and the optional error-type sub-code.
+ */
+struct tegra264_mc_fault {
+	const char *const *type_names;
+	u32 addr_hi_mask;
+	u32 addr_hi_reg;
+	u32 addr_hi_shift;
+	u32 addr_reg;
+	u32 rw_bit;
+	u32 sec_bit;
+	u32 status1_reg;
+	u32 status_reg;
+	u32 type_mask;
+	u32 type_shift;
+};
+
+/*
+ * Maps an MCF interrupt to a callback that fills in the register offsets and
+ * decoding parameters for that error type. The register offsets live in the
+ * per-SoC tegra_mc_regs, so they are collected at runtime rather than being
+ * stored in a static table.
+ */
+struct tegra264_mc_error_handler {
+	u32 mask;
+	void (*get_offsets)(struct tegra_mc *mc, struct tegra264_mc_fault *fault);
+};
+
+/*
+ * Description of a single HUB error type. addr_reg is zero for error types
+ * that don't latch a fault address.
+ */
+struct tegra264_hub_error_handler {
+	u32 addr_hi_reg;
+	u32 addr_reg;
+	u32 mask;
+	u32 status_reg;
+};
+
 /*
  * MC Client entries are sorted in the increasing order of the
  * override and security register offsets.
@@ -717,11 +758,18 @@ static const char *const tegra264_rt_error_names[16] = {
 };
 
 /*
- * MC instance aperture mapping for hubc registers
+ * On Tegra264 the memory controller error interrupts are distributed across
+ * several HUB/HUBC apertures. Each aperture is accessed like a broadcast
+ * channel through mc_ch_readl()/mc_ch_writel() using its fixed index.
  */
-static const int mc_hubc_aperture_number[5] = {
-	7, 8, 9, 10, 11
-};
+#define TEGRA264_MC_APERTURE_DISP	7
+#define TEGRA264_MC_APERTURE_SYSTEM	8
+#define TEGRA264_MC_APERTURE_VISION	9
+#define TEGRA264_MC_APERTURE_UPHY	10
+#define TEGRA264_MC_APERTURE_TOP	11
+
+/* Tegra264 splits the MCF common interrupt status across five slices. */
+#define TEGRA264_MC_NUM_SLICES 5
 
 /*
  * tegra264_mc_icc_set() - Pass MC client info to the BPMP-FW
@@ -807,115 +855,149 @@ static int tegra264_mc_icc_get_init_bw(struct icc_node *node, u32 *avg, u32 *pea
 	return 0;
 }
 
-static void mcf_log_fault(struct tegra_mc *mc, u32 channel, unsigned long mcf_ch_intstatus)
+static void tegra264_mc_fault_emem(struct tegra_mc *mc, struct tegra264_mc_fault *fault)
 {
-	unsigned int bit;
+	fault->status_reg = mc->soc->regs->err_status;
+	fault->addr_reg = mc->soc->regs->err_add;
+	fault->addr_hi_reg = mc->soc->regs->err_add_hi;
+	fault->type_mask = mc->soc->mc_err_status_type_mask;
+	fault->type_shift = MC_ERR_STATUS_TYPE_SHIFT;
+	fault->type_names = tegra264_mc_error_names;
+}
 
-	for_each_set_bit(bit, &mcf_ch_intstatus, 32) {
-		const char *client = "unknown", *desc = "NA";
-		u32 status_reg, status1_reg = 0, addr_reg, addr_hi_reg = 0, err_type_mask = 0;
-		u32 value, client_id, i, addr_hi_shift = 0, addr_hi_mask = 0, status1;
-		u32 mc_rw_bit = MC_ERR_STATUS_RW, mc_sec_bit = MC_ERR_STATUS_SECURITY;
-		phys_addr_t addr = 0;
-		u8 type;
-
-		switch (BIT(bit)) {
-		case MC_INT_DECERR_EMEM:
-		case MC_INT_SECURITY_VIOLATION:
-			status_reg = mc->soc->regs->err_status;
-			addr_reg = mc->soc->regs->err_add;
-			addr_hi_reg = mc->soc->regs->err_add_hi;
-			err_type_mask = mc->soc->mc_err_status_type_mask;
-			break;
+static void tegra264_mc_fault_vpr(struct tegra_mc *mc, struct tegra264_mc_fault *fault)
+{
+	fault->status_reg = mc->soc->regs->err_vpr_status;
+	fault->addr_reg = mc->soc->regs->err_vpr_add;
+	fault->addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
+	fault->addr_hi_mask = mc->soc->mc_addr_hi_mask;
+}
 
-		case MC_INT_DECERR_VPR:
-			status_reg = mc->soc->regs->err_vpr_status;
-			addr_reg = mc->soc->regs->err_vpr_add;
-			addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
-			addr_hi_mask = mc->soc->mc_addr_hi_mask;
-			break;
+static void tegra264_mc_fault_sec(struct tegra_mc *mc, struct tegra264_mc_fault *fault)
+{
+	fault->status_reg = mc->soc->regs->err_sec_status;
+	fault->addr_reg = mc->soc->regs->err_sec_add;
+	fault->addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
+	fault->addr_hi_mask = mc->soc->mc_addr_hi_mask;
+}
 
-		case MC_INT_SECERR_SEC:
-			status_reg = mc->soc->regs->err_sec_status;
-			addr_reg = mc->soc->regs->err_sec_add;
-			addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
-			addr_hi_mask = mc->soc->mc_addr_hi_mask;
-			break;
+static void tegra264_mc_fault_mts(struct tegra_mc *mc, struct tegra264_mc_fault *fault)
+{
+	fault->status_reg = mc->soc->regs->err_mts_status;
+	fault->addr_reg = mc->soc->regs->err_mts_add;
+	fault->addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
+	fault->addr_hi_mask = mc->soc->mc_addr_hi_mask;
+}
 
-		case MC_INT_DECERR_MTS:
-			status_reg = mc->soc->regs->err_mts_status;
-			addr_reg = mc->soc->regs->err_mts_add;
-			addr_hi_shift = MC_ERR_STATUS_ADR_HI_SHIFT;
-			addr_hi_mask = mc->soc->mc_addr_hi_mask;
-			break;
+static void tegra264_mc_fault_gsc(struct tegra_mc *mc, struct tegra264_mc_fault *fault)
+{
+	fault->status_reg = mc->soc->regs->err_gen_co_status;
+	fault->status1_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS_1_0;
+	fault->addr_reg = mc->soc->regs->err_gen_co_add;
+	fault->addr_hi_shift = MC_ERR_STATUS_GSC_ADR_HI_SHIFT;
+	fault->addr_hi_mask = MC_ERR_STATUS_GSC_ADR_HI_MASK;
+}
 
-		case MC_INT_DECERR_GENERALIZED_CARVEOUT:
-			status_reg = mc->soc->regs->err_gen_co_status;
-			status1_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS_1_0;
-			addr_reg = mc->soc->regs->err_gen_co_add;
-			addr_hi_shift = MC_ERR_STATUS_GSC_ADR_HI_SHIFT;
-			addr_hi_mask = MC_ERR_STATUS_GSC_ADR_HI_MASK;
-			break;
+static void tegra264_mc_fault_route(struct tegra_mc *mc, struct tegra264_mc_fault *fault)
+{
+	fault->status_reg = mc->soc->regs->err_route_status;
+	fault->addr_reg = mc->soc->regs->err_route_add;
+	fault->addr_hi_shift = MC_ERR_STATUS_RT_ADR_HI_SHIFT;
+	fault->addr_hi_mask = mc->soc->mc_addr_hi_mask;
+	fault->rw_bit = MC_ERR_ROUTE_SANITY_RW;
+	fault->sec_bit = MC_ERR_ROUTE_SANITY_SEC;
+	fault->type_mask = MC_ERR_STATUS_RT_TYPE_MASK;
+	fault->type_shift = MC_ERR_STATUS_RT_TYPE_SHIFT;
+	fault->type_names = tegra264_rt_error_names;
+}
 
-		case MC_INT_DECERR_ROUTE_SANITY:
-		case MC_INT_DECERR_ROUTE_SANITY_GIC_MSI:
-			status_reg = mc->soc->regs->err_route_status;
-			addr_reg = mc->soc->regs->err_route_add;
-			addr_hi_shift = MC_ERR_STATUS_RT_ADR_HI_SHIFT;
-			addr_hi_mask = mc->soc->mc_addr_hi_mask;
-			mc_sec_bit = MC_ERR_ROUTE_SANITY_SEC;
-			mc_rw_bit = MC_ERR_ROUTE_SANITY_RW;
-			err_type_mask = MC_ERR_STATUS_RT_TYPE_MASK;
-			break;
+static const struct tegra264_mc_error_handler tegra264_mc_error_handlers[] = {
+	{ MC_INT_DECERR_EMEM,			tegra264_mc_fault_emem },
+	{ MC_INT_SECURITY_VIOLATION,		tegra264_mc_fault_emem },
+	{ MC_INT_DECERR_VPR,			tegra264_mc_fault_vpr },
+	{ MC_INT_SECERR_SEC,			tegra264_mc_fault_sec },
+	{ MC_INT_DECERR_MTS,			tegra264_mc_fault_mts },
+	{ MC_INT_DECERR_GENERALIZED_CARVEOUT,	tegra264_mc_fault_gsc },
+	{ MC_INT_DECERR_ROUTE_SANITY,		tegra264_mc_fault_route },
+	{ MC_INT_DECERR_ROUTE_SANITY_GIC_MSI,	tegra264_mc_fault_route },
+};
 
-		default:
-			dev_err_ratelimited(mc->dev, "Incorrect MC interrupt mask\n");
-			return;
-		}
+static void tegra264_mc_report_fault(struct tegra_mc *mc, u32 channel,
+				     unsigned int interrupt,
+				     const struct tegra264_mc_fault *fault)
+{
+	const char *client = "unknown", *desc = "NA";
+	u32 value, client_id, status1 = 0;
+	phys_addr_t addr = 0;
+	unsigned int i;
+	u8 type;
+
+	value = mc_ch_readl(mc, channel, fault->status_reg);
+
+	if (fault->addr_hi_reg) {
+		addr = mc_ch_readl(mc, channel, fault->addr_hi_reg);
+	} else if (fault->status1_reg) {
+		status1 = mc_ch_readl(mc, channel, fault->status1_reg);
+		addr = (status1 >> fault->addr_hi_shift) & fault->addr_hi_mask;
+	} else {
+		addr = (value >> fault->addr_hi_shift) & fault->addr_hi_mask;
+	}
 
-		value = mc_ch_readl(mc, channel, status_reg);
-		if (addr_hi_reg) {
-			addr = mc_ch_readl(mc, channel, addr_hi_reg);
-		} else {
-			if (!status1_reg) {
-				addr = ((value >> addr_hi_shift) & addr_hi_mask);
-			} else {
-				status1 = mc_ch_readl(mc, channel, status1_reg);
-				addr = ((status1 >> addr_hi_shift) & addr_hi_mask);
-			}
+	addr <<= 32;
+	addr |= mc_ch_readl(mc, channel, fault->addr_reg);
+
+	client_id = value & mc->soc->client_id_mask;
+	for (i = 0; i < mc->soc->num_clients; i++) {
+		if (mc->soc->clients[i].id == client_id) {
+			client = mc->soc->clients[i].name;
+			break;
 		}
+	}
 
-		addr <<= 32;
-		addr |= mc_ch_readl(mc, channel, addr_reg);
+	if (fault->type_names) {
+		type = (value & fault->type_mask) >> fault->type_shift;
+		desc = fault->type_names[type];
+	}
 
-		client_id = value & mc->soc->client_id_mask;
-		for (i = 0; i < mc->soc->num_clients; i++) {
-			if (mc->soc->clients[i].id == client_id) {
-				client = mc->soc->clients[i].name;
+	dev_err_ratelimited(mc->dev, "%s: %s %s @%pa: %s (%s)\n",
+			    client, value & fault->sec_bit ? "secure" : "non-secure",
+			    value & fault->rw_bit ? "write" : "read", &addr,
+			    tegra_mc_status_names[interrupt] ?: "unknown", desc);
+
+	if (fault->status1_reg)
+		dev_err_ratelimited(mc->dev, "gsc_apr_id=%u gsc_co_apr_id=%u\n",
+				    (status1 >> ERR_GENERALIZED_APERTURE_ID_SHIFT) &
+				    ERR_GENERALIZED_APERTURE_ID_MASK,
+				    (status1 >> ERR_GENERALIZED_CARVEOUT_APERTURE_ID_SHIFT) &
+				    ERR_GENERALIZED_CARVEOUT_APERTURE_ID_MASK);
+}
+
+static void mcf_log_fault(struct tegra_mc *mc, u32 channel, unsigned long mcf_ch_intstatus)
+{
+	unsigned int interrupt;
+
+	for_each_set_bit(interrupt, &mcf_ch_intstatus, 32) {
+		const struct tegra264_mc_error_handler *handler = NULL;
+		struct tegra264_mc_fault fault = {
+			.rw_bit = MC_ERR_STATUS_RW,
+			.sec_bit = MC_ERR_STATUS_SECURITY,
+		};
+		unsigned int i;
+
+		for (i = 0; i < ARRAY_SIZE(tegra264_mc_error_handlers); i++) {
+			if (tegra264_mc_error_handlers[i].mask == BIT(interrupt)) {
+				handler = &tegra264_mc_error_handlers[i];
 				break;
 			}
 		}
 
-		if (err_type_mask == MC_ERR_STATUS_RT_TYPE_MASK) {
-			type = (value & err_type_mask) >>
-				MC_ERR_STATUS_RT_TYPE_SHIFT;
-			desc = tegra264_rt_error_names[type];
-		} else if (err_type_mask) {
-			type = (value & err_type_mask) >>
-					MC_ERR_STATUS_TYPE_SHIFT;
-			desc = tegra264_mc_error_names[type];
+		if (!handler) {
+			dev_err_ratelimited(mc->dev, "Incorrect MC interrupt mask\n");
+			return;
 		}
 
-		dev_err_ratelimited(mc->dev, "%s: %s %s @%pa: %s (%s)\n",
-				    client, value & mc_sec_bit ? "secure" : "non-secure",
-				    value & mc_rw_bit ? "write" : "read", &addr,
-				    tegra_mc_status_names[bit] ?: "unknown", desc);
-		if (status1_reg)
-			dev_err_ratelimited(mc->dev, "gsc_apr_id=%u gsc_co_apr_id=%u\n",
-					    ((status1 >> ERR_GENERALIZED_APERTURE_ID_SHIFT)
-					    & ERR_GENERALIZED_APERTURE_ID_MASK),
-					    ((status1 >> ERR_GENERALIZED_CARVEOUT_APERTURE_ID_SHIFT)
-					    & ERR_GENERALIZED_CARVEOUT_APERTURE_ID_MASK));
+		handler->get_offsets(mc, &fault);
+		tegra264_mc_report_fault(mc, channel, interrupt, &fault);
 	}
 
 	/* clear interrupts */
@@ -937,7 +1019,7 @@ static irqreturn_t handle_mcf_irq(int irq, void *data)
 
 	for_each_set_bit(slice, &common_intstat, 32) {
 		/* Find out the slice number on which interrupt occurred */
-		if (slice > 4) {
+		if (slice >= TEGRA264_MC_NUM_SLICES) {
 			dev_err(mc->dev, "Slice index out of bounds: %u\n", slice);
 			return IRQ_NONE;
 		}
@@ -950,53 +1032,69 @@ static irqreturn_t handle_mcf_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static const struct tegra264_hub_error_handler tegra264_hub_error_handlers[] = {
+	{
+		.mask = MSS_HUB_COALESCER_ERR_INTMASK,
+		.status_reg = MSS_HUB_COALESCE_ERR_STATUS_0,
+		.addr_reg = MSS_HUB_COALESCE_ERR_ADR_0,
+		.addr_hi_reg = MSS_HUB_COALESCE_ERR_ADR_HI_0,
+	},
+	{
+		.mask = MSS_HUB_SMMU_BYPASS_ALLOW_ERR_INTMASK,
+		.status_reg = MSS_HUB_SMMU_BYPASS_ALLOW_ERR_STATUS_0,
+	},
+	{
+		.mask = MSS_HUB_ILLEGAL_TBUGRP_ID_INTMASK,
+		.status_reg = MSS_HUB_ILLEGAL_TBUGRP_ID_ERR_STATUS_0,
+	},
+	{
+		.mask = MSS_HUB_MSI_ERR_INTMASK,
+		.status_reg = MSS_HUB_MSI_ERR_STATUS_0,
+	},
+	{
+		.mask = MSS_HUB_POISON_RSP_INTMASK,
+		.status_reg = MSS_HUB_POISON_RSP_STATUS_0,
+	},
+	{
+		.mask = MSS_HUB_RESTRICTED_ACCESS_ERR_INTMASK,
+		.status_reg = MSS_HUB_RESTRICTED_ACCESS_ERR_STATUS_0,
+	},
+	{
+		.mask = MSS_HUB_RESERVED_PA_ERR_INTMASK,
+		.status_reg = MSS_HUB_RESERVED_PA_ERR_STATUS_0,
+	},
+};
+
 static void hub_log_fault(struct tegra_mc *mc, u32 hub, unsigned long hub_intstat)
 {
-	unsigned int bit;
+	unsigned int interrupt;
 
-	for_each_set_bit(bit, &hub_intstat, 32) {
+	for_each_set_bit(interrupt, &hub_intstat, 32) {
+		const struct tegra264_hub_error_handler *handler = NULL;
 		const char *client = "unknown";
-		u32 client_id, status_reg, value, i;
+		u32 client_id, value;
 		phys_addr_t addr = 0;
+		unsigned int i;
 
-		switch (BIT(bit)) {
-		case MSS_HUB_COALESCER_ERR_INTMASK:
-			status_reg = MSS_HUB_COALESCE_ERR_STATUS_0;
-			addr = mc_ch_readl(mc, hub, MSS_HUB_COALESCE_ERR_ADR_HI_0);
-			addr <<= 32;
-			addr |= mc_ch_readl(mc, hub, MSS_HUB_COALESCE_ERR_ADR_0);
-			break;
-
-		case MSS_HUB_SMMU_BYPASS_ALLOW_ERR_INTMASK:
-			status_reg = MSS_HUB_SMMU_BYPASS_ALLOW_ERR_STATUS_0;
-			break;
-
-		case MSS_HUB_ILLEGAL_TBUGRP_ID_INTMASK:
-			status_reg = MSS_HUB_ILLEGAL_TBUGRP_ID_ERR_STATUS_0;
-			break;
-
-		case MSS_HUB_MSI_ERR_INTMASK:
-			status_reg = MSS_HUB_MSI_ERR_STATUS_0;
-			break;
-
-		case MSS_HUB_POISON_RSP_INTMASK:
-			status_reg = MSS_HUB_POISON_RSP_STATUS_0;
-			break;
-
-		case MSS_HUB_RESTRICTED_ACCESS_ERR_INTMASK:
-			status_reg = MSS_HUB_RESTRICTED_ACCESS_ERR_STATUS_0;
-			break;
-
-		case MSS_HUB_RESERVED_PA_ERR_INTMASK:
-			status_reg = MSS_HUB_RESERVED_PA_ERR_STATUS_0;
-			break;
+		for (i = 0; i < ARRAY_SIZE(tegra264_hub_error_handlers); i++) {
+			if (tegra264_hub_error_handlers[i].mask == BIT(interrupt)) {
+				handler = &tegra264_hub_error_handlers[i];
+				break;
+			}
+		}
 
-		default:
+		if (!handler) {
 			dev_err_ratelimited(mc->dev, "Incorrect HUB interrupt mask\n");
 			return;
 		}
 
-		value = mc_ch_readl(mc, hub, status_reg);
+		if (handler->addr_reg) {
+			addr = mc_ch_readl(mc, hub, handler->addr_hi_reg);
+			addr <<= 32;
+			addr |= mc_ch_readl(mc, hub, handler->addr_reg);
+		}
+
+		value = mc_ch_readl(mc, hub, handler->status_reg);
 
 		client_id = value & mc->soc->client_id_mask;
 		for (i = 0; i < mc->soc->num_clients; i++) {
@@ -1007,8 +1105,8 @@ static void hub_log_fault(struct tegra_mc *mc, u32 hub, unsigned long hub_intsta
 		}
 
 		dev_err_ratelimited(mc->dev, "%s: @%pa: %s status: 0x%x\n",
-				    client, &addr, tegra264_hub_error_names[bit] ?: "unknown",
-				    value);
+				    client, &addr,
+				    tegra264_hub_error_names[interrupt] ?: "unknown", value);
 	}
 
 	/* clear interrupts */
@@ -1057,27 +1155,27 @@ static irqreturn_t handle_hub_irq(int irq, void *data, int mc_hubc_aperture_numb
 
 static irqreturn_t handle_disp_hub_irq(int irq, void *data)
 {
-	return handle_hub_irq(irq, data, mc_hubc_aperture_number[0]);
+	return handle_hub_irq(irq, data, TEGRA264_MC_APERTURE_DISP);
 }
 
 static irqreturn_t handle_system_hub_irq(int irq, void *data)
 {
-	return handle_hub_irq(irq, data, mc_hubc_aperture_number[1]);
+	return handle_hub_irq(irq, data, TEGRA264_MC_APERTURE_SYSTEM);
 }
 
 static irqreturn_t handle_vision_hub_irq(int irq, void *data)
 {
-	return handle_hub_irq(irq, data, mc_hubc_aperture_number[2]);
+	return handle_hub_irq(irq, data, TEGRA264_MC_APERTURE_VISION);
 }
 
 static irqreturn_t handle_uphy_hub_irq(int irq, void *data)
 {
-	return handle_hub_irq(irq, data, mc_hubc_aperture_number[3]);
+	return handle_hub_irq(irq, data, TEGRA264_MC_APERTURE_UPHY);
 }
 
 static irqreturn_t handle_top_hub_irq(int irq, void *data)
 {
-	return handle_hub_irq(irq, data, mc_hubc_aperture_number[4]);
+	return handle_hub_irq(irq, data, TEGRA264_MC_APERTURE_TOP);
 }
 
 static irqreturn_t handle_generic_irq(struct tegra_mc *mc, unsigned long intstat_reg)
diff --git a/drivers/memstick/core/memstick.c b/drivers/memstick/core/memstick.c
index e03989c4e99ea1..0d15f48ef51d4f 100644
--- a/drivers/memstick/core/memstick.c
+++ b/drivers/memstick/core/memstick.c
@@ -370,9 +370,7 @@ int memstick_set_rw_addr(struct memstick_dev *card)
 {
 	card->next_request = h_memstick_set_rw_addr;
 	memstick_new_req(card->host);
-	if (!wait_for_completion_timeout(&card->mrq_complete,
-			msecs_to_jiffies(500)))
-		card->current_mrq.error = -ETIMEDOUT;
+	wait_for_completion(&card->mrq_complete);
 
 	return card->current_mrq.error;
 }
@@ -405,9 +403,7 @@ static struct memstick_dev *memstick_alloc_card(struct memstick_host *host)
 
 		card->next_request = h_memstick_read_dev_id;
 		memstick_new_req(host);
-		if (!wait_for_completion_timeout(&card->mrq_complete,
-				msecs_to_jiffies(500)))
-			card->current_mrq.error = -ETIMEDOUT;
+		wait_for_completion(&card->mrq_complete);
 
 		if (card->current_mrq.error)
 			goto err_out;
diff --git a/drivers/memstick/core/ms_block.c b/drivers/memstick/core/ms_block.c
index 65154e569a9e72..cd4ff1b79a6dae 100644
--- a/drivers/memstick/core/ms_block.c
+++ b/drivers/memstick/core/ms_block.c
@@ -720,7 +720,7 @@ static int h_msb_parallel_switch(struct memstick_dev *card,
 
 static int msb_switch_to_parallel(struct msb_data *msb);
 
-/* Reset the card, to guard against hw errors beeing treated as bad blocks */
+/* Reset the card, to guard against hw errors being treated as bad blocks */
 static int msb_reset(struct msb_data *msb, bool full)
 {
 
diff --git a/drivers/memstick/host/r592.c b/drivers/memstick/host/r592.c
index 605b2265536fbc..dc3de63901bed3 100644
--- a/drivers/memstick/host/r592.c
+++ b/drivers/memstick/host/r592.c
@@ -289,7 +289,7 @@ static int r592_transfer_fifo_dma(struct r592_device *dev)
 	dev->dma_error = 0;
 	reinit_completion(&dev->dma_done);
 
-	/* TODO: hidden assumption about nenth beeing always 1 */
+	/* TODO: hidden assumption about nenth being always 1 */
 	sg_count = dma_map_sg(&dev->pci_dev->dev, &dev->req->sg, 1, is_write ?
 			      DMA_TO_DEVICE : DMA_FROM_DEVICE);
 
diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index b55deb988ad922..bca05099718a03 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -196,20 +196,20 @@ mptfc_block_error_handler(struct fc_rport *rport)
 
 	hd = shost_priv(shost);
 	ioc = hd->ioc;
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	while ((ready = fc_remote_port_chkready(rport) >> 16) == DID_IMM_RETRY
 	 || (loops > 0 && ioc->active == 0)) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		dfcprintk (ioc, printk(MYIOC_s_DEBUG_FMT
 			"mptfc_block_error_handler.%d: %s, port status is "
 			"%x, active flag %d, deferring recovery.\n",
 			ioc->name, ioc->sh->host_no,
 			dev_name(&rport->dev), ready, ioc->active));
 		msleep(1000);
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		loops --;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (ready == DID_NO_CONNECT || ioc->active == 0) {
 		dfcprintk (ioc, printk(MYIOC_s_DEBUG_FMT
diff --git a/drivers/mfd/88pm886.c b/drivers/mfd/88pm886.c
index f8401d5e6dbe34..97904108d8d63e 100644
--- a/drivers/mfd/88pm886.c
+++ b/drivers/mfd/88pm886.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/mfd/core.h>
 #include <linux/module.h>
 #include <linux/notifier.h>
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index e4fd4572472f98..0a2778052519f1 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -1006,6 +1006,8 @@ config MFD_MAX77705
 	depends on I2C
 	select MFD_CORE
 	select MFD_SIMPLE_MFD_I2C
+	select REGMAP_I2C
+	select REGMAP_IRQ
 	help
 	  Say yes here to add support for Maxim Integrated MAX77705 PMIC.
 	  This is a Power Management IC with Charger, safe LDOs, Flash, Haptic
@@ -2226,7 +2228,7 @@ config MFD_ROHM_BD718XX
 	  and emergency shut down as well as 32,768KHz clock output.
 
 config MFD_ROHM_BD71828
-	tristate "ROHM BD718[15/28/79], BD72720 and BD73900 PMICs"
+	tristate "ROHM BD718[15/28/79], BD72720 and BD73[8/9]00 PMICs"
 	depends on I2C=y
 	depends on OF
 	select REGMAP_I2C
@@ -2235,14 +2237,17 @@ config MFD_ROHM_BD71828
 	select MFD_ROHM_PWRBUTTON
 	help
 	  Select this option to get support for the ROHM BD71815, BD71828,
-	  BD71879, BD72720 and BD73900 Power Management ICs (PMICs). These are
-	  single-chip Power Management ICs (PMIC), mainly for battery-powered
-	  portable devices.
+	  BD71879, BD72720, BD73800 and BD73900 Power Management ICs (PMICs).
+	  These are single-chip Power Management ICs (PMIC), mainly for
+	  battery-powered portable devices.
 	  The BD71815 has 5 bucks, 7 LDOs, and a boost for driving LEDs.
 	  The BD718[28/79] have 7 buck converters and 7 LDOs.
 	  The BD72720 and the BD73900 have 10 bucks and 11 LDOs.
-	  All ICs provide a single-cell linear charger, a Coulomb counter,
-	  a Real-Time Clock (RTC), GPIOs and a 32.768 kHz clock gate.
+	  All the above ICs provide a single-cell linear charger, and a Coulomb
+	  counter.
+	  The BD73800 has 8 bucks, 4 LDOs and ADC for power/current
+	  accumulation but no charging logic. All these PMICs integrate also a
+	  Real-Time Clock (RTC), GPIOs and a 32.768 kHz clock gate.
 
 config MFD_ROHM_BD957XMUF
 	tristate "ROHM BD9576MUF and BD9573MUF Power Management ICs"
diff --git a/drivers/mfd/ab8500-core.c b/drivers/mfd/ab8500-core.c
index 86fa99022cb356..ceea1ae34dd897 100644
--- a/drivers/mfd/ab8500-core.c
+++ b/drivers/mfd/ab8500-core.c
@@ -125,7 +125,7 @@ static u8 turn_on_stat_set;
 
 /*
  * Map interrupt numbers to the LATCH and MASK register offsets, Interrupt
- * numbers are indexed into this array with (num / 8). The interupts are
+ * numbers are indexed into this array with (num / 8). The interrupts are
  * defined in linux/mfd/ab8500.h
  *
  * This is one off from the register names, i.e. AB8500_IT_MASK1_REG is at
diff --git a/drivers/mfd/abx500-core.c b/drivers/mfd/abx500-core.c
index e896531d0354db..46f4d7dfcc6ae3 100644
--- a/drivers/mfd/abx500-core.c
+++ b/drivers/mfd/abx500-core.c
@@ -5,6 +5,7 @@
  * Author: Mattias Wallin <mattias.wallin@stericsson.com>
  */
 
+#include <linux/device.h>
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/err.h>
diff --git a/drivers/mfd/adp5585.c b/drivers/mfd/adp5585.c
index aad1d734baeba7..3ef0a3b0fb6d26 100644
--- a/drivers/mfd/adp5585.c
+++ b/drivers/mfd/adp5585.c
@@ -12,6 +12,7 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/gpio/consumer.h>
 #include <linux/mfd/adp5585.h>
 #include <linux/mfd/core.h>
diff --git a/drivers/mfd/cs40l50-core.c b/drivers/mfd/cs40l50-core.c
index 662d987b650b5b..56bb5f1499a360 100644
--- a/drivers/mfd/cs40l50-core.c
+++ b/drivers/mfd/cs40l50-core.c
@@ -8,6 +8,7 @@
  * Author: James Ogletree <james.ogletree@cirrus.com>
  */
 
+#include <linux/interrupt.h>
 #include <linux/firmware/cirrus/cs_dsp.h>
 #include <linux/firmware/cirrus/wmfw.h>
 #include <linux/mfd/core.h>
diff --git a/drivers/mfd/cs42l43-i2c.c b/drivers/mfd/cs42l43-i2c.c
index cbe05c3ea9100f..2f1862199618de 100644
--- a/drivers/mfd/cs42l43-i2c.c
+++ b/drivers/mfd/cs42l43-i2c.c
@@ -54,10 +54,18 @@ static int cs42l43_i2c_probe(struct i2c_client *i2c)
 	return cs42l43_dev_probe(cs42l43);
 }
 
+static void cs42l43_i2c_remove(struct i2c_client *i2c)
+{
+	struct cs42l43 *cs42l43 = dev_get_drvdata(&i2c->dev);
+
+	cs42l43_dev_remove(cs42l43);
+}
+
 #if IS_ENABLED(CONFIG_OF)
 static const struct of_device_id cs42l43_of_match[] = {
 	{ .compatible = "cirrus,cs42l43", .data = (void *)CS42L43_DEVID_VAL },
 	{ .compatible = "cirrus,cs42l43b", .data = (void *)CS42L43B_DEVID_VAL },
+	{ .compatible = "cirrus,cs42l44", .data = (void *)CS42L43B_DEVID_VAL },
 	{}
 };
 MODULE_DEVICE_TABLE(of, cs42l43_of_match);
@@ -67,6 +75,7 @@ MODULE_DEVICE_TABLE(of, cs42l43_of_match);
 static const struct acpi_device_id cs42l43_acpi_match[] = {
 	{ "CSC4243", CS42L43_DEVID_VAL },
 	{ "CSC2A3B", CS42L43B_DEVID_VAL },
+	{ "CSC4244", CS42L43B_DEVID_VAL },
 	{}
 };
 MODULE_DEVICE_TABLE(acpi, cs42l43_acpi_match);
@@ -81,6 +90,7 @@ static struct i2c_driver cs42l43_i2c_driver = {
 	},
 
 	.probe		= cs42l43_i2c_probe,
+	.remove		= cs42l43_i2c_remove,
 };
 module_i2c_driver(cs42l43_i2c_driver);
 
diff --git a/drivers/mfd/cs42l43-sdw.c b/drivers/mfd/cs42l43-sdw.c
index 0a6999453f325a..22e4c1fe929847 100644
--- a/drivers/mfd/cs42l43-sdw.c
+++ b/drivers/mfd/cs42l43-sdw.c
@@ -180,9 +180,17 @@ static int cs42l43_sdw_probe(struct sdw_slave *sdw, const struct sdw_device_id *
 	return cs42l43_dev_probe(cs42l43);
 }
 
+static void cs42l43_sdw_remove(struct sdw_slave *sdw)
+{
+	struct cs42l43 *cs42l43 = dev_get_drvdata(&sdw->dev);
+
+	cs42l43_dev_remove(cs42l43);
+}
+
 static const struct sdw_device_id cs42l43_sdw_id[] = {
 	SDW_SLAVE_ENTRY(0x01FA, 0x4243, (void *)CS42L43_DEVID_VAL),
 	SDW_SLAVE_ENTRY(0x01FA, 0x2A3B, (void *)CS42L43B_DEVID_VAL),
+	SDW_SLAVE_ENTRY(0x01FA, 0x4244, (void *)CS42L43B_DEVID_VAL),
 	{ }
 };
 MODULE_DEVICE_TABLE(sdw, cs42l43_sdw_id);
@@ -194,6 +202,7 @@ static struct sdw_driver cs42l43_sdw_driver = {
 	},
 
 	.probe		= cs42l43_sdw_probe,
+	.remove		= cs42l43_sdw_remove,
 	.id_table	= cs42l43_sdw_id,
 	.ops		= &cs42l43_sdw_ops,
 };
diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c
index 4212ebcca60b57..06ef1dc5c7c9db 100644
--- a/drivers/mfd/cs42l43.c
+++ b/drivers/mfd/cs42l43.c
@@ -14,6 +14,7 @@
 #include <linux/err.h>
 #include <linux/firmware.h>
 #include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
 #include <linux/jiffies.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/cs42l43.h>
@@ -959,6 +960,7 @@ static void cs42l43_boot_work(struct work_struct *work)
 {
 	struct cs42l43 *cs42l43 = container_of(work, struct cs42l43, boot_work);
 	unsigned int devid, revid, otp;
+	bool valid_id;
 	int ret;
 
 	ret = cs42l43_wait_for_attach(cs42l43);
@@ -973,19 +975,24 @@ static void cs42l43_boot_work(struct work_struct *work)
 
 	switch (devid) {
 	case CS42L43_DEVID_VAL:
+		valid_id = (cs42l43->variant_id == CS42L43_DEVID_VAL);
+		break;
 	case CS42L43B_DEVID_VAL:
-		if (devid != cs42l43->variant_id) {
-			dev_err(cs42l43->dev,
-				"Device ID (0x%06x) does not match variant ID (0x%06lx)\n",
-				devid, cs42l43->variant_id);
-			goto err;
-		}
+	case CS42L44_DEVID_VAL:
+		valid_id = (cs42l43->variant_id == CS42L43B_DEVID_VAL);
 		break;
 	default:
 		dev_err(cs42l43->dev, "Unrecognised devid: 0x%06x\n", devid);
 		goto err;
 	}
 
+	if (!valid_id) {
+		dev_err(cs42l43->dev,
+			"Device ID (0x%06x) does not match variant ID (0x%06lx)\n",
+			devid, cs42l43->variant_id);
+		goto err;
+	}
+
 	ret = regmap_read(cs42l43->regmap, CS42L43_REVID, &revid);
 	if (ret) {
 		dev_err(cs42l43->dev, "Failed to read rev: %d\n", ret);
@@ -1099,15 +1106,19 @@ static int cs42l43_power_down(struct cs42l43 *cs42l43)
 	return 0;
 }
 
-static void cs42l43_dev_remove(void *data)
+static void cs42l43_dev_power_down(void *data)
 {
 	struct cs42l43 *cs42l43 = data;
 
-	cancel_work_sync(&cs42l43->boot_work);
-
 	cs42l43_power_down(cs42l43);
 }
 
+void cs42l43_dev_remove(struct cs42l43 *cs42l43)
+{
+	flush_work(&cs42l43->boot_work);
+}
+EXPORT_SYMBOL_NS_GPL(cs42l43_dev_remove, "MFD_CS42L43");
+
 int cs42l43_dev_probe(struct cs42l43 *cs42l43)
 {
 	int i, ret;
@@ -1153,7 +1164,7 @@ int cs42l43_dev_probe(struct cs42l43 *cs42l43)
 	if (ret)
 		return ret;
 
-	ret = devm_add_action_or_reset(cs42l43->dev, cs42l43_dev_remove, cs42l43);
+	ret = devm_add_action_or_reset(cs42l43->dev, cs42l43_dev_power_down, cs42l43);
 	if (ret)
 		return ret;
 
diff --git a/drivers/mfd/cs42l43.h b/drivers/mfd/cs42l43.h
index a0068f6572e2c3..862bc6fb68cb73 100644
--- a/drivers/mfd/cs42l43.h
+++ b/drivers/mfd/cs42l43.h
@@ -25,5 +25,6 @@ bool cs42l43_precious_register(struct device *dev, unsigned int reg);
 bool cs42l43_volatile_register(struct device *dev, unsigned int reg);
 
 int cs42l43_dev_probe(struct cs42l43 *cs42l43);
+void cs42l43_dev_remove(struct cs42l43 *cs42l43);
 
 #endif /* CS42L43_CORE_INT_H */
diff --git a/drivers/mfd/da903x.c b/drivers/mfd/da903x.c
index e86b39de3303a2..e4ad48e1f139be 100644
--- a/drivers/mfd/da903x.c
+++ b/drivers/mfd/da903x.c
@@ -421,6 +421,14 @@ static irqreturn_t da903x_irq_handler(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static void da903x_cancel_irq_work(void *data)
+{
+	struct da903x_chip *chip = data;
+
+	disable_irq(chip->client->irq);
+	cancel_work_sync(&chip->irq_work);
+}
+
 static const struct da903x_chip_ops da903x_ops[] = {
 	[0] = {
 		.init_chip	= da9030_init_chip,
@@ -456,6 +464,13 @@ static int da903x_remove_subdevs(struct da903x_chip *chip)
 	return device_for_each_child(chip->dev, NULL, __remove_subdev);
 }
 
+static void da903x_remove_subdevs_action(void *data)
+{
+	struct da903x_chip *chip = data;
+
+	da903x_remove_subdevs(chip);
+}
+
 static int da903x_add_subdevs(struct da903x_chip *chip,
 					struct da903x_platform_data *pdata)
 {
@@ -484,7 +499,6 @@ static int da903x_add_subdevs(struct da903x_chip *chip,
 	return 0;
 
 failed:
-	da903x_remove_subdevs(chip);
 	return ret;
 }
 
@@ -520,6 +534,11 @@ static int da903x_probe(struct i2c_client *client)
 	chip->ops->mask_events(chip, chip->events_mask);
 	chip->ops->read_events(chip, &tmp);
 
+	ret = devm_add_action_or_reset(&client->dev,
+				       da903x_remove_subdevs_action, chip);
+	if (ret)
+		return ret;
+
 	ret = devm_request_irq(&client->dev, client->irq, da903x_irq_handler,
 			IRQF_TRIGGER_FALLING,
 			"da903x", chip);
@@ -529,14 +548,12 @@ static int da903x_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	return da903x_add_subdevs(chip, pdata);
-}
-
-static void da903x_remove(struct i2c_client *client)
-{
-	struct da903x_chip *chip = i2c_get_clientdata(client);
+	ret = devm_add_action_or_reset(&client->dev, da903x_cancel_irq_work,
+				       chip);
+	if (ret)
+		return ret;
 
-	da903x_remove_subdevs(chip);
+	return da903x_add_subdevs(chip, pdata);
 }
 
 static struct i2c_driver da903x_driver = {
@@ -544,7 +561,6 @@ static struct i2c_driver da903x_driver = {
 		.name	= "da903x",
 	},
 	.probe		= da903x_probe,
-	.remove		= da903x_remove,
 	.id_table	= da903x_id_table,
 };
 
diff --git a/drivers/mfd/da9062-core.c b/drivers/mfd/da9062-core.c
index 637c5f47a4b0c7..ad198183aff737 100644
--- a/drivers/mfd/da9062-core.c
+++ b/drivers/mfd/da9062-core.c
@@ -203,36 +203,36 @@ static const struct mfd_cell da9061_devs_noirq[] = {
 };
 
 static const struct resource da9062_core_resources[] = {
-	DEFINE_RES_NAMED(DA9062_IRQ_VDD_WARN, 1, "VDD_WARN", IORESOURCE_IRQ),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_VDD_WARN, "VDD_WARN"),
 };
 
 static const struct resource da9062_regulators_resources[] = {
-	DEFINE_RES_NAMED(DA9062_IRQ_LDO_LIM, 1, "LDO_LIM", IORESOURCE_IRQ),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_LDO_LIM, "LDO_LIM"),
 };
 
 static const struct resource da9062_thermal_resources[] = {
-	DEFINE_RES_NAMED(DA9062_IRQ_TEMP, 1, "THERMAL", IORESOURCE_IRQ),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_TEMP, "THERMAL"),
 };
 
 static const struct resource da9062_wdt_resources[] = {
-	DEFINE_RES_NAMED(DA9062_IRQ_WDG_WARN, 1, "WD_WARN", IORESOURCE_IRQ),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_WDG_WARN, "WD_WARN"),
 };
 
 static const struct resource da9062_rtc_resources[] = {
-	DEFINE_RES_NAMED(DA9062_IRQ_ALARM, 1, "ALARM", IORESOURCE_IRQ),
-	DEFINE_RES_NAMED(DA9062_IRQ_TICK, 1, "TICK", IORESOURCE_IRQ),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_ALARM, "ALARM"),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_TICK, "TICK"),
 };
 
 static const struct resource da9062_onkey_resources[] = {
-	DEFINE_RES_NAMED(DA9062_IRQ_ONKEY, 1, "ONKEY", IORESOURCE_IRQ),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_ONKEY, "ONKEY"),
 };
 
 static const struct resource da9062_gpio_resources[] = {
-	DEFINE_RES_NAMED(DA9062_IRQ_GPI0, 1, "GPI0", IORESOURCE_IRQ),
-	DEFINE_RES_NAMED(DA9062_IRQ_GPI1, 1, "GPI1", IORESOURCE_IRQ),
-	DEFINE_RES_NAMED(DA9062_IRQ_GPI2, 1, "GPI2", IORESOURCE_IRQ),
-	DEFINE_RES_NAMED(DA9062_IRQ_GPI3, 1, "GPI3", IORESOURCE_IRQ),
-	DEFINE_RES_NAMED(DA9062_IRQ_GPI4, 1, "GPI4", IORESOURCE_IRQ),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_GPI0, "GPI0"),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_GPI1, "GPI1"),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_GPI2, "GPI2"),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_GPI3, "GPI3"),
+	DEFINE_RES_IRQ_NAMED(DA9062_IRQ_GPI4, "GPI4"),
 };
 
 static const struct mfd_cell da9062_devs_irq[] = {
diff --git a/drivers/mfd/da9150-core.c b/drivers/mfd/da9150-core.c
index 5c59cc869fb3e2..19325d72a27e7e 100644
--- a/drivers/mfd/da9150-core.c
+++ b/drivers/mfd/da9150-core.c
@@ -450,7 +450,7 @@ static int da9150_probe(struct i2c_client *client)
 
 	da9150->irq_base = regmap_irq_chip_get_base(da9150->regmap_irq_data);
 
-	enable_irq_wake(da9150->irq);
+	da9150->irq_wake_enabled = !enable_irq_wake(da9150->irq);
 
 	ret = mfd_add_devices(da9150->dev, -1, da9150_devs,
 			      ARRAY_SIZE(da9150_devs), NULL,
@@ -463,6 +463,8 @@ static int da9150_probe(struct i2c_client *client)
 	return 0;
 
 mfd_fail:
+	if (da9150->irq_wake_enabled)
+		disable_irq_wake(da9150->irq);
 	regmap_del_irq_chip(da9150->irq, da9150->regmap_irq_data);
 regmap_irq_fail:
 	i2c_unregister_device(da9150->core_qif);
@@ -474,8 +476,10 @@ static void da9150_remove(struct i2c_client *client)
 {
 	struct da9150 *da9150 = i2c_get_clientdata(client);
 
-	regmap_del_irq_chip(da9150->irq, da9150->regmap_irq_data);
 	mfd_remove_devices(da9150->dev);
+	if (da9150->irq_wake_enabled)
+		disable_irq_wake(da9150->irq);
+	regmap_del_irq_chip(da9150->irq, da9150->regmap_irq_data);
 	i2c_unregister_device(da9150->core_qif);
 }
 
diff --git a/drivers/mfd/intel-lpss.c b/drivers/mfd/intel-lpss.c
index 63d6694f71457b..29f30ef0da9902 100644
--- a/drivers/mfd/intel-lpss.c
+++ b/drivers/mfd/intel-lpss.c
@@ -142,28 +142,21 @@ static void intel_lpss_cache_ltr(struct intel_lpss *lpss)
 	lpss->idle_ltr = readl(lpss->priv + LPSS_PRIV_IDLELTR);
 }
 
-static int intel_lpss_debugfs_add(struct intel_lpss *lpss)
+static void intel_lpss_debugfs_add(struct intel_lpss *lpss)
 {
-	struct dentry *dir;
-
-	dir = debugfs_create_dir(dev_name(lpss->dev), intel_lpss_debugfs);
-	if (IS_ERR(dir))
-		return PTR_ERR(dir);
+	lpss->debugfs = debugfs_create_dir(dev_name(lpss->dev), intel_lpss_debugfs);
 
 	/* Cache the values into lpss structure */
 	intel_lpss_cache_ltr(lpss);
 
-	debugfs_create_x32("capabilities", S_IRUGO, dir, &lpss->caps);
-	debugfs_create_x32("active_ltr", S_IRUGO, dir, &lpss->active_ltr);
-	debugfs_create_x32("idle_ltr", S_IRUGO, dir, &lpss->idle_ltr);
-
-	lpss->debugfs = dir;
-	return 0;
+	debugfs_create_x32("capabilities", 0444, lpss->debugfs, &lpss->caps);
+	debugfs_create_x32("active_ltr", 0444, lpss->debugfs, &lpss->active_ltr);
+	debugfs_create_x32("idle_ltr", 0444, lpss->debugfs, &lpss->idle_ltr);
 }
 
 static void intel_lpss_debugfs_remove(struct intel_lpss *lpss)
 {
-	debugfs_remove_recursive(lpss->debugfs);
+	debugfs_remove(lpss->debugfs);
 }
 
 static void intel_lpss_ltr_set(struct device *dev, s32 val)
@@ -432,10 +425,7 @@ int intel_lpss_probe(struct device *dev,
 		goto err_clk_register;
 
 	intel_lpss_ltr_expose(lpss);
-
-	ret = intel_lpss_debugfs_add(lpss);
-	if (ret)
-		dev_warn(dev, "Failed to create debugfs entries\n");
+	intel_lpss_debugfs_add(lpss);
 
 	if (intel_lpss_has_idma(lpss)) {
 		ret = mfd_add_devices(dev, lpss->devid, &intel_lpss_idma64_cell,
diff --git a/drivers/mfd/intel_quark_i2c_gpio.c b/drivers/mfd/intel_quark_i2c_gpio.c
index 9b9c76bd067b81..443a1a0cf7778d 100644
--- a/drivers/mfd/intel_quark_i2c_gpio.c
+++ b/drivers/mfd/intel_quark_i2c_gpio.c
@@ -35,11 +35,6 @@
 /* The Quark I2C controller source clock */
 #define INTEL_QUARK_I2C_CLK_HZ	33000000
 
-struct intel_quark_mfd {
-	struct clk		*i2c_clk;
-	struct clk_lookup	*i2c_clk_lookup;
-};
-
 static const struct property_entry intel_quark_i2c_controller_standard_properties[] = {
 	PROPERTY_ENTRY_U32("clock-frequency", I2C_MAX_STANDARD_MODE_FREQ),
 	{ }
@@ -160,37 +155,12 @@ MODULE_DEVICE_TABLE(pci, intel_quark_mfd_ids);
 
 static int intel_quark_register_i2c_clk(struct device *dev)
 {
-	struct intel_quark_mfd *quark_mfd = dev_get_drvdata(dev);
-	struct clk *i2c_clk;
-
-	i2c_clk = clk_register_fixed_rate(dev,
-					  INTEL_QUARK_I2C_CONTROLLER_CLK, NULL,
-					  0, INTEL_QUARK_I2C_CLK_HZ);
-	if (IS_ERR(i2c_clk))
-		return PTR_ERR(i2c_clk);
-
-	quark_mfd->i2c_clk = i2c_clk;
-	quark_mfd->i2c_clk_lookup = clkdev_create(i2c_clk, NULL,
-						INTEL_QUARK_I2C_CONTROLLER_CLK);
-
-	if (!quark_mfd->i2c_clk_lookup) {
-		clk_unregister(quark_mfd->i2c_clk);
-		dev_err(dev, "Fixed clk register failed\n");
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-static void intel_quark_unregister_i2c_clk(struct device *dev)
-{
-	struct intel_quark_mfd *quark_mfd = dev_get_drvdata(dev);
-
-	if (!quark_mfd->i2c_clk_lookup)
-		return;
+	struct clk_hw *hw;
 
-	clkdev_drop(quark_mfd->i2c_clk_lookup);
-	clk_unregister(quark_mfd->i2c_clk);
+	hw = devm_clk_hw_register_fixed_rate(dev, INTEL_QUARK_I2C_CONTROLLER_CLK,
+					     NULL, 0, INTEL_QUARK_I2C_CLK_HZ);
+	return devm_clk_hw_register_clkdev(dev, hw, NULL,
+					 INTEL_QUARK_I2C_CONTROLLER_CLK);
 }
 
 static int intel_quark_i2c_setup(struct pci_dev *pdev)
@@ -238,19 +208,12 @@ static int intel_quark_gpio_setup(struct pci_dev *pdev)
 static int intel_quark_mfd_probe(struct pci_dev *pdev,
 				 const struct pci_device_id *id)
 {
-	struct intel_quark_mfd *quark_mfd;
 	int ret;
 
 	ret = pcim_enable_device(pdev);
 	if (ret)
 		return ret;
 
-	quark_mfd = devm_kzalloc(&pdev->dev, sizeof(*quark_mfd), GFP_KERNEL);
-	if (!quark_mfd)
-		return -ENOMEM;
-
-	dev_set_drvdata(&pdev->dev, quark_mfd);
-
 	ret = intel_quark_register_i2c_clk(&pdev->dev);
 	if (ret)
 		return ret;
@@ -260,7 +223,7 @@ static int intel_quark_mfd_probe(struct pci_dev *pdev,
 	/* This driver only requires 1 IRQ vector */
 	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
 	if (ret < 0)
-		goto err_unregister_i2c_clk;
+		return ret;
 
 	ret = intel_quark_i2c_setup(pdev);
 	if (ret)
@@ -282,8 +245,6 @@ static int intel_quark_mfd_probe(struct pci_dev *pdev,
 	software_node_unregister_node_group(intel_quark_gpio_node_group);
 err_free_irq_vectors:
 	pci_free_irq_vectors(pdev);
-err_unregister_i2c_clk:
-	intel_quark_unregister_i2c_clk(&pdev->dev);
 	return ret;
 }
 
@@ -292,7 +253,6 @@ static void intel_quark_mfd_remove(struct pci_dev *pdev)
 	mfd_remove_devices(&pdev->dev);
 	software_node_unregister_node_group(intel_quark_gpio_node_group);
 	pci_free_irq_vectors(pdev);
-	intel_quark_unregister_i2c_clk(&pdev->dev);
 }
 
 static struct pci_driver intel_quark_mfd_driver = {
diff --git a/drivers/mfd/iqs62x.c b/drivers/mfd/iqs62x.c
index 412ae7777f7294..ef747a7b38455e 100644
--- a/drivers/mfd/iqs62x.c
+++ b/drivers/mfd/iqs62x.c
@@ -1015,7 +1015,7 @@ static void iqs62x_remove(struct i2c_client *client)
 	wait_for_completion(&iqs62x->fw_done);
 }
 
-static int __maybe_unused iqs62x_suspend(struct device *dev)
+static int iqs62x_suspend(struct device *dev)
 {
 	struct iqs62x_core *iqs62x = dev_get_drvdata(dev);
 	int ret;
@@ -1036,7 +1036,7 @@ static int __maybe_unused iqs62x_suspend(struct device *dev)
 				  IQS62X_PWR_SETTINGS_PWR_MODE_HALT);
 }
 
-static int __maybe_unused iqs62x_resume(struct device *dev)
+static int iqs62x_resume(struct device *dev)
 {
 	struct iqs62x_core *iqs62x = dev_get_drvdata(dev);
 	int ret;
@@ -1051,7 +1051,7 @@ static int __maybe_unused iqs62x_resume(struct device *dev)
 				  IQS62X_PWR_SETTINGS_DIS_AUTO, 0);
 }
 
-static SIMPLE_DEV_PM_OPS(iqs62x_pm, iqs62x_suspend, iqs62x_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(iqs62x_pm, iqs62x_suspend, iqs62x_resume);
 
 static const struct of_device_id iqs62x_of_match[] = {
 	{ .compatible = "azoteq,iqs620a" },
@@ -1067,7 +1067,7 @@ static struct i2c_driver iqs62x_i2c_driver = {
 	.driver = {
 		.name = "iqs62x",
 		.of_match_table = iqs62x_of_match,
-		.pm = &iqs62x_pm,
+		.pm = pm_sleep_ptr(&iqs62x_pm),
 	},
 	.probe = iqs62x_probe,
 	.remove = iqs62x_remove,
diff --git a/drivers/mfd/khadas-mcu.c b/drivers/mfd/khadas-mcu.c
index ba981a78869215..4b37e74e2b04ad 100644
--- a/drivers/mfd/khadas-mcu.c
+++ b/drivers/mfd/khadas-mcu.c
@@ -75,47 +75,116 @@ static const struct regmap_config khadas_mcu_regmap_config = {
 	.cache_type	= REGCACHE_MAPLE,
 };
 
-static struct mfd_cell khadas_mcu_fan_cells[] = {
+static const struct mfd_cell khadas_mcu_fan_cells[] = {
 	/* VIM1/2 Rev13+ and VIM3 only */
-	{ .name = "khadas-mcu-fan-ctrl", },
+	MFD_CELL_NAME("khadas-mcu-fan-ctrl"),
 };
 
-static struct mfd_cell khadas_mcu_cells[] = {
-	{ .name = "khadas-mcu-user-mem", },
+static const struct mfd_cell khadas_mcu_cells[] = {
+	MFD_CELL_NAME("khadas-mcu-user-mem"),
+};
+
+static bool khadas_mcu_vim4_reg_volatile(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case KHADAS_MCU_PWR_OFF_CMD_REG:
+	case KHADAS_MCU_VIM4_REST_CONF_REG:
+	case KHADAS_MCU_WOL_INIT_START_REG:
+	case KHADAS_MCU_VIM4_LED_ON_RAM_REG:
+	case KHADAS_MCU_VIM4_FAN_CTRL_REG:
+	case KHADAS_MCU_VIM4_WDT_EN_REG:
+	case KHADAS_MCU_VIM4_SYS_RST_REG:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool khadas_mcu_vim4_reg_writeable(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case KHADAS_MCU_VERSION_0_REG:
+	case KHADAS_MCU_VERSION_1_REG:
+	case KHADAS_MCU_SHUTDOWN_NORMAL_STATUS_REG:
+		return false;
+	default:
+		return true;
+	}
+}
+
+static const struct regmap_config khadas_mcu_vim4_regmap_config = {
+	.reg_bits	= 8,
+	.reg_stride	= 1,
+	.val_bits	= 8,
+	.max_register	= KHADAS_MCU_VIM4_SYS_RST_REG,
+	.volatile_reg	= khadas_mcu_vim4_reg_volatile,
+	.writeable_reg	= khadas_mcu_vim4_reg_writeable,
+	.cache_type	= REGCACHE_MAPLE,
+};
+
+static const struct mfd_cell khadas_mcu_vim4_fan_cells[] = {
+	MFD_CELL_NAME("khadas-mcu-vim4-fan"),
 };
 
 static int khadas_mcu_probe(struct i2c_client *client)
 {
+	const struct mfd_cell *cells, *fan_cells;
+	const struct regmap_config *regmap_cfg;
 	struct device *dev = &client->dev;
+	int ncells, nfan_cells, ret;
 	struct khadas_mcu *ddata;
-	int ret;
+	const void *mcu_variant;
 
 	ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
 	if (!ddata)
 		return -ENOMEM;
 
+	mcu_variant = i2c_get_match_data(client);
+	if (!mcu_variant)
+		return -ENODEV;
+
+	switch ((uintptr_t)mcu_variant) {
+	case KHADAS_MCU_GENERIC:
+		regmap_cfg	= &khadas_mcu_regmap_config;
+		cells		= khadas_mcu_cells;
+		ncells		= ARRAY_SIZE(khadas_mcu_cells);
+		fan_cells	= khadas_mcu_fan_cells;
+		nfan_cells	= ARRAY_SIZE(khadas_mcu_fan_cells);
+		break;
+	case KHADAS_MCU_VIM4:
+		regmap_cfg	= &khadas_mcu_vim4_regmap_config;
+		cells		= NULL;
+		ncells		= 0;
+		fan_cells	= khadas_mcu_vim4_fan_cells;
+		nfan_cells	= ARRAY_SIZE(khadas_mcu_vim4_fan_cells);
+		break;
+	default:
+		return -ENODEV;
+	}
+
 	i2c_set_clientdata(client, ddata);
 
 	ddata->dev = dev;
 
-	ddata->regmap = devm_regmap_init_i2c(client, &khadas_mcu_regmap_config);
+	ddata->regmap = devm_regmap_init_i2c(client, regmap_cfg);
 	if (IS_ERR(ddata->regmap)) {
 		ret = PTR_ERR(ddata->regmap);
-		dev_err(dev, "Failed to allocate register map: %d\n", ret);
-		return ret;
+		return dev_err_probe(dev, ret, "Failed to allocate register map\n");
 	}
 
-	ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE,
-				   khadas_mcu_cells,
-				   ARRAY_SIZE(khadas_mcu_cells),
-				   NULL, 0, NULL);
-	if (ret)
-		return ret;
+	if (cells && ncells) {
+		ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE,
+					   cells,
+					   ncells,
+					   NULL, 0, NULL);
+		if (ret)
+			return ret;
+	}
 
 	if (of_property_present(dev->of_node, "#cooling-cells"))
 		return devm_mfd_add_devices(dev, PLATFORM_DEVID_NONE,
-					    khadas_mcu_fan_cells,
-					    ARRAY_SIZE(khadas_mcu_fan_cells),
+					    fan_cells,
+					    nfan_cells,
 					    NULL, 0, NULL);
 
 	return 0;
@@ -123,7 +192,8 @@ static int khadas_mcu_probe(struct i2c_client *client)
 
 #ifdef CONFIG_OF
 static const struct of_device_id khadas_mcu_of_match[] = {
-	{ .compatible = "khadas,mcu", },
+	{ .compatible = "khadas,mcu", .data = (void *)KHADAS_MCU_GENERIC },
+	{ .compatible = "khadas,vim4-mcu", .data = (void *)KHADAS_MCU_VIM4 },
 	{},
 };
 MODULE_DEVICE_TABLE(of, khadas_mcu_of_match);
diff --git a/drivers/mfd/lm3533-core.c b/drivers/mfd/lm3533-core.c
index 0a2409d00b2e0b..9233da7ce09969 100644
--- a/drivers/mfd/lm3533-core.c
+++ b/drivers/mfd/lm3533-core.c
@@ -7,6 +7,7 @@
  * Author: Johan Hovold <jhovold@gmail.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -14,19 +15,24 @@
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/mfd/core.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
+#include <linux/units.h>
 
 #include <linux/mfd/lm3533.h>
 
 
 #define LM3533_BOOST_OVP_MASK		0x06
-#define LM3533_BOOST_OVP_SHIFT		1
+#define LM3533_BOOST_OVP_MIN		(16 * MICRO)
+#define LM3533_BOOST_OVP_MAX		(40 * MICRO)
 
 #define LM3533_BOOST_FREQ_MASK		0x01
-#define LM3533_BOOST_FREQ_SHIFT		0
+#define LM3533_BOOST_FREQ_MIN		(500 * HZ_PER_KHZ)
+#define LM3533_BOOST_FREQ_MAX		(1000 * HZ_PER_KHZ)
 
 #define LM3533_BL_ID_MASK		1
 #define LM3533_LED_ID_MASK		3
@@ -35,6 +41,7 @@
 
 #define LM3533_HVLED_ID_MAX		2
 #define LM3533_LVLED_ID_MAX		5
+#define LM3533_CELLS_MAX		7
 
 #define LM3533_REG_OUTPUT_CONF1		0x10
 #define LM3533_REG_OUTPUT_CONF2		0x11
@@ -42,125 +49,6 @@
 
 #define LM3533_REG_MAX			0xb2
 
-
-static struct mfd_cell lm3533_als_devs[] = {
-	{
-		.name	= "lm3533-als",
-		.id	= -1,
-	},
-};
-
-static struct mfd_cell lm3533_bl_devs[] = {
-	{
-		.name	= "lm3533-backlight",
-		.id	= 0,
-	},
-	{
-		.name	= "lm3533-backlight",
-		.id	= 1,
-	},
-};
-
-static struct mfd_cell lm3533_led_devs[] = {
-	{
-		.name	= "lm3533-leds",
-		.id	= 0,
-	},
-	{
-		.name	= "lm3533-leds",
-		.id	= 1,
-	},
-	{
-		.name	= "lm3533-leds",
-		.id	= 2,
-	},
-	{
-		.name	= "lm3533-leds",
-		.id	= 3,
-	},
-};
-
-int lm3533_read(struct lm3533 *lm3533, u8 reg, u8 *val)
-{
-	int tmp;
-	int ret;
-
-	ret = regmap_read(lm3533->regmap, reg, &tmp);
-	if (ret < 0) {
-		dev_err(lm3533->dev, "failed to read register %02x: %d\n",
-								reg, ret);
-		return ret;
-	}
-
-	*val = tmp;
-
-	dev_dbg(lm3533->dev, "read [%02x]: %02x\n", reg, *val);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(lm3533_read);
-
-int lm3533_write(struct lm3533 *lm3533, u8 reg, u8 val)
-{
-	int ret;
-
-	dev_dbg(lm3533->dev, "write [%02x]: %02x\n", reg, val);
-
-	ret = regmap_write(lm3533->regmap, reg, val);
-	if (ret < 0) {
-		dev_err(lm3533->dev, "failed to write register %02x: %d\n",
-								reg, ret);
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(lm3533_write);
-
-int lm3533_update(struct lm3533 *lm3533, u8 reg, u8 val, u8 mask)
-{
-	int ret;
-
-	dev_dbg(lm3533->dev, "update [%02x]: %02x/%02x\n", reg, val, mask);
-
-	ret = regmap_update_bits(lm3533->regmap, reg, mask, val);
-	if (ret < 0) {
-		dev_err(lm3533->dev, "failed to update register %02x: %d\n",
-								reg, ret);
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(lm3533_update);
-
-static int lm3533_set_boost_freq(struct lm3533 *lm3533,
-						enum lm3533_boost_freq freq)
-{
-	int ret;
-
-	ret = lm3533_update(lm3533, LM3533_REG_BOOST_PWM,
-					freq << LM3533_BOOST_FREQ_SHIFT,
-					LM3533_BOOST_FREQ_MASK);
-	if (ret)
-		dev_err(lm3533->dev, "failed to set boost frequency\n");
-
-	return ret;
-}
-
-
-static int lm3533_set_boost_ovp(struct lm3533 *lm3533,
-						enum lm3533_boost_ovp ovp)
-{
-	int ret;
-
-	ret = lm3533_update(lm3533, LM3533_REG_BOOST_PWM,
-					ovp << LM3533_BOOST_OVP_SHIFT,
-					LM3533_BOOST_OVP_MASK);
-	if (ret)
-		dev_err(lm3533->dev, "failed to set boost ovp\n");
-
-	return ret;
-}
-
 /*
  * HVLED output config -- output hvled controlled by backlight bl
  */
@@ -181,7 +69,8 @@ static int lm3533_set_hvled_config(struct lm3533 *lm3533, u8 hvled, u8 bl)
 	mask = LM3533_BL_ID_MASK << shift;
 	val = bl << shift;
 
-	ret = lm3533_update(lm3533, LM3533_REG_OUTPUT_CONF1, val, mask);
+	ret = regmap_update_bits(lm3533->regmap, LM3533_REG_OUTPUT_CONF1,
+				 mask, val);
 	if (ret)
 		dev_err(lm3533->dev, "failed to set hvled config\n");
 
@@ -216,21 +105,32 @@ static int lm3533_set_lvled_config(struct lm3533 *lm3533, u8 lvled, u8 led)
 	mask = LM3533_LED_ID_MASK << shift;
 	val = led << shift;
 
-	ret = lm3533_update(lm3533, reg, val, mask);
+	ret = regmap_update_bits(lm3533->regmap, reg, mask, val);
 	if (ret)
 		dev_err(lm3533->dev, "failed to set lvled config\n");
 
 	return ret;
 }
 
-static void lm3533_enable(struct lm3533 *lm3533)
+static int lm3533_enable(struct lm3533 *lm3533)
 {
+	int ret;
+
+	ret = regulator_enable(lm3533->vin_supply);
+	if (ret) {
+		dev_err(lm3533->dev, "failed to enable vin power supply\n");
+		return ret;
+	}
+
 	gpiod_set_value(lm3533->hwen, 1);
+
+	return 0;
 }
 
 static void lm3533_disable(struct lm3533 *lm3533)
 {
 	gpiod_set_value(lm3533->hwen, 0);
+	regulator_disable(lm3533->vin_supply);
 }
 
 enum lm3533_attribute_type {
@@ -258,7 +158,7 @@ static ssize_t show_output(struct device *dev,
 	struct lm3533_device_attribute *lattr = to_lm3533_dev_attr(attr);
 	int id = lattr->u.output.id;
 	u8 reg;
-	u8 val;
+	u32 val;
 	u8 mask;
 	int shift;
 	int ret;
@@ -278,7 +178,7 @@ static ssize_t show_output(struct device *dev,
 		mask = LM3533_LED_ID_MASK << shift;
 	}
 
-	ret = lm3533_read(lm3533, reg, &val);
+	ret = regmap_read(lm3533->regmap, reg, &val);
 	if (ret)
 		return ret;
 
@@ -376,136 +276,102 @@ static struct attribute_group lm3533_attribute_group = {
 	.attrs		= lm3533_attributes
 };
 
-static int lm3533_device_als_init(struct lm3533 *lm3533)
-{
-	struct lm3533_platform_data *pdata = dev_get_platdata(lm3533->dev);
-	int ret;
-
-	if (!pdata->als)
-		return 0;
-
-	lm3533_als_devs[0].platform_data = pdata->als;
-	lm3533_als_devs[0].pdata_size = sizeof(*pdata->als);
-
-	ret = mfd_add_devices(lm3533->dev, 0, lm3533_als_devs, 1, NULL,
-			      0, NULL);
-	if (ret) {
-		dev_err(lm3533->dev, "failed to add ALS device\n");
-		return ret;
-	}
-
-	lm3533->have_als = 1;
-
-	return 0;
-}
+static const struct attribute_group *lm3533_attribute_groups[] = {
+	&lm3533_attribute_group,
+	NULL,
+};
 
-static int lm3533_device_bl_init(struct lm3533 *lm3533)
+static int lm3533_device_init(struct lm3533 *lm3533)
 {
-	struct lm3533_platform_data *pdata = dev_get_platdata(lm3533->dev);
-	int i;
+	struct device *dev = lm3533->dev;
+	struct mfd_cell *lm3533_devices;
+	u32 reg, nchilds;
+	u32 count = 0;
 	int ret;
 
-	if (!pdata->backlights || pdata->num_backlights == 0)
-		return 0;
+	nchilds = device_get_child_node_count(dev);
+	if (!nchilds || nchilds > LM3533_CELLS_MAX)
+		return dev_err_probe(dev, -ENODEV,
+				     "num of child nodes is not supported\n");
 
-	if (pdata->num_backlights > ARRAY_SIZE(lm3533_bl_devs))
-		pdata->num_backlights = ARRAY_SIZE(lm3533_bl_devs);
+	lm3533_devices = devm_kcalloc(dev, nchilds, sizeof(*lm3533_devices),
+				      GFP_KERNEL);
+	if (!lm3533_devices)
+		return -ENOMEM;
 
-	for (i = 0; i < pdata->num_backlights; ++i) {
-		lm3533_bl_devs[i].platform_data = &pdata->backlights[i];
-		lm3533_bl_devs[i].pdata_size = sizeof(pdata->backlights[i]);
+	device_for_each_child_node_scoped(dev, child) {
+		if (count >= nchilds)
+			break;
+
+		if (fwnode_device_is_compatible(child, "ti,lm3533-als")) {
+			lm3533_devices[count].name = "lm3533-als";
+			lm3533_devices[count].of_compatible = "ti,lm3533-als";
+			lm3533_devices[count].id = PLATFORM_DEVID_NONE;
+
+			lm3533->have_als = true;
+			count++;
+		} else if (fwnode_device_is_compatible(child, "ti,lm3533-backlight")) {
+			ret = fwnode_property_read_u32(child, "reg", &reg);
+			if (ret || reg >= LM3533_HVLED_ID_MAX) {
+				dev_err(dev, "invalid backlight node %pfw\n", child);
+				continue;
+			}
+
+			lm3533_devices[count].name = "lm3533-backlight";
+			lm3533_devices[count].of_compatible = "ti,lm3533-backlight";
+			lm3533_devices[count].id = reg;
+			lm3533_devices[count].of_reg = reg;
+			lm3533_devices[count].use_of_reg = true;
+
+			lm3533->have_backlights = true;
+			count++;
+		} else if (fwnode_device_is_compatible(child, "ti,lm3533-leds")) {
+			ret = fwnode_property_read_u32(child, "reg", &reg);
+			if (ret || reg < LM3533_HVLED_ID_MAX ||
+			    reg > LM3533_LVLED_ID_MAX) {
+				dev_err(dev, "invalid LED node %pfw\n", child);
+				continue;
+			}
+
+			lm3533_devices[count].name = "lm3533-leds";
+			lm3533_devices[count].of_compatible = "ti,lm3533-leds";
+			lm3533_devices[count].id = reg - LM3533_HVLED_ID_MAX;
+			lm3533_devices[count].of_reg = reg;
+			lm3533_devices[count].use_of_reg = true;
+
+			lm3533->have_leds = true;
+			count++;
+		}
 	}
 
-	ret = mfd_add_devices(lm3533->dev, 0, lm3533_bl_devs,
-			      pdata->num_backlights, NULL, 0, NULL);
-	if (ret) {
-		dev_err(lm3533->dev, "failed to add backlight devices\n");
+	ret = lm3533_enable(lm3533);
+	if (ret)
 		return ret;
-	}
 
-	lm3533->have_backlights = 1;
-
-	return 0;
-}
-
-static int lm3533_device_led_init(struct lm3533 *lm3533)
-{
-	struct lm3533_platform_data *pdata = dev_get_platdata(lm3533->dev);
-	int i;
-	int ret;
-
-	if (!pdata->leds || pdata->num_leds == 0)
-		return 0;
-
-	if (pdata->num_leds > ARRAY_SIZE(lm3533_led_devs))
-		pdata->num_leds = ARRAY_SIZE(lm3533_led_devs);
-
-	for (i = 0; i < pdata->num_leds; ++i) {
-		lm3533_led_devs[i].platform_data = &pdata->leds[i];
-		lm3533_led_devs[i].pdata_size = sizeof(pdata->leds[i]);
-	}
-
-	ret = mfd_add_devices(lm3533->dev, 0, lm3533_led_devs,
-			      pdata->num_leds, NULL, 0, NULL);
+	ret = regmap_update_bits(lm3533->regmap, LM3533_REG_BOOST_PWM,
+				 LM3533_BOOST_FREQ_MASK,
+				 FIELD_PREP(LM3533_BOOST_FREQ_MASK, lm3533->boost_freq));
 	if (ret) {
-		dev_err(lm3533->dev, "failed to add LED devices\n");
-		return ret;
+		dev_err(dev, "failed to set boost frequency\n");
+		goto err_disable;
 	}
 
-	lm3533->have_leds = 1;
-
-	return 0;
-}
-
-static int lm3533_device_setup(struct lm3533 *lm3533,
-					struct lm3533_platform_data *pdata)
-{
-	int ret;
-
-	ret = lm3533_set_boost_freq(lm3533, pdata->boost_freq);
-	if (ret)
-		return ret;
-
-	return lm3533_set_boost_ovp(lm3533, pdata->boost_ovp);
-}
-
-static int lm3533_device_init(struct lm3533 *lm3533)
-{
-	struct lm3533_platform_data *pdata = dev_get_platdata(lm3533->dev);
-	int ret;
-
-	dev_dbg(lm3533->dev, "%s\n", __func__);
-
-	if (!pdata) {
-		dev_err(lm3533->dev, "no platform data\n");
-		return -EINVAL;
+	ret = regmap_update_bits(lm3533->regmap, LM3533_REG_BOOST_PWM,
+				 LM3533_BOOST_OVP_MASK,
+				 FIELD_PREP(LM3533_BOOST_OVP_MASK, lm3533->boost_ovp));
+	if (ret) {
+		dev_err(dev, "failed to set boost ovp\n");
+		goto err_disable;
 	}
 
-	lm3533->hwen = devm_gpiod_get(lm3533->dev, NULL, GPIOD_OUT_LOW);
-	if (IS_ERR(lm3533->hwen))
-		return dev_err_probe(lm3533->dev, PTR_ERR(lm3533->hwen), "failed to request HWEN GPIO\n");
-	gpiod_set_consumer_name(lm3533->hwen, "lm3533-hwen");
-
-	lm3533_enable(lm3533);
-
-	ret = lm3533_device_setup(lm3533, pdata);
-	if (ret)
+	ret = mfd_add_devices(dev, 0, lm3533_devices, count, NULL, 0, NULL);
+	if (ret) {
+		dev_err(dev, "failed to add MFD devices: %d\n", ret);
 		goto err_disable;
-
-	lm3533_device_als_init(lm3533);
-	lm3533_device_bl_init(lm3533);
-	lm3533_device_led_init(lm3533);
-
-	ret = sysfs_create_group(&lm3533->dev->kobj, &lm3533_attribute_group);
-	if (ret < 0) {
-		dev_err(lm3533->dev, "failed to create sysfs attributes\n");
-		goto err_unregister;
 	}
 
 	return 0;
 
-err_unregister:
-	mfd_remove_devices(lm3533->dev);
 err_disable:
 	lm3533_disable(lm3533);
 
@@ -516,8 +382,6 @@ static void lm3533_device_exit(struct lm3533 *lm3533)
 {
 	dev_dbg(lm3533->dev, "%s\n", __func__);
 
-	sysfs_remove_group(&lm3533->dev->kobj, &lm3533_attribute_group);
-
 	mfd_remove_devices(lm3533->dev);
 	lm3533_disable(lm3533);
 }
@@ -589,7 +453,35 @@ static int lm3533_i2c_probe(struct i2c_client *i2c)
 		return PTR_ERR(lm3533->regmap);
 
 	lm3533->dev = &i2c->dev;
-	lm3533->irq = i2c->irq;
+
+	lm3533->hwen = devm_gpiod_get_optional(lm3533->dev, "enable",
+					       GPIOD_OUT_LOW);
+	if (IS_ERR(lm3533->hwen))
+		return dev_err_probe(lm3533->dev, PTR_ERR(lm3533->hwen),
+				     "failed to get HWEN GPIO\n");
+
+	lm3533->vin_supply = devm_regulator_get(lm3533->dev, "vin");
+	if (IS_ERR(lm3533->vin_supply))
+		return dev_err_probe(lm3533->dev, PTR_ERR(lm3533->vin_supply),
+				     "failed to get vin-supply\n");
+
+	device_property_read_u32(lm3533->dev, "ti,boost-ovp-microvolt",
+				 &lm3533->boost_ovp);
+
+	lm3533->boost_ovp = clamp(lm3533->boost_ovp, LM3533_BOOST_OVP_MIN,
+				  LM3533_BOOST_OVP_MAX);
+	lm3533->boost_ovp = lm3533->boost_ovp / (8 * MICRO) - 2;
+
+	device_property_read_u32(lm3533->dev, "ti,boost-freq-hz",
+				 &lm3533->boost_freq);
+
+	lm3533->boost_freq = clamp(lm3533->boost_freq, LM3533_BOOST_FREQ_MIN,
+				   LM3533_BOOST_FREQ_MAX);
+	lm3533->boost_freq = lm3533->boost_freq / (500 * KILO) - 1;
+
+	/* LM3533 and child devices do not use DMA */
+	i2c->dev.coherent_dma_mask = 0;
+	i2c->dev.dma_mask = &i2c->dev.coherent_dma_mask;
 
 	return lm3533_device_init(lm3533);
 }
@@ -603,6 +495,12 @@ static void lm3533_i2c_remove(struct i2c_client *i2c)
 	lm3533_device_exit(lm3533);
 }
 
+static const struct of_device_id lm3533_match_table[] = {
+	{ .compatible = "ti,lm3533" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lm3533_match_table);
+
 static const struct i2c_device_id lm3533_i2c_ids[] = {
 	{ "lm3533" },
 	{ }
@@ -612,6 +510,8 @@ MODULE_DEVICE_TABLE(i2c, lm3533_i2c_ids);
 static struct i2c_driver lm3533_i2c_driver = {
 	.driver = {
 		   .name = "lm3533",
+		   .dev_groups = lm3533_attribute_groups,
+		   .of_match_table = lm3533_match_table,
 	},
 	.id_table	= lm3533_i2c_ids,
 	.probe		= lm3533_i2c_probe,
diff --git a/drivers/mfd/lm3533-ctrlbank.c b/drivers/mfd/lm3533-ctrlbank.c
index 2537dfade51cd9..3aab8ece4e8cb5 100644
--- a/drivers/mfd/lm3533-ctrlbank.c
+++ b/drivers/mfd/lm3533-ctrlbank.c
@@ -9,14 +9,10 @@
 
 #include <linux/device.h>
 #include <linux/module.h>
+#include <linux/regmap.h>
 
 #include <linux/mfd/lm3533.h>
 
-
-#define LM3533_MAX_CURRENT_MIN		5000
-#define LM3533_MAX_CURRENT_MAX		29800
-#define LM3533_MAX_CURRENT_STEP		800
-
 #define LM3533_PWM_MAX			0x3f
 
 #define LM3533_REG_PWM_BASE		0x14
@@ -38,8 +34,8 @@ int lm3533_ctrlbank_enable(struct lm3533_ctrlbank *cb)
 	dev_dbg(cb->dev, "%s - %d\n", __func__, cb->id);
 
 	mask = 1 << cb->id;
-	ret = lm3533_update(cb->lm3533, LM3533_REG_CTRLBANK_ENABLE,
-								mask, mask);
+	ret = regmap_set_bits(cb->regmap, LM3533_REG_CTRLBANK_ENABLE,
+			      mask);
 	if (ret)
 		dev_err(cb->dev, "failed to enable ctrlbank %d\n", cb->id);
 
@@ -55,7 +51,8 @@ int lm3533_ctrlbank_disable(struct lm3533_ctrlbank *cb)
 	dev_dbg(cb->dev, "%s - %d\n", __func__, cb->id);
 
 	mask = 1 << cb->id;
-	ret = lm3533_update(cb->lm3533, LM3533_REG_CTRLBANK_ENABLE, 0, mask);
+	ret = regmap_clear_bits(cb->regmap, LM3533_REG_CTRLBANK_ENABLE,
+				mask);
 	if (ret)
 		dev_err(cb->dev, "failed to disable ctrlbank %d\n", cb->id);
 
@@ -68,7 +65,7 @@ EXPORT_SYMBOL_GPL(lm3533_ctrlbank_disable);
  *
  * imax		5000 - 29800 uA (800 uA step)
  */
-int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb, u16 imax)
+int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb, u32 imax)
 {
 	u8 reg;
 	u8 val;
@@ -80,7 +77,7 @@ int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb, u16 imax)
 	val = (imax - LM3533_MAX_CURRENT_MIN) / LM3533_MAX_CURRENT_STEP;
 
 	reg = lm3533_ctrlbank_get_reg(cb, LM3533_REG_MAX_CURRENT_BASE);
-	ret = lm3533_write(cb->lm3533, reg, val);
+	ret = regmap_write(cb->regmap, reg, val);
 	if (ret)
 		dev_err(cb->dev, "failed to set max current\n");
 
@@ -88,13 +85,13 @@ int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb, u16 imax)
 }
 EXPORT_SYMBOL_GPL(lm3533_ctrlbank_set_max_current);
 
-int lm3533_ctrlbank_set_brightness(struct lm3533_ctrlbank *cb, u8 val)
+int lm3533_ctrlbank_set_brightness(struct lm3533_ctrlbank *cb, u32 val)
 {
 	u8 reg;
 	int ret;
 
 	reg = lm3533_ctrlbank_get_reg(cb, LM3533_REG_BRIGHTNESS_BASE);
-	ret = lm3533_write(cb->lm3533, reg, val);
+	ret = regmap_write(cb->regmap, reg, val);
 	if (ret)
 		dev_err(cb->dev, "failed to set brightness\n");
 
@@ -102,13 +99,13 @@ int lm3533_ctrlbank_set_brightness(struct lm3533_ctrlbank *cb, u8 val)
 }
 EXPORT_SYMBOL_GPL(lm3533_ctrlbank_set_brightness);
 
-int lm3533_ctrlbank_get_brightness(struct lm3533_ctrlbank *cb, u8 *val)
+int lm3533_ctrlbank_get_brightness(struct lm3533_ctrlbank *cb, u32 *val)
 {
 	u8 reg;
 	int ret;
 
 	reg = lm3533_ctrlbank_get_reg(cb, LM3533_REG_BRIGHTNESS_BASE);
-	ret = lm3533_read(cb->lm3533, reg, val);
+	ret = regmap_read(cb->regmap, reg, val);
 	if (ret)
 		dev_err(cb->dev, "failed to get brightness\n");
 
@@ -126,7 +123,7 @@ EXPORT_SYMBOL_GPL(lm3533_ctrlbank_get_brightness);
  *   bit 1 - PWM-input enabled in Zone 0
  *   bit 0 - PWM-input enabled
  */
-int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u8 val)
+int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u32 val)
 {
 	u8 reg;
 	int ret;
@@ -135,7 +132,7 @@ int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u8 val)
 		return -EINVAL;
 
 	reg = lm3533_ctrlbank_get_reg(cb, LM3533_REG_PWM_BASE);
-	ret = lm3533_write(cb->lm3533, reg, val);
+	ret = regmap_write(cb->regmap, reg, val);
 	if (ret)
 		dev_err(cb->dev, "failed to set PWM mask\n");
 
@@ -143,13 +140,13 @@ int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u8 val)
 }
 EXPORT_SYMBOL_GPL(lm3533_ctrlbank_set_pwm);
 
-int lm3533_ctrlbank_get_pwm(struct lm3533_ctrlbank *cb, u8 *val)
+int lm3533_ctrlbank_get_pwm(struct lm3533_ctrlbank *cb, u32 *val)
 {
 	u8 reg;
 	int ret;
 
 	reg = lm3533_ctrlbank_get_reg(cb, LM3533_REG_PWM_BASE);
-	ret = lm3533_read(cb->lm3533, reg, val);
+	ret = regmap_read(cb->regmap, reg, val);
 	if (ret)
 		dev_err(cb->dev, "failed to get PWM mask\n");
 
diff --git a/drivers/mfd/max77843.c b/drivers/mfd/max77843.c
index 2a48577b1a79f2..9ae6715642ae17 100644
--- a/drivers/mfd/max77843.c
+++ b/drivers/mfd/max77843.c
@@ -173,7 +173,7 @@ static const struct i2c_device_id max77843_id[] = {
 	{ },
 };
 
-static int __maybe_unused max77843_suspend(struct device *dev)
+static int max77843_suspend(struct device *dev)
 {
 	struct i2c_client *i2c = to_i2c_client(dev);
 	struct max77693_dev *max77843 = i2c_get_clientdata(i2c);
@@ -185,7 +185,7 @@ static int __maybe_unused max77843_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused max77843_resume(struct device *dev)
+static int max77843_resume(struct device *dev)
 {
 	struct i2c_client *i2c = to_i2c_client(dev);
 	struct max77693_dev *max77843 = i2c_get_clientdata(i2c);
@@ -197,12 +197,12 @@ static int __maybe_unused max77843_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(max77843_pm, max77843_suspend, max77843_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(max77843_pm, max77843_suspend, max77843_resume);
 
 static struct i2c_driver max77843_i2c_driver = {
 	.driver	= {
 		.name = "max77843",
-		.pm = &max77843_pm,
+		.pm = pm_sleep_ptr(&max77843_pm),
 		.of_match_table = max77843_dt_match,
 		.suppress_bind_attrs = true,
 	},
diff --git a/drivers/mfd/motorola-cpcap.c b/drivers/mfd/motorola-cpcap.c
index 4ed8704c985ab2..65d5eb18aa0044 100644
--- a/drivers/mfd/motorola-cpcap.c
+++ b/drivers/mfd/motorola-cpcap.c
@@ -11,6 +11,7 @@
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/sysfs.h>
 
@@ -29,6 +30,7 @@ struct cpcap_ddata {
 	struct regmap_irq_chip_data *irqdata[CPCAP_NR_IRQ_CHIPS];
 	const struct regmap_config *regmap_conf;
 	struct regmap *regmap;
+	enum cpcap_variant variant;
 };
 
 static int cpcap_sense_irq(struct regmap *regmap, int irq)
@@ -194,20 +196,6 @@ static int cpcap_init_irq(struct cpcap_ddata *cpcap)
 	return 0;
 }
 
-static const struct of_device_id cpcap_of_match[] = {
-	{ .compatible = "motorola,cpcap", },
-	{ .compatible = "st,6556002", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, cpcap_of_match);
-
-static const struct spi_device_id cpcap_spi_ids[] = {
-	{ .name = "cpcap" },
-	{ .name = "6556002" },
-	{ }
-};
-MODULE_DEVICE_TABLE(spi, cpcap_spi_ids);
-
 static const struct regmap_config cpcap_regmap_config = {
 	.reg_bits = 16,
 	.reg_stride = 4,
@@ -240,62 +228,74 @@ static int cpcap_resume(struct device *dev)
 
 static DEFINE_SIMPLE_DEV_PM_OPS(cpcap_pm, cpcap_suspend, cpcap_resume);
 
-static const struct mfd_cell cpcap_mfd_devices[] = {
-	{
-		.name          = "cpcap_adc",
-		.of_compatible = "motorola,mapphone-cpcap-adc",
-	}, {
-		.name          = "cpcap_battery",
-		.of_compatible = "motorola,cpcap-battery",
-	}, {
-		.name          = "cpcap-charger",
-		.of_compatible = "motorola,mapphone-cpcap-charger",
-	}, {
-		.name          = "cpcap-regulator",
-		.of_compatible = "motorola,mapphone-cpcap-regulator",
-	}, {
-		.name          = "cpcap-rtc",
-		.of_compatible = "motorola,cpcap-rtc",
-	}, {
-		.name          = "cpcap-pwrbutton",
-		.of_compatible = "motorola,cpcap-pwrbutton",
-	}, {
-		.name          = "cpcap-usb-phy",
-		.of_compatible = "motorola,mapphone-cpcap-usb-phy",
-	}, {
-		.name          = "cpcap-led",
-		.id            = 0,
-		.of_compatible = "motorola,cpcap-led-red",
-	}, {
-		.name          = "cpcap-led",
-		.id            = 1,
-		.of_compatible = "motorola,cpcap-led-green",
-	}, {
-		.name          = "cpcap-led",
-		.id            = 2,
-		.of_compatible = "motorola,cpcap-led-blue",
-	}, {
-		.name          = "cpcap-led",
-		.id            = 3,
-		.of_compatible = "motorola,cpcap-led-adl",
-	}, {
-		.name          = "cpcap-led",
-		.id            = 4,
-		.of_compatible = "motorola,cpcap-led-cp",
-	}, {
-		.name          = "cpcap-codec",
-	}
+static const struct mfd_cell cpcap_common_devices[] = {
+	MFD_CELL_OF("cpcap_battery", NULL, NULL, 0, 0, "motorola,cpcap-battery"),
+	MFD_CELL_OF("cpcap-rtc", NULL, NULL, 0, 0, "motorola,cpcap-rtc"),
+	MFD_CELL_OF("cpcap-pwrbutton", NULL, NULL, 0, 0, "motorola,cpcap-pwrbutton"),
+	MFD_CELL_OF("cpcap-led", NULL, NULL, 0, 0, "motorola,cpcap-led-red"),
+	MFD_CELL_OF("cpcap-led", NULL, NULL, 0, 1, "motorola,cpcap-led-green"),
+	MFD_CELL_OF("cpcap-led", NULL, NULL, 0, 2, "motorola,cpcap-led-blue"),
+	MFD_CELL_OF("cpcap-led", NULL, NULL, 0, 3, "motorola,cpcap-led-adl"),
+	MFD_CELL_NAME("cpcap-codec"),
+};
+
+static const struct mfd_cell cpcap_default_devices[] = {
+	MFD_CELL_OF("cpcap_adc", NULL, NULL, 0, 0, "motorola,cpcap-adc"),
+	MFD_CELL_OF("cpcap-regulator", NULL, NULL, 0, 0, "motorola,cpcap-regulator"),
+	MFD_CELL_OF("cpcap-usb-phy", NULL, NULL, 0, 0, "motorola,cpcap-usb-phy"),
+	MFD_CELL_OF("cpcap-led", NULL, NULL, 0, 4, "motorola,cpcap-led-cp"),
+};
+
+static const struct mfd_cell cpcap_mapphone_devices[] = {
+	MFD_CELL_OF("cpcap_adc", NULL, NULL, 0, 0, "motorola,mapphone-cpcap-adc"),
+	MFD_CELL_OF("cpcap-charger", NULL, NULL, 0, 0, "motorola,mapphone-cpcap-charger"),
+	MFD_CELL_OF("cpcap-regulator", NULL, NULL, 0, 0, "motorola,mapphone-cpcap-regulator"),
+	MFD_CELL_OF("cpcap-usb-phy", NULL, NULL, 0, 0, "motorola,mapphone-cpcap-usb-phy"),
+	MFD_CELL_OF("cpcap-led", NULL, NULL, 0, 4, "motorola,cpcap-led-cp"),
+};
+
+/*
+ * The Mot board features a USB-PHY and charger similar to the ones in
+ * Mapphone; however, because Mot is based on Tegra20, it is incompatible
+ * with the existing implementation, which is tightly interconnected with
+ * the OMAP USB PHY.
+ */
+static const struct mfd_cell cpcap_mot_devices[] = {
+	MFD_CELL_OF("cpcap_adc", NULL, NULL, 0, 0, "motorola,mot-cpcap-adc"),
+	MFD_CELL_OF("cpcap-regulator", NULL, NULL, 0, 0, "motorola,mot-cpcap-regulator"),
 };
 
 static int cpcap_probe(struct spi_device *spi)
 {
 	struct cpcap_ddata *cpcap;
+	const struct mfd_cell *cells;
+	unsigned int num_cells;
 	int ret;
 
 	cpcap = devm_kzalloc(&spi->dev, sizeof(*cpcap), GFP_KERNEL);
 	if (!cpcap)
 		return -ENOMEM;
 
+	cpcap->variant = (enum cpcap_variant)spi_get_device_match_data(spi);
+
+	switch (cpcap->variant) {
+	case CPCAP_DEFAULT:
+		cells = cpcap_default_devices;
+		num_cells = ARRAY_SIZE(cpcap_default_devices);
+		break;
+	case CPCAP_MAPPHONE:
+		cells = cpcap_mapphone_devices;
+		num_cells = ARRAY_SIZE(cpcap_mapphone_devices);
+		break;
+	case CPCAP_MOT:
+		cells = cpcap_mot_devices;
+		num_cells = ARRAY_SIZE(cpcap_mot_devices);
+		break;
+	default:
+		return dev_err_probe(&spi->dev, -ENODEV,
+				     "Unknown device %d\n", cpcap->variant);
+	}
+
 	cpcap->spi = spi;
 	spi_set_drvdata(spi, cpcap);
 
@@ -330,10 +330,31 @@ static int cpcap_probe(struct spi_device *spi)
 	spi->dev.coherent_dma_mask = 0;
 	spi->dev.dma_mask = &spi->dev.coherent_dma_mask;
 
-	return devm_mfd_add_devices(&spi->dev, 0, cpcap_mfd_devices,
-				    ARRAY_SIZE(cpcap_mfd_devices), NULL, 0, NULL);
+	ret = devm_mfd_add_devices(&spi->dev, 0, cpcap_common_devices,
+				   ARRAY_SIZE(cpcap_common_devices), NULL, 0, NULL);
+	if (ret)
+		return dev_err_probe(&spi->dev, ret,
+				     "Failed to add common child devices\n");
+
+	return devm_mfd_add_devices(&spi->dev, 0, cells, num_cells, NULL, 0, NULL);
 }
 
+static const struct of_device_id cpcap_of_match[] = {
+	{ .compatible = "motorola,cpcap", .data = (void *)CPCAP_DEFAULT },
+	{ .compatible = "motorola,mapphone-cpcap", .data = (void *)CPCAP_MAPPHONE },
+	{ .compatible = "motorola,mot-cpcap", .data = (void *)CPCAP_MOT },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, cpcap_of_match);
+
+static const struct spi_device_id cpcap_spi_ids[] = {
+	{ .name = "cpcap", .driver_data = CPCAP_DEFAULT },
+	{ .name = "mapphone-cpcap", .driver_data = CPCAP_MAPPHONE },
+	{ .name = "mot-cpcap", .driver_data = CPCAP_MOT },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(spi, cpcap_spi_ids);
+
 static struct spi_driver cpcap_driver = {
 	.driver = {
 		.name = "cpcap-core",
diff --git a/drivers/mfd/mt6360-core.c b/drivers/mfd/mt6360-core.c
index b9b1036c8ff463..8a97c30cad3fee 100644
--- a/drivers/mfd/mt6360-core.c
+++ b/drivers/mfd/mt6360-core.c
@@ -583,7 +583,7 @@ static int mt6360_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int __maybe_unused mt6360_suspend(struct device *dev)
+static int mt6360_suspend(struct device *dev)
 {
 	struct i2c_client *i2c = to_i2c_client(dev);
 
@@ -593,7 +593,7 @@ static int __maybe_unused mt6360_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused mt6360_resume(struct device *dev)
+static int mt6360_resume(struct device *dev)
 {
 
 	struct i2c_client *i2c = to_i2c_client(dev);
@@ -604,7 +604,7 @@ static int __maybe_unused mt6360_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(mt6360_pm_ops, mt6360_suspend, mt6360_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(mt6360_pm_ops, mt6360_suspend, mt6360_resume);
 
 static const struct of_device_id __maybe_unused mt6360_of_id[] = {
 	{ .compatible = "mediatek,mt6360", },
@@ -615,7 +615,7 @@ MODULE_DEVICE_TABLE(of, mt6360_of_id);
 static struct i2c_driver mt6360_driver = {
 	.driver = {
 		.name = "mt6360",
-		.pm = &mt6360_pm_ops,
+		.pm = pm_sleep_ptr(&mt6360_pm_ops),
 		.of_match_table = of_match_ptr(mt6360_of_id),
 	},
 	.probe = mt6360_probe,
diff --git a/drivers/mfd/rk8xx-core.c b/drivers/mfd/rk8xx-core.c
index 3dcf6abfda74ff..21182df45d1b6b 100644
--- a/drivers/mfd/rk8xx-core.c
+++ b/drivers/mfd/rk8xx-core.c
@@ -875,7 +875,7 @@ int rk8xx_probe(struct device *dev, int variant, unsigned int irq, struct regmap
 	if (device_property_read_bool(dev, "system-power-controller") ||
 	    device_property_read_bool(dev, "rockchip,system-power-controller")) {
 		ret = devm_register_sys_off_handler(dev,
-				    SYS_OFF_MODE_POWER_OFF_PREPARE, SYS_OFF_PRIO_HIGH,
+				    SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_HIGH,
 				    &rk808_power_off, rk808);
 		if (ret)
 			return dev_err_probe(dev, ret,
diff --git a/drivers/mfd/rk8xx-i2c.c b/drivers/mfd/rk8xx-i2c.c
index 2951b2911a37ae..161b25fbe146ec 100644
--- a/drivers/mfd/rk8xx-i2c.c
+++ b/drivers/mfd/rk8xx-i2c.c
@@ -251,7 +251,7 @@ static void rk8xx_i2c_shutdown(struct i2c_client *client)
 	rk8xx_shutdown(&client->dev);
 }
 
-static SIMPLE_DEV_PM_OPS(rk8xx_i2c_pm_ops, rk8xx_suspend, rk8xx_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(rk8xx_i2c_pm_ops, rk8xx_suspend, rk8xx_resume);
 
 static const struct of_device_id rk8xx_i2c_of_match[] = {
 	{ .compatible = "rockchip,rk801", .data = &rk801_data },
@@ -270,7 +270,7 @@ static struct i2c_driver rk8xx_i2c_driver = {
 	.driver = {
 		.name = "rk8xx-i2c",
 		.of_match_table = rk8xx_i2c_of_match,
-		.pm = &rk8xx_i2c_pm_ops,
+		.pm = pm_sleep_ptr(&rk8xx_i2c_pm_ops),
 	},
 	.probe = rk8xx_i2c_probe,
 	.shutdown  = rk8xx_i2c_shutdown,
diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c
index 23ca00d2c624d9..8c68dd16bb9e31 100644
--- a/drivers/mfd/rn5t618.c
+++ b/drivers/mfd/rn5t618.c
@@ -243,7 +243,7 @@ static void rn5t618_i2c_remove(struct i2c_client *i2c)
 	unregister_restart_handler(&rn5t618_restart_handler);
 }
 
-static int __maybe_unused rn5t618_i2c_suspend(struct device *dev)
+static int rn5t618_i2c_suspend(struct device *dev)
 {
 	struct rn5t618 *priv = dev_get_drvdata(dev);
 
@@ -253,7 +253,7 @@ static int __maybe_unused rn5t618_i2c_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused rn5t618_i2c_resume(struct device *dev)
+static int rn5t618_i2c_resume(struct device *dev)
 {
 	struct rn5t618 *priv = dev_get_drvdata(dev);
 
@@ -263,7 +263,7 @@ static int __maybe_unused rn5t618_i2c_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(rn5t618_i2c_dev_pm_ops,
+static DEFINE_SIMPLE_DEV_PM_OPS(rn5t618_i2c_dev_pm_ops,
 			rn5t618_i2c_suspend,
 			rn5t618_i2c_resume);
 
@@ -271,7 +271,7 @@ static struct i2c_driver rn5t618_i2c_driver = {
 	.driver = {
 		.name = "rn5t618",
 		.of_match_table = rn5t618_of_match,
-		.pm = &rn5t618_i2c_dev_pm_ops,
+		.pm = pm_sleep_ptr(&rn5t618_i2c_dev_pm_ops),
 	},
 	.probe = rn5t618_i2c_probe,
 	.remove = rn5t618_i2c_remove,
diff --git a/drivers/mfd/rohm-bd71828.c b/drivers/mfd/rohm-bd71828.c
index bf4df8e1b34c86..46faef7058f807 100644
--- a/drivers/mfd/rohm-bd71828.c
+++ b/drivers/mfd/rohm-bd71828.c
@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2019 ROHM Semiconductors
+ * Copyright (C) 2019, 2026 ROHM Semiconductor
  *
- * ROHM BD718[15/28/79] and BD72720 PMIC driver
+ * ROHM BD718[15/28/79], BD72720 and BD73[8/9]00 PMIC driver
  */
 
 #include <linux/i2c.h>
@@ -14,6 +14,7 @@
 #include <linux/mfd/rohm-bd71815.h>
 #include <linux/mfd/rohm-bd71828.h>
 #include <linux/mfd/rohm-bd72720.h>
+#include <linux/mfd/rohm-bd73800.h>
 #include <linux/mfd/rohm-generic.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -56,6 +57,12 @@ static const struct resource bd72720_rtc_irqs[] = {
 	DEFINE_RES_IRQ_NAMED(BD72720_INT_RTC2, "bd70528-rtc-alm-2"),
 };
 
+static const struct resource bd73800_rtc_irqs[] = {
+	DEFINE_RES_IRQ_NAMED(BD73800_INT_RTC0, "bd70528-rtc-alm-0"),
+	DEFINE_RES_IRQ_NAMED(BD73800_INT_RTC1, "bd70528-rtc-alm-1"),
+	DEFINE_RES_IRQ_NAMED(BD73800_INT_RTC2, "bd70528-rtc-alm-2"),
+};
+
 static const struct resource bd71815_power_irqs[] = {
 	DEFINE_RES_IRQ_NAMED(BD71815_INT_DCIN_RMV, "bd71815-dcin-rmv"),
 	DEFINE_RES_IRQ_NAMED(BD71815_INT_CLPS_OUT, "bd71815-dcin-clps-out"),
@@ -230,6 +237,17 @@ static const struct mfd_cell bd72720_mfd_cells[] = {
 	/* Power button is registered separately */
 };
 
+static const struct mfd_cell bd73800_mfd_cells[] = {
+	{ .name = "bd73800-pmic", },
+	{ .name = "bd73800-clk", },
+	{ .name = "bd73800-gpio", },
+	{
+		.name = "bd73800-rtc",
+		.num_resources = ARRAY_SIZE(bd73800_rtc_irqs),
+		.resources = bd73800_rtc_irqs,
+	},
+};
+
 static const struct regmap_range bd71815_volatile_ranges[] = {
 	regmap_reg_range(BD71815_REG_SEC, BD71815_REG_YEAR),
 	regmap_reg_range(BD71815_REG_CONF, BD71815_REG_BAT_TEMP),
@@ -339,6 +357,17 @@ static const struct regmap_range bd72720_volatile_ranges_4c[] = {
 	BD72720_UNWRAP_REG_RANGE(BD72720_REG_IMPCHK_CTRL, BD72720_REG_IMPCHK_CTRL),
 };
 
+static const struct regmap_range bd73800_volatile_ranges[] = {
+	regmap_reg_range(BD73800_REG_POR_REASON, BD73800_REG_POW_STATE),
+	regmap_reg_range(BD73800_REG_PS_CTRL_1, BD73800_REG_PS_CTRL_2),
+	regmap_reg_range(BD73800_REG_RCVNUM, BD73800_REG_RCVNUM),
+	regmap_reg_range(BD73800_REG_RTC_SEC, BD73800_REG_RTC_YEAR),
+	regmap_reg_range(BD73800_REG_RTC_CONF, BD73800_REG_RTC_CONF),
+	regmap_reg_range(BD73800_REG_ADC_ACCUM_KICK, BD73800_REG_ADC_TEMP_VAL0),
+	regmap_reg_range(BD73800_REG_INT_MAIN_STAT, BD73800_REG_INT_5_STAT),
+	regmap_reg_range(BD73800_REG_INT_MAIN_SRC, BD73800_REG_INT_5_SRC),
+};
+
 static const struct regmap_access_table bd71815_volatile_regs = {
 	.yes_ranges = &bd71815_volatile_ranges[0],
 	.n_yes_ranges = ARRAY_SIZE(bd71815_volatile_ranges),
@@ -364,6 +393,24 @@ static const struct regmap_access_table bd72720_volatile_regs_4c = {
 	.n_yes_ranges = ARRAY_SIZE(bd72720_volatile_ranges_4c),
 };
 
+static const struct regmap_access_table bd73800_volatile_regs = {
+	.yes_ranges = bd73800_volatile_ranges,
+	.n_yes_ranges = ARRAY_SIZE(bd73800_volatile_ranges),
+};
+
+static const struct regmap_range bd73800_read_only_ranges[] = {
+	regmap_reg_range(BD73800_REG_PRODUCT_ID, BD73800_REG_NVMVERSION),
+	regmap_reg_range(BD73800_REG_POW_STATE, BD73800_REG_POW_STATE),
+	regmap_reg_range(BD73800_REG_ADC_ACCUM_CNT2, BD73800_REG_ADC_TEMP_VAL0),
+	regmap_reg_range(BD73800_REG_INT_MAIN_STAT, BD73800_REG_INT_MAIN_STAT),
+	regmap_reg_range(BD73800_REG_INT_MAIN_SRC, BD73800_REG_INT_5_SRC),
+};
+
+static const struct regmap_access_table bd73800_ro_regs = {
+	.no_ranges = bd73800_read_only_ranges,
+	.n_no_ranges = ARRAY_SIZE(bd73800_read_only_ranges),
+};
+
 static const struct regmap_config bd71815_regmap = {
 	.reg_bits = 8,
 	.val_bits = 8,
@@ -448,6 +495,15 @@ static const struct regmap_config bd72720_regmap_4c = {
 	.cache_type = REGCACHE_MAPLE,
 };
 
+static const struct regmap_config bd73800_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.wr_table = &bd73800_ro_regs,
+	.volatile_table = &bd73800_volatile_regs,
+	.max_register = BD73800_MAX_REGISTER - 1,
+	.cache_type = REGCACHE_MAPLE,
+};
+
 /*
  * Mapping of main IRQ register bits to sub-IRQ register offsets so that we can
  * access corect sub-IRQ registers based on bits that are set in main IRQ
@@ -779,6 +835,60 @@ static int bd72720_set_type_config(unsigned int **buf, unsigned int type,
 	return regmap_irq_set_type_config_simple(buf, type, irq_data, idx, irq_drv_data);
 }
 
+static const struct regmap_irq bd73800_irqs[] = {
+	/* INT_STAT_1 register IRQs, ADC and RTC */
+	REGMAP_IRQ_REG(BD73800_INT_ADC_ACCUM_DONE, 0, BD73800_INT_ADC_ACCUM_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_ADC_ACCUM_OVF, 0, BD73800_INT_ADC_ACCUM_OVF_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_ADC_ACCUM_VAL, 0, BD73800_INT_ADC_ACCUM_VAL_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_ADC_ACCUM_TW, 0, BD73800_INT_ADC_ACCUM_TW_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_ADC_POW_VAL, 0, BD73800_INT_ADC_POW_VAL_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_RTC0, 0, BD73800_INT_RTC0_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_RTC1, 0, BD73800_INT_RTC1_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_RTC2, 0, BD73800_INT_RTC2_MASK),
+
+	/* BUCK reg interrupts */
+	/* INT_STAT_2 IRQs */
+	REGMAP_IRQ_REG(BD73800_INT_BUCK1_DVS_DONE, 1, BD73800_INT_BUCK1_DVS_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK2_DVS_DONE, 1, BD73800_INT_BUCK2_DVS_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK3_DVS_DONE, 1, BD73800_INT_BUCK3_DVS_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK4_DVS_DONE, 1, BD73800_INT_BUCK4_DVS_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK5_DVS_DONE, 1, BD73800_INT_BUCK5_DVS_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK6_DVS_DONE, 1, BD73800_INT_BUCK6_DVS_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK7_DVS_DONE, 1, BD73800_INT_BUCK7_DVS_DONE_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK8_DVS_DONE, 1, BD73800_INT_BUCK8_DVS_DONE_MASK),
+	/* INT_STAT_3 IRQs */
+	REGMAP_IRQ_REG(BD73800_INT_BUCK1_OCP, 2, BD73800_INT_BUCK1_OCP_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK2_OCP, 2, BD73800_INT_BUCK2_OCP_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK3_OCP, 2, BD73800_INT_BUCK3_OCP_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK4_OCP, 2, BD73800_INT_BUCK4_OCP_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK5_OCP, 2, BD73800_INT_BUCK5_OCP_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK6_OCP, 2, BD73800_INT_BUCK6_OCP_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK7_OCP, 2, BD73800_INT_BUCK7_OCP_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_BUCK8_OCP, 2, BD73800_INT_BUCK8_OCP_MASK),
+
+	/* INT_STAT_4 IRQs, power-button, WDG and reset */
+	REGMAP_IRQ_REG(BD73800_INT_PBTN_LONG_PRESS, 3, BD73800_INT_PBTN_LONG_PRESS_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_PBTN_MID_PRESS, 3, BD73800_INT_PBTN_MID_PRESS_MASK),
+	/*
+	 * The SHORT_PUSH is generated when button is first pressed (longer
+	 * than configured time limit), and then released before the MID_PRESS
+	 * time limit. The SHORT_PRESS is generated immediately when button is
+	 * pressed for longer than configured limit, whether it is released or
+	 * not.
+	 */
+	REGMAP_IRQ_REG(BD73800_INT_PBTN_SHORT_PUSH, 3, BD73800_INT_PBTN_SHORT_PUSH_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_PBTN_SHORT_PRESS, 3, BD73800_INT_PBTN_SHORT_PRESS_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_WDG, 3, BD73800_INT_WDG_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_SWRESET, 3, BD73800_INT_SWRESET_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_SEQ_DONE, 3, BD73800_INT_SEQ_DONE_MASK),
+
+	/* INT_STAT_5 IRQs, GPIO */
+	REGMAP_IRQ_REG(BD73800_INT_GPIO1, 4, BD73800_INT_GPIO1_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_GPIO2, 4, BD73800_INT_GPIO2_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_GPIO3, 4, BD73800_INT_GPIO3_MASK),
+	REGMAP_IRQ_REG(BD73800_INT_GPIO4, 4, BD73800_INT_GPIO4_MASK),
+};
+
 static const struct regmap_irq_chip bd71828_irq_chip = {
 	.name = "bd71828_irq",
 	.main_status = BD71828_REG_INT_MAIN,
@@ -833,6 +943,25 @@ static const struct regmap_irq_chip bd72720_irq_chip = {
 	.irq_reg_stride = 1,
 };
 
+static const struct regmap_irq_chip bd73800_irq_chip = {
+	.name = "bd73800_irq",
+	.main_status = BD73800_REG_INT_MAIN_STAT,
+	.irqs = bd73800_irqs,
+	.num_irqs = ARRAY_SIZE(bd73800_irqs),
+	.status_base = BD73800_REG_INT_1_STAT,
+	.unmask_base = BD73800_REG_INT_1_EN,
+	.ack_base = BD73800_REG_INT_1_STAT,
+	.init_ack_masked = true,
+	.num_regs = 5,
+	.num_main_regs = 1,
+	/*
+	 * Ignore mirrored bits [7:5]. They are handled as part of normal INT_4
+	 * handling.
+	 */
+	.num_main_status_bits = 5,
+	.irq_reg_stride = 1,
+};
+
 static int set_clk_mode(struct device *dev, struct regmap *regmap,
 			int clkmode_reg)
 {
@@ -971,6 +1100,17 @@ static int bd71828_i2c_probe(struct i2c_client *i2c)
 		main_lvl_val = BD72720_MASK_LVL1_EN_ALL;
 		break;
 	}
+	case ROHM_CHIP_TYPE_BD73800:
+		mfd = bd73800_mfd_cells;
+		cells = ARRAY_SIZE(bd73800_mfd_cells);
+		regmap_config = &bd73800_regmap;
+		irqchip = &bd73800_irq_chip;
+		clkmode_reg = BD73800_REG_OUT32K;
+		button_irq = BD73800_INT_PBTN_SHORT_PUSH;
+		main_lvl_mask_reg = BD73800_REG_INT_MAIN_EN;
+		main_lvl_val = BD73800_INT_MAIN_EN_ALL;
+		break;
+
 	default:
 		dev_err(&i2c->dev, "Unknown device type");
 		return -EINVAL;
@@ -1051,6 +1191,9 @@ static const struct of_device_id bd71828_of_match[] = {
 	}, {
 		.compatible = "rohm,bd72720",
 		.data = (void *)ROHM_CHIP_TYPE_BD72720,
+	}, {
+		.compatible = "rohm,bd73800",
+		.data = (void *)ROHM_CHIP_TYPE_BD73800,
 	 },
 	{ },
 };
diff --git a/drivers/mfd/rt5120.c b/drivers/mfd/rt5120.c
index a229eb292484b2..c468b1cf730b30 100644
--- a/drivers/mfd/rt5120.c
+++ b/drivers/mfd/rt5120.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/mfd/core.h>
 #include <linux/module.h>
diff --git a/drivers/mfd/ti_am335x_tscadc.c b/drivers/mfd/ti_am335x_tscadc.c
index 068c25401c6cda..81f07a1d780c53 100644
--- a/drivers/mfd/ti_am335x_tscadc.c
+++ b/drivers/mfd/ti_am335x_tscadc.c
@@ -313,7 +313,7 @@ static int __maybe_unused ti_tscadc_can_wakeup(struct device *dev, void *data)
 	return device_may_wakeup(dev);
 }
 
-static int __maybe_unused tscadc_suspend(struct device *dev)
+static int tscadc_suspend(struct device *dev)
 {
 	struct ti_tscadc_dev *tscadc = dev_get_drvdata(dev);
 
@@ -331,7 +331,7 @@ static int __maybe_unused tscadc_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused tscadc_resume(struct device *dev)
+static int tscadc_resume(struct device *dev)
 {
 	struct ti_tscadc_dev *tscadc = dev_get_drvdata(dev);
 
@@ -345,7 +345,7 @@ static int __maybe_unused tscadc_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(tscadc_pm_ops, tscadc_suspend, tscadc_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(tscadc_pm_ops, tscadc_suspend, tscadc_resume);
 
 static const struct ti_tscadc_data tscdata = {
 	.adc_feature_name = "TI-am335x-adc",
@@ -373,7 +373,7 @@ MODULE_DEVICE_TABLE(of, ti_tscadc_dt_ids);
 static struct platform_driver ti_tscadc_driver = {
 	.driver = {
 		.name   = "ti_am3359-tscadc",
-		.pm	= &tscadc_pm_ops,
+		.pm	= pm_sleep_ptr(&tscadc_pm_ops),
 		.of_match_table = ti_tscadc_dt_ids,
 	},
 	.probe	= ti_tscadc_probe,
diff --git a/drivers/mfd/tps65219.c b/drivers/mfd/tps65219.c
index e52fbf1481fefd..f8c08dd86618da 100644
--- a/drivers/mfd/tps65219.c
+++ b/drivers/mfd/tps65219.c
@@ -6,6 +6,7 @@
 // Copyright (C) 2024 Texas Instruments Incorporated - https://www.ti.com/
 
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/reboot.h>
 #include <linux/regmap.h>
 
diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c
index 3cfd2f02b62f3c..600186b1e8cb4e 100644
--- a/drivers/mfd/tps6586x.c
+++ b/drivers/mfd/tps6586x.c
@@ -618,7 +618,7 @@ static void tps6586x_i2c_remove(struct i2c_client *client)
 		free_irq(client->irq, tps6586x);
 }
 
-static int __maybe_unused tps6586x_i2c_suspend(struct device *dev)
+static int tps6586x_i2c_suspend(struct device *dev)
 {
 	struct tps6586x *tps6586x = dev_get_drvdata(dev);
 
@@ -628,7 +628,7 @@ static int __maybe_unused tps6586x_i2c_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused tps6586x_i2c_resume(struct device *dev)
+static int tps6586x_i2c_resume(struct device *dev)
 {
 	struct tps6586x *tps6586x = dev_get_drvdata(dev);
 
@@ -638,7 +638,7 @@ static int __maybe_unused tps6586x_i2c_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(tps6586x_pm_ops, tps6586x_i2c_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(tps6586x_pm_ops, tps6586x_i2c_suspend,
 			 tps6586x_i2c_resume);
 
 static const struct i2c_device_id tps6586x_id_table[] = {
@@ -651,7 +651,7 @@ static struct i2c_driver tps6586x_driver = {
 	.driver	= {
 		.name	= "tps6586x",
 		.of_match_table = of_match_ptr(tps6586x_of_match),
-		.pm	= &tps6586x_pm_ops,
+		.pm	= pm_sleep_ptr(&tps6586x_pm_ops),
 	},
 	.probe		= tps6586x_i2c_probe,
 	.remove		= tps6586x_i2c_remove,
diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c
index c024a28b057e39..f86df356d658d0 100644
--- a/drivers/mfd/twl-core.c
+++ b/drivers/mfd/twl-core.c
@@ -25,7 +25,7 @@
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
-#include <linux/irq.h>
+#include <linux/interrupt.h>
 #include <linux/irqdomain.h>
 
 #include <linux/regulator/machine.h>
@@ -719,6 +719,11 @@ static const struct mfd_cell twl6032_cells[] = {
 	{ .name = "twl6032-clk" },
 };
 
+static void twl_unregister_device(void *data)
+{
+	platform_device_unregister(data);
+}
+
 /* NOTE: This driver only handles a single twl4030/tps659x0 chip */
 static int
 twl_probe(struct i2c_client *client)
@@ -754,17 +759,22 @@ twl_probe(struct i2c_client *client)
 		return status;
 	}
 
+	status = devm_add_action_or_reset(&client->dev, twl_unregister_device,
+					  pdev);
+	if (status)
+		return status;
+
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
 		dev_dbg(&client->dev, "can't talk I2C?\n");
 		status = -EIO;
-		goto free;
+		goto out;
 	}
 
 	twl_priv = devm_kzalloc(&client->dev, sizeof(struct twl_private),
 				GFP_KERNEL);
 	if (!twl_priv) {
 		status = -ENOMEM;
-		goto free;
+		goto out;
 	}
 
 	if ((id->driver_data) & TWL6030_CLASS) {
@@ -784,7 +794,7 @@ twl_probe(struct i2c_client *client)
 					 GFP_KERNEL);
 	if (!twl_priv->twl_modules) {
 		status = -ENOMEM;
-		goto free;
+		goto out;
 	}
 
 	for (i = 0; i < num_slaves; i++) {
@@ -880,7 +890,7 @@ twl_probe(struct i2c_client *client)
 		status = devm_mfd_add_devices(&client->dev, PLATFORM_DEVID_NONE,
 					      cells, num_cells, NULL, 0, NULL);
 		if (status < 0)
-			goto free;
+			goto out;
 
 		if (of_device_is_system_power_controller(node)) {
 			if (!pm_power_off)
@@ -896,14 +906,11 @@ twl_probe(struct i2c_client *client)
 fail:
 	if (status < 0)
 		twl_remove(client);
-free:
-	if (status < 0)
-		platform_device_unregister(pdev);
-
+out:
 	return status;
 }
 
-static int __maybe_unused twl_suspend(struct device *dev)
+static int twl_suspend(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
 
@@ -913,7 +920,7 @@ static int __maybe_unused twl_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused twl_resume(struct device *dev)
+static int twl_resume(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
 
@@ -923,7 +930,7 @@ static int __maybe_unused twl_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(twl_dev_pm_ops, twl_suspend, twl_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(twl_dev_pm_ops, twl_suspend, twl_resume);
 
 static const struct i2c_device_id twl_ids[] = {
 	{ "twl4030", TWL4030_VAUX2 },	/* "Triton 2" */
@@ -942,7 +949,7 @@ static const struct i2c_device_id twl_ids[] = {
 /* One Client Driver , 4 Clients */
 static struct i2c_driver twl_driver = {
 	.driver.name	= DRIVER_NAME,
-	.driver.pm	= &twl_dev_pm_ops,
+	.driver.pm	= pm_sleep_ptr(&twl_dev_pm_ops),
 	.id_table	= twl_ids,
 	.probe		= twl_probe,
 	.remove		= twl_remove,
diff --git a/drivers/mfd/wcd934x.c b/drivers/mfd/wcd934x.c
index 3c3080e8c8cf7e..d57796933d8732 100644
--- a/drivers/mfd/wcd934x.c
+++ b/drivers/mfd/wcd934x.c
@@ -2,6 +2,7 @@
 // Copyright (c) 2019, Linaro Limited
 
 #include <linux/clk.h>
+#include <linux/dma-mapping.h>
 #include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
@@ -217,13 +218,18 @@ static int wcd934x_slim_probe(struct slim_device *sdev)
 	struct gpio_desc *reset_gpio;
 	int ret;
 
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to set DMA mask\n");
+
 	ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
 	if (!ddata)
 		return	-ENOMEM;
 
 	ddata->irq = of_irq_get(np, 0);
 	if (ddata->irq < 0)
-		return dev_err_probe(ddata->dev, ddata->irq,
+		return dev_err_probe(dev, ddata->irq,
 				     "Failed to get IRQ\n");
 
 	ddata->extclk = devm_clk_get(dev, "extclk");
diff --git a/drivers/mfd/wm831x-auxadc.c b/drivers/mfd/wm831x-auxadc.c
index 744e1d161c5a79..6280d4f3898154 100644
--- a/drivers/mfd/wm831x-auxadc.c
+++ b/drivers/mfd/wm831x-auxadc.c
@@ -132,7 +132,7 @@ static irqreturn_t wm831x_auxadc_irq(int irq, void *irq_data)
 			1 << input, 0);
 	wm831x->auxadc_active &= ~(1 << input);
 
-	/* Turn off the entire convertor if idle */
+	/* Turn off the entire converter if idle */
 	if (!wm831x->auxadc_active)
 		wm831x_reg_write(wm831x, WM831X_AUXADC_CONTROL, 0);
 
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
index d4fac2caca8678..af18ff1992eedb 100644
--- a/drivers/misc/fastrpc.c
+++ b/drivers/misc/fastrpc.c
@@ -70,8 +70,6 @@
 #define ADSP_MMAP_HEAP_ADDR  4
 /* MAP static DMA buffer on DSP User PD */
 #define ADSP_MMAP_DMA_BUFFER  6
-/* Add memory to static PD pool protection thru hypervisor */
-#define ADSP_MMAP_REMOTE_HEAP_ADDR  8
 /* Add memory to userPD pool, for user heap */
 #define ADSP_MMAP_ADD_PAGES 0x1000
 /* Add memory to userPD pool, for LLC heap */
@@ -314,10 +312,14 @@ struct fastrpc_channel_ctx {
 	struct kref refcount;
 	/* Flag if dsp attributes are cached */
 	bool valid_attributes;
+	/* Flag if audio PD init mem was allocated */
+	bool audio_init_mem;
+	/* Audio PD reserved remote heap region */
+	phys_addr_t remote_heap_addr;
+	u64 remote_heap_size;
 	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
 	struct fastrpc_device *secure_fdevice;
 	struct fastrpc_device *fdevice;
-	struct fastrpc_buf *remote_heap;
 	struct list_head invoke_interrupted_mmaps;
 	bool secure;
 	bool unsigned_support;
@@ -927,7 +929,7 @@ static int fastrpc_map_attach(struct fastrpc_user *fl, int fd,
 		dev_dbg(sess->dev, "Bad size passed len 0x%llx map size 0x%llx\n",
 				len, map->size);
 		err = -EINVAL;
-		goto map_err;
+		goto get_err;
 	}
 	map->va = sg_virt(map->table->sgl);
 	map->len = len;
@@ -950,7 +952,7 @@ static int fastrpc_map_attach(struct fastrpc_user *fl, int fd,
 			dev_err(sess->dev,
 				"Failed to assign memory with dma_addr %pad size 0x%llx err %d\n",
 				&map->dma_addr, map->len, err);
-			goto map_err;
+			goto get_err;
 		}
 	}
 	spin_lock(&fl->lock);
@@ -1454,15 +1456,24 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
 	struct fastrpc_init_create_static init;
 	struct fastrpc_invoke_args *args;
 	struct fastrpc_phy_page pages[1];
+	struct fastrpc_channel_ctx *cctx = fl->cctx;
 	char *name;
 	int err;
-	bool scm_done = false;
 	struct {
 		int client_id;
 		u32 namelen;
 		u32 pageslen;
 	} inbuf;
 	u32 sc;
+	unsigned long flags;
+	bool sent_heap = false;
+
+	if (!cctx->remote_heap_addr || !cctx->remote_heap_size) {
+		err = -ENOMEM;
+		dev_err(fl->sctx->dev,
+			"remote heap memory region is not added\n");
+		return err;
+	}
 
 	args = kzalloc_objs(*args, FASTRPC_CREATE_STATIC_PROCESS_NARGS);
 	if (!args)
@@ -1486,31 +1497,6 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
 	inbuf.client_id = fl->client_id;
 	inbuf.namelen = init.namelen;
 	inbuf.pageslen = 0;
-	if (!fl->cctx->remote_heap) {
-		err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,
-						&fl->cctx->remote_heap);
-		if (err)
-			goto err_name;
-
-		/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
-		if (fl->cctx->vmcount) {
-			u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
-
-			err = qcom_scm_assign_mem(fl->cctx->remote_heap->dma_addr,
-							(u64)fl->cctx->remote_heap->size,
-							&src_perms,
-							fl->cctx->vmperms, fl->cctx->vmcount);
-			if (err) {
-				dev_err(fl->sctx->dev,
-					"Failed to assign memory with dma_addr %pad size 0x%llx err %d\n",
-					&fl->cctx->remote_heap->dma_addr,
-					fl->cctx->remote_heap->size, err);
-				goto err_map;
-			}
-			scm_done = true;
-			inbuf.pageslen = 1;
-		}
-	}
 
 	fl->pd = USER_PD;
 
@@ -1522,8 +1508,25 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
 	args[1].length = inbuf.namelen;
 	args[1].fd = -1;
 
-	pages[0].addr = fl->cctx->remote_heap->dma_addr;
-	pages[0].size = fl->cctx->remote_heap->size;
+	/*
+	 * Audio PD is a static PD and retains the remote heap
+	 * information across daemon restarts. Therefore only
+	 * the first attach should provide heap information to
+	 * DSP. Subsequent attaches reuse the previously
+	 * initialized memory pool.
+	 */
+	spin_lock_irqsave(&cctx->lock, flags);
+	if (!cctx->audio_init_mem) {
+		pages[0].addr = cctx->remote_heap_addr;
+		pages[0].size = cctx->remote_heap_size;
+		cctx->audio_init_mem = true;
+		inbuf.pageslen = 1;
+		sent_heap = true;
+	} else {
+		pages[0].addr = 0;
+		pages[0].size = 0;
+	}
+	spin_unlock_irqrestore(&cctx->lock, flags);
 
 	args[2].ptr = (u64)(uintptr_t) pages;
 	args[2].length = sizeof(*pages);
@@ -1541,27 +1544,11 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
 
 	return 0;
 err_invoke:
-	if (fl->cctx->vmcount && scm_done) {
-		u64 src_perms = 0;
-		struct qcom_scm_vmperm dst_perms;
-		u32 i;
-
-		for (i = 0; i < fl->cctx->vmcount; i++)
-			src_perms |= BIT(fl->cctx->vmperms[i].vmid);
-
-		dst_perms.vmid = QCOM_SCM_VMID_HLOS;
-		dst_perms.perm = QCOM_SCM_PERM_RWX;
-		err = qcom_scm_assign_mem(fl->cctx->remote_heap->dma_addr,
-						(u64)fl->cctx->remote_heap->size,
-						&src_perms, &dst_perms, 1);
-		if (err)
-			dev_err(fl->sctx->dev, "Failed to assign memory dma_addr %pad size 0x%llx err %d\n",
-				&fl->cctx->remote_heap->dma_addr, fl->cctx->remote_heap->size, err);
+	if (sent_heap) {
+		spin_lock_irqsave(&cctx->lock, flags);
+		cctx->audio_init_mem = false;
+		spin_unlock_irqrestore(&cctx->lock, flags);
 	}
-err_map:
-	fastrpc_buf_free(fl->cctx->remote_heap);
-	fl->cctx->remote_heap = NULL;
-err_name:
 	kfree(name);
 err:
 	kfree(args);
@@ -1783,7 +1770,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp)
 
 	fl->sctx = fastrpc_session_alloc(fl);
 	if (!fl->sctx) {
-		dev_err(&cctx->rpdev->dev, "No session available\n");
+		dev_err_ratelimited(&cctx->rpdev->dev, "No session available\n");
 		mutex_destroy(&fl->mutex);
 		kfree(fl);
 		fastrpc_channel_ctx_put(cctx);
@@ -1937,7 +1924,7 @@ static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
 		kfree(dsp_attributes);
 		return -EOPNOTSUPP;
 	} else if (err) {
-		dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
+		dev_dbg(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
 		kfree(dsp_attributes);
 		return err;
 	}
@@ -2086,7 +2073,7 @@ static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
 	if (copy_from_user(&req, argp, sizeof(req)))
 		return -EFAULT;
 
-	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
+	if (req.flags != ADSP_MMAP_ADD_PAGES) {
 		dev_err(dev, "flag not supported 0x%x\n", req.flags);
 
 		return -EINVAL;
@@ -2097,10 +2084,7 @@ static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
 		return -EINVAL;
 	}
 
-	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
-		err = fastrpc_remote_heap_alloc(fl, dev, req.size, &buf);
-	else
-		err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
+	err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
 
 	if (err) {
 		dev_err(dev, "failed to allocate buffer\n");
@@ -2139,20 +2123,6 @@ static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
 	/* let the client know the address to use */
 	req.vaddrout = rsp_msg.vaddr;
 
-	/* Add memory to static PD pool, protection thru hypervisor */
-	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
-		u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
-
-		err = qcom_scm_assign_mem(buf->dma_addr, (u64)buf->size,
-			&src_perms, fl->cctx->vmperms, fl->cctx->vmcount);
-		if (err) {
-			dev_err(fl->sctx->dev,
-				"Failed to assign memory dma_addr %pad size 0x%llx err %d",
-				&buf->dma_addr, buf->size, err);
-			goto err_assign;
-		}
-	}
-
 	spin_lock(&fl->lock);
 	list_add_tail(&buf->node, &fl->mmaps);
 	spin_unlock(&fl->lock);
@@ -2533,6 +2503,45 @@ static const struct of_device_id fastrpc_poll_supported_machines[] __maybe_unuse
 	{},
 };
 
+static int fastrpc_init_reserved_mem(struct fastrpc_channel_ctx *cctx,
+				     struct device *rdev, u32 domain_id)
+{
+	struct resource res;
+	u64 src_perms;
+	int err;
+
+	/* Only SDSP and ADSP domains use a reserved remote heap region */
+	if (domain_id != SDSP_DOMAIN_ID && domain_id != ADSP_DOMAIN_ID)
+		return 0;
+
+	err = of_reserved_mem_region_to_resource(rdev->of_node, 0, &res);
+	if (err) {
+		/*
+		 * The reserved-memory region is optional at probe time. For
+		 * the Audio PD (ADSP) domain its absence is validated later in
+		 * fastrpc_init_create_static_process(), so keep probing here.
+		 */
+		return 0;
+	}
+
+	/*
+	 * Audio PD (ADSP) is a static PD: cache the region so its lifetime
+	 * is tied to the rpmsg channel instead of being controlled by
+	 * userspace.
+	 */
+	if (domain_id == ADSP_DOMAIN_ID) {
+		cctx->remote_heap_addr = res.start;
+		cctx->remote_heap_size = resource_size(&res);
+	}
+
+	if (!cctx->vmcount)
+		return 0;
+
+	src_perms = BIT(QCOM_SCM_VMID_HLOS);
+	return qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms,
+				  cctx->vmperms, cctx->vmcount);
+}
+
 static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
 {
 	struct device *rdev = &rpdev->dev;
@@ -2558,8 +2567,8 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
 		return -EINVAL;
 	}
 
-	if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0))
-		dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
+	if (devm_of_reserved_mem_device_init(rdev))
+		dev_dbg(rdev, "no reserved DMA memory for FASTRPC\n");
 
 	vmcount = of_property_read_variable_u32_array(rdev->of_node,
 				"qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);
@@ -2580,21 +2589,9 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
 		}
 	}
 
-	if (domain_id == SDSP_DOMAIN_ID) {
-		struct resource res;
-		u64 src_perms;
-
-		err = of_reserved_mem_region_to_resource(rdev->of_node, 0, &res);
-		if (!err) {
-			src_perms = BIT(QCOM_SCM_VMID_HLOS);
-
-			err = qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms,
-				    data->vmperms, data->vmcount);
-			if (err)
-				goto err_free_data;
-		}
-
-	}
+	err = fastrpc_init_reserved_mem(data, rdev, domain_id);
+	if (err)
+		goto err_free_data;
 
 	secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
 	data->secure = secure_dsp;
@@ -2694,8 +2691,25 @@ static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
 	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
 		list_del(&buf->node);
 
-	if (cctx->remote_heap)
-		fastrpc_buf_free(cctx->remote_heap);
+	if (cctx->remote_heap_size && cctx->vmcount) {
+		u64 src_perms = 0;
+		int err, i;
+		struct qcom_scm_vmperm dst_perms;
+
+		for (i = 0; i < cctx->vmcount; i++)
+			src_perms |= BIT(cctx->vmperms[i].vmid);
+
+		dst_perms.vmid = QCOM_SCM_VMID_HLOS;
+		dst_perms.perm = QCOM_SCM_PERM_RWX;
+
+		err = qcom_scm_assign_mem(cctx->remote_heap_addr,
+					  cctx->remote_heap_size, &src_perms,
+					  &dst_perms, 1);
+		if (err)
+			dev_err(&rpdev->dev,
+				"Failed to assign memory back to HLOS: addr %pa size %#llx err %d\n",
+				&cctx->remote_heap_addr, cctx->remote_heap_size, err);
+	}
 
 	of_platform_depopulate(&rpdev->dev);
 
diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c
index ededa32d674405..01bebcb33bd47e 100644
--- a/drivers/misc/lkdtm/core.c
+++ b/drivers/misc/lkdtm/core.c
@@ -236,11 +236,11 @@ static ssize_t lkdtm_debugfs_entry(struct file *f,
 	if (count >= PAGE_SIZE)
 		return -EINVAL;
 
-	buf = (char *)__get_free_page(GFP_KERNEL);
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 	if (copy_from_user(buf, user_buf, count)) {
-		free_page((unsigned long) buf);
+		kfree(buf);
 		return -EFAULT;
 	}
 	/* NULL-terminate and remove enter */
@@ -248,7 +248,7 @@ static ssize_t lkdtm_debugfs_entry(struct file *f,
 	strim(buf);
 
 	crashtype = find_crashtype(buf);
-	free_page((unsigned long)buf);
+	kfree(buf);
 
 	if (!crashtype)
 		return -EINVAL;
@@ -271,7 +271,7 @@ static ssize_t lkdtm_debugfs_read(struct file *f, char __user *user_buf,
 	ssize_t out;
 	char *buf;
 
-	buf = (char *)__get_free_page(GFP_KERNEL);
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (buf == NULL)
 		return -ENOMEM;
 
@@ -290,7 +290,7 @@ static ssize_t lkdtm_debugfs_read(struct file *f, char __user *user_buf,
 
 	out = simple_read_from_buffer(user_buf, count, off,
 				      buf, n);
-	free_page((unsigned long) buf);
+	kfree(buf);
 
 	return out;
 }
@@ -313,11 +313,11 @@ static ssize_t direct_entry(struct file *f, const char __user *user_buf,
 	if (count < 1)
 		return -EINVAL;
 
-	buf = (char *)__get_free_page(GFP_KERNEL);
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 	if (copy_from_user(buf, user_buf, count)) {
-		free_page((unsigned long) buf);
+		kfree(buf);
 		return -EFAULT;
 	}
 	/* NULL-terminate and remove enter */
@@ -325,7 +325,7 @@ static ssize_t direct_entry(struct file *f, const char __user *user_buf,
 	strim(buf);
 
 	crashtype = find_crashtype(buf);
-	free_page((unsigned long) buf);
+	kfree(buf);
 	if (!crashtype)
 		return -EINVAL;
 
diff --git a/drivers/misc/ntsync.c b/drivers/misc/ntsync.c
index 4a805919bb0c59..2857ae37d3c81c 100644
--- a/drivers/misc/ntsync.c
+++ b/drivers/misc/ntsync.c
@@ -724,9 +724,9 @@ static int ntsync_obj_get_fd(struct ntsync_obj *obj)
 {
 	FD_PREPARE(fdf, O_CLOEXEC,
 		   anon_inode_getfile("ntsync", &ntsync_obj_fops, obj, O_RDWR));
-	if (fdf.err)
-		return fdf.err;
-	obj->file = fd_prepare_file(fdf);
+	if (fdf->fd < 0)
+		return fdf->fd;
+	obj->file = fdf->file;
 	return fd_publish(fdf);
 }
 
diff --git a/drivers/misc/vmw_vmci/Kconfig b/drivers/misc/vmw_vmci/Kconfig
index b6d4d7fd686a52..ca03cbfdab1e1f 100644
--- a/drivers/misc/vmw_vmci/Kconfig
+++ b/drivers/misc/vmw_vmci/Kconfig
@@ -5,7 +5,7 @@
 
 config VMWARE_VMCI
 	tristate "VMware VMCI Driver"
-	depends on (X86 || ARM64) && !CPU_BIG_ENDIAN && PCI
+	depends on (X86 || ARM64) && PCI
 	help
 	  This is VMware's Virtual Machine Communication Interface.  It enables
 	  high-speed communication between host and guest in a virtual
diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
index e760d13ee325e7..a02bb2fee8745f 100644
--- a/drivers/mmc/core/block.c
+++ b/drivers/mmc/core/block.c
@@ -86,7 +86,7 @@ MODULE_ALIAS("mmc:block");
 static DEFINE_MUTEX(block_mutex);
 
 /*
- * The defaults come from config options but can be overriden by module
+ * The defaults come from config options but can be overridden by module
  * or bootarg options.
  */
 static int perdev_minors = CONFIG_MMC_BLOCK_MINORS;
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 29e80e5f928e95..920e4b18680f2a 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -13,7 +13,6 @@
 #include <linux/completion.h>
 #include <linux/device.h>
 #include <linux/delay.h>
-#include <linux/pagemap.h>
 #include <linux/err.h>
 #include <linux/leds.h>
 #include <linux/scatterlist.h>
@@ -589,8 +588,8 @@ EXPORT_SYMBOL(mmc_cqe_recovery);
  *	mmc_is_req_done() is used with requests that have
  *	mrq->cap_cmd_during_tfr = true. mmc_is_req_done() must be called after
  *	starting a request and before waiting for it to complete. That is,
- *	either in between calls to mmc_start_req(), or after mmc_wait_for_req()
- *	and before mmc_wait_for_req_done(). If it is called at other times the
+ *	after mmc_start_request() or mmc_wait_for_req() and before
+ *	mmc_wait_for_req_done(). If it is called at other times the
  *	result is not meaningful.
  */
 bool mmc_is_req_done(struct mmc_host *host, struct mmc_request *mrq)
@@ -1388,7 +1387,7 @@ void mmc_power_off(struct mmc_host *host)
 	 * XO-1.5, require a short delay after poweroff before the card
 	 * can be successfully turned on again.
 	 */
-	mmc_delay(1);
+	mmc_delay_us(host->ios.power_off_delay_us);
 }
 
 void mmc_power_cycle(struct mmc_host *host, u32 ocr)
@@ -2004,7 +2003,7 @@ unsigned int mmc_calc_max_discard(struct mmc_card *card)
 
 	/*
 	 * Without erase_group_def set, MMC erase timeout depends on clock
-	 * frequence which can change.  In that case, the best choice is
+	 * frequency which can change.  In that case, the best choice is
 	 * just the preferred erase size.
 	 */
 	if (mmc_card_mmc(card) && !(card->ext_csd.erase_group_def & 1))
diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h
index a028b48be16447..9be2875608cee9 100644
--- a/drivers/mmc/core/core.h
+++ b/drivers/mmc/core/core.h
@@ -73,6 +73,14 @@ static inline void mmc_delay(unsigned int ms)
 		msleep(ms);
 }
 
+static inline void mmc_delay_us(unsigned int us)
+{
+	if (us == 0)
+		return;
+
+	fsleep(us);
+}
+
 void mmc_rescan(struct work_struct *work);
 void mmc_start_host(struct mmc_host *host);
 void __mmc_stop_host(struct mmc_host *host);
@@ -162,7 +170,7 @@ int mmc_cqe_recovery(struct mmc_host *host);
  *	@host: MMC host to prepare command
  *	@mrq: MMC request to prepare for
  *
- *	mmc_pre_req() is called in prior to mmc_start_req() to let
+ *	mmc_pre_req() is called in prior to mmc_start_request() to let
  *	host prepare for the new request. Preparation of a request may be
  *	performed while another request is running on the host.
  */
diff --git a/drivers/mmc/core/crypto.c b/drivers/mmc/core/crypto.c
index fec4fbf16a5b64..a5a90bfc634ee5 100644
--- a/drivers/mmc/core/crypto.c
+++ b/drivers/mmc/core/crypto.c
@@ -15,7 +15,7 @@
 void mmc_crypto_set_initial_state(struct mmc_host *host)
 {
 	/* Reset might clear all keys, so reprogram all the keys. */
-	if (host->caps2 & MMC_CAP2_CRYPTO)
+	if ((host->caps2 & MMC_CAP2_CRYPTO) && !(host->caps2 & MMC_CAP2_CRYPTO_NO_REPROG))
 		blk_crypto_reprogram_all_keys(&host->crypto_profile);
 }
 
diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c
index 828078b3f9622d..d79ae6b5e0d478 100644
--- a/drivers/mmc/core/host.c
+++ b/drivers/mmc/core/host.c
@@ -13,7 +13,6 @@
 #include <linux/err.h>
 #include <linux/idr.h>
 #include <linux/of.h>
-#include <linux/pagemap.h>
 #include <linux/export.h>
 #include <linux/leds.h>
 #include <linux/slab.h>
@@ -421,6 +420,9 @@ int mmc_of_parse(struct mmc_host *host)
 	device_property_read_u32(dev, "post-power-on-delay-ms",
 				 &host->ios.power_delay_ms);
 
+	device_property_read_u32(dev, "power-off-delay-us",
+				 &host->ios.power_off_delay_us);
+
 	return mmc_pwrseq_alloc(host);
 }
 
@@ -502,14 +504,7 @@ static int mmc_first_nonreserved_index(void)
 	return max + 1;
 }
 
-/**
- *	mmc_alloc_host - initialise the per-host structure.
- *	@extra: sizeof private data structure
- *	@dev: pointer to host device model structure
- *
- *	Initialise the per-host structure.
- */
-struct mmc_host *mmc_alloc_host(int extra, struct device *dev)
+static struct mmc_host *mmc_alloc_host(struct device *dev, int extra)
 {
 	int index;
 	struct mmc_host *host;
@@ -574,12 +569,19 @@ struct mmc_host *mmc_alloc_host(int extra, struct device *dev)
 
 	host->fixed_drv_type = -EINVAL;
 	host->ios.power_delay_ms = 10;
+	host->ios.power_off_delay_us = 1000;
 	host->ios.power_mode = MMC_POWER_UNDEFINED;
 
 	return host;
 }
 
-EXPORT_SYMBOL(mmc_alloc_host);
+static void mmc_free_host(struct mmc_host *host)
+{
+	cancel_delayed_work_sync(&host->detect);
+	cancel_work_sync(&host->sdio_irq_work);
+	mmc_pwrseq_free(host);
+	put_device(&host->class_dev);
+}
 
 static void devm_mmc_host_release(struct device *dev, void *res)
 {
@@ -594,7 +596,7 @@ struct mmc_host *devm_mmc_alloc_host(struct device *dev, int extra)
 	if (!dr)
 		return NULL;
 
-	host = mmc_alloc_host(extra, dev);
+	host = mmc_alloc_host(dev, extra);
 	if (!host) {
 		devres_free(dr);
 		return NULL;
@@ -688,19 +690,3 @@ void mmc_remove_host(struct mmc_host *host)
 }
 
 EXPORT_SYMBOL(mmc_remove_host);
-
-/**
- *	mmc_free_host - free the host structure
- *	@host: mmc host
- *
- *	Free the host once all references to it have been dropped.
- */
-void mmc_free_host(struct mmc_host *host)
-{
-	cancel_delayed_work_sync(&host->detect);
-	cancel_work_sync(&host->sdio_irq_work);
-	mmc_pwrseq_free(host);
-	put_device(&host->class_dev);
-}
-
-EXPORT_SYMBOL(mmc_free_host);
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 05444ecf3909f8..a02b04938e1a0e 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -1015,7 +1015,7 @@ static void mmc_set_bus_speed(struct mmc_card *card)
 }
 
 /*
- * Select the bus width amoung 4-bit and 8-bit(SDR).
+ * Select the bus width among 4-bit and 8-bit(SDR).
  * If the bus width is changed successfully, return the selected width value.
  * Zero is returned instead of error value if the wide width is not supported.
  */
diff --git a/drivers/mmc/core/mmc_ops.c b/drivers/mmc/core/mmc_ops.c
index a952cc8265af8f..d59ff82fb89996 100644
--- a/drivers/mmc/core/mmc_ops.c
+++ b/drivers/mmc/core/mmc_ops.c
@@ -673,7 +673,8 @@ int mmc_switch(struct mmc_card *card, u8 set, u8 index, u8 value,
 }
 EXPORT_SYMBOL_GPL(mmc_switch);
 
-int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error)
+int mmc_send_tuning_timeout(struct mmc_host *host, u32 opcode, int *cmd_error,
+			    u32 timeout_ms)
 {
 	struct mmc_request mrq = {};
 	struct mmc_command cmd = {};
@@ -707,12 +708,7 @@ int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error)
 	data.blocks = 1;
 	data.flags = MMC_DATA_READ;
 
-	/*
-	 * According to the tuning specs, Tuning process
-	 * is normally shorter 40 executions of CMD19,
-	 * and timeout value should be shorter than 150 ms
-	 */
-	data.timeout_ns = 150 * NSEC_PER_MSEC;
+	data.timeout_ns = timeout_ms * NSEC_PER_MSEC;
 
 	data.sg = &sg;
 	data.sg_len = 1;
@@ -740,6 +736,18 @@ int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error)
 	kfree(data_buf);
 	return err;
 }
+EXPORT_SYMBOL_GPL(mmc_send_tuning_timeout);
+
+int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error)
+{
+	/*
+	 * The eMMC and SD specs give the tuning process an overall
+	 * budget of 150 ms.  mmc_send_tuning() simply applies that as
+	 * the timeout of every single tuning command.  Drivers which
+	 * want a different timeout can use mmc_send_tuning_timeout().
+	 */
+	return mmc_send_tuning_timeout(host, opcode, cmd_error, 150);
+}
 EXPORT_SYMBOL_GPL(mmc_send_tuning);
 
 int mmc_send_abort_tuning(struct mmc_host *host, u32 opcode)
diff --git a/drivers/mmc/core/sdio_cis.c b/drivers/mmc/core/sdio_cis.c
index afaa6cab1adc32..0b818cb478cee8 100644
--- a/drivers/mmc/core/sdio_cis.c
+++ b/drivers/mmc/core/sdio_cis.c
@@ -57,12 +57,11 @@ static int cistpl_vers_1(struct mmc_card *card, struct sdio_func *func,
 		return -ENOMEM;
 
 	string = (char*)(buffer + nr_strings);
+	memcpy(string, buf, size);
 
 	for (i = 0; i < nr_strings; i++) {
 		buffer[i] = string;
-		strcpy(string, buf);
-		string += strlen(string) + 1;
-		buf += strlen(buf) + 1;
+		string = strchr(string, 0) + 1;
 	}
 
 	if (func) {
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 4f060d3e563653..79bfc425e7c7c5 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -917,37 +917,6 @@ config MMC_JZ4740
 	  If you have a board based on such a SoC and with a SD/MMC slot,
 	  say Y or M here.
 
-config MMC_VUB300
-	tristate "VUB300 USB to SDIO/SD/MMC Host Controller support"
-	depends on USB
-	help
-	  This selects support for Elan Digital Systems' VUB300 chip.
-
-	  The VUB300 is a USB-SDIO Host Controller Interface chip
-	  that enables the host computer to use SDIO/SD/MMC cards
-	  via a USB 2.0 or USB 1.1 host.
-
-	  The VUB300 chip will be found in both physically separate
-	  USB to SDIO/SD/MMC adapters and embedded on some motherboards.
-
-	  The VUB300 chip supports SD and MMC memory cards in addition
-	  to single and multifunction SDIO cards.
-
-	  Some SDIO cards will need a firmware file to be loaded and
-	  sent to VUB300 chip in order to achieve better data throughput.
-	  Download these "Offload Pseudocode" from Elan Digital Systems'
-	  web-site http://www.elandigitalsystems.com/support/downloads.php
-	  and put them in /lib/firmware. Note that without these additional
-	  firmware files the VUB300 chip will still function, but not at
-	  the best obtainable data rate.
-
-	  To compile this mmc host controller driver as a module,
-	  choose M here: the module will be called vub300.
-
-	  If you have a computer with an embedded VUB300 chip
-	  or if you intend connecting a USB adapter based on a
-	  VUB300 chip say Y or M here.
-
 config MMC_USHC
 	tristate "USB SD Host Controller (USHC) support"
 	depends on USB
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index ee412e6b84d6c9..fd043afab57e64 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -60,7 +60,6 @@ obj-$(CONFIG_MMC_DW_ROCKCHIP)	+= dw_mmc-rockchip.o
 obj-$(CONFIG_MMC_DW_STARFIVE)	+= dw_mmc-starfive.o
 obj-$(CONFIG_MMC_SH_MMCIF)	+= sh_mmcif.o
 obj-$(CONFIG_MMC_JZ4740)	+= jz4740_mmc.o
-obj-$(CONFIG_MMC_VUB300)	+= vub300.o
 obj-$(CONFIG_MMC_USHC)		+= ushc.o
 obj-$(CONFIG_MMC_WMT)		+= wmt-sdmmc.o
 obj-$(CONFIG_MMC_MESON_GX)	+= meson-gx-mmc.o
@@ -80,6 +79,7 @@ obj-$(CONFIG_MMC_REALTEK_USB)	+= rtsx_usb_sdmmc.o
 
 obj-$(CONFIG_MMC_SDHCI_PLTFM)		+= sdhci-pltfm.o
 obj-$(CONFIG_MMC_SDHCI_CADENCE)		+= sdhci-cadence.o
+sdhci-cadence-y				+= sdhci-cadence-core.o sdhci-cadence-phy-v6.o
 obj-$(CONFIG_MMC_SDHCI_ESDHC_MCF)       += sdhci-esdhc-mcf.o
 obj-$(CONFIG_MMC_SDHCI_ESDHC_IMX)	+= sdhci-esdhc-imx.o
 obj-$(CONFIG_MMC_SDHCI_DOVE)		+= sdhci-dove.o
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index 8f4df250a77a04..98933be5f1744c 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -1677,10 +1677,7 @@ static void atmci_detect_change(struct timer_list *t)
 
 		spin_lock(&host->lock);
 
-		if (!present)
-			clear_bit(ATMCI_CARD_PRESENT, &slot->flags);
-		else
-			set_bit(ATMCI_CARD_PRESENT, &slot->flags);
+		assign_bit(ATMCI_CARD_PRESENT, &slot->flags, present);
 
 		/* Clean up queue if present */
 		mrq = slot->mrq;
diff --git a/drivers/mmc/host/bcm2835.c b/drivers/mmc/host/bcm2835.c
index c949a3eaae7942..efd1903c3ccd4d 100644
--- a/drivers/mmc/host/bcm2835.c
+++ b/drivers/mmc/host/bcm2835.c
@@ -1095,7 +1095,7 @@ static void bcm2835_set_clock(struct bcm2835_host *host, unsigned int clock)
 	int div;
 
 	/* The SDCDIV register has 11 bits, and holds (div - 2).  But
-	 * in data mode the max is 50MHz wihout a minimum, and only
+	 * in data mode the max is 50MHz without a minimum, and only
 	 * the bottom 3 bits are used. Since the switch over is
 	 * automatic (unless we have marked the card as slow...),
 	 * chosen values have to make sense in both modes.  Ident mode
diff --git a/drivers/mmc/host/cavium-octeon.c b/drivers/mmc/host/cavium-octeon.c
index 1bfc1a4f9fe9c6..671dd1a36a74ad 100644
--- a/drivers/mmc/host/cavium-octeon.c
+++ b/drivers/mmc/host/cavium-octeon.c
@@ -295,9 +295,12 @@ static void octeon_mmc_remove(struct platform_device *pdev)
 	u64 dma_cfg;
 	int i;
 
-	for (i = 0; i < CAVIUM_MAX_MMC; i++)
+	for (i = 0; i < CAVIUM_MAX_MMC; i++) {
 		if (host->slot[i])
 			cvm_mmc_of_slot_remove(host->slot[i]);
+		if (host->slot_pdev[i])
+			of_platform_device_destroy(&host->slot_pdev[i]->dev, NULL);
+	}
 
 	dma_cfg = readq(host->dma_base + MIO_EMM_DMA_CFG(host));
 	dma_cfg &= ~MIO_EMM_DMA_CFG_EN;
diff --git a/drivers/mmc/host/cavium-thunderx.c b/drivers/mmc/host/cavium-thunderx.c
index 84ff6d82ae3c0a..ee19ea52b27d0c 100644
--- a/drivers/mmc/host/cavium-thunderx.c
+++ b/drivers/mmc/host/cavium-thunderx.c
@@ -157,11 +157,8 @@ static int thunder_mmc_probe(struct pci_dev *pdev,
 	for (i = 0; i < CAVIUM_MAX_MMC; i++) {
 		if (host->slot[i])
 			cvm_mmc_of_slot_remove(host->slot[i]);
-		if (host->slot_pdev[i]) {
-			get_device(&host->slot_pdev[i]->dev);
+		if (host->slot_pdev[i])
 			of_platform_device_destroy(&host->slot_pdev[i]->dev, NULL);
-			put_device(&host->slot_pdev[i]->dev);
-		}
 	}
 	clk_disable_unprepare(host->clk);
 	return ret;
@@ -173,9 +170,12 @@ static void thunder_mmc_remove(struct pci_dev *pdev)
 	u64 dma_cfg;
 	int i;
 
-	for (i = 0; i < CAVIUM_MAX_MMC; i++)
+	for (i = 0; i < CAVIUM_MAX_MMC; i++) {
 		if (host->slot[i])
 			cvm_mmc_of_slot_remove(host->slot[i]);
+		if (host->slot_pdev[i])
+			of_platform_device_destroy(&host->slot_pdev[i]->dev, NULL);
+	}
 
 	dma_cfg = readq(host->dma_base + MIO_EMM_DMA_CFG(host));
 	dma_cfg &= ~MIO_EMM_DMA_CFG_EN;
diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
index cdb9fa94b56d06..c503e5f5c8865b 100644
--- a/drivers/mmc/host/davinci_mmc.c
+++ b/drivers/mmc/host/davinci_mmc.c
@@ -1249,6 +1249,10 @@ static int davinci_mmcsd_probe(struct platform_device *pdev)
 	host->use_dma = use_dma;
 	host->mmc_irq = irq;
 	host->sdio_irq = platform_get_irq_optional(pdev, 1);
+	if (host->sdio_irq < 0 && host->sdio_irq != -ENXIO) {
+		ret = host->sdio_irq;
+		goto parse_fail;
+	}
 
 	if (host->use_dma) {
 		ret = davinci_acquire_dma_channels(host);
@@ -1299,7 +1303,7 @@ static int davinci_mmcsd_probe(struct platform_device *pdev)
 	if (ret)
 		goto mmc_add_host_fail;
 
-	if (host->sdio_irq >= 0) {
+	if (host->sdio_irq > 0) {
 		ret = devm_request_irq(&pdev->dev, host->sdio_irq,
 				       mmc_davinci_sdio_irq, 0,
 				       mmc_hostname(mmc), host);
diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c
index 75c82ff20f1770..a07707a99a787b 100644
--- a/drivers/mmc/host/dw_mmc-rockchip.c
+++ b/drivers/mmc/host/dw_mmc-rockchip.c
@@ -327,7 +327,7 @@ static int dw_mci_rk3288_execute_tuning(struct dw_mci *host, u32 opcode)
 						i,
 						priv->num_phases));
 
-		v = !mmc_send_tuning(mmc, opcode, NULL);
+		v = !mmc_send_tuning_timeout(mmc, opcode, NULL, 5);
 
 		if (i == 0)
 			first_v = v;
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index cf5cecc5491d51..ff0b25d287a945 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -44,6 +44,137 @@
 #define DW_MCI_FREQ_MAX	200000000	/* unit: HZ */
 #define DW_MCI_FREQ_MIN	100000		/* unit: HZ */
 
+/* Event sets guarded by the central watchdog, see below. */
+#define DW_MCI_WD_CMD_EVENTS	BIT(EVENT_CMD_COMPLETE)
+#define DW_MCI_WD_DATA_EVENTS	(BIT(EVENT_DATA_ERROR) | \
+				 BIT(EVENT_DATA_COMPLETE))
+/* Extra watch rounds granted while the completion IRQ is merely late. */
+#define DW_MCI_WD_INFLIGHT_GRACE_MS	10
+
+/*
+ * Central watchdog
+ * ================
+ *
+ * A single hrtimer guards the legs of a request (command, data, voltage
+ * switch) against the hardware going silent: whenever a completion event
+ * that was expected within a bounded time does not arrive in time, the
+ * callback synthesizes it so that the request state machine can error out.
+ *
+ * @wd_events holds the pending_events bits still being waited for and
+ * @wd_states the host->state values for which the watch is valid; both
+ * are protected by irq_lock.  Every producer of the guarded events posts
+ * them under irq_lock as well, so a callback holding irq_lock can never
+ * miss an already-delivered event.  A stale callback which raced past
+ * all checks therefore degrades to at most one redundant state-machine
+ * run instead of completing a foreign leg.  Arming replaces whatever
+ * watch was outstanding before: command and data legs are strictly
+ * serial, so consecutive arms on the same path collapse onto the latest
+ * deadline.
+ *
+ * On expiry, before declaring a timeout, the callback double-checks that
+ * the completion interrupt is not already latched in hardware; if it is,
+ * only the interrupt latency is to blame and the watch is granted extra
+ * rounds instead of failing a transfer that is about to complete.
+ */
+static void dw_mci_wd_deliver(struct dw_mci *host, unsigned long events)
+{
+	host->wd_events &= ~events;
+	if (!host->wd_events)
+		hrtimer_try_to_cancel(&host->wd_timer);
+}
+
+/*
+ * Arm a new watch for @events valid while host->state is one of @states,
+ * expiring after @ms milliseconds.  @already lists bits whose delivery
+ * makes the watch pointless before it is even armed: arming is skipped
+ * when any of them has been posted concurrently.  Must be called with
+ * irq_lock held.
+ */
+static void dw_mci_wd_arm(struct dw_mci *host, unsigned long ms,
+			  unsigned long already, unsigned long events,
+			  unsigned long states)
+{
+	lockdep_assert_held(&host->irq_lock);
+
+	if (host->pending_events & already) {
+		/*
+		 * The event was delivered concurrently; no need for a
+		 * watch anymore.
+		 */
+		host->wd_events = 0;
+		hrtimer_try_to_cancel(&host->wd_timer);
+		return;
+	}
+
+	host->wd_events = events;
+	host->wd_states = states;
+	host->wd_deadline_ns = ktime_get_ns() + (u64)ms * NSEC_PER_MSEC;
+	hrtimer_start(&host->wd_timer, ms_to_ktime(ms), HRTIMER_MODE_REL);
+}
+
+/* Called from hardirq context on watchdog expiry. */
+static enum hrtimer_restart dw_mci_watchdog_fn(struct hrtimer *t)
+{
+	struct dw_mci *host = container_of(t, struct dw_mci, wd_timer);
+	unsigned long irqflags;
+	u32 pending;
+
+	spin_lock_irqsave(&host->irq_lock, irqflags);
+
+	/*
+	 * Last line of defense against a false timeout: when the completion
+	 * interrupt is already latched in MINTSTS but its handler has not
+	 * been scheduled yet, the leg is about to finish normally.  Warn
+	 * about the latency and keep watching instead of failing the
+	 * transfer; delivery by the real interrupt will stop the watch.
+	 * The retired per-timer callbacks went passive here and relied on
+	 * the latched interrupt arriving; rearming bounds the damage if
+	 * that assumption is ever broken.
+	 */
+	if (host->wd_events) {
+		pending = mci_readl(host, MINTSTS); /* read-only mask reg */
+		if ((host->wd_events == DW_MCI_WD_CMD_EVENTS &&
+		     (pending & (DW_MCI_CMD_ERROR_FLAGS |
+				 SDMMC_INT_CMD_DONE))) ||
+		    (host->wd_events == DW_MCI_WD_DATA_EVENTS &&
+		     (pending & (SDMMC_INT_DATA_OVER |
+				 DW_MCI_DATA_ERROR_FLAGS)))) {
+			dev_warn(host->dev,
+				 "Interrupt latency, state %d (watched %#lx)\n",
+				 host->state, host->wd_events);
+			hrtimer_forward_now(&host->wd_timer,
+					    ms_to_ktime(DW_MCI_WD_INFLIGHT_GRACE_MS));
+			spin_unlock_irqrestore(&host->irq_lock, irqflags);
+			return HRTIMER_RESTART;
+		}
+	}
+
+	if (host->wd_events == DW_MCI_WD_DATA_EVENTS) {
+		dev_warn(host->dev,
+			 "Data timeout, state %d\n", host->state);
+		host->data_status = SDMMC_INT_DRTO;
+		set_bit(EVENT_DATA_ERROR, &host->pending_events);
+		set_bit(EVENT_DATA_COMPLETE, &host->pending_events);
+	} else if (host->wd_events) {
+		dev_warn(host->dev,
+			 "Command timeout, state %d (watched %#lx)\n",
+			 host->state, host->wd_events);
+		host->cmd_status = SDMMC_INT_RTO;
+		set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
+	}
+
+	if (host->wd_events)
+		queue_work(system_bh_wq, &host->bh_work);
+
+	host->wd_events = 0;
+	host->wd_states = 0;
+	host->wd_deadline_ns = 0;
+
+	spin_unlock_irqrestore(&host->irq_lock, irqflags);
+
+	return HRTIMER_NORESTART;
+}
+
 #define IDMAC_INT_CLR		(SDMMC_IDMAC_INT_AI | SDMMC_IDMAC_INT_NI | \
 				 SDMMC_IDMAC_INT_CES | SDMMC_IDMAC_INT_DU | \
 				 SDMMC_IDMAC_INT_FBE | SDMMC_IDMAC_INT_RI | \
@@ -171,6 +302,10 @@ static void dw_mci_init_debugfs(struct dw_mci *host)
 			   &host->pending_events);
 	debugfs_create_xul("completed_events", 0400, root,
 			   &host->completed_events);
+	debugfs_create_x64("wd_deadline_ns", 0400, root,
+			   &host->wd_deadline_ns);
+	debugfs_create_xul("wd_events", 0400, root, &host->wd_events);
+	debugfs_create_xul("wd_states", 0400, root, &host->wd_states);
 #ifdef CONFIG_FAULT_INJECTION
 	fault_create_debugfs_attr("fail_data_crc", root, &host->fail_data_crc);
 #endif
@@ -366,18 +501,17 @@ static inline void dw_mci_set_cto(struct dw_mci *host)
 	 * extra careful about synchronization here.  Specifically in hardware a
 	 * command timeout is _at most_ 5.1 ms, so that means we expect an
 	 * interrupt (either command done or timeout) to come rather quickly
-	 * after the mci_writel.  ...but just in case we have a long interrupt
-	 * latency let's add a bit of paranoia.
+	 * after the mci_writel.
 	 *
-	 * In general we'll assume that at least an interrupt will be asserted
-	 * in hardware by the time the cto_timer runs.  ...and if it hasn't
-	 * been asserted in hardware by that time then we'll assume it'll never
-	 * come.
+	 * dw_mci_wd_arm() skips arming when the event was already delivered,
+	 * and the watchdog callback rechecks under the same lock before
+	 * declaring a timeout, so both orderings of the race are covered.
 	 */
 	spin_lock_irqsave(&host->irq_lock, irqflags);
-	if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))
-		mod_timer(&host->cto_timer,
-			jiffies + msecs_to_jiffies(cto_ms) + 1);
+	dw_mci_wd_arm(host, cto_ms, DW_MCI_WD_CMD_EVENTS,
+		      DW_MCI_WD_CMD_EVENTS,
+		      BIT(STATE_SENDING_CMD) | BIT(STATE_SENDING_STOP) |
+		      BIT(STATE_SENDING_CMD11));
 	spin_unlock_irqrestore(&host->irq_lock, irqflags);
 }
 
@@ -1265,17 +1399,16 @@ static void dw_mci_start_request(struct dw_mci *host, struct mmc_command *cmd)
 		/*
 		 * Databook says to fail after 2ms w/ no response, but evidence
 		 * shows that sometimes the cmd11 interrupt takes over 130ms.
-		 * We'll set to 500ms, plus an extra jiffy just in case jiffies
-		 * is just about to roll over.
+		 * We'll set to 500ms.
 		 *
 		 * We do this whole thing under spinlock and only if the
 		 * command hasn't already completed (indicating the irq
 		 * already ran so we don't want the timeout).
 		 */
 		spin_lock_irqsave(&host->irq_lock, irqflags);
-		if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))
-			mod_timer(&host->cmd11_timer,
-				jiffies + msecs_to_jiffies(500) + 1);
+		dw_mci_wd_arm(host, 500, DW_MCI_WD_CMD_EVENTS,
+			      DW_MCI_WD_CMD_EVENTS,
+			      BIT(STATE_SENDING_CMD11));
 		spin_unlock_irqrestore(&host->irq_lock, irqflags);
 	}
 
@@ -1870,9 +2003,9 @@ static void dw_mci_set_drto(struct dw_mci *host)
 	drto_ms += 10;
 
 	spin_lock_irqsave(&host->irq_lock, irqflags);
-	if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events))
-		mod_timer(&host->dto_timer,
-			  jiffies + msecs_to_jiffies(drto_ms));
+	dw_mci_wd_arm(host, drto_ms, BIT(EVENT_DATA_COMPLETE),
+		      DW_MCI_WD_DATA_EVENTS,
+		      BIT(STATE_SENDING_DATA) | BIT(STATE_DATA_BUSY));
 	spin_unlock_irqrestore(&host->irq_lock, irqflags);
 }
 
@@ -1881,14 +2014,6 @@ static bool dw_mci_clear_pending_cmd_complete(struct dw_mci *host)
 	if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))
 		return false;
 
-	/*
-	 * Really be certain that the timer has stopped.  This is a bit of
-	 * paranoia and could only really happen if we had really bad
-	 * interrupt latency and the interrupt routine and timeout were
-	 * running concurrently so that the timer_delete() in the interrupt
-	 * handler couldn't run.
-	 */
-	WARN_ON(timer_delete_sync(&host->cto_timer));
 	clear_bit(EVENT_CMD_COMPLETE, &host->pending_events);
 
 	return true;
@@ -1899,8 +2024,6 @@ static bool dw_mci_clear_pending_data_complete(struct dw_mci *host)
 	if (!test_bit(EVENT_DATA_COMPLETE, &host->pending_events))
 		return false;
 
-	/* Extra paranoia just like dw_mci_clear_pending_cmd_complete() */
-	WARN_ON(timer_delete_sync(&host->dto_timer));
 	clear_bit(EVENT_DATA_COMPLETE, &host->pending_events);
 
 	return true;
@@ -2633,7 +2756,7 @@ static void dw_mci_write_data_pio(struct dw_mci *host)
 
 static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)
 {
-	timer_delete(&host->cto_timer);
+	dw_mci_wd_deliver(host, BIT(EVENT_CMD_COMPLETE));
 
 	if (!host->cmd_status)
 		host->cmd_status = status;
@@ -2666,21 +2789,15 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 			mci_writel(host, RINTSTS, SDMMC_INT_VOLT_SWITCH);
 			pending &= ~SDMMC_INT_VOLT_SWITCH;
 
-			/*
-			 * Hold the lock; we know cmd11_timer can't be kicked
-			 * off after the lock is released, so safe to delete.
-			 */
 			spin_lock(&host->irq_lock);
 			dw_mci_cmd_interrupt(host, pending);
 			spin_unlock(&host->irq_lock);
-
-			timer_delete(&host->cmd11_timer);
 		}
 
 		if (pending & DW_MCI_CMD_ERROR_FLAGS) {
 			spin_lock(&host->irq_lock);
 
-			timer_delete(&host->cto_timer);
+			dw_mci_wd_deliver(host, BIT(EVENT_CMD_COMPLETE));
 			mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS);
 			host->cmd_status = pending;
 			smp_wmb(); /* drain writebuffer */
@@ -2693,7 +2810,7 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 			spin_lock(&host->irq_lock);
 
 			if (host->quirks & DW_MMC_QUIRK_EXTENDED_TMOUT)
-				timer_delete(&host->dto_timer);
+				dw_mci_wd_deliver(host, DW_MCI_WD_DATA_EVENTS);
 
 			/* if there is an error report DATA_ERROR */
 			mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS);
@@ -2714,7 +2831,7 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 		if (pending & SDMMC_INT_DATA_OVER) {
 			spin_lock(&host->irq_lock);
 
-			timer_delete(&host->dto_timer);
+			dw_mci_wd_deliver(host, DW_MCI_WD_DATA_EVENTS);
 
 			mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER);
 			if (!host->data_status)
@@ -2994,126 +3111,6 @@ static void dw_mci_init_dma(struct dw_mci *host)
 	host->use_dma = TRANS_MODE_PIO;
 }
 
-static void dw_mci_cmd11_timer(struct timer_list *t)
-{
-	struct dw_mci *host = timer_container_of(host, t, cmd11_timer);
-
-	if (host->state != STATE_SENDING_CMD11) {
-		dev_warn(host->dev, "Unexpected CMD11 timeout\n");
-		return;
-	}
-
-	host->cmd_status = SDMMC_INT_RTO;
-	set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
-	queue_work(system_bh_wq, &host->bh_work);
-}
-
-static void dw_mci_cto_timer(struct timer_list *t)
-{
-	struct dw_mci *host = timer_container_of(host, t, cto_timer);
-	unsigned long irqflags;
-	u32 pending;
-
-	spin_lock_irqsave(&host->irq_lock, irqflags);
-
-	/*
-	 * If somehow we have very bad interrupt latency it's remotely possible
-	 * that the timer could fire while the interrupt is still pending or
-	 * while the interrupt is midway through running.  Let's be paranoid
-	 * and detect those two cases.  Note that this is paranoia is somewhat
-	 * justified because in this function we don't actually cancel the
-	 * pending command in the controller--we just assume it will never come.
-	 */
-	pending = mci_readl(host, MINTSTS); /* read-only mask reg */
-	if (pending & (DW_MCI_CMD_ERROR_FLAGS | SDMMC_INT_CMD_DONE)) {
-		/* The interrupt should fire; no need to act but we can warn */
-		dev_warn(host->dev, "Unexpected interrupt latency\n");
-		goto exit;
-	}
-	if (test_bit(EVENT_CMD_COMPLETE, &host->pending_events)) {
-		/* Presumably interrupt handler couldn't delete the timer */
-		dev_warn(host->dev, "CTO timeout when already completed\n");
-		goto exit;
-	}
-
-	/*
-	 * Continued paranoia to make sure we're in the state we expect.
-	 * This paranoia isn't really justified but it seems good to be safe.
-	 */
-	switch (host->state) {
-	case STATE_SENDING_CMD11:
-	case STATE_SENDING_CMD:
-	case STATE_SENDING_STOP:
-		/*
-		 * If CMD_DONE interrupt does NOT come in sending command
-		 * state, we should notify the driver to terminate current
-		 * transfer and report a command timeout to the core.
-		 */
-		host->cmd_status = SDMMC_INT_RTO;
-		set_bit(EVENT_CMD_COMPLETE, &host->pending_events);
-		queue_work(system_bh_wq, &host->bh_work);
-		break;
-	default:
-		dev_warn(host->dev, "Unexpected command timeout, state %d\n",
-			 host->state);
-		break;
-	}
-
-exit:
-	spin_unlock_irqrestore(&host->irq_lock, irqflags);
-}
-
-static void dw_mci_dto_timer(struct timer_list *t)
-{
-	struct dw_mci *host = timer_container_of(host, t, dto_timer);
-	unsigned long irqflags;
-	u32 pending;
-
-	spin_lock_irqsave(&host->irq_lock, irqflags);
-
-	/*
-	 * The DTO timer is much longer than the CTO timer, so it's even less
-	 * likely that we'll these cases, but it pays to be paranoid.
-	 */
-	pending = mci_readl(host, MINTSTS); /* read-only mask reg */
-	if (pending & SDMMC_INT_DATA_OVER) {
-		/* The interrupt should fire; no need to act but we can warn */
-		dev_warn(host->dev, "Unexpected data interrupt latency\n");
-		goto exit;
-	}
-	if (test_bit(EVENT_DATA_COMPLETE, &host->pending_events)) {
-		/* Presumably interrupt handler couldn't delete the timer */
-		dev_warn(host->dev, "DTO timeout when already completed\n");
-		goto exit;
-	}
-
-	/*
-	 * Continued paranoia to make sure we're in the state we expect.
-	 * This paranoia isn't really justified but it seems good to be safe.
-	 */
-	switch (host->state) {
-	case STATE_SENDING_DATA:
-	case STATE_DATA_BUSY:
-		/*
-		 * If DTO interrupt does NOT come in sending data state,
-		 * we should notify the driver to terminate current transfer
-		 * and report a data timeout to the core.
-		 */
-		host->data_status = SDMMC_INT_DRTO;
-		set_bit(EVENT_DATA_ERROR, &host->pending_events);
-		set_bit(EVENT_DATA_COMPLETE, &host->pending_events);
-		queue_work(system_bh_wq, &host->bh_work);
-		break;
-	default:
-		dev_warn(host->dev, "Unexpected data timeout, state %d\n",
-			 host->state);
-		break;
-	}
-
-exit:
-	spin_unlock_irqrestore(&host->irq_lock, irqflags);
-}
-
 static int dw_mci_parse_dt(struct dw_mci *host)
 {
 	struct device *dev = host->dev;
@@ -3261,9 +3258,8 @@ int dw_mci_probe(struct dw_mci *host)
 		}
 	}
 
-	timer_setup(&host->cmd11_timer, dw_mci_cmd11_timer, 0);
-	timer_setup(&host->cto_timer, dw_mci_cto_timer, 0);
-	timer_setup(&host->dto_timer, dw_mci_dto_timer, 0);
+	hrtimer_setup(&host->wd_timer, dw_mci_watchdog_fn, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL);
 
 	spin_lock_init(&host->lock);
 	spin_lock_init(&host->irq_lock);
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index 38610c89d54a24..5c2a8d0e9fb4a1 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -69,7 +69,7 @@ struct dw_mci_dma_slave {
  * @cmd: The command currently being sent to the card, or NULL.
  * @data: The data currently being transferred, or NULL if no data
  *	transfer is in progress.
- * @stop_abort: The command currently prepared for stoping transfer.
+ * @stop_abort: The command currently prepared for stopping transfer.
  * @prev_blksz: The former transfer blksz record.
  * @timing: Record of current ios timing.
  * @use_dma: Which DMA channel is in use for the current transfer, zero
@@ -122,9 +122,11 @@ struct dw_mci_dma_slave {
  * @irq_flags: The flags to be passed to request_irq.
  * @irq: The irq value to be passed to request_irq.
  * @sdio_irq: SDIO interrupt bit in interrupt registers.
- * @cmd11_timer: Timer for SD3.0 voltage switch over scheme.
- * @cto_timer: Timer for broken command transfer over scheme.
- * @dto_timer: Timer for broken data transfer over scheme.
+ * @wd_timer: Central watchdog guarding request legs against hardware going
+ *	silent; fires when an expected completion event fails to arrive in
+ *	time so that the state machine can synthesize an error.
+ * @wd_events: pending_events bits still awaited by the armed watch.
+ * @wd_states: host->state values for which the armed watch is valid.
  * @mmc: The mmc_host representing this dw_mci.
  * @flags: Random state bits associated with the host.
  * @ctype: Card type for this host.
@@ -214,7 +216,6 @@ struct dw_mci {
 	void			*priv;
 	struct clk		*biu_clk;
 	struct clk		*ciu_clk;
-	struct dw_mci_slot	*slot;
 
 	/* FIFO push and pull */
 	int			fifo_depth;
@@ -235,9 +236,10 @@ struct dw_mci {
 
 	int			sdio_irq;
 
-	struct timer_list       cmd11_timer;
-	struct timer_list       cto_timer;
-	struct timer_list       dto_timer;
+	struct hrtimer		wd_timer;
+	unsigned long		wd_events;
+	unsigned long		wd_states;
+	u64			wd_deadline_ns;
 
 #ifdef CONFIG_FAULT_INJECTION
 	struct fault_attr	fail_data_crc;
diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
index 694bb443d5f3fe..7ef16d7c03cb62 100644
--- a/drivers/mmc/host/meson-gx-mmc.c
+++ b/drivers/mmc/host/meson-gx-mmc.c
@@ -139,6 +139,7 @@ struct meson_mmc_data {
 	unsigned int always_on;
 	unsigned int adjust;
 	unsigned int irq_sdio_sleep;
+	bool has_pipeline_clk;
 };
 
 struct sd_emmc_desc {
@@ -418,7 +419,7 @@ static int meson_mmc_clk_set(struct meson_host *host, unsigned long rate,
  */
 static int meson_mmc_clk_init(struct meson_host *host)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct clk_mux *mux;
 	struct clk_divider *div;
 	char clk_name[32];
@@ -1141,7 +1142,8 @@ static int meson_mmc_probe(struct platform_device *pdev)
 	struct meson_host *host;
 	struct mmc_host *mmc;
 	struct clk *core_clk;
-	int cd_irq, ret;
+	int cd_irq = -ENXIO;
+	int ret;
 
 	mmc = devm_mmc_alloc_host(&pdev->dev, sizeof(struct meson_host));
 	if (!mmc)
@@ -1185,7 +1187,11 @@ static int meson_mmc_probe(struct platform_device *pdev)
 	if (host->irq < 0)
 		return host->irq;
 
-	cd_irq = platform_get_irq_optional(pdev, 1);
+	ret = platform_get_irq_optional(pdev, 1);
+	if (ret < 0 && ret != -ENXIO)
+		return ret;
+	if (ret > 0)
+		cd_irq = ret;
 	mmc_gpio_set_cd_irq(mmc, cd_irq);
 
 	host->pinctrl = devm_pinctrl_get(&pdev->dev);
@@ -1204,6 +1210,15 @@ static int meson_mmc_probe(struct platform_device *pdev)
 	if (IS_ERR(core_clk))
 		return PTR_ERR(core_clk);
 
+	if (host->data->has_pipeline_clk) {
+		struct clk *pipe_clk;
+
+		pipe_clk = devm_clk_get_enabled(&pdev->dev, "pipeline");
+		if (IS_ERR(pipe_clk))
+			return dev_err_probe(&pdev->dev, PTR_ERR(pipe_clk),
+					     "missing pipeline clock\n");
+	}
+
 	ret = meson_mmc_clk_init(host);
 	if (ret)
 		return ret;
@@ -1322,12 +1337,22 @@ static const struct meson_mmc_data meson_axg_data = {
 	.irq_sdio_sleep	= CLK_V3_IRQ_SDIO_SLEEP,
 };
 
+static const struct meson_mmc_data meson_t7_data = {
+	.tx_delay_mask	= CLK_V3_TX_DELAY_MASK,
+	.rx_delay_mask	= CLK_V3_RX_DELAY_MASK,
+	.always_on	= CLK_V3_ALWAYS_ON,
+	.adjust		= SD_EMMC_V3_ADJUST,
+	.irq_sdio_sleep	= CLK_V3_IRQ_SDIO_SLEEP,
+	.has_pipeline_clk = true,
+};
+
 static const struct of_device_id meson_mmc_of_match[] = {
 	{ .compatible = "amlogic,meson-gx-mmc",		.data = &meson_gx_data },
 	{ .compatible = "amlogic,meson-gxbb-mmc", 	.data = &meson_gx_data },
 	{ .compatible = "amlogic,meson-gxl-mmc",	.data = &meson_gx_data },
 	{ .compatible = "amlogic,meson-gxm-mmc",	.data = &meson_gx_data },
 	{ .compatible = "amlogic,meson-axg-mmc",	.data = &meson_axg_data },
+	{ .compatible = "amlogic,t7-mmc",		.data = &meson_t7_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, meson_mmc_of_match);
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index 01ea3adbdf3b61..5b160fdad3f02d 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -3216,6 +3216,7 @@ static void msdc_drv_remove(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, NULL);
 	mmc_remove_host(mmc);
+	cancel_delayed_work_sync(&host->req_timeout);
 	msdc_deinit_hw(host);
 	msdc_gate_clock(host);
 
diff --git a/drivers/mmc/host/mvsdio.c b/drivers/mmc/host/mvsdio.c
index cf705b265e2114..7d86426c600155 100644
--- a/drivers/mmc/host/mvsdio.c
+++ b/drivers/mmc/host/mvsdio.c
@@ -717,7 +717,7 @@ static int mvsd_probe(struct platform_device *pdev)
 	/*
 	 * Some non-DT platforms do not pass a clock, and the clock
 	 * frequency is passed through platform_data. On DT platforms,
-	 * a clock must always be passed, even if there is no gatable
+	 * a clock must always be passed, even if there is no gateable
 	 * clock associated to the SDIO interface (it can simply be a
 	 * fixed rate clock).
 	 */
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
index 0c3967f758c2d2..033d8cd92726bd 100644
--- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
@@ -14,7 +14,6 @@
 #include <linux/mmc/host.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/pagemap.h>
 #include <linux/platform_data/tmio.h>
 #include <linux/platform_device.h>
 #include <linux/scatterlist.h>
diff --git a/drivers/mmc/host/renesas_sdhi_sys_dmac.c b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
index 426308b73b4902..c7df4da9e865bc 100644
--- a/drivers/mmc/host/renesas_sdhi_sys_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
@@ -14,7 +14,6 @@
 #include <linux/mmc/host.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/pagemap.h>
 #include <linux/platform_data/tmio.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index bf37fab233cf0c..d9b33aa7916508 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -980,6 +980,11 @@ static int sd_power_on(struct rtsx_usb_sdmmc *host)
 	}
 	dev_dbg(sdmmc_dev(host), "%s\n", __func__);
 	rtsx_usb_init_cmd(ucr);
+	/* Start SD init at 3.3V, like the old rts5139 driver. */
+	rtsx_usb_add_cmd(ucr, WRITE_REG_CMD, SD_PAD_CTL,
+			 SD_IO_USING_1V8, SD_IO_USING_3V3);
+	rtsx_usb_add_cmd(ucr, WRITE_REG_CMD, LDO_POWER_CFG,
+			 TUNE_SD18_MASK, TUNE_SD18_3V3);
 	rtsx_usb_add_cmd(ucr, WRITE_REG_CMD, CARD_SELECT, 0x07, SD_MOD_SEL);
 	rtsx_usb_add_cmd(ucr, WRITE_REG_CMD, CARD_SHARE_MODE,
 			CARD_SHARE_MASK, CARD_SHARE_SD);
diff --git a/drivers/mmc/host/sdhci-cadence-core.c b/drivers/mmc/host/sdhci-cadence-core.c
new file mode 100644
index 00000000000000..745455743b01ef
--- /dev/null
+++ b/drivers/mmc/host/sdhci-cadence-core.c
@@ -0,0 +1,830 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2016 Socionext Inc.
+ *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
+ * Copyright (C) 2026 Altera Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/dma-mapping.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/mmc.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+#include "sdhci-cadence.h"
+
+/* HRS - Host Register Set (specific to Cadence) */
+/* HRS04 (PHY access) bitfields (SD4HC) */
+#define   SDHCI_CDNS_HRS04_ACK			BIT(26)
+#define   SDHCI_CDNS_HRS04_RD			BIT(25)
+#define   SDHCI_CDNS_HRS04_WR			BIT(24)
+#define   SDHCI_CDNS_HRS04_RDATA		GENMASK(23, 16)
+#define   SDHCI_CDNS_HRS04_WDATA		GENMASK(15, 8)
+#define   SDHCI_CDNS_HRS04_ADDR			GENMASK(5, 0)
+
+#define SDHCI_CDNS_HRS06		0x18		/* eMMC control */
+#define   SDHCI_CDNS_HRS06_TUNE_UP		BIT(15)
+#define   SDHCI_CDNS_HRS06_TUNE			GENMASK(13, 8)
+#define   SDHCI_CDNS_HRS06_MODE			GENMASK(2, 0)
+#define   SDHCI_CDNS_HRS06_MODE_SD		0x0
+#define   SDHCI_CDNS_HRS06_MODE_MMC_SDR		0x2
+#define   SDHCI_CDNS_HRS06_MODE_MMC_DDR		0x3
+#define   SDHCI_CDNS_HRS06_MODE_MMC_HS200	0x4
+#define   SDHCI_CDNS_HRS06_MODE_MMC_HS400	0x5
+#define   SDHCI_CDNS_HRS06_MODE_MMC_HS400ES	0x6
+
+/* Read block gap */
+#define SDHCI_CDNS_HRS37		0x94	/* interface mode select */
+#define   SDHCI_CDNS_HRS37_MODE_DS		0x0
+#define   SDHCI_CDNS_HRS37_MODE_HS		0x1
+#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR12	0x8
+#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR25	0x9
+#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR50	0xa
+#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR104	0xb
+#define   SDHCI_CDNS_HRS37_MODE_UDS_DDR50	0xc
+#define   SDHCI_CDNS_HRS37_MODE_MMC_LEGACY	0x20
+#define   SDHCI_CDNS_HRS37_MODE_MMC_SDR		0x21
+#define   SDHCI_CDNS_HRS37_MODE_MMC_DDR		0x22
+#define   SDHCI_CDNS_HRS37_MODE_MMC_HS200	0x23
+#define   SDHCI_CDNS_HRS37_MODE_MMC_HS400	0x24
+#define   SDHCI_CDNS_HRS37_MODE_MMC_HS400ES	0x25
+#define SDHCI_CDNS_HRS38		0x98	/* Read block gap coefficient */
+#define   SDHCI_CDNS_HRS38_BLKGAP_MAX		0xf
+
+/* SRS - Slot Register Set (SDHCI-compatible) */
+#define SDHCI_CDNS_SRS_BASE		0x200
+
+/* PHY */
+#define SDHCI_CDNS_PHY_DLY_SD_HS	0x00
+#define SDHCI_CDNS_PHY_DLY_SD_DEFAULT	0x01
+#define SDHCI_CDNS_PHY_DLY_UHS_SDR12	0x02
+#define SDHCI_CDNS_PHY_DLY_UHS_SDR25	0x03
+#define SDHCI_CDNS_PHY_DLY_UHS_SDR50	0x04
+#define SDHCI_CDNS_PHY_DLY_UHS_DDR50	0x05
+#define SDHCI_CDNS_PHY_DLY_EMMC_LEGACY	0x06
+#define SDHCI_CDNS_PHY_DLY_EMMC_SDR	0x07
+#define SDHCI_CDNS_PHY_DLY_EMMC_DDR	0x08
+#define SDHCI_CDNS_PHY_DLY_SDCLK	0x0b
+#define SDHCI_CDNS_PHY_DLY_HSMMC	0x0c
+#define SDHCI_CDNS_PHY_DLY_STROBE	0x0d
+
+struct sdhci_cdns4_phy_param {
+	u8 addr;
+	u8 data;
+};
+
+struct sdhci_cdns4_phy {
+	unsigned int nr_phy_params;
+	struct sdhci_cdns4_phy_param phy_params[];
+};
+
+struct sdhci_cdns4_phy_cfg {
+	const char *property;
+	u8 addr;
+};
+
+struct sdhci_cdns_drv_data {
+	int (*init)(struct platform_device *pdev);
+	const struct sdhci_pltfm_data pltfm_data;
+	u64 dma_mask;
+};
+
+static const struct sdhci_cdns4_phy_cfg sdhci_cdns4_phy_cfgs[] = {
+	{ "cdns,phy-input-delay-sd-highspeed", SDHCI_CDNS_PHY_DLY_SD_HS, },
+	{ "cdns,phy-input-delay-legacy", SDHCI_CDNS_PHY_DLY_SD_DEFAULT, },
+	{ "cdns,phy-input-delay-sd-uhs-sdr12", SDHCI_CDNS_PHY_DLY_UHS_SDR12, },
+	{ "cdns,phy-input-delay-sd-uhs-sdr25", SDHCI_CDNS_PHY_DLY_UHS_SDR25, },
+	{ "cdns,phy-input-delay-sd-uhs-sdr50", SDHCI_CDNS_PHY_DLY_UHS_SDR50, },
+	{ "cdns,phy-input-delay-sd-uhs-ddr50", SDHCI_CDNS_PHY_DLY_UHS_DDR50, },
+	{ "cdns,phy-input-delay-mmc-highspeed", SDHCI_CDNS_PHY_DLY_EMMC_SDR, },
+	{ "cdns,phy-input-delay-mmc-ddr", SDHCI_CDNS_PHY_DLY_EMMC_DDR, },
+	{ "cdns,phy-dll-delay-sdclk", SDHCI_CDNS_PHY_DLY_SDCLK, },
+	{ "cdns,phy-dll-delay-sdclk-hsmmc", SDHCI_CDNS_PHY_DLY_HSMMC, },
+	{ "cdns,phy-dll-delay-strobe", SDHCI_CDNS_PHY_DLY_STROBE, },
+};
+
+static inline void cdns_writel(struct sdhci_cdns_priv *priv, u32 val,
+			       void __iomem *reg)
+{
+	writel(val, reg);
+}
+
+static int sdhci_cdns4_write_phy_reg(struct sdhci_cdns_priv *priv, u8 addr, u8 data)
+{
+	void __iomem *reg = priv->hrs_addr + SDHCI_CDNS_HRS04;
+	u32 tmp;
+	int ret;
+
+	ret = readl_poll_timeout(reg, tmp, !(tmp & SDHCI_CDNS_HRS04_ACK),
+				 0, 10);
+	if (ret)
+		return ret;
+
+	tmp = FIELD_PREP(SDHCI_CDNS_HRS04_WDATA, data) |
+	      FIELD_PREP(SDHCI_CDNS_HRS04_ADDR, addr);
+	priv->priv_writel(priv, tmp, reg);
+
+	tmp |= SDHCI_CDNS_HRS04_WR;
+	priv->priv_writel(priv, tmp, reg);
+
+	ret = readl_poll_timeout(reg, tmp, tmp & SDHCI_CDNS_HRS04_ACK, 0, 10);
+	if (ret)
+		return ret;
+
+	tmp &= ~SDHCI_CDNS_HRS04_WR;
+	priv->priv_writel(priv, tmp, reg);
+
+	ret = readl_poll_timeout(reg, tmp, !(tmp & SDHCI_CDNS_HRS04_ACK),
+				 0, 10);
+
+	return ret;
+}
+
+static unsigned int sdhci_cdns4_phy_param_count(struct device_node *np)
+{
+	unsigned int count = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sdhci_cdns4_phy_cfgs); i++)
+		if (of_property_present(np, sdhci_cdns4_phy_cfgs[i].property))
+			count++;
+
+	return count;
+}
+
+static void sdhci_cdns4_phy_param_parse(struct device_node *np, struct sdhci_cdns4_phy *phy)
+{
+	struct sdhci_cdns4_phy_param *p = phy->phy_params;
+	u32 val;
+	int ret, i;
+
+	for (i = 0; i < ARRAY_SIZE(sdhci_cdns4_phy_cfgs); i++) {
+		ret = of_property_read_u32(np, sdhci_cdns4_phy_cfgs[i].property, &val);
+		if (ret)
+			continue;
+
+		p->addr = sdhci_cdns4_phy_cfgs[i].addr;
+		p->data = val;
+		p++;
+	}
+}
+
+static int sdhci_cdns4_phy_init(struct sdhci_cdns_priv *priv)
+{
+	struct sdhci_cdns4_phy *phy = priv->phy;
+	int ret, i;
+
+	for (i = 0; i < phy->nr_phy_params; i++) {
+		ret = sdhci_cdns4_write_phy_reg(priv, phy->phy_params[i].addr,
+						phy->phy_params[i].data);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static unsigned int sdhci_cdns_get_timeout_clock(struct sdhci_host *host)
+{
+	/*
+	 * Cadence's spec says the Timeout Clock Frequency is the same as the
+	 * Base Clock Frequency.
+	 */
+	return host->max_clk;
+}
+
+static int sdhci_cdns_set_dma_mask(struct sdhci_host *host)
+{
+	const struct sdhci_cdns_drv_data *data;
+	struct device *dev = mmc_dev(host->mmc);
+	int ret;
+
+	data = of_device_get_match_data(dev);
+	if (!data || !data->dma_mask)
+		return 0;
+
+	ret = dma_set_mask_and_coherent(dev, data->dma_mask);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to set DMA mask\n");
+
+	return 0;
+}
+
+static void sdhci_cdns_set_emmc_mode(struct sdhci_cdns_priv *priv, u32 mode)
+{
+	u32 tmp;
+
+	/* The speed mode for eMMC is selected by HRS06 register */
+	tmp = readl(priv->hrs_addr + SDHCI_CDNS_HRS06);
+	tmp &= ~SDHCI_CDNS_HRS06_MODE;
+	tmp |= FIELD_PREP(SDHCI_CDNS_HRS06_MODE, mode);
+	priv->priv_writel(priv, tmp, priv->hrs_addr + SDHCI_CDNS_HRS06);
+}
+
+static u32 sdhci_cdns_get_emmc_mode(struct sdhci_cdns_priv *priv)
+{
+	u32 tmp;
+
+	tmp = readl(priv->hrs_addr + SDHCI_CDNS_HRS06);
+	return FIELD_GET(SDHCI_CDNS_HRS06_MODE, tmp);
+}
+
+static int sdhci_cdns_set_tune_val(struct sdhci_host *host, unsigned int val)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	void __iomem *reg = priv->hrs_addr + SDHCI_CDNS_HRS06;
+	u32 tmp;
+	int i, ret;
+
+	if (host->version >= SDHCI_SPEC_420)
+		return sdhci_cdns6_set_tune_val(host, val);
+
+	if (WARN_ON(!FIELD_FIT(SDHCI_CDNS_HRS06_TUNE, val)))
+		return -EINVAL;
+
+	tmp = readl(reg);
+	tmp &= ~SDHCI_CDNS_HRS06_TUNE;
+	tmp |= FIELD_PREP(SDHCI_CDNS_HRS06_TUNE, val);
+
+	/*
+	 * Workaround for IP errata:
+	 * The IP6116 SD/eMMC PHY design has a timing issue on receive data
+	 * path. Send tune request twice.
+	 */
+	for (i = 0; i < 2; i++) {
+		tmp |= SDHCI_CDNS_HRS06_TUNE_UP;
+		priv->priv_writel(priv, tmp, reg);
+
+		ret = readl_poll_timeout(reg, tmp,
+					 !(tmp & SDHCI_CDNS_HRS06_TUNE_UP),
+					 0, 1);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * sdhci_cdns_tune_blkgap() - tune multi-block read gap
+ * @mmc: MMC host
+ *
+ * Tune delay used in multi block read. To do so,
+ * try sending multi-block read command with incremented gap, unless
+ * it succeeds.
+ *
+ * Return: error code
+ */
+static int sdhci_cdns_tune_blkgap(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_cdns_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	void __iomem *hrs37_reg = priv->hrs_addr + SDHCI_CDNS_HRS37;
+	void __iomem *hrs38_reg = priv->hrs_addr + SDHCI_CDNS_HRS38;
+	int ret;
+	u32 gap;
+
+	/* Currently only needed in HS200 mode */
+	if (host->timing != MMC_TIMING_MMC_HS200)
+		return 0;
+
+	writel(SDHCI_CDNS_HRS37_MODE_MMC_HS200, hrs37_reg);
+
+	for (gap = 0; gap <= SDHCI_CDNS_HRS38_BLKGAP_MAX; gap++) {
+		writel(gap, hrs38_reg);
+		ret = mmc_read_tuning(mmc, 512, 32);
+		if (!ret)
+			break;
+	}
+
+	dev_dbg(mmc_dev(mmc), "read block gap tune %s, gap %d\n", ret ? "failed" : "OK", gap);
+	return ret;
+}
+
+/*
+ * In SD mode, software must not use the hardware tuning and instead perform
+ * an almost identical procedure to eMMC.
+ */
+static int sdhci_cdns_execute_tuning(struct sdhci_host *host, u32 opcode)
+{
+	int cur_streak = 0;
+	int max_streak = 0;
+	int end_of_streak = 0;
+	int i;
+	int ret;
+
+	/*
+	 * Do not execute tuning for UHS_SDR50 or UHS_DDR50.
+	 * The delay is set by probe, based on the DT properties.
+	 */
+	if (host->timing != MMC_TIMING_MMC_HS200 &&
+	    host->timing != MMC_TIMING_UHS_SDR104) {
+		dev_dbg(mmc_dev(host->mmc), "Tuning skipped (timing: %d)\n", host->timing);
+		return 0;
+	}
+
+	for (i = 0; i < SDHCI_CDNS_MAX_TUNING_LOOP; i++) {
+		if (sdhci_cdns_set_tune_val(host, i) ||
+		    mmc_send_tuning(host->mmc, opcode, NULL)) { /* bad */
+			cur_streak = 0;
+		} else { /* good */
+			cur_streak++;
+			if (cur_streak > max_streak) {
+				max_streak = cur_streak;
+				end_of_streak = i;
+			}
+		}
+	}
+
+	if (!max_streak) {
+		dev_err(mmc_dev(host->mmc), "no tuning point found\n");
+		return -EIO;
+	}
+
+	ret = sdhci_cdns_set_tune_val(host, end_of_streak - max_streak / 2);
+	if (ret)
+		return ret;
+
+	/* Block gap tuning is only required for SD4HC, not for SD6HC */
+	if (host->version >= SDHCI_SPEC_420)
+		return 0;
+
+	return sdhci_cdns_tune_blkgap(host->mmc);
+}
+
+static void sdhci_cdns_set_uhs_signaling(struct sdhci_host *host,
+					 unsigned int timing)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	u32 mode;
+
+	switch (timing) {
+	case MMC_TIMING_MMC_HS:
+		mode = SDHCI_CDNS_HRS06_MODE_MMC_SDR;
+		break;
+	case MMC_TIMING_MMC_DDR52:
+		mode = SDHCI_CDNS_HRS06_MODE_MMC_DDR;
+		break;
+	case MMC_TIMING_MMC_HS200:
+		mode = SDHCI_CDNS_HRS06_MODE_MMC_HS200;
+		break;
+	case MMC_TIMING_MMC_HS400:
+		if (priv->enhanced_strobe)
+			mode = SDHCI_CDNS_HRS06_MODE_MMC_HS400ES;
+		else
+			mode = SDHCI_CDNS_HRS06_MODE_MMC_HS400;
+		break;
+	default:
+		mode = SDHCI_CDNS_HRS06_MODE_SD;
+		break;
+	}
+
+	sdhci_cdns_set_emmc_mode(priv, mode);
+
+	/* For SD, fall back to the default handler */
+	if (mode == SDHCI_CDNS_HRS06_MODE_SD)
+		sdhci_set_uhs_signaling(host, timing);
+
+	/* For host controller V6, set SDHCI and PHY registers for UHS signaling */
+	if (host->version >= SDHCI_SPEC_420)
+		sdhci_cdns6_set_uhs_signaling(host, timing);
+}
+
+/* Elba control register bits [6:3] are byte-lane enables */
+#define ELBA_BYTE_ENABLE_MASK(x)	((x) << 3)
+
+/*
+ * The Pensando Elba SoC explicitly controls byte-lane enabling on writes
+ * which includes writes to the HRS registers.  The write lock (wrlock)
+ * is used to ensure byte-lane enable, using write control (ctl_addr),
+ * occurs before the data write.
+ */
+static void elba_priv_writel(struct sdhci_cdns_priv *priv, u32 val,
+			     void __iomem *reg)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->wrlock, flags);
+	writel(GENMASK(7, 3), priv->ctl_addr);
+	writel(val, reg);
+	spin_unlock_irqrestore(&priv->wrlock, flags);
+}
+
+static void elba_write_l(struct sdhci_host *host, u32 val, int reg)
+{
+	elba_priv_writel(sdhci_cdns_priv(host), val, host->ioaddr + reg);
+}
+
+static void elba_write_w(struct sdhci_host *host, u16 val, int reg)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	u32 shift = reg & GENMASK(1, 0);
+	unsigned long flags;
+	u32 byte_enables;
+
+	byte_enables = GENMASK(1, 0) << shift;
+	spin_lock_irqsave(&priv->wrlock, flags);
+	writel(ELBA_BYTE_ENABLE_MASK(byte_enables), priv->ctl_addr);
+	writew(val, host->ioaddr + reg);
+	spin_unlock_irqrestore(&priv->wrlock, flags);
+}
+
+static void elba_write_b(struct sdhci_host *host, u8 val, int reg)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	u32 shift = reg & GENMASK(1, 0);
+	unsigned long flags;
+	u32 byte_enables;
+
+	byte_enables = BIT(0) << shift;
+	spin_lock_irqsave(&priv->wrlock, flags);
+	writel(ELBA_BYTE_ENABLE_MASK(byte_enables), priv->ctl_addr);
+	writeb(val, host->ioaddr + reg);
+	spin_unlock_irqrestore(&priv->wrlock, flags);
+}
+
+static const struct sdhci_ops sdhci_elba_ops = {
+	.write_l = elba_write_l,
+	.write_w = elba_write_w,
+	.write_b = elba_write_b,
+	.set_clock = sdhci_set_clock,
+	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_reset,
+	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
+};
+
+static int elba_drv_init(struct platform_device *pdev)
+{
+	struct sdhci_host *host = platform_get_drvdata(pdev);
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	void __iomem *ioaddr;
+
+	host->mmc->caps |= MMC_CAP_1_8V_DDR | MMC_CAP_8_BIT_DATA;
+	spin_lock_init(&priv->wrlock);
+
+	/* Byte-lane control register */
+	ioaddr = devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(ioaddr))
+		return PTR_ERR(ioaddr);
+
+	priv->ctl_addr = ioaddr;
+	priv->priv_writel = elba_priv_writel;
+	writel(ELBA_BYTE_ENABLE_MASK(0xf), priv->ctl_addr);
+
+	return 0;
+}
+
+static const struct reset_control_bulk_data sdhci_cdns6_agilex5_reset_ids[] = {
+	{ .id = "sdhc-reset" },
+	{ .id = "combophy" },
+	{ .id = "sdmmc-ocp" },
+};
+
+static int sdhci_cdns6_agilex5_init(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct reset_control_bulk_data *resets;
+	int ret;
+
+	resets = devm_kmemdup(dev, sdhci_cdns6_agilex5_reset_ids,
+			      sizeof(sdhci_cdns6_agilex5_reset_ids), GFP_KERNEL);
+	if (!resets)
+		return -ENOMEM;
+
+	/*
+	 * Assert SDHCI, SoftPHY (combophy), and SDMMC OCP/AXI resets together
+	 * so their active periods overlap before all domains are released.
+	 * SoftPHY is shared with NAND, but only one of SDMMC
+	 * or NAND is enabled on a given board.
+	 */
+	ret = devm_reset_control_bulk_get_exclusive(dev, ARRAY_SIZE(sdhci_cdns6_agilex5_reset_ids),
+						    resets);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get resets\n");
+
+	ret = reset_control_bulk_assert(ARRAY_SIZE(sdhci_cdns6_agilex5_reset_ids), resets);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to assert resets\n");
+
+	/* Hold resets asserted long enough for all clock domains to capture. */
+	usleep_range(10, 20);
+
+	ret = reset_control_bulk_deassert(ARRAY_SIZE(sdhci_cdns6_agilex5_reset_ids), resets);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to deassert resets\n");
+
+	return 0;
+}
+
+static const struct sdhci_ops sdhci_cdns4_ops = {
+	.set_clock = sdhci_set_clock,
+	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_reset,
+	.platform_execute_tuning = sdhci_cdns_execute_tuning,
+	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
+};
+
+static const struct sdhci_ops sdhci_cdns6_ops = {
+	.set_clock = sdhci_cdns6_set_clock,
+	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_reset,
+	.platform_execute_tuning = sdhci_cdns_execute_tuning,
+	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
+	.hw_reset = sdhci_cdns6_hw_reset,
+};
+
+static const struct sdhci_ops sdhci_cdns6_agilex5_ops = {
+	.set_clock = sdhci_cdns6_set_clock,
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_reset,
+	.platform_execute_tuning = sdhci_cdns_execute_tuning,
+	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
+	.hw_reset = sdhci_cdns6_hw_reset,
+	.set_dma_mask = sdhci_cdns_set_dma_mask,
+};
+
+static const struct sdhci_cdns_drv_data sdhci_cdns_uniphier_drv_data = {
+	.pltfm_data = {
+		.ops = &sdhci_cdns4_ops,
+		.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+	},
+};
+
+static const struct sdhci_cdns_drv_data sdhci_elba_drv_data = {
+	.init = elba_drv_init,
+	.pltfm_data = {
+		.ops = &sdhci_elba_ops,
+	},
+};
+
+static const struct sdhci_cdns_drv_data sdhci_eyeq_drv_data = {
+	.pltfm_data = {
+		.ops = &sdhci_cdns4_ops,
+		.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+	},
+};
+
+static const struct sdhci_cdns_drv_data sdhci_cdns4_drv_data = {
+	.pltfm_data = {
+		.ops = &sdhci_cdns4_ops,
+	},
+};
+
+static const struct sdhci_cdns_drv_data sdhci_cdns6_agilex5_drv_data = {
+	.init = sdhci_cdns6_agilex5_init,
+	.pltfm_data = {
+		.ops = &sdhci_cdns6_agilex5_ops,
+		.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN |
+			  SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12,
+		.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+			   SDHCI_QUIRK2_ACMD23_BROKEN,
+	},
+	.dma_mask = DMA_BIT_MASK(40),
+};
+
+static const struct sdhci_cdns_drv_data sdhci_cdns6_drv_data = {
+	.pltfm_data = {
+		.ops = &sdhci_cdns6_ops,
+	},
+};
+
+static void sdhci_cdns_hs400_enhanced_strobe(struct mmc_host *mmc,
+					     struct mmc_ios *ios)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	u32 mode;
+
+	priv->enhanced_strobe = ios->enhanced_strobe;
+
+	mode = sdhci_cdns_get_emmc_mode(priv);
+
+	if (mode == SDHCI_CDNS_HRS06_MODE_MMC_HS400 && ios->enhanced_strobe)
+		sdhci_cdns_set_emmc_mode(priv,
+					 SDHCI_CDNS_HRS06_MODE_MMC_HS400ES);
+
+	if (mode == SDHCI_CDNS_HRS06_MODE_MMC_HS400ES && !ios->enhanced_strobe)
+		sdhci_cdns_set_emmc_mode(priv,
+					 SDHCI_CDNS_HRS06_MODE_MMC_HS400);
+
+	/*
+	 * Reprogram the SD6HC PHY for the current timing. Enhanced strobe
+	 * affects PHY delays, and this callback may run with either HS400
+	 * (enabling/disabling ES) or Legacy (initial bus state).
+	 */
+	if (host->version >= SDHCI_SPEC_420)
+		sdhci_cdns6_set_uhs_signaling(host, ios->timing);
+}
+
+static void sdhci_cdns_mmc_hw_reset(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+
+	dev_dbg(mmc_dev(host->mmc), "emmc hardware reset\n");
+
+	reset_control_assert(priv->rst_hw);
+	/* For eMMC, minimum is 1us but give it 3us for good measure */
+	udelay(3);
+
+	reset_control_deassert(priv->rst_hw);
+	/* For eMMC, minimum is 200us but give it 300us for good measure */
+	usleep_range(300, 1000);
+}
+
+static int sdhci_cdns4_phy_probe(struct platform_device *pdev, struct sdhci_cdns_priv *priv)
+{
+	struct device *dev = &pdev->dev;
+	struct sdhci_cdns4_phy *phy;
+	unsigned int nr_phy_params;
+
+	nr_phy_params = sdhci_cdns4_phy_param_count(dev->of_node);
+	phy = devm_kzalloc(dev, struct_size(phy, phy_params, nr_phy_params), GFP_KERNEL);
+	if (!phy)
+		return -ENOMEM;
+
+	phy->nr_phy_params = nr_phy_params;
+	sdhci_cdns4_phy_param_parse(dev->of_node, phy);
+	priv->phy = phy;
+
+	return sdhci_cdns4_phy_init(priv);
+}
+
+static int sdhci_cdns_probe(struct platform_device *pdev)
+{
+	struct sdhci_host *host;
+	const struct sdhci_cdns_drv_data *data;
+	struct sdhci_pltfm_host *pltfm_host;
+	struct sdhci_cdns_priv *priv;
+	struct clk *clk;
+	struct clk *biu_clk;
+	int ret;
+	struct device *dev = &pdev->dev;
+	static const u16 version = SDHCI_SPEC_400 << SDHCI_SPEC_VER_SHIFT;
+
+	if (of_device_is_compatible(dev->of_node, "cdns,sd6hc")) {
+		clk = devm_clk_get_enabled(dev, "ciu");
+		if (IS_ERR(clk))
+			return dev_err_probe(dev, PTR_ERR(clk), "failed to enable ciu clock\n");
+
+		biu_clk = devm_clk_get_enabled(dev, "biu");
+		if (IS_ERR(biu_clk))
+			return dev_err_probe(dev, PTR_ERR(biu_clk), "failed to enable biu clock\n");
+	} else {
+		clk = devm_clk_get_enabled(dev, NULL);
+		if (IS_ERR(clk))
+			return PTR_ERR(clk);
+	}
+
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return dev_err_probe(dev, -EINVAL, "missing platform driver data\n");
+
+	host = sdhci_pltfm_init(pdev, &data->pltfm_data, sizeof(*priv));
+	if (IS_ERR(host))
+		return PTR_ERR(host);
+
+	pltfm_host = sdhci_priv(host);
+	pltfm_host->clk = clk;
+
+	priv = sdhci_pltfm_priv(pltfm_host);
+	priv->hrs_addr = host->ioaddr;
+	priv->enhanced_strobe = false;
+	priv->priv_writel = cdns_writel;
+	host->ioaddr += SDHCI_CDNS_SRS_BASE;
+	host->mmc_host_ops.hs400_enhanced_strobe =
+				sdhci_cdns_hs400_enhanced_strobe;
+	if (data->init) {
+		ret = data->init(pdev);
+		if (ret)
+			return ret;
+	}
+	sdhci_enable_v4_mode(host);
+	sdhci_get_of_property(pdev);
+
+	ret = mmc_of_parse(host->mmc);
+	if (ret)
+		return ret;
+
+	/*
+	 * For SD4HC, read capabilities with fixed version override and set up
+	 * the optional eMMC card RST_n reset control.
+	 * For SD6HC, sdhci_add_host() will automatically read capabilities
+	 * and version from the host controller registers.
+	 */
+	if (of_device_is_compatible(dev->of_node, "cdns,sd4hc")) {
+		__sdhci_read_caps(host, &version, NULL, NULL);
+		ret = sdhci_cdns4_phy_probe(pdev, priv);
+		if (ret)
+			return ret;
+
+		if (host->mmc->caps & MMC_CAP_HW_RESET) {
+			priv->rst_hw = devm_reset_control_get_optional_exclusive(dev, NULL);
+			if (IS_ERR(priv->rst_hw))
+				return dev_err_probe(mmc_dev(host->mmc), PTR_ERR(priv->rst_hw),
+						    "reset controller error\n");
+			if (priv->rst_hw)
+				host->mmc_host_ops.card_hw_reset = sdhci_cdns_mmc_hw_reset;
+		}
+	} else {
+		ret = sdhci_cdns6_phy_probe(pdev, priv);
+		if (ret)
+			return ret;
+	}
+
+	return sdhci_add_host(host);
+}
+
+/*
+ * Only the CIU clock is gated on suspend. The SD6HC "biu" clock is not
+ * toggled here as it may be a shared bus clock; a dedicated biu clock
+ * would need explicit PM gating added here.
+ */
+static int sdhci_cdns_resume(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_cdns_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	int ret;
+
+	ret = clk_prepare_enable(pltfm_host->clk);
+	if (ret)
+		return ret;
+
+	if (host->version >= SDHCI_SPEC_420)
+		ret = sdhci_cdns6_phy_init(priv);
+	else
+		ret = sdhci_cdns4_phy_init(priv);
+
+	if (ret)
+		goto disable_clk;
+
+	ret = sdhci_resume_host(host);
+	if (ret)
+		goto disable_clk;
+
+	return 0;
+
+disable_clk:
+	clk_disable_unprepare(pltfm_host->clk);
+
+	return ret;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(sdhci_cdns_pm_ops, sdhci_pltfm_suspend, sdhci_cdns_resume);
+
+static const struct of_device_id sdhci_cdns_match[] = {
+	{
+		.compatible = "socionext,uniphier-sd4hc",
+		.data = &sdhci_cdns_uniphier_drv_data,
+	},
+	{
+		.compatible = "amd,pensando-elba-sd4hc",
+		.data = &sdhci_elba_drv_data,
+	},
+	{
+		.compatible = "mobileye,eyeq-sd4hc",
+		.data = &sdhci_eyeq_drv_data,
+	},
+	{
+		.compatible = "cdns,sd4hc",
+		.data = &sdhci_cdns4_drv_data,
+	},
+	{
+		.compatible = "altr,agilex5-sd6hc",
+		.data = &sdhci_cdns6_agilex5_drv_data,
+	},
+	{
+		.compatible = "cdns,sd6hc",
+		.data = &sdhci_cdns6_drv_data,
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sdhci_cdns_match);
+
+static struct platform_driver sdhci_cdns_driver = {
+	.driver = {
+		.name = "sdhci-cdns",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.pm = pm_sleep_ptr(&sdhci_cdns_pm_ops),
+		.of_match_table = sdhci_cdns_match,
+	},
+	.probe = sdhci_cdns_probe,
+	.remove = sdhci_pltfm_remove,
+};
+module_platform_driver(sdhci_cdns_driver);
+
+MODULE_AUTHOR("Masahiro Yamada <yamada.masahiro@socionext.com>");
+MODULE_DESCRIPTION("Cadence SD/SDIO/eMMC Host Controller Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mmc/host/sdhci-cadence-phy-v6.c b/drivers/mmc/host/sdhci-cadence-phy-v6.c
new file mode 100644
index 00000000000000..22d56bb46d75a2
--- /dev/null
+++ b/drivers/mmc/host/sdhci-cadence-phy-v6.c
@@ -0,0 +1,975 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * PHY and host controller support for Cadence SD6HC SDHCI
+ *
+ * This file provides support for Cadence's sixth-generation SDHCI controller (SD6HC).
+ * Implements PHY initialization, DLL management, per-speed-mode timing calculations,
+ * and host controller register programming for the SD6HC integrated combo-PHY.
+ *
+ * Copyright (C) 2026 Altera Corporation
+ *   Author: Tanmay Kathpalia <tanmay.kathpalia@altera.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/iopoll.h>
+#include <linux/of.h>
+
+#include "sdhci-cadence.h"
+
+/* IO Delay Information */
+#define SDHCI_CDNS_HRS07		0x1c
+#define   SDHCI_CDNS_HRS07_RW_COMPENSATE	GENMASK(20, 16)
+#define   SDHCI_CDNS_HRS07_IDELAY_VAL		GENMASK(4, 0)
+
+/* PHY Control and Status */
+#define SDHCI_CDNS_HRS09		0x24
+#define   SDHCI_CDNS_HRS09_RDDATA_EN		BIT(16)
+#define   SDHCI_CDNS_HRS09_RDCMD_EN		BIT(15)
+#define   SDHCI_CDNS_HRS09_EXTENDED_WR_MODE	BIT(3)
+#define   SDHCI_CDNS_HRS09_EXTENDED_RD_MODE	BIT(2)
+#define   SDHCI_CDNS_HRS09_PHY_INIT_COMPLETE	BIT(1)
+#define   SDHCI_CDNS_HRS09_PHY_SW_RESET		BIT(0)
+
+/* SDCLK start point adjustment */
+#define SDHCI_CDNS_HRS10		0x28
+#define   SDHCI_CDNS_HRS10_HCSDCLKADJ		GENMASK(19, 16)
+
+/* eMMC Control */
+#define SDHCI_CDNS_HRS11		0x2c
+#define   SDHCI_CDNS_HRS11_EMMC_RST		BIT(0) /* eMMC reset */
+
+/* CMD/DAT output delay */
+#define SDHCI_CDNS_HRS16		0x40
+#define   SDHCI_CDNS_HRS16_WRDATA1_SDCLK_DLY	GENMASK(31, 28)
+#define   SDHCI_CDNS_HRS16_WRDATA0_SDCLK_DLY	GENMASK(27, 24)
+#define   SDHCI_CDNS_HRS16_WRCMD1_SDCLK_DLY	GENMASK(23, 20)
+#define   SDHCI_CDNS_HRS16_WRCMD0_SDCLK_DLY	GENMASK(19, 16)
+#define   SDHCI_CDNS_HRS16_WRDATA1_DLY		GENMASK(15, 12)
+#define   SDHCI_CDNS_HRS16_WRDATA0_DLY		GENMASK(11, 8)
+#define   SDHCI_CDNS_HRS16_WRCMD1_DLY		GENMASK(7, 4)
+#define   SDHCI_CDNS_HRS16_WRCMD0_DLY		GENMASK(3, 0)
+
+/* PHY Special Function Registers */
+/* DQ timing */
+#define SDHCI_CDNS6_PHY_DQ_TIMING_REG			0x2000
+#define   SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_ALWAYS_ON		BIT(31)
+#define   SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_END			GENMASK(29, 27)
+#define   SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_START		GENMASK(26, 24)
+#define   SDHCI_CDNS6_PHY_DQ_TIMING_DATA_SELECT_OE_END		GENMASK(2, 0)
+
+/* DQS timing */
+#define SDHCI_CDNS6_PHY_DQS_TIMING_REG			0x2004
+#define   SDHCI_CDNS6_PHY_DQS_TIMING_USE_EXT_LPBK_DQS		BIT(22)
+#define   SDHCI_CDNS6_PHY_DQS_TIMING_USE_LPBK_DQS		BIT(21)
+#define   SDHCI_CDNS6_PHY_DQS_TIMING_USE_PHONY_DQS		BIT(20)
+#define   SDHCI_CDNS6_PHY_DQS_TIMING_USE_PHONY_DQS_CMD		BIT(19)
+
+/* Gate and loopback control */
+#define SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_REG		0x2008
+#define   SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_SYNC_METHOD		BIT(31)
+#define   SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_RD_DEL_SEL		GENMASK(24, 19)
+#define   SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_UNDERRUN_SUPPRESS	BIT(18)
+#define   SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_GATE_CFG_ALWAYS_ON	BIT(6)
+
+/* Master DLL logic */
+#define SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_REG		0x200c
+#define   SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_BYPASS_MODE		BIT(23)
+#define   SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_PHASE_DETECT_SEL	GENMASK(22, 20)
+#define   SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_DLL_LOCK_NUM		GENMASK(18, 16)
+#define   SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_DLL_START_POINT	GENMASK(7, 0)
+
+/* Slave DLL logic */
+#define SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_REG		0x2010
+#define   SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_CMD_DELAY	GENMASK(31, 24)
+#define   SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_CLK_WRDQS_DELAY	GENMASK(23, 16)
+#define   SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_CLK_WR_DELAY		GENMASK(15, 8)
+#define   SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_DELAY		GENMASK(7, 0)
+
+/* Global control settings */
+#define SDHCI_CDNS6_PHY_CTRL_REG			0x2080
+#define   SDHCI_CDNS6_PHY_CTRL_PHONY_DQS_TIMING			GENMASK(9, 4)
+
+/* Default PHY settings */
+#define SDHCI_CDNS6_PHY_DEFAULT_IOCELL_DELAY		2500
+#define SDHCI_CDNS6_PHY_DEFAULT_DELAY_ELEMENT		24
+#define SDHCI_CDNS6_PHY_DEFAULT_RD_DEL_SEL		52
+#define SDHCI_CDNS6_PHY_DEFAULT_DLL_START		4
+#define SDHCI_CDNS6_PHY_DEFAULT_PHASE_DETECT_SEL	2
+#define SDHCI_CDNS6_PHY_DEFAULT_DLL_LOCK_NUM		0
+#define SDHCI_CDNS6_PHY_DEFAULT_DATA_SELECT_OE_END	1
+
+/* Scale tuning tap (0..39) to 8-bit PHY DLL delay field (0..255) */
+#define SDHCI_CDNS6_PHY_DLL_FIELD_SIZE		256
+
+struct sdhci_cdns6_phy {
+	/*
+	 * Mode-specific timing constraints (in picoseconds)
+	 * These define valid output windows per SD/eMMC spec
+	 */
+	u32 t_cmd_output_min;
+	u32 t_cmd_output_max;
+	u32 t_dat_output_min;
+	u32 t_dat_output_max;
+
+	/*
+	 * PHY delay configuration (in picoseconds)
+	 * Derived from clock period and board-level IO cell delays
+	 */
+	u32 phy_sdclk_delay;
+	u32 phy_cmd_o_delay;
+	u32 phy_dat_o_delay;
+	u32 iocell_input_delay;
+	u32 iocell_output_delay;
+	/* Configured delay element (ps); preserved across clock changes */
+	u32 delay_element_org;
+	/* Active delay element (ps); doubled when one SDMCLK requires > 256 steps */
+	u32 delay_element;
+
+	/* PHY_DLL_SLAVE_CTRL register fields */
+	u8 cp_read_dqs_cmd_delay;	/* bits [31:24] */
+	u8 cp_clk_wrdqs_delay;		/* bits [23:16] */
+	u8 cp_clk_wr_delay;		/* bits [15:8] */
+	u8 cp_read_dqs_delay;		/* bits [7:0] */
+
+	/* PHY_DLL_MASTER_CTRL register fields */
+	bool cp_dll_bypass_mode;	/* bit [23] */
+
+	/* PHY_DQ_TIMING register fields */
+	u8 cp_io_mask_end;		/* bits [29:27] */
+	u8 cp_io_mask_start;		/* bits [26:24] */
+
+	/* PHY_DQS_TIMING register fields */
+	bool cp_use_phony_dqs;		/* bit [20] */
+	bool cp_use_phony_dqs_cmd;	/* bit [19] */
+
+	/* HRS07 register - IO delay Information */
+	u8 sdhc_rw_compensate;		/* bits [20:16] */
+	u8 sdhc_idelay_val;		/* bits [4:0] */
+
+	/* HRS09 register - PHY control and Status */
+	bool sdhc_extended_wr_mode;	/* bit [3] */
+	bool sdhc_extended_rd_mode;	/* bit [2] */
+
+	/* HRS10 register - SDCLK start point adjustment */
+	u8 sdhc_hcsdclkadj;		/* bits [19:16] */
+
+	/* HRS16 register fields - CMD/DAT output delay control */
+	u8 sdhc_wrdata1_sdclk_dly;	/* bits [31:28] */
+	u8 sdhc_wrdata0_sdclk_dly;	/* bits [27:24] */
+	u8 sdhc_wrcmd1_sdclk_dly;	/* bits [23:20] */
+	u8 sdhc_wrcmd0_sdclk_dly;	/* bits [19:16] */
+	u8 sdhc_wrdata1_dly;		/* bits [15:12] */
+	u8 sdhc_wrdata0_dly;		/* bits [11:8] */
+	u8 sdhc_wrcmd1_dly;		/* bits [7:4] */
+	u8 sdhc_wrcmd0_dly;		/* bits [3:0] */
+
+	/* DLL calculation intermediate values, used during PHY timing calculations */
+	u32 t_sdmclk_calc;	/* DLL-quantized SDMCLK period */
+	u32 dll_max_value;	/* DLL delay field ceiling */
+
+	/* Tuning value for HS200/HS400 modes */
+	u8 hs200_tune_val;
+
+	/* Clock periods (in picoseconds) */
+	u32 t_sdmclk;		/* Master clock period */
+	u32 t_sdclk;		/* SD card clock period */
+
+	/* Current operating state */
+	bool strobe_cmd;	/* Enhanced strobe for CMD line */
+	unsigned int mode;	/* Current MMC_TIMING_* mode */
+};
+
+/**
+ * init_ds() - Initialize PHY timing for Default Speed mode (25 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_ds(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 5000;
+	phy->t_cmd_output_max = t_sdclk - 5000;
+	phy->t_dat_output_min = 5000;
+	phy->t_dat_output_max = t_sdclk - 5000;
+}
+
+/**
+ * init_hs() - Initialize PHY timing for High Speed mode (50 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_hs(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 2000;
+	phy->t_cmd_output_max = t_sdclk - 6000;
+	phy->t_dat_output_min = 2000;
+	phy->t_dat_output_max = t_sdclk - 6000;
+}
+
+/**
+ * init_uhs_sdr12() - Initialize PHY timing for UHS SDR12 mode (25 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_uhs_sdr12(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 800;
+	phy->t_cmd_output_max = t_sdclk - 3000;
+	phy->t_dat_output_min = 800;
+	phy->t_dat_output_max = t_sdclk - 3000;
+}
+
+/**
+ * init_uhs_sdr25() - Initialize PHY timing for UHS SDR25 mode (50 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_uhs_sdr25(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 800;
+	phy->t_cmd_output_max = t_sdclk - 3000;
+	phy->t_dat_output_min = 800;
+	phy->t_dat_output_max = t_sdclk - 3000;
+}
+
+/**
+ * init_uhs_sdr50() - Initialize PHY timing for UHS SDR50 mode (100 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_uhs_sdr50(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 800;
+	phy->t_cmd_output_max = t_sdclk - 3000;
+	phy->t_dat_output_min = 800;
+	phy->t_dat_output_max = t_sdclk - 3000;
+}
+
+/**
+ * init_uhs_sdr104() - Initialize PHY timing for UHS SDR104 mode (200 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_uhs_sdr104(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 800;
+	phy->t_cmd_output_max = t_sdclk - 1400;
+	phy->t_dat_output_min = 800;
+	phy->t_dat_output_max = t_sdclk - 1400;
+}
+
+/**
+ * init_uhs_ddr50() - Initialize PHY timing for UHS DDR50 mode (50 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_uhs_ddr50(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 800;
+	phy->t_cmd_output_max = t_sdclk - 3000;
+	phy->t_dat_output_min = 800;
+	phy->t_dat_output_max = t_sdclk - 3000;
+}
+
+/**
+ * init_emmc_sdr() - Initialize PHY timing for eMMC legacy/SDR mode.
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_emmc_sdr(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 3000;
+	phy->t_cmd_output_max = t_sdclk - 3000;
+	phy->t_dat_output_min = 3000;
+	phy->t_dat_output_max = t_sdclk - 3000;
+}
+
+/**
+ * init_emmc_ddr() - Initialize PHY timing for eMMC DDR52 mode.
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_emmc_ddr(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 3000;
+	phy->t_cmd_output_max = t_sdclk - 3000;
+	phy->t_dat_output_min = 2500;
+	phy->t_dat_output_max = t_sdclk - 2500;
+}
+
+/**
+ * init_emmc_hs200() - Initialize PHY timing for eMMC HS200 mode (200 MHz).
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_emmc_hs200(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 800;
+	phy->t_cmd_output_max = t_sdclk - 1400;
+	phy->t_dat_output_min = 800;
+	phy->t_dat_output_max = t_sdclk - 1400;
+}
+
+/**
+ * init_emmc_hs400() - Initialize PHY timing for eMMC HS400/HS400ES mode.
+ * @phy: Pointer to SD6HC PHY state.
+ * @t_sdclk: SD clock period in picoseconds.
+ */
+static void init_emmc_hs400(struct sdhci_cdns6_phy *phy, u32 t_sdclk)
+{
+	phy->t_cmd_output_min = 800;
+	phy->t_cmd_output_max = t_sdclk - 1400;
+	phy->t_dat_output_min = 400;
+	phy->t_dat_output_max = t_sdclk - 400;
+}
+
+/*
+ * init_timings - PHY timing initializers indexed by MMC_TIMING_* value.
+ *
+ * Each entry corresponds to a MMC_TIMING_* constant and sets the appropriate cmd/dat
+ * output timing windows in the PHY state struct.
+ */
+static void (* const init_timings[])(struct sdhci_cdns6_phy *, u32) = {
+	[MMC_TIMING_LEGACY]     = init_ds,
+	[MMC_TIMING_MMC_HS]     = init_emmc_sdr,
+	[MMC_TIMING_SD_HS]      = init_hs,
+	[MMC_TIMING_UHS_SDR12]  = init_uhs_sdr12,
+	[MMC_TIMING_UHS_SDR25]  = init_uhs_sdr25,
+	[MMC_TIMING_UHS_SDR50]  = init_uhs_sdr50,
+	[MMC_TIMING_UHS_SDR104] = init_uhs_sdr104,
+	[MMC_TIMING_UHS_DDR50]  = init_uhs_ddr50,
+	[MMC_TIMING_MMC_DDR52]  = init_emmc_ddr,
+	[MMC_TIMING_MMC_HS200]  = init_emmc_hs200,
+	[MMC_TIMING_MMC_HS400]  = init_emmc_hs400,
+};
+
+static unsigned int sdhci_cdns6_read_phy_reg(struct sdhci_cdns_priv *priv, const u32 address)
+{
+	writel(address, priv->hrs_addr + SDHCI_CDNS_HRS04);
+	return readl(priv->hrs_addr + SDHCI_CDNS_HRS05);
+}
+
+static void sdhci_cdns6_write_phy_reg(struct sdhci_cdns_priv *priv, const u32 address,
+				      const u32 value)
+{
+	writel(address, priv->hrs_addr + SDHCI_CDNS_HRS04);
+	writel(value, priv->hrs_addr + SDHCI_CDNS_HRS05);
+}
+
+static int sdhci_cdns6_phy_lock_dll(struct sdhci_cdns6_phy *phy)
+{
+	u32 delay_element = phy->delay_element_org;
+	u32 delay_elements_in_sdmclk;
+
+	delay_elements_in_sdmclk = DIV_ROUND_UP(phy->t_sdmclk, delay_element);
+	if (delay_elements_in_sdmclk > 256) {
+		delay_element *= 2;
+		delay_elements_in_sdmclk = DIV_ROUND_UP(phy->t_sdmclk, delay_element);
+
+		if (delay_elements_in_sdmclk > 256)
+			return -EINVAL;
+
+		phy->dll_max_value = 127;
+	} else {
+		phy->dll_max_value = 255;
+	}
+
+	phy->t_sdmclk_calc = delay_element * delay_elements_in_sdmclk;
+	phy->delay_element = delay_element;
+	phy->cp_dll_bypass_mode = false;
+
+	return 0;
+}
+
+static void sdhci_cdns6_phy_dll_bypass(struct sdhci_cdns6_phy *phy)
+{
+	phy->dll_max_value = 255;
+	phy->cp_dll_bypass_mode = true;
+}
+
+static void sdhci_cdns6_phy_configure_dll(struct sdhci_cdns6_phy *phy)
+{
+	if (!phy->sdhc_extended_wr_mode) {
+		if (sdhci_cdns6_phy_lock_dll(phy) == 0)
+			return;
+	}
+	sdhci_cdns6_phy_dll_bypass(phy);
+}
+
+static void sdhci_cdns6_phy_calc_out(struct sdhci_cdns6_phy *phy, bool cmd_not_dat)
+{
+	u32 wr0_dly = 0, wr1_dly = 0, output_min, output_max, phy_o_delay,
+	    clk_wr_delay = 0, wr0_sdclk_dly = 0, wr1_sdclk_dly = 0;
+	bool ddr = (phy->mode == MMC_TIMING_UHS_DDR50) || (phy->mode == MMC_TIMING_MMC_DDR52) ||
+		   (phy->mode == MMC_TIMING_MMC_HS400);
+	bool data_ddr = ddr && !cmd_not_dat;
+	int t;
+
+	if (cmd_not_dat) {
+		output_min = phy->t_cmd_output_min;
+		output_max = phy->t_cmd_output_max;
+		phy_o_delay = phy->phy_cmd_o_delay;
+	} else {
+		output_min = phy->t_dat_output_min;
+		output_max = phy->t_dat_output_max;
+		phy_o_delay = phy->phy_dat_o_delay;
+	}
+
+	if (data_ddr) {
+		wr0_sdclk_dly = 1;
+		wr1_sdclk_dly = 1;
+	}
+
+	t = phy_o_delay - phy->phy_sdclk_delay - output_min;
+	if (t < 0 && phy->sdhc_extended_wr_mode) {
+		u32 n_half_cycle = DIV_ROUND_UP(-t * 2, phy->t_sdmclk);
+
+		wr0_dly = (n_half_cycle + 1) / 2;
+		if (data_ddr)
+			wr1_dly = (n_half_cycle + 1) / 2;
+		else
+			wr1_dly = (n_half_cycle + 1) % 2 + wr0_dly - 1;
+	}
+
+	if (!phy->sdhc_extended_wr_mode) {
+		u32 out_hold, out_setup, out_hold_margin;
+		u32 n;
+
+		if (!data_ddr)
+			wr0_dly = 1;
+
+		out_setup = output_max;
+		out_hold = output_min;
+		out_hold_margin = DIV_ROUND_UP(out_setup - out_hold, 4);
+		out_hold += out_hold_margin;
+
+		if (!phy->cp_dll_bypass_mode)
+			n = DIV_ROUND_UP(256 * out_hold, phy->t_sdmclk_calc);
+		else
+			n = DIV_ROUND_UP(out_hold, phy->delay_element) - 1;
+
+		if (n <= phy->dll_max_value)
+			clk_wr_delay = n;
+		else
+			clk_wr_delay = 255;
+	} else {
+		/* sdhc_extended_wr_mode set => PHY IO cell work in SDR mode */
+		clk_wr_delay = 0;
+	}
+
+	if (cmd_not_dat) {
+		phy->sdhc_wrcmd0_dly = wr0_dly;
+		phy->sdhc_wrcmd1_dly = wr1_dly;
+		phy->cp_clk_wrdqs_delay = clk_wr_delay;
+		phy->sdhc_wrcmd0_sdclk_dly = wr0_sdclk_dly;
+		phy->sdhc_wrcmd1_sdclk_dly = wr1_sdclk_dly;
+	} else {
+		phy->sdhc_wrdata0_dly = wr0_dly;
+		phy->sdhc_wrdata1_dly = wr1_dly;
+		phy->cp_clk_wr_delay = clk_wr_delay;
+		phy->sdhc_wrdata0_sdclk_dly = wr0_sdclk_dly;
+		phy->sdhc_wrdata1_sdclk_dly = wr1_sdclk_dly;
+	}
+}
+
+static void sdhci_cdns6_phy_calc_cmd_out(struct sdhci_cdns6_phy *phy)
+{
+	sdhci_cdns6_phy_calc_out(phy, true);
+}
+
+static void sdhci_cdns6_phy_calc_cmd_in(struct sdhci_cdns6_phy *phy)
+{
+	phy->cp_io_mask_end = ((phy->iocell_output_delay + phy->iocell_input_delay) * 2) /
+				phy->t_sdmclk;
+
+	/* cp_io_mask_end is a 3-bit field, clamp to max value of 7 */
+	phy->cp_io_mask_end = min_t(u8, phy->cp_io_mask_end, 7);
+
+	if (phy->strobe_cmd && phy->cp_io_mask_end > 0)
+		phy->cp_io_mask_end--;
+
+	if (phy->strobe_cmd) {
+		phy->cp_use_phony_dqs_cmd = false;
+		phy->cp_read_dqs_cmd_delay = 64;
+	} else {
+		phy->cp_use_phony_dqs_cmd = true;
+		phy->cp_read_dqs_cmd_delay = 0;
+	}
+
+	if ((phy->mode == MMC_TIMING_MMC_HS400 && !phy->strobe_cmd) ||
+	    phy->mode == MMC_TIMING_MMC_HS200)
+		phy->cp_read_dqs_cmd_delay = phy->hs200_tune_val;
+}
+
+static void sdhci_cdns6_phy_calc_dat_in(struct sdhci_cdns6_phy *phy)
+{
+	u32 hcsdclkadj = 0;
+	bool strobe_dat = (phy->mode == MMC_TIMING_MMC_HS400);
+
+	if (strobe_dat) {
+		phy->cp_use_phony_dqs = false;
+		phy->cp_read_dqs_delay = 64;
+	} else {
+		phy->cp_use_phony_dqs = true;
+		phy->cp_read_dqs_delay = 0;
+	}
+
+	if (phy->mode == MMC_TIMING_MMC_HS200)
+		phy->cp_read_dqs_delay = phy->hs200_tune_val;
+
+	if (strobe_dat) {
+		/* dqs loopback input via IO cell */
+		hcsdclkadj += phy->iocell_input_delay;
+		/* dfi_dqs_in: mem_dqs -> clean_dqs_mod; delay of hic_dll_dqs_nand2 */
+		hcsdclkadj += phy->delay_element / 2;
+		/* delay line */
+		hcsdclkadj += phy->t_sdclk / 2;
+		/* PHY FIFO write pointer */
+		hcsdclkadj += phy->t_sdclk / 2 + phy->delay_element;
+		/* 1st synchronizer */
+		hcsdclkadj += DIV_ROUND_UP(hcsdclkadj, phy->t_sdmclk) * phy->t_sdmclk - hcsdclkadj;
+		/*
+		 * 2nd synchronizer + PHY FIFO read pointer + PHY rddata
+		 * + PHY rddata registered, + FIFO 1st ciu_en
+		 */
+		hcsdclkadj += 5 * phy->t_sdmclk;
+		/* FIFO 2nd ciu_en */
+		hcsdclkadj += phy->t_sdclk;
+		hcsdclkadj /= phy->t_sdclk;
+	} else {
+		u32 n;
+
+		/* rebar PHY delay */
+		hcsdclkadj += 2 * phy->t_sdmclk;
+		/* rebar output via IO cell */
+		hcsdclkadj += phy->iocell_output_delay;
+		/* dqs loopback input via IO cell */
+		hcsdclkadj += phy->iocell_input_delay;
+		/* dfi_dqs_in: mem_dqs -> clean_dqs_mod delay of hic_dll_dqs_nand2 */
+		hcsdclkadj += phy->delay_element / 2;
+		/* dll: one delay element between SIGI_0 and SIGO_0 */
+		hcsdclkadj += phy->delay_element;
+		/* dfi_dqs_in: mem_dqs_delayed -> clk_dqs delay of hic_dll_dqs_nand2 */
+		hcsdclkadj += phy->delay_element / 2;
+		/* deskew DLL: clk_dqs -> clk_dqN: one delay element */
+		hcsdclkadj += phy->delay_element;
+
+		if (phy->t_sdclk == phy->t_sdmclk)
+			n = (hcsdclkadj - 2 * phy->t_sdmclk) / phy->t_sdclk;
+		else
+			n = hcsdclkadj / phy->t_sdclk;
+
+		/* phase shift within one t_sdclk clock cycle caused by rebar - lbk dqs delay */
+		hcsdclkadj = hcsdclkadj % phy->t_sdclk;
+		/* PHY FIFO write pointer */
+		hcsdclkadj += phy->t_sdclk / 2;
+		/* 1st synchronizer */
+		hcsdclkadj += DIV_ROUND_UP(hcsdclkadj, phy->t_sdmclk) * phy->t_sdmclk - hcsdclkadj;
+		/*
+		 * 2nd synchronizer + PHY FIFO read pointer + PHY rddata + PHY rddata registered
+		 */
+		hcsdclkadj += 4 * phy->t_sdmclk;
+
+		if ((phy->t_sdclk / phy->t_sdmclk) > 1) {
+			u32 tmp1, tmp2;
+
+			tmp1 = hcsdclkadj;
+			tmp2 = (hcsdclkadj / phy->t_sdclk) * phy->t_sdclk + phy->t_sdclk -
+				phy->t_sdmclk;
+			if (tmp1 == tmp2)
+				tmp2 += phy->t_sdclk;
+
+			/* FIFO aligns to clock cycle before ciu_en */
+			hcsdclkadj += tmp2 - tmp1;
+		}
+
+		/* FIFO 1st ciu_en */
+		hcsdclkadj += phy->t_sdmclk;
+		/* FIFO 2nd ciu_en */
+		hcsdclkadj += phy->t_sdclk;
+		hcsdclkadj /= phy->t_sdclk;
+		hcsdclkadj += n;
+
+		if ((phy->t_sdclk / phy->t_sdmclk) >= 2) {
+			if (phy->mode == MMC_TIMING_UHS_DDR50 || phy->mode == MMC_TIMING_MMC_DDR52)
+				hcsdclkadj -= 2;
+			else
+				hcsdclkadj -= 1;
+		} else if ((phy->t_sdclk / phy->t_sdmclk) == 1) {
+			hcsdclkadj += 2;
+		}
+
+		if (phy->mode == MMC_TIMING_UHS_SDR104 || phy->mode == MMC_TIMING_MMC_HS200)
+			hcsdclkadj -= 1;
+	}
+
+	/* hcsdclkadj is a 4-bit field, clamp to max value of 15 */
+	if (hcsdclkadj > 15)
+		hcsdclkadj = 15;
+
+	phy->sdhc_hcsdclkadj = hcsdclkadj;
+}
+
+static void sdhci_cdns6_phy_calc_dat_out(struct sdhci_cdns6_phy *phy)
+{
+	sdhci_cdns6_phy_calc_out(phy, false);
+}
+
+static void sdhci_cdns6_phy_calc_io(struct sdhci_cdns6_phy *phy)
+{
+	u32 rw_compensate;
+
+	rw_compensate = ((phy->iocell_input_delay + phy->iocell_output_delay) / phy->t_sdmclk) +
+			  phy->sdhc_wrdata0_dly + 5 + 3;
+
+	phy->sdhc_idelay_val = (2 * phy->iocell_input_delay) / phy->t_sdmclk;
+
+	phy->cp_io_mask_start = 0;
+	if (phy->t_sdclk == phy->t_sdmclk && rw_compensate > 10)
+		phy->cp_io_mask_start = 2 * (rw_compensate - 10);
+
+	if (phy->mode == MMC_TIMING_UHS_SDR104)
+		phy->cp_io_mask_start++;
+
+	if (phy->t_sdclk == phy->t_sdmclk && phy->mode == MMC_TIMING_UHS_SDR50)
+		phy->cp_io_mask_start++;
+
+	/* cp_io_mask_start is a 3-bit field, clamp to max value of 7 */
+	phy->cp_io_mask_start = min_t(u8, phy->cp_io_mask_start, 7);
+
+	phy->sdhc_rw_compensate = rw_compensate;
+}
+
+static void sdhci_cdns6_phy_calc_settings(struct sdhci_cdns6_phy *phy)
+{
+	sdhci_cdns6_phy_calc_cmd_out(phy);
+	sdhci_cdns6_phy_calc_cmd_in(phy);
+	sdhci_cdns6_phy_calc_dat_out(phy);
+	sdhci_cdns6_phy_calc_dat_in(phy);
+	sdhci_cdns6_phy_calc_io(phy);
+}
+
+static int sdhci_cdns6_dll_reset(struct sdhci_cdns_priv *priv, bool reset)
+{
+	u32 reg;
+	int ret = 0;
+
+	reg = readl(priv->hrs_addr + SDHCI_CDNS_HRS09);
+	if (reset)
+		reg &= ~SDHCI_CDNS_HRS09_PHY_SW_RESET;
+	else
+		reg |= SDHCI_CDNS_HRS09_PHY_SW_RESET;
+
+	writel(reg, priv->hrs_addr + SDHCI_CDNS_HRS09);
+
+	/* After releasing PHY from reset, wait until PHY_INIT_COMPLETE is set within 3000us */
+	if (!reset) {
+		ret = readl_poll_timeout(priv->hrs_addr + SDHCI_CDNS_HRS09, reg, (reg &
+					 SDHCI_CDNS_HRS09_PHY_INIT_COMPLETE), 0, 3000);
+	}
+
+	return ret;
+}
+
+int sdhci_cdns6_phy_init(struct sdhci_cdns_priv *priv)
+{
+	struct sdhci_cdns6_phy *phy = priv->phy;
+	u32 reg;
+	int ret;
+
+	sdhci_cdns6_dll_reset(priv, true);
+
+	reg = sdhci_cdns6_read_phy_reg(priv, SDHCI_CDNS6_PHY_DQS_TIMING_REG);
+	reg &= ~SDHCI_CDNS6_PHY_DQS_TIMING_USE_PHONY_DQS;
+	reg &= ~SDHCI_CDNS6_PHY_DQS_TIMING_USE_PHONY_DQS_CMD;
+	reg |= SDHCI_CDNS6_PHY_DQS_TIMING_USE_EXT_LPBK_DQS;
+	reg |= SDHCI_CDNS6_PHY_DQS_TIMING_USE_LPBK_DQS;
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DQS_TIMING_USE_PHONY_DQS, phy->cp_use_phony_dqs);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DQS_TIMING_USE_PHONY_DQS_CMD, phy->cp_use_phony_dqs_cmd);
+	sdhci_cdns6_write_phy_reg(priv, SDHCI_CDNS6_PHY_DQS_TIMING_REG, reg);
+
+	reg = sdhci_cdns6_read_phy_reg(priv, SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_REG);
+	reg &= ~SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_RD_DEL_SEL;
+	reg |= SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_UNDERRUN_SUPPRESS;
+	reg |= SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_GATE_CFG_ALWAYS_ON;
+	reg |= SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_SYNC_METHOD;
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_RD_DEL_SEL,
+			  SDHCI_CDNS6_PHY_DEFAULT_RD_DEL_SEL);
+	sdhci_cdns6_write_phy_reg(priv, SDHCI_CDNS6_PHY_GATE_LPBK_CTRL_REG, reg);
+
+	reg = FIELD_PREP(SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_BYPASS_MODE, phy->cp_dll_bypass_mode);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_PHASE_DETECT_SEL,
+			  SDHCI_CDNS6_PHY_DEFAULT_PHASE_DETECT_SEL);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_DLL_LOCK_NUM,
+			  SDHCI_CDNS6_PHY_DEFAULT_DLL_LOCK_NUM);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_DLL_START_POINT,
+			  SDHCI_CDNS6_PHY_DEFAULT_DLL_START);
+	sdhci_cdns6_write_phy_reg(priv, SDHCI_CDNS6_PHY_DLL_MASTER_CTRL_REG, reg);
+
+	reg = FIELD_PREP(SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_CMD_DELAY,
+			 phy->cp_read_dqs_cmd_delay);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_CLK_WRDQS_DELAY, phy->cp_clk_wrdqs_delay);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_CLK_WR_DELAY, phy->cp_clk_wr_delay);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_DELAY, phy->cp_read_dqs_delay);
+	sdhci_cdns6_write_phy_reg(priv, SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_REG, reg);
+
+	reg = sdhci_cdns6_read_phy_reg(priv, SDHCI_CDNS6_PHY_CTRL_REG);
+	reg &= ~SDHCI_CDNS6_PHY_CTRL_PHONY_DQS_TIMING;
+	sdhci_cdns6_write_phy_reg(priv, SDHCI_CDNS6_PHY_CTRL_REG, reg);
+
+	/*
+	 * Ensure all preceding PHY register writes complete and reach the controller before
+	 * releasing the PHY from reset. Without this, SDR104 has been observed to fail
+	 * intermittently on some boards.
+	 */
+	wmb();
+
+	ret = sdhci_cdns6_dll_reset(priv, false);
+	if (ret)
+		return ret;
+
+	reg = sdhci_cdns6_read_phy_reg(priv, SDHCI_CDNS6_PHY_DQ_TIMING_REG);
+	reg &= ~SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_ALWAYS_ON;
+	reg &= ~SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_END;
+	reg &= ~SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_START;
+	reg &= ~SDHCI_CDNS6_PHY_DQ_TIMING_DATA_SELECT_OE_END;
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_END, phy->cp_io_mask_end);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DQ_TIMING_IO_MASK_START, phy->cp_io_mask_start);
+	reg |= FIELD_PREP(SDHCI_CDNS6_PHY_DQ_TIMING_DATA_SELECT_OE_END,
+			  SDHCI_CDNS6_PHY_DEFAULT_DATA_SELECT_OE_END);
+	sdhci_cdns6_write_phy_reg(priv, SDHCI_CDNS6_PHY_DQ_TIMING_REG, reg);
+
+	/* Ensure DQ timing programming is visible before HRS09 follow-up writes */
+	wmb();
+
+	reg = readl(priv->hrs_addr + SDHCI_CDNS_HRS09);
+	if (phy->sdhc_extended_wr_mode)
+		reg |= SDHCI_CDNS_HRS09_EXTENDED_WR_MODE;
+	else
+		reg &= ~SDHCI_CDNS_HRS09_EXTENDED_WR_MODE;
+
+	if (phy->sdhc_extended_rd_mode)
+		reg |= SDHCI_CDNS_HRS09_EXTENDED_RD_MODE;
+	else
+		reg &= ~SDHCI_CDNS_HRS09_EXTENDED_RD_MODE;
+
+	reg |= SDHCI_CDNS_HRS09_RDDATA_EN;
+	reg |= SDHCI_CDNS_HRS09_RDCMD_EN;
+	writel(reg, priv->hrs_addr + SDHCI_CDNS_HRS09);
+
+	reg = FIELD_PREP(SDHCI_CDNS_HRS10_HCSDCLKADJ, phy->sdhc_hcsdclkadj);
+	writel(reg, priv->hrs_addr + SDHCI_CDNS_HRS10);
+
+	reg = FIELD_PREP(SDHCI_CDNS_HRS16_WRDATA1_SDCLK_DLY, phy->sdhc_wrdata1_sdclk_dly);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS16_WRDATA0_SDCLK_DLY, phy->sdhc_wrdata0_sdclk_dly);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS16_WRCMD1_SDCLK_DLY, phy->sdhc_wrcmd1_sdclk_dly);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS16_WRCMD0_SDCLK_DLY, phy->sdhc_wrcmd0_sdclk_dly);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS16_WRDATA1_DLY, phy->sdhc_wrdata1_dly);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS16_WRDATA0_DLY, phy->sdhc_wrdata0_dly);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS16_WRCMD1_DLY, phy->sdhc_wrcmd1_dly);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS16_WRCMD0_DLY, phy->sdhc_wrcmd0_dly);
+	writel(reg, priv->hrs_addr + SDHCI_CDNS_HRS16);
+
+	reg = FIELD_PREP(SDHCI_CDNS_HRS07_RW_COMPENSATE, phy->sdhc_rw_compensate);
+	reg |= FIELD_PREP(SDHCI_CDNS_HRS07_IDELAY_VAL, phy->sdhc_idelay_val);
+	writel(reg, priv->hrs_addr + SDHCI_CDNS_HRS07);
+
+	/* Allow 5 to 5.5 ms for clock and PHY signals to stabilize after configuration */
+	usleep_range(5000, 5500);
+
+	return 0;
+}
+
+int sdhci_cdns6_set_tune_val(struct sdhci_host *host, unsigned int val)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	struct sdhci_cdns6_phy *phy = priv->phy;
+	u32 tmp, tuneval;
+	int ret;
+
+	/*
+	 * Scale tuning tap (val in [0, SDHCI_CDNS_MAX_TUNING_LOOP-1]) to the
+	 * 8-bit PHY DLL slave delay field [0, 255]. With MAX_TUNING_LOOP=40
+	 * and FIELD_SIZE=256, the result fits in 8 bits.
+	 */
+	tuneval = (val * SDHCI_CDNS6_PHY_DLL_FIELD_SIZE) / SDHCI_CDNS_MAX_TUNING_LOOP;
+
+	phy->hs200_tune_val = tuneval;
+	phy->cp_read_dqs_cmd_delay = tuneval;
+	phy->cp_read_dqs_delay = tuneval;
+
+	tmp = sdhci_cdns6_read_phy_reg(priv, SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_REG);
+	tmp &= ~(SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_CMD_DELAY |
+		 SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_DELAY);
+	tmp |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_CMD_DELAY, tuneval);
+	tmp |= FIELD_PREP(SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_READ_DQS_DELAY, tuneval);
+
+	sdhci_cdns6_dll_reset(priv, true);
+	sdhci_cdns6_write_phy_reg(priv, SDHCI_CDNS6_PHY_DLL_SLAVE_CTRL_REG, tmp);
+
+	ret = sdhci_cdns6_dll_reset(priv, false);
+	if (ret)
+		dev_warn(mmc_dev(host->mmc), "%s: DLL reset release failed: %d\n", __func__, ret);
+
+	return ret;
+}
+
+static int sdhci_cdns6_phy_update_timings(struct sdhci_host *host)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	struct sdhci_cdns6_phy *phy = priv->phy;
+	u32 t_sdmclk = phy->t_sdmclk;
+
+	/* Validate mode is within supported range */
+	if (phy->mode >= ARRAY_SIZE(init_timings))
+		return -EINVAL;
+
+	/* initialize input */
+	init_timings[phy->mode](phy, phy->t_sdclk);
+
+	phy->strobe_cmd = false;
+
+	if (priv->enhanced_strobe)
+		phy->strobe_cmd = true;
+
+	phy->phy_sdclk_delay = 2 * t_sdmclk;
+
+	/*
+	 * CMD and DAT output delays are currently identical, but kept separate to allow
+	 * independent tuning for specific modes (e.g., HS400) or board-specific optimizations
+	 * in the future.
+	 */
+	phy->phy_cmd_o_delay = 2 * t_sdmclk + t_sdmclk / 2;
+	phy->phy_dat_o_delay = 2 * t_sdmclk + t_sdmclk / 2;
+
+	if (phy->t_sdclk == phy->t_sdmclk) {
+		phy->sdhc_extended_wr_mode = false;
+		phy->sdhc_extended_rd_mode = false;
+	} else {
+		phy->sdhc_extended_wr_mode = true;
+		phy->sdhc_extended_rd_mode = true;
+	}
+
+	sdhci_cdns6_phy_configure_dll(phy);
+	sdhci_cdns6_phy_calc_settings(phy);
+
+	return 0;
+}
+
+int sdhci_cdns6_phy_probe(struct platform_device *pdev, struct sdhci_cdns_priv *priv)
+{
+	struct device *dev = &pdev->dev;
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct sdhci_cdns6_phy *phy;
+	unsigned long val;
+	int ret;
+
+	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
+	if (!phy)
+		return -ENOMEM;
+
+	val = clk_get_rate(pltfm_host->clk);
+	if (!val)
+		return dev_err_probe(dev, -EINVAL, "failed to get controller clock rate\n");
+
+	phy->t_sdmclk = DIV_ROUND_DOWN_ULL(1000000000000ULL, val);
+
+	/*
+	 * Optional board-level PHY timing properties. When omitted, use the
+	 * Cadence-recommended defaults (2500 ps IO-cell delays, 24 ps DLL
+	 * delay element) that are valid for typical SoC packaging.
+	 */
+	ret = of_property_read_u32(dev->of_node, "cdns,iocell-input-delay-ps",
+				   &phy->iocell_input_delay);
+	if (ret)
+		phy->iocell_input_delay = SDHCI_CDNS6_PHY_DEFAULT_IOCELL_DELAY;
+
+	ret = of_property_read_u32(dev->of_node, "cdns,iocell-output-delay-ps",
+				   &phy->iocell_output_delay);
+	if (ret)
+		phy->iocell_output_delay = SDHCI_CDNS6_PHY_DEFAULT_IOCELL_DELAY;
+
+	ret = of_property_read_u32(dev->of_node, "cdns,delay-element-ps", &phy->delay_element);
+	if (ret) {
+		phy->delay_element = SDHCI_CDNS6_PHY_DEFAULT_DELAY_ELEMENT;
+	} else if (!phy->delay_element) {
+		dev_warn(dev, "cdns,delay-element-ps is 0, using default %u\n",
+			 SDHCI_CDNS6_PHY_DEFAULT_DELAY_ELEMENT);
+		phy->delay_element = SDHCI_CDNS6_PHY_DEFAULT_DELAY_ELEMENT;
+	}
+
+	phy->delay_element_org = phy->delay_element;
+
+	priv->phy = phy;
+
+	return 0;
+}
+
+static void sdhci_cdns6_phy_reconfigure(struct sdhci_host *host, unsigned int timing)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	struct sdhci_cdns6_phy *phy = priv->phy;
+	unsigned int rate = host->mmc->actual_clock ?: host->mmc->ios.clock;
+	u32 t_sdclk;
+	int ret;
+
+	if (!rate)
+		return;
+
+	t_sdclk = DIV_ROUND_DOWN_ULL(1000000000000ULL, rate);
+	if (phy->mode == timing && phy->t_sdclk == t_sdclk &&
+	    phy->strobe_cmd == priv->enhanced_strobe)
+		return;
+
+	phy->t_sdclk = t_sdclk;
+	phy->mode = timing;
+
+	ret = sdhci_cdns6_phy_update_timings(host);
+	if (ret) {
+		phy->t_sdclk = 0;
+		dev_warn(mmc_dev(host->mmc), "%s: update timings failed: %d\n", __func__, ret);
+		return;
+	}
+
+	ret = sdhci_cdns6_phy_init(priv);
+	if (ret) {
+		phy->t_sdclk = 0;
+		dev_warn(mmc_dev(host->mmc), "%s: phy init failed: %d\n", __func__, ret);
+	}
+}
+
+void sdhci_cdns6_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	struct sdhci_cdns6_phy *phy = priv->phy;
+
+	sdhci_set_clock(host, clock);
+
+	if (!clock) {
+		/* Force PHY reprogramming on the next non-zero clock. */
+		phy->t_sdclk = 0;
+		return;
+	}
+
+	sdhci_cdns6_phy_reconfigure(host, host->mmc->ios.timing);
+}
+
+void sdhci_cdns6_set_uhs_signaling(struct sdhci_host *host, unsigned int timing)
+{
+	sdhci_cdns6_phy_reconfigure(host, timing);
+}
+
+void sdhci_cdns6_hw_reset(struct sdhci_host *host)
+{
+	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
+	void __iomem *reg;
+
+	reg = priv->hrs_addr + SDHCI_CDNS_HRS11;
+	writel(SDHCI_CDNS_HRS11_EMMC_RST, reg);
+	/* eMMC HW reset assertion: spec requires >= 1us, give margin */
+	usleep_range(10, 20);
+	writel(0, reg);
+	/* For eMMC, minimum is 200us but give it 300us for good measure */
+	usleep_range(300, 1000);
+}
diff --git a/drivers/mmc/host/sdhci-cadence.c b/drivers/mmc/host/sdhci-cadence.c
deleted file mode 100644
index 435603c8c00b2a..00000000000000
--- a/drivers/mmc/host/sdhci-cadence.c
+++ /dev/null
@@ -1,675 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2016 Socionext Inc.
- *   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
- */
-
-#include <linux/bitfield.h>
-#include <linux/bits.h>
-#include <linux/iopoll.h>
-#include <linux/module.h>
-#include <linux/mmc/host.h>
-#include <linux/mmc/mmc.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/reset.h>
-
-#include "sdhci-pltfm.h"
-
-/* HRS - Host Register Set (specific to Cadence) */
-#define SDHCI_CDNS_HRS04		0x10		/* PHY access port */
-#define   SDHCI_CDNS_HRS04_ACK			BIT(26)
-#define   SDHCI_CDNS_HRS04_RD			BIT(25)
-#define   SDHCI_CDNS_HRS04_WR			BIT(24)
-#define   SDHCI_CDNS_HRS04_RDATA		GENMASK(23, 16)
-#define   SDHCI_CDNS_HRS04_WDATA		GENMASK(15, 8)
-#define   SDHCI_CDNS_HRS04_ADDR			GENMASK(5, 0)
-
-#define SDHCI_CDNS_HRS06		0x18		/* eMMC control */
-#define   SDHCI_CDNS_HRS06_TUNE_UP		BIT(15)
-#define   SDHCI_CDNS_HRS06_TUNE			GENMASK(13, 8)
-#define   SDHCI_CDNS_HRS06_MODE			GENMASK(2, 0)
-#define   SDHCI_CDNS_HRS06_MODE_SD		0x0
-#define   SDHCI_CDNS_HRS06_MODE_MMC_SDR		0x2
-#define   SDHCI_CDNS_HRS06_MODE_MMC_DDR		0x3
-#define   SDHCI_CDNS_HRS06_MODE_MMC_HS200	0x4
-#define   SDHCI_CDNS_HRS06_MODE_MMC_HS400	0x5
-#define   SDHCI_CDNS_HRS06_MODE_MMC_HS400ES	0x6
-
-/* Read block gap */
-#define SDHCI_CDNS_HRS37		0x94	/* interface mode select */
-#define   SDHCI_CDNS_HRS37_MODE_DS		0x0
-#define   SDHCI_CDNS_HRS37_MODE_HS		0x1
-#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR12	0x8
-#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR25	0x9
-#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR50	0xa
-#define   SDHCI_CDNS_HRS37_MODE_UDS_SDR104	0xb
-#define   SDHCI_CDNS_HRS37_MODE_UDS_DDR50	0xc
-#define   SDHCI_CDNS_HRS37_MODE_MMC_LEGACY	0x20
-#define   SDHCI_CDNS_HRS37_MODE_MMC_SDR		0x21
-#define   SDHCI_CDNS_HRS37_MODE_MMC_DDR		0x22
-#define   SDHCI_CDNS_HRS37_MODE_MMC_HS200	0x23
-#define   SDHCI_CDNS_HRS37_MODE_MMC_HS400	0x24
-#define   SDHCI_CDNS_HRS37_MODE_MMC_HS400ES	0x25
-#define SDHCI_CDNS_HRS38		0x98	/* Read block gap coefficient */
-#define   SDHCI_CDNS_HRS38_BLKGAP_MAX		0xf
-
-/* SRS - Slot Register Set (SDHCI-compatible) */
-#define SDHCI_CDNS_SRS_BASE		0x200
-
-/* PHY */
-#define SDHCI_CDNS_PHY_DLY_SD_HS	0x00
-#define SDHCI_CDNS_PHY_DLY_SD_DEFAULT	0x01
-#define SDHCI_CDNS_PHY_DLY_UHS_SDR12	0x02
-#define SDHCI_CDNS_PHY_DLY_UHS_SDR25	0x03
-#define SDHCI_CDNS_PHY_DLY_UHS_SDR50	0x04
-#define SDHCI_CDNS_PHY_DLY_UHS_DDR50	0x05
-#define SDHCI_CDNS_PHY_DLY_EMMC_LEGACY	0x06
-#define SDHCI_CDNS_PHY_DLY_EMMC_SDR	0x07
-#define SDHCI_CDNS_PHY_DLY_EMMC_DDR	0x08
-#define SDHCI_CDNS_PHY_DLY_SDCLK	0x0b
-#define SDHCI_CDNS_PHY_DLY_HSMMC	0x0c
-#define SDHCI_CDNS_PHY_DLY_STROBE	0x0d
-
-/*
- * The tuned val register is 6 bit-wide, but not the whole of the range is
- * available.  The range 0-42 seems to be available (then 43 wraps around to 0)
- * but I am not quite sure if it is official.  Use only 0 to 39 for safety.
- */
-#define SDHCI_CDNS_MAX_TUNING_LOOP	40
-
-struct sdhci_cdns_phy_param {
-	u8 addr;
-	u8 data;
-};
-
-struct sdhci_cdns_priv {
-	void __iomem *hrs_addr;
-	void __iomem *ctl_addr;	/* write control */
-	spinlock_t wrlock;	/* write lock */
-	bool enhanced_strobe;
-	void (*priv_writel)(struct sdhci_cdns_priv *priv, u32 val, void __iomem *reg);
-	struct reset_control *rst_hw;
-	unsigned int nr_phy_params;
-	struct sdhci_cdns_phy_param phy_params[];
-};
-
-struct sdhci_cdns_phy_cfg {
-	const char *property;
-	u8 addr;
-};
-
-struct sdhci_cdns_drv_data {
-	int (*init)(struct platform_device *pdev);
-	const struct sdhci_pltfm_data pltfm_data;
-};
-
-static const struct sdhci_cdns_phy_cfg sdhci_cdns_phy_cfgs[] = {
-	{ "cdns,phy-input-delay-sd-highspeed", SDHCI_CDNS_PHY_DLY_SD_HS, },
-	{ "cdns,phy-input-delay-legacy", SDHCI_CDNS_PHY_DLY_SD_DEFAULT, },
-	{ "cdns,phy-input-delay-sd-uhs-sdr12", SDHCI_CDNS_PHY_DLY_UHS_SDR12, },
-	{ "cdns,phy-input-delay-sd-uhs-sdr25", SDHCI_CDNS_PHY_DLY_UHS_SDR25, },
-	{ "cdns,phy-input-delay-sd-uhs-sdr50", SDHCI_CDNS_PHY_DLY_UHS_SDR50, },
-	{ "cdns,phy-input-delay-sd-uhs-ddr50", SDHCI_CDNS_PHY_DLY_UHS_DDR50, },
-	{ "cdns,phy-input-delay-mmc-highspeed", SDHCI_CDNS_PHY_DLY_EMMC_SDR, },
-	{ "cdns,phy-input-delay-mmc-ddr", SDHCI_CDNS_PHY_DLY_EMMC_DDR, },
-	{ "cdns,phy-dll-delay-sdclk", SDHCI_CDNS_PHY_DLY_SDCLK, },
-	{ "cdns,phy-dll-delay-sdclk-hsmmc", SDHCI_CDNS_PHY_DLY_HSMMC, },
-	{ "cdns,phy-dll-delay-strobe", SDHCI_CDNS_PHY_DLY_STROBE, },
-};
-
-static inline void cdns_writel(struct sdhci_cdns_priv *priv, u32 val,
-			       void __iomem *reg)
-{
-	writel(val, reg);
-}
-
-static int sdhci_cdns_write_phy_reg(struct sdhci_cdns_priv *priv,
-				    u8 addr, u8 data)
-{
-	void __iomem *reg = priv->hrs_addr + SDHCI_CDNS_HRS04;
-	u32 tmp;
-	int ret;
-
-	ret = readl_poll_timeout(reg, tmp, !(tmp & SDHCI_CDNS_HRS04_ACK),
-				 0, 10);
-	if (ret)
-		return ret;
-
-	tmp = FIELD_PREP(SDHCI_CDNS_HRS04_WDATA, data) |
-	      FIELD_PREP(SDHCI_CDNS_HRS04_ADDR, addr);
-	priv->priv_writel(priv, tmp, reg);
-
-	tmp |= SDHCI_CDNS_HRS04_WR;
-	priv->priv_writel(priv, tmp, reg);
-
-	ret = readl_poll_timeout(reg, tmp, tmp & SDHCI_CDNS_HRS04_ACK, 0, 10);
-	if (ret)
-		return ret;
-
-	tmp &= ~SDHCI_CDNS_HRS04_WR;
-	priv->priv_writel(priv, tmp, reg);
-
-	ret = readl_poll_timeout(reg, tmp, !(tmp & SDHCI_CDNS_HRS04_ACK),
-				 0, 10);
-
-	return ret;
-}
-
-static unsigned int sdhci_cdns_phy_param_count(struct device_node *np)
-{
-	unsigned int count = 0;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(sdhci_cdns_phy_cfgs); i++)
-		if (of_property_present(np, sdhci_cdns_phy_cfgs[i].property))
-			count++;
-
-	return count;
-}
-
-static void sdhci_cdns_phy_param_parse(struct device_node *np,
-				       struct sdhci_cdns_priv *priv)
-{
-	struct sdhci_cdns_phy_param *p = priv->phy_params;
-	u32 val;
-	int ret, i;
-
-	for (i = 0; i < ARRAY_SIZE(sdhci_cdns_phy_cfgs); i++) {
-		ret = of_property_read_u32(np, sdhci_cdns_phy_cfgs[i].property,
-					   &val);
-		if (ret)
-			continue;
-
-		p->addr = sdhci_cdns_phy_cfgs[i].addr;
-		p->data = val;
-		p++;
-	}
-}
-
-static int sdhci_cdns_phy_init(struct sdhci_cdns_priv *priv)
-{
-	int ret, i;
-
-	for (i = 0; i < priv->nr_phy_params; i++) {
-		ret = sdhci_cdns_write_phy_reg(priv, priv->phy_params[i].addr,
-					       priv->phy_params[i].data);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static void *sdhci_cdns_priv(struct sdhci_host *host)
-{
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-
-	return sdhci_pltfm_priv(pltfm_host);
-}
-
-static unsigned int sdhci_cdns_get_timeout_clock(struct sdhci_host *host)
-{
-	/*
-	 * Cadence's spec says the Timeout Clock Frequency is the same as the
-	 * Base Clock Frequency.
-	 */
-	return host->max_clk;
-}
-
-static void sdhci_cdns_set_emmc_mode(struct sdhci_cdns_priv *priv, u32 mode)
-{
-	u32 tmp;
-
-	/* The speed mode for eMMC is selected by HRS06 register */
-	tmp = readl(priv->hrs_addr + SDHCI_CDNS_HRS06);
-	tmp &= ~SDHCI_CDNS_HRS06_MODE;
-	tmp |= FIELD_PREP(SDHCI_CDNS_HRS06_MODE, mode);
-	priv->priv_writel(priv, tmp, priv->hrs_addr + SDHCI_CDNS_HRS06);
-}
-
-static u32 sdhci_cdns_get_emmc_mode(struct sdhci_cdns_priv *priv)
-{
-	u32 tmp;
-
-	tmp = readl(priv->hrs_addr + SDHCI_CDNS_HRS06);
-	return FIELD_GET(SDHCI_CDNS_HRS06_MODE, tmp);
-}
-
-static int sdhci_cdns_set_tune_val(struct sdhci_host *host, unsigned int val)
-{
-	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
-	void __iomem *reg = priv->hrs_addr + SDHCI_CDNS_HRS06;
-	u32 tmp;
-	int i, ret;
-
-	if (WARN_ON(!FIELD_FIT(SDHCI_CDNS_HRS06_TUNE, val)))
-		return -EINVAL;
-
-	tmp = readl(reg);
-	tmp &= ~SDHCI_CDNS_HRS06_TUNE;
-	tmp |= FIELD_PREP(SDHCI_CDNS_HRS06_TUNE, val);
-
-	/*
-	 * Workaround for IP errata:
-	 * The IP6116 SD/eMMC PHY design has a timing issue on receive data
-	 * path. Send tune request twice.
-	 */
-	for (i = 0; i < 2; i++) {
-		tmp |= SDHCI_CDNS_HRS06_TUNE_UP;
-		priv->priv_writel(priv, tmp, reg);
-
-		ret = readl_poll_timeout(reg, tmp,
-					 !(tmp & SDHCI_CDNS_HRS06_TUNE_UP),
-					 0, 1);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-/**
- * sdhci_cdns_tune_blkgap() - tune multi-block read gap
- * @mmc: MMC host
- *
- * Tune delay used in multi block read. To do so,
- * try sending multi-block read command with incremented gap, unless
- * it succeeds.
- *
- * Return: error code
- */
-static int sdhci_cdns_tune_blkgap(struct mmc_host *mmc)
-{
-	struct sdhci_host *host = mmc_priv(mmc);
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-	struct sdhci_cdns_priv *priv = sdhci_pltfm_priv(pltfm_host);
-	void __iomem *hrs37_reg = priv->hrs_addr + SDHCI_CDNS_HRS37;
-	void __iomem *hrs38_reg = priv->hrs_addr + SDHCI_CDNS_HRS38;
-	int ret;
-	u32 gap;
-
-	/* Currently only needed in HS200 mode */
-	if (host->timing != MMC_TIMING_MMC_HS200)
-		return 0;
-
-	writel(SDHCI_CDNS_HRS37_MODE_MMC_HS200, hrs37_reg);
-
-	for (gap = 0; gap <= SDHCI_CDNS_HRS38_BLKGAP_MAX; gap++) {
-		writel(gap, hrs38_reg);
-		ret = mmc_read_tuning(mmc, 512, 32);
-		if (!ret)
-			break;
-	}
-
-	dev_dbg(mmc_dev(mmc), "read block gap tune %s, gap %d\n", ret ? "failed" : "OK", gap);
-	return ret;
-}
-
-/*
- * In SD mode, software must not use the hardware tuning and instead perform
- * an almost identical procedure to eMMC.
- */
-static int sdhci_cdns_execute_tuning(struct sdhci_host *host, u32 opcode)
-{
-	int cur_streak = 0;
-	int max_streak = 0;
-	int end_of_streak = 0;
-	int i;
-	int ret;
-
-	/*
-	 * Do not execute tuning for UHS_SDR50 or UHS_DDR50.
-	 * The delay is set by probe, based on the DT properties.
-	 */
-	if (host->timing != MMC_TIMING_MMC_HS200 &&
-	    host->timing != MMC_TIMING_UHS_SDR104)
-		return 0;
-
-	for (i = 0; i < SDHCI_CDNS_MAX_TUNING_LOOP; i++) {
-		if (sdhci_cdns_set_tune_val(host, i) ||
-		    mmc_send_tuning(host->mmc, opcode, NULL)) { /* bad */
-			cur_streak = 0;
-		} else { /* good */
-			cur_streak++;
-			if (cur_streak > max_streak) {
-				max_streak = cur_streak;
-				end_of_streak = i;
-			}
-		}
-	}
-
-	if (!max_streak) {
-		dev_err(mmc_dev(host->mmc), "no tuning point found\n");
-		return -EIO;
-	}
-
-	ret = sdhci_cdns_set_tune_val(host, end_of_streak - max_streak / 2);
-	if (ret)
-		return ret;
-
-	return sdhci_cdns_tune_blkgap(host->mmc);
-}
-
-static void sdhci_cdns_set_uhs_signaling(struct sdhci_host *host,
-					 unsigned int timing)
-{
-	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
-	u32 mode;
-
-	switch (timing) {
-	case MMC_TIMING_MMC_HS:
-		mode = SDHCI_CDNS_HRS06_MODE_MMC_SDR;
-		break;
-	case MMC_TIMING_MMC_DDR52:
-		mode = SDHCI_CDNS_HRS06_MODE_MMC_DDR;
-		break;
-	case MMC_TIMING_MMC_HS200:
-		mode = SDHCI_CDNS_HRS06_MODE_MMC_HS200;
-		break;
-	case MMC_TIMING_MMC_HS400:
-		if (priv->enhanced_strobe)
-			mode = SDHCI_CDNS_HRS06_MODE_MMC_HS400ES;
-		else
-			mode = SDHCI_CDNS_HRS06_MODE_MMC_HS400;
-		break;
-	default:
-		mode = SDHCI_CDNS_HRS06_MODE_SD;
-		break;
-	}
-
-	sdhci_cdns_set_emmc_mode(priv, mode);
-
-	/* For SD, fall back to the default handler */
-	if (mode == SDHCI_CDNS_HRS06_MODE_SD)
-		sdhci_set_uhs_signaling(host, timing);
-}
-
-/* Elba control register bits [6:3] are byte-lane enables */
-#define ELBA_BYTE_ENABLE_MASK(x)	((x) << 3)
-
-/*
- * The Pensando Elba SoC explicitly controls byte-lane enabling on writes
- * which includes writes to the HRS registers.  The write lock (wrlock)
- * is used to ensure byte-lane enable, using write control (ctl_addr),
- * occurs before the data write.
- */
-static void elba_priv_writel(struct sdhci_cdns_priv *priv, u32 val,
-			     void __iomem *reg)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&priv->wrlock, flags);
-	writel(GENMASK(7, 3), priv->ctl_addr);
-	writel(val, reg);
-	spin_unlock_irqrestore(&priv->wrlock, flags);
-}
-
-static void elba_write_l(struct sdhci_host *host, u32 val, int reg)
-{
-	elba_priv_writel(sdhci_cdns_priv(host), val, host->ioaddr + reg);
-}
-
-static void elba_write_w(struct sdhci_host *host, u16 val, int reg)
-{
-	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
-	u32 shift = reg & GENMASK(1, 0);
-	unsigned long flags;
-	u32 byte_enables;
-
-	byte_enables = GENMASK(1, 0) << shift;
-	spin_lock_irqsave(&priv->wrlock, flags);
-	writel(ELBA_BYTE_ENABLE_MASK(byte_enables), priv->ctl_addr);
-	writew(val, host->ioaddr + reg);
-	spin_unlock_irqrestore(&priv->wrlock, flags);
-}
-
-static void elba_write_b(struct sdhci_host *host, u8 val, int reg)
-{
-	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
-	u32 shift = reg & GENMASK(1, 0);
-	unsigned long flags;
-	u32 byte_enables;
-
-	byte_enables = BIT(0) << shift;
-	spin_lock_irqsave(&priv->wrlock, flags);
-	writel(ELBA_BYTE_ENABLE_MASK(byte_enables), priv->ctl_addr);
-	writeb(val, host->ioaddr + reg);
-	spin_unlock_irqrestore(&priv->wrlock, flags);
-}
-
-static const struct sdhci_ops sdhci_elba_ops = {
-	.write_l = elba_write_l,
-	.write_w = elba_write_w,
-	.write_b = elba_write_b,
-	.set_clock = sdhci_set_clock,
-	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
-	.set_bus_width = sdhci_set_bus_width,
-	.reset = sdhci_reset,
-	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
-};
-
-static int elba_drv_init(struct platform_device *pdev)
-{
-	struct sdhci_host *host = platform_get_drvdata(pdev);
-	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
-	void __iomem *ioaddr;
-
-	host->mmc->caps |= MMC_CAP_1_8V_DDR | MMC_CAP_8_BIT_DATA;
-	spin_lock_init(&priv->wrlock);
-
-	/* Byte-lane control register */
-	ioaddr = devm_platform_ioremap_resource(pdev, 1);
-	if (IS_ERR(ioaddr))
-		return PTR_ERR(ioaddr);
-
-	priv->ctl_addr = ioaddr;
-	priv->priv_writel = elba_priv_writel;
-	writel(ELBA_BYTE_ENABLE_MASK(0xf), priv->ctl_addr);
-
-	return 0;
-}
-
-static const struct sdhci_ops sdhci_cdns_ops = {
-	.set_clock = sdhci_set_clock,
-	.get_timeout_clock = sdhci_cdns_get_timeout_clock,
-	.set_bus_width = sdhci_set_bus_width,
-	.reset = sdhci_reset,
-	.platform_execute_tuning = sdhci_cdns_execute_tuning,
-	.set_uhs_signaling = sdhci_cdns_set_uhs_signaling,
-};
-
-static const struct sdhci_cdns_drv_data sdhci_cdns_uniphier_drv_data = {
-	.pltfm_data = {
-		.ops = &sdhci_cdns_ops,
-		.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
-	},
-};
-
-static const struct sdhci_cdns_drv_data sdhci_elba_drv_data = {
-	.init = elba_drv_init,
-	.pltfm_data = {
-		.ops = &sdhci_elba_ops,
-	},
-};
-
-static const struct sdhci_cdns_drv_data sdhci_eyeq_drv_data = {
-	.pltfm_data = {
-		.ops = &sdhci_cdns_ops,
-		.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
-	},
-};
-
-static const struct sdhci_cdns_drv_data sdhci_cdns_drv_data = {
-	.pltfm_data = {
-		.ops = &sdhci_cdns_ops,
-	},
-};
-
-static void sdhci_cdns_hs400_enhanced_strobe(struct mmc_host *mmc,
-					     struct mmc_ios *ios)
-{
-	struct sdhci_host *host = mmc_priv(mmc);
-	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
-	u32 mode;
-
-	priv->enhanced_strobe = ios->enhanced_strobe;
-
-	mode = sdhci_cdns_get_emmc_mode(priv);
-
-	if (mode == SDHCI_CDNS_HRS06_MODE_MMC_HS400 && ios->enhanced_strobe)
-		sdhci_cdns_set_emmc_mode(priv,
-					 SDHCI_CDNS_HRS06_MODE_MMC_HS400ES);
-
-	if (mode == SDHCI_CDNS_HRS06_MODE_MMC_HS400ES && !ios->enhanced_strobe)
-		sdhci_cdns_set_emmc_mode(priv,
-					 SDHCI_CDNS_HRS06_MODE_MMC_HS400);
-}
-
-static void sdhci_cdns_mmc_hw_reset(struct mmc_host *mmc)
-{
-	struct sdhci_host *host = mmc_priv(mmc);
-	struct sdhci_cdns_priv *priv = sdhci_cdns_priv(host);
-
-	dev_dbg(mmc_dev(host->mmc), "emmc hardware reset\n");
-
-	reset_control_assert(priv->rst_hw);
-	/* For eMMC, minimum is 1us but give it 3us for good measure */
-	udelay(3);
-
-	reset_control_deassert(priv->rst_hw);
-	/* For eMMC, minimum is 200us but give it 300us for good measure */
-	usleep_range(300, 1000);
-}
-
-static int sdhci_cdns_probe(struct platform_device *pdev)
-{
-	struct sdhci_host *host;
-	const struct sdhci_cdns_drv_data *data;
-	struct sdhci_pltfm_host *pltfm_host;
-	struct sdhci_cdns_priv *priv;
-	struct clk *clk;
-	unsigned int nr_phy_params;
-	int ret;
-	struct device *dev = &pdev->dev;
-	static const u16 version = SDHCI_SPEC_400 << SDHCI_SPEC_VER_SHIFT;
-
-	clk = devm_clk_get_enabled(dev, NULL);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
-
-	data = of_device_get_match_data(dev);
-	if (!data)
-		data = &sdhci_cdns_drv_data;
-
-	nr_phy_params = sdhci_cdns_phy_param_count(dev->of_node);
-	host = sdhci_pltfm_init(pdev, &data->pltfm_data,
-				struct_size(priv, phy_params, nr_phy_params));
-	if (IS_ERR(host))
-		return PTR_ERR(host);
-
-	pltfm_host = sdhci_priv(host);
-	pltfm_host->clk = clk;
-
-	priv = sdhci_pltfm_priv(pltfm_host);
-	priv->nr_phy_params = nr_phy_params;
-	priv->hrs_addr = host->ioaddr;
-	priv->enhanced_strobe = false;
-	priv->priv_writel = cdns_writel;
-	host->ioaddr += SDHCI_CDNS_SRS_BASE;
-	host->mmc_host_ops.hs400_enhanced_strobe =
-				sdhci_cdns_hs400_enhanced_strobe;
-	if (data->init) {
-		ret = data->init(pdev);
-		if (ret)
-			return ret;
-	}
-	sdhci_enable_v4_mode(host);
-	__sdhci_read_caps(host, &version, NULL, NULL);
-
-	sdhci_get_of_property(pdev);
-
-	ret = mmc_of_parse(host->mmc);
-	if (ret)
-		return ret;
-
-	sdhci_cdns_phy_param_parse(dev->of_node, priv);
-
-	ret = sdhci_cdns_phy_init(priv);
-	if (ret)
-		return ret;
-
-	if (host->mmc->caps & MMC_CAP_HW_RESET) {
-		priv->rst_hw = devm_reset_control_get_optional_exclusive(dev, NULL);
-		if (IS_ERR(priv->rst_hw))
-			return dev_err_probe(mmc_dev(host->mmc), PTR_ERR(priv->rst_hw),
-					    "reset controller error\n");
-		if (priv->rst_hw)
-			host->mmc_host_ops.card_hw_reset = sdhci_cdns_mmc_hw_reset;
-	}
-
-	return sdhci_add_host(host);
-}
-
-static int sdhci_cdns_resume(struct device *dev)
-{
-	struct sdhci_host *host = dev_get_drvdata(dev);
-	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
-	struct sdhci_cdns_priv *priv = sdhci_pltfm_priv(pltfm_host);
-	int ret;
-
-	ret = clk_prepare_enable(pltfm_host->clk);
-	if (ret)
-		return ret;
-
-	ret = sdhci_cdns_phy_init(priv);
-	if (ret)
-		goto disable_clk;
-
-	ret = sdhci_resume_host(host);
-	if (ret)
-		goto disable_clk;
-
-	return 0;
-
-disable_clk:
-	clk_disable_unprepare(pltfm_host->clk);
-
-	return ret;
-}
-
-static DEFINE_SIMPLE_DEV_PM_OPS(sdhci_cdns_pm_ops, sdhci_pltfm_suspend, sdhci_cdns_resume);
-
-static const struct of_device_id sdhci_cdns_match[] = {
-	{
-		.compatible = "socionext,uniphier-sd4hc",
-		.data = &sdhci_cdns_uniphier_drv_data,
-	},
-	{
-		.compatible = "amd,pensando-elba-sd4hc",
-		.data = &sdhci_elba_drv_data,
-	},
-	{
-		.compatible = "mobileye,eyeq-sd4hc",
-		.data = &sdhci_eyeq_drv_data,
-	},
-	{ .compatible = "cdns,sd4hc" },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, sdhci_cdns_match);
-
-static struct platform_driver sdhci_cdns_driver = {
-	.driver = {
-		.name = "sdhci-cdns",
-		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
-		.pm = pm_sleep_ptr(&sdhci_cdns_pm_ops),
-		.of_match_table = sdhci_cdns_match,
-	},
-	.probe = sdhci_cdns_probe,
-	.remove = sdhci_pltfm_remove,
-};
-module_platform_driver(sdhci_cdns_driver);
-
-MODULE_AUTHOR("Masahiro Yamada <yamada.masahiro@socionext.com>");
-MODULE_DESCRIPTION("Cadence SD/SDIO/eMMC Host Controller Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/mmc/host/sdhci-cadence.h b/drivers/mmc/host/sdhci-cadence.h
new file mode 100644
index 00000000000000..2ac11befb53b9b
--- /dev/null
+++ b/drivers/mmc/host/sdhci-cadence.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2026 Altera Corporation
+ *   Author: Tanmay Kathpalia <tanmay.kathpalia@altera.com>
+ *
+ * Cadence SD/SDIO/eMMC Host Controller driver - common header
+ * Shared definitions and structures for the Cadence SDHCI driver.
+ * Contains private data and declarations for SD6HC-specific functions
+ * called by the main driver in sdhci-cadence-core.c.
+ */
+
+#ifndef _MMC_HOST_SDHCI_CADENCE_H
+#define _MMC_HOST_SDHCI_CADENCE_H
+
+#include <linux/compiler.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+#include "sdhci-pltfm.h"
+
+struct reset_control;
+
+/* HRS - Host Register Set (specific to Cadence) */
+#define SDHCI_CDNS_HRS04		0x10		/* PHY access: address port */
+#define SDHCI_CDNS_HRS05		0x14		/* PHY access: data port */
+
+/*
+ * The tuned val register is 6 bit-wide, but not the whole of the range is
+ * available.  The range 0-42 seems to be available (then 43 wraps around to 0)
+ * but I am not quite sure if it is official.  Use only 0 to 39 for safety.
+ */
+#define SDHCI_CDNS_MAX_TUNING_LOOP	40
+
+/**
+ * struct sdhci_cdns_priv - Cadence SDHCI private controller data
+ * @hrs_addr: Base address of Cadence Host Register Set (HRS) registers.
+ * @ctl_addr: Base address for write control registers.
+ *            Used only for "amd,pensando-elba-sd4hc" compatible controllers to enable
+ *            byte-lane writes.
+ * @wrlock: Spinlock for protecting register writes (Elba only).
+ * @enhanced_strobe: Flag indicating if Enhanced Strobe (HS400ES) is enabled.
+ * @priv_writel: Optional SoC-specific write function for register access.
+ *               Used for Elba to ensure correct byte-lane enable.
+ * @rst_hw: Hardware reset control for the eMMC card RST_n pin (SD4HC only).
+ * @phy: Opaque pointer to variant-specific PHY data.
+ *       For SD4HC: points to struct sdhci_cdns4_phy.
+ *       For SD6HC: points to struct sdhci_cdns6_phy.
+ */
+struct sdhci_cdns_priv {
+	void __iomem *hrs_addr;
+	void __iomem *ctl_addr;	/* write control */
+	spinlock_t wrlock;	/* write lock */
+	bool enhanced_strobe;
+	void (*priv_writel)(struct sdhci_cdns_priv *priv, u32 val, void __iomem *reg);
+	struct reset_control *rst_hw;
+	void *phy;
+};
+
+/**
+ * sdhci_cdns_priv - Helper to retrieve Cadence private data from sdhci_host
+ * @host: Pointer to struct sdhci_host.
+ *
+ * Return: Pointer to struct sdhci_cdns_priv.
+ */
+static inline void *sdhci_cdns_priv(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+
+	return sdhci_pltfm_priv(pltfm_host);
+}
+
+/**
+ * sdhci_cdns6_set_clock - Set SDCLK and reprogram SD6HC PHY timings.
+ * @host: Pointer to struct sdhci_host.
+ * @clock: Requested clock frequency in Hz (0 to disable).
+ */
+void sdhci_cdns6_set_clock(struct sdhci_host *host, unsigned int clock);
+
+/**
+ * sdhci_cdns6_set_uhs_signaling - Program PHY registers for a specific timing mode.
+ * @host: Pointer to struct sdhci_host.
+ * @timing: MMC timing mode (MMC_TIMING_*).
+ */
+void sdhci_cdns6_set_uhs_signaling(struct sdhci_host *host, unsigned int timing);
+
+/**
+ * sdhci_cdns6_set_tune_val - Set the PHY tuning value.
+ * @host: Pointer to struct sdhci_host.
+ * @val: Tuning value to program.
+ *
+ * Return: 0 on success, -ETIMEDOUT if DLL reset release times out.
+ */
+int sdhci_cdns6_set_tune_val(struct sdhci_host *host, unsigned int val);
+
+/**
+ * sdhci_cdns6_phy_probe - Probe and initialize Cadence SD6HC PHY parameters
+ * @pdev: Platform device pointer
+ * @priv: Pointer to Cadence private data structure
+ *
+ * Return: 0 on success or a negative error code.
+ */
+int sdhci_cdns6_phy_probe(struct platform_device *pdev, struct sdhci_cdns_priv *priv);
+
+/**
+ * sdhci_cdns6_hw_reset - Perform hardware reset of the Cadence SDHCI controller.
+ * @host: Pointer to struct sdhci_host.
+ */
+void sdhci_cdns6_hw_reset(struct sdhci_host *host);
+
+/**
+ * sdhci_cdns6_phy_init - Initialize the SD6HC PHY with current settings.
+ * @priv: Pointer to Cadence private data structure.
+ *
+ * Return: 0 on success, -ETIMEDOUT if PHY initialization times out.
+ */
+int sdhci_cdns6_phy_init(struct sdhci_cdns_priv *priv);
+
+#endif /* _MMC_HOST_SDHCI_CADENCE_H */
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 4aff965f0e2ea0..aecd425bf09511 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -1770,7 +1770,7 @@ static void sdhci_msm_handle_pwr_irq(struct sdhci_host *host, int irq)
 
 	/*
 	 * The driver has to acknowledge the interrupt, switch voltages and
-	 * report back if it succeded or not to this register. The voltage
+	 * report back if it succeeded or not to this register. The voltage
 	 * switches are handled by the sdhci core, so just report success.
 	 */
 	msm_host_writel(msm_host, irq_ack, host,
@@ -1959,6 +1959,7 @@ static int sdhci_msm_ice_init(struct sdhci_msm_host *msm_host,
 	}
 
 	mmc->caps2 |= MMC_CAP2_CRYPTO;
+	mmc->caps2 |= MMC_CAP2_CRYPTO_NO_REPROG;
 	return 0;
 }
 
@@ -2055,6 +2056,12 @@ static int sdhci_msm_ice_prepare_key(struct blk_crypto_profile *profile,
 	return qcom_ice_prepare_key(msm_host->ice, lt_key, lt_key_size, eph_key);
 }
 
+static void sdhci_msm_ice_restore(struct sdhci_host *host)
+{
+	if (host->mmc->caps2 & MMC_CAP2_CRYPTO)
+		blk_crypto_reprogram_all_keys(&host->mmc->crypto_profile);
+}
+
 static void sdhci_msm_non_cqe_ice_init(struct sdhci_host *host)
 {
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
@@ -2149,6 +2156,10 @@ sdhci_msm_ice_suspend(struct sdhci_msm_host *msm_host)
 {
 	return 0;
 }
+
+static inline void sdhci_msm_ice_restore(struct sdhci_host *host)
+{
+}
 #endif /* !CONFIG_MMC_CRYPTO */
 
 /*****************************************************************************\
@@ -2983,9 +2994,28 @@ static int sdhci_msm_runtime_resume(struct device *dev)
 	return ret;
 }
 
+static int sdhci_msm_restore(struct device *dev)
+{
+	struct sdhci_host *host = dev_get_drvdata(dev);
+	int ret;
+
+	ret = pm_runtime_force_resume(dev);
+	if (ret)
+		return ret;
+
+	sdhci_msm_ice_restore(host);
+
+	return ret;
+}
+
 static const struct dev_pm_ops sdhci_msm_pm_ops = {
-	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
 	RUNTIME_PM_OPS(sdhci_msm_runtime_suspend, sdhci_msm_runtime_resume, NULL)
+	.suspend	= pm_sleep_ptr(pm_runtime_force_suspend),
+	.resume		= pm_sleep_ptr(pm_runtime_force_resume),
+	.freeze		= pm_sleep_ptr(pm_runtime_force_suspend),
+	.restore	= pm_sleep_ptr(sdhci_msm_restore),
+	.thaw		= pm_sleep_ptr(pm_runtime_force_resume),
+	.poweroff	= pm_sleep_ptr(pm_runtime_force_suspend),
 };
 
 static struct platform_driver sdhci_msm_driver = {
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index 4ca73e7d799ead..0ee22923ad5681 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -1552,8 +1552,8 @@ sdhci_arasan_register_sdcardclk(struct sdhci_arasan_data *sdhci_arasan,
 				struct device *dev)
 {
 	struct sdhci_arasan_clk_data *clk_data = &sdhci_arasan->clk_data;
+	struct clk_init_data sdcardclk_init = {};
 	struct device_node *np = dev->of_node;
-	struct clk_init_data sdcardclk_init;
 	const char *parent_clk_name;
 	int ret;
 
@@ -1604,8 +1604,8 @@ sdhci_arasan_register_sampleclk(struct sdhci_arasan_data *sdhci_arasan,
 				struct device *dev)
 {
 	struct sdhci_arasan_clk_data *clk_data = &sdhci_arasan->clk_data;
+	struct clk_init_data sampleclk_init = {};
 	struct device_node *np = dev->of_node;
-	struct clk_init_data sampleclk_init;
 	const char *parent_clk_name;
 	int ret;
 
diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c
index 1baa2379c9c39c..09702022e70876 100644
--- a/drivers/mmc/host/sdhci-of-dwcmshc.c
+++ b/drivers/mmc/host/sdhci-of-dwcmshc.c
@@ -239,6 +239,8 @@
 					 SDHCI_TRNS_BLK_CNT_EN | \
 					 SDHCI_TRNS_DMA)
 
+#define CQHCI_SW_ERR_HALT_REQ_DISABLE	BIT(1)
+
 #define to_pltfm_data(priv, name) \
 	container_of((priv)->dwcmshc_pdata, struct name##_pltfm_data, dwcmshc_pdata)
 
@@ -702,6 +704,8 @@ static void rk35xx_sdhci_cqe_pre_enable(struct mmc_host *mmc)
 static void rk35xx_sdhci_cqe_enable(struct mmc_host *mmc)
 {
 	struct sdhci_host *host = mmc_priv(mmc);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct dwcmshc_priv *dwc_priv = sdhci_pltfm_priv(pltfm_host);
 	u32 reg;
 
 	reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
@@ -712,6 +716,15 @@ static void rk35xx_sdhci_cqe_enable(struct mmc_host *mmc)
 
 	sdhci_writew(host, DWCMSHC_SDHCI_CQE_TRNS_MODE, SDHCI_TRANSFER_MODE);
 
+	/*
+	 * Disable the halt request raised on an I/O error, otherwise a
+	 * doorbell write gets a slave response error while the halt request
+	 * is pending and hangs the CPU. This bit is reserved on older SoCs,
+	 * so the write is a no-op there.
+	 */
+	sdhci_writel(host, CQHCI_SW_ERR_HALT_REQ_DISABLE,
+		     dwc_priv->vendor_specific_area2 + CQHCI_CTL);
+
 	sdhci_cqe_enable(mmc);
 }
 
diff --git a/drivers/mmc/host/sdhci-of-esdhc.c b/drivers/mmc/host/sdhci-of-esdhc.c
index 8345e2c5a03413..1ca3562d8f7383 100644
--- a/drivers/mmc/host/sdhci-of-esdhc.c
+++ b/drivers/mmc/host/sdhci-of-esdhc.c
@@ -1381,7 +1381,7 @@ static void esdhc_init(struct platform_device *pdev, struct sdhci_host *host)
 	if (!IS_ERR(clk)) {
 		/*
 		 * esdhc->peripheral_clock would be assigned with a value
-		 * which is eSDHC base clock when use periperal clock.
+		 * which is eSDHC base clock when use peripheral clock.
 		 * For some platforms, the clock value got by common clk
 		 * API is peripheral clock while the eSDHC base clock is
 		 * 1/2 peripheral clock.
diff --git a/drivers/mmc/host/sdhci-omap.c b/drivers/mmc/host/sdhci-omap.c
index b5d7c1a80a92f2..7a198fb266280c 100644
--- a/drivers/mmc/host/sdhci-omap.c
+++ b/drivers/mmc/host/sdhci-omap.c
@@ -332,7 +332,7 @@ static int sdhci_omap_execute_tuning(struct mmc_host *mmc, u32 opcode)
 	u32 reg;
 	int i;
 
-	/* clock tuning is not needed for upto 52MHz */
+	/* clock tuning is not needed for up to 52MHz */
 	if (ios->clock <= 52000000)
 		return 0;
 
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index b121d896a80498..a9000f3bd4199f 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -2170,14 +2170,12 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot(
 
 	host->irq = pdev->irq;
 
-	ret = pcim_iomap_regions(pdev, BIT(bar), mmc_hostname(host->mmc));
-	if (ret) {
+	host->ioaddr = pcim_iomap_region(pdev, bar, mmc_hostname(host->mmc));
+	if (IS_ERR(host->ioaddr)) {
 		dev_err(&pdev->dev, "cannot request region\n");
-		return ERR_PTR(ret);
+		return ERR_CAST(host->ioaddr);
 	}
 
-	host->ioaddr = pcim_iomap_table(pdev)[bar];
-
 	if (chip->fixes && chip->fixes->probe_slot) {
 		ret = chip->fixes->probe_slot(slot);
 		if (ret)
diff --git a/drivers/mmc/host/sdhci-pxav3.c b/drivers/mmc/host/sdhci-pxav3.c
index d082c4e21aa9ed..f39764ffd4a056 100644
--- a/drivers/mmc/host/sdhci-pxav3.c
+++ b/drivers/mmc/host/sdhci-pxav3.c
@@ -123,7 +123,7 @@ static int mv_conf_mbus_windows(struct platform_device *pdev,
 static int armada_38x_quirks(struct platform_device *pdev,
 			     struct sdhci_host *host)
 {
-	struct device_node *np = pdev->dev.of_node;
+	struct device *dev = &pdev->dev;
 	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
 	struct sdhci_pxa *pxa = sdhci_pltfm_priv(pltfm_host);
 	struct resource *res;
@@ -155,7 +155,7 @@ static int armada_38x_quirks(struct platform_device *pdev,
 	 * controller has different capabilities than the ones shown
 	 * in its registers
 	 */
-	if (of_property_read_bool(np, "no-1-8-v")) {
+	if (device_property_present(dev, "no-1-8-v")) {
 		host->caps &= ~SDHCI_CAN_VDD_180;
 		host->mmc->caps &= ~MMC_CAP_1_8V_DDR;
 	} else {
@@ -284,7 +284,8 @@ static void pxav3_set_uhs_signaling(struct sdhci_host *host, unsigned int uhs)
 		    uhs == MMC_TIMING_UHS_DDR50) {
 			reg_val &= ~SDIO3_CONF_CLK_INV;
 			reg_val |= SDIO3_CONF_SD_FB_CLK;
-		} else if (uhs == MMC_TIMING_MMC_HS) {
+		} else if (uhs == MMC_TIMING_MMC_HS ||
+			   uhs == MMC_TIMING_SD_HS) {
 			reg_val &= ~SDIO3_CONF_CLK_INV;
 			reg_val &= ~SDIO3_CONF_SD_FB_CLK;
 		} else {
@@ -363,14 +364,13 @@ MODULE_DEVICE_TABLE(of, sdhci_pxav3_of_match);
 static struct sdhci_pxa_platdata *pxav3_get_mmc_pdata(struct device *dev)
 {
 	struct sdhci_pxa_platdata *pdata;
-	struct device_node *np = dev->of_node;
 	u32 clk_delay_cycles;
 
 	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
 		return NULL;
 
-	if (!of_property_read_u32(np, "mrvl,clk-delay-cycles",
+	if (!device_property_read_u32(dev, "mrvl,clk-delay-cycles",
 				  &clk_delay_cycles))
 		pdata->clk_delay_cycles = clk_delay_cycles;
 
@@ -401,7 +401,6 @@ static int sdhci_pxav3_probe(struct platform_device *pdev)
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_pxa_platdata *pdata = pdev->dev.platform_data;
 	struct device *dev = &pdev->dev;
-	struct device_node *np = pdev->dev.of_node;
 	struct sdhci_host *host = NULL;
 	struct sdhci_pxa *pxa = NULL;
 	const struct of_device_id *match;
@@ -432,7 +431,7 @@ static int sdhci_pxav3_probe(struct platform_device *pdev)
 	/* enable 1/8V DDR capable */
 	host->mmc->caps |= MMC_CAP_1_8V_DDR;
 
-	if (of_device_is_compatible(np, "marvell,armada-380-sdhci")) {
+	if (device_is_compatible(dev, "marvell,armada-380-sdhci")) {
 		ret = armada_38x_quirks(pdev, host);
 		if (ret < 0)
 			goto err_mbus_win;
diff --git a/drivers/mmc/host/sdhci-sprd.c b/drivers/mmc/host/sdhci-sprd.c
index 3584a2b314a9f9..0c8f389fe58471 100644
--- a/drivers/mmc/host/sdhci-sprd.c
+++ b/drivers/mmc/host/sdhci-sprd.c
@@ -890,6 +890,10 @@ static void sdhci_sprd_remove(struct platform_device *pdev)
 	struct sdhci_host *host = platform_get_drvdata(pdev);
 	struct sdhci_sprd_host *sprd_host = TO_SPRD_HOST(host);
 
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+
 	sdhci_remove_host(host, 0);
 
 	clk_disable_unprepare(sprd_host->clk_sdio);
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index f0fd81409a86c6..98c1fe20bfcdb1 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -409,7 +409,7 @@ struct sdhci_adma2_64_desc {
 #define SDHCI_MAX_MRQS		2
 
 /*
- * 48bit command and 136 bit response in 100KHz clock could take upto 2.48ms.
+ * 48bit command and 136 bit response in 100KHz clock could take up to 2.48ms.
  * However since the start time of the command, the time between
  * command and response, and the time between response and start of data is
  * not known, set the command transfer time to 10ms.
diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c
index 8bd9cdedafa4dc..a205c222a821d3 100644
--- a/drivers/mmc/host/sh_mmcif.c
+++ b/drivers/mmc/host/sh_mmcif.c
@@ -45,7 +45,6 @@
 #include <linux/mmc/sdio.h>
 #include <linux/mmc/slot-gpio.h>
 #include <linux/mutex.h>
-#include <linux/pagemap.h>
 #include <linux/platform_data/sh_mmcif.h>
 #include <linux/platform_device.h>
 #include <linux/pm_qos.h>
diff --git a/drivers/mmc/host/sunplus-mmc.c b/drivers/mmc/host/sunplus-mmc.c
index 63279760239cdc..e947c84eef9430 100644
--- a/drivers/mmc/host/sunplus-mmc.c
+++ b/drivers/mmc/host/sunplus-mmc.c
@@ -294,6 +294,11 @@ static void spmmc_set_bus_timing(struct spmmc_host *host, unsigned int timing)
 		value = readl(host->base + SPMMC_SD_CONFIG0_REG);
 		value |= SPMMC_SD_DDR_MODE;
 		writel(value, host->base + SPMMC_SD_CONFIG0_REG);
+		/* DDR mode needs rd_crc_dly=1 */
+		value = readl(host->base + SPMMC_SD_TIMING_CONFIG0_REG);
+		value &= ~SPMMC_SD_READ_CRC_DELAY;
+		value |= FIELD_PREP(SPMMC_SD_READ_CRC_DELAY, 1);
+		writel(value, host->base + SPMMC_SD_TIMING_CONFIG0_REG);
 	} else {
 		value = readl(host->base + SPMMC_SD_CONFIG0_REG);
 		value &= ~SPMMC_SD_DDR_MODE;
diff --git a/drivers/mmc/host/tmio_mmc.h b/drivers/mmc/host/tmio_mmc.h
index b9de03325c587b..3ab8c0d8a2bd07 100644
--- a/drivers/mmc/host/tmio_mmc.h
+++ b/drivers/mmc/host/tmio_mmc.h
@@ -18,7 +18,6 @@
 #include <linux/highmem.h>
 #include <linux/io.h>
 #include <linux/mutex.h>
-#include <linux/pagemap.h>
 #include <linux/scatterlist.h>
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c
index 775e0d9353d571..4116c4d29c6307 100644
--- a/drivers/mmc/host/tmio_mmc_core.c
+++ b/drivers/mmc/host/tmio_mmc_core.c
@@ -37,7 +37,6 @@
 #include <linux/mmc/slot-gpio.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/pagemap.h>
 #include <linux/platform_data/tmio.h>
 #include <linux/platform_device.h>
 #include <linux/pm_qos.h>
diff --git a/drivers/mmc/host/usdhi6rol0.c b/drivers/mmc/host/usdhi6rol0.c
index 3bccf800339ba2..8d9d4f362999fa 100644
--- a/drivers/mmc/host/usdhi6rol0.c
+++ b/drivers/mmc/host/usdhi6rol0.c
@@ -18,7 +18,6 @@
 #include <linux/mmc/sd.h>
 #include <linux/mmc/sdio.h>
 #include <linux/module.h>
-#include <linux/pagemap.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 #include <linux/scatterlist.h>
diff --git a/drivers/mmc/host/ushc.c b/drivers/mmc/host/ushc.c
index 7e8af5a06ac9b7..410e5bb627526d 100644
--- a/drivers/mmc/host/ushc.c
+++ b/drivers/mmc/host/ushc.c
@@ -388,10 +388,7 @@ static void ushc_enable_sdio_irq(struct mmc_host *mmc, int enable)
 {
 	struct ushc_data *ushc = mmc_priv(mmc);
 
-	if (enable)
-		set_bit(INT_EN, &ushc->flags);
-	else
-		clear_bit(INT_EN, &ushc->flags);
+	assign_bit(INT_EN, &ushc->flags, enable);
 }
 
 static void ushc_clean_up(struct ushc_data *ushc)
diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c
deleted file mode 100644
index 2dae474dcd0679..00000000000000
--- a/drivers/mmc/host/vub300.c
+++ /dev/null
@@ -1,2493 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Remote VUB300 SDIO/SDmem Host Controller Driver
- *
- * Copyright (C) 2010 Elan Digital Systems Limited
- *
- * based on USB Skeleton driver - 2.2
- *
- * Copyright (C) 2001-2004 Greg Kroah-Hartman (greg@kroah.com)
- *
- * VUB300: is a USB 2.0 client device with a single SDIO/SDmem/MMC slot
- *         Any SDIO/SDmem/MMC device plugged into the VUB300 will appear,
- *         by virtue of this driver, to have been plugged into a local
- *         SDIO host controller, similar to, say, a PCI Ricoh controller
- *         This is because this kernel device driver is both a USB 2.0
- *         client device driver AND an MMC host controller driver. Thus
- *         if there is an existing driver for the inserted SDIO/SDmem/MMC
- *         device then that driver will be used by the kernel to manage
- *         the device in exactly the same fashion as if it had been
- *         directly plugged into, say, a local pci bus Ricoh controller
- *
- * RANT: this driver was written using a display 128x48 - converting it
- *       to a line width of 80 makes it very difficult to support. In
- *       particular functions have been broken down into sub functions
- *       and the original meaningful names have been shortened into
- *       cryptic ones.
- *       The problem is that executing a fragment of code subject to
- *       two conditions means an indentation of 24, thus leaving only
- *       56 characters for a C statement. And that is quite ridiculous!
- *
- * Data types: data passed to/from the VUB300 is fixed to a number of
- *             bits and driver data fields reflect that limit by using
- *             u8, u16, u32
- */
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <linux/kref.h>
-#include <linux/uaccess.h>
-#include <linux/usb.h>
-#include <linux/mutex.h>
-#include <linux/mmc/host.h>
-#include <linux/mmc/card.h>
-#include <linux/mmc/sdio_func.h>
-#include <linux/mmc/sdio_ids.h>
-#include <linux/workqueue.h>
-#include <linux/ctype.h>
-#include <linux/firmware.h>
-#include <linux/scatterlist.h>
-
-struct host_controller_info {
-	u8 info_size;
-	u16 firmware_version;
-	u8 number_of_ports;
-} __packed;
-
-#define FIRMWARE_BLOCK_BOUNDARY 1024
-struct sd_command_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-	u8 command_type; /* Bit7 - Rd/Wr */
-	u8 command_index;
-	u8 transfer_size[4]; /* ReadSize + ReadSize */
-	u8 response_type;
-	u8 arguments[4];
-	u8 block_count[2];
-	u8 block_size[2];
-	u8 block_boundary[2];
-	u8 reserved[44]; /* to pad out to 64 bytes */
-} __packed;
-
-struct sd_irqpoll_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-	u8 command_type; /* Bit7 - Rd/Wr */
-	u8 padding[16]; /* don't ask why !! */
-	u8 poll_timeout_msb;
-	u8 poll_timeout_lsb;
-	u8 reserved[42]; /* to pad out to 64 bytes */
-} __packed;
-
-struct sd_common_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-} __packed;
-
-struct sd_response_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-	u8 command_type;
-	u8 command_index;
-	u8 command_response[];
-} __packed;
-
-struct sd_status_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-	u16 port_flags;
-	u32 sdio_clock;
-	u16 host_header_size;
-	u16 func_header_size;
-	u16 ctrl_header_size;
-} __packed;
-
-struct sd_error_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-	u8 error_code;
-} __packed;
-
-struct sd_interrupt_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-} __packed;
-
-struct offload_registers_access {
-	u8 command_byte[4];
-	u8 Respond_Byte[4];
-} __packed;
-
-#define INTERRUPT_REGISTER_ACCESSES 15
-struct sd_offloaded_interrupt {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-	struct offload_registers_access reg[INTERRUPT_REGISTER_ACCESSES];
-} __packed;
-
-struct sd_register_header {
-	u8 header_size;
-	u8 header_type;
-	u8 port_number;
-	u8 command_type;
-	u8 command_index;
-	u8 command_response[6];
-} __packed;
-
-#define PIGGYBACK_REGISTER_ACCESSES 14
-struct sd_offloaded_piggyback {
-	struct sd_register_header sdio;
-	struct offload_registers_access reg[PIGGYBACK_REGISTER_ACCESSES];
-} __packed;
-
-union sd_response {
-	struct sd_common_header common;
-	struct sd_status_header status;
-	struct sd_error_header error;
-	struct sd_interrupt_header interrupt;
-	struct sd_response_header response;
-	struct sd_offloaded_interrupt irq;
-	struct sd_offloaded_piggyback pig;
-} __packed;
-
-union sd_command {
-	struct sd_command_header head;
-	struct sd_irqpoll_header poll;
-} __packed;
-
-enum SD_RESPONSE_TYPE {
-	SDRT_UNSPECIFIED = 0,
-	SDRT_NONE,
-	SDRT_1,
-	SDRT_1B,
-	SDRT_2,
-	SDRT_3,
-	SDRT_4,
-	SDRT_5,
-	SDRT_5B,
-	SDRT_6,
-	SDRT_7,
-};
-
-#define RESPONSE_INTERRUPT			0x01
-#define RESPONSE_ERROR				0x02
-#define RESPONSE_STATUS				0x03
-#define RESPONSE_IRQ_DISABLED			0x05
-#define RESPONSE_IRQ_ENABLED			0x06
-#define RESPONSE_PIGGYBACKED			0x07
-#define RESPONSE_NO_INTERRUPT			0x08
-#define RESPONSE_PIG_DISABLED			0x09
-#define RESPONSE_PIG_ENABLED			0x0A
-#define SD_ERROR_1BIT_TIMEOUT			0x01
-#define SD_ERROR_4BIT_TIMEOUT			0x02
-#define SD_ERROR_1BIT_CRC_WRONG			0x03
-#define SD_ERROR_4BIT_CRC_WRONG			0x04
-#define SD_ERROR_1BIT_CRC_ERROR			0x05
-#define SD_ERROR_4BIT_CRC_ERROR			0x06
-#define SD_ERROR_NO_CMD_ENDBIT			0x07
-#define SD_ERROR_NO_1BIT_DATEND			0x08
-#define SD_ERROR_NO_4BIT_DATEND			0x09
-#define SD_ERROR_1BIT_UNEXPECTED_TIMEOUT	0x0A
-#define SD_ERROR_4BIT_UNEXPECTED_TIMEOUT	0x0B
-#define SD_ERROR_ILLEGAL_COMMAND		0x0C
-#define SD_ERROR_NO_DEVICE			0x0D
-#define SD_ERROR_TRANSFER_LENGTH		0x0E
-#define SD_ERROR_1BIT_DATA_TIMEOUT		0x0F
-#define SD_ERROR_4BIT_DATA_TIMEOUT		0x10
-#define SD_ERROR_ILLEGAL_STATE			0x11
-#define SD_ERROR_UNKNOWN_ERROR			0x12
-#define SD_ERROR_RESERVED_ERROR			0x13
-#define SD_ERROR_INVALID_FUNCTION		0x14
-#define SD_ERROR_OUT_OF_RANGE			0x15
-#define SD_ERROR_STAT_CMD			0x16
-#define SD_ERROR_STAT_DATA			0x17
-#define SD_ERROR_STAT_CMD_TIMEOUT		0x18
-#define SD_ERROR_SDCRDY_STUCK			0x19
-#define SD_ERROR_UNHANDLED			0x1A
-#define SD_ERROR_OVERRUN			0x1B
-#define SD_ERROR_PIO_TIMEOUT			0x1C
-
-#define FUN(c) (0x000007 & (c->arg>>28))
-#define REG(c) (0x01FFFF & (c->arg>>9))
-
-static bool limit_speed_to_24_MHz;
-module_param(limit_speed_to_24_MHz, bool, 0644);
-MODULE_PARM_DESC(limit_speed_to_24_MHz, "Limit Max SDIO Clock Speed to 24 MHz");
-
-static bool pad_input_to_usb_pkt;
-module_param(pad_input_to_usb_pkt, bool, 0644);
-MODULE_PARM_DESC(pad_input_to_usb_pkt,
-		 "Pad USB data input transfers to whole USB Packet");
-
-static bool disable_offload_processing;
-module_param(disable_offload_processing, bool, 0644);
-MODULE_PARM_DESC(disable_offload_processing, "Disable Offload Processing");
-
-static bool force_1_bit_data_xfers;
-module_param(force_1_bit_data_xfers, bool, 0644);
-MODULE_PARM_DESC(force_1_bit_data_xfers,
-		 "Force SDIO Data Transfers to 1-bit Mode");
-
-static bool force_polling_for_irqs;
-module_param(force_polling_for_irqs, bool, 0644);
-MODULE_PARM_DESC(force_polling_for_irqs, "Force Polling for SDIO interrupts");
-
-static int firmware_irqpoll_timeout = 1024;
-module_param(firmware_irqpoll_timeout, int, 0644);
-MODULE_PARM_DESC(firmware_irqpoll_timeout, "VUB300 firmware irqpoll timeout");
-
-static int force_max_req_size = 128;
-module_param(force_max_req_size, int, 0644);
-MODULE_PARM_DESC(force_max_req_size, "set max request size in kBytes");
-
-#ifdef SMSC_DEVELOPMENT_BOARD
-static int firmware_rom_wait_states = 0x04;
-#else
-static int firmware_rom_wait_states = 0x1C;
-#endif
-
-module_param(firmware_rom_wait_states, int, 0644);
-MODULE_PARM_DESC(firmware_rom_wait_states,
-		 "ROM wait states byte=RRRIIEEE (Reserved Internal External)");
-
-#define ELAN_VENDOR_ID		0x2201
-#define VUB300_VENDOR_ID	0x0424
-#define VUB300_PRODUCT_ID	0x012C
-static const struct usb_device_id vub300_table[] = {
-	{USB_DEVICE(ELAN_VENDOR_ID, VUB300_PRODUCT_ID)},
-	{USB_DEVICE(VUB300_VENDOR_ID, VUB300_PRODUCT_ID)},
-	{} /* Terminating entry */
-};
-MODULE_DEVICE_TABLE(usb, vub300_table);
-
-static struct workqueue_struct *cmndworkqueue;
-static struct workqueue_struct *pollworkqueue;
-static struct workqueue_struct *deadworkqueue;
-
-static inline int interface_to_InterfaceNumber(struct usb_interface *interface)
-{
-	if (!interface)
-		return -1;
-	if (!interface->cur_altsetting)
-		return -1;
-	return interface->cur_altsetting->desc.bInterfaceNumber;
-}
-
-struct sdio_register {
-	unsigned func_num:3;
-	unsigned sdio_reg:17;
-	unsigned activate:1;
-	unsigned prepared:1;
-	unsigned regvalue:8;
-	unsigned response:8;
-	unsigned sparebit:26;
-};
-
-struct vub300_mmc_host {
-	struct usb_device *udev;
-	struct usb_interface *interface;
-	struct kref kref;
-	struct mutex cmd_mutex;
-	struct mutex irq_mutex;
-	char vub_name[3 + (9 * 8) + 4 + 1]; /* max of 7 sdio fn's */
-	u8 cmnd_out_ep; /* EndPoint for commands */
-	u8 cmnd_res_ep; /* EndPoint for responses */
-	u8 data_out_ep; /* EndPoint for out data */
-	u8 data_inp_ep; /* EndPoint for inp data */
-	bool card_powered;
-	bool card_present;
-	bool read_only;
-	bool large_usb_packets;
-	bool app_spec; /* ApplicationSpecific */
-	bool irq_enabled; /* by the MMC CORE */
-	bool irq_disabled; /* in the firmware */
-	unsigned bus_width:4;
-	u8 total_offload_count;
-	u8 dynamic_register_count;
-	u8 resp_len;
-	u32 datasize;
-	int errors;
-	int usb_transport_fail;
-	int usb_timed_out;
-	int irqs_queued;
-	struct sdio_register sdio_register[16];
-	struct offload_interrupt_function_register {
-#define MAXREGBITS 4
-#define MAXREGS (1<<MAXREGBITS)
-#define MAXREGMASK (MAXREGS-1)
-		u8 offload_count;
-		u32 offload_point;
-		struct offload_registers_access reg[MAXREGS];
-	} fn[8];
-	u16 fbs[8]; /* Function Block Size */
-	struct mmc_command *cmd;
-	struct mmc_request *req;
-	struct mmc_data *data;
-	struct mmc_host *mmc;
-	struct urb *urb;
-	struct urb *command_out_urb;
-	struct urb *command_res_urb;
-	struct completion command_complete;
-	struct completion irqpoll_complete;
-	union sd_command cmnd;
-	union sd_response resp;
-	struct timer_list sg_transfer_timer;
-	struct usb_sg_request sg_request;
-	struct timer_list inactivity_timer;
-	struct work_struct deadwork;
-	struct work_struct cmndwork;
-	struct delayed_work pollwork;
-	struct host_controller_info hc_info;
-	struct sd_status_header system_port_status;
-	u8 padded_buffer[64];
-};
-
-#define kref_to_vub300_mmc_host(d) container_of(d, struct vub300_mmc_host, kref)
-#define SET_TRANSFER_PSEUDOCODE		21
-#define SET_INTERRUPT_PSEUDOCODE	20
-#define SET_FAILURE_MODE		18
-#define SET_ROM_WAIT_STATES		16
-#define SET_IRQ_ENABLE			13
-#define SET_CLOCK_SPEED			11
-#define SET_FUNCTION_BLOCK_SIZE		9
-#define SET_SD_DATA_MODE		6
-#define SET_SD_POWER			4
-#define ENTER_DFU_MODE			3
-#define GET_HC_INF0			1
-#define GET_SYSTEM_PORT_STATUS		0
-
-static void vub300_delete(struct kref *kref)
-{				/* kref callback - softirq */
-	struct vub300_mmc_host *vub300 = kref_to_vub300_mmc_host(kref);
-	struct mmc_host *mmc = vub300->mmc;
-
-	usb_free_urb(vub300->command_out_urb);
-	vub300->command_out_urb = NULL;
-	usb_free_urb(vub300->command_res_urb);
-	vub300->command_res_urb = NULL;
-	usb_put_dev(vub300->udev);
-	mmc_free_host(mmc);
-	/*
-	 * and hence also frees vub300
-	 * which is contained at the end of struct mmc
-	 */
-}
-
-static void vub300_queue_cmnd_work(struct vub300_mmc_host *vub300)
-{
-	kref_get(&vub300->kref);
-	if (queue_work(cmndworkqueue, &vub300->cmndwork)) {
-		/*
-		 * then the cmndworkqueue was not previously
-		 * running and the above get ref is obvious
-		 * required and will be put when the thread
-		 * terminates by a specific call
-		 */
-	} else {
-		/*
-		 * the cmndworkqueue was already running from
-		 * a previous invocation and thus to keep the
-		 * kref counts correct we must undo the get
-		 */
-		kref_put(&vub300->kref, vub300_delete);
-	}
-}
-
-static void vub300_queue_poll_work(struct vub300_mmc_host *vub300, int delay)
-{
-	kref_get(&vub300->kref);
-	if (queue_delayed_work(pollworkqueue, &vub300->pollwork, delay)) {
-		/*
-		 * then the pollworkqueue was not previously
-		 * running and the above get ref is obvious
-		 * required and will be put when the thread
-		 * terminates by a specific call
-		 */
-	} else {
-		/*
-		 * the pollworkqueue was already running from
-		 * a previous invocation and thus to keep the
-		 * kref counts correct we must undo the get
-		 */
-		kref_put(&vub300->kref, vub300_delete);
-	}
-}
-
-static void vub300_queue_dead_work(struct vub300_mmc_host *vub300)
-{
-	kref_get(&vub300->kref);
-	if (queue_work(deadworkqueue, &vub300->deadwork)) {
-		/*
-		 * then the deadworkqueue was not previously
-		 * running and the above get ref is obvious
-		 * required and will be put when the thread
-		 * terminates by a specific call
-		 */
-	} else {
-		/*
-		 * the deadworkqueue was already running from
-		 * a previous invocation and thus to keep the
-		 * kref counts correct we must undo the get
-		 */
-		kref_put(&vub300->kref, vub300_delete);
-	}
-}
-
-static void irqpoll_res_completed(struct urb *urb)
-{				/* urb completion handler - hardirq */
-	struct vub300_mmc_host *vub300 = (struct vub300_mmc_host *)urb->context;
-	if (urb->status)
-		vub300->usb_transport_fail = urb->status;
-	complete(&vub300->irqpoll_complete);
-}
-
-static void irqpoll_out_completed(struct urb *urb)
-{				/* urb completion handler - hardirq */
-	struct vub300_mmc_host *vub300 = (struct vub300_mmc_host *)urb->context;
-	if (urb->status) {
-		vub300->usb_transport_fail = urb->status;
-		complete(&vub300->irqpoll_complete);
-		return;
-	} else {
-		int ret;
-		unsigned int pipe =
-			usb_rcvbulkpipe(vub300->udev, vub300->cmnd_res_ep);
-		usb_fill_bulk_urb(vub300->command_res_urb, vub300->udev, pipe,
-				  &vub300->resp, sizeof(vub300->resp),
-				  irqpoll_res_completed, vub300);
-		vub300->command_res_urb->actual_length = 0;
-		ret = usb_submit_urb(vub300->command_res_urb, GFP_ATOMIC);
-		if (ret) {
-			vub300->usb_transport_fail = ret;
-			complete(&vub300->irqpoll_complete);
-		}
-		return;
-	}
-}
-
-static void send_irqpoll(struct vub300_mmc_host *vub300)
-{
-	/* cmd_mutex is held by vub300_pollwork_thread */
-	int retval;
-	int timeout = 0xFFFF & (0x0001FFFF - firmware_irqpoll_timeout);
-	vub300->cmnd.poll.header_size = 22;
-	vub300->cmnd.poll.header_type = 1;
-	vub300->cmnd.poll.port_number = 0;
-	vub300->cmnd.poll.command_type = 2;
-	vub300->cmnd.poll.poll_timeout_lsb = 0xFF & (unsigned)timeout;
-	vub300->cmnd.poll.poll_timeout_msb = 0xFF & (unsigned)(timeout >> 8);
-	usb_fill_bulk_urb(vub300->command_out_urb, vub300->udev,
-			  usb_sndbulkpipe(vub300->udev, vub300->cmnd_out_ep)
-			  , &vub300->cmnd, sizeof(vub300->cmnd)
-			  , irqpoll_out_completed, vub300);
-	retval = usb_submit_urb(vub300->command_out_urb, GFP_KERNEL);
-	if (0 > retval) {
-		vub300->usb_transport_fail = retval;
-		vub300_queue_poll_work(vub300, 1);
-		complete(&vub300->irqpoll_complete);
-		return;
-	} else {
-		return;
-	}
-}
-
-static void new_system_port_status(struct vub300_mmc_host *vub300)
-{
-	int old_card_present = vub300->card_present;
-	int new_card_present =
-		(0x0001 & vub300->system_port_status.port_flags) ? 1 : 0;
-	vub300->read_only =
-		(0x0010 & vub300->system_port_status.port_flags) ? 1 : 0;
-	if (new_card_present && !old_card_present) {
-		dev_info(&vub300->udev->dev, "card just inserted\n");
-		vub300->card_present = 1;
-		vub300->bus_width = 0;
-		if (disable_offload_processing)
-			strscpy(vub300->vub_name, "EMPTY Processing Disabled",
-				sizeof(vub300->vub_name));
-		else
-			vub300->vub_name[0] = 0;
-		mmc_detect_change(vub300->mmc, 1);
-	} else if (!new_card_present && old_card_present) {
-		dev_info(&vub300->udev->dev, "card just ejected\n");
-		vub300->card_present = 0;
-		mmc_detect_change(vub300->mmc, 0);
-	} else {
-		/* no change */
-	}
-}
-
-static void __add_offloaded_reg_to_fifo(struct vub300_mmc_host *vub300,
-					struct offload_registers_access
-					*register_access, u8 func)
-{
-	u8 r = vub300->fn[func].offload_point + vub300->fn[func].offload_count;
-	memcpy(&vub300->fn[func].reg[MAXREGMASK & r], register_access,
-	       sizeof(struct offload_registers_access));
-	vub300->fn[func].offload_count += 1;
-	vub300->total_offload_count += 1;
-}
-
-static void add_offloaded_reg(struct vub300_mmc_host *vub300,
-			      struct offload_registers_access *register_access)
-{
-	u32 Register = ((0x03 & register_access->command_byte[0]) << 15)
-			| ((0xFF & register_access->command_byte[1]) << 7)
-			| ((0xFE & register_access->command_byte[2]) >> 1);
-	u8 func = ((0x70 & register_access->command_byte[0]) >> 4);
-	u8 regs = vub300->dynamic_register_count;
-	u8 i = 0;
-	while (0 < regs-- && 1 == vub300->sdio_register[i].activate) {
-		if (vub300->sdio_register[i].func_num == func &&
-		    vub300->sdio_register[i].sdio_reg == Register) {
-			if (vub300->sdio_register[i].prepared == 0)
-				vub300->sdio_register[i].prepared = 1;
-			vub300->sdio_register[i].response =
-				register_access->Respond_Byte[2];
-			vub300->sdio_register[i].regvalue =
-				register_access->Respond_Byte[3];
-			return;
-		} else {
-			i += 1;
-			continue;
-		}
-	}
-	__add_offloaded_reg_to_fifo(vub300, register_access, func);
-}
-
-static void check_vub300_port_status(struct vub300_mmc_host *vub300)
-{
-	/*
-	 * cmd_mutex is held by vub300_pollwork_thread,
-	 * vub300_deadwork_thread or vub300_cmndwork_thread
-	 */
-	int retval;
-	retval =
-		usb_control_msg(vub300->udev, usb_rcvctrlpipe(vub300->udev, 0),
-				GET_SYSTEM_PORT_STATUS,
-				USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				0x0000, 0x0000, &vub300->system_port_status,
-				sizeof(vub300->system_port_status), 1000);
-	if (sizeof(vub300->system_port_status) == retval)
-		new_system_port_status(vub300);
-}
-
-static void __vub300_irqpoll_response(struct vub300_mmc_host *vub300)
-{
-	/* cmd_mutex is held by vub300_pollwork_thread */
-	if (vub300->command_res_urb->actual_length == 0)
-		return;
-
-	switch (vub300->resp.common.header_type) {
-	case RESPONSE_INTERRUPT:
-		mutex_lock(&vub300->irq_mutex);
-		if (vub300->irq_enabled)
-			mmc_signal_sdio_irq(vub300->mmc);
-		else
-			vub300->irqs_queued += 1;
-		vub300->irq_disabled = 1;
-		mutex_unlock(&vub300->irq_mutex);
-		break;
-	case RESPONSE_ERROR:
-		if (vub300->resp.error.error_code == SD_ERROR_NO_DEVICE)
-			check_vub300_port_status(vub300);
-		break;
-	case RESPONSE_STATUS:
-		vub300->system_port_status = vub300->resp.status;
-		new_system_port_status(vub300);
-		if (!vub300->card_present)
-			vub300_queue_poll_work(vub300, HZ / 5);
-		break;
-	case RESPONSE_IRQ_DISABLED:
-	{
-		int offloaded_data_length = vub300->resp.common.header_size - 3;
-		int register_count = offloaded_data_length >> 3;
-		int ri = 0;
-		while (register_count--) {
-			add_offloaded_reg(vub300, &vub300->resp.irq.reg[ri]);
-			ri += 1;
-		}
-		mutex_lock(&vub300->irq_mutex);
-		if (vub300->irq_enabled)
-			mmc_signal_sdio_irq(vub300->mmc);
-		else
-			vub300->irqs_queued += 1;
-		vub300->irq_disabled = 1;
-		mutex_unlock(&vub300->irq_mutex);
-		break;
-	}
-	case RESPONSE_IRQ_ENABLED:
-	{
-		int offloaded_data_length = vub300->resp.common.header_size - 3;
-		int register_count = offloaded_data_length >> 3;
-		int ri = 0;
-		while (register_count--) {
-			add_offloaded_reg(vub300, &vub300->resp.irq.reg[ri]);
-			ri += 1;
-		}
-		mutex_lock(&vub300->irq_mutex);
-		if (vub300->irq_enabled)
-			mmc_signal_sdio_irq(vub300->mmc);
-		else
-			vub300->irqs_queued += 1;
-		vub300->irq_disabled = 0;
-		mutex_unlock(&vub300->irq_mutex);
-		break;
-	}
-	case RESPONSE_NO_INTERRUPT:
-		vub300_queue_poll_work(vub300, 1);
-		break;
-	default:
-		break;
-	}
-}
-
-static void __do_poll(struct vub300_mmc_host *vub300)
-{
-	/* cmd_mutex is held by vub300_pollwork_thread */
-	unsigned long commretval;
-	mod_timer(&vub300->inactivity_timer, jiffies + HZ);
-	init_completion(&vub300->irqpoll_complete);
-	send_irqpoll(vub300);
-	commretval = wait_for_completion_timeout(&vub300->irqpoll_complete,
-						 msecs_to_jiffies(500));
-	if (vub300->usb_transport_fail) {
-		/* no need to do anything */
-	} else if (commretval == 0) {
-		vub300->usb_timed_out = 1;
-		usb_kill_urb(vub300->command_out_urb);
-		usb_kill_urb(vub300->command_res_urb);
-	} else { /* commretval > 0 */
-		__vub300_irqpoll_response(vub300);
-	}
-}
-
-/* this thread runs only when the driver
- * is trying to poll the device for an IRQ
- */
-static void vub300_pollwork_thread(struct work_struct *work)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300 = container_of(work,
-			      struct vub300_mmc_host, pollwork.work);
-	if (!vub300->interface) {
-		kref_put(&vub300->kref, vub300_delete);
-		return;
-	}
-	mutex_lock(&vub300->cmd_mutex);
-	if (vub300->cmd) {
-		vub300_queue_poll_work(vub300, 1);
-	} else if (!vub300->card_present) {
-		/* no need to do anything */
-	} else { /* vub300->card_present */
-		mutex_lock(&vub300->irq_mutex);
-		if (!vub300->irq_enabled) {
-			mutex_unlock(&vub300->irq_mutex);
-		} else if (vub300->irqs_queued) {
-			vub300->irqs_queued -= 1;
-			mmc_signal_sdio_irq(vub300->mmc);
-			mod_timer(&vub300->inactivity_timer, jiffies + HZ);
-			mutex_unlock(&vub300->irq_mutex);
-		} else { /* NOT vub300->irqs_queued */
-			mutex_unlock(&vub300->irq_mutex);
-			__do_poll(vub300);
-		}
-	}
-	mutex_unlock(&vub300->cmd_mutex);
-	kref_put(&vub300->kref, vub300_delete);
-}
-
-static void vub300_deadwork_thread(struct work_struct *work)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300 =
-		container_of(work, struct vub300_mmc_host, deadwork);
-	if (!vub300->interface) {
-		kref_put(&vub300->kref, vub300_delete);
-		return;
-	}
-	mutex_lock(&vub300->cmd_mutex);
-	if (vub300->cmd) {
-		/*
-		 * a command got in as the inactivity
-		 * timer expired - so we just let the
-		 * processing of the command show if
-		 * the device is dead
-		 */
-	} else if (vub300->card_present) {
-		check_vub300_port_status(vub300);
-	} else if (vub300->mmc && vub300->mmc->card) {
-		/*
-		 * the MMC core must not have responded
-		 * to the previous indication - lets
-		 * hope that it eventually does so we
-		 * will just ignore this for now
-		 */
-	} else {
-		check_vub300_port_status(vub300);
-	}
-	mod_timer(&vub300->inactivity_timer, jiffies + HZ);
-	mutex_unlock(&vub300->cmd_mutex);
-	kref_put(&vub300->kref, vub300_delete);
-}
-
-static void vub300_inactivity_timer_expired(struct timer_list *t)
-{				/* softirq */
-	struct vub300_mmc_host *vub300 = timer_container_of(vub300, t,
-							    inactivity_timer);
-	if (!vub300->interface) {
-		kref_put(&vub300->kref, vub300_delete);
-	} else if (vub300->cmd) {
-		mod_timer(&vub300->inactivity_timer, jiffies + HZ);
-	} else {
-		vub300_queue_dead_work(vub300);
-		mod_timer(&vub300->inactivity_timer, jiffies + HZ);
-	}
-}
-
-static int vub300_response_error(u8 error_code)
-{
-	switch (error_code) {
-	case SD_ERROR_PIO_TIMEOUT:
-	case SD_ERROR_1BIT_TIMEOUT:
-	case SD_ERROR_4BIT_TIMEOUT:
-		return -ETIMEDOUT;
-	case SD_ERROR_STAT_DATA:
-	case SD_ERROR_OVERRUN:
-	case SD_ERROR_STAT_CMD:
-	case SD_ERROR_STAT_CMD_TIMEOUT:
-	case SD_ERROR_SDCRDY_STUCK:
-	case SD_ERROR_UNHANDLED:
-	case SD_ERROR_1BIT_CRC_WRONG:
-	case SD_ERROR_4BIT_CRC_WRONG:
-	case SD_ERROR_1BIT_CRC_ERROR:
-	case SD_ERROR_4BIT_CRC_ERROR:
-	case SD_ERROR_NO_CMD_ENDBIT:
-	case SD_ERROR_NO_1BIT_DATEND:
-	case SD_ERROR_NO_4BIT_DATEND:
-	case SD_ERROR_1BIT_DATA_TIMEOUT:
-	case SD_ERROR_4BIT_DATA_TIMEOUT:
-	case SD_ERROR_1BIT_UNEXPECTED_TIMEOUT:
-	case SD_ERROR_4BIT_UNEXPECTED_TIMEOUT:
-		return -EILSEQ;
-	case 33:
-		return -EILSEQ;
-	case SD_ERROR_ILLEGAL_COMMAND:
-		return -EINVAL;
-	case SD_ERROR_NO_DEVICE:
-		return -ENOMEDIUM;
-	default:
-		return -ENODEV;
-	}
-}
-
-static void command_res_completed(struct urb *urb)
-{				/* urb completion handler - hardirq */
-	struct vub300_mmc_host *vub300 = (struct vub300_mmc_host *)urb->context;
-	if (urb->status) {
-		/* we have to let the initiator handle the error */
-	} else if (vub300->command_res_urb->actual_length == 0) {
-		/*
-		 * we have seen this happen once or twice and
-		 * we suspect a buggy USB host controller
-		 */
-	} else if (!vub300->data) {
-		/* this means that the command (typically CMD52) succeeded */
-	} else if (vub300->resp.common.header_type != 0x02) {
-		/*
-		 * this is an error response from the VUB300 chip
-		 * and we let the initiator handle it
-		 */
-	} else if (vub300->urb) {
-		vub300->cmd->error =
-			vub300_response_error(vub300->resp.error.error_code);
-		usb_unlink_urb(vub300->urb);
-	} else {
-		vub300->cmd->error =
-			vub300_response_error(vub300->resp.error.error_code);
-		usb_sg_cancel(&vub300->sg_request);
-	}
-	complete(&vub300->command_complete);	/* got_response_in */
-}
-
-static void command_out_completed(struct urb *urb)
-{				/* urb completion handler - hardirq */
-	struct vub300_mmc_host *vub300 = (struct vub300_mmc_host *)urb->context;
-	if (urb->status) {
-		complete(&vub300->command_complete);
-	} else {
-		int ret;
-		unsigned int pipe =
-			usb_rcvbulkpipe(vub300->udev, vub300->cmnd_res_ep);
-		usb_fill_bulk_urb(vub300->command_res_urb, vub300->udev, pipe,
-				  &vub300->resp, sizeof(vub300->resp),
-				  command_res_completed, vub300);
-		vub300->command_res_urb->actual_length = 0;
-		ret = usb_submit_urb(vub300->command_res_urb, GFP_ATOMIC);
-		if (ret == 0) {
-			/*
-			 * the urb completion handler will call
-			 * our completion handler
-			 */
-		} else {
-			/*
-			 * and thus we only call it directly
-			 * when it will not be called
-			 */
-			complete(&vub300->command_complete);
-		}
-	}
-}
-
-/*
- * the STUFF bits are masked out for the comparisons
- */
-static void snoop_block_size_and_bus_width(struct vub300_mmc_host *vub300,
-					   u32 cmd_arg)
-{
-	if ((0xFBFFFE00 & cmd_arg) == 0x80022200)
-		vub300->fbs[1] = (cmd_arg << 8) | (0x00FF & vub300->fbs[1]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x80022000)
-		vub300->fbs[1] = (0xFF & cmd_arg) | (0xFF00 & vub300->fbs[1]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x80042200)
-		vub300->fbs[2] = (cmd_arg << 8) | (0x00FF & vub300->fbs[2]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x80042000)
-		vub300->fbs[2] = (0xFF & cmd_arg) | (0xFF00 & vub300->fbs[2]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x80062200)
-		vub300->fbs[3] = (cmd_arg << 8) | (0x00FF & vub300->fbs[3]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x80062000)
-		vub300->fbs[3] = (0xFF & cmd_arg) | (0xFF00 & vub300->fbs[3]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x80082200)
-		vub300->fbs[4] = (cmd_arg << 8) | (0x00FF & vub300->fbs[4]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x80082000)
-		vub300->fbs[4] = (0xFF & cmd_arg) | (0xFF00 & vub300->fbs[4]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x800A2200)
-		vub300->fbs[5] = (cmd_arg << 8) | (0x00FF & vub300->fbs[5]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x800A2000)
-		vub300->fbs[5] = (0xFF & cmd_arg) | (0xFF00 & vub300->fbs[5]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x800C2200)
-		vub300->fbs[6] = (cmd_arg << 8) | (0x00FF & vub300->fbs[6]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x800C2000)
-		vub300->fbs[6] = (0xFF & cmd_arg) | (0xFF00 & vub300->fbs[6]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x800E2200)
-		vub300->fbs[7] = (cmd_arg << 8) | (0x00FF & vub300->fbs[7]);
-	else if ((0xFBFFFE00 & cmd_arg) == 0x800E2000)
-		vub300->fbs[7] = (0xFF & cmd_arg) | (0xFF00 & vub300->fbs[7]);
-	else if ((0xFBFFFE03 & cmd_arg) == 0x80000E00)
-		vub300->bus_width = 1;
-	else if ((0xFBFFFE03 & cmd_arg) == 0x80000E02)
-		vub300->bus_width = 4;
-}
-
-static void send_command(struct vub300_mmc_host *vub300)
-{
-	/* cmd_mutex is held by vub300_cmndwork_thread */
-	struct mmc_command *cmd = vub300->cmd;
-	struct mmc_data *data = vub300->data;
-	int retval;
-	int i;
-	u8 response_type;
-	if (vub300->app_spec) {
-		switch (cmd->opcode) {
-		case 6:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			if (0x00000000 == (0x00000003 & cmd->arg))
-				vub300->bus_width = 1;
-			else if (0x00000002 == (0x00000003 & cmd->arg))
-				vub300->bus_width = 4;
-			else
-				dev_err(&vub300->udev->dev,
-					"unexpected ACMD6 bus_width=%d\n",
-					0x00000003 & cmd->arg);
-			break;
-		case 13:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 22:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 23:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 41:
-			response_type = SDRT_3;
-			vub300->resp_len = 6;
-			break;
-		case 42:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 51:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 55:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		default:
-			vub300->resp_len = 0;
-			cmd->error = -EINVAL;
-			complete(&vub300->command_complete);
-			return;
-		}
-		vub300->app_spec = 0;
-	} else {
-		switch (cmd->opcode) {
-		case 0:
-			response_type = SDRT_NONE;
-			vub300->resp_len = 0;
-			break;
-		case 1:
-			response_type = SDRT_3;
-			vub300->resp_len = 6;
-			break;
-		case 2:
-			response_type = SDRT_2;
-			vub300->resp_len = 17;
-			break;
-		case 3:
-			response_type = SDRT_6;
-			vub300->resp_len = 6;
-			break;
-		case 4:
-			response_type = SDRT_NONE;
-			vub300->resp_len = 0;
-			break;
-		case 5:
-			response_type = SDRT_4;
-			vub300->resp_len = 6;
-			break;
-		case 6:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 7:
-			response_type = SDRT_1B;
-			vub300->resp_len = 6;
-			break;
-		case 8:
-			response_type = SDRT_7;
-			vub300->resp_len = 6;
-			break;
-		case 9:
-			response_type = SDRT_2;
-			vub300->resp_len = 17;
-			break;
-		case 10:
-			response_type = SDRT_2;
-			vub300->resp_len = 17;
-			break;
-		case 12:
-			response_type = SDRT_1B;
-			vub300->resp_len = 6;
-			break;
-		case 13:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 15:
-			response_type = SDRT_NONE;
-			vub300->resp_len = 0;
-			break;
-		case 16:
-			for (i = 0; i < ARRAY_SIZE(vub300->fbs); i++)
-				vub300->fbs[i] = 0xFFFF & cmd->arg;
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 17:
-		case 18:
-		case 24:
-		case 25:
-		case 27:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 28:
-		case 29:
-			response_type = SDRT_1B;
-			vub300->resp_len = 6;
-			break;
-		case 30:
-		case 32:
-		case 33:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 38:
-			response_type = SDRT_1B;
-			vub300->resp_len = 6;
-			break;
-		case 42:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		case 52:
-			response_type = SDRT_5;
-			vub300->resp_len = 6;
-			snoop_block_size_and_bus_width(vub300, cmd->arg);
-			break;
-		case 53:
-			response_type = SDRT_5;
-			vub300->resp_len = 6;
-			break;
-		case 55:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			vub300->app_spec = 1;
-			break;
-		case 56:
-			response_type = SDRT_1;
-			vub300->resp_len = 6;
-			break;
-		default:
-			vub300->resp_len = 0;
-			cmd->error = -EINVAL;
-			complete(&vub300->command_complete);
-			return;
-		}
-	}
-	/*
-	 * it is a shame that we can not use "sizeof(struct sd_command_header)"
-	 * this is because the packet _must_ be padded to 64 bytes
-	 */
-	vub300->cmnd.head.header_size = 20;
-	vub300->cmnd.head.header_type = 0x00;
-	vub300->cmnd.head.port_number = 0; /* "0" means port 1 */
-	vub300->cmnd.head.command_type = 0x00; /* standard read command */
-	vub300->cmnd.head.response_type = response_type;
-	vub300->cmnd.head.command_index = cmd->opcode;
-	vub300->cmnd.head.arguments[0] = cmd->arg >> 24;
-	vub300->cmnd.head.arguments[1] = cmd->arg >> 16;
-	vub300->cmnd.head.arguments[2] = cmd->arg >> 8;
-	vub300->cmnd.head.arguments[3] = cmd->arg >> 0;
-	if (cmd->opcode == 52) {
-		int fn = 0x7 & (cmd->arg >> 28);
-		vub300->cmnd.head.block_count[0] = 0;
-		vub300->cmnd.head.block_count[1] = 0;
-		vub300->cmnd.head.block_size[0] = (vub300->fbs[fn] >> 8) & 0xFF;
-		vub300->cmnd.head.block_size[1] = (vub300->fbs[fn] >> 0) & 0xFF;
-		vub300->cmnd.head.command_type = 0x00;
-		vub300->cmnd.head.transfer_size[0] = 0;
-		vub300->cmnd.head.transfer_size[1] = 0;
-		vub300->cmnd.head.transfer_size[2] = 0;
-		vub300->cmnd.head.transfer_size[3] = 0;
-	} else if (!data) {
-		vub300->cmnd.head.block_count[0] = 0;
-		vub300->cmnd.head.block_count[1] = 0;
-		vub300->cmnd.head.block_size[0] = (vub300->fbs[0] >> 8) & 0xFF;
-		vub300->cmnd.head.block_size[1] = (vub300->fbs[0] >> 0) & 0xFF;
-		vub300->cmnd.head.command_type = 0x00;
-		vub300->cmnd.head.transfer_size[0] = 0;
-		vub300->cmnd.head.transfer_size[1] = 0;
-		vub300->cmnd.head.transfer_size[2] = 0;
-		vub300->cmnd.head.transfer_size[3] = 0;
-	} else if (cmd->opcode == 53) {
-		int fn = 0x7 & (cmd->arg >> 28);
-		if (0x08 & vub300->cmnd.head.arguments[0]) { /* BLOCK MODE */
-			vub300->cmnd.head.block_count[0] =
-				(data->blocks >> 8) & 0xFF;
-			vub300->cmnd.head.block_count[1] =
-				(data->blocks >> 0) & 0xFF;
-			vub300->cmnd.head.block_size[0] =
-				(data->blksz >> 8) & 0xFF;
-			vub300->cmnd.head.block_size[1] =
-				(data->blksz >> 0) & 0xFF;
-		} else {	/* BYTE MODE */
-			vub300->cmnd.head.block_count[0] = 0;
-			vub300->cmnd.head.block_count[1] = 0;
-			vub300->cmnd.head.block_size[0] =
-				(vub300->datasize >> 8) & 0xFF;
-			vub300->cmnd.head.block_size[1] =
-				(vub300->datasize >> 0) & 0xFF;
-		}
-		vub300->cmnd.head.command_type =
-			(MMC_DATA_READ & data->flags) ? 0x00 : 0x80;
-		vub300->cmnd.head.transfer_size[0] =
-			(vub300->datasize >> 24) & 0xFF;
-		vub300->cmnd.head.transfer_size[1] =
-			(vub300->datasize >> 16) & 0xFF;
-		vub300->cmnd.head.transfer_size[2] =
-			(vub300->datasize >> 8) & 0xFF;
-		vub300->cmnd.head.transfer_size[3] =
-			(vub300->datasize >> 0) & 0xFF;
-		if (vub300->datasize < vub300->fbs[fn]) {
-			vub300->cmnd.head.block_count[0] = 0;
-			vub300->cmnd.head.block_count[1] = 0;
-		}
-	} else {
-		vub300->cmnd.head.block_count[0] = (data->blocks >> 8) & 0xFF;
-		vub300->cmnd.head.block_count[1] = (data->blocks >> 0) & 0xFF;
-		vub300->cmnd.head.block_size[0] = (data->blksz >> 8) & 0xFF;
-		vub300->cmnd.head.block_size[1] = (data->blksz >> 0) & 0xFF;
-		vub300->cmnd.head.command_type =
-			(MMC_DATA_READ & data->flags) ? 0x00 : 0x80;
-		vub300->cmnd.head.transfer_size[0] =
-			(vub300->datasize >> 24) & 0xFF;
-		vub300->cmnd.head.transfer_size[1] =
-			(vub300->datasize >> 16) & 0xFF;
-		vub300->cmnd.head.transfer_size[2] =
-			(vub300->datasize >> 8) & 0xFF;
-		vub300->cmnd.head.transfer_size[3] =
-			(vub300->datasize >> 0) & 0xFF;
-		if (vub300->datasize < vub300->fbs[0]) {
-			vub300->cmnd.head.block_count[0] = 0;
-			vub300->cmnd.head.block_count[1] = 0;
-		}
-	}
-	if (vub300->cmnd.head.block_size[0] || vub300->cmnd.head.block_size[1]) {
-		u16 block_size = vub300->cmnd.head.block_size[1] |
-			(vub300->cmnd.head.block_size[0] << 8);
-		u16 block_boundary = FIRMWARE_BLOCK_BOUNDARY -
-			(FIRMWARE_BLOCK_BOUNDARY % block_size);
-		vub300->cmnd.head.block_boundary[0] =
-			(block_boundary >> 8) & 0xFF;
-		vub300->cmnd.head.block_boundary[1] =
-			(block_boundary >> 0) & 0xFF;
-	} else {
-		vub300->cmnd.head.block_boundary[0] = 0;
-		vub300->cmnd.head.block_boundary[1] = 0;
-	}
-	usb_fill_bulk_urb(vub300->command_out_urb, vub300->udev,
-			  usb_sndbulkpipe(vub300->udev, vub300->cmnd_out_ep),
-			  &vub300->cmnd, sizeof(vub300->cmnd),
-			  command_out_completed, vub300);
-	retval = usb_submit_urb(vub300->command_out_urb, GFP_KERNEL);
-	if (retval < 0) {
-		cmd->error = retval;
-		complete(&vub300->command_complete);
-		return;
-	} else {
-		return;
-	}
-}
-
-/*
- * timer callback runs in atomic mode
- *       so it cannot call usb_kill_urb()
- */
-static void vub300_sg_timed_out(struct timer_list *t)
-{
-	struct vub300_mmc_host *vub300 = timer_container_of(vub300, t,
-							    sg_transfer_timer);
-	vub300->usb_timed_out = 1;
-	usb_sg_cancel(&vub300->sg_request);
-	usb_unlink_urb(vub300->command_out_urb);
-	usb_unlink_urb(vub300->command_res_urb);
-}
-
-static u16 roundup_to_multiple_of_64(u16 number)
-{
-	return 0xFFC0 & (0x3F + number);
-}
-
-/*
- * this is a separate function to solve the 80 column width restriction
- */
-static void __download_offload_pseudocode(struct vub300_mmc_host *vub300,
-					  const struct firmware *fw)
-{
-	u8 register_count = 0;
-	u16 ts = 0;
-	u16 interrupt_size = 0;
-	const u8 *data = fw->data;
-	int size = fw->size;
-	u8 c;
-	dev_info(&vub300->udev->dev, "using %s for SDIO offload processing\n",
-		 vub300->vub_name);
-	do {
-		c = *data++;
-	} while (size-- && c); /* skip comment */
-	dev_info(&vub300->udev->dev, "using offload firmware %s %s\n", fw->data,
-		 vub300->vub_name);
-	if (size < 4) {
-		dev_err(&vub300->udev->dev,
-			"corrupt offload pseudocode in firmware %s\n",
-			vub300->vub_name);
-		strscpy(vub300->vub_name, "corrupt offload pseudocode",
-			sizeof(vub300->vub_name));
-		return;
-	}
-	interrupt_size += *data++;
-	size -= 1;
-	interrupt_size <<= 8;
-	interrupt_size += *data++;
-	size -= 1;
-	if (interrupt_size < size) {
-		u16 xfer_length = roundup_to_multiple_of_64(interrupt_size);
-		u8 *xfer_buffer = kmalloc(xfer_length, GFP_KERNEL);
-		if (xfer_buffer) {
-			int retval;
-			memcpy(xfer_buffer, data, interrupt_size);
-			memset(xfer_buffer + interrupt_size, 0,
-			       xfer_length - interrupt_size);
-			size -= interrupt_size;
-			data += interrupt_size;
-			retval =
-				usb_control_msg(vub300->udev,
-						usb_sndctrlpipe(vub300->udev, 0),
-						SET_INTERRUPT_PSEUDOCODE,
-						USB_DIR_OUT | USB_TYPE_VENDOR |
-						USB_RECIP_DEVICE, 0x0000, 0x0000,
-						xfer_buffer, xfer_length, 1000);
-			kfree(xfer_buffer);
-			if (retval < 0)
-				goto copy_error_message;
-		} else {
-			dev_err(&vub300->udev->dev,
-				"not enough memory for xfer buffer to send"
-				" INTERRUPT_PSEUDOCODE for %s %s\n", fw->data,
-				vub300->vub_name);
-			strscpy(vub300->vub_name,
-				"SDIO interrupt pseudocode download failed",
-				sizeof(vub300->vub_name));
-			return;
-		}
-	} else {
-		dev_err(&vub300->udev->dev,
-			"corrupt interrupt pseudocode in firmware %s %s\n",
-			fw->data, vub300->vub_name);
-		strscpy(vub300->vub_name, "corrupt interrupt pseudocode",
-			sizeof(vub300->vub_name));
-		return;
-	}
-	ts += *data++;
-	size -= 1;
-	ts <<= 8;
-	ts += *data++;
-	size -= 1;
-	if (ts < size) {
-		u16 xfer_length = roundup_to_multiple_of_64(ts);
-		u8 *xfer_buffer = kmalloc(xfer_length, GFP_KERNEL);
-		if (xfer_buffer) {
-			int retval;
-			memcpy(xfer_buffer, data, ts);
-			memset(xfer_buffer + ts, 0,
-			       xfer_length - ts);
-			size -= ts;
-			data += ts;
-			retval =
-				usb_control_msg(vub300->udev,
-						usb_sndctrlpipe(vub300->udev, 0),
-						SET_TRANSFER_PSEUDOCODE,
-						USB_DIR_OUT | USB_TYPE_VENDOR |
-						USB_RECIP_DEVICE, 0x0000, 0x0000,
-						xfer_buffer, xfer_length, 1000);
-			kfree(xfer_buffer);
-			if (retval < 0)
-				goto copy_error_message;
-		} else {
-			dev_err(&vub300->udev->dev,
-				"not enough memory for xfer buffer to send"
-				" TRANSFER_PSEUDOCODE for %s %s\n", fw->data,
-				vub300->vub_name);
-			strscpy(vub300->vub_name,
-				"SDIO transfer pseudocode download failed",
-				sizeof(vub300->vub_name));
-			return;
-		}
-	} else {
-		dev_err(&vub300->udev->dev,
-			"corrupt transfer pseudocode in firmware %s %s\n",
-			fw->data, vub300->vub_name);
-		strscpy(vub300->vub_name, "corrupt transfer pseudocode",
-			sizeof(vub300->vub_name));
-		return;
-	}
-	register_count += *data++;
-	size -= 1;
-	if (register_count * 4 == size) {
-		int I = vub300->dynamic_register_count = register_count;
-		int i = 0;
-		while (I--) {
-			unsigned int func_num = 0;
-			vub300->sdio_register[i].func_num = *data++;
-			size -= 1;
-			func_num += *data++;
-			size -= 1;
-			func_num <<= 8;
-			func_num += *data++;
-			size -= 1;
-			func_num <<= 8;
-			func_num += *data++;
-			size -= 1;
-			vub300->sdio_register[i].sdio_reg = func_num;
-			vub300->sdio_register[i].activate = 1;
-			vub300->sdio_register[i].prepared = 0;
-			i += 1;
-		}
-		dev_info(&vub300->udev->dev,
-			 "initialized %d dynamic pseudocode registers\n",
-			 vub300->dynamic_register_count);
-		return;
-	} else {
-		dev_err(&vub300->udev->dev,
-			"corrupt dynamic registers in firmware %s\n",
-			vub300->vub_name);
-		strscpy(vub300->vub_name, "corrupt dynamic registers",
-			sizeof(vub300->vub_name));
-		return;
-	}
-
-copy_error_message:
-	strscpy(vub300->vub_name, "SDIO pseudocode download failed",
-		sizeof(vub300->vub_name));
-}
-
-/*
- * if the binary containing the EMPTY PseudoCode can not be found
- * vub300->vub_name is set anyway in order to prevent an automatic retry
- */
-static void download_offload_pseudocode(struct vub300_mmc_host *vub300)
-{
-	struct mmc_card *card = vub300->mmc->card;
-	int sdio_funcs = card->sdio_funcs;
-	const struct firmware *fw = NULL;
-	int l = snprintf(vub300->vub_name, sizeof(vub300->vub_name),
-			 "vub_%04X%04X", card->cis.vendor, card->cis.device);
-	int n = 0;
-	int retval;
-	for (n = 0; n < sdio_funcs; n++) {
-		struct sdio_func *sf = card->sdio_func[n];
-		l += scnprintf(vub300->vub_name + l,
-			      sizeof(vub300->vub_name) - l, "_%04X%04X",
-			      sf->vendor, sf->device);
-	}
-	snprintf(vub300->vub_name + l, sizeof(vub300->vub_name) - l, ".bin");
-	dev_info(&vub300->udev->dev, "requesting offload firmware %s\n",
-		 vub300->vub_name);
-	retval = request_firmware(&fw, vub300->vub_name, &card->dev);
-	if (retval < 0) {
-		strscpy(vub300->vub_name, "vub_default.bin",
-			sizeof(vub300->vub_name));
-		retval = request_firmware(&fw, vub300->vub_name, &card->dev);
-		if (retval < 0) {
-			strscpy(vub300->vub_name,
-				"no SDIO offload firmware found",
-				sizeof(vub300->vub_name));
-		} else {
-			__download_offload_pseudocode(vub300, fw);
-			release_firmware(fw);
-		}
-	} else {
-		__download_offload_pseudocode(vub300, fw);
-		release_firmware(fw);
-	}
-}
-
-static void vub300_usb_bulk_msg_completion(struct urb *urb)
-{				/* urb completion handler - hardirq */
-	complete((struct completion *)urb->context);
-}
-
-static int vub300_usb_bulk_msg(struct vub300_mmc_host *vub300,
-			       unsigned int pipe, void *data, int len,
-			       int *actual_length, int timeout_msecs)
-{
-	/* cmd_mutex is held by vub300_cmndwork_thread */
-	struct usb_device *usb_dev = vub300->udev;
-	struct completion done;
-	int retval;
-	vub300->urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!vub300->urb)
-		return -ENOMEM;
-	usb_fill_bulk_urb(vub300->urb, usb_dev, pipe, data, len,
-			  vub300_usb_bulk_msg_completion, NULL);
-	init_completion(&done);
-	vub300->urb->context = &done;
-	vub300->urb->actual_length = 0;
-	retval = usb_submit_urb(vub300->urb, GFP_KERNEL);
-	if (unlikely(retval))
-		goto out;
-	if (!wait_for_completion_timeout
-	    (&done, msecs_to_jiffies(timeout_msecs))) {
-		retval = -ETIMEDOUT;
-		usb_kill_urb(vub300->urb);
-	} else {
-		retval = vub300->urb->status;
-	}
-out:
-	*actual_length = vub300->urb->actual_length;
-	usb_free_urb(vub300->urb);
-	vub300->urb = NULL;
-	return retval;
-}
-
-static int __command_read_data(struct vub300_mmc_host *vub300,
-			       struct mmc_command *cmd, struct mmc_data *data)
-{
-	/* cmd_mutex is held by vub300_cmndwork_thread */
-	int linear_length = vub300->datasize;
-	int padded_length = vub300->large_usb_packets ?
-		((511 + linear_length) >> 9) << 9 :
-		((63 + linear_length) >> 6) << 6;
-	if ((padded_length == linear_length) || !pad_input_to_usb_pkt) {
-		int result;
-		unsigned pipe;
-		pipe = usb_rcvbulkpipe(vub300->udev, vub300->data_inp_ep);
-		result = usb_sg_init(&vub300->sg_request, vub300->udev,
-				     pipe, 0, data->sg,
-				     data->sg_len, 0, GFP_KERNEL);
-		if (result < 0) {
-			usb_unlink_urb(vub300->command_out_urb);
-			usb_unlink_urb(vub300->command_res_urb);
-			cmd->error = result;
-			data->bytes_xfered = 0;
-			return 0;
-		} else {
-			vub300->sg_transfer_timer.expires =
-				jiffies + msecs_to_jiffies(2000 +
-						  (linear_length / 16384));
-			add_timer(&vub300->sg_transfer_timer);
-			usb_sg_wait(&vub300->sg_request);
-			timer_delete(&vub300->sg_transfer_timer);
-			if (vub300->sg_request.status < 0) {
-				cmd->error = vub300->sg_request.status;
-				data->bytes_xfered = 0;
-				return 0;
-			} else {
-				data->bytes_xfered = vub300->datasize;
-				return linear_length;
-			}
-		}
-	} else {
-		u8 *buf = kmalloc(padded_length, GFP_KERNEL);
-		if (buf) {
-			int result;
-			unsigned pipe = usb_rcvbulkpipe(vub300->udev,
-							vub300->data_inp_ep);
-			int actual_length = 0;
-			result = vub300_usb_bulk_msg(vub300, pipe, buf,
-					     padded_length, &actual_length,
-					     2000 + (padded_length / 16384));
-			if (result < 0) {
-				cmd->error = result;
-				data->bytes_xfered = 0;
-				kfree(buf);
-				return 0;
-			} else if (actual_length < linear_length) {
-				cmd->error = -EREMOTEIO;
-				data->bytes_xfered = 0;
-				kfree(buf);
-				return 0;
-			} else {
-				sg_copy_from_buffer(data->sg, data->sg_len, buf,
-						    linear_length);
-				kfree(buf);
-				data->bytes_xfered = vub300->datasize;
-				return linear_length;
-			}
-		} else {
-			cmd->error = -ENOMEM;
-			data->bytes_xfered = 0;
-			return 0;
-		}
-	}
-}
-
-static int __command_write_data(struct vub300_mmc_host *vub300,
-				struct mmc_command *cmd, struct mmc_data *data)
-{
-	/* cmd_mutex is held by vub300_cmndwork_thread */
-	unsigned pipe = usb_sndbulkpipe(vub300->udev, vub300->data_out_ep);
-	int linear_length = vub300->datasize;
-	int modulo_64_length = linear_length & 0x003F;
-	int modulo_512_length = linear_length & 0x01FF;
-	if (linear_length < 64) {
-		int result;
-		int actual_length;
-		sg_copy_to_buffer(data->sg, data->sg_len,
-				  vub300->padded_buffer,
-				  sizeof(vub300->padded_buffer));
-		memset(vub300->padded_buffer + linear_length, 0,
-		       sizeof(vub300->padded_buffer) - linear_length);
-		result = vub300_usb_bulk_msg(vub300, pipe, vub300->padded_buffer,
-					     sizeof(vub300->padded_buffer),
-					     &actual_length, 2000 +
-					     (sizeof(vub300->padded_buffer) /
-					      16384));
-		if (result < 0) {
-			cmd->error = result;
-			data->bytes_xfered = 0;
-		} else {
-			data->bytes_xfered = vub300->datasize;
-		}
-	} else if ((!vub300->large_usb_packets && (0 < modulo_64_length)) ||
-		    (vub300->large_usb_packets && (64 > modulo_512_length))
-		) {		/* don't you just love these work-rounds */
-		int padded_length = ((63 + linear_length) >> 6) << 6;
-		u8 *buf = kmalloc(padded_length, GFP_KERNEL);
-		if (buf) {
-			int result;
-			int actual_length;
-			sg_copy_to_buffer(data->sg, data->sg_len, buf,
-					  padded_length);
-			memset(buf + linear_length, 0,
-			       padded_length - linear_length);
-			result =
-				vub300_usb_bulk_msg(vub300, pipe, buf,
-						    padded_length, &actual_length,
-						    2000 + padded_length / 16384);
-			kfree(buf);
-			if (result < 0) {
-				cmd->error = result;
-				data->bytes_xfered = 0;
-			} else {
-				data->bytes_xfered = vub300->datasize;
-			}
-		} else {
-			cmd->error = -ENOMEM;
-			data->bytes_xfered = 0;
-		}
-	} else {		/* no data padding required */
-		int result;
-		unsigned char buf[64 * 4];
-		sg_copy_to_buffer(data->sg, data->sg_len, buf, sizeof(buf));
-		result = usb_sg_init(&vub300->sg_request, vub300->udev,
-				     pipe, 0, data->sg,
-				     data->sg_len, 0, GFP_KERNEL);
-		if (result < 0) {
-			usb_unlink_urb(vub300->command_out_urb);
-			usb_unlink_urb(vub300->command_res_urb);
-			cmd->error = result;
-			data->bytes_xfered = 0;
-		} else {
-			vub300->sg_transfer_timer.expires =
-				jiffies + msecs_to_jiffies(2000 +
-							   linear_length / 16384);
-			add_timer(&vub300->sg_transfer_timer);
-			usb_sg_wait(&vub300->sg_request);
-			if (cmd->error) {
-				data->bytes_xfered = 0;
-			} else {
-				timer_delete(&vub300->sg_transfer_timer);
-				if (vub300->sg_request.status < 0) {
-					cmd->error = vub300->sg_request.status;
-					data->bytes_xfered = 0;
-				} else {
-					data->bytes_xfered = vub300->datasize;
-				}
-			}
-		}
-	}
-	return linear_length;
-}
-
-static bool __vub300_command_response(struct vub300_mmc_host *vub300,
-				      struct mmc_command *cmd,
-				      struct mmc_data *data, int data_length)
-{
-	/* cmd_mutex is held by vub300_cmndwork_thread */
-	long respretval;
-	int msec_timeout = 1000 + data_length / 4;
-	respretval =
-		wait_for_completion_timeout(&vub300->command_complete,
-					    msecs_to_jiffies(msec_timeout));
-	if (respretval == 0) { /* TIMED OUT */
-		/* we don't know which of "out" and "res" if any failed */
-		vub300->usb_timed_out = 1;
-		usb_kill_urb(vub300->command_out_urb);
-		usb_kill_urb(vub300->command_res_urb);
-		cmd->error = -ETIMEDOUT;
-		return true;
-	} else if (respretval < 0) {
-		/* we don't know which of "out" and "res" if any failed */
-		usb_kill_urb(vub300->command_out_urb);
-		usb_kill_urb(vub300->command_res_urb);
-		cmd->error = respretval;
-	} else if (cmd->error) {
-		/*
-		 * the error occurred sending the command
-		 * or receiving the response
-		 */
-	} else if (vub300->command_out_urb->status) {
-		vub300->usb_transport_fail = vub300->command_out_urb->status;
-		cmd->error = -EPROTO == vub300->command_out_urb->status ?
-			-ESHUTDOWN : vub300->command_out_urb->status;
-	} else if (vub300->command_res_urb->status) {
-		vub300->usb_transport_fail = vub300->command_res_urb->status;
-		cmd->error = -EPROTO == vub300->command_res_urb->status ?
-			-ESHUTDOWN : vub300->command_res_urb->status;
-	} else if (vub300->resp.common.header_type == 0x00) {
-		/*
-		 * the command completed successfully
-		 * and there was no piggybacked data
-		 */
-	} else if (vub300->resp.common.header_type == RESPONSE_ERROR) {
-		cmd->error =
-			vub300_response_error(vub300->resp.error.error_code);
-		if (vub300->data)
-			usb_sg_cancel(&vub300->sg_request);
-	} else if (vub300->resp.common.header_type == RESPONSE_PIGGYBACKED) {
-		int offloaded_data_length =
-			vub300->resp.common.header_size -
-			sizeof(struct sd_register_header);
-		int register_count = offloaded_data_length >> 3;
-		int ri = 0;
-		while (register_count--) {
-			add_offloaded_reg(vub300, &vub300->resp.pig.reg[ri]);
-			ri += 1;
-		}
-		vub300->resp.common.header_size =
-			sizeof(struct sd_register_header);
-		vub300->resp.common.header_type = 0x00;
-		cmd->error = 0;
-	} else if (vub300->resp.common.header_type == RESPONSE_PIG_DISABLED) {
-		int offloaded_data_length =
-			vub300->resp.common.header_size -
-			sizeof(struct sd_register_header);
-		int register_count = offloaded_data_length >> 3;
-		int ri = 0;
-		while (register_count--) {
-			add_offloaded_reg(vub300, &vub300->resp.pig.reg[ri]);
-			ri += 1;
-		}
-		mutex_lock(&vub300->irq_mutex);
-		if (vub300->irqs_queued) {
-			vub300->irqs_queued += 1;
-		} else if (vub300->irq_enabled) {
-			vub300->irqs_queued += 1;
-			vub300_queue_poll_work(vub300, 0);
-		} else {
-			vub300->irqs_queued += 1;
-		}
-		vub300->irq_disabled = 1;
-		mutex_unlock(&vub300->irq_mutex);
-		vub300->resp.common.header_size =
-			sizeof(struct sd_register_header);
-		vub300->resp.common.header_type = 0x00;
-		cmd->error = 0;
-	} else if (vub300->resp.common.header_type == RESPONSE_PIG_ENABLED) {
-		int offloaded_data_length =
-			vub300->resp.common.header_size -
-			sizeof(struct sd_register_header);
-		int register_count = offloaded_data_length >> 3;
-		int ri = 0;
-		while (register_count--) {
-			add_offloaded_reg(vub300, &vub300->resp.pig.reg[ri]);
-			ri += 1;
-		}
-		mutex_lock(&vub300->irq_mutex);
-		if (vub300->irqs_queued) {
-			vub300->irqs_queued += 1;
-		} else if (vub300->irq_enabled) {
-			vub300->irqs_queued += 1;
-			vub300_queue_poll_work(vub300, 0);
-		} else {
-			vub300->irqs_queued += 1;
-		}
-		vub300->irq_disabled = 0;
-		mutex_unlock(&vub300->irq_mutex);
-		vub300->resp.common.header_size =
-			sizeof(struct sd_register_header);
-		vub300->resp.common.header_type = 0x00;
-		cmd->error = 0;
-	} else {
-		cmd->error = -EINVAL;
-	}
-
-	return false;
-}
-
-static void construct_request_response(struct vub300_mmc_host *vub300,
-				       struct mmc_command *cmd)
-{
-	int resp_len = vub300->resp_len;
-	int less_cmd = (17 == resp_len) ? resp_len : resp_len - 1;
-	int bytes = 3 & less_cmd;
-	int words = less_cmd >> 2;
-	u8 *r = vub300->resp.response.command_response;
-
-	if (!resp_len)
-		return;
-	if (bytes == 3) {
-		cmd->resp[words] = (r[1 + (words << 2)] << 24)
-			| (r[2 + (words << 2)] << 16)
-			| (r[3 + (words << 2)] << 8);
-	} else if (bytes == 2) {
-		cmd->resp[words] = (r[1 + (words << 2)] << 24)
-			| (r[2 + (words << 2)] << 16);
-	} else if (bytes == 1) {
-		cmd->resp[words] = (r[1 + (words << 2)] << 24);
-	}
-	while (words-- > 0) {
-		cmd->resp[words] = (r[1 + (words << 2)] << 24)
-			| (r[2 + (words << 2)] << 16)
-			| (r[3 + (words << 2)] << 8)
-			| (r[4 + (words << 2)] << 0);
-	}
-	if ((cmd->opcode == 53) && (0x000000FF & cmd->resp[0]))
-		cmd->resp[0] &= 0xFFFFFF00;
-}
-
-/* this thread runs only when there is an upper level command req outstanding */
-static void vub300_cmndwork_thread(struct work_struct *work)
-{
-	struct vub300_mmc_host *vub300 =
-		container_of(work, struct vub300_mmc_host, cmndwork);
-	if (!vub300->interface) {
-		kref_put(&vub300->kref, vub300_delete);
-		return;
-	} else {
-		struct mmc_request *req = vub300->req;
-		struct mmc_command *cmd = vub300->cmd;
-		struct mmc_data *data = vub300->data;
-		bool reset_device;
-		int data_length;
-		mutex_lock(&vub300->cmd_mutex);
-		init_completion(&vub300->command_complete);
-		if (likely(vub300->vub_name[0]) || !vub300->mmc->card) {
-			/*
-			 * the name of the EMPTY Pseudo firmware file
-			 * is used as a flag to indicate that the file
-			 * has been already downloaded to the VUB300 chip
-			 */
-		} else if (0 == vub300->mmc->card->sdio_funcs) {
-			strscpy(vub300->vub_name, "SD memory device",
-				sizeof(vub300->vub_name));
-		} else {
-			download_offload_pseudocode(vub300);
-		}
-		send_command(vub300);
-		if (!data)
-			data_length = 0;
-		else if (MMC_DATA_READ & data->flags)
-			data_length = __command_read_data(vub300, cmd, data);
-		else
-			data_length = __command_write_data(vub300, cmd, data);
-		reset_device = __vub300_command_response(vub300, cmd,
-							 data, data_length);
-		vub300->req = NULL;
-		vub300->cmd = NULL;
-		vub300->data = NULL;
-		if (cmd->error) {
-			if (cmd->error == -ENOMEDIUM)
-				check_vub300_port_status(vub300);
-			mutex_unlock(&vub300->cmd_mutex);
-			if (reset_device) {
-				int result;
-
-				result = usb_lock_device_for_reset(vub300->udev,
-								   vub300->interface);
-				if (result == 0) {
-					result = usb_reset_device(vub300->udev);
-					usb_unlock_device(vub300->udev);
-				}
-			}
-			mmc_request_done(vub300->mmc, req);
-			kref_put(&vub300->kref, vub300_delete);
-			return;
-		} else {
-			construct_request_response(vub300, cmd);
-			vub300->resp_len = 0;
-			mutex_unlock(&vub300->cmd_mutex);
-			kref_put(&vub300->kref, vub300_delete);
-			mmc_request_done(vub300->mmc, req);
-			return;
-		}
-	}
-}
-
-static int examine_cyclic_buffer(struct vub300_mmc_host *vub300,
-				 struct mmc_command *cmd, u8 Function)
-{
-	/* cmd_mutex is held by vub300_mmc_request */
-	u8 cmd0 = 0xFF & (cmd->arg >> 24);
-	u8 cmd1 = 0xFF & (cmd->arg >> 16);
-	u8 cmd2 = 0xFF & (cmd->arg >> 8);
-	u8 cmd3 = 0xFF & (cmd->arg >> 0);
-	int first = MAXREGMASK & vub300->fn[Function].offload_point;
-	struct offload_registers_access *rf = &vub300->fn[Function].reg[first];
-	if (cmd0 == rf->command_byte[0] &&
-	    cmd1 == rf->command_byte[1] &&
-	    cmd2 == rf->command_byte[2] &&
-	    cmd3 == rf->command_byte[3]) {
-		u8 checksum = 0x00;
-		cmd->resp[1] = checksum << 24;
-		cmd->resp[0] = (rf->Respond_Byte[0] << 24)
-			| (rf->Respond_Byte[1] << 16)
-			| (rf->Respond_Byte[2] << 8)
-			| (rf->Respond_Byte[3] << 0);
-		vub300->fn[Function].offload_point += 1;
-		vub300->fn[Function].offload_count -= 1;
-		vub300->total_offload_count -= 1;
-		return 1;
-	} else {
-		int delta = 1;	/* because it does not match the first one */
-		u8 register_count = vub300->fn[Function].offload_count - 1;
-		u32 register_point = vub300->fn[Function].offload_point + 1;
-		while (0 < register_count) {
-			int point = MAXREGMASK & register_point;
-			struct offload_registers_access *r =
-				&vub300->fn[Function].reg[point];
-			if (cmd0 == r->command_byte[0] &&
-			    cmd1 == r->command_byte[1] &&
-			    cmd2 == r->command_byte[2] &&
-			    cmd3 == r->command_byte[3]) {
-				u8 checksum = 0x00;
-				cmd->resp[1] = checksum << 24;
-				cmd->resp[0] = (r->Respond_Byte[0] << 24)
-					| (r->Respond_Byte[1] << 16)
-					| (r->Respond_Byte[2] << 8)
-					| (r->Respond_Byte[3] << 0);
-				vub300->fn[Function].offload_point += delta;
-				vub300->fn[Function].offload_count -= delta;
-				vub300->total_offload_count -= delta;
-				return 1;
-			} else {
-				register_point += 1;
-				register_count -= 1;
-				delta += 1;
-				continue;
-			}
-		}
-		return 0;
-	}
-}
-
-static int satisfy_request_from_offloaded_data(struct vub300_mmc_host *vub300,
-					       struct mmc_command *cmd)
-{
-	/* cmd_mutex is held by vub300_mmc_request */
-	u8 regs = vub300->dynamic_register_count;
-	u8 i = 0;
-	u8 func = FUN(cmd);
-	u32 reg = REG(cmd);
-	while (0 < regs--) {
-		if ((vub300->sdio_register[i].func_num == func) &&
-		    (vub300->sdio_register[i].sdio_reg == reg)) {
-			if (!vub300->sdio_register[i].prepared) {
-				return 0;
-			} else if ((0x80000000 & cmd->arg) == 0x80000000) {
-				/*
-				 * a write to a dynamic register
-				 * nullifies our offloaded value
-				 */
-				vub300->sdio_register[i].prepared = 0;
-				return 0;
-			} else {
-				u8 checksum = 0x00;
-				u8 rsp0 = 0x00;
-				u8 rsp1 = 0x00;
-				u8 rsp2 = vub300->sdio_register[i].response;
-				u8 rsp3 = vub300->sdio_register[i].regvalue;
-				vub300->sdio_register[i].prepared = 0;
-				cmd->resp[1] = checksum << 24;
-				cmd->resp[0] = (rsp0 << 24)
-					| (rsp1 << 16)
-					| (rsp2 << 8)
-					| (rsp3 << 0);
-				return 1;
-			}
-		} else {
-			i += 1;
-			continue;
-		}
-	}
-	if (vub300->total_offload_count == 0)
-		return 0;
-	else if (vub300->fn[func].offload_count == 0)
-		return 0;
-	else
-		return examine_cyclic_buffer(vub300, cmd, func);
-}
-
-static void vub300_mmc_request(struct mmc_host *mmc, struct mmc_request *req)
-{				/* NOT irq */
-	struct mmc_command *cmd = req->cmd;
-	struct vub300_mmc_host *vub300 = mmc_priv(mmc);
-	if (!vub300->interface) {
-		cmd->error = -ESHUTDOWN;
-		mmc_request_done(mmc, req);
-		return;
-	} else {
-		struct mmc_data *data = req->data;
-		if (!vub300->card_powered) {
-			cmd->error = -ENOMEDIUM;
-			mmc_request_done(mmc, req);
-			return;
-		}
-		if (!vub300->card_present) {
-			cmd->error = -ENOMEDIUM;
-			mmc_request_done(mmc, req);
-			return;
-		}
-		if (vub300->usb_transport_fail) {
-			cmd->error = vub300->usb_transport_fail;
-			mmc_request_done(mmc, req);
-			return;
-		}
-		if (!vub300->interface) {
-			cmd->error = -ENODEV;
-			mmc_request_done(mmc, req);
-			return;
-		}
-		kref_get(&vub300->kref);
-		mutex_lock(&vub300->cmd_mutex);
-		mod_timer(&vub300->inactivity_timer, jiffies + HZ);
-		/*
-		 * for performance we have to return immediately
-		 * if the requested data has been offloaded
-		 */
-		if (cmd->opcode == 52 &&
-		    satisfy_request_from_offloaded_data(vub300, cmd)) {
-			cmd->error = 0;
-			mutex_unlock(&vub300->cmd_mutex);
-			kref_put(&vub300->kref, vub300_delete);
-			mmc_request_done(mmc, req);
-			return;
-		} else {
-			vub300->cmd = cmd;
-			vub300->req = req;
-			vub300->data = data;
-			if (data)
-				vub300->datasize = data->blksz * data->blocks;
-			else
-				vub300->datasize = 0;
-			vub300_queue_cmnd_work(vub300);
-			mutex_unlock(&vub300->cmd_mutex);
-			kref_put(&vub300->kref, vub300_delete);
-			/*
-			 * the kernel lock diagnostics complain
-			 * if the cmd_mutex * is "passed on"
-			 * to the cmndwork thread,
-			 * so we must release it now
-			 * and re-acquire it in the cmndwork thread
-			 */
-		}
-	}
-}
-
-static void __set_clock_speed(struct vub300_mmc_host *vub300, u8 buf[8],
-			      struct mmc_ios *ios)
-{
-	int buf_array_size = 8; /* ARRAY_SIZE(buf) does not work !!! */
-	int retval;
-	u32 kHzClock;
-	if (ios->clock >= 48000000)
-		kHzClock = 48000;
-	else if (ios->clock >= 24000000)
-		kHzClock = 24000;
-	else if (ios->clock >= 20000000)
-		kHzClock = 20000;
-	else if (ios->clock >= 15000000)
-		kHzClock = 15000;
-	else if (ios->clock >= 200000)
-		kHzClock = 200;
-	else
-		kHzClock = 0;
-	{
-		int i;
-		u64 c = kHzClock;
-		for (i = 0; i < buf_array_size; i++) {
-			buf[i] = c;
-			c >>= 8;
-		}
-	}
-	retval =
-		usb_control_msg(vub300->udev, usb_sndctrlpipe(vub300->udev, 0),
-				SET_CLOCK_SPEED,
-				USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				0x00, 0x00, buf, buf_array_size, 1000);
-	if (retval != 8) {
-		dev_err(&vub300->udev->dev, "SET_CLOCK_SPEED"
-			" %dkHz failed with retval=%d\n", kHzClock, retval);
-	} else {
-		dev_dbg(&vub300->udev->dev, "SET_CLOCK_SPEED"
-			" %dkHz\n", kHzClock);
-	}
-}
-
-static void vub300_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300 = mmc_priv(mmc);
-	if (!vub300->interface)
-		return;
-	kref_get(&vub300->kref);
-	mutex_lock(&vub300->cmd_mutex);
-	if ((ios->power_mode == MMC_POWER_OFF) && vub300->card_powered) {
-		vub300->card_powered = 0;
-		usb_control_msg(vub300->udev, usb_sndctrlpipe(vub300->udev, 0),
-				SET_SD_POWER,
-				USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				0x0000, 0x0000, NULL, 0, 1000);
-		/* must wait for the VUB300 u-proc to boot up */
-		msleep(600);
-	} else if ((ios->power_mode == MMC_POWER_UP) && !vub300->card_powered) {
-		usb_control_msg(vub300->udev, usb_sndctrlpipe(vub300->udev, 0),
-				SET_SD_POWER,
-				USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				0x0001, 0x0000, NULL, 0, 1000);
-		msleep(600);
-		vub300->card_powered = 1;
-	} else if (ios->power_mode == MMC_POWER_ON) {
-		u8 *buf = kmalloc(8, GFP_KERNEL);
-		if (buf) {
-			__set_clock_speed(vub300, buf, ios);
-			kfree(buf);
-		}
-	} else {
-		/* this should mean no change of state */
-	}
-	mutex_unlock(&vub300->cmd_mutex);
-	kref_put(&vub300->kref, vub300_delete);
-}
-
-static int vub300_mmc_get_ro(struct mmc_host *mmc)
-{
-	struct vub300_mmc_host *vub300 = mmc_priv(mmc);
-	return vub300->read_only;
-}
-
-static void vub300_enable_sdio_irq(struct mmc_host *mmc, int enable)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300 = mmc_priv(mmc);
-	if (!vub300->interface)
-		return;
-	kref_get(&vub300->kref);
-	if (enable) {
-		set_current_state(TASK_RUNNING);
-		mutex_lock(&vub300->irq_mutex);
-		if (vub300->irqs_queued) {
-			vub300->irqs_queued -= 1;
-			mmc_signal_sdio_irq(vub300->mmc);
-		} else if (vub300->irq_disabled) {
-			vub300->irq_disabled = 0;
-			vub300->irq_enabled = 1;
-			vub300_queue_poll_work(vub300, 0);
-		} else if (vub300->irq_enabled) {
-			/* this should not happen, so we will just ignore it */
-		} else {
-			vub300->irq_enabled = 1;
-			vub300_queue_poll_work(vub300, 0);
-		}
-		mutex_unlock(&vub300->irq_mutex);
-		set_current_state(TASK_INTERRUPTIBLE);
-	} else {
-		vub300->irq_enabled = 0;
-	}
-	kref_put(&vub300->kref, vub300_delete);
-}
-
-static const struct mmc_host_ops vub300_mmc_ops = {
-	.request = vub300_mmc_request,
-	.set_ios = vub300_mmc_set_ios,
-	.get_ro = vub300_mmc_get_ro,
-	.enable_sdio_irq = vub300_enable_sdio_irq,
-};
-
-static int vub300_probe(struct usb_interface *interface,
-			const struct usb_device_id *id)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300;
-	struct usb_host_interface *iface_desc;
-	struct usb_device *udev = usb_get_dev(interface_to_usbdev(interface));
-	int i;
-	int retval = -ENOMEM;
-	struct urb *command_out_urb;
-	struct urb *command_res_urb;
-	struct mmc_host *mmc;
-	char manufacturer[48];
-	char product[32];
-	char serial_number[32];
-	usb_string(udev, udev->descriptor.iManufacturer, manufacturer,
-		   sizeof(manufacturer));
-	usb_string(udev, udev->descriptor.iProduct, product, sizeof(product));
-	usb_string(udev, udev->descriptor.iSerialNumber, serial_number,
-		   sizeof(serial_number));
-	dev_info(&udev->dev, "probing VID:PID(%04X:%04X) %s %s %s\n",
-		 le16_to_cpu(udev->descriptor.idVendor),
-		 le16_to_cpu(udev->descriptor.idProduct),
-		 manufacturer, product, serial_number);
-	command_out_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!command_out_urb) {
-		retval = -ENOMEM;
-		goto err_put_udev;
-	}
-	command_res_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!command_res_urb) {
-		retval = -ENOMEM;
-		goto err_free_out_urb;
-	}
-	/* this also allocates memory for our VUB300 mmc host device */
-	mmc = mmc_alloc_host(sizeof(*vub300), &udev->dev);
-	if (!mmc) {
-		retval = -ENOMEM;
-		dev_err(&udev->dev, "not enough memory for the mmc_host\n");
-		goto err_free_res_urb;
-	}
-	/* MMC core transfer sizes tunable parameters */
-	mmc->caps = 0;
-	if (!force_1_bit_data_xfers)
-		mmc->caps |= MMC_CAP_4_BIT_DATA;
-	if (!force_polling_for_irqs)
-		mmc->caps |= MMC_CAP_SDIO_IRQ;
-	mmc->caps &= ~MMC_CAP_NEEDS_POLL;
-	/*
-	 * MMC_CAP_NEEDS_POLL causes core.c:mmc_rescan() to poll
-	 * for devices which results in spurious CMD7's being
-	 * issued which stops some SDIO cards from working
-	 */
-	if (limit_speed_to_24_MHz) {
-		mmc->caps |= MMC_CAP_MMC_HIGHSPEED;
-		mmc->caps |= MMC_CAP_SD_HIGHSPEED;
-		mmc->f_max = 24000000;
-		dev_info(&udev->dev, "limiting SDIO speed to 24_MHz\n");
-	} else {
-		mmc->caps |= MMC_CAP_MMC_HIGHSPEED;
-		mmc->caps |= MMC_CAP_SD_HIGHSPEED;
-		mmc->f_max = 48000000;
-	}
-	mmc->f_min = 200000;
-	mmc->max_blk_count = 511;
-	mmc->max_blk_size = 512;
-	mmc->max_segs = 128;
-	if (force_max_req_size)
-		mmc->max_req_size = force_max_req_size * 1024;
-	else
-		mmc->max_req_size = 64 * 1024;
-	mmc->max_seg_size = mmc->max_req_size;
-	mmc->ocr_avail = 0;
-	mmc->ocr_avail |= MMC_VDD_165_195;
-	mmc->ocr_avail |= MMC_VDD_20_21;
-	mmc->ocr_avail |= MMC_VDD_21_22;
-	mmc->ocr_avail |= MMC_VDD_22_23;
-	mmc->ocr_avail |= MMC_VDD_23_24;
-	mmc->ocr_avail |= MMC_VDD_24_25;
-	mmc->ocr_avail |= MMC_VDD_25_26;
-	mmc->ocr_avail |= MMC_VDD_26_27;
-	mmc->ocr_avail |= MMC_VDD_27_28;
-	mmc->ocr_avail |= MMC_VDD_28_29;
-	mmc->ocr_avail |= MMC_VDD_29_30;
-	mmc->ocr_avail |= MMC_VDD_30_31;
-	mmc->ocr_avail |= MMC_VDD_31_32;
-	mmc->ocr_avail |= MMC_VDD_32_33;
-	mmc->ocr_avail |= MMC_VDD_33_34;
-	mmc->ocr_avail |= MMC_VDD_34_35;
-	mmc->ocr_avail |= MMC_VDD_35_36;
-	mmc->ops = &vub300_mmc_ops;
-	vub300 = mmc_priv(mmc);
-	vub300->mmc = mmc;
-	vub300->card_powered = 0;
-	vub300->bus_width = 0;
-	vub300->cmnd.head.block_size[0] = 0x00;
-	vub300->cmnd.head.block_size[1] = 0x00;
-	vub300->app_spec = 0;
-	mutex_init(&vub300->cmd_mutex);
-	mutex_init(&vub300->irq_mutex);
-	vub300->command_out_urb = command_out_urb;
-	vub300->command_res_urb = command_res_urb;
-	vub300->usb_timed_out = 0;
-	vub300->dynamic_register_count = 0;
-
-	for (i = 0; i < ARRAY_SIZE(vub300->fn); i++) {
-		vub300->fn[i].offload_point = 0;
-		vub300->fn[i].offload_count = 0;
-	}
-
-	vub300->total_offload_count = 0;
-	vub300->irq_enabled = 0;
-	vub300->irq_disabled = 0;
-	vub300->irqs_queued = 0;
-
-	for (i = 0; i < ARRAY_SIZE(vub300->sdio_register); i++)
-		vub300->sdio_register[i++].activate = 0;
-
-	vub300->udev = udev;
-	vub300->interface = interface;
-	vub300->cmnd_res_ep = 0;
-	vub300->cmnd_out_ep = 0;
-	vub300->data_inp_ep = 0;
-	vub300->data_out_ep = 0;
-
-	for (i = 0; i < ARRAY_SIZE(vub300->fbs); i++)
-		vub300->fbs[i] = 512;
-
-	/*
-	 *      set up the endpoint information
-	 *
-	 * use the first pair of bulk-in and bulk-out
-	 *     endpoints for Command/Response+Interrupt
-	 *
-	 * use the second pair of bulk-in and bulk-out
-	 *     endpoints for Data In/Out
-	 */
-	vub300->large_usb_packets = 0;
-	iface_desc = interface->cur_altsetting;
-	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
-		struct usb_endpoint_descriptor *endpoint =
-			&iface_desc->endpoint[i].desc;
-		dev_info(&vub300->udev->dev,
-			 "vub300 testing %s EndPoint(%d) %02X\n",
-			 usb_endpoint_is_bulk_in(endpoint) ? "BULK IN" :
-			 usb_endpoint_is_bulk_out(endpoint) ? "BULK OUT" :
-			 "UNKNOWN", i, endpoint->bEndpointAddress);
-		if (endpoint->wMaxPacketSize > 64)
-			vub300->large_usb_packets = 1;
-		if (usb_endpoint_is_bulk_in(endpoint)) {
-			if (!vub300->cmnd_res_ep) {
-				vub300->cmnd_res_ep =
-					endpoint->bEndpointAddress;
-			} else if (!vub300->data_inp_ep) {
-				vub300->data_inp_ep =
-					endpoint->bEndpointAddress;
-			} else {
-				dev_warn(&vub300->udev->dev,
-					 "ignoring"
-					 " unexpected bulk_in endpoint");
-			}
-		} else if (usb_endpoint_is_bulk_out(endpoint)) {
-			if (!vub300->cmnd_out_ep) {
-				vub300->cmnd_out_ep =
-					endpoint->bEndpointAddress;
-			} else if (!vub300->data_out_ep) {
-				vub300->data_out_ep =
-					endpoint->bEndpointAddress;
-			} else {
-				dev_warn(&vub300->udev->dev,
-					 "ignoring"
-					 " unexpected bulk_out endpoint");
-			}
-		} else {
-			dev_warn(&vub300->udev->dev,
-				 "vub300 ignoring EndPoint(%d) %02X", i,
-				 endpoint->bEndpointAddress);
-		}
-	}
-	if (vub300->cmnd_res_ep && vub300->cmnd_out_ep &&
-	    vub300->data_inp_ep && vub300->data_out_ep) {
-		dev_info(&vub300->udev->dev,
-			 "vub300 %s packets"
-			 " using EndPoints %02X %02X %02X %02X\n",
-			 vub300->large_usb_packets ? "LARGE" : "SMALL",
-			 vub300->cmnd_out_ep, vub300->cmnd_res_ep,
-			 vub300->data_out_ep, vub300->data_inp_ep);
-		/* we have the expected EndPoints */
-	} else {
-		dev_err(&vub300->udev->dev,
-		    "Could not find two sets of bulk-in/out endpoint pairs\n");
-		retval = -EINVAL;
-		goto err_free_host;
-	}
-	retval =
-		usb_control_msg(vub300->udev, usb_rcvctrlpipe(vub300->udev, 0),
-				GET_HC_INF0,
-				USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				0x0000, 0x0000, &vub300->hc_info,
-				sizeof(vub300->hc_info), 1000);
-	if (retval < 0)
-		goto err_free_host;
-	retval =
-		usb_control_msg(vub300->udev, usb_sndctrlpipe(vub300->udev, 0),
-				SET_ROM_WAIT_STATES,
-				USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				firmware_rom_wait_states, 0x0000, NULL, 0, 1000);
-	if (retval < 0)
-		goto err_free_host;
-	dev_info(&vub300->udev->dev,
-		 "operating_mode = %s %s %d MHz %s %d byte USB packets\n",
-		 (mmc->caps & MMC_CAP_SDIO_IRQ) ? "IRQs" : "POLL",
-		 (mmc->caps & MMC_CAP_4_BIT_DATA) ? "4-bit" : "1-bit",
-		 mmc->f_max / 1000000,
-		 pad_input_to_usb_pkt ? "padding input data to" : "with",
-		 vub300->large_usb_packets ? 512 : 64);
-	retval =
-		usb_control_msg(vub300->udev, usb_rcvctrlpipe(vub300->udev, 0),
-				GET_SYSTEM_PORT_STATUS,
-				USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
-				0x0000, 0x0000, &vub300->system_port_status,
-				sizeof(vub300->system_port_status), 1000);
-	if (retval < 0) {
-		goto err_free_host;
-	} else if (sizeof(vub300->system_port_status) == retval) {
-		vub300->card_present =
-			(0x0001 & vub300->system_port_status.port_flags) ? 1 : 0;
-		vub300->read_only =
-			(0x0010 & vub300->system_port_status.port_flags) ? 1 : 0;
-	} else {
-		retval = -EINVAL;
-		goto err_free_host;
-	}
-	usb_set_intfdata(interface, vub300);
-	INIT_DELAYED_WORK(&vub300->pollwork, vub300_pollwork_thread);
-	INIT_WORK(&vub300->cmndwork, vub300_cmndwork_thread);
-	INIT_WORK(&vub300->deadwork, vub300_deadwork_thread);
-	kref_init(&vub300->kref);
-	timer_setup(&vub300->sg_transfer_timer, vub300_sg_timed_out, 0);
-	kref_get(&vub300->kref);
-	timer_setup(&vub300->inactivity_timer,
-		    vub300_inactivity_timer_expired, 0);
-	vub300->inactivity_timer.expires = jiffies + HZ;
-	add_timer(&vub300->inactivity_timer);
-	if (vub300->card_present)
-		dev_info(&vub300->udev->dev,
-			 "USB vub300 remote SDIO host controller[%d]"
-			 "connected with SD/SDIO card inserted\n",
-			 interface_to_InterfaceNumber(interface));
-	else
-		dev_info(&vub300->udev->dev,
-			 "USB vub300 remote SDIO host controller[%d]"
-			 "connected with no SD/SDIO card inserted\n",
-			 interface_to_InterfaceNumber(interface));
-	retval = mmc_add_host(mmc);
-	if (retval)
-		goto err_stop_io;
-
-	return 0;
-
-err_stop_io:
-	vub300->interface = NULL;
-	kref_put(&vub300->kref, vub300_delete);
-
-	return retval;
-
-err_free_host:
-	mmc_free_host(mmc);
-	/*
-	 * and hence also frees vub300
-	 * which is contained at the end of struct mmc
-	 */
-err_free_res_urb:
-	usb_free_urb(command_res_urb);
-err_free_out_urb:
-	usb_free_urb(command_out_urb);
-err_put_udev:
-	usb_put_dev(udev);
-
-	return retval;
-}
-
-static void vub300_disconnect(struct usb_interface *interface)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300 = usb_get_intfdata(interface);
-	if (!vub300 || !vub300->mmc) {
-		return;
-	} else {
-		struct mmc_host *mmc = vub300->mmc;
-		if (!vub300->mmc) {
-			return;
-		} else {
-			int ifnum = interface_to_InterfaceNumber(interface);
-			usb_set_intfdata(interface, NULL);
-			/* prevent more I/O from starting */
-			vub300->interface = NULL;
-			mmc_remove_host(mmc);
-			kref_put(&vub300->kref, vub300_delete);
-			pr_info("USB vub300 remote SDIO host controller[%d]"
-				" now disconnected", ifnum);
-			return;
-		}
-	}
-}
-
-#ifdef CONFIG_PM
-static int vub300_suspend(struct usb_interface *intf, pm_message_t message)
-{
-	return 0;
-}
-
-static int vub300_resume(struct usb_interface *intf)
-{
-	return 0;
-}
-#else
-#define vub300_suspend NULL
-#define vub300_resume NULL
-#endif
-static int vub300_pre_reset(struct usb_interface *intf)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300 = usb_get_intfdata(intf);
-	mutex_lock(&vub300->cmd_mutex);
-	return 0;
-}
-
-static int vub300_post_reset(struct usb_interface *intf)
-{				/* NOT irq */
-	struct vub300_mmc_host *vub300 = usb_get_intfdata(intf);
-	/* we are sure no URBs are active - no locking needed */
-	vub300->errors = -EPIPE;
-	mutex_unlock(&vub300->cmd_mutex);
-	return 0;
-}
-
-static struct usb_driver vub300_driver = {
-	.name = "vub300",
-	.probe = vub300_probe,
-	.disconnect = vub300_disconnect,
-	.suspend = vub300_suspend,
-	.resume = vub300_resume,
-	.pre_reset = vub300_pre_reset,
-	.post_reset = vub300_post_reset,
-	.id_table = vub300_table,
-	.supports_autosuspend = 1,
-};
-
-static int __init vub300_init(void)
-{				/* NOT irq */
-	int result;
-
-	pr_info("VUB300 Driver rom wait states = %02X irqpoll timeout = %04X",
-		firmware_rom_wait_states, 0x0FFFF & firmware_irqpoll_timeout);
-
-	cmndworkqueue = create_singlethread_workqueue("kvub300c");
-	if (!cmndworkqueue)
-		return -ENOMEM;
-
-	pollworkqueue = create_singlethread_workqueue("kvub300p");
-	if (!pollworkqueue) {
-		result = -ENOMEM;
-		goto err_destroy_cmdwq;
-	}
-
-	deadworkqueue = create_singlethread_workqueue("kvub300d");
-	if (!deadworkqueue) {
-		result = -ENOMEM;
-		goto err_destroy_pollwq;
-	}
-
-	result = usb_register(&vub300_driver);
-	if (result)
-		goto err_destroy_deadwq;
-
-	return 0;
-
-err_destroy_deadwq:
-	destroy_workqueue(deadworkqueue);
-err_destroy_pollwq:
-	destroy_workqueue(pollworkqueue);
-err_destroy_cmdwq:
-	destroy_workqueue(cmndworkqueue);
-
-	return result;
-}
-
-static void __exit vub300_exit(void)
-{
-	usb_deregister(&vub300_driver);
-	flush_workqueue(cmndworkqueue);
-	flush_workqueue(pollworkqueue);
-	flush_workqueue(deadworkqueue);
-	destroy_workqueue(cmndworkqueue);
-	destroy_workqueue(pollworkqueue);
-	destroy_workqueue(deadworkqueue);
-}
-
-module_init(vub300_init);
-module_exit(vub300_exit);
-
-MODULE_AUTHOR("Tony Olech <tony.olech@elandigitalsystems.com>");
-MODULE_DESCRIPTION("VUB300 USB to SD/MMC/SDIO adapter driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/mtd/chips/cfi_cmdset_0001.c b/drivers/mtd/chips/cfi_cmdset_0001.c
index 6049ba2d6bcb66..e5805eebeed683 100644
--- a/drivers/mtd/chips/cfi_cmdset_0001.c
+++ b/drivers/mtd/chips/cfi_cmdset_0001.c
@@ -1716,12 +1716,24 @@ static int cfi_intelext_write_words (struct mtd_info *mtd, loff_t to , size_t le
 }
 
 
+/*
+ * Keep noinline: inlined, the map_word temporaries put do_write_buffer() over
+ * the frame-size limit with MTD_MAP_BANK_WIDTH_32 and KASAN_STACK.
+ */
+static noinline void __xipram cfi_write_cmd(struct map_info *map,
+					    unsigned long cmd, unsigned long adr)
+{
+	struct cfi_private *cfi = map->fldrv_priv;
+
+	map_write(map, CMD(cmd), adr);
+}
+
 static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 				    unsigned long adr, const struct kvec **pvec,
 				    unsigned long *pvec_seek, int len)
 {
 	struct cfi_private *cfi = map->fldrv_priv;
-	map_word status, write_cmd, datum;
+	map_word status, datum;
 	unsigned long cmd_adr;
 	int ret, wbufsize, word_gap, words;
 	const struct kvec *vec;
@@ -1740,9 +1752,6 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 	if (is_LH28F640BF(cfi))
 		cmd_adr = adr;
 
-	/* Let's determine this according to the interleave only once */
-	write_cmd = (cfi->cfiq->P_ID != P_ID_INTEL_PERFORMANCE) ? CMD(0xe8) : CMD(0xe9);
-
 	mutex_lock(&chip->mutex);
 	ret = get_chip(map, chip, cmd_adr, FL_WRITING);
 	if (ret) {
@@ -1759,7 +1768,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 	   So we must check here and reset those bits if they're set. Otherwise
 	   we're just pissing in the wind */
 	if (chip->state != FL_STATUS) {
-		map_write(map, CMD(0x70), cmd_adr);
+		cfi_write_cmd(map, 0x70, cmd_adr);
 		chip->state = FL_STATUS;
 	}
 	status = map_read(map, cmd_adr);
@@ -1767,21 +1776,22 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 		xip_enable(map, chip, cmd_adr);
 		printk(KERN_WARNING "SR.4 or SR.5 bits set in buffer write (status %lx). Clearing.\n", status.x[0]);
 		xip_disable(map, chip, cmd_adr);
-		map_write(map, CMD(0x50), cmd_adr);
-		map_write(map, CMD(0x70), cmd_adr);
+		cfi_write_cmd(map, 0x50, cmd_adr);
+		cfi_write_cmd(map, 0x70, cmd_adr);
 	}
 
 	chip->state = FL_WRITING_TO_BUFFER;
-	map_write(map, write_cmd, cmd_adr);
+	cfi_write_cmd(map, (cfi->cfiq->P_ID != P_ID_INTEL_PERFORMANCE) ? 0xe8 : 0xe9,
+		      cmd_adr);
 	ret = WAIT_TIMEOUT(map, chip, cmd_adr, 0, 0);
 	if (ret) {
 		/* Argh. Not ready for write to buffer */
 		map_word Xstatus = map_read(map, cmd_adr);
-		map_write(map, CMD(0x70), cmd_adr);
+		cfi_write_cmd(map, 0x70, cmd_adr);
 		chip->state = FL_STATUS;
 		status = map_read(map, cmd_adr);
-		map_write(map, CMD(0x50), cmd_adr);
-		map_write(map, CMD(0x70), cmd_adr);
+		cfi_write_cmd(map, 0x50, cmd_adr);
+		cfi_write_cmd(map, 0x70, cmd_adr);
 		xip_enable(map, chip, cmd_adr);
 		printk(KERN_ERR "%s: Chip not ready for buffer write. Xstatus = %lx, status = %lx\n",
 				map->name, Xstatus.x[0], status.x[0]);
@@ -1800,7 +1810,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 	}
 
 	/* Write length of data to come */
-	map_write(map, CMD(words), cmd_adr );
+	cfi_write_cmd(map, words, cmd_adr);
 
 	/* Write data */
 	vec = *pvec;
@@ -1837,7 +1847,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 	*pvec_seek = vec_seek;
 
 	/* GO GO GO */
-	map_write(map, CMD(0xd0), cmd_adr);
+	cfi_write_cmd(map, 0xd0, cmd_adr);
 	chip->state = FL_WRITING;
 
 	ret = INVAL_CACHE_AND_WAIT(map, chip, cmd_adr,
@@ -1845,7 +1855,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 				   chip->buffer_write_time,
 				   chip->buffer_write_time_max);
 	if (ret) {
-		map_write(map, CMD(0x70), cmd_adr);
+		cfi_write_cmd(map, 0x70, cmd_adr);
 		chip->state = FL_STATUS;
 		xip_enable(map, chip, cmd_adr);
 		printk(KERN_ERR "%s: buffer write error (status timeout)\n", map->name);
@@ -1858,8 +1868,8 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 		unsigned long chipstatus = MERGESTATUS(status);
 
 		/* reset status */
-		map_write(map, CMD(0x50), cmd_adr);
-		map_write(map, CMD(0x70), cmd_adr);
+		cfi_write_cmd(map, 0x50, cmd_adr);
+		cfi_write_cmd(map, 0x70, cmd_adr);
 		xip_enable(map, chip, cmd_adr);
 
 		if (chipstatus & 0x02) {
@@ -2510,10 +2520,7 @@ static void cfi_intelext_save_locks(struct mtd_info *mtd)
 
 			status = cfi_varsize_frob(mtd,
 					do_getlockstatus_oneblock, adr, len, NULL);
-			if (status)
-				set_bit(block, region->lockmap);
-			else
-				clear_bit(block, region->lockmap);
+			assign_bit(block, region->lockmap, status);
 		}
 	}
 }
diff --git a/drivers/mtd/chips/cfi_cmdset_0002.c b/drivers/mtd/chips/cfi_cmdset_0002.c
index 517db2f2707f48..100cab95a17b16 100644
--- a/drivers/mtd/chips/cfi_cmdset_0002.c
+++ b/drivers/mtd/chips/cfi_cmdset_0002.c
@@ -283,6 +283,21 @@ static void fixup_use_write_buffers(struct mtd_info *mtd)
 		pr_debug("Using buffer write method\n");
 		mtd->_write = cfi_amdstd_write_buffers;
 	}
+
+	/*
+	 * The word count of the Write to Buffer command is a single bus
+	 * word per device, so an x8 device can be told to program at most
+	 * 256 bytes however large a buffer it advertises - including when
+	 * several of them are interleaved on a wider bus, where CMD()
+	 * replicates the count into each device's lane and it is truncated
+	 * there.
+	 */
+	if (cfi->device_type == CFI_DEVICETYPE_X8 &&
+	    cfi->cfiq->MaxBufWriteSize > 8) {
+		cfi->cfiq->MaxBufWriteSize = 8;
+		mtd->writebufsize = cfi_interleave(cfi) <<
+				    cfi->cfiq->MaxBufWriteSize;
+	}
 }
 #endif /* !FORCE_WORD_WRITE */
 
diff --git a/drivers/mtd/devices/block2mtd.c b/drivers/mtd/devices/block2mtd.c
index 03e80b2c4f5a72..349fa07be314cc 100644
--- a/drivers/mtd/devices/block2mtd.c
+++ b/drivers/mtd/devices/block2mtd.c
@@ -293,7 +293,7 @@ static struct block2mtd_dev *add_device(char *devname, int erase_size,
 	}
 
 	size = bdev_nr_bytes(bdev);
-	if ((long)size % erase_size) {
+	if (!erase_size || (long)size % erase_size) {
 		pr_err("erasesize must be a divisor of device size\n");
 		goto err_free_block2mtd;
 	}
diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c
index 603fd0efc2ea2d..3d89f810e2db2f 100644
--- a/drivers/mtd/devices/docg3.c
+++ b/drivers/mtd/devices/docg3.c
@@ -1884,7 +1884,7 @@ static void doc_release_device(struct mtd_info *mtd)
 
 /**
  * docg3_resume - Awakens docg3 floor
- * @pdev: platfrom device
+ * @pdev: platform device
  *
  * Returns 0 (always successful)
  */
@@ -1963,7 +1963,7 @@ static int docg3_suspend(struct platform_device *pdev, pm_message_t state)
  * @pdev: platform device
  *
  * Probes for a G3 chip at the specified IO space in the platform data
- * ressources. The floor 0 must be available.
+ * resources. The floor 0 must be available.
  *
  * Returns 0 on success, -ENOMEM, -ENXIO on error
  */
@@ -1971,25 +1971,15 @@ static int __init docg3_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct mtd_info *mtd;
-	struct resource *ress;
 	void __iomem *base;
 	int ret, floor;
 	struct docg3_cascade *cascade;
 
-	ret = -ENXIO;
-	ress = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!ress) {
-		dev_err(dev, "No I/O memory resource defined\n");
-		return ret;
-	}
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
 
 	ret = -ENOMEM;
-	base = devm_ioremap(dev, ress->start, DOC_IOSPACE_SIZE);
-	if (!base) {
-		dev_err(dev, "devm_ioremap dev failed\n");
-		return ret;
-	}
-
 	cascade = devm_kcalloc(dev, DOC_MAX_NBFLOORS, sizeof(*cascade),
 			       GFP_KERNEL);
 	if (!cascade)
diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c
index a566e86eb5e3a0..55a945c1a67add 100644
--- a/drivers/mtd/devices/mtd_intel_dg.c
+++ b/drivers/mtd/devices/mtd_intel_dg.c
@@ -387,7 +387,7 @@ idg_erase(struct intel_dg_nvm *nvm, u8 region, loff_t from, u64 len, u64 *fail_a
 	void __iomem *base2 = nvm->base2;
 	void __iomem *base = nvm->base;
 	const u32 block = 0x10;
-	u32 iter = 0;
+	u32 iter;
 	u32 reg;
 	u64 i;
 
@@ -396,6 +396,7 @@ idg_erase(struct intel_dg_nvm *nvm, u8 region, loff_t from, u64 len, u64 *fail_a
 		iowrite32(region << 24 | block, base + NVM_ERASE_REG);
 		if (nvm->non_posted_erase) {
 			/* Wait for Erase Done */
+			iter = 0;
 			reg = ioread32(base2 + NVM_DEBUG_REG);
 			while (!(reg & NVM_NON_POSTED_ERASE_DONE) &&
 			       ++iter < NVM_NON_POSTED_ERASE_DONE_ITER) {
diff --git a/drivers/mtd/devices/powernv_flash.c b/drivers/mtd/devices/powernv_flash.c
index a12427d9e20fa3..fe1d6684d0bd40 100644
--- a/drivers/mtd/devices/powernv_flash.c
+++ b/drivers/mtd/devices/powernv_flash.c
@@ -263,7 +263,7 @@ static int powernv_flash_probe(struct platform_device *pdev)
 }
 
 /**
- * op_release - Release the driver
+ * powernv_flash_release - Release the driver
  * @pdev: the platform device
  *
  * Returns 0
diff --git a/drivers/mtd/hyperbus/hbmc-am654.c b/drivers/mtd/hyperbus/hbmc-am654.c
index 9d31464046b221..8a14e908d0b539 100644
--- a/drivers/mtd/hyperbus/hbmc-am654.c
+++ b/drivers/mtd/hyperbus/hbmc-am654.c
@@ -78,14 +78,15 @@ static int am654_hbmc_dma_read(struct am654_hbmc_device_priv *priv, void *to,
 	struct dma_chan *rx_chan = priv->rx_chan;
 	struct dma_async_tx_descriptor *tx;
 	dma_addr_t dma_dst, dma_src;
+	struct device *rx_dev = dmaengine_get_dma_device(rx_chan);
 	dma_cookie_t cookie;
 	int ret;
 
 	if (!priv->rx_chan || !virt_addr_valid(to) || object_is_on_stack(to))
 		return -EINVAL;
 
-	dma_dst = dma_map_single(rx_chan->device->dev, to, len, DMA_FROM_DEVICE);
-	if (dma_mapping_error(rx_chan->device->dev, dma_dst)) {
+	dma_dst = dma_map_single(rx_dev, to, len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(rx_dev, dma_dst)) {
 		dev_dbg(priv->ctlr->dev, "DMA mapping failed\n");
 		return -EIO;
 	}
@@ -117,7 +118,7 @@ static int am654_hbmc_dma_read(struct am654_hbmc_device_priv *priv, void *to,
 	}
 
 unmap_dma:
-	dma_unmap_single(rx_chan->device->dev, dma_dst, len, DMA_FROM_DEVICE);
+	dma_unmap_single(rx_dev, dma_dst, len, DMA_FROM_DEVICE);
 	return ret;
 }
 
diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig
index 1bb3dba2631d8d..649fd145e15dc7 100644
--- a/drivers/mtd/maps/Kconfig
+++ b/drivers/mtd/maps/Kconfig
@@ -161,6 +161,23 @@ config MTD_AMD76XROM
 
 	  BE VERY CAREFUL.
 
+config MTD_INT0800
+	tristate "Read-only BIOS/firmware flash via ACPI INT0800"
+	depends on X86 && ACPI
+	select MTD_ROM
+	help
+	  Support for reading the system firmware (BIOS/EFI) flash chip
+	  through the memory window described by the ACPI INT0800
+	  "82802 firmware hub" device, present on most x86 machines.
+
+	  The flash is exposed read-only via the ROM chip driver, e.g.
+	  for firmware analysis or extracting option ROMs (such as
+	  video BIOS images embedded in EFI firmware volumes). There is
+	  no write or erase support.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called int0800.
+
 config MTD_ICHXROM
 	tristate "BIOS flash chip on Intel Controller Hub 2/3/4/5"
 	depends on X86 && MTD_JEDECPROBE
diff --git a/drivers/mtd/maps/Makefile b/drivers/mtd/maps/Makefile
index 01745eca1f7303..2f1a7375e13cdf 100644
--- a/drivers/mtd/maps/Makefile
+++ b/drivers/mtd/maps/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_MTD_L440GX)	+= l440gx.o
 obj-$(CONFIG_MTD_AMD76XROM)	+= amd76xrom.o
 obj-$(CONFIG_MTD_ESB2ROM)	+= esb2rom.o
 obj-$(CONFIG_MTD_ICHXROM)	+= ichxrom.o
+obj-$(CONFIG_MTD_INT0800)	+= int0800.o
 obj-$(CONFIG_MTD_CK804XROM)	+= ck804xrom.o
 obj-$(CONFIG_MTD_TSUNAMI)	+= tsunami_flash.o
 obj-$(CONFIG_MTD_PXA2XX)	+= pxa2xx-flash.o
diff --git a/drivers/mtd/maps/amd76xrom.c b/drivers/mtd/maps/amd76xrom.c
index 457e6ab7f3fb16..0c3e0eb08b8d8d 100644
--- a/drivers/mtd/maps/amd76xrom.c
+++ b/drivers/mtd/maps/amd76xrom.c
@@ -152,6 +152,8 @@ static int amd76xrom_init_one(struct pci_dev *pdev,
 		printk(KERN_ERR MOD_NAME
 		       " %s(): Unable to register resource %pR - kernel bug?\n",
 		       __func__, &window->rsrc);
+		pci_dev_put(window->pdev);
+		window->pdev = NULL;
 		return -EBUSY;
 	}
 
diff --git a/drivers/mtd/maps/int0800.c b/drivers/mtd/maps/int0800.c
new file mode 100644
index 00000000000000..c5f108aa347974
--- /dev/null
+++ b/drivers/mtd/maps/int0800.c
@@ -0,0 +1,115 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Read-only MTD access to the system firmware flash behind the ACPI
+ * INT0800 "Intel 82802 firmware hub" device.
+ *
+ * On x86 systems the boot flash is decoded into the physical address
+ * space below 4 GB, so a plain ioremap() is sufficient to read it -
+ * no SPI or LPC controller access is required.  The declared _CRS
+ * window may be larger than the real flash (the whole top-16MiB
+ * decode range is commonly claimed); undecoded holes read as 0xff.
+ *
+ * The device is exposed read-only via the ROM chip driver; there is
+ * deliberately no write or erase support.
+ */
+
+#include <linux/module.h>
+#include <linux/acpi.h>
+#include <linux/io.h>
+#include <linux/mtd/mtd.h>
+#include <linux/mtd/map.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/* top-of-4GB firmware decode, used when _CRS reports no window */
+#define INT0800_DEFAULT_PHYS	0xffe00000UL
+#define INT0800_DEFAULT_SIZE	SZ_2M
+
+struct int0800 {
+	struct map_info		map;
+	struct mtd_info		*mtd;
+};
+
+static int int0800_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct int0800 *fw;
+	resource_size_t end;
+	int ret;
+
+	fw = devm_kzalloc(&pdev->dev, sizeof(*fw), GFP_KERNEL);
+	if (!fw)
+		return -ENOMEM;
+
+	fw->map.name = dev_name(&pdev->dev);
+	fw->map.bankwidth = 1;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (res) {
+		fw->map.phys = res->start;
+		fw->map.size = resource_size(res);
+	} else {
+		fw->map.phys = INT0800_DEFAULT_PHYS;
+		fw->map.size = INT0800_DEFAULT_SIZE;
+	}
+
+	/*
+	 * Plain ioremap on purpose: the window is already claimed by the
+	 * ACPI/pnp resource reservation, so devm_ioremap_resource() would
+	 * fail with -EBUSY.
+	 */
+	fw->map.virt = devm_ioremap(&pdev->dev, fw->map.phys, fw->map.size);
+	if (!fw->map.virt)
+		return -ENOMEM;
+
+	simple_map_init(&fw->map);
+	fw->mtd = do_map_probe("map_rom", &fw->map);
+	if (!fw->mtd)
+		return -ENODEV;
+	fw->mtd->dev.parent = &pdev->dev;
+	platform_set_drvdata(pdev, fw);
+
+	end = fw->map.phys + fw->map.size - 1;
+	dev_info(&pdev->dev, "mapped firmware window %pa-%pa\n",
+		 &fw->map.phys, &end);
+
+	ret = mtd_device_register(fw->mtd, NULL, 0);
+	if (ret)
+		map_destroy(fw->mtd);
+	return ret;
+}
+
+static void int0800_remove(struct platform_device *pdev)
+{
+	struct int0800 *fw = platform_get_drvdata(pdev);
+
+	mtd_device_unregister(fw->mtd);
+	map_destroy(fw->mtd);
+}
+
+static const struct acpi_device_id int0800_ids[] = {
+	{ "INT0800", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, int0800_ids);
+
+static struct platform_driver int0800_driver = {
+	.probe	= int0800_probe,
+	.remove	= int0800_remove,
+	.driver	= {
+		.name			= "int0800",
+		.acpi_match_table	= int0800_ids,
+		/*
+		 * remove() frees the mtd_info and unmaps the window even
+		 * while /dev/mtdX fds are still open; the only safe
+		 * teardown path is module unload, which open fds block
+		 * via mtd->owner.
+		 */
+		.suppress_bind_attrs	= true,
+	},
+};
+module_platform_driver(int0800_driver);
+
+MODULE_AUTHOR("Stephen Bancroft <stevereaver@gmail.com>");
+MODULE_DESCRIPTION("Read-only MTD map over the INT0800 firmware flash window");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mtd/maps/l440gx.c b/drivers/mtd/maps/l440gx.c
index 832b880d1aaf82..1df5ea5fafcac1 100644
--- a/drivers/mtd/maps/l440gx.c
+++ b/drivers/mtd/maps/l440gx.c
@@ -110,8 +110,6 @@ static int __init init_l440gx(void)
 
 		/* Allocate the resource region */
 		if (pci_assign_resource(pm_dev, PIIXE_IOBASE_RESOURCE) != 0) {
-			pci_dev_put(dev);
-			pci_dev_put(pm_dev);
 			printk(KERN_WARNING "Could not allocate pm iobase resource\n");
 			iounmap(l440gx_map.virt);
 			return -ENXIO;
diff --git a/drivers/mtd/maps/pci.c b/drivers/mtd/maps/pci.c
index 34041cc8f173a4..f13a1f520d0434 100644
--- a/drivers/mtd/maps/pci.c
+++ b/drivers/mtd/maps/pci.c
@@ -250,7 +250,7 @@ static int mtd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	struct mtd_info *mtd = NULL;
 	int err;
 
-	err = pci_enable_device(dev);
+	err = pcim_enable_device(dev);
 	if (err)
 		goto out;
 
diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c
index 4d2e7b7774e90f..42f250344f0cd1 100644
--- a/drivers/mtd/mtd_blkdevs.c
+++ b/drivers/mtd/mtd_blkdevs.c
@@ -18,6 +18,7 @@
 #include <linux/spinlock.h>
 #include <linux/hdreg.h>
 #include <linux/mutex.h>
+#include <linux/sched/mm.h>
 #include <linux/uaccess.h>
 
 #include "mtdcore.h"
@@ -167,6 +168,7 @@ static blk_status_t mtd_queue_rq(struct blk_mq_hw_ctx *hctx,
 				 const struct blk_mq_queue_data *bd)
 {
 	struct mtd_blktrans_dev *dev;
+	unsigned int noio_flags;
 
 	dev = hctx->queue->queuedata;
 	if (!dev) {
@@ -174,10 +176,13 @@ static blk_status_t mtd_queue_rq(struct blk_mq_hw_ctx *hctx,
 		return BLK_STS_IOERR;
 	}
 
+	/* Reclaim must not recurse into I/O while processing requests. */
+	noio_flags = memalloc_noio_save();
 	spin_lock_irq(&dev->queue_lock);
 	list_add_tail(&bd->rq->queuelist, &dev->rq_list);
 	mtd_blktrans_work(dev);
 	spin_unlock_irq(&dev->queue_lock);
+	memalloc_noio_restore(noio_flags);
 
 	return BLK_STS_OK;
 }
diff --git a/drivers/mtd/mtd_virt_concat.c b/drivers/mtd/mtd_virt_concat.c
index 25cf33fe1ec18f..40b300021ddaa0 100644
--- a/drivers/mtd/mtd_virt_concat.c
+++ b/drivers/mtd/mtd_virt_concat.c
@@ -130,6 +130,7 @@ int mtd_virt_concat_destroy(struct mtd_info *mtd)
 			mtd_concat_destroy(&concat->mtd);
 		}
 
+		list_del(&item->head);
 		for (idx = 0; idx < item->count; idx++)
 			of_node_put(item->nodes[idx]);
 
@@ -201,6 +202,7 @@ void mtd_virt_concat_destroy_items(void)
 	int i;
 
 	list_for_each_entry_safe(item, temp, &concat_node_list, head) {
+		list_del(&item->head);
 		for (i = 0; i < item->count; i++)
 			of_node_put(item->nodes[i]);
 
diff --git a/drivers/mtd/mtdconcat.c b/drivers/mtd/mtdconcat.c
index d54d309c30a0cd..354cc71cd722d9 100644
--- a/drivers/mtd/mtdconcat.c
+++ b/drivers/mtd/mtdconcat.c
@@ -31,7 +31,7 @@
  * Given a pointer to the MTD object in the mtd_concat structure,
  * we can retrieve the pointer to that structure with this macro.
  */
-#define CONCAT(x)  ((struct mtd_concat *)(x))
+#define CONCAT(x)  container_of_const(x, struct mtd_concat, mtd)
 
 /*
  * MTD methods which look up the relevant subdevice, translate the
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 16629382a787bd..b170c3144ca8f0 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -563,7 +563,7 @@ int mtd_pairing_info_to_wunit(struct mtd_info *mtd,
 		return -EINVAL;
 
 	if (master->pairing && master->pairing->get_wunit)
-		return mtd->pairing->get_wunit(master, info);
+		return master->pairing->get_wunit(master, info);
 
 	return info->pair;
 }
@@ -1083,6 +1083,7 @@ static int mtd_otp_nvmem_add(struct mtd_info *mtd)
 
 err:
 	nvmem_unregister(mtd->otp_user_nvmem);
+	mtd->otp_user_nvmem = NULL;
 	/* Don't report error if OTP is not supported. */
 	if (err == -EOPNOTSUPP)
 		return 0;
@@ -1332,7 +1333,7 @@ int __get_mtd_device(struct mtd_info *mtd)
 	int err;
 
 	if (master->_get_device) {
-		err = master->_get_device(mtd);
+		err = master->_get_device(master);
 		if (err)
 			return err;
 	}
diff --git a/drivers/mtd/mtdsuper.c b/drivers/mtd/mtdsuper.c
index c709ff7aa6b575..02eede3fe582ef 100644
--- a/drivers/mtd/mtdsuper.c
+++ b/drivers/mtd/mtdsuper.c
@@ -31,8 +31,10 @@ static int mtd_get_sb(struct fs_context *fc,
 	int ret;
 
 	sb = sget_dev(fc, MKDEV(MTD_BLOCK_MAJOR, mtd->index));
-	if (IS_ERR(sb))
+	if (IS_ERR(sb)) {
+		put_mtd_device(mtd);
 		return PTR_ERR(sb);
+	}
 
 	if (sb->s_root) {
 		/* new mountpoint for an already mounted superblock */
diff --git a/drivers/mtd/nand/ecc.c b/drivers/mtd/nand/ecc.c
index 6ccdff3fc913be..1d809cbd47e6df 100644
--- a/drivers/mtd/nand/ecc.c
+++ b/drivers/mtd/nand/ecc.c
@@ -159,6 +159,13 @@ int nand_ecc_finish_io_req(struct nand_device *nand,
 }
 EXPORT_SYMBOL(nand_ecc_finish_io_req);
 
+bool nand_ecc_is_pipelined(const struct nand_device *nand)
+{
+	return nand->ecc.engine &&
+	       nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED;
+}
+EXPORT_SYMBOL(nand_ecc_is_pipelined);
+
 /* Define default OOB placement schemes for large and small page devices */
 static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section,
 				 struct mtd_oob_region *oobregion)
diff --git a/drivers/mtd/nand/onenand/onenand_base.c b/drivers/mtd/nand/onenand/onenand_base.c
index d08aeac86f9fb2..bca5a3bebd41db 100644
--- a/drivers/mtd/nand/onenand/onenand_base.c
+++ b/drivers/mtd/nand/onenand/onenand_base.c
@@ -3279,7 +3279,7 @@ static void onenand_check_features(struct mtd_info *mtd)
 			 *
 			 * Now it has as following
 			 * KFM4G16Q4M has NOP 4 with version ID 0x0131
-			 * KFM4G16Q5M has NOP 1 with versoin ID 0x013e
+			 * KFM4G16Q5M has NOP 1 with version ID 0x013e
 			 */
 			if ((this->version_id & 0xf) == 0xe)
 				this->options |= ONENAND_HAS_NOP_1;
diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c
index 8a19408abb6355..0bdbcbb50d94c3 100644
--- a/drivers/mtd/nand/raw/atmel/nand-controller.c
+++ b/drivers/mtd/nand/raw/atmel/nand-controller.c
@@ -2085,7 +2085,7 @@ static int atmel_nand_controller_init(struct atmel_nand_controller *nc,
 	if (!np) {
 		dev_err(dev, "Missing or invalid atmel,smc property\n");
 		ret = -EINVAL;
-		goto out_release_dma;
+		goto out_put_mck;
 	}
 
 	nc->smc = syscon_node_to_regmap(np);
@@ -2093,11 +2093,13 @@ static int atmel_nand_controller_init(struct atmel_nand_controller *nc,
 	if (IS_ERR(nc->smc)) {
 		ret = PTR_ERR(nc->smc);
 		dev_err(dev, "Could not get SMC regmap (err = %d)\n", ret);
-		goto out_release_dma;
+		goto out_put_mck;
 	}
 
 	return 0;
 
+out_put_mck:
+	clk_put(nc->mck);
 out_release_dma:
 	if (nc->dmac)
 		dma_release_channel(nc->dmac);
diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
index 5b9dadd5405e3b..4a1f5e736a1cf5 100644
--- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c
+++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
@@ -3272,6 +3272,8 @@ static int brcmnand_edu_setup(struct platform_device *pdev)
 
 		ctrl->edu_irq = platform_get_irq_optional(pdev, 1);
 		if (ctrl->edu_irq < 0) {
+			if (ctrl->edu_irq != -ENXIO)
+				return ctrl->edu_irq;
 			dev_warn(dev,
 				 "FLASH EDU enabled, using ctlrdy irq\n");
 		} else {
diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c
index d53b35a8b3cb24..4f5e6d2b128c74 100644
--- a/drivers/mtd/nand/raw/cadence-nand-controller.c
+++ b/drivers/mtd/nand/raw/cadence-nand-controller.c
@@ -3143,6 +3143,9 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
 		goto free_buf_desc;
 	}
 
+	spin_lock_init(&cdns_ctrl->irq_lock);
+	init_completion(&cdns_ctrl->complete);
+
 	if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr,
 			     IRQF_SHARED, "cadence-nand-controller",
 			     cdns_ctrl)) {
@@ -3151,9 +3154,6 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl)
 		goto free_buf;
 	}
 
-	spin_lock_init(&cdns_ctrl->irq_lock);
-	init_completion(&cdns_ctrl->complete);
-
 	ret = cadence_nand_hw_init(cdns_ctrl);
 	if (ret)
 		goto disable_irq;
@@ -3229,7 +3229,7 @@ static void cadence_nand_remove(struct cdns_nand_ctrl *cdns_ctrl)
 {
 	cadence_nand_chips_cleanup(cdns_ctrl);
 	if (cdns_ctrl->dmac)
-		dma_unmap_resource(cdns_ctrl->dmac->device->dev,
+		dma_unmap_resource(dmaengine_get_dma_device(cdns_ctrl->dmac),
 				   cdns_ctrl->io.iova_dma, cdns_ctrl->io.size,
 				   DMA_BIDIRECTIONAL, 0);
 	cadence_nand_irq_cleanup(cdns_ctrl->irq, cdns_ctrl);
diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c
index a88ac2cfaccdcc..be6bbc1c184d2c 100644
--- a/drivers/mtd/nand/raw/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c
@@ -12,7 +12,6 @@
 #include <linux/platform_device.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
-#include <linux/of_address.h>
 #include <linux/slab.h>
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/rawnand.h>
@@ -965,13 +964,6 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv)
 
 static int fsl_ifc_chip_remove(struct fsl_ifc_mtd *priv)
 {
-	struct mtd_info *mtd = nand_to_mtd(&priv->chip);
-
-	kfree(mtd->name);
-
-	if (priv->vbase)
-		iounmap(priv->vbase);
-
 	ifc_nand_ctrl->chips[priv->bank] = NULL;
 
 	return 0;
@@ -996,12 +988,12 @@ static int fsl_ifc_nand_probe(struct platform_device *dev)
 {
 	struct fsl_ifc_runtime __iomem *ifc;
 	struct fsl_ifc_mtd *priv;
-	struct resource res;
+	struct resource *res;
+	void __iomem *vbase;
 	static const char *part_probe_types[]
 		= { "cmdlinepart", "RedBoot", "ofpart", NULL };
 	int ret;
 	int bank;
-	struct device_node *node = dev->dev.of_node;
 	struct mtd_info *mtd;
 
 	if (!fsl_ifc_ctrl_dev || !fsl_ifc_ctrl_dev->rregs)
@@ -1009,15 +1001,13 @@ static int fsl_ifc_nand_probe(struct platform_device *dev)
 	ifc = fsl_ifc_ctrl_dev->rregs;
 
 	/* get, allocate and map the memory resource */
-	ret = of_address_to_resource(node, 0, &res);
-	if (ret) {
-		dev_err(&dev->dev, "%s: failed to get resource\n", __func__);
-		return ret;
-	}
+	vbase = devm_platform_get_and_ioremap_resource(dev, 0, &res);
+	if (IS_ERR(vbase))
+		return PTR_ERR(vbase);
 
 	/* find which chip select it is connected to */
 	for (bank = 0; bank < fsl_ifc_ctrl_dev->banks; bank++) {
-		if (match_bank(fsl_ifc_ctrl_dev->gregs, bank, res.start))
+		if (match_bank(fsl_ifc_ctrl_dev->gregs, bank, res->start))
 			break;
 	}
 
@@ -1054,13 +1044,7 @@ static int fsl_ifc_nand_probe(struct platform_device *dev)
 	priv->bank = bank;
 	priv->ctrl = fsl_ifc_ctrl_dev;
 	priv->dev = &dev->dev;
-
-	priv->vbase = ioremap(res.start, resource_size(&res));
-	if (!priv->vbase) {
-		dev_err(priv->dev, "%s: failed to map chip region\n", __func__);
-		ret = -ENOMEM;
-		goto err;
-	}
+	priv->vbase = vbase;
 
 	dev_set_drvdata(priv->dev, priv);
 
@@ -1076,7 +1060,7 @@ static int fsl_ifc_nand_probe(struct platform_device *dev)
 		  &ifc->ifc_nand.nand_evter_intr_en);
 
 	mtd = nand_to_mtd(&priv->chip);
-	mtd->name = kasprintf(GFP_KERNEL, "%llx.flash", (u64)res.start);
+	mtd->name = devm_kasprintf(priv->dev, GFP_KERNEL, "%llx.flash", (u64)res->start);
 	if (!mtd->name) {
 		ret = -ENOMEM;
 		goto err;
@@ -1098,7 +1082,7 @@ static int fsl_ifc_nand_probe(struct platform_device *dev)
 		goto cleanup_nand;
 
 	dev_info(priv->dev, "IFC NAND device at 0x%llx, bank %d\n",
-		 (unsigned long long)res.start, priv->bank);
+		 (unsigned long long)res->start, priv->bank);
 
 	return 0;
 
diff --git a/drivers/mtd/nand/raw/fsmc_nand.c b/drivers/mtd/nand/raw/fsmc_nand.c
index b13b2b0c3f300c..08ef4425559c33 100644
--- a/drivers/mtd/nand/raw/fsmc_nand.c
+++ b/drivers/mtd/nand/raw/fsmc_nand.c
@@ -461,7 +461,7 @@ static int fsmc_correct_ecc1(struct nand_chip *chip,
 				      chip->ecc.size, sm_order);
 }
 
-/* Count the number of 0's in buff upto a max of max_bits */
+/* Count the number of 0's in buff up to a max of max_bits */
 static int count_written_bits(u8 *buff, int size, int max_bits)
 {
 	int k, written_bits = 0;
diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
index 527165ccc839da..0f1528710a6a17 100644
--- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
+++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c
@@ -1387,7 +1387,7 @@ static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this)
 		goto error_alloc;
 
 	this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size,
-					&this->auxiliary_phys, GFP_DMA);
+					&this->auxiliary_phys, GFP_KERNEL);
 	if (!this->auxiliary_virt)
 		goto error_alloc;
 
@@ -2271,7 +2271,7 @@ static int nand_boot_init(struct gpmi_nand_data  *this)
 {
 	nand_boot_set_geometry(this);
 
-	/* This is ROM arch-specific initilization before the BBT scanning. */
+	/* This is ROM arch-specific initialization before the BBT scanning. */
 	if (GPMI_IS_MX23(this))
 		return mx23_boot_init(this);
 	return 0;
diff --git a/drivers/mtd/nand/raw/intel-nand-controller.c b/drivers/mtd/nand/raw/intel-nand-controller.c
index 01cefdaf115d13..0a4f5ff111334c 100644
--- a/drivers/mtd/nand/raw/intel-nand-controller.c
+++ b/drivers/mtd/nand/raw/intel-nand-controller.c
@@ -290,6 +290,7 @@ static int ebu_dma_start(struct ebu_nand_controller *ebu_host, u32 dir,
 	struct dma_async_tx_descriptor *tx;
 	struct completion *dma_completion;
 	dma_async_tx_callback callback;
+	struct device *dma_dev;
 	struct dma_chan *chan;
 	dma_cookie_t cookie;
 	unsigned long flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
@@ -307,8 +308,9 @@ static int ebu_dma_start(struct ebu_nand_controller *ebu_host, u32 dir,
 		callback = ebu_dma_tx_callback;
 	}
 
-	buf_dma = dma_map_single(chan->device->dev, (void *)buf, len, dir);
-	if (dma_mapping_error(chan->device->dev, buf_dma)) {
+	dma_dev = dmaengine_get_dma_device(chan);
+	buf_dma = dma_map_single(dma_dev, (void *)buf, len, dir);
+	if (dma_mapping_error(dma_dev, buf_dma)) {
 		dev_err(ebu_host->dev, "Failed to map DMA buffer\n");
 		ret = -EIO;
 		goto err_unmap;
diff --git a/drivers/mtd/nand/raw/internals.h b/drivers/mtd/nand/raw/internals.h
index b7162ced9efa61..b8b29341b88cdf 100644
--- a/drivers/mtd/nand/raw/internals.h
+++ b/drivers/mtd/nand/raw/internals.h
@@ -19,6 +19,7 @@
 #define NAND_MFR_AMD		0x01
 #define NAND_MFR_ATO		0x9b
 #define NAND_MFR_EON		0x92
+/* ESMT and GigaDevice share the same manufacturer ID. */
 #define NAND_MFR_ESMT		0xc8
 #define NAND_MFR_FUJITSU	0x04
 #define NAND_MFR_HYNIX		0xad
diff --git a/drivers/mtd/nand/raw/loongson-nand-controller.c b/drivers/mtd/nand/raw/loongson-nand-controller.c
index 8490412d5be191..935f66602de97c 100644
--- a/drivers/mtd/nand/raw/loongson-nand-controller.c
+++ b/drivers/mtd/nand/raw/loongson-nand-controller.c
@@ -391,7 +391,7 @@ static void loongson_nand_dma_callback(void *data)
 {
 	struct loongson_nand_host *host = (struct loongson_nand_host *)data;
 	struct dma_chan *chan = host->dma_chan;
-	struct device *dev = chan->device->dev;
+	struct device *dev = dmaengine_get_dma_device(chan);
 	enum dma_status status;
 
 	status = dmaengine_tx_status(chan, host->dma_cookie, NULL);
@@ -407,7 +407,7 @@ static int loongson_nand_dma_transfer(struct loongson_nand_host *host, struct lo
 {
 	struct nand_chip *chip = &host->chip;
 	struct dma_chan *chan = host->dma_chan;
-	struct device *dev = chan->device->dev;
+	struct device *dev = dmaengine_get_dma_device(chan);
 	struct dma_async_tx_descriptor *desc;
 	enum dma_data_direction data_dir = op->is_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 	enum dma_transfer_direction xfer_dir = op->is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
diff --git a/drivers/mtd/nand/raw/lpc32xx_mlc.c b/drivers/mtd/nand/raw/lpc32xx_mlc.c
index 8f6a89d9ba83fe..d03486dc411502 100644
--- a/drivers/mtd/nand/raw/lpc32xx_mlc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_mlc.c
@@ -26,7 +26,6 @@
 #include <linux/interrupt.h>
 #include <linux/of.h>
 #include <linux/gpio/consumer.h>
-#include <linux/mtd/lpc32xx_mlc.h>
 #include <linux/io.h>
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
@@ -174,7 +173,6 @@ static struct nand_bbt_descr lpc32xx_nand_bbt_mirror = {
 struct lpc32xx_nand_host {
 	struct platform_device	*pdev;
 	struct nand_chip	nand_chip;
-	struct lpc32xx_mlc_platform_data *pdata;
 	struct clk		*clk;
 	struct gpio_desc	*wp_gpio;
 	void __iomem		*io_base;
@@ -394,6 +392,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len,
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct lpc32xx_nand_host *host = nand_get_controller_data(chip);
+	struct device *dma_dev = dmaengine_get_dma_device(host->dma_chan);
 	struct dma_async_tx_descriptor *desc;
 	int flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 	unsigned long time_left;
@@ -401,8 +400,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len,
 
 	sg_init_one(&host->sgl, mem, len);
 
-	res = dma_map_sg(host->dma_chan->device->dev, &host->sgl, 1,
-			 DMA_BIDIRECTIONAL);
+	res = dma_map_sg(dma_dev, &host->sgl, 1, DMA_BIDIRECTIONAL);
 	if (res != 1) {
 		dev_err(mtd->dev.parent, "Failed to map sg list\n");
 		return -ENXIO;
@@ -430,12 +428,10 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, void *mem, int len,
 		goto out1;
 	}
 
-	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
-		     DMA_BIDIRECTIONAL);
+	dma_unmap_sg(dma_dev, &host->sgl, 1, DMA_BIDIRECTIONAL);
 	return 0;
 out1:
-	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
-		     DMA_BIDIRECTIONAL);
+	dma_unmap_sg(dma_dev, &host->sgl, 1, DMA_BIDIRECTIONAL);
 	return res;
 }
 
@@ -580,24 +576,11 @@ static void lpc32xx_ecc_enable(struct nand_chip *chip, int mode)
 static int lpc32xx_dma_setup(struct lpc32xx_nand_host *host)
 {
 	struct mtd_info *mtd = nand_to_mtd(&host->nand_chip);
-	dma_cap_mask_t mask;
 
 	host->dma_chan = dma_request_chan(mtd->dev.parent, "rx-tx");
 	if (IS_ERR(host->dma_chan)) {
-		/* fallback to request using platform data */
-		if (!host->pdata || !host->pdata->dma_filter) {
-			dev_err(mtd->dev.parent, "no DMA platform data\n");
-			return -ENOENT;
-		}
-
-		dma_cap_zero(mask);
-		dma_cap_set(DMA_SLAVE, mask);
-		host->dma_chan = dma_request_channel(mask, host->pdata->dma_filter, "nand-mlc");
-
-		if (!host->dma_chan) {
-			dev_err(mtd->dev.parent, "Failed to request DMA channel\n");
-			return -EBUSY;
-		}
+		dev_err(mtd->dev.parent, "Failed to request DMA channel\n");
+		return PTR_ERR(host->dma_chan);
 	}
 
 	/*
@@ -736,8 +719,6 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 
 	gpiod_set_consumer_name(host->wp_gpio, "NAND WP");
 
-	host->pdata = dev_get_platdata(&pdev->dev);
-
 	/* link the private data structures */
 	nand_set_controller_data(nand_chip, host);
 	nand_set_flash_node(nand_chip, pdev->dev.of_node);
diff --git a/drivers/mtd/nand/raw/lpc32xx_slc.c b/drivers/mtd/nand/raw/lpc32xx_slc.c
index 10c8080207f495..638267d44360d4 100644
--- a/drivers/mtd/nand/raw/lpc32xx_slc.c
+++ b/drivers/mtd/nand/raw/lpc32xx_slc.c
@@ -25,7 +25,6 @@
 #include <linux/dmaengine.h>
 #include <linux/gpio/consumer.h>
 #include <linux/of.h>
-#include <linux/mtd/lpc32xx_slc.h>
 
 #define LPC32XX_MODNAME		"lpc32xx-nand"
 
@@ -213,7 +212,6 @@ struct lpc32xx_nand_cfg_slc {
 
 struct lpc32xx_nand_host {
 	struct nand_chip	nand_chip;
-	struct lpc32xx_slc_platform_data *pdata;
 	struct clk		*clk;
 	struct gpio_desc	*wp_gpio;
 	void __iomem		*io_base;
@@ -427,6 +425,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, dma_addr_t dma,
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct lpc32xx_nand_host *host = nand_get_controller_data(chip);
+	struct device *dma_dev = dmaengine_get_dma_device(host->dma_chan);
 	struct dma_async_tx_descriptor *desc;
 	int flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 	int res;
@@ -448,8 +447,7 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, dma_addr_t dma,
 
 	sg_init_one(&host->sgl, mem, len);
 
-	res = dma_map_sg(host->dma_chan->device->dev, &host->sgl, 1,
-			 DMA_BIDIRECTIONAL);
+	res = dma_map_sg(dma_dev, &host->sgl, 1, DMA_BIDIRECTIONAL);
 	if (res != 1) {
 		dev_err(mtd->dev.parent, "Failed to map sg list\n");
 		return -ENXIO;
@@ -477,13 +475,11 @@ static int lpc32xx_xmit_dma(struct mtd_info *mtd, dma_addr_t dma,
 		res = 0;
 	}
 
-	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
-		     DMA_BIDIRECTIONAL);
+	dma_unmap_sg(dma_dev, &host->sgl, 1, DMA_BIDIRECTIONAL);
 
 	return res;
 out1:
-	dma_unmap_sg(host->dma_chan->device->dev, &host->sgl, 1,
-		     DMA_BIDIRECTIONAL);
+	dma_unmap_sg(dma_dev, &host->sgl, 1, DMA_BIDIRECTIONAL);
 	return -ENXIO;
 }
 
@@ -727,24 +723,11 @@ static int lpc32xx_nand_write_page_raw_syndrome(struct nand_chip *chip,
 static int lpc32xx_nand_dma_setup(struct lpc32xx_nand_host *host)
 {
 	struct mtd_info *mtd = nand_to_mtd(&host->nand_chip);
-	dma_cap_mask_t mask;
 
 	host->dma_chan = dma_request_chan(mtd->dev.parent, "rx-tx");
 	if (IS_ERR(host->dma_chan)) {
-		/* fallback to request using platform data */
-		if (!host->pdata || !host->pdata->dma_filter) {
-			dev_err(mtd->dev.parent, "no DMA platform data\n");
-			return -ENOENT;
-		}
-
-		dma_cap_zero(mask);
-		dma_cap_set(DMA_SLAVE, mask);
-		host->dma_chan = dma_request_channel(mask, host->pdata->dma_filter, "nand-slc");
-
-		if (!host->dma_chan) {
-			dev_err(mtd->dev.parent, "Failed to request DMA channel\n");
-			return -EBUSY;
-		}
+		dev_err(mtd->dev.parent, "Failed to request DMA channel\n");
+		return PTR_ERR(host->dma_chan);
 	}
 
 	return 0;
@@ -873,8 +856,6 @@ static int lpc32xx_nand_probe(struct platform_device *pdev)
 
 	gpiod_set_consumer_name(host->wp_gpio, "NAND WP");
 
-	host->pdata = dev_get_platdata(&pdev->dev);
-
 	chip = &host->nand_chip;
 	mtd = nand_to_mtd(chip);
 	nand_set_controller_data(chip, host);
diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c
index 38b7eb5b992c85..e5fa5540e80a04 100644
--- a/drivers/mtd/nand/raw/marvell_nand.c
+++ b/drivers/mtd/nand/raw/marvell_nand.c
@@ -868,6 +868,7 @@ static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
 				     unsigned int len)
 {
 	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
+	struct device *dma_dev = dmaengine_get_dma_device(nfc->dma_chan);
 	struct dma_async_tx_descriptor *tx;
 	struct scatterlist sg;
 	dma_cookie_t cookie;
@@ -876,7 +877,7 @@ static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
 	marvell_nfc_enable_dma(nfc);
 	/* Prepare the DMA transfer */
 	sg_init_one(&sg, nfc->dma_buf, dma_len);
-	ret = dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
+	ret = dma_map_sg(dma_dev, &sg, 1, direction);
 	if (!ret) {
 		dev_err(nfc->dev, "Could not map DMA S/G list\n");
 		return -ENXIO;
@@ -888,7 +889,7 @@ static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
 				     DMA_PREP_INTERRUPT);
 	if (!tx) {
 		dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
-		dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
+		dma_unmap_sg(dma_dev, &sg, 1, direction);
 		return -ENXIO;
 	}
 
@@ -900,7 +901,7 @@ static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
 
 	dma_async_issue_pending(nfc->dma_chan);
 	ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
-	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
+	dma_unmap_sg(dma_dev, &sg, 1, direction);
 	marvell_nfc_disable_dma(nfc);
 	if (ret) {
 		dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
diff --git a/drivers/mtd/nand/raw/mtk_nand.c b/drivers/mtd/nand/raw/mtk_nand.c
index 21c7e110274693..c439b7814ebb37 100644
--- a/drivers/mtd/nand/raw/mtk_nand.c
+++ b/drivers/mtd/nand/raw/mtk_nand.c
@@ -42,7 +42,7 @@
 #define		CON_BRD			BIT(8)  /* burst  read */
 #define		CON_BWR			BIT(9)	/* burst  write */
 #define		CON_SEC_SHIFT		(12)
-/* Timming control register */
+/* Timing control register */
 #define NFI_ACCCON		(0x0C)
 #define NFI_INTR_EN		(0x10)
 #define		INTR_AHB_DONE_EN	BIT(6)
diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c
index a5b278ab9384fa..a225b10f814573 100644
--- a/drivers/mtd/nand/raw/nand_base.c
+++ b/drivers/mtd/nand/raw/nand_base.c
@@ -6205,7 +6205,7 @@ static int nand_scan_tail(struct nand_chip *chip)
 	/*
 	 * FIXME: some NAND manufacturer drivers expect the first die to be
 	 * selected when manufacturer->init() is called. They should be fixed
-	 * to explictly select the relevant die when interacting with the NAND
+	 * to explicitly select the relevant die when interacting with the NAND
 	 * chip.
 	 */
 	nand_select_target(chip, 0);
diff --git a/drivers/mtd/nand/raw/nand_esmt.c b/drivers/mtd/nand/raw/nand_esmt.c
index 4412c407aef3f7..117b110fc54ad7 100644
--- a/drivers/mtd/nand/raw/nand_esmt.c
+++ b/drivers/mtd/nand/raw/nand_esmt.c
@@ -3,6 +3,9 @@
  * Copyright (C) 2018 Toradex AG
  *
  * Author: Marcel Ziswiler <marcel.ziswiler@toradex.com>
+ *
+ * ESMT and GigaDevice share the same JEDEC manufacturer ID (0xc8), so this
+ * file handles chips from both vendors.
  */
 
 #include <linux/mtd/rawnand.h>
@@ -38,6 +41,35 @@ static void esmt_nand_decode_id(struct nand_chip *chip)
 	nanddev_set_ecc_requirements(base, &requirements);
 }
 
+/*
+ * GigaDevice GD9FU2G8F3A does not implement the timing mode feature:
+ * GET_FEATURES does not report the mode that was written, and
+ * SET_FEATURES is ignored, even though the parameter page advertises
+ * support. The vendor confirmed the chip works at every advertised
+ * timing mode without being told, so unflag the feature and configure
+ * the host side only.
+ */
+static void esmt_nand_fix_broken_get_timings(struct nand_chip *chip)
+{
+	int i;
+	static const char * const broken_get_timings[] = {
+		"GD9FU2G8F3A",
+	};
+
+	if (!chip->parameters.supports_set_get_features)
+		return;
+
+	i = match_string(broken_get_timings, ARRAY_SIZE(broken_get_timings),
+			 chip->parameters.model);
+	if (i < 0)
+		return;
+
+	bitmap_clear(chip->parameters.get_feature_list,
+		     ONFI_FEATURE_ADDR_TIMING_MODE, 1);
+	bitmap_clear(chip->parameters.set_feature_list,
+		     ONFI_FEATURE_ADDR_TIMING_MODE, 1);
+}
+
 static int esmt_nand_init(struct nand_chip *chip)
 {
 	if (nand_is_slc(chip))
@@ -50,6 +82,8 @@ static int esmt_nand_init(struct nand_chip *chip)
 		chip->options |= NAND_BBM_FIRSTPAGE | NAND_BBM_SECONDPAGE |
 				 NAND_BBM_LASTPAGE;
 
+	esmt_nand_fix_broken_get_timings(chip);
+
 	return 0;
 }
 
diff --git a/drivers/mtd/nand/raw/nand_ids.c b/drivers/mtd/nand/raw/nand_ids.c
index 08707ce9d7c065..9b2dd246398fd2 100644
--- a/drivers/mtd/nand/raw/nand_ids.c
+++ b/drivers/mtd/nand/raw/nand_ids.c
@@ -185,7 +185,7 @@ static const struct nand_manufacturer_desc nand_manufacturer_descs[] = {
 	{NAND_MFR_AMD, "AMD/Spansion", &amd_nand_manuf_ops},
 	{NAND_MFR_ATO, "ATO"},
 	{NAND_MFR_EON, "Eon"},
-	{NAND_MFR_ESMT, "ESMT", &esmt_nand_manuf_ops},
+	{NAND_MFR_ESMT, "ESMT/GigaDevice", &esmt_nand_manuf_ops},
 	{NAND_MFR_FUJITSU, "Fujitsu"},
 	{NAND_MFR_HYNIX, "Hynix", &hynix_nand_manuf_ops},
 	{NAND_MFR_INTEL, "Intel"},
diff --git a/drivers/mtd/nand/raw/nand_legacy.c b/drivers/mtd/nand/raw/nand_legacy.c
index 97700f80d5b8f1..94d5c7723c5aa5 100644
--- a/drivers/mtd/nand/raw/nand_legacy.c
+++ b/drivers/mtd/nand/raw/nand_legacy.c
@@ -319,7 +319,7 @@ static void nand_command(struct nand_chip *chip, unsigned int command,
 				      NAND_CTRL_CLE | NAND_CTRL_CHANGE);
 		chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE,
 				      NAND_NCE | NAND_CTRL_CHANGE);
-		/* EZ-NAND can take upto 250ms as per ONFi v4.0 */
+		/* EZ-NAND can take up to 250ms as per ONFi v4.0 */
 		nand_wait_status_ready(chip, 250);
 		return;
 
@@ -459,7 +459,7 @@ static void nand_command_lp(struct nand_chip *chip, unsigned int command,
 				      NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE);
 		chip->legacy.cmd_ctrl(chip, NAND_CMD_NONE,
 				      NAND_NCE | NAND_CTRL_CHANGE);
-		/* EZ-NAND can take upto 250ms as per ONFi v4.0 */
+		/* EZ-NAND can take up to 250ms as per ONFi v4.0 */
 		nand_wait_status_ready(chip, 250);
 		return;
 
diff --git a/drivers/mtd/nand/raw/ndfc.c b/drivers/mtd/nand/raw/ndfc.c
index a48274297d3b12..d7b119028a7161 100644
--- a/drivers/mtd/nand/raw/ndfc.c
+++ b/drivers/mtd/nand/raw/ndfc.c
@@ -162,8 +162,8 @@ static int ndfc_chip_init(struct ndfc_controller *ndfc,
 		return -ENODEV;
 	nand_set_flash_node(chip, flash_np);
 
-	mtd->name = kasprintf(GFP_KERNEL, "%s.%pOFn", dev_name(&ndfc->ofdev->dev),
-			      flash_np);
+	mtd->name = devm_kasprintf(&ndfc->ofdev->dev, GFP_KERNEL, "%s.%pOFn",
+				   dev_name(&ndfc->ofdev->dev), flash_np);
 	if (!mtd->name) {
 		ret = -ENOMEM;
 		goto err;
@@ -177,18 +177,21 @@ static int ndfc_chip_init(struct ndfc_controller *ndfc,
 
 err:
 	of_node_put(flash_np);
-	if (ret)
-		kfree(mtd->name);
 	return ret;
 }
 
 static int ndfc_probe(struct platform_device *ofdev)
 {
 	struct ndfc_controller *ndfc;
+	void __iomem *ndfcbase;
 	const __be32 *reg;
 	u32 ccr;
 	u32 cs;
-	int err, len = 0;
+	int len = 0;
+
+	ndfcbase = devm_platform_ioremap_resource(ofdev, 0);
+	if (IS_ERR(ndfcbase))
+		return PTR_ERR(ndfcbase);
 
 	/* Read the reg property to get the chip select */
 	reg = of_get_property(ofdev->dev.of_node, "reg", &len);
@@ -210,11 +213,7 @@ static int ndfc_probe(struct platform_device *ofdev)
 	ndfc->ofdev = ofdev;
 	dev_set_drvdata(&ofdev->dev, ndfc);
 
-	ndfc->ndfcbase = of_iomap(ofdev->dev.of_node, 0);
-	if (!ndfc->ndfcbase) {
-		dev_err(&ofdev->dev, "failed to get memory\n");
-		return -EIO;
-	}
+	ndfc->ndfcbase = ndfcbase;
 
 	ccr = NDFC_CCR_BS(ndfc->chip_select);
 
@@ -232,13 +231,7 @@ static int ndfc_probe(struct platform_device *ofdev)
 		iowrite32be(be32_to_cpup(reg), ndfc->ndfcbase + offset);
 	}
 
-	err = ndfc_chip_init(ndfc, ofdev->dev.of_node);
-	if (err) {
-		iounmap(ndfc->ndfcbase);
-		return err;
-	}
-
-	return 0;
+	return ndfc_chip_init(ndfc, ofdev->dev.of_node);
 }
 
 static void ndfc_remove(struct platform_device *ofdev)
@@ -251,7 +244,6 @@ static void ndfc_remove(struct platform_device *ofdev)
 	ret = mtd_device_unregister(mtd);
 	WARN_ON(ret);
 	nand_cleanup(chip);
-	kfree(mtd->name);
 }
 
 static const struct of_device_id ndfc_match[] = {
diff --git a/drivers/mtd/nand/raw/nuvoton-ma35d1-nand-controller.c b/drivers/mtd/nand/raw/nuvoton-ma35d1-nand-controller.c
index 1a285cd8fad62a..f659d3a6b9ef46 100644
--- a/drivers/mtd/nand/raw/nuvoton-ma35d1-nand-controller.c
+++ b/drivers/mtd/nand/raw/nuvoton-ma35d1-nand-controller.c
@@ -993,7 +993,6 @@ static int ma35_nand_probe(struct platform_device *pdev)
 	ret = ma35_nand_chips_init(&pdev->dev, nand);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to init NAND chips\n");
-		clk_disable(nand->clk);
 		return ret;
 	}
 
diff --git a/drivers/mtd/nand/raw/omap2.c b/drivers/mtd/nand/raw/omap2.c
index 552f36da79fce4..4fa89aee934ebe 100644
--- a/drivers/mtd/nand/raw/omap2.c
+++ b/drivers/mtd/nand/raw/omap2.c
@@ -380,8 +380,8 @@ static inline int omap_nand_dma_transfer(struct nand_chip *chip,
 {
 	struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
 	struct dma_async_tx_descriptor *tx;
-	enum dma_data_direction dir = is_write ? DMA_TO_DEVICE :
-							DMA_FROM_DEVICE;
+	enum dma_data_direction dir = is_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct device *dma_dev = dmaengine_get_dma_device(info->dma);
 	struct scatterlist sg;
 	unsigned long tim, limit;
 	unsigned n;
@@ -392,7 +392,7 @@ static inline int omap_nand_dma_transfer(struct nand_chip *chip,
 		goto out_copy;
 
 	sg_init_one(&sg, addr, len);
-	n = dma_map_sg(info->dma->device->dev, &sg, 1, dir);
+	n = dma_map_sg(dma_dev, &sg, 1, dir);
 	if (n == 0) {
 		dev_err(&info->pdev->dev,
 			"Couldn't DMA map a %d byte buffer\n", len);
@@ -434,11 +434,11 @@ static inline int omap_nand_dma_transfer(struct nand_chip *chip,
 	/* disable and stop the PFPW engine */
 	omap_prefetch_reset(info->gpmc_cs, info);
 
-	dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
+	dma_unmap_sg(dma_dev, &sg, 1, dir);
 	return 0;
 
 out_copy_unmap:
-	dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
+	dma_unmap_sg(dma_dev, &sg, 1, dir);
 out_copy:
 	is_write == 0 ? omap_nand_data_in(chip, (void *)addr, len, false)
 		      : omap_nand_data_out(chip, addr, len, false);
@@ -666,9 +666,7 @@ static void gen_true_ecc(u8 *ecc_buf)
  * If there is no error, %0 is returned. If there is an error but it
  * was corrected, %1 is returned. Otherwise, %-1 is returned.
  */
-static int omap_compare_ecc(u8 *ecc_data1,	/* read from NAND memory */
-			    u8 *ecc_data2,	/* read from register */
-			    u8 *page_data)
+static int omap_compare_ecc(u8 *ecc_data1, u8 *ecc_data2, u8 *page_data)
 {
 	uint	i;
 	u8	tmp0_bit[8], tmp1_bit[8], tmp2_bit[8];
diff --git a/drivers/mtd/nand/raw/sh_flctl.c b/drivers/mtd/nand/raw/sh_flctl.c
index 97f733e481ffc2..0497801d7059fe 100644
--- a/drivers/mtd/nand/raw/sh_flctl.c
+++ b/drivers/mtd/nand/raw/sh_flctl.c
@@ -378,6 +378,7 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
 					int len, enum dma_data_direction dir)
 {
 	struct dma_async_tx_descriptor *desc = NULL;
+	struct device *dma_dev;
 	struct dma_chan *chan;
 	enum dma_transfer_direction tr_dir;
 	dma_addr_t dma_addr;
@@ -394,9 +395,10 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
 		tr_dir = DMA_MEM_TO_DEV;
 	}
 
-	dma_addr = dma_map_single(chan->device->dev, buf, len, dir);
+	dma_dev = dmaengine_get_dma_device(chan);
+	dma_addr = dma_map_single(dma_dev, buf, len, dir);
 
-	if (!dma_mapping_error(chan->device->dev, dma_addr))
+	if (!dma_mapping_error(dma_dev, dma_addr))
 		desc = dmaengine_prep_slave_single(chan, dma_addr, len,
 			tr_dir, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
@@ -440,7 +442,7 @@ static int flctl_dma_fifo0_transfer(struct sh_flctl *flctl, unsigned long *buf,
 	reg &= ~DREQ0EN;
 	writel(reg, FLINTDMACR(flctl));
 
-	dma_unmap_single(chan->device->dev, dma_addr, len, dir);
+	dma_unmap_single(dma_dev, dma_addr, len, dir);
 
 	/* ret == 0 is success */
 	return ret;
diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c
index c08d6b176372f6..9f42f8ab3212a8 100644
--- a/drivers/mtd/nand/raw/stm32_fmc2_nand.c
+++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c
@@ -2076,7 +2076,7 @@ static void stm32_fmc2_nfc_remove(struct platform_device *pdev)
 	stm32_fmc2_nfc_wp_enable(nand);
 }
 
-static int __maybe_unused stm32_fmc2_nfc_suspend(struct device *dev)
+static int stm32_fmc2_nfc_suspend(struct device *dev)
 {
 	struct stm32_fmc2_nfc *nfc = dev_get_drvdata(dev);
 	struct stm32_fmc2_nand *nand = &nfc->nand;
@@ -2090,7 +2090,7 @@ static int __maybe_unused stm32_fmc2_nfc_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused stm32_fmc2_nfc_resume(struct device *dev)
+static int stm32_fmc2_nfc_resume(struct device *dev)
 {
 	struct stm32_fmc2_nfc *nfc = dev_get_drvdata(dev);
 	struct stm32_fmc2_nand *nand = &nfc->nand;
@@ -2118,7 +2118,7 @@ static int __maybe_unused stm32_fmc2_nfc_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(stm32_fmc2_nfc_pm_ops, stm32_fmc2_nfc_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(stm32_fmc2_nfc_pm_ops, stm32_fmc2_nfc_suspend,
 			 stm32_fmc2_nfc_resume);
 
 static const struct stm32_fmc2_nfc_data stm32_fmc2_nfc_mp1_data = {
@@ -2153,7 +2153,7 @@ static struct platform_driver stm32_fmc2_nfc_driver = {
 	.driver	= {
 		.name = "stm32_fmc2_nfc",
 		.of_match_table = stm32_fmc2_nfc_match,
-		.pm = &stm32_fmc2_nfc_pm_ops,
+		.pm = pm_sleep_ptr(&stm32_fmc2_nfc_pm_ops),
 	},
 };
 module_platform_driver(stm32_fmc2_nfc_driver);
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index 45ccbce91551bd..f41feebe425715 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -222,6 +222,7 @@
  * USER_DATA_LEN registers.
  */
 #define USER_DATA_SZ 4
+#define SUNXI_NFC_MAX_USER_DATA_SZ 32
 
 /**
  * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
@@ -1003,11 +1004,10 @@ static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
 	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
 	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
 	unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, step);
-	u8 *user_data = NULL;
+	u8 user_data[SUNXI_NFC_MAX_USER_DATA_SZ] = {};
 
 	/* Randomize the Bad Block Marker. */
 	if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
-		user_data = kmalloc(user_data_sz, GFP_KERNEL);
 		memcpy(user_data, oob, user_data_sz);
 		sunxi_nfc_randomize_bbm(nand, page, user_data);
 		oob = user_data;
@@ -1040,7 +1040,6 @@ static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
 		}
 	}
 
-	kfree(user_data);
 }
 
 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
@@ -1991,7 +1990,7 @@ static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
 	struct nand_chip *nand = mtd_to_nand(mtd);
 	struct nand_ecc_ctrl *ecc = &nand->ecc;
 	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
-	unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, section);
+	unsigned int user_data_sz;
 
 	/*
 	 * The controller does not provide access to OOB bytes
@@ -2000,6 +1999,8 @@ static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
 	if (section >= ecc->steps)
 		return -ERANGE;
 
+	user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, section);
+
 	/*
 	 * The first 2 bytes are used for BB markers, hence we
 	 * only have user_data_sz - 2 bytes available in the first user data
@@ -2007,12 +2008,12 @@ static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
 	 */
 	if (section == 0) {
 		oobregion->offset = 2;
-		oobregion->length = user_data_sz - 2;
+		oobregion->length = user_data_sz > 2 ? user_data_sz - 2 : 0;
 
 		return 0;
 	}
 
-	oobregion->offset = sunxi_get_ecc_offset(sunxi_nand, ecc, section);
+	oobregion->offset = sunxi_get_oob_offset(sunxi_nand, ecc, section);
 	oobregion->length = user_data_sz;
 
 	return 0;
@@ -2041,6 +2042,9 @@ static int sunxi_nfc_maximize_user_data(struct nand_chip *nand, uint32_t oobsize
 	int remaining_bytes = oobsize - (ecc_bytes * nsectors);
 	int i, step;
 
+	if (nsectors <= 0)
+		return -EINVAL;
+
 	sunxi_nand->user_data_bytes = devm_kzalloc(nfc->dev, nsectors,
 						   GFP_KERNEL);
 	if (!sunxi_nand->user_data_bytes)
@@ -2056,6 +2060,8 @@ static int sunxi_nfc_maximize_user_data(struct nand_chip *nand, uint32_t oobsize
 		if (sunxi_nand->user_data_bytes[step] == 0)
 			break;
 	}
+	if (sunxi_nand->user_data_bytes[0] < USER_DATA_SZ)
+		return -EINVAL;
 
 	return 0;
 }
@@ -2072,11 +2078,15 @@ static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
 	int total_user_data_sz = 0;
 	int nsectors;
 	int ecc_mode;
+	int ret;
 	int i;
 
 	if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
 		int bytes = mtd->oobsize;
 
+		if (mtd->writesize < 1024)
+			return -EINVAL;
+
 		ecc->size = 1024;
 		nsectors = mtd->writesize / ecc->size;
 
@@ -2102,10 +2112,10 @@ static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
 			bytes -= total_user_data_sz;
 		} else {
 			/*
-			 * remove at least the BBM size before computing the
-			 * max ECC
+			 * User-data lengths are encoded in four-byte units. Reserve
+			 * the first word because it contains the two BBM bytes.
 			 */
-			bytes -= 2;
+			bytes -= USER_DATA_SZ;
 		}
 
 		/*
@@ -2169,9 +2179,12 @@ static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
 	 * The rationale for variable data length is to prioritize maximum ECC
 	 * strength, and then use the remaining space for user data.
 	 */
-	if (nfc->caps->reg_user_data_len)
-		sunxi_nfc_maximize_user_data(nand, mtd->oobsize, ecc->bytes,
-					     nsectors);
+	if (nfc->caps->reg_user_data_len) {
+		ret = sunxi_nfc_maximize_user_data(nand, mtd->oobsize,
+						   ecc->bytes, nsectors);
+		if (ret)
+			return ret;
+	}
 
 	if (total_user_data_sz == 0)
 		for (i = 0; i < nsectors; i++)
diff --git a/drivers/mtd/nand/raw/vf610_nfc.c b/drivers/mtd/nand/raw/vf610_nfc.c
index 9940681810cff2..1b9b370adfab61 100644
--- a/drivers/mtd/nand/raw/vf610_nfc.c
+++ b/drivers/mtd/nand/raw/vf610_nfc.c
@@ -505,6 +505,11 @@ static int vf610_nfc_exec_op(struct nand_chip *chip,
 				      check_only);
 }
 
+static unsigned int vf610_nfc_spare_size(struct mtd_info *mtd)
+{
+	return min_t(unsigned int, mtd->oobsize, 64);
+}
+
 static inline int vf610_nfc_correct_data(struct nand_chip *chip, uint8_t *dat,
 					 uint8_t *oob, int page)
 {
@@ -514,6 +519,7 @@ static inline int vf610_nfc_correct_data(struct nand_chip *chip, uint8_t *dat,
 	u8 ecc_status;
 	u8 ecc_count;
 	int flips_threshold = nfc->chip.ecc.strength / 2;
+	int ret;
 
 	ecc_status = vf610_nfc_read(nfc, ecc_status_off) & 0xff;
 	ecc_count = ecc_status & ECC_STATUS_ERR_COUNT;
@@ -521,16 +527,22 @@ static inline int vf610_nfc_correct_data(struct nand_chip *chip, uint8_t *dat,
 	if (!(ecc_status & ECC_STATUS_MASK))
 		return ecc_count;
 
+	/* The failed decode leaves a bogus correction in SRAM; re-read without ECC */
 	nfc->data_access = true;
-	nand_read_oob_op(&nfc->chip, page, 0, oob, mtd->oobsize);
+	ret = nand_read_page_op(&nfc->chip, page, 0, dat, mtd->writesize);
+	if (!ret)
+		ret = nand_read_oob_op(&nfc->chip, page, 0, oob,
+				       vf610_nfc_spare_size(mtd));
 	nfc->data_access = false;
+	if (ret)
+		return ret;
 
 	/*
-	 * On an erased page, bit count (including OOB) should be zero or
-	 * at least less then half of the ECC strength.
+	 * Run the erased-page check with the driver's historic threshold
+	 * of half the ECC strength.
 	 */
-	return nand_check_erased_ecc_chunk(dat, nfc->chip.ecc.size, oob,
-					   mtd->oobsize, NULL, 0,
+	return nand_check_erased_ecc_chunk(dat, mtd->writesize, oob,
+					   vf610_nfc_spare_size(mtd), NULL, 0,
 					   flips_threshold);
 }
 
@@ -551,7 +563,7 @@ static int vf610_nfc_read_page(struct nand_chip *chip, uint8_t *buf,
 {
 	struct vf610_nfc *nfc = chip_to_nfc(chip);
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	int trfr_sz = mtd->writesize + mtd->oobsize;
+	int trfr_sz = mtd->writesize + vf610_nfc_spare_size(mtd);
 	u32 row = 0, cmd1 = 0, cmd2 = 0, code = 0;
 	int stat;
 
@@ -577,11 +589,16 @@ static int vf610_nfc_read_page(struct nand_chip *chip, uint8_t *buf,
 	 */
 	vf610_nfc_rd_from_sram(buf, nfc->regs + NFC_MAIN_AREA(0),
 			       mtd->writesize, false);
-	if (oob_required)
+	if (oob_required) {
+		unsigned int spare = vf610_nfc_spare_size(mtd);
+
 		vf610_nfc_rd_from_sram(chip->oob_poi,
 				       nfc->regs + NFC_MAIN_AREA(0) +
 						   mtd->writesize,
-				       mtd->oobsize, false);
+				       spare, false);
+		/* Not transferred, and never written: reads back erased */
+		memset(chip->oob_poi + spare, 0xff, mtd->oobsize - spare);
+	}
 
 	stat = vf610_nfc_correct_data(chip, buf, chip->oob_poi, page);
 
@@ -599,7 +616,7 @@ static int vf610_nfc_write_page(struct nand_chip *chip, const uint8_t *buf,
 {
 	struct vf610_nfc *nfc = chip_to_nfc(chip);
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	int trfr_sz = mtd->writesize + mtd->oobsize;
+	int trfr_sz = mtd->writesize + vf610_nfc_spare_size(mtd);
 	u32 row = 0, cmd1 = 0, cmd2 = 0, code = 0;
 	u8 status;
 	int ret;
@@ -740,6 +757,49 @@ static void vf610_nfc_init_controller(struct vf610_nfc *nfc)
 	}
 }
 
+/*
+ * With 64 byte OOB chips the core's large page layout matches what
+ * U-Boot uses, and on chips with more, U-Boot and older kernels clamped
+ * mtd->oobsize to 64. Modifying the OOB size is no longer possible, the
+ * actual chip geometry must be respected, so to avoid breaking those
+ * existing setups use our own layout: the core's large page one,
+ * computed over the first 64 spare bytes only.
+ */
+static int vf610_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
+				   struct mtd_oob_region *oobregion)
+{
+	struct nand_device *nand = mtd_to_nanddev(mtd);
+	unsigned int total_ecc_bytes = nand->ecc.ctx.total;
+
+	if (section || !total_ecc_bytes)
+		return -ERANGE;
+
+	oobregion->length = total_ecc_bytes;
+	oobregion->offset = vf610_nfc_spare_size(mtd) - oobregion->length;
+
+	return 0;
+}
+
+static int vf610_nfc_ooblayout_free(struct mtd_info *mtd, int section,
+				    struct mtd_oob_region *oobregion)
+{
+	struct nand_device *nand = mtd_to_nanddev(mtd);
+	unsigned int total_ecc_bytes = nand->ecc.ctx.total;
+
+	if (section)
+		return -ERANGE;
+
+	oobregion->length = vf610_nfc_spare_size(mtd) - total_ecc_bytes - 2;
+	oobregion->offset = 2;
+
+	return 0;
+}
+
+static const struct mtd_ooblayout_ops vf610_nfc_ooblayout_ops = {
+	.ecc = vf610_nfc_ooblayout_ecc,
+	.free = vf610_nfc_ooblayout_free,
+};
+
 static int vf610_nfc_attach_chip(struct nand_chip *chip)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
@@ -770,12 +830,7 @@ static int vf610_nfc_attach_chip(struct nand_chip *chip)
 		return -ENXIO;
 	}
 
-	/* Only 64 byte ECC layouts known */
-	if (mtd->oobsize > 64)
-		mtd->oobsize = 64;
-
-	/* Use default large page ECC layout defined in NAND core */
-	mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());
+	mtd_set_ooblayout(mtd, &vf610_nfc_ooblayout_ops);
 	if (chip->ecc.strength == 32) {
 		nfc->ecc_mode = ECC_60_BYTE;
 		chip->ecc.bytes = 60;
diff --git a/drivers/mtd/nand/spi/Makefile b/drivers/mtd/nand/spi/Makefile
index b5ccb44860df86..f6b2a094fafc47 100644
--- a/drivers/mtd/nand/spi/Makefile
+++ b/drivers/mtd/nand/spi/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 spinand-objs := core.o otp.o
 spinand-objs += alliancememory.o ato.o dosilicon.o esmt.o fmsh.o foresee.o gigadevice.o heyangtek.o
-spinand-objs += macronix.o micron.o paragon.o skyhigh.o toshiba.o winbond.o xtx.o
+spinand-objs += issi.o macronix.o micron.o paragon.o skyhigh.o toshiba.o winbond.o xtx.o
 obj-$(CONFIG_MTD_SPI_NAND) += spinand.o
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c
index 8bf9301f25e752..133776ba53da09 100644
--- a/drivers/mtd/nand/spi/core.c
+++ b/drivers/mtd/nand/spi/core.c
@@ -281,8 +281,26 @@ static int spinand_init_cfg_cache(struct spinand_device *spinand)
 static int spinand_init_quad_enable(struct spinand_device *spinand,
 				    bool enable)
 {
-	return spinand_upd_cfg(spinand, CFG_QUAD_ENABLE,
-			       enable ? CFG_QUAD_ENABLE : 0);
+	struct nand_device *nand = spinand_to_nand(spinand);
+	unsigned int target;
+	int ret;
+
+	/*
+	 * QE is a per-die setting on some devices. Program each target
+	 * individually when enabling or disabling quad I/O mode.
+	 */
+	for (target = 0; target < nand->memorg.ntargets; target++) {
+		ret = spinand_select_target(spinand, target);
+		if (ret)
+			return ret;
+
+		ret = spinand_upd_cfg(spinand, CFG_QUAD_ENABLE,
+				      enable ? CFG_QUAD_ENABLE : 0);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int spinand_ecc_enable(struct spinand_device *spinand,
@@ -508,8 +526,7 @@ static int spinand_read_from_cache_op(struct spinand_device *spinand,
 	else
 		rdesc->info.op_tmpl = &rdesc->info.primary_op_tmpl;
 
-	if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED &&
-	    req->mode != MTD_OPS_RAW)
+	if (nand_ecc_is_pipelined(nand) && req->mode != MTD_OPS_RAW)
 		rdesc->info.op_tmpl->data.ecc = true;
 	else
 		rdesc->info.op_tmpl->data.ecc = false;
@@ -603,8 +620,7 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand,
 
 	wdesc = spinand->dirmaps[req->pos.plane].wdesc;
 
-	if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED &&
-	    req->mode != MTD_OPS_RAW)
+	if (nand_ecc_is_pipelined(nand) && req->mode != MTD_OPS_RAW)
 		wdesc->info.op_tmpl->data.ecc = true;
 	else
 		wdesc->info.op_tmpl->data.ecc = false;
@@ -1261,7 +1277,7 @@ static int spinand_create_dirmap(struct spinand_device *spinand,
 	struct spi_mem_dirmap_desc *desc;
 	bool enable_ecc = false, secondary_op = false;
 
-	if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED)
+	if (nand_ecc_is_pipelined(nand))
 		enable_ecc = true;
 
 	if (spinand->cont_read_possible && spinand->op_templates->cont_read_cache)
@@ -1360,6 +1376,7 @@ static const struct spinand_manufacturer *spinand_manufacturers[] = {
 	&foresee_spinand_manufacturer,
 	&gigadevice_spinand_manufacturer,
 	&heyangtek_spinand_manufacturer,
+	&issi_spinand_manufacturer,
 	&macronix_spinand_manufacturer,
 	&micron_spinand_manufacturer,
 	&paragon_spinand_manufacturer,
@@ -1797,11 +1814,11 @@ static int spinand_configure_chip(struct spinand_device *spinand)
 		    spinand->ssdr_op_templates.write_cache->data.buswidth == 4 ||
 		    spinand->ssdr_op_templates.update_cache->data.buswidth == 4)
 			quad_enable = true;
-	}
 
-	ret = spinand_init_quad_enable(spinand, quad_enable);
-	if (ret)
-		return ret;
+		ret = spinand_init_quad_enable(spinand, quad_enable);
+		if (ret)
+			return ret;
+	}
 
 	if (spinand->configure_chip) {
 		ret = spinand->configure_chip(spinand, SSDR);
@@ -1809,7 +1826,7 @@ static int spinand_configure_chip(struct spinand_device *spinand)
 			return ret;
 	}
 
-	return ret;
+	return 0;
 }
 
 static int spinand_init_flash(struct spinand_device *spinand)
@@ -1980,12 +1997,17 @@ static int spinand_init(struct spinand_device *spinand)
 			goto err_cleanup_ecc_engine;
 	}
 
-	if (nand->ecc.engine) {
-		ret = mtd_ooblayout_count_freebytes(mtd);
-		if (ret < 0)
-			goto err_cleanup_ecc_engine;
+	if (!nand->ecc.engine) {
+		if (spinand->eccinfo.ooblayout)
+			mtd_set_ooblayout(mtd, spinand->eccinfo.ooblayout);
+		else
+			mtd_set_ooblayout(mtd, &spinand_noecc_ooblayout);
 	}
 
+	ret = mtd_ooblayout_count_freebytes(mtd);
+	if (ret < 0)
+		goto err_cleanup_ecc_engine;
+
 	mtd->oobavail = ret;
 
 	/* Propagate ECC information to mtd_info */
diff --git a/drivers/mtd/nand/spi/issi.c b/drivers/mtd/nand/spi/issi.c
new file mode 100644
index 00000000000000..122b3a6e2f62e9
--- /dev/null
+++ b/drivers/mtd/nand/spi/issi.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 ISSI
+ *
+ * Authors:
+ *	Bill Lee <blee@issi.com>
+ *	Jeff Kim <jekim@issi.com>
+ * Co-Author:
+ *	Han Xu <han.xu@nxp.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/mtd/spinand.h>
+
+#define SPINAND_MFR_ISSI 0x9d
+
+#define ISSI_STATUS_ECC_MASK		GENMASK(6, 4)
+#define ISSI_STATUS_ECC_NO_BITFLIPS	0
+#define ISSI_STATUS_ECC_1TO3_BITFLIPS	1
+#define ISSI_STATUS_ECC_UNCOR_ERROR	2
+#define ISSI_STATUS_ECC_4TO6_BITFLIPS	3
+#define ISSI_STATUS_ECC_7TO8_BITFLIPS	5
+
+/*
+ * As per datasheet, die selection is done by the 7th bit of Drive
+ * Strength Register (Address 0xD0).
+ */
+#define ISSI_DIE_SELECT_REG 0xD0
+#define ISSI_SELECT_DIE_MASK BIT(7)
+#define ISSI_SELECT_DIE(x) ((x) << 7)
+
+static SPINAND_OP_VARIANTS(
+	quadio_read_cache_variants,
+	SPINAND_PAGE_READ_FROM_CACHE_1S_1S_4S_OP(0, 1, NULL, 0, 0),
+	SPINAND_PAGE_READ_FROM_CACHE_1S_1S_2S_OP(0, 1, NULL, 0, 0),
+	SPINAND_PAGE_READ_FROM_CACHE_FAST_1S_1S_1S_OP(0, 1, NULL, 0, 0),
+	SPINAND_PAGE_READ_FROM_CACHE_1S_1S_1S_OP(0, 1, NULL, 0, 0));
+
+static SPINAND_OP_VARIANTS(x4_write_cache_variants,
+			   SPINAND_PROG_LOAD_1S_1S_4S_OP(true, 0, NULL, 0),
+			   SPINAND_PROG_LOAD_1S_1S_1S_OP(true, 0, NULL, 0));
+
+static SPINAND_OP_VARIANTS(x4_update_cache_variants,
+			   SPINAND_PROG_LOAD_1S_1S_4S_OP(false, 0, NULL, 0),
+			   SPINAND_PROG_LOAD_1S_1S_1S_OP(false, 0, NULL, 0));
+
+static int issi_8_ooblayout_ecc(struct mtd_info *mtd, int section,
+				struct mtd_oob_region *region)
+{
+	if (section)
+		return -ERANGE;
+
+	region->offset = mtd->oobsize / 2;
+	region->length = mtd->oobsize / 2;
+
+	return 0;
+}
+
+static int issi_8_ooblayout_free(struct mtd_info *mtd, int section,
+				 struct mtd_oob_region *region)
+{
+	if (section)
+		return -ERANGE;
+
+	/* Reserve 2 bytes for the BBM. */
+	region->offset = 2;
+	region->length = (mtd->oobsize / 2) - 2;
+
+	return 0;
+}
+
+static const struct mtd_ooblayout_ops issi_8_ooblayout = {
+	.ecc = issi_8_ooblayout_ecc,
+	.free = issi_8_ooblayout_free,
+};
+
+static int issi_select_target(struct spinand_device *spinand,
+			      unsigned int target)
+{
+	int ret;
+	u8 regval;
+
+	if (target > 1)
+		return -EINVAL;
+
+	ret = spinand_read_reg_op(spinand, ISSI_DIE_SELECT_REG, &regval);
+	if (ret)
+		return ret;
+
+	regval &= ~ISSI_SELECT_DIE_MASK;
+	regval |= ISSI_SELECT_DIE(target);
+
+	return spinand_write_reg_op(spinand, ISSI_DIE_SELECT_REG, regval);
+}
+
+static int issi_8_ecc_get_status(struct spinand_device *spinand, u8 status)
+{
+	switch (FIELD_GET(ISSI_STATUS_ECC_MASK, status)) {
+	case ISSI_STATUS_ECC_NO_BITFLIPS:
+		return 0;
+
+	case ISSI_STATUS_ECC_1TO3_BITFLIPS:
+		return 3;
+
+	case ISSI_STATUS_ECC_4TO6_BITFLIPS:
+		return 6;
+
+	case ISSI_STATUS_ECC_7TO8_BITFLIPS:
+		return 8;
+
+	case ISSI_STATUS_ECC_UNCOR_ERROR:
+		return -EBADMSG;
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static const struct spinand_info issi_spinand_table[] = {
+	/* IS37/38SMW01G8B 1Gb 1.8V */
+	SPINAND_INFO("IS37/38SMW01G8B",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x15),
+		     NAND_MEMORG(1, 2048, 128, 64, 1024, 40, 1, 1, 1),
+		     NAND_ECCREQ(8, 512),
+		     SPINAND_INFO_OP_VARIANTS(&quadio_read_cache_variants,
+					      &x4_write_cache_variants,
+					      &x4_update_cache_variants),
+		     SPINAND_HAS_QE_BIT,
+		     SPINAND_ECCINFO(&issi_8_ooblayout, issi_8_ecc_get_status),
+		     SPINAND_SELECT_TARGET(issi_select_target)),
+
+	/* IS37/38SMW02G8B 2Gb 1.8V */
+	SPINAND_INFO("IS37/38SMW02G8B",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x25),
+		     NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 1),
+		     NAND_ECCREQ(8, 512),
+		     SPINAND_INFO_OP_VARIANTS(&quadio_read_cache_variants,
+					      &x4_write_cache_variants,
+					      &x4_update_cache_variants),
+		     SPINAND_HAS_QE_BIT,
+		     SPINAND_ECCINFO(&issi_8_ooblayout, issi_8_ecc_get_status),
+		     SPINAND_SELECT_TARGET(issi_select_target)),
+
+	/* IS37/38SML04G8B 4Gb 3.3V */
+	SPINAND_INFO("IS37/38SML04G8",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x34),
+		     NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 2),
+		     NAND_ECCREQ(8, 512),
+		     SPINAND_INFO_OP_VARIANTS(&quadio_read_cache_variants,
+					      &x4_write_cache_variants,
+					      &x4_update_cache_variants),
+		     SPINAND_HAS_QE_BIT,
+		     SPINAND_ECCINFO(&issi_8_ooblayout, issi_8_ecc_get_status),
+		     SPINAND_SELECT_TARGET(issi_select_target)),
+
+	/* IS37/38SMW04G8B 4Gb 1.8V */
+	SPINAND_INFO("IS37/38SMW04G8B",
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0x35),
+		     NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 2),
+		     NAND_ECCREQ(8, 512),
+		     SPINAND_INFO_OP_VARIANTS(&quadio_read_cache_variants,
+					      &x4_write_cache_variants,
+					      &x4_update_cache_variants),
+		     SPINAND_HAS_QE_BIT,
+		     SPINAND_ECCINFO(&issi_8_ooblayout, issi_8_ecc_get_status),
+		     SPINAND_SELECT_TARGET(issi_select_target)),
+};
+
+static int issi_spinand_init(struct spinand_device *spinand)
+{
+	return 0;
+}
+
+static const struct spinand_manufacturer_ops issi_spinand_manuf_ops = {
+	.init = issi_spinand_init,
+};
+
+const struct spinand_manufacturer issi_spinand_manufacturer = {
+	.id = SPINAND_MFR_ISSI,
+	.name = "ISSI",
+	.chips = issi_spinand_table,
+	.nchips = ARRAY_SIZE(issi_spinand_table),
+	.ops = &issi_spinand_manuf_ops,
+};
diff --git a/drivers/mtd/nand/spi/winbond.c b/drivers/mtd/nand/spi/winbond.c
index 9b78c1e6cbc9d2..810975808c97e3 100644
--- a/drivers/mtd/nand/spi/winbond.c
+++ b/drivers/mtd/nand/spi/winbond.c
@@ -211,6 +211,13 @@ static SPINAND_OP_VARIANTS(read_cache_variants,
 		SPINAND_PAGE_READ_FROM_CACHE_FAST_1S_1S_1S_OP(0, 1, NULL, 0, 0),
 		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_1S_OP(0, 1, NULL, 0, 0));
 
+static SPINAND_OP_VARIANTS(cont_read_cache_variants,
+		WINBOND_CONT_READ_FROM_CACHE_1S_4S_4S_OP(6, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1S_4S_OP(4, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_2S_2S_OP(4, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1S_2S_OP(4, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_FAST_1S_1S_1S_OP(4, NULL, 0, 0));
+
 static SPINAND_OP_VARIANTS(write_cache_variants,
 		SPINAND_PROG_LOAD_1S_1S_4S_OP(true, 0, NULL, 0),
 		SPINAND_PROG_LOAD_1S_1S_1S_OP(true, 0, NULL, 0));
@@ -314,10 +321,18 @@ static int w25n01kv_ooblayout_ecc(struct mtd_info *mtd, int section,
 static int w25n02kv_ooblayout_ecc(struct mtd_info *mtd, int section,
 				  struct mtd_oob_region *region)
 {
-	if (section > 3)
+	int num_sections = 4;
+	int offset = 64;
+
+	if (mtd->oobsize == 256) {
+		num_sections = 8;
+		offset = 128;
+	}
+
+	if (section >= num_sections)
 		return -ERANGE;
 
-	region->offset = 64 + (16 * section);
+	region->offset = offset + (16 * section);
 	region->length = 13;
 
 	return 0;
@@ -326,9 +341,21 @@ static int w25n02kv_ooblayout_ecc(struct mtd_info *mtd, int section,
 static int w25n02kv_ooblayout_free(struct mtd_info *mtd, int section,
 				   struct mtd_oob_region *region)
 {
-	if (section > 3)
+	int num_sections = 4;
+
+	if (mtd->oobsize == 256)
+		num_sections = 8;
+
+	if (section >= num_sections)
 		return -ERANGE;
 
+	/*
+	 * For at least W25N04KW and W25N04LW, this region is actually
+	 * split in two:
+	 * ECC-protected "User Data I" at + 4 offset, length 12
+	 * unprotected "User Data II" at + 2 offset, length 2
+	 * This returns the full region for backwards compatibility.
+	 */
 	region->offset = (16 * section) + 2;
 	region->length = 14;
 
@@ -345,6 +372,39 @@ static const struct mtd_ooblayout_ops w25n02kv_ooblayout = {
 	.free = w25n02kv_ooblayout_free,
 };
 
+static int w25n08lw_ooblayout_ecc(struct mtd_info *mtd, int section,
+				  struct mtd_oob_region *region)
+{
+	/*
+	 * With ecc enabled the parity bits are not accessible. So we can
+	 * only see page + 128. Without ecc the full page + 256 is accessible.
+	 * To make it simple just return the area as not accessible.
+	 */
+	return -ERANGE;
+}
+
+static int w25n08lw_ooblayout_free(struct mtd_info *mtd, int section,
+				   struct mtd_oob_region *region)
+{
+	if (section > 7)
+		return -ERANGE;
+
+	/*
+	 * Only expose the ECC protected "User Data I" bytes. The "User Data II"
+	 * bytes at offset + 2 are not covered by the on-die ECC engine, and the
+	 * first two bytes of the section hold the BBM.
+	 */
+	region->offset = (16 * section) + 4;
+	region->length = 12;
+
+	return 0;
+}
+
+static const struct mtd_ooblayout_ops w25n08lw_ooblayout = {
+	.ecc = w25n08lw_ooblayout_ecc,
+	.free = w25n08lw_ooblayout_free,
+};
+
 static int w25n01jw_ooblayout_ecc(struct mtd_info *mtd, int section,
 				  struct mtd_oob_region *region)
 {
@@ -768,6 +828,15 @@ static const struct spinand_info winbond_spinand_table[] = {
 					      &update_cache_variants),
 		     0,
 		     SPINAND_ECCINFO(&w25n02kv_ooblayout, w25n02kv_ecc_get_status)),
+	SPINAND_INFO("W25N04LW", /* 1.8V */
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xb2, 0x23),
+		     NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 1, 1),
+		     NAND_ECCREQ(8, 512),
+		     SPINAND_INFO_OP_VARIANTS(&read_cache_variants,
+					      &write_cache_variants,
+					      &update_cache_variants),
+		     0,
+		     SPINAND_ECCINFO(&w25n02kv_ooblayout, w25n02kv_ecc_get_status)),
 	SPINAND_INFO("W35N04JW", /* 1.8V */
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xdf, 0x23),
 		     NAND_MEMORG(1, 4096, 128, 64, 512, 10, 1, 4, 1),
@@ -781,6 +850,18 @@ static const struct spinand_info winbond_spinand_table[] = {
 		     SPINAND_ECCINFO(&w35n01jw_ooblayout, w25w35nxxjw_ecc_get_status),
 		     SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg),
 		     SPINAND_CONT_READ(w35n0xjw_set_cont_read)),
+	/* 8G-bit densities */
+	SPINAND_INFO("W25N08LW", /* 2x4G-bit 1.8V */
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xb3, 0x24),
+		     NAND_MEMORG(1, 4096, 256, 64, 2048, 40, 1, 2, 1),
+		     NAND_ECCREQ(8, 512),
+		     SPINAND_INFO_OP_VARIANTS_WITH_CONT(&read_cache_variants,
+							&write_cache_variants,
+							&update_cache_variants,
+							&cont_read_cache_variants),
+		     0,
+		     SPINAND_ECCINFO(&w25n08lw_ooblayout, w25n02kv_ecc_get_status),
+		     SPINAND_CONT_READ(w25n0xjw_set_cont_read)),
 };
 
 static int winbond_spinand_init(struct spinand_device *spinand)
diff --git a/drivers/mtd/spi-nor/atmel.c b/drivers/mtd/spi-nor/atmel.c
index 82c592f0a1e156..bccc702a2c4abb 100644
--- a/drivers/mtd/spi-nor/atmel.c
+++ b/drivers/mtd/spi-nor/atmel.c
@@ -23,18 +23,28 @@ static int at25fs_nor_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 
 static int at25fs_nor_unlock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
+	/* Write 0x00 to the status register to disable write protection */
+	u8 sr = 0;
 	int ret;
 
 	/* We only support unlocking the whole flash array */
 	if (ofs || len != nor->params->size)
 		return -EINVAL;
 
-	/* Write 0x00 to the status register to disable write protection */
-	ret = spi_nor_write_sr_and_check(nor, 0);
+	ret = spi_nor_write_sr1(nor, &sr);
+	if (ret)
+		return ret;
+
+	ret = spi_nor_read_sr1(nor, &sr);
 	if (ret)
+		return ret;
+
+	if (sr) {
 		dev_dbg(nor->dev, "unable to clear BP bits, WP# asserted?\n");
+		return -EIO;
+	}
 
-	return ret;
+	return 0;
 }
 
 static int at25fs_nor_is_locked(struct spi_nor *nor, loff_t ofs, u64 len)
@@ -50,6 +60,7 @@ static const struct spi_nor_locking_ops at25fs_nor_locking_ops = {
 
 static int at25fs_nor_late_init(struct spi_nor *nor)
 {
+	nor->params->opcodes.write_sr1 = SPINOR_OP_WRSR;
 	nor->params->locking_ops = &at25fs_nor_locking_ops;
 
 	return 0;
@@ -69,6 +80,7 @@ static const struct spi_nor_fixups at25fs_nor_fixups = {
  * Return: 0 on success, -error otherwise.
  */
 static int atmel_nor_set_global_protection(struct spi_nor *nor, loff_t ofs,
+
 					   u64 len, bool is_protect)
 {
 	int ret;
@@ -78,19 +90,24 @@ static int atmel_nor_set_global_protection(struct spi_nor *nor, loff_t ofs,
 	if (ofs || len != nor->params->size)
 		return -EINVAL;
 
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr1(nor, &sr);
 	if (ret)
 		return ret;
 
-	sr = nor->bouncebuf[0];
-
 	/* SRWD bit needs to be cleared, otherwise the protection doesn't change */
 	if (sr & SR_SRWD) {
 		sr &= ~SR_SRWD;
-		ret = spi_nor_write_sr_and_check(nor, sr);
-		if (ret) {
-			dev_dbg(nor->dev, "unable to clear SRWD bit, WP# asserted?\n");
+		ret = spi_nor_write_sr1(nor, &sr);
+		if (ret)
 			return ret;
+
+		ret = spi_nor_read_sr1(nor, &sr);
+		if (ret)
+			return ret;
+
+		if (sr & SR_SRWD) {
+			dev_dbg(nor->dev, "unable to clear SRWD bit, WP# asserted?\n");
+			return -EIO;
 		}
 	}
 
@@ -108,14 +125,7 @@ static int atmel_nor_set_global_protection(struct spi_nor *nor, loff_t ofs,
 		sr &= ~ATMEL_SR_GLOBAL_PROTECT_MASK;
 	}
 
-	nor->bouncebuf[0] = sr;
-
-	/*
-	 * We cannot use the spi_nor_write_sr_and_check() because this command
-	 * isn't really setting any bits, instead it is an pseudo command for
-	 * "Global Unprotect" or "Global Protect"
-	 */
-	return spi_nor_write_sr(nor, nor->bouncebuf, 1);
+	return spi_nor_write_sr1(nor, &sr);
 }
 
 static int atmel_nor_global_protect(struct spi_nor *nor, loff_t ofs, u64 len)
@@ -131,16 +141,17 @@ static int atmel_nor_global_unprotect(struct spi_nor *nor, loff_t ofs, u64 len)
 static int atmel_nor_is_global_protected(struct spi_nor *nor, loff_t ofs,
 					 u64 len)
 {
+	u8 sr;
 	int ret;
 
 	if (ofs >= nor->params->size || (ofs + len) > nor->params->size)
 		return -EINVAL;
 
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr1(nor, &sr);
 	if (ret)
 		return ret;
 
-	return ((nor->bouncebuf[0] & ATMEL_SR_GLOBAL_PROTECT_MASK) == ATMEL_SR_GLOBAL_PROTECT_MASK);
+	return ((sr & ATMEL_SR_GLOBAL_PROTECT_MASK) == ATMEL_SR_GLOBAL_PROTECT_MASK);
 }
 
 static const struct spi_nor_locking_ops atmel_nor_global_protection_ops = {
@@ -151,6 +162,7 @@ static const struct spi_nor_locking_ops atmel_nor_global_protection_ops = {
 
 static int atmel_nor_global_protection_late_init(struct spi_nor *nor)
 {
+	nor->params->opcodes.write_sr1 = SPINOR_OP_WRSR;
 	nor->params->locking_ops = &atmel_nor_global_protection_ops;
 
 	return 0;
@@ -182,47 +194,40 @@ static const struct flash_info atmel_nor_parts[] = {
 		.size = SZ_512K,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &atmel_nor_global_protection_fixups,
 	}, {
 		.id = SNOR_ID(0x1f, 0x45, 0x01),
 		.name = "at26df081a",
 		.size = SZ_1M,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &atmel_nor_global_protection_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x46, 0x01),
 		.name = "at26df161a",
 		.size = SZ_2M,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &atmel_nor_global_protection_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x47, 0x00),
 		.name = "at25df321",
 		.size = SZ_4M,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &atmel_nor_global_protection_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x47, 0x01),
 		.name = "at25df321a",
 		.size = SZ_4M,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &atmel_nor_global_protection_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x47, 0x08),
 		.name = "at25ff321a",
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
-		.fixups = &atmel_nor_global_protection_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x48, 0x00),
 		.name = "at25df641",
 		.size = SZ_8M,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &atmel_nor_global_protection_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x66, 0x01),
 		.name = "at25fs010",
@@ -230,14 +235,12 @@ static const struct flash_info atmel_nor_parts[] = {
 		.size = SZ_128K,
 		.flags = SPI_NOR_HAS_LOCK,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &at25fs_nor_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x66, 0x04),
 		.name = "at25fs040",
 		.size = SZ_512K,
 		.flags = SPI_NOR_HAS_LOCK,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &at25fs_nor_fixups
 	}, {
 		.id = SNOR_ID(0x1f, 0x87, 0x01),
 		.size = SZ_4M,
@@ -245,8 +248,22 @@ static const struct flash_info atmel_nor_parts[] = {
 	},
 };
 
+static const struct spi_nor_fixup atmel_fixups[] = {
+	{ .id = SNOR_ID(0x1f, 0x44, 0x01), .fixups = &atmel_nor_global_protection_fixups },
+	{ .id = SNOR_ID(0x1f, 0x45, 0x01), .fixups = &atmel_nor_global_protection_fixups },
+	{ .id = SNOR_ID(0x1f, 0x46, 0x01), .fixups = &atmel_nor_global_protection_fixups },
+	{ .id = SNOR_ID(0x1f, 0x47, 0x00), .fixups = &atmel_nor_global_protection_fixups },
+	{ .id = SNOR_ID(0x1f, 0x47, 0x01), .fixups = &atmel_nor_global_protection_fixups },
+	{ .id = SNOR_ID(0x1f, 0x47, 0x08), .fixups = &atmel_nor_global_protection_fixups },
+	{ .id = SNOR_ID(0x1f, 0x48, 0x00), .fixups = &atmel_nor_global_protection_fixups },
+	{ .id = SNOR_ID(0x1f, 0x66, 0x01), .fixups = &at25fs_nor_fixups },
+	{ .id = SNOR_ID(0x1f, 0x66, 0x04), .fixups = &at25fs_nor_fixups },
+};
+
 const struct spi_nor_manufacturer spi_nor_atmel = {
 	.name = "atmel",
 	.parts = atmel_nor_parts,
 	.nparts = ARRAY_SIZE(atmel_nor_parts),
+	.fixups = atmel_fixups,
+	.nfixups = ARRAY_SIZE(atmel_fixups),
 };
diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c
index ccf4396cdcd047..90524c58602e1c 100644
--- a/drivers/mtd/spi-nor/core.c
+++ b/drivers/mtd/spi-nor/core.c
@@ -59,7 +59,7 @@
 static u8 spi_nor_get_cmd_ext(const struct spi_nor *nor,
 			      const struct spi_mem_op *op)
 {
-	switch (nor->cmd_ext_type) {
+	switch (nor->params->cmd_ext_type) {
 	case SPI_NOR_EXT_INVERT:
 		return ~op->cmd.opcode;
 
@@ -83,6 +83,7 @@ void spi_nor_spimem_setup_op(const struct spi_nor *nor,
 			     struct spi_mem_op *op,
 			     const enum spi_nor_protocol proto)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	u8 ext;
 
 	op->cmd.buswidth = spi_nor_get_protocol_inst_nbits(proto);
@@ -116,7 +117,7 @@ void spi_nor_spimem_setup_op(const struct spi_nor *nor,
 		op->cmd.nbytes = 2;
 	}
 
-	if (proto == SNOR_PROTO_8_8_8_DTR && nor->flags & SNOR_F_SWAP16)
+	if (proto == SNOR_PROTO_8_8_8_DTR && params->flags & SNOR_F_SWAP16)
 		op->data.swap16 = true;
 }
 
@@ -443,75 +444,6 @@ int spi_nor_read_id(struct spi_nor *nor, u8 naddr, u8 ndummy, u8 *id,
 	return ret;
 }
 
-/**
- * spi_nor_read_sr() - Read the Status Register.
- * @nor:	pointer to 'struct spi_nor'.
- * @sr:		pointer to a DMA-able buffer where the value of the
- *              Status Register will be written. Should be at least 2 bytes.
- *
- * Return: 0 on success, -errno otherwise.
- */
-int spi_nor_read_sr(struct spi_nor *nor, u8 *sr)
-{
-	int ret;
-
-	if (nor->spimem) {
-		struct spi_mem_op op = SPI_NOR_RDSR_OP(sr);
-
-		if (nor->reg_proto == SNOR_PROTO_8_8_8_DTR) {
-			op.addr.nbytes = nor->params->rdsr_addr_nbytes;
-			op.dummy.nbytes = nor->params->rdsr_dummy;
-			/*
-			 * We don't want to read only one byte in DTR mode. So,
-			 * read 2 and then discard the second byte.
-			 */
-			op.data.nbytes = 2;
-		}
-
-		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
-
-		ret = spi_mem_exec_op(nor->spimem, &op);
-	} else {
-		ret = spi_nor_controller_ops_read_reg(nor, SPINOR_OP_RDSR, sr,
-						      1);
-	}
-
-	if (ret)
-		dev_dbg(nor->dev, "error %d reading SR\n", ret);
-
-	return ret;
-}
-
-/**
- * spi_nor_read_cr() - Read the Configuration Register using the
- * SPINOR_OP_RDCR (35h) command.
- * @nor:	pointer to 'struct spi_nor'
- * @cr:		pointer to a DMA-able buffer where the value of the
- *              Configuration Register will be written.
- *
- * Return: 0 on success, -errno otherwise.
- */
-int spi_nor_read_cr(struct spi_nor *nor, u8 *cr)
-{
-	int ret;
-
-	if (nor->spimem) {
-		struct spi_mem_op op = SPI_NOR_RDCR_OP(cr);
-
-		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
-
-		ret = spi_mem_exec_op(nor->spimem, &op);
-	} else {
-		ret = spi_nor_controller_ops_read_reg(nor, SPINOR_OP_RDCR, cr,
-						      1);
-	}
-
-	if (ret)
-		dev_dbg(nor->dev, "error %d reading CR\n", ret);
-
-	return ret;
-}
-
 /**
  * spi_nor_set_4byte_addr_mode_en4b_ex4b() - Enter/Exit 4-byte address mode
  *			using SPINOR_OP_EN4B/SPINOR_OP_EX4B. Typically used by
@@ -617,12 +549,13 @@ int spi_nor_set_4byte_addr_mode_brwr(struct spi_nor *nor, bool enable)
 int spi_nor_sr_ready(struct spi_nor *nor)
 {
 	int ret;
+	u8 sr;
 
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr1(nor, &sr);
 	if (ret)
 		return ret;
 
-	return !(nor->bouncebuf[0] & SR_WIP);
+	return !(sr & SR_WIP);
 }
 
 /**
@@ -633,7 +566,7 @@ int spi_nor_sr_ready(struct spi_nor *nor)
  */
 static bool spi_nor_use_parallel_locking(struct spi_nor *nor)
 {
-	return nor->flags & SNOR_F_RWW;
+	return nor->params->flags & SNOR_F_RWW;
 }
 
 /* Locking helpers for status read operations */
@@ -784,343 +717,294 @@ int spi_nor_global_block_unlock(struct spi_nor *nor)
 }
 
 /**
- * spi_nor_write_sr() - Write the Status Register.
+ * spi_nor_read_sr_ll() - Low-level Status Registers read.
  * @nor:	pointer to 'struct spi_nor'.
- * @sr:		pointer to DMA-able buffer to write to the Status Register.
- * @len:	number of bytes to write to the Status Register.
+ * @opcode:	opcode for the status register read operation.
+ * @sr:		pointer to buffer for storing the content of the status registers.
+ * @len:	number of status registers to read (1 or 2).
  *
  * Return: 0 on success, -errno otherwise.
  */
-int spi_nor_write_sr(struct spi_nor *nor, const u8 *sr, size_t len)
+int spi_nor_read_sr_ll(struct spi_nor *nor, u8 opcode, u8 *sr,
+		       unsigned int len)
 {
-	int ret;
+	int ret, i;
 
-	ret = spi_nor_write_enable(nor);
-	if (ret)
-		return ret;
+	if (len > 2)
+		return -EINVAL;
+
+	for (i = 0; i < len; i++)
+		nor->bouncebuf[i] = 0;
 
 	if (nor->spimem) {
-		struct spi_mem_op op = SPI_NOR_WRSR_OP(sr, len);
+		struct spi_mem_op op = SPI_NOR_RDSR_OP(opcode, nor->bouncebuf, len);
+
+		if (nor->reg_proto == SNOR_PROTO_8_8_8_DTR) {
+			op.addr.nbytes = nor->params->rdsr_addr_nbytes;
+			op.dummy.nbytes = nor->params->rdsr_dummy;
+			/*
+			 * We don't want to read only one byte in DTR mode. So,
+			 * read 2 and then discard the second byte.
+			 */
+			op.data.nbytes = 2;
+		}
 
 		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
 
 		ret = spi_mem_exec_op(nor->spimem, &op);
 	} else {
-		ret = spi_nor_controller_ops_write_reg(nor, SPINOR_OP_WRSR, sr,
-						       len);
+		ret = spi_nor_controller_ops_read_reg(nor, opcode, nor->bouncebuf, len);
 	}
 
-	if (ret) {
-		dev_dbg(nor->dev, "error %d writing SR\n", ret);
-		return ret;
-	}
-
-	return spi_nor_wait_till_ready(nor);
-}
-
-/**
- * spi_nor_write_sr1_and_check() - Write one byte to the Status Register 1 and
- * ensure that the byte written match the received value.
- * @nor:	pointer to a 'struct spi_nor'.
- * @sr1:	byte value to be written to the Status Register.
- *
- * Return: 0 on success, -errno otherwise.
- */
-static int spi_nor_write_sr1_and_check(struct spi_nor *nor, u8 sr1)
-{
-	int ret;
+	memcpy(sr, nor->bouncebuf, len);
 
-	nor->bouncebuf[0] = sr1;
-
-	ret = spi_nor_write_sr(nor, nor->bouncebuf, 1);
-	if (ret)
-		return ret;
-
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
 	if (ret)
-		return ret;
+		dev_dbg(nor->dev, "Error %d reading Status Registers\n", ret);
 
-	if (nor->bouncebuf[0] != sr1) {
-		dev_dbg(nor->dev, "SR1: read back test failed\n");
-		return -EIO;
-	}
-
-	return 0;
+	return ret;
 }
 
 /**
- * spi_nor_write_16bit_sr_and_check() - Write the Status Register 1 and the
- * Status Register 2 in one shot. Ensure that the byte written in the Status
- * Register 1 match the received value, and that the 16-bit Write did not
- * affect what was already in the Status Register 2.
- * @nor:	pointer to a 'struct spi_nor'.
- * @sr1:	byte value to be written to the Status Register 1.
+ * spi_nor_write_sr_ll() - Low-level Status Registers write.
+ * @nor:	pointer to 'struct spi_nor'.
+ * @opcode:	opcode for the status register write operation.
+ * @sr:		pointer to status registers buffer to write.
+ * @len:	number of status registers to write.
  *
  * Return: 0 on success, -errno otherwise.
  */
-static int spi_nor_write_16bit_sr_and_check(struct spi_nor *nor, u8 sr1)
+static int spi_nor_write_sr_ll(struct spi_nor *nor, u8 opcode,	const u8 *sr,
+			       unsigned int len)
 {
-	int ret;
-	u8 *sr_cr = nor->bouncebuf;
-	u8 cr_written;
-
-	/* Make sure we don't overwrite the contents of Status Register 2. */
-	if (!(nor->flags & SNOR_F_NO_READ_CR)) {
-		ret = spi_nor_read_cr(nor, &sr_cr[1]);
-		if (ret)
-			return ret;
-	} else if ((spi_nor_get_protocol_width(nor->read_proto) == 4 ||
-		    spi_nor_get_protocol_width(nor->write_proto) == 4) &&
-		   nor->params->quad_enable) {
-		/*
-		 * If the Status Register 2 Read command (35h) is not
-		 * supported, we should at least be sure we don't
-		 * change the value of the SR2 Quad Enable bit.
-		 *
-		 * When the Quad Enable method is set and the buswidth is 4, we
-		 * can safely assume that the value of the QE bit is one, as a
-		 * consequence of the nor->params->quad_enable() call.
-		 *
-		 * According to the JESD216 revB standard, BFPT DWORDS[15],
-		 * bits 22:20, the 16-bit Write Status (01h) command is
-		 * available just for the cases in which the QE bit is
-		 * described in SR2 at BIT(1).
-		 */
-		sr_cr[1] = SR2_QUAD_EN_BIT1;
-	} else {
-		sr_cr[1] = 0;
-	}
+	int ret, i;
 
-	sr_cr[0] = sr1;
+	if (len > 2)
+		return -EINVAL;
 
-	ret = spi_nor_write_sr(nor, sr_cr, 2);
+	ret = spi_nor_write_enable(nor);
 	if (ret)
 		return ret;
 
-	ret = spi_nor_read_sr(nor, sr_cr);
-	if (ret)
-		return ret;
+	for (i = 0; i < len; i++)
+		nor->bouncebuf[i] = sr[i];
 
-	if (sr1 != sr_cr[0]) {
-		dev_dbg(nor->dev, "SR: Read back test failed\n");
-		return -EIO;
-	}
+	if (nor->spimem) {
+		struct spi_mem_op op = SPI_NOR_WRSR_OP(opcode,
+						       nor->bouncebuf, len);
 
-	if (nor->flags & SNOR_F_NO_READ_CR)
-		return 0;
+		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
 
-	cr_written = sr_cr[1];
+		ret = spi_mem_exec_op(nor->spimem, &op);
+	} else {
+		ret = spi_nor_controller_ops_write_reg(nor, opcode,
+						       nor->bouncebuf, len);
+	}
 
-	ret = spi_nor_read_cr(nor, &sr_cr[1]);
-	if (ret)
+	if (ret) {
+		dev_dbg(nor->dev, "Error %d writing Status Registers\n", ret);
 		return ret;
-
-	if (cr_written != sr_cr[1]) {
-		dev_dbg(nor->dev, "CR: read back test failed\n");
-		return -EIO;
 	}
 
-	return 0;
+	return spi_nor_wait_till_ready(nor);
 }
 
 /**
- * spi_nor_write_16bit_cr_and_check() - Write the Status Register 1 and the
- * Configuration Register in one shot. Ensure that the byte written in the
- * Configuration Register match the received value, and that the 16-bit Write
- * did not affect what was already in the Status Register 1.
- * @nor:	pointer to a 'struct spi_nor'.
- * @cr:		byte value to be written to the Configuration Register.
+ * spi_nor_read_sr1() - Read SR1 only
+ * Useful for:
+ *   - The core to offer a generic "read SR1 and SR2" capability
+ *   - Manufacturer drivers (since they know the chip SR layout)
+ *   - Polling the BUSY bit in the core
  *
- * Return: 0 on success, -errno otherwise.
+ * @nor: the spi_nor structure
+ * @sr1: pointer to a valid SR1 buffer
+ *
+ * Return 0 or errno.
  */
-int spi_nor_write_16bit_cr_and_check(struct spi_nor *nor, u8 cr)
+int spi_nor_read_sr1(struct spi_nor *nor, u8 *sr1)
 {
-	int ret;
-	u8 *sr_cr = nor->bouncebuf;
-	u8 sr_written;
-
-	/* Keep the current value of the Status Register 1. */
-	ret = spi_nor_read_sr(nor, sr_cr);
-	if (ret)
-		return ret;
-
-	sr_cr[1] = cr;
-
-	ret = spi_nor_write_sr(nor, sr_cr, 2);
-	if (ret)
-		return ret;
-
-	sr_written = sr_cr[0];
-
-	ret = spi_nor_read_sr(nor, sr_cr);
-	if (ret)
-		return ret;
-
-	if (sr_written != sr_cr[0]) {
-		dev_dbg(nor->dev, "SR: Read back test failed\n");
-		return -EIO;
-	}
-
-	if (nor->flags & SNOR_F_NO_READ_CR)
-		return 0;
+	return spi_nor_read_sr_ll(nor, nor->params->opcodes.read_sr1, sr1, 1);
+}
 
-	ret = spi_nor_read_cr(nor, &sr_cr[1]);
-	if (ret)
-		return ret;
+/**
+ * spi_nor_read_sr2() - Read SR2 only
+ * Useful for:
+ *   - The core to offer a generic "read SR1 and SR2" capability
+ *   - Manufacturer drivers (since they know the chip SR layout)
+ *   - SR2 based OTP configuration
+ *
+ * @nor: the spi_nor structure
+ * @sr2: pointer to a valid SR2 buffer
+ *
+ * Return 0 or errno.
+ */
+int spi_nor_read_sr2(struct spi_nor *nor, u8 *sr2)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
 
-	if (cr != sr_cr[1]) {
-		dev_dbg(nor->dev, "CR: read back test failed\n");
-		return -EIO;
-	}
+	if (!params->opcodes.read_sr2)
+		return -EINVAL;
 
-	return 0;
+	return spi_nor_read_sr_ll(nor, params->opcodes.read_sr2, sr2, 1);
 }
 
 /**
- * spi_nor_write_16bit_sr_cr_and_check() - Write the Status Register 1 and the
- * Configuration Register in one shot. Ensure that the bytes written in both
- * registers match the received value.
- * @nor:	pointer to a 'struct spi_nor'.
- * @regs:	two-byte array with values to be written to the status and
- *		configuration registers.
+ * spi_nor_read_sr1_and_sr2() - Read SR1 then SR2
+ * General purpose helper, always safe to call. Will expectedly ignore
+ * SR2 on certain chips.
  *
- * Return: 0 on success, -errno otherwise.
+ * @nor: the spi_nor structure
+ * @sr: pointer to a valid 2-byte array
+ *
+ * Return 0 or errno.
  */
-static int spi_nor_write_16bit_sr_cr_and_check(struct spi_nor *nor, const u8 *regs)
+int spi_nor_read_sr1_and_sr2(struct spi_nor *nor, u8 *sr)
 {
-	u8 written_regs[2];
 	int ret;
 
-	written_regs[0] = regs[0];
-	written_regs[1] = regs[1];
-	nor->bouncebuf[0] = regs[0];
-	nor->bouncebuf[1] = regs[1];
-
-	ret = spi_nor_write_sr(nor, nor->bouncebuf, 2);
+	ret = spi_nor_read_sr1(nor, &sr[0]);
 	if (ret)
 		return ret;
 
-	ret = spi_nor_read_sr(nor, &nor->bouncebuf[0]);
-	if (ret)
-		return ret;
-
-	if (written_regs[0] != nor->bouncebuf[0]) {
-		dev_dbg(nor->dev, "SR: Read back test failed\n");
-		return -EIO;
-	}
-
-	if (nor->flags & SNOR_F_NO_READ_CR)
-		return 0;
+	if (nor->params->opcodes.read_sr2)
+		ret = spi_nor_read_sr2(nor, &sr[1]);
+	else
+		sr[1] = 0;
 
-	ret = spi_nor_read_cr(nor, &nor->bouncebuf[1]);
-	if (ret)
-		return ret;
+	return ret;
+}
 
-	if (written_regs[1] != nor->bouncebuf[1]) {
-		dev_dbg(nor->dev, "CR: read back test failed\n");
+/**
+ * spi_nor_write_sr1() - Write SR1 only
+ * Useful for:
+ *   - Manufacturer drivers (since they know the chip SR layout)
+ *
+ * @nor: the spi_nor structure
+ * @sr1: pointer to a valid SR1 buffer
+ *
+ * Return 0 or errno.
+ */
+int spi_nor_write_sr1(struct spi_nor *nor, const u8 *sr1)
+{
+	if (WARN_ONCE(!nor->params->opcodes.write_sr1,
+		      "Restricted helper use, write SR1 not supported"))
 		return -EIO;
-	}
 
-	return 0;
+	return spi_nor_write_sr_ll(nor, nor->params->opcodes.write_sr1, sr1, 1);
 }
 
 /**
- * spi_nor_write_sr_and_check() - Write the Status Register 1 and ensure that
- * the byte written match the received value without affecting other bits in the
- * Status Register 1 and 2.
- * @nor:	pointer to a 'struct spi_nor'.
- * @sr1:	byte value to be written to the Status Register.
+ * spi_nor_write_sr2() - Write SR2 only
+ * Useful for:
+ *   - Manufacturer drivers (since they know the chip SR layout)
+ *   - SR2 based OTP configuration
  *
- * Return: 0 on success, -errno otherwise.
+ * @nor: the spi_nor structure
+ * @sr2: pointer to a valid SR2 buffer
+ *
+ * Return 0 or errno.
  */
-int spi_nor_write_sr_and_check(struct spi_nor *nor, u8 sr1)
+int spi_nor_write_sr2(struct spi_nor *nor, const u8 *sr2)
 {
-	if (nor->flags & SNOR_F_HAS_16BIT_SR)
-		return spi_nor_write_16bit_sr_and_check(nor, sr1);
+	if (WARN_ONCE(!nor->params->opcodes.write_sr2,
+		      "Restricted helper use, write SR2 not supported"))
+		return -EIO;
 
-	return spi_nor_write_sr1_and_check(nor, sr1);
+	return spi_nor_write_sr_ll(nor, nor->params->opcodes.write_sr2, sr2, 1);
 }
 
 /**
- * spi_nor_write_sr_cr_and_check() - Write the Status Register 1 and ensure that
- * the byte written match the received value. Same for the Control Register if
- * available.
- * @nor:	pointer to a 'struct spi_nor'.
- * @regs:	byte array to be written to the registers.
+ * spi_nor_write_sr1_and_sr2() - Write SR1 then SR2
+ * General purpose helper, always safe to call. Will expectedly ignore
+ * SR2 on certain chips.
  *
- * Return: 0 on success, -errno otherwise.
+ * @nor: the spi_nor structure
+ * @sr: pointer to a valid 2-byte array
+ *
+ * Return 0 or errno.
  */
-int spi_nor_write_sr_cr_and_check(struct spi_nor *nor, const u8 *regs)
+static int spi_nor_write_sr1_and_sr2(struct spi_nor *nor, const u8 *sr)
 {
-	if (nor->flags & SNOR_F_HAS_16BIT_SR)
-		return spi_nor_write_16bit_sr_cr_and_check(nor, regs);
+	struct spi_nor_flash_parameter *params = nor->params;
+	int ret;
+
+	if (params->opcodes.write_sr1_and_sr2)
+		return spi_nor_write_sr_ll(nor,
+					   params->opcodes.write_sr1_and_sr2,
+					   sr, 2);
+
+	ret = spi_nor_write_sr1(nor, &sr[0]);
+	if (ret)
+		return ret;
+
+	if (params->opcodes.write_sr2)
+		return spi_nor_write_sr2(nor, &sr[1]);
 
-	return spi_nor_write_sr1_and_check(nor, regs[0]);
+	return 0;
 }
 
 /**
- * spi_nor_write_sr2() - Write the Status Register 2 using the
- * SPINOR_OP_WRSR2 (3eh) command.
- * @nor:	pointer to 'struct spi_nor'.
- * @sr2:	pointer to DMA-able buffer to write to the Status Register 2.
+ * spi_nor_write_sr1_and_sr2_and_check() - Write SR1 then SR2, then read
+ *					   them back and verifies
+ * General purpose helper, always safe to call. Will expectedly ignore
+ * SR2 on certain chips.
  *
- * Return: 0 on success, -errno otherwise.
+ * @nor: the spi_nor structure
+ * @sr: pointer to a valid 2-byte array
+ *
+ * Return 0 or errno.
  */
-static int spi_nor_write_sr2(struct spi_nor *nor, const u8 *sr2)
+int spi_nor_write_sr1_and_sr2_and_check(struct spi_nor *nor, const u8 *sr)
 {
+	u8 tmp[2];
 	int ret;
 
-	ret = spi_nor_write_enable(nor);
+	ret = spi_nor_write_sr1_and_sr2(nor, sr);
 	if (ret)
 		return ret;
 
-	if (nor->spimem) {
-		struct spi_mem_op op = SPI_NOR_WRSR2_OP(sr2);
-
-		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
+	ret = spi_nor_read_sr1_and_sr2(nor, tmp);
+	if (ret)
+		return ret;
 
-		ret = spi_mem_exec_op(nor->spimem, &op);
-	} else {
-		ret = spi_nor_controller_ops_write_reg(nor, SPINOR_OP_WRSR2,
-						       sr2, 1);
-	}
+	if (sr[0] != tmp[0])
+		return -EIO;
 
-	if (ret) {
-		dev_dbg(nor->dev, "error %d writing SR2\n", ret);
-		return ret;
-	}
+	if (nor->params->opcodes.read_sr2 && sr[1] != tmp[1])
+		return -EIO;
 
-	return spi_nor_wait_till_ready(nor);
+	return 0;
 }
 
 /**
- * spi_nor_read_sr2() - Read the Status Register 2 using the
- * SPINOR_OP_RDSR2 (3fh) command.
- * @nor:	pointer to 'struct spi_nor'.
- * @sr2:	pointer to DMA-able buffer where the value of the
- *		Status Register 2 will be written.
+ * spi_nor_generic_quad_enable() - Read the status registers,
+ *				   apply the QE bit mask,
+ *				   write the status registers,
+ *				   read them back and check the content.
+ *
+ * @nor:	pointer to 'struct spi_nor'
  *
  * Return: 0 on success, -errno otherwise.
  */
-static int spi_nor_read_sr2(struct spi_nor *nor, u8 *sr2)
+static int spi_nor_generic_quad_enable(struct spi_nor *nor)
 {
+	u8 *qe_mask = nor->params->qe_mask;
+	u8 sr[2] = {};
 	int ret;
 
-	if (nor->spimem) {
-		struct spi_mem_op op = SPI_NOR_RDSR2_OP(sr2);
+	if (!qe_mask[0] && !qe_mask[1])
+		return 0;
 
-		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
+	ret = spi_nor_read_sr1_and_sr2(nor, sr);
+	if (ret)
+		return ret;
 
-		ret = spi_mem_exec_op(nor->spimem, &op);
-	} else {
-		ret = spi_nor_controller_ops_read_reg(nor, SPINOR_OP_RDSR2, sr2,
-						      1);
-	}
+	if (sr[0] & qe_mask[0] || sr[1] & qe_mask[1])
+		return 0;
 
-	if (ret)
-		dev_dbg(nor->dev, "error %d reading SR2\n", ret);
+	sr[0] |= qe_mask[0];
+	sr[1] |= qe_mask[1];
 
-	return ret;
+	return spi_nor_write_sr1_and_sr2_and_check(nor, sr);
 }
 
 /**
@@ -1140,7 +1024,7 @@ static int spi_nor_erase_die(struct spi_nor *nor, loff_t addr, size_t die_size)
 
 	if (nor->spimem) {
 		struct spi_mem_op op =
-			SPI_NOR_DIE_ERASE_OP(nor->params->die_erase_opcode,
+			SPI_NOR_DIE_ERASE_OP(nor->params->opcodes.die_erase,
 					     nor->addr_nbytes, addr, multi_die);
 
 		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
@@ -1310,7 +1194,7 @@ static bool spi_nor_rww_start_exclusive(struct spi_nor *nor)
 {
 	struct spi_nor_rww *rww = &nor->rww;
 
-	mutex_lock(&nor->lock);
+	guard(mutex)(&nor->lock);
 
 	if (rww->ongoing_io || rww->ongoing_rd || rww->ongoing_pe)
 		return false;
@@ -1346,6 +1230,9 @@ int spi_nor_prep_and_lock(struct spi_nor *nor)
 		ret = wait_event_killable(nor->rww.wait,
 					  spi_nor_rww_start_exclusive(nor));
 
+	if (ret)
+		spi_nor_unprep(nor);
+
 	return ret;
 }
 
@@ -1417,6 +1304,9 @@ static int spi_nor_prep_and_lock_pe(struct spi_nor *nor, loff_t start, size_t le
 		ret = wait_event_killable(nor->rww.wait,
 					  spi_nor_rww_start_pe(nor, start, len));
 
+	if (ret)
+		spi_nor_unprep(nor);
+
 	return ret;
 }
 
@@ -1490,6 +1380,9 @@ static int spi_nor_prep_and_lock_rd(struct spi_nor *nor, loff_t start, size_t le
 		ret = wait_event_killable(nor->rww.wait,
 					  spi_nor_rww_start_rd(nor, start, len));
 
+	if (ret)
+		spi_nor_unprep(nor);
+
 	return ret;
 }
 
@@ -1852,8 +1745,7 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 		return ret;
 
 	/* chip (die) erase? */
-	if ((len == mtd->size && !(nor->flags & SNOR_F_NO_OP_CHIP_ERASE)) ||
-	    multi_die_erase) {
+	if (len == mtd->size || multi_die_erase) {
 		ret = spi_nor_erase_dice(nor, addr, len, die_size);
 		if (ret)
 			goto erase_err;
@@ -1904,106 +1796,6 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 	return ret;
 }
 
-/**
- * spi_nor_sr1_bit6_quad_enable() - Set the Quad Enable BIT(6) in the Status
- * Register 1.
- * @nor:	pointer to a 'struct spi_nor'
- *
- * Bit 6 of the Status Register 1 is the QE bit for Macronix like QSPI memories.
- *
- * Return: 0 on success, -errno otherwise.
- */
-int spi_nor_sr1_bit6_quad_enable(struct spi_nor *nor)
-{
-	int ret;
-
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
-	if (ret)
-		return ret;
-
-	if (nor->bouncebuf[0] & SR1_QUAD_EN_BIT6)
-		return 0;
-
-	nor->bouncebuf[0] |= SR1_QUAD_EN_BIT6;
-
-	return spi_nor_write_sr1_and_check(nor, nor->bouncebuf[0]);
-}
-
-/**
- * spi_nor_sr2_bit1_quad_enable() - set the Quad Enable BIT(1) in the Status
- * Register 2.
- * @nor:       pointer to a 'struct spi_nor'.
- *
- * Bit 1 of the Status Register 2 is the QE bit for Spansion like QSPI memories.
- *
- * Return: 0 on success, -errno otherwise.
- */
-int spi_nor_sr2_bit1_quad_enable(struct spi_nor *nor)
-{
-	int ret;
-
-	if (nor->flags & SNOR_F_NO_READ_CR)
-		return spi_nor_write_16bit_cr_and_check(nor, SR2_QUAD_EN_BIT1);
-
-	ret = spi_nor_read_cr(nor, nor->bouncebuf);
-	if (ret)
-		return ret;
-
-	if (nor->bouncebuf[0] & SR2_QUAD_EN_BIT1)
-		return 0;
-
-	nor->bouncebuf[0] |= SR2_QUAD_EN_BIT1;
-
-	return spi_nor_write_16bit_cr_and_check(nor, nor->bouncebuf[0]);
-}
-
-/**
- * spi_nor_sr2_bit7_quad_enable() - set QE bit in Status Register 2.
- * @nor:	pointer to a 'struct spi_nor'
- *
- * Set the Quad Enable (QE) bit in the Status Register 2.
- *
- * This is one of the procedures to set the QE bit described in the SFDP
- * (JESD216 rev B) specification but no manufacturer using this procedure has
- * been identified yet, hence the name of the function.
- *
- * Return: 0 on success, -errno otherwise.
- */
-int spi_nor_sr2_bit7_quad_enable(struct spi_nor *nor)
-{
-	u8 *sr2 = nor->bouncebuf;
-	int ret;
-	u8 sr2_written;
-
-	/* Check current Quad Enable bit value. */
-	ret = spi_nor_read_sr2(nor, sr2);
-	if (ret)
-		return ret;
-	if (*sr2 & SR2_QUAD_EN_BIT7)
-		return 0;
-
-	/* Update the Quad Enable bit. */
-	*sr2 |= SR2_QUAD_EN_BIT7;
-
-	ret = spi_nor_write_sr2(nor, sr2);
-	if (ret)
-		return ret;
-
-	sr2_written = *sr2;
-
-	/* Read back and check it. */
-	ret = spi_nor_read_sr2(nor, sr2);
-	if (ret)
-		return ret;
-
-	if (*sr2 != sr2_written) {
-		dev_dbg(nor->dev, "SR2: Read back test failed\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
 static const struct spi_nor_manufacturer *manufacturers[] = {
 	&spi_nor_atmel,
 	&spi_nor_eon,
@@ -2494,6 +2286,7 @@ spi_nor_spimem_adjust_hwcaps(struct spi_nor *nor, u32 *hwcaps)
 {
 	struct spi_nor_flash_parameter *params = nor->params;
 	unsigned int cap;
+	u8 opcode;
 
 	/* X-X-X modes are not supported yet, mask them all. */
 	*hwcaps &= ~SNOR_HWCAPS_X_X_X;
@@ -2502,7 +2295,7 @@ spi_nor_spimem_adjust_hwcaps(struct spi_nor *nor, u32 *hwcaps)
 	 * If the reset line is broken, we do not want to enter a stateful
 	 * mode.
 	 */
-	if (nor->flags & SNOR_F_BROKEN_RESET)
+	if (params->flags & SNOR_F_BROKEN_RESET)
 		*hwcaps &= ~(SNOR_HWCAPS_X_X_X | SNOR_HWCAPS_X_X_X_DTR);
 
 	for (cap = 0; cap < sizeof(*hwcaps) * BITS_PER_BYTE; cap++) {
@@ -2525,14 +2318,15 @@ spi_nor_spimem_adjust_hwcaps(struct spi_nor *nor, u32 *hwcaps)
 			*hwcaps &= ~BIT(cap);
 	}
 
-	/* Some SPI controllers might not support CR read opcode. */
-	if (!(nor->flags & SNOR_F_NO_READ_CR)) {
-		struct spi_mem_op op = SPI_NOR_RDCR_OP(nor->bouncebuf);
+	/* Some SPI controllers might not support reading SR2 */
+	opcode = nor->params->opcodes.read_sr2;
+	if (opcode) {
+		struct spi_mem_op op = SPI_NOR_RDSR_OP(opcode, nor->bouncebuf, 1);
 
 		spi_nor_spimem_setup_op(nor, &op, nor->reg_proto);
 
 		if (!spi_mem_supports_op(nor->spimem, &op))
-			nor->flags |= SNOR_F_NO_READ_CR;
+			nor->params->opcodes.read_sr2 = 0;
 	}
 }
 
@@ -2578,26 +2372,6 @@ void spi_nor_init_uniform_erase_map(struct spi_nor_erase_map *map,
 	map->n_regions = 1;
 }
 
-int spi_nor_post_bfpt_fixups(struct spi_nor *nor,
-			     const struct sfdp_parameter_header *bfpt_header,
-			     const struct sfdp_bfpt *bfpt)
-{
-	int ret;
-
-	if (nor->manufacturer && nor->manufacturer->fixups &&
-	    nor->manufacturer->fixups->post_bfpt) {
-		ret = nor->manufacturer->fixups->post_bfpt(nor, bfpt_header,
-							   bfpt);
-		if (ret)
-			return ret;
-	}
-
-	if (nor->info->fixups && nor->info->fixups->post_bfpt)
-		return nor->info->fixups->post_bfpt(nor, bfpt_header, bfpt);
-
-	return 0;
-}
-
 static int spi_nor_select_read(struct spi_nor *nor,
 			       u32 shared_hwcaps)
 {
@@ -2749,6 +2523,8 @@ static int spi_nor_select_erase(struct spi_nor *nor)
 
 static int spi_nor_set_addr_nbytes(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
+
 	if (nor->params->addr_nbytes) {
 		nor->addr_nbytes = nor->params->addr_nbytes;
 	} else if (nor->read_proto == SNOR_PROTO_8_8_8_DTR) {
@@ -2783,8 +2559,8 @@ static int spi_nor_set_addr_nbytes(struct spi_nor *nor)
 	}
 
 	/* Set 4byte opcodes when possible. */
-	if (nor->addr_nbytes == 4 && nor->flags & SNOR_F_4B_OPCODES &&
-	    !(nor->flags & SNOR_F_HAS_4BAIT))
+	if (nor->addr_nbytes == 4 && params->flags & SNOR_F_4B_OPCODES &&
+	    !(params->flags & SNOR_F_HAS_4BAIT))
 		spi_nor_set_4byte_opcodes(nor);
 
 	return 0;
@@ -2851,6 +2627,26 @@ static int spi_nor_setup(struct spi_nor *nor,
 	return spi_nor_set_addr_nbytes(nor);
 }
 
+bool spi_nor_fixup_match(const struct spi_nor *nor,
+			 const struct spi_nor_fixup *fixup)
+{
+	const struct spi_nor_id *id = nor->info ? nor->info->id : NULL;
+
+	/* Filter by ID first, if available */
+	if (fixup->id) {
+		if (!id || fixup->id->len > id->len ||
+		    memcmp(id->bytes, fixup->id->bytes, fixup->id->len))
+			return false;
+	}
+
+	/* Further filter with the match callback, if provided */
+	if (fixup->match)
+		return fixup->match(nor);
+
+	/* Either there was an ID and it matched, or it is a catch-all entry */
+	return true;
+}
+
 /**
  * spi_nor_manufacturer_init_params() - Initialize the flash's parameters and
  * settings based on MFR register and ->default_init() hook.
@@ -2858,21 +2654,27 @@ static int spi_nor_setup(struct spi_nor *nor,
  */
 static void spi_nor_manufacturer_init_params(struct spi_nor *nor)
 {
-	if (nor->manufacturer && nor->manufacturer->fixups &&
-	    nor->manufacturer->fixups->default_init)
-		nor->manufacturer->fixups->default_init(nor);
+	const struct spi_nor_fixup *fixups;
+	unsigned int i;
+
+	if (!nor->manufacturer || !nor->manufacturer->fixups)
+		return;
+
+	fixups = nor->manufacturer->fixups;
 
-	if (nor->info->fixups && nor->info->fixups->default_init)
-		nor->info->fixups->default_init(nor);
+	for (i = 0; i < nor->manufacturer->nfixups; i++) {
+		if (fixups[i].fixups && fixups[i].fixups->default_init &&
+		    spi_nor_fixup_match(nor, &fixups[i]))
+			fixups[i].fixups->default_init(nor);
+	}
 }
 
 /**
  * spi_nor_no_sfdp_init_params() - Initialize the flash's parameters and
- * settings based on nor->info->sfdp_flags. This method should be called only by
- * flashes that do not define SFDP tables. If the flash supports SFDP but the
- * information is wrong and the settings from this function can not be retrieved
- * by parsing SFDP, one should instead use the fixup hooks and update the wrong
- * bits.
+ * settings based on nor->info->sfdp_flags.
+ * If the flash supports SFDP but the information is wrong and the settings from
+ * this function can not be retrieved by parsing SFDP, one should instead use
+ * the fixup hooks and update the wrong bits.
  * @nor:	pointer to a 'struct spi_nor'.
  */
 static void spi_nor_no_sfdp_init_params(struct spi_nor *nor)
@@ -2947,39 +2749,40 @@ static void spi_nor_no_sfdp_init_params(struct spi_nor *nor)
  */
 static void spi_nor_init_flags(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	struct device_node *np = spi_nor_get_flash_node(nor);
 	const u16 flags = nor->info->flags;
 
 	if (of_property_read_bool(np, "broken-flash-reset"))
-		nor->flags |= SNOR_F_BROKEN_RESET;
+		params->flags |= SNOR_F_BROKEN_RESET;
 
 	if (of_property_read_bool(np, "no-wp"))
-		nor->flags |= SNOR_F_NO_WP;
+		params->flags |= SNOR_F_NO_WP;
 
 	if (flags & SPI_NOR_SWP_IS_VOLATILE)
-		nor->flags |= SNOR_F_SWP_IS_VOLATILE;
+		params->flags |= SNOR_F_SWP_IS_VOLATILE;
 
 	if (flags & SPI_NOR_HAS_LOCK)
-		nor->flags |= SNOR_F_HAS_LOCK;
+		params->flags |= SNOR_F_HAS_LOCK;
 
 	if (flags & SPI_NOR_HAS_TB) {
-		nor->flags |= SNOR_F_HAS_SR_TB;
+		params->flags |= SNOR_F_HAS_SR_TB;
 		if (flags & SPI_NOR_TB_SR_BIT6)
-			nor->flags |= SNOR_F_HAS_SR_TB_BIT6;
+			params->flags |= SNOR_F_HAS_SR_TB_BIT6;
 	}
 
 	if (flags & SPI_NOR_4BIT_BP) {
-		nor->flags |= SNOR_F_HAS_4BIT_BP;
+		params->flags |= SNOR_F_HAS_4BIT_BP;
 		if (flags & SPI_NOR_BP3_SR_BIT6)
-			nor->flags |= SNOR_F_HAS_SR_BP3_BIT6;
+			params->flags |= SNOR_F_HAS_SR_BP3_BIT6;
 	}
 
 	if (flags & SPI_NOR_HAS_CMP)
-		nor->flags |= SNOR_F_HAS_SR2_CMP_BIT6;
+		params->flags |= SNOR_F_HAS_SR2_CMP_BIT6;
 
 	if (flags & SPI_NOR_RWW && nor->params->n_banks > 1 &&
 	    !nor->controller_ops)
-		nor->flags |= SNOR_F_RWW;
+		params->flags |= SNOR_F_RWW;
 }
 
 /**
@@ -2992,13 +2795,25 @@ static void spi_nor_init_flags(struct spi_nor *nor)
  */
 static void spi_nor_init_fixup_flags(struct spi_nor *nor)
 {
-	const u8 fixup_flags = nor->info->fixup_flags;
+	struct spi_nor_flash_parameter *params = nor->params;
+	const struct spi_nor_fixup *fixups;
+	unsigned int i;
 
-	if (fixup_flags & SPI_NOR_4B_OPCODES)
-		nor->flags |= SNOR_F_4B_OPCODES;
+	if (!nor->manufacturer || !nor->manufacturer->fixups)
+		return;
 
-	if (fixup_flags & SPI_NOR_IO_MODE_EN_VOLATILE)
-		nor->flags |= SNOR_F_IO_MODE_EN_VOLATILE;
+	fixups = nor->manufacturer->fixups;
+
+	for (i = 0; i < nor->manufacturer->nfixups; i++) {
+		if (!fixups[i].fixup_flags ||
+		    !spi_nor_fixup_match(nor, &fixups[i]))
+			continue;
+
+		if (fixups[i].fixup_flags & SPI_NOR_4B_OPCODES)
+			params->flags |= SNOR_F_4B_OPCODES;
+		if (fixups[i].fixup_flags & SPI_NOR_IO_MODE_EN_VOLATILE)
+			params->flags |= SNOR_F_IO_MODE_EN_VOLATILE;
+	}
 }
 
 /**
@@ -3012,26 +2827,28 @@ static void spi_nor_init_fixup_flags(struct spi_nor *nor)
 static int spi_nor_late_init_params(struct spi_nor *nor)
 {
 	struct spi_nor_flash_parameter *params = nor->params;
+	const struct spi_nor_fixup *fixups;
+	unsigned int i;
 	int ret;
 
-	if (nor->manufacturer && nor->manufacturer->fixups &&
-	    nor->manufacturer->fixups->late_init) {
-		ret = nor->manufacturer->fixups->late_init(nor);
-		if (ret)
-			return ret;
-	}
-
-	/* Needed by some flashes late_init hooks. */
+	/* Needed by some late_init hooks */
 	spi_nor_init_flags(nor);
 
-	if (nor->info->fixups && nor->info->fixups->late_init) {
-		ret = nor->info->fixups->late_init(nor);
-		if (ret)
-			return ret;
+	if (nor->manufacturer && nor->manufacturer->fixups) {
+		fixups = nor->manufacturer->fixups;
+
+		for (i = 0; i < nor->manufacturer->nfixups; i++) {
+			if (fixups[i].fixups && fixups[i].fixups->late_init &&
+			    spi_nor_fixup_match(nor, &fixups[i])) {
+				ret = fixups[i].fixups->late_init(nor);
+				if (ret)
+					return ret;
+			}
+		}
 	}
 
-	if (!nor->params->die_erase_opcode)
-		nor->params->die_erase_opcode = SPINOR_OP_CHIP_ERASE;
+	if (!nor->params->opcodes.die_erase)
+		nor->params->opcodes.die_erase = SPINOR_OP_CHIP_ERASE;
 
 	/* Default method kept for backward compatibility. */
 	if (!params->set_4byte_addr_mode)
@@ -3043,7 +2860,7 @@ static int spi_nor_late_init_params(struct spi_nor *nor)
 	 * NOR protection support. When locking_ops are not provided, we pick
 	 * the default ones.
 	 */
-	if (nor->flags & SNOR_F_HAS_LOCK && !nor->params->locking_ops)
+	if (params->flags & SNOR_F_HAS_LOCK && !nor->params->locking_ops)
 		spi_nor_init_default_locking_ops(nor);
 
 	if (params->n_banks > 1)
@@ -3052,48 +2869,6 @@ static int spi_nor_late_init_params(struct spi_nor *nor)
 	return 0;
 }
 
-/**
- * spi_nor_sfdp_init_params_deprecated() - Deprecated way of initializing flash
- * parameters and settings based on JESD216 SFDP standard.
- * @nor:	pointer to a 'struct spi_nor'.
- *
- * The method has a roll-back mechanism: in case the SFDP parsing fails, the
- * legacy flash parameters and settings will be restored.
- */
-static void spi_nor_sfdp_init_params_deprecated(struct spi_nor *nor)
-{
-	struct spi_nor_flash_parameter sfdp_params;
-
-	memcpy(&sfdp_params, nor->params, sizeof(sfdp_params));
-
-	if (spi_nor_parse_sfdp(nor)) {
-		memcpy(nor->params, &sfdp_params, sizeof(*nor->params));
-		nor->flags &= ~SNOR_F_4B_OPCODES;
-	}
-}
-
-/**
- * spi_nor_init_params_deprecated() - Deprecated way of initializing flash
- * parameters and settings.
- * @nor:	pointer to a 'struct spi_nor'.
- *
- * The method assumes that flash doesn't support SFDP so it initializes flash
- * parameters in spi_nor_no_sfdp_init_params() which later on can be overwritten
- * when parsing SFDP, if supported.
- */
-static void spi_nor_init_params_deprecated(struct spi_nor *nor)
-{
-	spi_nor_no_sfdp_init_params(nor);
-
-	spi_nor_manufacturer_init_params(nor);
-
-	if (nor->info->no_sfdp_flags & (SPI_NOR_DUAL_READ |
-					SPI_NOR_QUAD_READ |
-					SPI_NOR_OCTAL_READ |
-					SPI_NOR_OCTAL_DTR_READ))
-		spi_nor_sfdp_init_params_deprecated(nor);
-}
-
 /**
  * spi_nor_init_default_params() - Default initialization of flash parameters
  * and settings. Done for all flashes, regardless is they define SFDP tables
@@ -3106,11 +2881,14 @@ static void spi_nor_init_default_params(struct spi_nor *nor)
 	const struct flash_info *info = nor->info;
 	struct device_node *np = spi_nor_get_flash_node(nor);
 
-	params->quad_enable = spi_nor_sr2_bit1_quad_enable;
-	params->otp.org = info->otp;
+	/* Default to 16-bit Read/Write Status commands */
+	params->opcodes.read_sr1 = SPINOR_OP_RDSR;
+	params->opcodes.read_sr2 = SPINOR_OP_RDCR;
+	params->opcodes.write_sr1_and_sr2 = SPINOR_OP_WRSR;
+	params->quad_enable = spi_nor_generic_quad_enable;
+	params->qe_mask[1] = BIT(1);
 
-	/* Default to 16-bit Write Status (01h) Command */
-	nor->flags |= SNOR_F_HAS_16BIT_SR;
+	params->otp.org = info->otp;
 
 	/* Set SPI NOR sizes. */
 	params->writesize = 1;
@@ -3154,7 +2932,8 @@ static void spi_nor_init_default_params(struct spi_nor *nor)
  *
  * 1/ Default flash parameters initialization. The initializations are done
  *    based on nor->info data:
- *		spi_nor_info_init_params()
+ *		spi_nor_init_default_params()
+ *		spi_nor_no_sfdp_init_params()
  *
  * which can be overwritten by:
  * 2/ Manufacturer flash parameters initialization. The initializations are
@@ -3165,7 +2944,7 @@ static void spi_nor_init_default_params(struct spi_nor *nor)
  * which can be overwritten by:
  * 3/ SFDP flash parameters initialization. JESD216 SFDP is a standard and
  *    should be more accurate that the above.
- *		spi_nor_parse_sfdp() or spi_nor_no_sfdp_init_params()
+ *		spi_nor_parse_sfdp()
  *
  *    Please note that there is a ->post_bfpt() fixup hook that can overwrite
  *    the flash parameters and settings immediately after parsing the Basic
@@ -3191,17 +2970,14 @@ static int spi_nor_init_params(struct spi_nor *nor)
 		return -ENOMEM;
 
 	spi_nor_init_default_params(nor);
+	spi_nor_no_sfdp_init_params(nor);
+	spi_nor_manufacturer_init_params(nor);
 
-	if (spi_nor_needs_sfdp(nor)) {
-		ret = spi_nor_parse_sfdp(nor);
-		if (ret) {
-			dev_err(nor->dev, "BFPT parsing failed. Please consider using SPI_NOR_SKIP_SFDP when declaring the flash\n");
-			return ret;
-		}
-	} else if (nor->info->no_sfdp_flags & SPI_NOR_SKIP_SFDP) {
-		spi_nor_no_sfdp_init_params(nor);
-	} else {
-		spi_nor_init_params_deprecated(nor);
+	ret = spi_nor_parse_sfdp(nor);
+	if (ret && spi_nor_needs_sfdp(nor)) {
+		dev_err(nor->dev,
+			"SFDP parsing failed. You need to manually declare the flash parameters.\n");
+		return ret;
 	}
 
 	ret = spi_nor_late_init_params(nor);
@@ -3231,7 +3007,7 @@ static int spi_nor_set_octal_dtr(struct spi_nor *nor, bool enable)
 	      nor->write_proto == SNOR_PROTO_8_8_8_DTR))
 		return 0;
 
-	if (!(nor->flags & SNOR_F_IO_MODE_EN_VOLATILE))
+	if (!(nor->params->flags & SNOR_F_IO_MODE_EN_VOLATILE))
 		return 0;
 
 	ret = nor->params->set_octal_dtr(nor, enable);
@@ -3284,7 +3060,7 @@ int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
 		 * reboots (e.g., crashes). Warn the user (or hopefully, system
 		 * designer) that this is bad.
 		 */
-		WARN_ONCE(nor->flags & SNOR_F_BROKEN_RESET,
+		WARN_ONCE(nor->params->flags & SNOR_F_BROKEN_RESET,
 			  "enabling reset hack; may not recover from unexpected reboots\n");
 	}
 
@@ -3305,6 +3081,7 @@ int spi_nor_set_4byte_addr_mode(struct spi_nor *nor, bool enable)
 
 static int spi_nor_init(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	int err;
 
 	err = spi_nor_set_octal_dtr(nor, true);
@@ -3332,13 +3109,13 @@ static int spi_nor_init(struct spi_nor *nor)
 	spi_nor_cache_sr_lock_bits(nor, NULL);
 	if (IS_ENABLED(CONFIG_MTD_SPI_NOR_SWP_DISABLE) ||
 	    (IS_ENABLED(CONFIG_MTD_SPI_NOR_SWP_DISABLE_ON_VOLATILE) &&
-	     nor->flags & SNOR_F_SWP_IS_VOLATILE)) {
+	     params->flags & SNOR_F_SWP_IS_VOLATILE)) {
 		spi_nor_try_unlock_all(nor);
 	}
 
 	if (nor->addr_nbytes == 4 &&
 	    nor->read_proto != SNOR_PROTO_8_8_8_DTR &&
-	    !(nor->flags & SNOR_F_4B_OPCODES))
+	    !(params->flags & SNOR_F_4B_OPCODES))
 		return spi_nor_set_4byte_addr_mode(nor, true);
 
 	return 0;
@@ -3453,11 +3230,12 @@ static void spi_nor_put_device(struct mtd_info *mtd)
 
 static void spi_nor_restore(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	int ret;
 
 	/* restore the addressing mode */
-	if (nor->addr_nbytes == 4 && !(nor->flags & SNOR_F_4B_OPCODES) &&
-	    nor->flags & SNOR_F_BROKEN_RESET) {
+	if (nor->addr_nbytes == 4 && !(params->flags & SNOR_F_4B_OPCODES) &&
+	    params->flags & SNOR_F_BROKEN_RESET) {
 		ret = spi_nor_set_4byte_addr_mode(nor, false);
 		if (ret)
 			/*
@@ -3468,7 +3246,7 @@ static void spi_nor_restore(struct spi_nor *nor)
 			dev_err(nor->dev, "Failed to exit 4-byte address mode, err = %d\n", ret);
 	}
 
-	if (nor->flags & SNOR_F_SOFT_RESET)
+	if (params->flags & SNOR_F_SOFT_RESET)
 		spi_nor_soft_reset(nor);
 }
 
@@ -3585,7 +3363,7 @@ static int spi_nor_set_mtd_info(struct spi_nor *nor)
 	mtd->type = MTD_NORFLASH;
 	mtd->flags = MTD_CAP_NORFLASH;
 	/* Unset BIT_WRITEABLE to enable JFFS2 write buffer for ECC'd NOR */
-	if (nor->flags & SNOR_F_ECC)
+	if (nor->params->flags & SNOR_F_ECC)
 		mtd->flags &= ~MTD_BIT_WRITEABLE;
 	if (nor->info->flags & SPI_NOR_NO_ERASE)
 		mtd->flags |= MTD_NO_ERASE;
@@ -3669,6 +3447,7 @@ int spi_nor_scan(struct spi_nor *nor, const char *name,
 		return PTR_ERR(info);
 
 	nor->info = info;
+	nor->partname = info->name;
 
 	mutex_init(&nor->lock);
 
diff --git a/drivers/mtd/spi-nor/core.h b/drivers/mtd/spi-nor/core.h
index ba2d1a862c9d4e..670182b3c2ad95 100644
--- a/drivers/mtd/spi-nor/core.h
+++ b/drivers/mtd/spi-nor/core.h
@@ -37,36 +37,18 @@
 		   SPI_MEM_OP_NO_DUMMY,					\
 		   SPI_MEM_OP_NO_DATA)
 
-#define SPI_NOR_RDSR_OP(buf)						\
-	SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDSR, 0),			\
+#define SPI_NOR_RDSR_OP(opcode, buf, len)				\
+	SPI_MEM_OP(SPI_MEM_OP_CMD(opcode, 0),				\
 		   SPI_MEM_OP_NO_ADDR,					\
 		   SPI_MEM_OP_NO_DUMMY,					\
-		   SPI_MEM_OP_DATA_IN(1, buf, 0))
+		   SPI_MEM_OP_DATA_IN(len, buf, 0))
 
-#define SPI_NOR_WRSR_OP(buf, len)					\
-	SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRSR, 0),			\
+#define SPI_NOR_WRSR_OP(opcode, buf, len)				\
+	SPI_MEM_OP(SPI_MEM_OP_CMD(opcode, 0),				\
 		   SPI_MEM_OP_NO_ADDR,					\
 		   SPI_MEM_OP_NO_DUMMY,					\
 		   SPI_MEM_OP_DATA_OUT(len, buf, 0))
 
-#define SPI_NOR_RDSR2_OP(buf)						\
-	SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDSR2, 0),			\
-		   SPI_MEM_OP_NO_ADDR,					\
-		   SPI_MEM_OP_NO_DUMMY,					\
-		   SPI_MEM_OP_DATA_OUT(1, buf, 0))
-
-#define SPI_NOR_WRSR2_OP(buf)						\
-	SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_WRSR2, 0),			\
-		   SPI_MEM_OP_NO_ADDR,					\
-		   SPI_MEM_OP_NO_DUMMY,					\
-		   SPI_MEM_OP_DATA_OUT(1, buf, 0))
-
-#define SPI_NOR_RDCR_OP(buf)						\
-	SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_RDCR, 0),			\
-		   SPI_MEM_OP_NO_ADDR,					\
-		   SPI_MEM_OP_NO_DUMMY,					\
-		   SPI_MEM_OP_DATA_IN(1, buf, 0))
-
 #define SPI_NOR_EN4B_EX4B_OP(enable)					\
 	SPI_MEM_OP(SPI_MEM_OP_CMD(enable ? SPINOR_OP_EN4B : SPINOR_OP_EX4B, 0),	\
 		   SPI_MEM_OP_NO_ADDR,					\
@@ -121,16 +103,16 @@
 		   SPI_MEM_OP_NO_ADDR,					\
 		   SPI_MEM_OP_NO_DATA)
 
-/* Keep these in sync with the list in debugfs.c */
+/*
+ * This could contain holes, if adding a new flag use the first free spot.
+ * Keep the flags in sync with the list in debugfs.c
+ */
 enum spi_nor_option_flags {
 	SNOR_F_HAS_SR_TB	= BIT(0),
-	SNOR_F_NO_OP_CHIP_ERASE	= BIT(1),
 	SNOR_F_BROKEN_RESET	= BIT(2),
 	SNOR_F_4B_OPCODES	= BIT(3),
 	SNOR_F_HAS_4BAIT	= BIT(4),
 	SNOR_F_HAS_LOCK		= BIT(5),
-	SNOR_F_HAS_16BIT_SR	= BIT(6),
-	SNOR_F_NO_READ_CR	= BIT(7),
 	SNOR_F_HAS_SR_TB_BIT6	= BIT(8),
 	SNOR_F_HAS_4BIT_BP      = BIT(9),
 	SNOR_F_HAS_SR_BP3_BIT6  = BIT(10),
@@ -340,6 +322,26 @@ struct spi_nor_otp {
 	const struct spi_nor_otp_ops *ops;
 };
 
+/**
+ * struct spi_nor_opcodes - SPI NOR flash specific opcodes.
+ * List of variable opcodes used by the chip.
+ *
+ * @die_erase: opcode for erasing a die, defaults to SPINOR_OP_CHIP_ERASE
+ * @read_sr1: opcode for reading SR1 alone
+ * @read_sr2: opcode for reading SR2 alone
+ * @write_sr1: opcode for writing SR1 alone
+ * @write_sr2: opcode for writing SR2 alone
+ * @write_sr1_and_sr2: opcode for writing SR1 then SR2 in one operation
+ */
+struct spi_nor_opcodes {
+	u8 die_erase;
+	u8 read_sr1;
+	u8 read_sr2;
+	u8 write_sr1;
+	u8 write_sr2;
+	u8 write_sr1_and_sr2;
+};
+
 /**
  * struct spi_nor_flash_parameter - SPI NOR flash parameters and settings.
  * Includes legacy flash parameters and settings that can be overwritten
@@ -348,6 +350,7 @@ struct spi_nor_otp {
  *
  * @bank_size:		the flash memory bank density in bytes.
  * @size:		the total flash memory density in bytes.
+ * @flags:		flag options for the current SPI NOR (SNOR_F_*)
  * @writesize		Minimal writable flash unit size. Defaults to 1. Set to
  *			ECC unit size for ECC-ed flashes.
  * @page_size:		the page size of the SPI NOR flash memory.
@@ -362,7 +365,6 @@ struct spi_nor_otp {
  *			command in octal DTR mode.
  * @n_banks:		number of banks.
  * @n_dice:		number of dice in the flash memory.
- * @die_erase_opcode:	die erase opcode. Defaults to SPINOR_OP_CHIP_ERASE.
  * @vreg_offset:	volatile register offset for each die.
  * @hwcaps:		describes the read and page program hardware
  *			capabilities.
@@ -370,11 +372,13 @@ struct spi_nor_otp {
  *                      in the array, the higher priority.
  * @page_programs:	page program capabilities ordered by priority: the
  *                      higher index in the array, the higher priority.
+ * @cmd_ext_type:	the command opcode extension type for DTR mode.
  * @erase_map:		the erase map parsed from the SFDP Sector Map Parameter
  *                      Table.
  * @otp:		SPI NOR OTP info.
  * @set_octal_dtr:	enables or disables SPI NOR octal DTR mode.
  * @quad_enable:	enables SPI NOR quad mode.
+ * @qe_mask:		two bytes mask used to set/clear the QE bit
  * @set_4byte_addr_mode: puts the SPI NOR in 4 byte addressing mode.
  * @ready:		(optional) flashes might use a different mechanism
  *			than reading the status register to indicate they
@@ -385,6 +389,7 @@ struct spi_nor_otp {
 struct spi_nor_flash_parameter {
 	u64				bank_size;
 	u64				size;
+	u32				flags;
 	u32				writesize;
 	u32				page_size;
 	u8				addr_nbytes;
@@ -393,18 +398,20 @@ struct spi_nor_flash_parameter {
 	u8				rdsr_addr_nbytes;
 	u8				n_banks;
 	u8				n_dice;
-	u8				die_erase_opcode;
 	u32				*vreg_offset;
 
 	struct spi_nor_hwcaps		hwcaps;
 	struct spi_nor_read_command	reads[SNOR_CMD_READ_MAX];
 	struct spi_nor_pp_command	page_programs[SNOR_CMD_PP_MAX];
+	enum spi_nor_cmd_ext		cmd_ext_type;
 
+	struct spi_nor_opcodes		opcodes;
 	struct spi_nor_erase_map        erase_map;
 	struct spi_nor_otp		otp;
 
 	int (*set_octal_dtr)(struct spi_nor *nor, bool enable);
 	int (*quad_enable)(struct spi_nor *nor);
+	u8				qe_mask[2];
 	int (*set_4byte_addr_mode)(struct spi_nor *nor, bool enable);
 	int (*ready)(struct spi_nor *nor);
 
@@ -445,6 +452,33 @@ struct spi_nor_fixups {
 	int (*late_init)(struct spi_nor *nor);
 };
 
+/**
+ * struct spi_nor_fixup - SPI NOR fixup registration.
+ * @id:		(optional) flash ID this fixup applies to, may only match the
+ *		ID prefix, eg. just the first few bytes to match a whole family
+ * @match:	(optional) custom match function (can be used together with @id)
+ * @fixup_flags: flags that indicate support that can be discovered via SFDP
+ *		 ideally, but can not be discovered for this particular flash
+ *		 because the SFDP table that indicates this support is not
+ *		 defined by the flash. In case the table for this support is
+ *		 defined but has wrong values, one should instead use a
+ *		 post_sfdp() hook to set the SNOR_F equivalent flag.
+ *
+ *	SPI_NOR_4B_OPCODES: use dedicated 4byte address op codes to support
+ *			    memory size above 128Mib.
+ *	SPI_NOR_IO_MODE_EN_VOLATILE: flash enables the best available I/O mode
+ *				     via a volatile bit.
+ * @fixups:	the fixup hooks to apply when this entry matches
+ */
+struct spi_nor_fixup {
+	const struct spi_nor_id *id;
+	bool (*match)(const struct spi_nor *nor);
+	u8 fixup_flags;
+#define SPI_NOR_4B_OPCODES		BIT(0)
+#define SPI_NOR_IO_MODE_EN_VOLATILE	BIT(1)
+	const struct spi_nor_fixups *fixups;
+};
+
 /**
  * struct spi_nor_id - SPI NOR flash ID.
  *
@@ -496,9 +530,7 @@ struct spi_nor_id {
  *                            be used with SPI_NOR_HAS_LOCK.
  *
  * @no_sfdp_flags:  flags that indicate support that can be discovered via SFDP.
- *                  Used when SFDP tables are not defined in the flash. These
- *                  flags are used together with the SPI_NOR_SKIP_SFDP flag.
- *   SPI_NOR_SKIP_SFDP:       skip parsing of SFDP tables.
+ *                  Used when SFDP tables are not defined in the flash.
  *   SECT_4K:                 SPINOR_OP_BE_4K works uniformly.
  *   SPI_NOR_DUAL_READ:       flash supports Dual Read.
  *   SPI_NOR_QUAD_READ:       flash supports Quad Read.
@@ -506,22 +538,10 @@ struct spi_nor_id {
  *   SPI_NOR_OCTAL_DTR_READ:  flash supports octal DTR Read.
  *   SPI_NOR_OCTAL_DTR_PP:    flash supports Octal DTR Page Program.
  *
- * @fixup_flags:    flags that indicate support that can be discovered via SFDP
- *                  ideally, but can not be discovered for this particular flash
- *                  because the SFDP table that indicates this support is not
- *                  defined by the flash. In case the table for this support is
- *                  defined but has wrong values, one should instead use a
- *                  post_sfdp() hook to set the SNOR_F equivalent flag.
- *
- *   SPI_NOR_4B_OPCODES:      use dedicated 4byte address op codes to support
- *                            memory size above 128Mib.
- *   SPI_NOR_IO_MODE_EN_VOLATILE: flash enables the best available I/O mode
- *                            via a volatile bit.
  * @mfr_flags:      manufacturer private flags. Used in the manufacturer fixup
  *                  hooks to differentiate support between flashes of the same
  *                  manufacturer.
  * @otp_org:        flash's OTP organization.
- * @fixups:         part specific fixup hooks.
  */
 struct flash_info {
 	char *name;
@@ -545,7 +565,6 @@ struct flash_info {
 #define SPI_NOR_HAS_CMP			BIT(10)
 
 	u8 no_sfdp_flags;
-#define SPI_NOR_SKIP_SFDP		BIT(0)
 #define SECT_4K				BIT(1)
 #define SPI_NOR_DUAL_READ		BIT(3)
 #define SPI_NOR_QUAD_READ		BIT(4)
@@ -553,14 +572,9 @@ struct flash_info {
 #define SPI_NOR_OCTAL_DTR_READ		BIT(6)
 #define SPI_NOR_OCTAL_DTR_PP		BIT(7)
 
-	u8 fixup_flags;
-#define SPI_NOR_4B_OPCODES		BIT(0)
-#define SPI_NOR_IO_MODE_EN_VOLATILE	BIT(1)
-
 	u8 mfr_flags;
 
 	const struct spi_nor_otp_organization *otp;
-	const struct spi_nor_fixups *fixups;
 };
 
 #define SNOR_ID(...)							\
@@ -582,13 +596,16 @@ struct flash_info {
  * @name: manufacturer name
  * @parts: array of parts supported by this manufacturer
  * @nparts: number of entries in the parts array
- * @fixups: hooks called at various points in time during spi_nor_scan()
+ * @fixups: list of fixups, each matched by ID (or a custom match function),
+ *          applied to any part of this manufacturer.
+ * @nfixups: number of entries in the fixups array
  */
 struct spi_nor_manufacturer {
 	const char *name;
 	const struct flash_info *parts;
 	unsigned int nparts;
-	const struct spi_nor_fixups *fixups;
+	const struct spi_nor_fixup *fixups;
+	unsigned int nfixups;
 };
 
 /**
@@ -619,6 +636,8 @@ extern const struct spi_nor_manufacturer spi_nor_xmc;
 
 extern const struct attribute_group *spi_nor_sysfs_groups[];
 
+bool spi_nor_fixup_match(const struct spi_nor *nor,
+			 const struct spi_nor_fixup *fixup);
 void spi_nor_spimem_setup_op(const struct spi_nor *nor,
 			     struct spi_mem_op *op,
 			     const enum spi_nor_protocol proto);
@@ -633,18 +652,17 @@ int spi_nor_wait_till_ready(struct spi_nor *nor);
 int spi_nor_global_block_unlock(struct spi_nor *nor);
 int spi_nor_prep_and_lock(struct spi_nor *nor);
 void spi_nor_unlock_and_unprep(struct spi_nor *nor);
-int spi_nor_sr1_bit6_quad_enable(struct spi_nor *nor);
-int spi_nor_sr2_bit1_quad_enable(struct spi_nor *nor);
-int spi_nor_sr2_bit7_quad_enable(struct spi_nor *nor);
 int spi_nor_read_id(struct spi_nor *nor, u8 naddr, u8 ndummy, u8 *id,
 		    enum spi_nor_protocol reg_proto);
-int spi_nor_read_sr(struct spi_nor *nor, u8 *sr);
 int spi_nor_sr_ready(struct spi_nor *nor);
-int spi_nor_read_cr(struct spi_nor *nor, u8 *cr);
-int spi_nor_write_sr(struct spi_nor *nor, const u8 *sr, size_t len);
-int spi_nor_write_sr_and_check(struct spi_nor *nor, u8 sr1);
-int spi_nor_write_16bit_cr_and_check(struct spi_nor *nor, u8 cr);
-int spi_nor_write_sr_cr_and_check(struct spi_nor *nor, const u8 *regs);
+int spi_nor_read_sr_ll(struct spi_nor *nor, u8 opcode, u8 *sr,
+		       unsigned int len);
+int spi_nor_read_sr1(struct spi_nor *nor, u8 *sr1);
+int spi_nor_read_sr2(struct spi_nor *nor, u8 *sr2);
+int spi_nor_read_sr1_and_sr2(struct spi_nor *nor, u8 *sr);
+int spi_nor_write_sr1(struct spi_nor *nor, const u8 *sr1);
+int spi_nor_write_sr2(struct spi_nor *nor, const u8 *sr2);
+int spi_nor_write_sr1_and_sr2_and_check(struct spi_nor *nor, const u8 *sr);
 
 ssize_t spi_nor_read_data(struct spi_nor *nor, loff_t from, size_t len,
 			  u8 *buf);
@@ -680,10 +698,6 @@ void spi_nor_mask_erase_type(struct spi_nor_erase_type *erase);
 void spi_nor_init_uniform_erase_map(struct spi_nor_erase_map *map,
 				    u8 erase_mask, u64 flash_size);
 
-int spi_nor_post_bfpt_fixups(struct spi_nor *nor,
-			     const struct sfdp_parameter_header *bfpt_header,
-			     const struct sfdp_bfpt *bfpt);
-
 void spi_nor_init_default_locking_ops(struct spi_nor *nor);
 bool spi_nor_has_default_locking_ops(struct spi_nor *nor);
 void spi_nor_try_unlock_all(struct spi_nor *nor);
diff --git a/drivers/mtd/spi-nor/debugfs.c b/drivers/mtd/spi-nor/debugfs.c
index 288e2866daedb2..3f8d586a7622bb 100644
--- a/drivers/mtd/spi-nor/debugfs.c
+++ b/drivers/mtd/spi-nor/debugfs.c
@@ -14,13 +14,10 @@
 #define SNOR_F_NAME(name) [ilog2(SNOR_F_##name)] = #name
 static const char *const snor_f_names[] = {
 	SNOR_F_NAME(HAS_SR_TB),
-	SNOR_F_NAME(NO_OP_CHIP_ERASE),
 	SNOR_F_NAME(BROKEN_RESET),
 	SNOR_F_NAME(4B_OPCODES),
 	SNOR_F_NAME(HAS_4BAIT),
 	SNOR_F_NAME(HAS_LOCK),
-	SNOR_F_NAME(HAS_16BIT_SR),
-	SNOR_F_NAME(NO_READ_CR),
 	SNOR_F_NAME(HAS_SR_TB_BIT6),
 	SNOR_F_NAME(HAS_4BIT_BP),
 	SNOR_F_NAME(HAS_SR_BP3_BIT6),
@@ -84,13 +81,12 @@ static int spi_nor_params_show(struct seq_file *s, void *data)
 	struct spi_nor_flash_parameter *params = nor->params;
 	struct spi_nor_erase_map *erase_map = &params->erase_map;
 	struct spi_nor_erase_region *region = erase_map->regions;
-	const struct flash_info *info = nor->info;
 	char buf[16], *str;
 	loff_t lock_start;
 	u64 lock_length;
 	unsigned int i;
 
-	seq_printf(s, "name\t\t%s\n", info->name);
+	seq_printf(s, "name\t\t%s\n", nor->partname);
 	seq_printf(s, "id\t\t%*ph\n", SPI_NOR_MAX_ID_LEN, nor->id);
 	string_get_size(params->size, 1, STRING_UNITS_2, buf, sizeof(buf));
 	seq_printf(s, "size\t\t%s\n", buf);
@@ -99,7 +95,7 @@ static int spi_nor_params_show(struct seq_file *s, void *data)
 	seq_printf(s, "address nbytes\t%u\n", nor->addr_nbytes);
 
 	seq_puts(s, "flags\t\t");
-	spi_nor_print_flags(s, nor->flags, snor_f_names,
+	spi_nor_print_flags(s, params->flags, snor_f_names,
 			    ARRAY_SIZE(snor_f_names));
 	seq_puts(s, "\n");
 
@@ -108,8 +104,23 @@ static int spi_nor_params_show(struct seq_file *s, void *data)
 	seq_printf(s, "  dummy cycles\t%u\n", nor->read_dummy);
 	seq_printf(s, " erase\t\t0x%02x\n", nor->erase_opcode);
 	seq_printf(s, " program\t0x%02x\n", nor->program_opcode);
-
-	switch (nor->cmd_ext_type) {
+	seq_printf(s, " SR1 read\t0x%02x\n", params->opcodes.read_sr1);
+	if (params->opcodes.read_sr2)
+		seq_printf(s, " SR2 read\t0x%02x\n", params->opcodes.read_sr2);
+	if (params->opcodes.write_sr1)
+		seq_printf(s, " SR1 write\t0x%02x\n", params->opcodes.write_sr1);
+	if (params->opcodes.write_sr2)
+		seq_printf(s, " SR2 write\t0x%02x\n", params->opcodes.write_sr2);
+	if (params->opcodes.write_sr1_and_sr2)
+		seq_printf(s, " SR1+SR2 write\t0x%02x\n",
+			   params->opcodes.write_sr1_and_sr2);
+
+	if (params->qe_mask[0] || params->qe_mask[1])
+		seq_printf(s, " QE\t\tSR%d bit %d\n",
+			   params->qe_mask[0] ? 1 : 2,
+			   ffs(params->qe_mask[0] | params->qe_mask[1]) - 1);
+
+	switch (params->cmd_ext_type) {
 	case SPI_NOR_EXT_NONE:
 		str = "none";
 		break;
@@ -144,10 +155,8 @@ static int spi_nor_params_show(struct seq_file *s, void *data)
 		}
 	}
 
-	if (!(nor->flags & SNOR_F_NO_OP_CHIP_ERASE)) {
-		string_get_size(params->size, 1, STRING_UNITS_2, buf, sizeof(buf));
-		seq_printf(s, " %02x (%s)\n", params->die_erase_opcode, buf);
-	}
+	string_get_size(params->size, 1, STRING_UNITS_2, buf, sizeof(buf));
+	seq_printf(s, " %02x (%s)\n", params->opcodes.die_erase, buf);
 
 	seq_puts(s, "\nsector map\n");
 	seq_puts(s, " region (in hex)   | erase mask | overlaid\n");
diff --git a/drivers/mtd/spi-nor/everspin.c b/drivers/mtd/spi-nor/everspin.c
index add37104d67368..864dffba0e9ca7 100644
--- a/drivers/mtd/spi-nor/everspin.c
+++ b/drivers/mtd/spi-nor/everspin.c
@@ -44,9 +44,14 @@ static const struct spi_nor_fixups everspin_nor_fixups = {
 	.default_init = everspin_nor_default_init,
 };
 
+static const struct spi_nor_fixup everspin_fixups[] = {
+	{ .fixups = &everspin_nor_fixups },
+};
+
 const struct spi_nor_manufacturer spi_nor_everspin = {
 	.name = "everspin",
 	.parts = everspin_nor_parts,
 	.nparts = ARRAY_SIZE(everspin_nor_parts),
-	.fixups = &everspin_nor_fixups,
+	.fixups = everspin_fixups,
+	.nfixups = ARRAY_SIZE(everspin_fixups),
 };
diff --git a/drivers/mtd/spi-nor/gigadevice.c b/drivers/mtd/spi-nor/gigadevice.c
index ef1edd0add70e6..6a2e6ebda14838 100644
--- a/drivers/mtd/spi-nor/gigadevice.c
+++ b/drivers/mtd/spi-nor/gigadevice.c
@@ -23,8 +23,10 @@ gd25q256_post_bfpt(struct spi_nor *nor,
 	 *      GD25Q256E      | SFDP_JESD216_MAJOR | SFDP_JESD216B_MINOR
 	 */
 	if (bfpt_header->major == SFDP_JESD216_MAJOR &&
-	    bfpt_header->minor == SFDP_JESD216_MINOR)
-		nor->params->quad_enable = spi_nor_sr1_bit6_quad_enable;
+	    bfpt_header->minor == SFDP_JESD216_MINOR) {
+		nor->params->qe_mask[0] = BIT(6);
+		nor->params->qe_mask[1] = 0;
+	}
 
 	return 0;
 }
@@ -62,8 +64,6 @@ static const struct flash_info gigadevice_nor_parts[] = {
 		.id = SNOR_ID(0xc8, 0x40, 0x19),
 		.name = "gd25q256",
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6,
-		.fixups = &gd25q256_fixups,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 	}, {
 		.id = SNOR_ID(0xc8, 0x60, 0x16),
 		.name = "gd25lq32",
@@ -85,8 +85,15 @@ static const struct flash_info gigadevice_nor_parts[] = {
 	},
 };
 
+static const struct spi_nor_fixup gigadevice_fixups[] = {
+	{ .id = SNOR_ID(0xc8, 0x40, 0x19), .fixups = &gd25q256_fixups,
+	  .fixup_flags = SPI_NOR_4B_OPCODES },
+};
+
 const struct spi_nor_manufacturer spi_nor_gigadevice = {
 	.name = "gigadevice",
 	.parts = gigadevice_nor_parts,
 	.nparts = ARRAY_SIZE(gigadevice_nor_parts),
+	.fixups = gigadevice_fixups,
+	.nfixups = ARRAY_SIZE(gigadevice_fixups),
 };
diff --git a/drivers/mtd/spi-nor/issi.c b/drivers/mtd/spi-nor/issi.c
index 18d9a00aa22eb2..2f057d731df2aa 100644
--- a/drivers/mtd/spi-nor/issi.c
+++ b/drivers/mtd/spi-nor/issi.c
@@ -52,13 +52,11 @@ static const struct flash_info issi_nor_parts[] = {
 		.sector_size = SZ_32K,
 		.size = SZ_64K,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &pm25lv_nor_fixups
 	}, {
 		.name = "pm25lv010",
 		.sector_size = SZ_32K,
 		.size = SZ_128K,
 		.no_sfdp_flags = SECT_4K,
-		.fixups = &pm25lv_nor_fixups
 	}, {
 		.id = SNOR_ID(0x7f, 0x9d, 0x20),
 		.name = "is25cd512",
@@ -103,8 +101,6 @@ static const struct flash_info issi_nor_parts[] = {
 	}, {
 		.id = SNOR_ID(0x9d, 0x60, 0x19),
 		.name = "is25lp256",
-		.fixups = &is25lp256_fixups,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 	}, {
 		.id = SNOR_ID(0x9d, 0x70, 0x16),
 		.name = "is25wp032",
@@ -124,23 +120,40 @@ static const struct flash_info issi_nor_parts[] = {
 		.id = SNOR_ID(0x9d, 0x70, 0x19),
 		.name = "is25wp256",
 		.flags = SPI_NOR_QUAD_PP,
-		.fixups = &is25lp256_fixups,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 	}
 };
 
 static void issi_nor_default_init(struct spi_nor *nor)
 {
-	nor->params->quad_enable = spi_nor_sr1_bit6_quad_enable;
+	nor->params->qe_mask[0] = BIT(6);
+	nor->params->qe_mask[1] = 0;
 }
 
 static const struct spi_nor_fixups issi_fixups = {
 	.default_init = issi_nor_default_init,
 };
 
+/* PM25LV parts have no JEDEC ID and are likely matched by name */
+static bool issi_pm25lv_match(const struct spi_nor *nor)
+{
+	const char *name = nor->info ? nor->info->name : NULL;
+
+	return name && strstarts(name, "pm25lv");
+}
+
+static const struct spi_nor_fixup issi_fixup_list[] = {
+	{ .fixups = &issi_fixups },
+	{ .match = issi_pm25lv_match, .fixups = &pm25lv_nor_fixups },
+	{ .id = SNOR_ID(0x9d, 0x60, 0x19), .fixups = &is25lp256_fixups,
+	  .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0x9d, 0x70, 0x19), .fixups = &is25lp256_fixups,
+	  .fixup_flags = SPI_NOR_4B_OPCODES },
+};
+
 const struct spi_nor_manufacturer spi_nor_issi = {
 	.name = "issi",
 	.parts = issi_nor_parts,
 	.nparts = ARRAY_SIZE(issi_nor_parts),
-	.fixups = &issi_fixups,
+	.fixups = issi_fixup_list,
+	.nfixups = ARRAY_SIZE(issi_fixup_list),
 };
diff --git a/drivers/mtd/spi-nor/macronix.c b/drivers/mtd/spi-nor/macronix.c
index e97f5cbd9aad23..b3fdb4b2fa1ca2 100644
--- a/drivers/mtd/spi-nor/macronix.c
+++ b/drivers/mtd/spi-nor/macronix.c
@@ -40,7 +40,7 @@ mx25l25635_post_bfpt_fixups(struct spi_nor *nor,
 	 * its BFPT table.
 	 */
 	if (bfpt->dwords[SFDP_DWORD(5)] & BFPT_DWORD5_FAST_READ_4_4_4)
-		nor->flags |= SNOR_F_4B_OPCODES;
+		nor->params->flags |= SNOR_F_4B_OPCODES;
 
 	return 0;
 }
@@ -65,11 +65,12 @@ mx25l3255e_late_init_fixups(struct spi_nor *nor)
 
 	/*
 	 * SFDP of MX25L3255E is JESD216, which does not include the Quad
-	 * Enable bit Requirement in BFPT. As a result, during BFPT parsing,
-	 * the quad_enable method is not set to spi_nor_sr1_bit6_quad_enable.
-	 * Therefore, it is necessary to correct this setting by late_init.
+	 * Enable bit Requirement in BFPT. As a result, during BFPT parsing
+	 * the quad_enable mask is reset. Therefore, it is necessary to
+	 * correct this setting by late_init.
 	 */
-	params->quad_enable = spi_nor_sr1_bit6_quad_enable;
+	params->qe_mask[0] = BIT(6);
+	params->qe_mask[1] = 0;
 
 	/*
 	 * In addition, MX25L3255E also supports 1-4-4 page program in 3-byte
@@ -83,6 +84,25 @@ mx25l3255e_late_init_fixups(struct spi_nor *nor)
 	return 0;
 }
 
+static int
+mx25l12805d_4pp3b_post_sfdp_fixups(struct spi_nor *nor)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
+
+	/*
+	 * JEDEC ID 0xc22018 is shared by MX25L12805D (no SFDP, no 4PP) and
+	 * MX25L12833F/MX25L12845G (support SFDP and 4PP in 3-byte mode).
+	 * The legacy 05D lacks SFDP and will not execute this hook. For
+	 * the newer flashes, 3-byte 1-4-4 PP is not defined in SFDP, so
+	 * we safely enable it here.
+	 */
+	params->hwcaps.mask |= SNOR_HWCAPS_PP_1_4_4;
+	spi_nor_set_pp_settings(&params->page_programs[SNOR_CMD_PP_1_4_4],
+				SPINOR_OP_PP_1_4_4, SNOR_PROTO_1_4_4);
+
+	return 0;
+}
+
 static const struct spi_nor_fixups mx25l25635_fixups = {
 	.post_bfpt = mx25l25635_post_bfpt_fixups,
 	.post_sfdp = macronix_qpp4b_post_sfdp_fixups,
@@ -96,6 +116,10 @@ static const struct spi_nor_fixups mx25l3255e_fixups = {
 	.late_init = mx25l3255e_late_init_fixups,
 };
 
+static const struct spi_nor_fixups mx25l12805d_4pp3b_fixups = {
+	.post_sfdp = mx25l12805d_4pp3b_post_sfdp_fixups,
+};
+
 static const struct flash_info macronix_nor_parts[] = {
 	{
 		.id = SNOR_ID(0xc2, 0x20, 0x10),
@@ -130,26 +154,23 @@ static const struct flash_info macronix_nor_parts[] = {
 		.size = SZ_8M,
 		.no_sfdp_flags = SECT_4K,
 	}, {
-		/* MX25L12805D */
+		/* MX25L12805D, MX25L12833F, MX25L12845G */
 		.id = SNOR_ID(0xc2, 0x20, 0x18),
+		.size = SZ_16M,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_4BIT_BP,
+		.no_sfdp_flags = SECT_4K,
 	}, {
 		/* MX25L25635E, MX25L25645G */
 		.id = SNOR_ID(0xc2, 0x20, 0x19),
-		.fixups = &mx25l25635_fixups
 	}, {
-		/* MX66L51235F */
+		/* MX25L51245G, MX25L51273G, MX66L51235F */
 		.id = SNOR_ID(0xc2, 0x20, 0x1a),
-		.fixup_flags = SPI_NOR_4B_OPCODES,
-		.fixups = &macronix_qpp4b_fixups,
 	}, {
 		/* MX66L1G45G */
 		.id = SNOR_ID(0xc2, 0x20, 0x1b),
-		.fixups = &macronix_qpp4b_fixups,
 	}, {
 		/* MX66L2G45G */
 		.id = SNOR_ID(0xc2, 0x20, 0x1c),
-		.fixups = &macronix_qpp4b_fixups,
 	}, {
 		.id = SNOR_ID(0xc2, 0x23, 0x14),
 		.name = "mx25v8035f",
@@ -186,17 +207,14 @@ static const struct flash_info macronix_nor_parts[] = {
 		.size = SZ_16M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
-		/* MX25U51245G */
+		/* MX25U51245G, MX25U51293G */
 		.id = SNOR_ID(0xc2, 0x25, 0x3a),
-		.fixups = &macronix_qpp4b_fixups,
 	}, {
-		/* MX66U1G45G */
+		/* MX66U1G45G, MX66U1G93G */
 		.id = SNOR_ID(0xc2, 0x25, 0x3b),
-		.fixups = &macronix_qpp4b_fixups,
 	}, {
 		/* MX66U2G45G */
 		.id = SNOR_ID(0xc2, 0x25, 0x3c),
-		.fixups = &macronix_qpp4b_fixups,
 	}, {
 		.id = SNOR_ID(0xc2, 0x26, 0x18),
 		.name = "mx25l12855e",
@@ -228,7 +246,6 @@ static const struct flash_info macronix_nor_parts[] = {
 	}, {
 		/* MX25L3255E */
 		.id = SNOR_ID(0xc2, 0x9e, 0x16),
-		.fixups = &mx25l3255e_fixups,
 	},
 	/*
 	 * This spares us of adding new flash entries for flashes that can be
@@ -314,7 +331,8 @@ static int macronix_nor_set_octal_dtr(struct spi_nor *nor, bool enable)
 
 static void macronix_nor_default_init(struct spi_nor *nor)
 {
-	nor->params->quad_enable = spi_nor_sr1_bit6_quad_enable;
+	nor->params->qe_mask[0] = BIT(6);
+	nor->params->qe_mask[1] = 0;
 }
 
 static int macronix_nor_late_init(struct spi_nor *nor)
@@ -331,9 +349,24 @@ static const struct spi_nor_fixups macronix_nor_fixups = {
 	.late_init = macronix_nor_late_init,
 };
 
+static const struct spi_nor_fixup macronix_fixups[] = {
+	{ .fixups = &macronix_nor_fixups },
+	{ .id = SNOR_ID(0xc2, 0x20, 0x18), .fixups = &mx25l12805d_4pp3b_fixups },
+	{ .id = SNOR_ID(0xc2, 0x20, 0x19), .fixups = &mx25l25635_fixups },
+	{ .id = SNOR_ID(0xc2, 0x20, 0x1a), .fixups = &macronix_qpp4b_fixups,
+	  .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0xc2, 0x20, 0x1b), .fixups = &macronix_qpp4b_fixups },
+	{ .id = SNOR_ID(0xc2, 0x20, 0x1c), .fixups = &macronix_qpp4b_fixups },
+	{ .id = SNOR_ID(0xc2, 0x25, 0x3a), .fixups = &macronix_qpp4b_fixups },
+	{ .id = SNOR_ID(0xc2, 0x25, 0x3b), .fixups = &macronix_qpp4b_fixups },
+	{ .id = SNOR_ID(0xc2, 0x25, 0x3c), .fixups = &macronix_qpp4b_fixups },
+	{ .id = SNOR_ID(0xc2, 0x9e, 0x16), .fixups = &mx25l3255e_fixups },
+};
+
 const struct spi_nor_manufacturer spi_nor_macronix = {
 	.name = "macronix",
 	.parts = macronix_nor_parts,
 	.nparts = ARRAY_SIZE(macronix_nor_parts),
-	.fixups = &macronix_nor_fixups,
+	.fixups = macronix_fixups,
+	.nfixups = ARRAY_SIZE(macronix_fixups),
 };
diff --git a/drivers/mtd/spi-nor/micron-st.c b/drivers/mtd/spi-nor/micron-st.c
index c75b0a1cd56761..f97f2dead8c2f1 100644
--- a/drivers/mtd/spi-nor/micron-st.c
+++ b/drivers/mtd/spi-nor/micron-st.c
@@ -131,7 +131,7 @@ static int micron_st_nor_four_die_late_init(struct spi_nor *nor)
 {
 	struct spi_nor_flash_parameter *params = nor->params;
 
-	params->die_erase_opcode = SPINOR_OP_MT_DIE_ERASE;
+	params->opcodes.die_erase = SPINOR_OP_MT_DIE_ERASE;
 	params->n_dice = 4;
 
 	/*
@@ -147,7 +147,7 @@ static int micron_st_nor_two_die_late_init(struct spi_nor *nor)
 {
 	struct spi_nor_flash_parameter *params = nor->params;
 
-	params->die_erase_opcode = SPINOR_OP_MT_DIE_ERASE;
+	params->opcodes.die_erase = SPINOR_OP_MT_DIE_ERASE;
 	params->n_dice = 2;
 
 	/*
@@ -177,7 +177,7 @@ static int mt35xu512aba_post_sfdp_fixup(struct spi_nor *nor)
 	spi_nor_set_pp_settings(&nor->params->page_programs[SNOR_CMD_PP_8_8_8_DTR],
 				SPINOR_OP_PP_4B, SNOR_PROTO_8_8_8_DTR);
 
-	nor->cmd_ext_type = SPI_NOR_EXT_REPEAT;
+	nor->params->cmd_ext_type = SPI_NOR_EXT_REPEAT;
 	nor->params->rdsr_dummy = 8;
 	nor->params->rdsr_addr_nbytes = 0;
 
@@ -205,14 +205,10 @@ static const struct flash_info micron_nor_parts[] = {
 		/* MT35XU512ABA */
 		.id = SNOR_ID(0x2c, 0x5b, 0x1a),
 		.mfr_flags = USE_FSR,
-		.fixup_flags = SPI_NOR_IO_MODE_EN_VOLATILE,
-		.fixups = &mt35xu512aba_fixups,
 	}, {
 		/* MT35XU01GBBA */
 		.id = SNOR_ID(0x2c, 0x5b, 0x1b),
 		.mfr_flags = USE_FSR,
-		.fixup_flags = SPI_NOR_IO_MODE_EN_VOLATILE,
-		.fixups = &mt35_two_die_fixups,
 	}, {
 		.id = SNOR_ID(0x2c, 0x5b, 0x1c),
 		.name = "mt35xu02g",
@@ -220,8 +216,6 @@ static const struct flash_info micron_nor_parts[] = {
 		.size = SZ_256M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_OCTAL_READ,
 		.mfr_flags = USE_FSR,
-		.fixup_flags = SPI_NOR_4B_OPCODES | SPI_NOR_IO_MODE_EN_VOLATILE,
-		.fixups = &mt35_two_die_fixups,
 	},
 };
 
@@ -229,10 +223,21 @@ static int mt25qu512a_post_bfpt_fixup(struct spi_nor *nor,
 				      const struct sfdp_parameter_header *bfpt_header,
 				      const struct sfdp_bfpt *bfpt)
 {
-	nor->flags &= ~SNOR_F_HAS_16BIT_SR;
+	nor->params->opcodes.write_sr1 = 0;
 	return 0;
 }
 
+/*
+ * n25q00a parts share the first same 3 ID bytes with mt25qu01g.
+ * In order to not mix the fixups, further filter out using the part name.
+ */
+static bool n25q00a_match(const struct spi_nor *nor)
+{
+	const char *name = nor->info ? nor->info->name : NULL;
+
+	return name && !strncmp(name, "n25q00a", 7);
+}
+
 static const struct spi_nor_fixups mt25qu512a_fixups = {
 	.post_bfpt = mt25qu512a_post_bfpt_fixup,
 };
@@ -395,7 +400,6 @@ static const struct flash_info st_nor_parts[] = {
 		.name = "mt25ql256a",
 		.size = SZ_32M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 		.mfr_flags = USE_FSR,
 	}, {
 		.id = SNOR_ID(0x20, 0xba, 0x19),
@@ -408,7 +412,6 @@ static const struct flash_info st_nor_parts[] = {
 		.name = "mt25ql512a",
 		.size = SZ_64M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 		.mfr_flags = USE_FSR,
 	}, {
 		.id = SNOR_ID(0x20, 0xba, 0x20),
@@ -426,14 +429,12 @@ static const struct flash_info st_nor_parts[] = {
 			 SPI_NOR_BP3_SR_BIT6,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_QUAD_READ,
 		.mfr_flags = USE_FSR,
-		.fixups = &n25q00_fixups,
 	}, {
 		.id = SNOR_ID(0x20, 0xba, 0x22),
 		.name = "mt25ql02g",
 		.size = SZ_256M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_QUAD_READ,
 		.mfr_flags = USE_FSR,
-		.fixups = &mt25q02_fixups,
 	}, {
 		.id = SNOR_ID(0x20, 0xbb, 0x15),
 		.name = "n25q016a",
@@ -466,7 +467,6 @@ static const struct flash_info st_nor_parts[] = {
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_4BIT_BP |
 			 SPI_NOR_BP3_SR_BIT6,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 		.mfr_flags = USE_FSR,
 	}, {
 		.id = SNOR_ID(0x20, 0xbb, 0x19),
@@ -480,7 +480,6 @@ static const struct flash_info st_nor_parts[] = {
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_4BIT_BP |
 			 SPI_NOR_BP3_SR_BIT6,
 		.mfr_flags = USE_FSR,
-		.fixups = &mt25qu512a_fixups,
 	}, {
 		.id = SNOR_ID(0x20, 0xbb, 0x20),
 		.name = "n25q512a",
@@ -493,21 +492,18 @@ static const struct flash_info st_nor_parts[] = {
 		.id = SNOR_ID(0x20, 0xbb, 0x21, 0x10, 0x44, 0x00),
 		.name = "mt25qu01g",
 		.mfr_flags = USE_FSR,
-		.fixups = &mt25q01_fixups,
 	}, {
 		.id = SNOR_ID(0x20, 0xbb, 0x21),
 		.name = "n25q00a",
 		.size = SZ_128M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_QUAD_READ,
 		.mfr_flags = USE_FSR,
-		.fixups = &n25q00_fixups,
 	}, {
 		.id = SNOR_ID(0x20, 0xbb, 0x22),
 		.name = "mt25qu02g",
 		.size = SZ_256M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 		.mfr_flags = USE_FSR,
-		.fixups = &mt25q02_fixups,
 	}
 };
 
@@ -632,8 +628,8 @@ static int micron_st_nor_ready(struct spi_nor *nor)
 
 static void micron_st_nor_default_init(struct spi_nor *nor)
 {
-	nor->flags |= SNOR_F_HAS_LOCK;
-	nor->flags &= ~SNOR_F_HAS_16BIT_SR;
+	nor->params->flags |= SNOR_F_HAS_LOCK;
+	nor->params->opcodes.write_sr1 = 0;
 	nor->params->quad_enable = NULL;
 }
 
@@ -657,16 +653,44 @@ static const struct spi_nor_fixups micron_st_nor_fixups = {
 	.late_init = micron_st_nor_late_init,
 };
 
+static const struct spi_nor_fixup micron_fixups[] = {
+	{ .fixups = &micron_st_nor_fixups },
+	{ .id = SNOR_ID(0x2c, 0x5b, 0x1a), .fixups = &mt35xu512aba_fixups,
+	  .fixup_flags = SPI_NOR_IO_MODE_EN_VOLATILE },
+	{ .id = SNOR_ID(0x2c, 0x5b, 0x1b), .fixups = &mt35_two_die_fixups,
+	  .fixup_flags = SPI_NOR_IO_MODE_EN_VOLATILE },
+	{ .id = SNOR_ID(0x2c, 0x5b, 0x1c), .fixups = &mt35_two_die_fixups,
+	  .fixup_flags = SPI_NOR_4B_OPCODES | SPI_NOR_IO_MODE_EN_VOLATILE },
+};
+
+static const struct spi_nor_fixup st_fixups[] = {
+	{ .fixups = &micron_st_nor_fixups },
+	{ .id = SNOR_ID(0x20, 0xba, 0x19, 0x10, 0x44, 0x00),
+	  .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0x20, 0xba, 0x20, 0x10, 0x44, 0x00),
+	  .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0x20, 0xbb, 0x19, 0x10, 0x44, 0x00),
+	  .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0x20, 0xba, 0x21), .fixups = &n25q00_fixups },
+	{ .id = SNOR_ID(0x20, 0xba, 0x22), .fixups = &mt25q02_fixups },
+	{ .id = SNOR_ID(0x20, 0xbb, 0x20, 0x10, 0x44, 0x00), .fixups = &mt25qu512a_fixups },
+	{ .id = SNOR_ID(0x20, 0xbb, 0x21, 0x10, 0x44, 0x00), .fixups = &mt25q01_fixups },
+	{ .id = SNOR_ID(0x20, 0xbb, 0x21), .match = n25q00a_match, .fixups = &n25q00_fixups },
+	{ .id = SNOR_ID(0x20, 0xbb, 0x22), .fixups = &mt25q02_fixups },
+};
+
 const struct spi_nor_manufacturer spi_nor_micron = {
 	.name = "micron",
 	.parts = micron_nor_parts,
 	.nparts = ARRAY_SIZE(micron_nor_parts),
-	.fixups = &micron_st_nor_fixups,
+	.fixups = micron_fixups,
+	.nfixups = ARRAY_SIZE(micron_fixups),
 };
 
 const struct spi_nor_manufacturer spi_nor_st = {
 	.name = "st",
 	.parts = st_nor_parts,
 	.nparts = ARRAY_SIZE(st_nor_parts),
-	.fixups = &micron_st_nor_fixups,
+	.fixups = st_fixups,
+	.nfixups = ARRAY_SIZE(st_fixups),
 };
diff --git a/drivers/mtd/spi-nor/otp.c b/drivers/mtd/spi-nor/otp.c
index 7d0b145d78d8a9..bb9a801b3ceb13 100644
--- a/drivers/mtd/spi-nor/otp.c
+++ b/drivers/mtd/spi-nor/otp.c
@@ -25,7 +25,7 @@
  * Read a security register by using the SPINOR_OP_RSECR commands.
  *
  * In Winbond/GigaDevice datasheets the term "security register" stands for
- * an one-time-programmable memory area, consisting of multiple bytes (usually
+ * a one-time-programmable memory area, consisting of multiple bytes (usually
  * 256). Thus one "security register" maps to one OTP region.
  *
  * This method is used on GigaDevice and Winbond flashes.
@@ -175,24 +175,24 @@ static int spi_nor_otp_lock_bit_cr(unsigned int region)
  */
 int spi_nor_otp_lock_sr2(struct spi_nor *nor, unsigned int region)
 {
-	u8 *cr = nor->bouncebuf;
 	int ret, lock_bit;
+	u8 sr[2];
 
 	lock_bit = spi_nor_otp_lock_bit_cr(region);
 	if (lock_bit < 0)
 		return lock_bit;
 
-	ret = spi_nor_read_cr(nor, cr);
+	ret = spi_nor_read_sr1_and_sr2(nor, sr);
 	if (ret)
 		return ret;
 
 	/* no need to write the register if region is already locked */
-	if (cr[0] & lock_bit)
+	if (sr[1] & lock_bit)
 		return 0;
 
-	cr[0] |= lock_bit;
+	sr[1] |= lock_bit;
 
-	return spi_nor_write_16bit_cr_and_check(nor, cr[0]);
+	return spi_nor_write_sr1_and_sr2_and_check(nor, sr);
 }
 
 /**
@@ -207,18 +207,18 @@ int spi_nor_otp_lock_sr2(struct spi_nor *nor, unsigned int region)
  */
 int spi_nor_otp_is_locked_sr2(struct spi_nor *nor, unsigned int region)
 {
-	u8 *cr = nor->bouncebuf;
 	int ret, lock_bit;
+	u8 sr2;
 
 	lock_bit = spi_nor_otp_lock_bit_cr(region);
 	if (lock_bit < 0)
 		return lock_bit;
 
-	ret = spi_nor_read_cr(nor, cr);
+	ret = spi_nor_read_sr2(nor, &sr2);
 	if (ret)
 		return ret;
 
-	return cr[0] & lock_bit;
+	return sr2 & lock_bit;
 }
 
 static loff_t spi_nor_otp_region_start(const struct spi_nor *nor, unsigned int region)
diff --git a/drivers/mtd/spi-nor/sfdp.c b/drivers/mtd/spi-nor/sfdp.c
index 4600983cb57990..beb0f5c66212cc 100644
--- a/drivers/mtd/spi-nor/sfdp.c
+++ b/drivers/mtd/spi-nor/sfdp.c
@@ -34,17 +34,6 @@
 
 #define SFDP_SIGNATURE		0x50444653U
 
-struct sfdp_header {
-	u32		signature; /* Ox50444653U <=> "SFDP" */
-	u8		minor;
-	u8		major;
-	u8		nph; /* 0-base number of parameter headers */
-	u8		unused;
-
-	/* Basic Flash Parameter Table. */
-	struct sfdp_parameter_header	bfpt_header;
-};
-
 /* Fast Read settings. */
 struct sfdp_bfpt_read {
 	/* The Fast Read x-y-z hardware capability in params->hwcaps.mask. */
@@ -126,6 +115,12 @@ struct sfdp_bfpt_erase {
 
 #define SFDP_4BAIT_DWORD_MAX	2
 
+/*
+ * Limit the total size of SFDP to a reasonable value to avoid allocating too
+ * much memory just of because the flash returned some insane values.
+ */
+#define SFDP_MAX_SIZE	SZ_16K
+
 struct sfdp_4bait {
 	/* The hardware capability. */
 	u32		hwcaps;
@@ -401,6 +396,42 @@ static void spi_nor_regions_sort_erase_types(struct spi_nor_erase_map *map)
 	}
 }
 
+/**
+ * spi_nor_post_bfpt_fixups() - Updates the flash's parameters and settings
+ * after BFPT has been parsed. Called only for flashes that define JESD216 SFDP
+ * tables.
+ * @nor:	pointer to a 'struct spi_nor'
+ * @bfpt_header:	pointer to the 'struct sfdp_parameter_header' describing
+ *			the Basic Flash Parameter Table length and version
+ *
+ * Used to tweak various flash parameters when information provided by the SFDP
+ * tables are wrong.
+ */
+static int spi_nor_post_bfpt_fixups(struct spi_nor *nor,
+				    const struct sfdp_parameter_header *bfpt_header,
+				    const struct sfdp_bfpt *bfpt)
+{
+	const struct spi_nor_fixup *fixups;
+	int ret;
+	unsigned int i;
+
+	if (!nor->manufacturer || !nor->manufacturer->fixups)
+		return 0;
+
+	fixups = nor->manufacturer->fixups;
+
+	for (i = 0; i < nor->manufacturer->nfixups; i++) {
+		if (fixups[i].fixups && fixups[i].fixups->post_bfpt &&
+		    spi_nor_fixup_match(nor, &fixups[i])) {
+			ret = fixups[i].fixups->post_bfpt(nor, bfpt_header, bfpt);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
 /**
  * spi_nor_parse_bfpt() - read and parse the Basic Flash Parameter Table.
  * @nor:		pointer to a 'struct spi_nor'
@@ -453,7 +484,7 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 		    bfpt_header->length * sizeof(u32));
 	addr = SFDP_PARAM_HEADER_PTP(bfpt_header);
 	memset(&bfpt, 0, sizeof(bfpt));
-	err = spi_nor_read_sfdp_dma_unsafe(nor,  addr, len, &bfpt);
+	err = spi_nor_read_sfdp_dma_unsafe(nor, addr, len, &bfpt);
 	if (err < 0)
 		return err;
 
@@ -561,34 +592,50 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 	val >>= BFPT_DWORD11_PAGE_SIZE_SHIFT;
 	params->page_size = 1U << val;
 
+	/*
+	 * The standard declares various read and write status methods, some of
+	 * them will be overloaded based on the QER field.
+	 */
+	params->opcodes.read_sr1 = SPINOR_OP_RDSR;
+	params->opcodes.read_sr2 = SPINOR_OP_RDCR;
+	params->opcodes.write_sr1 = SPINOR_OP_WRSR;
+	params->opcodes.write_sr1_and_sr2 = SPINOR_OP_WRSR;
+
+	params->qe_mask[0] = 0;
+	params->qe_mask[1] = 0;
+
 	/* Quad Enable Requirements. */
 	switch (bfpt.dwords[SFDP_DWORD(15)] & BFPT_DWORD15_QER_MASK) {
 	case BFPT_DWORD15_QER_NONE:
-		params->quad_enable = NULL;
 		break;
 
-	case BFPT_DWORD15_QER_SR2_BIT1_BUGGY:
+	case BFPT_DWORD15_QER_SR2_BIT1_NO_1B_WR:
 		/*
 		 * Writing only one byte to the Status Register has the
 		 * side-effect of clearing Status Register 2.
 		 */
+		fallthrough;
 	case BFPT_DWORD15_QER_SR2_BIT1_NO_RD:
 		/*
 		 * Read Configuration Register (35h) instruction is not
-		 * supported.
+		 * supported. 16-bit writes expected.
 		 */
-		nor->flags |= SNOR_F_HAS_16BIT_SR | SNOR_F_NO_READ_CR;
-		params->quad_enable = spi_nor_sr2_bit1_quad_enable;
+		params->opcodes.read_sr2 = 0;
+		params->opcodes.write_sr1 = 0;
+		params->qe_mask[1] = BIT(1);
 		break;
 
 	case BFPT_DWORD15_QER_SR1_BIT6:
-		nor->flags &= ~SNOR_F_HAS_16BIT_SR;
-		params->quad_enable = spi_nor_sr1_bit6_quad_enable;
+		params->opcodes.read_sr2 = 0;
+		params->opcodes.write_sr1_and_sr2 = 0;
+		params->qe_mask[0] = BIT(6);
 		break;
 
 	case BFPT_DWORD15_QER_SR2_BIT7:
-		nor->flags &= ~SNOR_F_HAS_16BIT_SR;
-		params->quad_enable = spi_nor_sr2_bit7_quad_enable;
+		params->opcodes.read_sr2 = SPINOR_OP_RDSR2;
+		params->opcodes.write_sr1_and_sr2 = 0;
+		params->opcodes.write_sr2 = SPINOR_OP_WRSR2;
+		params->qe_mask[1] = BIT(7);
 		break;
 
 	case BFPT_DWORD15_QER_SR2_BIT1:
@@ -598,9 +645,14 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 		 * Register 2, so let's be cautious and keep the default
 		 * assumption of a 16-bit Write Status (01h) command.
 		 */
-		nor->flags |= SNOR_F_HAS_16BIT_SR;
+		params->opcodes.write_sr1 = 0;
+		params->qe_mask[1] = BIT(1);
+		break;
 
-		params->quad_enable = spi_nor_sr2_bit1_quad_enable;
+	case BFPT_DWORD15_QER_SR2_BIT1_1B:
+		params->qe_mask[1] = BIT(1);
+		params->opcodes.write_sr1_and_sr2 = 0;
+		params->opcodes.write_sr2 = SPINOR_OP_WRSR2_ALT;
 		break;
 
 	default:
@@ -608,6 +660,11 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 		break;
 	}
 
+	/* opcodes sanity check */
+	WARN_ON(!params->opcodes.read_sr1 ||
+		(!params->opcodes.write_sr1 && !params->opcodes.write_sr1_and_sr2) ||
+		(!params->opcodes.read_sr2 && params->opcodes.write_sr2));
+
 	dword = bfpt.dwords[SFDP_DWORD(16)] & BFPT_DWORD16_4B_ADDR_MODE_MASK;
 	if (SFDP_MASK_CHECK(dword, BFPT_DWORD16_4B_ADDR_MODE_BRWR))
 		params->set_4byte_addr_mode = spi_nor_set_4byte_addr_mode_brwr;
@@ -620,7 +677,7 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 
 	/* Soft Reset support. */
 	if (bfpt.dwords[SFDP_DWORD(16)] & BFPT_DWORD16_SWRST_EN_RST)
-		nor->flags |= SNOR_F_SOFT_RESET;
+		params->flags |= SNOR_F_SOFT_RESET;
 
 	/* Stop here if not JESD216 rev C or later. */
 	if (bfpt_header->length == BFPT_DWORD_MAX_JESD216B)
@@ -655,11 +712,11 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 	/* 8D-8D-8D command extension. */
 	switch (bfpt.dwords[SFDP_DWORD(18)] & BFPT_DWORD18_CMD_EXT_MASK) {
 	case BFPT_DWORD18_CMD_EXT_REP:
-		nor->cmd_ext_type = SPI_NOR_EXT_REPEAT;
+		params->cmd_ext_type = SPI_NOR_EXT_REPEAT;
 		break;
 
 	case BFPT_DWORD18_CMD_EXT_INV:
-		nor->cmd_ext_type = SPI_NOR_EXT_INVERT;
+		params->cmd_ext_type = SPI_NOR_EXT_INVERT;
 		break;
 
 	case BFPT_DWORD18_CMD_EXT_RES:
@@ -673,7 +730,7 @@ static int spi_nor_parse_bfpt(struct spi_nor *nor,
 
 	/* Byte order in 8D-8D-8D mode */
 	if (bfpt.dwords[SFDP_DWORD(18)] & BFPT_DWORD18_BYTE_ORDER_SWAPPED)
-		nor->flags |= SNOR_F_SWAP16;
+		params->flags |= SNOR_F_SWAP16;
 
 	return spi_nor_post_bfpt_fixups(nor, bfpt_header, &bfpt);
 }
@@ -702,12 +759,19 @@ static u8 spi_nor_smpt_addr_nbytes(const struct spi_nor *nor, const u32 settings
 static void spi_nor_smpt_read_dummy_fixups(const struct spi_nor *nor,
 					   u8 *read_dummy)
 {
-	if (nor->manufacturer && nor->manufacturer->fixups &&
-	    nor->manufacturer->fixups->smpt_read_dummy)
-		nor->manufacturer->fixups->smpt_read_dummy(nor, read_dummy);
+	const struct spi_nor_fixup *fixups;
+	unsigned int i;
+
+	if (!nor->manufacturer || !nor->manufacturer->fixups)
+		return;
+
+	fixups = nor->manufacturer->fixups;
 
-	if (nor->info->fixups && nor->info->fixups->smpt_read_dummy)
-		nor->info->fixups->smpt_read_dummy(nor, read_dummy);
+	for (i = 0; i < nor->manufacturer->nfixups; i++) {
+		if (fixups[i].fixups && fixups[i].fixups->smpt_read_dummy &&
+		    spi_nor_fixup_match(nor, &fixups[i]))
+			fixups[i].fixups->smpt_read_dummy(nor, read_dummy);
+	}
 }
 
 /**
@@ -732,12 +796,19 @@ static u8 spi_nor_smpt_read_dummy(const struct spi_nor *nor, const u32 settings)
 
 static void spi_nor_smpt_map_id_fixups(const struct spi_nor *nor, u8 *map_id)
 {
-	if (nor->manufacturer && nor->manufacturer->fixups &&
-	    nor->manufacturer->fixups->smpt_map_id)
-		nor->manufacturer->fixups->smpt_map_id(nor, map_id);
+	const struct spi_nor_fixup *fixups;
+	unsigned int i;
+
+	if (!nor->manufacturer || !nor->manufacturer->fixups)
+		return;
 
-	if (nor->info->fixups && nor->info->fixups->smpt_map_id)
-		nor->info->fixups->smpt_map_id(nor, map_id);
+	fixups = nor->manufacturer->fixups;
+
+	for (i = 0; i < nor->manufacturer->nfixups; i++) {
+		if (fixups[i].fixups && fixups[i].fixups->smpt_map_id &&
+		    spi_nor_fixup_match(nor, &fixups[i]))
+			fixups[i].fixups->smpt_map_id(nor, map_id);
+	}
 }
 
 /**
@@ -1017,6 +1088,8 @@ static int spi_nor_parse_4bait(struct spi_nor *nor,
 		{ SNOR_HWCAPS_PP,		BIT(6) },
 		{ SNOR_HWCAPS_PP_1_1_4,		BIT(7) },
 		{ SNOR_HWCAPS_PP_1_4_4,		BIT(8) },
+		{ SNOR_HWCAPS_PP_1_1_8,		BIT(23) },
+		{ SNOR_HWCAPS_PP_1_8_8,		BIT(24) },
 	};
 	static const struct sfdp_4bait erases[SNOR_ERASE_TYPE_MAX] = {
 		{ 0u /* not used */,		BIT(9) },
@@ -1143,6 +1216,14 @@ static int spi_nor_parse_4bait(struct spi_nor *nor,
 		spi_nor_set_pp_settings(&params_pp[SNOR_CMD_PP_1_4_4],
 					SPINOR_OP_PP_1_4_4_4B,
 					SNOR_PROTO_1_4_4);
+	if (pp_hwcaps & SNOR_HWCAPS_PP_1_1_8)
+		spi_nor_set_pp_settings(&params_pp[SNOR_CMD_PP_1_1_8],
+					SPINOR_OP_PP_1_1_8_4B,
+					SNOR_PROTO_1_1_8);
+	if (pp_hwcaps & SNOR_HWCAPS_PP_1_8_8)
+		spi_nor_set_pp_settings(&params_pp[SNOR_CMD_PP_1_8_8],
+					SPINOR_OP_PP_1_8_8_4B,
+					SNOR_PROTO_1_8_8);
 
 	for (i = 0; i < SNOR_ERASE_TYPE_MAX; i++) {
 		if (erase_mask & BIT(i))
@@ -1160,7 +1241,7 @@ static int spi_nor_parse_4bait(struct spi_nor *nor,
 	 * SFDP compliant memories.
 	 */
 	params->addr_nbytes = 4;
-	nor->flags |= SNOR_F_4B_OPCODES | SNOR_F_HAS_4BAIT;
+	params->flags |= SNOR_F_4B_OPCODES | SNOR_F_HAS_4BAIT;
 
 	/* fall through */
 out:
@@ -1175,6 +1256,7 @@ static int spi_nor_parse_4bait(struct spi_nor *nor,
 #define PROFILE1_DWORD5_DUMMY_166MHZ		GENMASK(31, 27)
 #define PROFILE1_DWORD5_DUMMY_133MHZ		GENMASK(21, 17)
 #define PROFILE1_DWORD5_DUMMY_100MHZ		GENMASK(11, 7)
+#define SFDP_PROFILE1_DWORD_MIN			5
 
 /**
  * spi_nor_parse_profile1() - parse the xSPI Profile 1.0 table
@@ -1192,6 +1274,9 @@ static int spi_nor_parse_profile1(struct spi_nor *nor,
 	int ret;
 	u8 dummy, opcode;
 
+	if (profile1_header->length < SFDP_PROFILE1_DWORD_MIN)
+		return -EINVAL;
+
 	len = profile1_header->length * sizeof(*dwords);
 	dwords = kmalloc(len, GFP_KERNEL);
 	if (!dwords)
@@ -1262,6 +1347,7 @@ static int spi_nor_parse_profile1(struct spi_nor *nor,
 }
 
 #define SCCR_DWORD22_OCTAL_DTR_EN_VOLATILE		BIT(31)
+#define SFDP_SCCR_DWORD_MIN				22
 
 /**
  * spi_nor_parse_sccr() - Parse the Status, Control and Configuration Register
@@ -1280,6 +1366,9 @@ static int spi_nor_parse_sccr(struct spi_nor *nor,
 	size_t len;
 	int ret;
 
+	if (sccr_header->length < SFDP_SCCR_DWORD_MIN)
+		return -EINVAL;
+
 	len = sccr_header->length * sizeof(*dwords);
 	dwords = kmalloc(len, GFP_KERNEL);
 	if (!dwords)
@@ -1306,7 +1395,7 @@ static int spi_nor_parse_sccr(struct spi_nor *nor,
 
 	if (FIELD_GET(SCCR_DWORD22_OCTAL_DTR_EN_VOLATILE,
 		      dwords[SFDP_DWORD(22)]))
-		nor->flags |= SNOR_F_IO_MODE_EN_VOLATILE;
+		params->flags |= SNOR_F_IO_MODE_EN_VOLATILE;
 
 out:
 	kfree(dwords);
@@ -1381,17 +1470,23 @@ static int spi_nor_parse_sccr_mc(struct spi_nor *nor,
  */
 static int spi_nor_post_sfdp_fixups(struct spi_nor *nor)
 {
+	const struct spi_nor_fixup *fixups;
+	unsigned int i;
 	int ret;
 
-	if (nor->manufacturer && nor->manufacturer->fixups &&
-	    nor->manufacturer->fixups->post_sfdp) {
-		ret = nor->manufacturer->fixups->post_sfdp(nor);
-		if (ret)
-			return ret;
-	}
+	if (!nor->manufacturer || !nor->manufacturer->fixups)
+		return 0;
 
-	if (nor->info->fixups && nor->info->fixups->post_sfdp)
-		return nor->info->fixups->post_sfdp(nor);
+	fixups = nor->manufacturer->fixups;
+
+	for (i = 0; i < nor->manufacturer->nfixups; i++) {
+		if (fixups[i].fixups && fixups[i].fixups->post_sfdp &&
+		    spi_nor_fixup_match(nor, &fixups[i])) {
+			ret = fixups[i].fixups->post_sfdp(nor);
+			if (ret)
+				return ret;
+		}
+	}
 
 	return 0;
 }
@@ -1433,12 +1528,16 @@ int spi_nor_check_sfdp_signature(struct spi_nor *nor)
  * runtime the main parameters needed to perform basic SPI flash operations such
  * as Fast Read, Page Program or Sector Erase commands.
  *
+ * Because the parsing is optional, all the settings have to be reverted. IOW,
+ * nothing of struct spi_nor shall be changed.
+ *
  * Return: 0 on success, -errno otherwise.
  */
 int spi_nor_parse_sfdp(struct spi_nor *nor)
 {
 	const struct sfdp_parameter_header *param_header, *bfpt_header;
 	struct sfdp_parameter_header *param_headers = NULL;
+	struct spi_nor_flash_parameter params, params2;
 	struct sfdp_header header;
 	struct device *dev = nor->dev;
 	struct sfdp *sfdp;
@@ -1446,6 +1545,12 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 	size_t psize;
 	int i, err;
 
+	/*
+	 * Get a backup of all the parameter to roll back to in case of an
+	 * error.
+	 */
+	memcpy(&params, nor->params, sizeof(params));
+
 	/* Get the SFDP header. */
 	err = spi_nor_read_sfdp_dma_unsafe(nor, 0, sizeof(header), &header);
 	if (err < 0)
@@ -1490,7 +1595,7 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 					psize, param_headers);
 		if (err < 0) {
 			dev_dbg(dev, "failed to read SFDP parameter headers\n");
-			goto exit;
+			goto free_param_headers;
 		}
 	}
 
@@ -1505,20 +1610,16 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 				  SFDP_PARAM_HEADER_PARAM_LEN(param_header));
 	}
 
-	/*
-	 * Limit the total size to a reasonable value to avoid allocating too
-	 * much memory just of because the flash returned some insane values.
-	 */
-	if (sfdp_size > PAGE_SIZE) {
+	if (sfdp_size > SFDP_MAX_SIZE) {
 		dev_dbg(dev, "SFDP data (%zu) too big, truncating\n",
 			sfdp_size);
-		sfdp_size = PAGE_SIZE;
+		sfdp_size = SFDP_MAX_SIZE;
 	}
 
 	sfdp = devm_kzalloc(dev, sizeof(*sfdp), GFP_KERNEL);
 	if (!sfdp) {
 		err = -ENOMEM;
-		goto exit;
+		goto free_param_headers;
 	}
 
 	/*
@@ -1532,16 +1633,13 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 				    sizeof(*sfdp->dwords), GFP_KERNEL);
 	if (!sfdp->dwords) {
 		err = -ENOMEM;
-		devm_kfree(dev, sfdp);
-		goto exit;
+		goto free_sfdp;
 	}
 
 	err = spi_nor_read_sfdp(nor, 0, sfdp_size, sfdp->dwords);
 	if (err < 0) {
 		dev_dbg(dev, "failed to read SFDP data\n");
-		devm_kfree(dev, sfdp->dwords);
-		devm_kfree(dev, sfdp);
-		goto exit;
+		goto free_dwords;
 	}
 
 	nor->sfdp = sfdp;
@@ -1563,10 +1661,11 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 
 	err = spi_nor_parse_bfpt(nor, bfpt_header);
 	if (err)
-		goto exit;
+		goto clear_sfdp_ptr;
 
 	/* Parse optional parameter tables. */
 	for (i = 0; i < header.nph; i++) {
+		memcpy(&params2, nor->params, sizeof(params2));
 		param_header = &param_headers[i];
 
 		switch (SFDP_PARAM_HEADER_ID(param_header)) {
@@ -1600,15 +1699,43 @@ int spi_nor_parse_sfdp(struct spi_nor *nor)
 			/*
 			 * Let's not drop all information we extracted so far
 			 * if optional table parsers fail. In case of failing,
-			 * each optional parser is responsible to roll back to
-			 * the previously known spi_nor data.
+			 * roll back to the previously known
+			 * spi_nor_flash_parameter data.
 			 */
 			err = 0;
+			memcpy(nor->params, &params2, sizeof(*nor->params));
 		}
 	}
 
 	err = spi_nor_post_sfdp_fixups(nor);
-exit:
+	if (err)
+		goto clear_sfdp_ptr;
+
+	kfree(param_headers);
+
+	return 0;
+
+clear_sfdp_ptr:
+	nor->sfdp = NULL;
+free_dwords:
+	devm_kfree(dev, sfdp->dwords);
+free_sfdp:
+	devm_kfree(dev, sfdp);
+free_param_headers:
 	kfree(param_headers);
+	if (err)
+		memcpy(nor->params, &params, sizeof(*nor->params));
+
 	return err;
 }
+
+/**
+ * spi_nor_sfdp_get_header() - retrieves the SFDP header
+ * @nor: pointer to a 'struct spi_nor' with a valid SFDP field
+ *
+ * Return: the cached SFDP header.
+ */
+struct sfdp_header *spi_nor_sfdp_get_header(const struct spi_nor *nor)
+{
+	return (struct sfdp_header *)nor->sfdp->dwords;
+}
diff --git a/drivers/mtd/spi-nor/sfdp.h b/drivers/mtd/spi-nor/sfdp.h
index f74a0eb339ea79..1e49ed548a7e29 100644
--- a/drivers/mtd/spi-nor/sfdp.h
+++ b/drivers/mtd/spi-nor/sfdp.h
@@ -12,6 +12,7 @@
 #define SFDP_JESD216_MINOR	0
 #define SFDP_JESD216A_MINOR	5
 #define SFDP_JESD216B_MINOR	6
+#define SFDP_JESD216F_MINOR	10
 
 /* SFDP DWORDS are indexed from 1 but C arrays are indexed from 0. */
 #define SFDP_DWORD(i)		((i) - 1)
@@ -81,14 +82,21 @@ struct sfdp_bfpt {
  *         instruction 35h. QE is set via Write Status instruction 01h with
  *         two data bytes where bit 1 of the second byte is one.
  *         [...]
+ * (from JESD216 rev F)
+ * - 110b: QE is bit 1 of status register 2. Status register 1 is read using
+ *         Read Status instruction 05h. Status register2 is read using
+ *         instruction 35h, and status register 3 is read using instruction 15h.
+ *         QE is set via Write Status Register instruction 31h with one data
+ *         byte.
  */
 #define BFPT_DWORD15_QER_MASK			GENMASK(22, 20)
 #define BFPT_DWORD15_QER_NONE			(0x0UL << 20) /* Micron */
-#define BFPT_DWORD15_QER_SR2_BIT1_BUGGY		(0x1UL << 20)
+#define BFPT_DWORD15_QER_SR2_BIT1_NO_1B_WR	(0x1UL << 20)
 #define BFPT_DWORD15_QER_SR1_BIT6		(0x2UL << 20) /* Macronix */
 #define BFPT_DWORD15_QER_SR2_BIT7		(0x3UL << 20)
 #define BFPT_DWORD15_QER_SR2_BIT1_NO_RD		(0x4UL << 20)
 #define BFPT_DWORD15_QER_SR2_BIT1		(0x5UL << 20) /* Spansion */
+#define BFPT_DWORD15_QER_SR2_BIT1_1B		(0x6UL << 20) /* Winbond */
 
 #define BFPT_DWORD16_EN4B_MASK			GENMASK(31, 24)
 #define BFPT_DWORD16_EN4B_ALWAYS_4B		BIT(30)
@@ -141,4 +149,17 @@ struct sfdp_parameter_header {
 	u8		id_msb;
 };
 
+struct sfdp_header {
+	u32		signature; /* Ox50444653U <=> "SFDP" */
+	u8		minor;
+	u8		major;
+	u8		nph; /* 0-base number of parameter headers */
+	u8		unused;
+
+	/* Basic Flash Parameter Table. */
+	struct sfdp_parameter_header	bfpt_header;
+};
+
+struct sfdp_header *spi_nor_sfdp_get_header(const struct spi_nor *nor);
+
 #endif /* __LINUX_MTD_SFDP_H */
diff --git a/drivers/mtd/spi-nor/spansion.c b/drivers/mtd/spi-nor/spansion.c
index 65227d989de133..bc50881d564864 100644
--- a/drivers/mtd/spi-nor/spansion.c
+++ b/drivers/mtd/spi-nor/spansion.c
@@ -399,8 +399,9 @@ static int cypress_nor_determine_addr_mode_by_sr1(struct spi_nor *nor,
 					  nor->bouncebuf);
 	bool is3byte, is4byte;
 	int ret;
+	u8 sr;
 
-	ret = spi_nor_read_sr(nor, &nor->bouncebuf[1]);
+	ret = spi_nor_read_sr1(nor, &sr);
 	if (ret)
 		return ret;
 
@@ -408,7 +409,7 @@ static int cypress_nor_determine_addr_mode_by_sr1(struct spi_nor *nor,
 	if (ret)
 		return ret;
 
-	is3byte = (nor->bouncebuf[0] == nor->bouncebuf[1]);
+	is3byte = (nor->bouncebuf[0] == sr);
 
 	op = (struct spi_mem_op)
 		CYPRESS_NOR_RD_ANY_REG_OP(4, SPINOR_REG_CYPRESS_STR1V, 0,
@@ -417,7 +418,7 @@ static int cypress_nor_determine_addr_mode_by_sr1(struct spi_nor *nor,
 	if (ret)
 		return ret;
 
-	is4byte = (nor->bouncebuf[0] == nor->bouncebuf[1]);
+	is4byte = (nor->bouncebuf[0] == sr);
 
 	if (is3byte == is4byte)
 		return -EIO;
@@ -537,7 +538,7 @@ static void cypress_nor_ecc_init(struct spi_nor *nor)
 	 * same ECC data unit without an erase are not allowed.
 	 */
 	nor->params->writesize = 16;
-	nor->flags |= SNOR_F_ECC;
+	nor->params->flags |= SNOR_F_ECC;
 }
 
 static int
@@ -545,7 +546,6 @@ s25fs256t_post_bfpt_fixup(struct spi_nor *nor,
 			  const struct sfdp_parameter_header *bfpt_header,
 			  const struct sfdp_bfpt *bfpt)
 {
-	struct spi_mem_op op;
 	int ret;
 
 	/* Assign 4-byte address mode method that is not determined in BFPT */
@@ -555,19 +555,6 @@ s25fs256t_post_bfpt_fixup(struct spi_nor *nor,
 	if (ret)
 		return ret;
 
-	/* Read Architecture Configuration Register (ARCFN) */
-	op = (struct spi_mem_op)
-		CYPRESS_NOR_RD_ANY_REG_OP(nor->params->addr_mode_nbytes,
-					  SPINOR_REG_CYPRESS_ARCFN, 1,
-					  nor->bouncebuf);
-	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
-	if (ret)
-		return ret;
-
-	/* ARCFN value must be 0 if uniform sector is selected  */
-	if (nor->bouncebuf[0])
-		return -ENODEV;
-
 	return 0;
 }
 
@@ -598,6 +585,22 @@ static int s25fs256t_post_sfdp_fixup(struct spi_nor *nor)
 
 static int s25fs256t_late_init(struct spi_nor *nor)
 {
+	struct spi_mem_op op;
+	int ret;
+
+	/* Read Architecture Configuration Register (ARCFN) */
+	op = (struct spi_mem_op)
+		CYPRESS_NOR_RD_ANY_REG_OP(nor->params->addr_mode_nbytes,
+					  SPINOR_REG_CYPRESS_ARCFN, 1,
+					  nor->bouncebuf);
+	ret = spi_nor_read_any_reg(nor, &op, nor->reg_proto);
+	if (ret)
+		return ret;
+
+	/* ARCFN value must be 0 if uniform sector is selected  */
+	if (nor->bouncebuf[0])
+		return -ENODEV;
+
 	cypress_nor_ecc_init(nor);
 
 	return 0;
@@ -675,7 +678,7 @@ static int s25hx_t_late_init(struct spi_nor *nor)
 	cypress_nor_ecc_init(nor);
 
 	if (params->n_dice > 1)
-		params->die_erase_opcode = SPINOR_OP_CYPRESS_DIE_ERASE;
+		params->opcodes.die_erase = SPINOR_OP_CYPRESS_DIE_ERASE;
 
 	return 0;
 }
@@ -763,7 +766,7 @@ static int s28hx_t_late_init(struct spi_nor *nor)
 	cypress_nor_ecc_init(nor);
 
 	if (params->n_dice > 1)
-		params->die_erase_opcode = SPINOR_OP_CYPRESS_DIE_ERASE;
+		params->opcodes.die_erase = SPINOR_OP_CYPRESS_DIE_ERASE;
 
 	return 0;
 }
@@ -868,13 +871,6 @@ static const struct flash_info spansion_nor_parts[] = {
 		.name = "s25fl256s0",
 		.size = SZ_32M,
 		.sector_size = SZ_256K,
-		.no_sfdp_flags = SPI_NOR_SKIP_SFDP | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.mfr_flags = USE_CLSR,
-	}, {
-		.id = SNOR_ID(0x01, 0x02, 0x19, 0x4d, 0x00, 0x81),
-		.name = "s25fs256s0",
-		.size = SZ_32M,
-		.sector_size = SZ_256K,
 		.no_sfdp_flags = SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 		.mfr_flags = USE_CLSR,
 	}, {
@@ -904,7 +900,6 @@ static const struct flash_info spansion_nor_parts[] = {
 		.sector_size = SZ_256K,
 		.no_sfdp_flags = SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 		.mfr_flags = USE_CLSR,
-		.fixups = &s25fs_s_nor_fixups,
 	}, {
 		.id = SNOR_ID(0x01, 0x20, 0x18, 0x03, 0x00),
 		.name = "s25sl12800",
@@ -940,7 +935,6 @@ static const struct flash_info spansion_nor_parts[] = {
 		.size = SZ_16M,
 		.no_sfdp_flags = SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 		.mfr_flags = USE_CLSR,
-		.fixups = &s25fs_s_nor_fixups,
 	}, {
 		.id = SNOR_ID(0x01, 0x20, 0x18, 0x4d, 0x01),
 		.name = "s25fl129p1",
@@ -977,19 +971,16 @@ static const struct flash_info spansion_nor_parts[] = {
 		.name = "s25fl064l",
 		.size = SZ_8M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 	}, {
 		.id = SNOR_ID(0x01, 0x60, 0x18),
 		.name = "s25fl128l",
 		.size = SZ_16M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 	}, {
 		.id = SNOR_ID(0x01, 0x60, 0x19),
 		.name = "s25fl256l",
 		.size = SZ_32M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixup_flags = SPI_NOR_4B_OPCODES,
 	}, {
 		.id = SNOR_ID(0x04, 0x2c, 0xc2, 0x7f, 0x7f, 0x7f),
 		.name = "cy15x104q",
@@ -1000,76 +991,61 @@ static const struct flash_info spansion_nor_parts[] = {
 		.id = SNOR_ID(0x34, 0x2a, 0x1a, 0x0f, 0x03, 0x90),
 		.name = "s25hl512t",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s25hx_t_fixups
 	}, {
 		.id = SNOR_ID(0x34, 0x2a, 0x1b, 0x0f, 0x03, 0x90),
 		.name = "s25hl01gt",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s25hx_t_fixups
 	}, {
 		.id = SNOR_ID(0x34, 0x2a, 0x1c, 0x0f, 0x00, 0x90),
 		.name = "s25hl02gt",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s25hx_t_fixups
 	}, {
 		.id = SNOR_ID(0x34, 0x2b, 0x19, 0x0f, 0x08, 0x90),
 		.name = "s25fs256t",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s25fs256t_fixups
 	}, {
 		.id = SNOR_ID(0x34, 0x2b, 0x1a, 0x0f, 0x03, 0x90),
 		.name = "s25hs512t",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s25hx_t_fixups
 	}, {
 		.id = SNOR_ID(0x34, 0x2b, 0x1b, 0x0f, 0x03, 0x90),
 		.name = "s25hs01gt",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s25hx_t_fixups
 	}, {
 		.id = SNOR_ID(0x34, 0x2b, 0x1c, 0x0f, 0x00, 0x90),
 		.name = "s25hs02gt",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s25hx_t_fixups
 	}, {
 		/* S28HL256T */
 		.id = SNOR_ID(0x34, 0x5a, 0x19),
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		.id = SNOR_ID(0x34, 0x5a, 0x1a),
 		.name = "s28hl512t",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		.id = SNOR_ID(0x34, 0x5a, 0x1b),
 		.name = "s28hl01gt",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		/* S28HL02GT */
 		.id = SNOR_ID(0x34, 0x5a, 0x1c),
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		.id = SNOR_ID(0x34, 0x5b, 0x19),
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		.id = SNOR_ID(0x34, 0x5b, 0x1a),
 		.name = "s28hs512t",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		.id = SNOR_ID(0x34, 0x5b, 0x1b),
 		.name = "s28hs01gt",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		.id = SNOR_ID(0x34, 0x5b, 0x1c),
 		.name = "s28hs02gt",
 		.mfr_flags = USE_CLPEF,
-		.fixups = &s28hx_t_fixups,
 	}, {
 		.id = SNOR_ID(0xef, 0x40, 0x13),
 		.name = "s25fl004k",
@@ -1103,13 +1079,14 @@ static const struct flash_info spansion_nor_parts[] = {
 static int spansion_nor_sr_ready_and_clear(struct spi_nor *nor)
 {
 	int ret;
+	u8 sr;
 
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr1(nor, &sr);
 	if (ret)
 		return ret;
 
-	if (nor->bouncebuf[0] & (SR_E_ERR | SR_P_ERR)) {
-		if (nor->bouncebuf[0] & SR_E_ERR)
+	if (sr & (SR_E_ERR | SR_P_ERR)) {
+		if (sr & SR_E_ERR)
 			dev_err(nor->dev, "Erase Error occurred\n");
 		else
 			dev_err(nor->dev, "Programming Error occurred\n");
@@ -1129,7 +1106,7 @@ static int spansion_nor_sr_ready_and_clear(struct spi_nor *nor)
 		return -EIO;
 	}
 
-	return !(nor->bouncebuf[0] & SR_WIP);
+	return !(sr & SR_WIP);
 }
 
 static int spansion_nor_late_init(struct spi_nor *nor)
@@ -1139,7 +1116,7 @@ static int spansion_nor_late_init(struct spi_nor *nor)
 	u8 mfr_flags = nor->info->mfr_flags;
 
 	if (params->size > SZ_16M) {
-		nor->flags |= SNOR_F_4B_OPCODES;
+		params->flags |= SNOR_F_4B_OPCODES;
 		/* No small sector erase for 4-byte command set */
 		nor->erase_opcode = SPINOR_OP_SE;
 		nor->mtd.erasesize = nor->info->sector_size ?:
@@ -1168,9 +1145,35 @@ static const struct spi_nor_fixups spansion_nor_fixups = {
 	.late_init = spansion_nor_late_init,
 };
 
+static const struct spi_nor_fixup spansion_fixups[] = {
+	{ .fixups = &spansion_nor_fixups },
+	{ .id = SNOR_ID(0x01, 0x02, 0x20, 0x4d, 0x00, 0x81), .fixups = &s25fs_s_nor_fixups },
+	{ .id = SNOR_ID(0x01, 0x02, 0x19, 0x4d, 0x01, 0x81), .fixups = &s25fs_s_nor_fixups },
+	{ .id = SNOR_ID(0x01, 0x20, 0x18, 0x4d, 0x01, 0x81), .fixups = &s25fs_s_nor_fixups },
+	{ .id = SNOR_ID(0x01, 0x60, 0x17), .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0x01, 0x60, 0x18), .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0x01, 0x60, 0x19), .fixup_flags = SPI_NOR_4B_OPCODES },
+	{ .id = SNOR_ID(0x34, 0x2a, 0x1a, 0x0f, 0x03, 0x90), .fixups = &s25hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x2a, 0x1b, 0x0f, 0x03, 0x90), .fixups = &s25hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x2a, 0x1c, 0x0f, 0x00, 0x90), .fixups = &s25hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x2b, 0x19, 0x0f, 0x08, 0x90), .fixups = &s25fs256t_fixups },
+	{ .id = SNOR_ID(0x34, 0x2b, 0x1a, 0x0f, 0x03, 0x90), .fixups = &s25hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x2b, 0x1b, 0x0f, 0x03, 0x90), .fixups = &s25hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x2b, 0x1c, 0x0f, 0x00, 0x90), .fixups = &s25hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5a, 0x19), .fixups = &s28hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5a, 0x1a), .fixups = &s28hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5a, 0x1b), .fixups = &s28hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5a, 0x1c), .fixups = &s28hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5b, 0x19), .fixups = &s28hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5b, 0x1a), .fixups = &s28hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5b, 0x1b), .fixups = &s28hx_t_fixups },
+	{ .id = SNOR_ID(0x34, 0x5b, 0x1c), .fixups = &s28hx_t_fixups },
+};
+
 const struct spi_nor_manufacturer spi_nor_spansion = {
 	.name = "spansion",
 	.parts = spansion_nor_parts,
 	.nparts = ARRAY_SIZE(spansion_nor_parts),
-	.fixups = &spansion_nor_fixups,
+	.fixups = spansion_fixups,
+	.nfixups = ARRAY_SIZE(spansion_fixups),
 };
diff --git a/drivers/mtd/spi-nor/sst.c b/drivers/mtd/spi-nor/sst.c
index db02c14ba16faf..c460803b278bfd 100644
--- a/drivers/mtd/spi-nor/sst.c
+++ b/drivers/mtd/spi-nor/sst.c
@@ -21,16 +21,17 @@ static int sst26vf_nor_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 static int sst26vf_nor_unlock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	int ret;
+	u8 cr;
 
 	/* We only support unlocking the entire flash array. */
 	if (ofs != 0 || len != nor->params->size)
 		return -EINVAL;
 
-	ret = spi_nor_read_cr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr2(nor, &cr);
 	if (ret)
 		return ret;
 
-	if (!(nor->bouncebuf[0] & SST26VF_CR_BPNV)) {
+	if (!(cr & SST26VF_CR_BPNV)) {
 		dev_dbg(nor->dev, "Any block has been permanently locked\n");
 		return -EINVAL;
 	}
@@ -151,14 +152,12 @@ static const struct flash_info sst_nor_parts[] = {
 		.id = SNOR_ID(0xbf, 0x26, 0x42),
 		.name = "sst26vf032b",
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
-		.fixups = &sst26vf_nor_fixups,
 	}, {
 		.id = SNOR_ID(0xbf, 0x26, 0x43),
 		.name = "sst26vf064b",
 		.size = SZ_8M,
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_SWP_IS_VOLATILE,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixups = &sst26vf_nor_fixups,
 	}, {
 		.id = SNOR_ID(0xbf, 0x26, 0x51),
 		.name = "sst26wf016b",
@@ -277,9 +276,16 @@ static const struct spi_nor_fixups sst_nor_fixups = {
 	.late_init = sst_nor_late_init,
 };
 
+static const struct spi_nor_fixup sst_fixups[] = {
+	{ .fixups = &sst_nor_fixups },
+	{ .id = SNOR_ID(0xbf, 0x26, 0x42), .fixups = &sst26vf_nor_fixups },
+	{ .id = SNOR_ID(0xbf, 0x26, 0x43), .fixups = &sst26vf_nor_fixups },
+};
+
 const struct spi_nor_manufacturer spi_nor_sst = {
 	.name = "sst",
 	.parts = sst_nor_parts,
 	.nparts = ARRAY_SIZE(sst_nor_parts),
-	.fixups = &sst_nor_fixups,
+	.fixups = sst_fixups,
+	.nfixups = ARRAY_SIZE(sst_fixups),
 };
diff --git a/drivers/mtd/spi-nor/swp.c b/drivers/mtd/spi-nor/swp.c
index 235070b215d1e1..7e667e4ca84d51 100644
--- a/drivers/mtd/spi-nor/swp.c
+++ b/drivers/mtd/spi-nor/swp.c
@@ -13,12 +13,13 @@
 
 static u8 spi_nor_get_sr_bp_mask(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	u8 mask = SR_BP2 | SR_BP1 | SR_BP0;
 
-	if (nor->flags & SNOR_F_HAS_SR_BP3_BIT6)
+	if (params->flags & SNOR_F_HAS_SR_BP3_BIT6)
 		return mask | SR_BP3_BIT6;
 
-	if (nor->flags & SNOR_F_HAS_4BIT_BP)
+	if (params->flags & SNOR_F_HAS_4BIT_BP)
 		return mask | SR_BP3;
 
 	return mask;
@@ -26,9 +27,11 @@ static u8 spi_nor_get_sr_bp_mask(struct spi_nor *nor)
 
 static u8 spi_nor_get_sr_tb_mask(struct spi_nor *nor)
 {
-	if (nor->flags & SNOR_F_HAS_SR_TB_BIT6)
+	struct spi_nor_flash_parameter *params = nor->params;
+
+	if (params->flags & SNOR_F_HAS_SR_TB_BIT6)
 		return SR_TB_BIT6;
-	else if (nor->flags & SNOR_F_HAS_SR_TB)
+	else if (params->flags & SNOR_F_HAS_SR_TB)
 		return SR_TB_BIT5;
 	else
 		return 0;
@@ -36,8 +39,10 @@ static u8 spi_nor_get_sr_tb_mask(struct spi_nor *nor)
 
 static u8 spi_nor_get_sr_cmp_mask(struct spi_nor *nor)
 {
-	if (!(nor->flags & SNOR_F_NO_READ_CR) &&
-	    nor->flags & SNOR_F_HAS_SR2_CMP_BIT6)
+	struct spi_nor_flash_parameter *params = nor->params;
+
+	if (params->opcodes.read_sr2 &&
+	    params->flags & SNOR_F_HAS_SR2_CMP_BIT6)
 		return SR2_CMP_BIT6;
 	else
 		return 0;
@@ -67,15 +72,16 @@ u64 spi_nor_get_min_prot_length_sr(struct spi_nor *nor)
 void spi_nor_get_locked_range_sr(struct spi_nor *nor, const u8 *sr, loff_t *ofs,
 				 u64 *len)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	u64 min_prot_len;
 	u8 bp_mask = spi_nor_get_sr_bp_mask(nor);
 	u8 tb_mask = spi_nor_get_sr_tb_mask(nor);
 	u8 cmp_mask = spi_nor_get_sr_cmp_mask(nor);
 	u8 bp, val = sr[0] & bp_mask;
-	bool tb = (nor->flags & SNOR_F_HAS_SR_TB) ? sr[0] & tb_mask : 0;
+	bool tb = (params->flags & SNOR_F_HAS_SR_TB) ? sr[0] & tb_mask : 0;
 	bool cmp = sr[1] & cmp_mask;
 
-	if (nor->flags & SNOR_F_HAS_SR_BP3_BIT6 && val & SR_BP3_BIT6)
+	if (params->flags & SNOR_F_HAS_SR_BP3_BIT6 && val & SR_BP3_BIT6)
 		val = (val & ~SR_BP3_BIT6) | SR_BP3;
 
 	bp = val >> SR_BP_SHIFT;
@@ -153,10 +159,11 @@ static bool spi_nor_is_unlocked_sr(struct spi_nor *nor, loff_t ofs, u64 len,
 
 static int spi_nor_sr_set_bp_mask(struct spi_nor *nor, u8 *sr, u8 pow)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	u8 mask = spi_nor_get_sr_bp_mask(nor);
 	u8 val = pow << SR_BP_SHIFT;
 
-	if (nor->flags & SNOR_F_HAS_SR_BP3_BIT6 && val & SR_BP3)
+	if (params->flags & SNOR_F_HAS_SR_BP3_BIT6 && val & SR_BP3)
 		val = (val & ~SR_BP3) | SR_BP3_BIT6;
 
 	if (val & ~mask)
@@ -194,6 +201,45 @@ static int spi_nor_build_sr(struct spi_nor *nor, const u8 *old_sr, u8 *new_sr,
 	return 0;
 }
 
+/*
+ * Make sure we do our best to guess SR2. This has historically only be needed
+ * for swp.c, so let's keep this extra carefulness in this file.
+ */
+static int spi_nor_read_sr2_careful(struct spi_nor *nor, u8 *sr2)
+{
+	struct spi_nor_flash_parameter *params = nor->params;
+	int ret;
+
+	if (params->opcodes.read_sr2) {
+		ret = spi_nor_read_sr_ll(nor, params->opcodes.read_sr2, sr2, 1);
+		if (ret)
+			return ret;
+	} else if ((spi_nor_get_protocol_width(nor->read_proto) == 4 ||
+		   spi_nor_get_protocol_width(nor->write_proto) == 4) &&
+		   nor->params->quad_enable) {
+		/*
+		 * Make sure the QE bit is persistently kept. qe_mask[1] will be
+		 * 0 if the QE bit is in SR1.
+		 */
+		*sr2 = params->qe_mask[1];
+	} else {
+		return 0;
+	}
+
+	return 0;
+}
+
+static int spi_nor_read_sr1_and_sr2_careful(struct spi_nor *nor, u8 *sr)
+{
+	int ret;
+
+	ret = spi_nor_read_sr1(nor, &sr[0]);
+	if (ret)
+		return ret;
+
+	return spi_nor_read_sr2_careful(nor, &sr[1]);
+}
+
 /*
  * Keep a local cache containing all lock-related bits for debugfs use only.
  * This way, debugfs never needs to access the flash directly.
@@ -207,17 +253,9 @@ void spi_nor_cache_sr_lock_bits(struct spi_nor *nor, u8 *sr)
 
 
 	if (!sr) {
-		if (spi_nor_read_sr(nor, nor->bouncebuf))
+		if (spi_nor_read_sr1_and_sr2_careful(nor, sr_cr))
 			return;
 
-		sr_cr[0] = nor->bouncebuf[0];
-
-		if (!(nor->flags & SNOR_F_NO_READ_CR)) {
-			if (spi_nor_read_cr(nor, nor->bouncebuf))
-				return;
-		}
-
-		sr_cr[1] = nor->bouncebuf[0];
 		sr = sr_cr;
 	}
 
@@ -261,32 +299,23 @@ void spi_nor_cache_sr_lock_bits(struct spi_nor *nor, u8 *sr)
  */
 static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	u64 min_prot_len = spi_nor_get_min_prot_length_sr(nor);
 	u8 status_old[2] = {}, status_new[2] = {}, status_new_cmp[2] = {};
 	u8 *best_status_new = status_new;
 	loff_t ofs_old, ofs_new, ofs_new_cmp;
 	u64 len_old, len_new, len_new_cmp;
 	loff_t lock_len;
-	bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB,
+	bool can_be_top = true, can_be_bottom = params->flags & SNOR_F_HAS_SR_TB,
 		can_be_cmp = spi_nor_get_sr_cmp_mask(nor);
 	bool use_top;
 	int ret;
 	u8 pow;
 
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr1_and_sr2_careful(nor, status_old);
 	if (ret)
 		return ret;
 
-	status_old[0] = nor->bouncebuf[0];
-
-	if (!(nor->flags & SNOR_F_NO_READ_CR)) {
-		ret = spi_nor_read_cr(nor, nor->bouncebuf);
-		if (ret)
-			return ret;
-
-		status_old[1] = nor->bouncebuf[0];
-	}
-
 	/* If nothing in our range is unlocked, we don't need to do anything */
 	if (spi_nor_is_locked_sr(nor, ofs, len, status_old))
 		return 0;
@@ -313,7 +342,7 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 		lock_len = ofs + len;
 
 	if (lock_len == nor->params->size)
-		pow = (nor->flags & SNOR_F_HAS_4BIT_BP) ? GENMASK(3, 0) : GENMASK(2, 0);
+		pow = (params->flags & SNOR_F_HAS_4BIT_BP) ? GENMASK(3, 0) : GENMASK(2, 0);
 	else
 		pow = ilog2(lock_len) - ilog2(min_prot_len) + 1;
 
@@ -359,7 +388,7 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 	 * wrongly tied to GND (that includes internal pull-downs).
 	 * WP# pin hard strapped to GND can be a valid use case.
 	 */
-	if (!(nor->flags & SNOR_F_NO_WP))
+	if (!(params->flags & SNOR_F_NO_WP))
 		best_status_new[0] |= SR_SRWD;
 
 	spi_nor_get_locked_range_sr(nor, status_old, &ofs_old, &len_old);
@@ -378,10 +407,7 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 	    (ofs_old < ofs_new || (ofs_new + len_new) < (ofs_old + len_old)))
 		return -EINVAL;
 
-	if (nor->flags & SNOR_F_NO_READ_CR)
-		ret = spi_nor_write_sr_and_check(nor, best_status_new[0]);
-	else
-		ret = spi_nor_write_sr_cr_and_check(nor, best_status_new);
+	ret = spi_nor_write_sr1_and_sr2_and_check(nor, best_status_new);
 	if (ret)
 		return ret;
 
@@ -397,32 +423,23 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, u64 len)
  */
 static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	u64 min_prot_len = spi_nor_get_min_prot_length_sr(nor);
 	u8 status_old[2] = {}, status_new[2] = {}, status_new_cmp[2] = {};
 	u8 *best_status_new = status_new;
 	loff_t ofs_old, ofs_new, ofs_new_cmp;
 	u64 len_old, len_new, len_new_cmp;
 	loff_t lock_len;
-	bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB,
+	bool can_be_top = true, can_be_bottom = params->flags & SNOR_F_HAS_SR_TB,
 		can_be_cmp = spi_nor_get_sr_cmp_mask(nor);
 	bool use_top;
 	int ret;
 	u8 pow;
 
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr1_and_sr2_careful(nor, status_old);
 	if (ret)
 		return ret;
 
-	status_old[0] = nor->bouncebuf[0];
-
-	if (!(nor->flags & SNOR_F_NO_READ_CR)) {
-		ret = spi_nor_read_cr(nor, nor->bouncebuf);
-		if (ret)
-			return ret;
-
-		status_old[1] = nor->bouncebuf[0];
-	}
-
 	/* If nothing in our range is locked, we don't need to do anything */
 	if (spi_nor_is_unlocked_sr(nor, ofs, len, status_old))
 		return 0;
@@ -512,10 +529,7 @@ static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, u64 len)
 	    (ofs_new < ofs_old || (ofs_old + len_old) < (ofs_new + len_new)))
 		return -EINVAL;
 
-	if (nor->flags & SNOR_F_NO_READ_CR)
-		ret = spi_nor_write_sr_and_check(nor, best_status_new[0]);
-	else
-		ret = spi_nor_write_sr_cr_and_check(nor, best_status_new);
+	ret = spi_nor_write_sr1_and_sr2_and_check(nor, best_status_new);
 	if (ret)
 		return ret;
 
@@ -536,20 +550,10 @@ static int spi_nor_sr_is_locked(struct spi_nor *nor, loff_t ofs, u64 len)
 	u8 sr_cr[2] = {};
 	int ret;
 
-	ret = spi_nor_read_sr(nor, nor->bouncebuf);
+	ret = spi_nor_read_sr1_and_sr2_careful(nor, sr_cr);
 	if (ret)
 		return ret;
 
-	sr_cr[0] = nor->bouncebuf[0];
-
-	if (!(nor->flags & SNOR_F_NO_READ_CR)) {
-		ret = spi_nor_read_cr(nor, nor->bouncebuf);
-		if (ret)
-			return ret;
-
-		sr_cr[1] = nor->bouncebuf[0];
-	}
-
 	return spi_nor_is_locked_sr(nor, ofs, len, sr_cr);
 }
 
@@ -628,9 +632,10 @@ static int spi_nor_is_locked(struct mtd_info *mtd, loff_t ofs, u64 len)
  */
 void spi_nor_try_unlock_all(struct spi_nor *nor)
 {
+	struct spi_nor_flash_parameter *params = nor->params;
 	int ret;
 
-	if (!(nor->flags & SNOR_F_HAS_LOCK))
+	if (!(params->flags & SNOR_F_HAS_LOCK))
 		return;
 
 	dev_dbg(nor->dev, "Unprotecting entire flash array\n");
diff --git a/drivers/mtd/spi-nor/sysfs.c b/drivers/mtd/spi-nor/sysfs.c
index 643513ee891b7e..a2dabe4c6422a7 100644
--- a/drivers/mtd/spi-nor/sysfs.c
+++ b/drivers/mtd/spi-nor/sysfs.c
@@ -25,7 +25,7 @@ static ssize_t partname_show(struct device *dev,
 	struct spi_mem *spimem = spi_get_drvdata(spi);
 	struct spi_nor *nor = spi_mem_get_drvdata(spimem);
 
-	return sysfs_emit(buf, "%s\n", nor->info->name);
+	return sysfs_emit(buf, "%s\n", nor->partname);
 }
 static DEVICE_ATTR_RO(partname);
 
@@ -78,7 +78,7 @@ static umode_t spi_nor_sysfs_is_visible(struct kobject *kobj,
 
 	if (attr == &dev_attr_manufacturer.attr && !nor->manufacturer)
 		return 0;
-	if (attr == &dev_attr_partname.attr && !nor->info->name)
+	if (attr == &dev_attr_partname.attr && !nor->partname)
 		return 0;
 	if (attr == &dev_attr_jedec_id.attr && !nor->info->id && !nor->id)
 		return 0;
diff --git a/drivers/mtd/spi-nor/winbond.c b/drivers/mtd/spi-nor/winbond.c
index 8ebdbcec0b3fde..51f6bcd3f8e5b8 100644
--- a/drivers/mtd/spi-nor/winbond.c
+++ b/drivers/mtd/spi-nor/winbond.c
@@ -24,20 +24,94 @@
 		   SPI_MEM_OP_NO_DUMMY,					\
 		   SPI_MEM_OP_DATA_OUT(1, buf, 0))
 
-static int
-w25q128_post_bfpt_fixups(struct spi_nor *nor,
-			 const struct sfdp_parameter_header *bfpt_header,
-			 const struct sfdp_bfpt *bfpt)
+static bool is_w25qxxrv(const struct spi_nor *nor)
+{
+	struct sfdp_header *sfdp_h = spi_nor_sfdp_get_header(nor);
+
+	/*
+	 * W25QxxRV chips re-use the same ID as the W25QxxJV family.
+	 *
+	 * Chips are very similar, W25QxxRV brings mostly performance and power
+	 * consumption improvements. The RV family does not require the multi
+	 * die fixup.
+	 *
+	 * They can be distinguished based on their SFDP minor revision:
+	 * W25QxxJV:        JESD216A, minor revision == 05h
+	 * W25Q512/01/02JV: JESD216B, minor revision == 06h
+	 * W25QxxRV:        JESD216F, minor revision >= 0Ah
+	 */
+	return sfdp_h->minor >= SFDP_JESD216F_MINOR;
+}
+
+static bool is_zd25q128c(const struct spi_nor *nor,
+			 const struct sfdp_parameter_header *bfpt_header)
 {
 	/*
 	 * Zetta ZD25Q128C is a clone of the Winbond device. But the encoded
 	 * size is really wrong. It seems that they confused Mbit with MiB.
 	 * Thus the flash is discovered as a 2MiB device.
 	 */
-	if (bfpt_header->major == SFDP_JESD216_MAJOR &&
-	    bfpt_header->minor == SFDP_JESD216_MINOR &&
-	    nor->params->size == SZ_2M &&
-	    nor->params->erase_map.regions[0].size == SZ_2M) {
+	return bfpt_header->major == SFDP_JESD216_MAJOR &&
+	       bfpt_header->minor == SFDP_JESD216_MINOR &&
+	       nor->params->size == SZ_2M &&
+	       nor->params->erase_map.regions[0].size == SZ_2M;
+}
+
+/*
+ * Since SFDP is populated after ->default_init(), the match functions using
+ * nor->sfdp as discriminant cannot be used for this specific early fixup.
+ */
+static bool winbond_jv_match(const struct spi_nor *nor)
+{
+	return !nor->sfdp || !is_w25qxxrv(nor);
+}
+
+static bool winbond_rv_match(const struct spi_nor *nor)
+{
+	return nor->sfdp && is_w25qxxrv(nor);
+}
+
+static bool is_w25qxxpw(const struct spi_nor *nor)
+{
+	struct sfdp_header *sfdp_h = spi_nor_sfdp_get_header(nor);
+
+	/*
+	 * W25QxxPW chips re-use the same ID as the W25QxxJW/NW family.
+	 *
+	 * Chips are very similar, W25QxxPW brings mostly performance and power
+	 * consumption improvements. One key difference in behaviour is the
+	 * automatic 16-byte based error correction.
+	 *
+	 * They can be distinguished based on their SFDP minor revision:
+	 * W25QxxJW:       JESD216B, minor revision == 06h
+	 * W25Q51/01/02NW: JESD216B, minor revision == 06h
+	 * W25QxxPW:       JESD216F, minor revision >= 0Ah
+	 */
+	return sfdp_h->minor >= SFDP_JESD216F_MINOR;
+}
+
+static bool winbond_pw_match(const struct spi_nor *nor)
+{
+	return nor->sfdp && is_w25qxxpw(nor);
+}
+
+static bool winbond_pw_with_ecc_match(const struct spi_nor *nor)
+{
+	const struct spi_nor_id *id = nor->info->id;
+
+	if (!winbond_pw_match(nor))
+		return false;
+
+	/* W25QxxPW densities <= 32 Mbit (id[2] <= 0x16) do not have built-in ECC support */
+	return id->len == 3 && id->bytes[2] >= 0x17;
+}
+
+static int
+w25q128_post_bfpt_fixups(struct spi_nor *nor,
+			 const struct sfdp_parameter_header *bfpt_header,
+			 const struct sfdp_bfpt *bfpt)
+{
+	if (is_zd25q128c(nor, bfpt_header) || winbond_rv_match(nor)) {
 		nor->params->size = SZ_16M;
 		nor->params->erase_map.regions[0].size = SZ_16M;
 	}
@@ -64,7 +138,7 @@ w25q256_post_bfpt_fixups(struct spi_nor *nor,
 	 */
 	if (bfpt_header->major == SFDP_JESD216_MAJOR &&
 	    bfpt_header->minor == SFDP_JESD216A_MINOR)
-		nor->flags |= SNOR_F_4B_OPCODES;
+		nor->params->flags |= SNOR_F_4B_OPCODES;
 
 	return 0;
 }
@@ -73,26 +147,6 @@ static const struct spi_nor_fixups w25q256_fixups = {
 	.post_bfpt = w25q256_post_bfpt_fixups,
 };
 
-static int
-winbond_rdcr_post_bfpt_fixup(struct spi_nor *nor,
-			     const struct sfdp_parameter_header *bfpt_header,
-			     const struct sfdp_bfpt *bfpt)
-{
-	/*
-	 * W25H02NW, unlike its W25H512NW nor W25H01NW cousins, improperly sets
-	 * the QE BFPT configuration bits, indicating a non readable CR. This is
-	 * both incorrect and impractical, as the chip features a CMP bit for its
-	 * locking scheme that lays in the Control Register, and needs to be read.
-	 */
-	nor->flags &= ~SNOR_F_NO_READ_CR;
-
-	return 0;
-}
-
-static const struct spi_nor_fixups winbond_rdcr_fixup = {
-	.post_bfpt = winbond_rdcr_post_bfpt_fixup,
-};
-
 /**
  * winbond_nor_select_die() - Set active die.
  * @nor:	pointer to 'struct spi_nor'.
@@ -166,6 +220,38 @@ static const struct spi_nor_fixups winbond_nor_multi_die_fixups = {
 	.post_sfdp = winbond_nor_multi_die_post_sfdp_fixups,
 };
 
+static int winbond_nor_partname_post_sfdp_fixups(struct spi_nor *nor)
+{
+	/*
+	 * W25QxxRV parts re-use the JEDEC IDs of the JV family. Their name
+	 * being a legacy field, it is kept for the already established JV parts
+	 * but must not be exposed by the newer RV ones.
+	 */
+	nor->partname = NULL;
+
+	return 0;
+}
+
+static const struct spi_nor_fixups winbond_nor_partname_fixups = {
+	.post_sfdp = winbond_nor_partname_post_sfdp_fixups,
+};
+
+static int winbond_nor_ecc_configuration_post_sfdp_fixups(struct spi_nor *nor)
+{
+	/*
+	 * PW chips feature automatic error correction. Non 16-byte aligned
+	 * writes work, but disable error correction on the region until next erase.
+	 */
+	nor->params->writesize = 16;
+	nor->params->flags |= SNOR_F_ECC;
+
+	return 0;
+}
+
+static const struct spi_nor_fixups winbond_nor_ecc_configuration_fixups = {
+	.post_sfdp = winbond_nor_ecc_configuration_post_sfdp_fixups,
+};
+
 static const struct flash_info winbond_nor_parts[] = {
 	{
 		.id = SNOR_ID(0xef, 0x30, 0x10),
@@ -218,38 +304,53 @@ static const struct flash_info winbond_nor_parts[] = {
 		.size = SZ_1M,
 		.no_sfdp_flags = SECT_4K,
 	}, {
+		/* W25Q32JV-Q/N, W25Q32RV-Q/N */
 		.id = SNOR_ID(0xef, 0x40, 0x16),
 		.name = "w25q32",
 		.size = SZ_4M,
 		.no_sfdp_flags = SECT_4K,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q64JV-Q/N, W25Q64RV-Q/N */
 		.id = SNOR_ID(0xef, 0x40, 0x17),
 		.name = "w25q64",
 		.size = SZ_8M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q128JV-Q/N, W25Q12RV-Q/N */
 		.id = SNOR_ID(0xef, 0x40, 0x18),
 		/* Flavors w/ and w/o SFDP. */
 		.name = "w25q128",
 		.size = SZ_16M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixups = &w25q128_fixups,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q256JV-Q/N, W25Q25RV-Q/N */
 		.id = SNOR_ID(0xef, 0x40, 0x19),
 		.name = "w25q256",
 		.size = SZ_32M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
-		.fixups = &w25q256_fixups,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB |
+			 SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q512JV-Q/N, W25Q51RV-Q/N */
 		.id = SNOR_ID(0xef, 0x40, 0x20),
 		.name = "w25q512jvq",
 		.size = SZ_64M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
-		/* W25Q01JV */
+		/* W25Q01JV-Q/N, W25Q01RV-Q/N */
 		.id = SNOR_ID(0xef, 0x40, 0x21),
-		.fixups = &winbond_nor_multi_die_fixups,
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
+	}, {
+		/* W25Q02RV-Q/N */
+		.id = SNOR_ID(0xef, 0x40, 0x22),
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
 		.id = SNOR_ID(0xef, 0x50, 0x12),
 		.name = "w25q20bw",
@@ -272,133 +373,166 @@ static const struct flash_info winbond_nor_parts[] = {
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q32DW-P/G, W25Q32FW-G/Q, W25Q32JW-Q/N, W25Q33PW-Q/N */
 		.id = SNOR_ID(0xef, 0x60, 0x16),
 		.name = "w25q32dw",
 		.size = SZ_4M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 		.otp = SNOR_OTP(256, 3, 0x1000, 0x1000),
 	}, {
+		/* W25Q64DW-P/G, W25Q64FW-G/Q, W25Q64JW-Q/N, W25Q64PW-Q/N */
 		.id = SNOR_ID(0xef, 0x60, 0x17),
 		.name = "w25q64dw",
 		.size = SZ_8M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q128FW-G/Q, W25Q128JW-Q/N, W25Q12PW-Q/N */
 		.id = SNOR_ID(0xef, 0x60, 0x18),
 		.name = "w25q128fw",
 		.size = SZ_16M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q256JW-Q/N, W25Q25PW-Q/N */
 		.id = SNOR_ID(0xef, 0x60, 0x19),
 		.name = "w25q256jw",
 		.size = SZ_32M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP,
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q512NW-Q/N */
 		.id = SNOR_ID(0xef, 0x60, 0x20),
 		.name = "w25q512nwq",
 		.otp = SNOR_OTP(256, 3, 0x1000, 0x1000),
 	}, {
+		/* W25Q01NW-Q/N */
+		.id = SNOR_ID(0xef, 0x60, 0x21),
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
+	}, {
+		/* W25Q16JV-M */
 		.id = SNOR_ID(0xef, 0x70, 0x15),
 		.name = "w25q16jv-im/jm",
 		.size = SZ_2M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q32JV-M, W25Q32RV-M */
 		.id = SNOR_ID(0xef, 0x70, 0x16),
 		.name = "w25q32jv",
 		.size = SZ_4M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q64JV-M, W25Q64RV-M */
 		.id = SNOR_ID(0xef, 0x70, 0x17),
 		.name = "w25q64jvm",
 		.size = SZ_8M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
 		.no_sfdp_flags = SECT_4K,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q128JV-M, W25Q12RV-M */
 		.id = SNOR_ID(0xef, 0x70, 0x18),
 		.name = "w25q128jv",
 		.size = SZ_16M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 	}, {
+		/* W25Q256JV-M, W25Q25RV-M */
 		.id = SNOR_ID(0xef, 0x70, 0x19),
 		.name = "w25q256jvm",
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB |
+			 SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
-		/* W25Q02JV */
+		/* W25Q512JV-M, W25Q51RV-M */
+		.id = SNOR_ID(0xef, 0x70, 0x20),
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
+	}, {
+		/* W25Q01JV-M, W25Q01RV-M */
+		.id = SNOR_ID(0xef, 0x70, 0x21),
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
+	}, {
+		/* W25Q02JV-M, W25Q02RV-M */
 		.id = SNOR_ID(0xef, 0x70, 0x22),
-		.fixups = &winbond_nor_multi_die_fixups,
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
 		.id = SNOR_ID(0xef, 0x71, 0x19),
 		.name = "w25m512jv",
 		.size = SZ_64M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q32JW-M, W25Q33PW-M */
 		.id = SNOR_ID(0xef, 0x80, 0x16),
 		.name = "w25q32jwm",
 		.size = SZ_4M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 		.otp = SNOR_OTP(256, 3, 0x1000, 0x1000),
 	}, {
+		/* W25Q64JW-M, W25Q64PW-M */
 		.id = SNOR_ID(0xef, 0x80, 0x17),
 		.name = "w25q64jwm",
 		.size = SZ_8M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q128JW-M, W25Q12PW-M */
 		.id = SNOR_ID(0xef, 0x80, 0x18),
 		.name = "w25q128jwm",
 		.size = SZ_16M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB,
+		.flags = SPI_NOR_QUAD_PP | SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q256JW-M, W25Q25PW-M */
 		.id = SNOR_ID(0xef, 0x80, 0x19),
 		.name = "w25q256jwm",
 		.size = SZ_32M,
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 | SPI_NOR_4BIT_BP,
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
+			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
 	}, {
+		/* W25Q512NW-M */
 		.id = SNOR_ID(0xef, 0x80, 0x20),
 		.name = "w25q512nwm",
 		.otp = SNOR_OTP(256, 3, 0x1000, 0x1000),
 	}, {
-		/* W25Q01NWxxIQ */
-		.id = SNOR_ID(0xef, 0x60, 0x21),
-		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
-			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
-		.fixups = &winbond_rdcr_fixup,
-	}, {
-		/* W25Q01NWxxIM */
+		/* W25Q01NW-M */
 		.id = SNOR_ID(0xef, 0x80, 0x21),
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
 			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
-		/* W25Q02NWxxIM */
+		/* W25Q02NW-M */
 		.id = SNOR_ID(0xef, 0x80, 0x22),
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
 			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
-		.fixups = &winbond_rdcr_fixup,
 	}, {
-		/* W25H512NWxxAM */
+		/* W25H512NW-M */
 		.id = SNOR_ID(0xef, 0xa0, 0x20),
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
 			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
-		/* W25H01NWxxAM */
+		/* W25H01NW-M */
 		.id = SNOR_ID(0xef, 0xa0, 0x21),
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
 			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
 	}, {
-		/* W25H02NWxxAM */
+		/* W25H02NW-M */
 		.id = SNOR_ID(0xef, 0xa0, 0x22),
 		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_HAS_TB | SPI_NOR_TB_SR_BIT6 |
 			 SPI_NOR_4BIT_BP | SPI_NOR_HAS_CMP,
-		.fixups = &winbond_rdcr_fixup,
-	},
+	}, {
+		/*
+		 * Catch all entry to make sure all chips solely relying
+		 * on SFDP still go through the manufacturer hooks.
+		 */
+		.id = SNOR_ID(0xef),
+	}
 };
 
 /**
@@ -490,6 +624,38 @@ static int winbond_nor_late_init(struct spi_nor *nor)
 	 */
 	params->set_4byte_addr_mode = winbond_nor_set_4byte_addr_mode;
 
+	/*
+	 * All W25Q/W25H chips do set the BFPT_DWORD15_QER_SR2_BIT1_NO_RD bit in
+	 * their SFDP tables. The historical spi-nor assumption in this case has
+	 * been to declare CR reads as unsupported, whereas the Jedec
+	 * specification doesn't clearly state that. In practice, all these
+	 * chips do support reading back the CR, which is needed for SWP support,
+	 * so make sure that capability remains enabled.
+	 * In practice, only exclude the old W25X family (JEDEC ID: EF 30 xx)
+	 * which actually does not support this feature.
+	 */
+	if (nor->id[1] > 0x30)
+		params->opcodes.read_sr2 = SPINOR_OP_RDCR;
+
+	/*
+	 * Winbond has reused many IDs, up to four times at this
+	 * stage. In general, most of the chips with SFDP support are
+	 * correctly described by the ID table, but the non-SFDP chips,
+	 * however, are known to not feature as many capabilities. Make
+	 * sure we filter out those capabilities to keep backward
+	 * compatibility with these devices manufactured until ~2016.
+	 */
+	if (!nor->sfdp) {
+		struct spi_nor_flash_parameter *p = nor->params;
+
+		/* SPI_NOR_QUAD_PP was unsupported */
+		p->hwcaps.mask &= ~SNOR_HWCAPS_PP_1_1_4;
+		spi_nor_set_pp_settings(&p->page_programs[SNOR_CMD_PP_1_1_4], 0, 0);
+
+		/* SPI_NOR_HAS_CMP was unsupported */
+		nor->params->flags &= ~SNOR_F_HAS_SR2_CMP_BIT6;
+	}
+
 	return 0;
 }
 
@@ -497,9 +663,37 @@ static const struct spi_nor_fixups winbond_nor_fixups = {
 	.late_init = winbond_nor_late_init,
 };
 
+static const struct spi_nor_fixup winbond_fixups[] = {
+	{ .fixups = &winbond_nor_fixups },
+	{ .id = SNOR_ID(0xef, 0x40, 0x18), .fixups = &w25q128_fixups },
+	{ .id = SNOR_ID(0xef, 0x40, 0x19), .fixups = &w25q256_fixups },
+	{ .id = SNOR_ID(0xef, 0x40), .match = winbond_rv_match,
+	  .fixups = &winbond_nor_partname_fixups },
+	{ .id = SNOR_ID(0xef, 0x40, 0x21), .match = winbond_jv_match,
+	  .fixups = &winbond_nor_multi_die_fixups },
+	{ .id = SNOR_ID(0xef, 0x40, 0x22), .match = winbond_jv_match,
+	  .fixups = &winbond_nor_multi_die_fixups },
+	{ .id = SNOR_ID(0xef, 0x60), .match = winbond_pw_with_ecc_match,
+	  .fixups = &winbond_nor_ecc_configuration_fixups },
+	{ .id = SNOR_ID(0xef, 0x60), .match = winbond_pw_match,
+	  .fixups = &winbond_nor_partname_fixups },
+	{ .id = SNOR_ID(0xef, 0x70), .match = winbond_rv_match,
+	  .fixups = &winbond_nor_partname_fixups },
+	{ .id = SNOR_ID(0xef, 0x70, 0x18), .fixups = &w25q128_fixups },
+	{ .id = SNOR_ID(0xef, 0x70, 0x21), .match = winbond_jv_match,
+	  .fixups = &winbond_nor_multi_die_fixups },
+	{ .id = SNOR_ID(0xef, 0x70, 0x22), .match = winbond_jv_match,
+	  .fixups = &winbond_nor_multi_die_fixups },
+	{ .id = SNOR_ID(0xef, 0x80), .match = winbond_pw_with_ecc_match,
+	  .fixups = &winbond_nor_ecc_configuration_fixups },
+	{ .id = SNOR_ID(0xef, 0x80), .match = winbond_pw_match,
+	  .fixups = &winbond_nor_partname_fixups },
+};
+
 const struct spi_nor_manufacturer spi_nor_winbond = {
 	.name = "winbond",
 	.parts = winbond_nor_parts,
 	.nparts = ARRAY_SIZE(winbond_nor_parts),
-	.fixups = &winbond_nor_fixups,
+	.fixups = winbond_fixups,
+	.nfixups = ARRAY_SIZE(winbond_fixups),
 };
diff --git a/drivers/mtd/spi-nor/xmc.c b/drivers/mtd/spi-nor/xmc.c
index d5a06054b0dd8e..d3b52eb93e2c07 100644
--- a/drivers/mtd/spi-nor/xmc.c
+++ b/drivers/mtd/spi-nor/xmc.c
@@ -19,6 +19,7 @@ static const struct flash_info xmc_nor_parts[] = {
 		.name = "XM25QH128A",
 		.size = SZ_16M,
 		.no_sfdp_flags = SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ,
+		.flags = SPI_NOR_HAS_LOCK | SPI_NOR_4BIT_BP | SPI_NOR_HAS_TB,
 	},
 };
 
diff --git a/drivers/mux/Kconfig b/drivers/mux/Kconfig
index 6d17dfa25dad62..eb34457beaab31 100644
--- a/drivers/mux/Kconfig
+++ b/drivers/mux/Kconfig
@@ -70,6 +70,19 @@ config MUX_MMIO
 	  To compile the driver as a module, choose M here: the module will
 	  be called mux-mmio.
 
+config MUX_RZV2H_VBENCTL
+	tristate "Renesas RZ/V2H VBENCTL VBUS_SEL mux driver"
+	depends on RESET_RZV2H_USB2PHY || COMPILE_TEST
+	select REGMAP
+	select AUXILIARY_BUS
+	default RESET_RZV2H_USB2PHY
+	help
+	  Support for the VBENCTL VBUS_SEL mux implemented on Renesas
+	  RZ/V2H SoCs.
+
+	  To compile the driver as a module, choose M here: the module will
+	  be called mux-rzv2h-vbenctl.
+
 endmenu
 
 endif # MULTIPLEXER
diff --git a/drivers/mux/Makefile b/drivers/mux/Makefile
index 6e9fa47daf5663..0854c04613b919 100644
--- a/drivers/mux/Makefile
+++ b/drivers/mux/Makefile
@@ -8,9 +8,11 @@ mux-adg792a-objs		:= adg792a.o
 mux-adgs1408-objs		:= adgs1408.o
 mux-gpio-objs			:= gpio.o
 mux-mmio-objs			:= mmio.o
+mux-rzv2h-vbenctl-objs		:= rzv2h-vbenctl.o
 
 obj-$(CONFIG_MULTIPLEXER)	+= mux-core.o
 obj-$(CONFIG_MUX_ADG792A)	+= mux-adg792a.o
 obj-$(CONFIG_MUX_ADGS1408)	+= mux-adgs1408.o
 obj-$(CONFIG_MUX_GPIO)		+= mux-gpio.o
 obj-$(CONFIG_MUX_MMIO)		+= mux-mmio.o
+obj-$(CONFIG_MUX_RZV2H_VBENCTL)	+= mux-rzv2h-vbenctl.o
diff --git a/drivers/mux/rzv2h-vbenctl.c b/drivers/mux/rzv2h-vbenctl.c
new file mode 100644
index 00000000000000..2699458045f363
--- /dev/null
+++ b/drivers/mux/rzv2h-vbenctl.c
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Renesas RZ/V2H(P) VBENCTL VBUS_SEL mux driver
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/mux/driver.h>
+#include <linux/regmap.h>
+
+#define RZV2H_VBENCTL		0xf0c
+
+struct mux_rzv2h_vbenctl_priv {
+	struct regmap_field *field;
+};
+
+static int mux_rzv2h_vbenctl_set(struct mux_control *mux, int state)
+{
+	struct mux_rzv2h_vbenctl_priv *priv = mux_chip_priv(mux->chip);
+
+	return regmap_field_write(priv->field, state);
+}
+
+static const struct mux_control_ops mux_rzv2h_vbenctl_ops = {
+	.set = mux_rzv2h_vbenctl_set,
+};
+
+static int mux_rzv2h_vbenctl_probe(struct auxiliary_device *adev,
+				   const struct auxiliary_device_id *id)
+{
+	const struct reg_field reg_field = REG_FIELD(RZV2H_VBENCTL, 0, 0);
+	struct mux_rzv2h_vbenctl_priv *priv;
+	struct device *dev = &adev->dev;
+	struct mux_chip *mux_chip;
+	struct regmap *regmap;
+	int ret;
+
+	regmap = dev_get_regmap(dev->parent, NULL);
+	if (!regmap)
+		return -ENODEV;
+
+	mux_chip = devm_mux_chip_alloc(dev, 1, sizeof(*priv));
+	if (IS_ERR(mux_chip))
+		return PTR_ERR(mux_chip);
+
+	priv = mux_chip_priv(mux_chip);
+
+	priv->field = devm_regmap_field_alloc(dev, regmap, reg_field);
+	if (IS_ERR(priv->field))
+		return PTR_ERR(priv->field);
+
+	mux_chip->ops = &mux_rzv2h_vbenctl_ops;
+	mux_chip->mux[0].states = 2;
+	mux_chip->mux[0].idle_state = MUX_IDLE_AS_IS;
+
+	ret = devm_mux_chip_register(dev, mux_chip);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to register mux chip\n");
+
+	return 0;
+}
+
+static const struct auxiliary_device_id mux_rzv2h_vbenctl_ids[] = {
+	{ .name = "rzv2h_usb2phy_reset.vbenctl" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(auxiliary, mux_rzv2h_vbenctl_ids);
+
+static struct auxiliary_driver mux_rzv2h_vbenctl_driver = {
+	.name		= "vbenctl",
+	.probe		= mux_rzv2h_vbenctl_probe,
+	.id_table	= mux_rzv2h_vbenctl_ids,
+};
+module_auxiliary_driver(mux_rzv2h_vbenctl_driver);
+
+MODULE_DESCRIPTION("RZ/V2H VBENCTL VBUS_SEL mux driver");
+MODULE_AUTHOR("Tommaso Merciai <tommaso.merciai.xr@bp.renesas.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c
index 43ac8e28e41828..a5a0f19a22ac45 100644
--- a/drivers/net/bonding/bond_alb.c
+++ b/drivers/net/bonding/bond_alb.c
@@ -6,6 +6,7 @@
 #include <linux/skbuff.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
+#include <linux/ethtool.h>
 #include <linux/pkt_sched.h>
 #include <linux/spinlock.h>
 #include <linux/slab.h>
@@ -74,8 +75,8 @@ static inline u8 _simple_hash(const u8 *hash_start, int hash_size)
 static inline void tlb_init_table_entry(struct tlb_client_info *entry, int save_load)
 {
 	if (save_load) {
-		entry->load_history = 1 + entry->tx_bytes /
-				      BOND_TLB_REBALANCE_INTERVAL;
+		entry->load_history = 1 + div_u64(entry->tx_bytes,
+				      BOND_TLB_REBALANCE_INTERVAL);
 		entry->tx_bytes = 0;
 	}
 
@@ -133,6 +134,10 @@ static int tlb_initialize(struct bonding *bond)
 	if (!new_hashtbl)
 		return -ENOMEM;
 
+	bond_info->unbalanced_load = netdev_alloc_pcpu_stats(struct unbalanced_load_stats);
+	if (!bond_info->unbalanced_load)
+		goto out;
+
 	spin_lock_bh(&bond->mode_lock);
 
 	bond_info->tx_hashtbl = new_hashtbl;
@@ -143,6 +148,10 @@ static int tlb_initialize(struct bonding *bond)
 	spin_unlock_bh(&bond->mode_lock);
 
 	return 0;
+
+out:
+	kfree(new_hashtbl);
+	return -ENOMEM;
 }
 
 /* Must be called only after all slaves have been released */
@@ -154,14 +163,18 @@ static void tlb_deinitialize(struct bonding *bond)
 
 	kfree(bond_info->tx_hashtbl);
 	bond_info->tx_hashtbl = NULL;
+	free_percpu(bond_info->unbalanced_load);
+	bond_info->prev_total_unbalanced = 0;
 
 	spin_unlock_bh(&bond->mode_lock);
 }
 
 static long long compute_gap(struct slave *slave)
 {
-	return (s64) (slave->speed << 20) - /* Convert to Megabit per sec */
-	       (s64) (SLAVE_TLB_INFO(slave).load << 3); /* Bytes to bits */
+	u32 raw_speed = READ_ONCE(slave->speed);
+
+	return ((s64)raw_speed << 20) - /* Convert to bits per sec */
+	       ((s64)SLAVE_TLB_INFO(slave).load << 3); /* Bytes to bits */
 }
 
 static struct slave *tlb_get_least_loaded_slave(struct bonding *bond)
@@ -1350,8 +1363,14 @@ static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond,
 	if (!tx_slave) {
 		/* unbalanced or unassigned, send through primary */
 		tx_slave = rcu_dereference(bond->curr_active_slave);
-		if (bond->params.tlb_dynamic_lb)
-			bond_info->unbalanced_load += skb->len;
+		if (bond->params.tlb_dynamic_lb) {
+			struct unbalanced_load_stats *pcpu_load;
+
+			pcpu_load = this_cpu_ptr(bond_info->unbalanced_load);
+			u64_stats_update_begin(&pcpu_load->syncp);
+			u64_stats_add(&pcpu_load->tx_bytes, skb->len);
+			u64_stats_update_end(&pcpu_load->syncp);
+		}
 	}
 
 	if (tx_slave && bond_slave_can_tx(tx_slave)) {
@@ -1535,6 +1554,29 @@ netdev_tx_t bond_alb_xmit(struct sk_buff *skb, struct net_device *bond_dev)
 	return bond_do_alb_xmit(skb, bond, tx_slave);
 }
 
+static u64 reset_unbalanced_load(struct alb_bond_info *bond_info)
+{
+	u64 delta, tx_bytes, total_bytes = 0;
+	struct unbalanced_load_stats *p;
+	unsigned int start;
+	int i;
+
+	for_each_possible_cpu(i) {
+		p = per_cpu_ptr(bond_info->unbalanced_load, i);
+		do {
+			start = u64_stats_fetch_begin(&p->syncp);
+			tx_bytes = u64_stats_read(&p->tx_bytes);
+		} while (u64_stats_fetch_retry(&p->syncp, start));
+
+		total_bytes += tx_bytes;
+	}
+
+	delta = total_bytes - bond_info->prev_total_unbalanced;
+	bond_info->prev_total_unbalanced = total_bytes;
+
+	return div_u64(delta, BOND_TLB_REBALANCE_INTERVAL);
+}
+
 void bond_alb_monitor(struct work_struct *work)
 {
 	struct bonding *bond = container_of(work, struct bonding,
@@ -1577,10 +1619,11 @@ void bond_alb_monitor(struct work_struct *work)
 		bond_for_each_slave_rcu(bond, slave, iter) {
 			tlb_clear_slave(bond, slave, 1);
 			if (slave == rcu_access_pointer(bond->curr_active_slave)) {
-				SLAVE_TLB_INFO(slave).load =
-					bond_info->unbalanced_load /
-						BOND_TLB_REBALANCE_INTERVAL;
-				bond_info->unbalanced_load = 0;
+				u64 new_load = reset_unbalanced_load(bond_info);
+
+				spin_lock_bh(&bond->mode_lock);
+				SLAVE_TLB_INFO(slave).load = new_load;
+				spin_unlock_bh(&bond->mode_lock);
 			}
 		}
 		atomic_set(&bond_info->tx_rebalance_counter, 0);
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index a9bff7663eecad..de2489c3d9bf28 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -490,7 +490,7 @@ static int bond_ipsec_add_sa(struct net_device *bond_dev,
 	    !real_dev->xfrmdev_ops->xdo_dev_state_add ||
 	    netif_is_bond_master(real_dev)) {
 		NL_SET_ERR_MSG_MOD(extack, "Slave does not support ipsec offload");
-		err = -EINVAL;
+		err = -EOPNOTSUPP;
 		goto out;
 	}
 
diff --git a/drivers/net/can/bxcan.c b/drivers/net/can/bxcan.c
index baf494d20bef4a..4bddd015775bc3 100644
--- a/drivers/net/can/bxcan.c
+++ b/drivers/net/can/bxcan.c
@@ -1080,7 +1080,7 @@ static SIMPLE_DEV_PM_OPS(bxcan_pm_ops, bxcan_suspend, bxcan_resume);
 
 static const struct of_device_id bxcan_of_match[] = {
 	{.compatible = "st,stm32f4-bxcan"},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, bxcan_of_match);
 
diff --git a/drivers/net/can/c_can/c_can_platform.c b/drivers/net/can/c_can/c_can_platform.c
index 19c86b94a40e4f..75607b1afd2e99 100644
--- a/drivers/net/can/c_can/c_can_platform.c
+++ b/drivers/net/can/c_can/c_can_platform.c
@@ -239,7 +239,7 @@ static const struct platform_device_id c_can_id_table[] = {
 		.name = "d_can",
 		.driver_data = (kernel_ulong_t)&d_can_drvdata,
 	},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(platform, c_can_id_table);
 
@@ -249,7 +249,7 @@ static const struct of_device_id c_can_of_table[] = {
 	{ .compatible = "ti,dra7-d_can", .data = &dra7_dcan_drvdata },
 	{ .compatible = "ti,am3352-d_can", .data = &am3352_dcan_drvdata },
 	{ .compatible = "ti,am4372-d_can", .data = &am3352_dcan_drvdata },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, c_can_of_table);
 
diff --git a/drivers/net/can/flexcan/flexcan-core.c b/drivers/net/can/flexcan/flexcan-core.c
index f5d22c61503faa..86b40abe9e767e 100644
--- a/drivers/net/can/flexcan/flexcan-core.c
+++ b/drivers/net/can/flexcan/flexcan-core.c
@@ -2070,7 +2070,7 @@ static const struct of_device_id flexcan_of_match[] = {
 	{ .compatible = "fsl,ls1021ar2-flexcan", .data = &fsl_ls1021a_r2_devtype_data, },
 	{ .compatible = "fsl,lx2160ar1-flexcan", .data = &fsl_lx2160a_r1_devtype_data, },
 	{ .compatible = "nxp,s32g2-flexcan", .data = &nxp_s32g2_devtype_data, },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, flexcan_of_match);
 
diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c
index 0f83335e4d075e..9eecfe5a87bd12 100644
--- a/drivers/net/can/ifi_canfd/ifi_canfd.c
+++ b/drivers/net/can/ifi_canfd/ifi_canfd.c
@@ -1044,7 +1044,7 @@ static void ifi_canfd_plat_remove(struct platform_device *pdev)
 
 static const struct of_device_id ifi_canfd_of_table[] = {
 	{ .compatible = "ifi,canfd-1.0", .data = NULL },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, ifi_canfd_of_table);
 
diff --git a/drivers/net/can/m_can/m_can_platform.c b/drivers/net/can/m_can/m_can_platform.c
index 56da411878af01..51dd41dbf2c9a9 100644
--- a/drivers/net/can/m_can/m_can_platform.c
+++ b/drivers/net/can/m_can/m_can_platform.c
@@ -220,7 +220,7 @@ static const struct dev_pm_ops m_can_pmops = {
 
 static const struct of_device_id m_can_of_table[] = {
 	{ .compatible = "bosch,m_can", .data = NULL },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, m_can_of_table);
 
diff --git a/drivers/net/can/sja1000/f81601.c b/drivers/net/can/sja1000/f81601.c
index 8f25e95814eff8..18cf170db4575e 100644
--- a/drivers/net/can/sja1000/f81601.c
+++ b/drivers/net/can/sja1000/f81601.c
@@ -40,7 +40,7 @@ struct f81601_pci_card {
 
 static const struct pci_device_id f81601_pci_tbl[] = {
 	{ PCI_DEVICE(0x1c29, 0x1703) },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 
 MODULE_DEVICE_TABLE(pci, f81601_pci_tbl);
diff --git a/drivers/net/can/sja1000/sja1000_platform.c b/drivers/net/can/sja1000/sja1000_platform.c
index 2d555f854008b3..da4e1c820b6ec4 100644
--- a/drivers/net/can/sja1000/sja1000_platform.c
+++ b/drivers/net/can/sja1000/sja1000_platform.c
@@ -207,7 +207,7 @@ static const struct of_device_id sp_of_table[] = {
 	{ .compatible = "nxp,sja1000", .data = NULL, },
 	{ .compatible = "renesas,rzn1-sja1000", .data = &renesas_data, },
 	{ .compatible = "technologic,sja1000", .data = &technologic_data, },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sp_of_table);
 
diff --git a/drivers/net/can/slcan/slcan-core.c b/drivers/net/can/slcan/slcan-core.c
index 7439849d5c840e..c9414faafe664b 100644
--- a/drivers/net/can/slcan/slcan-core.c
+++ b/drivers/net/can/slcan/slcan-core.c
@@ -131,10 +131,7 @@ int slcan_enable_err_rst_on_open(struct net_device *ndev, bool on)
 	if (netif_running(ndev))
 		return -EBUSY;
 
-	if (on)
-		set_bit(CF_ERR_RST, &sl->cmd_flags);
-	else
-		clear_bit(CF_ERR_RST, &sl->cmd_flags);
+	assign_bit(CF_ERR_RST, &sl->cmd_flags, on);
 
 	return 0;
 }
diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
index 676fb7dffe14fa..fe8cd5338fda53 100644
--- a/drivers/net/dsa/Kconfig
+++ b/drivers/net/dsa/Kconfig
@@ -82,6 +82,8 @@ config NET_DSA_MV88E6060
 
 source "drivers/net/dsa/microchip/Kconfig"
 
+source "drivers/net/dsa/motorcomm/Kconfig"
+
 source "drivers/net/dsa/mv88e6xxx/Kconfig"
 
 source "drivers/net/dsa/mxl862xx/Kconfig"
@@ -110,10 +112,13 @@ config NET_DSA_RZN1_A5PSW
 config NET_DSA_KS8995
 	tristate "Micrel KS8995 family 5-ports 10/100 Ethernet switches"
 	depends on SPI
-	select NET_DSA_TAG_NONE
+	select NET_DSA_MICROCHIP_KSZ_COMMON
+	select NET_DSA_MICROCHIP_KSZ_SPI
 	help
-	  This driver supports the Micrel KS8995 family of 10/100 Mbit ethernet
-	  switches, managed over SPI.
+	  This is a transitional option for users who had NET_DSA_KS8995
+	  enabled. It automatically enables the new NET_DSA_MICROCHIP_KSZ_SPI
+	  driver which supersedes it. This option will be removed in a future
+	  kernel release.
 
 config NET_DSA_SMSC_LAN9303
 	tristate
@@ -168,12 +173,4 @@ config NET_DSA_VITESSE_VSC73XX_PLATFORM
 	  This enables support for the Vitesse VSC7385, VSC7388, VSC7395
 	  and VSC7398 SparX integrated ethernet switches, connected over
 	  a CPU-attached address bus and work in memory-mapped I/O mode.
-
-config NET_DSA_YT921X
-	tristate "Motorcomm YT9215 ethernet switch chip support"
-	select NET_DSA_TAG_YT921X
-	select NET_IEEE8021Q_HELPERS if DCB
-	help
-	  This enables support for the Motorcomm YT9215 ethernet switch
-	  chip.
 endmenu
diff --git a/drivers/net/dsa/Makefile b/drivers/net/dsa/Makefile
index 6ceb78a755d777..7e637d56b35c80 100644
--- a/drivers/net/dsa/Makefile
+++ b/drivers/net/dsa/Makefile
@@ -2,7 +2,6 @@
 obj-$(CONFIG_NET_DSA_BCM_SF2)	+= bcm-sf2.o
 bcm-sf2-objs			:= bcm_sf2.o bcm_sf2_cfp.o
 obj-$(CONFIG_NET_DSA_LOOP)	+= dsa_loop.o
-obj-$(CONFIG_NET_DSA_KS8995) 	+= ks8995.o
 obj-$(CONFIG_NET_DSA_MT7530)	+= mt7530.o
 obj-$(CONFIG_NET_DSA_MT7530_MDIO) += mt7530-mdio.o
 obj-$(CONFIG_NET_DSA_MT7530_MMIO) += mt7530-mmio.o
@@ -15,11 +14,11 @@ obj-$(CONFIG_NET_DSA_SMSC_LAN9303_MDIO) += lan9303_mdio.o
 obj-$(CONFIG_NET_DSA_VITESSE_VSC73XX) += vitesse-vsc73xx-core.o
 obj-$(CONFIG_NET_DSA_VITESSE_VSC73XX_PLATFORM) += vitesse-vsc73xx-platform.o
 obj-$(CONFIG_NET_DSA_VITESSE_VSC73XX_SPI) += vitesse-vsc73xx-spi.o
-obj-$(CONFIG_NET_DSA_YT921X) += yt921x.o
 obj-y				+= b53/
 obj-y				+= hirschmann/
 obj-y				+= lantiq/
 obj-y				+= microchip/
+obj-y				+= motorcomm/
 obj-y				+= mv88e6xxx/
 obj-y				+= mxl862xx/
 obj-y				+= netc/
diff --git a/drivers/net/dsa/b53/b53_mdio.c b/drivers/net/dsa/b53/b53_mdio.c
index 43a3b37b731bf8..fc1ce57a80a365 100644
--- a/drivers/net/dsa/b53/b53_mdio.c
+++ b/drivers/net/dsa/b53/b53_mdio.c
@@ -384,7 +384,7 @@ static const struct of_device_id b53_of_match[] = {
 	{ .compatible = "brcm,bcm5395" },
 	{ .compatible = "brcm,bcm5397" },
 	{ .compatible = "brcm,bcm5398" },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, b53_of_match);
 
diff --git a/drivers/net/dsa/b53/b53_srab.c b/drivers/net/dsa/b53/b53_srab.c
index b9939bbd2cd517..83a03fca142a02 100644
--- a/drivers/net/dsa/b53/b53_srab.c
+++ b/drivers/net/dsa/b53/b53_srab.c
@@ -515,7 +515,7 @@ static const struct of_device_id b53_srab_of_match[] = {
 	{ .compatible = "brcm,cygnus-srab", .data = (void *)BCM583XX_DEVICE_ID },
 	{ .compatible = "brcm,nsp-srab", .data = (void *)BCM58XX_DEVICE_ID },
 	{ .compatible = "brcm,omega-srab", .data = (void *)BCM583XX_DEVICE_ID },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, b53_srab_of_match);
 
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index de3efa3ce9a75f..9e5713015187b7 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -1358,7 +1358,7 @@ static const struct of_device_id bcm_sf2_of_match[] = {
 	{ .compatible = "brcm,bcm7278-switch-v4.8",
 	  .data = &bcm_sf2_7278_data
 	},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, bcm_sf2_of_match);
 
diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c
index 2c6cabf611436a..aec50414986e7e 100644
--- a/drivers/net/dsa/hirschmann/hellcreek.c
+++ b/drivers/net/dsa/hirschmann/hellcreek.c
@@ -2093,7 +2093,7 @@ static const struct of_device_id hellcreek_of_match[] = {
 		.compatible = "hirschmann,hellcreek-de1soc-r1",
 		.data	    = &de1soc_r1_pdata,
 	},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, hellcreek_of_match);
 
diff --git a/drivers/net/dsa/ks8995.c b/drivers/net/dsa/ks8995.c
deleted file mode 100644
index 77d8b842693c16..00000000000000
--- a/drivers/net/dsa/ks8995.c
+++ /dev/null
@@ -1,857 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * SPI driver for Micrel/Kendin KS8995M and KSZ8864RMN ethernet switches
- *
- * Copyright (C) 2008 Gabor Juhos <juhosg at openwrt.org>
- * Copyright (C) 2025 Linus Walleij <linus.walleij@linaro.org>
- *
- * This file was based on: drivers/spi/at25.c
- *     Copyright (C) 2006 David Brownell
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/bits.h>
-#include <linux/if_bridge.h>
-#include <linux/if_vlan.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/gpio/consumer.h>
-#include <linux/of.h>
-#include <linux/spi/spi.h>
-#include <net/dsa.h>
-
-#define DRV_VERSION		"0.1.1"
-#define DRV_DESC		"Micrel KS8995 Ethernet switch SPI driver"
-
-/* ------------------------------------------------------------------------ */
-
-#define KS8995_REG_ID0		0x00    /* Chip ID0 */
-#define KS8995_REG_ID1		0x01    /* Chip ID1 */
-
-#define KS8995_REG_GC0		0x02    /* Global Control 0 */
-
-#define KS8995_GC0_P5_PHY	BIT(3)	/* Port 5 PHY enabled */
-
-#define KS8995_REG_GC1		0x03    /* Global Control 1 */
-#define KS8995_REG_GC2		0x04    /* Global Control 2 */
-
-#define KS8995_GC2_HUGE		BIT(2)	/* Huge packet support */
-#define KS8995_GC2_LEGAL	BIT(1)	/* Legal size override */
-
-#define KS8995_REG_GC3		0x05    /* Global Control 3 */
-#define KS8995_REG_GC4		0x06    /* Global Control 4 */
-
-#define KS8995_GC4_10BT		BIT(4)	/* Force switch to 10Mbit */
-#define KS8995_GC4_MII_FLOW	BIT(5)	/* MII full-duplex flow control enable */
-#define KS8995_GC4_MII_HD	BIT(6)	/* MII half-duplex mode enable */
-
-#define KS8995_REG_GC5		0x07    /* Global Control 5 */
-#define KS8995_REG_GC6		0x08    /* Global Control 6 */
-#define KS8995_REG_GC7		0x09    /* Global Control 7 */
-#define KS8995_REG_GC8		0x0a    /* Global Control 8 */
-#define KS8995_REG_GC9		0x0b    /* Global Control 9 */
-
-#define KS8995_GC9_SPECIAL	BIT(0)	/* Special tagging mode (DSA) */
-
-/* In DSA the ports 1-4 are numbered 0-3 and the CPU port is port 4 */
-#define KS8995_REG_PC(p, r)	(0x10 + (0x10 * (p)) + (r)) /* Port Control */
-#define KS8995_REG_PS(p, r)	(0x1e + (0x10 * (p)) + (r)) /* Port Status */
-
-#define KS8995_REG_PC0		0x00    /* Port Control 0 */
-#define KS8995_REG_PC1		0x01    /* Port Control 1 */
-#define KS8995_REG_PC2		0x02    /* Port Control 2 */
-#define KS8995_REG_PC3		0x03    /* Port Control 3 */
-#define KS8995_REG_PC4		0x04    /* Port Control 4 */
-#define KS8995_REG_PC5		0x05    /* Port Control 5 */
-#define KS8995_REG_PC6		0x06    /* Port Control 6 */
-#define KS8995_REG_PC7		0x07    /* Port Control 7 */
-#define KS8995_REG_PC8		0x08    /* Port Control 8 */
-#define KS8995_REG_PC9		0x09    /* Port Control 9 */
-#define KS8995_REG_PC10		0x0a    /* Port Control 10 */
-#define KS8995_REG_PC11		0x0b    /* Port Control 11 */
-#define KS8995_REG_PC12		0x0c    /* Port Control 12 */
-#define KS8995_REG_PC13		0x0d    /* Port Control 13 */
-
-#define KS8995_PC0_TAG_INS	BIT(2)	/* Enable tag insertion on port */
-#define KS8995_PC0_TAG_REM	BIT(1)	/* Enable tag removal on port */
-#define KS8995_PC0_PRIO_EN	BIT(0)	/* Enable priority handling */
-
-#define KS8995_PC2_TXEN		BIT(2)	/* Enable TX on port */
-#define KS8995_PC2_RXEN		BIT(1)	/* Enable RX on port */
-#define KS8995_PC2_LEARN_DIS	BIT(0)	/* Disable learning on port */
-
-#define KS8995_PC13_TXDIS	BIT(6)	/* Disable transmitter */
-#define KS8995_PC13_PWDN	BIT(3)	/* Power down */
-
-#define KS8995_REG_TPC0		0x60    /* TOS Priority Control 0 */
-#define KS8995_REG_TPC1		0x61    /* TOS Priority Control 1 */
-#define KS8995_REG_TPC2		0x62    /* TOS Priority Control 2 */
-#define KS8995_REG_TPC3		0x63    /* TOS Priority Control 3 */
-#define KS8995_REG_TPC4		0x64    /* TOS Priority Control 4 */
-#define KS8995_REG_TPC5		0x65    /* TOS Priority Control 5 */
-#define KS8995_REG_TPC6		0x66    /* TOS Priority Control 6 */
-#define KS8995_REG_TPC7		0x67    /* TOS Priority Control 7 */
-
-#define KS8995_REG_MAC0		0x68    /* MAC address 0 */
-#define KS8995_REG_MAC1		0x69    /* MAC address 1 */
-#define KS8995_REG_MAC2		0x6a    /* MAC address 2 */
-#define KS8995_REG_MAC3		0x6b    /* MAC address 3 */
-#define KS8995_REG_MAC4		0x6c    /* MAC address 4 */
-#define KS8995_REG_MAC5		0x6d    /* MAC address 5 */
-
-#define KS8995_REG_IAC0		0x6e    /* Indirect Access Control 0 */
-#define KS8995_REG_IAC1		0x6f    /* Indirect Access Control 0 */
-#define KS8995_REG_IAD7		0x70    /* Indirect Access Data 7 */
-#define KS8995_REG_IAD6		0x71    /* Indirect Access Data 6 */
-#define KS8995_REG_IAD5		0x72    /* Indirect Access Data 5 */
-#define KS8995_REG_IAD4		0x73    /* Indirect Access Data 4 */
-#define KS8995_REG_IAD3		0x74    /* Indirect Access Data 3 */
-#define KS8995_REG_IAD2		0x75    /* Indirect Access Data 2 */
-#define KS8995_REG_IAD1		0x76    /* Indirect Access Data 1 */
-#define KS8995_REG_IAD0		0x77    /* Indirect Access Data 0 */
-
-#define KSZ8864_REG_ID1		0xfe	/* Chip ID in bit 7 */
-
-#define KS8995_REGS_SIZE	0x80
-#define KSZ8864_REGS_SIZE	0x100
-#define KSZ8795_REGS_SIZE	0x100
-
-#define ID1_CHIPID_M		0xf
-#define ID1_CHIPID_S		4
-#define ID1_REVISION_M		0x7
-#define ID1_REVISION_S		1
-#define ID1_START_SW		1	/* start the switch */
-
-#define FAMILY_KS8995		0x95
-#define FAMILY_KSZ8795		0x87
-#define CHIPID_M		0
-#define KS8995_CHIP_ID		0x00
-#define KSZ8864_CHIP_ID		0x01
-#define KSZ8795_CHIP_ID		0x09
-
-#define KS8995_CMD_WRITE	0x02U
-#define KS8995_CMD_READ		0x03U
-
-#define KS8995_CPU_PORT		4
-#define KS8995_NUM_PORTS	5 /* 5 ports including the CPU port */
-#define KS8995_RESET_DELAY	10 /* usec */
-
-enum ks8995_chip_variant {
-	ks8995,
-	ksz8864,
-	ksz8795,
-	max_variant
-};
-
-struct ks8995_chip_params {
-	char *name;
-	int family_id;
-	int chip_id;
-	int regs_size;
-	int addr_width;
-	int addr_shift;
-};
-
-static const struct ks8995_chip_params ks8995_chip[] = {
-	[ks8995] = {
-		.name = "KS8995MA",
-		.family_id = FAMILY_KS8995,
-		.chip_id = KS8995_CHIP_ID,
-		.regs_size = KS8995_REGS_SIZE,
-		.addr_width = 8,
-		.addr_shift = 0,
-	},
-	[ksz8864] = {
-		.name = "KSZ8864RMN",
-		.family_id = FAMILY_KS8995,
-		.chip_id = KSZ8864_CHIP_ID,
-		.regs_size = KSZ8864_REGS_SIZE,
-		.addr_width = 8,
-		.addr_shift = 0,
-	},
-	[ksz8795] = {
-		.name = "KSZ8795CLX",
-		.family_id = FAMILY_KSZ8795,
-		.chip_id = KSZ8795_CHIP_ID,
-		.regs_size = KSZ8795_REGS_SIZE,
-		.addr_width = 12,
-		.addr_shift = 1,
-	},
-};
-
-struct ks8995_switch {
-	struct spi_device	*spi;
-	struct device		*dev;
-	struct dsa_switch	*ds;
-	struct mutex		lock;
-	struct gpio_desc	*reset_gpio;
-	struct bin_attribute	regs_attr;
-	const struct ks8995_chip_params	*chip;
-	int			revision_id;
-	unsigned int max_mtu[KS8995_NUM_PORTS];
-};
-
-static const struct spi_device_id ks8995_id[] = {
-	{"ks8995", ks8995},
-	{"ksz8864", ksz8864},
-	{"ksz8795", ksz8795},
-	{ }
-};
-MODULE_DEVICE_TABLE(spi, ks8995_id);
-
-static const struct of_device_id ks8995_spi_of_match[] = {
-	{ .compatible = "micrel,ks8995" },
-	{ .compatible = "micrel,ksz8864" },
-	{ .compatible = "micrel,ksz8795" },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, ks8995_spi_of_match);
-
-static inline u8 get_chip_id(u8 val)
-{
-	return (val >> ID1_CHIPID_S) & ID1_CHIPID_M;
-}
-
-static inline u8 get_chip_rev(u8 val)
-{
-	return (val >> ID1_REVISION_S) & ID1_REVISION_M;
-}
-
-/* create_spi_cmd - create a chip specific SPI command header
- * @ks: pointer to switch instance
- * @cmd: SPI command for switch
- * @address: register address for command
- *
- * Different chip families use different bit pattern to address the switches
- * registers:
- *
- * KS8995: 8bit command + 8bit address
- * KSZ8795: 3bit command + 12bit address + 1bit TR (?)
- */
-static inline __be16 create_spi_cmd(struct ks8995_switch *ks, int cmd,
-				    unsigned address)
-{
-	u16 result = cmd;
-
-	/* make room for address (incl. address shift) */
-	result <<= ks->chip->addr_width + ks->chip->addr_shift;
-	/* add address */
-	result |= address << ks->chip->addr_shift;
-	/* SPI protocol needs big endian */
-	return cpu_to_be16(result);
-}
-/* ------------------------------------------------------------------------ */
-static int ks8995_read(struct ks8995_switch *ks, char *buf,
-		 unsigned offset, size_t count)
-{
-	__be16 cmd;
-	struct spi_transfer t[2];
-	struct spi_message m;
-	int err;
-
-	cmd = create_spi_cmd(ks, KS8995_CMD_READ, offset);
-	spi_message_init(&m);
-
-	memset(&t, 0, sizeof(t));
-
-	t[0].tx_buf = &cmd;
-	t[0].len = sizeof(cmd);
-	spi_message_add_tail(&t[0], &m);
-
-	t[1].rx_buf = buf;
-	t[1].len = count;
-	spi_message_add_tail(&t[1], &m);
-
-	mutex_lock(&ks->lock);
-	err = spi_sync(ks->spi, &m);
-	mutex_unlock(&ks->lock);
-
-	return err ? err : count;
-}
-
-static int ks8995_write(struct ks8995_switch *ks, char *buf,
-		 unsigned offset, size_t count)
-{
-	__be16 cmd;
-	struct spi_transfer t[2];
-	struct spi_message m;
-	int err;
-
-	cmd = create_spi_cmd(ks, KS8995_CMD_WRITE, offset);
-	spi_message_init(&m);
-
-	memset(&t, 0, sizeof(t));
-
-	t[0].tx_buf = &cmd;
-	t[0].len = sizeof(cmd);
-	spi_message_add_tail(&t[0], &m);
-
-	t[1].tx_buf = buf;
-	t[1].len = count;
-	spi_message_add_tail(&t[1], &m);
-
-	mutex_lock(&ks->lock);
-	err = spi_sync(ks->spi, &m);
-	mutex_unlock(&ks->lock);
-
-	return err ? err : count;
-}
-
-static inline int ks8995_read_reg(struct ks8995_switch *ks, u8 addr, u8 *buf)
-{
-	return ks8995_read(ks, buf, addr, 1) != 1;
-}
-
-static inline int ks8995_write_reg(struct ks8995_switch *ks, u8 addr, u8 val)
-{
-	char buf = val;
-
-	return ks8995_write(ks, &buf, addr, 1) != 1;
-}
-
-/* ------------------------------------------------------------------------ */
-
-static int ks8995_stop(struct ks8995_switch *ks)
-{
-	return ks8995_write_reg(ks, KS8995_REG_ID1, 0);
-}
-
-static int ks8995_start(struct ks8995_switch *ks)
-{
-	return ks8995_write_reg(ks, KS8995_REG_ID1, 1);
-}
-
-static int ks8995_reset(struct ks8995_switch *ks)
-{
-	int err;
-
-	err = ks8995_stop(ks);
-	if (err)
-		return err;
-
-	udelay(KS8995_RESET_DELAY);
-
-	return ks8995_start(ks);
-}
-
-/* ks8995_get_revision - get chip revision
- * @ks: pointer to switch instance
- *
- * Verify chip family and id and get chip revision.
- */
-static int ks8995_get_revision(struct ks8995_switch *ks)
-{
-	int err;
-	u8 id0, id1, ksz8864_id;
-
-	/* read family id */
-	err = ks8995_read_reg(ks, KS8995_REG_ID0, &id0);
-	if (err) {
-		err = -EIO;
-		goto err_out;
-	}
-
-	/* verify family id */
-	if (id0 != ks->chip->family_id) {
-		dev_err(&ks->spi->dev, "chip family id mismatch: expected 0x%02x but 0x%02x read\n",
-			ks->chip->family_id, id0);
-		err = -ENODEV;
-		goto err_out;
-	}
-
-	switch (ks->chip->family_id) {
-	case FAMILY_KS8995:
-		/* try reading chip id at CHIP ID1 */
-		err = ks8995_read_reg(ks, KS8995_REG_ID1, &id1);
-		if (err) {
-			err = -EIO;
-			goto err_out;
-		}
-
-		/* verify chip id */
-		if ((get_chip_id(id1) == CHIPID_M) &&
-		    (get_chip_id(id1) == ks->chip->chip_id)) {
-			/* KS8995MA */
-			ks->revision_id = get_chip_rev(id1);
-		} else if (get_chip_id(id1) != CHIPID_M) {
-			/* KSZ8864RMN */
-			err = ks8995_read_reg(ks, KS8995_REG_ID1, &ksz8864_id);
-			if (err) {
-				err = -EIO;
-				goto err_out;
-			}
-
-			if ((ksz8864_id & 0x80) &&
-			    (ks->chip->chip_id == KSZ8864_CHIP_ID)) {
-				ks->revision_id = get_chip_rev(id1);
-			}
-
-		} else {
-			dev_err(&ks->spi->dev, "unsupported chip id for KS8995 family: 0x%02x\n",
-				id1);
-			err = -ENODEV;
-		}
-		break;
-	case FAMILY_KSZ8795:
-		/* try reading chip id at CHIP ID1 */
-		err = ks8995_read_reg(ks, KS8995_REG_ID1, &id1);
-		if (err) {
-			err = -EIO;
-			goto err_out;
-		}
-
-		if (get_chip_id(id1) == ks->chip->chip_id) {
-			ks->revision_id = get_chip_rev(id1);
-		} else {
-			dev_err(&ks->spi->dev, "unsupported chip id for KSZ8795 family: 0x%02x\n",
-				id1);
-			err = -ENODEV;
-		}
-		break;
-	default:
-		dev_err(&ks->spi->dev, "unsupported family id: 0x%02x\n", id0);
-		err = -ENODEV;
-		break;
-	}
-err_out:
-	return err;
-}
-
-static int ks8995_check_config(struct ks8995_switch *ks)
-{
-	int ret;
-	u8 val;
-
-	ret = ks8995_read_reg(ks, KS8995_REG_GC0, &val);
-	if (ret) {
-		dev_err(ks->dev, "failed to read KS8995_REG_GC0\n");
-		return ret;
-	}
-
-	dev_dbg(ks->dev, "port 5 PHY %senabled\n",
-		(val & KS8995_GC0_P5_PHY) ? "" : "not ");
-
-	val |= KS8995_GC0_P5_PHY;
-	ret = ks8995_write_reg(ks, KS8995_REG_GC0, val);
-	if (ret)
-		dev_err(ks->dev, "failed to set KS8995_REG_GC0\n");
-
-	dev_dbg(ks->dev, "set KS8995_REG_GC0 to 0x%02x\n", val);
-
-	return 0;
-}
-
-static void
-ks8995_mac_config(struct phylink_config *config, unsigned int mode,
-		  const struct phylink_link_state *state)
-{
-}
-
-static void
-ks8995_mac_link_up(struct phylink_config *config, struct phy_device *phydev,
-		   unsigned int mode, phy_interface_t interface,
-		   int speed, int duplex, bool tx_pause, bool rx_pause)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct ks8995_switch *ks = dp->ds->priv;
-	int port = dp->index;
-	int ret;
-	u8 val;
-
-	/* Allow forcing the mode on the fixed CPU port, no autonegotiation.
-	 * We assume autonegotiation works on the PHY-facing ports.
-	 */
-	if (port != KS8995_CPU_PORT)
-		return;
-
-	dev_dbg(ks->dev, "MAC link up on CPU port (%d)\n", port);
-
-	ret = ks8995_read_reg(ks, KS8995_REG_GC4, &val);
-	if (ret) {
-		dev_err(ks->dev, "failed to read KS8995_REG_GC4\n");
-		return;
-	}
-
-	/* Conjure port config */
-	switch (speed) {
-	case SPEED_10:
-		dev_dbg(ks->dev, "set switch MII to 100Mbit mode\n");
-		val |= KS8995_GC4_10BT;
-		break;
-	case SPEED_100:
-	default:
-		dev_dbg(ks->dev, "set switch MII to 100Mbit mode\n");
-		val &= ~KS8995_GC4_10BT;
-		break;
-	}
-
-	if (duplex == DUPLEX_HALF) {
-		dev_dbg(ks->dev, "set switch MII to half duplex\n");
-		val |= KS8995_GC4_MII_HD;
-	} else {
-		dev_dbg(ks->dev, "set switch MII to full duplex\n");
-		val &= ~KS8995_GC4_MII_HD;
-	}
-
-	dev_dbg(ks->dev, "set KS8995_REG_GC4 to %02x\n", val);
-
-	/* Enable the CPU port */
-	ret = ks8995_write_reg(ks, KS8995_REG_GC4, val);
-	if (ret)
-		dev_err(ks->dev, "failed to set KS8995_REG_GC4\n");
-}
-
-static void
-ks8995_mac_link_down(struct phylink_config *config, unsigned int mode,
-		     phy_interface_t interface)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct ks8995_switch *ks = dp->ds->priv;
-	int port = dp->index;
-
-	if (port != KS8995_CPU_PORT)
-		return;
-
-	dev_dbg(ks->dev, "MAC link down on CPU port (%d)\n", port);
-
-	/* Disable the CPU port */
-}
-
-static const struct phylink_mac_ops ks8995_phylink_mac_ops = {
-	.mac_config = ks8995_mac_config,
-	.mac_link_up = ks8995_mac_link_up,
-	.mac_link_down = ks8995_mac_link_down,
-};
-
-static enum
-dsa_tag_protocol ks8995_get_tag_protocol(struct dsa_switch *ds,
-					 int port,
-					 enum dsa_tag_protocol mp)
-{
-	/* This switch actually uses the 6 byte KS8995 protocol */
-	return DSA_TAG_PROTO_NONE;
-}
-
-static int ks8995_setup(struct dsa_switch *ds)
-{
-	return 0;
-}
-
-static int ks8995_port_enable(struct dsa_switch *ds, int port,
-			      struct phy_device *phy)
-{
-	struct ks8995_switch *ks = ds->priv;
-
-	dev_dbg(ks->dev, "enable port %d\n", port);
-
-	return 0;
-}
-
-static void ks8995_port_disable(struct dsa_switch *ds, int port)
-{
-	struct ks8995_switch *ks = ds->priv;
-
-	dev_dbg(ks->dev, "disable port %d\n", port);
-}
-
-static int ks8995_port_pre_bridge_flags(struct dsa_switch *ds, int port,
-					struct switchdev_brport_flags flags,
-					struct netlink_ext_ack *extack)
-{
-	/* We support enabling/disabling learning */
-	if (flags.mask & ~(BR_LEARNING))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int ks8995_port_bridge_flags(struct dsa_switch *ds, int port,
-				    struct switchdev_brport_flags flags,
-				    struct netlink_ext_ack *extack)
-{
-	struct ks8995_switch *ks = ds->priv;
-	int ret;
-	u8 val;
-
-	if (flags.mask & BR_LEARNING) {
-		ret = ks8995_read_reg(ks, KS8995_REG_PC(port, KS8995_REG_PC2), &val);
-		if (ret) {
-			dev_err(ks->dev, "failed to read KS8995_REG_PC2 on port %d\n", port);
-			return ret;
-		}
-
-		if (flags.val & BR_LEARNING)
-			val &= ~KS8995_PC2_LEARN_DIS;
-		else
-			val |= KS8995_PC2_LEARN_DIS;
-
-		ret = ks8995_write_reg(ks, KS8995_REG_PC(port, KS8995_REG_PC2), val);
-		if (ret) {
-			dev_err(ks->dev, "failed to write KS8995_REG_PC2 on port %d\n", port);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-static void ks8995_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
-{
-	struct ks8995_switch *ks = ds->priv;
-	int ret;
-	u8 val;
-
-	ret = ks8995_read_reg(ks, KS8995_REG_PC(port, KS8995_REG_PC2), &val);
-	if (ret) {
-		dev_err(ks->dev, "failed to read KS8995_REG_PC2 on port %d\n", port);
-		return;
-	}
-
-	/* Set the bits for the different STP states in accordance with
-	 * the datasheet, pages 36-37 "Spanning tree support".
-	 */
-	switch (state) {
-	case BR_STATE_DISABLED:
-	case BR_STATE_BLOCKING:
-	case BR_STATE_LISTENING:
-		val &= ~KS8995_PC2_TXEN;
-		val &= ~KS8995_PC2_RXEN;
-		val |= KS8995_PC2_LEARN_DIS;
-		break;
-	case BR_STATE_LEARNING:
-		val &= ~KS8995_PC2_TXEN;
-		val &= ~KS8995_PC2_RXEN;
-		val &= ~KS8995_PC2_LEARN_DIS;
-		break;
-	case BR_STATE_FORWARDING:
-		val |= KS8995_PC2_TXEN;
-		val |= KS8995_PC2_RXEN;
-		val &= ~KS8995_PC2_LEARN_DIS;
-		break;
-	default:
-		dev_err(ks->dev, "unknown bridge state requested\n");
-		return;
-	}
-
-	ret = ks8995_write_reg(ks, KS8995_REG_PC(port, KS8995_REG_PC2), val);
-	if (ret) {
-		dev_err(ks->dev, "failed to write KS8995_REG_PC2 on port %d\n", port);
-		return;
-	}
-
-	dev_dbg(ks->dev, "set KS8995_REG_PC2 for port %d to %02x\n", port, val);
-}
-
-static void ks8995_phylink_get_caps(struct dsa_switch *dsa, int port,
-				    struct phylink_config *config)
-{
-	unsigned long *interfaces = config->supported_interfaces;
-
-	if (port == KS8995_CPU_PORT)
-		__set_bit(PHY_INTERFACE_MODE_MII, interfaces);
-
-	if (port <= 3) {
-		/* Internal PHYs */
-		__set_bit(PHY_INTERFACE_MODE_INTERNAL, interfaces);
-		/* phylib default */
-		__set_bit(PHY_INTERFACE_MODE_MII, interfaces);
-	}
-
-	config->mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100;
-}
-
-/* Huge packet support up to 1916 byte packages "inclusive"
- * which means that tags are included. If the bit is not set
- * it is 1536 bytes "inclusive". We present the length without
- * tags or ethernet headers. The setting affects all ports.
- */
-static int ks8995_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
-{
-	struct ks8995_switch *ks = ds->priv;
-	unsigned int max_mtu;
-	int ret;
-	u8 val;
-	int i;
-
-	ks->max_mtu[port] = new_mtu;
-
-	/* Roof out the MTU for the entire switch to the greatest
-	 * common denominator: the biggest set for any one port will
-	 * be the biggest MTU for the switch.
-	 */
-	max_mtu = ETH_DATA_LEN;
-	for (i = 0; i < KS8995_NUM_PORTS; i++) {
-		if (ks->max_mtu[i] > max_mtu)
-			max_mtu = ks->max_mtu[i];
-	}
-
-	/* Translate to layer 2 size.
-	 * Add ethernet and (possible) VLAN headers, and checksum to the size.
-	 * For ETH_DATA_LEN (1500 bytes) this will add up to 1522 bytes.
-	 */
-	max_mtu += VLAN_ETH_HLEN;
-	max_mtu += ETH_FCS_LEN;
-
-	ret = ks8995_read_reg(ks, KS8995_REG_GC2, &val);
-	if (ret) {
-		dev_err(ks->dev, "failed to read KS8995_REG_GC2\n");
-		return ret;
-	}
-
-	if (max_mtu <= 1522) {
-		val &= ~KS8995_GC2_HUGE;
-		val &= ~KS8995_GC2_LEGAL;
-	} else if (max_mtu > 1522 && max_mtu <= 1536) {
-		/* This accepts packets up to 1536 bytes */
-		val &= ~KS8995_GC2_HUGE;
-		val |= KS8995_GC2_LEGAL;
-	} else {
-		/* This accepts packets up to 1916 bytes */
-		val |= KS8995_GC2_HUGE;
-		val |= KS8995_GC2_LEGAL;
-	}
-
-	dev_dbg(ks->dev, "new max MTU %d bytes (inclusive)\n", max_mtu);
-
-	ret = ks8995_write_reg(ks, KS8995_REG_GC2, val);
-	if (ret)
-		dev_err(ks->dev, "failed to set KS8995_REG_GC2\n");
-
-	return ret;
-}
-
-static int ks8995_get_max_mtu(struct dsa_switch *ds, int port)
-{
-	return 1916 - ETH_HLEN - ETH_FCS_LEN;
-}
-
-static const struct dsa_switch_ops ks8995_ds_ops = {
-	.get_tag_protocol = ks8995_get_tag_protocol,
-	.setup = ks8995_setup,
-	.port_pre_bridge_flags = ks8995_port_pre_bridge_flags,
-	.port_bridge_flags = ks8995_port_bridge_flags,
-	.port_enable = ks8995_port_enable,
-	.port_disable = ks8995_port_disable,
-	.port_stp_state_set = ks8995_port_stp_state_set,
-	.port_change_mtu = ks8995_change_mtu,
-	.port_max_mtu = ks8995_get_max_mtu,
-	.phylink_get_caps = ks8995_phylink_get_caps,
-};
-
-/* ------------------------------------------------------------------------ */
-static int ks8995_probe(struct spi_device *spi)
-{
-	struct ks8995_switch *ks;
-	int err;
-	int variant = spi_get_device_id(spi)->driver_data;
-
-	if (variant >= max_variant) {
-		dev_err(&spi->dev, "bad chip variant %d\n", variant);
-		return -ENODEV;
-	}
-
-	ks = devm_kzalloc(&spi->dev, sizeof(*ks), GFP_KERNEL);
-	if (!ks)
-		return -ENOMEM;
-
-	mutex_init(&ks->lock);
-	ks->spi = spi;
-	ks->dev = &spi->dev;
-	ks->chip = &ks8995_chip[variant];
-
-	ks->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset",
-						 GPIOD_OUT_HIGH);
-	err = PTR_ERR_OR_ZERO(ks->reset_gpio);
-	if (err) {
-		dev_err(&spi->dev,
-			"failed to get reset gpio: %d\n", err);
-		return err;
-	}
-
-	err = gpiod_set_consumer_name(ks->reset_gpio, "switch-reset");
-	if (err)
-		return err;
-
-	if (ks->reset_gpio) {
-		/*
-		 * If a reset line was obtained, wait for 100us after
-		 * de-asserting RESET before accessing any registers, see
-		 * the KS8995MA datasheet, page 44.
-		 */
-		gpiod_set_value_cansleep(ks->reset_gpio, 0);
-		udelay(100);
-	}
-
-	spi_set_drvdata(spi, ks);
-
-	spi->mode = SPI_MODE_0;
-	spi->bits_per_word = 8;
-	err = spi_setup(spi);
-	if (err) {
-		dev_err(&spi->dev, "spi_setup failed, err=%d\n", err);
-		return err;
-	}
-
-	err = ks8995_get_revision(ks);
-	if (err)
-		return err;
-
-	err = ks8995_reset(ks);
-	if (err)
-		return err;
-
-	dev_info(&spi->dev, "%s device found, Chip ID:%x, Revision:%x\n",
-		 ks->chip->name, ks->chip->chip_id, ks->revision_id);
-
-	err = ks8995_check_config(ks);
-	if (err)
-		return err;
-
-	ks->ds = devm_kzalloc(&spi->dev, sizeof(*ks->ds), GFP_KERNEL);
-	if (!ks->ds)
-		return -ENOMEM;
-
-	ks->ds->dev = &spi->dev;
-	ks->ds->num_ports = KS8995_NUM_PORTS;
-	ks->ds->ops = &ks8995_ds_ops;
-	ks->ds->phylink_mac_ops = &ks8995_phylink_mac_ops;
-	ks->ds->priv = ks;
-
-	err = dsa_register_switch(ks->ds);
-	if (err)
-		return dev_err_probe(&spi->dev, err,
-				     "unable to register DSA switch\n");
-
-	return 0;
-}
-
-static void ks8995_remove(struct spi_device *spi)
-{
-	struct ks8995_switch *ks = spi_get_drvdata(spi);
-
-	dsa_unregister_switch(ks->ds);
-	/* assert reset */
-	gpiod_set_value_cansleep(ks->reset_gpio, 1);
-}
-
-/* ------------------------------------------------------------------------ */
-static struct spi_driver ks8995_driver = {
-	.driver = {
-		.name	    = "spi-ks8995",
-		.of_match_table = ks8995_spi_of_match,
-	},
-	.probe	  = ks8995_probe,
-	.remove	  = ks8995_remove,
-	.id_table = ks8995_id,
-};
-
-module_spi_driver(ks8995_driver);
-
-MODULE_DESCRIPTION(DRV_DESC);
-MODULE_VERSION(DRV_VERSION);
-MODULE_AUTHOR("Gabor Juhos <juhosg at openwrt.org>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/dsa/lan9303_i2c.c b/drivers/net/dsa/lan9303_i2c.c
index 3a09f3907f4414..7b70e22371fc40 100644
--- a/drivers/net/dsa/lan9303_i2c.c
+++ b/drivers/net/dsa/lan9303_i2c.c
@@ -96,7 +96,7 @@ MODULE_DEVICE_TABLE(i2c, lan9303_i2c_id);
 
 static const struct of_device_id lan9303_i2c_of_match[] = {
 	{ .compatible = "smsc,lan9303-i2c", },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, lan9303_i2c_of_match);
 
diff --git a/drivers/net/dsa/lan9303_mdio.c b/drivers/net/dsa/lan9303_mdio.c
index 0ac4857e5ee86e..d3b5062905b126 100644
--- a/drivers/net/dsa/lan9303_mdio.c
+++ b/drivers/net/dsa/lan9303_mdio.c
@@ -157,7 +157,7 @@ static void lan9303_mdio_shutdown(struct mdio_device *mdiodev)
 static const struct of_device_id lan9303_mdio_of_match[] = {
 	{ .compatible = "smsc,lan9303-mdio" },
 	{ .compatible = "microchip,lan9354-mdio" },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, lan9303_mdio_of_match);
 
diff --git a/drivers/net/dsa/lantiq/lantiq_gswip_common.c b/drivers/net/dsa/lantiq/lantiq_gswip_common.c
index 0e8eedf64d3a39..20ed487f37ca2a 100644
--- a/drivers/net/dsa/lantiq/lantiq_gswip_common.c
+++ b/drivers/net/dsa/lantiq/lantiq_gswip_common.c
@@ -1279,12 +1279,8 @@ static int gswip_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 	/* Enable MLEN for ports with non-standard MTUs, including the special
 	 * header on the CPU port added above.
 	 */
-	if (new_mtu != ETH_DATA_LEN)
-		regmap_set_bits(priv->gswip, GSWIP_MAC_CTRL_2p(port),
-				GSWIP_MAC_CTRL_2_MLEN);
-	else
-		regmap_clear_bits(priv->gswip, GSWIP_MAC_CTRL_2p(port),
-				  GSWIP_MAC_CTRL_2_MLEN);
+	regmap_assign_bits(priv->gswip, GSWIP_MAC_CTRL_2p(port),
+			   GSWIP_MAC_CTRL_2_MLEN, new_mtu != ETH_DATA_LEN);
 
 	return 0;
 }
@@ -1339,6 +1335,7 @@ static void gswip_port_set_speed(struct gswip_priv *priv, int port, int speed,
 		break;
 
 	case SPEED_1000:
+	case SPEED_2500:
 		mdio_phy = GSWIP_MDIO_PHY_SPEED_G1;
 
 		mii_cfg = GSWIP_MII_CFG_RATE_M125;
diff --git a/drivers/net/dsa/lantiq/mxl-gsw1xx.c b/drivers/net/dsa/lantiq/mxl-gsw1xx.c
index a1104b2f92a92d..66bf7e33190571 100644
--- a/drivers/net/dsa/lantiq/mxl-gsw1xx.c
+++ b/drivers/net/dsa/lantiq/mxl-gsw1xx.c
@@ -352,8 +352,22 @@ static int gsw1xx_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
 	/* mark PCS configuration as incomplete */
 	priv->tbi_interface = PHY_INTERFACE_MODE_NA;
 
-	if (!reconf)
+	if (!reconf) {
+		/* setup SerDes clock speed */
+		if (interface == PHY_INTERFACE_MODE_2500BASEX)
+			nco_ctrl = GSW1XX_SGMII_2G5 | GSW1XX_SGMII_2G5_NCO2;
+		else
+			nco_ctrl = GSW1XX_SGMII_1G | GSW1XX_SGMII_1G_NCO1;
+
+		ret = regmap_update_bits(priv->clk, GSW1XX_CLK_NCO_CTRL,
+					 GSW1XX_SGMII_HSP_MASK |
+					 GSW1XX_SGMII_SEL,
+					 nco_ctrl);
+		if (ret)
+			return ret;
+
 		ret = gsw1xx_pcs_reset(priv, interface);
+	}
 
 	if (ret)
 		return ret;
@@ -423,19 +437,6 @@ static int gsw1xx_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
 		return ret;
 
 	if (!reconf) {
-		/* setup SerDes clock speed */
-		if (interface == PHY_INTERFACE_MODE_2500BASEX)
-			nco_ctrl = GSW1XX_SGMII_2G5 | GSW1XX_SGMII_2G5_NCO2;
-		else
-			nco_ctrl = GSW1XX_SGMII_1G | GSW1XX_SGMII_1G_NCO1;
-
-		ret = regmap_update_bits(priv->clk, GSW1XX_CLK_NCO_CTRL,
-					 GSW1XX_SGMII_HSP_MASK |
-					 GSW1XX_SGMII_SEL,
-					 nco_ctrl);
-		if (ret)
-			return ret;
-
 		ret = gsw1xx_pcs_phy_xaui_write(priv, 0x30, 0x80);
 		if (ret)
 			return ret;
@@ -918,7 +919,7 @@ static const struct of_device_id gsw1xx_of_match[] = {
 	{ .compatible = "maxlinear,gsw140", .data = &gsw140_data },
 	{ .compatible = "maxlinear,gsw141", .data = &gsw141_data },
 	{ .compatible = "maxlinear,gsw145", .data = &gsw140_data },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 
 MODULE_DEVICE_TABLE(of, gsw1xx_of_match);
diff --git a/drivers/net/dsa/microchip/Kconfig b/drivers/net/dsa/microchip/Kconfig
index c71d3fd5dfeb86..75c9b2114afd36 100644
--- a/drivers/net/dsa/microchip/Kconfig
+++ b/drivers/net/dsa/microchip/Kconfig
@@ -2,6 +2,7 @@
 menuconfig NET_DSA_MICROCHIP_KSZ_COMMON
 	tristate "Microchip KSZ8XXX/KSZ9XXX/LAN937X series switch support"
 	depends on NET_DSA
+	select NET_DSA_TAG_KS8995
 	select NET_DSA_TAG_KSZ
 	select NET_DSA_TAG_NONE
 	select NET_IEEE8021Q_HELPERS
diff --git a/drivers/net/dsa/microchip/ksz8.c b/drivers/net/dsa/microchip/ksz8.c
index 941ae9f66f70ba..d7498132064e87 100644
--- a/drivers/net/dsa/microchip/ksz8.c
+++ b/drivers/net/dsa/microchip/ksz8.c
@@ -3,6 +3,7 @@
  * Microchip KSZ8XXX series switch driver
  *
  * It supports the following switches:
+ * - KSZ8995XA (the oldest ancestor)
  * - KSZ8463
  * - KSZ8863, KSZ8873 aka KSZ88X3
  * - KSZ8895, KSZ8864 aka KSZ8895 family
@@ -20,6 +21,7 @@
 #include <linux/export.h>
 #include <linux/gpio/consumer.h>
 #include <linux/if_vlan.h>
+#include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/platform_data/microchip-ksz.h>
@@ -253,7 +255,35 @@ static int ksz8463_reset_switch(struct ksz_device *dev)
 
 static int ksz8_reset_switch(struct ksz_device *dev)
 {
-	if (ksz_is_ksz88x3(dev)) {
+	if (ksz_is_ksz8995xa(dev)) {
+		unsigned int port, reg, val;
+		int ret;
+
+		/* The KSZ8995XA switch itself cannot be reset by software, but
+		 * often boot loaders have fun with ports, so reset them. This
+		 * reset bit is self-clearing.
+		 */
+		for (port = 0; port < dev->info->port_cnt; port++) {
+			ret = ksz_prmw8(dev, port, REG_PORT_STATUS_3,
+					PORT_PHY_SOFT_RESET,
+					PORT_PHY_SOFT_RESET);
+			if (ret)
+				return ret;
+		}
+
+		/* IEEE 802.3 allows up to 500 ms for a PHY reset. */
+		for (port = 0; port < dev->info->port_cnt; port++) {
+			reg = dev->dev_ops->get_port_addr(port, REG_PORT_STATUS_3);
+			ret = regmap_read_poll_timeout(ksz_regmap_8(dev), reg, val,
+						       !(val & PORT_PHY_SOFT_RESET),
+						       1000, 600000);
+			if (ret)
+				return ret;
+		}
+
+		/* Some PHYs need additional settling time after reset. */
+		usleep_range(1000, 2000);
+	} else if (ksz_is_ksz88x3(dev)) {
 		/* reset switch */
 		ksz_cfg(dev, KSZ8863_REG_SW_RESET,
 			KSZ8863_GLOBAL_SOFTWARE_RESET | KSZ8863_PCS_RESET, true);
@@ -269,6 +299,31 @@ static int ksz8_reset_switch(struct ksz_device *dev)
 	return 0;
 }
 
+static int ksz8995xa_change_mtu(struct dsa_switch *ds, int port, int mtu)
+{
+	struct ksz_device *dev = ds->priv;
+	int frame_size;
+	u8 ctrl2 = 0;
+
+	if (!dsa_is_cpu_port(dev->ds, port))
+		return 0;
+
+	frame_size = mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;
+
+	/* The legal packet size bit polarity documented in the KSZ8995XA
+	 * manual appears to be wrong. Practical tests show that the same
+	 * semantics as the other switches are correct; using the documented
+	 * polarity makes the switch hang on big packets.
+	 */
+	if (frame_size <= KSZ8995XA_LEGAL_PACKET_SIZE)
+		ctrl2 |= KSZ8863_LEGAL_PACKET_ENABLE;
+	else if (frame_size > KSZ8863_NORMAL_PACKET_SIZE)
+		ctrl2 |= KSZ8863_HUGE_PACKET_ENABLE;
+
+	return ksz_rmw8(dev, REG_SW_CTRL_2, KSZ8863_LEGAL_PACKET_ENABLE |
+			KSZ8863_HUGE_PACKET_ENABLE, ctrl2);
+}
+
 static int ksz88xx_change_mtu(struct dsa_switch *ds, int port, int mtu)
 {
 	struct ksz_device *dev = ds->priv;
@@ -325,8 +380,8 @@ static int ksz88xx_max_mtu(struct dsa_switch *ds, int port)
 
 static int ksz8_port_queue_split(struct ksz_device *dev, int port, int queues)
 {
-	u8 mask_4q, mask_2q;
-	u8 reg_4q, reg_2q;
+	u8 mask_4q = 0, mask_2q;
+	u8 reg_4q = 0, reg_2q;
 	u8 data_4q = 0;
 	u8 data_2q = 0;
 	int ret;
@@ -352,6 +407,10 @@ static int ksz8_port_queue_split(struct ksz_device *dev, int port, int queues)
 		mask_2q = KSZ8873_PORT_2QUEUE_SPLIT_EN;
 		reg_4q = P1CR1;
 		reg_2q = P1CR1 + 1;
+	} else if (ksz_is_ksz8995xa(dev)) {
+		/* This switch has no 4way split support */
+		mask_2q = KSZ8795_PORT_2QUEUE_SPLIT_EN;
+		reg_2q = REG_PORT_CTRL_0;
 	} else {
 		mask_4q = KSZ8795_PORT_4QUEUE_SPLIT_EN;
 		mask_2q = KSZ8795_PORT_2QUEUE_SPLIT_EN;
@@ -373,9 +432,11 @@ static int ksz8_port_queue_split(struct ksz_device *dev, int port, int queues)
 	else if (queues == 2)
 		data_2q = mask_2q;
 
-	ret = ksz_prmw8(dev, port, reg_4q, mask_4q, data_4q);
-	if (ret)
-		return ret;
+	if (mask_4q) {
+		ret = ksz_prmw8(dev, port, reg_4q, mask_4q, data_4q);
+		if (ret)
+			return ret;
+	}
 
 	return ksz_prmw8(dev, port, reg_2q, mask_2q, data_2q);
 }
@@ -970,6 +1031,10 @@ static int ksz8_r_phy_ctrl(struct ksz_device *dev, int port, u16 *val)
 	if (reg_val & PORT_MDIX_STATUS)
 		*val |= KSZ886X_CTRL_MDIX_STAT;
 
+	/* KSZ8995XA has no fancy features in register 0xA */
+	if (ksz_is_ksz8995xa(dev))
+		return 0;
+
 	ret = ksz_pread8(dev, port, REG_PORT_LINK_MD_CTRL, &reg_val);
 	if (ret < 0)
 		return ret;
@@ -1068,8 +1133,10 @@ static int ksz8_r_phy_bmcr(struct ksz_device *dev, u16 port, u16 *val)
 	if (ctrl & PORT_FORCE_FULL_DUPLEX)
 		*val |= BMCR_FULLDPLX;
 
-	if (speed & PORT_HP_MDIX)
-		*val |= KSZ886X_BMCR_HP_MDIX;
+	if (!ksz_is_ksz8995xa(dev)) {
+		if (speed & PORT_HP_MDIX)
+			*val |= KSZ886X_BMCR_HP_MDIX;
+	}
 
 	if (restart & PORT_FORCE_MDIX)
 		*val |= KSZ886X_BMCR_FORCE_MDI;
@@ -1164,6 +1231,9 @@ static int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 			data |= LPA_LPACK;
 		break;
 	case PHY_REG_LINK_MD:
+		if (ksz_is_ksz8995xa(dev))
+			return -EOPNOTSUPP;
+
 		ret = ksz_pread8(dev, p, REG_PORT_LINK_MD_CTRL, &val1);
 		if (ret)
 			return ret;
@@ -1307,13 +1377,15 @@ static int ksz8_w_phy_bmcr(struct ksz_device *dev, u16 port, u16 val)
 	if (val & BMCR_RESET)
 		return 0;
 
-	speed = 0;
-	if (val & KSZ886X_BMCR_HP_MDIX)
-		speed |= PORT_HP_MDIX;
+	if (!ksz_is_ksz8995xa(dev)) {
+		speed = 0;
+		if (val & KSZ886X_BMCR_HP_MDIX)
+			speed |= PORT_HP_MDIX;
 
-	ret = ksz_prmw8(dev, port, regs[P_SPEED_STATUS], PORT_HP_MDIX, speed);
-	if (ret)
-		return ret;
+		ret = ksz_prmw8(dev, port, regs[P_SPEED_STATUS], PORT_HP_MDIX, speed);
+		if (ret)
+			return ret;
+	}
 
 	ctrl = 0;
 	if (ksz_is_ksz88x3(dev)) {
@@ -1423,11 +1495,17 @@ static int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 		}
 		break;
 	case PHY_REG_LINK_MD:
+		if (ksz_is_ksz8995xa(dev))
+			return -EOPNOTSUPP;
+
 		if (val & PHY_START_CABLE_DIAG)
 			ksz_port_cfg(dev, p, REG_PORT_LINK_MD_CTRL, PORT_START_CABLE_DIAG, true);
 		break;
 
 	case PHY_REG_PHY_CTRL:
+		if (ksz_is_ksz8995xa(dev))
+			return -EOPNOTSUPP;
+
 		ret = ksz8_w_phy_ctrl(dev, p, val);
 		if (ret)
 			return ret;
@@ -1527,8 +1605,18 @@ static void ksz8_flush_dyn_mac_table(struct dsa_switch *ds, int port)
 	if (ksz_is_ksz8463(dev)) {
 		reg = KSZ8463_FLUSH_TABLE_CTRL;
 		mask = KSZ8463_FLUSH_DYN_MAC_TABLE;
+	} else if (ksz_is_ksz8995xa(dev)) {
+		reg = REG_SW_CTRL_1;
+		mask = SW_FAST_AGING;
 	}
 	ksz_cfg(dev, reg, mask, true);
+	if (ksz_is_ksz8995xa(dev)) {
+		/* Allow the 800 us fast-age cycle to finish before restoring
+		 * normal ageing and port learning.
+		 */
+		usleep_range(1000, 2000);
+		ksz_cfg(dev, reg, mask, false);
+	}
 	for (index = first; index < cnt; index++) {
 		if (!(learn[index] & PORT_LEARN_DISABLE))
 			ksz_pwrite8(dev, index, regs[P_STP_CTRL], learn[index]);
@@ -2043,12 +2131,14 @@ static void ksz8_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 
 	ksz8_port_queue_split(dev, port, dev->info->num_tx_queues);
 
-	/* replace priority */
-	offset = P_802_1P_CTRL;
-	if (ksz_is_ksz8463(dev))
-		offset = P1CR2;
-	ksz_port_cfg(dev, port, offset,
-		     masks[PORT_802_1P_REMAPPING], false);
+	if (!ksz_is_ksz8995xa(dev)) {
+		/* replace priority */
+		offset = P_802_1P_CTRL;
+		if (ksz_is_ksz8463(dev))
+			offset = P1CR2;
+		ksz_port_cfg(dev, port, offset,
+			     masks[PORT_802_1P_REMAPPING], false);
+	}
 
 	if (cpu_port)
 		member = dsa_user_ports(ds);
@@ -2057,6 +2147,20 @@ static void ksz8_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 
 	ksz8_cfg_port_member(dev, port, member);
 
+	if (ksz_is_ksz8995xa(dev)) {
+		/* Use VID 0 to identify untagged frames in the special tag */
+		ksz_pwrite16(dev, port, REG_PORT_CTRL_VID, 0);
+
+		/* The KSZ8995XA has a special tag format in the front of the frame
+		 * that need to be inserted by the CPU and then removed by each
+		 * port. PORT_REMOVE_TAG simply means "remove tags coming from the
+		 * CPU port" it does not affect ingress packets.
+		 */
+		ksz_prmw8(dev, port, REG_PORT_CTRL_0,
+			  PORT_INSERT_TAG | PORT_REMOVE_TAG,
+			  cpu_port ? PORT_INSERT_TAG : PORT_REMOVE_TAG);
+	}
+
 	/* Disable all WoL options by default. Otherwise
 	 * ksz_switch_macaddr_get/put logic will not work properly.
 	 * CPU port 4 has no WoL functionality.
@@ -2073,6 +2177,7 @@ static int ksz8_dsa_port_setup(struct dsa_switch *ds, int port)
 		return 0;
 
 	ksz8_port_setup(dev, port, false);
+
 	return ksz_dcb_init_port(dev, port);
 }
 
@@ -2147,7 +2252,9 @@ static void ksz8_config_cpu_port(struct dsa_switch *ds)
 	masks = dev->info->masks;
 	regs = dev->info->regs;
 
-	ksz_cfg(dev, regs[S_TAIL_TAG_CTRL], masks[SW_TAIL_TAG_ENABLE], true);
+	/* KSZ8995XA uses a tag in the header instead of the tail */
+	if (!ksz_is_ksz8995xa(dev))
+		ksz_cfg(dev, regs[S_TAIL_TAG_CTRL], masks[SW_TAIL_TAG_ENABLE], true);
 
 	ksz8_port_setup(dev, dev->cpu_port, true);
 
@@ -2323,6 +2430,10 @@ static int ksz8_enable_stp_addr(struct ksz_device *dev)
 {
 	struct alu_struct alu;
 
+	/* KSZ8995XA lacks STP */
+	if (ksz_is_ksz8995xa(dev))
+		return 0;
+
 	/* Setup STP address for STP operation. */
 	memset(&alu, 0, sizeof(alu));
 	ether_addr_copy(alu.mac, eth_stp_addr);
@@ -2650,13 +2761,17 @@ static int ksz8_setup(struct dsa_switch *ds)
 	struct ksz_port *p;
 	const u16 *regs;
 	int i, ret;
+	u8 val;
 
 	regs = dev->info->regs;
 
-	dev->vlan_cache = devm_kcalloc(dev->dev, sizeof(struct vlan_table),
-				       dev->info->num_vlans, GFP_KERNEL);
-	if (!dev->vlan_cache)
-		return -ENOMEM;
+	/* KSZ8995XA has no SW controlled VLAN handling */
+	if (!ksz_is_ksz8995xa(dev)) {
+		dev->vlan_cache = devm_kcalloc(dev->dev, sizeof(struct vlan_table),
+					       dev->info->num_vlans, GFP_KERNEL);
+		if (!dev->vlan_cache)
+			return -ENOMEM;
+	}
 
 	ret = ksz8_reset_switch(dev);
 	if (ret) {
@@ -2685,19 +2800,22 @@ static int ksz8_setup(struct dsa_switch *ds)
 
 	ksz_init_mib_timer(dev);
 
-	ds->configure_vlan_while_not_filtering = false;
 	ds->dscp_prio_mapping_is_global = true;
 	ds->mtu_enforcement_ingress = true;
 
-	/* We rely on software untagging on the CPU port, so that we
-	 * can support both tagged and untagged VLANs
-	 */
-	ds->untag_bridge_pvid = true;
+	if (!ksz_is_ksz8995xa(dev)) {
+		ds->configure_vlan_while_not_filtering = false;
 
-	/* VLAN filtering is partly controlled by the global VLAN
-	 * Enable flag
-	 */
-	ds->vlan_filtering_is_global = true;
+		/* We rely on software untagging on the CPU port, so that we
+		 * can support both tagged and untagged VLANs
+		 */
+		ds->untag_bridge_pvid = true;
+
+		/* VLAN filtering is partly controlled by the global VLAN
+		 * Enable flag
+		 */
+		ds->vlan_filtering_is_global = true;
+	}
 
 	/* Enable automatic fast aging when link changed detected. */
 	ksz_cfg(dev, S_LINK_AGING_CTRL, SW_LINK_AUTO_AGING, true);
@@ -2711,9 +2829,10 @@ static int ksz8_setup(struct dsa_switch *ds)
 	 * Make sure unicast VLAN boundary is set as default and
 	 * enable no excessive collision drop.
 	 */
-	ret = ksz_rmw8(dev, REG_SW_CTRL_2,
-		       UNICAST_VLAN_BOUNDARY | NO_EXC_COLLISION_DROP,
-		       UNICAST_VLAN_BOUNDARY | NO_EXC_COLLISION_DROP);
+	val = NO_EXC_COLLISION_DROP;
+	if (!ksz_is_ksz8995xa(dev))
+		val |= UNICAST_VLAN_BOUNDARY;
+	ret = ksz_rmw8(dev, REG_SW_CTRL_2, val, val);
 	if (ret)
 		return ret;
 
@@ -2721,11 +2840,15 @@ static int ksz8_setup(struct dsa_switch *ds)
 
 	ksz_cfg(dev, S_MIRROR_CTRL, SW_MIRROR_RX_TX, false);
 
-	if (!ksz_is_ksz88x3(dev))
+	if (ksz_is_ksz8995xa(dev))
+		ksz_cfg(dev, REG_SW_CTRL_9, SW_SPECIAL_TAG, true);
+	else if (!ksz_is_ksz88x3(dev))
 		ksz_cfg(dev, REG_SW_CTRL_19, SW_INS_TAG_ENABLE, true);
 
-	for (i = 0; i < (dev->info->num_vlans / 4); i++)
-		ksz8_r_vlan_entries(dev, i);
+	if (!ksz_is_ksz8995xa(dev)) {
+		for (i = 0; i < (dev->info->num_vlans / 4); i++)
+			ksz8_r_vlan_entries(dev, i);
+	}
 
 	/* Make sure PME (WoL) is not enabled. If requested, it will
 	 * be enabled by ksz_wol_pre_shutdown(). Otherwise, some PMICs
@@ -2976,6 +3099,13 @@ static int ksz8_switch_init(struct ksz_device *dev)
 	return 0;
 }
 
+static enum dsa_tag_protocol ksz8995xa_get_tag_protocol(struct dsa_switch *ds,
+							int port,
+							enum dsa_tag_protocol mp)
+{
+	return DSA_TAG_PROTO_KS8995;
+}
+
 static enum dsa_tag_protocol ksz8463_get_tag_protocol(struct dsa_switch *ds,
 						      int port,
 						      enum dsa_tag_protocol mp)
@@ -3072,6 +3202,16 @@ const struct phylink_mac_ops ksz8_phylink_mac_ops = {
 	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
 };
 
+/*
+ * The KS(Z)8995XA has no indirect access, meaning no MIB counters,
+ * no FDB access, and no VLAN handling.
+ */
+const struct ksz_dev_ops ksz8995xa_dev_ops = {
+	.get_port_addr = ksz8_get_port_addr,
+	.cfg_port_member = ksz8_cfg_port_member,
+	.init = ksz8_switch_init,
+};
+
 const struct ksz_dev_ops ksz8463_dev_ops = {
 	.get_port_addr = ksz8463_get_port_addr,
 	.cfg_port_member = ksz8_cfg_port_member,
@@ -3111,6 +3251,38 @@ const struct ksz_dev_ops ksz88xx_dev_ops = {
 	.pme_pwrite8 = ksz8_pme_pwrite8,
 };
 
+/*
+ * Restricted operations for KSZ8995XA, so many things are not supported
+ * by this old switch that we need diet DSA operations.
+ */
+const struct dsa_switch_ops ksz8995xa_switch_ops = {
+	.get_tag_protocol	= ksz8995xa_get_tag_protocol,
+	.setup			= ksz8_setup,
+	.teardown		= ksz_teardown,
+	.phy_read		= ksz8_phy_read16,
+	.phy_write		= ksz8_phy_write16,
+	.phylink_get_caps	= ksz8_phylink_get_caps,
+	.port_setup		= ksz8_dsa_port_setup,
+	.port_bridge_join	= ksz_port_bridge_join,
+	.port_bridge_leave	= ksz_port_bridge_leave,
+	.port_set_mac_address	= ksz_port_set_mac_address,
+	.port_stp_state_set	= ksz_port_stp_state_set,
+	.port_pre_bridge_flags	= ksz_port_pre_bridge_flags,
+	.port_bridge_flags	= ksz_port_bridge_flags,
+	.port_fast_age		= ksz8_flush_dyn_mac_table,
+	.port_mirror_add	= ksz8_port_mirror_add,
+	.port_mirror_del	= ksz8_port_mirror_del,
+	.port_change_mtu	= ksz8995xa_change_mtu,
+	.port_max_mtu		= ksz88xx_max_mtu,
+	.port_get_default_prio	= ksz_port_get_default_prio,
+	.port_set_default_prio	= ksz_port_set_default_prio,
+	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
+	.port_add_dscp_prio	= ksz_port_add_dscp_prio,
+	.port_del_dscp_prio	= ksz_port_del_dscp_prio,
+	.port_get_apptrust	= ksz_port_get_apptrust,
+	.port_set_apptrust	= ksz_port_set_apptrust,
+};
+
 const struct dsa_switch_ops ksz8463_switch_ops = {
 	.get_tag_protocol	= ksz8463_get_tag_protocol,
 	.connect_tag_protocol   = ksz8463_connect_tag_protocol,
diff --git a/drivers/net/dsa/microchip/ksz8.h b/drivers/net/dsa/microchip/ksz8.h
index bc371cc26c6f1d..9b37c56c9617e9 100644
--- a/drivers/net/dsa/microchip/ksz8.h
+++ b/drivers/net/dsa/microchip/ksz8.h
@@ -12,11 +12,13 @@
 #include <net/dsa.h>
 #include "ksz_common.h"
 
+extern const struct ksz_dev_ops ksz8995xa_dev_ops;
 extern const struct ksz_dev_ops ksz8463_dev_ops;
 extern const struct ksz_dev_ops ksz87xx_dev_ops;
 extern const struct ksz_dev_ops ksz88xx_dev_ops;
 extern const struct phylink_mac_ops ksz88x3_phylink_mac_ops;
 extern const struct phylink_mac_ops ksz8_phylink_mac_ops;
+extern const struct dsa_switch_ops ksz8995xa_switch_ops;
 extern const struct dsa_switch_ops ksz8463_switch_ops;
 extern const struct dsa_switch_ops ksz87xx_switch_ops;
 extern const struct dsa_switch_ops ksz88xx_switch_ops;
diff --git a/drivers/net/dsa/microchip/ksz8_reg.h b/drivers/net/dsa/microchip/ksz8_reg.h
index 6bc511da1f7ddd..f858bce84bfd43 100644
--- a/drivers/net/dsa/microchip/ksz8_reg.h
+++ b/drivers/net/dsa/microchip/ksz8_reg.h
@@ -95,6 +95,8 @@
 #define SW_LED_LINK_ACT_DUPLEX		2
 #define SW_LED_LINK_DUPLEX		3
 
+#define SW_SPECIAL_TAG			BIT(0) /* KSZ8995XA only */
+
 #define REG_SW_CTRL_10			0x0C
 
 #define SW_PASS_PAUSE			BIT(0)
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index 1a9d6f83a02328..f1b53439ff5829 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -446,6 +446,20 @@ static const u8 ksz8895_shifts[] = {
 	[DYNAMIC_MAC_SRC_PORT]		= 24,
 };
 
+static const u16 ksz8995xa_regs[] = {
+	[REG_SW_MAC_ADDR]		= 0x68,
+	[P_FORCE_CTRL]			= 0x0C,
+	[P_LINK_STATUS]			= 0x0E,
+	[P_LOCAL_CTRL]			= 0x0C,
+	[P_NEG_RESTART_CTRL]		= 0x0D,
+	[P_REMOTE_STATUS]		= 0x0E,
+	[P_SPEED_STATUS]		= 0x09,
+	[P_STP_CTRL]			= 0x02,
+	[S_START_CTRL]			= 0x01,
+	[S_BROADCAST_CTRL]		= 0x06,
+	[S_MULTICAST_CTRL]		= 0x04,
+};
+
 static const u16 ksz9477_regs[] = {
 	[REG_SW_MAC_ADDR]		= 0x0302,
 	[P_STP_CTRL]			= 0x0B04,
@@ -1377,6 +1391,21 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.internal_phy = {true, true, true, true, false},
 	},
 
+	[KSZ8995XA] = {
+		.chip_id = KSZ8995XA_CHIP_ID, /* Also known as KS8995XA */
+		.dev_name = "KSZ8995XA",
+		.cpu_ports = 0x10,	/* can be configured as cpu port */
+		.port_cnt = 5,		/* total cpu and user ports */
+		.num_tx_queues = 2,	/* low/hi priority queues, no more */
+		.num_ipms = 2,
+		.ops = &ksz8995xa_dev_ops,
+		.switch_ops = &ksz8995xa_switch_ops,
+		.phylink_mac_ops = &ksz88x3_phylink_mac_ops,
+		.regs = ksz8995xa_regs,
+		.supports_mii = {true, true, true, true, true},
+		.internal_phy = {true, true, true, true, false},
+	},
+
 	[KSZ9477] = {
 		.chip_id = KSZ9477_CHIP_ID,
 		.dev_name = "KSZ9477",
@@ -1486,6 +1515,41 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.gbit_capable	= {true, true, true, true, true, true, true},
 	},
 
+	[KSZ9897S] = {
+		.chip_id = KSZ9897S_CHIP_ID,
+		.dev_name = "KSZ9897S",
+		.num_vlans = 4096,
+		.num_alus = 4096,
+		.num_statics = 16,
+		.cpu_ports = 0x7F,	/* can be configured as cpu port */
+		.port_cnt = 7,		/* total physical port count */
+		.port_nirqs = 2,
+		.num_tx_queues = 4,
+		.num_ipms = 8,
+		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
+		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
+		.phy_errata_9477 = true,
+		.mib_names = ksz9477_mib_names,
+		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
+		.reg_mib_cnt = MIB_COUNTER_NUM,
+		.regs = ksz9477_regs,
+		.masks = ksz9477_masks,
+		.shifts = ksz9477_shifts,
+		.xmii_ctrl0 = ksz9477_xmii_ctrl0,
+		.xmii_ctrl1 = ksz9477_xmii_ctrl1,
+		.supports_mii	= {false, false, false, false,
+				   false, true, false},
+		.supports_rmii	= {false, false, false, false,
+				   false, true, false},
+		.supports_rgmii = {false, false, false, false,
+				   false, true, false},
+		.internal_phy	= {true, true, true, true,
+				   true, false, false},
+		.gbit_capable	= {true, true, true, true, true, true, true},
+		.sgmii_port = 7,
+	},
+
 	[KSZ9893] = {
 		.chip_id = KSZ9893_CHIP_ID,
 		.dev_name = "KSZ9893",
@@ -2649,6 +2713,10 @@ void ksz_init_mib_timer(struct ksz_device *dev)
 {
 	int i;
 
+	/* KSZ8995XA lacks MiB features */
+	if (ksz_is_ksz8995xa(dev))
+		return;
+
 	INIT_DELAYED_WORK(&dev->mib_read, ksz_mib_read_work);
 
 	for (i = 0; i < dev->info->port_cnt; i++) {
@@ -2944,11 +3012,15 @@ static int ksz_switch_detect(struct ksz_device *dev)
 			return -ENODEV;
 		break;
 	case KSZ8895_FAMILY_ID:
-		if (id2 == KSZ8895_CHIP_ID_95 ||
-		    id2 == KSZ8895_CHIP_ID_95R)
+		if (id2 == KSZ8895_CHIP_ID_95XA) {
+			dev->chip_id = KSZ8995XA_CHIP_ID;
+			break;
+		} else if (id2 == KSZ8895_CHIP_ID_95 ||
+			   id2 == KSZ8895_CHIP_ID_95R) {
 			dev->chip_id = KSZ8895_CHIP_ID;
-		else
+		} else {
 			return -ENODEV;
+		}
 		ret = ksz_read8(dev, REG_KSZ8864_CHIP_ID, &id4);
 		if (ret)
 			return ret;
@@ -3897,6 +3969,7 @@ int ksz_switch_register(struct ksz_device *dev)
 	struct device_node *ports;
 	phy_interface_t interface;
 	unsigned int port_num;
+	u32 lookup_chip_id;
 	int ret;
 	int i;
 
@@ -3933,7 +4006,30 @@ int ksz_switch_register(struct ksz_device *dev)
 	if (ret)
 		return ret;
 
-	info = ksz_lookup_info(dev->chip_id);
+	lookup_chip_id = dev->chip_id;
+
+	/* The KSZ9897S and the KSZ9897R report the same chip ID and differ
+	 * only in chip_data: only the S has the SGMII port 7, the R has a
+	 * second RGMII port instead. Bit 7 of the port 7 XMII control 0
+	 * register is read-only and tells them apart; it reads one on the S.
+	 * Compare the KSZ9897S data sheet DS00002394C section 5.2.4.1 with
+	 * the KSZ9897R data sheet DS00002330D section 5.2.3.1.
+	 *
+	 * Only the chip_data entry differs, so dev->chip_id keeps the shared
+	 * chip ID and nothing else has to know about the variant.
+	 */
+	if (dev->chip_id == KSZ9897_CHIP_ID) {
+		u8 val;
+
+		ret = ksz_read8(dev, KSZ9897_REG_PORT7_XMII_CTRL_0, &val);
+		if (ret)
+			return ret;
+
+		if (val & KSZ9897_PORT7_SGMII_SEL)
+			lookup_chip_id = KSZ9897S_CHIP_ID;
+	}
+
+	info = ksz_lookup_info(lookup_chip_id);
 	if (!info)
 		return -ENODEV;
 
@@ -4029,11 +4125,13 @@ int ksz_switch_register(struct ksz_device *dev)
 	if (ret)
 		return ret;
 
-	/* Read MIB counters every 30 seconds to avoid overflow. */
-	dev->mib_read_interval = msecs_to_jiffies(5000);
+	if (!ksz_is_ksz8995xa(dev)) {
+		/* Read MIB counters every 30 seconds to avoid overflow. */
+		dev->mib_read_interval = msecs_to_jiffies(5000);
 
-	/* Start the MIB timer. */
-	schedule_delayed_work(&dev->mib_read, 0);
+		/* Start the MIB timer. */
+		schedule_delayed_work(&dev->mib_read, 0);
+	}
 
 	return ret;
 }
diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h
index cbe98494578c37..091cfbdc486f87 100644
--- a/drivers/net/dsa/microchip/ksz_common.h
+++ b/drivers/net/dsa/microchip/ksz_common.h
@@ -281,9 +281,11 @@ enum ksz_model {
 	KSZ88X3,
 	KSZ8864,
 	KSZ8895,
+	KSZ8995XA,
 	KSZ9477,
 	KSZ9896,
 	KSZ9897,
+	KSZ9897S,
 	KSZ9893,
 	KSZ9563,
 	KSZ9567,
@@ -789,7 +791,13 @@ static inline bool ksz_is_ksz88x3(struct ksz_device *dev)
 static inline bool ksz_is_8895_family(struct ksz_device *dev)
 {
 	return dev->chip_id == KSZ8895_CHIP_ID ||
-	       dev->chip_id == KSZ8864_CHIP_ID;
+	       dev->chip_id == KSZ8864_CHIP_ID ||
+	       dev->chip_id == KSZ8995XA_CHIP_ID;
+}
+
+static inline bool ksz_is_ksz8995xa(struct ksz_device *dev)
+{
+	return dev->chip_id == KSZ8995XA_CHIP_ID;
 }
 
 static inline bool is_ksz8(struct ksz_device *dev)
@@ -860,6 +868,7 @@ static inline bool ksz_is_sgmii_port(struct ksz_device *dev, int port)
 #define KSZ87_CHIP_ID_94		0x6
 #define KSZ87_CHIP_ID_95		0x9
 #define KSZ88_CHIP_ID_63		0x3
+#define KSZ8895_CHIP_ID_95XA		0x0
 #define KSZ8895_CHIP_ID_95		0x4
 #define KSZ8895_CHIP_ID_95R		0x6
 
@@ -874,6 +883,10 @@ static inline bool ksz_is_sgmii_port(struct ksz_device *dev, int port)
 #define SKU_ID_KSZ8563			0x3c
 #define SKU_ID_KSZ9563			0x1c
 
+/* KSZ9897 specific register: port 7 XMII control 0 */
+#define KSZ9897_REG_PORT7_XMII_CTRL_0	0x7300
+#define KSZ9897_PORT7_SGMII_SEL		BIT(7)
+
 /* Driver set switch broadcast storm protection at 10% rate. */
 #define BROADCAST_STORM_PROT_RATE	10
 
@@ -923,6 +936,7 @@ static inline bool ksz_is_sgmii_port(struct ksz_device *dev, int port)
 #define KSZ8863_HUGE_PACKET_SIZE	1916
 #define KSZ8863_NORMAL_PACKET_SIZE	1536
 #define KSZ8_LEGAL_PACKET_SIZE		1518
+#define KSZ8995XA_LEGAL_PACKET_SIZE	1522
 #define KSZ9477_MAX_FRAME_SIZE		9000
 
 #define KSZ8873_REG_GLOBAL_CTRL_12	0x0e
diff --git a/drivers/net/dsa/microchip/ksz_dcb.c b/drivers/net/dsa/microchip/ksz_dcb.c
index 7131c5caac547b..aa0260f07252e8 100644
--- a/drivers/net/dsa/microchip/ksz_dcb.c
+++ b/drivers/net/dsa/microchip/ksz_dcb.c
@@ -20,10 +20,12 @@
 #define KSZ8_PORT_DIFFSERV_ENABLE		BIT(6)
 #define KSZ8_PORT_802_1P_ENABLE			BIT(5)
 #define KSZ8_PORT_BASED_PRIO_M			GENMASK(4, 3)
+#define KSZ8995XA_PORT_BASED_PRIO		BIT(4)
 
 #define KSZ8463_REG_TOS_DSCP_CTRL		0x16
 #define KSZ88X3_REG_TOS_DSCP_CTRL		0x60
 #define KSZ8765_REG_TOS_DSCP_CTRL		0x90
+#define KSZ8995XA_REG_TOS_DSCP_CTRL_7		0x67
 
 #define KSZ9477_REG_SW_MAC_TOS_CTRL		0x033e
 #define KSZ9477_SW_TOS_DSCP_REMAP		BIT(0)
@@ -98,8 +100,11 @@ static void ksz_get_default_port_prio_reg(struct ksz_device *dev, int *reg,
 {
 	if (is_ksz8(dev)) {
 		*reg = KSZ8_REG_PORT_1_CTRL_0;
-		*mask = KSZ8_PORT_BASED_PRIO_M;
-		*shift = __bf_shf(KSZ8_PORT_BASED_PRIO_M);
+		if (ksz_is_ksz8995xa(dev))
+			*mask = KSZ8995XA_PORT_BASED_PRIO;
+		else
+			*mask = KSZ8_PORT_BASED_PRIO_M;
+		*shift = __bf_shf(*mask);
 		if (ksz_is_ksz8463(dev))
 			*reg = KSZ8463_REG_PORT_1_CTRL_0;
 	} else {
@@ -112,31 +117,45 @@ static void ksz_get_default_port_prio_reg(struct ksz_device *dev, int *reg,
 /**
  * ksz_get_dscp_prio_reg - Retrieves the DSCP-to-priority-mapping register
  * @dev: Pointer to the KSZ switch device structure
+ * @dscp: DSCP value for which to retrieve the register
  * @reg: Pointer to the register address to be set
- * @per_reg: Pointer to the number of DSCP values per register
  * @mask: Pointer to the mask to be set
+ * @shift: Pointer to the bit shift to be set
  *
- * This function retrieves the DSCP to priority mapping register, the number of
- * DSCP values per register, and the mask to be set.
+ * This function retrieves the register, mask and shift for a DSCP to priority
+ * mapping entry.
  */
-static void ksz_get_dscp_prio_reg(struct ksz_device *dev, int *reg,
-				  int *per_reg, u8 *mask)
+static void ksz_get_dscp_prio_reg(struct ksz_device *dev, u8 dscp, int *reg,
+				  u8 *mask, int *shift)
 {
+	int per_reg;
+
+	if (ksz_is_ksz8995xa(dev)) {
+		/* KSZ8995XA stores DSCP groups in descending register order. */
+		*reg = KSZ8995XA_REG_TOS_DSCP_CTRL_7 - dscp / 8;
+		*mask = BIT(0);
+		*shift = dscp % 8;
+		return;
+	}
+
 	if (ksz_is_ksz87xx(dev) || ksz_is_8895_family(dev)) {
 		*reg = KSZ8765_REG_TOS_DSCP_CTRL;
-		*per_reg = 4;
+		per_reg = 4;
 		*mask = GENMASK(1, 0);
 	} else if (ksz_is_ksz88x3(dev) || ksz_is_ksz8463(dev)) {
 		*reg = KSZ88X3_REG_TOS_DSCP_CTRL;
-		*per_reg = 4;
+		per_reg = 4;
 		*mask = GENMASK(1, 0);
 		if (ksz_is_ksz8463(dev))
 			*reg = KSZ8463_REG_TOS_DSCP_CTRL;
 	} else {
 		*reg = KSZ9477_REG_DIFFSERV_PRIO_MAP;
-		*per_reg = 2;
+		per_reg = 2;
 		*mask = GENMASK(2, 0);
 	}
+
+	*reg += dscp / per_reg;
+	*shift = (dscp % per_reg) * (8 / per_reg);
 }
 
 /**
@@ -236,10 +255,10 @@ int ksz_port_set_default_prio(struct dsa_switch *ds, int port, u8 prio)
 int ksz_port_get_dscp_prio(struct dsa_switch *ds, int port, u8 dscp)
 {
 	struct ksz_device *dev = ds->priv;
-	int reg, per_reg, ret, shift;
+	int reg, ret, shift;
 	u8 data, mask;
 
-	ksz_get_dscp_prio_reg(dev, &reg, &per_reg, &mask);
+	ksz_get_dscp_prio_reg(dev, dscp, &reg, &mask, &shift);
 
 	/* If DSCP remapping is disabled, DSCP bits 3-5 are used as Internal
 	 * Priority Map (IPM)
@@ -260,13 +279,10 @@ int ksz_port_get_dscp_prio(struct dsa_switch *ds, int port, u8 dscp)
 	/* In case DSCP remapping is enabled, we need to write the DSCP to
 	 * priority mapping table.
 	 */
-	reg += dscp / per_reg;
 	ret = ksz_read8(dev, reg, &data);
 	if (ret)
 		return ret;
 
-	shift = (dscp % per_reg) * (8 / per_reg);
-
 	return (data >> shift) & mask;
 }
 
@@ -283,15 +299,12 @@ int ksz_port_get_dscp_prio(struct dsa_switch *ds, int port, u8 dscp)
  */
 static int ksz_set_global_dscp_entry(struct ksz_device *dev, u8 dscp, u8 ipm)
 {
-	int reg, per_reg, shift;
+	int reg, shift;
 	u8 mask;
 
-	ksz_get_dscp_prio_reg(dev, &reg, &per_reg, &mask);
+	ksz_get_dscp_prio_reg(dev, dscp, &reg, &mask, &shift);
 
-	shift = (dscp % per_reg) * (8 / per_reg);
-
-	return ksz_rmw8(dev, reg + (dscp / per_reg), mask << shift,
-			ipm << shift);
+	return ksz_rmw8(dev, reg, mask << shift, ipm << shift);
 }
 
 /**
@@ -441,6 +454,9 @@ static void ksz_apptrust_error(struct ksz_device *dev)
  * 2. IEEE_8021QAZ_APP_SEL_DSCP - Differentiated Services Code Point selector
  *   (lowest priority)
  *
+ * KSZ8995XA instead ORs the classification results, so it supports only
+ * one trusted selector at a time.
+ *
  * Return: 0 on success, or a negative error code on failure
  */
 static int ksz_port_set_apptrust_validate(struct ksz_device *dev, int port,
@@ -449,6 +465,12 @@ static int ksz_port_set_apptrust_validate(struct ksz_device *dev, int port,
 	int i, j, found;
 	int j_prev = 0;
 
+	if (ksz_is_ksz8995xa(dev) && nsel > 1) {
+		dev_err(dev->dev,
+			"KSZ8995XA supports only one apptrust selector at a time\n");
+		return -EOPNOTSUPP;
+	}
+
 	/* Iterate through the requested selectors */
 	for (i = 0; i < nsel; i++) {
 		found = 0;
@@ -552,6 +574,10 @@ int ksz_port_get_apptrust(struct dsa_switch *ds, int port, u8 *sel, int *nsel)
 	if (ret)
 		return ret;
 
+	/* XA's combined classification cannot be reported as a trust order. */
+	if (ksz_is_ksz8995xa(dev) && (data & mask) == mask)
+		return -EOPNOTSUPP;
+
 	*nsel = 0;
 	for (i = 0; i < ARRAY_SIZE(ksz_supported_apptrust); i++) {
 		if (data & map[i].bit)
diff --git a/drivers/net/dsa/microchip/ksz_spi.c b/drivers/net/dsa/microchip/ksz_spi.c
index 373e9054947cb7..e3bb9e604186df 100644
--- a/drivers/net/dsa/microchip/ksz_spi.c
+++ b/drivers/net/dsa/microchip/ksz_spi.c
@@ -161,7 +161,8 @@ static int ksz_spi_probe(struct spi_device *spi)
 		 chip->chip_id == KSZ8794_CHIP_ID ||
 		 chip->chip_id == KSZ8765_CHIP_ID)
 		regmap_config = ksz8795_regmap_config;
-	else if (chip->chip_id == KSZ8895_CHIP_ID ||
+	else if (chip->chip_id == KSZ8995XA_CHIP_ID ||
+		 chip->chip_id == KSZ8895_CHIP_ID ||
 		 chip->chip_id == KSZ8864_CHIP_ID)
 		regmap_config = ksz8863_regmap_config;
 	else
@@ -185,7 +186,10 @@ static int ksz_spi_probe(struct spi_device *spi)
 		dev->pdata = spi->dev.platform_data;
 
 	/* setup spi */
-	spi->mode = SPI_MODE_3;
+	if (chip->chip_id == KSZ8995XA_CHIP_ID)
+		spi->mode = SPI_MODE_0;
+	else
+		spi->mode = SPI_MODE_3;
 	ret = spi_setup(spi);
 	if (ret)
 		return ret;
@@ -224,6 +228,25 @@ static void ksz_spi_shutdown(struct spi_device *spi)
 }
 
 static const struct of_device_id ksz_dt_ids[] = {
+	/*
+	 * Legacy Micrel bindings. In 2015 Microchip acquired
+	 * Micrel which is the originator of the KSZ series, and
+	 * devices branded for Micrel already existed, as well as
+	 * some device tree bindings. These two products are identical
+	 * to the same Microchip products.
+	 */
+	{
+		.compatible = "micrel,ksz8864",
+		.data = &ksz_switch_chips[KSZ8864]
+	},
+	{
+		.compatible = "micrel,ksz8795",
+		.data = &ksz_switch_chips[KSZ8795]
+	},
+	{
+		.compatible = "micrel,ks8995",
+		.data = &ksz_switch_chips[KSZ8995XA]
+	},
 	{
 		.compatible = "microchip,ksz8463",
 		.data = &ksz_switch_chips[KSZ8463]
@@ -256,6 +279,10 @@ static const struct of_device_id ksz_dt_ids[] = {
 		.compatible = "microchip,ksz8895",
 		.data = &ksz_switch_chips[KSZ8895]
 	},
+	{
+		.compatible = "microchip,ksz8995xa",
+		.data = &ksz_switch_chips[KSZ8995XA]
+	},
 	{
 		.compatible = "microchip,ksz9477",
 		.data = &ksz_switch_chips[KSZ9477]
@@ -317,6 +344,7 @@ static const struct of_device_id ksz_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, ksz_dt_ids);
 
 static const struct spi_device_id ksz_spi_ids[] = {
+	{ "ks8995" },
 	{ "ksz8463" },
 	{ "ksz8765" },
 	{ "ksz8794" },
@@ -325,6 +353,7 @@ static const struct spi_device_id ksz_spi_ids[] = {
 	{ "ksz8864" },
 	{ "ksz8873" },
 	{ "ksz8895" },
+	{ "ksz8995xa" },
 	{ "ksz9477" },
 	{ "ksz9896" },
 	{ "ksz9897" },
diff --git a/drivers/net/dsa/motorcomm/Kconfig b/drivers/net/dsa/motorcomm/Kconfig
new file mode 100644
index 00000000000000..79cdd79a1fd25f
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/Kconfig
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config NET_DSA_YT921X
+	tristate "Motorcomm YT9215 ethernet switch chip support"
+	select NET_DSA_TAG_YT921X
+	select NET_IEEE8021Q_HELPERS if DCB
+	help
+	  This enables support for the Motorcomm YT9215 ethernet switch
+	  chip.
+
+config NET_DSA_YT921X_LEDS
+	bool "LED support for Motorcomm YT9215"
+	default y
+	depends on NET_DSA_YT921X
+	depends on LEDS_CLASS=y || LEDS_CLASS=NET_DSA_YT921X
+	help
+	  This enables support for controlling the LEDs attached to the
+	  Motorcomm YT9215 switch chips.
diff --git a/drivers/net/dsa/motorcomm/Makefile b/drivers/net/dsa/motorcomm/Makefile
new file mode 100644
index 00000000000000..1d2c1b3064c410
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_NET_DSA_YT921X) += yt921x.o
+yt921x-objs := chip.o
+yt921x-$(CONFIG_NET_DSA_YT921X_LEDS) += leds.o
+yt921x-objs += mdio_bus.o
+yt921x-objs += pcs-921x.o
+yt921x-objs += smi.o
diff --git a/drivers/net/dsa/motorcomm/chip.c b/drivers/net/dsa/motorcomm/chip.c
new file mode 100644
index 00000000000000..3ff2de383157a3
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/chip.c
@@ -0,0 +1,4510 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Motorcomm YT921x Switch
+ *
+ * Should work on YT9213/YT9214/YT9215/YT9218, but only tested on YT9215+SGMII,
+ * be sure to do your own checks before porting to another chip.
+ *
+ * Copyright (c) 2025 David Yang
+ */
+
+#include <linux/dcbnl.h>
+#include <linux/etherdevice.h>
+#include <linux/if_bridge.h>
+#include <linux/if_hsr.h>
+#include <linux/if_vlan.h>
+#include <linux/mdio.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+#include <linux/sort.h>
+
+#include <net/dsa.h>
+#include <net/dscp.h>
+#include <net/ieee8021q.h>
+#include <net/pkt_cls.h>
+
+#include "chip.h"
+#include "leds.h"
+#include "mdio_bus.h"
+#include "pcs.h"
+#include "smi.h"
+
+struct yt921x_mib_desc {
+	unsigned int size;
+	unsigned int offset;
+	const char *name;
+};
+
+#define MIB_DESC(_size, _offset, _name) \
+	{_size, _offset, _name}
+
+/* Must agree with yt921x_mib
+ *
+ * Unstructured fields (name != NULL) will appear in get_ethtool_stats(),
+ * structured go to their *_stats() methods, but we need their sizes and offsets
+ * to perform 32bit MIB overflow wraparound.
+ */
+static const struct yt921x_mib_desc yt921x_mib_descs[] = {
+	MIB_DESC(1, YT921X_MIB_DATA_RX_BROADCAST, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PAUSE, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_MULTICAST, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_CRC_ERR, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_RX_ALIGN_ERR, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_UNDERSIZE_ERR, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_FRAG_ERR, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_64, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_65_TO_127, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_128_TO_255, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_256_TO_511, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_512_TO_1023, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_1024_TO_1518, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_1519_TO_MAX, NULL),
+	MIB_DESC(2, YT921X_MIB_DATA_RX_GOOD_BYTES, NULL),
+
+	MIB_DESC(2, YT921X_MIB_DATA_RX_BAD_BYTES, "RxBadBytes"),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_OVERSIZE_ERR, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_RX_DROPPED, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_BROADCAST, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PAUSE, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_MULTICAST, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_TX_UNDERSIZE_ERR, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_64, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_65_TO_127, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_128_TO_255, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_256_TO_511, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_512_TO_1023, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_1024_TO_1518, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_1519_TO_MAX, NULL),
+
+	MIB_DESC(2, YT921X_MIB_DATA_TX_GOOD_BYTES, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_COLLISION, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_TX_EXCESSIVE_COLLISION, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_MULTIPLE_COLLISION, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_SINGLE_COLLISION, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT, NULL),
+
+	MIB_DESC(1, YT921X_MIB_DATA_TX_DEFERRED, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_LATE_COLLISION, NULL),
+	MIB_DESC(1, YT921X_MIB_DATA_RX_OAM, "RxOAM"),
+	MIB_DESC(1, YT921X_MIB_DATA_TX_OAM, "TxOAM"),
+};
+
+static const struct yt921x_info yt921x_infos[] = {
+	{
+		"YT9215SC", YT9215_MAJOR, 1, 0,
+		GENMASK(4, 0),
+		BIT(9),
+		BIT(8) | BIT(9),
+	},
+	{
+		"YT9215S", YT9215_MAJOR, 2, 0,
+		GENMASK(4, 0),
+		BIT(9),
+		BIT(8),
+	},
+	{
+		"YT9215RB", YT9215_MAJOR, 3, 0,
+		GENMASK(4, 0),
+		BIT(8) | BIT(9),
+		0,
+	},
+	{
+		"YT9214NB", YT9215_MAJOR, 3, 2,
+		BIT(1) | BIT(3),
+		BIT(9),
+		BIT(8),
+	},
+	{
+		"YT9213NB", YT9215_MAJOR, 3, 3,
+		BIT(1) | BIT(3),
+		BIT(9),
+		BIT(9),
+	},
+	{
+		"YT9218N", YT9218_MAJOR, 0, 0,
+		GENMASK(7, 0),
+		0,
+		0,
+	},
+	{
+		"YT9218MB", YT9218_MAJOR, 1, 0,
+		GENMASK(7, 0),
+		BIT(8) | BIT(9),
+		BIT(8) | BIT(9),
+	},
+	{}
+};
+
+#define YT921X_VID_UNWARE	4095
+
+/* The interval should be small enough to avoid overflow of 32bit MIBs.
+ *
+ * Until we can read MIBs from stats64 call directly (i.e. sleep
+ * there), we have to poll stats more frequently then it is actually needed.
+ * For overflow protection, normally, 100 sec interval should have been OK.
+ */
+#define YT921X_STATS_INTERVAL_JIFFIES	(3 * HZ)
+
+struct yt921x_reg_mdio {
+	struct mii_bus *bus;
+	int addr;
+	/* SWITCH_ID_1 / SWITCH_ID_0 of the device
+	 *
+	 * This is a way to multiplex multiple devices on the same MII phyaddr
+	 * and should be configurable in DT. However, MDIO core simply doesn't
+	 * allow multiple devices over one reg addr, so this is a fixed value
+	 * for now until a solution is found.
+	 *
+	 * Keep this because we need switchid to form MII regaddrs anyway.
+	 */
+	unsigned char switchid;
+};
+
+/* TODO: SPI/I2C */
+
+#define to_yt921x_priv(_ds) container_of_const(_ds, struct yt921x_priv, ds)
+#define to_device(priv) ((priv)->ds.dev)
+
+static u32 ethaddr_hi4_to_u32(const unsigned char *addr)
+{
+	return (addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) | addr[3];
+}
+
+static u32 ethaddr_lo2_to_u32(const unsigned char *addr)
+{
+	return (addr[4] << 8) | addr[5];
+}
+
+static int yt921x_reg_mdio_read(void *context, u32 reg, u32 *valp)
+{
+	struct yt921x_reg_mdio *mdio = context;
+	struct mii_bus *bus = mdio->bus;
+	int addr = mdio->addr;
+	u32 reg_addr;
+	u32 reg_data;
+	u32 val;
+	int res;
+
+	/* Hold the mdio bus lock to avoid (un)locking for 4 times */
+	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+
+	reg_addr = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_ADDR |
+		   YT921X_SMI_READ;
+	res = __mdiobus_write(bus, addr, reg_addr, (u16)(reg >> 16));
+	if (res)
+		goto end;
+	res = __mdiobus_write(bus, addr, reg_addr, (u16)reg);
+	if (res)
+		goto end;
+
+	reg_data = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_DATA |
+		   YT921X_SMI_READ;
+	res = __mdiobus_read(bus, addr, reg_data);
+	if (res < 0)
+		goto end;
+	val = (u16)res;
+	res = __mdiobus_read(bus, addr, reg_data);
+	if (res < 0)
+		goto end;
+	val = (val << 16) | (u16)res;
+
+	*valp = val;
+	res = 0;
+
+end:
+	mutex_unlock(&bus->mdio_lock);
+	return res;
+}
+
+static int yt921x_reg_mdio_write(void *context, u32 reg, u32 val)
+{
+	struct yt921x_reg_mdio *mdio = context;
+	struct mii_bus *bus = mdio->bus;
+	int addr = mdio->addr;
+	u32 reg_addr;
+	u32 reg_data;
+	int res;
+
+	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
+
+	reg_addr = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_ADDR |
+		   YT921X_SMI_WRITE;
+	res = __mdiobus_write(bus, addr, reg_addr, (u16)(reg >> 16));
+	if (res)
+		goto end;
+	res = __mdiobus_write(bus, addr, reg_addr, (u16)reg);
+	if (res)
+		goto end;
+
+	reg_data = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_DATA |
+		   YT921X_SMI_WRITE;
+	res = __mdiobus_write(bus, addr, reg_data, (u16)(val >> 16));
+	if (res)
+		goto end;
+	res = __mdiobus_write(bus, addr, reg_data, (u16)val);
+	if (res)
+		goto end;
+
+	res = 0;
+
+end:
+	mutex_unlock(&bus->mdio_lock);
+	return res;
+}
+
+static const struct yt921x_reg_ops yt921x_reg_ops_mdio = {
+	.read = yt921x_reg_mdio_read,
+	.write = yt921x_reg_mdio_write,
+};
+
+/* TODO: SPI/I2C */
+
+/* Read and handle overflow of 32bit MIBs. MIB buffer must be zeroed before. */
+static int yt921x_read_mib(struct yt921x_priv *priv, int port)
+{
+	struct yt921x_port *pp = &priv->ports[port];
+	struct device *dev = to_device(priv);
+	struct yt921x_mib *mib = &pp->mib;
+	int res = 0;
+
+	/* Reading of yt921x_port::mib is not protected by a lock and it's vain
+	 * to keep its consistency, since we have to read registers one by one
+	 * and there is no way to make a snapshot of MIB stats.
+	 *
+	 * Writing (by this function only) is and should be protected by
+	 * reg_lock.
+	 */
+
+	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
+		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
+		u32 reg = YT921X_MIBn_DATA0(port) + desc->offset;
+		u64 *valp = &((u64 *)mib)[i];
+		u32 val0;
+		u64 val;
+
+		res = yt921x_reg_read(priv, reg, &val0);
+		if (res)
+			break;
+
+		if (desc->size <= 1) {
+			u64 old_val = *valp;
+
+			val = (old_val & ~(u64)U32_MAX) | val0;
+			if (val < old_val)
+				val += 1ull << 32;
+		} else {
+			u32 val1;
+
+			res = yt921x_reg_read(priv, reg + 4, &val1);
+			if (res)
+				break;
+			val = ((u64)val1 << 32) | val0;
+		}
+
+		WRITE_ONCE(*valp, val);
+	}
+
+	pp->rx_frames = mib->rx_64byte + mib->rx_65_127byte +
+			mib->rx_128_255byte + mib->rx_256_511byte +
+			mib->rx_512_1023byte + mib->rx_1024_1518byte +
+			mib->rx_jumbo;
+	pp->tx_frames = mib->tx_64byte + mib->tx_65_127byte +
+			mib->tx_128_255byte + mib->tx_256_511byte +
+			mib->tx_512_1023byte + mib->tx_1024_1518byte +
+			mib->tx_jumbo;
+
+	if (res)
+		dev_err(dev, "Failed to %s port %d: %i\n", "read stats for",
+			port, res);
+	return res;
+}
+
+static void yt921x_poll_mib(struct work_struct *work)
+{
+	struct yt921x_port *pp = container_of_const(work, struct yt921x_port,
+						    mib_read.work);
+	unsigned long delay = YT921X_STATS_INTERVAL_JIFFIES;
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	int port = pp->index;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_read_mib(priv, port);
+	mutex_unlock(&priv->reg_lock);
+	if (res)
+		delay *= 4;
+
+	schedule_delayed_work(&pp->mib_read, delay);
+}
+
+static void
+yt921x_dsa_get_strings(struct dsa_switch *ds, int port, u32 stringset,
+		       uint8_t *data)
+{
+	if (stringset != ETH_SS_STATS)
+		return;
+
+	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
+		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
+
+		if (desc->name)
+			ethtool_puts(&data, desc->name);
+	}
+}
+
+static void
+yt921x_dsa_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_port *pp = &priv->ports[port];
+	struct yt921x_mib *mib = &pp->mib;
+	size_t j;
+
+	mutex_lock(&priv->reg_lock);
+	yt921x_read_mib(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	j = 0;
+	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
+		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
+
+		if (!desc->name)
+			continue;
+
+		data[j] = ((u64 *)mib)[i];
+		j++;
+	}
+}
+
+static int yt921x_dsa_get_sset_count(struct dsa_switch *ds, int port, int sset)
+{
+	int cnt = 0;
+
+	if (sset != ETH_SS_STATS)
+		return 0;
+
+	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
+		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
+
+		if (desc->name)
+			cnt++;
+	}
+
+	return cnt;
+}
+
+static void
+yt921x_dsa_get_eth_mac_stats(struct dsa_switch *ds, int port,
+			     struct ethtool_eth_mac_stats *mac_stats)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_port *pp = &priv->ports[port];
+	struct yt921x_mib *mib = &pp->mib;
+
+	mutex_lock(&priv->reg_lock);
+	yt921x_read_mib(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	mac_stats->FramesTransmittedOK = pp->tx_frames;
+	mac_stats->SingleCollisionFrames = mib->tx_single_collisions;
+	mac_stats->MultipleCollisionFrames = mib->tx_multiple_collisions;
+	mac_stats->FramesReceivedOK = pp->rx_frames;
+	mac_stats->FrameCheckSequenceErrors = mib->rx_crc_errors;
+	mac_stats->AlignmentErrors = mib->rx_alignment_errors;
+	mac_stats->OctetsTransmittedOK = mib->tx_good_bytes;
+	mac_stats->FramesWithDeferredXmissions = mib->tx_deferred;
+	mac_stats->LateCollisions = mib->tx_late_collisions;
+	mac_stats->FramesAbortedDueToXSColls = mib->tx_aborted_errors;
+	/* mac_stats->FramesLostDueToIntMACXmitError */
+	/* mac_stats->CarrierSenseErrors */
+	mac_stats->OctetsReceivedOK = mib->rx_good_bytes;
+	/* mac_stats->FramesLostDueToIntMACRcvError */
+	mac_stats->MulticastFramesXmittedOK = mib->tx_multicast;
+	mac_stats->BroadcastFramesXmittedOK = mib->tx_broadcast;
+	/* mac_stats->FramesWithExcessiveDeferral */
+	mac_stats->MulticastFramesReceivedOK = mib->rx_multicast;
+	mac_stats->BroadcastFramesReceivedOK = mib->rx_broadcast;
+	/* mac_stats->InRangeLengthErrors */
+	/* mac_stats->OutOfRangeLengthField */
+	mac_stats->FrameTooLongErrors = mib->rx_oversize_errors;
+}
+
+static void
+yt921x_dsa_get_eth_ctrl_stats(struct dsa_switch *ds, int port,
+			      struct ethtool_eth_ctrl_stats *ctrl_stats)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_port *pp = &priv->ports[port];
+	struct yt921x_mib *mib = &pp->mib;
+
+	mutex_lock(&priv->reg_lock);
+	yt921x_read_mib(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	ctrl_stats->MACControlFramesTransmitted = mib->tx_pause;
+	ctrl_stats->MACControlFramesReceived = mib->rx_pause;
+	/* ctrl_stats->UnsupportedOpcodesReceived */
+}
+
+static const struct ethtool_rmon_hist_range yt921x_rmon_ranges[] = {
+	{ 0, 64 },
+	{ 65, 127 },
+	{ 128, 255 },
+	{ 256, 511 },
+	{ 512, 1023 },
+	{ 1024, 1518 },
+	{ 1519, YT921X_FRAME_SIZE_MAX },
+	{}
+};
+
+static void
+yt921x_dsa_get_rmon_stats(struct dsa_switch *ds, int port,
+			  struct ethtool_rmon_stats *rmon_stats,
+			  const struct ethtool_rmon_hist_range **ranges)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_port *pp = &priv->ports[port];
+	struct yt921x_mib *mib = &pp->mib;
+
+	mutex_lock(&priv->reg_lock);
+	yt921x_read_mib(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	*ranges = yt921x_rmon_ranges;
+
+	rmon_stats->undersize_pkts = mib->rx_undersize_errors;
+	rmon_stats->oversize_pkts = mib->rx_oversize_errors;
+	rmon_stats->fragments = mib->rx_alignment_errors;
+	/* rmon_stats->jabbers */
+
+	rmon_stats->hist[0] = mib->rx_64byte;
+	rmon_stats->hist[1] = mib->rx_65_127byte;
+	rmon_stats->hist[2] = mib->rx_128_255byte;
+	rmon_stats->hist[3] = mib->rx_256_511byte;
+	rmon_stats->hist[4] = mib->rx_512_1023byte;
+	rmon_stats->hist[5] = mib->rx_1024_1518byte;
+	rmon_stats->hist[6] = mib->rx_jumbo;
+
+	rmon_stats->hist_tx[0] = mib->tx_64byte;
+	rmon_stats->hist_tx[1] = mib->tx_65_127byte;
+	rmon_stats->hist_tx[2] = mib->tx_128_255byte;
+	rmon_stats->hist_tx[3] = mib->tx_256_511byte;
+	rmon_stats->hist_tx[4] = mib->tx_512_1023byte;
+	rmon_stats->hist_tx[5] = mib->tx_1024_1518byte;
+	rmon_stats->hist_tx[6] = mib->tx_jumbo;
+}
+
+static void
+yt921x_dsa_get_stats64(struct dsa_switch *ds, int port,
+		       struct rtnl_link_stats64 *stats)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_port *pp = &priv->ports[port];
+	struct yt921x_mib *mib = &pp->mib;
+
+	stats->rx_length_errors = mib->rx_undersize_errors +
+				  mib->rx_fragment_errors;
+	stats->rx_over_errors = mib->rx_oversize_errors;
+	stats->rx_crc_errors = mib->rx_crc_errors;
+	stats->rx_frame_errors = mib->rx_alignment_errors;
+	/* stats->rx_fifo_errors */
+	/* stats->rx_missed_errors */
+
+	stats->tx_aborted_errors = mib->tx_aborted_errors;
+	/* stats->tx_carrier_errors */
+	stats->tx_fifo_errors = mib->tx_undersize_errors;
+	/* stats->tx_heartbeat_errors */
+	stats->tx_window_errors = mib->tx_late_collisions;
+
+	stats->rx_packets = pp->rx_frames;
+	stats->tx_packets = pp->tx_frames;
+	stats->rx_bytes = mib->rx_good_bytes - ETH_FCS_LEN * stats->rx_packets;
+	stats->tx_bytes = mib->tx_good_bytes - ETH_FCS_LEN * stats->tx_packets;
+	stats->rx_errors = stats->rx_length_errors + stats->rx_over_errors +
+			   stats->rx_crc_errors + stats->rx_frame_errors;
+	stats->tx_errors = stats->tx_aborted_errors + stats->tx_fifo_errors +
+			   stats->tx_window_errors;
+	stats->rx_dropped = mib->rx_dropped;
+	/* stats->tx_dropped */
+	stats->multicast = mib->rx_multicast;
+	stats->collisions = mib->tx_collisions;
+}
+
+static void
+yt921x_dsa_get_pause_stats(struct dsa_switch *ds, int port,
+			   struct ethtool_pause_stats *pause_stats)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_port *pp = &priv->ports[port];
+	struct yt921x_mib *mib = &pp->mib;
+
+	mutex_lock(&priv->reg_lock);
+	yt921x_read_mib(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	pause_stats->tx_pause_frames = mib->tx_pause;
+	pause_stats->rx_pause_frames = mib->rx_pause;
+}
+
+static int
+yt921x_set_eee(struct yt921x_priv *priv, int port, struct ethtool_keee *e)
+{
+	/* Poor datasheet for EEE operations; don't ask if you are confused */
+
+	bool enable = e->eee_enabled;
+	u16 new_mask;
+	int res;
+
+	/* Enable / disable global EEE */
+	new_mask = priv->eee_ports_mask;
+	new_mask &= ~BIT(port);
+	new_mask |= !enable ? 0 : BIT(port);
+
+	if (!!new_mask != !!priv->eee_ports_mask) {
+		res = yt921x_reg_toggle_bits(priv, YT921X_PON_STRAP_FUNC,
+					     YT921X_PON_STRAP_EEE, !!new_mask);
+		if (res)
+			return res;
+		res = yt921x_reg_toggle_bits(priv, YT921X_PON_STRAP_VAL,
+					     YT921X_PON_STRAP_EEE, !!new_mask);
+		if (res)
+			return res;
+	}
+
+	priv->eee_ports_mask = new_mask;
+
+	/* Enable / disable port EEE */
+	res = yt921x_reg_toggle_bits(priv, YT921X_EEE_CTRL,
+				     YT921X_EEE_CTRL_ENn(port), enable);
+	if (res)
+		return res;
+	res = yt921x_reg_toggle_bits(priv, YT921X_EEEn_VAL(port),
+				     YT921X_EEE_VAL_DATA, enable);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int
+yt921x_dsa_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_set_eee(priv, port, e);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int yt921x_mtu_fetch(struct yt921x_priv *priv, int port)
+{
+	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
+
+	return dp->user ? READ_ONCE(dp->user->mtu) : ETH_DATA_LEN;
+}
+
+static int
+yt921x_dsa_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
+{
+	/* Only serves as packet filter, since the frame size is always set to
+	 * maximum after reset
+	 */
+
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct dsa_port *dp = dsa_to_port(ds, port);
+	int frame_size;
+	int res;
+
+	frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;
+	if (dsa_port_is_cpu(dp))
+		frame_size += YT921X_TAG_LEN;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_update_bits(priv, YT921X_MACn_FRAME(port),
+				     YT921X_MAC_FRAME_SIZE_M,
+				     YT921X_MAC_FRAME_SIZE(frame_size));
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int yt921x_dsa_port_max_mtu(struct dsa_switch *ds, int port)
+{
+	/* Only called for user ports, exclude tag len here */
+	return YT921X_FRAME_SIZE_MAX - ETH_HLEN - ETH_FCS_LEN - YT921X_TAG_LEN;
+}
+
+/* v * 2^e */
+static u64 ldexpu64(u64 v, int e)
+{
+	return e >= 0 ? v << e : v >> -e;
+}
+
+/* slot (ns) * rate (/s) / 10^9 (ns/s) = 2^C * token * 4^unit */
+static u32 rate2token(u64 rate, unsigned int slot_ns, int unit, int C)
+{
+	int e = 2 * unit + C + YT921X_TOKEN_RATE_C;
+
+	return div_u64(ldexpu64(slot_ns * rate, -e), 1000000000);
+}
+
+static u64 token2rate(u32 token, unsigned int slot_ns, int unit, int C)
+{
+	int e = 2 * unit + C + YT921X_TOKEN_RATE_C;
+
+	return div_u64(ldexpu64(mul_u32_u32(1000000000, token), e), slot_ns);
+}
+
+/* burst = 2^C * token * 4^unit */
+static u32 burst2token(u64 burst, int unit, int C)
+{
+	return ldexpu64(burst, -(2 * unit + C));
+}
+
+static u64 token2burst(u32 token, int unit, int C)
+{
+	return ldexpu64(token, 2 * unit + C);
+}
+
+struct yt921x_marker {
+	u32 cir;
+	u32 cbs;
+	u32 ebs;
+	int unit;
+	bool pkt_mode;
+};
+
+#define YT921X_MARKER_PKT_MODE		BIT(0)
+#define YT921X_MARKER_SINGLE_BUCKET	BIT(1)
+
+static int
+yt921x_marker_tfm(struct yt921x_marker *marker, u64 rate, u64 burst,
+		  unsigned int flags, unsigned int slot_ns, u32 cir_max,
+		  u32 cbs_max, int unit_max, struct yt921x_priv *priv, int port,
+		  struct netlink_ext_ack *extack)
+{
+	const int C = flags & YT921X_MARKER_PKT_MODE ? YT921X_TOKEN_PKT_C :
+		      YT921X_TOKEN_BYTE_C;
+	struct device *dev = to_device(priv);
+	struct yt921x_marker m;
+	u64 burst_est;
+	u64 burst_sug;
+	u64 burst_max;
+	u64 rate_max;
+
+	m.unit = unit_max;
+	rate_max = token2rate(cir_max, slot_ns, m.unit, C);
+	burst_max = token2burst(cbs_max, m.unit, C);
+
+	/* Check for unusual values */
+	if (rate > rate_max || burst > burst_max) {
+		NL_SET_ERR_MSG_MOD(extack, "Unexpected tremendous rate");
+		return -ERANGE;
+	}
+
+	/* Check for matching burst */
+	burst_est = div_u64(slot_ns * rate, 1000000000);
+	burst_sug = burst_est;
+	if (flags & YT921X_MARKER_PKT_MODE)
+		burst_sug++;
+	else
+		burst_sug += ETH_HLEN + yt921x_mtu_fetch(priv, port) +
+			     ETH_FCS_LEN;
+	if (burst_sug > burst)
+		NL_SET_ERR_MSG_FMT_MOD(extack,
+				       "Consider match rate %llu with burst at least %llu",
+				       rate, burst_sug);
+
+	/* Select unit */
+	for (; m.unit > 0; m.unit--) {
+		if (rate > (rate_max >> 2) || burst > (burst_max >> 2))
+			break;
+		rate_max >>= 2;
+		burst_max >>= 2;
+	}
+
+	/* Calculate information rate and bucket size */
+	m.cir = rate2token(rate, slot_ns, m.unit, C);
+	if (!m.cir)
+		m.cir = 1;
+	else if (WARN_ON(m.cir > cir_max))
+		m.cir = cir_max;
+	m.cbs = burst2token(burst, m.unit, C);
+	if (!m.cbs)
+		m.cbs = 1;
+	else if (WARN_ON(m.cbs > cbs_max))
+		m.cbs = cbs_max;
+
+	/* Cut EBS */
+	m.ebs = 0;
+	if (!(flags & YT921X_MARKER_SINGLE_BUCKET)) {
+		/* We don't have a chance to adjust rate when MTU is changed */
+		if (flags & YT921X_MARKER_PKT_MODE)
+			burst_est++;
+		else
+			burst_est += YT921X_FRAME_SIZE_MAX;
+
+		if (burst_est < burst) {
+			u32 pbs = m.cbs;
+
+			m.cbs = burst2token(burst_est, m.unit, C);
+			if (!m.cbs)
+				m.cbs = 1;
+			else if (WARN_ON(m.cbs > cbs_max))
+				m.cbs = cbs_max;
+
+			if (pbs > m.cbs)
+				m.ebs = pbs - m.cbs;
+		}
+	}
+
+	dev_dbg(dev,
+		"slot %u ns, rate %llu, burst %llu -> unit %d, cir %u, cbs %u, ebs %u\n",
+		slot_ns, rate, burst, m.unit, m.cir, m.cbs, m.ebs);
+
+	m.pkt_mode = flags & YT921X_MARKER_PKT_MODE;
+	*marker = m;
+	return 0;
+}
+
+static int
+yt921x_marker_tfm_police(struct yt921x_marker *marker,
+			 const struct flow_action_police *police,
+			 unsigned int flags, struct yt921x_priv *priv, int port,
+			 struct netlink_ext_ack *extack)
+{
+	bool pkt_mode = !!police->rate_pkt_ps;
+	u64 burst;
+	u64 rate;
+
+	rate = pkt_mode ? police->rate_pkt_ps : police->rate_bytes_ps;
+	burst = pkt_mode ? police->burst_pkt : police->burst;
+	if (pkt_mode)
+		flags |= YT921X_MARKER_PKT_MODE;
+
+	return yt921x_marker_tfm(marker, rate, burst, flags,
+				 priv->meter_slot_ns, YT921X_METER_CIR_MAX,
+				 YT921X_METER_CBS_MAX, YT921X_METER_UNIT_MAX,
+				 priv, port, extack);
+}
+
+static int
+yt921x_marker_tfm_shape(struct yt921x_marker *marker, u64 rate, u64 burst,
+			unsigned int flags, struct yt921x_priv *priv, int port,
+			struct netlink_ext_ack *extack)
+{
+	return yt921x_marker_tfm(marker, rate, burst, flags,
+				 priv->port_shape_slot_ns, YT921X_SHAPE_CIR_MAX,
+				 YT921X_SHAPE_CBS_MAX, YT921X_SHAPE_UNIT_MAX,
+				 priv, port, extack);
+}
+
+static int
+yt921x_police_validate(const struct flow_action_police *police,
+		       const struct flow_action *action,
+		       const struct flow_action_entry *act,
+		       struct netlink_ext_ack *extack)
+{
+	if (police->exceed.act_id != FLOW_ACTION_DROP) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when exceed action is not drop");
+		return -EOPNOTSUPP;
+	}
+
+	if (police->notexceed.act_id != FLOW_ACTION_PIPE &&
+	    police->notexceed.act_id != FLOW_ACTION_ACCEPT) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when conform action is not pipe or ok");
+		return -EOPNOTSUPP;
+	}
+
+	if (police->notexceed.act_id == FLOW_ACTION_ACCEPT && action && act &&
+	    !flow_action_is_last_entry(action, act)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when conform action is ok, but action is not last");
+		return -EOPNOTSUPP;
+	}
+
+	/* mtu defaults to unlimited but we got 2040 here, don't know why */
+	if (police->peakrate_bytes_ps || police->avrate || police->overhead) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when peakrate/avrate/overhead is configured");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_meter_config(struct yt921x_priv *priv, unsigned int id,
+		    const struct yt921x_marker *marker)
+{
+	u32 ctrls[3];
+
+	ctrls[0] = 0;
+	ctrls[1] = YT921X_METER_CTRLb_CIR(marker->cir);
+	ctrls[2] = YT921X_METER_CTRLc_UNIT(marker->unit) |
+		   YT921X_METER_CTRLc_DROP_R |
+		   YT921X_METER_CTRLc_TOKEN_OVERFLOW_EN |
+		   YT921X_METER_CTRLc_METER_EN;
+	if (marker->pkt_mode)
+		ctrls[2] |= YT921X_METER_CTRLc_PKT_MODE;
+	update_ctrls_unaligned(&ctrls[0], &ctrls[1],
+			       YT921X_METER_CTRLab_EBS_M,
+			       YT921X_METER_CTRLab_EBS(marker->ebs));
+	update_ctrls_unaligned(&ctrls[1], &ctrls[2],
+			       YT921X_METER_CTRLbc_CBS_M,
+			       YT921X_METER_CTRLbc_CBS(marker->cbs));
+
+	return yt921x_reg96_write(priv, YT921X_METERn_CTRL(id), ctrls);
+}
+
+static void yt921x_dsa_port_policer_del(struct dsa_switch *ds, int port)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct device *dev = to_device(priv);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_write(priv, YT921X_PORTn_METER(port), 0);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s port %d: %i\n", "delete policer on",
+			port, res);
+}
+
+static int
+yt921x_dsa_port_policer_add(struct dsa_switch *ds, int port,
+			    const struct flow_action_police *police,
+			    struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_marker marker;
+	u32 ctrl;
+	int res;
+
+	res = yt921x_police_validate(police, NULL, NULL, extack);
+	if (res)
+		return res;
+
+	res = yt921x_marker_tfm_police(&marker, police, 0, priv, port, extack);
+	if (res)
+		return res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_meter_config(priv, port + YT921X_METER_NUM, &marker);
+	if (res)
+		goto end;
+
+	ctrl = YT921X_PORT_METER_ID(port) | YT921X_PORT_METER_EN;
+	res = yt921x_reg_write(priv, YT921X_PORTn_METER(port), ctrl);
+end:
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_setup_tc_tbf_port(struct dsa_switch *ds, int port,
+				  const struct tc_tbf_qopt_offload *qopt)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct netlink_ext_ack *extack = qopt->extack;
+	u32 ctrls[2];
+	int res;
+
+	if (qopt->parent != TC_H_ROOT)
+		return -EOPNOTSUPP;
+
+	switch (qopt->command) {
+	case TC_TBF_STATS:
+		/* Unfortunately the convention for TC_*_STATS is a mess,
+		 * neither 0 nor -EOPNOTSUPP is perfect here.
+		 */
+		return -EOPNOTSUPP;
+	case TC_TBF_DESTROY:
+		ctrls[0] = 0;
+		ctrls[1] = 0;
+		break;
+	case TC_TBF_REPLACE: {
+		const struct tc_tbf_qopt_offload_replace_params *p;
+		struct yt921x_marker marker;
+
+		p = &qopt->replace_params;
+
+		res = yt921x_marker_tfm_shape(&marker, p->rate.rate_bytes_ps,
+					      p->max_size,
+					      YT921X_MARKER_SINGLE_BUCKET,
+					      priv, port, extack);
+		if (res)
+			return res;
+
+		ctrls[0] = YT921X_PORT_SHAPE_CTRLa_CIR(marker.cir) |
+			   YT921X_PORT_SHAPE_CTRLa_CBS(marker.cbs);
+		ctrls[1] = YT921X_PORT_SHAPE_CTRLb_UNIT(marker.unit) |
+			   YT921X_PORT_SHAPE_CTRLb_EN;
+		break;
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg64_write(priv, YT921X_PORTn_SHAPE_CTRL(port), ctrls);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_setup_tc(struct dsa_switch *ds, int port,
+			 enum tc_setup_type type, void *type_data)
+{
+	switch (type) {
+	case TC_SETUP_QDISC_TBF: {
+		const struct tc_tbf_qopt_offload *qopt = type_data;
+
+		return yt921x_dsa_port_setup_tc_tbf_port(ds, port, qopt);
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+/* ACL: 48 blocks * 8 entries
+ *
+ * One rule can span multiple entries, but within a block.
+ */
+
+static void
+yt921x_acl_entry_set(struct yt921x_acl_entry *entry, unsigned int offset,
+		     u32 flags, bool set)
+{
+	if (set)
+		entry->key[offset] |= flags;
+	entry->mask[offset] |= flags;
+}
+
+static unsigned int
+yt921x_acl_entries_set_is_fragment(struct yt921x_acl_entry *entries,
+				   unsigned int size, bool set)
+{
+	for (unsigned int i = 0; i < size; i++)
+		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
+		case YT921X_ACL_TYPE_IPV4_DA:
+		case YT921X_ACL_TYPE_IPV4_SA:
+			yt921x_acl_entry_set(&entries[i], 1,
+					     YT921X_ACL_BINb_IPV4_FRAG, set);
+			return size;
+		case YT921X_ACL_TYPE_IPV6_DA3:
+		case YT921X_ACL_TYPE_IPV6_SA3:
+			yt921x_acl_entry_set(&entries[i], 1,
+					     YT921X_ACL_BINb_IPV6_xA3_FRAG,
+					     set);
+			return size;
+		case YT921X_ACL_TYPE_MISC:
+			yt921x_acl_entry_set(&entries[i], 1,
+					     YT921X_ACL_BINb_MISC_FRAG, set);
+			return size;
+		case YT921X_ACL_TYPE_L4:
+			yt921x_acl_entry_set(&entries[i], 1,
+					     YT921X_ACL_BINb_L4_FRAG, set);
+			return size;
+		}
+
+	if (size >= YT921X_ACL_ENT_PER_BLK)
+		return 0;
+
+	entries[size] = (typeof(*entries)){};
+	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
+	yt921x_acl_entry_set(&entries[size], 1, YT921X_ACL_BINb_MISC_FRAG, set);
+
+	return size + 1;
+}
+
+static unsigned int
+yt921x_acl_entries_set_first_frag(struct yt921x_acl_entry *entries,
+				  unsigned int size, bool set)
+{
+	for (unsigned int i = 0; i < size; i++)
+		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
+		case YT921X_ACL_TYPE_IPV6_DA2:
+		case YT921X_ACL_TYPE_IPV6_SA2:
+			yt921x_acl_entry_set(&entries[i], 1,
+					     YT921X_ACL_BINb_IPV6_xA2_FIRST_FRAG,
+					     set);
+			return size;
+		case YT921X_ACL_TYPE_MISC:
+			yt921x_acl_entry_set(&entries[i], 0,
+					     YT921X_ACL_BINa_MISC_FIRST_FRAG,
+					     set);
+			return size;
+		}
+
+	if (size >= YT921X_ACL_ENT_PER_BLK)
+		return 0;
+
+	entries[size] = (typeof(*entries)){};
+	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
+	yt921x_acl_entry_set(&entries[size], 0,
+			     YT921X_ACL_BINa_MISC_FIRST_FRAG, set);
+
+	return size + 1;
+}
+
+static unsigned int
+yt921x_acl_entries_set_l3_type(struct yt921x_acl_entry *entries,
+			       unsigned int size, enum yt921x_l3_type type)
+{
+	for (unsigned int i = 0; i < size; i++)
+		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
+		case YT921X_ACL_TYPE_MAC_DA0:
+		case YT921X_ACL_TYPE_MAC_SA0:
+			entries[i].key[1] |= YT921X_ACL_BINb_MAC_xA0_L3_TYPE(type);
+			entries[i].mask[1] |= YT921X_ACL_BINb_MAC_xA0_L3_TYPE_M;
+			return size;
+		case YT921X_ACL_TYPE_MISC:
+			entries[i].key[0] |= YT921X_ACL_BINa_MISC_L3_TYPE(type);
+			entries[i].mask[0] |= YT921X_ACL_BINa_MISC_L3_TYPE_M;
+			return size;
+		}
+
+	if (size >= YT921X_ACL_ENT_PER_BLK)
+		return 0;
+
+	entries[size] = (typeof(*entries)){};
+	entries[size].key[0] = YT921X_ACL_BINa_MISC_L3_TYPE(type);
+	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
+	entries[size].mask[0] = YT921X_ACL_BINa_MISC_L3_TYPE_M;
+
+	return size + 1;
+}
+
+static unsigned int
+yt921x_acl_entries_set_l4_type(struct yt921x_acl_entry *entries,
+			       unsigned int size, enum yt921x_l4_type type)
+{
+	for (unsigned int i = 0; i < size; i++)
+		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
+		case YT921X_ACL_TYPE_IPV4_DA:
+		case YT921X_ACL_TYPE_IPV4_SA:
+			entries[i].key[1] |= YT921X_ACL_BINb_IPV4_L4_TYPE(type);
+			entries[i].mask[1] |= YT921X_ACL_BINb_IPV4_L4_TYPE_M;
+			return size;
+		case YT921X_ACL_TYPE_IPV6_DA0:
+		case YT921X_ACL_TYPE_IPV6_DA1:
+		case YT921X_ACL_TYPE_IPV6_DA2:
+		case YT921X_ACL_TYPE_IPV6_DA3:
+		case YT921X_ACL_TYPE_IPV6_SA0:
+		case YT921X_ACL_TYPE_IPV6_SA1:
+		case YT921X_ACL_TYPE_IPV6_SA2:
+		case YT921X_ACL_TYPE_IPV6_SA3:
+			entries[i].key[1] |= YT921X_ACL_BINb_IPV6_L4_TYPE(type);
+			entries[i].mask[1] |= YT921X_ACL_BINb_IPV6_L4_TYPE_M;
+			return size;
+		case YT921X_ACL_TYPE_L4:
+			entries[i].key[1] |= YT921X_ACL_BINb_L4_TYPE(type);
+			entries[i].mask[1] |= YT921X_ACL_BINb_L4_TYPE_M;
+			return size;
+		case YT921X_ACL_TYPE_MISC:
+			entries[i].key[1] |= YT921X_ACL_BINb_MISC_L4_TYPE(type);
+			entries[i].mask[1] |= YT921X_ACL_BINb_MISC_L4_TYPE_M;
+			return size;
+		}
+
+	if (size >= YT921X_ACL_ENT_PER_BLK)
+		return 0;
+
+	entries[size] = (typeof(*entries)){};
+	entries[size].key[1] = YT921X_ACL_BINb_MISC_L4_TYPE(type) |
+			       YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
+	entries[size].mask[1] = YT921X_ACL_BINb_MISC_L4_TYPE_M;
+
+	return size + 1;
+}
+
+static struct yt921x_acl_entry *
+yt921x_acl_entries_new(struct yt921x_acl_entry *entries, unsigned int *sizep,
+		       u32 type)
+{
+	unsigned int size = *sizep;
+
+	if (size >= YT921X_ACL_ENT_PER_BLK)
+		return NULL;
+
+	entries[size] = (typeof(*entries)){};
+	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(type);
+
+	(*sizep)++;
+	return &entries[size];
+}
+
+static struct yt921x_acl_entry *
+yt921x_acl_entries_find(struct yt921x_acl_entry *entries, unsigned int *sizep,
+			u32 type)
+{
+	for (unsigned int i = 0; i < *sizep; i++)
+		if (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1]) ==
+		    type)
+			return &entries[i];
+	return yt921x_acl_entries_new(entries, sizep, type);
+}
+
+static void
+yt921x_acl_rule_set_ports(struct yt921x_acl_rule *aclrule, u16 ord,
+			  u16 ports_mask)
+{
+	struct yt921x_acl_entry *entries = aclrule->entries;
+
+	for (unsigned int i = 0; i < hweight8(aclrule->mask); i++) {
+		entries[i].key[1] |= YT921X_ACL_KEYb_SPORTS(ports_mask) |
+				     YT921X_ACL_KEYb_ORD(ord);
+	}
+}
+
+struct yt921x_acl_rule_ext {
+	struct yt921x_acl_rule r;
+
+	struct yt921x_marker marker;
+};
+
+static int
+yt921x_acl_rule_ext_parse_flow_entries(struct yt921x_acl_rule_ext *ruleext,
+				       const struct flow_cls_offload *cls)
+{
+	const struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	struct yt921x_acl_entry *entries = ruleext->r.entries;
+	struct netlink_ext_ack *extack = cls->common.extack;
+	const struct flow_dissector *dissector;
+	struct yt921x_acl_entry *entry;
+	unsigned int size = 0;
+	bool use_dport;
+	bool use_sport;
+
+	/* Incomplete and probably won't, since it supports custom u32 filters.
+	 * New adapters are welcome.
+	 */
+	dissector = rule->match.dissector;
+	if (dissector->used_keys &
+	    ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS_RANGE) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_IP) |
+	      BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
+		NL_SET_ERR_MSG_MOD(extack, "Unsupported keys used");
+		return -EOPNOTSUPP;
+	}
+
+	/* Entries */
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
+		struct flow_match_ipv4_addrs match;
+
+		flow_rule_match_ipv4_addrs(rule, &match);
+
+		if (match.mask->dst) {
+			entry = yt921x_acl_entries_new(entries, &size,
+						       YT921X_ACL_TYPE_IPV4_DA);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= ntohl(match.key->dst);
+			entry->mask[0] |= ntohl(match.mask->dst);
+		}
+
+		if (match.mask->src) {
+			entry = yt921x_acl_entries_new(entries, &size,
+						       YT921X_ACL_TYPE_IPV4_SA);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= ntohl(match.key->src);
+			entry->mask[0] |= ntohl(match.mask->src);
+		}
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
+		struct flow_match_ipv6_addrs match;
+
+		flow_rule_match_ipv6_addrs(rule, &match);
+
+		for (unsigned int i = 0; i < 4; i++) {
+			if (!match.mask->dst.s6_addr32[i])
+				continue;
+
+			entry = yt921x_acl_entries_new(entries, &size,
+						       YT921X_ACL_TYPE_IPV6_DA0 + i);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= ntohl(match.key->dst.s6_addr32[i]);
+			entry->mask[0] |= ntohl(match.mask->dst.s6_addr32[i]);
+		}
+
+		for (unsigned int i = 0; i < 4; i++) {
+			if (!match.mask->src.s6_addr32[i])
+				continue;
+
+			entry = yt921x_acl_entries_new(entries, &size,
+						       YT921X_ACL_TYPE_IPV6_SA0 + i);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= ntohl(match.key->src.s6_addr32[i]);
+			entry->mask[0] |= ntohl(match.mask->src.s6_addr32[i]);
+		}
+	}
+
+	use_dport = false;
+	use_sport = false;
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
+		struct flow_match_ports match;
+
+		entry = yt921x_acl_entries_new(entries, &size,
+					       YT921X_ACL_TYPE_L4);
+		if (!entry)
+			goto err;
+
+		flow_rule_match_ports(rule, &match);
+
+		use_dport = !!match.mask->dst;
+		use_sport = !!match.mask->src;
+
+		entry->key[0] |= (ntohs(match.key->dst) << 16) |
+				 ntohs(match.key->src);
+		entry->mask[0] |= (ntohs(match.mask->dst) << 16) |
+				  ntohs(match.mask->src);
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS_RANGE)) {
+		struct flow_match_ports_range match;
+
+		entry = yt921x_acl_entries_find(entries, &size,
+						YT921X_ACL_TYPE_L4);
+		if (!entry)
+			goto err;
+
+		flow_rule_match_ports_range(rule, &match);
+
+		if ((use_dport && match.mask->tp.dst) ||
+		    (use_sport && match.mask->tp.src)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Port mask and range are mutually exclusive");
+			return -EINVAL;
+		}
+
+		if (match.mask->tp.dst) {
+			entry->key[0] |= ntohs(match.key->tp_min.dst) << 16;
+			entry->key[1] |= YT921X_ACL_KEYb_L4_DPORT_RANGE_EN;
+			entry->mask[0] |= ntohs(match.key->tp_max.dst) << 16;
+		}
+
+		if (match.mask->tp.src) {
+			entry->key[0] |= ntohs(match.key->tp_min.src);
+			entry->key[1] |= YT921X_ACL_KEYb_L4_SPORT_RANGE_EN;
+			entry->mask[0] |= ntohs(match.key->tp_max.src);
+		}
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
+		struct flow_match_eth_addrs match;
+		u32 mask;
+
+		flow_rule_match_eth_addrs(rule, &match);
+
+		mask = ethaddr_hi4_to_u32(match.mask->dst);
+		if (mask) {
+			entry = yt921x_acl_entries_new(entries, &size,
+						       YT921X_ACL_TYPE_MAC_DA0);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= ethaddr_hi4_to_u32(match.key->dst);
+			entry->mask[0] |= mask;
+		}
+
+		mask = ethaddr_hi4_to_u32(match.mask->src);
+		if (mask) {
+			entry = yt921x_acl_entries_new(entries, &size,
+						       YT921X_ACL_TYPE_MAC_SA0);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= ethaddr_hi4_to_u32(match.key->src);
+			entry->mask[0] |= mask;
+		}
+
+		mask = (ethaddr_lo2_to_u32(match.mask->dst) << 16) |
+		       ethaddr_lo2_to_u32(match.mask->src);
+		if (mask) {
+			entry = yt921x_acl_entries_new(entries, &size,
+						       YT921X_ACL_TYPE_MAC_DA1_SA1);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= (ethaddr_lo2_to_u32(match.key->dst) << 16) |
+					 ethaddr_lo2_to_u32(match.key->src);
+			entry->mask[0] |= mask;
+		}
+	}
+
+	/* Entries + Misc */
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
+		struct flow_match_basic match;
+
+		flow_rule_match_basic(rule, &match);
+
+		if (match.mask->n_proto) {
+			enum yt921x_l3_type l3type = YT921X_L3_TYPE_OTHER;
+
+			if (match.mask->n_proto == htons(~0))
+				switch (match.key->n_proto) {
+				case htons(ETH_P_IP):
+					l3type = YT921X_L3_TYPE_IPV4;
+					break;
+				case htons(ETH_P_IPV6):
+					l3type = YT921X_L3_TYPE_IPV6;
+					break;
+				case htons(ETH_P_ARP):
+					l3type = YT921X_L3_TYPE_ARP;
+					break;
+				case htons(ETH_P_LLDP):
+					l3type = YT921X_L3_TYPE_LLDP;
+					break;
+				case htons(ETH_P_PAE):
+					l3type = YT921X_L3_TYPE_PAE;
+					break;
+				case htons(ETH_P_CFM):
+					l3type = YT921X_L3_TYPE_ERP;
+					break;
+				}
+
+			if (l3type != YT921X_L3_TYPE_OTHER) {
+				size = yt921x_acl_entries_set_l3_type(entries,
+								      size,
+								      l3type);
+				if (!size)
+					goto err;
+			} else {
+				entry = yt921x_acl_entries_new(entries, &size,
+							       YT921X_ACL_TYPE_ETHERTYPE);
+				if (!entry)
+					goto err;
+
+				entry->key[0] |= ntohs(match.key->n_proto);
+				entry->mask[0] |= ntohs(match.mask->n_proto);
+			}
+		}
+
+		if (match.mask->ip_proto) {
+			enum yt921x_l4_type l4type = YT921X_L4_TYPE_OTHER;
+
+			if (match.mask->ip_proto == (u8)~0)
+				switch (match.key->ip_proto) {
+				case IPPROTO_TCP:
+					l4type = YT921X_L4_TYPE_TCP;
+					break;
+				case IPPROTO_UDP:
+					l4type = YT921X_L4_TYPE_UDP;
+					break;
+				case IPPROTO_UDPLITE:
+					l4type = YT921X_L4_TYPE_UDPLITE;
+					break;
+				case IPPROTO_ICMP:
+					l4type = YT921X_L4_TYPE_ICMP;
+					break;
+				case IPPROTO_IGMP:
+					l4type = YT921X_L4_TYPE_IGMP;
+					break;
+				}
+
+			if (l4type != YT921X_L4_TYPE_OTHER) {
+				size = yt921x_acl_entries_set_l4_type(entries,
+								      size,
+								      l4type);
+				if (!size)
+					goto err;
+			} else {
+				entry = yt921x_acl_entries_find(entries, &size,
+								YT921X_ACL_TYPE_MISC);
+				if (!entry)
+					goto err;
+
+				entry->key[0] |= YT921X_ACL_BINa_MISC_IP_PROTO(match.key->ip_proto);
+				entry->mask[0] |= YT921X_ACL_BINa_MISC_IP_PROTO(match.mask->ip_proto);
+			}
+		}
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
+		u32 supp_flags = FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG;
+		struct flow_match_control match;
+
+		flow_rule_match_control(rule, &match);
+		if (!flow_rule_is_supp_control_flags(supp_flags,
+						     match.mask->flags, extack))
+			return -EOPNOTSUPP;
+
+		if (match.mask->flags & FLOW_DIS_IS_FRAGMENT) {
+			bool set = match.key->flags & FLOW_DIS_IS_FRAGMENT;
+
+			size = yt921x_acl_entries_set_is_fragment(entries, size,
+								  set);
+			if (!size)
+				goto err;
+		}
+		if (match.mask->flags & FLOW_DIS_FIRST_FRAG) {
+			bool set = match.key->flags & FLOW_DIS_FIRST_FRAG;
+
+			size = yt921x_acl_entries_set_first_frag(entries, size,
+								 set);
+			if (!size)
+				goto err;
+		}
+	}
+
+	/* Misc only */
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
+		struct flow_match_ip match;
+
+		flow_rule_match_ip(rule, &match);
+		if (match.mask->ttl) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Matching on TTL not supported");
+			return -EOPNOTSUPP;
+		}
+
+		if (match.mask->tos) {
+			entry = yt921x_acl_entries_find(entries, &size,
+							YT921X_ACL_TYPE_MISC);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= YT921X_ACL_BINa_MISC_TOS(match.key->tos);
+			entry->mask[0] |= YT921X_ACL_BINa_MISC_TOS(match.mask->tos);
+		}
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
+		struct flow_match_tcp match;
+
+		flow_rule_match_tcp(rule, &match);
+		if (match.mask->flags & htons(~0xff)) {
+			NL_SET_ERR_MSG_MOD(extack, "Unsupported TCP flags");
+			return -EOPNOTSUPP;
+		}
+
+		if (match.mask->flags) {
+			entry = yt921x_acl_entries_find(entries, &size,
+							YT921X_ACL_TYPE_MISC);
+			if (!entry)
+				goto err;
+
+			entry->key[0] |= YT921X_ACL_BINa_MISC_TCP_FLAGS(ntohs(match.key->flags));
+			entry->mask[0] |= YT921X_ACL_BINa_MISC_TCP_FLAGS(ntohs(match.mask->flags));
+		}
+	}
+
+	if (!size) {
+		NL_SET_ERR_MSG_MOD(extack, "Empty rule generated, this should not happen");
+		return -EOPNOTSUPP;
+	}
+
+	ruleext->r.mask = (1 << size) - 1;
+	return 0;
+
+err:
+	NL_SET_ERR_MSG_MOD(extack, "Rule too complex");
+	return -EOPNOTSUPP;
+}
+
+static int
+yt921x_acl_rule_ext_parse_flow_action(struct yt921x_acl_rule_ext *ruleext,
+				      const struct flow_cls_offload *cls,
+				      struct yt921x_priv *priv, int port)
+{
+	const struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
+	const struct flow_action *flow_action = &rule->action;
+	struct netlink_ext_ack *extack = cls->common.extack;
+	enum flow_action_id redir_act = NUM_FLOW_ACTIONS;
+	const struct flow_action_entry *act;
+	u32 *action = ruleext->r.action;
+	bool seen_priority = false;
+	const char *reason = NULL;
+	bool seen_police = false;
+	unsigned int i;
+	int res;
+
+	memset(action, 0, 3 * sizeof(*action));
+	flow_action_for_each(i, act, flow_action)
+		switch (act->id) {
+		case FLOW_ACTION_ACCEPT:
+		case FLOW_ACTION_DROP:
+		case FLOW_ACTION_REDIRECT:
+			if (redir_act != NUM_FLOW_ACTIONS &&
+			    redir_act != act->id) {
+				reason = "Different redirect actions";
+				goto fallback;
+			}
+			redir_act = act->id;
+
+			switch (act->id) {
+			case FLOW_ACTION_ACCEPT:
+				action[2] |= YT921X_ACL_ACTc_FWD_EN |
+					     YT921X_ACL_ACTc_FWD_FWD;
+				break;
+			case FLOW_ACTION_DROP:
+				action[2] |= YT921X_ACL_ACTc_FWD_EN |
+					     YT921X_ACL_ACTc_FWD_REDIR;
+				break;
+			case FLOW_ACTION_REDIRECT: {
+				struct dsa_port *to_dp;
+
+				to_dp = dsa_port_from_netdev(act->dev);
+				if (IS_ERR(to_dp) || to_dp->ds != &priv->ds) {
+					reason = "Redirect to non-local port";
+					goto fallback;
+				}
+
+				action[2] |= YT921X_ACL_ACTc_FWD_EN |
+					     YT921X_ACL_ACTc_FWD_REDIR |
+					     YT921X_ACL_ACTc_FWD_REDIR_DPORTn(to_dp->index);
+				break;
+			}
+			default:
+				break;
+			}
+			break;
+		case FLOW_ACTION_PRIORITY:
+			if (seen_priority) {
+				action[0] &= ~YT921X_ACL_ACTa_PRIO_EN;
+				action[1] &= ~YT921X_ACL_ACTb_PRIO_M;
+
+				reason = "Multiple priority actions";
+				goto fallback;
+			}
+			seen_priority = true;
+
+			if (act->priority >= YT921X_PRIO_NUM) {
+				NL_SET_ERR_MSG_MOD(extack,
+						   "Priority value is too high");
+				return -EOPNOTSUPP;
+			}
+			action[0] |= YT921X_ACL_ACTa_PRIO_EN;
+			action[1] |= YT921X_ACL_ACTb_PRIO(act->priority);
+			break;
+		case FLOW_ACTION_POLICE: {
+			const struct flow_action_police *police = &act->police;
+
+			if (seen_police) {
+				action[0] &= ~YT921X_ACL_ACTa_METER_EN;
+
+				reason = "Multiple police actions";
+				goto fallback;
+			}
+			seen_police = true;
+
+			res = yt921x_police_validate(police, flow_action, act,
+						     extack);
+			if (res)
+				return res;
+
+			res = yt921x_marker_tfm_police(&ruleext->marker, police,
+						       0, priv, port, extack);
+			if (res)
+				return res;
+
+			action[0] |= YT921X_ACL_ACTa_METER_EN;
+			break;
+		}
+		default:
+fallback:
+			if (cls->common.skip_sw) {
+				NL_SET_ERR_MSG_FMT_MOD(extack,
+						       "Action not supported when skip_sw: %s",
+						       reason);
+				return -EOPNOTSUPP;
+			}
+			fallthrough;
+		case FLOW_ACTION_TRAP:
+			redir_act = FLOW_ACTION_TRAP;
+
+			action[2] &= ~YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M &
+				     ~YT921X_ACL_ACTc_FWD_M;
+			action[2] |= YT921X_ACL_ACTc_FWD_EN |
+				     YT921X_ACL_ACTc_FWD_TRAP;
+			break;
+		}
+
+	ruleext->r.sw_assisted = !cls->common.skip_sw;
+	return 0;
+}
+
+static int
+yt921x_acl_rule_ext_parse_flow(struct yt921x_acl_rule_ext *ruleext, int port,
+			       const struct flow_cls_offload *cls, bool ingress,
+			       struct yt921x_priv *priv)
+{
+	struct netlink_ext_ack *extack = cls->common.extack;
+	int res;
+
+	if (!ingress) {
+		NL_SET_ERR_MSG_MOD(extack, "Only ingress is supported");
+		return -EOPNOTSUPP;
+	}
+
+	if (cls->common.chain_index) {
+		NL_SET_ERR_MSG(extack, "Only chain 0 is supported");
+		return -EOPNOTSUPP;
+	}
+
+	res = yt921x_acl_rule_ext_parse_flow_action(ruleext, cls, priv, port);
+	if (res)
+		return res;
+	res = yt921x_acl_rule_ext_parse_flow_entries(ruleext, cls);
+	if (res)
+		return res;
+
+	yt921x_acl_rule_set_ports(&ruleext->r, 0, BIT(port));
+	ruleext->r.tag = cls->cookie;
+	ruleext->r.type = TC_SETUP_CLSFLOWER;
+	return 0;
+}
+
+static unsigned int
+yt921x_acl_find(const struct yt921x_priv *priv, enum tc_setup_type type,
+		unsigned long tag)
+{
+	for (unsigned int blkid = 0; blkid < YT921X_ACL_BLK_NUM; blkid++) {
+		const struct yt921x_acl_blk *aclblk = priv->acl_blks[blkid];
+
+		if (!aclblk)
+			continue;
+
+		for (unsigned int i = 0; i < YT921X_ACL_ENT_PER_BLK; i++)
+			if (aclblk->rules[i] && aclblk->rules[i]->tag == tag &&
+			    aclblk->rules[i]->type == type)
+				return YT921X_ACL_ENT_PER_BLK * blkid + i;
+	}
+
+	return UINT_MAX;
+}
+
+static unsigned int
+yt921x_acl_reserve(struct yt921x_priv *priv, unsigned int entscnt,
+		   struct netlink_ext_ack *extack)
+{
+	int candidates[YT921X_ACL_ENT_PER_BLK + 1];
+	unsigned int acl_used_cnt = 0;
+
+	if (WARN_ON(entscnt > YT921X_ACL_ENT_PER_BLK))
+		return UINT_MAX;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(candidates); i++)
+		candidates[i] = -1;
+	for (unsigned int i = YT921X_ACL_BLK_NUM; i-- > 0;) {
+		unsigned int blk_used_cnt = hweight8(priv->acl_masks[i]);
+
+		candidates[blk_used_cnt] = i;
+		acl_used_cnt += blk_used_cnt;
+	}
+
+	if (acl_used_cnt >= YT921X_ACL_NUM) {
+		NL_SET_ERR_MSG_MOD(extack, "ACL entry limit reached");
+		return UINT_MAX;
+	}
+	if (acl_used_cnt + entscnt <= YT921X_ACL_NUM)
+		for (unsigned int i = YT921X_ACL_ENT_PER_BLK - entscnt + 1;
+		     i-- > 0;)
+			if (candidates[i] >= 0)
+				return YT921X_ACL_ENT_PER_BLK * candidates[i] +
+				       ffz(priv->acl_masks[candidates[i]]);
+
+	NL_SET_ERR_MSG_MOD(extack,
+			   "ACL entry allocation failed, simplify your rules or remove existing rules");
+	return UINT_MAX;
+}
+
+static int
+yt921x_acl_commit(struct yt921x_priv *priv, unsigned int entid, u8 entsmask)
+{
+	const struct yt921x_acl_rule *aclrule;
+	const struct yt921x_acl_blk *aclblk;
+	unsigned int blkid;
+	unsigned int binid;
+	unsigned long mask;
+	u32 zeros[3] = {};
+	unsigned int i;
+	unsigned int o;
+	u32 ctrl;
+	int res;
+
+	blkid = entid / YT921X_ACL_ENT_PER_BLK;
+	binid = entid % YT921X_ACL_ENT_PER_BLK;
+	aclblk = priv->acl_blks[blkid];
+	aclrule = aclblk->rules[binid];
+
+	/* Write actions */
+	res = yt921x_reg96_write(priv, YT921X_ACLn_ACT(entid),
+				 aclrule ? aclrule->action : zeros);
+	if (res)
+		return res;
+
+	/* Select the block */
+	ctrl = YT921X_ACL_BLK_CMD_MODIFY | YT921X_ACL_BLK_CMD_BLKID(blkid);
+	res = yt921x_reg_write(priv, YT921X_ACL_BLK_CMD, ctrl);
+	if (res)
+		return res;
+
+	/* Write keys and masks */
+	ctrl = 0;
+	for (unsigned int i = 0; i < YT921X_ACL_ENT_PER_BLK; i++)
+		ctrl |= YT921X_ACL_BLK_KEEP_KEEPn(i);
+
+	mask = entsmask;
+	i = 0;
+	for_each_set_bit(o, &mask, YT921X_ACL_ENT_PER_BLK) {
+		res = yt921x_reg64_write(priv, YT921X_ACLn_KEYm(blkid, o),
+					 aclrule ? aclrule->entries[i].key :
+					 zeros);
+		if (res)
+			return res;
+
+		res = yt921x_reg64_write(priv, YT921X_ACLn_MASKm(blkid, o),
+					 aclrule ? aclrule->entries[i].mask :
+					 zeros);
+		if (res)
+			return res;
+
+		ctrl &= ~YT921X_ACL_BLK_KEEP_KEEPn(o);
+		i++;
+	}
+
+	res = yt921x_reg_write(priv, YT921X_ACL_BLK_KEEP, ctrl);
+	if (res)
+		return res;
+
+	ctrl = 0;
+	for (unsigned int i = 0; i < YT921X_ACL_ENT_PER_BLK; i++) {
+		const struct yt921x_acl_rule *other = aclblk->rules[i];
+
+		if (!other)
+			continue;
+
+		mask = other->mask;
+		for_each_set_bit(o, &mask, YT921X_ACL_ENT_PER_BLK)
+			ctrl |= YT921X_ACL_ENTRY_ENm(o) |
+				YT921X_ACL_ENTRY_GRPIDm(o, i);
+	}
+	res = yt921x_reg_write(priv, YT921X_ACLn_ENTRY(blkid), ctrl);
+	if (res)
+		return res;
+
+	/* Commit the block */
+	ctrl = YT921X_ACL_BLK_CMD_BLKID(blkid);
+	res = yt921x_reg_write(priv, YT921X_ACL_BLK_CMD, ctrl);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int
+yt921x_acl_del(struct yt921x_priv *priv, enum tc_setup_type type,
+	       unsigned long tag)
+{
+	struct yt921x_acl_rule *aclrule;
+	struct yt921x_acl_blk *aclblk;
+	unsigned int binid;
+	unsigned int blkid;
+	unsigned int entid;
+	int res;
+
+	entid = yt921x_acl_find(priv, type, tag);
+	if (entid == UINT_MAX)
+		return -ENOENT;
+
+	blkid = entid / YT921X_ACL_ENT_PER_BLK;
+	binid = entid % YT921X_ACL_ENT_PER_BLK;
+	aclblk = priv->acl_blks[blkid];
+	aclrule = aclblk->rules[binid];
+
+	aclblk->rules[binid] = NULL;
+	res = yt921x_acl_commit(priv, entid, aclrule->mask);
+	/* the kernel never rolls back on failure */
+
+	if (aclrule->action[0] & YT921X_ACL_ACTa_METER_EN)
+		clear_bit(FIELD_GET(YT921X_ACL_ACTa_METER_ID_M,
+				    aclrule->action[0]),
+			  priv->meters_map);
+	priv->acl_masks[blkid] &= ~aclrule->mask;
+	kvfree(aclrule);
+	if (!priv->acl_masks[blkid]) {
+		kvfree(aclblk);
+		priv->acl_blks[blkid] = NULL;
+	}
+	return res;
+}
+
+static int
+yt921x_acl_add(struct yt921x_priv *priv,
+	       const struct yt921x_acl_rule_ext *ruleext,
+	       struct netlink_ext_ack *extack)
+{
+	unsigned int entscnt = hweight8(ruleext->r.mask);
+	struct yt921x_acl_rule *aclrule;
+	struct yt921x_acl_blk *aclblk;
+	bool use_trap = false;
+	unsigned int meterid;
+	unsigned long mask;
+	unsigned int binid;
+	unsigned int blkid;
+	unsigned int entid;
+	unsigned int o;
+	int res;
+
+	/* Allocate resources */
+	entid = yt921x_acl_reserve(priv, entscnt, extack);
+	if (entid == UINT_MAX)
+		return -EOPNOTSUPP;
+
+	if (!(ruleext->r.action[0] & YT921X_ACL_ACTa_METER_EN)) {
+		meterid = YT921X_METER_NUM;
+	} else {
+		meterid = find_first_zero_bit(priv->meters_map,
+					      YT921X_METER_NUM);
+		if (meterid < YT921X_METER_NUM) {
+			res = yt921x_meter_config(priv, meterid,
+						  &ruleext->marker);
+			if (res)
+				return res;
+		} else if (ruleext->r.sw_assisted) {
+			use_trap = true;
+		} else {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "No more meters available");
+			return -EOPNOTSUPP;
+		}
+	}
+
+	/* Prepare acl block ctrlblk */
+	blkid = entid / YT921X_ACL_ENT_PER_BLK;
+	binid = entid % YT921X_ACL_ENT_PER_BLK;
+	aclblk = priv->acl_blks[blkid];
+	if (!aclblk) {
+		aclblk = kvzalloc_obj(*aclblk);
+		if (!aclblk)
+			return -ENOMEM;
+		priv->acl_blks[blkid] = aclblk;
+	}
+
+	/* Prepare acl rule ctrlblk */
+	aclrule = kvmemdup(&ruleext->r,
+			   offsetof(struct yt921x_acl_rule, entries[entscnt]),
+			   GFP_KERNEL);
+	if (!aclrule) {
+		res = -ENOMEM;
+		goto err;
+	}
+
+	/* Replace the placeholder resource IDs */
+	aclrule->mask = 0;
+	mask = priv->acl_masks[blkid];
+	for_each_clear_bit(o, &mask, YT921X_ACL_ENT_PER_BLK) {
+		aclrule->mask |= BIT(o);
+		entscnt--;
+		if (!entscnt)
+			break;
+	}
+
+	if (use_trap) {
+		aclrule->action[2] &= ~YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M &
+				      ~YT921X_ACL_ACTc_FWD_M;
+		aclrule->action[2] |= YT921X_ACL_ACTc_FWD_EN |
+				      YT921X_ACL_ACTc_FWD_TRAP;
+	}
+	if (meterid < YT921X_METER_NUM)
+		aclrule->action[0] |= YT921X_ACL_ACTa_METER_ID(meterid);
+	else
+		aclrule->action[0] &= ~YT921X_ACL_ACTa_METER_EN;
+
+	/* Write rules */
+	aclblk->rules[binid] = aclrule;
+	res = yt921x_acl_commit(priv, entid, aclrule->mask);
+	if (res) {
+		aclblk->rules[binid] = NULL;
+		kvfree(aclrule);
+		goto err;
+	}
+
+	if (meterid < YT921X_METER_NUM)
+		set_bit(meterid, priv->meters_map);
+	priv->acl_masks[blkid] |= aclrule->mask;
+	return 0;
+
+err:
+	if (!priv->acl_masks[blkid]) {
+		kvfree(aclblk);
+		priv->acl_blks[blkid] = NULL;
+	}
+	return res;
+}
+
+static int
+yt921x_dsa_cls_flower_del(struct dsa_switch *ds, int port,
+			  struct flow_cls_offload *cls, bool ingress)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_acl_del(priv, TC_SETUP_CLSFLOWER, cls->cookie);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_cls_flower_add(struct dsa_switch *ds, int port,
+			  struct flow_cls_offload *cls, bool ingress)
+{
+	struct netlink_ext_ack *extack = cls->common.extack;
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_acl_rule_ext ruleext;
+	int res;
+
+	res = yt921x_acl_rule_ext_parse_flow(&ruleext, port, cls, ingress,
+					     priv);
+	if (res)
+		return res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_acl_add(priv, &ruleext, extack);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_mirror_del(struct yt921x_priv *priv, int port, bool ingress)
+{
+	u32 mask;
+
+	if (ingress)
+		mask = YT921X_MIRROR_IGR_PORTn(port);
+	else
+		mask = YT921X_MIRROR_EGR_PORTn(port);
+	return yt921x_reg_clear_bits(priv, YT921X_MIRROR, mask);
+}
+
+static int
+yt921x_mirror_add(struct yt921x_priv *priv, int port, bool ingress,
+		  int to_local_port, struct netlink_ext_ack *extack)
+{
+	u32 srcs;
+	u32 ctrl;
+	u32 val;
+	u32 dst;
+	int res;
+
+	if (ingress)
+		srcs = YT921X_MIRROR_IGR_PORTn(port);
+	else
+		srcs = YT921X_MIRROR_EGR_PORTn(port);
+	dst = YT921X_MIRROR_PORT(to_local_port);
+
+	res = yt921x_reg_read(priv, YT921X_MIRROR, &val);
+	if (res)
+		return res;
+
+	/* other mirror tasks & different dst port -> conflict */
+	if ((val & ~srcs & (YT921X_MIRROR_EGR_PORTS_M |
+			    YT921X_MIRROR_IGR_PORTS_M)) &&
+	    (val & YT921X_MIRROR_PORT_M) != dst) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Sniffer port is already configured, delete existing rules & retry");
+		return -EBUSY;
+	}
+
+	ctrl = val & ~YT921X_MIRROR_PORT_M;
+	ctrl |= srcs;
+	ctrl |= dst;
+
+	if (ctrl == val)
+		return 0;
+
+	return yt921x_reg_write(priv, YT921X_MIRROR, ctrl);
+}
+
+static void
+yt921x_dsa_port_mirror_del(struct dsa_switch *ds, int port,
+			   struct dsa_mall_mirror_tc_entry *mirror)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct device *dev = to_device(priv);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_mirror_del(priv, port, mirror->ingress);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s port %d: %i\n", "unmirror",
+			port, res);
+}
+
+static int
+yt921x_dsa_port_mirror_add(struct dsa_switch *ds, int port,
+			   struct dsa_mall_mirror_tc_entry *mirror,
+			   bool ingress, struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_mirror_add(priv, port, ingress,
+				mirror->to_local_port, extack);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int yt921x_lag_hash(struct yt921x_priv *priv, u32 ctrl, bool unique_lag,
+			   struct netlink_ext_ack *extack)
+{
+	u32 val;
+	int res;
+
+	/* Hash Mode is global. Make sure the same Hash Mode is set to all the
+	 * 2 possible lags.
+	 * If we are the unique LAG we can set whatever hash mode we want.
+	 * To change hash mode it's needed to remove all LAG and change the mode
+	 * with the latest.
+	 */
+	if (unique_lag) {
+		res = yt921x_reg_write(priv, YT921X_LAG_HASH, ctrl);
+		if (res)
+			return res;
+	} else {
+		res = yt921x_reg_read(priv, YT921X_LAG_HASH, &val);
+		if (res)
+			return res;
+
+		if (val != ctrl) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Mismatched Hash Mode across different lags is not supported");
+			return -EOPNOTSUPP;
+		}
+	}
+
+	return 0;
+}
+
+static int yt921x_lag_set(struct yt921x_priv *priv, u8 index, u16 ports_mask)
+{
+	unsigned long targets_mask = ports_mask;
+	unsigned int cnt;
+	u32 ctrl;
+	int port;
+	int res;
+
+	cnt = 0;
+	for_each_set_bit(port, &targets_mask, YT921X_PORT_NUM) {
+		ctrl = YT921X_LAG_MEMBER_PORT(port);
+		res = yt921x_reg_write(priv, YT921X_LAG_MEMBERnm(index, cnt),
+				       ctrl);
+		if (res)
+			return res;
+
+		cnt++;
+	}
+
+	ctrl = YT921X_LAG_GROUP_PORTS(ports_mask) |
+	       YT921X_LAG_GROUP_MEMBER_NUM(cnt);
+	return yt921x_reg_write(priv, YT921X_LAG_GROUPn(index), ctrl);
+}
+
+static int
+yt921x_dsa_port_lag_leave(struct dsa_switch *ds, int port, struct dsa_lag lag)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct dsa_port *dp;
+	u32 ctrl;
+	int res;
+
+	if (!lag.id)
+		return -EINVAL;
+
+	ctrl = 0;
+	dsa_lag_foreach_port(dp, ds->dst, &lag)
+		ctrl |= BIT(dp->index);
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_lag_set(priv, lag.id - 1, ctrl);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_lag_check(struct dsa_switch *ds, struct dsa_lag lag,
+			  struct netdev_lag_upper_info *info,
+			  struct netlink_ext_ack *extack)
+{
+	unsigned int members;
+	struct dsa_port *dp;
+
+	if (!lag.id)
+		return -EINVAL;
+
+	members = 0;
+	dsa_lag_foreach_port(dp, ds->dst, &lag)
+		/* Includes the port joining the LAG */
+		members++;
+
+	if (members > YT921X_LAG_PORT_NUM) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot offload more than 4 LAG ports");
+		return -EOPNOTSUPP;
+	}
+
+	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can only offload LAG using hash TX type");
+		return -EOPNOTSUPP;
+	}
+
+	if (info->hash_type != NETDEV_LAG_HASH_L2 &&
+	    info->hash_type != NETDEV_LAG_HASH_L23 &&
+	    info->hash_type != NETDEV_LAG_HASH_L34) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can only offload L2 or L2+L3 or L3+L4 TX hash");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_dsa_port_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag,
+			 struct netdev_lag_upper_info *info,
+			 struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct dsa_port *dp;
+	bool unique_lag;
+	unsigned int i;
+	u32 ctrl;
+	int res;
+
+	res = yt921x_dsa_port_lag_check(ds, lag, info, extack);
+	if (res)
+		return res;
+
+	ctrl = 0;
+	switch (info->hash_type) {
+	case NETDEV_LAG_HASH_L34:
+		ctrl |= YT921X_LAG_HASH_IP_DST;
+		ctrl |= YT921X_LAG_HASH_IP_SRC;
+		ctrl |= YT921X_LAG_HASH_IP_PROTO;
+
+		ctrl |= YT921X_LAG_HASH_L4_DPORT;
+		ctrl |= YT921X_LAG_HASH_L4_SPORT;
+		break;
+	case NETDEV_LAG_HASH_L23:
+		ctrl |= YT921X_LAG_HASH_MAC_DA;
+		ctrl |= YT921X_LAG_HASH_MAC_SA;
+
+		ctrl |= YT921X_LAG_HASH_IP_DST;
+		ctrl |= YT921X_LAG_HASH_IP_SRC;
+		ctrl |= YT921X_LAG_HASH_IP_PROTO;
+		break;
+	case NETDEV_LAG_HASH_L2:
+		ctrl |= YT921X_LAG_HASH_MAC_DA;
+		ctrl |= YT921X_LAG_HASH_MAC_SA;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	/* Check if we are the unique configured LAG */
+	unique_lag = true;
+	dsa_lags_foreach_id(i, ds->dst)
+		if (i != lag.id && dsa_lag_by_id(ds->dst, i)) {
+			unique_lag = false;
+			break;
+		}
+
+	mutex_lock(&priv->reg_lock);
+	do {
+		res = yt921x_lag_hash(priv, ctrl, unique_lag, extack);
+		if (res)
+			break;
+
+		ctrl = 0;
+		dsa_lag_foreach_port(dp, ds->dst, &lag)
+			ctrl |= BIT(dp->index);
+		res = yt921x_lag_set(priv, lag.id - 1, ctrl);
+	} while (0);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int yt921x_fdb_wait(struct yt921x_priv *priv, u32 *valp)
+{
+	struct device *dev = to_device(priv);
+	u32 val = YT921X_FDB_RESULT_DONE;
+	int res;
+
+	res = yt921x_reg_wait(priv, YT921X_FDB_RESULT, YT921X_FDB_RESULT_DONE,
+			      &val);
+	if (res) {
+		dev_err(dev, "FDB probably stuck\n");
+		return res;
+	}
+
+	*valp = val;
+	return 0;
+}
+
+static int
+yt921x_fdb_in01(struct yt921x_priv *priv, const unsigned char *addr,
+		u16 vid, u32 ctrl1)
+{
+	u32 ctrl;
+	int res;
+
+	ctrl = ethaddr_hi4_to_u32(addr);
+	res = yt921x_reg_write(priv, YT921X_FDB_IN0, ctrl);
+	if (res)
+		return res;
+
+	ctrl = ctrl1 | YT921X_FDB_IO1_FID(vid) | ethaddr_lo2_to_u32(addr);
+	return yt921x_reg_write(priv, YT921X_FDB_IN1, ctrl);
+}
+
+static int
+yt921x_fdb_has(struct yt921x_priv *priv, const unsigned char *addr, u16 vid,
+	       u16 *indexp)
+{
+	u32 ctrl;
+	u32 val;
+	int res;
+
+	res = yt921x_fdb_in01(priv, addr, vid, 0);
+	if (res)
+		return res;
+
+	ctrl = 0;
+	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
+	if (res)
+		return res;
+
+	ctrl = YT921X_FDB_OP_OP_GET_ONE | YT921X_FDB_OP_START;
+	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
+	if (res)
+		return res;
+
+	res = yt921x_fdb_wait(priv, &val);
+	if (res)
+		return res;
+	if (val & YT921X_FDB_RESULT_NOTFOUND) {
+		*indexp = YT921X_FDB_NUM;
+		return 0;
+	}
+
+	*indexp = FIELD_GET(YT921X_FDB_RESULT_INDEX_M, val);
+	return 0;
+}
+
+static int
+yt921x_fdb_read(struct yt921x_priv *priv, unsigned char *addr, u16 *vidp,
+		u16 *ports_maskp, u16 *indexp, u8 *statusp)
+{
+	struct device *dev = to_device(priv);
+	u16 index;
+	u32 data0;
+	u32 data1;
+	u32 data2;
+	u32 val;
+	int res;
+
+	res = yt921x_fdb_wait(priv, &val);
+	if (res)
+		return res;
+	if (val & YT921X_FDB_RESULT_NOTFOUND) {
+		*ports_maskp = 0;
+		return 0;
+	}
+	index = FIELD_GET(YT921X_FDB_RESULT_INDEX_M, val);
+
+	res = yt921x_reg_read(priv, YT921X_FDB_OUT1, &data1);
+	if (res)
+		return res;
+	if ((data1 & YT921X_FDB_IO1_STATUS_M) ==
+	    YT921X_FDB_IO1_STATUS_INVALID) {
+		*ports_maskp = 0;
+		return 0;
+	}
+
+	res = yt921x_reg_read(priv, YT921X_FDB_OUT0, &data0);
+	if (res)
+		return res;
+	res = yt921x_reg_read(priv, YT921X_FDB_OUT2, &data2);
+	if (res)
+		return res;
+
+	addr[0] = data0 >> 24;
+	addr[1] = data0 >> 16;
+	addr[2] = data0 >> 8;
+	addr[3] = data0;
+	addr[4] = data1 >> 8;
+	addr[5] = data1;
+	*vidp = FIELD_GET(YT921X_FDB_IO1_FID_M, data1);
+	*indexp = index;
+	*ports_maskp = FIELD_GET(YT921X_FDB_IO2_EGR_PORTS_M, data2);
+	*statusp = FIELD_GET(YT921X_FDB_IO1_STATUS_M, data1);
+
+	dev_dbg(dev,
+		"%s: index 0x%x, mac %02x:%02x:%02x:%02x:%02x:%02x, vid %d, ports 0x%x, status %d\n",
+		__func__, *indexp, addr[0], addr[1], addr[2], addr[3],
+		addr[4], addr[5], *vidp, *ports_maskp, *statusp);
+	return 0;
+}
+
+static int
+yt921x_fdb_dump(struct yt921x_priv *priv, u16 ports_mask,
+		dsa_fdb_dump_cb_t *cb, void *data)
+{
+	unsigned char addr[ETH_ALEN];
+	u8 status;
+	u16 pmask;
+	u16 index;
+	u32 ctrl;
+	u16 vid;
+	int res;
+
+	ctrl = YT921X_FDB_OP_INDEX(0) | YT921X_FDB_OP_MODE_INDEX |
+	       YT921X_FDB_OP_OP_GET_ONE | YT921X_FDB_OP_START;
+	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
+	if (res)
+		return res;
+	res = yt921x_fdb_read(priv, addr, &vid, &pmask, &index, &status);
+	if (res)
+		return res;
+	if ((pmask & ports_mask) && !is_multicast_ether_addr(addr)) {
+		res = cb(addr, vid,
+			 status == YT921X_FDB_ENTRY_STATUS_STATIC, data);
+		if (res)
+			return res;
+	}
+
+	ctrl = YT921X_FDB_IO2_EGR_PORTS(ports_mask);
+	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
+	if (res)
+		return res;
+
+	index = 0;
+	do {
+		ctrl = YT921X_FDB_OP_INDEX(index) | YT921X_FDB_OP_MODE_INDEX |
+		       YT921X_FDB_OP_NEXT_TYPE_UCAST_PORT |
+		       YT921X_FDB_OP_OP_GET_NEXT | YT921X_FDB_OP_START;
+		res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
+		if (res)
+			return res;
+
+		res = yt921x_fdb_read(priv, addr, &vid, &pmask, &index,
+				      &status);
+		if (res)
+			return res;
+		if (!pmask)
+			break;
+
+		if ((pmask & ports_mask) && !is_multicast_ether_addr(addr)) {
+			res = cb(addr, vid,
+				 status == YT921X_FDB_ENTRY_STATUS_STATIC,
+				 data);
+			if (res)
+				return res;
+		}
+
+		/* Never call GET_NEXT with 4095, otherwise it will hang
+		 * forever until a reset!
+		 */
+	} while (index < YT921X_FDB_NUM - 1);
+
+	return 0;
+}
+
+static int
+yt921x_fdb_flush_raw(struct yt921x_priv *priv, u16 ports_mask, u16 vid,
+		     bool flush_static)
+{
+	u32 ctrl;
+	u32 val;
+	int res;
+
+	if (vid < 4096) {
+		ctrl = YT921X_FDB_IO1_FID(vid);
+		res = yt921x_reg_write(priv, YT921X_FDB_IN1, ctrl);
+		if (res)
+			return res;
+	}
+
+	ctrl = YT921X_FDB_IO2_EGR_PORTS(ports_mask);
+	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
+	if (res)
+		return res;
+
+	ctrl = YT921X_FDB_OP_OP_FLUSH | YT921X_FDB_OP_START;
+	if (vid >= 4096)
+		ctrl |= YT921X_FDB_OP_FLUSH_PORT;
+	else
+		ctrl |= YT921X_FDB_OP_FLUSH_PORT_VID;
+	if (flush_static)
+		ctrl |= YT921X_FDB_OP_FLUSH_STATIC;
+	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
+	if (res)
+		return res;
+
+	res = yt921x_fdb_wait(priv, &val);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int
+yt921x_fdb_flush_port(struct yt921x_priv *priv, int port, bool flush_static)
+{
+	return yt921x_fdb_flush_raw(priv, BIT(port), 4096, flush_static);
+}
+
+static int
+yt921x_fdb_add_index_in12(struct yt921x_priv *priv, u16 index, u16 ctrl1,
+			  u16 ctrl2)
+{
+	u32 ctrl;
+	u32 val;
+	int res;
+
+	res = yt921x_reg_write(priv, YT921X_FDB_IN1, ctrl1);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl2);
+	if (res)
+		return res;
+
+	ctrl = YT921X_FDB_OP_INDEX(index) | YT921X_FDB_OP_MODE_INDEX |
+	       YT921X_FDB_OP_OP_ADD | YT921X_FDB_OP_START;
+	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
+	if (res)
+		return res;
+
+	return yt921x_fdb_wait(priv, &val);
+}
+
+static int
+yt921x_fdb_add(struct yt921x_priv *priv, const unsigned char *addr, u16 vid,
+	       u16 ports_mask)
+{
+	u32 ctrl;
+	u32 val;
+	int res;
+
+	ctrl = YT921X_FDB_IO1_STATUS_STATIC;
+	res = yt921x_fdb_in01(priv, addr, vid, ctrl);
+	if (res)
+		return res;
+
+	ctrl = YT921X_FDB_IO2_EGR_PORTS(ports_mask);
+	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
+	if (res)
+		return res;
+
+	ctrl = YT921X_FDB_OP_OP_ADD | YT921X_FDB_OP_START;
+	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
+	if (res)
+		return res;
+
+	return yt921x_fdb_wait(priv, &val);
+}
+
+static int
+yt921x_fdb_leave(struct yt921x_priv *priv, const unsigned char *addr,
+		 u16 vid, u16 ports_mask)
+{
+	u16 index;
+	u32 ctrl1;
+	u32 ctrl2;
+	u32 ctrl;
+	u32 val2;
+	u32 val;
+	int res;
+
+	/* Check for presence */
+	res = yt921x_fdb_has(priv, addr, vid, &index);
+	if (res)
+		return res;
+	if (index >= YT921X_FDB_NUM)
+		return 0;
+
+	/* Check if action required */
+	res = yt921x_reg_read(priv, YT921X_FDB_OUT2, &val2);
+	if (res)
+		return res;
+
+	ctrl2 = val2 & ~YT921X_FDB_IO2_EGR_PORTS(ports_mask);
+	if (ctrl2 == val2)
+		return 0;
+	if (!(ctrl2 & YT921X_FDB_IO2_EGR_PORTS_M)) {
+		ctrl = YT921X_FDB_OP_OP_DEL | YT921X_FDB_OP_START;
+		res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
+		if (res)
+			return res;
+
+		return yt921x_fdb_wait(priv, &val);
+	}
+
+	res = yt921x_reg_read(priv, YT921X_FDB_OUT1, &ctrl1);
+	if (res)
+		return res;
+
+	return yt921x_fdb_add_index_in12(priv, index, ctrl1, ctrl2);
+}
+
+static int
+yt921x_fdb_join(struct yt921x_priv *priv, const unsigned char *addr, u16 vid,
+		u16 ports_mask)
+{
+	u16 index;
+	u32 ctrl1;
+	u32 ctrl2;
+	u32 val1;
+	u32 val2;
+	int res;
+
+	/* Check for presence */
+	res = yt921x_fdb_has(priv, addr, vid, &index);
+	if (res)
+		return res;
+	if (index >= YT921X_FDB_NUM)
+		return yt921x_fdb_add(priv, addr, vid, ports_mask);
+
+	/* Check if action required */
+	res = yt921x_reg_read(priv, YT921X_FDB_OUT1, &val1);
+	if (res)
+		return res;
+	res = yt921x_reg_read(priv, YT921X_FDB_OUT2, &val2);
+	if (res)
+		return res;
+
+	ctrl1 = val1 & ~YT921X_FDB_IO1_STATUS_M;
+	ctrl1 |= YT921X_FDB_IO1_STATUS_STATIC;
+	ctrl2 = val2 | YT921X_FDB_IO2_EGR_PORTS(ports_mask);
+	if (ctrl1 == val1 && ctrl2 == val2)
+		return 0;
+
+	return yt921x_fdb_add_index_in12(priv, index, ctrl1, ctrl2);
+}
+
+static int
+yt921x_dsa_port_fdb_dump(struct dsa_switch *ds, int port,
+			 dsa_fdb_dump_cb_t *cb, void *data)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	/* Hardware FDB is shared for fdb and mdb, "bridge fdb show"
+	 * only wants to see unicast
+	 */
+	res = yt921x_fdb_dump(priv, BIT(port), cb, data);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static void yt921x_dsa_port_fast_age(struct dsa_switch *ds, int port)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct device *dev = to_device(priv);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_fdb_flush_port(priv, port, false);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s port %d: %i\n", "clear FDB for",
+			port, res);
+}
+
+static int
+yt921x_dsa_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u32 ctrl;
+	int res;
+
+	/* AGEING reg is set in 5s step */
+	ctrl = clamp(msecs / 5000, 1, U16_MAX);
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_write(priv, YT921X_AGEING, ctrl);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_fdb_del(struct dsa_switch *ds, int port,
+			const unsigned char *addr, u16 vid, struct dsa_db db)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_fdb_leave(priv, addr, vid, BIT(port));
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_fdb_add(struct dsa_switch *ds, int port,
+			const unsigned char *addr, u16 vid, struct dsa_db db)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_fdb_join(priv, addr, vid, BIT(port));
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_mdb_del(struct dsa_switch *ds, int port,
+			const struct switchdev_obj_port_mdb *mdb,
+			struct dsa_db db)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	const unsigned char *addr = mdb->addr;
+	u16 vid = mdb->vid;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_fdb_leave(priv, addr, vid, BIT(port));
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_mdb_add(struct dsa_switch *ds, int port,
+			const struct switchdev_obj_port_mdb *mdb,
+			struct dsa_db db)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	const unsigned char *addr = mdb->addr;
+	u16 vid = mdb->vid;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_fdb_join(priv, addr, vid, BIT(port));
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_vlan_aware_set(struct yt921x_priv *priv, int port, bool vlan_aware)
+{
+	u32 ctrl;
+
+	/* Abuse SVLAN for PCP parsing without polluting the FDB - it just works
+	 * despite YT921X_VLAN_CTRL_SVLAN_EN never being set
+	 */
+	if (!vlan_aware)
+		ctrl = YT921X_PORT_IGR_TPIDn_STAG(0);
+	else
+		ctrl = YT921X_PORT_IGR_TPIDn_CTAG(0);
+	return yt921x_reg_write(priv, YT921X_PORTn_IGR_TPID(port), ctrl);
+}
+
+static int
+yt921x_port_set_pvid(struct yt921x_priv *priv, int port, u16 vid)
+{
+	u32 mask;
+	u32 ctrl;
+
+	mask = YT921X_PORT_VLAN_CTRL_CVID_M;
+	ctrl = YT921X_PORT_VLAN_CTRL_CVID(vid);
+	return yt921x_reg_update_bits(priv, YT921X_PORTn_VLAN_CTRL(port),
+				      mask, ctrl);
+}
+
+static int
+yt921x_vlan_filtering(struct yt921x_priv *priv, int port, bool vlan_filtering)
+{
+	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
+	struct net_device *bdev;
+	u16 pvid;
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	bdev = dsa_port_bridge_dev_get(dp);
+
+	if (!bdev || !vlan_filtering)
+		pvid = YT921X_VID_UNWARE;
+	else
+		br_vlan_get_pvid(bdev, &pvid);
+	res = yt921x_port_set_pvid(priv, port, pvid);
+	if (res)
+		return res;
+
+	mask = YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_TAGGED |
+	       YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
+	ctrl = 0;
+	/* Do not drop tagged frames here; let VLAN_IGR_FILTER do it */
+	if (vlan_filtering && !pvid)
+		ctrl |= YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
+	res = yt921x_reg_update_bits(priv, YT921X_PORTn_VLAN_CTRL1(port),
+				     mask, ctrl);
+	if (res)
+		return res;
+
+	res = yt921x_reg_toggle_bits(priv, YT921X_VLAN_IGR_FILTER,
+				     YT921X_VLAN_IGR_FILTER_PORTn(port),
+				     vlan_filtering);
+	if (res)
+		return res;
+
+	res = yt921x_vlan_aware_set(priv, port, vlan_filtering);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int yt921x_vlan_del(struct yt921x_priv *priv, int port, u16 vid)
+{
+	u32 masks[2];
+
+	masks[0] = YT921X_VLAN_CTRLa_PORTn(port);
+	masks[1] = YT921X_VLAN_CTRLb_UNTAG_PORTn(port);
+
+	return yt921x_reg64_clear_bits(priv, YT921X_VLANn_CTRL(vid), masks);
+}
+
+static int
+yt921x_vlan_add(struct yt921x_priv *priv, int port, u16 vid, bool untagged)
+{
+	u32 masks[2];
+	u32 ctrls[2];
+
+	masks[0] = YT921X_VLAN_CTRLa_PORTn(port) |
+		   YT921X_VLAN_CTRLa_PORTS(priv->cpu_ports_mask);
+	ctrls[0] = masks[0];
+
+	masks[1] = YT921X_VLAN_CTRLb_UNTAG_PORTn(port);
+	ctrls[1] = untagged ? masks[1] : 0;
+
+	return yt921x_reg64_update_bits(priv, YT921X_VLANn_CTRL(vid),
+					masks, ctrls);
+}
+
+static int
+yt921x_pvid_clear(struct yt921x_priv *priv, int port)
+{
+	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
+	bool vlan_filtering;
+	u32 mask;
+	int res;
+
+	vlan_filtering = dsa_port_is_vlan_filtering(dp);
+
+	res = yt921x_port_set_pvid(priv, port,
+				   vlan_filtering ? 0 : YT921X_VID_UNWARE);
+	if (res)
+		return res;
+
+	if (vlan_filtering) {
+		mask = YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
+		res = yt921x_reg_set_bits(priv, YT921X_PORTn_VLAN_CTRL1(port),
+					  mask);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_pvid_set(struct yt921x_priv *priv, int port, u16 vid)
+{
+	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
+	bool vlan_filtering;
+	u32 mask;
+	int res;
+
+	vlan_filtering = dsa_port_is_vlan_filtering(dp);
+
+	if (vlan_filtering) {
+		res = yt921x_port_set_pvid(priv, port, vid);
+		if (res)
+			return res;
+	}
+
+	mask = YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
+	res = yt921x_reg_clear_bits(priv, YT921X_PORTn_VLAN_CTRL1(port), mask);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int
+yt921x_dsa_port_vlan_filtering(struct dsa_switch *ds, int port,
+			       bool vlan_filtering,
+			       struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	if (dsa_is_cpu_port(ds, port))
+		return 0;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_vlan_filtering(priv, port, vlan_filtering);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_vlan_del(struct dsa_switch *ds, int port,
+			 const struct switchdev_obj_port_vlan *vlan)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u16 vid = vlan->vid;
+	u16 pvid;
+	int res;
+
+	if (dsa_is_cpu_port(ds, port))
+		return 0;
+
+	mutex_lock(&priv->reg_lock);
+	do {
+		struct dsa_port *dp = dsa_to_port(ds, port);
+		struct net_device *bdev;
+
+		res = yt921x_vlan_del(priv, port, vid);
+		if (res)
+			break;
+
+		bdev = dsa_port_bridge_dev_get(dp);
+		if (bdev) {
+			br_vlan_get_pvid(bdev, &pvid);
+			if (pvid == vid)
+				res = yt921x_pvid_clear(priv, port);
+		}
+	} while (0);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_vlan_add(struct dsa_switch *ds, int port,
+			 const struct switchdev_obj_port_vlan *vlan,
+			 struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u16 vid = vlan->vid;
+	u16 pvid;
+	int res;
+
+	/* CPU port is supposed to be a member of every VLAN; see
+	 * yt921x_vlan_add() and yt921x_port_setup()
+	 */
+	if (dsa_is_cpu_port(ds, port))
+		return 0;
+
+	mutex_lock(&priv->reg_lock);
+	do {
+		struct dsa_port *dp = dsa_to_port(ds, port);
+		struct net_device *bdev;
+
+		res = yt921x_vlan_add(priv, port, vid,
+				      vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED);
+		if (res)
+			break;
+
+		bdev = dsa_port_bridge_dev_get(dp);
+		if (bdev) {
+			if (vlan->flags & BRIDGE_VLAN_INFO_PVID) {
+				res = yt921x_pvid_set(priv, port, vid);
+			} else {
+				br_vlan_get_pvid(bdev, &pvid);
+				if (pvid == vid)
+					res = yt921x_pvid_clear(priv, port);
+			}
+		}
+	} while (0);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int yt921x_userport_standalone(struct yt921x_priv *priv, int port)
+{
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	ctrl = ~priv->cpu_ports_mask;
+	res = yt921x_reg_write(priv, YT921X_PORTn_ISOLATION(port), ctrl);
+	if (res)
+		return res;
+
+	/* Turn off FDB learning to prevent FDB pollution */
+	mask = YT921X_PORT_LEARN_DIS;
+	res = yt921x_reg_set_bits(priv, YT921X_PORTn_LEARN(port), mask);
+	if (res)
+		return res;
+
+	/* Turn off VLAN awareness */
+	res = yt921x_vlan_aware_set(priv, port, false);
+	if (res)
+		return res;
+
+	/* Unrelated since learning is off and all packets are trapped;
+	 * set it anyway
+	 */
+	res = yt921x_port_set_pvid(priv, port, YT921X_VID_UNWARE);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int yt921x_userport_bridge(struct yt921x_priv *priv, int port)
+{
+	u32 mask;
+	int res;
+
+	mask = YT921X_PORT_LEARN_DIS;
+	res = yt921x_reg_clear_bits(priv, YT921X_PORTn_LEARN(port), mask);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int yt921x_isolate(struct yt921x_priv *priv, int port)
+{
+	u32 mask;
+	int res;
+
+	mask = BIT(port);
+	for (int i = 0; i < YT921X_PORT_NUM; i++) {
+		if ((BIT(i) & priv->cpu_ports_mask) || i == port)
+			continue;
+
+		res = yt921x_reg_set_bits(priv, YT921X_PORTn_ISOLATION(i),
+					  mask);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+/* Make sure to include the CPU port in ports_mask, or your bridge will
+ * not have it.
+ */
+static int yt921x_bridge(struct yt921x_priv *priv, u16 ports_mask)
+{
+	unsigned long targets_mask = ports_mask & ~priv->cpu_ports_mask;
+	u32 isolated_mask;
+	u32 ctrl;
+	int port;
+	int res;
+
+	isolated_mask = 0;
+	for_each_set_bit(port, &targets_mask, YT921X_PORT_NUM) {
+		struct yt921x_port *pp = &priv->ports[port];
+
+		if (pp->isolated)
+			isolated_mask |= BIT(port);
+	}
+
+	/* Block from non-cpu bridge ports ... */
+	for_each_set_bit(port, &targets_mask, YT921X_PORT_NUM) {
+		struct yt921x_port *pp = &priv->ports[port];
+
+		/* to non-bridge ports */
+		ctrl = ~ports_mask;
+		/* to isolated ports when isolated */
+		if (pp->isolated)
+			ctrl |= isolated_mask;
+		/* to itself when non-hairpin */
+		if (!pp->hairpin)
+			ctrl |= BIT(port);
+		else
+			ctrl &= ~BIT(port);
+
+		res = yt921x_reg_write(priv, YT921X_PORTn_ISOLATION(port),
+				       ctrl);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int yt921x_bridge_leave(struct yt921x_priv *priv, int port)
+{
+	int res;
+
+	res = yt921x_userport_standalone(priv, port);
+	if (res)
+		return res;
+
+	res = yt921x_isolate(priv, port);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int
+yt921x_bridge_join(struct yt921x_priv *priv, int port, u16 ports_mask)
+{
+	int res;
+
+	res = yt921x_userport_bridge(priv, port);
+	if (res)
+		return res;
+
+	res = yt921x_bridge(priv, ports_mask);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int
+yt921x_bridge_flags(struct yt921x_priv *priv, int port,
+		    struct switchdev_brport_flags flags)
+{
+	struct yt921x_port *pp = &priv->ports[port];
+	bool do_flush;
+	u32 mask;
+	int res;
+
+	if (flags.mask & BR_LEARNING) {
+		bool learning = flags.val & BR_LEARNING;
+
+		mask = YT921X_PORT_LEARN_DIS;
+		res = yt921x_reg_toggle_bits(priv, YT921X_PORTn_LEARN(port),
+					     mask, !learning);
+		if (res)
+			return res;
+	}
+
+	/* BR_FLOOD, BR_MCAST_FLOOD: see the comment where ACT_UNK_ACTn_TRAP
+	 * is set
+	 */
+
+	/* BR_BCAST_FLOOD: we can filter bcast, but cannot trap them */
+
+	do_flush = false;
+	if (flags.mask & BR_HAIRPIN_MODE) {
+		pp->hairpin = flags.val & BR_HAIRPIN_MODE;
+		do_flush = true;
+	}
+	if (flags.mask & BR_ISOLATED) {
+		pp->isolated = flags.val & BR_ISOLATED;
+		do_flush = true;
+	}
+	if (do_flush) {
+		struct dsa_switch *ds = &priv->ds;
+		struct dsa_port *dp = dsa_to_port(ds, port);
+		struct net_device *bdev;
+
+		bdev = dsa_port_bridge_dev_get(dp);
+		if (bdev) {
+			u32 ports_mask;
+
+			ports_mask = dsa_bridge_ports(ds, bdev);
+			ports_mask |= priv->cpu_ports_mask;
+			res = yt921x_bridge(priv, ports_mask);
+			if (res)
+				return res;
+		}
+	}
+
+	return 0;
+}
+
+static int
+yt921x_dsa_port_pre_bridge_flags(struct dsa_switch *ds, int port,
+				 struct switchdev_brport_flags flags,
+				 struct netlink_ext_ack *extack)
+{
+	if (flags.mask & ~(BR_HAIRPIN_MODE | BR_LEARNING | BR_FLOOD |
+			   BR_MCAST_FLOOD | BR_ISOLATED))
+		return -EINVAL;
+	return 0;
+}
+
+static int
+yt921x_dsa_port_bridge_flags(struct dsa_switch *ds, int port,
+			     struct switchdev_brport_flags flags,
+			     struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	if (dsa_is_cpu_port(ds, port))
+		return 0;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_bridge_flags(priv, port, flags);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static void
+yt921x_dsa_port_bridge_leave(struct dsa_switch *ds, int port,
+			     struct dsa_bridge bridge)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct device *dev = to_device(priv);
+	int res;
+
+	if (dsa_is_cpu_port(ds, port))
+		return;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_bridge_leave(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s port %d: %i\n", "unbridge",
+			port, res);
+}
+
+static int
+yt921x_dsa_port_bridge_join(struct dsa_switch *ds, int port,
+			    struct dsa_bridge bridge, bool *tx_fwd_offload,
+			    struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u16 ports_mask;
+	int res;
+
+	if (dsa_is_cpu_port(ds, port))
+		return 0;
+
+	ports_mask = dsa_bridge_ports(ds, bridge.dev);
+	ports_mask |= priv->cpu_ports_mask;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_bridge_join(priv, port, ports_mask);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_port_mst_state_set(struct dsa_switch *ds, int port,
+			      const struct switchdev_mst_state *st)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	mask = YT921X_STP_PORTn_M(port);
+	switch (st->state) {
+	case BR_STATE_DISABLED:
+		ctrl = YT921X_STP_PORTn_DISABLED(port);
+		break;
+	case BR_STATE_LISTENING:
+	case BR_STATE_LEARNING:
+		ctrl = YT921X_STP_PORTn_LEARNING(port);
+		break;
+	case BR_STATE_FORWARDING:
+	default:
+		ctrl = YT921X_STP_PORTn_FORWARD(port);
+		break;
+	case BR_STATE_BLOCKING:
+		ctrl = YT921X_STP_PORTn_BLOCKING(port);
+		break;
+	}
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_update_bits(priv, YT921X_STPn(st->msti), mask, ctrl);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_dsa_vlan_msti_set(struct dsa_switch *ds, struct dsa_bridge bridge,
+			 const struct switchdev_vlan_msti *msti)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u32 masks[2];
+	u32 ctrls[2];
+	int res;
+
+	if (!msti->vid)
+		return -EINVAL;
+	if (!msti->msti || msti->msti >= YT921X_MSTI_NUM)
+		return -EINVAL;
+
+	masks[0] = 0;
+	ctrls[0] = 0;
+	masks[1] = YT921X_VLAN_CTRLb_STP_ID_M;
+	ctrls[1] = YT921X_VLAN_CTRLb_STP_ID(msti->msti);
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg64_update_bits(priv, YT921X_VLANn_CTRL(msti->vid),
+				       masks, ctrls);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static void
+yt921x_dsa_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct dsa_port *dp = dsa_to_port(ds, port);
+	struct device *dev = to_device(priv);
+	bool learning;
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	mask = YT921X_STP_PORTn_M(port);
+	learning = false;
+	switch (state) {
+	case BR_STATE_DISABLED:
+		ctrl = YT921X_STP_PORTn_DISABLED(port);
+		break;
+	case BR_STATE_LISTENING:
+		ctrl = YT921X_STP_PORTn_LEARNING(port);
+		break;
+	case BR_STATE_LEARNING:
+		ctrl = YT921X_STP_PORTn_LEARNING(port);
+		learning = dp->learning;
+		break;
+	case BR_STATE_FORWARDING:
+	default:
+		ctrl = YT921X_STP_PORTn_FORWARD(port);
+		learning = dp->learning;
+		break;
+	case BR_STATE_BLOCKING:
+		ctrl = YT921X_STP_PORTn_BLOCKING(port);
+		break;
+	}
+
+	mutex_lock(&priv->reg_lock);
+	do {
+		res = yt921x_reg_update_bits(priv, YT921X_STPn(0), mask, ctrl);
+		if (res)
+			break;
+
+		mask = YT921X_PORT_LEARN_DIS;
+		ctrl = !learning ? YT921X_PORT_LEARN_DIS : 0;
+		res = yt921x_reg_update_bits(priv, YT921X_PORTn_LEARN(port),
+					     mask, ctrl);
+	} while (0);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s port %d: %i\n", "set STP state for",
+			port, res);
+}
+
+static int __maybe_unused
+yt921x_dsa_port_get_default_prio(struct dsa_switch *ds, int port)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u32 val;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_read(priv, YT921X_PORTn_QOS(port), &val);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		return res;
+
+	return FIELD_GET(YT921X_PORT_QOS_PRIO_M, val);
+}
+
+static int __maybe_unused
+yt921x_dsa_port_set_default_prio(struct dsa_switch *ds, int port, u8 prio)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	if (prio >= YT921X_PRIO_NUM)
+		return -EINVAL;
+
+	mutex_lock(&priv->reg_lock);
+	mask = YT921X_PORT_QOS_PRIO_M | YT921X_PORT_QOS_PRIO_EN;
+	ctrl = YT921X_PORT_QOS_PRIO(prio) | YT921X_PORT_QOS_PRIO_EN;
+	res = yt921x_reg_update_bits(priv, YT921X_PORTn_QOS(port), mask, ctrl);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int __maybe_unused appprios_cmp(const void *a, const void *b)
+{
+	return ((const u8 *)b)[1] - ((const u8 *)a)[1];
+}
+
+static int __maybe_unused
+yt921x_dsa_port_get_apptrust(struct dsa_switch *ds, int port, u8 *sel,
+			     int *nselp)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u8 appprios[2][2] = {};
+	int nsel;
+	u32 val;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_read(priv, YT921X_PORTn_PRIO_ORD(port), &val);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		return res;
+
+	appprios[0][0] = IEEE_8021QAZ_APP_SEL_DSCP;
+	appprios[0][1] = (val >> (3 * YT921X_APP_SEL_DSCP)) & 7;
+	appprios[1][0] = DCB_APP_SEL_PCP;
+	appprios[1][1] = (val >> (3 * YT921X_APP_SEL_CVLAN_PCP)) & 7;
+	sort(appprios, ARRAY_SIZE(appprios), sizeof(appprios[0]), appprios_cmp,
+	     NULL);
+
+	nsel = 0;
+	for (int i = 0; i < ARRAY_SIZE(appprios) && appprios[i][1]; i++) {
+		sel[nsel] = appprios[i][0];
+		nsel++;
+	}
+	*nselp = nsel;
+
+	return 0;
+}
+
+static int __maybe_unused
+yt921x_dsa_port_set_apptrust(struct dsa_switch *ds, int port, const u8 *sel,
+			     int nsel)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct device *dev = to_device(priv);
+	u32 ctrl;
+	int res;
+
+	if (nsel > YT921X_APP_SEL_NUM)
+		return -EINVAL;
+
+	ctrl = 0;
+	for (int i = 0; i < nsel; i++) {
+		switch (sel[i]) {
+		case IEEE_8021QAZ_APP_SEL_DSCP:
+			ctrl |= YT921X_PORT_PRIO_ORD_APPm(YT921X_APP_SEL_DSCP,
+							  7 - i);
+			break;
+		case DCB_APP_SEL_PCP:
+			ctrl |= YT921X_PORT_PRIO_ORD_APPm(YT921X_APP_SEL_CVLAN_PCP,
+							  7 - i);
+			ctrl |= YT921X_PORT_PRIO_ORD_APPm(YT921X_APP_SEL_SVLAN_PCP,
+							  7 - i);
+			break;
+		default:
+			dev_err(dev,
+				"Invalid apptrust selector (at %d-th). Supported: dscp, pcp\n",
+				i + 1);
+			return -EOPNOTSUPP;
+		}
+	}
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_write(priv, YT921X_PORTn_PRIO_ORD(port), ctrl);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int yt921x_port_down(struct yt921x_priv *priv, int port)
+{
+	const struct yt921x_info *info = priv->info;
+	u32 mask;
+	int res;
+
+	mask = YT921X_PORT_CTRL_LINK_AN | YT921X_PORT_RX_MAC_EN |
+	       YT921X_PORT_TX_MAC_EN;
+	res = yt921x_reg_clear_bits(priv, YT921X_PORTn_CTRL(port), mask);
+	if (res)
+		return res;
+
+	if (BIT(port) & info->serdes_mask) {
+		mask = YT921X_SERDES_LINK;
+		res = yt921x_reg_clear_bits(priv, YT921X_SERDESn(port), mask);
+		if (res)
+			return res;
+	}
+	if (BIT(port) & info->xmii_mask) {
+		mask = YT921X_XMII_LINK;
+		res = yt921x_reg_clear_bits(priv, YT921X_XMIIn(port), mask);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_port_up(struct yt921x_priv *priv, int port, unsigned int mode,
+	       phy_interface_t interface, int speed, int duplex,
+	       bool tx_pause, bool rx_pause)
+{
+	const struct yt921x_info *info = priv->info;
+	struct yt921x_port *pp = &priv->ports[port];
+	int ps = ethtool_speed_to_yt921x(speed);
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	if (ps == YT921X_SPEED_INVALID)
+		return -EINVAL;
+
+	mask = YT921X_PORT_SPEED_M | YT921X_PORT_TX_MAC_EN |
+	       YT921X_PORT_RX_MAC_EN | YT921X_PORT_TX_PAUSE |
+	       YT921X_PORT_RX_PAUSE | YT921X_PORT_DUPLEX_FULL |
+	       YT921X_PORT_CTRL_LINK_AN;
+	ctrl = YT921X_PORT_SPEED(ps);
+	if (duplex == DUPLEX_FULL)
+		ctrl |= YT921X_PORT_DUPLEX_FULL;
+	if (tx_pause)
+		ctrl |= YT921X_PORT_TX_PAUSE;
+	if (rx_pause)
+		ctrl |= YT921X_PORT_RX_PAUSE;
+	ctrl |= YT921X_PORT_RX_MAC_EN | YT921X_PORT_TX_MAC_EN;
+	if (pp->serdes && pp->inband)
+		ctrl |= YT921X_PORT_CTRL_LINK_AN;
+	res = yt921x_reg_update_bits(priv, YT921X_PORTn_CTRL(port), mask, ctrl);
+	if (res)
+		return res;
+
+	if (!(BIT(port) & (info->serdes_mask | info->xmii_mask)))
+		return 0;
+
+	if (pp->serdes) {
+		mask = YT921X_SERDES_SPEED_M;
+		ctrl = YT921X_SERDES_SPEED(ps);
+		mask |= YT921X_SERDES_DUPLEX_FULL;
+		if (duplex == DUPLEX_FULL)
+			ctrl |= YT921X_SERDES_DUPLEX_FULL;
+		mask |= YT921X_SERDES_TX_PAUSE;
+		if (tx_pause)
+			ctrl |= YT921X_SERDES_TX_PAUSE;
+		mask |= YT921X_SERDES_RX_PAUSE;
+		if (rx_pause)
+			ctrl |= YT921X_SERDES_RX_PAUSE;
+		mask |= YT921X_SERDES_LINK;
+		if (!pp->inband)
+			ctrl |= YT921X_SERDES_LINK;
+		res = yt921x_reg_update_bits(priv, YT921X_SERDESn(port),
+					     mask, ctrl);
+		if (res)
+			return res;
+	} else {
+		mask = YT921X_XMII_LINK;
+		res = yt921x_reg_set_bits(priv, YT921X_XMIIn(port), mask);
+		if (res)
+			return res;
+
+		ctrl = YT921X_MDIO_POLLING_SPEED(ps);
+		if (duplex == DUPLEX_FULL)
+			ctrl |= YT921X_MDIO_POLLING_DUPLEX_FULL;
+		ctrl |= YT921X_MDIO_POLLING_LINK;
+		res = yt921x_reg_write(priv, YT921X_MDIO_POLLINGn(port), ctrl);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_port_config(struct yt921x_priv *priv, int port, unsigned int mode,
+		   phy_interface_t interface)
+{
+	const struct yt921x_info *info = priv->info;
+	struct yt921x_port *pp = &priv->ports[port];
+	struct device *dev = to_device(priv);
+	u32 mask;
+	int res;
+
+	if (BIT(port) & info->internal_mask) {
+		if (interface != PHY_INTERFACE_MODE_INTERNAL)
+			goto err;
+		return 0;
+	}
+
+	switch (interface) {
+	/* SERDES */
+	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_100BASEX:
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_2500BASEX:
+		if (!(BIT(port) & info->serdes_mask))
+			goto err;
+
+		mask = YT921X_SERDES_CTRL_PORTn(port);
+		res = yt921x_reg_set_bits(priv, YT921X_SERDES_CTRL, mask);
+		if (res)
+			return res;
+
+		mask = YT921X_XMII_CTRL_PORTn(port);
+		res = yt921x_reg_clear_bits(priv, YT921X_XMII_CTRL, mask);
+		if (res)
+			return res;
+
+		/* The order is quite arbitrary - we can't return to a safe
+		 * state on IO errors.
+		 */
+		pp->serdes = true;
+
+		break;
+	/* add XMII support here */
+	default:
+		goto err;
+	}
+
+	return 0;
+
+err:
+	dev_err(dev, "Wrong mode %d on port %d\n", interface, port);
+	return -EINVAL;
+}
+
+static struct phylink_pcs *
+yt921x_phylink_mac_select_pcs(struct phylink_config *config,
+			      phy_interface_t interface)
+{
+	struct dsa_port *dp = dsa_phylink_to_port(config);
+	struct yt921x_priv *priv = to_yt921x_priv(dp->ds);
+	const struct yt921x_info *info = priv->info;
+	int port = dp->index;
+
+	if (!(BIT(port) & info->serdes_mask))
+		return NULL;
+
+	switch (interface) {
+	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_100BASEX:
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_2500BASEX:
+		return &priv->ports[port].pcs;
+	default:
+		return NULL;
+	}
+}
+
+static void
+yt921x_phylink_mac_link_down(struct phylink_config *config, unsigned int mode,
+			     phy_interface_t interface)
+{
+	struct dsa_port *dp = dsa_phylink_to_port(config);
+	struct yt921x_priv *priv = to_yt921x_priv(dp->ds);
+	int port = dp->index;
+	int res;
+
+	/* No need to sync; port control block is hold until device remove */
+	cancel_delayed_work(&priv->ports[port].mib_read);
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_port_down(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dp->ds->dev, "Failed to %s port %d: %i\n", "bring down",
+			port, res);
+}
+
+static void
+yt921x_phylink_mac_link_up(struct phylink_config *config,
+			   struct phy_device *phydev, unsigned int mode,
+			   phy_interface_t interface, int speed, int duplex,
+			   bool tx_pause, bool rx_pause)
+{
+	struct dsa_port *dp = dsa_phylink_to_port(config);
+	struct yt921x_priv *priv = to_yt921x_priv(dp->ds);
+	int port = dp->index;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_port_up(priv, port, mode, interface, speed, duplex,
+			     tx_pause, rx_pause);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dp->ds->dev, "Failed to %s port %d: %i\n", "bring up",
+			port, res);
+
+	schedule_delayed_work(&priv->ports[port].mib_read, 0);
+}
+
+static void
+yt921x_phylink_mac_config(struct phylink_config *config, unsigned int mode,
+			  const struct phylink_link_state *state)
+{
+	struct dsa_port *dp = dsa_phylink_to_port(config);
+	struct yt921x_priv *priv = to_yt921x_priv(dp->ds);
+	int port = dp->index;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_port_config(priv, port, mode, state->interface);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dp->ds->dev, "Failed to %s port %d: %i\n", "config",
+			port, res);
+}
+
+static void
+yt921x_dsa_phylink_get_caps(struct dsa_switch *ds, int port,
+			    struct phylink_config *config)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	const struct yt921x_info *info = priv->info;
+
+	config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
+				   MAC_10 | MAC_100 | MAC_1000;
+
+	if (info->internal_mask & BIT(port)) {
+		/* Port 10 for MCU should probably go here too. But since that
+		 * is untested yet, turn it down for the moment by letting it
+		 * fall to the default branch.
+		 */
+		__set_bit(PHY_INTERFACE_MODE_INTERNAL,
+			  config->supported_interfaces);
+		return;
+	}
+
+	if (BIT(port) & info->serdes_mask) {
+		__set_bit(PHY_INTERFACE_MODE_SGMII,
+			  config->supported_interfaces);
+		/* REVSGMII (SGMII in PHY role) should go here, once
+		 * PHY_INTERFACE_MODE_REVSGMII is introduced.
+		 */
+		__set_bit(PHY_INTERFACE_MODE_100BASEX,
+			  config->supported_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_1000BASEX,
+			  config->supported_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_2500BASEX,
+			  config->supported_interfaces);
+		config->mac_capabilities |= MAC_2500FD;
+	}
+	if (BIT(port) & info->xmii_mask) {
+		/* Not tested. To add support for XMII:
+		 *   - Add proper interface modes below
+		 *   - Handle them in yt921x_port_config()
+		 */
+	}
+	/* no such port: empty supported_interfaces causes phylink to turn it
+	 * down
+	 */
+}
+
+static int yt921x_port_setup(struct yt921x_priv *priv, int port)
+{
+	struct dsa_switch *ds = &priv->ds;
+	u32 ctrl;
+	int res;
+
+	res = yt921x_userport_standalone(priv, port);
+	if (res)
+		return res;
+
+	/* Clear prio order (even if DCB is not enabled) to avoid unsolicited
+	 * priorities
+	 */
+	res = yt921x_reg_write(priv, YT921X_PORTn_PRIO_ORD(port), 0);
+	if (res)
+		return res;
+
+	if (dsa_is_cpu_port(ds, port)) {
+		/* Egress of CPU port is supposed to be completely controlled
+		 * via tagging, so set to oneway isolated (drop all packets
+		 * without tag).
+		 */
+		ctrl = ~(u32)0;
+		res = yt921x_reg_write(priv, YT921X_PORTn_ISOLATION(port),
+				       ctrl);
+		if (res)
+			return res;
+
+		/* To simplify FDB "isolation" simulation, we also disable
+		 * learning on the CPU port, and let software identify packets
+		 * towarding CPU (either trapped or a static FDB entry is
+		 * matched, no matter which bridge that entry is for), which is
+		 * already done by yt921x_userport_standalone(). As a result,
+		 * VLAN-awareness becomes unrelated on the CPU port (set to
+		 * VLAN-unaware by the way).
+		 */
+	}
+
+	return 0;
+}
+
+static enum dsa_tag_protocol
+yt921x_dsa_get_tag_protocol(struct dsa_switch *ds, int port,
+			    enum dsa_tag_protocol m)
+{
+	return DSA_TAG_PROTO_YT921X;
+}
+
+static int yt921x_dsa_port_setup(struct dsa_switch *ds, int port)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_port_setup(priv, port);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+/* Not "port" - DSCP mapping is global */
+static int __maybe_unused
+yt921x_dsa_port_get_dscp_prio(struct dsa_switch *ds, int port, u8 dscp)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u32 val;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_read(priv, YT921X_IPM_DSCPn(dscp), &val);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		return res;
+
+	return FIELD_GET(YT921X_IPM_PRIO_M, val);
+}
+
+static int __maybe_unused
+yt921x_dsa_port_del_dscp_prio(struct dsa_switch *ds, int port, u8 dscp, u8 prio)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	u32 val;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	/* During a "dcb app replace" command, the new app table entry will be
+	 * added first, then the old one will be deleted. But the hardware only
+	 * supports one QoS class per DSCP value (duh), so if we blindly delete
+	 * the app table entry for this DSCP value, we end up deleting the
+	 * entry with the new priority. Avoid that by checking whether user
+	 * space wants to delete the priority which is currently configured, or
+	 * something else which is no longer current.
+	 */
+	res = yt921x_reg_read(priv, YT921X_IPM_DSCPn(dscp), &val);
+	if (!res && FIELD_GET(YT921X_IPM_PRIO_M, val) == prio)
+		res = yt921x_reg_write(priv, YT921X_IPM_DSCPn(dscp),
+				       YT921X_IPM_PRIO(IEEE8021Q_TT_BK));
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int __maybe_unused
+yt921x_dsa_port_add_dscp_prio(struct dsa_switch *ds, int port, u8 dscp, u8 prio)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	int res;
+
+	if (prio >= YT921X_PRIO_NUM)
+		return -EINVAL;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_write(priv, YT921X_IPM_DSCPn(dscp),
+			       YT921X_IPM_PRIO(prio));
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int yt921x_edata_wait(struct yt921x_priv *priv, u32 *valp)
+{
+	u32 val = YT921X_EDATA_DATA_IDLE;
+	int res;
+
+	res = yt921x_reg_wait(priv, YT921X_EDATA_DATA,
+			      YT921X_EDATA_DATA_STATUS_M, &val);
+	if (res)
+		return res;
+
+	*valp = val;
+	return 0;
+}
+
+static int
+yt921x_edata_read_cont(struct yt921x_priv *priv, u8 addr, u8 *valp)
+{
+	u32 ctrl;
+	u32 val;
+	int res;
+
+	ctrl = YT921X_EDATA_CTRL_ADDR(addr) | YT921X_EDATA_CTRL_READ;
+	res = yt921x_reg_write(priv, YT921X_EDATA_CTRL, ctrl);
+	if (res)
+		return res;
+	res = yt921x_edata_wait(priv, &val);
+	if (res)
+		return res;
+
+	*valp = FIELD_GET(YT921X_EDATA_DATA_DATA_M, val);
+	return 0;
+}
+
+static int yt921x_edata_read(struct yt921x_priv *priv, u8 addr, u8 *valp)
+{
+	u32 val;
+	int res;
+
+	res = yt921x_edata_wait(priv, &val);
+	if (res)
+		return res;
+	return yt921x_edata_read_cont(priv, addr, valp);
+}
+
+static void yt921x_dsa_teardown(struct dsa_switch *ds)
+{
+#if IS_ENABLED(CONFIG_NET_DSA_YT921X_LEDS)
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+
+	yt921x_leds_remove(priv);
+#endif
+}
+
+static int yt921x_chip_detect(struct yt921x_priv *priv)
+{
+	struct device *dev = to_device(priv);
+	const struct yt921x_info *info;
+	u8 extmode;
+	u32 chipid;
+	u32 major;
+	u32 mode;
+	u32 val;
+	int res;
+
+	res = yt921x_reg_read(priv, YT921X_CHIP_ID, &chipid);
+	if (res)
+		return res;
+
+	major = FIELD_GET(YT921X_CHIP_ID_MAJOR, chipid);
+
+	for (info = yt921x_infos; info->name; info++)
+		if (info->major == major)
+			break;
+	if (!info->name) {
+		dev_err(dev, "Unexpected chipid 0x%x\n", chipid);
+		return -ENODEV;
+	}
+
+	res = yt921x_reg_read(priv, YT921X_CHIP_MODE, &mode);
+	if (res)
+		return res;
+	res = yt921x_edata_read(priv, YT921X_EDATA_EXTMODE, &extmode);
+	if (res)
+		return res;
+
+	for (; info->name; info++)
+		if (info->major == major && info->mode == mode &&
+		    info->extmode == extmode)
+			break;
+	if (!info->name) {
+		dev_err(dev,
+			"Unsupported chipid 0x%x with chipmode 0x%x 0x%x\n",
+			chipid, mode, extmode);
+		return -ENODEV;
+	}
+
+	res = yt921x_reg_read(priv, YT921X_SYS_CLK, &val);
+	if (res)
+		return res;
+	switch (FIELD_GET(YT921X_SYS_CLK_SEL_M, val)) {
+	case 0:
+		priv->cycle_ns = info->major == YT9215_MAJOR ? 8 : 6;
+		break;
+	case YT921X_SYS_CLK_143M:
+		priv->cycle_ns = 7;
+		break;
+	default:
+		priv->cycle_ns = 8;
+	}
+
+	/* Print chipid here since we are interested in lower 16 bits */
+	dev_info(dev,
+		 "Motorcomm %s ethernet switch, chipid: 0x%x, chipmode: 0x%x 0x%x\n",
+		 info->name, chipid, mode, extmode);
+
+	priv->info = info;
+
+	return 0;
+}
+
+static int yt921x_chip_reset(struct yt921x_priv *priv)
+{
+	struct device *dev = to_device(priv);
+	u16 eth_p_tag;
+	u32 val;
+	int res;
+
+	res = yt921x_chip_detect(priv);
+	if (res)
+		return res;
+
+	/* Reset */
+	res = yt921x_reg_write(priv, YT921X_RST, YT921X_RST_HW);
+	if (res)
+		return res;
+
+	/* RST_HW is almost same as GPIO hard reset, so we need this delay. */
+	fsleep(YT921X_RST_DELAY_US);
+
+	val = 0;
+	res = yt921x_reg_wait(priv, YT921X_RST, ~0, &val);
+	if (res)
+		return res;
+
+	/* Check for tag EtherType; do it after reset in case you messed it up
+	 * before.
+	 */
+	res = yt921x_reg_read(priv, YT921X_CPU_TAG_TPID, &val);
+	if (res)
+		return res;
+	eth_p_tag = FIELD_GET(YT921X_CPU_TAG_TPID_TPID_M, val);
+	if (eth_p_tag != ETH_P_YT921X) {
+		dev_err(dev, "Tag type 0x%x != 0x%x\n", eth_p_tag,
+			ETH_P_YT921X);
+		/* Despite being possible, we choose not to set CPU_TAG_TPID,
+		 * since there is no way it can be different unless you have the
+		 * wrong chip.
+		 */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int yt921x_chip_setup_dsa(struct yt921x_priv *priv)
+{
+	struct dsa_switch *ds = &priv->ds;
+	unsigned long cpu_ports_mask;
+	u32 ctrls[2];
+	u32 ctrl;
+	int port;
+	int res;
+
+	/* Enable DSA */
+	priv->cpu_ports_mask = dsa_cpu_ports(ds);
+
+	ctrl = YT921X_EXT_CPU_PORT_TAG_EN | YT921X_EXT_CPU_PORT_PORT_EN |
+	       YT921X_EXT_CPU_PORT_PORT(__ffs(priv->cpu_ports_mask));
+	res = yt921x_reg_write(priv, YT921X_EXT_CPU_PORT, ctrl);
+	if (res)
+		return res;
+
+	/* Setup software switch */
+	ctrl = YT921X_CPU_COPY_TO_EXT_CPU;
+	res = yt921x_reg_write(priv, YT921X_CPU_COPY, ctrl);
+	if (res)
+		return res;
+
+	ctrl = GENMASK(10, 0);
+	res = yt921x_reg_write(priv, YT921X_FILTER_UNK_UCAST, ctrl);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_FILTER_UNK_MCAST, ctrl);
+	if (res)
+		return res;
+
+	/* YT921x does not support native DSA port bridging, so we use port
+	 * isolation to emulate it. However, be especially careful that port
+	 * isolation takes _after_ FDB lookups, i.e. if an FDB entry (from
+	 * another bridge) is matched and the destination port (in another
+	 * bridge) is blocked, the packet will be dropped instead of flooding to
+	 * the "bridged" ports, thus we need to trap and handle those packets by
+	 * software.
+	 *
+	 * If there is no more than one bridge, we might be able to drop them
+	 * directly given some conditions are met, but we trap them in all cases
+	 * for now.
+	 */
+	ctrl = 0;
+	for (int i = 0; i < YT921X_PORT_NUM; i++)
+		ctrl |= YT921X_ACT_UNK_ACTn_TRAP(i);
+	/* Except for CPU ports, if any packets are sent via CPU ports without
+	 * tag, they should be dropped.
+	 */
+	cpu_ports_mask = priv->cpu_ports_mask;
+	for_each_set_bit(port, &cpu_ports_mask, YT921X_PORT_NUM) {
+		ctrl &= ~YT921X_ACT_UNK_ACTn_M(port);
+		ctrl |= YT921X_ACT_UNK_ACTn_DROP(port);
+	}
+	res = yt921x_reg_write(priv, YT921X_ACT_UNK_UCAST, ctrl);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_ACT_UNK_MCAST, ctrl);
+	if (res)
+		return res;
+
+	/* Tagged VID 0 should be treated as untagged, which confuses the
+	 * hardware a lot
+	 */
+	ctrls[0] = YT921X_VLAN_CTRLa_LEARN_DIS | YT921X_VLAN_CTRLa_PORTS_M;
+	ctrls[1] = 0;
+	res = yt921x_reg64_write(priv, YT921X_VLANn_CTRL(0), ctrls);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int yt921x_chip_setup_tc(struct yt921x_priv *priv)
+{
+	unsigned int op_ns;
+	u32 ctrl;
+	int res;
+
+	op_ns = 8 * priv->cycle_ns;
+
+	ctrl = max(priv->meter_slot_ns / op_ns, YT921X_METER_SLOT_MIN);
+	res = yt921x_reg_write(priv, YT921X_METER_SLOT, ctrl);
+	if (res)
+		return res;
+	priv->meter_slot_ns = ctrl * op_ns;
+
+	ctrl = max(priv->port_shape_slot_ns / op_ns,
+		   YT921X_PORT_SHAPE_SLOT_MIN);
+	res = yt921x_reg_write(priv, YT921X_PORT_SHAPE_SLOT, ctrl);
+	if (res)
+		return res;
+	priv->port_shape_slot_ns = ctrl * op_ns;
+
+	return 0;
+}
+
+static int yt921x_chip_setup_acl(struct yt921x_priv *priv)
+{
+	u32 ctrl;
+	int res;
+
+	ctrl = YT921X_ACL_PERMIT_UNMATCH_PORTS_M;
+	res = yt921x_reg_write(priv, YT921X_ACL_PERMIT_UNMATCH, ctrl);
+	if (res)
+		return res;
+
+	ctrl = YT921X_ACL_PORT_PORTS_M;
+	res = yt921x_reg_write(priv, YT921X_ACL_PORT, ctrl);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int __maybe_unused yt921x_chip_setup_qos(struct yt921x_priv *priv)
+{
+	u32 ctrl;
+	int res;
+
+	/* DSCP to internal priorities */
+	for (u8 dscp = 0; dscp < DSCP_MAX; dscp++) {
+		int prio = ietf_dscp_to_ieee8021q_tt(dscp);
+
+		if (prio < 0)
+			return prio;
+
+		res = yt921x_reg_write(priv, YT921X_IPM_DSCPn(dscp),
+				       YT921X_IPM_PRIO(prio));
+		if (res)
+			return res;
+	}
+
+	/* 802.1Q QoS to internal priorities */
+	for (u8 pcp = 0; pcp < 8; pcp++)
+		for (u8 dei = 0; dei < 2; dei++) {
+			ctrl = YT921X_IPM_PRIO(pcp);
+			if (dei)
+				/* "Red" almost means drop, so it's not that
+				 * useful. Note that tc police does not support
+				 * Three-Color very well
+				 */
+				ctrl |= YT921X_IPM_COLOR_YELLOW;
+
+			for (u8 svlan = 0; svlan < 2; svlan++) {
+				u32 reg = YT921X_IPM_PCPn(svlan, dei, pcp);
+
+				res = yt921x_reg_write(priv, reg, ctrl);
+				if (res)
+					return res;
+			}
+		}
+
+	return 0;
+}
+
+static int yt921x_chip_setup(struct yt921x_priv *priv)
+{
+	u32 ctrl;
+	int res;
+
+	ctrl = YT921X_FUNC_MIB | YT921X_FUNC_ACL | YT921X_FUNC_METER;
+	res = yt921x_reg_set_bits(priv, YT921X_FUNC, ctrl);
+	if (res)
+		return res;
+
+	res = yt921x_chip_setup_dsa(priv);
+	if (res)
+		return res;
+
+	res = yt921x_chip_setup_tc(priv);
+	if (res)
+		return res;
+
+	res = yt921x_chip_setup_acl(priv);
+	if (res)
+		return res;
+
+#if IS_ENABLED(CONFIG_DCB)
+	res = yt921x_chip_setup_qos(priv);
+	if (res)
+		return res;
+#endif
+
+	/* Clear MIB */
+	ctrl = YT921X_MIB_CTRL_CLEAN | YT921X_MIB_CTRL_ALL_PORT;
+	res = yt921x_reg_write(priv, YT921X_MIB_CTRL, ctrl);
+	if (res)
+		return res;
+
+	/* Miscellaneous */
+	res = yt921x_reg_set_bits(priv, YT921X_SENSOR, YT921X_SENSOR_TEMP);
+	if (res)
+		return res;
+
+	res = yt921x_reg_clear_bits(priv, YT921X_SERDES_CTRL,
+				    YT921X_SERDES_CTRL_TEST);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int yt921x_dsa_setup(struct dsa_switch *ds)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct device *dev = to_device(priv);
+	struct device_node *np = dev->of_node;
+	struct device_node *child;
+	unsigned long mask;
+	int port;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_chip_reset(priv);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		return res;
+
+	/* Register the internal mdio bus. Nodes for internal ports should have
+	 * proper phy-handle pointing to their PHYs. Not enabling the internal
+	 * bus is possible, though pretty wired, if internal ports are not used.
+	 */
+	child = of_get_child_by_name(np, "mdio");
+	if (child) {
+		res = yt921x_mbus_int_init(priv, child);
+		of_node_put(child);
+		if (res)
+			return res;
+	}
+
+	/* External mdio bus is optional */
+	child = of_get_child_by_name(np, "mdio-external");
+	if (child) {
+		res = yt921x_mbus_ext_init(priv, child);
+		of_node_put(child);
+		if (res)
+			return res;
+
+		dev_err(dev, "Untested external mdio bus\n");
+		return -ENODEV;
+	}
+
+	mask = priv->info->serdes_mask;
+	for_each_set_bit(port, &mask, YT921X_PORT_NUM) {
+		struct yt921x_port *pp = &priv->ports[port];
+
+		pp->pcs.ops = &yt921x_phylink_pcs_ops;
+		pp->pcs.poll = true;
+
+		__set_bit(PHY_INTERFACE_MODE_SGMII,
+			  pp->pcs.supported_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_100BASEX,
+			  pp->pcs.supported_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_1000BASEX,
+			  pp->pcs.supported_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_2500BASEX,
+			  pp->pcs.supported_interfaces);
+	}
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_chip_setup(priv);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		return res;
+
+#if IS_ENABLED(CONFIG_NET_DSA_YT921X_LEDS)
+	res = yt921x_leds_setup(priv);
+	if (res)
+		dev_warn(dev, "Failed to setup LEDs: %d\n", res);
+#endif
+
+	return 0;
+}
+
+static const struct phylink_mac_ops yt921x_phylink_mac_ops = {
+	.mac_select_pcs	= yt921x_phylink_mac_select_pcs,
+	.mac_link_down	= yt921x_phylink_mac_link_down,
+	.mac_link_up	= yt921x_phylink_mac_link_up,
+	.mac_config	= yt921x_phylink_mac_config,
+};
+
+static const struct dsa_switch_ops yt921x_dsa_switch_ops = {
+	/* mib */
+	.get_strings		= yt921x_dsa_get_strings,
+	.get_ethtool_stats	= yt921x_dsa_get_ethtool_stats,
+	.get_sset_count		= yt921x_dsa_get_sset_count,
+	.get_eth_mac_stats	= yt921x_dsa_get_eth_mac_stats,
+	.get_eth_ctrl_stats	= yt921x_dsa_get_eth_ctrl_stats,
+	.get_rmon_stats		= yt921x_dsa_get_rmon_stats,
+	.get_stats64		= yt921x_dsa_get_stats64,
+	.get_pause_stats	= yt921x_dsa_get_pause_stats,
+	/* eee */
+	.support_eee		= dsa_supports_eee,
+	.set_mac_eee		= yt921x_dsa_set_mac_eee,
+	/* mtu */
+	.port_change_mtu	= yt921x_dsa_port_change_mtu,
+	.port_max_mtu		= yt921x_dsa_port_max_mtu,
+	/* rate */
+	.port_policer_del	= yt921x_dsa_port_policer_del,
+	.port_policer_add	= yt921x_dsa_port_policer_add,
+	.port_setup_tc		= yt921x_dsa_port_setup_tc,
+	/* acl */
+	.cls_flower_del		= yt921x_dsa_cls_flower_del,
+	.cls_flower_add		= yt921x_dsa_cls_flower_add,
+	/* hsr */
+	.port_hsr_leave		= dsa_port_simple_hsr_leave,
+	.port_hsr_join		= dsa_port_simple_hsr_join,
+	/* mirror */
+	.port_mirror_del	= yt921x_dsa_port_mirror_del,
+	.port_mirror_add	= yt921x_dsa_port_mirror_add,
+	/* lag */
+	.port_lag_leave		= yt921x_dsa_port_lag_leave,
+	.port_lag_join		= yt921x_dsa_port_lag_join,
+	/* fdb */
+	.port_fdb_dump		= yt921x_dsa_port_fdb_dump,
+	.port_fast_age		= yt921x_dsa_port_fast_age,
+	.set_ageing_time	= yt921x_dsa_set_ageing_time,
+	.port_fdb_del		= yt921x_dsa_port_fdb_del,
+	.port_fdb_add		= yt921x_dsa_port_fdb_add,
+	.port_mdb_del		= yt921x_dsa_port_mdb_del,
+	.port_mdb_add		= yt921x_dsa_port_mdb_add,
+	/* vlan */
+	.port_vlan_filtering	= yt921x_dsa_port_vlan_filtering,
+	.port_vlan_del		= yt921x_dsa_port_vlan_del,
+	.port_vlan_add		= yt921x_dsa_port_vlan_add,
+	/* bridge */
+	.port_pre_bridge_flags	= yt921x_dsa_port_pre_bridge_flags,
+	.port_bridge_flags	= yt921x_dsa_port_bridge_flags,
+	.port_bridge_leave	= yt921x_dsa_port_bridge_leave,
+	.port_bridge_join	= yt921x_dsa_port_bridge_join,
+	/* mst */
+	.port_mst_state_set	= yt921x_dsa_port_mst_state_set,
+	.vlan_msti_set		= yt921x_dsa_vlan_msti_set,
+	.port_stp_state_set	= yt921x_dsa_port_stp_state_set,
+#if IS_ENABLED(CONFIG_DCB)
+	/* dcb */
+	.port_get_default_prio	= yt921x_dsa_port_get_default_prio,
+	.port_set_default_prio	= yt921x_dsa_port_set_default_prio,
+	.port_get_apptrust	= yt921x_dsa_port_get_apptrust,
+	.port_set_apptrust	= yt921x_dsa_port_set_apptrust,
+#endif
+	/* port */
+	.get_tag_protocol	= yt921x_dsa_get_tag_protocol,
+	.phylink_get_caps	= yt921x_dsa_phylink_get_caps,
+	.port_setup		= yt921x_dsa_port_setup,
+#if IS_ENABLED(CONFIG_DCB)
+	/* dscp */
+	.port_get_dscp_prio	= yt921x_dsa_port_get_dscp_prio,
+	.port_del_dscp_prio	= yt921x_dsa_port_del_dscp_prio,
+	.port_add_dscp_prio	= yt921x_dsa_port_add_dscp_prio,
+#endif
+	/* chip */
+	.teardown		= yt921x_dsa_teardown,
+	.setup			= yt921x_dsa_setup,
+};
+
+static void yt921x_mdio_shutdown(struct mdio_device *mdiodev)
+{
+	struct yt921x_priv *priv = mdiodev_get_drvdata(mdiodev);
+
+	if (!priv)
+		return;
+
+	dsa_switch_shutdown(&priv->ds);
+}
+
+static void yt921x_mdio_remove(struct mdio_device *mdiodev)
+{
+	struct yt921x_priv *priv = mdiodev_get_drvdata(mdiodev);
+
+	if (!priv)
+		return;
+
+	for (size_t i = ARRAY_SIZE(priv->ports); i-- > 0; ) {
+		struct yt921x_port *pp = &priv->ports[i];
+
+		disable_delayed_work_sync(&pp->mib_read);
+	}
+
+	dsa_unregister_switch(&priv->ds);
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(priv->acl_blks); i++) {
+		struct yt921x_acl_blk *aclblk = priv->acl_blks[i];
+
+		if (!aclblk)
+			continue;
+		for (unsigned int j = 0; j < ARRAY_SIZE(aclblk->rules); j++) {
+			struct yt921x_acl_rule *aclrule = aclblk->rules[j];
+
+			if (!aclrule)
+				continue;
+			kvfree(aclrule);
+		}
+		kvfree(aclblk);
+	}
+	mutex_destroy(&priv->reg_lock);
+}
+
+static int yt921x_mdio_probe(struct mdio_device *mdiodev)
+{
+	struct device *dev = &mdiodev->dev;
+	struct yt921x_reg_mdio *mdio;
+	struct yt921x_priv *priv;
+	struct dsa_switch *ds;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	mdio = devm_kzalloc(dev, sizeof(*mdio), GFP_KERNEL);
+	if (!mdio)
+		return -ENOMEM;
+
+	mdio->bus = mdiodev->bus;
+	mdio->addr = mdiodev->addr;
+	mdio->switchid = 0;
+
+	mutex_init(&priv->reg_lock);
+
+	priv->reg_ops = &yt921x_reg_ops_mdio;
+	priv->reg_ctx = mdio;
+
+	for (size_t i = 0; i < ARRAY_SIZE(priv->ports); i++) {
+		struct yt921x_port *pp = &priv->ports[i];
+
+		pp->index = i;
+		INIT_DELAYED_WORK(&pp->mib_read, yt921x_poll_mib);
+	}
+
+	ds = &priv->ds;
+	ds->dev = dev;
+	ds->assisted_learning_on_cpu_port = true;
+	ds->dscp_prio_mapping_is_global = true;
+	ds->priv = priv;
+	ds->ops = &yt921x_dsa_switch_ops;
+	ds->ageing_time_min = 1 * 5000;
+	ds->ageing_time_max = U16_MAX * 5000;
+	ds->phylink_mac_ops = &yt921x_phylink_mac_ops;
+	ds->num_lag_ids = YT921X_LAG_NUM;
+	ds->num_ports = YT921X_PORT_NUM;
+
+	mdiodev_set_drvdata(mdiodev, priv);
+
+	return dsa_register_switch(ds);
+}
+
+static const struct of_device_id yt921x_of_match[] = {
+	{ .compatible = "motorcomm,yt9215" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, yt921x_of_match);
+
+static struct mdio_driver yt921x_mdio_driver = {
+	.probe = yt921x_mdio_probe,
+	.remove = yt921x_mdio_remove,
+	.shutdown = yt921x_mdio_shutdown,
+	.mdiodrv.driver = {
+		.name = YT921X_NAME,
+		.of_match_table = yt921x_of_match,
+	},
+};
+
+mdio_module_driver(yt921x_mdio_driver);
+
+MODULE_AUTHOR("David Yang <mmyangfl@gmail.com>");
+MODULE_DESCRIPTION("Driver for Motorcomm YT921x Switch");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/dsa/motorcomm/chip.h b/drivers/net/dsa/motorcomm/chip.h
new file mode 100644
index 00000000000000..28e4f26f9cc49b
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/chip.h
@@ -0,0 +1,1008 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2025 David Yang
+ */
+
+#ifndef __YT921X_H
+#define __YT921X_H
+
+#include <net/dsa.h>
+
+#define YT921X_SMI_SWITCHID_M		GENMASK(3, 2)
+#define  YT921X_SMI_SWITCHID(x)			FIELD_PREP(YT921X_SMI_SWITCHID_M, (x))
+#define YT921X_SMI_AD			BIT(1)
+#define  YT921X_SMI_ADDR			0
+#define  YT921X_SMI_DATA			YT921X_SMI_AD
+#define YT921X_SMI_RW			BIT(0)
+#define  YT921X_SMI_WRITE			0
+#define  YT921X_SMI_READ			YT921X_SMI_RW
+
+#define YT921X_SWITCHID_NUM		4
+
+#define YT921X_RST			0x80000
+#define  YT921X_RST_HW				BIT(31)
+#define  YT921X_RST_SW				BIT(1)
+#define YT921X_FUNC			0x80004
+#define  YT921X_FUNC_METER			BIT(4)
+#define  YT921X_FUNC_ACL			BIT(2)
+#define  YT921X_FUNC_MIB			BIT(1)
+#define YT921X_CHIP_ID			0x80008
+#define  YT921X_CHIP_ID_MAJOR			GENMASK(31, 16)
+#define YT921X_EXT_CPU_PORT		0x8000c
+#define  YT921X_EXT_CPU_PORT_TAG_EN		BIT(15)
+#define  YT921X_EXT_CPU_PORT_PORT_EN		BIT(14)
+#define  YT921X_EXT_CPU_PORT_PORT_M		GENMASK(3, 0)
+#define   YT921X_EXT_CPU_PORT_PORT(x)			FIELD_PREP(YT921X_EXT_CPU_PORT_PORT_M, (x))
+#define YT921X_CPU_TAG_TPID		0x80010
+#define  YT921X_CPU_TAG_TPID_TPID_M		GENMASK(15, 0)
+/* Same as ETH_P_YT921X, but this represents the true HW default, while the
+ * former is a local convention chosen by us.
+ */
+#define   YT921X_CPU_TAG_TPID_TPID_DEFAULT		0x9988
+#define YT921X_PVID_SEL			0x80014
+#define  YT921X_PVID_SEL_SVID_PORTn(port)	BIT(port)
+#define YT921X_SERDES_CTRL		0x80028
+#define  YT921X_SERDES_CTRL_TEST		BIT(6)
+#define  YT921X_SERDES_CTRL_PORTn(port)		BIT((port) - 8)
+#define YT921X_IO_LEVEL			0x80030
+#define  YT9215_IO_LEVEL_NORMAL_M		GENMASK(5, 4)
+#define   YT9215_IO_LEVEL_NORMAL(x)			FIELD_PREP(YT9215_IO_LEVEL_NORMAL_M, (x))
+#define   YT9215_IO_LEVEL_NORMAL_3V3			YT9215_IO_LEVEL_NORMAL(0)
+#define   YT9215_IO_LEVEL_NORMAL_1V8			YT9215_IO_LEVEL_NORMAL(3)
+#define  YT9215_IO_LEVEL_RGMII1_M		GENMASK(3, 2)
+#define   YT9215_IO_LEVEL_RGMII1(x)			FIELD_PREP(YT9215_IO_LEVEL_RGMII1_M, (x))
+#define   YT9215_IO_LEVEL_RGMII1_3V3			YT9215_IO_LEVEL_RGMII1(0)
+#define   YT9215_IO_LEVEL_RGMII1_2V5			YT9215_IO_LEVEL_RGMII1(1)
+#define   YT9215_IO_LEVEL_RGMII1_1V8			YT9215_IO_LEVEL_RGMII1(2)
+#define  YT9215_IO_LEVEL_RGMII0_M		GENMASK(1, 0)
+#define   YT9215_IO_LEVEL_RGMII0(x)			FIELD_PREP(YT9215_IO_LEVEL_RGMII0_M, (x))
+#define   YT9215_IO_LEVEL_RGMII0_3V3			YT9215_IO_LEVEL_RGMII0(0)
+#define   YT9215_IO_LEVEL_RGMII0_2V5			YT9215_IO_LEVEL_RGMII0(1)
+#define   YT9215_IO_LEVEL_RGMII0_1V8			YT9215_IO_LEVEL_RGMII0(2)
+#define  YT9218_IO_LEVEL_RGMII1_M		GENMASK(5, 4)
+#define   YT9218_IO_LEVEL_RGMII1(x)			FIELD_PREP(YT9218_IO_LEVEL_RGMII1_M, (x))
+#define   YT9218_IO_LEVEL_RGMII1_3V3			YT9218_IO_LEVEL_RGMII1(0)
+#define   YT9218_IO_LEVEL_RGMII1_2V5			YT9218_IO_LEVEL_RGMII1(1)
+#define   YT9218_IO_LEVEL_RGMII1_1V8			YT9218_IO_LEVEL_RGMII1(2)
+#define  YT9218_IO_LEVEL_RGMII0_M		GENMASK(3, 2)
+#define   YT9218_IO_LEVEL_RGMII0(x)			FIELD_PREP(YT9218_IO_LEVEL_RGMII0_M, (x))
+#define   YT9218_IO_LEVEL_RGMII0_3V3			YT9218_IO_LEVEL_RGMII0(0)
+#define   YT9218_IO_LEVEL_RGMII0_2V5			YT9218_IO_LEVEL_RGMII0(1)
+#define   YT9218_IO_LEVEL_RGMII0_1V8			YT9218_IO_LEVEL_RGMII0(2)
+#define  YT9218_IO_LEVEL_NORMAL_M		GENMASK(1, 0)
+#define   YT9218_IO_LEVEL_NORMAL(x)			FIELD_PREP(YT9218_IO_LEVEL_NORMAL_M, (x))
+#define   YT9218_IO_LEVEL_NORMAL_3V3			YT9218_IO_LEVEL_NORMAL(0)
+#define   YT9218_IO_LEVEL_NORMAL_1V8			YT9218_IO_LEVEL_NORMAL(3)
+#define YT921X_MAC_ADDR_HI2		0x80080
+#define YT921X_MAC_ADDR_LO4		0x80084
+#define YT921X_SERDESn(port)		(0x8008c + 4 * ((port) - 8))
+#define  YT921X_SERDES_MODE_M			GENMASK(9, 7)
+#define   YT921X_SERDES_MODE(x)				FIELD_PREP(YT921X_SERDES_MODE_M, (x))
+#define   YT921X_SERDES_MODE_SGMII			YT921X_SERDES_MODE(0)
+#define   YT921X_SERDES_MODE_REVSGMII			YT921X_SERDES_MODE(1)
+#define   YT921X_SERDES_MODE_1000BASEX			YT921X_SERDES_MODE(2)
+#define   YT921X_SERDES_MODE_100BASEX			YT921X_SERDES_MODE(3)
+#define   YT921X_SERDES_MODE_2500BASEX			YT921X_SERDES_MODE(4)
+#define  YT921X_SERDES_RX_PAUSE			BIT(6)
+#define  YT921X_SERDES_TX_PAUSE			BIT(5)
+#define  YT921X_SERDES_LINK			BIT(4)  /* force link */
+#define  YT921X_SERDES_DUPLEX_FULL		BIT(3)
+#define  YT921X_SERDES_SPEED_M			GENMASK(2, 0)
+#define   YT921X_SERDES_SPEED(x)			FIELD_PREP(YT921X_SERDES_SPEED_M, (x))
+#define YT921X_PORTn_CTRL(port)		(0x80100 + 4 * (port))  /* Bit 7-0 shared with PORT_STATUS */
+#define  YT921X_PORT_CTRL_PAUSE_AN		BIT(10)
+#define  YT921X_PORT_CTRL_LINK_AN		BIT(9)
+#define  YT921X_PORT_CTRL_HALF_PAUSE		BIT(8)  /* Half-duplex back pressure mode */
+#define YT921X_PORTn_STATUS(port)	(0x80200 + 4 * (port))
+#define  YT921X_PORT_LINK			BIT(8)
+#define  YT921X_PORT_DUPLEX_FULL		BIT(7)
+#define  YT921X_PORT_RX_PAUSE			BIT(6)
+#define  YT921X_PORT_TX_PAUSE			BIT(5)
+#define  YT921X_PORT_RX_MAC_EN			BIT(4)
+#define  YT921X_PORT_TX_MAC_EN			BIT(3)
+#define  YT921X_PORT_SPEED_M			GENMASK(2, 0)
+#define   YT921X_PORT_SPEED(x)				FIELD_PREP(YT921X_PORT_SPEED_M, (x))
+#define YT921X_PON_STRAP_FUNC		0x80320
+#define YT921X_PON_STRAP_VAL		0x80324
+#define YT921X_PON_STRAP_CAP		0x80328
+#define  YT921X_PON_STRAP_EEE			BIT(16)
+#define  YT921X_PON_STRAP_LOOP_DETECT		BIT(7)
+#define YT921X_MDIO_POLLINGn(port)	(0x80364 + 4 * ((port) - 8))
+#define  YT921X_MDIO_POLLING_DUPLEX_FULL	BIT(4)
+#define  YT921X_MDIO_POLLING_LINK		BIT(3)
+#define  YT921X_MDIO_POLLING_SPEED_M		GENMASK(2, 0)
+#define   YT921X_MDIO_POLLING_SPEED(x)			FIELD_PREP(YT921X_MDIO_POLLING_SPEED_M, (x))
+
+enum yt921x_speed {
+	YT921X_SPEED_10,
+	YT921X_SPEED_100,
+	YT921X_SPEED_1000,
+	YT921X_SPEED_10000,
+	YT921X_SPEED_2500,
+	YT921X_SPEED_INVALID
+};
+
+#define YT921X_SENSOR			0x8036c
+#define  YT921X_SENSOR_TEMP			BIT(18)
+#define YT921X_TEMP			0x80374
+#define YT921X_CHIP_MODE		0x80388
+#define  YT921X_CHIP_MODE_MODE			GENMASK(1, 0)
+#define YT921X_XMII_CTRL		0x80394
+#define  YT921X_XMII_CTRL_PORTn(port)		BIT(9 - (port))  /* Yes, it's reversed */
+#define YT921X_XMIIn(port)		(0x80400 + 8 * ((port) - 8))
+#define  YT921X_XMII_MODE_M			GENMASK(31, 29)
+#define   YT921X_XMII_MODE(x)				FIELD_PREP(YT921X_XMII_MODE_M, (x))
+#define   YT921X_XMII_MODE_MII				YT921X_XMII_MODE(0)
+#define   YT921X_XMII_MODE_REVMII			YT921X_XMII_MODE(1)
+#define   YT921X_XMII_MODE_RMII				YT921X_XMII_MODE(2)
+#define   YT921X_XMII_MODE_REVRMII			YT921X_XMII_MODE(3)
+#define   YT921X_XMII_MODE_RGMII			YT921X_XMII_MODE(4)
+#define   YT921X_XMII_MODE_DISABLE			YT921X_XMII_MODE(5)
+#define  YT921X_XMII_LINK			BIT(19)  /* force link */
+#define  YT921X_XMII_EN				BIT(18)
+#define  YT921X_XMII_SOFT_RST			BIT(17)
+#define  YT921X_XMII_RGMII_TX_DELAY_150PS_M	GENMASK(16, 13)
+#define   YT921X_XMII_RGMII_TX_DELAY_150PS(x)		FIELD_PREP(YT921X_XMII_RGMII_TX_DELAY_150PS_M, (x))
+#define  YT921X_XMII_TX_CLK_IN			BIT(11)
+#define  YT921X_XMII_RX_CLK_IN			BIT(10)
+#define  YT921X_XMII_RGMII_TX_DELAY_2NS		BIT(8)
+#define  YT921X_XMII_RGMII_TX_CLK_OUT		BIT(7)
+#define  YT921X_XMII_RGMII_RX_DELAY_150PS_M	GENMASK(6, 3)
+#define   YT921X_XMII_RGMII_RX_DELAY_150PS(x)		FIELD_PREP(YT921X_XMII_RGMII_RX_DELAY_150PS_M, (x))
+#define  YT921X_XMII_RMII_PHY_TX_CLK_OUT	BIT(2)
+#define  YT921X_XMII_REVMII_TX_CLK_OUT		BIT(1)
+#define  YT921X_XMII_REVMII_RX_CLK_OUT		BIT(0)
+
+#define YT921X_MACn_FRAME(port)		(0x81008 + 0x1000 * (port))
+#define  YT921X_MAC_FRAME_SIZE_M		GENMASK(21, 8)
+#define   YT921X_MAC_FRAME_SIZE(x)			FIELD_PREP(YT921X_MAC_FRAME_SIZE_M, (x))
+
+#define YT921X_EEEn_VAL(port)		(0xa0000 + 0x40 * (port))
+#define  YT921X_EEE_VAL_DATA			BIT(1)
+
+#define YT921X_EEE_CTRL			0xb0000
+#define  YT921X_EEE_CTRL_ENn(port)		BIT(port)
+
+#define YT921X_MIB_CTRL			0xc0004
+#define  YT921X_MIB_CTRL_CLEAN			BIT(30)
+#define  YT921X_MIB_CTRL_PORT_M			GENMASK(6, 3)
+#define   YT921X_MIB_CTRL_PORT(x)			FIELD_PREP(YT921X_MIB_CTRL_PORT_M, (x))
+#define  YT921X_MIB_CTRL_ONE_PORT		BIT(1)
+#define  YT921X_MIB_CTRL_ALL_PORT		BIT(0)
+#define YT921X_MIBn_DATA0(port)		(0xc0100 + 0x100 * (port))
+#define YT921X_MIBn_DATAm(port, x)	(YT921X_MIBn_DATA0(port) + 4 * (x))
+#define  YT921X_MIB_DATA_RX_BROADCAST		0x00
+#define  YT921X_MIB_DATA_RX_PAUSE		0x04
+#define  YT921X_MIB_DATA_RX_MULTICAST		0x08
+#define  YT921X_MIB_DATA_RX_CRC_ERR		0x0c
+
+#define  YT921X_MIB_DATA_RX_ALIGN_ERR		0x10
+#define  YT921X_MIB_DATA_RX_UNDERSIZE_ERR	0x14
+#define  YT921X_MIB_DATA_RX_FRAG_ERR		0x18
+#define  YT921X_MIB_DATA_RX_PKT_SZ_64		0x1c
+
+#define  YT921X_MIB_DATA_RX_PKT_SZ_65_TO_127	0x20
+#define  YT921X_MIB_DATA_RX_PKT_SZ_128_TO_255	0x24
+#define  YT921X_MIB_DATA_RX_PKT_SZ_256_TO_511	0x28
+#define  YT921X_MIB_DATA_RX_PKT_SZ_512_TO_1023	0x2c
+
+#define  YT921X_MIB_DATA_RX_PKT_SZ_1024_TO_1518	0x30
+#define  YT921X_MIB_DATA_RX_PKT_SZ_1519_TO_MAX	0x34
+/* 0x38: unused */
+#define  YT921X_MIB_DATA_RX_GOOD_BYTES		0x3c
+
+/* 0x40: 64 bytes */
+#define  YT921X_MIB_DATA_RX_BAD_BYTES		0x44
+/* 0x48: 64 bytes */
+#define  YT921X_MIB_DATA_RX_OVERSIZE_ERR	0x4c
+
+#define  YT921X_MIB_DATA_RX_DROPPED		0x50
+#define  YT921X_MIB_DATA_TX_BROADCAST		0x54
+#define  YT921X_MIB_DATA_TX_PAUSE		0x58
+#define  YT921X_MIB_DATA_TX_MULTICAST		0x5c
+
+#define  YT921X_MIB_DATA_TX_UNDERSIZE_ERR	0x60
+#define  YT921X_MIB_DATA_TX_PKT_SZ_64		0x64
+#define  YT921X_MIB_DATA_TX_PKT_SZ_65_TO_127	0x68
+#define  YT921X_MIB_DATA_TX_PKT_SZ_128_TO_255	0x6c
+
+#define  YT921X_MIB_DATA_TX_PKT_SZ_256_TO_511	0x70
+#define  YT921X_MIB_DATA_TX_PKT_SZ_512_TO_1023	0x74
+#define  YT921X_MIB_DATA_TX_PKT_SZ_1024_TO_1518	0x78
+#define  YT921X_MIB_DATA_TX_PKT_SZ_1519_TO_MAX	0x7c
+
+/* 0x80: unused */
+#define  YT921X_MIB_DATA_TX_GOOD_BYTES		0x84
+/* 0x88: 64 bytes */
+#define  YT921X_MIB_DATA_TX_COLLISION		0x8c
+
+#define  YT921X_MIB_DATA_TX_EXCESSIVE_COLLISION	0x90
+#define  YT921X_MIB_DATA_TX_MULTIPLE_COLLISION	0x94
+#define  YT921X_MIB_DATA_TX_SINGLE_COLLISION	0x98
+#define  YT921X_MIB_DATA_TX_PKT			0x9c
+
+#define  YT921X_MIB_DATA_TX_DEFERRED		0xa0
+#define  YT921X_MIB_DATA_TX_LATE_COLLISION	0xa4
+#define  YT921X_MIB_DATA_RX_OAM			0xa8
+#define  YT921X_MIB_DATA_TX_OAM			0xac
+
+#define YT921X_EDATA_CTRL		0xe0000
+#define  YT921X_EDATA_CTRL_ADDR_M		GENMASK(15, 8)
+#define   YT921X_EDATA_CTRL_ADDR(x)			FIELD_PREP(YT921X_EDATA_CTRL_ADDR_M, (x))
+#define  YT921X_EDATA_CTRL_OP_M			GENMASK(3, 0)
+#define   YT921X_EDATA_CTRL_OP(x)			FIELD_PREP(YT921X_EDATA_CTRL_OP_M, (x))
+#define   YT921X_EDATA_CTRL_READ			YT921X_EDATA_CTRL_OP(5)
+#define YT921X_EDATA_DATA		0xe0004
+#define  YT921X_EDATA_DATA_DATA_M			GENMASK(31, 24)
+#define  YT921X_EDATA_DATA_STATUS_M		GENMASK(3, 0)
+#define   YT921X_EDATA_DATA_STATUS(x)			FIELD_PREP(YT921X_EDATA_DATA_STATUS_M, (x))
+#define   YT921X_EDATA_DATA_IDLE			YT921X_EDATA_DATA_STATUS(3)
+#define YT921X_SYS_CLK			0xe0040
+#define  YT921X_SYS_CLK_SEL_M			GENMASK(1, 0)  /* unknown: 167M */
+#define   YT9215_SYS_CLK_125M				0
+#define   YT9218_SYS_CLK_167M				0
+#define   YT921X_SYS_CLK_143M				1
+
+#define YT921X_PORTn_EGR(port)		(0x100000 + 4 * (port))
+#define  YT921X_PORT_EGR_TPID_CTAG_M		GENMASK(5, 4)
+#define   YT921X_PORT_EGR_TPID_CTAG(x)			FIELD_PREP(YT921X_PORT_EGR_TPID_CTAG_M, (x))
+#define  YT921X_PORT_EGR_TPID_STAG_M		GENMASK(3, 2)
+#define   YT921X_PORT_EGR_TPID_STAG(x)			FIELD_PREP(YT921X_PORT_EGR_TPID_STAG_M, (x))
+#define YT921X_TPID_EGRn(x)		(0x100300 + 4 * (x))	/* [0, 3] */
+#define  YT921X_TPID_EGR_TPID_M			GENMASK(15, 0)
+
+#define YT921X_IPM_DSCPn(n)		(0x180000 + 4 * (n))	/* Internal Priority Map */
+#define YT921X_IPM_PCPn(map, dei, pcp)	(0x180100 + 4 * (16 * (map) + 8 * (dei) + (pcp)))
+#define  YT921X_IPM_PRIO_M			GENMASK(4, 2)
+#define   YT921X_IPM_PRIO(x)				FIELD_PREP(YT921X_IPM_PRIO_M, (x))
+#define  YT921X_IPM_COLOR_M			GENMASK(1, 0)
+#define   YT921X_IPM_COLOR(x)				FIELD_PREP(YT921X_IPM_COLOR_M, (x))
+#define   YT921X_IPM_COLOR_GREEN			YT921X_IPM_COLOR(0)
+#define   YT921X_IPM_COLOR_YELLOW			YT921X_IPM_COLOR(1)
+#define   YT921X_IPM_COLOR_RED				YT921X_IPM_COLOR(2)
+#define YT921X_PORTn_QOS(port)		(0x180180 + 4 * (port))
+#define  YT921X_PORT_QOS_CVLAN_PRIO_MAP_ID	BIT(5)
+#define  YT921X_PORT_QOS_SVLAN_PRIO_MAP_ID	BIT(4)
+#define  YT921X_PORT_QOS_PRIO_M			GENMASK(3, 1)
+#define   YT921X_PORT_QOS_PRIO(x)			FIELD_PREP(YT921X_PORT_QOS_PRIO_M, (x))
+#define  YT921X_PORT_QOS_PRIO_EN		BIT(0)
+#define YT921X_PORTn_PRIO_ORD(port)	(0x180200 + 4 * (port))
+#define  YT921X_PORT_PRIO_ORD_APPm_M(m)		GENMASK(3 * (m) + 2, 3 * (m))
+#define   YT921X_PORT_PRIO_ORD_APPm(m, x)		((x) << (3 * (m)))	/* 0: disabled, except PORT_QOS_PRIO */
+
+enum yt921x_app_selector {
+	YT921X_APP_SEL_MAC_SA,
+	YT921X_APP_SEL_MAC_DA,
+	YT921X_APP_SEL_VID,
+	YT921X_APP_SEL_ACL,
+	YT921X_APP_SEL_DSCP,
+	YT921X_APP_SEL_CVLAN_PCP,
+	YT921X_APP_SEL_SVLAN_PCP,
+	/* The physical port, i.e. YT921X_PORT_QOS_PRIO */
+	YT921X_APP_SEL_PORT,
+	YT921X_APP_SEL_NUM
+};
+
+#define YT921X_VLAN_IGR_FILTER		0x180280
+#define  YT921X_VLAN_IGR_FILTER_PORTn_BYPASS_IGMP(port)	BIT((port) + 11)
+#define  YT921X_VLAN_IGR_FILTER_PORTn(port)	BIT(port)
+#define YT921X_PORTn_ISOLATION(port)	(0x180294 + 4 * (port))
+#define  YT921X_PORT_ISOLATION_BLOCKn(port)	BIT(port)
+#define YT921X_STPn(n)			(0x18038c + 4 * (n))
+#define  YT921X_STP_PORTn_M(port)		GENMASK(2 * (port) + 1, 2 * (port))
+#define   YT921X_STP_PORTn(port, x)			((x) << (2 * (port)))
+#define   YT921X_STP_PORTn_DISABLED(port)		YT921X_STP_PORTn(port, 0)
+#define   YT921X_STP_PORTn_LEARNING(port)		YT921X_STP_PORTn(port, 1)
+#define   YT921X_STP_PORTn_BLOCKING(port)		YT921X_STP_PORTn(port, 2)
+#define   YT921X_STP_PORTn_FORWARD(port)		YT921X_STP_PORTn(port, 3)
+#define YT921X_PORTn_LEARN(port)	(0x1803d0 + 4 * (port))
+#define  YT921X_PORT_LEARN_VID_LEARN_MULTI_EN	BIT(22)
+#define  YT921X_PORT_LEARN_VID_LEARN_MODE	BIT(21)
+#define  YT921X_PORT_LEARN_VID_LEARN_EN		BIT(20)
+#define  YT921X_PORT_LEARN_SUSPEND_COPY_EN	BIT(19)
+#define  YT921X_PORT_LEARN_SUSPEND_DROP_EN	BIT(18)
+#define  YT921X_PORT_LEARN_DIS			BIT(17)
+#define  YT921X_PORT_LEARN_LIMIT_EN		BIT(16)
+#define  YT921X_PORT_LEARN_LIMIT_M		GENMASK(15, 8)
+#define   YT921X_PORT_LEARN_LIMIT(x)			FIELD_PREP(YT921X_PORT_LEARN_LIMIT_M, (x))
+#define  YT921X_PORT_LEARN_DROP_ON_EXCEEDED	BIT(2)
+#define  YT921X_PORT_LEARN_MODE_M		GENMASK(1, 0)
+#define   YT921X_PORT_LEARN_MODE(x)			FIELD_PREP(YT921X_PORT_LEARN_MODE_M, (x))
+#define   YT921X_PORT_LEARN_MODE_AUTO			YT921X_PORT_LEARN_MODE(0)
+#define   YT921X_PORT_LEARN_MODE_AUTO_AND_COPY		YT921X_PORT_LEARN_MODE(1)
+#define   YT921X_PORT_LEARN_MODE_CPU_CONTROL		YT921X_PORT_LEARN_MODE(2)
+#define YT921X_AGEING			0x180440
+#define  YT921X_AGEING_INTERVAL_M		GENMASK(15, 0)
+#define YT921X_FDB_IN0			0x180454
+#define YT921X_FDB_IN1			0x180458
+#define YT921X_FDB_IN2			0x18045c
+#define YT921X_FDB_OP			0x180460
+#define  YT921X_FDB_OP_INDEX_M			GENMASK(22, 11)
+#define   YT921X_FDB_OP_INDEX(x)			FIELD_PREP(YT921X_FDB_OP_INDEX_M, (x))
+#define  YT921X_FDB_OP_MODE_INDEX		BIT(10)  /* mac+fid / index */
+#define  YT921X_FDB_OP_FLUSH_MCAST		BIT(9)  /* ucast / mcast */
+#define  YT921X_FDB_OP_FLUSH_M			GENMASK(8, 7)
+#define   YT921X_FDB_OP_FLUSH(x)			FIELD_PREP(YT921X_FDB_OP_FLUSH_M, (x))
+#define   YT921X_FDB_OP_FLUSH_ALL			YT921X_FDB_OP_FLUSH(0)
+#define   YT921X_FDB_OP_FLUSH_PORT			YT921X_FDB_OP_FLUSH(1)
+#define   YT921X_FDB_OP_FLUSH_PORT_VID			YT921X_FDB_OP_FLUSH(2)
+#define   YT921X_FDB_OP_FLUSH_VID			YT921X_FDB_OP_FLUSH(3)
+#define  YT921X_FDB_OP_FLUSH_STATIC		BIT(6)
+#define  YT921X_FDB_OP_NEXT_TYPE_M		GENMASK(5, 4)
+#define   YT921X_FDB_OP_NEXT_TYPE(x)			FIELD_PREP(YT921X_FDB_OP_NEXT_TYPE_M, (x))
+#define   YT921X_FDB_OP_NEXT_TYPE_UCAST_PORT		YT921X_FDB_OP_NEXT_TYPE(0)
+#define   YT921X_FDB_OP_NEXT_TYPE_UCAST_VID		YT921X_FDB_OP_NEXT_TYPE(1)
+#define   YT921X_FDB_OP_NEXT_TYPE_UCAST			YT921X_FDB_OP_NEXT_TYPE(2)
+#define   YT921X_FDB_OP_NEXT_TYPE_MCAST			YT921X_FDB_OP_NEXT_TYPE(3)
+#define  YT921X_FDB_OP_OP_M			GENMASK(3, 1)
+#define   YT921X_FDB_OP_OP(x)				FIELD_PREP(YT921X_FDB_OP_OP_M, (x))
+#define   YT921X_FDB_OP_OP_ADD				YT921X_FDB_OP_OP(0)
+#define   YT921X_FDB_OP_OP_DEL				YT921X_FDB_OP_OP(1)
+#define   YT921X_FDB_OP_OP_GET_ONE			YT921X_FDB_OP_OP(2)
+#define   YT921X_FDB_OP_OP_GET_NEXT			YT921X_FDB_OP_OP(3)
+#define   YT921X_FDB_OP_OP_FLUSH			YT921X_FDB_OP_OP(4)
+#define  YT921X_FDB_OP_START			BIT(0)
+#define YT921X_FDB_RESULT		0x180464
+#define  YT921X_FDB_RESULT_DONE			BIT(15)
+#define  YT921X_FDB_RESULT_NOTFOUND		BIT(14)
+#define  YT921X_FDB_RESULT_OVERWRITED		BIT(13)
+#define  YT921X_FDB_RESULT_INDEX_M		GENMASK(11, 0)
+#define   YT921X_FDB_RESULT_INDEX(x)			FIELD_PREP(YT921X_FDB_RESULT_INDEX_M, (x))
+#define YT921X_FDB_OUT0			0x1804b0
+#define  YT921X_FDB_IO0_ADDR_HI4_M		GENMASK(31, 0)
+#define YT921X_FDB_OUT1			0x1804b4
+#define  YT921X_FDB_IO1_EGR_PRIO_EN		BIT(31)
+#define  YT921X_FDB_IO1_STATUS_M		GENMASK(30, 28)
+#define   YT921X_FDB_IO1_STATUS(x)			FIELD_PREP(YT921X_FDB_IO1_STATUS_M, (x))
+#define   YT921X_FDB_IO1_STATUS_INVALID			YT921X_FDB_IO1_STATUS(0)
+#define   YT921X_FDB_IO1_STATUS_MIN_TIME		YT921X_FDB_IO1_STATUS(1)
+#define   YT921X_FDB_IO1_STATUS_MOVE_AGING_MAX_TIME	YT921X_FDB_IO1_STATUS(3)
+#define   YT921X_FDB_IO1_STATUS_MAX_TIME		YT921X_FDB_IO1_STATUS(5)
+#define   YT921X_FDB_IO1_STATUS_PENDING			YT921X_FDB_IO1_STATUS(6)
+#define   YT921X_FDB_IO1_STATUS_STATIC			YT921X_FDB_IO1_STATUS(7)
+#define  YT921X_FDB_IO1_FID_M			GENMASK(27, 16)  /* filtering ID (VID) */
+#define   YT921X_FDB_IO1_FID(x)				FIELD_PREP(YT921X_FDB_IO1_FID_M, (x))
+#define  YT921X_FDB_IO1_ADDR_LO2_M		GENMASK(15, 0)
+#define YT921X_FDB_OUT2			0x1804b8
+#define  YT921X_FDB_IO2_MOVE_AGING_STATUS_M	GENMASK(31, 30)
+#define  YT921X_FDB_IO2_IGR_DROP		BIT(29)
+#define  YT921X_FDB_IO2_EGR_PORTS_M		GENMASK(28, 18)
+#define   YT921X_FDB_IO2_EGR_PORTS(x)			FIELD_PREP(YT921X_FDB_IO2_EGR_PORTS_M, (x))
+#define  YT921X_FDB_IO2_EGR_DROP		BIT(17)
+#define  YT921X_FDB_IO2_COPY_TO_CPU		BIT(16)
+#define  YT921X_FDB_IO2_IGR_PRIO_EN		BIT(15)
+#define  YT921X_FDB_IO2_PRIO_M			GENMASK(14, 12)
+#define   YT921X_FDB_IO2_PRIO(x)			FIELD_PREP(YT921X_FDB_IO2_PRIO_M, (x))
+#define  YT921X_FDB_IO2_NEW_VID_M		GENMASK(11, 0)
+#define   YT921X_FDB_IO2_NEW_VID(x)			FIELD_PREP(YT921X_FDB_IO2_NEW_VID_M, (x))
+#define YT921X_FILTER_UNK_UCAST		0x180508
+#define YT921X_FILTER_UNK_MCAST		0x18050c
+#define YT921X_FILTER_MCAST		0x180510
+#define YT921X_FILTER_BCAST		0x180514
+#define  YT921X_FILTER_PORTS_M			GENMASK(10, 0)
+#define   YT921X_FILTER_PORTS(x)			FIELD_PREP(YT921X_FILTER_PORTS_M, (x))
+#define  YT921X_FILTER_PORTn(port)		BIT(port)
+#define YT921X_VLAN_EGR_FILTER		0x180598
+#define  YT921X_VLAN_EGR_FILTER_PORTn(port)	BIT(port)
+#define YT921X_LAG_GROUPn(n)		(0x1805a8 + 4 * (n))
+#define  YT921X_LAG_GROUP_PORTS_M		GENMASK(13, 3)
+#define   YT921X_LAG_GROUP_PORTS(x)			FIELD_PREP(YT921X_LAG_GROUP_PORTS_M, (x))
+#define  YT921X_LAG_GROUP_MEMBER_NUM_M		GENMASK(2, 0)
+#define   YT921X_LAG_GROUP_MEMBER_NUM(x)		FIELD_PREP(YT921X_LAG_GROUP_MEMBER_NUM_M, (x))
+#define YT921X_LAG_MEMBERnm(n, m)	(0x1805b0 + 4 * (4 * (n) + (m)))
+#define  YT921X_LAG_MEMBER_PORT_M		GENMASK(3, 0)
+#define   YT921X_LAG_MEMBER_PORT(x)			FIELD_PREP(YT921X_LAG_MEMBER_PORT_M, (x))
+#define YT921X_CPU_COPY			0x180690
+#define  YT921X_CPU_COPY_FORCE_INT_PORT		BIT(2)
+#define  YT921X_CPU_COPY_TO_INT_CPU		BIT(1)
+#define  YT921X_CPU_COPY_TO_EXT_CPU		BIT(0)
+#define YT921X_ACL_PERMIT_UNMATCH	0x1806a0
+#define  YT921X_ACL_PERMIT_UNMATCH_PORTS_M	GENMASK(10, 0)
+#define   YT921X_ACL_PERMIT_UNMATCH_PORTS(x)		FIELD_PREP(YT921X_ACL_PERMIT_UNMATCH_PORTS_M, (x))
+#define  YT921X_ACL_PERMIT_UNMATCH_PORTn(port)	BIT(port)
+#define YT921X_ACT_UNK_UCAST		0x180734
+#define YT921X_ACT_UNK_MCAST		0x180738
+#define  YT921X_ACT_UNK_MCAST_BYPASS_DROP_RMA	BIT(23)
+#define  YT921X_ACT_UNK_MCAST_BYPASS_DROP_IGMP	BIT(22)
+#define  YT921X_ACT_UNK_ACTn_M(port)		GENMASK(2 * (port) + 1, 2 * (port))
+#define   YT921X_ACT_UNK_ACTn(port, x)			((x) << (2 * (port)))
+#define   YT921X_ACT_UNK_ACTn_FORWARD(port)		YT921X_ACT_UNK_ACTn(port, 0)  /* flood */
+#define   YT921X_ACT_UNK_ACTn_TRAP(port)		YT921X_ACT_UNK_ACTn(port, 1)  /* steer to CPU */
+#define   YT921X_ACT_UNK_ACTn_DROP(port)		YT921X_ACT_UNK_ACTn(port, 2)  /* discard */
+/* NEVER use this action; see comments in the tag driver */
+#define   YT921X_ACT_UNK_ACTn_COPY(port)		YT921X_ACT_UNK_ACTn(port, 3)  /* flood and copy */
+#define YT921X_FDB_HW_FLUSH		0x180958
+#define  YT921X_FDB_HW_FLUSH_ON_LINKDOWN	BIT(0)
+
+#define YT921X_VLANn_CTRL(vlan)		(0x188000 + 8 * (vlan))
+#define  YT921X_VLAN_CTRLb_UNTAG_PORTS_M	GENMASK(18, 8)
+#define   YT921X_VLAN_CTRLb_UNTAG_PORTS(x)		FIELD_PREP(YT921X_VLAN_CTRLb_UNTAG_PORTS_M, (x))
+#define  YT921X_VLAN_CTRLb_UNTAG_PORTn(port)	BIT((port) + 8)
+#define  YT921X_VLAN_CTRLb_STP_ID_M		GENMASK(7, 4)
+#define   YT921X_VLAN_CTRLb_STP_ID(x)			FIELD_PREP(YT921X_VLAN_CTRLb_STP_ID_M, (x))
+#define  YT921X_VLAN_CTRLb_SVLAN_EN		BIT(3)
+#define  YT921X_VLAN_CTRLab_FID_M		GENMASK_ULL(34, 23)
+#define   YT921X_VLAN_CTRLab_FID(x)			FIELD_PREP(YT921X_VLAN_CTRLab_FID_M, (x))
+#define  YT921X_VLAN_CTRLa_LEARN_DIS		BIT(22)
+#define  YT921X_VLAN_CTRLa_PRIO_EN		BIT(21)
+#define  YT921X_VLAN_CTRLa_PRIO_M		GENMASK(20, 18)
+#define   YT921X_VLAN_CTRLa_PRIO(x)			FIELD_PREP(YT921X_VLAN_CTRLa_PRIO_M, (x))
+#define  YT921X_VLAN_CTRLa_PORTS_M		GENMASK(17, 7)
+#define   YT921X_VLAN_CTRLa_PORTS(x)			FIELD_PREP(YT921X_VLAN_CTRLa_PORTS_M, (x))
+#define  YT921X_VLAN_CTRLa_PORTn(port)		BIT((port) + 7)
+#define  YT921X_VLAN_CTRLa_BYPASS_1X_AC		BIT(6)
+#define  YT921X_VLAN_CTRLa_METER_EN		BIT(5)
+#define  YT921X_VLAN_CTRLa_METER_ID_M		GENMASK(4, 0)
+
+#define YT921X_ACLn_ACT(n)		(0x1c0000 + 0x10 * (n))
+#define  YT921X_ACL_ACTc_STAG_M			GENMASK(26, 25)
+#define   YT921X_ACL_ACTc_STAG(x)			FIELD_PREP(YT921X_ACL_ACTc_STAG_M, (x))
+#define   YT921X_ACL_ACTc_STAG_DONTCARE			YT921X_ACL_ACTc_STAG(0)
+#define   YT921X_ACL_ACTc_STAG_UNTAG			YT921X_ACL_ACTc_STAG(1)
+#define   YT921X_ACL_ACTc_STAG_TAG			YT921X_ACL_ACTc_STAG(2)
+#define   YT921X_ACL_ACTc_STAG_KEEP			YT921X_ACL_ACTc_STAG(3)
+#define  YT921X_ACL_ACTc_CTAG_M			GENMASK(24, 23)
+#define   YT921X_ACL_ACTc_CTAG(x)			FIELD_PREP(YT921X_ACL_ACTc_CTAG_M, (x))
+#define   YT921X_ACL_ACTc_CTAG_DONTCARE			YT921X_ACL_ACTc_CTAG(0)
+#define   YT921X_ACL_ACTc_CTAG_UNTAG			YT921X_ACL_ACTc_CTAG(1)
+#define   YT921X_ACL_ACTc_CTAG_TAG			YT921X_ACL_ACTc_CTAG(2)
+#define   YT921X_ACL_ACTc_CTAG_KEEP			YT921X_ACL_ACTc_CTAG(3)
+#define  YT921X_ACL_ACTc_FWD_M			GENMASK(22, 21)
+#define   YT921X_ACL_ACTc_FWD(x)			FIELD_PREP(YT921X_ACL_ACTc_FWD_M, (x))
+#define   YT921X_ACL_ACTc_FWD_FWD			YT921X_ACL_ACTc_FWD(0)
+#define   YT921X_ACL_ACTc_FWD_COPY			YT921X_ACL_ACTc_FWD(1)
+#define   YT921X_ACL_ACTc_FWD_REDIR			YT921X_ACL_ACTc_FWD(2)
+#define   YT921X_ACL_ACTc_FWD_TRAP			YT921X_ACL_ACTc_FWD(3)
+#define  YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M	GENMASK(20, 10)
+#define   YT921X_ACL_ACTc_FWD_REDIR_DPORTS(x)		FIELD_PREP(YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M, (x))
+#define  YT921X_ACL_ACTc_FWD_REDIR_DPORTn(port)	BIT((port) + 10)
+#define  YT921X_ACL_ACTc_FWD_EN			BIT(9)
+#define  YT921X_ACL_ACTc_SDEI			BIT(8)
+#define  YT921X_ACL_ACTc_SDEI_REPLACE		BIT(7)
+#define  YT921X_ACL_ACTc_SPRI_M			GENMASK(6, 4)
+#define   YT921X_ACL_ACTc_SPRI(x)			FIELD_PREP(YT921X_ACL_ACTc_SPRI_M, (x))
+#define  YT921X_ACL_ACTc_SPRI_REPLACE		BIT(3)
+#define  YT921X_ACL_ACTbc_SVID_M		GENMASK_ULL(34, 23)
+#define   YT921X_ACL_ACTbc_SVID(x)			FIELD_PREP(YT921X_ACL_ACTbc_SVID_M, (x))
+#define  YT921X_ACL_ACTb_SVID_REPLACE		BIT(22)
+#define  YT921X_ACL_ACTb_CDEI			BIT(21)
+#define  YT921X_ACL_ACTb_CDEI_REPLACE		BIT(20)
+#define  YT921X_ACL_ACTb_CPRI_M			GENMASK(19, 17)
+#define   YT921X_ACL_ACTb_CPRI(x)			FIELD_PREP(YT921X_ACL_ACTb_CPRI_M, (x))
+#define  YT921X_ACL_ACTb_CPRI_REPLACE		BIT(16)
+#define  YT921X_ACL_ACTb_CVID_M			GENMASK(15, 4)
+#define   YT921X_ACL_ACTb_CVID(x)			FIELD_PREP(YT921X_ACL_ACTb_CVID_M, (x))
+#define  YT921X_ACL_ACTb_CVID_REPLACE		BIT(3)
+#define  YT921X_ACL_ACTb_PRIO_M			GENMASK(2, 0)
+#define   YT921X_ACL_ACTb_PRIO(x)			FIELD_PREP(YT921X_ACL_ACTb_PRIO_M, (x))
+#define  YT921X_ACL_ACTa_PRIO_EN		BIT(31)
+#define  YT921X_ACL_ACTa_COLOR_M		GENMASK(30, 29)
+#define   YT921X_ACL_ACTa_COLOR(x)			FIELD_PREP(YT921X_ACL_ACTa_COLOR_M, (x))
+#define   YT921X_ACL_ACTa_COLOR_GREEN			YT921X_ACL_ACTa_COLOR(0)
+#define   YT921X_ACL_ACTa_COLOR_YELLOW			YT921X_ACL_ACTa_COLOR(1)
+#define   YT921X_ACL_ACTa_COLOR_RED			YT921X_ACL_ACTa_COLOR(2)
+#define  YT921X_ACL_ACTa_COLOR_EN		BIT(28)
+#define  YT921X_ACL_ACTa_DSCP_M			GENMASK(27, 22)
+#define   YT921X_ACL_ACTa_DSCP(x)			FIELD_PREP(YT921X_ACL_ACTa_DSCP_M, (x))
+#define  YT921X_ACL_ACTa_DSCP_REPLACE		BIT(21)
+#define  YT921X_ACL_ACTa_METER_ID_M		GENMASK(20, 15)
+#define   YT921X_ACL_ACTa_METER_ID(x)			FIELD_PREP(YT921X_ACL_ACTa_METER_ID_M, (x))
+#define  YT921X_ACL_ACTa_METER_EN		BIT(14)
+#define  YT921X_ACL_ACTa_MIRROR_EN		BIT(13)
+#define  YT921X_ACL_ACTa_FLOWSTAT_EN		BIT(12)
+#define  YT921X_ACL_ACTa_FLOWSTAT_ID_M		GENMASK(11, 6)
+#define   YT921X_ACL_ACTa_FLOWSTAT_ID(x)		FIELD_PREP(YT921X_ACL_ACTa_FLOWSTAT_ID_M, (x))
+#define  YT921X_ACL_ACTa_GPIO_EN		BIT(5)
+#define  YT921X_ACL_ACTa_GPIO_PIN_M		GENMASK(4, 1)
+#define   YT921X_ACL_ACTa_GPIO_PIN(x)			FIELD_PREP(YT921X_ACL_ACTa_GPIO_PIN_M, (x))
+#define  YT921X_ACL_ACTa_INTR_EN		BIT(0)
+#define YT921X_ACL_BLK_KEEP		0x201000
+#define  YT921X_ACL_BLK_KEEP_GRPIDn_M(bin)	(7 << (4 * (bin) + 1))
+#define   YT921X_ACL_BLK_KEEP_GRPIDn(bin, x)		((x) << (4 * (bin) + 1))
+#define  YT921X_ACL_BLK_KEEP_KEEPn(bin)		BIT(4 * (bin))
+#define YT921X_ACL_PORT			0x202000
+#define  YT921X_ACL_PORT_PORTS_M		GENMASK(10, 0)
+#define   YT921X_ACL_PORT_PORTS(x)			FIELD_PREP(YT921X_ACL_PORT_PORTS_M, (x))
+#define  YT921X_ACL_PORT_PORTn(port)		BIT(port)
+#define YT921X_ACL_BLK_CMD		0x202004
+#define  YT921X_ACL_BLK_CMD_BLKID_M		GENMASK(6, 1)
+#define   YT921X_ACL_BLK_CMD_BLKID(x)			FIELD_PREP(YT921X_ACL_BLK_CMD_BLKID_M, (x))
+#define  YT921X_ACL_BLK_CMD_MODIFY		BIT(0)
+#define YT921X_ACLn_ENTRY(blk)		(0x203000 + 4 * (blk))
+#define  YT921X_ACL_ENTRY_GRPIDm_M(bin)		(7 << (4 * (bin) + 1))
+#define   YT921X_ACL_ENTRY_GRPIDm(bin, x)		((x) << (4 * (bin) + 1))
+#define  YT921X_ACL_ENTRY_ENm(bin)		BIT(4 * (bin))
+#define YT921X_ACLn_KEYm(blk, bin)	(0x204000 + 0x200 * (bin) + 8 * (blk))
+#define  YT921X_ACL_KEYb_ORD_M			GENMASK(29, 21)
+#define   YT921X_ACL_KEYb_ORD(x)			FIELD_PREP(YT921X_ACL_KEYb_ORD_M, (x))
+#define  YT921X_ACL_KEYb_SPORTS_M		GENMASK(20, 10)
+#define   YT921X_ACL_KEYb_SPORTS(x)			FIELD_PREP(YT921X_ACL_KEYb_SPORTS_M, (x))
+#define  YT921X_ACL_KEYb_SPORTn(port)		BIT((port) + 10)
+#define  YT921X_ACL_KEYb_REVERSE		BIT(9)	/* reverse match */
+#define  YT921X_ACL_KEYb_TYPE_M			GENMASK(8, 4)
+#define   YT921X_ACL_KEYb_TYPE(x)			FIELD_PREP(YT921X_ACL_KEYb_TYPE_M, (x))
+/* KEY_* fields need no masks */
+#define YT921X_ACLn_MASKm(blk, bin)	(0x205000 + 0x200 * (bin) + 8 * (blk))
+
+enum yt921x_acl_type {
+	YT921X_ACL_TYPE_NA,
+	YT921X_ACL_TYPE_MAC_DA0,
+	YT921X_ACL_TYPE_MAC_SA0,
+	YT921X_ACL_TYPE_MAC_DA1_SA1,
+	YT921X_ACL_TYPE_VLAN,
+	YT921X_ACL_TYPE_VTAG,
+	YT921X_ACL_TYPE_IPV4_DA,
+	YT921X_ACL_TYPE_IPV4_SA,
+	YT921X_ACL_TYPE_IPV6_DA0,
+	YT921X_ACL_TYPE_IPV6_DA1,
+	YT921X_ACL_TYPE_IPV6_DA2,
+	YT921X_ACL_TYPE_IPV6_DA3,
+	YT921X_ACL_TYPE_IPV6_SA0,
+	YT921X_ACL_TYPE_IPV6_SA1,
+	YT921X_ACL_TYPE_IPV6_SA2,
+	YT921X_ACL_TYPE_IPV6_SA3,
+	YT921X_ACL_TYPE_MISC,
+	YT921X_ACL_TYPE_L4,
+	YT921X_ACL_TYPE_UDF0,
+	YT921X_ACL_TYPE_UDF1,
+	YT921X_ACL_TYPE_UDF2,
+	YT921X_ACL_TYPE_UDF3,
+	YT921X_ACL_TYPE_UDF4,
+	YT921X_ACL_TYPE_UDF5,
+	YT921X_ACL_TYPE_UDF6,
+	YT921X_ACL_TYPE_UDF7,
+	YT921X_ACL_TYPE_ETHERTYPE,
+	YT921X_ACL_TYPE_NUM
+};
+
+/* Range: turn KEY:MASK into MIN:MAX */
+
+#define  YT921X_ACL_BINb_MAC_xA0_L3_TYPE_M	GENMASK(3, 0)
+#define   YT921X_ACL_BINb_MAC_xA0_L3_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_MAC_xA0_L3_TYPE_M, (x))
+#define  YT921X_ACL_BINa_MAC_xA0_MAC_xA0_M	GENMASK(31, 0)
+
+#define  YT921X_ACL_BINb_MAC_DA1_SA1_L2_TYPE_M	GENMASK(2, 0)
+#define   YT921X_ACL_BINb_MAC_DA1_SA1_L2_TYPE(x)	FIELD_PREP(YT921X_ACL_BINb_MAC_DA1_SA1_L2_TYPE_M, (x))
+#define  YT921X_ACL_BINa_MAC_DA1_SA1_MAC_DA1_M	GENMASK(31, 16)
+#define  YT921X_ACL_BINa_MAC_DA1_SA1_MAC_SA1_M	GENMASK(15, 0)
+
+#define  YT921X_ACL_KEYb_VLAN_SVID_RANGE_EN	BIT(31)
+#define  YT921X_ACL_KEYb_VLAN_CVID_RANGE_EN	BIT(30)
+#define  YT921X_ACL_BINb_VLAN_CDEI		BIT(3)
+#define  YT921X_ACL_BINb_VLAN_CPRI_M		GENMASK(2, 0)
+#define   YT921X_ACL_BINb_VLAN_CPRI(x)			FIELD_PREP(YT921X_ACL_BINb_VLAN_CPRI_M, (x))
+#define  YT921X_ACL_BINa_VLAN_CTAG_FMT_M	GENMASK(31, 30)
+#define   YT921X_ACL_BINa_VLAN_CTAG_FMT(x)		FIELD_PREP(YT921X_ACL_BINa_VLAN_CTAG_FMT_M, (x))
+#define  YT921X_ACL_BINa_VLAN_SDEI		BIT(29)
+#define  YT921X_ACL_BINa_VLAN_SPRI_M		GENMASK(28, 26)
+#define   YT921X_ACL_BINa_VLAN_SPRI(x)			FIELD_PREP(YT921X_ACL_BINa_VLAN_SPRI_M, (x))
+#define  YT921X_ACL_BINa_VLAN_STAG_FMT_M	GENMASK(25, 24)
+#define   YT921X_ACL_BINa_VLAN_STAG_FMT(x)		FIELD_PREP(YT921X_ACL_BINa_VLAN_STAG_FMT_M, (x))
+#define  YT921X_ACL_BINa_VLAN_SVID_M		GENMASK(23, 12)
+#define   YT921X_ACL_BINa_VLAN_SVID(x)			FIELD_PREP(YT921X_ACL_BINa_VLAN_SVID_M, (x))
+#define  YT921X_ACL_BINa_VLAN_CVID_M		GENMASK(11, 0)
+#define   YT921X_ACL_BINa_VLAN_CVID(x)			FIELD_PREP(YT921X_ACL_BINa_VLAN_CVID_M, (x))
+
+#define  YT921X_ACL_KEYb_VTAG_SVID_RANGE_EN	BIT(31)
+#define  YT921X_ACL_KEYb_VTAG_CVID_RANGE_EN	BIT(30)
+#define  YT921X_ACL_BINa_VTAG_CDEI		BIT(31)
+#define  YT921X_ACL_BINa_VTAG_CPRI_M		GENMASK(30, 28)
+#define   YT921X_ACL_BINa_VTAG_CPRI(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_CPRI_M, (x))
+#define  YT921X_ACL_BINa_VTAG_SDEI		BIT(27)
+#define  YT921X_ACL_BINa_VTAG_SPRI_M		GENMASK(26, 24)
+#define   YT921X_ACL_BINa_VTAG_SPRI(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_SPRI_M, (x))
+#define  YT921X_ACL_BINa_VTAG_SVID_M		GENMASK(23, 12)
+#define   YT921X_ACL_BINa_VTAG_SVID(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_SVID_M, (x))
+#define  YT921X_ACL_BINa_VTAG_CVID_M		GENMASK(11, 0)
+#define   YT921X_ACL_BINa_VTAG_CVID(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_CVID_M, (x))
+
+#define  YT921X_ACL_KEYb_IPV4_ADDR_RANGE_EN	BIT(30)
+#define  YT921X_ACL_BINb_IPV4_FRAG		BIT(3)
+#define  YT921X_ACL_BINb_IPV4_L4_TYPE_M		GENMASK(2, 0)
+#define   YT921X_ACL_BINb_IPV4_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_IPV4_L4_TYPE_M, (x))
+#define  YT921X_ACL_BINa_IPV4_ADDR_M		GENMASK(31, 0)
+
+#define  YT921X_ACL_BINb_IPV6_L4_TYPE_M		GENMASK(2, 0)
+#define   YT921X_ACL_BINb_IPV6_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_IPV6_L4_TYPE_M, (x))
+#define  YT921X_ACL_BINa_IPV6_ADDRx_M		GENMASK(31, 0)
+
+#define  YT921X_ACL_BINb_IPV6_xA1_IP_OPTION	BIT(3)
+
+#define  YT921X_ACL_BINb_IPV6_xA2_FIRST_FRAG	BIT(3)
+
+#define  YT921X_ACL_KEYb_IPV6_xA3_ADDR_RANGE_EN	BIT(30)
+#define  YT921X_ACL_BINb_IPV6_xA3_FRAG		BIT(3)
+
+#define  YT921X_ACL_BINb_MISC_FRAG		BIT(3)
+#define  YT921X_ACL_BINb_MISC_L4_TYPE_M		GENMASK(2, 0)
+#define   YT921X_ACL_BINb_MISC_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_MISC_L4_TYPE_M, (x))
+#define  YT921X_ACL_BINa_MISC_PPPOE_FLAG	BIT(30)
+#define  YT921X_ACL_BINa_MISC_FIRST_FRAG	BIT(29)
+#define  YT921X_ACL_BINa_MISC_IP_OPTION		BIT(28)
+#define  YT921X_ACL_BINa_MISC_TCP_FLAGS_M	GENMASK(27, 20)
+#define   YT921X_ACL_BINa_MISC_TCP_FLAGS(x)		FIELD_PREP(YT921X_ACL_BINa_MISC_TCP_FLAGS_M, (x))
+#define  YT921X_ACL_BINa_MISC_IP_PROTO_M	GENMASK(19, 12)
+#define   YT921X_ACL_BINa_MISC_IP_PROTO(x)		FIELD_PREP(YT921X_ACL_BINa_MISC_IP_PROTO_M, (x))
+#define  YT921X_ACL_BINa_MISC_TOS_M		GENMASK(11, 4)
+#define   YT921X_ACL_BINa_MISC_TOS(x)			FIELD_PREP(YT921X_ACL_BINa_MISC_TOS_M, (x))
+#define  YT921X_ACL_BINa_MISC_L3_TYPE_M		GENMASK(3, 0)
+#define   YT921X_ACL_BINa_MISC_L3_TYPE(x)		FIELD_PREP(YT921X_ACL_BINa_MISC_L3_TYPE_M, (x))
+
+#define  YT921X_ACL_KEYb_L4_DPORT_RANGE_EN	BIT(31)
+#define  YT921X_ACL_KEYb_L4_SPORT_RANGE_EN	BIT(30)
+#define  YT921X_ACL_BINb_L4_FRAG		BIT(3)
+#define  YT921X_ACL_BINb_L4_TYPE_M		GENMASK(2, 0)
+#define   YT921X_ACL_BINb_L4_TYPE(x)			FIELD_PREP(YT921X_ACL_BINb_L4_TYPE_M, (x))
+#define  YT921X_ACL_BINa_L4_DPORT_M		GENMASK(31, 16)
+#define  YT921X_ACL_BINa_L4_SPORT_M		GENMASK(15, 0)
+
+#define  YT921X_ACL_BINb_UDF_IS_IGMP		BIT(0)
+#define  YT921X_ACL_BINa_UDF_UDF0_M		GENMASK(31, 16)
+#define   YT921X_ACL_BINa_UDF_UDF0(x)			FIELD_PREP(YT921X_ACL_BINa_UDF_UDF0_M, (x))
+#define  YT921X_ACL_BINa_UDF_UDF1_M		GENMASK(15, 0)
+#define   YT921X_ACL_BINa_UDF_UDF1(x)			FIELD_PREP(YT921X_ACL_BINa_UDF_UDF1_M, (x))
+
+#define  YT921X_ACL_KEYb_ETHERTYPE_ETHERTYPE_RANGE_EN	BIT(30)
+#define  YT921X_ACL_BINb_ETHERTYPE_L4_TYPE_M	GENMASK(2, 0)
+#define   YT921X_ACL_BINb_ETHERTYPE_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_ETHERTYPE_L4_TYPE_M, (x))
+#define  YT921X_ACL_BINa_ETHERTYPE_ETHERTYPE_M	GENMASK(15, 0)
+#define   YT921X_ACL_BINa_ETHERTYPE_ETHERTYPE(x)	FIELD_PREP(YT921X_ACL_BINa_ETHERTYPE_ETHERTYPE_M, (x))
+
+enum yt921x_l2_type {
+	YT921X_L2_TYPE_ETH,
+	YT921X_L2_TYPE_ETHV2,
+	YT921X_L2_TYPE_ETHSAP,
+	YT921X_L2_TYPE_ETHSNAP,
+};
+
+enum yt921x_l3_type {
+	YT921X_L3_TYPE_OTHER,
+	YT921X_L3_TYPE_IPV4,
+	YT921X_L3_TYPE_IPV6,
+	YT921X_L3_TYPE_ARP,
+	YT921X_L3_TYPE_LLDP,
+	YT921X_L3_TYPE_PAE,
+	YT921X_L3_TYPE_ERP,
+	YT921X_L3_TYPE_SLOW_PROTOCOL,
+};
+
+enum yt921x_l4_type {
+	YT921X_L4_TYPE_OTHER,
+	YT921X_L4_TYPE_TCP,
+	YT921X_L4_TYPE_UDP,
+	YT921X_L4_TYPE_UDPLITE,
+	YT921X_L4_TYPE_ICMP,
+	YT921X_L4_TYPE_IGMP,
+	YT921X_L4_TYPE_MLD,
+	YT921X_L4_TYPE_ND,
+};
+
+#define YT921X_TPID_IGRn(x)		(0x210000 + 4 * (x))	/* [0, 3] */
+#define  YT921X_TPID_IGR_TPID_M			GENMASK(15, 0)
+#define YT921X_PORTn_IGR_TPID(port)	(0x210010 + 4 * (port))
+#define  YT921X_PORT_IGR_TPIDn_STAG_M		GENMASK(7, 4)
+#define  YT921X_PORT_IGR_TPIDn_STAG(x)		BIT((x) + 4)
+#define  YT921X_PORT_IGR_TPIDn_CTAG_M		GENMASK(3, 0)
+#define  YT921X_PORT_IGR_TPIDn_CTAG(x)		BIT(x)
+#define YT921X_LAG_HASH			0x210090
+#define  YT921X_LAG_HASH_L4_SPORT		BIT(7)
+#define  YT921X_LAG_HASH_L4_DPORT		BIT(6)
+#define  YT921X_LAG_HASH_IP_PROTO		BIT(5)
+#define  YT921X_LAG_HASH_IP_SRC			BIT(4)
+#define  YT921X_LAG_HASH_IP_DST			BIT(3)
+#define  YT921X_LAG_HASH_MAC_SA			BIT(2)
+#define  YT921X_LAG_HASH_MAC_DA			BIT(1)
+#define  YT921X_LAG_HASH_SRC_PORT		BIT(0)
+#define YT921X_UDFn_CTRL(x)		(0x210094 + 4 * (x))
+#define  YT921X_UDF_CTRL_UDF_TYPE_M		GENMASK(8, 7)
+#define   YT921X_UDF_CTRL_UDF_TYPE(x)			FIELD_PREP(YT921X_UDF_CTRL_UDF_TYPE_M, (x))
+#define   YT921X_UDF_CTRL_UDF_TYPE_ETH			YT921X_UDF_CTRL_UDF_TYPE(0)
+#define   YT921X_UDF_CTRL_UDF_TYPE_L3			YT921X_UDF_CTRL_UDF_TYPE(1)
+#define   YT921X_UDF_CTRL_UDF_TYPE_L4			YT921X_UDF_CTRL_UDF_TYPE(2)
+#define  YT921X_UDF_CTRL_UDF_OFFSET_M		GENMASK(6, 0)
+#define   YT921X_UDF_CTRL_UDF_OFFSET(x)			FIELD_PREP(YT921X_UDF_CTRL_UDF_OFFSET_M, (x))
+
+#define YT921X_PORTn_RATE(port)		(0x220000 + 4 * (port))
+#define  YT921X_PORT_RATE_GAP_VALUE		GENMASK(4, 0)	/* default 20 */
+#define YT921X_METER_SLOT		0x220104
+#define  YT921X_METER_SLOT_SLOT_M		GENMASK(11, 0)
+#define YT921X_PORTn_METER(port)	(0x220108 + 4 * (port))
+#define  YT921X_PORT_METER_EN			BIT(4)
+#define  YT921X_PORT_METER_ID_M			GENMASK(3, 0)
+#define   YT921X_PORT_METER_ID(x)			FIELD_PREP(YT921X_PORT_METER_ID_M, (x))
+#define YT921X_METERn_CTRL(x)		(0x220800 + 0x10 * (x))
+#define  YT921X_METER_CTRLc_METER_EN		BIT(14)
+#define  YT921X_METER_CTRLc_TOKEN_OVERFLOW_EN	BIT(13)	/* RFC4115: yellow use unused green bw */
+#define  YT921X_METER_CTRLc_DROP_M		GENMASK(12, 11)
+#define   YT921X_METER_CTRLc_DROP(x)			FIELD_PREP(YT921X_METER_CTRLc_DROP_M, (x))
+#define   YT921X_METER_CTRLc_DROP_GYR			YT921X_METER_CTRLc_DROP(0)
+#define   YT921X_METER_CTRLc_DROP_YR			YT921X_METER_CTRLc_DROP(1)
+#define   YT921X_METER_CTRLc_DROP_R			YT921X_METER_CTRLc_DROP(2)
+#define   YT921X_METER_CTRLc_DROP_NONE			YT921X_METER_CTRLc_DROP(3)
+#define  YT921X_METER_CTRLc_COLOR_BLIND		BIT(10)
+#define  YT921X_METER_CTRLc_UNIT_M		GENMASK(9, 7)
+#define   YT921X_METER_CTRLc_UNIT(x)			FIELD_PREP(YT921X_METER_CTRLc_UNIT_M, (x))
+#define  YT921X_METER_CTRLc_BYTE_MODE_INCLUDE_GAP	BIT(6)	/* +GAP_VALUE bytes each packet */
+#define  YT921X_METER_CTRLc_PKT_MODE		BIT(5)	/* 0: byte rate mode */
+#define  YT921X_METER_CTRLc_RFC2698		BIT(4)	/* 0: RFC4115 */
+#define  YT921X_METER_CTRLbc_CBS_M		GENMASK_ULL(35, 20)
+#define   YT921X_METER_CTRLbc_CBS(x)			FIELD_PREP(YT921X_METER_CTRLbc_CBS_M, (x))
+#define  YT921X_METER_CTRLb_CIR_M		GENMASK(19, 2)
+#define   YT921X_METER_CTRLb_CIR(x)			FIELD_PREP(YT921X_METER_CTRLb_CIR_M, (x))
+#define  YT921X_METER_CTRLab_EBS_M		GENMASK_ULL(33, 18)
+#define   YT921X_METER_CTRLab_EBS(x)			FIELD_PREP(YT921X_METER_CTRLab_EBS_M, (x))
+#define  YT921X_METER_CTRLa_EIR_M		GENMASK(17, 0)
+#define   YT921X_METER_CTRLa_EIR(x)			FIELD_PREP(YT921X_METER_CTRLa_EIR_M, (x))
+#define YT921X_METERn_STAT(x)		(0x221000 + 8 * (x))
+
+#define YT921X_PORTn_VLAN_CTRL(port)	(0x230010 + 4 * (port))
+#define  YT921X_PORT_VLAN_CTRL_SVLAN_PRIO_EN	BIT(31)
+#define  YT921X_PORT_VLAN_CTRL_CVLAN_PRIO_EN	BIT(30)
+#define  YT921X_PORT_VLAN_CTRL_SVID_M		GENMASK(29, 18)
+#define   YT921X_PORT_VLAN_CTRL_SVID(x)			FIELD_PREP(YT921X_PORT_VLAN_CTRL_SVID_M, (x))
+#define  YT921X_PORT_VLAN_CTRL_CVID_M		GENMASK(17, 6)
+#define   YT921X_PORT_VLAN_CTRL_CVID(x)			FIELD_PREP(YT921X_PORT_VLAN_CTRL_CVID_M, (x))
+#define  YT921X_PORT_VLAN_CTRL_SVLAN_PRIO_M	GENMASK(5, 3)
+#define  YT921X_PORT_VLAN_CTRL_CVLAN_PRIO_M	GENMASK(2, 0)
+#define YT921X_PORTn_VLAN_CTRL1(port)	(0x230080 + 4 * (port))
+#define  YT921X_PORT_VLAN_CTRL1_VLAN_RANGE_EN	BIT(8)
+#define  YT921X_PORT_VLAN_CTRL1_VLAN_RANGE_PROFILE_ID_M	GENMASK(7, 4)
+#define  YT921X_PORT_VLAN_CTRL1_SVLAN_DROP_TAGGED	BIT(3)
+#define  YT921X_PORT_VLAN_CTRL1_SVLAN_DROP_UNTAGGED	BIT(2)
+#define  YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_TAGGED	BIT(1)
+#define  YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED	BIT(0)
+
+#define YT921X_MIRROR			0x300300
+#define  YT921X_MIRROR_IGR_PORTS_M		GENMASK(26, 16)
+#define   YT921X_MIRROR_IGR_PORTS(x)			FIELD_PREP(YT921X_MIRROR_IGR_PORTS_M, (x))
+#define  YT921X_MIRROR_IGR_PORTn(port)		BIT((port) + 16)
+#define  YT921X_MIRROR_EGR_PORTS_M		GENMASK(14, 4)
+#define   YT921X_MIRROR_EGR_PORTS(x)			FIELD_PREP(YT921X_MIRROR_EGR_PORTS_M, (x))
+#define  YT921X_MIRROR_EGR_PORTn(port)		BIT((port) + 4)
+#define  YT921X_MIRROR_PORT_M			GENMASK(3, 0)
+#define   YT921X_MIRROR_PORT(x)				FIELD_PREP(YT921X_MIRROR_PORT_M, (x))
+
+#define YT921X_PORT_SHAPE_SLOT		0x34000c
+#define  YT921X_PORT_SHAPE_SLOT_SLOT_M		GENMASK(11, 0)
+#define YT921X_PORTn_SHAPE_CTRL(port)	(0x354000 + 8 * (port))
+#define  YT921X_PORT_SHAPE_CTRLb_EN		BIT(4)
+#define  YT921X_PORT_SHAPE_CTRLb_PKT_MODE	BIT(3)	/* 0: byte rate mode */
+#define  YT921X_PORT_SHAPE_CTRLb_UNIT_M		GENMASK(2, 0)
+#define   YT921X_PORT_SHAPE_CTRLb_UNIT(x)		FIELD_PREP(YT921X_PORT_SHAPE_CTRLb_UNIT_M, (x))
+#define  YT921X_PORT_SHAPE_CTRLa_CBS_M		GENMASK(31, 18)
+#define   YT921X_PORT_SHAPE_CTRLa_CBS(x)		FIELD_PREP(YT921X_PORT_SHAPE_CTRLa_CBS_M, (x))
+#define  YT921X_PORT_SHAPE_CTRLa_CIR_M		GENMASK(17, 0)
+#define   YT921X_PORT_SHAPE_CTRLa_CIR(x)		FIELD_PREP(YT921X_PORT_SHAPE_CTRLa_CIR_M, (x))
+#define YT921X_PORTn_SHAPE_STAT(port)	(0x356000 + 4 * (port))
+
+#define YT921X_EDATA_EXTMODE	0xfb
+#define YT921X_EDATA_LEN	0x100
+
+#define YT921X_FDB_NUM	4096
+
+enum yt921x_fdb_entry_status {
+	YT921X_FDB_ENTRY_STATUS_INVALID = 0,
+	YT921X_FDB_ENTRY_STATUS_MIN_TIME = 1,
+	YT921X_FDB_ENTRY_STATUS_MOVE_AGING_MAX_TIME = 3,
+	YT921X_FDB_ENTRY_STATUS_MAX_TIME = 5,
+	YT921X_FDB_ENTRY_STATUS_PENDING = 6,
+	YT921X_FDB_ENTRY_STATUS_STATIC = 7,
+};
+
+#define YT921X_MSTI_NUM		16
+
+#define YT921X_TOKEN_BYTE_C	1	/* 1 token = 2^1 byte */
+#define YT921X_TOKEN_PKT_C	-6	/* 1 token = 2^-6 packets */
+#define YT921X_TOKEN_RATE_C	-15
+/* Custom meters only, not including dedicated port meters (11) */
+#define YT921X_METER_NUM	64
+#define YT921X_METER_SLOT_MIN	80
+#define YT921X_METER_UNIT_MAX	((1 << 3) - 1)
+#define YT921X_METER_CIR_MAX	((1 << 18) - 1)
+#define YT921X_METER_CBS_MAX	((1 << 16) - 1)
+#define YT921X_PORT_SHAPE_SLOT_MIN	80
+#define YT921X_SHAPE_UNIT_MAX	((1 << 3) - 1)
+#define YT921X_SHAPE_CIR_MAX	((1 << 18) - 1)
+#define YT921X_SHAPE_CBS_MAX	((1 << 14) - 1)
+
+#define YT921X_LAG_NUM		2
+#define YT921X_LAG_PORT_NUM	4
+
+#define YT921X_PRIO_NUM	8
+
+#define YT9215_MAJOR	0x9002
+#define YT9218_MAJOR	0x9001
+
+/* required for a hard reset */
+#define YT921X_RST_DELAY_US	10000
+
+#define YT921X_FRAME_SIZE_MAX	0x2400  /* 9216 */
+
+#define YT921X_TAG_LEN	8
+
+#define YT921X_ACL_BLK_NUM	48
+#define YT921X_ACL_ENT_PER_BLK	8
+#define YT921X_ACL_NUM		(YT921X_ACL_BLK_NUM * YT921X_ACL_ENT_PER_BLK)
+#define YT921X_UDF_NUM		8
+
+#define YT921X_LED_GROUP_NUM	3
+#define YT921X_LED_PORT_NUM	10
+
+/* 8 internal + 2 external + 1 mcu */
+#define YT921X_PORT_NUM			11
+
+#define YT921X_NAME	"yt921x"
+
+struct yt921x_mib {
+	u64 rx_broadcast;
+	u64 rx_pause;
+	u64 rx_multicast;
+	u64 rx_crc_errors;
+
+	u64 rx_alignment_errors;
+	u64 rx_undersize_errors;
+	u64 rx_fragment_errors;
+	u64 rx_64byte;
+
+	u64 rx_65_127byte;
+	u64 rx_128_255byte;
+	u64 rx_256_511byte;
+	u64 rx_512_1023byte;
+
+	u64 rx_1024_1518byte;
+	u64 rx_jumbo;
+	u64 rx_good_bytes;
+
+	u64 rx_bad_bytes;
+	u64 rx_oversize_errors;
+
+	u64 rx_dropped;
+	u64 tx_broadcast;
+	u64 tx_pause;
+	u64 tx_multicast;
+
+	u64 tx_undersize_errors;
+	u64 tx_64byte;
+	u64 tx_65_127byte;
+	u64 tx_128_255byte;
+
+	u64 tx_256_511byte;
+	u64 tx_512_1023byte;
+	u64 tx_1024_1518byte;
+	u64 tx_jumbo;
+
+	u64 tx_good_bytes;
+	u64 tx_collisions;
+
+	u64 tx_aborted_errors;
+	u64 tx_multiple_collisions;
+	u64 tx_single_collisions;
+	u64 tx_good;
+
+	u64 tx_deferred;
+	u64 tx_late_collisions;
+	u64 rx_oam;
+	u64 tx_oam;
+};
+
+struct yt921x_acl_entry {
+	u32 key[2];
+	u32 mask[2];
+};
+
+struct yt921x_acl_rule {
+	unsigned long tag;
+	enum tc_setup_type type;
+
+	u32 action[3];
+	bool sw_assisted;
+
+	u8 mask;
+	struct yt921x_acl_entry entries[YT921X_ACL_ENT_PER_BLK];
+};
+
+struct yt921x_acl_blk {
+	struct yt921x_acl_rule *rules[YT921X_ACL_ENT_PER_BLK];
+};
+
+struct yt921x_port {
+	unsigned char index;
+
+	/* SerDes in use */
+	bool serdes:1;
+	/* Link from in-band status (PHYLINK_PCS_NEG_INBAND) */
+	bool inband:1;
+	/* BR_HAIRPIN_MODE */
+	bool hairpin:1;
+	/* BR_ISOLATED */
+	bool isolated:1;
+
+	struct delayed_work mib_read;
+	struct yt921x_mib mib;
+	u64 rx_frames;
+	u64 tx_frames;
+
+#if IS_ENABLED(CONFIG_NET_DSA_YT921X_LEDS)
+	unsigned char led_duty;
+	unsigned short led_cycle;
+
+	unsigned char led_duty_mask;
+	unsigned char led_cycle_mask;
+
+	struct yt921x_led *leds[YT921X_LED_GROUP_NUM];
+#endif
+
+	struct phylink_pcs pcs;
+};
+
+#define pcs_to_yt921x_port(_pcs) container_of((_pcs), struct yt921x_port, pcs)
+
+struct yt921x_reg_ops {
+	int (*read)(void *context, u32 reg, u32 *valp);
+	int (*write)(void *context, u32 reg, u32 val);
+};
+
+struct yt921x_info {
+	const char *name;
+	u16 major;
+	/* Unknown, seems to be plain enumeration */
+	u8 mode;
+	u8 extmode;
+	/* Ports with integral GbE PHYs, not including MCU Port 10 */
+	u16 internal_mask;
+	/* Note: xmii_mask and serdes_mask may overlap */
+	u16 xmii_mask;
+	u16 serdes_mask;
+};
+
+struct yt921x_priv {
+	struct dsa_switch ds;
+
+	const struct yt921x_info *info;
+	unsigned int meter_slot_ns;
+	unsigned int port_shape_slot_ns;
+	/* cache of dsa_cpu_ports(ds) */
+	u16 cpu_ports_mask;
+	unsigned char cycle_ns;
+
+	/* protect the access to the switch registers */
+	struct mutex reg_lock;
+	const struct yt921x_reg_ops *reg_ops;
+	void *reg_ctx;
+
+	/* mdio master bus */
+	struct mii_bus *mbus_int;
+	struct mii_bus *mbus_ext;
+
+	struct yt921x_port ports[YT921X_PORT_NUM];
+
+	u16 eee_ports_mask;
+
+	DECLARE_BITMAP(meters_map, YT921X_METER_NUM);
+
+	u8 acl_masks[YT921X_ACL_BLK_NUM];
+	struct yt921x_acl_blk *acl_blks[YT921X_ACL_BLK_NUM];
+};
+
+#define yt921x_port_to_priv(pp) \
+	container_of_const((pp), struct yt921x_priv, ports[(pp)->index])
+
+static inline int ethtool_speed_to_yt921x(int speed)
+{
+	switch (speed) {
+	case SPEED_10:
+		return YT921X_SPEED_10;
+	case SPEED_100:
+		return YT921X_SPEED_100;
+	case SPEED_1000:
+		return YT921X_SPEED_1000;
+	case SPEED_2500:
+		return YT921X_SPEED_2500;
+	case SPEED_10000:
+		return YT921X_SPEED_10000;
+	default:
+		return YT921X_SPEED_INVALID;
+	}
+}
+
+#endif
diff --git a/drivers/net/dsa/motorcomm/leds.c b/drivers/net/dsa/motorcomm/leds.c
new file mode 100644
index 00000000000000..6613fe3b0471e4
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/leds.c
@@ -0,0 +1,641 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#include <linux/uleds.h>
+
+#include "chip.h"
+#include "leds.h"
+#include "smi.h"
+
+#define to_yt921x_led(led_cdev) \
+	container_of_const((led_cdev), struct yt921x_led, cdev)
+#define to_yt921x_port(led) ((led)->port)
+#define to_device(priv) ((priv)->ds.dev)
+
+static u32 yt921x_led_regaddr(struct yt921x_priv *priv, int port, int group)
+{
+	switch (group) {
+	case 0:
+	default:
+		return YT921X_LED0_PORTn(port);
+	case 1:
+		return YT921X_LED1_PORTn(port);
+	case 2:
+		return YT921X_LED2_PORTn(port);
+	}
+}
+
+static int
+yt921x_led_force_set(struct yt921x_priv *priv, int port, int group, bool on)
+{
+	struct yt921x_port *pp = &priv->ports[port];
+	u32 ctrl;
+	u32 mask;
+
+	/* No reversion - LED is sacrificed in case of IO error */
+	pp->led_duty_mask &= ~BIT(group);
+	pp->led_cycle_mask &= ~BIT(group);
+
+	mask = YT921X_LED2_PORT_FORCEn_M(group);
+	ctrl = on ? YT921X_LED2_PORT_FORCEn_ON(group) :
+	       YT921X_LED2_PORT_FORCEn_OFF(group);
+	return yt921x_reg_update_bits(priv, YT921X_LED2_PORTn(port), mask,
+				      ctrl);
+}
+
+static int
+yt921x_led_blink_select(const struct yt921x_priv *priv, unsigned long on,
+			unsigned long off, unsigned short *cyclep,
+			unsigned char *dutyp)
+{
+	static const unsigned char dutys[] = {
+		YT921X_LED_DUTY(1, 6),
+		YT921X_LED_DUTY(1, 4),
+		YT921X_LED_DUTY(1, 3),
+		YT921X_LED_DUTY(1, 2),
+	};
+	unsigned int cycle_upper;
+	unsigned int cycle_req;
+	unsigned int duty_req;
+	unsigned int cycle;
+	unsigned int duty;
+
+	cycle = YT921X_LED_BLINK_MAX;
+	cycle_upper = cycle * 11585 / 8192 + 1;  /* M_SQRT2 * cycle */
+	if (check_add_overflow(on, off, &cycle_req) || cycle_req >= cycle_upper)
+		return -EOPNOTSUPP;
+
+	for (; cycle > YT921X_LED_BLINK_MIN; cycle_upper >>= 1, cycle >>= 1)
+		if (cycle_req >= cycle_upper >> 1)
+			break;
+	*cyclep = cycle;
+
+	duty_req = DIV_ROUND_CLOSEST(YT921X_LED_DUTY_DENOM *
+				     (on > off ? off : on), cycle_req);
+	for (unsigned int i = ARRAY_SIZE(dutys) - 1;; i--)
+		if (i == 0 || duty_req >= (dutys[i - 1] + dutys[i]) / 2) {
+			duty = dutys[i];
+			break;
+		}
+	if (on > off)
+		duty = YT921X_LED_DUTY_DENOM - duty;
+	*dutyp = duty;
+
+	return 0;
+}
+
+static int
+yt921x_led_blink_set(struct yt921x_priv *priv, int port, int group,
+		     unsigned long *onp, unsigned long *offp)
+{
+	struct yt921x_port *pp = &priv->ports[port];
+	unsigned short cycle;
+	unsigned char duty;
+	bool use_cycle;
+	u32 ctrl;
+	u32 mask;
+	u32 val;
+	int res;
+
+	if (!*onp && !*offp) {
+		cycle = YT921X_LED_BLINK_DEF;
+		duty = (pp->led_duty_mask & ~BIT(group)) ? pp->led_duty :
+		       YT921X_LED_DUTY(1, 2);
+
+		use_cycle = false;
+	} else if (!*onp || !*offp) {
+		return yt921x_led_force_set(priv, port, group, *onp);
+	} else {
+		res = yt921x_led_blink_select(priv, *onp, *offp, &cycle, &duty);
+		if (res)
+			return res;
+
+		use_cycle = cycle < YT921X_LED_BLINK_DEF;
+
+		if (use_cycle && cycle != pp->led_cycle &&
+		    (pp->led_cycle_mask & ~BIT(group)))
+			return -EOPNOTSUPP;
+		if (duty != pp->led_duty && (pp->led_duty_mask & ~BIT(group)))
+			return -EOPNOTSUPP;
+	}
+
+	pp->led_cycle_mask &= ~BIT(group);
+	pp->led_duty_mask &= ~BIT(group);
+
+	/* The chip seems to jam a while if changing duty directly */
+	res = yt921x_reg_read(priv, YT921X_LED2_PORTn(port), &val);
+	if (res)
+		return res;
+
+	ctrl = val & ~YT921X_LED2_PORT_FORCEn_M(group);
+	ctrl |= YT921X_LED2_PORT_FORCEn_OFF(group);
+	if (val != ctrl) {
+		res = yt921x_reg_write(priv, YT921X_LED2_PORTn(port), ctrl);
+		if (res)
+			return res;
+	}
+
+	mask = YT921X_LED1_PORT_BLINK_DUTY_M | YT921X_LED1_PORT_BLINK_DUTY_COMP;
+	switch (duty >= YT921X_LED_DUTY(1, 2) ? duty :
+		YT921X_LED_DUTY_DENOM - duty) {
+	default:
+		duty = YT921X_LED_DUTY(1, 2);
+		fallthrough;
+	case YT921X_LED_DUTY(1, 2):
+		ctrl = YT921X_LED1_PORT_BLINK_DUTY_1_2;
+		break;
+	case YT921X_LED_DUTY(2, 3):
+		ctrl = YT921X_LED1_PORT_BLINK_DUTY_2_3;
+		break;
+	case YT921X_LED_DUTY(3, 4):
+		ctrl = YT921X_LED1_PORT_BLINK_DUTY_3_4;
+		break;
+	case YT921X_LED_DUTY(5, 6):
+		ctrl = YT921X_LED1_PORT_BLINK_DUTY_5_6;
+		break;
+	}
+	if (duty < YT921X_LED_DUTY(1, 2))
+		ctrl |= YT921X_LED1_PORT_BLINK_DUTY_COMP;
+	if (use_cycle) {
+		mask |= YT921X_LED1_PORT_OTHER_BLINK_M;
+		ctrl |= YT921X_LED1_PORT_OTHER_BLINK(9 - __fls(cycle));
+	}
+	res = yt921x_reg_update_bits(priv, YT921X_LED1_PORTn(port), mask, ctrl);
+	if (res)
+		return res;
+
+	ctrl = val & ~(YT921X_LED2_PORT_FORCEn_M(group) |
+		       YT921X_LED2_PORT_FORCE_BLINKn_M(group));
+	ctrl |= YT921X_LED2_PORT_FORCEn_BLINK(group);
+	if (use_cycle)
+		ctrl |= YT921X_LED2_PORT_FORCE_BLINKn_OTHER(group);
+	else
+		ctrl |= YT921X_LED2_PORT_FORCE_BLINKn(group, __fls(cycle) - 9);
+	res = yt921x_reg_write(priv, YT921X_LED2_PORTn(port), ctrl);
+	if (res)
+		return res;
+
+	if (use_cycle) {
+		pp->led_cycle_mask |= BIT(group);
+		pp->led_cycle = cycle;
+	}
+	pp->led_duty_mask |= BIT(group);
+	pp->led_duty = duty;
+
+	*onp = DIV_ROUND_CLOSEST(duty * cycle, YT921X_LED_DUTY_DENOM);
+	*offp = cycle - *onp;
+	return 0;
+}
+
+struct yt921x_led_trigger_map {
+	unsigned long flags;
+	u32 mask;
+};
+
+static const struct yt921x_led_trigger_map yt921x_led_trigger_maps[] = {
+	{BIT(TRIGGER_NETDEV_LINK),
+	 YT921X_LEDx_PORT_ACT_DUPLEX_HALF | YT921X_LEDx_PORT_ACT_DUPLEX_FULL},
+	{BIT(TRIGGER_NETDEV_LINK_10), YT921X_LEDx_PORT_ACT_10M},
+	{BIT(TRIGGER_NETDEV_LINK_100), YT921X_LEDx_PORT_ACT_100M},
+	{BIT(TRIGGER_NETDEV_LINK_1000), YT921X_LEDx_PORT_ACT_1000M},
+	{BIT(TRIGGER_NETDEV_HALF_DUPLEX), YT921X_LEDx_PORT_ACT_DUPLEX_HALF},
+	{BIT(TRIGGER_NETDEV_FULL_DUPLEX), YT921X_LEDx_PORT_ACT_DUPLEX_FULL},
+	{BIT(TRIGGER_NETDEV_TX), YT921X_LEDx_PORT_ACT_TX_BLINK},
+	{BIT(TRIGGER_NETDEV_RX), YT921X_LEDx_PORT_ACT_RX_BLINK},
+	{BIT(TRIGGER_NETDEV_TX_ERR) | BIT(TRIGGER_NETDEV_RX_ERR),
+	 YT921X_LEDx_PORT_ACT_COLLISION_BLINK},
+};
+
+static bool
+yt921x_led_trigger_is_supported(const struct yt921x_priv *priv, int port,
+				int group, unsigned long flags)
+{
+	if (!flags)
+		return true;
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(yt921x_led_trigger_maps); i++) {
+		const struct yt921x_led_trigger_map *map = &yt921x_led_trigger_maps[i];
+
+		if ((flags & map->flags) == map->flags) {
+			flags &= ~map->flags;
+			if (!flags)
+				return true;
+		}
+	}
+
+	return false;
+}
+
+static int
+yt921x_led_trigger_get(struct yt921x_priv *priv, int port, int group,
+		       unsigned long *flagsp)
+{
+	u32 addr;
+	u32 val;
+	int res;
+
+	res = yt921x_reg_read(priv, YT921X_LED2_PORTn(port), &val);
+	if (res)
+		return res;
+
+	if ((val & YT921X_LED2_PORT_FORCEn_M(group)) !=
+	    YT921X_LED2_PORT_FORCEn_DONTCARE(group)) {
+		*flagsp = 0;
+		return 0;
+	}
+
+	if (group != 2) {
+		addr = yt921x_led_regaddr(priv, port, group);
+		res = yt921x_reg_read(priv, addr, &val);
+		if (res)
+			return res;
+	}
+
+	*flagsp = 0;
+	for (unsigned int i = 0; i < ARRAY_SIZE(yt921x_led_trigger_maps); i++) {
+		const struct yt921x_led_trigger_map *map = &yt921x_led_trigger_maps[i];
+
+		if ((val & map->mask) == map->mask)
+			*flagsp |= map->flags;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_led_trigger_set(struct yt921x_priv *priv, int port, int group,
+		       unsigned long flags)
+{
+	struct yt921x_port *pp = &priv->ports[port];
+	u32 addr;
+	u32 ctrl;
+	u32 mask;
+	int res;
+
+	ctrl = 0;
+	for (unsigned int i = 0; i < ARRAY_SIZE(yt921x_led_trigger_maps); i++) {
+		const struct yt921x_led_trigger_map *map = &yt921x_led_trigger_maps[i];
+
+		if ((flags & map->flags) == map->flags) {
+			flags &= ~map->flags;
+			ctrl |= map->mask;
+			if (!flags)
+				break;
+		}
+	}
+	if (flags)
+		return -EOPNOTSUPP;
+
+	pp->led_duty_mask &= ~BIT(group);
+	pp->led_cycle_mask &= ~BIT(group);
+
+	mask = !group ? YT921X_LED0_PORT_ACT_M : YT921X_LEDx_PORT_ACT_M;
+	if (group == 2) {
+		mask |= YT921X_LED2_PORT_FORCEn_M(group);
+		ctrl |= YT921X_LED2_PORT_FORCEn_DONTCARE(group);
+	}
+	addr = yt921x_led_regaddr(priv, port, group);
+	res = yt921x_reg_update_bits(priv, addr, mask, ctrl);
+	if (res)
+		return res;
+
+	if (group != 2) {
+		mask = YT921X_LED2_PORT_FORCEn_M(group);
+		ctrl = YT921X_LED2_PORT_FORCEn_DONTCARE(group);
+		res = yt921x_reg_update_bits(priv, YT921X_LED2_PORTn(port),
+					     mask, ctrl);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_cled_brightness_set_blocking(struct led_classdev *led_cdev,
+				    enum led_brightness brightness)
+{
+	struct yt921x_led *led = to_yt921x_led(led_cdev);
+	struct yt921x_port *pp = to_yt921x_port(led);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_led_force_set(priv, pp->index, led->group, brightness);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_cled_blink_set(struct led_classdev *led_cdev, unsigned long *delay_on,
+		      unsigned long *delay_off)
+{
+	struct yt921x_led *led = to_yt921x_led(led_cdev);
+	struct yt921x_port *pp = to_yt921x_port(led);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_led_blink_set(priv, pp->index, led->group, delay_on,
+				   delay_off);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static struct device __maybe_unused *
+yt921x_cled_hw_control_get_device(struct led_classdev *led_cdev)
+{
+	struct yt921x_led *led = to_yt921x_led(led_cdev);
+	struct yt921x_port *pp = to_yt921x_port(led);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	struct dsa_port *dp;
+
+	dp = dsa_to_port(&priv->ds, pp->index);
+	if (!dp)
+		return NULL;
+
+	if (dsa_port_is_user(dp))
+		return !dp->user ? NULL : &dp->user->dev;
+	if (dsa_port_is_cpu(dp))
+		return !dp->conduit ? NULL : &dp->conduit->dev;
+
+	return NULL;
+}
+
+static int __maybe_unused
+yt921x_cled_hw_control_is_supported(struct led_classdev *led_cdev,
+				    unsigned long flags)
+{
+	struct yt921x_led *led = to_yt921x_led(led_cdev);
+	struct yt921x_port *pp = to_yt921x_port(led);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+
+	if (yt921x_led_trigger_is_supported(priv, pp->index, led->group, flags))
+		return 0;
+	return -EOPNOTSUPP;
+}
+
+static int __maybe_unused
+yt921x_cled_hw_control_get(struct led_classdev *led_cdev, unsigned long *flagsp)
+{
+	struct yt921x_led *led = to_yt921x_led(led_cdev);
+	struct yt921x_port *pp = to_yt921x_port(led);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_led_trigger_get(priv, pp->index, led->group, flagsp);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int __maybe_unused
+yt921x_cled_hw_control_set(struct led_classdev *led_cdev, unsigned long flags)
+{
+	struct yt921x_led *led = to_yt921x_led(led_cdev);
+	struct yt921x_port *pp = to_yt921x_port(led);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_led_trigger_set(priv, pp->index, led->group, flags);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_led_setup(struct yt921x_priv *priv, int port,
+		 struct fwnode_handle *fwnode)
+{
+	struct yt921x_port *pp = &priv->ports[port];
+	struct device *dev = to_device(priv);
+	struct led_init_data init_data;
+	struct led_classdev *led_cdev;
+	char name[LED_MAX_NAME_SIZE];
+	enum led_default_state state;
+	struct yt921x_led *led;
+	bool force_high;
+	bool force_low;
+	u32 led2_val;
+	u32 inv_val;
+	u32 group;
+	u32 mask;
+	u32 ctrl;
+	bool on;
+	int res;
+	int ret;
+
+	res = fwnode_property_read_u32(fwnode, "reg", &group);
+	if (res) {
+		dev_err(dev, "Missing LED reg for %pfw\n", fwnode);
+		return res;
+	}
+	if (group >= YT921X_LED_GROUP_NUM) {
+		dev_err(dev, "Invalid LED reg %u for port %d\n", group, port);
+		return -EINVAL;
+	}
+	if (pp->leds[group]) {
+		res = -EEXIST;
+		goto err;
+	}
+
+	force_high = fwnode_property_read_bool(fwnode, "active-high");
+	force_low = fwnode_property_read_bool(fwnode, "active-low");
+	if (force_high && force_low) {
+		dev_err(dev, "Duplicate polarities for LED %02d:%02u\n",
+			port, group);
+		return -EINVAL;
+	}
+
+	led = devm_kzalloc(dev, sizeof(*led), GFP_KERNEL);
+	if (!led) {
+		res = -ENOMEM;
+		goto err;
+	}
+	pp->leds[group] = led;
+
+	led->port = pp;
+	led->group = group;
+
+	state = led_init_default_state_get(fwnode);
+
+	mutex_lock(&priv->reg_lock);
+
+	/* Inversion is internal - force on will give low logic.
+	 * In the rest of the file, treat LEDs as if active-low.
+	 */
+	inv_val = U32_MAX;
+	if (force_high || force_low) {
+		res = yt921x_reg_read(priv, YT921X_LED_PAR_INV, &inv_val);
+		if (res)
+			goto revoke_inv;
+
+		mask = YT921X_LED_PAR_INV_INVnm(group, port);
+		ctrl = force_high ? inv_val | mask : inv_val & ~mask;
+		res = yt921x_reg_write(priv, YT921X_LED_PAR_INV, ctrl);
+		if (res)
+			goto revoke_inv;
+	}
+
+	led2_val = U32_MAX;
+	res = yt921x_reg_read(priv, YT921X_LED2_PORTn(port), &led2_val);
+	if (res)
+		goto revoke_led2;
+	mask = YT921X_LED2_PORT_FORCEn_M(group);
+
+	switch (state) {
+	case LEDS_DEFSTATE_OFF:
+	case LEDS_DEFSTATE_ON:
+	default:
+		on = state == LEDS_DEFSTATE_ON;
+		ctrl = on ? YT921X_LED2_PORT_FORCEn_ON(group) :
+		       YT921X_LED2_PORT_FORCEn_OFF(group);
+		res = yt921x_reg_write(priv, YT921X_LED2_PORTn(port),
+				       (led2_val & ~mask) | ctrl);
+		if (res)
+			goto revoke_led2;
+		break;
+	case LEDS_DEFSTATE_KEEP:
+		on = (led2_val & mask) == YT921X_LED2_PORT_FORCEn_ON(group);
+		led2_val = U32_MAX;
+		break;
+	}
+
+	mutex_unlock(&priv->reg_lock);
+
+	led_cdev = &led->cdev;
+	led_cdev->brightness = on;
+	led_cdev->max_brightness = 1;
+	led_cdev->flags = LED_RETAIN_AT_SHUTDOWN;
+	led_cdev->brightness_set_blocking = yt921x_cled_brightness_set_blocking;
+	led_cdev->blink_set = yt921x_cled_blink_set;
+#ifdef CONFIG_LEDS_TRIGGERS
+	led_cdev->hw_control_trigger = "netdev";
+	led_cdev->hw_control_get_device = yt921x_cled_hw_control_get_device;
+	led_cdev->hw_control_is_supported = yt921x_cled_hw_control_is_supported;
+	led_cdev->hw_control_get = yt921x_cled_hw_control_get;
+	led_cdev->hw_control_set = yt921x_cled_hw_control_set;
+#endif
+
+	snprintf(name, sizeof(name), YT921X_NAME "-%u:%02d:%02u",
+		 priv->ds.index, port, group);
+	init_data = (typeof(init_data)){
+		.fwnode = fwnode,
+		.default_label = ":port",
+		.devicename = name,
+		.devname_mandatory = true,
+	};
+	res = devm_led_classdev_register_ext(dev, led_cdev, &init_data);
+	if (res)
+		goto revoke;
+
+	return 0;
+
+revoke:
+	mutex_lock(&priv->reg_lock);
+revoke_led2:
+	if (led2_val != U32_MAX) {
+		ret = yt921x_reg_write(priv, YT921X_LED2_PORTn(port), led2_val);
+		if (ret)
+			dev_err(dev,
+				"Failed to restore %s for LED %02d:%02u: %d\n",
+				"LED2_PORT", port, group, ret);
+	}
+revoke_inv:
+	if (inv_val != U32_MAX) {
+		ret = yt921x_reg_write(priv, YT921X_LED_PAR_INV, inv_val);
+		if (ret)
+			dev_err(dev,
+				"Failed to restore %s for LED %02d:%02u: %d\n",
+				"LED_PAR_INV", port, group, ret);
+	}
+	mutex_unlock(&priv->reg_lock);
+
+	pp->leds[group] = NULL;
+	devm_kfree(dev, led);
+
+err:
+	dev_err(dev, "Failed to initialize LED %02d:%02u: %d\n",
+		port, group, res);
+	return res;
+}
+
+static void yt921x_leds_remove_port(struct yt921x_priv *priv, int port)
+{
+	struct yt921x_port *pp = &priv->ports[port];
+	struct device *dev = to_device(priv);
+
+	for (int group = 0; group < YT921X_LED_GROUP_NUM; group++) {
+		struct yt921x_led *led = pp->leds[group];
+
+		if (led) {
+			devm_led_classdev_unregister(dev, &led->cdev);
+			pp->leds[group] = NULL;
+			devm_kfree(dev, led);
+		}
+	}
+
+	pp->led_duty_mask = 0;
+	pp->led_cycle_mask = 0;
+}
+
+static int yt921x_leds_setup_port(struct yt921x_priv *priv, int port)
+{
+	struct device *dev = to_device(priv);
+	struct dsa_switch *ds = &priv->ds;
+	struct device_node *leds_np;
+	struct dsa_port *dp;
+
+	dp = dsa_to_port(ds, port);
+	leds_np = of_get_child_by_name(dp->dn, "leds");
+	if (!leds_np)
+		return 0;
+
+	if (port >= YT921X_LED_PORT_NUM) {
+		dev_err(dev, "Cannot configure LEDs for port %d\n", port);
+		of_node_put(leds_np);
+		return -EINVAL;
+	}
+
+	for_each_child_of_node_scoped(leds_np, led_np) {
+		yt921x_led_setup(priv, port, of_fwnode_handle(led_np));
+		/* Allow partial configuration: LEDs are optional */
+	}
+
+	of_node_put(leds_np);
+	return 0;
+}
+
+void yt921x_leds_remove(struct yt921x_priv *priv)
+{
+	for (int port = 0; port < YT921X_LED_PORT_NUM; port++)
+		yt921x_leds_remove_port(priv, port);
+}
+
+int yt921x_leds_setup(struct yt921x_priv *priv)
+{
+	struct dsa_switch *ds = &priv->ds;
+	struct dsa_port *dp;
+
+	/* LEDs are always enabled. There is no way to disable them altogether
+	 * (as far as I know).
+	 */
+
+	dsa_switch_for_each_port(dp, ds) {
+		int port = dp->index;
+
+		if (!dp->dn)
+			continue;
+
+		yt921x_leds_setup_port(priv, port);
+		/* Allow partial configuration: LEDs are optional */
+	}
+
+	return 0;
+}
diff --git a/drivers/net/dsa/motorcomm/leds.h b/drivers/net/dsa/motorcomm/leds.h
new file mode 100644
index 00000000000000..11df479ee58103
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/leds.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#ifndef _YT_LEDS_H
+#define _YT_LEDS_H
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/kconfig.h>
+#include <linux/leds.h>
+
+#define YT921X_LED_CTRL			0xd0000
+#define  YT921X_LED_CTRL_UNK			BIT(21)
+#define  YT921X_LED_CTRL_LOOPDETECT_BLINK_M	GENMASK(20, 19)	/* cycle = 512 * x ms */
+#define   YT921X_LED_CTRL_LOOPDETECT_BLINK(x)		FIELD_PREP(YT921X_LED_CTRL_LOOPDETECT_BLINK_M, (x))
+#define  YT921X_LED_CTRL_PORT_NUM_M		GENMASK(16, 13)
+#define   YT921X_LED_CTRL_PORT_NUM(x)			FIELD_PREP(YT921X_LED_CTRL_PORT_NUM_M, (x))
+#define  YT921X_LED_CTRL_MODE_M			GENMASK(1, 0)
+#define   YT921X_LED_CTRL_MODE(x)			FIELD_PREP(YT921X_LED_CTRL_MODE_M, (x))
+#define   YT921X_LED_CTRL_MODE_PARALLEL			YT921X_LED_CTRL_MODE(0)
+#define   YT921X_LED_CTRL_MODE_SERIAL			YT921X_LED_CTRL_MODE(2)
+#define YT921X_LED0_PORTn(port)		(0xd0004 + 4 * (port))
+#define  YT921X_LED0_PORT_ACT_M			GENMASK(17, 0)
+#define  YT921X_LED0_PORT_ACT_LINK_TRY_DIS	BIT(17)
+#define  YT921X_LED0_PORT_ACT_COLLISION_BLINK_INDI	BIT(16)
+#define YT921X_LED1_PORTn(port)		(0xd0040 + 4 * (port))
+#define  YT921X_LED1_PORT_OTHER_BLINK_M		GENMASK(31, 30)	/* cycle = 512 >> x ms */
+#define   YT921X_LED1_PORT_OTHER_BLINK(x)		FIELD_PREP(YT921X_LED1_PORT_OTHER_BLINK_M, (x))
+#define  YT921X_LED1_PORT_EEE_BLINK_M		GENMASK(29, 28)	/* cycle = 512 >> x ms */
+#define   YT921X_LED1_PORT_EEE_BLINK(x)			FIELD_PREP(YT921X_LED1_PORT_EEE_BLINK_M, (x))
+#define  YT921X_LED1_PORT_BLINK_DUTY_COMP	BIT(27)
+#define  YT921X_LED1_PORT_BLINK_DUTY_M		GENMASK(26, 25)
+#define   YT921X_LED1_PORT_BLINK_DUTY(x)		FIELD_PREP(YT921X_LED1_PORT_BLINK_DUTY_M, (x))
+#define   YT921X_LED1_PORT_BLINK_DUTY_1_2		YT921X_LED1_PORT_BLINK_DUTY(0)
+#define   YT921X_LED1_PORT_BLINK_DUTY_2_3		YT921X_LED1_PORT_BLINK_DUTY(1)
+#define   YT921X_LED1_PORT_BLINK_DUTY_3_4		YT921X_LED1_PORT_BLINK_DUTY(2)
+#define   YT921X_LED1_PORT_BLINK_DUTY_5_6		YT921X_LED1_PORT_BLINK_DUTY(3)
+#define YT921X_LED2_PORTn(port)		(0xd0080 + 4 * (port))
+#define  YT921X_LED2_PORT_FORCEn_M(grp)		GENMASK(4 * (grp) + 19, 4 * (grp) + 18)
+#define   YT921X_LED2_PORT_FORCEn(grp, x)		((x) << (4 * (grp) + 18))
+#define   YT921X_LED2_PORT_FORCEn_DONTCARE(grp)		YT921X_LED2_PORT_FORCEn(grp, 0)
+#define   YT921X_LED2_PORT_FORCEn_BLINK(grp)		YT921X_LED2_PORT_FORCEn(grp, 1)
+#define   YT921X_LED2_PORT_FORCEn_ON(grp)		YT921X_LED2_PORT_FORCEn(grp, 2)
+#define   YT921X_LED2_PORT_FORCEn_OFF(grp)		YT921X_LED2_PORT_FORCEn(grp, 3)
+#define  YT921X_LED2_PORT_FORCE_BLINKn_M(grp)	GENMASK(4 * (grp) + 17, 4 * (grp) + 16)	/* cycle = 512 << x ms */
+#define   YT921X_LED2_PORT_FORCE_BLINKn(grp, x)		((x) << (4 * (grp) + 16))
+#define   YT921X_LED2_PORT_FORCE_BLINKn_OTHER(grp)	YT921X_LED2_PORT_FORCE_BLINKn(grp, 3)
+#define  YT921X_LEDx_PORT_ACT_M			GENMASK(15, 0)
+#define  YT921X_LEDx_PORT_ACT_EEE_BLINK		BIT(15)
+#define  YT921X_LEDx_PORT_ACT_LOOPDETECT_BLINK	BIT(14)
+#define  YT921X_LEDx_PORT_ACT_ACTIVE_BLINK	BIT(13)
+#define  YT921X_LEDx_PORT_ACT_DUPLEX_FULL	BIT(12)
+#define  YT921X_LEDx_PORT_ACT_DUPLEX_HALF	BIT(11)
+#define  YT921X_LEDx_PORT_ACT_TX_BLINK		BIT(10)
+#define  YT921X_LEDx_PORT_ACT_RX_BLINK		BIT(9)
+#define  YT921X_LEDx_PORT_ACT_TX		BIT(8)
+#define  YT921X_LEDx_PORT_ACT_RX		BIT(7)
+#define  YT921X_LEDx_PORT_ACT_1000M		BIT(6)
+#define  YT921X_LEDx_PORT_ACT_100M		BIT(5)
+#define  YT921X_LEDx_PORT_ACT_10M		BIT(4)
+#define  YT921X_LEDx_PORT_ACT_COLLISION_BLINK	BIT(3)
+#define  YT921X_LEDx_PORT_ACT_1000M_BLINK	BIT(2)
+#define  YT921X_LEDx_PORT_ACT_100M_BLINK	BIT(1)
+#define  YT921X_LEDx_PORT_ACT_10M_BLINK		BIT(0)
+#define YT921X_LED_SER_CTRL		0xd0100
+#define  YT921X_LED_SER_CTRL_UNK		GENMASK(25, 24)	/* delay? */
+#define  YT921X_LED_SER_CTRL_ACTIVE_LOW		BIT(4)
+#define  YT921X_LED_SER_CTRL_GRP_NUM_M		GENMASK(1, 0)	/* #grp - 1 */
+#define   YT921X_LED_SER_CTRL_GRP_NUM(x)		FIELD_PREP(YT921X_LED_SER_CTRL_GRP_NUM_M, (x))
+#define YT921X_LED_SER_MAPnm(grp, port)	(0xd0104 + 8 * (2 - (grp)) + 4 * ((port) / 5))
+#define  YT921X_LED_SER_MAP_DSTn_PORT_M(port)	GENMASK(6 * ((port) % 5) + 5, 6 * ((port) % 5) + 2)
+#define   YT921X_LED_SER_MAP_DSTn_PORT(port, x)		((x) << (6 * ((port) % 5) + 2))
+#define  YT921X_LED_SER_MAP_DSTn_LED_M(port)	GENMASK(6 * ((port) % 5) + 1, 6 * ((port) % 5))
+#define   YT921X_LED_SER_MAP_DSTn_LED(port, x)		((x) << (6 * ((port) % 5)))
+#define YT921X_LED_PAR_PORTS		0xd01c4
+#define YT921X_LED_PAR_INV		0xd01c8
+#define  YT921X_LED_PAR_INV_INVnm(grp, port)	BIT(10 * (grp) + (port))
+#define YT921X_LED_PAR_MAPn(port)	(0xd01d0 + 4 * (port))
+#define  YT921X_LED_PAR_MAP_DSTn_PORT_M(grp)	GENMASK(6 * (grp) + 5, 6 * (grp) + 2)
+#define   YT921X_LED_PAR_MAP_DSTn_PORT(grp, x)		((x) << (6 * (grp) + 2))
+#define  YT921X_LED_PAR_MAP_DSTn_LED_M(grp)	GENMASK(6 * (grp) + 1, 6 * (grp))
+#define   YT921X_LED_PAR_MAP_DSTn_LED(grp, x)		((x) << (6 * (grp)))
+
+#define YT921X_LED_BLINK_MIN	64
+#define YT921X_LED_BLINK_DEF	512
+#define YT921X_LED_BLINK_MAX	2048
+
+/* 2 * lcm(2, 3, 4, 6) */
+#define YT921X_LED_DUTY_DENOM		24
+#define YT921X_LED_DUTY(nom, denom)	(YT921X_LED_DUTY_DENOM * (nom) / (denom))
+
+struct yt921x_priv;
+
+struct yt921x_led {
+	struct led_classdev cdev;
+	struct yt921x_port *port;
+	unsigned char group;
+};
+
+#if IS_ENABLED(CONFIG_NET_DSA_YT921X_LEDS)
+
+void yt921x_leds_remove(struct yt921x_priv *priv);
+int yt921x_leds_setup(struct yt921x_priv *priv);
+
+#else
+
+static inline void yt921x_leds_remove(struct yt921x_priv *priv) {}
+
+static inline int yt921x_leds_setup(struct yt921x_priv *priv)
+{
+	return 0;
+}
+
+#endif
+
+#endif
diff --git a/drivers/net/dsa/motorcomm/mdio_bus.c b/drivers/net/dsa/motorcomm/mdio_bus.c
new file mode 100644
index 00000000000000..1a3f3cc68275de
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/mdio_bus.c
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#include <linux/of_mdio.h>
+
+#include "chip.h"
+#include "mdio_bus.h"
+#include "smi.h"
+
+#define to_device(priv) ((priv)->ds.dev)
+
+static int yt921x_intif_wait(struct yt921x_priv *priv)
+{
+	u32 val = 0;
+
+	return yt921x_reg_wait(priv, YT921X_INT_MBUS_OP, YT921X_MBUS_OP_START,
+			       &val);
+}
+
+int yt921x_intif_read(struct yt921x_priv *priv, int port, int reg, u16 *valp)
+{
+	struct device *dev = to_device(priv);
+	u32 mask;
+	u32 ctrl;
+	u32 val;
+	int res;
+
+	res = yt921x_intif_wait(priv);
+	if (res)
+		return res;
+
+	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
+	       YT921X_MBUS_CTRL_OP_M;
+	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
+	       YT921X_MBUS_CTRL_READ;
+	res = yt921x_reg_update_bits(priv, YT921X_INT_MBUS_CTRL, mask, ctrl);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_INT_MBUS_OP, YT921X_MBUS_OP_START);
+	if (res)
+		return res;
+
+	res = yt921x_intif_wait(priv);
+	if (res)
+		return res;
+	res = yt921x_reg_read(priv, YT921X_INT_MBUS_DIN, &val);
+	if (res)
+		return res;
+
+	if ((u16)val != val)
+		dev_info(dev,
+			 "%s: port %d, reg 0x%x: Expected u16, got 0x%08x\n",
+			 __func__, port, reg, val);
+	*valp = (u16)val;
+	return 0;
+}
+
+int yt921x_intif_write(struct yt921x_priv *priv, int port, int reg, u16 val)
+{
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	res = yt921x_intif_wait(priv);
+	if (res)
+		return res;
+
+	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
+	       YT921X_MBUS_CTRL_OP_M;
+	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
+	       YT921X_MBUS_CTRL_WRITE;
+	res = yt921x_reg_update_bits(priv, YT921X_INT_MBUS_CTRL, mask, ctrl);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_INT_MBUS_DOUT, val);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_INT_MBUS_OP, YT921X_MBUS_OP_START);
+	if (res)
+		return res;
+
+	return yt921x_intif_wait(priv);
+}
+
+int
+yt921x_intif_modify_changed(struct yt921x_priv *priv, int port, int reg,
+			    u16 mask, u16 val)
+{
+	int res;
+	u16 v;
+	u16 u;
+
+	res = yt921x_intif_read(priv, port, reg, &v);
+	if (res)
+		return res;
+
+	u = v;
+	u &= ~mask;
+	u |= val;
+	if (u == v)
+		return 0;
+
+	res = yt921x_intif_write(priv, port, reg, u);
+	if (res)
+		return res;
+
+	return 1;
+}
+
+static int yt921x_mbus_int_read(struct mii_bus *mbus, int port, int reg)
+{
+	struct yt921x_priv *priv = mbus->priv;
+	u16 val;
+	int res;
+
+	if (port >= YT921X_PORT_NUM)
+		return U16_MAX;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_intif_read(priv, port, reg, &val);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		return res;
+	return val;
+}
+
+static int
+yt921x_mbus_int_write(struct mii_bus *mbus, int port, int reg, u16 data)
+{
+	struct yt921x_priv *priv = mbus->priv;
+	int res;
+
+	if (port >= YT921X_PORT_NUM)
+		return -ENODEV;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_intif_write(priv, port, reg, data);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+int yt921x_mbus_int_init(struct yt921x_priv *priv, struct device_node *mnp)
+{
+	struct device *dev = to_device(priv);
+	struct mii_bus *mbus;
+	int res;
+
+	mbus = devm_mdiobus_alloc(dev);
+	if (!mbus)
+		return -ENOMEM;
+
+	mbus->name = "YT921x internal MDIO bus";
+	snprintf(mbus->id, MII_BUS_ID_SIZE, "%s", dev_name(dev));
+	mbus->priv = priv;
+	mbus->read = yt921x_mbus_int_read;
+	mbus->write = yt921x_mbus_int_write;
+	mbus->parent = dev;
+	mbus->phy_mask = (u32)~GENMASK(YT921X_PORT_NUM - 1, 0);
+
+	res = devm_of_mdiobus_register(dev, mbus, mnp);
+	if (res)
+		return res;
+
+	priv->mbus_int = mbus;
+
+	return 0;
+}
+
+static int yt921x_extif_wait(struct yt921x_priv *priv)
+{
+	u32 val = 0;
+
+	return yt921x_reg_wait(priv, YT921X_EXT_MBUS_OP, YT921X_MBUS_OP_START,
+			       &val);
+}
+
+static int
+yt921x_extif_read(struct yt921x_priv *priv, int port, int reg, u16 *valp)
+{
+	struct device *dev = to_device(priv);
+	u32 mask;
+	u32 ctrl;
+	u32 val;
+	int res;
+
+	res = yt921x_extif_wait(priv);
+	if (res)
+		return res;
+
+	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
+	       YT921X_MBUS_CTRL_TYPE_M | YT921X_MBUS_CTRL_OP_M;
+	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
+	       YT921X_MBUS_CTRL_TYPE_C22 | YT921X_MBUS_CTRL_READ;
+	res = yt921x_reg_update_bits(priv, YT921X_EXT_MBUS_CTRL, mask, ctrl);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_EXT_MBUS_OP, YT921X_MBUS_OP_START);
+	if (res)
+		return res;
+
+	res = yt921x_extif_wait(priv);
+	if (res)
+		return res;
+	res = yt921x_reg_read(priv, YT921X_EXT_MBUS_DIN, &val);
+	if (res)
+		return res;
+
+	if ((u16)val != val)
+		dev_info(dev,
+			 "%s: port %d, reg 0x%x: Expected u16, got 0x%08x\n",
+			 __func__, port, reg, val);
+	*valp = (u16)val;
+	return 0;
+}
+
+static int
+yt921x_extif_write(struct yt921x_priv *priv, int port, int reg, u16 val)
+{
+	u32 mask;
+	u32 ctrl;
+	int res;
+
+	res = yt921x_extif_wait(priv);
+	if (res)
+		return res;
+
+	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
+	       YT921X_MBUS_CTRL_TYPE_M | YT921X_MBUS_CTRL_OP_M;
+	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
+	       YT921X_MBUS_CTRL_TYPE_C22 | YT921X_MBUS_CTRL_WRITE;
+	res = yt921x_reg_update_bits(priv, YT921X_EXT_MBUS_CTRL, mask, ctrl);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_EXT_MBUS_DOUT, val);
+	if (res)
+		return res;
+	res = yt921x_reg_write(priv, YT921X_EXT_MBUS_OP, YT921X_MBUS_OP_START);
+	if (res)
+		return res;
+
+	return yt921x_extif_wait(priv);
+}
+
+static int yt921x_mbus_ext_read(struct mii_bus *mbus, int port, int reg)
+{
+	struct yt921x_priv *priv = mbus->priv;
+	u16 val;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_extif_read(priv, port, reg, &val);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		return res;
+	return val;
+}
+
+static int
+yt921x_mbus_ext_write(struct mii_bus *mbus, int port, int reg, u16 data)
+{
+	struct yt921x_priv *priv = mbus->priv;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_extif_write(priv, port, reg, data);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+int yt921x_mbus_ext_init(struct yt921x_priv *priv, struct device_node *mnp)
+{
+	struct device *dev = to_device(priv);
+	struct mii_bus *mbus;
+	int res;
+
+	mbus = devm_mdiobus_alloc(dev);
+	if (!mbus)
+		return -ENOMEM;
+
+	mbus->name = "YT921x external MDIO bus";
+	snprintf(mbus->id, MII_BUS_ID_SIZE, "%s@ext", dev_name(dev));
+	mbus->priv = priv;
+	/* TODO: c45? */
+	mbus->read = yt921x_mbus_ext_read;
+	mbus->write = yt921x_mbus_ext_write;
+	mbus->parent = dev;
+
+	res = devm_of_mdiobus_register(dev, mbus, mnp);
+	if (res)
+		return res;
+
+	priv->mbus_ext = mbus;
+
+	return 0;
+}
diff --git a/drivers/net/dsa/motorcomm/mdio_bus.h b/drivers/net/dsa/motorcomm/mdio_bus.h
new file mode 100644
index 00000000000000..e79b725d435bdd
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/mdio_bus.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#ifndef _YT_MDIO_BUS_H
+#define _YT_MDIO_BUS_H
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/of.h>
+
+#define YT921X_EXT_MBUS_OP		0x6a000
+#define YT921X_INT_MBUS_OP		0xf0000
+#define  YT921X_MBUS_OP_START			BIT(0)
+#define YT921X_EXT_MBUS_CTRL		0x6a004
+#define YT921X_INT_MBUS_CTRL		0xf0004
+#define  YT921X_MBUS_CTRL_PORT_M		GENMASK(25, 21)
+#define   YT921X_MBUS_CTRL_PORT(x)			FIELD_PREP(YT921X_MBUS_CTRL_PORT_M, (x))
+#define  YT921X_MBUS_CTRL_REG_M			GENMASK(20, 16)
+#define   YT921X_MBUS_CTRL_REG(x)			FIELD_PREP(YT921X_MBUS_CTRL_REG_M, (x))
+#define  YT921X_MBUS_CTRL_TYPE_M		GENMASK(11, 8)  /* wild guess */
+#define   YT921X_MBUS_CTRL_TYPE(x)			FIELD_PREP(YT921X_MBUS_CTRL_TYPE_M, (x))
+#define   YT921X_MBUS_CTRL_TYPE_C22			YT921X_MBUS_CTRL_TYPE(4)
+#define  YT921X_MBUS_CTRL_OP_M			GENMASK(3, 2)  /* wild guess */
+#define   YT921X_MBUS_CTRL_OP(x)			FIELD_PREP(YT921X_MBUS_CTRL_OP_M, (x))
+#define   YT921X_MBUS_CTRL_WRITE			YT921X_MBUS_CTRL_OP(1)
+#define   YT921X_MBUS_CTRL_READ				YT921X_MBUS_CTRL_OP(2)
+#define YT921X_EXT_MBUS_DOUT		0x6a008
+#define YT921X_INT_MBUS_DOUT		0xf0008
+#define YT921X_EXT_MBUS_DIN		0x6a00c
+#define YT921X_INT_MBUS_DIN		0xf000c
+
+struct yt921x_priv;
+
+int yt921x_intif_read(struct yt921x_priv *priv, int port, int reg, u16 *valp);
+int yt921x_intif_write(struct yt921x_priv *priv, int port, int reg, u16 val);
+int yt921x_intif_modify_changed(struct yt921x_priv *priv, int port, int reg,
+				u16 mask, u16 val);
+
+static inline int
+yt921x_intif_modify(struct yt921x_priv *priv, int port, int reg, u16 mask,
+		    u16 val)
+{
+	int res;
+
+	res = yt921x_intif_modify_changed(priv, port, reg, mask, val);
+	return res >= 0 ? 0 : res;
+}
+
+int yt921x_mbus_int_init(struct yt921x_priv *priv, struct device_node *mnp);
+int yt921x_mbus_ext_init(struct yt921x_priv *priv, struct device_node *mnp);
+
+#endif
diff --git a/drivers/net/dsa/motorcomm/pcs-921x.c b/drivers/net/dsa/motorcomm/pcs-921x.c
new file mode 100644
index 00000000000000..47467d803e136a
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/pcs-921x.c
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#include "chip.h"
+#include "mdio_bus.h"
+#include "pcs.h"
+#include "smi.h"
+
+#define to_device(priv) ((priv)->ds.dev)
+
+static int
+yt921x_serdes_config(struct yt921x_priv *priv, int port, unsigned int neg_mode,
+		     phy_interface_t interface,
+		     const unsigned long *advertising, bool permit_pause_to_mac)
+{
+	bool inband = neg_mode & PHYLINK_PCS_NEG_INBAND;
+	struct yt921x_port *pp = &priv->ports[port];
+	bool changed = false;
+	u16 bmcr;
+	u32 mask;
+	u32 ctrl;
+	u16 val;
+	int adv;
+	int res;
+
+	switch (interface) {
+	case PHY_INTERFACE_MODE_SGMII:
+		ctrl = YT921X_SERDES_MODE_SGMII;
+		break;
+	case PHY_INTERFACE_MODE_100BASEX:
+		ctrl = YT921X_SERDES_MODE_100BASEX;
+		break;
+	case PHY_INTERFACE_MODE_1000BASEX:
+		ctrl = YT921X_SERDES_MODE_1000BASEX;
+		break;
+	case PHY_INTERFACE_MODE_2500BASEX:
+		ctrl = YT921X_SERDES_MODE_2500BASEX;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	res = yt921x_reg_update_bits(priv, YT921X_SERDESn(port),
+				     YT921X_SERDES_MODE_M, ctrl);
+	if (res)
+		return res;
+
+	mask = YT921X_PORT_CTRL_LINK_AN | YT921X_PORT_CTRL_PAUSE_AN;
+	ctrl = 0;
+	if (inband)
+		ctrl |= YT921X_PORT_CTRL_LINK_AN;
+	if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED && permit_pause_to_mac &&
+	    interface != PHY_INTERFACE_MODE_SGMII)
+		ctrl |= YT921X_PORT_CTRL_PAUSE_AN;
+	res = yt921x_reg_update_bits(priv, YT921X_PORTn_CTRL(port), mask, ctrl);
+	if (res)
+		return res;
+
+	if (inband) {
+		res = yt921x_reg_clear_bits(priv, YT921X_SERDESn(port),
+					    YT921X_SERDES_LINK);
+		if (res)
+			return res;
+	}
+
+	adv = phylink_mii_c22_pcs_encode_advertisement(interface, advertising);
+	if (adv >= 0) {
+		res = yt921x_intif_modify_changed(priv, port, MII_ADVERTISE,
+						  U16_MAX, adv);
+		if (res < 0)
+			return res;
+
+		changed = !!res;
+	}
+
+	res = yt921x_intif_read(priv, port, MII_BMCR, &val);
+	if (res)
+		return res;
+
+	bmcr = val;
+	if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED)
+		bmcr |= BMCR_ANENABLE;
+	else
+		bmcr &= ~BMCR_ANENABLE;
+
+	/* If the ANENABLE bit was changed, the PHY will restart negotiation,
+	 * so we don't need to flag a change to trigger its own restart.
+	 */
+	if (bmcr != val)
+		changed = false;
+
+	bmcr &= ~BMCR_ISOLATE;
+	res = yt921x_intif_write(priv, port, MII_BMCR, bmcr);
+	if (res)
+		return res;
+
+	pp->inband = inband;
+	return changed;
+}
+
+static unsigned int
+yt921x_phylink_pcs_inband_caps(struct phylink_pcs *pcs,
+			       phy_interface_t interface)
+{
+	if (interface == PHY_INTERFACE_MODE_100BASEX)
+		return LINK_INBAND_DISABLE;
+
+	return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE;
+}
+
+static void
+yt921x_phylink_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode,
+			     struct phylink_link_state *state)
+{
+	struct yt921x_port *pp = pcs_to_yt921x_port(pcs);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	struct device *dev = to_device(priv);
+	int port = pp->index;
+	u16 bmsr;
+	u16 lpa;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+
+	res = yt921x_intif_read(priv, port, MII_BMSR, &bmsr);
+	if (res)
+		goto end;
+
+	res = yt921x_intif_read(priv, port, MII_LPA, &lpa);
+
+end:
+	mutex_unlock(&priv->reg_lock);
+
+	if (res) {
+		dev_err_ratelimited(dev, "Failed to %s PCS port %d: %i\n",
+				    "get state of", port, res);
+		state->link = false;
+		return;
+	}
+
+	if (state->interface == PHY_INTERFACE_MODE_100BASEX) {
+		state->speed = SPEED_100;
+		state->duplex = DUPLEX_FULL;
+		state->pause |= MLO_PAUSE_TX | MLO_PAUSE_RX;
+		state->link = !!(bmsr & BMSR_LSTATUS);
+		return;
+	}
+
+	phylink_mii_c22_pcs_decode_state(state, neg_mode, bmsr, lpa);
+}
+
+static void yt921x_phylink_pcs_an_restart(struct phylink_pcs *pcs)
+{
+	struct yt921x_port *pp = pcs_to_yt921x_port(pcs);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	struct device *dev = to_device(priv);
+	int port = pp->index;
+	u16 val;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+
+	res = yt921x_intif_read(priv, port, MII_BMCR, &val);
+	if (res)
+		goto end;
+
+	val |= BMCR_ANRESTART;
+	res = yt921x_intif_write(priv, port, MII_BMCR, val);
+
+end:
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s PCS port %d: %i\n", "restart",
+			port, res);
+}
+
+static void yt921x_phylink_pcs_disable(struct phylink_pcs *pcs)
+{
+	struct yt921x_port *pp = pcs_to_yt921x_port(pcs);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	struct device *dev = to_device(priv);
+	int port = pp->index;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_intif_modify(priv, port, MII_BMCR, BMCR_PDOWN, BMCR_PDOWN);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s PCS port %d: %i\n", "power down",
+			port, res);
+}
+
+static int yt921x_phylink_pcs_enable(struct phylink_pcs *pcs)
+{
+	struct yt921x_port *pp = pcs_to_yt921x_port(pcs);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	int port = pp->index;
+	u16 val;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+
+	res = yt921x_intif_read(priv, port, MII_BMCR, &val);
+	if (res)
+		goto end;
+
+	val &= ~BMCR_PDOWN;
+	val |= BMCR_ANRESTART;
+	res = yt921x_intif_write(priv, port, MII_BMCR, val);
+
+end:
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+static int
+yt921x_phylink_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
+			  phy_interface_t interface,
+			  const unsigned long *advertising,
+			  bool permit_pause_to_mac)
+{
+	struct yt921x_port *pp = pcs_to_yt921x_port(pcs);
+	struct yt921x_priv *priv = yt921x_port_to_priv(pp);
+	int port = pp->index;
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_serdes_config(priv, port, neg_mode, interface,
+				   advertising, permit_pause_to_mac);
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
+const struct phylink_pcs_ops yt921x_phylink_pcs_ops = {
+	.pcs_inband_caps	= yt921x_phylink_pcs_inband_caps,
+	.pcs_get_state		= yt921x_phylink_pcs_get_state,
+	.pcs_an_restart		= yt921x_phylink_pcs_an_restart,
+	.pcs_disable		= yt921x_phylink_pcs_disable,
+	.pcs_enable		= yt921x_phylink_pcs_enable,
+	.pcs_config		= yt921x_phylink_pcs_config,
+};
diff --git a/drivers/net/dsa/motorcomm/pcs.h b/drivers/net/dsa/motorcomm/pcs.h
new file mode 100644
index 00000000000000..42426558086af2
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/pcs.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#ifndef _YT_PCS_H
+#define _YT_PCS_H
+
+#include <linux/phylink.h>
+
+extern const struct phylink_pcs_ops yt921x_phylink_pcs_ops;
+
+#endif
diff --git a/drivers/net/dsa/motorcomm/smi.c b/drivers/net/dsa/motorcomm/smi.c
new file mode 100644
index 00000000000000..bf3adfd64165a5
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/smi.c
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#include <linux/iopoll.h>
+
+#include "chip.h"
+#include "smi.h"
+
+#define YT921X_POLL_SLEEP_US	10000
+#define YT921X_POLL_TIMEOUT_US	100000
+
+int yt921x_reg_read(struct yt921x_priv *priv, u32 reg, u32 *valp)
+{
+	lockdep_assert_held_once(&priv->reg_lock);
+
+	return priv->reg_ops->read(priv->reg_ctx, reg, valp);
+}
+
+int yt921x_reg_write(struct yt921x_priv *priv, u32 reg, u32 val)
+{
+	lockdep_assert_held_once(&priv->reg_lock);
+
+	return priv->reg_ops->write(priv->reg_ctx, reg, val);
+}
+
+int yt921x_reg_wait(struct yt921x_priv *priv, u32 reg, u32 mask, u32 *valp)
+{
+	u32 val;
+	int res;
+	int ret;
+
+	ret = read_poll_timeout(yt921x_reg_read, res,
+				res || (val & mask) == *valp,
+				YT921X_POLL_SLEEP_US, YT921X_POLL_TIMEOUT_US,
+				false, priv, reg, &val);
+	if (ret)
+		return ret;
+	if (res)
+		return res;
+
+	*valp = val;
+	return 0;
+}
+
+int yt921x_reg_update_bits(struct yt921x_priv *priv, u32 reg, u32 mask, u32 val)
+{
+	int res;
+	u32 v;
+	u32 u;
+
+	res = yt921x_reg_read(priv, reg, &v);
+	if (res)
+		return res;
+
+	u = v;
+	u &= ~mask;
+	u |= val;
+	if (u == v)
+		return 0;
+
+	return yt921x_reg_write(priv, reg, u);
+}
+
+static int
+yt921x_regs_read(struct yt921x_priv *priv, u32 reg, u32 *vals,
+		 unsigned int num_regs)
+{
+	int res;
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		res = yt921x_reg_read(priv, reg + 4 * i, &vals[i]);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_regs_write(struct yt921x_priv *priv, u32 reg, const u32 *vals,
+		  unsigned int num_regs)
+{
+	int res;
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		res = yt921x_reg_write(priv, reg + 4 * i, vals[i]);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_regs_update_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
+			const u32 *vals, unsigned int num_regs)
+{
+	bool changed = false;
+	u32 vs[4];
+	int res;
+
+	BUILD_BUG_ON(num_regs > ARRAY_SIZE(vs));
+
+	res = yt921x_regs_read(priv, reg, vs, num_regs);
+	if (res)
+		return res;
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		u32 u = vs[i];
+
+		u &= ~masks[i];
+		u |= vals[i];
+		if (u != vs[i])
+			changed = true;
+
+		vs[i] = u;
+	}
+
+	if (!changed)
+		return 0;
+
+	return yt921x_regs_write(priv, reg, vs, num_regs);
+}
+
+static int
+yt921x_regs_clear_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
+		       unsigned int num_regs)
+{
+	bool changed = false;
+	u32 vs[4];
+	int res;
+
+	BUILD_BUG_ON(num_regs > ARRAY_SIZE(vs));
+
+	res = yt921x_regs_read(priv, reg, vs, num_regs);
+	if (res)
+		return res;
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		u32 u = vs[i];
+
+		u &= ~masks[i];
+		if (u != vs[i])
+			changed = true;
+
+		vs[i] = u;
+	}
+
+	if (!changed)
+		return 0;
+
+	return yt921x_regs_write(priv, reg, vs, num_regs);
+}
+
+int
+yt921x_reg64_write(struct yt921x_priv *priv, u32 reg, const u32 *vals)
+{
+	return yt921x_regs_write(priv, reg, vals, 2);
+}
+
+int
+yt921x_reg64_update_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
+			 const u32 *vals)
+{
+	return yt921x_regs_update_bits(priv, reg, masks, vals, 2);
+}
+
+int
+yt921x_reg64_clear_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks)
+{
+	return yt921x_regs_clear_bits(priv, reg, masks, 2);
+}
+
+int
+yt921x_reg96_write(struct yt921x_priv *priv, u32 reg, const u32 *vals)
+{
+	return yt921x_regs_write(priv, reg, vals, 3);
+}
diff --git a/drivers/net/dsa/motorcomm/smi.h b/drivers/net/dsa/motorcomm/smi.h
new file mode 100644
index 00000000000000..212e20f71d8067
--- /dev/null
+++ b/drivers/net/dsa/motorcomm/smi.h
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (c) 2026 David Yang
+ */
+
+#ifndef _YT_SMI_H
+#define _YT_SMI_H
+
+#include <linux/types.h>
+#include <linux/wordpart.h>
+
+struct yt921x_priv;
+
+int yt921x_reg_read(struct yt921x_priv *priv, u32 reg, u32 *valp);
+int yt921x_reg_write(struct yt921x_priv *priv, u32 reg, u32 val);
+int yt921x_reg_wait(struct yt921x_priv *priv, u32 reg, u32 mask, u32 *valp);
+int yt921x_reg_update_bits(struct yt921x_priv *priv, u32 reg, u32 mask,
+			   u32 val);
+
+static inline int
+yt921x_reg_set_bits(struct yt921x_priv *priv, u32 reg, u32 mask)
+{
+	return yt921x_reg_update_bits(priv, reg, 0, mask);
+}
+
+static inline int
+yt921x_reg_clear_bits(struct yt921x_priv *priv, u32 reg, u32 mask)
+{
+	return yt921x_reg_update_bits(priv, reg, mask, 0);
+}
+
+static inline int
+yt921x_reg_toggle_bits(struct yt921x_priv *priv, u32 reg, u32 mask, bool set)
+{
+	return yt921x_reg_update_bits(priv, reg, mask, !set ? 0 : mask);
+}
+
+/* Some multi-word registers, like VLANn_CTRL, should be treated as a single
+ * long register. More specifically, writes to parts of its words won't become
+ * visible, until the last word is written.
+ *
+ * Here we require full read and write operations over these registers to
+ * eliminate potential issues, although partial reads/writes are also possible.
+ */
+
+static inline void update_ctrls_unaligned(u32 *lo, u32 *hi, u64 mask, u64 val)
+{
+	*lo &= ~lower_32_bits(mask);
+	*hi &= ~upper_32_bits(mask);
+	*lo |= lower_32_bits(val);
+	*hi |= upper_32_bits(val);
+}
+
+int yt921x_reg64_write(struct yt921x_priv *priv, u32 reg, const u32 *vals);
+int yt921x_reg64_update_bits(struct yt921x_priv *priv, u32 reg,
+			     const u32 *masks, const u32 *vals);
+int yt921x_reg64_clear_bits(struct yt921x_priv *priv, u32 reg,
+			    const u32 *masks);
+int yt921x_reg96_write(struct yt921x_priv *priv, u32 reg, const u32 *vals);
+
+#endif
diff --git a/drivers/net/dsa/mt7530-mdio.c b/drivers/net/dsa/mt7530-mdio.c
index 784dd58a71589f..0be107dc6759a0 100644
--- a/drivers/net/dsa/mt7530-mdio.c
+++ b/drivers/net/dsa/mt7530-mdio.c
@@ -48,6 +48,9 @@ mt7530_regmap_read(void *context, unsigned int reg, unsigned int *val)
 	u16 page, r, lo, hi;
 	int ret;
 
+	/* Callers do not check for errors, keep the value deterministic */
+	*val = 0;
+
 	page = (reg >> 6) & 0x3ff;
 	r = (reg >> 2) & 0xf;
 
@@ -139,7 +142,7 @@ static const struct of_device_id mt7530_of_match[] = {
 	{ .compatible = "mediatek,mt7621", .data = &mt753x_table[ID_MT7621], },
 	{ .compatible = "mediatek,mt7530", .data = &mt753x_table[ID_MT7530], },
 	{ .compatible = "mediatek,mt7531", .data = &mt753x_table[ID_MT7531], },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, mt7530_of_match);
 
@@ -148,12 +151,14 @@ static const struct regmap_config regmap_config = {
 	.val_bits = 32,
 	.reg_stride = 4,
 	.max_register = MT7530_CREV,
-	.disable_locking = true,
+	.lock = mt7530_mdio_regmap_lock,
+	.unlock = mt7530_mdio_regmap_unlock,
 };
 
 static int
 mt7530_probe(struct mdio_device *mdiodev)
 {
+	struct regmap_config rc = regmap_config;
 	struct mt7530_priv *priv;
 	struct device_node *dn;
 	int ret;
@@ -207,8 +212,9 @@ mt7530_probe(struct mdio_device *mdiodev)
 			return PTR_ERR(priv->io_pwr);
 	}
 
-	priv->regmap = devm_regmap_init(priv->dev, &mt7530_regmap_bus, priv,
-					&regmap_config);
+	rc.lock_arg = &priv->bus->mdio_lock;
+	priv->regmap = devm_regmap_init(priv->dev, &mt7530_regmap_bus,
+					priv, &rc);
 	if (IS_ERR(priv->regmap))
 		return PTR_ERR(priv->regmap);
 
@@ -227,15 +233,17 @@ mt7530_remove(struct mdio_device *mdiodev)
 	if (!priv)
 		return;
 
-	ret = regulator_disable(priv->core_pwr);
-	if (ret < 0)
-		dev_err(priv->dev,
-			"Failed to disable core power: %d\n", ret);
-
-	ret = regulator_disable(priv->io_pwr);
-	if (ret < 0)
-		dev_err(priv->dev, "Failed to disable io pwr: %d\n",
-			ret);
+	if (priv->id == ID_MT7530) {
+		ret = regulator_disable(priv->core_pwr);
+		if (ret < 0)
+			dev_err(priv->dev,
+				"Failed to disable core power: %d\n", ret);
+
+		ret = regulator_disable(priv->io_pwr);
+		if (ret < 0)
+			dev_err(priv->dev, "Failed to disable io pwr: %d\n",
+				ret);
+	}
 
 	mt7530_remove_common(priv);
 
diff --git a/drivers/net/dsa/mt7530-mmio.c b/drivers/net/dsa/mt7530-mmio.c
index fd68b1cd063068..cf3bdb23a2d546 100644
--- a/drivers/net/dsa/mt7530-mmio.c
+++ b/drivers/net/dsa/mt7530-mmio.c
@@ -14,7 +14,7 @@ static const struct of_device_id mt7988_of_match[] = {
 	{ .compatible = "airoha,en7581-switch", .data = &mt753x_table[ID_EN7581], },
 	{ .compatible = "econet,en7528-switch", .data = &mt753x_table[ID_EN7528], },
 	{ .compatible = "mediatek,mt7988-switch", .data = &mt753x_table[ID_MT7988], },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, mt7988_of_match);
 
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 3e61eb3c2b1e2d..7781a63b4e6fdf 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -154,97 +154,6 @@ core_clear(struct mt7530_priv *priv, u32 reg, u32 val)
 	core_rmw(priv, reg, val, 0);
 }
 
-static int
-mt7530_mii_write(struct mt7530_priv *priv, u32 reg, u32 val)
-{
-	int ret;
-
-	ret = regmap_write(priv->regmap, reg, val);
-
-	if (ret < 0)
-		dev_err(priv->dev,
-			"failed to write mt7530 register\n");
-
-	return ret;
-}
-
-static u32
-mt7530_mii_read(struct mt7530_priv *priv, u32 reg)
-{
-	int ret;
-	u32 val;
-
-	ret = regmap_read(priv->regmap, reg, &val);
-	if (ret) {
-		WARN_ON_ONCE(1);
-		dev_err(priv->dev,
-			"failed to read mt7530 register\n");
-		return 0;
-	}
-
-	return val;
-}
-
-static int
-mt7530_write(struct mt7530_priv *priv, u32 reg, u32 val)
-{
-	int ret;
-
-	mt7530_mutex_lock(priv);
-
-	ret = mt7530_mii_write(priv, reg, val);
-
-	mt7530_mutex_unlock(priv);
-
-	return ret;
-}
-
-static u32
-_mt7530_read(struct mt7530_dummy_poll *p)
-{
-	u32 val;
-
-	mt7530_mutex_lock(p->priv);
-
-	val = mt7530_mii_read(p->priv, p->reg);
-
-	mt7530_mutex_unlock(p->priv);
-
-	return val;
-}
-
-static u32
-mt7530_read(struct mt7530_priv *priv, u32 reg)
-{
-	struct mt7530_dummy_poll p;
-
-	INIT_MT7530_DUMMY_POLL(&p, priv, reg);
-	return _mt7530_read(&p);
-}
-
-static void
-mt7530_rmw(struct mt7530_priv *priv, u32 reg,
-	   u32 mask, u32 set)
-{
-	mt7530_mutex_lock(priv);
-
-	regmap_update_bits(priv->regmap, reg, mask, set);
-
-	mt7530_mutex_unlock(priv);
-}
-
-static void
-mt7530_set(struct mt7530_priv *priv, u32 reg, u32 val)
-{
-	mt7530_rmw(priv, reg, val, val);
-}
-
-static void
-mt7530_clear(struct mt7530_priv *priv, u32 reg, u32 val)
-{
-	mt7530_rmw(priv, reg, val, 0);
-}
-
 static int
 mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
 {
@@ -253,19 +162,15 @@ mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
 
 	/* Set the command operating upon the MAC address entries */
 	val = ATC_BUSY | ATC_MAT(0) | cmd;
-	ret = mt7530_write(priv, MT7530_ATC, val);
+	ret = regmap_write(priv->regmap, MT7530_ATC, val);
 	if (ret)
 		return ret;
 
-	mt7530_mutex_lock(priv);
-
 	ret = regmap_read_poll_timeout(priv->regmap, MT7530_ATC, val,
 				       !(val & ATC_BUSY), 20, 20000);
 	if (!ret)
 		ret = regmap_read(priv->regmap, MT7530_ATC, &val);
 
-	mt7530_mutex_unlock(priv);
-
 	if (ret < 0) {
 		dev_err(priv->dev, "reset timeout\n");
 		return ret;
@@ -283,6 +188,26 @@ mt7530_fdb_cmd(struct mt7530_priv *priv, enum mt7530_fdb_cmd cmd, u32 *rsp)
 	return 0;
 }
 
+static void mt7530_port_fast_age(struct dsa_switch *ds, int port)
+{
+	struct mt7530_priv *priv = ds->priv;
+	u32 val;
+	int ret;
+
+	mutex_lock(&priv->reg_mutex);
+
+	/* Flush all non-static MAC address entries */
+	val = ATC_BUSY | ATC_MAT_NON_STATIC_MAC | MT7530_FDB_FLUSH;
+	ret = regmap_write(priv->regmap, MT7530_ATC, val);
+	if (!ret)
+		ret = regmap_read_poll_timeout(priv->regmap, MT7530_ATC, val,
+					       !(val & ATC_BUSY), 20, 20000);
+	if (ret < 0)
+		dev_err(priv->dev, "fast age failed: %d\n", ret);
+
+	mutex_unlock(&priv->reg_mutex);
+}
+
 static void
 mt7530_fdb_read(struct mt7530_priv *priv, struct mt7530_fdb *fdb)
 {
@@ -291,22 +216,23 @@ mt7530_fdb_read(struct mt7530_priv *priv, struct mt7530_fdb *fdb)
 
 	/* Read from ARL table into an array */
 	for (i = 0; i < 3; i++) {
-		reg[i] = mt7530_read(priv, MT7530_TSRA1 + (i * 4));
+		regmap_read(priv->regmap, MT7530_TSRA1 + (i * 4),
+			    &reg[i]);
 
 		dev_dbg(priv->dev, "%s(%d) reg[%d]=0x%x\n",
 			__func__, __LINE__, i, reg[i]);
 	}
 
-	fdb->vid = (reg[1] >> CVID) & CVID_MASK;
-	fdb->aging = (reg[2] >> AGE_TIMER) & AGE_TIMER_MASK;
-	fdb->port_mask = (reg[2] >> PORT_MAP) & PORT_MAP_MASK;
-	fdb->mac[0] = (reg[0] >> MAC_BYTE_0) & MAC_BYTE_MASK;
-	fdb->mac[1] = (reg[0] >> MAC_BYTE_1) & MAC_BYTE_MASK;
-	fdb->mac[2] = (reg[0] >> MAC_BYTE_2) & MAC_BYTE_MASK;
-	fdb->mac[3] = (reg[0] >> MAC_BYTE_3) & MAC_BYTE_MASK;
-	fdb->mac[4] = (reg[1] >> MAC_BYTE_4) & MAC_BYTE_MASK;
-	fdb->mac[5] = (reg[1] >> MAC_BYTE_5) & MAC_BYTE_MASK;
-	fdb->noarp = ((reg[2] >> ENT_STATUS) & ENT_STATUS_MASK) == STATIC_ENT;
+	fdb->vid = FIELD_GET(CVID_MASK, reg[1]);
+	fdb->aging = FIELD_GET(AGE_TIMER_MASK, reg[2]);
+	fdb->port_mask = FIELD_GET(PORT_MAP_MASK, reg[2]);
+	fdb->mac[0] = FIELD_GET(MAC_BYTE_0_MASK, reg[0]);
+	fdb->mac[1] = FIELD_GET(MAC_BYTE_1_MASK, reg[0]);
+	fdb->mac[2] = FIELD_GET(MAC_BYTE_2_MASK, reg[0]);
+	fdb->mac[3] = FIELD_GET(MAC_BYTE_3_MASK, reg[0]);
+	fdb->mac[4] = FIELD_GET(MAC_BYTE_4_MASK, reg[1]);
+	fdb->mac[5] = FIELD_GET(MAC_BYTE_5_MASK, reg[1]);
+	fdb->noarp = FIELD_GET(ENT_STATUS_MASK, reg[2]) == STATIC_ENT;
 }
 
 static void
@@ -317,26 +243,26 @@ mt7530_fdb_write(struct mt7530_priv *priv, u16 vid,
 	u32 reg[3] = { 0 };
 	int i;
 
-	reg[1] |= vid & CVID_MASK;
+	reg[1] |= FIELD_PREP(CVID_MASK, vid);
 	reg[1] |= ATA2_IVL;
 	reg[1] |= ATA2_FID(FID_BRIDGED);
-	reg[2] |= (aging & AGE_TIMER_MASK) << AGE_TIMER;
-	reg[2] |= (port_mask & PORT_MAP_MASK) << PORT_MAP;
+	reg[2] |= FIELD_PREP(AGE_TIMER_MASK, aging);
+	reg[2] |= FIELD_PREP(PORT_MAP_MASK, port_mask);
 	/* STATIC_ENT indicate that entry is static wouldn't
 	 * be aged out and STATIC_EMP specified as erasing an
 	 * entry
 	 */
-	reg[2] |= (type & ENT_STATUS_MASK) << ENT_STATUS;
-	reg[1] |= mac[5] << MAC_BYTE_5;
-	reg[1] |= mac[4] << MAC_BYTE_4;
-	reg[0] |= mac[3] << MAC_BYTE_3;
-	reg[0] |= mac[2] << MAC_BYTE_2;
-	reg[0] |= mac[1] << MAC_BYTE_1;
-	reg[0] |= mac[0] << MAC_BYTE_0;
+	reg[2] |= FIELD_PREP(ENT_STATUS_MASK, type);
+	reg[1] |= FIELD_PREP(MAC_BYTE_5_MASK, mac[5]);
+	reg[1] |= FIELD_PREP(MAC_BYTE_4_MASK, mac[4]);
+	reg[0] |= FIELD_PREP(MAC_BYTE_3_MASK, mac[3]);
+	reg[0] |= FIELD_PREP(MAC_BYTE_2_MASK, mac[2]);
+	reg[0] |= FIELD_PREP(MAC_BYTE_1_MASK, mac[1]);
+	reg[0] |= FIELD_PREP(MAC_BYTE_0_MASK, mac[0]);
 
 	/* Write array into the ARL table */
 	for (i = 0; i < 3; i++)
-		mt7530_write(priv, MT7530_ATA1 + (i * 4), reg[i]);
+		regmap_write(priv->regmap, MT7530_ATA1 + (i * 4), reg[i]);
 }
 
 /* Set up switch core clock for MT7530 */
@@ -390,14 +316,16 @@ mt7530_setup_port6(struct dsa_switch *ds, phy_interface_t interface)
 	core_clear(priv, CORE_TRGMII_GSW_CLK_CG, REG_TRGMIICK_EN);
 
 	if (interface == PHY_INTERFACE_MODE_RGMII) {
-		mt7530_rmw(priv, MT7530_P6ECR, P6_INTF_MODE_MASK,
-			   P6_INTF_MODE(0));
+		regmap_update_bits(priv->regmap, MT7530_P6ECR,
+				   P6_INTF_MODE_MASK, P6_INTF_MODE(0));
 		return;
 	}
 
-	mt7530_rmw(priv, MT7530_P6ECR, P6_INTF_MODE_MASK, P6_INTF_MODE(1));
+	regmap_update_bits(priv->regmap, MT7530_P6ECR, P6_INTF_MODE_MASK,
+			   P6_INTF_MODE(1));
 
-	xtal = mt7530_read(priv, MT753X_MTRAP) & MT7530_XTAL_MASK;
+	regmap_read(priv->regmap, MT753X_MTRAP, &xtal);
+	xtal &= MT7530_XTAL_MASK;
 
 	if (xtal == MT7530_XTAL_25MHZ)
 		ssc_delta = 0x57;
@@ -441,9 +369,9 @@ mt7531_pll_setup(struct mt7530_priv *priv)
 	u32 hwstrap;
 	u32 val;
 
-	val = mt7530_read(priv, MT7531_CREV);
-	top_sig = mt7530_read(priv, MT7531_TOP_SIG_SR);
-	hwstrap = mt7530_read(priv, MT753X_TRAP);
+	regmap_read(priv->regmap, MT7531_CREV, &val);
+	regmap_read(priv->regmap, MT7531_TOP_SIG_SR, &top_sig);
+	regmap_read(priv->regmap, MT753X_TRAP, &hwstrap);
 	if ((val & CHIP_REV_M) > 0)
 		xtal = (top_sig & PAD_MCM_SMI_EN) ? MT7531_XTAL_FSEL_40MHZ :
 						    MT7531_XTAL_FSEL_25MHZ;
@@ -452,72 +380,72 @@ mt7531_pll_setup(struct mt7530_priv *priv)
 						   MT7531_XTAL_FSEL_40MHZ;
 
 	/* Step 1 : Disable MT7531 COREPLL */
-	val = mt7530_read(priv, MT7531_PLLGP_EN);
+	regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
 	val &= ~EN_COREPLL;
-	mt7530_write(priv, MT7531_PLLGP_EN, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
 
 	/* Step 2: switch to XTAL output */
-	val = mt7530_read(priv, MT7531_PLLGP_EN);
+	regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
 	val |= SW_CLKSW;
-	mt7530_write(priv, MT7531_PLLGP_EN, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
 
-	val = mt7530_read(priv, MT7531_PLLGP_CR0);
+	regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
 	val &= ~RG_COREPLL_EN;
-	mt7530_write(priv, MT7531_PLLGP_CR0, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
 
 	/* Step 3: disable PLLGP and enable program PLLGP */
-	val = mt7530_read(priv, MT7531_PLLGP_EN);
+	regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
 	val |= SW_PLLGP;
-	mt7530_write(priv, MT7531_PLLGP_EN, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
 
 	/* Step 4: program COREPLL output frequency to 500MHz */
-	val = mt7530_read(priv, MT7531_PLLGP_CR0);
-	val &= ~RG_COREPLL_POSDIV_M;
-	val |= 2 << RG_COREPLL_POSDIV_S;
-	mt7530_write(priv, MT7531_PLLGP_CR0, val);
+	regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
+	val &= ~RG_COREPLL_POSDIV_MASK;
+	val |= RG_COREPLL_POSDIV(2);
+	regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
 	usleep_range(25, 35);
 
 	switch (xtal) {
 	case MT7531_XTAL_FSEL_25MHZ:
-		val = mt7530_read(priv, MT7531_PLLGP_CR0);
-		val &= ~RG_COREPLL_SDM_PCW_M;
-		val |= 0x140000 << RG_COREPLL_SDM_PCW_S;
-		mt7530_write(priv, MT7531_PLLGP_CR0, val);
+		regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
+		val &= ~RG_COREPLL_SDM_PCW_MASK;
+		val |= RG_COREPLL_SDM_PCW(0x140000);
+		regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
 		break;
 	case MT7531_XTAL_FSEL_40MHZ:
-		val = mt7530_read(priv, MT7531_PLLGP_CR0);
-		val &= ~RG_COREPLL_SDM_PCW_M;
-		val |= 0x190000 << RG_COREPLL_SDM_PCW_S;
-		mt7530_write(priv, MT7531_PLLGP_CR0, val);
+		regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
+		val &= ~RG_COREPLL_SDM_PCW_MASK;
+		val |= RG_COREPLL_SDM_PCW(0x190000);
+		regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
 		break;
 	}
 
 	/* Set feedback divide ratio update signal to high */
-	val = mt7530_read(priv, MT7531_PLLGP_CR0);
+	regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
 	val |= RG_COREPLL_SDM_PCW_CHG;
-	mt7530_write(priv, MT7531_PLLGP_CR0, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
 	/* Wait for at least 16 XTAL clocks */
 	usleep_range(10, 20);
 
 	/* Step 5: set feedback divide ratio update signal to low */
-	val = mt7530_read(priv, MT7531_PLLGP_CR0);
+	regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
 	val &= ~RG_COREPLL_SDM_PCW_CHG;
-	mt7530_write(priv, MT7531_PLLGP_CR0, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
 
 	/* Enable 325M clock for SGMII */
-	mt7530_write(priv, MT7531_ANA_PLLGP_CR5, 0xad0000);
+	regmap_write(priv->regmap, MT7531_ANA_PLLGP_CR5, 0xad0000);
 
 	/* Enable 250SSC clock for RGMII */
-	mt7530_write(priv, MT7531_ANA_PLLGP_CR2, 0x4f40000);
+	regmap_write(priv->regmap, MT7531_ANA_PLLGP_CR2, 0x4f40000);
 
 	/* Step 6: Enable MT7531 PLL */
-	val = mt7530_read(priv, MT7531_PLLGP_CR0);
+	regmap_read(priv->regmap, MT7531_PLLGP_CR0, &val);
 	val |= RG_COREPLL_EN;
-	mt7530_write(priv, MT7531_PLLGP_CR0, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_CR0, val);
 
-	val = mt7530_read(priv, MT7531_PLLGP_EN);
+	regmap_read(priv->regmap, MT7531_PLLGP_EN, &val);
 	val |= EN_COREPLL;
-	mt7530_write(priv, MT7531_PLLGP_EN, val);
+	regmap_write(priv->regmap, MT7531_PLLGP_EN, val);
 	usleep_range(25, 35);
 }
 
@@ -526,8 +454,8 @@ mt7530_mib_reset(struct dsa_switch *ds)
 {
 	struct mt7530_priv *priv = ds->priv;
 
-	mt7530_write(priv, MT7530_MIB_CCR, CCR_MIB_FLUSH);
-	mt7530_write(priv, MT7530_MIB_CCR, CCR_MIB_ACTIVATE);
+	regmap_write(priv->regmap, MT7530_MIB_CCR, CCR_MIB_FLUSH);
+	regmap_write(priv->regmap, MT7530_MIB_CCR, CCR_MIB_ACTIVATE);
 }
 
 static int mt7530_phy_read_c22(struct mt7530_priv *priv, int port, int regnum)
@@ -560,7 +488,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
 	u32 reg, val;
 	int ret;
 
-	mt7530_mutex_lock(priv);
+	mutex_lock(&priv->reg_mutex);
 
 	ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val,
 				       !(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -571,7 +499,8 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
 
 	reg = MT7531_MDIO_CL45_ADDR | MT7531_MDIO_PHY_ADDR(port) |
 	      MT7531_MDIO_DEV_ADDR(devad) | regnum;
-	ret = mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+	ret = regmap_write(priv->regmap, MT7531_PHY_IAC,
+			   reg | MT7531_PHY_ACS_ST);
 	if (ret < 0)
 		goto out;
 
@@ -584,7 +513,8 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
 
 	reg = MT7531_MDIO_CL45_READ | MT7531_MDIO_PHY_ADDR(port) |
 	      MT7531_MDIO_DEV_ADDR(devad);
-	ret = mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+	ret = regmap_write(priv->regmap, MT7531_PHY_IAC,
+			   reg | MT7531_PHY_ACS_ST);
 	if (ret < 0)
 		goto out;
 
@@ -597,7 +527,7 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad,
 
 	ret = val & MT7531_MDIO_RW_DATA_MASK;
 out:
-	mt7530_mutex_unlock(priv);
+	mutex_unlock(&priv->reg_mutex);
 
 	return ret;
 }
@@ -609,7 +539,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
 	u32 val, reg;
 	int ret;
 
-	mt7530_mutex_lock(priv);
+	mutex_lock(&priv->reg_mutex);
 
 	ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val,
 				       !(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -620,7 +550,8 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
 
 	reg = MT7531_MDIO_CL45_ADDR | MT7531_MDIO_PHY_ADDR(port) |
 	      MT7531_MDIO_DEV_ADDR(devad) | regnum;
-	ret = mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+	ret = regmap_write(priv->regmap, MT7531_PHY_IAC,
+			   reg | MT7531_PHY_ACS_ST);
 	if (ret < 0)
 		goto out;
 
@@ -633,7 +564,8 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
 
 	reg = MT7531_MDIO_CL45_WRITE | MT7531_MDIO_PHY_ADDR(port) |
 	      MT7531_MDIO_DEV_ADDR(devad) | data;
-	ret = mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+	ret = regmap_write(priv->regmap, MT7531_PHY_IAC,
+			   reg | MT7531_PHY_ACS_ST);
 	if (ret < 0)
 		goto out;
 
@@ -645,7 +577,7 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad,
 	}
 
 out:
-	mt7530_mutex_unlock(priv);
+	mutex_unlock(&priv->reg_mutex);
 
 	return ret;
 }
@@ -656,7 +588,7 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
 	int ret;
 	u32 val;
 
-	mt7530_mutex_lock(priv);
+	mutex_lock(&priv->reg_mutex);
 
 	ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val,
 				       !(val & MT7531_PHY_ACS_ST), 20, 100000);
@@ -668,7 +600,8 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
 	val = MT7531_MDIO_CL22_READ | MT7531_MDIO_PHY_ADDR(port) |
 	      MT7531_MDIO_REG_ADDR(regnum);
 
-	ret = mt7530_mii_write(priv, MT7531_PHY_IAC, val | MT7531_PHY_ACS_ST);
+	ret = regmap_write(priv->regmap, MT7531_PHY_IAC,
+			   val | MT7531_PHY_ACS_ST);
 	if (ret < 0)
 		goto out;
 
@@ -681,7 +614,7 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum)
 
 	ret = val & MT7531_MDIO_RW_DATA_MASK;
 out:
-	mt7530_mutex_unlock(priv);
+	mutex_unlock(&priv->reg_mutex);
 
 	return ret;
 }
@@ -693,7 +626,7 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
 	int ret;
 	u32 reg;
 
-	mt7530_mutex_lock(priv);
+	mutex_lock(&priv->reg_mutex);
 
 	ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, reg,
 				       !(reg & MT7531_PHY_ACS_ST), 20, 100000);
@@ -705,7 +638,8 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
 	reg = MT7531_MDIO_CL22_WRITE | MT7531_MDIO_PHY_ADDR(port) |
 	      MT7531_MDIO_REG_ADDR(regnum) | data;
 
-	ret = mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST);
+	ret = regmap_write(priv->regmap, MT7531_PHY_IAC,
+			   reg | MT7531_PHY_ACS_ST);
 	if (ret < 0)
 		goto out;
 
@@ -717,7 +651,7 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum,
 	}
 
 out:
-	mt7530_mutex_unlock(priv);
+	mutex_unlock(&priv->reg_mutex);
 
 	return ret;
 }
@@ -774,11 +708,11 @@ mt7530_read_port_stats(struct mt7530_priv *priv, int port,
 {
 	u32 val, reg = MT7530_PORT_MIB_COUNTER(port) + offset;
 
-	val = mt7530_read(priv, reg);
+	regmap_read(priv->regmap, reg, &val);
 	*data = val;
 
 	if (size == 2) {
-		val = mt7530_read(priv, reg + 4);
+		regmap_read(priv->regmap, reg + 4, &val);
 		*data |= (u64)val << 32;
 	}
 }
@@ -1060,7 +994,8 @@ mt7530_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
 		}
 	}
 
-	mt7530_write(priv, MT7530_AAC, AGE_CNT(age_count) | AGE_UNIT(age_unit));
+	regmap_write(priv->regmap, MT7530_AAC,
+		     AGE_CNT(age_count) | AGE_UNIT(age_unit));
 
 	return 0;
 }
@@ -1081,11 +1016,11 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
 {
 	struct mt7530_priv *priv = ds->priv;
 	u8 tx_delay = 0;
-	int val;
+	u32 val;
 
 	mutex_lock(&priv->reg_mutex);
 
-	val = mt7530_read(priv, MT753X_MTRAP);
+	regmap_read(priv->regmap, MT753X_MTRAP, &val);
 
 	val &= ~MT7530_P5_PHY0_SEL & ~MT7530_P5_MAC_SEL & ~MT7530_P5_RGMII_MODE;
 
@@ -1098,7 +1033,7 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
 	/* MUX_PHY_P4: P4 -> P5 -> SoC MAC */
 	case MUX_PHY_P4:
 		/* Setup the MAC by default for the cpu port */
-		mt7530_write(priv, MT753X_PMCR_P(5), 0x56300);
+		regmap_write(priv->regmap, MT753X_PMCR_P(5), 0x56300);
 		break;
 
 	/* GMAC5: P5 -> SoC MAC or external PHY */
@@ -1112,7 +1047,8 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
 		val |= MT7530_P5_RGMII_MODE;
 
 		/* P5 RGMII RX Clock Control: delay setting for 1000M */
-		mt7530_write(priv, MT7530_P5RGMIIRXCR, CSR_RGMII_EDGE_ALIGN);
+		regmap_write(priv->regmap, MT7530_P5RGMIIRXCR,
+			     CSR_RGMII_EDGE_ALIGN);
 
 		/* Don't set delay in DSA mode */
 		if (!dsa_is_dsa_port(priv->ds, 5) &&
@@ -1121,15 +1057,15 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
 			tx_delay = 4; /* n * 0.5 ns */
 
 		/* P5 RGMII TX Clock Control: delay x */
-		mt7530_write(priv, MT7530_P5RGMIITXCR,
+		regmap_write(priv->regmap, MT7530_P5RGMIITXCR,
 			     CSR_RGMII_TXC_CFG(0x10 + tx_delay));
 
 		/* reduce P5 RGMII Tx driving, 8mA */
-		mt7530_write(priv, MT7530_IO_DRV_CR,
+		regmap_write(priv->regmap, MT7530_IO_DRV_CR,
 			     P5_IO_CLK_DRV(1) | P5_IO_DATA_DRV(1));
 	}
 
-	mt7530_write(priv, MT753X_MTRAP, val);
+	regmap_write(priv->regmap, MT753X_MTRAP, val);
 
 	dev_dbg(ds->dev, "Setup P5, HWTRAP=0x%x, mode=%s, phy-mode=%s\n", val,
 		mt7530_p5_mode_str(priv->p5_mode), phy_modes(interface));
@@ -1314,35 +1250,35 @@ mt753x_trap_frames(struct mt7530_priv *priv)
 	 * switch egress VLAN tag processing. This preserves VLAN tags
 	 * for reception on VLAN sub-interfaces.
 	 */
-	mt7530_rmw(priv, MT753X_BPC,
-		   PAE_BPDU_FR | PAE_EG_TAG_MASK | PAE_PORT_FW_MASK |
-			   BPDU_EG_TAG_MASK | BPDU_PORT_FW_MASK,
-		   PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_DISABLED) |
-			   PAE_PORT_FW(TO_CPU_FW_CPU_ONLY) |
-			   BPDU_EG_TAG(MT7530_VLAN_EG_DISABLED) |
-			   TO_CPU_FW_CPU_ONLY);
+	regmap_update_bits(priv->regmap, MT753X_BPC,
+			   PAE_BPDU_FR | PAE_EG_TAG_MASK | PAE_PORT_FW_MASK |
+				   BPDU_EG_TAG_MASK | BPDU_PORT_FW_MASK,
+			   PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+				   PAE_PORT_FW(TO_CPU_FW_CPU_ONLY) |
+				   BPDU_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+				   TO_CPU_FW_CPU_ONLY);
 
 	/* Trap frames with :01 and :02 MAC DAs to the CPU port(s) and
 	 * egress them with EG_TAG disabled.
 	 */
-	mt7530_rmw(priv, MT753X_RGAC1,
-		   R02_BPDU_FR | R02_EG_TAG_MASK | R02_PORT_FW_MASK |
-			   R01_BPDU_FR | R01_EG_TAG_MASK | R01_PORT_FW_MASK,
-		   R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_DISABLED) |
-			   R02_PORT_FW(TO_CPU_FW_CPU_ONLY) | R01_BPDU_FR |
-			   R01_EG_TAG(MT7530_VLAN_EG_DISABLED) |
-			   TO_CPU_FW_CPU_ONLY);
+	regmap_update_bits(priv->regmap, MT753X_RGAC1,
+			   R02_BPDU_FR | R02_EG_TAG_MASK | R02_PORT_FW_MASK |
+				   R01_BPDU_FR | R01_EG_TAG_MASK | R01_PORT_FW_MASK,
+			   R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+				   R02_PORT_FW(TO_CPU_FW_CPU_ONLY) | R01_BPDU_FR |
+				   R01_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+				   TO_CPU_FW_CPU_ONLY);
 
 	/* Trap frames with :03 and :0E MAC DAs to the CPU port(s) and
 	 * egress them with EG_TAG disabled.
 	 */
-	mt7530_rmw(priv, MT753X_RGAC2,
-		   R0E_BPDU_FR | R0E_EG_TAG_MASK | R0E_PORT_FW_MASK |
-			   R03_BPDU_FR | R03_EG_TAG_MASK | R03_PORT_FW_MASK,
-		   R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_DISABLED) |
-			   R0E_PORT_FW(TO_CPU_FW_CPU_ONLY) | R03_BPDU_FR |
-			   R03_EG_TAG(MT7530_VLAN_EG_DISABLED) |
-			   TO_CPU_FW_CPU_ONLY);
+	regmap_update_bits(priv->regmap, MT753X_RGAC2,
+			   R0E_BPDU_FR | R0E_EG_TAG_MASK | R0E_PORT_FW_MASK |
+				   R03_BPDU_FR | R03_EG_TAG_MASK | R03_PORT_FW_MASK,
+			   R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+				   R0E_PORT_FW(TO_CPU_FW_CPU_ONLY) | R03_BPDU_FR |
+				   R03_EG_TAG(MT7530_VLAN_EG_DISABLED) |
+				   TO_CPU_FW_CPU_ONLY);
 }
 
 static void
@@ -1351,12 +1287,11 @@ mt753x_cpu_port_enable(struct dsa_switch *ds, int port)
 	struct mt7530_priv *priv = ds->priv;
 
 	/* Enable Mediatek header mode on the cpu port */
-	mt7530_write(priv, MT7530_PVC_P(port),
-		     PORT_SPEC_TAG);
+	regmap_write(priv->regmap, MT7530_PVC_P(port), PORT_SPEC_TAG);
 
 	/* Enable flooding on the CPU port */
-	mt7530_set(priv, MT753X_MFC, BC_FFP(BIT(port)) | UNM_FFP(BIT(port)) |
-		   UNU_FFP(BIT(port)));
+	regmap_set_bits(priv->regmap, MT753X_MFC,
+			BC_FFP(BIT(port)) | UNM_FFP(BIT(port)) | UNU_FFP(BIT(port)));
 
 	/* Add the CPU port to the CPU port bitmap for MT7531 and the switch on
 	 * the MT7988 SoC. Trapped frames will be forwarded to the CPU port that
@@ -1364,17 +1299,18 @@ mt753x_cpu_port_enable(struct dsa_switch *ds, int port)
 	 */
 	if (priv->id == ID_MT7531 || priv->id == ID_MT7988 ||
 	    priv->id == ID_EN7581 || priv->id == ID_AN7583)
-		mt7530_set(priv, MT7531_CFC, MT7531_CPU_PMAP(BIT(port)));
+		regmap_set_bits(priv->regmap, MT7531_CFC,
+				MT7531_CPU_PMAP(BIT(port)));
 
 	/* CPU port gets connected to all user ports of
 	 * the switch.
 	 */
-	mt7530_write(priv, MT7530_PCR_P(port),
+	regmap_write(priv->regmap, MT7530_PCR_P(port),
 		     PCR_MATRIX(dsa_user_ports(priv->ds)));
 
 	/* Set to fallback mode for independent VLAN learning */
-	mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
-		   MT7530_PORT_FALLBACK_MODE);
+	regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+			   PCR_PORT_VLAN_MASK, MT7530_PORT_FALLBACK_MODE);
 }
 
 static int
@@ -1396,8 +1332,8 @@ mt7530_port_enable(struct dsa_switch *ds, int port,
 		priv->ports[port].pm |= PCR_MATRIX(BIT(cpu_dp->index));
 	}
 	priv->ports[port].enable = true;
-	mt7530_rmw(priv, MT7530_PCR_P(port), PCR_MATRIX_MASK,
-		   priv->ports[port].pm);
+	regmap_update_bits(priv->regmap, MT7530_PCR_P(port), PCR_MATRIX_MASK,
+			   priv->ports[port].pm);
 
 	mutex_unlock(&priv->reg_mutex);
 
@@ -1405,9 +1341,9 @@ mt7530_port_enable(struct dsa_switch *ds, int port,
 		return 0;
 
 	if (port == 5)
-		mt7530_clear(priv, MT753X_MTRAP, MT7530_P5_DIS);
+		regmap_clear_bits(priv->regmap, MT753X_MTRAP, MT7530_P5_DIS);
 	else if (port == 6)
-		mt7530_clear(priv, MT753X_MTRAP, MT7530_P6_DIS);
+		regmap_clear_bits(priv->regmap, MT753X_MTRAP, MT7530_P6_DIS);
 
 	return 0;
 }
@@ -1423,8 +1359,8 @@ mt7530_port_disable(struct dsa_switch *ds, int port)
 	 * enablement for the port.
 	 */
 	priv->ports[port].enable = false;
-	mt7530_rmw(priv, MT7530_PCR_P(port), PCR_MATRIX_MASK,
-		   PCR_MATRIX_CLR);
+	regmap_update_bits(priv->regmap, MT7530_PCR_P(port), PCR_MATRIX_MASK,
+			   PCR_MATRIX_CLR);
 
 	mutex_unlock(&priv->reg_mutex);
 
@@ -1433,9 +1369,9 @@ mt7530_port_disable(struct dsa_switch *ds, int port)
 
 	/* Do not set MT7530_P5_DIS when port 5 is being used for PHY muxing. */
 	if (port == 5 && priv->p5_mode == GMAC5)
-		mt7530_set(priv, MT753X_MTRAP, MT7530_P5_DIS);
+		regmap_set_bits(priv->regmap, MT753X_MTRAP, MT7530_P5_DIS);
 	else if (port == 6)
-		mt7530_set(priv, MT753X_MTRAP, MT7530_P6_DIS);
+		regmap_set_bits(priv->regmap, MT753X_MTRAP, MT7530_P6_DIS);
 }
 
 static int
@@ -1452,9 +1388,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 	if (!dsa_is_cpu_port(ds, port))
 		return 0;
 
-	mt7530_mutex_lock(priv);
-
-	val = mt7530_mii_read(priv, MT7530_GMACCR);
+	regmap_read(priv->regmap, MT7530_GMACCR, &val);
 	val &= ~MAX_RX_PKT_LEN_MASK;
 
 	/* RX length also includes Ethernet header, MTK tag, and FCS length */
@@ -1471,9 +1405,7 @@ mt7530_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 		val |= MAX_RX_PKT_LEN_JUMBO;
 	}
 
-	mt7530_mii_write(priv, MT7530_GMACCR, val);
-
-	mt7530_mutex_unlock(priv);
+	regmap_write(priv->regmap, MT7530_GMACCR, val);
 
 	return 0;
 }
@@ -1509,8 +1441,9 @@ mt7530_stp_state_set(struct dsa_switch *ds, int port, u8 state)
 		break;
 	}
 
-	mt7530_rmw(priv, MT7530_SSP_P(port), FID_PST_MASK(FID_BRIDGED),
-		   FID_PST(FID_BRIDGED, stp_state));
+	regmap_update_bits(priv->regmap, MT7530_SSP_P(port),
+			   FID_PST_MASK(FID_BRIDGED),
+			   FID_PST(FID_BRIDGED, stp_state));
 }
 
 static void mt7530_update_port_member(struct mt7530_priv *priv, int port,
@@ -1549,8 +1482,9 @@ static void mt7530_update_port_member(struct mt7530_priv *priv, int port,
 		}
 
 		if (other_p->enable)
-			mt7530_rmw(priv, MT7530_PCR_P(other_port),
-				   PCR_MATRIX_MASK, other_p->pm);
+			regmap_update_bits(priv->regmap,
+					   MT7530_PCR_P(other_port),
+					   PCR_MATRIX_MASK, other_p->pm);
 	}
 
 	/* Add/remove the all other ports to this port matrix. For !join
@@ -1559,7 +1493,8 @@ static void mt7530_update_port_member(struct mt7530_priv *priv, int port,
 	 */
 	p->pm = PCR_MATRIX(port_bitmap);
 	if (priv->ports[port].enable)
-		mt7530_rmw(priv, MT7530_PCR_P(port), PCR_MATRIX_MASK, p->pm);
+		regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+				   PCR_MATRIX_MASK, p->pm);
 }
 
 static int
@@ -1582,20 +1517,23 @@ mt7530_port_bridge_flags(struct dsa_switch *ds, int port,
 	struct mt7530_priv *priv = ds->priv;
 
 	if (flags.mask & BR_LEARNING)
-		mt7530_rmw(priv, MT7530_PSC_P(port), SA_DIS,
-			   flags.val & BR_LEARNING ? 0 : SA_DIS);
+		regmap_update_bits(priv->regmap, MT7530_PSC_P(port), SA_DIS,
+				   flags.val & BR_LEARNING ? 0 : SA_DIS);
 
 	if (flags.mask & BR_FLOOD)
-		mt7530_rmw(priv, MT753X_MFC, UNU_FFP(BIT(port)),
-			   flags.val & BR_FLOOD ? UNU_FFP(BIT(port)) : 0);
+		regmap_update_bits(priv->regmap, MT753X_MFC,
+				   UNU_FFP(BIT(port)),
+				   flags.val & BR_FLOOD ? UNU_FFP(BIT(port)) : 0);
 
 	if (flags.mask & BR_MCAST_FLOOD)
-		mt7530_rmw(priv, MT753X_MFC, UNM_FFP(BIT(port)),
-			   flags.val & BR_MCAST_FLOOD ? UNM_FFP(BIT(port)) : 0);
+		regmap_update_bits(priv->regmap, MT753X_MFC,
+				   UNM_FFP(BIT(port)),
+				   flags.val & BR_MCAST_FLOOD ? UNM_FFP(BIT(port)) : 0);
 
 	if (flags.mask & BR_BCAST_FLOOD)
-		mt7530_rmw(priv, MT753X_MFC, BC_FFP(BIT(port)),
-			   flags.val & BR_BCAST_FLOOD ? BC_FFP(BIT(port)) : 0);
+		regmap_update_bits(priv->regmap, MT753X_MFC,
+				   BC_FFP(BIT(port)),
+				   flags.val & BR_BCAST_FLOOD ? BC_FFP(BIT(port)) : 0);
 
 	if (flags.mask & BR_ISOLATED) {
 		struct dsa_port *dp = dsa_to_port(ds, port);
@@ -1623,8 +1561,8 @@ mt7530_port_bridge_join(struct dsa_switch *ds, int port,
 	mt7530_update_port_member(priv, port, bridge.dev, true);
 
 	/* Set to fallback mode for independent VLAN learning */
-	mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
-		   MT7530_PORT_FALLBACK_MODE);
+	regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+			   PCR_PORT_VLAN_MASK, MT7530_PORT_FALLBACK_MODE);
 
 	mutex_unlock(&priv->reg_mutex);
 
@@ -1637,20 +1575,16 @@ mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
 	u32 val;
 	int ret;
 
-	val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
-	ret = mt7530_write(priv, MT7530_VTCR, val);
+	val = VTCR_BUSY | VTCR_FUNC(cmd) | VTCR_VID(vid);
+	ret = regmap_write(priv->regmap, MT7530_VTCR, val);
 	if (ret)
 		return ret;
 
-	mt7530_mutex_lock(priv);
-
 	ret = regmap_read_poll_timeout(priv->regmap, MT7530_VTCR, val,
 				       !(val & VTCR_BUSY), 20, 20000);
 	if (!ret)
 		ret = regmap_read(priv->regmap, MT7530_VTCR, &val);
 
-	mt7530_mutex_unlock(priv);
-
 	if (ret < 0) {
 		dev_err(priv->dev, "poll timeout\n");
 		return ret;
@@ -1674,8 +1608,8 @@ mt7530_setup_vlan0(struct mt7530_priv *priv)
 	 */
 	val = IVL_MAC | EG_CON | PORT_MEM(MT7530_ALL_MEMBERS) | FID(FID_BRIDGED) |
 	      VLAN_VALID;
-	mt7530_write(priv, MT7530_VAWD1, val);
-	mt7530_write(priv, MT7530_VAWD2, 0);
+	regmap_write(priv->regmap, MT7530_VAWD1, val);
+	regmap_write(priv->regmap, MT7530_VAWD2, 0);
 
 	return mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, 0);
 }
@@ -1691,18 +1625,19 @@ mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
 	 * bridge. Don't set standalone ports to fallback mode.
 	 */
 	if (dsa_port_bridge_dev_get(dsa_to_port(ds, port)))
-		mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
-			   MT7530_PORT_FALLBACK_MODE);
-
-	mt7530_rmw(priv, MT7530_PVC_P(port),
-		   VLAN_ATTR_MASK | PVC_EG_TAG_MASK | ACC_FRM_MASK,
-		   VLAN_ATTR(MT7530_VLAN_TRANSPARENT) |
-		   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT) |
-		   MT7530_VLAN_ACC_ALL);
+		regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+				   PCR_PORT_VLAN_MASK,
+				   MT7530_PORT_FALLBACK_MODE);
+
+	regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+			   VLAN_ATTR_MASK | PVC_EG_TAG_MASK | ACC_FRM_MASK,
+			   VLAN_ATTR(MT7530_VLAN_TRANSPARENT) |
+			   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT) |
+			   MT7530_VLAN_ACC_ALL);
 
 	/* Set PVID to 0 */
-	mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
-		   G0_PORT_VID_DEF);
+	regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+			   G0_PORT_VID_MASK, G0_PORT_VID_DEF);
 
 	for (i = 0; i < priv->ds->num_ports; i++) {
 		if (i == port)
@@ -1733,24 +1668,27 @@ mt7530_port_set_vlan_aware(struct dsa_switch *ds, int port)
 	 * table lookup.
 	 */
 	if (dsa_is_user_port(ds, port)) {
-		mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
-			   MT7530_PORT_SECURITY_MODE);
-		mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
-			   G0_PORT_VID(priv->ports[port].pvid));
+		regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+				   PCR_PORT_VLAN_MASK,
+				   MT7530_PORT_SECURITY_MODE);
+		regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+				   G0_PORT_VID_MASK,
+				   G0_PORT_VID(priv->ports[port].pvid));
 
 		/* Only accept tagged frames if PVID is not set */
 		if (!priv->ports[port].pvid)
-			mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
-				   MT7530_VLAN_ACC_TAGGED);
+			regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+					   ACC_FRM_MASK,
+					   MT7530_VLAN_ACC_TAGGED);
 
 		/* Set the port as a user port which is to be able to recognize
 		 * VID from incoming packets before fetching entry within the
 		 * VLAN table.
 		 */
-		mt7530_rmw(priv, MT7530_PVC_P(port),
-			   VLAN_ATTR_MASK | PVC_EG_TAG_MASK,
-			   VLAN_ATTR(MT7530_VLAN_USER) |
-			   PVC_EG_TAG(MT7530_VLAN_EG_DISABLED));
+		regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+				   VLAN_ATTR_MASK | PVC_EG_TAG_MASK,
+				   VLAN_ATTR(MT7530_VLAN_USER) |
+				   PVC_EG_TAG(MT7530_VLAN_EG_DISABLED));
 	} else {
 		/* Also set CPU ports to the "user" VLAN port attribute, to
 		 * allow VLAN classification, but keep the EG_TAG attribute as
@@ -1759,8 +1697,9 @@ mt7530_port_set_vlan_aware(struct dsa_switch *ds, int port)
 		 * are forwarded to user ports as tagged, and untagged as
 		 * untagged.
 		 */
-		mt7530_rmw(priv, MT7530_PVC_P(port), VLAN_ATTR_MASK,
-			   VLAN_ATTR(MT7530_VLAN_USER));
+		regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+				   VLAN_ATTR_MASK,
+				   VLAN_ATTR(MT7530_VLAN_USER));
 	}
 }
 
@@ -1778,8 +1717,8 @@ mt7530_port_bridge_leave(struct dsa_switch *ds, int port,
 	 * back to the default as is at initial boot which is a VLAN-unaware
 	 * port.
 	 */
-	mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
-		   MT7530_PORT_MATRIX_MODE);
+	regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+			   PCR_PORT_VLAN_MASK, MT7530_PORT_MATRIX_MODE);
 
 	mutex_unlock(&priv->reg_mutex);
 }
@@ -1862,14 +1801,16 @@ mt7530_port_mdb_add(struct dsa_switch *ds, int port,
 	const u8 *addr = mdb->addr;
 	u16 vid = mdb->vid;
 	u8 port_mask = 0;
+	u32 val;
 	int ret;
 
 	mutex_lock(&priv->reg_mutex);
 
 	mt7530_fdb_write(priv, vid, 0, addr, 0, STATIC_EMP);
-	if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL))
-		port_mask = (mt7530_read(priv, MT7530_ATRD) >> PORT_MAP)
-			    & PORT_MAP_MASK;
+	if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL)) {
+		regmap_read(priv->regmap, MT7530_ATRD, &val);
+		port_mask = FIELD_GET(PORT_MAP_MASK, val);
+	}
 
 	port_mask |= BIT(port);
 	mt7530_fdb_write(priv, vid, port_mask, addr, -1, STATIC_ENT);
@@ -1889,14 +1830,16 @@ mt7530_port_mdb_del(struct dsa_switch *ds, int port,
 	const u8 *addr = mdb->addr;
 	u16 vid = mdb->vid;
 	u8 port_mask = 0;
+	u32 val;
 	int ret;
 
 	mutex_lock(&priv->reg_mutex);
 
 	mt7530_fdb_write(priv, vid, 0, addr, 0, STATIC_EMP);
-	if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL))
-		port_mask = (mt7530_read(priv, MT7530_ATRD) >> PORT_MAP)
-			    & PORT_MAP_MASK;
+	if (!mt7530_fdb_cmd(priv, MT7530_FDB_READ, NULL)) {
+		regmap_read(priv->regmap, MT7530_ATRD, &val);
+		port_mask = FIELD_GET(PORT_MAP_MASK, val);
+	}
 
 	port_mask &= ~BIT(port);
 	mt7530_fdb_write(priv, vid, port_mask, addr, -1,
@@ -1945,7 +1888,7 @@ mt7530_hw_vlan_add(struct mt7530_priv *priv,
 	 */
 	val = IVL_MAC | VTAG_EN | PORT_MEM(new_members) | FID(FID_BRIDGED) |
 	      VLAN_VALID;
-	mt7530_write(priv, MT7530_VAWD1, val);
+	regmap_write(priv->regmap, MT7530_VAWD1, val);
 
 	/* Decide whether adding tag or not for those outgoing packets from the
 	 * port inside the VLAN.
@@ -1960,9 +1903,9 @@ mt7530_hw_vlan_add(struct mt7530_priv *priv,
 		val = MT7530_VLAN_EGRESS_UNTAG;
 	else
 		val = MT7530_VLAN_EGRESS_TAG;
-	mt7530_rmw(priv, MT7530_VAWD2,
-		   ETAG_CTRL_P_MASK(entry->port),
-		   ETAG_CTRL_P(entry->port, val));
+	regmap_update_bits(priv->regmap, MT7530_VAWD2,
+			   ETAG_CTRL_P_MASK(entry->port),
+			   ETAG_CTRL_P(entry->port, val));
 }
 
 static void
@@ -1974,7 +1917,7 @@ mt7530_hw_vlan_del(struct mt7530_priv *priv,
 
 	new_members = entry->old_members & ~BIT(entry->port);
 
-	val = mt7530_read(priv, MT7530_VAWD1);
+	regmap_read(priv->regmap, MT7530_VAWD1, &val);
 	if (!(val & VLAN_VALID)) {
 		dev_err(priv->dev,
 			"Cannot be deleted due to invalid entry\n");
@@ -1984,10 +1927,10 @@ mt7530_hw_vlan_del(struct mt7530_priv *priv,
 	if (new_members) {
 		val = IVL_MAC | VTAG_EN | PORT_MEM(new_members) |
 		      VLAN_VALID;
-		mt7530_write(priv, MT7530_VAWD1, val);
+		regmap_write(priv->regmap, MT7530_VAWD1, val);
 	} else {
-		mt7530_write(priv, MT7530_VAWD1, 0);
-		mt7530_write(priv, MT7530_VAWD2, 0);
+		regmap_write(priv->regmap, MT7530_VAWD1, 0);
+		regmap_write(priv->regmap, MT7530_VAWD2, 0);
 	}
 }
 
@@ -2001,9 +1944,9 @@ mt7530_hw_vlan_update(struct mt7530_priv *priv, u16 vid,
 	/* Fetch entry */
 	mt7530_vlan_cmd(priv, MT7530_VTCR_RD_VID, vid);
 
-	val = mt7530_read(priv, MT7530_VAWD1);
+	regmap_read(priv->regmap, MT7530_VAWD1, &val);
 
-	entry->old_members = (val >> PORT_MEM_SHFT) & PORT_MEM_MASK;
+	entry->old_members = FIELD_GET(PORT_MEM_MASK, val);
 
 	/* Manipulate entry */
 	vlan_op(priv, entry);
@@ -2040,25 +1983,26 @@ mt7530_port_vlan_add(struct dsa_switch *ds, int port,
 		priv->ports[port].pvid = vlan->vid;
 
 		/* Accept all frames if PVID is set */
-		mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
-			   MT7530_VLAN_ACC_ALL);
+		regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+				   ACC_FRM_MASK, MT7530_VLAN_ACC_ALL);
 
 		/* Only configure PVID if VLAN filtering is enabled */
 		if (dsa_port_is_vlan_filtering(dsa_to_port(ds, port)))
-			mt7530_rmw(priv, MT7530_PPBV1_P(port),
-				   G0_PORT_VID_MASK,
-				   G0_PORT_VID(vlan->vid));
+			regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+					   G0_PORT_VID_MASK,
+					   G0_PORT_VID(vlan->vid));
 	} else if (vlan->vid && priv->ports[port].pvid == vlan->vid) {
 		/* This VLAN is overwritten without PVID, so unset it */
 		priv->ports[port].pvid = G0_PORT_VID_DEF;
 
 		/* Only accept tagged frames if the port is VLAN-aware */
 		if (dsa_port_is_vlan_filtering(dsa_to_port(ds, port)))
-			mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
-				   MT7530_VLAN_ACC_TAGGED);
+			regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+					   ACC_FRM_MASK,
+					   MT7530_VLAN_ACC_TAGGED);
 
-		mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
-			   G0_PORT_VID_DEF);
+		regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+				   G0_PORT_VID_MASK, G0_PORT_VID_DEF);
 	}
 
 	mutex_unlock(&priv->reg_mutex);
@@ -2092,11 +2036,12 @@ mt7530_port_vlan_del(struct dsa_switch *ds, int port,
 
 		/* Only accept tagged frames if the port is VLAN-aware */
 		if (dsa_port_is_vlan_filtering(dsa_to_port(ds, port)))
-			mt7530_rmw(priv, MT7530_PVC_P(port), ACC_FRM_MASK,
-				   MT7530_VLAN_ACC_TAGGED);
+			regmap_update_bits(priv->regmap, MT7530_PVC_P(port),
+					   ACC_FRM_MASK,
+					   MT7530_VLAN_ACC_TAGGED);
 
-		mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
-			   G0_PORT_VID_DEF);
+		regmap_update_bits(priv->regmap, MT7530_PPBV1_P(port),
+				   G0_PORT_VID_MASK, G0_PORT_VID_DEF);
 	}
 
 
@@ -2117,7 +2062,7 @@ static int mt753x_port_mirror_add(struct dsa_switch *ds, int port,
 	if ((ingress ? priv->mirror_rx : priv->mirror_tx) & BIT(port))
 		return -EEXIST;
 
-	val = mt7530_read(priv, MT753X_MIRROR_REG(priv->id));
+	regmap_read(priv->regmap, MT753X_MIRROR_REG(priv->id), &val);
 
 	/* MT7530 only supports one monitor port */
 	monitor_port = MT753X_MIRROR_PORT_GET(priv->id, val);
@@ -2128,9 +2073,9 @@ static int mt753x_port_mirror_add(struct dsa_switch *ds, int port,
 	val |= MT753X_MIRROR_EN(priv->id);
 	val &= ~MT753X_MIRROR_PORT_MASK(priv->id);
 	val |= MT753X_MIRROR_PORT_SET(priv->id, mirror->to_local_port);
-	mt7530_write(priv, MT753X_MIRROR_REG(priv->id), val);
+	regmap_write(priv->regmap, MT753X_MIRROR_REG(priv->id), val);
 
-	val = mt7530_read(priv, MT7530_PCR_P(port));
+	regmap_read(priv->regmap, MT7530_PCR_P(port), &val);
 	if (ingress) {
 		val |= PORT_RX_MIR;
 		priv->mirror_rx |= BIT(port);
@@ -2138,7 +2083,7 @@ static int mt753x_port_mirror_add(struct dsa_switch *ds, int port,
 		val |= PORT_TX_MIR;
 		priv->mirror_tx |= BIT(port);
 	}
-	mt7530_write(priv, MT7530_PCR_P(port), val);
+	regmap_write(priv->regmap, MT7530_PCR_P(port), val);
 
 	return 0;
 }
@@ -2149,7 +2094,7 @@ static void mt753x_port_mirror_del(struct dsa_switch *ds, int port,
 	struct mt7530_priv *priv = ds->priv;
 	u32 val;
 
-	val = mt7530_read(priv, MT7530_PCR_P(port));
+	regmap_read(priv->regmap, MT7530_PCR_P(port), &val);
 	if (mirror->ingress) {
 		val &= ~PORT_RX_MIR;
 		priv->mirror_rx &= ~BIT(port);
@@ -2157,12 +2102,12 @@ static void mt753x_port_mirror_del(struct dsa_switch *ds, int port,
 		val &= ~PORT_TX_MIR;
 		priv->mirror_tx &= ~BIT(port);
 	}
-	mt7530_write(priv, MT7530_PCR_P(port), val);
+	regmap_write(priv->regmap, MT7530_PCR_P(port), val);
 
 	if (!priv->mirror_rx && !priv->mirror_tx) {
-		val = mt7530_read(priv, MT753X_MIRROR_REG(priv->id));
+		regmap_read(priv->regmap, MT753X_MIRROR_REG(priv->id), &val);
 		val &= ~MT753X_MIRROR_EN(priv->id);
-		mt7530_write(priv, MT753X_MIRROR_REG(priv->id), val);
+		regmap_write(priv->regmap, MT753X_MIRROR_REG(priv->id), val);
 	}
 }
 
@@ -2192,8 +2137,11 @@ mt7530_gpio_get(struct gpio_chip *gc, unsigned int offset)
 {
 	struct mt7530_priv *priv = gpiochip_get_data(gc);
 	u32 bit = mt7530_gpio_to_bit(offset);
+	u32 val;
 
-	return !!(mt7530_read(priv, MT7530_LED_GPIO_DATA) & bit);
+	regmap_read(priv->regmap, MT7530_LED_GPIO_DATA, &val);
+
+	return !!(val & bit);
 }
 
 static int
@@ -2202,10 +2150,7 @@ mt7530_gpio_set(struct gpio_chip *gc, unsigned int offset, int value)
 	struct mt7530_priv *priv = gpiochip_get_data(gc);
 	u32 bit = mt7530_gpio_to_bit(offset);
 
-	if (value)
-		mt7530_set(priv, MT7530_LED_GPIO_DATA, bit);
-	else
-		mt7530_clear(priv, MT7530_LED_GPIO_DATA, bit);
+	regmap_assign_bits(priv->regmap, MT7530_LED_GPIO_DATA, bit, value);
 
 	return 0;
 }
@@ -2215,8 +2160,11 @@ mt7530_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
 {
 	struct mt7530_priv *priv = gpiochip_get_data(gc);
 	u32 bit = mt7530_gpio_to_bit(offset);
+	u32 val;
 
-	return (mt7530_read(priv, MT7530_LED_GPIO_DIR) & bit) ?
+	regmap_read(priv->regmap, MT7530_LED_GPIO_DIR, &val);
+
+	return (val & bit) ?
 		GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
 }
 
@@ -2226,8 +2174,8 @@ mt7530_gpio_direction_input(struct gpio_chip *gc, unsigned int offset)
 	struct mt7530_priv *priv = gpiochip_get_data(gc);
 	u32 bit = mt7530_gpio_to_bit(offset);
 
-	mt7530_clear(priv, MT7530_LED_GPIO_OE, bit);
-	mt7530_clear(priv, MT7530_LED_GPIO_DIR, bit);
+	regmap_clear_bits(priv->regmap, MT7530_LED_GPIO_OE, bit);
+	regmap_clear_bits(priv->regmap, MT7530_LED_GPIO_DIR, bit);
 
 	return 0;
 }
@@ -2238,14 +2186,11 @@ mt7530_gpio_direction_output(struct gpio_chip *gc, unsigned int offset, int valu
 	struct mt7530_priv *priv = gpiochip_get_data(gc);
 	u32 bit = mt7530_gpio_to_bit(offset);
 
-	mt7530_set(priv, MT7530_LED_GPIO_DIR, bit);
+	regmap_set_bits(priv->regmap, MT7530_LED_GPIO_DIR, bit);
 
-	if (value)
-		mt7530_set(priv, MT7530_LED_GPIO_DATA, bit);
-	else
-		mt7530_clear(priv, MT7530_LED_GPIO_DATA, bit);
+	regmap_assign_bits(priv->regmap, MT7530_LED_GPIO_DATA, bit, value);
 
-	mt7530_set(priv, MT7530_LED_GPIO_OE, bit);
+	regmap_set_bits(priv->regmap, MT7530_LED_GPIO_OE, bit);
 
 	return 0;
 }
@@ -2260,9 +2205,9 @@ mt7530_setup_gpio(struct mt7530_priv *priv)
 	if (!gc)
 		return -ENOMEM;
 
-	mt7530_write(priv, MT7530_LED_GPIO_OE, 0);
-	mt7530_write(priv, MT7530_LED_GPIO_DIR, 0);
-	mt7530_write(priv, MT7530_LED_IO_MODE, 0);
+	regmap_write(priv->regmap, MT7530_LED_GPIO_OE, 0);
+	regmap_write(priv->regmap, MT7530_LED_GPIO_DIR, 0);
+	regmap_write(priv->regmap, MT7530_LED_IO_MODE, 0);
 
 	gc->label = "mt7530";
 	gc->parent = dev;
@@ -2319,21 +2264,6 @@ static const struct regmap_irq mt7530_irqs[] = {
 	REGMAP_IRQ_REG_LINE(31, 32), /* ACL */
 };
 
-/* Serialize regmap-irq's mask sync like every other regmap user */
-static int mt7530_irq_mask_sync(int index, unsigned int mask_buf_def,
-				unsigned int mask_buf, void *irq_drv_data)
-{
-	struct mt7530_priv *priv = irq_drv_data;
-	int ret;
-
-	mt7530_mutex_lock(priv);
-	ret = regmap_update_bits(priv->regmap, MT7530_SYS_INT_EN,
-				 mask_buf_def, ~mask_buf);
-	mt7530_mutex_unlock(priv);
-
-	return ret;
-}
-
 static const struct regmap_irq_chip mt7530_regmap_irq_chip = {
 	.name = KBUILD_MODNAME,
 	.status_base = MT7530_SYS_INT_STS,
@@ -2343,14 +2273,12 @@ static const struct regmap_irq_chip mt7530_regmap_irq_chip = {
 	.irqs = mt7530_irqs,
 	.num_irqs = ARRAY_SIZE(mt7530_irqs),
 	.num_regs = 1,
-	.handle_mask_sync = mt7530_irq_mask_sync,
 };
 
 static int
 mt7530_setup_irq(struct mt7530_priv *priv)
 {
 	struct regmap_irq_chip_data *irq_data;
-	struct regmap_irq_chip *chip;
 	struct device *dev = priv->dev;
 	struct device_node *np = dev->of_node;
 	int irq, ret;
@@ -2368,19 +2296,13 @@ mt7530_setup_irq(struct mt7530_priv *priv)
 
 	/* This register must be set for MT7530 to properly fire interrupts */
 	if (priv->id == ID_MT7530 || priv->id == ID_MT7621)
-		mt7530_set(priv, MT7530_TOP_SIG_CTRL, TOP_SIG_CTRL_NORMAL);
-
-	chip = devm_kmemdup(dev, &mt7530_regmap_irq_chip, sizeof(*chip),
-			    GFP_KERNEL);
-	if (!chip)
-		return -ENOMEM;
-
-	chip->irq_drv_data = priv;
+		regmap_set_bits(priv->regmap, MT7530_TOP_SIG_CTRL,
+				TOP_SIG_CTRL_NORMAL);
 
 	ret = devm_regmap_add_irq_chip_fwnode(dev, dev_fwnode(dev),
 					      priv->regmap, irq,
 					      IRQF_ONESHOT,
-					      0, chip,
+					      0, &mt7530_regmap_irq_chip,
 					      &irq_data);
 	if (ret)
 		return ret;
@@ -2461,7 +2383,6 @@ mt7530_setup(struct dsa_switch *ds)
 	struct device_node *dn = NULL;
 	struct device_node *phy_node;
 	struct device_node *mac_np;
-	struct mt7530_dummy_poll p;
 	phy_interface_t interface;
 	struct dsa_port *cpu_dp;
 	u32 id, val;
@@ -2522,16 +2443,15 @@ mt7530_setup(struct dsa_switch *ds)
 	}
 
 	/* Waiting for MT7530 got to stable */
-	INIT_MT7530_DUMMY_POLL(&p, priv, MT753X_TRAP);
-	ret = readx_poll_timeout(_mt7530_read, &p, val, val != 0,
-				 20, 1000000);
+	ret = regmap_read_poll_timeout(priv->regmap, MT753X_TRAP, val,
+				       val != 0, 20, 1000000);
 	if (ret < 0) {
 		dev_err(priv->dev, "reset timeout\n");
 		return ret;
 	}
 
-	id = mt7530_read(priv, MT7530_CREV);
-	id >>= CHIP_NAME_SHIFT;
+	regmap_read(priv->regmap, MT7530_CREV, &id);
+	id = FIELD_GET(CHIP_NAME_MASK, id);
 	if (id != MT7530_ID) {
 		dev_err(priv->dev, "chip %x can't be supported\n", id);
 		return -ENODEV;
@@ -2544,24 +2464,24 @@ mt7530_setup(struct dsa_switch *ds)
 	}
 
 	/* Reset the switch through internal reset */
-	mt7530_write(priv, MT7530_SYS_CTRL,
-		     SYS_CTRL_PHY_RST | SYS_CTRL_SW_RST |
-		     SYS_CTRL_REG_RST);
+	regmap_write(priv->regmap, MT7530_SYS_CTRL,
+		     SYS_CTRL_PHY_RST | SYS_CTRL_SW_RST | SYS_CTRL_REG_RST);
 
 	/* Lower Tx driving for TRGMII path */
 	for (i = 0; i < NUM_TRGMII_CTRL; i++)
-		mt7530_write(priv, MT7530_TRGMII_TD_ODT(i),
+		regmap_write(priv->regmap, MT7530_TRGMII_TD_ODT(i),
 			     TD_DM_DRVP(8) | TD_DM_DRVN(8));
 
 	for (i = 0; i < NUM_TRGMII_CTRL; i++)
-		mt7530_rmw(priv, MT7530_TRGMII_RD(i),
-			   RD_TAP_MASK, RD_TAP(16));
+		regmap_update_bits(priv->regmap, MT7530_TRGMII_RD(i),
+				   RD_TAP_MASK, RD_TAP(16));
 
 	/* Allow modifying the trap and directly access PHY registers via the
 	 * MDIO bus the switch is on.
 	 */
-	mt7530_rmw(priv, MT753X_MTRAP, MT7530_CHG_TRAP |
-		   MT7530_PHY_INDIRECT_ACCESS, MT7530_CHG_TRAP);
+	regmap_update_bits(priv->regmap, MT753X_MTRAP,
+			   MT7530_CHG_TRAP | MT7530_PHY_INDIRECT_ACCESS,
+			   MT7530_CHG_TRAP);
 
 	if ((val & MT7530_XTAL_MASK) == MT7530_XTAL_40MHZ)
 		mt7530_pll_setup(priv);
@@ -2575,17 +2495,16 @@ mt7530_setup(struct dsa_switch *ds)
 		/* Clear link settings and enable force mode to force link down
 		 * on all ports until they're enabled later.
 		 */
-		mt7530_rmw(priv, MT753X_PMCR_P(i),
-			   PMCR_LINK_SETTINGS_MASK |
-			   MT753X_FORCE_MODE(priv->id),
-			   MT753X_FORCE_MODE(priv->id));
+		regmap_update_bits(priv->regmap, MT753X_PMCR_P(i),
+				   PMCR_LINK_SETTINGS_MASK | MT753X_FORCE_MODE(priv->id),
+				   MT753X_FORCE_MODE(priv->id));
 
 		/* Disable forwarding by default on all ports */
-		mt7530_rmw(priv, MT7530_PCR_P(i), PCR_MATRIX_MASK,
-			   PCR_MATRIX_CLR);
+		regmap_update_bits(priv->regmap, MT7530_PCR_P(i),
+				   PCR_MATRIX_MASK, PCR_MATRIX_CLR);
 
 		/* Disable learning by default on all ports */
-		mt7530_set(priv, MT7530_PSC_P(i), SA_DIS);
+		regmap_set_bits(priv->regmap, MT7530_PSC_P(i), SA_DIS);
 
 		if (dsa_is_cpu_port(ds, i)) {
 			mt753x_cpu_port_enable(ds, i);
@@ -2593,16 +2512,17 @@ mt7530_setup(struct dsa_switch *ds)
 			mt7530_port_disable(ds, i);
 
 			/* Set default PVID to 0 on all user ports */
-			mt7530_rmw(priv, MT7530_PPBV1_P(i), G0_PORT_VID_MASK,
-				   G0_PORT_VID_DEF);
+			regmap_update_bits(priv->regmap, MT7530_PPBV1_P(i),
+					   G0_PORT_VID_MASK, G0_PORT_VID_DEF);
 		}
 		/* Enable consistent egress tag */
-		mt7530_rmw(priv, MT7530_PVC_P(i), PVC_EG_TAG_MASK,
-			   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
+		regmap_update_bits(priv->regmap, MT7530_PVC_P(i),
+				   PVC_EG_TAG_MASK,
+				   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
 	}
 
 	/* Allow mirroring frames received on the local port (monitor port). */
-	mt7530_set(priv, MT753X_AGC, LOCAL_EN);
+	regmap_set_bits(priv->regmap, MT753X_AGC, LOCAL_EN);
 
 	/* Setup VLAN ID 0 for VLAN-unaware bridges */
 	ret = mt7530_setup_vlan0(priv);
@@ -2649,7 +2569,8 @@ mt7530_setup(struct dsa_switch *ds)
 
 		if (priv->p5_mode == MUX_PHY_P0 ||
 		    priv->p5_mode == MUX_PHY_P4) {
-			mt7530_clear(priv, MT753X_MTRAP, MT7530_P5_DIS);
+			regmap_clear_bits(priv->regmap, MT753X_MTRAP,
+					  MT7530_P5_DIS);
 			mt7530_setup_port5(ds, interface);
 		}
 	}
@@ -2688,26 +2609,26 @@ mt7531_setup_common(struct dsa_switch *ds)
 	mt7530_mib_reset(ds);
 
 	/* Disable flooding on all ports */
-	mt7530_clear(priv, MT753X_MFC, BC_FFP_MASK | UNM_FFP_MASK |
-		     UNU_FFP_MASK);
+	regmap_clear_bits(priv->regmap, MT753X_MFC,
+			  BC_FFP_MASK | UNM_FFP_MASK | UNU_FFP_MASK);
 
 	for (i = 0; i < priv->ds->num_ports; i++) {
 		/* Clear link settings and enable force mode to force link down
 		 * on all ports until they're enabled later.
 		 */
-		mt7530_rmw(priv, MT753X_PMCR_P(i),
-			   PMCR_LINK_SETTINGS_MASK |
-			   MT753X_FORCE_MODE(priv->id),
-			   MT753X_FORCE_MODE(priv->id));
+		regmap_update_bits(priv->regmap, MT753X_PMCR_P(i),
+				   PMCR_LINK_SETTINGS_MASK | MT753X_FORCE_MODE(priv->id),
+				   MT753X_FORCE_MODE(priv->id));
 
 		/* Disable forwarding by default on all ports */
-		mt7530_rmw(priv, MT7530_PCR_P(i), PCR_MATRIX_MASK,
-			   PCR_MATRIX_CLR);
+		regmap_update_bits(priv->regmap, MT7530_PCR_P(i),
+				   PCR_MATRIX_MASK, PCR_MATRIX_CLR);
 
 		/* Disable learning by default on all ports */
-		mt7530_set(priv, MT7530_PSC_P(i), SA_DIS);
+		regmap_set_bits(priv->regmap, MT7530_PSC_P(i), SA_DIS);
 
-		mt7530_set(priv, MT7531_DBG_CNT(i), MT7531_DIS_CLR);
+		regmap_set_bits(priv->regmap, MT7531_DBG_CNT(i),
+				MT7531_DIS_CLR);
 
 		if (dsa_is_cpu_port(ds, i)) {
 			mt753x_cpu_port_enable(ds, i);
@@ -2715,21 +2636,22 @@ mt7531_setup_common(struct dsa_switch *ds)
 			mt7530_port_disable(ds, i);
 
 			/* Set default PVID to 0 on all user ports */
-			mt7530_rmw(priv, MT7530_PPBV1_P(i), G0_PORT_VID_MASK,
-				   G0_PORT_VID_DEF);
+			regmap_update_bits(priv->regmap, MT7530_PPBV1_P(i),
+					   G0_PORT_VID_MASK, G0_PORT_VID_DEF);
 		}
 
 		/* Enable consistent egress tag */
-		mt7530_rmw(priv, MT7530_PVC_P(i), PVC_EG_TAG_MASK,
-			   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
+		regmap_update_bits(priv->regmap, MT7530_PVC_P(i),
+				   PVC_EG_TAG_MASK,
+				   PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
 	}
 
 	/* Allow mirroring frames received on the local port (monitor port). */
-	mt7530_set(priv, MT753X_AGC, LOCAL_EN);
+	regmap_set_bits(priv->regmap, MT753X_AGC, LOCAL_EN);
 
 	/* Enable Special Tag for rx frames */
 	if (priv->id == ID_EN7581 || priv->id == ID_AN7583)
-		mt7530_write(priv, MT753X_CPORT_SPTAG_CFG,
+		regmap_write(priv->regmap, MT753X_CPORT_SPTAG_CFG,
 			     CPORT_SW2FE_STAG_EN | CPORT_FE2SW_STAG_EN);
 
 	/* Flush the FDB table */
@@ -2745,7 +2667,6 @@ static int
 mt7531_setup(struct dsa_switch *ds)
 {
 	struct mt7530_priv *priv = ds->priv;
-	struct mt7530_dummy_poll p;
 	u32 val, id;
 	int ret, i;
 
@@ -2763,16 +2684,15 @@ mt7531_setup(struct dsa_switch *ds)
 	}
 
 	/* Waiting for MT7530 got to stable */
-	INIT_MT7530_DUMMY_POLL(&p, priv, MT753X_TRAP);
-	ret = readx_poll_timeout(_mt7530_read, &p, val, val != 0,
-				 20, 1000000);
+	ret = regmap_read_poll_timeout(priv->regmap, MT753X_TRAP, val,
+				       val != 0, 20, 1000000);
 	if (ret < 0) {
 		dev_err(priv->dev, "reset timeout\n");
 		return ret;
 	}
 
-	id = mt7530_read(priv, MT7531_CREV);
-	id >>= CHIP_NAME_SHIFT;
+	regmap_read(priv->regmap, MT7531_CREV, &id);
+	id = FIELD_GET(CHIP_NAME_MASK, id);
 
 	if (id != MT7531_ID) {
 		dev_err(priv->dev, "chip %x can't be supported\n", id);
@@ -2782,15 +2702,17 @@ mt7531_setup(struct dsa_switch *ds)
 	/* MT7531AE has got two SGMII units. One for port 5, one for port 6.
 	 * MT7531BE has got only one SGMII unit which is for port 6.
 	 */
-	val = mt7530_read(priv, MT7531_TOP_SIG_SR);
+	regmap_read(priv->regmap, MT7531_TOP_SIG_SR, &val);
 	priv->p5_sgmii = !!(val & PAD_DUAL_SGMII_EN);
 
 	/* Force link down on all ports before internal reset */
 	for (i = 0; i < priv->ds->num_ports; i++)
-		mt7530_write(priv, MT753X_PMCR_P(i), MT7531_FORCE_MODE_LNK);
+		regmap_write(priv->regmap, MT753X_PMCR_P(i),
+			     MT7531_FORCE_MODE_LNK);
 
 	/* Reset the switch through internal reset */
-	mt7530_write(priv, MT7530_SYS_CTRL, SYS_CTRL_SW_RST | SYS_CTRL_REG_RST);
+	regmap_write(priv->regmap, MT7530_SYS_CTRL,
+		     SYS_CTRL_SW_RST | SYS_CTRL_REG_RST);
 
 	if (!priv->p5_sgmii) {
 		mt7531_pll_setup(priv);
@@ -2799,14 +2721,16 @@ mt7531_setup(struct dsa_switch *ds)
 		 * MT7531AE. Set the GPIO 11-12 pins to function as MDC and MDIO
 		 * to expose the MDIO bus of the switch.
 		 */
-		mt7530_rmw(priv, MT7531_GPIO_MODE1, MT7531_GPIO11_RG_RXD2_MASK,
-			   MT7531_EXT_P_MDC_11);
-		mt7530_rmw(priv, MT7531_GPIO_MODE1, MT7531_GPIO12_RG_RXD3_MASK,
-			   MT7531_EXT_P_MDIO_12);
+		regmap_update_bits(priv->regmap, MT7531_GPIO_MODE1,
+				   MT7531_GPIO11_RG_RXD2_MASK,
+				   MT7531_EXT_P_MDC_11);
+		regmap_update_bits(priv->regmap, MT7531_GPIO_MODE1,
+				   MT7531_GPIO12_RG_RXD3_MASK,
+				   MT7531_EXT_P_MDIO_12);
 	}
 
-	mt7530_rmw(priv, MT7531_GPIO_MODE0, MT7531_GPIO0_MASK,
-		   MT7531_GPIO0_INTERRUPT);
+	regmap_update_bits(priv->regmap, MT7531_GPIO_MODE0, MT7531_GPIO0_MASK,
+			   MT7531_GPIO0_INTERRUPT);
 
 	/* Enable Energy-Efficient Ethernet (EEE) and PHY core PLL, since
 	 * phy_device has not yet been created provided for
@@ -2998,7 +2922,7 @@ static void mt7531_rgmii_setup(struct mt7530_priv *priv,
 {
 	u32 val;
 
-	val = mt7530_read(priv, MT7531_CLKGEN_CTRL);
+	regmap_read(priv->regmap, MT7531_CLKGEN_CTRL, &val);
 	val |= GP_CLK_EN;
 	val &= ~GP_MODE_MASK;
 	val |= GP_MODE(MT7531_GP_MODE_RGMII);
@@ -3029,7 +2953,7 @@ static void mt7531_rgmii_setup(struct mt7530_priv *priv,
 		}
 	}
 
-	mt7530_write(priv, MT7531_CLKGEN_CTRL, val);
+	regmap_write(priv->regmap, MT7531_CLKGEN_CTRL, val);
 }
 
 static void
@@ -3082,7 +3006,8 @@ mt753x_phylink_mac_config(struct phylink_config *config, unsigned int mode,
 
 	/* Are we connected to external phy */
 	if (port == 5 && dsa_is_user_port(ds, 5))
-		mt7530_set(priv, MT753X_PMCR_P(port), PMCR_EXT_PHY);
+		regmap_set_bits(priv->regmap, MT753X_PMCR_P(port),
+				PMCR_EXT_PHY);
 }
 
 static void mt753x_phylink_mac_link_down(struct phylink_config *config,
@@ -3092,7 +3017,8 @@ static void mt753x_phylink_mac_link_down(struct phylink_config *config,
 	struct dsa_port *dp = dsa_phylink_to_port(config);
 	struct mt7530_priv *priv = dp->ds->priv;
 
-	mt7530_clear(priv, MT753X_PMCR_P(dp->index), PMCR_LINK_SETTINGS_MASK);
+	regmap_clear_bits(priv->regmap, MT753X_PMCR_P(dp->index),
+			  PMCR_LINK_SETTINGS_MASK);
 }
 
 static void mt753x_phylink_mac_link_up(struct phylink_config *config,
@@ -3126,7 +3052,7 @@ static void mt753x_phylink_mac_link_up(struct phylink_config *config,
 			mcr |= PMCR_FORCE_RX_FC_EN;
 	}
 
-	mt7530_set(priv, MT753X_PMCR_P(dp->index), mcr);
+	regmap_set_bits(priv->regmap, MT753X_PMCR_P(dp->index), mcr);
 }
 
 static void mt753x_phylink_mac_disable_tx_lpi(struct phylink_config *config)
@@ -3134,8 +3060,8 @@ static void mt753x_phylink_mac_disable_tx_lpi(struct phylink_config *config)
 	struct dsa_port *dp = dsa_phylink_to_port(config);
 	struct mt7530_priv *priv = dp->ds->priv;
 
-	mt7530_clear(priv, MT753X_PMCR_P(dp->index),
-		     PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
+	regmap_clear_bits(priv->regmap, MT753X_PMCR_P(dp->index),
+			  PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
 }
 
 static int mt753x_phylink_mac_enable_tx_lpi(struct phylink_config *config,
@@ -3156,11 +3082,11 @@ static int mt753x_phylink_mac_enable_tx_lpi(struct phylink_config *config,
 	else
 		val = LPI_THRESH_MASK;
 
-	mt7530_rmw(priv, MT753X_PMEEECR_P(dp->index),
-		   LPI_THRESH_MASK | LPI_MODE_EN, val);
+	regmap_update_bits(priv->regmap, MT753X_PMEEECR_P(dp->index),
+			   LPI_THRESH_MASK | LPI_MODE_EN, val);
 
-	mt7530_set(priv, MT753X_PMCR_P(dp->index),
-		   PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
+	regmap_set_bits(priv->regmap, MT753X_PMCR_P(dp->index),
+			PMCR_FORCE_EEE1G | PMCR_FORCE_EEE100);
 
 	return 0;
 }
@@ -3182,7 +3108,9 @@ static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port,
 	 */
 	if (priv->id != ID_EN7528 &&
 	    config->mac_capabilities & (MAC_100FD | MAC_1000FD)) {
-		u32 eeecr = mt7530_read(priv, MT753X_PMEEECR_P(port));
+		u32 eeecr;
+
+		regmap_read(priv->regmap, MT753X_PMEEECR_P(port), &eeecr);
 
 		/* LPI above 1 Gbps is not supported */
 		config->lpi_capabilities = config->mac_capabilities &
@@ -3218,7 +3146,7 @@ static void mt7530_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode,
 	int port = pcs_to_mt753x_pcs(pcs)->port;
 	u32 pmsr;
 
-	pmsr = mt7530_read(priv, MT7530_PMSR_P(port));
+	regmap_read(priv->regmap, MT7530_PMSR_P(port), &pmsr);
 
 	state->link = (pmsr & PMSR_LINK);
 	state->an_complete = state->link;
@@ -3323,6 +3251,43 @@ static int mt753x_set_mac_eee(struct dsa_switch *ds, int port,
 	return 0;
 }
 
+static int
+mt753x_port_change_conduit(struct dsa_switch *ds, int port,
+			   struct net_device *conduit,
+			   struct netlink_ext_ack *extack)
+{
+	struct dsa_port *new_cpu_dp = conduit->dsa_ptr;
+	struct dsa_port *dp = dsa_to_port(ds, port);
+	struct mt7530_priv *priv = ds->priv;
+
+	if (priv->id != ID_MT7531) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Changing DSA conduit is only supported on MT7531");
+		return -EOPNOTSUPP;
+	}
+
+	if (new_cpu_dp->ds != ds) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot assign a conduit on a different switch");
+		return -EOPNOTSUPP;
+	}
+
+	mutex_lock(&priv->reg_mutex);
+
+	/* dp->cpu_dp still points to the old CPU port */
+	priv->ports[port].pm &= ~PCR_MATRIX(BIT(dp->cpu_dp->index));
+	priv->ports[port].pm |= PCR_MATRIX(BIT(new_cpu_dp->index));
+	if (priv->ports[port].enable)
+		regmap_update_bits(priv->regmap, MT7530_PCR_P(port),
+				   PCR_MATRIX_MASK, priv->ports[port].pm);
+
+	mutex_unlock(&priv->reg_mutex);
+
+	mt7530_port_fast_age(ds, port);
+
+	return 0;
+}
+
 static void
 mt753x_conduit_state_change(struct dsa_switch *ds,
 			    const struct net_device *conduit,
@@ -3353,7 +3318,8 @@ mt753x_conduit_state_change(struct dsa_switch *ds,
 		      MT7530_CPU_PORT(__ffs(priv->active_cpu_ports));
 	}
 
-	mt7530_rmw(priv, MT753X_MFC, MT7530_CPU_EN | MT7530_CPU_PORT_MASK, val);
+	regmap_update_bits(priv->regmap, MT753X_MFC,
+			   MT7530_CPU_EN | MT7530_CPU_PORT_MASK, val);
 }
 
 static int mt753x_tc_setup_qdisc_tbf(struct dsa_switch *ds, int port,
@@ -3370,8 +3336,8 @@ static int mt753x_tc_setup_qdisc_tbf(struct dsa_switch *ds, int port,
 	case TC_TBF_DESTROY: {
 		u32 val, tick;
 
-		mt7530_rmw(priv, MT753X_GERLCR, EGR_BC_MASK,
-			   EGR_BC_CRC_IPG_PREAMBLE);
+		regmap_update_bits(priv->regmap, MT753X_GERLCR, EGR_BC_MASK,
+				   EGR_BC_CRC_IPG_PREAMBLE);
 
 		/* if rate is greater than 10Mbps tick is 1/32 ms,
 		 * 1ms otherwise
@@ -3382,7 +3348,7 @@ static int mt753x_tc_setup_qdisc_tbf(struct dsa_switch *ds, int port,
 		      FIELD_PREP(ERLCR_EXP_MASK, tick) |
 		      ERLCR_TBF_MODE_MASK |
 		      FIELD_PREP(ERLCR_MANT_MASK, 0xf);
-		mt7530_write(priv, MT753X_ERLCR_P(port), val);
+		regmap_write(priv->regmap, MT753X_ERLCR_P(port), val);
 		break;
 	}
 	default:
@@ -3415,16 +3381,16 @@ static int mt7988_setup(struct dsa_switch *ds)
 
 	/* AN7583 require additional tweak to CONN_CFG */
 	if (priv->id == ID_AN7583)
-		mt7530_rmw(priv, AN7583_GEPHY_CONN_CFG,
-			   AN7583_CSR_DPHY_CKIN_SEL |
-			   AN7583_CSR_PHY_CORE_REG_CLK_SEL |
-			   AN7583_CSR_ETHER_AFE_PWD,
-			   AN7583_CSR_DPHY_CKIN_SEL |
-			   AN7583_CSR_PHY_CORE_REG_CLK_SEL |
-			   FIELD_PREP(AN7583_CSR_ETHER_AFE_PWD, 0));
+		regmap_update_bits(priv->regmap, AN7583_GEPHY_CONN_CFG,
+				   AN7583_CSR_DPHY_CKIN_SEL |
+				   AN7583_CSR_PHY_CORE_REG_CLK_SEL |
+				   AN7583_CSR_ETHER_AFE_PWD,
+				   AN7583_CSR_DPHY_CKIN_SEL |
+				   AN7583_CSR_PHY_CORE_REG_CLK_SEL |
+				   FIELD_PREP(AN7583_CSR_ETHER_AFE_PWD, 0));
 
 	/* Reset the switch PHYs */
-	mt7530_write(priv, MT7530_SYS_CTRL, SYS_CTRL_PHY_RST);
+	regmap_write(priv->regmap, MT7530_SYS_CTRL, SYS_CTRL_PHY_RST);
 
 	return mt7531_setup_common(ds);
 }
@@ -3434,6 +3400,7 @@ static const struct dsa_switch_ops mt7530_switch_ops = {
 	.setup			= mt753x_setup,
 	.teardown		= mt753x_teardown,
 	.preferred_default_local_cpu_port = mt753x_preferred_default_local_cpu_port,
+	.port_change_conduit	= mt753x_port_change_conduit,
 	.get_strings		= mt7530_get_strings,
 	.get_ethtool_stats	= mt7530_get_ethtool_stats,
 	.get_sset_count		= mt7530_get_sset_count,
@@ -3451,6 +3418,7 @@ static const struct dsa_switch_ops mt7530_switch_ops = {
 	.port_bridge_flags	= mt7530_port_bridge_flags,
 	.port_bridge_join	= mt7530_port_bridge_join,
 	.port_bridge_leave	= mt7530_port_bridge_leave,
+	.port_fast_age		= mt7530_port_fast_age,
 	.port_fdb_add		= mt7530_port_fdb_add,
 	.port_fdb_del		= mt7530_port_fdb_del,
 	.port_fdb_dump		= mt7530_port_fdb_dump,
@@ -3593,9 +3561,6 @@ EXPORT_SYMBOL_GPL(mt7530_probe_common);
 void
 mt7530_remove_common(struct mt7530_priv *priv)
 {
-	if (priv->irq_domain)
-		mt7530_free_mdio_irq(priv);
-
 	dsa_unregister_switch(priv->ds);
 
 	mutex_destroy(&priv->reg_mutex);
diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
index 5f1e841f42c0ed..2bbbe617b52eed 100644
--- a/drivers/net/dsa/mt7530.h
+++ b/drivers/net/dsa/mt7530.h
@@ -6,6 +6,8 @@
 #ifndef __MT7530_H
 #define __MT7530_H
 
+#include <linux/bitfield.h>
+
 #define MT7530_NUM_PORTS		7
 #define MT7530_NUM_PHYS			5
 #define MT7530_NUM_FDB_RECORDS		2048
@@ -147,20 +149,24 @@ enum mt753x_to_cpu_fw {
 #define  STATIC_ENT			3
 #define MT7530_ATA2			0x78
 #define  ATA2_IVL			BIT(15)
-#define  ATA2_FID(x)			(((x) & 0x7) << 12)
+#define  ATA2_FID_MASK			GENMASK(14, 12)
+#define  ATA2_FID(x)			FIELD_PREP(ATA2_FID_MASK, x)
 
 /* Register for address table write data */
 #define MT7530_ATWD			0x7c
 
 /* Register for address table control */
 #define MT7530_ATC			0x80
-#define  ATC_HASH			(((x) & 0xfff) << 16)
+#define  ATC_HASH_MASK			GENMASK(27, 16)
+#define  ATC_HASH(x)			FIELD_PREP(ATC_HASH_MASK, x)
 #define  ATC_BUSY			BIT(15)
 #define  ATC_SRCH_END			BIT(14)
 #define  ATC_SRCH_HIT			BIT(13)
 #define  ATC_INVALID			BIT(12)
-#define  ATC_MAT(x)			(((x) & 0xf) << 8)
+#define  ATC_MAT_MASK			GENMASK(11, 8)
+#define  ATC_MAT(x)			FIELD_PREP(ATC_MAT_MASK, x)
 #define  ATC_MAT_MACTAB			ATC_MAT(0)
+#define  ATC_MAT_NON_STATIC_MAC		ATC_MAT(4)
 
 enum mt7530_fdb_cmd {
 	MT7530_FDB_READ	= 0,
@@ -172,32 +178,29 @@ enum mt7530_fdb_cmd {
 
 /* Registers for table search read address */
 #define MT7530_TSRA1			0x84
-#define  MAC_BYTE_0			24
-#define  MAC_BYTE_1			16
-#define  MAC_BYTE_2			8
-#define  MAC_BYTE_3			0
-#define  MAC_BYTE_MASK			0xff
+#define  MAC_BYTE_0_MASK		GENMASK(31, 24)
+#define  MAC_BYTE_1_MASK		GENMASK(23, 16)
+#define  MAC_BYTE_2_MASK		GENMASK(15, 8)
+#define  MAC_BYTE_3_MASK		GENMASK(7, 0)
 
 #define MT7530_TSRA2			0x88
-#define  MAC_BYTE_4			24
-#define  MAC_BYTE_5			16
-#define  CVID				0
-#define  CVID_MASK			0xfff
+#define  MAC_BYTE_4_MASK		GENMASK(31, 24)
+#define  MAC_BYTE_5_MASK		GENMASK(23, 16)
+#define  CVID_MASK			GENMASK(11, 0)
 
 #define MT7530_ATRD			0x8C
-#define	 AGE_TIMER			24
-#define  AGE_TIMER_MASK			0xff
-#define  PORT_MAP			4
-#define  PORT_MAP_MASK			0xff
-#define  ENT_STATUS			2
-#define  ENT_STATUS_MASK		0x3
+#define  AGE_TIMER_MASK			GENMASK(31, 24)
+#define  PORT_MAP_MASK			GENMASK(11, 4)
+#define  ENT_STATUS_MASK		GENMASK(3, 2)
 
 /* Register for vlan table control */
 #define MT7530_VTCR			0x90
 #define  VTCR_BUSY			BIT(31)
 #define  VTCR_INVALID			BIT(16)
-#define  VTCR_FUNC(x)			(((x) & 0xf) << 12)
-#define  VTCR_VID			((x) & 0xfff)
+#define  VTCR_FUNC_MASK			GENMASK(15, 12)
+#define  VTCR_FUNC(x)			FIELD_PREP(VTCR_FUNC_MASK, x)
+#define  VTCR_VID_MASK			GENMASK(11, 0)
+#define  VTCR_VID(x)			FIELD_PREP(VTCR_VID_MASK, x)
 
 enum mt7530_vlan_cmd {
 	/* Read/Write the specified VID entry from VAWD register based
@@ -217,13 +220,13 @@ enum mt7530_vlan_cmd {
 /* Per VLAN Egress Tag Control */
 #define  VTAG_EN			BIT(28)
 /* VLAN Member Control */
-#define  PORT_MEM(x)			(((x) & 0xff) << 16)
+#define  PORT_MEM_MASK			GENMASK(23, 16)
+#define  PORT_MEM(x)			FIELD_PREP(PORT_MEM_MASK, x)
 /* Filter ID */
-#define  FID(x)				(((x) & 0x7) << 1)
+#define  FID_MASK			GENMASK(3, 1)
+#define  FID(x)				FIELD_PREP(FID_MASK, x)
 /* VLAN Entry Valid */
 #define  VLAN_VALID			BIT(0)
-#define  PORT_MEM_SHFT			16
-#define  PORT_MEM_MASK			0xff
 
 enum mt7530_fid {
 	FID_STANDALONE = 0,
@@ -248,11 +251,11 @@ enum mt7530_vlan_egress_attr {
 /* Age count */
 #define  AGE_CNT_MASK			GENMASK(19, 12)
 #define  AGE_CNT_MAX			0xff
-#define  AGE_CNT(x)			(AGE_CNT_MASK & ((x) << 12))
+#define  AGE_CNT(x)			FIELD_PREP(AGE_CNT_MASK, x)
 /* Age unit */
 #define  AGE_UNIT_MASK			GENMASK(11, 0)
 #define  AGE_UNIT_MAX			0xfff
-#define  AGE_UNIT(x)			(AGE_UNIT_MASK & (x))
+#define  AGE_UNIT(x)			FIELD_PREP(AGE_UNIT_MASK, x)
 
 #define MT753X_ERLCR_P(x)		(0x1040 + ((x) * 0x100))
 #define  ERLCR_CIR_MASK			GENMASK(31, 16)
@@ -283,30 +286,31 @@ enum mt7530_stp_state {
 #define MT7530_PCR_P(x)			(0x2004 + ((x) * 0x100))
 #define  PORT_TX_MIR			BIT(9)
 #define  PORT_RX_MIR			BIT(8)
-#define  PORT_VLAN(x)			((x) & 0x3)
+#define  PCR_PORT_VLAN_MASK		GENMASK(1, 0)
 
 enum mt7530_port_mode {
 	/* Port Matrix Mode: Frames are forwarded by the PCR_MATRIX members. */
-	MT7530_PORT_MATRIX_MODE = PORT_VLAN(0),
+	MT7530_PORT_MATRIX_MODE = 0,
 
 	/* Fallback Mode: Forward received frames with ingress ports that do
 	 * not belong to the VLAN member. Frames whose VID is not listed on
 	 * the VLAN table are forwarded by the PCR_MATRIX members.
 	 */
-	MT7530_PORT_FALLBACK_MODE = PORT_VLAN(1),
+	MT7530_PORT_FALLBACK_MODE = 1,
 
 	/* Security Mode: Discard any frame due to ingress membership
 	 * violation or VID missed on the VLAN table.
 	 */
-	MT7530_PORT_SECURITY_MODE = PORT_VLAN(3),
+	MT7530_PORT_SECURITY_MODE = 3,
 };
 
-#define  PCR_MATRIX(x)			(((x) & 0xff) << 16)
-#define  PORT_PRI(x)			(((x) & 0x7) << 24)
-#define  EG_TAG(x)			(((x) & 0x3) << 28)
-#define  PCR_MATRIX_MASK		PCR_MATRIX(0xff)
+#define  PCR_MATRIX_MASK		GENMASK(23, 16)
+#define  PCR_MATRIX(x)			FIELD_PREP(PCR_MATRIX_MASK, x)
+#define  PORT_PRI_MASK			GENMASK(26, 24)
+#define  PORT_PRI(x)			FIELD_PREP(PORT_PRI_MASK, x)
+#define  EG_TAG_MASK			GENMASK(29, 28)
+#define  EG_TAG(x)			FIELD_PREP(EG_TAG_MASK, x)
 #define  PCR_MATRIX_CLR			PCR_MATRIX(0)
-#define  PCR_PORT_VLAN_MASK		PORT_VLAN(3)
 
 /* Register for port security control */
 #define MT7530_PSC_P(x)			(0x200c + ((x) * 0x100))
@@ -315,10 +319,10 @@ enum mt7530_port_mode {
 /* Register for port vlan control */
 #define MT7530_PVC_P(x)			(0x2010 + ((x) * 0x100))
 #define  PORT_SPEC_TAG			BIT(5)
-#define  PVC_EG_TAG(x)			(((x) & 0x7) << 8)
-#define  PVC_EG_TAG_MASK		PVC_EG_TAG(7)
-#define  VLAN_ATTR(x)			(((x) & 0x3) << 6)
-#define  VLAN_ATTR_MASK			VLAN_ATTR(3)
+#define  PVC_EG_TAG_MASK		GENMASK(10, 8)
+#define  PVC_EG_TAG(x)			FIELD_PREP(PVC_EG_TAG_MASK, x)
+#define  VLAN_ATTR_MASK			GENMASK(7, 6)
+#define  VLAN_ATTR(x)			FIELD_PREP(VLAN_ATTR_MASK, x)
 #define  ACC_FRM_MASK			GENMASK(1, 0)
 
 enum mt7530_vlan_port_eg_tag {
@@ -338,12 +342,13 @@ enum mt7530_vlan_port_acc_frm {
 	MT7530_VLAN_ACC_UNTAGGED = 2,
 };
 
-#define  STAG_VPID			(((x) & 0xffff) << 16)
+#define  STAG_VPID_MASK			GENMASK(31, 16)
+#define  STAG_VPID(x)			FIELD_PREP(STAG_VPID_MASK, x)
 
 /* Register for port port-and-protocol based vlan 1 control */
 #define MT7530_PPBV1_P(x)		(0x2014 + ((x) * 0x100))
-#define  G0_PORT_VID(x)			(((x) & 0xfff) << 0)
-#define  G0_PORT_VID_MASK		G0_PORT_VID(0xfff)
+#define  G0_PORT_VID_MASK		GENMASK(11, 0)
+#define  G0_PORT_VID(x)			FIELD_PREP(G0_PORT_VID_MASK, x)
 #define  G0_PORT_VID_DEF		G0_PORT_VID(0)
 
 /* Register for port MAC control register */
@@ -419,8 +424,8 @@ enum mt7530_vlan_port_acc_frm {
 #define  MT7531_DIS_CLR			BIT(31)
 
 #define MT7530_GMACCR			0x30e0
-#define  MAX_RX_JUMBO(x)		((x) << 2)
 #define  MAX_RX_JUMBO_MASK		GENMASK(5, 2)
+#define  MAX_RX_JUMBO(x)		FIELD_PREP(MAX_RX_JUMBO_MASK, x)
 #define  MAX_RX_PKT_LEN_MASK		GENMASK(1, 0)
 #define  MAX_RX_PKT_LEN_1522		0x0
 #define  MAX_RX_PKT_LEN_1536		0x1
@@ -506,16 +511,16 @@ enum mt7530_vlan_port_acc_frm {
 /* Register for PHY Indirect Access Control */
 #define MT7531_PHY_IAC			0x701C
 #define  MT7531_PHY_ACS_ST		BIT(31)
-#define  MT7531_MDIO_REG_ADDR_MASK	(0x1f << 25)
-#define  MT7531_MDIO_PHY_ADDR_MASK	(0x1f << 20)
-#define  MT7531_MDIO_CMD_MASK		(0x3 << 18)
-#define  MT7531_MDIO_ST_MASK		(0x3 << 16)
-#define  MT7531_MDIO_RW_DATA_MASK	(0xffff)
-#define  MT7531_MDIO_REG_ADDR(x)	(((x) & 0x1f) << 25)
-#define  MT7531_MDIO_DEV_ADDR(x)	(((x) & 0x1f) << 25)
-#define  MT7531_MDIO_PHY_ADDR(x)	(((x) & 0x1f) << 20)
-#define  MT7531_MDIO_CMD(x)		(((x) & 0x3) << 18)
-#define  MT7531_MDIO_ST(x)		(((x) & 0x3) << 16)
+#define  MT7531_MDIO_REG_ADDR_MASK	GENMASK(29, 25)
+#define  MT7531_MDIO_PHY_ADDR_MASK	GENMASK(24, 20)
+#define  MT7531_MDIO_CMD_MASK		GENMASK(19, 18)
+#define  MT7531_MDIO_ST_MASK		GENMASK(17, 16)
+#define  MT7531_MDIO_RW_DATA_MASK	GENMASK(15, 0)
+#define  MT7531_MDIO_REG_ADDR(x)	FIELD_PREP(MT7531_MDIO_REG_ADDR_MASK, x)
+#define  MT7531_MDIO_DEV_ADDR(x)	FIELD_PREP(MT7531_MDIO_REG_ADDR_MASK, x)
+#define  MT7531_MDIO_PHY_ADDR(x)	FIELD_PREP(MT7531_MDIO_PHY_ADDR_MASK, x)
+#define  MT7531_MDIO_CMD(x)		FIELD_PREP(MT7531_MDIO_CMD_MASK, x)
+#define  MT7531_MDIO_ST(x)		FIELD_PREP(MT7531_MDIO_ST_MASK, x)
 
 enum mt7531_phy_iac_cmd {
 	MT7531_MDIO_ADDR = 0,
@@ -543,14 +548,14 @@ enum mt7531_mdio_st {
 
 /* Register for RGMII clock phase */
 #define MT7531_CLKGEN_CTRL		0x7500
-#define  CLK_SKEW_OUT(x)		(((x) & 0x3) << 8)
 #define  CLK_SKEW_OUT_MASK		GENMASK(9, 8)
-#define  CLK_SKEW_IN(x)			(((x) & 0x3) << 6)
+#define  CLK_SKEW_OUT(x)		FIELD_PREP(CLK_SKEW_OUT_MASK, x)
 #define  CLK_SKEW_IN_MASK		GENMASK(7, 6)
+#define  CLK_SKEW_IN(x)			FIELD_PREP(CLK_SKEW_IN_MASK, x)
 #define  RXCLK_NO_DELAY			BIT(5)
 #define  TXCLK_NO_REVERSE		BIT(4)
-#define  GP_MODE(x)			(((x) & 0x3) << 1)
 #define  GP_MODE_MASK			GENMASK(2, 1)
+#define  GP_MODE(x)			FIELD_PREP(GP_MODE_MASK, x)
 #define  GP_CLK_EN			BIT(0)
 
 enum mt7531_gp_mode {
@@ -600,8 +605,10 @@ enum mt7531_xtal_fsel {
 #define  PAD_MCM_SMI_EN			BIT(0)
 
 #define MT7530_IO_DRV_CR		0x7810
-#define  P5_IO_CLK_DRV(x)		((x) & 0x3)
-#define  P5_IO_DATA_DRV(x)		(((x) & 0x3) << 4)
+#define  P5_IO_CLK_DRV_MASK		GENMASK(1, 0)
+#define  P5_IO_CLK_DRV(x)		FIELD_PREP(P5_IO_CLK_DRV_MASK, x)
+#define  P5_IO_DATA_DRV_MASK		GENMASK(5, 4)
+#define  P5_IO_DATA_DRV(x)		FIELD_PREP(P5_IO_DATA_DRV_MASK, x)
 
 #define MT7531_CHIP_REV			0x781C
 
@@ -611,15 +618,15 @@ enum mt7531_xtal_fsel {
 #define  SW_PLLGP			BIT(0)
 
 #define MT7530_P6ECR			0x7830
-#define  P6_INTF_MODE_MASK		0x3
-#define  P6_INTF_MODE(x)		((x) & 0x3)
+#define  P6_INTF_MODE_MASK		GENMASK(1, 0)
+#define  P6_INTF_MODE(x)		FIELD_PREP(P6_INTF_MODE_MASK, x)
 
 #define MT7531_PLLGP_CR0		0x78a8
 #define  RG_COREPLL_EN			BIT(22)
-#define  RG_COREPLL_POSDIV_S		23
-#define  RG_COREPLL_POSDIV_M		0x3800000
-#define  RG_COREPLL_SDM_PCW_S		1
-#define  RG_COREPLL_SDM_PCW_M		0x3ffffe
+#define  RG_COREPLL_POSDIV_MASK		GENMASK(25, 23)
+#define  RG_COREPLL_POSDIV(x)		FIELD_PREP(RG_COREPLL_POSDIV_MASK, x)
+#define  RG_COREPLL_SDM_PCW_MASK	GENMASK(21, 1)
+#define  RG_COREPLL_SDM_PCW(x)		FIELD_PREP(RG_COREPLL_SDM_PCW_MASK, x)
 #define  RG_COREPLL_SDM_PCW_CHG		BIT(0)
 
 /* Registers for RGMII and SGMII PLL clock */
@@ -630,10 +637,10 @@ enum mt7531_xtal_fsel {
 #define MT7530_TRGMII_RCK_CTRL		0x7a00
 #define  RX_RST				BIT(31)
 #define  RXC_DQSISEL			BIT(30)
-#define  DQSI1_TAP_MASK			(0x7f << 8)
-#define  DQSI0_TAP_MASK			0x7f
-#define  DQSI1_TAP(x)			(((x) & 0x7f) << 8)
-#define  DQSI0_TAP(x)			((x) & 0x7f)
+#define  DQSI1_TAP_MASK			GENMASK(14, 8)
+#define  DQSI0_TAP_MASK			GENMASK(6, 0)
+#define  DQSI1_TAP(x)			FIELD_PREP(DQSI1_TAP_MASK, x)
+#define  DQSI0_TAP(x)			FIELD_PREP(DQSI0_TAP_MASK, x)
 
 #define MT7530_TRGMII_RCK_RTT		0x7a04
 #define  DQS1_GATE			BIT(31)
@@ -642,8 +649,8 @@ enum mt7531_xtal_fsel {
 #define MT7530_TRGMII_RD(x)		(0x7a10 + (x) * 8)
 #define  BSLIP_EN			BIT(31)
 #define  EDGE_CHK			BIT(30)
-#define  RD_TAP_MASK			0x7f
-#define  RD_TAP(x)			((x) & 0x7f)
+#define  RD_TAP_MASK			GENMASK(6, 0)
+#define  RD_TAP(x)			FIELD_PREP(RD_TAP_MASK, x)
 
 #define MT7530_TRGMII_TXCTRL		0x7a40
 #define  TRAIN_TXEN			BIT(31)
@@ -651,18 +658,23 @@ enum mt7531_xtal_fsel {
 #define  TX_RST				BIT(28)
 
 #define MT7530_TRGMII_TD_ODT(i)		(0x7a54 + 8 * (i))
-#define  TD_DM_DRVP(x)			((x) & 0xf)
-#define  TD_DM_DRVN(x)			(((x) & 0xf) << 4)
+#define  TD_DM_DRVP_MASK		GENMASK(3, 0)
+#define  TD_DM_DRVP(x)			FIELD_PREP(TD_DM_DRVP_MASK, x)
+#define  TD_DM_DRVN_MASK		GENMASK(7, 4)
+#define  TD_DM_DRVN(x)			FIELD_PREP(TD_DM_DRVN_MASK, x)
 
 #define MT7530_TRGMII_TCK_CTRL		0x7a78
-#define  TCK_TAP(x)			(((x) & 0xf) << 8)
+#define  TCK_TAP_MASK			GENMASK(11, 8)
+#define  TCK_TAP(x)			FIELD_PREP(TCK_TAP_MASK, x)
 
 #define MT7530_P5RGMIIRXCR		0x7b00
 #define  CSR_RGMII_EDGE_ALIGN		BIT(8)
-#define  CSR_RGMII_RXC_0DEG_CFG(x)	((x) & 0xf)
+#define  CSR_RGMII_RXC_0DEG_CFG_MASK	GENMASK(3, 0)
+#define  CSR_RGMII_RXC_0DEG_CFG(x)	FIELD_PREP(CSR_RGMII_RXC_0DEG_CFG_MASK, x)
 
 #define MT7530_P5RGMIITXCR		0x7b04
-#define  CSR_RGMII_TXC_CFG(x)		((x) & 0x1f)
+#define  CSR_RGMII_TXC_CFG_MASK		GENMASK(4, 0)
+#define  CSR_RGMII_TXC_CFG(x)		FIELD_PREP(CSR_RGMII_TXC_CFG_MASK, x)
 
 /* Registers for GPIO mode */
 #define MT7531_GPIO_MODE0		0x7c0c
@@ -671,9 +683,9 @@ enum mt7531_xtal_fsel {
 
 #define MT7531_GPIO_MODE1		0x7c10
 #define  MT7531_GPIO11_RG_RXD2_MASK	GENMASK(15, 12)
-#define  MT7531_EXT_P_MDC_11		(2 << 12)
+#define  MT7531_EXT_P_MDC_11		FIELD_PREP(MT7531_GPIO11_RG_RXD2_MASK, 2)
 #define  MT7531_GPIO12_RG_RXD3_MASK	GENMASK(19, 16)
-#define  MT7531_EXT_P_MDIO_12		(2 << 16)
+#define  MT7531_EXT_P_MDIO_12		FIELD_PREP(MT7531_GPIO12_RG_RXD3_MASK, 2)
 
 #define MT753X_CPORT_SPTAG_CFG		0x7c10
 #define  CPORT_SW2FE_STAG_EN		BIT(1)
@@ -705,7 +717,7 @@ enum mt7531_xtal_fsel {
 #define MT7530_LED_GPIO_DATA		0x7d18
 
 #define MT7530_CREV			0x7ffc
-#define  CHIP_NAME_SHIFT		16
+#define  CHIP_NAME_MASK			GENMASK(31, 16)
 #define  MT7530_ID			0x7530
 
 #define MT7531_CREV			0x781C
@@ -717,10 +729,13 @@ enum mt7531_xtal_fsel {
 #define  RG_SYSPLL_EN_NORMAL		BIT(15)
 #define  RG_SYSPLL_VODEN		BIT(14)
 #define  RG_SYSPLL_LF			BIT(13)
-#define  RG_SYSPLL_RST_DLY(x)		(((x) & 0x3) << 12)
+#define  RG_SYSPLL_RST_DLY_MASK		GENMASK(13, 12)
+#define  RG_SYSPLL_RST_DLY(x)		FIELD_PREP(RG_SYSPLL_RST_DLY_MASK, x)
 #define  RG_SYSPLL_LVROD_EN		BIT(10)
-#define  RG_SYSPLL_PREDIV(x)		(((x) & 0x3) << 8)
-#define  RG_SYSPLL_POSDIV(x)		(((x) & 0x3) << 5)
+#define  RG_SYSPLL_PREDIV_MASK		GENMASK(9, 8)
+#define  RG_SYSPLL_PREDIV(x)		FIELD_PREP(RG_SYSPLL_PREDIV_MASK, x)
+#define  RG_SYSPLL_POSDIV_MASK		GENMASK(6, 5)
+#define  RG_SYSPLL_POSDIV(x)		FIELD_PREP(RG_SYSPLL_POSDIV_MASK, x)
 #define  RG_SYSPLL_FBKSEL		BIT(4)
 #define  RT_SYSPLL_EN_AFE_OLT		BIT(0)
 
@@ -732,38 +747,48 @@ enum mt7531_xtal_fsel {
 #define  MT7531_PHY_PLL_OFF		BIT(5)
 #define  MT7531_PHY_PLL_BYPASS_MODE	BIT(4)
 
-#define MT753X_CTRL_PHY_ADDR(addr)	((addr + 1) & 0x1f)
+#define MT753X_CTRL_PHY_ADDR(addr)	(((addr) + 1) & (PHY_MAX_ADDR - 1))
 
 #define CORE_PLL_GROUP5			0x404
-#define  RG_LCDDS_PCW_NCPO1(x)		((x) & 0xffff)
+#define  RG_LCDDS_PCW_NCPO1_MASK	GENMASK(15, 0)
+#define  RG_LCDDS_PCW_NCPO1(x)		FIELD_PREP(RG_LCDDS_PCW_NCPO1_MASK, x)
 
 #define CORE_PLL_GROUP6			0x405
-#define  RG_LCDDS_PCW_NCPO0(x)		((x) & 0xffff)
+#define  RG_LCDDS_PCW_NCPO0_MASK	GENMASK(15, 0)
+#define  RG_LCDDS_PCW_NCPO0(x)		FIELD_PREP(RG_LCDDS_PCW_NCPO0_MASK, x)
 
 #define CORE_PLL_GROUP7			0x406
 #define  RG_LCDDS_PWDB			BIT(15)
 #define  RG_LCDDS_ISO_EN		BIT(13)
-#define  RG_LCCDS_C(x)			(((x) & 0x7) << 4)
+#define  RG_LCCDS_C_MASK		GENMASK(6, 4)
+#define  RG_LCCDS_C(x)			FIELD_PREP(RG_LCCDS_C_MASK, x)
 #define  RG_LCDDS_PCW_NCPO_CHG		BIT(3)
 
 #define CORE_PLL_GROUP10		0x409
-#define  RG_LCDDS_SSC_DELTA(x)		((x) & 0xfff)
+#define  RG_LCDDS_SSC_DELTA_MASK	GENMASK(11, 0)
+#define  RG_LCDDS_SSC_DELTA(x)		FIELD_PREP(RG_LCDDS_SSC_DELTA_MASK, x)
 
 #define CORE_PLL_GROUP11		0x40a
-#define  RG_LCDDS_SSC_DELTA1(x)		((x) & 0xfff)
+#define  RG_LCDDS_SSC_DELTA1_MASK	GENMASK(11, 0)
+#define  RG_LCDDS_SSC_DELTA1(x)		FIELD_PREP(RG_LCDDS_SSC_DELTA1_MASK, x)
 
 #define CORE_GSWPLL_GRP1		0x40d
-#define  RG_GSWPLL_PREDIV(x)		(((x) & 0x3) << 14)
-#define  RG_GSWPLL_POSDIV_200M(x)	(((x) & 0x3) << 12)
+#define  RG_GSWPLL_PREDIV_MASK		GENMASK(15, 14)
+#define  RG_GSWPLL_PREDIV(x)		FIELD_PREP(RG_GSWPLL_PREDIV_MASK, x)
+#define  RG_GSWPLL_POSDIV_200M_MASK	GENMASK(13, 12)
+#define  RG_GSWPLL_POSDIV_200M(x)	FIELD_PREP(RG_GSWPLL_POSDIV_200M_MASK, x)
 #define  RG_GSWPLL_EN_PRE		BIT(11)
 #define  RG_GSWPLL_FBKSEL		BIT(10)
 #define  RG_GSWPLL_BP			BIT(9)
 #define  RG_GSWPLL_BR			BIT(8)
-#define  RG_GSWPLL_FBKDIV_200M(x)	((x) & 0xff)
+#define  RG_GSWPLL_FBKDIV_200M_MASK	GENMASK(7, 0)
+#define  RG_GSWPLL_FBKDIV_200M(x)	FIELD_PREP(RG_GSWPLL_FBKDIV_200M_MASK, x)
 
 #define CORE_GSWPLL_GRP2		0x40e
-#define  RG_GSWPLL_POSDIV_500M(x)	(((x) & 0x3) << 8)
-#define  RG_GSWPLL_FBKDIV_500M(x)	((x) & 0xff)
+#define  RG_GSWPLL_POSDIV_500M_MASK	GENMASK(9, 8)
+#define  RG_GSWPLL_POSDIV_500M(x)	FIELD_PREP(RG_GSWPLL_POSDIV_500M_MASK, x)
+#define  RG_GSWPLL_FBKDIV_500M_MASK	GENMASK(7, 0)
+#define  RG_GSWPLL_FBKDIV_500M(x)	FIELD_PREP(RG_GSWPLL_FBKDIV_500M_MASK, x)
 
 #define CORE_TRGMII_GSW_CLK_CG		0x410
 #define  REG_GSWCK_EN			BIT(0)
@@ -933,18 +958,6 @@ struct mt7530_hw_stats {
 	u8		sizeof_stat;
 };
 
-struct mt7530_dummy_poll {
-	struct mt7530_priv *priv;
-	u32 reg;
-};
-
-static inline void INIT_MT7530_DUMMY_POLL(struct mt7530_dummy_poll *p,
-					  struct mt7530_priv *priv, u32 reg)
-{
-	p->priv = priv;
-	p->reg = reg;
-}
-
 int mt7530_probe_common(struct mt7530_priv *priv);
 void mt7530_remove_common(struct mt7530_priv *priv);
 
diff --git a/drivers/net/dsa/mv88e6060.c b/drivers/net/dsa/mv88e6060.c
index 9c8ac14cd4f5d3..3bb485a9042990 100644
--- a/drivers/net/dsa/mv88e6060.c
+++ b/drivers/net/dsa/mv88e6060.c
@@ -362,8 +362,9 @@ static const struct of_device_id mv88e6060_of_match[] = {
 	{
 		.compatible = "marvell,mv88e6060",
 	},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, mv88e6060_of_match);
 
 static struct mdio_driver mv88e6060_driver = {
 	.probe	= mv88e6060_probe,
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 7f68a0c5580265..2a975ecd79c743 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -2457,7 +2457,7 @@ static int mv88e6xxx_get_rxnfc(struct dsa_switch *ds, int port,
 	case ETHTOOL_GRXCLSRULE:
 		err = -ENOENT;
 		policy = idr_find(&chip->policies, fs->location);
-		if (policy) {
+		if (policy && policy->port == port) {
 			memcpy(fs, &policy->fs, sizeof(*fs));
 			err = 0;
 		}
@@ -2503,8 +2503,9 @@ static int mv88e6xxx_set_rxnfc(struct dsa_switch *ds, int port,
 		break;
 	case ETHTOOL_SRXCLSRLDEL:
 		err = -ENOENT;
-		policy = idr_remove(&chip->policies, fs->location);
-		if (policy) {
+		policy = idr_find(&chip->policies, fs->location);
+		if (policy && policy->port == port) {
+			idr_remove(&chip->policies, fs->location);
 			policy->action = MV88E6XXX_POLICY_ACTION_NORMAL;
 			err = mv88e6xxx_policy_apply(chip, port, policy);
 			devm_kfree(chip->dev, policy);
@@ -5639,6 +5640,68 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = {
 	.pcs_ops = &mv88e6390_pcs_ops,
 };
 
+static const struct mv88e6xxx_ops mv88e6191x_ops = {
+	/* MV88E6XXX_FAMILY_6393 without AVB and PTP: 6191X and 6193X */
+	.irl_init_all = mv88e6390_g2_irl_init_all,
+	.get_eeprom = mv88e6xxx_g2_get_eeprom8,
+	.set_eeprom = mv88e6xxx_g2_set_eeprom8,
+	.set_switch_mac = mv88e6xxx_g2_set_switch_mac,
+	.phy_read = mv88e6xxx_g2_smi_phy_read_c22,
+	.phy_write = mv88e6xxx_g2_smi_phy_write_c22,
+	.phy_read_c45 = mv88e6xxx_g2_smi_phy_read_c45,
+	.phy_write_c45 = mv88e6xxx_g2_smi_phy_write_c45,
+	.port_set_link = mv88e6xxx_port_set_link,
+	.port_sync_link = mv88e6xxx_port_sync_link,
+	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
+	.port_set_speed_duplex = mv88e6393x_port_set_speed_duplex,
+	.port_tag_remap = mv88e6390_port_tag_remap,
+	.port_set_policy = mv88e6393x_port_set_policy,
+	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
+	.port_set_ucast_flood = mv88e6352_port_set_ucast_flood,
+	.port_set_mcast_flood = mv88e6352_port_set_mcast_flood,
+	.port_set_ether_type = mv88e6393x_port_set_ether_type,
+	.port_set_jumbo_size = mv88e6165_port_set_jumbo_size,
+	.port_egress_rate_limiting = mv88e6097_port_egress_rate_limiting,
+	.port_pause_limit = mv88e6390_port_pause_limit,
+	.port_disable_learn_limit = mv88e6xxx_port_disable_learn_limit,
+	.port_disable_pri_override = mv88e6xxx_port_disable_pri_override,
+	.port_get_cmode = mv88e6352_port_get_cmode,
+	.port_set_cmode = mv88e6393x_port_set_cmode,
+	.port_setup_message_port = mv88e6xxx_setup_message_port,
+	.port_set_upstream_port = mv88e6393x_port_set_upstream_port,
+	.port_enable_tcam = mv88e6xxx_port_enable_tcam,
+	.stats_snapshot = mv88e6390_g1_stats_snapshot,
+	.stats_set_histogram = mv88e6390_g1_stats_set_histogram,
+	.stats_get_sset_count = mv88e6320_stats_get_sset_count,
+	.stats_get_strings = mv88e6320_stats_get_strings,
+	.stats_get_stat = mv88e6390_stats_get_stat,
+	/* .set_cpu_port is missing because this family does not support a global
+	 * CPU port, only per port CPU port which is set via
+	 * .port_set_upstream_port method.
+	 */
+	.set_egress_port = mv88e6393x_set_egress_port,
+	.watchdog_ops = &mv88e6393x_watchdog_ops,
+	.mgmt_rsvd2cpu = mv88e6393x_port_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
+	.hardware_reset_pre = mv88e6xxx_g2_eeprom_wait,
+	.hardware_reset_post = mv88e6xxx_g2_eeprom_wait,
+	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6390_g1_rmu_disable,
+	.atu_get_hash = mv88e6165_g1_atu_get_hash,
+	.atu_set_hash = mv88e6165_g1_atu_set_hash,
+	.vtu_getnext = mv88e6390_g1_vtu_getnext,
+	.vtu_loadpurge = mv88e6390_g1_vtu_loadpurge,
+	.stu_getnext = mv88e6390_g1_stu_getnext,
+	.stu_loadpurge = mv88e6390_g1_stu_loadpurge,
+	.serdes_get_lane = mv88e6393x_serdes_get_lane,
+	.serdes_irq_mapping = mv88e6390_serdes_irq_mapping,
+	/* TODO: serdes stats */
+	.gpio_ops = &mv88e6352_gpio_ops,
+	.phylink_get_caps = mv88e6393x_phylink_get_caps,
+	.pcs_ops = &mv88e6393x_pcs_ops,
+	.tcam_ops = &mv88e6393_tcam_ops,
+};
+
 static const struct mv88e6xxx_ops mv88e6393x_ops = {
 	/* MV88E6XXX_FAMILY_6393 */
 	.irl_init_all = mv88e6390_g2_irl_init_all,
@@ -6163,8 +6226,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.atu_move_port_mask = 0x1f,
 		.pvt = true,
 		.multi_chip = true,
-		.ptp_support = true,
-		.ops = &mv88e6393x_ops,
+		.ops = &mv88e6191x_ops,
 	},
 
 	[MV88E6193X] = {
@@ -6190,8 +6252,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.atu_move_port_mask = 0x1f,
 		.pvt = true,
 		.multi_chip = true,
-		.ptp_support = true,
-		.ops = &mv88e6393x_ops,
+		.ops = &mv88e6191x_ops,
 	},
 
 	[MV88E6220] = {
@@ -7521,7 +7582,7 @@ static const struct of_device_id mv88e6xxx_of_match[] = {
 		.compatible = "marvell,mv88e6250",
 		.data = &mv88e6xxx_table[MV88E6250],
 	},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 
 MODULE_DEVICE_TABLE(of, mv88e6xxx_of_match);
diff --git a/drivers/net/dsa/qca/qca8k-8xxx.c b/drivers/net/dsa/qca/qca8k-8xxx.c
index 4c928983b86231..60f2a615a5cc2f 100644
--- a/drivers/net/dsa/qca/qca8k-8xxx.c
+++ b/drivers/net/dsa/qca/qca8k-8xxx.c
@@ -2215,7 +2215,7 @@ static const struct of_device_id qca8k_of_match[] = {
 	{ .compatible = "qca,qca8328", .data = &qca8328 },
 	{ .compatible = "qca,qca8334", .data = &qca833x },
 	{ .compatible = "qca,qca8337", .data = &qca833x },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 
 static struct mdio_driver qca8kmdio_driver = {
diff --git a/drivers/net/dsa/qca/qca8k-common.c b/drivers/net/dsa/qca/qca8k-common.c
index 13005f10edb7d6..ab978cc8367f73 100644
--- a/drivers/net/dsa/qca/qca8k-common.c
+++ b/drivers/net/dsa/qca/qca8k-common.c
@@ -471,10 +471,7 @@ void qca8k_port_set_status(struct qca8k_priv *priv, int port, int enable)
 	if (port > 0 && port < 6)
 		mask |= QCA8K_PORT_STATUS_LINK_AUTO;
 
-	if (enable)
-		regmap_set_bits(priv->regmap, QCA8K_REG_PORT_STATUS(port), mask);
-	else
-		regmap_clear_bits(priv->regmap, QCA8K_REG_PORT_STATUS(port), mask);
+	regmap_assign_bits(priv->regmap, QCA8K_REG_PORT_STATUS(port), mask, enable);
 }
 
 void qca8k_get_strings(struct dsa_switch *ds, int port, u32 stringset,
@@ -562,14 +559,8 @@ static int qca8k_port_configure_learning(struct dsa_switch *ds, int port,
 {
 	struct qca8k_priv *priv = ds->priv;
 
-	if (learning)
-		return regmap_set_bits(priv->regmap,
-				       QCA8K_PORT_LOOKUP_CTRL(port),
-				       QCA8K_PORT_LOOKUP_LEARN);
-	else
-		return regmap_clear_bits(priv->regmap,
-					 QCA8K_PORT_LOOKUP_CTRL(port),
-					 QCA8K_PORT_LOOKUP_LEARN);
+	return regmap_assign_bits(priv->regmap, QCA8K_PORT_LOOKUP_CTRL(port),
+				  QCA8K_PORT_LOOKUP_LEARN, learning);
 }
 
 void qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
diff --git a/drivers/net/dsa/realtek/realtek.h b/drivers/net/dsa/realtek/realtek.h
index 6e0148cee8d87a..1c0ca6daf54544 100644
--- a/drivers/net/dsa/realtek/realtek.h
+++ b/drivers/net/dsa/realtek/realtek.h
@@ -156,6 +156,10 @@ struct realtek_ops {
 	int	(*phy_read)(struct realtek_priv *priv, int phy, int regnum);
 	int	(*phy_write)(struct realtek_priv *priv, int phy, int regnum,
 			     u16 val);
+	int	(*phy_read_c45)(struct realtek_priv *priv, int phy, int devad,
+				int regnum);
+	int	(*phy_write_c45)(struct realtek_priv *priv, int phy, int devad,
+				 int regnum, u16 val);
 };
 
 struct realtek_variant {
@@ -166,6 +170,9 @@ struct realtek_variant {
 	u8 cmd_read;
 	u8 cmd_write;
 	size_t chip_data_sz;
+	/* Regulator supplies to enable at probe, or NULL */
+	const char *const *supplies;
+	int num_supplies;
 };
 
 /* RTL8366 library helpers */
diff --git a/drivers/net/dsa/realtek/rtl8365mb_main.c b/drivers/net/dsa/realtek/rtl8365mb_main.c
index 728231d8f94cab..87ec924a061186 100644
--- a/drivers/net/dsa/realtek/rtl8365mb_main.c
+++ b/drivers/net/dsa/realtek/rtl8365mb_main.c
@@ -101,6 +101,7 @@
 #include <linux/regmap.h>
 #include <linux/if_bridge.h>
 #include <linux/if_vlan.h>
+#include <linux/mdio.h>
 #include <linux/phylink.h>
 
 #include "realtek.h"
@@ -199,9 +200,21 @@
 #define   RTL8365MB_GPHY_OCP_MSB_0_CFG_CPU_OCPADR_MASK	0x0FC0
 #define RTL8365MB_PHY_OCP_ADDR_PREFIX_MASK		0xFC00
 
+/* The full 16-bit OCP address is split across two registers: bits [15:10] are
+ * the prefix (RTL8365MB_PHY_OCP_ADDR_PREFIX_MASK above), and bits [9:1] go into
+ * the ADDRESS register as two fields, [5:1] and [9:6]. Bit 0 is always 0 - PHY
+ * OCP registers are 2-byte aligned.
+ */
+#define RTL8365MB_PHY_OCP_ADDR_5_1_MASK			GENMASK(5, 1)
+#define RTL8365MB_PHY_OCP_ADDR_9_6_MASK			GENMASK(9, 6)
+
 /* The PHY OCP addresses of PHY registers 0~31 start here */
 #define RTL8365MB_PHY_OCP_ADDR_PHYREG_BASE		0xA400
 
+#define RTL8365MB_PHY_OCP_ADDR_EEE_ABLE			0xA5C4
+#define RTL8365MB_PHY_OCP_ADDR_EEE_ADV			0xA5D0
+#define RTL8365MB_PHY_OCP_ADDR_EEE_LPABLE		0xA5D2
+
 /* External interface port mode values - used in DIGITAL_INTERFACE_SELECT */
 #define RTL8365MB_EXT_PORT_MODE_DISABLE		0
 #define RTL8365MB_EXT_PORT_MODE_RGMII		1
@@ -871,6 +884,8 @@ static int rtl8365mb_phy_poll_busy(struct realtek_priv *priv)
 static int rtl8365mb_phy_ocp_prepare(struct realtek_priv *priv, int phy,
 				     u32 ocp_addr)
 {
+	u16 ocp_addr_lo = FIELD_GET(RTL8365MB_PHY_OCP_ADDR_5_1_MASK, ocp_addr);
+	u16 ocp_addr_hi = FIELD_GET(RTL8365MB_PHY_OCP_ADDR_9_6_MASK, ocp_addr);
 	u32 val;
 	int ret;
 
@@ -887,9 +902,9 @@ static int rtl8365mb_phy_ocp_prepare(struct realtek_priv *priv, int phy,
 	val = RTL8365MB_PHY_BASE;
 	val |= FIELD_PREP(RTL8365MB_INDIRECT_ACCESS_ADDRESS_PHYNUM_MASK, phy);
 	val |= FIELD_PREP(RTL8365MB_INDIRECT_ACCESS_ADDRESS_OCPADR_5_1_MASK,
-			  ocp_addr >> 1);
+			  ocp_addr_lo);
 	val |= FIELD_PREP(RTL8365MB_INDIRECT_ACCESS_ADDRESS_OCPADR_9_6_MASK,
-			  ocp_addr >> 6);
+			  ocp_addr_hi);
 	ret = regmap_write(priv->map_nolock,
 			   RTL8365MB_INDIRECT_ACCESS_ADDRESS_REG, val);
 	if (ret)
@@ -1040,6 +1055,66 @@ static int rtl8365mb_phy_write(struct realtek_priv *priv, int phy, int regnum,
 	return 0;
 }
 
+static int rtl8365mb_phy_read_c45(struct realtek_priv *priv, int phy, int devad,
+				  int regnum)
+{
+	u32 ocp_addr;
+	u16 val;
+	int ret;
+
+	if (phy > RTL8365MB_PHYADDRMAX)
+		return -EINVAL;
+
+	if (devad == MDIO_MMD_PCS && regnum == MDIO_PCS_EEE_ABLE)
+		ocp_addr = RTL8365MB_PHY_OCP_ADDR_EEE_ABLE;
+	else if (devad == MDIO_MMD_AN && regnum == MDIO_AN_EEE_ADV)
+		ocp_addr = RTL8365MB_PHY_OCP_ADDR_EEE_ADV;
+	else if (devad == MDIO_MMD_AN && regnum == MDIO_AN_EEE_LPABLE)
+		ocp_addr = RTL8365MB_PHY_OCP_ADDR_EEE_LPABLE;
+	else
+		/* Only the EEE registers are mapped; others read as 0, as the
+		 * hardware does, so the generic MMD code is not tripped up by
+		 * an error.
+		 */
+		return 0;
+
+	ret = rtl8365mb_phy_ocp_read(priv, phy, ocp_addr, &val);
+	if (ret) {
+		dev_err(priv->dev,
+			"failed to read PHY%d OCP %04x, ret %d\n", phy, ocp_addr,
+			ret);
+		return ret;
+	}
+
+	return val;
+}
+
+static int rtl8365mb_phy_write_c45(struct realtek_priv *priv, int phy,
+				   int devad, int regnum, u16 val)
+{
+	int ret;
+
+	if (phy > RTL8365MB_PHYADDRMAX)
+		return -EINVAL;
+
+	/* Only the EEE advertisement register is writable; writes to other
+	 * registers are ignored, as the hardware does.
+	 */
+	if (devad != MDIO_MMD_AN || regnum != MDIO_AN_EEE_ADV)
+		return 0;
+
+	ret = rtl8365mb_phy_ocp_write(priv, phy, RTL8365MB_PHY_OCP_ADDR_EEE_ADV,
+				      val);
+	if (ret) {
+		dev_err(priv->dev,
+			"failed to write PHY%d OCP %04x, ret %d\n", phy,
+			RTL8365MB_PHY_OCP_ADDR_EEE_ADV, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 static const struct rtl8365mb_extint *
 rtl8365mb_get_port_extint(struct realtek_priv *priv, int port)
 {
@@ -1639,6 +1714,14 @@ static void rtl8365mb_phylink_get_caps(struct dsa_switch *ds, int port,
 		 */
 		__set_bit(PHY_INTERFACE_MODE_GMII,
 			  config->supported_interfaces);
+
+		/* Integrated PHYs support EEE at 100M/1G; the hardware manages
+		 * LPI on its own, so just advertise LPI awareness to phylink.
+		 */
+		memcpy(config->lpi_interfaces, config->supported_interfaces,
+		       sizeof(config->lpi_interfaces));
+		config->lpi_capabilities = MAC_100FD | MAC_1000FD;
+		config->eee_enabled_default = true;
 		return;
 	}
 
@@ -3275,18 +3358,66 @@ static int rtl8365mb_detect(struct realtek_priv *priv)
 	return 0;
 }
 
+static int rtl8365mb_phylink_mac_enable_tx_lpi(struct phylink_config *config,
+					       u32 timer, bool tx_clock_stop)
+{
+	/* The hardware manages LPI itself; there is no MAC-level LPI control.
+	 * This callback only signals LPI awareness to phylink.
+	 */
+	return 0;
+}
+
+static void rtl8365mb_phylink_mac_disable_tx_lpi(struct phylink_config *config)
+{
+}
+
 static const struct phylink_mac_ops rtl8365mb_phylink_mac_ops = {
 	.mac_select_pcs = rtl8365mb_phylink_mac_select_pcs,
 	.mac_config = rtl8365mb_phylink_mac_config,
 	.mac_link_down = rtl8365mb_phylink_mac_link_down,
 	.mac_link_up = rtl8365mb_phylink_mac_link_up,
+	.mac_enable_tx_lpi = rtl8365mb_phylink_mac_enable_tx_lpi,
+	.mac_disable_tx_lpi = rtl8365mb_phylink_mac_disable_tx_lpi,
 };
 
+static bool rtl8365mb_support_eee(struct dsa_switch *ds, int port)
+{
+	/* Only integrated-PHY ports support EEE, not the external RGMII ports. */
+	return !rtl8365mb_get_port_extint(ds->priv, port);
+}
+
+static int rtl8365mb_set_mac_eee(struct dsa_switch *ds, int port,
+				 struct ethtool_keee *e)
+{
+	struct realtek_priv *priv = ds->priv;
+
+	/* The only LPI timing control (tx_lpi_timer) is a single global
+	 * per-speed register shared by all ports, so it cannot be set from this
+	 * per-port callback; leave it at its reset default. Per-port EEE is
+	 * driven through the PHY advertisement. Reject the per-port TX LPI knobs
+	 * rather than silently ignoring them.
+	 */
+	if (!e->tx_lpi_enabled) {
+		dev_err(priv->dev, "disabling EEE TX LPI is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (e->tx_lpi_timer) {
+		dev_err(priv->dev,
+			"setting the EEE TX LPI timer is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
 static const struct dsa_switch_ops rtl8365mb_switch_ops = {
 	.get_tag_protocol = rtl8365mb_get_tag_protocol,
 	.change_tag_protocol = rtl8365mb_change_tag_protocol,
 	.setup = rtl8365mb_setup,
 	.teardown = rtl8365mb_teardown,
+	.support_eee = rtl8365mb_support_eee,
+	.set_mac_eee = rtl8365mb_set_mac_eee,
 	.phylink_get_caps = rtl8365mb_phylink_get_caps,
 	.port_bridge_join = rtl83xx_port_bridge_join,
 	.port_bridge_leave = rtl83xx_port_bridge_leave,
@@ -3332,6 +3463,12 @@ static const struct realtek_ops rtl8365mb_ops = {
 	.l2_flush = rtl8365mb_l2_flush,
 	.phy_read = rtl8365mb_phy_read,
 	.phy_write = rtl8365mb_phy_write,
+	.phy_read_c45 = rtl8365mb_phy_read_c45,
+	.phy_write_c45 = rtl8365mb_phy_write_c45,
+};
+
+static const char *const rtl8365mb_supplies[] = {
+	"avddh", "avddl", "dvddio", "dvddio1", "dvddl", "pllvddl",
 };
 
 const struct realtek_variant rtl8365mb_variant = {
@@ -3342,11 +3479,13 @@ const struct realtek_variant rtl8365mb_variant = {
 	.cmd_read = 0xb9,
 	.cmd_write = 0xb8,
 	.chip_data_sz = sizeof(struct rtl8365mb),
+	.supplies = rtl8365mb_supplies,
+	.num_supplies = ARRAY_SIZE(rtl8365mb_supplies),
 };
 
 static const struct of_device_id rtl8365mb_of_match[] = {
 	{ .compatible = "realtek,rtl8365mb", .data = &rtl8365mb_variant, },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, rtl8365mb_of_match);
 
diff --git a/drivers/net/dsa/realtek/rtl8366rb.c b/drivers/net/dsa/realtek/rtl8366rb.c
index f11831b66de887..093a06c599d48b 100644
--- a/drivers/net/dsa/realtek/rtl8366rb.c
+++ b/drivers/net/dsa/realtek/rtl8366rb.c
@@ -1839,7 +1839,7 @@ const struct realtek_variant rtl8366rb_variant = {
 
 static const struct of_device_id rtl8366rb_of_match[] = {
 	{ .compatible = "realtek,rtl8366rb", .data = &rtl8366rb_variant, },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, rtl8366rb_of_match);
 
diff --git a/drivers/net/dsa/realtek/rtl83xx.c b/drivers/net/dsa/realtek/rtl83xx.c
index a96beea72ef5d0..ad42c05d0a88dc 100644
--- a/drivers/net/dsa/realtek/rtl83xx.c
+++ b/drivers/net/dsa/realtek/rtl83xx.c
@@ -2,6 +2,7 @@
 
 #include <linux/module.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <linux/of_mdio.h>
 #include <linux/if_bridge.h>
 #include <linux/etherdevice.h>
@@ -61,6 +62,22 @@ static int rtl83xx_user_mdio_write(struct mii_bus *bus, int addr, int regnum,
 	return priv->ops->phy_write(priv, addr, regnum, val);
 }
 
+static int rtl83xx_user_mdio_read_c45(struct mii_bus *bus, int addr, int devad,
+				      int regnum)
+{
+	struct realtek_priv *priv = bus->priv;
+
+	return priv->ops->phy_read_c45(priv, addr, devad, regnum);
+}
+
+static int rtl83xx_user_mdio_write_c45(struct mii_bus *bus, int addr, int devad,
+				       int regnum, u16 val)
+{
+	struct realtek_priv *priv = bus->priv;
+
+	return priv->ops->phy_write_c45(priv, addr, devad, regnum, val);
+}
+
 /**
  * rtl83xx_setup_user_mdio() - register the user mii bus driver
  * @ds: DSA switch associated with this user_mii_bus
@@ -94,6 +111,10 @@ int rtl83xx_setup_user_mdio(struct dsa_switch *ds)
 	bus->name = "Realtek user MII";
 	bus->read = rtl83xx_user_mdio_read;
 	bus->write = rtl83xx_user_mdio_write;
+	if (priv->ops->phy_read_c45)
+		bus->read_c45 = rtl83xx_user_mdio_read_c45;
+	if (priv->ops->phy_write_c45)
+		bus->write_c45 = rtl83xx_user_mdio_write_c45;
 	snprintf(bus->id, MII_BUS_ID_SIZE, "%s:user_mii", dev_name(priv->dev));
 	bus->parent = priv->dev;
 
@@ -195,17 +216,24 @@ rtl83xx_probe(struct device *dev,
 	priv->leds_disabled = of_property_read_bool(dev->of_node,
 						    "realtek,disable-leds");
 
-	/* TODO: if power is software controlled, set up any regulators here */
+	/* Enable the supplies before the reset line is requested and driven,
+	 * so the chip is powered before its pins are driven.
+	 */
+	if (var->num_supplies) {
+		ret = devm_regulator_bulk_get_enable(dev, var->num_supplies,
+						     var->supplies);
+		if (ret)
+			return dev_err_ptr_probe(dev, ret, "failed to enable supplies\n");
+	}
+
 	priv->reset_ctl = devm_reset_control_get_optional(dev, NULL);
 	if (IS_ERR(priv->reset_ctl))
 		return dev_err_cast_probe(dev, priv->reset_ctl,
 					  "failed to get reset control\n");
 
 	priv->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(priv->reset)) {
-		dev_err(dev, "failed to get RESET GPIO\n");
-		return ERR_CAST(priv->reset);
-	}
+	if (IS_ERR(priv->reset))
+		return dev_err_cast_probe(dev, priv->reset, "failed to get RESET GPIO\n");
 
 	dev_set_drvdata(dev, priv);
 
@@ -216,6 +244,11 @@ rtl83xx_probe(struct device *dev,
 		rtl83xx_reset_deassert(priv);
 		msleep(REALTEK_HW_START_DELAY);
 		dev_dbg(dev, "deasserted RESET\n");
+	} else if (var->num_supplies) {
+		/* Powered but no reset line: still wait for the chip to boot
+		 * before the first register access.
+		 */
+		msleep(REALTEK_HW_START_DELAY);
 	}
 
 	return priv;
diff --git a/drivers/net/dsa/rzn1_a5psw.c b/drivers/net/dsa/rzn1_a5psw.c
index 4d857e3be10b95..85ff1fd2bbfbaa 100644
--- a/drivers/net/dsa/rzn1_a5psw.c
+++ b/drivers/net/dsa/rzn1_a5psw.c
@@ -1299,7 +1299,7 @@ static void a5psw_shutdown(struct platform_device *pdev)
 
 static const struct of_device_id a5psw_of_mtable[] = {
 	{ .compatible = "renesas,rzn1-a5psw", },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, a5psw_of_mtable);
 
diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c
index 1653e83d91230e..8984ed55ad594f 100644
--- a/drivers/net/dsa/sja1105/sja1105_main.c
+++ b/drivers/net/dsa/sja1105/sja1105_main.c
@@ -3374,7 +3374,7 @@ static const struct of_device_id sja1105_dt_ids[] = {
 	{ .compatible = "nxp,sja1110b", .data = &sja1110b_info },
 	{ .compatible = "nxp,sja1110c", .data = &sja1110c_info },
 	{ .compatible = "nxp,sja1110d", .data = &sja1110d_info },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sja1105_dt_ids);
 
diff --git a/drivers/net/dsa/vitesse-vsc73xx-spi.c b/drivers/net/dsa/vitesse-vsc73xx-spi.c
index 85b9a0f51dd894..7b784320c41e7a 100644
--- a/drivers/net/dsa/vitesse-vsc73xx-spi.c
+++ b/drivers/net/dsa/vitesse-vsc73xx-spi.c
@@ -153,10 +153,15 @@ static int vsc73xx_spi_probe(struct spi_device *spi)
 	ret = spi_setup(spi);
 	if (ret < 0) {
 		dev_err(dev, "spi setup failed.\n");
+		spi_dev_put(vsc_spi->spi);
 		return ret;
 	}
 
-	return vsc73xx_probe(&vsc_spi->vsc);
+	ret = vsc73xx_probe(&vsc_spi->vsc);
+	if (ret)
+		spi_dev_put(vsc_spi->spi);
+
+	return ret;
 }
 
 static void vsc73xx_spi_remove(struct spi_device *spi)
@@ -167,6 +172,8 @@ static void vsc73xx_spi_remove(struct spi_device *spi)
 		return;
 
 	vsc73xx_remove(&vsc_spi->vsc);
+
+	spi_dev_put(vsc_spi->spi);
 }
 
 static void vsc73xx_spi_shutdown(struct spi_device *spi)
diff --git a/drivers/net/dsa/yt921x.c b/drivers/net/dsa/yt921x.c
deleted file mode 100644
index 159b16606f6ca4..00000000000000
--- a/drivers/net/dsa/yt921x.c
+++ /dev/null
@@ -1,4982 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for Motorcomm YT921x Switch
- *
- * Should work on YT9213/YT9214/YT9215/YT9218, but only tested on YT9215+SGMII,
- * be sure to do your own checks before porting to another chip.
- *
- * Copyright (c) 2025 David Yang
- */
-
-#include <linux/dcbnl.h>
-#include <linux/etherdevice.h>
-#include <linux/if_bridge.h>
-#include <linux/if_hsr.h>
-#include <linux/if_vlan.h>
-#include <linux/iopoll.h>
-#include <linux/mdio.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_mdio.h>
-#include <linux/of_net.h>
-#include <linux/sort.h>
-
-#include <net/dsa.h>
-#include <net/dscp.h>
-#include <net/ieee8021q.h>
-#include <net/pkt_cls.h>
-
-#include "yt921x.h"
-
-struct yt921x_mib_desc {
-	unsigned int size;
-	unsigned int offset;
-	const char *name;
-};
-
-#define MIB_DESC(_size, _offset, _name) \
-	{_size, _offset, _name}
-
-/* Must agree with yt921x_mib
- *
- * Unstructured fields (name != NULL) will appear in get_ethtool_stats(),
- * structured go to their *_stats() methods, but we need their sizes and offsets
- * to perform 32bit MIB overflow wraparound.
- */
-static const struct yt921x_mib_desc yt921x_mib_descs[] = {
-	MIB_DESC(1, YT921X_MIB_DATA_RX_BROADCAST, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PAUSE, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_MULTICAST, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_CRC_ERR, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_RX_ALIGN_ERR, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_UNDERSIZE_ERR, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_FRAG_ERR, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_64, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_65_TO_127, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_128_TO_255, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_256_TO_511, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_512_TO_1023, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_1024_TO_1518, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_PKT_SZ_1519_TO_MAX, NULL),
-	MIB_DESC(2, YT921X_MIB_DATA_RX_GOOD_BYTES, NULL),
-
-	MIB_DESC(2, YT921X_MIB_DATA_RX_BAD_BYTES, "RxBadBytes"),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_OVERSIZE_ERR, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_RX_DROPPED, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_BROADCAST, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PAUSE, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_MULTICAST, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_TX_UNDERSIZE_ERR, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_64, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_65_TO_127, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_128_TO_255, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_256_TO_511, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_512_TO_1023, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_1024_TO_1518, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT_SZ_1519_TO_MAX, NULL),
-
-	MIB_DESC(2, YT921X_MIB_DATA_TX_GOOD_BYTES, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_COLLISION, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_TX_EXCESSIVE_COLLISION, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_MULTIPLE_COLLISION, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_SINGLE_COLLISION, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_PKT, NULL),
-
-	MIB_DESC(1, YT921X_MIB_DATA_TX_DEFERRED, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_LATE_COLLISION, NULL),
-	MIB_DESC(1, YT921X_MIB_DATA_RX_OAM, "RxOAM"),
-	MIB_DESC(1, YT921X_MIB_DATA_TX_OAM, "TxOAM"),
-};
-
-struct yt921x_info {
-	const char *name;
-	u16 major;
-	/* Unknown, seems to be plain enumeration */
-	u8 mode;
-	u8 extmode;
-	/* Ports with integral GbE PHYs, not including MCU Port 10 */
-	u16 internal_mask;
-	/* TODO: see comments in yt921x_dsa_phylink_get_caps() */
-	u16 external_mask;
-};
-
-#define YT921X_PORT_MASK_INTn(port)	BIT(port)
-#define YT921X_PORT_MASK_INT0_n(n)	GENMASK((n) - 1, 0)
-#define YT921X_PORT_MASK_EXT0		BIT(8)
-#define YT921X_PORT_MASK_EXT1		BIT(9)
-
-static const struct yt921x_info yt921x_infos[] = {
-	{
-		"YT9215SC", YT9215_MAJOR, 1, 0,
-		YT921X_PORT_MASK_INT0_n(5),
-		YT921X_PORT_MASK_EXT0 | YT921X_PORT_MASK_EXT1,
-	},
-	{
-		"YT9215S", YT9215_MAJOR, 2, 0,
-		YT921X_PORT_MASK_INT0_n(5),
-		YT921X_PORT_MASK_EXT0 | YT921X_PORT_MASK_EXT1,
-	},
-	{
-		"YT9215RB", YT9215_MAJOR, 3, 0,
-		YT921X_PORT_MASK_INT0_n(5),
-		YT921X_PORT_MASK_EXT0 | YT921X_PORT_MASK_EXT1,
-	},
-	{
-		"YT9214NB", YT9215_MAJOR, 3, 2,
-		YT921X_PORT_MASK_INTn(1) | YT921X_PORT_MASK_INTn(3),
-		YT921X_PORT_MASK_EXT0 | YT921X_PORT_MASK_EXT1,
-	},
-	{
-		"YT9213NB", YT9215_MAJOR, 3, 3,
-		YT921X_PORT_MASK_INTn(1) | YT921X_PORT_MASK_INTn(3),
-		YT921X_PORT_MASK_EXT1,
-	},
-	{
-		"YT9218N", YT9218_MAJOR, 0, 0,
-		YT921X_PORT_MASK_INT0_n(8),
-		0,
-	},
-	{
-		"YT9218MB", YT9218_MAJOR, 1, 0,
-		YT921X_PORT_MASK_INT0_n(8),
-		YT921X_PORT_MASK_EXT0 | YT921X_PORT_MASK_EXT1,
-	},
-	{}
-};
-
-#define YT921X_NAME	"yt921x"
-
-#define YT921X_VID_UNWARE	4095
-
-#define YT921X_POLL_SLEEP_US	10000
-#define YT921X_POLL_TIMEOUT_US	100000
-
-/* The interval should be small enough to avoid overflow of 32bit MIBs.
- *
- * Until we can read MIBs from stats64 call directly (i.e. sleep
- * there), we have to poll stats more frequently then it is actually needed.
- * For overflow protection, normally, 100 sec interval should have been OK.
- */
-#define YT921X_STATS_INTERVAL_JIFFIES	(3 * HZ)
-
-struct yt921x_reg_mdio {
-	struct mii_bus *bus;
-	int addr;
-	/* SWITCH_ID_1 / SWITCH_ID_0 of the device
-	 *
-	 * This is a way to multiplex multiple devices on the same MII phyaddr
-	 * and should be configurable in DT. However, MDIO core simply doesn't
-	 * allow multiple devices over one reg addr, so this is a fixed value
-	 * for now until a solution is found.
-	 *
-	 * Keep this because we need switchid to form MII regaddrs anyway.
-	 */
-	unsigned char switchid;
-};
-
-/* TODO: SPI/I2C */
-
-#define to_yt921x_priv(_ds) container_of_const(_ds, struct yt921x_priv, ds)
-#define to_device(priv) ((priv)->ds.dev)
-
-static u32 ethaddr_hi4_to_u32(const unsigned char *addr)
-{
-	return (addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) | addr[3];
-}
-
-static u32 ethaddr_lo2_to_u32(const unsigned char *addr)
-{
-	return (addr[4] << 8) | addr[5];
-}
-
-static int yt921x_reg_read(struct yt921x_priv *priv, u32 reg, u32 *valp)
-{
-	WARN_ON(!mutex_is_locked(&priv->reg_lock));
-
-	return priv->reg_ops->read(priv->reg_ctx, reg, valp);
-}
-
-static int yt921x_reg_write(struct yt921x_priv *priv, u32 reg, u32 val)
-{
-	WARN_ON(!mutex_is_locked(&priv->reg_lock));
-
-	return priv->reg_ops->write(priv->reg_ctx, reg, val);
-}
-
-static int
-yt921x_reg_wait(struct yt921x_priv *priv, u32 reg, u32 mask, u32 *valp)
-{
-	u32 val;
-	int res;
-	int ret;
-
-	ret = read_poll_timeout(yt921x_reg_read, res,
-				res || (val & mask) == *valp,
-				YT921X_POLL_SLEEP_US, YT921X_POLL_TIMEOUT_US,
-				false, priv, reg, &val);
-	if (ret)
-		return ret;
-	if (res)
-		return res;
-
-	*valp = val;
-	return 0;
-}
-
-static int
-yt921x_reg_update_bits(struct yt921x_priv *priv, u32 reg, u32 mask, u32 val)
-{
-	int res;
-	u32 v;
-	u32 u;
-
-	res = yt921x_reg_read(priv, reg, &v);
-	if (res)
-		return res;
-
-	u = v;
-	u &= ~mask;
-	u |= val;
-	if (u == v)
-		return 0;
-
-	return yt921x_reg_write(priv, reg, u);
-}
-
-static int yt921x_reg_set_bits(struct yt921x_priv *priv, u32 reg, u32 mask)
-{
-	return yt921x_reg_update_bits(priv, reg, 0, mask);
-}
-
-static int yt921x_reg_clear_bits(struct yt921x_priv *priv, u32 reg, u32 mask)
-{
-	return yt921x_reg_update_bits(priv, reg, mask, 0);
-}
-
-static int
-yt921x_reg_toggle_bits(struct yt921x_priv *priv, u32 reg, u32 mask, bool set)
-{
-	return yt921x_reg_update_bits(priv, reg, mask, !set ? 0 : mask);
-}
-
-/* Some multi-word registers, like VLANn_CTRL, should be treated as a single
- * long register. More specifically, writes to parts of its words won't become
- * visible, until the last word is written.
- *
- * Here we require full read and write operations over these registers to
- * eliminate potential issues, although partial reads/writes are also possible.
- */
-
-static void update_ctrls_unaligned(u32 *lo, u32 *hi, u64 mask, u64 val)
-{
-	*lo &= ~lower_32_bits(mask);
-	*hi &= ~upper_32_bits(mask);
-	*lo |= lower_32_bits(val);
-	*hi |= upper_32_bits(val);
-}
-
-static int
-yt921x_regs_read(struct yt921x_priv *priv, u32 reg, u32 *vals,
-		 unsigned int num_regs)
-{
-	int res;
-
-	for (unsigned int i = 0; i < num_regs; i++) {
-		res = yt921x_reg_read(priv, reg + 4 * i, &vals[i]);
-		if (res)
-			return res;
-	}
-
-	return 0;
-}
-
-static int
-yt921x_regs_write(struct yt921x_priv *priv, u32 reg, const u32 *vals,
-		  unsigned int num_regs)
-{
-	int res;
-
-	for (unsigned int i = 0; i < num_regs; i++) {
-		res = yt921x_reg_write(priv, reg + 4 * i, vals[i]);
-		if (res)
-			return res;
-	}
-
-	return 0;
-}
-
-static int
-yt921x_regs_update_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
-			const u32 *vals, unsigned int num_regs)
-{
-	bool changed = false;
-	u32 vs[4];
-	int res;
-
-	BUILD_BUG_ON(num_regs > ARRAY_SIZE(vs));
-
-	res = yt921x_regs_read(priv, reg, vs, num_regs);
-	if (res)
-		return res;
-
-	for (unsigned int i = 0; i < num_regs; i++) {
-		u32 u = vs[i];
-
-		u &= ~masks[i];
-		u |= vals[i];
-		if (u != vs[i])
-			changed = true;
-
-		vs[i] = u;
-	}
-
-	if (!changed)
-		return 0;
-
-	return yt921x_regs_write(priv, reg, vs, num_regs);
-}
-
-static int
-yt921x_regs_clear_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
-		       unsigned int num_regs)
-{
-	bool changed = false;
-	u32 vs[4];
-	int res;
-
-	BUILD_BUG_ON(num_regs > ARRAY_SIZE(vs));
-
-	res = yt921x_regs_read(priv, reg, vs, num_regs);
-	if (res)
-		return res;
-
-	for (unsigned int i = 0; i < num_regs; i++) {
-		u32 u = vs[i];
-
-		u &= ~masks[i];
-		if (u != vs[i])
-			changed = true;
-
-		vs[i] = u;
-	}
-
-	if (!changed)
-		return 0;
-
-	return yt921x_regs_write(priv, reg, vs, num_regs);
-}
-
-static int
-yt921x_reg64_write(struct yt921x_priv *priv, u32 reg, const u32 *vals)
-{
-	return yt921x_regs_write(priv, reg, vals, 2);
-}
-
-static int
-yt921x_reg64_update_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
-			 const u32 *vals)
-{
-	return yt921x_regs_update_bits(priv, reg, masks, vals, 2);
-}
-
-static int
-yt921x_reg64_clear_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks)
-{
-	return yt921x_regs_clear_bits(priv, reg, masks, 2);
-}
-
-static int
-yt921x_reg96_write(struct yt921x_priv *priv, u32 reg, const u32 *vals)
-{
-	return yt921x_regs_write(priv, reg, vals, 3);
-}
-
-static int yt921x_reg_mdio_read(void *context, u32 reg, u32 *valp)
-{
-	struct yt921x_reg_mdio *mdio = context;
-	struct mii_bus *bus = mdio->bus;
-	int addr = mdio->addr;
-	u32 reg_addr;
-	u32 reg_data;
-	u32 val;
-	int res;
-
-	/* Hold the mdio bus lock to avoid (un)locking for 4 times */
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
-
-	reg_addr = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_ADDR |
-		   YT921X_SMI_READ;
-	res = __mdiobus_write(bus, addr, reg_addr, (u16)(reg >> 16));
-	if (res)
-		goto end;
-	res = __mdiobus_write(bus, addr, reg_addr, (u16)reg);
-	if (res)
-		goto end;
-
-	reg_data = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_DATA |
-		   YT921X_SMI_READ;
-	res = __mdiobus_read(bus, addr, reg_data);
-	if (res < 0)
-		goto end;
-	val = (u16)res;
-	res = __mdiobus_read(bus, addr, reg_data);
-	if (res < 0)
-		goto end;
-	val = (val << 16) | (u16)res;
-
-	*valp = val;
-	res = 0;
-
-end:
-	mutex_unlock(&bus->mdio_lock);
-	return res;
-}
-
-static int yt921x_reg_mdio_write(void *context, u32 reg, u32 val)
-{
-	struct yt921x_reg_mdio *mdio = context;
-	struct mii_bus *bus = mdio->bus;
-	int addr = mdio->addr;
-	u32 reg_addr;
-	u32 reg_data;
-	int res;
-
-	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
-
-	reg_addr = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_ADDR |
-		   YT921X_SMI_WRITE;
-	res = __mdiobus_write(bus, addr, reg_addr, (u16)(reg >> 16));
-	if (res)
-		goto end;
-	res = __mdiobus_write(bus, addr, reg_addr, (u16)reg);
-	if (res)
-		goto end;
-
-	reg_data = YT921X_SMI_SWITCHID(mdio->switchid) | YT921X_SMI_DATA |
-		   YT921X_SMI_WRITE;
-	res = __mdiobus_write(bus, addr, reg_data, (u16)(val >> 16));
-	if (res)
-		goto end;
-	res = __mdiobus_write(bus, addr, reg_data, (u16)val);
-	if (res)
-		goto end;
-
-	res = 0;
-
-end:
-	mutex_unlock(&bus->mdio_lock);
-	return res;
-}
-
-static const struct yt921x_reg_ops yt921x_reg_ops_mdio = {
-	.read = yt921x_reg_mdio_read,
-	.write = yt921x_reg_mdio_write,
-};
-
-/* TODO: SPI/I2C */
-
-static int yt921x_intif_wait(struct yt921x_priv *priv)
-{
-	u32 val = 0;
-
-	return yt921x_reg_wait(priv, YT921X_INT_MBUS_OP, YT921X_MBUS_OP_START,
-			       &val);
-}
-
-static int
-yt921x_intif_read(struct yt921x_priv *priv, int port, int reg, u16 *valp)
-{
-	struct device *dev = to_device(priv);
-	u32 mask;
-	u32 ctrl;
-	u32 val;
-	int res;
-
-	res = yt921x_intif_wait(priv);
-	if (res)
-		return res;
-
-	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
-	       YT921X_MBUS_CTRL_OP_M;
-	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
-	       YT921X_MBUS_CTRL_READ;
-	res = yt921x_reg_update_bits(priv, YT921X_INT_MBUS_CTRL, mask, ctrl);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_INT_MBUS_OP, YT921X_MBUS_OP_START);
-	if (res)
-		return res;
-
-	res = yt921x_intif_wait(priv);
-	if (res)
-		return res;
-	res = yt921x_reg_read(priv, YT921X_INT_MBUS_DIN, &val);
-	if (res)
-		return res;
-
-	if ((u16)val != val)
-		dev_info(dev,
-			 "%s: port %d, reg 0x%x: Expected u16, got 0x%08x\n",
-			 __func__, port, reg, val);
-	*valp = (u16)val;
-	return 0;
-}
-
-static int
-yt921x_intif_write(struct yt921x_priv *priv, int port, int reg, u16 val)
-{
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	res = yt921x_intif_wait(priv);
-	if (res)
-		return res;
-
-	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
-	       YT921X_MBUS_CTRL_OP_M;
-	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
-	       YT921X_MBUS_CTRL_WRITE;
-	res = yt921x_reg_update_bits(priv, YT921X_INT_MBUS_CTRL, mask, ctrl);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_INT_MBUS_DOUT, val);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_INT_MBUS_OP, YT921X_MBUS_OP_START);
-	if (res)
-		return res;
-
-	return yt921x_intif_wait(priv);
-}
-
-static int yt921x_mbus_int_read(struct mii_bus *mbus, int port, int reg)
-{
-	struct yt921x_priv *priv = mbus->priv;
-	u16 val;
-	int res;
-
-	if (port >= YT921X_PORT_NUM)
-		return U16_MAX;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_intif_read(priv, port, reg, &val);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		return res;
-	return val;
-}
-
-static int
-yt921x_mbus_int_write(struct mii_bus *mbus, int port, int reg, u16 data)
-{
-	struct yt921x_priv *priv = mbus->priv;
-	int res;
-
-	if (port >= YT921X_PORT_NUM)
-		return -ENODEV;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_intif_write(priv, port, reg, data);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_mbus_int_init(struct yt921x_priv *priv, struct device_node *mnp)
-{
-	struct device *dev = to_device(priv);
-	struct mii_bus *mbus;
-	int res;
-
-	mbus = devm_mdiobus_alloc(dev);
-	if (!mbus)
-		return -ENOMEM;
-
-	mbus->name = "YT921x internal MDIO bus";
-	snprintf(mbus->id, MII_BUS_ID_SIZE, "%s", dev_name(dev));
-	mbus->priv = priv;
-	mbus->read = yt921x_mbus_int_read;
-	mbus->write = yt921x_mbus_int_write;
-	mbus->parent = dev;
-	mbus->phy_mask = (u32)~GENMASK(YT921X_PORT_NUM - 1, 0);
-
-	res = devm_of_mdiobus_register(dev, mbus, mnp);
-	if (res)
-		return res;
-
-	priv->mbus_int = mbus;
-
-	return 0;
-}
-
-static int yt921x_extif_wait(struct yt921x_priv *priv)
-{
-	u32 val = 0;
-
-	return yt921x_reg_wait(priv, YT921X_EXT_MBUS_OP, YT921X_MBUS_OP_START,
-			       &val);
-}
-
-static int
-yt921x_extif_read(struct yt921x_priv *priv, int port, int reg, u16 *valp)
-{
-	struct device *dev = to_device(priv);
-	u32 mask;
-	u32 ctrl;
-	u32 val;
-	int res;
-
-	res = yt921x_extif_wait(priv);
-	if (res)
-		return res;
-
-	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
-	       YT921X_MBUS_CTRL_TYPE_M | YT921X_MBUS_CTRL_OP_M;
-	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
-	       YT921X_MBUS_CTRL_TYPE_C22 | YT921X_MBUS_CTRL_READ;
-	res = yt921x_reg_update_bits(priv, YT921X_EXT_MBUS_CTRL, mask, ctrl);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_EXT_MBUS_OP, YT921X_MBUS_OP_START);
-	if (res)
-		return res;
-
-	res = yt921x_extif_wait(priv);
-	if (res)
-		return res;
-	res = yt921x_reg_read(priv, YT921X_EXT_MBUS_DIN, &val);
-	if (res)
-		return res;
-
-	if ((u16)val != val)
-		dev_info(dev,
-			 "%s: port %d, reg 0x%x: Expected u16, got 0x%08x\n",
-			 __func__, port, reg, val);
-	*valp = (u16)val;
-	return 0;
-}
-
-static int
-yt921x_extif_write(struct yt921x_priv *priv, int port, int reg, u16 val)
-{
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	res = yt921x_extif_wait(priv);
-	if (res)
-		return res;
-
-	mask = YT921X_MBUS_CTRL_PORT_M | YT921X_MBUS_CTRL_REG_M |
-	       YT921X_MBUS_CTRL_TYPE_M | YT921X_MBUS_CTRL_OP_M;
-	ctrl = YT921X_MBUS_CTRL_PORT(port) | YT921X_MBUS_CTRL_REG(reg) |
-	       YT921X_MBUS_CTRL_TYPE_C22 | YT921X_MBUS_CTRL_WRITE;
-	res = yt921x_reg_update_bits(priv, YT921X_EXT_MBUS_CTRL, mask, ctrl);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_EXT_MBUS_DOUT, val);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_EXT_MBUS_OP, YT921X_MBUS_OP_START);
-	if (res)
-		return res;
-
-	return yt921x_extif_wait(priv);
-}
-
-static int yt921x_mbus_ext_read(struct mii_bus *mbus, int port, int reg)
-{
-	struct yt921x_priv *priv = mbus->priv;
-	u16 val;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_extif_read(priv, port, reg, &val);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		return res;
-	return val;
-}
-
-static int
-yt921x_mbus_ext_write(struct mii_bus *mbus, int port, int reg, u16 data)
-{
-	struct yt921x_priv *priv = mbus->priv;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_extif_write(priv, port, reg, data);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_mbus_ext_init(struct yt921x_priv *priv, struct device_node *mnp)
-{
-	struct device *dev = to_device(priv);
-	struct mii_bus *mbus;
-	int res;
-
-	mbus = devm_mdiobus_alloc(dev);
-	if (!mbus)
-		return -ENOMEM;
-
-	mbus->name = "YT921x external MDIO bus";
-	snprintf(mbus->id, MII_BUS_ID_SIZE, "%s@ext", dev_name(dev));
-	mbus->priv = priv;
-	/* TODO: c45? */
-	mbus->read = yt921x_mbus_ext_read;
-	mbus->write = yt921x_mbus_ext_write;
-	mbus->parent = dev;
-
-	res = devm_of_mdiobus_register(dev, mbus, mnp);
-	if (res)
-		return res;
-
-	priv->mbus_ext = mbus;
-
-	return 0;
-}
-
-/* Read and handle overflow of 32bit MIBs. MIB buffer must be zeroed before. */
-static int yt921x_read_mib(struct yt921x_priv *priv, int port)
-{
-	struct yt921x_port *pp = &priv->ports[port];
-	struct device *dev = to_device(priv);
-	struct yt921x_mib *mib = &pp->mib;
-	int res = 0;
-
-	/* Reading of yt921x_port::mib is not protected by a lock and it's vain
-	 * to keep its consistency, since we have to read registers one by one
-	 * and there is no way to make a snapshot of MIB stats.
-	 *
-	 * Writing (by this function only) is and should be protected by
-	 * reg_lock.
-	 */
-
-	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
-		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
-		u32 reg = YT921X_MIBn_DATA0(port) + desc->offset;
-		u64 *valp = &((u64 *)mib)[i];
-		u32 val0;
-		u64 val;
-
-		res = yt921x_reg_read(priv, reg, &val0);
-		if (res)
-			break;
-
-		if (desc->size <= 1) {
-			u64 old_val = *valp;
-
-			val = (old_val & ~(u64)U32_MAX) | val0;
-			if (val < old_val)
-				val += 1ull << 32;
-		} else {
-			u32 val1;
-
-			res = yt921x_reg_read(priv, reg + 4, &val1);
-			if (res)
-				break;
-			val = ((u64)val1 << 32) | val0;
-		}
-
-		WRITE_ONCE(*valp, val);
-	}
-
-	pp->rx_frames = mib->rx_64byte + mib->rx_65_127byte +
-			mib->rx_128_255byte + mib->rx_256_511byte +
-			mib->rx_512_1023byte + mib->rx_1024_1518byte +
-			mib->rx_jumbo;
-	pp->tx_frames = mib->tx_64byte + mib->tx_65_127byte +
-			mib->tx_128_255byte + mib->tx_256_511byte +
-			mib->tx_512_1023byte + mib->tx_1024_1518byte +
-			mib->tx_jumbo;
-
-	if (res)
-		dev_err(dev, "Failed to %s port %d: %i\n", "read stats for",
-			port, res);
-	return res;
-}
-
-static void yt921x_poll_mib(struct work_struct *work)
-{
-	struct yt921x_port *pp = container_of_const(work, struct yt921x_port,
-						    mib_read.work);
-	struct yt921x_priv *priv = (void *)(pp - pp->index) -
-				   offsetof(struct yt921x_priv, ports);
-	unsigned long delay = YT921X_STATS_INTERVAL_JIFFIES;
-	int port = pp->index;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_read_mib(priv, port);
-	mutex_unlock(&priv->reg_lock);
-	if (res)
-		delay *= 4;
-
-	schedule_delayed_work(&pp->mib_read, delay);
-}
-
-static void
-yt921x_dsa_get_strings(struct dsa_switch *ds, int port, u32 stringset,
-		       uint8_t *data)
-{
-	if (stringset != ETH_SS_STATS)
-		return;
-
-	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
-		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
-
-		if (desc->name)
-			ethtool_puts(&data, desc->name);
-	}
-}
-
-static void
-yt921x_dsa_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_port *pp = &priv->ports[port];
-	struct yt921x_mib *mib = &pp->mib;
-	size_t j;
-
-	mutex_lock(&priv->reg_lock);
-	yt921x_read_mib(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	j = 0;
-	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
-		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
-
-		if (!desc->name)
-			continue;
-
-		data[j] = ((u64 *)mib)[i];
-		j++;
-	}
-}
-
-static int yt921x_dsa_get_sset_count(struct dsa_switch *ds, int port, int sset)
-{
-	int cnt = 0;
-
-	if (sset != ETH_SS_STATS)
-		return 0;
-
-	for (size_t i = 0; i < ARRAY_SIZE(yt921x_mib_descs); i++) {
-		const struct yt921x_mib_desc *desc = &yt921x_mib_descs[i];
-
-		if (desc->name)
-			cnt++;
-	}
-
-	return cnt;
-}
-
-static void
-yt921x_dsa_get_eth_mac_stats(struct dsa_switch *ds, int port,
-			     struct ethtool_eth_mac_stats *mac_stats)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_port *pp = &priv->ports[port];
-	struct yt921x_mib *mib = &pp->mib;
-
-	mutex_lock(&priv->reg_lock);
-	yt921x_read_mib(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	mac_stats->FramesTransmittedOK = pp->tx_frames;
-	mac_stats->SingleCollisionFrames = mib->tx_single_collisions;
-	mac_stats->MultipleCollisionFrames = mib->tx_multiple_collisions;
-	mac_stats->FramesReceivedOK = pp->rx_frames;
-	mac_stats->FrameCheckSequenceErrors = mib->rx_crc_errors;
-	mac_stats->AlignmentErrors = mib->rx_alignment_errors;
-	mac_stats->OctetsTransmittedOK = mib->tx_good_bytes;
-	mac_stats->FramesWithDeferredXmissions = mib->tx_deferred;
-	mac_stats->LateCollisions = mib->tx_late_collisions;
-	mac_stats->FramesAbortedDueToXSColls = mib->tx_aborted_errors;
-	/* mac_stats->FramesLostDueToIntMACXmitError */
-	/* mac_stats->CarrierSenseErrors */
-	mac_stats->OctetsReceivedOK = mib->rx_good_bytes;
-	/* mac_stats->FramesLostDueToIntMACRcvError */
-	mac_stats->MulticastFramesXmittedOK = mib->tx_multicast;
-	mac_stats->BroadcastFramesXmittedOK = mib->tx_broadcast;
-	/* mac_stats->FramesWithExcessiveDeferral */
-	mac_stats->MulticastFramesReceivedOK = mib->rx_multicast;
-	mac_stats->BroadcastFramesReceivedOK = mib->rx_broadcast;
-	/* mac_stats->InRangeLengthErrors */
-	/* mac_stats->OutOfRangeLengthField */
-	mac_stats->FrameTooLongErrors = mib->rx_oversize_errors;
-}
-
-static void
-yt921x_dsa_get_eth_ctrl_stats(struct dsa_switch *ds, int port,
-			      struct ethtool_eth_ctrl_stats *ctrl_stats)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_port *pp = &priv->ports[port];
-	struct yt921x_mib *mib = &pp->mib;
-
-	mutex_lock(&priv->reg_lock);
-	yt921x_read_mib(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	ctrl_stats->MACControlFramesTransmitted = mib->tx_pause;
-	ctrl_stats->MACControlFramesReceived = mib->rx_pause;
-	/* ctrl_stats->UnsupportedOpcodesReceived */
-}
-
-static const struct ethtool_rmon_hist_range yt921x_rmon_ranges[] = {
-	{ 0, 64 },
-	{ 65, 127 },
-	{ 128, 255 },
-	{ 256, 511 },
-	{ 512, 1023 },
-	{ 1024, 1518 },
-	{ 1519, YT921X_FRAME_SIZE_MAX },
-	{}
-};
-
-static void
-yt921x_dsa_get_rmon_stats(struct dsa_switch *ds, int port,
-			  struct ethtool_rmon_stats *rmon_stats,
-			  const struct ethtool_rmon_hist_range **ranges)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_port *pp = &priv->ports[port];
-	struct yt921x_mib *mib = &pp->mib;
-
-	mutex_lock(&priv->reg_lock);
-	yt921x_read_mib(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	*ranges = yt921x_rmon_ranges;
-
-	rmon_stats->undersize_pkts = mib->rx_undersize_errors;
-	rmon_stats->oversize_pkts = mib->rx_oversize_errors;
-	rmon_stats->fragments = mib->rx_alignment_errors;
-	/* rmon_stats->jabbers */
-
-	rmon_stats->hist[0] = mib->rx_64byte;
-	rmon_stats->hist[1] = mib->rx_65_127byte;
-	rmon_stats->hist[2] = mib->rx_128_255byte;
-	rmon_stats->hist[3] = mib->rx_256_511byte;
-	rmon_stats->hist[4] = mib->rx_512_1023byte;
-	rmon_stats->hist[5] = mib->rx_1024_1518byte;
-	rmon_stats->hist[6] = mib->rx_jumbo;
-
-	rmon_stats->hist_tx[0] = mib->tx_64byte;
-	rmon_stats->hist_tx[1] = mib->tx_65_127byte;
-	rmon_stats->hist_tx[2] = mib->tx_128_255byte;
-	rmon_stats->hist_tx[3] = mib->tx_256_511byte;
-	rmon_stats->hist_tx[4] = mib->tx_512_1023byte;
-	rmon_stats->hist_tx[5] = mib->tx_1024_1518byte;
-	rmon_stats->hist_tx[6] = mib->tx_jumbo;
-}
-
-static void
-yt921x_dsa_get_stats64(struct dsa_switch *ds, int port,
-		       struct rtnl_link_stats64 *stats)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_port *pp = &priv->ports[port];
-	struct yt921x_mib *mib = &pp->mib;
-
-	stats->rx_length_errors = mib->rx_undersize_errors +
-				  mib->rx_fragment_errors;
-	stats->rx_over_errors = mib->rx_oversize_errors;
-	stats->rx_crc_errors = mib->rx_crc_errors;
-	stats->rx_frame_errors = mib->rx_alignment_errors;
-	/* stats->rx_fifo_errors */
-	/* stats->rx_missed_errors */
-
-	stats->tx_aborted_errors = mib->tx_aborted_errors;
-	/* stats->tx_carrier_errors */
-	stats->tx_fifo_errors = mib->tx_undersize_errors;
-	/* stats->tx_heartbeat_errors */
-	stats->tx_window_errors = mib->tx_late_collisions;
-
-	stats->rx_packets = pp->rx_frames;
-	stats->tx_packets = pp->tx_frames;
-	stats->rx_bytes = mib->rx_good_bytes - ETH_FCS_LEN * stats->rx_packets;
-	stats->tx_bytes = mib->tx_good_bytes - ETH_FCS_LEN * stats->tx_packets;
-	stats->rx_errors = stats->rx_length_errors + stats->rx_over_errors +
-			   stats->rx_crc_errors + stats->rx_frame_errors;
-	stats->tx_errors = stats->tx_aborted_errors + stats->tx_fifo_errors +
-			   stats->tx_window_errors;
-	stats->rx_dropped = mib->rx_dropped;
-	/* stats->tx_dropped */
-	stats->multicast = mib->rx_multicast;
-	stats->collisions = mib->tx_collisions;
-}
-
-static void
-yt921x_dsa_get_pause_stats(struct dsa_switch *ds, int port,
-			   struct ethtool_pause_stats *pause_stats)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_port *pp = &priv->ports[port];
-	struct yt921x_mib *mib = &pp->mib;
-
-	mutex_lock(&priv->reg_lock);
-	yt921x_read_mib(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	pause_stats->tx_pause_frames = mib->tx_pause;
-	pause_stats->rx_pause_frames = mib->rx_pause;
-}
-
-static int
-yt921x_set_eee(struct yt921x_priv *priv, int port, struct ethtool_keee *e)
-{
-	/* Poor datasheet for EEE operations; don't ask if you are confused */
-
-	bool enable = e->eee_enabled;
-	u16 new_mask;
-	int res;
-
-	/* Enable / disable global EEE */
-	new_mask = priv->eee_ports_mask;
-	new_mask &= ~BIT(port);
-	new_mask |= !enable ? 0 : BIT(port);
-
-	if (!!new_mask != !!priv->eee_ports_mask) {
-		res = yt921x_reg_toggle_bits(priv, YT921X_PON_STRAP_FUNC,
-					     YT921X_PON_STRAP_EEE, !!new_mask);
-		if (res)
-			return res;
-		res = yt921x_reg_toggle_bits(priv, YT921X_PON_STRAP_VAL,
-					     YT921X_PON_STRAP_EEE, !!new_mask);
-		if (res)
-			return res;
-	}
-
-	priv->eee_ports_mask = new_mask;
-
-	/* Enable / disable port EEE */
-	res = yt921x_reg_toggle_bits(priv, YT921X_EEE_CTRL,
-				     YT921X_EEE_CTRL_ENn(port), enable);
-	if (res)
-		return res;
-	res = yt921x_reg_toggle_bits(priv, YT921X_EEEn_VAL(port),
-				     YT921X_EEE_VAL_DATA, enable);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int
-yt921x_dsa_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_set_eee(priv, port, e);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int yt921x_mtu_fetch(struct yt921x_priv *priv, int port)
-{
-	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
-
-	return dp->user ? READ_ONCE(dp->user->mtu) : ETH_DATA_LEN;
-}
-
-static int
-yt921x_dsa_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
-{
-	/* Only serves as packet filter, since the frame size is always set to
-	 * maximum after reset
-	 */
-
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct dsa_port *dp = dsa_to_port(ds, port);
-	int frame_size;
-	int res;
-
-	frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN;
-	if (dsa_port_is_cpu(dp))
-		frame_size += YT921X_TAG_LEN;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_update_bits(priv, YT921X_MACn_FRAME(port),
-				     YT921X_MAC_FRAME_SIZE_M,
-				     YT921X_MAC_FRAME_SIZE(frame_size));
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int yt921x_dsa_port_max_mtu(struct dsa_switch *ds, int port)
-{
-	/* Only called for user ports, exclude tag len here */
-	return YT921X_FRAME_SIZE_MAX - ETH_HLEN - ETH_FCS_LEN - YT921X_TAG_LEN;
-}
-
-/* v * 2^e */
-static u64 ldexpu64(u64 v, int e)
-{
-	return e >= 0 ? v << e : v >> -e;
-}
-
-/* slot (ns) * rate (/s) / 10^9 (ns/s) = 2^C * token * 4^unit */
-static u32 rate2token(u64 rate, unsigned int slot_ns, int unit, int C)
-{
-	int e = 2 * unit + C + YT921X_TOKEN_RATE_C;
-
-	return div_u64(ldexpu64(slot_ns * rate, -e), 1000000000);
-}
-
-static u64 token2rate(u32 token, unsigned int slot_ns, int unit, int C)
-{
-	int e = 2 * unit + C + YT921X_TOKEN_RATE_C;
-
-	return div_u64(ldexpu64(mul_u32_u32(1000000000, token), e), slot_ns);
-}
-
-/* burst = 2^C * token * 4^unit */
-static u32 burst2token(u64 burst, int unit, int C)
-{
-	return ldexpu64(burst, -(2 * unit + C));
-}
-
-static u64 token2burst(u32 token, int unit, int C)
-{
-	return ldexpu64(token, 2 * unit + C);
-}
-
-struct yt921x_marker {
-	u32 cir;
-	u32 cbs;
-	u32 ebs;
-	int unit;
-	bool pkt_mode;
-};
-
-#define YT921X_MARKER_PKT_MODE		BIT(0)
-#define YT921X_MARKER_SINGLE_BUCKET	BIT(1)
-
-static int
-yt921x_marker_tfm(struct yt921x_marker *marker, u64 rate, u64 burst,
-		  unsigned int flags, unsigned int slot_ns, u32 cir_max,
-		  u32 cbs_max, int unit_max, struct yt921x_priv *priv, int port,
-		  struct netlink_ext_ack *extack)
-{
-	const int C = flags & YT921X_MARKER_PKT_MODE ? YT921X_TOKEN_PKT_C :
-		      YT921X_TOKEN_BYTE_C;
-	struct device *dev = to_device(priv);
-	struct yt921x_marker m;
-	u64 burst_est;
-	u64 burst_sug;
-	u64 burst_max;
-	u64 rate_max;
-
-	m.unit = unit_max;
-	rate_max = token2rate(cir_max, slot_ns, m.unit, C);
-	burst_max = token2burst(cbs_max, m.unit, C);
-
-	/* Check for unusual values */
-	if (rate > rate_max || burst > burst_max) {
-		NL_SET_ERR_MSG_MOD(extack, "Unexpected tremendous rate");
-		return -ERANGE;
-	}
-
-	/* Check for matching burst */
-	burst_est = div_u64(slot_ns * rate, 1000000000);
-	burst_sug = burst_est;
-	if (flags & YT921X_MARKER_PKT_MODE)
-		burst_sug++;
-	else
-		burst_sug += ETH_HLEN + yt921x_mtu_fetch(priv, port) +
-			     ETH_FCS_LEN;
-	if (burst_sug > burst)
-		NL_SET_ERR_MSG_FMT_MOD(extack,
-				       "Consider match rate %llu with burst at least %llu",
-				       rate, burst_sug);
-
-	/* Select unit */
-	for (; m.unit > 0; m.unit--) {
-		if (rate > (rate_max >> 2) || burst > (burst_max >> 2))
-			break;
-		rate_max >>= 2;
-		burst_max >>= 2;
-	}
-
-	/* Calculate information rate and bucket size */
-	m.cir = rate2token(rate, slot_ns, m.unit, C);
-	if (!m.cir)
-		m.cir = 1;
-	else if (WARN_ON(m.cir > cir_max))
-		m.cir = cir_max;
-	m.cbs = burst2token(burst, m.unit, C);
-	if (!m.cbs)
-		m.cbs = 1;
-	else if (WARN_ON(m.cbs > cbs_max))
-		m.cbs = cbs_max;
-
-	/* Cut EBS */
-	m.ebs = 0;
-	if (!(flags & YT921X_MARKER_SINGLE_BUCKET)) {
-		/* We don't have a chance to adjust rate when MTU is changed */
-		if (flags & YT921X_MARKER_PKT_MODE)
-			burst_est++;
-		else
-			burst_est += YT921X_FRAME_SIZE_MAX;
-
-		if (burst_est < burst) {
-			u32 pbs = m.cbs;
-
-			m.cbs = burst2token(burst_est, m.unit, C);
-			if (!m.cbs)
-				m.cbs = 1;
-			else if (WARN_ON(m.cbs > cbs_max))
-				m.cbs = cbs_max;
-
-			if (pbs > m.cbs)
-				m.ebs = pbs - m.cbs;
-		}
-	}
-
-	dev_dbg(dev,
-		"slot %u ns, rate %llu, burst %llu -> unit %d, cir %u, cbs %u, ebs %u\n",
-		slot_ns, rate, burst, m.unit, m.cir, m.cbs, m.ebs);
-
-	m.pkt_mode = flags & YT921X_MARKER_PKT_MODE;
-	*marker = m;
-	return 0;
-}
-
-static int
-yt921x_marker_tfm_police(struct yt921x_marker *marker,
-			 const struct flow_action_police *police,
-			 unsigned int flags, struct yt921x_priv *priv, int port,
-			 struct netlink_ext_ack *extack)
-{
-	bool pkt_mode = !!police->rate_pkt_ps;
-	u64 burst;
-	u64 rate;
-
-	rate = pkt_mode ? police->rate_pkt_ps : police->rate_bytes_ps;
-	burst = pkt_mode ? police->burst_pkt : police->burst;
-	if (pkt_mode)
-		flags |= YT921X_MARKER_PKT_MODE;
-
-	return yt921x_marker_tfm(marker, rate, burst, flags,
-				 priv->meter_slot_ns, YT921X_METER_CIR_MAX,
-				 YT921X_METER_CBS_MAX, YT921X_METER_UNIT_MAX,
-				 priv, port, extack);
-}
-
-static int
-yt921x_marker_tfm_shape(struct yt921x_marker *marker, u64 rate, u64 burst,
-			unsigned int flags, struct yt921x_priv *priv, int port,
-			struct netlink_ext_ack *extack)
-{
-	return yt921x_marker_tfm(marker, rate, burst, flags,
-				 priv->port_shape_slot_ns, YT921X_SHAPE_CIR_MAX,
-				 YT921X_SHAPE_CBS_MAX, YT921X_SHAPE_UNIT_MAX,
-				 priv, port, extack);
-}
-
-static int
-yt921x_police_validate(const struct flow_action_police *police,
-		       const struct flow_action *action,
-		       const struct flow_action_entry *act,
-		       struct netlink_ext_ack *extack)
-{
-	if (police->exceed.act_id != FLOW_ACTION_DROP) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Offload not supported when exceed action is not drop");
-		return -EOPNOTSUPP;
-	}
-
-	if (police->notexceed.act_id != FLOW_ACTION_PIPE &&
-	    police->notexceed.act_id != FLOW_ACTION_ACCEPT) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Offload not supported when conform action is not pipe or ok");
-		return -EOPNOTSUPP;
-	}
-
-	if (police->notexceed.act_id == FLOW_ACTION_ACCEPT && action && act &&
-	    !flow_action_is_last_entry(action, act)) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Offload not supported when conform action is ok, but action is not last");
-		return -EOPNOTSUPP;
-	}
-
-	/* mtu defaults to unlimited but we got 2040 here, don't know why */
-	if (police->peakrate_bytes_ps || police->avrate || police->overhead) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Offload not supported when peakrate/avrate/overhead is configured");
-		return -EOPNOTSUPP;
-	}
-
-	return 0;
-}
-
-static int
-yt921x_meter_config(struct yt921x_priv *priv, unsigned int id,
-		    const struct yt921x_marker *marker)
-{
-	u32 ctrls[3];
-
-	ctrls[0] = 0;
-	ctrls[1] = YT921X_METER_CTRLb_CIR(marker->cir);
-	ctrls[2] = YT921X_METER_CTRLc_UNIT(marker->unit) |
-		   YT921X_METER_CTRLc_DROP_R |
-		   YT921X_METER_CTRLc_TOKEN_OVERFLOW_EN |
-		   YT921X_METER_CTRLc_METER_EN;
-	if (marker->pkt_mode)
-		ctrls[2] |= YT921X_METER_CTRLc_PKT_MODE;
-	update_ctrls_unaligned(&ctrls[0], &ctrls[1],
-			       YT921X_METER_CTRLab_EBS_M,
-			       YT921X_METER_CTRLab_EBS(marker->ebs));
-	update_ctrls_unaligned(&ctrls[1], &ctrls[2],
-			       YT921X_METER_CTRLbc_CBS_M,
-			       YT921X_METER_CTRLbc_CBS(marker->cbs));
-
-	return yt921x_reg96_write(priv, YT921X_METERn_CTRL(id), ctrls);
-}
-
-static void yt921x_dsa_port_policer_del(struct dsa_switch *ds, int port)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct device *dev = to_device(priv);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_write(priv, YT921X_PORTn_METER(port), 0);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dev, "Failed to %s port %d: %i\n", "delete policer on",
-			port, res);
-}
-
-static int
-yt921x_dsa_port_policer_add(struct dsa_switch *ds, int port,
-			    const struct flow_action_police *police,
-			    struct netlink_ext_ack *extack)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_marker marker;
-	u32 ctrl;
-	int res;
-
-	res = yt921x_police_validate(police, NULL, NULL, extack);
-	if (res)
-		return res;
-
-	res = yt921x_marker_tfm_police(&marker, police, 0, priv, port, extack);
-	if (res)
-		return res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_meter_config(priv, port + YT921X_METER_NUM, &marker);
-	if (res)
-		goto end;
-
-	ctrl = YT921X_PORT_METER_ID(port) | YT921X_PORT_METER_EN;
-	res = yt921x_reg_write(priv, YT921X_PORTn_METER(port), ctrl);
-end:
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_setup_tc_tbf_port(struct dsa_switch *ds, int port,
-				  const struct tc_tbf_qopt_offload *qopt)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct netlink_ext_ack *extack = qopt->extack;
-	u32 ctrls[2];
-	int res;
-
-	if (qopt->parent != TC_H_ROOT)
-		return -EOPNOTSUPP;
-
-	switch (qopt->command) {
-	case TC_TBF_STATS:
-		/* Unfortunately the convention for TC_*_STATS is a mess,
-		 * neither 0 nor -EOPNOTSUPP is perfect here.
-		 */
-		return -EOPNOTSUPP;
-	case TC_TBF_DESTROY:
-		ctrls[0] = 0;
-		ctrls[1] = 0;
-		break;
-	case TC_TBF_REPLACE: {
-		const struct tc_tbf_qopt_offload_replace_params *p;
-		struct yt921x_marker marker;
-
-		p = &qopt->replace_params;
-
-		res = yt921x_marker_tfm_shape(&marker, p->rate.rate_bytes_ps,
-					      p->max_size,
-					      YT921X_MARKER_SINGLE_BUCKET,
-					      priv, port, extack);
-		if (res)
-			return res;
-
-		ctrls[0] = YT921X_PORT_SHAPE_CTRLa_CIR(marker.cir) |
-			   YT921X_PORT_SHAPE_CTRLa_CBS(marker.cbs);
-		ctrls[1] = YT921X_PORT_SHAPE_CTRLb_UNIT(marker.unit) |
-			   YT921X_PORT_SHAPE_CTRLb_EN;
-		break;
-	}
-	default:
-		return -EOPNOTSUPP;
-	}
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg64_write(priv, YT921X_PORTn_SHAPE_CTRL(port), ctrls);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_setup_tc(struct dsa_switch *ds, int port,
-			 enum tc_setup_type type, void *type_data)
-{
-	switch (type) {
-	case TC_SETUP_QDISC_TBF: {
-		const struct tc_tbf_qopt_offload *qopt = type_data;
-
-		return yt921x_dsa_port_setup_tc_tbf_port(ds, port, qopt);
-	}
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-/* ACL: 48 blocks * 8 entries
- *
- * One rule can span multiple entries, but within a block.
- */
-
-static void
-yt921x_acl_entry_set(struct yt921x_acl_entry *entry, unsigned int offset,
-		     u32 flags, bool set)
-{
-	if (set)
-		entry->key[offset] |= flags;
-	entry->mask[offset] |= flags;
-}
-
-static unsigned int
-yt921x_acl_entries_set_is_fragment(struct yt921x_acl_entry *entries,
-				   unsigned int size, bool set)
-{
-	for (unsigned int i = 0; i < size; i++)
-		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
-		case YT921X_ACL_TYPE_IPV4_DA:
-		case YT921X_ACL_TYPE_IPV4_SA:
-			yt921x_acl_entry_set(&entries[i], 1,
-					     YT921X_ACL_BINb_IPV4_FRAG, set);
-			return size;
-		case YT921X_ACL_TYPE_IPV6_DA3:
-		case YT921X_ACL_TYPE_IPV6_SA3:
-			yt921x_acl_entry_set(&entries[i], 1,
-					     YT921X_ACL_BINb_IPV6_xA3_FRAG,
-					     set);
-			return size;
-		case YT921X_ACL_TYPE_MISC:
-			yt921x_acl_entry_set(&entries[i], 1,
-					     YT921X_ACL_BINb_MISC_FRAG, set);
-			return size;
-		case YT921X_ACL_TYPE_L4:
-			yt921x_acl_entry_set(&entries[i], 1,
-					     YT921X_ACL_BINb_L4_FRAG, set);
-			return size;
-		}
-
-	if (size >= YT921X_ACL_ENT_PER_BLK)
-		return 0;
-
-	entries[size] = (typeof(*entries)){};
-	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
-	yt921x_acl_entry_set(&entries[size], 1, YT921X_ACL_BINb_MISC_FRAG, set);
-
-	return size + 1;
-}
-
-static unsigned int
-yt921x_acl_entries_set_first_frag(struct yt921x_acl_entry *entries,
-				  unsigned int size, bool set)
-{
-	for (unsigned int i = 0; i < size; i++)
-		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
-		case YT921X_ACL_TYPE_IPV6_DA2:
-		case YT921X_ACL_TYPE_IPV6_SA2:
-			yt921x_acl_entry_set(&entries[i], 1,
-					     YT921X_ACL_BINb_IPV6_xA2_FIRST_FRAG,
-					     set);
-			return size;
-		case YT921X_ACL_TYPE_MISC:
-			yt921x_acl_entry_set(&entries[i], 0,
-					     YT921X_ACL_BINa_MISC_FIRST_FRAG,
-					     set);
-			return size;
-		}
-
-	if (size >= YT921X_ACL_ENT_PER_BLK)
-		return 0;
-
-	entries[size] = (typeof(*entries)){};
-	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
-	yt921x_acl_entry_set(&entries[size], 0,
-			     YT921X_ACL_BINa_MISC_FIRST_FRAG, set);
-
-	return size + 1;
-}
-
-static unsigned int
-yt921x_acl_entries_set_l3_type(struct yt921x_acl_entry *entries,
-			       unsigned int size, enum yt921x_l3_type type)
-{
-	for (unsigned int i = 0; i < size; i++)
-		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
-		case YT921X_ACL_TYPE_MAC_DA0:
-		case YT921X_ACL_TYPE_MAC_SA0:
-			entries[i].key[1] |= YT921X_ACL_BINb_MAC_xA0_L3_TYPE(type);
-			entries[i].mask[1] |= YT921X_ACL_BINb_MAC_xA0_L3_TYPE_M;
-			return size;
-		case YT921X_ACL_TYPE_MISC:
-			entries[i].key[0] |= YT921X_ACL_BINa_MISC_L3_TYPE(type);
-			entries[i].mask[0] |= YT921X_ACL_BINa_MISC_L3_TYPE_M;
-			return size;
-		}
-
-	if (size >= YT921X_ACL_ENT_PER_BLK)
-		return 0;
-
-	entries[size] = (typeof(*entries)){};
-	entries[size].key[0] = YT921X_ACL_BINa_MISC_L3_TYPE(type);
-	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
-	entries[size].mask[0] = YT921X_ACL_BINa_MISC_L3_TYPE_M;
-
-	return size + 1;
-}
-
-static unsigned int
-yt921x_acl_entries_set_l4_type(struct yt921x_acl_entry *entries,
-			       unsigned int size, enum yt921x_l4_type type)
-{
-	for (unsigned int i = 0; i < size; i++)
-		switch (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1])) {
-		case YT921X_ACL_TYPE_IPV4_DA:
-		case YT921X_ACL_TYPE_IPV4_SA:
-			entries[i].key[1] |= YT921X_ACL_BINb_IPV4_L4_TYPE(type);
-			entries[i].mask[1] |= YT921X_ACL_BINb_IPV4_L4_TYPE_M;
-			return size;
-		case YT921X_ACL_TYPE_IPV6_DA0:
-		case YT921X_ACL_TYPE_IPV6_DA1:
-		case YT921X_ACL_TYPE_IPV6_DA2:
-		case YT921X_ACL_TYPE_IPV6_DA3:
-		case YT921X_ACL_TYPE_IPV6_SA0:
-		case YT921X_ACL_TYPE_IPV6_SA1:
-		case YT921X_ACL_TYPE_IPV6_SA2:
-		case YT921X_ACL_TYPE_IPV6_SA3:
-			entries[i].key[1] |= YT921X_ACL_BINb_IPV6_L4_TYPE(type);
-			entries[i].mask[1] |= YT921X_ACL_BINb_IPV6_L4_TYPE_M;
-			return size;
-		case YT921X_ACL_TYPE_L4:
-			entries[i].key[1] |= YT921X_ACL_BINb_L4_TYPE(type);
-			entries[i].mask[1] |= YT921X_ACL_BINb_L4_TYPE_M;
-			return size;
-		case YT921X_ACL_TYPE_MISC:
-			entries[i].key[1] |= YT921X_ACL_BINb_MISC_L4_TYPE(type);
-			entries[i].mask[1] |= YT921X_ACL_BINb_MISC_L4_TYPE_M;
-			return size;
-		}
-
-	if (size >= YT921X_ACL_ENT_PER_BLK)
-		return 0;
-
-	entries[size] = (typeof(*entries)){};
-	entries[size].key[1] = YT921X_ACL_BINb_MISC_L4_TYPE(type) |
-			       YT921X_ACL_KEYb_TYPE(YT921X_ACL_TYPE_MISC);
-	entries[size].mask[1] = YT921X_ACL_BINb_MISC_L4_TYPE_M;
-
-	return size + 1;
-}
-
-static struct yt921x_acl_entry *
-yt921x_acl_entries_new(struct yt921x_acl_entry *entries, unsigned int *sizep,
-		       u32 type)
-{
-	unsigned int size = *sizep;
-
-	if (size >= YT921X_ACL_ENT_PER_BLK)
-		return NULL;
-
-	entries[size] = (typeof(*entries)){};
-	entries[size].key[1] = YT921X_ACL_KEYb_TYPE(type);
-
-	(*sizep)++;
-	return &entries[size];
-}
-
-static struct yt921x_acl_entry *
-yt921x_acl_entries_find(struct yt921x_acl_entry *entries, unsigned int *sizep,
-			u32 type)
-{
-	for (unsigned int i = 0; i < *sizep; i++)
-		if (FIELD_GET(YT921X_ACL_KEYb_TYPE_M, entries[i].key[1]) ==
-		    type)
-			return &entries[i];
-	return yt921x_acl_entries_new(entries, sizep, type);
-}
-
-static void
-yt921x_acl_rule_set_ports(struct yt921x_acl_rule *aclrule, u16 ord,
-			  u16 ports_mask)
-{
-	struct yt921x_acl_entry *entries = aclrule->entries;
-
-	for (unsigned int i = 0; i < hweight8(aclrule->mask); i++) {
-		entries[i].key[1] |= YT921X_ACL_KEYb_SPORTS(ports_mask) |
-				     YT921X_ACL_KEYb_ORD(ord);
-	}
-}
-
-struct yt921x_acl_rule_ext {
-	struct yt921x_acl_rule r;
-
-	struct yt921x_marker marker;
-};
-
-static int
-yt921x_acl_rule_ext_parse_flow_entries(struct yt921x_acl_rule_ext *ruleext,
-				       const struct flow_cls_offload *cls)
-{
-	const struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
-	struct yt921x_acl_entry *entries = ruleext->r.entries;
-	struct netlink_ext_ack *extack = cls->common.extack;
-	const struct flow_dissector *dissector;
-	struct yt921x_acl_entry *entry;
-	unsigned int size = 0;
-	bool use_dport;
-	bool use_sport;
-
-	/* Incomplete and probably won't, since it supports custom u32 filters.
-	 * New adapters are welcome.
-	 */
-	dissector = rule->match.dissector;
-	if (dissector->used_keys &
-	    ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS_RANGE) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_IP) |
-	      BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
-		NL_SET_ERR_MSG_MOD(extack, "Unsupported keys used");
-		return -EOPNOTSUPP;
-	}
-
-	/* Entries */
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
-		struct flow_match_ipv4_addrs match;
-
-		flow_rule_match_ipv4_addrs(rule, &match);
-
-		if (match.mask->dst) {
-			entry = yt921x_acl_entries_new(entries, &size,
-						       YT921X_ACL_TYPE_IPV4_DA);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= ntohl(match.key->dst);
-			entry->mask[0] |= ntohl(match.mask->dst);
-		}
-
-		if (match.mask->src) {
-			entry = yt921x_acl_entries_new(entries, &size,
-						       YT921X_ACL_TYPE_IPV4_SA);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= ntohl(match.key->src);
-			entry->mask[0] |= ntohl(match.mask->src);
-		}
-	}
-
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
-		struct flow_match_ipv6_addrs match;
-
-		flow_rule_match_ipv6_addrs(rule, &match);
-
-		for (unsigned int i = 0; i < 4; i++) {
-			if (!match.mask->dst.s6_addr32[i])
-				continue;
-
-			entry = yt921x_acl_entries_new(entries, &size,
-						       YT921X_ACL_TYPE_IPV6_DA0 + i);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= ntohl(match.key->dst.s6_addr32[i]);
-			entry->mask[0] |= ntohl(match.mask->dst.s6_addr32[i]);
-		}
-
-		for (unsigned int i = 0; i < 4; i++) {
-			if (!match.mask->src.s6_addr32[i])
-				continue;
-
-			entry = yt921x_acl_entries_new(entries, &size,
-						       YT921X_ACL_TYPE_IPV6_SA0 + i);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= ntohl(match.key->src.s6_addr32[i]);
-			entry->mask[0] |= ntohl(match.mask->src.s6_addr32[i]);
-		}
-	}
-
-	use_dport = false;
-	use_sport = false;
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
-		struct flow_match_ports match;
-
-		entry = yt921x_acl_entries_new(entries, &size,
-					       YT921X_ACL_TYPE_L4);
-		if (!entry)
-			goto err;
-
-		flow_rule_match_ports(rule, &match);
-
-		use_dport = !!match.mask->dst;
-		use_sport = !!match.mask->src;
-
-		entry->key[0] |= (ntohs(match.key->dst) << 16) |
-				 ntohs(match.key->src);
-		entry->mask[0] |= (ntohs(match.mask->dst) << 16) |
-				  ntohs(match.mask->src);
-	}
-
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS_RANGE)) {
-		struct flow_match_ports_range match;
-
-		entry = yt921x_acl_entries_find(entries, &size,
-						YT921X_ACL_TYPE_L4);
-		if (!entry)
-			goto err;
-
-		flow_rule_match_ports_range(rule, &match);
-
-		if ((use_dport && match.mask->tp.dst) ||
-		    (use_sport && match.mask->tp.src)) {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "Port mask and range are mutually exclusive");
-			return -EINVAL;
-		}
-
-		if (match.mask->tp.dst) {
-			entry->key[0] |= ntohs(match.key->tp_min.dst) << 16;
-			entry->key[1] |= YT921X_ACL_KEYb_L4_DPORT_RANGE_EN;
-			entry->mask[0] |= ntohs(match.key->tp_max.dst) << 16;
-		}
-
-		if (match.mask->tp.src) {
-			entry->key[0] |= ntohs(match.key->tp_min.src);
-			entry->key[1] |= YT921X_ACL_KEYb_L4_SPORT_RANGE_EN;
-			entry->mask[0] |= ntohs(match.key->tp_max.src);
-		}
-	}
-
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
-		struct flow_match_eth_addrs match;
-		u32 mask;
-
-		flow_rule_match_eth_addrs(rule, &match);
-
-		mask = ethaddr_hi4_to_u32(match.mask->dst);
-		if (mask) {
-			entry = yt921x_acl_entries_new(entries, &size,
-						       YT921X_ACL_TYPE_MAC_DA0);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= ethaddr_hi4_to_u32(match.key->dst);
-			entry->mask[0] |= mask;
-		}
-
-		mask = ethaddr_hi4_to_u32(match.mask->src);
-		if (mask) {
-			entry = yt921x_acl_entries_new(entries, &size,
-						       YT921X_ACL_TYPE_MAC_SA0);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= ethaddr_hi4_to_u32(match.key->src);
-			entry->mask[0] |= mask;
-		}
-
-		mask = (ethaddr_lo2_to_u32(match.mask->dst) << 16) |
-		       ethaddr_lo2_to_u32(match.mask->src);
-		if (mask) {
-			entry = yt921x_acl_entries_new(entries, &size,
-						       YT921X_ACL_TYPE_MAC_DA1_SA1);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= (ethaddr_lo2_to_u32(match.key->dst) << 16) |
-					 ethaddr_lo2_to_u32(match.key->src);
-			entry->mask[0] |= mask;
-		}
-	}
-
-	/* Entries + Misc */
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
-		struct flow_match_basic match;
-
-		flow_rule_match_basic(rule, &match);
-
-		if (match.mask->n_proto) {
-			enum yt921x_l3_type l3type = YT921X_L3_TYPE_OTHER;
-
-			if (match.mask->n_proto == htons(~0))
-				switch (match.key->n_proto) {
-				case htons(ETH_P_IP):
-					l3type = YT921X_L3_TYPE_IPV4;
-					break;
-				case htons(ETH_P_IPV6):
-					l3type = YT921X_L3_TYPE_IPV6;
-					break;
-				case htons(ETH_P_ARP):
-					l3type = YT921X_L3_TYPE_ARP;
-					break;
-				case htons(ETH_P_LLDP):
-					l3type = YT921X_L3_TYPE_LLDP;
-					break;
-				case htons(ETH_P_PAE):
-					l3type = YT921X_L3_TYPE_PAE;
-					break;
-				case htons(ETH_P_CFM):
-					l3type = YT921X_L3_TYPE_ERP;
-					break;
-				}
-
-			if (l3type != YT921X_L3_TYPE_OTHER) {
-				size = yt921x_acl_entries_set_l3_type(entries,
-								      size,
-								      l3type);
-				if (!size)
-					goto err;
-			} else {
-				entry = yt921x_acl_entries_new(entries, &size,
-							       YT921X_ACL_TYPE_ETHERTYPE);
-				if (!entry)
-					goto err;
-
-				entry->key[0] |= ntohs(match.key->n_proto);
-				entry->mask[0] |= ntohs(match.mask->n_proto);
-			}
-		}
-
-		if (match.mask->ip_proto) {
-			enum yt921x_l4_type l4type = YT921X_L4_TYPE_OTHER;
-
-			if (match.mask->ip_proto == (u8)~0)
-				switch (match.key->ip_proto) {
-				case IPPROTO_TCP:
-					l4type = YT921X_L4_TYPE_TCP;
-					break;
-				case IPPROTO_UDP:
-					l4type = YT921X_L4_TYPE_UDP;
-					break;
-				case IPPROTO_UDPLITE:
-					l4type = YT921X_L4_TYPE_UDPLITE;
-					break;
-				case IPPROTO_ICMP:
-					l4type = YT921X_L4_TYPE_ICMP;
-					break;
-				case IPPROTO_IGMP:
-					l4type = YT921X_L4_TYPE_IGMP;
-					break;
-				}
-
-			if (l4type != YT921X_L4_TYPE_OTHER) {
-				size = yt921x_acl_entries_set_l4_type(entries,
-								      size,
-								      l4type);
-				if (!size)
-					goto err;
-			} else {
-				entry = yt921x_acl_entries_find(entries, &size,
-								YT921X_ACL_TYPE_MISC);
-				if (!entry)
-					goto err;
-
-				entry->key[0] |= YT921X_ACL_BINa_MISC_IP_PROTO(match.key->ip_proto);
-				entry->mask[0] |= YT921X_ACL_BINa_MISC_IP_PROTO(match.mask->ip_proto);
-			}
-		}
-	}
-
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
-		u32 supp_flags = FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG;
-		struct flow_match_control match;
-
-		flow_rule_match_control(rule, &match);
-		if (!flow_rule_is_supp_control_flags(supp_flags,
-						     match.mask->flags, extack))
-			return -EOPNOTSUPP;
-
-		if (match.mask->flags & FLOW_DIS_IS_FRAGMENT) {
-			bool set = match.key->flags & FLOW_DIS_IS_FRAGMENT;
-
-			size = yt921x_acl_entries_set_is_fragment(entries, size,
-								  set);
-			if (!size)
-				goto err;
-		}
-		if (match.mask->flags & FLOW_DIS_FIRST_FRAG) {
-			bool set = match.key->flags & FLOW_DIS_FIRST_FRAG;
-
-			size = yt921x_acl_entries_set_first_frag(entries, size,
-								 set);
-			if (!size)
-				goto err;
-		}
-	}
-
-	/* Misc only */
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
-		struct flow_match_ip match;
-
-		flow_rule_match_ip(rule, &match);
-		if (match.mask->ttl) {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "Matching on TTL not supported");
-			return -EOPNOTSUPP;
-		}
-
-		if (match.mask->tos) {
-			entry = yt921x_acl_entries_find(entries, &size,
-							YT921X_ACL_TYPE_MISC);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= YT921X_ACL_BINa_MISC_TOS(match.key->tos);
-			entry->mask[0] |= YT921X_ACL_BINa_MISC_TOS(match.mask->tos);
-		}
-	}
-
-	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
-		struct flow_match_tcp match;
-
-		flow_rule_match_tcp(rule, &match);
-		if (match.mask->flags & htons(~0xff)) {
-			NL_SET_ERR_MSG_MOD(extack, "Unsupported TCP flags");
-			return -EOPNOTSUPP;
-		}
-
-		if (match.mask->flags) {
-			entry = yt921x_acl_entries_find(entries, &size,
-							YT921X_ACL_TYPE_MISC);
-			if (!entry)
-				goto err;
-
-			entry->key[0] |= YT921X_ACL_BINa_MISC_TCP_FLAGS(ntohs(match.key->flags));
-			entry->mask[0] |= YT921X_ACL_BINa_MISC_TCP_FLAGS(ntohs(match.mask->flags));
-		}
-	}
-
-	if (!size) {
-		NL_SET_ERR_MSG_MOD(extack, "Empty rule generated, this should not happen");
-		return -EOPNOTSUPP;
-	}
-
-	ruleext->r.mask = (1 << size) - 1;
-	return 0;
-
-err:
-	NL_SET_ERR_MSG_MOD(extack, "Rule too complex");
-	return -EOPNOTSUPP;
-}
-
-static int
-yt921x_acl_rule_ext_parse_flow_action(struct yt921x_acl_rule_ext *ruleext,
-				      const struct flow_cls_offload *cls,
-				      struct yt921x_priv *priv, int port)
-{
-	const struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
-	const struct flow_action *flow_action = &rule->action;
-	struct netlink_ext_ack *extack = cls->common.extack;
-	enum flow_action_id redir_act = NUM_FLOW_ACTIONS;
-	const struct flow_action_entry *act;
-	u32 *action = ruleext->r.action;
-	bool seen_priority = false;
-	const char *reason = NULL;
-	bool seen_police = false;
-	unsigned int i;
-	int res;
-
-	memset(action, 0, 3 * sizeof(*action));
-	flow_action_for_each(i, act, flow_action)
-		switch (act->id) {
-		case FLOW_ACTION_ACCEPT:
-		case FLOW_ACTION_DROP:
-		case FLOW_ACTION_REDIRECT:
-			if (redir_act != NUM_FLOW_ACTIONS &&
-			    redir_act != act->id) {
-				reason = "Different redirect actions";
-				goto fallback;
-			}
-			redir_act = act->id;
-
-			switch (act->id) {
-			case FLOW_ACTION_ACCEPT:
-				action[2] |= YT921X_ACL_ACTc_FWD_EN |
-					     YT921X_ACL_ACTc_FWD_FWD;
-				break;
-			case FLOW_ACTION_DROP:
-				action[2] |= YT921X_ACL_ACTc_FWD_EN |
-					     YT921X_ACL_ACTc_FWD_REDIR;
-				break;
-			case FLOW_ACTION_REDIRECT: {
-				struct dsa_port *to_dp;
-
-				to_dp = dsa_port_from_netdev(act->dev);
-				if (IS_ERR(to_dp) || to_dp->ds != &priv->ds) {
-					reason = "Redirect to non-local port";
-					goto fallback;
-				}
-
-				action[2] |= YT921X_ACL_ACTc_FWD_EN |
-					     YT921X_ACL_ACTc_FWD_REDIR |
-					     YT921X_ACL_ACTc_FWD_REDIR_DPORTn(to_dp->index);
-				break;
-			}
-			default:
-				break;
-			}
-			break;
-		case FLOW_ACTION_PRIORITY:
-			if (seen_priority) {
-				action[0] &= ~YT921X_ACL_ACTa_PRIO_EN;
-				action[1] &= ~YT921X_ACL_ACTb_PRIO_M;
-
-				reason = "Multiple priority actions";
-				goto fallback;
-			}
-			seen_priority = true;
-
-			if (act->priority >= YT921X_PRIO_NUM) {
-				NL_SET_ERR_MSG_MOD(extack,
-						   "Priority value is too high");
-				return -EOPNOTSUPP;
-			}
-			action[0] |= YT921X_ACL_ACTa_PRIO_EN;
-			action[1] |= YT921X_ACL_ACTb_PRIO(act->priority);
-			break;
-		case FLOW_ACTION_POLICE: {
-			const struct flow_action_police *police = &act->police;
-
-			if (seen_police) {
-				action[0] &= ~YT921X_ACL_ACTa_METER_EN;
-
-				reason = "Multiple police actions";
-				goto fallback;
-			}
-			seen_police = true;
-
-			res = yt921x_police_validate(police, flow_action, act,
-						     extack);
-			if (res)
-				return res;
-
-			res = yt921x_marker_tfm_police(&ruleext->marker, police,
-						       0, priv, port, extack);
-			if (res)
-				return res;
-
-			action[0] |= YT921X_ACL_ACTa_METER_EN;
-			break;
-		}
-		default:
-fallback:
-			if (cls->common.skip_sw) {
-				NL_SET_ERR_MSG_FMT_MOD(extack,
-						       "Action not supported when skip_sw: %s",
-						       reason);
-				return -EOPNOTSUPP;
-			}
-			fallthrough;
-		case FLOW_ACTION_TRAP:
-			redir_act = FLOW_ACTION_TRAP;
-
-			action[2] &= ~YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M &
-				     ~YT921X_ACL_ACTc_FWD_M;
-			action[2] |= YT921X_ACL_ACTc_FWD_EN |
-				     YT921X_ACL_ACTc_FWD_TRAP;
-			break;
-		}
-
-	ruleext->r.sw_assisted = !cls->common.skip_sw;
-	return 0;
-}
-
-static int
-yt921x_acl_rule_ext_parse_flow(struct yt921x_acl_rule_ext *ruleext, int port,
-			       const struct flow_cls_offload *cls, bool ingress,
-			       struct yt921x_priv *priv)
-{
-	struct netlink_ext_ack *extack = cls->common.extack;
-	int res;
-
-	if (!ingress) {
-		NL_SET_ERR_MSG_MOD(extack, "Only ingress is supported");
-		return -EOPNOTSUPP;
-	}
-
-	if (cls->common.chain_index) {
-		NL_SET_ERR_MSG(extack, "Only chain 0 is supported");
-		return -EOPNOTSUPP;
-	}
-
-	res = yt921x_acl_rule_ext_parse_flow_action(ruleext, cls, priv, port);
-	if (res)
-		return res;
-	res = yt921x_acl_rule_ext_parse_flow_entries(ruleext, cls);
-	if (res)
-		return res;
-
-	yt921x_acl_rule_set_ports(&ruleext->r, 0, BIT(port));
-	ruleext->r.tag = cls->cookie;
-	ruleext->r.type = TC_SETUP_CLSFLOWER;
-	return 0;
-}
-
-static unsigned int
-yt921x_acl_find(const struct yt921x_priv *priv, enum tc_setup_type type,
-		unsigned long tag)
-{
-	for (unsigned int blkid = 0; blkid < YT921X_ACL_BLK_NUM; blkid++) {
-		const struct yt921x_acl_blk *aclblk = priv->acl_blks[blkid];
-
-		if (!aclblk)
-			continue;
-
-		for (unsigned int i = 0; i < YT921X_ACL_ENT_PER_BLK; i++)
-			if (aclblk->rules[i] && aclblk->rules[i]->tag == tag &&
-			    aclblk->rules[i]->type == type)
-				return YT921X_ACL_ENT_PER_BLK * blkid + i;
-	}
-
-	return UINT_MAX;
-}
-
-static unsigned int
-yt921x_acl_reserve(struct yt921x_priv *priv, unsigned int entscnt,
-		   struct netlink_ext_ack *extack)
-{
-	int candidates[YT921X_ACL_ENT_PER_BLK + 1];
-	unsigned int acl_used_cnt = 0;
-
-	if (WARN_ON(entscnt > YT921X_ACL_ENT_PER_BLK))
-		return UINT_MAX;
-
-	for (unsigned int i = 0; i < ARRAY_SIZE(candidates); i++)
-		candidates[i] = -1;
-	for (unsigned int i = YT921X_ACL_BLK_NUM; i-- > 0;) {
-		unsigned int blk_used_cnt = hweight8(priv->acl_masks[i]);
-
-		candidates[blk_used_cnt] = i;
-		acl_used_cnt += blk_used_cnt;
-	}
-
-	if (acl_used_cnt >= YT921X_ACL_NUM) {
-		NL_SET_ERR_MSG_MOD(extack, "ACL entry limit reached");
-		return UINT_MAX;
-	}
-	if (acl_used_cnt + entscnt <= YT921X_ACL_NUM)
-		for (unsigned int i = YT921X_ACL_ENT_PER_BLK - entscnt + 1;
-		     i-- > 0;)
-			if (candidates[i] >= 0)
-				return YT921X_ACL_ENT_PER_BLK * candidates[i] +
-				       ffz(priv->acl_masks[candidates[i]]);
-
-	NL_SET_ERR_MSG_MOD(extack,
-			   "ACL entry allocation failed, simplify your rules or remove existing rules");
-	return UINT_MAX;
-}
-
-static int
-yt921x_acl_commit(struct yt921x_priv *priv, unsigned int entid, u8 entsmask)
-{
-	const struct yt921x_acl_rule *aclrule;
-	const struct yt921x_acl_blk *aclblk;
-	unsigned int blkid;
-	unsigned int binid;
-	unsigned long mask;
-	u32 zeros[3] = {};
-	unsigned int i;
-	unsigned int o;
-	u32 ctrl;
-	int res;
-
-	blkid = entid / YT921X_ACL_ENT_PER_BLK;
-	binid = entid % YT921X_ACL_ENT_PER_BLK;
-	aclblk = priv->acl_blks[blkid];
-	aclrule = aclblk->rules[binid];
-
-	/* Write actions */
-	res = yt921x_reg96_write(priv, YT921X_ACLn_ACT(entid),
-				 aclrule ? aclrule->action : zeros);
-	if (res)
-		return res;
-
-	/* Select the block */
-	ctrl = YT921X_ACL_BLK_CMD_MODIFY | YT921X_ACL_BLK_CMD_BLKID(blkid);
-	res = yt921x_reg_write(priv, YT921X_ACL_BLK_CMD, ctrl);
-	if (res)
-		return res;
-
-	/* Write keys and masks */
-	ctrl = 0;
-	for (unsigned int i = 0; i < YT921X_ACL_ENT_PER_BLK; i++)
-		ctrl |= YT921X_ACL_BLK_KEEP_KEEPn(i);
-
-	mask = entsmask;
-	i = 0;
-	for_each_set_bit(o, &mask, YT921X_ACL_ENT_PER_BLK) {
-		res = yt921x_reg64_write(priv, YT921X_ACLn_KEYm(blkid, o),
-					 aclrule ? aclrule->entries[i].key :
-					 zeros);
-		if (res)
-			return res;
-
-		res = yt921x_reg64_write(priv, YT921X_ACLn_MASKm(blkid, o),
-					 aclrule ? aclrule->entries[i].mask :
-					 zeros);
-		if (res)
-			return res;
-
-		ctrl &= ~YT921X_ACL_BLK_KEEP_KEEPn(o);
-		i++;
-	}
-
-	res = yt921x_reg_write(priv, YT921X_ACL_BLK_KEEP, ctrl);
-	if (res)
-		return res;
-
-	ctrl = 0;
-	for (unsigned int i = 0; i < YT921X_ACL_ENT_PER_BLK; i++) {
-		const struct yt921x_acl_rule *other = aclblk->rules[i];
-
-		if (!other)
-			continue;
-
-		mask = other->mask;
-		for_each_set_bit(o, &mask, YT921X_ACL_ENT_PER_BLK)
-			ctrl |= YT921X_ACL_ENTRY_ENm(o) |
-				YT921X_ACL_ENTRY_GRPIDm(o, i);
-	}
-	res = yt921x_reg_write(priv, YT921X_ACLn_ENTRY(blkid), ctrl);
-	if (res)
-		return res;
-
-	/* Commit the block */
-	ctrl = YT921X_ACL_BLK_CMD_BLKID(blkid);
-	res = yt921x_reg_write(priv, YT921X_ACL_BLK_CMD, ctrl);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int
-yt921x_acl_del(struct yt921x_priv *priv, enum tc_setup_type type,
-	       unsigned long tag)
-{
-	struct yt921x_acl_rule *aclrule;
-	struct yt921x_acl_blk *aclblk;
-	unsigned int binid;
-	unsigned int blkid;
-	unsigned int entid;
-	int res;
-
-	entid = yt921x_acl_find(priv, type, tag);
-	if (entid == UINT_MAX)
-		return -ENOENT;
-
-	blkid = entid / YT921X_ACL_ENT_PER_BLK;
-	binid = entid % YT921X_ACL_ENT_PER_BLK;
-	aclblk = priv->acl_blks[blkid];
-	aclrule = aclblk->rules[binid];
-
-	aclblk->rules[binid] = NULL;
-	res = yt921x_acl_commit(priv, entid, aclrule->mask);
-	/* the kernel never rolls back on failure */
-
-	if (aclrule->action[0] & YT921X_ACL_ACTa_METER_EN)
-		clear_bit(FIELD_GET(YT921X_ACL_ACTa_METER_ID_M,
-				    aclrule->action[0]),
-			  priv->meters_map);
-	priv->acl_masks[blkid] &= ~aclrule->mask;
-	kvfree(aclrule);
-	if (!priv->acl_masks[blkid]) {
-		kvfree(aclblk);
-		priv->acl_blks[blkid] = NULL;
-	}
-	return res;
-}
-
-static int
-yt921x_acl_add(struct yt921x_priv *priv,
-	       const struct yt921x_acl_rule_ext *ruleext,
-	       struct netlink_ext_ack *extack)
-{
-	unsigned int entscnt = hweight8(ruleext->r.mask);
-	struct yt921x_acl_rule *aclrule;
-	struct yt921x_acl_blk *aclblk;
-	bool use_trap = false;
-	unsigned int meterid;
-	unsigned long mask;
-	unsigned int binid;
-	unsigned int blkid;
-	unsigned int entid;
-	unsigned int o;
-	int res;
-
-	/* Allocate resources */
-	entid = yt921x_acl_reserve(priv, entscnt, extack);
-	if (entid == UINT_MAX)
-		return -EOPNOTSUPP;
-
-	if (!(ruleext->r.action[0] & YT921X_ACL_ACTa_METER_EN)) {
-		meterid = YT921X_METER_NUM;
-	} else {
-		meterid = find_first_zero_bit(priv->meters_map,
-					      YT921X_METER_NUM);
-		if (meterid < YT921X_METER_NUM) {
-			res = yt921x_meter_config(priv, meterid,
-						  &ruleext->marker);
-			if (res)
-				return res;
-		} else if (ruleext->r.sw_assisted) {
-			use_trap = true;
-		} else {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "No more meters available");
-			return -EOPNOTSUPP;
-		}
-	}
-
-	/* Prepare acl block ctrlblk */
-	blkid = entid / YT921X_ACL_ENT_PER_BLK;
-	binid = entid % YT921X_ACL_ENT_PER_BLK;
-	aclblk = priv->acl_blks[blkid];
-	if (!aclblk) {
-		aclblk = kvzalloc_obj(*aclblk);
-		if (!aclblk)
-			return -ENOMEM;
-		priv->acl_blks[blkid] = aclblk;
-	}
-
-	/* Prepare acl rule ctrlblk */
-	aclrule = kvmemdup(&ruleext->r,
-			   offsetof(struct yt921x_acl_rule, entries[entscnt]),
-			   GFP_KERNEL);
-	if (!aclrule) {
-		res = -ENOMEM;
-		goto err;
-	}
-
-	/* Replace the placeholder resource IDs */
-	aclrule->mask = 0;
-	mask = priv->acl_masks[blkid];
-	for_each_clear_bit(o, &mask, YT921X_ACL_ENT_PER_BLK) {
-		aclrule->mask |= BIT(o);
-		entscnt--;
-		if (!entscnt)
-			break;
-	}
-
-	if (use_trap) {
-		aclrule->action[2] &= ~YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M &
-				      ~YT921X_ACL_ACTc_FWD_M;
-		aclrule->action[2] |= YT921X_ACL_ACTc_FWD_EN |
-				      YT921X_ACL_ACTc_FWD_TRAP;
-	}
-	if (meterid < YT921X_METER_NUM)
-		aclrule->action[0] |= YT921X_ACL_ACTa_METER_ID(meterid);
-	else
-		aclrule->action[0] &= ~YT921X_ACL_ACTa_METER_EN;
-
-	/* Write rules */
-	aclblk->rules[binid] = aclrule;
-	res = yt921x_acl_commit(priv, entid, aclrule->mask);
-	if (res) {
-		aclblk->rules[binid] = NULL;
-		kvfree(aclrule);
-		goto err;
-	}
-
-	if (meterid < YT921X_METER_NUM)
-		set_bit(meterid, priv->meters_map);
-	priv->acl_masks[blkid] |= aclrule->mask;
-	return 0;
-
-err:
-	if (!priv->acl_masks[blkid]) {
-		kvfree(aclblk);
-		priv->acl_blks[blkid] = NULL;
-	}
-	return res;
-}
-
-static int
-yt921x_dsa_cls_flower_del(struct dsa_switch *ds, int port,
-			  struct flow_cls_offload *cls, bool ingress)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_acl_del(priv, TC_SETUP_CLSFLOWER, cls->cookie);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_cls_flower_add(struct dsa_switch *ds, int port,
-			  struct flow_cls_offload *cls, bool ingress)
-{
-	struct netlink_ext_ack *extack = cls->common.extack;
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct yt921x_acl_rule_ext ruleext;
-	int res;
-
-	res = yt921x_acl_rule_ext_parse_flow(&ruleext, port, cls, ingress,
-					     priv);
-	if (res)
-		return res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_acl_add(priv, &ruleext, extack);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_mirror_del(struct yt921x_priv *priv, int port, bool ingress)
-{
-	u32 mask;
-
-	if (ingress)
-		mask = YT921X_MIRROR_IGR_PORTn(port);
-	else
-		mask = YT921X_MIRROR_EGR_PORTn(port);
-	return yt921x_reg_clear_bits(priv, YT921X_MIRROR, mask);
-}
-
-static int
-yt921x_mirror_add(struct yt921x_priv *priv, int port, bool ingress,
-		  int to_local_port, struct netlink_ext_ack *extack)
-{
-	u32 srcs;
-	u32 ctrl;
-	u32 val;
-	u32 dst;
-	int res;
-
-	if (ingress)
-		srcs = YT921X_MIRROR_IGR_PORTn(port);
-	else
-		srcs = YT921X_MIRROR_EGR_PORTn(port);
-	dst = YT921X_MIRROR_PORT(to_local_port);
-
-	res = yt921x_reg_read(priv, YT921X_MIRROR, &val);
-	if (res)
-		return res;
-
-	/* other mirror tasks & different dst port -> conflict */
-	if ((val & ~srcs & (YT921X_MIRROR_EGR_PORTS_M |
-			    YT921X_MIRROR_IGR_PORTS_M)) &&
-	    (val & YT921X_MIRROR_PORT_M) != dst) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Sniffer port is already configured, delete existing rules & retry");
-		return -EBUSY;
-	}
-
-	ctrl = val & ~YT921X_MIRROR_PORT_M;
-	ctrl |= srcs;
-	ctrl |= dst;
-
-	if (ctrl == val)
-		return 0;
-
-	return yt921x_reg_write(priv, YT921X_MIRROR, ctrl);
-}
-
-static void
-yt921x_dsa_port_mirror_del(struct dsa_switch *ds, int port,
-			   struct dsa_mall_mirror_tc_entry *mirror)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct device *dev = to_device(priv);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_mirror_del(priv, port, mirror->ingress);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dev, "Failed to %s port %d: %i\n", "unmirror",
-			port, res);
-}
-
-static int
-yt921x_dsa_port_mirror_add(struct dsa_switch *ds, int port,
-			   struct dsa_mall_mirror_tc_entry *mirror,
-			   bool ingress, struct netlink_ext_ack *extack)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_mirror_add(priv, port, ingress,
-				mirror->to_local_port, extack);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int yt921x_lag_hash(struct yt921x_priv *priv, u32 ctrl, bool unique_lag,
-			   struct netlink_ext_ack *extack)
-{
-	u32 val;
-	int res;
-
-	/* Hash Mode is global. Make sure the same Hash Mode is set to all the
-	 * 2 possible lags.
-	 * If we are the unique LAG we can set whatever hash mode we want.
-	 * To change hash mode it's needed to remove all LAG and change the mode
-	 * with the latest.
-	 */
-	if (unique_lag) {
-		res = yt921x_reg_write(priv, YT921X_LAG_HASH, ctrl);
-		if (res)
-			return res;
-	} else {
-		res = yt921x_reg_read(priv, YT921X_LAG_HASH, &val);
-		if (res)
-			return res;
-
-		if (val != ctrl) {
-			NL_SET_ERR_MSG_MOD(extack,
-					   "Mismatched Hash Mode across different lags is not supported");
-			return -EOPNOTSUPP;
-		}
-	}
-
-	return 0;
-}
-
-static int yt921x_lag_set(struct yt921x_priv *priv, u8 index, u16 ports_mask)
-{
-	unsigned long targets_mask = ports_mask;
-	unsigned int cnt;
-	u32 ctrl;
-	int port;
-	int res;
-
-	cnt = 0;
-	for_each_set_bit(port, &targets_mask, YT921X_PORT_NUM) {
-		ctrl = YT921X_LAG_MEMBER_PORT(port);
-		res = yt921x_reg_write(priv, YT921X_LAG_MEMBERnm(index, cnt),
-				       ctrl);
-		if (res)
-			return res;
-
-		cnt++;
-	}
-
-	ctrl = YT921X_LAG_GROUP_PORTS(ports_mask) |
-	       YT921X_LAG_GROUP_MEMBER_NUM(cnt);
-	return yt921x_reg_write(priv, YT921X_LAG_GROUPn(index), ctrl);
-}
-
-static int
-yt921x_dsa_port_lag_leave(struct dsa_switch *ds, int port, struct dsa_lag lag)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct dsa_port *dp;
-	u32 ctrl;
-	int res;
-
-	if (!lag.id)
-		return -EINVAL;
-
-	ctrl = 0;
-	dsa_lag_foreach_port(dp, ds->dst, &lag)
-		ctrl |= BIT(dp->index);
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_lag_set(priv, lag.id - 1, ctrl);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_lag_check(struct dsa_switch *ds, struct dsa_lag lag,
-			  struct netdev_lag_upper_info *info,
-			  struct netlink_ext_ack *extack)
-{
-	unsigned int members;
-	struct dsa_port *dp;
-
-	if (!lag.id)
-		return -EINVAL;
-
-	members = 0;
-	dsa_lag_foreach_port(dp, ds->dst, &lag)
-		/* Includes the port joining the LAG */
-		members++;
-
-	if (members > YT921X_LAG_PORT_NUM) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Cannot offload more than 4 LAG ports");
-		return -EOPNOTSUPP;
-	}
-
-	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Can only offload LAG using hash TX type");
-		return -EOPNOTSUPP;
-	}
-
-	if (info->hash_type != NETDEV_LAG_HASH_L2 &&
-	    info->hash_type != NETDEV_LAG_HASH_L23 &&
-	    info->hash_type != NETDEV_LAG_HASH_L34) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Can only offload L2 or L2+L3 or L3+L4 TX hash");
-		return -EOPNOTSUPP;
-	}
-
-	return 0;
-}
-
-static int
-yt921x_dsa_port_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag,
-			 struct netdev_lag_upper_info *info,
-			 struct netlink_ext_ack *extack)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct dsa_port *dp;
-	bool unique_lag;
-	unsigned int i;
-	u32 ctrl;
-	int res;
-
-	res = yt921x_dsa_port_lag_check(ds, lag, info, extack);
-	if (res)
-		return res;
-
-	ctrl = 0;
-	switch (info->hash_type) {
-	case NETDEV_LAG_HASH_L34:
-		ctrl |= YT921X_LAG_HASH_IP_DST;
-		ctrl |= YT921X_LAG_HASH_IP_SRC;
-		ctrl |= YT921X_LAG_HASH_IP_PROTO;
-
-		ctrl |= YT921X_LAG_HASH_L4_DPORT;
-		ctrl |= YT921X_LAG_HASH_L4_SPORT;
-		break;
-	case NETDEV_LAG_HASH_L23:
-		ctrl |= YT921X_LAG_HASH_MAC_DA;
-		ctrl |= YT921X_LAG_HASH_MAC_SA;
-
-		ctrl |= YT921X_LAG_HASH_IP_DST;
-		ctrl |= YT921X_LAG_HASH_IP_SRC;
-		ctrl |= YT921X_LAG_HASH_IP_PROTO;
-		break;
-	case NETDEV_LAG_HASH_L2:
-		ctrl |= YT921X_LAG_HASH_MAC_DA;
-		ctrl |= YT921X_LAG_HASH_MAC_SA;
-		break;
-	default:
-		return -EOPNOTSUPP;
-	}
-
-	/* Check if we are the unique configured LAG */
-	unique_lag = true;
-	dsa_lags_foreach_id(i, ds->dst)
-		if (i != lag.id && dsa_lag_by_id(ds->dst, i)) {
-			unique_lag = false;
-			break;
-		}
-
-	mutex_lock(&priv->reg_lock);
-	do {
-		res = yt921x_lag_hash(priv, ctrl, unique_lag, extack);
-		if (res)
-			break;
-
-		ctrl = 0;
-		dsa_lag_foreach_port(dp, ds->dst, &lag)
-			ctrl |= BIT(dp->index);
-		res = yt921x_lag_set(priv, lag.id - 1, ctrl);
-	} while (0);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int yt921x_fdb_wait(struct yt921x_priv *priv, u32 *valp)
-{
-	struct device *dev = to_device(priv);
-	u32 val = YT921X_FDB_RESULT_DONE;
-	int res;
-
-	res = yt921x_reg_wait(priv, YT921X_FDB_RESULT, YT921X_FDB_RESULT_DONE,
-			      &val);
-	if (res) {
-		dev_err(dev, "FDB probably stuck\n");
-		return res;
-	}
-
-	*valp = val;
-	return 0;
-}
-
-static int
-yt921x_fdb_in01(struct yt921x_priv *priv, const unsigned char *addr,
-		u16 vid, u32 ctrl1)
-{
-	u32 ctrl;
-	int res;
-
-	ctrl = ethaddr_hi4_to_u32(addr);
-	res = yt921x_reg_write(priv, YT921X_FDB_IN0, ctrl);
-	if (res)
-		return res;
-
-	ctrl = ctrl1 | YT921X_FDB_IO1_FID(vid) | ethaddr_lo2_to_u32(addr);
-	return yt921x_reg_write(priv, YT921X_FDB_IN1, ctrl);
-}
-
-static int
-yt921x_fdb_has(struct yt921x_priv *priv, const unsigned char *addr, u16 vid,
-	       u16 *indexp)
-{
-	u32 ctrl;
-	u32 val;
-	int res;
-
-	res = yt921x_fdb_in01(priv, addr, vid, 0);
-	if (res)
-		return res;
-
-	ctrl = 0;
-	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
-	if (res)
-		return res;
-
-	ctrl = YT921X_FDB_OP_OP_GET_ONE | YT921X_FDB_OP_START;
-	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
-	if (res)
-		return res;
-
-	res = yt921x_fdb_wait(priv, &val);
-	if (res)
-		return res;
-	if (val & YT921X_FDB_RESULT_NOTFOUND) {
-		*indexp = YT921X_FDB_NUM;
-		return 0;
-	}
-
-	*indexp = FIELD_GET(YT921X_FDB_RESULT_INDEX_M, val);
-	return 0;
-}
-
-static int
-yt921x_fdb_read(struct yt921x_priv *priv, unsigned char *addr, u16 *vidp,
-		u16 *ports_maskp, u16 *indexp, u8 *statusp)
-{
-	struct device *dev = to_device(priv);
-	u16 index;
-	u32 data0;
-	u32 data1;
-	u32 data2;
-	u32 val;
-	int res;
-
-	res = yt921x_fdb_wait(priv, &val);
-	if (res)
-		return res;
-	if (val & YT921X_FDB_RESULT_NOTFOUND) {
-		*ports_maskp = 0;
-		return 0;
-	}
-	index = FIELD_GET(YT921X_FDB_RESULT_INDEX_M, val);
-
-	res = yt921x_reg_read(priv, YT921X_FDB_OUT1, &data1);
-	if (res)
-		return res;
-	if ((data1 & YT921X_FDB_IO1_STATUS_M) ==
-	    YT921X_FDB_IO1_STATUS_INVALID) {
-		*ports_maskp = 0;
-		return 0;
-	}
-
-	res = yt921x_reg_read(priv, YT921X_FDB_OUT0, &data0);
-	if (res)
-		return res;
-	res = yt921x_reg_read(priv, YT921X_FDB_OUT2, &data2);
-	if (res)
-		return res;
-
-	addr[0] = data0 >> 24;
-	addr[1] = data0 >> 16;
-	addr[2] = data0 >> 8;
-	addr[3] = data0;
-	addr[4] = data1 >> 8;
-	addr[5] = data1;
-	*vidp = FIELD_GET(YT921X_FDB_IO1_FID_M, data1);
-	*indexp = index;
-	*ports_maskp = FIELD_GET(YT921X_FDB_IO2_EGR_PORTS_M, data2);
-	*statusp = FIELD_GET(YT921X_FDB_IO1_STATUS_M, data1);
-
-	dev_dbg(dev,
-		"%s: index 0x%x, mac %02x:%02x:%02x:%02x:%02x:%02x, vid %d, ports 0x%x, status %d\n",
-		__func__, *indexp, addr[0], addr[1], addr[2], addr[3],
-		addr[4], addr[5], *vidp, *ports_maskp, *statusp);
-	return 0;
-}
-
-static int
-yt921x_fdb_dump(struct yt921x_priv *priv, u16 ports_mask,
-		dsa_fdb_dump_cb_t *cb, void *data)
-{
-	unsigned char addr[ETH_ALEN];
-	u8 status;
-	u16 pmask;
-	u16 index;
-	u32 ctrl;
-	u16 vid;
-	int res;
-
-	ctrl = YT921X_FDB_OP_INDEX(0) | YT921X_FDB_OP_MODE_INDEX |
-	       YT921X_FDB_OP_OP_GET_ONE | YT921X_FDB_OP_START;
-	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
-	if (res)
-		return res;
-	res = yt921x_fdb_read(priv, addr, &vid, &pmask, &index, &status);
-	if (res)
-		return res;
-	if ((pmask & ports_mask) && !is_multicast_ether_addr(addr)) {
-		res = cb(addr, vid,
-			 status == YT921X_FDB_ENTRY_STATUS_STATIC, data);
-		if (res)
-			return res;
-	}
-
-	ctrl = YT921X_FDB_IO2_EGR_PORTS(ports_mask);
-	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
-	if (res)
-		return res;
-
-	index = 0;
-	do {
-		ctrl = YT921X_FDB_OP_INDEX(index) | YT921X_FDB_OP_MODE_INDEX |
-		       YT921X_FDB_OP_NEXT_TYPE_UCAST_PORT |
-		       YT921X_FDB_OP_OP_GET_NEXT | YT921X_FDB_OP_START;
-		res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
-		if (res)
-			return res;
-
-		res = yt921x_fdb_read(priv, addr, &vid, &pmask, &index,
-				      &status);
-		if (res)
-			return res;
-		if (!pmask)
-			break;
-
-		if ((pmask & ports_mask) && !is_multicast_ether_addr(addr)) {
-			res = cb(addr, vid,
-				 status == YT921X_FDB_ENTRY_STATUS_STATIC,
-				 data);
-			if (res)
-				return res;
-		}
-
-		/* Never call GET_NEXT with 4095, otherwise it will hang
-		 * forever until a reset!
-		 */
-	} while (index < YT921X_FDB_NUM - 1);
-
-	return 0;
-}
-
-static int
-yt921x_fdb_flush_raw(struct yt921x_priv *priv, u16 ports_mask, u16 vid,
-		     bool flush_static)
-{
-	u32 ctrl;
-	u32 val;
-	int res;
-
-	if (vid < 4096) {
-		ctrl = YT921X_FDB_IO1_FID(vid);
-		res = yt921x_reg_write(priv, YT921X_FDB_IN1, ctrl);
-		if (res)
-			return res;
-	}
-
-	ctrl = YT921X_FDB_IO2_EGR_PORTS(ports_mask);
-	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
-	if (res)
-		return res;
-
-	ctrl = YT921X_FDB_OP_OP_FLUSH | YT921X_FDB_OP_START;
-	if (vid >= 4096)
-		ctrl |= YT921X_FDB_OP_FLUSH_PORT;
-	else
-		ctrl |= YT921X_FDB_OP_FLUSH_PORT_VID;
-	if (flush_static)
-		ctrl |= YT921X_FDB_OP_FLUSH_STATIC;
-	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
-	if (res)
-		return res;
-
-	res = yt921x_fdb_wait(priv, &val);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int
-yt921x_fdb_flush_port(struct yt921x_priv *priv, int port, bool flush_static)
-{
-	return yt921x_fdb_flush_raw(priv, BIT(port), 4096, flush_static);
-}
-
-static int
-yt921x_fdb_add_index_in12(struct yt921x_priv *priv, u16 index, u16 ctrl1,
-			  u16 ctrl2)
-{
-	u32 ctrl;
-	u32 val;
-	int res;
-
-	res = yt921x_reg_write(priv, YT921X_FDB_IN1, ctrl1);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl2);
-	if (res)
-		return res;
-
-	ctrl = YT921X_FDB_OP_INDEX(index) | YT921X_FDB_OP_MODE_INDEX |
-	       YT921X_FDB_OP_OP_ADD | YT921X_FDB_OP_START;
-	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
-	if (res)
-		return res;
-
-	return yt921x_fdb_wait(priv, &val);
-}
-
-static int
-yt921x_fdb_add(struct yt921x_priv *priv, const unsigned char *addr, u16 vid,
-	       u16 ports_mask)
-{
-	u32 ctrl;
-	u32 val;
-	int res;
-
-	ctrl = YT921X_FDB_IO1_STATUS_STATIC;
-	res = yt921x_fdb_in01(priv, addr, vid, ctrl);
-	if (res)
-		return res;
-
-	ctrl = YT921X_FDB_IO2_EGR_PORTS(ports_mask);
-	res = yt921x_reg_write(priv, YT921X_FDB_IN2, ctrl);
-	if (res)
-		return res;
-
-	ctrl = YT921X_FDB_OP_OP_ADD | YT921X_FDB_OP_START;
-	res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
-	if (res)
-		return res;
-
-	return yt921x_fdb_wait(priv, &val);
-}
-
-static int
-yt921x_fdb_leave(struct yt921x_priv *priv, const unsigned char *addr,
-		 u16 vid, u16 ports_mask)
-{
-	u16 index;
-	u32 ctrl1;
-	u32 ctrl2;
-	u32 ctrl;
-	u32 val2;
-	u32 val;
-	int res;
-
-	/* Check for presence */
-	res = yt921x_fdb_has(priv, addr, vid, &index);
-	if (res)
-		return res;
-	if (index >= YT921X_FDB_NUM)
-		return 0;
-
-	/* Check if action required */
-	res = yt921x_reg_read(priv, YT921X_FDB_OUT2, &val2);
-	if (res)
-		return res;
-
-	ctrl2 = val2 & ~YT921X_FDB_IO2_EGR_PORTS(ports_mask);
-	if (ctrl2 == val2)
-		return 0;
-	if (!(ctrl2 & YT921X_FDB_IO2_EGR_PORTS_M)) {
-		ctrl = YT921X_FDB_OP_OP_DEL | YT921X_FDB_OP_START;
-		res = yt921x_reg_write(priv, YT921X_FDB_OP, ctrl);
-		if (res)
-			return res;
-
-		return yt921x_fdb_wait(priv, &val);
-	}
-
-	res = yt921x_reg_read(priv, YT921X_FDB_OUT1, &ctrl1);
-	if (res)
-		return res;
-
-	return yt921x_fdb_add_index_in12(priv, index, ctrl1, ctrl2);
-}
-
-static int
-yt921x_fdb_join(struct yt921x_priv *priv, const unsigned char *addr, u16 vid,
-		u16 ports_mask)
-{
-	u16 index;
-	u32 ctrl1;
-	u32 ctrl2;
-	u32 val1;
-	u32 val2;
-	int res;
-
-	/* Check for presence */
-	res = yt921x_fdb_has(priv, addr, vid, &index);
-	if (res)
-		return res;
-	if (index >= YT921X_FDB_NUM)
-		return yt921x_fdb_add(priv, addr, vid, ports_mask);
-
-	/* Check if action required */
-	res = yt921x_reg_read(priv, YT921X_FDB_OUT1, &val1);
-	if (res)
-		return res;
-	res = yt921x_reg_read(priv, YT921X_FDB_OUT2, &val2);
-	if (res)
-		return res;
-
-	ctrl1 = val1 & ~YT921X_FDB_IO1_STATUS_M;
-	ctrl1 |= YT921X_FDB_IO1_STATUS_STATIC;
-	ctrl2 = val2 | YT921X_FDB_IO2_EGR_PORTS(ports_mask);
-	if (ctrl1 == val1 && ctrl2 == val2)
-		return 0;
-
-	return yt921x_fdb_add_index_in12(priv, index, ctrl1, ctrl2);
-}
-
-static int
-yt921x_dsa_port_fdb_dump(struct dsa_switch *ds, int port,
-			 dsa_fdb_dump_cb_t *cb, void *data)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	/* Hardware FDB is shared for fdb and mdb, "bridge fdb show"
-	 * only wants to see unicast
-	 */
-	res = yt921x_fdb_dump(priv, BIT(port), cb, data);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static void yt921x_dsa_port_fast_age(struct dsa_switch *ds, int port)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct device *dev = to_device(priv);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_fdb_flush_port(priv, port, false);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dev, "Failed to %s port %d: %i\n", "clear FDB for",
-			port, res);
-}
-
-static int
-yt921x_dsa_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u32 ctrl;
-	int res;
-
-	/* AGEING reg is set in 5s step */
-	ctrl = clamp(msecs / 5000, 1, U16_MAX);
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_write(priv, YT921X_AGEING, ctrl);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_fdb_del(struct dsa_switch *ds, int port,
-			const unsigned char *addr, u16 vid, struct dsa_db db)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_fdb_leave(priv, addr, vid, BIT(port));
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_fdb_add(struct dsa_switch *ds, int port,
-			const unsigned char *addr, u16 vid, struct dsa_db db)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_fdb_join(priv, addr, vid, BIT(port));
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_mdb_del(struct dsa_switch *ds, int port,
-			const struct switchdev_obj_port_mdb *mdb,
-			struct dsa_db db)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	const unsigned char *addr = mdb->addr;
-	u16 vid = mdb->vid;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_fdb_leave(priv, addr, vid, BIT(port));
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_mdb_add(struct dsa_switch *ds, int port,
-			const struct switchdev_obj_port_mdb *mdb,
-			struct dsa_db db)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	const unsigned char *addr = mdb->addr;
-	u16 vid = mdb->vid;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_fdb_join(priv, addr, vid, BIT(port));
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_vlan_aware_set(struct yt921x_priv *priv, int port, bool vlan_aware)
-{
-	u32 ctrl;
-
-	/* Abuse SVLAN for PCP parsing without polluting the FDB - it just works
-	 * despite YT921X_VLAN_CTRL_SVLAN_EN never being set
-	 */
-	if (!vlan_aware)
-		ctrl = YT921X_PORT_IGR_TPIDn_STAG(0);
-	else
-		ctrl = YT921X_PORT_IGR_TPIDn_CTAG(0);
-	return yt921x_reg_write(priv, YT921X_PORTn_IGR_TPID(port), ctrl);
-}
-
-static int
-yt921x_port_set_pvid(struct yt921x_priv *priv, int port, u16 vid)
-{
-	u32 mask;
-	u32 ctrl;
-
-	mask = YT921X_PORT_VLAN_CTRL_CVID_M;
-	ctrl = YT921X_PORT_VLAN_CTRL_CVID(vid);
-	return yt921x_reg_update_bits(priv, YT921X_PORTn_VLAN_CTRL(port),
-				      mask, ctrl);
-}
-
-static int
-yt921x_vlan_filtering(struct yt921x_priv *priv, int port, bool vlan_filtering)
-{
-	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
-	struct net_device *bdev;
-	u16 pvid;
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	bdev = dsa_port_bridge_dev_get(dp);
-
-	if (!bdev || !vlan_filtering)
-		pvid = YT921X_VID_UNWARE;
-	else
-		br_vlan_get_pvid(bdev, &pvid);
-	res = yt921x_port_set_pvid(priv, port, pvid);
-	if (res)
-		return res;
-
-	mask = YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_TAGGED |
-	       YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
-	ctrl = 0;
-	/* Do not drop tagged frames here; let VLAN_IGR_FILTER do it */
-	if (vlan_filtering && !pvid)
-		ctrl |= YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
-	res = yt921x_reg_update_bits(priv, YT921X_PORTn_VLAN_CTRL1(port),
-				     mask, ctrl);
-	if (res)
-		return res;
-
-	res = yt921x_reg_toggle_bits(priv, YT921X_VLAN_IGR_FILTER,
-				     YT921X_VLAN_IGR_FILTER_PORTn(port),
-				     vlan_filtering);
-	if (res)
-		return res;
-
-	res = yt921x_vlan_aware_set(priv, port, vlan_filtering);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int yt921x_vlan_del(struct yt921x_priv *priv, int port, u16 vid)
-{
-	u32 masks[2];
-
-	masks[0] = YT921X_VLAN_CTRLa_PORTn(port);
-	masks[1] = YT921X_VLAN_CTRLb_UNTAG_PORTn(port);
-
-	return yt921x_reg64_clear_bits(priv, YT921X_VLANn_CTRL(vid), masks);
-}
-
-static int
-yt921x_vlan_add(struct yt921x_priv *priv, int port, u16 vid, bool untagged)
-{
-	u32 masks[2];
-	u32 ctrls[2];
-
-	masks[0] = YT921X_VLAN_CTRLa_PORTn(port) |
-		   YT921X_VLAN_CTRLa_PORTS(priv->cpu_ports_mask);
-	ctrls[0] = masks[0];
-
-	masks[1] = YT921X_VLAN_CTRLb_UNTAG_PORTn(port);
-	ctrls[1] = untagged ? masks[1] : 0;
-
-	return yt921x_reg64_update_bits(priv, YT921X_VLANn_CTRL(vid),
-					masks, ctrls);
-}
-
-static int
-yt921x_pvid_clear(struct yt921x_priv *priv, int port)
-{
-	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
-	bool vlan_filtering;
-	u32 mask;
-	int res;
-
-	vlan_filtering = dsa_port_is_vlan_filtering(dp);
-
-	res = yt921x_port_set_pvid(priv, port,
-				   vlan_filtering ? 0 : YT921X_VID_UNWARE);
-	if (res)
-		return res;
-
-	if (vlan_filtering) {
-		mask = YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
-		res = yt921x_reg_set_bits(priv, YT921X_PORTn_VLAN_CTRL1(port),
-					  mask);
-		if (res)
-			return res;
-	}
-
-	return 0;
-}
-
-static int
-yt921x_pvid_set(struct yt921x_priv *priv, int port, u16 vid)
-{
-	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
-	bool vlan_filtering;
-	u32 mask;
-	int res;
-
-	vlan_filtering = dsa_port_is_vlan_filtering(dp);
-
-	if (vlan_filtering) {
-		res = yt921x_port_set_pvid(priv, port, vid);
-		if (res)
-			return res;
-	}
-
-	mask = YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED;
-	res = yt921x_reg_clear_bits(priv, YT921X_PORTn_VLAN_CTRL1(port), mask);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int
-yt921x_dsa_port_vlan_filtering(struct dsa_switch *ds, int port,
-			       bool vlan_filtering,
-			       struct netlink_ext_ack *extack)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	if (dsa_is_cpu_port(ds, port))
-		return 0;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_vlan_filtering(priv, port, vlan_filtering);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_vlan_del(struct dsa_switch *ds, int port,
-			 const struct switchdev_obj_port_vlan *vlan)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u16 vid = vlan->vid;
-	u16 pvid;
-	int res;
-
-	if (dsa_is_cpu_port(ds, port))
-		return 0;
-
-	mutex_lock(&priv->reg_lock);
-	do {
-		struct dsa_port *dp = dsa_to_port(ds, port);
-		struct net_device *bdev;
-
-		res = yt921x_vlan_del(priv, port, vid);
-		if (res)
-			break;
-
-		bdev = dsa_port_bridge_dev_get(dp);
-		if (bdev) {
-			br_vlan_get_pvid(bdev, &pvid);
-			if (pvid == vid)
-				res = yt921x_pvid_clear(priv, port);
-		}
-	} while (0);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_vlan_add(struct dsa_switch *ds, int port,
-			 const struct switchdev_obj_port_vlan *vlan,
-			 struct netlink_ext_ack *extack)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u16 vid = vlan->vid;
-	u16 pvid;
-	int res;
-
-	/* CPU port is supposed to be a member of every VLAN; see
-	 * yt921x_vlan_add() and yt921x_port_setup()
-	 */
-	if (dsa_is_cpu_port(ds, port))
-		return 0;
-
-	mutex_lock(&priv->reg_lock);
-	do {
-		struct dsa_port *dp = dsa_to_port(ds, port);
-		struct net_device *bdev;
-
-		res = yt921x_vlan_add(priv, port, vid,
-				      vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED);
-		if (res)
-			break;
-
-		bdev = dsa_port_bridge_dev_get(dp);
-		if (bdev) {
-			if (vlan->flags & BRIDGE_VLAN_INFO_PVID) {
-				res = yt921x_pvid_set(priv, port, vid);
-			} else {
-				br_vlan_get_pvid(bdev, &pvid);
-				if (pvid == vid)
-					res = yt921x_pvid_clear(priv, port);
-			}
-		}
-	} while (0);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int yt921x_userport_standalone(struct yt921x_priv *priv, int port)
-{
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	ctrl = ~priv->cpu_ports_mask;
-	res = yt921x_reg_write(priv, YT921X_PORTn_ISOLATION(port), ctrl);
-	if (res)
-		return res;
-
-	/* Turn off FDB learning to prevent FDB pollution */
-	mask = YT921X_PORT_LEARN_DIS;
-	res = yt921x_reg_set_bits(priv, YT921X_PORTn_LEARN(port), mask);
-	if (res)
-		return res;
-
-	/* Turn off VLAN awareness */
-	res = yt921x_vlan_aware_set(priv, port, false);
-	if (res)
-		return res;
-
-	/* Unrelated since learning is off and all packets are trapped;
-	 * set it anyway
-	 */
-	res = yt921x_port_set_pvid(priv, port, YT921X_VID_UNWARE);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int yt921x_userport_bridge(struct yt921x_priv *priv, int port)
-{
-	u32 mask;
-	int res;
-
-	mask = YT921X_PORT_LEARN_DIS;
-	res = yt921x_reg_clear_bits(priv, YT921X_PORTn_LEARN(port), mask);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int yt921x_isolate(struct yt921x_priv *priv, int port)
-{
-	u32 mask;
-	int res;
-
-	mask = BIT(port);
-	for (int i = 0; i < YT921X_PORT_NUM; i++) {
-		if ((BIT(i) & priv->cpu_ports_mask) || i == port)
-			continue;
-
-		res = yt921x_reg_set_bits(priv, YT921X_PORTn_ISOLATION(i),
-					  mask);
-		if (res)
-			return res;
-	}
-
-	return 0;
-}
-
-/* Make sure to include the CPU port in ports_mask, or your bridge will
- * not have it.
- */
-static int yt921x_bridge(struct yt921x_priv *priv, u16 ports_mask)
-{
-	unsigned long targets_mask = ports_mask & ~priv->cpu_ports_mask;
-	u32 isolated_mask;
-	u32 ctrl;
-	int port;
-	int res;
-
-	isolated_mask = 0;
-	for_each_set_bit(port, &targets_mask, YT921X_PORT_NUM) {
-		struct yt921x_port *pp = &priv->ports[port];
-
-		if (pp->isolated)
-			isolated_mask |= BIT(port);
-	}
-
-	/* Block from non-cpu bridge ports ... */
-	for_each_set_bit(port, &targets_mask, YT921X_PORT_NUM) {
-		struct yt921x_port *pp = &priv->ports[port];
-
-		/* to non-bridge ports */
-		ctrl = ~ports_mask;
-		/* to isolated ports when isolated */
-		if (pp->isolated)
-			ctrl |= isolated_mask;
-		/* to itself when non-hairpin */
-		if (!pp->hairpin)
-			ctrl |= BIT(port);
-		else
-			ctrl &= ~BIT(port);
-
-		res = yt921x_reg_write(priv, YT921X_PORTn_ISOLATION(port),
-				       ctrl);
-		if (res)
-			return res;
-	}
-
-	return 0;
-}
-
-static int yt921x_bridge_leave(struct yt921x_priv *priv, int port)
-{
-	int res;
-
-	res = yt921x_userport_standalone(priv, port);
-	if (res)
-		return res;
-
-	res = yt921x_isolate(priv, port);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int
-yt921x_bridge_join(struct yt921x_priv *priv, int port, u16 ports_mask)
-{
-	int res;
-
-	res = yt921x_userport_bridge(priv, port);
-	if (res)
-		return res;
-
-	res = yt921x_bridge(priv, ports_mask);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int
-yt921x_bridge_flags(struct yt921x_priv *priv, int port,
-		    struct switchdev_brport_flags flags)
-{
-	struct yt921x_port *pp = &priv->ports[port];
-	bool do_flush;
-	u32 mask;
-	int res;
-
-	if (flags.mask & BR_LEARNING) {
-		bool learning = flags.val & BR_LEARNING;
-
-		mask = YT921X_PORT_LEARN_DIS;
-		res = yt921x_reg_toggle_bits(priv, YT921X_PORTn_LEARN(port),
-					     mask, !learning);
-		if (res)
-			return res;
-	}
-
-	/* BR_FLOOD, BR_MCAST_FLOOD: see the comment where ACT_UNK_ACTn_TRAP
-	 * is set
-	 */
-
-	/* BR_BCAST_FLOOD: we can filter bcast, but cannot trap them */
-
-	do_flush = false;
-	if (flags.mask & BR_HAIRPIN_MODE) {
-		pp->hairpin = flags.val & BR_HAIRPIN_MODE;
-		do_flush = true;
-	}
-	if (flags.mask & BR_ISOLATED) {
-		pp->isolated = flags.val & BR_ISOLATED;
-		do_flush = true;
-	}
-	if (do_flush) {
-		struct dsa_switch *ds = &priv->ds;
-		struct dsa_port *dp = dsa_to_port(ds, port);
-		struct net_device *bdev;
-
-		bdev = dsa_port_bridge_dev_get(dp);
-		if (bdev) {
-			u32 ports_mask;
-
-			ports_mask = dsa_bridge_ports(ds, bdev);
-			ports_mask |= priv->cpu_ports_mask;
-			res = yt921x_bridge(priv, ports_mask);
-			if (res)
-				return res;
-		}
-	}
-
-	return 0;
-}
-
-static int
-yt921x_dsa_port_pre_bridge_flags(struct dsa_switch *ds, int port,
-				 struct switchdev_brport_flags flags,
-				 struct netlink_ext_ack *extack)
-{
-	if (flags.mask & ~(BR_HAIRPIN_MODE | BR_LEARNING | BR_FLOOD |
-			   BR_MCAST_FLOOD | BR_ISOLATED))
-		return -EINVAL;
-	return 0;
-}
-
-static int
-yt921x_dsa_port_bridge_flags(struct dsa_switch *ds, int port,
-			     struct switchdev_brport_flags flags,
-			     struct netlink_ext_ack *extack)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	if (dsa_is_cpu_port(ds, port))
-		return 0;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_bridge_flags(priv, port, flags);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static void
-yt921x_dsa_port_bridge_leave(struct dsa_switch *ds, int port,
-			     struct dsa_bridge bridge)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct device *dev = to_device(priv);
-	int res;
-
-	if (dsa_is_cpu_port(ds, port))
-		return;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_bridge_leave(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dev, "Failed to %s port %d: %i\n", "unbridge",
-			port, res);
-}
-
-static int
-yt921x_dsa_port_bridge_join(struct dsa_switch *ds, int port,
-			    struct dsa_bridge bridge, bool *tx_fwd_offload,
-			    struct netlink_ext_ack *extack)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u16 ports_mask;
-	int res;
-
-	if (dsa_is_cpu_port(ds, port))
-		return 0;
-
-	ports_mask = dsa_bridge_ports(ds, bridge.dev);
-	ports_mask |= priv->cpu_ports_mask;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_bridge_join(priv, port, ports_mask);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_port_mst_state_set(struct dsa_switch *ds, int port,
-			      const struct switchdev_mst_state *st)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	mask = YT921X_STP_PORTn_M(port);
-	switch (st->state) {
-	case BR_STATE_DISABLED:
-		ctrl = YT921X_STP_PORTn_DISABLED(port);
-		break;
-	case BR_STATE_LISTENING:
-	case BR_STATE_LEARNING:
-		ctrl = YT921X_STP_PORTn_LEARNING(port);
-		break;
-	case BR_STATE_FORWARDING:
-	default:
-		ctrl = YT921X_STP_PORTn_FORWARD(port);
-		break;
-	case BR_STATE_BLOCKING:
-		ctrl = YT921X_STP_PORTn_BLOCKING(port);
-		break;
-	}
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_update_bits(priv, YT921X_STPn(st->msti), mask, ctrl);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int
-yt921x_dsa_vlan_msti_set(struct dsa_switch *ds, struct dsa_bridge bridge,
-			 const struct switchdev_vlan_msti *msti)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u32 masks[2];
-	u32 ctrls[2];
-	int res;
-
-	if (!msti->vid)
-		return -EINVAL;
-	if (!msti->msti || msti->msti >= YT921X_MSTI_NUM)
-		return -EINVAL;
-
-	masks[0] = 0;
-	ctrls[0] = 0;
-	masks[1] = YT921X_VLAN_CTRLb_STP_ID_M;
-	ctrls[1] = YT921X_VLAN_CTRLb_STP_ID(msti->msti);
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg64_update_bits(priv, YT921X_VLANn_CTRL(msti->vid),
-				       masks, ctrls);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static void
-yt921x_dsa_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct dsa_port *dp = dsa_to_port(ds, port);
-	struct device *dev = to_device(priv);
-	bool learning;
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	mask = YT921X_STP_PORTn_M(port);
-	learning = false;
-	switch (state) {
-	case BR_STATE_DISABLED:
-		ctrl = YT921X_STP_PORTn_DISABLED(port);
-		break;
-	case BR_STATE_LISTENING:
-		ctrl = YT921X_STP_PORTn_LEARNING(port);
-		break;
-	case BR_STATE_LEARNING:
-		ctrl = YT921X_STP_PORTn_LEARNING(port);
-		learning = dp->learning;
-		break;
-	case BR_STATE_FORWARDING:
-	default:
-		ctrl = YT921X_STP_PORTn_FORWARD(port);
-		learning = dp->learning;
-		break;
-	case BR_STATE_BLOCKING:
-		ctrl = YT921X_STP_PORTn_BLOCKING(port);
-		break;
-	}
-
-	mutex_lock(&priv->reg_lock);
-	do {
-		res = yt921x_reg_update_bits(priv, YT921X_STPn(0), mask, ctrl);
-		if (res)
-			break;
-
-		mask = YT921X_PORT_LEARN_DIS;
-		ctrl = !learning ? YT921X_PORT_LEARN_DIS : 0;
-		res = yt921x_reg_update_bits(priv, YT921X_PORTn_LEARN(port),
-					     mask, ctrl);
-	} while (0);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dev, "Failed to %s port %d: %i\n", "set STP state for",
-			port, res);
-}
-
-static int __maybe_unused
-yt921x_dsa_port_get_default_prio(struct dsa_switch *ds, int port)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u32 val;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_read(priv, YT921X_PORTn_QOS(port), &val);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		return res;
-
-	return FIELD_GET(YT921X_PORT_QOS_PRIO_M, val);
-}
-
-static int __maybe_unused
-yt921x_dsa_port_set_default_prio(struct dsa_switch *ds, int port, u8 prio)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	if (prio >= YT921X_PRIO_NUM)
-		return -EINVAL;
-
-	mutex_lock(&priv->reg_lock);
-	mask = YT921X_PORT_QOS_PRIO_M | YT921X_PORT_QOS_PRIO_EN;
-	ctrl = YT921X_PORT_QOS_PRIO(prio) | YT921X_PORT_QOS_PRIO_EN;
-	res = yt921x_reg_update_bits(priv, YT921X_PORTn_QOS(port), mask, ctrl);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int __maybe_unused appprios_cmp(const void *a, const void *b)
-{
-	return ((const u8 *)b)[1] - ((const u8 *)a)[1];
-}
-
-static int __maybe_unused
-yt921x_dsa_port_get_apptrust(struct dsa_switch *ds, int port, u8 *sel,
-			     int *nselp)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u8 appprios[2][2] = {};
-	int nsel;
-	u32 val;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_read(priv, YT921X_PORTn_PRIO_ORD(port), &val);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		return res;
-
-	appprios[0][0] = IEEE_8021QAZ_APP_SEL_DSCP;
-	appprios[0][1] = (val >> (3 * YT921X_APP_SEL_DSCP)) & 7;
-	appprios[1][0] = DCB_APP_SEL_PCP;
-	appprios[1][1] = (val >> (3 * YT921X_APP_SEL_CVLAN_PCP)) & 7;
-	sort(appprios, ARRAY_SIZE(appprios), sizeof(appprios[0]), appprios_cmp,
-	     NULL);
-
-	nsel = 0;
-	for (int i = 0; i < ARRAY_SIZE(appprios) && appprios[i][1]; i++) {
-		sel[nsel] = appprios[i][0];
-		nsel++;
-	}
-	*nselp = nsel;
-
-	return 0;
-}
-
-static int __maybe_unused
-yt921x_dsa_port_set_apptrust(struct dsa_switch *ds, int port, const u8 *sel,
-			     int nsel)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct device *dev = to_device(priv);
-	u32 ctrl;
-	int res;
-
-	if (nsel > YT921X_APP_SEL_NUM)
-		return -EINVAL;
-
-	ctrl = 0;
-	for (int i = 0; i < nsel; i++) {
-		switch (sel[i]) {
-		case IEEE_8021QAZ_APP_SEL_DSCP:
-			ctrl |= YT921X_PORT_PRIO_ORD_APPm(YT921X_APP_SEL_DSCP,
-							  7 - i);
-			break;
-		case DCB_APP_SEL_PCP:
-			ctrl |= YT921X_PORT_PRIO_ORD_APPm(YT921X_APP_SEL_CVLAN_PCP,
-							  7 - i);
-			ctrl |= YT921X_PORT_PRIO_ORD_APPm(YT921X_APP_SEL_SVLAN_PCP,
-							  7 - i);
-			break;
-		default:
-			dev_err(dev,
-				"Invalid apptrust selector (at %d-th). Supported: dscp, pcp\n",
-				i + 1);
-			return -EOPNOTSUPP;
-		}
-	}
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_write(priv, YT921X_PORTn_PRIO_ORD(port), ctrl);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int yt921x_port_down(struct yt921x_priv *priv, int port)
-{
-	u32 mask;
-	int res;
-
-	mask = YT921X_PORT_LINK | YT921X_PORT_RX_MAC_EN | YT921X_PORT_TX_MAC_EN;
-	res = yt921x_reg_clear_bits(priv, YT921X_PORTn_CTRL(port), mask);
-	if (res)
-		return res;
-
-	if (yt921x_port_is_external(port)) {
-		mask = YT921X_SERDES_LINK;
-		res = yt921x_reg_clear_bits(priv, YT921X_SERDESn(port), mask);
-		if (res)
-			return res;
-
-		mask = YT921X_XMII_LINK;
-		res = yt921x_reg_clear_bits(priv, YT921X_XMIIn(port), mask);
-		if (res)
-			return res;
-	}
-
-	return 0;
-}
-
-static int
-yt921x_port_up(struct yt921x_priv *priv, int port, unsigned int mode,
-	       phy_interface_t interface, int speed, int duplex,
-	       bool tx_pause, bool rx_pause)
-{
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	switch (speed) {
-	case SPEED_10:
-		ctrl = YT921X_PORT_SPEED_10;
-		break;
-	case SPEED_100:
-		ctrl = YT921X_PORT_SPEED_100;
-		break;
-	case SPEED_1000:
-		ctrl = YT921X_PORT_SPEED_1000;
-		break;
-	case SPEED_2500:
-		ctrl = YT921X_PORT_SPEED_2500;
-		break;
-	case SPEED_10000:
-		ctrl = YT921X_PORT_SPEED_10000;
-		break;
-	default:
-		return -EINVAL;
-	}
-	if (duplex == DUPLEX_FULL)
-		ctrl |= YT921X_PORT_DUPLEX_FULL;
-	if (tx_pause)
-		ctrl |= YT921X_PORT_TX_PAUSE;
-	if (rx_pause)
-		ctrl |= YT921X_PORT_RX_PAUSE;
-	ctrl |= YT921X_PORT_RX_MAC_EN | YT921X_PORT_TX_MAC_EN;
-	res = yt921x_reg_write(priv, YT921X_PORTn_CTRL(port), ctrl);
-	if (res)
-		return res;
-
-	if (yt921x_port_is_external(port)) {
-		mask = YT921X_SERDES_SPEED_M;
-		switch (speed) {
-		case SPEED_10:
-			ctrl = YT921X_SERDES_SPEED_10;
-			break;
-		case SPEED_100:
-			ctrl = YT921X_SERDES_SPEED_100;
-			break;
-		case SPEED_1000:
-			ctrl = YT921X_SERDES_SPEED_1000;
-			break;
-		case SPEED_2500:
-			ctrl = YT921X_SERDES_SPEED_2500;
-			break;
-		case SPEED_10000:
-			ctrl = YT921X_SERDES_SPEED_10000;
-			break;
-		default:
-			return -EINVAL;
-		}
-		mask |= YT921X_SERDES_DUPLEX_FULL;
-		if (duplex == DUPLEX_FULL)
-			ctrl |= YT921X_SERDES_DUPLEX_FULL;
-		mask |= YT921X_SERDES_TX_PAUSE;
-		if (tx_pause)
-			ctrl |= YT921X_SERDES_TX_PAUSE;
-		mask |= YT921X_SERDES_RX_PAUSE;
-		if (rx_pause)
-			ctrl |= YT921X_SERDES_RX_PAUSE;
-		mask |= YT921X_SERDES_LINK;
-		ctrl |= YT921X_SERDES_LINK;
-		res = yt921x_reg_update_bits(priv, YT921X_SERDESn(port),
-					     mask, ctrl);
-		if (res)
-			return res;
-
-		mask = YT921X_XMII_LINK;
-		res = yt921x_reg_set_bits(priv, YT921X_XMIIn(port), mask);
-		if (res)
-			return res;
-
-		switch (speed) {
-		case SPEED_10:
-			ctrl = YT921X_MDIO_POLLING_SPEED_10;
-			break;
-		case SPEED_100:
-			ctrl = YT921X_MDIO_POLLING_SPEED_100;
-			break;
-		case SPEED_1000:
-			ctrl = YT921X_MDIO_POLLING_SPEED_1000;
-			break;
-		case SPEED_2500:
-			ctrl = YT921X_MDIO_POLLING_SPEED_2500;
-			break;
-		case SPEED_10000:
-			ctrl = YT921X_MDIO_POLLING_SPEED_10000;
-			break;
-		default:
-			return -EINVAL;
-		}
-		if (duplex == DUPLEX_FULL)
-			ctrl |= YT921X_MDIO_POLLING_DUPLEX_FULL;
-		ctrl |= YT921X_MDIO_POLLING_LINK;
-		res = yt921x_reg_write(priv, YT921X_MDIO_POLLINGn(port), ctrl);
-		if (res)
-			return res;
-	}
-
-	return 0;
-}
-
-static int
-yt921x_port_config(struct yt921x_priv *priv, int port, unsigned int mode,
-		   phy_interface_t interface)
-{
-	struct device *dev = to_device(priv);
-	u32 mask;
-	u32 ctrl;
-	int res;
-
-	if (!yt921x_port_is_external(port)) {
-		if (interface != PHY_INTERFACE_MODE_INTERNAL) {
-			dev_err(dev, "Wrong mode %d on port %d\n",
-				interface, port);
-			return -EINVAL;
-		}
-		return 0;
-	}
-
-	switch (interface) {
-	/* SERDES */
-	case PHY_INTERFACE_MODE_SGMII:
-	case PHY_INTERFACE_MODE_100BASEX:
-	case PHY_INTERFACE_MODE_1000BASEX:
-	case PHY_INTERFACE_MODE_2500BASEX:
-		mask = YT921X_SERDES_CTRL_PORTn(port);
-		res = yt921x_reg_set_bits(priv, YT921X_SERDES_CTRL, mask);
-		if (res)
-			return res;
-
-		mask = YT921X_XMII_CTRL_PORTn(port);
-		res = yt921x_reg_clear_bits(priv, YT921X_XMII_CTRL, mask);
-		if (res)
-			return res;
-
-		mask = YT921X_SERDES_MODE_M;
-		switch (interface) {
-		case PHY_INTERFACE_MODE_SGMII:
-			ctrl = YT921X_SERDES_MODE_SGMII;
-			break;
-		case PHY_INTERFACE_MODE_100BASEX:
-			ctrl = YT921X_SERDES_MODE_100BASEX;
-			break;
-		case PHY_INTERFACE_MODE_1000BASEX:
-			ctrl = YT921X_SERDES_MODE_1000BASEX;
-			break;
-		case PHY_INTERFACE_MODE_2500BASEX:
-			ctrl = YT921X_SERDES_MODE_2500BASEX;
-			break;
-		default:
-			return -EINVAL;
-		}
-		res = yt921x_reg_update_bits(priv, YT921X_SERDESn(port),
-					     mask, ctrl);
-		if (res)
-			return res;
-
-		break;
-	/* add XMII support here */
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static void
-yt921x_phylink_mac_link_down(struct phylink_config *config, unsigned int mode,
-			     phy_interface_t interface)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct yt921x_priv *priv = to_yt921x_priv(dp->ds);
-	int port = dp->index;
-	int res;
-
-	/* No need to sync; port control block is hold until device remove */
-	cancel_delayed_work(&priv->ports[port].mib_read);
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_port_down(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dp->ds->dev, "Failed to %s port %d: %i\n", "bring down",
-			port, res);
-}
-
-static void
-yt921x_phylink_mac_link_up(struct phylink_config *config,
-			   struct phy_device *phydev, unsigned int mode,
-			   phy_interface_t interface, int speed, int duplex,
-			   bool tx_pause, bool rx_pause)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct yt921x_priv *priv = to_yt921x_priv(dp->ds);
-	int port = dp->index;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_port_up(priv, port, mode, interface, speed, duplex,
-			     tx_pause, rx_pause);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dp->ds->dev, "Failed to %s port %d: %i\n", "bring up",
-			port, res);
-
-	schedule_delayed_work(&priv->ports[port].mib_read, 0);
-}
-
-static void
-yt921x_phylink_mac_config(struct phylink_config *config, unsigned int mode,
-			  const struct phylink_link_state *state)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct yt921x_priv *priv = to_yt921x_priv(dp->ds);
-	int port = dp->index;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_port_config(priv, port, mode, state->interface);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		dev_err(dp->ds->dev, "Failed to %s port %d: %i\n", "config",
-			port, res);
-}
-
-static void
-yt921x_dsa_phylink_get_caps(struct dsa_switch *ds, int port,
-			    struct phylink_config *config)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	const struct yt921x_info *info = priv->info;
-
-	config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
-				   MAC_10 | MAC_100 | MAC_1000;
-
-	if (info->internal_mask & BIT(port)) {
-		/* Port 10 for MCU should probably go here too. But since that
-		 * is untested yet, turn it down for the moment by letting it
-		 * fall to the default branch.
-		 */
-		__set_bit(PHY_INTERFACE_MODE_INTERNAL,
-			  config->supported_interfaces);
-	} else if (info->external_mask & BIT(port)) {
-		/* TODO: external ports may support SERDES only, XMII only, or
-		 * SERDES + XMII depending on the chip. However, we can't get
-		 * the accurate config table due to lack of document, thus
-		 * we simply declare SERDES + XMII and rely on the correctness
-		 * of devicetree for now.
-		 */
-
-		/* SERDES */
-		__set_bit(PHY_INTERFACE_MODE_SGMII,
-			  config->supported_interfaces);
-		/* REVSGMII (SGMII in PHY role) should go here, once
-		 * PHY_INTERFACE_MODE_REVSGMII is introduced.
-		 */
-		__set_bit(PHY_INTERFACE_MODE_100BASEX,
-			  config->supported_interfaces);
-		__set_bit(PHY_INTERFACE_MODE_1000BASEX,
-			  config->supported_interfaces);
-		__set_bit(PHY_INTERFACE_MODE_2500BASEX,
-			  config->supported_interfaces);
-		config->mac_capabilities |= MAC_2500FD;
-
-		/* XMII */
-
-		/* Not tested. To add support for XMII:
-		 *   - Add proper interface modes below
-		 *   - Handle them in yt921x_port_config()
-		 */
-	}
-	/* no such port: empty supported_interfaces causes phylink to turn it
-	 * down
-	 */
-}
-
-static int yt921x_port_setup(struct yt921x_priv *priv, int port)
-{
-	struct dsa_switch *ds = &priv->ds;
-	u32 ctrl;
-	int res;
-
-	res = yt921x_userport_standalone(priv, port);
-	if (res)
-		return res;
-
-	/* Clear prio order (even if DCB is not enabled) to avoid unsolicited
-	 * priorities
-	 */
-	res = yt921x_reg_write(priv, YT921X_PORTn_PRIO_ORD(port), 0);
-	if (res)
-		return res;
-
-	if (dsa_is_cpu_port(ds, port)) {
-		/* Egress of CPU port is supposed to be completely controlled
-		 * via tagging, so set to oneway isolated (drop all packets
-		 * without tag).
-		 */
-		ctrl = ~(u32)0;
-		res = yt921x_reg_write(priv, YT921X_PORTn_ISOLATION(port),
-				       ctrl);
-		if (res)
-			return res;
-
-		/* To simplify FDB "isolation" simulation, we also disable
-		 * learning on the CPU port, and let software identify packets
-		 * towarding CPU (either trapped or a static FDB entry is
-		 * matched, no matter which bridge that entry is for), which is
-		 * already done by yt921x_userport_standalone(). As a result,
-		 * VLAN-awareness becomes unrelated on the CPU port (set to
-		 * VLAN-unaware by the way).
-		 */
-	}
-
-	return 0;
-}
-
-static enum dsa_tag_protocol
-yt921x_dsa_get_tag_protocol(struct dsa_switch *ds, int port,
-			    enum dsa_tag_protocol m)
-{
-	return DSA_TAG_PROTO_YT921X;
-}
-
-static int yt921x_dsa_port_setup(struct dsa_switch *ds, int port)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_port_setup(priv, port);
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-/* Not "port" - DSCP mapping is global */
-static int __maybe_unused
-yt921x_dsa_port_get_dscp_prio(struct dsa_switch *ds, int port, u8 dscp)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u32 val;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_read(priv, YT921X_IPM_DSCPn(dscp), &val);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		return res;
-
-	return FIELD_GET(YT921X_IPM_PRIO_M, val);
-}
-
-static int __maybe_unused
-yt921x_dsa_port_del_dscp_prio(struct dsa_switch *ds, int port, u8 dscp, u8 prio)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u32 val;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	/* During a "dcb app replace" command, the new app table entry will be
-	 * added first, then the old one will be deleted. But the hardware only
-	 * supports one QoS class per DSCP value (duh), so if we blindly delete
-	 * the app table entry for this DSCP value, we end up deleting the
-	 * entry with the new priority. Avoid that by checking whether user
-	 * space wants to delete the priority which is currently configured, or
-	 * something else which is no longer current.
-	 */
-	res = yt921x_reg_read(priv, YT921X_IPM_DSCPn(dscp), &val);
-	if (!res && FIELD_GET(YT921X_IPM_PRIO_M, val) == prio)
-		res = yt921x_reg_write(priv, YT921X_IPM_DSCPn(dscp),
-				       YT921X_IPM_PRIO(IEEE8021Q_TT_BK));
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int __maybe_unused
-yt921x_dsa_port_add_dscp_prio(struct dsa_switch *ds, int port, u8 dscp, u8 prio)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	int res;
-
-	if (prio >= YT921X_PRIO_NUM)
-		return -EINVAL;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_reg_write(priv, YT921X_IPM_DSCPn(dscp),
-			       YT921X_IPM_PRIO(prio));
-	mutex_unlock(&priv->reg_lock);
-
-	return res;
-}
-
-static int yt921x_edata_wait(struct yt921x_priv *priv, u32 *valp)
-{
-	u32 val = YT921X_EDATA_DATA_IDLE;
-	int res;
-
-	res = yt921x_reg_wait(priv, YT921X_EDATA_DATA,
-			      YT921X_EDATA_DATA_STATUS_M, &val);
-	if (res)
-		return res;
-
-	*valp = val;
-	return 0;
-}
-
-static int
-yt921x_edata_read_cont(struct yt921x_priv *priv, u8 addr, u8 *valp)
-{
-	u32 ctrl;
-	u32 val;
-	int res;
-
-	ctrl = YT921X_EDATA_CTRL_ADDR(addr) | YT921X_EDATA_CTRL_READ;
-	res = yt921x_reg_write(priv, YT921X_EDATA_CTRL, ctrl);
-	if (res)
-		return res;
-	res = yt921x_edata_wait(priv, &val);
-	if (res)
-		return res;
-
-	*valp = FIELD_GET(YT921X_EDATA_DATA_DATA_M, val);
-	return 0;
-}
-
-static int yt921x_edata_read(struct yt921x_priv *priv, u8 addr, u8 *valp)
-{
-	u32 val;
-	int res;
-
-	res = yt921x_edata_wait(priv, &val);
-	if (res)
-		return res;
-	return yt921x_edata_read_cont(priv, addr, valp);
-}
-
-static int yt921x_chip_detect(struct yt921x_priv *priv)
-{
-	struct device *dev = to_device(priv);
-	const struct yt921x_info *info;
-	u8 extmode;
-	u32 chipid;
-	u32 major;
-	u32 mode;
-	u32 val;
-	int res;
-
-	res = yt921x_reg_read(priv, YT921X_CHIP_ID, &chipid);
-	if (res)
-		return res;
-
-	major = FIELD_GET(YT921X_CHIP_ID_MAJOR, chipid);
-
-	for (info = yt921x_infos; info->name; info++)
-		if (info->major == major)
-			break;
-	if (!info->name) {
-		dev_err(dev, "Unexpected chipid 0x%x\n", chipid);
-		return -ENODEV;
-	}
-
-	res = yt921x_reg_read(priv, YT921X_CHIP_MODE, &mode);
-	if (res)
-		return res;
-	res = yt921x_edata_read(priv, YT921X_EDATA_EXTMODE, &extmode);
-	if (res)
-		return res;
-
-	for (; info->name; info++)
-		if (info->major == major && info->mode == mode &&
-		    info->extmode == extmode)
-			break;
-	if (!info->name) {
-		dev_err(dev,
-			"Unsupported chipid 0x%x with chipmode 0x%x 0x%x\n",
-			chipid, mode, extmode);
-		return -ENODEV;
-	}
-
-	res = yt921x_reg_read(priv, YT921X_SYS_CLK, &val);
-	if (res)
-		return res;
-	switch (FIELD_GET(YT921X_SYS_CLK_SEL_M, val)) {
-	case 0:
-		priv->cycle_ns = info->major == YT9215_MAJOR ? 8 : 6;
-		break;
-	case YT921X_SYS_CLK_143M:
-		priv->cycle_ns = 7;
-		break;
-	default:
-		priv->cycle_ns = 8;
-	}
-
-	/* Print chipid here since we are interested in lower 16 bits */
-	dev_info(dev,
-		 "Motorcomm %s ethernet switch, chipid: 0x%x, chipmode: 0x%x 0x%x\n",
-		 info->name, chipid, mode, extmode);
-
-	priv->info = info;
-
-	return 0;
-}
-
-static int yt921x_chip_reset(struct yt921x_priv *priv)
-{
-	struct device *dev = to_device(priv);
-	u16 eth_p_tag;
-	u32 val;
-	int res;
-
-	res = yt921x_chip_detect(priv);
-	if (res)
-		return res;
-
-	/* Reset */
-	res = yt921x_reg_write(priv, YT921X_RST, YT921X_RST_HW);
-	if (res)
-		return res;
-
-	/* RST_HW is almost same as GPIO hard reset, so we need this delay. */
-	fsleep(YT921X_RST_DELAY_US);
-
-	val = 0;
-	res = yt921x_reg_wait(priv, YT921X_RST, ~0, &val);
-	if (res)
-		return res;
-
-	/* Check for tag EtherType; do it after reset in case you messed it up
-	 * before.
-	 */
-	res = yt921x_reg_read(priv, YT921X_CPU_TAG_TPID, &val);
-	if (res)
-		return res;
-	eth_p_tag = FIELD_GET(YT921X_CPU_TAG_TPID_TPID_M, val);
-	if (eth_p_tag != ETH_P_YT921X) {
-		dev_err(dev, "Tag type 0x%x != 0x%x\n", eth_p_tag,
-			ETH_P_YT921X);
-		/* Despite being possible, we choose not to set CPU_TAG_TPID,
-		 * since there is no way it can be different unless you have the
-		 * wrong chip.
-		 */
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int yt921x_chip_setup_dsa(struct yt921x_priv *priv)
-{
-	struct dsa_switch *ds = &priv->ds;
-	unsigned long cpu_ports_mask;
-	u32 ctrls[2];
-	u32 ctrl;
-	int port;
-	int res;
-
-	/* Enable DSA */
-	priv->cpu_ports_mask = dsa_cpu_ports(ds);
-
-	ctrl = YT921X_EXT_CPU_PORT_TAG_EN | YT921X_EXT_CPU_PORT_PORT_EN |
-	       YT921X_EXT_CPU_PORT_PORT(__ffs(priv->cpu_ports_mask));
-	res = yt921x_reg_write(priv, YT921X_EXT_CPU_PORT, ctrl);
-	if (res)
-		return res;
-
-	/* Setup software switch */
-	ctrl = YT921X_CPU_COPY_TO_EXT_CPU;
-	res = yt921x_reg_write(priv, YT921X_CPU_COPY, ctrl);
-	if (res)
-		return res;
-
-	ctrl = GENMASK(10, 0);
-	res = yt921x_reg_write(priv, YT921X_FILTER_UNK_UCAST, ctrl);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_FILTER_UNK_MCAST, ctrl);
-	if (res)
-		return res;
-
-	/* YT921x does not support native DSA port bridging, so we use port
-	 * isolation to emulate it. However, be especially careful that port
-	 * isolation takes _after_ FDB lookups, i.e. if an FDB entry (from
-	 * another bridge) is matched and the destination port (in another
-	 * bridge) is blocked, the packet will be dropped instead of flooding to
-	 * the "bridged" ports, thus we need to trap and handle those packets by
-	 * software.
-	 *
-	 * If there is no more than one bridge, we might be able to drop them
-	 * directly given some conditions are met, but we trap them in all cases
-	 * for now.
-	 */
-	ctrl = 0;
-	for (int i = 0; i < YT921X_PORT_NUM; i++)
-		ctrl |= YT921X_ACT_UNK_ACTn_TRAP(i);
-	/* Except for CPU ports, if any packets are sent via CPU ports without
-	 * tag, they should be dropped.
-	 */
-	cpu_ports_mask = priv->cpu_ports_mask;
-	for_each_set_bit(port, &cpu_ports_mask, YT921X_PORT_NUM) {
-		ctrl &= ~YT921X_ACT_UNK_ACTn_M(port);
-		ctrl |= YT921X_ACT_UNK_ACTn_DROP(port);
-	}
-	res = yt921x_reg_write(priv, YT921X_ACT_UNK_UCAST, ctrl);
-	if (res)
-		return res;
-	res = yt921x_reg_write(priv, YT921X_ACT_UNK_MCAST, ctrl);
-	if (res)
-		return res;
-
-	/* Tagged VID 0 should be treated as untagged, which confuses the
-	 * hardware a lot
-	 */
-	ctrls[0] = YT921X_VLAN_CTRLa_LEARN_DIS | YT921X_VLAN_CTRLa_PORTS_M;
-	ctrls[1] = 0;
-	res = yt921x_reg64_write(priv, YT921X_VLANn_CTRL(0), ctrls);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int yt921x_chip_setup_tc(struct yt921x_priv *priv)
-{
-	unsigned int op_ns;
-	u32 ctrl;
-	int res;
-
-	op_ns = 8 * priv->cycle_ns;
-
-	ctrl = max(priv->meter_slot_ns / op_ns, YT921X_METER_SLOT_MIN);
-	res = yt921x_reg_write(priv, YT921X_METER_SLOT, ctrl);
-	if (res)
-		return res;
-	priv->meter_slot_ns = ctrl * op_ns;
-
-	ctrl = max(priv->port_shape_slot_ns / op_ns,
-		   YT921X_PORT_SHAPE_SLOT_MIN);
-	res = yt921x_reg_write(priv, YT921X_PORT_SHAPE_SLOT, ctrl);
-	if (res)
-		return res;
-	priv->port_shape_slot_ns = ctrl * op_ns;
-
-	return 0;
-}
-
-static int yt921x_chip_setup_acl(struct yt921x_priv *priv)
-{
-	u32 ctrl;
-	int res;
-
-	ctrl = YT921X_ACL_PERMIT_UNMATCH_PORTS_M;
-	res = yt921x_reg_write(priv, YT921X_ACL_PERMIT_UNMATCH, ctrl);
-	if (res)
-		return res;
-
-	ctrl = YT921X_ACL_PORT_PORTS_M;
-	res = yt921x_reg_write(priv, YT921X_ACL_PORT, ctrl);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int __maybe_unused yt921x_chip_setup_qos(struct yt921x_priv *priv)
-{
-	u32 ctrl;
-	int res;
-
-	/* DSCP to internal priorities */
-	for (u8 dscp = 0; dscp < DSCP_MAX; dscp++) {
-		int prio = ietf_dscp_to_ieee8021q_tt(dscp);
-
-		if (prio < 0)
-			return prio;
-
-		res = yt921x_reg_write(priv, YT921X_IPM_DSCPn(dscp),
-				       YT921X_IPM_PRIO(prio));
-		if (res)
-			return res;
-	}
-
-	/* 802.1Q QoS to internal priorities */
-	for (u8 pcp = 0; pcp < 8; pcp++)
-		for (u8 dei = 0; dei < 2; dei++) {
-			ctrl = YT921X_IPM_PRIO(pcp);
-			if (dei)
-				/* "Red" almost means drop, so it's not that
-				 * useful. Note that tc police does not support
-				 * Three-Color very well
-				 */
-				ctrl |= YT921X_IPM_COLOR_YELLOW;
-
-			for (u8 svlan = 0; svlan < 2; svlan++) {
-				u32 reg = YT921X_IPM_PCPn(svlan, dei, pcp);
-
-				res = yt921x_reg_write(priv, reg, ctrl);
-				if (res)
-					return res;
-			}
-		}
-
-	return 0;
-}
-
-static int yt921x_chip_setup(struct yt921x_priv *priv)
-{
-	u32 ctrl;
-	int res;
-
-	ctrl = YT921X_FUNC_MIB | YT921X_FUNC_ACL | YT921X_FUNC_METER;
-	res = yt921x_reg_set_bits(priv, YT921X_FUNC, ctrl);
-	if (res)
-		return res;
-
-	res = yt921x_chip_setup_dsa(priv);
-	if (res)
-		return res;
-
-	res = yt921x_chip_setup_tc(priv);
-	if (res)
-		return res;
-
-	res = yt921x_chip_setup_acl(priv);
-	if (res)
-		return res;
-
-#if IS_ENABLED(CONFIG_DCB)
-	res = yt921x_chip_setup_qos(priv);
-	if (res)
-		return res;
-#endif
-
-	/* Clear MIB */
-	ctrl = YT921X_MIB_CTRL_CLEAN | YT921X_MIB_CTRL_ALL_PORT;
-	res = yt921x_reg_write(priv, YT921X_MIB_CTRL, ctrl);
-	if (res)
-		return res;
-
-	/* Miscellaneous */
-	res = yt921x_reg_set_bits(priv, YT921X_SENSOR, YT921X_SENSOR_TEMP);
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static int yt921x_dsa_setup(struct dsa_switch *ds)
-{
-	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	struct device *dev = to_device(priv);
-	struct device_node *np = dev->of_node;
-	struct device_node *child;
-	int res;
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_chip_reset(priv);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		return res;
-
-	/* Register the internal mdio bus. Nodes for internal ports should have
-	 * proper phy-handle pointing to their PHYs. Not enabling the internal
-	 * bus is possible, though pretty wired, if internal ports are not used.
-	 */
-	child = of_get_child_by_name(np, "mdio");
-	if (child) {
-		res = yt921x_mbus_int_init(priv, child);
-		of_node_put(child);
-		if (res)
-			return res;
-	}
-
-	/* External mdio bus is optional */
-	child = of_get_child_by_name(np, "mdio-external");
-	if (child) {
-		res = yt921x_mbus_ext_init(priv, child);
-		of_node_put(child);
-		if (res)
-			return res;
-
-		dev_err(dev, "Untested external mdio bus\n");
-		return -ENODEV;
-	}
-
-	mutex_lock(&priv->reg_lock);
-	res = yt921x_chip_setup(priv);
-	mutex_unlock(&priv->reg_lock);
-
-	if (res)
-		return res;
-
-	return 0;
-}
-
-static const struct phylink_mac_ops yt921x_phylink_mac_ops = {
-	.mac_link_down	= yt921x_phylink_mac_link_down,
-	.mac_link_up	= yt921x_phylink_mac_link_up,
-	.mac_config	= yt921x_phylink_mac_config,
-};
-
-static const struct dsa_switch_ops yt921x_dsa_switch_ops = {
-	/* mib */
-	.get_strings		= yt921x_dsa_get_strings,
-	.get_ethtool_stats	= yt921x_dsa_get_ethtool_stats,
-	.get_sset_count		= yt921x_dsa_get_sset_count,
-	.get_eth_mac_stats	= yt921x_dsa_get_eth_mac_stats,
-	.get_eth_ctrl_stats	= yt921x_dsa_get_eth_ctrl_stats,
-	.get_rmon_stats		= yt921x_dsa_get_rmon_stats,
-	.get_stats64		= yt921x_dsa_get_stats64,
-	.get_pause_stats	= yt921x_dsa_get_pause_stats,
-	/* eee */
-	.support_eee		= dsa_supports_eee,
-	.set_mac_eee		= yt921x_dsa_set_mac_eee,
-	/* mtu */
-	.port_change_mtu	= yt921x_dsa_port_change_mtu,
-	.port_max_mtu		= yt921x_dsa_port_max_mtu,
-	/* rate */
-	.port_policer_del	= yt921x_dsa_port_policer_del,
-	.port_policer_add	= yt921x_dsa_port_policer_add,
-	.port_setup_tc		= yt921x_dsa_port_setup_tc,
-	/* acl */
-	.cls_flower_del		= yt921x_dsa_cls_flower_del,
-	.cls_flower_add		= yt921x_dsa_cls_flower_add,
-	/* hsr */
-	.port_hsr_leave		= dsa_port_simple_hsr_leave,
-	.port_hsr_join		= dsa_port_simple_hsr_join,
-	/* mirror */
-	.port_mirror_del	= yt921x_dsa_port_mirror_del,
-	.port_mirror_add	= yt921x_dsa_port_mirror_add,
-	/* lag */
-	.port_lag_leave		= yt921x_dsa_port_lag_leave,
-	.port_lag_join		= yt921x_dsa_port_lag_join,
-	/* fdb */
-	.port_fdb_dump		= yt921x_dsa_port_fdb_dump,
-	.port_fast_age		= yt921x_dsa_port_fast_age,
-	.set_ageing_time	= yt921x_dsa_set_ageing_time,
-	.port_fdb_del		= yt921x_dsa_port_fdb_del,
-	.port_fdb_add		= yt921x_dsa_port_fdb_add,
-	.port_mdb_del		= yt921x_dsa_port_mdb_del,
-	.port_mdb_add		= yt921x_dsa_port_mdb_add,
-	/* vlan */
-	.port_vlan_filtering	= yt921x_dsa_port_vlan_filtering,
-	.port_vlan_del		= yt921x_dsa_port_vlan_del,
-	.port_vlan_add		= yt921x_dsa_port_vlan_add,
-	/* bridge */
-	.port_pre_bridge_flags	= yt921x_dsa_port_pre_bridge_flags,
-	.port_bridge_flags	= yt921x_dsa_port_bridge_flags,
-	.port_bridge_leave	= yt921x_dsa_port_bridge_leave,
-	.port_bridge_join	= yt921x_dsa_port_bridge_join,
-	/* mst */
-	.port_mst_state_set	= yt921x_dsa_port_mst_state_set,
-	.vlan_msti_set		= yt921x_dsa_vlan_msti_set,
-	.port_stp_state_set	= yt921x_dsa_port_stp_state_set,
-#if IS_ENABLED(CONFIG_DCB)
-	/* dcb */
-	.port_get_default_prio	= yt921x_dsa_port_get_default_prio,
-	.port_set_default_prio	= yt921x_dsa_port_set_default_prio,
-	.port_get_apptrust	= yt921x_dsa_port_get_apptrust,
-	.port_set_apptrust	= yt921x_dsa_port_set_apptrust,
-#endif
-	/* port */
-	.get_tag_protocol	= yt921x_dsa_get_tag_protocol,
-	.phylink_get_caps	= yt921x_dsa_phylink_get_caps,
-	.port_setup		= yt921x_dsa_port_setup,
-#if IS_ENABLED(CONFIG_DCB)
-	/* dscp */
-	.port_get_dscp_prio	= yt921x_dsa_port_get_dscp_prio,
-	.port_del_dscp_prio	= yt921x_dsa_port_del_dscp_prio,
-	.port_add_dscp_prio	= yt921x_dsa_port_add_dscp_prio,
-#endif
-	/* chip */
-	.setup			= yt921x_dsa_setup,
-};
-
-static void yt921x_mdio_shutdown(struct mdio_device *mdiodev)
-{
-	struct yt921x_priv *priv = mdiodev_get_drvdata(mdiodev);
-
-	if (!priv)
-		return;
-
-	dsa_switch_shutdown(&priv->ds);
-}
-
-static void yt921x_mdio_remove(struct mdio_device *mdiodev)
-{
-	struct yt921x_priv *priv = mdiodev_get_drvdata(mdiodev);
-
-	if (!priv)
-		return;
-
-	for (size_t i = ARRAY_SIZE(priv->ports); i-- > 0; ) {
-		struct yt921x_port *pp = &priv->ports[i];
-
-		disable_delayed_work_sync(&pp->mib_read);
-	}
-
-	dsa_unregister_switch(&priv->ds);
-
-	for (unsigned int i = 0; i < ARRAY_SIZE(priv->acl_blks); i++) {
-		struct yt921x_acl_blk *aclblk = priv->acl_blks[i];
-
-		if (!aclblk)
-			continue;
-		for (unsigned int j = 0; j < ARRAY_SIZE(aclblk->rules); j++) {
-			struct yt921x_acl_rule *aclrule = aclblk->rules[j];
-
-			if (!aclrule)
-				continue;
-			kvfree(aclrule);
-		}
-		kvfree(aclblk);
-	}
-	mutex_destroy(&priv->reg_lock);
-}
-
-static int yt921x_mdio_probe(struct mdio_device *mdiodev)
-{
-	struct device *dev = &mdiodev->dev;
-	struct yt921x_reg_mdio *mdio;
-	struct yt921x_priv *priv;
-	struct dsa_switch *ds;
-
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	mdio = devm_kzalloc(dev, sizeof(*mdio), GFP_KERNEL);
-	if (!mdio)
-		return -ENOMEM;
-
-	mdio->bus = mdiodev->bus;
-	mdio->addr = mdiodev->addr;
-	mdio->switchid = 0;
-
-	mutex_init(&priv->reg_lock);
-
-	priv->reg_ops = &yt921x_reg_ops_mdio;
-	priv->reg_ctx = mdio;
-
-	for (size_t i = 0; i < ARRAY_SIZE(priv->ports); i++) {
-		struct yt921x_port *pp = &priv->ports[i];
-
-		pp->index = i;
-		INIT_DELAYED_WORK(&pp->mib_read, yt921x_poll_mib);
-	}
-
-	ds = &priv->ds;
-	ds->dev = dev;
-	ds->assisted_learning_on_cpu_port = true;
-	ds->dscp_prio_mapping_is_global = true;
-	ds->priv = priv;
-	ds->ops = &yt921x_dsa_switch_ops;
-	ds->ageing_time_min = 1 * 5000;
-	ds->ageing_time_max = U16_MAX * 5000;
-	ds->phylink_mac_ops = &yt921x_phylink_mac_ops;
-	ds->num_lag_ids = YT921X_LAG_NUM;
-	ds->num_ports = YT921X_PORT_NUM;
-
-	mdiodev_set_drvdata(mdiodev, priv);
-
-	return dsa_register_switch(ds);
-}
-
-static const struct of_device_id yt921x_of_match[] = {
-	{ .compatible = "motorcomm,yt9215" },
-	{}
-};
-MODULE_DEVICE_TABLE(of, yt921x_of_match);
-
-static struct mdio_driver yt921x_mdio_driver = {
-	.probe = yt921x_mdio_probe,
-	.remove = yt921x_mdio_remove,
-	.shutdown = yt921x_mdio_shutdown,
-	.mdiodrv.driver = {
-		.name = YT921X_NAME,
-		.of_match_table = yt921x_of_match,
-	},
-};
-
-mdio_module_driver(yt921x_mdio_driver);
-
-MODULE_AUTHOR("David Yang <mmyangfl@gmail.com>");
-MODULE_DESCRIPTION("Driver for Motorcomm YT921x Switch");
-MODULE_LICENSE("GPL");
diff --git a/drivers/net/dsa/yt921x.h b/drivers/net/dsa/yt921x.h
deleted file mode 100644
index 5f3b99e189c412..00000000000000
--- a/drivers/net/dsa/yt921x.h
+++ /dev/null
@@ -1,977 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (c) 2025 David Yang
- */
-
-#ifndef __YT921X_H
-#define __YT921X_H
-
-#include <net/dsa.h>
-
-#define YT921X_SMI_SWITCHID_M		GENMASK(3, 2)
-#define  YT921X_SMI_SWITCHID(x)			FIELD_PREP(YT921X_SMI_SWITCHID_M, (x))
-#define YT921X_SMI_AD			BIT(1)
-#define  YT921X_SMI_ADDR			0
-#define  YT921X_SMI_DATA			YT921X_SMI_AD
-#define YT921X_SMI_RW			BIT(0)
-#define  YT921X_SMI_WRITE			0
-#define  YT921X_SMI_READ			YT921X_SMI_RW
-
-#define YT921X_SWITCHID_NUM		4
-
-#define YT921X_RST			0x80000
-#define  YT921X_RST_HW				BIT(31)
-#define  YT921X_RST_SW				BIT(1)
-#define YT921X_FUNC			0x80004
-#define  YT921X_FUNC_METER			BIT(4)
-#define  YT921X_FUNC_ACL			BIT(2)
-#define  YT921X_FUNC_MIB			BIT(1)
-#define YT921X_CHIP_ID			0x80008
-#define  YT921X_CHIP_ID_MAJOR			GENMASK(31, 16)
-#define YT921X_EXT_CPU_PORT		0x8000c
-#define  YT921X_EXT_CPU_PORT_TAG_EN		BIT(15)
-#define  YT921X_EXT_CPU_PORT_PORT_EN		BIT(14)
-#define  YT921X_EXT_CPU_PORT_PORT_M		GENMASK(3, 0)
-#define   YT921X_EXT_CPU_PORT_PORT(x)			FIELD_PREP(YT921X_EXT_CPU_PORT_PORT_M, (x))
-#define YT921X_CPU_TAG_TPID		0x80010
-#define  YT921X_CPU_TAG_TPID_TPID_M		GENMASK(15, 0)
-/* Same as ETH_P_YT921X, but this represents the true HW default, while the
- * former is a local convention chosen by us.
- */
-#define   YT921X_CPU_TAG_TPID_TPID_DEFAULT		0x9988
-#define YT921X_PVID_SEL			0x80014
-#define  YT921X_PVID_SEL_SVID_PORTn(port)	BIT(port)
-#define YT921X_SERDES_CTRL		0x80028
-#define  YT921X_SERDES_CTRL_PORTn_TEST(port)	BIT((port) - 3)
-#define  YT921X_SERDES_CTRL_PORTn(port)		BIT((port) - 8)
-#define YT921X_IO_LEVEL			0x80030
-#define  YT9215_IO_LEVEL_NORMAL_M		GENMASK(5, 4)
-#define   YT9215_IO_LEVEL_NORMAL(x)			FIELD_PREP(YT9215_IO_LEVEL_NORMAL_M, (x))
-#define   YT9215_IO_LEVEL_NORMAL_3V3			YT9215_IO_LEVEL_NORMAL(0)
-#define   YT9215_IO_LEVEL_NORMAL_1V8			YT9215_IO_LEVEL_NORMAL(3)
-#define  YT9215_IO_LEVEL_RGMII1_M		GENMASK(3, 2)
-#define   YT9215_IO_LEVEL_RGMII1(x)			FIELD_PREP(YT9215_IO_LEVEL_RGMII1_M, (x))
-#define   YT9215_IO_LEVEL_RGMII1_3V3			YT9215_IO_LEVEL_RGMII1(0)
-#define   YT9215_IO_LEVEL_RGMII1_2V5			YT9215_IO_LEVEL_RGMII1(1)
-#define   YT9215_IO_LEVEL_RGMII1_1V8			YT9215_IO_LEVEL_RGMII1(2)
-#define  YT9215_IO_LEVEL_RGMII0_M		GENMASK(1, 0)
-#define   YT9215_IO_LEVEL_RGMII0(x)			FIELD_PREP(YT9215_IO_LEVEL_RGMII0_M, (x))
-#define   YT9215_IO_LEVEL_RGMII0_3V3			YT9215_IO_LEVEL_RGMII0(0)
-#define   YT9215_IO_LEVEL_RGMII0_2V5			YT9215_IO_LEVEL_RGMII0(1)
-#define   YT9215_IO_LEVEL_RGMII0_1V8			YT9215_IO_LEVEL_RGMII0(2)
-#define  YT9218_IO_LEVEL_RGMII1_M		GENMASK(5, 4)
-#define   YT9218_IO_LEVEL_RGMII1(x)			FIELD_PREP(YT9218_IO_LEVEL_RGMII1_M, (x))
-#define   YT9218_IO_LEVEL_RGMII1_3V3			YT9218_IO_LEVEL_RGMII1(0)
-#define   YT9218_IO_LEVEL_RGMII1_2V5			YT9218_IO_LEVEL_RGMII1(1)
-#define   YT9218_IO_LEVEL_RGMII1_1V8			YT9218_IO_LEVEL_RGMII1(2)
-#define  YT9218_IO_LEVEL_RGMII0_M		GENMASK(3, 2)
-#define   YT9218_IO_LEVEL_RGMII0(x)			FIELD_PREP(YT9218_IO_LEVEL_RGMII0_M, (x))
-#define   YT9218_IO_LEVEL_RGMII0_3V3			YT9218_IO_LEVEL_RGMII0(0)
-#define   YT9218_IO_LEVEL_RGMII0_2V5			YT9218_IO_LEVEL_RGMII0(1)
-#define   YT9218_IO_LEVEL_RGMII0_1V8			YT9218_IO_LEVEL_RGMII0(2)
-#define  YT9218_IO_LEVEL_NORMAL_M		GENMASK(1, 0)
-#define   YT9218_IO_LEVEL_NORMAL(x)			FIELD_PREP(YT9218_IO_LEVEL_NORMAL_M, (x))
-#define   YT9218_IO_LEVEL_NORMAL_3V3			YT9218_IO_LEVEL_NORMAL(0)
-#define   YT9218_IO_LEVEL_NORMAL_1V8			YT9218_IO_LEVEL_NORMAL(3)
-#define YT921X_MAC_ADDR_HI2		0x80080
-#define YT921X_MAC_ADDR_LO4		0x80084
-#define YT921X_SERDESn(port)		(0x8008c + 4 * ((port) - 8))
-#define  YT921X_SERDES_MODE_M			GENMASK(9, 7)
-#define   YT921X_SERDES_MODE(x)				FIELD_PREP(YT921X_SERDES_MODE_M, (x))
-#define   YT921X_SERDES_MODE_SGMII			YT921X_SERDES_MODE(0)
-#define   YT921X_SERDES_MODE_REVSGMII			YT921X_SERDES_MODE(1)
-#define   YT921X_SERDES_MODE_1000BASEX			YT921X_SERDES_MODE(2)
-#define   YT921X_SERDES_MODE_100BASEX			YT921X_SERDES_MODE(3)
-#define   YT921X_SERDES_MODE_2500BASEX			YT921X_SERDES_MODE(4)
-#define  YT921X_SERDES_RX_PAUSE			BIT(6)
-#define  YT921X_SERDES_TX_PAUSE			BIT(5)
-#define  YT921X_SERDES_LINK			BIT(4)  /* force link */
-#define  YT921X_SERDES_DUPLEX_FULL		BIT(3)
-#define  YT921X_SERDES_SPEED_M			GENMASK(2, 0)
-#define   YT921X_SERDES_SPEED(x)			FIELD_PREP(YT921X_SERDES_SPEED_M, (x))
-#define   YT921X_SERDES_SPEED_10			YT921X_SERDES_SPEED(0)
-#define   YT921X_SERDES_SPEED_100			YT921X_SERDES_SPEED(1)
-#define   YT921X_SERDES_SPEED_1000			YT921X_SERDES_SPEED(2)
-#define   YT921X_SERDES_SPEED_10000			YT921X_SERDES_SPEED(3)
-#define   YT921X_SERDES_SPEED_2500			YT921X_SERDES_SPEED(4)
-#define YT921X_PORTn_CTRL(port)		(0x80100 + 4 * (port))
-#define  YT921X_PORT_CTRL_PAUSE_AN		BIT(10)
-#define YT921X_PORTn_STATUS(port)	(0x80200 + 4 * (port))
-#define  YT921X_PORT_LINK			BIT(9)  /* CTRL: auto negotiation */
-#define  YT921X_PORT_HALF_PAUSE			BIT(8)  /* Half-duplex back pressure mode */
-#define  YT921X_PORT_DUPLEX_FULL		BIT(7)
-#define  YT921X_PORT_RX_PAUSE			BIT(6)
-#define  YT921X_PORT_TX_PAUSE			BIT(5)
-#define  YT921X_PORT_RX_MAC_EN			BIT(4)
-#define  YT921X_PORT_TX_MAC_EN			BIT(3)
-#define  YT921X_PORT_SPEED_M			GENMASK(2, 0)
-#define   YT921X_PORT_SPEED(x)				FIELD_PREP(YT921X_PORT_SPEED_M, (x))
-#define   YT921X_PORT_SPEED_10				YT921X_PORT_SPEED(0)
-#define   YT921X_PORT_SPEED_100				YT921X_PORT_SPEED(1)
-#define   YT921X_PORT_SPEED_1000			YT921X_PORT_SPEED(2)
-#define   YT921X_PORT_SPEED_10000			YT921X_PORT_SPEED(3)
-#define   YT921X_PORT_SPEED_2500			YT921X_PORT_SPEED(4)
-#define YT921X_PON_STRAP_FUNC		0x80320
-#define YT921X_PON_STRAP_VAL		0x80324
-#define YT921X_PON_STRAP_CAP		0x80328
-#define  YT921X_PON_STRAP_EEE			BIT(16)
-#define  YT921X_PON_STRAP_LOOP_DETECT		BIT(7)
-#define YT921X_MDIO_POLLINGn(port)	(0x80364 + 4 * ((port) - 8))
-#define  YT921X_MDIO_POLLING_DUPLEX_FULL	BIT(4)
-#define  YT921X_MDIO_POLLING_LINK		BIT(3)
-#define  YT921X_MDIO_POLLING_SPEED_M		GENMASK(2, 0)
-#define   YT921X_MDIO_POLLING_SPEED(x)			FIELD_PREP(YT921X_MDIO_POLLING_SPEED_M, (x))
-#define   YT921X_MDIO_POLLING_SPEED_10			YT921X_MDIO_POLLING_SPEED(0)
-#define   YT921X_MDIO_POLLING_SPEED_100			YT921X_MDIO_POLLING_SPEED(1)
-#define   YT921X_MDIO_POLLING_SPEED_1000		YT921X_MDIO_POLLING_SPEED(2)
-#define   YT921X_MDIO_POLLING_SPEED_10000		YT921X_MDIO_POLLING_SPEED(3)
-#define   YT921X_MDIO_POLLING_SPEED_2500		YT921X_MDIO_POLLING_SPEED(4)
-#define YT921X_SENSOR			0x8036c
-#define  YT921X_SENSOR_TEMP			BIT(18)
-#define YT921X_TEMP			0x80374
-#define YT921X_CHIP_MODE		0x80388
-#define  YT921X_CHIP_MODE_MODE			GENMASK(1, 0)
-#define YT921X_XMII_CTRL		0x80394
-#define  YT921X_XMII_CTRL_PORTn(port)		BIT(9 - (port))  /* Yes, it's reversed */
-#define YT921X_XMIIn(port)		(0x80400 + 8 * ((port) - 8))
-#define  YT921X_XMII_MODE_M			GENMASK(31, 29)
-#define   YT921X_XMII_MODE(x)				FIELD_PREP(YT921X_XMII_MODE_M, (x))
-#define   YT921X_XMII_MODE_MII				YT921X_XMII_MODE(0)
-#define   YT921X_XMII_MODE_REVMII			YT921X_XMII_MODE(1)
-#define   YT921X_XMII_MODE_RMII				YT921X_XMII_MODE(2)
-#define   YT921X_XMII_MODE_REVRMII			YT921X_XMII_MODE(3)
-#define   YT921X_XMII_MODE_RGMII			YT921X_XMII_MODE(4)
-#define   YT921X_XMII_MODE_DISABLE			YT921X_XMII_MODE(5)
-#define  YT921X_XMII_LINK			BIT(19)  /* force link */
-#define  YT921X_XMII_EN				BIT(18)
-#define  YT921X_XMII_SOFT_RST			BIT(17)
-#define  YT921X_XMII_RGMII_TX_DELAY_150PS_M	GENMASK(16, 13)
-#define   YT921X_XMII_RGMII_TX_DELAY_150PS(x)		FIELD_PREP(YT921X_XMII_RGMII_TX_DELAY_150PS_M, (x))
-#define  YT921X_XMII_TX_CLK_IN			BIT(11)
-#define  YT921X_XMII_RX_CLK_IN			BIT(10)
-#define  YT921X_XMII_RGMII_TX_DELAY_2NS		BIT(8)
-#define  YT921X_XMII_RGMII_TX_CLK_OUT		BIT(7)
-#define  YT921X_XMII_RGMII_RX_DELAY_150PS_M	GENMASK(6, 3)
-#define   YT921X_XMII_RGMII_RX_DELAY_150PS(x)		FIELD_PREP(YT921X_XMII_RGMII_RX_DELAY_150PS_M, (x))
-#define  YT921X_XMII_RMII_PHY_TX_CLK_OUT	BIT(2)
-#define  YT921X_XMII_REVMII_TX_CLK_OUT		BIT(1)
-#define  YT921X_XMII_REVMII_RX_CLK_OUT		BIT(0)
-
-#define YT921X_MACn_FRAME(port)		(0x81008 + 0x1000 * (port))
-#define  YT921X_MAC_FRAME_SIZE_M		GENMASK(21, 8)
-#define   YT921X_MAC_FRAME_SIZE(x)			FIELD_PREP(YT921X_MAC_FRAME_SIZE_M, (x))
-
-#define YT921X_EEEn_VAL(port)		(0xa0000 + 0x40 * (port))
-#define  YT921X_EEE_VAL_DATA			BIT(1)
-
-#define YT921X_EEE_CTRL			0xb0000
-#define  YT921X_EEE_CTRL_ENn(port)		BIT(port)
-
-#define YT921X_MIB_CTRL			0xc0004
-#define  YT921X_MIB_CTRL_CLEAN			BIT(30)
-#define  YT921X_MIB_CTRL_PORT_M			GENMASK(6, 3)
-#define   YT921X_MIB_CTRL_PORT(x)			FIELD_PREP(YT921X_MIB_CTRL_PORT_M, (x))
-#define  YT921X_MIB_CTRL_ONE_PORT		BIT(1)
-#define  YT921X_MIB_CTRL_ALL_PORT		BIT(0)
-#define YT921X_MIBn_DATA0(port)		(0xc0100 + 0x100 * (port))
-#define YT921X_MIBn_DATAm(port, x)	(YT921X_MIBn_DATA0(port) + 4 * (x))
-#define  YT921X_MIB_DATA_RX_BROADCAST		0x00
-#define  YT921X_MIB_DATA_RX_PAUSE		0x04
-#define  YT921X_MIB_DATA_RX_MULTICAST		0x08
-#define  YT921X_MIB_DATA_RX_CRC_ERR		0x0c
-
-#define  YT921X_MIB_DATA_RX_ALIGN_ERR		0x10
-#define  YT921X_MIB_DATA_RX_UNDERSIZE_ERR	0x14
-#define  YT921X_MIB_DATA_RX_FRAG_ERR		0x18
-#define  YT921X_MIB_DATA_RX_PKT_SZ_64		0x1c
-
-#define  YT921X_MIB_DATA_RX_PKT_SZ_65_TO_127	0x20
-#define  YT921X_MIB_DATA_RX_PKT_SZ_128_TO_255	0x24
-#define  YT921X_MIB_DATA_RX_PKT_SZ_256_TO_511	0x28
-#define  YT921X_MIB_DATA_RX_PKT_SZ_512_TO_1023	0x2c
-
-#define  YT921X_MIB_DATA_RX_PKT_SZ_1024_TO_1518	0x30
-#define  YT921X_MIB_DATA_RX_PKT_SZ_1519_TO_MAX	0x34
-/* 0x38: unused */
-#define  YT921X_MIB_DATA_RX_GOOD_BYTES		0x3c
-
-/* 0x40: 64 bytes */
-#define  YT921X_MIB_DATA_RX_BAD_BYTES		0x44
-/* 0x48: 64 bytes */
-#define  YT921X_MIB_DATA_RX_OVERSIZE_ERR	0x4c
-
-#define  YT921X_MIB_DATA_RX_DROPPED		0x50
-#define  YT921X_MIB_DATA_TX_BROADCAST		0x54
-#define  YT921X_MIB_DATA_TX_PAUSE		0x58
-#define  YT921X_MIB_DATA_TX_MULTICAST		0x5c
-
-#define  YT921X_MIB_DATA_TX_UNDERSIZE_ERR	0x60
-#define  YT921X_MIB_DATA_TX_PKT_SZ_64		0x64
-#define  YT921X_MIB_DATA_TX_PKT_SZ_65_TO_127	0x68
-#define  YT921X_MIB_DATA_TX_PKT_SZ_128_TO_255	0x6c
-
-#define  YT921X_MIB_DATA_TX_PKT_SZ_256_TO_511	0x70
-#define  YT921X_MIB_DATA_TX_PKT_SZ_512_TO_1023	0x74
-#define  YT921X_MIB_DATA_TX_PKT_SZ_1024_TO_1518	0x78
-#define  YT921X_MIB_DATA_TX_PKT_SZ_1519_TO_MAX	0x7c
-
-/* 0x80: unused */
-#define  YT921X_MIB_DATA_TX_GOOD_BYTES		0x84
-/* 0x88: 64 bytes */
-#define  YT921X_MIB_DATA_TX_COLLISION		0x8c
-
-#define  YT921X_MIB_DATA_TX_EXCESSIVE_COLLISION	0x90
-#define  YT921X_MIB_DATA_TX_MULTIPLE_COLLISION	0x94
-#define  YT921X_MIB_DATA_TX_SINGLE_COLLISION	0x98
-#define  YT921X_MIB_DATA_TX_PKT			0x9c
-
-#define  YT921X_MIB_DATA_TX_DEFERRED		0xa0
-#define  YT921X_MIB_DATA_TX_LATE_COLLISION	0xa4
-#define  YT921X_MIB_DATA_RX_OAM			0xa8
-#define  YT921X_MIB_DATA_TX_OAM			0xac
-
-#define YT921X_EDATA_CTRL		0xe0000
-#define  YT921X_EDATA_CTRL_ADDR_M		GENMASK(15, 8)
-#define   YT921X_EDATA_CTRL_ADDR(x)			FIELD_PREP(YT921X_EDATA_CTRL_ADDR_M, (x))
-#define  YT921X_EDATA_CTRL_OP_M			GENMASK(3, 0)
-#define   YT921X_EDATA_CTRL_OP(x)			FIELD_PREP(YT921X_EDATA_CTRL_OP_M, (x))
-#define   YT921X_EDATA_CTRL_READ			YT921X_EDATA_CTRL_OP(5)
-#define YT921X_EDATA_DATA		0xe0004
-#define  YT921X_EDATA_DATA_DATA_M			GENMASK(31, 24)
-#define  YT921X_EDATA_DATA_STATUS_M		GENMASK(3, 0)
-#define   YT921X_EDATA_DATA_STATUS(x)			FIELD_PREP(YT921X_EDATA_DATA_STATUS_M, (x))
-#define   YT921X_EDATA_DATA_IDLE			YT921X_EDATA_DATA_STATUS(3)
-#define YT921X_SYS_CLK			0xe0040
-#define  YT921X_SYS_CLK_SEL_M			GENMASK(1, 0)  /* unknown: 167M */
-#define   YT9215_SYS_CLK_125M				0
-#define   YT9218_SYS_CLK_167M				0
-#define   YT921X_SYS_CLK_143M				1
-
-#define YT921X_EXT_MBUS_OP		0x6a000
-#define YT921X_INT_MBUS_OP		0xf0000
-#define  YT921X_MBUS_OP_START			BIT(0)
-#define YT921X_EXT_MBUS_CTRL		0x6a004
-#define YT921X_INT_MBUS_CTRL		0xf0004
-#define  YT921X_MBUS_CTRL_PORT_M		GENMASK(25, 21)
-#define   YT921X_MBUS_CTRL_PORT(x)			FIELD_PREP(YT921X_MBUS_CTRL_PORT_M, (x))
-#define  YT921X_MBUS_CTRL_REG_M			GENMASK(20, 16)
-#define   YT921X_MBUS_CTRL_REG(x)			FIELD_PREP(YT921X_MBUS_CTRL_REG_M, (x))
-#define  YT921X_MBUS_CTRL_TYPE_M		GENMASK(11, 8)  /* wild guess */
-#define   YT921X_MBUS_CTRL_TYPE(x)			FIELD_PREP(YT921X_MBUS_CTRL_TYPE_M, (x))
-#define   YT921X_MBUS_CTRL_TYPE_C22			YT921X_MBUS_CTRL_TYPE(4)
-#define  YT921X_MBUS_CTRL_OP_M			GENMASK(3, 2)  /* wild guess */
-#define   YT921X_MBUS_CTRL_OP(x)			FIELD_PREP(YT921X_MBUS_CTRL_OP_M, (x))
-#define   YT921X_MBUS_CTRL_WRITE			YT921X_MBUS_CTRL_OP(1)
-#define   YT921X_MBUS_CTRL_READ				YT921X_MBUS_CTRL_OP(2)
-#define YT921X_EXT_MBUS_DOUT		0x6a008
-#define YT921X_INT_MBUS_DOUT		0xf0008
-#define YT921X_EXT_MBUS_DIN		0x6a00c
-#define YT921X_INT_MBUS_DIN		0xf000c
-
-#define YT921X_PORTn_EGR(port)		(0x100000 + 4 * (port))
-#define  YT921X_PORT_EGR_TPID_CTAG_M		GENMASK(5, 4)
-#define   YT921X_PORT_EGR_TPID_CTAG(x)			FIELD_PREP(YT921X_PORT_EGR_TPID_CTAG_M, (x))
-#define  YT921X_PORT_EGR_TPID_STAG_M		GENMASK(3, 2)
-#define   YT921X_PORT_EGR_TPID_STAG(x)			FIELD_PREP(YT921X_PORT_EGR_TPID_STAG_M, (x))
-#define YT921X_TPID_EGRn(x)		(0x100300 + 4 * (x))	/* [0, 3] */
-#define  YT921X_TPID_EGR_TPID_M			GENMASK(15, 0)
-
-#define YT921X_IPM_DSCPn(n)		(0x180000 + 4 * (n))	/* Internal Priority Map */
-#define YT921X_IPM_PCPn(map, dei, pcp)	(0x180100 + 4 * (16 * (map) + 8 * (dei) + (pcp)))
-#define  YT921X_IPM_PRIO_M			GENMASK(4, 2)
-#define   YT921X_IPM_PRIO(x)				FIELD_PREP(YT921X_IPM_PRIO_M, (x))
-#define  YT921X_IPM_COLOR_M			GENMASK(1, 0)
-#define   YT921X_IPM_COLOR(x)				FIELD_PREP(YT921X_IPM_COLOR_M, (x))
-#define   YT921X_IPM_COLOR_GREEN			YT921X_IPM_COLOR(0)
-#define   YT921X_IPM_COLOR_YELLOW			YT921X_IPM_COLOR(1)
-#define   YT921X_IPM_COLOR_RED				YT921X_IPM_COLOR(2)
-#define YT921X_PORTn_QOS(port)		(0x180180 + 4 * (port))
-#define  YT921X_PORT_QOS_CVLAN_PRIO_MAP_ID	BIT(5)
-#define  YT921X_PORT_QOS_SVLAN_PRIO_MAP_ID	BIT(4)
-#define  YT921X_PORT_QOS_PRIO_M			GENMASK(3, 1)
-#define   YT921X_PORT_QOS_PRIO(x)			FIELD_PREP(YT921X_PORT_QOS_PRIO_M, (x))
-#define  YT921X_PORT_QOS_PRIO_EN		BIT(0)
-#define YT921X_PORTn_PRIO_ORD(port)	(0x180200 + 4 * (port))
-#define  YT921X_PORT_PRIO_ORD_APPm_M(m)		GENMASK(3 * (m) + 2, 3 * (m))
-#define   YT921X_PORT_PRIO_ORD_APPm(m, x)		((x) << (3 * (m)))	/* 0: disabled, except PORT_QOS_PRIO */
-
-enum yt921x_app_selector {
-	YT921X_APP_SEL_MAC_SA,
-	YT921X_APP_SEL_MAC_DA,
-	YT921X_APP_SEL_VID,
-	YT921X_APP_SEL_ACL,
-	YT921X_APP_SEL_DSCP,
-	YT921X_APP_SEL_CVLAN_PCP,
-	YT921X_APP_SEL_SVLAN_PCP,
-	/* The physical port, i.e. YT921X_PORT_QOS_PRIO */
-	YT921X_APP_SEL_PORT,
-	YT921X_APP_SEL_NUM
-};
-
-#define YT921X_VLAN_IGR_FILTER		0x180280
-#define  YT921X_VLAN_IGR_FILTER_PORTn_BYPASS_IGMP(port)	BIT((port) + 11)
-#define  YT921X_VLAN_IGR_FILTER_PORTn(port)	BIT(port)
-#define YT921X_PORTn_ISOLATION(port)	(0x180294 + 4 * (port))
-#define  YT921X_PORT_ISOLATION_BLOCKn(port)	BIT(port)
-#define YT921X_STPn(n)			(0x18038c + 4 * (n))
-#define  YT921X_STP_PORTn_M(port)		GENMASK(2 * (port) + 1, 2 * (port))
-#define   YT921X_STP_PORTn(port, x)			((x) << (2 * (port)))
-#define   YT921X_STP_PORTn_DISABLED(port)		YT921X_STP_PORTn(port, 0)
-#define   YT921X_STP_PORTn_LEARNING(port)		YT921X_STP_PORTn(port, 1)
-#define   YT921X_STP_PORTn_BLOCKING(port)		YT921X_STP_PORTn(port, 2)
-#define   YT921X_STP_PORTn_FORWARD(port)		YT921X_STP_PORTn(port, 3)
-#define YT921X_PORTn_LEARN(port)	(0x1803d0 + 4 * (port))
-#define  YT921X_PORT_LEARN_VID_LEARN_MULTI_EN	BIT(22)
-#define  YT921X_PORT_LEARN_VID_LEARN_MODE	BIT(21)
-#define  YT921X_PORT_LEARN_VID_LEARN_EN		BIT(20)
-#define  YT921X_PORT_LEARN_SUSPEND_COPY_EN	BIT(19)
-#define  YT921X_PORT_LEARN_SUSPEND_DROP_EN	BIT(18)
-#define  YT921X_PORT_LEARN_DIS			BIT(17)
-#define  YT921X_PORT_LEARN_LIMIT_EN		BIT(16)
-#define  YT921X_PORT_LEARN_LIMIT_M		GENMASK(15, 8)
-#define   YT921X_PORT_LEARN_LIMIT(x)			FIELD_PREP(YT921X_PORT_LEARN_LIMIT_M, (x))
-#define  YT921X_PORT_LEARN_DROP_ON_EXCEEDED	BIT(2)
-#define  YT921X_PORT_LEARN_MODE_M		GENMASK(1, 0)
-#define   YT921X_PORT_LEARN_MODE(x)			FIELD_PREP(YT921X_PORT_LEARN_MODE_M, (x))
-#define   YT921X_PORT_LEARN_MODE_AUTO			YT921X_PORT_LEARN_MODE(0)
-#define   YT921X_PORT_LEARN_MODE_AUTO_AND_COPY		YT921X_PORT_LEARN_MODE(1)
-#define   YT921X_PORT_LEARN_MODE_CPU_CONTROL		YT921X_PORT_LEARN_MODE(2)
-#define YT921X_AGEING			0x180440
-#define  YT921X_AGEING_INTERVAL_M		GENMASK(15, 0)
-#define YT921X_FDB_IN0			0x180454
-#define YT921X_FDB_IN1			0x180458
-#define YT921X_FDB_IN2			0x18045c
-#define YT921X_FDB_OP			0x180460
-#define  YT921X_FDB_OP_INDEX_M			GENMASK(22, 11)
-#define   YT921X_FDB_OP_INDEX(x)			FIELD_PREP(YT921X_FDB_OP_INDEX_M, (x))
-#define  YT921X_FDB_OP_MODE_INDEX		BIT(10)  /* mac+fid / index */
-#define  YT921X_FDB_OP_FLUSH_MCAST		BIT(9)  /* ucast / mcast */
-#define  YT921X_FDB_OP_FLUSH_M			GENMASK(8, 7)
-#define   YT921X_FDB_OP_FLUSH(x)			FIELD_PREP(YT921X_FDB_OP_FLUSH_M, (x))
-#define   YT921X_FDB_OP_FLUSH_ALL			YT921X_FDB_OP_FLUSH(0)
-#define   YT921X_FDB_OP_FLUSH_PORT			YT921X_FDB_OP_FLUSH(1)
-#define   YT921X_FDB_OP_FLUSH_PORT_VID			YT921X_FDB_OP_FLUSH(2)
-#define   YT921X_FDB_OP_FLUSH_VID			YT921X_FDB_OP_FLUSH(3)
-#define  YT921X_FDB_OP_FLUSH_STATIC		BIT(6)
-#define  YT921X_FDB_OP_NEXT_TYPE_M		GENMASK(5, 4)
-#define   YT921X_FDB_OP_NEXT_TYPE(x)			FIELD_PREP(YT921X_FDB_OP_NEXT_TYPE_M, (x))
-#define   YT921X_FDB_OP_NEXT_TYPE_UCAST_PORT		YT921X_FDB_OP_NEXT_TYPE(0)
-#define   YT921X_FDB_OP_NEXT_TYPE_UCAST_VID		YT921X_FDB_OP_NEXT_TYPE(1)
-#define   YT921X_FDB_OP_NEXT_TYPE_UCAST			YT921X_FDB_OP_NEXT_TYPE(2)
-#define   YT921X_FDB_OP_NEXT_TYPE_MCAST			YT921X_FDB_OP_NEXT_TYPE(3)
-#define  YT921X_FDB_OP_OP_M			GENMASK(3, 1)
-#define   YT921X_FDB_OP_OP(x)				FIELD_PREP(YT921X_FDB_OP_OP_M, (x))
-#define   YT921X_FDB_OP_OP_ADD				YT921X_FDB_OP_OP(0)
-#define   YT921X_FDB_OP_OP_DEL				YT921X_FDB_OP_OP(1)
-#define   YT921X_FDB_OP_OP_GET_ONE			YT921X_FDB_OP_OP(2)
-#define   YT921X_FDB_OP_OP_GET_NEXT			YT921X_FDB_OP_OP(3)
-#define   YT921X_FDB_OP_OP_FLUSH			YT921X_FDB_OP_OP(4)
-#define  YT921X_FDB_OP_START			BIT(0)
-#define YT921X_FDB_RESULT		0x180464
-#define  YT921X_FDB_RESULT_DONE			BIT(15)
-#define  YT921X_FDB_RESULT_NOTFOUND		BIT(14)
-#define  YT921X_FDB_RESULT_OVERWRITED		BIT(13)
-#define  YT921X_FDB_RESULT_INDEX_M		GENMASK(11, 0)
-#define   YT921X_FDB_RESULT_INDEX(x)			FIELD_PREP(YT921X_FDB_RESULT_INDEX_M, (x))
-#define YT921X_FDB_OUT0			0x1804b0
-#define  YT921X_FDB_IO0_ADDR_HI4_M		GENMASK(31, 0)
-#define YT921X_FDB_OUT1			0x1804b4
-#define  YT921X_FDB_IO1_EGR_PRIO_EN		BIT(31)
-#define  YT921X_FDB_IO1_STATUS_M		GENMASK(30, 28)
-#define   YT921X_FDB_IO1_STATUS(x)			FIELD_PREP(YT921X_FDB_IO1_STATUS_M, (x))
-#define   YT921X_FDB_IO1_STATUS_INVALID			YT921X_FDB_IO1_STATUS(0)
-#define   YT921X_FDB_IO1_STATUS_MIN_TIME		YT921X_FDB_IO1_STATUS(1)
-#define   YT921X_FDB_IO1_STATUS_MOVE_AGING_MAX_TIME	YT921X_FDB_IO1_STATUS(3)
-#define   YT921X_FDB_IO1_STATUS_MAX_TIME		YT921X_FDB_IO1_STATUS(5)
-#define   YT921X_FDB_IO1_STATUS_PENDING			YT921X_FDB_IO1_STATUS(6)
-#define   YT921X_FDB_IO1_STATUS_STATIC			YT921X_FDB_IO1_STATUS(7)
-#define  YT921X_FDB_IO1_FID_M			GENMASK(27, 16)  /* filtering ID (VID) */
-#define   YT921X_FDB_IO1_FID(x)				FIELD_PREP(YT921X_FDB_IO1_FID_M, (x))
-#define  YT921X_FDB_IO1_ADDR_LO2_M		GENMASK(15, 0)
-#define YT921X_FDB_OUT2			0x1804b8
-#define  YT921X_FDB_IO2_MOVE_AGING_STATUS_M	GENMASK(31, 30)
-#define  YT921X_FDB_IO2_IGR_DROP		BIT(29)
-#define  YT921X_FDB_IO2_EGR_PORTS_M		GENMASK(28, 18)
-#define   YT921X_FDB_IO2_EGR_PORTS(x)			FIELD_PREP(YT921X_FDB_IO2_EGR_PORTS_M, (x))
-#define  YT921X_FDB_IO2_EGR_DROP		BIT(17)
-#define  YT921X_FDB_IO2_COPY_TO_CPU		BIT(16)
-#define  YT921X_FDB_IO2_IGR_PRIO_EN		BIT(15)
-#define  YT921X_FDB_IO2_PRIO_M			GENMASK(14, 12)
-#define   YT921X_FDB_IO2_PRIO(x)			FIELD_PREP(YT921X_FDB_IO2_PRIO_M, (x))
-#define  YT921X_FDB_IO2_NEW_VID_M		GENMASK(11, 0)
-#define   YT921X_FDB_IO2_NEW_VID(x)			FIELD_PREP(YT921X_FDB_IO2_NEW_VID_M, (x))
-#define YT921X_FILTER_UNK_UCAST		0x180508
-#define YT921X_FILTER_UNK_MCAST		0x18050c
-#define YT921X_FILTER_MCAST		0x180510
-#define YT921X_FILTER_BCAST		0x180514
-#define  YT921X_FILTER_PORTS_M			GENMASK(10, 0)
-#define   YT921X_FILTER_PORTS(x)			FIELD_PREP(YT921X_FILTER_PORTS_M, (x))
-#define  YT921X_FILTER_PORTn(port)		BIT(port)
-#define YT921X_VLAN_EGR_FILTER		0x180598
-#define  YT921X_VLAN_EGR_FILTER_PORTn(port)	BIT(port)
-#define YT921X_LAG_GROUPn(n)		(0x1805a8 + 4 * (n))
-#define  YT921X_LAG_GROUP_PORTS_M		GENMASK(13, 3)
-#define   YT921X_LAG_GROUP_PORTS(x)			FIELD_PREP(YT921X_LAG_GROUP_PORTS_M, (x))
-#define  YT921X_LAG_GROUP_MEMBER_NUM_M		GENMASK(2, 0)
-#define   YT921X_LAG_GROUP_MEMBER_NUM(x)		FIELD_PREP(YT921X_LAG_GROUP_MEMBER_NUM_M, (x))
-#define YT921X_LAG_MEMBERnm(n, m)	(0x1805b0 + 4 * (4 * (n) + (m)))
-#define  YT921X_LAG_MEMBER_PORT_M		GENMASK(3, 0)
-#define   YT921X_LAG_MEMBER_PORT(x)			FIELD_PREP(YT921X_LAG_MEMBER_PORT_M, (x))
-#define YT921X_CPU_COPY			0x180690
-#define  YT921X_CPU_COPY_FORCE_INT_PORT		BIT(2)
-#define  YT921X_CPU_COPY_TO_INT_CPU		BIT(1)
-#define  YT921X_CPU_COPY_TO_EXT_CPU		BIT(0)
-#define YT921X_ACL_PERMIT_UNMATCH	0x1806a0
-#define  YT921X_ACL_PERMIT_UNMATCH_PORTS_M	GENMASK(10, 0)
-#define   YT921X_ACL_PERMIT_UNMATCH_PORTS(x)		FIELD_PREP(YT921X_ACL_PERMIT_UNMATCH_PORTS_M, (x))
-#define  YT921X_ACL_PERMIT_UNMATCH_PORTn(port)	BIT(port)
-#define YT921X_ACT_UNK_UCAST		0x180734
-#define YT921X_ACT_UNK_MCAST		0x180738
-#define  YT921X_ACT_UNK_MCAST_BYPASS_DROP_RMA	BIT(23)
-#define  YT921X_ACT_UNK_MCAST_BYPASS_DROP_IGMP	BIT(22)
-#define  YT921X_ACT_UNK_ACTn_M(port)		GENMASK(2 * (port) + 1, 2 * (port))
-#define   YT921X_ACT_UNK_ACTn(port, x)			((x) << (2 * (port)))
-#define   YT921X_ACT_UNK_ACTn_FORWARD(port)		YT921X_ACT_UNK_ACTn(port, 0)  /* flood */
-#define   YT921X_ACT_UNK_ACTn_TRAP(port)		YT921X_ACT_UNK_ACTn(port, 1)  /* steer to CPU */
-#define   YT921X_ACT_UNK_ACTn_DROP(port)		YT921X_ACT_UNK_ACTn(port, 2)  /* discard */
-/* NEVER use this action; see comments in the tag driver */
-#define   YT921X_ACT_UNK_ACTn_COPY(port)		YT921X_ACT_UNK_ACTn(port, 3)  /* flood and copy */
-#define YT921X_FDB_HW_FLUSH		0x180958
-#define  YT921X_FDB_HW_FLUSH_ON_LINKDOWN	BIT(0)
-
-#define YT921X_VLANn_CTRL(vlan)		(0x188000 + 8 * (vlan))
-#define  YT921X_VLAN_CTRLb_UNTAG_PORTS_M	GENMASK(18, 8)
-#define   YT921X_VLAN_CTRLb_UNTAG_PORTS(x)		FIELD_PREP(YT921X_VLAN_CTRLb_UNTAG_PORTS_M, (x))
-#define  YT921X_VLAN_CTRLb_UNTAG_PORTn(port)	BIT((port) + 8)
-#define  YT921X_VLAN_CTRLb_STP_ID_M		GENMASK(7, 4)
-#define   YT921X_VLAN_CTRLb_STP_ID(x)			FIELD_PREP(YT921X_VLAN_CTRLb_STP_ID_M, (x))
-#define  YT921X_VLAN_CTRLb_SVLAN_EN		BIT(3)
-#define  YT921X_VLAN_CTRLab_FID_M		GENMASK_ULL(34, 23)
-#define   YT921X_VLAN_CTRLab_FID(x)			FIELD_PREP(YT921X_VLAN_CTRLab_FID_M, (x))
-#define  YT921X_VLAN_CTRLa_LEARN_DIS		BIT(22)
-#define  YT921X_VLAN_CTRLa_PRIO_EN		BIT(21)
-#define  YT921X_VLAN_CTRLa_PRIO_M		GENMASK(20, 18)
-#define   YT921X_VLAN_CTRLa_PRIO(x)			FIELD_PREP(YT921X_VLAN_CTRLa_PRIO_M, (x))
-#define  YT921X_VLAN_CTRLa_PORTS_M		GENMASK(17, 7)
-#define   YT921X_VLAN_CTRLa_PORTS(x)			FIELD_PREP(YT921X_VLAN_CTRLa_PORTS_M, (x))
-#define  YT921X_VLAN_CTRLa_PORTn(port)		BIT((port) + 7)
-#define  YT921X_VLAN_CTRLa_BYPASS_1X_AC		BIT(6)
-#define  YT921X_VLAN_CTRLa_METER_EN		BIT(5)
-#define  YT921X_VLAN_CTRLa_METER_ID_M		GENMASK(4, 0)
-
-#define YT921X_ACLn_ACT(n)		(0x1c0000 + 0x10 * (n))
-#define  YT921X_ACL_ACTc_STAG_M			GENMASK(26, 25)
-#define   YT921X_ACL_ACTc_STAG(x)			FIELD_PREP(YT921X_ACL_ACTc_STAG_M, (x))
-#define   YT921X_ACL_ACTc_STAG_DONTCARE			YT921X_ACL_ACTc_STAG(0)
-#define   YT921X_ACL_ACTc_STAG_UNTAG			YT921X_ACL_ACTc_STAG(1)
-#define   YT921X_ACL_ACTc_STAG_TAG			YT921X_ACL_ACTc_STAG(2)
-#define   YT921X_ACL_ACTc_STAG_KEEP			YT921X_ACL_ACTc_STAG(3)
-#define  YT921X_ACL_ACTc_CTAG_M			GENMASK(24, 23)
-#define   YT921X_ACL_ACTc_CTAG(x)			FIELD_PREP(YT921X_ACL_ACTc_CTAG_M, (x))
-#define   YT921X_ACL_ACTc_CTAG_DONTCARE			YT921X_ACL_ACTc_CTAG(0)
-#define   YT921X_ACL_ACTc_CTAG_UNTAG			YT921X_ACL_ACTc_CTAG(1)
-#define   YT921X_ACL_ACTc_CTAG_TAG			YT921X_ACL_ACTc_CTAG(2)
-#define   YT921X_ACL_ACTc_CTAG_KEEP			YT921X_ACL_ACTc_CTAG(3)
-#define  YT921X_ACL_ACTc_FWD_M			GENMASK(22, 21)
-#define   YT921X_ACL_ACTc_FWD(x)			FIELD_PREP(YT921X_ACL_ACTc_FWD_M, (x))
-#define   YT921X_ACL_ACTc_FWD_FWD			YT921X_ACL_ACTc_FWD(0)
-#define   YT921X_ACL_ACTc_FWD_COPY			YT921X_ACL_ACTc_FWD(1)
-#define   YT921X_ACL_ACTc_FWD_REDIR			YT921X_ACL_ACTc_FWD(2)
-#define   YT921X_ACL_ACTc_FWD_TRAP			YT921X_ACL_ACTc_FWD(3)
-#define  YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M	GENMASK(20, 10)
-#define   YT921X_ACL_ACTc_FWD_REDIR_DPORTS(x)		FIELD_PREP(YT921X_ACL_ACTc_FWD_REDIR_DPORTS_M, (x))
-#define  YT921X_ACL_ACTc_FWD_REDIR_DPORTn(port)	BIT((port) + 10)
-#define  YT921X_ACL_ACTc_FWD_EN			BIT(9)
-#define  YT921X_ACL_ACTc_SDEI			BIT(8)
-#define  YT921X_ACL_ACTc_SDEI_REPLACE		BIT(7)
-#define  YT921X_ACL_ACTc_SPRI_M			GENMASK(6, 4)
-#define   YT921X_ACL_ACTc_SPRI(x)			FIELD_PREP(YT921X_ACL_ACTc_SPRI_M, (x))
-#define  YT921X_ACL_ACTc_SPRI_REPLACE		BIT(3)
-#define  YT921X_ACL_ACTbc_SVID_M		GENMASK_ULL(34, 23)
-#define   YT921X_ACL_ACTbc_SVID(x)			FIELD_PREP(YT921X_ACL_ACTbc_SVID_M, (x))
-#define  YT921X_ACL_ACTb_SVID_REPLACE		BIT(22)
-#define  YT921X_ACL_ACTb_CDEI			BIT(21)
-#define  YT921X_ACL_ACTb_CDEI_REPLACE		BIT(20)
-#define  YT921X_ACL_ACTb_CPRI_M			GENMASK(19, 17)
-#define   YT921X_ACL_ACTb_CPRI(x)			FIELD_PREP(YT921X_ACL_ACTb_CPRI_M, (x))
-#define  YT921X_ACL_ACTb_CPRI_REPLACE		BIT(16)
-#define  YT921X_ACL_ACTb_CVID_M			GENMASK(15, 4)
-#define   YT921X_ACL_ACTb_CVID(x)			FIELD_PREP(YT921X_ACL_ACTb_CVID_M, (x))
-#define  YT921X_ACL_ACTb_CVID_REPLACE		BIT(3)
-#define  YT921X_ACL_ACTb_PRIO_M			GENMASK(2, 0)
-#define   YT921X_ACL_ACTb_PRIO(x)			FIELD_PREP(YT921X_ACL_ACTb_PRIO_M, (x))
-#define  YT921X_ACL_ACTa_PRIO_EN		BIT(31)
-#define  YT921X_ACL_ACTa_COLOR_M		GENMASK(30, 29)
-#define   YT921X_ACL_ACTa_COLOR(x)			FIELD_PREP(YT921X_ACL_ACTa_COLOR_M, (x))
-#define   YT921X_ACL_ACTa_COLOR_GREEN			YT921X_ACL_ACTa_COLOR(0)
-#define   YT921X_ACL_ACTa_COLOR_YELLOW			YT921X_ACL_ACTa_COLOR(1)
-#define   YT921X_ACL_ACTa_COLOR_RED			YT921X_ACL_ACTa_COLOR(2)
-#define  YT921X_ACL_ACTa_COLOR_EN		BIT(28)
-#define  YT921X_ACL_ACTa_DSCP_M			GENMASK(27, 22)
-#define   YT921X_ACL_ACTa_DSCP(x)			FIELD_PREP(YT921X_ACL_ACTa_DSCP_M, (x))
-#define  YT921X_ACL_ACTa_DSCP_REPLACE		BIT(21)
-#define  YT921X_ACL_ACTa_METER_ID_M		GENMASK(20, 15)
-#define   YT921X_ACL_ACTa_METER_ID(x)			FIELD_PREP(YT921X_ACL_ACTa_METER_ID_M, (x))
-#define  YT921X_ACL_ACTa_METER_EN		BIT(14)
-#define  YT921X_ACL_ACTa_MIRROR_EN		BIT(13)
-#define  YT921X_ACL_ACTa_FLOWSTAT_EN		BIT(12)
-#define  YT921X_ACL_ACTa_FLOWSTAT_ID_M		GENMASK(11, 6)
-#define   YT921X_ACL_ACTa_FLOWSTAT_ID(x)		FIELD_PREP(YT921X_ACL_ACTa_FLOWSTAT_ID_M, (x))
-#define  YT921X_ACL_ACTa_GPIO_EN		BIT(5)
-#define  YT921X_ACL_ACTa_GPIO_PIN_M		GENMASK(4, 1)
-#define   YT921X_ACL_ACTa_GPIO_PIN(x)			FIELD_PREP(YT921X_ACL_ACTa_GPIO_PIN_M, (x))
-#define  YT921X_ACL_ACTa_INTR_EN		BIT(0)
-#define YT921X_ACL_BLK_KEEP		0x201000
-#define  YT921X_ACL_BLK_KEEP_GRPIDn_M(bin)	(7 << (4 * (bin) + 1))
-#define   YT921X_ACL_BLK_KEEP_GRPIDn(bin, x)		((x) << (4 * (bin) + 1))
-#define  YT921X_ACL_BLK_KEEP_KEEPn(bin)		BIT(4 * (bin))
-#define YT921X_ACL_PORT			0x202000
-#define  YT921X_ACL_PORT_PORTS_M		GENMASK(10, 0)
-#define   YT921X_ACL_PORT_PORTS(x)			FIELD_PREP(YT921X_ACL_PORT_PORTS_M, (x))
-#define  YT921X_ACL_PORT_PORTn(port)		BIT(port)
-#define YT921X_ACL_BLK_CMD		0x202004
-#define  YT921X_ACL_BLK_CMD_BLKID_M		GENMASK(6, 1)
-#define   YT921X_ACL_BLK_CMD_BLKID(x)			FIELD_PREP(YT921X_ACL_BLK_CMD_BLKID_M, (x))
-#define  YT921X_ACL_BLK_CMD_MODIFY		BIT(0)
-#define YT921X_ACLn_ENTRY(blk)		(0x203000 + 4 * (blk))
-#define  YT921X_ACL_ENTRY_GRPIDm_M(bin)		(7 << (4 * (bin) + 1))
-#define   YT921X_ACL_ENTRY_GRPIDm(bin, x)		((x) << (4 * (bin) + 1))
-#define  YT921X_ACL_ENTRY_ENm(bin)		BIT(4 * (bin))
-#define YT921X_ACLn_KEYm(blk, bin)	(0x204000 + 0x200 * (bin) + 8 * (blk))
-#define  YT921X_ACL_KEYb_ORD_M			GENMASK(29, 21)
-#define   YT921X_ACL_KEYb_ORD(x)			FIELD_PREP(YT921X_ACL_KEYb_ORD_M, (x))
-#define  YT921X_ACL_KEYb_SPORTS_M		GENMASK(20, 10)
-#define   YT921X_ACL_KEYb_SPORTS(x)			FIELD_PREP(YT921X_ACL_KEYb_SPORTS_M, (x))
-#define  YT921X_ACL_KEYb_SPORTn(port)		BIT((port) + 10)
-#define  YT921X_ACL_KEYb_REVERSE		BIT(9)	/* reverse match */
-#define  YT921X_ACL_KEYb_TYPE_M			GENMASK(8, 4)
-#define   YT921X_ACL_KEYb_TYPE(x)			FIELD_PREP(YT921X_ACL_KEYb_TYPE_M, (x))
-/* KEY_* fields need no masks */
-#define YT921X_ACLn_MASKm(blk, bin)	(0x205000 + 0x200 * (bin) + 8 * (blk))
-
-enum yt921x_acl_type {
-	YT921X_ACL_TYPE_NA,
-	YT921X_ACL_TYPE_MAC_DA0,
-	YT921X_ACL_TYPE_MAC_SA0,
-	YT921X_ACL_TYPE_MAC_DA1_SA1,
-	YT921X_ACL_TYPE_VLAN,
-	YT921X_ACL_TYPE_VTAG,
-	YT921X_ACL_TYPE_IPV4_DA,
-	YT921X_ACL_TYPE_IPV4_SA,
-	YT921X_ACL_TYPE_IPV6_DA0,
-	YT921X_ACL_TYPE_IPV6_DA1,
-	YT921X_ACL_TYPE_IPV6_DA2,
-	YT921X_ACL_TYPE_IPV6_DA3,
-	YT921X_ACL_TYPE_IPV6_SA0,
-	YT921X_ACL_TYPE_IPV6_SA1,
-	YT921X_ACL_TYPE_IPV6_SA2,
-	YT921X_ACL_TYPE_IPV6_SA3,
-	YT921X_ACL_TYPE_MISC,
-	YT921X_ACL_TYPE_L4,
-	YT921X_ACL_TYPE_UDF0,
-	YT921X_ACL_TYPE_UDF1,
-	YT921X_ACL_TYPE_UDF2,
-	YT921X_ACL_TYPE_UDF3,
-	YT921X_ACL_TYPE_UDF4,
-	YT921X_ACL_TYPE_UDF5,
-	YT921X_ACL_TYPE_UDF6,
-	YT921X_ACL_TYPE_UDF7,
-	YT921X_ACL_TYPE_ETHERTYPE,
-	YT921X_ACL_TYPE_NUM
-};
-
-/* Range: turn KEY:MASK into MIN:MAX */
-
-#define  YT921X_ACL_BINb_MAC_xA0_L3_TYPE_M	GENMASK(3, 0)
-#define   YT921X_ACL_BINb_MAC_xA0_L3_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_MAC_xA0_L3_TYPE_M, (x))
-#define  YT921X_ACL_BINa_MAC_xA0_MAC_xA0_M	GENMASK(31, 0)
-
-#define  YT921X_ACL_BINb_MAC_DA1_SA1_L2_TYPE_M	GENMASK(2, 0)
-#define   YT921X_ACL_BINb_MAC_DA1_SA1_L2_TYPE(x)	FIELD_PREP(YT921X_ACL_BINb_MAC_DA1_SA1_L2_TYPE_M, (x))
-#define  YT921X_ACL_BINa_MAC_DA1_SA1_MAC_DA1_M	GENMASK(31, 16)
-#define  YT921X_ACL_BINa_MAC_DA1_SA1_MAC_SA1_M	GENMASK(15, 0)
-
-#define  YT921X_ACL_KEYb_VLAN_SVID_RANGE_EN	BIT(31)
-#define  YT921X_ACL_KEYb_VLAN_CVID_RANGE_EN	BIT(30)
-#define  YT921X_ACL_BINb_VLAN_CDEI		BIT(3)
-#define  YT921X_ACL_BINb_VLAN_CPRI_M		GENMASK(2, 0)
-#define   YT921X_ACL_BINb_VLAN_CPRI(x)			FIELD_PREP(YT921X_ACL_BINb_VLAN_CPRI_M, (x))
-#define  YT921X_ACL_BINa_VLAN_CTAG_FMT_M	GENMASK(31, 30)
-#define   YT921X_ACL_BINa_VLAN_CTAG_FMT(x)		FIELD_PREP(YT921X_ACL_BINa_VLAN_CTAG_FMT_M, (x))
-#define  YT921X_ACL_BINa_VLAN_SDEI		BIT(29)
-#define  YT921X_ACL_BINa_VLAN_SPRI_M		GENMASK(28, 26)
-#define   YT921X_ACL_BINa_VLAN_SPRI(x)			FIELD_PREP(YT921X_ACL_BINa_VLAN_SPRI_M, (x))
-#define  YT921X_ACL_BINa_VLAN_STAG_FMT_M	GENMASK(25, 24)
-#define   YT921X_ACL_BINa_VLAN_STAG_FMT(x)		FIELD_PREP(YT921X_ACL_BINa_VLAN_STAG_FMT_M, (x))
-#define  YT921X_ACL_BINa_VLAN_SVID_M		GENMASK(23, 12)
-#define   YT921X_ACL_BINa_VLAN_SVID(x)			FIELD_PREP(YT921X_ACL_BINa_VLAN_SVID_M, (x))
-#define  YT921X_ACL_BINa_VLAN_CVID_M		GENMASK(11, 0)
-#define   YT921X_ACL_BINa_VLAN_CVID(x)			FIELD_PREP(YT921X_ACL_BINa_VLAN_CVID_M, (x))
-
-#define  YT921X_ACL_KEYb_VTAG_SVID_RANGE_EN	BIT(31)
-#define  YT921X_ACL_KEYb_VTAG_CVID_RANGE_EN	BIT(30)
-#define  YT921X_ACL_BINa_VTAG_CDEI		BIT(31)
-#define  YT921X_ACL_BINa_VTAG_CPRI_M		GENMASK(30, 28)
-#define   YT921X_ACL_BINa_VTAG_CPRI(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_CPRI_M, (x))
-#define  YT921X_ACL_BINa_VTAG_SDEI		BIT(27)
-#define  YT921X_ACL_BINa_VTAG_SPRI_M		GENMASK(26, 24)
-#define   YT921X_ACL_BINa_VTAG_SPRI(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_SPRI_M, (x))
-#define  YT921X_ACL_BINa_VTAG_SVID_M		GENMASK(23, 12)
-#define   YT921X_ACL_BINa_VTAG_SVID(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_SVID_M, (x))
-#define  YT921X_ACL_BINa_VTAG_CVID_M		GENMASK(11, 0)
-#define   YT921X_ACL_BINa_VTAG_CVID(x)			FIELD_PREP(YT921X_ACL_BINa_VTAG_CVID_M, (x))
-
-#define  YT921X_ACL_KEYb_IPV4_ADDR_RANGE_EN	BIT(30)
-#define  YT921X_ACL_BINb_IPV4_FRAG		BIT(3)
-#define  YT921X_ACL_BINb_IPV4_L4_TYPE_M		GENMASK(2, 0)
-#define   YT921X_ACL_BINb_IPV4_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_IPV4_L4_TYPE_M, (x))
-#define  YT921X_ACL_BINa_IPV4_ADDR_M		GENMASK(31, 0)
-
-#define  YT921X_ACL_BINb_IPV6_L4_TYPE_M		GENMASK(2, 0)
-#define   YT921X_ACL_BINb_IPV6_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_IPV6_L4_TYPE_M, (x))
-#define  YT921X_ACL_BINa_IPV6_ADDRx_M		GENMASK(31, 0)
-
-#define  YT921X_ACL_BINb_IPV6_xA1_IP_OPTION	BIT(3)
-
-#define  YT921X_ACL_BINb_IPV6_xA2_FIRST_FRAG	BIT(3)
-
-#define  YT921X_ACL_KEYb_IPV6_xA3_ADDR_RANGE_EN	BIT(30)
-#define  YT921X_ACL_BINb_IPV6_xA3_FRAG		BIT(3)
-
-#define  YT921X_ACL_BINb_MISC_FRAG		BIT(3)
-#define  YT921X_ACL_BINb_MISC_L4_TYPE_M		GENMASK(2, 0)
-#define   YT921X_ACL_BINb_MISC_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_MISC_L4_TYPE_M, (x))
-#define  YT921X_ACL_BINa_MISC_PPPOE_FLAG	BIT(30)
-#define  YT921X_ACL_BINa_MISC_FIRST_FRAG	BIT(29)
-#define  YT921X_ACL_BINa_MISC_IP_OPTION		BIT(28)
-#define  YT921X_ACL_BINa_MISC_TCP_FLAGS_M	GENMASK(27, 20)
-#define   YT921X_ACL_BINa_MISC_TCP_FLAGS(x)		FIELD_PREP(YT921X_ACL_BINa_MISC_TCP_FLAGS_M, (x))
-#define  YT921X_ACL_BINa_MISC_IP_PROTO_M	GENMASK(19, 12)
-#define   YT921X_ACL_BINa_MISC_IP_PROTO(x)		FIELD_PREP(YT921X_ACL_BINa_MISC_IP_PROTO_M, (x))
-#define  YT921X_ACL_BINa_MISC_TOS_M		GENMASK(11, 4)
-#define   YT921X_ACL_BINa_MISC_TOS(x)			FIELD_PREP(YT921X_ACL_BINa_MISC_TOS_M, (x))
-#define  YT921X_ACL_BINa_MISC_L3_TYPE_M		GENMASK(3, 0)
-#define   YT921X_ACL_BINa_MISC_L3_TYPE(x)		FIELD_PREP(YT921X_ACL_BINa_MISC_L3_TYPE_M, (x))
-
-#define  YT921X_ACL_KEYb_L4_DPORT_RANGE_EN	BIT(31)
-#define  YT921X_ACL_KEYb_L4_SPORT_RANGE_EN	BIT(30)
-#define  YT921X_ACL_BINb_L4_FRAG		BIT(3)
-#define  YT921X_ACL_BINb_L4_TYPE_M		GENMASK(2, 0)
-#define   YT921X_ACL_BINb_L4_TYPE(x)			FIELD_PREP(YT921X_ACL_BINb_L4_TYPE_M, (x))
-#define  YT921X_ACL_BINa_L4_DPORT_M		GENMASK(31, 16)
-#define  YT921X_ACL_BINa_L4_SPORT_M		GENMASK(15, 0)
-
-#define  YT921X_ACL_BINb_UDF_IS_IGMP		BIT(0)
-#define  YT921X_ACL_BINa_UDF_UDF0_M		GENMASK(31, 16)
-#define   YT921X_ACL_BINa_UDF_UDF0(x)			FIELD_PREP(YT921X_ACL_BINa_UDF_UDF0_M, (x))
-#define  YT921X_ACL_BINa_UDF_UDF1_M		GENMASK(15, 0)
-#define   YT921X_ACL_BINa_UDF_UDF1(x)			FIELD_PREP(YT921X_ACL_BINa_UDF_UDF1_M, (x))
-
-#define  YT921X_ACL_KEYb_ETHERTYPE_ETHERTYPE_RANGE_EN	BIT(30)
-#define  YT921X_ACL_BINb_ETHERTYPE_L4_TYPE_M	GENMASK(2, 0)
-#define   YT921X_ACL_BINb_ETHERTYPE_L4_TYPE(x)		FIELD_PREP(YT921X_ACL_BINb_ETHERTYPE_L4_TYPE_M, (x))
-#define  YT921X_ACL_BINa_ETHERTYPE_ETHERTYPE_M	GENMASK(15, 0)
-#define   YT921X_ACL_BINa_ETHERTYPE_ETHERTYPE(x)	FIELD_PREP(YT921X_ACL_BINa_ETHERTYPE_ETHERTYPE_M, (x))
-
-enum yt921x_l2_type {
-	YT921X_L2_TYPE_ETH,
-	YT921X_L2_TYPE_ETHV2,
-	YT921X_L2_TYPE_ETHSAP,
-	YT921X_L2_TYPE_ETHSNAP,
-};
-
-enum yt921x_l3_type {
-	YT921X_L3_TYPE_OTHER,
-	YT921X_L3_TYPE_IPV4,
-	YT921X_L3_TYPE_IPV6,
-	YT921X_L3_TYPE_ARP,
-	YT921X_L3_TYPE_LLDP,
-	YT921X_L3_TYPE_PAE,
-	YT921X_L3_TYPE_ERP,
-	YT921X_L3_TYPE_SLOW_PROTOCOL,
-};
-
-enum yt921x_l4_type {
-	YT921X_L4_TYPE_OTHER,
-	YT921X_L4_TYPE_TCP,
-	YT921X_L4_TYPE_UDP,
-	YT921X_L4_TYPE_UDPLITE,
-	YT921X_L4_TYPE_ICMP,
-	YT921X_L4_TYPE_IGMP,
-	YT921X_L4_TYPE_MLD,
-	YT921X_L4_TYPE_ND,
-};
-
-#define YT921X_TPID_IGRn(x)		(0x210000 + 4 * (x))	/* [0, 3] */
-#define  YT921X_TPID_IGR_TPID_M			GENMASK(15, 0)
-#define YT921X_PORTn_IGR_TPID(port)	(0x210010 + 4 * (port))
-#define  YT921X_PORT_IGR_TPIDn_STAG_M		GENMASK(7, 4)
-#define  YT921X_PORT_IGR_TPIDn_STAG(x)		BIT((x) + 4)
-#define  YT921X_PORT_IGR_TPIDn_CTAG_M		GENMASK(3, 0)
-#define  YT921X_PORT_IGR_TPIDn_CTAG(x)		BIT(x)
-#define YT921X_LAG_HASH			0x210090
-#define  YT921X_LAG_HASH_L4_SPORT		BIT(7)
-#define  YT921X_LAG_HASH_L4_DPORT		BIT(6)
-#define  YT921X_LAG_HASH_IP_PROTO		BIT(5)
-#define  YT921X_LAG_HASH_IP_SRC			BIT(4)
-#define  YT921X_LAG_HASH_IP_DST			BIT(3)
-#define  YT921X_LAG_HASH_MAC_SA			BIT(2)
-#define  YT921X_LAG_HASH_MAC_DA			BIT(1)
-#define  YT921X_LAG_HASH_SRC_PORT		BIT(0)
-#define YT921X_UDFn_CTRL(x)		(0x210094 + 4 * (x))
-#define  YT921X_UDF_CTRL_UDF_TYPE_M		GENMASK(8, 7)
-#define   YT921X_UDF_CTRL_UDF_TYPE(x)			FIELD_PREP(YT921X_UDF_CTRL_UDF_TYPE_M, (x))
-#define   YT921X_UDF_CTRL_UDF_TYPE_ETH			YT921X_UDF_CTRL_UDF_TYPE(0)
-#define   YT921X_UDF_CTRL_UDF_TYPE_L3			YT921X_UDF_CTRL_UDF_TYPE(1)
-#define   YT921X_UDF_CTRL_UDF_TYPE_L4			YT921X_UDF_CTRL_UDF_TYPE(2)
-#define  YT921X_UDF_CTRL_UDF_OFFSET_M		GENMASK(6, 0)
-#define   YT921X_UDF_CTRL_UDF_OFFSET(x)			FIELD_PREP(YT921X_UDF_CTRL_UDF_OFFSET_M, (x))
-
-#define YT921X_PORTn_RATE(port)		(0x220000 + 4 * (port))
-#define  YT921X_PORT_RATE_GAP_VALUE		GENMASK(4, 0)	/* default 20 */
-#define YT921X_METER_SLOT		0x220104
-#define  YT921X_METER_SLOT_SLOT_M		GENMASK(11, 0)
-#define YT921X_PORTn_METER(port)	(0x220108 + 4 * (port))
-#define  YT921X_PORT_METER_EN			BIT(4)
-#define  YT921X_PORT_METER_ID_M			GENMASK(3, 0)
-#define   YT921X_PORT_METER_ID(x)			FIELD_PREP(YT921X_PORT_METER_ID_M, (x))
-#define YT921X_METERn_CTRL(x)		(0x220800 + 0x10 * (x))
-#define  YT921X_METER_CTRLc_METER_EN		BIT(14)
-#define  YT921X_METER_CTRLc_TOKEN_OVERFLOW_EN	BIT(13)	/* RFC4115: yellow use unused green bw */
-#define  YT921X_METER_CTRLc_DROP_M		GENMASK(12, 11)
-#define   YT921X_METER_CTRLc_DROP(x)			FIELD_PREP(YT921X_METER_CTRLc_DROP_M, (x))
-#define   YT921X_METER_CTRLc_DROP_GYR			YT921X_METER_CTRLc_DROP(0)
-#define   YT921X_METER_CTRLc_DROP_YR			YT921X_METER_CTRLc_DROP(1)
-#define   YT921X_METER_CTRLc_DROP_R			YT921X_METER_CTRLc_DROP(2)
-#define   YT921X_METER_CTRLc_DROP_NONE			YT921X_METER_CTRLc_DROP(3)
-#define  YT921X_METER_CTRLc_COLOR_BLIND		BIT(10)
-#define  YT921X_METER_CTRLc_UNIT_M		GENMASK(9, 7)
-#define   YT921X_METER_CTRLc_UNIT(x)			FIELD_PREP(YT921X_METER_CTRLc_UNIT_M, (x))
-#define  YT921X_METER_CTRLc_BYTE_MODE_INCLUDE_GAP	BIT(6)	/* +GAP_VALUE bytes each packet */
-#define  YT921X_METER_CTRLc_PKT_MODE		BIT(5)	/* 0: byte rate mode */
-#define  YT921X_METER_CTRLc_RFC2698		BIT(4)	/* 0: RFC4115 */
-#define  YT921X_METER_CTRLbc_CBS_M		GENMASK_ULL(35, 20)
-#define   YT921X_METER_CTRLbc_CBS(x)			FIELD_PREP(YT921X_METER_CTRLbc_CBS_M, (x))
-#define  YT921X_METER_CTRLb_CIR_M		GENMASK(19, 2)
-#define   YT921X_METER_CTRLb_CIR(x)			FIELD_PREP(YT921X_METER_CTRLb_CIR_M, (x))
-#define  YT921X_METER_CTRLab_EBS_M		GENMASK_ULL(33, 18)
-#define   YT921X_METER_CTRLab_EBS(x)			FIELD_PREP(YT921X_METER_CTRLab_EBS_M, (x))
-#define  YT921X_METER_CTRLa_EIR_M		GENMASK(17, 0)
-#define   YT921X_METER_CTRLa_EIR(x)			FIELD_PREP(YT921X_METER_CTRLa_EIR_M, (x))
-#define YT921X_METERn_STAT(x)		(0x221000 + 8 * (x))
-
-#define YT921X_PORTn_VLAN_CTRL(port)	(0x230010 + 4 * (port))
-#define  YT921X_PORT_VLAN_CTRL_SVLAN_PRIO_EN	BIT(31)
-#define  YT921X_PORT_VLAN_CTRL_CVLAN_PRIO_EN	BIT(30)
-#define  YT921X_PORT_VLAN_CTRL_SVID_M		GENMASK(29, 18)
-#define   YT921X_PORT_VLAN_CTRL_SVID(x)			FIELD_PREP(YT921X_PORT_VLAN_CTRL_SVID_M, (x))
-#define  YT921X_PORT_VLAN_CTRL_CVID_M		GENMASK(17, 6)
-#define   YT921X_PORT_VLAN_CTRL_CVID(x)			FIELD_PREP(YT921X_PORT_VLAN_CTRL_CVID_M, (x))
-#define  YT921X_PORT_VLAN_CTRL_SVLAN_PRIO_M	GENMASK(5, 3)
-#define  YT921X_PORT_VLAN_CTRL_CVLAN_PRIO_M	GENMASK(2, 0)
-#define YT921X_PORTn_VLAN_CTRL1(port)	(0x230080 + 4 * (port))
-#define  YT921X_PORT_VLAN_CTRL1_VLAN_RANGE_EN	BIT(8)
-#define  YT921X_PORT_VLAN_CTRL1_VLAN_RANGE_PROFILE_ID_M	GENMASK(7, 4)
-#define  YT921X_PORT_VLAN_CTRL1_SVLAN_DROP_TAGGED	BIT(3)
-#define  YT921X_PORT_VLAN_CTRL1_SVLAN_DROP_UNTAGGED	BIT(2)
-#define  YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_TAGGED	BIT(1)
-#define  YT921X_PORT_VLAN_CTRL1_CVLAN_DROP_UNTAGGED	BIT(0)
-
-#define YT921X_MIRROR			0x300300
-#define  YT921X_MIRROR_IGR_PORTS_M		GENMASK(26, 16)
-#define   YT921X_MIRROR_IGR_PORTS(x)			FIELD_PREP(YT921X_MIRROR_IGR_PORTS_M, (x))
-#define  YT921X_MIRROR_IGR_PORTn(port)		BIT((port) + 16)
-#define  YT921X_MIRROR_EGR_PORTS_M		GENMASK(14, 4)
-#define   YT921X_MIRROR_EGR_PORTS(x)			FIELD_PREP(YT921X_MIRROR_EGR_PORTS_M, (x))
-#define  YT921X_MIRROR_EGR_PORTn(port)		BIT((port) + 4)
-#define  YT921X_MIRROR_PORT_M			GENMASK(3, 0)
-#define   YT921X_MIRROR_PORT(x)				FIELD_PREP(YT921X_MIRROR_PORT_M, (x))
-
-#define YT921X_PORT_SHAPE_SLOT		0x34000c
-#define  YT921X_PORT_SHAPE_SLOT_SLOT_M		GENMASK(11, 0)
-#define YT921X_PORTn_SHAPE_CTRL(port)	(0x354000 + 8 * (port))
-#define  YT921X_PORT_SHAPE_CTRLb_EN		BIT(4)
-#define  YT921X_PORT_SHAPE_CTRLb_PKT_MODE	BIT(3)	/* 0: byte rate mode */
-#define  YT921X_PORT_SHAPE_CTRLb_UNIT_M		GENMASK(2, 0)
-#define   YT921X_PORT_SHAPE_CTRLb_UNIT(x)		FIELD_PREP(YT921X_PORT_SHAPE_CTRLb_UNIT_M, (x))
-#define  YT921X_PORT_SHAPE_CTRLa_CBS_M		GENMASK(31, 18)
-#define   YT921X_PORT_SHAPE_CTRLa_CBS(x)		FIELD_PREP(YT921X_PORT_SHAPE_CTRLa_CBS_M, (x))
-#define  YT921X_PORT_SHAPE_CTRLa_CIR_M		GENMASK(17, 0)
-#define   YT921X_PORT_SHAPE_CTRLa_CIR(x)		FIELD_PREP(YT921X_PORT_SHAPE_CTRLa_CIR_M, (x))
-#define YT921X_PORTn_SHAPE_STAT(port)	(0x356000 + 4 * (port))
-
-#define YT921X_EDATA_EXTMODE	0xfb
-#define YT921X_EDATA_LEN	0x100
-
-#define YT921X_FDB_NUM	4096
-
-enum yt921x_fdb_entry_status {
-	YT921X_FDB_ENTRY_STATUS_INVALID = 0,
-	YT921X_FDB_ENTRY_STATUS_MIN_TIME = 1,
-	YT921X_FDB_ENTRY_STATUS_MOVE_AGING_MAX_TIME = 3,
-	YT921X_FDB_ENTRY_STATUS_MAX_TIME = 5,
-	YT921X_FDB_ENTRY_STATUS_PENDING = 6,
-	YT921X_FDB_ENTRY_STATUS_STATIC = 7,
-};
-
-#define YT921X_MSTI_NUM		16
-
-#define YT921X_TOKEN_BYTE_C	1	/* 1 token = 2^1 byte */
-#define YT921X_TOKEN_PKT_C	-6	/* 1 token = 2^-6 packets */
-#define YT921X_TOKEN_RATE_C	-15
-/* Custom meters only, not including dedicated port meters (11) */
-#define YT921X_METER_NUM	64
-#define YT921X_METER_SLOT_MIN	80
-#define YT921X_METER_UNIT_MAX	((1 << 3) - 1)
-#define YT921X_METER_CIR_MAX	((1 << 18) - 1)
-#define YT921X_METER_CBS_MAX	((1 << 16) - 1)
-#define YT921X_PORT_SHAPE_SLOT_MIN	80
-#define YT921X_SHAPE_UNIT_MAX	((1 << 3) - 1)
-#define YT921X_SHAPE_CIR_MAX	((1 << 18) - 1)
-#define YT921X_SHAPE_CBS_MAX	((1 << 14) - 1)
-
-#define YT921X_LAG_NUM		2
-#define YT921X_LAG_PORT_NUM	4
-
-#define YT921X_PRIO_NUM	8
-
-#define YT9215_MAJOR	0x9002
-#define YT9218_MAJOR	0x9001
-
-/* required for a hard reset */
-#define YT921X_RST_DELAY_US	10000
-
-#define YT921X_FRAME_SIZE_MAX	0x2400  /* 9216 */
-
-#define YT921X_TAG_LEN	8
-
-#define YT921X_ACL_BLK_NUM	48
-#define YT921X_ACL_ENT_PER_BLK	8
-#define YT921X_ACL_NUM		(YT921X_ACL_BLK_NUM * YT921X_ACL_ENT_PER_BLK)
-#define YT921X_UDF_NUM		8
-
-/* 8 internal + 2 external + 1 mcu */
-#define YT921X_PORT_NUM			11
-
-#define yt921x_port_is_internal(port) ((port) < 8)
-#define yt921x_port_is_external(port) ((port) == 8 || (port) == 9)
-
-struct yt921x_mib {
-	u64 rx_broadcast;
-	u64 rx_pause;
-	u64 rx_multicast;
-	u64 rx_crc_errors;
-
-	u64 rx_alignment_errors;
-	u64 rx_undersize_errors;
-	u64 rx_fragment_errors;
-	u64 rx_64byte;
-
-	u64 rx_65_127byte;
-	u64 rx_128_255byte;
-	u64 rx_256_511byte;
-	u64 rx_512_1023byte;
-
-	u64 rx_1024_1518byte;
-	u64 rx_jumbo;
-	u64 rx_good_bytes;
-
-	u64 rx_bad_bytes;
-	u64 rx_oversize_errors;
-
-	u64 rx_dropped;
-	u64 tx_broadcast;
-	u64 tx_pause;
-	u64 tx_multicast;
-
-	u64 tx_undersize_errors;
-	u64 tx_64byte;
-	u64 tx_65_127byte;
-	u64 tx_128_255byte;
-
-	u64 tx_256_511byte;
-	u64 tx_512_1023byte;
-	u64 tx_1024_1518byte;
-	u64 tx_jumbo;
-
-	u64 tx_good_bytes;
-	u64 tx_collisions;
-
-	u64 tx_aborted_errors;
-	u64 tx_multiple_collisions;
-	u64 tx_single_collisions;
-	u64 tx_good;
-
-	u64 tx_deferred;
-	u64 tx_late_collisions;
-	u64 rx_oam;
-	u64 tx_oam;
-};
-
-struct yt921x_acl_entry {
-	u32 key[2];
-	u32 mask[2];
-};
-
-struct yt921x_acl_rule {
-	unsigned long tag;
-	enum tc_setup_type type;
-
-	u32 action[3];
-	bool sw_assisted;
-
-	u8 mask;
-	struct yt921x_acl_entry entries[YT921X_ACL_ENT_PER_BLK];
-};
-
-struct yt921x_acl_blk {
-	struct yt921x_acl_rule *rules[YT921X_ACL_ENT_PER_BLK];
-};
-
-struct yt921x_port {
-	unsigned char index;
-
-	bool hairpin;
-	bool isolated;
-
-	struct delayed_work mib_read;
-	struct yt921x_mib mib;
-	u64 rx_frames;
-	u64 tx_frames;
-};
-
-struct yt921x_reg_ops {
-	int (*read)(void *context, u32 reg, u32 *valp);
-	int (*write)(void *context, u32 reg, u32 val);
-};
-
-struct yt921x_priv {
-	struct dsa_switch ds;
-
-	const struct yt921x_info *info;
-	unsigned int meter_slot_ns;
-	unsigned int port_shape_slot_ns;
-	/* cache of dsa_cpu_ports(ds) */
-	u16 cpu_ports_mask;
-	unsigned char cycle_ns;
-
-	/* protect the access to the switch registers */
-	struct mutex reg_lock;
-	const struct yt921x_reg_ops *reg_ops;
-	void *reg_ctx;
-
-	/* mdio master bus */
-	struct mii_bus *mbus_int;
-	struct mii_bus *mbus_ext;
-
-	struct yt921x_port ports[YT921X_PORT_NUM];
-
-	u16 eee_ports_mask;
-
-	DECLARE_BITMAP(meters_map, YT921X_METER_NUM);
-
-	u8 acl_masks[YT921X_ACL_BLK_NUM];
-	struct yt921x_acl_blk *acl_blks[YT921X_ACL_BLK_NUM];
-};
-
-#endif
diff --git a/drivers/net/ethernet/8390/pcnet_cs.c b/drivers/net/ethernet/8390/pcnet_cs.c
index 19f9c5db3f3ba7..0cf3c0f3ced7b2 100644
--- a/drivers/net/ethernet/8390/pcnet_cs.c
+++ b/drivers/net/ethernet/8390/pcnet_cs.c
@@ -1434,14 +1434,14 @@ static int setup_shmem_window(struct pcmcia_device *link, int start_pg,
     offset -= offset % window_size;
     ret = pcmcia_map_mem_page(link, link->resource[3], offset);
     if (ret)
-	    goto failed;
+	    goto release;
 
     /* Try scribbling on the buffer */
     info->base = ioremap(link->resource[3]->start,
 			resource_size(link->resource[3]));
     if (unlikely(!info->base)) {
 	    ret = -ENOMEM;
-	    goto failed;
+	    goto release;
     }
 
     for (i = 0; i < (TX_PAGES<<8); i += 2)
@@ -1452,9 +1452,8 @@ static int setup_shmem_window(struct pcmcia_device *link, int start_pg,
     pcnet_reset_8390(dev);
     if (i != (TX_PAGES<<8)) {
 	iounmap(info->base);
-	pcmcia_release_window(link, link->resource[3]);
 	info->base = NULL;
-	goto failed;
+	goto release;
     }
 
     ei_status.mem = info->base + offset;
@@ -1475,6 +1474,8 @@ static int setup_shmem_window(struct pcmcia_device *link, int start_pg,
     info->flags |= USE_SHMEM;
     return 0;
 
+release:
+	pcmcia_release_window(link, link->resource[3]);
 failed:
     return 1;
 }
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index 8581ccba1505e3..c2b0161d0beca6 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -130,6 +130,7 @@ config FEALNX
 
 source "drivers/net/ethernet/ni/Kconfig"
 source "drivers/net/ethernet/natsemi/Kconfig"
+source "drivers/net/ethernet/nebula-matrix/Kconfig"
 source "drivers/net/ethernet/netronome/Kconfig"
 source "drivers/net/ethernet/8390/Kconfig"
 source "drivers/net/ethernet/nvidia/Kconfig"
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index 2b1153d35b52b3..a306dae23bcb81 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -67,6 +67,7 @@ obj-$(CONFIG_NET_VENDOR_MUCSE) += mucse/
 obj-$(CONFIG_NET_VENDOR_MYRI) += myricom/
 obj-$(CONFIG_FEALNX) += fealnx.o
 obj-$(CONFIG_NET_VENDOR_NATSEMI) += natsemi/
+obj-$(CONFIG_NET_VENDOR_NEBULA_MATRIX) += nebula-matrix/
 obj-$(CONFIG_NET_VENDOR_NETRONOME) += netronome/
 obj-$(CONFIG_NET_VENDOR_NI) += ni/
 obj-$(CONFIG_NET_VENDOR_NVIDIA) += nvidia/
diff --git a/drivers/net/ethernet/adaptec/starfire.c b/drivers/net/ethernet/adaptec/starfire.c
index f1109d90e1fc19..b2f3998e128b48 100644
--- a/drivers/net/ethernet/adaptec/starfire.c
+++ b/drivers/net/ethernet/adaptec/starfire.c
@@ -634,7 +634,7 @@ static int starfire_init_one(struct pci_dev *pdev,
 	int i, irq, chip_idx = ent->driver_data;
 	struct net_device *dev;
 	u8 addr[ETH_ALEN];
-	long ioaddr;
+	resource_size_t ioaddr;
 	void __iomem *base;
 	int drv_flags, io_size;
 	int boguscnt;
@@ -664,8 +664,8 @@ static int starfire_init_one(struct pci_dev *pdev,
 
 	base = ioremap(ioaddr, io_size);
 	if (!base) {
-		dev_err(d, "cannot remap %#x @ %#lx, aborting\n",
-			io_size, ioaddr);
+		dev_err(d, "cannot remap %#x @ %pa, aborting\n",
+			io_size, &ioaddr);
 		goto err_out_free_res;
 	}
 
diff --git a/drivers/net/ethernet/adi/adin1140.c b/drivers/net/ethernet/adi/adin1140.c
index 93710baca1517a..adc6e2ef44f8c3 100644
--- a/drivers/net/ethernet/adi/adin1140.c
+++ b/drivers/net/ethernet/adi/adin1140.c
@@ -725,8 +725,8 @@ static int adin1140_probe(struct spi_device *spi)
 	tc6_quirks.quirk_flags = OA_TC6_BROKEN_PHY;
 
 	priv->tc6 = oa_tc6_init(spi, netdev, &tc6_quirks);
-	if (!priv->tc6)
-		return -ENODEV;
+	if (IS_ERR(priv->tc6))
+		return PTR_ERR(priv->tc6);
 
 	ret = devm_add_action_or_reset(&spi->dev, adin1140_oa_tc6_remove,
 				       priv->tc6);
diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
index 64619e9a704dd4..21ac80bdd084ac 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.c
+++ b/drivers/net/ethernet/airoha/airoha_eth.c
@@ -10,8 +10,10 @@
 #include <linux/tcp.h>
 #include <linux/u64_stats_sync.h>
 #include <net/dst_metadata.h>
+#include <net/ip6_checksum.h>
 #include <net/page_pool/helpers.h>
 #include <net/pkt_cls.h>
+#include <net/tcp.h>
 #include <uapi/linux/ppp_defs.h>
 
 #include "airoha_regs.h"
@@ -491,6 +493,88 @@ static void airoha_fe_crsn_qsel_init(struct airoha_eth *eth)
 				 CDM_CRSN_QSEL_Q1));
 }
 
+static void airoha_fe_lro_rxq_enable(struct airoha_eth *eth, int qdma_id,
+				     int lro_queue_index, int qid,
+				     int buf_size)
+{
+	int id = qdma_id + 1;
+
+	airoha_fe_rmw(eth, REG_CDM_LRO_LIMIT(id),
+		      CDM_LRO_AGG_NUM_MASK | CDM_LRO_AGG_SIZE_MASK,
+		      FIELD_PREP(CDM_LRO_AGG_SIZE_MASK, buf_size) |
+		      FIELD_PREP(CDM_LRO_AGG_NUM_MASK,
+				 AIROHA_RXQ_LRO_MAX_AGG_COUNT));
+	airoha_fe_rmw(eth, REG_CDM_LRO_AGE_TIME(id),
+		      CDM_LRO_AGE_TIME_MASK | CDM_LRO_AGG_TIME_MASK,
+		      FIELD_PREP(CDM_LRO_AGE_TIME_MASK,
+				 AIROHA_RXQ_LRO_MAX_AGE_TIME) |
+		      FIELD_PREP(CDM_LRO_AGG_TIME_MASK,
+				 AIROHA_RXQ_LRO_MAX_AGG_TIME));
+	airoha_fe_rmw(eth, REG_CDM_LRO_RXQ(id, lro_queue_index),
+		      LRO_RXQ_MASK(lro_queue_index),
+		      __field_prep(LRO_RXQ_MASK(lro_queue_index), qid));
+	airoha_fe_set(eth, REG_CDM_LRO_EN(id), BIT(lro_queue_index));
+}
+
+static void airoha_fe_lro_disable(struct airoha_eth *eth, int qdma_id)
+{
+	int i, id = qdma_id + 1;
+
+	airoha_fe_clear(eth, REG_CDM_LRO_EN(id), LRO_RXQ_EN_MASK);
+	airoha_fe_clear(eth, REG_CDM_LRO_LIMIT(id),
+			CDM_LRO_AGG_NUM_MASK | CDM_LRO_AGG_SIZE_MASK);
+	airoha_fe_clear(eth, REG_CDM_LRO_AGE_TIME(id),
+			CDM_LRO_AGE_TIME_MASK | CDM_LRO_AGG_TIME_MASK);
+	for (i = 0; i < AIROHA_MAX_NUM_LRO_QUEUES; i++)
+		airoha_fe_clear(eth, REG_CDM_LRO_RXQ(id, i), LRO_RXQ_MASK(i));
+}
+
+static bool airoha_fe_lro_is_enabled(struct airoha_eth *eth, int qdma_id)
+{
+	return airoha_fe_get(eth, REG_CDM_LRO_EN(qdma_id + 1),
+			     LRO_RXQ_EN_MASK);
+}
+
+static void airoha_dev_lro_enable(struct airoha_gdm_dev *dev)
+{
+	struct airoha_qdma *qdma = airoha_qdma_deref(dev);
+	struct airoha_eth *eth = qdma->eth;
+	int qdma_id = qdma - &eth->qdma[0];
+	int i, lro_queue_index = 0;
+
+	if (airoha_fe_lro_is_enabled(eth, qdma_id))
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) {
+		struct airoha_queue *q = &qdma->q_rx[i];
+		u32 size;
+
+		if (!q->ndesc)
+			continue;
+
+		if (!airoha_qdma_is_lro_queue(q))
+			continue;
+
+		size = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size));
+		size = min_t(u32, size, AIROHA_MAX_RX_SIZE);
+		airoha_fe_lro_rxq_enable(eth, qdma_id, lro_queue_index, i,
+					 size);
+		lro_queue_index++;
+	}
+}
+
+static void airoha_dev_lro_disable(struct airoha_gdm_dev *dev)
+{
+	struct airoha_qdma *qdma = airoha_qdma_deref(dev);
+	struct airoha_eth *eth = qdma->eth;
+	int qdma_id = qdma - &eth->qdma[0];
+
+	if (!airoha_fe_lro_is_enabled(eth, qdma_id))
+		return;
+
+	airoha_fe_lro_disable(eth, qdma_id);
+}
+
 static int airoha_fe_init(struct airoha_eth *eth)
 {
 	airoha_fe_maccr_init(eth);
@@ -616,6 +700,7 @@ static int airoha_qdma_fill_rx_queue(struct airoha_queue *q)
 		e->dma_addr = page_pool_get_dma_addr(page) + offset;
 		e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size));
 
+		WRITE_ONCE(desc->tcp_ts_reply, 0);
 		val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len);
 		WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
 		WRITE_ONCE(desc->addr, cpu_to_le32(e->dma_addr));
@@ -657,13 +742,146 @@ airoha_qdma_get_gdm_dev(struct airoha_eth *eth, struct airoha_qdma_desc *desc)
 	return port->devs[d] ? port->devs[d] : ERR_PTR(-ENODEV);
 }
 
+static int airoha_qdma_lro_rx_skb(struct airoha_queue *q,
+				  struct airoha_qdma_desc *desc,
+				  u32 msg1, u32 len)
+{
+	u32 th_off, tcp_ack_seq, data_off, agg_count = 1;
+	u32 msg2 = le32_to_cpu(READ_ONCE(desc->msg2));
+	struct skb_shared_info *shinfo;
+	struct sk_buff *skb = q->skb;
+	u16 tcp_win, l2_len;
+	struct tcphdr *th;
+	bool ipv4, ipv6;
+
+	if (airoha_qdma_is_lro_queue(q)) {
+		agg_count = FIELD_GET(QDMA_ETH_RXMSG_AGG_COUNT_MASK, msg2);
+		if (agg_count > AIROHA_RXQ_LRO_MAX_AGG_COUNT)
+			return -EINVAL;
+	}
+
+	if (likely(agg_count <= 1)) /* not LRO */
+		return 0;
+
+	ipv4 = FIELD_GET(QDMA_ETH_RXMSG_IP4_MASK, msg1);
+	ipv6 = FIELD_GET(QDMA_ETH_RXMSG_IP6_MASK, msg1);
+	if (!ipv4 && !ipv6)
+		return 0;
+
+	l2_len = FIELD_GET(QDMA_ETH_RXMSG_L2_LEN_MASK, msg2);
+
+	if (ipv4) {
+		struct iphdr *iph, _iph;
+
+		iph = skb_header_pointer(skb, l2_len, sizeof(*iph), &_iph);
+		if (!iph)
+			return -EINVAL;
+
+		if (iph->protocol != IPPROTO_TCP)
+			return -EINVAL;
+
+		if (iph->ihl < 5)
+			return -EINVAL;
+
+		th_off = l2_len + (iph->ihl << 2);
+		if (!pskb_may_pull(skb, th_off))
+			return -EINVAL;
+
+		iph = (struct iphdr *)(skb->data + l2_len);
+		iph->tot_len = cpu_to_be16(len - l2_len);
+		iph->check = 0;
+		iph->check = ip_fast_csum((void *)iph, iph->ihl);
+	} else {
+		struct ipv6hdr *ip6h;
+
+		th_off = l2_len + sizeof(*ip6h);
+		if (!pskb_may_pull(skb, th_off))
+			return -EINVAL;
+
+		ip6h = (struct ipv6hdr *)(skb->data + l2_len);
+		if (ip6h->nexthdr != NEXTHDR_TCP)
+			return -EINVAL;
+
+		ip6h->payload_len = cpu_to_be16(len - th_off);
+	}
+
+	data_off = th_off + sizeof(*th);
+	if (len < data_off)
+		return -EINVAL;
+
+	if (!pskb_may_pull(skb, data_off))
+		return -EINVAL;
+
+	th = (struct tcphdr *)(skb->data + th_off);
+	if (th->doff < 5)
+		return -EINVAL;
+
+	data_off = th_off + (th->doff << 2);
+	if (len <= data_off)
+		return -EINVAL;
+
+	tcp_win = FIELD_GET(QDMA_ETH_RXMSG_TCP_WIN_MASK,
+			    le32_to_cpu(READ_ONCE(desc->msg3)));
+	tcp_ack_seq = le32_to_cpu(READ_ONCE(desc->data));
+	th->ack_seq = cpu_to_be32(tcp_ack_seq);
+	th->window = cpu_to_be16(tcp_win);
+
+	/* Check tcp timestamp option */
+	if (th->doff == (sizeof(*th) + TCPOLEN_TSTAMP_ALIGNED) / 4) {
+		u32 topt;
+
+		if (!pskb_may_pull(skb, data_off))
+			return -EINVAL;
+
+		th = (struct tcphdr *)(skb->data + th_off);
+		topt = get_unaligned_be32(th + 1);
+		if (topt == ((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
+			     (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)) {
+			u8 *ptr = (u8 *)th + sizeof(*th) + 2 * sizeof(__be32);
+			__le32 tcp_ts_reply = READ_ONCE(desc->tcp_ts_reply);
+
+			/* The field is pre-zeroed in the posted descriptor,
+			 * so a non-zero value is the hardware confirming the
+			 * timestamp echo reply is valid. Leave TSecr
+			 * untouched otherwise, since the hardware may not
+			 * populate it for every aggregate.
+			 */
+			if (tcp_ts_reply)
+				put_unaligned_be32(le32_to_cpu(tcp_ts_reply),
+						   ptr);
+		}
+	}
+
+	if (ipv4) {
+		struct iphdr *iph = (struct iphdr *)(skb->data + l2_len);
+
+		th->check = ~tcp_v4_check(len - th_off, iph->saddr,
+					  iph->daddr, 0);
+	} else {
+		struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + l2_len);
+
+		th->check = ~tcp_v6_check(len - th_off, &ip6h->saddr,
+					  &ip6h->daddr, 0);
+	}
+
+	shinfo = skb_shinfo(skb);
+	shinfo->gso_type = ipv4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
+	shinfo->gso_type |= SKB_GSO_DODGY;
+	shinfo->gso_size = DIV_ROUND_UP(len - data_off, agg_count);
+	shinfo->gso_segs = agg_count;
+
+	skb->csum_start = skb_headroom(skb) + th_off;
+	skb->csum_offset = offsetof(struct tcphdr, check);
+	skb->ip_summed = CHECKSUM_PARTIAL;
+
+	return 0;
+}
+
 static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 {
 	enum dma_data_direction dir = page_pool_get_dma_dir(q->page_pool);
-	struct airoha_qdma *qdma = q->qdma;
-	struct airoha_eth *eth = qdma->eth;
-	int qid = q - &qdma->q_rx[0];
-	int done = 0;
+	int qid = q - &q->qdma->q_rx[0], done = 0;
+	struct airoha_eth *eth = q->qdma->eth;
 
 	while (done < budget) {
 		struct airoha_queue_entry *e = &q->entry[q->tail];
@@ -696,7 +914,9 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 		if (IS_ERR(dev))
 			goto free_frag;
 
+		msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
 		netdev = netdev_from_priv(dev);
+
 		if (!q->skb) { /* first buffer */
 			q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM,
 						q->buf_size);
@@ -707,9 +927,17 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 			__skb_put(q->skb, len);
 			skb_mark_for_recycle(q->skb);
 			q->skb->dev = netdev;
-			q->skb->protocol = eth_type_trans(q->skb, netdev);
 			q->skb->ip_summed = CHECKSUM_UNNECESSARY;
 			skb_record_rx_queue(q->skb, qid);
+
+			if (airoha_qdma_lro_rx_skb(q, desc, msg1, len)) {
+				dev_core_stats_rx_dropped_inc(netdev);
+				dev_kfree_skb(q->skb);
+				q->skb = NULL;
+				continue;
+			}
+
+			q->skb->protocol = eth_type_trans(q->skb, netdev);
 		} else { /* scattered frame */
 			struct skb_shared_info *shinfo = skb_shinfo(q->skb);
 			int nr_frags = shinfo->nr_frags;
@@ -742,7 +970,6 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 						  &port->dsa_meta[sptag]->dst);
 		}
 
-		msg1 = le32_to_cpu(READ_ONCE(desc->msg1));
 		hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1);
 		if (hash != AIROHA_RXD4_FOE_ENTRY)
 			skb_set_hash(q->skb, jhash_1word(hash, 0),
@@ -800,12 +1027,10 @@ static int airoha_qdma_rx_napi_poll(struct napi_struct *napi, int budget)
 static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 				     struct airoha_qdma *qdma, int ndesc)
 {
-	const struct page_pool_params pp_params = {
-		.order = 0,
+	struct page_pool_params pp_params = {
 		.pool_size = 256,
 		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
 		.dma_dir = DMA_FROM_DEVICE,
-		.max_len = PAGE_SIZE,
 		.nid = NUMA_NO_NODE,
 		.dev = qdma->eth->dev,
 		.napi = &q->napi,
@@ -813,9 +1038,10 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 	struct airoha_eth *eth = qdma->eth;
 	int qid = q - &qdma->q_rx[0], thr;
 	dma_addr_t dma_addr;
+	bool lro_q;
 
-	q->buf_size = PAGE_SIZE / 2;
 	q->qdma = qdma;
+	lro_q = airoha_qdma_is_lro_queue(q);
 
 	q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry),
 				GFP_KERNEL);
@@ -827,6 +1053,9 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 	if (!q->desc)
 		return -ENOMEM;
 
+	pp_params.order = lro_q ? AIROHA_LRO_PAGE_ORDER : 0;
+	pp_params.max_len = PAGE_SIZE << pp_params.order;
+
 	q->page_pool = page_pool_create(&pp_params);
 	if (IS_ERR(q->page_pool)) {
 		int err = PTR_ERR(q->page_pool);
@@ -835,6 +1064,7 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 		return err;
 	}
 
+	q->buf_size = lro_q ? pp_params.max_len : pp_params.max_len / 2;
 	q->ndesc = ndesc;
 	netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll);
 
@@ -848,7 +1078,12 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q,
 			FIELD_PREP(RX_RING_THR_MASK, thr));
 	airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK,
 			FIELD_PREP(RX_RING_DMA_IDX_MASK, q->head));
-	airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid), RX_RING_SG_EN_MASK);
+	if (lro_q)
+		airoha_qdma_clear(qdma, REG_RX_SCATTER_CFG(qid),
+				  RX_RING_SG_EN_MASK);
+	else
+		airoha_qdma_set(qdma, REG_RX_SCATTER_CFG(qid),
+				RX_RING_SG_EN_MASK);
 
 	airoha_qdma_fill_rx_queue(q);
 
@@ -870,6 +1105,7 @@ static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q)
 					page_pool_get_dma_dir(q->page_pool));
 		page_pool_put_full_page(q->page_pool, page, false);
 		/* Reset DMA descriptor */
+		WRITE_ONCE(desc->tcp_ts_reply, 0);
 		WRITE_ONCE(desc->ctrl, 0);
 		WRITE_ONCE(desc->addr, 0);
 		WRITE_ONCE(desc->data, 0);
@@ -1896,6 +2132,34 @@ static void airoha_update_hw_stats(struct airoha_gdm_dev *dev)
 	spin_unlock(&port->stats_lock);
 }
 
+static void airoha_update_netdev_features(struct airoha_gdm_dev *dev)
+{
+	struct airoha_eth *eth = dev->eth;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(eth->ports); i++) {
+		struct airoha_gdm_port *port = eth->ports[i];
+		int j;
+
+		if (!port)
+			continue;
+
+		for (j = 0; j < ARRAY_SIZE(port->devs); j++) {
+			struct airoha_gdm_dev *iter_dev = port->devs[j];
+			struct net_device *netdev;
+
+			if (!iter_dev)
+				continue;
+
+			netdev = netdev_from_priv(iter_dev);
+			if (netdev->reg_state != NETREG_REGISTERED)
+				continue;
+
+			netdev_update_features(netdev);
+		}
+	}
+}
+
 static void airoha_dev_set_xmit_frame_size(struct net_device *netdev)
 {
 	struct airoha_gdm_dev *dev = netdev_priv(netdev);
@@ -1913,14 +2177,28 @@ static int airoha_dev_open(struct net_device *netdev)
 	struct airoha_gdm_port *port = dev->port;
 	u32 pse_port = FE_PSE_PORT_PPE1;
 	struct airoha_qdma *qdma;
-	int err;
+	int qdma_id, err;
+
+	/* LRO is configured on the QDMA and it is shared between all
+	 * the devices using it. Refuse to open a second device on the
+	 * same QDMA if LRO is enabled on any device sharing it or if
+	 * this device has LRO enabled in its feature set.
+	 */
+	qdma = airoha_qdma_deref(dev);
+	qdma_id = qdma - &qdma->eth->qdma[0];
+
+	if (qdma->users > 1 && (airoha_fe_lro_is_enabled(qdma->eth, qdma_id) ||
+				(netdev->features & NETIF_F_LRO))) {
+		netdev_warn(netdev, "required to disable LRO on QDMA%d\n",
+			    qdma_id);
+		return -EBUSY;
+	}
 
 	netif_tx_start_all_queues(netdev);
 	err = airoha_set_vip_for_gdm_port(dev, true);
 	if (err)
 		return err;
 
-	qdma = airoha_qdma_deref(dev);
 	if (netdev_uses_dsa(netdev))
 		airoha_fe_set(qdma->eth, REG_GDM_INGRESS_CFG(port->id),
 			      GDM_STAG_EN_MASK);
@@ -1937,6 +2215,9 @@ static int airoha_dev_open(struct net_device *netdev)
 	airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id),
 				    pse_port);
 
+	if (netdev->features & NETIF_F_LRO)
+		airoha_dev_lro_enable(dev);
+
 	return 0;
 }
 
@@ -1954,6 +2235,10 @@ static int airoha_dev_stop(struct net_device *netdev)
 		airoha_set_gdm_port_fwd_cfg(dev->eth,
 					    REG_GDM_FWD_CFG(port->id),
 					    FE_PSE_PORT_DROP);
+
+	if (netdev->features & NETIF_F_LRO)
+		airoha_dev_lro_disable(dev);
+
 	return 0;
 }
 
@@ -2115,11 +2400,14 @@ static void airoha_dev_set_qdma(struct airoha_gdm_dev *dev)
 	qdma = &eth->qdma[!airoha_is_lan_gdm_dev(dev)];
 	cur_qdma = airoha_qdma_deref(dev);
 
-	if (cur_qdma)
+	if (cur_qdma) {
 		netif_tx_stop_all_queues(netdev);
+		airoha_dev_lro_disable(dev);
+	}
 
 	rcu_assign_pointer(dev->qdma, qdma);
 	netdev->irq = qdma->irq_banks[0].irq;
+	qdma->users++;
 	synchronize_rcu();
 
 	ppe_id = !airoha_is_lan_gdm_dev(dev) && airoha_ppe_is_enabled(eth, 1);
@@ -2136,8 +2424,14 @@ static void airoha_dev_set_qdma(struct airoha_gdm_dev *dev)
 			airoha_qdma_rr(qdma, REG_CNTR_VAL((i << 1) + 1));
 	}
 
-	if (cur_qdma)
+	if (cur_qdma) {
+		cur_qdma->users--;
 		netif_tx_wake_all_queues(netdev);
+	}
+
+	airoha_update_netdev_features(dev);
+	if (netdev->features & NETIF_F_LRO)
+		airoha_dev_lro_enable(dev);
 }
 
 static int airoha_dev_init(struct net_device *netdev)
@@ -2288,6 +2582,32 @@ int airoha_get_fe_port(struct airoha_gdm_dev *dev)
 	}
 }
 
+static netdev_features_t airoha_dev_fix_features(struct net_device *netdev,
+						 netdev_features_t features)
+{
+	struct airoha_gdm_dev *dev = netdev_priv(netdev);
+	struct airoha_qdma *qdma;
+
+	qdma = airoha_qdma_deref(dev);
+	if (qdma->users > 1)
+		features &= ~NETIF_F_LRO;
+
+	return features;
+}
+
+static int airoha_dev_set_features(struct net_device *netdev,
+				   netdev_features_t features)
+{
+	struct airoha_gdm_dev *dev = netdev_priv(netdev);
+
+	if (features & NETIF_F_LRO)
+		airoha_dev_lro_enable(dev);
+	else
+		airoha_dev_lro_disable(dev);
+
+	return 0;
+}
+
 static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb,
 				   struct net_device *netdev)
 {
@@ -3199,7 +3519,7 @@ static int airoha_enable_qos_for_gdm34(struct net_device *netdev,
 	struct airoha_gdm_dev *wan_dev, *dev = netdev_priv(netdev);
 	struct airoha_gdm_port *port = dev->port;
 	struct airoha_eth *eth = dev->eth;
-	int err = -EBUSY;
+	int err;
 
 	if (port->id != AIROHA_GDM3_IDX &&
 	    port->id != AIROHA_GDM4_IDX) {
@@ -3218,6 +3538,7 @@ static int airoha_enable_qos_for_gdm34(struct net_device *netdev,
 		    wan_dev->port->id == AIROHA_GDM2_IDX) {
 			NL_SET_ERR_MSG_MOD(extack,
 					   "QoS configured for WAN device");
+			err = -EBUSY;
 			goto error_unlock;
 		}
 		airoha_disable_qos_for_gdm34(netdev_from_priv(wan_dev));
@@ -3343,6 +3664,8 @@ static const struct net_device_ops airoha_netdev_ops = {
 	.ndo_stop		= airoha_dev_stop,
 	.ndo_change_mtu		= airoha_dev_change_mtu,
 	.ndo_select_queue	= airoha_dev_select_queue,
+	.ndo_fix_features	= airoha_dev_fix_features,
+	.ndo_set_features	= airoha_dev_set_features,
 	.ndo_start_xmit		= airoha_dev_xmit,
 	.ndo_get_stats64        = airoha_dev_get_stats64,
 	.ndo_set_mac_address	= airoha_dev_set_macaddr,
@@ -3430,11 +3753,9 @@ static int airoha_alloc_gdm_device(struct airoha_eth *eth,
 	netdev->ethtool_ops = &airoha_ethtool_ops;
 	netdev->max_mtu = AIROHA_MAX_MTU;
 	netdev->watchdog_timeo = 5 * HZ;
-	netdev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_TSO6 |
-			      NETIF_F_IPV6_CSUM | NETIF_F_SG | NETIF_F_TSO |
-			      NETIF_F_HW_TC;
-	netdev->features |= netdev->hw_features;
-	netdev->vlan_features = netdev->hw_features;
+	netdev->hw_features = AIROHA_HW_FEATURES | NETIF_F_LRO;
+	netdev->features |= AIROHA_HW_FEATURES;
+	netdev->vlan_features = AIROHA_HW_FEATURES;
 	SET_NETDEV_DEV(netdev, eth->dev);
 
 	/* reserve hw queues for HTB offloading */
diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
index 8277c1c87bb328..286f73e2e1e09d 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.h
+++ b/drivers/net/ethernet/airoha/airoha_eth.h
@@ -41,9 +41,22 @@
 #define TX_DSCP_NUM			1024
 #define RX_DSCP_NUM(_n)			\
 	((_n) ==  2 ? 128 :		\
+	 (_n) ==  4 ? 128 :		\
 	 (_n) == 11 ? 128 :		\
 	 (_n) == 15 ? 128 :		\
-	 (_n) ==  0 ? 1024 : 16)
+	 (_n) ==  0 ? 1024 : 32)
+
+#define AIROHA_LRO_PAGE_ORDER		get_order(SZ_16K)
+#define AIROHA_MAX_NUM_LRO_QUEUES	8
+#define AIROHA_RXQ_LRO_EN_MASK		GENMASK(31, 24)
+#define AIROHA_RXQ_LRO_MAX_AGG_COUNT	64
+#define AIROHA_RXQ_LRO_MAX_AGG_TIME	100
+#define AIROHA_RXQ_LRO_MAX_AGE_TIME	2000
+
+#define AIROHA_HW_FEATURES			\
+	(NETIF_F_IP_CSUM | NETIF_F_RXCSUM |	\
+	 NETIF_F_TSO6 | NETIF_F_IPV6_CSUM |	\
+	 NETIF_F_SG | NETIF_F_TSO | NETIF_F_HW_TC)
 
 #define PSE_RSV_PAGES			128
 #define PSE_QUEUE_RSV_PAGES		64
@@ -560,6 +573,8 @@ struct airoha_qdma {
 	struct airoha_eth *eth;
 	void __iomem *regs;
 
+	int users;
+
 	struct airoha_irq_bank irq_banks[AIROHA_MAX_NUM_IRQ_BANKS];
 
 	struct airoha_tx_irq_queue q_tx_irq[AIROHA_NUM_TX_IRQ];
@@ -714,6 +729,18 @@ static inline bool airoha_is_7583(struct airoha_eth *eth)
 	return eth->soc->version == 0x7583;
 }
 
+static inline bool airoha_qdma_is_lro_queue(struct airoha_queue *q)
+{
+	struct airoha_qdma *qdma = q->qdma;
+	int qid = q - &qdma->q_rx[0];
+
+	/* EN7581 SoC supports at most 8 LRO rx queues */
+	BUILD_BUG_ON(hweight32(AIROHA_RXQ_LRO_EN_MASK) >
+		     AIROHA_MAX_NUM_LRO_QUEUES);
+
+	return !!(AIROHA_RXQ_LRO_EN_MASK & BIT(qid));
+}
+
 int airoha_get_fe_port(struct airoha_gdm_dev *dev);
 bool airoha_is_valid_gdm_dev(struct airoha_eth *eth,
 			     struct airoha_gdm_dev *dev);
diff --git a/drivers/net/ethernet/airoha/airoha_npu.c b/drivers/net/ethernet/airoha/airoha_npu.c
index 5bb4817a898d1c..4d3195eb00f7ae 100644
--- a/drivers/net/ethernet/airoha/airoha_npu.c
+++ b/drivers/net/ethernet/airoha/airoha_npu.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/devcoredump.h>
+#include <linux/devm-helpers.h>
 #include <linux/firmware.h>
 #include <linux/platform_device.h>
 #include <linux/of_net.h>
@@ -751,12 +752,15 @@ static int airoha_npu_probe(struct platform_device *pdev)
 		if (irq < 0)
 			return irq;
 
+		err = devm_work_autocancel(dev, &core->wdt_work,
+					   airoha_npu_wdt_work);
+		if (err)
+			return err;
+
 		err = devm_request_irq(dev, irq, airoha_npu_wdt_handler,
 				       IRQF_SHARED, "airoha-npu-wdt", core);
 		if (err)
 			return err;
-
-		INIT_WORK(&core->wdt_work, airoha_npu_wdt_work);
 	}
 
 	/* wlan IRQ lines */
@@ -803,18 +807,8 @@ static int airoha_npu_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static void airoha_npu_remove(struct platform_device *pdev)
-{
-	struct airoha_npu *npu = platform_get_drvdata(pdev);
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(npu->cores); i++)
-		cancel_work_sync(&npu->cores[i].wdt_work);
-}
-
 static struct platform_driver airoha_npu_driver = {
 	.probe = airoha_npu_probe,
-	.remove = airoha_npu_remove,
 	.driver = {
 		.name = "airoha-npu",
 		.of_match_table = of_airoha_npu_match,
diff --git a/drivers/net/ethernet/airoha/airoha_regs.h b/drivers/net/ethernet/airoha/airoha_regs.h
index 442b48c9b991ef..07dd04859fde88 100644
--- a/drivers/net/ethernet/airoha/airoha_regs.h
+++ b/drivers/net/ethernet/airoha/airoha_regs.h
@@ -122,6 +122,20 @@
 #define CDM_CRSN_QSEL_REASON_MASK(_n)	\
 	GENMASK(4 + (((_n) % 4) << 3),	(((_n) % 4) << 3))
 
+#define REG_CDM_LRO_RXQ(_n, _m)		(CDM_BASE(_n) + 0x78 + ((_m) & 0x4))
+#define LRO_RXQ_MASK(_n)		GENMASK(4 + (((_n) & 0x3) << 3), ((_n) & 0x3) << 3)
+
+#define REG_CDM_LRO_EN(_n)		(CDM_BASE(_n) + 0x80)
+#define LRO_RXQ_EN_MASK			GENMASK(7, 0)
+
+#define REG_CDM_LRO_LIMIT(_n)		(CDM_BASE(_n) + 0x84)
+#define CDM_LRO_AGG_NUM_MASK		GENMASK(23, 16)
+#define CDM_LRO_AGG_SIZE_MASK		GENMASK(15, 0)
+
+#define REG_CDM_LRO_AGE_TIME(_n)	(CDM_BASE(_n) + 0x88)
+#define CDM_LRO_AGE_TIME_MASK		GENMASK(31, 16)
+#define CDM_LRO_AGG_TIME_MASK		GENMASK(15, 0)
+
 #define REG_GDM_FWD_CFG(_n)		GDM_BASE(_n)
 #define GDM_PAD_EN_MASK			BIT(28)
 #define GDM_DROP_CRC_ERR_MASK		BIT(23)
@@ -887,9 +901,16 @@
 #define QDMA_ETH_RXMSG_SPORT_MASK	GENMASK(25, 21)
 #define QDMA_ETH_RXMSG_CRSN_MASK	GENMASK(20, 16)
 #define QDMA_ETH_RXMSG_PPE_ENTRY_MASK	GENMASK(15, 0)
+/* RX MSG2 */
+#define QDMA_ETH_RXMSG_AGG_COUNT_MASK	GENMASK(31, 24)
+#define QDMA_ETH_RXMSG_L2_LEN_MASK	GENMASK(6, 0)
+/* RX MSG3 */
+/* max aggregated size in bytes */
+#define QDMA_ETH_RXMSG_AGG_LEN_MASK	GENMASK(31, 16)
+#define QDMA_ETH_RXMSG_TCP_WIN_MASK	GENMASK(15, 0)
 
 struct airoha_qdma_desc {
-	__le32 rsv;
+	__le32 tcp_ts_reply;
 	__le32 ctrl;
 	__le32 addr;
 	__le32 data;
diff --git a/drivers/net/ethernet/alacritech/slicoss.c b/drivers/net/ethernet/alacritech/slicoss.c
index c1949ca060ca80..14b74c09233e15 100644
--- a/drivers/net/ethernet/alacritech/slicoss.c
+++ b/drivers/net/ethernet/alacritech/slicoss.c
@@ -34,6 +34,7 @@ static const struct pci_device_id slic_id_tbl[] = {
 		     PCI_DEVICE_ID_ALACRITECH_OASIS) },
 	{ 0 }
 };
+MODULE_DEVICE_TABLE(pci, slic_id_tbl);
 
 static const char slic_stats_strings[][ETH_GSTRING_LEN] = {
 	"rx_packets",
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index ea89619039d89b..7eb6456ed0d556 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -4122,6 +4122,8 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 err_device_destroy:
 	ena_com_delete_host_info(ena_dev);
 	ena_com_admin_destroy(ena_dev);
+	ena_phc_destroy(adapter);
+	ena_com_mmio_reg_read_request_destroy(ena_dev);
 ena_devlink_destroy:
 	ena_devlink_free(devlink);
 err_metrics_destroy:
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
index 12770af031eb8b..c56587384d5731 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c
@@ -1202,10 +1202,7 @@ static int __xgbe_phy_config_aneg(struct xgbe_prv_data *pdata, bool set_mode)
 	xgbe_an_restart(pdata);
 
 out:
-	if (ret)
-		set_bit(XGBE_LINK_ERR, &pdata->dev_state);
-	else
-		clear_bit(XGBE_LINK_ERR, &pdata->dev_state);
+	assign_bit(XGBE_LINK_ERR, &pdata->dev_state, ret);
 
 	mutex_unlock(&pdata->an_mutex);
 
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
index 01792f7abf9627..42e0fe9248f288 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
@@ -1717,10 +1717,7 @@ int aq_nic_setup_tc_min_rate(struct aq_nic_s *self, const unsigned int tc,
 	if (tc >= AQ_CFG_TCS_MAX)
 		return -EINVAL;
 
-	if (min_rate)
-		set_bit(tc, &cfg->tc_min_rate_msk);
-	else
-		clear_bit(tc, &cfg->tc_min_rate_msk);
+	assign_bit(tc, &cfg->tc_min_rate_msk, min_rate);
 
 	if (min_rate && min_rate < 20) {
 		netdev_warn(self->ndev,
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
index 005d6ef4fc36fb..04b3ef840b369d 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
@@ -125,8 +125,6 @@ static int aq_pci_func_init(struct pci_dev *pdev)
 	int err;
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
-	if (err)
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 	if (err) {
 		err = -ENOSR;
 		goto err_exit;
diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c
index ab262e66f9869b..3ca69fd14a5c17 100644
--- a/drivers/net/ethernet/atheros/alx/main.c
+++ b/drivers/net/ethernet/atheros/alx/main.c
@@ -1727,15 +1727,12 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 * shared register for the high 32 bits, so only a single, aligned,
 	 * 4 GB physical address range can be used for descriptors.
 	 */
-	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
-		dev_dbg(&pdev->dev, "DMA to 64-BIT addresses\n");
-	} else {
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "No usable DMA config, aborting\n");
-			goto out_pci_disable;
-		}
+	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (err) {
+		dev_err(&pdev->dev, "No usable DMA config, aborting\n");
+		goto out_pci_disable;
 	}
+	dev_dbg(&pdev->dev, "DMA to 64-BIT addresses\n");
 
 	err = pci_request_mem_regions(pdev, alx_drv_name);
 	if (err) {
diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c.h b/drivers/net/ethernet/atheros/atl1c/atl1c.h
index 63ba64dbb73107..7898842614037f 100644
--- a/drivers/net/ethernet/atheros/atl1c/atl1c.h
+++ b/drivers/net/ethernet/atheros/atl1c/atl1c.h
@@ -29,7 +29,6 @@
 #include <linux/mii.h>
 #include <linux/io.h>
 #include <linux/vmalloc.h>
-#include <linux/pagemap.h>
 #include <linux/tcp.h>
 #include <linux/ethtool.h>
 #include <linux/if_vlan.h>
diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h b/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h
index c567c920628f13..8f03bb561d448c 100644
--- a/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h
+++ b/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h
@@ -659,7 +659,7 @@ void atl1c_post_phy_linkchg(struct atl1c_hw *hw, u16 link_speed);
 /* INT-triggle/SMB Control Register */
 #define REG_SMB_STAT_TIMER		0x15C4	/* 2us resolution */
 #define SMB_STAT_TIMER_MASK		0xFFFFFF
-#define REG_TINT_TPD_THRESH             0x15C8 /* tpd th to trig intrrupt */
+#define REG_TINT_TPD_THRESH             0x15C8 /* tpd th to trig interrupt */
 
 /* Mail box */
 #define MB_RFDX_PROD_IDX_MASK		0xFFFF
diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
index 7efa3fc257b395..e58f1d2c26bdfd 100644
--- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
+++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
@@ -1602,6 +1602,9 @@ static int atl1c_clean_tx(struct napi_struct *napi, int budget)
 	AT_READ_REGW(&adapter->hw, atl1c_qregs[tpd_ring->num].tpd_cons,
 		     &hw_next_to_clean);
 
+	if (unlikely(hw_next_to_clean >= tpd_ring->count))
+		hw_next_to_clean = next_to_clean;
+
 	while (next_to_clean != hw_next_to_clean) {
 		buffer_info = &tpd_ring->buffer_info[next_to_clean];
 		if (buffer_info->skb) {
diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e.h b/drivers/net/ethernet/atheros/atl1e/atl1e.h
index 9fcad783c939a4..ec0ef225e826f4 100644
--- a/drivers/net/ethernet/atheros/atl1e/atl1e.h
+++ b/drivers/net/ethernet/atheros/atl1e/atl1e.h
@@ -30,7 +30,6 @@
 #include <linux/mii.h>
 #include <linux/io.h>
 #include <linux/vmalloc.h>
-#include <linux/pagemap.h>
 #include <linux/tcp.h>
 #include <linux/ethtool.h>
 #include <linux/if_vlan.h>
@@ -381,7 +380,7 @@ struct atl1e_rx_page {
 	u8		*addr;   /* receive rage virtual address */
 	dma_addr_t	write_offset_dma;  /* the DMA address which contain the
 					      receive data offset in the page */
-	u32		*write_offset_addr; /* the virtaul address which contain
+	u32		*write_offset_addr; /* the virtual address which contain
 					     the receive data offset in the page */
 	u32		read_offset;       /* the offset where we have read */
 };
diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_hw.c b/drivers/net/ethernet/atheros/atl1e/atl1e_hw.c
index fa89bc27f62df4..25d3e74690583e 100644
--- a/drivers/net/ethernet/atheros/atl1e/atl1e_hw.c
+++ b/drivers/net/ethernet/atheros/atl1e/atl1e_hw.c
@@ -369,7 +369,7 @@ static int atl1e_phy_setup_autoneg_adv(struct atl1e_hw *hw)
  *
  * hw - Struct containing variables accessed by shared code
  *
- * Sets bit 15 and 12 of the MII control regiser (for F001 bug)
+ * Sets bit 15 and 12 of the MII control register (for F001 bug)
  */
 int atl1e_phy_commit(struct atl1e_hw *hw)
 {
diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
index 40290028580ba3..35b5fa7abecd43 100644
--- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
+++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
@@ -1234,6 +1234,9 @@ static bool atl1e_clean_tx_irq(struct atl1e_adapter *adapter)
 	u16 hw_next_to_clean = AT_READ_REGW(&adapter->hw, REG_TPD_CONS_IDX);
 	u16 next_to_clean = atomic_read(&tx_ring->next_to_clean);
 
+	if (unlikely(hw_next_to_clean >= tx_ring->count))
+		hw_next_to_clean = next_to_clean;
+
 	while (next_to_clean != hw_next_to_clean) {
 		tx_buffer = &tx_ring->tx_buffer[next_to_clean];
 		if (tx_buffer->dma) {
@@ -2117,7 +2120,7 @@ static int atl1e_suspend(struct pci_dev *pdev, pm_message_t state)
 			wol_ctrl_data |= WOL_MAGIC_EN | WOL_MAGIC_PME_EN;
 
 		if (wufc & AT_WUFC_LNKC) {
-		/* if orignal link status is link, just wait for retrive link */
+		/* if original link status is link, just wait for retrieve link */
 			if (mii_bmsr_data & BMSR_LSTATUS) {
 				for (i = 0; i < AT_SUSPEND_LINK_TIMEOUT; i++) {
 					msleep(100);
diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c
index 98a4d089270e41..a981c4bcf8ed2d 100644
--- a/drivers/net/ethernet/atheros/atlx/atl1.c
+++ b/drivers/net/ethernet/atheros/atlx/atl1.c
@@ -628,7 +628,7 @@ static s32 atl1_phy_leave_power_saving(struct atl1_hw *hw)
  * Resets the PHY and make all config validate
  * hw - Struct containing variables accessed by shared code
  *
- * Sets bit 15 and 12 of the MII Control regiser (for F001 bug)
+ * Sets bit 15 and 12 of the MII Control register (for F001 bug)
  */
 static s32 atl1_phy_reset(struct atl1_hw *hw)
 {
@@ -2066,6 +2066,9 @@ static int atl1_intr_tx(struct atl1_adapter *adapter)
 	sw_tpd_next_to_clean = atomic_read(&tpd_ring->next_to_clean);
 	cmb_tpd_next_to_clean = le16_to_cpu(adapter->cmb.cmb->tpd_cons_idx);
 
+	if (unlikely(cmb_tpd_next_to_clean >= tpd_ring->count))
+		cmb_tpd_next_to_clean = sw_tpd_next_to_clean;
+
 	while (cmb_tpd_next_to_clean != sw_tpd_next_to_clean) {
 		buffer_info = &tpd_ring->buffer_info[sw_tpd_next_to_clean];
 		if (buffer_info->dma) {
diff --git a/drivers/net/ethernet/atheros/atlx/atl2.c b/drivers/net/ethernet/atheros/atlx/atl2.c
index 5560d5cc316922..c9aca219ec9cdb 100644
--- a/drivers/net/ethernet/atheros/atlx/atl2.c
+++ b/drivers/net/ethernet/atheros/atlx/atl2.c
@@ -2591,7 +2591,7 @@ static s32 atl2_phy_setup_autoneg_adv(struct atl2_hw *hw)
  *
  * hw - Struct containing variables accessed by shared code
  *
- * Sets bit 15 and 12 of the MII Control regiser (for F001 bug)
+ * Sets bit 15 and 12 of the MII Control register (for F001 bug)
  */
 static s32 atl2_phy_commit(struct atl2_hw *hw)
 {
diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.c b/drivers/net/ethernet/broadcom/asp2/bcmasp.c
index 972474893a6bcb..3393b5dcbc9ae2 100644
--- a/drivers/net/ethernet/broadcom/asp2/bcmasp.c
+++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.c
@@ -1209,7 +1209,7 @@ static const struct of_device_id bcmasp_of_match[] = {
 	{ .compatible = "brcm,asp-v2.1", .data = &v21_plat_data },
 	{ .compatible = "brcm,asp-v2.2", .data = &v22_plat_data },
 	{ .compatible = "brcm,asp-v3.0", .data = &v30_plat_data },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, bcmasp_of_match);
 
@@ -1217,7 +1217,7 @@ static const struct of_device_id bcmasp_mdio_of_match[] = {
 	{ .compatible = "brcm,asp-v2.1-mdio", },
 	{ .compatible = "brcm,asp-v2.2-mdio", },
 	{ .compatible = "brcm,asp-v3.0-mdio", },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, bcmasp_mdio_of_match);
 
diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.h b/drivers/net/ethernet/broadcom/asp2/bcmasp.h
index 8c8ffaeadc79f8..b08a7ddc4c646d 100644
--- a/drivers/net/ethernet/broadcom/asp2/bcmasp.h
+++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.h
@@ -80,11 +80,6 @@
 #define  ASP_RX_FILTER_MDA_PAT_L(sel)		(((sel) * 0x14) + 0x108)
 #define  ASP_RX_FILTER_MDA_MSK_H(sel)		(((sel) * 0x14) + 0x10c)
 #define  ASP_RX_FILTER_MDA_MSK_L(sel)		(((sel) * 0x14) + 0x110)
-#define  ASP_RX_FILTER_MDA_CFG(sel)		(((sel) * 0x14) + 0x100)
-#define  ASP_RX_FILTER_MDA_PAT_H(sel)		(((sel) * 0x14) + 0x104)
-#define  ASP_RX_FILTER_MDA_PAT_L(sel)		(((sel) * 0x14) + 0x108)
-#define  ASP_RX_FILTER_MDA_MSK_H(sel)		(((sel) * 0x14) + 0x10c)
-#define  ASP_RX_FILTER_MDA_MSK_L(sel)		(((sel) * 0x14) + 0x110)
 #define  ASP_RX_FILTER_NET_CFG(sel)		(((sel) * 0xa04) + 0x400)
 #define   ASP_RX_FILTER_NET_CFG_CH(sel)		((sel) << 0)
 #define   ASP_RX_FILTER_NET_CFG_EN		BIT(9)
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
index 7ef1babfeca694..f6595fdc888d16 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
@@ -1564,7 +1564,7 @@ static int bcm_enet_set_pauseparam(struct net_device *dev,
 
 	if (priv->has_phy) {
 		if (ecmd->autoneg && (ecmd->rx_pause != ecmd->tx_pause)) {
-			/* asymetric pause mode not supported,
+			/* asymmetric pause mode not supported,
 			 * actually possible but integrated PHY has RO
 			 * asym_pause bit */
 			return -EINVAL;
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.h b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
index 78f1830fb3cb52..ba8f42cf5fb274 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.h
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
@@ -265,7 +265,7 @@ struct bcm_enet_priv {
 	/* number of available descriptor for tx */
 	int tx_desc_count;
 
-	/* next tx descriptor avaiable */
+	/* next tx descriptor available */
 	int tx_curr_desc;
 
 	/* next dirty tx descriptor to reclaim */
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index 4d06c6ba664188..2b2e8a7e8061a5 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -391,8 +391,10 @@ static void bcm_sysport_update_mib_counters(struct bcm_sysport_priv *priv)
 			if (priv->is_lite)
 				continue;
 
-			if (s->type != BCM_SYSPORT_STAT_MIB_RX)
+			if (s->type == BCM_SYSPORT_STAT_MIB_TX)
 				offset = UMAC_MIB_STAT_OFFSET;
+			else if (s->type == BCM_SYSPORT_STAT_RUNT)
+				offset = 2 * UMAC_MIB_STAT_OFFSET;
 			val = umac_readl(priv, UMAC_MIB_START + j + offset);
 			break;
 		case BCM_SYSPORT_STAT_RXCHK:
@@ -482,10 +484,10 @@ static void bcm_sysport_get_stats(struct net_device *dev,
 		    s->type == BCM_SYSPORT_STAT_NETDEV64) {
 			do {
 				start = u64_stats_fetch_begin(syncp);
-				data[i] = *(u64 *)p;
+				data[j] = *(u64 *)p;
 			} while (u64_stats_fetch_retry(syncp, start));
 		} else
-			data[i] = *(u32 *)p;
+			data[j] = *(u32 *)p;
 		j++;
 	}
 
@@ -1717,6 +1719,9 @@ static void bcm_sysport_fini_rx_ring(struct bcm_sysport_priv *priv)
 	if (!(reg & RDMA_DISABLED))
 		netdev_warn(priv->netdev, "RDMA not stopped!\n");
 
+	if (!priv->rx_cbs)
+		return;
+
 	for (i = 0; i < priv->num_rx_bds; i++) {
 		cb = &priv->rx_cbs[i];
 		if (dma_unmap_addr(cb, dma_addr))
@@ -2083,7 +2088,7 @@ static int bcm_sysport_stop(struct net_device *dev)
 
 	ret = tdma_enable_set(priv, 0);
 	if (ret) {
-		netdev_err(dev, "timeout disabling RDMA\n");
+		netdev_err(dev, "timeout disabling TDMA\n");
 		return ret;
 	}
 
@@ -2092,7 +2097,7 @@ static int bcm_sysport_stop(struct net_device *dev)
 
 	ret = rdma_enable_set(priv, 0);
 	if (ret) {
-		netdev_err(dev, "timeout disabling TDMA\n");
+		netdev_err(dev, "timeout disabling RDMA\n");
 		return ret;
 	}
 
@@ -2453,8 +2458,6 @@ static int bcm_sysport_probe(struct platform_device *pdev)
 		return -EINVAL;
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
-	if (ret)
-		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 	if (ret) {
 		dev_err(&pdev->dev, "unable to set DMA mask: %d\n", ret);
 		return ret;
@@ -2524,17 +2527,19 @@ static int bcm_sysport_probe(struct platform_device *pdev)
 	if (ret)
 		priv->phy_interface = PHY_INTERFACE_MODE_GMII;
 
+	priv->phy_dn = of_parse_phandle(dn, "phy-handle", 0);
+
 	/* In the case of a fixed PHY, the DT node associated
 	 * to the PHY is the Ethernet MAC DT node.
 	 */
-	if (of_phy_is_fixed_link(dn)) {
+	if (!priv->phy_dn && of_phy_is_fixed_link(dn)) {
 		ret = of_phy_register_fixed_link(dn);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to register fixed PHY\n");
 			goto err_free_netdev;
 		}
 
-		priv->phy_dn = dn;
+		priv->phy_dn = of_node_get(dn);
 	}
 
 	/* Initialize netdevice members */
@@ -2619,6 +2624,7 @@ static int bcm_sysport_probe(struct platform_device *pdev)
 err_deregister_fixed_link:
 	if (of_phy_is_fixed_link(dn))
 		of_phy_deregister_fixed_link(dn);
+	of_node_put(priv->phy_dn);
 err_free_netdev:
 	free_netdev(dev);
 	return ret;
@@ -2637,6 +2643,7 @@ static void bcm_sysport_remove(struct platform_device *pdev)
 	unregister_netdev(dev);
 	if (of_phy_is_fixed_link(dn))
 		of_phy_deregister_fixed_link(dn);
+	of_node_put(priv->phy_dn);
 	free_netdev(dev);
 	dev_set_drvdata(&pdev->dev, NULL);
 }
@@ -2645,7 +2652,7 @@ static int bcm_sysport_suspend_to_wol(struct bcm_sysport_priv *priv)
 {
 	struct net_device *ndev = priv->netdev;
 	unsigned int timeout = 1000;
-	unsigned int index, i = 0;
+	unsigned int index;
 	u32 reg;
 
 	reg = umac_readl(priv, UMAC_MPD_CTRL);
@@ -2675,10 +2682,8 @@ static int bcm_sysport_suspend_to_wol(struct bcm_sysport_priv *priv)
 		reg = rxchk_readl(priv, RXCHK_CONTROL);
 		reg &= ~(RXCHK_BRCM_TAG_MATCH_MASK <<
 			 RXCHK_BRCM_TAG_MATCH_SHIFT);
-		for_each_set_bit(index, priv->filters, RXCHK_BRCM_TAG_MAX) {
-			reg |= BIT(RXCHK_BRCM_TAG_MATCH_SHIFT + i);
-			i++;
-		}
+		for_each_set_bit(index, priv->filters, RXCHK_BRCM_TAG_MAX)
+			reg |= BIT(RXCHK_BRCM_TAG_MATCH_SHIFT + index);
 		reg |= RXCHK_EN | RXCHK_BRCM_TAG_EN;
 		rxchk_writel(priv, reg, RXCHK_CONTROL);
 	}
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.h b/drivers/net/ethernet/broadcom/bcmsysport.h
index a34296f989f12a..6f807e96dbd8f0 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.h
+++ b/drivers/net/ethernet/broadcom/bcmsysport.h
@@ -402,7 +402,7 @@ struct bcm_rsb {
 #define TDMA_DESC_RING_BASE(i)		(TDMA_DESC_RING_00_BASE + \
 					((i) * TDMA_DESC_RING_SIZE))
 
-/* Ring indexed register addreses */
+/* Ring indexed register addresses */
 #define TDMA_DESC_RING_HEAD_TAIL_PTR(i)	(TDMA_DESC_RING_BASE(i) + \
 					RING_HEAD_TAIL_PTR)
 #define TDMA_DESC_RING_COUNT(i)		(TDMA_DESC_RING_BASE(i) + \
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
index 9580ab83d387ce..28ea4a33f245cc 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x.h
@@ -280,7 +280,7 @@ enum {
  * (db_msg_sz 1 << 7 * cid + 0x40 DPM offset) it can come up with a valid number
  * We must avoid coming up with cid 8 for iscsi since according to this method
  * the designated UIO cid will come out 0 and it has a special handling for that
- * case which doesn't suit us. Therefore will will cieling to closes cid which
+ * case which doesn't suit us. Therefore will cieling to closes cid which
  * has least signigifcant nibble 8 and if it is 8 we will move forward to 0x18.
  */
 
@@ -666,7 +666,7 @@ struct bnx2x_fastpath {
 				 MAX_TX_DESC_CNT * NEXT_PAGE_TX_DESC_CNT)
 /* max BDs per tx packet w/o next_pages:
  * START_BD		- describes packed
- * START_BD(splitted)	- includes unpaged data segment for GSO
+ * START_BD(split)	- includes unpaged data segment for GSO
  * PARSING_BD		- for TSO and CSUM data
  * PARSING_BD2		- for encapsulation data
  * Frag BDs		- describes pages for frags
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_dcb.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_dcb.c
index 9af81630c8a445..3df8a598852194 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_dcb.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_dcb.c
@@ -1986,7 +1986,7 @@ static void bnx2x_dcbnl_set_pg_tccfg_tx(struct net_device *netdev, int prio,
 		return;
 
 	/**
-	 * bw_pct ignored -	band-width percentage devision between user
+	 * bw_pct ignored -	band-width percentage division between user
 	 *			priorities within the same group is not
 	 *			standard and hence not supported
 	 *
@@ -2038,7 +2038,7 @@ static void bnx2x_dcbnl_get_pg_tccfg_tx(struct net_device *netdev, int prio,
 	DP(BNX2X_MSG_DCB, "prio = %d\n", prio);
 
 	/**
-	 * bw_pct ignored -	band-width percentage devision between user
+	 * bw_pct ignored -	band-width percentage division between user
 	 *			priorities within the same group is not
 	 *			standard and hence not supported
 	 *
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h
index 611efee758340b..5297ab578a6f00 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_hsi.h
@@ -2342,7 +2342,7 @@ struct shmem2_region {
 		u32 oem_i2c_data_addr;
 	};
 
-	/* 9 entires for the C2S PCP map for each inner VLAN PCP + 1 default */
+	/* 9 entries for the C2S PCP map for each inner VLAN PCP + 1 default */
 	/* For PCP values 0-3 use the map lower */
 	/* 0xFF000000 - PCP 0, 0x00FF0000 - PCP 1,
 	 * 0x0000FF00 - PCP 2, 0x000000FF PCP 3
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h
index fc7fce642666ce..b5004aab124dc3 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_init_ops.h
@@ -582,7 +582,7 @@ static void bnx2x_init_pxp_arb(struct bnx2x *bp, int r_order,
 		REG_WR(bp, PXP2_REG_WR_CDU_MPS, val);
 	}
 
-	/* Validate number of tags suppoted by device */
+	/* Validate number of tags supported by device */
 #define PCIE_REG_PCIER_TL_HDR_FC_ST		0x2980
 	val = REG_RD(bp, PCIE_REG_PCIER_TL_HDR_FC_ST);
 	val &= 0xFF;
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.c
index ea310057fe3aff..84d002b31920b9 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.c
@@ -2536,7 +2536,7 @@ static int bnx2x_pbf_update(struct link_params *params, u32 flow_ctrl,
 }
 
 /**
- * bnx2x_get_emac_base - retrive emac base address
+ * bnx2x_get_emac_base - retrieve emac base address
  *
  * @bp:			driver handle
  * @mdc_mdio_access:	access type
@@ -3593,7 +3593,7 @@ static u8 bnx2x_ext_phy_resolve_fc(struct bnx2x_phy *phy,
 	u8 ret = 0;
 	vars->flow_ctrl = BNX2X_FLOW_CTRL_NONE;
 	if (phy->req_flow_ctrl != BNX2X_FLOW_CTRL_AUTO) {
-		/* Update the advertised flow-controled of LD/LP in AN */
+		/* Update the advertised flow-controlled of LD/LP in AN */
 		if (phy->req_line_speed == SPEED_AUTO_NEG)
 			bnx2x_ext_phy_update_adv_fc(phy, params, vars);
 		/* But set the flow-control result as the requested one */
@@ -3743,7 +3743,7 @@ static void bnx2x_warpcore_enable_AN_KR(struct bnx2x_phy *phy,
 		{MDIO_WC_DEVAD, MDIO_WC_REG_CL72_USERB0_CL72_TX_FIR_TAP, 0},
 	};
 	DP(NETIF_MSG_LINK, "Enable Auto Negotiation for KR\n");
-	/* Set to default registers that may be overriden by 10G force */
+	/* Set to default registers that may be overridden by 10G force */
 	for (i = 0; i < ARRAY_SIZE(reg_set); i++)
 		bnx2x_cl45_write(bp, phy, reg_set[i].devad, reg_set[i].reg,
 				 reg_set[i].val);
@@ -5425,7 +5425,7 @@ static void bnx2x_flow_ctrl_resolve(struct bnx2x_phy *phy,
 
 	/* Resolve from gp_status in case of AN complete and not sgmii */
 	if (phy->req_flow_ctrl != BNX2X_FLOW_CTRL_AUTO) {
-		/* Update the advertised flow-controled of LD/LP in AN */
+		/* Update the advertised flow-controlled of LD/LP in AN */
 		if (phy->req_line_speed == SPEED_AUTO_NEG)
 			bnx2x_update_adv_fc(phy, params, vars, gp_status);
 		/* But set the flow-control result as the requested one */
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index 39eb6ab5f8058b..277c7bd9c6e748 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -12669,7 +12669,7 @@ static int bnx2x_set_mc_list(struct bnx2x *bp)
 		if (rc)
 			return rc;
 
-		/* Override the curently configured set of mc filters */
+		/* Override the currently configured set of mc filters */
 		rc = bnx2x_config_mcast(bp, &rparam,
 					BNX2X_MCAST_CMD_SET);
 		if (rc < 0)
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h
index a018f251d19867..30943f34716420 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_reg.h
@@ -2236,7 +2236,7 @@ extern const u32 dmae_reg_go_c[];
 /* [RW 1] Output enable for TX PAUSE signal to the MAC. */
 #define NIG_REG_P0_MAC_PAUSE_OUT_EN				 0x185b4
 /* [RW 32] Eight 4-bit configurations for specifying which COS (0-15 for
- * future expansion) each priorty is to be mapped to. Bits 3:0 specify the
+ * future expansion) each priority is to be mapped to. Bits 3:0 specify the
  * COS for priority 0. Bits 31:28 specify the COS for priority 7. The 3-bit
  * priority field is extracted from the outer-most VLAN in receive packet.
  * Only COS 0 and COS 1 are supported in E2. */
@@ -2434,7 +2434,7 @@ extern const u32 dmae_reg_go_c[];
 /* [RW 1] Output enable for TX PAUSE signal to the MAC. */
 #define NIG_REG_P1_MAC_PAUSE_OUT_EN				 0x185c8
 /* [RW 32] Eight 4-bit configurations for specifying which COS (0-15 for
- * future expansion) each priorty is to be mapped to. Bits 3:0 specify the
+ * future expansion) each priority is to be mapped to. Bits 3:0 specify the
  * COS for priority 0. Bits 31:28 specify the COS for priority 7. The 3-bit
  * priority field is extracted from the outer-most VLAN in receive packet.
  * Only COS 0 and COS 1 are supported in E2. */
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
index b8af508f9c9853..b337e61f9c81cd 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
@@ -207,10 +207,7 @@ void bnx2x_vfop_qctor_prep(struct bnx2x *bp,
 	 */
 	__set_bit(BNX2X_Q_FLG_TX_SWITCH, &setup_p->flags);
 	__set_bit(BNX2X_Q_FLG_TX_SEC, &setup_p->flags);
-	if (vf->spoofchk)
-		__set_bit(BNX2X_Q_FLG_ANTI_SPOOF, &setup_p->flags);
-	else
-		__clear_bit(BNX2X_Q_FLG_ANTI_SPOOF, &setup_p->flags);
+	__assign_bit(BNX2X_Q_FLG_ANTI_SPOOF, &setup_p->flags, vf->spoofchk);
 
 	/* Setup-op rx parameters */
 	if (test_bit(BNX2X_Q_TYPE_HAS_RX, &q_type)) {
@@ -2370,12 +2367,8 @@ static int bnx2x_set_pf_tx_switching(struct bnx2x *bp, bool enable)
 	q_params.cmd = BNX2X_Q_CMD_UPDATE;
 	__set_bit(BNX2X_Q_UPDATE_TX_SWITCHING_CHNG,
 		  &q_params.params.update.update_flags);
-	if (enable)
-		__set_bit(BNX2X_Q_UPDATE_TX_SWITCHING,
-			  &q_params.params.update.update_flags);
-	else
-		__clear_bit(BNX2X_Q_UPDATE_TX_SWITCHING,
-			    &q_params.params.update.update_flags);
+	__assign_bit(BNX2X_Q_UPDATE_TX_SWITCHING,
+		     &q_params.params.update.update_flags, enable);
 
 	/* send the ramrod on all the queues of the PF */
 	for_each_eth_queue(bp, i) {
@@ -2764,10 +2757,7 @@ static void bnx2x_set_vf_vlan_acceptance(struct bnx2x *bp,
 
 	/* need to remove/add the VF's accept_any_vlan bit */
 	accept_flags = bnx2x_leading_vfq(vf, accept_flags);
-	if (accept)
-		set_bit(BNX2X_ACCEPT_ANY_VLAN, &accept_flags);
-	else
-		clear_bit(BNX2X_ACCEPT_ANY_VLAN, &accept_flags);
+	assign_bit(BNX2X_ACCEPT_ANY_VLAN, &accept_flags, accept);
 
 	bnx2x_vf_prep_rx_mode(bp, LEADING_IDX, &rx_ramrod, vf,
 			      accept_flags);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index d7728d0c5b6e63..ca99f4b1a63c33 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -15654,8 +15654,8 @@ static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
 		goto init_err_disable;
 	}
 
-	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
-	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
+	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (rc) {
 		dev_err(&pdev->dev, "System does not support DMA, aborting\n");
 		rc = -EIO;
 		goto init_err_release;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index 62bc9cae613c38..622e89587e5db4 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -5733,7 +5733,8 @@ const struct ethtool_ops bnxt_ethtool_ops = {
 	.op_needs_rtnl			= ETHTOOL_OP_NEEDS_RTNL_SCHANNELS |
 					  ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM |
 					  ETHTOOL_OP_NEEDS_RTNL_SCOALESCE |
-					  ETHTOOL_OP_NEEDS_RTNL_RSS,
+					  ETHTOOL_OP_NEEDS_RTNL_RSS |
+					  ETHTOOL_OP_NEEDS_RTNL_TEST,
 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
 				     ETHTOOL_COALESCE_MAX_FRAMES |
 				     ETHTOOL_COALESCE_USECS_IRQ |
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
index b916080f4ff176..4c9db2f9fc25fa 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
@@ -57,6 +57,9 @@
  */
 #define GENET_RSB_PAD		(sizeof(struct status_64) + 2)
 
+/* RX buffer plus the skb_shared_info napi_build_skb() places behind it */
+#define GENET_RX_BUF_SIZE	SKB_HEAD_ALIGN(RX_BUF_LENGTH)
+
 /* Tx/Rx DMA register offset, skip 256 descriptors */
 #define WORDS_PER_BD(p)		(p->hw_params->words_per_bd)
 #define DMA_DESC_SIZE		(WORDS_PER_BD(priv) * sizeof(u32))
@@ -852,7 +855,8 @@ static int bcmgenet_get_coalesce(struct net_device *dev,
 	ec->rx_max_coalesced_frames =
 		bcmgenet_rdma_ring_readl(priv, 0, DMA_MBUF_DONE_THRESH);
 	ec->rx_coalesce_usecs =
-		bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) * 8192 / 1000;
+		(bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) &
+		 DMA_TIMEOUT_MASK) * 8192 / 1000;
 
 	for (i = 0; i <= priv->hw_params->rx_queues; i++) {
 		ring = &priv->rx_rings[i];
@@ -1346,9 +1350,8 @@ static void bcmgenet_get_ethtool_stats(struct net_device *dev,
 				p = (char *)&stats64;
 
 			p += s->stat_offset;
-			if (sizeof(unsigned long) != sizeof(u32) &&
-				s->stat_sizeof == sizeof(unsigned long))
-				data[i] = *(unsigned long *)p;
+			if (s->stat_sizeof == sizeof(u64))
+				data[i] = *(u64 *)p;
 			else
 				data[i] = *(u32 *)p;
 		}
@@ -1763,13 +1766,12 @@ static int bcmgenet_power_up(struct bcmgenet_priv *priv,
 	int ret = 0;
 	u32 reg;
 
-	if (!bcmgenet_has_ext(priv))
-		return ret;
-
-	reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
-
 	switch (mode) {
 	case GENET_POWER_PASSIVE:
+		if (!bcmgenet_has_ext(priv))
+			break;
+
+		reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
 		reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS |
 			 EXT_ENERGY_DET_MASK);
 		if (GENET_IS_V5(priv) && !bcmgenet_has_ephy_16nm(priv)) {
@@ -1793,8 +1795,12 @@ static int bcmgenet_power_up(struct bcmgenet_priv *priv,
 		break;
 
 	case GENET_POWER_CABLE_SENSE:
+		if (!bcmgenet_has_ext(priv))
+			break;
+
 		/* enable APD */
 		if (!GENET_IS_V5(priv)) {
+			reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
 			reg |= EXT_PWR_DN_EN_LD;
 			bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
 		}
@@ -2017,25 +2023,20 @@ static int bcmgenet_tx_poll(struct napi_struct *napi, int budget)
 {
 	struct bcmgenet_tx_ring *ring =
 		container_of(napi, struct bcmgenet_tx_ring, napi);
-	unsigned int work_done = 0;
 	struct netdev_queue *txq;
 
 	spin_lock(&ring->lock);
-	work_done = __bcmgenet_tx_reclaim(ring->priv->dev, ring);
+	__bcmgenet_tx_reclaim(ring->priv->dev, ring);
 	if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {
 		txq = netdev_get_tx_queue(ring->priv->dev, ring->index);
 		netif_tx_wake_queue(txq);
 	}
 	spin_unlock(&ring->lock);
 
-	if (work_done == 0) {
-		napi_complete(napi);
+	if (budget && napi_complete_done(napi, 0))
 		bcmgenet_tx_ring_int_enable(ring);
 
-		return 0;
-	}
-
-	return budget;
+	return 0;
 }
 
 static void bcmgenet_tx_reclaim_all(struct net_device *dev)
@@ -2251,11 +2252,12 @@ static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring,
 			      struct enet_cb *cb)
 {
 	struct bcmgenet_priv *priv = ring->priv;
+	unsigned int size = GENET_RX_BUF_SIZE;
+	unsigned int offset;
 	dma_addr_t mapping;
 	struct page *page;
 
-	page = page_pool_alloc_pages(ring->page_pool,
-				     GFP_ATOMIC);
+	page = page_pool_dev_alloc(ring->page_pool, &offset, &size);
 	if (!page) {
 		priv->mib.alloc_rx_buff_failed++;
 		netif_err(priv, rx_err, priv->dev,
@@ -2264,9 +2266,13 @@ static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring,
 	}
 
 	/* page_pool handles DMA mapping via PP_FLAG_DMA_MAP */
-	mapping = page_pool_get_dma_addr(page);
+	mapping = page_pool_get_dma_addr(page) + offset;
+	dma_sync_single_for_device(&priv->pdev->dev, mapping, RX_BUF_LENGTH,
+				   DMA_FROM_DEVICE);
 
 	cb->rx_page = page;
+	cb->rx_offset = offset;
+	cb->rx_size = size;
 	dmadesc_set_addr(priv, cb->bd_addr, mapping);
 
 	return 0;
@@ -2320,6 +2326,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
 
 	while ((rxpktprocessed < rxpkttoprocess) &&
 	       (rxpktprocessed < budget)) {
+		unsigned int rx_offset, rx_size;
 		struct status_64 *status;
 		struct page *rx_page;
 		void *hard_start;
@@ -2329,6 +2336,8 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
 
 		/* Save the received page before refilling */
 		rx_page = cb->rx_page;
+		rx_offset = cb->rx_offset;
+		rx_size = cb->rx_size;
 
 		if (bcmgenet_rx_refill(ring, cb)) {
 			BCMGENET_STATS64_INC(stats, dropped);
@@ -2338,10 +2347,10 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
 		/* Sync the full buffer; the HW may have written anywhere
 		 * up to RX_BUF_LENGTH.
 		 */
-		page_pool_dma_sync_for_cpu(ring->page_pool, rx_page, 0,
+		page_pool_dma_sync_for_cpu(ring->page_pool, rx_page, rx_offset,
 					   RX_BUF_LENGTH);
 
-		hard_start = page_address(rx_page);
+		hard_start = page_address(rx_page) + rx_offset;
 		status = (struct status_64 *)hard_start;
 		dma_length_status = status->length_status;
 
@@ -2407,7 +2416,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
 		/* Build SKB from the page - data starts at hard_start,
 		 * frame begins after RSB(64) + pad(2) = 66 bytes.
 		 */
-		skb = napi_build_skb(hard_start, PAGE_SIZE);
+		skb = napi_build_skb(hard_start, rx_size);
 		if (unlikely(!skb)) {
 			BCMGENET_STATS64_INC(stats, dropped);
 			page_pool_put_full_page(ring->page_pool, rx_page,
@@ -2759,14 +2768,16 @@ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv,
 static int bcmgenet_rx_ring_create_pool(struct bcmgenet_priv *priv,
 					struct bcmgenet_rx_ring *ring)
 {
+	/* Buffers share a page. bcmgenet_rx_refill() syncs each one for the
+	 * device, PP_FLAG_DMA_SYNC_DEV would sync the whole page.
+	 */
 	struct page_pool_params pp_params = {
 		.order = 0,
-		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+		.flags = PP_FLAG_DMA_MAP,
 		.pool_size = ring->size,
 		.nid = NUMA_NO_NODE,
 		.dev = &priv->pdev->dev,
 		.dma_dir = DMA_FROM_DEVICE,
-		.max_len = RX_BUF_LENGTH,
 	};
 	int err;
 
@@ -3441,6 +3452,8 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy)
 {
 	struct bcmgenet_priv *priv = netdev_priv(dev);
 
+	/* Stop completion polling before it can wake a stopped queue */
+	bcmgenet_disable_tx_napi(priv);
 	netif_tx_disable(dev);
 
 	/* Disable MAC receive */
@@ -3455,7 +3468,6 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy)
 	/* Disable MAC transmit. TX DMA disabled must be done before this */
 	umac_enable_set(priv, CMD_TX_EN, false);
 
-	bcmgenet_disable_tx_napi(priv);
 	bcmgenet_disable_rx_napi(priv);
 	bcmgenet_intr_disable(priv);
 
@@ -3632,6 +3644,9 @@ static int bcmgenet_set_mac_addr(struct net_device *dev, void *p)
 	if (netif_running(dev))
 		return -EBUSY;
 
+	if (!is_valid_ether_addr(addr->sa_data))
+		return -EADDRNOTAVAIL;
+
 	eth_hw_addr_set(dev, addr->sa_data);
 
 	return 0;
@@ -4131,14 +4146,17 @@ static int bcmgenet_probe(struct platform_device *pdev)
 	netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1);
 
 	/* Set default coalescing parameters */
-	for (i = 0; i <= priv->hw_params->rx_queues; i++)
+	for (i = 0; i <= priv->hw_params->rx_queues; i++) {
+		priv->rx_rings[i].priv = priv;
+		priv->rx_rings[i].index = i;
 		priv->rx_rings[i].rx_max_coalesced_frames = 1;
+	}
 
 	/* Initialize u64 stats seq counter for 32bit machines */
-	for (i = 0; i <= priv->hw_params->rx_queues; i++)
+	for (i = 0; i <= GENET_MAX_MQ_CNT; i++) {
 		u64_stats_init(&priv->rx_rings[i].stats64.syncp);
-	for (i = 0; i <= priv->hw_params->tx_queues; i++)
 		u64_stats_init(&priv->tx_rings[i].stats64.syncp);
+	}
 
 	/* libphy will determine the link state */
 	netif_carrier_off(dev);
@@ -4320,6 +4338,8 @@ static int bcmgenet_suspend(struct device *d)
 	netif_device_detach(dev);
 
 	if (device_may_wakeup(d) && priv->wolopts) {
+		/* Stop completion polling before it can wake a stopped queue */
+		bcmgenet_disable_tx_napi(priv);
 		netif_tx_disable(dev);
 
 		/* Suspend non-wake Rx data flows */
@@ -4348,7 +4368,6 @@ static int bcmgenet_suspend(struct device *d)
 			netdev_warn(priv->dev,
 				    "Timed out while disabling TX DMA\n");
 
-		bcmgenet_disable_tx_napi(priv);
 		bcmgenet_disable_rx_napi(priv);
 		disable_irq(priv->irq1);
 		bcmgenet_tx_reclaim_all(dev);
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h
index 22a958ba99024b..86f2aed20dbe87 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h
@@ -470,6 +470,8 @@ struct bcmgenet_rx_stats64 {
 struct enet_cb {
 	struct sk_buff      *skb;
 	struct page         *rx_page;
+	unsigned int        rx_offset;
+	unsigned int        rx_size;
 	void __iomem *bd_addr;
 	DEFINE_DMA_UNMAP_ADDR(dma_addr);
 	DEFINE_DMA_UNMAP_LEN(dma_len);
diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c
index 73a4b569b03e31..8b6806a79edff8 100644
--- a/drivers/net/ethernet/broadcom/tg3.c
+++ b/drivers/net/ethernet/broadcom/tg3.c
@@ -17915,11 +17915,14 @@ static int tg3_init_one(struct pci_dev *pdev,
 
 	err = tg3_get_device_address(tp, addr);
 	if (err) {
-		dev_err(&pdev->dev,
-			"Could not obtain valid ethernet address, aborting\n");
-		goto err_out_apeunmap;
+		dev_warn_probe(&pdev->dev, err,
+			       "Could not obtain a valid ethernet address\n");
+		if (err == -EPROBE_DEFER)
+			goto err_out_apeunmap;
+		eth_hw_addr_random(dev);
+	} else {
+		eth_hw_addr_set(dev, addr);
 	}
-	eth_hw_addr_set(dev, addr);
 
 	intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW;
 	rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW;
@@ -18047,6 +18050,10 @@ static int tg3_init_one(struct pci_dev *pdev,
 	return 0;
 
 err_out_apeunmap:
+	if (tg3_flag(tp, USE_PHYLIB))
+		tg3_phy_fini(tp);
+	tg3_mdio_fini(tp);
+
 	if (tp->aperegs) {
 		iounmap(tp->aperegs);
 		tp->aperegs = NULL;
diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c
index 8b75004ba7c9d4..55dfd48967851e 100644
--- a/drivers/net/ethernet/brocade/bna/bnad.c
+++ b/drivers/net/ethernet/brocade/bna/bnad.c
@@ -2571,6 +2571,22 @@ bnad_ioceth_disable(struct bnad *bnad)
 	return err;
 }
 
+/*
+ * The IOC timers rearm one another, so deleting one cannot stop a
+ * sibling callback from arming it again.  Shut them down so a later
+ * mod_timer() is ignored.
+ */
+static void
+bnad_ioc_timers_shutdown(struct bnad *bnad)
+{
+	struct bfa_ioc *ioc = &bnad->bna.ioceth.ioc;
+
+	timer_shutdown_sync(&ioc->ioc_timer);
+	timer_shutdown_sync(&ioc->sem_timer);
+	timer_shutdown_sync(&ioc->hb_timer);
+	timer_shutdown_sync(&ioc->iocpf_timer);
+}
+
 static int
 bnad_ioceth_enable(struct bnad *bnad)
 {
@@ -3727,9 +3743,7 @@ bnad_pci_probe(struct pci_dev *pdev,
 	bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX);
 disable_ioceth:
 	bnad_ioceth_disable(bnad);
-	timer_delete_sync(&bnad->bna.ioceth.ioc.ioc_timer);
-	timer_delete_sync(&bnad->bna.ioceth.ioc.sem_timer);
-	timer_delete_sync(&bnad->bna.ioceth.ioc.hb_timer);
+	bnad_ioc_timers_shutdown(bnad);
 	spin_lock_irqsave(&bnad->bna_lock, flags);
 	bna_uninit(bna);
 	spin_unlock_irqrestore(&bnad->bna_lock, flags);
@@ -3770,9 +3784,7 @@ bnad_pci_remove(struct pci_dev *pdev)
 
 	mutex_lock(&bnad->conf_mutex);
 	bnad_ioceth_disable(bnad);
-	timer_delete_sync(&bnad->bna.ioceth.ioc.ioc_timer);
-	timer_delete_sync(&bnad->bna.ioceth.ioc.sem_timer);
-	timer_delete_sync(&bnad->bna.ioceth.ioc.hb_timer);
+	bnad_ioc_timers_shutdown(bnad);
 	spin_lock_irqsave(&bnad->bna_lock, flags);
 	bna_uninit(bna);
 	spin_unlock_irqrestore(&bnad->bna_lock, flags);
diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h
index d6931c41f39da0..1cb2778fe49e9a 100644
--- a/drivers/net/ethernet/cadence/macb.h
+++ b/drivers/net/ethernet/cadence/macb.h
@@ -1197,7 +1197,7 @@ struct macb_queue;
 struct macb_or_gem_ops {
 	int	(*mog_alloc_rx_buffers)(struct macb *bp);
 	void	(*mog_free_rx_buffers)(struct macb *bp);
-	void	(*mog_init_rings)(struct macb *bp);
+	int	(*mog_init_rings)(struct macb *bp);
 	int	(*mog_rx)(struct macb_queue *queue, struct napi_struct *napi,
 			  int budget);
 };
@@ -1382,6 +1382,11 @@ struct macb {
 	struct delayed_work	tx_lpi_work;
 	u32			tx_lpi_timer;
 
+	/* ISR must not drive NAPI & BH mechanisms. True when the interface
+	 * is closed. Protected by bp->lock.
+	 */
+	bool			irq_quiesced;
+
 	int	rx_bd_rd_prefetch;
 	int	tx_bd_rd_prefetch;
 
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index b8234ac4b60212..20fe3078983431 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -583,7 +583,12 @@ static void macb_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode,
 
 static void macb_pcs_an_restart(struct phylink_pcs *pcs)
 {
-	/* Not supported */
+	struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs);
+	u32 old, new;
+
+	old = gem_readl(bp, PCSCNTRL);
+	new = old | BMCR_ANRESTART;
+	gem_writel(bp, PCSCNTRL, new);
 }
 
 static int macb_pcs_config(struct phylink_pcs *pcs,
@@ -594,11 +599,17 @@ static int macb_pcs_config(struct phylink_pcs *pcs,
 {
 	struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs);
 	u32 old, new;
+	int ret = 0;
 
 	old = gem_readl(bp, PCSANADV);
 	new = phylink_mii_c22_pcs_encode_advertisement(interface, advertising);
-	if (new != -EINVAL && old != new)
+	if (new != -EINVAL && old != new) {
+		/* pcs_config() is supposed to return 1 if AN advertisement
+		 * has changed
+		 */
+		ret = 1;
 		gem_writel(bp, PCSANADV, new);
+	}
 
 	/* Disable AN if it's not to be used, enable otherwise.
 	 * Must be written after PCSSEL is set in NCFGR which is done in
@@ -612,7 +623,7 @@ static int macb_pcs_config(struct phylink_pcs *pcs,
 	if (old != new)
 		gem_writel(bp, PCSCNTRL, new);
 
-	return 0;
+	return ret;
 }
 
 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = {
@@ -750,7 +761,9 @@ static void macb_mac_config(struct phylink_config *config, unsigned int mode,
 		ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL));
 		ncr &= ~GEM_BIT(ENABLE_HS_MAC);
 
-		if (state->interface == PHY_INTERFACE_MODE_SGMII) {
+		if (state->interface == PHY_INTERFACE_MODE_1000BASEX) {
+			ctrl |= GEM_BIT(PCSSEL);
+		} else if (state->interface == PHY_INTERFACE_MODE_SGMII) {
 			ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
 		} else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {
 			ctrl |= GEM_BIT(PCSSEL);
@@ -957,7 +970,8 @@ static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config,
 
 	if (interface == PHY_INTERFACE_MODE_10GBASER)
 		return &bp->phylink_usx_pcs;
-	else if (interface == PHY_INTERFACE_MODE_SGMII)
+	else if (interface == PHY_INTERFACE_MODE_1000BASEX ||
+		 interface == PHY_INTERFACE_MODE_SGMII)
 		return &bp->phylink_sgmii_pcs;
 	else
 		return NULL;
@@ -1025,13 +1039,15 @@ static int macb_mii_probe(struct net_device *netdev)
 	struct macb *bp = netdev_priv(netdev);
 
 	bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops;
+	bp->phylink_sgmii_pcs.poll = true;
 	bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops;
 
 	bp->phylink_config.dev = &netdev->dev;
 	bp->phylink_config.type = PHYLINK_NETDEV;
 	bp->phylink_config.mac_managed_pm = true;
 
-	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+	if (bp->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
 		bp->phylink_config.poll_fixed_state = true;
 		bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;
 		/* The PCSAUTONEG bit in PCSCNTRL is on out of reset. Setting
@@ -1060,9 +1076,12 @@ static int macb_mii_probe(struct net_device *netdev)
 			  bp->phylink_config.supported_interfaces);
 		phy_interface_set_rgmii(bp->phylink_config.supported_interfaces);
 
-		if (bp->caps & MACB_CAPS_PCS)
+		if (bp->caps & MACB_CAPS_PCS) {
+			__set_bit(PHY_INTERFACE_MODE_1000BASEX,
+				  bp->phylink_config.supported_interfaces);
 			__set_bit(PHY_INTERFACE_MODE_SGMII,
 				  bp->phylink_config.supported_interfaces);
+		}
 
 		if (bp->caps & MACB_CAPS_HIGH_SPEED) {
 			__set_bit(PHY_INTERFACE_MODE_10GBASER,
@@ -1486,7 +1505,7 @@ static int macb_tx_complete(struct macb_queue *queue, int budget)
 	return packets;
 }
 
-static void gem_rx_refill(struct macb_queue *queue)
+static int gem_rx_refill(struct macb_queue *queue)
 {
 	struct macb *bp = queue->bp;
 	struct macb_dma_desc *desc;
@@ -1558,6 +1577,14 @@ static void gem_rx_refill(struct macb_queue *queue)
 
 	netdev_vdbg(bp->netdev, "rx ring: queue: %p, prepared head %d, tail %d\n",
 		    queue, queue->rx_prepared_head, queue->rx_tail);
+
+	/* Fail if queue has zero prepared descriptors. This is critical because
+	 * nothing will ever trigger a refill again.
+	 */
+	if (queue->rx_prepared_head == queue->rx_tail)
+		return -ENOMEM;
+
+	return 0;
 }
 
 /* Mark DMA descriptors from begin up to and not including end as unused */
@@ -1596,6 +1623,13 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi,
 		dma_addr_t addr;
 		bool rxused;
 
+		/* Only descriptors in [rx_tail, rx_prepared_head) were armed
+		 * for hardware. Outside, we might have RX_USED descriptors for
+		 * alloc failures.
+		 */
+		if (queue->rx_tail == queue->rx_prepared_head)
+			break;
+
 		entry = macb_rx_ring_wrap(bp, queue->rx_tail);
 		desc = macb_rx_desc(queue, entry);
 
@@ -1859,6 +1893,10 @@ static bool macb_rx_pending(struct macb_queue *queue)
 	struct macb_dma_desc *desc;
 	unsigned int entry;
 
+	/* No armed descriptor left: nothing can be pending. */
+	if (macb_is_gem(bp) && queue->rx_tail == queue->rx_prepared_head)
+		return false;
+
 	entry = macb_rx_ring_wrap(bp, queue->rx_tail);
 	desc = macb_rx_desc(queue, entry);
 
@@ -1990,6 +2028,53 @@ static int macb_tx_poll(struct napi_struct *napi, int budget)
 	return work_done;
 }
 
+static void macb_quiesce_start(struct macb *bp)
+{
+	struct macb_queue *queue;
+	unsigned long flags;
+	unsigned int q;
+
+	spin_lock_irqsave(&bp->lock, flags);
+	bp->irq_quiesced = true;
+	spin_unlock_irqrestore(&bp->lock, flags);
+
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
+		synchronize_irq(queue->irq);
+
+	cancel_work_sync(&bp->hresp_err_bh_work);
+
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		/* Must be done before NAPI is disabled: the task ends with a
+		 * napi_enable() call.
+		 */
+		cancel_work_sync(&queue->tx_error_task);
+
+		napi_disable(&queue->napi_rx);
+		napi_disable(&queue->napi_tx);
+	}
+
+	/* Must be done after napi_tx is disabled: its completion re-arms
+	 * the LPI timer.
+	 */
+	cancel_delayed_work_sync(&bp->tx_lpi_work);
+}
+
+static void macb_quiesce_end(struct macb *bp)
+{
+	struct macb_queue *queue;
+	unsigned long flags;
+	unsigned int q;
+
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		napi_enable(&queue->napi_rx);
+		napi_enable(&queue->napi_tx);
+	}
+
+	spin_lock_irqsave(&bp->lock, flags);
+	bp->irq_quiesced = false;
+	spin_unlock_irqrestore(&bp->lock, flags);
+}
+
 static void macb_hresp_error_task(struct work_struct *work)
 {
 	struct macb *bp = from_work(bp, work, hresp_err_bh_work);
@@ -2132,8 +2217,8 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id)
 	spin_lock(&bp->lock);
 
 	while (status) {
-		/* close possible race with dev_close */
-		if (unlikely(!netif_running(netdev))) {
+		/* self-disarm while the netdev is closed */
+		if (unlikely(bp->irq_quiesced)) {
 			queue_writel(queue, IDR, -1);
 			macb_queue_isr_clear(bp, queue, -1);
 			break;
@@ -2749,14 +2834,24 @@ static int macb_alloc(struct macb *bp)
 
 	size = bp->num_queues * macb_tx_ring_size_per_queue(bp);
 	tx = dma_alloc_coherent(dev, size, &tx_dma, GFP_KERNEL);
-	if (!tx || upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1))
+	if (!tx)
+		goto out_err;
+	/* Record the buffer so that the error path frees it. */
+	bp->queues[0].tx_ring = tx;
+	bp->queues[0].tx_ring_dma = tx_dma;
+	if (upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1))
 		goto out_err;
 	netdev_dbg(bp->netdev, "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n",
 		   size, bp->num_queues, (unsigned long)tx_dma, tx);
 
 	size = bp->num_queues * macb_rx_ring_size_per_queue(bp);
 	rx = dma_alloc_coherent(dev, size, &rx_dma, GFP_KERNEL);
-	if (!rx || upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1))
+	if (!rx)
+		goto out_err;
+	/* Record the buffer so that the error path frees it. */
+	bp->queues[0].rx_ring = rx;
+	bp->queues[0].rx_ring_dma = rx_dma;
+	if (upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1))
 		goto out_err;
 	netdev_dbg(bp->netdev, "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n",
 		   size, bp->num_queues, (unsigned long)rx_dma, rx);
@@ -2783,19 +2878,26 @@ static int macb_alloc(struct macb *bp)
 	return -ENOMEM;
 }
 
-static void gem_init_rx_ring(struct macb_queue *queue)
+static int gem_init_rx_ring(struct macb_queue *queue)
 {
+	unsigned int i;
+
 	queue->rx_tail = 0;
 	queue->rx_prepared_head = 0;
 
-	gem_rx_refill(queue);
+	for (i = 0; i < queue->bp->rx_ring_size; i++)
+		macb_rx_desc(queue, i)->addr |= MACB_BIT(RX_USED);
+
+	return gem_rx_refill(queue);
 }
 
-static void gem_init_rings(struct macb *bp)
+static int gem_init_rings(struct macb *bp)
 {
 	struct macb_queue *queue;
 	struct macb_dma_desc *desc = NULL;
+	int last_err = 0;
 	unsigned int q;
+	int err;
 	int i;
 
 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
@@ -2808,11 +2910,15 @@ static void gem_init_rings(struct macb *bp)
 		queue->tx_head = 0;
 		queue->tx_tail = 0;
 
-		gem_init_rx_ring(queue);
+		err = gem_init_rx_ring(queue);
+		if (err)
+			last_err = err;
 	}
+
+	return last_err;
 }
 
-static void macb_init_rings(struct macb *bp)
+static int macb_init_rings(struct macb *bp)
 {
 	int i;
 	struct macb_dma_desc *desc = NULL;
@@ -2827,6 +2933,8 @@ static void macb_init_rings(struct macb *bp)
 	bp->queues[0].tx_head = 0;
 	bp->queues[0].tx_tail = 0;
 	desc->ctrl |= MACB_BIT(TX_WRAP);
+
+	return 0;
 }
 
 static void macb_reset_hw(struct macb *bp)
@@ -3137,8 +3245,6 @@ static int macb_open(struct net_device *netdev)
 {
 	size_t bufsz = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;
 	struct macb *bp = netdev_priv(netdev);
-	struct macb_queue *queue;
-	unsigned int q;
 	int err;
 
 	netdev_dbg(bp->netdev, "open\n");
@@ -3157,13 +3263,12 @@ static int macb_open(struct net_device *netdev)
 		goto pm_exit;
 	}
 
-	bp->macbgem_ops.mog_init_rings(bp);
+	err = bp->macbgem_ops.mog_init_rings(bp);
+	if (err)
+		goto free_rings;
 	macb_init_buffers(bp);
 
-	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
-		napi_enable(&queue->napi_rx);
-		napi_enable(&queue->napi_tx);
-	}
+	macb_quiesce_end(bp);
 
 	macb_init_hw(bp);
 
@@ -3190,11 +3295,11 @@ static int macb_open(struct net_device *netdev)
 	phy_power_off(bp->phy);
 
 reset_hw:
+	/* The netdev stays down: quiesce and drain, as macb_close() does. */
+	macb_quiesce_start(bp);
+
 	macb_reset_hw(bp);
-	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
-		napi_disable(&queue->napi_rx);
-		napi_disable(&queue->napi_tx);
-	}
+free_rings:
 	macb_free(bp);
 pm_exit:
 	pm_runtime_put_sync(&bp->pdev->dev);
@@ -3204,19 +3309,17 @@ static int macb_open(struct net_device *netdev)
 static int macb_close(struct net_device *netdev)
 {
 	struct macb *bp = netdev_priv(netdev);
-	struct macb_queue *queue;
 	unsigned long flags;
 	unsigned int q;
 
-	netif_tx_stop_all_queues(netdev);
+	macb_quiesce_start(bp);
 
-	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
-		napi_disable(&queue->napi_rx);
-		napi_disable(&queue->napi_tx);
+	/* Drain the BH contexts before stopping the queues: NAPI completion
+	 * and tx_error_task wake them up.
+	 */
+	netif_tx_stop_all_queues(netdev);
+	for (q = 0; q < bp->num_queues; ++q)
 		netdev_tx_reset_queue(netdev_get_tx_queue(netdev, q));
-	}
-
-	cancel_delayed_work_sync(&bp->tx_lpi_work);
 
 	phylink_stop(bp->phylink);
 	phylink_disconnect_phy(bp->phylink);
@@ -4738,6 +4841,11 @@ static int macb_init_dflt(struct platform_device *pdev)
 	bp->tx_ring_size = DEFAULT_TX_RING_SIZE;
 	bp->rx_ring_size = DEFAULT_RX_RING_SIZE;
 
+	/* No locking needed because the IRQs are not requested yet. The
+	 * flag is cleared by macb_open() and re-armed by macb_close().
+	 */
+	bp->irq_quiesced = true;
+
 	/* set the queue register mapping once for all: queue0 has a special
 	 * register mapping but we don't want to test the queue index then
 	 * compute the corresponding register offset at run time.
@@ -4910,7 +5018,9 @@ static int macb_init_dflt(struct platform_device *pdev)
 	/* Set MII management clock divider */
 	val = macb_mdc_clk_div(bp);
 	val |= macb_dbw(bp);
-	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)
+	if (bp->phy_interface == PHY_INTERFACE_MODE_1000BASEX)
+		val |= GEM_BIT(PCSSEL);
+	else if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)
 		val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
 	macb_writel(bp, NCFGR, val);
 
@@ -5382,7 +5492,7 @@ static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk,
 			       struct clk **hclk, struct clk **tx_clk,
 			       struct clk **rx_clk, struct clk **tsu_clk)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int err = 0;
 
 	err = macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);
diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c
index 14ae57fa00cba7..2fd063394698ee 100644
--- a/drivers/net/ethernet/cadence/macb_ptp.c
+++ b/drivers/net/ethernet/cadence/macb_ptp.c
@@ -374,16 +374,6 @@ void gem_ptp_remove(struct net_device *netdev)
 		 GEM_PTP_TIMER_NAME);
 }
 
-static int gem_ptp_set_ts_mode(struct macb *bp,
-			       enum macb_bd_control tx_bd_control,
-			       enum macb_bd_control rx_bd_control)
-{
-	gem_writel(bp, TXBDCTRL, GEM_BF(TXTSMODE, tx_bd_control));
-	gem_writel(bp, RXBDCTRL, GEM_BF(RXTSMODE, rx_bd_control));
-
-	return 0;
-}
-
 int gem_get_hwtst(struct net_device *netdev,
 		  struct kernel_hwtstamp_config *tstamp_config)
 {
@@ -396,25 +386,17 @@ int gem_get_hwtst(struct net_device *netdev,
 	return 0;
 }
 
-static void gem_ptp_set_one_step_sync(struct macb *bp, u8 enable)
-{
-	u32 reg_val;
-
-	reg_val = macb_readl(bp, NCR);
-
-	if (enable)
-		macb_writel(bp, NCR, reg_val | MACB_BIT(OSSMODE));
-	else
-		macb_writel(bp, NCR, reg_val & ~MACB_BIT(OSSMODE));
-}
-
 int gem_set_hwtst(struct net_device *netdev,
 		  struct kernel_hwtstamp_config *tstamp_config,
 		  struct netlink_ext_ack *extack)
 {
 	enum macb_bd_control tx_bd_control = TSTAMP_DISABLED;
 	enum macb_bd_control rx_bd_control = TSTAMP_DISABLED;
+	int rx_filter = tstamp_config->rx_filter;
 	struct macb *bp = netdev_priv(netdev);
+	u32 ncr_mask = MACB_BIT(SRTSM);
+	unsigned long flags;
+	u32 ncr_bits = 0;
 	u32 regval;
 
 	if (!macb_dma_ptp(bp))
@@ -424,24 +406,21 @@ int gem_set_hwtst(struct net_device *netdev,
 	case HWTSTAMP_TX_OFF:
 		break;
 	case HWTSTAMP_TX_ONESTEP_SYNC:
-		gem_ptp_set_one_step_sync(bp, 1);
+		ncr_bits |= MACB_BIT(OSSMODE);
+		ncr_mask |= MACB_BIT(OSSMODE);
 		tx_bd_control = TSTAMP_ALL_FRAMES;
 		break;
 	case HWTSTAMP_TX_ON:
-		gem_ptp_set_one_step_sync(bp, 0);
+		ncr_mask |= MACB_BIT(OSSMODE);
 		tx_bd_control = TSTAMP_ALL_FRAMES;
 		break;
 	default:
 		return -ERANGE;
 	}
 
-	switch (tstamp_config->rx_filter) {
+	switch (rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		break;
-	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
-		break;
-	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
-		break;
 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
@@ -451,25 +430,32 @@ int gem_set_hwtst(struct net_device *netdev,
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
-		rx_bd_control =  TSTAMP_ALL_PTP_FRAMES;
-		tstamp_config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
-		regval = macb_readl(bp, NCR);
-		macb_writel(bp, NCR, (regval | MACB_BIT(SRTSM)));
+		rx_bd_control = TSTAMP_ALL_PTP_FRAMES;
+		rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
+		ncr_bits |= MACB_BIT(SRTSM);
 		break;
+	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 	case HWTSTAMP_FILTER_ALL:
 		rx_bd_control = TSTAMP_ALL_FRAMES;
-		tstamp_config->rx_filter = HWTSTAMP_FILTER_ALL;
+		rx_filter = HWTSTAMP_FILTER_ALL;
 		break;
 	default:
-		tstamp_config->rx_filter = HWTSTAMP_FILTER_NONE;
 		return -ERANGE;
 	}
 
-	bp->tstamp_config = *tstamp_config;
+	spin_lock_irqsave(&bp->lock, flags);
+	regval = macb_readl(bp, NCR);
+	regval = (regval & ~ncr_mask) | ncr_bits;
+	macb_writel(bp, NCR, regval);
 
-	if (gem_ptp_set_ts_mode(bp, tx_bd_control, rx_bd_control) != 0)
-		return -ERANGE;
+	gem_writel(bp, TXBDCTRL, GEM_BF(TXTSMODE, tx_bd_control));
+	gem_writel(bp, RXBDCTRL, GEM_BF(RXTSMODE, rx_bd_control));
+
+	tstamp_config->rx_filter = rx_filter;
+	bp->tstamp_config = *tstamp_config;
+	spin_unlock_irqrestore(&bp->lock, flags);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
index 93fccfec288d47..d3ebbb40579e44 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/cn66xx_device.c
@@ -177,7 +177,7 @@ static void lio_cn66xx_setup_pkt_ctl_regs(struct octeon_device *oct)
 
 	/* 66XX SPECIFIC */
 	if (CFG_GET_OQ_MAX_Q(cn6xxx->conf) <= 4)
-		/* Disable RING_EN if only upto 4 rings are used. */
+		/* Disable RING_EN if only up to 4 rings are used. */
 		pktctl &= ~(1 << 4);
 	else
 		pktctl |= (1 << 4);
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c
index 93b06d599eed62..bed450bfad3838 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_core.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c
@@ -565,7 +565,7 @@ static int octeon_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
 }
 
 /**
- * liquidio_push_packet - Routine to push packets arriving on Octeon interface upto network layer.
+ * liquidio_push_packet - Routine to push packets arriving on Octeon interface up to network layer.
  * @octeon_id:octeon device id.
  * @skbuff:   skbuff struct to be passed to network layer.
  * @len:      size of total data received.
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_config.h b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
index 24c21200121212..d2d64edaf3bf16 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_config.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_config.h
@@ -254,7 +254,7 @@ struct octeon_oq_config {
 	u64 pkts_per_intr:16;
 
 	/** Interrupt Coalescing (Time Interval). Octeon will interrupt the
-	 *  host if atleast one packet was sent in the time interval specified
+	 *  host if at least one packet was sent in the time interval specified
 	 *  by this field. The driver uses time interval interrupt coalescing
 	 *  by default. The time is specified in microseconds.
 	 */
@@ -296,7 +296,7 @@ struct octeon_oq_config {
 	u64 oq_intr_pkt:16;
 
 	/** Interrupt Coalescing (Time Interval). Octeon will interrupt the
-	 *  host if atleast one packet was sent in the time interval specified
+	 *  host if at least one packet was sent in the time interval specified
 	 *  by this field. The driver uses time interval interrupt coalescing
 	 *  by default.  The time is specified in microseconds.
 	 */
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
index a04f36a0e1a080..fdd7d3a77a2dc4 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_iq.h
@@ -70,7 +70,7 @@ struct oct_iq_stats {
 
 /** The instruction (input) queue.
  *  The input queue is used to post raw (instruction) mode data or packet
- *  data to Octeon device from the host. Each input queue (upto 4) for
+ *  data to Octeon device from the host. Each input queue (up to 4) for
  *  a Octeon device has one such structure to represent it.
  */
 struct octeon_instr_queue {
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_main.h b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
index 953edf0c7096c3..d2a52673cac58f 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_main.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_main.h
@@ -166,7 +166,7 @@ static inline int octeon_map_pci_barx(struct octeon_device *oct,
  * errno -EINTR: user abort the command.
  * errno -ETIME: user spefified timeout value has been expired.
  * errno -EBUSY: the response of the request does not return in
- *               resonable time (LIO_SC_MAX_TMO_MS).
+ *               reasonable time (LIO_SC_MAX_TMO_MS).
  *               the sc wll be move to zombie response list by
  *               lio_process_ordered_list()
  *
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
index c139fc423764ea..8341faaa0c2441 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_nic.h
@@ -277,7 +277,7 @@ int octnet_send_nic_data_pkt(struct octeon_device *oct,
 
 /** Send a NIC control packet to the device
  * @param oct - octeon device pointer
- * @param nctrl - control structure with command, timout, and callback info
+ * @param nctrl - control structure with command, timeout, and callback info
  * @returns IQ_FAILED if it failed to add to the input queue. IQ_STOP if the
  * queue should be stopped, and IQ_SEND_OK if it sent okay.
  */
diff --git a/drivers/net/ethernet/cavium/liquidio/request_manager.c b/drivers/net/ethernet/cavium/liquidio/request_manager.c
index d7cfb20eea00ca..a983e7f79461d4 100644
--- a/drivers/net/ethernet/cavium/liquidio/request_manager.c
+++ b/drivers/net/ethernet/cavium/liquidio/request_manager.c
@@ -40,7 +40,7 @@ static void  __check_db_timeout(struct octeon_device *oct, u64 iq_no);
 
 static void (*reqtype_free_fn[MAX_OCTEON_DEVICES][REQTYPE_LAST + 1]) (void *);
 
-/* Define this to return the request status comaptible to old code */
+/* Define this to return the request status compatible to old code */
 /*#define OCTEON_USE_OLD_REQ_STATUS*/
 
 /* Return 0 on success, 1 on failure */
diff --git a/drivers/net/ethernet/cavium/thunder/nic.h b/drivers/net/ethernet/cavium/thunder/nic.h
index 090d6b83982a2e..bb3f4fa6b9959c 100644
--- a/drivers/net/ethernet/cavium/thunder/nic.h
+++ b/drivers/net/ethernet/cavium/thunder/nic.h
@@ -97,7 +97,7 @@
  * This value ensures that once a packet has been "accepted"
  * for reception it will not get dropped due to non-availability
  * of CQ descriptor. An errata in HW mandates this value to be
- * atleast 0x100.
+ * at least 0x100.
  */
 #define NICPF_CQM_MIN_DROP_LEVEL       0x100
 
diff --git a/drivers/net/ethernet/cavium/thunder/nic_main.c b/drivers/net/ethernet/cavium/thunder/nic_main.c
index 0ec65ec634df57..086f32b61580dd 100644
--- a/drivers/net/ethernet/cavium/thunder/nic_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nic_main.c
@@ -595,7 +595,7 @@ static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg)
 	nic_reg_write(nic, idx_addr | (cpi_base << 3), cpi_cfg);
 }
 
-/* 4 level transmit side scheduler configutation
+/* 4 level transmit side scheduler configuration
  * for TNS bypass mode
  *
  * Sample configuration for SQ0 on 88xx
diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c
index 6b5824f45a14cd..9086e6dd558aad 100644
--- a/drivers/net/ethernet/cisco/enic/enic_main.c
+++ b/drivers/net/ethernet/cisco/enic/enic_main.c
@@ -3001,7 +3001,6 @@ static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct device *dev = &pdev->dev;
 	struct net_device *netdev;
 	struct enic *enic;
-	int using_dac = 0;
 	unsigned int i;
 	int err;
 #ifdef CONFIG_PCI_IOV
@@ -3044,20 +3043,11 @@ static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_master(pdev);
 
-	/* Query PCI controller on system for DMA addressing
-	 * limitation for the device.  Try 47-bit first, and
-	 * fail to 32-bit.
-	 */
-
+	/* The device supports DMA addresses up to 47 bits. */
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(47));
 	if (err) {
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(dev, "No usable DMA configuration, aborting\n");
-			goto err_out_release_regions;
-		}
-	} else {
-		using_dac = 1;
+		dev_err(dev, "No usable DMA configuration, aborting\n");
+		goto err_out_release_regions;
 	}
 
 	/* Map vNIC resources from BAR0-5
@@ -3330,8 +3320,7 @@ static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev->hw_features |= NETIF_F_NTUPLE;
 #endif
 
-	if (using_dac)
-		netdev->features |= NETIF_F_HIGHDMA;
+	netdev->features |= NETIF_F_HIGHDMA;
 
 	netdev->priv_flags |= IFF_UNICAST_FLT;
 
diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c
index 2dd2fa801829c1..2fe7fd0202d266 100644
--- a/drivers/net/ethernet/cortina/gemini.c
+++ b/drivers/net/ethernet/cortina/gemini.c
@@ -16,6 +16,7 @@
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/module.h>
+#include <linux/net.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 #include <linux/slab.h>
@@ -736,7 +737,8 @@ gmac_get_queue_page(struct gemini_ethernet *geth,
 	mapping = addr & PAGE_MASK;
 
 	if (!geth->freeq_pages) {
-		dev_err(geth->dev, "try to get page with no page list\n");
+		dev_err_ratelimited(geth->dev,
+				    "try to get page with no page list\n");
 		return NULL;
 	}
 
@@ -1441,7 +1443,7 @@ static struct sk_buff *gmac_skb_if_good_frame(struct gemini_ethernet_port *port,
 }
 
 static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget,
-			    unsigned int *freeq_consumed)
+			    unsigned int *freeq_consumed, bool *reschedule)
 {
 	struct gemini_ethernet_port *port = netdev_priv(netdev);
 	unsigned short m = (1 << port->rxq_order) - 1;
@@ -1449,6 +1451,8 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget,
 	void __iomem *ptr_reg = port->rxq_rwptr;
 	unsigned int frag_nr = port->rx_frag_nr;
 	struct sk_buff *skb = port->rx_skb;
+	/* Bound malformed chains while allowing maximum fragments per frame. */
+	unsigned int desc_limit = budget * MAX_SKB_FRAGS;
 	unsigned int consumed = 0;
 	unsigned int frame_len, frag_len;
 	struct gmac_rxdesc *rx = NULL;
@@ -1475,7 +1479,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget,
 	r = rw.bits.rptr;
 	w = rw.bits.wptr;
 
-	while (budget && w != r) {
+	while (budget && consumed < desc_limit && w != r) {
 		page = NULL;
 		rx = port->rxq_ring + r;
 		word0 = rx->word0;
@@ -1502,15 +1506,18 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget,
 		}
 
 		if (!mapping) {
-			netdev_err(netdev,
-				   "rxq[%u]: HW BUG: zero DMA desc\n", r);
+			if (net_ratelimit())
+				netdev_err(netdev,
+					   "rxq[%u]: HW BUG: zero DMA desc\n",
+					   r);
 			goto err_drop;
 		}
 
 		/* Freeq pointers are one page off */
 		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
 		if (!gpage) {
-			dev_err(geth->dev, "could not find mapping\n");
+			dev_err_ratelimited(geth->dev,
+					    "could not find mapping\n");
 			goto err_drop;
 		}
 		page = gpage->page;
@@ -1535,7 +1542,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget,
 		if (frag_nr == MAX_SKB_FRAGS)
 			goto err_drop;
 
-		if (frag_len == 0)
+		if (frag_len == 0 && net_ratelimit())
 			netdev_err(netdev, "Received fragment with len = 0\n");
 
 		skb_fill_page_desc(skb, frag_nr, page, page_offs, frag_len);
@@ -1579,6 +1586,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget,
 	port->rx_frag_nr = frag_nr;
 	port->rx_dropping = dropping;
 	*freeq_consumed = consumed;
+	*reschedule = budget && w != r;
 	writew(r, ptr_reg);
 	return received;
 }
@@ -1590,12 +1598,13 @@ static int gmac_napi_poll(struct napi_struct *napi, int budget)
 	unsigned int freeq_threshold;
 	unsigned int freeq_consumed;
 	unsigned int received;
+	bool reschedule;
 
 	freeq_threshold = 1 << (geth->freeq_order - 1);
 	u64_stats_update_begin(&port->rx_stats_syncp);
 
-	received = gmac_rx(napi->dev, budget, &freeq_consumed);
-	if (received < budget)
+	received = gmac_rx(napi->dev, budget, &freeq_consumed, &reschedule);
+	if (!reschedule && received < budget)
 		++port->rx_napi_exits;
 
 	u64_stats_update_end(&port->rx_stats_syncp);
@@ -1606,10 +1615,11 @@ static int gmac_napi_poll(struct napi_struct *napi, int budget)
 		geth_fill_freeq(geth, true);
 	}
 
-	if (received < budget && napi_complete_done(napi, received))
+	if (!reschedule && received < budget &&
+	    napi_complete_done(napi, received))
 		gmac_enable_rx_irq(napi->dev, 1);
 
-	return received;
+	return reschedule ? budget : received;
 }
 
 static void gmac_dump_dma_state(struct net_device *netdev)
diff --git a/drivers/net/ethernet/cortina/gemini.h b/drivers/net/ethernet/cortina/gemini.h
index 24bb989981f233..d359c633c5de34 100644
--- a/drivers/net/ethernet/cortina/gemini.h
+++ b/drivers/net/ethernet/cortina/gemini.h
@@ -726,7 +726,7 @@ union gmac_config0 {
 		unsigned int dis_rx:1;
 		/* 2: transmit data loopback enable */
 		unsigned int loop_back:1;
-		/* 3: flow control also trigged by Rx queues */
+		/* 3: flow control also triggered by Rx queues */
 		unsigned int flow_ctrl:1;
 		/* 4-7: adjust IFG from 96+/-56 */
 		unsigned int adj_ifg:4;
diff --git a/drivers/net/ethernet/davicom/dm9000.c b/drivers/net/ethernet/davicom/dm9000.c
index b87eaf0c250ce9..e294b365eb0661 100644
--- a/drivers/net/ethernet/davicom/dm9000.c
+++ b/drivers/net/ethernet/davicom/dm9000.c
@@ -1511,7 +1511,11 @@ dm9000_probe(struct platform_device *pdev)
 	}
 
 	db->irq_wake = platform_get_irq_optional(pdev, 1);
-	if (db->irq_wake >= 0) {
+	if (db->irq_wake < 0 && db->irq_wake != -ENXIO) {
+		ret = db->irq_wake;
+		goto out;
+	}
+	if (db->irq_wake > 0) {
 		dev_dbg(db->dev, "wakeup irq %d\n", db->irq_wake);
 
 		ret = request_irq(db->irq_wake, dm9000_wol_interrupt,
diff --git a/drivers/net/ethernet/dec/tulip/de2104x.c b/drivers/net/ethernet/dec/tulip/de2104x.c
index f38a10c79907f2..fa52e099409c52 100644
--- a/drivers/net/ethernet/dec/tulip/de2104x.c
+++ b/drivers/net/ethernet/dec/tulip/de2104x.c
@@ -373,7 +373,7 @@ static void de_rx_err_acct (struct de_private *de, unsigned rx_tail,
 		  rx_tail, status, len);
 
 	if ((status & 0x38000300) != 0x0300) {
-		/* Ingore earlier buffers. */
+		/* Ignore earlier buffers. */
 		if ((status & 0xffff) != 0x7fff) {
 			netif_warn(de, rx_err, de->dev,
 				   "Oversized Ethernet frame spanned multiple buffers, status %08x!\n",
diff --git a/drivers/net/ethernet/dec/tulip/interrupt.c b/drivers/net/ethernet/dec/tulip/interrupt.c
index 0a12cb9b3ba784..8c02771b28e4af 100644
--- a/drivers/net/ethernet/dec/tulip/interrupt.c
+++ b/drivers/net/ethernet/dec/tulip/interrupt.c
@@ -177,7 +177,7 @@ int tulip_poll(struct napi_struct *napi, int budget)
 			   pkt_len > 1518) {
 			       if ((status & (RxLengthOver2047 |
 					      RxWholePkt)) != RxWholePkt) {
-                                /* Ingore earlier buffers. */
+                                /* Ignore earlier buffers. */
                                        if ((status & 0xffff) != 0x7fff) {
                                                if (tulip_debug > 1)
                                                        dev_warn(&dev->dev,
@@ -408,7 +408,7 @@ static int tulip_rx(struct net_device *dev)
 		    pkt_len > 1518) {
 			if ((status & (RxLengthOver2047 |
 			     RxWholePkt))         != RxWholePkt) {
-				/* Ingore earlier buffers. */
+				/* Ignore earlier buffers. */
 				if ((status & 0xffff) != 0x7fff) {
 					if (tulip_debug > 1)
 						netdev_warn(dev,
diff --git a/drivers/net/ethernet/dec/tulip/media.c b/drivers/net/ethernet/dec/tulip/media.c
index 55d6fc99f40b6d..0c9694d2b38ae6 100644
--- a/drivers/net/ethernet/dec/tulip/media.c
+++ b/drivers/net/ethernet/dec/tulip/media.c
@@ -24,7 +24,7 @@
 
 /* Read and write the MII registers using software-generated serial
    MDIO protocol.  It is just different enough from the EEPROM protocol
-   to not share code.  The maxium data clock rate is 2.5 Mhz. */
+   to not share code.  The maximum data clock rate is 2.5 Mhz. */
 #define MDIO_SHIFT_CLK		0x10000
 #define MDIO_DATA_WRITE0	0x00000
 #define MDIO_DATA_WRITE1	0x20000
diff --git a/drivers/net/ethernet/dec/tulip/tulip.h b/drivers/net/ethernet/dec/tulip/tulip.h
index 5e010e1fa6f79c..e39e5c78d4ccd7 100644
--- a/drivers/net/ethernet/dec/tulip/tulip.h
+++ b/drivers/net/ethernet/dec/tulip/tulip.h
@@ -250,7 +250,7 @@ enum t21143_csr6_bits {
 	csr6_ttm = (1<<22),  /* Transmit Threshold Mode, set for 10baseT, 0 for 100BaseTX */
 	csr6_sf = (1<<21),   /* Store and forward. If set ignores TR bits */
 	csr6_hbd = (1<<19),  /* Heart beat disable. Disables SQE function in 10baseT */
-	csr6_ps = (1<<18),   /* Port Select. 0 (defualt) = 10baseT, 1 = 100baseTX: can't be set */
+	csr6_ps = (1<<18),   /* Port Select. 0 (default) = 10baseT, 1 = 100baseTX: can't be set */
 	csr6_ca = (1<<17),   /* Collision Offset Enable. If set uses special algorithm in low collision situations */
 	csr6_trh = (1<<15),  /* Transmit Threshold high bit */
 	csr6_trl = (1<<14),  /* Transmit Threshold low bit */
@@ -274,7 +274,7 @@ enum t21143_csr6_bits {
 	csr6_om_int_loop = (1<<10), /* internal (FIFO) loopback flag */
 	csr6_om_ext_loop = (1<<11), /* external (PMD) loopback flag */
 	/* set both and you get (PHY) loopback */
-	csr6_fd = (1<<9),    /* Full duplex mode, disables hearbeat, no loopback */
+	csr6_fd = (1<<9),    /* Full duplex mode, disables heartbeat, no loopback */
 	csr6_pm = (1<<7),    /* Pass All Multicast */
 	csr6_pr = (1<<6),    /* Promiscuous mode */
 	csr6_sb = (1<<5),    /* Start(1)/Stop(0) backoff counter */
diff --git a/drivers/net/ethernet/dec/tulip/winbond-840.c b/drivers/net/ethernet/dec/tulip/winbond-840.c
index a5581f1d014175..be9199f55a8a9b 100644
--- a/drivers/net/ethernet/dec/tulip/winbond-840.c
+++ b/drivers/net/ethernet/dec/tulip/winbond-840.c
@@ -1188,7 +1188,7 @@ static int netdev_rx(struct net_device *dev)
 			break;
 		if ((status & 0x38008300) != 0x0300) {
 			if ((status & 0x38000300) != 0x0300) {
-				/* Ingore earlier buffers. */
+				/* Ignore earlier buffers. */
 				if ((status & 0xffff) != 0x7fff) {
 					dev_warn(&dev->dev,
 						 "Oversized Ethernet frame spanned multiple buffers, entry %#x status %04x!\n",
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c
index 4497e3c0456d18..440319a1e77516 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ptp.c
@@ -232,6 +232,8 @@ static void dpaa2_ptp_remove(struct fsl_mc_device *mc_dev)
 	fsl_mc_free_irqs(mc_dev);
 	dprtc_close(mc_dev->mc_io, 0, mc_dev->mc_handle);
 	fsl_mc_portal_free(mc_dev->mc_io);
+	of_node_put(dev->of_node);
+	dev->of_node = NULL;
 }
 
 static const struct fsl_mc_device_id dpaa2_ptp_match_id_table[] = {
diff --git a/drivers/net/ethernet/freescale/enetc/Kconfig b/drivers/net/ethernet/freescale/enetc/Kconfig
index db5c17a4461372..f425f82a6213a8 100644
--- a/drivers/net/ethernet/freescale/enetc/Kconfig
+++ b/drivers/net/ethernet/freescale/enetc/Kconfig
@@ -69,6 +69,7 @@ config FSL_ENETC_VF
 	depends on PCI_MSI
 	select FSL_ENETC_CORE
 	select FSL_ENETC_MDIO
+	select NXP_NTMP
 	select PHYLINK
 	select DIMLIB
 	select CRC_ITU_T
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c
index 80f0082f6c6337..c6571b5cc84258 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc.c
@@ -88,10 +88,7 @@ void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable)
 
 	spin_lock(&si->gen_lock);
 
-	if (enable)
-		set_bit(ENETC_RXBDR_CM, &priv->flags);
-	else
-		clear_bit(ENETC_RXBDR_CM, &priv->flags);
+	assign_bit(ENETC_RXBDR_CM, &priv->flags, enable);
 
 	for (int i = 0; i < priv->num_rx_rings; i++) {
 		u32 old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR);
@@ -2935,11 +2932,31 @@ static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
 static int enetc_phylink_connect(struct net_device *ndev)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_si *si = priv->si;
 	struct ethtool_keee edata;
 	int err;
 
 	if (!priv->phylink) {
 		/* phy-less mode */
+		if (!si->ops->vf_reg_link_status_notifier)
+			goto carrier_on;
+
+		/* For phy-less VFs on ENETC v4, attempt to register a link
+		 * status notifier with the PF via the VSI-to-PSI messaging
+		 * channel. If registration succeeds, the PF will immediately
+		 * send the current link status and broadcast future link
+		 * transitions; carrier state is then managed in
+		 * enetc_vf_msg_handle_link_status(). If registration fails,
+		 * fall back to the LS1028A behaviour and assert carrier
+		 * unconditionally via netif_carrier_on().
+		 */
+		if (!si->ops->vf_reg_link_status_notifier(si))
+			return 0;
+
+		dev_warn(&ndev->dev,
+			 "Link status notifier registration failed\n");
+
+carrier_on:
 		netif_carrier_on(ndev);
 		return 0;
 	}
@@ -3024,10 +3041,6 @@ int enetc_open(struct net_device *ndev)
 	if (err)
 		goto err_setup_irqs;
 
-	err = enetc_phylink_connect(ndev);
-	if (err)
-		goto err_phy_connect;
-
 	tx_res = enetc_alloc_tx_resources(priv);
 	if (IS_ERR(tx_res)) {
 		err = PTR_ERR(tx_res);
@@ -3040,6 +3053,10 @@ int enetc_open(struct net_device *ndev)
 		goto err_alloc_rx;
 	}
 
+	err = enetc_phylink_connect(ndev);
+	if (err)
+		goto err_phy_connect;
+
 	enetc_tx_onestep_tstamp_init(priv);
 	enetc_assign_tx_resources(priv, tx_res);
 	enetc_assign_rx_resources(priv, rx_res);
@@ -3048,12 +3065,11 @@ int enetc_open(struct net_device *ndev)
 
 	return 0;
 
+err_phy_connect:
+	enetc_free_rx_resources(rx_res, priv->num_rx_rings);
 err_alloc_rx:
 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
 err_alloc_tx:
-	if (priv->phylink)
-		phylink_disconnect_phy(priv->phylink);
-err_phy_connect:
 	enetc_free_irqs(priv);
 err_setup_irqs:
 	clk_disable_unprepare(priv->ref_clk);
@@ -3093,6 +3109,7 @@ EXPORT_SYMBOL_GPL(enetc_stop);
 int enetc_close(struct net_device *ndev)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_si *si = priv->si;
 
 	enetc_stop(ndev);
 
@@ -3100,6 +3117,20 @@ int enetc_close(struct net_device *ndev)
 		phylink_stop(priv->phylink);
 		phylink_disconnect_phy(priv->phylink);
 	} else {
+		if (!si->ops->vf_unreg_link_status_notifier)
+			goto carrier_off;
+
+		/* No need to check whether the previous registration was
+		 * successful. Sending the deregistration message has no
+		 * impact; the PF side simply clears the corresponding bit
+		 * in link_status_ms_mask for the VF.
+		 */
+		if (!si->ops->vf_unreg_link_status_notifier(si))
+			goto carrier_off;
+
+		dev_warn(&ndev->dev,
+			 "Link status notifier unregistration failed\n");
+carrier_off:
 		netif_carrier_off(ndev);
 	}
 
@@ -3794,6 +3825,13 @@ static const struct enetc_drvdata enetc_vf_data = {
 	.eth_ops = &enetc_vf_ethtool_ops,
 };
 
+static const struct enetc_drvdata enetc4_vf_data = {
+	.sysclk_freq = ENETC_CLK_333M,
+	.tx_csum = true,
+	.max_frags = ENETC4_MAX_SKB_FRAGS,
+	.eth_ops = &enetc_vf_ethtool_ops,
+};
+
 static const struct enetc_platform_info enetc_info[] = {
 	{ .revision = ENETC_REV_1_0,
 	  .dev_id = ENETC_DEV_ID_PF,
@@ -3807,6 +3845,10 @@ static const struct enetc_platform_info enetc_info[] = {
 	  .dev_id = ENETC_DEV_ID_VF,
 	  .data = &enetc_vf_data,
 	},
+	{ .revision = ENETC_REV_4_1,
+	  .dev_id = NXP_ENETC_VF_DEV_ID,
+	  .data = &enetc4_vf_data,
+	},
 	{
 	  .revision = ENETC_REV_4_3,
 	  .dev_id = NXP_ENETC_PPM_DEV_ID,
@@ -3816,6 +3858,10 @@ static const struct enetc_platform_info enetc_info[] = {
 	  .dev_id = NXP_ENETC_PF_DEV_ID,
 	  .data = &enetc4_pf_data,
 	},
+	{ .revision = ENETC_REV_4_3,
+	  .dev_id = NXP_ENETC_VF_DEV_ID,
+	  .data = &enetc4_vf_data,
+	},
 };
 
 int enetc_get_driver_data(struct enetc_si *si)
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h
index d1e9d913005753..d9e91832a9c1d3 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc.h
@@ -25,6 +25,7 @@
 #define ENETC_CBD_DATA_MEM_ALIGN 64
 
 #define ENETC_MADDR_HASH_TBL_SZ	64
+#define ENETC_INT_NAME_MAX	(IFNAMSIZ + 8)
 
 enum enetc_mac_addr_type {UC, MC, MADDR_TYPE};
 
@@ -299,6 +300,10 @@ struct enetc_si_ops {
 	int (*set_rss_table)(struct enetc_si *si, const u32 *table, int count);
 	int (*setup_cbdr)(struct enetc_si *si);
 	void (*teardown_cbdr)(struct enetc_si *si);
+
+	/* VSI-specific hooks */
+	int (*vf_reg_link_status_notifier)(struct enetc_si *si);
+	int (*vf_unreg_link_status_notifier)(struct enetc_si *si);
 };
 
 /* PCI IEP device data */
@@ -333,6 +338,11 @@ struct enetc_si {
 
 	struct dentry *debugfs_root;
 	struct enetc_msg_swbd msg; /* Only valid for VSI */
+	struct workqueue_struct *workqueue;
+	struct work_struct msg_task;
+	char msg_int_name[ENETC_INT_NAME_MAX];
+
+	struct enetc_mac_filter mac_filter[MADDR_TYPE];
 };
 
 #define ENETC_SI_ALIGN	32
@@ -374,7 +384,6 @@ static inline bool enetc_is_pseudo_mac(struct enetc_si *si)
 }
 
 #define ENETC_MAX_NUM_TXQS	8
-#define ENETC_INT_NAME_MAX	(IFNAMSIZ + 8)
 
 struct enetc_int_vector {
 	void __iomem *rbier;
diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c b/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c
index 5029038bf99faa..c73722e2285f54 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc4_debugfs.c
@@ -6,24 +6,48 @@
 #include <linux/seq_file.h>
 #include <linux/string_choices.h>
 
-#include "enetc_pf.h"
+#include "enetc_pf_common.h"
 #include "enetc4_debugfs.h"
 
-static void enetc_show_si_mac_hash_filter(struct seq_file *s, int i)
+static void enetc_vf_state_lock(struct enetc_pf *pf, int vf_id)
 {
-	struct enetc_si *si = s->private;
-	struct enetc_hw *hw = &si->hw;
+	struct enetc_vf_state *vf_state;
+
+	if (vf_id < 0)
+		return;
+
+	vf_state = &pf->vf_state[vf_id];
+	mutex_lock(&vf_state->lock);
+}
+
+static void enetc_vf_state_unlock(struct enetc_pf *pf, int vf_id)
+{
+	struct enetc_vf_state *vf_state;
+
+	if (vf_id < 0)
+		return;
+
+	vf_state = &pf->vf_state[vf_id];
+	mutex_unlock(&vf_state->lock);
+}
+
+static void enetc_show_si_mac_hash_filter(struct seq_file *s, int si_id)
+{
+	struct enetc_pf *pf = enetc_si_priv(s->private);
+	struct enetc_hw *hw = &pf->si->hw;
 	u32 hash_h, hash_l;
 
-	hash_l = enetc_port_rd(hw, ENETC4_PSIUMHFR0(i));
-	hash_h = enetc_port_rd(hw, ENETC4_PSIUMHFR1(i));
+	enetc_vf_state_lock(pf, si_id - 1);
+	hash_l = enetc_port_rd(hw, ENETC4_PSIUMHFR0(si_id));
+	hash_h = enetc_port_rd(hw, ENETC4_PSIUMHFR1(si_id));
 	seq_printf(s, "SI %d unicast MAC hash filter: 0x%08x%08x\n",
-		   i, hash_h, hash_l);
+		   si_id, hash_h, hash_l);
 
-	hash_l = enetc_port_rd(hw, ENETC4_PSIMMHFR0(i));
-	hash_h = enetc_port_rd(hw, ENETC4_PSIMMHFR1(i));
+	hash_l = enetc_port_rd(hw, ENETC4_PSIMMHFR0(si_id));
+	hash_h = enetc_port_rd(hw, ENETC4_PSIMMHFR1(si_id));
 	seq_printf(s, "SI %d multicast MAC hash filter: 0x%08x%08x\n",
-		   i, hash_h, hash_l);
+		   si_id, hash_h, hash_l);
+	enetc_vf_state_unlock(pf, si_id - 1);
 }
 
 static int enetc_mac_filter_show(struct seq_file *s, void *data)
@@ -37,7 +61,11 @@ static int enetc_mac_filter_show(struct seq_file *s, void *data)
 	int err = 0;
 	int i;
 
+	/* Prevent concurrent access from causing PSIPMMR to be modified */
+	spin_lock(&pf->si->gen_lock);
 	val = enetc_port_rd(hw, ENETC4_PSIPMMR);
+	spin_unlock(&pf->si->gen_lock);
+
 	for (i = 0; i < num_si; i++) {
 		seq_printf(s, "SI %d Unicast Promiscuous mode: %s\n", i,
 			   str_enabled_disabled(PSIPMMR_SI_MAC_UP(i) & val));
@@ -45,13 +73,12 @@ static int enetc_mac_filter_show(struct seq_file *s, void *data)
 			   str_enabled_disabled(PSIPMMR_SI_MAC_MP(i) & val));
 	}
 
+	rtnl_lock();
 	/* MAC hash filter table */
 	for (i = 0; i < num_si; i++)
 		enetc_show_si_mac_hash_filter(s, i);
 
 	user = &pf->si->ntmp_user;
-	rtnl_lock();
-
 	if (bitmap_empty(user->maft_eid_bitmap, user->maft_num_entries))
 		goto unlock_rtnl;
 
diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h
index 09025e7a2a3aed..e23d8d82d2ba94 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h
@@ -12,6 +12,7 @@
 #define NXP_ENETC_VENDOR_ID		0x1131
 #define NXP_ENETC_PF_DEV_ID		0xe101
 #define NXP_ENETC_PPM_DEV_ID		0xe110
+#define NXP_ENETC_VF_DEV_ID		0xef00
 
 /**********************Station interface registers************************/
 /* Station interface LSO segmentation flag mask register 0/1 */
diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c
index 9bb1004548abc9..71c971618388b4 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc4_pf.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc4_pf.c
@@ -12,11 +12,6 @@
 
 #define ENETC_SI_MAX_RING_NUM	8
 
-#define ENETC_MAC_FILTER_TYPE_UC	BIT(0)
-#define ENETC_MAC_FILTER_TYPE_MC	BIT(1)
-#define ENETC_MAC_FILTER_TYPE_ALL	(ENETC_MAC_FILTER_TYPE_UC | \
-					 ENETC_MAC_FILTER_TYPE_MC)
-
 static void enetc4_get_port_caps(struct enetc_pf *pf)
 {
 	struct enetc_hw *hw = &pf->si->hw;
@@ -137,7 +132,7 @@ static int enetc4_pf_add_maft_entries(struct enetc_pf *pf,
 static void enetc4_pf_set_uc_hash_filter(struct enetc_pf *pf,
 					 struct netdev_hw_addr_list *uc)
 {
-	struct enetc_mac_filter *mac_filter = &pf->mac_filter[UC];
+	struct enetc_mac_filter *mac_filter = &pf->si->mac_filter[UC];
 	struct netdev_hw_addr *ha;
 	u64 hash;
 
@@ -172,7 +167,7 @@ static int enetc4_pf_set_uc_exact_filter(struct enetc_pf *pf,
 
 	err = enetc4_pf_add_maft_entries(pf, uc);
 	if (!err) {
-		enetc_reset_mac_addr_filter(&pf->mac_filter[UC]);
+		enetc_reset_mac_addr_filter(&si->mac_filter[UC]);
 		enetc_set_si_uc_hash_filter(si, 0, 0);
 	}
 
@@ -182,7 +177,7 @@ static int enetc4_pf_set_uc_exact_filter(struct enetc_pf *pf,
 static void enetc4_pf_set_mc_hash_filter(struct enetc_pf *pf,
 					 struct netdev_hw_addr_list *mc)
 {
-	struct enetc_mac_filter *mac_filter = &pf->mac_filter[MC];
+	struct enetc_mac_filter *mac_filter = &pf->si->mac_filter[MC];
 	struct netdev_hw_addr *ha;
 	u64 hash;
 
@@ -217,19 +212,43 @@ static void enetc4_pf_set_mac_filter(struct enetc_pf *pf, int type,
 		enetc4_pf_set_mc_hash_filter(pf, mc);
 }
 
+static void enetc4_pf_vf_flr_handler(struct enetc_pf *pf, int vf_id)
+{
+	struct enetc_vf_state *vf_state;
+	bool uc_promisc, mc_promisc;
+
+	vf_state = &pf->vf_state[vf_id];
+	mutex_lock(&vf_state->lock);
+
+	uc_promisc = !!(vf_state->flags & ENETC_VF_FLAG_UC_PROMISC);
+	mc_promisc = !!(vf_state->flags & ENETC_VF_FLAG_MC_PROMISC);
+
+	spin_lock(&pf->si->gen_lock);
+	enetc_set_si_uc_promisc(pf->si, vf_id + 1, uc_promisc);
+	enetc_set_si_mc_promisc(pf->si, vf_id + 1, mc_promisc);
+	spin_unlock(&pf->si->gen_lock);
+
+	mutex_unlock(&vf_state->lock);
+}
+
 static const struct enetc_pf_ops enetc4_pf_ops = {
 	.set_si_primary_mac = enetc4_pf_set_si_primary_mac,
 	.get_si_primary_mac = enetc4_pf_get_si_primary_mac,
+	.vf_flr_handler = enetc4_pf_vf_flr_handler,
 };
 
 static int enetc4_pf_struct_init(struct enetc_si *si)
 {
 	struct enetc_pf *pf = enetc_si_priv(si);
+	int err;
 
 	pf->si = si;
-	pf->total_vfs = pci_sriov_get_totalvfs(si->pdev);
 	pf->ops = &enetc4_pf_ops;
 
+	err = enetc_init_sriov_resources(pf);
+	if (err)
+		return err;
+
 	enetc4_get_port_caps(pf);
 	enetc4_get_psi_hw_features(si);
 
@@ -524,8 +543,10 @@ static int enetc4_pf_set_rx_mode(struct net_device *ndev,
 		type = ENETC_MAC_FILTER_TYPE_ALL;
 	}
 
+	spin_lock(&si->gen_lock);
 	enetc_set_si_uc_promisc(si, 0, uc_promisc);
 	enetc_set_si_mc_promisc(si, 0, mc_promisc);
+	spin_unlock(&si->gen_lock);
 
 	if (uc_promisc) {
 		enetc_set_si_uc_hash_filter(si, 0, 0);
@@ -574,6 +595,9 @@ static const struct net_device_ops enetc4_ndev_ops = {
 	.ndo_eth_ioctl		= enetc_ioctl,
 	.ndo_hwtstamp_get	= enetc_hwtstamp_get,
 	.ndo_hwtstamp_set	= enetc_hwtstamp_set,
+	.ndo_set_vf_trust	= enetc_pf_set_vf_trust,
+	.ndo_set_vf_mac		= enetc_pf_set_vf_mac,
+	.ndo_get_vf_config	= enetc_pf_get_vf_config,
 };
 
 static struct phylink_pcs *
@@ -894,6 +918,7 @@ static void enetc4_pl_mac_link_up(struct phylink_config *config,
 	enetc4_set_rx_pause(pf, rx_pause);
 	enetc4_mac_tx_enable(pf);
 	enetc4_mac_rx_enable(pf);
+	enetc_pf_notify_vf_link_up(pf);
 }
 
 static void enetc4_pl_mac_link_down(struct phylink_config *config,
@@ -902,6 +927,7 @@ static void enetc4_pl_mac_link_down(struct phylink_config *config,
 {
 	struct enetc_pf *pf = phylink_to_enetc_pf(config);
 
+	enetc_pf_notify_vf_link_down(pf);
 	enetc4_mac_rx_graceful_stop(pf);
 	enetc4_mac_tx_graceful_stop(pf);
 }
@@ -961,6 +987,36 @@ static void enetc4_link_deinit(struct enetc_ndev_priv *priv)
 	enetc_mdiobus_destroy(pf);
 }
 
+static void enetc4_pf_link_status_task(struct work_struct *work)
+{
+	struct enetc_pf *pf = container_of(work, struct enetc_pf,
+					   link_status_task);
+
+	enetc_pf_send_link_status_msg(pf);
+}
+
+static int enetc4_pf_wq_task_init(struct enetc_si *si)
+{
+	struct enetc_pf *pf = enetc_si_priv(si);
+
+	si->workqueue = alloc_ordered_workqueue("enetc-%s-wq", 0,
+						pci_name(si->pdev));
+	if (!si->workqueue)
+		return -ENOMEM;
+
+	INIT_WORK(&pf->link_status_task, enetc4_pf_link_status_task);
+
+	return 0;
+}
+
+static void enetc4_pf_wq_task_destroy(struct enetc_si *si)
+{
+	struct enetc_pf *pf = enetc_si_priv(si);
+
+	disable_work_sync(&pf->link_status_task);
+	destroy_workqueue(si->workqueue);
+}
+
 static int enetc4_pf_netdev_create(struct enetc_si *si)
 {
 	struct device *dev = &si->pdev->dev;
@@ -1001,6 +1057,10 @@ static int enetc4_pf_netdev_create(struct enetc_si *si)
 	if (err)
 		goto err_link_init;
 
+	err = enetc4_pf_wq_task_init(si);
+	if (err)
+		goto err_wq_init;
+
 	err = register_netdev(ndev);
 	if (err) {
 		dev_err(dev, "Failed to register netdev\n");
@@ -1010,6 +1070,8 @@ static int enetc4_pf_netdev_create(struct enetc_si *si)
 	return 0;
 
 err_reg_netdev:
+	enetc4_pf_wq_task_destroy(si);
+err_wq_init:
 	enetc4_link_deinit(priv);
 err_link_init:
 	enetc_free_msix(priv);
@@ -1027,6 +1089,7 @@ static void enetc4_pf_netdev_destroy(struct enetc_si *si)
 	struct net_device *ndev = si->ndev;
 
 	unregister_netdev(ndev);
+	enetc4_pf_wq_task_destroy(si);
 	enetc4_link_deinit(priv);
 	enetc_free_msix(priv);
 	free_netdev(ndev);
@@ -1096,6 +1159,9 @@ static void enetc4_pf_remove(struct pci_dev *pdev)
 	struct enetc_si *si = pci_get_drvdata(pdev);
 	struct enetc_pf *pf = enetc_si_priv(si);
 
+	if (pf->num_vfs)
+		enetc_sriov_configure(pdev, 0);
+
 	enetc_remove_debugfs(si);
 	enetc4_pf_netdev_destroy(si);
 	enetc4_pf_free(pf);
@@ -1113,6 +1179,7 @@ static struct pci_driver enetc4_pf_driver = {
 	.id_table = enetc4_pf_id_table,
 	.probe = enetc4_pf_probe,
 	.remove = enetc4_pf_remove,
+	.sriov_configure = enetc_sriov_configure,
 };
 module_pci_driver(enetc4_pf_driver);
 
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
index 07b7832f24276a..7965dfd06f5f61 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
@@ -859,6 +859,9 @@ static int enetc_get_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *rxnfc,
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 	int i, j;
 
+	if (!is_enetc_rev1(priv->si))
+		return -EOPNOTSUPP;
+
 	switch (rxnfc->cmd) {
 	case ETHTOOL_GRXCLSRLCNT:
 		/* total number of entries */
@@ -903,6 +906,9 @@ static int enetc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *rxnfc)
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 	int err;
 
+	if (!is_enetc_rev1(priv->si))
+		return -EOPNOTSUPP;
+
 	switch (rxnfc->cmd) {
 	case ETHTOOL_SRXCLSRLINS:
 		if (rxnfc->fs.location >= priv->si->num_fs_entries)
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
index 16da732dc5de82..2c9d9042eb0bbf 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
@@ -80,6 +80,14 @@ static inline u32 enetc_vsi_set_msize(u32 size)
 #define ENETC_SIMSGSR_SET_MC(val) ((val) << 16)
 #define ENETC_SIMSGSR_GET_MC(val) ((val) >> 16)
 
+#define ENETC_PSIMSGSR		0x208
+/* n is VF index, which is less than 15 */
+#define  PSIMSGSR_MS(n)		BIT((n) + 1)
+#define  PSIMSGSR_MC		GENMASK(31, 16)
+
+#define ENETC_VSIMSGRR		0x208
+#define  VSIMSGRR_MC		GENMASK(31, 16)
+
 /* SI statistics */
 #define ENETC_SIROCT	0x300
 #define ENETC_SIRFRM	0x308
@@ -103,8 +111,26 @@ static inline u32 enetc_vsi_set_msize(u32 size)
 #define ENETC_SICAPR0	0x900
 #define ENETC_SICAPR1	0x904
 
+#define ENETC_VSIIER	0xa00
+#define  VSIIER_MRIE	BIT(9)
+
+#define ENETC_VSIIDR	0xa08
+#define  VSIIDR_MR	BIT(9)
+
 #define ENETC_PSIIER	0xa00
 #define ENETC_PSIIDR	0xa08
+
+/* VF FLR interrupt mask, n is the active number of VFs.
+ * It is available for ENETC_PSIIER and ENETC_PSIIDR registers.
+ */
+#define ENETC_VFFLR_MASK(n)	\
+	({ typeof(n) _n = (n); (_n) ? GENMASK(16 + (_n), 17) : 0; })
+
+/* VF FLR interrupt bit, n is VF index. It is available
+ * for ENETC_PSIIER and ENETC_PSIIDR registers.
+ */
+#define ENETC_VFFLR_BIT(n)	BIT(17 + (n))
+
 #define ENETC_SITXIDR	0xa18
 #define ENETC_SIRXIDR	0xa28
 #define ENETC_SIMSIVR	0xa30
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_mailbox.h b/drivers/net/ethernet/freescale/enetc/enetc_mailbox.h
index d9677da3898920..6fa66c863748d0 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_mailbox.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_mailbox.h
@@ -66,8 +66,8 @@
  * 2) PSI_TX_control: PSIMSGSR[MC] - for PSI to VSI notification messages
  * (async mode)
  *
- * Note that for some GET messages, there is no COOKIE field, and the CLASS
- * CODE field is expanded to 8 bits.
+ * Note that for some PSI-to-VSI messages, there is no COOKIE field, and the
+ * CLASS CODE field is expanded to 8 bits.
  */
 
 #ifndef __ENETC_MAILBOX_H
@@ -87,10 +87,26 @@
 /* The fileds of PSI-to-VSI message, the message is only 16-bit */
 #define ENETC_PF_MSG_COOKIE			GENMASK(3, 0)
 #define ENETC_PF_MSG_CLASS_CODE			GENMASK(7, 4)
-/* Extend the class code to 8-bit for GET messages without COOKIE */
+/* Extend the class code to 8-bit for PSI-to-VSI messages without COOKIE
+ * The class code for the following messages is 8-bit.
+ * 1. Get IP revision messages
+ * 2. Link status messages
+ * 3. Link speed messages
+ */
 #define ENETC_PF_MSG_CLASS_CODE_U8		GENMASK(7, 0)
 #define ENETC_PF_MSG_CLASS_ID			GENMASK(15, 8)
 
+#define ENETC_MAC_HASH_TABLE_SIZE_64		0
+#define ENETC_MSG_MAC_HASH_SIZE			GENMASK(5, 0)
+#define ENETC_MSG_MAC_TYPE			GENMASK(7, 6)
+#define  ENETC_MAC_FILTER_TYPE_UC		BIT(0)
+#define  ENETC_MAC_FILTER_TYPE_MC		BIT(1)
+#define  ENETC_MAC_FILTER_TYPE_ALL		(ENETC_MAC_FILTER_TYPE_UC | \
+						 ENETC_MAC_FILTER_TYPE_MC)
+
+#define ENETC_MSG_MAC_FLUSH_MACS		BIT(0)
+#define ENETC_MSG_MAC_PROMISC_MODE		BIT(1)
+
 enum enetc_msg_class_id {
 	/* Class ID for PSI-to-VSI messages */
 	ENETC_MSG_CLASS_ID_CMD_SUCCESS		= 1,
@@ -107,20 +123,66 @@ enum enetc_msg_class_id {
 
 	/* Common Class ID for PSI-to-VSI and VSI-to-PSI messages */
 	ENETC_MSG_CLASS_ID_MAC_FILTER		= 0x20,
+	ENETC_MSG_CLASS_ID_LINK_STATUS		= 0x80,
+	ENETC_MSG_CLASS_ID_LINK_SPEED		= 0x81,
 	ENETC_MSG_CLASS_ID_IP_REVISION		= 0xf0,
 };
 
 enum enetc_msg_mac_filter_cmd_id {
 	ENETC_MSG_SET_PRIMARY_MAC,
+	ENETC_MSG_SET_MAC_HASH_TABLE		= 3,
+	ENETC_MSG_SET_MAC_PROMISC_MODE		= 5,
 };
 
 enum enetc_msg_ip_revision_cmd_id {
 	ENETC_MSG_GET_IP_MN			= 1,
 };
 
+enum enetc_msg_link_status_cmd_id {
+	ENETC_MSG_GET_CURRENT_LINK_STATUS,
+	ENETC_MSG_REGISTER_LINK_CHANGE_NOTIFIER,
+	ENETC_MSG_UNREGISTER_LINK_CHANGE_NOTIFIER,
+};
+
+enum enetc_msg_link_speed_cmd_id {
+	ENETC_MSG_GET_CURRENT_LINK_SPEED,
+	/* The following command IDs are not currently supported */
+	ENETC_MSG_REGISTER_SPEED_CHANGE_NOTIFIER,
+	ENETC_MSG_UNREGISTER_SPEED_CHANGE_NOTIFIER,
+};
+
 /* Class-specific error return codes of MAC filter */
 enum enetc_mac_filter_class_code {
 	ENETC_MF_CLASS_CODE_INVALID_MAC,
+	ENETC_MF_CLASS_CODE_INVALID_TYPE	= 4,
+	/* Unicast Filter Is Denied */
+	ENETC_MF_CLASS_CODE_UCF_DENY		= 5,
+};
+
+/* Class-specific notifications/codes of link status */
+#define ENETC_CLASS_CODE_LINK_DOWN		BIT(0)
+#define ENETC_CLASS_CODE_TX_PAUSE_EN		BIT(1)
+
+/* Class-specific notifications/codes of link speed */
+enum enetc_link_speed_class_code {
+	ENETC_MSG_SPEED_UNKNOWN,
+	ENETC_MSG_SPEED_10M_HD,
+	ENETC_MSG_SPEED_10M_FD,
+	ENETC_MSG_SPEED_100M_HD,
+	ENETC_MSG_SPEED_100M_FD,
+	ENETC_MSG_SPEED_1000M,
+	ENETC_MSG_SPEED_2500M,
+	ENETC_MSG_SPEED_5G,
+	/* Do not add enumeration values for any speed greater than
+	 * 5Gbps. For any speed greater than 5Gbps, its speed class
+	 * code should follow the formula below.
+	 *
+	 * SPEED = (link_speed - 5000) / 1000 + ENETC_MSG_SPEED_5G
+	 *
+	 * The unit of link_speed should be Mbps, the max SPEED
+	 * should <= ENETC_MSG_SPEED_MAX.
+	 */
+	ENETC_MSG_SPEED_MAX = 0xff,
 };
 
 struct enetc_msg_swbd {
@@ -158,9 +220,50 @@ struct enetc_msg_mac_exact_filter {
 	struct enetc_mac_addr mac[];
 };
 
+/* message format of class_id 0x20 for hash MAC filter.
+ * cmd_id 0x3: set MAC hash table
+ */
+struct enetc_msg_mac_hash_filter {
+	struct enetc_msg_header hdr;
+	/* bit 0 ~ 5: ENETC_MSG_MAC_HASH_SIZE
+	 * bit 6~7: ENETC_MSG_MAC_TYPE
+	 */
+	u8 sz_type;
+	u8 resv[3];
+	u32 hash_tbl[];
+};
+
+/* message format of class_id 0x20 for MAC promiscuous mode.
+ * cmd_id 0x5: set MAC promiscuous mode
+ */
+struct enetc_msg_mac_promisc_mode {
+	struct enetc_msg_header hdr;
+	/* bit 0: ENETC_MSG_MAC_FLUSH_MACS
+	 * bit 1: ENETC_MSG_MAC_PROMISC_MODE
+	 * bit 6~7: ENETC_MSG_MAC_TYPE
+	 */
+	u8 config;
+	u8 resv[15];
+};
+
 /* The generic message format applies to the following messages:
  * Get IP revision message, class_id 0xf0.
  * cmd_id 1: get IP minor revision
+ *
+ * Link status message, class id 0x80.
+ * cmd_id 0x0: get the current link status
+ * cmd_id 0x1: register link status change notification
+ * cmd_id 0x2: unregister link status change notification
+ *
+ * Link speed message, class_id 0x81.
+ * cmd_id 0x0: get the current link speed. Unlike the link status
+ *   query (class 0x80), this query is only permitted for trusted VFs;
+ *   an untrusted VF receives a permission-deny response. This is
+ *   because the PF must take rtnl_lock() to read the link speed, so
+ *   restricting it to trusted VFs avoids rtnl_lock contention on the
+ *   host from a misbehaving VF.
+ * cmd_id 0x1: register link speed change notification, not supported yet
+ * cmd_id 0x2: unregister link speed change notification, not supported yet
  */
 struct enetc_msg_generic {
 	struct enetc_msg_header hdr;
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_mdio.c b/drivers/net/ethernet/freescale/enetc/enetc_mdio.c
index 998aaa394e9c4c..a90a04a47befdd 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_mdio.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_mdio.c
@@ -24,7 +24,7 @@
 
 #define ENETC_EMDIO_CFG \
 	(MDIO_CFG_HOLD(2) | \
-	 MDIO_CFG_CLKDIV(258) | \
+	 MDIO_CFG_CLKDIV(66) | \
 	 MDIO_CFG_NEG)
 
 #define MDIO_CTL_DEV_ADDR(x)	((x) & 0x1f)
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_msg.c b/drivers/net/ethernet/freescale/enetc/enetc_msg.c
index edc1277bb58604..ff99937815c0a1 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_msg.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_msg.c
@@ -7,24 +7,28 @@
 					   ENETC_MSG_CLASS_ID_CMD_SUCCESS)
 #define ENETC_PF_MSG_NOTSUPP	FIELD_PREP(ENETC_PF_MSG_CLASS_ID, \
 					   ENETC_MSG_CLASS_ID_CMD_NOT_SUPPORT)
-
-static void enetc_msg_disable_mr_int(struct enetc_pf *pf)
+#define ENETC_PF_MSG_PERM_DENY	FIELD_PREP(ENETC_PF_MSG_CLASS_ID, \
+					   ENETC_MSG_CLASS_ID_PERMISSION_DENY)
+#define ENETC_PF_MSG_INV_LEN	FIELD_PREP(ENETC_PF_MSG_CLASS_ID, \
+					   ENETC_MSG_CLASS_ID_INVALID_MSG_LEN)
+#define ENETC_PF_MSG_MF(code)	(FIELD_PREP(ENETC_PF_MSG_CLASS_ID, \
+					    ENETC_MSG_CLASS_ID_MAC_FILTER) | \
+				 FIELD_PREP(ENETC_PF_MSG_CLASS_CODE, (code)))
+
+static void enetc_disable_psiier_interrupts(struct enetc_pf *pf)
 {
 	struct enetc_hw *hw = &pf->si->hw;
-	u32 psiier;
-
-	psiier = enetc_rd(hw, ENETC_PSIIER) & ~ENETC_PSIMR_MASK(pf->num_vfs);
 
-	/* disable MR int source(s) */
-	enetc_wr(hw, ENETC_PSIIER, psiier);
+	enetc_wr(hw, ENETC_PSIIER, 0);
 }
 
-static void enetc_msg_enable_mr_int(struct enetc_pf *pf)
+static void enetc_enable_psiier_interrupts(struct enetc_pf *pf)
 {
+	u32 psiier = ENETC_PSIMR_MASK(pf->num_vfs);
 	struct enetc_hw *hw = &pf->si->hw;
-	u32 psiier;
 
-	psiier = enetc_rd(hw, ENETC_PSIIER) | ENETC_PSIMR_MASK(pf->num_vfs);
+	if (pf->ops->vf_flr_handler)
+		psiier |= ENETC_VFFLR_MASK(pf->num_vfs);
 
 	enetc_wr(hw, ENETC_PSIIER, psiier);
 }
@@ -34,8 +38,8 @@ static irqreturn_t enetc_msg_psi_msix(int irq, void *data)
 	struct enetc_si *si = (struct enetc_si *)data;
 	struct enetc_pf *pf = enetc_si_priv(si);
 
-	enetc_msg_disable_mr_int(pf);
-	schedule_work(&pf->msg_task);
+	enetc_disable_psiier_interrupts(pf);
+	schedule_work(&si->msg_task);
 
 	return IRQ_HANDLED;
 }
@@ -61,31 +65,184 @@ static u16 enetc_msg_set_vf_primary_mac_addr(struct enetc_pf *pf, int vf_id,
 	struct enetc_vf_state *vf_state = &pf->vf_state[vf_id];
 	struct enetc_msg_mac_exact_filter *msg = vf_msg;
 	struct device *dev = &pf->si->pdev->dev;
+	u16 pf_msg = ENETC_PF_MSG_SUCCESS;
 	char *addr = msg->mac[0].addr;
 
+	mutex_lock(&vf_state->lock);
+
+	/* Untrusted VFs cannot set their MAC addresses by the mailbox
+	 * messages.
+	 */
+	if (!(vf_state->flags & ENETC_VF_FLAG_TRUSTED)) {
+		pf_msg = ENETC_PF_MSG_PERM_DENY;
+		goto vf_state_unlock;
+	}
+
 	if (!is_valid_ether_addr(addr)) {
 		dev_err_ratelimited(dev, "VF%d attempted to set invalid MAC\n",
 				    vf_id);
-		return (FIELD_PREP(ENETC_PF_MSG_CLASS_ID,
-				   ENETC_MSG_CLASS_ID_MAC_FILTER) |
-			FIELD_PREP(ENETC_PF_MSG_CLASS_CODE,
-				   ENETC_MF_CLASS_CODE_INVALID_MAC));
+		pf_msg = ENETC_PF_MSG_MF(ENETC_MF_CLASS_CODE_INVALID_MAC);
+		goto vf_state_unlock;
 	}
 
-	mutex_lock(&vf_state->lock);
+	/* PF has higher privileges. If PF has already modified the MAC
+	 * address for VF through .ndo_set_vf_mac() interface, VF is not
+	 * allowed to set its MAC address via mailbox messages, even if
+	 * it is trusted.
+	 */
 	if (vf_state->flags & ENETC_VF_FLAG_PF_SET_MAC) {
-		mutex_unlock(&vf_state->lock);
 		dev_err_ratelimited(dev,
 				    "VF%d attempted to override PF set MAC\n",
 				    vf_id);
-		return FIELD_PREP(ENETC_PF_MSG_CLASS_ID,
-				  ENETC_MSG_CLASS_ID_CMD_NOT_PERMITTED);
+		pf_msg = FIELD_PREP(ENETC_PF_MSG_CLASS_ID,
+				    ENETC_MSG_CLASS_ID_CMD_NOT_PERMITTED);
+		goto vf_state_unlock;
 	}
 
 	enetc_set_si_hw_addr(pf, vf_id + 1, addr);
+
+vf_state_unlock:
 	mutex_unlock(&vf_state->lock);
 
-	return ENETC_PF_MSG_SUCCESS;
+	return pf_msg;
+}
+
+static u16 enetc_msg_set_vf_mac_hash_filter(struct enetc_pf *pf, int vf_id,
+					    void *vf_msg)
+{
+	struct enetc_vf_state *vf_state = &pf->vf_state[vf_id];
+	struct enetc_msg_mac_hash_filter *msg = vf_msg;
+	u16 pf_msg = ENETC_PF_MSG_SUCCESS;
+	struct enetc_si *si = pf->si;
+	int si_id = vf_id + 1;
+	u64 uc_hash, mc_hash;
+	bool trusted;
+	int type;
+
+	/* Currently, hardware only supports 64 bits table size */
+	if (FIELD_GET(ENETC_MSG_MAC_HASH_SIZE, msg->sz_type) !=
+	    ENETC_MAC_HASH_TABLE_SIZE_64)
+		return ENETC_PF_MSG_NOTSUPP;
+
+	mutex_lock(&vf_state->lock);
+
+	/* For an untrusted VF, unicast MAC hash filtering is not permitted.
+	 * For multicast, the MAC hash filter is strictly limited to a maximum
+	 * of 8 bits to satisfy its basic multicast communication requirements
+	 * while preventing potential network abuse.
+	 */
+	trusted = !!(vf_state->flags & ENETC_VF_FLAG_TRUSTED);
+	type = FIELD_GET(ENETC_MSG_MAC_TYPE, msg->sz_type);
+	switch (type) {
+	case ENETC_MAC_FILTER_TYPE_UC:
+		if (!trusted) {
+			pf_msg = ENETC_PF_MSG_PERM_DENY;
+			goto vf_state_unlock;
+		}
+
+		uc_hash = (u64)msg->hash_tbl[1] << 32 | msg->hash_tbl[0];
+		enetc_set_si_uc_hash_filter(si, si_id, uc_hash);
+		break;
+	case ENETC_MAC_FILTER_TYPE_MC:
+		mc_hash = (u64)msg->hash_tbl[1] << 32 | msg->hash_tbl[0];
+		if (!trusted &&
+		    hweight64(mc_hash) > ENETC_VF_MC_HASH_BITS_MAX) {
+			pf_msg = ENETC_PF_MSG_PERM_DENY;
+			goto vf_state_unlock;
+		}
+
+		enetc_set_si_mc_hash_filter(si, si_id, mc_hash);
+		break;
+	case ENETC_MAC_FILTER_TYPE_ALL:
+		if (!msg->hdr.len) {
+			pf_msg = ENETC_PF_MSG_INV_LEN;
+			goto vf_state_unlock;
+		}
+
+		uc_hash = (u64)msg->hash_tbl[1] << 32 | msg->hash_tbl[0];
+		mc_hash = (u64)msg->hash_tbl[3] << 32 | msg->hash_tbl[2];
+
+		if (!trusted &&
+		    (hweight64(mc_hash) <= ENETC_VF_MC_HASH_BITS_MAX)) {
+			enetc_set_si_mc_hash_filter(si, si_id, mc_hash);
+			pf_msg = ENETC_PF_MSG_MF(ENETC_MF_CLASS_CODE_UCF_DENY);
+			goto vf_state_unlock;
+		}
+
+		if (!trusted) {
+			pf_msg = ENETC_PF_MSG_PERM_DENY;
+			goto vf_state_unlock;
+		}
+
+		enetc_set_si_uc_hash_filter(si, si_id, uc_hash);
+		enetc_set_si_mc_hash_filter(si, si_id, mc_hash);
+		break;
+	default:
+		pf_msg = ENETC_PF_MSG_MF(ENETC_MF_CLASS_CODE_INVALID_TYPE);
+	}
+
+vf_state_unlock:
+	mutex_unlock(&vf_state->lock);
+
+	return pf_msg;
+}
+
+static u16 enetc_msg_set_vf_mac_promisc_mode(struct enetc_pf *pf, int vf_id,
+					     void *vf_msg)
+{
+	struct enetc_vf_state *vf_state = &pf->vf_state[vf_id];
+	struct enetc_msg_mac_promisc_mode *msg = vf_msg;
+	u16 pf_msg = ENETC_PF_MSG_SUCCESS;
+	struct enetc_si *si = pf->si;
+	bool promisc, flush_macs;
+	int si_id = vf_id + 1;
+	int type;
+
+	flush_macs = !!(msg->config & ENETC_MSG_MAC_FLUSH_MACS);
+	type = FIELD_GET(ENETC_MSG_MAC_TYPE, msg->config);
+	if (!type)
+		return ENETC_PF_MSG_MF(ENETC_MF_CLASS_CODE_INVALID_TYPE);
+
+	mutex_lock(&vf_state->lock);
+
+	promisc = !!(msg->config & ENETC_MSG_MAC_PROMISC_MODE);
+	if (promisc && !(vf_state->flags & ENETC_VF_FLAG_TRUSTED)) {
+		pf_msg = ENETC_PF_MSG_PERM_DENY;
+		goto vf_state_unlock;
+	}
+
+	if (type & ENETC_MAC_FILTER_TYPE_UC) {
+		if (promisc)
+			vf_state->flags |= ENETC_VF_FLAG_UC_PROMISC;
+		else
+			vf_state->flags &= ~ENETC_VF_FLAG_UC_PROMISC;
+	}
+
+	if (type & ENETC_MAC_FILTER_TYPE_MC) {
+		if (promisc)
+			vf_state->flags |= ENETC_VF_FLAG_MC_PROMISC;
+		else
+			vf_state->flags &= ~ENETC_VF_FLAG_MC_PROMISC;
+	}
+
+	spin_lock(&si->gen_lock);
+	if (type & ENETC_MAC_FILTER_TYPE_UC)
+		enetc_set_si_uc_promisc(si, si_id, promisc);
+
+	if (type & ENETC_MAC_FILTER_TYPE_MC)
+		enetc_set_si_mc_promisc(si, si_id, promisc);
+	spin_unlock(&si->gen_lock);
+
+	if ((type & ENETC_MAC_FILTER_TYPE_UC) && flush_macs)
+		enetc_set_si_uc_hash_filter(si, si_id, 0);
+
+	if ((type & ENETC_MAC_FILTER_TYPE_MC) && flush_macs)
+		enetc_set_si_mc_hash_filter(si, si_id, 0);
+
+vf_state_unlock:
+	mutex_unlock(&vf_state->lock);
+
+	return pf_msg;
 }
 
 static u16 enetc_msg_handle_mac_filter(struct enetc_pf *pf, int vf_id,
@@ -96,6 +253,10 @@ static u16 enetc_msg_handle_mac_filter(struct enetc_pf *pf, int vf_id,
 	switch (msg_hdr->cmd_id) {
 	case ENETC_MSG_SET_PRIMARY_MAC:
 		return enetc_msg_set_vf_primary_mac_addr(pf, vf_id, vf_msg);
+	case ENETC_MSG_SET_MAC_HASH_TABLE:
+		return enetc_msg_set_vf_mac_hash_filter(pf, vf_id, vf_msg);
+	case ENETC_MSG_SET_MAC_PROMISC_MODE:
+		return enetc_msg_set_vf_mac_promisc_mode(pf, vf_id, vf_msg);
 	default:
 		return ENETC_PF_MSG_NOTSUPP;
 	}
@@ -116,6 +277,191 @@ static u16 enetc_msg_handle_ip_revision(struct enetc_pf *pf, void *vf_msg)
 	}
 }
 
+static void enetc_pf_reply_msg(struct enetc_hw *hw, int vf_id, u16 pf_msg)
+{
+	/* w1c to clear the corresponding VF MR bit */
+	enetc_wr(hw, ENETC_PSIIDR, ENETC_PSIMR_BIT(vf_id));
+	enetc_wr(hw, ENETC_PSIMSGRR, ENETC_SIMSGSR_SET_MC(pf_msg) |
+		 ENETC_PSIMR_BIT(vf_id));
+}
+
+static u16 enetc_build_link_status_msg(struct enetc_ndev_priv *priv,
+				       bool link_up)
+{
+	u8 status = 0;
+
+	if (link_up) {
+		if (test_bit(ENETC_RXBDR_CM, &priv->flags))
+			status |= ENETC_CLASS_CODE_TX_PAUSE_EN;
+	} else {
+		status |= ENETC_CLASS_CODE_LINK_DOWN;
+	}
+
+	return FIELD_PREP(ENETC_PF_MSG_CLASS_ID,
+			  ENETC_MSG_CLASS_ID_LINK_STATUS) |
+	       FIELD_PREP(ENETC_PF_MSG_CLASS_CODE_U8, status);
+}
+
+static void enetc_msg_get_link_status(struct enetc_pf *pf, int vf_id)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(pf->si->ndev);
+	struct enetc_si *si = pf->si;
+	u16 pf_msg;
+
+	spin_lock(&si->gen_lock);
+	pf_msg = enetc_build_link_status_msg(priv, pf->link_up);
+	enetc_pf_reply_msg(&si->hw, vf_id, pf_msg);
+	spin_unlock(&si->gen_lock);
+}
+
+static void enetc_msg_register_link_status_notifier(struct enetc_pf *pf,
+						    int vf_id)
+{
+	struct enetc_si *si = pf->si;
+
+	spin_lock(&si->gen_lock);
+	enetc_pf_reply_msg(&si->hw, vf_id, ENETC_PF_MSG_SUCCESS);
+
+	/* SR-IOV is being disabled if pf->sriov_enabled is false, so no
+	 * need to set link_status_ms_mask and notify the link status.
+	 */
+	if (!pf->sriov_enabled) {
+		spin_unlock(&si->gen_lock);
+		return;
+	}
+
+	pf->link_status_ms_mask |= PSIMSGSR_MS(vf_id);
+	spin_unlock(&si->gen_lock);
+
+	/* Notify VF the current link status */
+	queue_work(si->workqueue, &pf->link_status_task);
+}
+
+static void enetc_msg_unregister_link_status_notifier(struct enetc_pf *pf,
+						      int vf_id)
+{
+	spin_lock(&pf->si->gen_lock);
+	pf->link_status_ms_mask &= ~PSIMSGSR_MS(vf_id);
+	enetc_pf_reply_msg(&pf->si->hw, vf_id, ENETC_PF_MSG_SUCCESS);
+	spin_unlock(&pf->si->gen_lock);
+}
+
+static u16 enetc_msg_handle_link_status(struct enetc_pf *pf, int vf_id,
+					void *vf_msg)
+{
+	struct enetc_msg_header *msg_hdr = vf_msg;
+
+	switch (msg_hdr->cmd_id) {
+	case ENETC_MSG_GET_CURRENT_LINK_STATUS:
+		/* Currently, this message is intended only for
+		 * DPDK-owned VFs.
+		 */
+		enetc_msg_get_link_status(pf, vf_id);
+		break;
+	case ENETC_MSG_REGISTER_LINK_CHANGE_NOTIFIER:
+		enetc_msg_register_link_status_notifier(pf, vf_id);
+		break;
+	case ENETC_MSG_UNREGISTER_LINK_CHANGE_NOTIFIER:
+		enetc_msg_unregister_link_status_notifier(pf, vf_id);
+		break;
+	default:
+		return ENETC_PF_MSG_NOTSUPP;
+	}
+
+	return 0;
+}
+
+static u16 enetc_build_link_speed_msg(int speed, int duplex)
+{
+	u32 speed_code = ENETC_MSG_SPEED_UNKNOWN;
+
+	switch (speed) {
+	case SPEED_10:
+		if (duplex == DUPLEX_HALF)
+			speed_code = ENETC_MSG_SPEED_10M_HD;
+		else if (duplex == DUPLEX_FULL)
+			speed_code = ENETC_MSG_SPEED_10M_FD;
+		break;
+	case SPEED_100:
+		if (duplex == DUPLEX_HALF)
+			speed_code = ENETC_MSG_SPEED_100M_HD;
+		else if (duplex == DUPLEX_FULL)
+			speed_code = ENETC_MSG_SPEED_100M_FD;
+		break;
+	case SPEED_1000:
+		speed_code = ENETC_MSG_SPEED_1000M;
+		break;
+	case SPEED_2500:
+		speed_code = ENETC_MSG_SPEED_2500M;
+		break;
+	case SPEED_5000:
+		speed_code = ENETC_MSG_SPEED_5G;
+		break;
+	default:
+		if (speed < SPEED_5000)
+			break;
+
+		speed_code = (speed - SPEED_5000) / SPEED_1000 +
+			     ENETC_MSG_SPEED_5G;
+		if (speed_code > ENETC_MSG_SPEED_MAX)
+			speed_code = ENETC_MSG_SPEED_UNKNOWN;
+	}
+
+	return FIELD_PREP(ENETC_PF_MSG_CLASS_ID,
+			  ENETC_MSG_CLASS_ID_LINK_SPEED) |
+	       FIELD_PREP(ENETC_PF_MSG_CLASS_CODE_U8, speed_code);
+}
+
+static u16 enetc_msg_get_link_speed(struct enetc_pf *pf, int vf_id)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(pf->si->ndev);
+	struct enetc_vf_state *vf_state = &pf->vf_state[vf_id];
+	struct ethtool_link_ksettings link_info = {};
+
+	/* A malicious or malfunctioning VM could potentially spam these
+	 * messages in a tight loop causing global rtnl_lock contention,
+	 * which may severely starve other processes on the host that
+	 * require rtnl_lock for routine network configuration, resulting
+	 * in a system-wide control-plane denial of service. Therefore,
+	 * we expect the VF query for link speed to be trusted. There's no
+	 * need to consider the transition from trusted to untrusted here,
+	 * as this won't cause rtnl_lock() to be called frequently.
+	 */
+	mutex_lock(&vf_state->lock);
+	if (!(vf_state->flags & ENETC_VF_FLAG_TRUSTED)) {
+		mutex_unlock(&vf_state->lock);
+
+		return ENETC_PF_MSG_PERM_DENY;
+	}
+	mutex_unlock(&vf_state->lock);
+
+	rtnl_lock();
+	phylink_ethtool_ksettings_get(priv->phylink, &link_info);
+	rtnl_unlock();
+
+	return enetc_build_link_speed_msg(link_info.base.speed,
+					  link_info.base.duplex);
+}
+
+static u16 enetc_msg_handle_link_speed(struct enetc_pf *pf, int vf_id,
+				       void *vf_msg)
+{
+	struct enetc_msg_header *msg_hdr = vf_msg;
+
+	switch (msg_hdr->cmd_id) {
+	case ENETC_MSG_GET_CURRENT_LINK_SPEED:
+		return enetc_msg_get_link_speed(pf, vf_id);
+	case ENETC_MSG_REGISTER_SPEED_CHANGE_NOTIFIER:
+	case ENETC_MSG_UNREGISTER_SPEED_CHANGE_NOTIFIER:
+	default:
+		return ENETC_PF_MSG_NOTSUPP;
+	}
+}
+
+/* If *pf_msg is set to 0, it means that PF has responded to VF in
+ * enetc_msg_handle_rxmsg() through enetc_pf_reply_msg(), which also
+ * clears the corresponding VF MR bit in PSIIDR.
+ */
 static void enetc_msg_handle_rxmsg(struct enetc_pf *pf, int vf_id,
 				   u16 *pf_msg)
 {
@@ -128,8 +474,7 @@ static void enetc_msg_handle_rxmsg(struct enetc_pf *pf, int vf_id,
 	if (msg_size > ENETC_DEFAULT_MSG_SIZE) {
 		dev_err_ratelimited(dev,
 				    "Invalid message size: %u\n", msg_size);
-		*pf_msg = FIELD_PREP(ENETC_PF_MSG_CLASS_ID,
-				     ENETC_MSG_CLASS_ID_INVALID_MSG_LEN);
+		*pf_msg = ENETC_PF_MSG_INV_LEN;
 		return;
 	}
 
@@ -147,6 +492,14 @@ static void enetc_msg_handle_rxmsg(struct enetc_pf *pf, int vf_id,
 	}
 
 	memcpy(msg, msg_swbd->vaddr, msg_size);
+	msg_hdr = (struct enetc_msg_header *)msg;
+
+	/* Check message length whether is changed */
+	if (ENETC_MSG_SIZE(msg_hdr->len) != msg_size) {
+		*pf_msg = ENETC_PF_MSG_INV_LEN;
+		goto free_msg;
+	}
+
 	if (!enetc_msg_check_crc16(msg, msg_size)) {
 		dev_err_ratelimited(dev, "VSI to PSI Message CRC16 error\n");
 		*pf_msg = FIELD_PREP(ENETC_PF_MSG_CLASS_ID,
@@ -157,7 +510,6 @@ static void enetc_msg_handle_rxmsg(struct enetc_pf *pf, int vf_id,
 
 	/* Default to not supported */
 	*pf_msg = ENETC_PF_MSG_NOTSUPP;
-	msg_hdr = (struct enetc_msg_header *)msg;
 
 	/* Currently, asynchronous actions are not supported */
 	if (FIELD_GET(ENETC_VF_MSG_COOKIE, msg_hdr->cookie)) {
@@ -191,6 +543,12 @@ static void enetc_msg_handle_rxmsg(struct enetc_pf *pf, int vf_id,
 	case ENETC_MSG_CLASS_ID_IP_REVISION:
 		*pf_msg = enetc_msg_handle_ip_revision(pf, msg);
 		break;
+	case ENETC_MSG_CLASS_ID_LINK_STATUS:
+		*pf_msg = enetc_msg_handle_link_status(pf, vf_id, msg);
+		break;
+	case ENETC_MSG_CLASS_ID_LINK_SPEED:
+		*pf_msg = enetc_msg_handle_link_speed(pf, vf_id, msg);
+		break;
 	default:
 		dev_err_ratelimited(dev,
 				    "Unsupported message class ID: 0x%x\n",
@@ -201,21 +559,46 @@ static void enetc_msg_handle_rxmsg(struct enetc_pf *pf, int vf_id,
 	kfree(msg);
 }
 
-static void enetc_msg_task(struct work_struct *work)
+static void enetc_vf_flr_handler(struct enetc_pf *pf)
 {
-	struct enetc_pf *pf = container_of(work, struct enetc_pf, msg_task);
-	u32 mr_mask = ENETC_PSIMR_MASK(pf->num_vfs);
+	u32 flr_mask = ENETC_VFFLR_MASK(pf->num_vfs);
 	struct enetc_hw *hw = &pf->si->hw;
-	u32 mr_status;
+	u32 flr_status;
+
+	if (!pf->ops->vf_flr_handler)
+		return;
+
+	flr_status = enetc_rd(hw, ENETC_PSIIDR) & flr_mask;
+	if (!flr_status)
+		return;
+
+	for (int i = 0; i < pf->num_vfs; i++) {
+		if (!(ENETC_VFFLR_BIT(i) & flr_status))
+			continue;
+
+		/* Clear FLR interrupt status, W1C */
+		enetc_wr(hw, ENETC_PSIIDR, ENETC_VFFLR_BIT(i));
+		pf->ops->vf_flr_handler(pf, i);
+	}
+}
+
+static void enetc_msg_task(struct work_struct *work)
+{
+	struct enetc_si *si = container_of(work, struct enetc_si, msg_task);
+	struct enetc_pf *pf = enetc_si_priv(si);
+	struct enetc_hw *hw = &si->hw;
+	u32 mr_status, mr_mask;
 	int i;
 
+	enetc_vf_flr_handler(pf);
+
+	mr_mask = ENETC_PSIMR_MASK(pf->num_vfs);
 	mr_status = (enetc_rd(hw, ENETC_PSIMSGRR) & mr_mask) |
 		    (enetc_rd(hw, ENETC_PSIIDR) & mr_mask);
 	if (!mr_status)
 		goto out;
 
 	for (i = 0; i < pf->num_vfs; i++) {
-		u32 psimsgrr;
 		u16 msg_code;
 
 		if (!(ENETC_PSIMR_BIT(i) & mr_status))
@@ -223,16 +606,18 @@ static void enetc_msg_task(struct work_struct *work)
 
 		enetc_msg_handle_rxmsg(pf, i, &msg_code);
 
-		/* w1c to clear the corresponding VF MR bit */
-		enetc_wr(hw, ENETC_PSIIDR, ENETC_PSIMR_BIT(i));
+		/* If msg_code is 0, it means that PF has responded to VF
+		 * in enetc_msg_handle_rxmsg() through enetc_pf_reply_msg(),
+		 * which also clears the corresponding VF MR bit in PSIIDR.
+		 */
+		if (!msg_code)
+			continue;
 
-		psimsgrr = ENETC_SIMSGSR_SET_MC(msg_code);
-		psimsgrr |= ENETC_PSIMR_BIT(i); /* w1c */
-		enetc_wr(hw, ENETC_PSIMSGRR, psimsgrr);
+		enetc_pf_reply_msg(hw, i, msg_code);
 	}
 
 out:
-	enetc_msg_enable_mr_int(pf);
+	enetc_enable_psiier_interrupts(pf);
 }
 
 /* Init */
@@ -291,13 +676,13 @@ static int enetc_msg_psi_init(struct enetc_pf *pf)
 	}
 
 	/* initialize PSI mailbox */
-	INIT_WORK(&pf->msg_task, enetc_msg_task);
+	INIT_WORK(&si->msg_task, enetc_msg_task);
 
 	/* register message passing interrupt handler */
-	snprintf(pf->msg_int_name, sizeof(pf->msg_int_name), "%s-vfmsg",
+	snprintf(si->msg_int_name, sizeof(si->msg_int_name), "%s-vfmsg",
 		 si->ndev->name);
 	vector = pci_irq_vector(si->pdev, ENETC_SI_INT_IDX);
-	err = request_irq(vector, enetc_msg_psi_msix, 0, pf->msg_int_name, si);
+	err = request_irq(vector, enetc_msg_psi_msix, 0, si->msg_int_name, si);
 	if (err) {
 		dev_err(&si->pdev->dev,
 			"PSI messaging: request_irq() failed!\n");
@@ -307,8 +692,8 @@ static int enetc_msg_psi_init(struct enetc_pf *pf)
 	/* set one IRQ entry for PSI message receive notification (SI int) */
 	enetc_wr(&si->hw, ENETC_SIMSIVR, ENETC_SI_INT_IDX);
 
-	/* enable MR interrupts */
-	enetc_msg_enable_mr_int(pf);
+	/* enable PSIIER interrupts */
+	enetc_enable_psiier_interrupts(pf);
 
 	return 0;
 
@@ -319,24 +704,60 @@ static int enetc_msg_psi_init(struct enetc_pf *pf)
 	return err;
 }
 
+static void enetc_msg_clear_vf_config(struct enetc_pf *pf, int vf_id)
+{
+	struct enetc_vf_state *vf_state = &pf->vf_state[vf_id];
+	struct enetc_si *si = pf->si;
+	int si_id = vf_id + 1;
+
+	/* For ENETC v1, we only support setting the VF's MAC address via
+	 * VSI-to-PSI messages, so there is no configuration to clear.
+	 */
+	if (is_enetc_rev1(si))
+		return;
+
+	mutex_lock(&vf_state->lock);
+
+	/* VF may set these flags by mailbox messages, so need to clear these
+	 * flags when enetc_msg_psi_free() is called. PF-set flags (TRUSTED,
+	 * PF_SET_MAC) are not cleared, because these flags are unrelated to
+	 * whether SR-IOV is enabled or disabled.
+	 */
+	vf_state->flags &= ~(ENETC_VF_FLAG_UC_PROMISC |
+			     ENETC_VF_FLAG_MC_PROMISC);
+
+	spin_lock(&si->gen_lock);
+	vf_state->msg_fail_cnt = 0;
+	enetc_set_si_uc_promisc(si, si_id, false);
+	enetc_set_si_mc_promisc(si, si_id, false);
+	spin_unlock(&si->gen_lock);
+
+	enetc_set_si_uc_hash_filter(si, si_id, 0);
+	enetc_set_si_mc_hash_filter(si, si_id, 0);
+
+	mutex_unlock(&vf_state->lock);
+}
+
 static void enetc_msg_psi_free(struct enetc_pf *pf)
 {
 	struct enetc_si *si = pf->si;
 	int i;
 
-	/* disable MR interrupts */
-	enetc_msg_disable_mr_int(pf);
+	/* disable PSIIER interrupts */
+	enetc_disable_psiier_interrupts(pf);
 
 	/* de-register message passing interrupt handler */
 	free_irq(pci_irq_vector(si->pdev, ENETC_SI_INT_IDX), si);
 
-	cancel_work_sync(&pf->msg_task);
+	cancel_work_sync(&si->msg_task);
 
-	/* MR interrupts may be re-enabled by workqueue */
-	enetc_msg_disable_mr_int(pf);
+	/* PSIIER interrupts may be re-enabled by workqueue */
+	enetc_disable_psiier_interrupts(pf);
 
-	for (i = 0; i < pf->num_vfs; i++)
+	for (i = 0; i < pf->num_vfs; i++) {
 		enetc_msg_free_mbx(si, i);
+		enetc_msg_clear_vf_config(pf, i);
+	}
 }
 
 int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs)
@@ -346,6 +767,11 @@ int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	int err;
 
 	if (!num_vfs) {
+		spin_lock(&si->gen_lock);
+		pf->sriov_enabled = false;
+		pf->link_status_ms_mask = 0;
+		spin_unlock(&si->gen_lock);
+
 		pci_disable_sriov(pdev);
 		enetc_msg_psi_free(pf);
 		pf->num_vfs = 0;
@@ -358,6 +784,11 @@ int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 			goto err_msg_psi;
 		}
 
+		/* As PCI SR-IOV is not enabled at the moment, there is no
+		 * concurrent access to sriov_enabled. So no need to use
+		 * gen_lock to protect sriov_enabled.
+		 */
+		pf->sriov_enabled = true;
 		err = pci_enable_sriov(pdev, num_vfs);
 		if (err) {
 			dev_err(&pdev->dev, "pci_enable_sriov err %d\n", err);
@@ -368,6 +799,15 @@ int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	return num_vfs;
 
 err_en_sriov:
+	/* If pci_enable_sriov() fails after partially creating VFs, a VF
+	 * driver that successfully bound to one of the created VFs could
+	 * have sent a registration message, setting its bit in
+	 * link_status_ms_mask.
+	 */
+	spin_lock(&si->gen_lock);
+	pf->sriov_enabled = false;
+	pf->link_status_ms_mask = 0;
+	spin_unlock(&si->gen_lock);
 	enetc_msg_psi_free(pf);
 err_msg_psi:
 	pf->num_vfs = 0;
@@ -375,3 +815,114 @@ int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	return err;
 }
 EXPORT_SYMBOL_GPL(enetc_sriov_configure);
+
+void enetc_pf_send_link_status_msg(struct enetc_pf *pf)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(pf->si->ndev);
+	u16 pf_msg, ms_mask, new_ms_msk, ms_status;
+	struct enetc_si *si = pf->si;
+	int retry_num = 0;
+
+retry:
+	spin_lock(&si->gen_lock);
+	ms_mask = pf->link_status_ms_mask;
+	/* VFs have unregistered link status notification, return directly  */
+	if (!ms_mask)
+		goto unlock;
+
+	/* The MS bit is set, indicating that the corresponding VF has not
+	 * read the last message, PF cannot send new message to the VF. To
+	 * avoid sending messages to such a VF, the bit corresponding to VF
+	 * is cleared from ms_mask. Because the MS bit can only be written
+	 * as 1, writing a 0 has no effect. Writing a 1 when the bit is
+	 * already set is undefined.
+	 */
+	ms_status = enetc_rd(&si->hw, ENETC_PSIMSGSR) & 0xfffe;
+	if ((ms_mask & ms_status) && retry_num++ < 200) {
+		spin_unlock(&si->gen_lock);
+		/* Wait VFs to handle the last message */
+		usleep_range(1000, 1020);
+		goto retry;
+	}
+
+	/* None of the relevant VFs have processed the previous message, and
+	 * the PF has tried 200 times. This situation indicates that VF has
+	 * malfunctioned.
+	 */
+	new_ms_msk = ms_mask & (~ms_status);
+	if (!new_ms_msk) {
+		dev_err_ratelimited(&si->pdev->dev,
+				    "All registered VFs (MS: 0x%x) are busy\n",
+				    ms_mask);
+		goto ms_status_check;
+	}
+
+	if (new_ms_msk != ms_mask)
+		dev_warn_ratelimited(&si->pdev->dev,
+				     "Failed to notify link status to VFs (MS: 0x%x)\n",
+				     ms_mask ^ new_ms_msk);
+
+	pf_msg = enetc_build_link_status_msg(priv, pf->link_up);
+	enetc_wr(&si->hw, ENETC_PSIMSGSR,
+		 FIELD_PREP(PSIMSGSR_MC, pf_msg) | new_ms_msk);
+
+ms_status_check:
+	/* If the PF fails to send messages to the corresponding VF for 10
+	 * consecutive times, clear that VF's bit in link_status_ms_mask.
+	 */
+	for (int i = 0; i < pf->num_vfs; i++) {
+		struct enetc_vf_state *vf_state = &pf->vf_state[i];
+
+		if (!(PSIMSGSR_MS(i) & ms_mask))
+			continue;
+
+		if (!(PSIMSGSR_MS(i) & ms_status)) {
+			vf_state->msg_fail_cnt = 0;
+			continue;
+		}
+
+		if (vf_state->msg_fail_cnt++ < 10)
+			continue;
+
+		vf_state->msg_fail_cnt = 0;
+		pf->link_status_ms_mask &= ~PSIMSGSR_MS(i);
+		dev_warn_ratelimited(&si->pdev->dev,
+				     "Clear VF%d's link status MS bit\n", i);
+	}
+
+unlock:
+	spin_unlock(&si->gen_lock);
+}
+EXPORT_SYMBOL_GPL(enetc_pf_send_link_status_msg);
+
+static void enetc_pf_notify_vf_link_status(struct enetc_pf *pf,
+					   bool link_up)
+{
+	struct enetc_si *si = pf->si;
+
+	/* Currently we do not add link status message support for ENETC v1 */
+	if (!pf->total_vfs || is_enetc_rev1(si))
+		return;
+
+	spin_lock(&si->gen_lock);
+	pf->link_up = link_up;
+	if (!pf->link_status_ms_mask) {
+		spin_unlock(&si->gen_lock);
+		return;
+	}
+	spin_unlock(&si->gen_lock);
+
+	queue_work(si->workqueue, &pf->link_status_task);
+}
+
+void enetc_pf_notify_vf_link_up(struct enetc_pf *pf)
+{
+	enetc_pf_notify_vf_link_status(pf, true);
+}
+EXPORT_SYMBOL_GPL(enetc_pf_notify_vf_link_up);
+
+void enetc_pf_notify_vf_link_down(struct enetc_pf *pf)
+{
+	enetc_pf_notify_vf_link_status(pf, false);
+}
+EXPORT_SYMBOL_GPL(enetc_pf_notify_vf_link_down);
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
index 55c07c528f2232..d77a07cece28bc 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
@@ -60,10 +60,9 @@ static void enetc_add_mac_addr_em_filter(struct enetc_mac_filter *filter,
 	filter->mac_addr_cnt++;
 }
 
-static void enetc_sync_mac_filters(struct enetc_pf *pf)
+static void enetc_sync_mac_filters(struct enetc_si *si)
 {
-	struct enetc_mac_filter *f = pf->mac_filter;
-	struct enetc_si *si = pf->si;
+	struct enetc_mac_filter *f = si->mac_filter;
 	int i, pos;
 
 	pos = EMETC_MAC_ADDR_FILT_RES;
@@ -115,9 +114,9 @@ static void enetc_sync_mac_filters(struct enetc_pf *pf)
 static void enetc_pf_set_rx_mode(struct net_device *ndev)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
-	struct enetc_pf *pf = enetc_si_priv(priv->si);
 	bool uprom = false, mprom = false;
 	struct enetc_mac_filter *filter;
+	struct enetc_si *si = priv->si;
 	struct netdev_hw_addr *ha;
 	bool em;
 
@@ -133,7 +132,7 @@ static void enetc_pf_set_rx_mode(struct net_device *ndev)
 	/* first 2 filter entries belong to PF */
 	if (!uprom) {
 		/* Update unicast filters */
-		filter = &pf->mac_filter[UC];
+		filter = &si->mac_filter[UC];
 		enetc_reset_mac_addr_filter(filter);
 
 		em = (netdev_uc_count(ndev) == 1);
@@ -149,7 +148,7 @@ static void enetc_pf_set_rx_mode(struct net_device *ndev)
 
 	if (!mprom) {
 		/* Update multicast filters */
-		filter = &pf->mac_filter[MC];
+		filter = &si->mac_filter[MC];
 		enetc_reset_mac_addr_filter(filter);
 
 		netdev_for_each_mc_addr(ha, ndev) {
@@ -162,10 +161,10 @@ static void enetc_pf_set_rx_mode(struct net_device *ndev)
 
 	if (!uprom || !mprom)
 		/* update PF entries */
-		enetc_sync_mac_filters(pf);
+		enetc_sync_mac_filters(si);
 
-	enetc_set_si_uc_promisc(priv->si, 0, uprom);
-	enetc_set_si_mc_promisc(priv->si, 0, mprom);
+	enetc_set_si_uc_promisc(si, 0, uprom);
+	enetc_set_si_mc_promisc(si, 0, mprom);
 }
 
 static void enetc_set_loopback(struct net_device *ndev, bool en)
@@ -191,33 +190,12 @@ static void enetc_set_loopback(struct net_device *ndev, bool en)
 	}
 }
 
-static int enetc_pf_set_vf_mac(struct net_device *ndev, int vf, u8 *mac)
-{
-	struct enetc_ndev_priv *priv = netdev_priv(ndev);
-	struct enetc_pf *pf = enetc_si_priv(priv->si);
-	struct enetc_vf_state *vf_state;
-
-	if (vf >= pf->total_vfs)
-		return -EINVAL;
-
-	if (!is_valid_ether_addr(mac))
-		return -EADDRNOTAVAIL;
-
-	vf_state = &pf->vf_state[vf];
-
-	mutex_lock(&vf_state->lock);
-	vf_state->flags |= ENETC_VF_FLAG_PF_SET_MAC;
-	enetc_pf_set_primary_mac_addr(&priv->si->hw, vf + 1, mac);
-	mutex_unlock(&vf_state->lock);
-
-	return 0;
-}
-
 static int enetc_pf_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan,
 				u8 qos, __be16 proto)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 	struct enetc_pf *pf = enetc_si_priv(priv->si);
+	struct enetc_vf_state *vf_state;
 
 	if (priv->si->errata & ENETC_ERR_VLAN_ISOL)
 		return -EOPNOTSUPP;
@@ -230,6 +208,17 @@ static int enetc_pf_set_vf_vlan(struct net_device *ndev, int vf, u16 vlan,
 		return -EPROTONOSUPPORT;
 
 	enetc_set_isol_vlan(&priv->si->hw, vf + 1, vlan, qos);
+
+	vf_state = &pf->vf_state[vf];
+	mutex_lock(&vf_state->lock);
+	/* Currently only C-tags is supported, so tpid is always 0,
+	 * which indicates ETH_P_8021Q.
+	 */
+	vf_state->tpid = 0;
+	vf_state->qos = qos;
+	vf_state->vid = vlan;
+	mutex_unlock(&vf_state->lock);
+
 	return 0;
 }
 
@@ -237,6 +226,7 @@ static int enetc_pf_set_vf_spoofchk(struct net_device *ndev, int vf, bool en)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 	struct enetc_pf *pf = enetc_si_priv(priv->si);
+	struct enetc_vf_state *vf_state;
 	u32 cfgr;
 
 	if (vf >= pf->total_vfs)
@@ -246,6 +236,16 @@ static int enetc_pf_set_vf_spoofchk(struct net_device *ndev, int vf, bool en)
 	cfgr = (cfgr & ~ENETC_PSICFGR0_ASE) | (en ? ENETC_PSICFGR0_ASE : 0);
 	enetc_port_wr(&priv->si->hw, ENETC_PSICFGR0(vf + 1), cfgr);
 
+	vf_state = &pf->vf_state[vf];
+	mutex_lock(&vf_state->lock);
+
+	if (en)
+		vf_state->flags |= ENETC_VF_FLAG_SPOOFCHK;
+	else
+		vf_state->flags &= ~ENETC_VF_FLAG_SPOOFCHK;
+
+	mutex_unlock(&vf_state->lock);
+
 	return 0;
 }
 
@@ -488,6 +488,7 @@ static const struct net_device_ops enetc_ndev_ops = {
 	.ndo_set_rx_mode	= enetc_pf_set_rx_mode,
 	.ndo_vlan_rx_add_vid	= enetc_vlan_rx_add_vid,
 	.ndo_vlan_rx_kill_vid	= enetc_vlan_rx_del_vid,
+	.ndo_set_vf_trust	= enetc_pf_set_vf_trust,
 	.ndo_set_vf_mac		= enetc_pf_set_vf_mac,
 	.ndo_set_vf_vlan	= enetc_pf_set_vf_vlan,
 	.ndo_set_vf_spoofchk	= enetc_pf_set_vf_spoofchk,
@@ -498,6 +499,7 @@ static const struct net_device_ops enetc_ndev_ops = {
 	.ndo_xdp_xmit		= enetc_xdp_xmit,
 	.ndo_hwtstamp_get	= enetc_hwtstamp_get,
 	.ndo_hwtstamp_set	= enetc_hwtstamp_set,
+	.ndo_get_vf_config	= enetc_pf_get_vf_config,
 };
 
 static struct phylink_pcs *
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.h b/drivers/net/ethernet/freescale/enetc/enetc_pf.h
index 56d23a8a11a063..1f92d16d8ca810 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_pf.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.h
@@ -6,14 +6,24 @@
 
 #define ENETC_PF_NUM_RINGS	8
 #define ENETC_VLAN_HT_SIZE	64
+#define ENETC_VF_MC_HASH_BITS_MAX	8 /* For untrusted VFs */
 
 enum enetc_vf_flags {
 	ENETC_VF_FLAG_PF_SET_MAC	= BIT(0),
+	ENETC_VF_FLAG_TRUSTED		= BIT(1),
+	ENETC_VF_FLAG_UC_PROMISC	= BIT(2),
+	ENETC_VF_FLAG_MC_PROMISC	= BIT(3),
+	ENETC_VF_FLAG_SPOOFCHK		= BIT(4),
 };
 
 struct enetc_vf_state {
 	struct mutex lock; /* Prevent concurrent access */
 	enum enetc_vf_flags flags;
+	/* Number of consecutive failures to send PF-to-VF messages */
+	int msg_fail_cnt;
+	u8 tpid; /* SI-based VLAN TPID (0: 0x8100, 1: 0x88a8) */
+	u8 qos; /* SI-based VLAN QOS (priority) bits */
+	u16 vid; /* SI-based VLAN ID */
 };
 
 struct enetc_port_caps {
@@ -30,6 +40,7 @@ struct enetc_pf_ops {
 	struct phylink_pcs *(*create_pcs)(struct enetc_pf *pf, struct mii_bus *bus);
 	void (*destroy_pcs)(struct phylink_pcs *pcs);
 	int (*enable_psfp)(struct enetc_ndev_priv *priv);
+	void (*vf_flr_handler)(struct enetc_pf *pf, int vf_id);
 };
 
 struct enetc_pf {
@@ -37,12 +48,7 @@ struct enetc_pf {
 	int num_vfs; /* number of active VFs, after sriov_init */
 	int total_vfs; /* max number of VFs, set for PF at probe */
 	struct enetc_vf_state *vf_state;
-
-	struct enetc_mac_filter mac_filter[MADDR_TYPE];
-
 	struct enetc_msg_swbd *rxmsg;
-	struct work_struct msg_task;
-	char msg_int_name[ENETC_INT_NAME_MAX];
 
 	DECLARE_BITMAP(vlan_ht_filter, ENETC_VLAN_HT_SIZE);
 	DECLARE_BITMAP(active_vlans, VLAN_N_VID);
@@ -56,6 +62,11 @@ struct enetc_pf {
 
 	struct enetc_port_caps caps;
 	const struct enetc_pf_ops *ops;
+
+	struct work_struct link_status_task;
+	bool sriov_enabled;
+	bool link_up;
+	u16 link_status_ms_mask;
 };
 
 #define phylink_to_enetc_pf(config) \
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c
index d32a195a04c97e..8206884294a477 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.c
@@ -151,26 +151,44 @@ void enetc_set_si_uc_hash_filter(struct enetc_si *si, int si_id, u64 hash)
 }
 EXPORT_SYMBOL_GPL(enetc_set_si_uc_hash_filter);
 
-void enetc_set_si_mc_hash_filter(struct enetc_si *si, int si_id, u64 hash)
+static void enetc_get_psimmhfr_offsets(struct enetc_si *si, int si_id,
+				       int *psimmhfr0, int *psimmhfr1)
 {
-	int psimmhfr0_off, psimmhfr1_off;
-	struct enetc_hw *hw = &si->hw;
-
 	if (is_enetc_rev1(si)) {
 		bool err = si->errata & ENETC_ERR_UCMCSWP;
 
-		psimmhfr0_off = ENETC_PSIMMHFR0(si_id, err);
-		psimmhfr1_off = ENETC_PSIMMHFR1(si_id);
+		*psimmhfr0 = ENETC_PSIMMHFR0(si_id, err);
+		*psimmhfr1 = ENETC_PSIMMHFR1(si_id);
 	} else {
-		psimmhfr0_off = ENETC4_PSIMMHFR0(si_id);
-		psimmhfr1_off = ENETC4_PSIMMHFR1(si_id);
+		*psimmhfr0 = ENETC4_PSIMMHFR0(si_id);
+		*psimmhfr1 = ENETC4_PSIMMHFR1(si_id);
 	}
+}
+
+void enetc_set_si_mc_hash_filter(struct enetc_si *si, int si_id, u64 hash)
+{
+	int psimmhfr0_off, psimmhfr1_off;
+	struct enetc_hw *hw = &si->hw;
 
+	enetc_get_psimmhfr_offsets(si, si_id, &psimmhfr0_off, &psimmhfr1_off);
 	enetc_port_wr(hw, psimmhfr0_off, lower_32_bits(hash));
 	enetc_port_wr(hw, psimmhfr1_off, upper_32_bits(hash));
 }
 EXPORT_SYMBOL_GPL(enetc_set_si_mc_hash_filter);
 
+static u64 enetc_get_si_mc_hash_filter(struct enetc_si *si, int si_id)
+{
+	int psimmhfr0_off, psimmhfr1_off;
+	struct enetc_hw *hw = &si->hw;
+	u32 hash_h, hash_l;
+
+	enetc_get_psimmhfr_offsets(si, si_id, &psimmhfr0_off, &psimmhfr1_off);
+	hash_l = enetc_port_rd(hw, psimmhfr0_off);
+	hash_h = enetc_port_rd(hw, psimmhfr1_off);
+
+	return ((u64)hash_h << 32) | hash_l;
+}
+
 void enetc_set_si_vlan_promisc(struct enetc_si *si, int si_id, bool promisc)
 {
 	struct enetc_hw *hw = &si->hw;
@@ -586,5 +604,117 @@ int enetc_init_sriov_resources(struct enetc_pf *pf)
 }
 EXPORT_SYMBOL_GPL(enetc_init_sriov_resources);
 
+int enetc_pf_set_vf_trust(struct net_device *ndev, int vf, bool setting)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_pf *pf = enetc_si_priv(priv->si);
+	struct enetc_vf_state *vf_state;
+	struct enetc_si *si = priv->si;
+	int si_id = vf + 1;
+
+	if (vf >= pf->total_vfs)
+		return -EINVAL;
+
+	vf_state = &pf->vf_state[vf];
+	mutex_lock(&vf_state->lock);
+
+	if (setting) {
+		vf_state->flags |= ENETC_VF_FLAG_TRUSTED;
+	} else {
+		u64 hash;
+
+		vf_state->flags &= ~(ENETC_VF_FLAG_TRUSTED |
+				     ENETC_VF_FLAG_UC_PROMISC |
+				     ENETC_VF_FLAG_MC_PROMISC);
+
+		/* For ENETC v1, we only support setting the VF's MAC address
+		 * via VSI-to-PSI messages. Unicast and multicast promiscuous
+		 * mode and hash filters are not supported, so there is no need
+		 * to clear these configurations.
+		 */
+		if (is_enetc_rev1(si))
+			goto vf_state_unlock;
+
+		/* Disable unicast and multicast promiscuous modes */
+		spin_lock(&si->gen_lock);
+		enetc_set_si_uc_promisc(si, si_id, false);
+		enetc_set_si_mc_promisc(si, si_id, false);
+		spin_unlock(&si->gen_lock);
+
+		/* Clear unicast hash filter */
+		enetc_set_si_uc_hash_filter(si, si_id, 0);
+
+		/* Clear multicast hash filter if its set bits exceed
+		 * ENETC_VF_MC_HASH_BITS_MAX.
+		 */
+		hash = enetc_get_si_mc_hash_filter(si, si_id);
+		if (hweight64(hash) > ENETC_VF_MC_HASH_BITS_MAX)
+			enetc_set_si_mc_hash_filter(si, si_id, 0);
+	}
+
+vf_state_unlock:
+	mutex_unlock(&vf_state->lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(enetc_pf_set_vf_trust);
+
+int enetc_pf_set_vf_mac(struct net_device *ndev, int vf, u8 *mac)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_pf *pf = enetc_si_priv(priv->si);
+	struct enetc_vf_state *vf_state;
+
+	if (vf >= pf->total_vfs)
+		return -EINVAL;
+
+	if (!is_valid_ether_addr(mac))
+		return -EADDRNOTAVAIL;
+
+	vf_state = &pf->vf_state[vf];
+
+	mutex_lock(&vf_state->lock);
+	vf_state->flags |= ENETC_VF_FLAG_PF_SET_MAC;
+	enetc_set_si_hw_addr(pf, vf + 1, mac);
+	mutex_unlock(&vf_state->lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(enetc_pf_set_vf_mac);
+
+int enetc_pf_get_vf_config(struct net_device *ndev, int vf,
+			   struct ifla_vf_info *ivi)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_pf *pf = enetc_si_priv(priv->si);
+	struct enetc_vf_state *vf_state;
+
+	if (vf >= pf->total_vfs)
+		return -EINVAL;
+
+	vf_state = &pf->vf_state[vf];
+	mutex_lock(&vf_state->lock);
+
+	ivi->vf = vf;
+	/* ENETC v4 has not support spoofchk yet */
+	if (is_enetc_rev1(priv->si))
+		ivi->spoofchk = !!(vf_state->flags & ENETC_VF_FLAG_SPOOFCHK);
+
+	ivi->trusted = !!(vf_state->flags & ENETC_VF_FLAG_TRUSTED);
+	enetc_get_si_hw_addr(pf, vf + 1, ivi->mac);
+
+	if (vf_state->vid) {
+		ivi->vlan = vf_state->vid;
+		ivi->qos = vf_state->qos;
+		ivi->vlan_proto = vf_state->tpid ? htons(ETH_P_8021AD) :
+						   htons(ETH_P_8021Q);
+	}
+
+	mutex_unlock(&vf_state->lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(enetc_pf_get_vf_config);
+
 MODULE_DESCRIPTION("NXP ENETC PF common functionality driver");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h
index 8243ce0de57f64..c9eed879d5a3fa 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_pf_common.h
@@ -22,6 +22,10 @@ void enetc_set_si_mc_promisc(struct enetc_si *si, int si_id, bool promisc);
 void enetc_set_si_uc_hash_filter(struct enetc_si *si, int si_id, u64 hash);
 void enetc_set_si_mc_hash_filter(struct enetc_si *si, int si_id, u64 hash);
 void enetc_set_si_vlan_promisc(struct enetc_si *si, int si_id, bool promisc);
+int enetc_pf_set_vf_trust(struct net_device *ndev, int vf, bool setting);
+int enetc_pf_set_vf_mac(struct net_device *ndev, int vf, u8 *mac);
+int enetc_pf_get_vf_config(struct net_device *ndev, int vf,
+			   struct ifla_vf_info *ivi);
 
 static inline u16 enetc_get_ip_revision(struct enetc_hw *hw)
 {
@@ -30,9 +34,24 @@ static inline u16 enetc_get_ip_revision(struct enetc_hw *hw)
 
 #if IS_ENABLED(CONFIG_PCI_IOV)
 int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs);
+void enetc_pf_send_link_status_msg(struct enetc_pf *pf);
+void enetc_pf_notify_vf_link_up(struct enetc_pf *pf);
+void enetc_pf_notify_vf_link_down(struct enetc_pf *pf);
 #else
 static inline int enetc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 {
 	return 0;
 }
+
+static inline void enetc_pf_send_link_status_msg(struct enetc_pf *pf)
+{
+}
+
+static inline void enetc_pf_notify_vf_link_up(struct enetc_pf *pf)
+{
+}
+
+static inline void enetc_pf_notify_vf_link_down(struct enetc_pf *pf)
+{
+}
 #endif
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_vf.c b/drivers/net/ethernet/freescale/enetc/enetc_vf.c
index 7dcb4a0246f528..f70b6d4290f21d 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_vf.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_vf.c
@@ -48,6 +48,10 @@ static void enetc_msg_dma_free(struct device *dev, struct enetc_msg_swbd *msg)
 	}
 }
 
+/*
+ * Returns: 0 on success, a negative errno on failure, or the positive IP
+ * minor revision for an ENETC_MSG_GET_IP_MN request.
+ */
 static int enetc_msg_vsi_send(struct enetc_si *si, struct enetc_msg_swbd *msg)
 {
 	struct device *dev = &si->pdev->dev;
@@ -107,8 +111,12 @@ static int enetc_msg_vsi_send(struct enetc_si *si, struct enetc_msg_swbd *msg)
 		case ENETC_MSG_CLASS_ID_CMD_TIMEOUT:
 			err = -ETIME;
 			break;
-		case ENETC_MSG_CLASS_ID_INVALID_MSG_LEN:
 		case ENETC_MSG_CLASS_ID_MAC_FILTER:
+			if (FIELD_GET(ENETC_PF_MSG_CLASS_CODE, pf_msg) ==
+			    ENETC_MF_CLASS_CODE_UCF_DENY)
+				return -EACCES;
+			fallthrough;
+		case ENETC_MSG_CLASS_ID_INVALID_MSG_LEN:
 			err = -EINVAL;
 			break;
 		case ENETC_MSG_CLASS_ID_CMD_NOT_PERMITTED:
@@ -131,6 +139,37 @@ static int enetc_msg_vsi_send(struct enetc_si *si, struct enetc_msg_swbd *msg)
 	return err;
 }
 
+static int enetc_msg_link_status_notifier(struct enetc_si *si, bool reg)
+{
+	struct device *dev = &si->pdev->dev;
+	struct enetc_msg_swbd msg_swbd;
+	u8 cmd_id;
+
+	msg_swbd.size = ALIGN(sizeof(struct enetc_msg_generic),
+			      ENETC_MSG_ALIGN);
+	msg_swbd.vaddr = dma_alloc_coherent(dev, msg_swbd.size,
+					    &msg_swbd.dma, GFP_KERNEL);
+	if (!msg_swbd.vaddr)
+		return -ENOMEM;
+
+	cmd_id = reg ? ENETC_MSG_REGISTER_LINK_CHANGE_NOTIFIER :
+		       ENETC_MSG_UNREGISTER_LINK_CHANGE_NOTIFIER;
+	enetc_msg_fill_common_hdr(&msg_swbd, ENETC_MSG_CLASS_ID_LINK_STATUS,
+				  cmd_id, 0, 0);
+
+	return enetc_msg_vsi_send(si, &msg_swbd);
+}
+
+static int enetc_vf_reg_link_status_notifier(struct enetc_si *si)
+{
+	return enetc_msg_link_status_notifier(si, true);
+}
+
+static int enetc_vf_unreg_link_status_notifier(struct enetc_si *si)
+{
+	return enetc_msg_link_status_notifier(si, false);
+}
+
 static int enetc_msg_vsi_set_primary_mac_addr(struct enetc_ndev_priv *priv,
 					      struct sockaddr *saddr)
 {
@@ -209,6 +248,193 @@ static int enetc_vf_setup_tc(struct net_device *ndev, enum tc_setup_type type,
 	}
 }
 
+static int enetc_vf_set_mac_promisc(struct enetc_si *si, int type,
+				    bool promisc, bool flush_macs)
+{
+	struct enetc_msg_mac_promisc_mode *msg;
+	struct device *dev = &si->pdev->dev;
+	struct enetc_msg_swbd msg_swbd;
+
+	if (!(type & ENETC_MAC_FILTER_TYPE_ALL))
+		return -EINVAL;
+
+	msg_swbd.size = ALIGN(sizeof(*msg), ENETC_MSG_ALIGN);
+	msg_swbd.vaddr = dma_alloc_coherent(dev, msg_swbd.size,
+					    &msg_swbd.dma, GFP_KERNEL);
+	if (!msg_swbd.vaddr)
+		return -ENOMEM;
+
+	msg = (struct enetc_msg_mac_promisc_mode *)msg_swbd.vaddr;
+	msg->config = FIELD_PREP(ENETC_MSG_MAC_TYPE,
+				 type & ENETC_MAC_FILTER_TYPE_ALL);
+	msg->config |= FIELD_PREP(ENETC_MSG_MAC_PROMISC_MODE, promisc);
+	msg->config |= FIELD_PREP(ENETC_MSG_MAC_FLUSH_MACS, flush_macs);
+	enetc_msg_fill_common_hdr(&msg_swbd, ENETC_MSG_CLASS_ID_MAC_FILTER,
+				  ENETC_MSG_SET_MAC_PROMISC_MODE, 0, 0);
+
+	return enetc_msg_vsi_send(si, &msg_swbd);
+}
+
+static int enetc_vf_set_mac_hash_filter(struct enetc_si *si,
+					struct netdev_hw_addr_list *uc,
+					struct netdev_hw_addr_list *mc)
+{
+	struct enetc_msg_mac_hash_filter *msg;
+	struct enetc_mac_filter *mac_filter;
+	struct device *dev = &si->pdev->dev;
+	struct enetc_msg_swbd msg_swbd;
+	struct netdev_hw_addr *ha;
+	int mac_filter_type = 0;
+	u32 tbl_cnt = 0;
+	u32 msg_size;
+	int i = 0;
+
+	if (!uc && !mc)
+		return 0;
+
+	if (uc) {
+		tbl_cnt += 2;
+		mac_filter_type |= ENETC_MAC_FILTER_TYPE_UC;
+	}
+
+	if (mc) {
+		tbl_cnt += 2;
+		mac_filter_type |= ENETC_MAC_FILTER_TYPE_MC;
+	}
+
+	msg_size = struct_size(msg, hash_tbl, tbl_cnt);
+	msg_swbd.size = ALIGN(msg_size, ENETC_MSG_ALIGN);
+	msg_swbd.vaddr = dma_alloc_coherent(dev, msg_swbd.size,
+					    &msg_swbd.dma, GFP_KERNEL);
+	if (!msg_swbd.vaddr)
+		return -ENOMEM;
+
+	msg = (struct enetc_msg_mac_hash_filter *)msg_swbd.vaddr;
+	msg->sz_type = FIELD_PREP(ENETC_MSG_MAC_TYPE, mac_filter_type);
+	msg->sz_type |= FIELD_PREP(ENETC_MSG_MAC_HASH_SIZE,
+				   ENETC_MAC_HASH_TABLE_SIZE_64);
+
+	if (uc) {
+		mac_filter = &si->mac_filter[UC];
+		enetc_reset_mac_addr_filter(mac_filter);
+		netdev_hw_addr_list_for_each(ha, uc)
+			enetc_add_mac_addr_ht_filter(mac_filter, ha->addr);
+
+		bitmap_to_arr32(&msg->hash_tbl[i], mac_filter->mac_hash_table,
+				ENETC_MADDR_HASH_TBL_SZ);
+		i += 2;
+	}
+
+	if (mc) {
+		mac_filter = &si->mac_filter[MC];
+		enetc_reset_mac_addr_filter(mac_filter);
+		netdev_hw_addr_list_for_each(ha, mc)
+			enetc_add_mac_addr_ht_filter(mac_filter, ha->addr);
+
+		bitmap_to_arr32(&msg->hash_tbl[i], mac_filter->mac_hash_table,
+				ENETC_MADDR_HASH_TBL_SZ);
+	}
+
+	enetc_msg_fill_common_hdr(&msg_swbd, ENETC_MSG_CLASS_ID_MAC_FILTER,
+				  ENETC_MSG_SET_MAC_HASH_TABLE, 0, 0);
+
+	return enetc_msg_vsi_send(si, &msg_swbd);
+}
+
+static int enetc_vf_enable_iff_promisc(struct enetc_si *si)
+{
+	int err;
+
+	err = enetc_vf_set_mac_promisc(si, ENETC_MAC_FILTER_TYPE_ALL,
+				       true, true);
+	if (err)
+		dev_err(&si->pdev->dev,
+			"Failed to enable promiscuous mode, err: %pe\n",
+			ERR_PTR(err));
+
+	return err;
+}
+
+static int enetc_vf_disable_iff_promisc(struct enetc_si *si,
+					struct netdev_hw_addr_list *uc,
+					struct netdev_hw_addr_list *mc)
+{
+	int err;
+
+	err = enetc_vf_set_mac_hash_filter(si, uc, mc);
+	if (err) {
+		dev_err_once(&si->pdev->dev,
+			     "Failed to set MAC hash filters, err: %pe\n",
+			     ERR_PTR(err));
+		return err;
+	}
+
+	err = enetc_vf_set_mac_promisc(si, ENETC_MAC_FILTER_TYPE_ALL,
+				       false, false);
+	if (err)
+		dev_err_once(&si->pdev->dev,
+			     "Failed to disable promiscuous mode, err: %pe\n",
+			     ERR_PTR(err));
+
+	return err;
+}
+
+static int enetc_vf_enable_iff_allmulti(struct enetc_si *si,
+					struct netdev_hw_addr_list *uc)
+{
+	int err;
+
+	err = enetc_vf_set_mac_promisc(si, ENETC_MAC_FILTER_TYPE_MC,
+				       true, true);
+	if (err) {
+		dev_err(&si->pdev->dev,
+			"Failed to enable multicast promiscuous mode, err: %pe\n",
+			ERR_PTR(err));
+		return err;
+	}
+
+	err = enetc_vf_set_mac_hash_filter(si, uc, NULL);
+	if (err) {
+		dev_err(&si->pdev->dev,
+			"Failed to set unicast filter, err: %pe\n",
+		ERR_PTR(err));
+		return err;
+	}
+
+	err = enetc_vf_set_mac_promisc(si, ENETC_MAC_FILTER_TYPE_UC,
+				       false, false);
+	if (err)
+		dev_err(&si->pdev->dev,
+			"Failed to disable unicast promiscuous mode, err: %pe\n",
+		ERR_PTR(err));
+
+	return err;
+}
+
+static int enetc_vf_set_rx_mode(struct net_device *ndev,
+				struct netdev_hw_addr_list *uc,
+				struct netdev_hw_addr_list *mc)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_si *si = priv->si;
+	int err;
+
+	if (ndev->flags & IFF_PROMISC)
+		err = enetc_vf_enable_iff_promisc(si);
+	else if (ndev->flags & IFF_ALLMULTI)
+		err = enetc_vf_enable_iff_allmulti(si, uc);
+	else
+		err = enetc_vf_disable_iff_promisc(si, uc, mc);
+
+	/* If the error code is -EOPNOTSUPP or -EACCES or -EPERM, return 0
+	 * directly to avoid meaningless retries.
+	 */
+	if (err == -EOPNOTSUPP || err == -EACCES || err == -EPERM)
+		return 0;
+
+	return err;
+}
+
 /* Probing/ Init */
 static const struct net_device_ops enetc_ndev_ops = {
 	.ndo_open		= enetc_open,
@@ -223,6 +449,20 @@ static const struct net_device_ops enetc_ndev_ops = {
 	.ndo_hwtstamp_set	= enetc_hwtstamp_set,
 };
 
+static const struct net_device_ops enetc4_ndev_ops = {
+	.ndo_open		= enetc_open,
+	.ndo_stop		= enetc_close,
+	.ndo_start_xmit		= enetc_xmit,
+	.ndo_get_stats		= enetc_get_stats,
+	.ndo_set_mac_address	= enetc_vf_set_mac_addr,
+	.ndo_set_features	= enetc_vf_set_features,
+	.ndo_eth_ioctl		= enetc_ioctl,
+	.ndo_setup_tc		= enetc_vf_setup_tc,
+	.ndo_hwtstamp_get	= enetc_hwtstamp_get,
+	.ndo_hwtstamp_set	= enetc_hwtstamp_set,
+	.ndo_set_rx_mode_async	= enetc_vf_set_rx_mode,
+};
+
 static void enetc_vf_get_revision(struct enetc_si *si)
 {
 	int ip_mn;
@@ -276,15 +516,146 @@ static void enetc_vf_netdev_setup(struct enetc_si *si, struct net_device *ndev,
 	ndev->vlan_features = NETIF_F_SG | NETIF_F_HW_CSUM |
 			      NETIF_F_TSO | NETIF_F_TSO6;
 
+	if (!is_enetc_rev1(si))
+		ndev->priv_flags |= IFF_UNICAST_FLT;
+
 	if (si->num_rss) {
 		ndev->hw_features |= NETIF_F_RXHASH;
 		ndev->features |= NETIF_F_RXHASH;
 	}
 
+	if (si->drvdata->tx_csum)
+		priv->active_offloads |= ENETC_F_TXCSUM;
+
+	if (si->hw_features & ENETC_SI_F_LSO)
+		priv->active_offloads |= ENETC_F_LSO;
+
 	/* pick up primary MAC address from SI */
 	enetc_load_primary_mac_addr(&si->hw, ndev);
 }
 
+static void enetc_vf_enable_mr_int(struct enetc_si *si)
+{
+	if (is_enetc_rev1(si))
+		return;
+
+	enetc_wr(&si->hw, ENETC_VSIIER, VSIIER_MRIE);
+}
+
+static void enetc_vf_disable_mr_int(struct enetc_si *si)
+{
+	if (is_enetc_rev1(si))
+		return;
+
+	enetc_wr(&si->hw, ENETC_VSIIER, 0);
+}
+
+static void enetc_vf_msg_handle_link_status(struct enetc_si *si, u8 status)
+{
+	bool tx_pause = !!(status & ENETC_CLASS_CODE_TX_PAUSE_EN);
+	bool link_down = !!(status & ENETC_CLASS_CODE_LINK_DOWN);
+	struct enetc_ndev_priv *priv = netdev_priv(si->ndev);
+	struct net_device *ndev = si->ndev;
+
+	rtnl_lock();
+	if (!netif_running(ndev))
+		goto unlock_rtnl;
+
+	if (link_down) {
+		if (netif_carrier_ok(ndev)) {
+			netif_carrier_off(ndev);
+			netdev_info(ndev, "Link is Down\n");
+		}
+
+		goto unlock_rtnl;
+	}
+
+	/* Link is up */
+	enetc_set_congestion_mode(priv, tx_pause);
+
+	if (!netif_carrier_ok(ndev)) {
+		netif_carrier_on(ndev);
+		netdev_info(ndev, "Link is Up, tx pause %s\n",
+			    tx_pause ? "on" : "off");
+	}
+
+unlock_rtnl:
+	rtnl_unlock();
+}
+
+static void enetc_vf_msg_task(struct work_struct *work)
+{
+	struct enetc_si *si = container_of(work, struct enetc_si, msg_task);
+	struct enetc_hw *hw = &si->hw;
+	u8 class_id, class_code;
+	u16 pf_msg;
+
+	/* W1C to clear the message received interrupt event */
+	enetc_wr(hw, ENETC_VSIIDR, VSIIDR_MR);
+
+	/* Reading VSIMSGRR retrieves the message data and acknowledges to
+	 * the PF that the message was received and another message can be
+	 * sent.
+	 */
+	pf_msg = FIELD_GET(VSIMSGRR_MC, enetc_rd(hw, ENETC_VSIMSGRR));
+	class_id = FIELD_GET(ENETC_PF_MSG_CLASS_ID, pf_msg);
+
+	switch (class_id) {
+	case ENETC_MSG_CLASS_ID_LINK_STATUS:
+		class_code = FIELD_GET(ENETC_PF_MSG_CLASS_CODE_U8, pf_msg);
+		enetc_vf_msg_handle_link_status(si, class_code);
+		break;
+	default:
+		dev_err(&si->pdev->dev,
+			"Unsupported Message Class ID (0x%02x) from PF\n",
+			class_id);
+	}
+
+	enetc_vf_enable_mr_int(si);
+}
+
+static irqreturn_t enetc_vf_msg_msix_handler(int irq, void *data)
+{
+	struct enetc_si *si = (struct enetc_si *)data;
+
+	enetc_vf_disable_mr_int(si);
+	queue_work(si->workqueue, &si->msg_task);
+
+	return IRQ_HANDLED;
+}
+
+static int enetc_vf_register_msg_msix(struct enetc_si *si)
+{
+	int irq, err;
+
+	if (is_enetc_rev1(si))
+		return 0;
+
+	snprintf(si->msg_int_name, sizeof(si->msg_int_name), "%s-pfmsg",
+		 pci_name(si->pdev));
+	irq = pci_irq_vector(si->pdev, ENETC_SI_INT_IDX);
+	err = request_irq(irq, enetc_vf_msg_msix_handler, 0,
+			  si->msg_int_name, si);
+	if (err) {
+		dev_err(&si->pdev->dev,
+			"VF messaging: request_irq() failed!\n");
+		return err;
+	}
+
+	/* set one IRQ entry for PSI-to-VSI messaging */
+	enetc_wr(&si->hw, ENETC_SIMSIVR, ENETC_SI_INT_IDX);
+
+	return 0;
+}
+
+static void enetc_vf_free_msg_msix(struct enetc_si *si)
+{
+	if (is_enetc_rev1(si))
+		return;
+
+	free_irq(pci_irq_vector(si->pdev, ENETC_SI_INT_IDX), si);
+}
+
 static const struct enetc_si_ops enetc_vsi_ops = {
 	.get_rss_table = enetc_get_rss_table,
 	.set_rss_table = enetc_set_rss_table,
@@ -292,6 +663,46 @@ static const struct enetc_si_ops enetc_vsi_ops = {
 	.teardown_cbdr = enetc_teardown_cbdr,
 };
 
+static const struct enetc_si_ops enetc4_vsi_ops = {
+	.get_rss_table = enetc4_get_rss_table,
+	.set_rss_table = enetc4_set_rss_table,
+	.setup_cbdr = enetc4_setup_cbdr,
+	.teardown_cbdr = enetc4_teardown_cbdr,
+	.vf_reg_link_status_notifier = enetc_vf_reg_link_status_notifier,
+	.vf_unreg_link_status_notifier = enetc_vf_unreg_link_status_notifier,
+};
+
+static int enetc_vf_wq_task_init(struct enetc_si *si)
+{
+	if (is_enetc_rev1(si))
+		return 0;
+
+	si->workqueue = alloc_ordered_workqueue("enetc-%s-wq", WQ_MEM_RECLAIM,
+						pci_name(si->pdev));
+	if (!si->workqueue)
+		return -ENOMEM;
+
+	INIT_WORK(&si->msg_task, enetc_vf_msg_task);
+
+	return 0;
+}
+
+static void enetc_vf_wq_task_destroy(struct enetc_si *si)
+{
+	if (!si->workqueue)
+		return;
+
+	disable_work_sync(&si->msg_task);
+
+	/* Disable the MR interrupt */
+	enetc_vf_disable_mr_int(si);
+	enetc_wr(&si->hw, ENETC_VSIIDR, VSIIDR_MR);
+	/* Reading VSIMSGRR to clear the MS bit on the PF's side */
+	enetc_rd(&si->hw, ENETC_VSIMSGRR);
+
+	destroy_workqueue(si->workqueue);
+}
+
 static int enetc_vf_probe(struct pci_dev *pdev,
 			  const struct pci_device_id *ent)
 {
@@ -307,7 +718,11 @@ static int enetc_vf_probe(struct pci_dev *pdev,
 
 	si = pci_get_drvdata(pdev);
 	enetc_vf_get_revision(si);
-	si->ops = &enetc_vsi_ops;
+	if (is_enetc_rev1(si))
+		si->ops = &enetc_vsi_ops;
+	else
+		si->ops = &enetc4_vsi_ops;
+
 	err = enetc_get_driver_data(si);
 	if (err) {
 		dev_err_probe(&pdev->dev, err,
@@ -331,7 +746,10 @@ static int enetc_vf_probe(struct pci_dev *pdev,
 		goto err_alloc_netdev;
 	}
 
-	enetc_vf_netdev_setup(si, ndev, &enetc_ndev_ops);
+	if (is_enetc_rev1(si))
+		enetc_vf_netdev_setup(si, ndev, &enetc_ndev_ops);
+	else
+		enetc_vf_netdev_setup(si, ndev, &enetc4_ndev_ops);
 
 	priv = netdev_priv(ndev);
 
@@ -359,15 +777,32 @@ static int enetc_vf_probe(struct pci_dev *pdev,
 		goto err_alloc_msix;
 	}
 
+	err = enetc_vf_wq_task_init(si);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init workqueue\n");
+		goto err_wq_init;
+	}
+
+	err = enetc_vf_register_msg_msix(si);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to register msg irq\n");
+		goto err_register_msg_msix;
+	}
+
+	netif_carrier_off(ndev);
+	/* Enable message received interrupt */
+	enetc_vf_enable_mr_int(si);
 	err = register_netdev(ndev);
 	if (err)
 		goto err_reg_netdev;
 
-	netif_carrier_off(ndev);
-
 	return 0;
 
 err_reg_netdev:
+	enetc_vf_free_msg_msix(si);
+err_register_msg_msix:
+	enetc_vf_wq_task_destroy(si);
+err_wq_init:
 	enetc_free_msix(priv);
 err_config_si:
 err_alloc_msix:
@@ -395,6 +830,13 @@ static void enetc_vf_remove(struct pci_dev *pdev)
 	priv = netdev_priv(si->ndev);
 	unregister_netdev(si->ndev);
 
+	/* Disable promiscuous mode and clear MAC filters */
+	if (!is_enetc_rev1(si))
+		enetc_vf_set_mac_promisc(si, ENETC_MAC_FILTER_TYPE_ALL,
+					 false, true);
+
+	enetc_vf_free_msg_msix(si);
+	enetc_vf_wq_task_destroy(si);
 	enetc_free_msix(priv);
 
 	enetc_free_si_resources(priv);
@@ -409,6 +851,7 @@ static void enetc_vf_remove(struct pci_dev *pdev)
 
 static const struct pci_device_id enetc_vf_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, ENETC_DEV_ID_VF) },
+	{ PCI_DEVICE(NXP_ENETC_VENDOR_ID, NXP_ENETC_VF_DEV_ID) },
 	{ 0, } /* End of table. */
 };
 MODULE_DEVICE_TABLE(pci, enetc_vf_id_table);
diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h
index 7176803146f3d1..8831da37326b28 100644
--- a/drivers/net/ethernet/freescale/fec.h
+++ b/drivers/net/ethernet/freescale/fec.h
@@ -692,7 +692,7 @@ struct fec_enet_private {
 	u64 ethtool_stats[];
 };
 
-void fec_ptp_init(struct platform_device *pdev, int irq_idx);
+int fec_ptp_init(struct platform_device *pdev, int irq_idx);
 void fec_ptp_restore_state(struct fec_enet_private *fep);
 void fec_ptp_save_state(struct fec_enet_private *fep);
 void fec_ptp_stop(struct platform_device *pdev);
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index 794ec427b0ee64..f7636f23d1ebbb 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -5384,8 +5384,11 @@ fec_probe(struct platform_device *pdev)
 		goto failed_reset;
 
 	irq_cnt = fec_enet_get_irq_cnt(pdev);
-	if (fep->bufdesc_ex)
-		fec_ptp_init(pdev, irq_cnt);
+	if (fep->bufdesc_ex) {
+		ret = fec_ptp_init(pdev, irq_cnt);
+		if (ret)
+			goto failed_reset;
+	}
 
 	ret = fec_enet_init(ndev);
 	if (ret)
@@ -5394,12 +5397,17 @@ fec_probe(struct platform_device *pdev)
 	for (i = 0; i < irq_cnt; i++) {
 		snprintf(irq_name, sizeof(irq_name), "int%d", i);
 		irq = platform_get_irq_byname_optional(pdev, irq_name);
-		if (irq < 0)
-			irq = platform_get_irq(pdev, i);
-		if (irq < 0) {
+		if (irq < 0 && irq != -ENXIO) {
 			ret = irq;
 			goto failed_irq;
 		}
+		if (irq == -ENXIO) {
+			irq = platform_get_irq(pdev, i);
+			if (irq < 0) {
+				ret = irq;
+				goto failed_irq;
+			}
+		}
 		ret = devm_request_irq(&pdev->dev, irq, fec_enet_interrupt,
 				       0, pdev->name, ndev);
 		if (ret)
diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c
index 56801c2009d59a..0574a1a1870204 100644
--- a/drivers/net/ethernet/freescale/fec_ptp.c
+++ b/drivers/net/ethernet/freescale/fec_ptp.c
@@ -734,9 +734,11 @@ static irqreturn_t fec_pps_interrupt(int irq, void *dev_id)
  * This function performs the required steps for enabling ptp
  * support. If ptp support has already been loaded it simply calls the
  * cyclecounter init routine and exits.
+ *
+ * Return: 0 on success, or a negative error code on failure.
  */
 
-void fec_ptp_init(struct platform_device *pdev, int irq_idx)
+int fec_ptp_init(struct platform_device *pdev, int irq_idx)
 {
 	struct net_device *ndev = platform_get_drvdata(pdev);
 	struct fec_enet_private *fep = netdev_priv(ndev);
@@ -779,26 +781,32 @@ void fec_ptp_init(struct platform_device *pdev, int irq_idx)
 		      HRTIMER_MODE_REL);
 
 	irq = platform_get_irq_byname_optional(pdev, "pps");
-	if (irq < 0)
+	if (irq < 0 && irq != -ENXIO)
+		return irq;
+	if (irq == -ENXIO) {
 		irq = platform_get_irq_optional(pdev, irq_idx);
-	/* Failure to get an irq is not fatal,
-	 * only the PTP_CLOCK_PPS clock events should stop
-	 */
-	if (irq >= 0) {
+		if (irq < 0 && irq != -ENXIO)
+			return irq;
+	}
+
+	if (irq > 0) {
 		ret = devm_request_irq(&pdev->dev, irq, fec_pps_interrupt,
 				       0, pdev->name, ndev);
 		if (ret < 0)
-			dev_warn(&pdev->dev, "request for pps irq failed(%d)\n",
-				 ret);
+			return ret;
 	}
 
 	fep->ptp_clock = ptp_clock_register(&fep->ptp_caps, &pdev->dev);
 	if (IS_ERR(fep->ptp_clock)) {
+		ret = PTR_ERR(fep->ptp_clock);
 		fep->ptp_clock = NULL;
 		dev_err(&pdev->dev, "ptp_clock_register failed\n");
+		return ret;
 	}
 
 	schedule_delayed_work(&fep->time_keep, HZ);
+
+	return 0;
 }
 
 void fec_ptp_save_state(struct fec_enet_private *fep)
diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c
index 299bab043175b3..46cc28895e5653 100644
--- a/drivers/net/ethernet/freescale/fman/fman.c
+++ b/drivers/net/ethernet/freescale/fman/fman.c
@@ -2734,6 +2734,7 @@ static struct fman *read_dts_node(struct platform_device *of_dev)
 	}
 
 	clk_rate = clk_get_rate(clk);
+	clk_put(clk);
 	if (!clk_rate) {
 		err = -EINVAL;
 		dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n",
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
index d3c772ed5fc9b1..071e3f3de78c71 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
@@ -1007,6 +1007,7 @@ static void fs_enet_remove(struct platform_device *ofdev)
 	fep->ops->cleanup_data(ndev);
 	dev_set_drvdata(fep->dev, NULL);
 	phylink_destroy(fep->phylink);
+	kfree(fep->fpi);
 	free_netdev(ndev);
 }
 
diff --git a/drivers/net/ethernet/google/Kconfig b/drivers/net/ethernet/google/Kconfig
index 14c9431e15e544..a68327f47acb42 100644
--- a/drivers/net/ethernet/google/Kconfig
+++ b/drivers/net/ethernet/google/Kconfig
@@ -17,7 +17,7 @@ if NET_VENDOR_GOOGLE
 
 config GVE
 	tristate "Google Virtual NIC (gVNIC) support"
-	depends on (PCI_MSI && (X86 || CPU_LITTLE_ENDIAN))
+	depends on (PCI_MSI && (X86 || ARM64 || CPU_LITTLE_ENDIAN))
 	depends on PTP_1588_CLOCK_OPTIONAL
 	select PAGE_POOL
 	help
diff --git a/drivers/net/ethernet/google/gve/gve_desc_dqo.h b/drivers/net/ethernet/google/gve/gve_desc_dqo.h
index f7786b03c74447..d2c86c8eeae2a1 100644
--- a/drivers/net/ethernet/google/gve/gve_desc_dqo.h
+++ b/drivers/net/ethernet/google/gve/gve_desc_dqo.h
@@ -14,6 +14,11 @@
 #define GVE_TX_MAX_HDR_SIZE_DQO 255
 #define GVE_TX_MIN_TSO_MSS_DQO 88
 
+/* HW limit. This also has to fit in the 14 bits of the mss field of
+ * struct gve_tx_tso_context_desc_dqo.
+ */
+#define GVE_TX_MAX_TSO_MSS_DQO 9728
+
 #ifndef __LITTLE_ENDIAN_BITFIELD
 #error "Only little endian supported"
 #endif
diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
index 80ab0a449ff54e..ad99cbb77e291b 100644
--- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
@@ -577,40 +577,43 @@ static int gve_prep_tso(struct sk_buff *skb)
 	int header_len;
 	int err;
 
-	/* Note: HW requires MSS (gso_size) to be <= 9728 and the total length
-	 * of the TSO to be <= 262143.
+	/* Note: HW requires the total length of the TSO to be <= 262143,
+	 * this is enforced by netif_set_tso_max_size().
 	 *
-	 * However, we don't validate these because:
-	 * - Hypervisor enforces a limit of 9K MTU
-	 * - Kernel will not produce a TSO larger than 64k
+	 * MSS (gso_size) can not be trusted: packets forwarded from a tap or
+	 * injected by a packet socket can carry an arbitrary value, while the
+	 * mss field of the TSO context descriptor is only 14 bits wide.
+	 *
+	 * A too big MSS is dropped here instead of being rejected from
+	 * gve_features_check_dqo(), because software segmentation would
+	 * produce packets larger than the device can send.
 	 */
-
-	if (unlikely(shinfo->gso_size < GVE_TX_MIN_TSO_MSS_DQO))
+	if (unlikely(shinfo->gso_size > GVE_TX_MAX_TSO_MSS_DQO))
 		return -1;
 
 	/* Needed because we will modify header. */
 	err = skb_cow_head(skb, 0);
 	if (err < 0)
 		return err;
+	shinfo = skb_shinfo(skb);
 
 	l4_start = skb_transport_offset(skb);
 	paylen = skb->len - l4_start;
 
-	switch (shinfo->gso_type) {
-	case SKB_GSO_TCPV4:
-	case SKB_GSO_TCPV6:
+	/* gso_type is a bitmask: SKB_GSO_DODGY, SKB_GSO_TCP_ECN and
+	 * SKB_GSO_TCP_FIXEDID may be set alongside the protocol bit.
+	 */
+	if (shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) {
 		tcp = tcp_hdr(skb);
 		csum_replace_by_diff(&tcp->check,
 				     (__force __wsum)htonl(paylen));
 		header_len = skb_tcp_all_headers(skb);
-		break;
-	case SKB_GSO_UDP_L4:
+	} else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
 		udp = udp_hdr(skb);
 		csum_replace_by_diff(&udp->check,
 				     (__force __wsum)htonl(paylen));
 		header_len = sizeof(struct udphdr) + l4_start;
-		break;
-	default:
+	} else {
 		return -EINVAL;
 	}
 
@@ -918,13 +921,22 @@ static bool gve_can_send_tso(const struct sk_buff *skb)
 {
 	const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1;
 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
-	const int header_len = skb_tcp_all_headers(skb);
 	const int gso_size = shinfo->gso_size;
 	int cur_seg_num_bufs;
 	int prev_frag_size;
 	int cur_seg_size;
+	int header_len;
 	int i;
 
+	if (unlikely(gso_size < GVE_TX_MIN_TSO_MSS_DQO))
+		return false;
+
+	/* Must match the header length programmed by gve_prep_tso(). */
+	if (skb_is_gso_tcp(skb))
+		header_len = skb_tcp_all_headers(skb);
+	else
+		header_len = skb_transport_offset(skb) + sizeof(struct udphdr);
+
 	cur_seg_size = skb_headlen(skb) - header_len;
 	prev_frag_size = skb_headlen(skb);
 	cur_seg_num_bufs = cur_seg_size > 0;
@@ -966,7 +978,17 @@ netdev_features_t gve_features_check_dqo(struct sk_buff *skb,
 					 struct net_device *dev,
 					 netdev_features_t features)
 {
-	if (skb_is_gso(skb) && !gve_can_send_tso(skb))
+	if (!skb_is_gso(skb))
+		return features;
+
+	/* Keep the GSO bits for a too big MSS, so that gve_prep_tso() drops
+	 * the packet: software segmentation would give packets larger than
+	 * the device can send.
+	 */
+	if (skb_shinfo(skb)->gso_size > GVE_TX_MAX_TSO_MSS_DQO)
+		return features;
+
+	if (!gve_can_send_tso(skb))
 		return features & ~NETIF_F_GSO_MASK;
 
 	return features;
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_common.h b/drivers/net/ethernet/hisilicon/hibmcge/hbg_common.h
index 8e134da3e21758..9ca04909774dc5 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_common.h
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_common.h
@@ -152,7 +152,7 @@ struct hbg_mac_filter {
 	bool enabled;
 };
 
-/* saved for restore after rest */
+/* saved for restore after reset */
 struct hbg_user_def {
 	struct ethtool_pauseparam pause_param;
 };
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_hw.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_hw.c
index d6e8ce8e351a26..54ed1c17bc2dfe 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_hw.c
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_hw.c
@@ -190,8 +190,8 @@ void hbg_hw_set_mtu(struct hbg_priv *priv, u16 mtu)
 	hbg_hw_set_pcu_max_frame_len(priv, frame_len);
 	hbg_hw_set_mac_max_frame_len(priv, frame_len);
 
-	hbg_reg_write_field(priv, HBG_REG_BRUST_LENGTH_ADDR,
-			    HBG_REG_BRUST_LENGTH_B, burst_len_bit);
+	hbg_reg_write_field(priv, HBG_REG_BURST_LENGTH_ADDR,
+			    HBG_REG_BURST_LENGTH_B, burst_len_bit);
 }
 
 void hbg_hw_mac_enable(struct hbg_priv *priv, u32 enable)
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_irq.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_irq.c
index ae4cb35186d884..2d67c7557b72e0 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_irq.c
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_irq.c
@@ -37,9 +37,9 @@ static void hbg_irq_handle_rx_buf_val(struct hbg_priv *priv,
 
 #define HBG_IRQ_I(name, handle) \
 	{#name, HBG_INT_MSK_##name##_B, false, false, false, handle}
-#define HBG_ERR_IRQ_I(name, need_print, ndde_reset) \
+#define HBG_ERR_IRQ_I(name, need_print, need_reset) \
 	{#name, HBG_INT_MSK_##name##_B, true, need_print, \
-	ndde_reset, hbg_irq_handle_err}
+	need_reset, hbg_irq_handle_err}
 
 static const struct hbg_irq_info hbg_irqs[] = {
 	HBG_IRQ_I(RX, hbg_irq_handle_rx),
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_reg.h b/drivers/net/ethernet/hisilicon/hibmcge/hbg_reg.h
index 30b3903c8f2dee..0d2554ca3b5f69 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_reg.h
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_reg.h
@@ -191,8 +191,8 @@
 #define HBG_REG_TX_CFF_ADDR_2_ADDR		(HBG_REG_SGMII_BASE + 0x0490)
 #define HBG_REG_TX_CFF_ADDR_3_ADDR		(HBG_REG_SGMII_BASE + 0x0494)
 #define HBG_REG_RX_CFF_ADDR_ADDR		(HBG_REG_SGMII_BASE + 0x04A0)
-#define HBG_REG_BRUST_LENGTH_ADDR		(HBG_REG_SGMII_BASE + 0x04C4)
-#define HBG_REG_BRUST_LENGTH_B			BIT(29)
+#define HBG_REG_BURST_LENGTH_ADDR		(HBG_REG_SGMII_BASE + 0x04C4)
+#define HBG_REG_BURST_LENGTH_B			BIT(29)
 #define HBG_REG_RX_BUF_SIZE_ADDR		(HBG_REG_SGMII_BASE + 0x04E4)
 #define HBG_REG_RX_BUF_SIZE_M			GENMASK(15, 0)
 #define HBG_REG_BUS_CTRL_ADDR			(HBG_REG_SGMII_BASE + 0x04E8)
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c
index 207ada721dd9a7..2ea025fd964fdc 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c
@@ -26,10 +26,10 @@
 	(hbg_queue_used_num((head), (tail), (ring)) == 0)
 #define hbg_queue_is_full(head, tail, ring) \
 	(hbg_queue_left_num((head), (tail), (ring)) == 0)
-#define hbg_queue_next_prt(p, ring) (((p) + 1) % (ring)->len)
+#define hbg_queue_next_ptr(p, ring) (((p) + 1) % (ring)->len)
 #define hbg_queue_move_next(p, ring) ({ \
 	typeof(ring) _ring = (ring); \
-	_ring->p = hbg_queue_next_prt(_ring->p, _ring); })
+	_ring->p = hbg_queue_next_ptr(_ring->p, _ring); })
 
 #define hbg_get_page_order(ring) ({ \
 	typeof(ring) _ring = (ring); \
@@ -167,7 +167,7 @@ netdev_tx_t hbg_net_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 	/* This smp_store_release() pairs with smp_load_acquire() in
 	 * hbg_napi_tx_recycle() called in tx interrupt handle process.
 	 */
-	smp_store_release(&ring->ntu, hbg_queue_next_prt(ntu, ring));
+	smp_store_release(&ring->ntu, hbg_queue_next_ptr(ntu, ring));
 	dev_sw_netstats_tx_add(netdev, 1, skb->len);
 	return NETDEV_TX_OK;
 }
@@ -220,7 +220,7 @@ static int hbg_napi_tx_recycle(struct napi_struct *napi, int budget)
 			break;
 
 		hbg_buffer_free(buffer);
-		ntc = hbg_queue_next_prt(ntc, ring);
+		ntc = hbg_queue_next_ptr(ntc, ring);
 		packet_done++;
 	}
 
@@ -535,7 +535,7 @@ static int hbg_napi_rx_poll(struct napi_struct *napi, int budget)
 	return packet_done;
 }
 
-static void hbg_ring_page_pool_destory(struct hbg_ring *ring)
+static void hbg_ring_page_pool_destroy(struct hbg_ring *ring)
 {
 	if (!ring->page_pool)
 		return;
@@ -589,7 +589,7 @@ static void hbg_ring_uninit(struct hbg_ring *ring)
 		buffer->priv = NULL;
 	}
 
-	hbg_ring_page_pool_destory(ring);
+	hbg_ring_page_pool_destroy(ring);
 	dma_free_coherent(&ring->priv->pdev->dev,
 			  ring->len * sizeof(*ring->queue),
 			  ring->queue, ring->queue_dma);
diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c
index fc2c47dcfaabea..2920985144bfa5 100644
--- a/drivers/net/ethernet/hisilicon/hip04_eth.c
+++ b/drivers/net/ethernet/hisilicon/hip04_eth.c
@@ -527,13 +527,14 @@ hip04_mac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	priv->tx_skb[tx_head] = skb;
 	priv->tx_phys[tx_head] = phys;
 
-	desc->send_size = (__force u32)cpu_to_be32(skb->len);
 #if defined(CONFIG_HI13X1_GMAC)
+	desc->send_size = (__force u16)cpu_to_be16(skb->len);
 	desc->cfg = (__force u32)cpu_to_be32(TX_CLEAR_WB | TX_FINISH_CACHE_INV
 		| TX_RELEASE_TO_PPE | priv->port << TX_POOL_SHIFT);
-	desc->data_offset = (__force u32)cpu_to_be32(phys & SOC_CACHE_LINE_MASK);
+	desc->data_offset = (__force u16)cpu_to_be16(phys & SOC_CACHE_LINE_MASK);
 	desc->send_addr =  (__force u32)cpu_to_be32(phys & ~SOC_CACHE_LINE_MASK);
 #else
+	desc->send_size = (__force u32)cpu_to_be32(skb->len);
 	desc->cfg = (__force u32)cpu_to_be32(TX_CLEAR_WB | TX_FINISH_CACHE_INV);
 	desc->send_addr = (__force u32)cpu_to_be32(phys);
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
index bc6b269be29959..f1cb6d56e4b9f7 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
@@ -793,6 +793,7 @@ static int hns_mac_register_phy(struct hns_mac_cb *mac_cb)
 		dev_err(mac_cb->dev,
 			"mac%d mdio is NULL, dsaf will probe again later\n",
 			mac_cb->mac_id);
+		put_device(&pdev->dev);
 		return -EPROBE_DEFER;
 	}
 
@@ -801,6 +802,8 @@ static int hns_mac_register_phy(struct hns_mac_cb *mac_cb)
 		dev_dbg(mac_cb->dev, "mac%d register phy addr:%d\n",
 			mac_cb->mac_id, addr);
 
+	put_device(&pdev->dev);
+
 	return rc;
 }
 
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
index 91391a49fcea27..a59fd280a045c9 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
@@ -591,7 +591,7 @@ static int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, bool en)
 #define HILINK_ACCESS_SEL_CFG		0x40008
 			/* hilink4 & hilink3 use the same xge training and
 			 * xge u adaptor. There is a hilink access sel cfg
-			 * register to select which one to be configed
+			 * register to select which one to be configured
 			 */
 			if ((!HNS_DSAF_IS_DEBUG(mac_cb->dsaf_dev)) &&
 			    (mac_cb->mac_id <= 3))
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
index 5013beb4d28257..408d36d85fb135 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
@@ -139,7 +139,7 @@ static void hns_ppe_set_vlan_strip(struct hns_ppe_cb *ppe_cb, int en)
 }
 
 /**
- * hns_ppe_checksum_hw - set ppe checksum caculate
+ * hns_ppe_checksum_hw - set ppe checksum calculate
  * @ppe_cb: ppe device
  * @value: value
  */
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
index e905f10b894e88..d819d2bb345074 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
@@ -1230,7 +1230,7 @@ static void hns_nic_update_stats(struct net_device *netdev)
 	h->dev->ops->update_stats(h, &netdev->stats);
 }
 
-/* set mac addr if it is configed. or leave it to the AE driver */
+/* set mac addr if it is configured. or leave it to the AE driver */
 static void hns_init_mac_addr(struct net_device *ndev)
 {
 	struct hns_nic_priv *priv = netdev_priv(ndev);
@@ -2317,7 +2317,7 @@ static int hns_nic_dev_probe(struct platform_device *pdev)
 		ret = device_property_read_u32(dev, "port-id", &port_id);
 		if (ret)
 			goto out_read_prop_fail;
-		/* for old dts, we need to caculate the port offset */
+		/* for old dts, we need to calculate the port offset */
 		port_id = port_id < HNS_SRV_OFFSET ? port_id + HNS_DEBUG_OFFSET
 			: port_id - HNS_SRV_OFFSET;
 	}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
index e7318f23631576..7b92a3c73ef6dd 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
@@ -474,10 +474,7 @@ static void hns3_update_limit_promisc_mode(struct net_device *netdev,
 {
 	struct hnae3_handle *handle = hns3_get_handle(netdev);
 
-	if (enable)
-		set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->priv_flags);
-	else
-		clear_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->priv_flags);
+	assign_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->priv_flags, enable);
 
 	hns3_request_update_promisc_mode(handle);
 }
@@ -1210,10 +1207,7 @@ static int hns3_set_tx_push(struct net_device *netdev, u32 tx_push)
 	netdev_dbg(netdev, "Changing tx push from %s to %s\n",
 		   str_on_off(old_state), str_on_off(tx_push));
 
-	if (tx_push)
-		set_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state);
-	else
-		clear_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state);
+	assign_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state, tx_push);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index e6af939c1c19bf..d117a0b442d020 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -9156,13 +9156,8 @@ static int hclge_pci_init(struct hclge_dev *hdev)
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (ret) {
-		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (ret) {
-			dev_err(&pdev->dev,
-				"can't set consistent PCI DMA\n");
-			goto err_disable_device;
-		}
-		dev_warn(&pdev->dev, "set DMA mask to 32 bits\n");
+		dev_err(&pdev->dev, "can't set consistent PCI DMA\n");
+		goto err_disable_device;
 	}
 
 	ret = pci_request_regions(pdev, HCLGE_DRIVER_NAME);
@@ -10527,7 +10522,7 @@ static int hclge_get_link_diagnosis_info(struct hnae3_handle *handle,
 }
 
 /* After disable sriov, VF still has some config and info need clean,
- * which configed by PF.
+ * which configured by PF.
  */
 static void hclge_clear_vport_vf_info(struct hclge_vport *vport, int vfid)
 {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
index 30e38cb4843ac9..08040a4929855d 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
@@ -341,11 +341,8 @@ static void hclge_set_vf_promisc_mode(struct hclge_vport *vport,
 	vport->vf_info.request_mc_en = req->msg.en_mc;
 	vport->vf_info.request_bc_en = req->msg.en_bc;
 
-	if (req->msg.en_limit_promisc)
-		set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->priv_flags);
-	else
-		clear_bit(HNAE3_PFLAG_LIMIT_PROMISC,
-			  &handle->priv_flags);
+	assign_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->priv_flags,
+		   req->msg.en_limit_promisc);
 
 	set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state);
 	hclge_task_schedule(hdev, 0);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
index 00c3f2548bf6bf..bf533b8f46ae4b 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
@@ -35,7 +35,7 @@ enum hclge_shaper_level {
  * IR(Mbps) = -------------------------  *  CLOCK(1000Mbps)
  *		Tick * (2 ^ IR_s)
  *
- * @return: 0: calculate sucessful, negative: fail
+ * @return: 0: calculate successful, negative: fail
  */
 static int hclge_shaper_para_calc(u32 ir, u8 shaper_level,
 				  struct hclge_shaper_ir_para *ir_para,
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
index 063a692c2b41e2..5d14d29b02965e 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
@@ -1880,7 +1880,7 @@ static void hclgevf_reset_service_task(struct hclgevf_dev *hdev)
 		 * from PF as cmdq would be in unreliable state i.e. mailbox
 		 * communication between PF and VF would be broken.
 		 *
-		 * if we are never geting into pending state it means either:
+		 * if we are never getting into pending state it means either:
 		 * 1. PF is not receiving our request which could be due to IMP
 		 *    reset
 		 * 2. PF is screwed
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c
index f44b3064ab2eab..830bf9aadbf50b 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c
@@ -545,6 +545,10 @@ int hinic3_init_hwdev(struct pci_dev *pdev)
 	hwdev->dev = &pci_adapter->pdev->dev;
 	hwdev->func_state = 0;
 	hwdev->dev_id = hinic3_adev_idx_alloc();
+	if (hwdev->dev_id < 0) {
+		err = hwdev->dev_id;
+		goto err_free_hwdev_no_id;
+	}
 	spin_lock_init(&hwdev->channel_lock);
 
 	err = hinic3_init_hwif(hwdev);
@@ -602,8 +606,9 @@ int hinic3_init_hwdev(struct pci_dev *pdev)
 err_free_hwif:
 	hinic3_free_hwif(hwdev);
 err_free_hwdev:
-	pci_adapter->hwdev = NULL;
 	hinic3_adev_idx_free(hwdev->dev_id);
+err_free_hwdev_no_id:
+	pci_adapter->hwdev = NULL;
 	kfree(hwdev);
 
 	return err;
diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c
index 1d46cf6c2c127a..e7043523457c14 100644
--- a/drivers/net/ethernet/ibm/emac/core.c
+++ b/drivers/net/ethernet/ibm/emac/core.c
@@ -3044,6 +3044,15 @@ static int emac_probe(struct platform_device *ofdev)
 	if (err)
 		goto err_gone;
 
+	if (emac_phy_supports_gige(dev->phy_mode)) {
+		ndev->netdev_ops = &emac_gige_netdev_ops;
+		dev->commac.ops = &emac_commac_sg_ops;
+	} else {
+		ndev->netdev_ops = &emac_netdev_ops;
+		dev->commac.ops = &emac_commac_ops;
+	}
+	ndev->ethtool_ops = &emac_ethtool_ops;
+
 	dev->emacp = devm_platform_ioremap_resource(ofdev, 0);
 	if (IS_ERR(dev->emacp)) {
 		err = PTR_ERR(dev->emacp);
@@ -3076,7 +3085,6 @@ static int emac_probe(struct platform_device *ofdev)
 		dev->mdio_instance = platform_get_drvdata(dev->mdio_dev);
 
 	/* Register with MAL */
-	dev->commac.ops = &emac_commac_ops;
 	dev->commac.dev = dev;
 	dev->commac.tx_chan_mask = MAL_CHAN_MASK(dev->mal_tx_chan);
 	dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan);
@@ -3144,12 +3152,6 @@ static int emac_probe(struct platform_device *ofdev)
 		ndev->features |= ndev->hw_features | NETIF_F_RXCSUM;
 	}
 	ndev->watchdog_timeo = 5 * HZ;
-	if (emac_phy_supports_gige(dev->phy_mode)) {
-		ndev->netdev_ops = &emac_gige_netdev_ops;
-		dev->commac.ops = &emac_commac_sg_ops;
-	} else
-		ndev->netdev_ops = &emac_netdev_ops;
-	ndev->ethtool_ops = &emac_ethtool_ops;
 
 	/* MTU range: 46 - 1500 or whatever is in OF */
 	ndev->min_mtu = EMAC_MIN_MTU;
diff --git a/drivers/net/ethernet/intel/e1000/e1000.h b/drivers/net/ethernet/intel/e1000/e1000.h
index ea6ccf4b728be7..0b2ae5c4d4c3b4 100644
--- a/drivers/net/ethernet/intel/e1000/e1000.h
+++ b/drivers/net/ethernet/intel/e1000/e1000.h
@@ -24,7 +24,6 @@
 #include <linux/vmalloc.h>
 #include <linux/interrupt.h>
 #include <linux/string.h>
-#include <linux/pagemap.h>
 #include <linux/dma-mapping.h>
 #include <linux/bitops.h>
 #include <asm/io.h>
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index 844f31ab37ad44..a0142f656a1b64 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -8,7 +8,6 @@
 #include <linux/init.h>
 #include <linux/pci.h>
 #include <linux/vmalloc.h>
-#include <linux/pagemap.h>
 #include <linux/delay.h>
 #include <linux/netdevice.h>
 #include <linux/interrupt.h>
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
index f5b4d062709a5c..afe939f8f4efea 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
@@ -2111,8 +2111,6 @@ static int fm10k_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
-	if (err)
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 	if (err) {
 		dev_err(&pdev->dev,
 			"DMA configuration failed: %d\n", err);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c
index 8dadfef2c09f62..026b35e5a0003d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_common.c
@@ -676,7 +676,7 @@ int i40e_pf_reset(struct i40e_hw *hw)
 	grst_del = FIELD_GET(I40E_GLGEN_RSTCTL_GRSTDEL_MASK,
 			     rd32(hw, I40E_GLGEN_RSTCTL));
 
-	/* It can take upto 15 secs for GRST steady state.
+	/* It can take up to 15 secs for GRST steady state.
 	 * Bump it to 16 secs max to be safe.
 	 */
 	grst_del = grst_del * 20;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index 3da9ec49cc7457..6a992df06fa11b 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -2614,7 +2614,7 @@ static u64 i40e_eeprom_test(struct net_device *netdev, u64 *data)
 	netif_info(pf, hw, netdev, "eeprom test\n");
 	*data = i40e_diag_eeprom_test(&pf->hw);
 
-	/* forcebly clear the NVM Update state machine */
+	/* forcibly clear the NVM Update state machine */
 	pf->hw.nvmupd_state = I40E_NVMUPD_STATE_INIT;
 
 	return *data;
@@ -5263,10 +5263,7 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 		    test_bit(priv_flag->bitno, orig_flags) != new_val)
 			return -EOPNOTSUPP;
 
-		if (new_val)
-			set_bit(priv_flag->bitno, new_flags);
-		else
-			clear_bit(priv_flag->bitno, new_flags);
+		assign_bit(priv_flag->bitno, new_flags, new_val);
 	}
 
 	if (pf->hw.pf_id != 0)
@@ -5284,10 +5281,7 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 		    test_bit(priv_flag->bitno, orig_flags) != new_val)
 			return -EOPNOTSUPP;
 
-		if (new_val)
-			set_bit(priv_flag->bitno, new_flags);
-		else
-			clear_bit(priv_flag->bitno, new_flags);
+		assign_bit(priv_flag->bitno, new_flags, new_val);
 	}
 
 flags_complete:
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 0cd0e5597c909d..abbc71e815ae34 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -4,6 +4,7 @@
 #include <generated/utsrelease.h>
 #include <linux/crash_dump.h>
 #include <linux/net/intel/libie/pctype.h>
+#include <linux/net/intel/libie/rx.h>
 #include <linux/if_bridge.h>
 #include <linux/if_macvlan.h>
 #include <linux/module.h>
@@ -2411,10 +2412,12 @@ void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
 
 	if (fcnt != num_add) {
 		if (vsi->type == I40E_VSI_MAIN) {
-			set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
-			dev_warn(&vsi->back->pdev->dev,
-				 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
-				 libie_aq_str(aq_status), vsi_name);
+			if (!test_and_set_bit(__I40E_VSI_OVERFLOW_PROMISC,
+					      vsi->state)) {
+				dev_warn(&vsi->back->pdev->dev,
+					 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
+					 libie_aq_str(aq_status), vsi_name);
+			}
 		} else if (vsi->type == I40E_VSI_SRIOV ||
 			   vsi->type == I40E_VSI_VMDQ1 ||
 			   vsi->type == I40E_VSI_VMDQ2) {
@@ -2464,10 +2467,11 @@ i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
 	}
 
 	if (aq_ret) {
-		set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
-		dev_warn(&vsi->back->pdev->dev,
-			 "Error %s, forcing overflow promiscuous on %s\n",
-			 libie_aq_str(hw->aq.asq_last_status), vsi_name);
+		if (!test_and_set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state)) {
+			dev_warn(&vsi->back->pdev->dev,
+				 "Error %s, forcing overflow promiscuous on %s\n",
+				 libie_aq_str(hw->aq.asq_last_status), vsi_name);
+		}
 	}
 
 	return aq_ret;
@@ -3622,7 +3626,7 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
 	}
 
 skip:
-	xdp_init_buff(&ring->xdp, xdp_frame_sz, &ring->xdp_rxq);
+	xdp_init_buff(&ring->xdp_ctx.xdp, xdp_frame_sz, &ring->xdp_rxq);
 
 	rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
 				    BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
@@ -13582,6 +13586,53 @@ static int i40e_xdp(struct net_device *dev,
 	}
 }
 
+static int i40e_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
+			    enum xdp_rss_hash_type *rss_type)
+{
+	const struct i40e_xdp_buff *ctx = (const void *)_ctx;
+	const union i40e_rx_desc *desc = ctx->desc;
+	struct libeth_rx_pt rx_ptype;
+	u8 raw_rx_ptype;
+	u64 status;
+
+	status = le64_to_cpu(desc->wb.qword1.status_error_len);
+	raw_rx_ptype = FIELD_GET(I40E_RXD_QW1_PTYPE_MASK, status);
+	rx_ptype = libie_rx_pt_parse(raw_rx_ptype);
+
+	if (!libeth_rx_pt_has_hash(ctx->xdp.rxq->dev, rx_ptype) ||
+	    FIELD_GET(I40E_RX_DESC_STATUS_FLTSTAT_MASK, status) !=
+		    I40E_RX_DESC_FLTSTAT_RSS_HASH)
+		return -ENODATA;
+
+	*hash = le32_to_cpu(desc->wb.qword0.hi_dword.rss);
+	*rss_type = rx_ptype.hash_type;
+
+	return 0;
+}
+
+static int i40e_xdp_rx_vlan_tag(const struct xdp_md *_ctx, __be16 *vlan_proto,
+				u16 *vlan_tci)
+{
+	const struct i40e_xdp_buff *ctx = (const void *)_ctx;
+	const union i40e_rx_desc *desc = ctx->desc;
+	u64 status;
+
+	status = le64_to_cpu(desc->wb.qword1.status_error_len);
+
+	if (!(status & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)))
+		return -ENODATA;
+
+	*vlan_proto = cpu_to_be16(ETH_P_8021Q);
+	*vlan_tci = le16_to_cpu(desc->wb.qword0.lo_dword.l2tag1);
+
+	return 0;
+}
+
+static const struct xdp_metadata_ops i40e_xdp_metadata_ops = {
+	.xmo_rx_hash		= i40e_xdp_rx_hash,
+	.xmo_rx_vlan_tag	= i40e_xdp_rx_vlan_tag,
+};
+
 static const struct net_device_ops i40e_netdev_ops = {
 	.ndo_open		= i40e_open,
 	.ndo_stop		= i40e_close,
@@ -13784,6 +13835,7 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
 	i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
 
 	netdev->netdev_ops = &i40e_netdev_ops;
+	netdev->xdp_metadata_ops = &i40e_xdp_metadata_ops;
 	netdev->watchdog_timeo = 5 * HZ;
 	i40e_set_ethtool_ops(netdev);
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index ef5e657816f005..7f68adb5109b0f 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -2438,10 +2438,11 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget,
 			     unsigned int *rx_cleaned)
 {
 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
+	struct i40e_xdp_buff *xdp_ctx = &rx_ring->xdp_ctx;
 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
 	u16 clean_threshold = rx_ring->count / 2;
 	unsigned int offset = rx_ring->rx_offset;
-	struct xdp_buff *xdp = &rx_ring->xdp;
+	struct xdp_buff *xdp = &xdp_ctx->xdp;
 	unsigned int xdp_xmit = 0;
 	struct bpf_prog *xdp_prog;
 	bool failure = false;
@@ -2530,6 +2531,8 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget,
 		if (neop)
 			continue;
 
+		xdp_ctx->desc = rx_desc;
+
 		xdp_res = i40e_run_xdp(rx_ring, xdp, xdp_prog);
 
 		if (xdp_res) {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index 1e5fd63d47f47c..bb741ff3e5f2c6 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -283,6 +283,11 @@ struct i40e_rx_buffer {
 	__u32 page_count;
 };
 
+struct i40e_xdp_buff {
+	struct xdp_buff xdp;
+	const union i40e_rx_desc *desc;
+};
+
 struct i40e_queue_stats {
 	u64 packets;
 	u64 bytes;
@@ -345,7 +350,7 @@ struct i40e_ring {
 	 * and to resume packet building for this ring in the next call to
 	 * i40e_clean_rx_ring_irq().
 	 */
-	struct xdp_buff xdp;
+	struct i40e_xdp_buff xdp_ctx;
 
 	/* Next descriptor to be processed; next_to_clean is updated only on
 	 * processing EOP descriptor
diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h
index ed8bbdb586da9b..16a65c6e51537d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_type.h
@@ -4,6 +4,7 @@
 #ifndef _I40E_TYPE_H_
 #define _I40E_TYPE_H_
 
+#include <linux/bits.h>
 #include <uapi/linux/if_ether.h>
 #include "i40e_adminq.h"
 #include "i40e_hmc.h"
@@ -699,6 +700,10 @@ enum i40e_rx_desc_status_bits {
 	I40E_RX_DESC_STATUS_LAST /* this entry must be last!!! */
 };
 
+#define I40E_RX_DESC_STATUS_FLTSTAT_MASK                   \
+	GENMASK_ULL(I40E_RX_DESC_STATUS_FLTSTAT_SHIFT + 1, \
+		    I40E_RX_DESC_STATUS_FLTSTAT_SHIFT)
+
 #define I40E_RXD_QW1_STATUS_SHIFT	0
 #define I40E_RXD_QW1_STATUS_MASK	((BIT(I40E_RX_DESC_STATUS_LAST) - 1) \
 					 << I40E_RXD_QW1_STATUS_SHIFT)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index a26c3d47ec1561..0087f7ae0c4c00 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -1158,10 +1158,7 @@ static int i40e_alloc_vf_res(struct i40e_vf *vf)
 		pf->queues_left -=
 			total_queue_pairs - I40E_DEFAULT_QUEUES_PER_VF;
 
-	if (vf->trusted)
-		set_bit(I40E_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps);
-	else
-		clear_bit(I40E_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps);
+	assign_bit(I40E_VIRTCHNL_VF_CAP_PRIVILEGE, &vf->vf_caps, vf->trusted);
 
 	/* store the total qps number for the runtime
 	 * VF req validation
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index 9f47388eaba538..51a05ce4c7ce8d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -246,6 +246,8 @@ bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
 	u32 nb_buffs, i;
 	dma_addr_t dma;
 
+	XSK_CHECK_PRIV_TYPE(struct i40e_xdp_buff);
+
 	rx_desc = I40E_RX_DESC(rx_ring, ntu);
 	xdp = i40e_rx_bi(rx_ring, ntu);
 
@@ -396,6 +398,14 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring,
 	WARN_ON_ONCE(1);
 }
 
+static struct i40e_xdp_buff *xsk_buff_to_i40e_ctx(struct xdp_buff *xdp)
+{
+	/* xdp_buff pointer used by ZC code path is allocated as xdp_buff_xsk.
+	 * i40e_xdp_buff private fields overlap with xdp_buff_xsk->cb.
+	 */
+	return (struct i40e_xdp_buff *)xdp;
+}
+
 /**
  * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring
  * @rx_ring: Rx ring
@@ -472,6 +482,8 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
 		if (i40e_is_non_eop(rx_ring, rx_desc))
 			continue;
 
+		xsk_buff_to_i40e_ctx(first)->desc = rx_desc;
+
 		xdp_res = i40e_run_xdp_zc(rx_ring, first, xdp_prog);
 		i40e_handle_xdp_result_zc(rx_ring, first, rx_desc, &rx_packets,
 					  &rx_bytes, xdp_res, &failure);
diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile
index 95fd0c49800fde..a952bacb7ec728 100644
--- a/drivers/net/ethernet/intel/ice/Makefile
+++ b/drivers/net/ethernet/intel/ice/Makefile
@@ -24,7 +24,7 @@ ice-y := ice_main.o	\
 	 ice_vsi_vlan_ops.o \
 	 ice_vsi_vlan_lib.o \
 	 ice_fdir.o	\
-	 ice_ethtool_fdir.o \
+	 ice_ethtool_ntuple.o \
 	 ice_vlan_mode.o \
 	 ice_flex_pipe.o \
 	 ice_flow.o	\
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index db3c7015c56c43..a56e9b7c4dadf3 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -302,7 +302,6 @@ enum ice_pf_state {
 	ICE_CFG_BUSY,
 	ICE_SERVICE_SCHED,
 	ICE_SERVICE_DIS,
-	ICE_FD_FLUSH_REQ,
 	ICE_OICR_INTR_DIS,		/* Global OICR interrupt disabled */
 	ICE_MDD_VF_PRINT_PENDING,	/* set when MDD event handle */
 	ICE_VF_RESETS_DISABLED,	/* disable resets during ice_remove */
@@ -1018,11 +1017,11 @@ void ice_deinit_rdma(struct ice_pf *pf);
 bool ice_is_wol_supported(struct ice_hw *hw);
 void ice_fdir_del_all_fltrs(struct ice_vsi *vsi);
 int
-ice_fdir_write_fltr(struct ice_pf *pf, struct ice_fdir_fltr *input, bool add,
+ice_fdir_write_fltr(struct ice_pf *pf, struct ice_ntuple_fltr *input, bool add,
 		    bool is_tun);
 void ice_vsi_manage_fdir(struct ice_vsi *vsi, bool ena);
-int ice_add_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd);
-int ice_del_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd);
+int ice_add_ntuple_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd);
+int ice_del_ntuple_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd);
 int ice_get_ethtool_fdir_entry(struct ice_hw *hw, struct ethtool_rxnfc *cmd);
 int
 ice_get_fdir_fltr_ids(struct ice_hw *hw, struct ethtool_rxnfc *cmd,
@@ -1105,7 +1104,7 @@ extern const struct xdp_metadata_ops ice_xdp_md_ops;
  * ice_is_dual - Check if given config is multi-NAC
  * @hw: pointer to HW structure
  *
- * Return: true if the device is running in mutli-NAC (Network
+ * Return: true if the device is running in multi-NAC (Network
  * Acceleration Complex) configuration variant, false otherwise
  * (always false for non-E825 devices).
  */
diff --git a/drivers/net/ethernet/intel/ice/ice_arfs.c b/drivers/net/ethernet/intel/ice/ice_arfs.c
index 53b6e2b09eb974..e0335f5e18fe43 100644
--- a/drivers/net/ethernet/intel/ice/ice_arfs.c
+++ b/drivers/net/ethernet/intel/ice/ice_arfs.c
@@ -302,7 +302,7 @@ ice_arfs_build_entry(struct ice_vsi *vsi, const struct flow_keys *fk,
 		     u16 rxq_idx, u32 flow_id)
 {
 	struct ice_arfs_entry *arfs_entry;
-	struct ice_fdir_fltr *fltr_info;
+	struct ice_ntuple_fltr *fltr_info;
 	u8 ip_proto;
 
 	arfs_entry = devm_kzalloc(ice_pf_to_dev(vsi->back),
@@ -392,8 +392,8 @@ ice_arfs_is_perfect_flow_set(struct ice_hw *hw, __be16 l3_proto, u8 l4_proto)
  * * false	- fltr_info and fk refer to different flows.
  */
 static bool
-ice_arfs_cmp(const struct ice_fdir_fltr *fltr_info, const struct flow_keys *fk,
-	     __be16 n_proto, u8 ip_proto)
+ice_arfs_cmp(const struct ice_ntuple_fltr *fltr_info,
+	     const struct flow_keys *fk, __be16 n_proto, u8 ip_proto)
 {
 	/* Determine if the filter is for IPv4 or IPv6 based on flow_type,
 	 * which is one of ICE_FLTR_PTYPE_NONF_IPV{4,6}_{TCP,UDP}.
@@ -485,7 +485,7 @@ ice_rx_flow_steer(struct net_device *netdev, const struct sk_buff *skb,
 	spin_lock_bh(&vsi->arfs_lock);
 	hlist_for_each_entry(arfs_entry, &vsi->arfs_fltr_list[idx],
 			     list_entry) {
-		struct ice_fdir_fltr *fltr_info;
+		struct ice_ntuple_fltr *fltr_info;
 
 		/* keep searching for the already existing arfs_entry flow */
 		if (arfs_entry->flow_id != flow_id)
diff --git a/drivers/net/ethernet/intel/ice/ice_arfs.h b/drivers/net/ethernet/intel/ice/ice_arfs.h
index 9706293128c3b0..7393254b7e0a4f 100644
--- a/drivers/net/ethernet/intel/ice/ice_arfs.h
+++ b/drivers/net/ethernet/intel/ice/ice_arfs.h
@@ -13,7 +13,7 @@ enum ice_arfs_fltr_state {
 };
 
 struct ice_arfs_entry {
-	struct ice_fdir_fltr fltr_info;
+	struct ice_ntuple_fltr fltr_info;
 	struct hlist_node list_entry;
 	u64 time_activated;	/* only valid for UDP flows */
 	u32 flow_id;
diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c
index 85a74cd6ea1f3e..ade1fb3dd06743 100644
--- a/drivers/net/ethernet/intel/ice/ice_dpll.c
+++ b/drivers/net/ethernet/intel/ice/ice_dpll.c
@@ -22,6 +22,8 @@
 
 #define E825_RCLK_PARENT_0_PIN_IDX		0
 #define E825_RCLK_PARENT_1_PIN_IDX		1
+#define E825_DPLL_TSPLL_BASE_IDX		0
+#define E825_DPLL_TXCLK_BASE_IDX		1
 
 #define ICE_DPLL_PIN_SW_INPUT_ABS(in_idx) \
 	(ICE_DPLL_SW_PIN_INPUT_BASE_SFP + (in_idx))
@@ -84,6 +86,7 @@ static const char * const ice_dpll_sw_pin_sma[] = { "SMA1", "SMA2" };
 static const char * const ice_dpll_sw_pin_ufl[] = { "U.FL1", "U.FL2" };
 static const char * const ice_dpll_ext_eref_pin = "EXT_EREF0";
 static const char * const ice_dpll_fwnode_ext_synce = "clk_ref_synce";
+static const char * const ice_dpll_fwnode_time_ref = "time_ref";
 
 static const struct dpll_pin_frequency ice_esync_range[] = {
 	DPLL_PIN_FREQUENCY_RANGE(0, DPLL_PIN_FREQUENCY_1_HZ),
@@ -2814,6 +2817,160 @@ static const struct dpll_pin_ops ice_dpll_txclk_ops = {
 	.direction_get = ice_dpll_input_direction,
 };
 
+/**
+ * ice_dpll_tspll_lock_status_get - derive TSPLL state for dpll subsystem
+ * @pf: board private structure
+ * @use_cached: if true, read lock state from pf->ptp.tspll_locked (maintained
+ *              by the PTP periodic worker); if false, query hardware directly
+ *
+ * If TIME_REF is not selected, TSPLL is treated as unlocked from the dpll
+ * subsystem perspective, regardless of raw lock bit.
+ *
+ * Return: TSPLL lock status
+ */
+static enum dpll_lock_status
+ice_dpll_tspll_lock_status_get(struct ice_pf *pf, bool use_cached)
+{
+	enum ice_clk_src clk_src;
+	bool locked;
+
+	if (ice_tspll_get_clk_src(&pf->hw, &clk_src) ||
+	    clk_src != ICE_CLK_SRC_TIME_REF)
+		return DPLL_LOCK_STATUS_UNLOCKED;
+
+	if (use_cached) {
+		locked = READ_ONCE(pf->ptp.tspll_locked);
+	} else {
+		bool lock_lost;
+
+		if (ice_tspll_lost_lock_e825c(&pf->hw, &lock_lost))
+			return DPLL_LOCK_STATUS_UNLOCKED;
+		locked = !lock_lost;
+	}
+	return locked ? DPLL_LOCK_STATUS_LOCKED : DPLL_LOCK_STATUS_UNLOCKED;
+}
+
+/**
+ * ice_dpll_tspll_state_on_dpll_get - get TIME_REF pin state on TSPLL DPLL
+ * @pin: pointer to a pin
+ * @pin_priv: private data pointer passed on pin registration
+ * @dpll: registered dpll pointer
+ * @dpll_priv: private data pointer passed on dpll registration
+ * @state: on success holds pin state on parent dpll
+ * @extack: error reporting
+ *
+ * Dpll subsystem callback. Returns CONNECTED if the TSPLL is using
+ * TIME_REF as its clock source, DISCONNECTED otherwise.
+ *
+ * Return:
+ * * 0 - success
+ * * negative - failed to read clock source
+ */
+static int
+ice_dpll_tspll_state_on_dpll_get(const struct dpll_pin *pin, void *pin_priv,
+				 const struct dpll_device *dpll,
+				 void *dpll_priv,
+				 enum dpll_pin_state *state,
+				 struct netlink_ext_ack *extack)
+{
+	struct ice_dpll_pin *p = pin_priv;
+	enum ice_clk_src clk_src;
+	int err;
+
+	if (ice_dpll_is_reset(p->pf, extack))
+		return -EBUSY;
+
+	err = ice_tspll_get_clk_src(&p->pf->hw, &clk_src);
+	if (err)
+		return err;
+
+	*state = clk_src == ICE_CLK_SRC_TIME_REF ? DPLL_PIN_STATE_CONNECTED :
+						    DPLL_PIN_STATE_DISCONNECTED;
+	return 0;
+}
+
+/**
+ * ice_dpll_tspll_state_on_dpll_set - set TIME_REF pin state on TSPLL DPLL
+ * @pin: pointer to a pin
+ * @pin_priv: private data pointer passed on pin registration
+ * @dpll: registered dpll pointer
+ * @dpll_priv: private data pointer passed on dpll registration
+ * @state: requested state of the pin
+ * @extack: error reporting
+ *
+ * Dpll subsystem callback. Enables (CONNECTED) or disables (DISCONNECTED)
+ * the TIME_REF signal as the TSPLL clock source. Selects TCXO when disabled.
+ *
+ * Return:
+ * * 0 - success
+ * * negative - error
+ */
+static int
+ice_dpll_tspll_state_on_dpll_set(const struct dpll_pin *pin, void *pin_priv,
+				 const struct dpll_device *dpll,
+				 void *dpll_priv, enum dpll_pin_state state,
+				 struct netlink_ext_ack *extack)
+{
+	struct ice_dpll_pin *p = pin_priv;
+	struct ice_pf *pf = p->pf;
+	enum ice_clk_src clk_src;
+	bool changed = false;
+	struct ice_dpll *tp;
+	int ret;
+
+	if (ice_dpll_is_reset(pf, extack))
+		return -EBUSY;
+
+	if (state != DPLL_PIN_STATE_CONNECTED &&
+	    state != DPLL_PIN_STATE_DISCONNECTED) {
+		NL_SET_ERR_MSG(extack,
+			       "unsupported pin state for TSPLL reference clock");
+		return -EINVAL;
+	}
+
+	clk_src = (state == DPLL_PIN_STATE_CONNECTED) ? ICE_CLK_SRC_TIME_REF :
+							ICE_CLK_SRC_TCXO;
+	tp = &pf->dplls.tspll;
+	/* Serialize multi-register TSPLL reconfiguration with restart path. */
+	mutex_lock(&pf->dplls.lock);
+	ret = ice_tspll_set_cfg(&pf->hw, ICE_TSPLL_FREQ_156_250, clk_src);
+	if (!ret || ret == -EAGAIN) {
+		enum dpll_lock_status new_state;
+
+		new_state = ice_dpll_tspll_lock_status_get(pf, false);
+
+		/* Sync the cached lock state to what we just observed from HW
+		 * so ice_dpll_periodic_work_e825() (cached path) does not emit
+		 * a spurious UNLOCKED notification between now and the next
+		 * PTP monitor tick.
+		 */
+		WRITE_ONCE(pf->ptp.tspll_locked,
+			   new_state == DPLL_LOCK_STATUS_LOCKED);
+
+		if (tp->prev_dpll_state != new_state) {
+			tp->dpll_state = new_state;
+			tp->prev_dpll_state = new_state;
+			changed = true;
+		}
+	}
+	mutex_unlock(&pf->dplls.lock);
+	if (changed)
+		__dpll_device_change_ntf(tp->dpll);
+	/* TSPLL reconfiguration may complete before lock is reacquired.
+	 * PTP periodic monitoring tracks and restores the final lock state.
+	 */
+	if (ret == -EAGAIN)
+		ret = 0;
+
+	return ret;
+}
+
+static const struct dpll_pin_ops ice_dpll_tspll_in_ops = {
+	.state_on_dpll_set = ice_dpll_tspll_state_on_dpll_set,
+	.state_on_dpll_get = ice_dpll_tspll_state_on_dpll_get,
+	.direction_get = ice_dpll_input_direction,
+};
+
 static const struct dpll_pin_ops ice_dpll_pin_sma_ops = {
 	.state_on_dpll_set = ice_dpll_sma_pin_state_set,
 	.state_on_dpll_get = ice_dpll_sw_pin_state_get,
@@ -2902,6 +3059,68 @@ static u64 ice_generate_clock_id(struct ice_pf *pf)
 	return pci_get_dsn(pf->pdev);
 }
 
+/**
+ * ice_generate_dpll_clock_id - generate clock_id for a specific dpll device
+ * @pf: board private structure
+ * @d: dpll device context
+ * @type: dpll type being registered
+ *
+ * For E825 generic DPLLs, use per-interface permanent MAC as the clock_id
+ * basis so userspace can unambiguously map DPLL devices to interfaces.
+ * TX-CLK keeps plain MAC-derived ID, while TSPLL uses the same basis with
+ * a dedicated tag bit to remain distinct on source-timer owner PFs.
+ * Other DPLL objects keep board-level DSN-derived clock_id.
+ *
+ * Return: generated clock id for a dpll device
+ */
+static u64 ice_generate_dpll_clock_id(struct ice_pf *pf, struct ice_dpll *d,
+				      enum dpll_type type)
+{
+	struct ice_hw *hw = &pf->hw;
+	u64 mac_clock_id;
+
+	if (hw->mac_type == ICE_MAC_GENERIC_3K_E825 &&
+	    type == DPLL_TYPE_GENERIC &&
+	    hw->port_info &&
+	    is_valid_ether_addr(hw->port_info->mac.perm_addr)) {
+		mac_clock_id = ether_addr_to_u64(hw->port_info->mac.perm_addr);
+
+		if (d->dpll_idx >= E825_DPLL_TXCLK_BASE_IDX)
+			return mac_clock_id;
+
+		if (d->dpll_idx == E825_DPLL_TSPLL_BASE_IDX)
+			return mac_clock_id | BIT_ULL(63);
+	}
+
+	return pf->dplls.clock_id;
+}
+
+/**
+ * ice_dpll_is_own_dpll_clock_id - check if clock_id belongs to this pf's DPLLs
+ * @pf: board private structure
+ * @clock_id: clock_id from a DPLL notification
+ *
+ * Match info->src_clock_id from a DPLL pin notification against any DPLL
+ * device this PF has registered. Used to suppress self-notifications
+ * generated as a side effect of our own dpll_pin_register() and
+ * dpll_pin_unregister() calls on the fwnode-backed SYNCE and TIME_REF pins,
+ * whose DPLLs (TXC and TSPLL) use MAC-derived clock_ids on E825.
+ *
+ * Return: true if clock_id matches one of this PF's registered DPLL devices.
+ */
+static bool ice_dpll_is_own_dpll_clock_id(struct ice_pf *pf, u64 clock_id)
+{
+	if (clock_id == pf->dplls.clock_id)
+		return true;
+	if (pf->hw.mac_type != ICE_MAC_GENERIC_3K_E825)
+		return false;
+	if (clock_id == ice_generate_dpll_clock_id(pf, &pf->dplls.txc,
+						   DPLL_TYPE_GENERIC))
+		return true;
+	return clock_id == ice_generate_dpll_clock_id(pf, &pf->dplls.tspll,
+						      DPLL_TYPE_GENERIC);
+}
+
 /**
  * ice_dpll_pin_ntf - notify pin change including any SW pin wrappers
  * @dplls: pointer to dplls struct
@@ -3134,12 +3353,44 @@ ice_dpll_update_state(struct ice_pf *pf, struct ice_dpll *d, bool init)
 	return ret;
 }
 
+/**
+ * ice_dpll_periodic_work_e825 - TSPLL DPLL periodic update for E825
+ * @pf: board private structure
+ *
+ * Publish TSPLL lock status to the dpll subsystem. The PTP periodic worker
+ * owns TSPLL lock polling and recovery; this function consumes the cached
+ * result for dpll notifications.
+ *
+ * Context: Must be called without pf->dplls.lock held.
+ */
+static void ice_dpll_periodic_work_e825(struct ice_pf *pf)
+{
+	struct ice_dpll *tp = &pf->dplls.tspll;
+	enum dpll_lock_status new_state;
+	bool changed = false;
+
+	mutex_lock(&pf->dplls.lock);
+	new_state = ice_dpll_tspll_lock_status_get(pf, true);
+
+	if (tp->prev_dpll_state != new_state) {
+		tp->dpll_state = new_state;
+		tp->prev_dpll_state = new_state;
+		changed = true;
+	}
+	mutex_unlock(&pf->dplls.lock);
+	if (changed)
+		dpll_device_change_ntf(tp->dpll);
+}
+
 /**
  * ice_dpll_periodic_work - DPLLs periodic worker
  * @work: pointer to kthread_work structure
  *
- * DPLLs periodic worker is responsible for polling state of dpll.
- * Context: Holds pf->dplls.lock
+ * Periodic worker responsible for polling DPLL state. On E810 devices it
+ * polls the EEC and PPS DPLLs. On E825 devices, when this PF owns the
+ * source timer, it publishes TSPLL lock status to the dpll subsystem.
+ *
+ * Context: Acquires and releases pf->dplls.lock
  */
 static void ice_dpll_periodic_work(struct kthread_work *work)
 {
@@ -3147,38 +3398,43 @@ static void ice_dpll_periodic_work(struct kthread_work *work)
 	struct ice_pf *pf = container_of(d, struct ice_pf, dplls);
 	struct ice_dpll *de = &pf->dplls.eec;
 	struct ice_dpll *dp = &pf->dplls.pps;
-	u32 phase_offset_ntf = 0;
+	u32 ntf_mask = 0;
 	int ret = 0;
 
 	if (ice_is_reset_in_progress(pf->state))
 		goto resched;
-	mutex_lock(&pf->dplls.lock);
-	d->periodic_counter++;
-	ret = ice_dpll_update_state(pf, de, false);
-	if (!ret)
-		ret = ice_dpll_update_state(pf, dp, false);
-	if (!ret && dp->phase_offset_monitor_period &&
-	    d->periodic_counter % dp->phase_offset_monitor_period == 0)
-		ret = ice_dpll_pps_update_phase_offsets(pf, &phase_offset_ntf);
-	if (ret) {
-		/* EBUSY is expected during reset recovery */
-		if (pf->hw.adminq.sq_last_status != LIBIE_AQ_RC_EBUSY)
-			d->cgu_state_acq_err_num++;
-		/* stop rescheduling this worker */
-		if (d->cgu_state_acq_err_num >
-		    ICE_CGU_STATE_ACQ_ERR_THRESHOLD) {
-			dev_err(ice_pf_to_dev(pf),
-				"EEC/PPS DPLLs periodic work disabled\n");
-			mutex_unlock(&pf->dplls.lock);
-			return;
+
+	if (pf->hw.mac_type == ICE_MAC_GENERIC_3K_E825) {
+		ice_dpll_periodic_work_e825(pf);
+	} else {
+		mutex_lock(&pf->dplls.lock);
+		d->periodic_counter++;
+		ret = ice_dpll_update_state(pf, de, false);
+		if (!ret)
+			ret = ice_dpll_update_state(pf, dp, false);
+		if (!ret && dp->phase_offset_monitor_period &&
+		    d->periodic_counter % dp->phase_offset_monitor_period == 0)
+			ret = ice_dpll_pps_update_phase_offsets(pf, &ntf_mask);
+		if (ret) {
+			/* EBUSY is expected during reset recovery */
+			if (pf->hw.adminq.sq_last_status != LIBIE_AQ_RC_EBUSY)
+				d->cgu_state_acq_err_num++;
+			/* stop rescheduling this worker */
+			if (d->cgu_state_acq_err_num >
+			    ICE_CGU_STATE_ACQ_ERR_THRESHOLD) {
+				dev_err(ice_pf_to_dev(pf),
+					"EEC/PPS DPLLs periodic work disabled\n");
+				mutex_unlock(&pf->dplls.lock);
+				return;
+			}
 		}
+		mutex_unlock(&pf->dplls.lock);
+		ice_dpll_notify_changes(de);
+		ice_dpll_notify_changes(dp);
+		if (ntf_mask)
+			ice_dpll_pins_notify_mask(d, d->inputs, d->num_inputs,
+						  ntf_mask);
 	}
-	mutex_unlock(&pf->dplls.lock);
-	ice_dpll_notify_changes(de);
-	ice_dpll_notify_changes(dp);
-	if (phase_offset_ntf)
-		ice_dpll_pins_notify_mask(d, d->inputs, d->num_inputs,
-					  phase_offset_ntf);
 
 resched:
 	/* Run twice a second or reschedule if update failed */
@@ -3536,6 +3792,7 @@ static void ice_dpll_pin_notify_work(struct work_struct *work)
 						   work);
 	struct ice_dpll_pin *pin, *parent = w->pin;
 	bool is_tx_synce_parent = false;
+	bool is_tspll_time_ref = false;
 	struct ice_pf *pf = parent->pf;
 	bool is_rclk_parent = false;
 	int ret;
@@ -3556,7 +3813,11 @@ static void ice_dpll_pin_notify_work(struct work_struct *work)
 	is_tx_synce_parent =
 		ice_dpll_fwnode_eq(parent->fwnode,
 				   pf->dplls.txclks[E825_EXT_SYNCE_PIN_IDX].fwnode);
-	if (!is_rclk_parent && !is_tx_synce_parent)
+
+	is_tspll_time_ref =
+		ice_dpll_fwnode_eq(parent->fwnode, pf->dplls.tspll_in.fwnode);
+
+	if (!is_rclk_parent && !is_tx_synce_parent && !is_tspll_time_ref)
 		goto out;
 
 	switch (w->action) {
@@ -3597,6 +3858,18 @@ static void ice_dpll_pin_notify_work(struct work_struct *work)
 					ERR_PTR(ret));
 				goto drop_parent_ref;
 			}
+		} else if (is_tspll_time_ref) {
+			/* Register TIME_REF pin directly to TSPLL DPLL */
+			ret = dpll_pin_register(pf->dplls.tspll.dpll,
+						parent->pin,
+						&ice_dpll_tspll_in_ops,
+						parent);
+			if (ret) {
+				dev_err(ice_pf_to_dev(pf),
+					"TSPLL TIME_REF pin register failed: %pe\n",
+					ERR_PTR(ret));
+				goto drop_parent_ref;
+			}
 		}
 		break;
 	case DPLL_PIN_DELETED:
@@ -3615,6 +3888,11 @@ static void ice_dpll_pin_notify_work(struct work_struct *work)
 			pin = &pf->dplls.txclks[E825_EXT_SYNCE_PIN_IDX];
 			dpll_pin_unregister(pf->dplls.txc.dpll, pin->pin,
 					    &ice_dpll_txclk_ops, pin);
+		} else if (is_tspll_time_ref) {
+			/* Unregister TIME_REF pin from TSPLL DPLL */
+			dpll_pin_unregister(pf->dplls.tspll.dpll,
+					    parent->pin,
+					    &ice_dpll_tspll_in_ops, parent);
 		}
 drop_parent_ref:
 		/* Drop fwnode pin reference */
@@ -3642,10 +3920,12 @@ static int ice_dpll_pin_notify(struct notifier_block *nb, unsigned long action,
 	if (pin->fwnode != info->fwnode)
 		return NOTIFY_DONE; /* Not this pin */
 
-	/* Ignore notification which are the outcome of internal pin
-	 * registration/unregistration calls - synce pin case.
+	/* Ignore notifications that are a side effect of internal pin
+	 * registration/unregistration calls. E825 uses per-device
+	 * MAC-derived clock_ids for the TXC and TSPLL generic DPLLs, so
+	 * info->src_clock_id may not equal pf->dplls.clock_id.
 	 */
-	if (info->src_clock_id == pin->pf->dplls.clock_id)
+	if (ice_dpll_is_own_dpll_clock_id(pin->pf, info->src_clock_id))
 		return NOTIFY_DONE;
 
 	work = kzalloc_obj(*work);
@@ -3786,12 +4066,12 @@ ice_dpll_deinit_fwnode_pins(struct ice_pf *pf, struct ice_dpll_pin *pins,
 	destroy_workqueue(pf->dplls.wq);
 }
 
-static int ice_dpll_deinit_txclk_pins(struct ice_pf *pf)
+static int ice_dpll_deinit_txclk_pins(struct ice_pf *pf, bool flush)
 {
 	struct ice_dpll_pin *synce_pin = &pf->dplls.txclks[E825_EXT_SYNCE_PIN_IDX];
 	struct ice_dpll *dt = &pf->dplls.txc;
 
-	ice_dpll_stop_fwnode_pin_activity(synce_pin, true);
+	ice_dpll_stop_fwnode_pin_activity(synce_pin, flush);
 	ice_dpll_unregister_pins(dt->dpll, pf->dplls.txclks,
 				 &ice_dpll_txclk_ops,
 				 ARRAY_SIZE(pf->dplls.txclks));
@@ -3806,6 +4086,25 @@ static int ice_dpll_deinit_txclk_pins(struct ice_pf *pf)
 	return 0;
 }
 
+/**
+ * ice_dpll_deinit_tspll_pins - deinitialize TSPLL fwnode pin
+ * @pf: board private structure
+ *
+ * Stop notifier activity and release the TIME_REF fwnode pin, unregistering
+ * it from the TSPLL DPLL if it was registered.
+ */
+static void ice_dpll_deinit_tspll_pins(struct ice_pf *pf)
+{
+	struct ice_dpll_pin *time_ref = &pf->dplls.tspll_in;
+	struct ice_dpll *tp = &pf->dplls.tspll;
+
+	ice_dpll_stop_fwnode_pin_activity(time_ref, true);
+	if (!IS_ERR_OR_NULL(time_ref->pin))
+		dpll_pin_unregister(tp->dpll, time_ref->pin,
+				    &ice_dpll_tspll_in_ops, time_ref);
+	ice_dpll_release_fwnode_pin(time_ref);
+}
+
 /**
  * ice_dpll_deinit_pins - deinitialize direct pins
  * @pf: board private structure
@@ -3826,7 +4125,10 @@ static void ice_dpll_deinit_pins(struct ice_pf *pf, bool cgu)
 
 	ice_dpll_deinit_rclk_pin(pf);
 	if (pf->hw.mac_type == ICE_MAC_GENERIC_3K_E825) {
-		ice_dpll_deinit_txclk_pins(pf);
+		if (ice_pf_src_tmr_owned(pf) &&
+		    test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
+			ice_dpll_deinit_tspll_pins(pf);
+		ice_dpll_deinit_txclk_pins(pf, true);
 		ice_dpll_deinit_fwnode_pins(pf, pf->dplls.inputs, 0);
 	}
 	if (cgu) {
@@ -3919,7 +4221,6 @@ ice_dpll_init_fwnode_pin(struct ice_dpll_pin *pin, const char *name)
  * @pf: board private structure
  * @pins: pointer to pins array
  * @start_idx: starting index for pins
- * @count: number of pins to initialize
  *
  * Initialize input pins for E825 RCLK support. The parent pins (rclk0, rclk1)
  * are expected to be defined by the system firmware (ACPI). This function
@@ -3968,10 +4269,16 @@ static int ice_dpll_init_txclk_pins(struct ice_pf *pf, int start_idx)
 {
 	struct ice_dpll_pin *ref_pin = pf->dplls.txclks;
 	struct ice_dpll *txc = &pf->dplls.txc;
+	u64 clock_id;
 	int ret;
 
+	/* EXT_EREF0 is a non-fwnode pin; its clock_id must match the TX-CLK
+	 * DPLL device clock_id (see dpll_pin_register()).
+	 */
+	clock_id = ice_generate_dpll_clock_id(pf, txc, DPLL_TYPE_GENERIC);
+
 	/* Configure EXT_EREF0 pin */
-	ret = ice_dpll_get_pins(pf, ref_pin, start_idx, 1, pf->dplls.clock_id);
+	ret = ice_dpll_get_pins(pf, ref_pin, start_idx, 1, clock_id);
 	if (ret)
 		return ret;
 	ret = dpll_pin_register(txc->dpll, ref_pin->pin, &ice_dpll_txclk_ops,
@@ -4022,10 +4329,48 @@ static int ice_dpll_init_txclk_pins(struct ice_pf *pf, int start_idx)
 	return ret;
 }
 
+/**
+ * ice_dpll_init_tspll_pins - init and register TSPLL TIME_REF fwnode pin
+ * @pf: board private structure
+ *
+ * Initialize the fwnode-based TIME_REF input pin for the TSPLL DPLL on E825.
+ * If the pin is not yet available in the dpll subsystem, registration will
+ * be deferred via the notifier path.
+ *
+ * Return:
+ * * 0 - success
+ * * negative - initialization failure reason
+ */
+static int ice_dpll_init_tspll_pins(struct ice_pf *pf)
+{
+	struct ice_dpll_pin *time_ref = &pf->dplls.tspll_in;
+	struct ice_dpll *tp = &pf->dplls.tspll;
+	int ret;
+
+	time_ref->pf = pf;
+	ret = ice_dpll_init_fwnode_pin(time_ref, ice_dpll_fwnode_time_ref);
+	if (ret)
+		return ret;
+
+	if (IS_ERR_OR_NULL(time_ref->pin)) {
+		dev_dbg(ice_pf_to_dev(pf),
+			"TSPLL TIME_REF pin not registered yet\n");
+		return 0;
+	}
+
+	ret = dpll_pin_register(tp->dpll, time_ref->pin,
+				&ice_dpll_tspll_in_ops, time_ref);
+	if (ret) {
+		ice_dpll_stop_fwnode_pin_activity(time_ref, false);
+		ice_dpll_release_fwnode_pin(time_ref);
+	}
+
+	return ret;
+}
+
 /**
  * ice_dpll_init_pins_e825 - init pins and register pins with a dplls
  * @pf: board private structure
- * @cgu: if cgu is present and controlled by this NIC
  *
  * Initialize directly connected pf's pins within pf's dplls in a Linux dpll
  * subsystem.
@@ -4049,8 +4394,24 @@ static int ice_dpll_init_pins_e825(struct ice_pf *pf)
 		goto unregister_pins;
 
 	ret = ice_dpll_init_txclk_pins(pf, 0);
-	if (ret)
+	if (ret) {
 		ice_dpll_deinit_rclk_pin(pf);
+		goto unregister_pins;
+	}
+
+	if (ice_pf_src_tmr_owned(pf) &&
+	    test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) {
+		ret = ice_dpll_init_tspll_pins(pf);
+		if (ret) {
+			/* Avoid flushing pf->dplls.wq here: notifier work
+			 * items block on pf->dplls.dpll_init which is
+			 * completed by the unregister_pins path below.
+			 * destroy_workqueue() there drains queued items.
+			 */
+			ice_dpll_deinit_txclk_pins(pf, false);
+			ice_dpll_deinit_rclk_pin(pf);
+		}
+	}
 
 unregister_pins:
 	if (ret) {
@@ -4194,7 +4555,7 @@ static int
 ice_dpll_init_dpll(struct ice_pf *pf, struct ice_dpll *d, bool cgu,
 		   enum dpll_type type)
 {
-	u64 clock_id = pf->dplls.clock_id;
+	u64 clock_id = ice_generate_dpll_clock_id(pf, d, type);
 	int ret;
 
 	d->dpll = dpll_device_get(clock_id, d->dpll_idx, THIS_MODULE,
@@ -4237,6 +4598,7 @@ static void ice_dpll_deinit_worker(struct ice_pf *pf)
 
 	kthread_cancel_delayed_work_sync(&d->work);
 	kthread_destroy_worker(d->kworker);
+	d->kworker = NULL;
 }
 
 /**
@@ -4673,6 +5035,7 @@ static void ice_dpll_deinit_info(struct ice_pf *pf)
 static int ice_dpll_init_info_e825c(struct ice_pf *pf)
 {
 	struct ice_dplls *d = &pf->dplls;
+	struct ice_dpll *tp = &d->tspll;
 	struct ice_dpll *dt = &d->txc;
 	int ret = 0;
 	int i;
@@ -4681,7 +5044,11 @@ static int ice_dpll_init_info_e825c(struct ice_pf *pf)
 	d->num_inputs = ICE_SYNCE_CLK_NUM;
 	dt->dpll_state = ice_txclk_lock_status(pf->ptp.port.tx_clk);
 	dt->mode = DPLL_MODE_MANUAL;
-	dt->dpll_idx = pf->ptp.port.port_num;
+	dt->dpll_idx = E825_DPLL_TXCLK_BASE_IDX + pf->ptp.port.port_num;
+	tp->dpll_state = ice_dpll_tspll_lock_status_get(pf, false);
+	tp->prev_dpll_state = tp->dpll_state;
+	tp->mode = DPLL_MODE_MANUAL;
+	tp->dpll_idx = E825_DPLL_TSPLL_BASE_IDX;
 
 	d->inputs = kzalloc_objs(*d->inputs, d->num_inputs);
 	if (!d->inputs)
@@ -4824,7 +5191,8 @@ static int ice_dpll_init_info(struct ice_pf *pf, bool cgu)
  * resources and unregistering the dpll, pin and all resources used for
  * handling them.
  *
- * Context: Destroys pf->dplls.lock mutex. Call only if ICE_FLAG_DPLL was set.
+ * Context: Call only if ICE_FLAG_DPLL was set. pf->dplls.lock lifetime is
+ * managed by ice_init_features()/ice_deinit_features().
  */
 void ice_dpll_deinit(struct ice_pf *pf)
 {
@@ -4848,7 +5216,7 @@ void ice_dpll_deinit(struct ice_pf *pf)
 		up_write(&pf->dplls.txclk_notify_rwsem);
 	}
 
-	if (cgu)
+	if (pf->dplls.kworker)
 		ice_dpll_deinit_worker(pf);
 
 	if (pf->hw.mac_type == ICE_MAC_GENERIC_3K_E825)
@@ -4861,9 +5229,10 @@ void ice_dpll_deinit(struct ice_pf *pf)
 		ice_dpll_deinit_dpll(pf, &pf->dplls.eec, cgu);
 	if (!IS_ERR_OR_NULL(pf->dplls.txc.dpll))
 		ice_dpll_deinit_dpll(pf, &pf->dplls.txc, false);
+	if (!IS_ERR_OR_NULL(pf->dplls.tspll.dpll))
+		ice_dpll_deinit_dpll(pf, &pf->dplls.tspll, false);
 
 	ice_dpll_deinit_info(pf);
-	mutex_destroy(&pf->dplls.lock);
 }
 
 /**
@@ -4873,8 +5242,6 @@ void ice_dpll_deinit(struct ice_pf *pf)
  * Set up the device dplls, register them and pins connected within Linux dpll
  * subsystem. Allow userspace to obtain state of DPLL and handling of DPLL
  * configuration requests.
- *
- * Context: Initializes pf->dplls.lock mutex.
  */
 static void ice_dpll_init_e825(struct ice_pf *pf)
 {
@@ -4892,7 +5259,6 @@ static void ice_dpll_init_e825(struct ice_pf *pf)
 			pf->ptp.port.port_num = pf->hw.lane_num;
 	}
 
-	mutex_init(&d->lock);
 	/* Initialize the txclk worker and its notification rwsem before any
 	 * code path can fail: ice_dpll_deinit() runs unconditionally on
 	 * failure and calls cancel_work_sync() / down_write() on these.
@@ -4904,9 +5270,30 @@ static void ice_dpll_init_e825(struct ice_pf *pf)
 	err = ice_dpll_init_info_e825c(pf);
 	if (err)
 		goto err_exit;
+	if (ice_pf_src_tmr_owned(pf) &&
+	    test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) {
+		err = ice_dpll_init_dpll(pf, &pf->dplls.tspll, false,
+					 DPLL_TYPE_GENERIC);
+		if (err)
+			goto deinit_info;
+		/* Start the periodic worker before registering fwnode pins. The
+		 * fwnode notifier path (ice_dpll_pin_notify_work()) blocks on
+		 * pf->dplls.dpll_init, which is only signalled on the success
+		 * return below. Starting the worker before any fwnode notifier
+		 * is registered means a subsequent init failure cannot leave
+		 * notifier work items queued on pf->dplls.wq, so the deinit
+		 * flush_workqueue() paths in ice_dpll_deinit_pins() cannot
+		 * deadlock. The E825 periodic worker only touches
+		 * pf->dplls.tspll (already registered above) and pf->ptp state,
+		 * so it is safe to run before pins.
+		 */
+		err = ice_dpll_init_worker(pf);
+		if (err)
+			goto deinit_tspll;
+	}
 	err = ice_dpll_init_dpll(pf, &pf->dplls.txc, false, DPLL_TYPE_GENERIC);
 	if (err)
-		goto deinit_info;
+		goto deinit_worker;
 	err = ice_dpll_init_pins_e825(pf);
 	if (err)
 		goto deinit_txclk;
@@ -4917,10 +5304,15 @@ static void ice_dpll_init_e825(struct ice_pf *pf)
 
 deinit_txclk:
 	ice_dpll_deinit_dpll(pf, &pf->dplls.txc, false);
+deinit_worker:
+	if (pf->dplls.kworker)
+		ice_dpll_deinit_worker(pf);
+deinit_tspll:
+	if (!IS_ERR_OR_NULL(pf->dplls.tspll.dpll))
+		ice_dpll_deinit_dpll(pf, &pf->dplls.tspll, false);
 deinit_info:
 	ice_dpll_deinit_info(pf);
 err_exit:
-	mutex_destroy(&d->lock);
 	dev_warn(ice_pf_to_dev(pf), "DPLLs init failure err:%d\n", err);
 }
 
@@ -4931,16 +5323,12 @@ static void ice_dpll_init_e825(struct ice_pf *pf)
  * Set up the device dplls, register them and pins connected within Linux dpll
  * subsystem. Allow userspace to obtain state of DPLL and handling of DPLL
  * configuration requests.
- *
- * Context: Initializes pf->dplls.lock mutex.
  */
 static void ice_dpll_init_e810(struct ice_pf *pf)
 {
 	bool cgu = ice_is_feature_supported(pf, ICE_F_CGU);
-	struct ice_dplls *d = &pf->dplls;
 	int err = 0;
 
-	mutex_init(&d->lock);
 	err = ice_dpll_init_info(pf, cgu);
 	if (err)
 		goto err_exit;
@@ -4971,7 +5359,6 @@ static void ice_dpll_init_e810(struct ice_pf *pf)
 deinit_info:
 	ice_dpll_deinit_info(pf);
 err_exit:
-	mutex_destroy(&d->lock);
 	dev_warn(ice_pf_to_dev(pf), "DPLLs init failure err:%d\n", err);
 }
 
diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.h b/drivers/net/ethernet/intel/ice/ice_dpll.h
index 103ba3e490682c..bce0bea9a664bc 100644
--- a/drivers/net/ethernet/intel/ice/ice_dpll.h
+++ b/drivers/net/ethernet/intel/ice/ice_dpll.h
@@ -133,10 +133,12 @@ struct ice_dpll {
  * @eec: pointer to EEC dpll dev
  * @pps: pointer to PPS dpll dev
  * @txc: pointer to TXC dpll dev
+ * @tspll: pointer to TSPLL dpll dev (E825 only)
  * @inputs: input pins pointer
  * @outputs: output pins pointer
  * @rclk: recovered pins pointer
  * @txclks: TX clock reference pins pointer
+ * @tspll_in: TSPLL TIME_REF fwnode input pin (E825 only)
  * @num_inputs: number of input pins available on dpll
  * @num_outputs: number of output pins available on dpll
  * @cgu_state_acq_err_num: number of errors returned during periodic work
@@ -177,12 +179,14 @@ struct ice_dplls {
 	struct ice_dpll eec;
 	struct ice_dpll pps;
 	struct ice_dpll txc;
+	struct ice_dpll tspll;
 	struct ice_dpll_pin *inputs;
 	struct ice_dpll_pin *outputs;
 	struct ice_dpll_pin sma[ICE_DPLL_PIN_SW_NUM];
 	struct ice_dpll_pin ufl[ICE_DPLL_PIN_SW_NUM];
 	struct ice_dpll_pin rclk;
 	struct ice_dpll_pin txclks[ICE_DPLL_TXCLK_NUM_MAX];
+	struct ice_dpll_pin tspll_in;
 	u8 num_inputs;
 	u8 num_outputs;
 	u8 sma_data;
diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c
index bf9a821c543b7f..dffa213734f100 100644
--- a/drivers/net/ethernet/intel/ice/ice_ethtool.c
+++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c
@@ -1751,10 +1751,7 @@ static int ice_set_priv_flags(struct net_device *netdev, u32 flags)
 
 		priv_flag = &ice_gstrings_priv_flags[i];
 
-		if (flags & BIT(i))
-			set_bit(priv_flag->bitno, pf->flags);
-		else
-			clear_bit(priv_flag->bitno, pf->flags);
+		assign_bit(priv_flag->bitno, pf->flags, flags & BIT(i));
 	}
 
 	bitmap_xor(change_flags, pf->flags, orig_flags, ICE_PF_FLAGS_NBITS);
@@ -3088,9 +3085,9 @@ static int ice_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
 
 	switch (cmd->cmd) {
 	case ETHTOOL_SRXCLSRLINS:
-		return ice_add_fdir_ethtool(vsi, cmd);
+		return ice_add_ntuple_ethtool(vsi, cmd);
 	case ETHTOOL_SRXCLSRLDEL:
-		return ice_del_fdir_ethtool(vsi, cmd);
+		return ice_del_ntuple_ethtool(vsi, cmd);
 	default:
 		break;
 	}
@@ -3132,7 +3129,7 @@ ice_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
 
 	switch (cmd->cmd) {
 	case ETHTOOL_GRXCLSRLCNT:
-		cmd->rule_cnt = hw->fdir_active_fltr;
+		cmd->rule_cnt = hw->ntuple_active_fltr_cnt;
 		/* report total rule count */
 		cmd->data = ice_get_fdir_cnt_all(hw);
 		ret = 0;
@@ -3908,7 +3905,8 @@ static int ice_set_channels(struct net_device *dev, struct ethtool_channels *ch)
 		return -EOPNOTSUPP;
 	}
 
-	if (test_bit(ICE_FLAG_FD_ENA, pf->flags) && pf->hw.fdir_active_fltr) {
+	if (test_bit(ICE_FLAG_FD_ENA, pf->flags) &&
+	    pf->hw.ntuple_active_fltr_cnt) {
 		netdev_err(dev, "Cannot set channels when Flow Director filters are active\n");
 		return -EOPNOTSUPP;
 	}
diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c b/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c
deleted file mode 100644
index aceec184e89b2f..00000000000000
--- a/drivers/net/ethernet/intel/ice/ice_ethtool_fdir.c
+++ /dev/null
@@ -1,2080 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Copyright (C) 2018-2023, Intel Corporation. */
-
-/* flow director ethtool support for ice */
-
-#include "ice.h"
-#include "ice_lib.h"
-#include "ice_fdir.h"
-#include "ice_flow.h"
-
-static struct in6_addr full_ipv6_addr_mask = {
-	.in6_u = {
-		.u6_addr8 = {
-			0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
-			0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
-		}
-	}
-};
-
-static struct in6_addr zero_ipv6_addr_mask = {
-	.in6_u = {
-		.u6_addr8 = {
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		}
-	}
-};
-
-/* calls to ice_flow_add_prof require the number of segments in the array
- * for segs_cnt. In this code that is one more than the index.
- */
-#define TNL_SEG_CNT(_TNL_) ((_TNL_) + 1)
-
-/**
- * ice_fltr_to_ethtool_flow - convert filter type values to ethtool
- * flow type values
- * @flow: filter type to be converted
- *
- * Returns the corresponding ethtool flow type.
- */
-static int ice_fltr_to_ethtool_flow(enum ice_fltr_ptype flow)
-{
-	switch (flow) {
-	case ICE_FLTR_PTYPE_NONF_ETH:
-		return ETHER_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV4_TCP:
-		return TCP_V4_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV4_UDP:
-		return UDP_V4_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV4_SCTP:
-		return SCTP_V4_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV4_OTHER:
-		return IPV4_USER_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV6_TCP:
-		return TCP_V6_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV6_UDP:
-		return UDP_V6_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV6_SCTP:
-		return SCTP_V6_FLOW;
-	case ICE_FLTR_PTYPE_NONF_IPV6_OTHER:
-		return IPV6_USER_FLOW;
-	default:
-		/* 0 is undefined ethtool flow */
-		return 0;
-	}
-}
-
-/**
- * ice_ethtool_flow_to_fltr - convert ethtool flow type to filter enum
- * @eth: Ethtool flow type to be converted
- *
- * Returns flow enum
- */
-static enum ice_fltr_ptype ice_ethtool_flow_to_fltr(int eth)
-{
-	switch (eth) {
-	case ETHER_FLOW:
-		return ICE_FLTR_PTYPE_NONF_ETH;
-	case TCP_V4_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV4_TCP;
-	case UDP_V4_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV4_UDP;
-	case SCTP_V4_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV4_SCTP;
-	case IPV4_USER_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV4_OTHER;
-	case TCP_V6_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV6_TCP;
-	case UDP_V6_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV6_UDP;
-	case SCTP_V6_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV6_SCTP;
-	case IPV6_USER_FLOW:
-		return ICE_FLTR_PTYPE_NONF_IPV6_OTHER;
-	default:
-		return ICE_FLTR_PTYPE_NONF_NONE;
-	}
-}
-
-/**
- * ice_is_mask_valid - check mask field set
- * @mask: full mask to check
- * @field: field for which mask should be valid
- *
- * If the mask is fully set return true. If it is not valid for field return
- * false.
- */
-static bool ice_is_mask_valid(u64 mask, u64 field)
-{
-	return (mask & field) == field;
-}
-
-/**
- * ice_get_ethtool_fdir_entry - fill ethtool structure with fdir filter data
- * @hw: hardware structure that contains filter list
- * @cmd: ethtool command data structure to receive the filter data
- *
- * Returns 0 on success and -EINVAL on failure
- */
-int ice_get_ethtool_fdir_entry(struct ice_hw *hw, struct ethtool_rxnfc *cmd)
-{
-	struct ethtool_rx_flow_spec *fsp;
-	struct ice_fdir_fltr *rule;
-	int ret = 0;
-	u16 idx;
-
-	fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
-
-	mutex_lock(&hw->fdir_fltr_lock);
-
-	rule = ice_fdir_find_fltr_by_idx(hw, fsp->location);
-
-	if (!rule || fsp->location != rule->fltr_id) {
-		ret = -EINVAL;
-		goto release_lock;
-	}
-
-	fsp->flow_type = ice_fltr_to_ethtool_flow(rule->flow_type);
-
-	memset(&fsp->m_u, 0, sizeof(fsp->m_u));
-	memset(&fsp->m_ext, 0, sizeof(fsp->m_ext));
-
-	switch (fsp->flow_type) {
-	case ETHER_FLOW:
-		fsp->h_u.ether_spec = rule->eth;
-		fsp->m_u.ether_spec = rule->eth_mask;
-		break;
-	case IPV4_USER_FLOW:
-		fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4;
-		fsp->h_u.usr_ip4_spec.proto = 0;
-		fsp->h_u.usr_ip4_spec.l4_4_bytes = rule->ip.v4.l4_header;
-		fsp->h_u.usr_ip4_spec.tos = rule->ip.v4.tos;
-		fsp->h_u.usr_ip4_spec.ip4src = rule->ip.v4.src_ip;
-		fsp->h_u.usr_ip4_spec.ip4dst = rule->ip.v4.dst_ip;
-		fsp->m_u.usr_ip4_spec.ip4src = rule->mask.v4.src_ip;
-		fsp->m_u.usr_ip4_spec.ip4dst = rule->mask.v4.dst_ip;
-		fsp->m_u.usr_ip4_spec.ip_ver = 0xFF;
-		fsp->m_u.usr_ip4_spec.proto = 0;
-		fsp->m_u.usr_ip4_spec.l4_4_bytes = rule->mask.v4.l4_header;
-		fsp->m_u.usr_ip4_spec.tos = rule->mask.v4.tos;
-		break;
-	case TCP_V4_FLOW:
-	case UDP_V4_FLOW:
-	case SCTP_V4_FLOW:
-		fsp->h_u.tcp_ip4_spec.psrc = rule->ip.v4.src_port;
-		fsp->h_u.tcp_ip4_spec.pdst = rule->ip.v4.dst_port;
-		fsp->h_u.tcp_ip4_spec.ip4src = rule->ip.v4.src_ip;
-		fsp->h_u.tcp_ip4_spec.ip4dst = rule->ip.v4.dst_ip;
-		fsp->m_u.tcp_ip4_spec.psrc = rule->mask.v4.src_port;
-		fsp->m_u.tcp_ip4_spec.pdst = rule->mask.v4.dst_port;
-		fsp->m_u.tcp_ip4_spec.ip4src = rule->mask.v4.src_ip;
-		fsp->m_u.tcp_ip4_spec.ip4dst = rule->mask.v4.dst_ip;
-		break;
-	case IPV6_USER_FLOW:
-		fsp->h_u.usr_ip6_spec.l4_4_bytes = rule->ip.v6.l4_header;
-		fsp->h_u.usr_ip6_spec.tclass = rule->ip.v6.tc;
-		fsp->h_u.usr_ip6_spec.l4_proto = rule->ip.v6.proto;
-		memcpy(fsp->h_u.tcp_ip6_spec.ip6src, rule->ip.v6.src_ip,
-		       sizeof(struct in6_addr));
-		memcpy(fsp->h_u.tcp_ip6_spec.ip6dst, rule->ip.v6.dst_ip,
-		       sizeof(struct in6_addr));
-		memcpy(fsp->m_u.tcp_ip6_spec.ip6src, rule->mask.v6.src_ip,
-		       sizeof(struct in6_addr));
-		memcpy(fsp->m_u.tcp_ip6_spec.ip6dst, rule->mask.v6.dst_ip,
-		       sizeof(struct in6_addr));
-		fsp->m_u.usr_ip6_spec.l4_4_bytes = rule->mask.v6.l4_header;
-		fsp->m_u.usr_ip6_spec.tclass = rule->mask.v6.tc;
-		fsp->m_u.usr_ip6_spec.l4_proto = rule->mask.v6.proto;
-		break;
-	case TCP_V6_FLOW:
-	case UDP_V6_FLOW:
-	case SCTP_V6_FLOW:
-		memcpy(fsp->h_u.tcp_ip6_spec.ip6src, rule->ip.v6.src_ip,
-		       sizeof(struct in6_addr));
-		memcpy(fsp->h_u.tcp_ip6_spec.ip6dst, rule->ip.v6.dst_ip,
-		       sizeof(struct in6_addr));
-		fsp->h_u.tcp_ip6_spec.psrc = rule->ip.v6.src_port;
-		fsp->h_u.tcp_ip6_spec.pdst = rule->ip.v6.dst_port;
-		memcpy(fsp->m_u.tcp_ip6_spec.ip6src,
-		       rule->mask.v6.src_ip,
-		       sizeof(struct in6_addr));
-		memcpy(fsp->m_u.tcp_ip6_spec.ip6dst,
-		       rule->mask.v6.dst_ip,
-		       sizeof(struct in6_addr));
-		fsp->m_u.tcp_ip6_spec.psrc = rule->mask.v6.src_port;
-		fsp->m_u.tcp_ip6_spec.pdst = rule->mask.v6.dst_port;
-		fsp->h_u.tcp_ip6_spec.tclass = rule->ip.v6.tc;
-		fsp->m_u.tcp_ip6_spec.tclass = rule->mask.v6.tc;
-		break;
-	default:
-		break;
-	}
-
-	if (rule->dest_ctl == ICE_FLTR_PRGM_DESC_DEST_DROP_PKT)
-		fsp->ring_cookie = RX_CLS_FLOW_DISC;
-	else
-		fsp->ring_cookie = rule->orig_q_index;
-
-	idx = ice_ethtool_flow_to_fltr(fsp->flow_type);
-	if (idx == ICE_FLTR_PTYPE_NONF_NONE) {
-		dev_err(ice_hw_to_dev(hw), "Missing input index for flow_type %d\n",
-			rule->flow_type);
-		ret = -EINVAL;
-	}
-
-release_lock:
-	mutex_unlock(&hw->fdir_fltr_lock);
-	return ret;
-}
-
-/**
- * ice_get_fdir_fltr_ids - fill buffer with filter IDs of active filters
- * @hw: hardware structure containing the filter list
- * @cmd: ethtool command data structure
- * @rule_locs: ethtool array passed in from OS to receive filter IDs
- *
- * Returns 0 as expected for success by ethtool
- */
-int
-ice_get_fdir_fltr_ids(struct ice_hw *hw, struct ethtool_rxnfc *cmd,
-		      u32 *rule_locs)
-{
-	struct ice_fdir_fltr *f_rule;
-	unsigned int cnt = 0;
-	int val = 0;
-
-	/* report total rule count */
-	cmd->data = ice_get_fdir_cnt_all(hw);
-
-	mutex_lock(&hw->fdir_fltr_lock);
-
-	list_for_each_entry(f_rule, &hw->fdir_list_head, fltr_node) {
-		if (cnt == cmd->rule_cnt) {
-			val = -EMSGSIZE;
-			goto release_lock;
-		}
-		rule_locs[cnt] = f_rule->fltr_id;
-		cnt++;
-	}
-
-release_lock:
-	mutex_unlock(&hw->fdir_fltr_lock);
-	if (!val)
-		cmd->rule_cnt = cnt;
-	return val;
-}
-
-/**
- * ice_fdir_remap_entries - update the FDir entries in profile
- * @prof: FDir structure pointer
- * @tun: tunneled or non-tunneled packet
- * @idx: FDir entry index
- */
-static void
-ice_fdir_remap_entries(struct ice_fd_hw_prof *prof, int tun, int idx)
-{
-	if (idx != prof->cnt && tun < ICE_FD_HW_SEG_MAX) {
-		int i;
-
-		for (i = idx; i < (prof->cnt - 1); i++) {
-			u64 old_entry_h;
-
-			old_entry_h = prof->entry_h[i + 1][tun];
-			prof->entry_h[i][tun] = old_entry_h;
-			prof->vsi_h[i] = prof->vsi_h[i + 1];
-		}
-
-		prof->entry_h[i][tun] = 0;
-		prof->vsi_h[i] = 0;
-	}
-}
-
-/**
- * ice_fdir_rem_adq_chnl - remove an ADQ channel from HW filter rules
- * @hw: hardware structure containing filter list
- * @vsi_idx: VSI handle
- */
-void ice_fdir_rem_adq_chnl(struct ice_hw *hw, u16 vsi_idx)
-{
-	int status, flow;
-
-	if (!hw->fdir_prof)
-		return;
-
-	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) {
-		struct ice_fd_hw_prof *prof = hw->fdir_prof[flow];
-		int tun, i;
-
-		if (!prof || !prof->cnt)
-			continue;
-
-		for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
-			u64 prof_id = prof->prof_id[tun];
-
-			for (i = 0; i < prof->cnt; i++) {
-				if (prof->vsi_h[i] != vsi_idx)
-					continue;
-
-				prof->entry_h[i][tun] = 0;
-				prof->vsi_h[i] = 0;
-				break;
-			}
-
-			/* after clearing FDir entries update the remaining */
-			ice_fdir_remap_entries(prof, tun, i);
-
-			/* find flow profile corresponding to prof_id and clear
-			 * vsi_idx from bitmap.
-			 */
-			status = ice_flow_rem_vsi_prof(hw, vsi_idx, prof_id);
-			if (status) {
-				dev_err(ice_hw_to_dev(hw), "ice_flow_rem_vsi_prof() failed status=%d\n",
-					status);
-			}
-		}
-		prof->cnt--;
-	}
-}
-
-/**
- * ice_fdir_get_hw_prof - return the ice_fd_hw_proc associated with a flow
- * @hw: hardware structure containing the filter list
- * @blk: hardware block
- * @flow: FDir flow type to release
- */
-static struct ice_fd_hw_prof *
-ice_fdir_get_hw_prof(struct ice_hw *hw, enum ice_block blk, int flow)
-{
-	if (blk == ICE_BLK_FD && hw->fdir_prof)
-		return hw->fdir_prof[flow];
-
-	return NULL;
-}
-
-/**
- * ice_fdir_erase_flow_from_hw - remove a flow from the HW profile tables
- * @hw: hardware structure containing the filter list
- * @blk: hardware block
- * @flow: FDir flow type to release
- */
-static void
-ice_fdir_erase_flow_from_hw(struct ice_hw *hw, enum ice_block blk, int flow)
-{
-	struct ice_fd_hw_prof *prof = ice_fdir_get_hw_prof(hw, blk, flow);
-	int tun;
-
-	if (!prof)
-		return;
-
-	for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
-		u64 prof_id = prof->prof_id[tun];
-		int j;
-
-		for (j = 0; j < prof->cnt; j++) {
-			u16 vsi_num;
-
-			if (!prof->entry_h[j][tun] || !prof->vsi_h[j])
-				continue;
-			vsi_num = ice_get_hw_vsi_num(hw, prof->vsi_h[j]);
-			ice_rem_prof_id_flow(hw, blk, vsi_num, prof_id);
-			ice_flow_rem_entry(hw, blk, prof->entry_h[j][tun]);
-			prof->entry_h[j][tun] = 0;
-		}
-		ice_flow_rem_prof(hw, blk, prof_id);
-	}
-}
-
-/**
- * ice_fdir_rem_flow - release the ice_flow structures for a filter type
- * @hw: hardware structure containing the filter list
- * @blk: hardware block
- * @flow_type: FDir flow type to release
- */
-static void
-ice_fdir_rem_flow(struct ice_hw *hw, enum ice_block blk,
-		  enum ice_fltr_ptype flow_type)
-{
-	int flow = (int)flow_type & ~FLOW_EXT;
-	struct ice_fd_hw_prof *prof;
-	int tun, i;
-
-	prof = ice_fdir_get_hw_prof(hw, blk, flow);
-	if (!prof)
-		return;
-
-	ice_fdir_erase_flow_from_hw(hw, blk, flow);
-	for (i = 0; i < prof->cnt; i++)
-		prof->vsi_h[i] = 0;
-	for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
-		if (!prof->fdir_seg[tun])
-			continue;
-		devm_kfree(ice_hw_to_dev(hw), prof->fdir_seg[tun]);
-		prof->fdir_seg[tun] = NULL;
-	}
-	prof->cnt = 0;
-}
-
-/**
- * ice_fdir_release_flows - release all flows in use for later replay
- * @hw: pointer to HW instance
- */
-void ice_fdir_release_flows(struct ice_hw *hw)
-{
-	int flow;
-
-	/* release Flow Director HW table entries */
-	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++)
-		ice_fdir_erase_flow_from_hw(hw, ICE_BLK_FD, flow);
-}
-
-/**
- * ice_fdir_replay_flows - replay HW Flow Director filter info
- * @hw: pointer to HW instance
- */
-void ice_fdir_replay_flows(struct ice_hw *hw)
-{
-	int flow;
-
-	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) {
-		int tun;
-
-		if (!hw->fdir_prof[flow] || !hw->fdir_prof[flow]->cnt)
-			continue;
-		for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
-			struct ice_flow_prof *hw_prof;
-			struct ice_fd_hw_prof *prof;
-			int j;
-
-			prof = hw->fdir_prof[flow];
-			ice_flow_add_prof(hw, ICE_BLK_FD, ICE_FLOW_RX,
-					  prof->fdir_seg[tun], TNL_SEG_CNT(tun),
-					  false, &hw_prof);
-			for (j = 0; j < prof->cnt; j++) {
-				enum ice_flow_priority prio;
-				u64 entry_h = 0;
-				int err;
-
-				prio = ICE_FLOW_PRIO_NORMAL;
-				err = ice_flow_add_entry(hw, ICE_BLK_FD,
-							 hw_prof->id,
-							 prof->vsi_h[0],
-							 prof->vsi_h[j],
-							 prio, prof->fdir_seg,
-							 &entry_h);
-				if (err) {
-					dev_err(ice_hw_to_dev(hw), "Could not replay Flow Director, flow type %d\n",
-						flow);
-					continue;
-				}
-				prof->prof_id[tun] = hw_prof->id;
-				prof->entry_h[j][tun] = entry_h;
-			}
-		}
-	}
-}
-
-/**
- * ice_parse_rx_flow_user_data - deconstruct user-defined data
- * @fsp: pointer to ethtool Rx flow specification
- * @data: pointer to userdef data structure for storage
- *
- * Returns 0 on success, negative error value on failure
- */
-static int
-ice_parse_rx_flow_user_data(struct ethtool_rx_flow_spec *fsp,
-			    struct ice_rx_flow_userdef *data)
-{
-	u64 value, mask;
-
-	memset(data, 0, sizeof(*data));
-	if (!(fsp->flow_type & FLOW_EXT))
-		return 0;
-
-	value = be64_to_cpu(*((__force __be64 *)fsp->h_ext.data));
-	mask = be64_to_cpu(*((__force __be64 *)fsp->m_ext.data));
-	if (!mask)
-		return 0;
-
-#define ICE_USERDEF_FLEX_WORD_M	GENMASK_ULL(15, 0)
-#define ICE_USERDEF_FLEX_OFFS_S	16
-#define ICE_USERDEF_FLEX_OFFS_M	GENMASK_ULL(31, ICE_USERDEF_FLEX_OFFS_S)
-#define ICE_USERDEF_FLEX_FLTR_M	GENMASK_ULL(31, 0)
-
-	/* 0x1fe is the maximum value for offsets stored in the internal
-	 * filtering tables.
-	 */
-#define ICE_USERDEF_FLEX_MAX_OFFS_VAL 0x1fe
-
-	if (!ice_is_mask_valid(mask, ICE_USERDEF_FLEX_FLTR_M) ||
-	    value > ICE_USERDEF_FLEX_FLTR_M)
-		return -EINVAL;
-
-	data->flex_word = value & ICE_USERDEF_FLEX_WORD_M;
-	data->flex_offset = FIELD_GET(ICE_USERDEF_FLEX_OFFS_M, value);
-	if (data->flex_offset > ICE_USERDEF_FLEX_MAX_OFFS_VAL)
-		return -EINVAL;
-
-	data->flex_fltr = true;
-
-	return 0;
-}
-
-/**
- * ice_fdir_num_avail_fltr - return the number of unused flow director filters
- * @hw: pointer to hardware structure
- * @vsi: software VSI structure
- *
- * There are 2 filter pools: guaranteed and best effort(shared). Each VSI can
- * use filters from either pool. The guaranteed pool is divided between VSIs.
- * The best effort filter pool is common to all VSIs and is a device shared
- * resource pool. The number of filters available to this VSI is the sum of
- * the VSIs guaranteed filter pool and the global available best effort
- * filter pool.
- *
- * Returns the number of available flow director filters to this VSI
- */
-int ice_fdir_num_avail_fltr(struct ice_hw *hw, struct ice_vsi *vsi)
-{
-	u16 vsi_num = ice_get_hw_vsi_num(hw, vsi->idx);
-	u16 num_guar;
-	u16 num_be;
-
-	/* total guaranteed filters assigned to this VSI */
-	num_guar = vsi->num_gfltr;
-
-	/* total global best effort filters */
-	num_be = hw->func_caps.fd_fltr_best_effort;
-
-	/* Subtract the number of programmed filters from the global values */
-	switch (hw->mac_type) {
-	case ICE_MAC_E830:
-		num_guar -= FIELD_GET(E830_VSIQF_FD_CNT_FD_GCNT_M,
-				      rd32(hw, VSIQF_FD_CNT(vsi_num)));
-		num_be -= FIELD_GET(E830_GLQF_FD_CNT_FD_BCNT_M,
-				    rd32(hw, GLQF_FD_CNT));
-		break;
-	case ICE_MAC_E810:
-	default:
-		num_guar -= FIELD_GET(E800_VSIQF_FD_CNT_FD_GCNT_M,
-				      rd32(hw, VSIQF_FD_CNT(vsi_num)));
-		num_be -= FIELD_GET(E800_GLQF_FD_CNT_FD_BCNT_M,
-				    rd32(hw, GLQF_FD_CNT));
-	}
-
-	return num_guar + num_be;
-}
-
-/**
- * ice_fdir_alloc_flow_prof - allocate FDir flow profile structure(s)
- * @hw: HW structure containing the FDir flow profile structure(s)
- * @flow: flow type to allocate the flow profile for
- *
- * Allocate the fdir_prof and fdir_prof[flow] if not already created. Return 0
- * on success and negative on error.
- */
-static int
-ice_fdir_alloc_flow_prof(struct ice_hw *hw, enum ice_fltr_ptype flow)
-{
-	if (!hw)
-		return -EINVAL;
-
-	if (!hw->fdir_prof) {
-		hw->fdir_prof = devm_kcalloc(ice_hw_to_dev(hw),
-					     ICE_FLTR_PTYPE_MAX,
-					     sizeof(*hw->fdir_prof),
-					     GFP_KERNEL);
-		if (!hw->fdir_prof)
-			return -ENOMEM;
-	}
-
-	if (!hw->fdir_prof[flow]) {
-		hw->fdir_prof[flow] = devm_kzalloc(ice_hw_to_dev(hw),
-						   sizeof(**hw->fdir_prof),
-						   GFP_KERNEL);
-		if (!hw->fdir_prof[flow])
-			return -ENOMEM;
-	}
-
-	return 0;
-}
-
-/**
- * ice_fdir_prof_vsi_idx - find or insert a vsi_idx in structure
- * @prof: pointer to flow director HW profile
- * @vsi_idx: vsi_idx to locate
- *
- * return the index of the vsi_idx. if vsi_idx is not found insert it
- * into the vsi_h table.
- */
-static u16
-ice_fdir_prof_vsi_idx(struct ice_fd_hw_prof *prof, int vsi_idx)
-{
-	u16 idx = 0;
-
-	for (idx = 0; idx < prof->cnt; idx++)
-		if (prof->vsi_h[idx] == vsi_idx)
-			return idx;
-
-	if (idx == prof->cnt)
-		prof->vsi_h[prof->cnt++] = vsi_idx;
-	return idx;
-}
-
-/**
- * ice_fdir_set_hw_fltr_rule - Configure HW tables to generate a FDir rule
- * @pf: pointer to the PF structure
- * @seg: protocol header description pointer
- * @flow: filter enum
- * @tun: FDir segment to program
- */
-static int
-ice_fdir_set_hw_fltr_rule(struct ice_pf *pf, struct ice_flow_seg_info *seg,
-			  enum ice_fltr_ptype flow, enum ice_fd_hw_seg tun)
-{
-	struct device *dev = ice_pf_to_dev(pf);
-	struct ice_vsi *main_vsi, *ctrl_vsi;
-	struct ice_flow_seg_info *old_seg;
-	struct ice_flow_prof *prof = NULL;
-	struct ice_fd_hw_prof *hw_prof;
-	struct ice_hw *hw = &pf->hw;
-	u64 entry1_h = 0;
-	u64 entry2_h = 0;
-	bool del_last;
-	int err;
-	int idx;
-
-	main_vsi = ice_get_main_vsi(pf);
-	if (!main_vsi)
-		return -EINVAL;
-
-	ctrl_vsi = ice_get_ctrl_vsi(pf);
-	if (!ctrl_vsi)
-		return -EINVAL;
-
-	err = ice_fdir_alloc_flow_prof(hw, flow);
-	if (err)
-		return err;
-
-	hw_prof = hw->fdir_prof[flow];
-	old_seg = hw_prof->fdir_seg[tun];
-	if (old_seg) {
-		/* This flow_type already has a changed input set.
-		 * If it matches the requested input set then we are
-		 * done. Or, if it's different then it's an error.
-		 */
-		if (!memcmp(old_seg, seg, sizeof(*seg)))
-			return -EEXIST;
-
-		/* if there are FDir filters using this flow,
-		 * then return error.
-		 */
-		if (hw->fdir_fltr_cnt[flow]) {
-			dev_err(dev, "Failed to add filter. Flow director filters on each port must have the same input set.\n");
-			return -EINVAL;
-		}
-
-		if (ice_is_arfs_using_perfect_flow(hw, flow)) {
-			dev_err(dev, "aRFS using perfect flow type %d, cannot change input set\n",
-				flow);
-			return -EINVAL;
-		}
-
-		/* remove HW filter definition */
-		ice_fdir_rem_flow(hw, ICE_BLK_FD, flow);
-	}
-
-	/* Adding a profile, but there is only one header supported.
-	 * That is the final parameters are 1 header (segment), no
-	 * actions (NULL) and zero actions 0.
-	 */
-	err = ice_flow_add_prof(hw, ICE_BLK_FD, ICE_FLOW_RX, seg,
-				TNL_SEG_CNT(tun), false, &prof);
-	if (err)
-		return err;
-	err = ice_flow_add_entry(hw, ICE_BLK_FD, prof->id, main_vsi->idx,
-				 main_vsi->idx, ICE_FLOW_PRIO_NORMAL,
-				 seg, &entry1_h);
-	if (err)
-		goto err_prof;
-	err = ice_flow_add_entry(hw, ICE_BLK_FD, prof->id, main_vsi->idx,
-				 ctrl_vsi->idx, ICE_FLOW_PRIO_NORMAL,
-				 seg, &entry2_h);
-	if (err)
-		goto err_entry;
-
-	hw_prof->fdir_seg[tun] = seg;
-	hw_prof->prof_id[tun] = prof->id;
-	hw_prof->entry_h[0][tun] = entry1_h;
-	hw_prof->entry_h[1][tun] = entry2_h;
-	hw_prof->vsi_h[0] = main_vsi->idx;
-	hw_prof->vsi_h[1] = ctrl_vsi->idx;
-	if (!hw_prof->cnt)
-		hw_prof->cnt = 2;
-
-	for (idx = 1; idx < ICE_CHNL_MAX_TC; idx++) {
-		u16 vsi_idx;
-		u16 vsi_h;
-
-		if (!ice_is_adq_active(pf) || !main_vsi->tc_map_vsi[idx])
-			continue;
-
-		entry1_h = 0;
-		vsi_h = main_vsi->tc_map_vsi[idx]->idx;
-		err = ice_flow_add_entry(hw, ICE_BLK_FD, prof->id,
-					 main_vsi->idx, vsi_h,
-					 ICE_FLOW_PRIO_NORMAL, seg,
-					 &entry1_h);
-		if (err) {
-			dev_err(dev, "Could not add Channel VSI %d to flow group\n",
-				idx);
-			goto err_unroll;
-		}
-
-		vsi_idx = ice_fdir_prof_vsi_idx(hw_prof,
-						main_vsi->tc_map_vsi[idx]->idx);
-		hw_prof->entry_h[vsi_idx][tun] = entry1_h;
-	}
-
-	return 0;
-
-err_unroll:
-	entry1_h = 0;
-	hw_prof->fdir_seg[tun] = NULL;
-
-	/* The variable del_last will be used to determine when to clean up
-	 * the VSI group data. The VSI data is not needed if there are no
-	 * segments.
-	 */
-	del_last = true;
-	for (idx = 0; idx < ICE_FD_HW_SEG_MAX; idx++)
-		if (hw_prof->fdir_seg[idx]) {
-			del_last = false;
-			break;
-		}
-
-	for (idx = 0; idx < hw_prof->cnt; idx++) {
-		u16 vsi_num = ice_get_hw_vsi_num(hw, hw_prof->vsi_h[idx]);
-
-		if (!hw_prof->entry_h[idx][tun])
-			continue;
-		ice_rem_prof_id_flow(hw, ICE_BLK_FD, vsi_num, prof->id);
-		ice_flow_rem_entry(hw, ICE_BLK_FD, hw_prof->entry_h[idx][tun]);
-		hw_prof->entry_h[idx][tun] = 0;
-		if (del_last)
-			hw_prof->vsi_h[idx] = 0;
-	}
-	if (del_last)
-		hw_prof->cnt = 0;
-err_entry:
-	ice_rem_prof_id_flow(hw, ICE_BLK_FD,
-			     ice_get_hw_vsi_num(hw, main_vsi->idx), prof->id);
-	ice_flow_rem_entry(hw, ICE_BLK_FD, entry1_h);
-err_prof:
-	ice_flow_rem_prof(hw, ICE_BLK_FD, prof->id);
-	dev_err(dev, "Failed to add filter. Flow director filters on each port must have the same input set.\n");
-
-	return err;
-}
-
-/**
- * ice_set_init_fdir_seg
- * @seg: flow segment for programming
- * @l3_proto: ICE_FLOW_SEG_HDR_IPV4 or ICE_FLOW_SEG_HDR_IPV6
- * @l4_proto: ICE_FLOW_SEG_HDR_TCP or ICE_FLOW_SEG_HDR_UDP
- *
- * Set the configuration for perfect filters to the provided flow segment for
- * programming the HW filter. This is to be called only when initializing
- * filters as this function it assumes no filters exist.
- */
-static int
-ice_set_init_fdir_seg(struct ice_flow_seg_info *seg,
-		      enum ice_flow_seg_hdr l3_proto,
-		      enum ice_flow_seg_hdr l4_proto)
-{
-	enum ice_flow_field src_addr, dst_addr, src_port, dst_port;
-
-	if (!seg)
-		return -EINVAL;
-
-	if (l3_proto == ICE_FLOW_SEG_HDR_IPV4) {
-		src_addr = ICE_FLOW_FIELD_IDX_IPV4_SA;
-		dst_addr = ICE_FLOW_FIELD_IDX_IPV4_DA;
-	} else if (l3_proto == ICE_FLOW_SEG_HDR_IPV6) {
-		src_addr = ICE_FLOW_FIELD_IDX_IPV6_SA;
-		dst_addr = ICE_FLOW_FIELD_IDX_IPV6_DA;
-	} else {
-		return -EINVAL;
-	}
-
-	if (l4_proto == ICE_FLOW_SEG_HDR_TCP) {
-		src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT;
-	} else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) {
-		src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT;
-	} else {
-		return -EINVAL;
-	}
-
-	ICE_FLOW_SET_HDRS(seg, l3_proto | l4_proto);
-
-	/* IP source address */
-	ice_flow_set_fld(seg, src_addr, ICE_FLOW_FLD_OFF_INVAL,
-			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
-
-	/* IP destination address */
-	ice_flow_set_fld(seg, dst_addr, ICE_FLOW_FLD_OFF_INVAL,
-			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
-
-	/* Layer 4 source port */
-	ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL,
-			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
-
-	/* Layer 4 destination port */
-	ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL,
-			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
-
-	return 0;
-}
-
-/**
- * ice_create_init_fdir_rule
- * @pf: PF structure
- * @flow: filter enum
- *
- * Return error value or 0 on success.
- */
-static int
-ice_create_init_fdir_rule(struct ice_pf *pf, enum ice_fltr_ptype flow)
-{
-	struct ice_flow_seg_info *seg, *tun_seg;
-	struct device *dev = ice_pf_to_dev(pf);
-	struct ice_hw *hw = &pf->hw;
-	int ret;
-
-	/* if there is already a filter rule for kind return -EINVAL */
-	if (hw->fdir_prof && hw->fdir_prof[flow] &&
-	    hw->fdir_prof[flow]->fdir_seg[0])
-		return -EINVAL;
-
-	seg = devm_kzalloc(dev, sizeof(*seg), GFP_KERNEL);
-	if (!seg)
-		return -ENOMEM;
-
-	tun_seg = devm_kcalloc(dev, ICE_FD_HW_SEG_MAX, sizeof(*tun_seg),
-			       GFP_KERNEL);
-	if (!tun_seg) {
-		devm_kfree(dev, seg);
-		return -ENOMEM;
-	}
-
-	if (flow == ICE_FLTR_PTYPE_NONF_IPV4_TCP)
-		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV4,
-					    ICE_FLOW_SEG_HDR_TCP);
-	else if (flow == ICE_FLTR_PTYPE_NONF_IPV4_UDP)
-		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV4,
-					    ICE_FLOW_SEG_HDR_UDP);
-	else if (flow == ICE_FLTR_PTYPE_NONF_IPV6_TCP)
-		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV6,
-					    ICE_FLOW_SEG_HDR_TCP);
-	else if (flow == ICE_FLTR_PTYPE_NONF_IPV6_UDP)
-		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV6,
-					    ICE_FLOW_SEG_HDR_UDP);
-	else
-		ret = -EINVAL;
-	if (ret)
-		goto err_exit;
-
-	/* add filter for outer headers */
-	ret = ice_fdir_set_hw_fltr_rule(pf, seg, flow, ICE_FD_HW_SEG_NON_TUN);
-	if (ret)
-		/* could not write filter, free memory */
-		goto err_exit;
-
-	/* make tunneled filter HW entries if possible */
-	memcpy(&tun_seg[1], seg, sizeof(*seg));
-	ret = ice_fdir_set_hw_fltr_rule(pf, tun_seg, flow, ICE_FD_HW_SEG_TUN);
-	if (ret)
-		/* could not write tunnel filter, but outer header filter
-		 * exists
-		 */
-		devm_kfree(dev, tun_seg);
-
-	set_bit(flow, hw->fdir_perfect_fltr);
-	return ret;
-err_exit:
-	devm_kfree(dev, tun_seg);
-	devm_kfree(dev, seg);
-
-	return -EOPNOTSUPP;
-}
-
-/**
- * ice_set_fdir_ip4_seg
- * @seg: flow segment for programming
- * @tcp_ip4_spec: mask data from ethtool
- * @l4_proto: Layer 4 protocol to program
- * @perfect_fltr: only valid on success; returns true if perfect filter,
- *		  false if not
- *
- * Set the mask data into the flow segment to be used to program HW
- * table based on provided L4 protocol for IPv4
- */
-static int
-ice_set_fdir_ip4_seg(struct ice_flow_seg_info *seg,
-		     struct ethtool_tcpip4_spec *tcp_ip4_spec,
-		     enum ice_flow_seg_hdr l4_proto, bool *perfect_fltr)
-{
-	enum ice_flow_field src_port, dst_port;
-
-	/* make sure we don't have any empty rule */
-	if (!tcp_ip4_spec->psrc && !tcp_ip4_spec->ip4src &&
-	    !tcp_ip4_spec->pdst && !tcp_ip4_spec->ip4dst)
-		return -EINVAL;
-
-	/* filtering on TOS not supported */
-	if (tcp_ip4_spec->tos)
-		return -EOPNOTSUPP;
-
-	if (l4_proto == ICE_FLOW_SEG_HDR_TCP) {
-		src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT;
-	} else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) {
-		src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT;
-	} else if (l4_proto == ICE_FLOW_SEG_HDR_SCTP) {
-		src_port = ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_SCTP_DST_PORT;
-	} else {
-		return -EOPNOTSUPP;
-	}
-
-	*perfect_fltr = true;
-	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV4 | l4_proto);
-
-	/* IP source address */
-	if (tcp_ip4_spec->ip4src == htonl(0xFFFFFFFF))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_SA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!tcp_ip4_spec->ip4src)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	/* IP destination address */
-	if (tcp_ip4_spec->ip4dst == htonl(0xFFFFFFFF))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_DA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!tcp_ip4_spec->ip4dst)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	/* Layer 4 source port */
-	if (tcp_ip4_spec->psrc == htons(0xFFFF))
-		ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 false);
-	else if (!tcp_ip4_spec->psrc)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	/* Layer 4 destination port */
-	if (tcp_ip4_spec->pdst == htons(0xFFFF))
-		ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 false);
-	else if (!tcp_ip4_spec->pdst)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	return 0;
-}
-
-/**
- * ice_set_fdir_ip4_usr_seg
- * @seg: flow segment for programming
- * @usr_ip4_spec: ethtool userdef packet offset
- * @perfect_fltr: only valid on success; returns true if perfect filter,
- *		  false if not
- *
- * Set the offset data into the flow segment to be used to program HW
- * table for IPv4
- */
-static int
-ice_set_fdir_ip4_usr_seg(struct ice_flow_seg_info *seg,
-			 struct ethtool_usrip4_spec *usr_ip4_spec,
-			 bool *perfect_fltr)
-{
-	/* first 4 bytes of Layer 4 header */
-	if (usr_ip4_spec->l4_4_bytes)
-		return -EINVAL;
-	if (usr_ip4_spec->tos)
-		return -EINVAL;
-	if (usr_ip4_spec->ip_ver)
-		return -EINVAL;
-	/* Filtering on Layer 4 protocol not supported */
-	if (usr_ip4_spec->proto)
-		return -EOPNOTSUPP;
-	/* empty rules are not valid */
-	if (!usr_ip4_spec->ip4src && !usr_ip4_spec->ip4dst)
-		return -EINVAL;
-
-	*perfect_fltr = true;
-	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV4);
-
-	/* IP source address */
-	if (usr_ip4_spec->ip4src == htonl(0xFFFFFFFF))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_SA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!usr_ip4_spec->ip4src)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	/* IP destination address */
-	if (usr_ip4_spec->ip4dst == htonl(0xFFFFFFFF))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_DA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!usr_ip4_spec->ip4dst)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	return 0;
-}
-
-/**
- * ice_set_fdir_ip6_seg
- * @seg: flow segment for programming
- * @tcp_ip6_spec: mask data from ethtool
- * @l4_proto: Layer 4 protocol to program
- * @perfect_fltr: only valid on success; returns true if perfect filter,
- *		  false if not
- *
- * Set the mask data into the flow segment to be used to program HW
- * table based on provided L4 protocol for IPv6
- */
-static int
-ice_set_fdir_ip6_seg(struct ice_flow_seg_info *seg,
-		     struct ethtool_tcpip6_spec *tcp_ip6_spec,
-		     enum ice_flow_seg_hdr l4_proto, bool *perfect_fltr)
-{
-	enum ice_flow_field src_port, dst_port;
-
-	/* make sure we don't have any empty rule */
-	if (!memcmp(tcp_ip6_spec->ip6src, &zero_ipv6_addr_mask,
-		    sizeof(struct in6_addr)) &&
-	    !memcmp(tcp_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
-		    sizeof(struct in6_addr)) &&
-	    !tcp_ip6_spec->psrc && !tcp_ip6_spec->pdst)
-		return -EINVAL;
-
-	/* filtering on TC not supported */
-	if (tcp_ip6_spec->tclass)
-		return -EOPNOTSUPP;
-
-	if (l4_proto == ICE_FLOW_SEG_HDR_TCP) {
-		src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT;
-	} else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) {
-		src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT;
-	} else if (l4_proto == ICE_FLOW_SEG_HDR_SCTP) {
-		src_port = ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT;
-		dst_port = ICE_FLOW_FIELD_IDX_SCTP_DST_PORT;
-	} else {
-		return -EINVAL;
-	}
-
-	*perfect_fltr = true;
-	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV6 | l4_proto);
-
-	if (!memcmp(tcp_ip6_spec->ip6src, &full_ipv6_addr_mask,
-		    sizeof(struct in6_addr)))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_SA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!memcmp(tcp_ip6_spec->ip6src, &zero_ipv6_addr_mask,
-			 sizeof(struct in6_addr)))
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	if (!memcmp(tcp_ip6_spec->ip6dst, &full_ipv6_addr_mask,
-		    sizeof(struct in6_addr)))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_DA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!memcmp(tcp_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
-			 sizeof(struct in6_addr)))
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	/* Layer 4 source port */
-	if (tcp_ip6_spec->psrc == htons(0xFFFF))
-		ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 false);
-	else if (!tcp_ip6_spec->psrc)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	/* Layer 4 destination port */
-	if (tcp_ip6_spec->pdst == htons(0xFFFF))
-		ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 false);
-	else if (!tcp_ip6_spec->pdst)
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	return 0;
-}
-
-/**
- * ice_set_fdir_ip6_usr_seg
- * @seg: flow segment for programming
- * @usr_ip6_spec: ethtool userdef packet offset
- * @perfect_fltr: only valid on success; returns true if perfect filter,
- *		  false if not
- *
- * Set the offset data into the flow segment to be used to program HW
- * table for IPv6
- */
-static int
-ice_set_fdir_ip6_usr_seg(struct ice_flow_seg_info *seg,
-			 struct ethtool_usrip6_spec *usr_ip6_spec,
-			 bool *perfect_fltr)
-{
-	/* filtering on Layer 4 bytes not supported */
-	if (usr_ip6_spec->l4_4_bytes)
-		return -EOPNOTSUPP;
-	/* filtering on TC not supported */
-	if (usr_ip6_spec->tclass)
-		return -EOPNOTSUPP;
-	/* filtering on Layer 4 protocol not supported */
-	if (usr_ip6_spec->l4_proto)
-		return -EOPNOTSUPP;
-	/* empty rules are not valid */
-	if (!memcmp(usr_ip6_spec->ip6src, &zero_ipv6_addr_mask,
-		    sizeof(struct in6_addr)) &&
-	    !memcmp(usr_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
-		    sizeof(struct in6_addr)))
-		return -EINVAL;
-
-	*perfect_fltr = true;
-	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV6);
-
-	if (!memcmp(usr_ip6_spec->ip6src, &full_ipv6_addr_mask,
-		    sizeof(struct in6_addr)))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_SA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!memcmp(usr_ip6_spec->ip6src, &zero_ipv6_addr_mask,
-			 sizeof(struct in6_addr)))
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	if (!memcmp(usr_ip6_spec->ip6dst, &full_ipv6_addr_mask,
-		    sizeof(struct in6_addr)))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_DA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!memcmp(usr_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
-			 sizeof(struct in6_addr)))
-		*perfect_fltr = false;
-	else
-		return -EOPNOTSUPP;
-
-	return 0;
-}
-
-/**
- * ice_fdir_vlan_valid - validate VLAN data for Flow Director rule
- * @dev: network interface device structure
- * @fsp: pointer to ethtool Rx flow specification
- *
- * Return: true if vlan data is valid, false otherwise
- */
-static bool ice_fdir_vlan_valid(struct device *dev,
-				struct ethtool_rx_flow_spec *fsp)
-{
-	if (fsp->m_ext.vlan_etype && !eth_type_vlan(fsp->h_ext.vlan_etype))
-		return false;
-
-	if (fsp->m_ext.vlan_tci && ntohs(fsp->h_ext.vlan_tci) >= VLAN_N_VID)
-		return false;
-
-	/* proto and vlan must have vlan-etype defined */
-	if (fsp->m_u.ether_spec.h_proto && fsp->m_ext.vlan_tci &&
-	    !fsp->m_ext.vlan_etype) {
-		dev_warn(dev, "Filter with proto and vlan require also vlan-etype");
-		return false;
-	}
-
-	return true;
-}
-
-/**
- * ice_set_ether_flow_seg - set address and protocol segments for ether flow
- * @dev: network interface device structure
- * @seg: flow segment for programming
- * @eth_spec: mask data from ethtool
- *
- * Return: 0 on success and errno in case of error.
- */
-static int ice_set_ether_flow_seg(struct device *dev,
-				  struct ice_flow_seg_info *seg,
-				  struct ethhdr *eth_spec)
-{
-	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_ETH);
-
-	/* empty rules are not valid */
-	if (is_zero_ether_addr(eth_spec->h_source) &&
-	    is_zero_ether_addr(eth_spec->h_dest) &&
-	    !eth_spec->h_proto)
-		return -EINVAL;
-
-	/* Ethertype */
-	if (eth_spec->h_proto == htons(0xFFFF)) {
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_ETH_TYPE,
-				 ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	} else if (eth_spec->h_proto) {
-		dev_warn(dev, "Only 0x0000 or 0xffff proto mask is allowed for flow-type ether");
-		return -EOPNOTSUPP;
-	}
-
-	/* Source MAC address */
-	if (is_broadcast_ether_addr(eth_spec->h_source))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_ETH_SA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!is_zero_ether_addr(eth_spec->h_source))
-		goto err_mask;
-
-	/* Destination MAC address */
-	if (is_broadcast_ether_addr(eth_spec->h_dest))
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_ETH_DA,
-				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	else if (!is_zero_ether_addr(eth_spec->h_dest))
-		goto err_mask;
-
-	return 0;
-
-err_mask:
-	dev_warn(dev, "Only 00:00:00:00:00:00 or ff:ff:ff:ff:ff:ff MAC address mask is allowed for flow-type ether");
-	return -EOPNOTSUPP;
-}
-
-/**
- * ice_set_fdir_vlan_seg - set vlan segments for ether flow
- * @seg: flow segment for programming
- * @ext_masks: masks for additional RX flow fields
- *
- * Return: 0 on success and errno in case of error.
- */
-static int
-ice_set_fdir_vlan_seg(struct ice_flow_seg_info *seg,
-		      struct ethtool_flow_ext *ext_masks)
-{
-	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_VLAN);
-
-	if (ext_masks->vlan_etype) {
-		if (ext_masks->vlan_etype != htons(0xFFFF))
-			return -EOPNOTSUPP;
-
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_S_VLAN,
-				 ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	}
-
-	if (ext_masks->vlan_tci) {
-		if (ext_masks->vlan_tci != htons(0xFFFF))
-			return -EOPNOTSUPP;
-
-		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_C_VLAN,
-				 ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL,
-				 ICE_FLOW_FLD_OFF_INVAL, false);
-	}
-
-	return 0;
-}
-
-/**
- * ice_cfg_fdir_xtrct_seq - Configure extraction sequence for the given filter
- * @pf: PF structure
- * @fsp: pointer to ethtool Rx flow specification
- * @user: user defined data from flow specification
- *
- * Returns 0 on success.
- */
-static int
-ice_cfg_fdir_xtrct_seq(struct ice_pf *pf, struct ethtool_rx_flow_spec *fsp,
-		       struct ice_rx_flow_userdef *user)
-{
-	struct ice_flow_seg_info *seg, *tun_seg;
-	struct device *dev = ice_pf_to_dev(pf);
-	enum ice_fltr_ptype fltr_idx;
-	struct ice_hw *hw = &pf->hw;
-	bool perfect_filter = false;
-	int ret;
-
-	seg = devm_kzalloc(dev, sizeof(*seg), GFP_KERNEL);
-	if (!seg)
-		return -ENOMEM;
-
-	tun_seg = devm_kcalloc(dev, ICE_FD_HW_SEG_MAX, sizeof(*tun_seg),
-			       GFP_KERNEL);
-	if (!tun_seg) {
-		devm_kfree(dev, seg);
-		return -ENOMEM;
-	}
-
-	switch (fsp->flow_type & ~FLOW_EXT) {
-	case TCP_V4_FLOW:
-		ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec,
-					   ICE_FLOW_SEG_HDR_TCP,
-					   &perfect_filter);
-		break;
-	case UDP_V4_FLOW:
-		ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec,
-					   ICE_FLOW_SEG_HDR_UDP,
-					   &perfect_filter);
-		break;
-	case SCTP_V4_FLOW:
-		ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec,
-					   ICE_FLOW_SEG_HDR_SCTP,
-					   &perfect_filter);
-		break;
-	case IPV4_USER_FLOW:
-		ret = ice_set_fdir_ip4_usr_seg(seg, &fsp->m_u.usr_ip4_spec,
-					       &perfect_filter);
-		break;
-	case TCP_V6_FLOW:
-		ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec,
-					   ICE_FLOW_SEG_HDR_TCP,
-					   &perfect_filter);
-		break;
-	case UDP_V6_FLOW:
-		ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec,
-					   ICE_FLOW_SEG_HDR_UDP,
-					   &perfect_filter);
-		break;
-	case SCTP_V6_FLOW:
-		ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec,
-					   ICE_FLOW_SEG_HDR_SCTP,
-					   &perfect_filter);
-		break;
-	case IPV6_USER_FLOW:
-		ret = ice_set_fdir_ip6_usr_seg(seg, &fsp->m_u.usr_ip6_spec,
-					       &perfect_filter);
-		break;
-	case ETHER_FLOW:
-		ret = ice_set_ether_flow_seg(dev, seg, &fsp->m_u.ether_spec);
-		if (!ret && (fsp->m_ext.vlan_etype || fsp->m_ext.vlan_tci)) {
-			if (!ice_fdir_vlan_valid(dev, fsp)) {
-				ret = -EINVAL;
-				break;
-			}
-			ret = ice_set_fdir_vlan_seg(seg, &fsp->m_ext);
-		}
-		break;
-	default:
-		ret = -EINVAL;
-	}
-	if (ret)
-		goto err_exit;
-
-	/* tunnel segments are shifted up one. */
-	memcpy(&tun_seg[1], seg, sizeof(*seg));
-
-	if (user && user->flex_fltr) {
-		perfect_filter = false;
-		ice_flow_add_fld_raw(seg, user->flex_offset,
-				     ICE_FLTR_PRGM_FLEX_WORD_SIZE,
-				     ICE_FLOW_FLD_OFF_INVAL,
-				     ICE_FLOW_FLD_OFF_INVAL);
-		ice_flow_add_fld_raw(&tun_seg[1], user->flex_offset,
-				     ICE_FLTR_PRGM_FLEX_WORD_SIZE,
-				     ICE_FLOW_FLD_OFF_INVAL,
-				     ICE_FLOW_FLD_OFF_INVAL);
-	}
-
-	fltr_idx = ice_ethtool_flow_to_fltr(fsp->flow_type & ~FLOW_EXT);
-
-	assign_bit(fltr_idx, hw->fdir_perfect_fltr, perfect_filter);
-
-	/* add filter for outer headers */
-	ret = ice_fdir_set_hw_fltr_rule(pf, seg, fltr_idx,
-					ICE_FD_HW_SEG_NON_TUN);
-	if (ret == -EEXIST) {
-		/* Rule already exists, free memory and count as success */
-		ret = 0;
-		goto err_exit;
-	} else if (ret) {
-		/* could not write filter, free memory */
-		goto err_exit;
-	}
-
-	/* make tunneled filter HW entries if possible */
-	memcpy(&tun_seg[1], seg, sizeof(*seg));
-	ret = ice_fdir_set_hw_fltr_rule(pf, tun_seg, fltr_idx,
-					ICE_FD_HW_SEG_TUN);
-	if (ret == -EEXIST) {
-		/* Rule already exists, free memory and count as success */
-		devm_kfree(dev, tun_seg);
-		ret = 0;
-	} else if (ret) {
-		/* could not write tunnel filter, but outer filter exists */
-		devm_kfree(dev, tun_seg);
-	}
-
-	return ret;
-
-err_exit:
-	devm_kfree(dev, tun_seg);
-	devm_kfree(dev, seg);
-
-	return ret;
-}
-
-/**
- * ice_update_per_q_fltr
- * @vsi: ptr to VSI
- * @q_index: queue index
- * @inc: true to increment or false to decrement per queue filter count
- *
- * This function is used to keep track of per queue sideband filters
- */
-static void ice_update_per_q_fltr(struct ice_vsi *vsi, u32 q_index, bool inc)
-{
-	struct ice_rx_ring *rx_ring;
-
-	if (!vsi->num_rxq || q_index >= vsi->num_rxq)
-		return;
-
-	rx_ring = vsi->rx_rings[q_index];
-	if (!rx_ring || !rx_ring->ch)
-		return;
-
-	if (inc)
-		atomic_inc(&rx_ring->ch->num_sb_fltr);
-	else
-		atomic_dec_if_positive(&rx_ring->ch->num_sb_fltr);
-}
-
-/**
- * ice_fdir_write_fltr - send a flow director filter to the hardware
- * @pf: PF data structure
- * @input: filter structure
- * @add: true adds filter and false removed filter
- * @is_tun: true adds inner filter on tunnel and false outer headers
- *
- * returns 0 on success and negative value on error
- */
-int
-ice_fdir_write_fltr(struct ice_pf *pf, struct ice_fdir_fltr *input, bool add,
-		    bool is_tun)
-{
-	struct device *dev = ice_pf_to_dev(pf);
-	struct ice_hw *hw = &pf->hw;
-	struct ice_fltr_desc desc;
-	struct ice_vsi *ctrl_vsi;
-	u8 *pkt, *frag_pkt;
-	bool has_frag;
-	int err;
-
-	ctrl_vsi = ice_get_ctrl_vsi(pf);
-	if (!ctrl_vsi)
-		return -EINVAL;
-
-	pkt = devm_kzalloc(dev, ICE_FDIR_MAX_RAW_PKT_SIZE, GFP_KERNEL);
-	if (!pkt)
-		return -ENOMEM;
-	frag_pkt = devm_kzalloc(dev, ICE_FDIR_MAX_RAW_PKT_SIZE, GFP_KERNEL);
-	if (!frag_pkt) {
-		err = -ENOMEM;
-		goto err_free;
-	}
-
-	ice_fdir_get_prgm_desc(hw, input, &desc, add);
-	err = ice_fdir_get_gen_prgm_pkt(hw, input, pkt, false, is_tun);
-	if (err)
-		goto err_free_all;
-	err = ice_prgm_fdir_fltr(ctrl_vsi, &desc, pkt);
-	if (err)
-		goto err_free_all;
-
-	/* repeat for fragment packet */
-	has_frag = ice_fdir_has_frag(input->flow_type);
-	if (has_frag) {
-		/* does not return error */
-		ice_fdir_get_prgm_desc(hw, input, &desc, add);
-		err = ice_fdir_get_gen_prgm_pkt(hw, input, frag_pkt, true,
-						is_tun);
-		if (err)
-			goto err_frag;
-		err = ice_prgm_fdir_fltr(ctrl_vsi, &desc, frag_pkt);
-		if (err)
-			goto err_frag;
-	} else {
-		devm_kfree(dev, frag_pkt);
-	}
-
-	return 0;
-
-err_free_all:
-	devm_kfree(dev, frag_pkt);
-err_free:
-	devm_kfree(dev, pkt);
-	return err;
-
-err_frag:
-	devm_kfree(dev, frag_pkt);
-	return err;
-}
-
-/**
- * ice_fdir_write_all_fltr - send a flow director filter to the hardware
- * @pf: PF data structure
- * @input: filter structure
- * @add: true adds filter and false removed filter
- *
- * returns 0 on success and negative value on error
- */
-static int
-ice_fdir_write_all_fltr(struct ice_pf *pf, struct ice_fdir_fltr *input,
-			bool add)
-{
-	u16 port_num;
-	int tun;
-
-	for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
-		bool is_tun = tun == ICE_FD_HW_SEG_TUN;
-		int err;
-
-		if (is_tun && !ice_get_open_tunnel_port(&pf->hw, &port_num, TNL_ALL))
-			continue;
-		err = ice_fdir_write_fltr(pf, input, add, is_tun);
-		if (err)
-			return err;
-	}
-	return 0;
-}
-
-/**
- * ice_fdir_replay_fltrs - replay filters from the HW filter list
- * @pf: board private structure
- */
-void ice_fdir_replay_fltrs(struct ice_pf *pf)
-{
-	struct ice_fdir_fltr *f_rule;
-	struct ice_hw *hw = &pf->hw;
-
-	list_for_each_entry(f_rule, &hw->fdir_list_head, fltr_node) {
-		int err = ice_fdir_write_all_fltr(pf, f_rule, true);
-
-		if (err)
-			dev_dbg(ice_pf_to_dev(pf), "Flow Director error %d, could not reprogram filter %d\n",
-				err, f_rule->fltr_id);
-	}
-}
-
-/**
- * ice_fdir_create_dflt_rules - create default perfect filters
- * @pf: PF data structure
- *
- * Returns 0 for success or error.
- */
-int ice_fdir_create_dflt_rules(struct ice_pf *pf)
-{
-	static const enum ice_fltr_ptype dflt_rules[] = {
-		ICE_FLTR_PTYPE_NONF_IPV4_TCP, ICE_FLTR_PTYPE_NONF_IPV4_UDP,
-		ICE_FLTR_PTYPE_NONF_IPV6_TCP, ICE_FLTR_PTYPE_NONF_IPV6_UDP,
-	};
-	int err;
-
-	/* Create perfect TCP and UDP rules in hardware. */
-	for (int i = 0; i < ARRAY_SIZE(dflt_rules); i++) {
-		err = ice_create_init_fdir_rule(pf, dflt_rules[i]);
-
-		if (err)
-			break;
-	}
-
-	return err;
-}
-
-/**
- * ice_fdir_del_all_fltrs - Delete all flow director filters
- * @vsi: the VSI being changed
- *
- * This function needs to be called while holding hw->fdir_fltr_lock
- */
-void ice_fdir_del_all_fltrs(struct ice_vsi *vsi)
-{
-	struct ice_fdir_fltr *f_rule, *tmp;
-	struct ice_pf *pf = vsi->back;
-	struct ice_hw *hw = &pf->hw;
-
-	list_for_each_entry_safe(f_rule, tmp, &hw->fdir_list_head, fltr_node) {
-		ice_fdir_write_all_fltr(pf, f_rule, false);
-		ice_fdir_update_cntrs(hw, f_rule->flow_type, false);
-		list_del(&f_rule->fltr_node);
-		devm_kfree(ice_pf_to_dev(pf), f_rule);
-	}
-}
-
-/**
- * ice_vsi_manage_fdir - turn on/off flow director
- * @vsi: the VSI being changed
- * @ena: boolean value indicating if this is an enable or disable request
- */
-void ice_vsi_manage_fdir(struct ice_vsi *vsi, bool ena)
-{
-	struct ice_pf *pf = vsi->back;
-	struct ice_hw *hw = &pf->hw;
-	enum ice_fltr_ptype flow;
-
-	if (ena) {
-		set_bit(ICE_FLAG_FD_ENA, pf->flags);
-		ice_fdir_create_dflt_rules(pf);
-		return;
-	}
-
-	mutex_lock(&hw->fdir_fltr_lock);
-	if (!test_and_clear_bit(ICE_FLAG_FD_ENA, pf->flags))
-		goto release_lock;
-
-	ice_fdir_del_all_fltrs(vsi);
-
-	if (hw->fdir_prof)
-		for (flow = ICE_FLTR_PTYPE_NONF_NONE; flow < ICE_FLTR_PTYPE_MAX;
-		     flow++)
-			if (hw->fdir_prof[flow])
-				ice_fdir_rem_flow(hw, ICE_BLK_FD, flow);
-
-release_lock:
-	mutex_unlock(&hw->fdir_fltr_lock);
-}
-
-/**
- * ice_fdir_do_rem_flow - delete flow and possibly add perfect flow
- * @pf: PF structure
- * @flow_type: FDir flow type to release
- */
-static void
-ice_fdir_do_rem_flow(struct ice_pf *pf, enum ice_fltr_ptype flow_type)
-{
-	struct ice_hw *hw = &pf->hw;
-	bool need_perfect = false;
-
-	if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_TCP ||
-	    flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP ||
-	    flow_type == ICE_FLTR_PTYPE_NONF_IPV6_TCP ||
-	    flow_type == ICE_FLTR_PTYPE_NONF_IPV6_UDP)
-		need_perfect = true;
-
-	if (need_perfect && test_bit(flow_type, hw->fdir_perfect_fltr))
-		return;
-
-	ice_fdir_rem_flow(hw, ICE_BLK_FD, flow_type);
-	if (need_perfect)
-		ice_create_init_fdir_rule(pf, flow_type);
-}
-
-/**
- * ice_fdir_update_list_entry - add or delete a filter from the filter list
- * @pf: PF structure
- * @input: filter structure
- * @fltr_idx: ethtool index of filter to modify
- *
- * returns 0 on success and negative on errors
- */
-static int
-ice_fdir_update_list_entry(struct ice_pf *pf, struct ice_fdir_fltr *input,
-			   int fltr_idx)
-{
-	struct ice_fdir_fltr *old_fltr;
-	struct ice_hw *hw = &pf->hw;
-	struct ice_vsi *vsi;
-	int err = -ENOENT;
-
-	/* Do not update filters during reset */
-	if (ice_is_reset_in_progress(pf->state))
-		return -EBUSY;
-
-	vsi = ice_get_main_vsi(pf);
-	if (!vsi)
-		return -EINVAL;
-
-	old_fltr = ice_fdir_find_fltr_by_idx(hw, fltr_idx);
-	if (old_fltr) {
-		err = ice_fdir_write_all_fltr(pf, old_fltr, false);
-		if (err)
-			return err;
-		ice_fdir_update_cntrs(hw, old_fltr->flow_type, false);
-		/* update sb-filters count, specific to ring->channel */
-		ice_update_per_q_fltr(vsi, old_fltr->orig_q_index, false);
-		if (!input && !hw->fdir_fltr_cnt[old_fltr->flow_type])
-			/* we just deleted the last filter of flow_type so we
-			 * should also delete the HW filter info.
-			 */
-			ice_fdir_do_rem_flow(pf, old_fltr->flow_type);
-		list_del(&old_fltr->fltr_node);
-		devm_kfree(ice_hw_to_dev(hw), old_fltr);
-	}
-	if (!input)
-		return err;
-	ice_fdir_list_add_fltr(hw, input);
-	/* update sb-filters count, specific to ring->channel */
-	ice_update_per_q_fltr(vsi, input->orig_q_index, true);
-	ice_fdir_update_cntrs(hw, input->flow_type, true);
-	return 0;
-}
-
-/**
- * ice_del_fdir_ethtool - delete Flow Director filter
- * @vsi: pointer to target VSI
- * @cmd: command to add or delete Flow Director filter
- *
- * Returns 0 on success and negative values for failure
- */
-int ice_del_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd)
-{
-	struct ethtool_rx_flow_spec *fsp =
-		(struct ethtool_rx_flow_spec *)&cmd->fs;
-	struct ice_pf *pf = vsi->back;
-	struct ice_hw *hw = &pf->hw;
-	int val;
-
-	if (!test_bit(ICE_FLAG_FD_ENA, pf->flags))
-		return -EOPNOTSUPP;
-
-	/* Do not delete filters during reset */
-	if (ice_is_reset_in_progress(pf->state)) {
-		dev_err(ice_pf_to_dev(pf), "Device is resetting - deleting Flow Director filters not supported during reset\n");
-		return -EBUSY;
-	}
-
-	if (test_bit(ICE_FD_FLUSH_REQ, pf->state))
-		return -EBUSY;
-
-	mutex_lock(&hw->fdir_fltr_lock);
-	val = ice_fdir_update_list_entry(pf, NULL, fsp->location);
-	mutex_unlock(&hw->fdir_fltr_lock);
-
-	return val;
-}
-
-/**
- * ice_update_ring_dest_vsi - update dest ring and dest VSI
- * @vsi: pointer to target VSI
- * @dest_vsi: ptr to dest VSI index
- * @ring: ptr to dest ring
- *
- * This function updates destination VSI and queue if user specifies
- * target queue which falls in channel's (aka ADQ) queue region
- */
-static void
-ice_update_ring_dest_vsi(struct ice_vsi *vsi, u16 *dest_vsi, u32 *ring)
-{
-	struct ice_channel *ch;
-
-	list_for_each_entry(ch, &vsi->ch_list, list) {
-		if (!ch->ch_vsi)
-			continue;
-
-		/* make sure to locate corresponding channel based on "queue"
-		 * specified
-		 */
-		if ((*ring < ch->base_q) ||
-		    (*ring >= (ch->base_q + ch->num_rxq)))
-			continue;
-
-		/* update the dest_vsi based on channel */
-		*dest_vsi = ch->ch_vsi->idx;
-
-		/* update the "ring" to be correct based on channel */
-		*ring -= ch->base_q;
-	}
-}
-
-/**
- * ice_set_fdir_input_set - Set the input set for Flow Director
- * @vsi: pointer to target VSI
- * @fsp: pointer to ethtool Rx flow specification
- * @input: filter structure
- */
-static int
-ice_set_fdir_input_set(struct ice_vsi *vsi, struct ethtool_rx_flow_spec *fsp,
-		       struct ice_fdir_fltr *input)
-{
-	s16 q_index = ICE_FDIR_NO_QUEUE_IDX;
-	u16 orig_q_index = 0;
-	struct ice_pf *pf;
-	struct ice_hw *hw;
-	int flow_type;
-	u16 dest_vsi;
-	u8 dest_ctl;
-
-	if (!vsi || !fsp || !input)
-		return -EINVAL;
-
-	pf = vsi->back;
-	hw = &pf->hw;
-
-	dest_vsi = vsi->idx;
-	if (fsp->ring_cookie == RX_CLS_FLOW_DISC) {
-		dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DROP_PKT;
-	} else {
-		u32 ring = ethtool_get_flow_spec_ring(fsp->ring_cookie);
-		u8 vf = ethtool_get_flow_spec_ring_vf(fsp->ring_cookie);
-
-		if (vf) {
-			dev_err(ice_pf_to_dev(pf), "Failed to add filter. Flow director filters are not supported on VF queues.\n");
-			return -EINVAL;
-		}
-
-		if (ring >= vsi->num_rxq)
-			return -EINVAL;
-
-		orig_q_index = ring;
-		ice_update_ring_dest_vsi(vsi, &dest_vsi, &ring);
-		dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DIRECT_PKT_QINDEX;
-		q_index = ring;
-	}
-
-	input->fltr_id = fsp->location;
-	input->q_index = q_index;
-	flow_type = fsp->flow_type & ~FLOW_EXT;
-
-	/* Record the original queue index as specified by user.
-	 * with channel configuration 'q_index' becomes relative
-	 * to TC (channel).
-	 */
-	input->orig_q_index = orig_q_index;
-	input->dest_vsi = dest_vsi;
-	input->dest_ctl = dest_ctl;
-	input->fltr_status = ICE_FLTR_PRGM_DESC_FD_STATUS_FD_ID;
-	input->cnt_index = ICE_FD_SB_STAT_IDX(hw->fd_ctr_base);
-	input->flow_type = ice_ethtool_flow_to_fltr(flow_type);
-
-	if (fsp->flow_type & FLOW_EXT) {
-		memcpy(input->ext_data.usr_def, fsp->h_ext.data,
-		       sizeof(input->ext_data.usr_def));
-		input->ext_data.vlan_type = fsp->h_ext.vlan_etype;
-		input->ext_data.vlan_tag = fsp->h_ext.vlan_tci;
-		memcpy(input->ext_mask.usr_def, fsp->m_ext.data,
-		       sizeof(input->ext_mask.usr_def));
-		input->ext_mask.vlan_type = fsp->m_ext.vlan_etype;
-		input->ext_mask.vlan_tag = fsp->m_ext.vlan_tci;
-	}
-
-	switch (flow_type) {
-	case TCP_V4_FLOW:
-	case UDP_V4_FLOW:
-	case SCTP_V4_FLOW:
-		input->ip.v4.dst_port = fsp->h_u.tcp_ip4_spec.pdst;
-		input->ip.v4.src_port = fsp->h_u.tcp_ip4_spec.psrc;
-		input->ip.v4.dst_ip = fsp->h_u.tcp_ip4_spec.ip4dst;
-		input->ip.v4.src_ip = fsp->h_u.tcp_ip4_spec.ip4src;
-		input->mask.v4.dst_port = fsp->m_u.tcp_ip4_spec.pdst;
-		input->mask.v4.src_port = fsp->m_u.tcp_ip4_spec.psrc;
-		input->mask.v4.dst_ip = fsp->m_u.tcp_ip4_spec.ip4dst;
-		input->mask.v4.src_ip = fsp->m_u.tcp_ip4_spec.ip4src;
-		break;
-	case IPV4_USER_FLOW:
-		input->ip.v4.dst_ip = fsp->h_u.usr_ip4_spec.ip4dst;
-		input->ip.v4.src_ip = fsp->h_u.usr_ip4_spec.ip4src;
-		input->ip.v4.l4_header = fsp->h_u.usr_ip4_spec.l4_4_bytes;
-		input->ip.v4.proto = fsp->h_u.usr_ip4_spec.proto;
-		input->ip.v4.ip_ver = fsp->h_u.usr_ip4_spec.ip_ver;
-		input->ip.v4.tos = fsp->h_u.usr_ip4_spec.tos;
-		input->mask.v4.dst_ip = fsp->m_u.usr_ip4_spec.ip4dst;
-		input->mask.v4.src_ip = fsp->m_u.usr_ip4_spec.ip4src;
-		input->mask.v4.l4_header = fsp->m_u.usr_ip4_spec.l4_4_bytes;
-		input->mask.v4.proto = fsp->m_u.usr_ip4_spec.proto;
-		input->mask.v4.ip_ver = fsp->m_u.usr_ip4_spec.ip_ver;
-		input->mask.v4.tos = fsp->m_u.usr_ip4_spec.tos;
-		break;
-	case TCP_V6_FLOW:
-	case UDP_V6_FLOW:
-	case SCTP_V6_FLOW:
-		memcpy(input->ip.v6.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst,
-		       sizeof(struct in6_addr));
-		memcpy(input->ip.v6.src_ip, fsp->h_u.usr_ip6_spec.ip6src,
-		       sizeof(struct in6_addr));
-		input->ip.v6.dst_port = fsp->h_u.tcp_ip6_spec.pdst;
-		input->ip.v6.src_port = fsp->h_u.tcp_ip6_spec.psrc;
-		input->ip.v6.tc = fsp->h_u.tcp_ip6_spec.tclass;
-		memcpy(input->mask.v6.dst_ip, fsp->m_u.tcp_ip6_spec.ip6dst,
-		       sizeof(struct in6_addr));
-		memcpy(input->mask.v6.src_ip, fsp->m_u.tcp_ip6_spec.ip6src,
-		       sizeof(struct in6_addr));
-		input->mask.v6.dst_port = fsp->m_u.tcp_ip6_spec.pdst;
-		input->mask.v6.src_port = fsp->m_u.tcp_ip6_spec.psrc;
-		input->mask.v6.tc = fsp->m_u.tcp_ip6_spec.tclass;
-		break;
-	case IPV6_USER_FLOW:
-		memcpy(input->ip.v6.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst,
-		       sizeof(struct in6_addr));
-		memcpy(input->ip.v6.src_ip, fsp->h_u.usr_ip6_spec.ip6src,
-		       sizeof(struct in6_addr));
-		input->ip.v6.l4_header = fsp->h_u.usr_ip6_spec.l4_4_bytes;
-		input->ip.v6.tc = fsp->h_u.usr_ip6_spec.tclass;
-
-		/* if no protocol requested, use IPPROTO_NONE */
-		if (!fsp->m_u.usr_ip6_spec.l4_proto)
-			input->ip.v6.proto = IPPROTO_NONE;
-		else
-			input->ip.v6.proto = fsp->h_u.usr_ip6_spec.l4_proto;
-
-		memcpy(input->mask.v6.dst_ip, fsp->m_u.usr_ip6_spec.ip6dst,
-		       sizeof(struct in6_addr));
-		memcpy(input->mask.v6.src_ip, fsp->m_u.usr_ip6_spec.ip6src,
-		       sizeof(struct in6_addr));
-		input->mask.v6.l4_header = fsp->m_u.usr_ip6_spec.l4_4_bytes;
-		input->mask.v6.tc = fsp->m_u.usr_ip6_spec.tclass;
-		input->mask.v6.proto = fsp->m_u.usr_ip6_spec.l4_proto;
-		break;
-	case ETHER_FLOW:
-		input->eth = fsp->h_u.ether_spec;
-		input->eth_mask = fsp->m_u.ether_spec;
-		break;
-	default:
-		/* not doing un-parsed flow types */
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/**
- * ice_add_fdir_ethtool - Add/Remove Flow Director filter
- * @vsi: pointer to target VSI
- * @cmd: command to add or delete Flow Director filter
- *
- * Returns 0 on success and negative values for failure
- */
-int ice_add_fdir_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd)
-{
-	struct ice_rx_flow_userdef userdata;
-	struct ethtool_rx_flow_spec *fsp;
-	struct ice_fdir_fltr *input;
-	struct device *dev;
-	struct ice_pf *pf;
-	struct ice_hw *hw;
-	int fltrs_needed;
-	u32 max_location;
-	u16 tunnel_port;
-	int ret;
-
-	if (!vsi)
-		return -EINVAL;
-
-	pf = vsi->back;
-	hw = &pf->hw;
-	dev = ice_pf_to_dev(pf);
-
-	if (!test_bit(ICE_FLAG_FD_ENA, pf->flags))
-		return -EOPNOTSUPP;
-
-	/* Do not program filters during reset */
-	if (ice_is_reset_in_progress(pf->state)) {
-		dev_err(dev, "Device is resetting - adding Flow Director filters not supported during reset\n");
-		return -EBUSY;
-	}
-
-	fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
-
-	if (ice_parse_rx_flow_user_data(fsp, &userdata))
-		return -EINVAL;
-
-	if (fsp->flow_type & FLOW_MAC_EXT)
-		return -EINVAL;
-
-	ret = ice_cfg_fdir_xtrct_seq(pf, fsp, &userdata);
-	if (ret)
-		return ret;
-
-	max_location = ice_get_fdir_cnt_all(hw);
-	if (fsp->location >= max_location) {
-		dev_err(dev, "Failed to add filter. The number of ntuple filters or provided location exceed max %d.\n",
-			max_location);
-		return -ENOSPC;
-	}
-
-	/* return error if not an update and no available filters */
-	fltrs_needed = ice_get_open_tunnel_port(hw, &tunnel_port, TNL_ALL) ? 2 : 1;
-	if (!ice_fdir_find_fltr_by_idx(hw, fsp->location) &&
-	    ice_fdir_num_avail_fltr(hw, pf->vsi[vsi->idx]) < fltrs_needed) {
-		dev_err(dev, "Failed to add filter. The maximum number of flow director filters has been reached.\n");
-		return -ENOSPC;
-	}
-
-	input = devm_kzalloc(dev, sizeof(*input), GFP_KERNEL);
-	if (!input)
-		return -ENOMEM;
-
-	ret = ice_set_fdir_input_set(vsi, fsp, input);
-	if (ret)
-		goto free_input;
-
-	mutex_lock(&hw->fdir_fltr_lock);
-	if (ice_fdir_is_dup_fltr(hw, input)) {
-		ret = -EINVAL;
-		goto release_lock;
-	}
-
-	if (userdata.flex_fltr) {
-		input->flex_fltr = true;
-		input->flex_word = cpu_to_be16(userdata.flex_word);
-		input->flex_offset = userdata.flex_offset;
-	}
-
-	input->cnt_ena = ICE_FXD_FLTR_QW0_STAT_ENA_PKTS;
-	input->fdid_prio = ICE_FXD_FLTR_QW1_FDID_PRI_THREE;
-	input->comp_report = ICE_FXD_FLTR_QW0_COMP_REPORT_SW_FAIL;
-
-	/* input struct is added to the HW filter list */
-	ret = ice_fdir_update_list_entry(pf, input, fsp->location);
-	if (ret)
-		goto release_lock;
-
-	ret = ice_fdir_write_all_fltr(pf, input, true);
-	if (ret)
-		goto remove_sw_rule;
-
-	goto release_lock;
-
-remove_sw_rule:
-	ice_fdir_update_cntrs(hw, input->flow_type, false);
-	/* update sb-filters count, specific to ring->channel */
-	ice_update_per_q_fltr(vsi, input->orig_q_index, false);
-	list_del(&input->fltr_node);
-release_lock:
-	mutex_unlock(&hw->fdir_fltr_lock);
-free_input:
-	if (ret)
-		devm_kfree(dev, input);
-
-	return ret;
-}
diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool_ntuple.c b/drivers/net/ethernet/intel/ice/ice_ethtool_ntuple.c
new file mode 100644
index 00000000000000..516b57ff3b1caf
--- /dev/null
+++ b/drivers/net/ethernet/intel/ice/ice_ethtool_ntuple.c
@@ -0,0 +1,2098 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2018-2023, Intel Corporation. */
+
+/* flow director ethtool support for ice */
+
+#include "ice.h"
+#include "ice_lib.h"
+#include "ice_fdir.h"
+#include "ice_flow.h"
+
+static struct in6_addr full_ipv6_addr_mask = {
+	.in6_u = {
+		.u6_addr8 = {
+			0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+			0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+		}
+	}
+};
+
+static struct in6_addr zero_ipv6_addr_mask = {
+	.in6_u = {
+		.u6_addr8 = {
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		}
+	}
+};
+
+/* calls to ice_flow_add_prof require the number of segments in the array
+ * for segs_cnt. In this code that is one more than the index.
+ */
+#define TNL_SEG_CNT(_TNL_) ((_TNL_) + 1)
+
+/**
+ * ice_fltr_to_ethtool_flow - convert filter type values to ethtool
+ * flow type values
+ * @flow: filter type to be converted
+ *
+ * Returns the corresponding ethtool flow type.
+ */
+static int ice_fltr_to_ethtool_flow(enum ice_fltr_ptype flow)
+{
+	switch (flow) {
+	case ICE_FLTR_PTYPE_NONF_ETH:
+		return ETHER_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV4_TCP:
+		return TCP_V4_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV4_UDP:
+		return UDP_V4_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV4_SCTP:
+		return SCTP_V4_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV4_OTHER:
+		return IPV4_USER_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV6_TCP:
+		return TCP_V6_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV6_UDP:
+		return UDP_V6_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV6_SCTP:
+		return SCTP_V6_FLOW;
+	case ICE_FLTR_PTYPE_NONF_IPV6_OTHER:
+		return IPV6_USER_FLOW;
+	default:
+		/* 0 is undefined ethtool flow */
+		return 0;
+	}
+}
+
+/**
+ * ice_ethtool_flow_to_fltr - convert ethtool flow type to filter enum
+ * @eth: Ethtool flow type to be converted
+ *
+ * Returns flow enum
+ */
+static enum ice_fltr_ptype ice_ethtool_flow_to_fltr(int eth)
+{
+	switch (eth) {
+	case ETHER_FLOW:
+		return ICE_FLTR_PTYPE_NONF_ETH;
+	case TCP_V4_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV4_TCP;
+	case UDP_V4_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV4_UDP;
+	case SCTP_V4_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV4_SCTP;
+	case IPV4_USER_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV4_OTHER;
+	case TCP_V6_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV6_TCP;
+	case UDP_V6_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV6_UDP;
+	case SCTP_V6_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV6_SCTP;
+	case IPV6_USER_FLOW:
+		return ICE_FLTR_PTYPE_NONF_IPV6_OTHER;
+	default:
+		return ICE_FLTR_PTYPE_NONF_NONE;
+	}
+}
+
+/**
+ * ice_is_mask_valid - check mask field set
+ * @mask: full mask to check
+ * @field: field for which mask should be valid
+ *
+ * If the mask is fully set return true. If it is not valid for field return
+ * false.
+ */
+static bool ice_is_mask_valid(u64 mask, u64 field)
+{
+	return (mask & field) == field;
+}
+
+/**
+ * ice_get_ethtool_fdir_entry - fill ethtool structure with fdir filter data
+ * @hw: hardware structure that contains filter list
+ * @cmd: ethtool command data structure to receive the filter data
+ *
+ * Returns 0 on success and -EINVAL on failure
+ */
+int ice_get_ethtool_fdir_entry(struct ice_hw *hw, struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp;
+	struct ice_ntuple_fltr *rule;
+	int ret = 0;
+	u16 idx;
+
+	fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
+
+	mutex_lock(&hw->fdir_fltr_lock);
+
+	rule = ice_fdir_find_fltr_by_idx(hw, fsp->location);
+
+	if (!rule || fsp->location != rule->fltr_id) {
+		ret = -EINVAL;
+		goto release_lock;
+	}
+
+	fsp->flow_type = ice_fltr_to_ethtool_flow(rule->flow_type);
+
+	memset(&fsp->m_u, 0, sizeof(fsp->m_u));
+	memset(&fsp->m_ext, 0, sizeof(fsp->m_ext));
+
+	switch (fsp->flow_type) {
+	case ETHER_FLOW:
+		fsp->h_u.ether_spec = rule->eth;
+		fsp->m_u.ether_spec = rule->eth_mask;
+		break;
+	case IPV4_USER_FLOW:
+		fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4;
+		fsp->h_u.usr_ip4_spec.proto = 0;
+		fsp->h_u.usr_ip4_spec.l4_4_bytes = rule->ip.v4.l4_header;
+		fsp->h_u.usr_ip4_spec.tos = rule->ip.v4.tos;
+		fsp->h_u.usr_ip4_spec.ip4src = rule->ip.v4.src_ip;
+		fsp->h_u.usr_ip4_spec.ip4dst = rule->ip.v4.dst_ip;
+		fsp->m_u.usr_ip4_spec.ip4src = rule->mask.v4.src_ip;
+		fsp->m_u.usr_ip4_spec.ip4dst = rule->mask.v4.dst_ip;
+		fsp->m_u.usr_ip4_spec.ip_ver = 0xFF;
+		fsp->m_u.usr_ip4_spec.proto = 0;
+		fsp->m_u.usr_ip4_spec.l4_4_bytes = rule->mask.v4.l4_header;
+		fsp->m_u.usr_ip4_spec.tos = rule->mask.v4.tos;
+		break;
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case SCTP_V4_FLOW:
+		fsp->h_u.tcp_ip4_spec.psrc = rule->ip.v4.src_port;
+		fsp->h_u.tcp_ip4_spec.pdst = rule->ip.v4.dst_port;
+		fsp->h_u.tcp_ip4_spec.ip4src = rule->ip.v4.src_ip;
+		fsp->h_u.tcp_ip4_spec.ip4dst = rule->ip.v4.dst_ip;
+		fsp->m_u.tcp_ip4_spec.psrc = rule->mask.v4.src_port;
+		fsp->m_u.tcp_ip4_spec.pdst = rule->mask.v4.dst_port;
+		fsp->m_u.tcp_ip4_spec.ip4src = rule->mask.v4.src_ip;
+		fsp->m_u.tcp_ip4_spec.ip4dst = rule->mask.v4.dst_ip;
+		break;
+	case IPV6_USER_FLOW:
+		fsp->h_u.usr_ip6_spec.l4_4_bytes = rule->ip.v6.l4_header;
+		fsp->h_u.usr_ip6_spec.tclass = rule->ip.v6.tc;
+		fsp->h_u.usr_ip6_spec.l4_proto = rule->ip.v6.proto;
+		memcpy(fsp->h_u.tcp_ip6_spec.ip6src, rule->ip.v6.src_ip,
+		       sizeof(struct in6_addr));
+		memcpy(fsp->h_u.tcp_ip6_spec.ip6dst, rule->ip.v6.dst_ip,
+		       sizeof(struct in6_addr));
+		memcpy(fsp->m_u.tcp_ip6_spec.ip6src, rule->mask.v6.src_ip,
+		       sizeof(struct in6_addr));
+		memcpy(fsp->m_u.tcp_ip6_spec.ip6dst, rule->mask.v6.dst_ip,
+		       sizeof(struct in6_addr));
+		fsp->m_u.usr_ip6_spec.l4_4_bytes = rule->mask.v6.l4_header;
+		fsp->m_u.usr_ip6_spec.tclass = rule->mask.v6.tc;
+		fsp->m_u.usr_ip6_spec.l4_proto = rule->mask.v6.proto;
+		break;
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+	case SCTP_V6_FLOW:
+		memcpy(fsp->h_u.tcp_ip6_spec.ip6src, rule->ip.v6.src_ip,
+		       sizeof(struct in6_addr));
+		memcpy(fsp->h_u.tcp_ip6_spec.ip6dst, rule->ip.v6.dst_ip,
+		       sizeof(struct in6_addr));
+		fsp->h_u.tcp_ip6_spec.psrc = rule->ip.v6.src_port;
+		fsp->h_u.tcp_ip6_spec.pdst = rule->ip.v6.dst_port;
+		memcpy(fsp->m_u.tcp_ip6_spec.ip6src,
+		       rule->mask.v6.src_ip,
+		       sizeof(struct in6_addr));
+		memcpy(fsp->m_u.tcp_ip6_spec.ip6dst,
+		       rule->mask.v6.dst_ip,
+		       sizeof(struct in6_addr));
+		fsp->m_u.tcp_ip6_spec.psrc = rule->mask.v6.src_port;
+		fsp->m_u.tcp_ip6_spec.pdst = rule->mask.v6.dst_port;
+		fsp->h_u.tcp_ip6_spec.tclass = rule->ip.v6.tc;
+		fsp->m_u.tcp_ip6_spec.tclass = rule->mask.v6.tc;
+		break;
+	default:
+		break;
+	}
+
+	if (rule->dest_ctl == ICE_FLTR_PRGM_DESC_DEST_DROP_PKT)
+		fsp->ring_cookie = RX_CLS_FLOW_DISC;
+	else
+		fsp->ring_cookie = rule->orig_q_index;
+
+	idx = ice_ethtool_flow_to_fltr(fsp->flow_type);
+	if (idx == ICE_FLTR_PTYPE_NONF_NONE) {
+		dev_err(ice_hw_to_dev(hw), "Missing input index for flow_type %d\n",
+			rule->flow_type);
+		ret = -EINVAL;
+	}
+
+release_lock:
+	mutex_unlock(&hw->fdir_fltr_lock);
+	return ret;
+}
+
+/**
+ * ice_get_fdir_fltr_ids - fill buffer with filter IDs of active filters
+ * @hw: hardware structure containing the filter list
+ * @cmd: ethtool command data structure
+ * @rule_locs: ethtool array passed in from OS to receive filter IDs
+ *
+ * Returns 0 as expected for success by ethtool
+ */
+int
+ice_get_fdir_fltr_ids(struct ice_hw *hw, struct ethtool_rxnfc *cmd,
+		      u32 *rule_locs)
+{
+	struct ice_ntuple_fltr *f_rule;
+	unsigned int cnt = 0;
+	int val = 0;
+
+	/* report total rule count */
+	cmd->data = ice_get_fdir_cnt_all(hw);
+
+	mutex_lock(&hw->fdir_fltr_lock);
+
+	list_for_each_entry(f_rule, &hw->fdir_list_head, fltr_node) {
+		if (cnt == cmd->rule_cnt) {
+			val = -EMSGSIZE;
+			goto release_lock;
+		}
+		rule_locs[cnt] = f_rule->fltr_id;
+		cnt++;
+	}
+
+release_lock:
+	mutex_unlock(&hw->fdir_fltr_lock);
+	if (!val)
+		cmd->rule_cnt = cnt;
+	return val;
+}
+
+/**
+ * ice_fdir_remap_entries - update the FDir entries in profile
+ * @prof: FDir structure pointer
+ * @tun: tunneled or non-tunneled packet
+ * @idx: FDir entry index
+ */
+static void
+ice_fdir_remap_entries(struct ice_fd_hw_prof *prof, int tun, int idx)
+{
+	if (idx != prof->cnt && tun < ICE_FD_HW_SEG_MAX) {
+		int i;
+
+		for (i = idx; i < (prof->cnt - 1); i++) {
+			u64 old_entry_h;
+
+			old_entry_h = prof->entry_h[i + 1][tun];
+			prof->entry_h[i][tun] = old_entry_h;
+			prof->vsi_h[i] = prof->vsi_h[i + 1];
+		}
+
+		prof->entry_h[i][tun] = 0;
+		prof->vsi_h[i] = 0;
+	}
+}
+
+/**
+ * ice_fdir_rem_adq_chnl - remove an ADQ channel from HW filter rules
+ * @hw: hardware structure containing filter list
+ * @vsi_idx: VSI handle
+ */
+void ice_fdir_rem_adq_chnl(struct ice_hw *hw, u16 vsi_idx)
+{
+	int status, flow;
+
+	if (!hw->fdir_prof)
+		return;
+
+	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) {
+		struct ice_fd_hw_prof *prof = hw->fdir_prof[flow];
+		int tun, i;
+
+		if (!prof || !prof->cnt)
+			continue;
+
+		for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
+			u64 prof_id = prof->prof_id[tun];
+
+			for (i = 0; i < prof->cnt; i++) {
+				if (prof->vsi_h[i] != vsi_idx)
+					continue;
+
+				prof->entry_h[i][tun] = 0;
+				prof->vsi_h[i] = 0;
+				break;
+			}
+
+			/* after clearing FDir entries update the remaining */
+			ice_fdir_remap_entries(prof, tun, i);
+
+			/* find flow profile corresponding to prof_id and clear
+			 * vsi_idx from bitmap.
+			 */
+			status = ice_flow_rem_vsi_prof(hw, vsi_idx, prof_id);
+			if (status) {
+				dev_err(ice_hw_to_dev(hw), "ice_flow_rem_vsi_prof() failed status=%d\n",
+					status);
+			}
+		}
+		prof->cnt--;
+	}
+}
+
+/**
+ * ice_fdir_get_hw_prof - return the ice_fd_hw_proc associated with a flow
+ * @hw: hardware structure containing the filter list
+ * @blk: hardware block
+ * @flow: FDir flow type to release
+ */
+static struct ice_fd_hw_prof *
+ice_fdir_get_hw_prof(struct ice_hw *hw, enum ice_block blk, int flow)
+{
+	if (blk == ICE_BLK_FD && hw->fdir_prof)
+		return hw->fdir_prof[flow];
+
+	return NULL;
+}
+
+/**
+ * ice_fdir_erase_flow_from_hw - remove a flow from the HW profile tables
+ * @hw: hardware structure containing the filter list
+ * @blk: hardware block
+ * @flow: FDir flow type to release
+ */
+static void
+ice_fdir_erase_flow_from_hw(struct ice_hw *hw, enum ice_block blk, int flow)
+{
+	struct ice_fd_hw_prof *prof = ice_fdir_get_hw_prof(hw, blk, flow);
+	int tun;
+
+	if (!prof)
+		return;
+
+	for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
+		u64 prof_id = prof->prof_id[tun];
+		int j;
+
+		for (j = 0; j < prof->cnt; j++) {
+			u16 vsi_num;
+
+			if (!prof->entry_h[j][tun] || !prof->vsi_h[j])
+				continue;
+			vsi_num = ice_get_hw_vsi_num(hw, prof->vsi_h[j]);
+			ice_rem_prof_id_flow(hw, blk, vsi_num, prof_id);
+			ice_flow_rem_entry(hw, blk, prof->entry_h[j][tun]);
+			prof->entry_h[j][tun] = 0;
+		}
+		ice_flow_rem_prof(hw, blk, prof_id);
+	}
+}
+
+/**
+ * ice_fdir_rem_flow - release the ice_flow structures for a filter type
+ * @hw: hardware structure containing the filter list
+ * @blk: hardware block
+ * @flow_type: FDir flow type to release
+ */
+static void
+ice_fdir_rem_flow(struct ice_hw *hw, enum ice_block blk,
+		  enum ice_fltr_ptype flow_type)
+{
+	int flow = (int)flow_type & ~FLOW_EXT;
+	struct ice_fd_hw_prof *prof;
+	int tun, i;
+
+	prof = ice_fdir_get_hw_prof(hw, blk, flow);
+	if (!prof)
+		return;
+
+	ice_fdir_erase_flow_from_hw(hw, blk, flow);
+	for (i = 0; i < prof->cnt; i++)
+		prof->vsi_h[i] = 0;
+	for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
+		if (!prof->fdir_seg[tun])
+			continue;
+		devm_kfree(ice_hw_to_dev(hw), prof->fdir_seg[tun]);
+		prof->fdir_seg[tun] = NULL;
+	}
+	prof->cnt = 0;
+}
+
+/**
+ * ice_fdir_release_flows - release all flows in use for later replay
+ * @hw: pointer to HW instance
+ */
+void ice_fdir_release_flows(struct ice_hw *hw)
+{
+	int flow;
+
+	/* release Flow Director HW table entries */
+	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++)
+		ice_fdir_erase_flow_from_hw(hw, ICE_BLK_FD, flow);
+}
+
+/**
+ * ice_fdir_replay_flows - replay HW Flow Director filter info
+ * @hw: pointer to HW instance
+ */
+void ice_fdir_replay_flows(struct ice_hw *hw)
+{
+	int flow;
+
+	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) {
+		int tun;
+
+		if (!hw->fdir_prof[flow] || !hw->fdir_prof[flow]->cnt)
+			continue;
+		for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
+			struct ice_flow_prof *hw_prof;
+			struct ice_fd_hw_prof *prof;
+			int j;
+
+			prof = hw->fdir_prof[flow];
+			ice_flow_add_prof(hw, ICE_BLK_FD, ICE_FLOW_RX,
+					  prof->fdir_seg[tun], TNL_SEG_CNT(tun),
+					  false, &hw_prof);
+			for (j = 0; j < prof->cnt; j++) {
+				enum ice_flow_priority prio;
+				u64 entry_h = 0;
+				int err;
+
+				prio = ICE_FLOW_PRIO_NORMAL;
+				err = ice_flow_add_entry(hw, ICE_BLK_FD,
+							 hw_prof->id,
+							 prof->vsi_h[0],
+							 prof->vsi_h[j],
+							 prio, prof->fdir_seg,
+							 &entry_h);
+				if (err) {
+					dev_err(ice_hw_to_dev(hw), "Could not replay Flow Director, flow type %d\n",
+						flow);
+					continue;
+				}
+				prof->prof_id[tun] = hw_prof->id;
+				prof->entry_h[j][tun] = entry_h;
+			}
+		}
+	}
+}
+
+/**
+ * ice_parse_rx_flow_user_data - deconstruct user-defined data
+ * @fsp: pointer to ethtool Rx flow specification
+ * @data: pointer to userdef data structure for storage
+ *
+ * Returns 0 on success, negative error value on failure
+ */
+static int
+ice_parse_rx_flow_user_data(struct ethtool_rx_flow_spec *fsp,
+			    struct ice_rx_flow_userdef *data)
+{
+	u64 value, mask;
+
+	memset(data, 0, sizeof(*data));
+	if (!(fsp->flow_type & FLOW_EXT))
+		return 0;
+
+	value = be64_to_cpu(*((__force __be64 *)fsp->h_ext.data));
+	mask = be64_to_cpu(*((__force __be64 *)fsp->m_ext.data));
+	if (!mask)
+		return 0;
+
+#define ICE_USERDEF_FLEX_WORD_M	GENMASK_ULL(15, 0)
+#define ICE_USERDEF_FLEX_OFFS_S	16
+#define ICE_USERDEF_FLEX_OFFS_M	GENMASK_ULL(31, ICE_USERDEF_FLEX_OFFS_S)
+#define ICE_USERDEF_FLEX_FLTR_M	GENMASK_ULL(31, 0)
+
+	/* 0x1fe is the maximum value for offsets stored in the internal
+	 * filtering tables.
+	 */
+#define ICE_USERDEF_FLEX_MAX_OFFS_VAL 0x1fe
+
+	if (!ice_is_mask_valid(mask, ICE_USERDEF_FLEX_FLTR_M) ||
+	    value > ICE_USERDEF_FLEX_FLTR_M)
+		return -EINVAL;
+
+	data->flex_word = value & ICE_USERDEF_FLEX_WORD_M;
+	data->flex_offset = FIELD_GET(ICE_USERDEF_FLEX_OFFS_M, value);
+	if (data->flex_offset > ICE_USERDEF_FLEX_MAX_OFFS_VAL)
+		return -EINVAL;
+
+	data->flex_fltr = true;
+
+	return 0;
+}
+
+/**
+ * ice_fdir_num_avail_fltr - return the number of unused flow director filters
+ * @hw: pointer to hardware structure
+ * @vsi: software VSI structure
+ *
+ * There are 2 filter pools: guaranteed and best effort(shared). Each VSI can
+ * use filters from either pool. The guaranteed pool is divided between VSIs.
+ * The best effort filter pool is common to all VSIs and is a device shared
+ * resource pool. The number of filters available to this VSI is the sum of
+ * the VSIs guaranteed filter pool and the global available best effort
+ * filter pool.
+ *
+ * Returns the number of available flow director filters to this VSI
+ */
+int ice_fdir_num_avail_fltr(struct ice_hw *hw, struct ice_vsi *vsi)
+{
+	u16 vsi_num = ice_get_hw_vsi_num(hw, vsi->idx);
+	u16 num_guar;
+	u16 num_be;
+
+	/* total guaranteed filters assigned to this VSI */
+	num_guar = vsi->num_gfltr;
+
+	/* total global best effort filters */
+	num_be = hw->func_caps.fd_fltr_best_effort;
+
+	/* Subtract the number of programmed filters from the global values */
+	switch (hw->mac_type) {
+	case ICE_MAC_E830:
+		num_guar -= FIELD_GET(E830_VSIQF_FD_CNT_FD_GCNT_M,
+				      rd32(hw, VSIQF_FD_CNT(vsi_num)));
+		num_be -= FIELD_GET(E830_GLQF_FD_CNT_FD_BCNT_M,
+				    rd32(hw, GLQF_FD_CNT));
+		break;
+	case ICE_MAC_E810:
+	default:
+		num_guar -= FIELD_GET(E800_VSIQF_FD_CNT_FD_GCNT_M,
+				      rd32(hw, VSIQF_FD_CNT(vsi_num)));
+		num_be -= FIELD_GET(E800_GLQF_FD_CNT_FD_BCNT_M,
+				    rd32(hw, GLQF_FD_CNT));
+	}
+
+	return num_guar + num_be;
+}
+
+/**
+ * ice_fdir_alloc_flow_prof - allocate FDir flow profile structure(s)
+ * @hw: HW structure containing the FDir flow profile structure(s)
+ * @flow: flow type to allocate the flow profile for
+ *
+ * Allocate the fdir_prof and fdir_prof[flow] if not already created. Return 0
+ * on success and negative on error.
+ */
+static int
+ice_fdir_alloc_flow_prof(struct ice_hw *hw, enum ice_fltr_ptype flow)
+{
+	if (!hw)
+		return -EINVAL;
+
+	if (!hw->fdir_prof) {
+		hw->fdir_prof = devm_kcalloc(ice_hw_to_dev(hw),
+					     ICE_FLTR_PTYPE_MAX,
+					     sizeof(*hw->fdir_prof),
+					     GFP_KERNEL);
+		if (!hw->fdir_prof)
+			return -ENOMEM;
+	}
+
+	if (!hw->fdir_prof[flow]) {
+		hw->fdir_prof[flow] = devm_kzalloc(ice_hw_to_dev(hw),
+						   sizeof(**hw->fdir_prof),
+						   GFP_KERNEL);
+		if (!hw->fdir_prof[flow])
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * ice_fdir_prof_vsi_idx - find or insert a vsi_idx in structure
+ * @prof: pointer to flow director HW profile
+ * @vsi_idx: vsi_idx to locate
+ *
+ * return the index of the vsi_idx. if vsi_idx is not found insert it
+ * into the vsi_h table.
+ */
+static u16
+ice_fdir_prof_vsi_idx(struct ice_fd_hw_prof *prof, int vsi_idx)
+{
+	u16 idx = 0;
+
+	for (idx = 0; idx < prof->cnt; idx++)
+		if (prof->vsi_h[idx] == vsi_idx)
+			return idx;
+
+	if (idx == prof->cnt)
+		prof->vsi_h[prof->cnt++] = vsi_idx;
+	return idx;
+}
+
+/**
+ * ice_fdir_set_hw_fltr_rule - Configure HW tables to generate a FDir rule
+ * @pf: pointer to the PF structure
+ * @seg: protocol header description pointer
+ * @flow: filter enum
+ * @tun: FDir segment to program
+ */
+static int
+ice_fdir_set_hw_fltr_rule(struct ice_pf *pf, struct ice_flow_seg_info *seg,
+			  enum ice_fltr_ptype flow, enum ice_fd_hw_seg tun)
+{
+	struct device *dev = ice_pf_to_dev(pf);
+	struct ice_vsi *main_vsi, *ctrl_vsi;
+	struct ice_flow_seg_info *old_seg;
+	struct ice_flow_prof *prof = NULL;
+	struct ice_fd_hw_prof *hw_prof;
+	struct ice_hw *hw = &pf->hw;
+	u64 entry1_h = 0;
+	u64 entry2_h = 0;
+	bool del_last;
+	int err;
+	int idx;
+
+	main_vsi = ice_get_main_vsi(pf);
+	if (!main_vsi)
+		return -EINVAL;
+
+	ctrl_vsi = ice_get_ctrl_vsi(pf);
+	if (!ctrl_vsi)
+		return -EINVAL;
+
+	err = ice_fdir_alloc_flow_prof(hw, flow);
+	if (err)
+		return err;
+
+	hw_prof = hw->fdir_prof[flow];
+	old_seg = hw_prof->fdir_seg[tun];
+	if (old_seg) {
+		/* This flow_type already has a changed input set.
+		 * If it matches the requested input set then we are
+		 * done. Or, if it's different then it's an error.
+		 */
+		if (!memcmp(old_seg, seg, sizeof(*seg)))
+			return -EEXIST;
+
+		/* if there are FDir filters using this flow,
+		 * then return error.
+		 */
+		if (hw->fdir_fltr_cnt[flow]) {
+			dev_err(dev, "Failed to add filter. Flow director filters on each port must have the same input set.\n");
+			return -EINVAL;
+		}
+
+		if (ice_is_arfs_using_perfect_flow(hw, flow)) {
+			dev_err(dev, "aRFS using perfect flow type %d, cannot change input set\n",
+				flow);
+			return -EINVAL;
+		}
+
+		/* remove HW filter definition */
+		ice_fdir_rem_flow(hw, ICE_BLK_FD, flow);
+	}
+
+	/* Adding a profile, but there is only one header supported.
+	 * That is the final parameters are 1 header (segment), no
+	 * actions (NULL) and zero actions 0.
+	 */
+	err = ice_flow_add_prof(hw, ICE_BLK_FD, ICE_FLOW_RX, seg,
+				TNL_SEG_CNT(tun), false, &prof);
+	if (err)
+		return err;
+	err = ice_flow_add_entry(hw, ICE_BLK_FD, prof->id, main_vsi->idx,
+				 main_vsi->idx, ICE_FLOW_PRIO_NORMAL,
+				 seg, &entry1_h);
+	if (err)
+		goto err_prof;
+	err = ice_flow_add_entry(hw, ICE_BLK_FD, prof->id, main_vsi->idx,
+				 ctrl_vsi->idx, ICE_FLOW_PRIO_NORMAL,
+				 seg, &entry2_h);
+	if (err)
+		goto err_entry;
+
+	hw_prof->fdir_seg[tun] = seg;
+	hw_prof->prof_id[tun] = prof->id;
+	hw_prof->entry_h[0][tun] = entry1_h;
+	hw_prof->entry_h[1][tun] = entry2_h;
+	hw_prof->vsi_h[0] = main_vsi->idx;
+	hw_prof->vsi_h[1] = ctrl_vsi->idx;
+	if (!hw_prof->cnt)
+		hw_prof->cnt = 2;
+
+	for (idx = 1; idx < ICE_CHNL_MAX_TC; idx++) {
+		u16 vsi_idx;
+		u16 vsi_h;
+
+		if (!ice_is_adq_active(pf) || !main_vsi->tc_map_vsi[idx])
+			continue;
+
+		entry1_h = 0;
+		vsi_h = main_vsi->tc_map_vsi[idx]->idx;
+		err = ice_flow_add_entry(hw, ICE_BLK_FD, prof->id,
+					 main_vsi->idx, vsi_h,
+					 ICE_FLOW_PRIO_NORMAL, seg,
+					 &entry1_h);
+		if (err) {
+			dev_err(dev, "Could not add Channel VSI %d to flow group\n",
+				idx);
+			goto err_unroll;
+		}
+
+		vsi_idx = ice_fdir_prof_vsi_idx(hw_prof,
+						main_vsi->tc_map_vsi[idx]->idx);
+		hw_prof->entry_h[vsi_idx][tun] = entry1_h;
+	}
+
+	return 0;
+
+err_unroll:
+	entry1_h = 0;
+	hw_prof->fdir_seg[tun] = NULL;
+
+	/* The variable del_last will be used to determine when to clean up
+	 * the VSI group data. The VSI data is not needed if there are no
+	 * segments.
+	 */
+	del_last = true;
+	for (idx = 0; idx < ICE_FD_HW_SEG_MAX; idx++)
+		if (hw_prof->fdir_seg[idx]) {
+			del_last = false;
+			break;
+		}
+
+	for (idx = 0; idx < hw_prof->cnt; idx++) {
+		u16 vsi_num = ice_get_hw_vsi_num(hw, hw_prof->vsi_h[idx]);
+
+		if (!hw_prof->entry_h[idx][tun])
+			continue;
+		ice_rem_prof_id_flow(hw, ICE_BLK_FD, vsi_num, prof->id);
+		ice_flow_rem_entry(hw, ICE_BLK_FD, hw_prof->entry_h[idx][tun]);
+		hw_prof->entry_h[idx][tun] = 0;
+		if (del_last)
+			hw_prof->vsi_h[idx] = 0;
+	}
+	if (del_last)
+		hw_prof->cnt = 0;
+err_entry:
+	ice_rem_prof_id_flow(hw, ICE_BLK_FD,
+			     ice_get_hw_vsi_num(hw, main_vsi->idx), prof->id);
+	ice_flow_rem_entry(hw, ICE_BLK_FD, entry1_h);
+err_prof:
+	ice_flow_rem_prof(hw, ICE_BLK_FD, prof->id);
+	dev_err(dev, "Failed to add filter. Flow director filters on each port must have the same input set.\n");
+
+	return err;
+}
+
+/**
+ * ice_set_init_fdir_seg
+ * @seg: flow segment for programming
+ * @l3_proto: ICE_FLOW_SEG_HDR_IPV4 or ICE_FLOW_SEG_HDR_IPV6
+ * @l4_proto: ICE_FLOW_SEG_HDR_TCP or ICE_FLOW_SEG_HDR_UDP
+ *
+ * Set the configuration for perfect filters to the provided flow segment for
+ * programming the HW filter. This is to be called only when initializing
+ * filters as this function it assumes no filters exist.
+ */
+static int
+ice_set_init_fdir_seg(struct ice_flow_seg_info *seg,
+		      enum ice_flow_seg_hdr l3_proto,
+		      enum ice_flow_seg_hdr l4_proto)
+{
+	enum ice_flow_field src_addr, dst_addr, src_port, dst_port;
+
+	if (!seg)
+		return -EINVAL;
+
+	if (l3_proto == ICE_FLOW_SEG_HDR_IPV4) {
+		src_addr = ICE_FLOW_FIELD_IDX_IPV4_SA;
+		dst_addr = ICE_FLOW_FIELD_IDX_IPV4_DA;
+	} else if (l3_proto == ICE_FLOW_SEG_HDR_IPV6) {
+		src_addr = ICE_FLOW_FIELD_IDX_IPV6_SA;
+		dst_addr = ICE_FLOW_FIELD_IDX_IPV6_DA;
+	} else {
+		return -EINVAL;
+	}
+
+	if (l4_proto == ICE_FLOW_SEG_HDR_TCP) {
+		src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT;
+	} else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) {
+		src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT;
+	} else {
+		return -EINVAL;
+	}
+
+	ICE_FLOW_SET_HDRS(seg, l3_proto | l4_proto);
+
+	/* IP source address */
+	ice_flow_set_fld(seg, src_addr, ICE_FLOW_FLD_OFF_INVAL,
+			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
+
+	/* IP destination address */
+	ice_flow_set_fld(seg, dst_addr, ICE_FLOW_FLD_OFF_INVAL,
+			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
+
+	/* Layer 4 source port */
+	ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL,
+			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
+
+	/* Layer 4 destination port */
+	ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL,
+			 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL, false);
+
+	return 0;
+}
+
+/**
+ * ice_create_init_fdir_rule
+ * @pf: PF structure
+ * @flow: filter enum
+ *
+ * Return error value or 0 on success.
+ */
+static int
+ice_create_init_fdir_rule(struct ice_pf *pf, enum ice_fltr_ptype flow)
+{
+	struct ice_flow_seg_info *seg, *tun_seg;
+	struct device *dev = ice_pf_to_dev(pf);
+	struct ice_hw *hw = &pf->hw;
+	int ret;
+
+	/* if there is already a filter rule for kind return -EINVAL */
+	if (hw->fdir_prof && hw->fdir_prof[flow] &&
+	    hw->fdir_prof[flow]->fdir_seg[0])
+		return -EINVAL;
+
+	seg = devm_kzalloc(dev, sizeof(*seg), GFP_KERNEL);
+	if (!seg)
+		return -ENOMEM;
+
+	tun_seg = devm_kcalloc(dev, ICE_FD_HW_SEG_MAX, sizeof(*tun_seg),
+			       GFP_KERNEL);
+	if (!tun_seg) {
+		devm_kfree(dev, seg);
+		return -ENOMEM;
+	}
+
+	if (flow == ICE_FLTR_PTYPE_NONF_IPV4_TCP)
+		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV4,
+					    ICE_FLOW_SEG_HDR_TCP);
+	else if (flow == ICE_FLTR_PTYPE_NONF_IPV4_UDP)
+		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV4,
+					    ICE_FLOW_SEG_HDR_UDP);
+	else if (flow == ICE_FLTR_PTYPE_NONF_IPV6_TCP)
+		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV6,
+					    ICE_FLOW_SEG_HDR_TCP);
+	else if (flow == ICE_FLTR_PTYPE_NONF_IPV6_UDP)
+		ret = ice_set_init_fdir_seg(seg, ICE_FLOW_SEG_HDR_IPV6,
+					    ICE_FLOW_SEG_HDR_UDP);
+	else
+		ret = -EINVAL;
+	if (ret)
+		goto err_exit;
+
+	/* add filter for outer headers */
+	ret = ice_fdir_set_hw_fltr_rule(pf, seg, flow, ICE_FD_HW_SEG_NON_TUN);
+	if (ret)
+		/* could not write filter, free memory */
+		goto err_exit;
+
+	/* make tunneled filter HW entries if possible */
+	memcpy(&tun_seg[1], seg, sizeof(*seg));
+	ret = ice_fdir_set_hw_fltr_rule(pf, tun_seg, flow, ICE_FD_HW_SEG_TUN);
+	if (ret)
+		/* could not write tunnel filter, but outer header filter
+		 * exists
+		 */
+		devm_kfree(dev, tun_seg);
+
+	set_bit(flow, hw->fdir_perfect_fltr);
+	return ret;
+err_exit:
+	devm_kfree(dev, tun_seg);
+	devm_kfree(dev, seg);
+
+	return -EOPNOTSUPP;
+}
+
+/**
+ * ice_set_fdir_ip4_seg
+ * @seg: flow segment for programming
+ * @tcp_ip4_spec: mask data from ethtool
+ * @l4_proto: Layer 4 protocol to program
+ * @perfect_fltr: only valid on success; returns true if perfect filter,
+ *		  false if not
+ *
+ * Set the mask data into the flow segment to be used to program HW
+ * table based on provided L4 protocol for IPv4
+ */
+static int
+ice_set_fdir_ip4_seg(struct ice_flow_seg_info *seg,
+		     struct ethtool_tcpip4_spec *tcp_ip4_spec,
+		     enum ice_flow_seg_hdr l4_proto, bool *perfect_fltr)
+{
+	enum ice_flow_field src_port, dst_port;
+
+	/* make sure we don't have any empty rule */
+	if (!tcp_ip4_spec->psrc && !tcp_ip4_spec->ip4src &&
+	    !tcp_ip4_spec->pdst && !tcp_ip4_spec->ip4dst)
+		return -EINVAL;
+
+	/* filtering on TOS not supported */
+	if (tcp_ip4_spec->tos)
+		return -EOPNOTSUPP;
+
+	if (l4_proto == ICE_FLOW_SEG_HDR_TCP) {
+		src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT;
+	} else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) {
+		src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT;
+	} else if (l4_proto == ICE_FLOW_SEG_HDR_SCTP) {
+		src_port = ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_SCTP_DST_PORT;
+	} else {
+		return -EOPNOTSUPP;
+	}
+
+	*perfect_fltr = true;
+	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV4 | l4_proto);
+
+	/* IP source address */
+	if (tcp_ip4_spec->ip4src == htonl(0xFFFFFFFF))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_SA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!tcp_ip4_spec->ip4src)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	/* IP destination address */
+	if (tcp_ip4_spec->ip4dst == htonl(0xFFFFFFFF))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_DA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!tcp_ip4_spec->ip4dst)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	/* Layer 4 source port */
+	if (tcp_ip4_spec->psrc == htons(0xFFFF))
+		ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 false);
+	else if (!tcp_ip4_spec->psrc)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	/* Layer 4 destination port */
+	if (tcp_ip4_spec->pdst == htons(0xFFFF))
+		ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 false);
+	else if (!tcp_ip4_spec->pdst)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+/**
+ * ice_set_fdir_ip4_usr_seg
+ * @seg: flow segment for programming
+ * @usr_ip4_spec: ethtool userdef packet offset
+ * @perfect_fltr: only valid on success; returns true if perfect filter,
+ *		  false if not
+ *
+ * Set the offset data into the flow segment to be used to program HW
+ * table for IPv4
+ */
+static int
+ice_set_fdir_ip4_usr_seg(struct ice_flow_seg_info *seg,
+			 struct ethtool_usrip4_spec *usr_ip4_spec,
+			 bool *perfect_fltr)
+{
+	/* first 4 bytes of Layer 4 header */
+	if (usr_ip4_spec->l4_4_bytes)
+		return -EINVAL;
+	if (usr_ip4_spec->tos)
+		return -EINVAL;
+	if (usr_ip4_spec->ip_ver)
+		return -EINVAL;
+	/* Filtering on Layer 4 protocol not supported */
+	if (usr_ip4_spec->proto)
+		return -EOPNOTSUPP;
+	/* empty rules are not valid */
+	if (!usr_ip4_spec->ip4src && !usr_ip4_spec->ip4dst)
+		return -EINVAL;
+
+	*perfect_fltr = true;
+	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV4);
+
+	/* IP source address */
+	if (usr_ip4_spec->ip4src == htonl(0xFFFFFFFF))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_SA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!usr_ip4_spec->ip4src)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	/* IP destination address */
+	if (usr_ip4_spec->ip4dst == htonl(0xFFFFFFFF))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV4_DA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!usr_ip4_spec->ip4dst)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+/**
+ * ice_set_fdir_ip6_seg
+ * @seg: flow segment for programming
+ * @tcp_ip6_spec: mask data from ethtool
+ * @l4_proto: Layer 4 protocol to program
+ * @perfect_fltr: only valid on success; returns true if perfect filter,
+ *		  false if not
+ *
+ * Set the mask data into the flow segment to be used to program HW
+ * table based on provided L4 protocol for IPv6
+ */
+static int
+ice_set_fdir_ip6_seg(struct ice_flow_seg_info *seg,
+		     struct ethtool_tcpip6_spec *tcp_ip6_spec,
+		     enum ice_flow_seg_hdr l4_proto, bool *perfect_fltr)
+{
+	enum ice_flow_field src_port, dst_port;
+
+	/* make sure we don't have any empty rule */
+	if (!memcmp(tcp_ip6_spec->ip6src, &zero_ipv6_addr_mask,
+		    sizeof(struct in6_addr)) &&
+	    !memcmp(tcp_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
+		    sizeof(struct in6_addr)) &&
+	    !tcp_ip6_spec->psrc && !tcp_ip6_spec->pdst)
+		return -EINVAL;
+
+	/* filtering on TC not supported */
+	if (tcp_ip6_spec->tclass)
+		return -EOPNOTSUPP;
+
+	if (l4_proto == ICE_FLOW_SEG_HDR_TCP) {
+		src_port = ICE_FLOW_FIELD_IDX_TCP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_TCP_DST_PORT;
+	} else if (l4_proto == ICE_FLOW_SEG_HDR_UDP) {
+		src_port = ICE_FLOW_FIELD_IDX_UDP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_UDP_DST_PORT;
+	} else if (l4_proto == ICE_FLOW_SEG_HDR_SCTP) {
+		src_port = ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT;
+		dst_port = ICE_FLOW_FIELD_IDX_SCTP_DST_PORT;
+	} else {
+		return -EINVAL;
+	}
+
+	*perfect_fltr = true;
+	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV6 | l4_proto);
+
+	if (!memcmp(tcp_ip6_spec->ip6src, &full_ipv6_addr_mask,
+		    sizeof(struct in6_addr)))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_SA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!memcmp(tcp_ip6_spec->ip6src, &zero_ipv6_addr_mask,
+			 sizeof(struct in6_addr)))
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	if (!memcmp(tcp_ip6_spec->ip6dst, &full_ipv6_addr_mask,
+		    sizeof(struct in6_addr)))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_DA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!memcmp(tcp_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
+			 sizeof(struct in6_addr)))
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	/* Layer 4 source port */
+	if (tcp_ip6_spec->psrc == htons(0xFFFF))
+		ice_flow_set_fld(seg, src_port, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 false);
+	else if (!tcp_ip6_spec->psrc)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	/* Layer 4 destination port */
+	if (tcp_ip6_spec->pdst == htons(0xFFFF))
+		ice_flow_set_fld(seg, dst_port, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 false);
+	else if (!tcp_ip6_spec->pdst)
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+/**
+ * ice_set_fdir_ip6_usr_seg
+ * @seg: flow segment for programming
+ * @usr_ip6_spec: ethtool userdef packet offset
+ * @perfect_fltr: only valid on success; returns true if perfect filter,
+ *		  false if not
+ *
+ * Set the offset data into the flow segment to be used to program HW
+ * table for IPv6
+ */
+static int
+ice_set_fdir_ip6_usr_seg(struct ice_flow_seg_info *seg,
+			 struct ethtool_usrip6_spec *usr_ip6_spec,
+			 bool *perfect_fltr)
+{
+	/* filtering on Layer 4 bytes not supported */
+	if (usr_ip6_spec->l4_4_bytes)
+		return -EOPNOTSUPP;
+	/* filtering on TC not supported */
+	if (usr_ip6_spec->tclass)
+		return -EOPNOTSUPP;
+	/* filtering on Layer 4 protocol not supported */
+	if (usr_ip6_spec->l4_proto)
+		return -EOPNOTSUPP;
+	/* empty rules are not valid */
+	if (!memcmp(usr_ip6_spec->ip6src, &zero_ipv6_addr_mask,
+		    sizeof(struct in6_addr)) &&
+	    !memcmp(usr_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
+		    sizeof(struct in6_addr)))
+		return -EINVAL;
+
+	*perfect_fltr = true;
+	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_IPV6);
+
+	if (!memcmp(usr_ip6_spec->ip6src, &full_ipv6_addr_mask,
+		    sizeof(struct in6_addr)))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_SA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!memcmp(usr_ip6_spec->ip6src, &zero_ipv6_addr_mask,
+			 sizeof(struct in6_addr)))
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	if (!memcmp(usr_ip6_spec->ip6dst, &full_ipv6_addr_mask,
+		    sizeof(struct in6_addr)))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_IPV6_DA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!memcmp(usr_ip6_spec->ip6dst, &zero_ipv6_addr_mask,
+			 sizeof(struct in6_addr)))
+		*perfect_fltr = false;
+	else
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+/**
+ * ice_fdir_vlan_valid - validate VLAN data for Flow Director rule
+ * @dev: network interface device structure
+ * @fsp: pointer to ethtool Rx flow specification
+ *
+ * Return: true if vlan data is valid, false otherwise
+ */
+static bool ice_fdir_vlan_valid(struct device *dev,
+				struct ethtool_rx_flow_spec *fsp)
+{
+	if (fsp->m_ext.vlan_etype && !eth_type_vlan(fsp->h_ext.vlan_etype))
+		return false;
+
+	if (fsp->m_ext.vlan_tci && ntohs(fsp->h_ext.vlan_tci) >= VLAN_N_VID)
+		return false;
+
+	/* proto and vlan must have vlan-etype defined */
+	if (fsp->m_u.ether_spec.h_proto && fsp->m_ext.vlan_tci &&
+	    !fsp->m_ext.vlan_etype) {
+		dev_warn(dev, "Filter with proto and vlan require also vlan-etype");
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * ice_set_ether_flow_seg - set address and protocol segments for ether flow
+ * @dev: network interface device structure
+ * @seg: flow segment for programming
+ * @eth_spec: mask data from ethtool
+ *
+ * Return: 0 on success and errno in case of error.
+ */
+static int ice_set_ether_flow_seg(struct device *dev,
+				  struct ice_flow_seg_info *seg,
+				  struct ethhdr *eth_spec)
+{
+	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_ETH);
+
+	/* empty rules are not valid */
+	if (is_zero_ether_addr(eth_spec->h_source) &&
+	    is_zero_ether_addr(eth_spec->h_dest) &&
+	    !eth_spec->h_proto)
+		return -EINVAL;
+
+	/* Ethertype */
+	if (eth_spec->h_proto == htons(0xFFFF)) {
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_ETH_TYPE,
+				 ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	} else if (eth_spec->h_proto) {
+		dev_warn(dev, "Only 0x0000 or 0xffff proto mask is allowed for flow-type ether");
+		return -EOPNOTSUPP;
+	}
+
+	/* Source MAC address */
+	if (is_broadcast_ether_addr(eth_spec->h_source))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_ETH_SA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!is_zero_ether_addr(eth_spec->h_source))
+		goto err_mask;
+
+	/* Destination MAC address */
+	if (is_broadcast_ether_addr(eth_spec->h_dest))
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_ETH_DA,
+				 ICE_FLOW_FLD_OFF_INVAL, ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	else if (!is_zero_ether_addr(eth_spec->h_dest))
+		goto err_mask;
+
+	return 0;
+
+err_mask:
+	dev_warn(dev, "Only 00:00:00:00:00:00 or ff:ff:ff:ff:ff:ff MAC address mask is allowed for flow-type ether");
+	return -EOPNOTSUPP;
+}
+
+/**
+ * ice_set_fdir_vlan_seg - set vlan segments for ether flow
+ * @seg: flow segment for programming
+ * @ext_masks: masks for additional RX flow fields
+ *
+ * Return: 0 on success and errno in case of error.
+ */
+static int
+ice_set_fdir_vlan_seg(struct ice_flow_seg_info *seg,
+		      struct ethtool_flow_ext *ext_masks)
+{
+	ICE_FLOW_SET_HDRS(seg, ICE_FLOW_SEG_HDR_VLAN);
+
+	if (ext_masks->vlan_etype) {
+		if (ext_masks->vlan_etype != htons(0xFFFF))
+			return -EOPNOTSUPP;
+
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_S_VLAN,
+				 ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	}
+
+	if (ext_masks->vlan_tci) {
+		if (ext_masks->vlan_tci != htons(0xFFFF))
+			return -EOPNOTSUPP;
+
+		ice_flow_set_fld(seg, ICE_FLOW_FIELD_IDX_C_VLAN,
+				 ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL,
+				 ICE_FLOW_FLD_OFF_INVAL, false);
+	}
+
+	return 0;
+}
+
+/**
+ * ice_cfg_fdir_xtrct_seq - Configure extraction sequence for the given filter
+ * @pf: PF structure
+ * @fsp: pointer to ethtool Rx flow specification
+ * @user: user defined data from flow specification
+ *
+ * Returns 0 on success.
+ */
+static int
+ice_cfg_fdir_xtrct_seq(struct ice_pf *pf, struct ethtool_rx_flow_spec *fsp,
+		       struct ice_rx_flow_userdef *user)
+{
+	struct ice_flow_seg_info *seg, *tun_seg;
+	struct device *dev = ice_pf_to_dev(pf);
+	enum ice_fltr_ptype fltr_idx;
+	struct ice_hw *hw = &pf->hw;
+	bool perfect_filter = false;
+	int ret;
+
+	seg = devm_kzalloc(dev, sizeof(*seg), GFP_KERNEL);
+	if (!seg)
+		return -ENOMEM;
+
+	tun_seg = devm_kcalloc(dev, ICE_FD_HW_SEG_MAX, sizeof(*tun_seg),
+			       GFP_KERNEL);
+	if (!tun_seg) {
+		devm_kfree(dev, seg);
+		return -ENOMEM;
+	}
+
+	switch (fsp->flow_type & ~FLOW_EXT) {
+	case TCP_V4_FLOW:
+		ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec,
+					   ICE_FLOW_SEG_HDR_TCP,
+					   &perfect_filter);
+		break;
+	case UDP_V4_FLOW:
+		ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec,
+					   ICE_FLOW_SEG_HDR_UDP,
+					   &perfect_filter);
+		break;
+	case SCTP_V4_FLOW:
+		ret = ice_set_fdir_ip4_seg(seg, &fsp->m_u.tcp_ip4_spec,
+					   ICE_FLOW_SEG_HDR_SCTP,
+					   &perfect_filter);
+		break;
+	case IPV4_USER_FLOW:
+		ret = ice_set_fdir_ip4_usr_seg(seg, &fsp->m_u.usr_ip4_spec,
+					       &perfect_filter);
+		break;
+	case TCP_V6_FLOW:
+		ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec,
+					   ICE_FLOW_SEG_HDR_TCP,
+					   &perfect_filter);
+		break;
+	case UDP_V6_FLOW:
+		ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec,
+					   ICE_FLOW_SEG_HDR_UDP,
+					   &perfect_filter);
+		break;
+	case SCTP_V6_FLOW:
+		ret = ice_set_fdir_ip6_seg(seg, &fsp->m_u.tcp_ip6_spec,
+					   ICE_FLOW_SEG_HDR_SCTP,
+					   &perfect_filter);
+		break;
+	case IPV6_USER_FLOW:
+		ret = ice_set_fdir_ip6_usr_seg(seg, &fsp->m_u.usr_ip6_spec,
+					       &perfect_filter);
+		break;
+	case ETHER_FLOW:
+		ret = ice_set_ether_flow_seg(dev, seg, &fsp->m_u.ether_spec);
+		if (!ret && (fsp->m_ext.vlan_etype || fsp->m_ext.vlan_tci)) {
+			if (!ice_fdir_vlan_valid(dev, fsp)) {
+				ret = -EINVAL;
+				break;
+			}
+			ret = ice_set_fdir_vlan_seg(seg, &fsp->m_ext);
+		}
+		break;
+	default:
+		ret = -EINVAL;
+	}
+	if (ret)
+		goto err_exit;
+
+	/* tunnel segments are shifted up one. */
+	memcpy(&tun_seg[1], seg, sizeof(*seg));
+
+	if (user && user->flex_fltr) {
+		perfect_filter = false;
+		ice_flow_add_fld_raw(seg, user->flex_offset,
+				     ICE_FLTR_PRGM_FLEX_WORD_SIZE,
+				     ICE_FLOW_FLD_OFF_INVAL,
+				     ICE_FLOW_FLD_OFF_INVAL);
+		ice_flow_add_fld_raw(&tun_seg[1], user->flex_offset,
+				     ICE_FLTR_PRGM_FLEX_WORD_SIZE,
+				     ICE_FLOW_FLD_OFF_INVAL,
+				     ICE_FLOW_FLD_OFF_INVAL);
+	}
+
+	fltr_idx = ice_ethtool_flow_to_fltr(fsp->flow_type & ~FLOW_EXT);
+
+	assign_bit(fltr_idx, hw->fdir_perfect_fltr, perfect_filter);
+
+	/* add filter for outer headers */
+	ret = ice_fdir_set_hw_fltr_rule(pf, seg, fltr_idx,
+					ICE_FD_HW_SEG_NON_TUN);
+	if (ret == -EEXIST) {
+		/* Rule already exists, free memory and count as success */
+		ret = 0;
+		goto err_exit;
+	} else if (ret) {
+		/* could not write filter, free memory */
+		goto err_exit;
+	}
+
+	/* make tunneled filter HW entries if possible */
+	memcpy(&tun_seg[1], seg, sizeof(*seg));
+	ret = ice_fdir_set_hw_fltr_rule(pf, tun_seg, fltr_idx,
+					ICE_FD_HW_SEG_TUN);
+	if (ret == -EEXIST) {
+		/* Rule already exists, free memory and count as success */
+		devm_kfree(dev, tun_seg);
+		ret = 0;
+	} else if (ret) {
+		/* could not write tunnel filter, but outer filter exists */
+		devm_kfree(dev, tun_seg);
+	}
+
+	return ret;
+
+err_exit:
+	devm_kfree(dev, tun_seg);
+	devm_kfree(dev, seg);
+
+	return ret;
+}
+
+/**
+ * ice_update_per_q_fltr
+ * @vsi: ptr to VSI
+ * @q_index: queue index
+ * @inc: true to increment or false to decrement per queue filter count
+ *
+ * This function is used to keep track of per queue sideband filters
+ */
+static void ice_update_per_q_fltr(struct ice_vsi *vsi, u32 q_index, bool inc)
+{
+	struct ice_rx_ring *rx_ring;
+
+	if (!vsi->num_rxq || q_index >= vsi->num_rxq)
+		return;
+
+	rx_ring = vsi->rx_rings[q_index];
+	if (!rx_ring || !rx_ring->ch)
+		return;
+
+	if (inc)
+		atomic_inc(&rx_ring->ch->num_sb_fltr);
+	else
+		atomic_dec_if_positive(&rx_ring->ch->num_sb_fltr);
+}
+
+/**
+ * ice_fdir_write_fltr - send a flow director filter to the hardware
+ * @pf: PF data structure
+ * @input: filter structure
+ * @add: true adds filter and false removed filter
+ * @is_tun: true adds inner filter on tunnel and false outer headers
+ *
+ * Return: 0 on success and negative value on error
+ */
+int
+ice_fdir_write_fltr(struct ice_pf *pf, struct ice_ntuple_fltr *input, bool add,
+		    bool is_tun)
+{
+	struct device *dev = ice_pf_to_dev(pf);
+	struct ice_hw *hw = &pf->hw;
+	struct ice_fltr_desc desc;
+	struct ice_vsi *ctrl_vsi;
+	u8 *pkt, *frag_pkt;
+	bool has_frag;
+	int err;
+
+	ctrl_vsi = ice_get_ctrl_vsi(pf);
+	if (!ctrl_vsi)
+		return -EINVAL;
+
+	pkt = devm_kzalloc(dev, ICE_FDIR_MAX_RAW_PKT_SIZE, GFP_KERNEL);
+	if (!pkt)
+		return -ENOMEM;
+	frag_pkt = devm_kzalloc(dev, ICE_FDIR_MAX_RAW_PKT_SIZE, GFP_KERNEL);
+	if (!frag_pkt) {
+		err = -ENOMEM;
+		goto err_free;
+	}
+
+	ice_fdir_get_prgm_desc(hw, input, &desc, add);
+	err = ice_fdir_get_gen_prgm_pkt(hw, input, pkt, false, is_tun);
+	if (err)
+		goto err_free_all;
+	err = ice_prgm_fdir_fltr(ctrl_vsi, &desc, pkt);
+	if (err)
+		goto err_free_all;
+
+	/* repeat for fragment packet */
+	has_frag = ice_fdir_has_frag(input->flow_type);
+	if (has_frag) {
+		/* does not return error */
+		ice_fdir_get_prgm_desc(hw, input, &desc, add);
+		err = ice_fdir_get_gen_prgm_pkt(hw, input, frag_pkt, true,
+						is_tun);
+		if (err)
+			goto err_frag;
+		err = ice_prgm_fdir_fltr(ctrl_vsi, &desc, frag_pkt);
+		if (err)
+			goto err_frag;
+	} else {
+		devm_kfree(dev, frag_pkt);
+	}
+
+	return 0;
+
+err_free_all:
+	devm_kfree(dev, frag_pkt);
+err_free:
+	devm_kfree(dev, pkt);
+	return err;
+
+err_frag:
+	devm_kfree(dev, frag_pkt);
+	return err;
+}
+
+/**
+ * ice_fdir_write_all_fltr - send a flow director filter to the hardware
+ * @pf: PF data structure
+ * @input: filter structure
+ * @add: true adds filter and false removed filter
+ *
+ * Return: 0 on success and negative value on error
+ */
+static int
+ice_fdir_write_all_fltr(struct ice_pf *pf, struct ice_ntuple_fltr *input,
+			bool add)
+{
+	u16 port_num;
+	int tun;
+
+	for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
+		bool is_tun = tun == ICE_FD_HW_SEG_TUN;
+		int err;
+
+		if (is_tun && !ice_get_open_tunnel_port(&pf->hw, &port_num, TNL_ALL))
+			continue;
+		err = ice_fdir_write_fltr(pf, input, add, is_tun);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
+/**
+ * ice_fdir_replay_fltrs - replay filters from the HW filter list
+ * @pf: board private structure
+ */
+void ice_fdir_replay_fltrs(struct ice_pf *pf)
+{
+	struct ice_ntuple_fltr *f_rule;
+	struct ice_hw *hw = &pf->hw;
+
+	list_for_each_entry(f_rule, &hw->fdir_list_head, fltr_node) {
+		int err = ice_fdir_write_all_fltr(pf, f_rule, true);
+
+		if (err)
+			dev_dbg(ice_pf_to_dev(pf), "Flow Director error %d, could not reprogram filter %d\n",
+				err, f_rule->fltr_id);
+	}
+}
+
+/**
+ * ice_fdir_create_dflt_rules - create default perfect filters
+ * @pf: PF data structure
+ *
+ * Returns 0 for success or error.
+ */
+int ice_fdir_create_dflt_rules(struct ice_pf *pf)
+{
+	static const enum ice_fltr_ptype dflt_rules[] = {
+		ICE_FLTR_PTYPE_NONF_IPV4_TCP, ICE_FLTR_PTYPE_NONF_IPV4_UDP,
+		ICE_FLTR_PTYPE_NONF_IPV6_TCP, ICE_FLTR_PTYPE_NONF_IPV6_UDP,
+	};
+	int err;
+
+	/* Create perfect TCP and UDP rules in hardware. */
+	for (int i = 0; i < ARRAY_SIZE(dflt_rules); i++) {
+		err = ice_create_init_fdir_rule(pf, dflt_rules[i]);
+
+		if (err)
+			break;
+	}
+
+	return err;
+}
+
+/**
+ * ice_ntuple_update_cntrs - increment or decrement filter counter
+ * @hw: pointer to hardware structure
+ * @flow: filter flow type
+ * @add: true to increment, false to decrement
+ */
+static void ice_ntuple_update_cntrs(struct ice_hw *hw,
+				    enum ice_fltr_ptype flow, bool add)
+{
+	int incr = add ? 1 : -1;
+
+	hw->ntuple_active_fltr_cnt += incr;
+
+	if (flow == ICE_FLTR_PTYPE_NONF_NONE || flow >= ICE_FLTR_PTYPE_MAX)
+		ice_debug(hw, ICE_DBG_SW, "Unknown filter type %d\n", flow);
+	else
+		hw->fdir_fltr_cnt[flow] += incr;
+}
+
+/**
+ * ice_fdir_del_all_fltrs - Delete all flow director filters
+ * @vsi: the VSI being changed
+ *
+ * This function needs to be called while holding hw->fdir_fltr_lock
+ */
+void ice_fdir_del_all_fltrs(struct ice_vsi *vsi)
+{
+	struct ice_ntuple_fltr *f_rule, *tmp;
+	struct ice_pf *pf = vsi->back;
+	struct ice_hw *hw = &pf->hw;
+
+	list_for_each_entry_safe(f_rule, tmp, &hw->fdir_list_head, fltr_node) {
+		ice_fdir_write_all_fltr(pf, f_rule, false);
+		ice_ntuple_update_cntrs(hw, f_rule->flow_type, false);
+		list_del(&f_rule->fltr_node);
+		devm_kfree(ice_pf_to_dev(pf), f_rule);
+	}
+}
+
+/**
+ * ice_vsi_manage_fdir - turn on/off flow director
+ * @vsi: the VSI being changed
+ * @ena: boolean value indicating if this is an enable or disable request
+ */
+void ice_vsi_manage_fdir(struct ice_vsi *vsi, bool ena)
+{
+	struct ice_pf *pf = vsi->back;
+	struct ice_hw *hw = &pf->hw;
+	enum ice_fltr_ptype flow;
+
+	if (ena) {
+		set_bit(ICE_FLAG_FD_ENA, pf->flags);
+		ice_fdir_create_dflt_rules(pf);
+		return;
+	}
+
+	mutex_lock(&hw->fdir_fltr_lock);
+	if (!test_and_clear_bit(ICE_FLAG_FD_ENA, pf->flags))
+		goto release_lock;
+
+	ice_fdir_del_all_fltrs(vsi);
+
+	if (hw->fdir_prof)
+		for (flow = ICE_FLTR_PTYPE_NONF_NONE; flow < ICE_FLTR_PTYPE_MAX;
+		     flow++)
+			if (hw->fdir_prof[flow])
+				ice_fdir_rem_flow(hw, ICE_BLK_FD, flow);
+
+release_lock:
+	mutex_unlock(&hw->fdir_fltr_lock);
+}
+
+/**
+ * ice_fdir_do_rem_flow - delete flow and possibly add perfect flow
+ * @pf: PF structure
+ * @flow_type: FDir flow type to release
+ */
+static void
+ice_fdir_do_rem_flow(struct ice_pf *pf, enum ice_fltr_ptype flow_type)
+{
+	struct ice_hw *hw = &pf->hw;
+	bool need_perfect = false;
+
+	if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_TCP ||
+	    flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP ||
+	    flow_type == ICE_FLTR_PTYPE_NONF_IPV6_TCP ||
+	    flow_type == ICE_FLTR_PTYPE_NONF_IPV6_UDP)
+		need_perfect = true;
+
+	if (need_perfect && test_bit(flow_type, hw->fdir_perfect_fltr))
+		return;
+
+	ice_fdir_rem_flow(hw, ICE_BLK_FD, flow_type);
+	if (need_perfect)
+		ice_create_init_fdir_rule(pf, flow_type);
+}
+
+/**
+ * ice_ntuple_update_list_entry - add or delete a filter from the filter list
+ * @pf: PF structure
+ * @input: filter structure
+ * @fltr_idx: ethtool index of filter to modify
+ *
+ * Return: 0 on success and negative on errors
+ */
+static int
+ice_ntuple_update_list_entry(struct ice_pf *pf, struct ice_ntuple_fltr *input,
+			     int fltr_idx)
+{
+	struct ice_ntuple_fltr *old_fltr;
+	struct ice_hw *hw = &pf->hw;
+	struct ice_vsi *vsi;
+	int err = -ENOENT;
+
+	/* Do not update filters during reset */
+	if (ice_is_reset_in_progress(pf->state))
+		return -EBUSY;
+
+	vsi = ice_get_main_vsi(pf);
+	if (!vsi)
+		return -EINVAL;
+
+	old_fltr = ice_fdir_find_fltr_by_idx(hw, fltr_idx);
+	if (old_fltr) {
+		err = ice_fdir_write_all_fltr(pf, old_fltr, false);
+		if (err)
+			return err;
+		ice_ntuple_update_cntrs(hw, old_fltr->flow_type, false);
+		/* update sb-filters count, specific to ring->channel */
+		ice_update_per_q_fltr(vsi, old_fltr->orig_q_index, false);
+		if (!input && !hw->fdir_fltr_cnt[old_fltr->flow_type])
+			/* we just deleted the last filter of flow_type so we
+			 * should also delete the HW filter info.
+			 */
+			ice_fdir_do_rem_flow(pf, old_fltr->flow_type);
+		list_del(&old_fltr->fltr_node);
+		devm_kfree(ice_hw_to_dev(hw), old_fltr);
+	}
+	if (!input)
+		return err;
+	ice_fdir_list_add_fltr(hw, input);
+	/* update sb-filters count, specific to ring->channel */
+	ice_update_per_q_fltr(vsi, input->orig_q_index, true);
+	ice_ntuple_update_cntrs(hw, input->flow_type, true);
+	return 0;
+}
+
+/**
+ * ice_del_ntuple_ethtool - delete Flow Director or ACL filter
+ * @vsi: pointer to target VSI
+ * @cmd: command to add or delete the filter
+ *
+ * Return: 0 on success and negative values for failure
+ */
+int ice_del_ntuple_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+	struct ice_pf *pf = vsi->back;
+	struct ice_hw *hw = &pf->hw;
+	int val;
+
+	if (!test_bit(ICE_FLAG_FD_ENA, pf->flags))
+		return -EOPNOTSUPP;
+
+	/* Do not delete filters during reset */
+	if (ice_is_reset_in_progress(pf->state)) {
+		dev_err(ice_pf_to_dev(pf), "Device is resetting - deleting Flow Director filters not supported during reset\n");
+		return -EBUSY;
+	}
+
+	mutex_lock(&hw->fdir_fltr_lock);
+	val = ice_ntuple_update_list_entry(pf, NULL, fsp->location);
+	mutex_unlock(&hw->fdir_fltr_lock);
+
+	return val;
+}
+
+/**
+ * ice_update_ring_dest_vsi - update dest ring and dest VSI
+ * @vsi: pointer to target VSI
+ * @dest_vsi: ptr to dest VSI index
+ * @ring: ptr to dest ring
+ *
+ * This function updates destination VSI and queue if user specifies
+ * target queue which falls in channel's (aka ADQ) queue region
+ */
+static void
+ice_update_ring_dest_vsi(struct ice_vsi *vsi, u16 *dest_vsi, u32 *ring)
+{
+	struct ice_channel *ch;
+
+	list_for_each_entry(ch, &vsi->ch_list, list) {
+		if (!ch->ch_vsi)
+			continue;
+
+		/* make sure to locate corresponding channel based on "queue"
+		 * specified
+		 */
+		if ((*ring < ch->base_q) ||
+		    (*ring >= (ch->base_q + ch->num_rxq)))
+			continue;
+
+		/* update the dest_vsi based on channel */
+		*dest_vsi = ch->ch_vsi->idx;
+
+		/* update the "ring" to be correct based on channel */
+		*ring -= ch->base_q;
+	}
+}
+
+/**
+ * ice_ntuple_set_input_set - Set the input set for Flow Director
+ * @vsi: pointer to target VSI
+ * @fsp: pointer to ethtool Rx flow specification
+ * @input: filter structure
+ *
+ * Return: 0 on success, negative on failure
+ */
+static int
+ice_ntuple_set_input_set(struct ice_vsi *vsi, struct ethtool_rx_flow_spec *fsp,
+			 struct ice_ntuple_fltr *input)
+{
+	s16 q_index = ICE_FDIR_NO_QUEUE_IDX;
+	u16 orig_q_index = 0;
+	struct ice_pf *pf;
+	struct ice_hw *hw;
+	int flow_type;
+	u16 dest_vsi;
+	u8 dest_ctl;
+
+	if (!vsi || !fsp || !input)
+		return -EINVAL;
+
+	pf = vsi->back;
+	hw = &pf->hw;
+
+	dest_vsi = vsi->idx;
+	if (fsp->ring_cookie == RX_CLS_FLOW_DISC) {
+		dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DROP_PKT;
+	} else {
+		u32 ring = ethtool_get_flow_spec_ring(fsp->ring_cookie);
+		u8 vf = ethtool_get_flow_spec_ring_vf(fsp->ring_cookie);
+
+		if (vf) {
+			dev_err(ice_pf_to_dev(pf), "Failed to add filter. Flow director filters are not supported on VF queues.\n");
+			return -EINVAL;
+		}
+
+		if (ring >= vsi->num_rxq)
+			return -EINVAL;
+
+		orig_q_index = ring;
+		ice_update_ring_dest_vsi(vsi, &dest_vsi, &ring);
+		dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DIRECT_PKT_QINDEX;
+		q_index = ring;
+	}
+
+	input->fltr_id = fsp->location;
+	input->q_index = q_index;
+	flow_type = fsp->flow_type & ~FLOW_EXT;
+
+	/* Record the original queue index as specified by user.
+	 * with channel configuration 'q_index' becomes relative
+	 * to TC (channel).
+	 */
+	input->orig_q_index = orig_q_index;
+	input->dest_vsi = dest_vsi;
+	input->dest_ctl = dest_ctl;
+	input->fltr_status = ICE_FLTR_PRGM_DESC_FD_STATUS_FD_ID;
+	input->cnt_index = ICE_FD_SB_STAT_IDX(hw->fd_ctr_base);
+	input->flow_type = ice_ethtool_flow_to_fltr(flow_type);
+
+	if (fsp->flow_type & FLOW_EXT) {
+		memcpy(input->ext_data.usr_def, fsp->h_ext.data,
+		       sizeof(input->ext_data.usr_def));
+		input->ext_data.vlan_type = fsp->h_ext.vlan_etype;
+		input->ext_data.vlan_tag = fsp->h_ext.vlan_tci;
+		memcpy(input->ext_mask.usr_def, fsp->m_ext.data,
+		       sizeof(input->ext_mask.usr_def));
+		input->ext_mask.vlan_type = fsp->m_ext.vlan_etype;
+		input->ext_mask.vlan_tag = fsp->m_ext.vlan_tci;
+	}
+
+	switch (flow_type) {
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case SCTP_V4_FLOW:
+		input->ip.v4.dst_port = fsp->h_u.tcp_ip4_spec.pdst;
+		input->ip.v4.src_port = fsp->h_u.tcp_ip4_spec.psrc;
+		input->ip.v4.dst_ip = fsp->h_u.tcp_ip4_spec.ip4dst;
+		input->ip.v4.src_ip = fsp->h_u.tcp_ip4_spec.ip4src;
+		input->mask.v4.dst_port = fsp->m_u.tcp_ip4_spec.pdst;
+		input->mask.v4.src_port = fsp->m_u.tcp_ip4_spec.psrc;
+		input->mask.v4.dst_ip = fsp->m_u.tcp_ip4_spec.ip4dst;
+		input->mask.v4.src_ip = fsp->m_u.tcp_ip4_spec.ip4src;
+		break;
+	case IPV4_USER_FLOW:
+		input->ip.v4.dst_ip = fsp->h_u.usr_ip4_spec.ip4dst;
+		input->ip.v4.src_ip = fsp->h_u.usr_ip4_spec.ip4src;
+		input->ip.v4.l4_header = fsp->h_u.usr_ip4_spec.l4_4_bytes;
+		input->ip.v4.proto = fsp->h_u.usr_ip4_spec.proto;
+		input->ip.v4.ip_ver = fsp->h_u.usr_ip4_spec.ip_ver;
+		input->ip.v4.tos = fsp->h_u.usr_ip4_spec.tos;
+		input->mask.v4.dst_ip = fsp->m_u.usr_ip4_spec.ip4dst;
+		input->mask.v4.src_ip = fsp->m_u.usr_ip4_spec.ip4src;
+		input->mask.v4.l4_header = fsp->m_u.usr_ip4_spec.l4_4_bytes;
+		input->mask.v4.proto = fsp->m_u.usr_ip4_spec.proto;
+		input->mask.v4.ip_ver = fsp->m_u.usr_ip4_spec.ip_ver;
+		input->mask.v4.tos = fsp->m_u.usr_ip4_spec.tos;
+		break;
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+	case SCTP_V6_FLOW:
+		memcpy(input->ip.v6.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst,
+		       sizeof(struct in6_addr));
+		memcpy(input->ip.v6.src_ip, fsp->h_u.usr_ip6_spec.ip6src,
+		       sizeof(struct in6_addr));
+		input->ip.v6.dst_port = fsp->h_u.tcp_ip6_spec.pdst;
+		input->ip.v6.src_port = fsp->h_u.tcp_ip6_spec.psrc;
+		input->ip.v6.tc = fsp->h_u.tcp_ip6_spec.tclass;
+		memcpy(input->mask.v6.dst_ip, fsp->m_u.tcp_ip6_spec.ip6dst,
+		       sizeof(struct in6_addr));
+		memcpy(input->mask.v6.src_ip, fsp->m_u.tcp_ip6_spec.ip6src,
+		       sizeof(struct in6_addr));
+		input->mask.v6.dst_port = fsp->m_u.tcp_ip6_spec.pdst;
+		input->mask.v6.src_port = fsp->m_u.tcp_ip6_spec.psrc;
+		input->mask.v6.tc = fsp->m_u.tcp_ip6_spec.tclass;
+		break;
+	case IPV6_USER_FLOW:
+		memcpy(input->ip.v6.dst_ip, fsp->h_u.usr_ip6_spec.ip6dst,
+		       sizeof(struct in6_addr));
+		memcpy(input->ip.v6.src_ip, fsp->h_u.usr_ip6_spec.ip6src,
+		       sizeof(struct in6_addr));
+		input->ip.v6.l4_header = fsp->h_u.usr_ip6_spec.l4_4_bytes;
+		input->ip.v6.tc = fsp->h_u.usr_ip6_spec.tclass;
+
+		/* if no protocol requested, use IPPROTO_NONE */
+		if (!fsp->m_u.usr_ip6_spec.l4_proto)
+			input->ip.v6.proto = IPPROTO_NONE;
+		else
+			input->ip.v6.proto = fsp->h_u.usr_ip6_spec.l4_proto;
+
+		memcpy(input->mask.v6.dst_ip, fsp->m_u.usr_ip6_spec.ip6dst,
+		       sizeof(struct in6_addr));
+		memcpy(input->mask.v6.src_ip, fsp->m_u.usr_ip6_spec.ip6src,
+		       sizeof(struct in6_addr));
+		input->mask.v6.l4_header = fsp->m_u.usr_ip6_spec.l4_4_bytes;
+		input->mask.v6.tc = fsp->m_u.usr_ip6_spec.tclass;
+		input->mask.v6.proto = fsp->m_u.usr_ip6_spec.l4_proto;
+		break;
+	case ETHER_FLOW:
+		input->eth = fsp->h_u.ether_spec;
+		input->eth_mask = fsp->m_u.ether_spec;
+		break;
+	default:
+		/* not doing un-parsed flow types */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * ice_add_ntuple_ethtool - Add/Remove Flow Director or ACL filter
+ * @vsi: pointer to target VSI
+ * @cmd: command to add or delete the filter
+ *
+ * Return: 0 on success and negative values for failure
+ */
+int ice_add_ntuple_ethtool(struct ice_vsi *vsi, struct ethtool_rxnfc *cmd)
+{
+	struct ice_rx_flow_userdef userdata;
+	struct ethtool_rx_flow_spec *fsp;
+	struct ice_ntuple_fltr *input;
+	struct device *dev;
+	struct ice_pf *pf;
+	struct ice_hw *hw;
+	int fltrs_needed;
+	u32 max_location;
+	u16 tunnel_port;
+	int ret;
+
+	if (!vsi)
+		return -EINVAL;
+
+	pf = vsi->back;
+	hw = &pf->hw;
+	dev = ice_pf_to_dev(pf);
+
+	if (!test_bit(ICE_FLAG_FD_ENA, pf->flags))
+		return -EOPNOTSUPP;
+
+	/* Do not program filters during reset */
+	if (ice_is_reset_in_progress(pf->state)) {
+		dev_err(dev, "Device is resetting - adding Flow Director filters not supported during reset\n");
+		return -EBUSY;
+	}
+
+	fsp = (struct ethtool_rx_flow_spec *)&cmd->fs;
+
+	if (ice_parse_rx_flow_user_data(fsp, &userdata))
+		return -EINVAL;
+
+	if (fsp->flow_type & FLOW_MAC_EXT)
+		return -EINVAL;
+
+	ret = ice_cfg_fdir_xtrct_seq(pf, fsp, &userdata);
+	if (ret)
+		return ret;
+
+	max_location = ice_get_fdir_cnt_all(hw);
+	if (fsp->location >= max_location) {
+		dev_err(dev, "Failed to add filter. The number of ntuple filters or provided location exceed max %d.\n",
+			max_location);
+		return -ENOSPC;
+	}
+
+	/* return error if not an update and no available filters */
+	fltrs_needed = ice_get_open_tunnel_port(hw, &tunnel_port, TNL_ALL) ? 2 : 1;
+	if (!ice_fdir_find_fltr_by_idx(hw, fsp->location) &&
+	    ice_fdir_num_avail_fltr(hw, pf->vsi[vsi->idx]) < fltrs_needed) {
+		dev_err(dev, "Failed to add filter. The maximum number of flow director filters has been reached.\n");
+		return -ENOSPC;
+	}
+
+	input = devm_kzalloc(dev, sizeof(*input), GFP_KERNEL);
+	if (!input)
+		return -ENOMEM;
+
+	ret = ice_ntuple_set_input_set(vsi, fsp, input);
+	if (ret)
+		goto free_input;
+
+	mutex_lock(&hw->fdir_fltr_lock);
+	if (ice_fdir_is_dup_fltr(hw, input)) {
+		ret = -EINVAL;
+		goto release_lock;
+	}
+
+	if (userdata.flex_fltr) {
+		input->flex_fltr = true;
+		input->flex_word = cpu_to_be16(userdata.flex_word);
+		input->flex_offset = userdata.flex_offset;
+	}
+
+	input->cnt_ena = ICE_FXD_FLTR_QW0_STAT_ENA_PKTS;
+	input->fdid_prio = ICE_FXD_FLTR_QW1_FDID_PRI_THREE;
+	input->comp_report = ICE_FXD_FLTR_QW0_COMP_REPORT_SW_FAIL;
+
+	/* input struct is added to the HW filter list */
+	ret = ice_ntuple_update_list_entry(pf, input, fsp->location);
+	if (ret)
+		goto release_lock;
+
+	ret = ice_fdir_write_all_fltr(pf, input, true);
+	if (ret)
+		goto remove_sw_rule;
+
+	goto release_lock;
+
+remove_sw_rule:
+	ice_ntuple_update_cntrs(hw, input->flow_type, false);
+	/* update sb-filters count, specific to ring->channel */
+	ice_update_per_q_fltr(vsi, input->orig_q_index, false);
+	list_del(&input->fltr_node);
+release_lock:
+	mutex_unlock(&hw->fdir_fltr_lock);
+free_input:
+	if (ret)
+		devm_kfree(dev, input);
+
+	return ret;
+}
diff --git a/drivers/net/ethernet/intel/ice/ice_fdir.c b/drivers/net/ethernet/intel/ice/ice_fdir.c
index b29fbdec944245..1bc91cb4176970 100644
--- a/drivers/net/ethernet/intel/ice/ice_fdir.c
+++ b/drivers/net/ethernet/intel/ice/ice_fdir.c
@@ -648,7 +648,7 @@ ice_set_fd_desc_val(struct ice_fd_fltr_desc_ctx *ctx,
  * @add: if add is true, this is an add operation, false implies delete
  */
 void
-ice_fdir_get_prgm_desc(struct ice_hw *hw, struct ice_fdir_fltr *input,
+ice_fdir_get_prgm_desc(struct ice_hw *hw, struct ice_ntuple_fltr *input,
 		       struct ice_fltr_desc *fdesc, bool add)
 {
 	struct ice_fd_fltr_desc_ctx fdir_fltr_ctx = { 0 };
@@ -855,7 +855,7 @@ static void ice_pkt_insert_mac_addr(u8 *pkt, u8 *addr)
  * @tun: true implies generate a tunnel packet
  */
 int
-ice_fdir_get_gen_prgm_pkt(struct ice_hw *hw, struct ice_fdir_fltr *input,
+ice_fdir_get_gen_prgm_pkt(struct ice_hw *hw, struct ice_ntuple_fltr *input,
 			  u8 *pkt, bool frag, bool tun)
 {
 	enum ice_fltr_ptype flow;
@@ -1138,10 +1138,10 @@ bool ice_fdir_has_frag(enum ice_fltr_ptype flow)
  *
  * Returns pointer to filter if found or null
  */
-struct ice_fdir_fltr *
+struct ice_ntuple_fltr *
 ice_fdir_find_fltr_by_idx(struct ice_hw *hw, u32 fltr_idx)
 {
-	struct ice_fdir_fltr *rule;
+	struct ice_ntuple_fltr *rule;
 
 	list_for_each_entry(rule, &hw->fdir_list_head, fltr_node) {
 		/* rule ID found in the list */
@@ -1158,9 +1158,9 @@ ice_fdir_find_fltr_by_idx(struct ice_hw *hw, u32 fltr_idx)
  * @hw: hardware structure
  * @fltr: filter node to add to structure
  */
-void ice_fdir_list_add_fltr(struct ice_hw *hw, struct ice_fdir_fltr *fltr)
+void ice_fdir_list_add_fltr(struct ice_hw *hw, struct ice_ntuple_fltr *fltr)
 {
-	struct ice_fdir_fltr *rule, *parent = NULL;
+	struct ice_ntuple_fltr *rule, *parent = NULL;
 
 	list_for_each_entry(rule, &hw->fdir_list_head, fltr_node) {
 		/* rule ID found or pass its spot in the list */
@@ -1175,26 +1175,6 @@ void ice_fdir_list_add_fltr(struct ice_hw *hw, struct ice_fdir_fltr *fltr)
 		list_add(&fltr->fltr_node, &hw->fdir_list_head);
 }
 
-/**
- * ice_fdir_update_cntrs - increment / decrement filter counter
- * @hw: pointer to hardware structure
- * @flow: filter flow type
- * @add: true implies filters added
- */
-void
-ice_fdir_update_cntrs(struct ice_hw *hw, enum ice_fltr_ptype flow, bool add)
-{
-	int incr;
-
-	incr = add ? 1 : -1;
-	hw->fdir_active_fltr += incr;
-
-	if (flow == ICE_FLTR_PTYPE_NONF_NONE || flow >= ICE_FLTR_PTYPE_MAX)
-		ice_debug(hw, ICE_DBG_SW, "Unknown filter type %d\n", flow);
-	else
-		hw->fdir_fltr_cnt[flow] += incr;
-}
-
 /**
  * ice_cmp_ipv6_addr - compare 2 IP v6 addresses
  * @a: IP v6 address
@@ -1215,7 +1195,7 @@ static int ice_cmp_ipv6_addr(__be32 *a, __be32 *b)
  * Returns true if the filters match
  */
 static bool
-ice_fdir_comp_rules(struct ice_fdir_fltr *a,  struct ice_fdir_fltr *b)
+ice_fdir_comp_rules(struct ice_ntuple_fltr *a,  struct ice_ntuple_fltr *b)
 {
 	enum ice_fltr_ptype flow_type = a->flow_type;
 
@@ -1275,9 +1255,9 @@ ice_fdir_comp_rules(struct ice_fdir_fltr *a,  struct ice_fdir_fltr *b)
  *
  * Returns true if the filter is found in the list
  */
-bool ice_fdir_is_dup_fltr(struct ice_hw *hw, struct ice_fdir_fltr *input)
+bool ice_fdir_is_dup_fltr(struct ice_hw *hw, struct ice_ntuple_fltr *input)
 {
-	struct ice_fdir_fltr *rule;
+	struct ice_ntuple_fltr *rule;
 	bool ret = false;
 
 	list_for_each_entry(rule, &hw->fdir_list_head, fltr_node) {
diff --git a/drivers/net/ethernet/intel/ice/ice_fdir.h b/drivers/net/ethernet/intel/ice/ice_fdir.h
index 820023c0271fd5..54f51ae31b402f 100644
--- a/drivers/net/ethernet/intel/ice/ice_fdir.h
+++ b/drivers/net/ethernet/intel/ice/ice_fdir.h
@@ -160,7 +160,7 @@ struct ice_fdir_extra {
 	__be16 vlan_tag;	/* VLAN tag info */
 };
 
-struct ice_fdir_fltr {
+struct ice_ntuple_fltr {
 	struct list_head fltr_node;
 	enum ice_fltr_ptype flow_type;
 
@@ -216,18 +216,16 @@ int ice_free_fd_res_cntr(struct ice_hw *hw, u16 cntr_id);
 int ice_alloc_fd_guar_item(struct ice_hw *hw, u16 *cntr_id, u16 num_fltr);
 int ice_alloc_fd_shrd_item(struct ice_hw *hw, u16 *cntr_id, u16 num_fltr);
 void
-ice_fdir_get_prgm_desc(struct ice_hw *hw, struct ice_fdir_fltr *input,
+ice_fdir_get_prgm_desc(struct ice_hw *hw, struct ice_ntuple_fltr *input,
 		       struct ice_fltr_desc *fdesc, bool add);
 int
-ice_fdir_get_gen_prgm_pkt(struct ice_hw *hw, struct ice_fdir_fltr *input,
+ice_fdir_get_gen_prgm_pkt(struct ice_hw *hw, struct ice_ntuple_fltr *input,
 			  u8 *pkt, bool frag, bool tun);
 int ice_get_fdir_cnt_all(struct ice_hw *hw);
 int ice_fdir_num_avail_fltr(struct ice_hw *hw, struct ice_vsi *vsi);
-bool ice_fdir_is_dup_fltr(struct ice_hw *hw, struct ice_fdir_fltr *input);
+bool ice_fdir_is_dup_fltr(struct ice_hw *hw, struct ice_ntuple_fltr *input);
 bool ice_fdir_has_frag(enum ice_fltr_ptype flow);
-struct ice_fdir_fltr *
+struct ice_ntuple_fltr *
 ice_fdir_find_fltr_by_idx(struct ice_hw *hw, u32 fltr_idx);
-void
-ice_fdir_update_cntrs(struct ice_hw *hw, enum ice_fltr_ptype flow, bool add);
-void ice_fdir_list_add_fltr(struct ice_hw *hw, struct ice_fdir_fltr *input);
+void ice_fdir_list_add_fltr(struct ice_hw *hw, struct ice_ntuple_fltr *input);
 #endif /* _ICE_FDIR_H_ */
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index d88835482d3aac..707c7431b91a6c 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -4725,6 +4725,14 @@ static void ice_init_features(struct ice_pf *pf)
 	if (ice_is_safe_mode(pf))
 		return;
 
+	/* pf->dplls.lock guards TSPLL/CGU access shared between the DPLL
+	 * subsystem callbacks and the PTP periodic worker's TSPLL monitor.
+	 * Initialize it before ice_ptp_init() so the PTP kworker never sees
+	 * an uninitialized mutex, and destroy it in ice_deinit_features()
+	 * only after ice_ptp_release() has drained the kworker.
+	 */
+	mutex_init(&pf->dplls.lock);
+
 	/* initialize DDP driven features */
 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
 		ice_ptp_init(pf);
@@ -4769,6 +4777,7 @@ static void ice_deinit_features(struct ice_pf *pf)
 		ice_ptp_release(pf);
 	if (test_bit(ICE_FLAG_DPLL, pf->flags))
 		ice_dpll_deinit(pf);
+	mutex_destroy(&pf->dplls.lock);
 	if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV)
 		xa_destroy(&pf->eswitch.reprs);
 	ice_hwmon_exit(pf);
@@ -6202,7 +6211,7 @@ ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
  *
  * Features that need fixing:
  *	Cannot simultaneously enable CTAG and STAG stripping and/or insertion.
- *	These are mutually exlusive as the VSI context cannot support multiple
+ *	These are mutually exclusive as the VSI context cannot support multiple
  *	VLAN ethertypes simultaneously for stripping and/or insertion. If this
  *	is not done, then default to clearing the requested STAG offload
  *	settings.
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c
index eaec36ab6ae3f3..e54a98a0bdd08f 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp.c
+++ b/drivers/net/ethernet/intel/ice/ice_ptp.c
@@ -4,8 +4,11 @@
 #include "ice.h"
 #include "ice_lib.h"
 #include "ice_trace.h"
+#include "ice_tspll.h"
 #include "ice_txclk.h"
 
+#define ICE_TSPLL_LOG_INTERVAL		120
+
 static const char ice_pin_names[][64] = {
 	"SDP0",
 	"SDP1",
@@ -2849,6 +2852,79 @@ static void ice_ptp_maybe_trigger_tx_interrupt(struct ice_pf *pf)
 	}
 }
 
+/**
+ * ice_ptp_tspll_monitor - poll and recover TSPLL lock on E825 owner PFs
+ * @pf: Board private structure
+ *
+ * Called from the PTP periodic worker. On E825 devices that own the source
+ * timer, poll the TSPLL lock status via CGU registers and trigger a restart
+ * if the lock has been lost. The result is cached in @pf->ptp.tspll_locked
+ * so it can be consumed by the DPLL periodic worker via READ_ONCE().
+ *
+ * TSPLL lock is critical for PHC operation and must be monitored regardless
+ * of whether DPLL init succeeded or CONFIG_DPLL is enabled. Placing the
+ * monitor here makes recovery independent of the dpll subsystem.
+ *
+ * AQ read errors are rate-limited and do not stop monitoring. Lock-lost
+ * events are logged every 120 retries (~60 s at normal poll rate) to
+ * surface persistent failures without flooding the log.
+ */
+static void ice_ptp_tspll_monitor(struct ice_pf *pf)
+{
+	bool lock_lost;
+	int err;
+
+	if (pf->hw.mac_type != ICE_MAC_GENERIC_3K_E825 ||
+	    !ice_pf_src_tmr_owned(pf))
+		return;
+
+	/* Serialize the entire monitor tick against TSPLL userspace reconfig
+	 * (ice_dpll_tspll_state_on_dpll_set()). Both paths read HW state and
+	 * write pf->ptp.tspll_locked; without holding pf->dplls.lock across
+	 * the HW read here, a preempted monitor could observe stale HW state
+	 * and then overwrite an accurate cache update from the DPLL callback.
+	 * pf->dplls.lock is initialized in ice_init_features() before the PTP
+	 * kworker starts and destroyed in ice_deinit_features() only after
+	 * ice_ptp_release() has drained the kworker, so it is always valid.
+	 */
+	mutex_lock(&pf->dplls.lock);
+	err = ice_tspll_lost_lock_e825c(&pf->hw, &lock_lost);
+	if (err) {
+		mutex_unlock(&pf->dplls.lock);
+		dev_err_ratelimited(ice_pf_to_dev(pf),
+				    "Failed reading TimeSync PLL lock status (err: %d). Retrying.\n",
+				    err);
+		return;
+	}
+
+	if (lock_lost) {
+		WRITE_ONCE(pf->ptp.tspll_locked, false);
+		if (!(pf->ptp.tspll_lock_retries % ICE_TSPLL_LOG_INTERVAL))
+			dev_warn(ice_pf_to_dev(pf),
+				 "TimeSync PLL lock lost. Retrying to acquire lock.\n");
+		err = ice_tspll_restart_e825c(&pf->hw);
+		if (err)
+			dev_err_ratelimited(ice_pf_to_dev(pf),
+					    "Failed to restart TimeSync PLL (err: %d).\n",
+					    err);
+		pf->ptp.tspll_lock_retries++;
+	} else {
+		if (pf->ptp.tspll_lock_retries) {
+			const char *src_str = "unknown";
+			enum ice_clk_src clk_src;
+
+			if (!ice_tspll_get_clk_src(&pf->hw, &clk_src))
+				src_str = ice_tspll_clk_src_str(clk_src);
+			dev_info(ice_pf_to_dev(pf),
+				 "TimeSync PLL lock acquired with %s clock source after %u retries.\n",
+				 src_str, pf->ptp.tspll_lock_retries);
+		}
+		WRITE_ONCE(pf->ptp.tspll_locked, true);
+		pf->ptp.tspll_lock_retries = 0;
+	}
+	mutex_unlock(&pf->dplls.lock);
+}
+
 static void ice_ptp_periodic_work(struct kthread_work *work)
 {
 	struct ice_ptp *ptp = container_of(work, struct ice_ptp, work.work);
@@ -2858,6 +2934,8 @@ static void ice_ptp_periodic_work(struct kthread_work *work)
 	if (pf->ptp.state != ICE_PTP_READY)
 		return;
 
+	ice_ptp_tspll_monitor(pf);
+
 	err = ice_ptp_update_cached_phctime(pf);
 
 	ice_ptp_maybe_trigger_tx_interrupt(pf);
@@ -2975,6 +3053,9 @@ static int ice_ptp_rebuild_owner(struct ice_pf *pf)
 	err = ice_tspll_init(hw);
 	if (err)
 		return err;
+	/* Rebuild reinitialized TSPLL, so reset monitor retry state. */
+	WRITE_ONCE(ptp->tspll_locked, true);
+	ptp->tspll_lock_retries = 0;
 
 	/* Acquire the global hardware lock */
 	if (!ice_ptp_lock(hw)) {
@@ -3319,6 +3400,7 @@ void ice_ptp_init(struct ice_pf *pf)
 	}
 	ptp->port.port_num = hw->lane_num;
 
+	ptp->tspll_locked = true;
 	ice_ptp_init_hw(hw);
 
 	ice_ptp_init_tx_interrupt_mode(pf);
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h
index c4b0da7ce20ed6..0e40fef3b4e8a2 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp.h
+++ b/drivers/net/ethernet/intel/ice/ice_ptp.h
@@ -249,6 +249,15 @@ struct ice_ptp_pin_desc {
  * @tx_hwtstamp_discarded: number of Tx skbs discarded due to cached PHC time
  *                         being too old to correctly extend timestamp
  * @late_cached_phc_updates: number of times cached PHC update is late
+ * @tspll_locked: last observed TSPLL lock state on E825 owner PFs.
+ *	Written by the PTP periodic worker after polling the TSPLL and
+ *	intended to be read (without pf->dplls.lock) by the DPLL periodic
+ *	worker in a follow-up change. Access via READ_ONCE()/WRITE_ONCE();
+ *	precise synchronization is not required because both workers
+ *	converge on the same value within one poll period.
+ * @tspll_lock_retries: counts consecutive poll cycles in which the TSPLL
+ *	was found unlocked. Reset to zero when lock is re-acquired. Used to
+ *	rate-limit the lock-lost log message (~every 120 retries / ~60 s).
  */
 struct ice_ptp {
 	enum ice_ptp_state state;
@@ -274,6 +283,8 @@ struct ice_ptp {
 	u32 tx_hwtstamp_flushed;
 	u32 tx_hwtstamp_discarded;
 	u32 late_cached_phc_updates;
+	bool tspll_locked;
+	u32 tspll_lock_retries;
 };
 
 #define __ptp_port_to_ptp(p) \
diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c
index e04de021559665..9ef1aa66fa2b7d 100644
--- a/drivers/net/ethernet/intel/ice/ice_sriov.c
+++ b/drivers/net/ethernet/intel/ice/ice_sriov.c
@@ -924,7 +924,7 @@ static void ice_sriov_remap_vectors(struct ice_pf *pf, u16 restricted_id)
  * is correct etc. Then disable old mapping (MSI-X and queues registers), change
  * MSI-X and queues, rebuild VSI and enable new mapping.
  *
- * If it is possible (driver not binded to VF) try to remap also other VFs to
+ * If it is possible (driver not bound to VF) try to remap also other VFs to
  * linearize irqs register usage.
  */
 int ice_sriov_set_msix_vec_count(struct pci_dev *vf_dev, int msix_vec_count)
diff --git a/drivers/net/ethernet/intel/ice/ice_tspll.c b/drivers/net/ethernet/intel/ice/ice_tspll.c
index fd4b58eb9bc002..85bacb7c3a7903 100644
--- a/drivers/net/ethernet/intel/ice/ice_tspll.c
+++ b/drivers/net/ethernet/intel/ice/ice_tspll.c
@@ -129,7 +129,7 @@ static bool ice_tspll_check_params(struct ice_hw *hw,
  *
  * Return: specified clock source converted to its string name
  */
-static const char *ice_tspll_clk_src_str(enum ice_clk_src clk_src)
+const char *ice_tspll_clk_src_str(enum ice_clk_src clk_src)
 {
 	switch (clk_src) {
 	case ICE_CLK_SRC_TCXO:
@@ -334,8 +334,9 @@ static int ice_tspll_dis_sticky_bits_e82x(struct ice_hw *hw)
  * Return:
  * * %0       - success
  * * %-EINVAL - input parameters are incorrect
- * * %-EBUSY  - failed to lock TSPLL
- * * %other   - CGU read/write failure
+ * * %-EAGAIN - TSPLL configuration succeeded but lock is not acquired yet
+ * * %-EBUSY  - CGU access is busy (for example during reset)
+ * * %other   - other CGU read/write failures
  */
 static int ice_tspll_cfg_e825c(struct ice_hw *hw, enum ice_tspll_freq clk_freq,
 			       enum ice_clk_src clk_src)
@@ -467,7 +468,7 @@ static int ice_tspll_cfg_e825c(struct ice_hw *hw, enum ice_tspll_freq clk_freq,
 
 	if (!(val & ICE_CGU_RO_LOCK_TRUE_LOCK)) {
 		dev_warn(ice_hw_to_dev(hw), "CGU PLL failed to lock\n");
-		return -EBUSY;
+		return -EAGAIN;
 	}
 
 	err = ice_read_cgu_reg(hw, ICE_CGU_R9, &r9);
@@ -531,6 +532,62 @@ int ice_tspll_cfg_pps_out_e825c(struct ice_hw *hw, bool enable)
 	return ice_write_cgu_reg(hw, ICE_CGU_R9, val);
 }
 
+/**
+ * ice_tspll_lost_lock_e825c - check if TSPLL lost lock
+ * @hw: Pointer to the HW struct
+ * @lost_lock: Output flag for reporting lost lock
+ *
+ * Get E825 device TSPLL DPLL lock status.
+ *
+ * Return:
+ * * 0 - OK
+ * * negative - error
+ */
+int ice_tspll_lost_lock_e825c(struct ice_hw *hw, bool *lost_lock)
+{
+	u32 val;
+	int err;
+
+	err = ice_read_cgu_reg(hw, ICE_CGU_RO_LOCK, &val);
+	if (err)
+		return err;
+
+	*lost_lock = !FIELD_GET(ICE_CGU_RO_LOCK_TRUE_LOCK, val);
+
+	return 0;
+}
+
+/**
+ * ice_tspll_restart_e825c - trigger TSPLL restart
+ * @hw: Pointer to the HW struct
+ *
+ * Re-enable TSPLL for E825 device.
+ *
+ * Return:
+ * * 0 - OK
+ * * negative - error
+ */
+int ice_tspll_restart_e825c(struct ice_hw *hw)
+{
+	u32 val;
+	int err;
+
+	/* Read the initial values of r23 and disable the PLL */
+	err = ice_read_cgu_reg(hw, ICE_CGU_R23, &val);
+	if (err)
+		return err;
+
+	val &= ~ICE_CGU_R23_R24_TSPLL_ENABLE;
+	err = ice_write_cgu_reg(hw, ICE_CGU_R23, val);
+	if (err)
+		return err;
+
+	/* Wait at least 1 ms before reenabling PLL */
+	usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
+	val |= ICE_CGU_R23_R24_TSPLL_ENABLE;
+	return ice_write_cgu_reg(hw, ICE_CGU_R23, val);
+}
+
 /**
  * ice_tspll_cfg - Configure the Clock Generation Unit TSPLL
  * @hw: Pointer to the HW struct
@@ -577,6 +634,61 @@ static int ice_tspll_dis_sticky_bits(struct ice_hw *hw)
 	}
 }
 
+/**
+ * ice_tspll_get_clk_src - get current TSPLL clock source
+ * @hw: board private hw structure
+ * @clk_src: pointer to store clk_src value
+ *
+ * Get current TSPLL clock source settings.
+ *
+ * Return:
+ * * 0 - OK
+ * * negative - error
+ */
+int ice_tspll_get_clk_src(struct ice_hw *hw, enum ice_clk_src *clk_src)
+{
+	u32 val;
+	int err;
+
+	err = (hw->mac_type == ICE_MAC_GENERIC_3K_E825) ?
+		ice_read_cgu_reg(hw, ICE_CGU_R23, &val) :
+		ice_read_cgu_reg(hw, ICE_CGU_R24, &val);
+	if (err)
+		return err;
+
+	*clk_src = (enum ice_clk_src)FIELD_GET(ICE_CGU_R23_R24_TIME_REF_SEL,
+					       val);
+
+	return 0;
+}
+
+/**
+ * ice_tspll_set_cfg - configure TS PLL with new settings
+ * @hw: board private hw structure
+ * @clk_freq: clock frequency to program
+ * @clk_src: clock source to select (TIME_REF, or TCXO)
+ *
+ * Configure CGU with new clock source and clock frequency settings.
+ *
+ * Return:
+ * * 0 - OK
+ * * negative - error
+ */
+int ice_tspll_set_cfg(struct ice_hw *hw, enum ice_tspll_freq clk_freq,
+		      enum ice_clk_src clk_src)
+{
+	int ret;
+
+	if (!ice_tspll_check_params(hw, clk_freq, clk_src))
+		return -EINVAL;
+
+	ret = ice_tspll_dis_sticky_bits(hw);
+	if (ret)
+		return ret;
+
+	return ice_tspll_cfg(hw, clk_freq, clk_src);
+}
+
 /**
  * ice_tspll_init - Initialize TSPLL with settings from firmware
  * @hw: Pointer to the HW structure
diff --git a/drivers/net/ethernet/intel/ice/ice_tspll.h b/drivers/net/ethernet/intel/ice/ice_tspll.h
index d650867004d1fd..832c049e686cc9 100644
--- a/drivers/net/ethernet/intel/ice/ice_tspll.h
+++ b/drivers/net/ethernet/intel/ice/ice_tspll.h
@@ -32,6 +32,11 @@ struct ice_tspll_params_e82x {
 #define ICE_TSPLL_FBDIV_INTGR_E825		256
 
 int ice_tspll_cfg_pps_out_e825c(struct ice_hw *hw, bool enable);
+int ice_tspll_lost_lock_e825c(struct ice_hw *hw, bool *lost_lock);
+int ice_tspll_restart_e825c(struct ice_hw *hw);
+int ice_tspll_get_clk_src(struct ice_hw *hw, enum ice_clk_src *clk_src);
+int ice_tspll_set_cfg(struct ice_hw *hw, enum ice_tspll_freq clk_freq,
+		      enum ice_clk_src clk_src);
 int ice_tspll_init(struct ice_hw *hw);
 int ice_tspll_bypass_mux_active_e825c(struct ice_hw *hw, u8 port, bool *active,
 				      enum ice_synce_clk output);
@@ -39,4 +44,5 @@ int ice_tspll_cfg_bypass_mux_e825c(struct ice_hw *hw, bool ena, u32 port_num,
 				   enum ice_synce_clk output);
 int ice_tspll_cfg_synce_ethdiv_e825c(struct ice_hw *hw,
 				     enum ice_synce_clk output);
+const char *ice_tspll_clk_src_str(enum ice_clk_src clk_src);
 #endif /* _ICE_TSPLL_H_ */
diff --git a/drivers/net/ethernet/intel/ice/ice_txclk.c b/drivers/net/ethernet/intel/ice/ice_txclk.c
index 48459f971cbf92..3bca4443f03ab3 100644
--- a/drivers/net/ethernet/intel/ice/ice_txclk.c
+++ b/drivers/net/ethernet/intel/ice/ice_txclk.c
@@ -286,12 +286,9 @@ void ice_txclk_update_and_notify(struct ice_pf *pf)
 			if (ctrl_pf != pf)
 				mutex_lock(&ctrl_pf->dplls.lock);
 			for (int i = 0; i < ICE_REF_CLK_MAX; i++) {
-				if (clk == i)
-					set_bit(ptp_port->port_num,
-						&ctrl_pf->ptp.tx_refclks[phy][i]);
-				else
-					clear_bit(ptp_port->port_num,
-						  &ctrl_pf->ptp.tx_refclks[phy][i]);
+				assign_bit(ptp_port->port_num,
+					   &ctrl_pf->ptp.tx_refclks[phy][i],
+					   clk == i);
 			}
 			if (ctrl_pf != pf)
 				mutex_unlock(&ctrl_pf->dplls.lock);
@@ -315,13 +312,10 @@ void ice_txclk_update_and_notify(struct ice_pf *pf)
 	 */
 	if (ctrl_pf != pf)
 		mutex_lock(&ctrl_pf->dplls.lock);
-	for (int i = 0; i < ICE_REF_CLK_MAX; i++)
-		if (clk == i)
-			set_bit(ptp_port->port_num,
-				&ctrl_pf->ptp.tx_refclks[phy][i]);
-		else
-			clear_bit(ptp_port->port_num,
-				  &ctrl_pf->ptp.tx_refclks[phy][i]);
+	for (int i = 0; i < ICE_REF_CLK_MAX; i++) {
+		assign_bit(ptp_port->port_num,
+			   &ctrl_pf->ptp.tx_refclks[phy][i], clk == i);
+	}
 	if (ctrl_pf != pf)
 		mutex_unlock(&ctrl_pf->dplls.lock);
 
diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h
index cf147a21270712..f1a80da6239ebd 100644
--- a/drivers/net/ethernet/intel/ice/ice_type.h
+++ b/drivers/net/ethernet/intel/ice/ice_type.h
@@ -1019,8 +1019,8 @@ struct ice_hw {
 	struct mutex fl_profs_locks[ICE_BLK_COUNT];	/* lock fltr profiles */
 	struct list_head fl_profs[ICE_BLK_COUNT];
 
-	/* Flow Director filter info */
-	int fdir_active_fltr;
+	/* ntuple filter info */
+	int ntuple_active_fltr_cnt;
 
 	struct mutex fdir_fltr_lock;	/* protect Flow Director */
 	struct list_head fdir_list_head;
diff --git a/drivers/net/ethernet/intel/ice/virt/fdir.c b/drivers/net/ethernet/intel/ice/virt/fdir.c
index 4f1f3442e52c42..eca9eda04f3121 100644
--- a/drivers/net/ethernet/intel/ice/virt/fdir.c
+++ b/drivers/net/ethernet/intel/ice/virt/fdir.c
@@ -38,7 +38,7 @@ enum ice_fdir_tunnel_type {
 };
 
 struct virtchnl_fdir_fltr_conf {
-	struct ice_fdir_fltr input;
+	struct ice_ntuple_fltr input;
 	enum ice_fdir_tunnel_type ttype;
 	u64 inset_flag;
 	u32 flow_id;
@@ -567,12 +567,12 @@ static bool
 ice_vc_fdir_has_prof_conflict(struct ice_vf *vf,
 			      struct virtchnl_fdir_fltr_conf *conf)
 {
-	struct ice_fdir_fltr *desc;
+	struct ice_ntuple_fltr *desc;
 
 	list_for_each_entry(desc, &vf->fdir.fdir_rule_list, fltr_node) {
 		struct virtchnl_fdir_fltr_conf *existing_conf;
 		enum ice_fltr_ptype flow_type_a, flow_type_b;
-		struct ice_fdir_fltr *a, *b;
+		struct ice_ntuple_fltr *a, *b;
 
 		existing_conf = to_fltr_conf_from_desc(desc);
 		a = &existing_conf->input;
@@ -748,7 +748,7 @@ static int
 ice_vc_fdir_config_input_set(struct ice_vf *vf, struct virtchnl_fdir_add *fltr,
 			     struct virtchnl_fdir_fltr_conf *conf, int tun)
 {
-	struct ice_fdir_fltr *input = &conf->input;
+	struct ice_ntuple_fltr *input = &conf->input;
 	struct device *dev = ice_pf_to_dev(vf->pf);
 	struct ice_flow_seg_info *seg;
 	enum ice_fltr_ptype flow;
@@ -924,8 +924,8 @@ ice_vc_fdir_parse_pattern(struct ice_vf *vf, struct virtchnl_fdir_add *fltr,
 	struct virtchnl_proto_hdrs *proto = &fltr->rule_cfg.proto_hdrs;
 	enum virtchnl_proto_hdr_type l3 = VIRTCHNL_PROTO_HDR_NONE;
 	enum virtchnl_proto_hdr_type l4 = VIRTCHNL_PROTO_HDR_NONE;
+	struct ice_ntuple_fltr *input = &conf->input;
 	struct device *dev = ice_pf_to_dev(vf->pf);
-	struct ice_fdir_fltr *input = &conf->input;
 	int i;
 
 	if (proto->count > VIRTCHNL_MAX_NUM_PROTO_HDRS) {
@@ -1150,8 +1150,8 @@ ice_vc_fdir_parse_action(struct ice_vf *vf, struct virtchnl_fdir_add *fltr,
 			 struct virtchnl_fdir_fltr_conf *conf)
 {
 	struct virtchnl_filter_action_set *as = &fltr->rule_cfg.action_set;
+	struct ice_ntuple_fltr *input = &conf->input;
 	struct device *dev = ice_pf_to_dev(vf->pf);
-	struct ice_fdir_fltr *input = &conf->input;
 	u32 dest_num = 0;
 	u32 mark_num = 0;
 	int i;
@@ -1249,8 +1249,8 @@ static bool
 ice_vc_fdir_comp_rules(struct virtchnl_fdir_fltr_conf *conf_a,
 		       struct virtchnl_fdir_fltr_conf *conf_b)
 {
-	struct ice_fdir_fltr *a = &conf_a->input;
-	struct ice_fdir_fltr *b = &conf_b->input;
+	struct ice_ntuple_fltr *a = &conf_a->input;
+	struct ice_ntuple_fltr *b = &conf_b->input;
 
 	if (conf_a->ttype != conf_b->ttype)
 		return false;
@@ -1288,7 +1288,7 @@ ice_vc_fdir_comp_rules(struct virtchnl_fdir_fltr_conf *conf_a,
 static bool
 ice_vc_fdir_is_dup_fltr(struct ice_vf *vf, struct virtchnl_fdir_fltr_conf *conf)
 {
-	struct ice_fdir_fltr *desc;
+	struct ice_ntuple_fltr *desc;
 	bool ret;
 
 	list_for_each_entry(desc, &vf->fdir.fdir_rule_list, fltr_node) {
@@ -1317,7 +1317,7 @@ static int
 ice_vc_fdir_insert_entry(struct ice_vf *vf,
 			 struct virtchnl_fdir_fltr_conf *conf, u32 *id)
 {
-	struct ice_fdir_fltr *input = &conf->input;
+	struct ice_ntuple_fltr *input = &conf->input;
 	int i;
 
 	/* alloc ID corresponding with conf */
@@ -1341,7 +1341,7 @@ static void
 ice_vc_fdir_remove_entry(struct ice_vf *vf,
 			 struct virtchnl_fdir_fltr_conf *conf, u32 id)
 {
-	struct ice_fdir_fltr *input = &conf->input;
+	struct ice_ntuple_fltr *input = &conf->input;
 
 	idr_remove(&vf->fdir.fdir_rule_idr, id);
 	list_del(&input->fltr_node);
@@ -1367,7 +1367,7 @@ ice_vc_fdir_lookup_entry(struct ice_vf *vf, u32 id)
 static void ice_vc_fdir_flush_entry(struct ice_vf *vf)
 {
 	struct virtchnl_fdir_fltr_conf *conf;
-	struct ice_fdir_fltr *desc, *temp;
+	struct ice_ntuple_fltr *desc, *temp;
 
 	list_for_each_entry_safe(desc, temp,
 				 &vf->fdir.fdir_rule_list, fltr_node) {
@@ -1390,7 +1390,7 @@ static int ice_vc_fdir_write_fltr(struct ice_vf *vf,
 				  struct virtchnl_fdir_fltr_conf *conf,
 				  bool add, bool is_tun)
 {
-	struct ice_fdir_fltr *input = &conf->input;
+	struct ice_ntuple_fltr *input = &conf->input;
 	struct ice_vsi *vsi, *ctrl_vsi;
 	struct ice_fltr_desc desc;
 	struct device *dev;
@@ -2315,7 +2315,7 @@ int ice_vc_del_fdir_fltr(struct ice_vf *vf, u8 *msg)
 	struct virtchnl_fdir_fltr_conf *conf;
 	struct ice_vf_fdir *fdir = &vf->fdir;
 	enum virtchnl_status_code v_ret;
-	struct ice_fdir_fltr *input;
+	struct ice_ntuple_fltr *input;
 	enum ice_fltr_ptype flow;
 	struct device *dev;
 	struct ice_pf *pf;
diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h
index 470bc23c844c15..f214023095ee7c 100644
--- a/drivers/net/ethernet/intel/idpf/idpf.h
+++ b/drivers/net/ethernet/intel/idpf/idpf.h
@@ -604,6 +604,16 @@ struct idpf_vport_config {
 	DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS);
 };
 
+/**
+ * struct idpf_edt_caps_ilog2 - Host parsed EDT capabilities.
+ * @time_horizon_ns: Total time window in nanoseconds.
+ * @tstamp_granularity_pow2: Log2 of timestamp granularity in nanoseconds.
+ */
+struct idpf_edt_caps_ilog2 {
+	u32 time_horizon_ns;
+	u8 tstamp_granularity_pow2;
+};
+
 #define idpf_for_each_vport(adapter, iter) \
 	for (struct idpf_vport **__##iter = &(adapter)->vports[0], \
 	     *iter = (adapter)->max_vports ? *__##iter : NULL; \
@@ -657,6 +667,7 @@ struct idpf_vport_config {
  * @stats_task: Periodic statistics retrieval task
  * @stats_wq: Workqueue for statistics task
  * @caps: Negotiated capabilities with device
+ * @edt_caps: Negotiated EDT capabilities with device
  * @dev_ops: See idpf_dev_ops
  * @cdev_info: IDC core device info pointer
  * @num_vfs: Number of allocated VFs through sysfs. PF does not directly talk
@@ -720,6 +731,7 @@ struct idpf_adapter {
 	struct delayed_work stats_task;
 	struct workqueue_struct *stats_wq;
 	struct virtchnl2_get_capabilities caps;
+	struct idpf_edt_caps_ilog2 edt_caps;
 
 	struct idpf_dev_ops dev_ops;
 	struct iidc_rdma_core_dev_info *cdev_info;
diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c
index 827c795afcb621..cd1b173e3b2761 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_lib.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c
@@ -817,6 +817,21 @@ static void idpf_attach_and_open(struct idpf_adapter *adapter)
 	}
 }
 
+static void idpf_vport_set_pacing_offload(struct idpf_vport *vport,
+					  struct net_device *netdev)
+{
+	struct idpf_adapter *adapter = vport->adapter;
+	u32 max_horizon = 0;
+
+	if (adapter->edt_caps.time_horizon_ns &&
+	    idpf_is_queue_model_split(vport->dflt_qv_rsrc.txq_model) &&
+	    !idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS,
+			     VIRTCHNL2_CAP_SPLITQ_QSCHED))
+		max_horizon = adapter->edt_caps.time_horizon_ns;
+
+	WRITE_ONCE(netdev->max_pacing_offload_horizon, max_horizon);
+}
+
 /**
  * idpf_cfg_netdev - Allocate, configure and register a netdev
  * @vport: main vport structure
@@ -850,6 +865,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport)
 		np->max_tx_hdr_size = idpf_get_max_tx_hdr_size(adapter);
 		vport->netdev = netdev;
 
+		idpf_vport_set_pacing_offload(vport, netdev);
+
 		return idpf_init_mac_addr(vport, netdev);
 	}
 
@@ -890,6 +907,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport)
 	netdev->min_mtu = ETH_MIN_MTU;
 	netdev->max_mtu = vport->max_mtu;
 
+	idpf_vport_set_pacing_offload(vport, netdev);
+
 	dflt_features = NETIF_F_SG	|
 			NETIF_F_HIGHDMA;
 
diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
index 4311ffa30bb183..885df6294ad94c 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c
@@ -487,7 +487,6 @@ static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq,
 
 /**
  * idpf_rx_desc_rel_all - Free Rx Resources for All Queues
- * @vport: virtual port structure
  * @rsrc: pointer to queue and vector resources
  *
  * Free all rx queues resources
@@ -1394,7 +1393,6 @@ static void idpf_rxq_group_rel(struct idpf_q_vec_rsrc *rsrc)
 
 /**
  * idpf_vport_queue_grp_rel_all - Release all queue groups
- * @vport: vport to release queue groups for
  * @rsrc: pointer to queue and vector resources
  */
 static void idpf_vport_queue_grp_rel_all(struct idpf_q_vec_rsrc *rsrc)
@@ -1742,6 +1740,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport,
 			q->desc_count = rsrc->txq_desc_count;
 			q->tx_max_bufs = idpf_get_max_tx_bufs(adapter);
 			q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter);
+			q->ts_gran_pow2 = adapter->edt_caps.tstamp_granularity_pow2;
 			q->netdev = vport->netdev;
 			q->txq_grp = tx_qgrp;
 			q->rel_q_id = j;
@@ -2408,7 +2407,12 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc,
 				    struct idpf_tx_splitq_params *params,
 				    u16 td_cmd, u16 size)
 {
-	*(__le32 *)&desc->flow.qw1.cmd_dtype = cpu_to_le32((u8)(params->dtype | td_cmd));
+	desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd);
+
+	desc->flow.qw1.ts[0] = params->offload.desc_ts[0];
+	desc->flow.qw1.ts[1] = params->offload.desc_ts[1];
+	desc->flow.qw1.ts[2] = params->offload.desc_ts[2];
+
 	desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size);
 	desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag);
 }
@@ -3011,6 +3015,63 @@ static bool idpf_tx_splitq_need_re(struct idpf_tx_queue *tx_q)
 	return gap >= IDPF_TX_SPLITQ_RE_MIN_GAP;
 }
 
+static void idpf_tx_splitq_set_txtime(const struct sk_buff *skb,
+				      const struct idpf_tx_queue *tx_q,
+				      struct idpf_tx_splitq_params *tx_params)
+{
+	const int offload_slack_ns = 400;
+	u64 ts, now, horizon;
+
+	if (!tx_q->netdev->pacing_offload)
+		return;
+
+	horizon = READ_ONCE(tx_q->netdev->max_pacing_offload_horizon);
+	if (!horizon)
+		return;
+
+	switch (skb->tstamp_type) {
+	case SKB_CLOCK_REALTIME:
+		ts = ktime_to_ns(ktime_add(skb->tstamp,
+					   ktime_mono_to_any(0, TK_OFFS_TAI) -
+					   ktime_mono_to_any(0, TK_OFFS_REAL)));
+		break;
+	case SKB_CLOCK_MONOTONIC:
+		ts = ktime_to_ns(ktime_mono_to_any(skb->tstamp, TK_OFFS_TAI));
+		break;
+	case SKB_CLOCK_TAI:
+		ts = ktime_to_ns(skb->tstamp);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		return;
+	}
+
+	now = ktime_get_clocktai_ns();
+	if (ts < now + offload_slack_ns)
+		return;
+
+	/* beyond offload horizon? set overflow bit only */
+	if (ts > now + horizon) {
+		tx_params->offload.desc_ts[2] =
+			IDPF_TXD_FLOW_SCH_HORIZON_OVERFLOW_M;
+		return;
+	}
+
+	ts >>= tx_q->ts_gran_pow2;
+
+	/* 0 is valid 23b timestamp, but also means field unset.
+	 * Increase by one to avoid this case
+	 */
+	if ((ts & 0x7fffff) == 0) {
+		tx_params->offload.desc_ts[0] = 1;
+		return;
+	}
+
+	tx_params->offload.desc_ts[0] = ts & 0xff;
+	tx_params->offload.desc_ts[1] = (ts >> 8) & 0xff;
+	tx_params->offload.desc_ts[2] = ((ts >> 16) & 0x7f);
+}
+
 /**
  * idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors
  * @skb: send buffer
@@ -3097,6 +3158,10 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb,
 
 		tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE;
 		tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP;
+
+		if (skb->tstamp)
+			idpf_tx_splitq_set_txtime(skb, tx_q, &tx_params);
+
 		/* Set the RE bit periodically to "clean" the descriptor ring */
 		if (idpf_tx_splitq_need_re(tx_q)) {
 			tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE;
diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h
index 93547597efd2a6..fe0c913f9bb954 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h
+++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h
@@ -161,6 +161,7 @@ union idpf_tx_flex_desc {
  * @tso_segs: Number of segments to be sent
  * @tso_hdr_len: Length of headers to be duplicated
  * @td_cmd: Command field to be inserted into descriptor
+ * @desc_ts: Flow scheduling offload timestamp
  */
 struct idpf_tx_offload_params {
 	u32 tx_flags;
@@ -174,6 +175,7 @@ struct idpf_tx_offload_params {
 	u16 tso_hdr_len;
 
 	u16 td_cmd;
+	u8 desc_ts[3];
 };
 
 /**
@@ -608,6 +610,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue,
  *	 hot path TX pointers stored in vport. Used in both singleq/splitq.
  * @desc_count: Number of descriptors
  * @tx_min_pkt_len: Min supported packet length
+ * @ts_gran_pow2: Txtime timestamp granularity in nanoseconds (log2).
  * @thresh: XDP queue cleaning threshold
  * @netdev: &net_device corresponding to this queue
  * @next_to_use: Next descriptor to use
@@ -666,7 +669,10 @@ struct idpf_tx_queue {
 	u16 desc_count;
 
 	union {
-		u16 tx_min_pkt_len;
+		struct {
+			u16 tx_min_pkt_len;
+			u8 ts_gran_pow2;
+		};
 		u32 thresh;
 	};
 
diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c
index 1caf527069737f..928de868fa624e 100644
--- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c
+++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c
@@ -3,6 +3,7 @@
 
 #include <linux/export.h>
 #include <linux/net/intel/libie/pci.h>
+#include <linux/log2.h>
 #include <net/libeth/rx.h>
 
 #include "idpf.h"
@@ -553,7 +554,8 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter)
 			    VIRTCHNL2_CAP_SPLITQ_QSCHED		|
 			    VIRTCHNL2_CAP_PROMISC		|
 			    VIRTCHNL2_CAP_LOOPBACK		|
-			    VIRTCHNL2_CAP_PTP);
+			    VIRTCHNL2_CAP_PTP			|
+			    VIRTCHNL2_CAP_EDT);
 
 	err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps);
 	if (err)
@@ -573,6 +575,54 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter)
 	return err;
 }
 
+/**
+ * idpf_send_get_edt_caps_msg - Send virtchnl get EDT caps msg
+ * @adapter: Driver specific private struct
+ *
+ * Return: 0 on success or error code on failure.
+ */
+static int idpf_send_get_edt_caps_msg(struct idpf_adapter *adapter)
+{
+	struct libie_ctlq_xn_send_params xn_params = {
+		.timeout_ms	= IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC,
+		.chnl_opcode	= VIRTCHNL2_OP_GET_EDT_CAPS,
+	};
+	struct virtchnl2_edt_caps caps = {};
+	u64 gran_ns, horizon_ns;
+	int err;
+
+	err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps);
+	if (err)
+		return err;
+
+	if (xn_params.recv_mem.iov_len < sizeof(caps)) {
+		err = -EIO;
+		goto free_rx_buf;
+	}
+
+	memcpy(&caps, xn_params.recv_mem.iov_base, sizeof(caps));
+	horizon_ns = le64_to_cpu(caps.time_horizon_ns);
+	gran_ns = le64_to_cpu(caps.tstamp_granularity_ns);
+	if (horizon_ns > U32_MAX) {
+		dev_warn(&adapter->pdev->dev, "EDT horizon exceeds U32\n");
+		err = -EINVAL;
+		goto free_rx_buf;
+	}
+	if (!gran_ns || !is_power_of_2(gran_ns)) {
+		dev_warn(&adapter->pdev->dev, "Invalid EDT granularity\n");
+		err = -EINVAL;
+		goto free_rx_buf;
+	}
+
+	adapter->edt_caps.time_horizon_ns = horizon_ns;
+	adapter->edt_caps.tstamp_granularity_pow2 = ilog2(gran_ns);
+
+free_rx_buf:
+	libie_ctlq_release_rx_buf(&xn_params.recv_mem);
+
+	return err;
+}
+
 /**
  * idpf_mmio_region_non_static - Check if region is not static
  * @mmio_info: PCI resources info
@@ -3085,6 +3135,14 @@ int idpf_vc_core_init(struct idpf_adapter *adapter)
 		}
 	}
 
+	memset(&adapter->edt_caps, 0, sizeof(adapter->edt_caps));
+	if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_EDT)) {
+		err = idpf_send_get_edt_caps_msg(adapter);
+		if (err)
+			dev_err(&adapter->pdev->dev,
+				"EDT init failed, err=%d\n", err);
+	}
+
 	pci_sriov_set_totalvfs(adapter->pdev, idpf_get_max_vfs(adapter));
 	num_max_vports = idpf_get_max_vports(adapter);
 	adapter->vports = kzalloc_objs(*adapter->vports, num_max_vports);
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index d4a897a8c82c63..8b29f19a6deae8 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -8,7 +8,6 @@
 #include <linux/init.h>
 #include <linux/bitops.h>
 #include <linux/vmalloc.h>
-#include <linux/pagemap.h>
 #include <linux/netdevice.h>
 #include <linux/ipv6.h>
 #include <linux/slab.h>
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index c686ee120a143f..b187e5641202ce 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -12,7 +12,6 @@
 #include <linux/mii.h>
 #include <linux/module.h>
 #include <linux/netdevice.h>
-#include <linux/pagemap.h>
 #include <linux/pci.h>
 #include <linux/prefetch.h>
 #include <linux/sctp.h>
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 7ce46b4b482195..7eb4acf59dab68 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -3995,7 +3995,7 @@ static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
 	if (tx_flags & IXGBE_TX_FLAGS_CSUM)
 		olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
 
-	/* enble IPv4 checksum for TSO */
+	/* enable IPv4 checksum for TSO */
 	if (tx_flags & IXGBE_TX_FLAGS_IPV4)
 		olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
 
diff --git a/drivers/net/ethernet/marvell/mvmdio.c b/drivers/net/ethernet/marvell/mvmdio.c
index 2ccb8c8f5febd1..d59cf332d1ec35 100644
--- a/drivers/net/ethernet/marvell/mvmdio.c
+++ b/drivers/net/ethernet/marvell/mvmdio.c
@@ -373,8 +373,8 @@ static int orion_mdio_probe(struct platform_device *pdev)
 		writel(MVMDIO_ERR_INT_SMI_DONE,
 			dev->regs + MVMDIO_ERR_INT_MASK);
 
-	} else if (dev->err_interrupt == -EPROBE_DEFER) {
-		ret = -EPROBE_DEFER;
+	} else if (dev->err_interrupt < 0 && dev->err_interrupt != -ENXIO) {
+		ret = dev->err_interrupt;
 		goto out_mdio;
 	}
 
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index 543e566425c1e5..c2b1098d8d1db4 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -2403,7 +2403,7 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct page_pool *pool,
 	if (unlikely(xdp_buff_has_frags(xdp)))
 		num_frags = sinfo->nr_frags;
 
-	skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
+	skb = napi_build_skb(xdp->data_hard_start, PAGE_SIZE);
 	if (!skb)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h
index 061fcd444d503a..3ed5670f01c252 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h
@@ -729,7 +729,7 @@
 #define MVPP2_MAX_TXQ			8
 
 /* MVPP2_MAX_TSO_SEGS is the maximum number of fragments to allow in the GSO
- * skb. As we need a maxium of two descriptors per fragments (1 header, 1 data),
+ * skb. As we need a maximum of two descriptors per fragments (1 header, 1 data),
  * multiply this value by two to count the maximum number of skb descs needed.
  */
 #define MVPP2_MAX_TSO_SEGS		300
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
index 848ee655c6ea66..317af54369cfe2 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
@@ -5875,7 +5875,7 @@ static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port,
 	v->sw_thread_mask = *cpumask_bits(cpu_online_mask);
 	v->port = port;
 	v->irq = irq_of_parse_and_map(port_node, 0);
-	if (v->irq <= 0)
+	if (!v->irq)
 		return -EINVAL;
 	netif_napi_add(port->dev, &v->napi, mvpp2_poll);
 
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c
index 93e978bdf303c4..ad677cfddb4d7d 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c
@@ -382,7 +382,7 @@ static int mvpp2_prs_flow_find(struct mvpp2 *priv, int flow)
 	struct mvpp2_prs_entry pe;
 	int tid;
 
-	/* Go through the all entires with MVPP2_PRS_LU_FLOWS */
+	/* Go through the all entries with MVPP2_PRS_LU_FLOWS */
 	for (tid = MVPP2_PRS_TCAM_SRAM_SIZE - 1; tid >= 0; tid--) {
 		u8 bits;
 
@@ -1212,7 +1212,7 @@ static void mvpp2_prs_mh_init(struct mvpp2 *priv)
 	mvpp2_prs_hw_write(priv, &pe);
 }
 
-/* Set default entires (place holder) for promiscuous, non-promiscuous and
+/* Set default entries (place holder) for promiscuous, non-promiscuous and
  * multicast MAC addresses
  */
 static void mvpp2_prs_mac_init(struct mvpp2 *priv)
@@ -2241,7 +2241,7 @@ mvpp2_prs_mac_da_range_find(struct mvpp2 *priv, int pmap, const u8 *da,
 	struct mvpp2_prs_entry pe;
 	int tid;
 
-	/* Go through the all entires with MVPP2_PRS_LU_MAC */
+	/* Go through the all entries with MVPP2_PRS_LU_MAC */
 	for (tid = MVPP2_PE_MAC_RANGE_START;
 	     tid <= MVPP2_PE_MAC_RANGE_END; tid++) {
 		unsigned int entry_pmap;
@@ -2539,7 +2539,7 @@ int mvpp2_prs_def_flow(struct mvpp2_port *port)
 
 	/* Such entry not exist */
 	if (tid < 0) {
-		/* Go through the all entires from last to first */
+		/* Go through the all entries from last to first */
 		tid = mvpp2_prs_tcam_first_free(port->priv,
 						MVPP2_PE_LAST_FREE_TID,
 					       MVPP2_PE_FIRST_FREE_TID);
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c b/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c
index 0867fab61b1905..e4175d5bdb3bb1 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_pfvf_mbox.c
@@ -274,7 +274,6 @@ int octep_setup_pfvf_mbox(struct octep_device *oct)
 		if (!oct->mbox[ring])
 			goto free_mbox;
 
-		memset(oct->mbox[ring], 0, sizeof(struct octep_mbox));
 		memset(&oct->vf_info[i], 0, sizeof(struct octep_pfvf_info));
 		mutex_init(&oct->mbox[ring]->lock);
 		INIT_WORK(&oct->mbox[ring]->wk.work, octep_pfvf_mbox_work);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
index f5fd6138c352fd..ade394c2beba03 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
@@ -752,6 +752,16 @@ u64 cgx_features_get(void *cgxd)
 	return ((struct cgx *)cgxd)->hw_features;
 }
 
+u64 cgx_get_dmacflt_dropped_pktcnt(void *cgxd, int lmac_id)
+{
+	struct cgx *cgx = cgxd;
+
+	if (!is_lmac_valid(cgx, lmac_id))
+		return 0;
+
+	return cgx_read(cgx, lmac_id, CGXX_CMRX_RX_STAT4);
+}
+
 int cgx_stats_reset(void *cgxd, int lmac_id)
 {
 	struct cgx *cgx = cgxd;
@@ -971,15 +981,9 @@ int verify_lmac_fc_cfg(void *cgxd, int lmac_id, u8 tx_pause, u8 rx_pause,
 	if (!lmac)
 		return -ENODEV;
 
-	if (!rx_pause)
-		clear_bit(pfvf_idx, lmac->rx_fc_pfvf_bmap.bmap);
-	else
-		set_bit(pfvf_idx, lmac->rx_fc_pfvf_bmap.bmap);
+	assign_bit(pfvf_idx, lmac->rx_fc_pfvf_bmap.bmap, rx_pause);
 
-	if (!tx_pause)
-		clear_bit(pfvf_idx, lmac->tx_fc_pfvf_bmap.bmap);
-	else
-		set_bit(pfvf_idx, lmac->tx_fc_pfvf_bmap.bmap);
+	assign_bit(pfvf_idx, lmac->tx_fc_pfvf_bmap.bmap, tx_pause);
 
 	/* check if other pfvfs are using flow control */
 	if (!rx_pause && bitmap_weight(lmac->rx_fc_pfvf_bmap.bmap, lmac->rx_fc_pfvf_bmap.max)) {
@@ -1943,7 +1947,8 @@ static struct mac_ops	cgx_mac_ops    = {
 	.pfc_config =                   cgx_lmac_pfc_config,
 	.mac_get_pfc_frm_cfg   =        cgx_lmac_get_pfc_frm_cfg,
 	.mac_reset   =			cgx_lmac_reset,
-	.mac_stats_reset       =	cgx_stats_reset,
+	.get_dmacflt_dropped_pktcnt      =      cgx_get_dmacflt_dropped_pktcnt,
+	.mac_stats_reset		 =	cgx_stats_reset,
 	.mac_x2p_reset                   =      cgx_x2p_reset,
 	.mac_enadis_rx			 =      cgx_enadis_rx,
 };
@@ -2032,8 +2037,6 @@ static int cgx_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	}
 
 	list_add(&cgx->cgx_list, &cgx_list);
-
-
 	cgx_populate_features(cgx);
 
 	mutex_init(&cgx->lock);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.h b/drivers/net/ethernet/marvell/octeontx2/af/cgx.h
index 8411a75dd723f9..cd8858c4e41ce9 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.h
@@ -39,6 +39,7 @@
 #define CGXX_CMRX_INT_ENA_W1S		0x058
 #define CGXX_CMRX_RX_ID_MAP		0x060
 #define CGXX_CMRX_RX_STAT0		0x070
+#define CGXX_CMRX_RX_STAT4		0x090
 #define CGXX_CMRX_RX_LOGL_XON		0x100
 #define CGXX_CMRX_RX_LMACS		0x128
 #define CGXX_CMRX_RX_DMAC_CTL0		(0x1F8 + mac_ops->csr_offset)
@@ -187,5 +188,6 @@ int cgx_lmac_get_pfc_frm_cfg(void *cgxd, int lmac_id, u8 *tx_pause,
 int verify_lmac_fc_cfg(void *cgxd, int lmac_id, u8 tx_pause, u8 rx_pause,
 		       int pfvf_idx);
 int cgx_lmac_reset(void *cgxd, int lmac_id, u8 pf_req_flr);
+u64 cgx_get_dmacflt_dropped_pktcnt(void *cgx, int lmac_id);
 u32 cgx_get_fifo_len(void *cgxd);
 #endif /* CGX_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npa.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npa.c
index fe8f926c8b7504..8ebbf844f155ab 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npa.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npa.c
@@ -18,4 +18,3 @@ int rvu_mbox_handler_npa_cn20k_aq_enq(struct rvu *rvu,
 	return rvu_npa_aq_enq_inst(rvu, (struct npa_aq_enq_req *)req,
 				   (struct npa_aq_enq_rsp *)rsp);
 }
-EXPORT_SYMBOL(rvu_mbox_handler_npa_cn20k_aq_enq);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
index 2b67671a2bd8e5..7c829b1da3e52c 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c
@@ -857,10 +857,7 @@ npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
 	}
 
 update_en_map:
-	if (enable)
-		set_bit(index, npc_priv->en_map);
-	else
-		clear_bit(index, npc_priv->en_map);
+	assign_bit(index, npc_priv->en_map, enable);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/common.h b/drivers/net/ethernet/marvell/octeontx2/af/common.h
index 779413a383b743..78e42549d9908d 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/common.h
@@ -7,6 +7,10 @@
 #ifndef COMMON_H
 #define COMMON_H
 
+#include <linux/dma-mapping.h>
+#include <linux/gfp.h>
+#include <linux/mm.h>
+
 #include "rvu_struct.h"
 
 #define OTX2_ALIGN			128  /* Align to cacheline */
@@ -44,6 +48,33 @@ struct qmem {
 	u32		qsize;
 };
 
+static inline void *otx2_dma_alloc_coherent(struct device *dev, size_t size,
+					    dma_addr_t *dma_handle)
+{
+	dma_addr_t dma_addr;
+	void *vaddr;
+
+	vaddr = kzalloc(size, GFP_KERNEL);
+	if (!vaddr)
+		return NULL;
+
+	dma_addr = dma_map_single(dev, vaddr, size, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dev, dma_addr)) {
+		kfree(vaddr);
+		return NULL;
+	}
+
+	*dma_handle = dma_addr;
+	return vaddr;
+}
+
+static inline void otx2_dma_free_coherent(struct device *dev, size_t size,
+					  void *vaddr, dma_addr_t dma_handle)
+{
+	dma_unmap_single(dev, dma_handle, size, DMA_BIDIRECTIONAL);
+	kfree(vaddr);
+}
+
 static inline int qmem_alloc(struct device *dev, struct qmem **q,
 			     int qsize, int entry_sz)
 {
@@ -60,8 +91,11 @@ static inline int qmem_alloc(struct device *dev, struct qmem **q,
 
 	qmem->entry_sz = entry_sz;
 	qmem->alloc_sz = (qsize * entry_sz) + OTX2_ALIGN;
-	qmem->base = dma_alloc_attrs(dev, qmem->alloc_sz, &qmem->iova,
-				     GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS);
+
+	if (get_order(PAGE_ALIGN(qmem->alloc_sz)) > MAX_PAGE_ORDER)
+		return -ENOMEM;
+
+	qmem->base = otx2_dma_alloc_coherent(dev, qmem->alloc_sz, &qmem->iova);
 	if (!qmem->base)
 		return -ENOMEM;
 
@@ -80,10 +114,9 @@ static inline void qmem_free(struct device *dev, struct qmem *qmem)
 		return;
 
 	if (qmem->base)
-		dma_free_attrs(dev, qmem->alloc_sz,
-			       qmem->base - qmem->align,
-			       qmem->iova - qmem->align,
-			       DMA_ATTR_FORCE_CONTIGUOUS);
+		otx2_dma_free_coherent(dev, qmem->alloc_sz,
+				       qmem->base - qmem->align,
+				       qmem->iova - qmem->align);
 	devm_kfree(dev, qmem);
 }
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h
index 6180e68e1765a7..f07139e1f4cef8 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h
@@ -128,6 +128,8 @@ struct mac_ops {
 						       u8 *tx_pause, u8 *rx_pause);
 	int			(*mac_reset)(void *cgxd, int lmac_id, u8 pf_req_flr);
 
+	u64			(*get_dmacflt_dropped_pktcnt)(void *cgxd, int lmac_id);
+
 	/* FEC stats */
 	int			(*get_fec_stats)(void *cgxd, int lmac_id,
 						 struct cgx_fec_stats_rsp *rsp);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c
index 211c10aa5880f2..30c672d526a0cf 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c
@@ -314,7 +314,7 @@ int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir)
 	}
 
 	/* Clear SC stats */
-	for (id = 0; id < map->secy.max; id++) {
+	for (id = 0; id < map->sc.max; id++) {
 		if (map->sc2pf_map[id] != pcifunc)
 			continue;
 		mcs_clear_stats(mcs, MCS_SC_STATS, id, dir);
@@ -738,7 +738,7 @@ int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc)
 	}
 
 	/* free sc entries */
-	for (id = 0; id < map->secy.max; id++) {
+	for (id = 0; id < map->sc.max; id++) {
 		if (map->sc2pf_map[id] != pcifunc)
 			continue;
 		mcs_free_rsrc(&map->sc, map->sc2pf_map, id, pcifunc);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h
index f4c6de89002c1d..ec932a592707dd 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h
@@ -19,8 +19,6 @@
 		offset = 0xa28ull;			\
 	offset += (a) * 0x8ull;				\
 	offset; })
-
-
 #define MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(a) ({		\
 	u64 offset;					\
 							\
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
index 719b3618eeb527..309244fe875437 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h
@@ -84,7 +84,7 @@ enum npc_kpu_lc_ltype {
 	NPC_LT_LC_CUSTOM1 = 0xF,
 };
 
-/* Don't modify Ltypes upto SCTP, otherwise it will
+/* Don't modify Ltypes up to SCTP, otherwise it will
  * effect flow tag calculation and thus RSS.
  */
 enum npc_kpu_ld_ltype {
@@ -140,7 +140,7 @@ enum npc_kpu_lg_ltype {
 	NPC_LT_LG_CUSTOM1 = 0xF,
 };
 
-/* Don't modify Ltypes upto SCTP, otherwise it will
+/* Don't modify Ltypes up to SCTP, otherwise it will
  * effect flow tag calculation and thus RSS.
  */
 enum npc_kpu_lh_ltype {
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
index 2e9945446199ec..d46e8ec7230f72 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
@@ -29,18 +29,19 @@ static struct mac_ops		rpm_mac_ops   = {
 	.mac_get_tx_stats  =	rpm_get_tx_stats,
 	.get_fec_stats	   =	rpm_get_fec_stats,
 	.mac_enadis_rx_pause_fwding =	rpm_lmac_enadis_rx_pause_fwding,
-	.mac_get_pause_frm_status =	rpm_lmac_get_pause_frm_status,
-	.mac_enadis_pause_frm =		rpm_lmac_enadis_pause_frm,
-	.mac_pause_frm_config =		rpm_lmac_pause_frm_config,
-	.mac_enadis_ptp_config =	rpm_lmac_ptp_config,
-	.mac_rx_tx_enable =		rpm_lmac_rx_tx_enable,
-	.mac_tx_enable =		rpm_lmac_tx_enable,
-	.pfc_config =                   rpm_lmac_pfc_config,
-	.mac_get_pfc_frm_cfg   =        rpm_lmac_get_pfc_frm_cfg,
-	.mac_reset   =			rpm_lmac_reset,
-	.mac_stats_reset		 =	  rpm_stats_reset,
-	.mac_x2p_reset                   =        rpm_x2p_reset,
-	.mac_enadis_rx			 =        rpm_enadis_rx,
+	.mac_get_pause_frm_status   =	rpm_lmac_get_pause_frm_status,
+	.mac_enadis_pause_frm	    =	rpm_lmac_enadis_pause_frm,
+	.mac_pause_frm_config	    =	rpm_lmac_pause_frm_config,
+	.mac_enadis_ptp_config	    =	rpm_lmac_ptp_config,
+	.mac_rx_tx_enable	    =	rpm_lmac_rx_tx_enable,
+	.mac_tx_enable		    =	rpm_lmac_tx_enable,
+	.pfc_config		    =	rpm_lmac_pfc_config,
+	.mac_get_pfc_frm_cfg	    =	rpm_lmac_get_pfc_frm_cfg,
+	.mac_reset		    =	rpm_lmac_reset,
+	.get_dmacflt_dropped_pktcnt =   rpm_get_dmacflt_dropped_pktcnt,
+	.mac_stats_reset	    =	rpm_stats_reset,
+	.mac_x2p_reset              =   rpm_x2p_reset,
+	.mac_enadis_rx		    =   rpm_enadis_rx,
 };
 
 static struct mac_ops		rpm2_mac_ops   = {
@@ -73,6 +74,7 @@ static struct mac_ops		rpm2_mac_ops   = {
 	.pfc_config =                   rpm_lmac_pfc_config,
 	.mac_get_pfc_frm_cfg   =        rpm_lmac_get_pfc_frm_cfg,
 	.mac_reset   =			rpm_lmac_reset,
+	.get_dmacflt_dropped_pktcnt =   rpm_get_dmacflt_dropped_pktcnt,
 	.mac_stats_reset	    =	rpm_stats_reset,
 	.mac_x2p_reset              =   rpm_x2p_reset,
 	.mac_enadis_rx		    =   rpm_enadis_rx,
@@ -449,6 +451,20 @@ int rpm_get_tx_stats(void *rpmd, int lmac_id, int idx, u64 *tx_stat)
 	return 0;
 }
 
+u64 rpm_get_dmacflt_dropped_pktcnt(void *rpmd, int lmac_id)
+{
+	rpm_t *rpm = rpmd;
+	u64 dmac_flt_stat;
+
+	if (!is_lmac_valid(rpm, lmac_id))
+		return 0;
+
+	dmac_flt_stat = is_dev_rpm2(rpm) ? RPM2_CMRX_RX_STAT2 :
+			RPMX_CMRX_RX_STAT2;
+
+	return rpm_read(rpm, lmac_id, dmac_flt_stat);
+}
+
 int rpm_stats_reset(void *rpmd, int lmac_id)
 {
 	rpm_t *rpm = rpmd;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
index b8d3972e096aed..443481010abaee 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
@@ -60,6 +60,7 @@
 #define RPMX_MTI_STAT_RX_STAT_PAGES_COUNTERX 0x12000
 #define RPMX_MTI_STAT_TX_STAT_PAGES_COUNTERX 0x13000
 #define RPMX_MTI_STAT_DATA_HI_CDC            0x10038
+#define RPMX_CMRX_RX_STAT2		     0x4010
 
 #define RPM_LMAC_FWI			0xa
 #define RPM_TX_EN			BIT_ULL(0)
@@ -129,6 +130,7 @@ int rpm_lmac_enadis_pause_frm(void *rpmd, int lmac_id, u8 tx_pause,
 			      u8 rx_pause);
 int rpm_get_tx_stats(void *rpmd, int lmac_id, int idx, u64 *tx_stat);
 int rpm_get_rx_stats(void *rpmd, int lmac_id, int idx, u64 *rx_stat);
+u64 rpm_get_dmacflt_dropped_pktcnt(void *rpmd, int lmac_id);
 void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable);
 int rpm_lmac_rx_tx_enable(void *rpmd, int lmac_id, bool enable);
 int rpm_lmac_tx_enable(void *rpmd, int lmac_id, bool enable);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
index 937b085582b5f4..30e148291581cf 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
@@ -3371,7 +3371,7 @@ static int rvu_flr_init(struct rvu *rvu)
 	}
 
 	rvu->flr_wq = alloc_workqueue("rvu_afpf_flr",
-				      WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
+				      WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_PERCPU, 0);
 	if (!rvu->flr_wq)
 		return -ENOMEM;
 
@@ -3468,8 +3468,6 @@ int rvu_get_num_lbk_chans(void)
 	return ret;
 }
 
-#define PCI_DEVID_OCTEONTX2_RVU_AFVF	0xA0F8
-
 static int rvu_enable_sriov(struct rvu *rvu)
 {
 	struct pci_dev *pdev = rvu->pdev;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
index 094227404ef9f9..fb4870cd18e9e4 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
@@ -22,6 +22,7 @@
 
 /* PCI device IDs */
 #define	PCI_DEVID_OCTEONTX2_RVU_AF		0xA065
+#define	PCI_DEVID_OCTEONTX2_RVU_AFVF		0xA0F8
 #define	PCI_DEVID_OCTEONTX2_LBK			0xA061
 
 /* Subsystem Device ID */
@@ -882,11 +883,10 @@ static inline bool is_rvu_supports_nix1(struct rvu *rvu)
 /* Function Prototypes
  * RVU
  */
-#define	RVU_LBK_VF_DEVID	0xA0F8
 static inline bool is_lbk_vf(struct rvu *rvu, u16 pcifunc)
 {
 	return (!(pcifunc & ~RVU_PFVF_FUNC_MASK) &&
-		(rvu->vf_devid == RVU_LBK_VF_DEVID));
+		(rvu->vf_devid == PCI_DEVID_OCTEONTX2_RVU_AFVF));
 }
 
 static inline bool is_vf(u16 pcifunc)
@@ -1127,6 +1127,7 @@ int rvu_cgx_prio_flow_ctrl_cfg(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_
 			       u16 pfc_en);
 int rvu_cgx_cfg_pause_frm(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause);
 void rvu_mac_reset(struct rvu *rvu, u16 pcifunc);
+u64 rvu_cgx_get_dmacflt_dropped_pktcnt(void *cgxd, int lmac_id);
 u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac);
 void cgx_start_linkup(struct rvu *rvu);
 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam,
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c
index 87d21889dc49e2..41f157ea27f4b6 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c
@@ -1356,6 +1356,18 @@ void rvu_mac_reset(struct rvu *rvu, u16 pcifunc)
 		dev_err(rvu->dev, "Failed to reset MAC\n");
 }
 
+u64 rvu_cgx_get_dmacflt_dropped_pktcnt(void *cgxd, int lmac_id)
+{
+	struct mac_ops *mac_ops;
+
+	mac_ops = get_mac_ops(cgxd);
+
+	if (!mac_ops || !mac_ops->get_dmacflt_dropped_pktcnt)
+		return 0;
+
+	return mac_ops->get_dmacflt_dropped_pktcnt(cgxd, lmac_id);
+}
+
 /* Do not allow CGX-mapped VFs to overwrite PKIND when special parse kinds
  * (HiGig, EDSA, etc.) are in use on the shared LMAC. VFs must not program
  * NPC_TX_DEF_PKIND on NIX_AF_LFX_TX_PARSE_CFG in that case: the PF owns
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
index 904374baae6f32..33e3a627c3599c 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
@@ -714,110 +714,72 @@ static int get_max_column_width(struct rvu *rvu)
 }
 
 /* Dumps current provisioning status of all RVU block LFs */
-static ssize_t rvu_dbg_rsrc_attach_status(struct file *filp,
-					  char __user *buffer,
-					  size_t count, loff_t *ppos)
+static int rvu_dbg_rsrc_attach_status(struct seq_file *filp, void *unused)
 {
-	int index, off = 0, flag = 0, len = 0, i = 0;
-	struct rvu *rvu = filp->private_data;
-	int bytes_not_copied = 0;
+	struct rvu *rvu = filp->private;
+	int index, pf, vf, pcifunc;
 	struct rvu_block block;
-	int pf, vf, pcifunc;
-	int buf_size = 2048;
 	int lf_str_size;
 	char *lfs;
-	char *buf;
-
-	/* don't allow partial reads */
-	if (*ppos != 0)
-		return 0;
-
-	buf = kzalloc(buf_size, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
 
-	/* Get the maximum width of a column */
 	lf_str_size = get_max_column_width(rvu);
+	if (lf_str_size < 0)
+		return lf_str_size;
 
 	lfs = kzalloc(lf_str_size, GFP_KERNEL);
-	if (!lfs) {
-		kfree(buf);
+	if (!lfs)
 		return -ENOMEM;
-	}
-	off +=	scnprintf(&buf[off], buf_size - 1 - off, "%-*s", lf_str_size,
-			  "pcifunc");
-	for (index = 0; index < BLK_COUNT; index++)
-		if (strlen(rvu->hw->block[index].name)) {
-			off += scnprintf(&buf[off], buf_size - 1 - off,
-					 "%-*s", lf_str_size,
-					 rvu->hw->block[index].name);
-		}
 
-	off += scnprintf(&buf[off], buf_size - 1 - off, "\n");
-	bytes_not_copied = copy_to_user(buffer + (i * off), buf, off);
-	if (bytes_not_copied)
-		goto out;
+	seq_printf(filp, "%-*s", lf_str_size, "pcifunc");
+	for (index = 0; index < BLK_COUNT; index++)
+		if (strlen(rvu->hw->block[index].name))
+			seq_printf(filp, "%-*s", lf_str_size,
+				   rvu->hw->block[index].name);
 
-	i++;
-	*ppos += off;
+	seq_putc(filp, '\n');
 	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
 		for (vf = 0; vf <= rvu->hw->total_vfs; vf++) {
-			off = 0;
-			flag = 0;
 			pcifunc = rvu_make_pcifunc(rvu->pdev, pf, vf);
 			if (!pcifunc)
 				continue;
 
-			if (vf) {
+			for (index = 0; index < BLK_COUNT; index++) {
+				block = rvu->hw->block[index];
+				if (!strlen(block.name))
+					continue;
+				lfs[0] = '\0';
+				get_lf_str_list(&block, pcifunc, lfs);
+				if (strlen(lfs))
+					break;
+			}
+			if (index == BLK_COUNT)
+				continue;
+
+			if (vf)
 				sprintf(lfs, "PF%d:VF%d", pf, vf - 1);
-				off = scnprintf(&buf[off],
-						buf_size - 1 - off,
-						"%-*s", lf_str_size, lfs);
-			} else {
+			else
 				sprintf(lfs, "PF%d", pf);
-				off = scnprintf(&buf[off],
-						buf_size - 1 - off,
-						"%-*s", lf_str_size, lfs);
-			}
+			seq_printf(filp, "%-*s", lf_str_size, lfs);
 
 			for (index = 0; index < BLK_COUNT; index++) {
 				block = rvu->hw->block[index];
 				if (!strlen(block.name))
 					continue;
-				len = 0;
-				lfs[len] = '\0';
-				get_lf_str_list(&block, pcifunc, lfs);
-				if (strlen(lfs))
-					flag = 1;
 
-				off += scnprintf(&buf[off], buf_size - 1 - off,
-						 "%-*s", lf_str_size, lfs);
-			}
-			if (flag) {
-				off +=	scnprintf(&buf[off],
-						  buf_size - 1 - off, "\n");
-				bytes_not_copied = copy_to_user(buffer +
-								(i * off),
-								buf, off);
-				if (bytes_not_copied)
-					goto out;
-
-				i++;
-				*ppos += off;
+				lfs[0] = '\0';
+				get_lf_str_list(&block, pcifunc, lfs);
+				seq_printf(filp, "%-*s", lf_str_size, lfs);
 			}
+			seq_putc(filp, '\n');
 		}
 	}
 
-out:
 	kfree(lfs);
-	kfree(buf);
-	if (bytes_not_copied)
-		return -EFAULT;
 
-	return *ppos;
+	return 0;
 }
 
-RVU_DEBUG_FOPS(rsrc_status, rsrc_attach_status, NULL);
+RVU_DEBUG_SEQ_FOPS(rsrc_status, rsrc_attach_status, NULL);
 
 static int rvu_dbg_rvu_pf_cgx_map_display(struct seq_file *filp, void *unused)
 {
@@ -2969,8 +2931,10 @@ static int cgx_print_dmac_flt(struct seq_file *s, int lmac_id)
 	struct rvu_cgx_lmac_dbgfs_ctx *dctx = s->private;
 	struct rvu *rvu = dctx->rvu;
 	struct pci_dev *pdev = NULL;
+	struct mac_ops *mac_ops;
 	void *cgxd = dctx->cgxd;
 	char *bcast, *mcast;
+	u64 drop_cnt;
 	u16 index, domain;
 	u8 dmac[ETH_ALEN];
 	u64 cfg, mac;
@@ -3009,6 +2973,12 @@ static int cgx_print_dmac_flt(struct seq_file *s, int lmac_id)
 		}
 	}
 
+	mac_ops = get_mac_ops(cgxd);
+	if (mac_ops && mac_ops->get_dmacflt_dropped_pktcnt) {
+		drop_cnt = rvu_cgx_get_dmacflt_dropped_pktcnt(cgxd, lmac_id);
+		seq_printf(s, "\nDMAC filter drop count: %llu\n", drop_cnt);
+	}
+
 	pci_dev_put(pdev);
 	return 0;
 }
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
index 153eb57bad067f..d04540420bca96 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
@@ -324,15 +324,23 @@ static bool is_valid_txschq(struct rvu *rvu, int blkaddr,
 	if (lvl >= hw->cap.nix_tx_aggr_lvl) {
 		if ((nix_get_tx_link(rvu, map_func) !=
 		     nix_get_tx_link(rvu, pcifunc)) &&
-		     (rvu_get_pf(rvu->pdev, map_func) !=
-				rvu_get_pf(rvu->pdev, pcifunc)))
+		    (rvu_get_pf(rvu->pdev, map_func) !=
+		     rvu_get_pf(rvu->pdev, pcifunc))) {
+			dev_err_ratelimited(rvu->dev,
+					    "NIX TX schq %u lvl %d: pcifunc %x PF/link mismatch\n",
+					    schq, lvl, pcifunc);
 			return false;
-		else
+		} else {
 			return true;
+		}
 	}
 
-	if (map_func != pcifunc)
+	if (map_func != pcifunc) {
+		dev_err_ratelimited(rvu->dev,
+				    "NIX TX schq %u lvl %d: not owned by pcifunc %x\n",
+				    schq, lvl, pcifunc);
 		return false;
+	}
 
 	return true;
 }
@@ -1233,28 +1241,19 @@ static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw,
 			ena = (req->rq.ena & req->rq_mask.ena) |
 				(test_bit(req->qidx, pfvf->rq_bmap) &
 				~req->rq_mask.ena);
-			if (ena)
-				__set_bit(req->qidx, pfvf->rq_bmap);
-			else
-				__clear_bit(req->qidx, pfvf->rq_bmap);
+			__assign_bit(req->qidx, pfvf->rq_bmap, ena);
 		}
 		if (req->ctype == NIX_AQ_CTYPE_SQ) {
 			ena = (req->rq.ena & req->sq_mask.ena) |
 				(test_bit(req->qidx, pfvf->sq_bmap) &
 				~req->sq_mask.ena);
-			if (ena)
-				__set_bit(req->qidx, pfvf->sq_bmap);
-			else
-				__clear_bit(req->qidx, pfvf->sq_bmap);
+			__assign_bit(req->qidx, pfvf->sq_bmap, ena);
 		}
 		if (req->ctype == NIX_AQ_CTYPE_CQ) {
 			ena = (req->rq.ena & req->cq_mask.ena) |
 				(test_bit(req->qidx, pfvf->cq_bmap) &
 				~req->cq_mask.ena);
-			if (ena)
-				__set_bit(req->qidx, pfvf->cq_bmap);
-			else
-				__clear_bit(req->qidx, pfvf->cq_bmap);
+			__assign_bit(req->qidx, pfvf->cq_bmap, ena);
 		}
 	}
 
@@ -4974,7 +4973,7 @@ static void rvu_nix_setup_capabilities(struct rvu *rvu, int blkaddr)
 
 	/* On OcteonTx2 DWRR quantum is directly configured into each of
 	 * the transmit scheduler queues. And PF/VF drivers were free to
-	 * config any value upto 2^24.
+	 * config any value up to 2^24.
 	 * On CN10K, HW is modified, the quantum configuration at scheduler
 	 * queues is in terms of weight. And SW needs to setup a base DWRR MTU
 	 * at NIX_AF_DWRR_RPM_MTU / NIX_AF_DWRR_SDP_MTU. HW will do
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npa.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npa.c
index e2a33e46b48ad6..334dc3e888f485 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npa.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npa.c
@@ -169,10 +169,7 @@ int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req,
 			ena = (req->aura.ena & req->aura_mask.ena) |
 				(test_bit(req->aura_id, pfvf->aura_bmap) &
 				~req->aura_mask.ena);
-			if (ena)
-				__set_bit(req->aura_id, pfvf->aura_bmap);
-			else
-				__clear_bit(req->aura_id, pfvf->aura_bmap);
+			__assign_bit(req->aura_id, pfvf->aura_bmap, ena);
 		}
 	}
 
@@ -184,10 +181,7 @@ int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req,
 			ena = (req->pool.ena & req->pool_mask.ena) |
 				(test_bit(req->aura_id, pfvf->pool_bmap) &
 				~req->pool_mask.ena);
-			if (ena)
-				__set_bit(req->aura_id, pfvf->pool_bmap);
-			else
-				__clear_bit(req->aura_id, pfvf->pool_bmap);
+			__assign_bit(req->aura_id, pfvf->pool_bmap, ena);
 		}
 	}
 	spin_unlock(&aq->lock);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
index c34f8d86cc8a59..42a601976db36b 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c
@@ -1959,8 +1959,9 @@ static int npc_apply_custom_kpu_from_fs(struct rvu *rvu,
 	fw = rvu->kpu_fwdata;
 
 	/* Binary blob contains ikpu actions entries at start of data[0] */
-	profile->ikpu2 = devm_kcalloc(rvu->dev, 1,
-				      sizeof(ikpu_action_entries),
+	profile->ikpu2 = devm_kcalloc(rvu->dev,
+				      ARRAY_SIZE(ikpu_action_entries),
+				      sizeof(*profile->ikpu2),
 				      GFP_KERNEL);
 	if (!profile->ikpu2)
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
index d422bdd5e8f800..ec004ea5154cbb 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c
@@ -698,7 +698,6 @@ static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf)
 	proto_flags = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) |
 		       BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) |
 		       BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP) |
-		       BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP) |
 		       BIT_ULL(NPC_TYPE_ICMP) | BIT_ULL(NPC_CODE_ICMP) |
 		       BIT_ULL(NPC_TCP_FLAGS);
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
index 3e1bf22cba69c8..f98ef5431c906a 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
@@ -301,7 +301,7 @@ static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp,
 	 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket.
 	 * Here policer timeunit is 2 usecs and rate is in bits per sec.
 	 * Since floating point cannot be used below algorithm uses 1000000
-	 * scale factor to support rates upto 100Gbps.
+	 * scale factor to support rates up to 100Gbps.
 	 */
 	tmp = rate * 32 * 2;
 	if (tmp < 256000000) {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
index 8e41431c7f9c70..e92b77789f088c 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
@@ -242,15 +242,6 @@ int cn20k_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
 
 #define RQ_BP_LVL_AURA   (255 - ((85 * 256) / 100)) /* BP when 85% is full */
 
-static u8 cn20k_aura_bpid_idx(struct otx2_nic *pfvf, int aura_id)
-{
-#ifdef CONFIG_DCB
-	return pfvf->queue_to_pfc_map[aura_id];
-#else
-	return 0;
-#endif
-}
-
 static int cn20k_tc_get_entry_index(struct otx2_flow_config *flow_cfg,
 				    struct otx2_tc_flow *node)
 {
@@ -580,7 +571,7 @@ static int cn20k_aura_aq_init(struct otx2_nic *pfvf, int aura_id,
 		if (pfvf->nix_blkaddr == BLKADDR_NIX1)
 			aq->aura.bp_ena = 1;
 
-		bpid_idx = cn20k_aura_bpid_idx(pfvf, aura_id);
+		bpid_idx = otx2_get_bpid_idx(pfvf, aura_id);
 		aq->aura.bpid = pfvf->bpid[bpid_idx];
 
 		/* Set backpressure level for RQ's Aura */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 175992188c18a4..d90cc2fd26faa3 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -732,7 +732,6 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for
 
 	return otx2_sync_mbox_msg(&pfvf->mbox);
 }
-EXPORT_SYMBOL(otx2_txschq_config);
 
 int otx2_smq_flush(struct otx2_nic *pfvf, int smq)
 {
@@ -756,7 +755,6 @@ int otx2_smq_flush(struct otx2_nic *pfvf, int smq)
 	mutex_unlock(&pfvf->mbox.lock);
 	return rc;
 }
-EXPORT_SYMBOL(otx2_smq_flush);
 
 int otx2_txsch_alloc(struct otx2_nic *pfvf)
 {
@@ -828,7 +826,6 @@ void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq)
 
 	mutex_unlock(&pfvf->mbox.lock);
 }
-EXPORT_SYMBOL(otx2_txschq_free_one);
 
 void otx2_txschq_stop(struct otx2_nic *pfvf)
 {
@@ -1055,6 +1052,7 @@ int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
 	struct nix_aq_enq_req *aq;
 	struct otx2_cq_queue *cq;
 	struct otx2_pool *pool;
+	u8 bpid_idx;
 
 	cq = &qset->cq[qidx];
 	cq->cq_idx = qidx;
@@ -1132,11 +1130,8 @@ int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
 		if (!is_otx2_lbkvf(pfvf->pdev)) {
 			/* Enable receive CQ backpressure */
 			aq->cq.bp_ena = 1;
-#ifdef CONFIG_DCB
-			aq->cq.bpid = pfvf->bpid[pfvf->queue_to_pfc_map[qidx]];
-#else
-			aq->cq.bpid = pfvf->bpid[0];
-#endif
+			bpid_idx = otx2_get_bpid_idx(pfvf, qidx);
+			aq->cq.bpid = pfvf->bpid[bpid_idx];
 
 			/* Set backpressure level is same as cq pass level */
 			aq->cq.bp = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt);
@@ -1378,6 +1373,7 @@ int otx2_aura_aq_init(struct otx2_nic *pfvf, int aura_id,
 {
 	struct npa_aq_enq_req *aq;
 	struct otx2_pool *pool;
+	u8 bpid_idx;
 	int err;
 
 	pool = &pfvf->qset.pool[pool_id];
@@ -1433,11 +1429,8 @@ int otx2_aura_aq_init(struct otx2_nic *pfvf, int aura_id,
 		 */
 		if (pfvf->nix_blkaddr == BLKADDR_NIX1)
 			aq->aura.bp_ena = 1;
-#ifdef CONFIG_DCB
-		aq->aura.nix0_bpid = pfvf->bpid[pfvf->queue_to_pfc_map[aura_id]];
-#else
-		aq->aura.nix0_bpid = pfvf->bpid[0];
-#endif
+		bpid_idx = otx2_get_bpid_idx(pfvf, aura_id);
+		aq->aura.nix0_bpid = pfvf->bpid[bpid_idx];
 
 		/* Set backpressure level for RQ's Aura */
 		aq->aura.bp = RQ_BP_LVL_AURA;
@@ -1838,7 +1831,6 @@ int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable)
 
 	return otx2_sync_mbox_msg(&pfvf->mbox);
 }
-EXPORT_SYMBOL(otx2_nix_config_bp);
 
 int otx2_nix_cpt_config_bp(struct otx2_nic *pfvf, bool enable)
 {
@@ -1863,7 +1855,6 @@ int otx2_nix_cpt_config_bp(struct otx2_nic *pfvf, bool enable)
 
 	return otx2_sync_mbox_msg(&pfvf->mbox);
 }
-EXPORT_SYMBOL(otx2_nix_cpt_config_bp);
 
 /* Mbox message handlers */
 void mbox_handler_cgx_stats(struct otx2_nic *pfvf,
@@ -2110,8 +2101,7 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf,		\
 {									\
 	/* Nothing to do here */					\
 	return 0;							\
-}									\
-EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name);
+}
 MBOX_UP_CGX_MESSAGES
 MBOX_UP_MCS_MESSAGES
 #undef M
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index eecee612b7b2c9..fda6c4487c4fbe 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -42,7 +42,6 @@
 /* PCI device IDs */
 #define PCI_DEVID_OCTEONTX2_RVU_PF              0xA063
 #define PCI_DEVID_OCTEONTX2_RVU_VF		0xA064
-#define PCI_DEVID_OCTEONTX2_RVU_AFVF		0xA0F8
 
 #define PCI_SUBSYS_DEVID_96XX_RVU_PFVF		0xB200
 #define PCI_SUBSYS_DEVID_CN10K_A_RVU_PFVF	0xB900
@@ -1029,6 +1028,15 @@ static inline int otx2_tc_flower_rule_cnt(struct otx2_nic *pfvf)
 	return pfvf->flow_cfg->nr_flows;
 }
 
+static inline u8 otx2_get_bpid_idx(struct otx2_nic *pfvf, int qidx)
+{
+#ifdef CONFIG_DCB
+	if (pfvf->queue_to_pfc_map)
+		return pfvf->queue_to_pfc_map[qidx];
+#endif
+	return 0;
+}
+
 /* MSI-X APIs */
 void otx2_free_cints(struct otx2_nic *pfvf, int n);
 void otx2_set_cints_affinity(struct otx2_nic *pfvf);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
index f110dfa423605d..e3d54009180daa 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
@@ -54,7 +54,6 @@ int otx2_pfc_txschq_config(struct otx2_nic *pfvf)
 
 	return 0;
 }
-EXPORT_SYMBOL(otx2_pfc_txschq_config);
 
 static int otx2_pfc_txschq_alloc_one(struct otx2_nic *pfvf, u8 prio)
 {
@@ -67,7 +66,7 @@ static int otx2_pfc_txschq_alloc_one(struct otx2_nic *pfvf, u8 prio)
 	if (!req)
 		return -ENOMEM;
 
-	/* Request one schq per level upto max level as configured
+	/* Request one schq per level up to max level as configured
 	 * link config level. These rest of the scheduler can be
 	 * same as hw.txschq_list.
 	 */
@@ -123,7 +122,6 @@ int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf)
 
 	return 0;
 }
-EXPORT_SYMBOL(otx2_pfc_txschq_alloc);
 
 static int otx2_pfc_txschq_stop_one(struct otx2_nic *pfvf, u8 prio)
 {
@@ -262,7 +260,6 @@ int otx2_pfc_txschq_update(struct otx2_nic *pfvf)
 
 	return 0;
 }
-EXPORT_SYMBOL(otx2_pfc_txschq_update);
 
 int otx2_pfc_txschq_stop(struct otx2_nic *pfvf)
 {
@@ -285,7 +282,6 @@ int otx2_pfc_txschq_stop(struct otx2_nic *pfvf)
 
 	return 0;
 }
-EXPORT_SYMBOL(otx2_pfc_txschq_stop);
 
 int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf)
 {
@@ -395,7 +391,6 @@ void otx2_update_bpid_in_rqctx(struct otx2_nic *pfvf, int vlan_prio, int qidx,
 			 "Updating BPIDs in CQ and Aura contexts of RQ%d failed with err %d\n",
 			 qidx, err);
 }
-EXPORT_SYMBOL(otx2_update_bpid_in_rqctx);
 
 static int otx2_dcbnl_ieee_getpfc(struct net_device *dev, struct ieee_pfc *pfc)
 {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
index 9bee1b91eeaa67..a05dee0085a39d 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
@@ -1209,6 +1209,7 @@ static int otx2_get_link_ksettings(struct net_device *netdev,
 {
 	struct otx2_nic *pfvf = netdev_priv(netdev);
 	struct cgx_fw_data *rsp = NULL;
+	u8 port;
 
 	cmd->base.duplex  = pfvf->linfo.full_duplex;
 	cmd->base.speed   = pfvf->linfo.speed;
@@ -1231,6 +1232,23 @@ static int otx2_get_link_ksettings(struct net_device *netdev,
 				OTX2_MODE_SUPPORTED, cmd);
 	otx2_get_fec_info(rsp->fwdata.supported_fec,
 			  OTX2_MODE_SUPPORTED, cmd);
+
+	port = FIELD_GET(GENMASK(7, 0), rsp->fwdata.port);
+	switch (port) {
+	case PORT_TP:
+	case PORT_AUI:
+	case PORT_MII:
+	case PORT_FIBRE:
+	case PORT_BNC:
+	case PORT_DA:
+	case PORT_NONE:
+		cmd->base.port = port;
+		break;
+	default:
+		cmd->base.port = PORT_OTHER;
+		break;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
index 99d78fc5a2c4a9..6d095023d17b98 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c
@@ -281,7 +281,6 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count)
 			    count, allocated);
 	return allocated;
 }
-EXPORT_SYMBOL(otx2_alloc_mcam_entries);
 
 int otx2_mcam_entry_init(struct otx2_nic *pfvf)
 {
@@ -420,7 +419,6 @@ int otx2_mcam_entry_init(struct otx2_nic *pfvf)
 	refcount_set(&flow_cfg->mark_flows, 1);
 	return 0;
 }
-EXPORT_SYMBOL(otx2_mcam_entry_init);
 
 /* TODO : revisit on size */
 #define OTX2_DMAC_FLTR_BITMAP_SZ (4 * 2048 + 32)
@@ -654,7 +652,6 @@ int otx2_get_maxflows(struct otx2_flow_config *flow_cfg)
 	else
 		return flow_cfg->max_flows;
 }
-EXPORT_SYMBOL(otx2_get_maxflows);
 
 int otx2_get_flow(struct otx2_nic *pfvf, struct ethtool_rxnfc *nfc,
 		  u32 location)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index c995f29008590a..7dc0ce669d0b76 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -959,6 +959,7 @@ int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf,
 	otx2_handle_link_event(pf);
 	return 0;
 }
+EXPORT_SYMBOL(otx2_mbox_up_handler_cgx_link_event);
 
 static int otx2_process_mbox_msg_up(struct otx2_nic *pf,
 				    struct mbox_msghdr *req)
@@ -1167,10 +1168,9 @@ int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
 	}
 	err = otx2_sync_mbox_msg(&pf->mbox);
 	if (err) {
-		dev_warn(pf->dev,
-			 "AF not responding to mailbox, deferring probe\n");
 		otx2_disable_mbox_intr(pf);
-		return -EPROBE_DEFER;
+		return dev_err_probe(pf->dev, -EPROBE_DEFER,
+			 "AF not responding to mailbox, deferring probe\n");
 	}
 
 	return 0;
@@ -1524,6 +1524,34 @@ irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
 }
 EXPORT_SYMBOL(otx2_cq_intr_handler);
 
+static irqreturn_t otx2_nixlf_err_intr_handler(int irq, void *data)
+{
+	struct otx2_nic *pf = data;
+	u64 regval;
+
+	/* Clear interrupt */
+	regval = otx2_read64(pf, NIX_LF_ERR_INT);
+	otx2_write64(pf, NIX_LF_ERR_INT, regval);
+
+	dev_err_ratelimited(pf->dev, "NIXLF Error Interrupt: 0x%llx\n", regval);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t otx2_nixlf_poison_intr_handler(int irq, void *data)
+{
+	struct otx2_nic *pf = data;
+	u64 regval;
+
+	/* Clear interrupt */
+	regval = otx2_read64(pf, NIX_LF_RAS);
+	otx2_write64(pf, NIX_LF_RAS, regval);
+
+	dev_err_ratelimited(pf->dev, "NIXLF Poison Interrupt: 0x%llx\n", regval);
+
+	return IRQ_HANDLED;
+}
+
 void otx2_disable_napi(struct otx2_nic *pf)
 {
 	struct otx2_qset *qset = &pf->qset;
@@ -2080,6 +2108,34 @@ int otx2_open(struct net_device *netdev)
 
 	otx2_set_cints_affinity(pf);
 
+	/* Register NIXLF error IRQ handler */
+	vec = pf->hw.nix_msixoff + NIX_LF_ERR_VEC;
+	irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
+	snprintf(irq_name, NAME_SIZE, "%s-nixlf-err", pf->netdev->name);
+	err = request_irq(pci_irq_vector(pf->pdev, vec),
+			  otx2_nixlf_err_intr_handler, 0, irq_name, pf);
+	if (err) {
+		dev_err(pf->dev,
+			"RVUPF%d: IRQ registration failed for NIXLF ERR vector\n",
+			rvu_get_pf(pf->pdev, pf->pcifunc));
+		goto err_free_cints;
+	}
+	otx2_write64(pf, NIX_LF_ERR_INT_ENA_W1S, NIX_LF_ERR_INT_MASK);
+
+	/* Register NIXLF POISON interrupt handler */
+	vec = pf->hw.nix_msixoff + NIX_LF_POISON_VEC;
+	irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
+	snprintf(irq_name, NAME_SIZE, "%s-nixlf-poison", pf->netdev->name);
+	err = request_irq(pci_irq_vector(pf->pdev, vec),
+			  otx2_nixlf_poison_intr_handler, 0, irq_name, pf);
+	if (err) {
+		dev_err(pf->dev,
+			"RVUPF%d: IRQ registration failed for NIXLF POISON vector\n",
+			rvu_get_pf(pf->pdev, pf->pcifunc));
+		goto err_free_errint;
+	}
+	otx2_write64(pf, NIX_LF_RAS_ENA_W1S, NIX_LF_RAS_MASK);
+
 	if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
 		otx2_enable_rxvlan(pf, true);
 
@@ -2132,6 +2188,16 @@ int otx2_open(struct net_device *netdev)
 	netif_tx_stop_all_queues(netdev);
 	netif_carrier_off(netdev);
 	pf->flags |= OTX2_FLAG_INTF_DOWN;
+	/* free NIXLF POISON irq */
+	vec = pci_irq_vector(pf->pdev,
+			     pf->hw.nix_msixoff + NIX_LF_POISON_VEC);
+	otx2_write64(pf, NIX_LF_RAS_ENA_W1C, NIX_LF_RAS_MASK);
+	free_irq(vec, pf);
+err_free_errint:
+	vec = pci_irq_vector(pf->pdev,
+			     pf->hw.nix_msixoff + NIX_LF_ERR_VEC);
+	otx2_write64(pf, NIX_LF_ERR_INT_ENA_W1C, NIX_LF_ERR_INT_MASK);
+	free_irq(vec, pf);
 err_free_cints:
 	otx2_free_cints(pf, qidx);
 	vec = pci_irq_vector(pf->pdev,
@@ -2171,6 +2237,18 @@ int otx2_stop(struct net_device *netdev)
 	/* Clear RSS enable flag */
 	pf->hw.rss_info.enable = false;
 
+	/* Cleanup NIXLF Poison IRQ */
+	vec = pci_irq_vector(pf->pdev,
+			     pf->hw.nix_msixoff + NIX_LF_POISON_VEC);
+	otx2_write64(pf, NIX_LF_RAS_ENA_W1C, NIX_LF_RAS_MASK);
+	free_irq(vec, pf);
+
+	/* Cleanup NIXLF Error IRQ */
+	vec = pci_irq_vector(pf->pdev,
+			     pf->hw.nix_msixoff + NIX_LF_ERR_VEC);
+	otx2_write64(pf, NIX_LF_ERR_INT_ENA_W1C, NIX_LF_ERR_INT_MASK);
+	free_irq(vec, pf);
+
 	/* Cleanup Queue IRQ */
 	vec = pci_irq_vector(pf->pdev,
 			     pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
@@ -2993,11 +3071,9 @@ int otx2_check_pf_usable(struct otx2_nic *nic)
 	 * otherwise this driver probe should be deferred
 	 * until AF driver comes up.
 	 */
-	if (!rev) {
-		dev_warn(nic->dev,
+	if (!rev)
+		return dev_err_probe(nic->dev, -EPROBE_DEFER,
 			 "AF is not initialized, deferring probe\n");
-		return -EPROBE_DEFER;
-	}
 	return 0;
 }
 
@@ -3006,11 +3082,15 @@ int otx2_realloc_msix_vectors(struct otx2_nic *pf)
 	struct otx2_hw *hw = &pf->hw;
 	int num_vec, err;
 
-	/* NPA interrupts are inot registered, so alloc only
-	 * upto NIX vector offset.
+	/* Skip NPA vectors. Representors only use CINT vectors, so limit
+	 * the budget to that range. For PF/VF, allocate the full NIX LF
+	 * interrupt range (QINT, CINT, GINT, ERR and POISON vectors).
 	 */
 	num_vec = hw->nix_msixoff;
-	num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
+	if (pf->flags & OTX2_FLAG_REP_MODE_ENABLED)
+		num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
+	else
+		num_vec += NIX_LF_POISON_VEC + 1;
 
 	otx2_disable_mbox_intr(pf);
 	pci_free_irq_vectors(hw->pdev);
@@ -3023,7 +3103,6 @@ int otx2_realloc_msix_vectors(struct otx2_nic *pf)
 
 	return otx2_register_mbox_intr(pf, false);
 }
-EXPORT_SYMBOL(otx2_realloc_msix_vectors);
 
 static int otx2_sriov_vfcfg_init(struct otx2_nic *pf)
 {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h
index 1cd576fd09c56f..c144fd2bbe51f7 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_reg.h
@@ -121,10 +121,12 @@
 #define	NIX_LF_ERR_INT_W1S		(NIX_LFBASE | 0x228)
 #define	NIX_LF_ERR_INT_ENA_W1C		(NIX_LFBASE | 0x230)
 #define	NIX_LF_ERR_INT_ENA_W1S		(NIX_LFBASE | 0x238)
+#define	NIX_LF_ERR_INT_MASK		GENMASK_ULL(31, 0)
 #define	NIX_LF_RAS			(NIX_LFBASE | 0x240)
 #define	NIX_LF_RAS_W1S			(NIX_LFBASE | 0x248)
 #define	NIX_LF_RAS_ENA_W1C		(NIX_LFBASE | 0x250)
 #define	NIX_LF_RAS_ENA_W1S		(NIX_LFBASE | 0x258)
+#define	NIX_LF_RAS_MASK			GENMASK_ULL(10, 0)
 #define	NIX_LF_SQ_OP_ERR_DBG		(NIX_LFBASE | 0x260)
 #define	NIX_LF_MNQ_ERR_DBG		(NIX_LFBASE | 0x270)
 #define	NIX_LF_SEND_ERR_DBG		(NIX_LFBASE | 0x280)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
index 039fd47ebf5220..dee38d5c7f65b1 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c
@@ -1675,4 +1675,3 @@ void otx2_tc_apply_ingress_police_rules(struct otx2_nic *nic)
 			otx2_tc_config_ingress_rule(nic, node);
 	}
 }
-EXPORT_SYMBOL(otx2_tc_apply_ingress_police_rules);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
index fcdf891f90b580..f7765e19d78a9e 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
@@ -297,10 +297,9 @@ static int otx2vf_register_mbox_intr(struct otx2_nic *vf, bool probe_pf)
 
 	err = otx2_sync_mbox_msg(&vf->mbox);
 	if (err) {
-		dev_warn(vf->dev,
-			 "AF not responding to mailbox, deferring probe\n");
 		otx2vf_disable_mbox_intr(vf);
-		return -EPROBE_DEFER;
+		return dev_err_probe(vf->dev, -EPROBE_DEFER,
+			 "AF not responding to mailbox, deferring probe\n");
 	}
 	return 0;
 }
@@ -554,7 +553,7 @@ static int otx2vf_realloc_msix_vectors(struct otx2_nic *vf)
 	int num_vec, err;
 
 	num_vec = hw->nix_msixoff;
-	num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
+	num_vec += NIX_LF_POISON_VEC + 1;
 
 	otx2vf_disable_mbox_intr(vf);
 	pci_free_irq_vectors(hw->pdev);
@@ -597,7 +596,7 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	pci_set_master(pdev);
 
-	qcount = num_online_cpus();
+	qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT);
 	qos_txqs = min_t(int, qcount, OTX2_QOS_MAX_LEAF_NODES);
 	netdev = alloc_etherdev_mqs(sizeof(*vf), qcount + qos_txqs, qcount);
 	if (!netdev)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos.h b/drivers/net/ethernet/marvell/octeontx2/nic/qos.h
index 221bd0438f6034..bd12174bf94ef6 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/qos.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos.h
@@ -73,6 +73,4 @@ struct otx2_qos_node {
 	u8 level;
 	bool is_static;
 };
-
-
 #endif
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router.c b/drivers/net/ethernet/marvell/prestera/prestera_router.c
index 0c4f462baa6ef4..ba45b61b09bb0f 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_router.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_router.c
@@ -683,7 +683,7 @@ __prestera_k_arb_n_offload_set(struct prestera_switch *sw,
 {
 	struct neighbour *n;
 
-	n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4,
+	n = neigh_lookup(arp_table(&init_net), &nc->key.addr.u.ipv4,
 			 nc->key.dev);
 	if (!n)
 		return;
@@ -790,7 +790,7 @@ __prestera_k_arb_nc_kern_n_fetch(struct prestera_switch *sw,
 	int err;
 
 	memset(&nc->nh_neigh_info, 0, sizeof(nc->nh_neigh_info));
-	n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, nc->key.dev);
+	n = neigh_lookup(arp_table(&init_net), &nc->key.addr.u.ipv4, nc->key.dev);
 	if (!n)
 		goto out;
 
@@ -1052,10 +1052,12 @@ static void __prestera_k_arb_hw_state_upd(struct prestera_switch *sw,
 #endif /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */
 
 	if (nc->key.addr.v == PRESTERA_IPV4) {
-		n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4,
+		struct neigh_table *tbl = arp_table(&init_net);
+
+		n = neigh_lookup(tbl, &nc->key.addr.u.ipv4,
 				 nc->key.dev);
 		if (!n)
-			n = neigh_create(&arp_tbl, &nc->key.addr.u.ipv4,
+			n = neigh_create(tbl, &nc->key.addr.u.ipv4,
 					 nc->key.dev);
 	} else {
 		n = NULL;
diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c
index bbbbbba63e15db..2370b2ddba6b65 100644
--- a/drivers/net/ethernet/marvell/pxa168_eth.c
+++ b/drivers/net/ethernet/marvell/pxa168_eth.c
@@ -514,7 +514,7 @@ static int add_del_hash_entry(struct pxa168_eth_private *pep,
 /*
  * ----------------------------------------------------------------------------
  *  Create an addressTable entry from MAC address info
- *  found in the specifed net_device struct
+ *  found in the specified net_device struct
  *
  *  Input : pointer to ethernet interface network device structure
  *  Output : N/A
@@ -806,7 +806,7 @@ static int rxq_process(struct net_device *dev, int budget)
 		stats->rx_bytes += rx_desc->byte_cnt;
 		/*
 		 * In case received a packet without first / last bits on OR
-		 * the error summary bit is on, the packets needs to be droped.
+		 * the error summary bit is on, the packets needs to be dropped.
 		 */
 		if (((cmd_sts & (RX_FIRST_DESC | RX_LAST_DESC)) !=
 		     (RX_FIRST_DESC | RX_LAST_DESC))
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index fd7a49ae88d015..ad0fdc4133fb68 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -4509,6 +4509,10 @@ static int mtk_unreg_dev(struct mtk_eth *eth)
 		mac = netdev_priv(eth->netdev[i]);
 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA))
 			unregister_netdevice_notifier(&mac->device_notifier);
+
+		if (eth->netdev[i]->reg_state != NETREG_REGISTERED)
+			continue;
+
 		unregister_netdev(eth->netdev[i]);
 	}
 
@@ -5337,6 +5341,22 @@ static int mtk_probe(struct platform_device *pdev)
 		}
 	}
 
+	/* we run 2 devices on the same DMA ring so we need a dummy device
+	 * for NAPI to work. Allocate it before registering the netdevs so a
+	 * concurrent ndo_open (e.g. netifd bringing the first netdev up the
+	 * instant it is registered) never observes eth->dummy_dev == NULL in
+	 * mtk_dma_init() -> mtk_rx_alloc() -> __xdp_rxq_info_reg() (net/core/xdp.c
+	 * "Missing net_device from driver") and fails the first open.
+	 */
+	eth->dummy_dev = alloc_netdev_dummy(0);
+	if (!eth->dummy_dev) {
+		err = -ENOMEM;
+		dev_err(eth->dev, "failed to allocated dummy device\n");
+		goto err_unreg_netdev;
+	}
+	netif_napi_add(eth->dummy_dev, &eth->tx_napi, mtk_napi_tx);
+	netif_napi_add(eth->dummy_dev, &eth->rx_napi, mtk_napi_rx);
+
 	for (i = 0; i < MTK_MAX_DEVS; i++) {
 		if (!eth->netdev[i])
 			continue;
@@ -5344,25 +5364,13 @@ static int mtk_probe(struct platform_device *pdev)
 		err = register_netdev(eth->netdev[i]);
 		if (err) {
 			dev_err(eth->dev, "error bringing up device\n");
-			goto err_deinit_ppe;
+			goto err_unreg_netdev;
 		} else
 			netif_info(eth, probe, eth->netdev[i],
 				   "mediatek frame engine at 0x%08lx, irq %d\n",
 				   eth->netdev[i]->base_addr, eth->irq[MTK_FE_IRQ_SHARED]);
 	}
 
-	/* we run 2 devices on the same DMA ring so we need a dummy device
-	 * for NAPI to work
-	 */
-	eth->dummy_dev = alloc_netdev_dummy(0);
-	if (!eth->dummy_dev) {
-		err = -ENOMEM;
-		dev_err(eth->dev, "failed to allocated dummy device\n");
-		goto err_unreg_netdev;
-	}
-	netif_napi_add(eth->dummy_dev, &eth->tx_napi, mtk_napi_tx);
-	netif_napi_add(eth->dummy_dev, &eth->rx_napi, mtk_napi_rx);
-
 	platform_set_drvdata(pdev, eth);
 	schedule_delayed_work(&eth->reset.monitor_work,
 			      MTK_DMA_MONITOR_TIMEOUT);
@@ -5372,6 +5380,16 @@ static int mtk_probe(struct platform_device *pdev)
 err_unreg_netdev:
 	mtk_unreg_dev(eth);
 err_deinit_ppe:
+	/* A netdev opened while probe was still registering can have queued
+	 * the frame-engine reset worker, which takes rtnl and drives the
+	 * hardware; cancel it before anything below is torn down.
+	 */
+	cancel_work_sync(&eth->pending_work);
+	if (eth->dummy_dev) {
+		netif_napi_del(&eth->tx_napi);
+		netif_napi_del(&eth->rx_napi);
+		free_netdev(eth->dummy_dev);
+	}
 	mtk_ppe_deinit(eth);
 	mtk_mdio_cleanup(eth);
 err_free_dev:
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index c851daa5da9f5c..036135df23bf1a 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -3863,12 +3863,8 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data,
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (err) {
-		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
-			goto err_release_regions;
-		}
+		dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
+		goto err_release_regions;
 	}
 
 	/* Allow large DMA segments, up to the firmware limit of 1 GB */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index 723f66a6bd634e..17084d76209e8c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -259,7 +259,7 @@ static bool mlx5e_ptp_poll_ts_cq(struct mlx5e_cq *cq, int napi_budget)
 	mlx5_cqwq_update_db_record(cqwq);
 
 	/* ensure cq space is freed before enabling more cqes */
-	wmb();
+	dma_wmb();
 
 	while (metadata_buff_sz > 0)
 		mlx5e_ptp_metadata_fifo_push(&ptpsq->metadata_freelist,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
index baac38bece14a7..4b7b0a0fc2b2ff 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c
@@ -85,9 +85,16 @@ mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(struct net_device *dev, struct m
 	struct net_device *lower_dev;
 	struct list_head *iter;
 
-	if (netif_is_lag_master(dev) || mlx5e_eswitch_rep(dev))
-		return mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, esw, vport_num,
-								 esw_owner_vhca_id);
+	if (netif_is_lag_master(dev) || mlx5e_eswitch_rep(dev)) {
+		struct net_device *rep;
+
+		rep = mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, esw, vport_num,
+								esw_owner_vhca_id);
+		if (rep && !mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id,
+							esw->br_offloads))
+			return NULL;
+		return rep;
+	}
 
 	netdev_for_each_lower_dev(dev, lower_dev, iter) {
 		struct net_device *rep;
@@ -104,6 +111,28 @@ mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(struct net_device *dev, struct m
 	return NULL;
 }
 
+static bool mlx5_esw_bridge_rep_port_lookup(struct net_device *dev,
+					    struct mlx5_esw_bridge_offloads *br_offloads,
+					    u16 *vport_num, u16 *esw_owner_vhca_id)
+{
+	if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, vport_num,
+						       esw_owner_vhca_id))
+		return false;
+
+	return mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, br_offloads);
+}
+
+static bool mlx5_esw_bridge_lower_rep_port_lookup(struct net_device *dev,
+						  struct mlx5_esw_bridge_offloads *br_offloads,
+						  u16 *vport_num, u16 *esw_owner_vhca_id)
+{
+	if (!mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(dev, br_offloads->esw, vport_num,
+							     esw_owner_vhca_id))
+		return false;
+
+	return mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, br_offloads);
+}
+
 static bool mlx5_esw_bridge_is_local(struct net_device *dev, struct net_device *rep,
 				     struct mlx5_eswitch *esw)
 {
@@ -218,8 +247,7 @@ mlx5_esw_bridge_port_obj_add(struct net_device *dev,
 	u16 vport_num, esw_owner_vhca_id;
 	int err;
 
-	if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num,
-						       &esw_owner_vhca_id))
+	if (!mlx5_esw_bridge_rep_port_lookup(dev, br_offloads, &vport_num, &esw_owner_vhca_id))
 		return 0;
 
 	port_obj_info->handled = true;
@@ -251,8 +279,7 @@ mlx5_esw_bridge_port_obj_del(struct net_device *dev,
 	const struct switchdev_obj_port_mdb *mdb;
 	u16 vport_num, esw_owner_vhca_id;
 
-	if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num,
-						       &esw_owner_vhca_id))
+	if (!mlx5_esw_bridge_rep_port_lookup(dev, br_offloads, &vport_num, &esw_owner_vhca_id))
 		return 0;
 
 	port_obj_info->handled = true;
@@ -283,8 +310,8 @@ mlx5_esw_bridge_port_obj_attr_set(struct net_device *dev,
 	u16 vport_num, esw_owner_vhca_id;
 	int err = 0;
 
-	if (!mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num,
-							     &esw_owner_vhca_id))
+	if (!mlx5_esw_bridge_lower_rep_port_lookup(dev, br_offloads, &vport_num,
+						   &esw_owner_vhca_id))
 		return 0;
 
 	port_attr_info->handled = true;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c
index 648f4521c09644..6a8d8aac7d16fe 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c
@@ -17,20 +17,27 @@
 #include "fs_core.h"
 #include "diag/en_rep_tracepoint.h"
 
-static unsigned long mlx5e_rep_ipv6_interval(void)
+static unsigned long mlx5e_rep_ipv6_interval(struct net *net)
 {
 	if (IS_ENABLED(CONFIG_IPV6) && ipv6_mod_enabled())
-		return NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME);
+		return NEIGH_VAR(&nd_table(net)->parms, DELAY_PROBE_TIME);
 
 	return ~0UL;
 }
 
 static void mlx5e_rep_neigh_update_init_interval(struct mlx5e_rep_priv *rpriv)
 {
-	unsigned long ipv4_interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
-	unsigned long ipv6_interval = mlx5e_rep_ipv6_interval();
 	struct net_device *netdev = rpriv->netdev;
-	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct net *net = dev_net(netdev);
+	unsigned long ipv4_interval;
+	unsigned long ipv6_interval;
+	struct neigh_table *tbl;
+	struct mlx5e_priv *priv;
+
+	priv = netdev_priv(netdev);
+	tbl = arp_table(net);
+	ipv4_interval = NEIGH_VAR(&tbl->parms, DELAY_PROBE_TIME);
+	ipv6_interval = mlx5e_rep_ipv6_interval(net);
 
 	rpriv->neigh_update.min_interval = min_t(unsigned long, ipv6_interval, ipv4_interval);
 	mlx5_fc_update_sampling_interval(priv->mdev, rpriv->neigh_update.min_interval);
@@ -217,12 +224,6 @@ static int mlx5e_rep_netevent_event(struct notifier_block *nb,
 	switch (event) {
 	case NETEVENT_NEIGH_UPDATE:
 		n = ptr;
-#if IS_ENABLED(CONFIG_IPV6)
-		if (n->tbl != &nd_tbl && n->tbl != &arp_tbl)
-#else
-		if (n->tbl != &arp_tbl)
-#endif
-			return NOTIFY_DONE;
 
 		update_work = mlx5e_alloc_neigh_update_work(priv, n);
 		if (!update_work)
@@ -238,11 +239,7 @@ static int mlx5e_rep_netevent_event(struct notifier_block *nb,
 		 * changes in the default table, we only care about changes
 		 * done per device delay prob time parameter.
 		 */
-#if IS_ENABLED(CONFIG_IPV6)
-		if (!p->dev || (p->tbl != &nd_tbl && p->tbl != &arp_tbl))
-#else
-		if (!p->dev || p->tbl != &arp_tbl)
-#endif
+		if (!p->dev)
 			return NOTIFY_DONE;
 
 		rcu_read_lock();
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
index 6c87a1c7db0959..c1841ce74d9ac8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
@@ -1082,6 +1082,9 @@ mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv,
 
 	spin_unlock_bh(&ct_priv->ht_lock);
 
+	if (rev_entry)
+		mlx5_tc_ct_entry_put(rev_entry);
+
 create_counter:
 
 	shared_counter = mlx5_tc_ct_counter_create(ct_priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
index 7bfe7cdc577081..1ceebab4ce086d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h
@@ -97,18 +97,14 @@ struct mlx5e_tc_flow {
 	struct encap_flow_item encaps[MLX5_MAX_FLOW_FWD_VPORTS];
 	struct mlx5e_hairpin_entry *hpe; /* attached hairpin instance */
 	struct list_head hairpin; /* flows sharing the same hairpin */
-	struct list_head peer[MLX5_MAX_PORTS];    /* flows with peer flow */
-	DECLARE_BITMAP(peer_used, MLX5_MAX_PORTS); /* tracks populated peer
-						    * slots
-						    */
+	struct list_head peer; /* dup: node in origin esw's peer_flows */
 	struct list_head unready; /* flows not ready to be offloaded (e.g
 				   * due to missing route)
 				   */
-	struct list_head peer_flows; /* flows on peer */
-	int peer_index; /* peer-flow index pinned at add time, used at del
-			 * time so removal is independent of LAG state
-			 * changes between add and del.
-			 */
+
+	/* origin: its dups; dup: node on that list */
+	struct list_head peer_flows;
+	struct mlx5e_tc_flow *peer_orig; /* dup: back-ref to origin flow */
 	struct net_device *orig_dev; /* netdev adding flow first */
 	int tmp_entry_index;
 	struct list_head tmp_list; /* temporary flow list used by neigh update */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
index 8b827201935eb8..67c12ca19d59df 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c
@@ -393,20 +393,10 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
 	struct mlx5e_encap_entry *e = NULL;
 	struct mlx5e_tc_flow *flow;
 	struct mlx5_fc *counter;
-	struct neigh_table *tbl;
 	bool neigh_used = false;
 	struct neighbour *n;
 	u64 lastuse;
 
-	if (m_neigh->family == AF_INET)
-		tbl = &arp_tbl;
-#if IS_ENABLED(CONFIG_IPV6)
-	else if (m_neigh->family == AF_INET6)
-		tbl = &nd_tbl;
-#endif
-	else
-		return;
-
 	/* mlx5e_get_next_valid_encap() releases previous encap before returning
 	 * next one.
 	 */
@@ -447,12 +437,23 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
 	trace_mlx5e_tc_update_neigh_used_value(nhe, neigh_used);
 
 	if (neigh_used) {
+		struct net_device *dev = READ_ONCE(nhe->neigh_dev);
+		struct net *net = dev_net(dev);
+		struct neigh_table *tbl;
+
 		nhe->reported_lastuse = jiffies;
 
+#if IS_ENABLED(CONFIG_IPV6)
+		if (m_neigh->family != AF_INET)
+			tbl = nd_table(net);
+		else
+#endif
+			tbl = arp_table(net);
+
 		/* find the relevant neigh according to the cached device and
 		 * dst ip pair
 		 */
-		n = neigh_lookup(tbl, &m_neigh->dst_ip, READ_ONCE(nhe->neigh_dev));
+		n = neigh_lookup(tbl, &m_neigh->dst_ip, dev);
 		if (!n)
 			return;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
index 7a18a469961db8..467fbe43b89e9b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_vxlan.c
@@ -241,9 +241,16 @@ static bool mlx5e_tc_tun_encap_info_equal_vxlan(struct mlx5e_encap_key *a,
 static int mlx5e_tc_tun_get_remote_ifindex(struct net_device *mirred_dev)
 {
 	const struct vxlan_dev *vxlan = netdev_priv(mirred_dev);
-	const struct vxlan_rdst *dst = &vxlan->default_dst;
+	const struct vxlan_config *cfg;
+	int ifindex = 0;
 
-	return dst->remote_ifindex;
+	rcu_read_lock();
+	cfg = rcu_dereference(vxlan->cfg);
+	if (cfg)
+		ifindex = cfg->remote_ifindex;
+	rcu_read_unlock();
+
+	return ifindex;
 }
 
 struct mlx5e_tc_tunnel vxlan_tunnel = {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 77ea51bfbaae58..ee0986bd5c18dd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -817,7 +817,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
 	mlx5_cqwq_update_db_record(&cq->wq);
 
 	/* ensure cq space is freed before enabling more cqes */
-	wmb();
+	dma_wmb();
 
 	sq->cc = sqcc;
 	return (i == MLX5E_TX_CQ_POLL_BUDGET);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
index db260e3d1412fd..37a8ddee3ea1ea 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -262,6 +262,7 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry,
 	struct net_device *netdev = sa_entry->dev;
 	struct xfrm_state *x = sa_entry->x;
 	struct dst_entry *rt_dst_entry;
+	struct neigh_table *tbl;
 	struct flowi4 fl4 = {};
 	struct flowi6 fl6 = {};
 	struct neighbour *n;
@@ -364,9 +365,10 @@ static void mlx5e_ipsec_init_macs(struct mlx5e_ipsec_sa_entry *sa_entry,
 	return;
 
 neigh:
-	n = neigh_lookup(&arp_tbl, pkey, netdev);
+	tbl = arp_table(dev_net(netdev));
+	n = neigh_lookup(tbl, pkey, netdev);
 	if (!n) {
-		n = neigh_create(&arp_tbl, pkey, netdev);
+		n = neigh_create(tbl, pkey, netdev);
 		if (IS_ERR(n))
 			return;
 		neigh_event_send(n, NULL);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
index 329608c59313bc..8ffa8068e90ac9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
@@ -1564,14 +1564,14 @@ static void setup_fte_addr6(struct mlx5_flow_spec *spec,
 		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
 				    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16);
 		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
-				    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), dmask, 16);
+				    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), smask, 16);
 	}
 
 	if (!addr6_all_zero(daddr)) {
 		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
 				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16);
 		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
-				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), smask, 16);
+				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), dmask, 16);
 	}
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
index daff53ba7d09f9..38a3415acf7a38 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
@@ -1724,6 +1724,8 @@ void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv)
 	mlx5_core_dbg(priv->mdev, "mlx5e: MACsec acceleration enabled\n");
 	netdev->macsec_ops = &macsec_offload_ops;
 	netdev->features |= NETIF_F_HW_MACSEC;
+	netdev->hw_features |= NETIF_F_HW_MACSEC;
+	netdev->vlan_features |= NETIF_F_HW_MACSEC;
 	netif_keep_dst(netdev);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 3ed59ced0407aa..261c466a4d3621 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -2320,10 +2320,8 @@ static int set_pflag_rx_no_csum_complete(struct net_device *netdev, bool enable)
 
 	for (i = 0; i < channels->num; i++) {
 		c = channels->c[i];
-		if (enable)
-			__set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state);
-		else
-			__clear_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state);
+		__assign_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state,
+			     enable);
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index fc110a7d16e8da..f90b5a34e09725 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -5430,8 +5430,7 @@ static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
 	u8 addr[ETH_ALEN];
 
 	mlx5_query_mac_address(priv->mdev, addr);
-	if (is_zero_ether_addr(addr) &&
-	    !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
+	if (is_zero_ether_addr(addr)) {
 		eth_hw_addr_random(netdev);
 		mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
 		return;
@@ -5851,7 +5850,6 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	netdev->vlan_features    |= NETIF_F_SG;
 	netdev->vlan_features    |= NETIF_F_HW_CSUM;
-	netdev->vlan_features    |= NETIF_F_HW_MACSEC;
 	netdev->vlan_features    |= NETIF_F_GRO;
 	netdev->vlan_features    |= NETIF_F_TSO;
 	netdev->vlan_features    |= NETIF_F_TSO6;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 7bd0606a525385..e3f915beebe1b4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -2509,7 +2509,7 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
 	mlx5_cqwq_update_db_record(cqwq);
 
 	/* ensure cq space is freed before enabling more cqes */
-	wmb();
+	dma_wmb();
 
 	return work_done;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index b290beb4369ad5..c99e824b4c6dc1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -2116,43 +2116,35 @@ void mlx5e_put_flow_list(struct mlx5e_priv *priv, struct list_head *flow_list)
 		mlx5e_flow_put(priv, flow);
 }
 
-static void mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow *flow,
-				       int peer_index)
+static void mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow *peer_flow)
 {
-	struct mlx5_eswitch *esw = flow->priv->mdev->priv.eswitch;
-	struct mlx5e_tc_flow *peer_flow;
-	struct mlx5e_tc_flow *tmp;
-
-	if (!flow_flag_test(flow, ESWITCH) ||
-	    !flow_flag_test(flow, DUP))
-		return;
+	struct mlx5e_tc_flow *flow = peer_flow->peer_orig;
+	struct mlx5_eswitch *esw;
 
+	esw = flow->priv->mdev->priv.eswitch;
 	mutex_lock(&esw->offloads.peer_mutex);
-	list_del(&flow->peer[peer_index]);
-	clear_bit(peer_index, flow->peer_used);
+	list_del(&peer_flow->peer);
 	mutex_unlock(&esw->offloads.peer_mutex);
 
-	list_for_each_entry_safe(peer_flow, tmp, &flow->peer_flows, peer_flows) {
-		if (peer_index != peer_flow->peer_index)
-			continue;
-
-		list_del(&peer_flow->peer_flows);
-		if (refcount_dec_and_test(&peer_flow->refcnt)) {
-			mlx5e_tc_del_fdb_flow(peer_flow->priv, peer_flow);
-			kfree(peer_flow);
-		}
-	}
-
+	list_del(&peer_flow->peer_flows);
 	if (list_empty(&flow->peer_flows))
 		flow_flag_clear(flow, DUP);
+
+	if (refcount_dec_and_test(&peer_flow->refcnt)) {
+		mlx5e_tc_del_fdb_flow(peer_flow->priv, peer_flow);
+		kfree(peer_flow);
+	}
 }
 
 static void mlx5e_tc_del_fdb_peers_flow(struct mlx5e_tc_flow *flow)
 {
-	int i;
+	struct mlx5e_tc_flow *peer_flow, *tmp;
+
+	if (!flow_flag_test(flow, ESWITCH) || !flow_flag_test(flow, DUP))
+		return;
 
-	for_each_set_bit(i, flow->peer_used, MLX5_MAX_PORTS)
-		mlx5e_tc_del_fdb_peer_flow(flow, i);
+	list_for_each_entry_safe(peer_flow, tmp, &flow->peer_flows, peer_flows)
+		mlx5e_tc_del_fdb_peer_flow(peer_flow);
 }
 
 static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
@@ -4604,12 +4596,13 @@ static int mlx5e_tc_add_fdb_peer_flow(struct flow_cls_offload *f,
 				      unsigned long flow_flags,
 				      struct mlx5_eswitch *peer_esw)
 {
+	u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
 	struct mlx5e_priv *priv = flow->priv, *peer_priv;
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5_esw_flow_attr *attr = flow->attr->esw_attr;
 	struct mlx5e_tc_flow_parse_attr *parse_attr;
-	int i = mlx5_lag_get_dev_seq(peer_esw->dev);
 	struct mlx5e_rep_priv *peer_urpriv;
+	struct list_head *dup_peer_flows;
 	struct mlx5e_tc_flow *peer_flow;
 	struct mlx5_core_dev *in_mdev;
 	int err = 0;
@@ -4639,12 +4632,14 @@ static int mlx5e_tc_add_fdb_peer_flow(struct flow_cls_offload *f,
 		goto out;
 	}
 
-	peer_flow->peer_index = i;
+	peer_flow->peer_orig = flow;
 	list_add_tail(&peer_flow->peer_flows, &flow->peer_flows);
 	flow_flag_set(flow, DUP);
+	dup_peer_flows = xa_load(&esw->offloads.peer_flows, peer_vhca_id);
+	if (!dup_peer_flows)
+		return -ENODEV;
 	mutex_lock(&esw->offloads.peer_mutex);
-	list_add_tail(&flow->peer[i], &esw->offloads.peer_flows[i]);
-	set_bit(i, flow->peer_used);
+	list_add_tail(&peer_flow->peer, dup_peer_flows);
 	mutex_unlock(&esw->offloads.peer_mutex);
 
 out:
@@ -5539,22 +5534,14 @@ int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags)
 
 void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw)
 {
-	struct mlx5_devcom_comp_dev *devcom;
-	struct mlx5_devcom_comp_dev *pos;
-	struct mlx5e_tc_flow *flow, *tmp;
-	struct mlx5_eswitch *peer_esw;
-	int i;
-
-	devcom = esw->devcom;
-
-	mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) {
-		i = mlx5_lag_get_dev_seq(peer_esw->dev);
-		if (i < 0)
-			continue;
+	struct mlx5e_tc_flow *peer_flow, *tmp_peer_flow;
+	struct list_head *dup_peer_flows;
+	unsigned long index;
 
-		list_for_each_entry_safe(flow, tmp, &esw->offloads.peer_flows[i], peer[i])
-			mlx5e_tc_del_fdb_peers_flow(flow);
-	}
+	xa_for_each(&esw->offloads.peer_flows, index, dup_peer_flows)
+		list_for_each_entry_safe(peer_flow, tmp_peer_flow,
+					 dup_peer_flows, peer)
+			mlx5e_tc_del_fdb_peer_flow(peer_flow);
 }
 
 void mlx5e_tc_reoffload_flows_work(struct work_struct *work)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index 0b5e600e4a6a95..14479da98f427f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -849,7 +849,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
 	mlx5_cqwq_update_db_record(&cq->wq);
 
 	/* ensure cq space is freed before enabling more cqes */
-	wmb();
+	dma_wmb();
 
 	sq->dma_fifo_cc = dma_fifo_cc;
 	sq->cc = sqcc;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
index 87b5fd3495945f..b4cf3c5ac0dde8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c
@@ -1649,10 +1649,8 @@ int mlx5_esw_bridge_vport_unlink(struct net_device *br_netdev, u16 vport_num,
 	int err;
 
 	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
-	if (!port) {
-		NL_SET_ERR_MSG_MOD(extack, "Port is not attached to any bridge");
-		return -EINVAL;
-	}
+	if (!port)
+		return 0;
 	if (port->bridge->ifindex != br_netdev->ifindex) {
 		NL_SET_ERR_MSG_MOD(extack, "Port is attached to another bridge");
 		return -EINVAL;
@@ -1682,10 +1680,19 @@ int mlx5_esw_bridge_vport_peer_unlink(struct net_device *br_netdev, u16 vport_nu
 				      struct mlx5_esw_bridge_offloads *br_offloads,
 				      struct netlink_ext_ack *extack)
 {
+	if (!MLX5_CAP_ESW(br_offloads->esw->dev, merged_eswitch))
+		return 0;
+
 	return mlx5_esw_bridge_vport_unlink(br_netdev, vport_num, esw_owner_vhca_id, br_offloads,
 					    extack);
 }
 
+bool mlx5_esw_bridge_port_exists(u16 vport_num, u16 esw_owner_vhca_id,
+				 struct mlx5_esw_bridge_offloads *br_offloads)
+{
+	return mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
+}
+
 int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags,
 				  struct mlx5_esw_bridge_offloads *br_offloads,
 				  struct netlink_ext_ack *extack)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h
index d6f5391619930d..a4e59cc210894a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h
@@ -80,6 +80,8 @@ int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 pro
 				   struct mlx5_esw_bridge_offloads *br_offloads);
 int mlx5_esw_bridge_mcast_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable,
 			      struct mlx5_esw_bridge_offloads *br_offloads);
+bool mlx5_esw_bridge_port_exists(u16 vport_num, u16 esw_owner_vhca_id,
+				 struct mlx5_esw_bridge_offloads *br_offloads);
 int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags,
 				  struct mlx5_esw_bridge_offloads *br_offloads,
 				  struct netlink_ext_ack *extack);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index fc197d1dc9dfe1..2d289fb1353b0f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -1241,6 +1241,11 @@ static int mlx5_esw_host_functions_enabled_query(struct mlx5_eswitch *esw)
 	struct mlx5_esw_pf_info host_pf_info;
 	const u32 *query_host_out;
 
+	if (!mlx5_core_is_pf(esw->dev)) {
+		esw->esw_funcs.host_funcs_disabled = true;
+		return 0;
+	}
+
 	if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
 		return 0;
 
@@ -2460,13 +2465,20 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
 		}
 	}
 
-	if (mlx5_ecpf_vport_exists(dev) ||
-	    mlx5_core_is_ecpf_esw_manager(dev)) {
+	if (mlx5_ecpf_vport_exists(dev)) {
 		err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_ECPF);
 		if (err)
 			goto err;
 		idx++;
 	}
+
+	if (!xa_load(&esw->vports, esw->manager_vport)) {
+		err = mlx5_esw_vport_alloc(esw, idx, esw->manager_vport);
+		if (err)
+			goto err;
+		idx++;
+	}
+
 	err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_UPLINK);
 	if (err)
 		goto err;
@@ -2951,7 +2963,7 @@ bool mlx5_esw_hold(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_eswitch *esw = mdev->priv.eswitch;
 
-	/* e.g. VF doesn't have eswitch so nothing to do */
+	/* Not an eswitch manager, so there is no mode lock to take */
 	if (!mlx5_esw_allowed(esw))
 		return true;
 
@@ -3099,7 +3111,7 @@ void mlx5_eswitch_unblock_ipsec(struct mlx5_core_dev *dev)
 	struct mlx5_eswitch *esw = dev->priv.eswitch;
 
 	if (!mlx5_esw_allowed(esw))
-		/* Failure means no eswitch => core dev is not a PF */
+		/* Not an eswitch manager, so nothing was blocked */
 		return;
 
 	mutex_lock(&esw->state_lock);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 8b1f93b13ea9c5..e9cbcd73b23ffa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -319,7 +319,7 @@ struct mlx5_esw_offload {
 	struct mlx5_flow_handle *vport_rx_drop_rule;
 	struct mlx5_flow_table *ft_ipsec_tx_pol;
 	struct xarray vport_reps;
-	struct list_head peer_flows[MLX5_MAX_PORTS];
+	struct xarray peer_flows;
 	struct mutex peer_mutex;
 	struct mutex encap_tbl_lock; /* protects encap_tbl */
 	DECLARE_HASHTABLE(encap_tbl, 8);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index eb74b626016821..c712848b202db4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -3383,10 +3383,15 @@ static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw,
 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw,
 				     struct mlx5_eswitch *peer_esw)
 {
+	struct list_head *dup_peer_flows;
+
 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
 	mlx5e_tc_clean_fdb_peer_flows(esw);
 #endif
 	mlx5_esw_offloads_rep_event_unpair(esw, peer_esw);
+	dup_peer_flows = xa_erase(&esw->offloads.peer_flows,
+				  MLX5_CAP_GEN(peer_esw->dev, vhca_id));
+	kfree(dup_peer_flows);
 	esw_del_fdb_peer_miss_rules(esw, peer_esw->dev);
 }
 
@@ -3394,6 +3399,7 @@ static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
 				  struct mlx5_eswitch *peer_esw)
 {
 	const struct mlx5_eswitch_rep_ops *ops;
+	struct list_head *dup_peer_flows;
 	struct mlx5_eswitch_rep *rep;
 	unsigned long i;
 	u8 rep_type;
@@ -3403,6 +3409,20 @@ static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
 	if (err)
 		return err;
 
+	dup_peer_flows = kzalloc_obj(*dup_peer_flows);
+	if (!dup_peer_flows) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+	INIT_LIST_HEAD(dup_peer_flows);
+	err = xa_err(xa_store(&esw->offloads.peer_flows,
+			      MLX5_CAP_GEN(peer_esw->dev, vhca_id),
+			      dup_peer_flows, GFP_KERNEL));
+	if (err) {
+		kfree(dup_peer_flows);
+		goto err_out;
+	}
+
 	mlx5_esw_for_each_rep(esw, i, rep) {
 		for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
 			ops = esw->offloads.rep_ops[rep_type];
@@ -3548,10 +3568,9 @@ static int mlx5_esw_offloads_devcom_event(int event,
 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw,
 				   const struct mlx5_devcom_match_attr *attr)
 {
-	int i;
+	int err;
 
-	for (i = 0; i < MLX5_MAX_PORTS; i++)
-		INIT_LIST_HEAD(&esw->offloads.peer_flows[i]);
+	xa_init(&esw->offloads.peer_flows);
 	mutex_init(&esw->offloads.peer_mutex);
 
 	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
@@ -3572,10 +3591,12 @@ void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw,
 	if (!esw->devcom)
 		return;
 
-	mlx5_devcom_send_event(esw->devcom,
-			       ESW_OFFLOADS_DEVCOM_PAIR,
-			       ESW_OFFLOADS_DEVCOM_UNPAIR,
-			       esw);
+	err = mlx5_devcom_send_event(esw->devcom,
+				     ESW_OFFLOADS_DEVCOM_PAIR,
+				     ESW_OFFLOADS_DEVCOM_UNPAIR,
+				     esw);
+	if (err)
+		mlx5_esw_offloads_devcom_cleanup(esw);
 }
 
 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
@@ -3592,6 +3613,8 @@ void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
 	xa_destroy(&esw->paired);
 	xa_destroy(&esw->fdb_table.offloads.peer_miss_rules);
 	esw->devcom = NULL;
+	WARN_ON_ONCE(!xa_empty(&esw->offloads.peer_flows));
+	xa_destroy(&esw->offloads.peer_flows);
 }
 
 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
index 1f6bde5d762658..1341874ee58a3d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
@@ -384,7 +384,7 @@ static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn,
 
 	mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc);
 	/* ensure cq space is freed before enabling more cqes */
-	wmb();
+	dma_wmb();
 	mlx5_fpga_conn_arm_cq(conn);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
index 07440c58713a7d..7c0585fe1ee7a9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
@@ -66,10 +66,8 @@ static int mlx5_fw_reset_enable_remote_dev_reset_set(struct devlink *devlink, u3
 
 	fw_reset = dev->priv.fw_reset;
 
-	if (ctx->val.vbool)
-		clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
-	else
-		set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
+	assign_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
+		   &fw_reset->reset_flags, !ctx->val.vbool);
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c
index e9b8d79d2d21ed..b6125c7bd91bb6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c
@@ -101,16 +101,21 @@ static int flags_show(struct seq_file *file, void *priv)
 static int mapping_show(struct seq_file *file, void *priv)
 {
 	struct mlx5_core_dev *dev = file->private;
-	u8 ports[MLX5_MAX_PORTS] = {};
 	struct mlx5_lag *ldev;
 	bool hash = false;
 	bool lag_active;
 	int i, idx = 0;
 	int num_ports;
+	u8 *ports;
 
 	ldev = mlx5_lag_dev(dev);
+	ports = kcalloc(ldev->ports, sizeof(*ports), GFP_KERNEL);
+	if (!ports)
+		return -ENOMEM;
+
 	mutex_lock(&ldev->lock);
-	lag_active = __mlx5_lag_is_active(ldev);
+	lag_active = __mlx5_lag_is_active(ldev) &&
+		ldev->mode != MLX5_LAG_MODE_MPESW;
 	if (lag_active) {
 		if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) {
 			mlx5_infer_tx_enabled(&ldev->tracker, ldev, ports,
@@ -123,8 +128,10 @@ static int mapping_show(struct seq_file *file, void *priv)
 		}
 	}
 	mutex_unlock(&ldev->lock);
-	if (!lag_active)
+	if (!lag_active) {
+		kfree(ports);
 		return -EINVAL;
+	}
 
 	for (i = 0; i < num_ports; i++) {
 		if (hash)
@@ -133,6 +140,7 @@ static int mapping_show(struct seq_file *file, void *priv)
 			seq_printf(file, "%d:%d\n", i + 1, ports[i]);
 	}
 
+	kfree(ports);
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index c655f6e32e9b07..eb15334825786e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -63,19 +63,25 @@ static int get_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags)
 	return MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY;
 }
 
-static u8 lag_active_port_bits(struct mlx5_lag *ldev,
-			       struct lag_tracker *tracker)
+static int lag_active_port_bits(struct mlx5_lag *ldev,
+				struct lag_tracker *tracker)
 {
-	u8 enabled_ports[MLX5_MAX_PORTS] = {};
 	u8 active_port = 0;
+	u8 *enabled_ports;
 	int num_enabled;
 	int idx;
 
+	enabled_ports = kcalloc(ldev->ports, sizeof(*enabled_ports),
+				GFP_KERNEL);
+	if (!enabled_ports)
+		return -ENOMEM;
+
 	mlx5_infer_tx_enabled(tracker, ldev, enabled_ports,
 			      &num_enabled);
 	for (idx = 0; idx < num_enabled; idx++)
 		active_port |= BIT_MASK(enabled_ports[idx]);
 
+	kfree(enabled_ports);
 	return active_port;
 }
 
@@ -105,13 +111,21 @@ static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev,
 		MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, ports[idx0]);
 		MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, ports[idx1]);
 		break;
-	case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT:
+	case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT: {
+		u8 active_port;
+		int ret;
+
 		if (!MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass))
 			break;
 
-		MLX5_SET(lagc, lag_ctx, active_port,
-			 lag_active_port_bits(ldev, tracker));
+		ret = lag_active_port_bits(ldev, tracker);
+		if (ret < 0)
+			return ret;
+
+		active_port = ret;
+		MLX5_SET(lagc, lag_ctx, active_port, active_port);
 		break;
+	}
 	default:
 		break;
 	}
@@ -240,22 +254,31 @@ static void mlx5_lag_print_mapping(struct mlx5_core_dev *dev,
 				   struct lag_tracker *tracker,
 				   unsigned long flags)
 {
-	char buf[MLX5_MAX_PORTS * 10 + 1] = {};
-	u8 enabled_ports[MLX5_MAX_PORTS] = {};
+	u8 *enabled_ports = NULL;
 	int written = 0;
 	int num_enabled;
+	char *buf;
 	int idx;
 	int err;
 	int i;
 	int j;
 
+	buf = kcalloc(ldev->ports * 10 + 1, sizeof(*buf), GFP_KERNEL);
+	if (!buf)
+		return;
+
 	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
+		enabled_ports = kcalloc(ldev->ports, sizeof(*enabled_ports),
+					GFP_KERNEL);
+		if (!enabled_ports)
+			goto free_buf;
+
 		mlx5_infer_tx_enabled(tracker, ldev, enabled_ports,
 				      &num_enabled);
 		for (i = 0; i < num_enabled; i++) {
 			err = scnprintf(buf + written, 4, "%d, ", enabled_ports[i] + 1);
 			if (err != 3)
-				return;
+				goto free_enabled;
 			written += err;
 		}
 		buf[written - 2] = 0;
@@ -267,12 +290,17 @@ static void mlx5_lag_print_mapping(struct mlx5_core_dev *dev,
 				err = scnprintf(buf + written, 10,
 						" port %d:%d", i + 1, ldev->v2p_map[idx]);
 				if (err != 9)
-					return;
+					goto free_enabled;
 				written += err;
 			}
 		}
 		mlx5_core_info(dev, "lag map:%s\n", buf);
 	}
+
+free_enabled:
+	kfree(enabled_ports);
+free_buf:
+	kfree(buf);
 }
 
 static int mlx5_lag_netdev_event(struct notifier_block *this,
@@ -308,6 +336,7 @@ static void mlx5_ldev_free(struct kref *ref)
 	cancel_work_sync(&ldev->speed_update_work);
 	destroy_workqueue(ldev->wq);
 	mutex_destroy(&ldev->lock);
+	kfree(ldev->v2p_map);
 	kfree(ldev);
 }
 
@@ -330,11 +359,16 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev)
 	if (!ldev)
 		return NULL;
 
+	ldev->ports = MLX5_CAP_GEN(dev, num_lag_ports);
+	ldev->buckets = 1;
+	ldev->v2p_map = kcalloc(ldev->ports * MLX5_LAG_MAX_HASH_BUCKETS,
+				sizeof(*ldev->v2p_map), GFP_KERNEL);
+	if (!ldev->v2p_map)
+		goto err_v2p_map;
+
 	ldev->wq = create_singlethread_workqueue("mlx5_lag");
-	if (!ldev->wq) {
-		kfree(ldev);
-		return NULL;
-	}
+	if (!ldev->wq)
+		goto err_wq;
 
 	kref_init(&ldev->ref);
 	mutex_init(&ldev->lock);
@@ -358,10 +392,13 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev)
 		mlx5_core_err(dev, "Failed to init multipath lag err=%d\n",
 			      err);
 
-	ldev->ports = MLX5_CAP_GEN(dev, num_lag_ports);
-	ldev->buckets = 1;
-
 	return ldev;
+
+err_wq:
+	kfree(ldev->v2p_map);
+err_v2p_map:
+	kfree(ldev);
+	return NULL;
 }
 
 int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
@@ -659,20 +696,28 @@ static bool __mlx5_lag_is_sd_active(struct mlx5_lag *ldev,
  * If there are ports that are disabled fill the relevant slots
  * with mapping that points to active ports.
  */
-static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
-					   struct mlx5_lag *ldev,
-					   u8 buckets,
-					   u8 *ports)
+static int mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
+					  struct mlx5_lag *ldev,
+					  u8 buckets,
+					  u8 *ports)
 {
-	int disabled[MLX5_MAX_PORTS] = {};
-	int enabled[MLX5_MAX_PORTS] = {};
 	int disabled_ports_num = 0;
 	int enabled_ports_num = 0;
+	int *disabled;
+	int *enabled;
+	int err = 0;
 	int idx;
 	u32 rand;
 	int i;
 	int j;
 
+	enabled = kcalloc(ldev->ports, sizeof(*enabled), GFP_KERNEL);
+	disabled = kcalloc(ldev->ports, sizeof(*disabled), GFP_KERNEL);
+	if (!enabled || !disabled) {
+		err = -ENOMEM;
+		goto out;
+	}
+
 	mlx5_ldev_for_each(i, 0, ldev) {
 		if (tracker->netdev_state[i].tx_enabled &&
 		    tracker->netdev_state[i].link_up)
@@ -695,7 +740,7 @@ static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
 	/* If all ports are disabled/enabled keep native mapping */
 	if (enabled_ports_num == ldev->ports ||
 	    disabled_ports_num == ldev->ports)
-		return;
+		goto out;
 
 	/* Go over the disabled ports and for each assign a random active port */
 	for (i = 0; i < disabled_ports_num; i++) {
@@ -708,6 +753,11 @@ static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
 				mlx5_lag_xa_to_dev_idx(ldev, rand_xa_idx) + 1;
 		}
 	}
+
+out:
+	kfree(enabled);
+	kfree(disabled);
+	return err;
 }
 
 static bool mlx5_lag_has_drop_rule(struct mlx5_lag *ldev)
@@ -742,11 +792,8 @@ static void mlx5_lag_drop_rule_cleanup(struct mlx5_lag *ldev)
 static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
 				     struct lag_tracker *tracker)
 {
-	u8 disabled_ports[MLX5_MAX_PORTS] = {};
 	struct mlx5_core_dev *dev;
 	struct lag_func *pf;
-	int disabled_index;
-	int num_disabled;
 	int err;
 	int i;
 
@@ -758,11 +805,11 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
 	if (!ldev->tracker.has_inactive)
 		return;
 
-	mlx5_infer_tx_disabled(tracker, ldev, disabled_ports, &num_disabled);
-
-	for (i = 0; i < num_disabled; i++) {
-		disabled_index = disabled_ports[i];
-		pf = mlx5_lag_pf(ldev, disabled_index);
+	mlx5_ldev_for_each(i, 0, ldev) {
+		if (tracker->netdev_state[i].tx_enabled &&
+		    tracker->netdev_state[i].link_up)
+			continue;
+		pf = mlx5_lag_pf(ldev, i);
 		dev = pf->dev;
 		err = mlx5_esw_acl_ingress_vport_drop_rule_create(dev->priv.eswitch,
 								  MLX5_VPORT_UPLINK);
@@ -770,7 +817,8 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
 			pf->has_drop = true;
 		else
 			mlx5_core_err(dev,
-				      "Failed to create lag drop rule, error: %d", err);
+				      "Failed to create lag drop rule, error: %d",
+				      err);
 	}
 }
 
@@ -807,7 +855,10 @@ static int _mlx5_modify_lag(struct mlx5_lag *ldev,
 		    !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table_bypass))
 			return ret;
 
-		active_ports = lag_active_port_bits(ldev, tracker);
+		ret = lag_active_port_bits(ldev, tracker);
+		if (ret < 0)
+			return ret;
+		active_ports = ret;
 
 		return mlx5_cmd_modify_active_port(dev0, active_ports);
 	}
@@ -853,8 +904,8 @@ void mlx5_modify_lag(struct mlx5_lag *ldev,
 		     struct lag_tracker *tracker)
 {
 	int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
-	u8 ports[MLX5_MAX_PORTS * MLX5_LAG_MAX_HASH_BUCKETS] = {};
 	struct mlx5_core_dev *dev0;
+	u8 *ports;
 	int idx;
 	int err;
 	int i;
@@ -864,7 +915,20 @@ void mlx5_modify_lag(struct mlx5_lag *ldev,
 		return;
 
 	dev0 = mlx5_lag_pf(ldev, first_idx)->dev;
-	mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ports);
+
+	ports = kcalloc(ldev->ports * MLX5_LAG_MAX_HASH_BUCKETS,
+			sizeof(*ports), GFP_KERNEL);
+	if (!ports)
+		return;
+
+	err = mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets,
+					     ports);
+	if (err) {
+		mlx5_core_err(dev0,
+			      "mlx5_infer_tx_affinity_mapping failed, err = %d\n",
+			      err);
+		goto out;
+	}
 
 	mlx5_ldev_for_each(i, 0, ldev) {
 		for (j = 0; j < ldev->buckets; j++) {
@@ -876,9 +940,10 @@ void mlx5_modify_lag(struct mlx5_lag *ldev,
 				mlx5_core_err(dev0,
 					      "Failed to modify LAG (%d)\n",
 					      err);
-				return;
+				goto out;
 			}
-			memcpy(ldev->v2p_map, ports, sizeof(ports));
+			memcpy(ldev->v2p_map, ports,
+			       ldev->ports * MLX5_LAG_MAX_HASH_BUCKETS);
 
 			mlx5_lag_print_mapping(dev0, ldev, tracker,
 					       ldev->mode_flags);
@@ -899,6 +964,8 @@ void mlx5_modify_lag(struct mlx5_lag *ldev,
 					     ndev);
 		dev_put(ndev);
 	}
+out:
+	kfree(ports);
 }
 
 static int mlx5_lag_set_port_sel_mode(struct mlx5_lag *ldev,
@@ -1027,7 +1094,16 @@ int mlx5_activate_lag(struct mlx5_lag *ldev,
 		return err;
 
 	if (mode != MLX5_LAG_MODE_MPESW) {
-		mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ldev->v2p_map);
+		err = mlx5_infer_tx_affinity_mapping(tracker, ldev,
+						     ldev->buckets,
+						     ldev->v2p_map);
+		if (err) {
+			mlx5_core_err(dev0,
+				      "mlx5_infer_tx_affinity_mapping failed, err = %d\n",
+				      err);
+			return err;
+		}
+
 		if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &flags)) {
 			err = mlx5_lag_port_sel_create(ldev, tracker->hash_type,
 						       ldev->v2p_map);
@@ -1266,25 +1342,45 @@ void mlx5_lag_remove_devices(struct mlx5_lag *ldev)
 	mlx5_lag_remove_devices_filter(ldev, MLX5_LAG_FILTER_PORTS);
 }
 
+static int mlx5_lag_reload_ib_reps_idx(struct mlx5_lag *ldev, int idx,
+				       u32 flags)
+{
+	struct lag_func *pf = mlx5_lag_pf(ldev, idx);
+	struct mlx5_eswitch *esw;
+	int ret;
+
+	if (pf->dev->priv.flags & flags)
+		return 0;
+
+	esw = pf->dev->priv.eswitch;
+	mlx5_esw_reps_block(esw);
+	ret = mlx5_eswitch_reload_ib_reps(esw);
+	mlx5_esw_reps_unblock(esw);
+
+	return ret;
+}
+
 static int mlx5_lag_reload_ib_reps_unlocked(struct mlx5_lag *ldev, u32 flags,
 					    u32 filter, bool cont_on_fail)
 {
-	struct lag_func *pf;
+	int master_idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1,
+							     filter);
 	int ret;
 	int i;
 
+	if (master_idx < 0)
+		return -EINVAL;
+
+	ret = mlx5_lag_reload_ib_reps_idx(ldev, master_idx, flags);
+	if (ret && !cont_on_fail)
+		return ret;
+
 	mlx5_lag_for_each(i, 0, ldev, filter) {
-		pf = mlx5_lag_pf(ldev, i);
-		if (!(pf->dev->priv.flags & flags)) {
-			struct mlx5_eswitch *esw;
-
-			esw = pf->dev->priv.eswitch;
-			mlx5_esw_reps_block(esw);
-			ret = mlx5_eswitch_reload_ib_reps(esw);
-			mlx5_esw_reps_unblock(esw);
-			if (ret && !cont_on_fail)
-				return ret;
-		}
+		if (i == master_idx)
+			continue;
+		ret = mlx5_lag_reload_ib_reps_idx(ldev, i, flags);
+		if (ret && !cont_on_fail)
+			return ret;
 	}
 
 	return 0;
@@ -1923,9 +2019,11 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
 			}
 		}
 		if (i < MLX5_MAX_PORTS) {
-			slave = bond_slave_get_rcu(ndev_tmp);
-			if (slave)
-				has_inactive |= bond_is_slave_inactive(slave);
+			if (netif_is_bond_master(upper)) {
+				slave = bond_slave_get_rcu(ndev_tmp);
+				if (slave)
+					has_inactive |= bond_is_slave_inactive(slave);
+			}
 			bond_status |= (1 << idx);
 		}
 
@@ -2009,7 +2107,7 @@ static int mlx5_handle_changeinfodata_event(struct mlx5_lag *ldev,
 	bool has_inactive = 0;
 	int idx;
 
-	if (!netif_is_lag_master(ndev))
+	if (!netif_is_bond_master(ndev))
 		return 0;
 
 	rcu_read_lock();
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
index e9f0ef83ce1d25..8552792b9035fd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
@@ -85,7 +85,7 @@ struct mlx5_lag {
 	u8			  ports;
 	u8			  buckets;
 	int			  mode_changes_in_progress;
-	u8			  v2p_map[MLX5_MAX_PORTS * MLX5_LAG_MAX_HASH_BUCKETS];
+	u8			  *v2p_map;
 	struct kref               ref;
 	struct xarray             pfs;
 	struct lag_tracker        tracker;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c
index 6b4ad3c53f2f33..7c7e6c832a93d9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c
@@ -35,8 +35,7 @@ bool mlx5_lag_shared_fdb_supported_filter(struct mlx5_lag *ldev, u32 filter)
 
 	if (is_mdev_switchdev_mode(dev0) &&
 	    mlx5_eswitch_vport_match_metadata_enabled(dev0->priv.eswitch) &&
-	    mlx5_esw_offloads_devcom_is_ready(dev0->priv.eswitch) &&
-	    MLX5_CAP_ESW(dev0, esw_shared_ingress_acl))
+	    mlx5_esw_offloads_devcom_is_ready(dev0->priv.eswitch))
 		ret = true;
 
 	return ret;
@@ -270,6 +269,7 @@ int mlx5_lag_shared_fdb_create(struct mlx5_lag *ldev,
 			pf->sd_fdb_active = false;
 		}
 		mlx5_lag_destroy_single_fdb_filter(ldev, group_id);
+		mlx5_lag_unload_reps_from_locked(ldev, filter);
 	}
 err_add_devices:
 	mlx5_lag_add_devices_filter(ldev, filter);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c
index 614cd57a40d744..8f7a89ae4ad8c2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c
@@ -421,7 +421,7 @@ int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
 	mlx5_cqwq_update_db_record(&cq->wq);
 
 	/* ensure cq space is freed before enabling more cqes */
-	wmb();
+	dma_wmb();
 
 	if (with_data)
 		aso->cc += MLX5_ASO_WQEBBS_DATA;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
index 64f92427602de4..75855481522b11 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
@@ -37,6 +37,7 @@ struct mlx5_devcom_comp {
 	struct mlx5_devcom_key key;
 	mlx5_devcom_event_handler_t handler;
 	struct kref ref;
+	int nr_devs;
 	bool ready;
 	struct rw_semaphore sem;
 	struct lock_class_key lock_key;
@@ -170,6 +171,7 @@ devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc,
 
 	down_write(&comp->sem);
 	list_add_tail(&devcom->list, &comp->comp_dev_list_head);
+	WRITE_ONCE(comp->nr_devs, comp->nr_devs + 1);
 	up_write(&comp->sem);
 
 	return devcom;
@@ -182,6 +184,7 @@ devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom)
 
 	down_write(&comp->sem);
 	list_del(&devcom->list);
+	WRITE_ONCE(comp->nr_devs, comp->nr_devs - 1);
 	up_write(&comp->sem);
 
 	kref_put(&devcom->devc->ref, mlx5_devcom_dev_release);
@@ -284,7 +287,7 @@ int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom)
 {
 	struct mlx5_devcom_comp *comp = devcom->comp;
 
-	return kref_read(&comp->ref);
+	return READ_ONCE(comp->nr_devs);
 }
 
 int mlx5_devcom_locked_send_event(struct mlx5_devcom_comp_dev *devcom,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 5f28d906c35b61..3a3b0ec4aad1e8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -184,12 +184,8 @@ static int set_dma_caps(struct pci_dev *pdev)
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (err) {
-		dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
-			return err;
-		}
+		dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
+		return err;
 	}
 
 	dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
index 3bcf412a08c4c1..14a2bb9167a43d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
@@ -460,7 +460,9 @@ int mlx5hws_bwc_queue_poll(struct mlx5hws_context *ctx,
 		got_comp = !!ret;
 
 		if (unlikely(!got_comp && time_after(jiffies, timeout))) {
-			mlx5hws_err(ctx, "BWC poll error: polling queue %d - TIMEOUT\n", queue_id);
+			mlx5hws_err(ctx,
+				    "BWC poll error: polling queue %d - TIMEOUT (%d sec)\n",
+				    queue_id, MLX5HWS_BWC_POLLING_TIMEOUT);
 			return -ETIMEDOUT;
 		}
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c
index df3b93499eebb4..8b70040eaca030 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/send.c
@@ -111,9 +111,6 @@ static void hws_send_engine_post_ring(struct mlx5hws_send_ring_sq *sq,
 	wmb();
 
 	mlx5_write64((__be32 *)doorbell_cseg, sq->uar_map);
-
-	/* Ensure doorbell is written on uar_page before poll_cq */
-	WRITE_ONCE(doorbell_cseg, NULL);
 }
 
 static void
@@ -732,7 +729,7 @@ static int hws_send_ring_alloc_sq(struct mlx5_core_dev *mdev,
 		goto destroy_wq_cyc;
 	}
 
-	sq->wr_priv = kzalloc(sizeof(*sq->wr_priv) * buf_sz, GFP_KERNEL);
+	sq->wr_priv = kzalloc_objs(*sq->wr_priv, buf_sz);
 	if (!sq->wr_priv) {
 		err = -ENOMEM;
 		goto free_dep_wqe;
@@ -1192,6 +1189,8 @@ int mlx5hws_send_queue_action(struct mlx5hws_context *ctx,
 			      u16 queue_id,
 			      u32 actions)
 {
+	unsigned long timeout = jiffies +
+				secs_to_jiffies(MLX5HWS_BWC_POLLING_TIMEOUT);
 	struct mlx5hws_send_ring_sq *send_sq;
 	struct mlx5hws_send_engine *queue;
 	bool wait_comp = false;
@@ -1213,9 +1212,18 @@ int mlx5hws_send_queue_action(struct mlx5hws_context *ctx,
 			mlx5hws_send_engine_flush_queue(queue);
 
 		/* Poll queue until empty */
-		while (wait_comp && !mlx5hws_send_engine_empty(queue))
+		while (wait_comp && !mlx5hws_send_engine_empty(queue)) {
 			hws_send_engine_poll_cq(queue, NULL, &polled, 0);
 
+			if (unlikely(time_after(jiffies, timeout))) {
+				mlx5hws_err(ctx,
+					    "Error draining send queue %d - TIMEOUT (%d sec)\n",
+					    queue_id,
+					    MLX5HWS_BWC_POLLING_TIMEOUT);
+				return -ETIMEDOUT;
+			}
+		}
+
 		break;
 	default:
 		return -EINVAL;
@@ -1287,7 +1295,7 @@ hws_send_wqe_fw(struct mlx5_core_dev *mdev,
 send_wqe:
 	ret = mlx5hws_cmd_generate_wqe(mdev, &attr, &cqe);
 	if (ret) {
-		mlx5_core_err(mdev, "Failed to write WQE using command");
+		mlx5_core_err(mdev, "Failed to write WQE using command\n");
 		return ret;
 	}
 
@@ -1305,7 +1313,7 @@ hws_send_wqe_fw(struct mlx5_core_dev *mdev,
 		goto send_wqe;
 	}
 
-	return -1;
+	return -EIO;
 }
 
 void mlx5hws_send_stes_fw(struct mlx5hws_context *ctx,
@@ -1324,10 +1332,14 @@ void mlx5hws_send_stes_fw(struct mlx5hws_context *ctx,
 	pdn = ctx->pd_num;
 
 	/* Writing through FW can't HW fence, therefore we drain the queue */
-	if (send_attr->fence)
-		mlx5hws_send_queue_action(ctx,
-					  queue_id,
-					  MLX5HWS_SEND_QUEUE_ACTION_DRAIN_SYNC);
+	if (send_attr->fence) {
+		enum mlx5hws_send_queue_actions drain =
+			MLX5HWS_SEND_QUEUE_ACTION_DRAIN_SYNC;
+
+		ret = mlx5hws_send_queue_action(ctx, queue_id, drain);
+		if (ret)
+			goto fail_rule;
+	}
 
 	if (ste_attr->rtc_1) {
 		send_attr->id = ste_attr->rtc_1;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
index bfe3268dfdc17d..dceb69945d26e8 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
@@ -2429,11 +2429,8 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (err) {
-		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "dma_set_mask failed\n");
-			goto err_pci_set_dma_mask;
-		}
+		dev_err(&pdev->dev, "dma_set_mask_and_coherent failed\n");
+		goto err_pci_set_dma_mask;
 	}
 
 	if (pci_resource_len(pdev, 0) < MLXSW_PCI_BAR0_SIZE) {
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve_vxlan.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve_vxlan.c
index 52c2fe3644d4b9..21e7b499f3879c 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve_vxlan.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_nve_vxlan.c
@@ -59,8 +59,11 @@ static bool mlxsw_sp_nve_vxlan_can_offload(const struct mlxsw_sp_nve *nve,
 					   const struct mlxsw_sp_nve_params *params,
 					   struct netlink_ext_ack *extack)
 {
-	struct vxlan_dev *vxlan = netdev_priv(params->dev);
-	struct vxlan_config *cfg = &vxlan->cfg;
+	const struct vxlan_config *cfg;
+	struct vxlan_dev *vxlan;
+
+	vxlan = netdev_priv(params->dev);
+	cfg = rtnl_dereference(vxlan->cfg);
 
 	if (vxlan_addr_multicast(&cfg->remote_ip)) {
 		NL_SET_ERR_MSG_MOD(extack, "VxLAN: Multicast destination IP is not supported");
@@ -92,6 +95,11 @@ static bool mlxsw_sp_nve_vxlan_can_offload(const struct mlxsw_sp_nve *nve,
 		return false;
 	}
 
+	if (test_bit(VXLAN_DEV_F_MDB, &vxlan->flags)) {
+		NL_SET_ERR_MSG_MOD(extack, "VxLAN: MDB entries are not supported");
+		return false;
+	}
+
 	switch (cfg->saddr.sa.sa_family) {
 	case AF_INET:
 		if (!mlxsw_sp_nve_vxlan_ipv4_flags_check(cfg, extack))
@@ -148,8 +156,11 @@ static void mlxsw_sp_nve_vxlan_config(const struct mlxsw_sp_nve *nve,
 				      const struct mlxsw_sp_nve_params *params,
 				      struct mlxsw_sp_nve_config *config)
 {
-	struct vxlan_dev *vxlan = netdev_priv(params->dev);
-	struct vxlan_config *cfg = &vxlan->cfg;
+	const struct vxlan_config *cfg;
+	struct vxlan_dev *vxlan;
+
+	vxlan = netdev_priv(params->dev);
+	cfg = rtnl_dereference(vxlan->cfg);
 
 	config->type = MLXSW_SP_NVE_TYPE_VXLAN;
 	config->ttl = cfg->ttl;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 3d6fdbab05e012..f4a435885ba431 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -2397,14 +2397,15 @@ mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 static void
 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 {
+	struct net *net = mlxsw_sp_net(mlxsw_sp);
 	unsigned long interval;
 
 #if IS_ENABLED(CONFIG_IPV6)
 	interval = min_t(unsigned long,
-			 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
-			 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
+			 NEIGH_VAR(&arp_table(net)->parms, DELAY_PROBE_TIME),
+			 NEIGH_VAR(&nd_table(net)->parms, DELAY_PROBE_TIME));
 #else
-	interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
+	interval = NEIGH_VAR(&arp_table(net)->parms, DELAY_PROBE_TIME);
 #endif
 	mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
 }
@@ -2414,6 +2415,8 @@ static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 						   int ent_index)
 {
 	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
+	struct net *net = mlxsw_sp_net(mlxsw_sp);
+	struct neigh_table *tbl;
 	struct net_device *dev;
 	struct neighbour *n;
 	__be32 dipn;
@@ -2429,9 +2432,10 @@ static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 		return;
 	}
 
+	tbl = arp_table(net);
 	dipn = htonl(dip);
 	dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]);
-	n = neigh_lookup(&arp_tbl, &dipn, dev);
+	n = neigh_lookup(tbl, &dipn, dev);
 	if (!n)
 		return;
 
@@ -2445,6 +2449,8 @@ static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
 						   char *rauhtd_pl,
 						   int rec_index)
 {
+	struct net *net = mlxsw_sp_net(mlxsw_sp);
+	struct neigh_table *tbl;
 	struct net_device *dev;
 	struct neighbour *n;
 	struct in6_addr dip;
@@ -2458,8 +2464,9 @@ static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
 		return;
 	}
 
+	tbl = nd_table(net);
 	dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]);
-	n = neigh_lookup(&nd_tbl, &dip, dev);
+	n = neigh_lookup(tbl, &dip, dev);
 	if (!n)
 		return;
 
@@ -3014,16 +3021,17 @@ static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
 		.mlxsw_sp = mlxsw_sp,
 		.rif = rif,
 	};
+	struct net *net = mlxsw_sp_net(mlxsw_sp);
 
 	if (!mlxsw_sp_dev_lower_is_port(mlxsw_sp_rif_dev(rif)))
 		return 0;
 
-	neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms);
+	neigh_for_each(arp_table(net), mlxsw_sp_neigh_rif_made_sync_each, &rms);
 	if (rms.err)
 		goto err_arp;
 
 #if IS_ENABLED(CONFIG_IPV6)
-	neigh_for_each(&nd_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms);
+	neigh_for_each(nd_table(net), mlxsw_sp_neigh_rif_made_sync_each, &rms);
 #endif
 	if (rms.err)
 		goto err_nd;
@@ -4622,7 +4630,7 @@ static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
 	nh->nh_weight = 1;
 #endif
 	memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
-	nh->neigh_tbl = &arp_tbl;
+	nh->neigh_tbl = arp_table(mlxsw_sp_net(mlxsw_sp));
 	err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
 	if (err)
 		return err;
@@ -5112,6 +5120,7 @@ mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
 			  struct nh_notifier_single_info *nh_obj, int weight)
 {
 	struct net_device *dev = nh_obj->dev;
+	struct net *net = dev_net(dev);
 	int err;
 
 	nh->nhgi = nh_grp->nhgi;
@@ -5120,12 +5129,12 @@ mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
 	switch (nh_obj->gw_family) {
 	case AF_INET:
 		memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4));
-		nh->neigh_tbl = &arp_tbl;
+		nh->neigh_tbl = arp_table(net);
 		break;
 	case AF_INET6:
 		memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6));
 #if IS_ENABLED(CONFIG_IPV6)
-		nh->neigh_tbl = &nd_tbl;
+		nh->neigh_tbl = nd_table(net);
 #endif
 		break;
 	}
@@ -6981,7 +6990,7 @@ static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
 	nh->nh_weight = rt->fib6_nh->fib_nh_weight;
 	memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
 #if IS_ENABLED(CONFIG_IPV6)
-	nh->neigh_tbl = &nd_tbl;
+	nh->neigh_tbl = nd_table(mlxsw_sp_net(mlxsw_sp));
 #endif
 
 	err = mlxsw_sp_nexthop_counter_enable(mlxsw_sp, nh);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
index ae63d549b54299..1cd8347c274d49 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c
@@ -456,6 +456,7 @@ mlxsw_sp_span_entry_gretap4_parms(struct mlxsw_sp *mlxsw_sp,
 	bool inherit_tos = tparm.iph.tos & 0x1;
 	bool inherit_ttl = !tparm.iph.ttl;
 	union mlxsw_sp_l3addr gw = daddr;
+	struct neigh_table *tbl = NULL;
 	struct net_device *l3edev;
 
 	if (!(to_dev->flags & IFF_UP) ||
@@ -469,9 +470,11 @@ mlxsw_sp_span_entry_gretap4_parms(struct mlxsw_sp *mlxsw_sp,
 		return mlxsw_sp_span_entry_unoffloadable(sparmsp);
 
 	l3edev = mlxsw_sp_span_gretap4_route(to_dev, &saddr.addr4, &gw.addr4);
+	if (l3edev)
+		tbl = arp_table(dev_net(l3edev));
 	return mlxsw_sp_span_entry_tunnel_parms_common(l3edev, saddr, daddr, gw,
 						       tparm.iph.ttl,
-						       &arp_tbl, sparmsp);
+						       tbl, sparmsp);
 }
 
 static int
@@ -561,6 +564,7 @@ mlxsw_sp_span_entry_gretap6_parms(struct mlxsw_sp *mlxsw_sp,
 	union mlxsw_sp_l3addr daddr = { .addr6 = tparm.raddr };
 	bool inherit_ttl = !tparm.hop_limit;
 	union mlxsw_sp_l3addr gw = daddr;
+	struct neigh_table *tbl = NULL;
 	struct net_device *l3edev;
 
 	if (!(to_dev->flags & IFF_UP) ||
@@ -574,9 +578,11 @@ mlxsw_sp_span_entry_gretap6_parms(struct mlxsw_sp *mlxsw_sp,
 		return mlxsw_sp_span_entry_unoffloadable(sparmsp);
 
 	l3edev = mlxsw_sp_span_gretap6_route(to_dev, &saddr.addr6, &gw.addr6);
+	if (l3edev)
+		tbl = nd_table(dev_net(l3edev));
 	return mlxsw_sp_span_entry_tunnel_parms_common(l3edev, saddr, daddr, gw,
 						       tparm.hop_limit,
-						       &nd_tbl, sparmsp);
+						       tbl, sparmsp);
 }
 
 static int
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index fe45e533a4b2ef..de60b698bea982 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -2513,15 +2513,17 @@ mlxsw_sp_bridge_vlan_aware_vxlan_join(struct mlxsw_sp_bridge_device *bridge_devi
 {
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(bridge_device->dev);
 	struct vxlan_dev *vxlan = netdev_priv(vxlan_dev);
-	struct mlxsw_sp_nve_params params = {
-		.type = MLXSW_SP_NVE_TYPE_VXLAN,
-		.vni = vxlan->cfg.vni,
-		.dev = vxlan_dev,
-		.ethertype = ethertype,
-	};
+	struct mlxsw_sp_nve_params params;
+	const struct vxlan_config *cfg;
 	struct mlxsw_sp_fid *fid;
 	int err;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+	params.type = MLXSW_SP_NVE_TYPE_VXLAN;
+	params.vni = cfg->vni;
+	params.dev = vxlan_dev;
+	params.ethertype = ethertype;
+
 	/* If the VLAN is 0, we need to find the VLAN that is configured as
 	 * PVID and egress untagged on the bridge port of the VxLAN device.
 	 * It is possible no such VLAN exists
@@ -2704,15 +2706,17 @@ mlxsw_sp_bridge_8021d_vxlan_join(struct mlxsw_sp_bridge_device *bridge_device,
 {
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(bridge_device->dev);
 	struct vxlan_dev *vxlan = netdev_priv(vxlan_dev);
-	struct mlxsw_sp_nve_params params = {
-		.type = MLXSW_SP_NVE_TYPE_VXLAN,
-		.vni = vxlan->cfg.vni,
-		.dev = vxlan_dev,
-		.ethertype = ETH_P_8021Q,
-	};
+	struct mlxsw_sp_nve_params params;
+	const struct vxlan_config *cfg;
 	struct mlxsw_sp_fid *fid;
 	int err;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+	params.type = MLXSW_SP_NVE_TYPE_VXLAN;
+	params.vni = cfg->vni;
+	params.dev = vxlan_dev;
+	params.ethertype = ETH_P_8021Q;
+
 	fid = mlxsw_sp_fid_8021d_get(mlxsw_sp, bridge_device->dev->ifindex);
 	if (IS_ERR(fid)) {
 		NL_SET_ERR_MSG_MOD(extack, "Failed to create 802.1D FID");
@@ -2933,10 +2937,13 @@ static void __mlxsw_sp_bridge_vxlan_leave(struct mlxsw_sp *mlxsw_sp,
 					  const struct net_device *vxlan_dev)
 {
 	struct vxlan_dev *vxlan = netdev_priv(vxlan_dev);
+	const struct vxlan_config *cfg;
 	struct mlxsw_sp_fid *fid;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+
 	/* If the VxLAN device is down, then the FID does not have a VNI */
-	fid = mlxsw_sp_fid_lookup_by_vni(mlxsw_sp, vxlan->cfg.vni);
+	fid = mlxsw_sp_fid_lookup_by_vni(mlxsw_sp, cfg->vni);
 	if (!fid)
 		return;
 
@@ -3029,11 +3036,13 @@ static void mlxsw_sp_fdb_vxlan_call_notifiers(struct net_device *dev,
 	struct switchdev_notifier_vxlan_fdb_info info;
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	enum switchdev_notifier_type type;
+	const struct vxlan_config *cfg;
 
+	cfg = rtnl_dereference(vxlan->cfg);
 	type = adding ? SWITCHDEV_VXLAN_FDB_ADD_TO_BRIDGE :
 			SWITCHDEV_VXLAN_FDB_DEL_TO_BRIDGE;
 	mlxsw_sp_switchdev_addr_vxlan_convert(proto, addr, &info.remote_ip);
-	info.remote_port = vxlan->cfg.dst_port;
+	info.remote_port = cfg->dst_port;
 	info.remote_vni = vni;
 	info.remote_ifindex = 0;
 	ether_addr_copy(info.eth_addr, mac);
@@ -3236,8 +3245,10 @@ __mlxsw_sp_fdb_notify_mac_uc_tunnel_process(struct mlxsw_sp *mlxsw_sp,
 
 	if (adding && netif_is_vxlan(dev)) {
 		struct vxlan_dev *vxlan = netdev_priv(dev);
+		const struct vxlan_config *cfg;
 
-		if (!(vxlan->cfg.flags & VXLAN_F_LEARN))
+		cfg = rtnl_dereference(vxlan->cfg);
+		if (!(cfg->flags & VXLAN_F_LEARN))
 			return -EINVAL;
 	}
 
@@ -3722,9 +3733,11 @@ mlxsw_sp_switchdev_vxlan_work_prepare(struct mlxsw_sp_switchdev_event_work *
 {
 	struct vxlan_dev *vxlan = netdev_priv(switchdev_work->dev);
 	struct switchdev_notifier_vxlan_fdb_info *vxlan_fdb_info;
-	struct vxlan_config *cfg = &vxlan->cfg;
+	const struct vxlan_config *cfg;
 	struct netlink_ext_ack *extack;
 
+	cfg = rcu_dereference_rtnl(vxlan->cfg);
+
 	extack = switchdev_notifier_info_to_extack(info);
 	vxlan_fdb_info = container_of(info,
 				      struct switchdev_notifier_vxlan_fdb_info,
@@ -3851,11 +3864,15 @@ mlxsw_sp_switchdev_vxlan_vlan_add(struct mlxsw_sp *mlxsw_sp,
 				  struct netlink_ext_ack *extack)
 {
 	struct vxlan_dev *vxlan = netdev_priv(vxlan_dev);
-	__be32 vni = vxlan->cfg.vni;
+	const struct vxlan_config *cfg;
 	struct mlxsw_sp_fid *fid;
 	u16 old_vid;
+	__be32 vni;
 	int err;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+	vni = cfg->vni;
+
 	/* We cannot have the same VLAN as PVID and egress untagged on multiple
 	 * VxLAN devices. Note that we get this notification before the VLAN is
 	 * actually added to the bridge's database, so it is not possible for
@@ -3935,12 +3952,16 @@ mlxsw_sp_switchdev_vxlan_vlan_del(struct mlxsw_sp *mlxsw_sp,
 				  const struct net_device *vxlan_dev, u16 vid)
 {
 	struct vxlan_dev *vxlan = netdev_priv(vxlan_dev);
-	__be32 vni = vxlan->cfg.vni;
+	const struct vxlan_config *cfg;
 	struct mlxsw_sp_fid *fid;
+	__be32 vni;
 
 	if (!netif_running(vxlan_dev))
 		return;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+	vni = cfg->vni;
+
 	fid = mlxsw_sp_fid_lookup_by_vni(mlxsw_sp, vni);
 	if (!fid)
 		return;
diff --git a/drivers/net/ethernet/meta/Kconfig b/drivers/net/ethernet/meta/Kconfig
index ca5c7ac2a5bc22..f8b5fe7e31b734 100644
--- a/drivers/net/ethernet/meta/Kconfig
+++ b/drivers/net/ethernet/meta/Kconfig
@@ -35,4 +35,17 @@ config FBNIC
 	  To compile this driver as a module, choose M here. The module
 	  will be called fbnic.  MSI-X interrupt support is required.
 
+config MPNIC
+	tristate "Meta Platforms Network Interface Controller"
+	depends on 64BIT || COMPILE_TEST
+	depends on !S390
+	depends on PCI_MSI
+	select PAGE_POOL
+	help
+	  This driver supports the Meta Platforms Network Interface
+	  Controller.
+
+	  To compile this driver as a module, choose M here. The module
+	  will be called mpnic.
+
 endif # NET_VENDOR_META
diff --git a/drivers/net/ethernet/meta/Makefile b/drivers/net/ethernet/meta/Makefile
index 88804f3de96339..633973419c21b0 100644
--- a/drivers/net/ethernet/meta/Makefile
+++ b/drivers/net/ethernet/meta/Makefile
@@ -4,3 +4,4 @@
 #
 
 obj-$(CONFIG_FBNIC) += fbnic/
+obj-$(CONFIG_MPNIC) += mpnic/
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic.h b/drivers/net/ethernet/meta/fbnic/fbnic.h
index d0715695c43e75..0e7ae1def5bf3f 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic.h
@@ -57,6 +57,7 @@ struct fbnic_dev {
 	u64 dsn;
 	u32 mps;
 	u32 readrq;
+	u32 rx_cqe_nsecs;
 	u8 relaxed_ord;
 
 	/* Local copy of the devices TCAM */
@@ -250,6 +251,7 @@ int fbnic_csr_regs_len(struct fbnic_dev *fbd);
 
 void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm);
 void fbnic_config_rx_frames(struct fbnic_napi_vector *nv);
+void fbnic_config_rx_cqe_nsecs(struct fbnic_dev *fbd);
 
 enum fbnic_boards {
 	fbnic_board_asic
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
index 64b958df777443..baba3471bf5a42 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
@@ -974,6 +974,7 @@ enum {
 /* PUL User Registers */
 #define FBNIC_CSR_START_PUL_USER	0x31000	/* CSR section delimiter */
 #define FBNIC_PUL_OB_TLP_HDR_AW_CFG	0x3103d		/* 0xc40f4 */
+#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE	CSR_BIT(20)
 #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH	CSR_BIT(19)
 #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME		CSR_BIT(18)
 #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR	CSR_GENMASK(17, 15)
@@ -1215,6 +1216,10 @@ enum {
 #define FBNIC_IPC_MBX_DESC_LEN_MASK	DESC_GENMASK(63, 48)
 #define FBNIC_IPC_MBX_DESC_EOM		DESC_BIT(46)
 #define FBNIC_IPC_MBX_DESC_ADDR_MASK	DESC_GENMASK(45, 3)
+/* Set with FW_CMPL when the FW completed a descriptor without successfully
+ * processing it (e.g. a mailbox DMA error); the completion has no valid data.
+ */
+#define FBNIC_IPC_MBX_DESC_FW_ERR	DESC_BIT(2)
 #define FBNIC_IPC_MBX_DESC_FW_CMPL	DESC_BIT(1)
 #define FBNIC_IPC_MBX_DESC_HOST_CMPL	DESC_BIT(0)
 
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c
index 3c4563c8f403fa..6edfa0aa69f111 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c
@@ -539,8 +539,8 @@ static void fbnic_dbg_fw_mbx_display(struct seq_file *s,
 	/* Generate header */
 	seq_puts(s, mbx_idx == FBNIC_IPC_MBX_RX_IDX ? "Rx\n" : "Tx\n");
 
-	seq_printf(s, "Rdy: %d Head: %d Tail: %d\n",
-		   mbx->ready, mbx->head, mbx->tail);
+	seq_printf(s, "Rdy: %d Head: %d Tail: %d resp_error: %llu\n",
+		   mbx->ready, mbx->head, mbx->tail, mbx->resp_error);
 
 	snprintf(hdr, sizeof(hdr), "%3s %-4s %s %-12s %s %-3s %-16s\n",
 		 "Idx", "Len", "E", "Addr", "F", "H", "Raw");
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c
index 546e1c12d287c7..0b653c4c20a72a 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c
@@ -10,6 +10,7 @@
 #include "fbnic.h"
 #include "fbnic_fw.h"
 #include "fbnic_tlv.h"
+#include "fbnic_txrx.h"
 
 #define FBNIC_SN_STR_LEN	24
 
@@ -648,6 +649,7 @@ struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev)
 	fbd->mps = pcie_get_mps(pdev);
 	fbd->readrq = pcie_get_readrq(pdev);
 	fbd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev);
+	fbd->rx_cqe_nsecs = FBNIC_RX_CQE_NSECS_DEFAULT;
 
 	fbd->mac_addr_boundary = FBNIC_RPC_TCAM_MACDA_DEFAULT_BOUNDARY;
 
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
index 0e47088ec44baf..e774740fcb5d81 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c
@@ -246,6 +246,7 @@ static int fbnic_get_coalesce(struct net_device *netdev,
 	ec->tx_coalesce_usecs = fbn->tx_usecs;
 	ec->rx_coalesce_usecs = fbn->rx_usecs;
 	ec->rx_max_coalesced_frames = fbn->rx_max_frames;
+	kernel_coal->rx_cqe_nsecs = fbn->fbd->rx_cqe_nsecs;
 
 	return 0;
 }
@@ -256,6 +257,7 @@ static int fbnic_set_coalesce(struct net_device *netdev,
 			      struct netlink_ext_ack *extack)
 {
 	struct fbnic_net *fbn = netdev_priv(netdev);
+	struct fbnic_dev *fbd = fbn->fbd;
 
 	/* Verify against hardware limits */
 	if (ec->rx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT)) {
@@ -272,10 +274,20 @@ static int fbnic_set_coalesce(struct net_device *netdev,
 		NL_SET_ERR_MSG_MOD(extack, "rx_frames is above device max");
 		return -EINVAL;
 	}
+	if (kernel_coal->rx_cqe_nsecs < FBNIC_RX_CQE_NSECS_MIN ||
+	    kernel_coal->rx_cqe_nsecs > FBNIC_RX_CQE_NSECS_MAX) {
+		NL_SET_ERR_MSG_FMT_MOD(extack,
+				       "rx-cqe-nsecs must be between %u and %u",
+				       FBNIC_RX_CQE_NSECS_MIN,
+				       FBNIC_RX_CQE_NSECS_MAX);
+		return -EINVAL;
+	}
 
 	fbn->tx_usecs = ec->tx_coalesce_usecs;
 	fbn->rx_usecs = ec->rx_coalesce_usecs;
 	fbn->rx_max_frames = ec->rx_max_coalesced_frames;
+	fbd->rx_cqe_nsecs = kernel_coal->rx_cqe_nsecs;
+	fbnic_config_rx_cqe_nsecs(fbd);
 
 	if (netif_running(netdev)) {
 		int i;
@@ -313,6 +325,11 @@ fbnic_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring,
 	kernel_ring->hds_thresh = fbn->hds_thresh;
 }
 
+static u32 fbnic_ring_size_pow2(u32 size)
+{
+	return size ? roundup_pow_of_two(size) : 0;
+}
+
 static void fbnic_set_rings(struct fbnic_net *fbn,
 			    struct ethtool_ringparam *ring,
 			    struct kernel_ethtool_ringparam *kernel_ring)
@@ -334,10 +351,10 @@ fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring,
 	struct fbnic_net *clone;
 	int err;
 
-	ring->rx_pending	= roundup_pow_of_two(ring->rx_pending);
-	ring->rx_mini_pending	= roundup_pow_of_two(ring->rx_mini_pending);
-	ring->rx_jumbo_pending	= roundup_pow_of_two(ring->rx_jumbo_pending);
-	ring->tx_pending	= roundup_pow_of_two(ring->tx_pending);
+	ring->rx_pending	= fbnic_ring_size_pow2(ring->rx_pending);
+	ring->rx_mini_pending	= fbnic_ring_size_pow2(ring->rx_mini_pending);
+	ring->rx_jumbo_pending	= fbnic_ring_size_pow2(ring->rx_jumbo_pending);
+	ring->tx_pending	= fbnic_ring_size_pow2(ring->tx_pending);
 
 	/* These are absolute minimums allowing the device and driver to operate
 	 * but not necessarily guarantee reasonable performance. Settings below
@@ -2016,7 +2033,8 @@ static void fbnic_get_link_ext_stats(struct net_device *netdev,
 static const struct ethtool_ops fbnic_ethtool_ops = {
 	.cap_link_lanes_supported	= true,
 	.supported_coalesce_params	= ETHTOOL_COALESCE_USECS |
-					  ETHTOOL_COALESCE_RX_MAX_FRAMES,
+					  ETHTOOL_COALESCE_RX_MAX_FRAMES |
+					  ETHTOOL_COALESCE_RX_CQE_NSECS,
 	.supported_ring_params		= ETHTOOL_RING_USE_TCP_DATA_SPLIT |
 					  ETHTOOL_RING_USE_HDS_THRS,
 	.rxfh_max_num_contexts		= FBNIC_RPC_RSS_TBL_COUNT,
@@ -2025,7 +2043,8 @@ static const struct ethtool_ops fbnic_ethtool_ops = {
 					  ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM |
 					  ETHTOOL_OP_NEEDS_RTNL_SCHANNELS |
 					  ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM |
-					  ETHTOOL_OP_NEEDS_RTNL_GLINK,
+					  ETHTOOL_OP_NEEDS_RTNL_GLINK |
+					  ETHTOOL_OP_NEEDS_RTNL_TEST,
 	.get_drvinfo			= fbnic_get_drvinfo,
 	.get_regs_len			= fbnic_get_regs_len,
 	.get_regs			= fbnic_get_regs,
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c
index 283d25fae79e78..47894b45676b18 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c
@@ -60,8 +60,15 @@ static void fbnic_mbx_reset_desc_ring(struct fbnic_dev *fbd, int mbx_idx)
 	 */
 	switch (mbx_idx) {
 	case FBNIC_IPC_MBX_RX_IDX:
+		/* Clearing BME blocks the device from writing to the host
+		 * but leaves the requests parked in the write pipeline. The
+		 * write path only clears outstanding requests when both FLUSH
+		 * and FLUSH_MODE are set; FLUSH_MODE lets them drain without
+		 * landing on the host.
+		 */
 		wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AW_CFG,
-		     FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH);
+		     FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH |
+		     FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE);
 		break;
 	case FBNIC_IPC_MBX_TX_IDX:
 		wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AR_CFG,
@@ -285,6 +292,12 @@ static void fbnic_mbx_process_tx_msgs(struct fbnic_dev *fbd)
 		if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL))
 			break;
 
+		if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) {
+			tx_mbx->resp_error++;
+			dev_warn_ratelimited(fbd->dev,
+					     "FW completed a Tx mailbox request with an error\n");
+		}
+
 		fbnic_mbx_unmap_and_free_msg(fbd, FBNIC_IPC_MBX_TX_IDX, head);
 
 		head++;
@@ -1666,6 +1679,13 @@ static void fbnic_mbx_process_rx_msgs(struct fbnic_dev *fbd)
 		if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL))
 			break;
 
+		if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) {
+			rx_mbx->resp_error++;
+			dev_warn_ratelimited(fbd->dev,
+					     "FW reported an error on an Rx mailbox message; dropping\n");
+			goto next_page;
+		}
+
 		dma_sync_single_for_cpu(fbd->dev, rx_mbx->buf_info[head].addr,
 					FBNIC_RX_PAGE_SIZE, DMA_FROM_DEVICE);
 
@@ -1677,16 +1697,19 @@ static void fbnic_mbx_process_rx_msgs(struct fbnic_dev *fbd)
 		if (!length)
 			goto next_page;
 
-		/* Report descriptors with length greater than page size */
-		if (length > PAGE_SIZE) {
+		/* Report descriptors with invalid message extents. */
+		if (length < sizeof(msg->hdr) || length > PAGE_SIZE) {
 			dev_warn(fbd->dev,
 				 "Invalid mailbox descriptor length: %lld\n",
 				 length);
 			goto next_page;
 		}
 
-		if (le16_to_cpu(msg->hdr.len) * sizeof(u32) > length)
+		if (!le16_to_cpu(msg->hdr.len) ||
+		    le16_to_cpu(msg->hdr.len) * sizeof(u32) > length) {
 			dev_warn(fbd->dev, "Mailbox message length mismatch\n");
+			goto next_page;
+		}
 
 		/* If parsing fails dump contents of message to dmesg */
 		err = fbnic_tlv_msg_parse(fbd, msg, fbnic_fw_tlv_parser);
@@ -1733,7 +1756,9 @@ void fbnic_mbx_poll(struct fbnic_dev *fbd)
 int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd)
 {
 	struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX];
+	struct fbnic_fw_mbx *rx_mbx = &fbd->mbx[FBNIC_IPC_MBX_RX_IDX];
 	unsigned long timeout = jiffies + 10 * HZ + 1;
+	u64 tx_resp_error, rx_resp_error;
 	int err, i;
 
 	do {
@@ -1764,6 +1789,9 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd)
 	 * mgmt.version once we get the actual version from the firmware
 	 * in the capabilities request message.
 	 */
+send_cap_req:
+	tx_resp_error = tx_mbx->resp_error;
+	rx_resp_error = rx_mbx->resp_error;
 	err = fbnic_fw_xmit_simple_msg(fbd, FBNIC_TLV_MSG_ID_HOST_CAP_REQ);
 	if (err)
 		goto clean_mbx;
@@ -1781,9 +1809,27 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd)
 		msleep(20);
 		fbnic_mbx_poll(fbd);
 
+		/* A valid capabilities response ends the poll. Check it
+		 * before the FW_ERR retry below so a response parsed in the
+		 * same poll as an unrelated FW_ERR is not discarded.
+		 */
+		if (fbd->fw_cap.running.mgmt.version >= MIN_FW_VER_CODE)
+			break;
+
 		/* set err, but wait till mgmt.version check to report it */
-		if (!time_is_after_jiffies(timeout))
+		if (!time_is_after_jiffies(timeout)) {
 			err = -ETIMEDOUT;
+			continue;
+		}
+
+		/* The FW can flag our capabilities request (Tx) or its
+		 * response (Rx) with FW_ERR, in which case it produced no
+		 * usable response. The ring is not wedged, so re-issue the
+		 * request instead of spinning until the timeout.
+		 */
+		if (tx_mbx->resp_error != tx_resp_error ||
+		    rx_mbx->resp_error != rx_resp_error)
+			goto send_cap_req;
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h
index d84723e4cfa362..5f9969247e3059 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h
@@ -13,6 +13,7 @@ struct fbnic_tlv_msg;
 
 struct fbnic_fw_mbx {
 	u8 ready, head, tail;
+	u64 resp_error;
 	struct {
 		struct fbnic_tlv_msg	*msg;
 		dma_addr_t		addr;
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
index 53b7a938b4c277..16271c717a5aaa 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c
@@ -125,14 +125,7 @@ static void fbnic_mac_init_qm(struct fbnic_dev *fbd)
 	     FIELD_PREP(FBNIC_QM_TCQ_CTL0_COAL_WAIT,
 			clock_freq / 12500));
 
-	/* We will have the interrupt threshold timer tick once every
-	 * 1 usec and coalesce writes for up to 2 usecs.
-	 */
-	wr32(fbd, FBNIC_QM_RCQ_CTL0,
-	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES,
-			clock_freq / 1000000) |
-	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT,
-			clock_freq / 500000));
+	fbnic_config_rx_cqe_nsecs(fbd);
 
 	/* Configure spacer control to 64 beats. */
 	wr32(fbd, FBNIC_FAB_AXI4_AR_SPACER_2_CFG,
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c
index 8b9bc9e8ea56ca..ad234bb0f75d02 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c
@@ -276,8 +276,6 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
-	if (err)
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
 	if (err) {
 		dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
 		return err;
@@ -434,6 +432,7 @@ static int fbnic_pm_suspend(struct device *dev)
 {
 	struct fbnic_dev *fbd = dev_get_drvdata(dev);
 	struct net_device *netdev = fbd->netdev;
+	struct fbnic_net *fbn;
 
 	if (fbnic_init_failure(fbd))
 		goto null_uc_addr;
@@ -441,11 +440,16 @@ static int fbnic_pm_suspend(struct device *dev)
 	rtnl_lock();
 	netdev_lock(netdev);
 
+	fbn = netdev_priv(netdev);
+
 	netif_device_detach(netdev);
 
 	if (netif_running(netdev))
 		netdev->netdev_ops->ndo_stop(netdev);
 
+	/* The IRQs are about to be freed, so drop the napi vector count */
+	fbn->num_napi = 0;
+
 	netdev_unlock(netdev);
 	rtnl_unlock();
 
@@ -508,16 +512,20 @@ static int __fbnic_pm_resume(struct device *dev)
 	if (fbnic_init_failure(fbd))
 		return 0;
 
+	rtnl_lock();
+	netdev_lock(netdev);
+
 	fbn = netdev_priv(netdev);
 
 	/* Reset the queues if needed */
 	fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues);
 
-	rtnl_lock();
-	netdev_lock(netdev);
-
-	if (netif_running(netdev))
+	if (netif_running(netdev)) {
 		err = __fbnic_open(fbn);
+		/* On failure the vectors are freed, so drop the count */
+		if (err)
+			fbn->num_napi = 0;
+	}
 
 	netdev_unlock(netdev);
 	rtnl_unlock();
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c
index bc0f38b6a2b2b4..ab6e342ec3fa4b 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c
@@ -126,12 +126,8 @@ void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd,
 			set_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam);
 			mac_addr->state = FBNIC_TCAM_S_ADD;
 		}
-		if (enable_host)
-			set_bit(FBNIC_MAC_ADDR_T_ALLMULTI,
-				mac_addr->act_tcam);
-		else
-			clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI,
-				  mac_addr->act_tcam);
+		assign_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam,
+			   enable_host);
 	} else {
 		__fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BMC);
 		__fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_ALLMULTI);
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c
index c55d4f76a5fc0c..3ef3935c0419f2 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c
@@ -430,6 +430,10 @@ int fbnic_tlv_attr_parse_array(struct fbnic_tlv_msg *attr, int len,
 		u16 attr_len;
 		int err;
 
+		attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len));
+		if (!attr_len || attr_len > len)
+			return -EINVAL;
+
 		if (tlv_attr_id != attr_id)
 			return -EINVAL;
 
@@ -443,7 +447,6 @@ int fbnic_tlv_attr_parse_array(struct fbnic_tlv_msg *attr, int len,
 
 		results[i++] = attr;
 
-		attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len));
 		len -= attr_len;
 		attr += attr_len;
 	}
@@ -476,11 +479,16 @@ int fbnic_tlv_attr_parse(struct fbnic_tlv_msg *attr, int len,
 
 	/* Work through list of attributes, parsing them as necessary */
 	while (len > 0) {
-		int err = fbnic_tlv_attr_validate(attr, tlv_index);
 		u16 attr_id = attr->hdr.type;
 		u16 attr_len;
+		int err;
+
+		attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len));
+		if (!attr_len || attr_len > len)
+			return -EINVAL;
 
 		/* Stop parsing on full error */
+		err = fbnic_tlv_attr_validate(attr, tlv_index);
 		if (err < 0)
 			return err;
 
@@ -493,7 +501,6 @@ int fbnic_tlv_attr_parse(struct fbnic_tlv_msg *attr, int len,
 			results[attr_id] = attr;
 		}
 
-		attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len));
 		len -= attr_len;
 		attr += attr_len;
 	}
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
index e7918d3f6aba9c..b41ff7c88a0a4b 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c
@@ -6,7 +6,9 @@
 #include <linux/bpf_trace.h>
 #include <linux/iopoll.h>
 #include <linux/pci.h>
+#include <linux/time64.h>
 #include <net/netdev_queues.h>
+#include <net/netdev_rx_queue.h>
 #include <net/page_pool/helpers.h>
 #include <net/tcp.h>
 #include <net/xdp.h>
@@ -1044,6 +1046,7 @@ static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd,
 		pkt->add_frag_failed = true;
 		netdev_err_once(nv->napi.dev,
 				"Failed to add fragment to xdp_buff\n");
+		page_pool_put_full_netmem(qt->sub1.page_pool, netmem, true);
 	}
 }
 
@@ -1622,7 +1625,7 @@ fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt,
 	return 0;
 
 err_destroy_sub0:
-	page_pool_destroy(pp);
+	page_pool_destroy(qt->sub0.page_pool);
 	return PTR_ERR(pp);
 }
 
@@ -1791,7 +1794,7 @@ int fbnic_alloc_napi_vectors(struct fbnic_net *fbn)
 	int err;
 
 	/* Allocate 1 Tx queue per napi vector */
-	if (num_napi < FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) {
+	if (num_napi <= FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) {
 		while (num_tx) {
 			err = fbnic_alloc_napi_vector(fbd, fbn,
 						      num_napi, v_idx,
@@ -2667,6 +2670,22 @@ static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 r
 	fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_THRESHOLD, threshold);
 }
 
+void fbnic_config_rx_cqe_nsecs(struct fbnic_dev *fbd)
+{
+	u32 coal_wait;
+
+	coal_wait = DIV_ROUND_CLOSEST_ULL((u64)fbd->rx_cqe_nsecs *
+					  FBNIC_CLOCK_FREQ, NSEC_PER_SEC);
+
+	/* TICK_CYCLES controls the interrupt threshold timer. COAL_WAIT is
+	 * independent and measured in core clock cycles.
+	 */
+	wr32(fbd, FBNIC_QM_RCQ_CTL0,
+	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES,
+			FBNIC_CLOCK_FREQ / USEC_PER_SEC) |
+	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT, coal_wait));
+}
+
 void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm)
 {
 	struct fbnic_net *fbn = netdev_priv(nv->napi.dev);
@@ -2853,6 +2872,17 @@ void fbnic_napi_depletion_check(struct net_device *netdev)
 	fbnic_wrfl(fbd);
 }
 
+/* Returns the napi vector servicing an Rx queue, or NULL if the datapath
+ * is torn down.  The association is published by fbnic_set_netif_napi()
+ * and cleared by fbnic_reset_netif_napi(), both under the instance lock.
+ */
+static struct fbnic_napi_vector *fbnic_rxq_nv(struct net_device *dev, int idx)
+{
+	struct napi_struct *napi = __netif_get_rx_queue(dev, idx)->napi;
+
+	return napi ? container_of(napi, struct fbnic_napi_vector, napi) : NULL;
+}
+
 static int fbnic_queue_mem_alloc(struct net_device *dev,
 				 struct netdev_queue_config *qcfg,
 				 void *qmem, int idx)
@@ -2865,8 +2895,16 @@ static int fbnic_queue_mem_alloc(struct net_device *dev,
 	if (!netif_running(dev))
 		return fbnic_alloc_qt_page_pools(fbn, qt, idx);
 
+	/* A failed PCIe recovery or resume can leave the datapath torn down
+	 * while netif_running() is still true.  This ndo runs before
+	 * netdev_rx_queue_restart() checks netif_running(), so bail out
+	 * rather than touching rings and vectors that are already freed.
+	 */
+	nv = fbnic_rxq_nv(dev, idx);
+	if (!nv)
+		return -ENETDOWN;
+
 	real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl);
-	nv = fbn->napi[idx % fbn->num_napi];
 
 	fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx,
 			real->sub0.flags);
@@ -2917,7 +2955,7 @@ static int fbnic_queue_start(struct net_device *dev,
 	struct fbnic_q_triad *real;
 
 	real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl);
-	nv = fbn->napi[idx % fbn->num_napi];
+	nv = fbnic_rxq_nv(dev, idx);
 
 	fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0);
 	fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1);
@@ -2939,7 +2977,7 @@ static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx)
 	int err;
 
 	real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl);
-	nv = fbn->napi[idx % fbn->num_napi];
+	nv = fbnic_rxq_nv(dev, idx);
 	fbnic_dbg_nv_exit(nv);
 
 	napi_disable_locked(&nv->napi);
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
index f5899446dcc510..66c287a5404a99 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h
@@ -4,12 +4,16 @@
 #ifndef _FBNIC_TXRX_H_
 #define _FBNIC_TXRX_H_
 
+#include <linux/bitfield.h>
 #include <linux/netdevice.h>
 #include <linux/skbuff.h>
+#include <linux/time64.h>
 #include <linux/types.h>
 #include <linux/u64_stats_sync.h>
 #include <net/xdp.h>
 
+#include "fbnic_csr.h"
+
 struct fbnic_net;
 
 /* Guarantee we have space needed for storing the buffer
@@ -49,6 +53,12 @@ struct fbnic_net;
 #define FBNIC_RX_USECS_DEFAULT		30
 #define FBNIC_RX_FRAMES_DEFAULT		0
 
+#define FBNIC_RX_CQE_NSECS_MIN		1000
+#define FBNIC_RX_CQE_NSECS_DEFAULT	2000
+#define FBNIC_RX_CQE_NSECS_MAX \
+	((u32)(((u64)FIELD_MAX(FBNIC_QM_RCQ_CTL0_COAL_WAIT) * NSEC_PER_SEC) / \
+	       FBNIC_CLOCK_FREQ))
+
 #define FBNIC_RX_TROOM \
 	SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
 #define FBNIC_RX_HROOM_PAD		128
diff --git a/drivers/net/ethernet/meta/mpnic/Makefile b/drivers/net/ethernet/meta/mpnic/Makefile
new file mode 100644
index 00000000000000..d5bdbd5bd1c7b2
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/Makefile
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+
+#
+# Makefile for the Meta(R) Platforms Network Interface Controller
+#
+
+obj-$(CONFIG_MPNIC) += mpnic.o
+
+mpnic-y := \
+	mpnic_init.o \
+	mpnic_irq.o \
+	mpnic_netdev.o \
+	mpnic_pci.o \
+	mpnic_txrx.o \
+# End of mpnic-y
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic.h b/drivers/net/ethernet/meta/mpnic/mpnic.h
new file mode 100644
index 00000000000000..78359ab6abb122
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#ifndef _MPNIC_H_
+#define _MPNIC_H_
+
+#include <linux/interrupt.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/types.h>
+
+#include "mpnic_csr.h"
+
+#define MPNIC_DRV_NAME		"mpnic"
+
+#define MPNIC_MAX_TXQS		1024u
+#define MPNIC_MAX_RXQS		1024u
+
+/* misc IRQ entries are allocated before the completion queue IRQs */
+enum {
+	MPNIC_FW_MSIX_ENTRY,
+	MPNIC_NON_NAPI_VECTORS
+};
+
+struct mpnic_dev {
+	struct device *dev;
+	struct net_device *netdev;
+
+	u32 __iomem *uc_addr0;
+
+	u16 num_irqs;
+
+	u64 dsn;
+	u32 mps;
+	u32 readrq;
+	u8 relaxed_ord;
+};
+
+u64 mpnic_rd64(struct mpnic_dev *mpd, u32 reg);
+
+int mpnic_dev_init(struct mpnic_dev *mpd);
+
+int mpnic_request_irq(struct mpnic_dev *mpd, int nr, irq_handler_t handler,
+		      unsigned long flags, const char *name, void *data);
+void mpnic_free_irq(struct mpnic_dev *mpd, int nr, void *data);
+void mpnic_free_irqs(struct mpnic_dev *mpd);
+int mpnic_alloc_irqs(struct mpnic_dev *mpd);
+
+static inline void mpnic_wr64(struct mpnic_dev *mpd, u32 reg, u64 val)
+{
+	u32 __iomem *csr = READ_ONCE(mpd->uc_addr0);
+
+	if (csr)
+		writeq(val, csr + reg);
+}
+
+static inline void mpnic_wrfl(struct mpnic_dev *mpd)
+{
+	mpnic_rd64(mpd, MPNIC_BDQ_SPARE);
+}
+
+static inline bool mpnic_present(struct mpnic_dev *mpd)
+{
+	return !!READ_ONCE(mpd->uc_addr0);
+}
+
+#endif /* _MPNIC_H_ */
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_csr.h b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h
new file mode 100644
index 00000000000000..d6ab9df5816838
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_csr.h
@@ -0,0 +1,319 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#ifndef _MPNIC_CSR_H_
+#define _MPNIC_CSR_H_
+
+#include <linux/bits.h>
+
+#define CSR_BIT(nr)		BIT_ULL(nr)
+#define CSR_GENMASK(h, l)	GENMASK_ULL(h, l)
+
+#define DESC_BIT(nr)		BIT_ULL(nr)
+#define DESC_GENMASK(h, l)	GENMASK_ULL(h, l)
+
+/* Transmit Work Descriptor Format */
+#define MPNIC_TWD_FLAG_REQ_COMPLETION	DESC_BIT(37)
+#define MPNIC_TWD_FLAG_DEST_MAC		DESC_BIT(43)
+#define MPNIC_TWD_TYPE			DESC_GENMASK(47, 46)
+enum {
+	MPNIC_TWD_TYPE_META	= 0,
+	MPNIC_TWD_TYPE_AL	= 2,
+	MPNIC_TWD_TYPE_LAST_AL	= 3,
+};
+
+#define MPNIC_TWD_ADDR			DESC_GENMASK(45, 0)
+#define MPNIC_TWD_LEN			DESC_GENMASK(63, 48)
+
+/* Tx Completion Descriptor Format */
+#define MPNIC_TCD_TYPE0_HEAD0		DESC_GENMASK(15, 0)
+#define MPNIC_TCD_DONE			DESC_BIT(63)
+
+/* Rx Buffer Descriptor Format */
+#define MPNIC_BD_DESC_ADDR		DESC_GENMASK(39, 2)
+#define MPNIC_BD_DESC_ID		DESC_GENMASK(57, 40)
+#define MPNIC_BD_DESC_BUF_SZ_LOG2	DESC_GENMASK(62, 58)
+
+/* Rx Completion Queue Descriptors */
+#define MPNIC_RCD_TYPE			DESC_GENMASK(62, 61)
+enum {
+	MPNIC_RCD_TYPE_HDR_AL	= 0,
+	MPNIC_RCD_TYPE_PAY_AL	= 1,
+	MPNIC_RCD_TYPE_META	= 3,
+};
+
+#define MPNIC_RCD_DONE			DESC_BIT(63)
+
+#define MPNIC_RCD_HDR_SUBTYPE		DESC_GENMASK(60, 59)
+enum {
+	MPNIC_RCD_HDR_SUBTYPE_HDR	= 2,
+};
+
+/* Address/Length Completion Descriptors */
+#define MPNIC_RCD_AL_BUFF_OFF		DESC_GENMASK(15, 0)
+#define MPNIC_RCD_AL_BUFF_ID		DESC_GENMASK(33, 16)
+#define MPNIC_RCD_AL_BUFF_LEN		DESC_GENMASK(47, 34)
+#define MPNIC_RCD_AL_PAGE_FIN		DESC_BIT(53)
+
+/* Metadata Completion Descriptors */
+#define MPNIC_RCD_META_ERR_MAC_EOP		DESC_BIT(53)
+#define MPNIC_RCD_META_ERR_TRUNCATED_FRAME	DESC_BIT(54)
+#define MPNIC_RCD_META_UNCORRECTABLE_ERR_MASK	\
+	(MPNIC_RCD_META_ERR_MAC_EOP | MPNIC_RCD_META_ERR_TRUNCATED_FRAME)
+
+/* Common fields for all DESC_CFG CSRs */
+#define MPNIC_DESC_CFG_NUM_DESCS	CSR_GENMASK(2, 0)
+#define MPNIC_DESC_CFG_START_ADDR	CSR_GENMASK(19, 8)
+
+/* Register Definitions
+ *
+ * The register file is addressed as an array of le32, so the byte address of
+ * a register is 4 times the index below. Each register is listed with its
+ * name, index and byte address.
+ *
+ *	Name				Index			Address
+ *****************************************************************************/
+
+/* NIC_CORE_TDF */
+#define MPNIC_TWQ_CTL(i, j)		(0x0 + 1024 * (i) + 2 * (j))
+								/* 0x0 */
+#define MPNIC_TWQ_CTL_RESET			CSR_BIT(0)
+#define MPNIC_TWQ_CTL_ENABLE			CSR_BIT(1)
+#define MPNIC_TWQ_TAIL(i, j)		(0x4 + 1024 * (i) + 2 * (j))
+								/* 0x10 */
+#define MPNIC_TWQ_SIZE(i, j)		(0x10 + 1024 * (i) + 2 * (j))
+								/* 0x40 */
+#define MPNIC_TWQ_SIZE_SIZE			CSR_GENMASK(3, 0)
+#define MPNIC_TWQ_BASE_ADDR(i, j)	(0x1c + 1024 * (i) + 2 * (j))
+								/* 0x70 */
+
+/* NIC_CORE_TCM */
+#define MPNIC_TCQ_CTL(i)		(0x80 + 1024 * (i))	/* 0x200 */
+#define MPNIC_TCQ_CTL_RESET			CSR_BIT(0)
+#define MPNIC_TCQ_CTL_ENABLE			CSR_BIT(1)
+#define MPNIC_TCQ_BASE_ADDR(i)		(0x86 + 1024 * (i))	/* 0x218 */
+#define MPNIC_TCQ_HEAD(i)		(0x8e + 1024 * (i))	/* 0x238 */
+#define MPNIC_TCQ_SIZE(i)		(0x94 + 1024 * (i))	/* 0x250 */
+#define MPNIC_TCQ_SIZE_SIZE			CSR_GENMASK(4, 0)
+
+/* NIC_CORE_TIM */
+#define MPNIC_TIM_CTL1(i)		(0xc0 + 1024 * (i))	/* 0x300 */
+#define MPNIC_TIM_CTL1_UPD_IGN_LONG_EVENT_CNT	CSR_BIT(48)
+#define MPNIC_TIM_CTL1_UPD_IGN_LONG_TIME_CNT	CSR_BIT(49)
+#define MPNIC_TIM_CTL1_UPD_IGN_SHORT_TIME_CNT	CSR_BIT(50)
+#define MPNIC_TIM_CTL1_MASK			CSR_BIT(51)
+#define MPNIC_TIM_CTL1_MASK_EN			CSR_BIT(52)
+#define MPNIC_TIM_CTL1_TRIGGER			CSR_BIT(53)
+#define MPNIC_TIM_INTR_MASK(i)		(0xc8 + 1024 * (i))	/* 0x320 */
+#define MPNIC_TIM_INTR_MASK_MASK		CSR_BIT(0)
+
+/* NIC_CORE_RBP */
+#define MPNIC_BDQ_CTL(i)		(0x200 + 1024 * (i))	/* 0x800 */
+#define MPNIC_BDQ_CTL_RESET			CSR_BIT(0)
+#define MPNIC_BDQ_CTL_ENABLE			CSR_BIT(1)
+#define MPNIC_BDQ_CTL_ENABLE_PPQ		CSR_BIT(3)
+#define MPNIC_HPQ_TAIL(i)		(0x202 + 1024 * (i))	/* 0x808 */
+#define MPNIC_PPQ_TAIL(i)		(0x204 + 1024 * (i))	/* 0x810 */
+#define MPNIC_HPQ_SIZE(i)		(0x20a + 1024 * (i))	/* 0x828 */
+#define MPNIC_HPQ_SIZE_SIZE			CSR_GENMASK(4, 0)
+#define MPNIC_PPQ_SIZE(i)		(0x20c + 1024 * (i))	/* 0x830 */
+#define MPNIC_PPQ_SIZE_SIZE			CSR_GENMASK(4, 0)
+#define MPNIC_HPQ_BASE_ADDR(i)		(0x216 + 1024 * (i))	/* 0x858 */
+#define MPNIC_PPQ_BASE_ADDR(i)		(0x218 + 1024 * (i))	/* 0x860 */
+
+/* NIC_CORE_RCM */
+#define MPNIC_RCQ_CTL(i)		(0x280 + 1024 * (i))	/* 0xa00 */
+#define MPNIC_RCQ_CTL_RESET			CSR_BIT(0)
+#define MPNIC_RCQ_CTL_ENABLE			CSR_BIT(1)
+#define MPNIC_RCQ_BASE_ADDR(i)		(0x286 + 1024 * (i))	/* 0xa18 */
+#define MPNIC_RCQ_HEAD(i)		(0x28e + 1024 * (i))	/* 0xa38 */
+#define MPNIC_RCQ_SIZE(i)		(0x294 + 1024 * (i))	/* 0xa50 */
+#define MPNIC_RCQ_SIZE_SIZE			CSR_GENMASK(4, 0)
+
+/* NIC_CORE_RIM */
+#define MPNIC_RIM_INTR_MASK(i)		(0x2c8 + 1024 * (i))	/* 0xb20 */
+#define MPNIC_RIM_INTR_MASK_MASK		CSR_BIT(0)
+
+/* NIC_CORE_TIM_PRV */
+#define MPNIC_TIM_CTL(i)		(0x100100 + 1024 * (i))	/* 0x400400 */
+
+/* NIC_CORE_RDE */
+#define MPNIC_RDE_CFG(i)		(0x10021c + 1024 * (i))	/* 0x400870 */
+#define MPNIC_RDE_CFG_MIN_TAIL_ROOM		CSR_GENMASK(9, 0)
+#define MPNIC_RDE_CFG_MIN_HEAD_ROOM		CSR_GENMASK(18, 10)
+#define MPNIC_RDE_CFG_MAX_HEADER_BYTES		CSR_GENMASK(45, 32)
+
+/* NIC_CORE_RIM_PRV */
+#define MPNIC_RIM_CTL(i)		(0x100280 + 1024 * (i))	/* 0x400a00 */
+
+/* NIC_CORE_RBP_HP_GLBL */
+#define MPNIC_HPQ_IDLE(i)		(0x420000 + 2 * (i))	/* 0x1080000 */
+#define MPNIC_HPQ_IDLE_CNT		16
+#define MPNIC_PPQ_IDLE(i)		(0x420060 + 2 * (i))	/* 0x1080180 */
+#define MPNIC_PPQ_IDLE_CNT		16
+#define MPNIC_BDQ_GLBL_CTL0		0x420080		/* 0x1080200 */
+#define MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE	CSR_GENMASK(26, 18)
+#define MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH \
+						CSR_GENMASK(42, 32)
+#define MPNIC_RDE_CTL			0x420082		/* 0x1080208 */
+#define MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD	CSR_GENMASK(10, 0)
+#define MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD	CSR_GENMASK(21, 11)
+#define MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD	CSR_GENMASK(42, 32)
+#define MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD	CSR_GENMASK(53, 43)
+#define MPNIC_BDQ_MEM_INIT_REQ		0x42013a		/* 0x10804e8 */
+#define MPNIC_BDQ_MEM_INIT_DONE		0x42013c		/* 0x10804f0 */
+#define MPNIC_BDQ_SPARE			0x42013e		/* 0x10804f8 */
+#define MPNIC_HPQ_DESC_CFG(i)		(0x420140 + 2 * (i))	/* 0x1080500 */
+#define MPNIC_PPQ_DESC_CFG(i)		(0x420940 + 2 * (i))	/* 0x1082500 */
+
+/* NIC_CORE_RDE_GLBL */
+#define MPNIC_RDE_MEM_INIT_REQ		0x4240e6		/* 0x1090398 */
+#define MPNIC_RDE_MEM_INIT_DONE		0x4240e8		/* 0x10903a0 */
+
+/* NIC_CORE_RCM_GLBL */
+#define MPNIC_RCQ_IDLE(i)		(0x42505e + 2 * (i))	/* 0x1094178 */
+#define MPNIC_RCQ_IDLE_CNT		16
+#define MPNIC_RCM_MEM_INIT_REQ		0x42507e		/* 0x10941f8 */
+#define MPNIC_RCM_MEM_INIT_DONE		0x425080		/* 0x1094200 */
+
+/* NIC_CORE_RNI_GLBL */
+#define MPNIC_RNI_RBP_CTL		0x427000		/* 0x109c000 */
+#define MPNIC_RNI_RDE_CTL		0x427002		/* 0x109c008 */
+#define MPNIC_RNI_RDE_CTL_MPS			CSR_GENMASK(1, 0)
+#define MPNIC_RNI_RDE_CTL_CLS			CSR_GENMASK(3, 2)
+#define MPNIC_RNI_RCM_CTL		0x427004		/* 0x109c010 */
+
+/* NIC_CORE_TDF_GLBL */
+#define MPNIC_TWQ_IDLE(i)		(0x428042 + 2 * (i))	/* 0x10a0108 */
+#define MPNIC_TWQ_IDLE_CNT		32
+#define MPNIC_TWQ_DEF_PRI_TWD		0x428082		/* 0x10a0208 */
+#define MPNIC_TDF_MEM_INIT_REQ		0x42813a		/* 0x10a04e8 */
+#define MPNIC_TDF_MEM_INIT_DONE		0x42813c		/* 0x10a04f0 */
+#define MPNIC_TDF_DESC_CFG(i)		(0x428140 + 2 * (i))	/* 0x10a0500 */
+
+/* NIC_CORE_TQS_GLBL */
+#define MPNIC_TQS_GLBL_CTL0		0x42a000		/* 0x10a8000 */
+#define MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN	CSR_BIT(2)
+#define MPNIC_TQS_GLBL_P0_0		0x42a002		/* 0x10a8008 */
+#define MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0	CSR_GENMASK(63, 48)
+#define MPNIC_TQS_GLBL_P0_1		0x42a004		/* 0x10a8010 */
+#define MPNIC_TQS_GLBL_BMC		0x42a012		/* 0x10a8048 */
+#define MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS	CSR_GENMASK(15, 0)
+#define MPNIC_TQS_SLOWDOWN_CTL		0x42a026		/* 0x10a8098 */
+#define MPNIC_TQS_SLOWDOWN_CTL_ENABLE		CSR_BIT(6)
+#define MPNIC_TQS_MTU_CTL0		0x42a030		/* 0x10a80c0 */
+#define MPNIC_TQS_MTU_CTL1		0x42a032		/* 0x10a80c8 */
+#define MPNIC_TQS_IDLE(i)		(0x42a040 + 2 * (i))	/* 0x10a8100 */
+#define MPNIC_TQS_IDLE_CNT		32
+#define MPNIC_TQS_SET_P0_MAP0(i)	(0x42a082 + 2 * (i))	/* 0x10a8208 */
+#define MPNIC_TQS_SET_P0_MAP1(i)	(0x42a092 + 2 * (i))	/* 0x10a8248 */
+#define MPNIC_TQS_GLBL_SHAPING		0x42a108		/* 0x10a8420 */
+#define MPNIC_TQS_GLBL_SHAPING_DISABLE		CSR_BIT(0)
+#define MPNIC_TQS_ARB_CTL		0x42a122		/* 0x10a8488 */
+#define MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN	CSR_BIT(9)
+#define MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN	CSR_BIT(8)
+#define MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN	CSR_BIT(5)
+#define MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN	CSR_BIT(4)
+#define MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN	CSR_BIT(1)
+#define MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN	CSR_BIT(0)
+#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0 \
+					0x42a124		/* 0x10a8490 */
+#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE	CSR_GENMASK(19, 0)
+#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP	CSR_GENMASK(59, 32)
+#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1 \
+					0x42a126		/* 0x10a8498 */
+#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET	CSR_GENMASK(27, 0)
+#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT	CSR_GENMASK(63, 32)
+#define MPNIC_TQS_SRAM_INIT_CTL		0x42a128		/* 0x10a84a0 */
+#define MPNIC_TQS_SRAM_INIT_CTL_QUANTUM		CSR_GENMASK(31, 20)
+#define MPNIC_TQS_SRAM_INIT_CTL_INIT		CSR_BIT(32)
+#define MPNIC_TQS_GROUP_INIT_CTL	0x42a12a		/* 0x10a84a8 */
+#define MPNIC_TQS_GROUP_INIT_CTL_QUANTUM	CSR_GENMASK(51, 32)
+#define MPNIC_TQS_GROUP_INIT_CTL_INIT		CSR_BIT(52)
+#define MPNIC_TQS_SET_INIT_CTL		0x42a12c		/* 0x10a84b0 */
+#define MPNIC_TQS_SET_INIT_CTL_QUANTUM		CSR_GENMASK(51, 32)
+#define MPNIC_TQS_SET_INIT_CTL_INIT		CSR_BIT(52)
+#define MPNIC_TQS_PORT_INIT_CTL		0x42a12e		/* 0x10a84b8 */
+#define MPNIC_TQS_PORT_INIT_CTL_QUANTUM		CSR_GENMASK(55, 32)
+#define MPNIC_TQS_PORT_INIT_CTL_INIT		CSR_BIT(56)
+#define MPNIC_TQS_SRAM_STS		0x42a130		/* 0x10a84c0 */
+#define MPNIC_TQS_PORT_CTL(i)		(0x42a1e4 + 2 * (i))	/* 0x10a8790 */
+
+/* NIC_CORE_TDE_GLBL */
+#define MPNIC_TDE_IDLE(i)		(0x42b000 + 2 * (i))	/* 0x10ac000 */
+#define MPNIC_TDE_IDLE_CNT		32
+#define MPNIC_TDE_MEM_INIT_REQ		0x42b1ee		/* 0x10ac7b8 */
+#define MPNIC_TDE_MEM_INIT_DONE		0x42b1f0		/* 0x10ac7c0 */
+
+/* NIC_CORE_TCM_GLBL */
+#define MPNIC_TCQ_IDLE(i)		(0x42c09e + 2 * (i))	/* 0x10b0278 */
+#define MPNIC_TCQ_IDLE_CNT		16
+#define MPNIC_TCM_MEM_INIT_REQ		0x42c0be		/* 0x10b02f8 */
+#define MPNIC_TCM_MEM_INIT_DONE		0x42c0c0		/* 0x10b0300 */
+
+/* NIC_CORE_TNI_GLBL */
+#define MPNIC_TNI_GLBL_TDF_CTL		0x42e000		/* 0x10b8000 */
+#define MPNIC_TNI_GLBL_TDF_CTL_MRRS		CSR_GENMASK(2, 0)
+#define MPNIC_TNI_GLBL_TDF_CTL_CLS		CSR_GENMASK(5, 3)
+#define MPNIC_TNI_GLBL_TDE_CTL		0x42e002		/* 0x10b8008 */
+#define MPNIC_TNI_GLBL_TCM_CTL		0x42e004		/* 0x10b8010 */
+
+/* NIC_CORE_TXB */
+#define MPNIC_TXB_PORT_CONFIG		0x600000		/* 0x1800000 */
+#define MPNIC_TXB_PORT_CONFIG_PORT_MODE		CSR_GENMASK(15, 13)
+#define MPNIC_TXB_BMC			0x600122		/* 0x1800488 */
+#define MPNIC_TXB_P0(i)			(0x600124 + 2 * (i))	/* 0x1800490 */
+#define MPNIC_TXB_P0_CNT		17
+#define MPNIC_TXB_P0_THRESH(i)		(0x60025e + 2 * (i))	/* 0x1800978 */
+#define MPNIC_TXB_P0_ARB_WEIGHTS(i)	(0x600378 + 2 * (i))	/* 0x1800de0 */
+
+/* NIC_CORE_RXB */
+#define MPNIC_RXB_MEM_INIT_REQ		0x620002		/* 0x1880008 */
+#define MPNIC_RXB_MEM_INIT_DONE		0x620004		/* 0x1880010 */
+#define MPNIC_RXB_PORT_CFG(i)		(0x620006 + 2 * (i))	/* 0x1880018 */
+#define MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE	CSR_GENMASK(22, 22)
+enum {
+	MPNIC_FCS_MODE_KEEP	= 0,
+	MPNIC_FCS_MODE_STRIP	= 1,
+};
+
+#define MPNIC_RXB_PORT_CLASS_CFG(i)	(0x620020 + 2 * (i))	/* 0x1880080 */
+#define MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION \
+						CSR_GENMASK(0, 0)
+enum {
+	MPNIC_L2_ACTION_DROP	= 0,
+	MPNIC_L2_ACTION_PASS	= 1,
+};
+
+#define MPNIC_RXB_TC_CRDTS(i)		(0x620418 + 2 * (i))	/* 0x1881060 */
+#define MPNIC_RXB_POOL_COMMON_CRDTS(i)	(0x620458 + 2 * (i))	/* 0x1881160 */
+#define MPNIC_RXB_COMMON_CRDT_CTRL_TC(i) \
+					(0x620472 + 2 * (i))	/* 0x18811c8 */
+#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH	CSR_GENMASK(15, 0)
+#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN	CSR_BIT(29)
+#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS	CSR_GENMASK(47, 32)
+#define MPNIC_RXB_HOST_DROP_THRESH(i)	(0x6205b0 + 2 * (i))	/* 0x18816c0 */
+#define MPNIC_RXB_BMC_CRDTS(i)		(0x6205d0 + 2 * (i))	/* 0x1881740 */
+
+/* NIC_CORE_RPC */
+#define MPNIC_RPC_MEM_INIT_REQ		0x780440		/* 0x1e01100 */
+#define MPNIC_RPC_MEM_INIT_DONE		0x780442		/* 0x1e01108 */
+
+/* NIC_CORE_ROF */
+#define MPNIC_RSC_GLOBAL_CONF		0x7e2002		/* 0x1f88008 */
+#define MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE	CSR_BIT(0)
+
+/* NIC_CORE_TOF */
+#define MPNIC_TOF_TCAM_DEST_REMAP	0x7e3022		/* 0x1f8c088 */
+
+/* PEMO_WRAPPER */
+#define MPNIC_OB_ATTR_RO			CSR_BIT(1)
+#define MPNIC_OB_ATTR_TDE_H		0x9a000e		/* 0x2680038 */
+#define MPNIC_OB_ATTR_TDE_P		0x9a0010		/* 0x2680040 */
+#define MPNIC_OB_ATTR_TDF		0x9a0012		/* 0x2680048 */
+#define MPNIC_OB_ATTR_RBP_HPQ		0x9a0014		/* 0x2680050 */
+#define MPNIC_OB_ATTR_RBP_PPQ		0x9a0016		/* 0x2680058 */
+#define MPNIC_OB_ATTR_RDE_H		0x9a0018		/* 0x2680060 */
+#define MPNIC_OB_ATTR_RDE_P		0x9a001a		/* 0x2680068 */
+
+#endif /* _MPNIC_CSR_H_ */
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_init.c b/drivers/net/ethernet/meta/mpnic/mpnic_init.c
new file mode 100644
index 00000000000000..2daed458f5fc27
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_init.c
@@ -0,0 +1,556 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cache.h>
+#include <linux/if_ether.h>
+#include <linux/iopoll.h>
+#include <linux/log2.h>
+#include <linux/sizes.h>
+
+#include "mpnic.h"
+
+#define MPNIC_MEM_INIT_POLL_US		500
+#define MPNIC_MEM_INIT_TO_US		5000
+
+/* BDQ mem init:
+ *  bit 1: fifo_wrptr_mem
+ *  bit 0: fifo_rdptr_mem
+ */
+#define MPNIC_MEM_INIT_BDQ_VAL		0x3
+
+/* RCM mem init:
+ *  bit 3: cq_base_addr
+ *  bit 2: cd_fifo_rptr_stats
+ *  bit 1: cd_fifo_wptr_stats
+ *  bit 0: cq_head_ptr_stats
+ */
+#define MPNIC_MEM_INIT_RCM_VAL		0xf
+
+/* RDE mem init, bits 0-18. Bits 0-12 are the per-queue packet, error and
+ * drop counters, bits 13-18 the two context memories of each of the HPQ,
+ * PPQ and SPQ descriptor prefetchers.
+ */
+#define MPNIC_MEM_INIT_RDE_VAL		0x7ffff
+
+/* RPC mem init, bit 0 covers the whole classifier. */
+#define MPNIC_MEM_INIT_RPC_VAL		0x1
+
+/* TCM mem init:
+ *  bit 3: cq_head_ptr_stats
+ *  bit 2: cd_fifo_wptr_stats
+ *  bit 1: cd_fifo_rptr_stats
+ *  bit 0: cq_base_addr
+ */
+#define MPNIC_MEM_INIT_TCM_VAL		0xf
+
+/* TDE mem init:
+ *  bit 1: stats mem
+ *  bit 0: dma_head_ptr SRAM
+ */
+#define MPNIC_MEM_INIT_TDE_VAL		0x3
+
+/* TDF mem init, bit 0 covers the descriptor fetch SRAM. */
+#define MPNIC_MEM_INIT_TDF_VAL		0x1
+
+/* RXB mem init, bit 0 covers the DMAC TCAM statistics RAM. */
+#define MPNIC_MEM_INIT_RXB_VAL		0x1
+
+/* TQS arbiter SRAM init done, one bit per DWRR level. */
+#define MPNIC_TQS_ARB_INIT_VAL		0xf
+
+/* On-chip SRAM allocated to each queue for descriptor fetch, in units of
+ * descriptors. Valid values are 64, 128, 256, 512 and 1024.
+ *
+ * The partition sizes are chosen for the maximum number of queues the
+ * device supports, so they do not have to be adjusted when the active
+ * queue count changes:
+ *
+ *   BDQ: 1 MiB / (8 B/DESC) / (1024 HPQ + 1024 PPQ) =  64 DESC/QUEUE
+ *   TWQ: 2 MiB / (8 B/DESC) / (1024 TXQ * 2 TWQ)    = 128 DESC/QUEUE
+ */
+#define MPNIC_BDQ_SRAM_DESCS		64u
+#define MPNIC_TDF_SRAM_DESCS		128u
+
+/* A total of 1 MiB worth of Tx credits is available, in units of 128 B.
+ * The BMC to MAC and host to BMC paths each get a guaranteed share of them
+ * whether or not anything is routed their way, everything else goes to
+ * MAC TC0.
+ */
+#define MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL		800
+#define MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL	\
+	(SZ_1M / 128 - 2 * MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL)
+
+/* The recommended lower bound for the TXB threshold is 80, based on a 10K
+ * MTU. Round up by 25% to stay on the defensive side. The same reasoning
+ * applies to the arbitration weight, which has to exceed the full packet
+ * size.
+ */
+#define MPNIC_TXB_INIT_BMC_THRESH		100
+#define MPNIC_TXB_INIT_P0_THRESH		100
+#define MPNIC_TXB_INIT_P0_ARB_WEIGHTS		0x64
+
+/* RXB host drop threshold in units of 128 B beats. Packets targeting a
+ * queue are dropped when the available credits fall below it. 80 beats is
+ * 10 KB, which is also the largest frame the device is configured for.
+ */
+#define MPNIC_RXB_INIT_HOST_DROP_THRESH		0x50
+
+/* A total of 8 MiB of Rx buffer is available. The recommended per-TC pool
+ * for a 800G configuration is 420 KB with all 8 TCs enabled. Only one TC
+ * is in use, so give it 8 * 420 KB (in units of 128 B) and push the rest
+ * to the common pool. Start drawing from the common pool as soon as the
+ * TC0 credits fall below one max sized frame. The BMC keeps a small
+ * reserve of its own whether or not the host talks to it.
+ */
+#define MPNIC_RXB_INIT_POOL_TC_CRDTS_P0		(0xd20 * 8)
+#define MPNIC_RXB_INIT_BMC_CRDTS		0x20
+#define MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH	\
+	(SZ_8M / 128 - MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 - \
+	 MPNIC_RXB_INIT_BMC_CRDTS)
+#define MPNIC_RXB_INIT_COMMON_CRDT_THRSH	0x50
+
+/* TXB port mode selects the number of active MAC ports for Tx buffer
+ * credit distribution and arbitration. The hardware only supports single,
+ * dual and quad port, encoded as b'001, b'010 and b'100.
+ */
+#define MPNIC_TXB_PORT_MODE_SINGLE	1
+
+/* The MAC traffic classes start at index 8 in the TXB arrays, the BMC
+ * sits above them.
+ */
+#define MPNIC_TXB_TC_IDX_MAC_0		8
+#define MPNIC_TXB_TC_IDX_BMC		16
+
+/* Largest frame the Tx queue scheduler will pass through. Anything above
+ * it gets truncated.
+ */
+#define MPNIC_TQS_MTU_CTL0_MAX		0x2800
+
+/* The unit of the DWRR quantum is 256 B. It has to be large enough for at
+ * least 11 MTUs to be transmitted in one quantum; use 15 for headroom.
+ */
+#define MPNIC_TQS_DWRR_INIT_QUANTUM	(15 * MPNIC_TQS_MTU_CTL0_MAX / 256)
+
+/* Lower bound on the credit available to a queue, group, set or port
+ * before the scheduler stops issuing requests for it. 15 MTUs, matching
+ * the DWRR quantum above.
+ */
+#define MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH	(15 * MPNIC_TQS_MTU_CTL0_MAX)
+
+/* The TQS view of the TXB partitioning between the MAC and the BMC, in
+ * units of 1 KiB. It has to match the TXB private credits above.
+ */
+#define MPNIC_TQS_GLBL_TXB_CRDT_UNIT	SZ_1K
+#define MPNIC_TQS_GLBL_TXB_CRDT_BMC	\
+	(MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL * 128 / MPNIC_TQS_GLBL_TXB_CRDT_UNIT)
+#define MPNIC_TQS_GLBL_TXB_CRDT_MAC	\
+	(MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL * 128 / \
+	 MPNIC_TQS_GLBL_TXB_CRDT_UNIT)
+
+struct mpnic_init_poll {
+	u64 exp_val;
+	u32 addr;
+};
+
+struct mpnic_poll_state {
+	int poll_idx;
+	u64 val;
+};
+
+static void mpnic_tdf_glbl_init(struct mpnic_dev *mpd)
+{
+	/* Default metadata descriptor, used for frames the driver did not
+	 * prepend one to.
+	 */
+	mpnic_wr64(mpd, MPNIC_TWQ_DEF_PRI_TWD, MPNIC_TWD_FLAG_REQ_COMPLETION);
+}
+
+static void mpnic_txb_init(struct mpnic_dev *mpd)
+{
+	int i;
+
+	mpnic_wr64(mpd, MPNIC_TXB_BMC, MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL);
+
+	/* Zero the private credits of every traffic class, then hand the
+	 * unreserved ones to MAC TC0.
+	 */
+	for (i = 0; i < MPNIC_TXB_P0_CNT; i++)
+		mpnic_wr64(mpd, MPNIC_TXB_P0(i), 0);
+	mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_MAC_0),
+		   MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL);
+	mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_BMC),
+		   MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL);
+
+	mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_BMC),
+		   MPNIC_TXB_INIT_BMC_THRESH);
+
+	mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_MAC_0),
+		   MPNIC_TXB_INIT_P0_THRESH);
+	mpnic_wr64(mpd, MPNIC_TXB_P0_ARB_WEIGHTS(MPNIC_TXB_TC_IDX_MAC_0),
+		   MPNIC_TXB_INIT_P0_ARB_WEIGHTS);
+	mpnic_wr64(mpd, MPNIC_TXB_PORT_CONFIG,
+		   FIELD_PREP(MPNIC_TXB_PORT_CONFIG_PORT_MODE,
+			      MPNIC_TXB_PORT_MODE_SINGLE));
+}
+
+static void mpnic_rxb_init(struct mpnic_dev *mpd)
+{
+	/* Accept all packets that miss dmac tcam, until l2 filtering is
+	 * implemented.
+	 */
+	mpnic_wr64(mpd, MPNIC_RXB_PORT_CLASS_CFG(0),
+		   FIELD_PREP(MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION,
+			      MPNIC_L2_ACTION_PASS));
+	mpnic_wr64(mpd, MPNIC_RXB_PORT_CFG(0),
+		   FIELD_PREP(MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE,
+			      MPNIC_FCS_MODE_STRIP));
+
+	mpnic_wr64(mpd, MPNIC_RXB_HOST_DROP_THRESH(0),
+		   MPNIC_RXB_INIT_HOST_DROP_THRESH);
+
+	mpnic_wr64(mpd, MPNIC_RXB_TC_CRDTS(0),
+		   MPNIC_RXB_INIT_POOL_TC_CRDTS_P0);
+	mpnic_wr64(mpd, MPNIC_RXB_COMMON_CRDT_CTRL_TC(0),
+		   FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH,
+			      MPNIC_RXB_INIT_COMMON_CRDT_THRSH) |
+		   FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS,
+			      MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH) |
+		   MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN);
+
+	/* Only pool 0 is used, it gets all of the common credits */
+	mpnic_wr64(mpd, MPNIC_RXB_POOL_COMMON_CRDTS(0),
+		   MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH);
+
+	mpnic_wr64(mpd, MPNIC_RXB_BMC_CRDTS(0), MPNIC_RXB_INIT_BMC_CRDTS);
+
+	mpnic_wr64(mpd, MPNIC_RXB_MEM_INIT_REQ, MPNIC_MEM_INIT_RXB_VAL);
+}
+
+static u64 mpnic_desc_cfg(unsigned int sram_descs, unsigned int q_idx)
+{
+	/* The hardware encodes the partition size as 64 * 2^n descriptors,
+	 * and its start address in units of 64 descriptors.
+	 */
+	return FIELD_PREP(MPNIC_DESC_CFG_NUM_DESCS, __ffs(sram_descs) - 6) |
+	       FIELD_PREP(MPNIC_DESC_CFG_START_ADDR,
+			  q_idx * (sram_descs >> 6));
+}
+
+static void mpnic_desc_sram_init(struct mpnic_dev *mpd)
+{
+	int i;
+
+	for (i = 0; i < MPNIC_MAX_TXQS * 2; i++)
+		mpnic_wr64(mpd, MPNIC_TDF_DESC_CFG(i),
+			   mpnic_desc_cfg(MPNIC_TDF_SRAM_DESCS, i));
+
+	for (i = 0; i < MPNIC_MAX_RXQS; i++) {
+		mpnic_wr64(mpd, MPNIC_HPQ_DESC_CFG(i),
+			   mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS, i));
+		mpnic_wr64(mpd, MPNIC_PPQ_DESC_CFG(i),
+			   mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS,
+					  MPNIC_MAX_RXQS + i));
+	}
+}
+
+static void mpnic_rxglb_init(struct mpnic_dev *mpd)
+{
+	/* Descriptor prefetch reads are only issued once 32 descriptors
+	 * worth of FIFO space is available, and no more than 64 descriptors
+	 * are fetched for one queue at a time so that a single queue cannot
+	 * monopolize the bus. Both have to be multiples of 16 to keep the
+	 * reads 128 B aligned.
+	 */
+	mpnic_wr64(mpd, MPNIC_BDQ_GLBL_CTL0,
+		   FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH, 32) |
+		   FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE, 64));
+
+	/* Minimum number of descriptors which has to be available before
+	 * the descriptor engine considers a queue usable, globally and in
+	 * the per-queue prefetch FIFO.
+	 */
+	mpnic_wr64(mpd, MPNIC_RDE_CTL,
+		   FIELD_PREP(MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD, 16) |
+		   FIELD_PREP(MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD, 16) |
+		   FIELD_PREP(MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD, 16) |
+		   FIELD_PREP(MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD, 16));
+
+	/* Receive side coalescing is not supported yet */
+	mpnic_wr64(mpd, MPNIC_RSC_GLOBAL_CONF,
+		   MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE);
+
+	mpnic_wr64(mpd, MPNIC_BDQ_MEM_INIT_REQ, MPNIC_MEM_INIT_BDQ_VAL);
+	mpnic_wr64(mpd, MPNIC_RCM_MEM_INIT_REQ, MPNIC_MEM_INIT_RCM_VAL);
+	mpnic_wr64(mpd, MPNIC_RDE_MEM_INIT_REQ, MPNIC_MEM_INIT_RDE_VAL);
+	mpnic_wr64(mpd, MPNIC_RPC_MEM_INIT_REQ, MPNIC_MEM_INIT_RPC_VAL);
+}
+
+static void mpnic_txglb_init(struct mpnic_dev *mpd)
+{
+	/* Nothing is redirected to the BMC until the Tx offload TCAM gets
+	 * programmed with its addresses.
+	 */
+	mpnic_wr64(mpd, MPNIC_TOF_TCAM_DEST_REMAP, 0);
+
+	mpnic_wr64(mpd, MPNIC_TCM_MEM_INIT_REQ, MPNIC_MEM_INIT_TCM_VAL);
+	mpnic_wr64(mpd, MPNIC_TDE_MEM_INIT_REQ, MPNIC_MEM_INIT_TDE_VAL);
+	mpnic_wr64(mpd, MPNIC_TDF_MEM_INIT_REQ, MPNIC_MEM_INIT_TDF_VAL);
+}
+
+/* Fill the DWRR arbiter memories. Setting the INIT bit makes the hardware
+ * write the given credit and quantum into every entry at the queue, group,
+ * set and port level, so nothing has to be programmed per queue.
+ */
+static void mpnic_tqs_sram_init(struct mpnic_dev *mpd)
+{
+	mpnic_wr64(mpd, MPNIC_TQS_GROUP_INIT_CTL,
+		   FIELD_PREP(MPNIC_TQS_GROUP_INIT_CTL_QUANTUM,
+			      MPNIC_TQS_DWRR_INIT_QUANTUM) |
+		   MPNIC_TQS_GROUP_INIT_CTL_INIT);
+	mpnic_wr64(mpd, MPNIC_TQS_SET_INIT_CTL,
+		   FIELD_PREP(MPNIC_TQS_SET_INIT_CTL_QUANTUM,
+			      MPNIC_TQS_DWRR_INIT_QUANTUM) |
+		   MPNIC_TQS_SET_INIT_CTL_INIT);
+	mpnic_wr64(mpd, MPNIC_TQS_PORT_INIT_CTL,
+		   FIELD_PREP(MPNIC_TQS_PORT_INIT_CTL_QUANTUM,
+			      MPNIC_TQS_DWRR_INIT_QUANTUM) |
+		   MPNIC_TQS_PORT_INIT_CTL_INIT);
+	mpnic_wr64(mpd, MPNIC_TQS_SRAM_INIT_CTL,
+		   FIELD_PREP(MPNIC_TQS_SRAM_INIT_CTL_QUANTUM,
+			      MPNIC_TQS_DWRR_INIT_QUANTUM) |
+		   MPNIC_TQS_SRAM_INIT_CTL_INIT);
+}
+
+static void mpnic_tqs_init(struct mpnic_dev *mpd)
+{
+	u64 val;
+
+	/* Initialize to the largest frame we support, the scheduler
+	 * truncates anything above it. The BMC gets the same limit.
+	 */
+	mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL0, MPNIC_TQS_MTU_CTL0_MAX);
+	mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL1, MPNIC_TQS_MTU_CTL0_MAX);
+
+	mpnic_wr64(mpd, MPNIC_TQS_GLBL_CTL0,
+		   MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN);
+
+	mpnic_tqs_sram_init(mpd);
+
+	/* Only port 0 is used. A single traffic class is in use as well, so
+	 * TC0 gets all of the Tx buffer credits not reserved for the BMC.
+	 */
+	mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_0,
+		   FIELD_PREP(MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0,
+			      MPNIC_TQS_GLBL_TXB_CRDT_MAC));
+	mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_1, 0);
+	mpnic_wr64(mpd, MPNIC_TQS_GLBL_BMC,
+		   FIELD_PREP(MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS,
+			      MPNIC_TQS_GLBL_TXB_CRDT_BMC));
+
+	mpnic_wr64(mpd, MPNIC_TQS_PORT_CTL(0), 0);
+
+	/* Map all sets to port 0 */
+	mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP0(0), ~0ULL);
+	mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP1(0), ~0ULL);
+
+	mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_0,
+		   FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE,
+			      MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) |
+		   FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP,
+			      MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH));
+	mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_1,
+		   FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET,
+			      MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) |
+		   FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT,
+			      MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH));
+
+	/* The rate limiters are left uninitialized, so shaping has to stay
+	 * off or nothing would ever get scheduled.
+	 */
+	mpnic_wr64(mpd, MPNIC_TQS_GLBL_SHAPING, MPNIC_TQS_GLBL_SHAPING_DISABLE);
+
+	/* Enable fairness protection (phantom eligibility). Read modify
+	 * write so that the reset default slowdown cycle is preserved.
+	 */
+	val = mpnic_rd64(mpd, MPNIC_TQS_SLOWDOWN_CTL);
+	val |= MPNIC_TQS_SLOWDOWN_CTL_ENABLE;
+	mpnic_wr64(mpd, MPNIC_TQS_SLOWDOWN_CTL, val);
+
+	/* Use immediate credit decrement at every DWRR level so that the
+	 * credit reflects a grant in the same cycle.
+	 */
+	val = mpnic_rd64(mpd, MPNIC_TQS_ARB_CTL);
+	val |= MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN |
+	       MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN |
+	       MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN |
+	       MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN |
+	       MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN |
+	       MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN;
+	mpnic_wr64(mpd, MPNIC_TQS_ARB_CTL, val);
+}
+
+/* The MPS and CLS fields sit at the same bit positions in every block, so
+ * one set of masks covers both the RNI and the TNI registers.
+ */
+static void mpnic_mps_init(struct mpnic_dev *mpd, u32 reg, unsigned int mps,
+			   unsigned int cls)
+{
+	u64 val = mpnic_rd64(mpd, reg);
+
+	val &= ~(MPNIC_RNI_RDE_CTL_MPS | MPNIC_RNI_RDE_CTL_CLS);
+	val |= FIELD_PREP(MPNIC_RNI_RDE_CTL_MPS, mps) |
+	       FIELD_PREP(MPNIC_RNI_RDE_CTL_CLS, cls);
+
+	mpnic_wr64(mpd, reg, val);
+}
+
+/* Likewise for the MRRS and CLS fields, which have their own common
+ * layout.
+ */
+static void mpnic_mrrs_init(struct mpnic_dev *mpd, u32 reg, unsigned int mrrs,
+			    unsigned int cls)
+{
+	u64 val = mpnic_rd64(mpd, reg);
+
+	val &= ~(MPNIC_TNI_GLBL_TDF_CTL_MRRS | MPNIC_TNI_GLBL_TDF_CTL_CLS);
+	val |= FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_MRRS, mrrs) |
+	       FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_CLS, cls);
+
+	mpnic_wr64(mpd, reg, val);
+}
+
+/**
+ * mpnic_axi_init - Configure AXI bus parameters from host PCIe capabilities
+ * @mpd: Device to configure
+ *
+ * Programs the max read request size, max payload size and cache line size
+ * of the DMA engines. The hardware encodes all three as a power of 2 index,
+ * MRRS and MPS relative to 128 B and CLS relative to 64 B.
+ *
+ * MAX_OT and MAX_OB are left at their hardware defaults.
+ */
+static void mpnic_axi_init(struct mpnic_dev *mpd)
+{
+	int mps, cls, mrrs;
+
+	mps = clamp(ilog2(mpd->mps) - 7, 0, 3);
+	cls = clamp(ilog2(L1_CACHE_BYTES) - 6, 0, 3);
+
+	mpnic_mps_init(mpd, MPNIC_RNI_RDE_CTL, mps, cls);
+	mpnic_mps_init(mpd, MPNIC_RNI_RCM_CTL, mps, cls);
+	mpnic_mps_init(mpd, MPNIC_TNI_GLBL_TCM_CTL, mps, cls);
+
+	mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 3);
+	mpnic_mrrs_init(mpd, MPNIC_RNI_RBP_CTL, mrrs, cls);
+	mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDF_CTL, mrrs, cls);
+
+	/* TDE supports a wider range of MRRS encodings. */
+	mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 5);
+	mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDE_CTL, mrrs, cls);
+}
+
+/**
+ * mpnic_ro_init - Set relaxed ordering on the outbound TLP attributes
+ * @mpd: Device to configure
+ *
+ * Completions must stay ordered so that they are not observed before the
+ * payload DMA they describe has landed, so RCM and TCM are left alone.
+ */
+static void mpnic_ro_init(struct mpnic_dev *mpd)
+{
+	u64 attr = mpd->relaxed_ord ? MPNIC_OB_ATTR_RO : 0;
+
+	mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_H, attr);
+	mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_P, attr);
+	mpnic_wr64(mpd, MPNIC_OB_ATTR_TDF, attr);
+	mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_HPQ, attr);
+	mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_PPQ, attr);
+	mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_H, attr);
+	mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_P, attr);
+}
+
+static bool mpnic_init_status_ready(struct mpnic_dev *mpd,
+				    const struct mpnic_init_poll *polls,
+				    struct mpnic_poll_state *state)
+{
+	u64 val;
+	int i;
+
+	for (i = state->poll_idx; polls[i].addr; i++) {
+		val = mpnic_rd64(mpd, polls[i].addr);
+
+		if ((val & polls[i].exp_val) != polls[i].exp_val) {
+			state->poll_idx = i;
+			state->val = val;
+			return false;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * mpnic_mem_init_poll - Wait for the memory initializations to complete
+ * @mpd: Device to poll
+ *
+ * The blocks initialize their memories in parallel, so walk the status
+ * registers in order and only go back to sleep on the first one which is
+ * not done yet.
+ *
+ * Return: 0 on success, -ETIMEDOUT if not everything completed in time
+ */
+static int mpnic_mem_init_poll(struct mpnic_dev *mpd)
+{
+	static const struct mpnic_init_poll polls[] = {
+		{ MPNIC_TQS_ARB_INIT_VAL, MPNIC_TQS_SRAM_STS },
+		{ MPNIC_MEM_INIT_BDQ_VAL, MPNIC_BDQ_MEM_INIT_DONE },
+		{ MPNIC_MEM_INIT_RCM_VAL, MPNIC_RCM_MEM_INIT_DONE },
+		{ MPNIC_MEM_INIT_RDE_VAL, MPNIC_RDE_MEM_INIT_DONE },
+		{ MPNIC_MEM_INIT_RPC_VAL, MPNIC_RPC_MEM_INIT_DONE },
+		{ MPNIC_MEM_INIT_TCM_VAL, MPNIC_TCM_MEM_INIT_DONE },
+		{ MPNIC_MEM_INIT_TDE_VAL, MPNIC_TDE_MEM_INIT_DONE },
+		{ MPNIC_MEM_INIT_TDF_VAL, MPNIC_TDF_MEM_INIT_DONE },
+		{ MPNIC_MEM_INIT_RXB_VAL, MPNIC_RXB_MEM_INIT_DONE },
+		{ 0 },
+	};
+	struct mpnic_poll_state state = {};
+	bool done;
+	int err;
+
+	err = read_poll_timeout(mpnic_init_status_ready, done, done,
+				MPNIC_MEM_INIT_POLL_US, MPNIC_MEM_INIT_TO_US,
+				false, mpd, polls, &state);
+	if (err)
+		dev_err(mpd->dev, "Poll timeout for reg 0x%x: 0x%llx\n",
+			polls[state.poll_idx].addr, state.val);
+
+	return err;
+}
+
+int mpnic_dev_init(struct mpnic_dev *mpd)
+{
+	int err;
+
+	mpnic_tdf_glbl_init(mpd);
+	mpnic_txb_init(mpd);
+	mpnic_rxb_init(mpd);
+	mpnic_desc_sram_init(mpd);
+	mpnic_axi_init(mpd);
+	mpnic_ro_init(mpd);
+	mpnic_rxglb_init(mpd);
+	mpnic_txglb_init(mpd);
+	mpnic_tqs_init(mpd);
+
+	err = mpnic_mem_init_poll(mpd);
+	if (err) {
+		dev_err(mpd->dev, "Device initialization failed: %d\n", err);
+		return err;
+	}
+
+	if (!mpnic_present(mpd))
+		return -EIO;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_irq.c b/drivers/net/ethernet/meta/mpnic/mpnic_irq.c
new file mode 100644
index 00000000000000..bcc33655cbbea8
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_irq.c
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#include <linux/cpumask.h>
+#include <linux/interrupt.h>
+#include <linux/minmax.h>
+#include <linux/pci.h>
+
+#include "mpnic.h"
+
+int mpnic_request_irq(struct mpnic_dev *mpd, int nr, irq_handler_t handler,
+		      unsigned long flags, const char *name, void *data)
+{
+	struct pci_dev *pdev = to_pci_dev(mpd->dev);
+	int irq = pci_irq_vector(pdev, nr);
+
+	if (irq < 0)
+		return irq;
+
+	return request_irq(irq, handler, flags, name, data);
+}
+
+void mpnic_free_irq(struct mpnic_dev *mpd, int nr, void *data)
+{
+	struct pci_dev *pdev = to_pci_dev(mpd->dev);
+	int irq = pci_irq_vector(pdev, nr);
+
+	if (irq < 0)
+		return;
+
+	free_irq(irq, data);
+}
+
+void mpnic_free_irqs(struct mpnic_dev *mpd)
+{
+	struct pci_dev *pdev = to_pci_dev(mpd->dev);
+
+	mpd->num_irqs = 0;
+	pci_free_irq_vectors(pdev);
+}
+
+int mpnic_alloc_irqs(struct mpnic_dev *mpd)
+{
+	unsigned int wanted_irqs = MPNIC_NON_NAPI_VECTORS;
+	struct pci_dev *pdev = to_pci_dev(mpd->dev);
+	int num_irqs;
+
+	wanted_irqs += min_t(unsigned int, num_online_cpus(), MPNIC_MAX_RXQS);
+	num_irqs = pci_alloc_irq_vectors(pdev, MPNIC_NON_NAPI_VECTORS + 1,
+					 wanted_irqs, PCI_IRQ_MSIX);
+	if (num_irqs < 0) {
+		dev_err(mpd->dev, "Failed to allocate MSI-X entries: %d\n",
+			num_irqs);
+		return num_irqs;
+	}
+
+	if (num_irqs < wanted_irqs)
+		dev_warn(mpd->dev, "Allocated %d IRQs, expected %u\n",
+			 num_irqs, wanted_irqs);
+
+	mpd->num_irqs = num_irqs;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_netdev.c b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.c
new file mode 100644
index 00000000000000..2c0309d0c22131
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.c
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#include <linux/etherdevice.h>
+#include <linux/ipv6.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+
+#include "mpnic.h"
+#include "mpnic_netdev.h"
+#include "mpnic_txrx.h"
+
+static int mpnic_open(struct net_device *netdev)
+{
+	struct mpnic_net *mpn = netdev_priv(netdev);
+	int err;
+
+	err = mpnic_alloc_napi_vectors(mpn);
+	if (err)
+		return err;
+
+	err = mpnic_alloc_resources(mpn);
+	if (err)
+		goto err_free_napi_vectors;
+
+	err = mpnic_set_netif_queues(mpn);
+	if (err)
+		goto err_free_resources;
+
+	mpnic_enable(mpn);
+	mpnic_fill(mpn);
+	mpnic_napi_enable(mpn);
+
+	netif_tx_wake_all_queues(netdev);
+	netif_carrier_on(netdev);
+
+	return 0;
+
+err_free_resources:
+	mpnic_free_resources(mpn);
+err_free_napi_vectors:
+	mpnic_free_napi_vectors(mpn);
+	return err;
+}
+
+static int mpnic_stop(struct net_device *netdev)
+{
+	struct mpnic_net *mpn = netdev_priv(netdev);
+
+	netif_carrier_off(netdev);
+
+	mpnic_napi_disable(mpn);
+	netif_tx_disable(netdev);
+
+	mpnic_disable(mpn);
+	mpnic_wait_all_queues_idle(mpn->mpd);
+	mpnic_flush(mpn);
+
+	mpnic_reset_netif_queues(mpn);
+	mpnic_free_resources(mpn);
+	mpnic_free_napi_vectors(mpn);
+
+	return 0;
+}
+
+static const struct net_device_ops mpnic_netdev_ops = {
+	.ndo_open		= mpnic_open,
+	.ndo_stop		= mpnic_stop,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_start_xmit		= mpnic_xmit_frame,
+};
+
+/**
+ * mpnic_netdev_free - Free the netdev associated with mpnic
+ * @mpd: Driver specific structure to free netdev from
+ **/
+void mpnic_netdev_free(struct mpnic_dev *mpd)
+{
+	free_netdev(mpd->netdev);
+	mpd->netdev = NULL;
+}
+
+/**
+ * mpnic_netdev_alloc - Allocate a netdev and associate it with mpnic
+ * @mpd: Driver specific structure to associate the netdev with
+ *
+ * Return: NULL on failure.
+ **/
+struct net_device *mpnic_netdev_alloc(struct mpnic_dev *mpd)
+{
+	struct net_device *netdev;
+	struct mpnic_net *mpn;
+	unsigned int queues;
+
+	netdev = alloc_etherdev_mq(sizeof(*mpn), MPNIC_MAX_RXQS);
+	if (!netdev)
+		return NULL;
+
+	SET_NETDEV_DEV(netdev, mpd->dev);
+	mpd->netdev = netdev;
+
+	netdev->netdev_ops = &mpnic_netdev_ops;
+	netdev->request_ops_lock = true;
+
+	mpn = netdev_priv(netdev);
+	mpn->netdev = netdev;
+	mpn->mpd = mpd;
+
+	mpn->txq_size = MPNIC_TXQ_SIZE_DEFAULT;
+	mpn->hpq_size = MPNIC_HPQ_SIZE_DEFAULT;
+	mpn->ppq_size = MPNIC_PPQ_SIZE_DEFAULT;
+	mpn->rcq_size = MPNIC_RCQ_SIZE_DEFAULT;
+
+	queues = min(netif_get_num_default_rss_queues(),
+		     mpd->num_irqs - MPNIC_NON_NAPI_VECTORS);
+	mpn->num_tx_queues = queues;
+	mpn->num_rx_queues = queues;
+	mpn->num_napi = queues;
+
+	netdev->features |= NETIF_F_SG;
+	netdev->hw_features |= netdev->features;
+	netdev->vlan_features |= netdev->features;
+
+	netdev->min_mtu = IPV6_MIN_MTU;
+	netdev->max_mtu = MPNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;
+
+	netif_carrier_off(netdev);
+	netif_tx_stop_all_queues(netdev);
+
+	return netdev;
+}
+
+static int mpnic_dsn_to_mac_addr(u64 dsn, u8 *addr)
+{
+	addr[0] = (dsn >> 56) & 0xFF;
+	addr[1] = (dsn >> 48) & 0xFF;
+	addr[2] = (dsn >> 40) & 0xFF;
+	addr[3] = (dsn >> 16) & 0xFF;
+	addr[4] = (dsn >> 8) & 0xFF;
+	addr[5] = dsn & 0xFF;
+
+	return is_valid_ether_addr(addr) ? 0 : -EINVAL;
+}
+
+/**
+ * mpnic_netdev_register - Assign the MAC address and register the netdev
+ * @netdev: Netdev to register
+ *
+ * The permanent address is derived from the PCIe device serial number, the
+ * same way the firmware and the BMC derive it. A random address would break
+ * provisioning, so refuse to spawn the interface if the serial number does
+ * not yield a valid one.
+ *
+ * Return: non-zero on failure.
+ **/
+int mpnic_netdev_register(struct net_device *netdev)
+{
+	struct mpnic_net *mpn = netdev_priv(netdev);
+	struct mpnic_dev *mpd = mpn->mpd;
+	u8 addr[ETH_ALEN];
+	int err;
+
+	err = mpnic_dsn_to_mac_addr(mpd->dsn, addr);
+	if (err) {
+		dev_err(mpd->dev, "MAC addr %pM invalid\n", addr);
+		return err;
+	}
+
+	ether_addr_copy(netdev->perm_addr, addr);
+	eth_hw_addr_set(netdev, addr);
+
+	err = netif_set_real_num_queues(netdev, mpn->num_tx_queues,
+					mpn->num_rx_queues);
+	if (err)
+		return err;
+
+	/* Abort if MMIO has failed. This has to be the last check before
+	 * registration, the register accessors can only detach the device
+	 * once it has been registered.
+	 */
+	if (!mpnic_present(mpd))
+		return -EIO;
+
+	return register_netdev(netdev);
+}
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h
new file mode 100644
index 00000000000000..ccb0929f9180ad
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_netdev.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#ifndef _MPNIC_NETDEV_H_
+#define _MPNIC_NETDEV_H_
+
+#include <linux/types.h>
+
+#include "mpnic.h"
+#include "mpnic_txrx.h"
+
+struct mpnic_net {
+	struct mpnic_ring *tx[MPNIC_MAX_TXQS];
+	struct mpnic_ring *rx[MPNIC_MAX_RXQS];
+
+	struct mpnic_napi_vector *napi[MPNIC_MAX_NAPI_VECTORS];
+
+	struct net_device *netdev;
+	struct mpnic_dev *mpd;
+
+	u32 txq_size;
+	u32 hpq_size;
+	u32 ppq_size;
+	u32 rcq_size;
+
+	u16 num_napi;
+	u16 num_tx_queues;
+	u16 num_rx_queues;
+};
+
+struct net_device *mpnic_netdev_alloc(struct mpnic_dev *mpd);
+void mpnic_netdev_free(struct mpnic_dev *mpd);
+int mpnic_netdev_register(struct net_device *netdev);
+
+#endif /* _MPNIC_NETDEV_H_ */
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_pci.c b/drivers/net/ethernet/meta/mpnic/mpnic_pci.c
new file mode 100644
index 00000000000000..968cd611b8eab2
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_pci.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "mpnic.h"
+#include "mpnic_netdev.h"
+
+#define PCI_DEVICE_ID_META_MPNIC	0x0014
+
+static void mpnic_mmio_err(struct mpnic_dev *mpd, u32 reg)
+{
+	/* Hardware is giving us all 1's reads, assume it is gone */
+	WRITE_ONCE(mpd->uc_addr0, NULL);
+
+	dev_err(mpd->dev,
+		"Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
+		reg, reg << 2);
+
+	/* Tell the stack the device has lost its PCIe link */
+	if (mpd->netdev)
+		netif_device_detach(mpd->netdev);
+}
+
+u64 mpnic_rd64(struct mpnic_dev *mpd, u32 reg)
+{
+	u32 __iomem *csr = READ_ONCE(mpd->uc_addr0);
+	u64 value;
+
+	if (!csr)
+		return ~0ULL;
+
+	value = readq(csr + reg);
+
+	/* If any bits are 0 value should be valid */
+	if (~value)
+		return value;
+
+	/* All ones can be a valid value, so confirm against a register
+	 * which never reads that way on a live device.
+	 */
+	if (reg != MPNIC_BDQ_SPARE && ~readq(csr + MPNIC_BDQ_SPARE))
+		return value;
+
+	mpnic_mmio_err(mpd, reg);
+
+	return ~0ULL;
+}
+
+static struct mpnic_dev *mpnic_alloc(struct pci_dev *pdev)
+{
+	struct mpnic_dev *mpd;
+
+	mpd = kzalloc_obj(*mpd);
+	if (!mpd)
+		return NULL;
+
+	pci_set_drvdata(pdev, mpd);
+	mpd->dev = &pdev->dev;
+
+	mpd->dsn = pci_get_dsn(pdev);
+	mpd->mps = pcie_get_mps(pdev);
+	mpd->readrq = pcie_get_readrq(pdev);
+	mpd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev);
+
+	return mpd;
+}
+
+/**
+ * mpnic_probe - Device initialization routine
+ * @pdev: PCI device information struct
+ * @ent: entry in mpnic_pci_tbl
+ *
+ * Return: 0 on success, negative on failure
+ **/
+static int mpnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct net_device *netdev;
+	void __iomem *uc_addr0;
+	struct mpnic_dev *mpd;
+	int err;
+
+	if (pdev->error_state != pci_channel_io_normal) {
+		dev_err(&pdev->dev,
+			"PCI device still in an error state. Unable to load...\n");
+		return -EIO;
+	}
+
+	err = pcim_enable_device(pdev);
+	if (err) {
+		dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
+		return err;
+	}
+
+	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
+	if (err) {
+		dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
+		return err;
+	}
+
+	mpd = mpnic_alloc(pdev);
+	if (!mpd)
+		return -ENOMEM;
+
+	uc_addr0 = pcim_iomap_region(pdev, 0, MPNIC_DRV_NAME);
+	if (IS_ERR(uc_addr0)) {
+		err = PTR_ERR(uc_addr0);
+		dev_err(&pdev->dev, "Mapping the register file failed: %d\n",
+			err);
+		goto err_free_mpd;
+	}
+	mpd->uc_addr0 = uc_addr0;
+
+	pci_set_master(pdev);
+	pci_save_state(pdev);
+
+	err = mpnic_alloc_irqs(mpd);
+	if (err)
+		goto err_free_mpd;
+
+	err = mpnic_dev_init(mpd);
+	if (err)
+		goto err_free_irqs;
+
+	netdev = mpnic_netdev_alloc(mpd);
+	if (!netdev) {
+		dev_err(&pdev->dev, "Netdev allocation failed\n");
+		err = -ENOMEM;
+		goto err_free_irqs;
+	}
+
+	err = mpnic_netdev_register(netdev);
+	if (err) {
+		dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
+		goto err_free_netdev;
+	}
+
+	return 0;
+
+err_free_netdev:
+	mpnic_netdev_free(mpd);
+err_free_irqs:
+	mpnic_free_irqs(mpd);
+err_free_mpd:
+	kfree(mpd);
+
+	return err;
+}
+
+/**
+ * mpnic_remove - Device removal routine
+ * @pdev: PCI device information struct
+ **/
+static void mpnic_remove(struct pci_dev *pdev)
+{
+	struct mpnic_dev *mpd = pci_get_drvdata(pdev);
+
+	unregister_netdev(mpd->netdev);
+	mpnic_netdev_free(mpd);
+	mpnic_free_irqs(mpd);
+	kfree(mpd);
+}
+
+static const struct pci_device_id mpnic_pci_tbl[] = {
+	{ PCI_VDEVICE(META, PCI_DEVICE_ID_META_MPNIC) },
+	/* required last entry */
+	{}
+};
+MODULE_DEVICE_TABLE(pci, mpnic_pci_tbl);
+
+static struct pci_driver mpnic_driver = {
+	.name		= MPNIC_DRV_NAME,
+	.id_table	= mpnic_pci_tbl,
+	.probe		= mpnic_probe,
+	.remove		= mpnic_remove,
+};
+
+module_pci_driver(mpnic_driver);
+
+MODULE_DESCRIPTION("Meta Platforms Network Interface Controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c
new file mode 100644
index 00000000000000..edb07b7f334bdb
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.c
@@ -0,0 +1,1395 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bitfield.h>
+#include <linux/dma-mapping.h>
+#include <linux/iopoll.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <net/page_pool/helpers.h>
+
+#include "mpnic.h"
+#include "mpnic_netdev.h"
+#include "mpnic_txrx.h"
+
+struct mpnic_xmit_cb {
+	u32 bytecount;
+	u8 desc_count;
+};
+
+#define MPNIC_XMIT_CB(__skb) ((struct mpnic_xmit_cb *)((__skb)->cb))
+#define MPNIC_TWD_TYPE_MASK(_type) \
+	cpu_to_le64(FIELD_PREP(MPNIC_TWD_TYPE, MPNIC_TWD_TYPE_##_type))
+
+/* Leave the interrupt moderation counters alone when arming or masking */
+#define MPNIC_TIM_PARAM_CFG_PRESERVE_MASK \
+	(MPNIC_TIM_CTL1_UPD_IGN_LONG_EVENT_CNT | \
+	 MPNIC_TIM_CTL1_UPD_IGN_LONG_TIME_CNT | \
+	 MPNIC_TIM_CTL1_UPD_IGN_SHORT_TIME_CNT)
+
+static void mpnic_nv_irq_disable(struct mpnic_napi_vector *nv)
+{
+	mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx),
+		   MPNIC_TIM_PARAM_CFG_PRESERVE_MASK |
+		   MPNIC_TIM_CTL1_MASK_EN | MPNIC_TIM_CTL1_MASK);
+}
+
+static void mpnic_nv_irq_rearm(struct mpnic_napi_vector *nv)
+{
+	/* Rearming a single queue on a given IRQ rearms all the other
+	 * queues mapped to the same IRQ.
+	 */
+	mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx),
+		   MPNIC_TIM_PARAM_CFG_PRESERVE_MASK | MPNIC_TIM_CTL1_MASK_EN);
+}
+
+static void mpnic_nv_irq_trigger(struct mpnic_napi_vector *nv)
+{
+	mpnic_wr64(nv->mpd, MPNIC_TIM_CTL1(nv->qt[0].cmpl.q_idx),
+		   MPNIC_TIM_PARAM_CFG_PRESERVE_MASK |
+		   MPNIC_TIM_CTL1_MASK_EN | MPNIC_TIM_CTL1_TRIGGER);
+}
+
+static unsigned int mpnic_desc_unused(struct mpnic_ring *ring)
+{
+	return (ring->head - ring->tail - 1) & ring->size_mask;
+}
+
+static struct netdev_queue *mpnic_txring_txq(const struct net_device *dev,
+					     const struct mpnic_ring *ring)
+{
+	return netdev_get_tx_queue(dev, ring->q_idx);
+}
+
+static void mpnic_tx_doorbell(struct mpnic_ring *ring, __le64 *meta)
+{
+	*meta |= cpu_to_le64(MPNIC_TWD_FLAG_REQ_COMPLETION);
+	ring->deferred_meta = -1;
+
+	/* Force DMA writes to flush before writing to tail */
+	dma_wmb();
+
+	writeq(ring->tail, ring->doorbell);
+}
+
+/* Packets handed to us with xmit_more set are left in the ring without a
+ * doorbell, and without a completion request, in the expectation that the
+ * packet ending the burst will ring for all of them. If that packet gets
+ * dropped instead we have to ring here, otherwise the descriptors sit in
+ * the ring until the next transmit, which may never come.
+ */
+static void mpnic_tx_flush_doorbell(struct mpnic_ring *ring)
+{
+	if (ring->deferred_meta >= 0)
+		mpnic_tx_doorbell(ring, &ring->desc[ring->deferred_meta]);
+}
+
+static void mpnic_unmap_single_twd(struct device *dev, __le64 *twd)
+{
+	u64 raw_twd = le64_to_cpu(*twd);
+
+	dma_unmap_single(dev, FIELD_GET(MPNIC_TWD_ADDR, raw_twd),
+			 FIELD_GET(MPNIC_TWD_LEN, raw_twd), DMA_TO_DEVICE);
+}
+
+static void mpnic_unmap_page_twd(struct device *dev, __le64 *twd)
+{
+	u64 raw_twd = le64_to_cpu(*twd);
+
+	dma_unmap_page(dev, FIELD_GET(MPNIC_TWD_ADDR, raw_twd),
+		       FIELD_GET(MPNIC_TWD_LEN, raw_twd), DMA_TO_DEVICE);
+}
+
+static bool
+mpnic_tx_map(struct mpnic_ring *ring, struct sk_buff *skb, __le64 *meta)
+{
+	struct device *dev = skb->dev->dev.parent;
+	unsigned int tail = ring->tail, first;
+	unsigned int size, data_len;
+	skb_frag_t *frag;
+	dma_addr_t dma;
+	__le64 *twd;
+
+	tail++;
+	tail &= ring->size_mask;
+	first = tail;
+
+	size = skb_headlen(skb);
+	data_len = skb->data_len;
+
+	if (size > FIELD_MAX(MPNIC_TWD_LEN))
+		goto err_dma;
+
+	dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
+
+	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
+		twd = &ring->desc[tail];
+
+		if (dma_mapping_error(dev, dma))
+			goto err_dma;
+
+		*twd = cpu_to_le64(FIELD_PREP(MPNIC_TWD_ADDR, dma) |
+				   FIELD_PREP(MPNIC_TWD_LEN, size) |
+				   FIELD_PREP(MPNIC_TWD_TYPE,
+					      MPNIC_TWD_TYPE_AL));
+
+		tail++;
+		tail &= ring->size_mask;
+
+		if (!data_len)
+			break;
+
+		size = skb_frag_size(frag);
+		data_len -= size;
+
+		if (size > FIELD_MAX(MPNIC_TWD_LEN))
+			goto err_dma;
+
+		dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
+	}
+
+	*twd |= MPNIC_TWD_TYPE_MASK(LAST_AL);
+
+	MPNIC_XMIT_CB(skb)->desc_count = ((twd - meta) + 1) & ring->size_mask;
+
+	skb_tx_timestamp(skb);
+
+	ring->tail = tail;
+
+	/* Verify there is room for another packet */
+	netif_txq_maybe_stop(mpnic_txring_txq(skb->dev, ring),
+			     mpnic_desc_unused(ring), MPNIC_MAX_SKB_DESC,
+			     MPNIC_TX_DESC_WAKEUP);
+
+	if (__netdev_tx_sent_queue(mpnic_txring_txq(skb->dev, ring),
+				   MPNIC_XMIT_CB(skb)->bytecount,
+				   netdev_xmit_more()))
+		mpnic_tx_doorbell(ring, meta);
+	else
+		ring->deferred_meta = meta - ring->desc;
+
+	return false;
+err_dma:
+	if (net_ratelimit())
+		netdev_err(skb->dev, "TX DMA map failed\n");
+
+	while (tail != first) {
+		tail--;
+		tail &= ring->size_mask;
+		twd = &ring->desc[tail];
+		if (tail == first)
+			mpnic_unmap_single_twd(dev, twd);
+		else
+			mpnic_unmap_page_twd(dev, twd);
+	}
+
+	return true;
+}
+
+#define MPNIC_MIN_FRAME_LEN	60
+
+static netdev_tx_t mpnic_xmit_frame_ring(struct sk_buff *skb,
+					 struct mpnic_ring *ring)
+{
+	__le64 *meta = &ring->desc[ring->tail];
+	u32 tail = ring->tail;
+
+	if (skb_put_padto(skb, MPNIC_MIN_FRAME_LEN))
+		goto err_drop;
+
+	if (!netif_txq_maybe_stop(mpnic_txring_txq(skb->dev, ring),
+				  mpnic_desc_unused(ring), MPNIC_MAX_SKB_DESC,
+				  MPNIC_TX_DESC_WAKEUP)) {
+		mpnic_tx_flush_doorbell(ring);
+		return NETDEV_TX_BUSY;
+	}
+
+	ring->tx_buf[tail] = skb;
+	*meta = cpu_to_le64(MPNIC_TWD_FLAG_DEST_MAC);
+
+	MPNIC_XMIT_CB(skb)->bytecount = skb->len;
+	MPNIC_XMIT_CB(skb)->desc_count = 0;
+
+	if (mpnic_tx_map(ring, skb, meta))
+		goto err_free;
+
+	return NETDEV_TX_OK;
+
+err_free:
+	dev_kfree_skb_any(skb);
+	ring->tx_buf[tail] = NULL;
+	ring->tail = tail;
+err_drop:
+	mpnic_tx_flush_doorbell(ring);
+
+	return NETDEV_TX_OK;
+}
+
+netdev_tx_t mpnic_xmit_frame(struct sk_buff *skb, struct net_device *dev)
+{
+	struct mpnic_net *mpn = netdev_priv(dev);
+
+	return mpnic_xmit_frame_ring(skb, mpn->tx[skb_get_queue_mapping(skb)]);
+}
+
+static void mpnic_clean_twq0(struct mpnic_napi_vector *nv, int napi_budget,
+			     struct mpnic_ring *ring, bool discard,
+			     unsigned int hw_head)
+{
+	u64 total_bytes = 0, total_packets = 0;
+	unsigned int head = ring->head;
+	struct netdev_queue *txq;
+	unsigned int clean_desc;
+
+	clean_desc = (hw_head - head) & ring->size_mask;
+
+	while (clean_desc) {
+		struct sk_buff *skb = ring->tx_buf[head];
+		unsigned int desc_cnt;
+
+		desc_cnt = MPNIC_XMIT_CB(skb)->desc_count;
+		if (desc_cnt > clean_desc)
+			break;
+
+		ring->tx_buf[head] = NULL;
+
+		clean_desc -= desc_cnt;
+
+		/* Step over the metadata descriptor */
+		head++;
+		head &= ring->size_mask;
+		desc_cnt--;
+
+		mpnic_unmap_single_twd(nv->dev, &ring->desc[head]);
+		head++;
+		head &= ring->size_mask;
+		desc_cnt--;
+
+		while (desc_cnt--) {
+			mpnic_unmap_page_twd(nv->dev, &ring->desc[head]);
+			head++;
+			head &= ring->size_mask;
+		}
+
+		total_bytes += MPNIC_XMIT_CB(skb)->bytecount;
+		total_packets++;
+
+		napi_consume_skb(skb, napi_budget);
+	}
+
+	if (!total_bytes)
+		return;
+
+	ring->head = head;
+
+	if (discard)
+		return;
+
+	txq = mpnic_txring_txq(nv->napi.dev, ring);
+	netif_txq_completed_wake(txq, total_packets, total_bytes,
+				 mpnic_desc_unused(ring),
+				 MPNIC_TX_DESC_WAKEUP);
+}
+
+static void mpnic_commit_cq_head(struct mpnic_ring *cmpl)
+{
+	u32 head = cmpl->head;
+
+	/* The tail shadows the last value written to the doorbell, so a
+	 * completion queue which has not moved costs no MMIO write.
+	 */
+	if (cmpl->tail != head) {
+		cmpl->tail = head;
+		writeq(head & cmpl->size_mask, cmpl->doorbell);
+	}
+}
+
+static void mpnic_clean_tcq(struct mpnic_napi_vector *nv,
+			    struct mpnic_q_triad *qt, int napi_budget)
+{
+	struct mpnic_ring *cmpl = &qt->cmpl;
+	__le64 *raw_tcd, done;
+	u32 head = cmpl->head;
+	s32 head0 = -1;
+
+	done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(MPNIC_TCD_DONE);
+	raw_tcd = &cmpl->desc[head & cmpl->size_mask];
+
+	/* Walk the completion queue collecting the heads reported by NIC.
+	 * Only the first work queue is enabled and no packet asks for a
+	 * timestamp, so every completion is a plain head update and the
+	 * descriptor type does not have to be decoded.
+	 */
+	while ((*raw_tcd & cpu_to_le64(MPNIC_TCD_DONE)) == done) {
+		u64 tcd;
+
+		dma_rmb();
+
+		tcd = le64_to_cpu(*raw_tcd);
+		head0 = FIELD_GET(MPNIC_TCD_TYPE0_HEAD0, tcd);
+
+		raw_tcd++;
+		head++;
+
+		if (unlikely(!(head & cmpl->size_mask))) {
+			done ^= cpu_to_le64(MPNIC_TCD_DONE);
+			raw_tcd = &cmpl->desc[0];
+		}
+	}
+
+	cmpl->head = head;
+
+	if (head0 >= 0)
+		mpnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0);
+}
+
+static void mpnic_bd_prep(struct mpnic_ring *bdq, u32 idx, struct page *page)
+{
+	dma_addr_t dma = page_pool_get_dma_addr(page);
+
+	bdq->desc[idx] = cpu_to_le64(FIELD_PREP(MPNIC_BD_DESC_ADDR, dma >> 10) |
+				     FIELD_PREP(MPNIC_BD_DESC_ID, idx) |
+				     FIELD_PREP(MPNIC_BD_DESC_BUF_SZ_LOG2,
+						page_shift(page) - 10));
+}
+
+static unsigned int mpnic_bdq_desc_unused(struct mpnic_ring *bdq)
+{
+	return (ALIGN_DOWN(bdq->head - 1, MPNIC_BDQ_BATCH_SIZE) - bdq->tail) &
+	       bdq->size_mask;
+}
+
+static unsigned int __mpnic_fill_bdq(struct mpnic_ring *bdq)
+{
+	unsigned int i = bdq->tail;
+	unsigned int count;
+
+	for (count = mpnic_bdq_desc_unused(bdq); count; count--) {
+		struct page *page;
+
+		page = page_pool_dev_alloc_pages(bdq->page_pool);
+		if (!page)
+			break;
+
+		bdq->rx_buf[i] = page;
+		mpnic_bd_prep(bdq, i, page);
+
+		i++;
+		i &= bdq->size_mask;
+	}
+
+	return i;
+}
+
+static void __mpnic_bdq_commit_tail(struct mpnic_ring *bdq, unsigned int tail)
+{
+	if (bdq->tail != tail) {
+		bdq->tail = tail;
+
+		writeq(tail, bdq->doorbell);
+	}
+}
+
+static void mpnic_fill_qt_bdqs(struct mpnic_q_triad *qt)
+{
+	unsigned int ppq_i = __mpnic_fill_bdq(&qt->sub1);
+	unsigned int hpq_i = __mpnic_fill_bdq(&qt->sub0);
+
+	/* Force DMA writes to flush before writing to tail(s) */
+	dma_wmb();
+
+	/* Flush out the completions we are done with */
+	mpnic_commit_cq_head(&qt->cmpl);
+
+	__mpnic_bdq_commit_tail(&qt->sub0, hpq_i);
+	__mpnic_bdq_commit_tail(&qt->sub1, ppq_i);
+}
+
+/* Take one of the references batched on the page at @idx. If the device
+ * has moved on to a new page, first drop the unused references left on
+ * the previous one.
+ */
+static struct page *
+mpnic_page_pool_get(struct mpnic_pg_ctxt *pg_ctxt, struct mpnic_ring *ring,
+		    u32 idx)
+{
+	struct page *page = pg_ctxt->page;
+
+	if (unlikely(pg_ctxt->idx != idx)) {
+		if (pg_ctxt->pagecnt_bias &&
+		    !page_pool_unref_page(page, pg_ctxt->pagecnt_bias))
+			page_pool_put_unrefed_page(page->pp, page, -1, true);
+
+		page = ring->rx_buf[idx];
+		page_pool_fragment_page(page, MPNIC_PAGECNT_BIAS_MAX);
+
+		pg_ctxt->page = page;
+		pg_ctxt->pagecnt_bias = MPNIC_PAGECNT_BIAS_MAX;
+		pg_ctxt->idx = idx;
+	}
+
+	pg_ctxt->pagecnt_bias--;
+
+	return page;
+}
+
+static void mpnic_flush_pg_ctxt(struct mpnic_pg_ctxt *ctxt, bool napi)
+{
+	long pagecnt_bias = ctxt->pagecnt_bias;
+
+	if (pagecnt_bias) {
+		struct page *page = ctxt->page;
+
+		if (!page_pool_unref_page(page, pagecnt_bias))
+			page_pool_put_unrefed_page(page->pp, page, -1, napi);
+	}
+}
+
+static unsigned int mpnic_hdr_pg_start(unsigned int pg_off)
+{
+	/* The headroom of the first header may be larger than
+	 * MPNIC_RX_HROOM due to alignment. So account for that by just
+	 * making the page offset 0 if we are starting at the first header.
+	 */
+	if (ALIGN(MPNIC_RX_HROOM, 128) > MPNIC_RX_HROOM &&
+	    pg_off == ALIGN(MPNIC_RX_HROOM, 128))
+		return 0;
+
+	return pg_off - MPNIC_RX_HROOM;
+}
+
+static unsigned int mpnic_hdr_pg_end(unsigned int pg_off, unsigned int len)
+{
+	/* Determine the end of the buffer by finding the start of the next
+	 * and then subtracting the headroom from that frame.
+	 */
+	pg_off += len + MPNIC_RX_TROOM + MPNIC_RX_HROOM;
+
+	return ALIGN(pg_off, 128) - MPNIC_RX_HROOM;
+}
+
+static void
+mpnic_pkt_prepare(u64 rcd, struct mpnic_rcq_state *state,
+		  struct mpnic_q_triad *qt)
+{
+	unsigned int pg_off = FIELD_GET(MPNIC_RCD_AL_BUFF_OFF, rcd);
+	unsigned int pg_idx = FIELD_GET(MPNIC_RCD_AL_BUFF_ID, rcd);
+	unsigned int len = FIELD_GET(MPNIC_RCD_AL_BUFF_LEN, rcd);
+	bool fin = FIELD_GET(MPNIC_RCD_AL_PAGE_FIN, rcd);
+	unsigned int frame_sz, pg_start, pg_end;
+	struct xdp_buff *buff = &state->pkt.buff;
+	struct page *page;
+
+	pg_start = mpnic_hdr_pg_start(pg_off);
+
+	page = mpnic_page_pool_get(&state->hdr, &qt->sub0, pg_idx);
+	qt->sub0.head = (pg_idx + 1) & qt->sub0.size_mask;
+
+	/* Short-cut the end calculation if the page is fully consumed */
+	pg_end = fin ? page_size(page) : mpnic_hdr_pg_end(pg_off, len);
+	frame_sz = pg_end - pg_start;
+
+	page_pool_dma_sync_for_cpu(qt->sub0.page_pool, page, pg_start,
+				   frame_sz);
+
+	xdp_init_buff(buff, frame_sz, &qt->xdp_rxq);
+	xdp_prepare_buff(buff, page_address(page) + pg_start,
+			 pg_off - pg_start, len, true);
+	net_prefetch(buff->data);
+
+	state->pkt.add_frag_failed = false;
+}
+
+static void
+mpnic_add_rx_frag(u64 rcd, struct mpnic_rcq_state *state,
+		  struct mpnic_q_triad *qt)
+{
+	unsigned int pg_off = FIELD_GET(MPNIC_RCD_AL_BUFF_OFF, rcd);
+	unsigned int pg_idx = FIELD_GET(MPNIC_RCD_AL_BUFF_ID, rcd);
+	unsigned int len = FIELD_GET(MPNIC_RCD_AL_BUFF_LEN, rcd);
+	bool fin = FIELD_GET(MPNIC_RCD_AL_PAGE_FIN, rcd);
+	struct xdp_buff *buff = &state->pkt.buff;
+	unsigned int truesz;
+	struct page *page;
+
+	page = mpnic_page_pool_get(&state->payld, &qt->sub1, pg_idx);
+	qt->sub1.head = (pg_idx + 1) & qt->sub1.size_mask;
+
+	truesz = (fin ? page_size(page) : ALIGN(pg_off + len, 128)) - pg_off;
+
+	page_pool_dma_sync_for_cpu(qt->sub1.page_pool, page, pg_off, truesz);
+
+	if (!xdp_buff_add_frag(buff, page_to_netmem(page), pg_off, len,
+			       truesz)) {
+		state->payld.pagecnt_bias++;
+		state->pkt.add_frag_failed = true;
+	}
+}
+
+static void mpnic_put_pkt_buff(struct mpnic_pkt_ctxt *ctxt, bool napi)
+{
+	struct xdp_buff *buff = &ctxt->buff;
+	struct page *page;
+
+	if (!buff->data_hard_start)
+		return;
+
+	if (unlikely(xdp_buff_has_frags(buff))) {
+		struct skb_shared_info *shinfo;
+		int nr_frags;
+
+		shinfo = xdp_get_shared_info_from_buff(buff);
+		nr_frags = shinfo->nr_frags;
+
+		while (nr_frags--) {
+			page = skb_frag_page(&shinfo->frags[nr_frags]);
+			page_pool_put_full_page(page->pp, page, napi);
+		}
+	}
+
+	page = virt_to_head_page(buff->data_hard_start);
+	page_pool_put_full_page(page->pp, page, napi);
+}
+
+static int mpnic_clean_rcq(struct mpnic_napi_vector *nv,
+			   struct mpnic_q_triad *qt, int budget)
+{
+	struct mpnic_ring *rcq = &qt->cmpl;
+	struct mpnic_rcq_state *state;
+	unsigned int packets = 0;
+	__le64 *raw_rcd, done;
+	u32 head = rcq->head;
+
+	done = (head & (rcq->size_mask + 1)) ? 0 : cpu_to_le64(MPNIC_RCD_DONE);
+	raw_rcd = &rcq->desc[head & rcq->size_mask];
+	state = rcq->state;
+
+	while (packets < budget) {
+		u64 rcd;
+
+		if ((*raw_rcd & cpu_to_le64(MPNIC_RCD_DONE)) != done)
+			break;
+
+		dma_rmb();
+
+		rcd = le64_to_cpu(*raw_rcd);
+
+		switch (FIELD_GET(MPNIC_RCD_TYPE, rcd)) {
+		case MPNIC_RCD_TYPE_HDR_AL:
+			if (FIELD_GET(MPNIC_RCD_HDR_SUBTYPE, rcd) ==
+			    MPNIC_RCD_HDR_SUBTYPE_HDR)
+				mpnic_pkt_prepare(rcd, state, qt);
+			break;
+		case MPNIC_RCD_TYPE_PAY_AL:
+			mpnic_add_rx_frag(rcd, state, qt);
+			break;
+		case MPNIC_RCD_TYPE_META: {
+			struct sk_buff *skb = NULL;
+
+			if (likely(!(rcd &
+				     MPNIC_RCD_META_UNCORRECTABLE_ERR_MASK) &&
+				   !state->pkt.add_frag_failed))
+				skb = xdp_build_skb_from_buff(&state->pkt.buff);
+
+			if (likely(skb))
+				napi_gro_receive(&nv->napi, skb);
+			else
+				mpnic_put_pkt_buff(&state->pkt, true);
+
+			state->pkt.buff.data_hard_start = NULL;
+			packets++;
+			break;
+		}
+		}
+
+		raw_rcd++;
+		head++;
+
+		if (unlikely(!(head & rcq->size_mask))) {
+			done ^= cpu_to_le64(MPNIC_RCD_DONE);
+			raw_rcd = &rcq->desc[0];
+		}
+	}
+
+	rcq->head = head;
+
+	/* Allocate buffers, force dma_wmb(), and then start writing tails */
+	mpnic_fill_qt_bdqs(qt);
+
+	return packets;
+}
+
+static int mpnic_poll(struct napi_struct *napi, int budget)
+{
+	struct mpnic_napi_vector *nv = container_of(napi,
+						    struct mpnic_napi_vector,
+						    napi);
+	int i, j, work_done = 0;
+
+	for (i = 0; i < nv->txt_count; i++)
+		mpnic_clean_tcq(nv, &nv->qt[i], budget);
+
+	if (likely(budget))
+		for (j = 0; j < nv->rxt_count; j++, i++)
+			work_done += mpnic_clean_rcq(nv, &nv->qt[i],
+						     budget - work_done);
+
+	for (i = 0; i < nv->txt_count; i++)
+		mpnic_commit_cq_head(&nv->qt[i].cmpl);
+
+	if (work_done >= budget)
+		return budget;
+
+	if (likely(napi_complete_done(napi, work_done)))
+		mpnic_nv_irq_rearm(nv);
+
+	return work_done;
+}
+
+static irqreturn_t mpnic_msix_clean_rings(int __always_unused irq, void *data)
+{
+	struct mpnic_napi_vector *nv = data;
+
+	napi_schedule_irqoff(&nv->napi);
+
+	return IRQ_HANDLED;
+}
+
+static void mpnic_free_napi_vector(struct mpnic_net *mpn,
+				   struct mpnic_napi_vector *nv)
+{
+	int i, j;
+
+	for (i = 0; i < nv->txt_count; i++)
+		mpn->tx[nv->qt[i].sub0.q_idx] = NULL;
+
+	for (j = 0; j < nv->rxt_count; j++, i++)
+		mpn->rx[nv->qt[i].cmpl.q_idx] = NULL;
+
+	mpnic_free_irq(nv->mpd, nv->v_idx, nv);
+	netif_napi_del_locked(&nv->napi);
+	mpn->napi[nv->v_idx - MPNIC_NON_NAPI_VECTORS] = NULL;
+	kfree(nv);
+}
+
+void mpnic_free_napi_vectors(struct mpnic_net *mpn)
+{
+	int i;
+
+	for (i = 0; i < mpn->num_napi; i++)
+		if (mpn->napi[i])
+			mpnic_free_napi_vector(mpn, mpn->napi[i]);
+}
+
+static void mpnic_ring_init(struct mpnic_ring *ring, u32 __iomem *doorbell,
+			    int q_idx)
+{
+	ring->doorbell = doorbell;
+	ring->q_idx = q_idx;
+}
+
+static int mpnic_alloc_napi_vector(struct mpnic_dev *mpd,
+				   struct mpnic_net *mpn, unsigned int idx)
+{
+	u32 __iomem *uc_addr = READ_ONCE(mpd->uc_addr0);
+	struct mpnic_napi_vector *nv;
+	int err;
+
+	/* Doorbells are plain pointers into the register window, they have
+	 * no way of noticing that it went away.
+	 */
+	if (!uc_addr)
+		return -EIO;
+
+	nv = kzalloc_flex(*nv, qt, 2);
+	if (!nv)
+		return -ENOMEM;
+
+	nv->txt_count = 1;
+	nv->rxt_count = 1;
+	nv->mpd = mpd;
+	nv->dev = mpd->dev;
+	nv->v_idx = idx + MPNIC_NON_NAPI_VECTORS;
+
+	mpn->napi[idx] = nv;
+	netif_napi_add_config_locked(mpn->netdev, &nv->napi, mpnic_poll, idx);
+	netif_napi_set_irq_locked(&nv->napi,
+				  pci_irq_vector(to_pci_dev(mpd->dev),
+						 nv->v_idx));
+
+	snprintf(nv->name, sizeof(nv->name), "%s-TxRx-%u",
+		 mpn->netdev->name, idx);
+
+	err = mpnic_request_irq(mpd, nv->v_idx, mpnic_msix_clean_rings, 0,
+				nv->name, nv);
+	if (err)
+		goto err_napi_del;
+
+	mpnic_ring_init(&nv->qt[0].sub0, &uc_addr[MPNIC_TWQ_TAIL(idx, 0)], idx);
+	mpnic_ring_init(&nv->qt[0].cmpl, &uc_addr[MPNIC_TCQ_HEAD(idx)], idx);
+	mpn->tx[idx] = &nv->qt[0].sub0;
+
+	mpnic_ring_init(&nv->qt[1].sub0, &uc_addr[MPNIC_HPQ_TAIL(idx)], idx);
+	mpnic_ring_init(&nv->qt[1].sub1, &uc_addr[MPNIC_PPQ_TAIL(idx)], idx);
+	mpnic_ring_init(&nv->qt[1].cmpl, &uc_addr[MPNIC_RCQ_HEAD(idx)], idx);
+	mpn->rx[idx] = &nv->qt[1].cmpl;
+
+	return 0;
+
+err_napi_del:
+	netif_napi_del_locked(&nv->napi);
+	mpn->napi[idx] = NULL;
+	kfree(nv);
+	return err;
+}
+
+int mpnic_alloc_napi_vectors(struct mpnic_net *mpn)
+{
+	unsigned int i;
+	int err;
+
+	for (i = 0; i < mpn->num_napi; i++) {
+		err = mpnic_alloc_napi_vector(mpn->mpd, mpn, i);
+		if (err)
+			goto err_free_vectors;
+	}
+
+	return 0;
+
+err_free_vectors:
+	mpnic_free_napi_vectors(mpn);
+
+	return err;
+}
+
+static void mpnic_free_ring_resources(struct device *dev,
+				      struct mpnic_ring *ring)
+{
+	kvfree(ring->buffer);
+	ring->buffer = NULL;
+
+	/* If size is not set there are no descriptors present */
+	if (!ring->size)
+		return;
+
+	dma_free_coherent(dev, ring->size, ring->desc, ring->dma);
+	ring->size_mask = 0;
+	ring->size = 0;
+}
+
+static int mpnic_alloc_ring_desc(struct mpnic_net *mpn,
+				 struct mpnic_ring *ring, u32 count)
+{
+	struct device *dev = mpn->netdev->dev.parent;
+	size_t size;
+
+	size = ALIGN(array_size(sizeof(*ring->desc), count), 4096);
+
+	ring->desc = dma_alloc_coherent(dev, size, &ring->dma,
+					GFP_KERNEL | __GFP_NOWARN);
+	if (!ring->desc)
+		return -ENOMEM;
+
+	ring->size_mask = count - 1;
+	ring->size = size;
+
+	return 0;
+}
+
+static void mpnic_free_tx_qt_resources(struct mpnic_net *mpn,
+				       struct mpnic_q_triad *qt)
+{
+	struct device *dev = mpn->netdev->dev.parent;
+
+	mpnic_free_ring_resources(dev, &qt->cmpl);
+	mpnic_free_ring_resources(dev, &qt->sub0);
+}
+
+static int mpnic_alloc_tx_qt_resources(struct mpnic_net *mpn,
+				       struct mpnic_q_triad *qt)
+{
+	int err;
+
+	err = mpnic_alloc_ring_desc(mpn, &qt->sub0, mpn->txq_size);
+	if (err)
+		return err;
+
+	qt->sub0.tx_buf = kvzalloc_objs(*qt->sub0.tx_buf, mpn->txq_size,
+					GFP_KERNEL | __GFP_NOWARN);
+	if (!qt->sub0.tx_buf) {
+		err = -ENOMEM;
+		goto err_free_qt;
+	}
+
+	err = mpnic_alloc_ring_desc(mpn, &qt->cmpl, mpn->txq_size);
+	if (err)
+		goto err_free_qt;
+
+	return 0;
+
+err_free_qt:
+	mpnic_free_tx_qt_resources(mpn, qt);
+	return err;
+}
+
+static int
+mpnic_alloc_qt_page_pool(struct mpnic_net *mpn, struct mpnic_napi_vector *nv,
+			 struct mpnic_q_triad *qt)
+{
+	struct page_pool_params pp_params = {
+		.flags		= PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+		.pool_size	= min(mpn->hpq_size + mpn->ppq_size, 32768u),
+		.nid		= NUMA_NO_NODE,
+		.dev		= nv->dev,
+		.dma_dir	= DMA_FROM_DEVICE,
+		.max_len	= PAGE_SIZE,
+		.napi		= &nv->napi,
+		.netdev		= mpn->netdev,
+		.queue_idx	= qt->cmpl.q_idx,
+	};
+	struct page_pool *pp;
+
+	pp = page_pool_create(&pp_params);
+	if (IS_ERR(pp))
+		return PTR_ERR(pp);
+
+	qt->sub0.page_pool = pp;
+	page_pool_get(pp);
+	qt->sub1.page_pool = pp;
+
+	return 0;
+}
+
+static void mpnic_free_rx_qt_resources(struct mpnic_net *mpn,
+				       struct mpnic_q_triad *qt)
+{
+	struct device *dev = mpn->netdev->dev.parent;
+
+	mpnic_free_ring_resources(dev, &qt->cmpl);
+	mpnic_free_ring_resources(dev, &qt->sub1);
+	mpnic_free_ring_resources(dev, &qt->sub0);
+
+	if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) {
+		xdp_rxq_info_unreg(&qt->xdp_rxq);
+		page_pool_destroy(qt->sub1.page_pool);
+		page_pool_destroy(qt->sub0.page_pool);
+	}
+}
+
+static int mpnic_alloc_rx_qt_resources(struct mpnic_net *mpn,
+				       struct mpnic_napi_vector *nv,
+				       struct mpnic_q_triad *qt)
+{
+	int err;
+
+	err = mpnic_alloc_qt_page_pool(mpn, nv, qt);
+	if (err)
+		return err;
+
+	err = xdp_rxq_info_reg(&qt->xdp_rxq, mpn->netdev, qt->cmpl.q_idx,
+			       nv->napi.napi_id);
+	if (err)
+		goto err_free_page_pool;
+
+	err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL,
+					 qt->sub0.page_pool);
+	if (err)
+		goto err_unreg_rxq;
+
+	err = mpnic_alloc_ring_desc(mpn, &qt->sub0, mpn->hpq_size);
+	if (err)
+		goto err_unreg_mm;
+
+	qt->sub0.rx_buf = kvzalloc_objs(*qt->sub0.rx_buf, mpn->hpq_size,
+					GFP_KERNEL | __GFP_NOWARN);
+	if (!qt->sub0.rx_buf) {
+		err = -ENOMEM;
+		goto err_free_qt;
+	}
+
+	err = mpnic_alloc_ring_desc(mpn, &qt->sub1, mpn->ppq_size);
+	if (err)
+		goto err_free_qt;
+
+	qt->sub1.rx_buf = kvzalloc_objs(*qt->sub1.rx_buf, mpn->ppq_size,
+					GFP_KERNEL | __GFP_NOWARN);
+	if (!qt->sub1.rx_buf) {
+		err = -ENOMEM;
+		goto err_free_qt;
+	}
+
+	err = mpnic_alloc_ring_desc(mpn, &qt->cmpl, mpn->rcq_size);
+	if (err)
+		goto err_free_qt;
+
+	qt->cmpl.state = kvzalloc_obj(*qt->cmpl.state,
+				      GFP_KERNEL | __GFP_NOWARN);
+	if (!qt->cmpl.state) {
+		err = -ENOMEM;
+		goto err_free_qt;
+	}
+
+	return 0;
+
+err_free_qt:
+	mpnic_free_rx_qt_resources(mpn, qt);
+	return err;
+err_unreg_mm:
+	xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq);
+err_unreg_rxq:
+	xdp_rxq_info_unreg(&qt->xdp_rxq);
+err_free_page_pool:
+	page_pool_destroy(qt->sub1.page_pool);
+	page_pool_destroy(qt->sub0.page_pool);
+	return err;
+}
+
+static void mpnic_free_nv_resources(struct mpnic_net *mpn,
+				    struct mpnic_napi_vector *nv)
+{
+	int i, j;
+
+	for (i = 0; i < nv->txt_count; i++)
+		mpnic_free_tx_qt_resources(mpn, &nv->qt[i]);
+
+	for (j = 0; j < nv->rxt_count; j++, i++)
+		mpnic_free_rx_qt_resources(mpn, &nv->qt[i]);
+}
+
+static int mpnic_alloc_nv_resources(struct mpnic_net *mpn,
+				    struct mpnic_napi_vector *nv)
+{
+	int i, j, err;
+
+	for (i = 0; i < nv->txt_count; i++) {
+		err = mpnic_alloc_tx_qt_resources(mpn, &nv->qt[i]);
+		if (err)
+			goto err_free_qt_resources;
+	}
+
+	for (j = 0; j < nv->rxt_count; j++, i++) {
+		err = mpnic_alloc_rx_qt_resources(mpn, nv, &nv->qt[i]);
+		if (err)
+			goto err_free_qt_resources;
+	}
+
+	return 0;
+
+err_free_qt_resources:
+	while (i--) {
+		if (i < nv->txt_count)
+			mpnic_free_tx_qt_resources(mpn, &nv->qt[i]);
+		else
+			mpnic_free_rx_qt_resources(mpn, &nv->qt[i]);
+	}
+	return err;
+}
+
+void mpnic_free_resources(struct mpnic_net *mpn)
+{
+	int i;
+
+	for (i = 0; i < mpn->num_napi; i++)
+		mpnic_free_nv_resources(mpn, mpn->napi[i]);
+}
+
+int mpnic_alloc_resources(struct mpnic_net *mpn)
+{
+	int i, err;
+
+	for (i = 0; i < mpn->num_napi; i++) {
+		err = mpnic_alloc_nv_resources(mpn, mpn->napi[i]);
+		if (err)
+			goto err_free_resources;
+	}
+
+	return 0;
+
+err_free_resources:
+	while (i--)
+		mpnic_free_nv_resources(mpn, mpn->napi[i]);
+
+	return err;
+}
+
+static void mpnic_set_netif_napi(struct mpnic_napi_vector *nv,
+				 struct napi_struct *napi)
+{
+	int i, j;
+
+	for (i = 0; i < nv->txt_count; i++)
+		netif_queue_set_napi(nv->napi.dev, nv->qt[i].sub0.q_idx,
+				     NETDEV_QUEUE_TYPE_TX, napi);
+
+	for (j = 0; j < nv->rxt_count; j++, i++)
+		netif_queue_set_napi(nv->napi.dev, nv->qt[i].cmpl.q_idx,
+				     NETDEV_QUEUE_TYPE_RX, napi);
+}
+
+int mpnic_set_netif_queues(struct mpnic_net *mpn)
+{
+	int i, err;
+
+	err = netif_set_real_num_queues(mpn->netdev, mpn->num_tx_queues,
+					mpn->num_rx_queues);
+	if (err)
+		return err;
+
+	for (i = 0; i < mpn->num_napi; i++)
+		mpnic_set_netif_napi(mpn->napi[i], &mpn->napi[i]->napi);
+
+	return 0;
+}
+
+void mpnic_reset_netif_queues(struct mpnic_net *mpn)
+{
+	int i;
+
+	for (i = 0; i < mpn->num_napi; i++)
+		mpnic_set_netif_napi(mpn->napi[i], NULL);
+}
+
+static void mpnic_enable_twq(struct mpnic_dev *mpd, struct mpnic_ring *twq)
+{
+	u32 log_size = fls(twq->size_mask);
+	u32 i = twq->q_idx;
+
+	/* Reset head/tail */
+	mpnic_wr64(mpd, MPNIC_TWQ_CTL(i, 0), MPNIC_TWQ_CTL_RESET);
+	twq->tail = 0;
+	twq->head = 0;
+	twq->deferred_meta = -1;
+
+	/* Store descriptor ring address and size */
+	mpnic_wr64(mpd, MPNIC_TWQ_BASE_ADDR(i, 0), twq->dma);
+	mpnic_wr64(mpd, MPNIC_TWQ_SIZE(i, 0), log_size & MPNIC_TWQ_SIZE_SIZE);
+
+	mpnic_wr64(mpd, MPNIC_TWQ_CTL(i, 0), MPNIC_TWQ_CTL_ENABLE);
+}
+
+static void mpnic_enable_tcq(struct mpnic_dev *mpd,
+			     struct mpnic_napi_vector *nv,
+			     struct mpnic_ring *tcq)
+{
+	u32 log_size = fls(tcq->size_mask);
+	u32 i = tcq->q_idx;
+
+	/* Reset head/tail */
+	mpnic_wr64(mpd, MPNIC_TCQ_CTL(i), MPNIC_TCQ_CTL_RESET);
+	tcq->tail = 0;
+	tcq->head = 0;
+
+	/* Store descriptor ring address and size */
+	mpnic_wr64(mpd, MPNIC_TCQ_BASE_ADDR(i), tcq->dma);
+	mpnic_wr64(mpd, MPNIC_TCQ_SIZE(i), log_size & MPNIC_TCQ_SIZE_SIZE);
+
+	/* Store interrupt information for the completion queue */
+	mpnic_wr64(mpd, MPNIC_TIM_CTL(i), nv->v_idx);
+	mpnic_wr64(mpd, MPNIC_TIM_INTR_MASK(i), 0);
+
+	mpnic_wr64(mpd, MPNIC_TCQ_CTL(i), MPNIC_TCQ_CTL_ENABLE);
+}
+
+static void mpnic_enable_bdq(struct mpnic_dev *mpd, struct mpnic_ring *hpq,
+			     struct mpnic_ring *ppq)
+{
+	u32 hpq_log_size = fls(hpq->size_mask);
+	u32 ppq_log_size = fls(ppq->size_mask);
+	u32 i = hpq->q_idx;
+
+	/* Reset head/tail */
+	mpnic_wr64(mpd, MPNIC_BDQ_CTL(i), MPNIC_BDQ_CTL_RESET);
+	hpq->tail = 0;
+	hpq->head = 0;
+	ppq->tail = 0;
+	ppq->head = 0;
+
+	/* Store descriptor ring addresses and sizes */
+	mpnic_wr64(mpd, MPNIC_HPQ_BASE_ADDR(i), hpq->dma);
+	mpnic_wr64(mpd, MPNIC_HPQ_SIZE(i), hpq_log_size & MPNIC_HPQ_SIZE_SIZE);
+	mpnic_wr64(mpd, MPNIC_PPQ_BASE_ADDR(i), ppq->dma);
+	mpnic_wr64(mpd, MPNIC_PPQ_SIZE(i), ppq_log_size & MPNIC_PPQ_SIZE_SIZE);
+
+	mpnic_wr64(mpd, MPNIC_BDQ_CTL(i),
+		   MPNIC_BDQ_CTL_ENABLE | MPNIC_BDQ_CTL_ENABLE_PPQ);
+}
+
+static void mpnic_set_rde_cfg(struct mpnic_dev *mpd, struct mpnic_ring *rcq)
+{
+	BUILD_BUG_ON(FIELD_MAX(MPNIC_RDE_CFG_MIN_HEAD_ROOM) < MPNIC_RX_HROOM);
+	BUILD_BUG_ON(FIELD_MAX(MPNIC_RDE_CFG_MIN_TAIL_ROOM) < MPNIC_RX_TROOM);
+
+	mpnic_wr64(mpd, MPNIC_RDE_CFG(rcq->q_idx),
+		   FIELD_PREP(MPNIC_RDE_CFG_MIN_HEAD_ROOM, MPNIC_RX_HROOM) |
+		   FIELD_PREP(MPNIC_RDE_CFG_MIN_TAIL_ROOM, MPNIC_RX_TROOM) |
+		   FIELD_PREP(MPNIC_RDE_CFG_MAX_HEADER_BYTES,
+			      MPNIC_RX_MAX_HDR));
+}
+
+static void mpnic_enable_rcq(struct mpnic_dev *mpd,
+			     struct mpnic_napi_vector *nv,
+			     struct mpnic_ring *rcq)
+{
+	u32 log_size = fls(rcq->size_mask);
+	u32 i = rcq->q_idx;
+
+	mpnic_set_rde_cfg(mpd, rcq);
+
+	/* Reset head/tail */
+	mpnic_wr64(mpd, MPNIC_RCQ_CTL(i), MPNIC_RCQ_CTL_RESET);
+	rcq->head = 0;
+	rcq->tail = 0;
+
+	/* Store descriptor ring address and size */
+	mpnic_wr64(mpd, MPNIC_RCQ_BASE_ADDR(i), rcq->dma);
+	mpnic_wr64(mpd, MPNIC_RCQ_SIZE(i), log_size & MPNIC_RCQ_SIZE_SIZE);
+
+	/* Store interrupt information for the completion queue */
+	mpnic_wr64(mpd, MPNIC_RIM_CTL(i), nv->v_idx);
+	mpnic_wr64(mpd, MPNIC_RIM_INTR_MASK(i), 0);
+
+	mpnic_wr64(mpd, MPNIC_RCQ_CTL(i), MPNIC_RCQ_CTL_ENABLE);
+}
+
+void mpnic_enable(struct mpnic_net *mpn)
+{
+	struct mpnic_dev *mpd = mpn->mpd;
+	int i, j, t;
+
+	for (i = 0; i < mpn->num_napi; i++) {
+		struct mpnic_napi_vector *nv = mpn->napi[i];
+
+		for (t = 0; t < nv->txt_count; t++) {
+			mpnic_enable_twq(mpd, &nv->qt[t].sub0);
+			mpnic_enable_tcq(mpd, nv, &nv->qt[t].cmpl);
+		}
+
+		for (j = 0; j < nv->rxt_count; j++, t++) {
+			mpnic_enable_bdq(mpd, &nv->qt[t].sub0, &nv->qt[t].sub1);
+			mpnic_enable_rcq(mpd, nv, &nv->qt[t].cmpl);
+		}
+	}
+
+	mpnic_wrfl(mpd);
+}
+
+static void mpnic_disable_twq(struct mpnic_dev *mpd, struct mpnic_ring *txr)
+{
+	u64 twq_ctl = mpnic_rd64(mpd, MPNIC_TWQ_CTL(txr->q_idx, 0));
+
+	twq_ctl &= ~MPNIC_TWQ_CTL_ENABLE;
+	mpnic_wr64(mpd, MPNIC_TWQ_CTL(txr->q_idx, 0), twq_ctl);
+}
+
+static void mpnic_disable_tcq(struct mpnic_dev *mpd, struct mpnic_ring *txr)
+{
+	mpnic_wr64(mpd, MPNIC_TCQ_CTL(txr->q_idx), 0);
+	mpnic_wr64(mpd, MPNIC_TIM_INTR_MASK(txr->q_idx),
+		   MPNIC_TIM_INTR_MASK_MASK);
+}
+
+static void mpnic_disable_bdq(struct mpnic_dev *mpd, struct mpnic_ring *hpq)
+{
+	u64 bdq_ctl = mpnic_rd64(mpd, MPNIC_BDQ_CTL(hpq->q_idx));
+
+	bdq_ctl &= ~(MPNIC_BDQ_CTL_ENABLE | MPNIC_BDQ_CTL_ENABLE_PPQ);
+	mpnic_wr64(mpd, MPNIC_BDQ_CTL(hpq->q_idx), bdq_ctl);
+}
+
+static void mpnic_disable_rcq(struct mpnic_dev *mpd, struct mpnic_ring *rcq)
+{
+	mpnic_wr64(mpd, MPNIC_RCQ_CTL(rcq->q_idx), 0);
+	mpnic_wr64(mpd, MPNIC_RIM_INTR_MASK(rcq->q_idx),
+		   MPNIC_RIM_INTR_MASK_MASK);
+}
+
+void mpnic_disable(struct mpnic_net *mpn)
+{
+	struct mpnic_dev *mpd = mpn->mpd;
+	int i, j, t;
+
+	for (i = 0; i < mpn->num_napi; i++) {
+		struct mpnic_napi_vector *nv = mpn->napi[i];
+
+		for (t = 0; t < nv->txt_count; t++) {
+			mpnic_disable_twq(mpd, &nv->qt[t].sub0);
+			mpnic_disable_tcq(mpd, &nv->qt[t].cmpl);
+		}
+
+		for (j = 0; j < nv->rxt_count; j++, t++) {
+			mpnic_disable_bdq(mpd, &nv->qt[t].sub0);
+			mpnic_disable_rcq(mpd, &nv->qt[t].cmpl);
+		}
+	}
+
+	mpnic_wrfl(mpd);
+}
+
+struct mpnic_idle_regs {
+	u32 reg_base;
+	u8 reg_cnt;
+	char name[4];
+};
+
+static u32 mpnic_non_idle_queues(struct mpnic_dev *mpd,
+				 const struct mpnic_idle_regs *regs,
+				 unsigned int nregs)
+{
+	u32 non_idle_bitmap = 0;
+	unsigned int i, j;
+
+	for (i = 0; i < nregs; i++) {
+		for (j = 0; j < regs[i].reg_cnt; j++) {
+			if (mpnic_rd64(mpd, regs[i].reg_base + 2 * j) !=
+			    ~0ULL) {
+				non_idle_bitmap |= BIT(i);
+				break;
+			}
+		}
+	}
+
+	return non_idle_bitmap;
+}
+
+static void mpnic_idle_dump(struct mpnic_dev *mpd,
+			    const struct mpnic_idle_regs *regs,
+			    unsigned int nregs, u32 non_idle_bitmap, int err)
+{
+	unsigned int i, j;
+
+	dev_err(mpd->dev, "error waiting for queues idle %d\n", err);
+	for (i = 0; i < nregs; i++) {
+		if (!(non_idle_bitmap & BIT(i)))
+			continue;
+
+		dev_err(mpd->dev, "%s block not idle:\n", regs[i].name);
+		for (j = 0; j < regs[i].reg_cnt; j++)
+			dev_err(mpd->dev, "  0x%04x: %016llx\n",
+				regs[i].reg_base + 2 * j,
+				mpnic_rd64(mpd, regs[i].reg_base + 2 * j));
+	}
+}
+
+void mpnic_wait_all_queues_idle(struct mpnic_dev *mpd)
+{
+	static const struct mpnic_idle_regs queues[] = {
+		{ MPNIC_TWQ_IDLE(0), MPNIC_TWQ_IDLE_CNT, "TWQ" },
+		{ MPNIC_TQS_IDLE(0), MPNIC_TQS_IDLE_CNT, "TQS" },
+		{ MPNIC_TDE_IDLE(0), MPNIC_TDE_IDLE_CNT, "TDE" },
+		{ MPNIC_TCQ_IDLE(0), MPNIC_TCQ_IDLE_CNT, "TCQ" },
+		{ MPNIC_HPQ_IDLE(0), MPNIC_HPQ_IDLE_CNT, "HPQ" },
+		{ MPNIC_PPQ_IDLE(0), MPNIC_PPQ_IDLE_CNT, "PPQ" },
+		{ MPNIC_RCQ_IDLE(0), MPNIC_RCQ_IDLE_CNT, "RCQ" },
+	};
+	u32 non_idle_bitmap;
+	int err;
+
+	err = read_poll_timeout(mpnic_non_idle_queues, non_idle_bitmap,
+				!non_idle_bitmap, 20, 500000, false, mpd,
+				queues, ARRAY_SIZE(queues));
+	if (err)
+		mpnic_idle_dump(mpd, queues, ARRAY_SIZE(queues),
+				non_idle_bitmap, err);
+}
+
+static void mpnic_clean_bdq(struct mpnic_ring *bdq)
+{
+	unsigned int head = bdq->head;
+
+	while (head != bdq->tail) {
+		struct page *page = bdq->rx_buf[head];
+
+		page_pool_put_full_page(page->pp, page, false);
+
+		head++;
+		head &= bdq->size_mask;
+	}
+
+	bdq->head = head;
+}
+
+void mpnic_flush(struct mpnic_net *mpn)
+{
+	int i, j, t;
+
+	for (i = 0; i < mpn->num_napi; i++) {
+		struct mpnic_napi_vector *nv = mpn->napi[i];
+
+		for (t = 0; t < nv->txt_count; t++) {
+			struct mpnic_q_triad *qt = &nv->qt[t];
+			struct netdev_queue *txq;
+
+			/* Clean the work queue of unprocessed work */
+			mpnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail);
+
+			txq = netdev_get_tx_queue(mpn->netdev, qt->sub0.q_idx);
+			netdev_tx_reset_queue(txq);
+		}
+
+		for (j = 0; j < nv->rxt_count; j++, t++) {
+			struct mpnic_q_triad *qt = &nv->qt[t];
+			struct mpnic_rcq_state *state = qt->cmpl.state;
+
+			/* Release the partially assembled frame and the
+			 * pages the queues are still handing out.
+			 */
+			mpnic_put_pkt_buff(&state->pkt, false);
+			mpnic_flush_pg_ctxt(&state->hdr, false);
+			mpnic_flush_pg_ctxt(&state->payld, false);
+			memset(state, 0, sizeof(*state));
+
+			mpnic_clean_bdq(&qt->sub0);
+			mpnic_clean_bdq(&qt->sub1);
+		}
+	}
+}
+
+void mpnic_fill(struct mpnic_net *mpn)
+{
+	int i, j, t;
+
+	for (i = 0; i < mpn->num_napi; i++) {
+		struct mpnic_napi_vector *nv = mpn->napi[i];
+
+		for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) {
+			struct mpnic_q_triad *qt = &nv->qt[t];
+			struct mpnic_rcq_state *state = qt->cmpl.state;
+
+			/* Point the page contexts at an index the device
+			 * cannot report, so the first buffer coming out of
+			 * either queue is not taken for a page we hold.
+			 */
+			state->hdr.idx = UINT_MAX;
+			state->payld.idx = UINT_MAX;
+
+			mpnic_fill_qt_bdqs(qt);
+		}
+	}
+}
+
+void mpnic_napi_disable(struct mpnic_net *mpn)
+{
+	int i;
+
+	for (i = 0; i < mpn->num_napi; i++) {
+		napi_disable_locked(&mpn->napi[i]->napi);
+
+		mpnic_nv_irq_disable(mpn->napi[i]);
+	}
+}
+
+void mpnic_napi_enable(struct mpnic_net *mpn)
+{
+	int i;
+
+	for (i = 0; i < mpn->num_napi; i++)
+		napi_enable_locked(&mpn->napi[i]->napi);
+
+	/* Force the first interrupt on each vector to guarantee that any
+	 * completions posted during bringup are processed.
+	 */
+	for (i = 0; i < mpn->num_napi; i++)
+		mpnic_nv_irq_trigger(mpn->napi[i]);
+
+	mpnic_wrfl(mpn->mpd);
+}
diff --git a/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h
new file mode 100644
index 00000000000000..834eb9bfffd197
--- /dev/null
+++ b/drivers/net/ethernet/meta/mpnic/mpnic_txrx.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#ifndef _MPNIC_TXRX_H_
+#define _MPNIC_TXRX_H_
+
+#include <linux/if_ether.h>
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+#include <linux/types.h>
+#include <net/netdev_queues.h>
+#include <net/xdp.h>
+
+#include "mpnic.h"
+
+struct mpnic_net;
+
+/* Space we have to have available in a work queue to take a packet:
+ *	1 descriptor per page
+ *	+ 1 descriptor for the skb head
+ *	+ 1 descriptor for the metadata
+ *	+ 7 descriptors to keep the tail out of the head's cacheline
+ * If we cannot guarantee that we return NETDEV_TX_BUSY.
+ */
+#define MPNIC_MAX_SKB_DESC		(MAX_SKB_FRAGS + 9)
+#define MPNIC_TX_DESC_WAKEUP		(MPNIC_MAX_SKB_DESC * 2)
+
+#define MPNIC_MAX_NAPI_VECTORS		1024u
+
+/* Number of buffer descriptors the driver posts before ringing the
+ * doorbell. The device consumes whatever the doorbell points at, this is
+ * purely to keep the driver from writing the CSR for every descriptor.
+ */
+#define MPNIC_BDQ_BATCH_SIZE		64u
+
+#define MPNIC_TXQ_SIZE_DEFAULT		1024
+#define MPNIC_HPQ_SIZE_DEFAULT		256
+#define MPNIC_PPQ_SIZE_DEFAULT		256
+#define MPNIC_RCQ_SIZE_DEFAULT		1024
+
+/* Room the device has to leave in front of and behind every header so the
+ * driver can build an skb around it in place. The headroom is padded out
+ * so that consecutive headers in one page start 128 B aligned.
+ */
+#define MPNIC_RX_TROOM \
+	SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
+#define MPNIC_RX_HROOM \
+	(ALIGN(MPNIC_RX_TROOM + XDP_PACKET_HEADROOM, 128) - MPNIC_RX_TROOM)
+
+/* Headers longer than this are split off into the payload queue */
+#define MPNIC_RX_MAX_HDR		1536
+
+/* A page is handed out to many packets, each of which takes one reference.
+ * Rather than a locked increment per packet the driver takes a batch of
+ * references up front and returns whatever is left when the page is done.
+ */
+#define MPNIC_PAGECNT_BIAS_MAX		(PAGE_SIZE + 1)
+
+#define MPNIC_MAX_JUMBO_FRAME_SIZE	9742
+
+/* The page a buffer descriptor queue is currently handing out. Records
+ * how many of the references taken on it are still unused.
+ */
+struct mpnic_pg_ctxt {
+	struct page	*page;
+	long		pagecnt_bias;
+	u32		idx;
+};
+
+struct mpnic_pkt_ctxt {
+	struct xdp_buff buff;
+	bool add_frag_failed;
+};
+
+struct mpnic_rcq_state {
+	struct mpnic_pkt_ctxt pkt;
+	struct mpnic_pg_ctxt hdr;
+	struct mpnic_pg_ctxt payld;
+};
+
+struct mpnic_ring {
+	union {
+		struct mpnic_rcq_state *state;	/* RCQ */
+		struct page **rx_buf;		/* BDQ */
+		void **tx_buf;			/* TWQ */
+		void *buffer;			/* Generic pointer */
+	};
+
+	u32 __iomem *doorbell;		/* Pointer to CSR space for ring */
+	__le64 *desc;			/* Descriptor ring memory */
+	u16 size_mask;			/* Size of ring in descriptors - 1 */
+	u16 q_idx;			/* Hardware queue index */
+
+	u32 head, tail;			/* Head/Tail of ring */
+
+	union {
+		/* BDQ only */
+		struct page_pool *page_pool;
+
+		/* TWQ only, index of the metadata descriptor of the last
+		 * packet placed in the ring without ringing the doorbell,
+		 * -1 if the doorbell is in sync with the tail.
+		 */
+		s32 deferred_meta;
+	};
+
+	/* Slow path fields follow */
+	dma_addr_t dma;			/* Phys addr of descriptor memory */
+	size_t size;			/* Size of descriptor ring in memory */
+};
+
+/* The device pairs two work queues with one completion queue. On the Rx
+ * side they are the header and the payload buffer descriptor queues; on
+ * the Tx side only the first one is used for now, the second one becomes
+ * the XDP ring.
+ */
+struct mpnic_q_triad {
+	struct xdp_rxq_info xdp_rxq;
+	struct mpnic_ring sub0, sub1, cmpl;
+};
+
+struct mpnic_napi_vector {
+	struct napi_struct napi;
+	struct device *dev;		/* Device for DMA unmapping */
+	struct mpnic_dev *mpd;
+
+	u16 v_idx;
+	u16 txt_count;
+	u16 rxt_count;
+
+	char name[IFNAMSIZ + 11];
+
+	struct mpnic_q_triad qt[];
+};
+
+netdev_tx_t mpnic_xmit_frame(struct sk_buff *skb, struct net_device *dev);
+int mpnic_alloc_napi_vectors(struct mpnic_net *mpn);
+void mpnic_free_napi_vectors(struct mpnic_net *mpn);
+int mpnic_alloc_resources(struct mpnic_net *mpn);
+void mpnic_free_resources(struct mpnic_net *mpn);
+int mpnic_set_netif_queues(struct mpnic_net *mpn);
+void mpnic_reset_netif_queues(struct mpnic_net *mpn);
+void mpnic_napi_enable(struct mpnic_net *mpn);
+void mpnic_napi_disable(struct mpnic_net *mpn);
+void mpnic_enable(struct mpnic_net *mpn);
+void mpnic_disable(struct mpnic_net *mpn);
+void mpnic_wait_all_queues_idle(struct mpnic_dev *mpd);
+void mpnic_flush(struct mpnic_net *mpn);
+void mpnic_fill(struct mpnic_net *mpn);
+
+#endif /* _MPNIC_TXRX_H_ */
diff --git a/drivers/net/ethernet/microchip/lan865x/lan865x.c b/drivers/net/ethernet/microchip/lan865x/lan865x.c
index 26a2761332a5a1..127afb9e9f1416 100644
--- a/drivers/net/ethernet/microchip/lan865x/lan865x.c
+++ b/drivers/net/ethernet/microchip/lan865x/lan865x.c
@@ -347,8 +347,8 @@ static int lan865x_probe(struct spi_device *spi)
 	INIT_WORK(&priv->multicast_work, lan865x_multicast_work_handler);
 
 	priv->tc6 = oa_tc6_init(spi, netdev, NULL);
-	if (!priv->tc6) {
-		ret = -ENODEV;
+	if (IS_ERR(priv->tc6)) {
+		ret = PTR_ERR(priv->tc6);
 		goto free_netdev;
 	}
 
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c
index a16ec8136d6d63..5953e04ef14e6f 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c
@@ -673,6 +673,9 @@ void sparx5_ptp_deinit(struct sparx5 *sparx5)
 	struct sparx5_port *port;
 	int i;
 
+	if (!sparx5->ptp)
+		return;
+
 	if (sparx5->ptp_irq >= 0) {
 		disable_irq(sparx5->ptp_irq);
 		sparx5->ptp_irq = -ENXIO;
diff --git a/drivers/net/ethernet/microchip/vcap/Kconfig b/drivers/net/ethernet/microchip/vcap/Kconfig
index 97f43fd4473f4f..1af30a358a156b 100644
--- a/drivers/net/ethernet/microchip/vcap/Kconfig
+++ b/drivers/net/ethernet/microchip/vcap/Kconfig
@@ -40,7 +40,6 @@ config VCAP_KUNIT_TEST
 	bool "KUnit test for VCAP library" if !KUNIT_ALL_TESTS
 	depends on KUNIT
 	depends on KUNIT=y && VCAP=y && y
-	select DEBUG_FS
 	default KUNIT_ALL_TESTS
 	help
 	  This builds unit tests for the VCAP library.
diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api.c b/drivers/net/ethernet/microchip/vcap/vcap_api.c
index 788c0728d76368..0f8d98f2094d0e 100644
--- a/drivers/net/ethernet/microchip/vcap/vcap_api.c
+++ b/drivers/net/ethernet/microchip/vcap/vcap_api.c
@@ -3527,8 +3527,8 @@ int vcap_filter_rule_keys(struct vcap_rule *rule,
 				if (key == keylist[idx]) {
 					list_del(&ckf->ctrl.list);
 					kfree(ckf);
-					idx++;
 					err = 0;
+					break;
 				}
 		}
 	}
diff --git a/drivers/net/ethernet/microsoft/Kconfig b/drivers/net/ethernet/microsoft/Kconfig
index e9be18c92ca5f2..c4ad3d85875a67 100644
--- a/drivers/net/ethernet/microsoft/Kconfig
+++ b/drivers/net/ethernet/microsoft/Kconfig
@@ -18,7 +18,7 @@ if NET_VENDOR_MICROSOFT
 config MICROSOFT_MANA
 	tristate "Microsoft Azure Network Adapter (MANA) support"
 	depends on PCI_MSI
-	depends on X86_64 || (ARM64 && !CPU_BIG_ENDIAN)
+	depends on X86_64 || ARM64
 	depends on PCI_HYPERV
 	select AUXILIARY_BUS
 	select DIMLIB
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index 8e9bfc1d6a2ae1..740005782fa3cf 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -559,11 +559,13 @@ static int mana_gd_disable_queue(struct gdma_queue *queue)
 
 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,
 				  enum gdma_queue_type q_type, u32 qid,
-				  u32 tail_ptr, u8 num_req)
+				  u32 tail_ptr, u8 num_req, u8 client_offset)
 {
 	void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index;
 	union gdma_doorbell_entry e = {};
 
+	addr += client_offset;
+
 	switch (q_type) {
 	case GDMA_EQ:
 		e.eq.id = qid;
@@ -623,10 +625,18 @@ void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue)
 	 * wqe_cnt for Receive Queues. This value is not used in Send Queues.
 	 */
 	mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type,
-			      queue->id, queue->head * GDMA_WQE_BU_SIZE, 0);
+			      queue->id, queue->head * GDMA_WQE_BU_SIZE, 0, 0);
 }
 EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell, "NET_MANA");
 
+void mana_gd_wq_ring_doorbell_ext(struct gdma_context *gc, struct gdma_queue *queue,
+				  u32 tail_ptr, u8 wqe_cnt, u8 client_offset)
+{
+	mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type,
+			      queue->id, tail_ptr, wqe_cnt, client_offset);
+}
+EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell_ext, "NET_MANA");
+
 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit)
 {
 	struct gdma_context *gc = cq->gdma_dev->gdma_context;
@@ -636,7 +646,7 @@ void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit)
 	u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS);
 
 	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id,
-			      head, arm_bit);
+			      head, arm_bit, 0);
 }
 EXPORT_SYMBOL_NS(mana_gd_ring_cq, "NET_MANA");
 
@@ -653,7 +663,7 @@ void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
 		dim_val |= MANA_INTR_MODR_USEC_VLD;
 
 	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, GDMA_DIM, cq->id,
-			      dim_val, mod_comps_vld);
+			      dim_val, mod_comps_vld, 0);
 }
 EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA");
 
@@ -1023,7 +1033,7 @@ static void mana_gd_process_eq_events(void *arg)
 	head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS);
 
 	mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id,
-			      head, SET_ARM_BIT);
+			      head, SET_ARM_BIT, 0);
 }
 
 static int mana_gd_register_irq(struct gdma_queue *queue,
@@ -1738,6 +1748,24 @@ static void mana_gd_write_sgl(struct gdma_queue *wq, u32 sgl_offset,
 	memcpy(mana_gd_ring_ptr(wq, sgl_offset), address, sgl_size);
 }
 
+static void mana_gd_write_ib_sgl(struct gdma_queue *wq, u32 sgl_offset,
+				 const struct gdma_wqe_request *wqe_req)
+{
+	const struct ib_sge *sge = wqe_req->ib_sgl;
+	struct gdma_sge *gdma_sgl;
+	u32 i;
+
+	for (i = 0; i < wqe_req->num_sge; ++i, ++sge) {
+		gdma_sgl = mana_gd_ring_ptr(wq, sgl_offset);
+		gdma_sgl->address = sge->addr;
+		gdma_sgl->size = sge->length;
+		gdma_sgl->mem_key = sge->lkey;
+		sgl_offset += sizeof(*gdma_sgl);
+		if (sgl_offset == wq->queue_size)
+			sgl_offset = 0;
+	}
+}
+
 int mana_gd_post_work_request(struct gdma_queue *wq,
 			      const struct gdma_wqe_request *wqe_req,
 			      struct gdma_posted_wqe_info *wqe_info)
@@ -1779,10 +1807,12 @@ int mana_gd_post_work_request(struct gdma_queue *wq,
 	if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq))
 		return -ENOSPC;
 
-	if (wqe_info)
+	head = wq->head;
+	if (wqe_info) {
 		wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE;
+		wqe_info->wqe_offset = head;
+	}
 
-	head = wq->head;
 	wqe_offset = (head * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1);
 	wqe_ptr = mana_gd_get_wqe_ptr(wq, head);
 	oob_len = mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size,
@@ -1792,7 +1822,10 @@ int mana_gd_post_work_request(struct gdma_queue *wq,
 	if (sgl_offset >= wq->queue_size)
 		sgl_offset -= wq->queue_size;
 
-	mana_gd_write_sgl(wq, sgl_offset, wqe_req);
+	if (wqe_req->flags & GDMA_WR_IB_SGL)
+		mana_gd_write_ib_sgl(wq, sgl_offset, wqe_req);
+	else
+		mana_gd_write_sgl(wq, sgl_offset, wqe_req);
 
 	wq->head += wqe_size / GDMA_WQE_BU_SIZE;
 
diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c
index 263e7c4e293418..3bca4b683134b8 100644
--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c
+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c
@@ -170,12 +170,12 @@ static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
 			hwc->txq->msg_buf->gpa_mkey = val;
 			break;
 
-		case HWC_INIT_DATA_PF_DEST_RQ_ID:
-			hwc->pf_dest_vrq_id = val;
+		case HWC_INIT_DATA_DEST_RQ_ID:
+			hwc->dest_vrq_id = val;
 			break;
 
-		case HWC_INIT_DATA_PF_DEST_CQ_ID:
-			hwc->pf_dest_vrcq_id = val;
+		case HWC_INIT_DATA_DEST_CQ_ID:
+			hwc->dest_vrcq_id = val;
 			break;
 		}
 
@@ -855,13 +855,12 @@ void mana_hwc_destroy_channel(struct gdma_context *gc)
 int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 			  const void *req, u32 resp_len, void *resp)
 {
-	struct gdma_context *gc = hwc->gdma_dev->gdma_context;
 	struct hwc_work_request *tx_wr;
 	struct hwc_wq *txq = hwc->txq;
 	struct gdma_req_hdr *req_msg;
 	struct hwc_caller_ctx *ctx;
-	u32 dest_vrcq = 0;
-	u32 dest_vrq = 0;
+	u32 dest_vrcq;
+	u32 dest_vrq;
 	u32 command;
 	u16 msg_id;
 	int err;
@@ -890,10 +889,13 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 	tx_wr->msg_size = req_len;
 	command = req_msg->req.msg_type;
 
-	if (gc->is_pf) {
-		dest_vrq = hwc->pf_dest_vrq_id;
-		dest_vrcq = hwc->pf_dest_vrcq_id;
-	}
+	/* The hardware reports the HWC destination queues through
+	 * HWC_INIT_DATA_DEST_RQ_ID and HWC_INIT_DATA_DEST_CQ_ID, and
+	 * always supplies values that are valid for this function, so no
+	 * PF-specific handling is needed here.
+	 */
+	dest_vrq = hwc->dest_vrq_id;
+	dest_vrcq = hwc->dest_vrcq_id;
 
 	err = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false);
 	if (err) {
diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
index 5c9961ee9747ab..9ef42b74048b2c 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
@@ -254,7 +254,6 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
 {
 	struct netlink_ext_ack *extack = bpf->extack;
-	int ret;
 
 	switch (bpf->command) {
 	case XDP_SETUP_PROG:
@@ -263,6 +262,4 @@ int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
 	default:
 		return -EOPNOTSUPP;
 	}
-
-	return ret;
 }
diff --git a/drivers/net/ethernet/nebula-matrix/Kconfig b/drivers/net/ethernet/nebula-matrix/Kconfig
new file mode 100644
index 00000000000000..dd0e91d14131f8
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/Kconfig
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Nebula-matrix network device configuration
+#
+
+config NET_VENDOR_NEBULA_MATRIX
+	bool "Nebula-matrix devices"
+	default y
+	help
+	  If you have a network (Ethernet) card belonging to this class, say Y.
+	  Note that the answer to this question doesn't directly affect the
+	  kernel: saying N will just cause the configurator to skip all
+	  the questions about Nebula-matrix cards. If you say Y, you will be asked
+	  for your specific card in the following questions.
+
+if NET_VENDOR_NEBULA_MATRIX
+
+config NBL
+	tristate "Nebula-matrix Ethernet Controller m18110/m18000 support"
+	depends on PCI && (64BIT || COMPILE_TEST) && !CPU_BIG_ENDIAN
+	help
+	  This driver provides the core infrastructure for Nebula-matrix
+	  Ethernet Controller m18110/m18000 Family of devices, including
+	  PCI enumeration, firmware mailbox and channel communication.
+
+	  More specific information on configuring the driver is in
+	  <file:Documentation/networking/device_drivers/ethernet/nebula-matrix/nbl.rst>.
+
+	  To compile this driver as a module, choose M here. The module
+	  will be called nbl.
+
+endif # NET_VENDOR_NEBULA_MATRIX
diff --git a/drivers/net/ethernet/nebula-matrix/Makefile b/drivers/net/ethernet/nebula-matrix/Makefile
new file mode 100644
index 00000000000000..42cdf2db8f0cf6
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the Nebula-matrix network device drivers.
+#
+
+obj-$(CONFIG_NBL) += nbl/
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/Makefile b/drivers/net/ethernet/nebula-matrix/nbl/Makefile
new file mode 100644
index 00000000000000..cc060cf8bf7596
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2026 Nebula Matrix Limited.
+
+obj-$(CONFIG_NBL) := nbl.o
+
+nbl-objs +=	nbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \
+		nbl_main.o
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
new file mode 100644
index 00000000000000..1cd6587a8fcb66
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_CORE_H_
+#define _NBL_CORE_H_
+
+#include <linux/pci.h>
+#include "nbl_include/nbl_include.h"
+#include "nbl_include/nbl_def_common.h"
+
+enum {
+	NBL_CAP_HAS_NET_BIT,
+};
+
+struct nbl_core {
+	struct nbl_hw_mgt *hw_mgt;
+};
+
+struct nbl_adapter {
+	struct pci_dev *pdev;
+	struct nbl_core core;
+	struct nbl_common_info common;
+};
+
+struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,
+				  struct nbl_init_param *param);
+void nbl_core_remove(struct nbl_adapter *adapter);
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
new file mode 100644
index 00000000000000..cf40ddc4519211
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+#include <linux/device.h>
+#include <linux/pci.h>
+#include <linux/bits.h>
+#include <linux/io.h>
+#include "nbl_hw_leonis.h"
+
+/* Structure starts here, adding an op should not modify anything below */
+static struct nbl_hw_mgt *nbl_hw_setup_hw_mgt(struct nbl_common_info *common)
+{
+	struct device *dev = common->dev;
+	struct nbl_hw_mgt *hw_mgt;
+
+	hw_mgt = devm_kzalloc(dev, sizeof(*hw_mgt), GFP_KERNEL);
+	if (!hw_mgt)
+		return ERR_PTR(-ENOMEM);
+
+	hw_mgt->common = common;
+
+	return hw_mgt;
+}
+
+static int nbl_pcim_request_selected_bars(struct pci_dev *pdev, u32 mask,
+					  const char *name)
+{
+	int bar;
+	int ret;
+
+	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
+		if (!(mask & BIT(bar)))
+			continue;
+		ret = pcim_request_region(pdev, bar, name);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+int nbl_hw_init_leonis(struct nbl_adapter *adapter)
+{
+	resource_size_t expect_sz = NBL_MEM_BAR_TOTAL_SIZE;
+	struct nbl_common_info *common = &adapter->common;
+	struct pci_dev *pdev = common->pdev;
+	struct nbl_hw_mgt *hw_mgt = NULL;
+	resource_size_t bar_len;
+	resource_size_t hw_size;
+	u32 bar_mask;
+	int ret;
+
+	hw_mgt = nbl_hw_setup_hw_mgt(common);
+	if (IS_ERR(hw_mgt)) {
+		ret = PTR_ERR(hw_mgt);
+		goto setup_mgt_fail;
+	}
+	bar_mask = BIT(NBL_MEMORY_BAR) | BIT(NBL_MAILBOX_BAR);
+	ret = nbl_pcim_request_selected_bars(pdev, bar_mask, NBL_DRIVER_NAME);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"Request memory bar failed, err = %d\n",
+			ret);
+		goto setup_mgt_fail;
+	}
+
+	bar_len = pci_resource_len(pdev, NBL_MEMORY_BAR);
+	if (!(pci_resource_flags(pdev, NBL_MEMORY_BAR) & IORESOURCE_MEM)) {
+		dev_err(&pdev->dev, "MEMORY BAR is not memory resource\n");
+		ret = -EINVAL;
+		goto setup_mgt_fail;
+	}
+	if (common->has_ctrl) {
+		/*
+		 * Hardware layout: MEMORY BAR total size is 64M.
+		 * The tail NBL_RDMA_NOTIFY_LEN bytes of the 64M BAR are
+		 * reserved exclusively for RDMA notify hardware.
+		 * Ethernet driver must avoid mapping this reserved tail
+		 * to prevent x86 PAT aliasing conflict between eth net
+		 * mapping and RDMA driver WC mapping. Mapping starts
+		 * at BAR offset 0.
+		 *
+		 * Skip trailing NBL_RDMA_NOTIFY_LEN bytes at BAR tail.
+		 * Round size down to page boundary to avoid ioremap
+		 * rounding up and accidentally including RDMA reserved
+		 * region when PAGE_SIZE > 8KiB.
+		 */
+		if (bar_len < NBL_MEM_BAR_TOTAL_SIZE) {
+			dev_err(&pdev->dev,
+				"MEMORY BAR len %pa smaller than expected %pa\n",
+				&bar_len, &expect_sz);
+			ret = -EINVAL;
+			goto setup_mgt_fail;
+		}
+		hw_size = PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE -
+					  NBL_RDMA_NOTIFY_LEN);
+		hw_mgt->hw_addr =
+			pcim_iomap(pdev, NBL_MEMORY_BAR,
+				   hw_size);
+	} else {
+		if (bar_len < NBL_REG_NET_ONLY_LEN) {
+			dev_err(&pdev->dev,
+				"MEMORY BAR len %pa too small for net only reg space\n",
+				&bar_len);
+			ret = -EINVAL;
+			goto setup_mgt_fail;
+		}
+		hw_size = NBL_REG_NET_ONLY_LEN;
+		hw_mgt->hw_addr = pcim_iomap(pdev, NBL_MEMORY_BAR,
+					     hw_size);
+	}
+	if (!hw_mgt->hw_addr) {
+		dev_err(&pdev->dev, "MEMORY BAR pcim_iomap failed\n");
+		ret = -EIO;
+		goto setup_mgt_fail;
+	}
+
+	bar_len = pci_resource_len(pdev, NBL_MAILBOX_BAR);
+	if (!(pci_resource_flags(pdev, NBL_MAILBOX_BAR) & IORESOURCE_MEM)) {
+		dev_err(&pdev->dev, "MAILBOX BAR is not memory resource\n");
+		ret = -EINVAL;
+		goto setup_mgt_fail;
+	}
+	if (bar_len < NBL_BAR2_MAX_LEN) {
+		dev_err(&pdev->dev, "MAILBOX BAR length %pa too small\n",
+			&bar_len);
+		ret = -EINVAL;
+		goto setup_mgt_fail;
+	}
+	hw_mgt->mailbox_bar_hw_addr = pcim_iomap(pdev, NBL_MAILBOX_BAR,
+						 bar_len);
+	if (!hw_mgt->mailbox_bar_hw_addr) {
+		dev_err(&pdev->dev, "MAILBOX BAR pcim_iomap failed\n");
+		ret = -EIO;
+		goto setup_mgt_fail;
+	}
+
+	hw_mgt->mailbox_bar_size = bar_len;
+
+	adapter->core.hw_mgt = hw_mgt;
+
+	return 0;
+
+setup_mgt_fail:
+	return ret;
+}
+
+void nbl_hw_remove_leonis(struct nbl_adapter *adapter)
+{
+	/* All BAR mappings & PCI regions are managed by pcim/devres,
+	 * no manual iounmap / release required
+	 */
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h
new file mode 100644
index 00000000000000..1f9e509dc6312f
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_HW_LEONIS_H_
+#define _NBL_HW_LEONIS_H_
+
+#include <linux/types.h>
+
+#include "../../nbl_include/nbl_include.h"
+#include "../nbl_hw_reg.h"
+
+#define NBL_BAR2_MAX_LEN		0x300
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h
new file mode 100644
index 00000000000000..e281109f502e67
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_HW_REG_H_
+#define _NBL_HW_REG_H_
+
+#include <linux/types.h>
+
+#include "../nbl_include/nbl_def_hw.h"
+#include "../nbl_include/nbl_def_common.h"
+#include "../nbl_core.h"
+
+#define NBL_MEMORY_BAR				0
+#define NBL_MAILBOX_BAR				2
+#define NBL_RDMA_NOTIFY_LEN			(8ULL << 10)
+#define NBL_REG_NET_ONLY_LEN			(8ULL << 10)
+/*
+ * PCI MEMORY BAR total size: 64MiB.
+ */
+#define NBL_MEM_BAR_TOTAL_SIZE			(64ULL << 20)
+
+struct nbl_hw_mgt {
+	struct nbl_common_info *common;
+	u8 __iomem *hw_addr;
+	u8 __iomem *mailbox_bar_hw_addr;
+	resource_size_t mailbox_bar_size;
+};
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h
new file mode 100644
index 00000000000000..ea646d1efc2dfb
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DEF_COMMON_H_
+#define _NBL_DEF_COMMON_H_
+
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/device.h>
+#include "nbl_include.h"
+
+struct nbl_common_info {
+	struct pci_dev *pdev;
+	struct device *dev;
+	u16 vsi_id;
+	u8 eth_id;
+	u8 logic_eth_id;
+	u8 eth_num;
+
+	u8 function;
+	u8 devid;
+	u8 bus;
+	u8 hw_bus;
+
+	u8 has_ctrl;
+	u8 has_net;
+};
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
new file mode 100644
index 00000000000000..ecbf440e436667
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DEF_HW_H_
+#define _NBL_DEF_HW_H_
+
+#include <linux/types.h>
+
+struct nbl_hw_mgt;
+struct nbl_adapter;
+
+int nbl_hw_init_leonis(struct nbl_adapter *adapter);
+void nbl_hw_remove_leonis(struct nbl_adapter *adapter);
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h
new file mode 100644
index 00000000000000..14e7b19f9a4cf8
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_INCLUDE_H_
+#define _NBL_INCLUDE_H_
+
+#include <linux/types.h>
+
+/*  ------  Basic definitions  -------  */
+#define NBL_DRIVER_NAME					"nbl"
+struct nbl_func_caps {
+	u32 has_ctrl:1;
+	u32 has_net:1;
+	u32 rsv:30;
+};
+
+struct nbl_init_param {
+	struct nbl_func_caps caps;
+};
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c
new file mode 100644
index 00000000000000..f2552bc7329376
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#include <linux/device.h>
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/bits.h>
+#include "nbl_include/nbl_include.h"
+#include "nbl_include/nbl_def_hw.h"
+#include "nbl_include/nbl_def_common.h"
+#include "nbl_core.h"
+
+struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,
+				  struct nbl_init_param *param)
+{
+	struct nbl_common_info *common;
+	struct nbl_adapter *adapter;
+	int ret;
+
+	adapter = devm_kzalloc(&pdev->dev, sizeof(*adapter), GFP_KERNEL);
+	if (!adapter)
+		return ERR_PTR(-ENOMEM);
+
+	adapter->pdev = pdev;
+	common = &adapter->common;
+
+	common->pdev = pdev;
+	common->dev = &pdev->dev;
+	common->has_ctrl = param->caps.has_ctrl;
+	common->has_net = param->caps.has_net;
+	common->function = PCI_FUNC(pdev->devfn);
+	common->devid = PCI_SLOT(pdev->devfn);
+	common->bus = pdev->bus->number;
+
+	ret = nbl_hw_init_leonis(adapter);
+	if (ret)
+		goto hw_init_fail;
+
+	return adapter;
+hw_init_fail:
+	return ERR_PTR(ret);
+}
+
+void nbl_core_remove(struct nbl_adapter *adapter)
+{
+	nbl_hw_remove_leonis(adapter);
+}
+
+static void nbl_get_func_param(struct pci_dev *pdev, kernel_ulong_t driver_data,
+			       struct nbl_init_param *param)
+{
+	param->caps.has_net = !!(driver_data & BIT(NBL_CAP_HAS_NET_BIT));
+
+	/*
+	 * Hardware fixed rule: physical PF0 is the only management PF with
+	 * global ctrl capability. All PFs share identical PCI device ID, so
+	 * distinguish control PF via physical function ID.
+	 *
+	 * Hardware & firmware design FORBID passing any PF through to virtual
+	 * machines, there is no scenario where a non-management PF appears
+	 * as Func 0 inside guest. Thus using PCI_FUNC(pdev->devfn) to identify
+	 *  control PF is safe on our platform.
+	 */
+	if ((PCI_FUNC(pdev->devfn) == 0) && !pdev->is_virtfn)
+		param->caps.has_ctrl = 1;
+}
+
+static int nbl_probe(struct pci_dev *pdev,
+		     const struct pci_device_id *id)
+{
+	struct nbl_init_param param = { { 0 } };
+	struct device *dev = &pdev->dev;
+	struct nbl_adapter *adapter;
+	int err;
+
+	err = pcim_enable_device(pdev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to enable PCI dev, err=%d\n", err);
+		return err;
+	}
+
+	nbl_get_func_param(pdev, id->driver_data, &param);
+	/* never return fail when DMA_BIT_MASK(64) */
+	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
+
+	pci_set_master(pdev);
+
+	adapter = nbl_core_init(pdev, &param);
+	if (IS_ERR(adapter)) {
+		dev_err(dev, "Nbl adapter init fail: %pe\n", adapter);
+		err = PTR_ERR(adapter);
+		goto adapter_init_err;
+	}
+	pci_set_drvdata(pdev, adapter);
+	return 0;
+adapter_init_err:
+	pci_clear_master(pdev);
+	return err;
+}
+
+static void nbl_remove(struct pci_dev *pdev)
+{
+	struct nbl_adapter *adapter = pci_get_drvdata(pdev);
+
+	if (!adapter)
+		return;
+	pci_set_drvdata(pdev, NULL);
+	nbl_core_remove(adapter);
+
+	pci_clear_master(pdev);
+}
+
+/*
+ * PCI Device IDs for Leonis/NBL Network Controllers
+ *
+ * Vendor ID: 0x1F0F
+ * SNIC v3r1 product Device IDs range: 0x3403-0x3412
+ */
+#define NBL_VENDOR_ID				0x1F0F
+
+#define NBL_DEVICE_ID_M18110			0x3403
+#define NBL_DEVICE_ID_M18110_LX			0x3404
+#define NBL_DEVICE_ID_M18110_BASE_T		0x3405
+#define NBL_DEVICE_ID_M18110_LX_BASE_T		0x3406
+#define NBL_DEVICE_ID_M18110_OCP		0x3407
+#define NBL_DEVICE_ID_M18110_LX_OCP		0x3408
+#define NBL_DEVICE_ID_M18110_BASE_T_OCP		0x3409
+#define NBL_DEVICE_ID_M18110_LX_BASE_T_OCP	0x340a
+#define NBL_DEVICE_ID_M18000			0x340b
+#define NBL_DEVICE_ID_M18000_LX			0x340c
+#define NBL_DEVICE_ID_M18000_BASE_T		0x340d
+#define NBL_DEVICE_ID_M18000_LX_BASE_T		0x340e
+#define NBL_DEVICE_ID_M18000_OCP		0x340f
+#define NBL_DEVICE_ID_M18000_LX_OCP		0x3410
+#define NBL_DEVICE_ID_M18000_BASE_T_OCP		0x3411
+#define NBL_DEVICE_ID_M18000_LX_BASE_T_OCP	0x3412
+
+/* All below IDs belong to Leonis ASIC family, different form-factor variants,
+ * share the same hardware initialization flow without differentiated ops.
+ */
+static const struct pci_device_id nbl_id_table[] = {
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_BASE_T),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX_BASE_T),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_BASE_T_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX_BASE_T_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_LX),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_BASE_T),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_LX_BASE_T),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_LX_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_BASE_T_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_LX_BASE_T_OCP),
+	  .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },
+	/* required as sentinel */
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, nbl_id_table);
+
+static struct pci_driver nbl_driver = {
+	.name = NBL_DRIVER_NAME,
+	.id_table = nbl_id_table,
+	.probe = nbl_probe,
+	.remove = nbl_remove,
+};
+
+module_pci_driver(nbl_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Nebula Matrix Network Driver");
+MODULE_AUTHOR("Illusion Wang <illusion.wang@nebula-matrix.com>");
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 1caa87da72b51c..9ddbfe9d601947 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -105,7 +105,7 @@ static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env)
 		unsigned int soff;
 
 		soff = -(off + i) - 1;
-		stack_entry = &state->stack[soff / BPF_REG_SIZE];
+		stack_entry = bpf_stack_slot(state, soff / BPF_REG_SIZE);
 		if (stack_entry->slot_type[soff % BPF_REG_SIZE] == STACK_ZERO)
 			continue;
 
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
index 9e7c285eaa6bca..960d7513aa8dd8 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
+++ b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c
@@ -625,11 +625,12 @@ int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb)
 
 	xa_lock(&nn->xa_ipsec);
 	x = xa_load(&nn->xa_ipsec, saidx);
+	if (x)
+		xfrm_state_hold(x);
 	xa_unlock(&nn->xa_ipsec);
 	if (!x)
 		return -EINVAL;
 
-	xfrm_state_hold(x);
 	sp->xvec[sp->len++] = x;
 	sp->olen++;
 	xo = xfrm_offload(skb);
diff --git a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
index ca30702f88780e..d650d33e3709c0 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
@@ -209,6 +209,7 @@ void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb)
 {
 	struct nfp_tun_active_tuns *payload;
 	struct net_device *netdev;
+	struct neigh_table *tbl;
 	int count, i, pay_len;
 	struct neighbour *n;
 	__be32 ipv4_addr;
@@ -235,7 +236,8 @@ void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb)
 		if (!netdev)
 			continue;
 
-		n = neigh_lookup(&arp_tbl, &ipv4_addr, netdev);
+		tbl = arp_table(dev_net(netdev));
+		n = neigh_lookup(tbl, &ipv4_addr, netdev);
 		if (!n)
 			continue;
 
@@ -251,6 +253,7 @@ void nfp_tunnel_keep_alive_v6(struct nfp_app *app, struct sk_buff *skb)
 #if IS_ENABLED(CONFIG_IPV6)
 	struct nfp_tun_active_tuns_v6 *payload;
 	struct net_device *netdev;
+	struct neigh_table *tbl;
 	int count, i, pay_len;
 	struct neighbour *n;
 	void *ipv6_add;
@@ -277,7 +280,8 @@ void nfp_tunnel_keep_alive_v6(struct nfp_app *app, struct sk_buff *skb)
 		if (!netdev)
 			continue;
 
-		n = neigh_lookup(&nd_tbl, ipv6_add, netdev);
+		tbl = nd_table(dev_net(netdev));
+		n = neigh_lookup(tbl, ipv6_add, netdev);
 		if (!n)
 			continue;
 
@@ -729,12 +733,6 @@ nfp_tun_neigh_event_handler(struct notifier_block *nb, unsigned long event,
 	default:
 		return NOTIFY_DONE;
 	}
-#if IS_ENABLED(CONFIG_IPV6)
-	if (n->tbl != &nd_tbl && n->tbl != &arp_tbl)
-#else
-	if (n->tbl != &arp_tbl)
-#endif
-		return NOTIFY_DONE;
 
 	app_priv = container_of(nb, struct nfp_flower_priv, tun.neigh_nb);
 	app = app_priv->app;
diff --git a/drivers/net/ethernet/oa_tc6.c b/drivers/net/ethernet/oa_tc6.c
index 6fcc5f561d560b..364027c39fa4a9 100644
--- a/drivers/net/ethernet/oa_tc6.c
+++ b/drivers/net/ethernet/oa_tc6.c
@@ -6,6 +6,8 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/iopoll.h>
 #include <linux/interrupt.h>
 #include <linux/mdio.h>
@@ -88,6 +90,7 @@ struct oa_tc6 {
 	bool disable_traffic;
 	bool prot_ctrl;
 	enum oa_tc6_quirk_flag quirk_flags;
+	struct gpio_desc *reset_gpio;
 };
 
 enum oa_tc6_header_type {
@@ -1453,7 +1456,7 @@ static int oa_tc6_check_ctrl_protection(struct oa_tc6 *tc6)
  * @quirks: device specific modifiers for the OA TC6 protocol.
  *
  * Return: pointer reference to the oa_tc6 structure if the MAC-PHY
- * initialization is successful otherwise NULL.
+ * initialization is successful otherwise an ERR_PTR.
  */
 struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev,
 			   struct oa_tc6_quirks *quirks)
@@ -1463,7 +1466,7 @@ struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev,
 
 	tc6 = devm_kzalloc(&spi->dev, sizeof(*tc6), GFP_KERNEL);
 	if (!tc6)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	tc6->spi = spi;
 	tc6->netdev = netdev;
@@ -1476,60 +1479,77 @@ struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev,
 
 	/* Set the SPI controller to pump at realtime priority */
 	tc6->spi->rt = true;
-	if (spi_setup(tc6->spi) < 0)
-		return NULL;
+	ret = spi_setup(tc6->spi);
+	if (ret < 0)
+		return ERR_PTR(ret);
 
 	tc6->spi_ctrl_tx_buf = devm_kzalloc(&tc6->spi->dev,
 					    OA_TC6_CTRL_SPI_BUF_SIZE,
 					    GFP_KERNEL);
 	if (!tc6->spi_ctrl_tx_buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	tc6->spi_ctrl_rx_buf = devm_kzalloc(&tc6->spi->dev,
 					    OA_TC6_CTRL_SPI_BUF_SIZE,
 					    GFP_KERNEL);
 	if (!tc6->spi_ctrl_rx_buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	tc6->spi_data_tx_buf = devm_kzalloc(&tc6->spi->dev,
 					    OA_TC6_SPI_DATA_BUF_SIZE,
 					    GFP_KERNEL);
 	if (!tc6->spi_data_tx_buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	tc6->spi_data_rx_buf = devm_kzalloc(&tc6->spi->dev,
 					    OA_TC6_SPI_DATA_BUF_SIZE,
 					    GFP_KERNEL);
 	if (!tc6->spi_data_rx_buf)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
+
+	tc6->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset",
+						  GPIOD_OUT_HIGH);
+	if (IS_ERR(tc6->reset_gpio))
+		return ERR_PTR(dev_err_probe(&spi->dev,
+					     PTR_ERR(tc6->reset_gpio),
+					     "failed to get reset gpio\n"));
+
+	if (tc6->reset_gpio) {
+		/* Keep the reset asserted for 10 us and then allow 1 ms of
+		 * settle time for the crystal oscillator startup.
+		 */
+		fsleep(10);
+		gpiod_set_value_cansleep(tc6->reset_gpio, 0);
+		fsleep(1000);
+	}
 
 	/* Check the PROTE bit status so that we can reset the device */
 	ret = oa_tc6_check_ctrl_protection(tc6);
 	if (ret) {
 		dev_err(&tc6->spi->dev,
 			"Failed to check the protection mode: %d\n", ret);
-		return NULL;
+		return ERR_PTR(ret);
 	}
 
 	ret = oa_tc6_sw_reset_macphy(tc6);
 	if (ret) {
 		dev_err(&tc6->spi->dev,
 			"MAC-PHY software reset failed: %d\n", ret);
-		return NULL;
+		return ERR_PTR(ret);
 	}
 
 	ret = oa_tc6_unmask_macphy_error_interrupts(tc6);
 	if (ret) {
 		dev_err(&tc6->spi->dev,
 			"MAC-PHY error interrupts unmask failed: %d\n", ret);
-		return NULL;
+		return ERR_PTR(ret);
 	}
 
 	ret = oa_tc6_phy_init(tc6);
 	if (ret) {
 		dev_err(&tc6->spi->dev,
 			"MAC internal PHY initialization failed: %d\n", ret);
-		return NULL;
+		return ERR_PTR(ret);
 	}
 
 	ret = oa_tc6_enable_data_transfer(tc6);
@@ -1570,7 +1590,7 @@ struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev,
 
 phy_exit:
 	oa_tc6_phy_exit(tc6);
-	return NULL;
+	return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(oa_tc6_init);
 
diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
index 88c5c52e0e3813..ba3d10f9aec6b9 100644
--- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
+++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
@@ -2504,12 +2504,10 @@ static int pch_gbe_probe(struct pci_dev *pdev,
 	if (ret)
 		return ret;
 
-	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
-		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (ret) {
-			dev_err(&pdev->dev, "ERR: No usable DMA configuration, aborting\n");
-			return ret;
-		}
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		dev_err(&pdev->dev, "ERR: No usable DMA configuration, aborting\n");
+		return ret;
 	}
 
 	ret = pcim_iomap_regions(pdev, 1 << PCH_GBE_PCI_BAR, pci_name(pdev));
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
index 05f19489ec5c83..666efe972de784 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
@@ -424,6 +424,7 @@ static void ionic_remove(struct pci_dev *pdev)
 		ionic_auxbus_unregister(ionic->lif);
 		ionic_lif_unregister(ionic->lif);
 		ionic_devlink_unregister(ionic);
+		ionic_vf_dealloc(ionic);
 		ionic_lif_deinit(ionic->lif);
 		ionic_lif_free(ionic->lif);
 		ionic->lif = NULL;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
index fc8c50e8f36517..119ac720c0787a 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
@@ -655,15 +655,9 @@ static int ionic_cmb_rings_toggle(struct ionic_lif *lif, bool cmb_tx, bool cmb_r
 	if (pages_used < 0)
 		return pages_used;
 
-	if (cmb_tx)
-		set_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state);
-	else
-		clear_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state);
+	assign_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state, cmb_tx);
 
-	if (cmb_rx)
-		set_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state);
-	else
-		clear_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state);
+	assign_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state, cmb_rx);
 
 	if (cmb_tx || cmb_rx)
 		netdev_info(lif->netdev, "Enabling CMB %s %s rings - %d pages\n",
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h
index 15f2bc3f404345..e911fad545d672 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_if.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h
@@ -7,6 +7,7 @@
 #define IONIC_DEV_INFO_SIGNATURE		0x44455649      /* 'DEVI' */
 #define IONIC_DEV_INFO_VERSION			1
 #define IONIC_IFNAMSIZ				16
+#define IONIC_CREATE_CQ_CMD_V2_MAGIC		-1
 
 /*
  * enum ionic_cmd_opcode - Device commands
@@ -513,6 +514,23 @@ enum ionic_lif_rdma_cap_stats {
 	IONIC_LIF_RDMA_STAT_QP = BIT(1),
 };
 
+/**
+ * enum ionic_lif_rdma_alloc_qid - RDMA QID allocation capability
+ *
+ * Bitmask of RDMA queue types for which firmware can automatically
+ * allocate queue IDs. When a bit is set, the driver can request the
+ * firmware to allocate QIDs for that queue type during creation.
+ *
+ * @IONIC_LIF_RDMA_ALLOC_QID_CQ:  Firmware can allocate QIDs for
+ *                                Completion Queues
+ * @IONIC_LIF_RDMA_ALLOC_QID_SRQ: Firmware can allocate QIDs for
+ *                                Shared Receive Queues
+ */
+enum ionic_lif_rdma_alloc_qid {
+	IONIC_LIF_RDMA_ALLOC_QID_CQ = BIT(0),
+	IONIC_LIF_RDMA_ALLOC_QID_SRQ = BIT(1),
+};
+
 /**
  * struct ionic_lif_identity - LIF identity information (type-specific)
  *
@@ -557,6 +575,10 @@ enum ionic_lif_rdma_cap_stats {
  *	                       (enum ionic_lif_rdma_cap_stats)
  *	@rdma.rsvd:            Reserved byte
  *	@rdma.rcq_sign_bit:    RCQ sign bit
+ *	@rdma.srq_qtype:       RDMA Shared Receive Qtype
+ *	@rdma.rsvd2:           Reserved byte(s)
+ *	@rdma.alloc_qid_cap:   RDMA queue type QID allocation capability
+ *	                       (bitmask of enum ionic_lif_rdma_alloc_qid)
  *	@rdma.rsvd1:           Reserved byte(s)
  * @words:               word access to struct contents
  */
@@ -604,7 +626,10 @@ union ionic_lif_identity {
 			__le16 stats_type;
 			u8 rsvd;
 			u8 rcq_sign_bit;
-			u8 rsvd1[160];
+			struct ionic_lif_logical_qtype srq_qtype;
+			u8 rsvd2[5];
+			u8 alloc_qid_cap;
+			u8 rsvd1[142];
 		} __packed rdma;
 	} __packed;
 	__le32 words[478];
@@ -2608,6 +2633,7 @@ struct ionic_rdma_reset_cmd {
  * @depth_log2:    log base two of queue depth
  * @stride_log2:   log base two of queue stride
  * @dma_addr:      address of the queue memory
+ * @udma_idx:      udma index
  * @rsvd2:         reserved byte(s)
  *
  * The same command struct is used to create an RDMA event queue, completion
@@ -2637,7 +2663,8 @@ struct ionic_rdma_queue_cmd {
 	u8     depth_log2;
 	u8     stride_log2;
 	__le64 dma_addr;
-	u8     rsvd2[40];
+	u8     udma_idx;
+	u8     rsvd2[39];
 };
 
 /******************************************************************
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ctx.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ctx.c
index fef4b2b0b1f268..79d6a819bf9051 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ctx.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ctx.c
@@ -17,7 +17,7 @@ netxen_poll_rsp(struct netxen_adapter *adapter)
 	int	timeout = 0;
 
 	do {
-		/* give atleast 1ms for firmware to respond */
+		/* give at least 1ms for firmware to respond */
 		msleep(1);
 
 		if (++timeout > NX_OS_CRB_RETRY_COUNT)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
index b70262e70baf6e..89b947f82cf00e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
@@ -561,7 +561,7 @@ int qed_cxt_cfg_ilt_compute(struct qed_hwfn *p_hwfn, u32 *line_count)
 			 *      block in the ILT client. This is later used to
 			 *      configure the CDU segment offset registers and
 			 *      results in an FL command for TIDs of this
-			 *      segement behaves as regular load commands
+			 *      segment behaves as regular load commands
 			 *      (loading TIDs from the working memory).
 			 */
 			line = p_cli->pf_blks[CDUT_SEG_BLK(i)].start_line;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
index 9bd0565fe8abc1..50699d600b8e3b 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h
@@ -783,7 +783,7 @@ struct qed_mcp_info {
 	u16					mfw_mb_length;
 	u32					mcp_hist;
 
-	/* Capabilties negotiated with the MFW */
+	/* Capabilities negotiated with the MFW */
 	u32					capabilities;
 
 	/* S/N for debug data mailbox commands */
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index 647f30a16a94dc..037882a34203b1 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -492,10 +492,8 @@ static int qede_set_priv_flags(struct net_device *dev, u32 flags)
 	if (dflags & ~BIT(QEDE_PRI_FLAG_RECOVER_ON_ERROR))
 		return -EINVAL;
 
-	if (flags & BIT(QEDE_PRI_FLAG_RECOVER_ON_ERROR))
-		set_bit(QEDE_ERR_IS_RECOVERABLE, &edev->err_flags);
-	else
-		clear_bit(QEDE_ERR_IS_RECOVERABLE, &edev->err_flags);
+	assign_bit(QEDE_ERR_IS_RECOVERABLE, &edev->err_flags,
+		   flags & BIT(QEDE_PRI_FLAG_RECOVER_ON_ERROR));
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/qlogic/qla3xxx.c b/drivers/net/ethernet/qlogic/qla3xxx.c
index 861a13ad7e1ac5..1466ec85de247a 100644
--- a/drivers/net/ethernet/qlogic/qla3xxx.c
+++ b/drivers/net/ethernet/qlogic/qla3xxx.c
@@ -1581,10 +1581,8 @@ static void ql_link_state_machine_work(struct work_struct *work)
  */
 static void ql_get_phy_owner(struct ql3_adapter *qdev)
 {
-	if (ql_this_adapter_controls_port(qdev))
-		set_bit(QL_LINK_MASTER, &qdev->flags);
-	else
-		clear_bit(QL_LINK_MASTER, &qdev->flags);
+	assign_bit(QL_LINK_MASTER, &qdev->flags,
+		   ql_this_adapter_controls_port(qdev));
 }
 
 /*
@@ -3343,10 +3341,8 @@ static void ql_set_mac_info(struct ql3_adapter *qdev)
 		qdev->mac_ob_opcode = OUTBOUND_MAC_IOCB | func_number;
 		qdev->mb_bit_mask = FN0_MA_BITS_MASK;
 		qdev->PHYAddr = PORT0_PHY_ADDRESS;
-		if (port_status & PORT_STATUS_SM0)
-			set_bit(QL_LINK_OPTICAL, &qdev->flags);
-		else
-			clear_bit(QL_LINK_OPTICAL, &qdev->flags);
+		assign_bit(QL_LINK_OPTICAL, &qdev->flags,
+			   port_status & PORT_STATUS_SM0);
 		break;
 
 	case ISP_CONTROL_FN1_NET:
@@ -3354,10 +3350,8 @@ static void ql_set_mac_info(struct ql3_adapter *qdev)
 		qdev->mac_ob_opcode = OUTBOUND_MAC_IOCB | func_number;
 		qdev->mb_bit_mask = FN1_MA_BITS_MASK;
 		qdev->PHYAddr = PORT1_PHY_ADDRESS;
-		if (port_status & PORT_STATUS_SM1)
-			set_bit(QL_LINK_OPTICAL, &qdev->flags);
-		else
-			clear_bit(QL_LINK_OPTICAL, &qdev->flags);
+		assign_bit(QL_LINK_OPTICAL, &qdev->flags,
+			   port_status & PORT_STATUS_SM1);
 		break;
 
 	case ISP_CONTROL_FN0_SCSI:
@@ -3916,6 +3910,7 @@ static void ql3xxx_remove(struct pci_dev *pdev)
 	iounmap(qdev->mem_map_registers);
 	pci_release_regions(pdev);
 	free_netdev(ndev);
+	pci_disable_device(pdev);
 }
 
 static struct pci_driver ql3xxx_driver = {
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
index d7c8fadb49de1f..9728d583b8919f 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h
@@ -909,7 +909,7 @@ struct qlcnic_mac_vlan_list {
 
 #define QLCNIC_LRO_REQUEST_CLEANUP	4
 
-/* Capabilites received */
+/* Capabilities received */
 #define QLCNIC_FW_CAPABILITY_TSO		BIT_1
 #define QLCNIC_FW_CAPABILITY_BDG		BIT_8
 #define QLCNIC_FW_CAPABILITY_FVLANTX		BIT_9
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c
index 47cd9ec665eeb4..ff337a917243b1 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c
@@ -1957,9 +1957,9 @@ static void qlcnic_83xx_template_end(struct qlcnic_adapter *p_dev)
 * qlcnic_83xx_exec_template_cmd
 *
 * @p_dev: adapter structure
-* @p_buff: Poiter to instruction template
+* @p_buff: Pointer to instruction template
 *
-* Template provides instructions to stop, restart and initalize firmware.
+* Template provides instructions to stop, restart and initialize firmware.
 * These instructions are abstracted as a series of read, write and
 * poll operations on hardware registers. Register information and operation
 * specifics are not exposed to the driver. Driver reads the template from
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ctx.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ctx.c
index eb827b86ecae81..1b94c42d669b35 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ctx.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ctx.c
@@ -95,7 +95,7 @@ qlcnic_poll_rsp(struct qlcnic_adapter *adapter)
 	int timeout = 0, err = 0;
 
 	do {
-		/* give atleast 1ms for firmware to respond */
+		/* give at least 1ms for firmware to respond */
 		mdelay(1);
 
 		if (++timeout > QLCNIC_OS_CRB_RETRY_COUNT)
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.c
index 576340315e0dbd..ef4a0376bafffe 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_dcb.c
@@ -644,10 +644,7 @@ static void qlcnic_83xx_dcb_aen_handler(struct qlcnic_dcb *dcb, void *data)
 	if (test_and_set_bit(QLCNIC_DCB_AEN_MODE, &dcb->state))
 		return;
 
-	if (*val & BIT_8)
-		set_bit(QLCNIC_DCB_STATE, &dcb->state);
-	else
-		clear_bit(QLCNIC_DCB_STATE, &dcb->state);
+	assign_bit(QLCNIC_DCB_STATE, &dcb->state, *val & BIT_8);
 
 	queue_delayed_work(dcb->wq, &dcb->aen_work, 0);
 }
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
index ff4f7cd20c7916..23ae76ca81dbe0 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c
@@ -640,7 +640,7 @@ static int qlcnic_max_rings(struct qlcnic_adapter *adapter, u8 ring_cnt,
 
 void qlcnic_set_tx_ring_count(struct qlcnic_adapter *adapter, u8 tx_cnt)
 {
-	/* 83xx adapter does not have max_tx_rings intialized in probe */
+	/* 83xx adapter does not have max_tx_rings initialized in probe */
 	if (adapter->max_tx_rings)
 		adapter->drv_tx_rings = qlcnic_max_rings(adapter, tx_cnt,
 							 QLCNIC_TX_QUEUE);
@@ -650,7 +650,7 @@ void qlcnic_set_tx_ring_count(struct qlcnic_adapter *adapter, u8 tx_cnt)
 
 void qlcnic_set_sds_ring_count(struct qlcnic_adapter *adapter, u8 rx_cnt)
 {
-	/* 83xx adapter does not have max_sds_rings intialized in probe */
+	/* 83xx adapter does not have max_sds_rings initialized in probe */
 	if (adapter->max_sds_rings)
 		adapter->drv_sds_rings = qlcnic_max_rings(adapter, rx_cnt,
 							  QLCNIC_RX_QUEUE);
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
index 0704dbc52d822c..a106f961e177b3 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_common.c
@@ -1488,10 +1488,8 @@ static int qlcnic_sriov_channel_cfg_cmd(struct qlcnic_adapter *adapter, u8 cmd_o
 		goto out;
 	}
 
-	if (cmd_op == QLCNIC_BC_CMD_CHANNEL_INIT)
-		set_bit(QLC_BC_VF_STATE, &vf->state);
-	else
-		clear_bit(QLC_BC_VF_STATE, &vf->state);
+	assign_bit(QLC_BC_VF_STATE, &vf->state,
+		   cmd_op == QLCNIC_BC_CMD_CHANNEL_INIT);
 
 out:
 	qlcnic_free_mbx_args(&cmd);
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
index 7052d34aad4595..c69b41e317d262 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_sriov_pf.c
@@ -773,10 +773,8 @@ static int qlcnic_sriov_pf_channel_cfg_cmd(struct qlcnic_bc_trans *trans,
 
 	cmd->rsp.arg[0] |= (1 << 25);
 
-	if (trans->req_hdr->cmd_op == QLCNIC_BC_CMD_CHANNEL_INIT)
-		set_bit(QLC_BC_VF_STATE, &vf->state);
-	else
-		clear_bit(QLC_BC_VF_STATE, &vf->state);
+	assign_bit(QLC_BC_VF_STATE, &vf->state,
+		   trans->req_hdr->cmd_op == QLCNIC_BC_CMD_CHANNEL_INIT);
 
 	return err;
 
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
index bed6f63facf250..61b04c6c0390e8 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
@@ -176,7 +176,7 @@ static int rmnet_newlink(struct net_device *dev,
 	}
 
 	netdev_dbg(dev, "data format [0x%08X]\n", data_format);
-	port->data_format = data_format;
+	WRITE_ONCE(port->data_format, data_format);
 
 	return 0;
 
@@ -312,6 +312,12 @@ static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (!rmnet_is_real_dev_registered(real_dev))
 		return -ENODEV;
 
+	if (!rtnl_dev_link_net_capable(dev, dev_net(real_dev))) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "request modifies device in another netns");
+		return -EPERM;
+	}
+
 	port = rmnet_get_port_rtnl(real_dev);
 
 	if (data[IFLA_RMNET_MUX_ID]) {
@@ -331,25 +337,27 @@ static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[],
 			}
 
 			hlist_del_init_rcu(&ep->hlnode);
+			WRITE_ONCE(ep->mux_id, mux_id);
 			hlist_add_head_rcu(&ep->hlnode,
 					   &port->muxed_ep[mux_id]);
 
-			ep->mux_id = mux_id;
-			priv->mux_id = mux_id;
+			WRITE_ONCE(priv->mux_id, mux_id);
 		}
 	}
 
 	if (data[IFLA_RMNET_FLAGS]) {
 		struct ifla_rmnet_flags *flags;
 		u32 old_data_format;
+		u32 data_format;
 
 		old_data_format = port->data_format;
 		flags = nla_data(data[IFLA_RMNET_FLAGS]);
-		port->data_format &= ~flags->mask;
-		port->data_format |= flags->flags & flags->mask;
+		data_format = old_data_format & ~flags->mask;
+		data_format |= flags->flags & flags->mask;
+		WRITE_ONCE(port->data_format, data_format);
 
 		if (rmnet_vnd_update_dev_mtu(port, real_dev)) {
-			port->data_format = old_data_format;
+			WRITE_ONCE(port->data_format, old_data_format);
 			NL_SET_ERR_MSG_MOD(extack, "Invalid MTU on real dev");
 			return -EINVAL;
 		}
@@ -369,32 +377,24 @@ static size_t rmnet_get_size(const struct net_device *dev)
 
 static int rmnet_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
-	struct rmnet_priv *priv = netdev_priv(dev);
-	struct net_device *real_dev;
+	const struct rmnet_priv *priv = netdev_priv(dev);
+	const struct rmnet_port *port;
 	struct ifla_rmnet_flags f;
-	struct rmnet_port *port;
 
-	real_dev = priv->real_dev;
-
-	if (nla_put_u16(skb, IFLA_RMNET_MUX_ID, priv->mux_id))
-		goto nla_put_failure;
+	if (nla_put_u16(skb, IFLA_RMNET_MUX_ID, READ_ONCE(priv->mux_id)))
+		return -EMSGSIZE;
 
-	if (rmnet_is_real_dev_registered(real_dev)) {
-		port = rmnet_get_port_rtnl(real_dev);
-		f.flags = port->data_format;
-	} else {
-		f.flags = 0;
-	}
+	rcu_read_lock();
+	port = rmnet_get_port_rcu(priv->real_dev);
+	f.flags = port ? READ_ONCE(port->data_format) : 0;
+	rcu_read_unlock();
 
 	f.mask  = ~0;
 
 	if (nla_put(skb, IFLA_RMNET_FLAGS, sizeof(f), &f))
-		goto nla_put_failure;
+		return -EMSGSIZE;
 
 	return 0;
-
-nla_put_failure:
-	return -EMSGSIZE;
 }
 
 struct rtnl_link_ops rmnet_link_ops __read_mostly = {
@@ -411,12 +411,16 @@ struct rtnl_link_ops rmnet_link_ops __read_mostly = {
 	.fill_info	= rmnet_fill_info,
 };
 
-struct rmnet_port *rmnet_get_port_rcu(struct net_device *real_dev)
+/* Can be called from a RCU read-side critical section, with or
+ * without BH disabled.
+ */
+struct rmnet_port *rmnet_get_port_rcu(const struct net_device *real_dev)
 {
-	if (rmnet_is_real_dev_registered(real_dev))
-		return rcu_dereference_bh(real_dev->rx_handler_data);
-	else
+	if (!rmnet_is_real_dev_registered(real_dev))
 		return NULL;
+
+	return rcu_dereference_check(real_dev->rx_handler_data,
+				     rcu_read_lock_bh_held());
 }
 
 struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id)
@@ -425,7 +429,7 @@ struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id)
 
 	hlist_for_each_entry_rcu(ep, &port->muxed_ep[mux_id], hlnode,
 				 lockdep_rtnl_is_held()) {
-		if (ep->mux_id == mux_id)
+		if (READ_ONCE(ep->mux_id) == mux_id)
 			return ep;
 	}
 
@@ -441,6 +445,12 @@ int rmnet_add_bridge(struct net_device *rmnet_dev,
 	struct rmnet_port *port, *slave_port;
 	int err;
 
+	if (!rtnl_dev_link_net_capable(slave_dev, dev_net(real_dev))) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "request modifies device in another netns");
+		return -EPERM;
+	}
+
 	port = rmnet_get_port_rtnl(real_dev);
 
 	/* If there is more than one rmnet dev attached, its probably being
@@ -489,7 +499,14 @@ int rmnet_add_bridge(struct net_device *rmnet_dev,
 int rmnet_del_bridge(struct net_device *rmnet_dev,
 		     struct net_device *slave_dev)
 {
-	struct rmnet_port *port = rmnet_get_port_rtnl(slave_dev);
+	struct rmnet_priv *priv = netdev_priv(rmnet_dev);
+	struct net_device *real_dev = priv->real_dev;
+	struct rmnet_port *port;
+
+	if (!rtnl_dev_link_net_capable(slave_dev, dev_net(real_dev)))
+		return -EPERM;
+
+	port = rmnet_get_port_rtnl(slave_dev);
 
 	rmnet_unregister_bridge(port);
 
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
index f50fae1c6bdd5d..5adda0323dda77 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
@@ -90,7 +90,7 @@ struct rmnet_priv {
 	struct rmnet_priv_stats stats;
 };
 
-struct rmnet_port *rmnet_get_port_rcu(struct net_device *real_dev);
+struct rmnet_port *rmnet_get_port_rcu(const struct net_device *real_dev);
 struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id);
 int rmnet_add_bridge(struct net_device *rmnet_dev,
 		     struct net_device *slave_dev,
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
index d055a2628d8c9d..aa5523f4618eb3 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
@@ -54,7 +54,8 @@ rmnet_deliver_skb(struct sk_buff *skb)
 
 static void
 __rmnet_map_ingress_handler(struct sk_buff *skb,
-			    struct rmnet_port *port)
+			    struct rmnet_port *port,
+			    u32 data_format)
 {
 	struct rmnet_map_header *map_header = (void *)skb->data;
 	struct rmnet_endpoint *ep;
@@ -63,8 +64,8 @@ __rmnet_map_ingress_handler(struct sk_buff *skb,
 
 	if (map_header->flags & MAP_CMD_FLAG) {
 		/* Packet contains a MAP command (not data) */
-		if (port->data_format & RMNET_FLAGS_INGRESS_MAP_COMMANDS)
-			return rmnet_map_command(skb, port);
+		if (data_format & RMNET_FLAGS_INGRESS_MAP_COMMANDS)
+			return rmnet_map_command(skb, port, data_format);
 
 		goto free_skb;
 	}
@@ -82,7 +83,7 @@ __rmnet_map_ingress_handler(struct sk_buff *skb,
 
 	skb->dev = ep->egress_dev;
 
-	if ((port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) &&
+	if ((data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) &&
 	    (map_header->flags & MAP_NEXT_HEADER_FLAG)) {
 		if (rmnet_map_process_next_hdr_packet(skb, len))
 			goto free_skb;
@@ -92,7 +93,7 @@ __rmnet_map_ingress_handler(struct sk_buff *skb,
 		/* Subtract MAP header */
 		skb_pull(skb, sizeof(*map_header));
 		rmnet_set_skb_proto(skb);
-		if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4 &&
+		if (data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4 &&
 		    !rmnet_map_checksum_downlink_packet(skb, len + pad))
 			skb->ip_summed = CHECKSUM_UNNECESSARY;
 	}
@@ -110,6 +111,7 @@ rmnet_map_ingress_handler(struct sk_buff *skb,
 			  struct rmnet_port *port)
 {
 	struct sk_buff *skbn;
+	u32 data_format;
 
 	if (skb->dev->type == ARPHRD_ETHER) {
 		if (pskb_expand_head(skb, ETH_HLEN, 0, GFP_ATOMIC)) {
@@ -120,14 +122,16 @@ rmnet_map_ingress_handler(struct sk_buff *skb,
 		skb_push(skb, ETH_HLEN);
 	}
 
-	if (port->data_format & RMNET_FLAGS_INGRESS_DEAGGREGATION) {
-		while ((skbn = rmnet_map_deaggregate(skb, port)) != NULL)
-			__rmnet_map_ingress_handler(skbn, port);
+	data_format = READ_ONCE(port->data_format);
+
+	if (data_format & RMNET_FLAGS_INGRESS_DEAGGREGATION) {
+		while ((skbn = rmnet_map_deaggregate(skb, data_format)) != NULL)
+			__rmnet_map_ingress_handler(skbn, port, data_format);
 
 		consume_skb(skb);
 	} else {
-		if (rmnet_map_validate_packet_len(skb, port))
-			__rmnet_map_ingress_handler(skb, port);
+		if (rmnet_map_validate_packet_len(skb, data_format))
+			__rmnet_map_ingress_handler(skb, port, data_format);
 		else
 			kfree_skb(skb);
 	}
@@ -139,14 +143,16 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
 {
 	int required_headroom, additional_header_len, csum_type = 0;
 	struct rmnet_map_header *map_header;
+	u32 data_format;
 
 	additional_header_len = 0;
 	required_headroom = sizeof(struct rmnet_map_header);
 
-	if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) {
+	data_format = READ_ONCE(port->data_format);
+	if (data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) {
 		additional_header_len = sizeof(struct rmnet_map_ul_csum_header);
 		csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV4;
-	} else if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) {
+	} else if (data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) {
 		additional_header_len = sizeof(struct rmnet_map_v5_csum_header);
 		csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV5;
 	}
@@ -161,7 +167,7 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
 						 csum_type);
 
 	map_header = rmnet_map_add_map_header(skb, additional_header_len,
-					      port, 0);
+					      data_format, 0);
 	if (!map_header)
 		return -ENOMEM;
 
@@ -255,7 +261,7 @@ void rmnet_egress_handler(struct sk_buff *skb)
 	orig_dev = skb->dev;
 	priv = netdev_priv(orig_dev);
 	skb->dev = priv->real_dev;
-	mux_id = priv->mux_id;
+	mux_id = READ_ONCE(priv->mux_id);
 
 	port = rmnet_get_port_rcu(skb->dev);
 	if (!port)
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
index 60ca8b780c88ad..0977e495f59155 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
@@ -41,12 +41,13 @@ enum rmnet_map_commands {
 #define RMNET_MAP_ADD_PAD_BYTES       1
 
 struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
-				      struct rmnet_port *port);
+				      u32 data_format);
 struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
 						  int hdrlen,
-						  struct rmnet_port *port,
+						  u32 data_format,
 						  int pad);
-void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port);
+void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port,
+		       u32 data_format);
 int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len);
 void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
 				      struct rmnet_port *port,
@@ -59,6 +60,6 @@ void rmnet_map_tx_aggregate_init(struct rmnet_port *port);
 void rmnet_map_tx_aggregate_exit(struct rmnet_port *port);
 void rmnet_map_update_ul_agg_config(struct rmnet_port *port, u32 size,
 				    u32 count, u32 time);
-u32 rmnet_map_validate_packet_len(struct sk_buff *skb, struct rmnet_port *port);
+u32 rmnet_map_validate_packet_len(struct sk_buff *skb, u32 data_format);
 
 #endif /* _RMNET_MAP_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
index add0f5ade2e617..d334727e1f5256 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
@@ -48,13 +48,13 @@ static u8 rmnet_map_do_flow_control(struct sk_buff *skb,
 
 static void rmnet_map_send_ack(struct sk_buff *skb,
 			       unsigned char type,
-			       struct rmnet_port *port)
+			       u32 data_format)
 {
 	struct rmnet_map_header *map_header = (void *)skb->data;
 	struct rmnet_map_control_command *cmd;
 	struct net_device *dev = skb->dev;
 
-	if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4)
+	if (data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4)
 		skb_trim(skb,
 			 skb->len - sizeof(struct rmnet_map_dl_csum_trailer));
 
@@ -72,7 +72,8 @@ static void rmnet_map_send_ack(struct sk_buff *skb,
 /* Process MAP command frame and send N/ACK message as appropriate. Message cmd
  * name is decoded here and appropriate handler is called.
  */
-void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port)
+void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port,
+		       u32 data_format)
 {
 	struct rmnet_map_header *map_header = (void *)skb->data;
 	struct rmnet_map_control_command *cmd;
@@ -98,5 +99,5 @@ void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port)
 		break;
 	}
 	if (rc == RMNET_MAP_COMMAND_ACK)
-		rmnet_map_send_ack(skb, rc, port);
+		rmnet_map_send_ack(skb, rc, data_format);
 }
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
index e6f48dc9fe64d7..39d6d084e73f00 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
@@ -294,7 +294,7 @@ static void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb,
  */
 struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
 						  int hdrlen,
-						  struct rmnet_port *port,
+						  u32 data_format,
 						  int pad)
 {
 	struct rmnet_map_header *map_header;
@@ -306,7 +306,7 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
 	memset(map_header, 0, sizeof(struct rmnet_map_header));
 
 	/* Set next_hdr bit for csum offload packets */
-	if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5)
+	if (data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5)
 		map_header->flags |= MAP_NEXT_HEADER_FLAG;
 
 	if (pad == RMNET_MAP_NO_PAD_BYTES) {
@@ -333,7 +333,7 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
 	return map_header;
 }
 
-u32 rmnet_map_validate_packet_len(struct sk_buff *skb, struct rmnet_port *port)
+u32 rmnet_map_validate_packet_len(struct sk_buff *skb, u32 data_format)
 {
 	struct rmnet_map_v5_csum_header *next_hdr = NULL;
 	struct rmnet_map_header *maph;
@@ -351,9 +351,9 @@ u32 rmnet_map_validate_packet_len(struct sk_buff *skb, struct rmnet_port *port)
 
 	packet_len = ntohs(maph->pkt_len) + sizeof(*maph);
 
-	if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) {
+	if (data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) {
 		packet_len += sizeof(struct rmnet_map_dl_csum_trailer);
-	} else if ((port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) &&
+	} else if ((data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) &&
 		   !(maph->flags & MAP_CMD_FLAG)) {
 		/* Mapv5 data pkt without csum hdr is invalid */
 		if (!(maph->flags & MAP_NEXT_HEADER_FLAG))
@@ -381,12 +381,12 @@ u32 rmnet_map_validate_packet_len(struct sk_buff *skb, struct rmnet_port *port)
  * is responsible for freeing the original skb.
  */
 struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
-				      struct rmnet_port *port)
+				      u32 data_format)
 {
 	struct sk_buff *skbn;
 	u32 packet_len;
 
-	packet_len = rmnet_map_validate_packet_len(skb, port);
+	packet_len = rmnet_map_validate_packet_len(skb, data_format);
 	if (!packet_len)
 		return NULL;
 
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
index 4f0ddcedfa9790..5f921cddf82b46 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
@@ -4,9 +4,11 @@
  * RMNET Data virtual network driver
  */
 
+#include <linux/capability.h>
 #include <linux/etherdevice.h>
 #include <linux/ethtool.h>
 #include <linux/if_arp.h>
+#include <linux/netlink.h>
 #include <net/pkt_sched.h>
 #include "rmnet_config.h"
 #include "rmnet_handlers.h"
@@ -240,9 +242,16 @@ static int rmnet_set_coalesce(struct net_device *dev,
 			      struct netlink_ext_ack *extack)
 {
 	struct rmnet_priv *priv = netdev_priv(dev);
+	struct net_device *real_dev = priv->real_dev;
 	struct rmnet_port *port;
 
-	port = rmnet_get_port_rtnl(priv->real_dev);
+	if (!ns_capable(dev_net(real_dev)->user_ns, CAP_NET_ADMIN)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "request modifies device in another netns");
+		return -EPERM;
+	}
+
+	port = rmnet_get_port_rtnl(real_dev);
 
 	if (kernel_coal->tx_aggr_max_frames < 1 || kernel_coal->tx_aggr_max_frames > 64)
 		return -EINVAL;
@@ -332,7 +341,7 @@ int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev,
 
 		rmnet_dev->rtnl_link_ops = &rmnet_link_ops;
 
-		priv->mux_id = id;
+		WRITE_ONCE(priv->mux_id, id);
 
 		netdev_dbg(rmnet_dev, "rmnet dev created\n");
 	}
diff --git a/drivers/net/ethernet/realtek/Kconfig b/drivers/net/ethernet/realtek/Kconfig
index 9b0f4f9631dbaa..5818a9a500a86d 100644
--- a/drivers/net/ethernet/realtek/Kconfig
+++ b/drivers/net/ethernet/realtek/Kconfig
@@ -86,8 +86,8 @@ config R8169
 	depends on PCI
 	select FW_LOADER
 	select CRC32
-	select PHYLIB
 	select REALTEK_PHY
+	select PHYLINK
 	help
 	  Say Y here if you have a Realtek Ethernet adapter belonging to
 	  the following families:
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index ec4fc21fa21fb0..1d06ddff983271 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -26,12 +26,12 @@
 #include <linux/dma-mapping.h>
 #include <linux/pm_runtime.h>
 #include <linux/bitfield.h>
+#include <linux/phylink.h>
 #include <linux/prefetch.h>
 #include <linux/ipv6.h>
 #include <linux/unaligned.h>
 #include <net/ip6_checksum.h>
 #include <net/netdev_queues.h>
-#include <net/phy/realtek_phy.h>
 
 #include "r8169.h"
 #include "r8169_firmware.h"
@@ -96,6 +96,26 @@
 #define JUMBO_9K	(9 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
 #define JUMBO_16K	(SZ_16K - VLAN_ETH_HLEN - ETH_FCS_LEN)
 
+#define R8127_SDS_CMD		0x2348
+#define R8127_SDS_ADDR		0x234a
+#define R8127_SDS_DATA_IN	0x234c
+#define R8127_SDS_DATA_OUT	0x234e
+#define R8127_SDS_CMD_EXEC	BIT(0)
+#define R8127_SDS_CMD_WE	BIT(1)
+#define OCP_SDS_ADDR_REG	0xeb10
+#define OCP_SDS_CMD_REG		0xeb0e
+#define OCP_SDS_DATA_REG	0xeb14
+#define SDS_CMD_READ		0x0001
+#define RTL_SDS_C22_BASE	0x40
+#define RTL_SDS_C45_BASE	0x0080
+#define RTL_PKG_DETECT		0xdc00
+#define RTL_PKG_DETECT_MASK	0x0078
+#define RTL_PKG_DETECT_8116AF	0x0030
+#define RTL_INT_HW_ID		0xd006
+#define RTL_INT_HW_ID_MASK	0x00ff
+#define RTL_INT_HW_ID_8116AF	0x0000
+#define RTL_PM_CTRL_D3HOT	0x03
+
 static const struct rtl_chip_info {
 	u32 mask;
 	u32 val;
@@ -334,11 +354,13 @@ enum rtl_registers {
 	ALDPS_LTR	= 0xe0a2,
 	LTR_OBFF_LOCK	= 0xe032,
 	LTR_SNOOP	= 0xe034,
+	SEND_LTR_MSG	= 0xe038,
 
 #define ALDPS_LTR_EN			BIT(0)
 #define LTR_OBFF_LOCK_EN		BIT(0)
 #define LINK_SPEED_CHANGE_EN		BIT(14)
 #define LTR_SNOOP_EN			GENMASK(15, 14)
+#define LTR_MSG_EN			BIT(0)
 };
 
 enum rtl8168_8101_registers {
@@ -728,6 +750,12 @@ enum rtl_dash_type {
 	RTL_DASH_25_BP,
 };
 
+enum rtl_sfp_mode {
+	RTL_SFP_NONE,
+	RTL_SFP_8168_AF,
+	RTL_SFP_8127_ATF,
+};
+
 struct rtl8169_private {
 	void __iomem *mmio_addr;	/* memory map physical address */
 	struct pci_dev *pci_dev;
@@ -736,6 +764,7 @@ struct rtl8169_private {
 	struct napi_struct napi;
 	enum mac_version mac_version;
 	enum rtl_dash_type dash_type;
+	enum rtl_sfp_mode sfp_mode;
 	u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
 	u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
 	u32 dirty_tx;
@@ -750,6 +779,7 @@ struct rtl8169_private {
 	u32 irq_mask;
 	int irq;
 	struct clk *clk;
+	int speed;
 
 	struct {
 		DECLARE_BITMAP(flags, RTL_FLAG_MAX);
@@ -762,7 +792,6 @@ struct rtl8169_private {
 	unsigned supports_gmii:1;
 	unsigned aspm_manageable:1;
 	unsigned dash_enabled:1;
-	bool sfp_mode:1;
 	dma_addr_t counters_phys_addr;
 	struct rtl8169_counters *counters;
 	struct rtl8169_tc_offsets tc_offset;
@@ -774,6 +803,9 @@ struct rtl8169_private {
 	struct r8169_led_classdev *leds;
 
 	u32 ocp_base;
+	struct phylink *phylink;
+	struct phylink_config phylink_config;
+	struct phylink_pcs pcs;
 };
 
 typedef void (*rtl_generic_fct)(struct rtl8169_private *tp);
@@ -1128,10 +1160,6 @@ static int r8168_phy_ocp_read(struct rtl8169_private *tp, u32 reg)
 	if (rtl_ocp_reg_failure(reg))
 		return 0;
 
-	/* Return dummy MII_PHYSID2 in SFP mode to match SFP PHY driver */
-	if (tp->sfp_mode && reg == (OCP_STD_PHY_BASE + 2 * MII_PHYSID2))
-		return PHY_ID_RTL_DUMMY_SFP & 0xffff;
-
 	RTL_W32(tp, GPHY_OCP, reg << 15);
 
 	return rtl_loop_wait_high(tp, &rtl_ocp_gphy_cond, 25, 10) ?
@@ -1199,6 +1227,75 @@ static void r8127_sfp_sds_phy_reset(struct rtl8169_private *tp)
 	usleep_range(10, 20);
 }
 
+DECLARE_RTL_COND(r8127_sds_cmd_cond)
+{
+	return RTL_R16(tp, R8127_SDS_CMD) & R8127_SDS_CMD_EXEC;
+}
+
+static int r8127_sds_read(struct rtl8169_private *tp, u16 index,
+			  u16 page, u16 reg)
+{
+	u16 addr = (index << 11) | (page << 5) | reg;
+
+	RTL_W16(tp, R8127_SDS_ADDR, addr);
+	RTL_W16(tp, R8127_SDS_CMD, R8127_SDS_CMD_EXEC);
+
+	if (rtl_loop_wait_low(tp, &r8127_sds_cmd_cond, 10, 100))
+		return RTL_R16(tp, R8127_SDS_DATA_OUT);
+
+	return -ETIMEDOUT;
+}
+
+static int r8127_sds_write(struct rtl8169_private *tp, u16 index, u16 page,
+			   u16 reg, u16 val)
+{
+	u16 addr = (index << 11) | (page << 5) | reg;
+
+	RTL_W16(tp, R8127_SDS_DATA_IN, val);
+	RTL_W16(tp, R8127_SDS_ADDR, addr);
+	RTL_W16(tp, R8127_SDS_CMD, R8127_SDS_CMD_EXEC | R8127_SDS_CMD_WE);
+
+	if (rtl_loop_wait_low(tp, &r8127_sds_cmd_cond, 10, 100))
+		return 0;
+
+	return -ETIMEDOUT;
+}
+
+static void r8127_sds_modify(struct rtl8169_private *tp, u16 index, u16 page,
+			     u16 reg, u16 clearmask, u16 setmask)
+{
+	int val = r8127_sds_read(tp, index, page, reg);
+
+	if (val < 0)
+		return;
+
+	val = (val & ~clearmask) | setmask;
+	r8127_sds_write(tp, index, page, reg, val);
+}
+
+static void r8127_sfp_init_1g(struct rtl8169_private *tp)
+{
+	int val;
+
+	r8127_sfp_sds_phy_reset(tp);
+
+	r8127_sds_modify(tp, 0, 1, 31, 0, BIT(3));
+	r8127_sds_modify(tp, 0, 2,  0, BIT(13) | BIT(12) | BIT(6),
+			 BIT(12) | BIT(6));
+	r8127_sds_modify(tp, 0, 0,  4, 0, BIT(2));
+
+	RTL_W16(tp, 0x233a, 0x8004);
+	RTL_W16(tp, 0x233e, (RTL_R16(tp, 0x233e) & ~0x3003) | 0x0002);
+
+	r8168_phy_ocp_write(tp, 0xc40a, 0x0000);
+	r8168_phy_ocp_write(tp, 0xc466, 0x0000);
+	r8168_phy_ocp_write(tp, 0xc808, 0x0000);
+	r8168_phy_ocp_write(tp, 0xc80a, 0x0000);
+
+	val = r8168_phy_ocp_read(tp, 0xc804);
+	r8168_phy_ocp_write(tp, 0xc804, (val & ~0x000f) | 0x000c);
+}
+
 static void r8127_sfp_init_10g(struct rtl8169_private *tp)
 {
 	int val;
@@ -1217,12 +1314,6 @@ static void r8127_sfp_init_10g(struct rtl8169_private *tp)
 	r8168_phy_ocp_write(tp, 0xc804, (val & ~0x000f) | 0x000c);
 }
 
-static void rtl_sfp_init(struct rtl8169_private *tp)
-{
-	if (tp->mac_version == RTL_GIGA_MAC_VER_80)
-		r8127_sfp_init_10g(tp);
-}
-
 static void rtl_sfp_reset(struct rtl8169_private *tp)
 {
 	if (tp->mac_version == RTL_GIGA_MAC_VER_80)
@@ -1283,6 +1374,15 @@ static void mac_mcu_write(struct rtl8169_private *tp, int reg, int value)
 	r8168_mac_ocp_write(tp, tp->ocp_base + reg, value);
 }
 
+static bool rtl_is_8116af(struct rtl8169_private *tp)
+{
+	return tp->mac_version == RTL_GIGA_MAC_VER_52 &&
+		(r8168_mac_ocp_read(tp, RTL_PKG_DETECT) & RTL_PKG_DETECT_MASK) ==
+		RTL_PKG_DETECT_8116AF &&
+		(r8168_mac_ocp_read(tp, RTL_INT_HW_ID) & RTL_INT_HW_ID_MASK) ==
+		RTL_INT_HW_ID_8116AF;
+}
+
 static int mac_mcu_read(struct rtl8169_private *tp, int reg)
 {
 	return r8168_mac_ocp_read(tp, tp->ocp_base + reg);
@@ -1578,6 +1678,20 @@ static bool rtl_dash_is_enabled(struct rtl8169_private *tp)
 	}
 }
 
+static enum rtl_sfp_mode rtl_get_sfp_mode(struct rtl8169_private *tp)
+{
+	if (rtl_is_8125(tp)) {
+		u16 data = r8168_mac_ocp_read(tp, RTL_INT_HW_ID);
+
+		if ((data & 0xff) == 0x07)
+			return RTL_SFP_8127_ATF;
+	} else if (rtl_is_8116af(tp)) {
+		return RTL_SFP_8168_AF;
+	}
+
+	return RTL_SFP_NONE;
+}
+
 static enum rtl_dash_type rtl_get_dash_type(struct rtl8169_private *tp)
 {
 	switch (tp->mac_version) {
@@ -1673,16 +1787,14 @@ static void rtl8169_irq_mask_and_ack(struct rtl8169_private *tp)
 	rtl_pci_commit(tp);
 }
 
-static void rtl_link_chg_patch(struct rtl8169_private *tp)
+static void rtl_link_chg_patch(struct rtl8169_private *tp, int speed)
 {
-	struct phy_device *phydev = tp->phydev;
-
 	if (tp->mac_version == RTL_GIGA_MAC_VER_34 ||
 	    tp->mac_version == RTL_GIGA_MAC_VER_38) {
-		if (phydev->speed == SPEED_1000) {
+		if (speed == SPEED_1000) {
 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
-		} else if (phydev->speed == SPEED_100) {
+		} else if (speed == SPEED_100) {
 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
 		} else {
@@ -1692,7 +1804,7 @@ static void rtl_link_chg_patch(struct rtl8169_private *tp)
 		rtl_reset_packet_filter(tp);
 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_35 ||
 		   tp->mac_version == RTL_GIGA_MAC_VER_36) {
-		if (phydev->speed == SPEED_1000) {
+		if (speed == SPEED_1000) {
 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
 		} else {
@@ -1700,7 +1812,7 @@ static void rtl_link_chg_patch(struct rtl8169_private *tp)
 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x0000003f);
 		}
 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_37) {
-		if (phydev->speed == SPEED_10) {
+		if (speed == SPEED_10) {
 			rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x4d02);
 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_0011, 0x0060a);
 		} else {
@@ -2074,11 +2186,11 @@ rtl_coalesce_info(struct rtl8169_private *tp)
 		ci = rtl_coalesce_info_8168_8136;
 
 	/* if speed is unknown assume highest one */
-	if (tp->phydev->speed == SPEED_UNKNOWN)
+	if (tp->speed == SPEED_UNKNOWN)
 		return ci;
 
 	for (; ci->speed; ci++) {
-		if (tp->phydev->speed == ci->speed)
+		if (tp->speed == ci->speed)
 			return ci;
 	}
 
@@ -2236,7 +2348,7 @@ static void rtl_set_eee_txidle_timer(struct rtl8169_private *tp)
 
 static unsigned int r8169_get_tx_lpi_timer_us(struct rtl8169_private *tp)
 {
-	unsigned int speed = tp->phydev->speed;
+	unsigned int speed = tp->speed;
 	unsigned int timer = tp->tx_lpi_timer;
 
 	if (!timer || speed == SPEED_UNKNOWN)
@@ -2254,7 +2366,7 @@ static int rtl8169_get_eee(struct net_device *dev, struct ethtool_keee *data)
 	if (!rtl_supports_eee(tp))
 		return -EOPNOTSUPP;
 
-	ret = phy_ethtool_get_eee(tp->phydev, data);
+	ret = phylink_ethtool_get_eee(tp->phylink, data);
 	if (ret)
 		return ret;
 
@@ -2270,7 +2382,7 @@ static int rtl8169_set_eee(struct net_device *dev, struct ethtool_keee *data)
 	if (!rtl_supports_eee(tp))
 		return -EOPNOTSUPP;
 
-	return phy_ethtool_set_eee(tp->phydev, data);
+	return phylink_ethtool_set_eee(tp->phylink, data);
 }
 
 static void rtl8169_get_ringparam(struct net_device *dev,
@@ -2301,13 +2413,8 @@ static void rtl8169_get_pauseparam(struct net_device *dev,
 				   struct ethtool_pauseparam *data)
 {
 	struct rtl8169_private *tp = netdev_priv(dev);
-	bool tx_pause, rx_pause;
 
-	phy_get_pause(tp->phydev, &tx_pause, &rx_pause);
-
-	data->autoneg = tp->phydev->autoneg;
-	data->tx_pause = tx_pause ? 1 : 0;
-	data->rx_pause = rx_pause ? 1 : 0;
+	phylink_ethtool_get_pauseparam(tp->phylink, data);
 }
 
 static int rtl8169_set_pauseparam(struct net_device *dev,
@@ -2315,12 +2422,10 @@ static int rtl8169_set_pauseparam(struct net_device *dev,
 {
 	struct rtl8169_private *tp = netdev_priv(dev);
 
-	if (dev->mtu > ETH_DATA_LEN)
+	if (dev->mtu > ETH_DATA_LEN || data->autoneg == AUTONEG_DISABLE)
 		return -EOPNOTSUPP;
 
-	phy_set_asym_pause(tp->phydev, data->rx_pause, data->tx_pause);
-
-	return 0;
+	return phylink_ethtool_set_pauseparam(tp->phylink, data);
 }
 
 static void rtl8169_get_eth_mac_stats(struct net_device *dev,
@@ -2386,34 +2491,27 @@ static void rtl8169_get_eth_ctrl_stats(struct net_device *dev,
 		le32_to_cpu(tp->counters->rx_unknown_opcode);
 }
 
-static int rtl8169_set_link_ksettings(struct net_device *ndev,
-				      const struct ethtool_link_ksettings *cmd)
+static int rtl8169_get_link_ksettings(struct net_device *ndev,
+				      struct ethtool_link_ksettings *cmd)
 {
 	struct rtl8169_private *tp = netdev_priv(ndev);
-	struct phy_device *phydev = tp->phydev;
-	int duplex = cmd->base.duplex;
-	int speed = cmd->base.speed;
-
-	if (!tp->sfp_mode)
-		return phy_ethtool_ksettings_set(phydev, cmd);
-
-	if (cmd->base.autoneg != AUTONEG_DISABLE)
-		return -EINVAL;
 
-	if (!phy_check_valid(speed, duplex, phydev->supported))
-		return -EINVAL;
-
-	mutex_lock(&phydev->lock);
+	return phylink_ethtool_ksettings_get(tp->phylink, cmd);
+}
 
-	phydev->autoneg = AUTONEG_DISABLE;
-	phydev->speed = speed;
-	phydev->duplex = duplex;
+static int rtl8169_set_link_ksettings(struct net_device *ndev,
+				      const struct ethtool_link_ksettings *cmd)
+{
+	struct rtl8169_private *tp = netdev_priv(ndev);
 
-	rtl_sfp_init(tp);
+	return phylink_ethtool_ksettings_set(tp->phylink, cmd);
+}
 
-	mutex_unlock(&phydev->lock);
+static int rtl8169_nway_reset(struct net_device *dev)
+{
+	struct rtl8169_private *tp = netdev_priv(dev);
 
-	return 0;
+	return phylink_ethtool_nway_reset(tp->phylink);
 }
 
 static const struct ethtool_ops rtl8169_ethtool_ops = {
@@ -2431,10 +2529,10 @@ static const struct ethtool_ops rtl8169_ethtool_ops = {
 	.get_sset_count		= rtl8169_get_sset_count,
 	.get_ethtool_stats	= rtl8169_get_ethtool_stats,
 	.get_ts_info		= ethtool_op_get_ts_info,
-	.nway_reset		= phy_ethtool_nway_reset,
+	.nway_reset		= rtl8169_nway_reset,
 	.get_eee		= rtl8169_get_eee,
 	.set_eee		= rtl8169_set_eee,
-	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
+	.get_link_ksettings	= rtl8169_get_link_ksettings,
 	.set_link_ksettings	= rtl8169_set_link_ksettings,
 	.get_ringparam		= rtl8169_get_ringparam,
 	.get_pause_stats	= rtl8169_get_pause_stats,
@@ -2499,9 +2597,10 @@ void r8169_apply_firmware(struct rtl8169_private *tp)
 		tp->ocp_base = OCP_STD_PHY_BASE;
 
 		/* PHY soft reset may still be in progress */
-		phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
-				      !(val & BMCR_RESET),
-				      50000, 600000, true);
+		if (tp->phydev)
+			phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
+					      !(val & BMCR_RESET),
+					      50000, 600000, true);
 	}
 }
 
@@ -2538,6 +2637,8 @@ static void rtl_schedule_task(struct rtl8169_private *tp, enum rtl_flag flag)
 
 static void rtl8169_init_phy(struct rtl8169_private *tp)
 {
+	phy_init_hw(tp->phydev);
+	phy_resume(tp->phydev);
 	r8169_hw_phy_config(tp, tp->phydev, tp->mac_version);
 
 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06) {
@@ -2552,9 +2653,6 @@ static void rtl8169_init_phy(struct rtl8169_private *tp)
 	    tp->pci_dev->subsystem_device == 0xe000)
 		phy_write_paged(tp->phydev, 0x0001, 0x10, 0xf01b);
 
-	if (tp->sfp_mode)
-		rtl_sfp_init(tp);
-
 	/* We may have called phy_speed_down before */
 	phy_speed_up(tp->phydev);
 
@@ -2657,15 +2755,6 @@ static void rtl_jumbo_config(struct rtl8169_private *tp)
 
 	if (pci_is_pcie(tp->pci_dev) && tp->supports_gmii)
 		pcie_set_readrq(tp->pci_dev, readrq);
-
-	/* Chip doesn't support pause in jumbo mode */
-	if (jumbo) {
-		linkmode_clear_bit(ETHTOOL_LINK_MODE_Pause_BIT,
-				   tp->phydev->advertising);
-		linkmode_clear_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT,
-				   tp->phydev->advertising);
-		phy_start_aneg(tp->phydev);
-	}
 }
 
 DECLARE_RTL_COND(rtl_chipcmd_cond)
@@ -2780,7 +2869,7 @@ static void rtl_prepare_power_down(struct rtl8169_private *tp)
 		rtl_ephy_write(tp, 0x19, 0xff64);
 
 	if (device_may_wakeup(tp_to_dev(tp))) {
-		phy_speed_down(tp->phydev, false);
+		phylink_speed_down(tp->phylink, false);
 		rtl_wol_enable_rx(tp);
 	}
 }
@@ -2832,6 +2921,16 @@ static void rtl8169_set_magic_reg(struct rtl8169_private *tp)
 	RTL_W32(tp, 0x7c, val);
 }
 
+static int rtl8169_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+{
+	struct rtl8169_private *tp = netdev_priv(dev);
+
+	if (!netif_running(dev))
+		return -ENODEV;
+
+	return phylink_mii_ioctl(tp->phylink, ifr, cmd);
+}
+
 static void rtl_set_rx_mode(struct net_device *dev)
 {
 	u32 rx_mode = AcceptBroadcast | AcceptMyPhys | AcceptMulticast;
@@ -3098,8 +3197,23 @@ static void rtl_enable_ltr(struct rtl8169_private *tp)
 		r8168_mac_ocp_write(tp, 0xcdf2, 0x9003);
 		r8168_mac_ocp_modify(tp, LTR_OBFF_LOCK, 0x0000, LINK_SPEED_CHANGE_EN);
 		break;
-	case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
 	case RTL_GIGA_MAC_VER_52:
+		r8168_mac_ocp_write(tp, 0xcdd0, 0x9003);
+		r8168_mac_ocp_modify(tp, LTR_SNOOP, 0x0000, LTR_SNOOP_EN);
+		r8168_mac_ocp_write(tp, 0xe02c, 0x1880);
+		r8168_mac_ocp_write(tp, 0xe02e, 0x4880);
+		r8168_mac_ocp_modify(tp, ALDPS_LTR, 0x0000, ALDPS_LTR_EN);
+		r8168_mac_ocp_write(tp, 0xcdd8, 0x9003);
+		r8168_mac_ocp_write(tp, 0xcdda, 0x9003);
+		r8168_mac_ocp_write(tp, 0xcddc, 0x9003);
+		r8168_mac_ocp_write(tp, 0xcdd2, 0x883c);
+		r8168_mac_ocp_write(tp, 0xcdd4, 0x8c12);
+		r8168_mac_ocp_write(tp, 0xcdd6, 0x9003);
+		r8168_mac_ocp_write(tp, 0xe0a6, 0x9003);
+		r8168_mac_ocp_write(tp, 0xe0a8, 0x9003);
+		r8168_mac_ocp_modify(tp, LTR_OBFF_LOCK, 0x0000, LINK_SPEED_CHANGE_EN);
+		break;
+	case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
 		r8168_mac_ocp_modify(tp, ALDPS_LTR, 0x0000, ALDPS_LTR_EN);
 		RTL_W8(tp, COMBO_LTR_EXTEND, RTL_R8(tp, COMBO_LTR_EXTEND) | COMBO_LTR_EXTEND_EN);
 		fallthrough;
@@ -3119,6 +3233,7 @@ static void rtl_enable_ltr(struct rtl8169_private *tp)
 	}
 	/* chip can trigger LTR */
 	r8168_mac_ocp_modify(tp, LTR_OBFF_LOCK, 0x0003, LTR_OBFF_LOCK_EN);
+	r8168_mac_ocp_modify(tp, SEND_LTR_MSG, 0x0000, LTR_MSG_EN);
 }
 
 static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
@@ -3152,6 +3267,7 @@ static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
 		rtl_enable_ltr(tp);
 		switch (tp->mac_version) {
 		case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
+		case RTL_GIGA_MAC_VER_52:
 		case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_LAST:
 			/* reset ephy tx/rx disable timer */
 			r8168_mac_ocp_modify(tp, 0xe094, 0xff00, 0);
@@ -3164,6 +3280,7 @@ static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
 	} else {
 		switch (tp->mac_version) {
 		case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
+		case RTL_GIGA_MAC_VER_52:
 		case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_LAST:
 			r8168_mac_ocp_modify(tp, 0xe092, 0x00ff, 0);
 			break;
@@ -3657,6 +3774,41 @@ static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
 }
 
+static void rtl_lowpower_hidden_functions(struct pci_dev *pdev)
+{
+	unsigned int slot = PCI_SLOT(pdev->devfn);
+	struct pci_bus *bus = pdev->bus;
+	int func, pos;
+	u16 val;
+
+	for (func = 2; func < 8; func++) {
+		unsigned int devfn = PCI_DEVFN(slot, func);
+
+		pos = pci_bus_find_capability(bus, devfn, PCI_CAP_ID_EXP);
+		if (pos) {
+			pci_bus_read_config_word(bus, devfn, pos + PCI_EXP_LNKCTL, &val);
+
+			if (PCI_POSSIBLE_ERROR(val))
+				continue;
+
+			val |= (PCI_EXP_LNKCTL_ASPMC | PCI_EXP_LNKCTL_CLKREQ_EN);
+			pci_bus_write_config_word(bus, devfn, pos + PCI_EXP_LNKCTL, val);
+		}
+
+		pos = pci_bus_find_capability(bus, devfn, PCI_CAP_ID_PM);
+		if (pos) {
+			pci_bus_read_config_word(bus, devfn, pos + PCI_PM_CTRL, &val);
+
+			if (PCI_POSSIBLE_ERROR(val))
+				continue;
+
+			val &= ~PCI_PM_CTRL_STATE_MASK;
+			val |= (RTL_PM_CTRL_D3HOT | PCI_PM_CTRL_PME_STATUS);
+			pci_bus_write_config_word(bus, devfn, pos + PCI_PM_CTRL, val);
+		}
+	}
+}
+
 static void rtl_hw_start_8117(struct rtl8169_private *tp)
 {
 	static const struct ephy_info e_info_8117[] = {
@@ -3693,12 +3845,14 @@ static void rtl_hw_start_8117(struct rtl8169_private *tp)
 
 	rtl_pcie_state_l2l3_disable(tp);
 
-	rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
-	if (rg_saw_cnt > 0) {
-		u16 sw_cnt_1ms_ini;
+	if (tp->phydev) {
+		rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
+		if (rg_saw_cnt > 0) {
+			u16 sw_cnt_1ms_ini;
 
-		sw_cnt_1ms_ini = (16000000 / rg_saw_cnt) & 0x0fff;
-		r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
+			sw_cnt_1ms_ini = (16000000 / rg_saw_cnt) & 0x0fff;
+			r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
+		}
 	}
 
 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0000);
@@ -4139,12 +4293,23 @@ static void rtl_hw_start(struct  rtl8169_private *tp)
 static int rtl8169_change_mtu(struct net_device *dev, int new_mtu)
 {
 	struct rtl8169_private *tp = netdev_priv(dev);
+	bool jumbo_before = dev->mtu > ETH_DATA_LEN;
+	bool jumbo_after = new_mtu > ETH_DATA_LEN;
 
 	WRITE_ONCE(dev->mtu, new_mtu);
 	netdev_update_features(dev);
 	rtl_jumbo_config(tp);
 	rtl_set_eee_txidle_timer(tp);
 
+	if (jumbo_before != jumbo_after) {
+		unsigned long caps = 0;
+
+		if (!jumbo_after)
+			caps |= (MAC_SYM_PAUSE | MAC_ASYM_PAUSE);
+
+		phylink_update_mac_pause_capabilities(tp->phylink, caps);
+	}
+
 	return 0;
 }
 
@@ -4869,8 +5034,13 @@ static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
 		goto out;
 	}
 
-	if (status & LinkChg)
-		phy_mac_interrupt(tp->phydev);
+	if (status & LinkChg) {
+		if (tp->phydev)
+			phy_mac_interrupt(tp->phydev);
+		else if (tp->sfp_mode)
+			phylink_mac_change(tp->phylink,
+					   !!(RTL_R8(tp, PHYstatus) & LinkStatus));
+	}
 
 	rtl_irq_disable(tp);
 	napi_schedule(&tp->napi);
@@ -4930,9 +5100,6 @@ static int rtl8169_poll(struct napi_struct *napi, int budget)
 
 static void rtl_enable_tx_lpi(struct rtl8169_private *tp, bool enable)
 {
-	if (!rtl_supports_eee(tp))
-		return;
-
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_52:
 		/* Adjust EEE LED frequency */
@@ -4963,40 +5130,15 @@ static void rtl_enable_tx_lpi(struct rtl8169_private *tp, bool enable)
 	}
 }
 
-static void r8169_phylink_handler(struct net_device *ndev)
-{
-	struct rtl8169_private *tp = netdev_priv(ndev);
-	struct device *d = tp_to_dev(tp);
-
-	if (netif_carrier_ok(ndev)) {
-		rtl_link_chg_patch(tp);
-		rtl_enable_tx_lpi(tp, tp->phydev->enable_tx_lpi);
-		pm_request_resume(d);
-	} else {
-		pm_runtime_idle(d);
-	}
-
-	phy_print_status(tp->phydev);
-}
-
 static int r8169_phy_connect(struct rtl8169_private *tp)
 {
-	struct phy_device *phydev = tp->phydev;
-	phy_interface_t phy_mode;
 	int ret;
 
-	phy_mode = tp->supports_gmii ? PHY_INTERFACE_MODE_GMII :
-		   PHY_INTERFACE_MODE_MII;
-
-	ret = phy_connect_direct(tp->dev, phydev, r8169_phylink_handler,
-				 phy_mode);
-	if (ret)
+	ret = phylink_connect_phy(tp->phylink, tp->phydev);
+	if (ret) {
+		netdev_err(tp->dev, "failed to connect phy\n");
 		return ret;
-
-	if (!tp->supports_gmii)
-		phy_set_max_speed(phydev, SPEED_100);
-
-	phy_attached_info(phydev);
+	}
 
 	return 0;
 }
@@ -5007,10 +5149,8 @@ static void rtl8169_down(struct rtl8169_private *tp)
 	/* Clear all task flags */
 	bitmap_zero(tp->wk.flags, RTL_FLAG_MAX);
 
-	phy_stop(tp->phydev);
-
 	/* Reset SerDes PHY to bring down fiber link */
-	if (tp->sfp_mode)
+	if (tp->sfp_mode == RTL_SFP_8127_ATF)
 		rtl_sfp_reset(tp);
 
 	rtl8169_update_counters(tp);
@@ -5032,14 +5172,12 @@ static void rtl8169_up(struct rtl8169_private *tp)
 		rtl8168_driver_start(tp);
 
 	pci_set_master(tp->pci_dev);
-	phy_init_hw(tp->phydev);
-	phy_resume(tp->phydev);
-	rtl8169_init_phy(tp);
+	if (tp->phydev)
+		rtl8169_init_phy(tp);
+
 	napi_enable(&tp->napi);
 	enable_work(&tp->wk.work);
 	rtl_reset_work(tp);
-
-	phy_start(tp->phydev);
 }
 
 static int rtl8169_close(struct net_device *dev)
@@ -5049,13 +5187,14 @@ static int rtl8169_close(struct net_device *dev)
 
 	pm_runtime_get_sync(&pdev->dev);
 
+	phylink_stop(tp->phylink);
 	netif_stop_queue(dev);
 	rtl8169_down(tp);
 	rtl8169_rx_clear(tp);
 
 	free_irq(tp->irq, tp);
 
-	phy_disconnect(tp->phydev);
+	phylink_disconnect_phy(tp->phylink);
 
 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
 			  tp->RxPhyAddr);
@@ -5112,11 +5251,13 @@ static int rtl_open(struct net_device *dev)
 	if (retval < 0)
 		goto err_release_fw_2;
 
-	retval = r8169_phy_connect(tp);
-	if (retval)
-		goto err_free_irq;
-
+	if (tp->phydev) {
+		retval = r8169_phy_connect(tp);
+		if (retval)
+			goto err_free_irq;
+	}
 	rtl8169_up(tp);
+	phylink_start(tp->phylink);
 	rtl8169_init_counter_offsets(tp);
 	netif_start_queue(dev);
 out:
@@ -5221,6 +5362,9 @@ static int rtl8169_resume(struct device *device)
 	/* Some chip versions may truncate packets without this initialization */
 	rtl_init_rxcfg(tp);
 
+	if (rtl_is_8116af(tp))
+		rtl_lowpower_hidden_functions(tp->pci_dev);
+
 	return rtl8169_runtime_resume(device);
 }
 
@@ -5288,6 +5432,7 @@ static void rtl_remove_one(struct pci_dev *pdev)
 		r8169_remove_leds(tp->leds);
 
 	unregister_netdev(tp->dev);
+	phylink_destroy(tp->phylink);
 
 	if (tp->dash_type != RTL_DASH_NONE)
 		rtl8168_driver_stop(tp);
@@ -5310,7 +5455,7 @@ static const struct net_device_ops rtl_netdev_ops = {
 	.ndo_fix_features	= rtl8169_fix_features,
 	.ndo_set_features	= rtl8169_set_features,
 	.ndo_set_mac_address	= rtl_set_mac_address,
-	.ndo_eth_ioctl		= phy_do_ioctl_running,
+	.ndo_eth_ioctl		= rtl8169_ioctl,
 	.ndo_set_rx_mode	= rtl_set_rx_mode,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= rtl8169_netpoll,
@@ -5474,16 +5619,6 @@ static int r8169_mdio_register(struct rtl8169_private *tp)
 		return -EUNATCH;
 	}
 
-	tp->phydev->mac_managed_pm = true;
-	if (rtl_supports_eee(tp))
-		phy_support_eee(tp->phydev);
-	phy_support_asym_pause(tp->phydev);
-
-	/* mimic behavior of r8125/r8126 vendor drivers */
-	if (tp->mac_version == RTL_GIGA_MAC_VER_61)
-		phy_disable_eee_mode(tp->phydev,
-				     ETHTOOL_LINK_MODE_2500baseT_Full_BIT);
-
 	/* PHY will be woken up in rtl_open() */
 	phy_suspend(tp->phydev);
 
@@ -5599,6 +5734,278 @@ static bool rtl_aspm_is_safe(struct rtl8169_private *tp)
 	return false;
 }
 
+static void rtl_mac_link_down(struct phylink_config *config, unsigned int mode,
+			      phy_interface_t interface)
+{
+	struct rtl8169_private *tp = container_of(config,
+						  struct rtl8169_private,
+						  phylink_config);
+
+	tp->speed = SPEED_UNKNOWN;
+	pm_runtime_idle(tp_to_dev(tp));
+}
+
+static void rtl_mac_link_up(struct phylink_config *config,
+			    struct phy_device *phydev,
+			    unsigned int mode,
+			    phy_interface_t interface,
+			    int speed, int duplex,
+			    bool tx_pause, bool rx_pause)
+{
+	struct rtl8169_private *tp = container_of(config,
+						  struct rtl8169_private,
+						  phylink_config);
+	struct device *d = tp_to_dev(tp);
+
+	tp->speed = speed;
+	rtl_link_chg_patch(tp, speed);
+
+	/*
+	 * Note: This hardware does not support forcing flow control.
+	 * Tx/Rx pause is completely hardwired: if autoneg is ON, hardware
+	 * handles it automatically based on PHY result; if autoneg is OFF,
+	 * hardware forces pause to be disabled. No software override exists.
+	 */
+	pm_request_resume(d);
+}
+
+static struct phylink_pcs *rtl_mac_select_pcs(struct phylink_config *config,
+					      phy_interface_t interface)
+{
+	struct rtl8169_private *tp = container_of(config,
+						  struct rtl8169_private,
+						  phylink_config);
+
+	if (!tp->pcs.ops)
+		return NULL;
+
+	if (interface == PHY_INTERFACE_MODE_1000BASEX ||
+	    interface == PHY_INTERFACE_MODE_10GBASER)
+		return &tp->pcs;
+
+	return NULL;
+}
+
+static void rtl_mac_config(struct phylink_config *config, unsigned int mode,
+			   const struct phylink_link_state *state)
+{
+}
+
+static u16 rtl8169_sds_read(struct rtl8169_private *tp, u16 sds_reg)
+{
+	unsigned long flags;
+	u16 val = 0;
+
+	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
+	__r8168_mac_ocp_write(tp, OCP_SDS_ADDR_REG, sds_reg);
+	__r8168_mac_ocp_write(tp, OCP_SDS_CMD_REG, SDS_CMD_READ);
+	val = __r8168_mac_ocp_read(tp, OCP_SDS_DATA_REG);
+	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
+
+	return val;
+}
+
+static void rtl8169_pcs_get_state(struct phylink_pcs *pcs,
+				  unsigned int neg_mode,
+				  struct phylink_link_state *state)
+{
+	struct rtl8169_private *tp = container_of(pcs, struct rtl8169_private,
+						  pcs);
+
+	if (tp->sfp_mode == RTL_SFP_8127_ATF) {
+		u16 stat1;
+
+		stat1 = rtl8169_sds_read(tp, RTL_SDS_C45_BASE + MDIO_STAT1);
+
+		if (!(stat1 & MDIO_STAT1_LSTATUS))
+			stat1 = rtl8169_sds_read(tp, RTL_SDS_C45_BASE + MDIO_STAT1);
+
+		state->link = !!(stat1 & MDIO_STAT1_LSTATUS);
+		if (!state->link)
+			return;
+
+		if (state->interface == PHY_INTERFACE_MODE_1000BASEX)
+			state->speed = SPEED_1000;
+		else
+			state->speed = SPEED_10000;
+
+		state->duplex = DUPLEX_FULL;
+	} else {
+		u16 bmsr, lpa;
+
+		bmsr = rtl8169_sds_read(tp, RTL_SDS_C22_BASE + MII_BMSR);
+		lpa = rtl8169_sds_read(tp, RTL_SDS_C22_BASE + MII_LPA);
+		phylink_mii_c22_pcs_decode_state(state, neg_mode, bmsr, lpa);
+	}
+}
+
+static int rtl8169_pcs_config(struct phylink_pcs *pcs, unsigned int mode,
+			      phy_interface_t interface,
+			      const unsigned long *advertising,
+			      bool permit_pause_to_mac)
+{
+	struct rtl8169_private *tp = container_of(pcs, struct rtl8169_private,
+						  pcs);
+
+	if (tp->sfp_mode == RTL_SFP_8127_ATF) {
+		switch (interface) {
+		case PHY_INTERFACE_MODE_10GBASER:
+			r8127_sfp_init_10g(tp);
+			break;
+		case PHY_INTERFACE_MODE_1000BASEX:
+			r8127_sfp_init_1g(tp);
+			break;
+		default:
+			netdev_err(tp->dev, "Unsupported SFP interface mode: %s\n",
+				   phy_modes(interface));
+			return -EOPNOTSUPP;
+		}
+	}
+
+	return 0;
+}
+
+static int rtl8169_pcs_validate(struct phylink_pcs *pcs,
+				unsigned long *supported,
+				const struct phylink_link_state *state)
+{
+	return 0;
+}
+
+static void rtl8169_pcs_an_restart(struct phylink_pcs *pcs)
+{
+}
+
+static void rtl_mac_disable_tx_lpi(struct phylink_config *config)
+{
+	struct rtl8169_private *tp = container_of(config,
+						  struct rtl8169_private,
+						  phylink_config);
+
+	rtl_enable_tx_lpi(tp, false);
+}
+
+static int rtl_mac_enable_tx_lpi(struct phylink_config *config,
+				 u32 timer,
+				 bool tx_clk_stop)
+{
+	struct rtl8169_private *tp = container_of(config,
+						  struct rtl8169_private,
+						  phylink_config);
+
+	rtl_enable_tx_lpi(tp, true);
+
+	return 0;
+}
+
+static const struct phylink_mac_ops rtl_phylink_mac_ops = {
+	.mac_select_pcs = rtl_mac_select_pcs,
+	.mac_config = rtl_mac_config,
+	.mac_link_down  = rtl_mac_link_down,
+	.mac_link_up    = rtl_mac_link_up,
+	.mac_disable_tx_lpi = rtl_mac_disable_tx_lpi,
+	.mac_enable_tx_lpi = rtl_mac_enable_tx_lpi,
+};
+
+static unsigned long rtl8169_get_lpi_caps(struct rtl8169_private *tp)
+{
+	unsigned long caps = 0;
+
+	if (!rtl_supports_eee(tp) || tp->sfp_mode == RTL_SFP_8127_ATF)
+		return 0;
+
+	caps |= MAC_100FD | MAC_1000FD;
+
+	/* mimic behavior of r8125/r8126 vendor drivers
+	 * RTL_GIGA_MAC_VER_61 doesn't support 2.5G eee
+	 */
+	if (tp->mac_version >= RTL_GIGA_MAC_VER_63)
+		caps |= MAC_2500FD;
+	if (tp->mac_version >= RTL_GIGA_MAC_VER_70)
+		caps |= MAC_5000FD;
+	if (tp->mac_version == RTL_GIGA_MAC_VER_80)
+		caps |= MAC_10000FD;
+
+	return caps;
+}
+
+static const struct phylink_pcs_ops r8169_pcs_ops = {
+	.pcs_validate = rtl8169_pcs_validate,
+	.pcs_get_state = rtl8169_pcs_get_state,
+	.pcs_config = rtl8169_pcs_config,
+	.pcs_an_restart = rtl8169_pcs_an_restart,
+};
+
+static int rtl_init_phylink(struct rtl8169_private *tp)
+{
+	struct phylink *pl;
+	phy_interface_t phy_mode;
+
+	tp->phylink_config.dev = &tp->dev->dev;
+	tp->phylink_config.type = PHYLINK_NETDEV;
+	tp->phylink_config.mac_managed_pm = true;
+	tp->phylink_config.lpi_capabilities = rtl8169_get_lpi_caps(tp);
+	tp->phylink_config.eee_enabled_default = !!tp->phylink_config.lpi_capabilities;
+	tp->phylink_config.mac_capabilities |= MAC_ASYM_PAUSE | MAC_SYM_PAUSE;
+
+	switch (tp->sfp_mode) {
+	case RTL_SFP_8168_AF:
+		tp->pcs.ops = &r8169_pcs_ops;
+		tp->phylink_config.default_an_inband = true;
+		phy_mode = PHY_INTERFACE_MODE_1000BASEX;
+		tp->phylink_config.mac_capabilities |= MAC_1000FD;
+		break;
+	case RTL_SFP_8127_ATF:
+		tp->pcs.ops = &r8169_pcs_ops;
+		phy_mode = PHY_INTERFACE_MODE_10GBASER;
+		tp->phylink_config.default_an_inband = true;
+		tp->phylink_config.mac_capabilities |= MAC_1000FD | MAC_10000FD;
+		__set_bit(PHY_INTERFACE_MODE_10GBASER,
+			  tp->phylink_config.supported_interfaces);
+		__set_bit(PHY_INTERFACE_MODE_1000BASEX,
+			  tp->phylink_config.supported_interfaces);
+		break;
+	default:
+		tp->phylink_config.mac_capabilities |= MAC_10 | MAC_100;
+
+		if (tp->mac_version == RTL_GIGA_MAC_VER_80)
+			tp->phylink_config.mac_capabilities |= MAC_1000FD |
+							       MAC_2500FD |
+							       MAC_5000FD |
+							       MAC_10000FD;
+		else if (tp->mac_version == RTL_GIGA_MAC_VER_70)
+			tp->phylink_config.mac_capabilities |= MAC_1000FD |
+							       MAC_2500FD |
+							       MAC_5000FD;
+		else if (tp->mac_version >= RTL_GIGA_MAC_VER_61)
+			tp->phylink_config.mac_capabilities |= MAC_1000FD |
+							       MAC_2500FD;
+		else
+			if (tp->supports_gmii)
+				tp->phylink_config.mac_capabilities |= MAC_1000FD;
+
+		if (tp->mac_version < RTL_GIGA_MAC_VER_61)
+			phy_mode = tp->supports_gmii ? PHY_INTERFACE_MODE_GMII :
+				   PHY_INTERFACE_MODE_MII;
+		else
+			phy_mode = PHY_INTERFACE_MODE_INTERNAL;
+		break;
+	}
+
+	__set_bit(phy_mode, tp->phylink_config.supported_interfaces);
+	if (tp->phylink_config.lpi_capabilities)
+		__set_bit(phy_mode, tp->phylink_config.lpi_interfaces);
+
+	pl = phylink_create(&tp->phylink_config, tp_to_dev(tp)->fwnode,
+			    phy_mode, &rtl_phylink_mac_ops);
+	if (IS_ERR(pl))
+		return PTR_ERR(pl);
+
+	tp->phylink = pl;
+
+	return 0;
+}
+
 static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	const struct rtl_chip_info *chip;
@@ -5667,6 +6074,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 				     ext_xid_str, xid);
 	tp->mac_version = chip->mac_version;
 	tp->fw_name = chip->fw_name;
+	tp->speed = SPEED_UNKNOWN;
 
 	/* Disable ASPM L1 as that cause random device stop working
 	 * problems as well as full system hangs for some PCIe devices users.
@@ -5679,12 +6087,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 	tp->aspm_manageable = !rc;
 
-	if (rtl_is_8125(tp)) {
-		u16 data = r8168_mac_ocp_read(tp, 0xd006);
-
-		if ((data & 0xff) == 0x07)
-			tp->sfp_mode = true;
-	}
+	tp->sfp_mode = rtl_get_sfp_mode(tp);
 
 	tp->dash_type = rtl_get_dash_type(tp);
 	tp->dash_enabled = rtl_dash_is_enabled(tp);
@@ -5788,13 +6191,23 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_drvdata(pdev, tp);
 
-	rc = r8169_mdio_register(tp);
+	rc = rtl_init_phylink(tp);
 	if (rc)
 		return rc;
 
+	if (tp->sfp_mode == RTL_SFP_NONE) {
+		rc = r8169_mdio_register(tp);
+		if (rc) {
+			phylink_destroy(tp->phylink);
+			return rc;
+		}
+	}
+
 	rc = register_netdev(dev);
-	if (rc)
+	if (rc) {
+		phylink_destroy(tp->phylink);
 		return rc;
+	}
 
 	if (IS_ENABLED(CONFIG_R8169_LEDS)) {
 		if (rtl_is_8125(tp))
@@ -5817,6 +6230,9 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		rtl8168_driver_start(tp);
 	}
 
+	if (rtl_is_8116af(tp))
+		rtl_lowpower_hidden_functions(tp->pci_dev);
+
 	if (pci_dev_run_wake(pdev))
 		pm_runtime_put_sync(&pdev->dev);
 
diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h
index 3ee4c61081890c..5d35b40d7c584b 100644
--- a/drivers/net/ethernet/renesas/ravb.h
+++ b/drivers/net/ethernet/renesas/ravb.h
@@ -249,6 +249,8 @@ enum APSR_BIT {
 	APSR_RDM	= 0x00002000,
 	APSR_TDM	= 0x00004000,
 	APSR_MIISELECT	= 0x01000000,	/* R-Car V4M only */
+	APSR_GPTPTIMER_SOURCE = BIT(25), /* Gen4 */
+	APSR_GPTPCLOCK	= BIT(29),	/* Gen4 */
 };
 
 /* RCR */
@@ -1028,13 +1030,37 @@ struct ravb_ptp_perout {
 struct ravb_ptp {
 	struct ptp_clock *clock;
 	struct ptp_clock_info info;
-	int phc_index;
 	u32 default_addend;
 	u32 current_addend;
 	int extts[N_EXT_TS];
 	struct ravb_ptp_perout perout[N_PER_OUT];
 };
 
+/**
+ * struct ravb_gptp_info - Platform specific gPTP behavior
+ *
+ * Each generation of RAVB have slightly different behaviors when interacting
+ * with the gPTP clock. This struct provides the callbacks to be called at
+ * critical points in the RAVB driver.
+ *
+ * @probe:		Probe the gPTP clock
+ * @clock_index:	Get the PTP clock index, if any
+ * @set_config_mode:	Enter config mode
+ * @dmac_start:		Called when the DMAC starts
+ * @dmac_stop:		Called when the DMAC stops
+ * @ndev_open:		Called when the ndev is opened
+ * @ndev_close:		Called when the ndev is closed
+ */
+struct ravb_gptp_info {
+	int (*probe)(struct net_device *ndev);
+	int (*clock_index)(struct net_device *ndev);
+	int (*set_config_mode)(struct net_device *ndev);
+	int (*dmac_start)(struct net_device *ndev);
+	void (*dmac_stop)(struct net_device *ndev);
+	int (*ndev_open)(struct net_device *ndev);
+	void (*ndev_close)(struct net_device *ndev);
+};
+
 struct ravb_hw_info {
 	int (*receive)(struct net_device *ndev, int budget, int q);
 	void (*set_rate)(struct net_device *ndev);
@@ -1053,6 +1079,7 @@ struct ravb_hw_info {
 	u32 rx_buffer_size;
 	u32 rx_desc_size;
 	u32 dbat_entry_num;
+	const struct ravb_gptp_info *ptp; /* Callbacks to handle gPTP interactions. */
 	unsigned aligned_tx: 1;
 	unsigned coalesce_irqs:1;	/* Needs software IRQ coalescing */
 
@@ -1063,9 +1090,6 @@ struct ravb_hw_info {
 	unsigned multi_irqs:1;		/* AVB-DMAC and E-MAC has multiple irqs */
 	unsigned irq_en_dis:1;		/* Has separate irq enable and disable regs */
 	unsigned err_mgmt_irqs:1;	/* Line1 (Err) and Line2 (Mgmt) irqs are separate */
-	unsigned gptp:1;		/* AVB-DMAC has gPTP support */
-	unsigned ccc_gac:1;		/* AVB-DMAC has gPTP support active in config mode */
-	unsigned gptp_ref_clk:1;	/* gPTP has separate reference clock */
 	unsigned nc_queues:1;		/* AVB-DMAC has RX and TX NC queues */
 	unsigned magic_pkt:1;		/* E-MAC supports magic packet detection */
 	unsigned half_duplex:1;		/* E-MAC supports half duplex mode */
@@ -1110,6 +1134,7 @@ struct ravb_private {
 	struct list_head ts_skb_list;
 	u32 ts_skb_tag;
 	struct ravb_ptp ptp;
+	struct device_node *of_gptp;	/* Reference to external gPTP clock, if any. */
 	spinlock_t lock;		/* Register access lock */
 	u32 cur_rx[NUM_RX_QUEUE];	/* Consumer ring indices */
 	u32 dirty_rx[NUM_RX_QUEUE];	/* Producer ring indices */
@@ -1163,7 +1188,7 @@ void ravb_modify(struct net_device *ndev, enum ravb_reg reg, u32 clear,
 int ravb_wait(struct net_device *ndev, enum ravb_reg reg, u32 mask, u32 value);
 
 void ravb_ptp_interrupt(struct net_device *ndev);
-void ravb_ptp_init(struct net_device *ndev, struct platform_device *pdev);
+int ravb_ptp_init(struct net_device *ndev);
 void ravb_ptp_stop(struct net_device *ndev);
 
 #endif	/* #ifndef __RAVB_H__ */
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index ea1c7e536791e8..5a07cc0180d8b5 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -707,7 +707,15 @@ static int ravb_dmac_init(struct net_device *ndev)
 		return error;
 
 	/* Setting the control will start the AVB-DMAC process. */
-	return ravb_set_opmode(ndev, CCC_OPC_OPERATION);
+	error = ravb_set_opmode(ndev, CCC_OPC_OPERATION);
+	if (error)
+		return error;
+
+	/* Initialise PTP Clock driver */
+	if (info->ptp && info->ptp->dmac_start)
+		return info->ptp->dmac_start(ndev);
+
+	return 0;
 }
 
 static void ravb_get_tx_tstamp(struct net_device *ndev)
@@ -1115,6 +1123,10 @@ static int ravb_stop_dma(struct net_device *ndev)
 			netdev_err(ndev, "failed to stop AXI BUS\n");
 	}
 
+	/* Stop PTP Clock driver */
+	if (info->ptp && info->ptp->dmac_stop)
+		info->ptp->dmac_stop(ndev);
+
 	/* Stop AVB-DMAC process */
 	return ravb_set_opmode(ndev, CCC_OPC_CONFIG);
 }
@@ -1719,9 +1731,7 @@ static int ravb_set_ringparam(struct net_device *ndev,
 
 	if (netif_running(ndev)) {
 		netif_device_detach(ndev);
-		/* Stop PTP Clock driver */
-		if (info->gptp)
-			ravb_ptp_stop(ndev);
+
 		/* Wait for DMA stopping */
 		error = ravb_stop_dma(ndev);
 		if (error) {
@@ -1752,10 +1762,6 @@ static int ravb_set_ringparam(struct net_device *ndev,
 
 		ravb_emac_init(ndev);
 
-		/* Initialise PTP Clock driver */
-		if (info->gptp)
-			ravb_ptp_init(ndev, priv->pdev);
-
 		netif_device_attach(ndev);
 	}
 
@@ -1767,8 +1773,13 @@ static int ravb_get_ts_info(struct net_device *ndev,
 {
 	struct ravb_private *priv = netdev_priv(ndev);
 	const struct ravb_hw_info *hw_info = priv->info;
+	int index = -1;
+
+	if (hw_info->ptp && hw_info->ptp->clock_index)
+		index = hw_info->ptp->clock_index(ndev);
 
-	if (hw_info->gptp || hw_info->ccc_gac) {
+	/* Only advertise ptp clock if present. */
+	if (index >= 0) {
 		info->so_timestamping =
 			SOF_TIMESTAMPING_TX_SOFTWARE |
 			SOF_TIMESTAMPING_TX_HARDWARE |
@@ -1779,7 +1790,7 @@ static int ravb_get_ts_info(struct net_device *ndev,
 			(1 << HWTSTAMP_FILTER_NONE) |
 			(1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
 			(1 << HWTSTAMP_FILTER_ALL);
-		info->phc_index = READ_ONCE(priv->ptp.phc_index);
+		info->phc_index = index;
 	}
 
 	return 0;
@@ -1826,55 +1837,24 @@ static const struct ethtool_ops ravb_ethtool_ops = {
 };
 
 static int ravb_set_config_mode(struct net_device *ndev)
-{
-	struct ravb_private *priv = netdev_priv(ndev);
-	const struct ravb_hw_info *info = priv->info;
-	int error;
-
-	if (info->gptp) {
-		error = ravb_set_opmode(ndev, CCC_OPC_CONFIG);
-		if (error)
-			return error;
-		/* Set CSEL value */
-		ravb_modify(ndev, CCC, CCC_CSEL, CCC_CSEL_HPB);
-	} else if (info->ccc_gac) {
-		error = ravb_set_opmode(ndev, CCC_OPC_CONFIG | CCC_GAC | CCC_CSEL_HPB);
-	} else {
-		error = ravb_set_opmode(ndev, CCC_OPC_CONFIG);
-	}
-
-	return error;
-}
-
-static void ravb_set_gti(struct net_device *ndev)
 {
 	struct ravb_private *priv = netdev_priv(ndev);
 	const struct ravb_hw_info *info = priv->info;
 
-	if (!(info->gptp || info->ccc_gac))
-		return;
+	if (info->ptp && info->ptp->set_config_mode)
+		return info->ptp->set_config_mode(ndev);
 
-	ravb_write(ndev, priv->gti_tiv, GTI);
-
-	/* Request GTI loading */
-	ravb_modify(ndev, GCCR, GCCR_LTI, GCCR_LTI);
+	return ravb_set_opmode(ndev, CCC_OPC_CONFIG);
 }
 
-static int ravb_compute_gti(struct net_device *ndev)
+static int ravb_compute_gti(struct net_device *ndev, struct clk *clk)
 {
 	struct ravb_private *priv = netdev_priv(ndev);
-	const struct ravb_hw_info *info = priv->info;
 	struct device *dev = ndev->dev.parent;
 	unsigned long rate;
 	u64 inc;
 
-	if (!(info->gptp || info->ccc_gac))
-		return 0;
-
-	if (info->gptp_ref_clk)
-		rate = clk_get_rate(priv->gptp_clk);
-	else
-		rate = clk_get_rate(priv->clk);
+	rate = clk_get_rate(clk);
 	if (!rate)
 		return -EINVAL;
 
@@ -1974,11 +1954,12 @@ static int ravb_open(struct net_device *ndev)
 
 	ravb_emac_init(ndev);
 
-	ravb_set_gti(ndev);
-
 	/* Initialise PTP Clock driver */
-	if (info->gptp || info->ccc_gac)
-		ravb_ptp_init(ndev, priv->pdev);
+	if (info->ptp && info->ptp->ndev_open) {
+		error = info->ptp->ndev_open(ndev);
+		if (error)
+			goto out_ptp_stop;
+	}
 
 	/* PHY control start */
 	error = ravb_phy_start(ndev);
@@ -1991,8 +1972,8 @@ static int ravb_open(struct net_device *ndev)
 
 out_ptp_stop:
 	/* Stop PTP Clock driver */
-	if (info->gptp || info->ccc_gac)
-		ravb_ptp_stop(ndev);
+	if (info->ptp && info->ptp->ndev_close)
+		info->ptp->ndev_close(ndev);
 	ravb_stop_dma(ndev);
 out_set_reset:
 	ravb_set_opmode(ndev, CCC_OPC_RESET);
@@ -2036,10 +2017,6 @@ static void ravb_tx_timeout_work(struct work_struct *work)
 
 	netif_tx_stop_all_queues(ndev);
 
-	/* Stop PTP Clock driver */
-	if (info->gptp)
-		ravb_ptp_stop(ndev);
-
 	/* Wait for DMA stopping */
 	if (ravb_stop_dma(ndev)) {
 		/* If ravb_stop_dma() fails, the hardware is still operating
@@ -2072,10 +2049,6 @@ static void ravb_tx_timeout_work(struct work_struct *work)
 	ravb_emac_init(ndev);
 
 out:
-	/* Initialise PTP Clock driver */
-	if (info->gptp)
-		ravb_ptp_init(ndev, priv->pdev);
-
 	netif_tx_start_all_queues(ndev);
 
 out_unlock:
@@ -2208,7 +2181,7 @@ static netdev_tx_t ravb_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	desc->dptr = cpu_to_le32(dma_addr);
 
 	/* TX timestamp required */
-	if (info->gptp || info->ccc_gac) {
+	if (info->ptp) {
 		if (q == RAVB_NC) {
 			ts_skb = kmalloc_obj(*ts_skb, GFP_ATOMIC);
 			if (!ts_skb) {
@@ -2390,8 +2363,8 @@ static int ravb_close(struct net_device *ndev)
 	}
 
 	/* Stop PTP Clock driver */
-	if (info->gptp || info->ccc_gac)
-		ravb_ptp_stop(ndev);
+	if (info->ptp && info->ptp->ndev_close)
+		info->ptp->ndev_close(ndev);
 
 	/* Set the config mode to stop the AVB-DMAC's processes */
 	if (ravb_stop_dma(ndev) < 0)
@@ -2399,7 +2372,7 @@ static int ravb_close(struct net_device *ndev)
 			   "device will be stopped after h/w processes are done.\n");
 
 	/* Clear the timestamp list */
-	if (info->gptp || info->ccc_gac) {
+	if (info->ptp) {
 		list_for_each_entry_safe(ts_skb, ts_skb2, &priv->ts_skb_list, list) {
 			list_del(&ts_skb->list);
 			kfree_skb(ts_skb->skb);
@@ -2681,6 +2654,45 @@ static int ravb_mdio_release(struct ravb_private *priv)
 	return 0;
 }
 
+static int ravb_gen2_ptp_probe(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	return ravb_compute_gti(ndev, priv->clk);
+}
+
+static int ravb_gen2_ptp_clock_index(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	if (!priv->ptp.clock)
+		return -ENODEV;
+
+	return ptp_clock_index(priv->ptp.clock);
+}
+
+static int ravb_gen2_ptp_set_config_mode(struct net_device *ndev)
+{
+	int ret;
+
+	ret = ravb_set_opmode(ndev, CCC_OPC_CONFIG);
+	if (ret)
+		return ret;
+
+	/* gPTP Clock Select High-speed peripheral bus clock. */
+	ravb_modify(ndev, CCC, CCC_CSEL, CCC_CSEL_HPB);
+
+	return 0;
+}
+
+static const struct ravb_gptp_info ravb_gen2_ptp_info = {
+	.probe = ravb_gen2_ptp_probe,
+	.clock_index = ravb_gen2_ptp_clock_index,
+	.set_config_mode = ravb_gen2_ptp_set_config_mode,
+	.dmac_start = ravb_ptp_init,
+	.dmac_stop = ravb_ptp_stop,
+};
+
 static const struct ravb_hw_info ravb_gen2_hw_info = {
 	.receive = ravb_rx_rcar,
 	.set_rate = ravb_set_rate_rcar,
@@ -2699,12 +2711,26 @@ static const struct ravb_hw_info ravb_gen2_hw_info = {
 			  SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
 	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
 	.dbat_entry_num = 22,
+	.ptp = &ravb_gen2_ptp_info,
 	.aligned_tx = 1,
-	.gptp = 1,
 	.nc_queues = 1,
 	.magic_pkt = 1,
 };
 
+static int ravb_gen3_ptp_set_config_mode(struct net_device *ndev)
+{
+	/* Enable gPTP Clock and Select High-speed peripheral bus clock. */
+	return ravb_set_opmode(ndev, CCC_OPC_CONFIG | CCC_GAC | CCC_CSEL_HPB);
+}
+
+static const struct ravb_gptp_info ravb_gen3_ptp_info = {
+	.probe = ravb_gen2_ptp_probe,
+	.clock_index = ravb_gen2_ptp_clock_index,
+	.set_config_mode = ravb_gen3_ptp_set_config_mode,
+	.ndev_open = ravb_ptp_init,
+	.ndev_close = ravb_ptp_stop,
+};
+
 static const struct ravb_hw_info ravb_gen3_hw_info = {
 	.receive = ravb_rx_rcar,
 	.set_rate = ravb_set_rate_rcar,
@@ -2723,15 +2749,66 @@ static const struct ravb_hw_info ravb_gen3_hw_info = {
 			  SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
 	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
 	.dbat_entry_num = 22,
+	.ptp = &ravb_gen3_ptp_info,
 	.internal_delay = 1,
 	.tx_counters = 1,
 	.multi_irqs = 1,
 	.irq_en_dis = 1,
-	.ccc_gac = 1,
 	.nc_queues = 1,
 	.magic_pkt = 1,
 };
 
+static int ravb_gen4_ptp_probe(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	priv->of_gptp = of_parse_phandle(priv->pdev->dev.of_node, "ptp-timer", 0);
+	if (!priv->of_gptp)
+		return 0;
+
+	if (!of_device_is_available(priv->of_gptp)) {
+		of_node_put(priv->of_gptp);
+		priv->of_gptp = NULL;
+	}
+
+	return 0;
+}
+
+static int ravb_gen4_ptp_clock_index(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	/* If no clock, mimic ptp_clock_index_by_of_node() fail and return -1 */
+	if (!priv->of_gptp)
+		return -1;
+
+	return ptp_clock_index_by_of_node(priv->of_gptp);
+}
+
+static int ravb_gen4_ptp_set_config_mode(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+	int ret;
+
+	/* Enable gPTP Clock and Select High-speed peripheral bus clock. */
+	ret = ravb_set_opmode(ndev, CCC_OPC_CONFIG | CCC_GAC | CCC_CSEL_HPB);
+	if (ret)
+		return ret;
+
+	/* Set PTP source to GPTP module, only option on Gen4. */
+	if (priv->of_gptp)
+		ravb_modify(ndev, APSR, APSR_GPTPTIMER_SOURCE | APSR_GPTPCLOCK,
+			    APSR_GPTPTIMER_SOURCE | APSR_GPTPCLOCK);
+
+	return 0;
+}
+
+static const struct ravb_gptp_info ravb_gen4_ptp_info = {
+	.probe = ravb_gen4_ptp_probe,
+	.clock_index = ravb_gen4_ptp_clock_index,
+	.set_config_mode = ravb_gen4_ptp_set_config_mode,
+};
+
 static const struct ravb_hw_info ravb_gen4_hw_info = {
 	.receive = ravb_rx_rcar,
 	.set_rate = ravb_set_rate_rcar,
@@ -2754,11 +2831,30 @@ static const struct ravb_hw_info ravb_gen4_hw_info = {
 	.tx_counters = 1,
 	.multi_irqs = 1,
 	.irq_en_dis = 1,
-	.ccc_gac = 1,
+	.ptp = &ravb_gen4_ptp_info,
 	.nc_queues = 1,
 	.magic_pkt = 1,
 };
 
+static int ravb_rzv2m_ptp_probe(struct net_device *ndev)
+{
+	struct ravb_private *priv = netdev_priv(ndev);
+
+	priv->gptp_clk = devm_clk_get(&priv->pdev->dev, "gptp");
+	if (IS_ERR(priv->gptp_clk))
+		return PTR_ERR(priv->gptp_clk);
+
+	return ravb_compute_gti(ndev, priv->gptp_clk);
+}
+
+static const struct ravb_gptp_info ravb_rzv2m_ptp_info = {
+	.probe = ravb_rzv2m_ptp_probe,
+	.clock_index = ravb_gen2_ptp_clock_index,
+	.set_config_mode = ravb_gen2_ptp_set_config_mode,
+	.dmac_start = ravb_ptp_init,
+	.dmac_stop = ravb_ptp_stop,
+};
+
 static const struct ravb_hw_info ravb_rzv2m_hw_info = {
 	.receive = ravb_rx_rcar,
 	.set_rate = ravb_set_rate_rcar,
@@ -2779,8 +2875,7 @@ static const struct ravb_hw_info ravb_rzv2m_hw_info = {
 	.dbat_entry_num = 22,
 	.multi_irqs = 1,
 	.err_mgmt_irqs = 1,
-	.gptp = 1,
-	.gptp_ref_clk = 1,
+	.ptp = &ravb_rzv2m_ptp_info,
 	.nc_queues = 1,
 	.magic_pkt = 1,
 };
@@ -2955,7 +3050,6 @@ static int ravb_probe(struct platform_device *pdev)
 	priv->rstc = rstc;
 	priv->ndev = ndev;
 	priv->pdev = pdev;
-	priv->ptp.phc_index = -1;
 	priv->num_tx_ring[RAVB_BE] = BE_TX_RING_SIZE;
 	priv->num_rx_ring[RAVB_BE] = BE_RX_RING_SIZE;
 	if (info->nc_queues) {
@@ -2973,12 +3067,10 @@ static int ravb_probe(struct platform_device *pdev)
 		goto out_reset_assert;
 	}
 
-	if (info->gptp_ref_clk) {
-		priv->gptp_clk = devm_clk_get(&pdev->dev, "gptp");
-		if (IS_ERR(priv->gptp_clk)) {
-			error = PTR_ERR(priv->gptp_clk);
+	if (info->ptp && info->ptp->probe) {
+		error = info->ptp->probe(ndev);
+		if (error)
 			goto out_reset_assert;
-		}
 	}
 
 	priv->refclk = devm_clk_get_optional(&pdev->dev, "refclk");
@@ -3031,10 +3123,6 @@ static int ravb_probe(struct platform_device *pdev)
 	ndev->netdev_ops = &ravb_netdev_ops;
 	ndev->ethtool_ops = &ravb_ethtool_ops;
 
-	error = ravb_compute_gti(ndev);
-	if (error)
-		goto out_rpm_put;
-
 	ravb_parse_delay_mode(np, ndev);
 
 	/* Allocate descriptor base address table */
@@ -3157,6 +3245,7 @@ static void ravb_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(dev);
 	clk_unprepare(priv->refclk);
+	of_node_put(priv->of_gptp);
 	reset_control_assert(priv->rstc);
 	free_netdev(ndev);
 	platform_set_drvdata(pdev, NULL);
@@ -3182,9 +3271,6 @@ static int ravb_wol_setup(struct net_device *ndev)
 	/* Enable MagicPacket */
 	ravb_modify(ndev, ECMR, ECMR_MPDE, ECMR_MPDE);
 
-	if (priv->info->ccc_gac)
-		ravb_ptp_stop(ndev);
-
 	return enable_irq_wake(priv->emac_irq);
 }
 
@@ -3204,9 +3290,6 @@ static int ravb_wol_restore(struct net_device *ndev)
 	if (error)
 		return error;
 
-	if (priv->info->ccc_gac)
-		ravb_ptp_init(ndev, priv->pdev);
-
 	if (info->nc_queues)
 		napi_enable(&priv->napi[RAVB_NC]);
 	napi_enable(&priv->napi[RAVB_BE]);
diff --git a/drivers/net/ethernet/renesas/ravb_ptp.c b/drivers/net/ethernet/renesas/ravb_ptp.c
index 43218bc15b1519..407ae97b76bbb1 100644
--- a/drivers/net/ethernet/renesas/ravb_ptp.c
+++ b/drivers/net/ethernet/renesas/ravb_ptp.c
@@ -313,31 +313,39 @@ void ravb_ptp_interrupt(struct net_device *ndev)
 	ravb_write(ndev, ~(gis | GIS_RESERVED), GIS);
 }
 
-void ravb_ptp_init(struct net_device *ndev, struct platform_device *pdev)
+int ravb_ptp_init(struct net_device *ndev)
 {
 	struct ravb_private *priv = netdev_priv(ndev);
-	struct ptp_clock *clock;
 	unsigned long flags;
+	int ret = 0;
 
 	priv->ptp.info = ravb_ptp_info;
 
-	priv->ptp.default_addend = ravb_read(ndev, GTI);
+	priv->ptp.default_addend = priv->gti_tiv;
 	priv->ptp.current_addend = priv->ptp.default_addend;
 
 	spin_lock_irqsave(&priv->lock, flags);
+
+	/* Set gPTP Timer Increment Value. */
+	ravb_write(ndev, priv->ptp.default_addend, GTI);
+
+	/* Request GTI loading. */
+	ravb_modify(ndev, GCCR, GCCR_LTI, GCCR_LTI);
+
+	/* Wait for GIT loading to complete by checking there are no requests. */
 	ravb_wait(ndev, GCCR, GCCR_TCR, GCCR_TCR_NOREQ);
+
 	ravb_modify(ndev, GCCR, GCCR_TCSS, GCCR_TCSS_ADJGPTP);
 	spin_unlock_irqrestore(&priv->lock, flags);
 
-	clock = ptp_clock_register(&priv->ptp.info, &pdev->dev);
-	if (IS_ERR(clock)) {
-		netdev_err(ndev, "failed to register PTP clock: %pe\n", clock);
-		clock = NULL;
+	priv->ptp.clock = ptp_clock_register(&priv->ptp.info, &priv->pdev->dev);
+	if (IS_ERR(priv->ptp.clock)) {
+		ret = PTR_ERR(priv->ptp.clock);
+		priv->ptp.clock = NULL;
+		ravb_ptp_stop(ndev);
 	}
 
-	WRITE_ONCE(priv->ptp.clock, clock);
-	if (clock)
-		WRITE_ONCE(priv->ptp.phc_index, ptp_clock_index(clock));
+	return ret;
 }
 
 static void ravb_ptp_disable(struct net_device *ndev)
@@ -360,14 +368,12 @@ static void ravb_ptp_sync_irqs(struct net_device *ndev)
 void ravb_ptp_stop(struct net_device *ndev)
 {
 	struct ravb_private *priv = netdev_priv(ndev);
-	struct ptp_clock *clock;
-
-	WRITE_ONCE(priv->ptp.phc_index, -1);
-	clock = xchg(&priv->ptp.clock, NULL);
 
 	ravb_ptp_disable(ndev);
 	ravb_ptp_sync_irqs(ndev);
 
-	if (clock)
-		ptp_clock_unregister(clock);
+	if (priv->ptp.clock) {
+		ptp_clock_unregister(priv->ptp.clock);
+		priv->ptp.clock = NULL;
+	}
 }
diff --git a/drivers/net/ethernet/renesas/rswitch_main.c b/drivers/net/ethernet/renesas/rswitch_main.c
index 755232994fccb5..64b821ca02f8d5 100644
--- a/drivers/net/ethernet/renesas/rswitch_main.c
+++ b/drivers/net/ethernet/renesas/rswitch_main.c
@@ -2162,11 +2162,8 @@ static int renesas_eth_sw_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
-	if (ret < 0) {
-		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (ret < 0)
-			return ret;
-	}
+	if (ret)
+		return ret;
 
 	priv->gwca.index = AGENT_INDEX_GWCA;
 	priv->gwca.num_queues = min(RSWITCH_NUM_PORTS * NUM_QUEUES_PER_NDEV,
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index 84a55f2b48ffe1..0fcd65fc0af7fc 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -3134,7 +3134,7 @@ static int rocker_netevent_event(struct notifier_block *unused,
 
 	switch (event) {
 	case NETEVENT_NEIGH_UPDATE:
-		if (n->tbl != &arp_tbl)
+		if (n->tbl != arp_table(&init_net))
 			return NOTIFY_DONE;
 		dev = n->dev;
 		if (!rocker_port_dev_check(dev))
diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c
index 15d19a8a171014..ead8b447b89c40 100644
--- a/drivers/net/ethernet/rocker/rocker_ofdpa.c
+++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c
@@ -1336,7 +1336,7 @@ static int ofdpa_port_ipv4_resolve(struct ofdpa_port *ofdpa_port,
 	int err = 0;
 
 	if (!n) {
-		n = neigh_create(&arp_tbl, &ip_addr, dev);
+		n = neigh_create(arp_table(&init_net), &ip_addr, dev);
 		if (IS_ERR(n))
 			return PTR_ERR(n);
 	}
diff --git a/drivers/net/ethernet/sfc/Kconfig b/drivers/net/ethernet/sfc/Kconfig
index 979f2801e2a8e3..e2898ec2e6e92c 100644
--- a/drivers/net/ethernet/sfc/Kconfig
+++ b/drivers/net/ethernet/sfc/Kconfig
@@ -68,7 +68,7 @@ config SFC_MCDI_LOGGING
 	  a sysfs file 'mcdi_logging' under the PCI device.
 config SFC_CXL
 	bool "Solarflare SFC9100-family CXL support"
-	depends on SFC && CXL_BUS >= SFC
+	depends on SFC && CXL_MEM >= SFC
 	default SFC
 	help
 	  This enables SFC CXL support if the kernel is configuring CXL for
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 3806cd3dd7f485..953f1d90b9b1a3 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -904,6 +904,10 @@ static const struct pci_device_id efx_pci_table[] = {
 	 .driver_data = (unsigned long)&efx_x4_nic_type},
 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x2c03),  /* X4 PF (FF only) */
 	 .driver_data = (unsigned long)&efx_x4_nic_type},
+	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x8c03),  /* X4D PF (FF/LL) */
+	 .driver_data = (unsigned long)&efx_x4_nic_type},
+	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0xac03),  /* X4D PF (FF only) */
+	 .driver_data = (unsigned long)&efx_x4_nic_type},
 	{0}			/* end of list */
 };
 
diff --git a/drivers/net/ethernet/sfc/tc_counters.c b/drivers/net/ethernet/sfc/tc_counters.c
index b84235e93ffe99..793a562fc1f7ce 100644
--- a/drivers/net/ethernet/sfc/tc_counters.c
+++ b/drivers/net/ethernet/sfc/tc_counters.c
@@ -91,6 +91,7 @@ static void efx_tc_counter_work(struct work_struct *work)
 	struct efx_tc_action_set *act;
 	unsigned long touched;
 	struct neighbour *n;
+	struct net *net;
 
 	spin_lock_bh(&cnt->lock);
 	touched = READ_ONCE(cnt->touched);
@@ -103,16 +104,19 @@ static void efx_tc_counter_work(struct work_struct *work)
 			continue;
 		if (time_after_eq(encap->neigh->used, touched))
 			continue;
+
 		encap->neigh->used = touched;
+		net = encap->neigh->net;
+
 		/* We have passed traffic using this ARP entry, so
 		 * indicate to the ARP cache that it's still active
 		 */
 		if (encap->neigh->dst_ip)
-			n = neigh_lookup(&arp_tbl, &encap->neigh->dst_ip,
+			n = neigh_lookup(arp_table(net), &encap->neigh->dst_ip,
 					 encap->neigh->egdev);
 		else
 #if IS_ENABLED(CONFIG_IPV6)
-			n = neigh_lookup(&nd_tbl,
+			n = neigh_lookup(nd_table(net),
 					 &encap->neigh->dst_ip6,
 					 encap->neigh->egdev);
 #else
diff --git a/drivers/net/ethernet/sfc/tc_encap_actions.c b/drivers/net/ethernet/sfc/tc_encap_actions.c
index c2ad3a358d2012..152c4639ecc6f2 100644
--- a/drivers/net/ethernet/sfc/tc_encap_actions.c
+++ b/drivers/net/ethernet/sfc/tc_encap_actions.c
@@ -527,10 +527,10 @@ static int efx_neigh_event(struct efx_nic *efx, struct neighbour *n)
 	if (WARN_ON(!efx->tc))
 		return NOTIFY_DONE;
 
-	if (n->tbl == &arp_tbl) {
+	if (n->tbl->family == AF_INET) {
 		keysize = sizeof(keys.dst_ip);
 #if IS_ENABLED(CONFIG_IPV6)
-	} else if (n->tbl == &nd_tbl) {
+	} else if (n->tbl->family == AF_INET6) {
 		ipv6 = true;
 		keysize = sizeof(keys.dst_ip6);
 #endif
diff --git a/drivers/net/ethernet/smsc/Kconfig b/drivers/net/ethernet/smsc/Kconfig
index 66bca803b19cb9..99e22d3e16ef56 100644
--- a/drivers/net/ethernet/smsc/Kconfig
+++ b/drivers/net/ethernet/smsc/Kconfig
@@ -29,8 +29,7 @@ config SMC91X
 	help
 	  This is a driver for SMC's 91x series of Ethernet chipsets,
 	  including the SMC91C94 and the SMC91C111. Say Y if you want it
-	  compiled into the kernel, and read the file
-	  <file:Documentation/networking/device_drivers/ethernet/smsc/smc9.rst>.
+	  compiled into the kernel.
 
 	  This driver is also available as a module ( = code which can be
 	  inserted in and removed from the running kernel whenever you want).
diff --git a/drivers/net/ethernet/smsc/smc9194.h b/drivers/net/ethernet/smsc/smc9194.h
deleted file mode 100644
index cf69d0a5a1cb2d..00000000000000
--- a/drivers/net/ethernet/smsc/smc9194.h
+++ /dev/null
@@ -1,241 +0,0 @@
-/*------------------------------------------------------------------------
- . smc9194.h
- . Copyright (C) 1996 by Erik Stahlman
- .
- . This software may be used and distributed according to the terms
- . of the GNU General Public License, incorporated herein by reference.
- .
- . This file contains register information and access macros for
- . the SMC91xxx chipset.
- .
- . Information contained in this file was obtained from the SMC91C94
- . manual from SMC.  To get a copy, if you really want one, you can find
- . information under www.smc.com in the components division.
- . ( this thanks to advice from Donald Becker ).
- .
- . Authors
- . 	Erik Stahlman				( erik@vt.edu )
- .
- . History
- . 01/06/96		 Erik Stahlman   moved definitions here from main .c file
- . 01/19/96		 Erik Stahlman	  polished this up some, and added better
- .										  error handling
- .
- ---------------------------------------------------------------------------*/
-#ifndef _SMC9194_H_
-#define _SMC9194_H_
-
-/* I want some simple types */
-
-typedef unsigned char			byte;
-typedef unsigned short			word;
-typedef unsigned long int 		dword;
-
-
-/* Because of bank switching, the SMC91xxx uses only 16 I/O ports */
-
-#define SMC_IO_EXTENT	16
-
-
-/*---------------------------------------------------------------
- .
- . A description of the SMC registers is probably in order here,
- . although for details, the SMC datasheet is invaluable.
- .
- . Basically, the chip has 4 banks of registers ( 0 to 3 ), which
- . are accessed by writing a number into the BANK_SELECT register
- . ( I also use a SMC_SELECT_BANK macro for this ).
- .
- . The banks are configured so that for most purposes, bank 2 is all
- . that is needed for simple run time tasks.
- -----------------------------------------------------------------------*/
-
-/*
- . Bank Select Register:
- .
- .		yyyy yyyy 0000 00xx
- .		xx 		= bank number
- .		yyyy yyyy	= 0x33, for identification purposes.
-*/
-#define	BANK_SELECT		14
-
-/* BANK 0  */
-
-#define	TCR 		0    	/* transmit control register */
-#define TCR_ENABLE	0x0001	/* if this is 1, we can transmit */
-#define TCR_FDUPLX    	0x0800  /* receive packets sent out */
-#define TCR_STP_SQET	0x1000	/* stop transmitting if Signal quality error */
-#define	TCR_MON_CNS	0x0400	/* monitors the carrier status */
-#define	TCR_PAD_ENABLE	0x0080	/* pads short packets to 64 bytes */
-
-#define	TCR_CLEAR	0	/* do NOTHING */
-/* the normal settings for the TCR register : */
-/* QUESTION: do I want to enable padding of short packets ? */
-#define	TCR_NORMAL  	TCR_ENABLE
-
-
-#define EPH_STATUS	2
-#define ES_LINK_OK	0x4000	/* is the link integrity ok ? */
-
-#define	RCR		4
-#define RCR_SOFTRESET	0x8000 	/* resets the chip */
-#define	RCR_STRIP_CRC	0x200	/* strips CRC */
-#define RCR_ENABLE	0x100	/* IFF this is set, we can receive packets */
-#define RCR_ALMUL	0x4 	/* receive all multicast packets */
-#define	RCR_PROMISC	0x2	/* enable promiscuous mode */
-
-/* the normal settings for the RCR register : */
-#define	RCR_NORMAL	(RCR_STRIP_CRC | RCR_ENABLE)
-#define RCR_CLEAR	0x0		/* set it to a base state */
-
-#define	COUNTER		6
-#define	MIR		8
-#define	MCR		10
-/* 12 is reserved */
-
-/* BANK 1 */
-#define CONFIG			0
-#define CFG_AUI_SELECT	 	0x100
-#define	BASE			2
-#define	ADDR0			4
-#define	ADDR1			6
-#define	ADDR2			8
-#define	GENERAL			10
-#define	CONTROL			12
-#define	CTL_POWERDOWN		0x2000
-#define	CTL_LE_ENABLE		0x80
-#define	CTL_CR_ENABLE		0x40
-#define	CTL_TE_ENABLE		0x0020
-#define CTL_AUTO_RELEASE	0x0800
-#define	CTL_EPROM_ACCESS	0x0003 /* high if Eprom is being read */
-
-/* BANK 2 */
-#define MMU_CMD		0
-#define MC_BUSY		1	/* only readable bit in the register */
-#define MC_NOP		0
-#define	MC_ALLOC	0x20  	/* or with number of 256 byte packets */
-#define	MC_RESET	0x40
-#define	MC_REMOVE	0x60  	/* remove the current rx packet */
-#define MC_RELEASE  	0x80  	/* remove and release the current rx packet */
-#define MC_FREEPKT  	0xA0  	/* Release packet in PNR register */
-#define MC_ENQUEUE	0xC0 	/* Enqueue the packet for transmit */
-
-#define	PNR_ARR		2
-#define FIFO_PORTS	4
-
-#define FP_RXEMPTY  0x8000
-#define FP_TXEMPTY  0x80
-
-#define	POINTER		6
-#define PTR_READ	0x2000
-#define	PTR_RCV		0x8000
-#define	PTR_AUTOINC 	0x4000
-#define PTR_AUTO_INC	0x0040
-
-#define	DATA_1		8
-#define	DATA_2		10
-#define	INTERRUPT	12
-
-#define INT_MASK	13
-#define IM_RCV_INT	0x1
-#define	IM_TX_INT	0x2
-#define	IM_TX_EMPTY_INT	0x4
-#define	IM_ALLOC_INT	0x8
-#define	IM_RX_OVRN_INT	0x10
-#define	IM_EPH_INT	0x20
-#define	IM_ERCV_INT	0x40 /* not on SMC9192 */
-
-/* BANK 3 */
-#define	MULTICAST1	0
-#define	MULTICAST2	2
-#define	MULTICAST3	4
-#define	MULTICAST4	6
-#define	MGMT		8
-#define	REVISION	10 /* ( hi: chip id   low: rev # ) */
-
-
-/* this is NOT on SMC9192 */
-#define	ERCV		12
-
-#define CHIP_9190	3
-#define CHIP_9194	4
-#define CHIP_9195	5
-#define CHIP_91100	7
-
-static const char * chip_ids[ 15 ] =  {
-	NULL, NULL, NULL,
-	/* 3 */ "SMC91C90/91C92",
-	/* 4 */ "SMC91C94",
-	/* 5 */ "SMC91C95",
-	NULL,
-	/* 7 */ "SMC91C100",
-	/* 8 */ "SMC91C100FD",
-	NULL, NULL, NULL,
-	NULL, NULL, NULL};
-
-/*
- . Transmit status bits
-*/
-#define TS_SUCCESS 0x0001
-#define TS_LOSTCAR 0x0400
-#define TS_LATCOL  0x0200
-#define TS_16COL   0x0010
-
-/*
- . Receive status bits
-*/
-#define RS_ALGNERR	0x8000
-#define RS_BADCRC	0x2000
-#define RS_ODDFRAME	0x1000
-#define RS_TOOLONG	0x0800
-#define RS_TOOSHORT	0x0400
-#define RS_MULTICAST	0x0001
-#define RS_ERRORS	(RS_ALGNERR | RS_BADCRC | RS_TOOLONG | RS_TOOSHORT)
-
-static const char * interfaces[ 2 ] = { "TP", "AUI" };
-
-/*-------------------------------------------------------------------------
- .  I define some macros to make it easier to do somewhat common
- . or slightly complicated, repeated tasks.
- --------------------------------------------------------------------------*/
-
-/* select a register bank, 0 to 3  */
-
-#define SMC_SELECT_BANK(x)  { outw( x, ioaddr + BANK_SELECT ); }
-
-/* define a small delay for the reset */
-#define SMC_DELAY() { inw( ioaddr + RCR );\
-			inw( ioaddr + RCR );\
-			inw( ioaddr + RCR );  }
-
-/* this enables an interrupt in the interrupt mask register */
-#define SMC_ENABLE_INT(x) {\
-		unsigned char mask;\
-		SMC_SELECT_BANK(2);\
-		mask = inb( ioaddr + INT_MASK );\
-		mask |= (x);\
-		outb( mask, ioaddr + INT_MASK ); \
-}
-
-/* this disables an interrupt from the interrupt mask register */
-
-#define SMC_DISABLE_INT(x) {\
-		unsigned char mask;\
-		SMC_SELECT_BANK(2);\
-		mask = inb( ioaddr + INT_MASK );\
-		mask &= ~(x);\
-		outb( mask, ioaddr + INT_MASK ); \
-}
-
-/*----------------------------------------------------------------------
- . Define the interrupts that I want to receive from the card
- .
- . I want:
- .  IM_EPH_INT, for nasty errors
- .  IM_RCV_INT, for happy received packets
- .  IM_RX_OVRN_INT, because I have to kick the receiver
- --------------------------------------------------------------------------*/
-#define SMC_INTERRUPT_MASK   (IM_EPH_INT | IM_RX_OVRN_INT | IM_RCV_INT)
-
-#endif  /* _SMC_9194_H_ */
-
diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c
index f7f16397a2c2ac..71a80838bffd0b 100644
--- a/drivers/net/ethernet/spacemit/k1_emac.c
+++ b/drivers/net/ethernet/spacemit/k1_emac.c
@@ -803,6 +803,9 @@ static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb)
 	while (i != head) {
 		emac_free_tx_buf(priv, i);
 
+		tx_desc_addr = &((struct emac_desc *)tx_ring->desc_addr)[i];
+		memset(tx_desc_addr, 0, sizeof(*tx_desc_addr));
+
 		if (++i == tx_ring->total_cnt)
 			i = 0;
 	}
@@ -1161,7 +1164,7 @@ static const struct ethtool_rmon_hist_range emac_rmon_hist_ranges[] = {
 	{  512, 1023 },
 	{ 1024, 1518 },
 	{ 1519, 4096 },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 
 /* Like dev_fetch_dstats(), but we only use tx_drops */
@@ -2066,7 +2069,7 @@ static const struct dev_pm_ops emac_pm_ops = {
 
 static const struct of_device_id emac_of_match[] = {
 	{ .compatible = "spacemit,k1-emac" },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, emac_of_match);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index e3dd5adda5acaf..ab3c4cf964239d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -319,6 +319,17 @@ config DWMAC_THEAD
 	  the stmmac device driver. This driver is used for T-HEAD TH1520
 	  ethernet controller.
 
+config DWMAC_ULTRARISC
+	tristate "UltraRISC DWMAC support"
+	default ARCH_ULTRARISC
+	depends on OF && (ARCH_ULTRARISC || COMPILE_TEST)
+	help
+	  Support for the Ethernet controller on the UltraRISC DP1000 SoC.
+
+	  This selects the UltraRISC platform specific glue layer support
+	  for the stmmac device driver. This driver is used for the DP1000
+	  GMAC Ethernet controller.
+
 config DWMAC_IMX8
 	tristate "NXP IMX8 DWMAC support"
 	default ARCH_MXC
diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile
index a1cea2f57252e6..b16a456b730ec1 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Makefile
+++ b/drivers/net/ethernet/stmicro/stmmac/Makefile
@@ -36,6 +36,7 @@ obj-$(CONFIG_DWMAC_SUNXI)	+= dwmac-sunxi.o
 obj-$(CONFIG_DWMAC_SUN8I)	+= dwmac-sun8i.o
 obj-$(CONFIG_DWMAC_SUN55I)	+= dwmac-sun55i.o
 obj-$(CONFIG_DWMAC_THEAD)	+= dwmac-thead.o
+obj-$(CONFIG_DWMAC_ULTRARISC)	+= dwmac-ultrarisc.o
 obj-$(CONFIG_DWMAC_DWC_QOS_ETH)	+= dwmac-dwc-qos-eth.o
 obj-$(CONFIG_DWMAC_INTEL_PLAT)	+= dwmac-intel-plat.o
 obj-$(CONFIG_DWMAC_LOONGSON1)	+= dwmac-loongson1.o
diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h
index 927ea6230073f4..1729fb29ece385 100644
--- a/drivers/net/ethernet/stmicro/stmmac/common.h
+++ b/drivers/net/ethernet/stmicro/stmmac/common.h
@@ -419,6 +419,8 @@ struct dma_features {
 	unsigned int half_duplex;
 	unsigned int hash_filter;
 	unsigned int multi_addr;
+	unsigned int additional_32_addr;
+	unsigned int additional_64_addr;
 	unsigned int pcs;
 	unsigned int sma_mdio;
 	unsigned int pmt_remote_wake_up;
@@ -627,6 +629,7 @@ struct mac_device_info {
 	void __iomem *pcsr;     /* vpointer to device CSRs */
 	unsigned int multicast_filter_bins;
 	unsigned int unicast_filter_entries;
+	unsigned int multi_addr;
 	unsigned int mcast_bits_log2;
 	unsigned int rx_csum;
 	unsigned int num_vlan;
@@ -635,6 +638,8 @@ struct mac_device_info {
 	u8 vlan_fail_q;
 	bool hw_vlan_en;
 	bool reverse_sgmii_enable;
+	bool additional_32_addr;
+	bool additional_64_addr;
 
 	/* This spinlock protects read-modify-write of the interrupt
 	 * mask/enable registers.
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
index eab8c13fbdcc07..0d654ec07f33f7 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-eic7700.c
@@ -198,18 +198,9 @@ static int eic7700_dwmac_probe(struct platform_device *pdev)
 	/*
 	 * The MAC silicon unconditionally adds ~2 ns TX delay; prevent
 	 * the PHY from also adding TX delay to avoid doubling it.
-	 *
-	 * DT specifies rgmii-id (TX from MAC silicon, RX from PHY);
-	 * override to rgmii-rxid so the PHY only adds its RX delay.
+	 * The common platform code adjusts the PHY interface accordingly.
 	 */
-	if (data->has_internal_tx_delay) {
-		plat_dat->phy_interface =
-				 phy_fix_phy_mode_for_mac_delays(plat_dat->phy_interface,
-								 true, false);
-		if (plat_dat->phy_interface == PHY_INTERFACE_MODE_NA)
-			return dev_err_probe(&pdev->dev, -EINVAL,
-				"phy interface mode is NA\n");
-	}
+	plat_dat->has_internal_tx_delay = data->has_internal_tx_delay;
 
 	/* Read rx-internal-delay-ps and update rx_clk delay */
 	if (!of_property_read_u32(pdev->dev.of_node,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
index 8d7042e6892619..72bdbcb5e863a7 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
@@ -1162,8 +1162,11 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable)
 				return ret;
 
 			ret = clk_prepare_enable(bsp_priv->clk_phy);
-			if (ret)
+			if (ret) {
+				clk_bulk_disable_unprepare(bsp_priv->num_clks,
+							   bsp_priv->clks);
 				return ret;
+			}
 
 			rk_configure_io_clksel(bsp_priv);
 			rk_ungate_rmii_clock(bsp_priv);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c
index e02ad3762b5f73..f62c803665ad47 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sophgo.c
@@ -55,14 +55,10 @@ static int sophgo_dwmac_probe(struct platform_device *pdev)
 		return ret;
 
 	data = device_get_match_data(&pdev->dev);
-	if (data && data->has_internal_rx_delay) {
-		plat_dat->phy_interface = phy_fix_phy_mode_for_mac_delays(plat_dat->phy_interface,
-									  false, true);
-		if (plat_dat->phy_interface == PHY_INTERFACE_MODE_NA)
-			return -EINVAL;
-	}
-
-	return stmmac_dvr_probe(dev, plat_dat, &stmmac_res);
+	if (data)
+		plat_dat->has_internal_rx_delay = data->has_internal_rx_delay;
+
+	return stmmac_pltfr_probe(pdev, plat_dat, &stmmac_res);
 }
 
 static const struct sophgo_dwmac_data sg2042_dwmac_data = {
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
index 48c52eb9623356..47954537b29e42 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
@@ -81,6 +81,13 @@ static const struct reg_field sun8i_syscon_reg_field = {
 	.msb = 31,
 };
 
+/* EMAC1 clock register @ 0x34 in the "system control" address range */
+static const struct reg_field sun8i_syscon_reg_field_emac1 = {
+	.reg = 0x34,
+	.lsb = 0,
+	.msb = 31,
+};
+
 /* EMAC clock register @ 0x164 in the CCU address range */
 static const struct reg_field sun8i_ccu_reg_field = {
 	.reg = 0x164,
@@ -143,6 +150,13 @@ static const struct emac_variant emac_variant_h6 = {
 	.tx_delay_max = 7,
 };
 
+static const struct emac_variant emac_variant_h616_emac1 = {
+	.syscon_field = &sun8i_syscon_reg_field_emac1,
+	/* The co-packaged AC200/AC300 PHY does not use the H3 PHY controls. */
+	.soc_has_internal_phy = false,
+	.support_rmii = true,
+};
+
 #define EMAC_BASIC_CTL0 0x00
 #define EMAC_BASIC_CTL1 0x04
 #define EMAC_INT_STA    0x08
@@ -269,11 +283,33 @@ static const struct emac_variant emac_variant_h6 = {
 #define SYSCON_ETCS_EXT_GMII	0x1
 #define SYSCON_ETCS_INT_GMII	0x2
 
+static int sun8i_dwmac_reset(void __iomem *ioaddr)
+{
+	u32 v;
+
+	v = readl(ioaddr + EMAC_BASIC_CTL1);
+	writel(v | 0x01, ioaddr + EMAC_BASIC_CTL1);
+
+	/* The timeout was previously set to 10ms, but some board (OrangePI0)
+	 * need more if no cable plugged. 100ms seems OK
+	 */
+	return readl_poll_timeout(ioaddr + EMAC_BASIC_CTL1, v,
+				  !(v & 0x01), 100, 100000);
+}
+
 /* sun8i_dwmac_dma_reset() - reset the EMAC
  * Called from stmmac via stmmac_dma_ops->reset
  */
 static int sun8i_dwmac_dma_reset(void __iomem *ioaddr)
 {
+	int ret;
+
+	writel(0, ioaddr + EMAC_INT_EN);
+
+	/* The PHY receive clock must be running for the reset to complete. */
+	ret = sun8i_dwmac_reset(ioaddr);
+
+	/* Leave DMA and interrupts disabled even if the reset timed out. */
 	writel(0, ioaddr + EMAC_RX_CTL1);
 	writel(0, ioaddr + EMAC_TX_CTL1);
 	writel(0, ioaddr + EMAC_RX_FRM_FLT);
@@ -281,7 +317,7 @@ static int sun8i_dwmac_dma_reset(void __iomem *ioaddr)
 	writel(0, ioaddr + EMAC_TX_DESC_LIST);
 	writel(0, ioaddr + EMAC_INT_EN);
 	writel(0x1FFFFFF, ioaddr + EMAC_INT_STA);
-	return 0;
+	return ret;
 }
 
 /* sun8i_dwmac_dma_init() - initialize the EMAC
@@ -738,27 +774,6 @@ static void sun8i_dwmac_flow_ctrl(struct mac_device_info *hw,
 	writel(v, ioaddr + EMAC_TX_FLOW_CTL);
 }
 
-static int sun8i_dwmac_reset(struct stmmac_priv *priv)
-{
-	u32 v;
-	int err;
-
-	v = readl(priv->ioaddr + EMAC_BASIC_CTL1);
-	writel(v | 0x01, priv->ioaddr + EMAC_BASIC_CTL1);
-
-	/* The timeout was previously set to 10ms, but some board (OrangePI0)
-	 * need more if no cable plugged. 100ms seems OK
-	 */
-	err = readl_poll_timeout(priv->ioaddr + EMAC_BASIC_CTL1, v,
-				 !(v & 0x01), 100, 100000);
-
-	if (err) {
-		dev_err(priv->device, "EMAC reset timeout\n");
-		return err;
-	}
-	return 0;
-}
-
 /* Search in mdio-mux node for internal PHY node and get its clk/reset */
 static int get_ephy_nodes(struct stmmac_priv *priv)
 {
@@ -783,18 +798,22 @@ static int get_ephy_nodes(struct stmmac_priv *priv)
 
 	/* Seek for internal PHY */
 	for_each_child_of_node_scoped(mdio_internal, iphynode) {
-		gmac->ephy_clk = of_clk_get(iphynode, 0);
-		if (IS_ERR(gmac->ephy_clk))
+		struct clk *ephy_clk;
+
+		ephy_clk = of_clk_get(iphynode, 0);
+		if (IS_ERR(ephy_clk))
 			continue;
 		gmac->rst_ephy = of_reset_control_get_exclusive(iphynode, NULL);
 		if (IS_ERR(gmac->rst_ephy)) {
 			ret = PTR_ERR(gmac->rst_ephy);
+			clk_put(ephy_clk);
 			if (ret == -EPROBE_DEFER) {
 				of_node_put(mdio_internal);
 				return ret;
 			}
 			continue;
 		}
+		gmac->ephy_clk = ephy_clk;
 		dev_info(priv->device, "Found internal PHY node\n");
 		of_node_put(mdio_internal);
 		return 0;
@@ -895,7 +914,9 @@ static int mdio_mux_syscon_switch_fn(int current_child, int desired_child,
 		/* After changing syscon value, the MAC need reset or it will
 		 * use the last value (and so the last PHY set).
 		 */
-		ret = sun8i_dwmac_reset(priv);
+		ret = sun8i_dwmac_reset(priv->ioaddr);
+		if (ret)
+			dev_err(priv->device, "EMAC reset timeout\n");
 	}
 	return ret;
 }
@@ -1217,10 +1238,6 @@ static int sun8i_dwmac_probe(struct platform_device *pdev)
 			dev_err(&pdev->dev, "Failed to register mux\n");
 			goto dwmac_mux;
 		}
-	} else {
-		ret = sun8i_dwmac_reset(priv);
-		if (ret)
-			goto dwmac_remove;
 	}
 
 	pm_runtime_put(&pdev->dev);
@@ -1278,6 +1295,8 @@ static const struct of_device_id sun8i_dwmac_match[] = {
 		.data = &emac_variant_a64 },
 	{ .compatible = "allwinner,sun50i-h6-emac",
 		.data = &emac_variant_h6 },
+	{ .compatible = "allwinner,sun50i-h616-emac1",
+		.data = &emac_variant_h616_emac1 },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun8i_dwmac_match);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ultrarisc.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ultrarisc.c
new file mode 100644
index 00000000000000..0b62a75b782a21
--- /dev/null
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ultrarisc.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * UltraRISC DWMAC platform driver
+ *
+ * Copyright (C) 2026 UltraRISC Technology (Shanghai) Co., Ltd.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "stmmac_platform.h"
+
+static int ultrarisc_dwmac_probe(struct platform_device *pdev)
+{
+	struct plat_stmmacenet_data *plat_dat;
+	struct stmmac_resources stmmac_res;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	ret = stmmac_get_platform_resources(pdev, &stmmac_res);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to get resources\n");
+
+	plat_dat = devm_stmmac_probe_config_dt(pdev, stmmac_res.mac);
+	if (IS_ERR(plat_dat))
+		return dev_err_probe(dev, PTR_ERR(plat_dat),
+				     "failed to parse DT parameters\n");
+
+	plat_dat->has_internal_tx_delay = true;
+	plat_dat->has_internal_rx_delay = true;
+
+	return devm_stmmac_pltfr_probe(pdev, plat_dat, &stmmac_res);
+}
+
+static const struct of_device_id ultrarisc_dwmac_match[] = {
+	{ .compatible = "ultrarisc,dp1000-gmac" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ultrarisc_dwmac_match);
+
+static struct platform_driver ultrarisc_dwmac_driver = {
+	.probe = ultrarisc_dwmac_probe,
+	.driver = {
+		.name = "ultrarisc-dwmac",
+		.pm = &stmmac_pltfr_pm_ops,
+		.of_match_table = ultrarisc_dwmac_match,
+	},
+};
+module_platform_driver(ultrarisc_dwmac_driver);
+
+MODULE_DESCRIPTION("UltraRISC DWMAC platform driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
index 6382836828bac6..69684653122957 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
@@ -75,8 +75,6 @@
 #define GMAC_PACKET_FILTER_IPFE		BIT(20)
 #define GMAC_PACKET_FILTER_RA		BIT(31)
 
-#define GMAC_MAX_PERFECT_ADDRESSES	128
-
 /* MAC RX Queue Enable */
 #define GMAC_RX_QUEUE_CLEAR(queue)	~(GENMASK(1, 0) << ((queue) * 2))
 #define GMAC_RX_AV_QUEUE_ENABLE(queue)	BIT((queue) * 2)
@@ -176,7 +174,9 @@ enum power_event {
 
 /* MAC HW features0 bitmap */
 #define GMAC_HW_FEAT_SAVLANINS		BIT(27)
-#define GMAC_HW_FEAT_ADDMAC		BIT(18)
+#define GMAC_HW_FEAT_MACADR64SEL	BIT(24)
+#define GMAC_HW_FEAT_MACADR32SEL	BIT(23)
+#define GMAC_HW_FEAT_ADDMAC		GENMASK(22, 18)
 #define GMAC_HW_FEAT_RXCOESEL		BIT(16)
 #define GMAC_HW_FEAT_TXCOSEL		BIT(14)
 #define GMAC_HW_FEAT_EEESEL		BIT(13)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index 18b357b257cc09..cd6c5d6f42b544 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -365,22 +365,63 @@ static void dwmac4_pmt(struct mac_device_info *hw, unsigned long mode)
 	writel(pmt, ioaddr + GMAC_PMT);
 }
 
+/**
+ * dwmac4_umac_addr_slot - Get the UC filter slot for the Nth MAC address
+ * @hw: The MAC device info
+ * @reg_n: The MAC address's index in the UC list
+ *
+ * On dwmac4 the slots available for Unicast MAC filtering are configured when
+ * integrating the IP.
+ *
+ *  - Slot 0 is always available, used to store the primary MAC address
+ *  - From 1 to 31, the number of implemented slots is set in hw->multi_addr
+ *  - From 32 to 63, the range is available if hw->additional_32_addr
+ *    is set
+ *  - From 64 to 127, the range is available if hw->additional_64_addr is set
+ *
+ * All the ranges can be configured independently (e.g. 64 -> 127 can be
+ * available, but not 32 -> 63)
+ *
+ * Returns: The physical slot index in the UC filter
+ */
+static unsigned int dwmac4_umac_addr_slot(struct mac_device_info *hw,
+					  unsigned int reg_n)
+{
+	unsigned int empty_slots = 0;
+
+	/* reg_n is in the 1-31 bank : 1 to 1 mapping */
+	if (reg_n < (hw->multi_addr + 1))
+		return reg_n;
+
+	/* Gap between the last address in the 1->31 range and the next slot */
+	if (hw->additional_32_addr)
+		empty_slots = 32 - (hw->multi_addr + 1);
+	else if (hw->additional_64_addr)
+		empty_slots = 64 - (hw->multi_addr + 1);
+
+	return reg_n + empty_slots;
+}
+
 static void dwmac4_set_umac_addr(struct mac_device_info *hw,
 				 const unsigned char *addr, unsigned int reg_n)
 {
 	void __iomem *ioaddr = hw->pcsr;
+	unsigned int slot;
 
-	stmmac_dwmac4_set_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
-				   GMAC_ADDR_LOW(reg_n));
+	slot = dwmac4_umac_addr_slot(hw, reg_n);
+	stmmac_dwmac4_set_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(slot),
+				   GMAC_ADDR_LOW(slot));
 }
 
 static void dwmac4_get_umac_addr(struct mac_device_info *hw,
 				 unsigned char *addr, unsigned int reg_n)
 {
 	void __iomem *ioaddr = hw->pcsr;
+	unsigned int slot;
 
-	stmmac_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n),
-			    GMAC_ADDR_LOW(reg_n));
+	slot = dwmac4_umac_addr_slot(hw, reg_n);
+	stmmac_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(slot),
+			    GMAC_ADDR_LOW(slot));
 }
 
 static int dwmac4_set_lpi_mode(struct mac_device_info *hw,
@@ -522,9 +563,6 @@ static void dwmac4_set_filter(struct mac_device_info *hw,
 
 	/* Handle multiple unicast addresses */
 	if (netdev_uc_count(dev) + 1 > hw->unicast_filter_entries) {
-		/* Switch to promiscuous mode if more than 128 addrs
-		 * are required
-		 */
 		value |= GMAC_PACKET_FILTER_PR;
 	} else {
 		struct netdev_hw_addr *ha;
@@ -535,9 +573,11 @@ static void dwmac4_set_filter(struct mac_device_info *hw,
 			reg++;
 		}
 
-		while (reg < GMAC_MAX_PERFECT_ADDRESSES) {
-			writel(0, ioaddr + GMAC_ADDR_HIGH(reg));
-			writel(0, ioaddr + GMAC_ADDR_LOW(reg));
+		while (reg < hw->unicast_filter_entries) {
+			unsigned int slot = dwmac4_umac_addr_slot(hw, reg);
+
+			writel(0, ioaddr + GMAC_ADDR_HIGH(slot));
+			writel(0, ioaddr + GMAC_ADDR_LOW(slot));
 			reg++;
 		}
 	}
@@ -779,22 +819,6 @@ static void dwmac4_sarc_configure(void __iomem *ioaddr, int val)
 	writel(value, ioaddr + GMAC_CONFIG);
 }
 
-static void dwmac4_set_arp_offload(struct mac_device_info *hw, bool en,
-				   u32 addr)
-{
-	void __iomem *ioaddr = hw->pcsr;
-	u32 value;
-
-	writel(addr, ioaddr + GMAC_ARP_ADDR);
-
-	value = readl(ioaddr + GMAC_CONFIG);
-	if (en)
-		value |= GMAC_CONFIG_ARPEN;
-	else
-		value &= ~GMAC_CONFIG_ARPEN;
-	writel(value, ioaddr + GMAC_CONFIG);
-}
-
 static int dwmac4_config_l3_filter(struct mac_device_info *hw, u32 filter_no,
 				   bool en, bool ipv6, bool sa, bool inv,
 				   u32 match)
@@ -926,7 +950,6 @@ const struct stmmac_ops dwmac4_ops = {
 	.set_filter = dwmac4_set_filter,
 	.set_mac_loopback = dwmac4_set_mac_loopback,
 	.sarc_configure = dwmac4_sarc_configure,
-	.set_arp_offload = dwmac4_set_arp_offload,
 	.config_l3_filter = dwmac4_config_l3_filter,
 	.config_l4_filter = dwmac4_config_l4_filter,
 };
@@ -963,7 +986,6 @@ const struct stmmac_ops dwmac410_ops = {
 	.flex_pps_config = dwmac5_flex_pps_config,
 	.set_mac_loopback = dwmac4_set_mac_loopback,
 	.sarc_configure = dwmac4_sarc_configure,
-	.set_arp_offload = dwmac4_set_arp_offload,
 	.config_l3_filter = dwmac4_config_l3_filter,
 	.config_l4_filter = dwmac4_config_l4_filter,
 	.fpe_map_preemption_class = dwmac5_fpe_map_preemption_class,
@@ -1005,7 +1027,6 @@ const struct stmmac_ops dwmac510_ops = {
 	.flex_pps_config = dwmac5_flex_pps_config,
 	.set_mac_loopback = dwmac4_set_mac_loopback,
 	.sarc_configure = dwmac4_sarc_configure,
-	.set_arp_offload = dwmac4_set_arp_offload,
 	.config_l3_filter = dwmac4_config_l3_filter,
 	.config_l4_filter = dwmac4_config_l4_filter,
 	.fpe_map_preemption_class = dwmac5_fpe_map_preemption_class,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
index 2994df41ec2c4e..c6a8f8d7350150 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
@@ -474,11 +474,11 @@ static void dwmac4_set_sarc(struct dma_desc *p, u32 sarc_type)
 					  sarc_type));
 }
 
-static int set_16kib_bfsize(int mtu)
+static int set_16kib_bfsize(int len)
 {
 	int ret = 0;
 
-	if (unlikely(mtu >= BUF_SIZE_8KiB))
+	if (unlikely(len > BUF_SIZE_8KiB))
 		ret = BUF_SIZE_16KiB;
 	return ret;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
index 23ffe1adcd0d37..14ac3f0e51f779 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c
@@ -388,6 +388,8 @@ static int dwmac4_get_hw_feature(void __iomem *ioaddr,
 	dma_cap->half_duplex = (hw_cap & GMAC_HW_FEAT_HDSEL) >> 2;
 	dma_cap->vlhash = (hw_cap & GMAC_HW_FEAT_VLHASH) >> 4;
 	dma_cap->multi_addr = (hw_cap & GMAC_HW_FEAT_ADDMAC) >> 18;
+	dma_cap->additional_32_addr = (hw_cap & GMAC_HW_FEAT_MACADR32SEL) >> 23;
+	dma_cap->additional_64_addr = (hw_cap & GMAC_HW_FEAT_MACADR64SEL) >> 24;
 	dma_cap->pcs = (hw_cap & GMAC_HW_FEAT_PCSSEL) >> 3;
 	dma_cap->sma_mdio = (hw_cap & GMAC_HW_FEAT_SMASEL) >> 5;
 	dma_cap->pmt_remote_wake_up = (hw_cap & GMAC_HW_FEAT_RWKSEL) >> 6;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
index fc6ddb51c6820f..1a88cbaed70ca1 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c
@@ -1410,22 +1410,6 @@ static int dwxgmac2_config_l4_filter(struct mac_device_info *hw, u32 filter_no,
 	return 0;
 }
 
-static void dwxgmac2_set_arp_offload(struct mac_device_info *hw, bool en,
-				     u32 addr)
-{
-	void __iomem *ioaddr = hw->pcsr;
-	u32 value;
-
-	writel(addr, ioaddr + XGMAC_ARP_ADDR);
-
-	value = readl(ioaddr + XGMAC_RX_CONFIG);
-	if (en)
-		value |= XGMAC_CONFIG_ARPEN;
-	else
-		value &= ~XGMAC_CONFIG_ARPEN;
-	writel(value, ioaddr + XGMAC_RX_CONFIG);
-}
-
 const struct stmmac_ops dwxgmac210_ops = {
 	.core_init = dwxgmac2_core_init,
 	.irq_modify = dwxgmac2_irq_modify,
@@ -1464,7 +1448,6 @@ const struct stmmac_ops dwxgmac210_ops = {
 	.sarc_configure = dwxgmac2_sarc_configure,
 	.config_l3_filter = dwxgmac2_config_l3_filter,
 	.config_l4_filter = dwxgmac2_config_l4_filter,
-	.set_arp_offload = dwxgmac2_set_arp_offload,
 	.fpe_map_preemption_class = dwxgmac3_fpe_map_preemption_class,
 };
 
@@ -1519,7 +1502,6 @@ const struct stmmac_ops dwxlgmac2_ops = {
 	.sarc_configure = dwxgmac2_sarc_configure,
 	.config_l3_filter = dwxgmac2_config_l3_filter,
 	.config_l4_filter = dwxgmac2_config_l4_filter,
-	.set_arp_offload = dwxgmac2_set_arp_offload,
 	.fpe_map_preemption_class = dwxgmac3_fpe_map_preemption_class,
 };
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h
index 9314bcb85c221d..a8a5c8fdd5edbd 100644
--- a/drivers/net/ethernet/stmicro/stmmac/hwif.h
+++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h
@@ -407,7 +407,6 @@ struct stmmac_ops {
 	int (*config_l4_filter)(struct mac_device_info *hw, u32 filter_no,
 				bool en, bool udp, bool sa, bool inv,
 				u32 match);
-	void (*set_arp_offload)(struct mac_device_info *hw, bool en, u32 addr);
 	int (*fpe_map_preemption_class)(struct net_device *ndev,
 					struct netlink_ext_ack *extack,
 					u32 pclass);
@@ -491,8 +490,6 @@ struct stmmac_ops {
 	stmmac_do_callback(__priv, mac, config_l3_filter, __args)
 #define stmmac_config_l4_filter(__priv, __args...) \
 	stmmac_do_callback(__priv, mac, config_l4_filter, __args)
-#define stmmac_set_arp_offload(__priv, __args...) \
-	stmmac_do_void_callback(__priv, mac, set_arp_offload, __args)
 #define stmmac_fpe_map_preemption_class(__priv, __args...) \
 	stmmac_do_callback(__priv, mac, fpe_map_preemption_class, __args)
 
@@ -540,7 +537,7 @@ struct stmmac_mode_ops {
 	bool (*is_jumbo_frm)(unsigned int len, bool enh_desc);
 	int (*jumbo_frm)(struct stmmac_tx_queue *tx_q, struct sk_buff *skb,
 			 int csum);
-	int (*set_16kib_bfsize)(int mtu);
+	int (*set_16kib_bfsize)(int len);
 	void (*init_desc3)(struct dma_desc *p);
 	void (*refill_desc3)(struct stmmac_rx_queue *rx_q, struct dma_desc *p);
 	void (*clean_desc3)(struct stmmac_tx_queue *tx_q, struct dma_desc *p);
diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
index f7949419eb9fdc..d2f0c321661d99 100644
--- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
+++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c
@@ -124,10 +124,10 @@ static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p)
 		p->des3 = 0;
 }
 
-static int set_16kib_bfsize(int mtu)
+static int set_16kib_bfsize(int len)
 {
 	int ret = 0;
-	if (unlikely(mtu > BUF_SIZE_8KiB))
+	if (unlikely(len > BUF_SIZE_8KiB))
 		ret = BUF_SIZE_16KiB;
 	return ret;
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
index 7582fca637419d..4fc96b317d79c6 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
@@ -329,6 +329,8 @@ struct stmmac_priv {
 	struct kernel_hwtstamp_config tstamp_config;
 	struct ptp_clock *ptp_clock;
 	struct ptp_clock_info ptp_clock_ops;
+	bool ptp_enabled;
+
 	unsigned int default_addend;
 	u32 sub_second_inc;
 	u32 systime_flags;
@@ -419,6 +421,11 @@ int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i,
 
 struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev);
 
+static inline bool stmmac_check_timestamp_cap(struct stmmac_priv *priv)
+{
+	return priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp;
+}
+
 static inline bool stmmac_xdp_is_enabled(struct stmmac_priv *priv)
 {
 	return !!priv->xdp_prog;
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
index 1be5310ca766c5..1cf0f8820b337a 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
@@ -1007,8 +1007,7 @@ static int stmmac_get_ts_info(struct net_device *dev,
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
 
-	if ((priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) {
-
+	if (stmmac_check_timestamp_cap(priv)) {
 		info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
 					SOF_TIMESTAMPING_TX_HARDWARE |
 					SOF_TIMESTAMPING_RX_HARDWARE |
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 1fb5f804ea2345..2cbf502c5769da 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -653,7 +653,8 @@ static int stmmac_hwtstamp_set(struct net_device *dev,
 	u32 ts_master_en = 0;
 	u32 ts_event_en = 0;
 
-	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
+	if (!priv->plat->clk_ptp_rate ||
+	    !(priv->dma_cap.time_stamp || priv->adv_ts)) {
 		NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping");
 		priv->hwts_tx_en = 0;
 		priv->hwts_rx_en = 0;
@@ -843,7 +844,7 @@ static int stmmac_hwtstamp_get(struct net_device *dev,
 {
 	struct stmmac_priv *priv = netdev_priv(dev);
 
-	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
+	if (!stmmac_check_timestamp_cap(priv))
 		return -EOPNOTSUPP;
 
 	*config = priv->tstamp_config;
@@ -866,11 +867,6 @@ static int stmmac_init_tstamp_counter(struct stmmac_priv *priv,
 {
 	struct timespec64 now;
 
-	if (!priv->plat->clk_ptp_rate) {
-		netdev_err(priv->dev, "Invalid PTP clock rate");
-		return -EINVAL;
-	}
-
 	stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);
 	priv->systime_flags = systime_flags;
 
@@ -885,26 +881,37 @@ static int stmmac_init_tstamp_counter(struct stmmac_priv *priv,
 	return 0;
 }
 
+static int stmmac_init_ptp_clk_freq(struct stmmac_priv *priv)
+{
+	if (priv->plat->ptp_clk_freq_config)
+		priv->plat->ptp_clk_freq_config(priv);
+
+	if (!priv->plat->clk_ptp_rate) {
+		netdev_info(priv->dev, "PTP clock rate not configured\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 /**
  * stmmac_init_timestamping - initialise timestamping
  * @priv: driver private structure
- * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
- * This is done by looking at the HW cap. register.
- * This function also registers the ptp driver.
+ *
+ * Description: initialise the hardware timestamping counter, reset the
+ * timestamping configuration and derive the advanced timestamping flags from
+ * the HW capabilities. The caller must have ensured a valid PTP reference
+ * clock rate (see stmmac_init_ptp_clk_freq()); the configured state is valid
+ * as long as the interface is open and not suspended, and this function is
+ * re-run on resume.
+ *
+ * Return: 0 on success, a negative errno otherwise.
  */
 static int stmmac_init_timestamping(struct stmmac_priv *priv)
 {
 	bool xmac = dwmac_is_xmac(priv->plat->core_type);
 	int ret;
 
-	if (priv->plat->ptp_clk_freq_config)
-		priv->plat->ptp_clk_freq_config(priv);
-
-	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) {
-		netdev_info(priv->dev, "PTP not supported by HW\n");
-		return -EOPNOTSUPP;
-	}
-
 	ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE |
 					       PTP_TCR_TSCFUPDT);
 	if (ret) {
@@ -937,24 +944,48 @@ static int stmmac_init_timestamping(struct stmmac_priv *priv)
 	return 0;
 }
 
-static void stmmac_setup_ptp(struct stmmac_priv *priv)
+static int stmmac_setup_ptp(struct stmmac_priv *priv)
 {
 	int ret;
 
+	if (!stmmac_check_timestamp_cap(priv)) {
+		netdev_info(priv->dev, "PTP not supported\n");
+		return 0;
+	}
+
 	ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
-	if (ret < 0)
+	if (ret < 0) {
 		netdev_warn(priv->dev,
 			    "failed to enable PTP reference clock: %pe\n",
 			    ERR_PTR(ret));
+		return ret;
+	}
+
+	if (stmmac_init_ptp_clk_freq(priv)) {
+		clk_disable_unprepare(priv->plat->clk_ptp_ref);
+		return 0;
+	}
+
+	ret = stmmac_init_timestamping(priv);
+	if (ret) {
+		clk_disable_unprepare(priv->plat->clk_ptp_ref);
+		return ret;
+	}
+
+	stmmac_ptp_register(priv);
+	priv->ptp_enabled = true;
 
-	if (stmmac_init_timestamping(priv) == 0)
-		stmmac_ptp_register(priv);
+	return 0;
 }
 
 static void stmmac_release_ptp(struct stmmac_priv *priv)
 {
+	if (!priv->ptp_enabled)
+		return;
+
 	stmmac_ptp_unregister(priv);
 	clk_disable_unprepare(priv->plat->clk_ptp_ref);
+	priv->ptp_enabled = false;
 }
 
 static void stmmac_legacy_serdes_power_down(struct stmmac_priv *priv)
@@ -1536,17 +1567,17 @@ static unsigned int stmmac_rx_offset(struct stmmac_priv *priv)
 	return NET_SKB_PAD + NET_IP_ALIGN;
 }
 
-static int stmmac_set_bfsize(int mtu)
+static int stmmac_set_bfsize(int len)
 {
 	int ret;
 
-	if (mtu >= BUF_SIZE_8KiB)
+	if (len > BUF_SIZE_8KiB)
 		ret = BUF_SIZE_16KiB;
-	else if (mtu >= BUF_SIZE_4KiB)
+	else if (len > BUF_SIZE_4KiB)
 		ret = BUF_SIZE_8KiB;
-	else if (mtu >= BUF_SIZE_2KiB)
+	else if (len > BUF_SIZE_2KiB)
 		ret = BUF_SIZE_4KiB;
-	else if (mtu > DEFAULT_BUFSIZE)
+	else if (len > DEFAULT_BUFSIZE)
 		ret = BUF_SIZE_2KiB;
 	else
 		ret = DEFAULT_BUFSIZE;
@@ -1629,6 +1660,16 @@ static void stmmac_clear_descriptors(struct stmmac_priv *priv,
 		stmmac_clear_tx_descriptors(priv, dma_conf, queue);
 }
 
+static bool stmmac_rx_check_buf2_cap(struct stmmac_priv *priv)
+{
+	/* Only cores that can program an independent secondary RX buffer
+	 * (used for scatter-gather overflow or split-header payload) back
+	 * buffer2. Legacy cores have no set_sec_addr op, so buffer2 is
+	 * never handed to the hardware there.
+	 */
+	return priv->hw->desc && priv->hw->desc->set_sec_addr;
+}
+
 /**
  * stmmac_init_rx_buffers - init the RX descriptor buffer.
  * @priv: driver private structure
@@ -1659,16 +1700,13 @@ static int stmmac_init_rx_buffers(struct stmmac_priv *priv,
 		buf->page_offset = stmmac_rx_offset(priv);
 	}
 
-	if (priv->sph_active && !buf->sec_page) {
+	if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) {
 		buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
 		if (!buf->sec_page)
 			return -ENOMEM;
 
 		buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
 		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
-	} else {
-		buf->sec_page = NULL;
-		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
 	}
 
 	buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
@@ -2256,13 +2294,8 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
 	pp_params.nid = dev_to_node(priv->device);
 	pp_params.dev = priv->device;
 	pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
-	pp_params.offset = stmmac_rx_offset(priv);
-	pp_params.max_len = dma_conf->dma_buf_sz;
-
-	if (priv->sph_active) {
-		pp_params.offset = 0;
-		pp_params.max_len += stmmac_rx_offset(priv);
-	}
+	pp_params.offset = 0;
+	pp_params.max_len = dma_conf->dma_buf_sz + stmmac_rx_offset(priv);
 
 	rx_q->page_pool = page_pool_create(&pp_params);
 	if (IS_ERR(rx_q->page_pool)) {
@@ -4063,7 +4096,7 @@ static struct stmmac_dma_conf *
 stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)
 {
 	struct stmmac_dma_conf *dma_conf;
-	int bfsize, ret;
+	int bfsize, len, ret;
 	u8 chan;
 
 	dma_conf = kzalloc_obj(*dma_conf);
@@ -4073,13 +4106,15 @@ stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	/* Returns 0 or BUF_SIZE_16KiB if mtu > 8KiB and dwmac4 or ring mode */
-	bfsize = stmmac_set_16kib_bfsize(priv, mtu);
+	len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN;
+
+	/* Returns 0 or BUF_SIZE_16KiB if len > 8KiB and dwmac4 or ring mode */
+	bfsize = stmmac_set_16kib_bfsize(priv, len);
 	if (bfsize < 0)
 		bfsize = 0;
 
 	if (bfsize < BUF_SIZE_16KiB)
-		bfsize = stmmac_set_bfsize(mtu);
+		bfsize = stmmac_set_bfsize(len);
 
 	dma_conf->dma_buf_sz = bfsize;
 	/* Chose the tx/rx size from the already defined one in the
@@ -4161,10 +4196,12 @@ static int __stmmac_open(struct net_device *dev,
 	ret = stmmac_hw_setup(dev);
 	if (ret < 0) {
 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
-		goto init_error;
+		return ret;
 	}
 
-	stmmac_setup_ptp(priv);
+	ret = stmmac_setup_ptp(priv);
+	if (ret)
+		goto ptp_error;
 
 	stmmac_init_coalesce(priv);
 
@@ -4185,11 +4222,16 @@ static int __stmmac_open(struct net_device *dev,
 irq_error:
 	phylink_stop(priv->phylink);
 
+	stmmac_stop_all_dma(priv);
+
 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);
 
 	stmmac_release_ptp(priv);
-init_error:
+ptp_error:
+	stmmac_stop_all_dma(priv);
+	stmmac_mac_set(priv, priv->ioaddr, false);
+
 	return ret;
 }
 
@@ -5097,7 +5139,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 				break;
 		}
 
-		if (priv->sph_active && !buf->sec_page) {
+		if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) {
 			buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp);
 			if (!buf->sec_page)
 				break;
@@ -5108,10 +5150,8 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
 		buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
 
 		stmmac_set_desc_addr(priv, p, buf->addr);
-		if (priv->sph_active)
-			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
-		else
-			stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
+		stmmac_set_desc_sec_addr(priv, p, buf->sec_addr,
+					 stmmac_rx_check_buf2_cap(priv));
 		stmmac_refill_desc3(priv, rx_q, p);
 
 		rx_q->rx_count_frames++;
@@ -5142,7 +5182,9 @@ static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
 	unsigned int plen = 0, hlen = 0;
 	int coe = priv->hw->rx_csum;
 
-	/* Not first descriptor, buffer is always zero */
+	/* Not first descriptor, SPH enabled: buffer1 only carries the
+	 * split header of the first descriptor, so it is zero here.
+	 */
 	if (priv->sph_active && len)
 		return 0;
 
@@ -5153,14 +5195,16 @@ static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
 		return hlen;
 	}
 
-	/* First descriptor, not last descriptor and not split header */
+	/* Not last descriptor and not split header: buffer1 is fully filled */
 	if (status & rx_not_ls)
 		return priv->dma_conf.dma_buf_sz;
 
 	plen = stmmac_get_rx_frame_len(priv, p, coe);
 
-	/* First descriptor and last descriptor and not split header */
-	return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen);
+	/* Last descriptor and not split header: buffer1 holds the remaining
+	 * bytes of the frame, up to dma_buf_sz
+	 */
+	return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen - len);
 }
 
 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
@@ -5170,8 +5214,7 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
 	int coe = priv->hw->rx_csum;
 	unsigned int plen = 0;
 
-	/* Not split header, buffer is not available */
-	if (!priv->sph_active)
+	if (!stmmac_rx_check_buf2_cap(priv))
 		return 0;
 
 	/* For GMAC4, when split header is enabled, in some rare cases, the
@@ -5188,14 +5231,15 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
 	 * Thus 'plen - len' always gives the correct length of buf2.
 	 */
 
-	/* Not GMAC4 and not last descriptor */
-	if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls))
+	/* Not GMAC4, or non-SPH and not last descriptor */
+	if ((priv->plat->core_type != DWMAC_CORE_GMAC4 || !priv->sph_active) &&
+	    (status & rx_not_ls))
 		return priv->dma_conf.dma_buf_sz;
 
 	/* GMAC4 or last descriptor */
 	plen = stmmac_get_rx_frame_len(priv, p, coe);
 
-	return plen - len;
+	return plen > len ? plen - len : 0;
 }
 
 static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
@@ -5718,6 +5762,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 	unsigned int desc_size;
 	struct sk_buff *skb = NULL;
 	struct stmmac_xdp_buff ctx;
+	bool fcs_stripped = false;
 	int xdp_status = 0;
 	int bufsz;
 
@@ -5807,24 +5852,25 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 		len += buf2_len;
 
 		/* ACS is disabled; strip manually. */
-		if (likely(!(status & rx_not_ls))) {
-			if (buf2_len) {
-				buf2_len -= ETH_FCS_LEN;
-				len -= ETH_FCS_LEN;
-			} else if (buf1_len) {
-				buf1_len -= ETH_FCS_LEN;
-				len -= ETH_FCS_LEN;
-			}
-		}
+		if (likely(!(status & rx_not_ls)))
+			len -= ETH_FCS_LEN;
 
 		if (!skb) {
 			unsigned int pre_len, sync_len;
 
+			/* Each frame starts here: reset the FCS handling */
+			fcs_stripped = false;
+
 			dma_sync_single_for_cpu(priv->device, buf->addr,
 						buf1_len, dma_dir);
 			net_prefetch(page_address(buf->page) +
 				     buf->page_offset);
 
+			if (stmmac_xdp_is_enabled(priv) && !buf2_len) {
+				buf1_len -= ETH_FCS_LEN;
+				fcs_stripped = true;
+			}
+
 			xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq);
 			xdp_prepare_buff(&ctx.xdp, page_address(buf->page),
 					 buf->page_offset, buf1_len, true);
@@ -5887,6 +5933,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 			if (!skb) {
 				page_pool_recycle_direct(rx_q->page_pool,
 							 buf->page);
+				buf->page = NULL;
 				rx_dropped++;
 				count++;
 				goto drain_data;
@@ -5924,6 +5971,15 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 
 		/* Got entire packet into SKB. Finish it. */
 
+		/* Remove FCS if needed */
+		if (!fcs_stripped && pskb_trim(skb, len)) {
+			dev_kfree_skb_any(skb);
+			skb = NULL;
+			rx_dropped++;
+			count++;
+			continue;
+		}
+
 		stmmac_get_rx_hwtstamp(priv, p, np, skb);
 
 		if (priv->hw->hw_vlan_en)
@@ -6468,24 +6524,6 @@ static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
 	}
 }
 
-static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
-			       struct net_device *sb_dev)
-{
-	int gso = skb_shinfo(skb)->gso_type;
-
-	if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
-		/*
-		 * There is no way to determine the number of TSO/USO
-		 * capable Queues. Let's use always the Queue 0
-		 * because if TSO/USO is supported then at least this
-		 * one will be capable.
-		 */
-		return 0;
-	}
-
-	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
-}
-
 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
 {
 	struct stmmac_priv *priv = netdev_priv(ndev);
@@ -6622,6 +6660,18 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
 		seq_printf(seq,
 			   "\tNumber of Additional MAC address registers: %d\n",
 			   priv->dma_cap.multi_addr);
+	} else if (priv->plat->core_type == DWMAC_CORE_GMAC4) {
+		seq_printf(seq,
+			   "\tNumber of MAC address registers (1-31): %d\n",
+			   priv->dma_cap.multi_addr);
+		seq_printf(seq,
+			   "\tAdditional 32 MAC address registers (32-63): %s\n",
+			   priv->dma_cap.additional_32_addr ? "Y" : "N");
+		seq_printf(seq,
+			   "\tAdditional 64 MAC address registers (64-127): %s\n",
+			   priv->dma_cap.additional_64_addr ? "Y" : "N");
+		seq_printf(seq, "\tHash Filter: %s\n",
+			   (priv->dma_cap.hash_filter) ? "Y" : "N");
 	} else {
 		seq_printf(seq, "\tHash Filter: %s\n",
 			   (priv->dma_cap.hash_filter) ? "Y" : "N");
@@ -7229,6 +7279,8 @@ int stmmac_xdp_open(struct net_device *dev)
 	return 0;
 
 irq_error:
+	stmmac_stop_all_dma(priv);
+
 	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
 		hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer);
 
@@ -7340,7 +7392,6 @@ static const struct net_device_ops stmmac_netdev_ops = {
 	.ndo_eth_ioctl = stmmac_ioctl,
 	.ndo_get_stats64 = stmmac_get_stats64,
 	.ndo_setup_tc = stmmac_setup_tc,
-	.ndo_select_queue = stmmac_select_queue,
 	.ndo_set_mac_address = stmmac_set_mac_address,
 	.ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
 	.ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
@@ -7522,6 +7573,33 @@ static int stmmac_hw_init(struct stmmac_priv *priv)
 		priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size;
 	}
 
+	/* On DWMAC4 we can get the exact number of perfect filter entries from
+	 * the HW_Features.
+	 */
+	if (priv->plat->core_type == DWMAC_CORE_GMAC4) {
+		priv->hw->multi_addr = priv->dma_cap.multi_addr;
+		priv->hw->additional_32_addr =
+			!!priv->dma_cap.additional_32_addr;
+		priv->hw->additional_64_addr =
+			!!priv->dma_cap.additional_64_addr;
+
+		/* We always have one slot for the primary MAC */
+		priv->hw->unicast_filter_entries = 1;
+
+		/* How many slots in the 1 -> 31 range */
+		priv->hw->unicast_filter_entries += priv->hw->multi_addr;
+
+		/* Additional 32 entries in the 32 -> 63 range */
+		if (priv->hw->additional_32_addr)
+			priv->hw->unicast_filter_entries += 32;
+
+		/* Additional 64 entries in the 64 -> 127 range, can be enabled
+		 * independently of the 32 -> 63 range
+		 */
+		if (priv->hw->additional_64_addr)
+			priv->hw->unicast_filter_entries += 64;
+	}
+
 	priv->hw->vlan_fail_q_en =
 		(priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN);
 	priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;
@@ -7740,8 +7818,7 @@ static int stmmac_register_devlink(struct stmmac_priv *priv)
 	/* For now, what is exposed over devlink is only relevant when
 	 * timestamping is available and we have a valid ptp clock rate
 	 */
-	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp) ||
-	    !priv->plat->clk_ptp_rate)
+	if (!stmmac_check_timestamp_cap(priv) || !priv->plat->clk_ptp_rate)
 		return 0;
 
 	priv->devlink = devlink_alloc(&stmmac_devlink_ops, sizeof(*dl_priv),
@@ -8346,14 +8423,19 @@ int stmmac_resume(struct device *dev)
 	ret = stmmac_hw_setup(ndev);
 	if (ret < 0) {
 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
-		stmmac_legacy_serdes_power_down(priv);
-		mutex_unlock(&priv->lock);
-		rtnl_unlock();
-		return ret;
+		goto error_unlock;
 	}
 
-	stmmac_init_timestamping(priv);
+	if (priv->ptp_enabled) {
+		if (stmmac_init_ptp_clk_freq(priv))
+			goto init_coalesce;
 
+		ret = stmmac_init_timestamping(priv);
+		if (ret)
+			goto error_stop_dma;
+	}
+
+init_coalesce:
 	stmmac_init_coalesce(priv);
 	phylink_rx_clk_stop_block(priv->phylink);
 	stmmac_set_rx_mode(ndev);
@@ -8376,6 +8458,16 @@ int stmmac_resume(struct device *dev)
 	netif_device_attach(ndev);
 
 	return 0;
+
+error_stop_dma:
+	stmmac_stop_all_dma(priv);
+	stmmac_mac_set(priv, priv->ioaddr, false);
+error_unlock:
+	stmmac_legacy_serdes_power_down(priv);
+	mutex_unlock(&priv->lock);
+	rtnl_unlock();
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(stmmac_resume);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index 6128ed1bd521a4..3ea16d6182151f 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -16,6 +16,7 @@
 #include <linux/of.h>
 #include <linux/of_net.h>
 #include <linux/of_mdio.h>
+#include <linux/phy.h>
 
 #include "stmmac.h"
 #include "stmmac_platform.h"
@@ -847,6 +848,16 @@ int stmmac_pltfr_probe(struct platform_device *pdev,
 		       struct plat_stmmacenet_data *plat,
 		       struct stmmac_resources *res)
 {
+	if (plat->has_internal_tx_delay || plat->has_internal_rx_delay) {
+		plat->phy_interface =
+			phy_fix_phy_mode_for_mac_delays(plat->phy_interface,
+							plat->has_internal_tx_delay,
+							plat->has_internal_rx_delay);
+		if (plat->phy_interface == PHY_INTERFACE_MODE_NA)
+			return dev_err_probe(&pdev->dev, -EINVAL,
+					     "unsupported phy interface mode\n");
+	}
+
 	if (!plat->suspend && plat->exit)
 		plat->suspend = stmmac_plat_suspend;
 	if (!plat->resume && plat->init)
@@ -959,7 +970,8 @@ static int __maybe_unused stmmac_pltfr_noirq_suspend(struct device *dev)
 
 	if (!priv->wolopts) {
 		/* Disable clock in case of PWM is off */
-		clk_disable_unprepare(priv->plat->clk_ptp_ref);
+		if (priv->ptp_enabled)
+			clk_disable_unprepare(priv->plat->clk_ptp_ref);
 
 		ret = pm_runtime_force_suspend(dev);
 		if (ret)
@@ -984,6 +996,9 @@ static int __maybe_unused stmmac_pltfr_noirq_resume(struct device *dev)
 		if (ret)
 			return ret;
 
+		if (!priv->ptp_enabled)
+			return 0;
+
 		ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
 		if (ret < 0) {
 			netdev_warn(priv->dev,
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c
index 6372ec7c3f3170..6097f312fce447 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c
@@ -12,6 +12,7 @@
 #include <linux/ethtool.h>
 #include <linux/ip.h>
 #include <linux/udp.h>
+#include <net/dsa.h>
 #include <net/pkt_cls.h>
 #include <net/pkt_sched.h>
 #include <net/tcp.h>
@@ -29,6 +30,7 @@ struct stmmachdr {
 			      sizeof(struct stmmachdr))
 #define STMMAC_TEST_PKT_MAGIC	0xdeadcafecafedeadULL
 #define STMMAC_LB_TIMEOUT	msecs_to_jiffies(200)
+#define STMMAC_SFT_MAX_LPI	(5 * USEC_PER_SEC)
 
 struct stmmac_packet_attrs {
 	int vlan;
@@ -215,28 +217,14 @@ static struct sk_buff *stmmac_test_get_udp_skb(struct stmmac_priv *priv,
 	return skb;
 }
 
-static struct sk_buff *stmmac_test_get_arp_skb(struct stmmac_priv *priv,
-					       struct stmmac_packet_attrs *attr)
-{
-	__be32 ip_src = htonl(attr->ip_src);
-	__be32 ip_dst = htonl(attr->ip_dst);
-	struct sk_buff *skb = NULL;
-
-	skb = arp_create(ARPOP_REQUEST, ETH_P_ARP, ip_dst, priv->dev, ip_src,
-			 NULL, attr->src, attr->dst);
-	if (!skb)
-		return NULL;
-
-	skb->pkt_type = PACKET_HOST;
-	skb->dev = priv->dev;
-
-	return skb;
-}
-
 struct stmmac_test_priv {
 	struct stmmac_packet_attrs *packet;
 	struct packet_type pt;
 	struct completion comp;
+	__be16 packet_type;
+	int (*func)(struct sk_buff *skb, struct net_device *ndev,
+		    struct packet_type *pt, struct net_device *orig_ndev);
+	bool capture_all;
 	int double_vlan;
 	int vlan_id;
 	int ok;
@@ -316,6 +304,52 @@ static int stmmac_test_loopback_validate(struct sk_buff *skb,
 	return 0;
 }
 
+static int stmmac_sft_filter(struct sk_buff *skb, struct net_device *ndev,
+			     struct packet_type *pt,
+			     struct net_device *orig_ndev)
+{
+	struct stmmac_test_priv *tpriv = pt->af_packet_priv;
+	struct ethhdr *hdr = eth_hdr(skb);
+	int ret = 0;
+
+	if (hdr->h_proto == tpriv->packet_type) {
+		struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
+
+		if (nskb)
+			ret = tpriv->func(nskb, ndev, pt, orig_ndev);
+	}
+
+	kfree_skb(skb);
+	return ret;
+}
+
+static void stmmac_sft_add_pack(struct packet_type *pt)
+{
+	struct stmmac_test_priv *tpriv = pt->af_packet_priv;
+
+	if (netdev_uses_dsa(tpriv->pt.dev) || tpriv->capture_all) {
+		tpriv->packet_type = tpriv->pt.type;
+		tpriv->func = tpriv->pt.func;
+
+		/* DSA conduit will report ETH_P_XDSA, so our packet handler
+		 * won't match. Let's register a ETH_P_ALL match and filter
+		 * manually in stmmac_sft_filter. This is also useful for
+		 * VLAN tests, to capture packets otherwise marked as
+		 * OTHERHOST.
+		 */
+		tpriv->pt.type = htons(ETH_P_ALL);
+		tpriv->pt.func = stmmac_sft_filter;
+		tpriv->pt.ignore_outgoing = true;
+	}
+
+	dev_add_pack(pt);
+}
+
+static void stmmac_sft_remove_pack(struct packet_type *pt)
+{
+	dev_remove_pack(pt);
+}
+
 static int __stmmac_test_loopback(struct stmmac_priv *priv,
 				  struct stmmac_packet_attrs *attr)
 {
@@ -337,7 +371,7 @@ static int __stmmac_test_loopback(struct stmmac_priv *priv,
 	tpriv->packet = attr;
 
 	if (!attr->dont_wait)
-		dev_add_pack(&tpriv->pt);
+		stmmac_sft_add_pack(&tpriv->pt);
 
 	skb = stmmac_test_get_udp_skb(priv, attr);
 	if (!skb) {
@@ -360,7 +394,7 @@ static int __stmmac_test_loopback(struct stmmac_priv *priv,
 
 cleanup:
 	if (!attr->dont_wait)
-		dev_remove_pack(&tpriv->pt);
+		stmmac_sft_remove_pack(&tpriv->pt);
 	kfree(tpriv);
 	return ret;
 }
@@ -414,12 +448,16 @@ static int stmmac_test_mmc(struct stmmac_priv *priv)
 static int stmmac_test_eee(struct stmmac_priv *priv)
 {
 	struct stmmac_extra_stats *initial, *final;
-	int retries = 10;
+	unsigned long timeout, max_duration;
 	int ret;
 
 	if (!priv->dma_cap.eee || !priv->eee_active)
 		return -EOPNOTSUPP;
 
+	/* Bail out if the configured LPI timer is too long */
+	if (priv->tx_lpi_timer > STMMAC_SFT_MAX_LPI)
+		return -EOPNOTSUPP;
+
 	initial = kzalloc_obj(*initial);
 	if (!initial)
 		return -ENOMEM;
@@ -430,14 +468,21 @@ static int stmmac_test_eee(struct stmmac_priv *priv)
 		goto out_free_initial;
 	}
 
+	/* Snapshot stats, we want to count the in_lpi events. We may enter
+	 * LPI just after the packet was sent.
+	 */
 	memcpy(initial, &priv->xstats, sizeof(*initial));
 
+	/* Send a frame, then wait to enter LPI */
 	ret = stmmac_test_mac_loopback(priv);
 	if (ret)
 		goto out_free_final;
 
+	max_duration = usecs_to_jiffies(2 * priv->tx_lpi_timer);
+
 	/* We have no traffic in the line so, sooner or later it will go LPI */
-	while (--retries) {
+	timeout = jiffies + max_duration;
+	while (!time_after(jiffies, timeout)) {
 		memcpy(final, &priv->xstats, sizeof(*final));
 
 		if (final->irq_tx_path_in_lpi_mode_n >
@@ -446,20 +491,38 @@ static int stmmac_test_eee(struct stmmac_priv *priv)
 		msleep(100);
 	}
 
-	if (!retries) {
+	memcpy(final, &priv->xstats, sizeof(*final));
+	if (final->irq_tx_path_in_lpi_mode_n <=
+	    initial->irq_tx_path_in_lpi_mode_n) {
 		ret = -ETIMEDOUT;
 		goto out_free_final;
 	}
 
-	if (final->irq_tx_path_in_lpi_mode_n <=
-	    initial->irq_tx_path_in_lpi_mode_n) {
-		ret = -EINVAL;
+	/* Re-snapshot, as we want to measure exit_lpi events. We should be
+	 * in LPI right now.
+	 */
+	memcpy(initial, &priv->xstats, sizeof(*initial));
+
+	/* TX something so we go out of LPI */
+	ret = stmmac_test_mac_loopback(priv);
+	if (ret)
 		goto out_free_final;
+
+	/* Wait for the exit LPI interrupt */
+	timeout = jiffies + max_duration;
+	while (!time_after(jiffies, timeout)) {
+		memcpy(final, &priv->xstats, sizeof(*final));
+
+		if (final->irq_tx_path_exit_lpi_mode_n >
+		    initial->irq_tx_path_exit_lpi_mode_n)
+			break;
+		msleep(100);
 	}
 
+	memcpy(final, &priv->xstats, sizeof(*final));
 	if (final->irq_tx_path_exit_lpi_mode_n <=
 	    initial->irq_tx_path_exit_lpi_mode_n) {
-		ret = -EINVAL;
+		ret = -ETIMEDOUT;
 		goto out_free_final;
 	}
 
@@ -767,7 +830,7 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv)
 	tpriv->pt.func = stmmac_test_flowctrl_validate;
 	tpriv->pt.dev = priv->dev;
 	tpriv->pt.af_packet_priv = tpriv;
-	dev_add_pack(&tpriv->pt);
+	stmmac_sft_add_pack(&tpriv->pt);
 
 	/* Compute minimum number of packets to make FIFO full */
 	pkt_count = rx_fifo_size;
@@ -823,7 +886,7 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv)
 cleanup:
 	dev_mc_del(priv->dev, paddr);
 	dev_set_promiscuity(priv->dev, -1);
-	dev_remove_pack(&tpriv->pt);
+	stmmac_sft_remove_pack(&tpriv->pt);
 	kfree(tpriv);
 	return ret;
 }
@@ -865,6 +928,11 @@ static int stmmac_test_vlan_validate(struct sk_buff *skb,
 		goto out;
 	if (skb_headlen(skb) < (STMMAC_TEST_PKT_SIZE - ETH_HLEN))
 		goto out;
+
+	ehdr = (struct ethhdr *)skb_mac_header(skb);
+	if (!ether_addr_equal_unaligned(ehdr->h_dest, tpriv->packet->dst))
+		goto out;
+
 	if (tpriv->vlan_id) {
 		if (skb->vlan_proto != htons(proto))
 			goto out;
@@ -876,10 +944,6 @@ static int stmmac_test_vlan_validate(struct sk_buff *skb,
 		}
 	}
 
-	ehdr = (struct ethhdr *)skb_mac_header(skb);
-	if (!ether_addr_equal_unaligned(ehdr->h_dest, tpriv->packet->dst))
-		goto out;
-
 	ihdr = ip_hdr(skb);
 	if (tpriv->double_vlan)
 		ihdr = (struct iphdr *)(skb_network_header(skb) + 4);
@@ -921,6 +985,7 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv)
 	tpriv->pt.dev = priv->dev;
 	tpriv->pt.af_packet_priv = tpriv;
 	tpriv->packet = &attr;
+	tpriv->capture_all = true;
 
 	/*
 	 * As we use HASH filtering, false positives may appear. This is a
@@ -928,18 +993,20 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv)
 	 * HASH values.
 	 */
 	tpriv->vlan_id = 0x123;
-	dev_add_pack(&tpriv->pt);
 
 	ret = vlan_vid_add(priv->dev, htons(ETH_P_8021Q), tpriv->vlan_id);
 	if (ret)
 		goto cleanup;
 
+	attr.vlan = 1;
+	attr.dst = priv->dev->dev_addr;
+	attr.sport = 9;
+	attr.dport = 9;
+
+	stmmac_sft_add_pack(&tpriv->pt);
+
 	for (i = 0; i < 4; i++) {
-		attr.vlan = 1;
 		attr.vlan_id_out = tpriv->vlan_id + i;
-		attr.dst = priv->dev->dev_addr;
-		attr.sport = 9;
-		attr.dport = 9;
 
 		skb = stmmac_test_get_udp_skb(priv, &attr);
 		if (!skb) {
@@ -966,9 +1033,9 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv)
 	}
 
 vlan_del:
+	stmmac_sft_remove_pack(&tpriv->pt);
 	vlan_vid_del(priv->dev, htons(ETH_P_8021Q), tpriv->vlan_id);
 cleanup:
-	dev_remove_pack(&tpriv->pt);
 	kfree(tpriv);
 	return ret;
 }
@@ -1015,6 +1082,7 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv)
 	tpriv->pt.dev = priv->dev;
 	tpriv->pt.af_packet_priv = tpriv;
 	tpriv->packet = &attr;
+	tpriv->capture_all = true;
 
 	/*
 	 * As we use HASH filtering, false positives may appear. This is a
@@ -1022,18 +1090,20 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv)
 	 * HASH values.
 	 */
 	tpriv->vlan_id = 0x123;
-	dev_add_pack(&tpriv->pt);
 
 	ret = vlan_vid_add(priv->dev, htons(ETH_P_8021AD), tpriv->vlan_id);
 	if (ret)
 		goto cleanup;
 
+	attr.vlan = 2;
+	attr.dst = priv->dev->dev_addr;
+	attr.sport = 9;
+	attr.dport = 9;
+
+	stmmac_sft_add_pack(&tpriv->pt);
+
 	for (i = 0; i < 4; i++) {
-		attr.vlan = 2;
 		attr.vlan_id_out = tpriv->vlan_id + i;
-		attr.dst = priv->dev->dev_addr;
-		attr.sport = 9;
-		attr.dport = 9;
 
 		skb = stmmac_test_get_udp_skb(priv, &attr);
 		if (!skb) {
@@ -1060,9 +1130,9 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv)
 	}
 
 vlan_del:
+	stmmac_sft_remove_pack(&tpriv->pt);
 	vlan_vid_del(priv->dev, htons(ETH_P_8021AD), tpriv->vlan_id);
 cleanup:
-	dev_remove_pack(&tpriv->pt);
 	kfree(tpriv);
 	return ret;
 }
@@ -1293,7 +1363,7 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan)
 	tpriv->pt.af_packet_priv = tpriv;
 	tpriv->packet = &attr;
 	tpriv->vlan_id = 0x123;
-	dev_add_pack(&tpriv->pt);
+	tpriv->capture_all = true;
 
 	ret = vlan_vid_add(priv->dev, htons(proto), tpriv->vlan_id);
 	if (ret)
@@ -1301,6 +1371,8 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan)
 
 	attr.dst = priv->dev->dev_addr;
 
+	stmmac_sft_add_pack(&tpriv->pt);
+
 	skb = stmmac_test_get_udp_skb(priv, &attr);
 	if (!skb) {
 		ret = -ENOMEM;
@@ -1318,9 +1390,9 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan)
 	ret = tpriv->ok ? 0 : -ETIMEDOUT;
 
 vlan_del:
+	stmmac_sft_remove_pack(&tpriv->pt);
 	vlan_vid_del(priv->dev, htons(proto), tpriv->vlan_id);
 cleanup:
-	dev_remove_pack(&tpriv->pt);
 	kfree(tpriv);
 	return ret;
 }
@@ -1332,7 +1404,7 @@ static int stmmac_test_vlanoff(struct stmmac_priv *priv)
 
 static int stmmac_test_svlanoff(struct stmmac_priv *priv)
 {
-	if (!priv->dma_cap.dvlan)
+	if (!(priv->dev->features & NETIF_F_HW_VLAN_STAG_TX))
 		return -EOPNOTSUPP;
 	return stmmac_test_vlanoff_common(priv, true);
 }
@@ -1603,102 +1675,14 @@ static int stmmac_test_l4filt_sa_udp(struct stmmac_priv *priv)
 	return __stmmac_test_l4filt(priv, 0, dummy_port, 0, ~0, true);
 }
 
-static int stmmac_test_arp_validate(struct sk_buff *skb,
-				    struct net_device *ndev,
-				    struct packet_type *pt,
-				    struct net_device *orig_ndev)
-{
-	struct stmmac_test_priv *tpriv = pt->af_packet_priv;
-	struct ethhdr *ehdr;
-	struct arphdr *ahdr;
-
-	ehdr = (struct ethhdr *)skb_mac_header(skb);
-	if (!ether_addr_equal_unaligned(ehdr->h_dest, tpriv->packet->src))
-		goto out;
-
-	ahdr = arp_hdr(skb);
-	if (ahdr->ar_op != htons(ARPOP_REPLY))
-		goto out;
-
-	tpriv->ok = true;
-	complete(&tpriv->comp);
-out:
-	kfree_skb(skb);
-	return 0;
-}
-
-static int stmmac_test_arpoffload(struct stmmac_priv *priv)
-{
-	unsigned char src[ETH_ALEN] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06};
-	unsigned char dst[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	struct stmmac_packet_attrs attr = { };
-	struct stmmac_test_priv *tpriv;
-	struct sk_buff *skb = NULL;
-	u32 ip_addr = 0xdeadcafe;
-	u32 ip_src = 0xdeadbeef;
-	int ret;
-
-	if (!priv->dma_cap.arpoffsel)
-		return -EOPNOTSUPP;
-
-	tpriv = kzalloc_obj(*tpriv);
-	if (!tpriv)
-		return -ENOMEM;
-
-	tpriv->ok = false;
-	init_completion(&tpriv->comp);
-
-	tpriv->pt.type = htons(ETH_P_ARP);
-	tpriv->pt.func = stmmac_test_arp_validate;
-	tpriv->pt.dev = priv->dev;
-	tpriv->pt.af_packet_priv = tpriv;
-	tpriv->packet = &attr;
-	dev_add_pack(&tpriv->pt);
-
-	attr.src = src;
-	attr.ip_src = ip_src;
-	attr.dst = dst;
-	attr.ip_dst = ip_addr;
-
-	skb = stmmac_test_get_arp_skb(priv, &attr);
-	if (!skb) {
-		ret = -ENOMEM;
-		goto cleanup;
-	}
-
-	ret = stmmac_set_arp_offload(priv, priv->hw, true, ip_addr);
-	if (ret) {
-		kfree_skb(skb);
-		goto cleanup;
-	}
-
-	ret = dev_set_promiscuity(priv->dev, 1);
-	if (ret) {
-		kfree_skb(skb);
-		goto cleanup;
-	}
-
-	ret = dev_direct_xmit(skb, 0);
-	if (ret)
-		goto cleanup_promisc;
-
-	wait_for_completion_timeout(&tpriv->comp, STMMAC_LB_TIMEOUT);
-	ret = tpriv->ok ? 0 : -ETIMEDOUT;
-
-cleanup_promisc:
-	dev_set_promiscuity(priv->dev, -1);
-cleanup:
-	stmmac_set_arp_offload(priv, priv->hw, false, 0x0);
-	dev_remove_pack(&tpriv->pt);
-	kfree(tpriv);
-	return ret;
-}
-
 static int __stmmac_test_jumbo(struct stmmac_priv *priv, u16 queue)
 {
 	struct stmmac_packet_attrs attr = { };
 	int size = priv->dma_conf.dma_buf_sz;
 
+	if (!dwmac_is_xmac(priv->plat->core_type))
+		size -= NET_IP_ALIGN;
+
 	attr.dst = priv->dev->dev_addr;
 	attr.max_size = size - ETH_FCS_LEN;
 	attr.queue_mapping = queue;
@@ -1911,9 +1895,6 @@ static const struct stmmac_test {
 	}, {
 		.name = "L4 SA UDP Filtering        ",
 		.fn = stmmac_test_l4filt_sa_udp,
-	}, {
-		.name = "ARP Offload                ",
-		.fn = stmmac_test_arpoffload,
 	}, {
 		.name = "Jumbo Frame                ",
 		.fn = stmmac_test_jumbo,
diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c
index 54dd7281191da5..c74a97fe54641b 100644
--- a/drivers/net/ethernet/sun/niu.c
+++ b/drivers/net/ethernet/sun/niu.c
@@ -9835,15 +9835,11 @@ static int niu_pci_init_one(struct pci_dev *pdev,
 		PCI_EXP_DEVCTL_RELAX_EN);
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));
-	if (!err)
-		dev->features |= NETIF_F_HIGHDMA;
 	if (err) {
-		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "No usable DMA configuration, aborting\n");
-			goto err_out_release_parent;
-		}
+		dev_err(&pdev->dev, "No usable DMA configuration, aborting\n");
+		goto err_out_release_parent;
 	}
+	dev->features |= NETIF_F_HIGHDMA;
 
 	niu_set_basic_features(dev);
 
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index 6330540979d70e..2551554add17a4 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -3506,7 +3506,7 @@ static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
 	{ .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
 	{ .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata},
 	{ .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
 
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index aa3531e844e871..63b64faeaca0b8 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -1529,7 +1529,7 @@ static const struct of_device_id cpsw_of_mtable[] = {
 	{ .compatible = "ti,am335x-cpsw"},
 	{ .compatible = "ti,am4372-cpsw"},
 	{ .compatible = "ti,dra7-cpsw"},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
 
diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c
index b6e25a4658d1aa..07832cad5e3c21 100644
--- a/drivers/net/ethernet/ti/cpsw_new.c
+++ b/drivers/net/ethernet/ti/cpsw_new.c
@@ -1875,7 +1875,7 @@ static const struct of_device_id cpsw_of_mtable[] = {
 	{ .compatible = "ti,am335x-cpsw-switch"},
 	{ .compatible = "ti,am4372-cpsw-switch"},
 	{ .compatible = "ti,dra7-cpsw-switch"},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, cpsw_of_mtable);
 
diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c
index 48f85a3649b265..5bec72f2f51dc5 100644
--- a/drivers/net/ethernet/ti/davinci_mdio.c
+++ b/drivers/net/ethernet/ti/davinci_mdio.c
@@ -510,7 +510,7 @@ static const struct soc_device_attribute k3_mdio_socinfo[] = {
 	{ .family = "J7200", .data = &am65_mdio_soc_data },
 	{ .family = "J721E", .data = &am65_mdio_soc_data },
 	{ .family = "J721S2", .data = &am65_mdio_soc_data },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 
 #if IS_ENABLED(CONFIG_OF)
@@ -521,7 +521,7 @@ static const struct davinci_mdio_of_param of_cpsw_mdio_data = {
 static const struct of_device_id davinci_mdio_of_mtable[] = {
 	{ .compatible = "ti,davinci_mdio", },
 	{ .compatible = "ti,cpsw-mdio", .data = &of_cpsw_mdio_data},
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, davinci_mdio_of_mtable);
 #endif
diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c
index eb8fc2ed05f45b..4f9e20468bbbfe 100644
--- a/drivers/net/ethernet/ti/netcp_core.c
+++ b/drivers/net/ethernet/ti/netcp_core.c
@@ -2225,7 +2225,7 @@ static int netcp_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	pm_runtime_enable(&pdev->dev);
-	ret = pm_runtime_get_sync(&pdev->dev);
+	ret = pm_runtime_resume_and_get(&pdev->dev);
 	if (ret < 0) {
 		dev_err(dev, "Failed to enable NETCP power-domain\n");
 		pm_runtime_disable(&pdev->dev);
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
index 122c4952d203f0..111aadf792089b 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
@@ -2518,9 +2518,10 @@ int wx_sw_init(struct wx *wx)
 	}
 
 	spin_lock_init(&wx->hw_stats_lock);
+	spin_lock_init(&wx->ptp_tx_lock);
 	mutex_init(&wx->reset_lock);
 	bitmap_zero(wx->state, WX_STATE_NBITS);
-	bitmap_zero(wx->flags, WX_PF_FLAGS_NBITS);
+	bitmap_zero(wx->flags, WX_FLAGS_NBITS);
 	set_bit(WX_STATE_DOWN, wx->state);
 	wx->misc_irq_domain = false;
 
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c
index ed5aad7857bd9b..dcbf5811046e96 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c
@@ -1200,9 +1200,11 @@ static int wx_tx_map(struct wx_ring *tx_ring,
 		i--;
 	}
 
-	dev_kfree_skb_any(first->skb);
-	first->skb = NULL;
-
+	/* first->skb is released by the caller, which keeps a reference on it
+	 * until the PTP cleanup has compared it against wx->ptp_tx_skb. That
+	 * prevents the address from being reused by a newer request while the
+	 * comparison is pending.
+	 */
 	tx_ring->next_to_use = i;
 
 	return -ENOMEM;
@@ -1649,9 +1651,11 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
 
 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
 	    wx->ptp_clock) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&wx->ptp_tx_lock, flags);
 		if (wx->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
-		    !test_and_set_bit_lock(WX_STATE_PTP_TX_IN_PROGRESS,
-					   wx->state)) {
+		    !test_and_set_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) {
 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 			tx_flags |= WX_TX_FLAGS_TSTAMP;
 			wx->ptp_tx_skb = skb_get(skb);
@@ -1659,6 +1663,7 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
 		} else {
 			wx->tx_hwtstamp_skipped++;
 		}
+		spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
 	}
 
 	/* record initial flags and protocol */
@@ -1677,19 +1682,35 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,
 		wx->atr(tx_ring, first, ptype);
 
 	if (wx_tx_map(tx_ring, first, hdr_len))
-		goto cleanup_tx_tstamp;
+		goto out_drop;
 
 	return NETDEV_TX_OK;
 out_drop:
-	dev_kfree_skb_any(first->skb);
-	first->skb = NULL;
-cleanup_tx_tstamp:
+	/* The frame never reached the hardware, so no timestamp will ever be
+	 * reported for it and the request has to be cancelled. The slot is
+	 * shared, though: wx_ptp_clear_tx_timestamp() or wx_ptp_tx_hang() may
+	 * have dropped our request already, and a transmit on another queue
+	 * can have claimed the slot since. Only cancel it while it is still
+	 * ours, otherwise we would free somebody else's skb and release their
+	 * in-progress bit.
+	 */
 	if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) {
-		dev_kfree_skb_any(wx->ptp_tx_skb);
-		wx->ptp_tx_skb = NULL;
-		wx->tx_hwtstamp_errors++;
-		clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
+		struct sk_buff *ptp_tx_skb = NULL;
+		unsigned long flags;
+
+		spin_lock_irqsave(&wx->ptp_tx_lock, flags);
+		if (wx->ptp_tx_skb == skb) {
+			ptp_tx_skb = wx->ptp_tx_skb;
+			wx->ptp_tx_skb = NULL;
+			clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
+			wx->tx_hwtstamp_errors++;
+		}
+		spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
+
+		dev_kfree_skb_any(ptp_tx_skb);
 	}
+	dev_kfree_skb_any(first->skb);
+	first->skb = NULL;
 
 	return NETDEV_TX_OK;
 }
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c
index 4708e7f3958f75..e2d60d6bcade93 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c
@@ -129,6 +129,34 @@ static int wx_ptp_settime64(struct ptp_clock_info *ptp,
 	return 0;
 }
 
+/**
+ * __wx_ptp_detach_tx_skb - detach the skb tracking the Tx timestamp request
+ * @wx: the private board structure
+ *
+ * Detach the skb of the outstanding request and release the in-progress bit,
+ * so that a new request can be submitted.
+ *
+ * This performs no register access. Callers that need a timestamp the hardware
+ * may have left latched must unlatch it themselves, while the device is known
+ * to be alive. wx_ptp_quiesce() runs during PCIe error recovery, where MMIO is
+ * not reliable, and therefore deliberately skips the unlatch.
+ *
+ * Context: Expects wx->ptp_tx_lock to be held by the caller.
+ * Return: the detached skb, or NULL if no request was outstanding. The caller
+ * owns the returned reference and must release it once the lock is dropped.
+ */
+static struct sk_buff *__wx_ptp_detach_tx_skb(struct wx *wx)
+{
+	struct sk_buff *skb = wx->ptp_tx_skb;
+
+	lockdep_assert_held(&wx->ptp_tx_lock);
+
+	wx->ptp_tx_skb = NULL;
+	clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
+
+	return skb;
+}
+
 /**
  * wx_ptp_clear_tx_timestamp - utility function to clear Tx timestamp state
  * @wx: the private board structure
@@ -139,12 +167,16 @@ static int wx_ptp_settime64(struct ptp_clock_info *ptp,
  */
 static void wx_ptp_clear_tx_timestamp(struct wx *wx)
 {
+	struct sk_buff *skb;
+	unsigned long flags;
+
+	spin_lock_irqsave(&wx->ptp_tx_lock, flags);
+	/* Unlatch a timestamp the hardware may have left pending. */
 	rd32ptp(wx, WX_TSC_1588_STMPH);
-	if (wx->ptp_tx_skb) {
-		dev_kfree_skb_any(wx->ptp_tx_skb);
-		wx->ptp_tx_skb = NULL;
-	}
-	clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
+	skb = __wx_ptp_detach_tx_skb(wx);
+	spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
+
+	dev_kfree_skb_any(skb);
 }
 
 /**
@@ -175,49 +207,54 @@ static void wx_ptp_convert_to_hwtstamp(struct wx *wx,
 }
 
 /**
- * wx_ptp_tx_hwtstamp - utility function which checks for TX time stamp
+ * wx_ptp_tx_hwtstamp_work - check for a pending Tx time stamp
  * @wx: the private board struct
  *
- * if the timestamp is valid, we convert it into the timecounter ns
- * value, then store that result into the shhwtstamps structure which
- * is passed up the network stack
+ * If a Tx timestamp request is outstanding and the hardware has latched a
+ * valid value, we convert it into the timecounter ns value, then store that
+ * result into the shhwtstamps structure which is passed up the network stack.
+ *
+ * Return: 0 when there is nothing left to poll for, -1 when the timestamp is
+ * not available yet and the caller should poll again.
  */
-static void wx_ptp_tx_hwtstamp(struct wx *wx)
+static int wx_ptp_tx_hwtstamp_work(struct wx *wx)
 {
 	struct skb_shared_hwtstamps shhwtstamps;
-	struct sk_buff *skb = wx->ptp_tx_skb;
+	unsigned long flags;
+	struct sk_buff *skb;
+	u32 tsynctxctl;
 	u64 regval = 0;
 
-	regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPL);
-	regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPH) << 32;
-
-	wx_ptp_convert_to_hwtstamp(wx, &shhwtstamps, regval);
-
-	wx->ptp_tx_skb = NULL;
-	clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
-	skb_tstamp_tx(skb, &shhwtstamps);
-	dev_kfree_skb_any(skb);
-	wx->tx_hwtstamp_pkts++;
-}
-
-static int wx_ptp_tx_hwtstamp_work(struct wx *wx)
-{
-	u32 tsynctxctl;
+	spin_lock_irqsave(&wx->ptp_tx_lock, flags);
 
 	/* we have to have a valid skb to poll for a timestamp */
 	if (!wx->ptp_tx_skb) {
-		wx_ptp_clear_tx_timestamp(wx);
+		rd32ptp(wx, WX_TSC_1588_STMPH);
+		__wx_ptp_detach_tx_skb(wx);
+		spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
 		return 0;
 	}
 
 	/* stop polling once we have a valid timestamp */
 	tsynctxctl = rd32ptp(wx, WX_TSC_1588_CTL);
-	if (tsynctxctl & WX_TSC_1588_CTL_VALID) {
-		wx_ptp_tx_hwtstamp(wx);
-		return 0;
+	if (!(tsynctxctl & WX_TSC_1588_CTL_VALID)) {
+		spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
+		return -1;
 	}
 
-	return -1;
+	regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPL);
+	regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPH) << 32;
+	skb = wx->ptp_tx_skb;
+	wx->ptp_tx_skb = NULL;
+	clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
+	spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
+
+	wx_ptp_convert_to_hwtstamp(wx, &shhwtstamps, regval);
+	skb_tstamp_tx(skb, &shhwtstamps);
+	dev_kfree_skb_any(skb);
+	wx->tx_hwtstamp_pkts++;
+
+	return 0;
 }
 
 /**
@@ -296,24 +333,29 @@ static void wx_ptp_rx_hang(struct wx *wx)
  */
 static void wx_ptp_tx_hang(struct wx *wx)
 {
-	bool timeout = time_is_before_jiffies(wx->ptp_tx_start +
-					      WX_PTP_TX_TIMEOUT);
-
-	if (!wx->ptp_tx_skb)
-		return;
+	struct sk_buff *skb = NULL;
+	unsigned long flags;
 
-	if (!test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state))
-		return;
+	spin_lock_irqsave(&wx->ptp_tx_lock, flags);
 
 	/* If we haven't received a timestamp within the timeout, it is
 	 * reasonable to assume that it will never occur, so we can unlock the
 	 * timestamp bit when this occurs.
 	 */
-	if (timeout) {
-		wx_ptp_clear_tx_timestamp(wx);
-		wx->tx_hwtstamp_timeouts++;
-		dev_warn(&wx->pdev->dev, "clearing Tx timestamp hang\n");
+	if (wx->ptp_tx_skb &&
+	    test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state) &&
+	    time_is_before_jiffies(wx->ptp_tx_start + WX_PTP_TX_TIMEOUT)) {
+		rd32ptp(wx, WX_TSC_1588_STMPH);
+		skb = __wx_ptp_detach_tx_skb(wx);
 	}
+	spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
+
+	if (!skb)
+		return;
+
+	dev_kfree_skb_any(skb);
+	wx->tx_hwtstamp_timeouts++;
+	dev_warn(&wx->pdev->dev, "clearing Tx timestamp hang\n");
 }
 
 static long wx_ptp_do_aux_work(struct ptp_clock_info *ptp)
@@ -450,10 +492,7 @@ static int wx_ptp_feature_enable(struct ptp_clock_info *ptp,
 		wx->pps_width = 120000000;
 	}
 
-	if (on)
-		set_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags);
-	else
-		clear_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags);
+	assign_bit(WX_FLAG_PTP_PPS_ENABLED, wx->flags, on);
 
 	return wx->ptp_setup_sdp(wx);
 }
@@ -841,6 +880,9 @@ EXPORT_SYMBOL(wx_ptp_stop);
 
 void wx_ptp_quiesce(struct wx *wx)
 {
+	struct sk_buff *skb;
+	unsigned long flags;
+
 	if (!test_and_clear_bit(WX_STATE_PTP_RUNNING, wx->state))
 		return;
 
@@ -849,11 +891,14 @@ void wx_ptp_quiesce(struct wx *wx)
 	if (wx->ptp_clock)
 		ptp_cancel_worker_sync(wx->ptp_clock);
 
-	if (wx->ptp_tx_skb) {
-		dev_kfree_skb_any(wx->ptp_tx_skb);
-		wx->ptp_tx_skb = NULL;
-	}
-	clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state);
+	/* Drop a pending Tx timestamp request. Do not touch the registers
+	 * here: quiesce runs during PCIe error recovery, where the device may
+	 * already be gone and MMIO is not reliable.
+	 */
+	spin_lock_irqsave(&wx->ptp_tx_lock, flags);
+	skb = __wx_ptp_detach_tx_skb(wx);
+	spin_unlock_irqrestore(&wx->ptp_tx_lock, flags);
+	dev_kfree_skb_any(skb);
 
 	if (wx->ptp_clock) {
 		ptp_clock_unregister(wx->ptp_clock);
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h
index 9454e90258d8ed..a7f3c48e9603cb 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_type.h
+++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h
@@ -1269,7 +1269,7 @@ struct wx_rss_flow_map {
 	u8 flag;
 };
 
-enum wx_pf_flags {
+enum wx_flags {
 	WX_FLAG_MULTI_64_FUNC,
 	WX_FLAG_SWFW_RING,
 	WX_FLAG_VMDQ_ENABLED,
@@ -1291,13 +1291,13 @@ enum wx_pf_flags {
 	WX_FLAG_RX_MERGE_ENABLED,
 	WX_FLAG_TXHEAD_WB_ENABLED,
 	WX_FLAG_NEED_PCIE_RECOVERY,
-	WX_PF_FLAGS_NBITS               /* must be last */
+	WX_FLAGS_NBITS               /* must be last */
 };
 
 struct wx {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
 	DECLARE_BITMAP(state, WX_STATE_NBITS);
-	DECLARE_BITMAP(flags, WX_PF_FLAGS_NBITS);
+	DECLARE_BITMAP(flags, WX_FLAGS_NBITS);
 
 	void *priv;
 	u8 __iomem *hw_addr;
@@ -1430,6 +1430,8 @@ struct wx {
 	unsigned long last_overflow_check;
 	unsigned long last_rx_ptp_check;
 	unsigned long ptp_tx_start;
+	/* protects ptp_tx_skb, ptp_tx_start and the in-progress state bit */
+	spinlock_t ptp_tx_lock;
 	seqlock_t hw_tc_lock; /* seqlock for ptp */
 	struct cyclecounter hw_cc;
 	struct timecounter hw_tc;
diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c
index a840108285517f..59a47532618c89 100644
--- a/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c
+++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c
@@ -591,15 +591,24 @@ static void txgbe_fdir_filter_restore(struct wx *wx)
 			queue = TXGBE_RDB_FDIR_DROP_QUEUE;
 		} else {
 			u32 ring = ethtool_get_flow_spec_ring(filter->action);
+			u8 vf = ethtool_get_flow_spec_ring_vf(filter->action);
 
-			if (ring >= wx->num_rx_queues) {
+			if (!vf && ring >= wx->num_rx_queues) {
 				wx_err(wx, "FDIR restore failed, ring:%u\n",
 				       ring);
 				continue;
+			} else if (vf && (vf > wx->num_vfs ||
+				  ring >= wx->num_rx_queues_per_pool)) {
+				wx_err(wx, "FDIR restore failed, vf:%u, ring:%u\n",
+				       vf, ring);
+				continue;
 			}
 
 			/* Map the ring onto the absolute queue index */
-			queue = wx->rx_ring[ring]->reg_idx;
+			if (!vf)
+				queue = wx->rx_ring[ring]->reg_idx;
+			else
+				queue = ((vf - 1) * wx->num_rx_queues_per_pool) + ring;
 		}
 
 		ret = txgbe_fdir_write_perfect_filter(wx,
diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c
index dc9f243146580b..c81d485221d56d 100644
--- a/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c
+++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c
@@ -434,7 +434,7 @@ static int txgbe_clock_register(struct txgbe *txgbe)
 
 	clock = clkdev_create(clk, NULL, "%s", clk_name);
 	if (!clock) {
-		clk_unregister(clk);
+		clk_unregister_fixed_rate(clk);
 		return -ENOMEM;
 	}
 
@@ -637,7 +637,7 @@ int txgbe_init_phy(struct txgbe *txgbe)
 	platform_device_unregister(txgbe->i2c_dev);
 err_unregister_clk:
 	clkdev_drop(txgbe->clock);
-	clk_unregister(txgbe->clk);
+	clk_unregister_fixed_rate(txgbe->clk);
 err_destroy_phylink:
 	phylink_destroy(wx->phylink);
 err_destroy_xpcs:
@@ -671,7 +671,7 @@ void txgbe_remove_phy(struct txgbe *txgbe)
 	platform_device_unregister(txgbe->sfp_dev);
 	platform_device_unregister(txgbe->i2c_dev);
 	clkdev_drop(txgbe->clock);
-	clk_unregister(txgbe->clk);
+	clk_unregister_fixed_rate(txgbe->clk);
 	phylink_destroy(txgbe->wx->phylink);
 	xpcs_destroy_pcs(txgbe->pcs);
 	software_node_unregister_node_group(txgbe->nodes.group);
diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c
index 53d8dc642fbd60..dff39ba8103c47 100644
--- a/drivers/net/ethernet/wiznet/w5100.c
+++ b/drivers/net/ethernet/wiznet/w5100.c
@@ -18,9 +18,11 @@
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/mutex.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/interrupt.h>
+#include <linux/property.h>
 #include <linux/irq.h>
 
 #include "w5100.h"
@@ -124,6 +126,8 @@ MODULE_LICENSE("GPL");
  */
 #define W5500_SIMR		0x0018 /* Socket Interrupt Mask Register */
 #define W5500_RTR		0x0019 /* Retry Time-value Register */
+#define W5500_PHYCFGR		0x002e /* PHY Configuration Register */
+#define   PHYCFGR_LNK		  0x01 /* Link status */
 
 #define W5500_S0_REGS		0x10000
 
@@ -154,6 +158,9 @@ struct w5100_priv {
 	u16 s0_rx_buf_size;
 
 	int irq;
+	int link_irq;
+	/* Protects link state and carrier updates */
+	struct mutex link_lock;
 
 	struct napi_struct napi;
 	struct net_device *ndev;
@@ -345,6 +352,82 @@ static void w5500_memory_configure(struct w5100_priv *priv)
 	}
 }
 
+static int w5500_get_phycfgr_lnk(struct net_device *ndev)
+{
+	struct w5100_priv *priv = netdev_priv(ndev);
+	int retries = 3;
+	int ret;
+
+	do {
+		ret = w5100_read(priv, W5500_PHYCFGR);
+		if (ret < 0)
+			netif_err(priv, link, ndev,
+				  "failed to read link status: %d\n", ret);
+	} while (ret < 0 && --retries);
+
+	if (ret < 0)
+		return ret;
+
+	return ret & PHYCFGR_LNK;
+}
+
+static void w5500_report_carrier_state(struct net_device *ndev)
+{
+	struct w5100_priv *priv = netdev_priv(ndev);
+	int state;
+
+	mutex_lock(&priv->link_lock);
+
+	state = w5500_get_phycfgr_lnk(ndev);
+	if (state > 0) {
+		netif_info(priv, link, ndev, "link is up\n");
+		netif_carrier_on(ndev);
+	} else if (state == 0) {
+		netif_info(priv, link, ndev, "link is down\n");
+		netif_carrier_off(ndev);
+	}
+
+	mutex_unlock(&priv->link_lock);
+}
+
+static irqreturn_t w5500_detect_link_interrupt(int irq, void *ndev_instance)
+{
+	struct net_device *ndev = ndev_instance;
+
+	if (netif_running(ndev))
+		w5500_report_carrier_state(ndev);
+
+	return IRQ_HANDLED;
+}
+
+static u32 w5500_get_link(struct net_device *ndev)
+{
+	struct w5100_priv *priv = netdev_priv(ndev);
+	int state;
+	bool link;
+
+	mutex_lock(&priv->link_lock);
+
+	if (!netif_device_present(ndev)) {
+		link = netif_carrier_ok(ndev);
+		goto out;
+	}
+
+	state = w5500_get_phycfgr_lnk(ndev);
+
+	if (state < 0) {
+		link = netif_carrier_ok(ndev);
+		goto out;
+	}
+
+	link = state > 0;
+
+out:
+	mutex_unlock(&priv->link_lock);
+
+	return link;
+}
+
 static int w5100_hw_reset(struct w5100_priv *priv)
 {
 	u32 rtr;
@@ -448,12 +531,28 @@ static void w5100_restart(struct net_device *ndev)
 {
 	struct w5100_priv *priv = netdev_priv(ndev);
 
+	if (!netif_running(ndev) || !netif_device_present(ndev))
+		return;
+
+	disable_irq(priv->irq);
+	if (priv->link_irq > 0)
+		disable_irq(priv->link_irq);
+
 	netif_stop_queue(ndev);
+	w5100_hw_close(priv);
+	flush_work(&priv->rx_work);
+	flush_work(&priv->tx_work);
 	w5100_hw_reset(priv);
+	enable_irq(priv->irq);
 	w5100_hw_start(priv);
 	ndev->stats.tx_errors++;
 	netif_trans_update(ndev);
 	netif_wake_queue(ndev);
+
+	if (priv->link_irq > 0) {
+		enable_irq(priv->link_irq);
+		w5500_report_carrier_state(ndev);
+	}
 }
 
 static void w5100_restart_work(struct work_struct *work)
@@ -468,10 +567,7 @@ static void w5100_tx_timeout(struct net_device *ndev, unsigned int txqueue)
 {
 	struct w5100_priv *priv = netdev_priv(ndev);
 
-	if (priv->ops->may_sleep)
-		schedule_work(&priv->restart_work);
-	else
-		w5100_restart(ndev);
+	schedule_work(&priv->restart_work);
 }
 
 static void w5100_tx_skb(struct net_device *ndev, struct sk_buff *skb)
@@ -564,7 +660,8 @@ static void w5100_rx_work(struct work_struct *work)
 	while ((skb = w5100_rx_skb(priv->ndev)))
 		netif_rx(skb);
 
-	w5100_enable_intr(priv);
+	if (netif_running(priv->ndev))
+		w5100_enable_intr(priv);
 }
 
 static int w5100_napi_poll(struct napi_struct *napi, int budget)
@@ -656,9 +753,17 @@ static int w5100_open(struct net_device *ndev)
 	struct w5100_priv *priv = netdev_priv(ndev);
 
 	netif_info(priv, ifup, ndev, "enabling\n");
+	w5100_hw_reset(priv);
+	enable_irq(priv->irq);
 	w5100_hw_start(priv);
 	napi_enable(&priv->napi);
 	netif_start_queue(ndev);
+
+	if (priv->link_irq > 0) {
+		enable_irq(priv->link_irq);
+		w5500_report_carrier_state(ndev);
+	}
+
 	return 0;
 }
 
@@ -667,13 +772,40 @@ static int w5100_stop(struct net_device *ndev)
 	struct w5100_priv *priv = netdev_priv(ndev);
 
 	netif_info(priv, ifdown, ndev, "shutting down\n");
-	w5100_hw_close(priv);
-	netif_carrier_off(ndev);
+
+	disable_irq(priv->irq);
+	if (priv->link_irq > 0)
+		disable_irq(priv->link_irq);
+
 	netif_stop_queue(ndev);
 	napi_disable(&priv->napi);
+
+	cancel_work_sync(&priv->restart_work);
+	cancel_work_sync(&priv->setrx_work);
+
+	w5100_hw_close(priv);
+
+	flush_work(&priv->rx_work);
+	flush_work(&priv->tx_work);
+
+	if (priv->link_irq > 0)  {
+		mutex_lock(&priv->link_lock);
+		netif_carrier_off(ndev);
+		mutex_unlock(&priv->link_lock);
+	}
+
 	return 0;
 }
 
+static const struct ethtool_ops w5500_ethtool_ops = {
+	.get_drvinfo		= w5100_get_drvinfo,
+	.get_msglevel		= w5100_get_msglevel,
+	.set_msglevel		= w5100_set_msglevel,
+	.get_link		= w5500_get_link,
+	.get_regs_len		= w5100_get_regs_len,
+	.get_regs		= w5100_get_regs,
+};
+
 static const struct ethtool_ops w5100_ethtool_ops = {
 	.get_drvinfo		= w5100_get_drvinfo,
 	.get_msglevel		= w5100_get_msglevel,
@@ -721,6 +853,8 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops,
 	dev_set_drvdata(dev, ndev);
 	priv = netdev_priv(ndev);
 
+	mutex_init(&priv->link_lock);
+
 	switch (ops->chip_id) {
 	case W5100:
 		priv->s0_regs = W5100_S0_REGS;
@@ -745,15 +879,27 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops,
 		break;
 	default:
 		err = -EINVAL;
-		goto err_register;
+		goto err_mutex;
 	}
 
 	priv->ndev = ndev;
 	priv->ops = ops;
 	priv->irq = irq;
 
+	priv->link_irq = -EINVAL;
+	if (ops->chip_id == W5500) {
+		priv->link_irq = fwnode_irq_get(dev_fwnode(dev), 1);
+		if (priv->link_irq == -EPROBE_DEFER) {
+			err = dev_err_probe(dev, priv->link_irq,
+					    "failed to get link irq\n");
+			goto err_mutex;
+		}
+	}
+
 	ndev->netdev_ops = &w5100_netdev_ops;
-	ndev->ethtool_ops = &w5100_ethtool_ops;
+	ndev->ethtool_ops = priv->link_irq > 0 ? &w5500_ethtool_ops :
+						 &w5100_ethtool_ops;
+
 	netif_napi_add_weight(ndev, &priv->napi, w5100_napi_poll, 16);
 
 	/* This chip doesn't support VLAN packets with normal MTU,
@@ -761,15 +907,11 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops,
 	 */
 	ndev->features |= NETIF_F_VLAN_CHALLENGED;
 
-	err = register_netdev(ndev);
-	if (err < 0)
-		goto err_register;
-
 	priv->xfer_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_PERCPU, 0,
-					netdev_name(ndev));
+					dev_name(dev));
 	if (!priv->xfer_wq) {
 		err = -ENOMEM;
-		goto err_wq;
+		goto err_mutex;
 	}
 
 	INIT_WORK(&priv->rx_work, w5100_rx_work);
@@ -794,22 +936,45 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops,
 
 	if (ops->may_sleep) {
 		err = request_threaded_irq(priv->irq, NULL, w5100_interrupt,
-					   IRQF_TRIGGER_LOW | IRQF_ONESHOT,
-					   netdev_name(ndev), ndev);
+					   IRQF_TRIGGER_LOW | IRQF_ONESHOT |
+					   IRQF_NO_AUTOEN,
+					   dev_name(dev), ndev);
 	} else {
 		err = request_irq(priv->irq, w5100_interrupt,
-				  IRQF_TRIGGER_LOW, netdev_name(ndev), ndev);
+				  IRQF_TRIGGER_LOW | IRQF_NO_AUTOEN,
+				  dev_name(dev), ndev);
 	}
 	if (err)
 		goto err_hw;
 
+	if (priv->link_irq > 0) {
+		err = request_threaded_irq(priv->link_irq, NULL,
+					   w5500_detect_link_interrupt,
+					   IRQF_TRIGGER_RISING |
+					   IRQF_TRIGGER_FALLING |
+					   IRQF_ONESHOT | IRQF_NO_AUTOEN,
+					   "w5100-link", ndev);
+		if (err < 0)
+			goto err_irq;
+
+		netif_carrier_off(ndev);
+	}
+
+	err = register_netdev(ndev);
+	if (err < 0)
+		goto err_link_irq;
+
 	return 0;
 
+err_link_irq:
+	if (priv->link_irq > 0)
+		free_irq(priv->link_irq, ndev);
+err_irq:
+	free_irq(priv->irq, ndev);
 err_hw:
 	destroy_workqueue(priv->xfer_wq);
-err_wq:
-	unregister_netdev(ndev);
-err_register:
+err_mutex:
+	mutex_destroy(&priv->link_lock);
 	free_netdev(ndev);
 	return err;
 }
@@ -820,14 +985,21 @@ void w5100_remove(struct device *dev)
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct w5100_priv *priv = netdev_priv(ndev);
 
-	w5100_hw_reset(priv);
+	unregister_netdev(ndev);
+
+	if (priv->link_irq > 0)
+		free_irq(priv->link_irq, ndev);
 	free_irq(priv->irq, ndev);
 
-	flush_work(&priv->setrx_work);
-	flush_work(&priv->restart_work);
-	destroy_workqueue(priv->xfer_wq);
+	cancel_work_sync(&priv->setrx_work);
+	cancel_work_sync(&priv->restart_work);
+	flush_work(&priv->rx_work);
+	flush_work(&priv->tx_work);
 
-	unregister_netdev(ndev);
+	w5100_hw_reset(priv);
+
+	destroy_workqueue(priv->xfer_wq);
+	mutex_destroy(&priv->link_lock);
 	free_netdev(ndev);
 }
 EXPORT_SYMBOL_GPL(w5100_remove);
@@ -839,10 +1011,11 @@ static int w5100_suspend(struct device *dev)
 	struct w5100_priv *priv = netdev_priv(ndev);
 
 	if (netif_running(ndev)) {
-		netif_carrier_off(ndev);
+		mutex_lock(&priv->link_lock);
 		netif_device_detach(ndev);
+		mutex_unlock(&priv->link_lock);
 
-		w5100_hw_close(priv);
+		w5100_stop(ndev);
 	}
 	return 0;
 }
@@ -853,10 +1026,11 @@ static int w5100_resume(struct device *dev)
 	struct w5100_priv *priv = netdev_priv(ndev);
 
 	if (netif_running(ndev)) {
-		w5100_hw_reset(priv);
-		w5100_hw_start(priv);
+		w5100_open(ndev);
 
+		mutex_lock(&priv->link_lock);
 		netif_device_attach(ndev);
+		mutex_unlock(&priv->link_lock);
 	}
 	return 0;
 }
diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c
index 711ed9c2631b08..303b4da66399d9 100644
--- a/drivers/net/ethernet/xilinx/ll_temac_main.c
+++ b/drivers/net/ethernet/xilinx/ll_temac_main.c
@@ -1579,10 +1579,9 @@ static int temac_probe(struct platform_device *pdev)
 	if (temac_np) {
 		/* Retrieve the MAC address */
 		rc = of_get_mac_address(temac_np, addr);
-		if (rc) {
-			dev_err(&pdev->dev, "could not find MAC address\n");
-			return -ENODEV;
-		}
+		if (rc)
+			return dev_err_probe(&pdev->dev, rc,
+					     "could not find MAC address\n");
 		temac_init_mac_address(ndev, addr);
 	} else if (pdata) {
 		temac_init_mac_address(ndev, pdata->mac_addr);
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
index 1722b7038f3400..782f903d318f33 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
@@ -2971,10 +2971,16 @@ static int axienet_probe(struct platform_device *pdev)
 			dev_err(&pdev->dev, "could not map DMA regs\n");
 			return PTR_ERR(lp->dma_regs);
 		}
-		if (lp->rx_irq <= 0 || lp->tx_irq <= 0) {
+		if (!lp->rx_irq || !lp->tx_irq) {
 			dev_err(&pdev->dev, "could not determine irqs\n");
-			return -ENOMEM;
+			return -EINVAL;
 		}
+		if (lp->rx_irq < 0)
+			return lp->rx_irq;
+		if (lp->tx_irq < 0)
+			return lp->tx_irq;
+		if (lp->eth_irq < 0 && lp->eth_irq != -ENXIO)
+			return lp->eth_irq;
 
 		/* Reset core now that clocks are enabled, prior to accessing MDIO */
 		ret = __axienet_device_reset(lp);
@@ -3050,7 +3056,7 @@ static int axienet_probe(struct platform_device *pdev)
 		ndev->ethtool_ops = &axienet_ethtool_ops;
 	}
 	/* Check for Ethernet core IRQ (optional) */
-	if (lp->eth_irq <= 0)
+	if (lp->eth_irq < 0)
 		dev_info(&pdev->dev, "Ethernet core IRQ not defined\n");
 
 	/* Retrieve the MAC address */
diff --git a/drivers/net/ethernet/xscale/ptp_ixp46x.c b/drivers/net/ethernet/xscale/ptp_ixp46x.c
index 558c4f8d23f7a7..c805d1df5d4e06 100644
--- a/drivers/net/ethernet/xscale/ptp_ixp46x.c
+++ b/drivers/net/ethernet/xscale/ptp_ixp46x.c
@@ -307,6 +307,7 @@ static const struct of_device_id ptp_ixp_match[] = {
 	},
 	{ },
 };
+MODULE_DEVICE_TABLE(of, ptp_ixp_match);
 
 static struct platform_driver ptp_ixp_driver = {
 	.driver = {
diff --git a/drivers/net/fddi/skfp/cfm.c b/drivers/net/fddi/skfp/cfm.c
index 4eea3408034be0..b722a195d831f2 100644
--- a/drivers/net/fddi/skfp/cfm.c
+++ b/drivers/net/fddi/skfp/cfm.c
@@ -17,7 +17,7 @@
 */
 
 /*
- *	Hardware independent state machine implemantation
+ *	Hardware independent state machine implementation
  *	The following external SMT functions are referenced :
  *
  *		queue_event()
diff --git a/drivers/net/fddi/skfp/ecm.c b/drivers/net/fddi/skfp/ecm.c
index 2f5f5f26bb4348..e418ab65cb611c 100644
--- a/drivers/net/fddi/skfp/ecm.c
+++ b/drivers/net/fddi/skfp/ecm.c
@@ -17,7 +17,7 @@
 */
 
 /*
- * Hardware independent state machine implemantation
+ * Hardware independent state machine implementation
  * The following external SMT functions are referenced :
  *
  * 		queue_event()
diff --git a/drivers/net/fddi/skfp/fplustm.c b/drivers/net/fddi/skfp/fplustm.c
index 036062376c066a..0d4201345f9bef 100644
--- a/drivers/net/fddi/skfp/fplustm.c
+++ b/drivers/net/fddi/skfp/fplustm.c
@@ -150,7 +150,7 @@ static u_long read_mdr(struct s_smc *smc, unsigned int addr)
 	CHECK_NPP() ;
 	MARR(addr) ;
 	outpw(FM_A(FM_CMDREG1),FM_IRMEMWO) ;
-	CHECK_NPP() ;	/* needed for PCI to prevent from timeing violations */
+	CHECK_NPP() ;	/* needed for PCI to prevent from timing violations */
 /*	p = MDRR() ; */	/* bad read values if the workaround */
 			/* smc->hw.mc_dummy = *((short volatile far *)(addr)))*/
 			/* is used */
@@ -572,7 +572,7 @@ Note	After any ring operational change the transmit complete
 	The operating system dependent module must enable
 	the transmit complete interrupt of a queue,
 		- when it queues the first frame,
-		  because of no transmit resources are beeing
+		  because of no transmit resources are being
 		  available and
 		- when it escapes from the function llc_restart_tx
 		  while some frames are still queued.
diff --git a/drivers/net/fddi/skfp/h/hwmtm.h b/drivers/net/fddi/skfp/h/hwmtm.h
index e97db826cdd45b..0ab49cc0a8bda7 100644
--- a/drivers/net/fddi/skfp/h/hwmtm.h
+++ b/drivers/net/fddi/skfp/h/hwmtm.h
@@ -113,7 +113,7 @@ struct hw_modul {
 	struct	s_mbuf_pool	mbuf_pool ;
 	struct	hwm_r	r ;
 
-	union s_fp_descr volatile *descr_p ; /* points to the desriptor area */
+	union s_fp_descr volatile *descr_p ; /* points to the descriptor area */
 
 	u_short pass_SMT ;		/* pass SMT frames */
 	u_short pass_NSA ;		/* pass all NSA frames */
diff --git a/drivers/net/fddi/skfp/h/sba.h b/drivers/net/fddi/skfp/h/sba.h
index f30301fcd022c0..7fe99c12e2015c 100644
--- a/drivers/net/fddi/skfp/h/sba.h
+++ b/drivers/net/fddi/skfp/h/sba.h
@@ -94,7 +94,7 @@ struct s_sba {
 	long		msg_mib_ov ;		/* Current Overhead for this Path*/
 	long		msg_category ;		/* Category of the Allocation */
 	u_long		msg_max_t_neg ;		/* longest T_Neg acceptable */
-	u_long		msg_min_seg_siz ;	/* minimum segement size */
+	u_long		msg_min_seg_siz ;	/* minimum segment size */
 	struct smt_header	*sm ;		/* points to the rec message */
 	struct fddi_addr	*msg_alloc_addr ;	/* Allocation Address */
 
diff --git a/drivers/net/fddi/skfp/pcmplc.c b/drivers/net/fddi/skfp/pcmplc.c
index 90e8df6d9a88ac..745285d4d6d2ed 100644
--- a/drivers/net/fddi/skfp/pcmplc.c
+++ b/drivers/net/fddi/skfp/pcmplc.c
@@ -16,7 +16,7 @@
 */
 
 /*
- * Hardware independent state machine implemantation
+ * Hardware independent state machine implementation
  * The following external SMT functions are referenced :
  *
  * 		queue_event()
diff --git a/drivers/net/fddi/skfp/rmt.c b/drivers/net/fddi/skfp/rmt.c
index 314623650a84ba..5708124f05d2b4 100644
--- a/drivers/net/fddi/skfp/rmt.c
+++ b/drivers/net/fddi/skfp/rmt.c
@@ -16,7 +16,7 @@
 */
 
 /*
- * Hardware independent state machine implemantation
+ * Hardware independent state machine implementation
  * The following external SMT functions are referenced :
  *
  * 		queue_event()
diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c
index 69fe5717846b5b..4aff23bcfdd1a5 100644
--- a/drivers/net/gtp.c
+++ b/drivers/net/gtp.c
@@ -770,7 +770,7 @@ static int gtp_parse_exthdrs(struct sk_buff *skb, unsigned int *hdrlen)
 	 *
 	 * This length field includes length field size itself (1 byte),
 	 * payload (variable length) and next type (1 byte). The extension
-	 * header is aligned to to 4 bytes.
+	 * header is aligned to 4 bytes.
 	 */
 
 	do {
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index 5cd084e5696cd1..fc6548b16c91ea 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -625,10 +625,7 @@ static int negotiate_nvsp_ver(struct hv_device *device,
 	init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1;
 
 	if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) {
-		if (hv_is_isolation_supported())
-			netdev_info(ndev, "SR-IOV not advertised by guests on the host supporting isolation\n");
-		else
-			init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1;
+		init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1;
 
 		/* Teaming bit is needed to receive link speed updates */
 		init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1;
@@ -1665,10 +1662,7 @@ static void netvsc_receive_inband(struct net_device *ndev,
 		break;
 
 	case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION:
-		if (hv_is_isolation_supported())
-			netdev_err(ndev, "Ignore VF_ASSOCIATION msg from the host supporting isolation\n");
-		else
-			netvsc_send_vf(ndev, nvmsg, msglen);
+		netvsc_send_vf(ndev, nvmsg, msglen);
 		break;
 	}
 }
diff --git a/drivers/net/ieee802154/at86rf230.c b/drivers/net/ieee802154/at86rf230.c
index 6156cf3799164d..f1adca942e3dc1 100644
--- a/drivers/net/ieee802154/at86rf230.c
+++ b/drivers/net/ieee802154/at86rf230.c
@@ -1370,7 +1370,7 @@ static int at86rf230_hw_init(struct at86rf230_local *lp, u8 xtal_trim)
 	 *	  but this is different on each board setup. You need to fine
 	 *	  tuning this value via CTRIM.
 	 * CTRIM = variable capacitor setting. Resolution is 0.3 pF range is
-	 *	   0 pF upto 4.5 pF.
+	 *	   0 pF up to 4.5 pF.
 	 *
 	 * Examples:
 	 * atben transceiver:
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index 6f9f3aceffaad3..af8c23a322b8a5 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -19,6 +19,7 @@
 #include <net/gro_cells.h>
 #include <net/macsec.h>
 #include <net/dst_metadata.h>
+#include <net/rtnetlink.h>
 #include <net/netdev_lock.h>
 #include <linux/phy.h>
 #include <linux/byteorder/generic.h>
@@ -3539,6 +3540,22 @@ static int macsec_dev_init(struct net_device *dev)
 	if (err)
 		return err;
 
+	err = -ENOMEM;
+	macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats);
+	if (!macsec->stats)
+		goto destroy_gro_cells;
+
+	macsec->secy.tx_sc.stats =
+		netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats);
+	if (!macsec->secy.tx_sc.stats)
+		goto free_secy_stats;
+
+	macsec->secy.tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC,
+						       GFP_KERNEL);
+	if (!macsec->secy.tx_sc.md_dst)
+		goto free_tx_sc_stats;
+	macsec->secy.tx_sc.md_dst->u.macsec_info.sci = macsec->secy.sci;
+
 	macsec_inherit_tso_max(dev);
 
 	dev->hw_features = real_dev->hw_features & MACSEC_OFFLOAD_FEATURES;
@@ -3551,8 +3568,6 @@ static int macsec_dev_init(struct net_device *dev)
 
 	macsec_set_head_tail_room(dev);
 
-	if (is_zero_ether_addr(dev->dev_addr))
-		eth_hw_addr_inherit(dev, real_dev);
 	if (is_zero_ether_addr(dev->broadcast))
 		memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len);
 
@@ -3560,6 +3575,14 @@ static int macsec_dev_init(struct net_device *dev)
 	netdev_hold(real_dev, &macsec->dev_tracker, GFP_KERNEL);
 
 	return 0;
+
+free_tx_sc_stats:
+	free_percpu(macsec->secy.tx_sc.stats);
+free_secy_stats:
+	free_percpu(macsec->stats);
+destroy_gro_cells:
+	gro_cells_destroy(&macsec->gro_cells);
+	return err;
 }
 
 static void macsec_dev_uninit(struct net_device *dev)
@@ -3945,6 +3968,33 @@ static int macsec_changelink_common(struct net_device *dev,
 	return 0;
 }
 
+/* Both a change of the offload state (mdo_add_secy()/mdo_del_secy()) and
+ * any request against an already offloaded device (mdo_upd_secy()) reach
+ * the driver of real_dev, which may live in a different network namespace
+ * from the macsec device. Require CAP_NET_ADMIN over real_dev's namespace
+ * for those; local attributes of a non-offloaded device are unaffected.
+ */
+static int macsec_changelink_check_netns(struct net_device *dev,
+					 struct nlattr *data[],
+					 struct netlink_ext_ack *extack)
+{
+	struct macsec_dev *macsec = macsec_priv(dev);
+	bool reaches_real_dev;
+
+	reaches_real_dev = macsec_is_offloaded(macsec);
+	if (data[IFLA_MACSEC_OFFLOAD] &&
+	    nla_get_u8(data[IFLA_MACSEC_OFFLOAD]) != macsec->offload)
+		reaches_real_dev = true;
+
+	if (!reaches_real_dev ||
+	    rtnl_dev_link_net_capable(dev, dev_net(macsec->real_dev)))
+		return 0;
+
+	NL_SET_ERR_MSG(extack,
+		       "Changing a MACsec device whose real device is in another network namespace requires CAP_NET_ADMIN in that namespace");
+	return -EPERM;
+}
+
 static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
 			     struct nlattr *data[],
 			     struct netlink_ext_ack *extack)
@@ -3959,6 +4009,10 @@ static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (!data)
 		return 0;
 
+	ret = macsec_changelink_check_netns(dev, data, extack);
+	if (ret)
+		return ret;
+
 	if (data[IFLA_MACSEC_CIPHER_SUITE] ||
 	    data[IFLA_MACSEC_ICV_LEN] ||
 	    data[IFLA_MACSEC_SCI] ||
@@ -4116,26 +4170,11 @@ static sci_t dev_to_sci(struct net_device *dev, __be16 port)
 	return make_sci(dev->dev_addr, port);
 }
 
-static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
+static void macsec_init_secy(struct net_device *dev, sci_t sci, u8 icv_len)
 {
 	struct macsec_dev *macsec = macsec_priv(dev);
 	struct macsec_secy *secy = &macsec->secy;
 
-	macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats);
-	if (!macsec->stats)
-		return -ENOMEM;
-
-	secy->tx_sc.stats = netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats);
-	if (!secy->tx_sc.stats)
-		return -ENOMEM;
-
-	secy->tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL);
-	if (!secy->tx_sc.md_dst)
-		/* macsec and secy percpu stats will be freed when unregistering
-		 * net_device in macsec_free_netdev()
-		 */
-		return -ENOMEM;
-
 	if (sci == MACSEC_UNDEF_SCI)
 		sci = dev_to_sci(dev, MACSEC_PORT_ES);
 
@@ -4149,15 +4188,12 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
 	secy->xpn = DEFAULT_XPN;
 
 	secy->sci = sci;
-	secy->tx_sc.md_dst->u.macsec_info.sci = sci;
 	secy->tx_sc.active = true;
 	secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA;
 	secy->tx_sc.encrypt = DEFAULT_ENCRYPT;
 	secy->tx_sc.send_sci = DEFAULT_SEND_SCI;
 	secy->tx_sc.end_station = false;
 	secy->tx_sc.scb = false;
-
-	return 0;
 }
 
 static struct lock_class_key macsec_netdev_addr_lock_key;
@@ -4220,6 +4256,24 @@ static int macsec_newlink(struct net_device *dev,
 	if (rx_handler && rx_handler != macsec_handle_frame)
 		return -EBUSY;
 
+	if (is_zero_ether_addr(dev->dev_addr))
+		eth_hw_addr_inherit(dev, real_dev);
+
+	if (data && data[IFLA_MACSEC_SCI])
+		sci = nla_get_sci(data[IFLA_MACSEC_SCI]);
+	else if (data && data[IFLA_MACSEC_PORT])
+		sci = dev_to_sci(dev, nla_get_be16(data[IFLA_MACSEC_PORT]));
+	else
+		sci = dev_to_sci(dev, MACSEC_PORT_ES);
+
+	/* Registration can notify listeners before returning. */
+	macsec_init_secy(dev, sci, icv_len);
+	if (data) {
+		err = macsec_changelink_common(dev, data);
+		if (err)
+			return err;
+	}
+
 	err = register_netdevice(dev);
 	if (err < 0)
 		return err;
@@ -4232,31 +4286,11 @@ static int macsec_newlink(struct net_device *dev,
 	if (err < 0)
 		goto unregister;
 
-	/* need to be already registered so that ->init has run and
-	 * the MAC addr is set
-	 */
-	if (data && data[IFLA_MACSEC_SCI])
-		sci = nla_get_sci(data[IFLA_MACSEC_SCI]);
-	else if (data && data[IFLA_MACSEC_PORT])
-		sci = dev_to_sci(dev, nla_get_be16(data[IFLA_MACSEC_PORT]));
-	else
-		sci = dev_to_sci(dev, MACSEC_PORT_ES);
-
 	if (rx_handler && sci_exists(real_dev, sci)) {
 		err = -EBUSY;
 		goto unlink;
 	}
 
-	err = macsec_add_dev(dev, sci, icv_len);
-	if (err)
-		goto unlink;
-
-	if (data) {
-		err = macsec_changelink_common(dev, data);
-		if (err)
-			goto del_dev;
-	}
-
 	/* If h/w offloading is available, propagate to the device */
 	if (macsec_is_offloaded(macsec)) {
 		const struct macsec_ops *ops;
diff --git a/drivers/net/mdio/Kconfig b/drivers/net/mdio/Kconfig
index a05229838cb459..d44278f26fabeb 100644
--- a/drivers/net/mdio/Kconfig
+++ b/drivers/net/mdio/Kconfig
@@ -172,11 +172,11 @@ config MDIO_IPQ8064
 	  interface units of the IPQ8064 SoC
 
 config MDIO_REALTEK_RTL9300
-	tristate "Realtek RTL9300 MDIO interface support"
+	tristate "Realtek RTL83xx/RTL93xx MDIO interface support"
 	depends on MACH_REALTEK_RTL || COMPILE_TEST
 	help
 	  This driver supports the MDIO interface found in the Realtek
-	  RTL9300 family of Ethernet switches with integrated SoC.
+	  RTL83xx/RTL93xx family of Ethernet switches with integrated SoC.
 
 config MDIO_REGMAP
 	tristate
diff --git a/drivers/net/mdio/mdio-bcm-iproc.c b/drivers/net/mdio/mdio-bcm-iproc.c
index 91690b496793c0..22473c988e3b10 100644
--- a/drivers/net/mdio/mdio-bcm-iproc.c
+++ b/drivers/net/mdio/mdio-bcm-iproc.c
@@ -195,7 +195,7 @@ static const struct dev_pm_ops iproc_mdio_pm_ops = {
 
 static const struct of_device_id iproc_mdio_of_match[] = {
 	{ .compatible = "brcm,iproc-mdio", },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, iproc_mdio_of_match);
 
diff --git a/drivers/net/mdio/mdio-bcm-unimac.c b/drivers/net/mdio/mdio-bcm-unimac.c
index 37e35f282d9ad8..31e396cc9fbf2a 100644
--- a/drivers/net/mdio/mdio-bcm-unimac.c
+++ b/drivers/net/mdio/mdio-bcm-unimac.c
@@ -345,7 +345,7 @@ static const struct of_device_id unimac_mdio_ids[] = {
 	{ .compatible = "brcm,genet-mdio-v2", },
 	{ .compatible = "brcm,genet-mdio-v1", },
 	{ .compatible = "brcm,unimac-mdio", },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, unimac_mdio_ids);
 
diff --git a/drivers/net/mdio/mdio-mux-meson-g12a.c b/drivers/net/mdio/mdio-mux-meson-g12a.c
index 08d6a6c93fb8bf..e6c9ef3641afa2 100644
--- a/drivers/net/mdio/mdio-mux-meson-g12a.c
+++ b/drivers/net/mdio/mdio-mux-meson-g12a.c
@@ -223,7 +223,7 @@ static int g12a_ephy_glue_clk_register(struct device *dev)
 {
 	struct g12a_mdio_mux *priv = dev_get_drvdata(dev);
 	const char *parent_names[PLL_MUX_NUM_PARENT];
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	struct g12a_ephy_pll *pll;
 	struct clk_mux *mux;
 	struct clk *clk;
diff --git a/drivers/net/mdio/mdio-realtek-rtl9300.c b/drivers/net/mdio/mdio-realtek-rtl9300.c
index afd52a1cd7f831..a6daf4a5a66459 100644
--- a/drivers/net/mdio/mdio-realtek-rtl9300.c
+++ b/drivers/net/mdio/mdio-realtek-rtl9300.c
@@ -35,11 +35,87 @@
  *
  * The driver works out the mapping based on the MDIO bus described in device tree and phandles on
  * the ethernet-ports property.
+ *
+ * The devices have a hardware polling unit that runs in the background without any CPU load. It
+ * constantly scans the MDIO bus and the attached PHYs and updates the MAC status registers.
+ *
+ * How does the polling work?
+ *
+ * Each device has a SMI_POLL_CTRL register. A per-port bitmask decides if the hardware polling of
+ * the associated bus/address is active or not. The hardware runs a tight loop over this and for
+ * each set polling bit it issues a status check for the PHY. Attaching a logic analyzer to the
+ * MDIO bus of an RTL8380 and RTL8393 gives the following commands (in kernel notation):
+ *
+ *	RTL8380				RTL8393
+ *	---------------------------	---------------------------
+ *	phy_write(phy, 31, 0x0);	phy_read(phy, 0);
+ *	phy_write(phy, 13, 0x7);	phy_read(phy, 1);
+ *	phy_write(phy, 14, 0x3c);	phy_read(phy, 4);
+ *	phy_write(phy, 13, 0x8007);	phy_read(phy, 5);
+ *	phy_read(phy, 14);		phy_read(phy, 6);
+ *	phy_write(phy, 13, 0x7);	phy_read(phy, 9);
+ *	phy_write(phy, 14, 0x3d);	phy_read(phy, 10);
+ *	phy_write(phy, 13, 0x8007);	phy_read(phy, 15);
+ *	phy_read(phy, 14);		phy_write(phy, 13, 0x7);
+ *	phy_read(phy, 9);		phy_write(phy, 14, 0x3c);
+ *	phy_read(phy, 10);		phy_write(phy, 13, 0x4007);
+ *	phy_read(phy, 15);		phy_read(phy, 14);
+ *	phy_read(phy, 0);		phy_write(phy, 13, 0x7);
+ *	phy_read(phy, 1);		phy_write(phy, 14, 0x3d);
+ *	phy_read(phy, 4);		phy_write(phy, 13, 0x4007);
+ *	phy_read(phy, 5);		phy_read(phy, 14);
+ *	phy_read(phy, 6);
+ *
+ * After one PHY status is read, the polling engine goes over to the next PHY. If one bus is fully
+ * scanned it switches over to the next bus. Basically the polling system is always busy and the
+ * MAC state is updated in real-time.
+ *
+ * This is a glimpse look at the complexity of the polling and leaves out the C45 case and the MAC
+ * register update logic afterwards. Important to note:
+ *
+ * - 100 MBit downshifts (broken cables) are identified and propagated to the MAC correctly
+ * - Access to MDIO_AN_EEE_ADV and MDIO_AN_EEE_LPABLE works via C45 over C22.
+ * - It is unclear if these sequences change for different PHYs.
+ * - Access to Realtek reserved register 26 (link speed) has not yet been seen.
+ *
+ * How does MDIO access from kernel work?
+ *
+ * When issuing MDIO accesses via an MMIO based interface the final write to the command register
+ * sets a "run command now" bit. Between two polling sequences for different PHYs the hardware
+ * checks if a user command needs to run and sends it onto the bus. Afterwards it simply continues
+ * its polling work. Inspecting the command sequence for a paged write on the logic analyzer gives:
+ *
+ *	RTL8380				RTL8393
+ *	---------------------------	---------------------------
+ *	phy_write(phy, 31, page);	phy_write(phy, 31, page);
+ *	phy_write(phy, reg, value);	phy_write(phy, reg, value);
+ *					phy_write(phy, 31, 0);
+ *
+ * What does this mean?
+ *
+ * There are slight differences in polling and PHY access between the models but the challenge
+ * stays the same. On the one hand that greatly simplifies the MAC layer, on the other hand it
+ * has some implications for the kernel PHY subsystem.
+ *
+ * - Without the polling and a proper MAC status, some of the link handling features do not work.
+ *   Especially an unpopulated MAC_LINK_STS register cancels operations to other MAC registers.
+ * - The Realtek page register 31 is magically modified in the background so that polling will
+ *   read the right data. On the RTL838x polling simply resets it to zero. Other devices seem
+ *   to track the page access "magically" in the background.
+ * - A C45 over C22 kernel access sequence is most likely to fail because chances are high that
+ *   the polling engine overwrites registers 13/14 in between.
+ * - PHY firmware loading can have issues. Especially if a PHY is designed to expect a clean
+ *   sequence of registers and values without deviation.
+ * - An access to one PHY will need to wait for the next free slot of the polling engine.
+ *
+ * Conclusion: The Realtek MDIO bus driver PHY access must know and handle any interference that
+ * arises from the above described hardware polling.
  */
 
 #include <linux/bitfield.h>
 #include <linux/bitmap.h>
 #include <linux/bits.h>
+#include <linux/device.h>
 #include <linux/find.h>
 #include <linux/mdio.h>
 #include <linux/mfd/syscon.h>
@@ -50,6 +126,50 @@
 #include <linux/property.h>
 #include <linux/regmap.h>
 
+#define RTL8380_NUM_BUSES			1
+#define RTL8380_NUM_PAGES			4096
+#define RTL8380_NUM_PORTS			28
+#define RTL8380_SMI_GLB_CTRL			0xa100
+#define   RTL8380_SMI_PHY_PATCH_DONE		BIT(15)
+#define RTL8380_SMI_ACCESS_PHY_CTRL_0		0xa1b8
+#define RTL8380_SMI_ACCESS_PHY_CTRL_1		0xa1bc
+#define   RTL8380_PHY_CTRL_REG_ADDR		GENMASK(24, 20)
+#define   RTL8380_PHY_CTRL_PARK_PAGE		GENMASK(19, 15)
+#define   RTL8380_PHY_CTRL_MAIN_PAGE		GENMASK(14, 3)
+#define   RTL8380_PHY_CTRL_WRITE		BIT(2)
+#define   RTL8380_PHY_CTRL_READ			0
+#define   RTL8380_PHY_CTRL_TYPE_C45		BIT(1)
+#define   RTL8380_PHY_CTRL_TYPE_C22		0
+#define   RTL8380_PHY_CTRL_FAIL			0 /* no fail indicator */
+#define RTL8380_SMI_ACCESS_PHY_CTRL_2		0xa1c0
+#define   RTL8380_PHY_CTRL_INDATA		GENMASK(31, 16)
+#define   RTL8380_PHY_CTRL_DATA			GENMASK(15, 0)
+#define RTL8380_SMI_ACCESS_PHY_CTRL_3		0xa1c4
+#define RTL8380_SMI_POLL_CTRL			0xa17c
+#define RTL8380_SMI_PORT0_5_ADDR_CTRL		0xa1c8
+
+#define RTL8390_NUM_BUSES			2
+#define RTL8390_NUM_PAGES			8192
+#define RTL8390_NUM_PORTS			52
+#define RTL8390_BCAST_PHYID_CTRL		0x03ec
+#define RTL8390_PHYREG_ACCESS_CTRL		0x03dc
+#define   RTL8390_PHY_CTRL_REG_ADDR		GENMASK(9, 5)
+#define   RTL8390_PHY_CTRL_MAIN_PAGE		GENMASK(22, 10)
+#define   RTL8390_PHY_CTRL_FAIL			BIT(1)
+#define   RTL8390_PHY_CTRL_WRITE		BIT(3)
+#define   RTL8390_PHY_CTRL_READ			0
+#define   RTL8390_PHY_CTRL_TYPE_C45		BIT(2)
+#define   RTL8390_PHY_CTRL_TYPE_C22		0
+#define RTL8390_PHYREG_CTRL			0x03e0
+#define   RTL8390_PHY_CTRL_EXT_PAGE		GENMASK(8, 0)
+#define RTL8390_PHYREG_DATA_CTRL		0x03f0
+#define   RTL8390_PHY_CTRL_INDATA		GENMASK(31, 16)
+#define   RTL8390_PHY_CTRL_DATA			GENMASK(15, 0)
+#define RTL8390_PHYREG_MMD_CTRL			0x03f4
+#define RTL8390_PHYREG_PORT_CTRL_LOW		0x03e4
+#define RTL8390_PHYREG_PORT_CTRL_HIGH		0x03e8
+#define RTL8390_SMI_PORT_POLLING_CTRL		0x03fc
+
 #define RTL9300_NUM_BUSES			4
 #define RTL9300_NUM_PAGES			4096
 #define RTL9300_NUM_PORTS			28
@@ -70,6 +190,7 @@
 #define   RTL9300_PHY_CTRL_INDATA		GENMASK(31, 16)
 #define   RTL9300_PHY_CTRL_DATA			GENMASK(15, 0)
 #define RTL9300_SMI_ACCESS_PHY_CTRL_3		0xcb7c
+#define RTL9300_SMI_POLL_CTRL			0xca90
 #define RTL9300_SMI_PORT0_5_ADDR_CTRL		0xcb80
 
 #define RTL9310_NUM_BUSES			4
@@ -88,6 +209,8 @@
 #define RTL9310_SMI_INDRT_ACCESS_BC_PHYID_CTRL	0x0c14
 #define   RTL9310_BC_PORT_ID			GENMASK(10, 5)
 #define RTL9310_SMI_INDRT_ACCESS_CTRL_1		0x0c04
+#define   RTL9310_SMI_INDRT_EXT_PAGE		GENMASK(8, 0)
+#define   RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE	0x1ff
 #define RTL9310_SMI_INDRT_ACCESS_CTRL_2_LOW	0x0c08
 #define RTL9310_SMI_INDRT_ACCESS_CTRL_2_HIGH	0x0c0c
 #define RTL9310_SMI_INDRT_ACCESS_CTRL_3		0x0c10 /* I/O fields flipped */
@@ -95,12 +218,16 @@
 #define   RTL9310_PHY_CTRL_INDATA		GENMASK(15, 0)
 #define RTL9310_SMI_INDRT_ACCESS_MMD_CTRL	0x0c18
 #define RTL9310_SMI_PORT_ADDR_CTRL		0x0c74
+#define RTL9310_SMI_PORT_POLLING_CTRL		0x0ccc
 #define RTL9310_SMI_PORT_POLLING_SEL		0x0c9c
 
 #define PHY_CTRL_CMD				BIT(0)
 #define PHY_CTRL_MMD_DEVAD			GENMASK(20, 16)
 #define PHY_CTRL_MMD_REG			GENMASK(15, 0)
 
+#define RTL_VENDOR_ID				0x001cc800
+#define RTL_PAGE_SELECT				31
+
 #define MAP_ADDRS_PER_REG			6
 #define MAP_BITS_PER_ADDR			5
 #define MAP_BITS_PER_BUS			2
@@ -126,6 +253,7 @@ struct otto_emdio_priv {
 	struct regmap *regmap;
 	struct mutex lock; /* protect HW access */
 	DECLARE_BITMAP(valid_ports, MAX_PORTS);
+	u16 page[MAX_PORTS];
 	u8 smi_bus[MAX_PORTS];
 	u8 smi_addr[MAX_PORTS];
 	bool smi_bus_is_c45[MAX_SMI_BUSSES];
@@ -142,6 +270,7 @@ struct otto_emdio_info {
 	u8 num_buses;
 	u8 num_ports;
 	u16 num_pages;
+	u32 poll_ctrl;
 	int (*setup_controller)(struct otto_emdio_priv *priv);
 	int (*read_c22)(struct mii_bus *bus, int port, int regnum, u32 *value);
 	int (*read_c45)(struct mii_bus *bus, int port, int dev_addr, int regnum, u32 *value);
@@ -177,6 +306,12 @@ static struct otto_emdio_priv *otto_emdio_bus_to_priv(struct mii_bus *bus)
 	return chan->priv;
 }
 
+static int otto_emdio_set_port_polling(struct otto_emdio_priv *priv, int port, bool active)
+{
+	return regmap_assign_bits(priv->regmap, priv->info->poll_ctrl + (port / 32) * 4,
+				  BIT(port % 32), active);
+}
+
 static int otto_emdio_run_cmd(struct mii_bus *bus, u32 cmd,
 			      struct otto_emdio_cmd_regs *cmd_data)
 {
@@ -185,9 +320,9 @@ static int otto_emdio_run_cmd(struct mii_bus *bus, u32 cmd,
 	u32 cmdstate;
 	int ret;
 
-	/* Defensive pre check just in case something goes horrible wrong */
+	/* Defensive pre check just in case something goes horribly wrong */
 	ret = regmap_read_poll_timeout(priv->regmap, info->cmd_regs.c22_data,
-				       cmdstate, !(cmdstate & PHY_CTRL_CMD), 10, 1000);
+				       cmdstate, !(cmdstate & PHY_CTRL_CMD), 10, 10000);
 	if (ret)
 		return ret;
 
@@ -227,7 +362,7 @@ static int otto_emdio_run_cmd(struct mii_bus *bus, u32 cmd,
 		return ret;
 
 	ret = regmap_read_poll_timeout(priv->regmap, info->cmd_regs.c22_data,
-				       cmdstate, !(cmdstate & PHY_CTRL_CMD), 10, 1000);
+				       cmdstate, !(cmdstate & PHY_CTRL_CMD), 10, 10000);
 	if (ret)
 		return ret;
 
@@ -264,13 +399,123 @@ static int otto_emdio_write_cmd(struct mii_bus *bus, u32 cmd,
 	return otto_emdio_run_cmd(bus, cmd | priv->info->cmd_write, cmd_data);
 }
 
+static int otto_emdio_8380_read_c22(struct mii_bus *bus, int port, int regnum, u32 *value)
+{
+	struct otto_emdio_priv *priv = otto_emdio_bus_to_priv(bus);
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c22_data	= FIELD_PREP(RTL8380_PHY_CTRL_REG_ADDR, regnum) |
+				  FIELD_PREP(RTL8380_PHY_CTRL_PARK_PAGE, 0x1f) |
+				  FIELD_PREP(RTL8380_PHY_CTRL_MAIN_PAGE, priv->page[port]),
+		.io_data	= FIELD_PREP(RTL8380_PHY_CTRL_INDATA, port),
+	};
+
+	return otto_emdio_read_cmd(bus, RTL8380_PHY_CTRL_TYPE_C22, &cmd_data,
+				   RTL8380_PHY_CTRL_DATA, value);
+}
+
+static int otto_emdio_8380_write_c22(struct mii_bus *bus, int port, int regnum, u16 value)
+{
+	struct otto_emdio_priv *priv = otto_emdio_bus_to_priv(bus);
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c22_data	= FIELD_PREP(RTL8380_PHY_CTRL_REG_ADDR, regnum) |
+				  FIELD_PREP(RTL8380_PHY_CTRL_PARK_PAGE, 0x1f) |
+				  FIELD_PREP(RTL8380_PHY_CTRL_MAIN_PAGE, priv->page[port]),
+		.io_data	= FIELD_PREP(RTL8380_PHY_CTRL_INDATA, value),
+		.port_mask_low	= BIT(port),
+	};
+
+	return otto_emdio_write_cmd(bus, RTL8380_PHY_CTRL_TYPE_C22, &cmd_data);
+}
+
+static int otto_emdio_8380_read_c45(struct mii_bus *bus, int port,
+				    int dev_addr, int regnum, u32 *value)
+{
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c45_data	= FIELD_PREP(PHY_CTRL_MMD_DEVAD, dev_addr) |
+				  FIELD_PREP(PHY_CTRL_MMD_REG, regnum),
+		.io_data	= FIELD_PREP(RTL8380_PHY_CTRL_INDATA, port),
+	};
+
+	return otto_emdio_read_cmd(bus, RTL8380_PHY_CTRL_TYPE_C45, &cmd_data,
+				   RTL8380_PHY_CTRL_DATA, value);
+}
+
+static int otto_emdio_8380_write_c45(struct mii_bus *bus, int port,
+				     int dev_addr, int regnum, u16 value)
+{
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c45_data	= FIELD_PREP(PHY_CTRL_MMD_DEVAD, dev_addr) |
+				  FIELD_PREP(PHY_CTRL_MMD_REG, regnum),
+		.io_data	= FIELD_PREP(RTL8380_PHY_CTRL_INDATA, value),
+		.port_mask_low	= BIT(port),
+	};
+
+	return otto_emdio_write_cmd(bus, RTL8380_PHY_CTRL_TYPE_C45, &cmd_data);
+}
+
+static int otto_emdio_8390_read_c22(struct mii_bus *bus, int port, int regnum, u32 *value)
+{
+	struct otto_emdio_priv *priv = otto_emdio_bus_to_priv(bus);
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c22_data	= FIELD_PREP(RTL8390_PHY_CTRL_REG_ADDR, regnum) |
+				  FIELD_PREP(RTL8390_PHY_CTRL_MAIN_PAGE, priv->page[port]),
+		.ext_page	= FIELD_PREP(RTL8390_PHY_CTRL_EXT_PAGE, 0x1ff),
+		.io_data	= FIELD_PREP(RTL8390_PHY_CTRL_INDATA, port),
+	};
+
+	return otto_emdio_read_cmd(bus, RTL8390_PHY_CTRL_TYPE_C22, &cmd_data,
+				   RTL8390_PHY_CTRL_DATA, value);
+}
+
+static int otto_emdio_8390_write_c22(struct mii_bus *bus, int port, int regnum, u16 value)
+{
+	struct otto_emdio_priv *priv = otto_emdio_bus_to_priv(bus);
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c22_data	= FIELD_PREP(RTL8390_PHY_CTRL_REG_ADDR, regnum) |
+				  FIELD_PREP(RTL8390_PHY_CTRL_MAIN_PAGE, priv->page[port]),
+		.ext_page	= FIELD_PREP(RTL8390_PHY_CTRL_EXT_PAGE, 0x1ff),
+		.io_data	= FIELD_PREP(RTL8390_PHY_CTRL_INDATA, value),
+		.port_mask_high	= (u32)(BIT_ULL(port) >> 32),
+		.port_mask_low	= (u32)(BIT_ULL(port)),
+	};
+
+	return otto_emdio_write_cmd(bus, RTL8390_PHY_CTRL_TYPE_C22, &cmd_data);
+}
+
+static int otto_emdio_8390_read_c45(struct mii_bus *bus, int port,
+				    int dev_addr, int regnum, u32 *value)
+{
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c45_data	= FIELD_PREP(PHY_CTRL_MMD_DEVAD, dev_addr) |
+				  FIELD_PREP(PHY_CTRL_MMD_REG, regnum),
+		.io_data	= FIELD_PREP(RTL8390_PHY_CTRL_INDATA, port),
+	};
+
+	return otto_emdio_read_cmd(bus, RTL8390_PHY_CTRL_TYPE_C45, &cmd_data,
+				   RTL8390_PHY_CTRL_DATA, value);
+}
+
+static int otto_emdio_8390_write_c45(struct mii_bus *bus, int port,
+				     int dev_addr, int regnum, u16 value)
+{
+	struct otto_emdio_cmd_regs cmd_data = {
+		.c45_data	= FIELD_PREP(PHY_CTRL_MMD_DEVAD, dev_addr) |
+				  FIELD_PREP(PHY_CTRL_MMD_REG, regnum),
+		.io_data	= FIELD_PREP(RTL8390_PHY_CTRL_INDATA, value),
+		.port_mask_high	= (u32)(BIT_ULL(port) >> 32),
+		.port_mask_low	= (u32)(BIT_ULL(port)),
+	};
+
+	return otto_emdio_write_cmd(bus, RTL8390_PHY_CTRL_TYPE_C45, &cmd_data);
+}
+
 static int otto_emdio_9300_read_c22(struct mii_bus *bus, int port, int regnum, u32 *value)
 {
 	struct otto_emdio_priv *priv = otto_emdio_bus_to_priv(bus);
 	struct otto_emdio_cmd_regs cmd_data = {
 		.c22_data	= FIELD_PREP(RTL9300_PHY_CTRL_REG_ADDR, regnum) |
 				  FIELD_PREP(RTL9300_PHY_CTRL_PARK_PAGE, 0x1f) |
-				  FIELD_PREP(RTL9300_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)),
+				  FIELD_PREP(RTL9300_PHY_CTRL_MAIN_PAGE, priv->page[port]),
 		.io_data	= FIELD_PREP(RTL9300_PHY_CTRL_INDATA, port),
 	};
 
@@ -284,7 +529,7 @@ static int otto_emdio_9300_write_c22(struct mii_bus *bus, int port, int regnum,
 	struct otto_emdio_cmd_regs cmd_data = {
 		.c22_data	= FIELD_PREP(RTL9300_PHY_CTRL_REG_ADDR, regnum) |
 				  FIELD_PREP(RTL9300_PHY_CTRL_PARK_PAGE, 0x1f) |
-				  FIELD_PREP(RTL9300_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)),
+				  FIELD_PREP(RTL9300_PHY_CTRL_MAIN_PAGE, priv->page[port]),
 		.io_data	= FIELD_PREP(RTL9300_PHY_CTRL_INDATA, value),
 		.port_mask_low	= BIT(port),
 	};
@@ -324,7 +569,9 @@ static int otto_emdio_9310_read_c22(struct mii_bus *bus, int port, int regnum, u
 	struct otto_emdio_cmd_regs cmd_data = {
 		.broadcast	= FIELD_PREP(RTL9310_BC_PORT_ID, port),
 		.c22_data	= FIELD_PREP(RTL9310_PHY_CTRL_REG_ADDR, regnum) |
-				  FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)),
+				  FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, priv->page[port]),
+		.ext_page	= FIELD_PREP(RTL9310_SMI_INDRT_EXT_PAGE,
+					     RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE),
 	};
 
 	return otto_emdio_read_cmd(bus, RTL9310_PHY_CTRL_TYPE_C22, &cmd_data,
@@ -336,7 +583,9 @@ static int otto_emdio_9310_write_c22(struct mii_bus *bus, int port, int regnum,
 	struct otto_emdio_priv *priv = otto_emdio_bus_to_priv(bus);
 	struct otto_emdio_cmd_regs cmd_data = {
 		.c22_data	= FIELD_PREP(RTL9310_PHY_CTRL_REG_ADDR, regnum) |
-				  FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)),
+				  FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, priv->page[port]),
+		.ext_page	= FIELD_PREP(RTL9310_SMI_INDRT_EXT_PAGE,
+					     RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE),
 		.io_data	= FIELD_PREP(RTL9310_PHY_CTRL_INDATA, value),
 		.port_mask_high	= (u32)(BIT_ULL(port) >> 32),
 		.port_mask_low	= (u32)(BIT_ULL(port)),
@@ -378,27 +627,53 @@ static int otto_emdio_read_c22(struct mii_bus *bus, int phy_id, int regnum)
 	int ret, port;
 	u32 value;
 
+	if (regnum == MII_MMD_CTRL || regnum == MII_MMD_DATA) {
+		dev_warn_once(&bus->dev,
+			      "C45 over C22 read access broken due to polling\n");
+		return -EOPNOTSUPP;
+	}
+
 	port = otto_emdio_phy_to_port(bus, phy_id);
 	if (port < 0)
 		return port;
 
-	scoped_guard(mutex, &priv->lock)
+	scoped_guard(mutex, &priv->lock) {
+		if (regnum == RTL_PAGE_SELECT)
+			return priv->page[port];
+
 		ret = priv->info->read_c22(bus, port, regnum, &value);
+	}
 
 	return ret ? ret : value;
 }
 
-static int otto_emdio_write_c22(struct mii_bus *bus, int phy_id, int regnum, u16 value)
+static int otto_emdio_write_c22(struct mii_bus *bus, int phy_id, int regnum,
+				u16 value)
 {
 	struct otto_emdio_priv *priv = otto_emdio_bus_to_priv(bus);
 	int ret, port;
 
+	if (regnum == MII_MMD_CTRL || regnum == MII_MMD_DATA) {
+		dev_warn_once(&bus->dev,
+			      "C45 over C22 write access broken due to polling\n");
+		return -EOPNOTSUPP;
+	}
+
 	port = otto_emdio_phy_to_port(bus, phy_id);
 	if (port < 0)
 		return port;
 
-	scoped_guard(mutex, &priv->lock)
+	scoped_guard(mutex, &priv->lock) {
+		if (regnum == RTL_PAGE_SELECT) {
+			if (value >= RAW_PAGE(priv))
+				return -EINVAL;
+
+			priv->page[port] = value;
+			return 0;
+		}
+
 		ret = priv->info->write_c22(bus, port, regnum, value);
+	}
 
 	return ret;
 }
@@ -471,6 +746,15 @@ static int otto_emdio_setup_topology(struct otto_emdio_priv *priv)
 	return 0;
 }
 
+static int otto_emdio_8380_setup_controller(struct otto_emdio_priv *priv)
+{
+	/*
+	 * PHY_PATCH_DONE enables PHY control via SoC. This is required for PHY access, including
+	 * patching and must be set before the PHYs are probed.
+	 */
+	return regmap_set_bits(priv->regmap, RTL8380_SMI_GLB_CTRL, RTL8380_SMI_PHY_PATCH_DONE);
+}
+
 static int otto_emdio_9300_setup_controller(struct otto_emdio_priv *priv)
 {
 	u32 glb_ctrl_mask = 0, glb_ctrl_val = 0;
@@ -507,6 +791,51 @@ static int otto_emdio_9310_setup_controller(struct otto_emdio_priv *priv)
 	return 0;
 }
 
+static int otto_emdio_notify_phy_attach(struct phy_device *phydev)
+{
+	int port = otto_emdio_phy_to_port(phydev->mdio.bus, phydev->mdio.addr);
+	struct otto_emdio_chan *chan = phydev->mdio.bus->priv;
+	struct otto_emdio_priv *priv = chan->priv;
+
+	if (port < 0) {
+		/* All subsequent bus operations will fail */
+		phydev_err(phydev, "PHY is not mapped to a valid switch port\n");
+		return port;
+	}
+
+	/* "sync" page in case of previously failed attachment */
+	scoped_guard(mutex, &priv->lock)
+		priv->page[port] = 0;
+
+	if (!priv->smi_bus_is_c45[chan->mdio_bus] &&
+	    !phy_id_compare_vendor(phydev->phy_id, RTL_VENDOR_ID)) {
+		phydev_err(phydev, "Only Realtek PHYs allowed on C22 bus\n");
+		return -EOPNOTSUPP;
+	}
+
+	return otto_emdio_set_port_polling(priv, port, true);
+}
+
+static void otto_emdio_notify_phy_detach(struct phy_device *phydev)
+{
+	struct mii_bus *bus = phydev->mdio.bus;
+	struct otto_emdio_priv *priv;
+	int port;
+
+	priv = otto_emdio_bus_to_priv(phydev->mdio.bus);
+	port = otto_emdio_phy_to_port(phydev->mdio.bus, phydev->mdio.addr);
+
+	if (port < 0)
+		return;
+
+	/* "sync" page for next attachment */
+	scoped_guard(mutex, &priv->lock)
+		priv->page[port] = 0;
+
+	if (otto_emdio_set_port_polling(priv, port, false))
+		dev_err(bus->parent, "failed to disable polling for port %d\n", port);
+}
+
 static int otto_emdio_probe_one(struct device *dev, struct otto_emdio_priv *priv,
 				 struct fwnode_handle *node)
 {
@@ -528,14 +857,14 @@ static int otto_emdio_probe_one(struct device *dev, struct otto_emdio_priv *priv
 		return -ENOMEM;
 
 	bus->name = "Realtek Switch MDIO Bus";
-	if (priv->smi_bus_is_c45[mdio_bus]) {
-		bus->read_c45 = otto_emdio_read_c45;
-		bus->write_c45 = otto_emdio_write_c45;
-	} else {
-		bus->read = otto_emdio_read_c22;
-		bus->write = otto_emdio_write_c22;
-	}
+	bus->read_c45 = otto_emdio_read_c45;
+	bus->write_c45 = otto_emdio_write_c45;
+	bus->read = otto_emdio_read_c22;
+	bus->write = otto_emdio_write_c22;
 	bus->parent = dev;
+	bus->notify_phy_attach = otto_emdio_notify_phy_attach;
+	bus->notify_phy_detach = otto_emdio_notify_phy_detach;
+
 	chan = bus->priv;
 	chan->mdio_bus = mdio_bus;
 	chan->priv = priv;
@@ -652,6 +981,19 @@ static int otto_emdio_map_ports(struct device *dev)
 	return err;
 }
 
+static int otto_emdio_init_polling(struct otto_emdio_priv *priv)
+{
+	int err;
+
+	for (int port = 0; port < priv->info->num_ports; port++) {
+		err = otto_emdio_set_port_polling(priv, port, false);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
 static int otto_emdio_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -671,6 +1013,10 @@ static int otto_emdio_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->regmap))
 		return PTR_ERR(priv->regmap);
 
+	err = otto_emdio_init_polling(priv);
+	if (err)
+		return err;
+
 	platform_set_drvdata(pdev, priv);
 
 	err = otto_emdio_map_ports(dev);
@@ -696,6 +1042,51 @@ static int otto_emdio_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct otto_emdio_info otto_emdio_8380_info = {
+	.addr_map_base = RTL8380_SMI_PORT0_5_ADDR_CTRL,
+	.cmd_fail = RTL8380_PHY_CTRL_FAIL,
+	.cmd_read = RTL8380_PHY_CTRL_READ,
+	.cmd_write = RTL8380_PHY_CTRL_WRITE,
+	.cmd_regs = {
+		.c22_data = RTL8380_SMI_ACCESS_PHY_CTRL_1,
+		.c45_data = RTL8380_SMI_ACCESS_PHY_CTRL_3,
+		.io_data = RTL8380_SMI_ACCESS_PHY_CTRL_2,
+		.port_mask_low = RTL8380_SMI_ACCESS_PHY_CTRL_0,
+	},
+	.num_buses = RTL8380_NUM_BUSES,
+	.num_pages = RTL8380_NUM_PAGES,
+	.num_ports = RTL8380_NUM_PORTS,
+	.poll_ctrl = RTL8380_SMI_POLL_CTRL,
+	.setup_controller = otto_emdio_8380_setup_controller,
+	.read_c22 = otto_emdio_8380_read_c22,
+	.read_c45 = otto_emdio_8380_read_c45,
+	.write_c22 = otto_emdio_8380_write_c22,
+	.write_c45 = otto_emdio_8380_write_c45,
+};
+
+static const struct otto_emdio_info otto_emdio_8390_info = {
+	.cmd_fail = RTL8390_PHY_CTRL_FAIL,
+	.cmd_read = RTL8390_PHY_CTRL_READ,
+	.cmd_write = RTL8390_PHY_CTRL_WRITE,
+	.cmd_regs = {
+		.broadcast = RTL8390_BCAST_PHYID_CTRL,
+		.c22_data = RTL8390_PHYREG_ACCESS_CTRL,
+		.c45_data = RTL8390_PHYREG_MMD_CTRL,
+		.ext_page = RTL8390_PHYREG_CTRL,
+		.io_data = RTL8390_PHYREG_DATA_CTRL,
+		.port_mask_low = RTL8390_PHYREG_PORT_CTRL_LOW,
+		.port_mask_high = RTL8390_PHYREG_PORT_CTRL_HIGH,
+	},
+	.num_buses = RTL8390_NUM_BUSES,
+	.num_pages = RTL8390_NUM_PAGES,
+	.num_ports = RTL8390_NUM_PORTS,
+	.poll_ctrl = RTL8390_SMI_PORT_POLLING_CTRL,
+	.read_c22 = otto_emdio_8390_read_c22,
+	.read_c45 = otto_emdio_8390_read_c45,
+	.write_c22 = otto_emdio_8390_write_c22,
+	.write_c45 = otto_emdio_8390_write_c45,
+};
+
 static const struct otto_emdio_info otto_emdio_9300_info = {
 	.addr_map_base = RTL9300_SMI_PORT0_5_ADDR_CTRL,
 	.bus_map_base = RTL9300_SMI_PORT0_15_POLLING_SEL,
@@ -711,6 +1102,7 @@ static const struct otto_emdio_info otto_emdio_9300_info = {
 	.num_buses = RTL9300_NUM_BUSES,
 	.num_ports = RTL9300_NUM_PORTS,
 	.num_pages = RTL9300_NUM_PAGES,
+	.poll_ctrl = RTL9300_SMI_POLL_CTRL,
 	.setup_controller = otto_emdio_9300_setup_controller,
 	.read_c22 = otto_emdio_9300_read_c22,
 	.read_c45 = otto_emdio_9300_read_c45,
@@ -736,6 +1128,7 @@ static const struct otto_emdio_info otto_emdio_9310_info = {
 	.num_buses = RTL9310_NUM_BUSES,
 	.num_pages = RTL9310_NUM_PAGES,
 	.num_ports = RTL9310_NUM_PORTS,
+	.poll_ctrl = RTL9310_SMI_PORT_POLLING_CTRL,
 	.setup_controller = otto_emdio_9310_setup_controller,
 	.read_c22 = otto_emdio_9310_read_c22,
 	.read_c45 = otto_emdio_9310_read_c45,
@@ -744,6 +1137,8 @@ static const struct otto_emdio_info otto_emdio_9310_info = {
 };
 
 static const struct of_device_id otto_emdio_ids[] = {
+	{ .compatible = "realtek,rtl8380-mdio", .data = &otto_emdio_8380_info },
+	{ .compatible = "realtek,rtl8391-mdio", .data = &otto_emdio_8390_info },
 	{ .compatible = "realtek,rtl9301-mdio", .data = &otto_emdio_9300_info },
 	{ .compatible = "realtek,rtl9311-mdio", .data = &otto_emdio_9310_info },
 	{}
@@ -755,10 +1150,11 @@ static struct platform_driver otto_emdio_driver = {
 	.driver = {
 		.name = "mdio-rtl9300",
 		.of_match_table = otto_emdio_ids,
+		.suppress_bind_attrs = true,
 	},
 };
 
 module_platform_driver(otto_emdio_driver);
 
-MODULE_DESCRIPTION("RTL9300 MDIO driver");
+MODULE_DESCRIPTION("RTL83xx/RTL93xx MDIO driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c
index b358e5c3673510..267254f046de9a 100644
--- a/drivers/net/netconsole.c
+++ b/drivers/net/netconsole.c
@@ -151,6 +151,15 @@ enum target_state {
 	STATE_DEACTIVATED,
 };
 
+struct inet_addr {
+	/* Address family: AF_UNSPEC when unset, else AF_INET or AF_INET6 */
+	u8			family;
+	union {
+		__be32		ip;
+		struct in6_addr	in6;
+	};
+};
+
 /**
  * struct netcons_userdata - Formatted userdata payload of a target.
  * @rcu:	Used to free the payload after a grace period.
@@ -195,7 +204,6 @@ struct netcons_userdata {
  *		local_mac	(read-only)
  * @local_ip:	Source IP address of the target (read-write).
  * @remote_ip:	Destination IP address of the target (read-write).
- * @ipv6:	Whether the target addresses are IPv6 (read-write).
  * @local_port:	Source UDP port of the target (read-write).
  * @remote_port: Destination UDP port of the target (read-write).
  * @remote_mac:	Destination ethernet address of the target (read-write).
@@ -225,8 +233,7 @@ struct netconsole_target {
 	bool			extended;
 	bool			release;
 	struct netpoll		np;
-	union inet_addr		local_ip, remote_ip;
-	bool			ipv6;
+	struct inet_addr	local_ip, remote_ip;
 	u16			local_port, remote_port;
 	u8			remote_mac[ETH_ALEN];
 	/* protected by target_list_lock; +1 gives scnprintf() room for its
@@ -428,6 +435,7 @@ static int netcons_take_ipv6(struct netconsole_target *nt,
 				continue;
 			/* Got the IP, let's return */
 			nt->local_ip.in6 = ifp->addr;
+			nt->local_ip.family = AF_INET6;
 			err = 0;
 			break;
 		}
@@ -469,28 +477,12 @@ static int netcons_take_ipv4(struct netconsole_target *nt,
 	}
 
 	nt->local_ip.ip = ifa->ifa_local;
+	nt->local_ip.family = AF_INET;
 	np_info(np, "local IP %pI4\n", &nt->local_ip.ip);
 
 	return 0;
 }
 
-/*
- * Test whether the caller left nt->local_ip unset, so that
- * netcons_netpoll_setup() should auto-populate it from the egress device.
- *
- * nt->local_ip is a union of __be32 (IPv4) and struct in6_addr (IPv6),
- * so an IPv6 address whose first 4 bytes are zero (e.g. ::1, ::2,
- * IPv4-mapped ::ffff:a.b.c.d) must not be tested via the IPv4 arm —
- * doing so would misclassify a caller-supplied address as unset and
- * silently overwrite it with whatever address the device exposes.
- */
-static bool netcons_local_ip_unset(const struct netconsole_target *nt)
-{
-	if (nt->ipv6)
-		return ipv6_addr_any(&nt->local_ip.in6);
-	return !nt->local_ip.ip;
-}
-
 static int netcons_netpoll_setup(struct netconsole_target *nt)
 {
 	struct net *net = current->nsproxy->net_ns;
@@ -500,6 +492,17 @@ static int netcons_netpoll_setup(struct netconsole_target *nt)
 	bool ip_overwritten = false;
 	int err;
 
+	if (nt->remote_ip.family == AF_UNSPEC) {
+		np_err(np, "remote IP address not configured, aborting\n");
+		return -EDESTADDRREQ;
+	}
+
+	if (nt->local_ip.family != AF_UNSPEC &&
+	    nt->local_ip.family != nt->remote_ip.family) {
+		np_err(np, "local and remote IP address families differ, aborting\n");
+		return -EINVAL;
+	}
+
 	rtnl_lock();
 	if (np->dev_name[0])
 		ndev = __dev_get_by_name(net, np->dev_name);
@@ -536,16 +539,13 @@ static int netcons_netpoll_setup(struct netconsole_target *nt)
 		rtnl_lock();
 	}
 
-	if (netcons_local_ip_unset(nt)) {
-		if (!nt->ipv6) {
-			err = netcons_take_ipv4(nt, ndev);
-			if (err)
-				goto put;
-		} else {
+	if (nt->local_ip.family == AF_UNSPEC) {
+		if (nt->remote_ip.family == AF_INET6)
 			err = netcons_take_ipv6(nt, ndev);
-			if (err)
-				goto put;
-		}
+		else
+			err = netcons_take_ipv4(nt, ndev);
+		if (err)
+			goto put;
 		ip_overwritten = true;
 	}
 
@@ -727,24 +727,28 @@ static void netconsole_print_banner(struct netconsole_target *nt)
 	struct netpoll *np = &nt->np;
 
 	np_info(np, "local port %d\n", nt->local_port);
-	if (nt->ipv6)
+	if (nt->local_ip.family == AF_UNSPEC)
+		np_info(np, "local IP unset\n");
+	else if (nt->local_ip.family == AF_INET6)
 		np_info(np, "local IPv6 address %pI6c\n", &nt->local_ip.in6);
 	else
 		np_info(np, "local IPv4 address %pI4\n", &nt->local_ip.ip);
 	np_info(np, "interface name '%s'\n", np->dev_name);
 	np_info(np, "local ethernet address '%pM'\n", np->dev_mac);
 	np_info(np, "remote port %d\n", nt->remote_port);
-	if (nt->ipv6)
+	if (nt->remote_ip.family == AF_UNSPEC)
+		np_info(np, "remote IP unset\n");
+	else if (nt->remote_ip.family == AF_INET6)
 		np_info(np, "remote IPv6 address %pI6c\n", &nt->remote_ip.in6);
 	else
 		np_info(np, "remote IPv4 address %pI4\n", &nt->remote_ip.ip);
 	np_info(np, "remote ethernet address %pM\n", nt->remote_mac);
 }
 
-/* Parse the string and populate the `inet_addr` union. Return 0 if IPv4 is
- * populated, 1 if IPv6 is populated, and -1 upon failure.
+/* Parse the string and populate the `inet_addr` struct. Return 0 on success
+ * and -1 upon failure.
  */
-static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
+static int netpoll_parse_ip_addr(const char *str, struct inet_addr *addr)
 {
 	const char *end = NULL;
 	int len;
@@ -756,14 +760,18 @@ static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
 	if (str[len - 1] == '\n')
 		len -= 1;
 
-	if (in4_pton(str, len, (void *)addr, -1, &end) > 0 &&
-	    (!end || *end == 0 || *end == '\n'))
+	if (in4_pton(str, len, (void *)&addr->ip, -1, &end) > 0 &&
+	    (!end || *end == 0 || *end == '\n')) {
+		addr->family = AF_INET;
 		return 0;
+	}
 
 	if (IS_ENABLED(CONFIG_IPV6) &&
-	    in6_pton(str, len, (void *)addr, -1, &end) > 0 &&
-	    (!end || *end == 0 || *end == '\n'))
-		return 1;
+	    in6_pton(str, len, (void *)&addr->in6, -1, &end) > 0 &&
+	    (!end || *end == 0 || *end == '\n')) {
+		addr->family = AF_INET6;
+		return 0;
+	}
 
 	return -1;
 }
@@ -868,20 +876,24 @@ static ssize_t local_ip_show(struct config_item *item, char *buf)
 {
 	struct netconsole_target *nt = to_target(item);
 
-	if (nt->ipv6)
+	if (nt->local_ip.family == AF_UNSPEC)
+		return sysfs_emit(buf, "\n");
+	if (nt->local_ip.family == AF_INET6)
 		return sysfs_emit(buf, "%pI6c\n", &nt->local_ip.in6);
 	else
-		return sysfs_emit(buf, "%pI4\n", &nt->local_ip);
+		return sysfs_emit(buf, "%pI4\n", &nt->local_ip.ip);
 }
 
 static ssize_t remote_ip_show(struct config_item *item, char *buf)
 {
 	struct netconsole_target *nt = to_target(item);
 
-	if (nt->ipv6)
+	if (nt->remote_ip.family == AF_UNSPEC)
+		return sysfs_emit(buf, "\n");
+	if (nt->remote_ip.family == AF_INET6)
 		return sysfs_emit(buf, "%pI6c\n", &nt->remote_ip.in6);
 	else
-		return sysfs_emit(buf, "%pI4\n", &nt->remote_ip);
+		return sysfs_emit(buf, "%pI4\n", &nt->remote_ip.ip);
 }
 
 static ssize_t local_mac_show(struct config_item *item, char *buf)
@@ -1226,7 +1238,6 @@ static ssize_t local_ip_store(struct config_item *item, const char *buf,
 {
 	struct netconsole_target *nt = to_target(item);
 	ssize_t ret = -EINVAL;
-	int ipv6;
 
 	dynamic_netconsole_mutex_lock();
 	if (nt->state == STATE_ENABLED) {
@@ -1235,10 +1246,8 @@ static ssize_t local_ip_store(struct config_item *item, const char *buf,
 		goto out_unlock;
 	}
 
-	ipv6 = netpoll_parse_ip_addr(buf, &nt->local_ip);
-	if (ipv6 == -1)
+	if (netpoll_parse_ip_addr(buf, &nt->local_ip) < 0)
 		goto out_unlock;
-	nt->ipv6 = !!ipv6;
 
 	ret = count;
 out_unlock:
@@ -1251,7 +1260,6 @@ static ssize_t remote_ip_store(struct config_item *item, const char *buf,
 {
 	struct netconsole_target *nt = to_target(item);
 	ssize_t ret = -EINVAL;
-	int ipv6;
 
 	dynamic_netconsole_mutex_lock();
 	if (nt->state == STATE_ENABLED) {
@@ -1260,10 +1268,8 @@ static ssize_t remote_ip_store(struct config_item *item, const char *buf,
 		goto out_unlock;
 	}
 
-	ipv6 = netpoll_parse_ip_addr(buf, &nt->remote_ip);
-	if (ipv6 == -1)
+	if (netpoll_parse_ip_addr(buf, &nt->remote_ip) < 0)
 		goto out_unlock;
-	nt->ipv6 = !!ipv6;
 
 	ret = count;
 out_unlock:
@@ -2072,7 +2078,7 @@ static void netpoll_udp_checksum(struct netconsole_target *nt,
 
 	/* check needs to be set, since it will be consumed in csum_partial */
 	udph->check = 0;
-	if (nt->ipv6)
+	if (nt->remote_ip.family == AF_INET6)
 		udph->check = csum_ipv6_magic(&nt->local_ip.in6,
 					      &nt->remote_ip.in6,
 					      udp_len, IPPROTO_UDP,
@@ -2113,7 +2119,7 @@ static void push_eth(struct netconsole_target *nt, struct sk_buff *skb)
 	skb_reset_mac_header(skb);
 	ether_addr_copy(eth->h_source, np->dev->dev_addr);
 	ether_addr_copy(eth->h_dest, nt->remote_mac);
-	if (nt->ipv6)
+	if (nt->remote_ip.family == AF_INET6)
 		eth->h_proto = htons(ETH_P_IPV6);
 	else
 		eth->h_proto = htons(ETH_P_IP);
@@ -2182,7 +2188,7 @@ static int netpoll_send_udp(struct netconsole_target *nt, const char *msg,
 		WARN_ON_ONCE(!irqs_disabled());
 
 	udp_len = len + sizeof(struct udphdr);
-	if (nt->ipv6)
+	if (nt->remote_ip.family == AF_INET6)
 		ip_len = udp_len + sizeof(struct ipv6hdr);
 	else
 		ip_len = udp_len + sizeof(struct iphdr);
@@ -2198,7 +2204,7 @@ static int netpoll_send_udp(struct netconsole_target *nt, const char *msg,
 	skb_put(skb, len);
 
 	push_udp(nt, skb, len);
-	if (nt->ipv6)
+	if (nt->remote_ip.family == AF_INET6)
 		push_ipv6(nt, skb, len);
 	else
 		push_ipv4(nt, skb, len);
@@ -2527,10 +2533,8 @@ __releases(&target_list_lock)
 static int netconsole_parser_cmdline(struct netconsole_target *nt, char *opt)
 {
 	struct netpoll *np = &nt->np;
-	bool ipversion_set = false;
 	char *cur = opt;
 	char *delim;
-	int ipv6;
 
 	if (*cur != '@') {
 		delim = strchr(cur, '@');
@@ -2544,16 +2548,12 @@ static int netconsole_parser_cmdline(struct netconsole_target *nt, char *opt)
 	cur++;
 
 	if (*cur != '/') {
-		ipversion_set = true;
 		delim = strchr(cur, '/');
 		if (!delim)
 			goto parse_failed;
 		*delim = 0;
-		ipv6 = netpoll_parse_ip_addr(cur, &nt->local_ip);
-		if (ipv6 < 0)
+		if (netpoll_parse_ip_addr(cur, &nt->local_ip) < 0)
 			goto parse_failed;
-		else
-			nt->ipv6 = (bool)ipv6;
 		cur = delim;
 	}
 	cur++;
@@ -2595,13 +2595,11 @@ static int netconsole_parser_cmdline(struct netconsole_target *nt, char *opt)
 	if (!delim)
 		goto parse_failed;
 	*delim = 0;
-	ipv6 = netpoll_parse_ip_addr(cur, &nt->remote_ip);
-	if (ipv6 < 0)
+	if (netpoll_parse_ip_addr(cur, &nt->remote_ip) < 0)
 		goto parse_failed;
-	else if (ipversion_set && nt->ipv6 != (bool)ipv6)
+	if (nt->local_ip.family != AF_UNSPEC &&
+	    nt->local_ip.family != nt->remote_ip.family)
 		goto parse_failed;
-	else
-		nt->ipv6 = (bool)ipv6;
 	cur = delim + 1;
 
 	if (*cur != 0) {
diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c
index 5c55c308487b42..b792ee10b1c1a7 100644
--- a/drivers/net/netdevsim/bus.c
+++ b/drivers/net/netdevsim/bus.c
@@ -194,7 +194,6 @@ new_device_store(const struct bus_type *bus, const char *buf, size_t count)
 		goto err;
 	}
 
-	refcount_inc(&nsim_bus_devs);
 	/* Allow using nsim_bus_dev */
 	smp_store_release(&nsim_bus_dev->init, true);
 
@@ -465,6 +464,7 @@ nsim_bus_dev_new(unsigned int id, unsigned int port_count, unsigned int num_queu
 	/* Disallow using nsim_bus_dev */
 	smp_store_release(&nsim_bus_dev->init, false);
 
+	refcount_inc(&nsim_bus_devs);
 	err = device_register(&nsim_bus_dev->dev);
 	if (err)
 		goto err_nsim_bus_dev_id_free;
@@ -520,8 +520,6 @@ void nsim_bus_exit(void)
 
 	/* Disallow using resources */
 	smp_store_release(&nsim_bus_enable, false);
-	if (refcount_dec_and_test(&nsim_bus_devs))
-		complete(&nsim_bus_devs_released);
 
 	mutex_lock(&nsim_bus_dev_list_lock);
 	list_for_each_entry_safe(nsim_bus_dev, tmp, &nsim_bus_dev_list, list) {
@@ -530,6 +528,9 @@ void nsim_bus_exit(void)
 	}
 	mutex_unlock(&nsim_bus_dev_list_lock);
 
+	if (refcount_dec_and_test(&nsim_bus_devs))
+		complete(&nsim_bus_devs_released);
+
 	wait_for_completion(&nsim_bus_devs_released);
 
 	driver_unregister(&nsim_driver);
diff --git a/drivers/net/netdevsim/ethtool.c b/drivers/net/netdevsim/ethtool.c
index 025ea79879f3a9..c3d2bc783d5571 100644
--- a/drivers/net/netdevsim/ethtool.c
+++ b/drivers/net/netdevsim/ethtool.c
@@ -205,7 +205,59 @@ static int nsim_get_ts_info(struct net_device *dev,
 	return 0;
 }
 
+static u32 nsim_get_rxfh_key_size(struct net_device *dev)
+{
+	return NETDEV_RSS_KEY_LEN;
+}
+
+static u32 nsim_get_rxfh_indir_size(struct net_device *dev)
+{
+	return NSIM_RSS_INDIR_SIZE;
+}
+
+static int nsim_get_rxfh(struct net_device *dev,
+			 struct ethtool_rxfh_param *rxfh)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+	u32 i;
+
+	rxfh->hfunc = ETH_RSS_HASH_TOP;
+	if (rxfh->indir) {
+		for (i = 0; i < NSIM_RSS_INDIR_SIZE; i++)
+			rxfh->indir[i] = ethtool_rxfh_indir_default(i, ns->ethtool.channels);
+	}
+	if (rxfh->key)
+		memcpy(rxfh->key, ns->ethtool.rss_key,
+		       sizeof(ns->ethtool.rss_key));
+
+	return 0;
+}
+
+static int nsim_get_rxfh_fields(struct net_device *dev,
+				struct ethtool_rxfh_fields *info)
+{
+	switch (info->flow_type) {
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+		info->data = RXH_IP_SRC | RXH_IP_DST |
+			     RXH_L4_B_0_1 | RXH_L4_B_2_3;
+		return 0;
+	case IPV4_FLOW:
+	case IPV6_FLOW:
+		info->data = RXH_IP_SRC | RXH_IP_DST;
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static const struct ethtool_ops nsim_ethtool_ops = {
+	.get_rxfh_key_size		= nsim_get_rxfh_key_size,
+	.get_rxfh_indir_size		= nsim_get_rxfh_indir_size,
+	.get_rxfh			= nsim_get_rxfh,
+	.get_rxfh_fields		= nsim_get_rxfh_fields,
 	.supported_coalesce_params	= ETHTOOL_COALESCE_ALL_PARAMS,
 	.supported_ring_params		= ETHTOOL_RING_USE_TCP_DATA_SPLIT |
 					  ETHTOOL_RING_USE_HDS_THRS,
@@ -250,6 +302,7 @@ void nsim_ethtool_init(struct netdevsim *ns)
 	ns->ethtool.fec.active_fec = ETHTOOL_FEC_NONE;
 
 	ns->ethtool.channels = ns->nsim_bus_dev->num_queues;
+	netdev_rss_key_fill(ns->ethtool.rss_key, sizeof(ns->ethtool.rss_key));
 
 	ethtool = debugfs_create_dir("ethtool", ns->nsim_dev_port->ddir);
 	ns->ethtool_ddir = ethtool;
diff --git a/drivers/net/netdevsim/ipsec.c b/drivers/net/netdevsim/ipsec.c
index 36a1be4923d61f..5722837bcda36c 100644
--- a/drivers/net/netdevsim/ipsec.c
+++ b/drivers/net/netdevsim/ipsec.c
@@ -50,11 +50,11 @@ static ssize_t nsim_dbg_netdev_ops_read(struct file *filp,
 		p += scnprintf(p, bufsize - (p - buf),
 			       "sa[%i]    spi=0x%08x proto=0x%x salt=0x%08x crypt=%d\n",
 			       i, be32_to_cpu(sap->xs->id.spi),
-			       sap->xs->id.proto, sap->salt, sap->crypt);
+			       sap->xs->id.proto, be32_to_cpu(sap->salt), sap->crypt);
 		p += scnprintf(p, bufsize - (p - buf),
 			       "sa[%i]    key=0x%08x %08x %08x %08x\n",
-			       i, sap->key[0], sap->key[1],
-			       sap->key[2], sap->key[3]);
+			       i, be32_to_cpu(sap->key[0]), be32_to_cpu(sap->key[1]),
+			       be32_to_cpu(sap->key[2]), be32_to_cpu(sap->key[3]));
 	}
 
 	len = simple_read_from_buffer(buffer, count, ppos, buf, p - buf);
@@ -87,7 +87,7 @@ static int nsim_ipsec_find_empty_idx(struct nsim_ipsec *ipsec)
 
 static int nsim_ipsec_parse_proto_keys(struct net_device *dev,
 				       struct xfrm_state *xs,
-				       u32 *mykey, u32 *mysalt)
+				       __be32 *mykey, __be32 *mysalt)
 {
 	const char aes_gcm_name[] = "rfc4106(gcm(aes))";
 	unsigned char *key_data;
@@ -117,7 +117,7 @@ static int nsim_ipsec_parse_proto_keys(struct net_device *dev,
 
 	/* 160 accounts for 16 byte key and 4 byte salt */
 	if (key_len > NSIM_IPSEC_AUTH_BITS) {
-		*mysalt = ((u32 *)key_data)[4];
+		*mysalt = ((__be32 *)key_data)[4];
 	} else if (key_len == NSIM_IPSEC_AUTH_BITS) {
 		*mysalt = 0;
 	} else {
diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c
index b4a99f3ceac60b..eebc02ccc4a929 100644
--- a/drivers/net/netdevsim/netdev.c
+++ b/drivers/net/netdevsim/netdev.c
@@ -185,6 +185,14 @@ static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	return NETDEV_TX_OK;
 }
 
+static netdev_tx_t nsim_start_xmit_vf(struct sk_buff *skb,
+				      struct net_device *dev)
+{
+	dev_dstats_tx_dropped(dev);
+	kfree_skb(skb);
+	return NETDEV_TX_OK;
+}
+
 static int nsim_set_rx_mode(struct net_device *dev,
 			    struct netdev_hw_addr_list *uc,
 			    struct netdev_hw_addr_list *mc)
@@ -651,7 +659,7 @@ static const struct net_device_ops nsim_netdev_ops = {
 };
 
 static const struct net_device_ops nsim_vf_netdev_ops = {
-	.ndo_start_xmit		= nsim_start_xmit,
+	.ndo_start_xmit		= nsim_start_xmit_vf,
 	.ndo_set_rx_mode_async	= nsim_set_rx_mode,
 	.ndo_set_mac_address	= eth_mac_addr,
 	.ndo_validate_addr	= eth_validate_addr,
diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h
index 55aec41237b9bd..fd5599cd3bb9f1 100644
--- a/drivers/net/netdevsim/netdevsim.h
+++ b/drivers/net/netdevsim/netdevsim.h
@@ -42,8 +42,8 @@
 struct nsim_sa {
 	struct xfrm_state *xs;
 	__be32 ipaddr[4];
-	u32 key[4];
-	u32 salt;
+	__be32 key[4];
+	__be32 salt;
 	bool used;
 	bool crypt;
 	bool rx;
@@ -87,10 +87,13 @@ struct nsim_ethtool_pauseparam {
 	bool report_stats_tx;
 };
 
+#define NSIM_RSS_INDIR_SIZE		128
+
 struct nsim_ethtool {
 	u32 get_err;
 	u32 set_err;
 	u32 channels;
+	u8 rss_key[NETDEV_RSS_KEY_LEN];
 	struct nsim_ethtool_pauseparam pauseparam;
 	struct ethtool_coalesce coalesce;
 	struct ethtool_ringparam ring;
@@ -122,7 +125,6 @@ struct netdevsim {
 		struct dentry *rereg;
 		struct mutex rereg_lock;
 		u32 spi;
-		u32 assoc_cnt;
 	} psp;
 
 	struct nsim_bus_dev *nsim_bus_dev;
diff --git a/drivers/net/netdevsim/psp.c b/drivers/net/netdevsim/psp.c
index 6b3532b5e360d5..b7452c49b581d1 100644
--- a/drivers/net/netdevsim/psp.c
+++ b/drivers/net/netdevsim/psp.c
@@ -23,7 +23,6 @@ nsim_do_psp(struct sk_buff *skb, struct netdevsim *ns,
 	struct psp_assoc *pas;
 	struct net *net;
 	int psp_len;
-	void **ptr;
 
 	rcu_read_lock();
 	pas = psp_skb_get_assoc_rcu(skb);
@@ -37,12 +36,6 @@ nsim_do_psp(struct sk_buff *skb, struct netdevsim *ns,
 		goto out_unlock;
 	}
 
-	ptr = psp_assoc_drv_data(pas);
-	if (*ptr != ns) {
-		rc = SKB_DROP_REASON_PSP_OUTPUT;
-		goto out_unlock;
-	}
-
 	net = sock_net(skb->sk);
 	if (!psp_dev_encapsulate(net, skb, pas->tx.spi, pas->version, 0)) {
 		rc = SKB_DROP_REASON_PSP_OUTPUT;
@@ -149,19 +142,6 @@ nsim_rx_spi_alloc(struct psp_dev *psd, u32 version,
 	return 0;
 }
 
-static int nsim_assoc_add(struct psp_dev *psd, struct psp_assoc *pas,
-			  struct netlink_ext_ack *extack)
-{
-	struct netdevsim *ns = psd->drv_priv;
-	void **ptr = psp_assoc_drv_data(pas);
-
-	/* Copy drv_priv from psd to assoc */
-	*ptr = psd->drv_priv;
-	ns->psp.assoc_cnt++;
-
-	return 0;
-}
-
 static int nsim_key_rotate(struct psp_dev *psd, struct netlink_ext_ack *extack)
 {
 	struct netdevsim *ns = psd->drv_priv;
@@ -177,15 +157,6 @@ static int nsim_key_rotate(struct psp_dev *psd, struct netlink_ext_ack *extack)
 	return 0;
 }
 
-static void nsim_assoc_del(struct psp_dev *psd, struct psp_assoc *pas)
-{
-	struct netdevsim *ns = psd->drv_priv;
-	void **ptr = psp_assoc_drv_data(pas);
-
-	*ptr = NULL;
-	ns->psp.assoc_cnt--;
-}
-
 static void nsim_get_stats(struct psp_dev *psd, struct psp_dev_stats *stats)
 {
 	struct netdevsim *ns = psd->drv_priv;
@@ -204,8 +175,6 @@ static void nsim_get_stats(struct psp_dev *psd, struct psp_dev_stats *stats)
 static struct psp_dev_ops nsim_psp_ops = {
 	.set_config	= nsim_psp_set_config,
 	.rx_spi_alloc	= nsim_rx_spi_alloc,
-	.tx_key_add	= nsim_assoc_add,
-	.tx_key_del	= nsim_assoc_del,
 	.key_rotate	= nsim_key_rotate,
 	.get_stats	= nsim_get_stats,
 };
@@ -215,7 +184,6 @@ static struct psp_dev_caps nsim_psp_caps = {
 		    1 << PSP_VERSION_HDR0_AES_GMAC_128 |
 		    1 << PSP_VERSION_HDR0_AES_GCM_256 |
 		    1 << PSP_VERSION_HDR0_AES_GMAC_256,
-	.assoc_drv_spc = sizeof(void *),
 };
 
 static void __nsim_psp_uninit(struct netdevsim *ns, bool teardown)
@@ -230,7 +198,6 @@ static void __nsim_psp_uninit(struct netdevsim *ns, bool teardown)
 		synchronize_rcu();
 		psp_dev_unregister(psd);
 	}
-	WARN_ON(ns->psp.assoc_cnt);
 }
 
 void nsim_psp_uninit(struct netdevsim *ns)
diff --git a/drivers/net/netkit.c b/drivers/net/netkit.c
index a3931cd821321c..82e608e733d9ee 100644
--- a/drivers/net/netkit.c
+++ b/drivers/net/netkit.c
@@ -1209,12 +1209,12 @@ static size_t netkit_get_size(const struct net_device *dev)
 
 static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
-	struct netkit *nk = netkit_priv(dev);
-	struct net_device *peer = rtnl_dereference(nk->peer);
+	const struct netkit *nk = netkit_priv(dev);
+	const struct net_device *peer;
 
 	if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary))
 		return -EMSGSIZE;
-	if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy))
+	if (nla_put_u32(skb, IFLA_NETKIT_POLICY, READ_ONCE(nk->policy)))
 		return -EMSGSIZE;
 	if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode))
 		return -EMSGSIZE;
@@ -1228,13 +1228,18 @@ static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev)
 	if (nla_put_u32(skb, IFLA_NETKIT_PAIRING, nk->pair))
 		return -EMSGSIZE;
 
+	rcu_read_lock();
+	peer = rcu_dereference(nk->peer);
 	if (peer) {
 		nk = netkit_priv(peer);
-		if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy))
-			return -EMSGSIZE;
-		if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub))
+		if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY,
+				READ_ONCE(nk->policy)) ||
+		    nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub)) {
+			rcu_read_unlock();
 			return -EMSGSIZE;
+		}
 	}
+	rcu_read_unlock();
 
 	return 0;
 }
diff --git a/drivers/net/ovpn/netlink.c b/drivers/net/ovpn/netlink.c
index 4dad852941982e..5432bc2eb8e80c 100644
--- a/drivers/net/ovpn/netlink.c
+++ b/drivers/net/ovpn/netlink.c
@@ -100,6 +100,8 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs,
 	struct sockaddr_in6 *sin6;
 	struct sockaddr_in *sin;
 	struct in6_addr *in6;
+	struct nlattr *scope;
+	u32 scope_id = 0;
 	__be16 port = 0;
 	__be32 *in;
 
@@ -114,6 +116,9 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs,
 	} else if (attrs[OVPN_A_PEER_REMOTE_IPV6]) {
 		ss->ss_family = AF_INET6;
 		in6 = nla_data(attrs[OVPN_A_PEER_REMOTE_IPV6]);
+		scope = attrs[OVPN_A_PEER_REMOTE_IPV6_SCOPE_ID];
+		if (scope)
+			scope_id = nla_get_u32(scope);
 	} else {
 		return false;
 	}
@@ -126,6 +131,7 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs,
 		if (!ipv6_addr_v4mapped(in6)) {
 			sin6 = (struct sockaddr_in6 *)ss;
 			sin6->sin6_port = port;
+			sin6->sin6_scope_id = scope_id;
 			memcpy(&sin6->sin6_addr, in6, sizeof(*in6));
 			break;
 		}
@@ -179,6 +185,39 @@ static sa_family_t ovpn_nl_family_get(struct nlattr *addr4,
 	return AF_UNSPEC;
 }
 
+static int ovpn_nl_peer_check_vpn_addrs(const struct in_addr *addr4,
+					const struct in6_addr *addr6,
+					struct genl_info *info)
+{
+	int addr6_type;
+
+	if (addr4->s_addr == htonl(INADDR_ANY) && ipv6_addr_any(addr6)) {
+		NL_SET_ERR_MSG_MOD(info->extack,
+				   "at least one VPN IP must be configured in MP mode");
+		return -EINVAL;
+	}
+
+	if (ipv4_is_multicast(addr4->s_addr) || ipv4_is_lbcast(addr4->s_addr) ||
+	    ipv4_is_loopback(addr4->s_addr)) {
+		NL_SET_ERR_MSG_MOD(info->extack,
+				   "VPN IPv4 address must be valid unicast or any");
+		return -EADDRNOTAVAIL;
+	}
+
+	if (!ipv6_addr_any(addr6)) {
+		addr6_type = ipv6_addr_type(addr6);
+
+		if (!(addr6_type & IPV6_ADDR_UNICAST) ||
+		    (addr6_type & (IPV6_ADDR_LOOPBACK | IPV6_ADDR_COMPATv4))) {
+			NL_SET_ERR_MSG_MOD(info->extack,
+					   "VPN IPv6 address must be valid unicast or any");
+			return -EADDRNOTAVAIL;
+		}
+	}
+
+	return 0;
+}
+
 static int ovpn_nl_peer_precheck(struct ovpn_priv *ovpn,
 				 struct genl_info *info,
 				 struct nlattr **attrs)
@@ -346,8 +385,10 @@ static int ovpn_nl_peer_modify(struct ovpn_peer *peer, struct genl_info *info,
 
 int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct nlattr *attrs[OVPN_A_PEER_MAX + 1];
+	struct in_addr vpn_addr4 = { .s_addr = htonl(INADDR_ANY) };
+	struct in6_addr vpn_addr6 = IN6ADDR_ANY_INIT;
 	struct ovpn_priv *ovpn = info->user_ptr[0];
+	struct nlattr *attrs[OVPN_A_PEER_MAX + 1];
 	struct ovpn_socket *ovpn_sock;
 	struct socket *sock = NULL;
 	struct ovpn_peer *peer;
@@ -371,11 +412,18 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
 		return -EINVAL;
 
 	/* in MP mode VPN IPs are required for selecting the right peer */
-	if (ovpn->mode == OVPN_MODE_MP && !attrs[OVPN_A_PEER_VPN_IPV4] &&
-	    !attrs[OVPN_A_PEER_VPN_IPV6]) {
-		NL_SET_ERR_MSG_FMT_MOD(info->extack,
-				       "VPN IP must be provided in MP mode");
-		return -EINVAL;
+	if (ovpn->mode == OVPN_MODE_MP) {
+		if (attrs[OVPN_A_PEER_VPN_IPV4])
+			vpn_addr4.s_addr =
+				nla_get_in_addr(attrs[OVPN_A_PEER_VPN_IPV4]);
+		if (attrs[OVPN_A_PEER_VPN_IPV6])
+			vpn_addr6 =
+				nla_get_in6_addr(attrs[OVPN_A_PEER_VPN_IPV6]);
+
+		ret = ovpn_nl_peer_check_vpn_addrs(&vpn_addr4, &vpn_addr6,
+						   info);
+		if (ret < 0)
+			return ret;
 	}
 
 	peer_id = nla_get_u32(attrs[OVPN_A_PEER_ID]);
@@ -474,8 +522,10 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
 
 int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info)
 {
-	struct nlattr *attrs[OVPN_A_PEER_MAX + 1];
 	struct ovpn_priv *ovpn = info->user_ptr[0];
+	struct nlattr *attrs[OVPN_A_PEER_MAX + 1];
+	struct in6_addr vpn_addr6;
+	struct in_addr vpn_addr4;
 	struct ovpn_socket *sock;
 	struct ovpn_peer *peer;
 	u32 peer_id;
@@ -522,28 +572,58 @@ int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info)
 	rcu_read_unlock();
 
 	spin_lock_bh(&ovpn->lock);
-	ret = ovpn_nl_peer_modify(peer, info, attrs);
-	if (ret < 0) {
-		spin_unlock_bh(&ovpn->lock);
-		ovpn_peer_put(peer);
-		return ret;
+
+	vpn_addr4 = peer->vpn_addrs.ipv4;
+	vpn_addr6 = peer->vpn_addrs.ipv6;
+
+	/* reject peer with conflicting VPN address */
+	if (attrs[OVPN_A_PEER_VPN_IPV4]) {
+		vpn_addr4.s_addr = nla_get_in_addr(attrs[OVPN_A_PEER_VPN_IPV4]);
+		if (ovpn_peer_vpn_addr_conflict4(ovpn, peer, &vpn_addr4))
+			goto addr_conflict;
+	}
+	if (attrs[OVPN_A_PEER_VPN_IPV6]) {
+		vpn_addr6 = nla_get_in6_addr(attrs[OVPN_A_PEER_VPN_IPV6]);
+		if (ovpn_peer_vpn_addr_conflict6(ovpn, peer, &vpn_addr6))
+			goto addr_conflict;
 	}
 
+	/* in MP mode VPN IPs are required for selecting the right peer */
+	if (ovpn->mode == OVPN_MODE_MP) {
+		ret = ovpn_nl_peer_check_vpn_addrs(&vpn_addr4, &vpn_addr6,
+						   info);
+		if (ret < 0)
+			goto unlock;
+	}
+
+	ret = ovpn_nl_peer_modify(peer, info, attrs);
+	if (ret < 0)
+		goto unlock;
+
 	/* ret == 1 means that VPN IPv4/6 has been modified and rehashing
 	 * is required
 	 */
-	if (ret > 0)
+	if (ret > 0) {
 		ovpn_peer_hash_vpn_ip(peer);
+		ret = 0;
+	}
 	/* if the remote endpoint was updated, the by_transp_addr hash bucket
 	 * also needs to be refreshed, otherwise incoming packets from the new
 	 * remote address would fail the lockless lookup
 	 */
 	if (attrs[OVPN_A_PEER_REMOTE_IPV4] || attrs[OVPN_A_PEER_REMOTE_IPV6])
 		ovpn_peer_hash_transp_addr(peer);
+
+unlock:
 	spin_unlock_bh(&ovpn->lock);
 	ovpn_peer_put(peer);
 
-	return 0;
+	return ret;
+addr_conflict:
+	NL_SET_ERR_MSG_FMT_MOD(info->extack,
+			       "VPN IP is already assigned to another peer");
+	ret = -EADDRINUSE;
+	goto unlock;
 }
 
 static int ovpn_nl_send_peer(struct sk_buff *skb, const struct genl_info *info,
diff --git a/drivers/net/ovpn/peer.c b/drivers/net/ovpn/peer.c
index c95656ca7c3571..2067825bb5b614 100644
--- a/drivers/net/ovpn/peer.c
+++ b/drivers/net/ovpn/peer.c
@@ -113,6 +113,7 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id)
 	RCU_INIT_POINTER(peer->bind, NULL);
 	ovpn_crypto_state_init(&peer->crypto);
 	spin_lock_init(&peer->lock);
+	seqcount_spinlock_init(&peer->route_key_seq, &peer->lock);
 	kref_init(&peer->refcount);
 	ovpn_peer_stats_init(&peer->vpn_stats);
 	ovpn_peer_stats_init(&peer->link_stats);
@@ -199,13 +200,12 @@ static void __ovpn_peer_hash_transp_addr(struct ovpn_peer *peer,
  */
 void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb)
 {
+	const void *local_ip = NULL;
 	struct sockaddr_storage ss;
 	struct sockaddr_in6 *sa6;
-	bool reset_cache = false;
 	struct sockaddr_in *sa;
 	struct ovpn_bind *bind;
-	const void *local_ip;
-	size_t salen = 0;
+	bool floated = false;
 
 	spin_lock_bh(&peer->lock);
 	bind = rcu_dereference_protected(peer->bind,
@@ -232,8 +232,7 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb)
 				.sin_addr.s_addr = ip_hdr(skb)->saddr,
 				.sin_port = udp_hdr(skb)->source,
 			};
-			salen = sizeof(*sa);
-			reset_cache = true;
+			floated = true;
 			break;
 		}
 
@@ -245,10 +244,12 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb)
 					    netdev_name(peer->ovpn->dev),
 					    peer->id, &bind->local.ipv4.s_addr,
 					    &ip_hdr(skb)->daddr);
-			bind->local.ipv4.s_addr = ip_hdr(skb)->daddr;
-			reset_cache = true;
+			local_ip = &ip_hdr(skb)->daddr;
+			memcpy(&ss, &bind->remote, sizeof(struct sockaddr_in));
+			break;
 		}
-		break;
+		/* nothing changed */
+		goto unlock;
 	case htons(ETH_P_IPV6):
 		/* float check */
 		if (unlikely(!ovpn_bind_skb_src_match(bind, skb))) {
@@ -270,8 +271,7 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb)
 					ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr,
 							    skb->skb_iif),
 			};
-			salen = sizeof(*sa6);
-			reset_cache = true;
+			floated = true;
 			break;
 		}
 
@@ -284,26 +284,30 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb)
 					    netdev_name(peer->ovpn->dev),
 					    peer->id, &bind->local.ipv6,
 					    &ipv6_hdr(skb)->daddr);
-			bind->local.ipv6 = ipv6_hdr(skb)->daddr;
-			reset_cache = true;
+			local_ip = &ipv6_hdr(skb)->daddr;
+			memcpy(&ss, &bind->remote, sizeof(struct sockaddr_in6));
+			break;
 		}
-		break;
+		/* nothing changed */
+		goto unlock;
 	default:
 		goto unlock;
 	}
 
-	if (unlikely(reset_cache))
-		dst_cache_reset(&peer->dst_cache);
-
-	/* if the peer did not float, we can bail out now */
-	if (likely(!salen))
-		goto unlock;
-
 	if (unlikely(ovpn_peer_reset_sockaddr(peer,
 					      (struct sockaddr_storage *)&ss,
 					      local_ip) < 0))
 		goto unlock;
 
+	/* reset the cache only after a successful bind update to avoid useless
+	 * cache misses on concurrent TX
+	 */
+	dst_cache_reset(&peer->dst_cache);
+
+	/* if only the local address changed, bail out now */
+	if (!floated)
+		goto unlock;
+
 	net_dbg_ratelimited("%s: peer %d floated to %pIScp",
 			    netdev_name(peer->ovpn->dev), peer->id, &ss);
 
@@ -484,7 +488,7 @@ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr4(struct ovpn_priv *ovpn,
  * Return: the peer if found or NULL otherwise
  */
 static struct ovpn_peer *ovpn_peer_get_by_vpn_addr6(struct ovpn_priv *ovpn,
-						    struct in6_addr *addr)
+						    const struct in6_addr *addr)
 {
 	struct hlist_nulls_head *nhead;
 	struct hlist_nulls_node *ntmp;
@@ -509,6 +513,64 @@ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr6(struct ovpn_priv *ovpn,
 	return NULL;
 }
 
+/**
+ * ovpn_peer_vpn_addr_conflict4 - check if the VPN v4 address is already in use
+ * @ovpn: the openvpn instance to search
+ * @peer: peer being added or updated, or NULL
+ * @addr: VPN IPv4 address to check
+ *
+ * Check whether @addr is already assigned to another peer. @peer is ignored
+ * when found, allowing peer updates that keep an existing address.
+ * Unspecified addresses are ignored.
+ *
+ * Note: the caller must hold @ovpn->lock.
+ *
+ * Return: true on conflict, false otherwise.
+ */
+bool ovpn_peer_vpn_addr_conflict4(struct ovpn_priv *ovpn,
+				  const struct ovpn_peer *peer,
+				  const struct in_addr *addr)
+{
+	struct ovpn_peer *tmp = NULL;
+
+	lockdep_assert_held(&ovpn->lock);
+
+	/* we don't hash INADDR_ANY, no conflict in that case */
+	if (addr->s_addr != htonl(INADDR_ANY))
+		tmp = ovpn_peer_get_by_vpn_addr4(ovpn, addr->s_addr);
+
+	return tmp && tmp != peer;
+}
+
+/**
+ * ovpn_peer_vpn_addr_conflict6 - check if the VPN v6 address is already in use
+ * @ovpn: the openvpn instance to search
+ * @peer: peer being added or updated, or NULL
+ * @addr: VPN IPv6 address to check
+ *
+ * Check whether @addr is already assigned to another peer. @peer is ignored
+ * when found, allowing peer updates that keep an existing address.
+ * Unspecified addresses are ignored.
+ *
+ * Note: the caller must hold @ovpn->lock.
+ *
+ * Return: true on conflict, false otherwise.
+ */
+bool ovpn_peer_vpn_addr_conflict6(struct ovpn_priv *ovpn,
+				  const struct ovpn_peer *peer,
+				  const struct in6_addr *addr)
+{
+	struct ovpn_peer *tmp = NULL;
+
+	lockdep_assert_held(&ovpn->lock);
+
+	/* we don't hash ::, no conflict in that case */
+	if (!ipv6_addr_any(addr))
+		tmp = ovpn_peer_get_by_vpn_addr6(ovpn, addr);
+
+	return tmp && tmp != peer;
+}
+
 /**
  * ovpn_peer_transp_match - check if sockaddr and peer binding match
  * @peer: the peer to get the binding from
@@ -990,10 +1052,11 @@ void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer)
 	if (hlist_unhashed(&peer->hash_entry_id))
 		return;
 
-	if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) {
-		/* remove potential old hashing */
-		hlist_nulls_del_init_rcu(&peer->hash_entry_addr4);
+	/* remove potential old hashing */
+	hlist_nulls_del_init_rcu(&peer->hash_entry_addr4);
+	hlist_nulls_del_init_rcu(&peer->hash_entry_addr6);
 
+	if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) {
 		nhead = ovpn_get_hash_head(peer->ovpn->peers->by_vpn_addr4,
 					   &peer->vpn_addrs.ipv4,
 					   sizeof(peer->vpn_addrs.ipv4));
@@ -1001,9 +1064,6 @@ void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer)
 	}
 
 	if (!ipv6_addr_any(&peer->vpn_addrs.ipv6)) {
-		/* remove potential old hashing */
-		hlist_nulls_del_init_rcu(&peer->hash_entry_addr6);
-
 		nhead = ovpn_get_hash_head(peer->ovpn->peers->by_vpn_addr6,
 					   &peer->vpn_addrs.ipv6,
 					   sizeof(peer->vpn_addrs.ipv6));
@@ -1038,6 +1098,13 @@ static int ovpn_peer_add_mp(struct ovpn_priv *ovpn, struct ovpn_peer *peer)
 		goto out;
 	}
 
+	/* reject peer with conflicting VPN address */
+	if (ovpn_peer_vpn_addr_conflict4(ovpn, NULL, &peer->vpn_addrs.ipv4) ||
+	    ovpn_peer_vpn_addr_conflict6(ovpn, NULL, &peer->vpn_addrs.ipv6)) {
+		ret = -EADDRINUSE;
+		goto out;
+	}
+
 	bind = rcu_dereference_protected(peer->bind, true);
 	/* peers connected via TCP have bind == NULL */
 	if (bind) {
diff --git a/drivers/net/ovpn/peer.h b/drivers/net/ovpn/peer.h
index dfa5c0037e02b0..1879bfb76992d1 100644
--- a/drivers/net/ovpn/peer.h
+++ b/drivers/net/ovpn/peer.h
@@ -10,6 +10,7 @@
 #ifndef _NET_OVPN_OVPNPEER_H_
 #define _NET_OVPN_OVPNPEER_H_
 
+#include <linux/seqlock.h>
 #include <net/dst_cache.h>
 #include <net/strparser.h>
 
@@ -17,6 +18,16 @@
 #include "socket.h"
 #include "stats.h"
 
+/**
+ * struct ovpn_route_key - route key used for the peer dst cache
+ * @mark: fwmark used for route lookup
+ * @sport: UDP source port used for route lookup
+ */
+struct ovpn_route_key {
+	u32 mark;
+	__be16 sport;
+};
+
 /**
  * struct ovpn_peer - the main remote peer object
  * @ovpn: main openvpn instance this peer belongs to
@@ -45,6 +56,8 @@
  * @tcp.sk_cb.ops: pointer to the original prot_ops object (TCP only)
  * @crypto: the crypto configuration (ciphers, keys, etc..)
  * @dst_cache: cache for dst_entry used to send to peer
+ * @route_key: route key matching the current dst cache contents
+ * @route_key_seq: seqcount protecting lockless route_key reads
  * @bind: remote peer binding
  * @keepalive_interval: seconds after which a new keepalive should be sent
  * @keepalive_xmit_exp: future timestamp when next keepalive should be sent
@@ -55,7 +68,7 @@
  * @vpn_stats: per-peer in-VPN TX/RX stats
  * @link_stats: per-peer link/transport TX/RX stats
  * @delete_reason: why peer was deleted (i.e. timeout, transport error, ..)
- * @lock: protects binding to peer (bind) and keepalive* fields
+ * @lock: protects binding to peer (bind), route_key and keepalive* fields
  * @refcount: reference counter
  * @rcu: used to free peer in an RCU safe way
  * @release_entry: entry for the socket release list
@@ -99,6 +112,8 @@ struct ovpn_peer {
 	} tcp;
 	struct ovpn_crypto_state crypto;
 	struct dst_cache dst_cache;
+	struct ovpn_route_key route_key;
+	seqcount_spinlock_t route_key_seq;
 	struct ovpn_bind __rcu *bind;
 	unsigned long keepalive_interval;
 	unsigned long keepalive_xmit_exp;
@@ -109,7 +124,7 @@ struct ovpn_peer {
 	struct ovpn_peer_stats vpn_stats;
 	struct ovpn_peer_stats link_stats;
 	enum ovpn_del_peer_reason delete_reason;
-	spinlock_t lock; /* protects bind  and keepalive* */
+	spinlock_t lock; /* protects bind, route_key and keepalive* */
 	struct kref refcount;
 	struct rcu_head rcu;
 	struct llist_node release_entry;
@@ -149,6 +164,12 @@ struct ovpn_peer *ovpn_peer_get_by_transp_addr(struct ovpn_priv *ovpn,
 struct ovpn_peer *ovpn_peer_get_by_id(struct ovpn_priv *ovpn, u32 peer_id);
 struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn,
 				       struct sk_buff *skb);
+bool ovpn_peer_vpn_addr_conflict4(struct ovpn_priv *ovpn,
+				  const struct ovpn_peer *peer,
+				  const struct in_addr *addr);
+bool ovpn_peer_vpn_addr_conflict6(struct ovpn_priv *ovpn,
+				  const struct ovpn_peer *peer,
+				  const struct in6_addr *addr);
 void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer);
 void ovpn_peer_hash_transp_addr(struct ovpn_peer *peer);
 bool ovpn_peer_check_by_src(struct ovpn_priv *ovpn, struct sk_buff *skb,
diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c
index 7f69e8890b5b5c..055cdb1bee13b0 100644
--- a/drivers/net/ovpn/udp.c
+++ b/drivers/net/ovpn/udp.c
@@ -131,6 +131,77 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
 	return 0;
 }
 
+static bool ovpn_route_key_equal(const struct ovpn_route_key *a,
+				 const struct ovpn_route_key *b)
+{
+	return a->mark == b->mark && a->sport == b->sport;
+}
+
+/**
+ * ovpn_dst_cache_check_key - reset peer dst cache after key changes
+ * @peer: the peer owning the dst cache
+ * @cache: the cache that might need to be reset
+ * @key: the route key for the packet being transmitted
+ *
+ * Reset the peer dst cache if it was populated for a different route key.
+ */
+static void ovpn_dst_cache_check_key(struct ovpn_peer *peer,
+				     struct dst_cache *cache,
+				     const struct ovpn_route_key *key)
+{
+	struct ovpn_route_key old_key;
+	unsigned int seq;
+
+	/* snapshot the saved key before deciding whether the cache matches */
+	do {
+		seq = read_seqcount_begin(&peer->route_key_seq);
+		old_key = peer->route_key;
+	} while (read_seqcount_retry(&peer->route_key_seq, seq));
+
+	/* nothing changed: the current cache can be reused */
+	if (likely(ovpn_route_key_equal(&old_key, key)))
+		return;
+
+	/* recheck under lock because another path may have updated the key */
+	spin_lock_bh(&peer->lock);
+	if (!ovpn_route_key_equal(&peer->route_key, key)) {
+		write_seqcount_begin(&peer->route_key_seq);
+		peer->route_key = *key;
+		dst_cache_reset(cache);
+		write_seqcount_end(&peer->route_key_seq);
+	}
+	spin_unlock_bh(&peer->lock);
+}
+
+/**
+ * ovpn_dst_cache_current - check whether a route lookup matches peer state
+ * @peer: the peer owning the bind and dst cache
+ * @bind: the RCU bind used for the route lookup
+ * @key: the route key used for the route lookup
+ *
+ * Check that @bind is still the current peer bind and that @key still matches
+ * the peer route key. The caller must hold @peer->lock. The TX path keeps
+ * @bind inside an RCU read-side critical section, so pointer identity is enough
+ * to detect whether the bind was replaced while the route lookup was running.
+ *
+ * Return: true if the lookup result still matches the current peer state and
+ * may update the dst cache or replace the bind.
+ */
+static bool ovpn_dst_cache_current(const struct ovpn_peer *peer,
+				   const struct ovpn_bind *bind,
+				   const struct ovpn_route_key *key)
+{
+	const struct ovpn_bind *curr_bind;
+
+	lockdep_assert_held(&peer->lock);
+
+	curr_bind = rcu_dereference_protected(peer->bind,
+					      lockdep_is_held(&peer->lock));
+
+	return curr_bind == bind &&
+	       ovpn_route_key_equal(key, &peer->route_key);
+}
+
 /**
  * ovpn_udp4_output - send IPv4 packet over udp socket
  * @peer: the destination peer
@@ -138,21 +209,26 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
  * @cache: dst cache
  * @sk: the socket to send the packet over
  * @skb: the packet to send
+ * @key: the route key snapshot used for cache validation and flow lookup
  *
  * Return: 0 on success or a negative error code otherwise
  */
 static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 			    struct dst_cache *cache, struct sock *sk,
-			    struct sk_buff *skb)
+			    struct sk_buff *skb,
+			    const struct ovpn_route_key *key)
 {
+	struct sockaddr_storage remote;
+	struct in_addr local = {};
+	bool reset_local = false;
 	struct rtable *rt;
 	struct flowi4 fl = {
 		.saddr = bind->local.ipv4.s_addr,
 		.daddr = bind->remote.in4.sin_addr.s_addr,
-		.fl4_sport = inet_sk(sk)->inet_sport,
+		.fl4_sport = key->sport,
 		.fl4_dport = bind->remote.in4.sin_port,
 		.flowi4_proto = sk->sk_protocol,
-		.flowi4_mark = sk->sk_mark,
+		.flowi4_mark = key->mark,
 	};
 	int ret;
 
@@ -161,26 +237,19 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 	if (rt)
 		goto transmit;
 
-	if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr,
-					RT_SCOPE_HOST))) {
-		/* we may end up here when the cached address is not usable
-		 * anymore. In this case we reset address/cache and perform a
-		 * new look up
+	if (fl.saddr && unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0,
+						    fl.saddr, RT_SCOPE_HOST))) {
+		/* The learned local address is not usable anymore.
+		 * Retry with source address autoselection.
 		 */
 		fl.saddr = 0;
-		spin_lock_bh(&peer->lock);
-		bind->local.ipv4.s_addr = 0;
-		spin_unlock_bh(&peer->lock);
-		dst_cache_reset(cache);
+		reset_local = true;
 	}
 
 	rt = ip_route_output_flow(sock_net(sk), &fl, sk);
 	if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) {
 		fl.saddr = 0;
-		spin_lock_bh(&peer->lock);
-		bind->local.ipv4.s_addr = 0;
-		spin_unlock_bh(&peer->lock);
-		dst_cache_reset(cache);
+		reset_local = true;
 
 		rt = ip_route_output_flow(sock_net(sk), &fl, sk);
 	}
@@ -193,7 +262,30 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 				    ret);
 		goto err;
 	}
-	dst_cache_set_ip4(cache, &rt->dst, fl.saddr);
+
+	/* avoid storing a stale cache or local address */
+	spin_lock_bh(&peer->lock);
+	if (likely(ovpn_dst_cache_current(peer, bind, key))) {
+		if (!reset_local) {
+			dst_cache_set_ip4(cache, &rt->dst, fl.saddr);
+			spin_unlock_bh(&peer->lock);
+			goto transmit;
+		}
+
+		/* invalidate per-CPU dst entries that may still carry
+		 * the stale source
+		 */
+		dst_cache_reset(cache);
+
+		/* preserve the current remote */
+		memcpy(&remote, &bind->remote, sizeof(struct sockaddr_in));
+		/* The current packet already has a valid wildcard-source route.
+		 * If replacing the bind fails, leave the stale local in place;
+		 * a later cache miss will retry the repair.
+		 */
+		ovpn_peer_reset_sockaddr(peer, &remote, &local);
+	}
+	spin_unlock_bh(&peer->lock);
 
 transmit:
 	udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0,
@@ -213,23 +305,28 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
  * @cache: dst cache
  * @sk: the socket to send the packet over
  * @skb: the packet to send
+ * @key: the route key snapshot used for cache validation and flow lookup
  *
  * Return: 0 on success or a negative error code otherwise
  */
 static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 			    struct dst_cache *cache, struct sock *sk,
-			    struct sk_buff *skb)
+			    struct sk_buff *skb,
+			    const struct ovpn_route_key *key)
 {
+	struct in6_addr local = in6addr_any;
+	struct sockaddr_storage remote;
+	bool reset_local = false;
 	struct dst_entry *dst;
 	int ret;
 
 	struct flowi6 fl = {
 		.saddr = bind->local.ipv6,
 		.daddr = bind->remote.in6.sin6_addr,
-		.fl6_sport = inet_sk(sk)->inet_sport,
+		.fl6_sport = key->sport,
 		.fl6_dport = bind->remote.in6.sin6_port,
 		.flowi6_proto = sk->sk_protocol,
-		.flowi6_mark = sk->sk_mark,
+		.flowi6_mark = key->mark,
 		.flowi6_oif = bind->remote.in6.sin6_scope_id,
 	};
 
@@ -238,16 +335,13 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 	if (dst)
 		goto transmit;
 
-	if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) {
-		/* we may end up here when the cached address is not usable
-		 * anymore. In this case we reset address/cache and perform a
-		 * new look up
+	if (!ipv6_addr_any(&fl.saddr) &&
+	    unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) {
+		/* The learned local address is not usable anymore.
+		 * Retry with source address autoselection.
 		 */
 		fl.saddr = in6addr_any;
-		spin_lock_bh(&peer->lock);
-		bind->local.ipv6 = in6addr_any;
-		spin_unlock_bh(&peer->lock);
-		dst_cache_reset(cache);
+		reset_local = true;
 	}
 
 	dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL);
@@ -258,7 +352,30 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
 				    &bind->remote.in6, ret);
 		goto err;
 	}
-	dst_cache_set_ip6(cache, dst, &fl.saddr);
+
+	/* avoid storing a stale cache or local address */
+	spin_lock_bh(&peer->lock);
+	if (likely(ovpn_dst_cache_current(peer, bind, key))) {
+		if (!reset_local) {
+			dst_cache_set_ip6(cache, dst, &fl.saddr);
+			spin_unlock_bh(&peer->lock);
+			goto transmit;
+		}
+
+		/* invalidate per-CPU dst entries that may still carry
+		 * the stale source
+		 */
+		dst_cache_reset(cache);
+
+		/* preserve the current remote */
+		memcpy(&remote, &bind->remote, sizeof(struct sockaddr_in6));
+		/* The current packet already has a valid wildcard-source route.
+		 * If replacing the bind fails, leave the stale local in place;
+		 * a later cache miss will retry the repair.
+		 */
+		ovpn_peer_reset_sockaddr(peer, &remote, &local);
+	}
+	spin_unlock_bh(&peer->lock);
 
 transmit:
 	/* user IPv6 packets may be larger than the transport interface
@@ -287,6 +404,7 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
  * @cache: dst cache
  * @sk: the socket to send the packet over
  * @skb: the packet to send
+ * @key: route key snapshot used for cache validation and flow lookup
  *
  * rcu_read_lock should be held on entry.
  * On return, the skb is consumed.
@@ -294,7 +412,8 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind,
  * Return: 0 on success or a negative error code otherwise
  */
 static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache,
-			   struct sock *sk, struct sk_buff *skb)
+			   struct sock *sk, struct sk_buff *skb,
+			   struct ovpn_route_key *key)
 {
 	struct ovpn_bind *bind;
 	int ret;
@@ -314,11 +433,11 @@ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache,
 
 	switch (bind->remote.in4.sin_family) {
 	case AF_INET:
-		ret = ovpn_udp4_output(peer, bind, cache, sk, skb);
+		ret = ovpn_udp4_output(peer, bind, cache, sk, skb, key);
 		break;
 #if IS_ENABLED(CONFIG_IPV6)
 	case AF_INET6:
-		ret = ovpn_udp6_output(peer, bind, cache, sk, skb);
+		ret = ovpn_udp6_output(peer, bind, cache, sk, skb, key);
 		break;
 #endif
 	default:
@@ -340,15 +459,21 @@ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache,
 void ovpn_udp_send_skb(struct ovpn_peer *peer, struct sock *sk,
 		       struct sk_buff *skb)
 {
+	struct ovpn_route_key key = {
+		.mark = READ_ONCE(sk->sk_mark),
+		.sport = READ_ONCE(inet_sk(sk)->inet_sport),
+	};
 	int ret;
 
 	skb->dev = peer->ovpn->dev;
-	skb->mark = READ_ONCE(sk->sk_mark);
+	skb->mark = key.mark;
 	/* no checksum performed at this layer */
 	skb->ip_summed = CHECKSUM_NONE;
 
+	ovpn_dst_cache_check_key(peer, &peer->dst_cache, &key);
+
 	/* crypto layer -> transport (UDP) */
-	ret = ovpn_udp_output(peer, &peer->dst_cache, sk, skb);
+	ret = ovpn_udp_output(peer, &peer->dst_cache, sk, skb, &key);
 	if (unlikely(ret < 0))
 		kfree_skb(skb);
 }
diff --git a/drivers/net/pcs/pcs-lynx.c b/drivers/net/pcs/pcs-lynx.c
index a92081560e641a..6d94f92a333295 100644
--- a/drivers/net/pcs/pcs-lynx.c
+++ b/drivers/net/pcs/pcs-lynx.c
@@ -20,6 +20,12 @@
 #define IF_MODE_SPEED_MSK		GENMASK(3, 2)
 #define IF_MODE_HALF_DUPLEX		BIT(4)
 
+/* USXGMII replicator link timer step is 3.2 ns (312.5M XGMII columns per sec)
+ * for single port mode. For quad port mode, it is 1/4 of that.
+ */
+#define LINK_TIMER_VAL_USXGMII(ns)	((u32)((ns) * 10 / 32))
+#define LINK_TIMER_VAL_10G_QXGMII(ns)	((u32)((ns) * 10 / 128))
+
 struct lynx_pcs {
 	struct phylink_pcs pcs;
 	struct mdio_device *mdio;
@@ -46,6 +52,7 @@ static unsigned int lynx_pcs_inband_caps(struct phylink_pcs *pcs,
 		return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE;
 
 	case PHY_INTERFACE_MODE_10GBASER:
+	case PHY_INTERFACE_MODE_25GBASER:
 		return LINK_INBAND_DISABLE;
 
 	case PHY_INTERFACE_MODE_USXGMII:
@@ -97,6 +104,7 @@ static void lynx_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode,
 		lynx_pcs_get_state_usxgmii(lynx->mdio, state);
 		break;
 	case PHY_INTERFACE_MODE_10GBASER:
+	case PHY_INTERFACE_MODE_25GBASER:
 		phylink_mii_c45_pcs_get_state(lynx->mdio, state);
 		break;
 	default:
@@ -156,6 +164,9 @@ static int lynx_pcs_config_usxgmii(struct mdio_device *pcs,
 {
 	struct mii_bus *bus = pcs->bus;
 	int addr = pcs->addr;
+	int link_timer_ns;
+	u32 link_timer;
+	int ret;
 
 	if (neg_mode != PHYLINK_PCS_NEG_INBAND_ENABLED) {
 		dev_err(&pcs->dev, "%s only supports in-band AN for now\n",
@@ -164,10 +175,46 @@ static int lynx_pcs_config_usxgmii(struct mdio_device *pcs,
 	}
 
 	/* Configure device ability for the USXGMII Replicator */
-	return mdiobus_c45_write(bus, addr, MDIO_MMD_VEND2, MII_ADVERTISE,
-				 MDIO_USXGMII_10G | MDIO_USXGMII_LINK |
-				 MDIO_USXGMII_FULL_DUPLEX |
-				 ADVERTISE_SGMII | ADVERTISE_LPACK);
+	ret = mdiobus_c45_write(bus, addr, MDIO_MMD_VEND2, MII_ADVERTISE,
+				MDIO_USXGMII_10G | MDIO_USXGMII_LINK |
+				MDIO_USXGMII_FULL_DUPLEX |
+				ADVERTISE_SGMII | ADVERTISE_LPACK);
+	if (ret < 0) {
+		dev_err(&pcs->dev, "could not set USXGMII replicator config\n");
+		return ret;
+	}
+
+	link_timer_ns = phylink_get_link_timer_ns(interface);
+	if (link_timer_ns > 0) {
+		if (interface == PHY_INTERFACE_MODE_10G_QXGMII)
+			link_timer = LINK_TIMER_VAL_10G_QXGMII(link_timer_ns);
+		else
+			link_timer = LINK_TIMER_VAL_USXGMII(link_timer_ns);
+
+		ret = mdiobus_c45_write(bus, addr, MDIO_MMD_VEND2,
+					LINK_TIMER_LO, link_timer & 0xffff);
+		if (ret < 0) {
+			dev_err(&pcs->dev, "could not set USXGMII Link Timer 1\n");
+			return ret;
+		}
+
+		ret = mdiobus_c45_write(bus, addr, MDIO_MMD_VEND2,
+					LINK_TIMER_HI, (link_timer >> 16) & 0x1f);
+		if (ret < 0) {
+			dev_err(&pcs->dev, "could not set USXGMII Link Timer 2\n");
+			return ret;
+		}
+	}
+
+	/* Configure autonegotiation */
+	ret = mdiobus_c45_write(bus, addr, MDIO_MMD_VEND2, MII_BMCR,
+				BMCR_RESET | BMCR_ANENABLE | BMCR_ANRESTART);
+	if (ret < 0) {
+		dev_err(&pcs->dev, "could not set USXGMII replicator control config\n");
+		return ret;
+	}
+
+	return ret;
 }
 
 static int lynx_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
@@ -188,7 +235,8 @@ static int lynx_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
 		return lynx_pcs_config_usxgmii(lynx->mdio, ifmode, advertising,
 					       neg_mode);
 	case PHY_INTERFACE_MODE_10GBASER:
-		/* Nothing to do here for 10GBASER */
+	case PHY_INTERFACE_MODE_25GBASER:
+		/* Nothing to do here for 10GBASER and 25GBASER */
 		break;
 	default:
 		return -EOPNOTSUPP;
@@ -279,6 +327,7 @@ static const phy_interface_t lynx_interfaces[] = {
 	PHY_INTERFACE_MODE_1000BASEX,
 	PHY_INTERFACE_MODE_2500BASEX,
 	PHY_INTERFACE_MODE_10GBASER,
+	PHY_INTERFACE_MODE_25GBASER,
 	PHY_INTERFACE_MODE_USXGMII,
 	PHY_INTERFACE_MODE_10G_QXGMII,
 };
diff --git a/drivers/net/pcs/pcs-rzn1-miic.c b/drivers/net/pcs/pcs-rzn1-miic.c
index 2b72fa98ddf13e..3aa5134670609e 100644
--- a/drivers/net/pcs/pcs-rzn1-miic.c
+++ b/drivers/net/pcs/pcs-rzn1-miic.c
@@ -270,6 +270,9 @@ static struct miic_port *phylink_pcs_to_miic_port(struct phylink_pcs *pcs)
 
 static void miic_unlock_regs(struct miic *miic)
 {
+	/* Clear protection state */
+	writel(0x0000, miic->base + MIIC_PRCMD);
+
 	/* Unprotect register writes */
 	writel(0x00A5, miic->base + MIIC_PRCMD);
 	writel(0x0001, miic->base + MIIC_PRCMD);
@@ -683,7 +686,8 @@ static int miic_parse_dt(struct miic *miic, u32 *mode_cfg)
 	if (!dt_val)
 		return -ENOMEM;
 
-	memset(dt_val, MIIC_MODCTRL_CONF_NONE, sizeof(*dt_val));
+	memset(dt_val, MIIC_MODCTRL_CONF_NONE,
+	       sizeof(*dt_val) * miic->of_data->conf_conv_count);
 
 	if (of_property_read_u32(np, "renesas,miic-switch-portin", &conf) == 0)
 		dt_val[0] = conf;
diff --git a/drivers/net/pcs/pcs-xpcs-plat.c b/drivers/net/pcs/pcs-xpcs-plat.c
index f4b1b8246ce968..e669a7b248ac46 100644
--- a/drivers/net/pcs/pcs-xpcs-plat.c
+++ b/drivers/net/pcs/pcs-xpcs-plat.c
@@ -438,7 +438,7 @@ static const struct of_device_id xpcs_of_ids[] = {
 	{ .compatible = "snps,dw-xpcs-gen4-6g", .data = &xpcs_pma_gen4_6g },
 	{ .compatible = "snps,dw-xpcs-gen5-10g", .data = &xpcs_pma_gen5_10g },
 	{ .compatible = "snps,dw-xpcs-gen5-12g", .data = &xpcs_pma_gen5_12g },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, xpcs_of_ids);
 
diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c
index 0337e2bcc01258..b415b93d77c154 100644
--- a/drivers/net/pcs/pcs-xpcs.c
+++ b/drivers/net/pcs/pcs-xpcs.c
@@ -1545,8 +1545,10 @@ static int xpcs_init_clks(struct dw_xpcs *xpcs)
 		return dev_err_probe(dev, ret, "Failed to get clocks\n");
 
 	ret = clk_bulk_prepare_enable(DW_XPCS_NUM_CLKS, xpcs->clks);
-	if (ret)
+	if (ret) {
+		clk_bulk_put(DW_XPCS_NUM_CLKS, xpcs->clks);
 		return dev_err_probe(dev, ret, "Failed to enable clocks\n");
+	}
 
 	return 0;
 }
diff --git a/drivers/net/pfcp.c b/drivers/net/pfcp.c
index e1cca779d2eccd..3ecd43cafdf03d 100644
--- a/drivers/net/pfcp.c
+++ b/drivers/net/pfcp.c
@@ -105,6 +105,9 @@ static int pfcp_encap_recv(struct sock *sk, struct sk_buff *skb)
 
 static void pfcp_del_sock(struct pfcp_dev *pfcp)
 {
+	if (!pfcp->sk)
+		return;
+
 	udp_tunnel_sock_release(pfcp->sk);
 	pfcp->sk = NULL;
 }
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index b4ef927fd4a67c..d3835597e3795a 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -484,6 +484,17 @@ config VITESSE_PHY
 	help
 	  Currently supports the vsc8244
 
+config XPOWERS_ACX00_PHY
+	tristate "X-Powers AC200/AC300 Ethernet PHY"
+	depends on COMMON_CLK && NVMEM && OF_MDIO && REGULATOR
+	depends on I2C || !I2C
+	select PHY_PACKAGE
+	help
+	  Enable the Fast Ethernet PHY driver shared by the X-Powers AC200
+	  and AC300 companion ICs. The driver configures their common MDIO PHY
+	  registers and the package-specific AC200 I2C or AC300 MDIO control
+	  registers.
+
 config XILINX_GMII2RGMII
 	tristate "Xilinx GMII2RGMII converter driver"
 	help
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index 25c4a3c2429f79..579066a638f074 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -8,7 +8,7 @@ libphy-y			:= phy.o phy-c45.o phy-core.o phy_device.o \
 
 ifdef CONFIG_PHYLIB
 # built-in whenever PHYLIB is built-in or module
-obj-y				+= stubs.o
+obj-y				+= stubs.o phy_fixup.o
 endif
 
 libphy-$(CONFIG_SWPHY)		+= swphy.o
@@ -101,4 +101,5 @@ obj-$(CONFIG_SMSC_PHY)		+= smsc.o
 obj-$(CONFIG_STE10XP)		+= ste10Xp.o
 obj-$(CONFIG_TERANETICS_PHY)	+= teranetics.o
 obj-$(CONFIG_VITESSE_PHY)	+= vitesse.o
+obj-$(CONFIG_XPOWERS_ACX00_PHY)	+= xpowers/
 obj-$(CONFIG_XILINX_GMII2RGMII) += xilinx_gmii2rgmii.o
diff --git a/drivers/net/phy/air_en8811h.c b/drivers/net/phy/air_en8811h.c
index 0eeb7b9a4e260b..a03d63dd4ac916 100644
--- a/drivers/net/phy/air_en8811h.c
+++ b/drivers/net/phy/air_en8811h.c
@@ -145,10 +145,22 @@
 
 #define AN8811HB_GPIO_OUTPUT		0x5cf8b8
 #define   AN8811HB_GPIO_OUTPUT_345		(BIT(3) | BIT(4) | BIT(5))
+#define   AN8811HB_GPIO_OUTPUT_0115		(BIT(0) | BIT(1) | BIT(15))
+
+#define AN8811HB_GPIO_SEL1		0x5cf8bc
+#define   AN8811HB_GPIO_SEL1_0_MASK		GENMASK(3, 0)
+#define   AN8811HB_GPIO_SEL1_1_MASK		GENMASK(7, 4)
+#define   AN8811HB_GPIO_SEL1_0			BIT(0)
+#define   AN8811HB_GPIO_SEL1_1			0
+
+#define AN8811HB_GPIO_SEL2		0x5cf8c0
+#define   AN8811HB_GPIO_SEL2_15_MASK		GENMASK(31, 28)
+#define   AN8811HB_GPIO_SEL2_15			BIT(29)
 
 #define AN8811HB_HWTRAP1		0x5cf910
 #define AN8811HB_HWTRAP2		0x5cf914
 #define   AN8811HB_HWTRAP2_CKO			BIT(28)
+#define   AN8811HB_HWTRAP2_PKG			GENMASK(14, 12)
 
 #define AN8811HB_CLK_DRV		0x5cf9e4
 #define AN8811HB_CLK_DRV_CKO_MASK		GENMASK(14, 12)
@@ -202,6 +214,7 @@ struct en8811h_priv {
 	struct phy_device	*phydev;
 	unsigned int		cko_is_enabled;
 	struct mdio_device	*pbusdev;
+	bool			is_an8811hbcn;
 };
 
 enum {
@@ -312,6 +325,12 @@ static int air_write_buf(struct phy_device *phydev, u32 address,
 	int saved_page;
 	int ret = 0;
 
+	if (fw->size % 4) {
+		phydev_err(phydev, "firmware size %zu is not a multiple of 4\n",
+			   fw->size);
+		return -EINVAL;
+	}
+
 	saved_page = phy_select_page(phydev, AIR_PHY_PAGE_EXTENDED_4);
 
 	if (saved_page >= 0) {
@@ -932,7 +951,7 @@ static const struct clk_ops an8811hb_clk_ops = {
 
 static int an8811hb_clk_provider_setup(struct device *dev, struct clk_hw *hw)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int ret;
 
 	if (!IS_ENABLED(CONFIG_COMMON_CLK))
@@ -1031,7 +1050,7 @@ static const struct clk_ops en8811h_clk_ops = {
 
 static int en8811h_clk_provider_setup(struct device *dev, struct clk_hw *hw)
 {
-	struct clk_init_data init;
+	struct clk_init_data init = {};
 	int ret;
 
 	if (!IS_ENABLED(CONFIG_COMMON_CLK))
@@ -1071,10 +1090,49 @@ static int en8811h_leds_setup(struct phy_device *phydev)
 	return ret;
 }
 
+static int an8811hb_led_gpio_setup(struct phy_device *phydev)
+{
+	struct en8811h_priv *priv = phydev->priv;
+	int ret;
+
+	if (priv->is_an8811hbcn) {
+		/* AN8811HBCN: LED GPIOs are 0, 1, 15 */
+		ret = air_phy_buckpbus_reg_modify(phydev, AN8811HB_GPIO_OUTPUT,
+						  AN8811HB_GPIO_OUTPUT_0115,
+						  AN8811HB_GPIO_OUTPUT_0115);
+		if (ret < 0)
+			return ret;
+
+		ret = air_phy_buckpbus_reg_modify(phydev, AN8811HB_GPIO_SEL1,
+						  AN8811HB_GPIO_SEL1_0_MASK |
+						  AN8811HB_GPIO_SEL1_1_MASK,
+						  AN8811HB_GPIO_SEL1_0 |
+						  AN8811HB_GPIO_SEL1_1);
+		if (ret < 0)
+			return ret;
+
+		ret = air_phy_buckpbus_reg_modify(phydev, AN8811HB_GPIO_SEL2,
+						  AN8811HB_GPIO_SEL2_15_MASK,
+						  AN8811HB_GPIO_SEL2_15);
+		if (ret < 0)
+			return ret;
+	} else {
+		/* AN8811HBN: LED GPIOs are 3, 4, 5 */
+		ret = air_phy_buckpbus_reg_modify(phydev, AN8811HB_GPIO_OUTPUT,
+						  AN8811HB_GPIO_OUTPUT_345,
+						  AN8811HB_GPIO_OUTPUT_345);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
 static int an8811hb_probe(struct phy_device *phydev)
 {
 	struct mdio_device *mdiodev;
 	struct en8811h_priv *priv;
+	u32 reg_val;
 	int ret;
 
 	priv = devm_kzalloc(&phydev->mdio.dev, sizeof(struct en8811h_priv),
@@ -1115,6 +1173,12 @@ static int an8811hb_probe(struct phy_device *phydev)
 	/* MDIO_DEVS1/2 empty, so set mmds_present bits here */
 	phydev->c45_ids.mmds_present |= MDIO_DEVS_PMAPMD | MDIO_DEVS_AN;
 
+	/* Detect package variant */
+	ret = air_phy_buckpbus_reg_read(phydev, AN8811HB_HWTRAP2, &reg_val);
+	if (ret < 0)
+		goto err_dev_create;
+	priv->is_an8811hbcn = FIELD_GET(AN8811HB_HWTRAP2_PKG, reg_val);
+
 	ret = en8811h_leds_setup(phydev);
 	if (ret < 0)
 		goto err_dev_create;
@@ -1125,13 +1189,6 @@ static int an8811hb_probe(struct phy_device *phydev)
 	if (ret)
 		goto err_dev_create;
 
-	/* Configure led gpio pins as output */
-	ret = air_phy_buckpbus_reg_modify(phydev, AN8811HB_GPIO_OUTPUT,
-					  AN8811HB_GPIO_OUTPUT_345,
-					  AN8811HB_GPIO_OUTPUT_345);
-	if (ret < 0)
-		goto err_dev_create;
-
 	return 0;
 
 err_dev_create:
@@ -1265,9 +1322,16 @@ static int an8811hb_config_init(struct phy_device *phydev)
 
 	ret = air_leds_init(phydev, EN8811H_LED_COUNT, AIR_PHY_LED_DUR,
 			    AIR_LED_MODE_USER_DEFINE);
-	if (ret < 0)
+	if (ret < 0) {
 		phydev_err(phydev, "Failed to initialize leds: %d\n", ret);
+		return ret;
+	}
 
+	ret = an8811hb_led_gpio_setup(phydev);
+	if (ret < 0) {
+		phydev_err(phydev, "Failed to set up LED GPIO: %d\n", ret);
+		return ret;
+	}
 	return ret;
 }
 
diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c
index d1a4edb34ad2ef..b515e38755041d 100644
--- a/drivers/net/phy/broadcom.c
+++ b/drivers/net/phy/broadcom.c
@@ -537,6 +537,13 @@ static int bcm54xx_config_init(struct phy_device *phydev)
 
 	bcm54xx_ptp_config_init(phydev);
 
+	/* Transmit is limited to 4.5 KB without it, see the BCM54210PE
+	 * datasheet section 5.4.12.14.
+	 */
+	err = bcm_phy_enable_jumbo(phydev);
+	if (err < 0)
+		return err;
+
 	/* Acknowledge any left over interrupt and charge the device for
 	 * wake-up.
 	 */
diff --git a/drivers/net/phy/dp83848.c b/drivers/net/phy/dp83848.c
index d88b1999d59629..e65651ac567a7a 100644
--- a/drivers/net/phy/dp83848.c
+++ b/drivers/net/phy/dp83848.c
@@ -117,6 +117,9 @@ static int dp83848_config_init(struct phy_device *phydev)
 	 * we check initial value of BMCR Auto negotiation enable bit
 	 */
 	val = phy_read(phydev, MII_BMCR);
+	if (val < 0)
+		return val;
+
 	if (!(val & BMCR_ANENABLE))
 		phydev->autoneg = AUTONEG_DISABLE;
 
diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c
index 61a941aa02d944..a3cd5cb11ea0ac 100644
--- a/drivers/net/phy/dp83867.c
+++ b/drivers/net/phy/dp83867.c
@@ -8,6 +8,7 @@
 #include <linux/kernel.h>
 #include <linux/mii.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/phy.h>
 #include <linux/delay.h>
@@ -196,6 +197,16 @@ struct dp83867_private {
 	bool set_clk_output;
 	u32 clk_output_sel;
 	bool sgmii_ref_clk_en;
+
+	/* Shadow of the LED registers, replayed after a soft reset. led_lock
+	 * serializes the shadow and its replay against the LED callbacks,
+	 * because dp83867_config_init() does the replay off phydev->lock.
+	 */
+	struct mutex led_lock;
+	u16 ledcr1;
+	u16 ledcr1_mask;
+	u16 ledcr2;
+	u16 ledcr2_mask;
 };
 
 static int dp83867_ack_interrupt(struct phy_device *phydev)
@@ -722,6 +733,7 @@ static int dp83867_resume(struct phy_device *phydev)
 static int dp83867_probe(struct phy_device *phydev)
 {
 	struct dp83867_private *dp83867;
+	int ret;
 
 	dp83867 = devm_kzalloc(&phydev->mdio.dev, sizeof(*dp83867),
 			       GFP_KERNEL);
@@ -730,9 +742,60 @@ static int dp83867_probe(struct phy_device *phydev)
 
 	phydev->priv = dp83867;
 
+	ret = devm_mutex_init(&phydev->mdio.dev, &dp83867->led_lock);
+	if (ret)
+		return ret;
+
 	return dp83867_of_init(phydev);
 }
 
+/* Update an LED register and mirror the change into the shadow, so that
+ * dp83867_config_init() can replay it after a soft reset. Caller must hold
+ * dp83867->led_lock.
+ */
+static int __dp83867_led_modify(struct phy_device *phydev, u32 reg,
+				u16 mask, u16 val)
+{
+	struct dp83867_private *dp83867 = phydev->priv;
+	int ret;
+
+	ret = phy_modify(phydev, reg, mask, val);
+	if (ret)
+		return ret;
+
+	if (reg == DP83867_LEDCR1) {
+		dp83867->ledcr1 = (dp83867->ledcr1 & ~mask) | (val & mask);
+		dp83867->ledcr1_mask |= mask;
+	} else if (reg == DP83867_LEDCR2) {
+		dp83867->ledcr2 = (dp83867->ledcr2 & ~mask) | (val & mask);
+		dp83867->ledcr2_mask |= mask;
+	} else {
+		WARN_ON_ONCE(1);
+	}
+
+	return 0;
+}
+
+/* Restore the LED registers the driver has programmed, cleared by the soft
+ * reset in dp83867_phy_reset().
+ */
+static int dp83867_led_restore(struct phy_device *phydev)
+{
+	struct dp83867_private *dp83867 = phydev->priv;
+	int ret = 0;
+
+	mutex_lock(&dp83867->led_lock);
+	if (dp83867->ledcr1_mask)
+		ret = phy_modify(phydev, DP83867_LEDCR1,
+				 dp83867->ledcr1_mask, dp83867->ledcr1);
+	if (!ret && dp83867->ledcr2_mask)
+		ret = phy_modify(phydev, DP83867_LEDCR2,
+				 dp83867->ledcr2_mask, dp83867->ledcr2);
+	mutex_unlock(&dp83867->led_lock);
+
+	return ret;
+}
+
 static int dp83867_config_init(struct phy_device *phydev)
 {
 	struct dp83867_private *dp83867 = phydev->priv;
@@ -896,6 +959,10 @@ static int dp83867_config_init(struct phy_device *phydev)
 			       mask, val);
 	}
 
+	ret = dp83867_led_restore(phydev);
+	if (ret)
+		phydev_warn(phydev, "failed to restore LED config: %d\n", ret);
+
 	return 0;
 }
 
@@ -1004,7 +1071,9 @@ static int
 dp83867_led_brightness_set(struct phy_device *phydev,
 			   u8 index, enum led_brightness brightness)
 {
-	u32 val;
+	struct dp83867_private *dp83867 = phydev->priv;
+	u16 val;
+	int ret;
 
 	if (index >= DP83867_LED_COUNT)
 		return -EINVAL;
@@ -1015,10 +1084,13 @@ dp83867_led_brightness_set(struct phy_device *phydev,
 	if (brightness)
 		val |= DP83867_LED_DRV_VAL(index);
 
-	return phy_modify(phydev, DP83867_LEDCR2,
-			  DP83867_LED_DRV_VAL(index) |
-			  DP83867_LED_DRV_EN(index),
-			  val);
+	mutex_lock(&dp83867->led_lock);
+	ret = __dp83867_led_modify(phydev, DP83867_LEDCR2,
+				   DP83867_LED_DRV_VAL(index) |
+				   DP83867_LED_DRV_EN(index), val);
+	mutex_unlock(&dp83867->led_lock);
+
+	return ret;
 }
 
 static int dp83867_led_mode(u8 index, unsigned long rules)
@@ -1069,18 +1141,24 @@ static int dp83867_led_hw_is_supported(struct phy_device *phydev, u8 index,
 static int dp83867_led_hw_control_set(struct phy_device *phydev, u8 index,
 				      unsigned long rules)
 {
+	struct dp83867_private *dp83867 = phydev->priv;
 	int mode, ret;
 
 	mode = dp83867_led_mode(index, rules);
 	if (mode < 0)
 		return mode;
 
-	ret = phy_modify(phydev, DP83867_LEDCR1, DP83867_LED_FN_MASK(index),
-			 DP83867_LED_FN(index, mode));
-	if (ret)
-		return ret;
-
-	return phy_modify(phydev, DP83867_LEDCR2, DP83867_LED_DRV_EN(index), 0);
+	mutex_lock(&dp83867->led_lock);
+	ret = __dp83867_led_modify(phydev, DP83867_LEDCR1,
+				   DP83867_LED_FN_MASK(index),
+				   DP83867_LED_FN(index, mode));
+	if (!ret)
+		ret = __dp83867_led_modify(phydev, DP83867_LEDCR2,
+					   DP83867_LED_DRV_EN(index) |
+					   DP83867_LED_DRV_VAL(index), 0);
+	mutex_unlock(&dp83867->led_lock);
+
+	return ret;
 }
 
 static int dp83867_led_hw_control_get(struct phy_device *phydev, u8 index,
@@ -1141,9 +1219,11 @@ static int dp83867_led_hw_control_get(struct phy_device *phydev, u8 index,
 static int dp83867_led_polarity_set(struct phy_device *phydev, int index,
 				    unsigned long modes)
 {
+	struct dp83867_private *dp83867 = phydev->priv;
 	/* Default active high */
 	u16 polarity = DP83867_LED_POLARITY(index);
 	u32 mode;
+	int ret;
 
 	for_each_set_bit(mode, &modes, __PHY_LED_MODES_NUM) {
 		switch (mode) {
@@ -1157,8 +1237,13 @@ static int dp83867_led_polarity_set(struct phy_device *phydev, int index,
 			return -EINVAL;
 		}
 	}
-	return phy_modify(phydev, DP83867_LEDCR2,
-			  DP83867_LED_POLARITY(index), polarity);
+
+	mutex_lock(&dp83867->led_lock);
+	ret = __dp83867_led_modify(phydev, DP83867_LEDCR2,
+				   DP83867_LED_POLARITY(index), polarity);
+	mutex_unlock(&dp83867->led_lock);
+
+	return ret;
 }
 
 static unsigned int dp83867_inband_caps(struct phy_device *phydev,
diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c
index afbcec71174433..3cee31bb931f23 100644
--- a/drivers/net/phy/intel-xway.c
+++ b/drivers/net/phy/intel-xway.c
@@ -16,6 +16,11 @@
 #define XWAY_MDIO_ISTAT			0x1A	/* interrupt status */
 #define XWAY_MDIO_LED			0x1B	/* led control */
 
+#define XWAY_MDIO_GCTRL_TM_MASK		GENMASK(15, 13)
+#define XWAY_MDIO_GCTRL_TM(mode)	FIELD_PREP(XWAY_MDIO_GCTRL_TM_MASK, (mode))
+#define XWAY_MDIO_GCTRL_TM_NOP		XWAY_MDIO_GCTRL_TM(0)	/* Normal operation */
+#define XWAY_MDIO_GCTRL_TM_CDIAG	XWAY_MDIO_GCTRL_TM(6)	/* Cable diagnostics */
+
 #define XWAY_MDIO_ERRCNT_SEL		GENMASK(11, 8)
 #define XWAY_MDIO_ERRCNT_COUNT		GENMASK(7, 0)
 #define XWAY_MDIO_ERRCNT_SEL_RXERR	0
@@ -326,6 +331,28 @@ static int xway_gphy_probe(struct phy_device *phydev)
 	return 0;
 }
 
+static int xway_11g_int_config_init(struct phy_device *phydev)
+{
+	int err;
+
+	/* An issue has been sporadically observed after device power-on on the
+	 * first link-up attempt in 100BASE-TX mode resulting in either the
+	 * link-up taking a long time, or failing to link-up altogether.
+	 *
+	 * Workaround:
+	 * After power-on, enable Cable Diagnostic Mode for all ports and
+	 * disable it.
+	 */
+	err = phy_modify(phydev, MII_CTRL1000, XWAY_MDIO_GCTRL_TM_MASK, XWAY_MDIO_GCTRL_TM_CDIAG);
+	if (err)
+		return err;
+	err = phy_modify(phydev, MII_CTRL1000, XWAY_MDIO_GCTRL_TM_MASK, XWAY_MDIO_GCTRL_TM_NOP);
+	if (err)
+		return err;
+
+	return xway_gphy_config_init(phydev);
+}
+
 static int xway_gphy14_config_aneg(struct phy_device *phydev)
 {
 	int reg, err;
@@ -735,7 +762,7 @@ static struct phy_driver xway_gphy[] = {
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY11G (xRX v1.2 integrated)",
 		/* PHY_GBIT_FEATURES */
-		.config_init	= xway_gphy_config_init,
+		.config_init	= xway_11g_int_config_init,
 		.probe		= xway_gphy_probe,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
diff --git a/drivers/net/phy/mediatek/mtk-phy-lib.c b/drivers/net/phy/mediatek/mtk-phy-lib.c
index 608072fbfde97e..4c4eca870bde1c 100644
--- a/drivers/net/phy/mediatek/mtk-phy-lib.c
+++ b/drivers/net/phy/mediatek/mtk-phy-lib.c
@@ -136,22 +136,15 @@ int mtk_phy_led_hw_ctrl_get(struct phy_device *phydev, u8 index,
 	if (blink < 0)
 		return -EIO;
 
-	if ((on & (on_set | MTK_PHY_LED_ON_FDX |
-		   MTK_PHY_LED_ON_HDX | MTK_PHY_LED_ON_LINKDOWN)) ||
-	    (blink & (rx_blink_set | tx_blink_set)))
-		set_bit(bit_netdev, &priv->led_state);
-	else
-		clear_bit(bit_netdev, &priv->led_state);
+	assign_bit(bit_netdev, &priv->led_state,
+		   (on & (on_set | MTK_PHY_LED_ON_FDX | MTK_PHY_LED_ON_HDX |
+		    MTK_PHY_LED_ON_LINKDOWN)) ||
+		    (blink & (rx_blink_set | tx_blink_set)));
 
-	if (on & MTK_PHY_LED_ON_FORCE_ON)
-		set_bit(bit_on, &priv->led_state);
-	else
-		clear_bit(bit_on, &priv->led_state);
+	assign_bit(bit_on, &priv->led_state, on & MTK_PHY_LED_ON_FORCE_ON);
 
-	if (blink & MTK_PHY_LED_BLINK_FORCE_BLINK)
-		set_bit(bit_blink, &priv->led_state);
-	else
-		clear_bit(bit_blink, &priv->led_state);
+	assign_bit(bit_blink, &priv->led_state,
+		   blink & MTK_PHY_LED_BLINK_FORCE_BLINK);
 
 	if (!rules)
 		return 0;
@@ -254,10 +247,7 @@ int mtk_phy_led_hw_ctrl_set(struct phy_device *phydev, u8 index,
 		}
 	}
 
-	if (blink || on)
-		set_bit(bit_netdev, &priv->led_state);
-	else
-		clear_bit(bit_netdev, &priv->led_state);
+	assign_bit(bit_netdev, &priv->led_state, blink || on);
 
 	ret = phy_modify_mmd(phydev, MDIO_MMD_VEND2, index ?
 			     MTK_PHY_LED1_ON_CTRL : MTK_PHY_LED0_ON_CTRL,
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index ae830781824b26..5c8461db7b4b6e 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -6344,6 +6344,7 @@ static int lanphy_write_reg_data(struct phy_device *phydev,
 					    data->val);
 		if (ret)
 			break;
+		data++;
 	}
 
 	return ret;
diff --git a/drivers/net/phy/microchip_t1.c b/drivers/net/phy/microchip_t1.c
index 3292b2235c8fa8..e38d20bf6aa4c8 100644
--- a/drivers/net/phy/microchip_t1.c
+++ b/drivers/net/phy/microchip_t1.c
@@ -1285,14 +1285,16 @@ static int lan887x_phy_init(struct phy_device *phydev)
 		if (IS_ERR(priv->clock))
 			return PTR_ERR(priv->clock);
 
-		/* Enable pin mux for EVT */
-		phy_modify_mmd(phydev, MDIO_MMD_VEND1,
-			       LAN887X_MX_CHIP_TOP_REG_CONTROL1,
-			       LAN887X_MX_CHIP_TOP_REG_CONTROL1_EVT_EN,
-			       LAN887X_MX_CHIP_TOP_REG_CONTROL1_EVT_EN);
-
-		/* Initialize pin numbers specific to PEROUT */
-		priv->clock->event_pin = 3;
+		if (priv->clock) {
+			/* Enable pin mux for EVT */
+			phy_modify_mmd(phydev, MDIO_MMD_VEND1,
+				       LAN887X_MX_CHIP_TOP_REG_CONTROL1,
+				       LAN887X_MX_CHIP_TOP_REG_CONTROL1_EVT_EN,
+				       LAN887X_MX_CHIP_TOP_REG_CONTROL1_EVT_EN);
+
+			/* Initialize pin numbers specific to PEROUT */
+			priv->clock->event_pin = 3;
+		}
 
 		priv->init_done = true;
 	}
diff --git a/drivers/net/phy/motorcomm.c b/drivers/net/phy/motorcomm.c
index c5a2cda8d31bd9..90a4f86f2758b1 100644
--- a/drivers/net/phy/motorcomm.c
+++ b/drivers/net/phy/motorcomm.c
@@ -144,6 +144,11 @@
 #define YT8521_CLOCK_GATING_REG			0xC
 #define YT8521_CGR_RX_CLK_EN			BIT(12)
 
+/* Analog front-end control register 3 */
+#define YT8531S_EXT_AFE_CTRL3			0x12
+/* Analog front-end DAC clock enable */
+#define YT8531S_AFE_CTRL3_CLKDAC_AON		BIT(13)
+
 #define YT8521_EXTREG_SLEEP_CONTROL1_REG	0x27
 #define YT8521_ESC1R_SLEEP_SW			BIT(15)
 #define YT8521_ESC1R_PLLON_SLP			BIT(14)
@@ -1680,27 +1685,16 @@ static int yt8521_resume(struct phy_device *phydev)
 	return yt8521_modify_utp_fiber_bmcr(phydev, BMCR_PDOWN, 0);
 }
 
-/**
- * yt8521_config_init() - called to initialize the PHY
- * @phydev: a pointer to a &struct phy_device
- *
- * returns 0 or negative errno code
- */
-static int yt8521_config_init(struct phy_device *phydev)
+static int __yt8521_config_init(struct phy_device *phydev)
 {
 	struct device *dev = &phydev->mdio.dev;
-	int old_page;
 	int ret = 0;
 
-	old_page = phy_select_page(phydev, YT8521_RSSR_UTP_SPACE);
-	if (old_page < 0)
-		goto err_restore_page;
-
 	/* set rgmii delay mode */
 	if (phydev->interface != PHY_INTERFACE_MODE_SGMII) {
 		ret = ytphy_rgmii_clk_delay_config(phydev);
 		if (ret < 0)
-			goto err_restore_page;
+			return ret;
 	}
 
 	if (device_property_read_bool(dev, "motorcomm,auto-sleep-disabled")) {
@@ -1708,20 +1702,58 @@ static int yt8521_config_init(struct phy_device *phydev)
 		ret = ytphy_modify_ext(phydev, YT8521_EXTREG_SLEEP_CONTROL1_REG,
 				       YT8521_ESC1R_SLEEP_SW, 0);
 		if (ret < 0)
-			goto err_restore_page;
+			return ret;
 	}
 
-	if (device_property_read_bool(dev, "motorcomm,keep-pll-enabled")) {
+	if (device_property_read_bool(dev, "motorcomm,keep-pll-enabled"))
 		/* enable RXC clock when no wire plug */
-		ret = ytphy_modify_ext(phydev, YT8521_CLOCK_GATING_REG,
-				       YT8521_CGR_RX_CLK_EN, 0);
-		if (ret < 0)
+		return ytphy_modify_ext(phydev, YT8521_CLOCK_GATING_REG,
+					YT8521_CGR_RX_CLK_EN, 0);
+
+	return 0;
+}
+
+/**
+ * yt8521_config_init() - called to initialize the PHY
+ * @phydev: a pointer to a &struct phy_device
+ *
+ * returns 0 or negative errno code
+ */
+static int yt8521_config_init(struct phy_device *phydev)
+{
+	int old_page, ret = 0;
+
+	old_page = phy_select_page(phydev, YT8521_RSSR_UTP_SPACE);
+	if (old_page < 0)
+		goto err_restore_page;
+
+	ret = __yt8521_config_init(phydev);
+
+err_restore_page:
+	return phy_restore_page(phydev, old_page, ret);
+}
+
+static int yt8531s_config_init(struct phy_device *phydev)
+{
+	int old_page, ret = 0;
+
+	old_page = phy_select_page(phydev, YT8521_RSSR_UTP_SPACE);
+	if (old_page < 0)
+		goto err_restore_page;
+
+	ret = __yt8521_config_init(phydev);
+	if (ret)
+		goto err_restore_page;
+
+	if (phy_interface_is_rgmii(phydev)) {
+		ret = yt8531_set_ds(phydev);
+		if (ret)
 			goto err_restore_page;
 	}
 
-	if (phy_interface_is_rgmii(phydev) &&
-	    phydev_id_compare(phydev, PHY_ID_YT8531S))
-		ret = yt8531_set_ds(phydev);
+	if (phydev->interface == PHY_INTERFACE_MODE_GMII)
+		ret = ytphy_modify_ext(phydev, YT8531S_EXT_AFE_CTRL3,
+				       0, YT8531S_AFE_CTRL3_CLKDAC_AON);
 
 err_restore_page:
 	return phy_restore_page(phydev, old_page, ret);
@@ -3135,7 +3167,7 @@ static struct phy_driver motorcomm_phy_drvs[] = {
 		.set_wol	= ytphy_set_wol,
 		.config_aneg	= yt8521_config_aneg,
 		.aneg_done	= yt8521_aneg_done,
-		.config_init	= yt8521_config_init,
+		.config_init	= yt8531s_config_init,
 		.read_status	= yt8521_read_status,
 		.soft_reset	= yt8521_soft_reset,
 		.suspend	= yt8521_suspend,
diff --git a/drivers/net/phy/mxl-gpy.c b/drivers/net/phy/mxl-gpy.c
index 0da2d4e9d85481..089a1a9e5bc613 100644
--- a/drivers/net/phy/mxl-gpy.c
+++ b/drivers/net/phy/mxl-gpy.c
@@ -1405,7 +1405,7 @@ static struct phy_driver gpy_drivers[] = {
 		PHY_ID_MATCH_MODEL(PHY_ID_MXL86211C),
 		.name		= "Maxlinear Ethernet MxL86211C",
 		.get_features	= genphy_c45_pma_read_abilities,
-		.config_init	= gpy_config_init,
+		.config_init	= gpy21x_config_init,
 		.probe		= gpy_probe,
 		.inband_caps	= gpy_inband_caps,
 		.config_inband	= gpy_config_inband,
diff --git a/drivers/net/phy/nxp-c45-tja11xx.c b/drivers/net/phy/nxp-c45-tja11xx.c
index f526528d2e3289..8df152f6671fc1 100644
--- a/drivers/net/phy/nxp-c45-tja11xx.c
+++ b/drivers/net/phy/nxp-c45-tja11xx.c
@@ -2180,7 +2180,7 @@ module_phy_driver(nxp_c45_driver);
 static const struct mdio_device_id __maybe_unused nxp_c45_tbl[] = {
 	{ PHY_ID_MATCH_MODEL(PHY_ID_TJA_1103) },
 	{ PHY_ID_MATCH_MODEL(PHY_ID_TJA_1120) },
-	{ /*sentinel*/ },
+	{ /*sentinel*/ }
 };
 
 MODULE_DEVICE_TABLE(mdio, nxp_c45_tbl);
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index 94b2e85e00a379..5b13a74e2fa921 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -49,14 +49,6 @@ MODULE_DESCRIPTION("PHY library");
 MODULE_AUTHOR("Andy Fleming");
 MODULE_LICENSE("GPL");
 
-struct phy_fixup {
-	struct list_head list;
-	char bus_id[MII_BUS_ID_SIZE + 3];
-	u32 phy_uid;
-	u32 phy_uid_mask;
-	int (*run)(struct phy_device *phydev);
-};
-
 static struct phy_driver genphy_c45_driver = {
 	.phy_id         = 0xffffffff,
 	.phy_id_mask    = 0xffffffff,
@@ -237,9 +229,6 @@ static void phy_mdio_device_remove(struct mdio_device *mdiodev)
 
 static struct phy_driver genphy_driver;
 
-static LIST_HEAD(phy_fixup_list);
-static DEFINE_MUTEX(phy_fixup_lock);
-
 static bool phy_drv_wol_enabled(struct phy_device *phydev)
 {
 	struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
@@ -427,85 +416,6 @@ static __maybe_unused int mdio_bus_phy_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(mdio_bus_phy_pm_ops, mdio_bus_phy_suspend,
 			 mdio_bus_phy_resume);
 
-/**
- * phy_register_fixup - creates a new phy_fixup and adds it to the list
- * @bus_id: A string which matches phydev->mdio.dev.bus_id (or NULL)
- * @phy_uid: Used to match against phydev->phy_id (the UID of the PHY)
- * @phy_uid_mask: Applied to phydev->phy_id and fixup->phy_uid before
- *	comparison (or 0 to disable id-based matching)
- * @run: The actual code to be run when a matching PHY is found
- */
-static int phy_register_fixup(const char *bus_id, u32 phy_uid, u32 phy_uid_mask,
-			      int (*run)(struct phy_device *))
-{
-	struct phy_fixup *fixup = kzalloc_obj(*fixup);
-
-	if (!fixup)
-		return -ENOMEM;
-
-	if (bus_id)
-		strscpy(fixup->bus_id, bus_id, sizeof(fixup->bus_id));
-	fixup->phy_uid = phy_uid;
-	fixup->phy_uid_mask = phy_uid_mask;
-	fixup->run = run;
-
-	mutex_lock(&phy_fixup_lock);
-	list_add_tail(&fixup->list, &phy_fixup_list);
-	mutex_unlock(&phy_fixup_lock);
-
-	return 0;
-}
-
-/* Registers a fixup to be run on any PHY with the UID in phy_uid */
-int phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask,
-			       int (*run)(struct phy_device *))
-{
-	return phy_register_fixup(NULL, phy_uid, phy_uid_mask, run);
-}
-EXPORT_SYMBOL(phy_register_fixup_for_uid);
-
-/* Registers a fixup to be run on the PHY with id string bus_id */
-int phy_register_fixup_for_id(const char *bus_id,
-			      int (*run)(struct phy_device *))
-{
-	return phy_register_fixup(bus_id, 0, 0, run);
-}
-EXPORT_SYMBOL(phy_register_fixup_for_id);
-
-static bool phy_needs_fixup(struct phy_device *phydev, struct phy_fixup *fixup)
-{
-	if (!strcmp(fixup->bus_id, phydev_name(phydev)))
-		return true;
-
-	if (fixup->phy_uid_mask &&
-	    phy_id_compare(phydev->phy_id, fixup->phy_uid, fixup->phy_uid_mask))
-		return true;
-
-	return false;
-}
-
-/* Runs any matching fixups for this phydev */
-static int phy_scan_fixups(struct phy_device *phydev)
-{
-	struct phy_fixup *fixup;
-
-	mutex_lock(&phy_fixup_lock);
-	list_for_each_entry(fixup, &phy_fixup_list, list) {
-		if (phy_needs_fixup(phydev, fixup)) {
-			int err = fixup->run(phydev);
-
-			if (err < 0) {
-				mutex_unlock(&phy_fixup_lock);
-				return err;
-			}
-			phydev->has_fixups = true;
-		}
-	}
-	mutex_unlock(&phy_fixup_lock);
-
-	return 0;
-}
-
 /**
  * genphy_match_phy_device - match a PHY device with a PHY driver
  * @phydev: target phy_device struct
@@ -1723,17 +1633,144 @@ static int phy_sfp_probe(struct phy_device *phydev)
 			phydev->sfp_bus = NULL;
 	}
 
-	if (!ret && phydev->sfp_bus)
+	if (!ret && phydev->sfp_bus) {
 		ret = phy_setup_sfp_port(phydev);
+		if (ret) {
+			sfp_bus_del_upstream(phydev->sfp_bus);
+			phydev->sfp_bus = NULL;
+		}
+	}
 
 	return ret;
 }
 
+/**
+ * phy_sfp_release - release resources set up by phy_sfp_probe()
+ * @phydev: the PHY device
+ *
+ * Release the SFP resources set up by a successful phy_sfp_probe(). Unregister
+ * the upstream before destroying its phy_port, so SFP upstream callbacks cannot
+ * race with port destruction.
+ */
+static void phy_sfp_release(struct phy_device *phydev)
+{
+	struct phy_port *port, *tmp;
+
+	sfp_bus_del_upstream(phydev->sfp_bus);
+	phydev->sfp_bus = NULL;
+
+	list_for_each_entry_safe(port, tmp, &phydev->ports, head) {
+		if (!port->is_sfp)
+			continue;
+
+		phy_del_port(phydev, port);
+		phy_port_destroy(port);
+	}
+}
+
 static bool phy_drv_supports_irq(const struct phy_driver *phydrv)
 {
 	return phydrv->config_intr && phydrv->handle_interrupt;
 }
 
+/**
+ * phy_detach_internal - detach a PHY device from its network device
+ * @phydev: target phy_device struct
+ * @notify_bus: whether to notify the MDIO bus about the PHY detach
+ *
+ * This detaches the phy device from its network device and the phy
+ * driver, and drops the reference count taken in phy_attach_direct().
+ */
+static void phy_detach_internal(struct phy_device *phydev, bool notify_bus)
+{
+	struct net_device *dev = phydev->attached_dev;
+	struct module *ndev_owner = NULL;
+	struct mii_bus *bus;
+
+	if (phydev->devlink) {
+		device_link_del(phydev->devlink);
+		phydev->devlink = NULL;
+	}
+
+	if (phydev->sysfs_links) {
+		if (dev)
+			sysfs_remove_link(&dev->dev.kobj, "phydev");
+		sysfs_remove_link(&phydev->mdio.dev.kobj, "attached_dev");
+	}
+
+	if (!phydev->attached_dev)
+		sysfs_remove_file(&phydev->mdio.dev.kobj,
+				  &dev_attr_phy_standalone.attr);
+
+	phy_suspend(phydev);
+
+	if (notify_bus && phydev->mdio.bus->notify_phy_detach)
+		phydev->mdio.bus->notify_phy_detach(phydev);
+
+	if (dev) {
+		struct hwtstamp_provider *hwprov;
+
+		/* hwprov may technically be protected by ops lock but
+		 * not for devices with a phydev, see phy_link_topo_add_phy()
+		 */
+		hwprov = rtnl_dereference(dev->hwprov);
+		/* Disable timestamp if it is the one selected */
+		if (hwprov && hwprov->phydev == phydev) {
+			rcu_assign_pointer(dev->hwprov, NULL);
+			kfree_rcu(hwprov, rcu_head);
+		}
+
+		phydev->attached_dev->phydev = NULL;
+		phydev->attached_dev = NULL;
+		phy_link_topo_del_phy(dev, phydev);
+	}
+
+	phydev->phy_link_change = NULL;
+	phydev->phylink = NULL;
+
+	if (phydev->mdio.dev.driver)
+		module_put(phydev->mdio.dev.driver->owner);
+
+	/* If the device had no specific driver before (i.e. - it
+	 * was using the generic driver), we unbind the device
+	 * from the generic driver so that there's a chance a
+	 * real driver could be loaded
+	 */
+	if (phydev->is_genphy_driven) {
+		device_release_driver(&phydev->mdio.dev);
+		phydev->is_genphy_driven = 0;
+	}
+
+	/* Assert the reset signal */
+	phy_device_reset(phydev, 1);
+
+	/*
+	 * The phydev might go away on the put_device() below, so avoid
+	 * a use-after-free bug by reading the underlying bus first.
+	 */
+	bus = phydev->mdio.bus;
+
+	put_device(&phydev->mdio.dev);
+	if (dev)
+		ndev_owner = dev->dev.parent->driver->owner;
+	if (ndev_owner != bus->owner)
+		module_put(bus->owner);
+}
+
+/**
+ * phy_detach - detach a PHY device from its network device
+ * @phydev: target phy_device struct
+ *
+ * This detaches the phy device from its network device and the phy
+ * driver, and drops the reference count taken in phy_attach_direct().
+ */
+void phy_detach(struct phy_device *phydev)
+{
+	/* cleanup including bus notification */
+	phy_detach_internal(phydev, true);
+}
+EXPORT_SYMBOL(phy_detach);
+
 /**
  * phy_attach_direct - attach a network device to a given PHY device pointer
  * @dev: network device to attach
@@ -1876,6 +1913,12 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 	if (err)
 		goto error;
 
+	if (phydev->mdio.bus->notify_phy_attach) {
+		err = phydev->mdio.bus->notify_phy_attach(phydev);
+		if (err)
+			goto error;
+	}
+
 	phy_resume(phydev);
 
 	/**
@@ -1890,8 +1933,8 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 	return err;
 
 error:
-	/* phy_detach() does all of the cleanup below */
-	phy_detach(phydev);
+	/* phy_detach_internal() does all of the cleanup below */
+	phy_detach_internal(phydev, false);
 	return err;
 
 error_module_put:
@@ -1906,86 +1949,6 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 }
 EXPORT_SYMBOL(phy_attach_direct);
 
-/**
- * phy_detach - detach a PHY device from its network device
- * @phydev: target phy_device struct
- *
- * This detaches the phy device from its network device and the phy
- * driver, and drops the reference count taken in phy_attach_direct().
- */
-void phy_detach(struct phy_device *phydev)
-{
-	struct net_device *dev = phydev->attached_dev;
-	struct module *ndev_owner = NULL;
-	struct mii_bus *bus;
-
-	if (phydev->devlink) {
-		device_link_del(phydev->devlink);
-		phydev->devlink = NULL;
-	}
-
-	if (phydev->sysfs_links) {
-		if (dev)
-			sysfs_remove_link(&dev->dev.kobj, "phydev");
-		sysfs_remove_link(&phydev->mdio.dev.kobj, "attached_dev");
-	}
-
-	if (!phydev->attached_dev)
-		sysfs_remove_file(&phydev->mdio.dev.kobj,
-				  &dev_attr_phy_standalone.attr);
-
-	phy_suspend(phydev);
-	if (dev) {
-		struct hwtstamp_provider *hwprov;
-
-		/* hwprov may technically be protected by ops lock but
-		 * not for devices with a phydev, see phy_link_topo_add_phy()
-		 */
-		hwprov = rtnl_dereference(dev->hwprov);
-		/* Disable timestamp if it is the one selected */
-		if (hwprov && hwprov->phydev == phydev) {
-			rcu_assign_pointer(dev->hwprov, NULL);
-			kfree_rcu(hwprov, rcu_head);
-		}
-
-		phydev->attached_dev->phydev = NULL;
-		phydev->attached_dev = NULL;
-		phy_link_topo_del_phy(dev, phydev);
-	}
-
-	phydev->phy_link_change = NULL;
-	phydev->phylink = NULL;
-
-	if (phydev->mdio.dev.driver)
-		module_put(phydev->mdio.dev.driver->owner);
-
-	/* If the device had no specific driver before (i.e. - it
-	 * was using the generic driver), we unbind the device
-	 * from the generic driver so that there's a chance a
-	 * real driver could be loaded
-	 */
-	if (phydev->is_genphy_driven) {
-		device_release_driver(&phydev->mdio.dev);
-		phydev->is_genphy_driven = 0;
-	}
-
-	/* Assert the reset signal */
-	phy_device_reset(phydev, 1);
-
-	/*
-	 * The phydev might go away on the put_device() below, so avoid
-	 * a use-after-free bug by reading the underlying bus first.
-	 */
-	bus = phydev->mdio.bus;
-
-	put_device(&phydev->mdio.dev);
-	if (dev)
-		ndev_owner = dev->dev.parent->driver->owner;
-	if (ndev_owner != bus->owner)
-		module_put(bus->owner);
-}
-EXPORT_SYMBOL(phy_detach);
-
 int phy_suspend(struct phy_device *phydev)
 {
 	struct net_device *netdev = phydev->attached_dev;
@@ -3454,6 +3417,7 @@ static int phy_default_setup_single_port(struct phy_device *phydev)
 {
 	struct phy_port *port = phy_port_alloc();
 	unsigned long mode;
+	int ret;
 
 	if (!port)
 		return -ENOMEM;
@@ -3480,9 +3444,11 @@ static int phy_default_setup_single_port(struct phy_device *phydev)
 		port->pairs = max_t(int, port->pairs,
 				    ethtool_linkmode_n_pairs(mode));
 
-	phy_add_port(phydev, port);
+	ret = phy_add_port(phydev, port);
+	if (ret)
+		phy_port_destroy(port);
 
-	return 0;
+	return ret;
 }
 
 static int of_phy_ports(struct phy_device *phydev)
@@ -3547,13 +3513,13 @@ static int phy_setup_ports(struct phy_device *phydev)
 	if (!phydev->is_genphy_driven) {
 		ret = phy_sfp_probe(phydev);
 		if (ret)
-			goto out;
+			goto err_ports;
 	}
 
 	if (phydev->n_ports < phydev->max_n_ports) {
 		ret = phy_default_setup_single_port(phydev);
 		if (ret)
-			goto out;
+			goto err_sfp;
 	}
 
 	linkmode_zero(ports_supported);
@@ -3580,7 +3546,9 @@ static int phy_setup_ports(struct phy_device *phydev)
 
 	return 0;
 
-out:
+err_sfp:
+	phy_sfp_release(phydev);
+err_ports:
 	phy_cleanup_ports(phydev);
 	return ret;
 }
@@ -3706,7 +3674,7 @@ static int phy_probe(struct device *dev)
 	if (phydev->drv->probe) {
 		err = phydev->drv->probe(phydev);
 		if (err)
-			goto out;
+			goto out_reset;
 	}
 
 	phy_disable_interrupts(phydev);
@@ -3727,7 +3695,7 @@ static int phy_probe(struct device *dev)
 		err = genphy_read_abilities(phydev);
 
 	if (err)
-		goto out;
+		goto out_remove;
 
 	if (!linkmode_test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT,
 			       phydev->supported))
@@ -3744,7 +3712,7 @@ static int phy_probe(struct device *dev)
 
 	err = phy_setup_ports(phydev);
 	if (err)
-		goto out;
+		goto out_remove;
 
 	phy_advertise_supported(phydev);
 
@@ -3753,7 +3721,7 @@ static int phy_probe(struct device *dev)
 	 */
 	err = genphy_c45_read_eee_adv(phydev, phydev->advertising_eee);
 	if (err)
-		goto out;
+		goto out_sfp_release;
 
 	/* Get the EEE modes we want to prohibit. */
 	of_set_phy_eee_broken(phydev);
@@ -3793,9 +3761,6 @@ static int phy_probe(struct device *dev)
 				 phydev->supported);
 	}
 
-	/* Set the state to READY by default */
-	phydev->state = PHY_READY;
-
 	/* Register the PHY LED triggers */
 	if (!phydev->is_on_sfp_module)
 		phy_led_triggers_register(phydev);
@@ -3806,20 +3771,27 @@ static int phy_probe(struct device *dev)
 	if (IS_ENABLED(CONFIG_PHYLIB_LEDS) && !phy_driver_is_genphy(phydev)) {
 		err = of_phy_leds(phydev);
 		if (err)
-			goto out;
+			goto out_unreg_led_triggers;
 	}
 
+	/* Set the state to READY by default */
+	phydev->state = PHY_READY;
+
 	return 0;
 
-out:
-	sfp_bus_del_upstream(phydev->sfp_bus);
-	phydev->sfp_bus = NULL;
+out_unreg_led_triggers:
+	if (!phydev->is_on_sfp_module)
+		phy_led_triggers_unregister(phydev);
 
+out_sfp_release:
+	phy_sfp_release(phydev);
 	phy_cleanup_ports(phydev);
 
-	if (!phydev->is_on_sfp_module)
-		phy_led_triggers_unregister(phydev);
+out_remove:
+	if (phydev->drv->remove)
+		phydev->drv->remove(phydev);
 
+out_reset:
 	/* Re-assert the reset signal on error */
 	phy_device_reset(phydev, 1);
 
@@ -3840,9 +3812,7 @@ static int phy_remove(struct device *dev)
 
 	phydev->state = PHY_DOWN;
 
-	sfp_bus_del_upstream(phydev->sfp_bus);
-	phydev->sfp_bus = NULL;
-
+	phy_sfp_release(phydev);
 	phy_cleanup_ports(phydev);
 
 	if (phydev->drv && phydev->drv->remove)
diff --git a/drivers/net/phy/phy_fixup.c b/drivers/net/phy/phy_fixup.c
new file mode 100644
index 00000000000000..75ed9b142ccc30
--- /dev/null
+++ b/drivers/net/phy/phy_fixup.c
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * PHY fixup support
+ */
+
+#include <linux/list.h>
+#include <linux/phy.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "phylib-internal.h"
+
+static struct list_head phy_fixup_list __ro_after_init =
+	LIST_HEAD_INIT(phy_fixup_list);
+
+struct phy_fixup {
+	struct list_head list;
+	char bus_id[MII_BUS_ID_SIZE + 3];
+	u32 phy_uid;
+	u32 phy_uid_mask;
+	int (*run)(struct phy_device *phydev);
+};
+
+/**
+ * phy_register_fixup - creates a new phy_fixup and adds it to the list
+ * @bus_id: A string which matches phydev->mdio.dev.bus_id (or NULL)
+ * @phy_uid: Used to match against phydev->phy_id (the UID of the PHY)
+ * @phy_uid_mask: Applied to phydev->phy_id and fixup->phy_uid before
+ *	comparison (or 0 to disable id-based matching)
+ * @run: The actual code to be run when a matching PHY is found
+ */
+static void __init phy_register_fixup(const char *bus_id, u32 phy_uid,
+				      u32 phy_uid_mask,
+				      int (*run)(struct phy_device *))
+{
+	struct phy_fixup *fixup = kzalloc_obj(*fixup);
+
+	if (!fixup)
+		return;
+
+	if (bus_id)
+		strscpy(fixup->bus_id, bus_id);
+	fixup->phy_uid = phy_uid;
+	fixup->phy_uid_mask = phy_uid_mask;
+	fixup->run = run;
+
+	list_add_tail(&fixup->list, &phy_fixup_list);
+}
+
+/* Registers a fixup to be run on any PHY with the UID in phy_uid */
+void __init phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask,
+				       int (*run)(struct phy_device *))
+{
+	phy_register_fixup(NULL, phy_uid, phy_uid_mask, run);
+}
+
+/* Registers a fixup to be run on the PHY with id string bus_id */
+void __init phy_register_fixup_for_id(const char *bus_id,
+				      int (*run)(struct phy_device *))
+{
+	phy_register_fixup(bus_id, 0, 0, run);
+}
+
+static bool phy_needs_fixup(struct phy_device *phydev, struct phy_fixup *fixup)
+{
+	if (!strcmp(fixup->bus_id, phydev_name(phydev)))
+		return true;
+
+	if (fixup->phy_uid_mask &&
+	    phy_id_compare(phydev->phy_id, fixup->phy_uid, fixup->phy_uid_mask))
+		return true;
+
+	return false;
+}
+
+/**
+ * phy_scan_fixups - runs any matching fixups for this phydev
+ * @phydev: the phydev to search and run fixups for
+ * Returns: 0 or an errno
+ */
+int phy_scan_fixups(struct phy_device *phydev)
+{
+	struct phy_fixup *fixup;
+
+	list_for_each_entry(fixup, &phy_fixup_list, list) {
+		if (phy_needs_fixup(phydev, fixup)) {
+			int err = fixup->run(phydev);
+
+			if (err < 0)
+				return err;
+
+			phydev->has_fixups = true;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(phy_scan_fixups);
diff --git a/drivers/net/phy/phy_led_triggers.c b/drivers/net/phy/phy_led_triggers.c
index 4eb7716bb9d6c0..ff6e518395be0f 100644
--- a/drivers/net/phy/phy_led_triggers.c
+++ b/drivers/net/phy/phy_led_triggers.c
@@ -126,6 +126,7 @@ int phy_led_triggers_register(struct phy_device *phy)
 	while (i--)
 		phy_led_trigger_unregister(&phy->phy_led_triggers[i]);
 	kfree(phy->phy_led_triggers);
+	phy->phy_led_triggers = NULL;
 out_unreg_link:
 	phy_led_trigger_unregister(phy->led_link_trigger);
 out_free_link:
@@ -141,10 +142,12 @@ void phy_led_triggers_unregister(struct phy_device *phy)
 {
 	int i;
 
-	for (i = 0; i < phy->phy_num_led_triggers; i++)
-		phy_led_trigger_unregister(&phy->phy_led_triggers[i]);
-	kfree(phy->phy_led_triggers);
-	phy->phy_led_triggers = NULL;
+	if (phy->phy_led_triggers) {
+		for (i = 0; i < phy->phy_num_led_triggers; i++)
+			phy_led_trigger_unregister(&phy->phy_led_triggers[i]);
+		kfree(phy->phy_led_triggers);
+		phy->phy_led_triggers = NULL;
+	}
 
 	if (phy->led_link_trigger) {
 		phy_led_trigger_unregister(phy->led_link_trigger);
diff --git a/drivers/net/phy/phylib-internal.h b/drivers/net/phy/phylib-internal.h
index 664ed7faa5184c..38ee294f3c962b 100644
--- a/drivers/net/phy/phylib-internal.h
+++ b/drivers/net/phy/phylib-internal.h
@@ -23,6 +23,7 @@ void of_set_phy_eee_broken(struct phy_device *phydev);
 void of_set_phy_timing_role(struct phy_device *phydev);
 int phy_speed_down_core(struct phy_device *phydev);
 void phy_check_downshift(struct phy_device *phydev);
+int phy_scan_fixups(struct phy_device *phydev);
 
 int mdiobus_register_device(struct mdio_device *mdiodev);
 int mdiobus_unregister_device(struct mdio_device *mdiodev);
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
index a1458da8111b6c..a7d086cdc9b251 100644
--- a/drivers/net/phy/phylink.c
+++ b/drivers/net/phy/phylink.c
@@ -1831,6 +1831,126 @@ int phylink_set_fixed_link(struct phylink *pl,
 }
 EXPORT_SYMBOL_GPL(phylink_set_fixed_link);
 
+/**
+ * phylink_update_pause_state() - Update the phylink pause frame configuration
+ * @pl: a pointer to a &struct phylink instance
+ * @pause_state: bitmask indicating the new pause state
+ *
+ * Update the MAC pause frame (flow control) state for the phylink instance.
+ */
+static void phylink_update_pause_state(struct phylink *pl, int pause_state)
+{
+	struct phylink_link_state *config = &pl->link_config;
+	bool tx_pause = !!(pause_state & MLO_PAUSE_TX);
+	bool rx_pause = !!(pause_state & MLO_PAUSE_RX);
+	bool manual_changed;
+
+	mutex_lock(&pl->state_mutex);
+
+	/*
+	 * See the comments for linkmode_set_pause(), wrt the deficiencies
+	 * with the current implementation.  A solution to this issue would
+	 * be:
+	 * ethtool  Local device
+	 *  rx  tx  Pause AsymDir
+	 *  0   0   0     0
+	 *  1   0   1     1
+	 *  0   1   0     1
+	 *  1   1   1     1
+	 * and then use the ethtool rx/tx enablement status to mask the
+	 * rx/tx pause resolution.
+	 */
+	linkmode_set_pause(config->advertising, tx_pause,
+			   rx_pause);
+
+	manual_changed = (config->pause ^ pause_state) & MLO_PAUSE_AN ||
+			 (!(pause_state & MLO_PAUSE_AN) &&
+			   (config->pause ^ pause_state) & MLO_PAUSE_TXRX_MASK);
+
+	config->pause = pause_state;
+
+	/* Update our in-band advertisement, triggering a renegotiation if
+	 * the advertisement changed.
+	 */
+	if (!pl->phydev)
+		phylink_change_inband_advert(pl);
+
+	mutex_unlock(&pl->state_mutex);
+
+	/* If we have a PHY, a change of the pause frame advertisement will
+	 * cause phylib to renegotiate (if AN is enabled) which will in turn
+	 * call our phylink_phy_change() and trigger a resolve.  Note that
+	 * we can't hold our state mutex while calling phy_set_asym_pause().
+	 */
+	if (pl->phydev)
+		phy_set_asym_pause(pl->phydev, rx_pause, tx_pause);
+
+	/* If the manual pause settings changed, make sure we trigger a
+	 * resolve to update their state; we can not guarantee that the
+	 * link will cycle.
+	 */
+	if (manual_changed) {
+		pl->link_failed = true;
+		phylink_run_resolve(pl);
+	}
+}
+
+/**
+ * phylink_update_mac_pause_capabilities() - Dynamically update MAC pause
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @mac_pause: the new MAC pause capabilities mask
+ *
+ * This function allows a MAC driver to dynamically change its pause state,
+ * such as losing/gaining Pause frame support based on MTU size.
+ * It recalculates supported link modes and triggers renegotiation if needed.
+ */
+void phylink_update_mac_pause_capabilities(struct phylink *pl, unsigned long mac_pause)
+{
+	struct phylink_link_state *config = &pl->link_config;
+	unsigned long old_pause;
+	int pause_state;
+
+	ASSERT_RTNL();
+
+	if (mac_pause & ~(MAC_SYM_PAUSE | MAC_ASYM_PAUSE)) {
+		phylink_err(pl, "Attempted to dynamically change non-pause MAC capabilities\n");
+		return;
+	}
+
+	old_pause = pl->config->mac_capabilities & (MAC_SYM_PAUSE | MAC_ASYM_PAUSE);
+	if (old_pause == mac_pause)
+		return;
+
+	mutex_lock(&pl->state_mutex);
+
+	pl->config->mac_capabilities &= ~(MAC_SYM_PAUSE | MAC_ASYM_PAUSE);
+	pl->config->mac_capabilities |= mac_pause;
+
+	phylink_set(pl->supported, Pause);
+	phylink_set(pl->supported, Asym_Pause);
+
+	if (pl->phydev)
+		linkmode_and(pl->supported, pl->supported, pl->phydev->supported);
+	else if (pl->sfp_bus)
+		linkmode_and(pl->supported, pl->supported, pl->sfp_support);
+
+	phylink_validate(pl, pl->supported, config);
+
+	pause_state = config->pause;
+
+	if (!phylink_test(pl->supported, Pause)) {
+		pause_state &= ~(MLO_PAUSE_RX | MLO_PAUSE_TX);
+	} else if (!phylink_test(pl->supported, Asym_Pause)) {
+		if ((pause_state & MLO_PAUSE_RX) ^ (pause_state & MLO_PAUSE_TX))
+			pause_state &= ~(MLO_PAUSE_RX | MLO_PAUSE_TX);
+	}
+
+	mutex_unlock(&pl->state_mutex);
+
+	phylink_update_pause_state(pl, pause_state);
+}
+EXPORT_SYMBOL_GPL(phylink_update_mac_pause_capabilities);
+
 /**
  * phylink_create() - create a phylink instance
  * @config: a pointer to the target &struct phylink_config
@@ -2129,7 +2249,6 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy,
 	mutex_lock(&pl->phydev_mutex);
 	mutex_lock(&phy->lock);
 	mutex_lock(&pl->state_mutex);
-	pl->phydev = phy;
 	pl->phy_state.interface = interface;
 	pl->phy_state.pause = MLO_PAUSE_NONE;
 	pl->phy_state.speed = SPEED_UNKNOWN;
@@ -2196,10 +2315,25 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy,
 			ret = 0;
 	}
 
-	if (ret == 0 && phy_interrupt_is_valid(phy))
+	if (ret)
+		return ret;
+
+	/* Nothing below can fail, so the PHY can be recorded now. Doing it
+	 * here rather than above keeps a failed bringup from leaving
+	 * pl->phydev pointing at a PHY the caller is about to detach.
+	 */
+	mutex_lock(&pl->phydev_mutex);
+	mutex_lock(&phy->lock);
+	mutex_lock(&pl->state_mutex);
+	pl->phydev = phy;
+	mutex_unlock(&pl->state_mutex);
+	mutex_unlock(&phy->lock);
+	mutex_unlock(&pl->phydev_mutex);
+
+	if (phy_interrupt_is_valid(phy))
 		phy_request_interrupt(phy);
 
-	return ret;
+	return 0;
 }
 
 static int phylink_attach_phy(struct phylink *pl, struct phy_device *phy,
@@ -3190,8 +3324,6 @@ EXPORT_SYMBOL_GPL(phylink_ethtool_get_pauseparam);
 int phylink_ethtool_set_pauseparam(struct phylink *pl,
 				   struct ethtool_pauseparam *pause)
 {
-	struct phylink_link_state *config = &pl->link_config;
-	bool manual_changed;
 	int pause_state;
 
 	ASSERT_RTNL();
@@ -3215,54 +3347,7 @@ int phylink_ethtool_set_pauseparam(struct phylink *pl,
 	if (pause->tx_pause)
 		pause_state |= MLO_PAUSE_TX;
 
-	mutex_lock(&pl->state_mutex);
-	/*
-	 * See the comments for linkmode_set_pause(), wrt the deficiencies
-	 * with the current implementation.  A solution to this issue would
-	 * be:
-	 * ethtool  Local device
-	 *  rx  tx  Pause AsymDir
-	 *  0   0   0     0
-	 *  1   0   1     1
-	 *  0   1   0     1
-	 *  1   1   1     1
-	 * and then use the ethtool rx/tx enablement status to mask the
-	 * rx/tx pause resolution.
-	 */
-	linkmode_set_pause(config->advertising, pause->tx_pause,
-			   pause->rx_pause);
-
-	manual_changed = (config->pause ^ pause_state) & MLO_PAUSE_AN ||
-			 (!(pause_state & MLO_PAUSE_AN) &&
-			   (config->pause ^ pause_state) & MLO_PAUSE_TXRX_MASK);
-
-	config->pause = pause_state;
-
-	/* Update our in-band advertisement, triggering a renegotiation if
-	 * the advertisement changed.
-	 */
-	if (!pl->phydev)
-		phylink_change_inband_advert(pl);
-
-	mutex_unlock(&pl->state_mutex);
-
-	/* If we have a PHY, a change of the pause frame advertisement will
-	 * cause phylib to renegotiate (if AN is enabled) which will in turn
-	 * call our phylink_phy_change() and trigger a resolve.  Note that
-	 * we can't hold our state mutex while calling phy_set_asym_pause().
-	 */
-	if (pl->phydev)
-		phy_set_asym_pause(pl->phydev, pause->rx_pause,
-				   pause->tx_pause);
-
-	/* If the manual pause settings changed, make sure we trigger a
-	 * resolve to update their state; we can not guarantee that the
-	 * link will cycle.
-	 */
-	if (manual_changed) {
-		pl->link_failed = true;
-		phylink_run_resolve(pl);
-	}
+	phylink_update_pause_state(pl, pause_state);
 
 	return 0;
 }
@@ -4362,6 +4447,11 @@ void phylink_mii_c45_pcs_get_state(struct mdio_device *pcs,
 		state->duplex = DUPLEX_FULL;
 		break;
 
+	case PHY_INTERFACE_MODE_25GBASER:
+		state->speed = SPEED_25000;
+		state->duplex = DUPLEX_FULL;
+		break;
+
 	default:
 		break;
 	}
diff --git a/drivers/net/phy/qcom/at803x.c b/drivers/net/phy/qcom/at803x.c
index 6872dbf7785628..cacbadf1f48410 100644
--- a/drivers/net/phy/qcom/at803x.c
+++ b/drivers/net/phy/qcom/at803x.c
@@ -1051,11 +1051,15 @@ static int ipq5018_config_init(struct phy_device *phydev)
 	if (priv->set_short_cable_dac) {
 		/* setting MDAC (Multi-level Digital-to-Analog Converter) in MMD1 */
 		phy_modify_mmd(phydev, MDIO_MMD_PMAPMD, IPQ5018_PHY_MMD1_MDAC,
-			       IPQ5018_PHY_DAC_MASK, IPQ5018_PHY_MMD1_MDAC_VAL);
+			       IPQ5018_PHY_DAC_MASK,
+			       FIELD_PREP(IPQ5018_PHY_DAC_MASK,
+					  IPQ5018_PHY_MMD1_MDAC_VAL));
 
 		/* setting EDAC (Error-detection and Correction) in debug register */
 		at803x_debug_reg_mask(phydev, IPQ5018_PHY_DEBUG_EDAC,
-				      IPQ5018_PHY_DAC_MASK, IPQ5018_PHY_DEBUG_EDAC_VAL);
+				      IPQ5018_PHY_DAC_MASK,
+				      FIELD_PREP(IPQ5018_PHY_DAC_MASK,
+						 IPQ5018_PHY_DEBUG_EDAC_VAL));
 	}
 
 	return 0;
diff --git a/drivers/net/phy/qcom/qca808x.c b/drivers/net/phy/qcom/qca808x.c
index 8eb51b1a006c4c..bab26e4c61408c 100644
--- a/drivers/net/phy/qcom/qca808x.c
+++ b/drivers/net/phy/qcom/qca808x.c
@@ -87,6 +87,9 @@
 
 #define QCA8081_PHY_ID				0x004dd101
 
+/* led_polarity_mode otherwise holds a PHY_LED_ACTIVE_* value */
+#define QCA808X_PHY_LED_UNSET			-1
+
 MODULE_DESCRIPTION("Qualcomm Atheros QCA808X PHY driver");
 MODULE_AUTHOR("Matus Ujhelyi");
 MODULE_LICENSE("GPL");
@@ -187,8 +190,7 @@ static int qca808x_probe(struct phy_device *phydev)
 	if (!priv)
 		return -ENOMEM;
 
-	/* Init LED polarity mode to -1 */
-	priv->led_polarity_mode = -1;
+	priv->led_polarity_mode = QCA808X_PHY_LED_UNSET;
 
 	phydev->priv = priv;
 
@@ -200,8 +202,8 @@ static int qca808x_config_init(struct phy_device *phydev)
 	struct qca808x_priv *priv = phydev->priv;
 	int ret;
 
-	/* Default to LED Active High if active-low not in DT */
-	if (priv->led_polarity_mode == -1) {
+	/* Set LED Active High unless active-low was requested in DT */
+	if (priv->led_polarity_mode != PHY_LED_ACTIVE_LOW) {
 		ret = phy_set_bits_mmd(phydev, MDIO_MMD_AN,
 				       QCA808X_MMD7_LED_POLARITY_CTRL,
 				       QCA808X_LED_ACTIVE_HIGH);
@@ -603,6 +605,9 @@ static int qca808x_led_polarity_set(struct phy_device *phydev, int index,
 		case PHY_LED_ACTIVE_LOW:
 			active_low = true;
 			break;
+		case PHY_LED_ACTIVE_HIGH:
+			active_low = false;
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -612,7 +617,7 @@ static int qca808x_led_polarity_set(struct phy_device *phydev, int index,
 	 * To detect this, check if last requested polarity mode
 	 * match the new one.
 	 */
-	if (priv->led_polarity_mode >= 0 &&
+	if (priv->led_polarity_mode != QCA808X_PHY_LED_UNSET &&
 	    priv->led_polarity_mode != active_low) {
 		phydev_err(phydev, "PHY polarity is global. Mismatched polarity on different LED\n");
 		return -EINVAL;
diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c
index 177b62a7b2d160..3b3352e2cdc6a7 100644
--- a/drivers/net/phy/realtek/realtek_main.c
+++ b/drivers/net/phy/realtek/realtek_main.c
@@ -19,7 +19,6 @@
 #include <linux/delay.h>
 #include <linux/clk.h>
 #include <linux/string_choices.h>
-#include <net/phy/realtek_phy.h>
 
 #include "../phylib.h"
 #include "realtek.h"
@@ -258,6 +257,8 @@
 #define RTL_8251B				0x001cc862
 #define RTL_8261C				0x001cc890
 #define RTL_8261C_CG				0x001cc898
+#define RTL_8261CE_CG				0x001cc899
+#define RTL_8261D_VM				0x001cc89a
 
 #define RTL8261C_CE_MODEL		0x00
 #define RTL8261D_MODEL			0x81
@@ -286,11 +287,15 @@
 
 #define FW_MAIN_MAGIC			0x52544C38
 #define FW_SUB_MAGIC_8261C		0x32363143
+#define FW_SUB_MAGIC_8261D		0x32363144
 #define RTL8261X_POLL_TIMEOUT_MS	100
 #define RTL8261X_MAX_MMD_DEV		31
 
 #define RTL8261C_CE_FW_NAME	"rtl_nic/rtl8261c.bin"
+#define RTL8261D_FW_NAME	"rtl_nic/rtl8261d.bin"
+
 MODULE_FIRMWARE(RTL8261C_CE_FW_NAME);
+MODULE_FIRMWARE(RTL8261D_FW_NAME);
 
 enum rtl8261x_fw_op {
 	OP_WRITE = 0x00,	/* Write */
@@ -342,6 +347,7 @@ struct rtl821x_priv {
 
 struct rtl8261x_priv {
 	const char *fw_name;
+	u32 fw_sub_magic;
 	bool fw_loaded;
 };
 
@@ -411,10 +417,13 @@ static int rtl8261x_probe(struct phy_device *phydev)
 	switch (sub_phy_id) {
 	case RTL8261C_CE_MODEL:
 		priv->fw_name = RTL8261C_CE_FW_NAME;
+		priv->fw_sub_magic = FW_SUB_MAGIC_8261C;
 		phydev_info(phydev, "RTL8261C detected (sub_id 0x%02x)\n", sub_phy_id);
 		break;
 
 	case RTL8261D_MODEL:
+		priv->fw_name = RTL8261D_FW_NAME;
+		priv->fw_sub_magic = FW_SUB_MAGIC_8261D;
 		phydev_info(phydev, "RTL8261D detected (sub_id 0x%02x)\n", sub_phy_id);
 		break;
 
@@ -473,6 +482,7 @@ static int rtl8261x_read_status(struct phy_device *phydev)
 
 static int rtl8261x_verify_firmware(struct phy_device *phydev, const struct firmware *fw)
 {
+	struct rtl8261x_priv *priv = phydev->priv;
 	const struct rtl8261x_fw_header *hdr;
 	u32 main_magic, sub_magic;
 	u32 calc_crc, file_crc;
@@ -493,7 +503,7 @@ static int rtl8261x_verify_firmware(struct phy_device *phydev, const struct firm
 	}
 
 	sub_magic = le32_to_cpu(hdr->sub_magic);
-	if (sub_magic != FW_SUB_MAGIC_8261C) {
+	if (sub_magic != priv->fw_sub_magic) {
 		phydev_err(phydev, "Invalid sub magic: 0x%08x\n", sub_magic);
 		return -EINVAL;
 	}
@@ -549,8 +559,12 @@ static int rtl8261x_fw_execute_entry(struct phy_device *phydev,
 
 	switch (entry->type) {
 	case OP_WRITE:
-		ret = phy_modify_mmd(phydev, dev, addr,
-				     GENMASK(msb, lsb), (value << lsb) & GENMASK(msb, lsb));
+		if (msb != 15 || lsb != 0)
+			ret = phy_modify_mmd(phydev, dev, addr, GENMASK(msb, lsb),
+					     (value << lsb) & GENMASK(msb, lsb));
+		else
+			ret = phy_write_mmd(phydev, dev, addr, value);
+
 		if (ret)
 			return ret;
 		break;
@@ -3044,45 +3058,6 @@ static irqreturn_t rtl8221b_handle_interrupt(struct phy_device *phydev)
 	return IRQ_HANDLED;
 }
 
-static int rtlgen_sfp_get_features(struct phy_device *phydev)
-{
-	linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
-			 phydev->supported);
-
-	/* set default mode */
-	phydev->speed = SPEED_10000;
-	phydev->duplex = DUPLEX_FULL;
-
-	phydev->port = PORT_FIBRE;
-
-	return 0;
-}
-
-static int rtlgen_sfp_read_status(struct phy_device *phydev)
-{
-	int val, err;
-
-	err = genphy_update_link(phydev);
-	if (err)
-		return err;
-
-	if (!phydev->link)
-		return 0;
-
-	val = phy_read(phydev, RTL_PHYSR);
-	if (val < 0)
-		return val;
-
-	rtlgen_decode_physr(phydev, val);
-
-	return 0;
-}
-
-static int rtlgen_sfp_config_aneg(struct phy_device *phydev)
-{
-	return 0;
-}
-
 static struct phy_driver realtek_drvs[] = {
 	{
 		PHY_ID_MATCH_EXACT(0x00008201),
@@ -3332,20 +3307,6 @@ static struct phy_driver realtek_drvs[] = {
 		.write_page	= rtl821x_write_page,
 		.read_mmd	= rtl822x_read_mmd,
 		.write_mmd	= rtl822x_write_mmd,
-	}, {
-		PHY_ID_MATCH_EXACT(PHY_ID_RTL_DUMMY_SFP),
-		.name		= "Realtek SFP PHY Mode",
-		.flags		= PHY_IS_INTERNAL,
-		.probe		= rtl822x_probe,
-		.get_features	= rtlgen_sfp_get_features,
-		.config_aneg	= rtlgen_sfp_config_aneg,
-		.read_status	= rtlgen_sfp_read_status,
-		.suspend	= genphy_suspend,
-		.resume		= rtlgen_resume,
-		.read_page	= rtl821x_read_page,
-		.write_page	= rtl821x_write_page,
-		.read_mmd	= rtl822x_read_mmd,
-		.write_mmd	= rtl822x_write_mmd,
 	}, {
 		PHY_ID_MATCH_EXACT(0x001ccad0),
 		.name		= "RTL8224 2.5Gbps PHY",
@@ -3388,6 +3349,8 @@ static struct phy_driver realtek_drvs[] = {
 	}, {
 		PHY_ID_MATCH_EXACT(0x001cc942),
 		.name		= "RTL8365MB-VC Gigabit Ethernet",
+		.read_mmd	= genphy_read_mmd_c45,
+		.write_mmd	= genphy_write_mmd_c45,
 		/* Interrupt handling analogous to RTL8366RB */
 		.config_intr	= genphy_no_config_intr,
 		.handle_interrupt = genphy_handle_interrupt_no_ack,
@@ -3413,6 +3376,32 @@ static struct phy_driver realtek_drvs[] = {
 		.soft_reset		= genphy_c45_pma_soft_reset,
 		.suspend		= genphy_c45_pma_suspend,
 		.resume			= genphy_c45_pma_resume,
+	}, {
+		PHY_ID_MATCH_EXACT(RTL_8261CE_CG),
+		.name			= "Realtek RTL8261CE 10Gbps PHY",
+		.probe			= rtl8261x_probe,
+		.config_init		= rtl8261x_config_init,
+		.get_features		= rtl8261x_get_features,
+		.config_aneg		= rtl8261x_config_aneg,
+		.read_status		= rtl8261x_read_status,
+		.config_intr		= rtl8261x_config_intr,
+		.handle_interrupt	= rtl8261x_handle_interrupt,
+		.soft_reset		= genphy_c45_pma_soft_reset,
+		.suspend		= genphy_c45_pma_suspend,
+		.resume			= genphy_c45_pma_resume,
+	}, {
+		PHY_ID_MATCH_EXACT(RTL_8261D_VM),
+		.name			= "Realtek RTL8261D_VM 10Gbps PHY",
+		.probe			= rtl8261x_probe,
+		.config_init		= rtl8261x_config_init,
+		.get_features		= rtl8261x_get_features,
+		.config_aneg		= rtl8261x_config_aneg,
+		.read_status		= rtl8261x_read_status,
+		.config_intr		= rtl8261x_config_intr,
+		.handle_interrupt	= rtl8261x_handle_interrupt,
+		.soft_reset		= genphy_c45_pma_soft_reset,
+		.suspend		= genphy_c45_pma_suspend,
+		.resume			= genphy_c45_pma_resume,
 	},
 };
 
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
index 2ec91466acdf3a..96361584791b45 100644
--- a/drivers/net/phy/sfp.c
+++ b/drivers/net/phy/sfp.c
@@ -432,6 +432,19 @@ static void sfp_fixup_rollball_wait4s(struct sfp *sfp)
 	sfp->module_t_wait = msecs_to_jiffies(4000);
 }
 
+static void sfp_fixup_xikestor_2_5g(struct sfp *sfp)
+{
+	sfp_fixup_rollball(sfp);
+
+	/* This module does not immediately respond to Rollball commands. Add
+	 * a small delay to avoid unnecessary PHY access attempts. Note that
+	 * the delay should not be too long, as the RTL8221B-VB-CG PHY inside
+	 * breaks when reading some registers from MMD 30 if it has already
+	 * established a link (which takes about 4 seconds after reset).
+	 */
+	sfp->module_t_wait = msecs_to_jiffies(1000);
+}
+
 static void sfp_fixup_fs_10gt(struct sfp *sfp)
 {
 	sfp_fixup_10gbaset_30m(sfp);
@@ -581,17 +594,17 @@ static const struct sfp_quirk sfp_quirks[] = {
 	// can operate at 2500base-X, but reports 1000BASE-LX / 1300MBd in its
 	// EEPROM
 	SFP_QUIRK("Hisense-Leox", "LXT-010S-H", sfp_quirk_2500basex,
-		  sfp_fixup_ignore_tx_fault),
+		  sfp_fixup_ignore_tx_fault_and_los),
 
 	// Hisense ZNID-GPON-2311NA can operate at 2500base-X, but reports
 	// 1000BASE-LX / 1300MBd in its EEPROM
 	SFP_QUIRK("Hisense", "ZNID-GPON-2311NA", sfp_quirk_2500basex,
-		  sfp_fixup_ignore_tx_fault),
+		  sfp_fixup_ignore_tx_fault_and_los),
 
 	// HSGQ HSGQ-XPON-Stick can operate at 2500base-X, but reports
 	// 1000BASE-LX / 1300MBd in its EEPROM
 	SFP_QUIRK("HSGQ", "HSGQ-XPON-Stick", sfp_quirk_2500basex,
-		  sfp_fixup_ignore_tx_fault),
+		  sfp_fixup_ignore_tx_fault_and_los),
 
 	// Lantech 8330-262D-E and 8330-265D can operate at 2500base-X, but
 	// incorrectly report 2500MBd NRZ in their EEPROM.
@@ -636,6 +649,8 @@ static const struct sfp_quirk sfp_quirks[] = {
 	SFP_QUIRK_F("Turris", "RTSFP-10", sfp_fixup_rollball),
 	SFP_QUIRK_F("Turris", "RTSFP-10G", sfp_fixup_rollball),
 
+	SFP_QUIRK_F("XikeStor", "SKT-2.5G-100M", sfp_fixup_xikestor_2_5g),
+
 	SFP_QUIRK_S("ZOERAX", "SFP-2.5G-T", sfp_quirk_oem_2_5g),
 };
 
diff --git a/drivers/net/phy/xpowers/Makefile b/drivers/net/phy/xpowers/Makefile
new file mode 100644
index 00000000000000..ae977a7958325f
--- /dev/null
+++ b/drivers/net/phy/xpowers/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+xpowers-acx00-y := acx00.o ac200.o ac300.o
+obj-$(CONFIG_XPOWERS_ACX00_PHY) += xpowers-acx00.o
diff --git a/drivers/net/phy/xpowers/ac200.c b/drivers/net/phy/xpowers/ac200.c
new file mode 100644
index 00000000000000..afdd2a3dfbe41d
--- /dev/null
+++ b/drivers/net/phy/xpowers/ac200.c
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * X-Powers AC200 Ethernet PHY package backend
+ *
+ * Copyright (c) 2022 Arm Ltd. (Andre Przywara <andre.przywara@arm.com>)
+ * Copyright (C) 2026 James Hilliard <james.hilliard1@gmail.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/i2c.h>
+#include <linux/of.h>
+#include <linux/phy.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+#include "acx00.h"
+
+#define AC200_EPHY_BPS_EFFUSE_OFFSET	3
+#define AC200_EPHY_CLK_RATE_24_MHZ	24000000
+#define AC200_EPHY_CLK_RATE_27_MHZ	27000000
+
+#define AC200_SYS_EPHY_CTL0_REG			0x0014
+#define AC200_EPHY_RESET_DEASSERT		BIT(0)
+#define AC200_EPHY_SYSCLK_ENABLE			BIT(1)
+
+#define AC200_SYS_EPHY_CTL1_REG			0x0016
+#define AC200_EPHY_MII_IO_ENABLE			BIT(0)
+
+/* AC200-internal copy of the Ethernet PHY calibration eFuse. */
+#define AC200_EFUSE_EPHY_REG			0x8004
+
+#define AC200_EPHY_CTL_REG			0x6000
+#define AC200_EPHY_SHUTDOWN			BIT(0)
+#define AC200_EPHY_CLK_SEL_24_MHZ		BIT(2)
+#define AC200_EPHY_PHY_ADDR_MASK			GENMASK(8, 4)
+#define AC200_EPHY_RMII_SEL			BIT(11)
+#define AC200_EPHY_BPS_EFFUSE_MASK		GENMASK(15, 12)
+
+struct ac200_ephy_ctl {
+	struct acx00_ephy_control control;
+	struct regmap *regmap;
+	struct regulator *vcc;
+	struct device *dev;
+	u16 ephy_ctl;
+	unsigned int phy_addr;
+	phy_interface_t interface;
+	bool supply_enabled;
+	bool powered;
+};
+
+static u16 ac200_ephy_ctl_config(const struct ac200_ephy_ctl *priv)
+{
+	return priv->ephy_ctl |
+		(priv->interface == PHY_INTERFACE_MODE_RMII ?
+		 AC200_EPHY_RMII_SEL : 0) |
+		FIELD_PREP(AC200_EPHY_PHY_ADDR_MASK, priv->phy_addr);
+}
+
+static int ac200_ephy_ctl_write(struct ac200_ephy_ctl *priv,
+				unsigned int reg, u16 value)
+{
+	int ret;
+
+	ret = regmap_write(priv->regmap, reg, value);
+	if (ret)
+		dev_err(priv->dev, "failed to write register %#x: %pe\n",
+			reg, ERR_PTR(ret));
+
+	return ret;
+}
+
+static int ac200_ephy_ctl_disable(struct ac200_ephy_ctl *priv)
+{
+	int err;
+	int ret = 0;
+
+	if (priv->powered) {
+		ret = ac200_ephy_ctl_write(priv, AC200_EPHY_CTL_REG,
+					   ac200_ephy_ctl_config(priv) |
+					   AC200_EPHY_SHUTDOWN);
+		err = ac200_ephy_ctl_write(priv, AC200_SYS_EPHY_CTL1_REG, 0);
+		if (!ret)
+			ret = err;
+		err = ac200_ephy_ctl_write(priv, AC200_SYS_EPHY_CTL0_REG, 0);
+		if (!ret)
+			ret = err;
+
+		priv->powered = false;
+	}
+
+	if (priv->supply_enabled) {
+		err = regulator_disable(priv->vcc);
+		if (!err)
+			priv->supply_enabled = false;
+		else if (!ret)
+			ret = err;
+	}
+
+	return ret;
+}
+
+static int ac200_ephy_ctl_power_off(struct acx00_ephy_control *control)
+{
+	struct ac200_ephy_ctl *priv =
+		container_of(control, struct ac200_ephy_ctl, control);
+
+	return ac200_ephy_ctl_disable(priv);
+}
+
+static int
+ac200_ephy_ctl_set_interface(struct acx00_ephy_control *control,
+			     phy_interface_t interface)
+{
+	struct ac200_ephy_ctl *priv =
+		container_of(control, struct ac200_ephy_ctl, control);
+	int ret = 0;
+
+	if (interface != PHY_INTERFACE_MODE_MII &&
+	    interface != PHY_INTERFACE_MODE_RMII)
+		return -EINVAL;
+
+	if (priv->interface == interface)
+		return 0;
+
+	if (priv->powered)
+		ret = regmap_update_bits(priv->regmap, AC200_EPHY_CTL_REG,
+					 AC200_EPHY_RMII_SEL,
+					 interface == PHY_INTERFACE_MODE_RMII ?
+					 AC200_EPHY_RMII_SEL : 0);
+	if (!ret)
+		priv->interface = interface;
+
+	return ret;
+}
+
+static int ac200_ephy_ctl_power_on(struct acx00_ephy_control *control,
+				   unsigned int phy_addr)
+{
+	struct ac200_ephy_ctl *priv =
+		container_of(control, struct ac200_ephy_ctl, control);
+	u16 ephy_ctl;
+	int ret;
+
+	if (phy_addr > FIELD_MAX(AC200_EPHY_PHY_ADDR_MASK))
+		return -EINVAL;
+
+	if (priv->powered && priv->phy_addr == phy_addr)
+		return 0;
+
+	if (priv->powered) {
+		ret = ac200_ephy_ctl_disable(priv);
+		if (ret)
+			return ret;
+	}
+	priv->phy_addr = phy_addr;
+
+	if (!priv->supply_enabled) {
+		ret = regulator_enable(priv->vcc);
+		if (ret)
+			return ret;
+		priv->supply_enabled = true;
+	}
+
+	ephy_ctl = ac200_ephy_ctl_config(priv);
+
+	/*
+	 * Start from a disabled state before applying the configuration. AC200
+	 * specifies no additional delay between these controls. The companion
+	 * MFD provider has completed its input-clock startup delay before
+	 * publishing the regmap.
+	 */
+	ret = ac200_ephy_ctl_write(priv, AC200_SYS_EPHY_CTL0_REG, 0);
+	if (ret)
+		goto err_disable;
+
+	ret = ac200_ephy_ctl_write(priv, AC200_SYS_EPHY_CTL1_REG,
+				   AC200_EPHY_MII_IO_ENABLE);
+	if (ret)
+		goto err_disable;
+
+	ret = ac200_ephy_ctl_write(priv, AC200_EPHY_CTL_REG,
+				   ephy_ctl | AC200_EPHY_SHUTDOWN);
+	if (ret)
+		goto err_disable;
+
+	ret = ac200_ephy_ctl_write(priv, AC200_SYS_EPHY_CTL0_REG,
+				   AC200_EPHY_RESET_DEASSERT |
+				   AC200_EPHY_SYSCLK_ENABLE);
+	if (ret)
+		goto err_disable;
+
+	ret = ac200_ephy_ctl_write(priv, AC200_EPHY_CTL_REG, ephy_ctl);
+	if (ret)
+		goto err_disable;
+
+	priv->powered = true;
+	return 0;
+
+err_disable:
+	/* Attempt every step of the shutdown sequence after a partial start. */
+	priv->powered = true;
+	ac200_ephy_ctl_disable(priv);
+
+	return ret;
+}
+
+struct acx00_ephy_control *
+ac200_ephy_ctl_create(struct phy_device *phydev,
+		      struct device_node *package_node,
+		      bool has_calibration, u8 calibration)
+{
+	struct device *dev = &phydev->mdio.dev;
+	unsigned int internal_calibration;
+	struct device_node *ac200_node;
+	struct ac200_ephy_ctl *priv;
+	struct i2c_client *client;
+	unsigned long clk_rate;
+	u8 bps_effuse_code;
+	struct clk *clk;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return ERR_PTR(-ENOMEM);
+	priv->dev = dev;
+	priv->vcc = devm_of_regulator_get(dev, package_node, "vcc");
+	if (IS_ERR(priv->vcc))
+		return ERR_PTR(dev_err_probe(dev, PTR_ERR(priv->vcc),
+					     "failed to get package supply\n"));
+
+	ac200_node = of_parse_phandle(package_node, "x-powers,ac200", 0);
+	if (!ac200_node)
+		return ERR_PTR(dev_err_probe(dev, -EINVAL,
+					     "missing x-powers,ac200 reference\n"));
+	client = of_find_i2c_device_by_node(ac200_node);
+	of_node_put(ac200_node);
+	if (!client) {
+		ret = IS_ENABLED(CONFIG_I2C) ? -EPROBE_DEFER : -ENODEV;
+		return ERR_PTR(dev_err_probe(dev, ret,
+					     "AC200 device is not registered\n"));
+	}
+
+	if (!device_link_add(dev, &client->dev,
+			     DL_FLAG_AUTOREMOVE_CONSUMER)) {
+		ret = dev_err_probe(dev, -EINVAL,
+				    "failed to link AC200 device\n");
+		goto out_put_client;
+	}
+
+	device_lock(&client->dev);
+	if (device_is_bound(&client->dev))
+		priv->regmap = dev_get_regmap(&client->dev, NULL);
+	device_unlock(&client->dev);
+	if (!priv->regmap) {
+		ret = dev_err_probe(dev, -EPROBE_DEFER,
+				    "AC200 driver is not ready\n");
+		goto out_put_client;
+	}
+
+	if (!has_calibration) {
+		ret = regmap_read(priv->regmap, AC200_EFUSE_EPHY_REG,
+				  &internal_calibration);
+		if (ret)
+			goto out_error;
+		calibration = internal_calibration;
+	}
+
+	/* The vendor driver supplies no transfer function beyond this offset. */
+	bps_effuse_code = (calibration + AC200_EPHY_BPS_EFFUSE_OFFSET) &
+			   FIELD_MAX(AC200_EPHY_BPS_EFFUSE_MASK);
+	priv->ephy_ctl =
+		FIELD_PREP(AC200_EPHY_BPS_EFFUSE_MASK, bps_effuse_code);
+	/* EPHY_MODE and BIST_CLK_EN stay clear for normal operation. */
+
+	clk = clk_get(&client->dev, NULL);
+	if (IS_ERR(clk)) {
+		ret = PTR_ERR(clk);
+		goto out_error;
+	}
+
+	/* The bound AC200 provider enables and exclusively pins this rate. */
+	clk_rate = clk_get_rate(clk);
+	clk_put(clk);
+
+	switch (clk_rate) {
+	case AC200_EPHY_CLK_RATE_24_MHZ:
+		priv->ephy_ctl |= AC200_EPHY_CLK_SEL_24_MHZ;
+		break;
+	case AC200_EPHY_CLK_RATE_27_MHZ:
+		break;
+	default:
+		ret = dev_err_probe(dev, -EINVAL,
+				    "unsupported AC200 clock rate %lu Hz\n",
+				    clk_rate);
+		goto out_put_client;
+	}
+
+	priv->control.power_on = ac200_ephy_ctl_power_on;
+	priv->control.power_off = ac200_ephy_ctl_power_off;
+	priv->control.set_interface = ac200_ephy_ctl_set_interface;
+	/* MII is the reset default used until the MAC supplies its interface. */
+	priv->interface = PHY_INTERFACE_MODE_MII;
+	put_device(&client->dev);
+
+	return &priv->control;
+
+out_error:
+	ret = dev_err_probe(dev, ret, "failed to initialize AC200 control\n");
+out_put_client:
+	put_device(&client->dev);
+	return ERR_PTR(ret);
+}
diff --git a/drivers/net/phy/xpowers/ac300.c b/drivers/net/phy/xpowers/ac300.c
new file mode 100644
index 00000000000000..aa560432a14c9c
--- /dev/null
+++ b/drivers/net/phy/xpowers/ac300.c
@@ -0,0 +1,387 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * X-Powers AC300 Ethernet PHY package backend
+ *
+ * Copyright (C) 2026 James Hilliard <james.hilliard1@gmail.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/phy.h>
+#include <linux/regulator/consumer.h>
+
+#include "../phylib.h"
+#include "acx00.h"
+
+#define AC300_EPHY_BGS_EFFUSE_OFFSET	3
+#define AC300_EPHY_CLK_RATE_24_MHZ	24000000
+#define AC300_EPHY_CLK_RATE_25_MHZ	25000000
+#define AC300_EPHY_CLK_RATE_27_MHZ	27000000
+#define AC300_SYS_CONTROL_REG			0x00
+#define AC300_PACKAGE_STATUS_MASK		GENMASK(11, 8)
+#define AC300_EPHY_CLK_SEL_MASK			GENMASK(7, 6)
+#define AC300_EPHY_CLK_SEL_25_MHZ		FIELD_PREP(AC300_EPHY_CLK_SEL_MASK, 0)
+#define AC300_EPHY_CLK_SEL_27_MHZ		FIELD_PREP(AC300_EPHY_CLK_SEL_MASK, 1)
+#define AC300_EPHY_CLK_SEL_24_MHZ		FIELD_PREP(AC300_EPHY_CLK_SEL_MASK, 2)
+#define AC300_EFUSE_CLK_ENABLE			BIT(5)
+#define AC300_EPHY_REG_CLK_ENABLE		BIT(4)
+#define AC300_CLKIN_GATING_ENABLE		BIT(2)
+#define AC300_EPHY_RESET_DEASSERT		BIT(1)
+#define AC300_CHIP_RESET_DEASSERT		BIT(0)
+
+#define AC300_PACKAGE_POR_INTERNAL_DLDO		BIT(3)
+#define AC300_PACKAGE_PHY_ADDR_MASK		GENMASK(2, 0)
+
+#define AC300_SYS_BIAS1_REG			0x02
+#define AC300_INTERNAL_DLDO_ENABLE		BIT(15)
+
+#define AC300_SYS_IO_REG			0x05
+#define AC300_MDIO_DRV_MASK			GENMASK(15, 14)
+#define AC300_MII_DRV_MASK			GENMASK(11, 10)
+#define AC300_IO_DRV_LEVEL_2			2
+#define AC300_CLKIN_PAD_ENABLE			BIT(4)
+#define AC300_EPHY_MII_IO_ENABLE			BIT(0)
+
+#define AC300_EPHY_CONFIG_REG			0x06
+#define AC300_EPHY_BGS_EFFUSE_MASK		GENMASK(15, 12)
+#define AC300_EPHY_RMII_SEL			BIT(11)
+#define AC300_EPHY_SHUTDOWN			BIT(0)
+
+#define AC300_SYS_CONTROL_ENABLE_BITS \
+	(AC300_EFUSE_CLK_ENABLE | AC300_EPHY_REG_CLK_ENABLE | \
+	 AC300_CLKIN_GATING_ENABLE | AC300_EPHY_RESET_DEASSERT | \
+	 AC300_CHIP_RESET_DEASSERT)
+
+#define AC300_SYS_IO_VALUE \
+	(FIELD_PREP(AC300_MDIO_DRV_MASK, AC300_IO_DRV_LEVEL_2) | \
+	 FIELD_PREP(AC300_MII_DRV_MASK, AC300_IO_DRV_LEVEL_2) | \
+	 AC300_CLKIN_PAD_ENABLE | AC300_EPHY_MII_IO_ENABLE)
+
+struct ac300_ephy_ctl {
+	struct acx00_ephy_control control;
+	struct phy_device *phydev;
+	struct regulator *vcc;
+	struct clk *clk;
+	u16 sys_control;
+	u16 ephy_config;
+	phy_interface_t interface;
+	bool package_known;
+	bool internal_dldo;
+	bool supply_enabled;
+	bool powered;
+};
+
+static unsigned int
+ac300_ephy_ctl_link_addr(const struct ac300_ephy_ctl *priv)
+{
+	return priv->phydev->mdio.addr;
+}
+
+static int ac300_ephy_ctl_read(struct ac300_ephy_ctl *priv, u32 regnum)
+{
+	int ret;
+
+	phy_lock_mdio_bus(priv->phydev);
+	ret = __phy_package_read(priv->phydev,
+				 AC300_EPHY_CONTROL_ADDR_OFFSET, regnum);
+	phy_unlock_mdio_bus(priv->phydev);
+
+	return ret;
+}
+
+static int ac300_ephy_ctl_write(struct ac300_ephy_ctl *priv, u32 regnum,
+				u16 val)
+{
+	int ret;
+
+	phy_lock_mdio_bus(priv->phydev);
+	ret = __phy_package_write(priv->phydev,
+				  AC300_EPHY_CONTROL_ADDR_OFFSET, regnum, val);
+	phy_unlock_mdio_bus(priv->phydev);
+
+	return ret;
+}
+
+static int ac300_ephy_ctl_modify(struct ac300_ephy_ctl *priv, u32 regnum,
+				 u16 mask, u16 set)
+{
+	int ret;
+
+	phy_lock_mdio_bus(priv->phydev);
+	ret = __phy_package_read(priv->phydev,
+				 AC300_EPHY_CONTROL_ADDR_OFFSET, regnum);
+	if (ret >= 0) {
+		u16 val = (ret & ~mask) | (set & mask);
+
+		ret = val == ret ? 0 :
+			__phy_package_write(priv->phydev,
+					    AC300_EPHY_CONTROL_ADDR_OFFSET,
+					    regnum, val);
+	}
+	phy_unlock_mdio_bus(priv->phydev);
+
+	return ret;
+}
+
+static u16 ac300_ephy_ctl_config(const struct ac300_ephy_ctl *priv)
+{
+	return priv->ephy_config |
+		(priv->interface == PHY_INTERFACE_MODE_RMII ?
+		 AC300_EPHY_RMII_SEL : 0);
+}
+
+static u16 ac300_ephy_ctl_reset_value(const struct ac300_ephy_ctl *priv)
+{
+	/*
+	 * A chip reset restores DLDOEN to one. Until PKG_STATUS has identified
+	 * the supply arrangement, preserve any external-VDD setup left by the
+	 * bootloader by asserting only the EPHY reset.
+	 */
+	return !priv->package_known || !priv->internal_dldo ?
+		AC300_CHIP_RESET_DEASSERT : 0;
+}
+
+static int ac300_ephy_ctl_disable(struct ac300_ephy_ctl *priv)
+{
+	int err;
+	int ret = 0;
+
+	if (priv->powered) {
+		ret = ac300_ephy_ctl_write(priv, AC300_EPHY_CONFIG_REG,
+					   ac300_ephy_ctl_config(priv) |
+					   AC300_EPHY_SHUTDOWN);
+		err = ac300_ephy_ctl_write(priv, AC300_SYS_IO_REG, 0);
+		if (!ret)
+			ret = err;
+		err = ac300_ephy_ctl_write(priv, AC300_SYS_CONTROL_REG,
+					   ac300_ephy_ctl_reset_value(priv));
+		if (!ret)
+			ret = err;
+
+		clk_disable_unprepare(priv->clk);
+		priv->powered = false;
+	}
+
+	if (priv->supply_enabled) {
+		err = regulator_disable(priv->vcc);
+		if (!err)
+			priv->supply_enabled = false;
+		else if (!ret)
+			ret = err;
+	}
+
+	return ret;
+}
+
+static int ac300_ephy_ctl_power_off(struct acx00_ephy_control *control)
+{
+	struct ac300_ephy_ctl *priv =
+		container_of(control, struct ac300_ephy_ctl, control);
+
+	return ac300_ephy_ctl_disable(priv);
+}
+
+static int
+ac300_ephy_ctl_set_interface(struct acx00_ephy_control *control,
+			     phy_interface_t interface)
+{
+	struct ac300_ephy_ctl *priv =
+		container_of(control, struct ac300_ephy_ctl, control);
+	int ret = 0;
+
+	if (interface != PHY_INTERFACE_MODE_MII &&
+	    interface != PHY_INTERFACE_MODE_RMII)
+		return -EINVAL;
+
+	if (priv->interface == interface)
+		return 0;
+
+	if (priv->powered)
+		ret = ac300_ephy_ctl_modify(priv, AC300_EPHY_CONFIG_REG,
+					    AC300_EPHY_RMII_SEL,
+					    interface == PHY_INTERFACE_MODE_RMII ?
+					    AC300_EPHY_RMII_SEL : 0);
+	if (!ret)
+		priv->interface = interface;
+
+	return ret;
+}
+
+static int ac300_ephy_ctl_power_on(struct acx00_ephy_control *control,
+				   unsigned int phy_addr)
+{
+	struct ac300_ephy_ctl *priv =
+		container_of(control, struct ac300_ephy_ctl, control);
+	u8 package_status;
+	u16 reset_value;
+	int sys_control;
+	int ret;
+
+	if (phy_addr != ac300_ephy_ctl_link_addr(priv))
+		return -EINVAL;
+
+	if (priv->powered)
+		return 0;
+
+	if (!priv->supply_enabled) {
+		ret = regulator_enable(priv->vcc);
+		if (ret)
+			return ret;
+		priv->supply_enabled = true;
+
+		/* Wait for the power-on reset interval specified by the manual. */
+		fsleep(10000);
+	}
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		goto err_power_off;
+	priv->powered = true;
+
+	/* Keep the external-supply configuration across chip resets. */
+	reset_value = ac300_ephy_ctl_reset_value(priv);
+	ret = ac300_ephy_ctl_write(priv, AC300_SYS_CONTROL_REG, reset_value);
+	if (ret)
+		goto err_power_off;
+
+	/* The manual requires both resets to be released before the clocks. */
+	ret = ac300_ephy_ctl_write(priv, AC300_SYS_CONTROL_REG,
+				   AC300_EPHY_RESET_DEASSERT |
+				  AC300_CHIP_RESET_DEASSERT);
+	if (ret)
+		goto err_power_off;
+
+	/* Retain the vendor clock-enable defaults, including the eFuse clock. */
+	ret = ac300_ephy_ctl_write(priv, AC300_SYS_CONTROL_REG,
+				   priv->sys_control);
+	if (ret)
+		goto err_power_off;
+
+	sys_control = ac300_ephy_ctl_read(priv, AC300_SYS_CONTROL_REG);
+	if (sys_control < 0) {
+		ret = sys_control;
+		goto err_power_off;
+	}
+
+	package_status = FIELD_GET(AC300_PACKAGE_STATUS_MASK, sys_control);
+	if ((~package_status & AC300_PACKAGE_PHY_ADDR_MASK) !=
+	    ac300_ephy_ctl_link_addr(priv)) {
+		ret = -EINVAL;
+		goto err_power_off;
+	}
+
+	priv->internal_dldo = package_status & AC300_PACKAGE_POR_INTERNAL_DLDO;
+	priv->package_known = true;
+	ret = ac300_ephy_ctl_modify(priv, AC300_SYS_BIAS1_REG,
+				    AC300_INTERNAL_DLDO_ENABLE,
+				   priv->internal_dldo ?
+				   AC300_INTERNAL_DLDO_ENABLE : 0);
+	if (ret)
+		goto err_power_off;
+
+	/* Keep the documented default drive level and leave the IRQ disabled. */
+	ret = ac300_ephy_ctl_write(priv, AC300_SYS_IO_REG,
+				   AC300_SYS_IO_VALUE);
+	if (ret)
+		goto err_power_off;
+
+	fsleep(10000);
+
+	ret = ac300_ephy_ctl_write(priv, AC300_EPHY_CONFIG_REG,
+				   ac300_ephy_ctl_config(priv) |
+				  AC300_EPHY_SHUTDOWN);
+	if (ret)
+		goto err_power_off;
+
+	fsleep(10000);
+
+	ret = ac300_ephy_ctl_write(priv, AC300_EPHY_CONFIG_REG,
+				   ac300_ephy_ctl_config(priv));
+	if (ret)
+		goto err_power_off;
+
+	return 0;
+
+err_power_off:
+	ac300_ephy_ctl_disable(priv);
+
+	return ret;
+}
+
+static void ac300_ephy_clk_put(void *data)
+{
+	clk_put(data);
+}
+
+struct acx00_ephy_control *
+ac300_ephy_ctl_create(struct phy_device *phydev,
+		      struct device_node *package_node, u8 calibration)
+{
+	struct device *dev = &phydev->mdio.dev;
+	struct ac300_ephy_ctl *priv;
+	unsigned long clk_rate;
+	u8 bgs_effuse_code;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return ERR_PTR(-ENOMEM);
+	if (phydev->mdio.addr > FIELD_MAX(AC300_PACKAGE_PHY_ADDR_MASK))
+		return ERR_PTR(dev_err_probe(dev, -EINVAL,
+					     "link address is outside the package range\n"));
+	priv->phydev = phydev;
+
+	priv->vcc = devm_of_regulator_get(dev, package_node, "vcc");
+	if (IS_ERR(priv->vcc))
+		return ERR_PTR(dev_err_probe(dev, PTR_ERR(priv->vcc),
+					     "failed to get package supply\n"));
+
+	priv->clk = of_clk_get(package_node, 0);
+	if (IS_ERR(priv->clk))
+		return ERR_PTR(dev_err_probe(dev, PTR_ERR(priv->clk),
+					     "failed to get input clock\n"));
+
+	ret = devm_add_action_or_reset(dev, ac300_ephy_clk_put, priv->clk);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = devm_clk_rate_exclusive_get(dev, priv->clk);
+	if (ret)
+		return ERR_PTR(dev_err_probe(dev, ret,
+					     "failed to lock clock rate\n"));
+
+	clk_rate = clk_get_rate(priv->clk);
+	switch (clk_rate) {
+	case AC300_EPHY_CLK_RATE_24_MHZ:
+		priv->sys_control = AC300_EPHY_CLK_SEL_24_MHZ;
+		break;
+	case AC300_EPHY_CLK_RATE_25_MHZ:
+		priv->sys_control = AC300_EPHY_CLK_SEL_25_MHZ;
+		break;
+	case AC300_EPHY_CLK_RATE_27_MHZ:
+		priv->sys_control = AC300_EPHY_CLK_SEL_27_MHZ;
+		break;
+	default:
+		return ERR_PTR(dev_err_probe(dev, -EINVAL,
+					     "unsupported input clock rate %lu Hz\n",
+					     clk_rate));
+	}
+	priv->sys_control |= AC300_SYS_CONTROL_ENABLE_BITS;
+
+	/* The vendor driver supplies no transfer function beyond this offset. */
+	bgs_effuse_code = (calibration + AC300_EPHY_BGS_EFFUSE_OFFSET) &
+			   FIELD_MAX(AC300_EPHY_BGS_EFFUSE_MASK);
+	priv->ephy_config =
+		FIELD_PREP(AC300_EPHY_BGS_EFFUSE_MASK, bgs_effuse_code);
+	/* EPHY_MODE and BIST_CLK_EN stay clear for normal operation. */
+
+	priv->control.power_on = ac300_ephy_ctl_power_on;
+	priv->control.power_off = ac300_ephy_ctl_power_off;
+	priv->control.set_interface = ac300_ephy_ctl_set_interface;
+	/* MII is the reset default used until the MAC supplies its interface. */
+	priv->interface = PHY_INTERFACE_MODE_MII;
+
+	return &priv->control;
+}
diff --git a/drivers/net/phy/xpowers/acx00.c b/drivers/net/phy/xpowers/acx00.c
new file mode 100644
index 00000000000000..d19ba6d8069a4a
--- /dev/null
+++ b/drivers/net/phy/xpowers/acx00.c
@@ -0,0 +1,633 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * X-Powers AC200/AC300 Ethernet PHY driver
+ *
+ * Copyright (C) 2019 Jernej Skrabec <jernej.skrabec@gmail.com>
+ * Copyright (C) 2026 James Hilliard <james.hilliard1@gmail.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/phy.h>
+#include <linux/slab.h>
+
+#include "../phylib.h"
+#include "acx00.h"
+
+#define ACX00_EPHY_ID				0x00441400
+
+#define ACX00_EPHY_CONFIG_CALIBRATION_MASK	GENMASK(3, 0)
+#define ACX00_EPHY_CONFIG_VARIANT_AC300		BIT(8)
+#define ACX00_EPHY_CONFIG_CALIBRATION_LOW	BIT(9)
+
+#define ACX00_PAGE_SELECT_REG			0x1f
+#define ACX00_PAGE_SELECT_MASK			GENMASK(12, 8)
+#define ACX00_PAGE_0				0
+#define ACX00_PAGE_1				1
+#define ACX00_PAGE_2				2
+#define ACX00_PAGE_6				6
+#define ACX00_PAGE_8				8
+
+#define ACX00_PAGE0_GLOBAL_CONFIG_REG		0x13
+#define ACX00_PAGE0_XMII_RX_CLOCK_INVERT	BIT(12)
+#define ACX00_PAGE0_MDI_MODE_MASK		GENMASK(1, 0)
+#define ACX00_PAGE0_MDI_MODE_AUTO		2
+
+#define ACX00_PAGE1_APS_CONTROL_REG		0x12
+#define ACX00_PAGE1_APS_DISABLED_4S_VALUE	0x4824
+#define ACX00_PAGE1_UAPS_CONTROL_REG		0x13
+#define ACX00_PAGE1_UAPS_ENABLE			BIT(15)
+#define ACX00_PAGE1_INTELLIGENT_EEE_CONTROL_REG	0x17
+#define ACX00_PAGE1_INTELLIGENT_EEE_ENABLE	BIT(3)
+
+#define ACX00_PAGE2_TX_DATA_CONTROL_REG		0x18
+#define ACX00_PAGE2_10BT_FIR_SELECT_MASK		GENMASK(14, 12)
+#define ACX00_PAGE2_10BT_FIR_SELECT_DEFAULT	0
+
+#define ACX00_PAGE6_ADC_CONTROL_REG		0x10
+#define ACX00_PAGE6_ADC_CONTROL_LOW_CAL_VALUE	0x5523
+#define ACX00_PAGE6_AFE_RX_CONTROL_REG		0x13
+#define ACX00_PAGE6_AFE_RX_CONTROL_VALUE		0xf000
+#define ACX00_PAGE6_AFE_EQ_RX_DETECT_CONTROL_REG	0x14
+#define AC200_PAGE6_AFE_EQ_RX_DETECT_VALUE	0x708f
+#define AC300_PAGE6_AFE_EQ_RX_DETECT_VALUE	0x708b
+#define ACX00_PAGE6_AFE_EQ_RX_DETECT_LOW_CAL_VALUE 0x7809
+#define ACX00_PAGE6_TX_LEVEL_REG			0x15
+#define ACX00_PAGE6_TX_LEVEL_100M_MASK		GENMASK(15, 8)
+#define ACX00_PAGE6_TX_LEVEL_10M_MASK		GENMASK(7, 0)
+#define ACX00_PAGE6_TX_LEVEL_VALUE(_100m, _10m) \
+	(FIELD_PREP(ACX00_PAGE6_TX_LEVEL_100M_MASK, (_100m)) | \
+	 FIELD_PREP(ACX00_PAGE6_TX_LEVEL_10M_MASK, (_10m)))
+#define ACX00_PAGE6_TX_LEVEL_DEFAULT_VALUE \
+	ACX00_PAGE6_TX_LEVEL_VALUE(0x15, 0x30)
+#define ACX00_PAGE6_TX_LEVEL_LOW_CAL_VALUE \
+	ACX00_PAGE6_TX_LEVEL_VALUE(0x35, 0x33)
+
+#define ACX00_PAGE8_AFE_CONTROL_REG		0x18
+#define ACX00_PAGE8_AFE_CONTROL_VALUE		0x00bc
+#define ACX00_PAGE8_AUTO_CAL_CONTROL_REG		0x1d
+#define ACX00_PAGE8_AUTO_CAL_TX_LEVEL_ADJUST_BYPASS BIT(11)
+#define ACX00_PAGE8_AUTO_CAL_LOW_CAL_OPAQUE_BITS	0x0044
+#define ACX00_PAGE8_AUTO_CAL_LOW_VALUE \
+	(ACX00_PAGE8_AUTO_CAL_TX_LEVEL_ADJUST_BYPASS | \
+	 ACX00_PAGE8_AUTO_CAL_LOW_CAL_OPAQUE_BITS)
+
+/*
+ * Another integration of this exact-ID PHY documents its digital vendor
+ * register map, which also matches the observed ACx00 reset values. The ACx00
+ * analog-page field encodings remain unpublished, so keep those as opaque
+ * vendor initialization values instead of inventing bit definitions.
+ */
+
+struct acx00_ephy_priv {
+	struct phy_device *phydev;
+	struct acx00_ephy_control *control;
+	/* Serializes package state and multi-register power sequences. */
+	struct mutex state_lock;
+	bool is_ac300;
+	bool use_low_calibration_tuning;
+	bool xmii_rx_clock_inverted;
+};
+
+static int acx00_ephy_read_page(struct phy_device *phydev)
+{
+	int ret;
+
+	ret = __phy_read(phydev, ACX00_PAGE_SELECT_REG);
+	if (ret < 0)
+		return ret;
+
+	return FIELD_GET(ACX00_PAGE_SELECT_MASK, ret);
+}
+
+static int acx00_ephy_write_page(struct phy_device *phydev, int page)
+{
+	return __phy_write(phydev, ACX00_PAGE_SELECT_REG,
+			   FIELD_PREP(ACX00_PAGE_SELECT_MASK, page));
+}
+
+static int acx00_ephy_control_power_on(struct acx00_ephy_priv *priv)
+{
+	lockdep_assert_held(&priv->state_lock);
+
+	return priv->control->power_on(priv->control,
+				       priv->phydev->mdio.addr);
+}
+
+static int acx00_ephy_control_power_off(struct acx00_ephy_priv *priv)
+{
+	lockdep_assert_held(&priv->state_lock);
+
+	return priv->control->power_off(priv->control);
+}
+
+static int acx00_ephy_set_interface(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+
+	lockdep_assert_held(&priv->state_lock);
+
+	if (phydev->interface == PHY_INTERFACE_MODE_NA)
+		return 0;
+	if (phydev->interface != PHY_INTERFACE_MODE_MII &&
+	    phydev->interface != PHY_INTERFACE_MODE_RMII)
+		return -EINVAL;
+
+	return priv->control->set_interface(priv->control,
+					    phydev->interface);
+}
+
+static void acx00_ephy_control_power_off_warn(struct acx00_ephy_priv *priv)
+{
+	int ret;
+
+	ret = acx00_ephy_control_power_off(priv);
+	if (ret)
+		phydev_warn(priv->phydev,
+			    "failed to power off control block: %pe\n",
+			    ERR_PTR(ret));
+}
+
+static void acx00_ephy_control_release(void *data)
+{
+	struct acx00_ephy_priv *priv = data;
+
+	mutex_lock(&priv->state_lock);
+	acx00_ephy_control_power_off_warn(priv);
+	mutex_unlock(&priv->state_lock);
+}
+
+static int acx00_ephy_read_nvmem_u16(struct device_node *node,
+				     const char *name, size_t min_len,
+				     u16 *value)
+{
+	struct nvmem_cell *cell;
+	size_t i;
+	size_t len;
+	u8 *buf;
+	u16 val = 0;
+
+	cell = of_nvmem_cell_get(node, name);
+	if (IS_ERR(cell))
+		return PTR_ERR(cell);
+
+	buf = nvmem_cell_read(cell, &len);
+	nvmem_cell_put(cell);
+	if (IS_ERR(buf))
+		return PTR_ERR(buf);
+	if (len < min_len || len > sizeof(*value)) {
+		kfree(buf);
+		return len > sizeof(*value) ? -ERANGE : -EINVAL;
+	}
+
+	for (i = 0; i < len; i++)
+		val |= (u16)buf[i] << (8 * i);
+	kfree(buf);
+	*value = val;
+
+	return 0;
+}
+
+static int acx00_ephy_init_package(struct phy_device *phydev,
+				   struct acx00_ephy_priv *priv)
+{
+	struct device *dev = &phydev->mdio.dev;
+	struct device_node *package_node;
+	bool selectable;
+	bool fixed_ac300;
+	bool has_configuration;
+	u8 calibration;
+	u32 base_addr;
+	u16 configuration = 0;
+	int ret;
+
+	package_node = of_get_parent(dev->of_node);
+	if (!package_node)
+		return -EINVAL;
+	if (!of_node_name_eq(package_node, "ethernet-phy-package")) {
+		ret = dev_err_probe(dev, -EINVAL,
+				    "PHY is not in an Ethernet PHY package\n");
+		goto out_put_node;
+	}
+
+	ret = of_property_read_u32(package_node, "reg", &base_addr);
+	if (ret || base_addr != phydev->mdio.addr) {
+		ret = dev_err_probe(dev, -EINVAL,
+				    "package and link PHY addresses differ\n");
+		goto out_put_node;
+	}
+
+	selectable = of_device_is_compatible(package_node,
+					     "x-powers,acx00-ephy-package");
+	fixed_ac300 = of_device_is_compatible(package_node,
+					      "x-powers,ac300-ephy-package");
+	if (!selectable && !fixed_ac300 &&
+	    !of_device_is_compatible(package_node,
+				     "x-powers,ac200-ephy-package")) {
+		ret = dev_err_probe(dev, -EINVAL,
+				    "unsupported Ethernet PHY package\n");
+		goto out_put_node;
+	}
+
+	ret = devm_of_phy_package_join(dev, phydev, 0);
+	if (ret)
+		goto out_put_node;
+
+	has_configuration =
+		of_property_match_string(package_node, "nvmem-cell-names",
+					 "configuration") >= 0;
+	if (has_configuration) {
+		ret = acx00_ephy_read_nvmem_u16(package_node, "configuration",
+						selectable || fixed_ac300 ? 2 : 1,
+						&configuration);
+		if (ret) {
+			ret = dev_err_probe(dev, ret,
+					    "failed to read package configuration\n");
+			goto out_put_node;
+		}
+	} else if (selectable || fixed_ac300) {
+		ret = dev_err_probe(dev, -EINVAL,
+				    "package configuration is required\n");
+		goto out_put_node;
+	}
+
+	if (selectable) {
+		priv->is_ac300 =
+			configuration & ACX00_EPHY_CONFIG_VARIANT_AC300;
+	} else {
+		priv->is_ac300 = fixed_ac300;
+		if (has_configuration &&
+		    !!(configuration & ACX00_EPHY_CONFIG_VARIANT_AC300) !=
+		    priv->is_ac300) {
+			ret = dev_err_probe(dev, -EINVAL,
+					    "configuration does not match package\n");
+			goto out_put_node;
+		}
+	}
+
+	calibration = FIELD_GET(ACX00_EPHY_CONFIG_CALIBRATION_MASK,
+				configuration);
+
+	priv->use_low_calibration_tuning =
+		priv->is_ac300 &&
+		!!(configuration & ACX00_EPHY_CONFIG_CALIBRATION_LOW);
+	priv->xmii_rx_clock_inverted =
+		of_property_read_bool(package_node,
+				      "x-powers,xmii-rx-clock-inverted");
+
+	if (priv->is_ac300)
+		priv->control =
+			ac300_ephy_ctl_create(phydev, package_node,
+					      calibration);
+	else
+		priv->control =
+			ac200_ephy_ctl_create(phydev, package_node,
+					      has_configuration,
+					      calibration);
+	ret = PTR_ERR_OR_ZERO(priv->control);
+
+out_put_node:
+	of_node_put(package_node);
+	return ret;
+}
+
+static int acx00_ephy_disable_autonomous_eee(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+	int ret;
+
+	mutex_lock(&priv->state_lock);
+	ret = phy_modify_paged(phydev, ACX00_PAGE_1,
+			       ACX00_PAGE1_INTELLIGENT_EEE_CONTROL_REG,
+			       ACX00_PAGE1_INTELLIGENT_EEE_ENABLE, 0);
+	mutex_unlock(&priv->state_lock);
+
+	return ret;
+}
+
+static int acx00_ephy_probe(struct phy_device *phydev)
+{
+	struct device *dev = &phydev->mdio.dev;
+	struct acx00_ephy_priv *priv;
+	int ret;
+
+	if (!dev->of_node)
+		return -ENODEV;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->phydev = phydev;
+	mutex_init(&priv->state_lock);
+	ret = acx00_ephy_init_package(phydev, priv);
+	if (ret)
+		return ret;
+
+	phydev->priv = priv;
+	ret = devm_add_action_or_reset(dev, acx00_ephy_control_release, priv);
+	if (ret)
+		return ret;
+
+	mutex_lock(&priv->state_lock);
+	ret = acx00_ephy_control_power_on(priv);
+	mutex_unlock(&priv->state_lock);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to power on control block\n");
+
+	return 0;
+}
+
+static int acx00_ephy_soft_reset(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+	int ret;
+
+	mutex_lock(&priv->state_lock);
+
+	ret = acx00_ephy_set_interface(phydev);
+	if (ret)
+		goto out_power_off;
+
+	ret = acx00_ephy_control_power_on(priv);
+	if (ret)
+		goto out_power_off;
+
+	/* ACx00 can acknowledge reset in power-down without restarting. */
+	ret = genphy_resume(phydev);
+	if (ret)
+		goto out_power_off;
+
+	ret = genphy_soft_reset(phydev);
+	if (ret)
+		goto out_power_off;
+
+	goto out_unlock;
+
+out_power_off:
+	acx00_ephy_control_power_off_warn(priv);
+out_unlock:
+	mutex_unlock(&priv->state_lock);
+
+	return ret;
+}
+
+static int __acx00_ephy_config_init(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+	u16 afe_eq_rx_detect = priv->is_ac300 ?
+		AC300_PAGE6_AFE_EQ_RX_DETECT_VALUE :
+		AC200_PAGE6_AFE_EQ_RX_DETECT_VALUE;
+	u16 tx_level_value = ACX00_PAGE6_TX_LEVEL_DEFAULT_VALUE;
+	u16 global_config;
+	int oldpage;
+	int ret;
+
+	lockdep_assert_held(&priv->state_lock);
+
+	global_config = FIELD_PREP(ACX00_PAGE0_MDI_MODE_MASK,
+				   ACX00_PAGE0_MDI_MODE_AUTO);
+	if (priv->xmii_rx_clock_inverted)
+		global_config |= ACX00_PAGE0_XMII_RX_CLOCK_INVERT;
+
+	ret = phy_modify_paged(phydev, ACX00_PAGE_0,
+			       ACX00_PAGE0_GLOBAL_CONFIG_REG,
+			       ACX00_PAGE0_XMII_RX_CLOCK_INVERT |
+			       ACX00_PAGE0_MDI_MODE_MASK, global_config);
+	if (ret)
+		return ret;
+
+	if (priv->is_ac300 && priv->use_low_calibration_tuning) {
+		afe_eq_rx_detect =
+			ACX00_PAGE6_AFE_EQ_RX_DETECT_LOW_CAL_VALUE;
+		tx_level_value = ACX00_PAGE6_TX_LEVEL_LOW_CAL_VALUE;
+	}
+
+	oldpage = phy_select_page(phydev, ACX00_PAGE_1);
+	if (oldpage < 0)
+		return phy_restore_page(phydev, oldpage, oldpage);
+
+	ret = __phy_write(phydev, ACX00_PAGE1_APS_CONTROL_REG,
+			  ACX00_PAGE1_APS_DISABLED_4S_VALUE);
+	if (ret)
+		goto out_restore_page;
+	ret = __phy_modify(phydev, ACX00_PAGE1_UAPS_CONTROL_REG,
+			   ACX00_PAGE1_UAPS_ENABLE, 0);
+	if (ret)
+		goto out_restore_page;
+	ret = __phy_modify(phydev, ACX00_PAGE1_INTELLIGENT_EEE_CONTROL_REG,
+			   ACX00_PAGE1_INTELLIGENT_EEE_ENABLE, 0);
+	if (ret)
+		goto out_restore_page;
+
+	ret = acx00_ephy_write_page(phydev, ACX00_PAGE_2);
+	if (ret)
+		goto out_restore_page;
+	ret = __phy_modify(phydev, ACX00_PAGE2_TX_DATA_CONTROL_REG,
+			   ACX00_PAGE2_10BT_FIR_SELECT_MASK,
+			   FIELD_PREP(ACX00_PAGE2_10BT_FIR_SELECT_MASK,
+				      ACX00_PAGE2_10BT_FIR_SELECT_DEFAULT));
+	if (ret)
+		goto out_restore_page;
+
+	ret = acx00_ephy_write_page(phydev, ACX00_PAGE_6);
+	if (ret)
+		goto out_restore_page;
+	ret = __phy_write(phydev, ACX00_PAGE6_AFE_EQ_RX_DETECT_CONTROL_REG,
+			  afe_eq_rx_detect);
+	if (ret)
+		goto out_restore_page;
+	ret = __phy_write(phydev, ACX00_PAGE6_AFE_RX_CONTROL_REG,
+			  ACX00_PAGE6_AFE_RX_CONTROL_VALUE);
+	if (ret)
+		goto out_restore_page;
+	if (priv->is_ac300 && priv->use_low_calibration_tuning) {
+		ret = __phy_write(phydev, ACX00_PAGE6_ADC_CONTROL_REG,
+				  ACX00_PAGE6_ADC_CONTROL_LOW_CAL_VALUE);
+		if (ret)
+			goto out_restore_page;
+	}
+	ret = __phy_write(phydev, ACX00_PAGE6_TX_LEVEL_REG, tx_level_value);
+	if (ret)
+		goto out_restore_page;
+
+	ret = acx00_ephy_write_page(phydev, ACX00_PAGE_8);
+	if (ret)
+		goto out_restore_page;
+	if (priv->is_ac300 && priv->use_low_calibration_tuning) {
+		ret = __phy_write(phydev, ACX00_PAGE8_AUTO_CAL_CONTROL_REG,
+				  ACX00_PAGE8_AUTO_CAL_LOW_VALUE);
+		if (ret)
+			goto out_restore_page;
+	}
+	ret = __phy_write(phydev, ACX00_PAGE8_AFE_CONTROL_REG,
+			  ACX00_PAGE8_AFE_CONTROL_VALUE);
+
+out_restore_page:
+	ret = phy_restore_page(phydev, oldpage, ret);
+	if (ret)
+		return ret;
+
+	/* Restore the standard EEE policy retained by phylib across resets. */
+	ret = genphy_c45_an_config_eee_aneg(phydev);
+	if (ret <= 0)
+		return ret;
+
+	return genphy_restart_aneg(phydev);
+}
+
+static int acx00_ephy_config_init(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+	int ret;
+
+	mutex_lock(&priv->state_lock);
+	ret = __acx00_ephy_config_init(phydev);
+	if (ret)
+		acx00_ephy_control_power_off_warn(priv);
+	mutex_unlock(&priv->state_lock);
+
+	return ret;
+}
+
+static int __acx00_ephy_power_on_and_resume(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+	int ret;
+
+	lockdep_assert_held(&phydev->lock);
+	lockdep_assert_held(&priv->state_lock);
+
+	ret = acx00_ephy_set_interface(phydev);
+	if (ret)
+		goto out_power_off;
+
+	ret = acx00_ephy_control_power_on(priv);
+	if (ret)
+		goto out_power_off;
+
+	ret = genphy_resume(phydev);
+	if (ret)
+		goto out_power_off;
+
+	/* Powering off the control block loses the vendor-page state. */
+	ret = __acx00_ephy_config_init(phydev);
+	if (ret)
+		goto out_power_off;
+
+	return 0;
+
+out_power_off:
+	acx00_ephy_control_power_off_warn(priv);
+
+	return ret;
+}
+
+static int acx00_ephy_resume(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+	int ret;
+
+	mutex_lock(&priv->state_lock);
+	ret = __acx00_ephy_power_on_and_resume(phydev);
+	mutex_unlock(&priv->state_lock);
+
+	return ret;
+}
+
+static int acx00_ephy_suspend(struct phy_device *phydev)
+{
+	struct acx00_ephy_priv *priv = phydev->priv;
+	int resume_ret;
+	int ret;
+
+	/* The suspend callback is invoked without the phylib lock held. */
+	mutex_lock(&phydev->lock);
+	mutex_lock(&priv->state_lock);
+
+	ret = genphy_suspend(phydev);
+	if (ret)
+		goto out_unlock;
+
+	ret = acx00_ephy_control_power_off(priv);
+	if (ret) {
+		resume_ret = __acx00_ephy_power_on_and_resume(phydev);
+		if (!resume_ret) {
+			/*
+			 * The package reset also loses the standard link settings.
+			 * A failed suspend is not followed by resume, so restore
+			 * advertisement or forced speed/duplex before returning.
+			 */
+			resume_ret = genphy_config_aneg(phydev);
+			if (resume_ret)
+				acx00_ephy_control_power_off_warn(priv);
+		}
+		if (resume_ret)
+			phydev_warn(phydev,
+				    "failed to recover from suspend error: %pe\n",
+				    ERR_PTR(resume_ret));
+	}
+
+out_unlock:
+	mutex_unlock(&priv->state_lock);
+	mutex_unlock(&phydev->lock);
+
+	return ret;
+}
+
+static int acx00_ephy_match_phy_device(struct phy_device *phydev,
+				       const struct phy_driver *phydrv)
+{
+	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device_node *package_node;
+	bool match;
+
+	if (!genphy_match_phy_device(phydev, phydrv) || !node)
+		return 0;
+
+	/* RK630 reports the same PHY ID, so also match the package identity. */
+	package_node = of_get_parent(node);
+	if (!package_node)
+		return 0;
+	match = of_device_is_compatible(package_node,
+					"x-powers,ac200-ephy-package") ||
+		of_device_is_compatible(package_node,
+					"x-powers,ac300-ephy-package") ||
+		of_device_is_compatible(package_node,
+					"x-powers,acx00-ephy-package");
+	of_node_put(package_node);
+
+	return match;
+}
+
+static struct phy_driver acx00_ephy_driver[] = {
+	{
+		PHY_ID_MATCH_MODEL(ACX00_EPHY_ID),
+		.name = "X-Powers AC200/AC300 EPHY",
+		.match_phy_device = acx00_ephy_match_phy_device,
+		.probe = acx00_ephy_probe,
+		.read_page = acx00_ephy_read_page,
+		.write_page = acx00_ephy_write_page,
+		.soft_reset = acx00_ephy_soft_reset,
+		.config_init = acx00_ephy_config_init,
+		.disable_autonomous_eee = acx00_ephy_disable_autonomous_eee,
+		.suspend = acx00_ephy_suspend,
+		.resume = acx00_ephy_resume,
+	},
+};
+module_phy_driver(acx00_ephy_driver);
+
+static const struct mdio_device_id __maybe_unused acx00_ephy_tbl[] = {
+	{ PHY_ID_MATCH_MODEL(ACX00_EPHY_ID) },
+	{ }
+};
+MODULE_DEVICE_TABLE(mdio, acx00_ephy_tbl);
+
+MODULE_AUTHOR("James Hilliard <james.hilliard1@gmail.com>");
+MODULE_DESCRIPTION("X-Powers AC200/AC300 Ethernet PHY driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/xpowers/acx00.h b/drivers/net/phy/xpowers/acx00.h
new file mode 100644
index 00000000000000..7df6084c7a7729
--- /dev/null
+++ b/drivers/net/phy/xpowers/acx00.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __DRIVERS_NET_PHY_XPOWERS_ACX00_H
+#define __DRIVERS_NET_PHY_XPOWERS_ACX00_H
+
+#include <linux/err.h>
+#include <linux/phy.h>
+
+#define AC300_EPHY_CONTROL_ADDR_OFFSET	16
+
+struct device_node;
+
+struct acx00_ephy_control {
+	int (*power_on)(struct acx00_ephy_control *control,
+			unsigned int phy_addr);
+	int (*power_off)(struct acx00_ephy_control *control);
+	int (*set_interface)(struct acx00_ephy_control *control,
+			     phy_interface_t interface);
+};
+
+struct acx00_ephy_control *
+ac200_ephy_ctl_create(struct phy_device *phydev,
+		      struct device_node *package_node,
+		      bool has_calibration, u8 calibration);
+struct acx00_ephy_control *
+ac300_ephy_ctl_create(struct phy_device *phydev,
+		      struct device_node *package_node, u8 calibration);
+
+#endif
diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c
index ebd62a7ab54b37..b90ae0583c8401 100644
--- a/drivers/net/ppp/ppp_synctty.c
+++ b/drivers/net/ppp/ppp_synctty.c
@@ -436,10 +436,9 @@ ppp_sync_txmunge(struct syncppp *ap, struct sk_buff *skb)
 	int islcp;
 
 	/* Ensure we can safely access protocol field and LCP code */
-	if (!pskb_may_pull(skb, 3)) {
-		kfree_skb(skb);
-		return NULL;
-	}
+	if (!pskb_may_pull(skb, 3))
+		goto free_skb;
+
 	data  = skb->data;
 	proto = get_unaligned_be16(data);
 
@@ -455,10 +454,9 @@ ppp_sync_txmunge(struct syncppp *ap, struct sk_buff *skb)
 
 	/* prepend address/control fields if necessary */
 	if ((ap->flags & SC_COMP_AC) == 0 || islcp) {
-		if (skb_cow_head(skb, 2)) {
-			kfree_skb(skb);
-			return NULL;
-		}
+		if (skb_cow_head(skb, 2))
+			goto free_skb;
+
 		skb_push(skb,2);
 		skb->data[0] = PPP_ALLSTATIONS;
 		skb->data[1] = PPP_UI;
@@ -466,10 +464,14 @@ ppp_sync_txmunge(struct syncppp *ap, struct sk_buff *skb)
 
 	ap->last_xmit = jiffies;
 
-	if (skb && ap->flags & SC_LOG_OUTPKT)
+	if (ap->flags & SC_LOG_OUTPKT)
 		ppp_print_buffer ("send buffer", skb->data, skb->len);
 
 	return skb;
+
+free_skb:
+	kfree_skb(skb);
+	return NULL;
 }
 
 /*
diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c
index d9fb587a62c5e1..cf51b9c39f4ee9 100644
--- a/drivers/net/thunderbolt/main.c
+++ b/drivers/net/thunderbolt/main.c
@@ -540,11 +540,12 @@ static int tbnet_alloc_rx_buffers(struct tbnet *net, unsigned int nbuffers)
 		trace_tbnet_alloc_rx_frame(index, tf->page, dma_addr,
 					   DMA_FROM_DEVICE);
 
-		tb_ring_rx(ring->ring, &tf->frame);
+		tb_ring_rx_more(ring->ring, &tf->frame);
 
 		ring->prod++;
 	}
 
+	tb_ring_notify(ring->ring);
 	return 0;
 
 err_free:
@@ -1243,7 +1244,8 @@ static netdev_tx_t tbnet_start_xmit(struct sk_buff *skb,
 		goto err_drop;
 
 	for (i = 0; i < frame_index + 1; i++)
-		tb_ring_tx(net->tx_ring.ring, &frames[i]->frame);
+		tb_ring_tx_more(net->tx_ring.ring, &frames[i]->frame);
+	tb_ring_notify(net->tx_ring.ring);
 
 	if (net->svc->prtcstns & TBNET_MATCH_FRAGS_ID)
 		atomic_inc(&net->frame_id);
diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c
index 81d8412ce8e2fa..2ed46ab62c0065 100644
--- a/drivers/net/usb/ax88179_178a.c
+++ b/drivers/net/usb/ax88179_178a.c
@@ -32,8 +32,8 @@
 #define AX_ACCESS_EEPROM			0x04
 #define AX_ACCESS_EFUS				0x05
 #define AX_RELOAD_EEPROM_EFUSE			0x06
-#define AX_PAUSE_WATERLVL_HIGH			0x54
-#define AX_PAUSE_WATERLVL_LOW			0x55
+#define AX_PAUSE_WATERLVL_LOW			0x54
+#define AX_PAUSE_WATERLVL_HIGH			0x55
 
 #define PHYSICAL_LINK_STATUS			0x02
 	#define	AX_USB_SS		0x04
@@ -1619,11 +1619,10 @@ static int ax88179_reset(struct usbnet *dev)
 	dev->rx_urb_size = 1024 * 20;
 
 	*tmp = 0x34;
-	ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PAUSE_WATERLVL_LOW, 1, 1, tmp);
+	ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PAUSE_WATERLVL_HIGH, 1, 1, tmp);
 
 	*tmp = 0x52;
-	ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PAUSE_WATERLVL_HIGH,
-			  1, 1, tmp);
+	ax88179_write_cmd(dev, AX_ACCESS_MAC, AX_PAUSE_WATERLVL_LOW, 1, 1, tmp);
 
 	/* Enable checksum offload */
 	*tmp = AX_RXCOE_IP | AX_RXCOE_TCP | AX_RXCOE_UDP |
diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c
index 96e82f94edcf81..39b678f175dd95 100644
--- a/drivers/net/usb/catc.c
+++ b/drivers/net/usb/catc.c
@@ -233,17 +233,26 @@ static void catc_rx_done(struct urb *urb)
 	}
 
 	do {
-		if(!catc->is_f5u011) {
-			pkt_len = le16_to_cpup((__le16*)pkt_start);
-			if (pkt_len > urb->actual_length) {
+		int remaining = urb->actual_length -
+				(pkt_start - (u8 *)urb->transfer_buffer);
+
+		if (!catc->is_f5u011) {
+			if (remaining < pkt_offset) {
 				catc->netdev->stats.rx_length_errors++;
 				catc->netdev->stats.rx_errors++;
 				break;
 			}
+			pkt_len = le16_to_cpup((__le16 *)pkt_start);
 		} else {
 			pkt_len = urb->actual_length;
 		}
 
+		if (pkt_len < ETH_HLEN || pkt_len + pkt_offset > remaining) {
+			catc->netdev->stats.rx_length_errors++;
+			catc->netdev->stats.rx_errors++;
+			break;
+		}
+
 		if (!(skb = dev_alloc_skb(pkt_len)))
 			return;
 
diff --git a/drivers/net/usb/cdc_mbim.c b/drivers/net/usb/cdc_mbim.c
index 877fb0ed7d3d71..a7010a0664c7d4 100644
--- a/drivers/net/usb/cdc_mbim.c
+++ b/drivers/net/usb/cdc_mbim.c
@@ -635,6 +635,11 @@ static const struct usb_device_id mbim_devs[] = {
 	  .driver_info = (unsigned long)&cdc_mbim_info,
 	},
 
+	/* MeiG Smart SRM821 ZLP conformance */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d53, USB_CLASS_COMM, USB_CDC_SUBCLASS_MBIM, USB_CDC_PROTO_NONE),
+	  .driver_info = (unsigned long)&cdc_mbim_info,
+	},
+
 	/* Some Huawei devices, ME906s-158 (12d1:15c1) and E3372
 	 * (12d1:157d), are known to fail unless the NDP is placed
 	 * after the IP packets.  Applying the quirk to all Huawei
diff --git a/drivers/net/usb/ch9200.c b/drivers/net/usb/ch9200.c
index a206ffa76f1b93..5cd825f264207f 100644
--- a/drivers/net/usb/ch9200.c
+++ b/drivers/net/usb/ch9200.c
@@ -168,8 +168,6 @@ static int control_write(struct usbnet *dev, unsigned char request,
 		err = -EINVAL;
 	kfree(buf);
 
-	return 0;
-
 err_out:
 	return err;
 }
@@ -360,32 +358,44 @@ static int ch9200_bind(struct usbnet *dev, struct usb_interface *intf)
 	data[1] = 0x0F;
 	retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_THRESHOLD, data,
 			       0x02, CONTROL_TIMEOUT_MS);
+	if (retval < 0)
+		return retval;
 
 	data[0] = 0xA0;
 	data[1] = 0x90;
 	retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_FIFO_DEPTH, data,
 			       0x02, CONTROL_TIMEOUT_MS);
+	if (retval < 0)
+		return retval;
 
 	data[0] = 0x30;
 	data[1] = 0x00;
 	retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_PAUSE, data,
 			       0x02, CONTROL_TIMEOUT_MS);
+	if (retval < 0)
+		return retval;
 
 	data[0] = 0x17;
 	data[1] = 0xD8;
 	retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_FLOW_CONTROL,
 			       data, 0x02, CONTROL_TIMEOUT_MS);
+	if (retval < 0)
+		return retval;
 
 	/* Undocumented register */
 	data[0] = 0x01;
 	data[1] = 0x00;
 	retval = control_write(dev, REQUEST_WRITE, 0, 254, data, 0x02,
 			       CONTROL_TIMEOUT_MS);
+	if (retval < 0)
+		return retval;
 
 	data[0] = 0x5F;
 	data[1] = 0x0D;
 	retval = control_write(dev, REQUEST_WRITE, 0, MAC_REG_CTRL, data, 0x02,
 			       CONTROL_TIMEOUT_MS);
+	if (retval < 0)
+		return retval;
 
 	retval = get_mac_address(dev, addr);
 	eth_hw_addr_set(dev->net, addr);
diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
index cb782d81d84f2a..5655941f147830 100644
--- a/drivers/net/usb/lan78xx.c
+++ b/drivers/net/usb/lan78xx.c
@@ -5239,10 +5239,12 @@ static bool lan78xx_submit_deferred_urbs(struct lan78xx_net *dev)
 		    !netif_carrier_ok(dev->net) ||
 		    pipe_halted) {
 			lan78xx_release_tx_buf(dev, skb);
+			usb_put_urb(urb);
 			continue;
 		}
 
 		ret = usb_submit_urb(urb, GFP_ATOMIC);
+		usb_put_urb(urb);
 
 		if (ret == 0) {
 			netif_trans_update(dev->net);
diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c
index f51cf9cb9421c3..f8b771dcc9e031 100644
--- a/drivers/net/usb/qmi_wwan.c
+++ b/drivers/net/usb/qmi_wwan.c
@@ -1086,6 +1086,7 @@ static const struct usb_device_id products[] = {
 	{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0125)},	/* Quectel EC25, EC20 R2.0  Mini PCIe */
 	{QMI_MATCH_FF_FF_FF(0x2c7c, 0x013d)},	/* Quectel RG660QB */
 	{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0306)},	/* Quectel EP06/EG06/EM06 */
+	{QMI_MATCH_FF_FF_FF(0x2c7c, 0x030b)},	/* Quectel EM060K/EG120K-EA */
 	{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0512)},	/* Quectel EG12/EM12 */
 	{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0620)},	/* Quectel EM160R-GL */
 	{QMI_MATCH_FF_FF_FF(0x2c7c, 0x0800)},	/* Quectel RM500Q-GL */
@@ -1447,6 +1448,7 @@ static const struct usb_device_id products[] = {
 	{QMI_QUIRK_SET_DTR(0x2c7c, 0x0316, 3)},	/* Quectel RG255C */
 	{QMI_QUIRK_SET_DTR(0x2cb7, 0x0104, 4)},	/* Fibocom NL678 series */
 	{QMI_QUIRK_SET_DTR(0x2cb7, 0x0112, 0)},	/* Fibocom FG132 */
+	{QMI_FIXED_INTF(0x04b7, 0x4d22, 5)},	/* Compal EXC-T1 */
 	{QMI_QUIRK_SET_DTR(0x04b7, 0x8217, 8)},	/* Compal EXM-G1x */
 	{QMI_FIXED_INTF(0x0489, 0xe0b4, 0)},	/* Foxconn T77W968 LTE */
 	{QMI_FIXED_INTF(0x0489, 0xe0b5, 0)},	/* Foxconn T77W968 LTE with eSIM support*/
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index f61686433031c7..2077ddfa5eb010 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -1432,24 +1432,15 @@ static int generic_ocp_read(struct r8152 *tp, u16 index, u16 size,
 		return -EPERM;
 
 	while (size) {
-		if (size > limit) {
-			ret = get_registers(tp, index, type, limit, data);
-			if (ret < 0)
-				break;
-
-			index += limit;
-			data += limit;
-			size -= limit;
-		} else {
-			ret = get_registers(tp, index, type, size, data);
-			if (ret < 0)
-				break;
+		u16 n = min(size, limit);
 
-			index += size;
-			data += size;
-			size = 0;
+		ret = get_registers(tp, index, type, n, data);
+		if (ret < 0)
 			break;
-		}
+
+		index += n;
+		data += n;
+		size -= n;
 	}
 
 	if (ret == -ENODEV)
@@ -1499,28 +1490,16 @@ static int generic_ocp_write(struct r8152 *tp, u16 index, u16 byteen,
 			size -= 4;
 
 		while (size) {
-			if (size > limit) {
-				ret = set_registers(tp, index,
-						    type | BYTE_EN_DWORD,
-						    limit, data);
-				if (ret < 0)
-					goto error1;
-
-				index += limit;
-				data += limit;
-				size -= limit;
-			} else {
-				ret = set_registers(tp, index,
-						    type | BYTE_EN_DWORD,
-						    size, data);
-				if (ret < 0)
-					goto error1;
-
-				index += size;
-				data += size;
-				size = 0;
-				break;
-			}
+			u16 n = min(size, limit);
+
+			ret = set_registers(tp, index, type | BYTE_EN_DWORD,
+					    n, data);
+			if (ret < 0)
+				goto error1;
+
+			index += n;
+			data += n;
+			size -= n;
 		}
 
 		/* Set the last DWORD */
@@ -10495,7 +10474,7 @@ static int __init rtl8152_driver_init(void)
 {
 	int ret;
 
-	ret = usb_register_device_driver(&rtl8152_cfgselector_driver, THIS_MODULE);
+	ret = usb_register_device_driver(&rtl8152_cfgselector_driver);
 	if (ret)
 		return ret;
 
diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c
index 937e6fef3ac6d2..50981a28376a53 100644
--- a/drivers/net/usb/sr9700.c
+++ b/drivers/net/usb/sr9700.c
@@ -355,7 +355,8 @@ static int sr9700_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
 		/* ignore the CRC length */
 		len = (skb->data[1] | (skb->data[2] << 8)) - 4;
 
-		if (len > ETH_FRAME_LEN || len > skb->len || len < 0)
+		if (len > ETH_FRAME_LEN || len < 0 ||
+		    len > skb->len - SR_RX_OVERHEAD)
 			return 0;
 
 		/* the last packet of current skb */
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 6ed3ee81153fbe..71227d0389aa5d 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -1054,6 +1054,7 @@ static int __veth_napi_enable_range(struct net_device *dev, int start, int end)
 	for (i = start; i < end; i++) {
 		struct veth_rq *rq = &priv->rq[i];
 
+		rcu_assign_pointer(rq->xdp_prog, priv->_xdp_prog);
 		napi_enable(&rq->xdp_napi);
 		rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi);
 	}
@@ -1088,6 +1089,7 @@ static void veth_napi_del_range(struct net_device *dev, int start, int end)
 
 		rcu_assign_pointer(priv->rq[i].napi, NULL);
 		napi_disable(&rq->xdp_napi);
+		rcu_assign_pointer(rq->xdp_prog, NULL);
 		__netif_napi_del(&rq->xdp_napi);
 	}
 	synchronize_net();
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index e34c52d059d398..f6c5eacbacae77 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -125,6 +125,7 @@ struct virtnet_rq_stats {
 	u64_stats_t packets;
 	u64_stats_t bytes;
 	u64_stats_t drops;
+	u64_stats_t alloc_fail;
 	u64_stats_t xdp_packets;
 	u64_stats_t xdp_tx;
 	u64_stats_t xdp_redirects;
@@ -173,8 +174,9 @@ static const struct virtnet_stat_desc virtnet_sq_stats_desc_qstat[] = {
 };
 
 static const struct virtnet_stat_desc virtnet_rq_stats_desc_qstat[] = {
-	VIRTNET_RQ_STAT_QSTAT("packets", packets),
-	VIRTNET_RQ_STAT_QSTAT("bytes",   bytes),
+	VIRTNET_RQ_STAT_QSTAT("packets",    packets),
+	VIRTNET_RQ_STAT_QSTAT("bytes",      bytes),
+	VIRTNET_RQ_STAT_QSTAT("alloc_fail", alloc_fail),
 };
 
 #define VIRTNET_STATS_DESC_CQ(name) \
@@ -1917,8 +1919,10 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
 	}
 
 	skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len);
-	if (unlikely(!skb))
+	if (unlikely(!skb)) {
+		u64_stats_inc(&stats->alloc_fail);
 		goto err;
+	}
 
 	if (metasize)
 		skb_metadata_set(skb, metasize);
@@ -1985,6 +1989,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
 		return skb;
 	}
 
+	u64_stats_inc(&stats->alloc_fail);
 err:
 	u64_stats_inc(&stats->drops);
 	page_pool_put_page(rq->page_pool, page, -1, true);
@@ -2016,8 +2021,10 @@ static struct sk_buff *receive_big(struct net_device *dev,
 
 	skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0);
 	u64_stats_add(&stats->bytes, len - vi->hdr_len);
-	if (unlikely(!skb))
+	if (unlikely(!skb)) {
+		u64_stats_inc(&stats->alloc_fail);
 		goto err;
+	}
 
 	return skb;
 
@@ -2298,8 +2305,10 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
 	switch (act) {
 	case XDP_PASS:
 		head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
-		if (unlikely(!head_skb))
+		if (unlikely(!head_skb)) {
+			u64_stats_inc(&stats->alloc_fail);
 			break;
+		}
 
 		skb_mark_for_recycle(head_skb);
 		return head_skb;
@@ -2414,8 +2423,10 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom);
 	curr_skb = head_skb;
 
-	if (unlikely(!curr_skb))
+	if (unlikely(!curr_skb)) {
+		u64_stats_inc(&stats->alloc_fail);
 		goto err_skb;
+	}
 
 	skb_mark_for_recycle(head_skb);
 	while (--num_buf) {
@@ -2444,8 +2455,10 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		truesize = mergeable_ctx_to_truesize(ctx);
 		curr_skb  = virtnet_skb_append_frag(rq, head_skb, curr_skb, page,
 						    buf, len, truesize);
-		if (!curr_skb)
+		if (!curr_skb) {
+			u64_stats_inc(&stats->alloc_fail);
 			goto err_skb;
+		}
 	}
 
 	ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len);
@@ -2928,12 +2941,14 @@ static int virtnet_receive(struct receive_queue *rq, int budget,
 
 	u64_stats_set(&stats.packets, packets);
 	if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) {
-		if (!try_fill_recv(vi, rq, GFP_ATOMIC))
+		if (!try_fill_recv(vi, rq, GFP_ATOMIC)) {
 			/* We need to retry refilling in the next NAPI poll so
 			 * we must return budget to make sure the NAPI is
 			 * repolled.
 			 */
 			packets = budget;
+			u64_stats_inc(&stats.alloc_fail);
+		}
 	}
 
 	u64_stats_update_begin(&rq->stats.syncp);
@@ -2948,6 +2963,7 @@ static int virtnet_receive(struct receive_queue *rq, int budget,
 
 	u64_stats_add(&rq->stats.packets, u64_stats_read(&stats.packets));
 	u64_stats_add(&rq->stats.bytes, u64_stats_read(&stats.bytes));
+	u64_stats_add(&rq->stats.alloc_fail, u64_stats_read(&stats.alloc_fail));
 
 	u64_stats_update_end(&rq->stats.syncp);
 
@@ -3349,6 +3365,14 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
 	else
 		virtqueue_disable_cb(sq->vq);
 
+	if (!use_napi &&
+	    unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) {
+		DEV_STATS_INC(dev, tx_dropped);
+		dev_kfree_skb_any(skb);
+		kick = !xmit_more || netif_xmit_stopped(txq);
+		goto kick_vq;
+	}
+
 	/* timestamp packet in software */
 	skb_tx_timestamp(skb);
 
@@ -3381,6 +3405,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	kick = use_napi ? __netdev_tx_sent_queue(txq, skb->len, xmit_more) :
 			  !xmit_more || netif_xmit_stopped(txq);
+kick_vq:
 	if (kick) {
 		if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) {
 			u64_stats_update_begin(&sq->stats.syncp);
@@ -5666,6 +5691,7 @@ static void virtnet_get_base_stats(struct net_device *dev,
 	 */
 	rx->bytes = 0;
 	rx->packets = 0;
+	rx->alloc_fail = 0;
 
 	if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) {
 		rx->hw_drops = 0;
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index a0557a3a70260a..320f3584a43b20 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -616,7 +616,7 @@ static int vrf_finish_output6(struct net *net, struct sock *sk,
 	nexthop = rt6_nexthop(dst_rt6_info(dst), &ipv6_hdr(skb)->daddr);
 	neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
 	if (unlikely(!neigh))
-		neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
+		neigh = __neigh_create(nd_table(net), nexthop, dst->dev, false);
 	if (!IS_ERR(neigh)) {
 		sock_confirm_neigh(skb, neigh);
 		ret = neigh_output(neigh, skb, false);
@@ -1175,8 +1175,6 @@ static int vrf_prepare_mac_header(struct sk_buff *skb,
 	skb->protocol = eth->h_proto;
 	skb->pkt_type = PACKET_HOST;
 
-	skb_postpush_rcsum(skb, skb->data, ETH_HLEN);
-
 	skb_pull_inline(skb, ETH_HLEN);
 
 	return 0;
diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
index c1d54339fa2b15..27b0b6567d5210 100644
--- a/drivers/net/vxlan/vxlan_core.c
+++ b/drivers/net/vxlan/vxlan_core.c
@@ -110,20 +110,23 @@ static struct vxlan_dev *vxlan_vs_find_vni(struct vxlan_sock *vs,
 		vni = 0;
 
 	hlist_for_each_entry_rcu(node, vni_head(vs, vni), hlist) {
+		const struct vxlan_config *cfg;
+
 		if (!node->vxlan)
 			continue;
+
+		cfg = rcu_dereference(node->vxlan->cfg);
+
 		vnode = NULL;
-		if (node->vxlan->cfg.flags & VXLAN_F_VNIFILTER) {
+		if (cfg->flags & VXLAN_F_VNIFILTER) {
 			vnode = vxlan_vnifilter_lookup(node->vxlan, vni);
 			if (!vnode)
 				continue;
-		} else if (node->vxlan->default_dst.remote_vni != vni) {
+		} else if (cfg->vni != vni) {
 			continue;
 		}
 
 		if (IS_ENABLED(CONFIG_IPV6)) {
-			const struct vxlan_config *cfg = &node->vxlan->cfg;
-
 			if ((cfg->flags & VXLAN_F_IPV6_LINKLOCAL) &&
 			    cfg->remote_ifindex != ifindex)
 				continue;
@@ -157,6 +160,7 @@ static int vxlan_fdb_info(struct sk_buff *skb, struct vxlan_dev *vxlan,
 			  u32 portid, u32 seq, int type, unsigned int flags,
 			  const struct vxlan_rdst *rdst)
 {
+	const struct vxlan_config *cfg = rcu_dereference_rtnl(vxlan->cfg);
 	unsigned long now = jiffies;
 	struct nda_cacheinfo ci;
 	bool send_ip, send_eth;
@@ -216,10 +220,10 @@ static int vxlan_fdb_info(struct sk_buff *skb, struct vxlan_dev *vxlan,
 			goto nla_put_failure;
 
 		if (rdst->remote_port &&
-		    rdst->remote_port != vxlan->cfg.dst_port &&
+		    rdst->remote_port != cfg->dst_port &&
 		    nla_put_be16(skb, NDA_PORT, rdst->remote_port))
 			goto nla_put_failure;
-		if (rdst->remote_vni != vxlan->default_dst.remote_vni &&
+		if (rdst->remote_vni != cfg->vni &&
 		    nla_put_u32(skb, NDA_VNI, be32_to_cpu(rdst->remote_vni)))
 			goto nla_put_failure;
 		if (rdst->remote_ifindex &&
@@ -227,7 +231,7 @@ static int vxlan_fdb_info(struct sk_buff *skb, struct vxlan_dev *vxlan,
 			goto nla_put_failure;
 	}
 
-	if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) && fdb->key.vni &&
+	if ((cfg->flags & VXLAN_F_COLLECT_METADATA) && fdb->key.vni &&
 	    nla_put_u32(skb, NDA_SRC_VNI,
 			be32_to_cpu(fdb->key.vni)))
 		goto nla_put_failure;
@@ -377,14 +381,15 @@ static void vxlan_fdb_miss(struct vxlan_dev *vxlan, const u8 eth_addr[ETH_ALEN])
 
 /* Look up Ethernet address in forwarding table */
 static struct vxlan_fdb *vxlan_find_mac_rcu(struct vxlan_dev *vxlan,
+					    const struct vxlan_config *cfg,
 					    const u8 *mac, __be32 vni)
 {
 	struct vxlan_fdb_key key;
 
 	memset(&key, 0, sizeof(key));
 	memcpy(key.eth_addr, mac, sizeof(key.eth_addr));
-	if (!(vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA))
-		key.vni = vxlan->default_dst.remote_vni;
+	if (!(cfg->flags & VXLAN_F_COLLECT_METADATA))
+		key.vni = cfg->vni;
 	else
 		key.vni = vni;
 
@@ -393,11 +398,12 @@ static struct vxlan_fdb *vxlan_find_mac_rcu(struct vxlan_dev *vxlan,
 }
 
 static struct vxlan_fdb *vxlan_find_mac_tx(struct vxlan_dev *vxlan,
+					   const struct vxlan_config *cfg,
 					   const u8 *mac, __be32 vni)
 {
 	struct vxlan_fdb *f;
 
-	f = vxlan_find_mac_rcu(vxlan, mac, vni);
+	f = vxlan_find_mac_rcu(vxlan, cfg, mac, vni);
 	if (f) {
 		unsigned long now = jiffies;
 
@@ -416,7 +422,7 @@ static struct vxlan_fdb *vxlan_find_mac(struct vxlan_dev *vxlan,
 	lockdep_assert_held_once(&vxlan->hash_lock);
 
 	rcu_read_lock();
-	f = vxlan_find_mac_rcu(vxlan, mac, vni);
+	f = vxlan_find_mac_rcu(vxlan, rcu_dereference(vxlan->cfg), mac, vni);
 	rcu_read_unlock();
 
 	return f;
@@ -457,7 +463,7 @@ int vxlan_fdb_find_uc(struct net_device *dev, const u8 *mac, __be32 vni,
 
 	rcu_read_lock();
 
-	f = vxlan_find_mac_rcu(vxlan, eth_addr, vni);
+	f = vxlan_find_mac_rcu(vxlan, rcu_dereference(vxlan->cfg), eth_addr, vni);
 	if (f)
 		rdst = first_remote_rcu(f);
 	if (!rdst) {
@@ -798,6 +804,7 @@ static int vxlan_fdb_nh_update(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
 			       u32 nhid, struct netlink_ext_ack *extack)
 {
 	struct nexthop *old_nh = rtnl_dereference(fdb->nh);
+	const struct vxlan_config *cfg;
 	struct nexthop *nh;
 	int err = -EINVAL;
 
@@ -826,7 +833,8 @@ static int vxlan_fdb_nh_update(struct vxlan_dev *vxlan, struct vxlan_fdb *fdb,
 	}
 
 	/* check nexthop group family */
-	switch (vxlan->default_dst.remote_ip.sa.sa_family) {
+	cfg = rtnl_dereference(vxlan->cfg);
+	switch (cfg->remote_ip.sa.sa_family) {
 	case AF_INET:
 		if (!nexthop_has_v4(nh)) {
 			err = -EAFNOSUPPORT;
@@ -863,12 +871,13 @@ int vxlan_fdb_create(struct vxlan_dev *vxlan,
 		     u32 nhid, struct vxlan_fdb **fdb,
 		     struct netlink_ext_ack *extack)
 {
+	const struct vxlan_config *cfg = rcu_dereference_rtnl(vxlan->cfg);
 	struct vxlan_rdst *rd = NULL;
 	struct vxlan_fdb *f;
 	int rc;
 
-	if (vxlan->cfg.addrmax &&
-	    vxlan->addrcnt >= vxlan->cfg.addrmax)
+	if (cfg->addrmax &&
+	    vxlan->addrcnt >= cfg->addrmax)
 		return -ENOSPC;
 
 	netdev_dbg(vxlan->dev, "add %pM -> %pIS\n", mac, ip);
@@ -1154,6 +1163,7 @@ static int vxlan_fdb_parse(struct nlattr *tb[], struct vxlan_dev *vxlan,
 			   __be32 *vni, u32 *ifindex, u32 *nhid,
 			   struct netlink_ext_ack *extack)
 {
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 	struct net *net = dev_net(vxlan->dev);
 	int err;
 
@@ -1170,7 +1180,7 @@ static int vxlan_fdb_parse(struct nlattr *tb[], struct vxlan_dev *vxlan,
 			return err;
 		}
 	} else {
-		union vxlan_addr *remote = &vxlan->default_dst.remote_ip;
+		const union vxlan_addr *remote = &cfg->remote_ip;
 
 		if (remote->sa.sa_family == AF_INET) {
 			ip->sin.sin_addr.s_addr = htonl(INADDR_ANY);
@@ -1190,7 +1200,7 @@ static int vxlan_fdb_parse(struct nlattr *tb[], struct vxlan_dev *vxlan,
 		}
 		*port = nla_get_be16(tb[NDA_PORT]);
 	} else {
-		*port = vxlan->cfg.dst_port;
+		*port = cfg->dst_port;
 	}
 
 	if (tb[NDA_VNI]) {
@@ -1200,7 +1210,7 @@ static int vxlan_fdb_parse(struct nlattr *tb[], struct vxlan_dev *vxlan,
 		}
 		*vni = cpu_to_be32(nla_get_u32(tb[NDA_VNI]));
 	} else {
-		*vni = vxlan->default_dst.remote_vni;
+		*vni = cfg->vni;
 	}
 
 	if (tb[NDA_SRC_VNI]) {
@@ -1210,7 +1220,7 @@ static int vxlan_fdb_parse(struct nlattr *tb[], struct vxlan_dev *vxlan,
 		}
 		*src_vni = cpu_to_be32(nla_get_u32(tb[NDA_SRC_VNI]));
 	} else {
-		*src_vni = vxlan->default_dst.remote_vni;
+		*src_vni = cfg->vni;
 	}
 
 	if (tb[NDA_IFINDEX]) {
@@ -1241,6 +1251,7 @@ static int vxlan_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
 			 const unsigned char *addr, u16 vid, u16 flags,
 			 bool *notified, struct netlink_ext_ack *extack)
 {
+	const struct vxlan_config *cfg;
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	/* struct net *net = dev_net(vxlan->dev); */
 	union vxlan_addr ip;
@@ -1268,7 +1279,8 @@ static int vxlan_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
 		return -EINVAL;
 	}
 
-	if (vxlan->default_dst.remote_ip.sa.sa_family != ip.sa.sa_family)
+	cfg = rtnl_dereference(vxlan->cfg);
+	if (cfg->remote_ip.sa.sa_family != ip.sa.sa_family)
 		return -EAFNOSUPPORT;
 
 	spin_lock_bh(&vxlan->hash_lock);
@@ -1405,18 +1417,21 @@ static int vxlan_fdb_get(struct sk_buff *skb,
 			 struct netlink_ext_ack *extack)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
+	const struct vxlan_config *cfg;
 	struct vxlan_fdb *f;
 	__be32 vni;
 	int err;
 
+	cfg = rcu_dereference_rtnl(vxlan->cfg);
+
 	if (tb[NDA_VNI])
 		vni = cpu_to_be32(nla_get_u32(tb[NDA_VNI]));
 	else
-		vni = vxlan->default_dst.remote_vni;
+		vni = cfg->vni;
 
 	rcu_read_lock();
 
-	f = vxlan_find_mac_rcu(vxlan, addr, vni);
+	f = vxlan_find_mac_rcu(vxlan, cfg, addr, vni);
 	if (!f) {
 		NL_SET_ERR_MSG(extack, "Fdb entry not found");
 		err = -ENOENT;
@@ -1434,6 +1449,7 @@ static int vxlan_fdb_get(struct sk_buff *skb,
  * and Tunnel endpoint.
  */
 static enum skb_drop_reason vxlan_snoop(struct net_device *dev,
+					const struct vxlan_config *cfg,
 					union vxlan_addr *src_ip,
 					const u8 *src_mac, u32 src_ifindex,
 					__be32 vni)
@@ -1452,7 +1468,7 @@ static enum skb_drop_reason vxlan_snoop(struct net_device *dev,
 		ifindex = src_ifindex;
 #endif
 
-	f = vxlan_find_mac_rcu(vxlan, src_mac, vni);
+	f = vxlan_find_mac_rcu(vxlan, cfg, src_mac, vni);
 	if (likely(f)) {
 		struct vxlan_rdst *rdst = first_remote_rcu(f);
 		unsigned long now = jiffies;
@@ -1488,9 +1504,9 @@ static enum skb_drop_reason vxlan_snoop(struct net_device *dev,
 			vxlan_fdb_update(vxlan, src_mac, src_ip,
 					 NUD_REACHABLE,
 					 NLM_F_EXCL|NLM_F_CREATE,
-					 vxlan->cfg.dst_port,
+					 cfg->dst_port,
 					 vni,
-					 vxlan->default_dst.remote_vni,
+					 cfg->vni,
 					 ifindex, NTF_SELF, 0, true, NULL);
 		spin_unlock(&vxlan->hash_lock);
 	}
@@ -1518,6 +1534,7 @@ static bool __vxlan_sock_release_prep(struct vxlan_sock *vs)
 
 static void vxlan_sock_release(struct vxlan_dev *vxlan)
 {
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 	struct vxlan_sock *sock4 = rtnl_dereference(vxlan->vn4_sock);
 #if IS_ENABLED(CONFIG_IPV6)
 	struct vxlan_sock *sock6 = rtnl_dereference(vxlan->vn6_sock);
@@ -1527,7 +1544,7 @@ static void vxlan_sock_release(struct vxlan_dev *vxlan)
 
 	RCU_INIT_POINTER(vxlan->vn4_sock, NULL);
 
-	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
+	if (cfg->flags & VXLAN_F_VNIFILTER)
 		vxlan_vs_del_vnigrp(vxlan);
 	else
 		vxlan_vs_del_dev(vxlan);
@@ -1598,6 +1615,7 @@ static void vxlan_parse_gbp_hdr(struct sk_buff *skb, u32 vxflags,
 }
 
 static enum skb_drop_reason vxlan_set_mac(struct vxlan_dev *vxlan,
+					  const struct vxlan_config *cfg,
 					  struct vxlan_sock *vs,
 					  struct sk_buff *skb, __be32 vni)
 {
@@ -1623,10 +1641,10 @@ static enum skb_drop_reason vxlan_set_mac(struct vxlan_dev *vxlan,
 #endif
 	}
 
-	if (!(vxlan->cfg.flags & VXLAN_F_LEARN))
+	if (!(cfg->flags & VXLAN_F_LEARN))
 		return SKB_NOT_DROPPED_YET;
 
-	return vxlan_snoop(skb->dev, &saddr, eth_hdr(skb)->h_source,
+	return vxlan_snoop(skb->dev, cfg, &saddr, eth_hdr(skb)->h_source,
 			   ifindex, vni);
 }
 
@@ -1657,18 +1675,21 @@ static bool vxlan_ecn_decapsulate(struct vxlan_sock *vs, void *oiph,
 static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 {
 	struct vxlan_vni_node *vninode = NULL;
-	const struct vxlanhdr *vh;
-	struct vxlan_dev *vxlan;
-	struct vxlan_sock *vs;
-	struct vxlan_metadata _md;
-	struct vxlan_metadata *md = &_md;
 	__be16 protocol = htons(ETH_P_TEB);
+	const struct vxlan_config *cfg;
 	enum skb_drop_reason reason;
+	const struct vxlanhdr *vh;
+	struct vxlan_metadata *md;
+	struct vxlan_metadata _md;
+	struct vxlan_dev *vxlan;
 	bool raw_proto = false;
-	void *oiph;
+	struct vxlan_sock *vs;
 	__be32 vni = 0;
+	void *oiph;
 	int nh;
 
+	md = &_md;
+
 	/* Need UDP and VXLAN header to be present */
 	reason = pskb_may_pull_reason(skb, VXLAN_HLEN);
 	if (reason)
@@ -1696,8 +1717,10 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 		goto drop;
 	}
 
-	if (vh->vx_flags & vxlan->cfg.reserved_bits.vx_flags ||
-	    vh->vx_vni & vxlan->cfg.reserved_bits.vx_vni) {
+	cfg = rcu_dereference(vxlan->cfg);
+
+	if (vh->vx_flags & cfg->reserved_bits.vx_flags ||
+	    vh->vx_vni & cfg->reserved_bits.vx_vni) {
 		/* If the header uses bits besides those enabled by the
 		 * netdevice configuration, treat this as a malformed packet.
 		 * This behavior diverges from VXLAN RFC (RFC7348) which
@@ -1709,12 +1732,12 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 		reason = SKB_DROP_REASON_VXLAN_INVALID_HDR;
 		DEV_STATS_INC(vxlan->dev, rx_frame_errors);
 		DEV_STATS_INC(vxlan->dev, rx_errors);
-		vxlan_vnifilter_count(vxlan, vni, vninode,
+		vxlan_vnifilter_count(vxlan, cfg, vni, vninode,
 				      VXLAN_VNI_STATS_RX_ERRORS, 0);
 		goto drop;
 	}
 
-	if (vxlan->cfg.flags & VXLAN_F_GPE) {
+	if (cfg->flags & VXLAN_F_GPE) {
 		if (!vxlan_parse_gpe_proto(vh, &protocol))
 			goto drop;
 		raw_proto = true;
@@ -1726,8 +1749,8 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 		goto drop;
 	}
 
-	if (vxlan->cfg.flags & VXLAN_F_REMCSUM_RX) {
-		reason = vxlan_remcsum(skb, vxlan->cfg.flags);
+	if (cfg->flags & VXLAN_F_REMCSUM_RX) {
+		reason = vxlan_remcsum(skb, cfg->flags);
 		if (unlikely(reason))
 			goto drop;
 	}
@@ -1752,14 +1775,14 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 		memset(md, 0, sizeof(*md));
 	}
 
-	if (vxlan->cfg.flags & VXLAN_F_GBP)
-		vxlan_parse_gbp_hdr(skb, vxlan->cfg.flags, md);
+	if (cfg->flags & VXLAN_F_GBP)
+		vxlan_parse_gbp_hdr(skb, cfg->flags, md);
 	/* Note that GBP and GPE can never be active together. This is
 	 * ensured in vxlan_dev_configure.
 	 */
 
 	if (!raw_proto) {
-		reason = vxlan_set_mac(vxlan, vs, skb, vni);
+		reason = vxlan_set_mac(vxlan, cfg, vs, skb, vni);
 		if (reason)
 			goto drop;
 	} else {
@@ -1780,7 +1803,7 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 	if (reason) {
 		DEV_STATS_INC(vxlan->dev, rx_length_errors);
 		DEV_STATS_INC(vxlan->dev, rx_errors);
-		vxlan_vnifilter_count(vxlan, vni, vninode,
+		vxlan_vnifilter_count(vxlan, cfg, vni, vninode,
 				      VXLAN_VNI_STATS_RX_ERRORS, 0);
 		goto drop;
 	}
@@ -1792,7 +1815,7 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 		reason = SKB_DROP_REASON_IP_TUNNEL_ECN;
 		DEV_STATS_INC(vxlan->dev, rx_frame_errors);
 		DEV_STATS_INC(vxlan->dev, rx_errors);
-		vxlan_vnifilter_count(vxlan, vni, vninode,
+		vxlan_vnifilter_count(vxlan, cfg, vni, vninode,
 				      VXLAN_VNI_STATS_RX_ERRORS, 0);
 		goto drop;
 	}
@@ -1802,14 +1825,15 @@ static int vxlan_rcv(struct sock *sk, struct sk_buff *skb)
 	if (unlikely(!(vxlan->dev->flags & IFF_UP))) {
 		rcu_read_unlock();
 		dev_dstats_rx_dropped(vxlan->dev);
-		vxlan_vnifilter_count(vxlan, vni, vninode,
+		vxlan_vnifilter_count(vxlan, cfg, vni, vninode,
 				      VXLAN_VNI_STATS_RX_DROPS, 0);
 		reason = SKB_DROP_REASON_DEV_READY;
 		goto drop;
 	}
 
 	dev_dstats_rx_add(vxlan->dev, skb->len);
-	vxlan_vnifilter_count(vxlan, vni, vninode, VXLAN_VNI_STATS_RX, skb->len);
+	vxlan_vnifilter_count(vxlan, cfg, vni, vninode, VXLAN_VNI_STATS_RX,
+			      skb->len);
 	gro_cells_receive(&vxlan->gro_cells, skb);
 
 	rcu_read_unlock();
@@ -1850,20 +1874,22 @@ static int vxlan_err_lookup(struct sock *sk, struct sk_buff *skb)
 	return 0;
 }
 
-static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
+static int arp_reduce(struct net_device *dev, struct sk_buff *skb,
+		      const struct vxlan_config *cfg, __be32 vni)
 {
+	struct neigh_table *tbl = arp_table(dev_net(dev));
 	struct vxlan_dev *vxlan = netdev_priv(dev);
+	struct neighbour *n;
 	struct arphdr *parp;
 	u8 *arpptr, *sha;
 	__be32 sip, tip;
-	struct neighbour *n;
 
 	if (dev->flags & IFF_NOARP)
 		goto out;
 
 	if (!pskb_network_may_pull(skb, arp_hdr_len(dev))) {
 		dev_dstats_tx_dropped(dev);
-		vxlan_vnifilter_count(vxlan, vni, NULL,
+		vxlan_vnifilter_count(vxlan, cfg, vni, NULL,
 				      VXLAN_VNI_STATS_TX_DROPS, 0);
 		goto out;
 	}
@@ -1888,7 +1914,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 	    ipv4_is_multicast(tip))
 		goto out;
 
-	n = neigh_lookup(&arp_tbl, &tip, dev);
+	n = neigh_lookup(tbl, &tip, dev);
 
 	if (n) {
 		struct vxlan_rdst *rdst = NULL;
@@ -1904,7 +1930,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 		neigh_ha_snapshot(ha, n, n->dev);
 
 		rcu_read_lock();
-		f = vxlan_find_mac_tx(vxlan, ha, vni);
+		f = vxlan_find_mac_tx(vxlan, cfg, ha, vni);
 		if (f)
 			rdst = first_remote_rcu(f);
 		if (rdst && vxlan_addr_any(&rdst->remote_ip)) {
@@ -1930,11 +1956,11 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 
 		if (netif_rx(reply) == NET_RX_DROP) {
 			dev_dstats_rx_dropped(dev);
-			vxlan_vnifilter_count(vxlan, vni, NULL,
+			vxlan_vnifilter_count(vxlan, cfg, vni, NULL,
 					      VXLAN_VNI_STATS_RX_DROPS, 0);
 		}
 
-	} else if (vxlan->cfg.flags & VXLAN_F_L3MISS) {
+	} else if (cfg->flags & VXLAN_F_L3MISS) {
 		union vxlan_addr ipa = {
 			.sin.sin_addr.s_addr = tip,
 			.sin.sin_family = AF_INET,
@@ -1958,13 +1984,15 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request,
 	struct ipv6hdr *pip6;
 	u8 *daddr;
 	int na_olen = 8; /* opt hdr + ETH_ALEN for target */
+	int headroom;
 	int ns_olen;
 	int i, len;
 
 	if (dev == NULL || !pskb_may_pull(request, request->len))
 		return NULL;
 
-	len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) +
+	headroom = LL_RESERVED_SPACE(dev);
+	len = headroom + sizeof(struct ipv6hdr) +
 		sizeof(*na) + na_olen + dev->needed_tailroom;
 	reply = alloc_skb(len, GFP_ATOMIC);
 	if (reply == NULL)
@@ -1972,7 +2000,7 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request,
 
 	reply->protocol = htons(ETH_P_IPV6);
 	reply->dev = dev;
-	skb_reserve(reply, LL_RESERVED_SPACE(request->dev));
+	skb_reserve(reply, headroom);
 	skb_push(reply, sizeof(struct ethhdr));
 	skb_reset_mac_header(reply);
 
@@ -2042,7 +2070,8 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request,
 	return reply;
 }
 
-static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
+static int neigh_reduce(struct net_device *dev, struct sk_buff *skb,
+			const struct vxlan_config *cfg, __be32 vni)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	const struct in6_addr *daddr;
@@ -2062,7 +2091,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 	    ipv6_addr_is_multicast(&msg->target))
 		goto out;
 
-	n = neigh_lookup(&nd_tbl, &msg->target, dev);
+	n = neigh_lookup(nd_table(dev_net(dev)), &msg->target, dev);
 
 	if (n) {
 		struct vxlan_rdst *rdst = NULL;
@@ -2076,7 +2105,7 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 		}
 
 		neigh_ha_snapshot(ha, n, n->dev);
-		f = vxlan_find_mac_tx(vxlan, ha, vni);
+		f = vxlan_find_mac_tx(vxlan, cfg, ha, vni);
 		if (f)
 			rdst = first_remote_rcu(f);
 		if (rdst && vxlan_addr_any(&rdst->remote_ip)) {
@@ -2095,10 +2124,10 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 
 		if (netif_rx(reply) == NET_RX_DROP) {
 			dev_dstats_rx_dropped(dev);
-			vxlan_vnifilter_count(vxlan, vni, NULL,
+			vxlan_vnifilter_count(vxlan, cfg, vni, NULL,
 					      VXLAN_VNI_STATS_RX_DROPS, 0);
 		}
-	} else if (vxlan->cfg.flags & VXLAN_F_L3MISS) {
+	} else if (cfg->flags & VXLAN_F_L3MISS) {
 		union vxlan_addr ipa = {
 			.sin6.sin6_addr = msg->target,
 			.sin6.sin6_family = AF_INET6,
@@ -2114,9 +2143,10 @@ static int neigh_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni)
 }
 #endif
 
-static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb)
+static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb,
+			       const struct vxlan_config *cfg)
 {
-	struct vxlan_dev *vxlan = netdev_priv(dev);
+	struct neigh_table *tbl;
 	struct neighbour *n;
 
 	if (is_multicast_ether_addr(eth_hdr(skb)->h_dest))
@@ -2130,9 +2160,11 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb)
 
 		if (!pskb_network_may_pull(skb, sizeof(struct iphdr)))
 			return false;
+
+		tbl = arp_table(dev_net(dev));
 		pip = ip_hdr(skb);
-		n = neigh_lookup(&arp_tbl, &pip->daddr, dev);
-		if (!n && (vxlan->cfg.flags & VXLAN_F_L3MISS)) {
+		n = neigh_lookup(tbl, &pip->daddr, dev);
+		if (!n && (cfg->flags & VXLAN_F_L3MISS)) {
 			union vxlan_addr ipa = {
 				.sin.sin_addr.s_addr = pip->daddr,
 				.sin.sin_family = AF_INET,
@@ -2156,9 +2188,11 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb)
 			return false;
 		if (!pskb_network_may_pull(skb, sizeof(struct ipv6hdr)))
 			return false;
+
+		tbl = nd_table(dev_net(dev));
 		pip6 = ipv6_hdr(skb);
-		n = neigh_lookup(&nd_tbl, &pip6->daddr, dev);
-		if (!n && (vxlan->cfg.flags & VXLAN_F_L3MISS)) {
+		n = neigh_lookup(tbl, &pip6->daddr, dev);
+		if (!n && (cfg->flags & VXLAN_F_L3MISS)) {
 			union vxlan_addr ipa = {
 				.sin6.sin6_addr = pip6->daddr,
 				.sin6.sin6_family = AF_INET6,
@@ -2276,20 +2310,28 @@ static int vxlan_build_skb(struct sk_buff *skb, struct dst_entry *dst,
 
 /* Bypass encapsulation if the destination is local */
 static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan,
-			       struct vxlan_dev *dst_vxlan, __be32 vni,
-			       bool snoop)
+			       struct vxlan_dev *dst_vxlan,
+			       const struct vxlan_config *src_cfg,
+			       __be32 vni, bool snoop)
 {
+	const struct vxlan_config *dst_cfg;
 	union vxlan_addr loopback;
-	union vxlan_addr *remote_ip = &dst_vxlan->default_dst.remote_ip;
 	unsigned int len = skb->len;
-	struct net_device *dev;
+	struct net_device *dev = dst_vxlan->dev;
 
 	skb->pkt_type = PACKET_HOST;
 	skb->encapsulation = 0;
-	skb->dev = dst_vxlan->dev;
+	skb->dev = dev;
 	__skb_pull(skb, skb_network_offset(skb));
 
-	if (remote_ip->sa.sa_family == AF_INET) {
+	rcu_read_lock();
+	dst_cfg = rcu_dereference(dst_vxlan->cfg);
+	if (unlikely(!(dev->flags & IFF_UP))) {
+		kfree_skb_reason(skb, SKB_DROP_REASON_DEV_READY);
+		goto drop;
+	}
+
+	if (dst_cfg->remote_ip.sa.sa_family == AF_INET) {
 		loopback.sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 		loopback.sa.sa_family =  AF_INET;
 #if IS_ENABLED(CONFIG_IPV6)
@@ -2299,27 +2341,20 @@ static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan,
 #endif
 	}
 
-	rcu_read_lock();
-	dev = skb->dev;
-	if (unlikely(!(dev->flags & IFF_UP))) {
-		kfree_skb_reason(skb, SKB_DROP_REASON_DEV_READY);
-		goto drop;
-	}
-
-	if ((dst_vxlan->cfg.flags & VXLAN_F_LEARN) && snoop)
-		vxlan_snoop(dev, &loopback, eth_hdr(skb)->h_source, 0, vni);
+	if ((dst_cfg->flags & VXLAN_F_LEARN) && snoop)
+		vxlan_snoop(dev, dst_cfg, &loopback, eth_hdr(skb)->h_source, 0, vni);
 
 	dev_dstats_tx_add(src_vxlan->dev, len);
-	vxlan_vnifilter_count(src_vxlan, vni, NULL, VXLAN_VNI_STATS_TX, len);
+	vxlan_vnifilter_count(src_vxlan, src_cfg, vni, NULL, VXLAN_VNI_STATS_TX, len);
 
 	if (__netif_rx(skb) == NET_RX_SUCCESS) {
 		dev_dstats_rx_add(dst_vxlan->dev, len);
-		vxlan_vnifilter_count(dst_vxlan, vni, NULL, VXLAN_VNI_STATS_RX,
+		vxlan_vnifilter_count(dst_vxlan, dst_cfg, vni, NULL, VXLAN_VNI_STATS_RX,
 				      len);
 	} else {
 drop:
 		dev_dstats_rx_dropped(dev);
-		vxlan_vnifilter_count(dst_vxlan, vni, NULL,
+		vxlan_vnifilter_count(dst_vxlan, dst_cfg, vni, NULL,
 				      VXLAN_VNI_STATS_RX_DROPS, 0);
 	}
 	rcu_read_unlock();
@@ -2327,6 +2362,7 @@ static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan,
 
 static int encap_bypass_if_local(struct sk_buff *skb, struct net_device *dev,
 				 struct vxlan_dev *vxlan,
+				 const struct vxlan_config *cfg,
 				 int addr_family,
 				 __be16 dst_port, int dst_ifindex, __be32 vni,
 				 struct dst_entry *dst,
@@ -2342,22 +2378,22 @@ static int encap_bypass_if_local(struct sk_buff *skb, struct net_device *dev,
 	/* Bypass encapsulation if the destination is local */
 	if (rt_flags & RTCF_LOCAL &&
 	    !(rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
-	    vxlan->cfg.flags & VXLAN_F_LOCALBYPASS) {
+	    cfg->flags & VXLAN_F_LOCALBYPASS) {
 		struct vxlan_dev *dst_vxlan;
 
 		dst_release(dst);
 		dst_vxlan = vxlan_find_vni(vxlan->net, dst_ifindex, vni,
 					   addr_family, dst_port,
-					   vxlan->cfg.flags);
+					   cfg->flags);
 		if (!dst_vxlan) {
 			DEV_STATS_INC(dev, tx_errors);
-			vxlan_vnifilter_count(vxlan, vni, NULL,
+			vxlan_vnifilter_count(vxlan, cfg, vni, NULL,
 					      VXLAN_VNI_STATS_TX_ERRORS, 0);
 			kfree_skb_reason(skb, SKB_DROP_REASON_VXLAN_VNI_NOT_FOUND);
 
 			return -ENOENT;
 		}
-		vxlan_encap_bypass(skb, vxlan, dst_vxlan, vni, true);
+		vxlan_encap_bypass(skb, vxlan, dst_vxlan, cfg, vni, true);
 		return 1;
 	}
 
@@ -2365,30 +2401,35 @@ static int encap_bypass_if_local(struct sk_buff *skb, struct net_device *dev,
 }
 
 void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
+		    const struct vxlan_config *cfg,
 		    __be32 default_vni, struct vxlan_rdst *rdst, bool did_rsc)
 {
+	unsigned int pkt_len = skb->len;
+	struct vxlan_metadata _md = {};
+	__be16 src_port = 0, dst_port;
+	struct dst_entry *ndst = NULL;
+	enum skb_drop_reason reason;
 	struct dst_cache *dst_cache;
+	const struct iphdr *old_iph;
 	struct ip_tunnel_info *info;
 	struct ip_tunnel_key *pkey;
+	struct vxlan_metadata *md;
 	struct ip_tunnel_key key;
-	struct vxlan_dev *vxlan = netdev_priv(dev);
-	const struct iphdr *old_iph;
-	struct vxlan_metadata _md = {};
-	struct vxlan_metadata *md = &_md;
-	unsigned int pkt_len = skb->len;
-	__be16 src_port = 0, dst_port;
-	struct dst_entry *ndst = NULL;
+	struct vxlan_dev *vxlan;
+	u32 flags = cfg->flags;
+	bool udp_sum = false;
+	bool no_eth_encap;
 	int addr_family;
+	bool use_cache;
+	__be32 vni = 0;
 	__u8 tos, ttl;
 	int ifindex;
 	int err = 0;
-	u32 flags = vxlan->cfg.flags;
-	bool use_cache;
-	bool udp_sum = false;
-	bool xnet = !net_eq(vxlan->net, dev_net(vxlan->dev));
-	enum skb_drop_reason reason;
-	bool no_eth_encap;
-	__be32 vni = 0;
+	bool xnet;
+
+	vxlan = netdev_priv(dev);
+	xnet = !net_eq(vxlan->net, dev_net(vxlan->dev));
+	md = &_md;
 
 	no_eth_encap = flags & VXLAN_F_GPE && skb->protocol != htons(ETH_P_TEB);
 	reason = skb_vlan_inet_prepare(skb, no_eth_encap);
@@ -2408,23 +2449,23 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		if (vxlan_addr_any(&rdst->remote_ip)) {
 			if (did_rsc) {
 				/* short-circuited back to local bridge */
-				vxlan_encap_bypass(skb, vxlan, vxlan,
+				vxlan_encap_bypass(skb, vxlan, vxlan, cfg,
 						   default_vni, true);
 				return;
 			}
 			goto drop;
 		}
 
-		addr_family = vxlan->cfg.saddr.sa.sa_family;
-		dst_port = rdst->remote_port ? rdst->remote_port : vxlan->cfg.dst_port;
+		addr_family = cfg->saddr.sa.sa_family;
+		dst_port = rdst->remote_port ? rdst->remote_port : cfg->dst_port;
 		vni = (rdst->remote_vni) ? : default_vni;
 		ifindex = rdst->remote_ifindex;
 
 		if (addr_family == AF_INET) {
-			key.u.ipv4.src = vxlan->cfg.saddr.sin.sin_addr.s_addr;
+			key.u.ipv4.src = cfg->saddr.sin.sin_addr.s_addr;
 			key.u.ipv4.dst = rdst->remote_ip.sin.sin_addr.s_addr;
 		} else {
-			key.u.ipv6.src = vxlan->cfg.saddr.sin6.sin6_addr;
+			key.u.ipv6.src = cfg->saddr.sin6.sin6_addr;
 			key.u.ipv6.dst = rdst->remote_ip.sin6.sin6_addr;
 		}
 
@@ -2433,11 +2474,11 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		if (flags & VXLAN_F_TTL_INHERIT) {
 			ttl = ip_tunnel_get_ttl(old_iph, skb);
 		} else {
-			ttl = vxlan->cfg.ttl;
+			ttl = cfg->ttl;
 			if (!ttl && vxlan_addr_multicast(&rdst->remote_ip))
 				ttl = 1;
 		}
-		tos = vxlan->cfg.tos;
+		tos = cfg->tos;
 		if (tos == 1)
 			tos = ip_tunnel_get_dsfield(old_iph, skb);
 		if (tos && !info)
@@ -2448,9 +2489,9 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		else
 			udp_sum = !(flags & VXLAN_F_UDP_ZERO_CSUM6_TX);
 #if IS_ENABLED(CONFIG_IPV6)
-		switch (vxlan->cfg.label_policy) {
+		switch (cfg->label_policy) {
 		case VXLAN_LABEL_FIXED:
-			key.label = vxlan->cfg.label;
+			key.label = cfg->label;
 			break;
 		case VXLAN_LABEL_INHERIT:
 			key.label = ip_tunnel_get_flowlabel(old_iph, skb);
@@ -2468,7 +2509,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		}
 		pkey = &info->key;
 		addr_family = ip_tunnel_info_af(info);
-		dst_port = info->key.tp_dst ? : vxlan->cfg.dst_port;
+		dst_port = info->key.tp_dst ? : cfg->dst_port;
 		vni = tunnel_id_to_key32(info->key.tun_id);
 		ifindex = 0;
 		dst_cache = &info->dst_cache;
@@ -2481,8 +2522,8 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		tos = info->key.tos;
 		udp_sum = test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
 	}
-	src_port = udp_flow_src_port(dev_net(dev), skb, vxlan->cfg.port_min,
-				     vxlan->cfg.port_max, true);
+	src_port = udp_flow_src_port(dev_net(dev), skb, cfg->port_min,
+				     cfg->port_max, true);
 
 	rcu_read_lock();
 	if (addr_family == AF_INET) {
@@ -2515,15 +2556,15 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 
 		if (!info) {
 			/* Bypass encapsulation if the destination is local */
-			err = encap_bypass_if_local(skb, dev, vxlan, AF_INET,
+			err = encap_bypass_if_local(skb, dev, vxlan, cfg, AF_INET,
 						    dst_port, ifindex, vni,
 						    &rt->dst, rt->rt_flags);
 			if (err)
 				goto out_unlock;
 
-			if (vxlan->cfg.df == VXLAN_DF_SET) {
+			if (cfg->df == VXLAN_DF_SET) {
 				df = htons(IP_DF);
-			} else if (vxlan->cfg.df == VXLAN_DF_INHERIT) {
+			} else if (cfg->df == VXLAN_DF_INHERIT) {
 				struct ethhdr *eth = eth_hdr(skb);
 
 				if (ntohs(eth->h_proto) == ETH_P_IPV6 ||
@@ -2552,7 +2593,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 				unclone->key.u.ipv4.src = pkey->u.ipv4.dst;
 				unclone->key.u.ipv4.dst = saddr;
 			}
-			vxlan_encap_bypass(skb, vxlan, vxlan, vni, false);
+			vxlan_encap_bypass(skb, vxlan, vxlan, cfg, vni, false);
 			dst_release(ndst);
 			goto out_unlock;
 		}
@@ -2602,7 +2643,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		if (!info) {
 			u32 rt6i_flags = dst_rt6_info(ndst)->rt6i_flags;
 
-			err = encap_bypass_if_local(skb, dev, vxlan, AF_INET6,
+			err = encap_bypass_if_local(skb, dev, vxlan, cfg, AF_INET6,
 						    dst_port, ifindex, vni,
 						    ndst, rt6i_flags);
 			if (err)
@@ -2626,7 +2667,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 				unclone->key.u.ipv6.dst = saddr;
 			}
 
-			vxlan_encap_bypass(skb, vxlan, vxlan, vni, false);
+			vxlan_encap_bypass(skb, vxlan, vxlan, cfg, vni, false);
 			dst_release(ndst);
 			goto out_unlock;
 		}
@@ -2647,14 +2688,14 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 				     ip6cb_flags);
 #endif
 	}
-	vxlan_vnifilter_count(vxlan, vni, NULL, VXLAN_VNI_STATS_TX, pkt_len);
+	vxlan_vnifilter_count(vxlan, cfg, vni, NULL, VXLAN_VNI_STATS_TX, pkt_len);
 out_unlock:
 	rcu_read_unlock();
 	return;
 
 drop:
 	dev_dstats_tx_dropped(dev);
-	vxlan_vnifilter_count(vxlan, vni, NULL, VXLAN_VNI_STATS_TX_DROPS, 0);
+	vxlan_vnifilter_count(vxlan, cfg, vni, NULL, VXLAN_VNI_STATS_TX_DROPS, 0);
 	kfree_skb_reason(skb, reason);
 	return;
 
@@ -2666,11 +2707,12 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		DEV_STATS_INC(dev, tx_carrier_errors);
 	dst_release(ndst);
 	DEV_STATS_INC(dev, tx_errors);
-	vxlan_vnifilter_count(vxlan, vni, NULL, VXLAN_VNI_STATS_TX_ERRORS, 0);
+	vxlan_vnifilter_count(vxlan, cfg, vni, NULL, VXLAN_VNI_STATS_TX_ERRORS, 0);
 	kfree_skb_reason(skb, reason);
 }
 
 static void vxlan_xmit_nh(struct sk_buff *skb, struct net_device *dev,
+			  const struct vxlan_config *cfg,
 			  struct vxlan_fdb *f, __be32 vni, bool did_rsc)
 {
 	struct vxlan_rdst nh_rdst;
@@ -2687,7 +2729,7 @@ static void vxlan_xmit_nh(struct sk_buff *skb, struct net_device *dev,
 	do_xmit = vxlan_fdb_nh_path_select(nh, hash, &nh_rdst);
 
 	if (likely(do_xmit))
-		vxlan_xmit_one(skb, dev, vni, &nh_rdst, did_rsc);
+		vxlan_xmit_one(skb, dev, cfg, vni, &nh_rdst, did_rsc);
 	else
 		goto drop;
 
@@ -2695,15 +2737,15 @@ static void vxlan_xmit_nh(struct sk_buff *skb, struct net_device *dev,
 
 drop:
 	dev_dstats_tx_dropped(dev);
-	vxlan_vnifilter_count(netdev_priv(dev), vni, NULL,
+	vxlan_vnifilter_count(netdev_priv(dev), cfg, vni, NULL,
 			      VXLAN_VNI_STATS_TX_DROPS, 0);
 	dev_kfree_skb(skb);
 }
 
 static netdev_tx_t vxlan_xmit_nhid(struct sk_buff *skb, struct net_device *dev,
-				   u32 nhid, __be32 vni)
+				   u32 nhid, __be32 vni,
+				   const struct vxlan_config *cfg)
 {
-	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct vxlan_rdst nh_rdst;
 	struct nexthop *nh;
 	bool do_xmit;
@@ -2721,11 +2763,11 @@ static netdev_tx_t vxlan_xmit_nhid(struct sk_buff *skb, struct net_device *dev,
 	do_xmit = vxlan_fdb_nh_path_select(nh, hash, &nh_rdst);
 	rcu_read_unlock();
 
-	if (vxlan->cfg.saddr.sa.sa_family != nh_rdst.remote_ip.sa.sa_family)
+	if (cfg->saddr.sa.sa_family != nh_rdst.remote_ip.sa.sa_family)
 		goto drop;
 
 	if (likely(do_xmit))
-		vxlan_xmit_one(skb, dev, vni, &nh_rdst, false);
+		vxlan_xmit_one(skb, dev, cfg, vni, &nh_rdst, false);
 	else
 		goto drop;
 
@@ -2733,7 +2775,7 @@ static netdev_tx_t vxlan_xmit_nhid(struct sk_buff *skb, struct net_device *dev,
 
 drop:
 	dev_dstats_tx_dropped(dev);
-	vxlan_vnifilter_count(netdev_priv(dev), vni, NULL,
+	vxlan_vnifilter_count(netdev_priv(dev), cfg, vni, NULL,
 			      VXLAN_VNI_STATS_TX_DROPS, 0);
 	dev_kfree_skb(skb);
 	return NETDEV_TX_OK;
@@ -2750,34 +2792,47 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct vxlan_rdst *rdst, *fdst = NULL;
 	const struct ip_tunnel_info *info;
+	const struct vxlan_config *cfg;
+	__be32 default_vni;
 	struct vxlan_fdb *f;
 	struct ethhdr *eth;
 	__be32 vni = 0;
-	u32 nhid = 0;
 	bool did_rsc;
+	u32 nhid = 0;
+	u32 flags;
+
+	rcu_read_lock();
+	cfg = rcu_dereference(vxlan->cfg);
+	flags = cfg->flags;
+	default_vni = cfg->vni;
 
 	info = skb_tunnel_info(skb);
 
 	skb_reset_mac_header(skb);
 
-	if (vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) {
+	if (flags & VXLAN_F_COLLECT_METADATA) {
 		if (info && info->mode & IP_TUNNEL_INFO_BRIDGE &&
 		    info->mode & IP_TUNNEL_INFO_TX) {
 			vni = tunnel_id_to_key32(info->key.tun_id);
 			nhid = info->key.nhid;
 		} else {
 			if (info && info->mode & IP_TUNNEL_INFO_TX)
-				vxlan_xmit_one(skb, dev, vni, NULL, false);
+				vxlan_xmit_one(skb, dev, cfg, vni, NULL, false);
 			else
 				kfree_skb_reason(skb, SKB_DROP_REASON_TUNNEL_TXINFO);
+			rcu_read_unlock();
 			return NETDEV_TX_OK;
 		}
 	}
 
-	if (vxlan->cfg.flags & VXLAN_F_PROXY) {
+	if (flags & VXLAN_F_PROXY) {
 		eth = eth_hdr(skb);
-		if (ntohs(eth->h_proto) == ETH_P_ARP)
-			return arp_reduce(dev, skb, vni);
+		if (ntohs(eth->h_proto) == ETH_P_ARP) {
+			netdev_tx_t res = arp_reduce(dev, skb, cfg, vni);
+
+			rcu_read_unlock();
+			return res;
+		}
 #if IS_ENABLED(CONFIG_IPV6)
 		else if (ntohs(eth->h_proto) == ETH_P_IPV6 &&
 			 pskb_network_may_pull(skb, sizeof(struct ipv6hdr) +
@@ -2786,53 +2841,57 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
 			struct nd_msg *m = (struct nd_msg *)(ipv6_hdr(skb) + 1);
 
 			if (m->icmph.icmp6_code == 0 &&
-			    m->icmph.icmp6_type == NDISC_NEIGHBOUR_SOLICITATION)
-				return neigh_reduce(dev, skb, vni);
+			    m->icmph.icmp6_type == NDISC_NEIGHBOUR_SOLICITATION) {
+				netdev_tx_t res = neigh_reduce(dev, skb, cfg, vni);
+
+				rcu_read_unlock();
+				return res;
+			}
 		}
 #endif
 	}
 
-	if (nhid)
-		return vxlan_xmit_nhid(skb, dev, nhid, vni);
+	if (nhid) {
+		netdev_tx_t res = vxlan_xmit_nhid(skb, dev, nhid, vni, cfg);
 
-	if (vxlan->cfg.flags & VXLAN_F_MDB) {
+		rcu_read_unlock();
+		return res;
+	}
+
+	if (test_bit(VXLAN_DEV_F_MDB, &vxlan->flags)) {
 		struct vxlan_mdb_entry *mdb_entry;
 
-		rcu_read_lock();
-		mdb_entry = vxlan_mdb_entry_skb_get(vxlan, skb, vni);
+		mdb_entry = vxlan_mdb_entry_skb_get(vxlan, cfg, skb, vni);
 		if (mdb_entry) {
-			netdev_tx_t ret;
+			netdev_tx_t ret = vxlan_mdb_xmit(vxlan, cfg, mdb_entry, skb);
 
-			ret = vxlan_mdb_xmit(vxlan, mdb_entry, skb);
 			rcu_read_unlock();
 			return ret;
 		}
-		rcu_read_unlock();
 	}
 
 	eth = eth_hdr(skb);
-	rcu_read_lock();
-	f = vxlan_find_mac_tx(vxlan, eth->h_dest, vni);
+	f = vxlan_find_mac_tx(vxlan, cfg, eth->h_dest, vni);
 	did_rsc = false;
 
-	if (f && (f->flags & NTF_ROUTER) && (vxlan->cfg.flags & VXLAN_F_RSC) &&
+	if (f && (f->flags & NTF_ROUTER) && (flags & VXLAN_F_RSC) &&
 	    (ntohs(eth->h_proto) == ETH_P_IP ||
 	     ntohs(eth->h_proto) == ETH_P_IPV6)) {
-		did_rsc = route_shortcircuit(dev, skb);
+		did_rsc = route_shortcircuit(dev, skb, cfg);
 		eth = eth_hdr(skb);
 		if (did_rsc)
-			f = vxlan_find_mac_tx(vxlan, eth->h_dest, vni);
+			f = vxlan_find_mac_tx(vxlan, cfg, eth->h_dest, vni);
 	}
 
 	if (f == NULL) {
-		f = vxlan_find_mac_tx(vxlan, all_zeros_mac, vni);
+		f = vxlan_find_mac_tx(vxlan, cfg, all_zeros_mac, vni);
 		if (f == NULL) {
-			if ((vxlan->cfg.flags & VXLAN_F_L2MISS) &&
+			if ((flags & VXLAN_F_L2MISS) &&
 			    !is_multicast_ether_addr(eth->h_dest))
 				vxlan_fdb_miss(vxlan, eth->h_dest);
 
 			dev_dstats_tx_dropped(dev);
-			vxlan_vnifilter_count(vxlan, vni, NULL,
+			vxlan_vnifilter_count(vxlan, cfg, vni, NULL,
 					      VXLAN_VNI_STATS_TX_DROPS, 0);
 			kfree_skb_reason(skb, SKB_DROP_REASON_NO_TX_TARGET);
 			goto out;
@@ -2840,8 +2899,8 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
 	}
 
 	if (rcu_access_pointer(f->nh)) {
-		vxlan_xmit_nh(skb, dev, f,
-			      (vni ? : vxlan->default_dst.remote_vni), did_rsc);
+		vxlan_xmit_nh(skb, dev, cfg, f,
+			      (vni ? : default_vni), did_rsc);
 	} else {
 		list_for_each_entry_rcu(rdst, &f->remotes, list) {
 			struct sk_buff *skb1;
@@ -2852,10 +2911,10 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev)
 			}
 			skb1 = skb_clone(skb, GFP_ATOMIC);
 			if (skb1)
-				vxlan_xmit_one(skb1, dev, vni, rdst, did_rsc);
+				vxlan_xmit_one(skb1, dev, cfg, vni, rdst, did_rsc);
 		}
 		if (fdst)
-			vxlan_xmit_one(skb, dev, vni, fdst, did_rsc);
+			vxlan_xmit_one(skb, dev, cfg, vni, fdst, did_rsc);
 		else
 			kfree_skb_reason(skb, SKB_DROP_REASON_NO_TX_TARGET);
 	}
@@ -2870,12 +2929,15 @@ static void vxlan_cleanup(struct timer_list *t)
 {
 	struct vxlan_dev *vxlan = timer_container_of(vxlan, t, age_timer);
 	unsigned long next_timer = jiffies + FDB_AGE_INTERVAL;
+	const struct vxlan_config *cfg;
 	struct vxlan_fdb *f;
 
 	if (!netif_running(vxlan->dev))
 		return;
 
 	rcu_read_lock();
+	cfg = rcu_dereference(vxlan->cfg);
+
 	hlist_for_each_entry_rcu(f, &vxlan->fdb_list, fdb_node) {
 		unsigned long timeout;
 
@@ -2885,7 +2947,7 @@ static void vxlan_cleanup(struct timer_list *t)
 		if (f->flags & NTF_EXT_LEARNED)
 			continue;
 
-		timeout = READ_ONCE(f->updated) + vxlan->cfg.age_interval * HZ;
+		timeout = READ_ONCE(f->updated) + cfg->age_interval * HZ;
 		if (time_before_eq(timeout, jiffies)) {
 			spin_lock(&vxlan->hash_lock);
 			if (!hlist_unhashed(&f->fdb_node)) {
@@ -2917,10 +2979,14 @@ static void vxlan_vs_del_dev(struct vxlan_dev *vxlan)
 static void vxlan_vs_add_dev(struct vxlan_sock *vs, struct vxlan_dev *vxlan,
 			     struct vxlan_dev_node *node)
 {
-	__be32 vni = vxlan->default_dst.remote_vni;
+	const struct vxlan_config *cfg;
+	__be32 vni;
 
 	ASSERT_RTNL();
 
+	cfg = rtnl_dereference(vxlan->cfg);
+	vni = cfg->vni;
+
 	node->vxlan = vxlan;
 	hlist_add_head_rcu(&node->hlist, vni_head(vs, vni));
 }
@@ -2929,13 +2995,16 @@ static void vxlan_vs_add_dev(struct vxlan_sock *vs, struct vxlan_dev *vxlan,
 static int vxlan_init(struct net_device *dev)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
+	const struct vxlan_config *cfg;
 	int err;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+
 	err = rhashtable_init(&vxlan->fdb_hash_tbl, &vxlan_fdb_rht_params);
 	if (err)
 		return err;
 
-	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER) {
+	if (cfg->flags & VXLAN_F_VNIFILTER) {
 		err = vxlan_vnigroup_init(vxlan);
 		if (err)
 			goto err_rhashtable_destroy;
@@ -2955,7 +3024,7 @@ static int vxlan_init(struct net_device *dev)
 err_gro_cells_destroy:
 	gro_cells_destroy(&vxlan->gro_cells);
 err_vnigroup_uninit:
-	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
+	if (cfg->flags & VXLAN_F_VNIFILTER)
 		vxlan_vnigroup_uninit(vxlan);
 err_rhashtable_destroy:
 	rhashtable_destroy(&vxlan->fdb_hash_tbl);
@@ -2965,10 +3034,13 @@ static int vxlan_init(struct net_device *dev)
 static void vxlan_uninit(struct net_device *dev)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
+	const struct vxlan_config *cfg;
+
+	cfg = rtnl_dereference(vxlan->cfg);
 
 	vxlan_mdb_fini(vxlan);
 
-	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
+	if (cfg->flags & VXLAN_F_VNIFILTER)
 		vxlan_vnigroup_uninit(vxlan);
 
 	gro_cells_destroy(&vxlan->gro_cells);
@@ -2980,6 +3052,7 @@ static void vxlan_uninit(struct net_device *dev)
 static int vxlan_open(struct net_device *dev)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
+	const struct vxlan_config *cfg;
 	int ret;
 
 	ret = vxlan_sock_add(vxlan);
@@ -2992,7 +3065,8 @@ static int vxlan_open(struct net_device *dev)
 		return ret;
 	}
 
-	if (vxlan->cfg.age_interval)
+	cfg = rtnl_dereference(vxlan->cfg);
+	if (cfg->age_interval)
 		mod_timer(&vxlan->age_timer, jiffies + FDB_AGE_INTERVAL);
 
 	return ret;
@@ -3014,8 +3088,10 @@ struct vxlan_fdb_flush_desc {
 static bool vxlan_fdb_is_default_entry(const struct vxlan_fdb *f,
 				       const struct vxlan_dev *vxlan)
 {
+	const struct vxlan_config *cfg = rcu_dereference_rtnl(vxlan->cfg);
+
 	return is_zero_ether_addr(f->key.eth_addr) &&
-	       f->key.vni == vxlan->cfg.vni;
+	       f->key.vni == cfg->vni;
 }
 
 static bool vxlan_fdb_nhid_matches(const struct vxlan_fdb *f, u32 nhid)
@@ -3232,15 +3308,18 @@ static void vxlan_set_multicast_list(struct net_device *dev)
 static int vxlan_change_mtu(struct net_device *dev, int new_mtu)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
-	struct vxlan_rdst *dst = &vxlan->default_dst;
-	struct net_device *lowerdev = __dev_get_by_index(vxlan->net,
-							 dst->remote_ifindex);
+	const struct vxlan_config *cfg;
+	struct net_device *lowerdev;
+
+	cfg = rtnl_dereference(vxlan->cfg);
+
+	lowerdev = __dev_get_by_index(vxlan->net, cfg->remote_ifindex);
 
 	/* This check is different than dev->max_mtu, because it looks at
 	 * the lowerdev->mtu, rather than the static dev->max_mtu
 	 */
 	if (lowerdev) {
-		int max_mtu = lowerdev->mtu - vxlan_headroom(vxlan->cfg.flags);
+		int max_mtu = lowerdev->mtu - vxlan_headroom(cfg->flags);
 		if (new_mtu > max_mtu)
 			return -EINVAL;
 	}
@@ -3253,11 +3332,14 @@ static int vxlan_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct ip_tunnel_info *info = skb_tunnel_info(skb);
+	const struct vxlan_config *cfg;
 	__be16 sport, dport;
 
-	sport = udp_flow_src_port(dev_net(dev), skb, vxlan->cfg.port_min,
-				  vxlan->cfg.port_max, true);
-	dport = info->key.tp_dst ? : vxlan->cfg.dst_port;
+	cfg = rcu_dereference(vxlan->cfg);
+
+	sport = udp_flow_src_port(dev_net(dev), skb, cfg->port_min,
+				  cfg->port_max, true);
+	dport = info->key.tp_dst ? : cfg->dst_port;
 
 	if (ip_tunnel_info_af(info) == AF_INET) {
 		struct vxlan_sock *sock4 = rcu_dereference(vxlan->vn4_sock);
@@ -3367,6 +3449,15 @@ static void vxlan_offload_rx_ports(struct net_device *dev, bool push)
 	}
 }
 
+static void vxlan_free_dev(struct net_device *dev)
+{
+	struct vxlan_dev *vxlan = netdev_priv(dev);
+	struct vxlan_config *cfg = rcu_dereference_protected(vxlan->cfg, 1);
+
+	RCU_INIT_POINTER(vxlan->cfg, NULL);
+	kfree(cfg);
+}
+
 /* Initialize the device structure. */
 static void vxlan_setup(struct net_device *dev)
 {
@@ -3375,6 +3466,8 @@ static void vxlan_setup(struct net_device *dev)
 	eth_hw_addr_random(dev);
 	ether_setup(dev);
 
+	dev->priv_destructor = vxlan_free_dev;
+
 	dev->needs_free_netdev = true;
 	SET_NETDEV_DEVTYPE(dev, &vxlan_type);
 
@@ -3549,9 +3642,11 @@ static int vxlan_get_link_ksettings(struct net_device *dev,
 				    struct ethtool_link_ksettings *cmd)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
-	struct vxlan_rdst *dst = &vxlan->default_dst;
-	struct net_device *lowerdev = __dev_get_by_index(vxlan->net,
-							 dst->remote_ifindex);
+	const struct vxlan_config *cfg;
+	struct net_device *lowerdev;
+
+	cfg = rtnl_dereference(vxlan->cfg);
+	lowerdev = __dev_get_by_index(vxlan->net, cfg->remote_ifindex);
 
 	if (!lowerdev) {
 		cmd->base.duplex = DUPLEX_UNKNOWN;
@@ -3657,21 +3752,22 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, bool ipv6,
 
 static int __vxlan_sock_add(struct vxlan_dev *vxlan, bool ipv6)
 {
-	bool metadata = vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA;
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
+	bool metadata = cfg->flags & VXLAN_F_COLLECT_METADATA;
 	struct vxlan_sock *vs = NULL;
 	struct vxlan_dev_node *node;
 	int l3mdev_index = 0;
 
 	ASSERT_RTNL();
 
-	if (vxlan->cfg.remote_ifindex)
+	if (cfg->remote_ifindex)
 		l3mdev_index = l3mdev_master_upper_ifindex_by_index(
-			vxlan->net, vxlan->cfg.remote_ifindex);
+			vxlan->net, cfg->remote_ifindex);
 
-	if (!vxlan->cfg.no_share) {
+	if (!cfg->no_share) {
 		rcu_read_lock();
 		vs = vxlan_find_sock(vxlan->net, ipv6 ? AF_INET6 : AF_INET,
-				     vxlan->cfg.dst_port, vxlan->cfg.flags,
+				     cfg->dst_port, cfg->flags,
 				     l3mdev_index);
 		if (vs && !refcount_inc_not_zero(&vs->refcnt)) {
 			rcu_read_unlock();
@@ -3681,7 +3777,7 @@ static int __vxlan_sock_add(struct vxlan_dev *vxlan, bool ipv6)
 	}
 	if (!vs)
 		vs = vxlan_socket_create(vxlan->net, ipv6,
-					 vxlan->cfg.dst_port, vxlan->cfg.flags,
+					 cfg->dst_port, cfg->flags,
 					 l3mdev_index);
 	if (IS_ERR(vs))
 		return PTR_ERR(vs);
@@ -3696,7 +3792,7 @@ static int __vxlan_sock_add(struct vxlan_dev *vxlan, bool ipv6)
 		node = &vxlan->hlist4;
 	}
 
-	if (metadata && (vxlan->cfg.flags & VXLAN_F_VNIFILTER))
+	if (metadata && (cfg->flags & VXLAN_F_VNIFILTER))
 		vxlan_vs_add_vnigrp(vxlan, vs, ipv6);
 	else
 		vxlan_vs_add_dev(vs, vxlan, node);
@@ -3706,11 +3802,14 @@ static int __vxlan_sock_add(struct vxlan_dev *vxlan, bool ipv6)
 
 static int vxlan_sock_add(struct vxlan_dev *vxlan)
 {
-	bool metadata = vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA;
-	bool ipv6 = vxlan->cfg.flags & VXLAN_F_IPV6 || metadata;
-	bool ipv4 = !ipv6 || metadata;
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
+	bool metadata, ipv6, ipv4;
 	int ret = 0;
 
+	metadata = cfg->flags & VXLAN_F_COLLECT_METADATA;
+	ipv6 = (cfg->flags & VXLAN_F_IPV6) || metadata;
+	ipv4 = !ipv6 || metadata;
+
 	RCU_INIT_POINTER(vxlan->vn4_sock, NULL);
 #if IS_ENABLED(CONFIG_IPV6)
 	RCU_INIT_POINTER(vxlan->vn6_sock, NULL);
@@ -3728,28 +3827,33 @@ static int vxlan_sock_add(struct vxlan_dev *vxlan)
 }
 
 int vxlan_vni_in_use(struct net *src_net, struct vxlan_dev *vxlan,
-		     struct vxlan_config *conf, __be32 vni)
+		     const struct vxlan_config *conf, __be32 vni)
 {
 	struct vxlan_net *vn = net_generic(src_net, vxlan_net_id);
 	struct vxlan_dev *tmp;
 
 	list_for_each_entry(tmp, &vn->vxlan_list, next) {
+		const struct vxlan_config *tmp_cfg;
+
 		if (tmp == vxlan)
 			continue;
-		if (tmp->cfg.flags & VXLAN_F_VNIFILTER) {
+
+		tmp_cfg = rtnl_dereference(tmp->cfg);
+
+		if (tmp_cfg->flags & VXLAN_F_VNIFILTER) {
 			if (!vxlan_vnifilter_lookup(tmp, vni))
 				continue;
-		} else if (tmp->cfg.vni != vni) {
+		} else if (tmp_cfg->vni != vni) {
 			continue;
 		}
-		if (tmp->cfg.dst_port != conf->dst_port)
+		if (tmp_cfg->dst_port != conf->dst_port)
 			continue;
-		if ((tmp->cfg.flags & (VXLAN_F_RCV_FLAGS | VXLAN_F_IPV6)) !=
+		if ((tmp_cfg->flags & (VXLAN_F_RCV_FLAGS | VXLAN_F_IPV6)) !=
 		    (conf->flags & (VXLAN_F_RCV_FLAGS | VXLAN_F_IPV6)))
 			continue;
 
 		if ((conf->flags & VXLAN_F_IPV6_LINKLOCAL) &&
-		    tmp->cfg.remote_ifindex != conf->remote_ifindex)
+		    tmp_cfg->remote_ifindex != conf->remote_ifindex)
 			continue;
 
 		return -EEXIST;
@@ -3880,6 +3984,13 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 			return -EINVAL;
 		}
 
+		if ((conf->flags & VXLAN_F_VNIFILTER) && old &&
+		    vxlan_vnifilter_has_multicast(old)) {
+			NL_SET_ERR_MSG(extack,
+				       "Local interface required for multicast remote group");
+			return -EINVAL;
+		}
+
 #if IS_ENABLED(CONFIG_IPV6)
 		if (conf->flags & VXLAN_F_IPV6_LINKLOCAL) {
 			NL_SET_ERR_MSG(extack,
@@ -3911,16 +4022,16 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 }
 
 static void vxlan_config_apply(struct net_device *dev,
-			       struct vxlan_config *conf,
+			       struct vxlan_config *new_cfg,
 			       struct net_device *lowerdev,
 			       struct net *src_net,
-			       bool changelink)
+			       bool changelink, bool lowerdev_changed)
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
-	struct vxlan_rdst *dst = &vxlan->default_dst;
 	unsigned short needed_headroom = ETH_HLEN;
+	struct vxlan_config *old_cfg;
 	int max_mtu = ETH_MAX_MTU;
-	u32 flags = conf->flags;
+	u32 flags = new_cfg->flags;
 
 	if (!changelink) {
 		if (flags & VXLAN_F_GPE)
@@ -3928,19 +4039,14 @@ static void vxlan_config_apply(struct net_device *dev,
 		else
 			vxlan_ether_setup(dev);
 
-		if (conf->mtu)
-			dev->mtu = conf->mtu;
+		if (new_cfg->mtu)
+			WRITE_ONCE(dev->mtu, new_cfg->mtu);
 
 		vxlan->net = src_net;
 	}
 
-	dst->remote_vni = conf->vni;
-
-	memcpy(&dst->remote_ip, &conf->remote_ip, sizeof(conf->remote_ip));
-
+	dev->needed_tailroom = 0;
 	if (lowerdev) {
-		dst->remote_ifindex = conf->remote_ifindex;
-
 		netif_inherit_tso_max(dev, lowerdev);
 
 		needed_headroom = lowerdev->hard_header_len;
@@ -3948,23 +4054,27 @@ static void vxlan_config_apply(struct net_device *dev,
 
 		dev->needed_tailroom = lowerdev->needed_tailroom;
 
-		max_mtu = lowerdev->mtu - vxlan_headroom(flags);
+		max_mtu = READ_ONCE(lowerdev->mtu) - vxlan_headroom(flags);
 		if (max_mtu < ETH_MIN_MTU)
 			max_mtu = ETH_MIN_MTU;
 
-		if (!changelink && !conf->mtu)
-			dev->mtu = max_mtu;
+		if (!changelink && !new_cfg->mtu)
+			WRITE_ONCE(dev->mtu, max_mtu);
 	}
 
-	if (dev->mtu > max_mtu)
-		dev->mtu = max_mtu;
+	/* A changelink leaving the lower device alone must not shrink the mtu */
+	if (lowerdev_changed && READ_ONCE(dev->mtu) > max_mtu)
+		WRITE_ONCE(dev->mtu, max_mtu);
 
 	if (flags & VXLAN_F_COLLECT_METADATA)
 		flags |= VXLAN_F_IPV6;
 	needed_headroom += vxlan_headroom(flags);
 	dev->needed_headroom = needed_headroom;
 
-	memcpy(&vxlan->cfg, conf, sizeof(*conf));
+	old_cfg = rtnl_dereference(vxlan->cfg);
+	rcu_assign_pointer(vxlan->cfg, new_cfg);
+	if (old_cfg)
+		kfree_rcu(old_cfg, rcu);
 }
 
 static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
@@ -3973,13 +4083,18 @@ static int vxlan_dev_configure(struct net *src_net, struct net_device *dev,
 {
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct net_device *lowerdev;
+	struct vxlan_config *new_cfg;
 	int ret;
 
 	ret = vxlan_config_validate(src_net, conf, &lowerdev, vxlan, extack);
 	if (ret)
 		return ret;
 
-	vxlan_config_apply(dev, conf, lowerdev, src_net, false);
+	new_cfg = kmemdup(conf, sizeof(*conf), GFP_KERNEL);
+	if (!new_cfg)
+		return -ENOMEM;
+
+	vxlan_config_apply(dev, new_cfg, lowerdev, src_net, false, true);
 
 	return 0;
 }
@@ -3991,22 +4106,25 @@ static int vxlan_dev_create(struct net *net, struct net_device *dev,
 	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct net_device *remote_dev = NULL;
-	struct vxlan_rdst *dst;
+	const struct vxlan_config *cfg;
 	int err;
 
-	dst = &vxlan->default_dst;
 	err = vxlan_dev_configure(net, dev, conf, extack);
 	if (err)
 		return err;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+
 	dev->ethtool_ops = &vxlan_ethtool_ops;
 
 	err = register_netdevice(dev);
-	if (err)
+	if (err) {
+		vxlan_free_dev(dev);
 		return err;
+	}
 
-	if (dst->remote_ifindex) {
-		remote_dev = __dev_get_by_index(net, dst->remote_ifindex);
+	if (cfg->remote_ifindex) {
+		remote_dev = __dev_get_by_index(net, cfg->remote_ifindex);
 		if (!remote_dev) {
 			err = -ENODEV;
 			goto unregister;
@@ -4016,7 +4134,7 @@ static int vxlan_dev_create(struct net *net, struct net_device *dev,
 		if (err)
 			goto unregister;
 
-		dst->remote_dev = remote_dev;
+		vxlan->lowerdev = remote_dev;
 	}
 
 	err = rtnl_configure_link(dev, NULL, 0, NULL);
@@ -4024,16 +4142,18 @@ static int vxlan_dev_create(struct net *net, struct net_device *dev,
 		goto unlink;
 
 	/* create an fdb entry for a valid default destination */
-	if (!vxlan_addr_any(&dst->remote_ip)) {
+	if (!vxlan_addr_any(&cfg->remote_ip)) {
+		union vxlan_addr rip = cfg->remote_ip;
+
 		spin_lock_bh(&vxlan->hash_lock);
 		err = vxlan_fdb_update(vxlan, all_zeros_mac,
-				       &dst->remote_ip,
+				       &rip,
 				       NUD_REACHABLE | NUD_PERMANENT,
 				       NLM_F_EXCL | NLM_F_CREATE,
-				       vxlan->cfg.dst_port,
-				       dst->remote_vni,
-				       dst->remote_vni,
-				       dst->remote_ifindex,
+				       cfg->dst_port,
+				       cfg->vni,
+				       cfg->vni,
+				       cfg->remote_ifindex,
 				       NTF_SELF, 0, true, extack);
 		spin_unlock_bh(&vxlan->hash_lock);
 		if (err)
@@ -4094,8 +4214,12 @@ static int vxlan_nl2conf(struct nlattr *tb[], struct nlattr *data[],
 	memset(conf, 0, sizeof(*conf));
 
 	/* if changelink operation, start with old existing cfg */
-	if (changelink)
-		memcpy(conf, &vxlan->cfg, sizeof(*conf));
+	if (changelink) {
+		const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
+
+		if (cfg)
+			memcpy(conf, cfg, sizeof(*conf));
+	}
 
 	if (data[IFLA_VXLAN_ID]) {
 		__be32 vni = cpu_to_be32(nla_get_u32(data[IFLA_VXLAN_ID]));
@@ -4441,17 +4565,19 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],
 			    struct nlattr *data[],
 			    struct netlink_ext_ack *extack)
 {
+	bool lowerdev_changed, rem_ip_changed, change_igmp;
 	struct vxlan_dev *vxlan = netdev_priv(dev);
-	bool rem_ip_changed, change_igmp;
+	const struct vxlan_config *cfg;
+	struct vxlan_config *new_cfg;
 	struct net_device *lowerdev;
 	struct vxlan_config conf;
-	struct vxlan_rdst *dst;
 	int err;
 
+	cfg = rtnl_dereference(vxlan->cfg);
+
 	if (!rtnl_dev_link_net_capable(dev, vxlan->net))
 		return -EPERM;
 
-	dst = &vxlan->default_dst;
 	err = vxlan_nl2conf(tb, data, dev, &conf, true, extack);
 	if (err)
 		return err;
@@ -4461,76 +4587,108 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (err)
 		return err;
 
-	if (dst->remote_dev == lowerdev)
-		lowerdev = NULL;
+	new_cfg = kmemdup(&conf, sizeof(conf), GFP_KERNEL);
+	if (!new_cfg)
+		return -ENOMEM;
 
-	err = netdev_adjacent_change_prepare(dst->remote_dev, lowerdev, dev,
-					     extack);
-	if (err)
-		return err;
+	lowerdev_changed = vxlan->lowerdev != lowerdev;
+	if (lowerdev_changed) {
+		err = netdev_adjacent_change_prepare(vxlan->lowerdev, lowerdev,
+						     dev, extack);
+		if (err) {
+			kfree(new_cfg);
+			return err;
+		}
+	}
 
-	rem_ip_changed = !vxlan_addr_equal(&conf.remote_ip, &dst->remote_ip);
+	rem_ip_changed = !vxlan_addr_equal(&conf.remote_ip, &cfg->remote_ip);
 	change_igmp = vxlan->dev->flags & IFF_UP &&
 		      (rem_ip_changed ||
-		       dst->remote_ifindex != conf.remote_ifindex);
+		       cfg->remote_ifindex != conf.remote_ifindex);
 
 	/* handle default dst entry */
-	if (rem_ip_changed) {
+	if (rem_ip_changed || cfg->remote_ifindex != conf.remote_ifindex) {
 		spin_lock_bh(&vxlan->hash_lock);
 		if (!vxlan_addr_any(&conf.remote_ip)) {
 			err = vxlan_fdb_update(vxlan, all_zeros_mac,
 					       &conf.remote_ip,
 					       NUD_REACHABLE | NUD_PERMANENT,
 					       NLM_F_APPEND | NLM_F_CREATE,
-					       vxlan->cfg.dst_port,
+					       cfg->dst_port,
 					       conf.vni, conf.vni,
 					       conf.remote_ifindex,
 					       NTF_SELF, 0, true, extack);
 			if (err) {
 				spin_unlock_bh(&vxlan->hash_lock);
-				netdev_adjacent_change_abort(dst->remote_dev,
-							     lowerdev, dev);
+				if (lowerdev_changed)
+					netdev_adjacent_change_abort(vxlan->lowerdev,
+								     lowerdev, dev);
+				kfree(new_cfg);
 				return err;
 			}
 		}
-		if (!vxlan_addr_any(&dst->remote_ip))
+		if (!vxlan_addr_any(&cfg->remote_ip))
 			__vxlan_fdb_delete(vxlan, all_zeros_mac,
-					   dst->remote_ip,
-					   vxlan->cfg.dst_port,
-					   dst->remote_vni,
-					   dst->remote_vni,
-					   dst->remote_ifindex,
+					   cfg->remote_ip,
+					   cfg->dst_port,
+					   cfg->vni,
+					   cfg->vni,
+					   cfg->remote_ifindex,
 					   true);
 		spin_unlock_bh(&vxlan->hash_lock);
 
-		/* If vni filtering device, also update fdb entries of
-		 * all vnis that were using default remote ip
+		/* If vni filtering device, also update default fdb entries of
+		 * all vnis
 		 */
-		if (vxlan->cfg.flags & VXLAN_F_VNIFILTER) {
-			err = vxlan_vnilist_update_group(vxlan, &dst->remote_ip,
-							 &conf.remote_ip, extack);
+		if (cfg->flags & VXLAN_F_VNIFILTER) {
+			err = vxlan_vnilist_update_group(vxlan, &cfg->remote_ip,
+							 &conf.remote_ip,
+							 cfg->remote_ifindex,
+							 conf.remote_ifindex,
+							 extack);
 			if (err) {
-				netdev_adjacent_change_abort(dst->remote_dev,
-							     lowerdev, dev);
+				vxlan_update_default_fdb_entry(vxlan, conf.vni,
+							       &conf.remote_ip,
+							       &cfg->remote_ip,
+							       conf.remote_ifindex,
+							       cfg->remote_ifindex,
+							       NULL);
+				if (lowerdev_changed)
+					netdev_adjacent_change_abort(vxlan->lowerdev,
+								     lowerdev, dev);
+				kfree(new_cfg);
 				return err;
 			}
 		}
 	}
 
-	if (change_igmp && vxlan_addr_multicast(&dst->remote_ip))
+	if (change_igmp &&
+	    (vxlan_addr_multicast(&cfg->remote_ip) ||
+	     (cfg->flags & VXLAN_F_VNIFILTER)))
 		err = vxlan_multicast_leave(vxlan);
 
-	if (netif_running(dev) && conf.age_interval != vxlan->cfg.age_interval)
+	if (netif_running(dev) && conf.age_interval != cfg->age_interval)
 		mod_timer(&vxlan->age_timer, jiffies);
 
-	netdev_adjacent_change_commit(dst->remote_dev, lowerdev, dev);
-	if (lowerdev && lowerdev != dst->remote_dev)
-		dst->remote_dev = lowerdev;
-	vxlan_config_apply(dev, &conf, lowerdev, vxlan->net, true);
+	if (lowerdev_changed) {
+		if (lowerdev)
+			netdev_adjacent_change_commit(vxlan->lowerdev, lowerdev,
+						      dev);
+		else
+			netdev_upper_dev_unlink(vxlan->lowerdev, dev);
+		vxlan->lowerdev = lowerdev;
+	}
+	vxlan_config_apply(dev, new_cfg, lowerdev, vxlan->net, true,
+			   lowerdev_changed);
+
+	if (change_igmp &&
+	    (vxlan_addr_multicast(&new_cfg->remote_ip) ||
+	     (new_cfg->flags & VXLAN_F_VNIFILTER))) {
+		int join_err = vxlan_multicast_join(vxlan);
 
-	if (!err && change_igmp &&
-	    vxlan_addr_multicast(&dst->remote_ip))
-		err = vxlan_multicast_join(vxlan);
+		if (join_err)
+			err = join_err;
+	}
 
 	return err;
 }
@@ -4544,8 +4702,8 @@ static void vxlan_dellink(struct net_device *dev, struct list_head *head)
 
 	list_del(&vxlan->next);
 	unregister_netdevice_queue(dev, head);
-	if (vxlan->default_dst.remote_dev)
-		netdev_upper_dev_unlink(vxlan->default_dst.remote_dev, dev);
+	if (vxlan->lowerdev)
+		netdev_upper_dev_unlink(vxlan->lowerdev, dev);
 }
 
 static size_t vxlan_get_size(const struct net_device *dev)
@@ -4589,110 +4747,114 @@ static size_t vxlan_get_size(const struct net_device *dev)
 static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
 	const struct vxlan_dev *vxlan = netdev_priv(dev);
-	const struct vxlan_rdst *dst = &vxlan->default_dst;
-	struct ifla_vxlan_port_range ports = {
-		.low =  htons(vxlan->cfg.port_min),
-		.high = htons(vxlan->cfg.port_max),
-	};
+	struct ifla_vxlan_port_range ports;
+	const struct vxlan_config *cfg;
+
+	rcu_read_lock();
+	cfg = rcu_dereference(vxlan->cfg);
 
-	if (nla_put_u32(skb, IFLA_VXLAN_ID, be32_to_cpu(dst->remote_vni)))
+	if (nla_put_u32(skb, IFLA_VXLAN_ID, be32_to_cpu(cfg->vni)))
 		goto nla_put_failure;
 
-	if (!vxlan_addr_any(&dst->remote_ip)) {
-		if (dst->remote_ip.sa.sa_family == AF_INET) {
+	if (!vxlan_addr_any(&cfg->remote_ip)) {
+		if (cfg->remote_ip.sa.sa_family == AF_INET) {
 			if (nla_put_in_addr(skb, IFLA_VXLAN_GROUP,
-					    dst->remote_ip.sin.sin_addr.s_addr))
+					    cfg->remote_ip.sin.sin_addr.s_addr))
 				goto nla_put_failure;
 #if IS_ENABLED(CONFIG_IPV6)
 		} else {
 			if (nla_put_in6_addr(skb, IFLA_VXLAN_GROUP6,
-					     &dst->remote_ip.sin6.sin6_addr))
+					     &cfg->remote_ip.sin6.sin6_addr))
 				goto nla_put_failure;
 #endif
 		}
 	}
 
-	if (dst->remote_ifindex && nla_put_u32(skb, IFLA_VXLAN_LINK, dst->remote_ifindex))
+	if (cfg->remote_ifindex && nla_put_u32(skb, IFLA_VXLAN_LINK, cfg->remote_ifindex))
 		goto nla_put_failure;
 
-	if (!vxlan_addr_any(&vxlan->cfg.saddr)) {
-		if (vxlan->cfg.saddr.sa.sa_family == AF_INET) {
+	if (!vxlan_addr_any(&cfg->saddr)) {
+		if (cfg->saddr.sa.sa_family == AF_INET) {
 			if (nla_put_in_addr(skb, IFLA_VXLAN_LOCAL,
-					    vxlan->cfg.saddr.sin.sin_addr.s_addr))
+					    cfg->saddr.sin.sin_addr.s_addr))
 				goto nla_put_failure;
 #if IS_ENABLED(CONFIG_IPV6)
 		} else {
 			if (nla_put_in6_addr(skb, IFLA_VXLAN_LOCAL6,
-					     &vxlan->cfg.saddr.sin6.sin6_addr))
+					     &cfg->saddr.sin6.sin6_addr))
 				goto nla_put_failure;
 #endif
 		}
 	}
 
-	if (nla_put_u8(skb, IFLA_VXLAN_TTL, vxlan->cfg.ttl) ||
+	if (nla_put_u8(skb, IFLA_VXLAN_TTL, cfg->ttl) ||
 	    nla_put_u8(skb, IFLA_VXLAN_TTL_INHERIT,
-		       !!(vxlan->cfg.flags & VXLAN_F_TTL_INHERIT)) ||
-	    nla_put_u8(skb, IFLA_VXLAN_TOS, vxlan->cfg.tos) ||
-	    nla_put_u8(skb, IFLA_VXLAN_DF, vxlan->cfg.df) ||
-	    nla_put_be32(skb, IFLA_VXLAN_LABEL, vxlan->cfg.label) ||
-	    nla_put_u32(skb, IFLA_VXLAN_LABEL_POLICY, vxlan->cfg.label_policy) ||
+		       !!(cfg->flags & VXLAN_F_TTL_INHERIT)) ||
+	    nla_put_u8(skb, IFLA_VXLAN_TOS, cfg->tos) ||
+	    nla_put_u8(skb, IFLA_VXLAN_DF, cfg->df) ||
+	    nla_put_be32(skb, IFLA_VXLAN_LABEL, cfg->label) ||
+	    nla_put_u32(skb, IFLA_VXLAN_LABEL_POLICY, cfg->label_policy) ||
 	    nla_put_u8(skb, IFLA_VXLAN_LEARNING,
-		       !!(vxlan->cfg.flags & VXLAN_F_LEARN)) ||
+		       !!(cfg->flags & VXLAN_F_LEARN)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_PROXY,
-		       !!(vxlan->cfg.flags & VXLAN_F_PROXY)) ||
+		       !!(cfg->flags & VXLAN_F_PROXY)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_RSC,
-		       !!(vxlan->cfg.flags & VXLAN_F_RSC)) ||
+		       !!(cfg->flags & VXLAN_F_RSC)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_L2MISS,
-		       !!(vxlan->cfg.flags & VXLAN_F_L2MISS)) ||
+		       !!(cfg->flags & VXLAN_F_L2MISS)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_L3MISS,
-		       !!(vxlan->cfg.flags & VXLAN_F_L3MISS)) ||
+		       !!(cfg->flags & VXLAN_F_L3MISS)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_COLLECT_METADATA,
-		       !!(vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA)) ||
-	    nla_put_u32(skb, IFLA_VXLAN_AGEING, vxlan->cfg.age_interval) ||
-	    nla_put_u32(skb, IFLA_VXLAN_LIMIT, vxlan->cfg.addrmax) ||
-	    nla_put_be16(skb, IFLA_VXLAN_PORT, vxlan->cfg.dst_port) ||
+		       !!(cfg->flags & VXLAN_F_COLLECT_METADATA)) ||
+	    nla_put_u32(skb, IFLA_VXLAN_AGEING, cfg->age_interval) ||
+	    nla_put_u32(skb, IFLA_VXLAN_LIMIT, cfg->addrmax) ||
+	    nla_put_be16(skb, IFLA_VXLAN_PORT, cfg->dst_port) ||
 	    nla_put_u8(skb, IFLA_VXLAN_UDP_CSUM,
-		       !(vxlan->cfg.flags & VXLAN_F_UDP_ZERO_CSUM_TX)) ||
+		       !(cfg->flags & VXLAN_F_UDP_ZERO_CSUM_TX)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_TX,
-		       !!(vxlan->cfg.flags & VXLAN_F_UDP_ZERO_CSUM6_TX)) ||
+		       !!(cfg->flags & VXLAN_F_UDP_ZERO_CSUM6_TX)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_RX,
-		       !!(vxlan->cfg.flags & VXLAN_F_UDP_ZERO_CSUM6_RX)) ||
+		       !!(cfg->flags & VXLAN_F_UDP_ZERO_CSUM6_RX)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_REMCSUM_TX,
-		       !!(vxlan->cfg.flags & VXLAN_F_REMCSUM_TX)) ||
+		       !!(cfg->flags & VXLAN_F_REMCSUM_TX)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_REMCSUM_RX,
-		       !!(vxlan->cfg.flags & VXLAN_F_REMCSUM_RX)) ||
+		       !!(cfg->flags & VXLAN_F_REMCSUM_RX)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_LOCALBYPASS,
-		       !!(vxlan->cfg.flags & VXLAN_F_LOCALBYPASS)))
+		       !!(cfg->flags & VXLAN_F_LOCALBYPASS)))
 		goto nla_put_failure;
 
+	ports.low = htons(cfg->port_min);
+	ports.high = htons(cfg->port_max);
 	if (nla_put(skb, IFLA_VXLAN_PORT_RANGE, sizeof(ports), &ports))
 		goto nla_put_failure;
 
-	if (vxlan->cfg.flags & VXLAN_F_GBP &&
+	if (cfg->flags & VXLAN_F_GBP &&
 	    nla_put_flag(skb, IFLA_VXLAN_GBP))
 		goto nla_put_failure;
 
-	if (vxlan->cfg.flags & VXLAN_F_GPE &&
+	if (cfg->flags & VXLAN_F_GPE &&
 	    nla_put_flag(skb, IFLA_VXLAN_GPE))
 		goto nla_put_failure;
 
-	if (vxlan->cfg.flags & VXLAN_F_REMCSUM_NOPARTIAL &&
+	if (cfg->flags & VXLAN_F_REMCSUM_NOPARTIAL &&
 	    nla_put_flag(skb, IFLA_VXLAN_REMCSUM_NOPARTIAL))
 		goto nla_put_failure;
 
-	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER &&
+	if (cfg->flags & VXLAN_F_VNIFILTER &&
 	    nla_put_u8(skb, IFLA_VXLAN_VNIFILTER,
-		       !!(vxlan->cfg.flags & VXLAN_F_VNIFILTER)))
+		       !!(cfg->flags & VXLAN_F_VNIFILTER)))
 		goto nla_put_failure;
 
 	if (nla_put(skb, IFLA_VXLAN_RESERVED_BITS,
-		    sizeof(vxlan->cfg.reserved_bits),
-		    &vxlan->cfg.reserved_bits))
+		    sizeof(cfg->reserved_bits),
+		    &cfg->reserved_bits))
 		goto nla_put_failure;
 
+	rcu_read_unlock();
 	return 0;
 
 nla_put_failure:
+	rcu_read_unlock();
 	return -EMSGSIZE;
 }
 
@@ -4725,7 +4887,7 @@ static void vxlan_handle_lowerdev_unregister(struct vxlan_net *vn,
 	LIST_HEAD(list_kill);
 
 	list_for_each_entry_safe(vxlan, next, &vn->vxlan_list, next) {
-		struct vxlan_rdst *dst = &vxlan->default_dst;
+		const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 
 		/* In case we created vxlan device with carrier
 		 * and we loose the carrier due to module unload
@@ -4733,7 +4895,7 @@ static void vxlan_handle_lowerdev_unregister(struct vxlan_net *vn,
 		 * cases, it's not necessary and remote_ifindex
 		 * is 0 here, so no matches.
 		 */
-		if (dst->remote_ifindex == dev->ifindex)
+		if (cfg->remote_ifindex == dev->ifindex)
 			vxlan_dellink(vxlan->dev, &list_kill);
 	}
 
@@ -4746,6 +4908,10 @@ static int vxlan_netdevice_event(struct notifier_block *unused,
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 	struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
 
+	if ((event == NETDEV_REGISTER || event == NETDEV_UNREGISTER) &&
+	    netif_is_vxlan(dev))
+		vxlan_vnifilter_seq_inc(dev_net(dev));
+
 	if (event == NETDEV_UNREGISTER)
 		vxlan_handle_lowerdev_unregister(vn, dev);
 	else if (event == NETDEV_UDP_TUNNEL_PUSH_INFO)
diff --git a/drivers/net/vxlan/vxlan_mdb.c b/drivers/net/vxlan/vxlan_mdb.c
index 841f42ffecb9aa..c1a0551990555c 100644
--- a/drivers/net/vxlan/vxlan_mdb.c
+++ b/drivers/net/vxlan/vxlan_mdb.c
@@ -165,6 +165,7 @@ static int vxlan_mdb_entry_info_fill(const struct vxlan_dev *vxlan,
 				     const struct vxlan_mdb_entry *mdb_entry,
 				     const struct vxlan_mdb_remote *remote)
 {
+	const struct vxlan_config *cfg = rcu_dereference_rtnl(vxlan->cfg);
 	struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
 	struct br_mdb_entry e;
 	struct nlattr *nest;
@@ -189,12 +190,12 @@ static int vxlan_mdb_entry_info_fill(const struct vxlan_dev *vxlan,
 	    vxlan_nla_put_addr(skb, MDBA_MDB_EATTR_DST, &rd->remote_ip))
 		goto nest_err;
 
-	if (rd->remote_port && rd->remote_port != vxlan->cfg.dst_port &&
+	if (rd->remote_port && rd->remote_port != cfg->dst_port &&
 	    nla_put_u16(skb, MDBA_MDB_EATTR_DST_PORT,
 			be16_to_cpu(rd->remote_port)))
 		goto nest_err;
 
-	if (rd->remote_vni != vxlan->default_dst.remote_vni &&
+	if (rd->remote_vni != cfg->vni &&
 	    nla_put_u32(skb, MDBA_MDB_EATTR_VNI, be32_to_cpu(rd->remote_vni)))
 		goto nest_err;
 
@@ -202,7 +203,7 @@ static int vxlan_mdb_entry_info_fill(const struct vxlan_dev *vxlan,
 	    nla_put_u32(skb, MDBA_MDB_EATTR_IFINDEX, rd->remote_ifindex))
 		goto nest_err;
 
-	if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) &&
+	if ((cfg->flags & VXLAN_F_COLLECT_METADATA) &&
 	    mdb_entry->key.vni && nla_put_u32(skb, MDBA_MDB_EATTR_SRC_VNI,
 					      be32_to_cpu(mdb_entry->key.vni)))
 		goto nest_err;
@@ -613,16 +614,19 @@ static int vxlan_mdb_config_init(struct vxlan_mdb_config *cfg,
 {
 	struct br_mdb_entry *entry = nla_data(tb[MDBA_SET_ENTRY]);
 	struct vxlan_dev *vxlan = netdev_priv(dev);
+	const struct vxlan_config *vcfg;
+
+	vcfg = rtnl_dereference(vxlan->cfg);
 
 	memset(cfg, 0, sizeof(*cfg));
 	cfg->vxlan = vxlan;
-	cfg->group.vni = vxlan->default_dst.remote_vni;
+	cfg->group.vni = vcfg->vni;
 	INIT_LIST_HEAD(&cfg->src_list);
 	cfg->nlflags = nlmsg_flags;
 	cfg->filter_mode = MCAST_EXCLUDE;
 	cfg->rt_protocol = RTPROT_STATIC;
-	cfg->remote_vni = vxlan->default_dst.remote_vni;
-	cfg->remote_port = vxlan->cfg.dst_port;
+	cfg->remote_vni = vcfg->vni;
+	cfg->remote_port = vcfg->dst_port;
 
 	if (entry->ifindex != dev->ifindex) {
 		NL_SET_ERR_MSG_MOD(extack, "Port net device must be the VXLAN net device");
@@ -957,11 +961,12 @@ vxlan_mdb_nlmsg_remote_size(const struct vxlan_dev *vxlan,
 			    const struct vxlan_mdb_entry *mdb_entry,
 			    const struct vxlan_mdb_remote *remote)
 {
+	const struct vxlan_config *cfg = rcu_dereference_rtnl(vxlan->cfg);
 	const struct vxlan_mdb_entry_key *group = &mdb_entry->key;
 	struct vxlan_rdst *rd = rtnl_dereference(remote->rd);
 	size_t nlmsg_size;
 
-		     /* MDBA_MDB_ENTRY_INFO */
+	/* MDBA_MDB_ENTRY_INFO */
 	nlmsg_size = nla_total_size(sizeof(struct br_mdb_entry)) +
 		     /* MDBA_MDB_EATTR_TIMER */
 		     nla_total_size(sizeof(u32));
@@ -978,16 +983,16 @@ vxlan_mdb_nlmsg_remote_size(const struct vxlan_dev *vxlan,
 	/* MDBA_MDB_EATTR_DST */
 	nlmsg_size += nla_total_size(vxlan_addr_size(&rd->remote_ip));
 	/* MDBA_MDB_EATTR_DST_PORT */
-	if (rd->remote_port && rd->remote_port != vxlan->cfg.dst_port)
+	if (rd->remote_port && rd->remote_port != cfg->dst_port)
 		nlmsg_size += nla_total_size(sizeof(u16));
 	/* MDBA_MDB_EATTR_VNI */
-	if (rd->remote_vni != vxlan->default_dst.remote_vni)
+	if (rd->remote_vni != cfg->vni)
 		nlmsg_size += nla_total_size(sizeof(u32));
 	/* MDBA_MDB_EATTR_IFINDEX */
 	if (rd->remote_ifindex)
 		nlmsg_size += nla_total_size(sizeof(u32));
 	/* MDBA_MDB_EATTR_SRC_VNI */
-	if ((vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA) && group->vni)
+	if ((cfg->flags & VXLAN_F_COLLECT_METADATA) && group->vni)
 		nlmsg_size += nla_total_size(sizeof(u32));
 
 	return nlmsg_size;
@@ -1216,7 +1221,7 @@ vxlan_mdb_entry_get(struct vxlan_dev *vxlan,
 		goto err_free_entry;
 
 	if (hlist_is_singular_node(&mdb_entry->mdb_node, &vxlan->mdb_list))
-		vxlan->cfg.flags |= VXLAN_F_MDB;
+		set_bit(VXLAN_DEV_F_MDB, &vxlan->flags);
 
 	return mdb_entry;
 
@@ -1233,7 +1238,7 @@ static void vxlan_mdb_entry_put(struct vxlan_dev *vxlan,
 		return;
 
 	if (hlist_is_singular_node(&mdb_entry->mdb_node, &vxlan->mdb_list))
-		vxlan->cfg.flags &= ~VXLAN_F_MDB;
+		clear_bit(VXLAN_DEV_F_MDB, &vxlan->flags);
 
 	rhashtable_remove_fast(&vxlan->mdb_tbl, &mdb_entry->rhnode,
 			       vxlan_mdb_rht_params);
@@ -1483,11 +1488,13 @@ static int vxlan_mdb_get_parse(struct net_device *dev, struct nlattr *tb[],
 {
 	struct br_mdb_entry *entry = nla_data(tb[MDBA_GET_ENTRY]);
 	struct nlattr *mdbe_attrs[MDBE_ATTR_MAX + 1];
+	const struct vxlan_config *cfg;
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	int err;
 
+	cfg = rtnl_dereference(vxlan->cfg);
 	memset(group, 0, sizeof(*group));
-	group->vni = vxlan->default_dst.remote_vni;
+	group->vni = cfg->vni;
 
 	if (!tb[MDBA_GET_ENTRY_ATTRS]) {
 		vxlan_mdb_group_set(group, entry, NULL);
@@ -1621,6 +1628,7 @@ int vxlan_mdb_get(struct net_device *dev, struct nlattr *tb[], u32 portid,
 }
 
 struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan,
+						const struct vxlan_config *cfg,
 						struct sk_buff *skb,
 						__be32 src_vni)
 {
@@ -1634,8 +1642,8 @@ struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan,
 	/* When not in collect metadata mode, 'src_vni' is zero, but MDB
 	 * entries are stored with the VNI of the VXLAN device.
 	 */
-	if (!(vxlan->cfg.flags & VXLAN_F_COLLECT_METADATA))
-		src_vni = vxlan->default_dst.remote_vni;
+	if (!(cfg->flags & VXLAN_F_COLLECT_METADATA))
+		src_vni = cfg->vni;
 
 	memset(&group, 0, sizeof(group));
 	group.vni = src_vni;
@@ -1700,6 +1708,7 @@ struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan,
 }
 
 netdev_tx_t vxlan_mdb_xmit(struct vxlan_dev *vxlan,
+			   const struct vxlan_config *cfg,
 			   const struct vxlan_mdb_entry *mdb_entry,
 			   struct sk_buff *skb)
 {
@@ -1721,12 +1730,12 @@ netdev_tx_t vxlan_mdb_xmit(struct vxlan_dev *vxlan,
 
 		skb1 = skb_clone(skb, GFP_ATOMIC);
 		if (skb1)
-			vxlan_xmit_one(skb1, vxlan->dev, src_vni,
+			vxlan_xmit_one(skb1, vxlan->dev, cfg, src_vni,
 				       rcu_dereference(remote->rd), false);
 	}
 
 	if (fremote)
-		vxlan_xmit_one(skb, vxlan->dev, src_vni,
+		vxlan_xmit_one(skb, vxlan->dev, cfg, src_vni,
 			       rcu_dereference(fremote->rd), false);
 	else
 		kfree_skb_reason(skb, SKB_DROP_REASON_NO_TX_TARGET);
@@ -1757,7 +1766,7 @@ void vxlan_mdb_fini(struct vxlan_dev *vxlan)
 	struct vxlan_mdb_flush_desc desc = {};
 
 	vxlan_mdb_flush(vxlan, &desc);
-	WARN_ON_ONCE(vxlan->cfg.flags & VXLAN_F_MDB);
+	WARN_ON_ONCE(test_bit(VXLAN_DEV_F_MDB, &vxlan->flags));
 	rhashtable_free_and_destroy(&vxlan->mdb_tbl, vxlan_mdb_check_empty,
 				    NULL);
 }
diff --git a/drivers/net/vxlan/vxlan_multicast.c b/drivers/net/vxlan/vxlan_multicast.c
index 3b75b48dc726df..a5ef6f66f00d4b 100644
--- a/drivers/net/vxlan/vxlan_multicast.c
+++ b/drivers/net/vxlan/vxlan_multicast.c
@@ -14,11 +14,12 @@
 /* Update multicast group membership when first VNI on
  * multicast address is brought up
  */
-int vxlan_igmp_join(struct vxlan_dev *vxlan, union vxlan_addr *rip,
+int vxlan_igmp_join(struct vxlan_dev *vxlan, const union vxlan_addr *rip,
 		    int rifindex)
 {
-	union vxlan_addr *ip = (rip ? : &vxlan->default_dst.remote_ip);
-	int ifindex = (rifindex ? : vxlan->default_dst.remote_ifindex);
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
+	const union vxlan_addr *ip = (rip ? : &cfg->remote_ip);
+	int ifindex = (rifindex ? : cfg->remote_ifindex);
 	int ret = -EINVAL;
 	struct sock *sk;
 
@@ -47,11 +48,12 @@ int vxlan_igmp_join(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 	return ret;
 }
 
-int vxlan_igmp_leave(struct vxlan_dev *vxlan, union vxlan_addr *rip,
+int vxlan_igmp_leave(struct vxlan_dev *vxlan, const union vxlan_addr *rip,
 		     int rifindex)
 {
-	union vxlan_addr *ip = (rip ? : &vxlan->default_dst.remote_ip);
-	int ifindex = (rifindex ? : vxlan->default_dst.remote_ifindex);
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
+	const union vxlan_addr *ip = (rip ? : &cfg->remote_ip);
+	int ifindex = (rifindex ? : cfg->remote_ifindex);
 	int ret = -EINVAL;
 	struct sock *sk;
 
@@ -80,8 +82,8 @@ int vxlan_igmp_leave(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 	return ret;
 }
 
-static bool vxlan_group_used_match(union vxlan_addr *ip, int ifindex,
-				   union vxlan_addr *rip, int rifindex)
+static bool vxlan_group_used_match(const union vxlan_addr *ip, int ifindex,
+				   const union vxlan_addr *rip, int rifindex)
 {
 	if (!vxlan_addr_multicast(rip))
 		return false;
@@ -96,14 +98,16 @@ static bool vxlan_group_used_match(union vxlan_addr *ip, int ifindex,
 }
 
 static bool vxlan_group_used_by_vnifilter(struct vxlan_dev *vxlan,
-					  union vxlan_addr *ip, int ifindex)
+					  const struct vxlan_config *cfg,
+					  const union vxlan_addr *ip,
+					  int ifindex)
 {
 	struct vxlan_vni_group *vg = rtnl_dereference(vxlan->vnigrp);
 	struct vxlan_vni_node *v, *tmp;
 
 	if (vxlan_group_used_match(ip, ifindex,
-				   &vxlan->default_dst.remote_ip,
-				   vxlan->default_dst.remote_ifindex))
+				   &cfg->remote_ip,
+				   cfg->remote_ifindex))
 		return true;
 
 	list_for_each_entry_safe(v, tmp, &vg->vni_list, vlist) {
@@ -112,7 +116,7 @@ static bool vxlan_group_used_by_vnifilter(struct vxlan_dev *vxlan,
 
 		if (vxlan_group_used_match(ip, ifindex,
 					   &v->remote_ip,
-					   vxlan->default_dst.remote_ifindex))
+					   cfg->remote_ifindex))
 			return true;
 	}
 
@@ -121,16 +125,17 @@ static bool vxlan_group_used_by_vnifilter(struct vxlan_dev *vxlan,
 
 /* See if multicast group is already in use by other ID */
 bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev,
-		      __be32 vni, union vxlan_addr *rip, int rifindex)
+		      __be32 vni, const union vxlan_addr *rip, int rifindex)
 {
-	union vxlan_addr *ip = (rip ? : &dev->default_dst.remote_ip);
-	int ifindex = (rifindex ? : dev->default_dst.remote_ifindex);
+	const struct vxlan_config *dev_cfg = rtnl_dereference(dev->cfg);
+	const union vxlan_addr *ip = (rip ? : &dev_cfg->remote_ip);
+	int ifindex = (rifindex ? : dev_cfg->remote_ifindex);
 	struct vxlan_dev *vxlan;
 	struct vxlan_sock *sock4;
 #if IS_ENABLED(CONFIG_IPV6)
 	struct vxlan_sock *sock6;
 #endif
-	unsigned short family = dev->default_dst.remote_ip.sa.sa_family;
+	unsigned short family = dev_cfg->remote_ip.sa.sa_family;
 
 	sock4 = rtnl_dereference(dev->vn4_sock);
 
@@ -147,6 +152,8 @@ bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev,
 #endif
 
 	list_for_each_entry(vxlan, &vn->vxlan_list, next) {
+		const struct vxlan_config *cfg;
+
 		if (!netif_running(vxlan->dev) || vxlan == dev)
 			continue;
 
@@ -158,13 +165,15 @@ bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev,
 		    rtnl_dereference(vxlan->vn6_sock) != sock6)
 			continue;
 #endif
-		if (vxlan->cfg.flags & VXLAN_F_VNIFILTER) {
-			if (!vxlan_group_used_by_vnifilter(vxlan, ip, ifindex))
+		cfg = rtnl_dereference(vxlan->cfg);
+
+		if (cfg->flags & VXLAN_F_VNIFILTER) {
+			if (!vxlan_group_used_by_vnifilter(vxlan, cfg, ip, ifindex))
 				continue;
 		} else {
 			if (!vxlan_group_used_match(ip, ifindex,
-						    &vxlan->default_dst.remote_ip,
-						    vxlan->default_dst.remote_ifindex))
+						    &cfg->remote_ip,
+						    cfg->remote_ifindex))
 				continue;
 		}
 
@@ -174,7 +183,8 @@ bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev,
 	return false;
 }
 
-static int vxlan_multicast_join_vnigrp(struct vxlan_dev *vxlan)
+static int vxlan_multicast_join_vnigrp(struct vxlan_dev *vxlan,
+				       const struct vxlan_config *cfg)
 {
 	struct vxlan_vni_group *vg = rtnl_dereference(vxlan->vnigrp);
 	struct vxlan_vni_node *v, *tmp, *vgood = NULL;
@@ -185,7 +195,7 @@ static int vxlan_multicast_join_vnigrp(struct vxlan_dev *vxlan)
 			continue;
 		/* skip if address is same as default address */
 		if (vxlan_addr_equal(&v->remote_ip,
-				     &vxlan->default_dst.remote_ip))
+				     &cfg->remote_ip))
 			continue;
 		ret = vxlan_igmp_join(vxlan, &v->remote_ip, 0);
 		if (ret == -EADDRINUSE)
@@ -200,7 +210,7 @@ static int vxlan_multicast_join_vnigrp(struct vxlan_dev *vxlan)
 			if (!vxlan_addr_multicast(&v->remote_ip))
 				continue;
 			if (vxlan_addr_equal(&v->remote_ip,
-					     &vxlan->default_dst.remote_ip))
+					     &cfg->remote_ip))
 				continue;
 			vxlan_igmp_leave(vxlan, &v->remote_ip, 0);
 			if (v == vgood)
@@ -211,7 +221,8 @@ static int vxlan_multicast_join_vnigrp(struct vxlan_dev *vxlan)
 	return ret;
 }
 
-static int vxlan_multicast_leave_vnigrp(struct vxlan_dev *vxlan)
+static int vxlan_multicast_leave_vnigrp(struct vxlan_dev *vxlan,
+					const struct vxlan_config *cfg)
 {
 	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
 	struct vxlan_vni_group *vg = rtnl_dereference(vxlan->vnigrp);
@@ -219,10 +230,17 @@ static int vxlan_multicast_leave_vnigrp(struct vxlan_dev *vxlan)
 	int last_err = 0, ret;
 
 	list_for_each_entry_safe(v, tmp, &vg->vni_list, vlist) {
-		if (vxlan_addr_multicast(&v->remote_ip) &&
-		    !vxlan_group_used(vn, vxlan, v->vni, &v->remote_ip,
+		if (!vxlan_addr_multicast(&v->remote_ip))
+			continue;
+		/* skip if address is same as default address */
+		if (vxlan_addr_equal(&v->remote_ip,
+				     &cfg->remote_ip))
+			continue;
+		if (!vxlan_group_used(vn, vxlan, v->vni, &v->remote_ip,
 				      0)) {
 			ret = vxlan_igmp_leave(vxlan, &v->remote_ip, 0);
+			if (ret == -EADDRNOTAVAIL)
+				ret = 0;
 			if (ret)
 				last_err = ret;
 		}
@@ -233,38 +251,42 @@ static int vxlan_multicast_leave_vnigrp(struct vxlan_dev *vxlan)
 
 int vxlan_multicast_join(struct vxlan_dev *vxlan)
 {
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 	int ret = 0;
 
-	if (vxlan_addr_multicast(&vxlan->default_dst.remote_ip)) {
-		ret = vxlan_igmp_join(vxlan, &vxlan->default_dst.remote_ip,
-				      vxlan->default_dst.remote_ifindex);
+	if (vxlan_addr_multicast(&cfg->remote_ip)) {
+		ret = vxlan_igmp_join(vxlan, &cfg->remote_ip,
+				      cfg->remote_ifindex);
 		if (ret == -EADDRINUSE)
 			ret = 0;
 		if (ret)
 			return ret;
 	}
 
-	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
-		return vxlan_multicast_join_vnigrp(vxlan);
+	if (cfg->flags & VXLAN_F_VNIFILTER)
+		return vxlan_multicast_join_vnigrp(vxlan, cfg);
 
 	return 0;
 }
 
 int vxlan_multicast_leave(struct vxlan_dev *vxlan)
 {
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
 	int ret = 0;
 
-	if (vxlan_addr_multicast(&vxlan->default_dst.remote_ip) &&
+	if (vxlan_addr_multicast(&cfg->remote_ip) &&
 	    !vxlan_group_used(vn, vxlan, 0, NULL, 0)) {
-		ret = vxlan_igmp_leave(vxlan, &vxlan->default_dst.remote_ip,
-				       vxlan->default_dst.remote_ifindex);
+		ret = vxlan_igmp_leave(vxlan, &cfg->remote_ip,
+				       cfg->remote_ifindex);
+		if (ret == -EADDRNOTAVAIL)
+			ret = 0;
 		if (ret)
 			return ret;
 	}
 
-	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
-		return vxlan_multicast_leave_vnigrp(vxlan);
+	if (cfg->flags & VXLAN_F_VNIFILTER)
+		return vxlan_multicast_leave_vnigrp(vxlan, cfg);
 
 	return 0;
 }
diff --git a/drivers/net/vxlan/vxlan_private.h b/drivers/net/vxlan/vxlan_private.h
index b1eec221636088..f0ee85f83732a2 100644
--- a/drivers/net/vxlan/vxlan_private.h
+++ b/drivers/net/vxlan/vxlan_private.h
@@ -22,6 +22,8 @@ struct vxlan_net {
 	/* sock_list is protected by rtnl lock */
 	struct hlist_head sock_list[PORT_HASH_SIZE];
 	struct notifier_block nexthop_notifier_block;
+	/* Generation counter for RTM_GETTUNNEL dumps */
+	atomic_t vnifilter_seq;
 };
 
 struct vxlan_fdb_key {
@@ -177,6 +179,13 @@ vxlan_vnifilter_lookup(struct vxlan_dev *vxlan, __be32 vni)
 				      vxlan_vni_rht_params);
 }
 
+static inline void vxlan_vnifilter_seq_inc(const struct net *net)
+{
+	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
+
+	atomic_inc(&vn->vnifilter_seq);
+}
+
 /* vxlan_core.c */
 int vxlan_fdb_create(struct vxlan_dev *vxlan,
 		     const u8 *mac, union vxlan_addr *ip,
@@ -195,9 +204,10 @@ int vxlan_fdb_update(struct vxlan_dev *vxlan,
 		     __u32 ifindex, __u16 ndm_flags, u32 nhid,
 		     bool swdev_notify, struct netlink_ext_ack *extack);
 void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
+		    const struct vxlan_config *cfg,
 		    __be32 default_vni, struct vxlan_rdst *rdst, bool did_rsc);
 int vxlan_vni_in_use(struct net *src_net, struct vxlan_dev *vxlan,
-		     struct vxlan_config *conf, __be32 vni);
+		     const struct vxlan_config *conf, __be32 vni);
 
 /* vxlan_vnifilter.c */
 int vxlan_vnigroup_init(struct vxlan_dev *vxlan);
@@ -205,7 +215,8 @@ void vxlan_vnigroup_uninit(struct vxlan_dev *vxlan);
 
 int vxlan_vnifilter_init(void);
 void vxlan_vnifilter_uninit(void);
-void vxlan_vnifilter_count(struct vxlan_dev *vxlan, __be32 vni,
+void vxlan_vnifilter_count(struct vxlan_dev *vxlan,
+			   const struct vxlan_config *cfg, __be32 vni,
 			   struct vxlan_vni_node *vninode,
 			   int type, unsigned int len);
 
@@ -213,9 +224,16 @@ void vxlan_vs_add_vnigrp(struct vxlan_dev *vxlan,
 			 struct vxlan_sock *vs,
 			 bool ipv6);
 void vxlan_vs_del_vnigrp(struct vxlan_dev *vxlan);
+bool vxlan_vnifilter_has_multicast(const struct vxlan_dev *vxlan);
+int vxlan_update_default_fdb_entry(struct vxlan_dev *vxlan, __be32 vni,
+				   const union vxlan_addr *old_remote_ip,
+				   const union vxlan_addr *remote_ip,
+				   u32 old_ifindex, u32 new_ifindex,
+				   struct netlink_ext_ack *extack);
 int vxlan_vnilist_update_group(struct vxlan_dev *vxlan,
-			       union vxlan_addr *old_remote_ip,
-			       union vxlan_addr *new_remote_ip,
+			       const union vxlan_addr *old_remote_ip,
+			       const union vxlan_addr *new_remote_ip,
+			       u32 old_ifindex, u32 new_ifindex,
 			       struct netlink_ext_ack *extack);
 
 
@@ -223,10 +241,10 @@ int vxlan_vnilist_update_group(struct vxlan_dev *vxlan,
 int vxlan_multicast_join(struct vxlan_dev *vxlan);
 int vxlan_multicast_leave(struct vxlan_dev *vxlan);
 bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev,
-		      __be32 vni, union vxlan_addr *rip, int rifindex);
-int vxlan_igmp_join(struct vxlan_dev *vxlan, union vxlan_addr *rip,
+		      __be32 vni, const union vxlan_addr *rip, int rifindex);
+int vxlan_igmp_join(struct vxlan_dev *vxlan, const union vxlan_addr *rip,
 		    int rifindex);
-int vxlan_igmp_leave(struct vxlan_dev *vxlan, union vxlan_addr *rip,
+int vxlan_igmp_leave(struct vxlan_dev *vxlan, const union vxlan_addr *rip,
 		     int rifindex);
 
 /* vxlan_mdb.c */
@@ -241,9 +259,11 @@ int vxlan_mdb_del_bulk(struct net_device *dev, struct nlattr *tb[],
 int vxlan_mdb_get(struct net_device *dev, struct nlattr *tb[], u32 portid,
 		  u32 seq, struct netlink_ext_ack *extack);
 struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan,
+						const struct vxlan_config *cfg,
 						struct sk_buff *skb,
 						__be32 src_vni);
 netdev_tx_t vxlan_mdb_xmit(struct vxlan_dev *vxlan,
+			   const struct vxlan_config *cfg,
 			   const struct vxlan_mdb_entry *mdb_entry,
 			   struct sk_buff *skb);
 int vxlan_mdb_init(struct vxlan_dev *vxlan);
diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c
index dd94085e088656..41095096b698dd 100644
--- a/drivers/net/vxlan/vxlan_vnifilter.c
+++ b/drivers/net/vxlan/vxlan_vnifilter.c
@@ -171,13 +171,14 @@ static void vxlan_vnifilter_stats_add(struct vxlan_vni_node *vninode,
 	u64_stats_update_end(&pstats->syncp);
 }
 
-void vxlan_vnifilter_count(struct vxlan_dev *vxlan, __be32 vni,
+void vxlan_vnifilter_count(struct vxlan_dev *vxlan,
+			   const struct vxlan_config *cfg, __be32 vni,
 			   struct vxlan_vni_node *vninode,
 			   int type, unsigned int len)
 {
 	struct vxlan_vni_node *vnode;
 
-	if (!(vxlan->cfg.flags & VXLAN_F_VNIFILTER))
+	if (!(cfg->flags & VXLAN_F_VNIFILTER))
 		return;
 
 	if (vninode) {
@@ -333,22 +334,36 @@ static int vxlan_vnifilter_dump_dev(const struct net_device *dev,
 				    struct sk_buff *skb,
 				    struct netlink_callback *cb)
 {
-	struct vxlan_vni_node *tmp, *v, *vbegin = NULL, *vend = NULL;
+	struct vxlan_vni_node *v, *vbegin = NULL, *vend = NULL;
 	struct vxlan_dev *vxlan = netdev_priv(dev);
 	struct tunnel_msg *new_tmsg, *tmsg;
-	int idx = 0, s_idx = cb->args[1];
+	const struct vxlan_config *cfg;
 	struct vxlan_vni_group *vg;
 	struct nlmsghdr *nlh;
+	int idx = 0, s_idx;
 	bool dump_stats;
 	int err = 0;
 
-	if (!(vxlan->cfg.flags & VXLAN_F_VNIFILTER))
+	if (cb->args[2] != dev->ifindex) {
+		cb->args[1] = 0;
+		cb->args[2] = dev->ifindex;
+	}
+	s_idx = cb->args[1];
+
+	cfg = rcu_dereference(vxlan->cfg);
+	if (!(cfg->flags & VXLAN_F_VNIFILTER)) {
+		cb->args[1] = 0;
+		cb->args[2] = 0;
 		return -EINVAL;
+	}
 
 	/* RCU needed because of the vni locking rules (rcu || rtnl) */
 	vg = rcu_dereference(vxlan->vnigrp);
-	if (!vg || !vg->num_vnis)
+	if (!vg || !vg->num_vnis) {
+		cb->args[1] = 0;
+		cb->args[2] = 0;
 		return 0;
+	}
 
 	tmsg = nlmsg_data(cb->nlh);
 	dump_stats = !!(tmsg->flags & TUNNEL_MSG_FLAG_STATS);
@@ -362,7 +377,7 @@ static int vxlan_vnifilter_dump_dev(const struct net_device *dev,
 	new_tmsg->family = PF_BRIDGE;
 	new_tmsg->ifindex = dev->ifindex;
 
-	list_for_each_entry_safe(v, tmp, &vg->vni_list, vlist) {
+	list_for_each_entry_rcu(v, &vg->vni_list, vlist) {
 		if (idx < s_idx) {
 			idx++;
 			continue;
@@ -394,12 +409,26 @@ static int vxlan_vnifilter_dump_dev(const struct net_device *dev,
 	}
 
 	cb->args[1] = err ? idx : 0;
+	cb->args[2] = err ? dev->ifindex : 0;
 
 	nlmsg_end(skb, nlh);
 
+	nl_dump_check_consistent(cb, nlh);
+
 	return err;
 }
 
+static u32 vxlan_vnifilter_base_seq(const struct net *net)
+{
+	const struct vxlan_net *vn = net_generic(net, vxlan_net_id);
+	u32 res = atomic_read(&vn->vnifilter_seq);
+
+	/* Must not return 0 (see nl_dump_check_consistent()) */
+	if (!res)
+		res = 0x80000000;
+	return res;
+}
+
 static int vxlan_vnifilter_dump(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	int idx = 0, err = 0, s_idx = cb->args[0];
@@ -419,6 +448,9 @@ static int vxlan_vnifilter_dump(struct sk_buff *skb, struct netlink_callback *cb
 	}
 
 	rcu_read_lock();
+
+	cb->seq = vxlan_vnifilter_base_seq(net);
+
 	if (tmsg->ifindex) {
 		dev = dev_get_by_index_rcu(net, tmsg->ifindex);
 		if (!dev) {
@@ -470,26 +502,50 @@ static const struct nla_policy vni_filter_policy[VXLAN_VNIFILTER_MAX + 1] = {
 	[VXLAN_VNIFILTER_ENTRY] = { .type = NLA_NESTED },
 };
 
-static int vxlan_update_default_fdb_entry(struct vxlan_dev *vxlan, __be32 vni,
-					  union vxlan_addr *old_remote_ip,
-					  union vxlan_addr *remote_ip,
-					  struct netlink_ext_ack *extack)
+bool vxlan_vnifilter_has_multicast(const struct vxlan_dev *vxlan)
 {
-	struct vxlan_rdst *dst = &vxlan->default_dst;
+	struct vxlan_vni_group *vg = rtnl_dereference(vxlan->vnigrp);
+	struct vxlan_vni_node *v;
+
+	if (!vg)
+		return false;
+
+	list_for_each_entry(v, &vg->vni_list, vlist) {
+		if (vxlan_addr_multicast(&v->remote_ip))
+			return true;
+	}
+
+	return false;
+}
+
+int vxlan_update_default_fdb_entry(struct vxlan_dev *vxlan, __be32 vni,
+				   const union vxlan_addr *old_remote_ip,
+				   const union vxlan_addr *remote_ip,
+				   u32 old_ifindex, u32 new_ifindex,
+				   struct netlink_ext_ack *extack)
+{
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 	int err = 0;
 
+	if (old_remote_ip && remote_ip &&
+	    vxlan_addr_equal(old_remote_ip, remote_ip) &&
+	    old_ifindex == new_ifindex)
+		return 0;
+
 	spin_lock_bh(&vxlan->hash_lock);
 	if (remote_ip && !vxlan_addr_any(remote_ip)) {
+		union vxlan_addr rip = *remote_ip;
+
 		err = vxlan_fdb_update(vxlan, all_zeros_mac,
-				       remote_ip,
+				       &rip,
 				       NUD_REACHABLE | NUD_PERMANENT,
 				       NLM_F_APPEND | NLM_F_CREATE,
-				       vxlan->cfg.dst_port,
+				       cfg->dst_port,
 				       vni,
 				       vni,
-				       dst->remote_ifindex,
+				       new_ifindex,
 				       NTF_SELF, 0, true, extack);
-		if (err) {
+		if (err && extack) {
 			spin_unlock_bh(&vxlan->hash_lock);
 			return err;
 		}
@@ -498,9 +554,9 @@ static int vxlan_update_default_fdb_entry(struct vxlan_dev *vxlan, __be32 vni,
 	if (old_remote_ip && !vxlan_addr_any(old_remote_ip)) {
 		__vxlan_fdb_delete(vxlan, all_zeros_mac,
 				   *old_remote_ip,
-				   vxlan->cfg.dst_port,
+				   cfg->dst_port,
 				   vni, vni,
-				   dst->remote_ifindex,
+				   old_ifindex,
 				   true);
 	}
 	spin_unlock_bh(&vxlan->hash_lock);
@@ -515,8 +571,8 @@ static int vxlan_vni_update_group(struct vxlan_dev *vxlan,
 				  struct netlink_ext_ack *extack)
 {
 	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
-	struct vxlan_rdst *dst = &vxlan->default_dst;
-	union vxlan_addr *newrip = NULL, *oldrip = NULL;
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
+	const union vxlan_addr *newrip = NULL, *oldrip = NULL;
 	union vxlan_addr old_remote_ip;
 	int ret = 0;
 
@@ -528,15 +584,16 @@ static int vxlan_vni_update_group(struct vxlan_dev *vxlan,
 	if (group && !vxlan_addr_any(group)) {
 		newrip = group;
 	} else {
-		if (!vxlan_addr_any(&dst->remote_ip))
-			newrip = &dst->remote_ip;
+		if (!vxlan_addr_any(&cfg->remote_ip))
+			newrip = &cfg->remote_ip;
 	}
 
-	/* if old rip exists, and no newrip,
-	 * explicitly delete old rip
-	 */
-	if (!newrip && !vxlan_addr_any(&old_remote_ip))
-		oldrip = &old_remote_ip;
+	if (!create) {
+		if (!vxlan_addr_any(&old_remote_ip))
+			oldrip = &old_remote_ip;
+		else if (!vxlan_addr_any(&cfg->remote_ip))
+			oldrip = &cfg->remote_ip;
+	}
 
 	if (!newrip && !oldrip)
 		return 0;
@@ -546,20 +603,27 @@ static int vxlan_vni_update_group(struct vxlan_dev *vxlan,
 
 	ret = vxlan_update_default_fdb_entry(vxlan, vninode->vni,
 					     oldrip, newrip,
+					     cfg->remote_ifindex,
+					     cfg->remote_ifindex,
 					     extack);
 	if (ret)
 		goto out;
 
-	if (group)
+	if (group) {
 		memcpy(&vninode->remote_ip, group, sizeof(vninode->remote_ip));
+		if (!create)
+			vxlan_vnifilter_seq_inc(dev_net(vxlan->dev));
+	}
 
 	if (vxlan->dev->flags & IFF_UP) {
 		if (vxlan_addr_multicast(&old_remote_ip) &&
 		    !vxlan_group_used(vn, vxlan, vninode->vni,
 				      &old_remote_ip,
-				      vxlan->default_dst.remote_ifindex)) {
+				      cfg->remote_ifindex)) {
 			ret = vxlan_igmp_leave(vxlan, &old_remote_ip,
 					       0);
+			if (ret == -EADDRNOTAVAIL)
+				ret = 0;
 			if (ret)
 				goto out;
 		}
@@ -581,10 +645,12 @@ static int vxlan_vni_update_group(struct vxlan_dev *vxlan,
 }
 
 int vxlan_vnilist_update_group(struct vxlan_dev *vxlan,
-			       union vxlan_addr *old_remote_ip,
-			       union vxlan_addr *new_remote_ip,
+			       const union vxlan_addr *old_remote_ip,
+			       const union vxlan_addr *new_remote_ip,
+			       u32 old_ifindex, u32 new_ifindex,
 			       struct netlink_ext_ack *extack)
 {
+	const union vxlan_addr *oldrip, *newrip;
 	struct list_head *headp, *hpos;
 	struct vxlan_vni_group *vg;
 	struct vxlan_vni_node *vent;
@@ -595,37 +661,66 @@ int vxlan_vnilist_update_group(struct vxlan_dev *vxlan,
 	headp = &vg->vni_list;
 	list_for_each_prev(hpos, headp) {
 		vent = list_entry(hpos, struct vxlan_vni_node, vlist);
+
 		if (vxlan_addr_any(&vent->remote_ip)) {
-			ret = vxlan_update_default_fdb_entry(vxlan, vent->vni,
-							     old_remote_ip,
-							     new_remote_ip,
-							     extack);
-			if (ret)
-				return ret;
+			oldrip = old_remote_ip;
+			newrip = new_remote_ip;
+		} else {
+			/* A vni with its own group keeps it, but its fdb entry
+			 * is still keyed on the device remote_ifindex.
+			 */
+			oldrip = &vent->remote_ip;
+			newrip = &vent->remote_ip;
 		}
+
+		ret = vxlan_update_default_fdb_entry(vxlan, vent->vni,
+						     oldrip, newrip,
+						     old_ifindex, new_ifindex,
+						     extack);
+		if (ret)
+			goto err_unwind;
 	}
 
 	return 0;
+
+err_unwind:
+	list_for_each_continue(hpos, headp) {
+		vent = list_entry(hpos, struct vxlan_vni_node, vlist);
+
+		if (vxlan_addr_any(&vent->remote_ip)) {
+			oldrip = old_remote_ip;
+			newrip = new_remote_ip;
+		} else {
+			oldrip = &vent->remote_ip;
+			newrip = &vent->remote_ip;
+		}
+
+		vxlan_update_default_fdb_entry(vxlan, vent->vni,
+					       newrip, oldrip,
+					       new_ifindex, old_ifindex,
+					       NULL);
+	}
+	return ret;
 }
 
 static void vxlan_vni_delete_group(struct vxlan_dev *vxlan,
 				   struct vxlan_vni_node *vninode)
 {
 	struct vxlan_net *vn = net_generic(vxlan->net, vxlan_net_id);
-	struct vxlan_rdst *dst = &vxlan->default_dst;
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 
 	/* if per vni remote_ip not present, delete the
 	 * default dst remote_ip previously added for this vni
 	 */
 	if (!vxlan_addr_any(&vninode->remote_ip) ||
-	    !vxlan_addr_any(&dst->remote_ip)) {
+	    !vxlan_addr_any(&cfg->remote_ip)) {
 		spin_lock_bh(&vxlan->hash_lock);
 		__vxlan_fdb_delete(vxlan, all_zeros_mac,
 				   (vxlan_addr_any(&vninode->remote_ip) ?
-				   dst->remote_ip : vninode->remote_ip),
-				   vxlan->cfg.dst_port,
+				   cfg->remote_ip : vninode->remote_ip),
+				   cfg->dst_port,
 				   vninode->vni, vninode->vni,
-				   dst->remote_ifindex,
+				   cfg->remote_ifindex,
 				   true);
 		spin_unlock_bh(&vxlan->hash_lock);
 	}
@@ -634,7 +729,7 @@ static void vxlan_vni_delete_group(struct vxlan_dev *vxlan,
 		if (vxlan_addr_multicast(&vninode->remote_ip) &&
 		    !vxlan_group_used(vn, vxlan, vninode->vni,
 				      &vninode->remote_ip,
-				      dst->remote_ifindex)) {
+				      cfg->remote_ifindex)) {
 			vxlan_igmp_leave(vxlan, &vninode->remote_ip, 0);
 		}
 	}
@@ -665,7 +760,8 @@ static int vxlan_vni_update(struct vxlan_dev *vxlan,
 	return 0;
 }
 
-static void __vxlan_vni_add_list(struct vxlan_vni_group *vg,
+static void __vxlan_vni_add_list(struct vxlan_dev *vxlan,
+				 struct vxlan_vni_group *vg,
 				 struct vxlan_vni_node *v)
 {
 	struct list_head *headp, *hpos;
@@ -681,13 +777,16 @@ static void __vxlan_vni_add_list(struct vxlan_vni_group *vg,
 	}
 	list_add_rcu(&v->vlist, hpos);
 	vg->num_vnis++;
+	vxlan_vnifilter_seq_inc(dev_net(vxlan->dev));
 }
 
-static void __vxlan_vni_del_list(struct vxlan_vni_group *vg,
+static void __vxlan_vni_del_list(struct vxlan_dev *vxlan,
+				 struct vxlan_vni_group *vg,
 				 struct vxlan_vni_node *v)
 {
 	list_del_rcu(&v->vlist);
 	vg->num_vnis--;
+	vxlan_vnifilter_seq_inc(dev_net(vxlan->dev));
 }
 
 static struct vxlan_vni_node *vxlan_vni_alloc(struct vxlan_dev *vxlan,
@@ -723,6 +822,7 @@ static int vxlan_vni_add(struct vxlan_dev *vxlan,
 			 u32 vni, union vxlan_addr *group,
 			 struct netlink_ext_ack *extack)
 {
+	const struct vxlan_config *cfg = rtnl_dereference(vxlan->cfg);
 	struct vxlan_vni_node *vninode;
 	__be32 v = cpu_to_be32(vni);
 	bool changed = false;
@@ -731,7 +831,7 @@ static int vxlan_vni_add(struct vxlan_dev *vxlan,
 	if (vxlan_vnifilter_lookup(vxlan, v))
 		return vxlan_vni_update(vxlan, vg, v, group, &changed, extack);
 
-	err = vxlan_vni_in_use(vxlan->net, vxlan, &vxlan->cfg, v);
+	err = vxlan_vni_in_use(vxlan->net, vxlan, cfg, v);
 	if (err) {
 		NL_SET_ERR_MSG(extack, "VNI in use");
 		return err;
@@ -749,7 +849,7 @@ static int vxlan_vni_add(struct vxlan_dev *vxlan,
 		return err;
 	}
 
-	__vxlan_vni_add_list(vg, vninode);
+	__vxlan_vni_add_list(vxlan, vg, vninode);
 
 	if (vxlan->dev->flags & IFF_UP)
 		vxlan_vs_add_del_vninode(vxlan, vninode, false);
@@ -795,7 +895,7 @@ static int vxlan_vni_del(struct vxlan_dev *vxlan,
 	if (err)
 		goto out;
 
-	__vxlan_vni_del_list(vg, vninode);
+	__vxlan_vni_del_list(vxlan, vg, vninode);
 
 	vxlan_vnifilter_notify(vxlan, vninode, RTM_DELTUNNEL);
 
@@ -844,6 +944,7 @@ static int vxlan_process_vni_filter(struct vxlan_dev *vxlan,
 				    int cmd, struct netlink_ext_ack *extack)
 {
 	struct nlattr *vattrs[VXLAN_VNIFILTER_ENTRY_MAX + 1];
+	const struct vxlan_config *cfg;
 	u32 vni_start = 0, vni_end = 0;
 	union vxlan_addr group;
 	int err;
@@ -881,7 +982,8 @@ static int vxlan_process_vni_filter(struct vxlan_dev *vxlan,
 		memset(&group, 0, sizeof(group));
 	}
 
-	if (vxlan_addr_multicast(&group) && !vxlan->default_dst.remote_ifindex) {
+	cfg = rtnl_dereference(vxlan->cfg);
+	if (vxlan_addr_multicast(&group) && !cfg->remote_ifindex) {
 		NL_SET_ERR_MSG(extack,
 			       "Local interface required for multicast remote group");
 
@@ -909,7 +1011,7 @@ void vxlan_vnigroup_uninit(struct vxlan_dev *vxlan)
 #if IS_ENABLED(CONFIG_IPV6)
 		hlist_del_init_rcu(&v->hlist6.hlist);
 #endif
-		__vxlan_vni_del_list(vg, v);
+		__vxlan_vni_del_list(vxlan, vg, v);
 		vxlan_vnifilter_notify(vxlan, v, RTM_DELTUNNEL);
 		call_rcu(&v->rcu, vxlan_vni_node_rcu_free);
 	}
@@ -940,6 +1042,7 @@ static int vxlan_vnifilter_process(struct sk_buff *skb, struct nlmsghdr *nlh,
 				   struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
+	const struct vxlan_config *cfg;
 	struct tunnel_msg *tmsg;
 	struct vxlan_dev *vxlan;
 	struct net_device *dev;
@@ -964,8 +1067,9 @@ static int vxlan_vnifilter_process(struct sk_buff *skb, struct nlmsghdr *nlh,
 	}
 
 	vxlan = netdev_priv(dev);
+	cfg = rtnl_dereference(vxlan->cfg);
 
-	if (!(vxlan->cfg.flags & VXLAN_F_VNIFILTER))
+	if (!(cfg->flags & VXLAN_F_VNIFILTER))
 		return -EOPNOTSUPP;
 
 	nlmsg_for_each_attr_type(attr, VXLAN_VNIFILTER_ENTRY, nlh,
diff --git a/drivers/net/wan/slic_ds26522.c b/drivers/net/wan/slic_ds26522.c
index cbb99fc5ea9fe7..c57e7cb98410ae 100644
--- a/drivers/net/wan/slic_ds26522.c
+++ b/drivers/net/wan/slic_ds26522.c
@@ -219,7 +219,7 @@ static int slic_ds26522_probe(struct spi_device *spi)
 
 static const struct spi_device_id slic_ds26522_id[] = {
 	{ .name = "ds26522" },
-	{ /* sentinel */ },
+	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(spi, slic_ds26522_id);
 
diff --git a/drivers/net/wireless/ath/ath10k/Kconfig b/drivers/net/wireless/ath/ath10k/Kconfig
index efb9f022d8c662..eac3ec5a4e78f5 100644
--- a/drivers/net/wireless/ath/ath10k/Kconfig
+++ b/drivers/net/wireless/ath/ath10k/Kconfig
@@ -17,7 +17,7 @@ config ATH10K_CE
 
 config ATH10K_PCI
 	tristate "Atheros ath10k PCI support"
-	depends on ATH10K && PCI
+	depends on ATH10K && PCI && PCIEASPM
 	help
 	  This module adds support for PCIE bus
 
diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c
index fb61e53ff4c347..e11eb7a6cdf26a 100644
--- a/drivers/net/wireless/ath/ath10k/debug.c
+++ b/drivers/net/wireless/ath/ath10k/debug.c
@@ -2004,10 +2004,7 @@ static ssize_t ath10k_write_btcoex(struct file *file,
 		ath10k_core_start_recovery(ar);
 	}
 
-	if (val)
-		set_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags);
-	else
-		clear_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags);
+	assign_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags, val);
 
 	ret = count;
 
@@ -2118,10 +2115,7 @@ static ssize_t ath10k_write_peer_stats(struct file *file,
 		goto exit;
 	}
 
-	if (val)
-		set_bit(ATH10K_FLAG_PEER_STATS, &ar->dev_flags);
-	else
-		clear_bit(ATH10K_FLAG_PEER_STATS, &ar->dev_flags);
+	assign_bit(ATH10K_FLAG_PEER_STATS, &ar->dev_flags, val);
 
 	ath10k_info(ar, "restarting firmware due to Peer stats change");
 
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index 335bc7c488e419..2e27a70065ccec 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -1965,9 +1965,7 @@ static int ath10k_pci_hif_start(struct ath10k *ar)
 	ath10k_pci_irq_enable(ar);
 	ath10k_pci_rx_post(ar);
 
-	pcie_capability_clear_and_set_word(ar_pci->pdev, PCI_EXP_LNKCTL,
-					   PCI_EXP_LNKCTL_ASPMC,
-					   ar_pci->link_ctl & PCI_EXP_LNKCTL_ASPMC);
+	pci_force_enable_link_state(ar_pci->pdev, ar_pci->aspm_states);
 
 	return 0;
 }
@@ -2822,10 +2820,9 @@ static int ath10k_pci_hif_power_up(struct ath10k *ar,
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT, "boot hif power up\n");
 
-	pcie_capability_read_word(ar_pci->pdev, PCI_EXP_LNKCTL,
-				  &ar_pci->link_ctl);
-	pcie_capability_clear_word(ar_pci->pdev, PCI_EXP_LNKCTL,
-				   PCI_EXP_LNKCTL_ASPMC);
+	ar_pci->aspm_states = pcie_aspm_enabled(ar_pci->pdev);
+
+	pci_disable_link_state(ar_pci->pdev, PCIE_LINK_STATE_ASPM_ALL);
 
 	/*
 	 * Bring the target up cleanly.
diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h
index d114778edb4173..122e365b7cdd84 100644
--- a/drivers/net/wireless/ath/ath10k/pci.h
+++ b/drivers/net/wireless/ath/ath10k/pci.h
@@ -128,10 +128,9 @@ struct ath10k_pci {
 	struct timer_list rx_post_retry;
 
 	/* Due to HW quirks it is recommended to disable ASPM during device
-	 * bootup. To do that the original PCI-E Link Control is stored before
-	 * device bootup is executed and re-programmed later.
+	 * bootup. To do that the ASPM states are saved and re-programmed later.
 	 */
-	u16 link_ctl;
+	u32 aspm_states;
 
 	/* Protects ps_awake and ps_wake_refcount */
 	spinlock_t ps_lock;
diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c
index 8de133fe2a72f7..8f171ab6f4de85 100644
--- a/drivers/net/wireless/ath/ath10k/snoc.c
+++ b/drivers/net/wireless/ath/ath10k/snoc.c
@@ -1532,10 +1532,8 @@ static int ath10k_snoc_modem_notify(struct notifier_block *nb, unsigned long act
 	case QCOM_SSR_BEFORE_SHUTDOWN:
 		ath10k_dbg(ar, ATH10K_DBG_SNOC, "received modem %s event\n",
 			   notify_data->crashed ? "crashed" : "stopping");
-		if (!notify_data->crashed)
-			set_bit(ATH10K_SNOC_FLAG_MODEM_STOPPED, &ar_snoc->flags);
-		else
-			clear_bit(ATH10K_SNOC_FLAG_MODEM_STOPPED, &ar_snoc->flags);
+		assign_bit(ATH10K_SNOC_FLAG_MODEM_STOPPED, &ar_snoc->flags,
+			   !notify_data->crashed);
 		break;
 
 	case QCOM_SSR_AFTER_SHUTDOWN:
diff --git a/drivers/net/wireless/ath/ath11k/Kconfig b/drivers/net/wireless/ath/ath11k/Kconfig
index 122726f84492a3..ac398246d530a4 100644
--- a/drivers/net/wireless/ath/ath11k/Kconfig
+++ b/drivers/net/wireless/ath/ath11k/Kconfig
@@ -14,12 +14,15 @@ config ATH11K_AHB
 	tristate "Atheros ath11k AHB support"
 	depends on ATH11K
 	depends on REMOTEPROC
+	select RPMSG
+	select QRTR
+	select QRTR_SMD
 	help
 	  This module adds support for AHB bus
 
 config ATH11K_PCI
 	tristate "Atheros ath11k PCI support"
-	depends on ATH11K && PCI
+	depends on ATH11K && PCI && PCIEASPM
 	select MHI_BUS
 	select QRTR
 	select QRTR_MHI
diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c
index ec5bf5c9fd7928..27d01411b6e9c5 100644
--- a/drivers/net/wireless/ath/ath11k/ahb.c
+++ b/drivers/net/wireless/ath/ath11k/ahb.c
@@ -391,12 +391,17 @@ static void ath11k_ahb_ext_irq_disable(struct ath11k_base *ab)
 	ath11k_ahb_sync_ext_irqs(ab);
 }
 
-static void ath11k_ahb_stop(struct ath11k_base *ab)
+static void ath11k_ahb_ce_irq_disable_sync(struct ath11k_base *ab)
 {
 	if (!test_bit(ATH11K_FLAG_CRASH_FLUSH, &ab->dev_flags))
 		ath11k_ahb_ce_irqs_disable(ab);
 	ath11k_ahb_sync_ce_irqs(ab);
 	ath11k_ahb_kill_tasklets(ab);
+}
+
+static void ath11k_ahb_stop(struct ath11k_base *ab)
+{
+	ath11k_ahb_ce_irq_disable_sync(ab);
 	timer_delete_sync(&ab->rx_replenish_retry);
 	ath11k_ce_cleanup_pipes(ab);
 }
@@ -804,6 +809,8 @@ static const struct ath11k_hif_ops ath11k_ahb_hif_ops_ipq8074 = {
 	.map_service_to_pipe = ath11k_ahb_map_service_to_pipe,
 	.power_down = ath11k_ahb_power_down,
 	.power_up = ath11k_ahb_power_up,
+	.ce_irq_enable = ath11k_ahb_ce_irqs_enable,
+	.ce_irq_disable = ath11k_ahb_ce_irq_disable_sync,
 };
 
 static const struct ath11k_hif_ops ath11k_ahb_hif_ops_wcn6750 = {
@@ -1296,14 +1303,16 @@ static void ath11k_ahb_remove(struct platform_device *pdev)
 {
 	struct ath11k_base *ab = platform_get_drvdata(pdev);
 
-	if (test_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags)) {
+	ath11k_ahb_remove_prepare(ab);
+
+	if (!test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) {
 		ath11k_ahb_power_down(ab, false);
 		ath11k_debugfs_soc_destroy(ab);
 		ath11k_qmi_deinit_service(ab);
+		ath11k_core_pm_notifier_unregister(ab);
 		goto qmi_fail;
 	}
 
-	ath11k_ahb_remove_prepare(ab);
 	ath11k_core_deinit(ab);
 
 qmi_fail:
@@ -1343,5 +1352,6 @@ static struct platform_driver ath11k_ahb_driver = {
 
 module_platform_driver(ath11k_ahb_driver);
 
+MODULE_SOFTDEP("pre: qrtr_smd");
 MODULE_DESCRIPTION("Driver support for Qualcomm Technologies 802.11ax WLAN AHB devices");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/ath/ath11k/ce.c b/drivers/net/wireless/ath/ath11k/ce.c
index c56dd8a8676065..1e08b0970b148c 100644
--- a/drivers/net/wireless/ath/ath11k/ce.c
+++ b/drivers/net/wireless/ath/ath11k/ce.c
@@ -325,7 +325,7 @@ static int ath11k_ce_rx_post_pipe(struct ath11k_ce_pipe *pipe)
 	dma_addr_t paddr;
 	int ret = 0;
 
-	if (!(pipe->dest_ring || pipe->status_ring))
+	if (!pipe->dest_ring || !pipe->status_ring)
 		return 0;
 
 	spin_lock_bh(&ab->ce.ce_lock);
diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c
index 8039124e783275..d2ed6a0ea7e362 100644
--- a/drivers/net/wireless/ath/ath11k/core.c
+++ b/drivers/net/wireless/ath/ath11k/core.c
@@ -2334,6 +2334,16 @@ static int ath11k_core_reconfigure_on_crash(struct ath11k_base *ab)
 
 	mutex_lock(&ab->core_lock);
 	ath11k_thermal_unregister(ab);
+
+	/*
+	 * ath11k_core_reset() already disabled the interrupts on the reset
+	 * path; only the firmware crash path reaches here with them live.
+	 */
+	if (!ab->is_reset) {
+		ath11k_hif_irq_disable(ab);
+		ath11k_hif_ce_irq_disable(ab);
+	}
+
 	ath11k_dp_pdev_free(ab);
 	ath11k_cfr_deinit(ab);
 	ath11k_spectral_deinit(ab);
diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c
index f389b97acbdd5d..be65d73b904b61 100644
--- a/drivers/net/wireless/ath/ath11k/dp.c
+++ b/drivers/net/wireless/ath/ath11k/dp.c
@@ -313,6 +313,7 @@ int ath11k_dp_srng_setup(struct ath11k_base *ab, struct dp_srng *ring,
 	case HAL_RXDMA_DIR_BUF:
 		break;
 	default:
+		ath11k_dp_srng_cleanup(ab, ring);
 		ath11k_warn(ab, "Not a valid ring type in dp :%d\n", type);
 		return -EINVAL;
 	}
@@ -324,6 +325,7 @@ int ath11k_dp_srng_setup(struct ath11k_base *ab, struct dp_srng *ring,
 
 	ret = ath11k_hal_srng_setup(ab, type, ring_num, mac_id, &params);
 	if (ret < 0) {
+		ath11k_dp_srng_cleanup(ab, ring);
 		ath11k_warn(ab, "failed to setup srng: %d ring_id %d\n",
 			    ret, ring_num);
 		return ret;
diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
index 33425707c08426..6d95f7b343a55c 100644
--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
@@ -762,6 +762,9 @@ static void ath11k_dp_rx_tid_del_func(struct ath11k_dp *dp, void *ctx,
 	struct ath11k_base *ab = dp->ab;
 	struct dp_rx_tid *rx_tid = ctx;
 	struct dp_reo_cache_flush_elem *elem, *tmp;
+	struct list_head flush_list;
+
+	INIT_LIST_HEAD(&flush_list);
 
 	if (status == HAL_REO_CMD_DRAIN) {
 		goto free_desc;
@@ -783,23 +786,25 @@ static void ath11k_dp_rx_tid_del_func(struct ath11k_dp *dp, void *ctx,
 	list_add_tail(&elem->list, &dp->reo_cmd_cache_flush_list);
 	dp->reo_cmd_cache_flush_count++;
 
-	/* Flush and invalidate aged REO desc from HW cache */
+	/* identify aged REO desc that needs removal */
 	list_for_each_entry_safe(elem, tmp, &dp->reo_cmd_cache_flush_list,
 				 list) {
 		if (dp->reo_cmd_cache_flush_count > DP_REO_DESC_FREE_THRESHOLD ||
 		    time_after(jiffies, elem->ts +
 			       msecs_to_jiffies(DP_REO_DESC_FREE_TIMEOUT_MS))) {
-			list_del(&elem->list);
+			list_move_tail(&elem->list, &flush_list);
 			dp->reo_cmd_cache_flush_count--;
-			spin_unlock_bh(&dp->reo_cmd_lock);
-
-			ath11k_dp_reo_cache_flush(ab, &elem->data);
-			kfree(elem);
-			spin_lock_bh(&dp->reo_cmd_lock);
 		}
 	}
 	spin_unlock_bh(&dp->reo_cmd_lock);
 
+	/* remove aged REO desc from HW */
+	list_for_each_entry_safe(elem, tmp, &flush_list, list) {
+		ath11k_dp_reo_cache_flush(ab, &elem->data);
+		list_del(&elem->list);
+		kfree(elem);
+	}
+
 	return;
 free_desc:
 	dma_free_noncoherent(ab->dev, rx_tid->unaligned_size,
@@ -2500,7 +2505,7 @@ static void ath11k_dp_rx_deliver_msdu(struct ath11k *ar, struct napi_struct *nap
 	    !(is_mcbc && rx_status->flag & RX_FLAG_DECRYPTED))
 		rx_status->flag |= RX_FLAG_8023;
 
-	ieee80211_rx_napi(ar->hw, pubsta, msdu, napi);
+	ieee80211_rx_napi(ar->hw, pubsta ? &pubsta->deflink : NULL, msdu, napi);
 }
 
 static bool ath11k_dp_rx_check_nwifi_hdr_len_valid(struct ath11k_base *ab,
diff --git a/drivers/net/wireless/ath/ath11k/htc.c b/drivers/net/wireless/ath/ath11k/htc.c
index 5f9ee5e633da84..07fc959ed74c99 100644
--- a/drivers/net/wireless/ath/ath11k/htc.c
+++ b/drivers/net/wireless/ath/ath11k/htc.c
@@ -244,10 +244,7 @@ static void ath11k_htc_suspend_complete(struct ath11k_base *ab, bool ack)
 {
 	ath11k_dbg(ab, ATH11K_DBG_BOOT, "suspend complete %d\n", ack);
 
-	if (ack)
-		set_bit(ATH11K_FLAG_HTC_SUSPEND_COMPLETE, &ab->dev_flags);
-	else
-		clear_bit(ATH11K_FLAG_HTC_SUSPEND_COMPLETE, &ab->dev_flags);
+	assign_bit(ATH11K_FLAG_HTC_SUSPEND_COMPLETE, &ab->dev_flags, ack);
 
 	complete(&ab->htc_suspend);
 }
diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c
index ae91b57c8422f1..c1fa42edd3630a 100644
--- a/drivers/net/wireless/ath/ath11k/mac.c
+++ b/drivers/net/wireless/ath/ath11k/mac.c
@@ -6716,6 +6716,7 @@ static int ath11k_mac_op_start(struct ieee80211_hw *hw)
 	ar->num_started_vdevs = 0;
 	ar->num_created_vdevs = 0;
 	ar->num_peers = 0;
+	ar->num_stations = 0;
 	ar->allocated_vdev_map = 0;
 
 	/* Configure monitor status ring with default rx_filter to get rx status
diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c
index a163168f361764..32dbb844a47b88 100644
--- a/drivers/net/wireless/ath/ath11k/pci.c
+++ b/drivers/net/wireless/ath/ath11k/pci.c
@@ -608,19 +608,9 @@ static void ath11k_pci_free_region(struct ath11k_pci *ab_pci)
 
 static void ath11k_pci_aspm_disable(struct ath11k_pci *ab_pci)
 {
-	struct ath11k_base *ab = ab_pci->ab;
-
-	pcie_capability_read_word(ab_pci->pdev, PCI_EXP_LNKCTL,
-				  &ab_pci->link_ctl);
-
-	ath11k_dbg(ab, ATH11K_DBG_PCI, "link_ctl 0x%04x L0s %d L1 %d\n",
-		   ab_pci->link_ctl,
-		   u16_get_bits(ab_pci->link_ctl, PCI_EXP_LNKCTL_ASPM_L0S),
-		   u16_get_bits(ab_pci->link_ctl, PCI_EXP_LNKCTL_ASPM_L1));
+	ab_pci->aspm_states = pcie_aspm_enabled(ab_pci->pdev);
 
-	/* disable L0s and L1 */
-	pcie_capability_clear_word(ab_pci->pdev, PCI_EXP_LNKCTL,
-				   PCI_EXP_LNKCTL_ASPMC);
+	pci_disable_link_state(ab_pci->pdev, PCIE_LINK_STATE_ASPM_ALL);
 
 	set_bit(ATH11K_PCI_ASPM_RESTORE, &ab_pci->flags);
 }
@@ -628,10 +618,7 @@ static void ath11k_pci_aspm_disable(struct ath11k_pci *ab_pci)
 static void ath11k_pci_aspm_restore(struct ath11k_pci *ab_pci)
 {
 	if (test_and_clear_bit(ATH11K_PCI_ASPM_RESTORE, &ab_pci->flags))
-		pcie_capability_clear_and_set_word(ab_pci->pdev, PCI_EXP_LNKCTL,
-						   PCI_EXP_LNKCTL_ASPMC,
-						   ab_pci->link_ctl &
-						   PCI_EXP_LNKCTL_ASPMC);
+		pci_force_enable_link_state(ab_pci->pdev, ab_pci->aspm_states);
 }
 
 #ifdef CONFIG_DEV_COREDUMP
diff --git a/drivers/net/wireless/ath/ath11k/pci.h b/drivers/net/wireless/ath/ath11k/pci.h
index 1e3005a4b64c68..474c1874141871 100644
--- a/drivers/net/wireless/ath/ath11k/pci.h
+++ b/drivers/net/wireless/ath/ath11k/pci.h
@@ -72,7 +72,8 @@ struct ath11k_pci {
 
 	/* enum ath11k_pci_flags */
 	unsigned long flags;
-	u16 link_ctl;
+	/* Cached PCIe ASPM states */
+	u32 aspm_states;
 	u64 dma_mask;
 };
 
diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c
index bbca275a8289a3..08fd6795ed9e71 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.c
+++ b/drivers/net/wireless/ath/ath11k/wmi.c
@@ -5136,6 +5136,15 @@ static int ath11k_service_ready_ext_event(struct ath11k_base *ab,
 
 err:
 	kfree(svc_rdy_ext.mac_phy_caps);
+
+	/*
+	 * During TLV iteration the WMI_TAG_SOC_HAL_REG_CAPABILITIES
+	 * handler may have allocated ab->reg_info_store, so free it
+	 * on error to avoid a memory leak
+	 */
+	kfree(ab->reg_info_store);
+	ab->reg_info_store = NULL;
+
 	ath11k_wmi_free_dbring_caps(ab);
 	return ret;
 }
diff --git a/drivers/net/wireless/ath/ath12k/Kconfig b/drivers/net/wireless/ath/ath12k/Kconfig
index 0d5d1c55bfc136..4662d38c88c0cc 100644
--- a/drivers/net/wireless/ath/ath12k/Kconfig
+++ b/drivers/net/wireless/ath/ath12k/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 config ATH12K
 	tristate "Qualcomm Wi-Fi 7 support (ath12k)"
-	depends on MAC80211 && HAS_DMA && PCI
+	depends on MAC80211 && HAS_DMA && PCI && PCIEASPM
 	select QCOM_QMI_HELPERS
 	select MHI_BUS
 	select QRTR
diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c
index d89a49c6ebb742..85360365aa4b22 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.c
+++ b/drivers/net/wireless/ath/ath12k/ahb.c
@@ -347,35 +347,60 @@ static void ath12k_ahb_stop(struct ath12k_base *ab)
 	ath12k_ce_cleanup_pipes(ab);
 }
 
+static int ath12k_ahb_get_fw_load_region(struct ath12k_base *ab)
+{
+	struct ath12k_ahb *ab_ahb = ath12k_ab_to_ahb(ab);
+	struct ath12k_ahb_rproc_info *rproc_info = ab_ahb->rproc_info;
+	struct device *dev = ab->dev;
+	struct resource res;
+	int ret;
+
+	if (rproc_info->mem_region)
+		return 0;
+
+	ret = of_reserved_mem_region_to_resource_byname(dev->of_node, "q6-region", &res);
+	if (ret)
+		return ret;
+
+	rproc_info->mem_phys = res.start;
+	rproc_info->mem_size = resource_size(&res);
+	rproc_info->mem_region = memremap(rproc_info->mem_phys, rproc_info->mem_size,
+					  MEMREMAP_WC);
+	if (!rproc_info->mem_region) {
+		ath12k_err(ab, "unable to map memory region: %pa+%zx\n",
+			   &res.start, rproc_info->mem_size);
+		rproc_info->mem_phys = 0;
+		rproc_info->mem_size = 0;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void ath12k_ahb_put_fw_load_region(struct ath12k_ahb_rproc_info *rproc_info)
+{
+	memunmap(rproc_info->mem_region);
+	rproc_info->mem_region = NULL;
+	rproc_info->mem_phys = 0;
+	rproc_info->mem_size = 0;
+}
+
 static int ath12k_ahb_power_up(struct ath12k_base *ab)
 {
 	struct ath12k_ahb *ab_ahb = ath12k_ab_to_ahb(ab);
+	struct ath12k_ahb_rproc_info *rproc_info = ab_ahb->rproc_info;
 	char fw_name[ATH12K_USERPD_FW_NAME_LEN];
 	char fw2_name[ATH12K_USERPD_FW_NAME_LEN];
 	struct device *dev = ab->dev;
 	const struct firmware *fw, *fw2;
 	unsigned long time_left;
-	phys_addr_t mem_phys;
-	struct resource res;
-	void *mem_region;
-	size_t mem_size;
 	u32 pasid;
 	int ret;
 
-	ret = of_reserved_mem_region_to_resource_byname(dev->of_node, "q6-region",
-							&res);
+	ret = ath12k_ahb_get_fw_load_region(ab);
 	if (ret)
 		return ret;
 
-	mem_phys = res.start;
-	mem_size = resource_size(&res);
-	mem_region = devm_memremap(dev, mem_phys, mem_size, MEMREMAP_WC);
-	if (IS_ERR(mem_region)) {
-		ath12k_err(ab, "unable to map memory region: %pa+%zx\n",
-			   &res.start, mem_size);
-		return PTR_ERR(mem_region);
-	}
-
 	snprintf(fw_name, sizeof(fw_name), "%s/%s/%s%d%s", ATH12K_FW_DIR,
 		 ab->hw_params->fw.dir, ATH12K_AHB_FW_PREFIX, ab_ahb->userpd_id,
 		 ATH12K_AHB_FW_SUFFIX);
@@ -400,11 +425,13 @@ static int ath12k_ahb_power_up(struct ath12k_base *ab)
 
 	/* Load FW image to a reserved memory location */
 	if (ab_ahb->scm_auth_enabled)
-		ret = qcom_mdt_load(dev, fw, fw_name, pasid, mem_region,
-				    mem_phys, mem_size, &mem_phys);
+		ret = qcom_mdt_load(dev, fw, fw_name, pasid, rproc_info->mem_region,
+				    rproc_info->mem_phys, rproc_info->mem_size,
+				    NULL);
 	else
-		ret = qcom_mdt_load_no_init(dev, fw, fw_name, mem_region,
-					    mem_phys, mem_size, &mem_phys);
+		ret = qcom_mdt_load_no_init(dev, fw, fw_name, rproc_info->mem_region,
+					    rproc_info->mem_phys, rproc_info->mem_size,
+					    NULL);
 	if (ret) {
 		ath12k_err(ab, "Failed to load MDT segments: %d\n", ret);
 		goto err_fw;
@@ -428,8 +455,9 @@ static int ath12k_ahb_power_up(struct ath12k_base *ab)
 		goto err_fw2;
 	}
 
-	ret = qcom_mdt_load_no_init(dev, fw2, fw2_name, mem_region, mem_phys,
-				    mem_size, &mem_phys);
+	ret = qcom_mdt_load_no_init(dev, fw2, fw2_name, rproc_info->mem_region,
+				    rproc_info->mem_phys, rproc_info->mem_size,
+				    NULL);
 	if (ret) {
 		ath12k_err(ab, "Failed to load MDT segments: %d\n", ret);
 		goto err_fw2;
@@ -877,6 +905,7 @@ static struct ath12k_ahb_rproc_info *ath12k_ahb_rproc_info_alloc(struct ath12k_b
 	rproc_info->rootpd_booted_by_driver = false;
 	rproc_info->userpd[ab_ahb->userpd_id - 1] = ab_ahb;
 	rproc_info->num_userpd = 1;
+	rproc_info->shared_fw_loaded = false;
 	init_completion(&rproc_info->rootpd_ready);
 	ab_ahb->rproc_info = rproc_info;
 
@@ -961,6 +990,76 @@ static int ath12k_ahb_boot_root_pd(struct ath12k_base *ab)
 	return 0;
 }
 
+static int ath12k_ahb_load_auth_shared_fw(struct ath12k_base *ab,
+					  struct ath12k_ahb_rproc_info *rproc_info,
+					  const char *fw_name, u32 pasid)
+{
+	int ret;
+
+	const struct firmware *fw __free(firmware) = NULL;
+	ret = request_firmware(&fw, fw_name, ab->dev);
+	if (ret) {
+		ath12k_err(ab, "failed to request shared firmware %s: %d\n",
+			   fw_name, ret);
+		return ret;
+	}
+
+	if (!fw->size) {
+		ath12k_err(ab, "Invalid firmware size\n");
+		return -EINVAL;
+	}
+
+	ath12k_dbg(ab, ATH12K_DBG_AHB, "loading firmware %s, size %zd\n", fw_name,
+		   fw->size);
+
+	ret = qcom_mdt_load(ab->dev, fw, fw_name, pasid, rproc_info->mem_region,
+			    rproc_info->mem_phys, rproc_info->mem_size, NULL);
+	if (ret) {
+		ath12k_err(ab, "failed to load RO firmware %s: %d\n", fw_name, ret);
+		return ret;
+	}
+
+	ret = qcom_pas_auth_and_reset(pasid);
+	if (ret)
+		ath12k_err(ab, "failed to authenticate and boot shared firmware: %d\n",
+			   ret);
+	return ret;
+}
+
+static int ath12k_ahb_load_shared_firmware(struct ath12k_base *ab)
+{
+	struct ath12k_ahb *ab_ahb = ath12k_ab_to_ahb(ab);
+	struct ath12k_ahb_rproc_info *rproc_info = ab_ahb->rproc_info;
+	char fw_name[ATH12K_USERPD_FW_NAME_LEN];
+	u32 pasid;
+	int ret;
+
+	lockdep_assert_held(&ath12k_rproc_info_lock);
+
+	if (!ab_ahb->supports_multipd)
+		return 0;
+
+	ret = ath12k_ahb_get_fw_load_region(ab);
+	if (ret)
+		return ret;
+
+	snprintf(fw_name, sizeof(fw_name), "%s/%s/%s%d%s", ATH12K_FW_DIR,
+		 ab->hw_params->fw.dir, ATH12K_AHB_FW_PREFIX, ATH12K_AHB_RO_ID,
+		 ATH12K_AHB_FW_SUFFIX);
+
+	pasid = u32_encode_bits(ATH12K_AHB_RO_ID, ATH12K_USERPD_ID_MASK) |
+		ATH12K_AHB_UPD_SWID;
+
+	ret = ath12k_ahb_load_auth_shared_fw(ab, rproc_info, fw_name, pasid);
+	if (ret) {
+		ath12k_ahb_put_fw_load_region(rproc_info);
+		return ret;
+	}
+
+	rproc_info->shared_fw_loaded = true;
+	return 0;
+}
+
 static int ath12k_ahb_configure_rproc(struct ath12k_base *ab)
 {
 	int ret;
@@ -989,6 +1088,15 @@ static int ath12k_ahb_configure_rproc(struct ath12k_base *ab)
 		g_rproc_info->rootpd_booted_by_driver = true;
 	}
 
+	if (!g_rproc_info->shared_fw_loaded) {
+		ret = ath12k_ahb_load_shared_firmware(ab);
+		if (ret) {
+			if (g_rproc_info->rootpd_booted_by_driver)
+				rproc_shutdown(g_rproc_info->tgt_rproc);
+			goto err_unreg_notifier;
+		}
+	}
+
 	mutex_unlock(&ath12k_rproc_info_lock);
 	return 0;
 
@@ -1012,6 +1120,8 @@ static void ath12k_ahb_deconfigure_rproc(struct ath12k_base *ab)
 {
 	struct ath12k_ahb *ab_ahb = ath12k_ab_to_ahb(ab);
 	struct ath12k_ahb_rproc_info *rproc_info = ab_ahb->rproc_info;
+	u32 pasid;
+	int ret;
 
 	lockdep_assert_held(&ath12k_rproc_info_lock);
 
@@ -1023,6 +1133,20 @@ static void ath12k_ahb_deconfigure_rproc(struct ath12k_base *ab)
 	if (!g_rproc_info->num_userpd) {
 		ath12k_ahb_unregister_rproc_notifier();
 
+		if (g_rproc_info->shared_fw_loaded) {
+			pasid = u32_encode_bits(ATH12K_AHB_RO_ID, ATH12K_USERPD_ID_MASK) |
+				ATH12K_AHB_UPD_SWID;
+			ret = qcom_pas_shutdown(pasid);
+			if (ret)
+				ath12k_err(ab, "pas shutdown failed for shared firmware: %d\n",
+					   ret);
+
+			g_rproc_info->shared_fw_loaded = false;
+		}
+
+		if (g_rproc_info->mem_region)
+			ath12k_ahb_put_fw_load_region(g_rproc_info);
+
 		if (g_rproc_info->rootpd_booted_by_driver &&
 		    g_rproc_info->tgt_rproc->state == RPROC_RUNNING)
 			rproc_shutdown(g_rproc_info->tgt_rproc);
diff --git a/drivers/net/wireless/ath/ath12k/ahb.h b/drivers/net/wireless/ath/ath12k/ahb.h
index cdb58b07338fd8..46a4420d41bdc2 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.h
+++ b/drivers/net/wireless/ath/ath12k/ahb.h
@@ -30,6 +30,9 @@
 #define ATH12K_USERPD_ID_MASK			GENMASK(10, 8)
 #define ATH12K_USERPD_FW_NAME_LEN		35
 
+/* Shared read-only firmware ID, mandated by MultiPD firmware architecture */
+#define ATH12K_AHB_RO_ID			4
+
 enum ath12k_ahb_userpd_id {
 	ATH12K_AHB_USERPD_ID_0 = 1,
 	ATH12K_AHB_USERPD_ID_1,
@@ -46,6 +49,7 @@ struct ath12k_ahb_desc {
 	enum ath12k_hw_rev hw_rev;
 	bool auth_enabled;
 	const struct ath12k_hif_ops *ops;
+	bool supports_multipd;
 };
 
 enum ath12k_ahb_smp2p_msg_id {
@@ -77,6 +81,10 @@ struct ath12k_ahb_rproc_info {
 	u8 num_userpd;
 	bool rootpd_booted_by_driver;
 	struct ath12k_ahb *userpd[ATH12K_MAX_DEVICES];
+	bool shared_fw_loaded;
+	phys_addr_t mem_phys;
+	size_t mem_size;
+	void *mem_region;
 };
 
 struct ath12k_ahb {
@@ -95,6 +103,7 @@ struct ath12k_ahb {
 	const struct ath12k_ahb_device_family_ops *device_family_ops;
 	bool scm_auth_enabled;
 	struct ath12k_ahb_rproc_info *rproc_info;
+	bool supports_multipd;
 };
 
 struct ath12k_ahb_driver {
diff --git a/drivers/net/wireless/ath/ath12k/ce.c b/drivers/net/wireless/ath/ath12k/ce.c
index 67bbc6d8d0c914..c99e915e3885bb 100644
--- a/drivers/net/wireless/ath/ath12k/ce.c
+++ b/drivers/net/wireless/ath/ath12k/ce.c
@@ -269,7 +269,7 @@ static void ath12k_ce_srng_msi_ring_params_setup(struct ath12k_base *ab, u32 ce_
 	ath12k_hif_get_ce_msi_idx(ab, ce_id, &msi_data_idx);
 
 	ring_params->msi_addr = addr_lo;
-	ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32);
+	ring_params->msi_addr |= (dma_addr_t)(((u64)addr_hi) << 32);
 	ring_params->msi_data = (msi_data_idx % msi_data_count) + msi_data_start;
 	ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR;
 }
diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c
index 262a2045309b1f..0c72948f3f56b6 100644
--- a/drivers/net/wireless/ath/ath12k/core.c
+++ b/drivers/net/wireless/ath/ath12k/core.c
@@ -792,6 +792,29 @@ int ath12k_core_check_smbios(struct ath12k_base *ab)
 	return 0;
 }
 
+int ath12k_core_check_dt(struct ath12k_base *ab)
+{
+	size_t max_len = sizeof(ab->qmi.target.bdf_ext);
+	const char *variant = NULL;
+	struct device_node *node;
+
+	node = ab->dev->of_node;
+	if (!node)
+		return -ENOENT;
+
+	of_property_read_string(node, "qcom,calibration-variant",
+				&variant);
+	if (!variant)
+		return -ENODATA;
+
+	if (strscpy(ab->qmi.target.bdf_ext, variant, max_len) < 0)
+		ath12k_dbg(ab, ATH12K_DBG_BOOT,
+			   "bdf variant string is longer than the buffer can accommodate (variant: %s)\n",
+			   variant);
+
+	return 0;
+}
+
 static int ath12k_core_soc_create(struct ath12k_base *ab)
 {
 	int ret;
diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h
index a98fc6e0699d76..4718da494aab13 100644
--- a/drivers/net/wireless/ath/ath12k/core.h
+++ b/drivers/net/wireless/ath/ath12k/core.h
@@ -928,15 +928,6 @@ struct ath12k_wsi_info {
 	u32 hw_link_id_base;
 };
 
-struct ath12k_dp_profile_params {
-	u32 tx_comp_ring_size;
-	u32 rxdma_monitor_buf_ring_size;
-	u32 rxdma_monitor_dst_ring_size;
-	u32 num_pool_tx_desc;
-	u32 rx_desc_count;
-	u32 rx_release_ring_size;
-};
-
 struct ath12k_mem_profile_based_param {
 	u32 num_vdevs;
 	u32 max_client_single;
diff --git a/drivers/net/wireless/ath/ath12k/debugfs.c b/drivers/net/wireless/ath/ath12k/debugfs.c
index d54995b7adb278..12791e267d1a8b 100644
--- a/drivers/net/wireless/ath/ath12k/debugfs.c
+++ b/drivers/net/wireless/ath/ath12k/debugfs.c
@@ -1071,6 +1071,20 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 		[HAL_REO_DEST_RING_ERROR_CODE_PN_ERR_FLAG_SET] = "PN err",
 		[HAL_REO_DEST_RING_ERROR_CODE_DESC_BLOCKED] = "Desc blocked"};
 
+	static const char *wbm_rx_drop[WBM_ERR_DROP_MAX] = {
+		[WBM_ERR_DROP_GET_SW_DESC] = "SW desc error",
+		[WBM_ERR_DROP_DESC_PARSE] = "Desc parse error",
+		[WBM_ERR_DROP_INV_HW_ID] = "Invalid hw id",
+		[WBM_ERR_DROP_NULL_PRTNR_DP] = "Null Partner dp",
+		[WBM_ERR_DROP_NULL_PROC_DP] = "Process Null Partner dp",
+		[WBM_ERR_DROP_NULL_PDEV] = "Null Pdev",
+		[WBM_ERR_DROP_NULL_AR] = "Null ar",
+		[WBM_ERR_DROP_CAC_RUNNING] = "CAC Running",
+		[WBM_ERR_DROP_SG] = "Scatter Gather",
+		[WBM_ERR_DROP_INV_NWIFI_HDR] = "Invalid NWifi Hdr len",
+		[WBM_ERR_DROP_REO_GENERIC] = "REO Generic",
+		[WBM_ERR_DROP_RXDMA_GENERIC] = "RXDMA Generic"};
+
 	static const char *wbm_rel_src[HAL_WBM_REL_SRC_MODULE_MAX] = {
 		[HAL_WBM_REL_SRC_MODULE_TQM] = "TQM",
 		[HAL_WBM_REL_SRC_MODULE_RXDMA] = "Rxdma",
@@ -1095,13 +1109,32 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 
 	for (i = 0; i < HAL_REO_ENTR_RING_RXDMA_ECODE_MAX; i++)
 		len += scnprintf(buf + len, size - len, "%s: %u\n",
-				 rxdma_err[i], device_stats->rxdma_error[i]);
+				 rxdma_err[i], device_stats->wbm_err.rxdma_error[i]);
 
 	len += scnprintf(buf + len, size - len, "\nREO errors:\n");
 
 	for (i = 0; i < HAL_REO_DEST_RING_ERROR_CODE_MAX; i++)
 		len += scnprintf(buf + len, size - len, "%s: %u\n",
-				 reo_err[i], device_stats->reo_error[i]);
+				 reo_err[i], device_stats->wbm_err.reo_error[i]);
+
+	len += scnprintf(buf + len, size - len, "\nWBM Rx Drop Count:\n");
+	for (i = 0; i < WBM_ERR_DROP_MAX; i++)
+		len += scnprintf(buf + len, size - len, "%s: %u\n",
+				 wbm_rx_drop[i], device_stats->wbm_err.drop[i]);
+
+	len += scnprintf(buf + len, size - len, "\nREO sent to stack:\n");
+	for (i = 0; i < DP_REO_DST_RING_MAX; i++) {
+		len += scnprintf(buf + len, size - len, "ring%d:", i);
+		for (j = 0; j < ab->ag->num_devices; j++)
+			len += scnprintf(buf + len, size - len,
+					 "\t%d:%u", j,
+					 device_stats->sent_to_stack[i][j]);
+		len += scnprintf(buf + len, size - len, "\n");
+	}
+
+	len += scnprintf(buf + len, size - len,
+			 "\nWBM SW desc fallback (HW CC not done): %u\n",
+			 device_stats->wbm_err.sw_desc_fallback);
 
 	len += scnprintf(buf + len, size - len, "\nHAL REO errors:\n");
 
@@ -1117,6 +1150,11 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 		len += scnprintf(buf + len, size - len, "ring%d: %u\n",
 				 i, device_stats->tx_err.desc_na[i]);
 
+	len += scnprintf(buf + len, size - len, "\nTX Descriptor Pool Alloc Failures:\n");
+	for (i = 0; i < ATH12K_HW_MAX_QUEUES; i++)
+		len += scnprintf(buf + len, size - len, "pool%d: %u\n",
+				 i, device_stats->tx_err.txbuf_na[i]);
+
 	len += scnprintf(buf + len, size - len,
 			 "\nMisc Transmit Failures: %d\n",
 			 atomic_read(&device_stats->tx_err.misc_fail));
@@ -1181,7 +1219,7 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 	len += scnprintf(buf + len, size - len, "\nREO Rx Received:\n");
 
 	for (i = 0; i < DP_REO_DST_RING_MAX; i++) {
-		len += scnprintf(buf + len, size - len, "Ring%d:", i + 1);
+		len += scnprintf(buf + len, size - len, "ring%d:", i);
 
 		for (j = 0; j < ab->ag->num_devices; j++) {
 			len += scnprintf(buf + len, size - len,
@@ -1215,8 +1253,40 @@ static ssize_t ath12k_debugfs_dump_device_dp_stats(struct file *file,
 	return ret;
 }
 
+static ssize_t
+ath12k_debugfs_write_device_dp_stats(struct file *file,
+				     const char __user *user_buf,
+				     size_t count, loff_t *ppos)
+{
+	struct ath12k_base *ab = file->private_data;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_device_dp_stats *device_stats = &dp->device_stats;
+	char buf[20] = {};
+	int ret;
+
+	/* filter partial writes and invalid commands */
+	if (*ppos != 0 || count >= sizeof(buf) || count == 0)
+		return -EINVAL;
+
+	ret = simple_write_to_buffer(buf, sizeof(buf) - 1, ppos, user_buf, count);
+	if (ret < 0)
+		return ret;
+
+	/* drop the possible '\n' from the end */
+	if (buf[*ppos - 1] == '\n')
+		buf[*ppos - 1] = '\0';
+
+	if (!strcmp(buf, "reset")) {
+		memset(device_stats, 0, sizeof(*device_stats));
+		return count;
+	}
+
+	return -EINVAL;
+}
+
 static const struct file_operations fops_device_dp_stats = {
 	.read = ath12k_debugfs_dump_device_dp_stats,
+	.write = ath12k_debugfs_write_device_dp_stats,
 	.open = simple_open,
 	.owner = THIS_MODULE,
 	.llseek = default_llseek,
@@ -1227,7 +1297,7 @@ void ath12k_debugfs_pdev_create(struct ath12k_base *ab)
 	debugfs_create_file("simulate_fw_crash", 0600, ab->debugfs_soc, ab,
 			    &fops_simulate_fw_crash);
 
-	debugfs_create_file("device_dp_stats", 0400, ab->debugfs_soc, ab,
+	debugfs_create_file("device_dp_stats", 0600, ab->debugfs_soc, ab,
 			    &fops_device_dp_stats);
 }
 
diff --git a/drivers/net/wireless/ath/ath12k/dp.c b/drivers/net/wireless/ath/ath12k/dp.c
index f9b37d75956d3d..83f9edfabf6965 100644
--- a/drivers/net/wireless/ath/ath12k/dp.c
+++ b/drivers/net/wireless/ath/ath12k/dp.c
@@ -224,7 +224,7 @@ static void ath12k_dp_srng_msi_setup(struct ath12k_base *ab,
 	ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi);
 
 	ring_params->msi_addr = addr_lo;
-	ring_params->msi_addr |= (dma_addr_t)(((uint64_t)addr_hi) << 32);
+	ring_params->msi_addr |= (dma_addr_t)(((u64)addr_hi) << 32);
 	ring_params->msi_data = (msi_group_number % msi_data_count)
 		+ msi_data_start;
 	ring_params->flags |= HAL_SRNG_FLAGS_MSI_INTR;
@@ -439,6 +439,7 @@ static void ath12k_dp_srng_common_cleanup(struct ath12k_base *ab)
 
 static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
 {
+	const struct ath12k_dp_profile_params *dp_params = &ab->profile_param->dp_params;
 	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	const struct ath12k_hal_tcl_to_wbm_rbm_map *map;
 	struct hal_srng *srng;
@@ -469,7 +470,7 @@ static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
 
 		ret = ath12k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring,
 					   HAL_WBM2SW_RELEASE, tx_comp_ring_num, 0,
-					   DP_TX_COMP_RING_SIZE(ab));
+					   ath12k_dp_tx_comp_ring_size(dp_params));
 		if (ret) {
 			ath12k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n",
 				    tx_comp_ring_num, ret);
@@ -487,7 +488,7 @@ static int ath12k_dp_srng_common_setup(struct ath12k_base *ab)
 
 	ret = ath12k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE,
 				   HAL_WBM2SW_REL_ERR_RING_NUM, 0,
-				   DP_RX_RELEASE_RING_SIZE(ab));
+				   ath12k_dp_rx_release_ring_size(dp_params));
 	if (ret) {
 		ath12k_warn(ab, "failed to set up rx_rel ring :%d\n", ret);
 		goto err;
@@ -966,14 +967,15 @@ void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif)
 
 static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
 {
-	struct ath12k_rx_desc_info *desc_info;
+	const struct ath12k_dp_profile_params *dp_params = &ab->profile_param->dp_params;
+	u32 pool_id, tx_spt_page, tx_spt_pages_per_pool, num_rx_spt_pages;
 	struct ath12k_tx_desc_info *tx_desc_info, *tmp1;
 	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	struct ath12k_rx_desc_info *desc_info;
 	struct ath12k_skb_cb *skb_cb;
 	struct sk_buff *skb;
 	struct ath12k *ar;
 	int i, j;
-	u32 pool_id, tx_spt_page;
 
 	if (!dp->spt_info)
 		return;
@@ -981,8 +983,10 @@ static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
 	/* RX Descriptor cleanup */
 	spin_lock_bh(&dp->rx_desc_lock);
 
+	num_rx_spt_pages = ath12k_dp_num_rx_spt_pages(dp_params);
+
 	if (dp->rxbaddr) {
-		for (i = 0; i < ATH12K_NUM_RX_SPT_PAGES(ab); i++) {
+		for (i = 0; i < num_rx_spt_pages; i++) {
 			if (!dp->rxbaddr[i])
 				continue;
 
@@ -998,10 +1002,11 @@ static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
 				if (!skb)
 					continue;
 
-				dma_unmap_single(ab->dev,
-						 ATH12K_SKB_RXCB(skb)->paddr,
-						 skb->len + skb_tailroom(skb),
-						 DMA_FROM_DEVICE);
+				if (!ATH12K_SKB_RXCB(skb)->unmapped)
+					dma_unmap_single(ab->dev,
+							 ATH12K_SKB_RXCB(skb)->paddr,
+							 skb->len + skb_tailroom(skb),
+							 DMA_FROM_DEVICE);
 				dev_kfree_skb_any(skb);
 			}
 
@@ -1055,12 +1060,13 @@ static void ath12k_dp_cc_cleanup(struct ath12k_base *ab)
 	}
 
 	if (dp->txbaddr) {
+		tx_spt_pages_per_pool = ath12k_dp_tx_spt_pages_per_pool(dp_params);
+
 		for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
 			spin_lock_bh(&dp->tx_desc_lock[pool_id]);
 
-			for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) {
-				tx_spt_page = i + pool_id *
-					      ATH12K_TX_SPT_PAGES_PER_POOL(ab);
+			for (i = 0; i < tx_spt_pages_per_pool; i++) {
+				tx_spt_page = i + pool_id * tx_spt_pages_per_pool;
 				if (!dp->txbaddr[tx_spt_page])
 					continue;
 
@@ -1154,14 +1160,16 @@ static void *ath12k_dp_cc_get_desc_addr_ptr(struct ath12k_dp *dp,
 struct ath12k_rx_desc_info *ath12k_dp_get_rx_desc(struct ath12k_dp *dp,
 						  u32 cookie)
 {
+	const struct ath12k_dp_profile_params *dp_params;
 	struct ath12k_rx_desc_info **desc_addr_ptr;
 	u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx;
 
 	ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT);
 	spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT);
+	dp_params = &dp->ab->profile_param->dp_params;
 
-	start_ppt_idx = dp->rx_ppt_base + ATH12K_RX_SPT_PAGE_OFFSET(dp->ab);
-	end_ppt_idx = start_ppt_idx + ATH12K_NUM_RX_SPT_PAGES(dp->ab);
+	start_ppt_idx = dp->rx_ppt_base + ath12k_dp_rx_spt_page_offset(dp_params);
+	end_ppt_idx = start_ppt_idx + ath12k_dp_num_rx_spt_pages(dp_params);
 
 	if (ppt_idx < start_ppt_idx ||
 	    ppt_idx >= end_ppt_idx ||
@@ -1178,15 +1186,16 @@ EXPORT_SYMBOL(ath12k_dp_get_rx_desc);
 struct ath12k_tx_desc_info *ath12k_dp_get_tx_desc(struct ath12k_dp *dp,
 						  u32 cookie)
 {
+	const struct ath12k_dp_profile_params *dp_params;
 	struct ath12k_tx_desc_info **desc_addr_ptr;
 	u16 start_ppt_idx, end_ppt_idx, ppt_idx, spt_idx;
 
 	ppt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_PPT);
 	spt_idx = u32_get_bits(cookie, ATH12K_DP_CC_COOKIE_SPT);
+	dp_params = &dp->ab->profile_param->dp_params;
 
 	start_ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET;
-	end_ppt_idx = start_ppt_idx +
-		      (ATH12K_TX_SPT_PAGES_PER_POOL(dp->ab) * ATH12K_HW_MAX_QUEUES);
+	end_ppt_idx = start_ppt_idx + ath12k_dp_num_tx_spt_pages(dp_params);
 
 	if (ppt_idx < start_ppt_idx ||
 	    ppt_idx >= end_ppt_idx ||
@@ -1201,12 +1210,13 @@ EXPORT_SYMBOL(ath12k_dp_get_tx_desc);
 
 static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 {
-	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
+	const struct ath12k_dp_profile_params *dp_params = &ab->profile_param->dp_params;
+	u32 num_rx_spt_pages = ath12k_dp_num_rx_spt_pages(dp_params);
 	struct ath12k_rx_desc_info *rx_descs, **rx_desc_addr;
 	struct ath12k_tx_desc_info *tx_descs, **tx_desc_addr;
-	u32 num_rx_spt_pages = ATH12K_NUM_RX_SPT_PAGES(ab);
-	u32 i, j, pool_id, tx_spt_page;
-	u32 ppt_idx, cookie_ppt_idx;
+	u32 i, j, pool_id, tx_spt_page, tx_spt_pages_per_pool;
+	u32 ppt_idx, cookie_ppt_idx, rx_spt_page_offset;
+	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 
 	spin_lock_bh(&dp->rx_desc_lock);
 
@@ -1218,9 +1228,9 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 		return -ENOMEM;
 	}
 
-	/* First ATH12K_NUM_RX_SPT_PAGES(ab) of allocated SPT pages are used for
-	 * RX
-	 */
+	rx_spt_page_offset = ath12k_dp_rx_spt_page_offset(dp_params);
+
+	/* First num_rx_spt_pages of allocated SPT pages are used for RX */
 	for (i = 0; i < num_rx_spt_pages; i++) {
 		rx_descs = kzalloc_objs(*rx_descs, ATH12K_MAX_SPT_ENTRIES,
 					GFP_ATOMIC);
@@ -1230,7 +1240,7 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 			return -ENOMEM;
 		}
 
-		ppt_idx = ATH12K_RX_SPT_PAGE_OFFSET(ab) + i;
+		ppt_idx = rx_spt_page_offset + i;
 		cookie_ppt_idx = dp->rx_ppt_base + ppt_idx;
 		dp->rxbaddr[i] = &rx_descs[0];
 
@@ -1249,14 +1259,17 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 	spin_unlock_bh(&dp->rx_desc_lock);
 
 	dp->txbaddr = kzalloc_objs(struct ath12k_tx_desc_info *,
-				   ATH12K_NUM_TX_SPT_PAGES(ab), GFP_ATOMIC);
+				   ath12k_dp_num_tx_spt_pages(dp_params),
+				   GFP_ATOMIC);
 
 	if (!dp->txbaddr)
 		return -ENOMEM;
 
+	tx_spt_pages_per_pool = ath12k_dp_tx_spt_pages_per_pool(dp_params);
+
 	for (pool_id = 0; pool_id < ATH12K_HW_MAX_QUEUES; pool_id++) {
 		spin_lock_bh(&dp->tx_desc_lock[pool_id]);
-		for (i = 0; i < ATH12K_TX_SPT_PAGES_PER_POOL(ab); i++) {
+		for (i = 0; i < tx_spt_pages_per_pool; i++) {
 			tx_descs = kzalloc_objs(*tx_descs,
 						ATH12K_MAX_SPT_ENTRIES,
 						GFP_ATOMIC);
@@ -1267,8 +1280,7 @@ static int ath12k_dp_cc_desc_init(struct ath12k_base *ab)
 				return -ENOMEM;
 			}
 
-			tx_spt_page = i + pool_id *
-				      ATH12K_TX_SPT_PAGES_PER_POOL(ab);
+			tx_spt_page = i + pool_id * tx_spt_pages_per_pool;
 			ppt_idx = ATH12K_TX_SPT_PAGE_OFFSET + tx_spt_page;
 
 			dp->txbaddr[tx_spt_page] = &tx_descs[0];
@@ -1294,6 +1306,7 @@ static int ath12k_dp_cmem_init(struct ath12k_base *ab,
 			       struct ath12k_dp *dp,
 			       enum ath12k_dp_desc_type type)
 {
+	const struct ath12k_dp_profile_params *dp_params = &ab->profile_param->dp_params;
 	u32 cmem_base;
 	int i, start, end;
 
@@ -1302,12 +1315,12 @@ static int ath12k_dp_cmem_init(struct ath12k_base *ab,
 	switch (type) {
 	case ATH12K_DP_TX_DESC:
 		start = ATH12K_TX_SPT_PAGE_OFFSET;
-		end = start + ATH12K_NUM_TX_SPT_PAGES(ab);
+		end = start + ath12k_dp_num_tx_spt_pages(dp_params);
 		break;
 	case ATH12K_DP_RX_DESC:
 		cmem_base += ATH12K_PPT_ADDR_OFFSET(dp->rx_ppt_base);
-		start = ATH12K_RX_SPT_PAGE_OFFSET(ab);
-		end = start + ATH12K_NUM_RX_SPT_PAGES(ab);
+		start = ath12k_dp_rx_spt_page_offset(dp_params);
+		end = start + ath12k_dp_num_rx_spt_pages(dp_params);
 		break;
 	default:
 		ath12k_err(ab, "invalid descriptor type %d in cmem init\n", type);
@@ -1337,7 +1350,10 @@ void ath12k_dp_partner_cc_init(struct ath12k_base *ab)
 
 static u32 ath12k_dp_get_num_spt_pages(struct ath12k_base *ab)
 {
-	return ATH12K_NUM_RX_SPT_PAGES(ab) + ATH12K_NUM_TX_SPT_PAGES(ab);
+	const struct ath12k_dp_profile_params *dp_params = &ab->profile_param->dp_params;
+
+	return ath12k_dp_num_rx_spt_pages(dp_params) +
+	       ath12k_dp_num_tx_spt_pages(dp_params);
 }
 
 static int ath12k_dp_cc_init(struct ath12k_base *ab)
@@ -1365,7 +1381,8 @@ static int ath12k_dp_cc_init(struct ath12k_base *ab)
 		return -ENOMEM;
 	}
 
-	dp->rx_ppt_base = ab->device_id * ATH12K_NUM_RX_SPT_PAGES(ab);
+	dp->rx_ppt_base = ab->device_id *
+			  ath12k_dp_num_rx_spt_pages(&ab->profile_param->dp_params);
 
 	for (i = 0; i < dp->num_spt_pages; i++) {
 		dp->spt_info[i].vaddr = dma_alloc_coherent(ab->dev,
@@ -1465,6 +1482,7 @@ static int ath12k_dp_reoq_lut_setup(struct ath12k_base *ab)
 
 static int ath12k_dp_setup(struct ath12k_base *ab)
 {
+	const struct ath12k_dp_profile_params *dp_params;
 	struct ath12k_dp *dp;
 	struct hal_srng *srng = NULL;
 	size_t size = 0;
@@ -1474,6 +1492,7 @@ static int ath12k_dp_setup(struct ath12k_base *ab)
 
 	dp = ath12k_ab_to_dp(ab);
 	dp->ab = ab;
+	dp_params = &ab->profile_param->dp_params;
 
 	INIT_LIST_HEAD(&dp->reo_cmd_list);
 	INIT_LIST_HEAD(&dp->reo_cmd_cache_flush_list);
@@ -1528,7 +1547,7 @@ static int ath12k_dp_setup(struct ath12k_base *ab)
 		goto fail_dp_bank_profiles_cleanup;
 
 	size = ab->hal.hal_wbm_release_ring_tx_size *
-	       DP_TX_COMP_RING_SIZE(ab);
+	       ath12k_dp_tx_comp_ring_size(dp_params);
 
 	ret = ath12k_dp_reoq_lut_setup(ab);
 	if (ret) {
@@ -1540,7 +1559,8 @@ static int ath12k_dp_setup(struct ath12k_base *ab)
 		dp->tx_ring[i].tcl_data_ring_id = i;
 
 		dp->tx_ring[i].tx_status_head = 0;
-		dp->tx_ring[i].tx_status_tail = DP_TX_COMP_RING_SIZE(ab) - 1;
+		dp->tx_ring[i].tx_status_tail =
+			ath12k_dp_tx_comp_ring_size(dp_params) - 1;
 		dp->tx_ring[i].tx_status = kmalloc(size, GFP_KERNEL);
 		if (!dp->tx_ring[i].tx_status) {
 			ret = -ENOMEM;
diff --git a/drivers/net/wireless/ath/ath12k/dp.h b/drivers/net/wireless/ath/ath12k/dp.h
index bef0f2ba0560fa..c53eac20b9898e 100644
--- a/drivers/net/wireless/ath/ath12k/dp.h
+++ b/drivers/net/wireless/ath/ath12k/dp.h
@@ -7,6 +7,7 @@
 #ifndef ATH12K_DP_H
 #define ATH12K_DP_H
 
+#include "dp_stats.h"
 #include "hw.h"
 #include "dp_htt.h"
 #include "dp_cmn.h"
@@ -23,6 +24,15 @@ struct ath12k_ext_irq_grp;
 struct ath12k_dp_rx_tid;
 struct ath12k_dp_rx_tid_rxq;
 
+struct ath12k_dp_profile_params {
+	u32 tx_comp_ring_size;
+	u32 rxdma_monitor_buf_ring_size;
+	u32 rxdma_monitor_dst_ring_size;
+	u32 num_pool_tx_desc;
+	u32 rx_desc_count;
+	u32 rx_release_ring_size;
+};
+
 #define DP_MON_PURGE_TIMEOUT_MS     100
 #define DP_MON_SERVICE_BUDGET       128
 
@@ -52,7 +62,7 @@ struct dp_rxdma_ring {
 	int bufs_max;
 };
 
-#define ATH12K_TX_COMPL_NEXT(ab, x)	(((x) + 1) % DP_TX_COMP_RING_SIZE(ab))
+#define ATH12K_TX_COMPL_NEXT(ring_size, x)	(((x) + 1) % (ring_size))
 
 struct dp_tx_ring {
 	u8 tcl_data_ring_id;
@@ -192,16 +202,11 @@ struct ath12k_pdev_dp {
 
 #define DP_WBM_RELEASE_RING_SIZE	64
 #define DP_TCL_DATA_RING_SIZE		512
-#define DP_TX_COMP_RING_SIZE(ab) \
-	((ab)->profile_param->dp_params.tx_comp_ring_size)
-#define DP_TX_IDR_SIZE(ab)		DP_TX_COMP_RING_SIZE(ab)
 #define DP_TCL_CMD_RING_SIZE		32
 #define DP_TCL_STATUS_RING_SIZE		32
 #define DP_REO_DST_RING_MAX		8
 #define DP_REO_DST_RING_SIZE		2048
 #define DP_REO_REINJECT_RING_SIZE	32
-#define DP_RX_RELEASE_RING_SIZE(ab) \
-	((ab)->profile_param->dp_params.rx_release_ring_size)
 #define DP_REO_EXCEPTION_RING_SIZE	128
 #define DP_REO_CMD_RING_SIZE		256
 #define DP_REO_STATUS_RING_SIZE		2048
@@ -210,10 +215,6 @@ struct ath12k_pdev_dp {
 #define DP_RXDMA_REFILL_RING_SIZE	2048
 #define DP_RXDMA_ERR_DST_RING_SIZE	1024
 #define DP_RXDMA_MON_STATUS_RING_SIZE	1024
-#define DP_RXDMA_MONITOR_BUF_RING_SIZE(ab) \
-	((ab)->profile_param->dp_params.rxdma_monitor_buf_ring_size)
-#define DP_RXDMA_MONITOR_DST_RING_SIZE(ab) \
-	((ab)->profile_param->dp_params.rxdma_monitor_dst_ring_size)
 #define DP_RXDMA_MONITOR_DESC_RING_SIZE	4096
 #define DP_TX_MONITOR_BUF_RING_SIZE	4096
 #define DP_TX_MONITOR_DEST_RING_SIZE	2048
@@ -247,12 +248,6 @@ struct ath12k_pdev_dp {
 #define ATH12K_SHADOW_DP_TIMER_INTERVAL 20
 #define ATH12K_SHADOW_CTRL_TIMER_INTERVAL 10
 
-#define ATH12K_NUM_POOL_TX_DESC(ab) \
-	((ab)->profile_param->dp_params.num_pool_tx_desc)
-/* TODO: revisit this count during testing */
-#define ATH12K_RX_DESC_COUNT(ab) \
-	((ab)->profile_param->dp_params.rx_desc_count)
-
 #define ATH12K_PAGE_SIZE	PAGE_SIZE
 
 /* Total 1024 entries in PPT, i.e 4K/4 considering 4K aligned
@@ -263,21 +258,7 @@ struct ath12k_pdev_dp {
 /* Total 512 entries in a SPT, i.e 4K Page/8 */
 #define ATH12K_MAX_SPT_ENTRIES	512
 
-#define ATH12K_NUM_RX_SPT_PAGES(ab)	((ATH12K_RX_DESC_COUNT(ab)) / \
-					  ATH12K_MAX_SPT_ENTRIES)
-
-#define ATH12K_TX_SPT_PAGES_PER_POOL(ab) (ATH12K_NUM_POOL_TX_DESC(ab) / \
-					  ATH12K_MAX_SPT_ENTRIES)
-#define ATH12K_NUM_TX_SPT_PAGES(ab)	(ATH12K_TX_SPT_PAGES_PER_POOL(ab) * \
-					 ATH12K_HW_MAX_QUEUES)
-
 #define ATH12K_TX_SPT_PAGE_OFFSET 0
-#define ATH12K_RX_SPT_PAGE_OFFSET(ab) ATH12K_NUM_TX_SPT_PAGES(ab)
-
-/* The SPT pages are divided for RX and TX, first block for RX
- * and remaining for TX
- */
-#define ATH12K_NUM_TX_SPT_PAGE_START(ab) ATH12K_NUM_RX_SPT_PAGES(ab)
 
 #define ATH12K_DP_RX_DESC_MAGIC	0xBABABABA
 
@@ -429,19 +410,27 @@ struct ath12k_dp_arch_ops {
 struct ath12k_device_dp_tx_err_stats {
 	/* TCL Ring Descriptor unavailable */
 	u32 desc_na[DP_TCL_NUM_RING_MAX];
+	/* TX descriptor pool exhausted (per pool_id / traffic class) */
+	u32 txbuf_na[ATH12K_HW_MAX_QUEUES];
 	/* Other failures during dp_tx due to mem allocation failure
 	 * idr unavailable etc.
 	 */
 	atomic_t misc_fail;
 };
 
+struct ath12k_device_dp_rx_wbm_err_stats {
+	u32 rxdma_error[HAL_REO_ENTR_RING_RXDMA_ECODE_MAX];
+	u32 reo_error[HAL_REO_DEST_RING_ERROR_CODE_MAX];
+	u32 drop[WBM_ERR_DROP_MAX];
+	u32 sw_desc_fallback;
+};
+
 struct ath12k_device_dp_stats {
 	u32 err_ring_pkts;
 	u32 invalid_rbm;
-	u32 rxdma_error[HAL_REO_ENTR_RING_RXDMA_ECODE_MAX];
-	u32 reo_error[HAL_REO_DEST_RING_ERROR_CODE_MAX];
 	u32 hal_reo_error[DP_REO_DST_RING_MAX];
 	struct ath12k_device_dp_tx_err_stats tx_err;
+	struct ath12k_device_dp_rx_wbm_err_stats wbm_err;
 	u32 reo_rx[DP_REO_DST_RING_MAX][ATH12K_MAX_DEVICES];
 	u32 rx_wbm_rel_source[HAL_WBM_REL_SRC_MODULE_MAX][ATH12K_MAX_DEVICES];
 	u32 tqm_rel_reason[MAX_TQM_RELEASE_REASON];
@@ -450,6 +439,7 @@ struct ath12k_device_dp_stats {
 	u32 tx_enqueued[DP_TCL_NUM_RING_MAX];
 	u32 tx_completed[DP_TCL_NUM_RING_MAX];
 	u32 reo_excep_msdu_buf_type;
+	u32 sent_to_stack[DP_REO_DST_RING_MAX][ATH12K_MAX_DEVICES];
 };
 
 struct ath12k_dp {
@@ -680,6 +670,66 @@ ath12k_dp_to_pdev_dp(struct ath12k_dp *dp, u8 pdev_idx)
 	return rcu_dereference(dp->dp_pdevs[pdev_idx]);
 }
 
+static inline u32
+ath12k_dp_tx_comp_ring_size(const struct ath12k_dp_profile_params *p)
+{
+	return p->tx_comp_ring_size;
+}
+
+static inline u32
+ath12k_dp_rxdma_monitor_buf_ring_size(const struct ath12k_dp_profile_params *p)
+{
+	return p->rxdma_monitor_buf_ring_size;
+}
+
+static inline u32
+ath12k_dp_rxdma_monitor_dst_ring_size(const struct ath12k_dp_profile_params *p)
+{
+	return p->rxdma_monitor_dst_ring_size;
+}
+
+static inline u32
+ath12k_dp_num_pool_tx_desc(const struct ath12k_dp_profile_params *p)
+{
+	return p->num_pool_tx_desc;
+}
+
+static inline u32
+ath12k_dp_tx_spt_pages_per_pool(const struct ath12k_dp_profile_params *p)
+{
+	return ath12k_dp_num_pool_tx_desc(p) / ATH12K_MAX_SPT_ENTRIES;
+}
+
+static inline u32
+ath12k_dp_num_tx_spt_pages(const struct ath12k_dp_profile_params *p)
+{
+	return ath12k_dp_tx_spt_pages_per_pool(p) * ATH12K_HW_MAX_QUEUES;
+}
+
+static inline u32
+ath12k_dp_rx_spt_page_offset(const struct ath12k_dp_profile_params *p)
+{
+	return ath12k_dp_num_tx_spt_pages(p);
+}
+
+static inline u32
+ath12k_dp_rx_desc_count(const struct ath12k_dp_profile_params *p)
+{
+	return p->rx_desc_count;
+}
+
+static inline u32
+ath12k_dp_num_rx_spt_pages(const struct ath12k_dp_profile_params *p)
+{
+	return ath12k_dp_rx_desc_count(p) / ATH12K_MAX_SPT_ENTRIES;
+}
+
+static inline u32
+ath12k_dp_rx_release_ring_size(const struct ath12k_dp_profile_params *p)
+{
+	return p->rx_release_ring_size;
+}
+
 void ath12k_dp_vdev_tx_attach(struct ath12k *ar, struct ath12k_link_vif *arvif);
 void ath12k_dp_partner_cc_init(struct ath12k_base *ab);
 int ath12k_dp_pdev_alloc(struct ath12k_base *ab);
diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.c b/drivers/net/wireless/ath/ath12k/dp_mon.c
index 7d5be77b081f82..709507d44d8e51 100644
--- a/drivers/net/wireless/ath/ath12k/dp_mon.c
+++ b/drivers/net/wireless/ath/ath12k/dp_mon.c
@@ -81,24 +81,23 @@ struct sk_buff
 }
 EXPORT_SYMBOL(ath12k_dp_rx_alloc_mon_status_buf);
 
-u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id)
+bool ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id)
 {
-	u32 ret = 0;
-
 	if ((*ppdu_id < msdu_ppdu_id) &&
 	    ((msdu_ppdu_id - *ppdu_id) < DP_NOT_PPDU_ID_WRAP_AROUND)) {
 		/* Hold on mon dest ring, and reap mon status ring. */
 		*ppdu_id = msdu_ppdu_id;
-		ret = msdu_ppdu_id;
-	} else if ((*ppdu_id > msdu_ppdu_id) &&
-		((*ppdu_id - msdu_ppdu_id) > DP_NOT_PPDU_ID_WRAP_AROUND)) {
+		return true;
+	}
+	if ((*ppdu_id > msdu_ppdu_id) &&
+	    ((*ppdu_id - msdu_ppdu_id) > DP_NOT_PPDU_ID_WRAP_AROUND)) {
 		/* PPDU ID has exceeded the maximum value and will
 		 * restart from 0.
 		 */
 		*ppdu_id = msdu_ppdu_id;
-		ret = msdu_ppdu_id;
+		return true;
 	}
-	return ret;
+	return false;
 }
 EXPORT_SYMBOL(ath12k_dp_mon_comp_ppduid);
 
@@ -503,8 +502,6 @@ void ath12k_dp_mon_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev,
 	struct ieee80211_rx_status *rx_status;
 	struct ieee80211_radiotap_he *he = NULL;
 
-	status->link_valid = 0;
-
 	if ((status->encoding == RX_ENC_HE) && !(status->flag & RX_FLAG_RADIOTAP_HE) &&
 	    !(status->flag & RX_FLAG_SKIP_MONITOR)) {
 		he = skb_push(msdu, sizeof(known));
diff --git a/drivers/net/wireless/ath/ath12k/dp_mon.h b/drivers/net/wireless/ath/ath12k/dp_mon.h
index 162cdcaa57a77b..f352655eae9de2 100644
--- a/drivers/net/wireless/ath/ath12k/dp_mon.h
+++ b/drivers/net/wireless/ath/ath12k/dp_mon.h
@@ -100,7 +100,7 @@ struct sk_buff
 *ath12k_dp_rx_alloc_mon_status_buf(struct ath12k_base *ab,
 				   struct dp_rxdma_mon_ring *rx_ring,
 				   int *buf_id);
-u32 ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id);
+bool ath12k_dp_mon_comp_ppduid(u32 msdu_ppdu_id, u32 *ppdu_id);
 int
 ath12k_dp_mon_parse_status_buf(struct ath12k_pdev_dp *dp_pdev,
 			       struct ath12k_mon_data *pmon,
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c
index 8fa0e90b453160..cfaf26ae2720b8 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.c
@@ -330,16 +330,20 @@ static int ath12k_dp_rx_pdev_srng_alloc(struct ath12k *ar)
 {
 	struct ath12k_pdev_dp *dp = &ar->dp;
 	struct ath12k_base *ab = ar->ab;
+	u32 monitor_dst_ring_size;
+	u32 mac_id = dp->mac_id;
 	int i;
 	int ret;
-	u32 mac_id = dp->mac_id;
+
+	monitor_dst_ring_size =
+		ath12k_dp_rxdma_monitor_dst_ring_size(&ab->profile_param->dp_params);
 
 	for (i = 0; i < ab->hw_params->num_rxdma_per_pdev; i++) {
 		ret = ath12k_dp_srng_setup(ar->ab,
 					   &dp->rxdma_mon_dst_ring[i],
 					   HAL_RXDMA_MONITOR_DST,
 					   0, mac_id + i,
-					   DP_RXDMA_MONITOR_DST_RING_SIZE(ab));
+					   monitor_dst_ring_size);
 		if (ret) {
 			ath12k_warn(ar->ab,
 				    "failed to setup HAL_RXDMA_MONITOR_DST\n");
@@ -1200,27 +1204,6 @@ void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu
 }
 EXPORT_SYMBOL(ath12k_dp_rx_h_undecap);
 
-struct ath12k_dp_link_peer *
-ath12k_dp_rx_h_find_link_peer(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
-			      struct hal_rx_desc_data *rx_info)
-{
-	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
-	struct ath12k_dp_link_peer *peer = NULL;
-	struct ath12k_dp *dp = dp_pdev->dp;
-
-	lockdep_assert_held(&dp->dp_lock);
-
-	peer = ath12k_dp_link_peer_find_by_peerid(dp_pdev, rxcb->peer_id);
-
-	if (peer)
-		return peer;
-
-	if (rx_info->addr2_present)
-		peer = ath12k_dp_link_peer_find_by_addr(dp, rx_info->addr2);
-
-	return peer;
-}
-
 static void ath12k_dp_rx_h_rate(struct ath12k_pdev_dp *dp_pdev,
 				struct hal_rx_desc_data *rx_info)
 {
@@ -1375,6 +1358,7 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 	struct ath12k_dp *dp = dp_pdev->dp;
 	struct ieee80211_rx_status *rx_status;
 	struct ieee80211_sta *pubsta;
+	struct ieee80211_link_sta *link_pubsta = NULL;
 	struct ath12k_dp_peer *peer;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	struct ieee80211_rx_status *status = rx_info->rx_status;
@@ -1384,9 +1368,19 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 
 	pubsta = peer ? peer->sta : NULL;
 
-	status->link_valid = 0;
-	if (pubsta && pubsta->valid_links)
-		ath12k_hw_set_rx_link_id(dp->hw_params, peer, rxcb, status);
+	if (pubsta) {
+		if (pubsta->valid_links) {
+			int link_id =
+				ath12k_hw_get_rx_link_id(dp->hw_params, peer,
+							 rxcb, status);
+
+			if (link_id >= 0)
+				link_pubsta =
+					rcu_dereference(pubsta->link[link_id]);
+		} else {
+			link_pubsta = &pubsta->deflink;
+		}
+	}
 
 	ath12k_dbg(dp->ab, ATH12K_DBG_DATA,
 		   "rx skb %p len %u peer %pM %d %s sn %u %s%s%s%s%s%s%s%s%s%s rate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n",
@@ -1422,7 +1416,8 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 
 	/* TODO: trace rx packet */
 
-	ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), pubsta, msdu, napi);
+	ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), link_pubsta, msdu,
+			  napi);
 }
 EXPORT_SYMBOL(ath12k_dp_rx_deliver_msdu);
 
@@ -1692,6 +1687,8 @@ int ath12k_dp_rx_htt_setup(struct ath12k_base *ab)
 
 int ath12k_dp_rx_alloc(struct ath12k_base *ab)
 {
+	const struct ath12k_dp_profile_params *dp_params = &ab->profile_param->dp_params;
+	u32 monitor_buf_ring_size = ath12k_dp_rxdma_monitor_buf_ring_size(dp_params);
 	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	struct dp_srng *srng;
 	int i, ret;
@@ -1736,7 +1733,7 @@ int ath12k_dp_rx_alloc(struct ath12k_base *ab)
 		ret = ath12k_dp_srng_setup(ab,
 					   &dp->rxdma_mon_buf_ring.refill_buf_ring,
 					   HAL_RXDMA_MONITOR_BUF, 0, 0,
-					   DP_RXDMA_MONITOR_BUF_RING_SIZE(ab));
+					   monitor_buf_ring_size);
 		if (ret) {
 			ath12k_warn(ab, "failed to setup HAL_RXDMA_MONITOR_BUF\n");
 			return ret;
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.h b/drivers/net/wireless/ath/ath12k/dp_rx.h
index 0660db070e9217..82700474548a5b 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.h
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.h
@@ -237,9 +237,6 @@ int ath12k_dp_rx_peer_frag_setup(struct ath12k *ar, const u8 *peer_mac, int vdev
 
 u8 ath12k_dp_rx_h_l3pad(struct ath12k_base *ab,
 			struct hal_rx_desc *desc);
-struct ath12k_dp_link_peer *
-ath12k_dp_rx_h_find_link_peer(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu,
-			      struct hal_rx_desc_data *rx_info);
 u8 ath12k_dp_rx_h_decap_type(struct ath12k_base *ab,
 			     struct hal_rx_desc *desc);
 u32 ath12k_dp_rx_h_mpdu_err(struct ath12k_base *ab,
diff --git a/drivers/net/wireless/ath/ath12k/dp_stats.h b/drivers/net/wireless/ath/ath12k/dp_stats.h
new file mode 100644
index 00000000000000..1a05256dbf7407
--- /dev/null
+++ b/drivers/net/wireless/ath/ath12k/dp_stats.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef ATH12K_DP_STATS_H
+#define ATH12K_DP_STATS_H
+
+enum ath12k_wbm_err_drop_reason {
+	WBM_ERR_DROP_GET_SW_DESC,
+	WBM_ERR_DROP_DESC_PARSE,
+	WBM_ERR_DROP_INV_HW_ID,
+	WBM_ERR_DROP_NULL_PRTNR_DP,
+	WBM_ERR_DROP_NULL_PROC_DP,
+	WBM_ERR_DROP_NULL_PDEV,
+	WBM_ERR_DROP_NULL_AR,
+	WBM_ERR_DROP_CAC_RUNNING,
+	WBM_ERR_DROP_SG,
+	WBM_ERR_DROP_INV_NWIFI_HDR,
+	WBM_ERR_DROP_REO_GENERIC,
+	WBM_ERR_DROP_RXDMA_GENERIC,
+	WBM_ERR_DROP_MAX,
+};
+
+#endif /* ATH12K_DP_STATS_H */
diff --git a/drivers/net/wireless/ath/ath12k/htc.c b/drivers/net/wireless/ath/ath12k/htc.c
index 92138caa2a82fd..7227bed91ade7a 100644
--- a/drivers/net/wireless/ath/ath12k/htc.c
+++ b/drivers/net/wireless/ath/ath12k/htc.c
@@ -236,10 +236,7 @@ static void ath12k_htc_suspend_complete(struct ath12k_base *ab, bool ack)
 {
 	ath12k_dbg(ab, ATH12K_DBG_BOOT, "boot suspend complete %d\n", ack);
 
-	if (ack)
-		set_bit(ATH12K_FLAG_HTC_SUSPEND_COMPLETE, &ab->dev_flags);
-	else
-		clear_bit(ATH12K_FLAG_HTC_SUSPEND_COMPLETE, &ab->dev_flags);
+	assign_bit(ATH12K_FLAG_HTC_SUSPEND_COMPLETE, &ab->dev_flags, ack);
 
 	complete(&ab->htc_suspend);
 }
diff --git a/drivers/net/wireless/ath/ath12k/hw.h b/drivers/net/wireless/ath/ath12k/hw.h
index 011bb22e21c7b0..1e652dc0d2471b 100644
--- a/drivers/net/wireless/ath/ath12k/hw.h
+++ b/drivers/net/wireless/ath/ath12k/hw.h
@@ -166,6 +166,7 @@ struct ath12k_hw_params {
 	u8 max_radios;
 	bool single_pdev_only:1;
 	u32 qmi_service_ins_id;
+	u32 qmi_max_chunk_size;
 	bool internal_sleep_clock:1;
 
 	const struct ath12k_hw_ops *hw_ops;
@@ -249,9 +250,9 @@ struct ath12k_hw_ops {
 	bool (*dp_srng_is_tx_comp_ring)(int ring_num);
 	bool (*is_frame_link_agnostic)(struct ath12k_link_vif *arvif,
 				       struct ieee80211_mgmt *mgmt);
-	void (*set_rx_link_id)(struct ath12k_dp_peer *dp_peer,
-			       struct ath12k_skb_rxcb *rxcb,
-			       struct ieee80211_rx_status *status);
+	int (*get_rx_link_id)(struct ath12k_dp_peer *dp_peer,
+			      struct ath12k_skb_rxcb *rxcb,
+			      struct ieee80211_rx_status *status);
 };
 
 static inline
@@ -282,13 +283,15 @@ static inline int ath12k_hw_mac_id_to_srng_id(const struct ath12k_hw_params *hw,
 	return 0;
 }
 
-static inline void ath12k_hw_set_rx_link_id(const struct ath12k_hw_params *hw,
-					    struct ath12k_dp_peer *dp_peer,
-					    struct ath12k_skb_rxcb *rxcb,
-					    struct ieee80211_rx_status *status)
+static inline int ath12k_hw_get_rx_link_id(const struct ath12k_hw_params *hw,
+					   struct ath12k_dp_peer *dp_peer,
+					   struct ath12k_skb_rxcb *rxcb,
+					   struct ieee80211_rx_status *status)
 {
-	if (hw->hw_ops->set_rx_link_id)
-		hw->hw_ops->set_rx_link_id(dp_peer, rxcb, status);
+	if (hw->hw_ops->get_rx_link_id)
+		return hw->hw_ops->get_rx_link_id(dp_peer, rxcb, status);
+
+	return -1;
 }
 
 struct ath12k_fw_ie {
diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c
index 99bf5cf79d1028..196b807210de71 100644
--- a/drivers/net/wireless/ath/ath12k/mac.c
+++ b/drivers/net/wireless/ath/ath12k/mac.c
@@ -1506,6 +1506,13 @@ static int ath12k_mac_monitor_start(struct ath12k *ar)
 		return ret;
 	}
 
+	/*
+	 * A previous monitor session may have stopped mid-PPDU, leaving
+	 * ppdu_continuation set. Clear it so the first status buffer of
+	 * this session is not merged with stale state from before.
+	 */
+	ar->dp.mon_data.mon_ppdu_info.ppdu_continuation = false;
+
 	ar->monitor_started = true;
 	ar->num_started_vdevs++;
 
@@ -3517,7 +3524,7 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar,
 							   IEEE80211_EHT_MCS_NSS_RX));
 	}
 
-	max_nss = min(max_nss, (uint8_t)eht_nss);
+	max_nss = min(max_nss, (u8)eht_nss);
 
 	arg->peer_nss = min(link_sta->rx_nss, max_nss);
 
@@ -9732,6 +9739,7 @@ static int ath12k_mac_config_mon_status_default(struct ath12k *ar, bool enable)
 
 static int ath12k_mac_start(struct ath12k *ar)
 {
+	const struct ath12k_dp_profile_params *dp_params;
 	struct ath12k_hw *ah = ar->ah;
 	struct ath12k_base *ab = ar->ab;
 	struct ath12k_pdev *pdev = ar->pdev;
@@ -9740,6 +9748,8 @@ static int ath12k_mac_start(struct ath12k *ar)
 	lockdep_assert_held(&ah->hw_mutex);
 	lockdep_assert_wiphy(ath12k_ar_to_hw(ar)->wiphy);
 
+	dp_params = &ab->profile_param->dp_params;
+
 	ret = ath12k_wmi_pdev_set_param(ar, WMI_PDEV_PARAM_PMF_QOS,
 					1, pdev->pdev_id);
 
@@ -9788,7 +9798,7 @@ static int ath12k_mac_start(struct ath12k *ar)
 	if (ab->hw_params->supports_cong_ctrl_max_msdus) {
 		ret = ath12k_wmi_pdev_set_param(ar,
 						WMI_PDEV_PARAM_SET_CONG_CTRL_MAX_MSDUS,
-						ATH12K_NUM_POOL_TX_DESC(ab),
+						ath12k_dp_num_pool_tx_desc(dp_params),
 						pdev->pdev_id);
 		if (ret) {
 			ath12k_err(ab,
@@ -14294,6 +14304,7 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar,
 					   sizeof(ath12k_6ghz_channels), GFP_KERNEL);
 			if (!channels) {
 				kfree(ar->mac.sbands[NL80211_BAND_2GHZ].channels);
+				ar->mac.sbands[NL80211_BAND_2GHZ].channels = NULL;
 				return -ENOMEM;
 			}
 
@@ -14344,7 +14355,9 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar,
 					   GFP_KERNEL);
 			if (!channels) {
 				kfree(ar->mac.sbands[NL80211_BAND_2GHZ].channels);
+				ar->mac.sbands[NL80211_BAND_2GHZ].channels = NULL;
 				kfree(ar->mac.sbands[NL80211_BAND_6GHZ].channels);
+				ar->mac.sbands[NL80211_BAND_6GHZ].channels = NULL;
 				return -ENOMEM;
 			}
 
@@ -14384,7 +14397,7 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar,
 					kfree(ar->mac.sbands[NL80211_BAND_2GHZ].channels);
 					ar->mac.sbands[NL80211_BAND_2GHZ].channels = NULL;
 					kfree(ar->mac.sbands[NL80211_BAND_6GHZ].channels);
-					ar->mac.sbands[NL80211_BAND_2GHZ].channels = NULL;
+					ar->mac.sbands[NL80211_BAND_6GHZ].channels = NULL;
 					kfree(channels);
 					band->channels = NULL;
 					return ret;
@@ -15043,28 +15056,49 @@ static int ath12k_mac_hw_register(struct ath12k_hw *ah)
 		wiphy->interface_modes &= ~BIT(NL80211_IFTYPE_MONITOR);
 
 	for_each_ar(ah, ar, i) {
+		struct ath12k_base *this_ab = ar->ab;
+
 		/* Apply the regd received during initialization */
 		ret = ath12k_regd_update(ar, true);
 		if (ret) {
-			ath12k_err(ar->ab, "ath12k regd update failed: %d\n", ret);
+			ath12k_err(this_ab, "ath12k regd update failed: %d\n", ret);
 			goto err_unregister_hw;
 		}
 
-		if (ar->ab->hw_params->current_cc_support && ab->new_alpha2[0]) {
+		if (this_ab->hw_params->current_cc_support) {
 			struct wmi_set_current_country_arg current_cc = {};
+			struct ieee80211_regdomain *default_regd;
+			bool same_cc = false;
 
-			memcpy(&current_cc.alpha2, ab->new_alpha2, 2);
-			memcpy(&ar->alpha2, ab->new_alpha2, 2);
+			spin_lock_bh(&this_ab->base_lock);
+			memcpy(&current_cc.alpha2, this_ab->new_alpha2, 2);
+			spin_unlock_bh(&this_ab->base_lock);
+
+			if (!current_cc.alpha2[0])
+				goto fw_stats_init;
+
+			memcpy(&ar->alpha2, current_cc.alpha2, 2);
+
+			spin_lock_bh(&this_ab->base_lock);
+			default_regd = this_ab->default_regd[ar->pdev_idx];
+			if (default_regd)
+				same_cc = !memcmp(default_regd->alpha2,
+						  current_cc.alpha2, 2);
+			spin_unlock_bh(&this_ab->base_lock);
+
+			if (same_cc)
+				goto fw_stats_init;
 
 			reinit_completion(&ar->regd_update_completed);
 
 			ret = ath12k_wmi_send_set_current_country_cmd(ar, &current_cc);
 			if (ret)
-				ath12k_warn(ar->ab,
+				ath12k_warn(this_ab,
 					    "failed set cc code for mac register: %d\n",
 					    ret);
 		}
 
+fw_stats_init:
 		ath12k_fw_stats_init(ar);
 		ath12k_debugfs_register(ar);
 	}
diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c
index 6441927b538224..58e9de62315135 100644
--- a/drivers/net/wireless/ath/ath12k/pci.c
+++ b/drivers/net/wireless/ath/ath12k/pci.c
@@ -313,11 +313,24 @@ static void ath12k_pci_sw_reset(struct ath12k_base *ab, bool power_on)
 	ath12k_mhi_set_mhictrl_reset(ab);
 }
 
-static void ath12k_pci_free_ext_irq(struct ath12k_base *ab)
+static void ath12k_pci_free_ce_irq(struct ath12k_base *ab, int num_ce)
+{
+	int i, irq_idx;
+
+	for (i = 0; i < num_ce; i++) {
+		if (ath12k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
+			continue;
+
+		irq_idx = ATH12K_PCI_IRQ_CE0_OFFSET + i;
+		free_irq(ab->irq_num[irq_idx], &ab->ce.ce_pipe[i]);
+	}
+}
+
+static void ath12k_pci_free_ext_irq(struct ath12k_base *ab, int num_ext_irq_grp)
 {
 	int i, j;
 
-	for (i = 0; i < ATH12K_EXT_IRQ_GRP_NUM_MAX; i++) {
+	for (i = 0; i < num_ext_irq_grp; i++) {
 		struct ath12k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
 
 		for (j = 0; j < irq_grp->num_irq; j++)
@@ -330,16 +343,8 @@ static void ath12k_pci_free_ext_irq(struct ath12k_base *ab)
 
 static void ath12k_pci_free_irq(struct ath12k_base *ab)
 {
-	int i, irq_idx;
-
-	for (i = 0; i < ab->hw_params->ce_count; i++) {
-		if (ath12k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
-			continue;
-		irq_idx = ATH12K_PCI_IRQ_CE0_OFFSET + i;
-		free_irq(ab->irq_num[irq_idx], &ab->ce.ce_pipe[i]);
-	}
-
-	ath12k_pci_free_ext_irq(ab);
+	ath12k_pci_free_ce_irq(ab, ab->hw_params->ce_count);
+	ath12k_pci_free_ext_irq(ab, ATH12K_EXT_IRQ_GRP_NUM_MAX);
 }
 
 static void ath12k_pci_ce_irq_enable(struct ath12k_base *ab, u16 ce_id)
@@ -595,8 +600,6 @@ static int ath12k_pci_ext_irq_config(struct ath12k_base *ab)
 
 			irq = ath12k_pci_get_msi_irq(ab->dev, vector);
 
-			ab->irq_num[irq_idx] = irq;
-
 			ath12k_dbg(ab, ATH12K_DBG_PCI,
 				   "irq:%d group:%d\n", irq, i);
 
@@ -607,22 +610,24 @@ static int ath12k_pci_ext_irq_config(struct ath12k_base *ab)
 			if (ret) {
 				ath12k_err(ab, "failed request irq %d: %d\n",
 					   vector, ret);
-				goto fail_request;
+
+				for (n = 0; n < j; n++)
+					free_irq(ab->irq_num[irq_grp->irqs[n]], irq_grp);
+
+				netif_napi_del(&ab->ext_irq_grp[i].napi);
+				free_netdev(ab->ext_irq_grp[i].napi_ndev);
+				goto fail_allocate;
 			}
+
+			ab->irq_num[irq_idx] = irq;
 		}
 		ath12k_pci_ext_grp_disable(irq_grp);
 	}
 
 	return 0;
 
-fail_request:
-	/* i ->napi_ndev was properly allocated. Free it also */
-	i += 1;
 fail_allocate:
-	for (n = 0; n < i; n++) {
-		irq_grp = &ab->ext_irq_grp[n];
-		free_netdev(irq_grp->napi_ndev);
-	}
+	ath12k_pci_free_ext_irq(ab, i);
 	return ret;
 }
 
@@ -671,6 +676,8 @@ static int ath12k_pci_config_irq(struct ath12k_base *ab)
 		if (ret) {
 			ath12k_err(ab, "failed to request irq %d: %d\n",
 				   irq_idx, ret);
+
+			ath12k_pci_free_ce_irq(ab, i);
 			return ret;
 		}
 
@@ -681,8 +688,10 @@ static int ath12k_pci_config_irq(struct ath12k_base *ab)
 	}
 
 	ret = ath12k_pci_ext_irq_config(ab);
-	if (ret)
+	if (ret) {
+		ath12k_pci_free_ce_irq(ab, ab->hw_params->ce_count);
 		return ret;
+	}
 
 	return 0;
 }
@@ -899,19 +908,9 @@ static void ath12k_pci_free_region(struct ath12k_pci *ab_pci)
 
 static void ath12k_pci_aspm_disable(struct ath12k_pci *ab_pci)
 {
-	struct ath12k_base *ab = ab_pci->ab;
-
-	pcie_capability_read_word(ab_pci->pdev, PCI_EXP_LNKCTL,
-				  &ab_pci->link_ctl);
-
-	ath12k_dbg(ab, ATH12K_DBG_PCI, "pci link_ctl 0x%04x L0s %d L1 %d\n",
-		   ab_pci->link_ctl,
-		   u16_get_bits(ab_pci->link_ctl, PCI_EXP_LNKCTL_ASPM_L0S),
-		   u16_get_bits(ab_pci->link_ctl, PCI_EXP_LNKCTL_ASPM_L1));
+	ab_pci->aspm_states = pcie_aspm_enabled(ab_pci->pdev);
 
-	/* disable L0s and L1 */
-	pcie_capability_clear_word(ab_pci->pdev, PCI_EXP_LNKCTL,
-				   PCI_EXP_LNKCTL_ASPMC);
+	pci_disable_link_state(ab_pci->pdev, PCIE_LINK_STATE_ASPM_ALL);
 
 	set_bit(ATH12K_PCI_ASPM_RESTORE, &ab_pci->flags);
 }
@@ -940,10 +939,7 @@ static void ath12k_pci_aspm_restore(struct ath12k_pci *ab_pci)
 {
 	if (ab_pci->ab->hw_params->supports_aspm &&
 	    test_and_clear_bit(ATH12K_PCI_ASPM_RESTORE, &ab_pci->flags))
-		pcie_capability_clear_and_set_word(ab_pci->pdev, PCI_EXP_LNKCTL,
-						   PCI_EXP_LNKCTL_ASPMC,
-						   ab_pci->link_ctl &
-						   PCI_EXP_LNKCTL_ASPMC);
+		pci_force_enable_link_state(ab_pci->pdev, ab_pci->aspm_states);
 }
 
 static void ath12k_pci_cancel_workqueue(struct ath12k_base *ab)
diff --git a/drivers/net/wireless/ath/ath12k/pci.h b/drivers/net/wireless/ath/ath12k/pci.h
index 0e0e2020c6ae58..409ef063cd69af 100644
--- a/drivers/net/wireless/ath/ath12k/pci.h
+++ b/drivers/net/wireless/ath/ath12k/pci.h
@@ -128,7 +128,9 @@ struct ath12k_pci {
 
 	/* enum ath12k_pci_flags */
 	unsigned long flags;
-	u16 link_ctl;
+
+	/* Cached PCIe ASPM states */
+	u32 aspm_states;
 	unsigned long irq_flags;
 	const struct ath12k_pci_ops *pci_ops;
 	u32 qmi_instance;
diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c
index 280e50a1f31d89..817bd820623d25 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.c
+++ b/drivers/net/wireless/ath/ath12k/qmi.c
@@ -14,11 +14,11 @@
 #include <linux/of_address.h>
 #include <linux/ioport.h>
 #include <linux/of_reserved_mem.h>
+#include <linux/sizes.h>
 
 #define SLEEP_CLOCK_SELECT_INTERNAL_BIT	0x02
 #define HOST_CSTATE_BIT			0x04
 #define PLATFORM_CAP_PCIE_GLOBAL_RESET	0x08
-#define ATH12K_QMI_MAX_CHUNK_SIZE	2097152
 
 static const struct qmi_elem_info wlfw_host_mlo_chip_info_s_v01_ei[] = {
 	{
@@ -2584,46 +2584,63 @@ static void ath12k_qmi_free_mlo_mem_chunk(struct ath12k_base *ab,
 		mlo_chunk->v.ioaddr = NULL;
 	} else if (mlo_chunk->v.addr) {
 		dma_free_coherent(ab->dev,
-				  mlo_chunk->size,
-				  mlo_chunk->v.addr,
-				  mlo_chunk->paddr);
+				  mlo_chunk->total_size,
+				  mlo_chunk->vaddr_unaligned,
+				  mlo_chunk->paddr_unaligned);
+		mlo_chunk->vaddr_unaligned = NULL;
 		mlo_chunk->v.addr = NULL;
 	}
 
 	mlo_chunk->paddr = 0;
+	mlo_chunk->paddr_unaligned = 0;
 	mlo_chunk->size = 0;
-	if (fixed_mem)
+	mlo_chunk->total_size = 0;
+
+	if (fixed_mem) {
 		chunk->v.ioaddr = NULL;
-	else
+	} else {
 		chunk->v.addr = NULL;
+		chunk->vaddr_unaligned = NULL;
+	}
 	chunk->paddr = 0;
+	chunk->paddr_unaligned = 0;
 	chunk->size = 0;
+	chunk->total_size = 0;
 }
 
 static void ath12k_qmi_free_target_mem_chunk(struct ath12k_base *ab)
 {
 	struct ath12k_hw_group *ag = ab->ag;
+	struct target_mem_chunk *chunk;
 	int i, mlo_idx;
 
 	for (i = 0, mlo_idx = 0; i < ab->qmi.mem_seg_count; i++) {
-		if (ab->qmi.target_mem[i].type == MLO_GLOBAL_MEM_REGION_TYPE) {
+		chunk = &ab->qmi.target_mem[i];
+
+		if (chunk->type == MLO_GLOBAL_MEM_REGION_TYPE) {
 			ath12k_qmi_free_mlo_mem_chunk(ab,
-						      &ab->qmi.target_mem[i],
+						      chunk,
 						      mlo_idx++);
 		} else {
 			if (test_bit(ATH12K_FLAG_FIXED_MEM_REGION, &ab->dev_flags) &&
-			    ab->qmi.target_mem[i].v.ioaddr) {
-				iounmap(ab->qmi.target_mem[i].v.ioaddr);
-				ab->qmi.target_mem[i].v.ioaddr = NULL;
+			    chunk->v.ioaddr) {
+				iounmap(chunk->v.ioaddr);
+				chunk->v.ioaddr = NULL;
 			} else {
-				if (!ab->qmi.target_mem[i].v.addr)
+				if (!chunk->v.addr)
 					continue;
 				dma_free_coherent(ab->dev,
-						  ab->qmi.target_mem[i].prev_size,
-						  ab->qmi.target_mem[i].v.addr,
-						  ab->qmi.target_mem[i].paddr);
-				ab->qmi.target_mem[i].v.addr = NULL;
+						  chunk->total_size,
+						  chunk->vaddr_unaligned,
+						  chunk->paddr_unaligned);
+				chunk->vaddr_unaligned = NULL;
+				chunk->v.addr = NULL;
 			}
+
+			chunk->paddr = 0;
+			chunk->paddr_unaligned = 0;
+			chunk->size = 0;
+			chunk->total_size = 0;
 		}
 	}
 
@@ -2636,6 +2653,10 @@ static void ath12k_qmi_free_target_mem_chunk(struct ath12k_base *ab)
 static int ath12k_qmi_alloc_chunk(struct ath12k_base *ab,
 				  struct target_mem_chunk *chunk)
 {
+	dma_addr_t paddr;
+	size_t size;
+	void *vaddr;
+
 	/* Firmware reloads in recovery/resume.
 	 * In such cases, no need to allocate memory for FW again.
 	 */
@@ -2645,29 +2666,56 @@ static int ath12k_qmi_alloc_chunk(struct ath12k_base *ab,
 			goto this_chunk_done;
 
 		/* cannot reuse the existing chunk */
-		dma_free_coherent(ab->dev, chunk->prev_size,
-				  chunk->v.addr, chunk->paddr);
+		dma_free_coherent(ab->dev, chunk->total_size,
+				  chunk->vaddr_unaligned, chunk->paddr_unaligned);
 		chunk->v.addr = NULL;
+		chunk->vaddr_unaligned = NULL;
+		chunk->paddr_unaligned = 0;
+		chunk->total_size = 0;
 	}
 
-	chunk->v.addr = dma_alloc_coherent(ab->dev,
-					   chunk->size,
-					   &chunk->paddr,
+	/*
+	 * Each unaligned chunk costs the firmware extra TLB entries when
+	 * mapping it, and too many unaligned chunks exhaust the TLB and
+	 * crash the firmware, so align the base to 64 KB. The DMA
+	 * allocator only guarantees page alignment, but a natural
+	 * allocation is often already 64 KB aligned. Try the exact size
+	 * first and keep it when it happens to be aligned; only fall back
+	 * to over-allocating and rounding up when it is not.
+	 */
+	size = chunk->size;
+	vaddr = dma_alloc_coherent(ab->dev, size, &paddr,
+				   GFP_KERNEL | __GFP_NOWARN);
+	if (vaddr && !IS_ALIGNED(paddr, SZ_64K)) {
+		dma_free_coherent(ab->dev, size, vaddr, paddr);
+
+		size = chunk->size + SZ_64K - PAGE_SIZE;
+		vaddr = dma_alloc_coherent(ab->dev, size, &paddr,
 					   GFP_KERNEL | __GFP_NOWARN);
-	if (!chunk->v.addr) {
-		if (chunk->size > ATH12K_QMI_MAX_CHUNK_SIZE) {
+	}
+
+	if (!vaddr) {
+		if (chunk->size > ab->hw_params->qmi_max_chunk_size) {
 			ab->qmi.target_mem_delayed = true;
 			ath12k_warn(ab,
-				    "qmi dma allocation failed (%u B type %u), will try later with small size\n",
-				    chunk->size,
+				    "qmi dma allocation failed (%zu B type %u), will try later with small size\n",
+				    size,
 				    chunk->type);
 			ath12k_qmi_free_target_mem_chunk(ab);
 			return -EAGAIN;
 		}
-		ath12k_warn(ab, "memory allocation failure for %u size: %u\n",
-			    chunk->type, chunk->size);
+		ath12k_warn(ab, "memory allocation failure for %u size: %zu\n",
+			    chunk->type, size);
 		return -ENOMEM;
 	}
+
+	chunk->vaddr_unaligned = vaddr;
+	chunk->paddr_unaligned = paddr;
+	chunk->total_size = size;
+
+	chunk->paddr = ALIGN(paddr, SZ_64K);
+	chunk->v.addr = (u8 *)vaddr + (chunk->paddr - paddr);
+
 	chunk->prev_type = chunk->type;
 	chunk->prev_size = chunk->size;
 this_chunk_done:
@@ -2972,6 +3020,10 @@ int ath12k_qmi_request_target_cap(struct ath12k_base *ab)
 	if (r)
 		ath12k_dbg(ab, ATH12K_DBG_QMI, "SMBIOS bdf variant name not set.\n");
 
+	r = ath12k_core_check_dt(ab);
+	if (r)
+		ath12k_dbg(ab, ATH12K_DBG_QMI, "DT bdf variant name not set.\n");
+
 	r = ath12k_acpi_start(ab);
 	if (r)
 		/* ACPI is optional so continue in case of an error */
diff --git a/drivers/net/wireless/ath/ath12k/qmi.h b/drivers/net/wireless/ath/ath12k/qmi.h
index 6da10f3cb597c3..1b1c88906f1e94 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.h
+++ b/drivers/net/wireless/ath/ath12k/qmi.h
@@ -100,6 +100,10 @@ struct target_mem_chunk {
 		void __iomem *ioaddr;
 		void *addr;
 	} v;
+
+	dma_addr_t paddr_unaligned;
+	void *vaddr_unaligned;
+	u32 total_size;
 };
 
 struct target_info {
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/ahb.c b/drivers/net/wireless/ath/ath12k/wifi7/ahb.c
index 6e9e9034cba1a3..70372db6abae89 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/ahb.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/ahb.c
@@ -20,11 +20,13 @@ static const struct ath12k_ahb_desc ath12k_wifi7_ahb_desc[] = {
 		.hw_rev = ATH12K_HW_IPQ5332_HW10,
 		.auth_enabled = true,
 		.ops = &ath12k_ahb_hif_ops,
+		.supports_multipd = true,
 	},
 	[ATH12K_HW_IPQ5424_HW10] = {
 		.hw_rev = ATH12K_HW_IPQ5424_HW10,
 		.auth_enabled = false,
 		.ops = &ath12k_ahb_hif_ops,
+		.supports_multipd = false,
 	},
 };
 
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
index 016b0c38e51e20..1e29c13ad66d9c 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c
@@ -2675,6 +2675,8 @@ ath12k_wifi7_dp_rx_mon_mpdu_pop(struct ath12k *ar, int mac_id,
 		if (pmon->mon_last_linkdesc_paddr == paddr) {
 			pmon->rx_mon_stats.dup_mon_linkdesc_cnt++;
 			spin_unlock_bh(&pmon->mon_lock);
+			kfree_skb_list(*head_msdu);
+			*head_msdu = NULL;
 			return rx_bufs_used;
 		}
 
@@ -2718,7 +2720,6 @@ ath12k_wifi7_dp_rx_mon_mpdu_pop(struct ath12k *ar, int mac_id,
 					   i, (unsigned long)rxcb->paddr,
 					   (unsigned long)msdu_list.paddr[i]);
 				drop_mpdu = true;
-				continue;
 			}
 			if (!rxcb->unmapped) {
 				dma_unmap_single(ar->ab->dev, rxcb->paddr,
@@ -2801,7 +2802,7 @@ ath12k_wifi7_dp_rx_mon_mpdu_pop(struct ath12k *ar, int mac_id,
 	if (last)
 		last->next = NULL;
 
-	*tail_msdu = msdu;
+	*tail_msdu = last;
 
 	if (msdu_cnt == 0)
 		*npackets = 1;
@@ -2826,7 +2827,6 @@ ath12k_wifi7_dp_rx_mon_dest_process(struct ath12k *ar, int mac_id,
 	struct ath12k_base *ab = ar->ab;
 	struct ath12k_dp *dp = ath12k_ab_to_dp(ab);
 	void *ring_entry, *mon_dst_srng;
-	struct dp_mon_mpdu *tmp_mpdu;
 	LIST_HEAD(rx_desc_used_list);
 	struct hal_srng *srng;
 
@@ -2892,18 +2892,16 @@ ath12k_wifi7_dp_rx_mon_dest_process(struct ath12k *ar, int mac_id,
 		}
 
 		if (head_msdu && tail_msdu) {
-			tmp_mpdu = kzalloc_obj(*tmp_mpdu, GFP_ATOMIC);
-			if (!tmp_mpdu)
-				break;
-
-			tmp_mpdu->head = head_msdu;
-			tmp_mpdu->tail = tail_msdu;
-			tmp_mpdu->err_bitmap = pmon->err_bitmap;
-			tmp_mpdu->decap_format = pmon->decap_format;
-			ath12k_wifi7_dp_mon_rx_deliver(&ar->dp, tmp_mpdu,
+			struct dp_mon_mpdu tmp_mpdu = {
+				.head = head_msdu,
+				.tail = tail_msdu,
+				.err_bitmap = pmon->err_bitmap,
+				.decap_format = pmon->decap_format,
+			};
+
+			ath12k_wifi7_dp_mon_rx_deliver(&ar->dp, &tmp_mpdu,
 						       &pmon->mon_ppdu_info, napi);
 			rx_mon_stats->dest_mpdu_done++;
-			kfree(tmp_mpdu);
 		}
 
 		ring_entry = ath12k_hal_srng_dst_get_next_entry(ar->ab,
@@ -3211,18 +3209,23 @@ __ath12k_wifi7_dp_mon_process_ring(struct ath12k *ar, int mac_id,
 		goto exit;
 
 	while ((skb = __skb_dequeue(&skb_list))) {
-		memset(ppdu_info, 0, sizeof(*ppdu_info));
-		ppdu_info->peer_id = HAL_INVALID_PEERID;
+		if (!ppdu_info->ppdu_continuation)
+			ath12k_wifi7_dp_mon_rx_memset_ppdu_info(ppdu_info);
 
 		hal_status = ath12k_wifi7_dp_mon_parse_rx_dest(&ar->dp, pmon, skb);
 
-		if (ar->monitor_started &&
-		    pmon->mon_ppdu_status == DP_PPDU_STATUS_START &&
-		    hal_status == HAL_TLV_STATUS_PPDU_DONE) {
-			rx_mon_stats->status_ppdu_done++;
-			pmon->mon_ppdu_status = DP_PPDU_STATUS_DONE;
-			ath12k_wifi7_dp_rx_mon_dest_process(ar, mac_id, *budget, napi);
-			pmon->mon_ppdu_status = DP_PPDU_STATUS_START;
+		if (hal_status == HAL_TLV_STATUS_PPDU_DONE) {
+			ppdu_info->ppdu_continuation = false;
+			if (ar->monitor_started &&
+			    pmon->mon_ppdu_status == DP_PPDU_STATUS_START) {
+				rx_mon_stats->status_ppdu_done++;
+				pmon->mon_ppdu_status = DP_PPDU_STATUS_DONE;
+				ath12k_wifi7_dp_rx_mon_dest_process(ar, mac_id,
+								    *budget, napi);
+				pmon->mon_ppdu_status = DP_PPDU_STATUS_START;
+			}
+		} else {
+			ppdu_info->ppdu_continuation = true;
 		}
 
 		dev_kfree_skb_any(skb);
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
index 95d87dd67872f7..a32c804403e8a0 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.c
@@ -12,6 +12,15 @@
 #include "hal_wcn7850.h"
 #include "hal_qcc2072.h"
 
+static void
+ath12k_wifi7_dp_rx_wbm_err_free_skb(struct ath12k_dp *dp,
+				    struct sk_buff *msdu,
+				    enum ath12k_wbm_err_drop_reason drop_reason)
+{
+	dp->device_stats.wbm_err.drop[drop_reason]++;
+	dev_kfree_skb_any(msdu);
+}
+
 static u16 ath12k_wifi7_dp_rx_get_peer_id(struct ath12k_dp *dp,
 					  enum ath12k_peer_metadata_version ver,
 					  __le32 peer_metadata)
@@ -642,7 +651,10 @@ ath12k_wifi7_dp_rx_process_received_packets(struct ath12k_dp *dp,
 			dev_kfree_skb_any(msdu);
 			continue;
 		}
-
+		if (likely(ring_id < DP_REO_DST_RING_MAX))
+			dp->device_stats.sent_to_stack[ring_id][partner_dp->device_id]++;
+		else
+			WARN_ON_ONCE(1);
 		ath12k_dp_rx_deliver_msdu(dp_pdev, napi, msdu, &rx_info);
 	}
 
@@ -1614,6 +1626,7 @@ static int ath12k_wifi7_dp_rx_h_null_q_desc(struct ath12k_pdev_dp *dp_pdev,
 		/* First buffer will be freed by the caller, so deduct it's length */
 		msdu_len = msdu_len - (DP_RX_BUFFER_SIZE - hal_rx_desc_sz);
 		ath12k_wifi7_dp_rx_null_q_desc_sg_drop(dp, msdu_len, msdu_list);
+		dp->device_stats.wbm_err.drop[WBM_ERR_DROP_SG]++;
 		return -EINVAL;
 	}
 
@@ -1649,8 +1662,10 @@ static int ath12k_wifi7_dp_rx_h_null_q_desc(struct ath12k_pdev_dp *dp_pdev,
 		skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len);
 		skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes);
 	}
-	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, msdu, rx_info)))
+	if (unlikely(!ath12k_dp_rx_check_nwifi_hdr_len_valid(dp, msdu, rx_info))) {
+		dp->device_stats.wbm_err.drop[WBM_ERR_DROP_INV_NWIFI_HDR]++;
 		return -EINVAL;
+	}
 
 	ath12k_dp_rx_h_ppdu(dp_pdev, rx_info);
 	ret = ath12k_wifi7_dp_rx_h_mpdu(dp_pdev, msdu, rx_info);
@@ -1763,7 +1778,10 @@ static bool ath12k_wifi7_dp_rx_h_rxdma_err(struct ath12k_pdev_dp *dp_pdev,
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	bool drop = false;
 
-	dp->device_stats.rxdma_error[rxcb->err_code]++;
+	if (likely(rxcb->err_code < HAL_REO_ENTR_RING_RXDMA_ECODE_MAX))
+		dp->device_stats.wbm_err.rxdma_error[rxcb->err_code]++;
+	else
+		WARN_ON_ONCE(1);
 
 	switch (rxcb->err_code) {
 	case HAL_REO_ENTR_RING_RXDMA_ECODE_UNAUTH_WDS_ERR:
@@ -1780,6 +1798,7 @@ static bool ath12k_wifi7_dp_rx_h_rxdma_err(struct ath12k_pdev_dp *dp_pdev,
 		/* TODO: Review other rxdma error code to check if anything is
 		 * worth reporting to mac80211
 		 */
+		dp->device_stats.wbm_err.drop[WBM_ERR_DROP_RXDMA_GENERIC]++;
 		drop = true;
 		break;
 	}
@@ -1796,7 +1815,10 @@ static bool ath12k_wifi7_dp_rx_h_reo_err(struct ath12k_pdev_dp *dp_pdev,
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	bool drop = false;
 
-	dp->device_stats.reo_error[rxcb->err_code]++;
+	if (likely(rxcb->err_code < HAL_REO_DEST_RING_ERROR_CODE_MAX))
+		dp->device_stats.wbm_err.reo_error[rxcb->err_code]++;
+	else
+		WARN_ON_ONCE(1);
 
 	switch (rxcb->err_code) {
 	case HAL_REO_DEST_RING_ERROR_CODE_DESC_ADDR_ZERO:
@@ -1813,6 +1835,7 @@ static bool ath12k_wifi7_dp_rx_h_reo_err(struct ath12k_pdev_dp *dp_pdev,
 		/* TODO: Review other errors and process them to mac80211
 		 * as appropriate.
 		 */
+		dp->device_stats.wbm_err.drop[WBM_ERR_DROP_REO_GENERIC]++;
 		drop = true;
 		break;
 	}
@@ -1935,6 +1958,7 @@ int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
 		ret = ath12k_wifi7_hal_wbm_desc_parse_err(dp, rx_desc,
 							  &err_info);
 		if (ret) {
+			dp->device_stats.wbm_err.drop[WBM_ERR_DROP_DESC_PARSE]++;
 			ath12k_warn(ab, "failed to parse rx error in wbm_rel ring desc %d\n",
 				    ret);
 			continue;
@@ -1944,8 +1968,10 @@ int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
 
 		/* retry manual desc retrieval if hw cc is not done */
 		if (!desc_info) {
+			dp->device_stats.wbm_err.sw_desc_fallback++;
 			desc_info = ath12k_dp_get_rx_desc(dp, err_info.cookie);
 			if (!desc_info) {
+				dp->device_stats.wbm_err.drop[WBM_ERR_DROP_GET_SW_DESC]++;
 				ath12k_warn(ab, "Invalid cookie in DP WBM rx error descriptor retrieval: 0x%x\n",
 					    err_info.cookie);
 				continue;
@@ -1961,7 +1987,8 @@ int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
 		device_id = desc_info->device_id;
 		partner_dp = ath12k_dp_hw_grp_to_dp(dp_hw_grp, device_id);
 		if (unlikely(!partner_dp)) {
-			dev_kfree_skb_any(msdu);
+			ath12k_wifi7_dp_rx_wbm_err_free_skb(dp, msdu,
+							    WBM_ERR_DROP_NULL_PRTNR_DP);
 
 			/* In any case continuation bit is set
 			 * in the previous record, cleanup scatter_msdu_list
@@ -2007,7 +2034,8 @@ int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
 		hw_link_id = ath12k_dp_rx_get_msdu_src_link(partner_dp->hal,
 							    msdu_data);
 		if (hw_link_id >= ATH12K_GROUP_MAX_RADIO) {
-			dev_kfree_skb_any(msdu);
+			ath12k_wifi7_dp_rx_wbm_err_free_skb(dp, msdu,
+							    WBM_ERR_DROP_INV_HW_ID);
 
 			/* In any case continuation bit is set
 			 * in the previous record, cleanup scatter_msdu_list
@@ -2068,7 +2096,8 @@ int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
 			ath12k_dbg(ab, ATH12K_DBG_DATA,
 				   "Unable to process WBM error msdu due to invalid hw link id %d device id %d\n",
 				   hw_link_id, device_id);
-			dev_kfree_skb_any(msdu);
+			ath12k_wifi7_dp_rx_wbm_err_free_skb(dp, msdu,
+							    WBM_ERR_DROP_NULL_PROC_DP);
 			continue;
 		}
 
@@ -2077,18 +2106,21 @@ int ath12k_wifi7_dp_rx_process_wbm_err(struct ath12k_dp *dp,
 
 		dp_pdev = ath12k_dp_to_pdev_dp(partner_dp, pdev_idx);
 		if (!dp_pdev) {
-			dev_kfree_skb_any(msdu);
+			ath12k_wifi7_dp_rx_wbm_err_free_skb(dp, msdu,
+							    WBM_ERR_DROP_NULL_PDEV);
 			continue;
 		}
 		ar = ath12k_pdev_dp_to_ar(dp_pdev);
 
 		if (!ar || !rcu_dereference(ar->ab->pdevs_active[pdev_idx])) {
-			dev_kfree_skb_any(msdu);
+			ath12k_wifi7_dp_rx_wbm_err_free_skb(dp, msdu,
+							    WBM_ERR_DROP_NULL_AR);
 			continue;
 		}
 
 		if (test_bit(ATH12K_FLAG_CAC_RUNNING, &ar->dev_flags)) {
-			dev_kfree_skb_any(msdu);
+			ath12k_wifi7_dp_rx_wbm_err_free_skb(dp, msdu,
+							    WBM_ERR_DROP_CAC_RUNNING);
 			continue;
 		}
 
@@ -2317,17 +2349,16 @@ ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
 	return tlv_tag == HAL_RX_MPDU_START;
 }
 
-void
-ath12k_wifi7_dp_rx_set_link_id_qcn9274(struct ath12k_dp_peer *dp_peer,
+int
+ath12k_wifi7_dp_rx_get_link_id_qcn9274(struct ath12k_dp_peer *dp_peer,
 				       struct ath12k_skb_rxcb *rxcb,
 				       struct ieee80211_rx_status *status)
 {
-	status->link_valid = 1;
-	status->link_id = dp_peer->hw_links[rxcb->hw_link_id];
+	return dp_peer->hw_links[rxcb->hw_link_id];
 }
 
-void
-ath12k_wifi7_dp_rx_set_link_id_wcn7850(struct ath12k_dp_peer *dp_peer,
+int
+ath12k_wifi7_dp_rx_get_link_id_wcn7850(struct ath12k_dp_peer *dp_peer,
 				       struct ath12k_skb_rxcb *rxcb,
 				       struct ieee80211_rx_status *status)
 {
@@ -2341,10 +2372,9 @@ ath12k_wifi7_dp_rx_set_link_id_wcn7850(struct ath12k_dp_peer *dp_peer,
 	links_map = READ_ONCE(dp_peer->link_peers_map);
 	for_each_set_bit(i, &links_map, ATH12K_NUM_MAX_LINKS) {
 		link_peer = rcu_dereference(dp_peer->link_peers[i]);
-		if (link_peer && link_peer->peer_id == rxcb->peer_id) {
-			status->link_valid = 1;
-			status->link_id = link_peer->link_id;
-			return;
-		}
+		if (link_peer && link_peer->peer_id == rxcb->peer_id)
+			return link_peer->link_id;
 	}
+
+	return -1;
 }
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
index 1d3a4788a2dd91..d68489ff7b0373 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_rx.h
@@ -57,10 +57,10 @@ ath12k_wifi7_dp_rxdesc_mpdu_valid(struct ath12k_base *ab,
 				  struct hal_rx_desc *rx_desc);
 int ath12k_wifi7_dp_rx_tid_delete_handler(struct ath12k_base *ab,
 					  struct ath12k_dp_rx_tid_rxq *rx_tid);
-void ath12k_wifi7_dp_rx_set_link_id_qcn9274(struct ath12k_dp_peer *dp_peer,
-					    struct ath12k_skb_rxcb *rxcb,
-					    struct ieee80211_rx_status *status);
-void ath12k_wifi7_dp_rx_set_link_id_wcn7850(struct ath12k_dp_peer *dp_peer,
-					    struct ath12k_skb_rxcb *rxcb,
-					    struct ieee80211_rx_status *status);
+int ath12k_wifi7_dp_rx_get_link_id_qcn9274(struct ath12k_dp_peer *dp_peer,
+					   struct ath12k_skb_rxcb *rxcb,
+					   struct ieee80211_rx_status *status);
+int ath12k_wifi7_dp_rx_get_link_id_wcn7850(struct ath12k_dp_peer *dp_peer,
+					   struct ath12k_skb_rxcb *rxcb,
+					   struct ieee80211_rx_status *status);
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c
index 587d58eeccfa5d..41abf18657028e 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_tx.c
@@ -63,7 +63,7 @@ ath12k_wifi7_hal_tx_cmd_ext_desc_setup(struct ath12k_base *ab,
 {
 	tcl_ext_cmd->info0 = le32_encode_bits(ti->paddr,
 					      HAL_TX_MSDU_EXT_INFO0_BUF_PTR_LO);
-	tcl_ext_cmd->info1 = le32_encode_bits(0x0,
+	tcl_ext_cmd->info1 = le32_encode_bits((u64)ti->paddr >> HAL_ADDR_MSB_REG_SHIFT,
 					      HAL_TX_MSDU_EXT_INFO1_BUF_PTR_HI) |
 			       le32_encode_bits(ti->data_len,
 						HAL_TX_MSDU_EXT_INFO1_BUF_LEN);
@@ -167,8 +167,10 @@ int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *a
 	tx_ring = &dp->tx_ring[ti.ring_id];
 
 	tx_desc = ath12k_dp_tx_assign_buffer(dp, pool_id);
-	if (!tx_desc)
+	if (!tx_desc) {
+		dp->device_stats.tx_err.txbuf_na[pool_id]++;
 		return -ENOMEM;
+	}
 
 	dp_link_vif = ath12k_dp_vif_to_dp_link_vif(&ahvif->dp_vif, arvif->link_id);
 
@@ -313,10 +315,15 @@ int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *a
 			goto map;
 		}
 
-		/* hdr is pointing to a wrong place after alignment,
-		 * so refresh it for later use.
+		/*
+		 * The payload may have been shifted or even the entire buffer may have
+		 * been reallocated for alignment. In that case, hdr, eth and skb_cb
+		 * are stale pointers. Refresh them now for later dereference.
 		 */
 		hdr = (void *)skb->data;
+		if (eth)
+			eth = (struct ethhdr *)skb->data;
+		skb_cb = ATH12K_SKB_CB(skb);
 	}
 map:
 	ti.paddr = dma_map_single(dp->dev, skb->data, skb->len, DMA_TO_DEVICE);
@@ -449,15 +456,17 @@ int ath12k_wifi7_dp_tx(struct ath12k_pdev_dp *dp_pdev, struct ath12k_link_vif *a
 	return 0;
 
 fail_unmap_dma_ext:
-	if (skb_cb->paddr_ext_desc)
+	if (skb_cb->paddr_ext_desc) {
 		dma_unmap_single(dp->dev, skb_cb->paddr_ext_desc,
 				 skb_ext_desc->len,
 				 DMA_TO_DEVICE);
+		skb_cb->paddr_ext_desc = 0;
+	}
 fail_free_ext_skb:
 	kfree_skb(skb_ext_desc);
 
 fail_unmap_dma:
-	dma_unmap_single(dp->dev, ti.paddr, ti.data_len, DMA_TO_DEVICE);
+	dma_unmap_single(dp->dev, skb_cb->paddr, skb->len, DMA_TO_DEVICE);
 
 fail_remove_tx_buf:
 	ath12k_dp_tx_release_txbuf(dp, tx_desc, pool_id);
@@ -575,18 +584,19 @@ ath12k_dp_tx_process_htt_tx_complete(struct ath12k_dp *dp, void *desc,
 {
 	struct htt_tx_wbm_completion *status_desc;
 	struct ath12k_dp_htt_wbm_tx_status ts = {};
-	enum hal_wbm_htt_tx_comp_status wbm_status;
+	enum hal_wbm_htt_tx_comp_status htt_status;
 	u16 peer_id;
 
 	status_desc = desc;
 
-	wbm_status = le32_get_bits(status_desc->info0,
+	htt_status = le32_get_bits(status_desc->info0,
 				   HTT_TX_WBM_COMP_INFO0_STATUS);
-	dp->device_stats.fw_tx_status[wbm_status]++;
+	if (likely(htt_status < MAX_FW_TX_STATUS))
+		dp->device_stats.fw_tx_status[htt_status]++;
 
-	switch (wbm_status) {
+	switch (htt_status) {
 	case HAL_WBM_REL_HTT_TX_COMP_STATUS_OK:
-		ts.acked = (wbm_status == HAL_WBM_REL_HTT_TX_COMP_STATUS_OK);
+		ts.acked = true;
 		ts.ack_rssi = le32_get_bits(status_desc->info2,
 					    HTT_TX_WBM_COMP_INFO2_ACK_RSSI);
 
@@ -608,7 +618,7 @@ ath12k_dp_tx_process_htt_tx_complete(struct ath12k_dp *dp, void *desc,
 		 */
 		break;
 	default:
-		ath12k_warn(dp->ab, "Unknown htt wbm tx status %d\n", wbm_status);
+		ath12k_warn(dp->ab, "Unknown htt tx status %d\n", htt_status);
 		break;
 	}
 }
@@ -922,12 +932,14 @@ void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id)
 	u64 desc_va;
 	enum hal_wbm_rel_src_module buf_rel_source;
 	enum hal_wbm_tqm_rel_reason rel_status;
+	u32 ring_size;
 
 	spin_lock_bh(&status_ring->lock);
 
+	ring_size = ath12k_dp_tx_comp_ring_size(&ab->profile_param->dp_params);
 	ath12k_hal_srng_access_begin(ab, status_ring);
 
-	while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) !=
+	while (ATH12K_TX_COMPL_NEXT(ring_size, tx_ring->tx_status_head) !=
 	       tx_ring->tx_status_tail) {
 		desc = ath12k_hal_srng_dst_get_next_entry(ab, status_ring);
 		if (!desc)
@@ -936,11 +948,11 @@ void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id)
 		memcpy(&tx_ring->tx_status[tx_ring->tx_status_head],
 		       desc, sizeof(*desc));
 		tx_ring->tx_status_head =
-			ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head);
+			ATH12K_TX_COMPL_NEXT(ring_size, tx_ring->tx_status_head);
 	}
 
 	if (ath12k_hal_srng_dst_peek(ab, status_ring) &&
-	    (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_head) ==
+	    (ATH12K_TX_COMPL_NEXT(ring_size, tx_ring->tx_status_head) ==
 	     tx_ring->tx_status_tail)) {
 		/* TODO: Process pending tx_status messages when kfifo_is_full() */
 		ath12k_warn(ab, "Unable to process some of the tx_status ring desc because status_fifo is full\n");
@@ -950,13 +962,13 @@ void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id)
 
 	spin_unlock_bh(&status_ring->lock);
 
-	while (ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail) !=
+	while (ATH12K_TX_COMPL_NEXT(ring_size, tx_ring->tx_status_tail) !=
 	       tx_ring->tx_status_head) {
 		struct hal_wbm_completion_ring_tx *tx_status;
 		u32 desc_id;
 
 		tx_ring->tx_status_tail =
-			ATH12K_TX_COMPL_NEXT(ab, tx_ring->tx_status_tail);
+			ATH12K_TX_COMPL_NEXT(ring_size, tx_ring->tx_status_tail);
 		tx_status = &tx_ring->tx_status[tx_ring->tx_status_tail];
 		ath12k_wifi7_dp_tx_status_parse(dp, tx_status, &ts);
 
@@ -984,11 +996,17 @@ void ath12k_wifi7_dp_tx_completion_handler(struct ath12k_dp *dp, int ring_id)
 		/* Find the HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE value */
 		buf_rel_source = le32_get_bits(tx_status->info0,
 					       HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE);
-		dp->device_stats.tx_wbm_rel_source[buf_rel_source]++;
+		if (likely(buf_rel_source < HAL_WBM_REL_SRC_MODULE_MAX))
+			dp->device_stats.tx_wbm_rel_source[buf_rel_source]++;
+		else
+			WARN_ON_ONCE(1);
 
 		rel_status = le32_get_bits(tx_status->info0,
 					   HAL_WBM_COMPL_TX_INFO0_TQM_RELEASE_REASON);
-		dp->device_stats.tqm_rel_reason[rel_status]++;
+		if (likely(rel_status < MAX_TQM_RELEASE_REASON))
+			dp->device_stats.tqm_rel_reason[rel_status]++;
+		else
+			WARN_ON_ONCE(1);
 
 		/* Release descriptor as soon as extracting necessary info
 		 * to reduce contention
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c
index 49c69328970941..60e7a36097f12d 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.c
@@ -329,7 +329,10 @@ int ath12k_wifi7_hal_desc_reo_parse_err(struct ath12k_dp *dp,
 				    HAL_REO_DEST_RING_INFO0_PUSH_REASON);
 	err_code = le32_get_bits(desc->info0,
 				 HAL_REO_DEST_RING_INFO0_ERROR_CODE);
-	dp->device_stats.reo_error[err_code]++;
+	if (likely(err_code < HAL_REO_DEST_RING_ERROR_CODE_MAX))
+		dp->device_stats.wbm_err.reo_error[err_code]++;
+	else
+		WARN_ON_ONCE(1);
 
 	if (push_reason != HAL_REO_DEST_RING_PUSH_REASON_ERR_DETECTED &&
 	    push_reason != HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) {
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c
index eeabe9db776ebb..def54052987282 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_tx.c
@@ -36,7 +36,7 @@ void ath12k_wifi7_hal_tx_cmd_desc_setup(struct ath12k_base *ab,
 	tcl_cmd->buf_addr_info.info0 =
 		le32_encode_bits(ti->paddr, BUFFER_ADDR_INFO0_ADDR);
 	tcl_cmd->buf_addr_info.info1 =
-		le32_encode_bits(((uint64_t)ti->paddr >> HAL_ADDR_MSB_REG_SHIFT),
+		le32_encode_bits(((u64)ti->paddr >> HAL_ADDR_MSB_REG_SHIFT),
 				 BUFFER_ADDR_INFO1_ADDR);
 	tcl_cmd->buf_addr_info.info1 |=
 		le32_encode_bits((ti->rbm_id), BUFFER_ADDR_INFO1_RET_BUF_MGR) |
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hw.c b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
index 7b86f324b2e682..aec7c652808c3c 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hw.c
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hw.c
@@ -7,6 +7,7 @@
 #include <linux/types.h>
 #include <linux/bitops.h>
 #include <linux/bitfield.h>
+#include <linux/sizes.h>
 
 #include "../debug.h"
 #include "../core.h"
@@ -158,7 +159,7 @@ static const struct ath12k_hw_ops qcn9274_ops = {
 	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_qcn9274,
 	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_qcn9274,
 	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_qcn9274,
-	.set_rx_link_id = ath12k_wifi7_dp_rx_set_link_id_qcn9274,
+	.get_rx_link_id = ath12k_wifi7_dp_rx_get_link_id_qcn9274,
 };
 
 static const struct ath12k_hw_ops wcn7850_ops = {
@@ -169,7 +170,7 @@ static const struct ath12k_hw_ops wcn7850_ops = {
 	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_wcn7850,
 	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_wcn7850,
 	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_wcn7850,
-	.set_rx_link_id = ath12k_wifi7_dp_rx_set_link_id_wcn7850,
+	.get_rx_link_id = ath12k_wifi7_dp_rx_get_link_id_wcn7850,
 };
 
 static const struct ath12k_hw_ops qcc2072_ops = {
@@ -180,7 +181,7 @@ static const struct ath12k_hw_ops qcc2072_ops = {
 	.get_ring_selector = ath12k_wifi7_hw_get_ring_selector_wcn7850,
 	.dp_srng_is_tx_comp_ring = ath12k_wifi7_dp_srng_is_comp_ring_wcn7850,
 	.is_frame_link_agnostic = ath12k_wifi7_is_frame_link_agnostic_wcn7850,
-	.set_rx_link_id = ath12k_wifi7_dp_rx_set_link_id_wcn7850,
+	.get_rx_link_id = ath12k_wifi7_dp_rx_get_link_id_wcn7850,
 };
 
 #define ATH12K_TX_RING_MASK_0 0x1
@@ -369,6 +370,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 		.max_radios = 1,
 		.single_pdev_only = false,
 		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274,
+		.qmi_max_chunk_size = SZ_2M,
 		.internal_sleep_clock = false,
 
 		.hw_ops = &qcn9274_ops,
@@ -461,6 +463,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 		.max_radios = 1,
 		.single_pdev_only = true,
 		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850,
+		.qmi_max_chunk_size = SZ_512K,
 		.internal_sleep_clock = true,
 
 		.hw_ops = &wcn7850_ops,
@@ -553,6 +556,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 		.max_radios = 2,
 		.single_pdev_only = false,
 		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_QCN9274,
+		.qmi_max_chunk_size = SZ_2M,
 		.internal_sleep_clock = false,
 
 		.hw_ops = &qcn9274_ops,
@@ -643,6 +647,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 		.max_radios = 1,
 		.single_pdev_only = false,
 		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_IPQ5332,
+		.qmi_max_chunk_size = SZ_2M,
 		.internal_sleep_clock = false,
 
 		.hw_ops = &qcn9274_ops,
@@ -664,7 +669,8 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
 				   BIT(NL80211_IFTYPE_AP) |
-				   BIT(NL80211_IFTYPE_MESH_POINT),
+				   BIT(NL80211_IFTYPE_MESH_POINT) |
+				   BIT(NL80211_IFTYPE_AP_VLAN),
 		.supports_monitor = true,
 		.supports_cong_ctrl_max_msdus = true,
 
@@ -728,6 +734,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 		.max_radios = 1,
 		.single_pdev_only = true,
 		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850,
+		.qmi_max_chunk_size = SZ_512K,
 		.internal_sleep_clock = true,
 
 		.hw_ops = &qcc2072_ops,
@@ -821,6 +828,7 @@ static const struct ath12k_hw_params ath12k_wifi7_hw_params[] = {
 		.max_radios = 1,
 		.single_pdev_only = false,
 		.qmi_service_ins_id = ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_IPQ5332,
+		.qmi_max_chunk_size = SZ_2M,
 		.internal_sleep_clock = false,
 
 		.hw_ops = &qcn9274_ops,
diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c
index d5160af60e00d1..3149b6376c9493 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.c
+++ b/drivers/net/wireless/ath/ath12k/wmi.c
@@ -6975,6 +6975,14 @@ static int ath12k_reg_chan_list_event(struct ath12k_base *ab, struct sk_buff *sk
 		goto mem_free;
 	}
 
+	/*
+	 * Set the valid pdev_idx before validating so that, even when the
+	 * event is dropped or falls back, the completion the caller in
+	 * ath12k_mac_hw_register() may be waiting on is still signalled at
+	 * the end and it does not time out.
+	 */
+	pdev_idx = reg_info->phy_id;
+
 	ret = ath12k_reg_validate_reg_info(ab, reg_info);
 	if (ret == ATH12K_REG_STATUS_FALLBACK) {
 		ath12k_warn(ab, "failed to validate reg info %d\n", ret);
@@ -6991,7 +6999,6 @@ static int ath12k_reg_chan_list_event(struct ath12k_base *ab, struct sk_buff *sk
 	}
 
 	/* free old reg_info if it exist */
-	pdev_idx = reg_info->phy_id;
 	if (ab->reg_info[pdev_idx]) {
 		ath12k_reg_reset_reg_info(ab->reg_info[pdev_idx]);
 		kfree(ab->reg_info[pdev_idx]);
@@ -7030,7 +7037,7 @@ static int ath12k_reg_chan_list_event(struct ath12k_base *ab, struct sk_buff *sk
 
 out:
 	/* In some error cases, even a valid pdev_idx might not be available */
-	if (pdev_idx != 255)
+	if (pdev_idx < ab->num_radios)
 		ar = ab->pdevs[pdev_idx].ar;
 
 	/* During the boot-time update, 'ar' might not be allocated,
@@ -7271,7 +7278,6 @@ static void ath12k_mgmt_rx_event(struct ath12k_base *ab, struct sk_buff *skb)
 	struct ath12k_wmi_mgmt_rx_arg rx_ev = {};
 	struct ath12k *ar;
 	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
-	struct ieee80211_sta *pubsta = NULL;
 	struct ath12k_dp_link_peer *peer;
 	struct ieee80211_hdr *hdr;
 	bool is_4addr_null_pkt;
@@ -7373,11 +7379,6 @@ static void ath12k_mgmt_rx_event(struct ath12k_base *ab, struct sk_buff *skb)
 			dev_kfree_skb(skb);
 			goto exit;
 		}
-		pubsta = peer->sta;
-		if (pubsta && pubsta->valid_links) {
-			status->link_valid = 1;
-			status->link_id = peer->link_id;
-		}
 		spin_unlock_bh(&dp->dp_lock);
 		goto send_rx;
 	}
@@ -9992,11 +9993,6 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab,
 	u32 event_count;
 	int ret;
 
-	if (!skb->data) {
-		ath12k_warn(ab, "No data present in tpc stats event\n");
-		return;
-	}
-
 	if (skb->len < (sizeof(*fixed_param) + TLV_HDR_SIZE)) {
 		ath12k_warn(ab, "TPC stats event size invalid\n");
 		return;
diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c
index 8bbe554a9b3641..a9a35476fecd72 100644
--- a/drivers/net/wireless/ath/ath6kl/cfg80211.c
+++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c
@@ -1130,13 +1130,15 @@ void ath6kl_cfg80211_ch_switch_notify(struct ath6kl_vif *vif, int freq,
 }
 
 static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-				   int link_id, u8 key_index, bool pairwise,
+				   int link_id, u8 key_index,
+				   enum nl80211_key_type type,
 				   const u8 *mac_addr,
 				   struct key_params *params)
 {
 	struct ath6kl *ar = ath6kl_priv(wdev->netdev);
 	struct ath6kl_vif *vif = netdev_priv(wdev->netdev);
 	struct ath6kl_key *key = NULL;
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	int seq_len;
 	u8 key_usage;
 	u8 key_type;
@@ -1255,7 +1257,8 @@ static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wde
 }
 
 static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-				   int link_id, u8 key_index, bool pairwise,
+				   int link_id, u8 key_index,
+				   enum nl80211_key_type type,
 				   const u8 *mac_addr)
 {
 	struct ath6kl *ar = ath6kl_priv(wdev->netdev);
@@ -1285,7 +1288,8 @@ static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wde
 }
 
 static int ath6kl_cfg80211_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-				   int link_id, u8 key_index, bool pairwise,
+				   int link_id, u8 key_index,
+				   enum nl80211_key_type type,
 				   const u8 *mac_addr, void *cookie,
 				   void (*callback) (void *cookie,
 						     struct key_params *))
diff --git a/drivers/net/wireless/ath/ath9k/ar9003_calib.c b/drivers/net/wireless/ath/ath9k/ar9003_calib.c
index 2224cb74b1d41d..5ea18c87e12ee8 100644
--- a/drivers/net/wireless/ath/ath9k/ar9003_calib.c
+++ b/drivers/net/wireless/ath/ath9k/ar9003_calib.c
@@ -968,10 +968,7 @@ static void ar9003_hw_tx_iq_cal_outlier_detection(struct ath_hw *ah,
 		      AR_PHY_RX_IQCAL_CORR_B0_LOOPBACK_IQCORR_EN, 0x1);
 
 	if (caldata) {
-		if (is_reusable)
-			set_bit(TXIQCAL_DONE, &caldata->cal_flags);
-		else
-			clear_bit(TXIQCAL_DONE, &caldata->cal_flags);
+		assign_bit(TXIQCAL_DONE, &caldata->cal_flags, is_reusable);
 	}
 
 	return;
diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
index d3491ff08e6ef5..7d2f292e20c52b 100644
--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
+++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
@@ -14,6 +14,7 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
+#include <linux/math.h>
 #include <linux/unaligned.h>
 #include "htc.h"
 
@@ -328,11 +329,14 @@ static int __hif_usb_tx(struct hif_device_usb *hif_dev)
 	tx_skb_cnt = min_t(u16, hif_dev->tx.tx_skb_cnt, MAX_TX_AGGR_NUM);
 
 	for (i = 0; i < tx_skb_cnt; i++) {
-		nskb = __skb_dequeue(&hif_dev->tx.tx_skb_queue);
+		nskb = skb_peek(&hif_dev->tx.tx_skb_queue);
+		if (!nskb)
+			break;
 
-		/* Should never be NULL */
-		BUG_ON(!nskb);
+		if (tx_buf->offset + nskb->len + 4 > MAX_TX_BUF_SIZE)
+			break;
 
+		nskb = __skb_dequeue(&hif_dev->tx.tx_skb_queue);
 		hif_dev->tx.tx_skb_cnt--;
 
 		buf = tx_buf->buf;
@@ -342,13 +346,8 @@ static int __hif_usb_tx(struct hif_device_usb *hif_dev)
 		*hdr++ = cpu_to_le16(ATH_USB_TX_STREAM_MODE_TAG);
 		buf += 4;
 		memcpy(buf, nskb->data, nskb->len);
-		tx_buf->len = nskb->len + 4;
-
-		if (i < (tx_skb_cnt - 1))
-			tx_buf->offset += (((tx_buf->len - 1) / 4) + 1) * 4;
-
-		if (i == (tx_skb_cnt - 1))
-			tx_buf->len += tx_buf->offset;
+		tx_buf->len = tx_buf->offset + nskb->len + 4;
+		tx_buf->offset += round_up(nskb->len + 4, 4);
 
 		__skb_queue_tail(&tx_buf->skb_queue, nskb);
 		TX_STAT_INC(hif_dev, skb_queued);
diff --git a/drivers/net/wireless/ath/ath9k/htc.h b/drivers/net/wireless/ath/ath9k/htc.h
index 6c33e898b30006..4d2b75d0212efa 100644
--- a/drivers/net/wireless/ath/ath9k/htc.h
+++ b/drivers/net/wireless/ath/ath9k/htc.h
@@ -281,7 +281,6 @@ struct ath9k_htc_rxbuf {
 struct ath9k_htc_rx {
 	struct list_head rxbuf;
 	spinlock_t rxbuflock;
-	bool initialized;
 };
 
 #define ATH9K_HTC_TX_CLEANUP_INTERVAL 50 /* ms */
diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_init.c b/drivers/net/wireless/ath/ath9k/htc_drv_init.c
index 6de78ae8572687..911eaa4f776abc 100644
--- a/drivers/net/wireless/ath/ath9k/htc_drv_init.c
+++ b/drivers/net/wireless/ath/ath9k/htc_drv_init.c
@@ -910,6 +910,9 @@ static int ath9k_init_device(struct ath9k_htc_priv *priv,
 	ath9k_init_leds(priv);
 	ath9k_start_rfkill_poll(priv);
 
+	/* signal completion to ath9k_htc_rxep() and ath9k_wmi_event_tasklet() */
+	smp_store_release(&priv->initialized, true);
+
 	return 0;
 
 err_world:
@@ -966,10 +969,6 @@ int ath9k_htc_probe_device(struct htc_target *htc_handle, struct device *dev,
 
 	htc_handle->drv_priv = priv;
 
-	/* Allow ath9k_wmi_event_tasklet() to operate. */
-	smp_wmb();
-	priv->initialized = true;
-
 	return 0;
 
 err_init:
diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c
index bed7ea2425a0e2..95f48c7dca154b 100644
--- a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c
+++ b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c
@@ -1141,8 +1141,8 @@ void ath9k_htc_rxep(void *drv_priv, struct sk_buff *skb,
 	struct ath9k_htc_rxbuf *rxbuf = NULL, *tmp_buf = NULL;
 	unsigned long flags;
 
-	/* Check if ath9k_rx_init() completed. */
-	if (!data_race(priv->rx.initialized))
+	/* Check if ath9k_init_device() completed. */
+	if (!smp_load_acquire(&priv->initialized))
 		goto err;
 
 	spin_lock_irqsave(&priv->rx.rxbuflock, flags);
@@ -1200,10 +1200,6 @@ int ath9k_rx_init(struct ath9k_htc_priv *priv)
 		list_add_tail(&rxbuf->list, &priv->rx.rxbuf);
 	}
 
-	/* Allow ath9k_htc_rxep() to operate. */
-	smp_wmb();
-	priv->rx.initialized = true;
-
 	return 0;
 
 err:
diff --git a/drivers/net/wireless/ath/ath9k/mci.c b/drivers/net/wireless/ath/ath9k/mci.c
index f82bb23499284b..62ad761d6a765c 100644
--- a/drivers/net/wireless/ath/ath9k/mci.c
+++ b/drivers/net/wireless/ath/ath9k/mci.c
@@ -348,10 +348,7 @@ static u8 ath_mci_process_status(struct ath_softc *sc,
 	if (status->conn_handle >= ATH_MCI_MAX_PROFILE)
 		return 0;
 
-	if (status->is_critical)
-		__set_bit(status->conn_handle, mci->status);
-	else
-		__clear_bit(status->conn_handle, mci->status);
+	__assign_bit(status->conn_handle, mci->status, status->is_critical);
 
 	mci->num_mgmt = 0;
 	do {
diff --git a/drivers/net/wireless/ath/ath9k/wmi.c b/drivers/net/wireless/ath/ath9k/wmi.c
index 284e8c13b04309..4a08babb6d9d67 100644
--- a/drivers/net/wireless/ath/ath9k/wmi.c
+++ b/drivers/net/wireless/ath/ath9k/wmi.c
@@ -146,6 +146,12 @@ void ath9k_wmi_event_tasklet(struct tasklet_struct *t)
 	unsigned long flags;
 	u16 cmd_id;
 
+	/* Check if ath9k_init_device() completed. */
+	if (!smp_load_acquire(&priv->initialized)) {
+		tasklet_schedule(t);
+		return;
+	}
+
 	do {
 		spin_lock_irqsave(&wmi->wmi_lock, flags);
 		skb = __skb_dequeue(&wmi->wmi_event_queue);
@@ -155,12 +161,6 @@ void ath9k_wmi_event_tasklet(struct tasklet_struct *t)
 		}
 		spin_unlock_irqrestore(&wmi->wmi_lock, flags);
 
-		/* Check if ath9k_htc_probe_device() completed. */
-		if (!data_race(priv->initialized)) {
-			kfree_skb(skb);
-			continue;
-		}
-
 		hdr = (struct wmi_cmd_hdr *) skb->data;
 		cmd_id = be16_to_cpu(hdr->command_id);
 		wmi_event = skb_pull(skb, sizeof(struct wmi_cmd_hdr));
@@ -206,9 +206,16 @@ static void ath9k_wmi_rsp_callback(struct wmi *wmi, struct sk_buff *skb)
 {
 	skb_pull(skb, sizeof(struct wmi_cmd_hdr));
 
-	if (wmi->cmd_rsp_buf != NULL && wmi->cmd_rsp_len != 0)
+	if (wmi->cmd_rsp_buf && wmi->cmd_rsp_len) {
+		if (skb->len < wmi->cmd_rsp_len) {
+			wmi->cmd_rsp_status = -EMSGSIZE;
+			goto complete;
+		}
 		memcpy(wmi->cmd_rsp_buf, skb->data, wmi->cmd_rsp_len);
+	}
+	wmi->cmd_rsp_status = 0;
 
+complete:
 	complete(&wmi->cmd_wait);
 }
 
@@ -300,6 +307,7 @@ static int ath9k_wmi_cmd_issue(struct wmi *wmi,
 	/* record the rsp buffer and length */
 	wmi->cmd_rsp_buf = rsp_buf;
 	wmi->cmd_rsp_len = rsp_len;
+	wmi->cmd_rsp_status = 0;
 
 	wmi->last_seq_id = wmi->tx_seq_id;
 	spin_unlock_irqrestore(&wmi->wmi_lock, flags);
@@ -356,9 +364,13 @@ int ath9k_wmi_cmd(struct wmi *wmi, enum wmi_cmd_id cmd_id,
 		return -ETIMEDOUT;
 	}
 
+	spin_lock_irqsave(&wmi->wmi_lock, flags);
+	ret = wmi->cmd_rsp_status;
+	spin_unlock_irqrestore(&wmi->wmi_lock, flags);
+
 	mutex_unlock(&wmi->op_mutex);
 
-	return 0;
+	return ret;
 
 out:
 	ath_dbg(common, WMI, "WMI failure for: %s\n", wmi_cmd_to_name(cmd_id));
diff --git a/drivers/net/wireless/ath/ath9k/wmi.h b/drivers/net/wireless/ath/ath9k/wmi.h
index 5c3b710b8f3190..0bb9d31f1a1cc4 100644
--- a/drivers/net/wireless/ath/ath9k/wmi.h
+++ b/drivers/net/wireless/ath/ath9k/wmi.h
@@ -157,6 +157,7 @@ struct wmi {
 	u16 tx_seq_id;
 	u8 *cmd_rsp_buf;
 	u32 cmd_rsp_len;
+	int cmd_rsp_status;
 	bool stopped;
 
 	struct list_head pending_tx_events;
diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c
index 89d8b31787846b..3413bb4906d4db 100644
--- a/drivers/net/wireless/ath/ath9k/xmit.c
+++ b/drivers/net/wireless/ath/ath9k/xmit.c
@@ -168,7 +168,7 @@ static bool ath_merge_ratetbl(struct ieee80211_sta *sta, struct ath_buf *bf,
 	if (!sta)
 		return false;
 
-	ratetbl = rcu_dereference(sta->rates);
+	ratetbl = rcu_dereference_bh(sta->rates);
 	if (!ratetbl)
 		return false;
 
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index 9ac33d827f8010..bf060281c9a2ce 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -1620,11 +1620,13 @@ static void wil_del_rx_key(u8 key_index, enum wmi_key_usage key_usage,
 
 static int wil_cfg80211_add_key(struct wiphy *wiphy,
 				struct wireless_dev *wdev, int link_id,
-				u8 key_index, bool pairwise,
+				u8 key_index,
+				enum nl80211_key_type type,
 				const u8 *mac_addr,
 				struct key_params *params)
 {
 	int rc;
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
 	enum wmi_key_usage key_usage = wil_detect_key_usage(wdev, pairwise);
@@ -1695,12 +1697,14 @@ static int wil_cfg80211_add_key(struct wiphy *wiphy,
 
 static int wil_cfg80211_del_key(struct wiphy *wiphy,
 				struct wireless_dev *wdev, int link_id,
-				u8 key_index, bool pairwise,
+				u8 key_index,
+				enum nl80211_key_type type,
 				const u8 *mac_addr)
 {
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	struct wil6210_vif *vif = wdev_to_vif(wil, wdev);
-	enum wmi_key_usage key_usage = wil_detect_key_usage(wdev, pairwise);
+	enum wmi_key_usage key_usage =
+		wil_detect_key_usage(wdev, type == NL80211_KEYTYPE_PAIRWISE);
 	struct wil_sta_info *cs = wil_find_sta_by_key_usage(wil, vif->mid,
 							    key_usage,
 							    mac_addr);
@@ -2071,7 +2075,8 @@ void wil_cfg80211_ap_recovery(struct wil6210_priv *wil)
 		key_params.seq_len = IEEE80211_GCMP_PN_LEN;
 		rc = wil_cfg80211_add_key(wiphy, vif_to_wdev(vif), -1,
 					  vif->gtk_index,
-					  false, NULL, &key_params);
+					  NL80211_KEYTYPE_PAIRWISE,
+					  NULL, &key_params);
 		if (rc)
 			wil_err(wil, "vif %d recovery add key failed (%d)\n",
 				i, rc);
diff --git a/drivers/net/wireless/broadcom/b43legacy/dma.c b/drivers/net/wireless/broadcom/b43legacy/dma.c
index a9557356c9abbe..6c3d847242bbaf 100644
--- a/drivers/net/wireless/broadcom/b43legacy/dma.c
+++ b/drivers/net/wireless/broadcom/b43legacy/dma.c
@@ -610,25 +610,25 @@ struct b43legacy_dmaring *b43legacy_setup_dmaring(struct b43legacy_wldev *dev,
 	int nr_slots;
 	dma_addr_t dma_test;
 
-	ring = kzalloc_obj(*ring);
+	if (for_tx)
+		nr_slots = B43legacy_TXRING_SLOTS;
+	else
+		nr_slots = B43legacy_RXRING_SLOTS;
+
+	ring = kzalloc_flex(*ring, meta, nr_slots);
 	if (!ring)
 		goto out;
+
+	ring->nr_slots = nr_slots;
 	ring->type = type;
 	ring->dev = dev;
 
-	nr_slots = B43legacy_RXRING_SLOTS;
-	if (for_tx)
-		nr_slots = B43legacy_TXRING_SLOTS;
-
-	ring->meta = kzalloc_objs(struct b43legacy_dmadesc_meta, nr_slots);
-	if (!ring->meta)
-		goto err_kfree_ring;
 	if (for_tx) {
 		ring->txhdr_cache = kcalloc(nr_slots,
 					sizeof(struct b43legacy_txhdr_fw3),
 					GFP_KERNEL);
 		if (!ring->txhdr_cache)
-			goto err_kfree_meta;
+			goto err_kfree_ring;
 
 		/* test for ability to dma to txhdr_cache */
 		dma_test = dma_map_single(dev->dev->dma_dev, ring->txhdr_cache,
@@ -643,7 +643,7 @@ struct b43legacy_dmaring *b43legacy_setup_dmaring(struct b43legacy_wldev *dev,
 					sizeof(struct b43legacy_txhdr_fw3),
 					GFP_KERNEL | GFP_DMA);
 			if (!ring->txhdr_cache)
-				goto err_kfree_meta;
+				goto err_kfree_txhdr_cache;
 
 			dma_test = dma_map_single(dev->dev->dma_dev,
 					ring->txhdr_cache,
@@ -660,7 +660,6 @@ struct b43legacy_dmaring *b43legacy_setup_dmaring(struct b43legacy_wldev *dev,
 				 DMA_TO_DEVICE);
 	}
 
-	ring->nr_slots = nr_slots;
 	ring->mmio_base = b43legacy_dmacontroller_base(type, controller_index);
 	ring->index = controller_index;
 	if (for_tx) {
@@ -694,8 +693,6 @@ struct b43legacy_dmaring *b43legacy_setup_dmaring(struct b43legacy_wldev *dev,
 	free_ringmemory(ring);
 err_kfree_txhdr_cache:
 	kfree(ring->txhdr_cache);
-err_kfree_meta:
-	kfree(ring->meta);
 err_kfree_ring:
 	kfree(ring);
 	ring = NULL;
@@ -720,7 +717,6 @@ static void b43legacy_destroy_dmaring(struct b43legacy_dmaring *ring)
 	free_ringmemory(ring);
 
 	kfree(ring->txhdr_cache);
-	kfree(ring->meta);
 	kfree(ring);
 }
 
diff --git a/drivers/net/wireless/broadcom/b43legacy/dma.h b/drivers/net/wireless/broadcom/b43legacy/dma.h
index b5c1a51db2a469..edd06225b64ffa 100644
--- a/drivers/net/wireless/broadcom/b43legacy/dma.h
+++ b/drivers/net/wireless/broadcom/b43legacy/dma.h
@@ -122,8 +122,6 @@ enum b43legacy_dmatype {
 struct b43legacy_dmaring {
 	/* Kernel virtual base address of the ring memory. */
 	void *descbase;
-	/* Meta data about all descriptors. */
-	struct b43legacy_dmadesc_meta *meta;
 	/* Cache of TX headers for each slot.
 	 * This is to avoid an allocation on each TX.
 	 * This is NULL for an RX ring.
@@ -161,6 +159,8 @@ struct b43legacy_dmaring {
 	/* Last time we injected a ring overflow. */
 	unsigned long last_injected_overflow;
 #endif /* CONFIG_B43LEGACY_DEBUG*/
+	/* Meta data about all descriptors. */
+	struct b43legacy_dmadesc_meta meta[] __counted_by(nr_slots);
 };
 
 
diff --git a/drivers/net/wireless/broadcom/b43legacy/main.c b/drivers/net/wireless/broadcom/b43legacy/main.c
index 6b3e0b2bbca7d8..dfff3710dea3d5 100644
--- a/drivers/net/wireless/broadcom/b43legacy/main.c
+++ b/drivers/net/wireless/broadcom/b43legacy/main.c
@@ -3207,7 +3207,8 @@ static void b43legacy_wireless_core_exit(struct b43legacy_wldev *dev)
 	ssb_bus_may_powerdown(dev->dev->bus);
 }
 
-static void prepare_phy_data_for_init(struct b43legacy_wldev *dev)
+static noinline_for_stack void
+prepare_phy_data_for_init(struct b43legacy_wldev *dev)
 {
 	struct b43legacy_phy *phy = &dev->phy;
 	int i;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index 872c48806d0916..ede76a9e12c87c 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -2112,9 +2112,11 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme)
 	struct brcmf_pub *drvr = ifp->drvr;
 	s32 val;
 	s32 err;
+	s32 okc_enable;
 	const struct brcmf_tlv *rsn_ie;
 	const u8 *ie;
 	u32 ie_len;
+	bool fwsup_roam;
 	u32 offset;
 	u16 rsn_cap;
 	u32 mfp;
@@ -2122,6 +2124,9 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme)
 
 	profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE;
 	profile->is_ft = false;
+	profile->is_okc = false;
+	fwsup_roam = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FBT) ||
+		     brcmf_feat_is_enabled(ifp, BRCMF_FEAT_OKC);
 
 	if (!sme->crypto.n_akm_suites)
 		return 0;
@@ -2138,6 +2143,8 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme)
 			val = WPA_AUTH_UNSPECIFIED;
 			if (sme->want_1x)
 				profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X;
+			else if (fwsup_roam)
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM;
 			break;
 		case WLAN_AKM_SUITE_PSK:
 			val = WPA_AUTH_PSK;
@@ -2153,11 +2160,15 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme)
 			val = WPA2_AUTH_UNSPECIFIED;
 			if (sme->want_1x)
 				profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X;
+			else if (fwsup_roam)
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM;
 			break;
 		case WLAN_AKM_SUITE_8021X_SHA256:
 			val = WPA2_AUTH_1X_SHA256;
 			if (sme->want_1x)
 				profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X;
+			else if (fwsup_roam)
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM;
 			break;
 		case WLAN_AKM_SUITE_PSK_SHA256:
 			val = WPA2_AUTH_PSK_SHA256;
@@ -2170,10 +2181,16 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme)
 			profile->is_ft = true;
 			if (sme->want_1x)
 				profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X;
+			else if (fwsup_roam)
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM;
 			break;
 		case WLAN_AKM_SUITE_FT_PSK:
 			val = WPA2_AUTH_PSK | WPA2_AUTH_FT;
 			profile->is_ft = true;
+			if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP))
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK;
+			else if (fwsup_roam)
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM;
 			break;
 		case WLAN_AKM_SUITE_WFA_DPP:
 			val = WFA_AUTH_DPP;
@@ -2204,8 +2221,22 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme)
 
 	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X)
 		brcmf_dbg(INFO, "using 1X offload\n");
+
+	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X ||
+	    profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM) {
+		err = brcmf_fil_bsscfg_int_get(ifp, "okc_enable",
+					       &okc_enable);
+		if (err) {
+			bphy_err(drvr, "get okc_enable failed (%d)\n", err);
+		} else {
+			brcmf_dbg(INFO, "okc_enable (%d)\n", okc_enable);
+			profile->is_okc = okc_enable;
+		}
+	}
 	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE)
 		brcmf_dbg(INFO, "using SAE offload\n");
+	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM)
+		brcmf_dbg(INFO, "using roaming offload\n");
 
 	if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MFP))
 		goto skip_mfp_config;
@@ -2494,13 +2525,14 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev,
 
 		if (sme->crypto.psk && !is_sae_akm &&
 		    profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) {
-			if (WARN_ON(profile->use_fwsup !=
-				    BRCMF_PROFILE_FWSUP_NONE)) {
+			if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_NONE) {
+				brcmf_dbg(INFO, "using PSK offload\n");
+				profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK;
+			} else if (WARN_ON(profile->use_fwsup !=
+					   BRCMF_PROFILE_FWSUP_PSK)) {
 				err = -EINVAL;
 				goto done;
 			}
-			brcmf_dbg(INFO, "using PSK offload\n");
-			profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK;
 		}
 		if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE) {
 			/* enable firmware supplicant for this interface */
@@ -2770,7 +2802,8 @@ brcmf_cfg80211_config_default_key(struct wiphy *wiphy, struct net_device *ndev,
 
 static s32
 brcmf_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-		       int link_id, u8 key_idx, bool pairwise,
+		       int link_id, u8 key_idx,
+		       enum nl80211_key_type type,
 		       const u8 *mac_addr)
 {
 	struct brcmf_if *ifp = netdev_priv(wdev->netdev);
@@ -2808,7 +2841,8 @@ brcmf_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 
 static s32
 brcmf_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-		       int link_id, u8 key_idx, bool pairwise,
+		       int link_id, u8 key_idx,
+		       enum nl80211_key_type type,
 		       const u8 *mac_addr, struct key_params *params)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
@@ -2834,7 +2868,7 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 
 	if (params->key_len == 0)
 		return brcmf_cfg80211_del_key(wiphy, wdev, -1, key_idx,
-					      pairwise, mac_addr);
+					      type, mac_addr);
 
 	if (params->key_len > sizeof(key->data)) {
 		bphy_err(drvr, "Too long key length (%u)\n", params->key_len);
@@ -2930,7 +2964,8 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 
 static s32
 brcmf_cfg80211_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-		       int link_id, u8 key_idx, bool pairwise,
+		       int link_id, u8 key_idx,
+		       enum nl80211_key_type type,
 		       const u8 *mac_addr, void *cookie,
 		       void (*callback)(void *cookie,
 					struct key_params *params))
@@ -5953,17 +5988,29 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev,
 				  const struct cfg80211_pmk_conf *conf)
 {
 	struct brcmf_if *ifp;
+	struct brcmf_pub *drvr;
+	int ret;
 
 	brcmf_dbg(TRACE, "enter\n");
 
-	/* expect using firmware supplicant for 1X */
 	ifp = netdev_priv(dev);
-	if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X))
+	drvr = ifp->drvr;
+	if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X &&
+		    !ifp->vif->profile.is_ft &&
+		    !ifp->vif->profile.is_okc))
 		return -EINVAL;
 
 	if (conf->pmk_len > BRCMF_WSEC_MAX_PSK_LEN)
 		return -ERANGE;
 
+	if (ifp->vif->profile.is_okc) {
+		ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk",
+					       conf->pmk, conf->pmk_len);
+		if (ret < 0)
+			bphy_err(drvr, "okc_info_pmk iovar failed: ret=%d\n",
+				 ret);
+	}
+
 	return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len);
 }
 
@@ -6422,6 +6469,47 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg,
 	return err;
 }
 
+static bool brcmf_has_pmkid(const u8 *parse, u32 len)
+{
+	const struct brcmf_tlv *rsn_ie;
+	const u8 *ie;
+	u32 ie_len;
+	u32 offset;
+	u16 count;
+
+	if (!parse)
+		return false;
+
+	rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN);
+	if (!rsn_ie)
+		return false;
+
+	ie = (const u8 *)rsn_ie;
+	ie_len = rsn_ie->len + TLV_HDR_LEN;
+
+	offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN;
+	if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len)
+		return false;
+
+	count = ie[offset] + (ie[offset + 1] << 8);
+	offset += WPA_IE_SUITE_COUNT_LEN + count * WPA_IE_MIN_OUI_LEN;
+	if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len)
+		return false;
+
+	count = ie[offset] + (ie[offset + 1] << 8);
+	offset += WPA_IE_SUITE_COUNT_LEN + count * WPA_IE_MIN_OUI_LEN;
+	if (offset + RSN_CAP_LEN >= ie_len)
+		return false;
+
+	offset += RSN_CAP_LEN;
+	if (offset + RSN_PMKID_COUNT_LEN > ie_len)
+		return false;
+
+	count = ie[offset] + (ie[offset + 1] << 8);
+
+	return count > 0;
+}
+
 static s32
 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg,
 		       struct net_device *ndev,
@@ -6436,6 +6524,7 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg,
 	struct brcmf_bss_info_le *bi;
 	struct brcmu_chan ch;
 	struct cfg80211_roam_info roam_info = {};
+	bool authorized = false;
 	u32 freq;
 	s32 err = 0;
 	u8 *buf;
@@ -6482,14 +6571,18 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg,
 	roam_info.resp_ie = conn_info->resp_ie;
 	roam_info.resp_ie_len = conn_info->resp_ie_len;
 
+	if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X ||
+	     profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM) &&
+	    (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) ||
+	     profile->is_ft || profile->is_okc))
+		authorized = true;
+
 	cfg80211_roamed(ndev, &roam_info, GFP_KERNEL);
+	if (authorized)
+		cfg80211_port_authorized(ndev, profile->bssid, NULL, 0,
+					 GFP_KERNEL);
 	brcmf_dbg(CONN, "Report roaming result\n");
 
-	if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) {
-		cfg80211_port_authorized(ndev, profile->bssid, NULL, 0, GFP_KERNEL);
-		brcmf_dbg(CONN, "Report port authorized\n");
-	}
-
 	set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state);
 	brcmf_dbg(TRACE, "Exit\n");
 	return err;
@@ -6504,6 +6597,7 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg,
 	struct brcmf_cfg80211_profile *profile = &ifp->vif->profile;
 	struct brcmf_cfg80211_connect_info *conn_info = cfg_to_conn(cfg);
 	struct cfg80211_connect_resp_params conn_params;
+	bool authorized;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
@@ -6522,13 +6616,23 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg,
 			clear_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS,
 				  &ifp->vif->sme_state);
 			conn_params.status = WLAN_STATUS_AUTH_TIMEOUT;
+			bphy_err(ifp->drvr, "connect failed: event %u status %u reason %u\n",
+				 e->event_code, e->status, e->reason);
 		}
 		conn_params.links[0].bssid = profile->bssid;
 		conn_params.req_ie = conn_info->req_ie;
 		conn_params.req_ie_len = conn_info->req_ie_len;
 		conn_params.resp_ie = conn_info->resp_ie;
 		conn_params.resp_ie_len = conn_info->resp_ie_len;
+		authorized = completed &&
+			     (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X ||
+			      profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM) &&
+			     brcmf_has_pmkid(conn_params.req_ie,
+					     conn_params.req_ie_len);
 		cfg80211_connect_done(ndev, &conn_params, GFP_KERNEL);
+		if (authorized)
+			cfg80211_port_authorized(ndev, profile->bssid, NULL, 0,
+						 GFP_KERNEL);
 		brcmf_dbg(CONN, "Report connect result - connection %s\n",
 			  completed ? "succeeded" : "failed");
 	}
@@ -7731,6 +7835,10 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp)
 	}
 	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE_EXT))
 		wiphy->features |= NL80211_FEATURE_SAE;
+	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FBT) ||
+	    brcmf_feat_is_enabled(ifp, BRCMF_FEAT_OKC))
+		wiphy_ext_feature_set(wiphy,
+				      NL80211_EXT_FEATURE_FAST_ROAM_OFFLOAD);
 	wiphy->mgmt_stypes = brcmf_txrx_stypes;
 	wiphy->max_remain_on_channel_duration = 5000;
 	if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
index 63e534523f5188..1aa99390a9510b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h
@@ -127,7 +127,8 @@ enum brcmf_profile_fwsup {
 	BRCMF_PROFILE_FWSUP_NONE,
 	BRCMF_PROFILE_FWSUP_PSK,
 	BRCMF_PROFILE_FWSUP_1X,
-	BRCMF_PROFILE_FWSUP_SAE
+	BRCMF_PROFILE_FWSUP_SAE,
+	BRCMF_PROFILE_FWSUP_ROAM
 };
 
 /**
@@ -164,6 +165,7 @@ enum brcmf_mgmt_tx_status {
  * @bssid: bssid of joined/joining ibss.
  * @sec: security information.
  * @key: key information
+ * @is_okc: OKC is used for current connection.
  */
 struct brcmf_cfg80211_profile {
 	u8 bssid[ETH_ALEN];
@@ -172,6 +174,7 @@ struct brcmf_cfg80211_profile {
 	enum brcmf_profile_fwsup use_fwsup;
 	u16 use_fwauth;
 	bool is_ft;
+	bool is_okc;
 };
 
 /**
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
index 488364ef8ff2a1..cc19e5e7ebf572 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
@@ -45,6 +45,8 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = {
 	{ BRCMF_FEAT_SAE, "sae " },
 	{ BRCMF_FEAT_FWAUTH, "idauth" },
 	{ BRCMF_FEAT_SAE_EXT, "sae_ext" },
+	{ BRCMF_FEAT_FBT, "fbt " },
+	{ BRCMF_FEAT_OKC, "okc" },
 };
 
 #ifdef DEBUG
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h
index 31f8695ca41765..8165a286b6308c 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h
@@ -59,7 +59,9 @@
 	BRCMF_FEAT_DEF(SCAN_V2) \
 	BRCMF_FEAT_DEF(PMKID_V2) \
 	BRCMF_FEAT_DEF(PMKID_V3) \
-	BRCMF_FEAT_DEF(SAE_EXT)
+	BRCMF_FEAT_DEF(SAE_EXT) \
+	BRCMF_FEAT_DEF(FBT) \
+	BRCMF_FEAT_DEF(OKC)
 
 /*
  * Quirks:
diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c
index cbaf250626c5c3..f7ca182d14943a 100644
--- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c
+++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c
@@ -3807,6 +3807,7 @@ il3945_pci_remove(struct pci_dev *pdev)
 
 	il_free_channel_map(il);
 	il_free_geos(il);
+	il_eeprom_free(il);
 	kfree(il->scan_cmd);
 	dev_kfree_skb(il->beacon_skb);
 	ieee80211_free_hw(il->hw);
diff --git a/drivers/net/wireless/intel/iwlwifi/Makefile b/drivers/net/wireless/intel/iwlwifi/Makefile
index 2a74efe09cf018..6746281350e0ee 100644
--- a/drivers/net/wireless/intel/iwlwifi/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/Makefile
@@ -9,9 +9,9 @@ iwlwifi-objs		+= iwl-utils.o
 iwlwifi-objs		+= iwl-phy-db.o iwl-nvm-parse.o
 
 # Bus
-iwlwifi-objs		+= pcie/ctxt-info.o pcie/ctxt-info-v2.o pcie/drv.o pcie/utils.o
-iwlwifi-objs		+= pcie/gen1_2/rx.o pcie/gen1_2/tx.o pcie/gen1_2/trans.o
-iwlwifi-objs		+= pcie/gen1_2/trans-gen2.o pcie/gen1_2/tx-gen2.o
+iwlwifi-objs		+= pcie/drv.o pcie/rx.o pcie/tx.o pcie/trans.o
+iwlwifi-objs		+= pcie/ctxt-info.o pcie/ctxt-info-v2.o pcie/trans-gen2.o pcie/tx-gen2.o
+iwlwifi-objs		+= pcie/utils.o
 
 CFLAGS_pcie/drv.o += -Wno-override-init
 
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/bz.c b/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
index 1099d8cfe6aa40..3cc3ffd0aa1c76 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
@@ -10,14 +10,11 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware core release supported */
-#define IWL_BZ_UCODE_CORE_MAX	107
+#define IWL_BZ_UCODE_CORE_MAX	108
 
 /* Lowest firmware core release supported */
 #define IWL_BZ_UCODE_CORE_MIN	102
 
-#define IWL_BZ_A_FM_B_FW_PRE		"iwlwifi-bz-a0-fm-b0"
-#define IWL_BZ_A_FM_C_FW_PRE		"iwlwifi-bz-a0-fm-c0"
-#define IWL_BZ_A_FM4_B_FW_PRE		"iwlwifi-bz-a0-fm4-b0"
 #define IWL_GL_B_FM_B_FW_PRE		"iwlwifi-gl-b0-fm-b0"
 #define IWL_GL_C_FM_C_FW_PRE		"iwlwifi-gl-c0-fm-c0"
 
@@ -92,8 +89,5 @@ const struct iwl_mac_cfg iwl_gl_mac_cfg = {
 	.low_latency_xtal = true,
 };
 
-IWL_CORE_FW(IWL_BZ_A_FM_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
-IWL_CORE_FW(IWL_BZ_A_FM_C_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
-IWL_CORE_FW(IWL_BZ_A_FM4_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
 IWL_CORE_FW(IWL_GL_B_FM_B_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
 IWL_CORE_FW(IWL_GL_C_FM_C_FW_PRE, IWL_BZ_UCODE_CORE_MAX);
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/dr.c b/drivers/net/wireless/intel/iwlwifi/cfg/dr.c
index a89a2c75687f2f..f6dd1f9733fb5a 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/dr.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/dr.c
@@ -9,7 +9,7 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware core release supported */
-#define IWL_DR_UCODE_CORE_MAX	107
+#define IWL_DR_UCODE_CORE_MAX	108
 
 /* Lowest firmware core release supported */
 #define IWL_DR_UCODE_CORE_MIN	102
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
index 35b1618f3474ae..a7770cd612137e 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
@@ -5,6 +5,8 @@
  */
 #include "iwl-config.h"
 
+#define IWL_BZ_A_FM_C_FW_PRE		"iwlwifi-bz-a0-fm-c0"
+
 #define IWL_DEVICE_FM							\
 	.ht_params = {							\
 		.stbc = true,						\
@@ -48,3 +50,5 @@ const char iwl_be201_name[] = "Intel(R) Wi-Fi 7 BE201 320MHz";
 const char iwl_be200_name[] = "Intel(R) Wi-Fi 7 BE200 320MHz";
 const char iwl_be202_name[] = "Intel(R) Wi-Fi 7 BE202 160MHz";
 const char iwl_be401_name[] = "Intel(R) Wi-Fi 7 BE401 320MHz";
+
+IWL_CORE_FW(IWL_BZ_A_FM_C_FW_PRE, 107);
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/sc.c b/drivers/net/wireless/intel/iwlwifi/cfg/sc.c
index 16abf0147fe42f..467888c8d2cba8 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/sc.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/sc.c
@@ -10,7 +10,7 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware core release supported */
-#define IWL_SC_UCODE_CORE_MAX	107
+#define IWL_SC_UCODE_CORE_MAX	108
 
 /* Lowest firmware core release supported */
 #define IWL_SC_UCODE_CORE_MIN	102
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
index 71e0950d01932c..7b11316a3062c9 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c
@@ -525,6 +525,83 @@ iwl_acpi_parse_chains_table(union acpi_object *table,
 	return 0;
 }
 
+static int
+iwl_acpi_parse_extended_sar_profiles(union acpi_object *wifi_pkg,
+				     struct iwl_sar_profile *sar_profiles,
+				     u8 num_sub_bands, bool has_cdb)
+{
+	int n_profiles, pos, ret;
+	bool enabled;
+
+	if (wifi_pkg->package.elements[1].type != ACPI_TYPE_INTEGER ||
+	    wifi_pkg->package.elements[2].type != ACPI_TYPE_INTEGER)
+		return -EINVAL;
+
+	if (WARN_ON(ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands >
+		    ARRAY_SIZE(sar_profiles[0].chains) *
+		    ARRAY_SIZE(sar_profiles[0].chains[0].subbands)))
+		return -EINVAL;
+
+	enabled = !!(wifi_pkg->package.elements[1].integer.value);
+	n_profiles = wifi_pkg->package.elements[2].integer.value;
+
+	if (n_profiles >= BIOS_SAR_MAX_PROFILE_NUM)
+		return -EINVAL;
+
+	/* the tables start at element 3 */
+	pos = 3;
+
+	BUILD_BUG_ON(ACPI_SAR_NUM_CHAINS_REV0 != ACPI_SAR_NUM_CHAINS_REV1);
+	BUILD_BUG_ON(ACPI_SAR_NUM_CHAINS_REV2 != 2 * ACPI_SAR_NUM_CHAINS_REV0);
+
+	/* parse non-cdb chains for all profiles */
+	for (int i = 0; i < n_profiles; i++) {
+		union acpi_object *table = &wifi_pkg->package.elements[pos];
+
+		ret = iwl_acpi_parse_chains_table(table,
+						  sar_profiles[i + 1].chains,
+						  ACPI_SAR_NUM_CHAINS_REV0,
+						  num_sub_bands);
+		if (ret < 0)
+			return ret;
+
+		/* go to the next table */
+		pos += ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands;
+	}
+
+	if (!has_cdb)
+		goto set_enabled;
+
+	if (WARN_ON(ACPI_SAR_NUM_CHAINS_REV0 * 2 * num_sub_bands >
+		    ARRAY_SIZE(sar_profiles[0].chains) *
+		    ARRAY_SIZE(sar_profiles[0].chains[0].subbands)))
+		return -EINVAL;
+
+	/* parse cdb chains for all profiles */
+	for (int i = 0; i < n_profiles; i++) {
+		struct iwl_sar_profile_chain *chains;
+		union acpi_object *table;
+
+		table = &wifi_pkg->package.elements[pos];
+		chains = &sar_profiles[i + 1].chains[ACPI_SAR_NUM_CHAINS_REV0];
+		ret = iwl_acpi_parse_chains_table(table,
+						  chains,
+						  ACPI_SAR_NUM_CHAINS_REV0,
+						  num_sub_bands);
+		if (ret < 0)
+			return ret;
+
+		/* go to the next table */
+		pos += ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands;
+	}
+
+set_enabled:
+	for (int i = 0; i < n_profiles; i++)
+		sar_profiles[i + 1].enabled = enabled;
+
+	return 0;
+}
+
 int iwl_acpi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 {
 	union acpi_object *wifi_pkg, *table, *data;
@@ -629,8 +706,12 @@ int iwl_acpi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 	 */
 	ret = iwl_acpi_parse_chains_table(table, fwrt->sar_profiles[0].chains,
 					  num_chains, num_sub_bands);
-	if (!ret && flags & IWL_SAR_ENABLE_MSK)
+	if (ret)
+		goto out_free;
+
+	if (flags & IWL_SAR_ENABLE_MSK)
 		fwrt->sar_profiles[0].enabled = true;
+	fwrt->wrds_table_revision = tbl_rev;
 
 out_free:
 	kfree(data);
@@ -640,8 +721,7 @@ int iwl_acpi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 {
 	union acpi_object *wifi_pkg, *data;
-	bool enabled;
-	int i, n_profiles, tbl_rev, pos;
+	int tbl_rev;
 	int ret = 0;
 	u8 num_sub_bands;
 
@@ -713,93 +793,147 @@ int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 	goto out_free;
 
 read_table:
-	if (wifi_pkg->package.elements[1].type != ACPI_TYPE_INTEGER ||
-	    wifi_pkg->package.elements[2].type != ACPI_TYPE_INTEGER) {
-		ret = -EINVAL;
+	ret = iwl_acpi_parse_extended_sar_profiles(wifi_pkg,
+						   fwrt->sar_profiles,
+						   num_sub_bands,
+						   tbl_rev >= 2);
+	if (ret)
 		goto out_free;
+
+	fwrt->ewrd_table_revision = tbl_rev;
+
+out_free:
+	kfree(data);
+	return ret;
+}
+
+int iwl_acpi_get_wsss_table(struct iwl_fw_runtime *fwrt)
+{
+	union acpi_object *data __free(kfree) = NULL;
+	union acpi_object *wifi_pkg, *table;
+	u8 num_chains, num_sub_bands;
+	int ret, tbl_rev;
+	u32 flags;
+
+	if (fwrt->wrds_table_revision == IWL_BIOS_REVISION_UNSET) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: WRDS table was not read\n");
+		return -EINVAL;
 	}
 
-	if (WARN_ON(ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands >
-		    ARRAY_SIZE(fwrt->sar_profiles[0].chains) *
-		    ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands))) {
-		ret = -EINVAL;
-		goto out_free;
+	data = iwl_acpi_get_object(fwrt->dev, ACPI_WSSS_METHOD);
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	switch (fwrt->wrds_table_revision) {
+	case 3:
+		wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+						 ACPI_WRDS_WIFI_DATA_SIZE_REV3,
+						 &tbl_rev);
+		num_chains = ACPI_SAR_NUM_CHAINS_REV2;
+		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV3;
+		break;
+	case 2:
+		wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+						 ACPI_WRDS_WIFI_DATA_SIZE_REV2,
+						 &tbl_rev);
+		num_chains = ACPI_SAR_NUM_CHAINS_REV2;
+		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV2;
+		break;
+	default:
+		return -EINVAL;
 	}
 
-	enabled = !!(wifi_pkg->package.elements[1].integer.value);
-	n_profiles = wifi_pkg->package.elements[2].integer.value;
+	if (IS_ERR(wifi_pkg))
+		return PTR_ERR(wifi_pkg);
 
-	/*
-	 * Check the validity of n_profiles.  The EWRD profiles start
-	 * from index 1, so the maximum value allowed here is
-	 * ACPI_SAR_PROFILES_NUM - 1.
-	 */
-	if (n_profiles >= BIOS_SAR_MAX_PROFILE_NUM) {
-		ret = -EINVAL;
-		goto out_free;
+	if (fwrt->wrds_table_revision != tbl_rev) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: WRDS/WSSS revision mismatch (WRDS rev %d, WSSS rev %d)\n",
+				fwrt->wrds_table_revision,
+				tbl_rev);
+		return -EINVAL;
 	}
 
-	/* the tables start at element 3 */
-	pos = 3;
+	if (wifi_pkg->package.elements[1].type != ACPI_TYPE_INTEGER)
+		return -EINVAL;
 
-	BUILD_BUG_ON(ACPI_SAR_NUM_CHAINS_REV0 != ACPI_SAR_NUM_CHAINS_REV1);
-	BUILD_BUG_ON(ACPI_SAR_NUM_CHAINS_REV2 != 2 * ACPI_SAR_NUM_CHAINS_REV0);
+	if (WARN_ON(num_chains * num_sub_bands >
+		    ARRAY_SIZE(fwrt->sar_standalone_profiles[0].chains) *
+		    ARRAY_SIZE(fwrt->sar_standalone_profiles[0].chains[0].subbands)))
+		return -EINVAL;
 
-	/* parse non-cdb chains for all profiles */
-	for (i = 0; i < n_profiles; i++) {
-		union acpi_object *table = &wifi_pkg->package.elements[pos];
+	IWL_DEBUG_RADIO(fwrt, "Reading WSSS (WRDS Standalone) tbl_rev=%d\n",
+			tbl_rev);
 
-		/* The EWRD profiles officially go from 2 to 4, but we
-		 * save them in sar_profiles[1-3] (because we don't
-		 * have profile 0).  So in the array we start from 1.
-		 */
-		ret = iwl_acpi_parse_chains_table(table,
-						  fwrt->sar_profiles[i + 1].chains,
-						  ACPI_SAR_NUM_CHAINS_REV0,
-						  num_sub_bands);
-		if (ret < 0)
-			goto out_free;
+	flags = wifi_pkg->package.elements[1].integer.value;
 
-		/* go to the next table */
-		pos += ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands;
-	}
+	/* position of the actual table */
+	table = &wifi_pkg->package.elements[2];
 
-	/* non-cdb table revisions */
-	if (tbl_rev < 2)
-		goto set_enabled;
+	ret = iwl_acpi_parse_chains_table(table,
+					  fwrt->sar_standalone_profiles[0].chains,
+					  num_chains, num_sub_bands);
+	if (ret)
+		return ret;
 
-	if (WARN_ON(ACPI_SAR_NUM_CHAINS_REV0 * 2 * num_sub_bands >
-		    ARRAY_SIZE(fwrt->sar_profiles[0].chains) *
-		    ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands))) {
-		ret = -EINVAL;
-		goto out_free;
+	if (flags & IWL_SAR_ENABLE_MSK)
+		fwrt->sar_standalone_profiles[0].enabled = true;
+
+	return ret;
+}
+
+int iwl_acpi_get_ewss_table(struct iwl_fw_runtime *fwrt)
+{
+	union acpi_object *data __free(kfree) = NULL;
+	union acpi_object *wifi_pkg;
+	u8 num_sub_bands;
+	int tbl_rev;
+
+	if (fwrt->ewrd_table_revision == IWL_BIOS_REVISION_UNSET) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: EWRD table was not read\n");
+		return -EINVAL;
 	}
 
-	/* parse cdb chains for all profiles */
-	for (i = 0; i < n_profiles; i++) {
-		struct iwl_sar_profile_chain *chains;
-		union acpi_object *table;
+	data = iwl_acpi_get_object(fwrt->dev, ACPI_EWSS_METHOD);
+	if (IS_ERR(data))
+		return PTR_ERR(data);
 
-		table = &wifi_pkg->package.elements[pos];
-		chains = &fwrt->sar_profiles[i + 1].chains[ACPI_SAR_NUM_CHAINS_REV0];
-		ret = iwl_acpi_parse_chains_table(table,
-						  chains,
-						  ACPI_SAR_NUM_CHAINS_REV0,
-						  num_sub_bands);
-		if (ret < 0)
-			goto out_free;
+	switch (fwrt->ewrd_table_revision) {
+	case 3:
+		wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+						 ACPI_EWRD_WIFI_DATA_SIZE_REV3,
+						 &tbl_rev);
+		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV3;
+		break;
+	case 2:
+		wifi_pkg = iwl_acpi_get_wifi_pkg(fwrt->dev, data,
+						 ACPI_EWRD_WIFI_DATA_SIZE_REV2,
+						 &tbl_rev);
+		num_sub_bands = ACPI_SAR_NUM_SUB_BANDS_REV2;
+		break;
+	default:
+		return -EINVAL;
+	}
 
-		/* go to the next table */
-		pos += ACPI_SAR_NUM_CHAINS_REV0 * num_sub_bands;
+	if (IS_ERR(wifi_pkg))
+		return PTR_ERR(wifi_pkg);
+
+	if (fwrt->ewrd_table_revision != tbl_rev) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: EWRD/EWSS revision mismatch (EWRD rev %d, EWSS rev %d)\n",
+				fwrt->ewrd_table_revision, tbl_rev);
+		return -EINVAL;
 	}
 
-set_enabled:
-	for (i = 0; i < n_profiles; i++)
-		fwrt->sar_profiles[i + 1].enabled = enabled;
+	IWL_DEBUG_RADIO(fwrt, "Reading EWSS (EWRD Standalone) tbl_rev=%d\n",
+			tbl_rev);
 
-out_free:
-	kfree(data);
-	return ret;
+	return iwl_acpi_parse_extended_sar_profiles(wifi_pkg,
+						    fwrt->sar_standalone_profiles,
+						    num_sub_bands,
+						    fwrt->ewrd_table_revision >= 2);
 }
 
 int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
@@ -1153,7 +1287,8 @@ void iwl_acpi_get_guid_lock_status(struct iwl_fw_runtime *fwrt)
 	}
 
 	if (wifi_pkg->package.elements[1].type != ACPI_TYPE_INTEGER ||
-	    wifi_pkg->package.elements[1].integer.value > ACPI_GLAI_MAX_STATUS)
+	    wifi_pkg->package.elements[1].integer.value >
+	    UEFI_CNV_GUID_TEST_MODE)
 		goto out_free;
 
 	fwrt->uefi_tables_lock_status =
@@ -1165,7 +1300,6 @@ void iwl_acpi_get_guid_lock_status(struct iwl_fw_runtime *fwrt)
 out_free:
 	kfree(data);
 }
-IWL_EXPORT_SYMBOL(iwl_acpi_get_guid_lock_status);
 
 int iwl_acpi_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value)
 {
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h
index 45eb35ffb63760..b722beae2f0029 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h
@@ -14,6 +14,8 @@
 
 #define ACPI_WRDS_METHOD	"WRDS"
 #define ACPI_EWRD_METHOD	"EWRD"
+#define ACPI_WSSS_METHOD	"WSSS"
+#define ACPI_EWSS_METHOD	"EWSS"
 #define ACPI_WGDS_METHOD	"WGDS"
 #define ACPI_WRDD_METHOD	"WRDD"
 #define ACPI_SPLC_METHOD	"SPLC"
@@ -89,7 +91,6 @@
  * and one for the status
  */
 #define ACPI_GLAI_WIFI_DATA_SIZE	2
-#define ACPI_GLAI_MAX_STATUS		2
 /*
  * TAS size: 1 elelment for type,
  *	     1 element for enabled field,
@@ -111,9 +112,6 @@
 #define ACPI_PPAG_WIFI_DATA_SIZE_V3	((ACPI_PPAG_NUM_CHAINS * \
 					  ACPI_PPAG_NUM_BANDS_V3) + 2)
 
-/* The Inidcator whether UEFI WIFI GUID tables are locked is read from ACPI */
-#define UEFI_WIFI_GUID_UNLOCKED		0
-
 #define ACPI_DSM_REV 0
 
 #define DSM_INTERNAL_FUNC_GET_PLAT_INFO	1
@@ -179,6 +177,10 @@ int iwl_acpi_get_wrds_table(struct iwl_fw_runtime *fwrt);
 
 int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt);
 
+int iwl_acpi_get_wsss_table(struct iwl_fw_runtime *fwrt);
+
+int iwl_acpi_get_ewss_table(struct iwl_fw_runtime *fwrt);
+
 int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt);
 
 int iwl_acpi_get_tas_table(struct iwl_fw_runtime *fwrt,
@@ -233,6 +235,16 @@ static inline int iwl_acpi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 	return -ENOENT;
 }
 
+static inline int iwl_acpi_get_wsss_table(struct iwl_fw_runtime *fwrt)
+{
+	return -ENOENT;
+}
+
+static inline int iwl_acpi_get_ewss_table(struct iwl_fw_runtime *fwrt)
+{
+	return -ENOENT;
+}
+
 static inline int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 {
 	return 1;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
index c8eb016e1a7ee7..3fbcb2e4bfe069 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
@@ -522,17 +522,34 @@ struct iwl_rx_baid_cfg_cmd_alloc {
 } __packed; /* RX_BAID_ALLOCATION_ADD_CMD_API_S_VER_1 */
 
 /**
- * struct iwl_rx_baid_cfg_cmd_modify - BAID modification data
+ * struct iwl_rx_baid_cfg_cmd_modify_v2 - BAID modification data
  * @old_sta_id_mask: old station ID mask
  * @new_sta_id_mask: new station ID mask
  * @tid: TID of the BAID
  */
-struct iwl_rx_baid_cfg_cmd_modify {
+struct iwl_rx_baid_cfg_cmd_modify_v2 {
 	__le32 old_sta_id_mask;
 	__le32 new_sta_id_mask;
 	__le32 tid;
 } __packed; /* RX_BAID_ALLOCATION_MODIFY_CMD_API_S_VER_2 */
 
+/**
+ * struct iwl_rx_baid_cfg_cmd_modify - BAID modification data
+ * @old_sta_id_mask: old station ID mask
+ * @new_sta_id_mask: new station ID mask
+ * @tid: TID of the BAID
+ * @reserved: reserved
+ * @win_size: RX BA session window size to apply to the modified BAID,
+ *	0 means keep the current one
+ */
+struct iwl_rx_baid_cfg_cmd_modify {
+	__le32 old_sta_id_mask;
+	__le32 new_sta_id_mask;
+	u8 tid;
+	u8 reserved;
+	__le16 win_size;
+} __packed; /* RX_BAID_ALLOCATION_MODIFY_CMD_API_S_VER_3 */
+
 /**
  * struct iwl_rx_baid_cfg_cmd_remove_v1 - BAID removal data
  * @baid: the BAID to remove
@@ -555,7 +572,8 @@ struct iwl_rx_baid_cfg_cmd_remove {
  * struct iwl_rx_baid_cfg_cmd - BAID allocation/config command
  * @action: the action, from &enum iwl_rx_baid_action
  * @alloc: allocation data
- * @modify: modify data
+ * @modify_v2: modify data (version 2)
+ * @modify: modify data (version 3)
  * @remove_v1: remove data (version 1)
  * @remove: remove data
  */
@@ -563,11 +581,12 @@ struct iwl_rx_baid_cfg_cmd {
 	__le32 action;
 	union {
 		struct iwl_rx_baid_cfg_cmd_alloc alloc;
+		struct iwl_rx_baid_cfg_cmd_modify_v2 modify_v2;
 		struct iwl_rx_baid_cfg_cmd_modify modify;
 		struct iwl_rx_baid_cfg_cmd_remove_v1 remove_v1;
 		struct iwl_rx_baid_cfg_cmd_remove remove;
-	}; /* RX_BAID_ALLOCATION_OPERATION_API_U_VER_2 */
-} __packed; /* RX_BAID_ALLOCATION_CONFIG_CMD_API_S_VER_2 */
+	}; /* RX_BAID_ALLOCATION_OPERATION_API_U_VER_3 */
+} __packed; /* RX_BAID_ALLOCATION_CONFIG_CMD_API_S_VER_3 */
 
 /**
  * struct iwl_rx_baid_cfg_resp - BAID allocation response
@@ -643,6 +662,7 @@ struct iwl_scd_queue_cfg_cmd {
  * @IWL_SEC_KEY_FLAG_MFP: MFP is in used for this key
  * @IWL_SEC_KEY_FLAG_MCAST_KEY: this is a multicast key
  * @IWL_SEC_KEY_FLAG_SPP_AMSDU: SPP A-MSDU should be used
+ * @IWL_SEC_KEY_FLAG_ASSOC_ENC: use this key for (Re)Assoc Req/Resp encryption
  */
 enum iwl_sec_key_flags {
 	IWL_SEC_KEY_FLAG_CIPHER_MASK	= 0x07,
@@ -655,6 +675,7 @@ enum iwl_sec_key_flags {
 	IWL_SEC_KEY_FLAG_MFP		= 0x20,
 	IWL_SEC_KEY_FLAG_MCAST_KEY	= 0x40,
 	IWL_SEC_KEY_FLAG_SPP_AMSDU	= 0x80,
+	IWL_SEC_KEY_FLAG_ASSOC_ENC	= 0x200,
 };
 
 #define IWL_SEC_WEP_KEY_OFFSET	3
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
index 09b2cddc4ad2de..9579e691cc8772 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
@@ -7,6 +7,8 @@
 #ifndef __iwl_fw_api_mac_cfg_h__
 #define __iwl_fw_api_mac_cfg_h__
 
+#include <linux/ieee80211-p2p.h>
+
 #include "mac.h"
 #include "phy-ctxt.h"
 
@@ -663,7 +665,13 @@ struct iwl_npca_params {
  * @puncture_mask: puncture mask for EHT (removed in VER_3)
  * @frame_time_rts_th: HE duration RTS threshold, in units of 32us
  * @flags: a combination from &enum iwl_link_ctx_flags
- * @flags_mask: what of %flags have changed. Also &enum iwl_link_ctx_flags
+ * @flags_mask: what of %flags have changed. Also &enum iwl_link_ctx_flags.
+ *	Removed in _VER_6, reused as @reserved4 / @max_bssid_indicator
+ *	in _VER_9.
+ * @reserved4: reserved
+ * @max_bssid_indicator: N where 2**N is the max number of BSSIDs the physical
+ *	AP can support (multi-BSS or EMCCA set). 0 disables the feature.
+ *	Since _VER_9.
  * Below fields are for multi-bssid:
  * @ref_bssid_addr: reference BSSID used by the AP
  * @reserved_for_ref_bssid_addr: reserved
@@ -704,17 +712,23 @@ struct iwl_link_config_cmd {
 	/* MAC_QOS_PARAM_API_S_VER_1 */
 	__le32 qos_flags;
 	struct iwl_ac_qos ac[AC_NUM + 1];
-	u8 htc_trig_based_pkt_ext;
-	u8 rand_alloc_ecwmin;
-	u8 rand_alloc_ecwmax;
+	u8 htc_trig_based_pkt_ext; /* removed in _VER_9 */
+	u8 rand_alloc_ecwmin;      /* removed in _VER_9 */
+	u8 rand_alloc_ecwmax;      /* removed in _VER_9 */
 	u8 ndp_fdbk_buff_th_exp;
 	struct iwl_he_backoff_conf trig_based_txf[AC_NUM];
 	__le32 bi;
 	__le32 dtim_interval;
 	__le16 puncture_mask; /* removed in _VER_3 */
-	__le16 frame_time_rts_th;
+	__le16 frame_time_rts_th; /* removed in _VER_9 */
 	__le32 flags;
-	__le32 flags_mask; /* removed in _VER_6 */
+	union {
+		__le32 flags_mask; /* removed in _VER_6 */
+		struct {
+			u8 reserved4[3];
+			u8 max_bssid_indicator; /* since _VER_9 */
+		};
+	};
 	/* The below fields are for multi-bssid */
 	u8 ref_bssid_addr[6];
 	__le16 reserved_for_ref_bssid_addr;
@@ -728,7 +742,9 @@ struct iwl_link_config_cmd {
 	struct iwl_ac_qos prio_edca_params; /* since _VER_7 */
 	__le32 reserved3[4];
 } __packed; /* LINK_CONTEXT_CONFIG_CMD_API_S_VER_1, _VER_2, _VER_3, _VER_4,
-	     *				    _VER_5, _VER_6, _VER_7, _VER_8 */
+	     *				    _VER_5, _VER_6, _VER_7, _VER_8,
+	     *				    _VER_9
+	     */
 
 /* Currently FW supports link ids in the range 0-3 and can have
  * at most two active links for each vif.
@@ -868,8 +884,8 @@ struct iwl_sta_cfg_cmd_v1 {
  * @dps_pad_time: DPS (Dynamic Power Save) padding delay resolution to ensure
  *	proper timing alignment
  * @dps_trans_delay: DPS minimal time that takes the peer to return to low power
- * @mic_prep_pad_delay: MIC prep time padding
- * @mic_compute_pad_delay: MIC compute time padding
+ * @mic_prep_pad_delay: MIC prep time padding (4 us resolution)
+ * @mic_compute_pad_delay: MIC compute time padding (4 us resolution)
  * @reserved: Reserved for alignment
  */
 struct iwl_sta_cfg_cmd_v2 {
@@ -942,8 +958,8 @@ struct iwl_sta_cfg_cmd_v2 {
  *	proper timing alignment
  * @dps_trans_delay: DPS minimal time that takes the peer to return to low power
  * @dps_enabled: flag indicating whether or not DPS is enabled
- * @mic_prep_pad_delay: MIC prep time padding
- * @mic_compute_pad_delay: MIC compute time padding
+ * @mic_prep_pad_delay: MIC prep time padding (4 us resolution)
+ * @mic_compute_pad_delay: MIC compute time padding (4 us resolution)
  * @nmi_sta_id: for an NDI peer STA, the NMI peer STA ID it relates to
  * @ndi_local_addr: for an NDI peer STA or NAN multicast data station,
  *	the local NDI interface MAC address
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
index 115e65ba19f8c4..4d54a95b98ca67 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
@@ -300,7 +300,8 @@ enum iwl_dev_tx_power_cmd_mode {
 	IWL_TX_POWER_MODE_SET_ACK = 3,
 	IWL_TX_POWER_MODE_SET_SAR_TIMER = 4,
 	IWL_TX_POWER_MODE_SET_SAR_TIMER_DEFAULT_TABLE = 5,
-}; /* TX_POWER_REDUCED_FLAGS_TYPE_API_E_VER_5 */;
+	IWL_TX_POWER_MODE_SET_STANDALONE_CHAINS = 8, /* cmd ver 12+ */
+}; /* TX_POWER_REDUCED_FLAGS_TYPE_API_E_VER_2 */;
 
 #define IWL_NUM_CHAIN_TABLES	1
 #define IWL_NUM_CHAIN_TABLES_V2	2
@@ -464,31 +465,34 @@ struct iwl_dev_tx_power_cmd_v10 {
 	__le32 flags;
 } __packed; /* TX_REDUCED_POWER_API_S_VER_10 */
 
-struct iwl_dev_tx_power_cmd_v11 {
+struct iwl_dev_tx_power_cmd_v12 {
 	__le16 per_chain[IWL_NUM_CHAIN_TABLES_V2][IWL_NUM_CHAIN_LIMITS][IWL_NUM_SUB_BANDS_V3];
 	u8 per_chain_restriction_changed;
 	u8 reserved;
 	__le32 timer_period;
 	__le32 flags;
-} __packed; /* TX_REDUCED_POWER_API_S_VER_11 */
+} __packed; /* TX_REDUCED_POWER_API_S_VER_11
+	     * TX_REDUCED_POWER_API_S_VER_12
+	     */
 
 /*
  * struct iwl_dev_tx_power_cmd - TX power reduction command (multiversion)
  * @common: common part of the command
  * @v9: version 9 part of the command
  * @v10: version 10 part of the command
- * @v11: version 11 part of the command
+ * @v12: version 12 part of the command (also covers v11)
  */
 struct iwl_dev_tx_power_cmd {
 	struct iwl_dev_tx_power_common common;
 	union {
 		struct iwl_dev_tx_power_cmd_v9 v9;
 		struct iwl_dev_tx_power_cmd_v10 v10;
-		struct iwl_dev_tx_power_cmd_v11 v11;
+		struct iwl_dev_tx_power_cmd_v12 v12;
 	};
 } __packed; /* TX_REDUCED_POWER_API_S_VER_9
 	     * TX_REDUCED_POWER_API_S_VER_10
 	     * TX_REDUCED_POWER_API_S_VER_11
+	     * TX_REDUCED_POWER_API_S_VER_12
 	     */
 
 #define IWL_NUM_GEO_PROFILES		3
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
index 699343cf027993..20460534fc99d0 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
@@ -583,7 +583,11 @@ struct iwl_rx_mpdu_desc_v3 {
 	/**
 	 * @reserved_xsum: reserved high bits in the raw checksum
 	 */
-	__le16 reserved_xsum;
+	u8 reserved_xsum;
+	/**
+	 * @mac_context: MAC context mask, in this DW only from API version 10
+	 */
+	u8 mac_context;
 	/* DW11 */
 	/**
 	 * @rate_n_flags: RX rate/flags encoding
@@ -603,9 +607,10 @@ struct iwl_rx_mpdu_desc_v3 {
 	 */
 	u8 channel;
 	/**
-	 * @mac_context: MAC context mask
+	 * @ru: RU/MRU/DRU used for the frame, in trigger-like format;
+	 *	only from API version 10, this byte held @mac_context before
 	 */
-	u8 mac_context;
+	u8 ru;
 	/* DW13 */
 	/**
 	 * @gp2_on_air_rise: GP2 timer value on air rise (INA)
@@ -647,7 +652,11 @@ struct iwl_rx_mpdu_desc_v3 {
 	__le32 reserved[1];
 } __packed; /* RX_MPDU_RES_START_API_S_VER_3,
 	     * RX_MPDU_RES_START_API_S_VER_5,
-	     * RX_MPDU_RES_START_API_S_VER_6
+	     * RX_MPDU_RES_START_API_S_VER_6,
+	     * RX_MPDU_RES_START_API_S_VER_7,
+	     * RX_MPDU_RES_START_API_S_VER_8,
+	     * RX_MPDU_RES_START_API_S_VER_9,
+	     * RX_MPDU_RES_START_API_S_VER_10
 	     */
 
 /**
@@ -737,6 +746,8 @@ struct iwl_rx_mpdu_desc {
 	     * RX_MPDU_RES_START_API_S_VER_6
 	     * RX_MPDU_RES_START_API_S_VER_7
 	     * RX_MPDU_RES_START_API_S_VER_8
+	     * RX_MPDU_RES_START_API_S_VER_9
+	     * RX_MPDU_RES_START_API_S_VER_10
 	     */
 
 #define IWL_RX_DESC_SIZE_V1 offsetofend(struct iwl_rx_mpdu_desc, v1)
@@ -1347,7 +1358,7 @@ struct iwl_uhr_elr_sigs {
 #define OFDM_RX_VECTOR_UHR_ELR_PPDU_TYPE		0x00000003
 #define OFDM_RX_VECTOR_UHR_ELR_USIG_STA_ID		0x00001ffc
 #define OFDM_RX_VECTOR_UHR_ELR_VALIDATE			0x0000e000
-#define OFDM_RX_VECTOR_UHR_ELR_CRC_OK			0x00010000
+#define OFDM_RX_VECTOR_UHR_ELR_CRC_OK			0x40000000
 	__le32 usig_a2_uhr_elr;
 #define OFDM_RX_VECTOR_UHR_ELR_VER_ID			0x00000007
 #define OFDM_RX_VECTOR_UHR_ELR_UPLINK_FLAG		0x00000008
@@ -1363,7 +1374,7 @@ struct iwl_uhr_elr_sigs {
 #define OFDM_RX_VECTOR_UHR_ELR_SIG_ID_INDX		0x00000e00
 #define OFDM_RX_VECTOR_UHR_ELR_STA_RU			0x000ff000
 #define OFDM_RX_VECTOR_UHR_ELR_STA_RU_PS160		0x00100000
-#define OFDM_RX_VECTOR_UHR_ELR_SIG2_CRC_OK		0x00200000
+#define OFDM_RX_VECTOR_UHR_ELR_SIG2_CRC_OK		0x40000000
 	__le32 uhr_sig_elr2;
 };
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
index ee78371082b094..8994a7b840ef82 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
@@ -987,6 +987,8 @@ struct iwl_scan_probe_params_v4 {
 #define SCAN_MAX_NUM_CHANS_V3 67
 #define SCAN_MAX_NUM_CHANS_V4 68
 
+#define SCAN_MAX_NUM_NOTIF_RESULTS 112
+
 /**
  * struct iwl_scan_channel_params_v4 - channel params
  * @flags: channel flags &enum iwl_scan_channel_flags
@@ -1232,7 +1234,8 @@ struct iwl_umac_scan_complete {
 } __packed; /* SCAN_COMPLETE_NTF_UMAC_API_S_VER_1 */
 
 #define SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V1 5
-#define SCAN_OFFLOAD_MATCHING_CHANNELS_LEN    7
+#define SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V2 7
+#define SCAN_OFFLOAD_MATCHING_CHANNELS_LEN    16
 
 /**
  * struct iwl_scan_offload_profile_match_v1 - match information
@@ -1278,11 +1281,31 @@ struct iwl_scan_offload_profiles_query_v1 {
 	struct iwl_scan_offload_profile_match_v1 matches[];
 } __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_2 */
 
+/**
+ * struct iwl_scan_offload_profile_match_v2 - match information
+ * @bssid: matched bssid
+ * @reserved: reserved
+ * @channel: channel where the match occurred
+ * @energy: energy
+ * @matching_feature: feature matches
+ * @matching_channels: bitmap of channels that matched, referencing
+ *	the channels passed in the scan offload request.
+ */
+struct iwl_scan_offload_profile_match_v2 {
+	u8 bssid[ETH_ALEN];
+	__le16 reserved;
+	u8 channel;
+	u8 energy;
+	u8 matching_feature;
+	u8 matching_channels[SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V2];
+} __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_2 */
+
 /**
  * struct iwl_scan_offload_profile_match - match information
  * @bssid: matched bssid
  * @reserved: reserved
  * @channel: channel where the match occurred
+ * @band: band where the match occurred
  * @energy: energy
  * @matching_feature: feature matches
  * @matching_channels: bitmap of channels that matched, referencing
@@ -1292,10 +1315,11 @@ struct iwl_scan_offload_profile_match {
 	u8 bssid[ETH_ALEN];
 	__le16 reserved;
 	u8 channel;
+	u8 band;
 	u8 energy;
 	u8 matching_feature;
 	u8 matching_channels[SCAN_OFFLOAD_MATCHING_CHANNELS_LEN];
-} __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_2 */
+} __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_3 */
 
 /**
  * struct iwl_scan_offload_match_info - match results information
@@ -1320,12 +1344,13 @@ struct iwl_scan_offload_match_info {
 	u8 self_recovery;
 	__le16 reserved;
 	struct iwl_scan_offload_profile_match matches[IWL_SCAN_MAX_PROFILES_V2];
-} __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_3 and
+} __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_5 and
 	     * SCAN_OFFLOAD_MATCH_INFO_NOTIFICATION_S_VER_1
 	     */
 
 /**
- * struct iwl_umac_scan_iter_complete_notif - notifies end of scanning iteration
+ * struct iwl_umac_scan_iter_complete_notif_v2 - notifies end of scanning
+ *	iteration
  * @uid: scan id, &enum iwl_umac_scan_uid_offsets
  * @scanned_channels: number of channels scanned and number of valid elements in
  *	results array
@@ -1336,7 +1361,7 @@ struct iwl_scan_offload_match_info {
  *	in &struct iwl_scan_req_umac.
  * @results: array of scan results, length in @scanned_channels
  */
-struct iwl_umac_scan_iter_complete_notif {
+struct iwl_umac_scan_iter_complete_notif_v2 {
 	__le32 uid;
 	u8 scanned_channels;
 	u8 status;
@@ -1346,6 +1371,39 @@ struct iwl_umac_scan_iter_complete_notif {
 	struct iwl_scan_results_notif results[];
 } __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_2 */
 
+/**
+ * struct iwl_umac_scan_iter_complete_notif - notifies end of scanning
+ *	iteration
+ * @uid: scan id, &enum iwl_umac_scan_uid_offsets
+ * @scanned_channels: number of channels scanned and number of valid elements in
+ *	results array
+ * @status: one of SCAN_COMP_STATUS_*
+ * @bt_status: BT on/off status
+ * @last_channel: last channel that was scanned
+ * @start_tsf: TSF timer in usecs of the scan start time for the mac specified
+ *	in &struct iwl_scan_req_umac.
+ * @results: array of scan results, only @scanned_channels of them are valid
+ * @ebs_status: EBS completion status, &enum iwl_scan_ebs_status
+ * @reserved: reserved for alignment
+ * @ebs_channel_status: only valid when @ebs_status is %IWL_SCAN_EBS_SUCCESS.
+ *	Bitmap of the 5 GHz channels on which EBS found energy:
+ *	bit i in [0..27]:  channel = 36 + 4 * i   (channels 36 - 144)
+ *	bit i in [28..35]: channel = 37 + 4 * i   (channels 149 - 177)
+ *	bit 36: channel 181 (reserved, not currently covered by EBS)
+ */
+struct iwl_umac_scan_iter_complete_notif {
+	__le32 uid;
+	u8 scanned_channels;
+	u8 status;
+	u8 bt_status;
+	u8 last_channel;
+	__le64 start_tsf;
+	struct iwl_scan_results_notif results[SCAN_MAX_NUM_NOTIF_RESULTS];
+	u8 ebs_status;
+	u8 reserved[3];
+	__le64 ebs_channel_status;
+} __packed; /* SCAN_ITER_COMPLETE_NTF_UMAC_API_S_VER_3 */
+
 /**
  * struct iwl_umac_scan_channel_survey_notif - data for survey
  * @channel: the channel scanned
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
index 8d16782a129ce7..e3bf5d08c7e7f9 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
@@ -615,6 +615,37 @@ struct iwl_stats_ntfy_coex {
 	__le16 reserved;
 } __packed; /* STATISTICS_FW_NTFY_COEX_TELEMETRY_API_S_VER_1 */
 
+/**
+ * struct iwl_stats_ntfy_per_phy_v2 - per PHY statistics
+ * @channel_load: channel load
+ * @channel_load_by_us: device contribution to MCLM
+ * @channel_load_not_by_us: other devices' contribution to MCLM
+ * @clt: CLT HW timer (TIM_CH_LOAD2)
+ * @act: active accumulator SW
+ * @elp: elapsed time accumulator SW
+ * @rx_detected_per_ch_width: number of deferred TX per channel width,
+ *	0 - 20, 1/2/3 - 40/80/160
+ * @success_per_ch_width: number of frames that got ACK/BACK/CTS
+ *	per channel BW. note, BACK counted as 1
+ * @fail_per_ch_width: number of frames that didn't get ACK/BACK/CTS
+ *	per channel BW. note BACK counted as 1
+ * @last_tx_ch_width_indx: last txed frame channel width index
+ * @coex: coex related data
+ */
+struct iwl_stats_ntfy_per_phy_v2 {
+	__le32 channel_load;
+	__le32 channel_load_by_us;
+	__le32 channel_load_not_by_us;
+	__le32 clt;
+	__le32 act;
+	__le32 elp;
+	__le32 rx_detected_per_ch_width[IWL_STATS_MAX_BW_INDEX];
+	__le32 success_per_ch_width[IWL_STATS_MAX_BW_INDEX];
+	__le32 fail_per_ch_width[IWL_STATS_MAX_BW_INDEX];
+	__le32 last_tx_ch_width_indx;
+	struct iwl_stats_ntfy_coex coex;
+} __packed; /* STATISTICS_NTFY_PER_PHY_API_S_VER_2 */
+
 /**
  * struct iwl_stats_ntfy_per_phy - per PHY statistics
  * @channel_load: channel load
@@ -631,6 +662,10 @@ struct iwl_stats_ntfy_coex {
  *	per channel BW. note BACK counted as 1
  * @last_tx_ch_width_indx: last txed frame channel width index
  * @coex: coex related data
+ * @channel_load_npca_by_us: device NPCA contribution to MCLM
+ * @channel_load_npca_obss: the time FW identified for NPCA, not necessarily
+ *	entered to NPCA state, but can be used as an indication for NPCA
+ *	opportunities in the environment
  */
 struct iwl_stats_ntfy_per_phy {
 	__le32 channel_load;
@@ -644,7 +679,9 @@ struct iwl_stats_ntfy_per_phy {
 	__le32 fail_per_ch_width[IWL_STATS_MAX_BW_INDEX];
 	__le32 last_tx_ch_width_indx;
 	struct iwl_stats_ntfy_coex coex;
-} __packed; /* STATISTICS_NTFY_PER_PHY_API_S_VER_2 */
+	__le32 channel_load_npca_by_us;
+	__le32 channel_load_npca_obss;
+} __packed; /* STATISTICS_NTFY_PER_PHY_API_S_VER_3 */
 
 /* unknown channel load (due to not being active on channel) */
 #define IWL_STATS_UNKNOWN_CHANNEL_LOAD	0xffffffff
@@ -675,12 +712,27 @@ struct iwl_system_statistics_notif_oper_v3 {
 	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
 } __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_3 */
 
+/**
+ * struct iwl_system_statistics_notif_oper_v4 - statistics notification
+ *
+ * @time_stamp: time when the notification is sent from firmware
+ * @per_link: per link statistics, &struct iwl_stats_ntfy_per_link
+ * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy_v2
+ * @per_sta: per sta statistics, &struct iwl_stats_ntfy_per_sta
+ */
+struct iwl_system_statistics_notif_oper_v4 {
+	__le32 time_stamp;
+	struct iwl_stats_ntfy_per_link per_link[IWL_FW_MAX_LINKS];
+	struct iwl_stats_ntfy_per_phy_v2 per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
+	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
+} __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_4 */
+
 /**
  * struct iwl_system_statistics_notif_oper - statistics notification
  *
  * @time_stamp: time when the notification is sent from firmware
  * @per_link: per link statistics, &struct iwl_stats_ntfy_per_link
- * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy_v1
+ * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy
  * @per_sta: per sta statistics, &struct iwl_stats_ntfy_per_sta
  */
 struct iwl_system_statistics_notif_oper {
@@ -688,7 +740,7 @@ struct iwl_system_statistics_notif_oper {
 	struct iwl_stats_ntfy_per_link per_link[IWL_FW_MAX_LINKS];
 	struct iwl_stats_ntfy_per_phy per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
 	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
-} __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_4 */
+} __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_5 */
 
 /**
  * struct iwl_system_statistics_part1_notif_oper - part1 stats notification
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
index 6621ef9358bce2..897538066b7dea 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
@@ -344,6 +344,9 @@ iwl_dump_ini_mon_dram_iter(struct iwl_fw_runtime *fwrt,
 	struct iwl_dram_data *frag;
 	u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
 
+	if (WARN_ON_ONCE(alloc_id >= ARRAY_SIZE(fwrt->trans->dbg.fw_mon_ini)))
+		return -EINVAL;
+
 	frag = &fwrt->trans->dbg.fw_mon_ini[alloc_id].frags[idx];
 
 	range->dram_base_addr = cpu_to_le64(frag->physical);
@@ -957,6 +960,9 @@ iwl_dump_ini_mon_dram_ranges(struct iwl_fw_runtime *fwrt,
 	u32 ranges = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
 	int i;
 
+	if (WARN_ON_ONCE(alloc_id >= ARRAY_SIZE(fwrt->trans->dbg.fw_mon_ini)))
+		return 0;
+
 	fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
 
 	for (i = 0; i < fw_mon->num_frags; i++) {
@@ -1016,8 +1022,16 @@ static u32 iwl_dump_ini_mem_get_size(struct iwl_fw_runtime *fwrt,
 	if (!size || !ranges)
 		return 0;
 
-	return sizeof(struct iwl_fw_ini_error_dump) + ranges *
-		(size + sizeof(struct iwl_fw_ini_error_dump_range));
+	if (check_add_overflow(size,
+			       (u32)sizeof(struct iwl_fw_ini_error_dump_range),
+			       &size) ||
+	    check_mul_overflow(ranges, size, &size) ||
+	    check_add_overflow(size,
+			       (u32)sizeof(struct iwl_fw_ini_error_dump),
+			       &size))
+		return 0;
+
+	return size;
 }
 
 static u32
@@ -1028,15 +1042,24 @@ iwl_dump_ini_mem_block_get_size(struct iwl_fw_runtime *fwrt,
 	struct iwl_fw_ini_addr_size *pairs = (void *)reg->addrs;
 	u32 ranges = iwl_dump_ini_mem_block_ranges(fwrt, reg_data);
 	u32 size = sizeof(struct iwl_fw_ini_error_dump);
+	u32 range_hdrs;
 	int range;
 
 	if (!ranges)
 		return 0;
 
 	for (range = 0; range < ranges; range++)
-		size += le32_to_cpu(pairs[range].size);
+		if (check_add_overflow(size, le32_to_cpu(pairs[range].size),
+				       &size))
+			return 0;
 
-	return size + ranges * sizeof(struct iwl_fw_ini_error_dump_range);
+	if (check_mul_overflow(ranges,
+			       (u32)sizeof(struct iwl_fw_ini_error_dump_range),
+			       &range_hdrs) ||
+	    check_add_overflow(size, range_hdrs, &size))
+		return 0;
+
+	return size;
 }
 
 static u32
@@ -1068,6 +1091,9 @@ iwl_dump_ini_mon_dram_get_size(struct iwl_fw_runtime *fwrt,
 	u32 size = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
 	int i;
 
+	if (WARN_ON_ONCE(alloc_id >= ARRAY_SIZE(fwrt->trans->dbg.fw_mon_ini)))
+		return 0;
+
 	fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
 
 	for (i = 0; i < fw_mon->num_frags; i++) {
@@ -1112,8 +1138,16 @@ static u32 iwl_dump_ini_mon_dbgi_get_size(struct iwl_fw_runtime *fwrt,
 	if (!size || !ranges)
 		return 0;
 
-	return sizeof(struct iwl_fw_ini_monitor_dump) + ranges *
-		(size + sizeof(struct iwl_fw_ini_error_dump_range));
+	if (check_add_overflow(size,
+			       (u32)sizeof(struct iwl_fw_ini_error_dump_range),
+			       &size) ||
+	    check_mul_overflow(ranges, size, &size) ||
+	    check_add_overflow(size,
+			       (u32)sizeof(struct iwl_fw_ini_monitor_dump),
+			       &size))
+		return 0;
+
+	return size;
 }
 
 static u32 iwl_dump_ini_txf_get_size(struct iwl_fw_runtime *fwrt,
@@ -2021,7 +2055,7 @@ int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
 		if (trigger->flags & IWL_FW_DBG_FORCE_RESTART) {
 			IWL_WARN(fwrt, "Force restart: trigger %d fired.\n",
 				 trig);
-			iwl_force_nmi(fwrt->trans);
+			iwl_trans_force_nmi(fwrt->trans);
 			return 0;
 		}
 
@@ -2199,7 +2233,8 @@ static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
 	}
 
 	if (fwrt->trans->dbg.last_tp_resetfw == IWL_FW_INI_RESET_FW_MODE_STOP_FW_ONLY)
-		iwl_force_nmi(fwrt->trans);
+		iwl_trans_force_nmi(fwrt->trans);
+
 out:
 	if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
 		iwl_fw_error_dump_data_free(dump_data);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index 664ac25754e9f4..3a986b394dd822 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -452,6 +452,7 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
  *	table data. For LARI_CONFIG_CHANGE command version 13 and above, this
  *	capability is obsolete since raw DSM values are accepted by default.
  * @IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT: Supports NAN synchronization
+ * @IWL_UCODE_TLV_CAPA_ASSOC_ENC_SUPPORT: firmware supports association frame encryption
  *
  * @NUM_IWL_UCODE_TLV_CAPA: number of bits used
  */
@@ -556,6 +557,7 @@ enum iwl_ucode_tlv_capa {
 	IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT		= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 0),
 	IWL_UCODE_TLV_CAPA_FW_ACCEPTS_RAW_DSM_TABLE 	= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 1),
 	IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT		= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 2),
+	IWL_UCODE_TLV_CAPA_ASSOC_ENC_SUPPORT		= (__force iwl_ucode_tlv_capa_t)(4 * 32 + 4),
 	NUM_IWL_UCODE_TLV_CAPA
 /*
  * This construction make both sparse (which cannot increment the previous
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/init.c b/drivers/net/wireless/intel/iwlwifi/fw/init.c
index d1d8058ad29f46..f62c6a69f0dc28 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/init.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/init.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2019-2021, 2024-2025 Intel Corporation
+ * Copyright (C) 2019-2021, 2024-2026 Intel Corporation
  */
 #include "iwl-drv.h"
 #include "runtime.h"
@@ -23,6 +23,8 @@ void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
 	int i;
 
 	memset(fwrt, 0, sizeof(*fwrt));
+	fwrt->wrds_table_revision = IWL_BIOS_REVISION_UNSET;
+	fwrt->ewrd_table_revision = IWL_BIOS_REVISION_UNSET;
 	fwrt->trans = trans;
 	fwrt->fw = fw;
 	fwrt->dev = trans->dev;
@@ -31,6 +33,8 @@ void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
 	fwrt->sanitize_ops = sanitize_ops;
 	fwrt->sanitize_ctx = sanitize_ctx;
 	fwrt->ops_ctx = ops_ctx;
+	/* default until the connectivity GUID lock status is resolved */
+	fwrt->uefi_tables_lock_status = UEFI_CNV_GUID_UNKNOWN;
 	for (i = 0; i < IWL_FW_RUNTIME_DUMP_WK_NUM; i++) {
 		fwrt->dump.wks[i].idx = i;
 		INIT_DELAYED_WORK(&fwrt->dump.wks[i].wk, iwl_fw_error_dump_wk);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
index 1d6d38ee55b499..c7de45323935e8 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
@@ -11,9 +11,7 @@
 
 #define GET_BIOS_TABLE(__name, ...)					\
 do {									\
-	int ret = -ENOENT;						\
-	if (fwrt->uefi_tables_lock_status > UEFI_WIFI_GUID_UNLOCKED)	\
-		ret = iwl_uefi_get_ ## __name(__VA_ARGS__);		\
+	int ret = iwl_uefi_get_ ## __name(__VA_ARGS__);			\
 	if (ret < 0)							\
 		ret = iwl_acpi_get_ ## __name(__VA_ARGS__);		\
 	return ret;							\
@@ -32,6 +30,8 @@ IWL_EXPORT_SYMBOL(iwl_bios_get_ ## __name)
 
 IWL_BIOS_TABLE_LOADER(wrds_table);
 IWL_BIOS_TABLE_LOADER(ewrd_table);
+IWL_BIOS_TABLE_LOADER(wsss_table);
+IWL_BIOS_TABLE_LOADER(ewss_table);
 IWL_BIOS_TABLE_LOADER(wgds_table);
 IWL_BIOS_TABLE_LOADER(ppag_table);
 IWL_BIOS_TABLE_LOADER(phy_filters);
@@ -49,11 +49,16 @@ static const struct dmi_system_id dmi_ppag_approved_list[] = {
 			DMI_MATCH(DMI_SYS_VENDOR, "HP"),
 		},
 	},
-	{ .ident = "SAMSUNG",
+	{ .ident = "SAMSUNG_ELECTRONICS",
 	  .matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD"),
 		},
 	},
+	{ .ident = "SAMSUNG",
+	  .matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Samsung"),
+		},
+	},
 	{ .ident = "MSFT",
 	  .matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Microsoft Corporation"),
@@ -126,10 +131,15 @@ static const struct dmi_system_id dmi_tas_approved_list[] = {
 			DMI_MATCH(DMI_SYS_VENDOR, "HP"),
 		},
 	},
-	{ .ident = "SAMSUNG",
+	{ .ident = "SAMSUNG_ELECTRONICS",
 	  .matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD"),
 		},
+	},
+	{ .ident = "SAMSUNG",
+	  .matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Samsung"),
+		},
 	},
 		{ .ident = "LENOVO",
 	  .matches = {
@@ -167,6 +177,12 @@ static const struct dmi_system_id dmi_tas_approved_list[] = {
 			DMI_MATCH(DMI_BOARD_VENDOR, "HP"),
 		},
 	},
+	{ .ident = "GOOGLE-ASUS",
+	  .matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Google"),
+			DMI_MATCH(DMI_BOARD_VENDOR, "ASUS"),
+		},
+	},
 	{ .ident = "MSI",
 	  .matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Micro-Star International Co., Ltd."),
@@ -241,17 +257,22 @@ IWL_EXPORT_SYMBOL(iwl_sar_geo_fill_table);
 
 static int iwl_sar_fill_table(struct iwl_fw_runtime *fwrt,
 			      __le16 *per_chain, u32 n_subbands,
-			      int prof_a, int prof_b)
+			      int prof_a, int prof_b,
+			      const struct iwl_sar_profile *profiles,
+			      bool is_standalone)
 {
 	int profs[BIOS_SAR_NUM_CHAINS] = { prof_a, prof_b };
 	int i, j;
 
+	BUILD_BUG_ON(ARRAY_SIZE(fwrt->sar_standalone_profiles[0].chains[0].subbands) !=
+		     ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands));
+
 	if (WARN_ON_ONCE(n_subbands >
 			 ARRAY_SIZE(fwrt->sar_profiles[0].chains[0].subbands)))
 		return -EINVAL;
 
 	for (i = 0; i < BIOS_SAR_NUM_CHAINS; i++) {
-		struct iwl_sar_profile *prof;
+		const struct iwl_sar_profile *prof;
 
 		/* don't allow SAR to be disabled (profile 0 means disable) */
 		if (profs[i] == 0)
@@ -262,11 +283,13 @@ static int iwl_sar_fill_table(struct iwl_fw_runtime *fwrt,
 			return -EINVAL;
 
 		/* profiles go from 1 to 4, so decrement to access the array */
-		prof = &fwrt->sar_profiles[profs[i] - 1];
+		prof = &profiles[profs[i] - 1];
 
 		/* if the profile is disabled, do nothing */
 		if (!prof->enabled) {
-			IWL_DEBUG_RADIO(fwrt, "SAR profile %d is disabled.\n",
+			IWL_DEBUG_RADIO(fwrt,
+					"SAR %sprofile %d is disabled.\n",
+					is_standalone ? "standalone " : "",
 					profs[i]);
 			/*
 			 * if one of the profiles is disabled, we
@@ -276,8 +299,8 @@ static int iwl_sar_fill_table(struct iwl_fw_runtime *fwrt,
 			return 1;
 		}
 
-		IWL_DEBUG_INFO(fwrt,
-			       "SAR EWRD: chain %d profile index %d\n",
+		IWL_DEBUG_INFO(fwrt, "SAR %s: chain %d profile index %d\n",
+			       is_standalone ? "standalone" : "EWRD",
 			       i, profs[i]);
 		IWL_DEBUG_RADIO(fwrt, "  Chain[%d]:\n", i);
 		for (j = 0; j < n_subbands; j++) {
@@ -293,14 +316,17 @@ static int iwl_sar_fill_table(struct iwl_fw_runtime *fwrt,
 
 int iwl_sar_fill_profile(struct iwl_fw_runtime *fwrt,
 			 __le16 *per_chain, u32 n_tables, u32 n_subbands,
-			 int prof_a, int prof_b)
+			 int prof_a, int prof_b,
+			 const struct iwl_sar_profile *profiles,
+			 bool is_standalone)
 {
 	int i, ret = 0;
 
 	for (i = 0; i < n_tables; i++) {
 		ret = iwl_sar_fill_table(fwrt,
 			&per_chain[i * n_subbands * BIOS_SAR_NUM_CHAINS],
-			n_subbands, prof_a, prof_b);
+			n_subbands, prof_a, prof_b, profiles,
+			is_standalone);
 		if (ret)
 			break;
 	}
@@ -447,3 +473,27 @@ bool iwl_rfi_is_enabled_in_bios(struct iwl_fw_runtime *fwrt)
 	return false;
 }
 IWL_EXPORT_SYMBOL(iwl_rfi_is_enabled_in_bios);
+
+/**
+ * iwl_bios_get_guid_lock_status - resolve connectivity variables lock status
+ * @fwrt: the firmware runtime context
+ *
+ * Determine the root-of-trust for the connectivity UEFI variables. The UEFI
+ * GUID Lock Indicator (GLUI) is preferred: if it attests the tables as locked
+ * or in test mode, that result is used. Otherwise (GLUI is missing, invalid or
+ * reports unlocked) the ACPI GUID Lock Indicator is consulted and can override
+ * the GLUI result.
+ *
+ * This only caches the result in &fwrt->uefi_tables_lock_status
+ * (see &enum iwl_uefi_cnv_guid_status).
+ */
+void iwl_bios_get_guid_lock_status(struct iwl_fw_runtime *fwrt)
+{
+	iwl_uefi_get_guid_lock_status(fwrt);
+	if (fwrt->uefi_tables_lock_status == UEFI_CNV_GUID_LOCKED ||
+	    fwrt->uefi_tables_lock_status == UEFI_CNV_GUID_TEST_MODE)
+		return;
+
+	iwl_acpi_get_guid_lock_status(fwrt);
+}
+IWL_EXPORT_SYMBOL(iwl_bios_get_guid_lock_status);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h
index 22c97c75b83c56..e32e107812b1ad 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.h
@@ -31,6 +31,7 @@
 
 #define IWL_SAR_ENABLE_MSK		BIT(0)
 #define IWL_REDUCE_POWER_FLAGS_POS	1
+#define IWL_BIOS_REVISION_UNSET		0xFF
 
 /* PPAG gain value bounds in 1/8 dBm */
 #define IWL_PPAG_MIN_LB	-16
@@ -47,6 +48,22 @@
 #define IWL_WTAS_CANADA_UHB_MSK		BIT(15)
 #define IWL_WTAS_USA_UHB_MSK		BIT(16)
 
+/**
+ * enum iwl_uefi_cnv_guid_status - lock status of the connectivity
+ * UEFI tables
+ * @UEFI_CNV_GUID_UNLOCKED: the tables are unlocked
+ * @UEFI_CNV_GUID_LOCKED: the tables are locked
+ * @UEFI_CNV_GUID_TEST_MODE: the tables are locked in test mode
+ * @UEFI_CNV_GUID_UNKNOWN: the lock status could not be determined.
+ *	This value is a driver internal, not a possible BIOS reported one
+ */
+enum iwl_uefi_cnv_guid_status {
+	UEFI_CNV_GUID_UNLOCKED = 0,
+	UEFI_CNV_GUID_LOCKED = 1,
+	UEFI_CNV_GUID_TEST_MODE = 2,
+	UEFI_CNV_GUID_UNKNOWN = 3,
+};
+
 struct iwl_tas_selection_data {
 	u8 override_tas_iec:1,
 	   enable_tas_iec:1,
@@ -193,7 +210,9 @@ int iwl_sar_geo_fill_table(struct iwl_fw_runtime *fwrt,
 
 int iwl_sar_fill_profile(struct iwl_fw_runtime *fwrt,
 			 __le16 *per_chain, u32 n_tables, u32 n_subbands,
-			 int prof_a, int prof_b);
+			 int prof_a, int prof_b,
+			 const struct iwl_sar_profile *profiles,
+			 bool is_standalone);
 
 bool iwl_is_ppag_approved(struct iwl_fw_runtime *fwrt);
 
@@ -207,6 +226,10 @@ int iwl_bios_get_wrds_table(struct iwl_fw_runtime *fwrt);
 
 int iwl_bios_get_ewrd_table(struct iwl_fw_runtime *fwrt);
 
+int iwl_bios_get_wsss_table(struct iwl_fw_runtime *fwrt);
+
+int iwl_bios_get_ewss_table(struct iwl_fw_runtime *fwrt);
+
 int iwl_bios_get_wgds_table(struct iwl_fw_runtime *fwrt);
 
 int iwl_bios_get_ppag_table(struct iwl_fw_runtime *fwrt);
@@ -242,6 +265,7 @@ bool iwl_puncturing_is_allowed_in_bios(u32 puncturing, u16 mcc);
 
 int iwl_bios_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value);
 int iwl_bios_get_phy_filters(struct iwl_fw_runtime *fwrt);
+void iwl_bios_get_guid_lock_status(struct iwl_fw_runtime *fwrt);
 
 static inline void iwl_bios_setup_step(struct iwl_trans *trans,
 				       struct iwl_fw_runtime *fwrt)
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
index ac01ac0092eea3..33ce06f1a60ad3 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
@@ -108,10 +108,14 @@ struct iwl_txf_iter_data {
  * @dump: debug dump data
  * @ap_type_cmd: AP type tables (for enablement on 6 GHz)
  * @ap_type_cmd_valid: if &ap_type_cmd is valid
- * @uefi_tables_lock_status: The status of the WIFI GUID UEFI variables lock:
- *	0: Unlocked, 1 and 2: Locked.
- *	Only read the UEFI variables if locked.
+ * @uefi_tables_lock_status: the status of the WIFI GUID UEFI variables lock,
+ *	see &enum iwl_uefi_cnv_guid_status.
+ *	Trust the UEFI variables if locked or in test mode.
  * @sar_profiles: sar profiles as read from WRDS/EWRD BIOS tables
+ * @sar_standalone_profiles: standalone sar profiles as read from
+ *	WSSS/EWSS BIOS tables (for Wi-Fi HB/UHB without concurrent BT)
+ * @wrds_table_revision: revision of WRDS BIOS table
+ * @ewrd_table_revision: revision of EWRD BIOS table
  * @geo_profiles: geographic profiles as read from WGDS BIOS table
  * @geo_bios_source: see &enum bios_source
  * @phy_filters: specific phy filters as read from WPFC BIOS table
@@ -206,6 +210,9 @@ struct iwl_fw_runtime {
 	bool tpc_enabled;
 #endif /* CONFIG_IWLWIFI_DEBUGFS */
 	struct iwl_sar_profile sar_profiles[BIOS_SAR_MAX_PROFILE_NUM];
+	struct iwl_sar_profile sar_standalone_profiles[BIOS_SAR_MAX_PROFILE_NUM];
+	u8 wrds_table_revision;
+	u8 ewrd_table_revision;
 	u8 sar_chain_a_profile;
 	u8 sar_chain_b_profile;
 	u8 reduced_power_flags;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
index bf6a31e6f1499b..dffd5e68f41da9 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.c
@@ -20,6 +20,10 @@
 					 0x8d, 0x03, 0x77, 0x2e,	\
 					 0xcc, 0x3d, 0xa5, 0x31)
 
+#define IWL_EFI_GLUI_GUID	EFI_GUID(0x6c5bed75, 0x0ee8, 0x4d45,	\
+					 0x98, 0x0b, 0x4d, 0x81,	\
+					 0x31, 0xba, 0x84, 0xb5)
+
 struct iwl_uefi_pnvm_mem_desc {
 	__le32 addr;
 	__le32 size;
@@ -27,9 +31,10 @@ struct iwl_uefi_pnvm_mem_desc {
 } __packed;
 
 static void *iwl_uefi_get_variable(efi_char16_t *name, efi_guid_t *guid,
-				   unsigned long *data_size)
+				   u32 *attributes, unsigned long *data_size)
 {
 	efi_status_t status;
+	u32 attr = 0;
 	void *data;
 
 	if (!data_size)
@@ -48,15 +53,28 @@ static void *iwl_uefi_get_variable(efi_char16_t *name, efi_guid_t *guid,
 	if (!data)
 		return ERR_PTR(-ENOMEM);
 
-	status = efi.get_variable(name, guid, NULL, data_size, data);
+	status = efi.get_variable(name, guid, &attr, data_size, data);
 	if (status != EFI_SUCCESS) {
 		kfree(data);
 		return ERR_PTR(-ENOENT);
 	}
 
+	if (attributes)
+		*attributes = attr;
+
 	return data;
 }
 
+static efi_status_t iwl_uefi_set_variable(efi_char16_t *name, efi_guid_t *guid,
+					  u32 attributes, void *data,
+					  unsigned long data_size)
+{
+	if (!efi_rt_services_supported(EFI_RT_SUPPORTED_SET_VARIABLE))
+		return EFI_UNSUPPORTED;
+
+	return efi.set_variable(name, guid, attributes, data_size, data);
+}
+
 void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 {
 	unsigned long package_size;
@@ -65,7 +83,7 @@ void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 	*len = 0;
 
 	data = iwl_uefi_get_variable(IWL_UEFI_OEM_PNVM_NAME, &IWL_EFI_WIFI_GUID,
-				     &package_size);
+				     NULL, &package_size);
 	if (IS_ERR(data)) {
 		IWL_DEBUG_FW(trans,
 			     "PNVM UEFI variable not found 0x%lx (len %lu)\n",
@@ -80,17 +98,18 @@ void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 }
 
 static void *
-iwl_uefi_get_verified_variable_guid(struct iwl_trans *trans,
-				    efi_guid_t *guid,
-				    efi_char16_t *uefi_var_name,
-				    char *var_name,
-				    unsigned int expected_size,
-				    unsigned long *size)
+iwl_uefi_get_variable_guid(struct iwl_trans *trans,
+			   efi_guid_t *guid,
+			   efi_char16_t *uefi_var_name,
+			   char *var_name,
+			   unsigned int expected_size,
+			   unsigned long *size,
+			   u32 *attributes)
 {
 	void *var;
 	unsigned long var_size;
 
-	var = iwl_uefi_get_variable(uefi_var_name, guid, &var_size);
+	var = iwl_uefi_get_variable(uefi_var_name, guid, attributes, &var_size);
 
 	if (IS_ERR(var)) {
 		IWL_DEBUG_RADIO(trans,
@@ -116,13 +135,72 @@ iwl_uefi_get_verified_variable_guid(struct iwl_trans *trans,
 }
 
 static void *
-iwl_uefi_get_verified_variable(struct iwl_trans *trans,
+iwl_uefi_get_verified_variable_guid(struct iwl_fw_runtime *fwrt,
+				    efi_guid_t *guid,
+				    efi_char16_t *uefi_var_name,
+				    char *var_name,
+				    unsigned int expected_size,
+				    unsigned long *size)
+{
+	unsigned long var_size;
+	u32 attributes = 0;
+	void *var;
+
+	if (fwrt->uefi_tables_lock_status == UEFI_CNV_GUID_UNLOCKED)
+		return ERR_PTR(-EINVAL);
+
+	/* In x86 either GLUI or GLAI should exist */
+	if ((IS_ENABLED(CONFIG_X86) || IS_ENABLED(CONFIG_X86_64) ||
+	     IS_ENABLED(CONFIG_X86_32)) &&
+	    fwrt->uefi_tables_lock_status == UEFI_CNV_GUID_UNKNOWN)
+		return ERR_PTR(-EINVAL);
+
+	var = iwl_uefi_get_variable_guid(fwrt->trans,
+					 guid,
+					 uefi_var_name, var_name,
+					 expected_size, &var_size,
+					 &attributes);
+	if (IS_ERR(var))
+		return var;
+
+	if (fwrt->uefi_tables_lock_status != UEFI_CNV_GUID_UNKNOWN)
+		goto out;
+
+	/*
+	 * The lock status is unknown, so probe it: rewrite the variable
+	 * with the data just read and trust it only if the firmware
+	 * refuses the write. Treat a zero attributes value as untrusted,
+	 * since a SetVariable() with no attributes would delete it.
+	 */
+	if (attributes) {
+		efi_status_t status;
+
+		status = iwl_uefi_set_variable(uefi_var_name, guid, attributes,
+					       var, var_size);
+		if (status == EFI_WRITE_PROTECTED ||
+		    status == EFI_SECURITY_VIOLATION ||
+		    status == EFI_ACCESS_DENIED)
+			goto out;
+	}
+
+	/* unprobeable (e.g. EFI_UNSUPPORTED) or writable: not trusted */
+	kfree(var);
+	return ERR_PTR(-EINVAL);
+
+out:
+	if (size)
+		*size = var_size;
+	return var;
+}
+
+static void *
+iwl_uefi_get_verified_wifi_var(struct iwl_fw_runtime *fwrt,
 			       efi_char16_t *uefi_var_name,
 			       char *var_name,
 			       unsigned int expected_size,
 			       unsigned long *size)
 {
-	return iwl_uefi_get_verified_variable_guid(trans, &IWL_EFI_WIFI_GUID,
+	return iwl_uefi_get_verified_variable_guid(fwrt, &IWL_EFI_WIFI_GUID,
 						   uefi_var_name, var_name,
 						   expected_size, size);
 }
@@ -173,15 +251,24 @@ static int iwl_uefi_reduce_power_section(struct iwl_trans *trans,
 	memset(pnvm_data, 0, sizeof(*pnvm_data));
 
 	while (len >= sizeof(*tlv)) {
-		u32 tlv_len, tlv_type;
+		u32 tlv_len, tlv_type, tlv_len_aligned;
 
 		len -= sizeof(*tlv);
 		tlv = (const void *)data;
 
 		tlv_len = le32_to_cpu(tlv->length);
 		tlv_type = le32_to_cpu(tlv->type);
+		tlv_len_aligned = ALIGN(tlv_len, 4);
 
-		if (len < tlv_len) {
+		/* Make sure ALIGN() did not overflow for a malformed TLV length. */
+		if (tlv_len_aligned < tlv_len) {
+			IWL_ERR(trans,
+				"TLV len overflows on alignment: %u\n",
+				tlv_len);
+			return -EINVAL;
+		}
+
+		if (len < tlv_len_aligned) {
 			IWL_ERR(trans, "invalid TLV len: %zd/%u\n",
 				len, tlv_len);
 			return -EINVAL;
@@ -200,13 +287,13 @@ static int iwl_uefi_reduce_power_section(struct iwl_trans *trans,
 				     "New REDUCE_POWER section started, stop parsing.\n");
 			goto done;
 		default:
-			IWL_DEBUG_FW(trans, "Found TLV 0x%0x, len %d\n",
+			IWL_DEBUG_FW(trans, "Found TLV 0x%0x, len %u\n",
 				     tlv_type, tlv_len);
 			break;
 		}
 
-		len -= ALIGN(tlv_len, 4);
-		data += ALIGN(tlv_len, 4);
+		len -= tlv_len_aligned;
+		data += tlv_len_aligned;
 	}
 
 done:
@@ -227,15 +314,24 @@ int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 	IWL_DEBUG_FW(trans, "Parsing REDUCE_POWER data\n");
 
 	while (len >= sizeof(*tlv)) {
-		u32 tlv_len, tlv_type;
+		u32 tlv_len, tlv_type, tlv_len_aligned;
 
 		len -= sizeof(*tlv);
 		tlv = (const void *)data;
 
 		tlv_len = le32_to_cpu(tlv->length);
 		tlv_type = le32_to_cpu(tlv->type);
+		tlv_len_aligned = ALIGN(tlv_len, 4);
+
+		/* Make sure ALIGN() did not overflow for a malformed TLV length. */
+		if (tlv_len_aligned < tlv_len) {
+			IWL_ERR(trans,
+				"TLV len overflows on alignment: %u\n",
+				tlv_len);
+			return -EINVAL;
+		}
 
-		if (len < tlv_len) {
+		if (len < tlv_len_aligned) {
 			IWL_ERR(trans, "invalid TLV len: %zd/%u\n",
 				len, tlv_len);
 			return -EINVAL;
@@ -245,8 +341,15 @@ int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 			const struct iwl_sku_id *tlv_sku_id =
 				(const void *)(data + sizeof(*tlv));
 
+			if (tlv_len < sizeof(*tlv_sku_id)) {
+				IWL_ERR(trans,
+					"Invalid IWL_UCODE_TLV_PNVM_SKU len %u\n",
+					tlv_len);
+				return -EINVAL;
+			}
+
 			IWL_DEBUG_FW(trans,
-				     "Got IWL_UCODE_TLV_PNVM_SKU len %d\n",
+				     "Got IWL_UCODE_TLV_PNVM_SKU len %u\n",
 				     tlv_len);
 			if (tlv_len < sizeof(*tlv_sku_id)) {
 				IWL_ERR(trans, "invalid PNVM SKU TLV len: %u\n",
@@ -259,8 +362,8 @@ int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 				     le32_to_cpu(tlv_sku_id->data[1]),
 				     le32_to_cpu(tlv_sku_id->data[2]));
 
-			data += sizeof(*tlv) + ALIGN(tlv_len, 4);
-			len -= ALIGN(tlv_len, 4);
+			data += sizeof(*tlv) + tlv_len_aligned;
+			len -= tlv_len_aligned;
 
 			if (sku_id[0] == tlv_sku_id->data[0] &&
 			    sku_id[1] == tlv_sku_id->data[1] &&
@@ -274,8 +377,8 @@ int iwl_uefi_reduce_power_parse(struct iwl_trans *trans,
 				IWL_DEBUG_FW(trans, "SKU ID didn't match!\n");
 			}
 		} else {
-			data += sizeof(*tlv) + ALIGN(tlv_len, 4);
-			len -= ALIGN(tlv_len, 4);
+			data += sizeof(*tlv) + tlv_len_aligned;
+			len -= tlv_len_aligned;
 		}
 	}
 
@@ -288,11 +391,12 @@ u8 *iwl_uefi_get_reduced_power(struct iwl_trans *trans, size_t *len)
 	unsigned long package_size;
 	u8 *data;
 
-	package = iwl_uefi_get_verified_variable(trans,
-						 IWL_UEFI_REDUCED_POWER_NAME,
-						 "Reduced Power",
-						 sizeof(*package),
-						 &package_size);
+	package =
+		iwl_uefi_get_variable_guid(trans, &IWL_EFI_WIFI_GUID,
+					   IWL_UEFI_REDUCED_POWER_NAME,
+					   "Reduced Power",
+					   sizeof(*package),
+					   &package_size, NULL);
 	if (IS_ERR(package))
 		return ERR_CAST(package);
 
@@ -334,9 +438,9 @@ void iwl_uefi_get_step_table(struct iwl_trans *trans)
 	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 		return;
 
-	data = iwl_uefi_get_verified_variable_guid(trans, &IWL_EFI_WIFI_BT_GUID,
-						   IWL_UEFI_STEP_NAME,
-						   "STEP", sizeof(*data), NULL);
+	data = iwl_uefi_get_variable_guid(trans, &IWL_EFI_WIFI_BT_GUID,
+					  IWL_UEFI_STEP_NAME, "STEP",
+					  sizeof(*data), NULL, NULL);
 	if (IS_ERR(data))
 		return;
 
@@ -389,8 +493,9 @@ void iwl_uefi_get_sgom_table(struct iwl_trans *trans,
 	if (!fwrt->geo_enabled)
 		return;
 
-	data = iwl_uefi_get_verified_variable(trans, IWL_UEFI_SGOM_NAME,
-					      "SGOM", sizeof(*data), NULL);
+	data = iwl_uefi_get_variable_guid(trans, &IWL_EFI_WIFI_GUID,
+					  IWL_UEFI_SGOM_NAME, "SGOM",
+					  sizeof(*data), NULL, NULL);
 	if (IS_ERR(data))
 		return;
 
@@ -423,8 +528,9 @@ void iwl_uefi_get_uats_table(struct iwl_trans *trans,
 	struct uefi_cnv_wlan_uats_data *data;
 	int ret;
 
-	data = iwl_uefi_get_verified_variable(trans, IWL_UEFI_UATS_NAME,
-					      "UATS", sizeof(*data), NULL);
+	data = iwl_uefi_get_variable_guid(trans, &IWL_EFI_WIFI_GUID,
+					  IWL_UEFI_UATS_NAME, "UATS",
+					  sizeof(*data), NULL, NULL);
 	if (IS_ERR(data))
 		return;
 
@@ -440,8 +546,9 @@ void iwl_uefi_get_uneb_table(struct iwl_trans *trans,
 {
 	struct uefi_cnv_wlan_uneb_data *data;
 
-	data = iwl_uefi_get_verified_variable(trans, IWL_UEFI_UNEB_NAME,
-					      "UNEB", sizeof(*data), NULL);
+	data = iwl_uefi_get_variable_guid(trans, &IWL_EFI_WIFI_GUID,
+					  IWL_UEFI_UNEB_NAME, "UNEB",
+					  sizeof(*data), NULL, NULL);
 	if (IS_ERR(data))
 		return;
 
@@ -465,11 +572,11 @@ void iwl_uefi_get_uneb_table(struct iwl_trans *trans,
 }
 IWL_EXPORT_SYMBOL(iwl_uefi_get_uneb_table);
 
-static void iwl_uefi_set_sar_profile(struct iwl_fw_runtime *fwrt,
+static void iwl_uefi_set_sar_profile(struct iwl_sar_profile *profiles,
 				     const u8 *vals, u8 prof_index,
-				     u8 num_subbands, bool enabled)
+				     u8 num_subbands, u32 mode)
 {
-	struct iwl_sar_profile *sar_prof = &fwrt->sar_profiles[prof_index];
+	struct iwl_sar_profile *sar_prof = &profiles[prof_index];
 
 	/*
 	 * Make sure fwrt has enough room to hold the data
@@ -491,7 +598,7 @@ static void iwl_uefi_set_sar_profile(struct iwl_fw_runtime *fwrt,
 				vals[chain * num_subbands + subband];
 	}
 
-	fwrt->sar_profiles[prof_index].enabled = enabled & IWL_SAR_ENABLE_MSK;
+	sar_prof->enabled = mode & IWL_SAR_ENABLE_MSK;
 }
 
 int iwl_uefi_get_wrds_table(struct iwl_fw_runtime *fwrt)
@@ -502,8 +609,8 @@ int iwl_uefi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 	int num_subbands;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WRDS_NAME,
-					      "WRDS",
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_WRDS_NAME, "WRDS",
 					      UEFI_SAR_WRDS_TABLE_SIZE_REV2,
 					      &size);
 
@@ -535,8 +642,9 @@ int iwl_uefi_get_wrds_table(struct iwl_fw_runtime *fwrt)
 	/* The profile from WRDS is officially profile 1, but goes
 	 * into sar_profiles[0] (because we don't have a profile 0).
 	 */
-	iwl_uefi_set_sar_profile(fwrt, data->vals, 0,
+	iwl_uefi_set_sar_profile(fwrt->sar_profiles, data->vals, 0,
 				 num_subbands, data->mode);
+	fwrt->wrds_table_revision = data->revision;
 out:
 	kfree(data);
 	return ret;
@@ -551,8 +659,8 @@ int iwl_uefi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 	int num_subbands;
 	int profile_size;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_EWRD_NAME,
-					      "EWRD",
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_EWRD_NAME, "EWRD",
 					      UEFI_SAR_EWRD_TABLE_SIZE_REV2,
 					      &size);
 	if (IS_ERR(data))
@@ -588,14 +696,136 @@ int iwl_uefi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 		 * save them in sar_profiles[1-3] (because we don't
 		 * have profile 0).  So in the array we start from 1.
 		 */
-		iwl_uefi_set_sar_profile(fwrt, &data->vals[i * profile_size],
+		iwl_uefi_set_sar_profile(fwrt->sar_profiles,
+					 &data->vals[i * profile_size],
 					 i + 1, num_subbands, data->mode);
 
+	fwrt->ewrd_table_revision = data->revision;
+
 out:
 	kfree(data);
 	return ret;
 }
 
+int iwl_uefi_get_wsss_table(struct iwl_fw_runtime *fwrt)
+{
+	struct uefi_cnv_var_wrds *data __free(kfree) = NULL;
+	unsigned long expected_size;
+	unsigned long size;
+	int num_subbands;
+
+	if (fwrt->wrds_table_revision == IWL_BIOS_REVISION_UNSET) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: WRDS table was not read\n");
+		return -EINVAL;
+	}
+
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_WSSS_NAME,
+					      "WSSS",
+					      UEFI_SAR_WRDS_TABLE_SIZE_REV2,
+					      &size);
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	switch (data->revision) {
+	case 2:
+		expected_size = UEFI_SAR_WRDS_TABLE_SIZE_REV2;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV2;
+		break;
+	case 3:
+		expected_size = UEFI_SAR_WRDS_TABLE_SIZE_REV3;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV3;
+		break;
+	default:
+		IWL_DEBUG_RADIO(fwrt,
+				"Unsupported UEFI WSSS revision:%d\n",
+				data->revision);
+		return -EINVAL;
+	}
+
+	if (size != expected_size)
+		return -EINVAL;
+
+	if (fwrt->wrds_table_revision != data->revision) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: WRDS/WSSS revision mismatch (WRDS rev %d, WSSS rev %d)\n",
+				fwrt->wrds_table_revision,
+				data->revision);
+		return -EINVAL;
+	}
+
+	IWL_DEBUG_RADIO(fwrt, "Reading WSSS (WRDS Standalone) tbl_rev=%d\n",
+			data->revision);
+	iwl_uefi_set_sar_profile(fwrt->sar_standalone_profiles,
+				 data->vals, 0,
+				 num_subbands,
+				 data->mode);
+	return 0;
+}
+
+int iwl_uefi_get_ewss_table(struct iwl_fw_runtime *fwrt)
+{
+	struct uefi_cnv_var_ewrd *data __free(kfree) = NULL;
+	unsigned long expected_size;
+	unsigned long size;
+	int num_subbands;
+	int profile_size;
+	int i;
+
+	if (fwrt->ewrd_table_revision == IWL_BIOS_REVISION_UNSET) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: EWRD table was not read\n");
+		return -EINVAL;
+	}
+
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_EWSS_NAME,
+					      "EWSS",
+					      UEFI_SAR_EWRD_TABLE_SIZE_REV2,
+					      &size);
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	switch (data->revision) {
+	case 3:
+		expected_size = UEFI_SAR_EWRD_TABLE_SIZE_REV3;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV3;
+		profile_size = UEFI_SAR_PROFILE_SIZE_REV3;
+		break;
+	case 2:
+		expected_size = UEFI_SAR_EWRD_TABLE_SIZE_REV2;
+		num_subbands = UEFI_SAR_SUB_BANDS_NUM_REV2;
+		profile_size = UEFI_SAR_PROFILE_SIZE_REV2;
+		break;
+	default:
+		IWL_DEBUG_RADIO(fwrt,
+				"Unsupported UEFI EWSS revision:%d\n",
+				data->revision);
+		return -EINVAL;
+	}
+
+	if (size != expected_size ||
+	    data->num_profiles >= BIOS_SAR_MAX_PROFILE_NUM)
+		return -EINVAL;
+
+	if (fwrt->ewrd_table_revision != data->revision) {
+		IWL_DEBUG_RADIO(fwrt,
+				"Skipping standalone SAR: EWRD/EWSS revision mismatch (EWRD rev %d, EWSS rev %d)\n",
+				fwrt->ewrd_table_revision,
+				data->revision);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < data->num_profiles; i++)
+		iwl_uefi_set_sar_profile(fwrt->sar_standalone_profiles,
+					 &data->vals[i * profile_size],
+					 i + 1, num_subbands,
+					 data->mode);
+
+	return 0;
+}
+
 int iwl_uefi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 {
 	struct uefi_cnv_var_wgds *data;
@@ -605,9 +835,9 @@ int iwl_uefi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 	int n_subbands;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WGDS_NAME,
-					      "WGDS", UEFI_WGDS_TABLE_SIZE_REV3,
-					      &size);
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_WGDS_NAME, "WGDS",
+					      UEFI_WGDS_TABLE_SIZE_REV3, &size);
 	if (IS_ERR(data))
 		return -EINVAL;
 
@@ -680,9 +910,9 @@ int iwl_uefi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 	u32 valid_rev;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_PPAG_NAME,
-					      "PPAG", UEFI_PPAG_DATA_SIZE_V5,
-					      NULL);
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_PPAG_NAME, "PPAG",
+					      UEFI_PPAG_DATA_SIZE_V5, NULL);
 	if (!IS_ERR(data)) {
 		n_subbands = UEFI_PPAG_SUB_BANDS_NUM_REV5;
 		valid_rev = BIT(5);
@@ -690,7 +920,7 @@ int iwl_uefi_get_ppag_table(struct iwl_fw_runtime *fwrt)
 		goto parse_table;
 	}
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans,
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
 					      IWL_UEFI_PPAG_NAME,
 					      "PPAG",
 					      UEFI_PPAG_DATA_SIZE_V4,
@@ -748,8 +978,9 @@ int iwl_uefi_get_tas_table(struct iwl_fw_runtime *fwrt,
 	struct uefi_cnv_var_wtas *uefi_tas;
 	int ret, enabled;
 
-	uefi_tas = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WTAS_NAME,
-						  "WTAS", sizeof(*uefi_tas), NULL);
+	uefi_tas = iwl_uefi_get_verified_wifi_var(fwrt,
+						  IWL_UEFI_WTAS_NAME, "WTAS",
+						  sizeof(*uefi_tas), NULL);
 	if (IS_ERR(uefi_tas))
 		return -EINVAL;
 
@@ -801,7 +1032,7 @@ int iwl_uefi_get_pwr_limit(struct iwl_fw_runtime *fwrt,
 	struct uefi_cnv_var_splc *data;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_SPLC_NAME,
+	data = iwl_uefi_get_verified_wifi_var(fwrt, IWL_UEFI_SPLC_NAME,
 					      "SPLC", sizeof(*data), NULL);
 	if (IS_ERR(data))
 		return -EINVAL;
@@ -823,8 +1054,9 @@ int iwl_uefi_get_mcc(struct iwl_fw_runtime *fwrt, char *mcc)
 	struct uefi_cnv_var_wrdd *data;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WRDD_NAME,
-					      "WRDD", sizeof(*data), NULL);
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_WRDD_NAME, "WRDD",
+					      sizeof(*data), NULL);
 	if (IS_ERR(data))
 		return -EINVAL;
 
@@ -854,7 +1086,7 @@ int iwl_uefi_get_eckv(struct iwl_fw_runtime *fwrt, u32 *extl_clk)
 	struct uefi_cnv_var_eckv *data;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable_guid(fwrt->trans,
+	data = iwl_uefi_get_verified_variable_guid(fwrt,
 						   &IWL_EFI_WIFI_BT_GUID,
 						   IWL_UEFI_ECKV_NAME,
 						   "ECKV", sizeof(*data), NULL);
@@ -878,8 +1110,9 @@ int iwl_uefi_get_wbem(struct iwl_fw_runtime *fwrt, u32 *value)
 	struct uefi_cnv_wlan_wbem_data *data;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WBEM_NAME,
-					      "WBEM", sizeof(*data), NULL);
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_WBEM_NAME, "WBEM",
+					      sizeof(*data), NULL);
 	if (IS_ERR(data))
 		return -EINVAL;
 
@@ -905,7 +1138,7 @@ static int iwl_uefi_load_dsm_values(struct iwl_fw_runtime *fwrt)
 
 	BUILD_BUG_ON(ARRAY_SIZE(data->functions) < ARRAY_SIZE(fwrt->dsm_values));
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_DSM_NAME,
+	data = iwl_uefi_get_verified_wifi_var(fwrt, IWL_UEFI_DSM_NAME,
 					      "DSM", sizeof(*data), NULL);
 	if (IS_ERR(data))
 		return -EINVAL;
@@ -915,6 +1148,12 @@ static int iwl_uefi_load_dsm_values(struct iwl_fw_runtime *fwrt)
 				data->revision);
 		goto out;
 	}
+	if (!data->functions[DSM_FUNC_QUERY]) {
+		IWL_DEBUG_RADIO(fwrt,
+				"UEFI DSM query bitmap is empty, revision:%d\n",
+				data->revision);
+		goto out;
+	}
 	fwrt->dsm_revision = data->revision;
 	fwrt->dsm_source = BIOS_SOURCE_UEFI;
 
@@ -980,10 +1219,10 @@ int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt)
 	struct uefi_cnv_var_puncturing_data *data;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans,
-					      IWL_UEFI_PUNCTURING_NAME,
-					      "UefiCnvWlanPuncturing",
-					      sizeof(*data), NULL);
+	data = iwl_uefi_get_variable_guid(fwrt->trans, &IWL_EFI_WIFI_GUID,
+					  IWL_UEFI_PUNCTURING_NAME,
+					  "UefiCnvWlanPuncturing",
+					  sizeof(*data), NULL, NULL);
 	if (IS_ERR(data))
 		return -EINVAL;
 
@@ -1009,7 +1248,7 @@ int iwl_uefi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value)
 	struct uefi_cnv_wlan_dsbr_data *data;
 	int ret = 0;
 
-	data = iwl_uefi_get_verified_variable_guid(fwrt->trans,
+	data = iwl_uefi_get_verified_variable_guid(fwrt,
 						   &IWL_EFI_WIFI_BT_GUID,
 						   IWL_UEFI_DSBR_NAME, "DSBR",
 						   sizeof(*data), NULL);
@@ -1035,8 +1274,9 @@ int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt)
 	struct uefi_cnv_wpfc_data *data __free(kfree);
 	struct iwl_phy_specific_cfg *filters = &fwrt->phy_filters;
 
-	data = iwl_uefi_get_verified_variable(fwrt->trans, IWL_UEFI_WPFC_NAME,
-					      "WPFC", sizeof(*data), NULL);
+	data = iwl_uefi_get_verified_wifi_var(fwrt,
+					      IWL_UEFI_WPFC_NAME, "WPFC",
+					      sizeof(*data), NULL);
 	if (IS_ERR(data))
 		return -EINVAL;
 
@@ -1057,3 +1297,33 @@ int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt)
 	IWL_DEBUG_RADIO(fwrt, "Loaded WPFC config from UEFI\n");
 	return 0;
 }
+
+void iwl_uefi_get_guid_lock_status(struct iwl_fw_runtime *fwrt)
+{
+	struct uefi_cnv_var_glui *data __free(kfree) =
+		iwl_uefi_get_variable_guid(fwrt->trans,
+					   &IWL_EFI_GLUI_GUID,
+					   IWL_UEFI_GLUI_NAME, "GLUI",
+					   sizeof(*data), NULL, NULL);
+
+	if (IS_ERR(data))
+		return;
+
+	if (data->revision != IWL_UEFI_GLUI_REVISION) {
+		IWL_DEBUG_RADIO(fwrt, "Unsupported UEFI GLUI revision:%d\n",
+				data->revision);
+		return;
+	}
+
+	/* UEFI_CNV_GUID_TEST_MODE is the max value the BIOS can report */
+	if (data->guid_lock_status > UEFI_CNV_GUID_TEST_MODE) {
+		IWL_DEBUG_RADIO(fwrt, "Invalid UEFI GUID lock status:%d\n",
+				data->guid_lock_status);
+		return;
+	}
+
+	fwrt->uefi_tables_lock_status = data->guid_lock_status;
+
+	IWL_DEBUG_RADIO(fwrt, "Loaded UEFI WIFI GUID lock status: %d\n",
+			fwrt->uefi_tables_lock_status);
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/uefi.h b/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
index 386ca37444089e..f4f7a447e52871 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/uefi.h
@@ -13,7 +13,10 @@
 #define IWL_UEFI_STEP_NAME		L"UefiCnvCommonSTEP"
 #define IWL_UEFI_UATS_NAME		L"CnvUefiWlanUATS"
 #define IWL_UEFI_WRDS_NAME		L"UefiCnvWlanWRDS"
+#define IWL_UEFI_GLUI_NAME		L"UefiCnvCommonGLUI"
 #define IWL_UEFI_EWRD_NAME		L"UefiCnvWlanEWRD"
+#define IWL_UEFI_WSSS_NAME		L"UefiCnvWlanWSSS"
+#define IWL_UEFI_EWSS_NAME		L"UefiCnvWlanEWSS"
 #define IWL_UEFI_WGDS_NAME		L"UefiCnvWlanWGDS"
 #define IWL_UEFI_PPAG_NAME		L"UefiCnvWlanPPAG"
 #define IWL_UEFI_WTAS_NAME		L"UefiCnvWlanWTAS"
@@ -40,6 +43,7 @@
 #define IWL_UEFI_DSM_REVISION		4
 #define IWL_UEFI_PUNCTURING_REVISION	0
 #define IWL_UEFI_DSBR_REVISION		1
+#define IWL_UEFI_GLUI_REVISION		0
 
 struct pnvm_sku_package {
 	u8 rev;
@@ -54,6 +58,11 @@ struct uefi_cnv_wlan_sgom_data {
 	u8 offset_map[IWL_SGOM_MAP_SIZE - 1];
 } __packed;
 
+struct uefi_cnv_var_glui {
+	u8 revision;
+	u8 guid_lock_status;
+} __packed;
+
 struct uefi_cnv_wlan_uats_data {
 	u8 revision;
 	u8 mcc_to_ap_type_map[IWL_UATS_MAP_SIZE - 1];
@@ -330,6 +339,8 @@ int iwl_uefi_handle_tlv_mem_desc(struct iwl_trans *trans, const u8 *data,
 				 u32 tlv_len, struct iwl_pnvm_image *pnvm_data);
 int iwl_uefi_get_wrds_table(struct iwl_fw_runtime *fwrt);
 int iwl_uefi_get_ewrd_table(struct iwl_fw_runtime *fwrt);
+int iwl_uefi_get_wsss_table(struct iwl_fw_runtime *fwrt);
+int iwl_uefi_get_ewss_table(struct iwl_fw_runtime *fwrt);
 int iwl_uefi_get_wgds_table(struct iwl_fw_runtime *fwrt);
 int iwl_uefi_get_ppag_table(struct iwl_fw_runtime *fwrt);
 int iwl_uefi_get_tas_table(struct iwl_fw_runtime *fwrt,
@@ -349,6 +360,7 @@ void iwl_uefi_get_uneb_table(struct iwl_trans *trans,
 int iwl_uefi_get_puncturing(struct iwl_fw_runtime *fwrt);
 int iwl_uefi_get_dsbr(struct iwl_fw_runtime *fwrt, u32 *value);
 int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt);
+void iwl_uefi_get_guid_lock_status(struct iwl_fw_runtime *fwrt);
 #else /* CONFIG_EFI */
 static inline void *iwl_uefi_get_pnvm(struct iwl_trans *trans, size_t *len)
 {
@@ -391,6 +403,16 @@ static inline int iwl_uefi_get_ewrd_table(struct iwl_fw_runtime *fwrt)
 	return -ENOENT;
 }
 
+static inline int iwl_uefi_get_wsss_table(struct iwl_fw_runtime *fwrt)
+{
+	return -ENOENT;
+}
+
+static inline int iwl_uefi_get_ewss_table(struct iwl_fw_runtime *fwrt)
+{
+	return -ENOENT;
+}
+
 static inline int iwl_uefi_get_wgds_table(struct iwl_fw_runtime *fwrt)
 {
 	return -ENOENT;
@@ -466,5 +488,9 @@ static inline int iwl_uefi_get_phy_filters(struct iwl_fw_runtime *fwrt)
 {
 	return -ENOENT;
 }
+
+static inline void iwl_uefi_get_guid_lock_status(struct iwl_fw_runtime *fwrt)
+{
+}
 #endif /* CONFIG_EFI */
 #endif /* __iwl_fw_uefi__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
index 11763dec77eca6..4372a171172b0e 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c
@@ -167,11 +167,13 @@ static int iwl_dbg_tlv_alloc_hcmd(struct iwl_trans *trans,
 				  const struct iwl_ucode_tlv *tlv)
 {
 	const struct iwl_fw_ini_hcmd_tlv *hcmd = (const void *)tlv->data;
-	u32 tp = le32_to_cpu(hcmd->time_point);
+	u32 tp;
 
 	if (le32_to_cpu(tlv->length) <= sizeof(*hcmd))
 		return -EINVAL;
 
+	tp = le32_to_cpu(hcmd->time_point);
+
 	/* Host commands can not be sent in early time point since the FW
 	 * is not ready
 	 */
@@ -194,9 +196,14 @@ static int iwl_dbg_tlv_alloc_region(struct iwl_trans *trans,
 {
 	const struct iwl_fw_ini_region_tlv *reg = (const void *)tlv->data;
 	struct iwl_ucode_tlv **active_reg;
-	u32 id = le32_to_cpu(reg->id);
-	u8 type = reg->type;
 	u32 tlv_len = sizeof(*tlv) + le32_to_cpu(tlv->length);
+	u32 id;
+	u8 type;
+
+	if (le32_to_cpu(tlv->length) < sizeof(*reg))
+		return -EINVAL;
+
+	id = le32_to_cpu(reg->id);
 
 	/*
 	 * The higher part of the ID from version 2 is debug policy.
@@ -205,9 +212,6 @@ static int iwl_dbg_tlv_alloc_region(struct iwl_trans *trans,
 	if (le32_to_cpu(reg->hdr.version) >= 2)
 		id &= IWL_FW_INI_REGION_ID_MASK;
 
-	if (le32_to_cpu(tlv->length) < sizeof(*reg))
-		return -EINVAL;
-
 	/* for safe use of a string from FW, limit it to IWL_FW_INI_MAX_NAME */
 	IWL_DEBUG_FW(trans, "WRT: parsing region: %.*s\n",
 		     IWL_FW_INI_MAX_NAME, reg->name);
@@ -217,12 +221,26 @@ static int iwl_dbg_tlv_alloc_region(struct iwl_trans *trans,
 		return -EINVAL;
 	}
 
+	type = reg->type;
+
 	if (type <= IWL_FW_INI_REGION_INVALID ||
 	    type >= IWL_FW_INI_REGION_NUM) {
 		IWL_ERR(trans, "WRT: Invalid region type %u\n", type);
 		return -EINVAL;
 	}
 
+	if (type == IWL_FW_INI_REGION_DRAM_BUFFER) {
+		u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
+
+		if (alloc_id <= IWL_FW_INI_ALLOCATION_INVALID ||
+		    alloc_id >= ARRAY_SIZE(trans->dbg.fw_mon_ini)) {
+			IWL_ERR(trans,
+				"WRT: Invalid dram_alloc_id %u for region %u\n",
+				alloc_id, id);
+			return -EINVAL;
+		}
+	}
+
 	if (type == IWL_FW_INI_REGION_INTERNAL_BUFFER) {
 		trans->dbg.imr_data.sram_addr =
 			le32_to_cpu(reg->internal_buffer.base_addr);
@@ -251,13 +269,14 @@ static int iwl_dbg_tlv_alloc_trigger(struct iwl_trans *trans,
 				     const struct iwl_ucode_tlv *tlv)
 {
 	const struct iwl_fw_ini_trigger_tlv *trig = (const void *)tlv->data;
-	u32 tp = le32_to_cpu(trig->time_point);
-	u32 rf = le32_to_cpu(trig->reset_fw);
+	u32 tp, rf;
 	struct iwl_ucode_tlv *new_tlv;
 
 	if (le32_to_cpu(tlv->length) < sizeof(*trig))
 		return -EINVAL;
 
+	tp = le32_to_cpu(trig->time_point);
+
 	if (tp <= IWL_FW_INI_TIME_POINT_INVALID ||
 	    tp >= IWL_FW_INI_TIME_POINT_NUM) {
 		IWL_ERR(trans,
@@ -266,6 +285,8 @@ static int iwl_dbg_tlv_alloc_trigger(struct iwl_trans *trans,
 		return -EINVAL;
 	}
 
+	rf = le32_to_cpu(trig->reset_fw);
+
 	IWL_DEBUG_FW(trans,
 		     "WRT: time point %u for trigger TLV with reset_fw %u\n",
 		     tp, rf);
@@ -288,8 +309,13 @@ static int iwl_dbg_tlv_config_set(struct iwl_trans *trans,
 				  const struct iwl_ucode_tlv *tlv)
 {
 	const struct iwl_fw_ini_conf_set_tlv *conf_set = (const void *)tlv->data;
-	u32 tp = le32_to_cpu(conf_set->time_point);
-	u32 type = le32_to_cpu(conf_set->set_type);
+	u32 tp, type;
+
+	if (le32_to_cpu(tlv->length) < sizeof(*conf_set))
+		return -EINVAL;
+
+	tp = le32_to_cpu(conf_set->time_point);
+	type = le32_to_cpu(conf_set->set_type);
 
 	if (tp <= IWL_FW_INI_TIME_POINT_INVALID ||
 	    tp >= IWL_FW_INI_TIME_POINT_NUM) {
@@ -593,7 +619,7 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt,
 	u32 num_frags, remain_pages, frag_pages;
 	int i;
 
-	if (alloc_id < IWL_FW_INI_ALLOCATION_INVALID ||
+	if (alloc_id <= IWL_FW_INI_ALLOCATION_INVALID ||
 	    alloc_id >= IWL_FW_INI_ALLOCATION_NUM)
 		return -EIO;
 
@@ -676,7 +702,7 @@ static int iwl_dbg_tlv_apply_buffer(struct iwl_fw_runtime *fwrt,
 			 IWL_UCODE_TLV_CAPA_DBG_BUF_ALLOC_CMD_SUPP))
 		return 0;
 
-	if (alloc_id < IWL_FW_INI_ALLOCATION_INVALID ||
+	if (alloc_id <= IWL_FW_INI_ALLOCATION_INVALID ||
 	    alloc_id >= IWL_FW_INI_ALLOCATION_NUM)
 		return -EIO;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
index 7f1515dfc5bf9d..0982e60842b1c6 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
@@ -1103,10 +1103,17 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			const struct iwl_fw_dbg_dest_tlv_v1 *dest_v1 = NULL;
 			u8 mon_mode;
 
+			if (tlv_len < sizeof(*pieces->dbg_dest_ver))
+				goto invalid_tlv_len;
+
 			pieces->dbg_dest_ver = (const u8 *)tlv_data;
 			if (*pieces->dbg_dest_ver == 1) {
+				if (tlv_len < sizeof(*dest))
+					goto invalid_tlv_len;
 				dest = (const void *)tlv_data;
 			} else if (*pieces->dbg_dest_ver == 0) {
+				if (tlv_len < sizeof(*dest_v1))
+					goto invalid_tlv_len;
 				dest_v1 = (const void *)tlv_data;
 			} else {
 				IWL_ERR(drv,
@@ -2039,8 +2046,6 @@ void iwl_drv_stop(struct iwl_drv *drv)
 	mutex_unlock(&iwlwifi_opmode_table_mtx);
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
-	iwl_trans_debugfs_cleanup(drv->trans);
-
 	debugfs_remove_recursive(drv->dbgfs_drv);
 #endif
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.c b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
index bb746112ddadf5..cc6bf4ea5055d5 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-io.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
@@ -10,8 +10,6 @@
 #include "iwl-io.h"
 #include "iwl-csr.h"
 #include "iwl-debug.h"
-#include "iwl-prph.h"
-#include "iwl-fh.h"
 
 void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val)
 {
@@ -217,232 +215,3 @@ void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
 	}
 }
 IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);
-
-void iwl_force_nmi(struct iwl_trans *trans)
-{
-	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
-		iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,
-				     DEVICE_SET_NMI_VAL_DRV, 1);
-	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
-		iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
-				UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);
-	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
-		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
-				    UREG_DOORBELL_TO_ISR6_NMI_BIT);
-	else
-		iwl_write32(trans, CSR_DOORBELL_VECTOR,
-			    UREG_DOORBELL_TO_ISR6_NMI_BIT);
-}
-IWL_EXPORT_SYMBOL(iwl_force_nmi);
-
-static const char *get_rfh_string(int cmd)
-{
-#define IWL_CMD(x) case x: return #x
-#define IWL_CMD_MQ(arg, reg, q) { if (arg == reg(q)) return #reg; }
-
-	int i;
-
-	for (i = 0; i < IWL_MAX_RX_HW_QUEUES; i++) {
-		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_BA_LSB, i);
-		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i);
-		IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i);
-		IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i);
-	}
-
-	switch (cmd) {
-	IWL_CMD(RFH_RXF_DMA_CFG);
-	IWL_CMD(RFH_GEN_CFG);
-	IWL_CMD(RFH_GEN_STATUS);
-	IWL_CMD(FH_TSSR_TX_STATUS_REG);
-	IWL_CMD(FH_TSSR_TX_ERROR_REG);
-	default:
-		return "UNKNOWN";
-	}
-#undef IWL_CMD_MQ
-}
-
-struct reg {
-	u32 addr;
-	bool is64;
-};
-
-static int iwl_dump_rfh(struct iwl_trans *trans, char **buf)
-{
-	int i, q;
-	int num_q = trans->info.num_rxqs;
-	static const u32 rfh_tbl[] = {
-		RFH_RXF_DMA_CFG,
-		RFH_GEN_CFG,
-		RFH_GEN_STATUS,
-		FH_TSSR_TX_STATUS_REG,
-		FH_TSSR_TX_ERROR_REG,
-	};
-	static const struct reg rfh_mq_tbl[] = {
-		{ RFH_Q0_FRBDCB_BA_LSB, true },
-		{ RFH_Q0_FRBDCB_WIDX, false },
-		{ RFH_Q0_FRBDCB_RIDX, false },
-		{ RFH_Q0_URBD_STTS_WPTR_LSB, true },
-	};
-
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-	if (buf) {
-		int pos = 0;
-		/*
-		 * Register (up to 34 for name + 8 blank/q for MQ): 40 chars
-		 * Colon + space: 2 characters
-		 * 0X%08x: 10 characters
-		 * New line: 1 character
-		 * Total of 53 characters
-		 */
-		size_t bufsz = ARRAY_SIZE(rfh_tbl) * 53 +
-			       ARRAY_SIZE(rfh_mq_tbl) * 53 * num_q + 40;
-
-		*buf = kmalloc(bufsz, GFP_KERNEL);
-		if (!*buf)
-			return -ENOMEM;
-
-		pos += scnprintf(*buf + pos, bufsz - pos,
-				"RFH register values:\n");
-
-		for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
-			pos += scnprintf(*buf + pos, bufsz - pos,
-				"%40s: 0X%08x\n",
-				get_rfh_string(rfh_tbl[i]),
-				iwl_read_prph(trans, rfh_tbl[i]));
-
-		for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
-			for (q = 0; q < num_q; q++) {
-				u32 addr = rfh_mq_tbl[i].addr;
-
-				addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
-				pos += scnprintf(*buf + pos, bufsz - pos,
-					"%34s(q %2d): 0X%08x\n",
-					get_rfh_string(addr), q,
-					iwl_read_prph(trans, addr));
-			}
-
-		return pos;
-	}
-#endif
-
-	IWL_ERR(trans, "RFH register values:\n");
-	for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
-		IWL_ERR(trans, "  %34s: 0X%08x\n",
-			get_rfh_string(rfh_tbl[i]),
-			iwl_read_prph(trans, rfh_tbl[i]));
-
-	for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
-		for (q = 0; q < num_q; q++) {
-			u32 addr = rfh_mq_tbl[i].addr;
-
-			addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
-			IWL_ERR(trans, "  %34s(q %d): 0X%08x\n",
-				get_rfh_string(addr), q,
-				iwl_read_prph(trans, addr));
-		}
-
-	return 0;
-}
-
-static const char *get_fh_string(int cmd)
-{
-	switch (cmd) {
-	IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
-	IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
-	IWL_CMD(FH_RSCSR_CHNL0_WPTR);
-	IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
-	IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
-	IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
-	IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
-	IWL_CMD(FH_TSSR_TX_STATUS_REG);
-	IWL_CMD(FH_TSSR_TX_ERROR_REG);
-	default:
-		return "UNKNOWN";
-	}
-#undef IWL_CMD
-}
-
-int iwl_dump_fh(struct iwl_trans *trans, char **buf)
-{
-	int i;
-	static const u32 fh_tbl[] = {
-		FH_RSCSR_CHNL0_STTS_WPTR_REG,
-		FH_RSCSR_CHNL0_RBDCB_BASE_REG,
-		FH_RSCSR_CHNL0_WPTR,
-		FH_MEM_RCSR_CHNL0_CONFIG_REG,
-		FH_MEM_RSSR_SHARED_CTRL_REG,
-		FH_MEM_RSSR_RX_STATUS_REG,
-		FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
-		FH_TSSR_TX_STATUS_REG,
-		FH_TSSR_TX_ERROR_REG
-	};
-
-	if (trans->mac_cfg->mq_rx_supported)
-		return iwl_dump_rfh(trans, buf);
-
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-	if (buf) {
-		int pos = 0;
-		size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;
-
-		*buf = kmalloc(bufsz, GFP_KERNEL);
-		if (!*buf)
-			return -ENOMEM;
-
-		pos += scnprintf(*buf + pos, bufsz - pos,
-				"FH register values:\n");
-
-		for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)
-			pos += scnprintf(*buf + pos, bufsz - pos,
-				"  %34s: 0X%08x\n",
-				get_fh_string(fh_tbl[i]),
-				iwl_read_direct32(trans, fh_tbl[i]));
-
-		return pos;
-	}
-#endif
-
-	IWL_ERR(trans, "FH register values:\n");
-	for (i = 0; i <  ARRAY_SIZE(fh_tbl); i++)
-		IWL_ERR(trans, "  %34s: 0X%08x\n",
-			get_fh_string(fh_tbl[i]),
-			iwl_read_direct32(trans, fh_tbl[i]));
-
-	return 0;
-}
-
-void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
-				  u32 sw_err_bit)
-{
-	unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;
-	bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);
-
-	/* if the interrupts were already disabled, there is no point in
-	 * calling iwl_disable_interrupts
-	 */
-	if (interrupts_enabled)
-		iwl_trans_interrupts(trans, false);
-
-	iwl_force_nmi(trans);
-	while (time_after(timeout, jiffies)) {
-		u32 inta_hw = iwl_read32(trans, inta_addr);
-
-		/* Error detected by uCode */
-		if (inta_hw & sw_err_bit) {
-			/* Clear causes register */
-			iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);
-			break;
-		}
-
-		mdelay(1);
-	}
-
-	/* enable interrupts only if there were already enabled before this
-	 * function to avoid a case were the driver enable interrupts before
-	 * proper configurations were made
-	 */
-	if (interrupts_enabled)
-		iwl_trans_interrupts(trans, true);
-
-	iwl_trans_fw_error(trans, IWL_ERR_TYPE_NMI_FORCED);
-}
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.h b/drivers/net/wireless/intel/iwlwifi/iwl-io.h
index 6dce2e5267a645..e4a9f2ba0de93d 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-io.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.h
@@ -57,10 +57,6 @@ void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask);
 void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs,
 			    u32 bits, u32 mask);
 void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask);
-void iwl_force_nmi(struct iwl_trans *trans);
-
-/* Error handling */
-int iwl_dump_fh(struct iwl_trans *trans, char **buf);
 
 /*
  * UMAC periphery address space changed from 0xA00000 to 0xD00000 starting from
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
index 863d5e358152db..413dd55115f8f5 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
@@ -612,7 +612,6 @@ static const struct ieee80211_sband_iftype_data iwl_iftype_cap[] = {
 					IEEE80211_HE_PHY_CAP7_POWER_BOOST_FACTOR_SUPP |
 					IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI,
 				.phy_cap_info[8] =
-					IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI |
 					IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G |
 					IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU |
 					IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU |
@@ -710,7 +709,8 @@ static const struct ieee80211_sband_iftype_data iwl_iftype_cap[] = {
 			.has_uhr = true,
 			/* Note: asymmetry is fixed later */
 			.phy.cap = cpu_to_le32(IEEE80211_UHR_PHY_CAP_ELR_RX |
-					       IEEE80211_UHR_PHY_CAP_ELR_TX),
+					       IEEE80211_UHR_PHY_CAP_ELR_TX |
+					       IEEE80211_UHR_PHY_CAP_2XLDPC_RX),
 			.mac.mac_cap = {
 				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP |
 				      IEEE80211_UHR_MAC_CAP0_DPS_SUPP,
@@ -746,7 +746,6 @@ static const struct ieee80211_sband_iftype_data iwl_iftype_cap[] = {
 				.phy_cap_info[7] =
 					IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI,
 				.phy_cap_info[8] =
-					IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI |
 					IEEE80211_HE_PHY_CAP8_DCM_MAX_RU_242,
 				.phy_cap_info[9] =
 					IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU |
@@ -820,7 +819,8 @@ static const struct ieee80211_sband_iftype_data iwl_iftype_cap[] = {
 			.has_uhr = true,
 			/* Note: asymmetry is fixed later */
 			.phy.cap = cpu_to_le32(IEEE80211_UHR_PHY_CAP_ELR_RX |
-					       IEEE80211_UHR_PHY_CAP_ELR_TX),
+					       IEEE80211_UHR_PHY_CAP_ELR_TX |
+					       IEEE80211_UHR_PHY_CAP_2XLDPC_RX),
 		},
 	},
 };
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
index 73aae11250421f..0a6b7b264aa230 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
@@ -10,9 +10,10 @@
 
 #include "iwl-trans.h"
 #include "iwl-drv.h"
+#include "iwl-prph.h"
 #include <linux/dmapool.h>
 #include "fw/api/commands.h"
-#include "pcie/gen1_2/internal.h"
+#include "pcie/internal.h"
 #include "pcie/iwl-context-info-v2.h"
 
 struct iwl_trans_dev_restart_data {
@@ -333,14 +334,14 @@ IWL_EXPORT_SYMBOL(iwl_trans_send_cmd);
 
 struct iwl_device_tx_cmd *iwl_trans_alloc_tx_cmd(struct iwl_trans *trans)
 {
-	return iwl_pcie_gen1_2_alloc_tx_cmd(trans);
+	return iwl_pcie_alloc_tx_cmd(trans);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_alloc_tx_cmd);
 
 void iwl_trans_free_tx_cmd(struct iwl_trans *trans,
 			   struct iwl_device_tx_cmd *dev_cmd)
 {
-	iwl_pcie_gen1_2_free_tx_cmd(trans, dev_cmd);
+	iwl_pcie_free_tx_cmd(trans, dev_cmd);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_free_tx_cmd);
 
@@ -395,7 +396,7 @@ void iwl_trans_op_mode_enter(struct iwl_trans *trans,
 
 	WARN_ON_ONCE(!trans->conf.rx_mpdu_cmd);
 
-	iwl_pcie_gen1_2_op_mode_enter(trans);
+	iwl_trans_pcie_op_mode_enter(trans);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_op_mode_enter);
 
@@ -452,6 +453,23 @@ void iwl_trans_write_prph(struct iwl_trans *trans, u32 ofs, u32 val)
 	return iwl_trans_pcie_write_prph(trans, ofs, val);
 }
 
+void iwl_trans_force_nmi(struct iwl_trans *trans)
+{
+	if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
+		iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,
+				     DEVICE_SET_NMI_VAL_DRV, 1);
+	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
+		iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
+				    UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);
+	else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
+		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
+				    UREG_DOORBELL_TO_ISR6_NMI_BIT);
+	else
+		iwl_write32(trans, CSR_DOORBELL_VECTOR,
+			    UREG_DOORBELL_TO_ISR6_NMI_BIT);
+}
+IWL_EXPORT_SYMBOL(iwl_trans_force_nmi);
+
 int iwl_trans_read_mem(struct iwl_trans *trans, u32 addr,
 		       void *buf, int dwords)
 {
@@ -523,11 +541,6 @@ int iwl_trans_d3_resume(struct iwl_trans *trans, bool reset)
 }
 IWL_EXPORT_SYMBOL(iwl_trans_d3_resume);
 
-void iwl_trans_interrupts(struct iwl_trans *trans, bool enable)
-{
-	iwl_trans_pci_interrupts(trans, enable);
-}
-
 void iwl_trans_sync_nmi(struct iwl_trans *trans)
 {
 	iwl_trans_pcie_sync_nmi(trans);
@@ -563,8 +576,7 @@ void iwl_trans_resched_with_nic_access(struct iwl_trans *trans)
 	iwl_trans_pcie_resched_with_nic_access(trans);
 }
 
-void __releases(nic_access)
-iwl_trans_release_nic_access(struct iwl_trans *trans)
+void iwl_trans_release_nic_access(struct iwl_trans *trans)
 {
 	iwl_trans_pcie_release_nic_access(trans);
 }
@@ -746,13 +758,6 @@ void iwl_trans_txq_set_shared_mode(struct iwl_trans *trans,
 }
 IWL_EXPORT_SYMBOL(iwl_trans_txq_set_shared_mode);
 
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-void iwl_trans_debugfs_cleanup(struct iwl_trans *trans)
-{
-	iwl_trans_pcie_debugfs_cleanup(trans);
-}
-#endif
-
 void iwl_trans_set_q_ptrs(struct iwl_trans *trans, int queue, int ptr)
 {
 	if (WARN_ONCE(trans->state != IWL_TRANS_FW_ALIVE,
@@ -828,13 +833,13 @@ IWL_EXPORT_SYMBOL(iwl_trans_is_pm_supported);
 
 bool iwl_trans_is_ltr_enabled(struct iwl_trans *trans)
 {
-	return iwl_pcie_gen1_2_is_ltr_enabled(trans);
+	return iwl_pcie_is_ltr_enabled(trans);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_is_ltr_enabled);
 
 int iwl_trans_activate_nic(struct iwl_trans *trans)
 {
-	return iwl_pcie_gen1_2_activate_nic(trans);
+	return iwl_pcie_activate_nic(trans);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_activate_nic);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index c581cc8f5bcb89..56c5e61b614ad0 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -934,10 +934,6 @@ int iwl_trans_read_mem_no_grab(struct iwl_trans *trans, u32 addr,
 int iwl_trans_read_config32(struct iwl_trans *trans, u32 ofs,
 			    u32 *val);
 
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-void iwl_trans_debugfs_cleanup(struct iwl_trans *trans);
-#endif
-
 #define iwl_trans_read_mem_bytes(trans, addr, buf, bufsize)	\
 	({							\
 		if (__builtin_constant_p(bufsize))		\
@@ -996,8 +992,7 @@ bool iwl_trans_grab_nic_access(struct iwl_trans *trans);
  */
 void iwl_trans_resched_with_nic_access(struct iwl_trans *trans);
 
-void __releases(nic_access)
-iwl_trans_release_nic_access(struct iwl_trans *trans);
+void iwl_trans_release_nic_access(struct iwl_trans *trans);
 
 static inline void iwl_trans_schedule_reset(struct iwl_trans *trans,
 					    enum iwl_fw_error_type type)
@@ -1059,8 +1054,7 @@ static inline bool iwl_trans_fw_running(struct iwl_trans *trans)
 
 void iwl_trans_sync_nmi(struct iwl_trans *trans);
 
-void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
-				  u32 sw_err_bit);
+void iwl_trans_force_nmi(struct iwl_trans *trans);
 
 int iwl_trans_load_pnvm(struct iwl_trans *trans,
 			const struct iwl_pnvm_image *pnvm_data,
@@ -1082,8 +1076,6 @@ static inline bool iwl_trans_dbg_ini_valid(struct iwl_trans *trans)
 		trans->dbg.external_ini_cfg != IWL_INI_CFG_STATE_NOT_LOADED;
 }
 
-void iwl_trans_interrupts(struct iwl_trans *trans, bool enable);
-
 int iwl_trans_activate_nic(struct iwl_trans *trans);
 
 static inline void iwl_trans_finish_sw_reset(struct iwl_trans *trans)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/agg.c b/drivers/net/wireless/intel/iwlwifi/mld/agg.c
index 96102d2af2cf97..ea1f02380ad4ef 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/agg.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/agg.c
@@ -7,7 +7,8 @@
 #include "hcmd.h"
 
 static void
-iwl_mld_reorder_release_frames(struct iwl_mld *mld, struct ieee80211_sta *sta,
+iwl_mld_reorder_release_frames(struct iwl_mld *mld,
+			       struct ieee80211_link_sta *link_sta,
 			       struct napi_struct *napi,
 			       struct iwl_mld_baid_data *baid_data,
 			       struct iwl_mld_reorder_buffer *reorder_buf,
@@ -32,7 +33,7 @@ iwl_mld_reorder_release_frames(struct iwl_mld *mld, struct ieee80211_sta *sta,
 		while ((skb = __skb_dequeue(skb_list))) {
 			iwl_mld_pass_packet_to_mac80211(mld, napi, skb,
 							reorder_buf->queue,
-							sta);
+							link_sta);
 			reorder_buf->num_stored--;
 		}
 	}
@@ -74,7 +75,7 @@ static void iwl_mld_release_frames_from_notif(struct iwl_mld *mld,
 
 	reorder_buf = &ba_data->reorder_buf[queue];
 
-	iwl_mld_reorder_release_frames(mld, link_sta->sta, napi, ba_data,
+	iwl_mld_reorder_release_frames(mld, link_sta, napi, ba_data,
 				       reorder_buf, nssn);
 out_unlock:
 	rcu_read_unlock();
@@ -180,7 +181,7 @@ void iwl_mld_del_ba(struct iwl_mld *mld, int queue,
 	reorder_buf = &ba_data->reorder_buf[queue];
 
 	/* release all frames that are in the reorder buffer to the stack */
-	iwl_mld_reorder_release_frames(mld, link_sta->sta, NULL,
+	iwl_mld_reorder_release_frames(mld, link_sta, NULL,
 				       ba_data, reorder_buf,
 				       ieee80211_sn_add(reorder_buf->head_sn,
 							ba_data->buf_size));
@@ -193,7 +194,7 @@ void iwl_mld_del_ba(struct iwl_mld *mld, int queue,
  */
 enum iwl_mld_reorder_result
 iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
-		int queue, struct ieee80211_sta *sta,
+		int queue, struct ieee80211_link_sta *link_sta,
 		struct sk_buff *skb, struct iwl_rx_mpdu_desc *desc)
 {
 	struct ieee80211_hdr *hdr = (void *)skb_mac_header(skb);
@@ -228,12 +229,12 @@ iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
 		return IWL_MLD_PASS_SKB;
 
 	/* no sta yet */
-	if (IWL_FW_CHECK(mld, !sta,
+	if (IWL_FW_CHECK(mld, !link_sta,
 			 "Got valid BAID without a valid station assigned - %d\n",
 			 baid))
 		return IWL_MLD_PASS_SKB;
 
-	mld_sta = iwl_mld_sta_from_mac80211(sta);
+	mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
 
 	/* not a data packet */
 	if (!ieee80211_is_data_qos(hdr->frame_control) ||
@@ -324,7 +325,7 @@ iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
 	 * will be released when the frame release notification arrives.
 	 */
 	if (!amsdu || last_subframe)
-		iwl_mld_reorder_release_frames(mld, sta, napi, baid_data,
+		iwl_mld_reorder_release_frames(mld, link_sta, napi, baid_data,
 					       buffer, nssn);
 	else if (buffer->num_stored == 1)
 		buffer->head_sn = nssn;
@@ -659,6 +660,7 @@ int iwl_mld_update_sta_baids(struct iwl_mld *mld,
 		.modify.new_sta_id_mask = cpu_to_le32(new_sta_mask),
 	};
 	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, RX_BAID_ALLOCATION_CONFIG_CMD);
+	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 2);
 	int baid;
 
 	/* mac80211 will remove sessions later, but we ignore all that */
@@ -666,6 +668,7 @@ int iwl_mld_update_sta_baids(struct iwl_mld *mld,
 		return 0;
 
 	BUILD_BUG_ON(sizeof(struct iwl_rx_baid_cfg_resp) != sizeof(baid));
+	BUILD_BUG_ON(sizeof(cmd.modify) != sizeof(cmd.modify_v2));
 
 	for (baid = 0; baid < ARRAY_SIZE(mld->fw_id_to_ba); baid++) {
 		struct iwl_mld_baid_data *data;
@@ -682,7 +685,10 @@ int iwl_mld_update_sta_baids(struct iwl_mld *mld,
 			  "BAID data for %d corrupted - expected 0x%x found 0x%x\n",
 			  baid, old_sta_mask, data->sta_mask);
 
-		cmd.modify.tid = cpu_to_le32(data->tid);
+		if (cmd_ver >= 3)
+			cmd.modify.tid = data->tid;
+		else
+			cmd.modify_v2.tid = cpu_to_le32(data->tid);
 
 		ret = iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd);
 		if (ret)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/agg.h b/drivers/net/wireless/intel/iwlwifi/mld/agg.h
index 651c80d1c7cdaf..c6cd5fa219bed9 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/agg.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/agg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024, 2026 Intel Corporation
  */
 #ifndef __iwl_agg_h__
 #define __iwl_agg_h__
@@ -106,7 +106,7 @@ int iwl_mld_ampdu_rx_stop(struct iwl_mld *mld, struct ieee80211_sta *sta,
 
 enum iwl_mld_reorder_result
 iwl_mld_reorder(struct iwl_mld *mld, struct napi_struct *napi,
-		int queue, struct ieee80211_sta *sta,
+		int queue, struct ieee80211_link_sta *link_sta,
 		struct sk_buff *skb, struct iwl_rx_mpdu_desc *desc);
 
 void iwl_mld_handle_frame_release_notif(struct iwl_mld *mld,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.c b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
index 55f9b809e764d8..6d8ede336d59bf 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
@@ -1138,7 +1138,8 @@ iwl_mld_add_mcast_rekey(struct ieee80211_vif *vif,
 		return;
 
 	key_config = ieee80211_gtk_rekey_add(vif, key_data->id, key_data->key,
-					     sizeof(key_data->key), link_id);
+					     sizeof(key_data->key), link_id,
+					     false);
 	if (IS_ERR(key_config))
 		return;
 
@@ -1220,7 +1221,8 @@ static void iwl_mld_mlo_rekey(struct iwl_mld *mld,
 				 mlo_key->idx, link_id);
 
 		key = ieee80211_gtk_rekey_add(vif, mlo_key->idx, mlo_key->key,
-					      sizeof(mlo_key->key), link_id);
+					      sizeof(mlo_key->key), link_id,
+					      false);
 
 		if (IS_ERR(key))
 			continue;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
index 351a4f177e920f..dd4bf7cb6a7911 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
@@ -69,7 +69,7 @@ static ssize_t iwl_dbgfs_fw_nmi_write(struct iwl_mld *mld, char *buf,
 	if (count == 6 && !strcmp(buf, "nolog\n"))
 		mld->fw_status.do_not_dump_once = true;
 
-	iwl_force_nmi(mld->trans);
+	iwl_trans_force_nmi(mld->trans);
 
 	return count;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/fw.c b/drivers/net/wireless/intel/iwlwifi/mld/fw.c
index 7b1fb84a641c79..e5866d89176d1e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/fw.c
@@ -396,8 +396,10 @@ void iwl_mld_send_recovery_cmd(struct iwl_mld *mld, u32 flags)
 
 	if (flags & ERROR_RECOVERY_UPDATE_DB) {
 		/* no buf was allocated upon NIC error */
-		if (!mld->error_recovery_buf)
+		if (!mld->error_recovery_buf) {
+			IWL_ERR(mld, "Recovering without recovery buffer. Expect bugs\n");
 			return;
+		}
 
 		cmd.data[1] = mld->error_recovery_buf;
 		cmd.len[1] =  error_log_size;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h b/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h
index 19e1c9e509cfc0..761ef73dd9a59f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/hcmd.h
@@ -5,6 +5,8 @@
 #ifndef __iwl_mld_hcmd_h__
 #define __iwl_mld_hcmd_h__
 
+#include "mld.h"
+
 static inline int iwl_mld_send_cmd(struct iwl_mld *mld, struct iwl_host_cmd *cmd)
 {
 	/* No commands, including the d3 related commands, should be sent
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/key.c b/drivers/net/wireless/intel/iwlwifi/mld/key.c
index bf80b4770b5af1..f2c87f830ba306 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/key.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/key.c
@@ -37,6 +37,12 @@ static u32 iwl_mld_get_key_flags(struct iwl_mld *mld,
 		break;
 	}
 
+	/* Note that sta->mfp might not be set yet, as it is set only
+	 * after association.
+	 */
+	if (sta && pairwise && sta->epp_peer)
+		flags |= IWL_SEC_KEY_FLAG_ASSOC_ENC | IWL_SEC_KEY_FLAG_MFP;
+
 	/* If we are installing an iGTK (in AP or STA mode), we need to tell
 	 * the firmware this key will en/decrypt MGMT frames.
 	 * Same goes if we are installing a pairwise key for an MFP station.
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/link.c b/drivers/net/wireless/intel/iwlwifi/mld/link.c
index 96e06940b34c41..c7d50ba30b36a7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/link.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/link.c
@@ -361,13 +361,6 @@ iwl_mld_change_link_in_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link,
 			flags |= LINK_FLG_MLPM;
 	}
 
-	cmd.htc_trig_based_pkt_ext = link->htc_trig_based_pkt_ext;
-
-	if (link->uora_exists) {
-		cmd.rand_alloc_ecwmin = link->uora_ocw_range & 0x7;
-		cmd.rand_alloc_ecwmax = (link->uora_ocw_range >> 3) & 0x7;
-	}
-
 	if (iwl_mld_fill_mu_edca(mld, mld_link, cmd.trig_based_txf))
 		flags |= LINK_FLG_MU_EDCA_CW;
 
@@ -376,8 +369,6 @@ iwl_mld_change_link_in_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link,
 	if (!link->he_bss_color.enabled)
 		flags |= LINK_FLG_BSS_COLOR_DIS;
 
-	cmd.frame_time_rts_th = cpu_to_le16(link->frame_time_rts_th);
-
 	/* Block 26-tone RU OFDMA transmissions */
 	if (mld_link->he_ru_2mhz_block)
 		flags |= LINK_FLG_RU_2MHZ_BLOCK;
@@ -387,6 +378,11 @@ iwl_mld_change_link_in_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link,
 		cmd.bssid_index = link->bssid_index;
 	}
 
+	if (iwl_fw_lookup_cmd_ver(mld->fw,
+				  WIDE_ID(MAC_CONF_GROUP, LINK_CONFIG_CMD),
+				  0) >= 9)
+		cmd.max_bssid_indicator = link->bssid_indicator;
+
 	/* The only EHT parameter is puncturing, and starting from PHY cmd
 	 * version 6 - it is sent there. For older versions of the PHY cmd,
 	 * puncturing is not needed at all.
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
index 3a4c8fda68d0a4..0b121114d47ca1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
@@ -395,6 +395,10 @@ static void iwl_mac_hw_set_wiphy(struct iwl_mld *mld)
 	if (fw_has_capa(ucode_capa, IWL_UCODE_TLV_CAPA_PROTECTED_TWT))
 		wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_PROTECTED_TWT);
 
+	if (fw_has_capa(ucode_capa, IWL_UCODE_TLV_CAPA_ASSOC_ENC_SUPPORT))
+		wiphy_ext_feature_set(wiphy,
+				      NL80211_EXT_FEATURE_ASSOC_FRAME_ENCRYPTION);
+
 	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM)
 		wiphy->flags |= WIPHY_FLAG_PS_ON_BY_DEFAULT;
 	else
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.c b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
index 6e991fb99ff394..b7b1e3686336c2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mld.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
@@ -621,6 +621,13 @@ static void iwl_mld_read_error_recovery_buffer(struct iwl_mld *mld)
 	if (!src_size)
 		return;
 
+	/*
+	 * If we have a the recovery buffer from a previous (failing) recovery -
+	 * free it here
+	 */
+	kfree(mld->error_recovery_buf);
+	mld->error_recovery_buf = NULL;
+
 	recovery_buf = kzalloc(src_size, GFP_ATOMIC);
 	if (!recovery_buf)
 		return;
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/nan.c b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
index dbe7a54a38b71c..10df33a238d1ab 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/nan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
@@ -5,6 +5,7 @@
 
 #include "mld.h"
 #include "iface.h"
+#include "key.h"
 #include "link.h"
 #include "mlo.h"
 #include "tlc.h"
@@ -107,6 +108,20 @@ static void iwl_mld_nan_flush(struct iwl_mld *mld, struct ieee80211_vif *vif)
 	iwl_mld_flush_link_sta_txqs(mld, sta->sta_id);
 }
 
+static void iwl_mld_remove_nan_keys_iter(struct ieee80211_hw *hw,
+					 struct ieee80211_vif *vif,
+					 struct ieee80211_sta *sta,
+					 struct ieee80211_key_conf *key,
+					 void *data)
+{
+	struct iwl_mld *mld = data;
+
+	if (sta || key->hw_key_idx == STA_KEY_IDX_INVALID)
+		return;
+
+	iwl_mld_remove_key(mld, vif, NULL, key);
+}
+
 static void iwl_mld_nan_remove_stations(struct iwl_mld *mld,
 					struct ieee80211_vif *vif)
 {
@@ -296,6 +311,8 @@ int iwl_mld_stop_nan(struct ieee80211_hw *hw,
 	if (ret)
 		IWL_ERR(mld, "NAN: Failed to stop NAN. ret=%d\n", ret);
 
+	ieee80211_iter_keys(mld->hw, vif, iwl_mld_remove_nan_keys_iter, mld);
+
 	/* assume that higher layer guarantees that no additional frames are
 	 * added before calling this callback
 	 */
@@ -576,21 +593,11 @@ static int iwl_mld_nan_link_set_active(struct iwl_mld *mld,
 				       bool active)
 {
 	struct iwl_link_config_cmd cmd;
-	struct ieee80211_sta *sta;
 	int ret;
 
 	if (nan_link->active == active)
 		return 0;
 
-	if (active) {
-		for_each_station(sta, mld->hw) {
-			struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
-
-			if (mld_sta->sta_type == STATION_TYPE_NAN_PEER_NDI)
-				iwl_mld_config_tlc(mld, mld_sta->vif, sta);
-		}
-	}
-
 	nan_link->active = active;
 
 	iwl_mld_nan_link_prep_cmd(mld, nan_link, &cmd,
@@ -602,10 +609,8 @@ static int iwl_mld_nan_link_set_active(struct iwl_mld *mld,
 		return ret;
 	}
 
-	if (!active) {
+	if (!active)
 		nan_link->chanctx = NULL;
-		/* TODO: when FW is ready, Update phy in TLC to invalid after */
-	}
 
 	return 0;
 }
@@ -847,6 +852,10 @@ void iwl_mld_nan_vif_cfg_changed(struct iwl_mld *mld,
 			if (mld_sta->sta_type == STATION_TYPE_NAN_PEER_NMI ||
 			    mld_sta->sta_type == STATION_TYPE_NAN_PEER_NDI)
 				iwl_mld_add_modify_sta_cmd(mld, &sta->deflink);
+
+			if (added_links &&
+			    mld_sta->sta_type == STATION_TYPE_NAN_PEER_NDI)
+				iwl_mld_config_tlc(mld, mld_sta->vif, sta);
 		}
 
 		/*
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.c b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
index 90733bbaf73377..570b0011fd9e94 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/notif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
@@ -292,7 +292,8 @@ CMD_VERSIONS(scan_start_notif,
 CMD_VERSIONS(scan_complete_notif,
 	     CMD_VER_ENTRY(1, iwl_umac_scan_complete))
 CMD_VERSIONS(scan_iter_complete_notif,
-	     CMD_VER_ENTRY(2, iwl_umac_scan_iter_complete_notif))
+	     CMD_VER_ENTRY(2, iwl_umac_scan_iter_complete_notif_v2)
+	     CMD_VER_ENTRY(3, iwl_umac_scan_iter_complete_notif))
 CMD_VERSIONS(channel_survey_notif,
 	     CMD_VER_ENTRY(1, iwl_umac_scan_channel_survey_notif))
 CMD_VERSIONS(mfuart_notif,
@@ -329,7 +330,8 @@ CMD_VERSIONS(datapath_monitor_notif,
 	     CMD_VER_ENTRY(1, iwl_datapath_monitor_notif))
 CMD_VERSIONS(stats_oper_notif,
 	     CMD_VER_ENTRY(3, iwl_system_statistics_notif_oper_v3)
-	     CMD_VER_ENTRY(4, iwl_system_statistics_notif_oper))
+	     CMD_VER_ENTRY(4, iwl_system_statistics_notif_oper_v4)
+	     CMD_VER_ENTRY(5, iwl_system_statistics_notif_oper))
 CMD_VERSIONS(stats_oper_part1_notif,
 	     CMD_VER_ENTRY(4, iwl_system_statistics_part1_notif_oper))
 CMD_VERSIONS(bt_coex_notif,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/power.c b/drivers/net/wireless/intel/iwlwifi/mld/power.c
index 2387f3da5185a8..6fcbb1a2e8a8f2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/power.c
@@ -525,7 +525,7 @@ int iwl_mld_set_tx_power(struct iwl_mld *mld,
 		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_LINK),
 		.common.pwr_restriction = cpu_to_le16(u_tx_power),
 	};
-	int len = sizeof(cmd.common) + sizeof(cmd.v11);
+	int len = sizeof(cmd.common) + sizeof(cmd.v12);
 
 	if (iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 10) == 10)
 		len = sizeof(cmd.common) + sizeof(cmd.v10);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
index 6db84c2117f25b..8a20d7622329cd 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.c
@@ -6,7 +6,6 @@
 #include <linux/dmi.h>
 
 #include "fw/regulatory.h"
-#include "fw/acpi.h"
 #include "fw/uefi.h"
 
 #include "regulatory.h"
@@ -37,7 +36,7 @@ void iwl_mld_get_bios_tables(struct iwl_mld *mld)
 {
 	int ret;
 
-	iwl_acpi_get_guid_lock_status(&mld->fwrt);
+	iwl_bios_get_guid_lock_status(&mld->fwrt);
 
 	ret = iwl_bios_get_ppag_table(&mld->fwrt);
 	if (ret < 0) {
@@ -75,6 +74,21 @@ void iwl_mld_get_bios_tables(struct iwl_mld *mld)
 					"EWRD SAR BIOS table invalid or unavailable. (%d)\n",
 					ret);
 
+		ret = iwl_bios_get_wsss_table(&mld->fwrt);
+		if (ret < 0)
+			IWL_DEBUG_RADIO(mld,
+					"WSSS (WRDS Standalone) SAR BIOS table invalid or unavailable. (%d)\n",
+					ret);
+
+		if (!ret) {
+			/* EWSS only extends the standalone SAR profiles from WSSS. */
+			ret = iwl_bios_get_ewss_table(&mld->fwrt);
+			if (ret < 0)
+				IWL_DEBUG_RADIO(mld,
+						"EWSS (EWRD Standalone) SAR BIOS table invalid or unavailable. (%d)\n",
+						ret);
+		}
+
 		ret = iwl_bios_get_wgds_table(&mld->fwrt);
 		if (ret < 0)
 			IWL_DEBUG_RADIO(mld,
@@ -136,7 +150,9 @@ static int iwl_mld_geo_sar_init(struct iwl_mld *mld)
 	return iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd, cmd_size);
 }
 
-int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b)
+static int
+iwl_mld_config_sar_concurrent_profile(struct iwl_mld *mld, int prof_a,
+				      int prof_b)
 {
 	struct iwl_dev_tx_power_cmd cmd = {
 		.common.set_mode = cpu_to_le32(IWL_TX_POWER_MODE_SET_CHAINS),
@@ -153,8 +169,9 @@ int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b)
 		num_subbands = IWL_NUM_SUB_BANDS_V2;
 		break;
 	case 11:
-		cmd.v11.flags = cpu_to_le32(mld->fwrt.reduced_power_flags);
-		cmd_size = sizeof(cmd.common) + sizeof(cmd.v11);
+	case 12:
+		cmd.v12.flags = cpu_to_le32(mld->fwrt.reduced_power_flags);
+		cmd_size = sizeof(cmd.common) + sizeof(cmd.v12);
 		num_subbands = IWL_NUM_SUB_BANDS_V3;
 		break;
 	default:
@@ -164,12 +181,13 @@ int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b)
 	}
 
 	/* TODO: CDB - support IWL_NUM_CHAIN_TABLES_V2 */
-	/* v10 and v11 have the same position for per_chain */
-	BUILD_BUG_ON(offsetof(typeof(cmd), v11.per_chain) !=
+	/* v10 and v12 have the same position for per_chain */
+	BUILD_BUG_ON(offsetof(typeof(cmd), v12.per_chain) !=
 		     offsetof(typeof(cmd), v10.per_chain));
-	ret = iwl_sar_fill_profile(&mld->fwrt, &cmd.v11.per_chain[0][0][0],
+	ret = iwl_sar_fill_profile(&mld->fwrt, &cmd.v12.per_chain[0][0][0],
 				   IWL_NUM_CHAIN_TABLES, num_subbands,
-				   prof_a, prof_b);
+				   prof_a, prof_b,
+				   mld->fwrt.sar_profiles, false);
 	/* return on error or if the profile is disabled (positive number) */
 	if (ret)
 		return ret;
@@ -177,6 +195,51 @@ int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b)
 	return iwl_mld_send_cmd_pdu(mld, REDUCE_TX_POWER_CMD, &cmd, cmd_size);
 }
 
+static int iwl_mld_config_sar_standalone_profile(struct iwl_mld *mld,
+						 int prof_a, int prof_b)
+{
+	struct iwl_dev_tx_power_cmd cmd = {
+		.common.set_mode =
+			cpu_to_le32(IWL_TX_POWER_MODE_SET_STANDALONE_CHAINS),
+	};
+	int ret;
+
+	/* standalone per-chain SAR limits require cmd version 12 */
+	if (iwl_fw_lookup_cmd_ver(mld->fw, REDUCE_TX_POWER_CMD, 10) != 12)
+		return 0;
+
+	ret = iwl_sar_fill_profile(&mld->fwrt,
+				   &cmd.v12.per_chain[0][0][0],
+				   IWL_NUM_CHAIN_TABLES,
+				   IWL_NUM_SUB_BANDS_V3,
+				   prof_a, prof_b,
+				   mld->fwrt.sar_standalone_profiles,
+				   true);
+	/* return on error or if the standalone profile is disabled */
+	if (ret)
+		return ret;
+
+	return iwl_mld_send_cmd_pdu(mld, REDUCE_TX_POWER_CMD, &cmd,
+				    sizeof(cmd.common) + sizeof(cmd.v12));
+}
+
+int iwl_mld_config_sar_profiles(struct iwl_mld *mld, int prof_a, int prof_b)
+{
+	int ret;
+
+	ret = iwl_mld_config_sar_concurrent_profile(mld, prof_a, prof_b);
+	/* return on error or if the profile is disabled (positive number) */
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_config_sar_standalone_profile(mld, prof_a, prof_b);
+	/* standalone profile is optional and may be disabled (positive number) */
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
 int iwl_mld_init_sar(struct iwl_mld *mld)
 {
 	int chain_a_prof = 1;
@@ -191,7 +254,7 @@ int iwl_mld_init_sar(struct iwl_mld *mld)
 		chain_b_prof = mld->fwrt.sar_chain_b_profile;
 	}
 
-	ret = iwl_mld_config_sar_profile(mld, chain_a_prof, chain_b_prof);
+	ret = iwl_mld_config_sar_profiles(mld, chain_a_prof, chain_b_prof);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h
index 5498c19789f4e4..0877b951f2e27f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/regulatory.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifndef __iwl_mld_regulatory_h__
 #define __iwl_mld_regulatory_h__
@@ -18,6 +18,6 @@ int iwl_mld_init_sgom(struct iwl_mld *mld);
 
 int iwl_mld_init_sar(struct iwl_mld *mld);
 
-int iwl_mld_config_sar_profile(struct iwl_mld *mld, int prof_a, int prof_b);
+int iwl_mld_config_sar_profiles(struct iwl_mld *mld, int prof_a, int prof_b);
 
 #endif /* __iwl_mld_regulatory_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/rx.c b/drivers/net/wireless/intel/iwlwifi/mld/rx.c
index 269439d789f46a..5f18aa551af52f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/rx.c
@@ -47,7 +47,8 @@ iwl_mld_fill_phy_data_from_mpdu(struct iwl_mld *mld,
 }
 
 static inline int iwl_mld_check_pn(struct iwl_mld *mld, struct sk_buff *skb,
-				   int queue, struct ieee80211_sta *sta)
+				   int queue,
+				   struct ieee80211_link_sta *link_sta)
 {
 	struct ieee80211_hdr *hdr = (void *)skb_mac_header(skb);
 	struct ieee80211_rx_status *stats = IEEE80211_SKB_RXCB(skb);
@@ -71,13 +72,13 @@ static inline int iwl_mld_check_pn(struct iwl_mld *mld, struct sk_buff *skb,
 		return 0;
 
 	/* if we are here - this for sure is either CCMP or GCMP */
-	if (!sta) {
+	if (!link_sta) {
 		IWL_DEBUG_DROP(mld,
 			       "expected hw-decrypted unicast frame for station\n");
 		return -1;
 	}
 
-	mld_sta = iwl_mld_sta_from_mac80211(sta);
+	mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
 
 	extiv = (u8 *)hdr + ieee80211_hdrlen(hdr->frame_control);
 	keyidx = extiv[3] >> 6;
@@ -119,17 +120,17 @@ static inline int iwl_mld_check_pn(struct iwl_mld *mld, struct sk_buff *skb,
 void iwl_mld_pass_packet_to_mac80211(struct iwl_mld *mld,
 				     struct napi_struct *napi,
 				     struct sk_buff *skb, int queue,
-				     struct ieee80211_sta *sta)
+				     struct ieee80211_link_sta *link_sta)
 {
 	KUNIT_STATIC_STUB_REDIRECT(iwl_mld_pass_packet_to_mac80211,
-				   mld, napi, skb, queue, sta);
+				   mld, napi, skb, queue, link_sta);
 
-	if (unlikely(iwl_mld_check_pn(mld, skb, queue, sta))) {
+	if (unlikely(iwl_mld_check_pn(mld, skb, queue, link_sta))) {
 		kfree_skb(skb);
 		return;
 	}
 
-	ieee80211_rx_napi(mld->hw, sta, skb, napi);
+	ieee80211_rx_napi(mld->hw, link_sta, skb, napi);
 }
 EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_pass_packet_to_mac80211);
 
@@ -817,7 +818,8 @@ static void iwl_mld_decode_eht_usig_tb(struct iwl_mld_rx_phy_data *phy_data,
 	__le32 usig_a2 = phy_data->ntfy->sigs.eht_tb.usig_a2_eht;
 
 	IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a1,
-				    OFDM_RX_FRAME_EHT_USIG1_DISREGARD,
+				    OFDM_RX_FRAME_EHT_USIG1_DISREGARD |
+				    OFDM_RX_FRAME_EHT_USIG1_VALIDATE,
 				    IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD);
 	IWL_MLD_ENC_USIG_VALUE_MASK(usig, usig_a2,
 				    OFDM_RX_FRAME_EHT_PPDU_TYPE,
@@ -1728,7 +1730,7 @@ static void iwl_mld_update_last_rx_timestamp(struct iwl_mld *mld, u8 baid)
  * Sets *drop to true if the packet should be dropped.
  * Returns the station if found, or NULL otherwise.
  */
-static struct ieee80211_sta *
+static struct ieee80211_link_sta *
 iwl_mld_rx_with_sta(struct iwl_mld *mld, struct ieee80211_hdr *hdr,
 		    struct sk_buff *skb,
 		    const struct iwl_rx_mpdu_desc *mpdu_desc,
@@ -1764,11 +1766,6 @@ iwl_mld_rx_with_sta(struct iwl_mld *mld, struct ieee80211_hdr *hdr,
 
 	rx_status = IEEE80211_SKB_RXCB(skb);
 
-	if (link_sta && sta->valid_links) {
-		rx_status->link_valid = true;
-		rx_status->link_id = link_sta->link_id;
-	}
-
 	/* fill checksum */
 	if (ieee80211_is_data(hdr->frame_control) &&
 	    pkt->len_n_flags & cpu_to_le32(FH_RSCSR_RPA_EN)) {
@@ -1803,10 +1800,10 @@ iwl_mld_rx_with_sta(struct iwl_mld *mld, struct ieee80211_hdr *hdr,
 							    queue);
 	}
 
-	return sta;
+	return link_sta;
 }
 
-static int iwl_mld_rx_mgmt_prot(struct ieee80211_sta *sta,
+static int iwl_mld_rx_mgmt_prot(struct ieee80211_link_sta *link_sta,
 				struct ieee80211_hdr *hdr,
 				struct ieee80211_rx_status *rx_status,
 				u32 mpdu_status,
@@ -1820,7 +1817,6 @@ static int iwl_mld_rx_mgmt_prot(struct ieee80211_sta *sta,
 	struct ieee80211_key_conf *key;
 	const u8 *frame = (void *)hdr;
 	const u8 *mmie;
-	u8 link_id;
 
 	if ((mpdu_status & IWL_RX_MPDU_STATUS_SEC_MASK) ==
 	     IWL_RX_MPDU_STATUS_SEC_NONE)
@@ -1836,10 +1832,10 @@ static int iwl_mld_rx_mgmt_prot(struct ieee80211_sta *sta,
 	if (!ieee80211_is_beacon(hdr->frame_control))
 		return 0;
 
-	if (!sta)
+	if (!link_sta)
 		return -1;
 
-	mld_sta = iwl_mld_sta_from_mac80211(sta);
+	mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
 	mld_vif = iwl_mld_vif_from_mac80211(mld_sta->vif);
 
 	/* key mismatch - will also report !MIC_OK but we shouldn't count it */
@@ -1853,8 +1849,7 @@ static int iwl_mld_rx_mgmt_prot(struct ieee80211_sta *sta,
 		return 0;
 	}
 
-	link_id = rx_status->link_valid ? rx_status->link_id : 0;
-	link = rcu_dereference(mld_vif->link[link_id]);
+	link = rcu_dereference(mld_vif->link[link_sta->link_id]);
 	if (WARN_ON_ONCE(!link))
 		return -1;
 
@@ -1905,7 +1900,7 @@ static int iwl_mld_rx_mgmt_prot(struct ieee80211_sta *sta,
 }
 
 static int iwl_mld_rx_crypto(struct iwl_mld *mld,
-			     struct ieee80211_sta *sta,
+			     struct ieee80211_link_sta *link_sta,
 			     struct ieee80211_hdr *hdr,
 			     struct ieee80211_rx_status *rx_status,
 			     struct iwl_rx_mpdu_desc *desc, int queue,
@@ -1915,7 +1910,7 @@ static int iwl_mld_rx_crypto(struct iwl_mld *mld,
 
 	if (unlikely(ieee80211_is_mgmt(hdr->frame_control) &&
 		     !ieee80211_has_protected(hdr->frame_control)))
-		return iwl_mld_rx_mgmt_prot(sta, hdr, rx_status, status,
+		return iwl_mld_rx_mgmt_prot(link_sta, hdr, rx_status, status,
 					    le16_to_cpu(desc->mpdu_len));
 
 	if (!ieee80211_has_protected(hdr->frame_control) ||
@@ -2023,7 +2018,7 @@ void iwl_mld_rx_mpdu(struct iwl_mld *mld, struct napi_struct *napi,
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_mld_rx_phy_data phy_data = {};
 	struct iwl_rx_mpdu_desc *mpdu_desc = (void *)pkt->data;
-	struct ieee80211_sta *sta;
+	struct ieee80211_link_sta *link_sta;
 	struct ieee80211_hdr *hdr;
 	struct sk_buff *skb;
 	size_t mpdu_desc_size = sizeof(*mpdu_desc);
@@ -2086,7 +2081,8 @@ void iwl_mld_rx_mpdu(struct iwl_mld *mld, struct napi_struct *napi,
 
 	rcu_read_lock();
 
-	sta = iwl_mld_rx_with_sta(mld, hdr, skb, mpdu_desc, pkt, queue, &drop);
+	link_sta = iwl_mld_rx_with_sta(mld, hdr, skb, mpdu_desc, pkt, queue,
+				       &drop);
 	if (drop)
 		goto drop;
 
@@ -2127,7 +2123,7 @@ void iwl_mld_rx_mpdu(struct iwl_mld *mld, struct napi_struct *napi,
 
 	iwl_mld_rx_fill_status(mld, link_id, hdr, skb, &phy_data);
 
-	if (iwl_mld_rx_crypto(mld, sta, hdr, rx_status, mpdu_desc, queue,
+	if (iwl_mld_rx_crypto(mld, link_sta, hdr, rx_status, mpdu_desc, queue,
 			      le32_to_cpu(pkt->len_n_flags), &crypto_len))
 		goto drop;
 
@@ -2140,7 +2136,8 @@ void iwl_mld_rx_mpdu(struct iwl_mld *mld, struct napi_struct *napi,
 	if (iwl_mld_time_sync_frame(mld, skb, hdr->addr2))
 		goto out;
 
-	reorder_res = iwl_mld_reorder(mld, napi, queue, sta, skb, mpdu_desc);
+	reorder_res = iwl_mld_reorder(mld, napi, queue, link_sta, skb,
+				      mpdu_desc);
 	switch (reorder_res) {
 	case IWL_MLD_PASS_SKB:
 		break;
@@ -2153,7 +2150,7 @@ void iwl_mld_rx_mpdu(struct iwl_mld *mld, struct napi_struct *napi,
 		goto drop;
 	}
 
-	iwl_mld_pass_packet_to_mac80211(mld, napi, skb, queue, sta);
+	iwl_mld_pass_packet_to_mac80211(mld, napi, skb, queue, link_sta);
 
 	goto out;
 
@@ -2285,7 +2282,7 @@ void iwl_mld_handle_rsc_notif(struct iwl_mld *mld,
 #endif /* CONFIG_PM_SLEEP */
 
 static void iwl_mld_no_data_rx(struct iwl_mld *mld,
-			       struct napi_struct *napi,
+			       struct napi_struct *napi, u8 status,
 			       struct iwl_rx_phy_air_sniffer_ntfy *ntfy)
 {
 	struct ieee80211_rx_status *rx_status;
@@ -2310,7 +2307,7 @@ static void iwl_mld_no_data_rx(struct iwl_mld *mld,
 	/* 0-length PSDU */
 	rx_status->flag |= RX_FLAG_NO_PSDU;
 
-	switch (ntfy->status) {
+	switch (status) {
 	case IWL_SNIF_STAT_PLCP_RX_OK:
 		/* we only get here with sounding PPDUs */
 		rx_status->zero_length_psdu_type =
@@ -2369,11 +2366,9 @@ void iwl_mld_handle_phy_air_sniffer_notif(struct iwl_mld *mld,
 		return;
 
 	/* check if there's an old one to release as errored */
-	if (mld->monitor.phy.valid && !mld->monitor.phy.used) {
-		/* didn't capture data, so override status */
-		mld->monitor.phy.data.status = IWL_SNIF_STAT_AID_NOT_FOR_US;
-		iwl_mld_no_data_rx(mld, napi, &mld->monitor.phy.data);
-	}
+	if (mld->monitor.phy.valid && !mld->monitor.phy.used)
+		iwl_mld_no_data_rx(mld, napi, IWL_SNIF_STAT_AID_NOT_FOR_US,
+				   &mld->monitor.phy.data);
 
 	/* old data is no longer valid now */
 	mld->monitor.phy.valid = false;
@@ -2410,7 +2405,7 @@ void iwl_mld_handle_phy_air_sniffer_notif(struct iwl_mld *mld,
 	}
 
 	if (ntfy->status != IWL_SNIF_STAT_PLCP_RX_OK || is_ndp) {
-		iwl_mld_no_data_rx(mld, napi, ntfy);
+		iwl_mld_no_data_rx(mld, napi, ntfy->status, ntfy);
 		return;
 	}
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/rx.h b/drivers/net/wireless/intel/iwlwifi/mld/rx.h
index 573b89c3c9c610..b08c5ade3db8fc 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/rx.h
@@ -69,7 +69,7 @@ void iwl_mld_handle_rsc_notif(struct iwl_mld *mld,
 void iwl_mld_pass_packet_to_mac80211(struct iwl_mld *mld,
 				     struct napi_struct *napi,
 				     struct sk_buff *skb, int queue,
-				     struct ieee80211_sta *sta);
+				     struct ieee80211_link_sta *link_sta);
 
 void iwl_mld_handle_phy_air_sniffer_notif(struct iwl_mld *mld,
 					  struct napi_struct *napi,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/scan.c b/drivers/net/wireless/intel/iwlwifi/mld/scan.c
index 0faca402bc140e..e44b24904c19ab 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/scan.c
@@ -1992,7 +1992,14 @@ void iwl_mld_int_mlo_scan(struct iwl_mld *mld, struct ieee80211_vif *vif)
 void iwl_mld_handle_scan_iter_complete_notif(struct iwl_mld *mld,
 					     struct iwl_rx_packet *pkt)
 {
-	struct iwl_umac_scan_iter_complete_notif *notif = (void *)pkt->data;
+	/*
+	 * The v3 notification is a superset of v2, so the fields that are
+	 * common to both versions can always be read from the v3 layout.
+	 */
+	const struct iwl_umac_scan_iter_complete_notif *notif =
+		(const void *)pkt->data;
+	u8 notif_ver = iwl_fw_lookup_notif_ver(mld->fw, LEGACY_GROUP,
+					       SCAN_ITERATION_COMPLETE_UMAC, 2);
 	u32 uid = __le32_to_cpu(notif->uid);
 
 	if (IWL_FW_CHECK(mld, uid >= ARRAY_SIZE(mld->scan.uid_status),
@@ -2006,6 +2013,12 @@ void iwl_mld_handle_scan_iter_complete_notif(struct iwl_mld *mld,
 		       "UMAC Scan iteration complete: status=0x%x scanned_channels=%d\n",
 		       notif->status, notif->scanned_channels);
 
+	if (notif_ver >= 3)
+		IWL_DEBUG_SCAN(mld,
+			       "UMAC Scan iteration complete: EBS=%s, EBS channel status=0x%llx\n",
+			       iwl_mld_scan_ebs_status_str(notif->ebs_status),
+			       le64_to_cpu(notif->ebs_channel_status));
+
 	if (mld->scan.pass_all_sched_res == SCHED_SCAN_PASS_ALL_STATE_FOUND) {
 		IWL_DEBUG_SCAN(mld, "Pass all scheduled scan results found\n");
 		ieee80211_sched_scan_results(mld->hw);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/stats.c b/drivers/net/wireless/intel/iwlwifi/mld/stats.c
index e7b283cbe19913..3b6cd4f382f6ba 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/stats.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/stats.c
@@ -43,12 +43,17 @@ iwl_mld_fill_stats_from_oper_notif(struct iwl_mld *mld,
 	const struct iwl_stats_ntfy_per_sta *per_sta;
 	struct ieee80211_link_sta *link_sta;
 	struct iwl_mld_link_sta *mld_link_sta;
+	u8 notif_ver = iwl_fw_lookup_notif_ver(mld->fw, STATISTICS_GROUP,
+					       STATISTICS_OPER_NOTIF, 3);
 
-	if (iwl_fw_lookup_notif_ver(mld->fw, STATISTICS_GROUP,
-				    STATISTICS_OPER_NOTIF, 3) >= 4) {
+	if (notif_ver >= 5) {
 		const struct iwl_system_statistics_notif_oper *notif =
 			(void *)&pkt->data;
 		per_sta = &notif->per_sta[fw_sta_id];
+	} else if (notif_ver == 4) {
+		const struct iwl_system_statistics_notif_oper_v4 *notif =
+			(void *)&pkt->data;
+		per_sta = &notif->per_sta[fw_sta_id];
 	} else {
 		const struct iwl_system_statistics_notif_oper_v3 *notif =
 			(void *)&pkt->data;
@@ -581,17 +586,39 @@ void iwl_mld_handle_stats_oper_notif(struct iwl_mld *mld,
 	struct iwl_system_statistics_notif_oper *_notif __free(kfree) = NULL;
 	const struct iwl_system_statistics_notif_oper *notif =
 		(void *)&pkt->data;
+	u8 notif_ver = iwl_fw_lookup_notif_ver(mld->fw, STATISTICS_GROUP,
+					       STATISTICS_OPER_NOTIF, 3);
 
 	BUILD_BUG_ON(ARRAY_SIZE(notif->per_sta) != IWL_STATION_COUNT_MAX);
 	BUILD_BUG_ON(ARRAY_SIZE(notif->per_link) <
 		     ARRAY_SIZE(mld->fw_id_to_bss_conf));
 
-	if (iwl_fw_lookup_notif_ver(mld->fw, STATISTICS_GROUP,
-				    STATISTICS_OPER_NOTIF, 3) == 3) {
+	if (notif_ver == 3) {
 		const struct iwl_system_statistics_notif_oper_v3 *stats =
 			(void *)&pkt->data;
 		_notif = kzalloc_obj(*_notif);
 
+		if (!_notif)
+			return;
+
+		_notif->time_stamp = stats->time_stamp;
+		for (int i = 0; i < ARRAY_SIZE(_notif->per_link); i++)
+			_notif->per_link[i] = stats->per_link[i];
+
+		BUILD_BUG_ON(sizeof(_notif->per_phy[0]) <
+			     sizeof(stats->per_phy[0]));
+		for (int i = 0; i < ARRAY_SIZE(_notif->per_phy); i++)
+			memcpy(&_notif->per_phy[i], &stats->per_phy[i],
+			       sizeof(stats->per_phy[i]));
+		for (int i = 0; i < ARRAY_SIZE(_notif->per_sta); i++)
+			_notif->per_sta[i] = stats->per_sta[i];
+
+		notif = _notif;
+	} else if (notif_ver == 4) {
+		const struct iwl_system_statistics_notif_oper_v4 *stats =
+			(void *)&pkt->data;
+		_notif = kzalloc_obj(*_notif);
+
 		if (!_notif)
 			return;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tests/agg.c b/drivers/net/wireless/intel/iwlwifi/mld/tests/agg.c
index 29b0248cec3dff..e9efe2996f0712 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tests/agg.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tests/agg.c
@@ -2,7 +2,7 @@
 /*
  * KUnit tests for channel helper functions
  *
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include <kunit/test.h>
 #include <kunit/static_stub.h>
@@ -441,7 +441,7 @@ static void
 fake_iwl_mld_pass_packet_to_mac80211(struct iwl_mld *mld,
 				     struct napi_struct *napi,
 				     struct sk_buff *skb, int queue,
-				     struct ieee80211_sta *sta)
+				     struct ieee80211_link_sta *link_sta)
 {
 	__skb_queue_tail(&g_released_skbs, skb);
 	g_num_released_skbs++;
@@ -630,7 +630,8 @@ static void test_reorder_buffer(struct kunit *test)
 	mpdu_desc = setup_mpdu_desc();
 
 	rcu_read_lock();
-	reorder_res = iwl_mld_reorder(mld, NULL, QUEUE, sta, skb, mpdu_desc);
+	reorder_res = iwl_mld_reorder(mld, NULL, QUEUE, &sta->deflink, skb,
+				      mpdu_desc);
 	rcu_read_unlock();
 
 	KUNIT_ASSERT_EQ(test, reorder_res, param->expected.reorder_res);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tlc.c b/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
index b6c2fa3664cbc9..0e448b24a6a560 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
@@ -49,9 +49,60 @@ struct iwl_mld_tlc_sta_capa {
 	const struct ieee80211_sta_uhr_cap *own_uhr_cap;
 };
 
+static bool iwl_mld_eht_ldpc_support(const struct ieee80211_sta_he_cap *he_cap,
+				     const struct ieee80211_sta_eht_cap *eht_cap,
+				     bool from_ap)
+{
+	u8 nss;
+
+	/* higher bandwidth implies LDPC */
+	if (he_cap->he_cap_elem.phy_cap_info[0] &
+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G)
+		return true;
+	if (he_cap->he_cap_elem.phy_cap_info[0] &
+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G)
+		return true;
+	if (he_cap->he_cap_elem.phy_cap_info[0] &
+	    IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G)
+		return true;
+	if (eht_cap->eht_cap_elem.phy_cap_info[0] &
+	    IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ)
+		return true;
+
+	/* MCS 10-14 imply LDPC support, we only check 10-13 here */
+	if (!from_ap) {
+		const struct ieee80211_eht_mcs_nss_supp_20mhz_only *mcs_nss;
+
+		mcs_nss = &eht_cap->eht_mcs_nss_supp.only_20mhz;
+
+		if (mcs_nss->rx_tx_mcs11_max_nss ||
+		    mcs_nss->rx_tx_mcs13_max_nss)
+			return true;
+
+		nss = mcs_nss->rx_tx_mcs7_max_nss;
+	} else {
+		const struct ieee80211_eht_mcs_nss_supp_bw *mcs_nss;
+
+		mcs_nss = &eht_cap->eht_mcs_nss_supp.bw._80;
+
+		if (mcs_nss->rx_tx_mcs11_max_nss ||
+		    mcs_nss->rx_tx_mcs13_max_nss)
+			return true;
+
+		nss = mcs_nss->rx_tx_mcs9_max_nss;
+	}
+
+	/* NSS > 4 implies LDPC support */
+	return u8_get_bits(nss, IEEE80211_EHT_MCS_NSS_RX) > 4 ||
+	       u8_get_bits(nss, IEEE80211_EHT_MCS_NSS_TX) > 4;
+}
+
 static __le16
 iwl_mld_get_tlc_cmd_flags(struct iwl_mld *mld,
-			  struct iwl_mld_tlc_sta_capa *capa)
+			  struct ieee80211_vif *vif,
+			  struct iwl_mld_tlc_sta_capa *capa,
+			  enum iwl_tlc_mng_cfg_mode mode,
+			  enum iwl_tlc_mng_cfg_cw max_bw)
 {
 	const struct ieee80211_sta_ht_cap *ht_cap = capa->ht_cap;
 	const struct ieee80211_sta_vht_cap *vht_cap = capa->vht_cap;
@@ -61,64 +112,79 @@ iwl_mld_get_tlc_cmd_flags(struct iwl_mld *mld,
 	const struct ieee80211_sta_he_cap *own_he_cap = capa->own_he_cap;
 	const struct ieee80211_sta_eht_cap *own_eht_cap = capa->own_eht_cap;
 	const struct ieee80211_sta_uhr_cap *own_uhr_cap = capa->own_uhr_cap;
-	bool has_vht = vht_cap && vht_cap->vht_supported;
+	bool stbc_possible = mld->cfg->ht_params.stbc &&
+			     hweight8(iwl_mld_get_valid_tx_ant(mld)) > 1;
 	u16 flags = 0;
 
-	/* STBC flags */
-	if (mld->cfg->ht_params.stbc &&
-	    (hweight8(iwl_mld_get_valid_tx_ant(mld)) > 1)) {
-		if (he_cap && he_cap->has_he &&
-		    he_cap->he_cap_elem.phy_cap_info[2] &
-				IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ)
+	switch (mode) {
+	case IWL_TLC_MNG_MODE_NON_HT:
+		break;
+	case IWL_TLC_MNG_MODE_HT:
+		if (WARN_ON(!ht_cap || !ht_cap->ht_supported))
+			return 0;
+		if (stbc_possible &&
+		    ht_cap->cap & IEEE80211_HT_CAP_RX_STBC)
 			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
-		else if (vht_cap &&
-			 vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK)
+		if (mld->cfg->ht_params.ldpc &&
+		    ht_cap->cap & IEEE80211_HT_CAP_LDPC_CODING)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+		break;
+	case IWL_TLC_MNG_MODE_VHT:
+		if (WARN_ON(!vht_cap || !vht_cap->vht_supported))
+			return 0;
+		if (stbc_possible &&
+		    vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK)
 			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
-		else if (ht_cap && ht_cap->cap & IEEE80211_HT_CAP_RX_STBC)
+		if (mld->cfg->ht_params.ldpc &&
+		    vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+		break;
+	case IWL_TLC_MNG_MODE_HE:
+		if (WARN_ON(!he_cap || !he_cap->has_he || !own_he_cap))
+			return 0;
+		if (stbc_possible &&
+		    he_cap->he_cap_elem.phy_cap_info[2] &
+				IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ &&
+		    (max_bw <= IWL_TLC_MNG_CH_WIDTH_80MHZ ||
+		     he_cap->he_cap_elem.phy_cap_info[7] &
+				IEEE80211_HE_PHY_CAP7_STBC_RX_ABOVE_80MHZ))
 			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
+		if (he_cap->he_cap_elem.phy_cap_info[1] &
+				IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD &&
+		    own_he_cap->he_cap_elem.phy_cap_info[1] &
+				IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+		if (he_cap->he_cap_elem.phy_cap_info[3] &
+				IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_MASK &&
+		    own_he_cap->he_cap_elem.phy_cap_info[3] &
+				IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_MASK)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_HE_DCM_NSS_1_MSK;
+		break;
+	case IWL_TLC_MNG_MODE_UHR:
+		if (WARN_ON(!uhr_cap || !uhr_cap->has_uhr || !own_uhr_cap))
+			return 0;
+		if (uhr_cap->phy.cap & cpu_to_le32(IEEE80211_UHR_PHY_CAP_ELR_RX) &&
+		    own_uhr_cap->phy.cap & cpu_to_le32(IEEE80211_UHR_PHY_CAP_ELR_TX))
+			flags |= IWL_TLC_MNG_CFG_FLAGS_UHR_ELR_1_5_MBPS_MSK |
+				 IWL_TLC_MNG_CFG_FLAGS_UHR_ELR_3_MBPS_MSK;
+		/* uses most config from EHT */
+		fallthrough;
+	case IWL_TLC_MNG_MODE_EHT:
+		if (WARN_ON(!own_he_cap || !he_cap || !he_cap->has_he ||
+			    !own_eht_cap || !eht_cap || !eht_cap->has_eht))
+			return 0;
+		if (own_eht_cap->eht_cap_elem.phy_cap_info[5] &
+				IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF &&
+		    eht_cap->eht_cap_elem.phy_cap_info[5] &
+				IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF)
+			flags |= IWL_TLC_MNG_CFG_FLAGS_EHT_EXTRA_LTF_MSK;
+		/* no STBC in EHT */
+		if (iwl_mld_eht_ldpc_support(he_cap, eht_cap,
+					     vif->type == NL80211_IFTYPE_STATION))
+			flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+		break;
 	}
 
-	/* LDPC */
-	if (mld->cfg->ht_params.ldpc &&
-	    ((ht_cap && ht_cap->cap & IEEE80211_HT_CAP_LDPC_CODING) ||
-	     (has_vht && (vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC))))
-		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
-
-	if (he_cap && he_cap->has_he &&
-	    (he_cap->he_cap_elem.phy_cap_info[1] &
-			IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
-		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
-
-	if (own_he_cap &&
-	    !(own_he_cap->he_cap_elem.phy_cap_info[1] &
-			IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
-		flags &= ~IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
-
-	/* DCM */
-	if (he_cap && he_cap->has_he &&
-	    (he_cap->he_cap_elem.phy_cap_info[3] &
-	     IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_MASK &&
-	     own_he_cap &&
-	     own_he_cap->he_cap_elem.phy_cap_info[3] &
-			IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_MASK))
-		flags |= IWL_TLC_MNG_CFG_FLAGS_HE_DCM_NSS_1_MSK;
-
-	/* Extra EHT LTF */
-	if (own_eht_cap &&
-	    own_eht_cap->eht_cap_elem.phy_cap_info[5] &
-			IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF &&
-	    eht_cap && eht_cap->has_eht &&
-	    eht_cap->eht_cap_elem.phy_cap_info[5] &
-			IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF) {
-		flags |= IWL_TLC_MNG_CFG_FLAGS_EHT_EXTRA_LTF_MSK;
-	}
-
-	if (uhr_cap && uhr_cap->has_uhr && own_uhr_cap &&
-	    uhr_cap->phy.cap & cpu_to_le32(IEEE80211_UHR_PHY_CAP_ELR_RX) &&
-	    own_uhr_cap->phy.cap & cpu_to_le32(IEEE80211_UHR_PHY_CAP_ELR_TX))
-		flags |= IWL_TLC_MNG_CFG_FLAGS_UHR_ELR_1_5_MBPS_MSK |
-			 IWL_TLC_MNG_CFG_FLAGS_UHR_ELR_3_MBPS_MSK;
-
 	return cpu_to_le16(flags);
 }
 
@@ -478,7 +544,6 @@ static void iwl_mld_send_tlc_cmd(struct iwl_mld *mld,
 		.max_ch_width = mld_sta->sta_state > IEEE80211_STA_ASSOC ?
 			iwl_mld_fw_bw_from_sta_bw(capa->bandwidth) :
 			IWL_TLC_MNG_CH_WIDTH_20MHZ,
-		.flags = iwl_mld_get_tlc_cmd_flags(mld, capa),
 		.chains = iwl_mld_get_fw_chains(mld),
 		.sgi_ch_width_supp = iwl_mld_get_fw_sgi(capa),
 		.max_mpdu_len = cpu_to_le16(capa->max_amsdu_len),
@@ -493,6 +558,8 @@ static void iwl_mld_send_tlc_cmd(struct iwl_mld *mld,
 	cmd.phy_id = cpu_to_le32(phy_id);
 
 	iwl_mld_fill_supp_rates(mld, vif, capa, &cmd);
+	cmd.flags = iwl_mld_get_tlc_cmd_flags(mld, vif, capa, cmd.mode,
+					      cmd.max_ch_width);
 
 	if (cmd_ver == 6) {
 		cmd_ptr = &cmd;
@@ -824,8 +891,13 @@ iwl_mld_get_amsdu_size_of_tid(struct iwl_mld *mld,
 
 	txf = iwl_mld_mac80211_ac_to_fw_tx_fifo(ac);
 
-	/* Only one link: take the lmac according to the band */
-	if (hweight16(sta->valid_links) <= 1) {
+	/*
+	 * Only one link: take the lmac according to the band
+	 * In NAN, we don't have a link_conf.
+	 * TODO: once we have a TLC object, we can know better the band.
+	 */
+	if (hweight16(sta->valid_links) <= 1 &&
+	    vif->type != NL80211_IFTYPE_NAN_DATA) {
 		enum nl80211_band band;
 		struct ieee80211_bss_conf *link =
 			wiphy_dereference(mld->wiphy,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
index 6b11fa32ea5c6b..2abc4bc3eb3497 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
@@ -1887,7 +1887,8 @@ static bool iwl_mvm_gtk_rekey(struct iwl_wowlan_status_data *status,
 
 		key = ieee80211_gtk_rekey_add(vif, status->gtk[i].id,
 					      status->gtk[i].key,
-					      sizeof(status->gtk[i].key), -1);
+					      sizeof(status->gtk[i].key), -1,
+					      false);
 		if (IS_ERR(key)) {
 			/* FW may send also the old keys */
 			if (PTR_ERR(key) == -EALREADY)
@@ -1922,7 +1923,8 @@ iwl_mvm_d3_igtk_bigtk_rekey_add(struct iwl_wowlan_status_data *status,
 		return true;
 
 	key_config = ieee80211_gtk_rekey_add(vif, keyidx, key_data->key,
-					     sizeof(key_data->key), -1);
+					     sizeof(key_data->key), -1,
+					     false);
 	if (IS_ERR(key_config)) {
 		/* FW may send also the old keys */
 		return PTR_ERR(key_config) == -EALREADY;
@@ -2438,6 +2440,12 @@ struct iwl_mvm_nd_results {
 	u8 matches[ND_QUERY_BUF_LEN];
 };
 
+static bool iwl_mvm_nd_match_info_v3(struct iwl_mvm *mvm)
+{
+	return iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
+				       SCAN_OFFLOAD_PROFILES_QUERY_CMD, 0) >= 4;
+}
+
 static int
 iwl_mvm_netdetect_query_results(struct iwl_mvm *mvm,
 				struct iwl_mvm_nd_results *results)
@@ -2457,12 +2465,17 @@ iwl_mvm_netdetect_query_results(struct iwl_mvm *mvm,
 		return ret;
 	}
 
-	if (fw_has_api(&mvm->fw->ucode_capa,
-		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
+	if (iwl_mvm_nd_match_info_v3(mvm)) {
 		matches_len = sizeof(struct iwl_scan_offload_profile_match) *
 			max_profiles;
 		query_len = offsetof(struct iwl_scan_offload_match_info,
 				     matches) + matches_len;
+	} else if (fw_has_api(&mvm->fw->ucode_capa,
+			      IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
+		matches_len = sizeof(struct iwl_scan_offload_profile_match_v2) *
+			max_profiles;
+		query_len = offsetof(struct iwl_scan_offload_match_info,
+				     matches) + matches_len;
 	} else {
 		matches_len = sizeof(struct iwl_scan_offload_profile_match_v1) *
 			max_profiles;
@@ -2497,13 +2510,19 @@ static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm,
 {
 	int n_chans = 0, i;
 
-	if (fw_has_api(&mvm->fw->ucode_capa,
-		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
+	if (iwl_mvm_nd_match_info_v3(mvm)) {
 		struct iwl_scan_offload_profile_match *matches =
 			(void *)results->matches;
 
 		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN; i++)
 			n_chans += hweight8(matches[idx].matching_channels[i]);
+	} else if (fw_has_api(&mvm->fw->ucode_capa,
+			      IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
+		struct iwl_scan_offload_profile_match_v2 *matches =
+			(void *)results->matches;
+
+		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V2; i++)
+			n_chans += hweight8(matches[idx].matching_channels[i]);
 	} else {
 		struct iwl_scan_offload_profile_match_v1 *matches =
 			(void *)results->matches;
@@ -2515,34 +2534,58 @@ static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm,
 	return n_chans;
 }
 
+static void iwl_mvm_set_matching_freqs(struct iwl_mvm *mvm,
+				       const u8 *matching_channels,
+				       size_t num_bytes,
+				       struct cfg80211_wowlan_nd_match *match)
+{
+	int n_channels = 0;
+
+	for (int i = 0; i < num_bytes * 8; i++) {
+		if (!(matching_channels[i / 8] & BIT(i % 8)))
+			continue;
+		if (IWL_FW_CHECK(mvm, i >= mvm->n_nd_channels,
+				 "FW matched channel bit %d beyond n_nd_channels %d\n",
+				 i, mvm->n_nd_channels))
+			break;
+		match->channels[n_channels++] =
+			mvm->nd_channels[i]->center_freq;
+	}
+	/* We may have ended up with fewer channels than we allocated. */
+	match->n_channels = n_channels;
+}
+
 static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
 				    struct iwl_mvm_nd_results *results,
 				    struct cfg80211_wowlan_nd_match *match,
 				    int idx)
 {
-	int i;
-	int n_channels = 0;
-
-	if (fw_has_api(&mvm->fw->ucode_capa,
-		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
+	if (iwl_mvm_nd_match_info_v3(mvm)) {
 		struct iwl_scan_offload_profile_match *matches =
 			 (void *)results->matches;
 
-		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN * 8; i++)
-			if (matches[idx].matching_channels[i / 8] & (BIT(i % 8)))
-				match->channels[n_channels++] =
-					mvm->nd_channels[i]->center_freq;
+		iwl_mvm_set_matching_freqs(mvm,
+					   matches[idx].matching_channels,
+					   sizeof(matches[idx].matching_channels),
+					   match);
+	} else if (fw_has_api(&mvm->fw->ucode_capa,
+			      IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
+		struct iwl_scan_offload_profile_match_v2 *matches =
+			 (void *)results->matches;
+
+		iwl_mvm_set_matching_freqs(mvm,
+					   matches[idx].matching_channels,
+					   sizeof(matches[idx].matching_channels),
+					   match);
 	} else {
 		struct iwl_scan_offload_profile_match_v1 *matches =
 			 (void *)results->matches;
 
-		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V1 * 8; i++)
-			if (matches[idx].matching_channels[i / 8] & (BIT(i % 8)))
-				match->channels[n_channels++] =
-					mvm->nd_channels[i]->center_freq;
+		iwl_mvm_set_matching_freqs(mvm,
+					   matches[idx].matching_channels,
+					   sizeof(matches[idx].matching_channels),
+					   match);
 	}
-	/* We may have ended up with fewer channels than we allocated. */
-	match->n_channels = n_channels;
 }
 
 /**
@@ -2815,8 +2858,14 @@ static void iwl_mvm_nd_match_info_handler(struct iwl_mvm *mvm,
 	struct iwl_wowlan_status_data *status = d3_data->status;
 	struct ieee80211_vif *vif = iwl_mvm_get_bss_vif(mvm);
 	struct iwl_mvm_nd_results *results = d3_data->nd_results;
-	size_t i, matches_len = sizeof(struct iwl_scan_offload_profile_match) *
-		iwl_umac_scan_get_max_profiles(mvm->fw);
+	size_t i, matches_len;
+
+	if (iwl_mvm_nd_match_info_v3(mvm))
+		matches_len = sizeof(struct iwl_scan_offload_profile_match) *
+			iwl_umac_scan_get_max_profiles(mvm->fw);
+	else
+		matches_len = sizeof(struct iwl_scan_offload_profile_match_v2) *
+			iwl_umac_scan_get_max_profiles(mvm->fw);
 
 	if (IS_ERR_OR_NULL(vif))
 		return;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index e6b9896dc4ac3c..06502f57e823e3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -153,72 +153,6 @@ static ssize_t iwl_dbgfs_tx_flush_write(struct iwl_mvm *mvm, char *buf,
 	return ret;
 }
 
-static ssize_t iwl_dbgfs_sram_read(struct file *file, char __user *user_buf,
-				   size_t count, loff_t *ppos)
-{
-	struct iwl_mvm *mvm = file->private_data;
-	const struct fw_img *img;
-	unsigned int ofs, len;
-	size_t ret;
-	u8 *ptr;
-
-	if (!iwl_mvm_firmware_running(mvm))
-		return -EINVAL;
-
-	/* default is to dump the entire data segment */
-	img = &mvm->fw->img[mvm->fwrt.cur_fw_img];
-	ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
-	len = img->sec[IWL_UCODE_SECTION_DATA].len;
-
-	if (mvm->dbgfs_sram_len) {
-		ofs = mvm->dbgfs_sram_offset;
-		len = mvm->dbgfs_sram_len;
-	}
-
-	ptr = kzalloc(len, GFP_KERNEL);
-	if (!ptr)
-		return -ENOMEM;
-
-	iwl_trans_read_mem_bytes(mvm->trans, ofs, ptr, len);
-
-	ret = simple_read_from_buffer(user_buf, count, ppos, ptr, len);
-
-	kfree(ptr);
-
-	return ret;
-}
-
-static ssize_t iwl_dbgfs_sram_write(struct iwl_mvm *mvm, char *buf,
-				    size_t count, loff_t *ppos)
-{
-	const struct fw_img *img;
-	u32 offset, len;
-	u32 img_offset, img_len;
-
-	if (!iwl_mvm_firmware_running(mvm))
-		return -EINVAL;
-
-	img = &mvm->fw->img[mvm->fwrt.cur_fw_img];
-	img_offset = img->sec[IWL_UCODE_SECTION_DATA].offset;
-	img_len = img->sec[IWL_UCODE_SECTION_DATA].len;
-
-	if (sscanf(buf, "%x,%x", &offset, &len) == 2) {
-		if ((offset & 0x3) || (len & 0x3))
-			return -EINVAL;
-
-		if (offset + len > img_offset + img_len)
-			return -EINVAL;
-
-		mvm->dbgfs_sram_offset = offset;
-		mvm->dbgfs_sram_len = len;
-	} else {
-		mvm->dbgfs_sram_offset = 0;
-		mvm->dbgfs_sram_len = 0;
-	}
-
-	return count;
-}
-
 static ssize_t iwl_dbgfs_set_nic_temperature_read(struct file *file,
 						  char __user *user_buf,
 						  size_t count, loff_t *ppos)
@@ -591,16 +525,24 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 	if (ret < 0)
 		return ret;
 
+	if (iwl_rx_packet_payload_len(hcmd.resp_pkt) != sizeof(*rsp)) {
+		iwl_free_resp(&hcmd);
+		return -EIO;
+	}
+
 	buff = kzalloc(bufsz, GFP_KERNEL);
-	if (!buff)
+	if (!buff) {
+		iwl_free_resp(&hcmd);
 		return -ENOMEM;
+	}
 	pos = buff;
 	endpos = pos + bufsz;
 
 	rsp = (void *)hcmd.resp_pkt->data;
 
 	pos += scnprintf(pos, endpos - pos, "TAS Conclusion:\n");
-	for (i = 0; i < rsp->in_dual_radio + 1; i++) {
+	for (i = 0; i < min_t(int, rsp->in_dual_radio + 1,
+			      ARRAY_SIZE(rsp->tas_status_mac)); i++) {
 		if (rsp->tas_status_mac[i].dynamic_status &
 		    BIT(TAS_DYNA_ACTIVE)) {
 			pos += scnprintf(pos, endpos - pos, "\tON for ");
@@ -655,7 +597,8 @@ static ssize_t iwl_dbgfs_tas_get_status_read(struct file *file,
 			 "\tDo TAS Support Dual Radio?: %s\n",
 			 rsp->in_dual_radio ? "TRUE" : "FALSE");
 
-	for (i = 0; i < rsp->in_dual_radio + 1; i++) {
+	for (i = 0; i < min_t(int, rsp->in_dual_radio + 1,
+			      ARRAY_SIZE(rsp->tas_status_mac)); i++) {
 		if (rsp->tas_status_mac[i].static_status == 0) {
 			pos += scnprintf(pos, endpos - pos,
 					 "Static status: disabled\n");
@@ -1159,7 +1102,7 @@ static ssize_t iwl_dbgfs_fw_nmi_write(struct iwl_mvm *mvm, char *buf,
 	if (count == 6 && !strcmp(buf, "nolog\n"))
 		set_bit(IWL_MVM_STATUS_SUPPRESS_ERROR_LOG_ONCE, &mvm->status);
 
-	iwl_force_nmi(mvm->trans);
+	iwl_trans_force_nmi(mvm->trans);
 
 	return count;
 }
@@ -1211,53 +1154,6 @@ iwl_dbgfs_scan_ant_rxchain_write(struct iwl_mvm *mvm, char *buf,
 	return count;
 }
 
-static ssize_t iwl_dbgfs_indirection_tbl_write(struct iwl_mvm *mvm,
-					       char *buf, size_t count,
-					       loff_t *ppos)
-{
-	struct iwl_rss_config_cmd cmd = {
-		.flags = cpu_to_le32(IWL_RSS_ENABLE),
-		.hash_mask = IWL_RSS_HASH_TYPE_IPV4_TCP |
-			     IWL_RSS_HASH_TYPE_IPV4_UDP |
-			     IWL_RSS_HASH_TYPE_IPV4_PAYLOAD |
-			     IWL_RSS_HASH_TYPE_IPV6_TCP |
-			     IWL_RSS_HASH_TYPE_IPV6_UDP |
-			     IWL_RSS_HASH_TYPE_IPV6_PAYLOAD,
-	};
-	int ret, i, num_repeats, nbytes = count / 2;
-
-	ret = hex2bin(cmd.indirection_table, buf, nbytes);
-	if (ret)
-		return ret;
-
-	/*
-	 * The input is the redirection table, partial or full.
-	 * Repeat the pattern if needed.
-	 * For example, input of 01020F will be repeated 42 times,
-	 * indirecting RSS hash results to queues 1, 2, 15 (skipping
-	 * queues 3 - 14).
-	 */
-	num_repeats = ARRAY_SIZE(cmd.indirection_table) / nbytes;
-	for (i = 1; i < num_repeats; i++)
-		memcpy(&cmd.indirection_table[i * nbytes],
-		       cmd.indirection_table, nbytes);
-	/* handle cut in the middle pattern for the last places */
-	memcpy(&cmd.indirection_table[i * nbytes], cmd.indirection_table,
-	       ARRAY_SIZE(cmd.indirection_table) % nbytes);
-
-	netdev_rss_key_fill(cmd.secret_key, sizeof(cmd.secret_key));
-
-	mutex_lock(&mvm->mutex);
-	if (iwl_mvm_firmware_running(mvm))
-		ret = iwl_mvm_send_cmd_pdu(mvm, RSS_CONFIG_CMD, 0,
-					   sizeof(cmd), &cmd);
-	else
-		ret = 0;
-	mutex_unlock(&mvm->mutex);
-
-	return ret ?: count;
-}
-
 static ssize_t iwl_dbgfs_inject_packet_write(struct iwl_mvm *mvm,
 					     char *buf, size_t count,
 					     loff_t *ppos)
@@ -1916,7 +1812,6 @@ MVM_DEBUGFS_WRITE_FILE_OPS(start_ctdp, 8);
 MVM_DEBUGFS_WRITE_FILE_OPS(force_ctkill, 8);
 MVM_DEBUGFS_WRITE_FILE_OPS(tx_flush, 16);
 MVM_DEBUGFS_WRITE_FILE_OPS(send_echo_cmd, 8);
-MVM_DEBUGFS_READ_WRITE_FILE_OPS(sram, 64);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(set_nic_temperature, 64);
 MVM_DEBUGFS_READ_FILE_OPS(nic_temp);
 MVM_DEBUGFS_READ_FILE_OPS(stations);
@@ -1933,8 +1828,6 @@ MVM_DEBUGFS_READ_WRITE_FILE_OPS(scan_ant_rxchain, 8);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(fw_dbg_conf, 8);
 MVM_DEBUGFS_WRITE_FILE_OPS(fw_dbg_clear, 64);
 MVM_DEBUGFS_WRITE_FILE_OPS(dbg_time_point, 64);
-MVM_DEBUGFS_WRITE_FILE_OPS(indirection_tbl,
-			   (IWL_RSS_INDIRECTION_TABLE_SIZE * 2));
 MVM_DEBUGFS_WRITE_FILE_OPS(inject_packet, 512);
 MVM_DEBUGFS_WRITE_FILE_OPS(inject_beacon_ie, 512);
 MVM_DEBUGFS_WRITE_FILE_OPS(inject_beacon_ie_restore, 512);
@@ -2114,7 +2007,6 @@ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm)
 	spin_lock_init(&mvm->drv_stats_lock);
 
 	MVM_DEBUGFS_ADD_FILE(tx_flush, mvm->debugfs_dir, 0200);
-	MVM_DEBUGFS_ADD_FILE(sram, mvm->debugfs_dir, 0600);
 	MVM_DEBUGFS_ADD_FILE(set_nic_temperature, mvm->debugfs_dir, 0600);
 	MVM_DEBUGFS_ADD_FILE(nic_temp, mvm->debugfs_dir, 0400);
 	MVM_DEBUGFS_ADD_FILE(ctdp_budget, mvm->debugfs_dir, 0400);
@@ -2134,7 +2026,6 @@ void iwl_mvm_dbgfs_register(struct iwl_mvm *mvm)
 	MVM_DEBUGFS_ADD_FILE(fw_dbg_clear, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(dbg_time_point, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(send_echo_cmd, mvm->debugfs_dir, 0200);
-	MVM_DEBUGFS_ADD_FILE(indirection_tbl, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(inject_packet, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(inject_beacon_ie, mvm->debugfs_dir, 0200);
 	MVM_DEBUGFS_ADD_FILE(inject_beacon_ie_restore, mvm->debugfs_dir, 0200);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index 43a7230f1ab406..c3402f5e47a031 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -911,7 +911,8 @@ int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b)
 
 	ret = iwl_sar_fill_profile(&mvm->fwrt, per_chain,
 				   IWL_NUM_CHAIN_TABLES,
-				   n_subbands, prof_a, prof_b);
+				   n_subbands, prof_a, prof_b,
+				   mvm->fwrt.sar_profiles, false);
 
 	/* return on error or if the profile is disabled (positive number) */
 	if (ret)
@@ -1526,7 +1527,7 @@ static void iwl_mvm_send_lari_cfg_extension(struct iwl_mvm *mvm)
 	u32 value;
 	int ret;
 
-	if (iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 0) < 1)
+	if (iwl_fw_lookup_cmd_ver(mvm->fw, cmd_id, 0) != 1)
 		return;
 
 	ret = iwl_bios_get_dsm(fwrt, DSM_FUNC_REGULATORY_CONFIG, &value);
@@ -1576,7 +1577,7 @@ void iwl_mvm_get_bios_tables(struct iwl_mvm *mvm)
 {
 	int ret;
 
-	iwl_acpi_get_guid_lock_status(&mvm->fwrt);
+	iwl_bios_get_guid_lock_status(&mvm->fwrt);
 
 	/* read PPAG table */
 	ret = iwl_bios_get_ppag_table(&mvm->fwrt);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index 5bd246e3794327..a4f3d3c2bd350a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -5258,7 +5258,7 @@ iwl_mvm_switch_vif_chanctx_swap(struct iwl_mvm *mvm,
 
 out_restart:
 	/* things keep failing, better restart the hw */
-	iwl_force_nmi(mvm->trans);
+	iwl_trans_force_nmi(mvm->trans);
 	return ret;
 }
 
@@ -5294,7 +5294,7 @@ iwl_mvm_switch_vif_chanctx_reassign(struct iwl_mvm *mvm,
 
 out_restart:
 	/* things keep failing, better restart the hw */
-	iwl_force_nmi(mvm->trans);
+	iwl_trans_force_nmi(mvm->trans);
 	return ret;
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index c64e4ed00986a8..ae2ee8771f1679 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -42,7 +42,7 @@
 #define IWL_MVM_MAX_ADDRESSES		5
 /* RSSI offset for WkP */
 #define IWL_RSSI_OFFSET 50
-#define IWL_MVM_MISSED_BEACONS_SINCE_RX_THOLD 4
+#define IWL_MVM_MISSED_BEACONS_SINCE_RX_THOLD 6
 #define IWL_MVM_MISSED_BEACONS_THRESHOLD 8
 #define IWL_MVM_MISSED_BEACONS_THRESHOLD_LONG 19
 
@@ -1053,7 +1053,6 @@ struct iwl_mvm {
 	 */
 	struct dentry *debugfs_dir;
 #ifdef CONFIG_IWLWIFI_DEBUGFS
-	u32 dbgfs_sram_offset, dbgfs_sram_len;
 	u32 dbgfs_prph_reg_addr;
 	bool disable_power_off;
 	bool disable_power_off_d3;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
index f76e57399c1f33..54006d5e15a5f3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/nvm.c
@@ -86,6 +86,7 @@ static int iwl_nvm_read_chunk(struct iwl_mvm *mvm, u16 section,
 		.data = { &nvm_access_cmd, },
 	};
 	int ret, bytes_read, offset_read;
+	unsigned int pkt_len;
 	u8 *resp_data;
 
 	cmd.len[0] = sizeof(struct iwl_nvm_access_cmd);
@@ -95,6 +96,12 @@ static int iwl_nvm_read_chunk(struct iwl_mvm *mvm, u16 section,
 		return ret;
 
 	pkt = cmd.resp_pkt;
+	pkt_len = iwl_rx_packet_payload_len(pkt);
+	if (pkt_len < sizeof(*nvm_resp)) {
+		IWL_ERR(mvm, "NVM ACCESS response too short: %u\n", pkt_len);
+		ret = -EINVAL;
+		goto exit;
+	}
 
 	/* Extract NVM response */
 	nvm_resp = (void *)pkt->data;
@@ -133,6 +140,14 @@ static int iwl_nvm_read_chunk(struct iwl_mvm *mvm, u16 section,
 		goto exit;
 	}
 
+	if (bytes_read > length ||
+	    pkt_len < struct_size(nvm_resp, data, bytes_read)) {
+		IWL_ERR(mvm, "NVM ACCESS response with invalid length %d\n",
+			bytes_read);
+		ret = -EINVAL;
+		goto exit;
+	}
+
 	/* Write data to NVM */
 	memcpy(data + offset, resp_data, bytes_read);
 	ret = bytes_read;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index 1dd27292af9b81..93d5cab87f6452 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -374,7 +374,7 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 		   struct iwl_umac_scan_complete),
 	RX_HANDLER(SCAN_ITERATION_COMPLETE_UMAC,
 		   iwl_mvm_rx_umac_scan_iter_complete_notif, RX_HANDLER_SYNC,
-		   struct iwl_umac_scan_iter_complete_notif),
+		   struct iwl_umac_scan_iter_complete_notif_v2),
 
 	RX_HANDLER(MISSED_BEACONS_NOTIFICATION,
 		   iwl_mvm_rx_missed_beacons_notif_legacy,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
index ab1eb2eb0c3cdf..8c197a090c2b1a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
@@ -90,7 +90,7 @@ static void iwl_mvm_pass_packet_to_mac80211(struct iwl_mvm *mvm,
 				fraglen, rxb->truesize);
 	}
 
-	ieee80211_rx_napi(mvm->hw, sta, skb, napi);
+	ieee80211_rx_napi(mvm->hw, sta ? &sta->deflink : NULL, skb, napi);
 }
 
 /*
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
index 7f0b4f5daa2143..c01c704929bad4 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -99,10 +99,14 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 	u8 mic_crc_len = u8_get_bits(desc->mac_flags1,
 				     IWL_RX_MPDU_MFLG1_MIC_CRC_LEN_MASK) << 1;
 
-	if (desc->mac_flags2 & IWL_RX_MPDU_MFLG2_PAD) {
-		len -= 2;
-		pad_len = 2;
-	}
+	pad_len = desc->mac_flags2 & IWL_RX_MPDU_MFLG2_PAD ? 2 : 0;
+
+	if (IWL_FW_CHECK(mvm, len < hdrlen + crypt_len + pad_len + mic_crc_len,
+			 "invalid len (%d): hdrlen=%u, crypt_len=%u, pad_len=%u, mic_crc_len=%u\n",
+			 len, hdrlen, crypt_len, pad_len, mic_crc_len))
+		return -EINVAL;
+
+	len -= pad_len;
 
 	/*
 	 * For non monitor interface strip the bytes the RADA might not have
@@ -110,7 +114,7 @@ static int iwl_mvm_create_skb(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 * interface cannot exist with other interfaces, this removal is safe
 	 * and sufficient, in monitor mode there's no decryption being done.
 	 */
-	if (len > mic_crc_len && !ieee80211_hw_check(mvm->hw, RX_INCLUDES_FCS))
+	if (!ieee80211_hw_check(mvm->hw, RX_INCLUDES_FCS))
 		len -= mic_crc_len;
 
 	/* If frame is small enough to fit in skb->head, pull it completely.
@@ -243,7 +247,7 @@ static void iwl_mvm_pass_packet_to_mac80211(struct iwl_mvm *mvm,
 		return;
 	}
 
-	ieee80211_rx_napi(mvm->hw, sta, skb, napi);
+	ieee80211_rx_napi(mvm->hw, sta ? &sta->deflink : NULL, skb, napi);
 }
 
 static bool iwl_mvm_used_average_energy(struct iwl_mvm *mvm,
@@ -2164,7 +2168,17 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	phy_data.phy_info = le16_to_cpu(desc->phy_info);
 	phy_data.d4 = desc->phy_data4;
 
+	if (IWL_FW_CHECK(mvm, len < sizeof(hdr->frame_control),
+			 "MPDU len (%u) is smaller than frame control\n",
+			 len))
+		return;
+
 	hdr = (void *)(pkt->data + desc_size);
+	if (IWL_FW_CHECK(mvm, len < ieee80211_hdrlen(hdr->frame_control),
+			 "MPDU len (%u) is smaller than header length (%u)\n",
+			 len, ieee80211_hdrlen(hdr->frame_control)))
+		return;
+
 	/* Dont use dev_alloc_skb(), we'll have enough headroom once
 	 * ieee80211_hdr pulled.
 	 */
@@ -2285,7 +2299,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		struct iwl_fw_dbg_trigger_tlv *trig;
 		struct ieee80211_vif *vif = mvmsta->vif;
 
-		if (!mvm->tcm.paused && len >= sizeof(*hdr) &&
+		if (!mvm->tcm.paused &&
 		    !is_multicast_ether_addr(hdr->addr1) &&
 		    ieee80211_is_data(hdr->frame_control) &&
 		    time_after(jiffies, mvm->tcm.ts + MVM_TCM_PERIOD))
@@ -2528,7 +2542,7 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	}
 
 	rcu_read_lock();
-	ieee80211_rx_napi(mvm->hw, sta, skb, napi);
+	ieee80211_rx_napi(mvm->hw, sta ? &sta->deflink : NULL, skb, napi);
 	rcu_read_unlock();
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index 3831b3c27e0f37..a94eaca0b3ed51 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -2703,7 +2703,7 @@ void iwl_mvm_scan_timeout_wk(struct work_struct *work)
 
 	IWL_ERR(mvm, "regular scan timed out\n");
 
-	iwl_force_nmi(mvm->trans);
+	iwl_trans_force_nmi(mvm->trans);
 }
 
 static void iwl_mvm_fill_scan_type(struct iwl_mvm *mvm,
@@ -3282,7 +3282,7 @@ void iwl_mvm_rx_umac_scan_iter_complete_notif(struct iwl_mvm *mvm,
 					      struct iwl_rx_cmd_buffer *rxb)
 {
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_umac_scan_iter_complete_notif *notif = (void *)pkt->data;
+	struct iwl_umac_scan_iter_complete_notif_v2 *notif = (void *)pkt->data;
 
 	mvm->scan_start = le64_to_cpu(notif->start_tsf);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
index 2d0811be734628..a66128a62546b7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2015, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2015, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -2792,7 +2792,11 @@ static int iwl_mvm_fw_baid_op_sta(struct iwl_mvm *mvm,
 		if (WARN_ON(start && iwl_mvm_has_new_rx_api(mvm) &&
 			    !(status & IWL_ADD_STA_BAID_VALID_MASK)))
 			return -EINVAL;
-		return u32_get_bits(status, IWL_ADD_STA_BAID_MASK);
+		ret = u32_get_bits(status, IWL_ADD_STA_BAID_MASK);
+		if (IWL_FW_CHECK(mvm, start && ret >= ARRAY_SIZE(mvm->baid_map),
+				 "invalid BAID %d from FW\n", ret))
+			return -EINVAL;
+		return ret;
 	case ADD_STA_IMMEDIATE_BA_FAILURE:
 		IWL_WARN(mvm, "RX BA Session refused by fw\n");
 		return -ENOSPC;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info-v2.c b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info-v2.c
index 06be929a3ca5a4..c75210829ec94e 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info-v2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info-v2.c
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include <linux/dmi.h>
 #include "iwl-trans.h"
 #include "iwl-fh.h"
 #include "iwl-context-info-v2.h"
-#include "gen1_2/internal.h"
+#include "internal.h"
 #include "iwl-prph.h"
 
 static const struct dmi_system_id dmi_force_scu_active_approved_list[] = {
@@ -112,6 +112,14 @@ int iwl_pcie_ctxt_info_v2_alloc(struct iwl_trans *trans,
 	int cmdq_size = max_t(u32, IWL_CMD_QUEUE_SIZE,
 			      trans->mac_cfg->base->min_txq_size);
 
+	if (WARN_ON_ONCE(trans_pcie->ctxt_info_v2 || trans_pcie->prph_info ||
+			 trans_pcie->prph_scratch || trans_pcie->iml ||
+			 trans_pcie->ctxt_info_dma_addr ||
+			 trans_pcie->prph_info_dma_addr ||
+			 trans_pcie->prph_scratch_dma_addr ||
+			 trans_pcie->iml_dma_addr || trans_pcie->iml_len))
+		return -EBUSY;
+
 	switch (trans->conf.rx_buf_size) {
 	case IWL_AMSDU_DEF:
 		return -EINVAL;
@@ -263,7 +271,6 @@ int iwl_pcie_ctxt_info_v2_alloc(struct iwl_trans *trans,
 	trans_pcie->prph_scratch = prph_scratch;
 
 	/* Allocate IML */
-	trans_pcie->iml_len = fw->iml_len;
 	trans_pcie->iml = dma_alloc_coherent(trans->dev, fw->iml_len,
 					     &trans_pcie->iml_dma_addr,
 					     GFP_KERNEL);
@@ -271,6 +278,7 @@ int iwl_pcie_ctxt_info_v2_alloc(struct iwl_trans *trans,
 		ret = -ENOMEM;
 		goto err_free_ctxt_info;
 	}
+	trans_pcie->iml_len = fw->iml_len;
 
 	memcpy(trans_pcie->iml, fw->iml, fw->iml_len);
 
@@ -280,16 +288,21 @@ int iwl_pcie_ctxt_info_v2_alloc(struct iwl_trans *trans,
 	dma_free_coherent(trans->dev, sizeof(*trans_pcie->ctxt_info_v2),
 			  trans_pcie->ctxt_info_v2,
 			  trans_pcie->ctxt_info_dma_addr);
+	trans_pcie->ctxt_info_dma_addr = 0;
 	trans_pcie->ctxt_info_v2 = NULL;
 err_free_prph_info:
 	dma_free_coherent(trans->dev, PAGE_SIZE, prph_info,
 			  trans_pcie->prph_info_dma_addr);
+	trans_pcie->prph_info_dma_addr = 0;
+	trans_pcie->prph_info = NULL;
 
 err_free_prph_scratch:
 	dma_free_coherent(trans->dev,
 			  sizeof(*prph_scratch),
 			prph_scratch,
 			trans_pcie->prph_scratch_dma_addr);
+	trans_pcie->prph_scratch_dma_addr = 0;
+	trans_pcie->prph_scratch = NULL;
 	return ret;
 
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
index 7f886200d69333..23ec968577bd9d 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/ctxt-info.c
@@ -6,7 +6,7 @@
 #include "iwl-trans.h"
 #include "iwl-fh.h"
 #include "iwl-context-info.h"
-#include "gen1_2/internal.h"
+#include "internal.h"
 #include "iwl-prph.h"
 
 static void *_iwl_pcie_ctxt_info_dma_alloc_coherent(struct iwl_trans *trans,
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index a3e6c9e09a3be6..905482419a9afd 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -15,7 +15,7 @@
 #include "iwl-trans.h"
 #include "iwl-drv.h"
 #include "iwl-prph.h"
-#include "gen1_2/internal.h"
+#include "internal.h"
 
 #define _IS_A(cfg, _struct) __builtin_types_compatible_p(typeof(cfg),	\
 							 struct _struct)
@@ -1018,6 +1018,7 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info iwl_dev_info_table[] = {
 /* GF RF */
 	IWL_DEV_INFO(iwl_rf_gf, iwl_ax211_name, RF_TYPE(GF)),
 	IWL_DEV_INFO(iwl_rf_ot, iwl_ax231_name, RF_TYPE(GF), SUBDEV(0x0294)),
+	IWL_DEV_INFO(iwl_rf_ot, iwl_ax231_name, RF_TYPE(GF), SUBDEV(0x4294)),
 	IWL_DEV_INFO(iwl_rf_gf, iwl_ax411_name, RF_TYPE(GF), CDB),
 	IWL_DEV_INFO(iwl_rf_gf, iwl_ax210_name, DEVICE(0x2725)),
 
@@ -1187,17 +1188,22 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		return -EIO;
 	}
 
-	return iwl_pci_gen1_2_probe(pdev, ent, mac_cfg, hw_base, hw_rev);
+	return _iwl_pci_probe(pdev, ent, mac_cfg, hw_base, hw_rev);
 }
 
 static void iwl_pci_remove(struct pci_dev *pdev)
 {
 	struct iwl_trans *trans = pci_get_drvdata(pdev);
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	if (!trans)
 		return;
 
-	iwl_pcie_gen1_2_remove(trans);
+	cancel_delayed_work_sync(&trans_pcie->me_recheck_wk);
+
+	iwl_drv_stop(trans->drv);
+
+	iwl_trans_pcie_free(trans);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
similarity index 95%
rename from drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h
rename to drivers/net/wireless/intel/iwlwifi/pcie/internal.h
index d84c7c1efee1c0..e6b1cb559a0a68 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
@@ -22,7 +22,7 @@
 #include "iwl-io.h"
 #include "iwl-op-mode.h"
 #include "iwl-drv.h"
-#include "pcie/iwl-context-info.h"
+#include "iwl-context-info.h"
 
 /*
  * RX related structures and functions
@@ -206,23 +206,6 @@ static inline u16 iwl_get_closed_rb_stts(struct iwl_trans *trans,
 	}
 }
 
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-/**
- * enum iwl_fw_mon_dbgfs_state - the different states of the monitor_data
- * debugfs file
- *
- * @IWL_FW_MON_DBGFS_STATE_CLOSED: the file is closed.
- * @IWL_FW_MON_DBGFS_STATE_OPEN: the file is open.
- * @IWL_FW_MON_DBGFS_STATE_DISABLED: the file is disabled, once this state is
- *	set the file can no longer be used.
- */
-enum iwl_fw_mon_dbgfs_state {
-	IWL_FW_MON_DBGFS_STATE_CLOSED,
-	IWL_FW_MON_DBGFS_STATE_OPEN,
-	IWL_FW_MON_DBGFS_STATE_DISABLED,
-};
-#endif
-
 /**
  * enum iwl_shared_irq_flags - level of sharing for irq
  * @IWL_SHARED_IRQ_NON_RX: interrupt vector serves non rx causes.
@@ -245,26 +228,6 @@ enum iwl_image_response_code {
 	IWL_IMAGE_RESP_FAIL		= 2,
 };
 
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-/**
- * struct cont_rec: continuous recording data structure
- * @prev_wr_ptr: the last address that was read in monitor_data
- *	debugfs file
- * @prev_wrap_cnt: the wrap count that was used during the last read in
- *	monitor_data debugfs file
- * @state: the state of monitor_data debugfs file as described
- *	in &iwl_fw_mon_dbgfs_state enum
- * @mutex: locked while reading from monitor_data debugfs file
- */
-struct cont_rec {
-	u32 prev_wr_ptr;
-	u32 prev_wrap_cnt;
-	u8  state;
-	/* Used to sync monitor_data debugfs file with driver unload flow */
-	struct mutex mutex;
-};
-#endif
-
 enum iwl_pcie_fw_reset_state {
 	FW_RESET_IDLE,
 	FW_RESET_REQUESTED,
@@ -451,7 +414,6 @@ struct iwl_pcie_txqs {
  * @rx_buf_bytes: RX buffer (RB) size in bytes
  * @reg_lock: protect hw register access
  * @mutex: to protect stop_device / start_fw / start_hw
- * @fw_mon_data: fw continuous recording data
  * @cmd_hold_nic_awake: indicates NIC is held awake for APMG workaround
  *	during commands in flight
  * @msix_entries: array of MSI-X entries
@@ -577,10 +539,6 @@ struct iwl_trans_pcie {
 	spinlock_t reg_lock;
 	bool cmd_hold_nic_awake;
 
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-	struct cont_rec fw_mon_data;
-#endif
-
 	struct msix_entry msix_entries[IWL_MAX_RX_HW_QUEUES];
 	bool msix_enabled;
 	u8 shared_vec_mask;
@@ -892,7 +850,7 @@ static inline u16 iwl_txq_gen1_tfd_tb_get_len(struct iwl_trans *trans,
 }
 
 static inline struct iwl_device_tx_cmd *
-iwl_pcie_gen1_2_alloc_tx_cmd(struct iwl_trans *trans)
+iwl_pcie_alloc_tx_cmd(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
@@ -900,8 +858,8 @@ iwl_pcie_gen1_2_alloc_tx_cmd(struct iwl_trans *trans)
 }
 
 static inline void
-iwl_pcie_gen1_2_free_tx_cmd(struct iwl_trans *trans,
-			    struct iwl_device_tx_cmd *dev_cmd)
+iwl_pcie_free_tx_cmd(struct iwl_trans *trans,
+		     struct iwl_device_tx_cmd *dev_cmd)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
@@ -1167,7 +1125,6 @@ void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state, bool from_irq);
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans);
-void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans);
 #else
 static inline void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans) { }
 #endif
@@ -1175,7 +1132,7 @@ static inline void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans) { }
 void iwl_pcie_rx_allocator_work(struct work_struct *data);
 
 /* common trans ops for all generations transports */
-void iwl_pcie_gen1_2_op_mode_enter(struct iwl_trans *trans);
+void iwl_trans_pcie_op_mode_enter(struct iwl_trans *trans);
 int _iwl_trans_pcie_start_hw(struct iwl_trans *trans);
 int iwl_trans_pcie_start_hw(struct iwl_trans *trans);
 void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans);
@@ -1204,14 +1161,11 @@ int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
 				 u32 *val);
 bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans);
 void iwl_trans_pcie_resched_with_nic_access(struct iwl_trans *trans);
-void __releases(nic_access_nobh)
-iwl_trans_pcie_release_nic_access(struct iwl_trans *trans);
+void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans);
 void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power);
-int iwl_pci_gen1_2_probe(struct pci_dev *pdev,
-			 const struct pci_device_id *ent,
-			 const struct iwl_mac_cfg *mac_cfg,
-			 u8 __iomem *hw_base, u32 hw_rev);
-void iwl_pcie_gen1_2_remove(struct iwl_trans *trans);
+int _iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent,
+		   const struct iwl_mac_cfg *mac_cfg,
+		   u8 __iomem *hw_base, u32 hw_rev);
 
 /* transport gen 1 exported functions */
 void iwl_trans_pcie_fw_alive(struct iwl_trans *trans);
@@ -1236,7 +1190,7 @@ int iwl_pcie_alloc_dma_ptr(struct iwl_trans *trans,
 			   struct iwl_dma_ptr *ptr, size_t size);
 void iwl_pcie_free_dma_ptr(struct iwl_trans *trans, struct iwl_dma_ptr *ptr);
 void iwl_pcie_apply_destination(struct iwl_trans *trans);
-int iwl_pcie_gen1_2_activate_nic(struct iwl_trans *trans);
+int iwl_pcie_activate_nic(struct iwl_trans *trans);
 
 /* transport gen 2 exported functions */
 int iwl_trans_pcie_gen2_start_fw(struct iwl_trans *trans,
@@ -1266,7 +1220,7 @@ static inline bool iwl_pcie_gen1_is_pm_supported(struct iwl_trans *trans)
 	return trans_pcie->pm_support;
 }
 
-static inline bool iwl_pcie_gen1_2_is_ltr_enabled(struct iwl_trans *trans)
+static inline bool iwl_pcie_is_ltr_enabled(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
similarity index 99%
rename from drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c
rename to drivers/net/wireless/intel/iwlwifi/pcie/rx.c
index 4631e11f2a964d..db3933b0d1979e 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
@@ -12,7 +12,7 @@
 #include "iwl-io.h"
 #include "internal.h"
 #include "iwl-op-mode.h"
-#include "pcie/iwl-context-info-v2.h"
+#include "iwl-context-info-v2.h"
 #include "fw/dbg.h"
 
 /******************************************************************************
@@ -1493,7 +1493,7 @@ static struct iwl_rx_mem_buffer *iwl_pcie_get_rxb(struct iwl_trans *trans,
 
 out_err:
 	WARN(1, "Invalid rxb from HW %u\n", (u32)vid);
-	iwl_force_nmi(trans);
+	iwl_trans_force_nmi(trans);
 	return NULL;
 }
 
@@ -1519,6 +1519,11 @@ static int iwl_pcie_rx_handle(struct iwl_trans *trans, int queue, int budget)
 	r = iwl_get_closed_rb_stts(trans, rxq);
 	i = rxq->read;
 
+	/* Order the read of the write pointer before any read of the
+	 * completion descriptors and of the RB contents it makes visible.
+	 */
+	dma_rmb();
+
 	/* W/A 9000 device step A0 wrap-around bug */
 	r &= (rxq->queue_size - 1);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
similarity index 99%
rename from drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
rename to drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
index f41cbe62e7aa57..39dd963920b31c 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
@@ -5,8 +5,8 @@
  */
 #include "iwl-trans.h"
 #include "iwl-prph.h"
-#include "pcie/iwl-context-info.h"
-#include "pcie/iwl-context-info-v2.h"
+#include "iwl-context-info.h"
+#include "iwl-context-info-v2.h"
 #include "internal.h"
 #include "fw/dbg.h"
 
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
similarity index 95%
rename from drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
rename to drivers/net/wireless/intel/iwlwifi/pcie/trans.c
index 28b276c659526a..bda069a6f7ebfd 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
@@ -28,7 +28,7 @@
 #include "mei/iwl-mei.h"
 #include "internal.h"
 #include "iwl-fh.h"
-#include "pcie/iwl-context-info-v2.h"
+#include "iwl-context-info-v2.h"
 #include "pcie/utils.h"
 
 #define IWL_HOST_MON_BLOCK_PEMON	0x00
@@ -1903,7 +1903,7 @@ void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr, u32 val)
 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
 }
 
-void iwl_pcie_gen1_2_op_mode_enter(struct iwl_trans *trans)
+void iwl_trans_pcie_op_mode_enter(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
@@ -2402,10 +2402,10 @@ bool _iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent)
 
 out:
 	/*
-	 * Fool sparse by faking we release the lock - sparse will
-	 * track nic_access anyway.
+	 * Deliberately return with reg_lock held; the caller must drop it via
+	 * iwl_trans_pcie_release_nic_access(), or explicitly if it wants to
+	 * keep the NIC awake past the critical section (cmd_hold_nic_awake).
 	 */
-	__release(&trans_pcie->reg_lock);
 	return true;
 }
 
@@ -2427,24 +2427,19 @@ void iwl_trans_pcie_resched_with_nic_access(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
+	lockdep_assert_held(&trans_pcie->reg_lock);
+
 	spin_unlock_bh(&trans_pcie->reg_lock);
 	cond_resched();
 	spin_lock_bh(&trans_pcie->reg_lock);
 }
 
-void __releases(nic_access_nobh)
-iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
+void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 
 	lockdep_assert_held(&trans_pcie->reg_lock);
 
-	/*
-	 * Fool sparse by faking we acquiring the lock - sparse will
-	 * track nic_access anyway.
-	 */
-	__acquire(&trans_pcie->reg_lock);
-
 	if (trans_pcie->cmd_hold_nic_awake)
 		goto out;
 	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
@@ -2460,7 +2455,6 @@ iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
 	 * scheduled on different CPUs (after we drop reg_lock).
 	 */
 out:
-	__release(nic_access_nobh);
 	spin_unlock_bh(&trans_pcie->reg_lock);
 }
 
@@ -3016,24 +3010,6 @@ static ssize_t iwl_dbgfs_csr_write(struct file *file,
 	return count;
 }
 
-static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
-				     char __user *user_buf,
-				     size_t count, loff_t *ppos)
-{
-	struct iwl_trans *trans = file->private_data;
-	char *buf = NULL;
-	ssize_t ret;
-
-	ret = iwl_dump_fh(trans, &buf);
-	if (ret < 0)
-		return ret;
-	if (!buf)
-		return -EINVAL;
-	ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
-	kfree(buf);
-	return ret;
-}
-
 static ssize_t iwl_dbgfs_rfkill_read(struct file *file,
 				     char __user *user_buf,
 				     size_t count, loff_t *ppos)
@@ -3073,137 +3049,6 @@ static ssize_t iwl_dbgfs_rfkill_write(struct file *file,
 	return count;
 }
 
-static int iwl_dbgfs_monitor_data_open(struct inode *inode,
-				       struct file *file)
-{
-	struct iwl_trans *trans = inode->i_private;
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-
-	if (!trans->dbg.dest_tlv ||
-	    trans->dbg.dest_tlv->monitor_mode != EXTERNAL_MODE) {
-		IWL_ERR(trans, "Debug destination is not set to DRAM\n");
-		return -ENOENT;
-	}
-
-	if (trans_pcie->fw_mon_data.state != IWL_FW_MON_DBGFS_STATE_CLOSED)
-		return -EBUSY;
-
-	trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_OPEN;
-	return simple_open(inode, file);
-}
-
-static int iwl_dbgfs_monitor_data_release(struct inode *inode,
-					  struct file *file)
-{
-	struct iwl_trans_pcie *trans_pcie =
-		IWL_TRANS_GET_PCIE_TRANS(inode->i_private);
-
-	if (trans_pcie->fw_mon_data.state == IWL_FW_MON_DBGFS_STATE_OPEN)
-		trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
-	return 0;
-}
-
-static bool iwl_write_to_user_buf(char __user *user_buf, ssize_t count,
-				  void *buf, ssize_t *size,
-				  ssize_t *bytes_copied)
-{
-	ssize_t buf_size_left = count - *bytes_copied;
-
-	buf_size_left = buf_size_left - (buf_size_left % sizeof(u32));
-	if (*size > buf_size_left)
-		*size = buf_size_left;
-
-	*size -= copy_to_user(user_buf, buf, *size);
-	*bytes_copied += *size;
-
-	if (buf_size_left == *size)
-		return true;
-	return false;
-}
-
-static ssize_t iwl_dbgfs_monitor_data_read(struct file *file,
-					   char __user *user_buf,
-					   size_t count, loff_t *ppos)
-{
-	struct iwl_trans *trans = file->private_data;
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	u8 *cpu_addr = (void *)trans->dbg.fw_mon.block, *curr_buf;
-	struct cont_rec *data = &trans_pcie->fw_mon_data;
-	u32 write_ptr_addr, wrap_cnt_addr, write_ptr, wrap_cnt;
-	ssize_t size, bytes_copied = 0;
-	bool b_full;
-
-	if (trans->dbg.dest_tlv) {
-		write_ptr_addr =
-			le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg);
-		wrap_cnt_addr = le32_to_cpu(trans->dbg.dest_tlv->wrap_count);
-	} else {
-		write_ptr_addr = MON_BUFF_WRPTR;
-		wrap_cnt_addr = MON_BUFF_CYCLE_CNT;
-	}
-
-	if (unlikely(!trans->dbg.rec_on))
-		return 0;
-
-	mutex_lock(&data->mutex);
-	if (data->state ==
-	    IWL_FW_MON_DBGFS_STATE_DISABLED) {
-		mutex_unlock(&data->mutex);
-		return 0;
-	}
-
-	/* write_ptr position in bytes rather then DW */
-	write_ptr = iwl_read_prph(trans, write_ptr_addr) * sizeof(u32);
-	wrap_cnt = iwl_read_prph(trans, wrap_cnt_addr);
-
-	if (data->prev_wrap_cnt == wrap_cnt) {
-		size = write_ptr - data->prev_wr_ptr;
-		curr_buf = cpu_addr + data->prev_wr_ptr;
-		b_full = iwl_write_to_user_buf(user_buf, count,
-					       curr_buf, &size,
-					       &bytes_copied);
-		data->prev_wr_ptr += size;
-
-	} else if (data->prev_wrap_cnt == wrap_cnt - 1 &&
-		   write_ptr < data->prev_wr_ptr) {
-		size = trans->dbg.fw_mon.size - data->prev_wr_ptr;
-		curr_buf = cpu_addr + data->prev_wr_ptr;
-		b_full = iwl_write_to_user_buf(user_buf, count,
-					       curr_buf, &size,
-					       &bytes_copied);
-		data->prev_wr_ptr += size;
-
-		if (!b_full) {
-			size = write_ptr;
-			b_full = iwl_write_to_user_buf(user_buf, count,
-						       cpu_addr, &size,
-						       &bytes_copied);
-			data->prev_wr_ptr = size;
-			data->prev_wrap_cnt++;
-		}
-	} else {
-		if (data->prev_wrap_cnt == wrap_cnt - 1 &&
-		    write_ptr > data->prev_wr_ptr)
-			IWL_WARN(trans,
-				 "write pointer passed previous write pointer, start copying from the beginning\n");
-		else if (!unlikely(data->prev_wrap_cnt == 0 &&
-				   data->prev_wr_ptr == 0))
-			IWL_WARN(trans,
-				 "monitor data is out of sync, start copying from the beginning\n");
-
-		size = write_ptr;
-		b_full = iwl_write_to_user_buf(user_buf, count,
-					       cpu_addr, &size,
-					       &bytes_copied);
-		data->prev_wr_ptr = size;
-		data->prev_wrap_cnt = wrap_cnt;
-	}
-
-	mutex_unlock(&data->mutex);
-
-	return bytes_copied;
-}
-
 static ssize_t iwl_dbgfs_rf_read(struct file *file,
 				 char __user *user_buf,
 				 size_t count, loff_t *ppos)
@@ -3264,7 +3109,6 @@ static ssize_t iwl_dbgfs_reset_write(struct file *file,
 }
 
 DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
-DEBUGFS_READ_FILE_OPS(fh_reg);
 DEBUGFS_READ_FILE_OPS(rx_queue);
 DEBUGFS_WRITE_FILE_OPS(csr);
 DEBUGFS_READ_WRITE_FILE_OPS(rfkill);
@@ -3279,12 +3123,6 @@ static const struct file_operations iwl_dbgfs_tx_queue_ops = {
 	.release = seq_release_private,
 };
 
-static const struct file_operations iwl_dbgfs_monitor_data_ops = {
-	.read = iwl_dbgfs_monitor_data_read,
-	.open = iwl_dbgfs_monitor_data_open,
-	.release = iwl_dbgfs_monitor_data_release,
-};
-
 /* Create the debugfs files and directories */
 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
 {
@@ -3294,22 +3132,10 @@ void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
 	DEBUGFS_ADD_FILE(tx_queue, dir, 0400);
 	DEBUGFS_ADD_FILE(interrupt, dir, 0600);
 	DEBUGFS_ADD_FILE(csr, dir, 0200);
-	DEBUGFS_ADD_FILE(fh_reg, dir, 0400);
 	DEBUGFS_ADD_FILE(rfkill, dir, 0600);
-	DEBUGFS_ADD_FILE(monitor_data, dir, 0400);
 	DEBUGFS_ADD_FILE(rf, dir, 0400);
 	DEBUGFS_ADD_FILE(reset, dir, 0200);
 }
-
-void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans)
-{
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	struct cont_rec *data = &trans_pcie->fw_mon_data;
-
-	mutex_lock(&data->mutex);
-	data->state = IWL_FW_MON_DBGFS_STATE_DISABLED;
-	mutex_unlock(&data->mutex);
-}
 #endif /*CONFIG_IWLWIFI_DEBUGFS */
 
 static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd)
@@ -3742,8 +3568,10 @@ void iwl_trans_pci_interrupts(struct iwl_trans *trans, bool enable)
 
 void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans)
 {
-	u32 inta_addr, sw_err_bit;
+	bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+	unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;
+	u32 inta_addr, sw_err_bit;
 
 	if (trans_pcie->msix_enabled) {
 		inta_addr = CSR_MSIX_HW_INT_CAUSES_AD;
@@ -3756,7 +3584,34 @@ void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans)
 		sw_err_bit = CSR_INT_BIT_SW_ERR;
 	}
 
-	iwl_trans_sync_nmi_with_addr(trans, inta_addr, sw_err_bit);
+	/* if the interrupts were already disabled, there is no point in
+	 * calling iwl_disable_interrupts
+	 */
+	if (interrupts_enabled)
+		iwl_trans_pci_interrupts(trans, false);
+
+	iwl_trans_force_nmi(trans);
+	while (time_after(timeout, jiffies)) {
+		u32 inta_hw = iwl_read32(trans, inta_addr);
+
+		/* Error detected by uCode */
+		if (inta_hw & sw_err_bit) {
+			/* Clear causes register */
+			iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);
+			break;
+		}
+
+		mdelay(1);
+	}
+
+	/* enable interrupts only if there were already enabled before this
+	 * function to avoid a case were the driver enable interrupts before
+	 * proper configurations were made
+	 */
+	if (interrupts_enabled)
+		iwl_trans_pci_interrupts(trans, true);
+
+	iwl_trans_fw_error(trans, IWL_ERR_TYPE_NMI_FORCED);
 }
 
 static int iwl_trans_pcie_alloc_txcmd_pool(struct iwl_trans *trans)
@@ -3974,11 +3829,6 @@ iwl_trans_pcie_alloc(struct pci_dev *pdev,
 		}
 	 }
 
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-	trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
-	mutex_init(&trans_pcie->fw_mon_data.mutex);
-#endif
-
 	iwl_dbg_tlv_init(trans);
 
 	return trans;
@@ -4229,10 +4079,9 @@ static void iwl_pcie_check_me_status(struct iwl_trans *trans)
 	schedule_delayed_work(&trans_pcie->me_recheck_wk, HZ);
 }
 
-int iwl_pci_gen1_2_probe(struct pci_dev *pdev,
-			 const struct pci_device_id *ent,
-			 const struct iwl_mac_cfg *mac_cfg,
-			 u8 __iomem *hw_base, u32 hw_rev)
+int _iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent,
+		   const struct iwl_mac_cfg *mac_cfg,
+		   u8 __iomem *hw_base, u32 hw_rev)
 {
 	const struct iwl_dev_info *dev_info;
 	struct iwl_trans_info info = {
@@ -4382,18 +4231,7 @@ int iwl_pci_gen1_2_probe(struct pci_dev *pdev,
 	return ret;
 }
 
-void iwl_pcie_gen1_2_remove(struct iwl_trans *trans)
-{
-	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-
-	cancel_delayed_work_sync(&trans_pcie->me_recheck_wk);
-
-	iwl_drv_stop(trans->drv);
-
-	iwl_trans_pcie_free(trans);
-}
-
-int iwl_pcie_gen1_2_activate_nic(struct iwl_trans *trans)
+int iwl_pcie_activate_nic(struct iwl_trans *trans)
 {
 	const struct iwl_mac_cfg *mac_cfg = trans->mac_cfg;
 	u32 poll_ready;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
similarity index 100%
rename from drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx-gen2.c
rename to drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
similarity index 98%
rename from drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c
rename to drivers/net/wireless/intel/iwlwifi/pcie/tx.c
index dc2fe0c4f68c8c..496435ea93ee83 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
@@ -715,7 +715,7 @@ static void iwl_txq_stuck_timer(struct timer_list *t)
 
 	iwl_txq_log_scd_error(trans, txq);
 
-	iwl_force_nmi(trans);
+	iwl_trans_force_nmi(trans);
 }
 
 int iwl_pcie_txq_alloc(struct iwl_trans *trans, struct iwl_txq *txq,
@@ -1105,7 +1105,7 @@ static void iwl_pcie_cmdq_reclaim(struct iwl_trans *trans, int txq_id, int idx)
 		if (nfreed++ > 0) {
 			IWL_ERR(trans, "HCMD skipped: index (%d) %d %d\n",
 				idx, txq->write_ptr, r);
-			iwl_force_nmi(trans);
+			iwl_trans_force_nmi(trans);
 		}
 	}
 
@@ -1276,9 +1276,17 @@ void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id,
 				bool configure_scd)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	u32 stts_addr = trans_pcie->scd_base_addr +
-			SCD_TX_STTS_QUEUE_OFFSET(txq_id);
 	static const u32 zero_val[4] = {};
+	u32 stts_addr;
+
+	if (WARN_ON(txq_id < 0 ||
+		    txq_id >= trans->mac_cfg->base->num_of_queues ||
+		    txq_id == trans->conf.cmd_queue ||
+		    !trans_pcie->txqs.txq[txq_id]))
+		return;
+
+	stts_addr = trans_pcie->scd_base_addr +
+		    SCD_TX_STTS_QUEUE_OFFSET(txq_id);
 
 	trans_pcie->txqs.txq[txq_id]->frozen_expiry_remainder = 0;
 	trans_pcie->txqs.txq[txq_id]->frozen = false;
@@ -1894,18 +1902,27 @@ struct sg_table *iwl_pcie_prep_tso(struct iwl_trans *trans, struct sk_buff *skb,
 	/* Only map the data, not the header (it is copied to the TSO page) */
 	orig_nents = skb_to_sgvec(skb, sgt->sgl, offset, skb->len - offset);
 	if (WARN_ON_ONCE(orig_nents <= 0))
-		return NULL;
+		goto err_cleanup;
 
 	sgt->orig_nents = orig_nents;
 
 	/* And map the entire SKB */
 	if (dma_map_sgtable(trans->dev, sgt, DMA_TO_DEVICE, 0) < 0)
-		return NULL;
+		goto err_cleanup;
 
 	/* Store non-zero (i.e. valid) offset for unmapping */
 	cmd_meta->sg_offset = (unsigned long) sgt & ~PAGE_MASK;
 
 	return sgt;
+
+err_cleanup:
+	/*
+	 * Since cmd_meta->sg_offset is not set yet, iwl_pcie_free_tso_pages
+	 * will only free the page without unmapping the SG table (which hasn't
+	 * been successfully mapped yet anyway).
+	 */
+	iwl_pcie_free_tso_pages(trans, skb, cmd_meta);
+	return NULL;
 }
 
 static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb,
@@ -2552,6 +2569,7 @@ void iwl_pcie_freeze_txq_timer(struct iwl_trans *trans,
 	}
 }
 
+#define HOST_COMPLETE_QUICK_TIMEOUT	msecs_to_jiffies(200)
 #define HOST_COMPLETE_TIMEOUT	(2 * HZ)
 
 static int iwl_trans_pcie_send_hcmd_sync(struct iwl_trans *trans,
@@ -2585,10 +2603,32 @@ static int iwl_trans_pcie_send_hcmd_sync(struct iwl_trans *trans,
 		return ret;
 	}
 
-	ret = wait_event_timeout(trans_pcie->wait_command_queue,
-				 !test_bit(STATUS_SYNC_HCMD_ACTIVE,
-					   &trans->status),
-				 HOST_COMPLETE_TIMEOUT);
+	/*
+	 * There is a race between the PCI link power save flows and the
+	 * notification / MSI-X interrupt in the firmware.
+	 * If we didn't get a response after 200ms, poke the config space
+	 * to force a wake-up of the PCI link.
+	 */
+	for (int i = 0; i < 2; i++) {
+		ret = wait_event_timeout(trans_pcie->wait_command_queue,
+					 !test_bit(STATUS_SYNC_HCMD_ACTIVE,
+						   &trans->status),
+					 i == 0 ? HOST_COMPLETE_QUICK_TIMEOUT :
+						  HOST_COMPLETE_TIMEOUT);
+		if (ret)
+			break;
+
+		if (i == 0) {
+			u32 val;
+
+			pci_read_config_dword(trans_pcie->pci_dev,
+					      PCI_VENDOR_ID, &val);
+			IWL_DEBUG_HC(trans,
+				     "write_ptr: %d: no response after %ums poking config space\n",
+				     txq->write_ptr,
+				     jiffies_to_msecs(HOST_COMPLETE_QUICK_TIMEOUT));
+		}
+	}
 	if (!ret) {
 		IWL_ERR(trans, "Error sending %s: time out after %dms.\n",
 			cmd_str, jiffies_to_msecs(HOST_COMPLETE_TIMEOUT));
diff --git a/drivers/net/wireless/intersil/p54/fwio.c b/drivers/net/wireless/intersil/p54/fwio.c
index a3d9053f043c68..f028a7bfa73456 100644
--- a/drivers/net/wireless/intersil/p54/fwio.c
+++ b/drivers/net/wireless/intersil/p54/fwio.c
@@ -52,6 +52,13 @@ int p54_parse_firmware(struct ieee80211_hw *dev, const struct firmware *fw)
 		u32 code = le32_to_cpu(bootrec->code);
 		switch (code) {
 		case BR_CODE_COMPONENT_ID:
+			if (len < sizeof(struct bootrec_comp_id) /
+				sizeof(*bootrec->data)) {
+				wiphy_err(priv->hw->wiphy,
+					  "firmware component ID is too short\n");
+				return -EINVAL;
+			}
+
 			priv->fw_interface = be32_to_cpup((__be32 *)
 					     bootrec->data);
 			switch (priv->fw_interface) {
@@ -72,16 +79,41 @@ int p54_parse_firmware(struct ieee80211_hw *dev, const struct firmware *fw)
 			}
 			break;
 		case BR_CODE_COMPONENT_VERSION:
+			if (len < DIV_ROUND_UP(sizeof(struct bootrec_comp_ver),
+					       sizeof(*bootrec->data))) {
+				wiphy_err(priv->hw->wiphy,
+					  "firmware component version is too short\n");
+				return -EINVAL;
+			}
+
 			/* 24 bytes should be enough for all firmwares */
-			if (strnlen((unsigned char *) bootrec->data, 24) < 24)
-				fw_version = (unsigned char *) bootrec->data;
+			if (strnlen((unsigned char *)bootrec->data,
+				    sizeof(struct bootrec_comp_ver)) <
+				    sizeof(struct bootrec_comp_ver))
+				fw_version = (unsigned char *)bootrec->data;
 			break;
 		case BR_CODE_DESCR: {
 			struct bootrec_desc *desc =
 				(struct bootrec_desc *)bootrec->data;
-			priv->rx_start = le32_to_cpu(desc->rx_start);
-			/* FIXME add sanity checking */
-			priv->rx_end = le32_to_cpu(desc->rx_end) - 0x3500;
+			u32 rx_start, rx_end;
+
+			/* 0xa is the shortest descriptor in supported firmware. */
+			if (len < 0xa) {
+				wiphy_err(priv->hw->wiphy,
+					  "firmware descriptor is too short\n");
+				return -EINVAL;
+			}
+
+			rx_start = le32_to_cpu(desc->rx_start);
+			rx_end = le32_to_cpu(desc->rx_end);
+			if (rx_end < 0x3500 || rx_end - 0x3500 <= rx_start) {
+				wiphy_err(priv->hw->wiphy,
+					  "firmware descriptor has invalid RX range\n");
+				return -EINVAL;
+			}
+
+			priv->rx_start = rx_start;
+			priv->rx_end = rx_end - 0x3500;
 			priv->headroom = desc->headroom;
 			priv->tailroom = desc->tailroom;
 			priv->privacy_caps = desc->privacy_caps;
diff --git a/drivers/net/wireless/marvell/libertas/cfg.c b/drivers/net/wireless/marvell/libertas/cfg.c
index 72c92f72469deb..e015cfa1950c13 100644
--- a/drivers/net/wireless/marvell/libertas/cfg.c
+++ b/drivers/net/wireless/marvell/libertas/cfg.c
@@ -1508,7 +1508,8 @@ static int lbs_cfg_set_default_key(struct wiphy *wiphy,
 
 
 static int lbs_cfg_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			   int link_id, u8 idx, bool pairwise,
+			   int link_id, u8 idx,
+			   enum nl80211_key_type type,
 			   const u8 *mac_addr, struct key_params *params)
 {
 	struct lbs_private *priv = wiphy_priv(wiphy);
@@ -1569,7 +1570,8 @@ static int lbs_cfg_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 
 
 static int lbs_cfg_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			   int link_id, u8 key_index, bool pairwise,
+			   int link_id, u8 key_index,
+			   enum nl80211_key_type type,
 			   const u8 *mac_addr)
 {
 
diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c
index 5cc0c5cac25745..74870c9bd091c3 100644
--- a/drivers/net/wireless/marvell/libertas/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas/if_usb.c
@@ -798,10 +798,18 @@ static int check_fwfile_format(const uint8_t *data, uint32_t totlen)
 	exit = len = 0;
 
 	do {
-		struct fwheader *fwh = (void *)data;
+		struct fwheader *fwh;
+
+		if (totlen - len < sizeof(*fwh))
+			break;
+
+		fwh = (void *)data;
 
 		bincmd = le32_to_cpu(fwh->dnldcmd);
 		blksize = le32_to_cpu(fwh->datalength);
+		if (blksize > totlen - len - sizeof(*fwh))
+			break;
+
 		switch (bincmd) {
 		case FW_HAS_DATA_TO_RECV:
 			offset = sizeof(struct fwheader) + blksize;
diff --git a/drivers/net/wireless/marvell/libertas/mesh.c b/drivers/net/wireless/marvell/libertas/mesh.c
index fced7485d015c7..686b2e18a0e915 100644
--- a/drivers/net/wireless/marvell/libertas/mesh.c
+++ b/drivers/net/wireless/marvell/libertas/mesh.c
@@ -65,19 +65,6 @@ static int __lbs_mesh_config_send(struct lbs_private *priv,
 	return ret;
 }
 
-static int lbs_mesh_config_send(struct lbs_private *priv,
-			 struct cmd_ds_mesh_config *cmd,
-			 uint16_t action, uint16_t type)
-{
-	int ret;
-
-	if (!(priv->fwcapinfo & FW_CAPINFO_PERSISTENT_CONFIG))
-		return -EOPNOTSUPP;
-
-	ret = __lbs_mesh_config_send(priv, cmd, action, type);
-	return ret;
-}
-
 /* This function is the CMD_MESH_CONFIG legacy function.  It only handles the
  * START and STOP actions.  The extended actions supported by CMD_MESH_CONFIG
  * are all handled by preparing a struct cmd_ds_mesh_config and passing it to
@@ -333,497 +320,6 @@ static const struct attribute_group lbs_mesh_attr_group = {
 	.attrs = lbs_mesh_sysfs_entries,
 };
 
-
-/***************************************************************************
- * Persistent configuration support
- */
-
-static int mesh_get_default_parameters(struct device *dev,
-				       struct mrvl_mesh_defaults *defs)
-{
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	struct cmd_ds_mesh_config cmd;
-	int ret;
-
-	memset(&cmd, 0, sizeof(struct cmd_ds_mesh_config));
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_GET,
-				   CMD_TYPE_MESH_GET_DEFAULTS);
-
-	if (ret)
-		return -EOPNOTSUPP;
-
-	memcpy(defs, &cmd.data[0], sizeof(struct mrvl_mesh_defaults));
-
-	return 0;
-}
-
-/**
- * bootflag_show - Get function for sysfs attribute bootflag
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer where data will be returned
- */
-static ssize_t bootflag_show(struct device *dev,
-			     struct device_attribute *attr, char *buf)
-{
-	struct mrvl_mesh_defaults defs;
-	int ret;
-
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	if (ret)
-		return ret;
-
-	return sysfs_emit(buf, "%d\n", le32_to_cpu(defs.bootflag));
-}
-
-/**
- * bootflag_store - Set function for sysfs attribute bootflag
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer that contains new attribute value
- * @count: size of buffer
- */
-static ssize_t bootflag_store(struct device *dev, struct device_attribute *attr,
-			      const char *buf, size_t count)
-{
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	struct cmd_ds_mesh_config cmd;
-	uint32_t datum;
-	int ret;
-
-	ret = kstrtouint(buf, 10, &datum);
-	if (ret)
-		return ret;
-	if (datum > 1)
-		return -EINVAL;
-
-	memset(&cmd, 0, sizeof(cmd));
-	*((__le32 *)&cmd.data[0]) = cpu_to_le32(!!datum);
-	cmd.length = cpu_to_le16(sizeof(uint32_t));
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_SET,
-				   CMD_TYPE_MESH_SET_BOOTFLAG);
-	if (ret)
-		return ret;
-
-	return strlen(buf);
-}
-
-/**
- * boottime_show - Get function for sysfs attribute boottime
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer where data will be returned
- */
-static ssize_t boottime_show(struct device *dev,
-			     struct device_attribute *attr, char *buf)
-{
-	struct mrvl_mesh_defaults defs;
-	int ret;
-
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	if (ret)
-		return ret;
-
-	return sysfs_emit(buf, "%d\n", defs.boottime);
-}
-
-/**
- * boottime_store - Set function for sysfs attribute boottime
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer that contains new attribute value
- * @count: size of buffer
- */
-static ssize_t boottime_store(struct device *dev,
-			      struct device_attribute *attr,
-			      const char *buf, size_t count)
-{
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	struct cmd_ds_mesh_config cmd;
-	uint32_t datum;
-	int ret;
-
-	ret = kstrtouint(buf, 10, &datum);
-	if (ret)
-		return ret;
-	if (datum > 255)
-		return -EINVAL;
-
-	memset(&cmd, 0, sizeof(cmd));
-
-	/* A too small boot time will result in the device booting into
-	 * standalone (no-host) mode before the host can take control of it,
-	 * so the change will be hard to revert.  This may be a desired
-	 * feature (e.g to configure a very fast boot time for devices that
-	 * will not be attached to a host), but dangerous.  So I'm enforcing a
-	 * lower limit of 20 seconds:  remove and recompile the driver if this
-	 * does not work for you.
-	 */
-	datum = (datum < 20) ? 20 : datum;
-	cmd.data[0] = datum;
-	cmd.length = cpu_to_le16(sizeof(uint8_t));
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_SET,
-				   CMD_TYPE_MESH_SET_BOOTTIME);
-	if (ret)
-		return ret;
-
-	return strlen(buf);
-}
-
-/**
- * channel_show - Get function for sysfs attribute channel
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer where data will be returned
- */
-static ssize_t channel_show(struct device *dev,
-			    struct device_attribute *attr, char *buf)
-{
-	struct mrvl_mesh_defaults defs;
-	int ret;
-
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	if (ret)
-		return ret;
-
-	return sysfs_emit(buf, "%d\n", le16_to_cpu(defs.channel));
-}
-
-/**
- * channel_store - Set function for sysfs attribute channel
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer that contains new attribute value
- * @count: size of buffer
- */
-static ssize_t channel_store(struct device *dev, struct device_attribute *attr,
-			     const char *buf, size_t count)
-{
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	struct cmd_ds_mesh_config cmd;
-	uint32_t datum;
-	int ret;
-
-	ret = kstrtouint(buf, 10, &datum);
-	if (ret)
-		return ret;
-	if (datum < 1 || datum > 11)
-		return -EINVAL;
-
-	memset(&cmd, 0, sizeof(cmd));
-	*((__le16 *)&cmd.data[0]) = cpu_to_le16(datum);
-	cmd.length = cpu_to_le16(sizeof(uint16_t));
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_SET,
-				   CMD_TYPE_MESH_SET_DEF_CHANNEL);
-	if (ret)
-		return ret;
-
-	return strlen(buf);
-}
-
-/**
- * mesh_id_show - Get function for sysfs attribute mesh_id
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer where data will be returned
- */
-static ssize_t mesh_id_show(struct device *dev, struct device_attribute *attr,
-			    char *buf)
-{
-	struct mrvl_mesh_defaults defs;
-	int ret;
-
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	if (ret)
-		return ret;
-
-	if (defs.meshie.val.mesh_id_len > IEEE80211_MAX_SSID_LEN) {
-		dev_err(dev, "inconsistent mesh ID length\n");
-		defs.meshie.val.mesh_id_len = IEEE80211_MAX_SSID_LEN;
-	}
-
-	memcpy(buf, defs.meshie.val.mesh_id, defs.meshie.val.mesh_id_len);
-	buf[defs.meshie.val.mesh_id_len] = '\n';
-	buf[defs.meshie.val.mesh_id_len + 1] = '\0';
-
-	return defs.meshie.val.mesh_id_len + 1;
-}
-
-/**
- * mesh_id_store - Set function for sysfs attribute mesh_id
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer that contains new attribute value
- * @count: size of buffer
- */
-static ssize_t mesh_id_store(struct device *dev, struct device_attribute *attr,
-			     const char *buf, size_t count)
-{
-	struct cmd_ds_mesh_config cmd;
-	struct mrvl_mesh_defaults defs;
-	struct mrvl_meshie *ie;
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	int len;
-	int ret;
-
-	if (count < 2 || count > IEEE80211_MAX_SSID_LEN + 1)
-		return -EINVAL;
-
-	memset(&cmd, 0, sizeof(struct cmd_ds_mesh_config));
-	ie = (struct mrvl_meshie *) &cmd.data[0];
-
-	/* fetch all other Information Element parameters */
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	cmd.length = cpu_to_le16(sizeof(struct mrvl_meshie));
-
-	/* transfer IE elements */
-	memcpy(ie, &defs.meshie, sizeof(struct mrvl_meshie));
-
-	len = count - 1;
-	memcpy(ie->val.mesh_id, buf, len);
-	/* SSID len */
-	ie->val.mesh_id_len = len;
-	/* IE len */
-	ie->len = sizeof(struct mrvl_meshie_val) - IEEE80211_MAX_SSID_LEN + len;
-
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_SET,
-				   CMD_TYPE_MESH_SET_MESH_IE);
-	if (ret)
-		return ret;
-
-	return strlen(buf);
-}
-
-/**
- * protocol_id_show - Get function for sysfs attribute protocol_id
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer where data will be returned
- */
-static ssize_t protocol_id_show(struct device *dev,
-				struct device_attribute *attr,
-				char *buf)
-{
-	struct mrvl_mesh_defaults defs;
-	int ret;
-
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	if (ret)
-		return ret;
-
-	return sysfs_emit(buf, "%d\n", defs.meshie.val.active_protocol_id);
-}
-
-/**
- * protocol_id_store - Set function for sysfs attribute protocol_id
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer that contains new attribute value
- * @count: size of buffer
- */
-static ssize_t protocol_id_store(struct device *dev,
-				 struct device_attribute *attr,
-				 const char *buf, size_t count)
-{
-	struct cmd_ds_mesh_config cmd;
-	struct mrvl_mesh_defaults defs;
-	struct mrvl_meshie *ie;
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	uint32_t datum;
-	int ret;
-
-	ret = kstrtouint(buf, 10, &datum);
-	if (ret)
-		return ret;
-	if (datum > 255)
-		return -EINVAL;
-
-	memset(&cmd, 0, sizeof(cmd));
-
-	/* fetch all other Information Element parameters */
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	cmd.length = cpu_to_le16(sizeof(struct mrvl_meshie));
-
-	/* transfer IE elements */
-	ie = (struct mrvl_meshie *) &cmd.data[0];
-	memcpy(ie, &defs.meshie, sizeof(struct mrvl_meshie));
-	/* update protocol id */
-	ie->val.active_protocol_id = datum;
-
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_SET,
-				   CMD_TYPE_MESH_SET_MESH_IE);
-	if (ret)
-		return ret;
-
-	return strlen(buf);
-}
-
-/**
- * metric_id_show - Get function for sysfs attribute metric_id
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer where data will be returned
- */
-static ssize_t metric_id_show(struct device *dev,
-			      struct device_attribute *attr, char *buf)
-{
-	struct mrvl_mesh_defaults defs;
-	int ret;
-
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	if (ret)
-		return ret;
-
-	return sysfs_emit(buf, "%d\n", defs.meshie.val.active_metric_id);
-}
-
-/**
- * metric_id_store - Set function for sysfs attribute metric_id
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer that contains new attribute value
- * @count: size of buffer
- */
-static ssize_t metric_id_store(struct device *dev,
-			       struct device_attribute *attr,
-			       const char *buf, size_t count)
-{
-	struct cmd_ds_mesh_config cmd;
-	struct mrvl_mesh_defaults defs;
-	struct mrvl_meshie *ie;
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	uint32_t datum;
-	int ret;
-
-	memset(&cmd, 0, sizeof(cmd));
-	ret = sscanf(buf, "%d", &datum);
-	if ((ret != 1) || (datum > 255))
-		return -EINVAL;
-
-	/* fetch all other Information Element parameters */
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	cmd.length = cpu_to_le16(sizeof(struct mrvl_meshie));
-
-	/* transfer IE elements */
-	ie = (struct mrvl_meshie *) &cmd.data[0];
-	memcpy(ie, &defs.meshie, sizeof(struct mrvl_meshie));
-	/* update metric id */
-	ie->val.active_metric_id = datum;
-
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_SET,
-				   CMD_TYPE_MESH_SET_MESH_IE);
-	if (ret)
-		return ret;
-
-	return strlen(buf);
-}
-
-/**
- * capability_show - Get function for sysfs attribute capability
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer where data will be returned
- */
-static ssize_t capability_show(struct device *dev,
-			       struct device_attribute *attr, char *buf)
-{
-	struct mrvl_mesh_defaults defs;
-	int ret;
-
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	if (ret)
-		return ret;
-
-	return sysfs_emit(buf, "%d\n", defs.meshie.val.mesh_capability);
-}
-
-/**
- * capability_store - Set function for sysfs attribute capability
- * @dev: the &struct device
- * @attr: device attributes
- * @buf: buffer that contains new attribute value
- * @count: size of buffer
- */
-static ssize_t capability_store(struct device *dev,
-				struct device_attribute *attr,
-				const char *buf, size_t count)
-{
-	struct cmd_ds_mesh_config cmd;
-	struct mrvl_mesh_defaults defs;
-	struct mrvl_meshie *ie;
-	struct lbs_private *priv = to_net_dev(dev)->ml_priv;
-	uint32_t datum;
-	int ret;
-
-	memset(&cmd, 0, sizeof(cmd));
-	ret = sscanf(buf, "%d", &datum);
-	if ((ret != 1) || (datum > 255))
-		return -EINVAL;
-
-	/* fetch all other Information Element parameters */
-	ret = mesh_get_default_parameters(dev, &defs);
-
-	cmd.length = cpu_to_le16(sizeof(struct mrvl_meshie));
-
-	/* transfer IE elements */
-	ie = (struct mrvl_meshie *) &cmd.data[0];
-	memcpy(ie, &defs.meshie, sizeof(struct mrvl_meshie));
-	/* update value */
-	ie->val.mesh_capability = datum;
-
-	ret = lbs_mesh_config_send(priv, &cmd, CMD_ACT_MESH_CONFIG_SET,
-				   CMD_TYPE_MESH_SET_MESH_IE);
-	if (ret)
-		return ret;
-
-	return strlen(buf);
-}
-
-
-static DEVICE_ATTR_RW(bootflag);
-static DEVICE_ATTR_RW(boottime);
-static DEVICE_ATTR_RW(channel);
-static DEVICE_ATTR_RW(mesh_id);
-static DEVICE_ATTR_RW(protocol_id);
-static DEVICE_ATTR_RW(metric_id);
-static DEVICE_ATTR_RW(capability);
-
-static struct attribute *boot_opts_attrs[] = {
-	&dev_attr_bootflag.attr,
-	&dev_attr_boottime.attr,
-	&dev_attr_channel.attr,
-	NULL
-};
-
-static const struct attribute_group boot_opts_group = {
-	.name = "boot_options",
-	.attrs = boot_opts_attrs,
-};
-
-static struct attribute *mesh_ie_attrs[] = {
-	&dev_attr_mesh_id.attr,
-	&dev_attr_protocol_id.attr,
-	&dev_attr_metric_id.attr,
-	&dev_attr_capability.attr,
-	NULL
-};
-
-static const struct attribute_group mesh_ie_group = {
-	.name = "mesh_ie",
-	.attrs = mesh_ie_attrs,
-};
-
-
 /***************************************************************************
  * Initializing and starting, stopping mesh
  */
@@ -1019,8 +515,6 @@ static int lbs_add_mesh(struct lbs_private *priv)
 
 	mesh_dev->flags |= IFF_BROADCAST | IFF_MULTICAST;
 	mesh_dev->sysfs_groups[0] = &lbs_mesh_attr_group;
-	mesh_dev->sysfs_groups[1] = &boot_opts_group;
-	mesh_dev->sysfs_groups[2] = &mesh_ie_group;
 
 	/* Register virtual mesh interface */
 	ret = register_netdev(mesh_dev);
diff --git a/drivers/net/wireless/marvell/libertas/rx.c b/drivers/net/wireless/marvell/libertas/rx.c
index c34d30f7cbe03c..a9d1320d90b15b 100644
--- a/drivers/net/wireless/marvell/libertas/rx.c
+++ b/drivers/net/wireless/marvell/libertas/rx.c
@@ -89,6 +89,15 @@ int lbs_process_rxed_packet(struct lbs_private *priv, struct sk_buff *skb)
 		goto done;
 	}
 
+	if (le32_to_cpu(p_rx_pd->pkt_ptr) >
+	    skb->len - sizeof(struct rxpackethdr)) {
+		lbs_deb_rx("rx err: pkt_ptr beyond packet\n");
+		dev->stats.rx_length_errors++;
+		ret = -EINVAL;
+		dev_kfree_skb(skb);
+		goto done;
+	}
+
 	lbs_deb_rx("rx data: skb->len - pkt_ptr = %d-%zd = %zd\n",
 		skb->len, (size_t)le32_to_cpu(p_rx_pd->pkt_ptr),
 		skb->len - (size_t)le32_to_cpu(p_rx_pd->pkt_ptr));
diff --git a/drivers/net/wireless/marvell/libertas_tf/if_usb.c b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
index b85c6d783bf7b2..b9d1a33bddfcea 100644
--- a/drivers/net/wireless/marvell/libertas_tf/if_usb.c
+++ b/drivers/net/wireless/marvell/libertas_tf/if_usb.c
@@ -769,10 +769,18 @@ static int check_fwfile_format(const u8 *data, u32 totlen)
 	exit = len = 0;
 
 	do {
-		struct fwheader *fwh = (void *) data;
+		struct fwheader *fwh;
+
+		if (totlen - len < sizeof(*fwh))
+			break;
+
+		fwh = (void *)data;
 
 		bincmd = le32_to_cpu(fwh->dnldcmd);
 		blksize = le32_to_cpu(fwh->datalength);
+		if (blksize > totlen - len - sizeof(*fwh))
+			break;
+
 		switch (bincmd) {
 		case FW_HAS_DATA_TO_RECV:
 			offset = sizeof(struct fwheader) + blksize;
diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
index 9deb47f22a6174..06b00552507841 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
+++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
@@ -213,7 +213,6 @@ mwifiex_del_rx_reorder_entry(struct mwifiex_private *priv,
 	list_del(&tbl->list);
 	spin_unlock_bh(&priv->rx_reorder_tbl_lock);
 
-	kfree(tbl->rx_reorder_ptr);
 	kfree(tbl);
 
 	spin_lock_bh(&priv->adapter->rx_proc_lock);
@@ -329,7 +328,6 @@ static void
 mwifiex_11n_create_rx_reorder_tbl(struct mwifiex_private *priv, u8 *ta,
 				  int tid, int win_size, int seq_num)
 {
-	int i;
 	struct mwifiex_rx_reorder_tbl *tbl, *new_node;
 	u16 last_seq = 0;
 	struct mwifiex_sta_node *node;
@@ -344,10 +342,12 @@ mwifiex_11n_create_rx_reorder_tbl(struct mwifiex_private *priv, u8 *ta,
 		return;
 	}
 	/* if !tbl then create one */
-	new_node = kzalloc_obj(struct mwifiex_rx_reorder_tbl);
+	new_node = kzalloc_flex(*new_node, rx_reorder_ptr, win_size);
 	if (!new_node)
 		return;
 
+	new_node->win_size = win_size;
+
 	INIT_LIST_HEAD(&new_node->list);
 	new_node->tid = tid;
 	memcpy(new_node->ta, ta, ETH_ALEN);
@@ -381,26 +381,12 @@ mwifiex_11n_create_rx_reorder_tbl(struct mwifiex_private *priv, u8 *ta,
 		new_node->flags |= RXREOR_INIT_WINDOW_SHIFT;
 	}
 
-	new_node->win_size = win_size;
-
-	new_node->rx_reorder_ptr = kcalloc(win_size, sizeof(void *),
-					   GFP_KERNEL);
-	if (!new_node->rx_reorder_ptr) {
-		kfree(new_node);
-		mwifiex_dbg(priv->adapter, ERROR,
-			    "%s: failed to alloc reorder_ptr\n", __func__);
-		return;
-	}
-
 	new_node->timer_context.ptr = new_node;
 	new_node->timer_context.priv = priv;
 	new_node->timer_context.timer_is_set = false;
 
 	timer_setup(&new_node->timer_context.timer, mwifiex_flush_data, 0);
 
-	for (i = 0; i < win_size; ++i)
-		new_node->rx_reorder_ptr[i] = NULL;
-
 	spin_lock_bh(&priv->rx_reorder_tbl_lock);
 	list_add_tail(&new_node->list, &priv->rx_reorder_tbl_ptr);
 	spin_unlock_bh(&priv->rx_reorder_tbl_lock);
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index 936939ea9c4723..a30fd0c936e12a 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -142,12 +142,14 @@ static void *mwifiex_cfg80211_get_adapter(struct wiphy *wiphy)
  */
 static int
 mwifiex_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			 int link_id, u8 key_index, bool pairwise,
+			 int link_id, u8 key_index,
+			enum nl80211_key_type type,
 			 const u8 *mac_addr)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 	static const u8 bc_mac[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	const u8 *peer_mac = pairwise ? mac_addr : bc_mac;
+	const u8 *peer_mac =
+		(type == NL80211_KEYTYPE_PAIRWISE) ? mac_addr : bc_mac;
 
 	if (mwifiex_set_encode(priv, NULL, NULL, 0, key_index, peer_mac, 1)) {
 		mwifiex_dbg(priv->adapter, ERROR, "deleting the crypto keys\n");
@@ -480,13 +482,15 @@ mwifiex_cfg80211_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
  */
 static int
 mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			 int link_id, u8 key_index, bool pairwise,
+			 int link_id, u8 key_index,
+			 enum nl80211_key_type type,
 			 const u8 *mac_addr, struct key_params *params)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 	struct mwifiex_wep_key *wep_key;
 	static const u8 bc_mac[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	const u8 *peer_mac = pairwise ? mac_addr : bc_mac;
+	const u8 *peer_mac =
+		(type == NL80211_KEYTYPE_PAIRWISE) ? mac_addr : bc_mac;
 
 	if (GET_BSS_ROLE(priv) == MWIFIEX_BSS_ROLE_UAP &&
 	    (params->cipher == WLAN_CIPHER_SUITE_WEP40 ||
@@ -3562,6 +3566,14 @@ static int mwifiex_cfg80211_suspend(struct wiphy *wiphy,
 		mwifiex_dbg(adapter, ERROR, "Failed to set HS params\n");
 
 done:
+	if (ret) {
+		for (i = 0; i < adapter->priv_num; i++) {
+			priv = adapter->priv[i];
+			if (priv->netdev)
+				netif_device_attach(priv->netdev);
+		}
+	}
+
 	sta_priv->scan_aborting = false;
 	return ret;
 }
@@ -4905,7 +4917,14 @@ int mwifiex_register_cfg80211(struct mwifiex_adapter *adapter)
 				country_code =
 					mwifiex_11d_code_2_region(
 						adapter->region_code);
+
+				/* If the reg hint in ROM conflicts with platform
+				 * configuration, it should be ignored, so the
+				 * platform regulatory domain can be used.
+				 */
 				if (country_code &&
+				    !device_property_read_bool(adapter->dev,
+							       "marvell,invalid-reg-hint-in-rom") &&
 				    regulatory_hint(wiphy, country_code))
 					mwifiex_dbg(priv->adapter, ERROR,
 						    "regulatory_hint() failed\n");
diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h
index 27559e2ddc3175..8e6238c01aa67c 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.h
+++ b/drivers/net/wireless/marvell/mwifiex/main.h
@@ -708,10 +708,10 @@ struct mwifiex_rx_reorder_tbl {
 	int init_win;
 	int start_win;
 	int win_size;
-	void **rx_reorder_ptr;
 	struct reorder_tmr_cnxt timer_context;
 	u8 amsdu;
 	u8 flags;
+	struct sk_buff *rx_reorder_ptr[] __counted_by(win_size);
 };
 
 struct mwifiex_bss_prio_node {
diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.c b/drivers/net/wireless/marvell/mwifiex/sdio.c
index f039d6f19183a5..c15d6e1dd8f633 100644
--- a/drivers/net/wireless/marvell/mwifiex/sdio.c
+++ b/drivers/net/wireless/marvell/mwifiex/sdio.c
@@ -2795,6 +2795,9 @@ static void mwifiex_sdio_fw_dump(struct mwifiex_adapter *adapter)
 		mwifiex_dbg(adapter, ERROR, "SDIO read memory length err\n");
 		goto done;
 	}
+	/* Limit the dump count to the memory table size */
+	if (dump_num > ARRAY_SIZE(mem_type_mapping_tbl))
+		dump_num = ARRAY_SIZE(mem_type_mapping_tbl);
 
 	/* Read the length of every memory which will dump */
 	for (idx = 0; idx < dump_num; idx++) {
diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c
index abbe65cbcd89e2..cda5c5d51aab0a 100644
--- a/drivers/net/wireless/mediatek/mt76/mac80211.c
+++ b/drivers/net/wireless/mediatek/mt76/mac80211.c
@@ -1255,11 +1255,12 @@ EXPORT_SYMBOL(mt76_rx_signal);
 static void
 mt76_rx_convert(struct mt76_dev *dev, struct sk_buff *skb,
 		struct ieee80211_hw **hw,
-		struct ieee80211_sta **sta)
+		struct ieee80211_link_sta **link_sta)
 {
 	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_hdr *hdr = mt76_skb_get_hdr(skb);
 	struct mt76_rx_status mstat;
+	struct ieee80211_sta *sta;
 
 	mstat = *((struct mt76_rx_status *)skb->cb);
 	memset(status, 0, sizeof(*status));
@@ -1301,12 +1302,14 @@ mt76_rx_convert(struct mt76_dev *dev, struct sk_buff *skb,
 	memcpy(status->chain_signal, mstat.chain_signal,
 	       sizeof(mstat.chain_signal));
 
-	if (mstat.wcid) {
-		status->link_valid = mstat.wcid->link_valid;
-		status->link_id = mstat.wcid->link_id;
-	}
+	sta = wcid_to_sta(mstat.wcid);
+	if (!sta)
+		*link_sta = NULL;
+	else if (mstat.wcid->link_valid)
+		*link_sta = rcu_dereference(sta->link[mstat.wcid->link_id]);
+	else
+		*link_sta = &sta->deflink;
 
-	*sta = wcid_to_sta(mstat.wcid);
 	*hw = mt76_phy_hw(dev, mstat.phy_idx);
 }
 
@@ -1530,7 +1533,7 @@ mt76_check_sta(struct mt76_dev *dev, struct sk_buff *skb)
 void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
 		      struct napi_struct *napi)
 {
-	struct ieee80211_sta *sta;
+	struct ieee80211_link_sta *link_sta;
 	struct ieee80211_hw *hw;
 	struct sk_buff *skb, *tmp;
 	LIST_HEAD(list);
@@ -1541,8 +1544,8 @@ void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
 
 		mt76_check_ccmp_pn(skb);
 		skb_shinfo(skb)->frag_list = NULL;
-		mt76_rx_convert(dev, skb, &hw, &sta);
-		ieee80211_rx_list(hw, sta, skb, &list);
+		mt76_rx_convert(dev, skb, &hw, &link_sta);
+		ieee80211_rx_list(hw, link_sta, skb, &list);
 
 		/* subsequent amsdu frames */
 		while (nskb) {
@@ -1550,8 +1553,8 @@ void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
 			nskb = nskb->next;
 			skb->next = NULL;
 
-			mt76_rx_convert(dev, skb, &hw, &sta);
-			ieee80211_rx_list(hw, sta, skb, &list);
+			mt76_rx_convert(dev, skb, &hw, &link_sta);
+			ieee80211_rx_list(hw, link_sta, skb, &list);
 		}
 	}
 	spin_unlock(&dev->rx_lock);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
index 2f925d22b9aa73..2c49f3b493bdb3 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
@@ -2034,10 +2034,7 @@ int mt76_connac_mcu_sched_scan_enable(struct mt76_phy *phy,
 		.active = !enable,
 	};
 
-	if (enable)
-		set_bit(MT76_HW_SCHED_SCANNING, &phy->state);
-	else
-		clear_bit(MT76_HW_SCHED_SCANNING, &phy->state);
+	assign_bit(MT76_HW_SCHED_SCANNING, &phy->state, enable);
 
 	return mt76_mcu_send_msg(phy->dev, MCU_CE_CMD(SCHED_SCAN_ENABLE),
 				 &req, sizeof(req), false);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
index fa29c486a4553e..aeb017516186f9 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
@@ -3467,10 +3467,7 @@ mt7925_mcu_sched_scan_enable(struct mt76_phy *phy,
 	req = (struct scan_sched_enable *)tlv;
 	req->active = !enable;
 
-	if (enable)
-		set_bit(MT76_HW_SCHED_SCANNING, &phy->state);
-	else
-		clear_bit(MT76_HW_SCHED_SCANNING, &phy->state);
+	assign_bit(MT76_HW_SCHED_SCANNING, &phy->state, enable);
 
 	return mt76_mcu_skb_send_msg(mdev, skb, MCU_UNI_CMD(SCAN_REQ),
 				     true);
diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
index 9ad21d41e99995..3db290875ff47f 100644
--- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
@@ -535,10 +535,13 @@ static int wilc_wfi_cfg_copy_wpa_info(struct wilc_wfi_key *key_info,
 }
 
 static int add_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
-		   u8 key_index, bool pairwise, const u8 *mac_addr,
+		   u8 key_index,
+		   enum nl80211_key_type type,
+		   const u8 *mac_addr,
 		   struct key_params *params)
 
 {
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	int ret = 0, keylen = params->key_len;
 	const u8 *rx_mic = NULL;
 	const u8 *tx_mic = NULL;
@@ -641,9 +644,10 @@ static int add_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
 
 static int del_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
 		   u8 key_index,
-		   bool pairwise,
+		   enum nl80211_key_type type,
 		   const u8 *mac_addr)
 {
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	struct wilc_priv *priv = &vif->priv;
 
@@ -681,10 +685,11 @@ static int del_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
 }
 
 static int get_key(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
-		   u8 key_index, bool pairwise, const u8 *mac_addr,
-		   void *cookie,
+		   u8 key_index, enum nl80211_key_type type,
+		   const u8 *mac_addr, void *cookie,
 		   void (*callback)(void *cookie, struct key_params *))
 {
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	struct wilc_vif *vif = netdev_priv(wdev->netdev);
 	struct wilc_priv *priv = &vif->priv;
 	struct  key_params key_params;
diff --git a/drivers/net/wireless/morsemicro/mm81x/core.h b/drivers/net/wireless/morsemicro/mm81x/core.h
index 2fd4b4786e777c..75e998e8e3241a 100644
--- a/drivers/net/wireless/morsemicro/mm81x/core.h
+++ b/drivers/net/wireless/morsemicro/mm81x/core.h
@@ -192,7 +192,12 @@ struct mm81x_sta {
 	bool tid_tx[IEEE80211_NUM_TIDS];
 	bool tid_start_tx[IEEE80211_NUM_TIDS];
 	u8 tid_params[IEEE80211_NUM_TIDS];
-	int max_bw_mhz;
+
+	/*
+	 * We really should be using link_sta::bandwidth but
+	 * that requires non-trivial work.
+	 */
+	int max_rx_bw_mhz;
 	struct mm81x_rc_sta rc;
 	struct mmrc_rate last_sta_tx_rate;
 	s16 avg_rssi;
diff --git a/drivers/net/wireless/morsemicro/mm81x/fw.c b/drivers/net/wireless/morsemicro/mm81x/fw.c
index d6d2ad086c3285..a2cb97cefc4e96 100644
--- a/drivers/net/wireless/morsemicro/mm81x/fw.c
+++ b/drivers/net/wireless/morsemicro/mm81x/fw.c
@@ -117,13 +117,15 @@ static int mm81x_fw_load_fw(struct mm81x *mors, const struct firmware *fw)
 
 	if (mm81x_fw_get_header(fw->data, &ehdr)) {
 		dev_err(mors->dev, "Wrong file format");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out_free;
 	}
 
 	if (mm81x_fw_get_section_header(fw->data, &ehdr, &sh_strtab,
 					ehdr.e_shstrndx)) {
 		dev_err(mors->dev, "Invalid firmware. Missing string table");
-		return -ENOENT;
+		ret = -ENOENT;
+		goto out_free;
 	}
 
 	sh_strs = (const char *)fw->data + sh_strtab.sh_offset;
@@ -160,7 +162,7 @@ static int mm81x_fw_load_fw(struct mm81x *mors, const struct firmware *fw)
 			mm81x_release_bus(mors);
 			if (status) {
 				ret = -EIO;
-				break;
+				goto out_free;
 			}
 		}
 	}
@@ -179,6 +181,7 @@ static int mm81x_fw_load_fw(struct mm81x *mors, const struct firmware *fw)
 	if (ehdr.e_entry)
 		ret = mm81x_fw_set_boot_addr(mors, ehdr.e_entry);
 
+out_free:
 	devm_kfree(mors->dev, fw_buf);
 	return ret;
 }
diff --git a/drivers/net/wireless/morsemicro/mm81x/mac.c b/drivers/net/wireless/morsemicro/mm81x/mac.c
index 0fa80b1488aacc..1a7382d27475e8 100644
--- a/drivers/net/wireless/morsemicro/mm81x/mac.c
+++ b/drivers/net/wireless/morsemicro/mm81x/mac.c
@@ -66,6 +66,9 @@
 /* HW restart delay time before terminating hardware IF work items */
 #define MM81X_HW_RESTART_DELAY_MS 20
 
+/* Maximum number of multicast frames to release after a DTIM beacon */
+#define MM81X_MAX_MC_FRAMES_AFTER_DTIM 4
+
 /* clang-format off */
 
 /* mm81x chips do not support 16MHz */
@@ -322,7 +325,7 @@ static int mm81x_mac_ops_start(struct ieee80211_hw *hw)
 	return 0;
 }
 
-static int mm81x_tx_h_get_max_bw(struct mm81x *mors)
+static int mm81x_tx_h_get_max_tx_bw(struct mm81x *mors)
 {
 	return MM81X_FW_SUPP(&mors->fw_caps, 8MHZ) ? 8 :
 	       MM81X_FW_SUPP(&mors->fw_caps, 4MHZ) ? 4 :
@@ -445,39 +448,6 @@ static void mm81x_beacon_h_fill_tx_info(struct mm81x *mors,
 			cpu_to_le32(MM81X_TX_CONF_FLAGS_IMMEDIATE_REPORT);
 }
 
-static void mm81x_mac_beacon_work(struct work_struct *work)
-{
-	struct mm81x_vif *mors_vif =
-		from_work(mors_vif, work, u.ap.beacon_work);
-	struct mm81x *mors = mm81x_vif_to_mors(mors_vif);
-	struct mm81x_skbq *mq;
-	struct sk_buff *beacon;
-	struct ieee80211_vif *vif = mm81x_vif_to_ieee80211_vif(mors_vif);
-	struct mm81x_skb_tx_info tx_info = { 0 };
-	int num_bcn_vifs = atomic_read(&mors->num_bcn_vifs);
-
-	mq = mm81x_hif_get_tx_beacon_queue(mors);
-	if (!mq) {
-		dev_err(mors->dev, "no matching beacon Q found");
-		return;
-	}
-
-	if (mm81x_skbq_count(mq) >= num_bcn_vifs) {
-		dev_err(mors->dev,
-			"previous beacon not consumed, dropping req [id:%d]",
-			mors_vif->id);
-		return;
-	}
-
-	beacon = ieee80211_beacon_get(mors->hw, vif, false);
-	if (!beacon)
-		return;
-
-	mm81x_beacon_h_fill_tx_info(mors, &tx_info, mors_vif,
-				    cfg80211_chandef_s1g_pri_width(&mors->chandef));
-	mm81x_skbq_skb_tx(mq, &beacon, &tx_info, MM81X_SKB_CHAN_BEACON);
-}
-
 void mm81x_mac_beacon_irq_handle(struct mm81x *mors, u32 status)
 {
 	int vif_id;
@@ -497,15 +467,6 @@ void mm81x_mac_beacon_irq_handle(struct mm81x *mors, u32 status)
 	}
 }
 
-static void mm81x_mac_beacon_init(struct mm81x_vif *mors_vif)
-{
-	struct mm81x *mors = mm81x_vif_to_mors(mors_vif);
-
-	INIT_WORK(&mors_vif->u.ap.beacon_work, mm81x_mac_beacon_work);
-	mm81x_mac_beacon_irq_enable(mors_vif, true);
-	atomic_inc(&mors->num_bcn_vifs);
-}
-
 static struct hw_scan_tlv_hdr mm81x_hw_scan_h_pack_tlv_hdr(u16 tag, u16 len)
 {
 	struct hw_scan_tlv_hdr hdr = { .tag = cpu_to_le16(tag),
@@ -1281,38 +1242,54 @@ static void mm81x_tx_h_fill_info(struct mm81x *mors,
 	}
 }
 
+static int mm81x_tx_h_get_bw(struct mm81x *mors, struct ieee80211_sta *sta,
+			     struct sk_buff *skb, bool is_mgmt)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct mm81x_sta *mors_sta = NULL;
+	int tx_bw_mhz;
+
+	if (ieee80211_is_probe_resp(hdr->frame_control))
+		return 1;
+
+	if (is_mgmt || info->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO)
+		return cfg80211_chandef_s1g_pri_width(&mors->chandef);
+
+	/*
+	 * In AP mode group addressed frames go out at the primary width so
+	 * that every associated STA can receive them, including any that
+	 * cannot support the operating width.
+	 */
+	if (info->control.vif->type == NL80211_IFTYPE_AP &&
+	    is_multicast_ether_addr(ieee80211_get_DA(hdr)))
+		return cfg80211_chandef_s1g_pri_width(&mors->chandef);
+
+	if (sta)
+		mors_sta = (struct mm81x_sta *)sta->drv_priv;
+
+	tx_bw_mhz = min(mm81x_tx_h_get_max_tx_bw(mors),
+			cfg80211_chandef_get_width(&mors->chandef));
+	if (mors_sta && mors_sta->max_rx_bw_mhz)
+		tx_bw_mhz = min(tx_bw_mhz, mors_sta->max_rx_bw_mhz);
+
+	return tx_bw_mhz;
+}
+
 static void mm81x_mac_ops_tx(struct ieee80211_hw *hw,
 			     struct ieee80211_tx_control *control,
 			     struct sk_buff *skb)
 {
-	struct mm81x *mors = hw->priv;
-	struct mm81x_skbq *mq = NULL;
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-	struct ieee80211_vif *vif = info->control.vif;
-	struct mm81x_skb_tx_info tx_info = { 0 };
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	bool is_mgmt = ieee80211_is_mgmt(hdr->frame_control);
-	int tx_bw_mhz = cfg80211_chandef_get_width(&mors->chandef);
 	struct ieee80211_sta *sta = control->sta;
-	int max_tx_bw = 0, sta_max_bw_mhz = 0;
-
-	if (sta) {
-		struct mm81x_sta *mors_sta = (struct mm81x_sta *)sta->drv_priv;
-
-		sta_max_bw_mhz = mors_sta->max_bw_mhz;
-	}
-
-	max_tx_bw = mm81x_tx_h_get_max_bw(mors);
-	tx_bw_mhz = min(max_tx_bw, tx_bw_mhz);
-
-	if (is_mgmt)
-		tx_bw_mhz = cfg80211_chandef_s1g_pri_width(&mors->chandef);
-	if (sta_max_bw_mhz)
-		tx_bw_mhz = min(tx_bw_mhz, sta_max_bw_mhz);
-	if (ieee80211_is_probe_resp(hdr->frame_control))
-		tx_bw_mhz = 1;
+	struct mm81x_skb_tx_info tx_info = { 0 };
+	struct mm81x *mors = hw->priv;
+	struct mm81x_skbq *mq;
 
-	mm81x_tx_h_fill_info(mors, &tx_info, skb, vif, tx_bw_mhz, sta);
+	mm81x_tx_h_fill_info(mors, &tx_info, skb, info->control.vif,
+			     mm81x_tx_h_get_bw(mors, sta, skb, is_mgmt), sta);
 
 	if (mm81x_tx_h_ps_filtered_for_sta(mors, skb, sta))
 		return;
@@ -1328,6 +1305,65 @@ static void mm81x_mac_ops_tx(struct ieee80211_hw *hw,
 				      MM81X_SKB_CHAN_DATA);
 }
 
+static void mm81x_mac_send_buffered_bc(struct mm81x *mors,
+				       struct ieee80211_vif *vif)
+{
+	int count = MM81X_MAX_MC_FRAMES_AFTER_DTIM;
+	struct ieee80211_tx_control control = { 0 };
+	struct sk_buff *bc_frame;
+
+	while (count-- &&
+	       (bc_frame = ieee80211_get_buffered_bc(mors->hw, vif))) {
+		IEEE80211_SKB_CB(bc_frame)->control.vif = vif;
+		mm81x_mac_ops_tx(mors->hw, &control, bc_frame);
+	}
+}
+
+static void mm81x_mac_beacon_work(struct work_struct *work)
+{
+	struct mm81x_vif *mors_vif =
+		from_work(mors_vif, work, u.ap.beacon_work);
+	struct mm81x *mors = mm81x_vif_to_mors(mors_vif);
+	struct mm81x_skbq *mq;
+	struct sk_buff *beacon;
+	struct ieee80211_vif *vif = mm81x_vif_to_ieee80211_vif(mors_vif);
+	struct mm81x_skb_tx_info tx_info = { 0 };
+	int num_bcn_vifs = atomic_read(&mors->num_bcn_vifs);
+
+	mq = mm81x_hif_get_tx_beacon_queue(mors);
+	if (!mq) {
+		dev_err(mors->dev, "no matching beacon Q found");
+		return;
+	}
+
+	if (mm81x_skbq_count(mq) >= num_bcn_vifs) {
+		dev_err(mors->dev,
+			"previous beacon not consumed, dropping req [id:%d]",
+			mors_vif->id);
+		return;
+	}
+
+	beacon = ieee80211_beacon_get(mors->hw, vif, false);
+	if (!beacon)
+		return;
+
+	mm81x_beacon_h_fill_tx_info(mors, &tx_info, mors_vif,
+				    cfg80211_chandef_s1g_pri_width(&mors->chandef));
+	mm81x_skbq_skb_tx(mq, &beacon, &tx_info, MM81X_SKB_CHAN_BEACON);
+
+	if (!test_bit(MM81X_STATE_DATA_QS_STOPPED, &mors->state_flags))
+		mm81x_mac_send_buffered_bc(mors, vif);
+}
+
+static void mm81x_mac_beacon_init(struct mm81x_vif *mors_vif)
+{
+	struct mm81x *mors = mm81x_vif_to_mors(mors_vif);
+
+	INIT_WORK(&mors_vif->u.ap.beacon_work, mm81x_mac_beacon_work);
+	mm81x_mac_beacon_irq_enable(mors_vif, true);
+	atomic_inc(&mors->num_bcn_vifs);
+}
+
 static void mm81x_mac_ops_stop(struct ieee80211_hw *hw, bool suspend)
 {
 	struct mm81x *mors = hw->priv;
@@ -1690,6 +1726,9 @@ static int mm81x_mac_ops_sta_state(struct ieee80211_hw *hw,
 			mors_vif->u.ap.num_stas++;
 		else if (vif->type == NL80211_IFTYPE_STATION)
 			mors_vif->u.sta.is_assoc = true;
+
+		mors_sta->max_rx_bw_mhz =
+			S1G_SUPP_CH_WIDTH_MAX(sta->deflink.s1g_cap.cap);
 	}
 
 	if (new_state < old_state && new_state == IEEE80211_STA_NONE) {
@@ -2014,6 +2053,13 @@ static void mm81x_mac_ops_flush(struct ieee80211_hw *hw,
 		mm81x_mac_wait_queues(mors);
 }
 
+static bool mm81x_mac_ops_tx_frames_pending(struct ieee80211_hw *hw)
+{
+	struct mm81x *mors = hw->priv;
+
+	return mm81x_mac_has_tx_pending(mors);
+}
+
 static int mm81x_mac_ops_set_rts_threshold(struct ieee80211_hw *hw,
 					   int radio_idx, u32 value)
 {
@@ -2239,6 +2285,7 @@ static const struct ieee80211_ops mm81x_ops = {
 	.configure_filter = mm81x_mac_ops_configure_filter,
 	.sta_state = mm81x_mac_ops_sta_state,
 	.flush = mm81x_mac_ops_flush,
+	.tx_frames_pending = mm81x_mac_ops_tx_frames_pending,
 	.set_frag_threshold = mm81x_mac_set_frag_threshold,
 	.set_rts_threshold = mm81x_mac_ops_set_rts_threshold,
 	.sta_statistics = mm81x_mac_ops_sta_statistics,
diff --git a/drivers/net/wireless/morsemicro/mm81x/sdio.c b/drivers/net/wireless/morsemicro/mm81x/sdio.c
index 96fce187dd3522..65277399bf189a 100644
--- a/drivers/net/wireless/morsemicro/mm81x/sdio.c
+++ b/drivers/net/wireless/morsemicro/mm81x/sdio.c
@@ -381,7 +381,7 @@ static int mm81x_sdio_dm_read(struct mm81x *mors, u32 address, u8 *data,
 
 static int mm81x_sdio_reg32_write(struct mm81x *mors, u32 address, u32 val)
 {
-	ssize_t ret = 0;
+	int ret = 0;
 	u32 original_address = address;
 	struct mm81x_sdio *sdio = (struct mm81x_sdio *)mors->drv_priv;
 	struct sdio_func *func1 = sdio->func->card->sdio_func[0];
@@ -391,7 +391,7 @@ static int mm81x_sdio_reg32_write(struct mm81x *mors, u32 address, u32 val)
 
 	address &= 0x0000FFFF;
 	sdio_writel(func1, (__force u32)cpu_to_le32(val),
-		    (__force u32)cpu_to_le32(address), (int *)&ret);
+		    (__force u32)cpu_to_le32(address), &ret);
 	if (ret)
 		goto error;
 
@@ -411,7 +411,7 @@ static int mm81x_sdio_reg32_write(struct mm81x *mors, u32 address, u32 val)
 static int mm81x_sdio_reg32_read(struct mm81x *mors, u32 address, u32 *val)
 {
 	u32 value;
-	ssize_t ret = 0;
+	int ret = 0;
 	struct mm81x_sdio *sdio = (struct mm81x_sdio *)mors->drv_priv;
 	struct sdio_func *func1 = sdio->func->card->sdio_func[0];
 
@@ -419,8 +419,7 @@ static int mm81x_sdio_reg32_read(struct mm81x *mors, u32 address, u32 *val)
 					 MM81X_CONFIG_ACCESS_4BYTE);
 
 	address &= 0x0000FFFF;
-	value = sdio_readl(func1, (__force u32)cpu_to_le32(address),
-			   (int *)&ret);
+	value = sdio_readl(func1, (__force u32)cpu_to_le32(address), &ret);
 	if (ret)
 		return ret;
 
diff --git a/drivers/net/wireless/morsemicro/mm81x/skbq.c b/drivers/net/wireless/morsemicro/mm81x/skbq.c
index 25655bd56d140b..3bf15e7c38d803 100644
--- a/drivers/net/wireless/morsemicro/mm81x/skbq.c
+++ b/drivers/net/wireless/morsemicro/mm81x/skbq.c
@@ -231,10 +231,9 @@ static bool mm81x_tx_h_is_ps_filtered(struct mm81x_skbq *mq,
 	WARN_ON_ONCE(!(le32_to_cpu(tx_sts->flags) &
 		       MM81X_TX_STATUS_FLAGS_PS_FILTERED));
 
-	if (vif->type == NL80211_IFTYPE_AP) {
-		__mm81x_skbq_drop_pending_skb(mq, skb);
-		return true;
-	}
+	/* mac80211 buffers frames in AP mode */
+	if (vif->type == NL80211_IFTYPE_AP)
+		return false;
 
 	if (vif->type == NL80211_IFTYPE_STATION) {
 		mm81x_skbq_insert_pending(mq, skb, tx_sts->pkt_id);
diff --git a/drivers/net/wireless/nxp/nxpwifi/cfg80211.c b/drivers/net/wireless/nxp/nxpwifi/cfg80211.c
index 86304ea3331c74..8805439c0b4b48 100644
--- a/drivers/net/wireless/nxp/nxpwifi/cfg80211.c
+++ b/drivers/net/wireless/nxp/nxpwifi/cfg80211.c
@@ -99,12 +99,14 @@ static void *nxpwifi_cfg80211_get_adapter(struct wiphy *wiphy)
 /* cfg80211 operation handler to delete a network key. */
 static int
 nxpwifi_cfg80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			 int link_id, u8 key_index, bool pairwise,
+			 int link_id, u8 key_index,
+			 enum nl80211_key_type type,
 			 const u8 *mac_addr)
 {
 	struct nxpwifi_private *priv = nxpwifi_netdev_get_priv(wdev->netdev);
 	static const u8 bc_mac[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-	const u8 *peer_mac = pairwise ? mac_addr : bc_mac;
+	const u8 *peer_mac =
+		(type == NL80211_KEYTYPE_PAIRWISE) ? mac_addr : bc_mac;
 	int ret;
 
 	ret = nxpwifi_set_encode(priv, NULL, NULL, 0, key_index, peer_mac, 1);
@@ -426,7 +428,8 @@ nxpwifi_cfg80211_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
 /* cfg80211 handler for adding an 802.11 encryption key. */
 static int
 nxpwifi_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			 int link_id, u8 key_index, bool pairwise,
+			 int link_id, u8 key_index,
+			 enum nl80211_key_type type,
 			 const u8 *mac_addr, struct key_params *params)
 {
 	struct nxpwifi_private *priv = nxpwifi_netdev_get_priv(wdev->netdev);
@@ -436,7 +439,7 @@ nxpwifi_cfg80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 	int ret;
 
 	eth_broadcast_addr(bc_mac);
-	peer_mac = pairwise ? mac_addr : bc_mac;
+	peer_mac = (type == NL80211_KEYTYPE_PAIRWISE) ? mac_addr : bc_mac;
 
 	if (GET_BSS_ROLE(priv) == NXPWIFI_BSS_ROLE_UAP &&
 	    (params->cipher == WLAN_CIPHER_SUITE_WEP40 ||
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
index 45e2b7ae963332..d9c84d5ce2548b 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
@@ -531,10 +531,12 @@ qtnf_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev,
 }
 
 static int qtnf_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			int link_id, u8 key_index, bool pairwise,
+			int link_id, u8 key_index,
+			enum nl80211_key_type type,
 			const u8 *mac_addr, struct key_params *params)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	int ret;
 
 	ret = qtnf_cmd_send_add_key(vif, key_index, pairwise, mac_addr, params);
@@ -547,10 +549,12 @@ static int qtnf_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 }
 
 static int qtnf_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			int link_id, u8 key_index, bool pairwise,
+			int link_id, u8 key_index,
+			enum nl80211_key_type type,
 			const u8 *mac_addr)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev);
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	int ret;
 
 	ret = qtnf_cmd_send_del_key(vif, key_index, pairwise, mac_addr);
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00pci.c b/drivers/net/wireless/ralink/rt2x00/rt2x00pci.c
index cabeef0dde45d5..d4635b5314818e 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00pci.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00pci.c
@@ -19,84 +19,50 @@
 #include "rt2x00.h"
 #include "rt2x00pci.h"
 
-/*
- * PCI driver handlers.
- */
-static void rt2x00pci_free_reg(struct rt2x00_dev *rt2x00dev)
-{
-	kfree(rt2x00dev->rf);
-	rt2x00dev->rf = NULL;
-
-	kfree(rt2x00dev->eeprom);
-	rt2x00dev->eeprom = NULL;
-
-	if (rt2x00dev->csr.base) {
-		iounmap(rt2x00dev->csr.base);
-		rt2x00dev->csr.base = NULL;
-	}
-}
-
-static int rt2x00pci_alloc_reg(struct rt2x00_dev *rt2x00dev)
-{
-	struct pci_dev *pci_dev = to_pci_dev(rt2x00dev->dev);
-
-	rt2x00dev->csr.base = pci_ioremap_bar(pci_dev, 0);
-	if (!rt2x00dev->csr.base)
-		goto exit;
-
-	rt2x00dev->eeprom = kzalloc(rt2x00dev->ops->eeprom_size, GFP_KERNEL);
-	if (!rt2x00dev->eeprom)
-		goto exit;
-
-	rt2x00dev->rf = kzalloc(rt2x00dev->ops->rf_size, GFP_KERNEL);
-	if (!rt2x00dev->rf)
-		goto exit;
-
-	return 0;
-
-exit:
-	rt2x00_probe_err("Failed to allocate registers\n");
-
-	rt2x00pci_free_reg(rt2x00dev);
-
-	return -ENOMEM;
-}
-
 int rt2x00pci_probe(struct pci_dev *pci_dev, const struct rt2x00_ops *ops)
 {
-	struct ieee80211_hw *hw;
 	struct rt2x00_dev *rt2x00dev;
+	struct ieee80211_hw *hw;
+	void __iomem *base;
+	__le16 *eeprom;
 	int retval;
 	u16 chip;
+	u32 *rf;
 
-	retval = pci_enable_device(pci_dev);
+	retval = pcim_enable_device(pci_dev);
 	if (retval) {
 		rt2x00_probe_err("Enable device failed\n");
 		return retval;
 	}
 
-	retval = pci_request_regions(pci_dev, pci_name(pci_dev));
-	if (retval) {
-		rt2x00_probe_err("PCI request regions failed\n");
-		goto exit_disable_device;
+	base = pcim_iomap_region(pci_dev, 0, pci_name(pci_dev));
+	if (IS_ERR(base)) {
+		rt2x00_probe_err("PCI iomap region failed\n");
+		return PTR_ERR(base);
 	}
 
 	pci_set_master(pci_dev);
 
-	if (pci_set_mwi(pci_dev))
+	if (pcim_set_mwi(pci_dev))
 		rt2x00_probe_err("MWI not available\n");
 
 	if (dma_set_mask(&pci_dev->dev, DMA_BIT_MASK(32))) {
 		rt2x00_probe_err("PCI DMA not supported\n");
-		retval = -EIO;
-		goto exit_release_regions;
+		return -EIO;
 	}
 
+	eeprom = devm_kzalloc(&pci_dev->dev, ops->eeprom_size, GFP_KERNEL);
+	if (!eeprom)
+		return -ENOMEM;
+
+	rf = devm_kzalloc(&pci_dev->dev, ops->rf_size, GFP_KERNEL);
+	if (!rf)
+		return -ENOMEM;
+
 	hw = ieee80211_alloc_hw(sizeof(struct rt2x00_dev), ops->hw);
 	if (!hw) {
 		rt2x00_probe_err("Failed to allocate hardware\n");
-		retval = -ENOMEM;
-		goto exit_release_regions;
+		return -ENOMEM;
 	}
 
 	pci_set_drvdata(pci_dev, hw);
@@ -105,18 +71,17 @@ int rt2x00pci_probe(struct pci_dev *pci_dev, const struct rt2x00_ops *ops)
 	rt2x00dev->dev = &pci_dev->dev;
 	rt2x00dev->ops = ops;
 	rt2x00dev->hw = hw;
+	rt2x00dev->csr.base = base;
 	rt2x00dev->irq = pci_dev->irq;
 	rt2x00dev->name = ops->name;
+	rt2x00dev->eeprom = eeprom;
+	rt2x00dev->rf = rf;
 
 	if (pci_is_pcie(pci_dev))
 		rt2x00_set_chip_intf(rt2x00dev, RT2X00_CHIP_INTF_PCIE);
 	else
 		rt2x00_set_chip_intf(rt2x00dev, RT2X00_CHIP_INTF_PCI);
 
-	retval = rt2x00pci_alloc_reg(rt2x00dev);
-	if (retval)
-		goto exit_free_device;
-
 	/*
 	 * Because rt3290 chip use different efuse offset to read efuse data.
 	 * So before read efuse it need to indicate it is the
@@ -127,23 +92,13 @@ int rt2x00pci_probe(struct pci_dev *pci_dev, const struct rt2x00_ops *ops)
 
 	retval = rt2x00lib_probe_dev(rt2x00dev);
 	if (retval)
-		goto exit_free_reg;
+		goto exit_free_device;
 
 	return 0;
 
-exit_free_reg:
-	rt2x00pci_free_reg(rt2x00dev);
-
 exit_free_device:
 	ieee80211_free_hw(hw);
 
-exit_release_regions:
-	pci_clear_mwi(pci_dev);
-	pci_release_regions(pci_dev);
-
-exit_disable_device:
-	pci_disable_device(pci_dev);
-
 	return retval;
 }
 EXPORT_SYMBOL_GPL(rt2x00pci_probe);
@@ -153,19 +108,8 @@ void rt2x00pci_remove(struct pci_dev *pci_dev)
 	struct ieee80211_hw *hw = pci_get_drvdata(pci_dev);
 	struct rt2x00_dev *rt2x00dev = hw->priv;
 
-	/*
-	 * Free all allocated data.
-	 */
 	rt2x00lib_remove_dev(rt2x00dev);
-	rt2x00pci_free_reg(rt2x00dev);
 	ieee80211_free_hw(hw);
-
-	/*
-	 * Free the PCI device data.
-	 */
-	pci_clear_mwi(pci_dev);
-	pci_disable_device(pci_dev);
-	pci_release_regions(pci_dev);
 }
 EXPORT_SYMBOL_GPL(rt2x00pci_remove);
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
index 47e427ea86221a..c9fbc3710fd8d4 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
@@ -755,55 +755,31 @@ void rt2x00usb_uninitialize(struct rt2x00_dev *rt2x00dev)
 }
 EXPORT_SYMBOL_GPL(rt2x00usb_uninitialize);
 
-/*
- * USB driver handlers.
- */
-static void rt2x00usb_free_reg(struct rt2x00_dev *rt2x00dev)
-{
-	kfree(rt2x00dev->rf);
-	rt2x00dev->rf = NULL;
-
-	kfree(rt2x00dev->eeprom);
-	rt2x00dev->eeprom = NULL;
-
-	kfree(rt2x00dev->csr.cache);
-	rt2x00dev->csr.cache = NULL;
-}
-
-static int rt2x00usb_alloc_reg(struct rt2x00_dev *rt2x00dev)
-{
-	rt2x00dev->csr.cache = kzalloc(CSR_CACHE_SIZE, GFP_KERNEL);
-	if (!rt2x00dev->csr.cache)
-		goto exit;
-
-	rt2x00dev->eeprom = kzalloc(rt2x00dev->ops->eeprom_size, GFP_KERNEL);
-	if (!rt2x00dev->eeprom)
-		goto exit;
-
-	rt2x00dev->rf = kzalloc(rt2x00dev->ops->rf_size, GFP_KERNEL);
-	if (!rt2x00dev->rf)
-		goto exit;
-
-	return 0;
-
-exit:
-	rt2x00_probe_err("Failed to allocate registers\n");
-
-	rt2x00usb_free_reg(rt2x00dev);
-
-	return -ENOMEM;
-}
-
 int rt2x00usb_probe(struct usb_interface *usb_intf,
 		    const struct rt2x00_ops *ops)
 {
 	struct usb_device *usb_dev = interface_to_usbdev(usb_intf);
-	struct ieee80211_hw *hw;
 	struct rt2x00_dev *rt2x00dev;
+	struct ieee80211_hw *hw;
+	__le16 *eeprom;
+	void *cache;
 	int retval;
+	u32 *rf;
 
 	usb_reset_device(usb_dev);
 
+	cache = devm_kzalloc(&usb_intf->dev, CSR_CACHE_SIZE, GFP_KERNEL);
+	if (!cache)
+		return -ENOMEM;
+
+	eeprom = devm_kzalloc(&usb_intf->dev, ops->eeprom_size, GFP_KERNEL);
+	if (!eeprom)
+		return -ENOMEM;
+
+	rf = devm_kzalloc(&usb_intf->dev, ops->rf_size, GFP_KERNEL);
+	if (!rf)
+		return -ENOMEM;
+
 	hw = ieee80211_alloc_hw(struct_size(rt2x00dev, anchor, 1), ops->hw);
 	if (!hw) {
 		rt2x00_probe_err("Failed to allocate hardware\n");
@@ -816,16 +792,15 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 	rt2x00dev->dev = &usb_intf->dev;
 	rt2x00dev->ops = ops;
 	rt2x00dev->hw = hw;
+	rt2x00dev->csr.cache = cache;
+	rt2x00dev->eeprom = eeprom;
+	rt2x00dev->rf = rf;
 
 	rt2x00_set_chip_intf(rt2x00dev, RT2X00_CHIP_INTF_USB);
 
 	INIT_WORK(&rt2x00dev->rxdone_work, rt2x00usb_work_rxdone);
 	INIT_WORK(&rt2x00dev->txdone_work, rt2x00usb_work_txdone);
 
-	retval = rt2x00usb_alloc_reg(rt2x00dev);
-	if (retval)
-		goto exit_free_device;
-
 	init_usb_anchor(rt2x00dev->anchor);
 
 	retval = rt2x00lib_probe_dev(rt2x00dev);
@@ -836,9 +811,6 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 
 exit_free_anchor:
 	usb_kill_anchored_urbs(rt2x00dev->anchor);
-	rt2x00usb_free_reg(rt2x00dev);
-
-exit_free_device:
 	ieee80211_free_hw(hw);
 	usb_set_intfdata(usb_intf, NULL);
 
@@ -855,7 +827,6 @@ void rt2x00usb_disconnect(struct usb_interface *usb_intf)
 	 * Free all allocated data.
 	 */
 	rt2x00lib_remove_dev(rt2x00dev);
-	rt2x00usb_free_reg(rt2x00dev);
 	ieee80211_free_hw(hw);
 
 	usb_set_intfdata(usb_intf, NULL);
diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c
index b67d69b01f879b..3a10f1bbe0e0c7 100644
--- a/drivers/net/wireless/realtek/rtw88/debug.c
+++ b/drivers/net/wireless/realtek/rtw88/debug.c
@@ -1051,10 +1051,7 @@ static ssize_t rtw_debugfs_set_force_lowest_basic_rate(struct file *filp,
 	if (err)
 		return err;
 
-	if (input)
-		set_bit(RTW_FLAG_FORCE_LOWEST_RATE, rtwdev->flags);
-	else
-		clear_bit(RTW_FLAG_FORCE_LOWEST_RATE, rtwdev->flags);
+	assign_bit(RTW_FLAG_FORCE_LOWEST_RATE, rtwdev->flags, input);
 
 	return count;
 }
diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c
index 397ebbfcac09af..dacc2d889a4677 100644
--- a/drivers/net/wireless/realtek/rtw89/core.c
+++ b/drivers/net/wireless/realtek/rtw89/core.c
@@ -3073,10 +3073,8 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 	struct rtw89_vif *rtwvif = vif_to_rtwvif(vif);
 	struct rtw89_rx_desc_info *desc_info = iter_data->desc_info;
 	struct sk_buff *skb = iter_data->skb;
-	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct rtw89_rx_phy_ppdu *phy_ppdu = iter_data->phy_ppdu;
-	bool is_mld = ieee80211_vif_is_mld(vif);
 	struct ieee80211_bss_conf *bss_conf;
 	struct rtw89_vif_link *rtwvif_link;
 	const u8 *bssid = iter_data->bssid;
@@ -3110,11 +3108,6 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac,
 	if (!ether_addr_equal(target_bssid, bssid))
 		goto out;
 
-	if (is_mld) {
-		rx_status->link_valid = true;
-		rx_status->link_id = rtwvif_link->link_id;
-	}
-
 	bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx);
 	pkt_stat = &bb->cur_pkt_stat;
 
@@ -7372,10 +7365,7 @@ void rtw89_core_rfkill_poll(struct rtw89_dev *rtwdev, bool force)
 	rtw89_info(rtwdev, "rfkill hardware state changed to %s\n",
 		   str_enable_disable(!blocked));
 
-	if (blocked)
-		set_bit(RTW89_FLAG_HW_RFKILL_STATE, rtwdev->flags);
-	else
-		clear_bit(RTW89_FLAG_HW_RFKILL_STATE, rtwdev->flags);
+	assign_bit(RTW89_FLAG_HW_RFKILL_STATE, rtwdev->flags, blocked);
 
 	wiphy_rfkill_set_hw_state(rtwdev->hw->wiphy, blocked);
 }
diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c
index df396fbfca2684..68da8908f14cfb 100644
--- a/drivers/net/wireless/realtek/rtw89/mac.c
+++ b/drivers/net/wireless/realtek/rtw89/mac.c
@@ -6046,10 +6046,7 @@ rtw89_mac_c2h_pwr_int_notify(struct rtw89_dev *rtwdev, struct sk_buff *skb, u32
 		goto out;
 
 	rtwsta = rtwsta_link->rtwsta;
-	if (ps)
-		set_bit(RTW89_REMOTE_STA_IN_PS, rtwsta->flags);
-	else
-		clear_bit(RTW89_REMOTE_STA_IN_PS, rtwsta->flags);
+	assign_bit(RTW89_REMOTE_STA_IN_PS, rtwsta->flags, ps);
 
 	sta = rtwsta_to_sta(rtwsta);
 	ieee80211_sta_ps_transition(sta, ps);
@@ -7004,10 +7001,7 @@ void rtw89_mac_bf_monitor_calc(struct rtw89_dev *rtwdev,
 					  &data);
 
 	rtw89_debug(rtwdev, RTW89_DBG_BF, "bfee STA count=%d\n", data.count);
-	if (data.count)
-		set_bit(RTW89_FLAG_BFEE_MON, rtwdev->flags);
-	else
-		clear_bit(RTW89_FLAG_BFEE_MON, rtwdev->flags);
+	assign_bit(RTW89_FLAG_BFEE_MON, rtwdev->flags, data.count);
 }
 
 void _rtw89_mac_bf_monitor_track(struct rtw89_dev *rtwdev)
diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c
index 14fab6b1a74a75..b9a5ce218b1daa 100644
--- a/drivers/net/wireless/realtek/rtw89/regd.c
+++ b/drivers/net/wireless/realtek/rtw89/regd.c
@@ -421,10 +421,7 @@ static void __rtw89_regd_setup_policy_6ghz(struct rtw89_dev *rtwdev, bool block,
 		return;
 	}
 
-	if (block)
-		set_bit(index, regulatory->block_6ghz);
-	else
-		clear_bit(index, regulatory->block_6ghz);
+	assign_bit(index, regulatory->block_6ghz, block);
 }
 
 static void rtw89_regd_setup_policy_6ghz(struct rtw89_dev *rtwdev)
diff --git a/drivers/net/wireless/realtek/rtw89/wow.c b/drivers/net/wireless/realtek/rtw89/wow.c
index 2dfa24c54e7d72..2e37e1fafbef5b 100644
--- a/drivers/net/wireless/realtek/rtw89/wow.c
+++ b/drivers/net/wireless/realtek/rtw89/wow.c
@@ -669,10 +669,12 @@ static struct ieee80211_key_conf *rtw89_wow_gtk_rekey(struct rtw89_dev *rtwdev,
 	if (ieee80211_vif_is_mld(wow_vif))
 		key = ieee80211_gtk_rekey_add(wow_vif, keyidx, gtk,
 					      cipher_info->len,
-					      rtwvif_link->link_id);
+					      rtwvif_link->link_id,
+					      false);
 	else
 		key = ieee80211_gtk_rekey_add(wow_vif, keyidx, gtk,
-					      cipher_info->len, -1);
+					      cipher_info->len, -1,
+					      false);
 
 	kfree(rekey_conf);
 	if (IS_ERR(key)) {
diff --git a/drivers/net/wireless/silabs/wfx/bh.c b/drivers/net/wireless/silabs/wfx/bh.c
index 21dfdcf9cc2734..0071659bc9c98f 100644
--- a/drivers/net/wireless/silabs/wfx/bh.c
+++ b/drivers/net/wireless/silabs/wfx/bh.c
@@ -284,10 +284,10 @@ void wfx_bh_request_tx(struct wfx_dev *wdev)
 }
 
 /* If IRQ is not available, this function allow to manually poll the control register and simulate
- * an IRQ ahen an event happened.
+ * an IRQ when an event happened.
  *
- * Note that the device has a bug: If an IRQ raise while host read control register, the IRQ is
- * lost. So, use this function carefully (only duing device initialisation).
+ * Note that the device has a bug: If an IRQ raise while host read control register, the device is
+ * lost. Unfortunately, this is the only way to initialize the SDIO with out-of-band IRQ.
  */
 void wfx_bh_poll_irq(struct wfx_dev *wdev)
 {
diff --git a/drivers/net/wireless/silabs/wfx/bus_sdio.c b/drivers/net/wireless/silabs/wfx/bus_sdio.c
index ab0793b9908f42..1ca2cc262feb21 100644
--- a/drivers/net/wireless/silabs/wfx/bus_sdio.c
+++ b/drivers/net/wireless/silabs/wfx/bus_sdio.c
@@ -279,6 +279,7 @@ static int wfx_sdio_probe(struct sdio_func *func, const struct sdio_device_id *i
 		goto sdio_release;
 	}
 
+	bus->core->poll_irq = !!bus->of_irq;
 	ret = wfx_probe(bus->core);
 	if (ret)
 		goto sdio_release;
diff --git a/drivers/net/wireless/silabs/wfx/hif_tx.c b/drivers/net/wireless/silabs/wfx/hif_tx.c
index 9f403d275cb136..9efb108bd877df 100644
--- a/drivers/net/wireless/silabs/wfx/hif_tx.c
+++ b/drivers/net/wireless/silabs/wfx/hif_tx.c
@@ -207,9 +207,11 @@ int wfx_hif_read_mib(struct wfx_dev *wdev, int vif_id, u16 mib_id, void *val, si
 		dev_warn(wdev->dev, "%s: confirmation mismatch request\n", __func__);
 		ret = -EIO;
 	}
-	if (ret == -ENOMEM)
+	if (!ret && le16_to_cpu(reply->length) > val_len) {
 		dev_err(wdev->dev, "buffer is too small to receive %s (%zu < %d)\n",
 			wfx_get_mib_name(mib_id), val_len, le16_to_cpu(reply->length));
+		ret = -EIO;
+	}
 	if (!ret)
 		memcpy(val, &reply->mib_data, le16_to_cpu(reply->length));
 	else
diff --git a/drivers/net/wireless/silabs/wfx/main.c b/drivers/net/wireless/silabs/wfx/main.c
index dda36e41eed135..16d5b51f6f03a9 100644
--- a/drivers/net/wireless/silabs/wfx/main.c
+++ b/drivers/net/wireless/silabs/wfx/main.c
@@ -362,7 +362,6 @@ int wfx_probe(struct wfx_dev *wdev)
 	 */
 	gpio_saved = wdev->pdata.gpio_wakeup;
 	wdev->pdata.gpio_wakeup = NULL;
-	wdev->poll_irq = true;
 
 	wdev->bh_wq = alloc_workqueue("wfx_bh_wq", WQ_HIGHPRI | WQ_PERCPU, 0);
 	if (!wdev->bh_wq)
@@ -370,16 +369,24 @@ int wfx_probe(struct wfx_dev *wdev)
 
 	wfx_bh_register(wdev);
 
+	if (!wdev->poll_irq) {
+		err = wdev->hwbus_ops->irq_subscribe(wdev->hwbus_priv);
+		if (err)
+			goto bh_unregister;
+	}
+
 	err = wfx_init_device(wdev);
 	if (err)
-		goto bh_unregister;
+		goto irq_unsubscribe;
+
+	if (wdev->poll_irq)
+		wfx_bh_poll_irq(wdev);
 
-	wfx_bh_poll_irq(wdev);
 	err = wait_for_completion_timeout(&wdev->firmware_ready, 1 * HZ);
 	if (err == 0) {
 		dev_err(wdev->dev, "timeout while waiting for startup indication\n");
 		err = -ETIMEDOUT;
-		goto bh_unregister;
+		goto irq_unsubscribe;
 	}
 
 	/* FIXME: fill wiphy::hw_version */
@@ -399,12 +406,13 @@ int wfx_probe(struct wfx_dev *wdev)
 		dev_err(wdev->dev, "unsupported firmware API version (expect 1 while firmware returns %d)\n",
 			wdev->hw_caps.api_version_major);
 		err = -EOPNOTSUPP;
-		goto bh_unregister;
+		goto irq_unsubscribe;
 	}
 
 	if (wdev->hw_caps.link_mode == SEC_LINK_ENFORCED) {
 		dev_err(wdev->dev, "chip require secure_link, but can't negotiate it\n");
-		goto bh_unregister;
+		err = -EOPNOTSUPP;
+		goto irq_unsubscribe;
 	}
 
 	if (wdev->hw_caps.region_sel_mode) {
@@ -420,12 +428,14 @@ int wfx_probe(struct wfx_dev *wdev)
 	dev_dbg(wdev->dev, "sending configuration file %s\n", wdev->pdata.file_pds);
 	err = wfx_send_pdata_pds(wdev);
 	if (err < 0 && err != -ENOENT)
-		goto bh_unregister;
+		goto irq_unsubscribe;
 
-	wdev->poll_irq = false;
-	err = wdev->hwbus_ops->irq_subscribe(wdev->hwbus_priv);
-	if (err)
-		goto bh_unregister;
+	if (wdev->poll_irq) {
+		err = wdev->hwbus_ops->irq_subscribe(wdev->hwbus_priv);
+		if (err)
+			goto bh_unregister;
+		wdev->poll_irq = false;
+	}
 
 	err = wfx_hif_use_multi_tx_conf(wdev, true);
 	if (err)
@@ -474,9 +484,11 @@ int wfx_probe(struct wfx_dev *wdev)
 ieee80211_unregister:
 	ieee80211_unregister_hw(wdev->hw);
 irq_unsubscribe:
-	wdev->hwbus_ops->irq_unsubscribe(wdev->hwbus_priv);
+	if (!wdev->poll_irq)
+		wdev->hwbus_ops->irq_unsubscribe(wdev->hwbus_priv);
 bh_unregister:
 	wfx_bh_unregister(wdev);
+	cancel_delayed_work_sync(&wdev->cooling_timeout_work);
 	destroy_workqueue(wdev->bh_wq);
 	return err;
 }
@@ -487,6 +499,7 @@ void wfx_release(struct wfx_dev *wdev)
 	wfx_hif_shutdown(wdev);
 	wdev->hwbus_ops->irq_unsubscribe(wdev->hwbus_priv);
 	wfx_bh_unregister(wdev);
+	cancel_delayed_work_sync(&wdev->cooling_timeout_work);
 	destroy_workqueue(wdev->bh_wq);
 }
 
diff --git a/drivers/net/wireless/st/cw1200/bh.c b/drivers/net/wireless/st/cw1200/bh.c
index b034bab4b48979..9367a981f1dee9 100644
--- a/drivers/net/wireless/st/cw1200/bh.c
+++ b/drivers/net/wireless/st/cw1200/bh.c
@@ -344,6 +344,7 @@ static int cw1200_bh_tx_helper(struct cw1200_common *priv,
 			       int *tx_burst)
 {
 	size_t tx_len;
+	size_t aligned_len;
 	u8 *data;
 	int ret;
 	struct wsm_hdr *wsm;
@@ -376,9 +377,14 @@ static int cw1200_bh_tx_helper(struct cw1200_common *priv,
 
 	atomic_inc(&priv->bh_tx);
 
-	tx_len = priv->hwbus_ops->align_size(
+	aligned_len = priv->hwbus_ops->align_size(
 		priv->hwbus_priv, tx_len);
 
+	/* Zero the bus alignment padding so no stale data is sent. */
+	if (aligned_len > tx_len)
+		memset(data + tx_len, 0, aligned_len - tx_len);
+	tx_len = aligned_len;
+
 	/* Check if not exceeding CW1200 capabilities */
 	if (WARN_ON_ONCE(tx_len > EFFECTIVE_BUF_SIZE))
 		pr_debug("Write aligned len: %zu\n", tx_len);
diff --git a/drivers/net/wireless/st/cw1200/txrx.c b/drivers/net/wireless/st/cw1200/txrx.c
index 084d52b11f5b03..33b02e562b79e9 100644
--- a/drivers/net/wireless/st/cw1200/txrx.c
+++ b/drivers/net/wireless/st/cw1200/txrx.c
@@ -14,6 +14,7 @@
 #include "wsm.h"
 #include "bh.h"
 #include "sta.h"
+#include "hwbus.h"
 #include "debug.h"
 
 #define CW1200_INVALID_RATE_ID (0xFF)
@@ -716,6 +717,7 @@ void cw1200_tx(struct ieee80211_hw *dev,
 	struct ieee80211_sta *sta;
 	struct wsm_tx *wsm;
 	bool tid_update = false;
+	size_t pad_len;
 	u8 flags = 0;
 	int ret;
 
@@ -764,6 +766,18 @@ void cw1200_tx(struct ieee80211_hw *dev,
 
 	sta = t.sta;
 
+	/*
+	 * The bus transfer is padded up to the bus alignment and the
+	 * padding is transferred from the frame buffer.  Make sure the
+	 * buffer has room for the padding so the device never receives
+	 * data past the end of the frame.
+	 */
+	pad_len = priv->hwbus_ops->align_size(priv->hwbus_priv, skb->len) -
+		skb->len;
+	if (skb_tailroom(skb) < pad_len &&
+	    pskb_expand_head(skb, 0, pad_len, GFP_ATOMIC))
+		goto drop;
+
 	spin_lock_bh(&priv->ps_state_lock);
 	{
 		tid_update = cw1200_tx_h_pm_state(priv, &t);
@@ -1036,7 +1050,7 @@ void cw1200_rx_cb(struct cw1200_common *priv,
 		schedule_work(&priv->linkid_reset_work);
 	}
 
-	if (link_id && p2p &&
+	if (link_id && link_id <= CW1200_MAX_STA_IN_AP_MODE && p2p &&
 	    ieee80211_is_action(frame->frame_control) &&
 	    (mgmt->u.action.category == WLAN_CATEGORY_PUBLIC)) {
 		/* Link ID already exists for the ACTION frame.
diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c
index edf6ca23c6c35f..26aabf4ea12785 100644
--- a/drivers/net/wireless/ti/wlcore/main.c
+++ b/drivers/net/wireless/ti/wlcore/main.c
@@ -6809,8 +6809,10 @@ void wlcore_remove(struct platform_device *pdev)
 
 	if (pdev_data->family && pdev_data->family->nvs_name)
 		wait_for_completion(&wl->nvs_loading_complete);
-	if (!wl->initialized)
+	if (!wl->initialized) {
+		pm_runtime_put_noidle(wl->dev);
 		return;
+	}
 
 	if (wl->wakeirq >= 0) {
 		dev_pm_clear_wake_irq(wl->dev);
diff --git a/drivers/net/wireless/ti/wlcore/rx.c b/drivers/net/wireless/ti/wlcore/rx.c
index 307fab21050b74..4d6f5a04778289 100644
--- a/drivers/net/wireless/ti/wlcore/rx.c
+++ b/drivers/net/wireless/ti/wlcore/rx.c
@@ -316,10 +316,7 @@ int wl1271_rx_filter_enable(struct wl1271 *wl,
 		return ret;
 	}
 
-	if (enable)
-		__set_bit(index, wl->rx_filter_enabled);
-	else
-		__clear_bit(index, wl->rx_filter_enabled);
+	__assign_bit(index, wl->rx_filter_enabled, enable);
 
 	return 0;
 }
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.h b/drivers/net/wireless/virtual/mac80211_hwsim.h
index a022cd5c0f1c73..1d2fd06c585a93 100644
--- a/drivers/net/wireless/virtual/mac80211_hwsim.h
+++ b/drivers/net/wireless/virtual/mac80211_hwsim.h
@@ -3,7 +3,7 @@
  * mac80211_hwsim - software simulator of 802.11 radio(s) for mac80211
  * Copyright (c) 2008, Jouni Malinen <j@w1.fi>
  * Copyright (c) 2011, Javier Lopez <jlopex@gmail.com>
- * Copyright (C) 2020, 2022-2025 Intel Corporation
+ * Copyright (C) 2020, 2022-2026 Intel Corporation
  */
 
 #ifndef __MAC80211_HWSIM_H
@@ -162,6 +162,8 @@ enum hwsim_commands {
  *	each radio instead of for the wiphy.
  * @HWSIM_ATTR_SUPPORT_NAN_DEVICE: support NAN Device virtual interface (flag)
  * @HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR: background radar/CAC support (flag)
+ * @HWSIM_ATTR_NO_MONITOR: skip monitor report for this TX status, e.g. in case
+ *	the frame was never reported to any receiver (flag)
  * @__HWSIM_ATTR_MAX: enum limit
  */
 enum hwsim_attrs {
@@ -197,6 +199,7 @@ enum hwsim_attrs {
 	HWSIM_ATTR_MULTI_RADIO,
 	HWSIM_ATTR_SUPPORT_NAN_DEVICE,
 	HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR,
+	HWSIM_ATTR_NO_MONITOR,
 	__HWSIM_ATTR_MAX,
 };
 #define HWSIM_ATTR_MAX (__HWSIM_ATTR_MAX - 1)
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_main.c b/drivers/net/wireless/virtual/mac80211_hwsim_main.c
index b9446577ff4997..762f99eb15e4c8 100644
--- a/drivers/net/wireless/virtual/mac80211_hwsim_main.c
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_main.c
@@ -895,6 +895,7 @@ static const struct nla_policy hwsim_genl_policy[HWSIM_ATTR_MAX + 1] = {
 	[HWSIM_ATTR_MULTI_RADIO] = { .type = NLA_FLAG },
 	[HWSIM_ATTR_SUPPORT_NAN_DEVICE] = { .type = NLA_FLAG },
 	[HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_NO_MONITOR] = { .type = NLA_FLAG },
 };
 
 #if IS_REACHABLE(CONFIG_VIRTIO)
@@ -1337,7 +1338,7 @@ mac80211_hwsim_get_tx_rate(struct ieee80211_hw *hw,
 
 static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw,
 				      struct sk_buff *tx_skb,
-				      struct ieee80211_channel *chan)
+				      u32 freq)
 {
 	struct mac80211_hwsim_data *data = hw->priv;
 	struct sk_buff *skb;
@@ -1369,7 +1370,7 @@ static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw,
 	hdr->rt_tsft = __mac80211_hwsim_get_tsf(data);
 	hdr->rt_flags = 0;
 	hdr->rt_rate = bitrate / 5;
-	hdr->rt_channel = cpu_to_le16(chan->center_freq);
+	hdr->rt_channel = cpu_to_le16(freq);
 	flags = IEEE80211_CHAN_2GHZ;
 	if (txrate && txrate->flags & IEEE80211_RATE_ERP_G)
 		flags |= IEEE80211_CHAN_OFDM;
@@ -1387,8 +1388,7 @@ static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw,
 }
 
 
-static void mac80211_hwsim_monitor_ack(struct ieee80211_channel *chan,
-				       const u8 *addr)
+static void mac80211_hwsim_monitor_ack(u32 freq, const u8 *addr)
 {
 	struct sk_buff *skb;
 	struct hwsim_radiotap_ack_hdr *hdr;
@@ -1410,7 +1410,7 @@ static void mac80211_hwsim_monitor_ack(struct ieee80211_channel *chan,
 					  (1 << IEEE80211_RADIOTAP_CHANNEL));
 	hdr->rt_flags = 0;
 	hdr->pad = 0;
-	hdr->rt_channel = cpu_to_le16(chan->center_freq);
+	hdr->rt_channel = cpu_to_le16(freq);
 	flags = IEEE80211_CHAN_2GHZ;
 	hdr->rt_chbitmask = cpu_to_le16(flags);
 
@@ -1735,6 +1735,9 @@ static void mac80211_hwsim_tx_frame_nl(struct ieee80211_hw *hw,
 	if (nla_put_u64_64bit(skb, HWSIM_ATTR_COOKIE, cookie, HWSIM_ATTR_PAD))
 		goto nla_put_failure;
 
+	/* track the frequency */
+	info->rate_driver_data[1] = (void *)(uintptr_t)channel->center_freq;
+
 	genlmsg_end(skb, msg_head);
 
 	if (hwsim_virtio_enabled) {
@@ -1880,9 +1883,6 @@ static void mac80211_hwsim_rx(struct mac80211_hwsim_data *data,
 				sp->active_links_rx &= ~BIT(link_id);
 			else
 				sp->active_links_rx |= BIT(link_id);
-
-			rx_status->link_valid = true;
-			rx_status->link_id = link_id;
 		}
 		rcu_read_unlock();
 	}
@@ -1912,7 +1912,7 @@ static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw,
 
 	mac80211_hwsim_write_tsf(data, skb, sim_tsf);
 
-	mac80211_hwsim_monitor_rx(hw, skb, chan);
+	mac80211_hwsim_monitor_rx(hw, skb, chan->center_freq);
 
 	memset(&rx_status, 0, sizeof(rx_status));
 	rx_status.flag |= RX_FLAG_MACTIME_START;
@@ -2276,7 +2276,7 @@ static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
 	ack = mac80211_hwsim_tx_frame_no_nl(hw, skb, channel);
 
 	if (ack && skb->len >= 16)
-		mac80211_hwsim_monitor_ack(channel, hdr->addr2);
+		mac80211_hwsim_monitor_ack(channel->center_freq, hdr->addr2);
 
 	ieee80211_tx_info_clear_status(txi);
 
@@ -4401,15 +4401,15 @@ struct hwsim_new_radio_params {
 	bool background_radar;
 };
 
-static void hwsim_mcast_config_msg(struct sk_buff *mcast_skb,
+static void hwsim_mcast_config_msg(struct sk_buff *mcast_skb, struct net *net,
 				   struct genl_info *info)
 {
 	if (info)
 		genl_notify(&hwsim_genl_family, mcast_skb, info,
 			    HWSIM_MCGRP_CONFIG, GFP_KERNEL);
 	else
-		genlmsg_multicast(&hwsim_genl_family, mcast_skb, 0,
-				  HWSIM_MCGRP_CONFIG, GFP_KERNEL);
+		genlmsg_multicast_netns(&hwsim_genl_family, net, mcast_skb, 0,
+					HWSIM_MCGRP_CONFIG, GFP_KERNEL);
 }
 
 static int append_radio_msg(struct sk_buff *skb, int id,
@@ -4493,7 +4493,8 @@ static int append_radio_msg(struct sk_buff *skb, int id,
 	return 0;
 }
 
-static void hwsim_mcast_new_radio(int id, struct genl_info *info,
+static void hwsim_mcast_new_radio(int id, struct net *net,
+				  struct genl_info *info,
 				  struct hwsim_new_radio_params *param)
 {
 	struct sk_buff *mcast_skb;
@@ -4513,7 +4514,7 @@ static void hwsim_mcast_new_radio(int id, struct genl_info *info,
 
 	genlmsg_end(mcast_skb, data);
 
-	hwsim_mcast_config_msg(mcast_skb, info);
+	hwsim_mcast_config_msg(mcast_skb, net, info);
 	return;
 
 out_err:
@@ -5649,6 +5650,14 @@ static void mac80211_hwsim_sband_capab(struct ieee80211_supported_band *sband)
 	 BIT(NL80211_IFTYPE_MESH_POINT) | \
 	 BIT(NL80211_IFTYPE_OCB))
 
+static const u8 iftypes_ext_capa[] = {
+	 [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
+	 [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
+	 [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF |
+	       WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB,
+	 [8] = WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB,
+};
+
 static const u8 iftypes_ext_capa_ap[] = {
 	 [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
 	 [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
@@ -5673,6 +5682,33 @@ static const struct wiphy_iftype_ext_capab mac80211_hwsim_iftypes_ext_capa[] = {
 		.eml_capabilities = IEEE80211_EML_CAP_EMLSR_SUPP |
 				    IEEE80211_EML_CAP_EMLMR_SUPPORT,
 		.mld_capa_and_ops = MAC80211_HWSIM_MLD_CAPA_OPS,
+		/* CIP works as hwsim does not have control frames */
+		.cip_supported = true,
+		.cip_capabilities = 0,
+	},
+	{
+		.iftype = NL80211_IFTYPE_STATION,
+		.extended_capabilities = iftypes_ext_capa,
+		.extended_capabilities_mask = iftypes_ext_capa,
+		.extended_capabilities_len = sizeof(iftypes_ext_capa),
+		.cip_supported = true,
+		.cip_capabilities = 0,
+	},
+	{
+		.iftype = NL80211_IFTYPE_P2P_CLIENT,
+		.extended_capabilities = iftypes_ext_capa,
+		.extended_capabilities_mask = iftypes_ext_capa,
+		.extended_capabilities_len = sizeof(iftypes_ext_capa),
+		.cip_supported = true,
+		.cip_capabilities = 0,
+	},
+	{
+		.iftype = NL80211_IFTYPE_P2P_GO,
+		.extended_capabilities = iftypes_ext_capa,
+		.extended_capabilities_mask = iftypes_ext_capa,
+		.extended_capabilities_len = sizeof(iftypes_ext_capa),
+		.cip_supported = true,
+		.cip_capabilities = 0,
 	},
 };
 
@@ -6195,7 +6231,7 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 	hwsim_radios_generation++;
 	spin_unlock_bh(&hwsim_radio_lock);
 
-	hwsim_mcast_new_radio(idx, info, param);
+	hwsim_mcast_new_radio(idx, wiphy_net(data->hw->wiphy), info, param);
 
 	return idx;
 
@@ -6213,7 +6249,7 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 }
 
 static void hwsim_mcast_del_radio(int id, const char *hwname,
-				  struct genl_info *info)
+				  struct net *net, struct genl_info *info)
 {
 	struct sk_buff *skb;
 	void *data;
@@ -6239,7 +6275,7 @@ static void hwsim_mcast_del_radio(int id, const char *hwname,
 
 	genlmsg_end(skb, data);
 
-	hwsim_mcast_config_msg(skb, info);
+	hwsim_mcast_config_msg(skb, net, info);
 
 	return;
 
@@ -6251,7 +6287,7 @@ static void mac80211_hwsim_del_radio(struct mac80211_hwsim_data *data,
 				     const char *hwname,
 				     struct genl_info *info)
 {
-	hwsim_mcast_del_radio(data->idx, hwname, info);
+	hwsim_mcast_del_radio(data->idx, hwname, wiphy_net(data->hw->wiphy), info);
 	debugfs_remove_recursive(data->debugfs);
 	ieee80211_unregister_hw(data->hw);
 	device_release_driver(data->dev);
@@ -6396,6 +6432,8 @@ static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
 	int i;
 	unsigned long flags;
 	bool found = false;
+	bool no_monitor;
+	u32 freq;
 
 	if (!info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER] ||
 	    !info->attrs[HWSIM_ATTR_FLAGS] ||
@@ -6407,6 +6445,7 @@ static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
 	src = (void *)nla_data(info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]);
 	hwsim_flags = nla_get_u32(info->attrs[HWSIM_ATTR_FLAGS]);
 	ret_skb_cookie = nla_get_u64(info->attrs[HWSIM_ATTR_COOKIE]);
+	no_monitor = nla_get_flag(info->attrs[HWSIM_ATTR_NO_MONITOR]);
 
 	data2 = get_hwsim_data_ref_from_addr(src);
 	if (!data2)
@@ -6441,7 +6480,11 @@ static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
 	if (!found)
 		goto out;
 
-	mac80211_hwsim_monitor_rx(data2->hw, skb, data2->channel);
+	txi = IEEE80211_SKB_CB(skb);
+	freq = (uintptr_t)txi->rate_driver_data[1];
+
+	if (!no_monitor)
+		mac80211_hwsim_monitor_rx(data2->hw, skb, freq);
 
 	/* Tx info received because the frame was broadcasted on user space,
 	 so we get all the necessary info: tx attempts and skb control buff */
@@ -6450,8 +6493,6 @@ static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
 		       info->attrs[HWSIM_ATTR_TX_INFO]);
 
 	/* now send back TX status */
-	txi = IEEE80211_SKB_CB(skb);
-
 	ieee80211_tx_info_clear_status(txi);
 
 	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
@@ -6471,8 +6512,7 @@ static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
 	if (!(hwsim_flags & HWSIM_TX_CTL_NO_ACK) &&
 	   (hwsim_flags & HWSIM_TX_STAT_ACK)) {
 		if (skb->len >= 16)
-			mac80211_hwsim_monitor_ack(data2->channel,
-						   hdr->addr2);
+			mac80211_hwsim_monitor_ack(freq, hdr->addr2);
 		txi->flags |= IEEE80211_TX_STAT_ACK;
 	}
 
@@ -6547,6 +6587,51 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2,
 	 */
 	mutex_lock(&data2->mutex);
 
+	/* look for the skb matching the cookie passed back from user */
+	if (info->attrs[HWSIM_ATTR_COOKIE] &&
+	    info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]) {
+		u64 cookie = nla_get_u64(info->attrs[HWSIM_ATTR_COOKIE]);
+		struct sk_buff *orig_skb, *found = NULL;
+		struct mac80211_hwsim_data *txdata;
+		struct ieee80211_tx_info *txi;
+		const u8 *transmitter;
+		unsigned long flags;
+
+		transmitter = nla_data(info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]);
+		txdata = get_hwsim_data_ref_from_addr(transmitter);
+		if (!txdata || txdata->netgroup != data2->netgroup)
+			goto out_unlock;
+
+		spin_lock_irqsave(&txdata->pending.lock, flags);
+		skb_queue_walk(&txdata->pending, orig_skb) {
+			uintptr_t skb_cookie;
+
+			txi = IEEE80211_SKB_CB(orig_skb);
+			skb_cookie = (uintptr_t)txi->rate_driver_data[0];
+
+			if (skb_cookie == cookie) {
+				found = orig_skb;
+				break;
+			}
+		}
+
+		/* that's weird */
+		if (!found) {
+			spin_unlock_irqrestore(&txdata->pending.lock, flags);
+			goto out_unlock;
+		}
+
+		if (frame_data_len > found->len &&
+		    pskb_expand_head(found, 0, frame_data_len - found->len,
+				     GFP_ATOMIC)) {
+			spin_unlock_irqrestore(&txdata->pending.lock, flags);
+			goto out_unlock;
+		}
+		skb_trim(found, 0);
+		skb_put_data(found, frame_data, frame_data_len);
+		spin_unlock_irqrestore(&txdata->pending.lock, flags);
+	}
+
 	/* check if radio is configured properly */
 
 	if ((data2->idle && !data2->tmp_chan) || !data2->started)
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_nan.c b/drivers/net/wireless/virtual/mac80211_hwsim_nan.c
index 7be64c45babf94..2d9f6d7b92514f 100644
--- a/drivers/net/wireless/virtual/mac80211_hwsim_nan.c
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_nan.c
@@ -906,6 +906,18 @@ mac80211_hwsim_nan_discovery_beacon_timer(struct hrtimer *timer)
 	return HRTIMER_RESTART;
 }
 
+static void
+mac80211_hwsim_nan_sched_update_work(struct wiphy *wiphy,
+				     struct wiphy_work *work)
+{
+	struct mac80211_hwsim_data *data =
+		container_of(work, struct mac80211_hwsim_data,
+			     nan.sched_update_work.work);
+
+	if (data->nan.device_vif)
+		ieee80211_nan_sched_update_done(data->nan.device_vif);
+}
+
 int mac80211_hwsim_nan_start(struct ieee80211_hw *hw,
 			     struct ieee80211_vif *vif,
 			     struct cfg80211_nan_conf *conf)
@@ -922,6 +934,9 @@ int mac80211_hwsim_nan_start(struct ieee80211_hw *hw,
 	data->nan.device_vif = vif;
 	data->nan.bands = conf->bands;
 
+	wiphy_delayed_work_init(&data->nan.sched_update_work,
+				mac80211_hwsim_nan_sched_update_work);
+
 	scoped_guard(spinlock_bh, &data->nan.state_lock) {
 		/* Start in the "scan" phase and stay there for a little bit */
 		data->nan.phase = MAC80211_HWSIM_NAN_PHASE_SCAN;
@@ -960,6 +975,7 @@ int mac80211_hwsim_nan_stop(struct ieee80211_hw *hw,
 	hrtimer_cancel(&data->nan.slot_timer);
 	hrtimer_cancel(&data->nan.resume_txqs_timer);
 	hrtimer_cancel(&data->nan.discovery_beacon_timer);
+	wiphy_delayed_work_cancel(hw->wiphy, &data->nan.sched_update_work);
 	data->nan.device_vif = NULL;
 
 	return 0;
@@ -1296,6 +1312,12 @@ void mac80211_hwsim_nan_local_sched_changed(struct ieee80211_hw *hw,
 	}
 
 	spin_unlock_bh(&data->nan.state_lock);
+
+	/* Simulate firmware completing a deferred schedule update */
+	if (vif->cfg.nan_sched.deferred)
+		wiphy_delayed_work_queue(hw->wiphy,
+					 &data->nan.sched_update_work,
+					 msecs_to_jiffies(10));
 }
 
 int mac80211_hwsim_nan_peer_sched_changed(struct ieee80211_hw *hw,
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_nan.h b/drivers/net/wireless/virtual/mac80211_hwsim_nan.h
index 81e105ac7b8eda..2801a1d8dcf07d 100644
--- a/drivers/net/wireless/virtual/mac80211_hwsim_nan.h
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_nan.h
@@ -59,6 +59,9 @@ struct mac80211_hwsim_nan_data {
 	 * available in that slot (except DW which is implicit).
 	 */
 	struct cfg80211_chan_def local_sched[CFG80211_NAN_SCHED_NUM_TIME_SLOTS];
+
+	/* Simulates the firmware completing a deferred schedule update */
+	struct wiphy_delayed_work sched_update_work;
 };
 
 enum hrtimer_restart
diff --git a/drivers/nfc/microread/microread.c b/drivers/nfc/microread/microread.c
index dfa2490db545c1..2bfafa94e83d6c 100644
--- a/drivers/nfc/microread/microread.c
+++ b/drivers/nfc/microread/microread.c
@@ -251,9 +251,9 @@ static int microread_start_poll(struct nfc_hci_dev *hdev,
 		param[1] |= (1 << 1);
 
 	if ((im_protocols | tm_protocols) & NFC_PROTO_NFC_DEP_MASK) {
-		hdev->gb = nfc_get_local_general_bytes(hdev->ndev,
-						       &hdev->gb_len);
-		if (hdev->gb == NULL || hdev->gb_len == 0) {
+		nfc_get_local_general_bytes(hdev->ndev, hdev->gb,
+					    sizeof(hdev->gb), &hdev->gb_len);
+		if (hdev->gb_len == 0) {
 			im_protocols &= ~NFC_PROTO_NFC_DEP_MASK;
 			tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK;
 		}
diff --git a/drivers/nfc/nfcmrvl/fw_dnld.c b/drivers/nfc/nfcmrvl/fw_dnld.c
index 2b8f401d8fd7a6..8b9d5257320dcc 100644
--- a/drivers/nfc/nfcmrvl/fw_dnld.c
+++ b/drivers/nfc/nfcmrvl/fw_dnld.c
@@ -263,9 +263,14 @@ static int process_state_fw_dnld(struct nfcmrvl_private *priv,
 		 * B8..N: payload
 		 */
 
-		/* Remove NCI HDR */
-		skb_pull(skb, 3);
-		if (skb->data[0] != HELPER_CMD_PACKET_FORMAT || skb->len != 5) {
+		if (skb->len != NCI_DATA_HDR_SIZE + 5) {
+			nfc_err(priv->dev, "bad command");
+			return -EINVAL;
+		}
+
+		/* Remove NCI header */
+		skb_pull(skb, NCI_DATA_HDR_SIZE);
+		if (skb->data[0] != HELPER_CMD_PACKET_FORMAT) {
 			nfc_err(priv->dev, "bad command");
 			return -EINVAL;
 		}
diff --git a/drivers/nfc/pn533/pn533.c b/drivers/nfc/pn533/pn533.c
index f5a6a7c20d5af3..b0133e51dce963 100644
--- a/drivers/nfc/pn533/pn533.c
+++ b/drivers/nfc/pn533/pn533.c
@@ -1357,10 +1357,11 @@ static int pn533_poll_dep(struct nfc_dev *nfc_dev)
 	u8 *next, nfcid3[NFC_NFCID3_MAXSIZE];
 	u8 passive_data[PASSIVE_DATA_LEN] = {0x00, 0xff, 0xff, 0x00, 0x3};
 
-	if (!dev->gb) {
-		dev->gb = nfc_get_local_general_bytes(nfc_dev, &dev->gb_len);
-
-		if (!dev->gb || !dev->gb_len) {
+	if (!dev->gb_len) {
+		nfc_get_local_general_bytes(nfc_dev, dev->gb,
+					    sizeof(dev->gb),
+					    &dev->gb_len);
+		if (!dev->gb_len) {
 			dev->poll_dep = 0;
 			queue_work(dev->wq, &dev->rf_work);
 		}
@@ -1658,8 +1659,9 @@ static int pn533_start_poll(struct nfc_dev *nfc_dev,
 	}
 
 	if (tm_protocols) {
-		dev->gb = nfc_get_local_general_bytes(nfc_dev, &dev->gb_len);
-		if (dev->gb == NULL)
+		nfc_get_local_general_bytes(nfc_dev, dev->gb,
+					    sizeof(dev->gb), &dev->gb_len);
+		if (dev->gb_len == 0)
 			tm_protocols = 0;
 	}
 
diff --git a/drivers/nfc/pn533/pn533.h b/drivers/nfc/pn533/pn533.h
index 09e35b8693f5a6..5ab668e05121ae 100644
--- a/drivers/nfc/pn533/pn533.h
+++ b/drivers/nfc/pn533/pn533.h
@@ -6,6 +6,8 @@
  * Copyright (C) 2012-2013 Tieto Poland
  */
 
+#include <net/nfc/nfc.h>
+
 #define PN533_DEVICE_STD		0x1
 #define PN533_DEVICE_PASORI		0x2
 #define PN533_DEVICE_ACR122U		0x3
@@ -166,7 +168,7 @@ struct pn533 {
 	struct timer_list listen_timer;
 	int cancel_listen;
 
-	u8 *gb;
+	u8 gb[NFC_MAX_GT_LEN];
 	size_t gb_len;
 
 	u8 tgt_available_prots;
diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c
index efb07f944fce29..972eaac09e5924 100644
--- a/drivers/nfc/pn533/usb.c
+++ b/drivers/nfc/pn533/usb.c
@@ -319,7 +319,9 @@ static bool pn533_acr122_is_rx_frame_valid(void *_frame, struct pn533 *dev)
 	if (frame->ccid.type != 0x83)
 		return false;
 
-	if (!frame->ccid.datalen)
+	if (frame->ccid.datalen < 2 ||
+	    frame->ccid.datalen > PN533_ACR122_FRAME_MAX_PAYLOAD_LEN +
+	    PN533_ACR122_RX_FRAME_TAIL_LEN)
 		return false;
 
 	if (frame->data[frame->ccid.datalen - 2] == 0x63)
diff --git a/drivers/nfc/pn544/pn544.c b/drivers/nfc/pn544/pn544.c
index 9d0a16ac465e3d..c4fa70e45c14fb 100644
--- a/drivers/nfc/pn544/pn544.c
+++ b/drivers/nfc/pn544/pn544.c
@@ -377,10 +377,9 @@ static int pn544_hci_start_poll(struct nfc_hci_dev *hdev,
 		return r;
 
 	if ((im_protocols | tm_protocols) & NFC_PROTO_NFC_DEP_MASK) {
-		hdev->gb = nfc_get_local_general_bytes(hdev->ndev,
-							&hdev->gb_len);
-		pr_debug("generate local bytes %p\n", hdev->gb);
-		if (hdev->gb == NULL || hdev->gb_len == 0) {
+		nfc_get_local_general_bytes(hdev->ndev, hdev->gb,
+					    sizeof(hdev->gb), &hdev->gb_len);
+		if (hdev->gb_len == 0) {
 			im_protocols &= ~NFC_PROTO_NFC_DEP_MASK;
 			tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK;
 		}
diff --git a/drivers/nfc/port100.c b/drivers/nfc/port100.c
index b613f5e2fd57a4..e769a30b8b5c7e 100644
--- a/drivers/nfc/port100.c
+++ b/drivers/nfc/port100.c
@@ -636,6 +636,13 @@ static void port100_recv_response(struct urb *urb)
 
 	in_frame = dev->in_urb->transfer_buffer;
 
+	if (urb->actual_length < PORT100_FRAME_HEADER_LEN ||
+	    urb->actual_length < port100_rx_frame_size(in_frame)) {
+		nfc_err(&dev->interface->dev, "Received a truncated frame\n");
+		cmd->status = -EIO;
+		goto sched_wq;
+	}
+
 	if (!port100_rx_frame_is_valid(in_frame)) {
 		nfc_err(&dev->interface->dev, "Received an invalid frame\n");
 		cmd->status = -EIO;
diff --git a/drivers/nfc/st21nfca/core.c b/drivers/nfc/st21nfca/core.c
index fd39a05c962221..b5c1ca3acfbe12 100644
--- a/drivers/nfc/st21nfca/core.c
+++ b/drivers/nfc/st21nfca/core.c
@@ -351,10 +351,10 @@ static int st21nfca_hci_start_poll(struct nfc_hci_dev *hdev,
 			if (r < 0)
 				return r;
 		} else {
-			hdev->gb = nfc_get_local_general_bytes(hdev->ndev,
-							       &hdev->gb_len);
-
-			if (hdev->gb == NULL || hdev->gb_len == 0) {
+			nfc_get_local_general_bytes(hdev->ndev, hdev->gb,
+						    sizeof(hdev->gb),
+						    &hdev->gb_len);
+			if (hdev->gb_len == 0) {
 				im_protocols &= ~NFC_PROTO_NFC_DEP_MASK;
 				tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK;
 			}
@@ -577,9 +577,7 @@ static int st21nfca_get_iso15693_inventory(struct nfc_hci_dev *hdev,
 	if (r < 0)
 		goto exit;
 
-	skb_pull(inventory_skb, 2);
-
-	if (inventory_skb->len == 0 ||
+	if (!skb_pull(inventory_skb, 2) || inventory_skb->len < 2 ||
 	    inventory_skb->len > NFC_ISO15693_UID_MAXSIZE) {
 		r = -EPROTO;
 		goto exit;
diff --git a/drivers/nfc/st21nfca/i2c.c b/drivers/nfc/st21nfca/i2c.c
index a4c93ff7c5b0c2..0f44c783bd0417 100644
--- a/drivers/nfc/st21nfca/i2c.c
+++ b/drivers/nfc/st21nfca/i2c.c
@@ -289,27 +289,36 @@ static int check_crc(u8 *buf, int buflen)
  */
 static int st21nfca_hci_i2c_repack(struct sk_buff *skb)
 {
-	int i, j, r, size;
+	int read, write, r, size;
 
-	if (skb->len < 1 || (skb->len > 1 && skb->data[1] != 0))
+	if (skb->len < ST21NFCA_FRAME_HEADROOM ||
+	    !IS_START_OF_FRAME(skb->data))
 		return -EBADMSG;
 
 	size = get_frame_size(skb->data, skb->len);
 	if (size > 0) {
+		if (size < ST21NFCA_FRAME_HEADROOM + 2)
+			return -EBADMSG;
+
 		skb_trim(skb, size);
 		/* remove ST21NFCA byte stuffing for upper layer */
-		for (i = 1, j = 0; i < skb->len; i++) {
-			if (skb->data[i + j] ==
+		for (read = 1, write = 1; read < skb->len;) {
+			if (skb->data[read] ==
 					(u8) ST21NFCA_ESCAPE_BYTE_STUFFING) {
-				skb->data[i] = skb->data[i + j + 1]
-						| ST21NFCA_BYTE_STUFFING_MASK;
-				i++;
-				j++;
+				if (read + 1 == skb->len)
+					return -EBADMSG;
+
+				skb->data[write++] = skb->data[read + 1]
+						     | ST21NFCA_BYTE_STUFFING_MASK;
+				read += 2;
+			} else {
+				skb->data[write++] = skb->data[read++];
 			}
-			skb->data[i] = skb->data[i + j];
 		}
 		/* remove byte stuffing useless byte */
-		skb_trim(skb, i - j);
+		skb_trim(skb, write);
+		if (skb->len < ST21NFCA_FRAME_HEADROOM + 2)
+			return -EBADMSG;
 		/* remove ST21NFCA_SOF_EOF from head */
 		skb_pull(skb, 1);
 
diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c
index 60883001fa5d94..ddfc58c2996495 100644
--- a/drivers/nfc/trf7970a.c
+++ b/drivers/nfc/trf7970a.c
@@ -1997,8 +1997,10 @@ static int trf7970a_startup(struct trf7970a *trf)
 		return ret;
 
 	ret = trf7970a_update_rx_gain_reduction(trf);
-	if (ret)
+	if (ret) {
+		trf7970a_power_down(trf);
 		return ret;
+	}
 
 	pm_runtime_set_active(trf->dev);
 	pm_runtime_enable(trf->dev);
diff --git a/drivers/nfc/virtual_ncidev.c b/drivers/nfc/virtual_ncidev.c
index 8eeb447ac96e9b..e51c647b27ebed 100644
--- a/drivers/nfc/virtual_ncidev.c
+++ b/drivers/nfc/virtual_ncidev.c
@@ -195,7 +195,8 @@ static const struct file_operations virtual_ncidev_fops = {
 	.write = virtual_ncidev_write,
 	.open = virtual_ncidev_open,
 	.release = virtual_ncidev_close,
-	.unlocked_ioctl = virtual_ncidev_ioctl
+	.unlocked_ioctl = virtual_ncidev_ioctl,
+	.compat_ioctl = compat_ptr_ioctl,
 };
 
 static struct miscdevice miscdev = {
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index 30a51c365ce8ba..5fb86595e8bd7f 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -80,7 +80,7 @@ static void pmem_mkpage_present(struct pmem_device *pmem, phys_addr_t offset,
 		 * here since we're in the driver I/O path and
 		 * outstanding I/O requests pin the dev_pagemap.
 		 */
-		if (test_and_clear_pmem_poison(page))
+		if (TestClearPageHWPoison(page))
 			clear_mce_nospec(pfn);
 	}
 }
diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h
index a48509f901968e..76870505dd7971 100644
--- a/drivers/nvdimm/pmem.h
+++ b/drivers/nvdimm/pmem.h
@@ -1,7 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef __NVDIMM_PMEM_H__
 #define __NVDIMM_PMEM_H__
-#include <linux/page-flags.h>
 #include <linux/badblocks.h>
 #include <linux/memremap.h>
 #include <linux/types.h>
@@ -31,15 +30,4 @@ long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
 		long nr_pages, enum dax_access_mode mode, void **kaddr,
 		unsigned long *pfn);
 
-#ifdef CONFIG_MEMORY_FAILURE
-static inline bool test_and_clear_pmem_poison(struct page *page)
-{
-	return TestClearPageHWPoison(page);
-}
-#else
-static inline bool test_and_clear_pmem_poison(struct page *page)
-{
-	return false;
-}
-#endif
 #endif /* __NVDIMM_PMEM_H__ */
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index beea23d04a702b..9bcab3dc4c1184 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -598,7 +598,6 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 		break;
 	case NVME_CTRL_RESETTING:
 		switch (old_state) {
-		case NVME_CTRL_NEW:
 		case NVME_CTRL_LIVE:
 			changed = true;
 			atomic_long_inc(&ctrl->nr_reset);
@@ -3195,9 +3194,16 @@ static void nvme_release_subsystem(struct device *dev)
 {
 	struct nvme_subsystem *subsys =
 		container_of(dev, struct nvme_subsystem, dev);
+	struct nvme_effects_log *cel;
+	unsigned long i;
 
 	if (subsys->instance >= 0)
 		ida_free(&nvme_instance_ida, subsys->instance);
+	xa_for_each(&subsys->cels, i, cel) {
+		xa_erase(&subsys->cels, i);
+		kfree(cel);
+	}
+	xa_destroy(&subsys->cels);
 	kfree(subsys);
 }
 
@@ -3311,6 +3317,7 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 	kref_init(&subsys->ref);
 	INIT_LIST_HEAD(&subsys->ctrls);
 	INIT_LIST_HEAD(&subsys->nsheads);
+	xa_init(&subsys->cels);
 	nvme_init_subnqn(subsys, ctrl, id);
 	memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
 	memcpy(subsys->model, id->mn, sizeof(subsys->model));
@@ -3413,7 +3420,7 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
 				struct nvme_effects_log **log)
 {
-	struct nvme_effects_log *old, *cel = xa_load(&ctrl->cels, csi);
+	struct nvme_effects_log *old, *cel = xa_load(&ctrl->subsys->cels, csi);
 	int ret;
 
 	if (cel)
@@ -3430,7 +3437,7 @@ static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
 		return ret;
 	}
 
-	old = xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
+	old = xa_store(&ctrl->subsys->cels, csi, cel, GFP_KERNEL);
 	if (xa_is_err(old)) {
 		kfree(cel);
 		return xa_err(old);
@@ -3505,7 +3512,7 @@ static int nvme_init_effects_log(struct nvme_ctrl *ctrl,
 	if (!effects)
 		return -ENOMEM;
 
-	old = xa_store(&ctrl->cels, csi, effects, GFP_KERNEL);
+	old = xa_store(&ctrl->subsys->cels, csi, effects, GFP_KERNEL);
 	if (xa_is_err(old)) {
 		kfree(effects);
 		return xa_err(old);
@@ -3551,23 +3558,27 @@ static int nvme_init_effects(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 {
 	int ret = 0;
 
+	mutex_lock(&ctrl->subsys->lock);
+	ctrl->effects = xa_load(&ctrl->subsys->cels, NVME_CSI_NVM);
 	if (ctrl->effects)
-		return 0;
+		goto out_unlock;
 
 	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
 		ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
 		if (ret < 0)
-			return ret;
+			goto out_unlock;
 	}
 
 	if (!ctrl->effects) {
 		ret = nvme_init_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
 		if (ret < 0)
-			return ret;
+			goto out_unlock;
 	}
 
 	nvme_init_known_nvm_effects(ctrl);
-	return 0;
+out_unlock:
+	mutex_unlock(&ctrl->subsys->lock);
+	return ret;
 }
 
 static int nvme_check_ctrl_fabric_info(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
@@ -3931,9 +3942,9 @@ void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
 	put_device(cdev_device);
 }
 
-int nvme_cdev_add(const char *name, struct cdev *cdev,
-		struct device *cdev_device,
-		const struct file_operations *fops, struct module *owner)
+int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
+		const struct file_operations *fops, struct module *owner,
+		int ctrl, int head)
 {
 	int minor, ret;
 
@@ -3941,7 +3952,7 @@ int nvme_cdev_add(const char *name, struct cdev *cdev,
 	if (minor < 0)
 		return minor;
 
-	ret = dev_set_name(cdev_device, name);
+	ret = dev_set_name(cdev_device, "ng%dn%d", ctrl, head);
 	if (ret) {
 		ida_free(&nvme_ns_chr_minor_ida, minor);
 		return ret;
@@ -3982,17 +3993,14 @@ static const struct file_operations nvme_ns_chr_fops = {
 
 static void nvme_add_ns_cdev(struct nvme_ns *ns)
 {
-	char name[32];
-
 	ns->cdev_device.parent = ns->ctrl->device;
-	snprintf(name, sizeof(name), "ng%dn%d", ns->ctrl->instance,
-		 ns->head->instance);
 
 	nvme_get_ns(ns); /* Undone in nvme_cdev_rel() */
-	if (nvme_cdev_add(name, &ns->cdev, &ns->cdev_device,
-			&nvme_ns_chr_fops, ns->ctrl->ops->module)) {
-		dev_err(ns->ctrl->device, "Unable to create the %s device\n",
-			name);
+	if (nvme_cdev_add(&ns->cdev, &ns->cdev_device,
+			&nvme_ns_chr_fops, ns->ctrl->ops->module,
+			ns->ctrl->instance, ns->head->instance)) {
+		dev_err(ns->ctrl->device, "Unable to create the ng%dn%d device\n",
+			ns->ctrl->instance, ns->head->instance);
 		nvme_put_ns(ns);
 		return;
 	}
@@ -5155,19 +5163,6 @@ void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
 }
 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
 
-static void nvme_free_cels(struct nvme_ctrl *ctrl)
-{
-	struct nvme_effects_log	*cel;
-	unsigned long i;
-
-	xa_for_each(&ctrl->cels, i, cel) {
-		xa_erase(&ctrl->cels, i);
-		kfree(cel);
-	}
-
-	xa_destroy(&ctrl->cels);
-}
-
 static void nvme_free_ctrl(struct device *dev)
 {
 	struct nvme_ctrl *ctrl =
@@ -5180,7 +5175,6 @@ static void nvme_free_ctrl(struct device *dev)
 		blk_put_queue(ctrl->fabrics_q);
 	if (!subsys || ctrl->instance != subsys->instance)
 		ida_free(&nvme_instance_ida, ctrl->instance);
-	nvme_free_cels(ctrl);
 	nvme_mpath_uninit(ctrl);
 	cleanup_srcu_struct(&ctrl->srcu);
 	nvme_auth_stop(ctrl);
@@ -5226,7 +5220,6 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
 
 	mutex_init(&ctrl->scan_lock);
 	INIT_LIST_HEAD(&ctrl->namespaces);
-	xa_init(&ctrl->cels);
 	ctrl->dev = dev;
 	ctrl->ops = ops;
 	ctrl->quirks = quirks;
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 48454cb7a0fcc9..3a4030bcfab7d9 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -3338,7 +3338,7 @@ nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
 				ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
 					(ctrl->ctrl.opts->max_reconnects *
 					 ctrl->ctrl.opts->reconnect_delay)));
-		WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
+		nvme_delete_ctrl(&ctrl->ctrl);
 	}
 }
 
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 6539d475009865..3aa5d2f2dfbbda 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -423,8 +423,6 @@ struct nvme_uring_data {
 struct nvme_uring_cmd_pdu {
 	struct request *req;
 	struct bio *bio;
-	u64 result;
-	int status;
 };
 
 static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
@@ -440,8 +438,7 @@ static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 
 	if (pdu->bio)
 		blk_rq_unmap_user(pdu->bio);
-	io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
-			    IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+	__io_uring_cmd_done(ioucmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 }
 
 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
@@ -450,15 +447,17 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 {
 	struct io_uring_cmd *ioucmd = req->end_io_data;
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
+	u64 result = le64_to_cpu(nvme_req(req)->result.u64);
+	int status;
 
 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
-		pdu->status = -EINTR;
+		status = -EINTR;
 	} else {
-		pdu->status = nvme_req(req)->status;
-		if (!pdu->status)
-			pdu->status = blk_status_to_errno(err);
+		status = nvme_req(req)->status;
+		if (!status)
+			status = blk_status_to_errno(err);
 	}
-	pdu->result = le64_to_cpu(nvme_req(req)->result.u64);
+	io_uring_cmd_set_res32(ioucmd, status, result);
 
 	/*
 	 * For IOPOLL, check if this completion is happening in the context
@@ -471,7 +470,7 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 	    iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
 		if (pdu->bio)
 			blk_rq_unmap_user(pdu->bio);
-		io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 0);
+		__io_uring_cmd_done(ioucmd, 0);
 	} else {
 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
 	}
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 75dbb58286a32c..99ecc73393e585 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -646,17 +646,15 @@ static const struct file_operations nvme_ns_head_chr_fops = {
 
 static void nvme_add_ns_head_cdev(struct nvme_ns_head *head)
 {
-	char name[32];
-
 	head->cdev_device.parent = &head->subsys->dev;
-	snprintf(name, sizeof(name), "ng%dn%d", head->subsys->instance,
-		 head->instance);
 
 	nvme_get_ns_head(head); /* Undone in nvme_cdev_rel() */
-	if (nvme_cdev_add(name, &head->cdev, &head->cdev_device,
-			&nvme_ns_head_chr_fops, THIS_MODULE)) {
+	if (nvme_cdev_add(&head->cdev, &head->cdev_device,
+			&nvme_ns_head_chr_fops, THIS_MODULE,
+			head->subsys->instance, head->instance)) {
 		dev_err(disk_to_dev(head->disk),
-			"Unable to create the %s device\n", name);
+			"Unable to create the ng%dn%d device\n",
+			head->subsys->instance, head->instance);
 		nvme_put_ns_head(head);
 		return;
 	}
@@ -763,8 +761,6 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 	lim.dma_alignment = 3;
 	lim.features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT |
 		BLK_FEAT_POLL | BLK_FEAT_ATOMIC_WRITES | BLK_FEAT_PCI_P2PDMA;
-	if (head->ids.csi == NVME_CSI_ZNS)
-		lim.features |= BLK_FEAT_ZONED;
 
 	head->disk = blk_alloc_disk(&lim, ctrl->numa_node);
 	if (IS_ERR(head->disk))
@@ -845,7 +841,8 @@ static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 		u32 nr_nsids;
 		size_t nsid_buf_size;
 
-		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
+		if (WARN_ON_ONCE(offset > ctrl->ana_log_size ||
+				 sizeof(*desc) > ctrl->ana_log_size - offset))
 			return -EINVAL;
 
 		nr_nsids = le32_to_cpu(desc->nnsids);
@@ -861,7 +858,7 @@ static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 			return -EINVAL;
 
 		offset += sizeof(*desc);
-		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
+		if (WARN_ON_ONCE(nsid_buf_size > ctrl->ana_log_size - offset))
 			return -EINVAL;
 
 		error = cb(ctrl, desc, data);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 2cff9fcbf7404e..a643c4723a69b9 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -415,7 +415,6 @@ struct nvme_ctrl {
 	unsigned long quirks;
 	struct nvme_id_power_state psd[32];
 	struct nvme_effects_log *effects;
-	struct xarray cels;
 	struct work_struct scan_work;
 	struct work_struct async_event_work;
 	struct delayed_work ka_work;
@@ -509,6 +508,8 @@ struct nvme_subsystem {
 		__guarded_by(&nvme_subsystems_lock);
 	struct list_head	nsheads
 		__guarded_by(&lock);
+	/* Command effects logs, indexed by CSI and protected by lock. */
+	struct xarray		cels;
 	char			subnqn[NVMF_NQN_SIZE];
 	char			serial[20];
 	char			model[40];
@@ -1013,9 +1014,9 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
 void nvme_get_ns_head(struct nvme_ns_head *head);
 bool nvme_tryget_ns_head(struct nvme_ns_head *head);
 void nvme_put_ns_head(struct nvme_ns_head *head);
-int nvme_cdev_add(const char *name, struct cdev *cdev,
-		struct device *cdev_device,
-		const struct file_operations *fops, struct module *owner);
+int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
+		const struct file_operations *fops, struct module *owner,
+		int ctrl, int head);
 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device);
 int nvme_ioctl(struct block_device *bdev, blk_mode_t mode,
 		unsigned int cmd, unsigned long arg);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 5440cf18b55b28..dbcfc7ddb78aa0 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -3747,7 +3747,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev,
 	 */
 	dev->ctrl.max_hw_sectors = min_t(u32,
 			NVME_MAX_BYTES >> SECTOR_SHIFT,
-			dma_opt_mapping_size(&pdev->dev) >> 9);
+			dma_max_opt_mapping_size(&pdev->dev) >> SECTOR_SHIFT);
 	dev->ctrl.max_segments = NVME_MAX_SEGS;
 	dev->ctrl.max_integrity_segments = 1;
 	return dev;
@@ -4196,6 +4196,8 @@ static const struct pci_device_id nvme_id_table[] = {
 		.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN },
 	 { PCI_DEVICE(0x1344, 0x6001),   /* Micron Nitro NVMe */
 		 .driver_data = NVME_QUIRK_BOGUS_NID, },
+	{ PCI_DEVICE(0x1344, 0x6004),	/* Micron 4100AT */
+		.driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, },
 	{ PCI_DEVICE(0x1c5c, 0x1504),   /* SK Hynix PC400 */
 		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
 	{ PCI_DEVICE(0x1c5c, 0x174a),   /* SK Hynix P31 SSD */
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 02a2490a9ed75f..e1e3dcfd084be4 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -679,7 +679,7 @@ static ssize_t cntrltype_show(struct device *dev,
 	if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype])
 		return sysfs_emit(buf, "reserved\n");
 
-	return sysfs_emit(buf, type[ctrl->cntrltype]);
+	return sysfs_emit(buf, "%s", type[ctrl->cntrltype]);
 }
 static DEVICE_ATTR_RO(cntrltype);
 
@@ -696,7 +696,7 @@ static ssize_t dctype_show(struct device *dev,
 	if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype])
 		return sysfs_emit(buf, "reserved\n");
 
-	return sysfs_emit(buf, type[ctrl->dctype]);
+	return sysfs_emit(buf, "%s", type[ctrl->dctype]);
 }
 static DEVICE_ATTR_RO(dctype);
 
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 921934028e0b22..7eec983c0eaded 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -144,11 +144,6 @@ struct nvme_tcp_queue {
 	void (*state_change)(struct sock *);
 	void (*data_ready)(struct sock *);
 	void (*write_space)(struct sock *);
-
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-	struct lock_class_key nvme_tcp_sk_key;
-	struct lock_class_key nvme_tcp_slock_key;
-#endif
 };
 
 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
@@ -185,9 +180,12 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
  * a separate class prevents lockdep from conflating nvme-tcp socket use with
  * user-space socket API use.
  */
-static void nvme_tcp_reclassify_socket(struct nvme_tcp_queue *queue)
+static struct lock_class_key nvme_tcp_sk_key[2];
+static struct lock_class_key nvme_tcp_slock_key[2];
+
+static void nvme_tcp_reclassify_socket(struct socket *sock)
 {
-	struct sock *sk = queue->sock->sk;
+	struct sock *sk = sock->sk;
 
 	if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
 		return;
@@ -195,20 +193,22 @@ static void nvme_tcp_reclassify_socket(struct nvme_tcp_queue *queue)
 	switch (sk->sk_family) {
 	case AF_INET:
 		sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
-					      &queue->nvme_tcp_slock_key,
+					      &nvme_tcp_slock_key[0],
 					      "sk_lock-AF_INET-NVME",
-					      &queue->nvme_tcp_sk_key);
+					      &nvme_tcp_sk_key[0]);
 		break;
 	case AF_INET6:
 		sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
-					      &queue->nvme_tcp_slock_key,
+					      &nvme_tcp_slock_key[1],
 					      "sk_lock-AF_INET6-NVME",
-					      &queue->nvme_tcp_sk_key);
+					      &nvme_tcp_sk_key[1]);
 		break;
 	default:
 		WARN_ON_ONCE(1);
 	}
 }
+#else
+static void nvme_tcp_reclassify_socket(struct socket *sock) { }
 #endif
 
 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
@@ -1523,11 +1523,6 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
 	mutex_destroy(&queue->send_mutex);
 	mutex_destroy(&queue->queue_lock);
 	mutex_destroy(&queue->pf_cache_lock);
-
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-	lockdep_unregister_key(&queue->nvme_tcp_sk_key);
-	lockdep_unregister_key(&queue->nvme_tcp_slock_key);
-#endif
 }
 
 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
@@ -1875,12 +1870,6 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 
 	sk_net_refcnt_upgrade(queue->sock->sk);
 
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-	lockdep_register_key(&queue->nvme_tcp_sk_key);
-	lockdep_register_key(&queue->nvme_tcp_slock_key);
-	nvme_tcp_reclassify_socket(queue);
-#endif
-
 	/* Single syn retry */
 	tcp_sock_set_syncnt(queue->sock->sk, 1);
 
@@ -1968,6 +1957,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 			goto err_init_connect;
 	}
 
+	nvme_tcp_reclassify_socket(queue->sock);
+
 	ret = nvme_tcp_init_connection(queue);
 	if (ret)
 		goto err_init_connect;
@@ -1984,10 +1975,6 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 	/* Use sync variant - see nvme_tcp_free_queue() for explanation */
 	__fput_sync(queue->sock->file);
 	queue->sock = NULL;
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-	lockdep_unregister_key(&queue->nvme_tcp_sk_key);
-	lockdep_unregister_key(&queue->nvme_tcp_slock_key);
-#endif
 err_destroy_mutex:
 	mutex_destroy(&queue->send_mutex);
 	mutex_destroy(&queue->queue_lock);
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index 6286e38436ddf8..b03b5d1c2dc3d2 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -534,6 +534,7 @@ static ssize_t nvmet_ns_device_path_store(struct config_item *item,
 	struct nvmet_subsys *subsys = ns->subsys;
 	size_t len;
 	int ret;
+	char *new_path = NULL;
 
 	mutex_lock(&subsys->lock);
 	ret = -EBUSY;
@@ -545,12 +546,14 @@ static ssize_t nvmet_ns_device_path_store(struct config_item *item,
 	if (!len)
 		goto out_unlock;
 
-	kfree(ns->device_path);
 	ret = -ENOMEM;
-	ns->device_path = kmemdup_nul(page, len, GFP_KERNEL);
-	if (!ns->device_path)
+	new_path = kmemdup_nul(page, len, GFP_KERNEL);
+	if (!new_path)
 		goto out_unlock;
 
+	kfree(ns->device_path);
+	ns->device_path = new_path;
+
 	mutex_unlock(&subsys->lock);
 	return count;
 
diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 43871a8f56ca36..8eea0a5043080c 100644
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -446,20 +446,27 @@ u16 nvmet_req_find_ns(struct nvmet_req *req)
 {
 	u32 nsid = le32_to_cpu(req->cmd->common.nsid);
 	struct nvmet_subsys *subsys = nvmet_req_subsys(req);
+	u16 status = NVME_SC_SUCCESS;
 
+	rcu_read_lock();
 	req->ns = xa_load(&subsys->namespaces, nsid);
-	if (unlikely(!req->ns || !req->ns->enabled)) {
+	if (unlikely(!req->ns) ||
+	    !test_bit(NVMET_NS_IO_LIVE, &req->ns->flags) ||
+	    !percpu_ref_tryget_live_rcu(&req->ns->ref)) {
 		req->error_loc = offsetof(struct nvme_common_command, nsid);
-		if (!req->ns) /* ns doesn't exist! */
-			return NVME_SC_INVALID_NS | NVME_STATUS_DNR;
+		if (!req->ns) { /* ns doesn't exist! */
+			status = NVME_SC_INVALID_NS | NVME_STATUS_DNR;
+			goto unlock;
+		}
 
 		/* ns exists but it's disabled */
 		req->ns = NULL;
-		return NVME_SC_INTERNAL_PATH_ERROR;
+		status = NVME_SC_INTERNAL_PATH_ERROR;
 	}
+unlock:
+	rcu_read_unlock();
 
-	percpu_ref_get(&req->ns->ref);
-	return NVME_SC_SUCCESS;
+	return status;
 }
 
 static void nvmet_destroy_namespace(struct percpu_ref *ref)
@@ -623,6 +630,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns)
 	ns->enabled = true;
 	xa_set_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED);
 	nvmet_debugfs_ns_setup(ns);
+	set_bit(NVMET_NS_IO_LIVE, &ns->flags);
 	ret = 0;
 out_unlock:
 	mutex_unlock(&subsys->lock);
@@ -643,11 +651,11 @@ void nvmet_ns_disable(struct nvmet_ns *ns)
 	struct nvmet_subsys *subsys = ns->subsys;
 	struct nvmet_ctrl *ctrl;
 
+	if (!test_and_clear_bit(NVMET_NS_IO_LIVE, &ns->flags))
+		return;
+
 	mutex_lock(&subsys->lock);
-	if (!ns->enabled)
-		goto out_unlock;
 
-	ns->enabled = false;
 	xa_clear_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED);
 	nvmet_debugfs_ns_free(ns);
 
@@ -675,7 +683,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns)
 	mutex_lock(&subsys->lock);
 	nvmet_ns_changed(subsys, ns->nsid);
 	nvmet_ns_dev_disable(ns);
-out_unlock:
+	ns->enabled = false;
 	mutex_unlock(&subsys->lock);
 }
 
@@ -1648,6 +1656,8 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args)
 	INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer);
 
 	memcpy(ctrl->hostnqn, args->hostnqn, NVMF_NQN_SIZE);
+	if (args->hostid)
+		uuid_copy(&ctrl->hostid, args->hostid);
 
 	kref_init(&ctrl->ref);
 	ctrl->subsys = subsys;
@@ -1706,9 +1716,6 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args)
 
 	nvmet_start_keep_alive_timer(ctrl);
 
-	if (args->hostid)
-		uuid_copy(&ctrl->hostid, args->hostid);
-
 	dhchap_status = nvmet_setup_auth(ctrl, args->sq, false);
 	if (dhchap_status) {
 		pr_err("Failed to setup authentication, dhchap status %u\n",
diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c
index 92f8a76f10ff50..dccdb55a367c50 100644
--- a/drivers/nvme/target/fabrics-cmd-auth.c
+++ b/drivers/nvme/target/fabrics-cmd-auth.c
@@ -427,7 +427,7 @@ static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
 	int ret = 0;
 	int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
-	int data_size = sizeof(*d) + hash_len;
+	int data_size = sizeof(*data) + hash_len;
 
 	if (ctrl->dh_tfm)
 		data_size += ctrl->dh_keysize;
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index dbda55895f4fb9..162e2fdd848e00 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -112,6 +112,8 @@ struct nvmet_ns {
 
 	bool			buffered_io;
 	bool			enabled;
+#define NVMET_NS_IO_LIVE	0
+	unsigned long		flags;
 	struct nvmet_subsys	*subsys;
 	const char		*device_path;
 
diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c
index 9c28367435a599..c683450df42278 100644
--- a/drivers/nvme/target/pci-epf.c
+++ b/drivers/nvme/target/pci-epf.c
@@ -978,6 +978,9 @@ nvmet_pci_epf_get_sgl_segment(struct nvmet_pci_epf_ctrl *ctrl,
 	int nr_descs, ret;
 	void *buf;
 
+	if (length < sizeof(struct nvme_sgl_desc))
+		return NULL;
+
 	buf = kmalloc(length, GFP_KERNEL);
 	if (!buf)
 		return NULL;
diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig
index c36c2a4c2a0b72..083a5e528960ee 100644
--- a/drivers/nvmem/Kconfig
+++ b/drivers/nvmem/Kconfig
@@ -328,6 +328,17 @@ config NVMEM_MTK_EFUSE
 	  This driver can also be built as a module. If so, the module
 	  will be called efuse-mtk.
 
+config NVMEM_MT6323_EFUSE
+	tristate "MediaTek MT6323 PMIC EFUSE support"
+	depends on ARCH_MEDIATEK || COMPILE_TEST
+	depends on MFD_MT6397
+	help
+	  This is a driver to access hardware related data like sensor
+	  calibration, etc.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called efuse-mt6323.
+
 config NVMEM_MXS_OCOTP
 	tristate "Freescale MXS On-Chip OTP Memory Support"
 	depends on ARCH_MXS || COMPILE_TEST
@@ -541,8 +552,9 @@ config NVMEM_U_BOOT_ENV
 
 config NVMEM_UNIPHIER_EFUSE
 	tristate "UniPhier SoCs eFuse support"
-	depends on ARCH_UNIPHIER || COMPILE_TEST
+	depends on ARCH_UNIPHIER || ARCH_K3 || COMPILE_TEST
 	depends on HAS_IOMEM
+	default y if ARCH_K3
 	help
 	  This is a simple driver to dump specified values of UniPhier SoC
 	  from eFuse.
diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile
index 2bbfb9ff188590..5e70bad9f0f9e3 100644
--- a/drivers/nvmem/Makefile
+++ b/drivers/nvmem/Makefile
@@ -53,6 +53,8 @@ obj-$(CONFIG_NVMEM_MESON_MX_EFUSE)	+= nvmem_meson_mx_efuse.o
 nvmem_meson_mx_efuse-y			:= meson-mx-efuse.o
 obj-$(CONFIG_NVMEM_MICROCHIP_OTPC)	+= nvmem-microchip-otpc.o
 nvmem-microchip-otpc-y			:= microchip-otpc.o
+obj-$(CONFIG_NVMEM_MT6323_EFUSE)	+= nvmem_mt6323-efuse.o
+nvmem_mt6323-efuse-y			:= mt6323-efuse.o
 obj-$(CONFIG_NVMEM_MTK_EFUSE)		+= nvmem_mtk-efuse.o
 nvmem_mtk-efuse-y			:= mtk-efuse.o
 obj-$(CONFIG_NVMEM_MXS_OCOTP)		+= nvmem-mxs-ocotp.o
diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
index 0556d140170a41..7d4cce8e27424a 100644
--- a/drivers/nvmem/core.c
+++ b/drivers/nvmem/core.c
@@ -57,7 +57,12 @@ static BLOCKING_NOTIFIER_HEAD(nvmem_notifier);
 static int __nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset,
 			    void *val, size_t bytes)
 {
-	struct nvmem_operations *ops = nvmem->ops;
+	struct nvmem_operations *ops;
+
+	guard(srcu)(&nvmem->srcu);
+	ops = srcu_dereference(nvmem->ops, &nvmem->srcu);
+	if (!ops)
+		return -ENODEV;
 
 	if (!ops->reg_read)
 		return -EOPNOTSUPP;
@@ -68,9 +73,14 @@ static int __nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset,
 static int __nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset,
 			     void *val, size_t bytes)
 {
-	struct nvmem_operations *ops = nvmem->ops;
+	struct nvmem_operations *ops;
 	int ret, wr_ok;
 
+	guard(srcu)(&nvmem->srcu);
+	ops = srcu_dereference(nvmem->ops, &nvmem->srcu);
+	if (!ops)
+		return -ENODEV;
+
 	if (!ops->reg_write)
 		return -EOPNOTSUPP;
 
@@ -289,7 +299,7 @@ static ssize_t bin_attr_nvmem_write(struct file *filp, struct kobject *kobj,
 
 static umode_t nvmem_bin_attr_get_umode(struct nvmem_device *nvmem)
 {
-	struct nvmem_operations *ops = nvmem->ops;
+	struct nvmem_operations *ops = rcu_dereference_raw(nvmem->ops);
 
 	umode_t mode = 0400;
 
@@ -333,7 +343,7 @@ static umode_t nvmem_attr_is_visible(struct kobject *kobj,
 {
 	struct device *dev = kobj_to_dev(kobj);
 	struct nvmem_device *nvmem = to_nvmem_device(dev);
-	struct nvmem_operations *ops = nvmem->ops;
+	struct nvmem_operations *ops = rcu_dereference_raw(nvmem->ops);
 
 	/*
 	 * If the device has no .reg_write operation, do not allow
@@ -485,7 +495,7 @@ static int nvmem_populate_sysfs_cells(struct nvmem_device *nvmem)
 	/* Allocate an array of attributes with a sentinel */
 	ncells = list_count_nodes(&nvmem->cells);
 	pattrs = devm_kcalloc(&nvmem->dev, ncells + 1,
-			      sizeof(struct bin_attribute *), GFP_KERNEL);
+			      sizeof(*pattrs), GFP_KERNEL);
 	if (!pattrs)
 		return -ENOMEM;
 
@@ -535,24 +545,6 @@ static void nvmem_sysfs_remove_compat(struct nvmem_device *nvmem)
 
 #endif /* CONFIG_NVMEM_SYSFS */
 
-static void nvmem_release(struct device *dev)
-{
-	struct nvmem_device *nvmem = to_nvmem_device(dev);
-
-	ida_free(&nvmem_ida, nvmem->id);
-	gpiod_put(nvmem->wp_gpio);
-	kfree(nvmem->ops);
-	kfree(nvmem);
-}
-
-static const struct device_type nvmem_provider_type = {
-	.release	= nvmem_release,
-};
-
-static const struct bus_type nvmem_bus_type = {
-	.name		= "nvmem",
-};
-
 static void nvmem_cell_entry_drop(struct nvmem_cell_entry *cell)
 {
 	blocking_notifier_call_chain(&nvmem_notifier, NVMEM_CELL_REMOVE, cell);
@@ -571,6 +563,25 @@ static void nvmem_device_remove_all_cells(const struct nvmem_device *nvmem)
 		nvmem_cell_entry_drop(cell);
 }
 
+static void nvmem_release(struct device *dev)
+{
+	struct nvmem_device *nvmem = to_nvmem_device(dev);
+
+	gpiod_put(nvmem->wp_gpio);
+	nvmem_device_remove_all_cells(nvmem);
+	ida_free(&nvmem_ida, nvmem->id);
+	cleanup_srcu_struct(&nvmem->srcu);
+	kfree(nvmem);
+}
+
+static const struct device_type nvmem_provider_type = {
+	.release	= nvmem_release,
+};
+
+static const struct bus_type nvmem_bus_type = {
+	.name		= "nvmem",
+};
+
 static void nvmem_cell_entry_add(struct nvmem_cell_entry *cell)
 {
 	scoped_guard(mutex, &nvmem_mutex)
@@ -918,7 +929,21 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	nvmem->dev.type = &nvmem_provider_type;
 	nvmem->dev.bus = &nvmem_bus_type;
 	nvmem->dev.parent = config->dev;
-	nvmem->ops = ops;
+	INIT_LIST_HEAD(&nvmem->cells);
+
+	/*
+	 * Must happen before we assign the release() callback in
+	 * device_initialize().
+	 */
+	rval = init_srcu_struct(&nvmem->srcu);
+	if (rval) {
+		ida_free(&nvmem_ida, nvmem->id);
+		kfree(ops);
+		kfree(nvmem);
+		return ERR_PTR(rval);
+	}
+
+	rcu_assign_pointer(nvmem->ops, ops);
 
 	device_initialize(&nvmem->dev);
 
@@ -931,8 +956,6 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 		goto err_put_device;
 	}
 
-	kref_init(&nvmem->refcnt);
-	INIT_LIST_HEAD(&nvmem->cells);
 	nvmem->fixup_dt_cell_info = config->fixup_dt_cell_info;
 
 	ops->reg_read = config->reg_read;
@@ -993,20 +1016,20 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	if (config->cells) {
 		rval = nvmem_add_cells(nvmem, config->cells, config->ncells);
 		if (rval)
-			goto err_remove_cells;
+			goto err_remove_compat;
 	}
 
 	if (config->add_legacy_fixed_of_cells) {
 		rval = nvmem_add_cells_from_legacy_of(nvmem);
 		if (rval)
-			goto err_remove_cells;
+			goto err_remove_compat;
 	}
 
 	dev_dbg(&nvmem->dev, "Registering nvmem device %s\n", config->name);
 
 	rval = device_add(&nvmem->dev);
 	if (rval)
-		goto err_remove_cells;
+		goto err_remove_compat;
 
 	rval = nvmem_populate_layout(nvmem);
 	if (rval)
@@ -1032,40 +1055,40 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 #endif
 err_remove_dev:
 	device_del(&nvmem->dev);
-err_remove_cells:
-	nvmem_device_remove_all_cells(nvmem);
+err_remove_compat:
 	nvmem_sysfs_remove_compat(nvmem);
 err_put_device:
+	ops = rcu_replace_pointer(nvmem->ops, NULL, true);
+	synchronize_srcu(&nvmem->srcu);
 	put_device(&nvmem->dev);
+	kfree(ops);
 
 	return ERR_PTR(rval);
 }
 EXPORT_SYMBOL_GPL(nvmem_register);
 
-static void nvmem_device_release(struct kref *kref)
+/**
+ * nvmem_unregister() - Unregister previously registered nvmem device
+ *
+ * @nvmem: Pointer to previously registered nvmem device.
+ */
+void nvmem_unregister(struct nvmem_device *nvmem)
 {
-	struct nvmem_device *nvmem;
+	struct nvmem_operations *ops;
 
-	nvmem = container_of(kref, struct nvmem_device, refcnt);
+	if (!nvmem)
+		return;
 
 	blocking_notifier_call_chain(&nvmem_notifier, NVMEM_REMOVE, nvmem);
 
-	nvmem_sysfs_remove_compat(nvmem);
+	ops = rcu_replace_pointer(nvmem->ops, NULL, true);
+	synchronize_srcu(&nvmem->srcu);
 
-	nvmem_device_remove_all_cells(nvmem);
+	nvmem_sysfs_remove_compat(nvmem);
 	nvmem_destroy_layout(nvmem);
-	device_unregister(&nvmem->dev);
-}
+	kfree(ops);
 
-/**
- * nvmem_unregister() - Unregister previously registered nvmem device
- *
- * @nvmem: Pointer to previously registered nvmem device.
- */
-void nvmem_unregister(struct nvmem_device *nvmem)
-{
-	if (nvmem)
-		kref_put(&nvmem->refcnt, nvmem_device_release);
+	device_unregister(&nvmem->dev);
 }
 EXPORT_SYMBOL_GPL(nvmem_unregister);
 
@@ -1126,8 +1149,6 @@ static struct nvmem_device *nvmem_device_match(void *data,
 		return ERR_PTR(-EINVAL);
 	}
 
-	kref_get(&nvmem->refcnt);
-
 	return nvmem;
 }
 
@@ -1243,9 +1264,8 @@ EXPORT_SYMBOL_GPL(devm_nvmem_device_put);
  */
 void nvmem_device_put(struct nvmem_device *nvmem)
 {
-	put_device(&nvmem->dev);
 	module_put(nvmem->owner);
-	kref_put(&nvmem->refcnt, nvmem_device_release);
+	put_device(&nvmem->dev);
 }
 EXPORT_SYMBOL_GPL(nvmem_device_put);
 
@@ -1598,12 +1618,14 @@ static void nvmem_shift_read_buffer_in_place(struct nvmem_cell_entry *cell, void
 		*p = *b++ >> bit_offset;
 
 		/* setup rest of the bytes if any */
-		for (i = 1; i < cell->bytes; i++) {
+		for (i = 1; i < (cell->bytes - bytes_offset); i++) {
 			/* Get bits from next byte and shift them towards msb */
 			*p++ |= *b << (BITS_PER_BYTE - bit_offset);
 
 			*p = *b++ >> bit_offset;
 		}
+		/* point to end of the buffer unused bits will be cleared */
+		p = buf + cell->bytes - 1;
 	} else if (p != b) {
 		memmove(p, b, cell->bytes - bytes_offset);
 		p += cell->bytes - 1;
diff --git a/drivers/nvmem/internals.h b/drivers/nvmem/internals.h
index 4e610deeaa7b3c..bc7a99f5aefbff 100644
--- a/drivers/nvmem/internals.h
+++ b/drivers/nvmem/internals.h
@@ -6,6 +6,7 @@
 #include <linux/device.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/nvmem-provider.h>
+#include <linux/srcu.h>
 
 /* Hold pointers to callbacks owned by the nvmem provider module. */
 struct nvmem_operations {
@@ -16,10 +17,10 @@ struct nvmem_operations {
 struct nvmem_device {
 	struct module		*owner;
 	struct device		dev;
+	struct srcu_struct	srcu;
 	int			stride;
 	int			word_size;
 	int			id;
-	struct kref		refcnt;
 	size_t			size;
 	bool			read_only;
 	bool			root_only;
@@ -34,7 +35,7 @@ struct nvmem_device {
 	unsigned int		nkeepout;
 	struct gpio_desc	*wp_gpio;
 	struct nvmem_layout	*layout;
-	struct nvmem_operations	*ops;
+	struct nvmem_operations __rcu *ops;
 	void *priv;
 	bool			sysfs_cells_populated;
 };
diff --git a/drivers/nvmem/layouts.c b/drivers/nvmem/layouts.c
index 07a34be9669ca1..409909e3244fb4 100644
--- a/drivers/nvmem/layouts.c
+++ b/drivers/nvmem/layouts.c
@@ -156,7 +156,18 @@ static int nvmem_layout_bus_populate(struct nvmem_device *nvmem,
 
 struct device_node *of_nvmem_layout_get_container(struct nvmem_device *nvmem)
 {
-	return of_get_child_by_name(nvmem->dev.of_node, "nvmem-layout");
+	struct device_node *np;
+
+	/* Search for nvmem-layout child */
+	np = of_get_child_by_name(nvmem->dev.of_node, "nvmem-layout");
+	if (np)
+		return np;
+
+	/* The nvmem of_node is itself a fixed-layout node */
+	if (of_device_is_compatible(nvmem->dev.of_node, "fixed-layout"))
+		return of_node_get(nvmem->dev.of_node);
+
+	return NULL;
 }
 EXPORT_SYMBOL_GPL(of_nvmem_layout_get_container);
 
diff --git a/drivers/nvmem/layouts/onie-tlv.c b/drivers/nvmem/layouts/onie-tlv.c
index 8b0f3c1b8a0e9c..4da3a704388e9c 100644
--- a/drivers/nvmem/layouts/onie-tlv.c
+++ b/drivers/nvmem/layouts/onie-tlv.c
@@ -128,6 +128,7 @@ static int onie_tlv_add_cells(struct device *dev, struct nvmem_device *nvmem,
 
 		ret = nvmem_add_one_cell(nvmem, &cell);
 		if (ret) {
+			of_node_put(cell.np);
 			of_node_put(layout);
 			return ret;
 		}
diff --git a/drivers/nvmem/layouts/sl28vpd.c b/drivers/nvmem/layouts/sl28vpd.c
index e93b020b08369d..79de1e6947d096 100644
--- a/drivers/nvmem/layouts/sl28vpd.c
+++ b/drivers/nvmem/layouts/sl28vpd.c
@@ -126,6 +126,7 @@ static int sl28vpd_add_cells(struct nvmem_layout *layout)
 
 		ret = nvmem_add_one_cell(nvmem, &info);
 		if (ret) {
+			of_node_put(info.np);
 			of_node_put(layout_np);
 			return ret;
 		}
diff --git a/drivers/nvmem/mt6323-efuse.c b/drivers/nvmem/mt6323-efuse.c
new file mode 100644
index 00000000000000..de5e30215c22df
--- /dev/null
+++ b/drivers/nvmem/mt6323-efuse.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (c) 2026 Roman Vivchar <rva333@protonmail.com>
+ */
+
+#include <linux/err.h>
+#include <linux/mfd/mt6397/core.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#include <linux/mfd/mt6323/registers.h>
+
+#define MT6323_EFUSE_DOUT_BASE	MT6323_EFUSE_DOUT_0_15
+#define MT6323_EFUSE_SIZE	24
+
+static int mt6323_efuse_read(void *context, unsigned int offset, void *val,
+			     size_t bytes)
+{
+	struct regmap *map = context;
+	u16 *buf = val;
+	u32 tmp;
+	int ret;
+
+	/*
+	 * A manual loop using regmap_read is required because PWRAP is not
+	 * a continuous MMIO space, but rather a FSM that doesn't implement the
+	 * necessary read callback for the regmap_read_raw and regmap_read_bulk
+	 * functions.
+	 */
+	for (size_t i = 0; i < bytes; i += sizeof(*buf)) {
+		ret = regmap_read(map, MT6323_EFUSE_DOUT_BASE + offset + i, &tmp);
+		if (ret)
+			return ret;
+
+		*buf++ = tmp;
+	}
+
+	return 0;
+}
+
+static int mt6323_efuse_probe(struct platform_device *pdev)
+{
+	struct mt6397_chip *mt6323 = dev_get_drvdata(pdev->dev.parent);
+	struct device *dev = &pdev->dev;
+	struct nvmem_config config = {
+		.name = "mt6323-efuse",
+		.stride = 2,
+		.word_size = 2,
+		.size = MT6323_EFUSE_SIZE,
+		.reg_read = mt6323_efuse_read,
+	};
+	struct nvmem_device *nvmem;
+
+	if (!mt6323 || !mt6323->regmap)
+		return dev_err_probe(dev, -ENODEV, "failed to get parent\n");
+
+	config.dev = dev;
+	config.priv = mt6323->regmap;
+
+	nvmem = devm_nvmem_register(dev, &config);
+	return PTR_ERR_OR_ZERO(nvmem);
+}
+
+static const struct of_device_id mt6323_efuse_of_match[] = {
+	{ .compatible = "mediatek,mt6323-efuse" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mt6323_efuse_of_match);
+
+static struct platform_driver mt6323_efuse_driver = {
+	.probe = mt6323_efuse_probe,
+	.driver = {
+		.name = "mt6323-efuse",
+		.of_match_table = mt6323_efuse_of_match,
+	},
+};
+module_platform_driver(mt6323_efuse_driver);
+
+MODULE_DESCRIPTION("MediaTek MT6323 PMIC EFUSE driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/rockchip-otp.c b/drivers/nvmem/rockchip-otp.c
index 2c0feb036f3fd3..f8a8c7cece98d1 100644
--- a/drivers/nvmem/rockchip-otp.c
+++ b/drivers/nvmem/rockchip-otp.c
@@ -12,6 +12,7 @@
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/nvmem-provider.h>
 #include <linux/reset.h>
 #include <linux/slab.h>
@@ -80,6 +81,8 @@ struct rockchip_otp {
 	void __iomem *base;
 	struct reset_control *rst;
 	const struct rockchip_data *data;
+	/* Serializes access to the OTP controller state machine */
+	struct mutex mutex;
 	struct clk_bulk_data clks[];
 };
 
@@ -272,10 +275,12 @@ static int rockchip_otp_read(void *context, unsigned int offset,
 	if (!otp->data || !otp->data->reg_read)
 		return -EINVAL;
 
+	mutex_lock(&otp->mutex);
+
 	ret = clk_bulk_prepare_enable(otp->data->num_clks, otp->clks);
 	if (ret < 0) {
 		dev_err(otp->dev, "failed to prepare/enable clks\n");
-		return ret;
+		goto unlock;
 	}
 
 	offset += otp->data->read_offset;
@@ -308,6 +313,9 @@ static int rockchip_otp_read(void *context, unsigned int offset,
 err:
 	clk_bulk_disable_unprepare(otp->data->num_clks, otp->clks);
 
+unlock:
+	mutex_unlock(&otp->mutex);
+
 	return ret;
 }
 
@@ -431,6 +439,11 @@ static int rockchip_otp_probe(struct platform_device *pdev)
 
 	otp->data = data;
 	otp->dev = dev;
+
+	ret = devm_mutex_init(dev, &otp->mutex);
+	if (ret)
+		return ret;
+
 	otp->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(otp->base))
 		return dev_err_probe(dev, PTR_ERR(otp->base),
diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
index a64afc3ded3d44..ecc9231026224c 100644
--- a/drivers/of/fdt.c
+++ b/drivers/of/fdt.c
@@ -929,7 +929,7 @@ void __init early_init_dt_check_for_usable_mem_range(void)
 static void __init early_init_dt_check_kho(void)
 {
 	unsigned long node = chosen_node_offset;
-	u64 fdt_start, fdt_size, scratch_start, scratch_size;
+	u64 fdt_start, fdt_size, bootmem_start, bootmem_size;
 
 	if (!IS_ENABLED(CONFIG_KEXEC_HANDOVER) || (long)node < 0)
 		return;
@@ -938,11 +938,11 @@ static void __init early_init_dt_check_kho(void)
 				      &fdt_start, &fdt_size))
 		return;
 
-	if (!of_flat_dt_get_addr_size(node, "linux,kho-scratch",
-				      &scratch_start, &scratch_size))
+	if (!of_flat_dt_get_addr_size(node, "linux,kho-bootmem",
+				      &bootmem_start, &bootmem_size))
 		return;
 
-	kho_populate(fdt_start, fdt_size, scratch_start, scratch_size);
+	kho_populate(fdt_start, fdt_size, bootmem_start, bootmem_size);
 }
 
 #ifdef CONFIG_SERIAL_EARLYCON
diff --git a/drivers/of/fdt_address.c b/drivers/of/fdt_address.c
index f358d2c807540e..2e4e46a58cb6fc 100644
--- a/drivers/of/fdt_address.c
+++ b/drivers/of/fdt_address.c
@@ -172,7 +172,7 @@ static u64 __init fdt_translate_address(const void *blob, int node_offset)
 		goto bail;
 	bus = &of_busses[0];
 
-	/* Cound address cells & copy address locally */
+	/* Count address cells & copy address locally */
 	bus->count_cells(blob, parent, &na, &ns);
 	if (!OF_CHECK_COUNTS(na, ns)) {
 		pr_err("Bad cell count for %s\n",
diff --git a/drivers/of/irq.c b/drivers/of/irq.c
index ec035367c9500c..4fbbd13f4d0a8b 100644
--- a/drivers/of/irq.c
+++ b/drivers/of/irq.c
@@ -765,34 +765,27 @@ void __init of_irq_init(const struct of_device_id *matches)
 	}
 }
 
-static int of_check_msi_parent(struct device_node *dev_node, struct device_node **msi_node)
+static int of_check_msi_parent(struct device_node *dev_node, struct device_node **msi_node,
+			       struct of_phandle_args *msi_spec)
 {
-	struct of_phandle_args msi_spec;
 	int ret;
 
 	/*
 	 * An msi-parent phandle with a missing or == 0 #msi-cells
 	 * property identifies a 1:1 ID translation mapping.
 	 *
-	 * Set the msi controller node if the firmware matches this
-	 * condition.
+	 * @msi_spec keeps a reference to the target node whenever the
+	 * phandle parses, -EINVAL included, and the caller releases it.
 	 */
 	ret = of_parse_phandle_with_optional_args(dev_node, "msi-parent", "#msi-cells",
-						  0, &msi_spec);
+						  0, msi_spec);
 	if (ret)
 		return ret;
 
-	if ((*msi_node && *msi_node != msi_spec.np) || msi_spec.args_count != 0)
-		ret = -EINVAL;
-
-	if (!ret) {
-		/* Return with a node reference held */
-		*msi_node = msi_spec.np;
-		return 0;
-	}
-	of_node_put(msi_spec.np);
+	if ((*msi_node && *msi_node != msi_spec->np) || msi_spec->args_count != 0)
+		return -EINVAL;
 
-	return ret;
+	return 0;
 }
 
 /**
@@ -806,7 +799,9 @@ static int of_check_msi_parent(struct device_node *dev_node, struct device_node
  * @id_in: Device ID.
  *
  * Walk up the device hierarchy looking for devices with a "msi-map"
- * or "msi-parent" property. If found, apply the mapping to @id_in.
+ * or "msi-parent" property. If found, apply the mapping to @id_in. With
+ * @msi_np non-NULL, a device declaring an msi-parent ends the walk, usable
+ * or not.
  *
  * Returns: The mapped MSI id.
  */
@@ -821,6 +816,7 @@ u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in)
 	 */
 	for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent) {
 		struct of_phandle_args msi_spec = {};
+		int ret;
 
 		if (!of_map_msi_id(parent_dev->of_node, id_in, msi_np, &msi_spec)) {
 			if (msi_spec.np) {
@@ -835,8 +831,17 @@ u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in)
 			break;
 		}
 		/* -ENODEV: msi-map absent → check for msi-parent */
-		if (msi_np && !of_check_msi_parent(parent_dev->of_node, msi_np))
+		if (!msi_np)
+			continue;
+
+		ret = of_check_msi_parent(parent_dev->of_node, msi_np, &msi_spec);
+		if (msi_spec.np) {
+			/* A declared msi-parent names the controller, usable or not */
+			if (!ret && !*msi_np)
+				*msi_np = of_node_get(msi_spec.np);
+			of_node_put(msi_spec.np);
 			break;
+		}
 	}
 	return id_out;
 }
diff --git a/drivers/of/kexec.c b/drivers/of/kexec.c
index 029903b986cbd3..bb4e0d9bfd384d 100644
--- a/drivers/of/kexec.c
+++ b/drivers/of/kexec.c
@@ -261,23 +261,23 @@ static int kho_add_chosen(const struct kimage *image, void *fdt, int chosen_node
 #ifdef CONFIG_KEXEC_HANDOVER
 	phys_addr_t fdt_mem = 0;
 	phys_addr_t fdt_len = 0;
-	phys_addr_t scratch_mem = 0;
-	phys_addr_t scratch_len = 0;
+	phys_addr_t bootmem_arr = 0;
+	phys_addr_t bootmem_len = 0;
 
 	ret = fdt_delprop(fdt, chosen_node, "linux,kho-fdt");
 	if (ret && ret != -FDT_ERR_NOTFOUND)
 		return ret;
-	ret = fdt_delprop(fdt, chosen_node, "linux,kho-scratch");
+	ret = fdt_delprop(fdt, chosen_node, "linux,kho-bootmem");
 	if (ret && ret != -FDT_ERR_NOTFOUND)
 		return ret;
 
-	if (!image->kho.fdt || !image->kho.scratch)
+	if (!image->kho.fdt || !image->kho.bootmem)
 		return 0;
 
 	fdt_mem = image->kho.fdt;
 	fdt_len = PAGE_SIZE;
-	scratch_mem = image->kho.scratch->mem;
-	scratch_len = image->kho.scratch->bufsz;
+	bootmem_arr = image->kho.bootmem->mem;
+	bootmem_len = image->kho.bootmem->bufsz;
 
 	pr_debug("Adding kho metadata to DT");
 
@@ -285,8 +285,8 @@ static int kho_add_chosen(const struct kimage *image, void *fdt, int chosen_node
 				       fdt_mem, fdt_len);
 	if (ret)
 		return ret;
-	ret = fdt_appendprop_addrrange(fdt, 0, chosen_node, "linux,kho-scratch",
-				       scratch_mem, scratch_len);
+	ret = fdt_appendprop_addrrange(fdt, 0, chosen_node, "linux,kho-bootmem",
+				       bootmem_arr, bootmem_len);
 
 #endif /* CONFIG_KEXEC_HANDOVER */
 	return ret;
diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c
index 08d5351746be65..83c56dd9b5d83e 100644
--- a/drivers/of/overlay.c
+++ b/drivers/of/overlay.c
@@ -256,6 +256,9 @@ static struct property *dup_and_fixup_symbol_prop(
 	if (!target_path)
 		return NULL;
 	target_path_len = strlen(target_path);
+	/* a root target renders as "/"; drop it to avoid "//" results */
+	if (target_path_len == 1 && target_path[0] == '/' && path_tail_len)
+		target_path_len = 0;
 
 	new_prop = kzalloc_obj(*new_prop);
 	if (!new_prop)
@@ -358,12 +361,13 @@ static int add_changeset_property(struct overlay_changeset *ovcs,
 		return -ENOMEM;
 
 	if (!prop) {
-		if (!target->in_livetree) {
+		ret = of_changeset_add_property(&ovcs->cset, target->np,
+						new_prop);
+		/* the detached node owns the property until the apply */
+		if (!ret && !target->in_livetree) {
 			new_prop->next = target->np->deadprops;
 			target->np->deadprops = new_prop;
 		}
-		ret = of_changeset_add_property(&ovcs->cset, target->np,
-						new_prop);
 	} else {
 		ret = of_changeset_update_property(&ovcs->cset, target->np,
 						   new_prop);
@@ -853,6 +857,10 @@ static int init_overlay_changeset(struct overlay_changeset *ovcs,
 err_out:
 	pr_err("%s() failed, ret = %d\n", __func__, ret);
 
+	/* let free_overlay_changeset() put the fragments set up so far */
+	if (ovcs->fragments)
+		ovcs->count = cnt;
+
 	return ret;
 }
 
@@ -863,7 +871,8 @@ static void free_overlay_changeset(struct overlay_changeset *ovcs)
 	if (ovcs->cset.entries.next)
 		of_changeset_destroy(&ovcs->cset);
 
-	if (ovcs->id) {
+	/* a failed idr_alloc() leaves its negative error in ovcs->id */
+	if (ovcs->id > 0) {
 		idr_remove(&ovcs_idr, ovcs->id);
 		list_del(&ovcs->ovcs_list);
 		ovcs->id = 0;
diff --git a/drivers/of/property.c b/drivers/of/property.c
index 72cf12907de034..45e55f61848801 100644
--- a/drivers/of/property.c
+++ b/drivers/of/property.c
@@ -74,7 +74,7 @@ bool of_graph_is_present(const struct device_node *node)
 
 	struct device_node *port __free(device_node) = of_get_child_by_name(node, "port");
 
-	return !!port;
+	return port;
 }
 EXPORT_SYMBOL(of_graph_is_present);
 
@@ -1471,6 +1471,22 @@ static struct device_node *parse_gpio_compat(struct device_node *np,
 	return sup_args.np;
 }
 
+static struct device_node *parse_i2c_parent(struct device_node *np,
+					    const char *prop_name, int index)
+{
+	if (strcmp(prop_name, "i2c-parent"))
+		return NULL;
+
+	/* tc9563 (pci1179,0623) is <phandle addr>; every other user is bare phandles */
+	if (of_device_is_compatible(np, "pci1179,0623")) {
+		if (index)
+			return NULL;
+		return of_parse_phandle(np, prop_name, 0);
+	}
+
+	return of_parse_phandle(np, prop_name, index);
+}
+
 static struct device_node *parse_interrupts(struct device_node *np,
 					    const char *prop_name, int index)
 {
@@ -1562,6 +1578,7 @@ static const struct supplier_bindings of_supplier_bindings[] = {
 	{ .parse_prop = parse_resets, },
 	{ .parse_prop = parse_leds, },
 	{ .parse_prop = parse_backlight, },
+	{ .parse_prop = parse_i2c_parent, },
 	{ .parse_prop = parse_panel, },
 	{ .parse_prop = parse_msi_parent, },
 	{ .parse_prop = parse_pses, },
@@ -1659,12 +1676,12 @@ static bool of_is_fwnode_add_links_supported(void)
 		return true;
 
 	if (is_supported != -1)
-		return !!is_supported;
+		return is_supported;
 
 	is_supported = !((match_property_by_path("/soc", "compatible", "intel,ce4100-cp") >= 0) ||
 			 (match_property_by_path("/", "architecture", "OLPC") >= 0));
 
-	return !!is_supported;
+	return is_supported;
 }
 
 static int of_fwnode_add_links(struct fwnode_handle *fwnode)
diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c
index dc1b6d11f9abb1..a85efc41ce27b2 100644
--- a/drivers/parisc/ccio-dma.c
+++ b/drivers/parisc/ccio-dma.c
@@ -415,7 +415,7 @@ ccio_alloc_range(struct ioc *ioc, struct device *dev, size_t size)
  * @pages_mapped: The requested number of pages to be freed from the
  * I/O Pdir.
  *
- * This function frees the resouces allocated for the iova.
+ * This function frees the resources allocated for the iova.
  */
 static void
 ccio_free_range(struct ioc *ioc, dma_addr_t iova, unsigned long pages_mapped)
diff --git a/drivers/parisc/dino.c b/drivers/parisc/dino.c
index 4b96d9fda49a9a..286f4f75dba738 100644
--- a/drivers/parisc/dino.c
+++ b/drivers/parisc/dino.c
@@ -590,7 +590,7 @@ dino_card_fixup(struct pci_dev *dev)
 	dino_cfg_write(dev->bus, dev->devfn, PCI_INTERRUPT_LINE, 1, dev->irq); 
 }
 
-/* The alignment contraints for PCI bridges under dino */
+/* The alignment constraints for PCI bridges under dino */
 #define DINO_BRIDGE_ALIGN 0x100000
 
 
diff --git a/drivers/parisc/pdc_stable.c b/drivers/parisc/pdc_stable.c
index 16f4496bca9511..6b567a74fc2cb2 100644
--- a/drivers/parisc/pdc_stable.c
+++ b/drivers/parisc/pdc_stable.c
@@ -586,7 +586,7 @@ static ssize_t pdcs_autosearch_read(struct kobject *kobj,
  * @attr: The kobject attributes.
  * @buf: The output buffer to write to.
  *
- * The value of the timer field correponds to a number of seconds in powers of 2.
+ * The value of the timer field corresponds to a number of seconds in powers of 2.
  */
 static ssize_t pdcs_timer_read(struct kobject *kobj,
 			       struct kobj_attribute *attr, char *buf)
diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c
index e0a148ab5876c6..ecc7bb7a846826 100644
--- a/drivers/parisc/sba_iommu.c
+++ b/drivers/parisc/sba_iommu.c
@@ -599,7 +599,7 @@ sba_io_pdir_entry(__le64 *pdir_ptr, space_t sid, phys_addr_t pba,
  * corresponding IO TLB entry. The Ike PCOM (Purge Command Register)
  * is to purge stale entries in the IO TLB when unmapping entries.
  *
- * The PCOM register supports purging of multiple pages, with a minium
+ * The PCOM register supports purging of multiple pages, with a minimum
  * of 1 page and a maximum of 2GB. Hardware requires the address be
  * aligned to the size of the range being purged. The size of the range
  * must be a power of 2. The "Cool perf optimization" in the
diff --git a/drivers/parport/parport_gsc.c b/drivers/parport/parport_gsc.c
index e5e8054a1af800..0a9f7fdb25bc8d 100644
--- a/drivers/parport/parport_gsc.c
+++ b/drivers/parport/parport_gsc.c
@@ -348,7 +348,7 @@ static int __init parport_init_chip(struct parisc_device *dev)
 	 */
 	if (boot_cpu_data.cpu_type > pcxt && !pdc_add_valid(port+4)) {
 
-		/* Initialize bidirectional-mode (0x10) & data-tranfer-mode #1 (0x20) */
+		/* Initialize bidirectional-mode (0x10) & data-transfer-mode #1 (0x20) */
 		pr_info("%s: initialize bidirectional-mode\n", __func__);
 		parport_writeb ( (0x10 + 0x20), port + 4);
 
diff --git a/drivers/parport/procfs.c b/drivers/parport/procfs.c
index 3880460e67f25a..1bfafc2a5724bf 100644
--- a/drivers/parport/procfs.c
+++ b/drivers/parport/procfs.c
@@ -26,7 +26,7 @@
 
 #include <linux/uaccess.h>
 
-#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
+#ifdef CONFIG_SYSCTL
 
 #define PARPORT_MIN_TIMESLICE_VALUE 1ul 
 #define PARPORT_MAX_TIMESLICE_VALUE ((unsigned long) HZ)
@@ -560,7 +560,7 @@ static void __exit parport_default_proc_unregister(void)
 	parport_bus_exit();
 }
 
-#else /* no sysctl or no procfs*/
+#else /* CONFIG_SYSCTL */
 
 int parport_proc_register(struct parport *pp)
 {
@@ -591,7 +591,7 @@ static void __exit parport_default_proc_unregister (void)
 {
 	parport_bus_exit();
 }
-#endif
+#endif /* CONFIG_SYSCTL */
 
 subsys_initcall(parport_default_proc_register)
 module_exit(parport_default_proc_unregister)
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 0c7408509ba24b..16fbd4212e0f55 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -271,6 +271,21 @@ config VGA_ARB_MAX_GPUS
 	  Reserves space in the kernel to maintain resource locking for
 	  multiple GPUS.  The overhead for each GPU is very small.
 
+config PCI_LIVEUPDATE
+	bool "PCI Live Update Support"
+	depends on PCI && LIVEUPDATE && 64BIT && !CARDBUS
+	help
+	  Enable PCI core support for preserving PCI devices across Live
+	  Update. This, in combination with support in a device's driver,
+	  enables PCI devices to run and perform memory transactions
+	  uninterrupted during a kexec for Live Update.
+
+	  This option should only be enabled by users who plan to use Live
+	  Update for kernel upgrades and require preserving PCI devices during
+	  those upgrades.
+
+	  If unsure, say N.
+
 source "drivers/pci/hotplug/Kconfig"
 source "drivers/pci/controller/Kconfig"
 source "drivers/pci/endpoint/Kconfig"
diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index 41ebc3b9a51824..e8d003cb675763 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -16,6 +16,7 @@ obj-$(CONFIG_PROC_FS)		+= proc.o
 obj-$(CONFIG_SYSFS)		+= pci-sysfs.o slot.o
 obj-$(CONFIG_ACPI)		+= pci-acpi.o
 obj-$(CONFIG_GENERIC_PCI_IOMAP) += iomap.o
+obj-$(CONFIG_PCI_LIVEUPDATE)	+= liveupdate.o
 endif
 
 obj-$(CONFIG_OF)		+= of.o
diff --git a/drivers/pci/controller/cadence/pcie-cadence-debugfs.c b/drivers/pci/controller/cadence/pcie-cadence-debugfs.c
index 0a308f95e9f6c5..7b8354052aeabf 100644
--- a/drivers/pci/controller/cadence/pcie-cadence-debugfs.c
+++ b/drivers/pci/controller/cadence/pcie-cadence-debugfs.c
@@ -210,24 +210,21 @@ static const char *cdns_pcie_hpa_ltssm_status_string(enum cdns_pcie_hpa_ltssm lt
 static int ltssm_status_show(struct seq_file *s, void *v)
 {
 	struct cdns_pcie *pci = s->private;
-	enum cdns_pcie_lga_ltssm lga_ltssm;
-	enum cdns_pcie_hpa_ltssm hpa_ltssm;
 	const char *str_ltssm;
-	u32 val;
+	u32 val, ltssm_val;
 
 	if (pci->is_hpa) {
 		val = cdns_pcie_hpa_readl(pci, REG_BANK_IP_REG,
 					  CDNS_PCIE_HPA_PHY_DBG_STS_REG0);
-		hpa_ltssm = FIELD_GET(CDNS_PCIE_HPA_LTSSM_STATUS_MASK, val);
-		str_ltssm = cdns_pcie_hpa_ltssm_status_string(hpa_ltssm);
+		ltssm_val = FIELD_GET(CDNS_PCIE_HPA_LTSSM_STATUS_MASK, val);
+		str_ltssm = cdns_pcie_hpa_ltssm_status_string(ltssm_val);
 	} else {
 		val = cdns_pcie_readl(pci, CDNS_PCIE_LM_BASE);
-		lga_ltssm = FIELD_GET(CDNS_PCIE_LGA_LTSSM_STATUS_MASK, val);
-		str_ltssm = cdns_pcie_lga_ltssm_status_string(lga_ltssm);
+		ltssm_val = FIELD_GET(CDNS_PCIE_LGA_LTSSM_STATUS_MASK, val);
+		str_ltssm = cdns_pcie_lga_ltssm_status_string(ltssm_val);
 	}
 
-	seq_printf(s, "%s (0x%02x)\n", str_ltssm,
-		   pci->is_hpa ? hpa_ltssm : lga_ltssm);
+	seq_printf(s, "%s (0x%02x)\n", str_ltssm, ltssm_val);
 
 	return 0;
 }
diff --git a/drivers/pci/controller/cadence/pcie-cadence-plat.c b/drivers/pci/controller/cadence/pcie-cadence-plat.c
index a1ea24fc3b63f2..13edc6be21f597 100644
--- a/drivers/pci/controller/cadence/pcie-cadence-plat.c
+++ b/drivers/pci/controller/cadence/pcie-cadence-plat.c
@@ -41,7 +41,6 @@ static int cdns_plat_pcie_probe(struct platform_device *pdev)
 	struct pci_host_bridge *bridge;
 	struct cdns_pcie_ep *ep;
 	struct cdns_pcie_rc *rc;
-	int phy_count;
 	bool is_rc;
 	int ret;
 
@@ -122,11 +121,8 @@ static int cdns_plat_pcie_probe(struct platform_device *pdev)
 	pm_runtime_put_sync(dev);
 	pm_runtime_disable(dev);
 	cdns_pcie_disable_phy(cdns_plat_pcie->pcie);
-	phy_count = cdns_plat_pcie->pcie->phy_count;
-	while (phy_count--)
-		device_link_del(cdns_plat_pcie->pcie->link[phy_count]);
 
-	return 0;
+	return ret;
 }
 
 static void cdns_plat_pcie_shutdown(struct platform_device *pdev)
diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c
index a1eada56edba76..0ac98024994141 100644
--- a/drivers/pci/controller/cadence/pcie-cadence.c
+++ b/drivers/pci/controller/cadence/pcie-cadence.c
@@ -222,7 +222,6 @@ int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie)
 	struct device_node *np = dev->of_node;
 	int phy_count;
 	struct phy **phy;
-	struct device_link **link;
 	int i;
 	int ret;
 	const char *name;
@@ -238,10 +237,6 @@ int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie)
 	if (!phy)
 		return -ENOMEM;
 
-	link = devm_kcalloc(dev, phy_count, sizeof(*link), GFP_KERNEL);
-	if (!link)
-		return -ENOMEM;
-
 	for (i = 0; i < phy_count; i++) {
 		of_property_read_string_index(np, "phy-names", i, &name);
 		phy[i] = devm_phy_get(dev, name);
@@ -249,17 +244,10 @@ int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie)
 			ret = PTR_ERR(phy[i]);
 			goto err_phy;
 		}
-		link[i] = device_link_add(dev, &phy[i]->dev, DL_FLAG_STATELESS);
-		if (!link[i]) {
-			devm_phy_put(dev, phy[i]);
-			ret = -EINVAL;
-			goto err_phy;
-		}
 	}
 
 	pcie->phy_count = phy_count;
 	pcie->phy = phy;
-	pcie->link = link;
 
 	ret =  cdns_pcie_enable_phy(pcie);
 	if (ret)
@@ -268,10 +256,8 @@ int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie)
 	return 0;
 
 err_phy:
-	while (--i >= 0) {
-		device_link_del(link[i]);
+	while (--i >= 0)
 		devm_phy_put(dev, phy[i]);
-	}
 
 	return ret;
 }
diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h
index b4d51adefea536..4a96e1869e5359 100644
--- a/drivers/pci/controller/cadence/pcie-cadence.h
+++ b/drivers/pci/controller/cadence/pcie-cadence.h
@@ -260,7 +260,6 @@ struct cdns_plat_pcie_of_data {
  * @is_hpa: indicates if the architecture is HPA
  * @phy_count: number of supported PHY devices
  * @phy: list of pointers to specific PHY control blocks
- * @link: list of pointers to corresponding device link representations
  * @ops: Platform-specific ops to control various inputs from Cadence PCIe
  *       wrapper
  * @cdns_pcie_reg_offsets: Register bank offsets for different SoC
@@ -276,7 +275,6 @@ struct cdns_pcie {
 	bool				     is_hpa;
 	int			             phy_count;
 	struct phy		             **phy;
-	struct device_link	             **link;
 	const  struct cdns_pcie_ops          *ops;
 	const  struct cdns_plat_pcie_of_data *cdns_pcie_reg_offsets;
 	int				     max_link_speed;
diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
index 6ae5b27e27b323..3460a83b439593 100644
--- a/drivers/pci/controller/dwc/pci-dra7xx.c
+++ b/drivers/pci/controller/dwc/pci-dra7xx.c
@@ -9,7 +9,6 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/device.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
@@ -680,7 +679,6 @@ static int dra7xx_pcie_probe(struct platform_device *pdev)
 	int i;
 	int phy_count;
 	struct phy **phy;
-	struct device_link **link;
 	void __iomem *base;
 	struct dw_pcie *pci;
 	struct dra7xx_pcie *dra7xx;
@@ -728,30 +726,16 @@ static int dra7xx_pcie_probe(struct platform_device *pdev)
 	if (!phy)
 		return -ENOMEM;
 
-	link = devm_kcalloc(dev, phy_count, sizeof(*link), GFP_KERNEL);
-	if (!link)
-		return -ENOMEM;
-
-	dra7xx->clk = devm_clk_get_optional(dev, NULL);
+	dra7xx->clk = devm_clk_get_optional_enabled(dev, NULL);
 	if (IS_ERR(dra7xx->clk))
 		return dev_err_probe(dev, PTR_ERR(dra7xx->clk),
 				     "clock request failed");
 
-	ret = clk_prepare_enable(dra7xx->clk);
-	if (ret)
-		return ret;
-
 	for (i = 0; i < phy_count; i++) {
 		snprintf(name, sizeof(name), "pcie-phy%d", i);
 		phy[i] = devm_phy_get(dev, name);
 		if (IS_ERR(phy[i]))
 			return PTR_ERR(phy[i]);
-
-		link[i] = device_link_add(dev, &phy[i]->dev, DL_FLAG_STATELESS);
-		if (!link[i]) {
-			ret = -EINVAL;
-			goto err_link;
-		}
 	}
 
 	dra7xx->base = base;
@@ -853,10 +837,6 @@ static int dra7xx_pcie_probe(struct platform_device *pdev)
 	pm_runtime_disable(dev);
 	dra7xx_pcie_disable_phy(dra7xx);
 
-err_link:
-	while (--i >= 0)
-		device_link_del(link[i]);
-
 	return ret;
 }
 
diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c
index f7a2eb257c1691..28fed010b503f5 100644
--- a/drivers/pci/controller/dwc/pci-imx6.c
+++ b/drivers/pci/controller/dwc/pci-imx6.c
@@ -80,6 +80,18 @@
 #define IMX95_SID_MASK				GENMASK(5, 0)
 #define IMX95_MAX_LUT				32
 
+#define IMX95_PCIE_PHY_REG_ADDR			0x3008
+#define IMX95_PCIE_PHY_REG_EN			BIT(31)
+#define IMX95_PCIE_PHY_REG_ADDR_MASK		GENMASK(15, 0)
+
+#define IMX95_PCIE_PHY_REG_DATA			0x300c
+
+#define IMX95_PCIE_PHY_MPLLB_OVRD_IN		0x2004
+#define IMX95_PCIE_PHY_MPLLB_OVRD_BW_EN		BIT(10)
+
+#define IMX95_PCIE_PHY_MPLLB_BW_IN		0x2005
+#define IMX95_PCIE_PHY_MPLLB_BW_VAL		0x8c
+
 #define IMX95_PCIE_RST_CTRL			0x3010
 #define IMX95_PCIE_COLD_RST			BIT(0)
 
@@ -180,8 +192,12 @@ struct imx_pcie {
 	struct imx_lut_data	luts[IMX95_MAX_LUT];
 	/* power domain for pcie */
 	struct device		*pd_pcie;
+	/* device link for pcie power domain */
+	struct device_link	*pd_link;
 	/* power domain for pcie phy */
 	struct device		*pd_pcie_phy;
+	/* device link for pcie phy power domain */
+	struct device_link	*pd_phy_link;
 	struct phy		*phy;
 	const struct imx_pcie_drvdata *drvdata;
 
@@ -270,8 +286,19 @@ static int imx95_pcie_select_ref_clk_src(struct imx_pcie *imx_pcie)
 	return 0;
 }
 
+static void imx95_pcie_phy_write(struct imx_pcie *imx_pcie, int addr, u16 data)
+{
+	regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_REG_ADDR,
+			   IMX95_PCIE_PHY_REG_EN, IMX95_PCIE_PHY_REG_EN);
+	regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_REG_ADDR,
+			   IMX95_PCIE_PHY_REG_ADDR_MASK, addr);
+	regmap_write(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_REG_DATA, data);
+}
+
 static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie)
 {
+	u32 val;
+
 	/*
 	 * ERR051624: The Controller Without Vaux Cannot Exit L23 Ready
 	 * Through Beacon or PERST# De-assertion
@@ -290,6 +317,21 @@ static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie)
 			IMX95_PCIE_PHY_CR_PARA_SEL,
 			IMX95_PCIE_PHY_CR_PARA_SEL);
 
+	/* Flush the IMX95_PCIE_PHY_CR_PARA_SEL update */
+	regmap_read(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0, &val);
+
+	/*
+	 * A delay is required between the assertion of
+	 * IMX95_PCIE_PHY_CR_PARA_SEL and subsequent PHY register write
+	 * operation to ensure values are successfully written.
+	 */
+	udelay(200);
+
+	imx95_pcie_phy_write(imx_pcie, IMX95_PCIE_PHY_MPLLB_BW_IN,
+			     IMX95_PCIE_PHY_MPLLB_BW_VAL);
+	imx95_pcie_phy_write(imx_pcie, IMX95_PCIE_PHY_MPLLB_OVRD_IN,
+			     IMX95_PCIE_PHY_MPLLB_OVRD_BW_EN);
+
 	return 0;
 }
 
@@ -639,10 +681,34 @@ static int imx6q_pcie_abort_handler(unsigned long addr,
 }
 #endif
 
+static void imx_pcie_detach_pd_action(void *data)
+{
+	struct imx_pcie *imx_pcie = data;
+
+	if (!IS_ERR_OR_NULL(imx_pcie->pd_phy_link)) {
+		device_link_del(imx_pcie->pd_phy_link);
+		imx_pcie->pd_phy_link = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(imx_pcie->pd_link)) {
+		device_link_del(imx_pcie->pd_link);
+		imx_pcie->pd_link = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(imx_pcie->pd_pcie_phy)) {
+		dev_pm_domain_detach(imx_pcie->pd_pcie_phy, true);
+		imx_pcie->pd_pcie_phy = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(imx_pcie->pd_pcie)) {
+		dev_pm_domain_detach(imx_pcie->pd_pcie, true);
+		imx_pcie->pd_pcie = NULL;
+	}
+}
+
 static int imx_pcie_attach_pd(struct device *dev)
 {
 	struct imx_pcie *imx_pcie = dev_get_drvdata(dev);
-	struct device_link *link;
 
 	/* Do nothing when in a single power domain */
 	if (dev->pm_domain)
@@ -654,11 +720,11 @@ static int imx_pcie_attach_pd(struct device *dev)
 	/* Do nothing when power domain missing */
 	if (!imx_pcie->pd_pcie)
 		return 0;
-	link = device_link_add(dev, imx_pcie->pd_pcie,
-			DL_FLAG_STATELESS |
-			DL_FLAG_PM_RUNTIME |
-			DL_FLAG_RPM_ACTIVE);
-	if (!link) {
+	imx_pcie->pd_link = device_link_add(dev, imx_pcie->pd_pcie,
+					    DL_FLAG_STATELESS |
+					    DL_FLAG_PM_RUNTIME |
+					    DL_FLAG_RPM_ACTIVE);
+	if (!imx_pcie->pd_link) {
 		dev_err(dev, "Failed to add device_link to pcie pd\n");
 		return -EINVAL;
 	}
@@ -667,11 +733,11 @@ static int imx_pcie_attach_pd(struct device *dev)
 	if (IS_ERR(imx_pcie->pd_pcie_phy))
 		return PTR_ERR(imx_pcie->pd_pcie_phy);
 
-	link = device_link_add(dev, imx_pcie->pd_pcie_phy,
-			DL_FLAG_STATELESS |
-			DL_FLAG_PM_RUNTIME |
-			DL_FLAG_RPM_ACTIVE);
-	if (!link) {
+	imx_pcie->pd_phy_link = device_link_add(dev, imx_pcie->pd_pcie_phy,
+						DL_FLAG_STATELESS |
+						DL_FLAG_PM_RUNTIME |
+						DL_FLAG_RPM_ACTIVE);
+	if (!imx_pcie->pd_phy_link) {
 		dev_err(dev, "Failed to add device_link to pcie_phy pd\n");
 		return -EINVAL;
 	}
@@ -1951,6 +2017,10 @@ static int imx_pcie_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, imx_pcie);
 
+	ret = devm_add_action_or_reset(dev, imx_pcie_detach_pd_action, imx_pcie);
+	if (ret)
+		return ret;
+
 	ret = imx_pcie_attach_pd(dev);
 	if (ret)
 		return ret;
@@ -1975,7 +2045,7 @@ static int imx_pcie_probe(struct platform_device *pdev)
 			pm_runtime_no_callbacks(dev);
 			ret = devm_pm_runtime_set_active_enabled(dev);
 			if (ret < 0)
-				return ret;
+				goto err_pwrctrl_destroy;
 		}
 
 		if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY))
diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c
index 602516239a573d..b8e303243c0341 100644
--- a/drivers/pci/controller/dwc/pci-keystone.c
+++ b/drivers/pci/controller/dwc/pci-keystone.c
@@ -24,6 +24,7 @@
 #include <linux/of_pci.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/resource.h>
 #include <linux/signal.h>
@@ -129,7 +130,6 @@ struct keystone_pcie {
 	int			num_lanes;
 	u32			num_viewport;
 	struct phy		**phy;
-	struct device_link	**link;
 	struct			device_node *msi_intc_np;
 	struct irq_domain	*intx_irq_domain;
 	struct device_node	*np;
@@ -1131,7 +1131,6 @@ static int ks_pcie_probe(struct platform_device *pdev)
 	enum dw_pcie_device_mode mode;
 	struct dw_pcie *pci;
 	struct keystone_pcie *ks_pcie;
-	struct device_link **link;
 	struct gpio_desc *gpiod;
 	struct resource *res;
 	void __iomem *base;
@@ -1202,31 +1201,17 @@ static int ks_pcie_probe(struct platform_device *pdev)
 	if (!phy)
 		return -ENOMEM;
 
-	link = devm_kcalloc(dev, num_lanes, sizeof(*link), GFP_KERNEL);
-	if (!link)
-		return -ENOMEM;
-
 	for (i = 0; i < num_lanes; i++) {
 		snprintf(name, sizeof(name), "pcie-phy%d", i);
 		phy[i] = devm_phy_optional_get(dev, name);
 		if (IS_ERR(phy[i])) {
 			ret = PTR_ERR(phy[i]);
-			goto err_link;
-		}
-
-		if (!phy[i])
-			continue;
-
-		link[i] = device_link_add(dev, &phy[i]->dev, DL_FLAG_STATELESS);
-		if (!link[i]) {
-			ret = -EINVAL;
-			goto err_link;
+			goto err;
 		}
 	}
 
 	ks_pcie->np = np;
 	ks_pcie->pci = pci;
-	ks_pcie->link = link;
 	ks_pcie->num_lanes = num_lanes;
 	ks_pcie->phy = phy;
 
@@ -1236,7 +1221,7 @@ static int ks_pcie_probe(struct platform_device *pdev)
 		ret = PTR_ERR(gpiod);
 		if (ret != -EPROBE_DEFER)
 			dev_err(dev, "Failed to get reset GPIO\n");
-		goto err_link;
+		goto err;
 	}
 
 	/* Obtain references to the PHYs */
@@ -1251,7 +1236,7 @@ static int ks_pcie_probe(struct platform_device *pdev)
 
 	if (ret) {
 		dev_err(dev, "failed to enable phy\n");
-		goto err_link;
+		goto err;
 	}
 
 	platform_set_drvdata(pdev, ks_pcie);
@@ -1338,25 +1323,18 @@ static int ks_pcie_probe(struct platform_device *pdev)
 	pm_runtime_disable(dev);
 	ks_pcie_disable_phy(ks_pcie);
 
-err_link:
-	while (--i >= 0 && link[i])
-		device_link_del(link[i]);
-
+err:
 	return ret;
 }
 
 static void ks_pcie_remove(struct platform_device *pdev)
 {
 	struct keystone_pcie *ks_pcie = platform_get_drvdata(pdev);
-	struct device_link **link = ks_pcie->link;
-	int num_lanes = ks_pcie->num_lanes;
 	struct device *dev = &pdev->dev;
 
 	pm_runtime_put(dev);
 	pm_runtime_disable(dev);
 	ks_pcie_disable_phy(ks_pcie);
-	while (num_lanes--)
-		device_link_del(link[num_lanes]);
 }
 
 static struct platform_driver ks_pcie_driver = {
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index de8ee3db43601d..1a3491b5003ec3 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -1194,6 +1194,9 @@ void dw_pcie_ep_deinit(struct dw_pcie_ep *ep)
 			      epc->mem->window.page_size);
 
 	pci_epc_mem_exit(epc);
+
+	if (ep->ops->post_deinit)
+		ep->ops->post_deinit(ep);
 }
 EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit);
 
@@ -1553,7 +1556,7 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 			       ep->page_size);
 	if (ret < 0) {
 		dev_err(dev, "Failed to initialize address space\n");
-		return ret;
+		goto err_deinit;
 	}
 
 	ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys,
@@ -1568,6 +1571,9 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 
 err_exit_epc_mem:
 	pci_epc_mem_exit(epc);
+err_deinit:
+	if (ep->ops->post_deinit)
+		ep->ops->post_deinit(ep);
 
 	return ret;
 }
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index 593388f29bdd48..ea29c9833491bd 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -1039,6 +1039,7 @@ static void dw_pcie_edma_init_data(struct dw_pcie *pci)
 		pci->edma.ops = &dw_pcie_edma_ops;
 
 	pci->edma.flags |= DW_EDMA_CHIP_LOCAL;
+	pci->edma.ch_space_sz = 256;
 }
 
 static int dw_pcie_edma_find_mf(struct dw_pcie *pci)
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 0735ae94092404..a53ac27cd2447c 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -475,6 +475,7 @@ struct dw_pcie_rp {
 
 struct dw_pcie_ep_ops {
 	int	(*pre_init)(struct dw_pcie_ep *ep);
+	void	(*post_deinit)(struct dw_pcie_ep *ep);
 	int	(*init)(struct dw_pcie_ep *ep);
 	int	(*raise_irq)(struct dw_pcie_ep *ep, u8 func_no,
 			     unsigned int type, u16 interrupt_num);
diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c
index af26a07c0c9e5b..14976861f8efed 100644
--- a/drivers/pci/controller/dwc/pcie-dw-rockchip.c
+++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c
@@ -21,6 +21,7 @@
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <linux/reset.h>
 #include <linux/workqueue.h>
 #include <trace/events/pci_controller.h>
diff --git a/drivers/pci/controller/dwc/pcie-histb.c b/drivers/pci/controller/dwc/pcie-histb.c
index a52071589377ae..432a54c5bfceaa 100644
--- a/drivers/pci/controller/dwc/pcie-histb.c
+++ b/drivers/pci/controller/dwc/pcie-histb.c
@@ -18,6 +18,7 @@
 #include <linux/pci.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
 #include <linux/resource.h>
 #include <linux/reset.h>
 
diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
index 56184e6ca6e690..8364696a1b980a 100644
--- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
+++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
@@ -19,6 +19,7 @@
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/reset.h>
 #include <linux/module.h>
diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
index b58a607b713f58..fe0cf6b6d966dd 100644
--- a/drivers/pci/controller/dwc/pcie-qcom.c
+++ b/drivers/pci/controller/dwc/pcie-qcom.c
@@ -20,6 +20,7 @@
 #include <linux/kernel.h>
 #include <linux/limits.h>
 #include <linux/init.h>
+#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_pci.h>
 #include <linux/pci.h>
@@ -73,6 +74,23 @@
 #define PARF_BDF_TO_SID_TABLE_N			0x2000
 #define PARF_BDF_TO_SID_CFG			0x2c00
 
+/*
+ * ECAM blocker range registers. The blocked range has a write pair
+ * (WR_BASE/WR_LIMIT) and a read pair (RD_BASE/RD_LIMIT); each address is
+ * split into a low (32-bit) and a HI (upper 32-bit) register.
+ */
+#define PARF_BLOCK_SLV_AXI_WR_BASE		0x360
+#define PARF_BLOCK_SLV_AXI_WR_BASE_HI		0x364
+#define PARF_BLOCK_SLV_AXI_WR_LIMIT		0x368
+#define PARF_BLOCK_SLV_AXI_WR_LIMIT_HI		0x36c
+#define PARF_BLOCK_SLV_AXI_RD_BASE		0x370
+#define PARF_BLOCK_SLV_AXI_RD_BASE_HI		0x374
+#define PARF_BLOCK_SLV_AXI_RD_LIMIT		0x378
+#define PARF_BLOCK_SLV_AXI_RD_LIMIT_HI		0x37c
+
+#define PARF_ECAM_BASE				0x380
+#define PARF_ECAM_BASE_HI			0x384
+
 /* ELBI registers */
 #define ELBI_SYS_CTRL				0x04
 #define ELBI_SYS_STTS				0x08
@@ -90,6 +108,7 @@
 
 /* PARF_SYS_CTRL register fields */
 #define MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN	BIT(29)
+#define ECAM_BLOCKER_EN				BIT(26)
 #define MST_WAKEUP_EN				BIT(13)
 #define SLV_WAKEUP_EN				BIT(12)
 #define MSTR_ACLK_CGC_DIS			BIT(10)
@@ -308,6 +327,7 @@ struct qcom_pcie {
 	struct gpio_desc *reset;
 	int global_irq;
 	bool use_pm_opp;
+	struct mutex reset_lock;
 };
 
 #define to_qcom_pcie(x)		dev_get_drvdata((x)->dev)
@@ -445,6 +465,25 @@ static void qcom_pcie_configure_dbi_atu_base(struct qcom_pcie *pcie)
 	}
 }
 
+static void qcom_pcie_init_ecam_blocker(struct qcom_pcie *pcie)
+{
+	struct dw_pcie *pci = pcie->pci;
+
+	/* ECAM base must match the DBI base address */
+	writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + PARF_ECAM_BASE);
+	writel(upper_32_bits(pci->dbi_phys_addr), pcie->parf + PARF_ECAM_BASE_HI);
+
+	writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_WR_BASE);
+	writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_WR_BASE_HI);
+	writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_WR_LIMIT);
+	writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_WR_LIMIT_HI);
+
+	writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_RD_BASE);
+	writel(0, pcie->parf + PARF_BLOCK_SLV_AXI_RD_BASE_HI);
+	writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_RD_LIMIT);
+	writel(U32_MAX, pcie->parf + PARF_BLOCK_SLV_AXI_RD_LIMIT_HI);
+}
+
 static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie)
 {
 	struct dw_pcie *pci = pcie->pci;
@@ -990,6 +1029,8 @@ static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie)
 
 	dw_pcie_dbi_ro_wr_dis(pci);
 
+	qcom_pcie_init_ecam_blocker(pcie);
+
 	return 0;
 }
 
@@ -1104,16 +1145,13 @@ static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie)
 
 	qcom_pcie_set_slot_cap(pcie->pci);
 
+	qcom_pcie_init_ecam_blocker(pcie);
+
 	return 0;
 }
 
 static int qcom_pcie_enable_aspm(struct pci_dev *pdev, void *userdata)
 {
-	/*
-	 * Downstream devices need to be in D0 state before enabling PCI PM
-	 * substates.
-	 */
-	pci_set_power_state_locked(pdev, PCI_D0);
 	pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL);
 
 	return 0;
@@ -1143,50 +1181,90 @@ static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie)
 
 static int qcom_pcie_config_sid_1_9_0(struct qcom_pcie *pcie)
 {
-	/* iommu map structure */
-	struct {
-		u32 bdf;
-		u32 phandle;
-		u32 smmu_sid;
-		u32 smmu_sid_len;
-	} *map;
 	void __iomem *bdf_to_sid_base = pcie->parf + PARF_BDF_TO_SID_TABLE_N;
 	struct device *dev = pcie->pci->dev;
+	struct device_node *iommu_np;
 	u8 qcom_pcie_crc8_table[CRC8_TABLE_SIZE];
-	int i, nr_map, size = 0;
-	u32 smmu_sid_base;
+	const __be32 *map;
+	u32 iommu_cells, entry_cells, phandle, smmu_sid_base;
+	int i, nr_cells, nr_map, size = 0;
 	u32 val;
 
-	of_get_property(dev->of_node, "iommu-map", &size);
-	if (!size)
+	map = of_get_property(dev->of_node, "iommu-map", &size);
+	if (!map || !size)
 		return 0;
 
+	if (size % sizeof(*map)) {
+		dev_err(dev, "Malformed iommu-map property\n");
+		return -EINVAL;
+	}
+	nr_cells = size / sizeof(*map);
+
+	/*
+	 * Each iommu-map entry is: rid-base (1 cell), phandle (1 cell),
+	 * IOMMU specifier (#iommu-cells cells), length (1 cell). Read
+	 * #iommu-cells from the IOMMU provider referenced by the first
+	 * entry to compute the per-entry stride.
+	 */
+	phandle = be32_to_cpu(map[1]);
+	iommu_np = of_find_node_by_phandle(phandle);
+	if (!iommu_np) {
+		dev_err(dev, "Failed to find IOMMU node in iommu-map\n");
+		return -ENODEV;
+	}
+
+	if (of_property_read_u32(iommu_np, "#iommu-cells", &iommu_cells))
+		iommu_cells = 1;
+	of_node_put(iommu_np);
+
+	entry_cells = 3 + iommu_cells;
+
+	/*
+	 * Retain backward compatibility with DTs that describe iommu-map
+	 * with 4-cell entries against an IOMMU declaring #iommu-cells = 2,
+	 * matching the fallback in drivers/of/base.c::of_check_bad_map().
+	 */
+	if (iommu_cells == 2 && !(nr_cells % 4)) {
+		bool legacy = true;
+
+		for (i = 0; i < nr_cells; i += 4) {
+			if (be32_to_cpu(map[i + 1]) != phandle ||
+			    be32_to_cpu(map[i + 3]) != 1) {
+				legacy = false;
+				break;
+			}
+		}
+
+		if (legacy) {
+			dev_warn_once(dev, "iommu-map has 1-cell entries with #iommu-cells=2, using 1-cell\n");
+			entry_cells = 4;
+		}
+	}
+
+	if (nr_cells % entry_cells) {
+		dev_err(dev, "Malformed iommu-map property\n");
+		return -EINVAL;
+	}
+	nr_map = nr_cells / entry_cells;
+
 	/* Enable BDF to SID translation by disabling bypass mode (default) */
 	val = readl(pcie->parf + PARF_BDF_TO_SID_CFG);
 	val &= ~BDF_TO_SID_BYPASS;
 	writel(val, pcie->parf + PARF_BDF_TO_SID_CFG);
 
-	map = kzalloc(size, GFP_KERNEL);
-	if (!map)
-		return -ENOMEM;
-
-	of_property_read_u32_array(dev->of_node, "iommu-map", (u32 *)map,
-				   size / sizeof(u32));
-
-	nr_map = size / (sizeof(*map));
-
 	crc8_populate_msb(qcom_pcie_crc8_table, QCOM_PCIE_CRC8_POLYNOMIAL);
 
 	/* Registers need to be zero out first */
 	memset_io(bdf_to_sid_base, 0, CRC8_TABLE_SIZE * sizeof(u32));
 
 	/* Extract the SMMU SID base from the first entry of iommu-map */
-	smmu_sid_base = map[0].smmu_sid;
+	smmu_sid_base = be32_to_cpu(map[2]);
 
 	/* Look for an available entry to hold the mapping */
 	for (i = 0; i < nr_map; i++) {
-		__be16 bdf_be = cpu_to_be16(map[i].bdf);
-		u32 val;
+		u32 bdf = be32_to_cpu(map[i * entry_cells]);
+		u32 sid = be32_to_cpu(map[i * entry_cells + 2]);
+		__be16 bdf_be = cpu_to_be16(bdf);
 		u8 hash;
 
 		hash = crc8(qcom_pcie_crc8_table, (u8 *)&bdf_be, sizeof(bdf_be), 0);
@@ -1208,12 +1286,10 @@ static int qcom_pcie_config_sid_1_9_0(struct qcom_pcie *pcie)
 		}
 
 		/* BDF [31:16] | SID [15:8] | NEXT [7:0] */
-		val = map[i].bdf << 16 | (map[i].smmu_sid - smmu_sid_base) << 8 | 0;
+		val = bdf << 16 | (sid - smmu_sid_base) << 8 | 0;
 		writel(val, bdf_to_sid_base + hash * sizeof(u32));
 	}
 
-	kfree(map);
-
 	return 0;
 }
 
@@ -1322,6 +1398,8 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie)
 	for (i = 0; i < 256; i++)
 		writel(0, pcie->parf + PARF_BDF_TO_SID_TABLE_N + (4 * i));
 
+	qcom_pcie_init_ecam_blocker(pcie);
+
 	return 0;
 }
 
@@ -1382,6 +1460,18 @@ static void qcom_pcie_configure_ports(struct qcom_pcie *pcie)
 		dw_pcie_program_t_power_on(pcie->pci, port->l1ss_t_power_on);
 }
 
+static void qcom_pcie_enable_ecam_blocker(struct qcom_pcie *pcie)
+{
+	u32 sys_ctrl;
+
+	sys_ctrl = readl(pcie->parf + PARF_SYS_CTRL);
+	sys_ctrl |= ECAM_BLOCKER_EN;
+	writel(sys_ctrl, pcie->parf + PARF_SYS_CTRL);
+
+	/* Flush the write so the blocker is enabled before this function returns */
+	readl(pcie->parf + PARF_SYS_CTRL);
+}
+
 static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
@@ -1775,6 +1865,8 @@ static int qcom_pcie_reset_root_port(struct pci_host_bridge *bridge,
 	u32 val;
 	int ret;
 
+	guard(mutex)(&pcie->reset_lock);
+
 	/* Wait for the pending transactions to be completed */
 	ret = readl_relaxed_poll_timeout(pcie->parf + PARF_STATUS, val,
 					 val & FLUSH_COMPLETED, 10,
@@ -1878,6 +1970,11 @@ static irqreturn_t qcom_pcie_global_irq_thread(int irq, void *data)
 
 	if (test_and_clear_bit(INT_ALL_LINK_DOWN, &status)) {
 		dev_dbg(dev, "Received Link down event\n");
+
+		mutex_lock(&pcie->reset_lock);
+		qcom_pcie_enable_ecam_blocker(pcie);
+		mutex_unlock(&pcie->reset_lock);
+
 		for_each_pci_bridge(port, pp->bridge->bus) {
 			if (pci_pcie_type(port) == PCI_EXP_TYPE_ROOT_PORT)
 				pci_host_handle_link_down(port);
@@ -1953,7 +2050,7 @@ static bool qcom_pcie_is_child_node(struct device *dev,
 	return false;
 }
 
-/* Parse PERST# from all nodes in depth first manner starting from @np */
+/* Collect PERST# GPIOs from PCI bridge nodes depth-first, starting at @np */
 static int qcom_pcie_parse_perst(struct qcom_pcie *pcie,
 				 struct qcom_pcie_port *port,
 				 struct device_node *np)
@@ -2019,6 +2116,9 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie,
 
 parse_child_node:
 	for_each_available_child_of_node_scoped(np, child) {
+		if (!of_node_is_type(child, "pci"))
+			continue;
+
 		ret = qcom_pcie_parse_perst(pcie, port, child);
 		if (ret)
 			return ret;
@@ -2204,6 +2304,10 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 
 	INIT_LIST_HEAD(&pcie->ports);
 
+	ret = devm_mutex_init(dev, &pcie->reset_lock);
+	if (ret)
+		goto err_pm_runtime_put;
+
 	pci->dev = dev;
 	pci->ops = &dw_pcie_ops;
 	pp = &pci->pp;
@@ -2341,6 +2445,10 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
 {
 	struct qcom_pcie *pcie;
 	int ret = 0;
+	const struct qcom_pcie_cfg *pcie_cfg = of_device_get_match_data(dev);
+
+	if (pcie_cfg && pcie_cfg->firmware_managed)
+		return 0;
 
 	pcie = dev_get_drvdata(dev);
 	if (!pcie)
@@ -2399,6 +2507,10 @@ static int qcom_pcie_resume_noirq(struct device *dev)
 {
 	struct qcom_pcie *pcie;
 	int ret;
+	const struct qcom_pcie_cfg *pcie_cfg = of_device_get_match_data(dev);
+
+	if (pcie_cfg && pcie_cfg->firmware_managed)
+		return 0;
 
 	pcie = dev_get_drvdata(dev);
 	if (!pcie)
diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c
index fbe465a29068fd..21c3dd9b88910d 100644
--- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c
+++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c
@@ -19,6 +19,7 @@
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/pci.h>
+#include <linux/phy/phy.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
@@ -36,6 +37,7 @@
 
 /* MSI Capability */
 #define MSICAP0			0x0050
+#define MSICAP0_MMESCAP_MASK	GENMASK(19, 17)
 #define MSICAP0_MSIE		BIT(16)
 
 /* PCIe Interrupt Status 0 */
@@ -74,6 +76,11 @@
 #define PCIEPWRMNGCTRL		0x0070
 #define APP_CLK_REQ_N		BIT(11)
 #define APP_CLK_PM_EN		BIT(10)
+#define APP_READY_ENTR_L23	BIT(6)
+#define APP_REQ_ENTR_L1		BIT(5)
+
+/* PCI Express capability */
+#define EXPCAP(x)		(0x0070 + (x))
 
 #define RCAR_NUM_SPEED_CHANGE_RETRIES	10
 #define RCAR_MAX_LINK_SPEED		4
@@ -87,8 +94,10 @@ MODULE_FIRMWARE(RCAR_GEN4_PCIE_FIRMWARE_NAME);
 
 struct rcar_gen4_pcie;
 struct rcar_gen4_pcie_drvdata {
-	void (*additional_common_init)(struct rcar_gen4_pcie *rcar);
+	int (*init)(struct rcar_gen4_pcie *rcar);
+	void (*deinit)(struct rcar_gen4_pcie *rcar);
 	int (*ltssm_control)(struct rcar_gen4_pcie *rcar, bool enable);
+	int (*speed_control)(struct rcar_gen4_pcie *rcar);
 	enum dw_pcie_device_mode mode;
 };
 
@@ -96,7 +105,9 @@ struct rcar_gen4_pcie {
 	struct dw_pcie dw;
 	void __iomem *base;
 	void __iomem *phy_base;
+	struct phy *phy;
 	struct platform_device *pdev;
+	struct reset_control *perst;
 	const struct rcar_gen4_pcie_drvdata *drvdata;
 };
 #define to_rcar_gen4_pcie(_dw)	container_of(_dw, struct rcar_gen4_pcie, dw)
@@ -140,20 +151,10 @@ static int rcar_gen4_pcie_speed_change(struct dw_pcie *dw)
 	return -ETIMEDOUT;
 }
 
-/*
- * Enable LTSSM of this controller and manually initiate the speed change.
- * Always return 0.
- */
-static int rcar_gen4_pcie_start_link(struct dw_pcie *dw)
+static int rcar_gen4_pcie_speed_control(struct rcar_gen4_pcie *rcar)
 {
-	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
-	int i, changes, ret;
-
-	if (rcar->drvdata->ltssm_control) {
-		ret = rcar->drvdata->ltssm_control(rcar, true);
-		if (ret)
-			return ret;
-	}
+	struct dw_pcie *dw = &rcar->dw;
+	int i, changes;
 
 	/*
 	 * Require direct speed change with retrying here if the max_link_speed
@@ -177,6 +178,45 @@ static int rcar_gen4_pcie_start_link(struct dw_pcie *dw)
 	return 0;
 }
 
+static int rcar_gen5_pcie_speed_control(struct rcar_gen4_pcie *rcar)
+{
+	struct dw_pcie *dw = &rcar->dw;
+	u32 lnkcap = dw_pcie_readl_dbi(dw, EXPCAP(PCI_EXP_LNKCAP));
+	u32 lnksta = dw_pcie_readw_dbi(dw, EXPCAP(PCI_EXP_LNKSTA));
+	u32 val;
+
+	if ((lnksta & PCI_EXP_LNKSTA_CLS) == (lnkcap & PCI_EXP_LNKCAP_SLS))
+		return 0;
+
+	/* Retrain link */
+	val = dw_pcie_readw_dbi(dw, EXPCAP(PCI_EXP_LNKCTL));
+	val |= PCI_EXP_LNKCTL_RL;
+	dw_pcie_writew_dbi(dw, EXPCAP(PCI_EXP_LNKCTL), val);
+
+	/* Wait for link retrain, 500ms must be enough for all link rates. */
+	return read_poll_timeout(dw_pcie_readw_dbi, lnksta, !(lnksta & PCI_EXP_LNKSTA_LT),
+				 1000, 5 * PCIE_RESET_CONFIG_WAIT_MS * USEC_PER_MSEC,
+				 false, dw, EXPCAP(PCI_EXP_LNKSTA));
+}
+
+/*
+ * Enable LTSSM of this controller and manually initiate the speed change.
+ * Always return 0.
+ */
+static int rcar_gen4_pcie_start_link(struct dw_pcie *dw)
+{
+	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
+	int ret;
+
+	if (rcar->drvdata->ltssm_control) {
+		ret = rcar->drvdata->ltssm_control(rcar, true);
+		if (ret)
+			return ret;
+	}
+
+	return rcar->drvdata->speed_control(rcar);
+}
+
 static void rcar_gen4_pcie_stop_link(struct dw_pcie *dw)
 {
 	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
@@ -197,6 +237,8 @@ static int rcar_gen4_pcie_common_init(struct rcar_gen4_pcie *rcar)
 		return ret;
 	}
 
+	reset_control_deassert(dw->core_rsts[DW_PCIE_CORE_RST].rstc);
+
 	if (!reset_control_status(dw->core_rsts[DW_PCIE_PWR_RST].rstc)) {
 		reset_control_assert(dw->core_rsts[DW_PCIE_PWR_RST].rstc);
 		/*
@@ -241,8 +283,74 @@ static int rcar_gen4_pcie_common_init(struct rcar_gen4_pcie *rcar)
 	reset_control_status(dw->core_rsts[DW_PCIE_PWR_RST].rstc);
 	fsleep(1000);
 
-	if (rcar->drvdata->additional_common_init)
-		rcar->drvdata->additional_common_init(rcar);
+	return 0;
+
+err_unprepare:
+	reset_control_assert(dw->core_rsts[DW_PCIE_CORE_RST].rstc);
+	clk_bulk_disable_unprepare(DW_PCIE_NUM_CORE_CLKS, dw->core_clks);
+
+	return ret;
+}
+
+static int rcar_gen4_v4h_v4m_pcie_init(struct rcar_gen4_pcie *rcar)
+{
+	struct dw_pcie *dw = &rcar->dw;
+	u32 val;
+	int ret;
+
+	/* R-Car Gen4 common initialization. */
+	ret = rcar_gen4_pcie_common_init(rcar);
+	if (ret)
+		return ret;
+
+	/* R-Car V4H and V4M specific additional initialization. */
+	val = dw_pcie_readl_dbi(dw, PCIE_PORT_LANE_SKEW);
+	val &= ~PORT_LANE_SKEW_INSERT_MASK;
+	if (dw->num_lanes < 4)
+		val |= BIT(6);
+	dw_pcie_writel_dbi(dw, PCIE_PORT_LANE_SKEW, val);
+
+	val = readl(rcar->base + PCIEPWRMNGCTRL);
+	val |= APP_CLK_REQ_N | APP_CLK_PM_EN;
+	writel(val, rcar->base + PCIEPWRMNGCTRL);
+
+	return 0;
+}
+
+static int rcar_gen5_pcie_init(struct rcar_gen4_pcie *rcar)
+{
+	struct dw_pcie *dw = &rcar->dw;
+	int ret;
+	u32 val;
+
+	/* R-Car Gen4 and Gen5 common initialization. */
+	ret = rcar_gen4_pcie_common_init(rcar);
+	if (ret)
+		return ret;
+
+	/* R-Car Gen5 specific additional initialization. */
+	ret = phy_init(rcar->phy);
+	if (ret)
+		goto err_unprepare;
+
+	dw_pcie_dbi_ro_wr_en(dw);
+
+	val = dw_pcie_readl_dbi(dw, PCIE_PORT_LANE_SKEW);
+	val &= ~PORT_LANE_SKEW_INSERT_MASK;
+	if (dw->num_lanes < 8)
+		val |= BIT(6);
+	dw_pcie_writel_dbi(dw, PCIE_PORT_LANE_SKEW, val);
+
+	val = dw_pcie_readl_dbi(dw, MSICAP0);
+	FIELD_MODIFY(MSICAP0_MMESCAP_MASK, &val, 4);
+	dw_pcie_writel_dbi(dw, MSICAP0, val);
+
+	dw_pcie_dbi_ro_wr_dis(dw);
+
+	val = readl(rcar->base + PCIEPWRMNGCTRL);
+	val |= APP_CLK_REQ_N | APP_CLK_PM_EN |
+	       APP_READY_ENTR_L23 | APP_REQ_ENTR_L1;
+	writel(val, rcar->base + PCIEPWRMNGCTRL);
 
 	return 0;
 
@@ -257,9 +365,16 @@ static void rcar_gen4_pcie_common_deinit(struct rcar_gen4_pcie *rcar)
 	struct dw_pcie *dw = &rcar->dw;
 
 	reset_control_assert(dw->core_rsts[DW_PCIE_PWR_RST].rstc);
+	reset_control_assert(dw->core_rsts[DW_PCIE_CORE_RST].rstc);
 	clk_bulk_disable_unprepare(DW_PCIE_NUM_CORE_CLKS, dw->core_clks);
 }
 
+static void rcar_gen5_pcie_deinit(struct rcar_gen4_pcie *rcar)
+{
+	phy_exit(rcar->phy);
+	rcar_gen4_pcie_common_deinit(rcar);
+}
+
 static int rcar_gen4_pcie_prepare(struct rcar_gen4_pcie *rcar)
 {
 	struct device *dev = rcar->dw.dev;
@@ -285,12 +400,29 @@ static void rcar_gen4_pcie_unprepare(struct rcar_gen4_pcie *rcar)
 
 static int rcar_gen4_pcie_get_resources(struct rcar_gen4_pcie *rcar)
 {
+	struct device *dev = rcar->dw.dev;
+	struct device_node *root_port;
+
 	rcar->phy_base = devm_platform_ioremap_resource_byname(rcar->pdev, "phy");
-	if (IS_ERR(rcar->phy_base))
-		return PTR_ERR(rcar->phy_base);
+	if (IS_ERR(rcar->phy_base)) {
+		rcar->phy_base = NULL;
+		rcar->phy = devm_phy_get(dev, NULL);
+		if (IS_ERR(rcar->phy))
+			return PTR_ERR(rcar->phy);
+	}
+
+	root_port = of_get_next_available_child(dev->of_node, NULL);
+	if (root_port) {
+		rcar->perst = of_reset_control_get_optional_exclusive(root_port, "perst");
+		of_node_put(root_port);
+		if (IS_ERR(rcar->perst))
+			return dev_err_probe(dev, PTR_ERR(rcar->perst), "Failed to get PERST#\n");
+	}
 
 	/* Renesas-specific registers */
 	rcar->base = devm_platform_ioremap_resource_byname(rcar->pdev, "app");
+	if (IS_ERR(rcar->base))
+		reset_control_put(rcar->perst);
 
 	return PTR_ERR_OR_ZERO(rcar->base);
 }
@@ -411,6 +543,72 @@ static int rcar_gen4_pcie_host_msi_init(struct dw_pcie_rp *pp)
 	return ret;
 }
 
+static int rcar_gen4_pcie_enable_device(struct pci_host_bridge *bridge,
+					struct pci_dev *dev)
+{
+	/*
+	 * R-Car Gen4 PCIe controller has a hardware limitation of 256 Bytes
+	 * Max_Payload_Size (MPS). PCIe specification indicates that the MPS
+	 * must not exceed minimum MPS of any element along the packet path.
+	 * The controller reports Max_Payload_Size_Supported (MPSS) 256 Bytes
+	 * for header type 0 and 128 Bytes for header type 1. The PCIe core
+	 * will not allow MPS to be set higher than MPSS; warn here in case
+	 * something went very wrong in the core.
+	 *
+	 * For details, refer to chapter "104.1.1 Features" in either of:
+	 * R-Car S4 R19UH0161EJ0140 Rev.1.40 Jul. 31, 2026 or
+	 * R-Car V4H R19UH0186EJ0140 Rev.1.40 Aug. 7, 2026 or
+	 * R-Car V4M R19UH0217EJ0110 Rev.1.10 Jun. 30, 2026.
+	 */
+	WARN_ON(pcie_get_mps(dev) > 256);
+
+	/*
+	 * R-Car Gen4 Reference Manual, chapter 104.4.8 Usage notes for
+	 * MRRS (Max Read Request Size) states:
+	 *
+	 *   Please set "Max Read Request Size" to 128 bytes or 256 bytes.
+	 *   If "Max Read Request Size" is set to anything other than the
+	 *   above, the transferred data will not match the expected value.
+	 *
+	 * This limitation also seems to apply to devices issuing MRd TLPs.
+	 * This limitation can be triggered by using non-HMB NVMe SSD with
+	 * Max_Read_Request_Size 512 Bytes, for example Crucial P5 Plus.
+	 * Any write into the SSD (MRd TLP issued by the SSD) longer than
+	 * 256 Bytes wraps around at 256 Byte boundary, and the same data
+	 * are written into the SSD starting at offset 0 and at 256 Bytes.
+	 *
+	 * Limit Max_Read_Request_Size to at most 256 Bytes for each
+	 * device connected to this PCIe controller to avoid this behavior.
+	 *
+	 * For details, refer to aforementioned chapter in either of:
+	 * R-Car S4 R19UH0161EJ0140 Rev.1.40 Jul. 31, 2026 or
+	 * R-Car V4H R19UH0186EJ0140 Rev.1.40 Aug. 7, 2026 or
+	 * R-Car V4M R19UH0217EJ0110 Rev.1.10 Jun. 30, 2026.
+	 */
+	bridge->no_inc_mrrs = 1;
+	if (pcie_get_readrq(dev) > 256) {
+		pci_info(dev, "Limiting MRRS to 256 bytes\n");
+		pcie_set_readrq(dev, 256);
+	}
+
+	return 0;
+}
+
+static void rcar_gen4_pcie_host_perst_assert(struct dw_pcie_rp *pp, bool assert)
+{
+	struct dw_pcie *dw = to_dw_pcie_from_pp(pp);
+	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
+
+	if (dw->pe_rst) {
+		gpiod_set_value_cansleep(dw->pe_rst, assert);
+	} else {
+		if (assert)
+			reset_control_assert(rcar->perst);
+		else
+			reset_control_deassert(rcar->perst);
+	}
+}
+
 /* Host mode */
 static int rcar_gen4_pcie_host_init(struct dw_pcie_rp *pp)
 {
@@ -418,9 +616,12 @@ static int rcar_gen4_pcie_host_init(struct dw_pcie_rp *pp)
 	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
 	int ret;
 
-	gpiod_set_value_cansleep(dw->pe_rst, 1);
+	if (pp->bridge)
+		pp->bridge->enable_device = rcar_gen4_pcie_enable_device;
 
-	ret = rcar_gen4_pcie_common_init(rcar);
+	rcar_gen4_pcie_host_perst_assert(pp, true);
+
+	ret = rcar->drvdata->init(rcar);
 	if (ret)
 		return ret;
 
@@ -439,12 +640,12 @@ static int rcar_gen4_pcie_host_init(struct dw_pcie_rp *pp)
 
 	msleep(PCIE_T_PVPERL_MS);	/* pe_rst requires 100msec delay */
 
-	gpiod_set_value_cansleep(dw->pe_rst, 0);
+	rcar_gen4_pcie_host_perst_assert(pp, false);
 
 	return 0;
 
 err:
-	rcar_gen4_pcie_common_deinit(rcar);
+	rcar->drvdata->deinit(rcar);
 	return ret;
 }
 
@@ -453,8 +654,8 @@ static void rcar_gen4_pcie_host_deinit(struct dw_pcie_rp *pp)
 	struct dw_pcie *dw = to_dw_pcie_from_pp(pp);
 	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
 
-	gpiod_set_value_cansleep(dw->pe_rst, 1);
-	rcar_gen4_pcie_common_deinit(rcar);
+	rcar_gen4_pcie_host_perst_assert(pp, true);
+	rcar->drvdata->deinit(rcar);
 }
 
 static const struct dw_pcie_host_ops rcar_gen4_pcie_host_ops = {
@@ -487,7 +688,9 @@ static int rcar_gen4_pcie_ep_pre_init(struct dw_pcie_ep *ep)
 	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
 	int ret;
 
-	ret = rcar_gen4_pcie_common_init(rcar);
+	writel(0, rcar->base + PCIEDMAINTSTSEN);
+
+	ret = rcar->drvdata->init(rcar);
 	if (ret)
 		return ret;
 
@@ -496,10 +699,13 @@ static int rcar_gen4_pcie_ep_pre_init(struct dw_pcie_ep *ep)
 	return 0;
 }
 
-static void rcar_gen4_pcie_ep_deinit(struct rcar_gen4_pcie *rcar)
+static void rcar_gen4_pcie_ep_post_deinit(struct dw_pcie_ep *ep)
 {
+	struct dw_pcie *dw = to_dw_pcie_from_ep(ep);
+	struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw);
+
 	writel(0, rcar->base + PCIEDMAINTSTSEN);
-	rcar_gen4_pcie_common_deinit(rcar);
+	rcar->drvdata->deinit(rcar);
 }
 
 static int rcar_gen4_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
@@ -552,6 +758,7 @@ static unsigned int rcar_gen4_pcie_ep_get_dbi2_offset(struct dw_pcie_ep *ep,
 
 static const struct dw_pcie_ep_ops pcie_ep_ops = {
 	.pre_init = rcar_gen4_pcie_ep_pre_init,
+	.post_deinit = rcar_gen4_pcie_ep_post_deinit,
 	.raise_irq = rcar_gen4_pcie_ep_raise_irq,
 	.get_features = rcar_gen4_pcie_ep_get_features,
 	.get_dbi_offset = rcar_gen4_pcie_ep_get_dbi_offset,
@@ -570,16 +777,13 @@ static int rcar_gen4_add_dw_pcie_ep(struct rcar_gen4_pcie *rcar)
 	ep->ops = &pcie_ep_ops;
 
 	ret = dw_pcie_ep_init(ep);
-	if (ret) {
-		rcar_gen4_pcie_ep_deinit(rcar);
+	if (ret)
 		return ret;
-	}
 
 	ret = dw_pcie_ep_init_registers(ep);
 	if (ret) {
 		dev_err(dev, "Failed to initialize DWC endpoint registers\n");
 		dw_pcie_ep_deinit(ep);
-		rcar_gen4_pcie_ep_deinit(rcar);
 	}
 
 	pci_epc_init_notify(ep->epc);
@@ -590,7 +794,6 @@ static int rcar_gen4_add_dw_pcie_ep(struct rcar_gen4_pcie *rcar)
 static void rcar_gen4_remove_dw_pcie_ep(struct rcar_gen4_pcie *rcar)
 {
 	dw_pcie_ep_deinit(&rcar->dw.ep);
-	rcar_gen4_pcie_ep_deinit(rcar);
 }
 
 /* Common */
@@ -625,7 +828,7 @@ static int rcar_gen4_pcie_probe(struct platform_device *pdev)
 
 	err = rcar_gen4_pcie_prepare(rcar);
 	if (err)
-		return err;
+		goto err_prepare;
 
 	err = rcar_gen4_add_dw_pcie(rcar);
 	if (err)
@@ -636,6 +839,9 @@ static int rcar_gen4_pcie_probe(struct platform_device *pdev)
 err_unprepare:
 	rcar_gen4_pcie_unprepare(rcar);
 
+err_prepare:
+	reset_control_put(rcar->perst);
+
 	return err;
 }
 
@@ -659,6 +865,7 @@ static void rcar_gen4_pcie_remove(struct platform_device *pdev)
 
 	rcar_gen4_remove_dw_pcie(rcar);
 	rcar_gen4_pcie_unprepare(rcar);
+	reset_control_put(rcar->perst);
 }
 
 static int r8a779f0_pcie_ltssm_control(struct rcar_gen4_pcie *rcar, bool enable)
@@ -683,20 +890,26 @@ static int r8a779f0_pcie_ltssm_control(struct rcar_gen4_pcie *rcar, bool enable)
 	return 0;
 }
 
-static void rcar_gen4_pcie_additional_common_init(struct rcar_gen4_pcie *rcar)
+static int rcar_gen5_pcie_ltssm_control(struct rcar_gen4_pcie *rcar, bool enable)
 {
-	struct dw_pcie *dw = &rcar->dw;
 	u32 val;
 
-	val = dw_pcie_readl_dbi(dw, PCIE_PORT_LANE_SKEW);
-	val &= ~PORT_LANE_SKEW_INSERT_MASK;
-	if (dw->num_lanes < 4)
-		val |= BIT(6);
-	dw_pcie_writel_dbi(dw, PCIE_PORT_LANE_SKEW, val);
+	val = readl(rcar->base + PCIERSTCTRL1);
+	if (enable) {
+		val |= APP_LTSSM_ENABLE;
+		val &= ~APP_HOLD_PHY_RST;
+	} else {
+		val &= ~APP_LTSSM_ENABLE;
+		val |= APP_HOLD_PHY_RST;
+	}
+	writel(val, rcar->base + PCIERSTCTRL1);
 
-	val = readl(rcar->base + PCIEPWRMNGCTRL);
-	val |= APP_CLK_REQ_N | APP_CLK_PM_EN;
-	writel(val, rcar->base + PCIEPWRMNGCTRL);
+	if (enable)
+		phy_power_on(rcar->phy);
+	else
+		phy_power_off(rcar->phy);
+
+	return 0;
 }
 
 static void rcar_gen4_pcie_phy_reg_update_bits(struct rcar_gen4_pcie *rcar,
@@ -850,27 +1063,45 @@ static int rcar_gen4_pcie_ltssm_control(struct rcar_gen4_pcie *rcar, bool enable
 }
 
 static struct rcar_gen4_pcie_drvdata drvdata_r8a779f0_pcie = {
+	.init = rcar_gen4_pcie_common_init,
+	.deinit = rcar_gen4_pcie_common_deinit,
 	.ltssm_control = r8a779f0_pcie_ltssm_control,
+	.speed_control = rcar_gen4_pcie_speed_control,
 	.mode = DW_PCIE_RC_TYPE,
 };
 
 static struct rcar_gen4_pcie_drvdata drvdata_r8a779f0_pcie_ep = {
+	.init = rcar_gen4_pcie_common_init,
+	.deinit = rcar_gen4_pcie_common_deinit,
 	.ltssm_control = r8a779f0_pcie_ltssm_control,
+	.speed_control = rcar_gen4_pcie_speed_control,
 	.mode = DW_PCIE_EP_TYPE,
 };
 
 static struct rcar_gen4_pcie_drvdata drvdata_rcar_gen4_pcie = {
-	.additional_common_init = rcar_gen4_pcie_additional_common_init,
+	.init = rcar_gen4_v4h_v4m_pcie_init,
+	.deinit = rcar_gen4_pcie_common_deinit,
 	.ltssm_control = rcar_gen4_pcie_ltssm_control,
+	.speed_control = rcar_gen4_pcie_speed_control,
 	.mode = DW_PCIE_RC_TYPE,
 };
 
 static struct rcar_gen4_pcie_drvdata drvdata_rcar_gen4_pcie_ep = {
-	.additional_common_init = rcar_gen4_pcie_additional_common_init,
+	.init = rcar_gen4_v4h_v4m_pcie_init,
+	.deinit = rcar_gen4_pcie_common_deinit,
 	.ltssm_control = rcar_gen4_pcie_ltssm_control,
+	.speed_control = rcar_gen4_pcie_speed_control,
 	.mode = DW_PCIE_EP_TYPE,
 };
 
+static struct rcar_gen4_pcie_drvdata drvdata_rcar_gen5_pcie = {
+	.init = rcar_gen5_pcie_init,
+	.deinit = rcar_gen5_pcie_deinit,
+	.ltssm_control = rcar_gen5_pcie_ltssm_control,
+	.speed_control = rcar_gen5_pcie_speed_control,
+	.mode = DW_PCIE_RC_TYPE,
+};
+
 static const struct of_device_id rcar_gen4_pcie_of_match[] = {
 	{
 		.compatible = "renesas,r8a779f0-pcie",
@@ -888,6 +1119,10 @@ static const struct of_device_id rcar_gen4_pcie_of_match[] = {
 		.compatible = "renesas,rcar-gen4-pcie-ep",
 		.data = &drvdata_rcar_gen4_pcie_ep,
 	},
+	{
+		.compatible = "renesas,rcar-gen5-pcie4",
+		.data = &drvdata_rcar_gen5_pcie,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, rcar_gen4_pcie_of_match);
diff --git a/drivers/pci/controller/dwc/pcie-spacemit-k1.c b/drivers/pci/controller/dwc/pcie-spacemit-k1.c
index 0564c46e2f484c..92a71ea90ac313 100644
--- a/drivers/pci/controller/dwc/pcie-spacemit-k1.c
+++ b/drivers/pci/controller/dwc/pcie-spacemit-k1.c
@@ -12,9 +12,12 @@
 #include <linux/err.h>
 #include <linux/gfp.h>
 #include <linux/mfd/syscon.h>
+#include <linux/of.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <linux/reset.h>
 #include <linux/types.h>
 
diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
index 63a0f56cad8ea3..8ab23d1d7bd6b3 100644
--- a/drivers/pci/controller/dwc/pcie-tegra194.c
+++ b/drivers/pci/controller/dwc/pcie-tegra194.c
@@ -27,6 +27,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/random.h>
+#include <linux/regulator/consumer.h>
 #include <linux/reset.h>
 #include <linux/resource.h>
 #include <linux/types.h>
diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
index 89816a2bd7cd3a..85d826f8f68dba 100644
--- a/drivers/pci/controller/pci-hyperv.c
+++ b/drivers/pci/controller/pci-hyperv.c
@@ -1040,19 +1040,38 @@ static void put_pcichild(struct hv_pci_dev *hpdev)
 
 /*
  * There is no good way to get notified from vmbus_onoffer_rescind(),
- * so let's use polling here, since this is not a hot path.
+ * so let's use polling here, since this is not a hot path. If
+ * wait_for_response() has been polling for 2 minutes
+ * without either a rescind or completion, add a warning.
  */
+#define PCI_RESPONSE_WARN_POLL_COUNT 1200
+
 static int wait_for_response(struct hv_device *hdev,
-			     struct completion *comp)
+			     struct completion *comp,
+			     const char *msg_type)
 {
+	u64 counter = 0;
+
 	while (true) {
 		if (hdev->channel->rescind) {
 			dev_warn_once(&hdev->device, "The device is gone.\n");
 			return -ENODEV;
 		}
 
-		if (wait_for_completion_timeout(comp, HZ / 10))
+		counter++;
+
+		if (wait_for_completion_timeout(comp, HZ / 10)) {
+			if (counter > PCI_RESPONSE_WARN_POLL_COUNT)
+				dev_warn(&hdev->device,
+					 "Late %s completion arrived.\n", msg_type);
 			break;
+		}
+
+		if (counter == PCI_RESPONSE_WARN_POLL_COUNT) {
+			dev_err(&hdev->device,
+				"%s stuck waiting for response, relid = %u\n",
+				msg_type, hdev->channel->offermsg.child_relid);
+		}
 	}
 
 	return 0;
@@ -1518,7 +1537,8 @@ static int hv_read_config_block(struct pci_dev *pdev, void *buf,
 	if (ret)
 		return ret;
 
-	ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event);
+	ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event,
+				"PCI_READ_BLOCK");
 	if (ret)
 		return ret;
 
@@ -1607,7 +1627,8 @@ static int hv_write_config_block(struct pci_dev *pdev, void *buf,
 	if (ret)
 		return ret;
 
-	ret = wait_for_response(hbus->hdev, &comp_pkt.host_event);
+	ret = wait_for_response(hbus->hdev, &comp_pkt.host_event,
+				"PCI_WRITE_BLOCK");
 	if (ret)
 		return ret;
 
@@ -2624,7 +2645,8 @@ static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus,
 	if (ret)
 		goto error;
 
-	if (wait_for_response(hbus->hdev, &comp_pkt.host_event))
+	if (wait_for_response(hbus->hdev, &comp_pkt.host_event,
+			      "PCI_QUERY_RESOURCE_REQUIREMENTS"))
 		goto error;
 
 	hpdev->desc = *desc;
@@ -3256,7 +3278,8 @@ static int hv_pci_protocol_negotiation(struct hv_device *hdev,
 				(unsigned long)pkt, VM_PKT_DATA_INBAND,
 				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 		if (!ret)
-			ret = wait_for_response(hdev, &comp_pkt.host_event);
+			ret = wait_for_response(hdev, &comp_pkt.host_event,
+						"PCI_QUERY_PROTOCOL_VERSION");
 
 		if (ret) {
 			dev_err(&hdev->device,
@@ -3476,7 +3499,8 @@ static int hv_pci_enter_d0(struct hv_device *hdev)
 			       (unsigned long)pkt, VM_PKT_DATA_INBAND,
 			       VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 	if (!ret)
-		ret = wait_for_response(hdev, &comp_pkt.host_event);
+		ret = wait_for_response(hdev, &comp_pkt.host_event,
+					"PCI_BUS_D0ENTRY");
 
 	if (ret)
 		goto exit;
@@ -3553,7 +3577,8 @@ static int hv_pci_query_relations(struct hv_device *hdev)
 	ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message),
 			       0, VM_PKT_DATA_INBAND, 0);
 	if (!ret)
-		ret = wait_for_response(hdev, &comp);
+		ret = wait_for_response(hdev, &comp,
+					"PCI_QUERY_BUS_RELATIONS");
 
 	/*
 	 * In the case of fast device addition/removal, it's possible that
@@ -3644,7 +3669,8 @@ static int hv_send_resources_allocated(struct hv_device *hdev)
 				VM_PKT_DATA_INBAND,
 				VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED);
 		if (!ret)
-			ret = wait_for_response(hdev, &comp_pkt.host_event);
+			ret = wait_for_response(hdev, &comp_pkt.host_event,
+						"PCI_RESOURCE_ASSIGNED");
 		if (ret)
 			break;
 
@@ -4155,6 +4181,9 @@ static struct hv_driver hv_pci_drv = {
 	.remove		= hv_pci_remove,
 	.suspend	= hv_pci_suspend,
 	.resume		= hv_pci_resume,
+	.driver = {
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
 };
 
 static void __exit exit_hv_pci_drv(void)
diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c
index 59b56446c507ee..96bd522fa67b38 100644
--- a/drivers/pci/controller/pci-tegra.c
+++ b/drivers/pci/controller/pci-tegra.c
@@ -36,6 +36,7 @@
 #include <linux/phy/phy.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/reset.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c
index 83c9a2930eecaf..a5b8bbc04a2214 100644
--- a/drivers/pci/controller/pci-xgene.c
+++ b/drivers/pci/controller/pci-xgene.c
@@ -320,18 +320,11 @@ static void xgene_pcie_linkup(struct xgene_pcie *port,
 static int xgene_pcie_init_port(struct xgene_pcie *port)
 {
 	struct device *dev = port->dev;
-	int rc;
 
-	port->clk = clk_get(dev, NULL);
+	port->clk = devm_clk_get_enabled(dev, NULL);
 	if (IS_ERR(port->clk)) {
 		dev_err(dev, "clock not available\n");
-		return -ENODEV;
-	}
-
-	rc = clk_prepare_enable(port->clk);
-	if (rc) {
-		dev_err(dev, "clock enable failed\n");
-		return rc;
+		return PTR_ERR(port->clk);
 	}
 
 	return 0;
diff --git a/drivers/pci/controller/pcie-aspeed.c b/drivers/pci/controller/pcie-aspeed.c
index 74b03a55643fa7..ecb7562f84c01f 100644
--- a/drivers/pci/controller/pcie-aspeed.c
+++ b/drivers/pci/controller/pcie-aspeed.c
@@ -761,16 +761,21 @@ static int aspeed_pcie_port_init(struct aspeed_pcie_port *port)
 				     port->slot);
 
 	ret = phy_init(port->phy);
-	if (ret)
+	if (ret) {
+		clk_disable_unprepare(port->clk);
 		return dev_err_probe(dev, ret,
 				     "failed to init phy pcie for slot (%d)\n",
 				     port->slot);
+	}
 
 	ret = phy_set_mode_ext(port->phy, PHY_MODE_PCIE, PHY_MODE_PCIE_RC);
-	if (ret)
+	if (ret) {
+		phy_exit(port->phy);
+		clk_disable_unprepare(port->clk);
 		return dev_err_probe(dev, ret,
 				     "failed to set phy mode for slot (%d)\n",
 				     port->slot);
+	}
 
 	reset_control_deassert(port->perst);
 	msleep(PCIE_RESET_CONFIG_WAIT_MS);
diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c
index bcbd08c82a5ce4..e2fb6e372ede5f 100644
--- a/drivers/pci/controller/pcie-mediatek-gen3.c
+++ b/drivers/pci/controller/pcie-mediatek-gen3.c
@@ -7,6 +7,8 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/bits.h>
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/delay.h>
@@ -334,10 +336,10 @@ static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie,
 
 	while (remaining && (*num < PCIE_MAX_TRANS_TABLES)) {
 		/* Table size needs to be a power of 2 */
-		table_size = BIT(fls(remaining) - 1);
+		table_size = BIT_ULL(fls64(remaining) - 1);
 
 		if (cpu_addr > 0) {
-			addr_align = BIT(ffs(cpu_addr) - 1);
+			addr_align = BIT_ULL(__ffs64(cpu_addr));
 			table_size = min(table_size, addr_align);
 		}
 
@@ -349,7 +351,8 @@ static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie,
 		}
 
 		table = pcie->base + PCIE_TRANS_TABLE_BASE_REG + *num * PCIE_ATR_TLB_SET_OFFSET;
-		writel_relaxed(lower_32_bits(cpu_addr) | PCIE_ATR_SIZE(fls(table_size) - 1), table);
+		writel_relaxed(lower_32_bits(cpu_addr) |
+		PCIE_ATR_SIZE(fls64(table_size) - 1), table);
 		writel_relaxed(upper_32_bits(cpu_addr), table + PCIE_ATR_SRC_ADDR_MSB_OFFSET);
 		writel_relaxed(lower_32_bits(pci_addr), table + PCIE_ATR_TRSL_ADDR_LSB_OFFSET);
 		writel_relaxed(upper_32_bits(pci_addr), table + PCIE_ATR_TRSL_ADDR_MSB_OFFSET);
@@ -375,9 +378,11 @@ static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie,
 		(*num)++;
 	}
 
-	if (remaining)
-		dev_warn(pcie->dev, "not enough translate table for addr: %#llx, limited to [%d]\n",
+	if (remaining) {
+		dev_err(pcie->dev, "not enough translate table for addr: %#llx, limited to [%d]\n",
 			 (unsigned long long)cpu_addr, PCIE_MAX_TRANS_TABLES);
+		return -ENOMEM;
+	}
 
 	return 0;
 }
diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
index a60d1ae076f802..3a3e91cd0c2113 100644
--- a/drivers/pci/controller/pcie-mediatek.c
+++ b/drivers/pci/controller/pcie-mediatek.c
@@ -583,9 +583,9 @@ static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port,
 	int ret;
 
 	/* Setup INTx */
-	pcie_intc_node = of_get_next_child(node, NULL);
+	pcie_intc_node = of_get_child_by_name(node, "interrupt-controller");
 	if (!pcie_intc_node) {
-		dev_err(dev, "no PCIe Intc node found\n");
+		dev_err(dev, "no PCIe INTx interrupt-controller node found\n");
 		return -ENODEV;
 	}
 
diff --git a/drivers/pci/controller/pcie-rockchip-host.c b/drivers/pci/controller/pcie-rockchip-host.c
index d203c4876d3041..be8ff99b243f93 100644
--- a/drivers/pci/controller/pcie-rockchip-host.c
+++ b/drivers/pci/controller/pcie-rockchip-host.c
@@ -24,6 +24,7 @@
 #include <linux/of_pci.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
 
 #include "../pci.h"
 #include "pcie-rockchip.h"
diff --git a/drivers/pci/controller/pcie-rzg3s-host.c b/drivers/pci/controller/pcie-rzg3s-host.c
index a1a78bdb7af119..077cfb0834b378 100644
--- a/drivers/pci/controller/pcie-rzg3s-host.c
+++ b/drivers/pci/controller/pcie-rzg3s-host.c
@@ -1006,22 +1006,38 @@ static const struct irq_domain_ops rzg3s_pcie_intx_domain_ops = {
 	.xlate = irq_domain_xlate_onetwocell,
 };
 
+static void rzg3s_pcie_teardown_intx(struct rzg3s_pcie_host *host,
+				     int count)
+{
+	while (--count >= 0) {
+		irq_set_chained_handler_and_data(host->intx_irqs[count], NULL,
+						 NULL);
+	}
+
+	if (host->intx_domain)
+		irq_domain_remove(host->intx_domain);
+}
+
 static int rzg3s_pcie_init_irqdomain(struct rzg3s_pcie_host *host)
 {
 	struct device *dev = host->dev;
 	struct platform_device *pdev = to_platform_device(dev);
+	int i, ret;
 
-	for (int i = 0; i < PCI_NUM_INTX; i++) {
+	for (i = 0; i < PCI_NUM_INTX; i++) {
 		char irq_name[5] = {0};
 		int irq;
 
 		scnprintf(irq_name, ARRAY_SIZE(irq_name), "int%c", 'a' + i);
 
 		irq = platform_get_irq_byname(pdev, irq_name);
-		if (irq < 0)
-			return dev_err_probe(dev, -EINVAL,
-					     "Failed to parse and map INT%c IRQ\n",
-					     'A' + i);
+		if (irq < 0) {
+			ret = irq;
+			dev_err_probe(dev, ret,
+				      "Failed to parse and map INT%c IRQ\n",
+				      'A' + i);
+			goto teardown_intx;
+		}
 
 		host->intx_irqs[i] = irq;
 		irq_set_chained_handler_and_data(irq,
@@ -1033,21 +1049,27 @@ static int rzg3s_pcie_init_irqdomain(struct rzg3s_pcie_host *host)
 						     PCI_NUM_INTX,
 						     &rzg3s_pcie_intx_domain_ops,
 						     host);
-	if (!host->intx_domain)
-		return dev_err_probe(dev, -EINVAL,
-				     "Failed to add irq domain for INTx IRQs\n");
+	if (!host->intx_domain) {
+		ret = -EINVAL;
+		dev_err_probe(dev, ret,
+			      "Failed to add irq domain for INTx IRQs\n");
+		goto teardown_intx;
+	}
 	irq_domain_update_bus_token(host->intx_domain, DOMAIN_BUS_WIRED);
 
 	if (IS_ENABLED(CONFIG_PCI_MSI)) {
-		int ret = rzg3s_pcie_init_msi(host);
+		ret = rzg3s_pcie_init_msi(host);
 
-		if (ret) {
-			irq_domain_remove(host->intx_domain);
-			return ret;
-		}
+		if (ret)
+			goto teardown_intx;
 	}
 
 	return 0;
+
+teardown_intx:
+	rzg3s_pcie_teardown_intx(host, i);
+
+	return ret;
 }
 
 static void rzg3s_pcie_teardown_irqdomain(struct rzg3s_pcie_host *host)
@@ -1055,7 +1077,7 @@ static void rzg3s_pcie_teardown_irqdomain(struct rzg3s_pcie_host *host)
 	if (IS_ENABLED(CONFIG_PCI_MSI))
 		rzg3s_pcie_teardown_msi(host);
 
-	irq_domain_remove(host->intx_domain);
+	rzg3s_pcie_teardown_intx(host, PCI_NUM_INTX);
 }
 
 static int rzg3s_pcie_set_max_link_speed(struct rzg3s_pcie_host *host)
@@ -1077,7 +1099,7 @@ static int rzg3s_pcie_set_max_link_speed(struct rzg3s_pcie_host *host)
 				 FIELD_GET(RZG3S_PCI_PCSTAT1_LTSSM_STATE, tmp) == ltssm_state_l0,
 				 PCIE_LINK_WAIT_SLEEP_MS * MILLI,
 				 PCIE_LINK_WAIT_SLEEP_MS * MILLI *
-				 PCIE_LINK_WAIT_MAX_RETRIES);
+				 PCIE_LINK_WAIT_MAX_RETRIES * 10);
 	if (ret)
 		return ret;
 
@@ -1897,6 +1919,7 @@ static int rzg3s_pcie_probe(struct platform_device *pdev)
 	return 0;
 
 host_probe_teardown:
+	clk_disable_unprepare(host->port.refclk);
 	rzg3s_pcie_teardown_irqdomain(host);
 	host->data->config_deinit(host);
 rpm_put:
@@ -2074,6 +2097,25 @@ static const struct rzg3s_pcie_soc_data rzg3e_soc_data = {
 	},
 };
 
+static const struct rzg3s_pcie_soc_data rzg3l_soc_data = {
+	.power_resets = rzg3e_soc_power_resets,
+	.num_power_resets = ARRAY_SIZE(rzg3e_soc_power_resets),
+	.num_pcie_controllers = 1,
+	.config_pre_init = rzg3e_pcie_config_pre_init,
+	.config_post_init = rzg3e_pcie_config_post_init,
+	.config_deinit = rzg3e_pcie_config_deinit,
+	.sysc_info = {
+		[RZG3S_PCIE_CONTROLLER_ID_0] = {
+			.functions = {
+				[RZG3S_SYSC_FUNC_ID_L1_ALLOW] = {
+					.offset = 0x3a0,
+					.mask = BIT(8),
+				},
+			},
+		},
+	},
+};
+
 static const struct rzg3s_pcie_soc_data rzv2h_soc_data = {
 	.power_resets = rzg3e_soc_power_resets,
 	.num_power_resets = ARRAY_SIZE(rzg3e_soc_power_resets),
@@ -2123,6 +2165,10 @@ static const struct of_device_id rzg3s_pcie_of_match[] = {
 		.compatible = "renesas,r9a08g045-pcie",
 		.data = &rzg3s_soc_data,
 	},
+	{
+		.compatible = "renesas,r9a08g046-pcie",
+		.data = &rzg3l_soc_data,
+	},
 	{
 		.compatible = "renesas,r9a09g047-pcie",
 		.data = &rzg3e_soc_data,
diff --git a/drivers/pci/controller/pcie-xilinx-cpm.c b/drivers/pci/controller/pcie-xilinx-cpm.c
index d38f27e2076190..1fdd7f9205909c 100644
--- a/drivers/pci/controller/pcie-xilinx-cpm.c
+++ b/drivers/pci/controller/pcie-xilinx-cpm.c
@@ -6,6 +6,8 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/irqchip.h>
@@ -16,6 +18,7 @@
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
 #include <linux/of_platform.h>
+#include <linux/reset.h>
 
 #include "../pci.h"
 #include "pcie-xilinx-common.h"
@@ -113,6 +116,8 @@ struct xilinx_cpm_variant {
  * @irq: Error interrupt number
  * @lock: lock protecting shared register access
  * @variant: CPM version check pointer
+ * @perst_gpio: GPIO descriptor for PERST# signal handling
+ * @rstc: Pointer to the PCIe controller reset
  */
 struct xilinx_cpm_pcie {
 	struct device			*dev;
@@ -125,6 +130,8 @@ struct xilinx_cpm_pcie {
 	int				irq;
 	raw_spinlock_t			lock;
 	const struct xilinx_cpm_variant   *variant;
+	struct gpio_desc		*perst_gpio;
+	struct reset_control		*rstc;
 };
 
 static u32 pcie_read(struct xilinx_cpm_pcie *port, u32 reg)
@@ -389,7 +396,7 @@ static int xilinx_cpm_pcie_init_irq_domain(struct xilinx_cpm_pcie *port)
 	struct device_node *pcie_intc_node;
 
 	/* Setup INTx */
-	pcie_intc_node = of_get_next_child(node, NULL);
+	pcie_intc_node = of_get_child_by_name(node, "interrupt-controller");
 	if (!pcie_intc_node) {
 		dev_err(dev, "No PCIe Intc node found\n");
 		return -EINVAL;
@@ -469,6 +476,22 @@ static int xilinx_cpm_setup_irq(struct xilinx_cpm_pcie *port)
 	return 0;
 }
 
+/**
+ * xilinx_cpm_pcie_reset - Reset the PCIe controller and deassert PERST#
+ * @port: PCIe port information
+ *
+ * Reset the PCIe controller and then release the PERST# signal so that the
+ * link can train once the bridge is enabled.
+ */
+static void xilinx_cpm_pcie_reset(struct xilinx_cpm_pcie *port)
+{
+	reset_control_assert(port->rstc);
+	udelay(PCIE_T_PERST_US);
+	reset_control_deassert(port->rstc);
+	gpiod_set_value_cansleep(port->perst_gpio, 0);
+	msleep(PCIE_RESET_CONFIG_WAIT_MS);
+}
+
 /**
  * xilinx_cpm_pcie_init_port - Initialize hardware
  * @port: PCIe port information
@@ -480,6 +503,9 @@ static void xilinx_cpm_pcie_init_port(struct xilinx_cpm_pcie *port)
 	if (variant->version == CPM5NC_HOST)
 		return;
 
+	if (port->perst_gpio || port->rstc)
+		xilinx_cpm_pcie_reset(port);
+
 	if (cpm_pcie_link_up(port))
 		dev_info(port->dev, "PCIe Link is UP\n");
 	else
@@ -512,6 +538,42 @@ static void xilinx_cpm_pcie_init_port(struct xilinx_cpm_pcie *port)
 		   XILINX_CPM_PCIE_REG_RPSC);
 }
 
+/**
+ * xilinx_cpm_pcie_parse_port - Parse the PCIe Root Port child node
+ * @port: PCIe port information
+ *
+ * Read the PERST# GPIO from the Root Port child node.
+ *
+ * Return: '0' on success and error value on failure
+ */
+static int xilinx_cpm_pcie_parse_port(struct xilinx_cpm_pcie *port)
+{
+	struct device *dev = port->dev;
+
+	/*
+	 * This platform currently supports only one Root Port, so the loop
+	 * will execute only once.
+	 * TODO: Enhance the driver to handle multiple Root Ports in the future.
+	 */
+	for_each_child_of_node_with_prefix(dev->of_node, pcie_port_node, "pcie") {
+		port->perst_gpio = devm_fwnode_gpiod_get(dev,
+							 of_fwnode_handle(pcie_port_node),
+							 "reset", GPIOD_OUT_HIGH,
+							 NULL);
+		if (IS_ERR(port->perst_gpio)) {
+			if (PTR_ERR(port->perst_gpio) == -ENOENT) {
+				port->perst_gpio = NULL;
+				return 0;
+			}
+			return dev_err_probe(dev, PTR_ERR(port->perst_gpio),
+					     "Failed to request reset GPIO\n");
+		}
+		return 0;
+	}
+
+	return 0;
+}
+
 /**
  * xilinx_cpm_pcie_parse_dt - Parse Device tree
  * @port: PCIe port information
@@ -525,6 +587,19 @@ static int xilinx_cpm_pcie_parse_dt(struct xilinx_cpm_pcie *port,
 	struct device *dev = port->dev;
 	struct platform_device *pdev = to_platform_device(dev);
 	struct resource *res;
+	int ret;
+
+	/* CPM5NC does not support PERST# handling yet */
+	if (port->variant->version != CPM5NC_HOST) {
+		port->rstc = devm_reset_control_get_optional_exclusive(dev, NULL);
+		if (IS_ERR(port->rstc))
+			return dev_err_probe(dev, PTR_ERR(port->rstc),
+					     "Failed to request reset\n");
+
+		ret = xilinx_cpm_pcie_parse_port(port);
+		if (ret)
+			return ret;
+	}
 
 	port->cpm_base = devm_platform_ioremap_resource_byname(pdev,
 							       "cpm_slcr");
diff --git a/drivers/pci/controller/plda/pcie-starfive.c b/drivers/pci/controller/plda/pcie-starfive.c
index 02d5aa6dc14bcf..b693eabdf85ba3 100644
--- a/drivers/pci/controller/plda/pcie-starfive.c
+++ b/drivers/pci/controller/plda/pcie-starfive.c
@@ -21,6 +21,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <linux/reset.h>
 #include "../../pci.h"
 
diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index 241023ecf67741..d5845ea647f6ef 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -92,6 +92,22 @@ enum vmd_features {
 	 * referred to as MEMBAR2 or MSI-X BAR.
 	 */
 	VMD_FEAT_USE_BIOS_INFO		= (1 << 6),
+
+	/*
+	 * Meteor Lake VMD (device ID 0x7d0b) is affected by erratum MTL016:
+	 * the VMD may signal its MSI before the posted writes that carry the
+	 * child device's DMA data have landed in memory.  The demuxed handler
+	 * then runs against a not-yet-coherent completion queue and misses the
+	 * completion entirely, so the I/O is only recovered when the block
+	 * layer timeout fires and polls the queue ("timeout, completion
+	 * polled").  Intel's documented workaround is to issue a dummy read
+	 * to the MSI initiator (the child device) before handling the
+	 * interrupt: the read completion cannot pass the device's earlier
+	 * posted writes, so it pulls them into memory per PCIe ordering
+	 * rules.  A read that terminates at the VMD itself does not order
+	 * against the child's writes and is not sufficient (measured).
+	 */
+	VMD_FEAT_INTERRUPT_QUIRK	= (1 << 7),
 };
 
 #define VMD_BIOS_PM_QUIRK_LTR	0x1003	/* 3145728 ns */
@@ -114,6 +130,9 @@ static DEFINE_RAW_SPINLOCK(list_lock);
  * @irq:	back pointer to parent.
  * @enabled:	true if driver enabled IRQ
  * @virq:	the virtual IRQ value provided to the requesting driver.
+ * @flush_addr:	config space address of the initiating device, read before
+ *		demuxing to flush its posted writes (MTL016); NULL if the
+ *		VMD is not affected.
  *
  * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to
  * a VMD IRQ using this structure.
@@ -123,8 +142,14 @@ struct vmd_irq {
 	struct vmd_irq_list	*irq;
 	bool			enabled;
 	unsigned int		virq;
+	void __iomem		*flush_addr;
 };
 
+struct vmd_dev;
+
+static void __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,
+				  unsigned int devfn, int reg, int len);
+
 /**
  * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector
  * @irq_list:	the list of irq's the VMD one demuxes to.
@@ -132,12 +157,15 @@ struct vmd_irq {
  * @count:	number of child IRQs assigned to this vector; used to track
  *		sharing.
  * @virq:	The underlying VMD Linux interrupt number
+ * @vmd:	back pointer to the owning VMD device; serializes the MTL016
+ *		flush read against other config space access.
  */
 struct vmd_irq_list {
 	struct list_head	irq_list;
 	struct srcu_struct	srcu;
 	unsigned int		count;
 	unsigned int		virq;
+	struct vmd_dev		*vmd;
 };
 
 struct vmd_dev {
@@ -295,6 +323,13 @@ static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq,
 		INIT_LIST_HEAD(&vmdirq->node);
 		vmdirq->irq = vmd_next_irq(vmd, desc);
 		vmdirq->virq = virq + i;
+		if (vmd->features & VMD_FEAT_INTERRUPT_QUIRK) {
+			struct pci_dev *pdev = msi_desc_to_pci_dev(desc);
+
+			vmdirq->flush_addr = vmd_cfg_addr(vmd, pdev->bus,
+							  pdev->devfn,
+							  PCI_VENDOR_ID, 2);
+		}
 
 		irq_domain_set_info(domain, virq + i, vmdirq->irq->virq,
 				    &vmd_msi_controller, vmdirq,
@@ -756,8 +791,20 @@ static irqreturn_t vmd_irq(int irq, void *data)
 	int idx;
 
 	idx = srcu_read_lock(&irqs->srcu);
-	list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)
+	list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) {
+		/*
+		 * MTL016: the MSI may have outrun the initiating device's
+		 * posted writes (e.g. its NVMe completion entry).  A read
+		 * that completes at the initiator flushes them, so the
+		 * demuxed handler observes a coherent completion queue.
+		 * The value is discarded; only the ordering matters.
+		 */
+		if (vmdirq->flush_addr) {
+			guard(raw_spinlock)(&irqs->vmd->cfg_lock);
+			readw(vmdirq->flush_addr);
+		}
 		generic_handle_irq(vmdirq->virq);
+	}
 	srcu_read_unlock(&irqs->srcu, idx);
 
 	return IRQ_HANDLED;
@@ -788,6 +835,7 @@ static int vmd_alloc_irqs(struct vmd_dev *vmd)
 			return err;
 
 		INIT_LIST_HEAD(&vmd->irqs[i].irq_list);
+		vmd->irqs[i].vmd = vmd;
 		vmd->irqs[i].virq = pci_irq_vector(dev, i);
 		err = devm_request_irq(&dev->dev, vmd->irqs[i].virq,
 				       vmd_irq, IRQF_NO_THREAD,
@@ -896,11 +944,6 @@ static int vmd_pm_enable_quirk(struct pci_dev *pdev, void *userdata)
 	pci_info(pdev, "VMD: Default LTR value set by driver\n");
 
 out_state_change:
-	/*
-	 * Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per
-	 * PCIe r6.0, sec 5.5.4.
-	 */
-	pci_set_power_state_locked(pdev, PCI_D0);
 	pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL);
 	return 0;
 }
@@ -1250,7 +1293,7 @@ static const struct pci_device_id vmd_ids[] = {
 	{PCI_VDEVICE(INTEL, 0xa77f),
 		.driver_data = VMD_FEATS_CLIENT,},
 	{PCI_VDEVICE(INTEL, 0x7d0b),
-		.driver_data = VMD_FEATS_CLIENT,},
+		.driver_data = VMD_FEATS_CLIENT | VMD_FEAT_INTERRUPT_QUIRK,},
 	{PCI_VDEVICE(INTEL, 0xad0b),
 		.driver_data = VMD_FEATS_CLIENT,},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B),
diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
index c3caec927d7488..07ba338732d4e7 100644
--- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
@@ -132,7 +132,7 @@ struct epf_ntb_ctrl {
 } __packed;
 
 struct epf_ntb {
-	struct ntb_dev ntb;
+	struct ntb_dev *ntb;
 	struct pci_epf *epf;
 	struct config_group group;
 
@@ -166,10 +166,15 @@ struct epf_ntb {
 	void __iomem *vpci_mw_addr[MAX_MW];
 
 	struct delayed_work cmd_handler;
+	struct pci_bus *vpci_bus;
 };
 
 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
-#define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)
+
+static struct epf_ntb *ntb_ndev(struct ntb_dev *ntb)
+{
+	return ntb->pdev->sysdata;
+}
 
 static struct pci_epf_header epf_ntb_header = {
 	.vendorid	= PCI_ANY_ID,
@@ -195,7 +200,7 @@ static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
 	else
 		ntb->reg->link_status &= ~LINK_STATUS_UP;
 
-	ntb_link_event(&ntb->ntb);
+	ntb_link_event(ntb->ntb);
 	return 0;
 }
 
@@ -284,7 +289,7 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
 	     i++) {
 		if (ntb->epf_db[i]) {
 			atomic64_or(1 << (i - EPF_IRQ_DB_START), &ntb->db);
-			ntb_db_event(&ntb->ntb, i - EPF_IRQ_DB_START);
+			ntb_db_event(ntb->ntb, i - EPF_IRQ_DB_START);
 			ntb->epf_db[i] = 0;
 		}
 	}
@@ -348,12 +353,18 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
 static irqreturn_t epf_ntb_doorbell_handler(int irq, void *data)
 {
 	struct epf_ntb *ntb = data;
+	struct ntb_dev *ndev;
 	int i;
 
+	/* Pair with smp_store_release() in pci_vntb_probe() */
+	ndev = smp_load_acquire(&ntb->ntb);
+	if (!ndev)
+		return IRQ_HANDLED;
+
 	for (i = EPF_IRQ_DB_START; i < ntb->db_count; i++)
 		if (irq == ntb->epf->db_msg[i].virq) {
 			atomic64_or(1 << (i - EPF_IRQ_DB_START), &ntb->db);
-			ntb_db_event(&ntb->ntb, i - EPF_IRQ_DB_START);
+			ntb_db_event(ndev, i - EPF_IRQ_DB_START);
 		}
 
 	return IRQ_HANDLED;
@@ -985,7 +996,6 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb)
 	}
 
 	INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);
-	queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);
 
 	atomic64_set(&ntb->peer_db_pending, 0);
 	enable_work(&ntb->peer_db_work);
@@ -1332,14 +1342,20 @@ static int vpci_scan_bus(void *sysdata)
 	struct pci_bus *vpci_bus;
 	struct epf_ntb *ndev = sysdata;
 
+	pci_lock_rescan_remove();
+
 	vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata);
 	if (!vpci_bus) {
 		pr_err("create pci bus failed\n");
+		pci_unlock_rescan_remove();
 		return -EINVAL;
 	}
+	ndev->vpci_bus = vpci_bus;
 
 	pci_bus_add_devices(vpci_bus);
 
+	pci_unlock_rescan_remove();
+
 	return 0;
 }
 
@@ -1420,14 +1436,15 @@ static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
 	int ret;
 	struct device *dev;
 
-	dev = &ntb->ntb.dev;
+	dev = &ndev->dev;
 	barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
 	epf_bar = &ntb->epf->bar[barno];
 	epf_bar->phys_addr = addr;
 	epf_bar->barno = barno;
 	epf_bar->size = size;
 
-	ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);
+	ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
+			      ntb->epf->vfunc_no, epf_bar);
 	if (ret) {
 		dev_err(dev, "failure set mw trans\n");
 		return ret;
@@ -1558,7 +1575,7 @@ static void vntb_epf_peer_db_work(struct work_struct *work)
 			ret = pci_epc_raise_irq(epf->epc, func_no, vfunc_no,
 						PCI_IRQ_MSI, interrupt_num);
 			if (ret)
-				dev_err(&ntb->ntb.dev,
+				dev_err(&epf->dev,
 					"Failed to raise IRQ for interrupt_num %u: %d\n",
 					interrupt_num, ret);
 		}
@@ -1676,13 +1693,18 @@ static const struct ntb_dev_ops vntb_epf_ops = {
 
 static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
-	int ret;
 	struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata;
 	struct device *dev = &pdev->dev;
+	struct ntb_dev *ntb;
+	int ret;
+
+	ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
 
-	ndev->ntb.pdev = pdev;
-	ndev->ntb.topo = NTB_TOPO_NONE;
-	ndev->ntb.ops =  &vntb_epf_ops;
+	ntb->pdev = pdev;
+	ntb->topo = NTB_TOPO_NONE;
+	ntb->ops = &vntb_epf_ops;
 
 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 	if (ret) {
@@ -1690,16 +1712,41 @@ static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		return ret;
 	}
 
-	ret = ntb_register_device(&ndev->ntb);
+	ret = ntb_register_device(ntb);
 	if (ret) {
 		dev_err(dev, "Failed to register NTB device\n");
 		return ret;
 	}
 
+	/* Publish after ntb_register_device() succeeds */
+	smp_store_release(&ndev->ntb, ntb);
+	queue_delayed_work(kpcintb_workqueue, &ndev->cmd_handler, 0);
+
 	dev_dbg(dev, "PCI Virtual NTB driver loaded\n");
 	return 0;
 }
 
+static void pci_vntb_remove(struct pci_dev *pdev)
+{
+	struct epf_ntb *ndev = pdev->sysdata;
+	struct ntb_dev *ntb;
+	unsigned int i;
+
+	/* Stop the work reader, then close and drain the IRQ gate */
+	cancel_delayed_work_sync(&ndev->cmd_handler);
+
+	ntb = xchg(&ndev->ntb, NULL);
+
+	if (ndev->msi_doorbell)
+		for (i = 0; i < ndev->db_count; i++) {
+			if (epf_ntb_db_irq_is_duplicated(ndev->epf, i))
+				continue;
+			synchronize_irq(ndev->epf->db_msg[i].virq);
+		}
+
+	ntb_unregister_device(ntb);
+}
+
 static struct pci_device_id pci_vntb_table[] = {
 	{
 		PCI_DEVICE(0xffff, 0xffff),
@@ -1711,6 +1758,7 @@ static struct pci_driver vntb_pci_driver = {
 	.name           = "pci-vntb",
 	.id_table       = pci_vntb_table,
 	.probe          = pci_vntb_probe,
+	.remove         = pci_vntb_remove,
 };
 
 /* ============ PCIe EPF Driver Bind ====================*/
@@ -1791,12 +1839,25 @@ static int epf_ntb_bind(struct pci_epf *epf)
  */
 static void epf_ntb_unbind(struct pci_epf *epf)
 {
+	struct pci_host_bridge *bridge;
 	struct epf_ntb *ntb = epf_get_drvdata(epf);
 
+	pci_unregister_driver(&vntb_pci_driver);
+
+	if (ntb->vpci_bus) {
+		bridge = to_pci_host_bridge(ntb->vpci_bus->bridge);
+
+		pci_lock_rescan_remove();
+		pci_stop_root_bus(ntb->vpci_bus);
+		pci_remove_root_bus(ntb->vpci_bus);
+		ntb->vpci_bus = NULL;
+		pci_unlock_rescan_remove();
+
+		pci_free_host_bridge(bridge);
+	}
+
 	epf_ntb_epc_cleanup(ntb);
 	epf_ntb_config_spad_bar_free(ntb);
-
-	pci_unregister_driver(&vntb_pci_driver);
 }
 
 // EPF driver probe
diff --git a/drivers/pci/endpoint/pci-ep-msi.c b/drivers/pci/endpoint/pci-ep-msi.c
index 9c13f193537df6..2fe48326a4c630 100644
--- a/drivers/pci/endpoint/pci-ep-msi.c
+++ b/drivers/pci/endpoint/pci-ep-msi.c
@@ -10,7 +10,6 @@
 #include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/export.h>
-#include <linux/interrupt.h>
 #include <linux/irqdomain.h>
 #include <linux/module.h>
 #include <linux/msi.h>
@@ -112,8 +111,11 @@ static int pci_epf_alloc_doorbell_embedded(struct pci_epf *epf, u16 num_db)
 	 *
 	 * Still, pci_epf_alloc_doorbell() allows requesting multiple doorbells.
 	 * For such backends we replicate the same address/data for each entry
-	 * and mark the IRQ as shared (IRQF_SHARED). Consumers must treat them
-	 * as equivalent "kick" doorbells.
+	 * as equivalent "kick" doorbells. Consumers must request each distinct
+	 * IRQ only once.
+	 *
+	 * Keep the IRQ exclusive because the source cannot identify which EPF
+	 * rang the doorbell.
 	 */
 	for (i = 0; i < num_db; i++)
 		msg[i] = (struct pci_epf_doorbell_msg) {
@@ -121,7 +123,6 @@ static int pci_epf_alloc_doorbell_embedded(struct pci_epf *epf, u16 num_db)
 			.msg.address_hi = (u32)(addr >> 32),
 			.msg.data = doorbell->u.db_mmio.data,
 			.virq = doorbell->u.db_mmio.irq,
-			.irq_flags = IRQF_SHARED,
 			.type = PCI_EPF_DOORBELL_EMBEDDED,
 			.bar = doorbell->bar,
 			.offset = (doorbell->bar == NO_BAR) ? 0 :
@@ -198,28 +199,31 @@ static int pci_epf_alloc_doorbell_msi(struct pci_epf *epf, u16 num_db)
 int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
 {
 	struct pci_epc *epc = epf->epc;
+	struct pci_epf *first_epf;
 	struct device *dev = &epf->dev;
 	int ret;
 
-	/* TODO: Multi-EPF support */
-	if (list_first_entry_or_null(&epc->pci_epf, struct pci_epf, list) != epf) {
-		dev_err(dev, "Doorbell doesn't support multiple EPF\n");
-		return -EINVAL;
-	}
-
 	if (epf->db_msg)
 		return -EBUSY;
 
-	ret = pci_epf_alloc_doorbell_msi(epf, num_db);
-	if (!ret)
-		return 0;
-
 	/*
-	 * Fall back to embedded doorbell only when platform MSI is unavailable
-	 * for this EPC.
+	 * The MSI-backed doorbell path currently targets the first EPF attached
+	 * to the EPC. Let non-first EPFs try the embedded doorbell instead.
 	 */
-	if (ret != -ENODEV)
-		return ret;
+	first_epf = list_first_entry_or_null(&epc->pci_epf, struct pci_epf,
+					     list);
+	if (first_epf == epf) {
+		ret = pci_epf_alloc_doorbell_msi(epf, num_db);
+		if (!ret)
+			return 0;
+
+		/*
+		 * Fall back to embedded doorbell only when platform MSI is
+		 * unavailable for this EPC.
+		 */
+		if (ret != -ENODEV)
+			return ret;
+	}
 
 	ret = pci_epf_alloc_doorbell_embedded(epf, num_db);
 	if (ret) {
diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c
index 2cafd3b26f344f..5ce32dc7e178fb 100644
--- a/drivers/pci/hotplug/pciehp_core.c
+++ b/drivers/pci/hotplug/pciehp_core.c
@@ -38,7 +38,7 @@ int pciehp_poll_time;
  * not really modular, but the easiest way to keep compat with existing
  * bootargs behaviour is to continue using module_param here.
  */
-module_param(pciehp_poll_mode, bool, 0644);
+module_param(pciehp_poll_mode, bool, 0444);
 module_param(pciehp_poll_time, int, 0644);
 MODULE_PARM_DESC(pciehp_poll_mode, "Using polling mechanism for hot-plug events or not");
 MODULE_PARM_DESC(pciehp_poll_time, "Polling mechanism frequency, in seconds");
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 9d408fb8ac250c..9185934a900f25 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -179,29 +179,6 @@ bool pci_iov_is_memory_decoding_enabled(struct pci_dev *dev)
 	return cmd & PCI_SRIOV_CTRL_MSE;
 }
 
-static void pci_read_vf_config_common(struct pci_dev *virtfn)
-{
-	struct pci_dev *physfn = virtfn->physfn;
-
-	/*
-	 * Some config registers are the same across all associated VFs.
-	 * Read them once from VF0 so we can skip reading them from the
-	 * other VFs.
-	 *
-	 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to
-	 * have the same Revision ID and Subsystem ID, but we assume they
-	 * do.
-	 */
-	pci_read_config_dword(virtfn, PCI_CLASS_REVISION,
-			      &physfn->sriov->class);
-	pci_read_config_byte(virtfn, PCI_HEADER_TYPE,
-			     &physfn->sriov->hdr_type);
-	pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID,
-			     &physfn->sriov->subsystem_vendor);
-	pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID,
-			     &physfn->sriov->subsystem_device);
-}
-
 int pci_iov_sysfs_link(struct pci_dev *dev,
 		struct pci_dev *virtfn, int id)
 {
@@ -329,8 +306,6 @@ static struct pci_dev *pci_iov_scan_device(struct pci_dev *dev, int id,
 	virtfn->physfn = pci_dev_get(dev);
 	virtfn->no_command_memory = 1;
 
-	if (id == 0)
-		pci_read_vf_config_common(virtfn);
 
 	rc = pci_setup_device(virtfn);
 	if (rc) {
diff --git a/drivers/pci/liveupdate.c b/drivers/pci/liveupdate.c
new file mode 100644
index 00000000000000..51695782ed4271
--- /dev/null
+++ b/drivers/pci/liveupdate.c
@@ -0,0 +1,1015 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2026, Google LLC.
+ * David Matlack <dmatlack@google.com>
+ */
+
+/**
+ * DOC: PCI Live Update
+ *
+ * The PCI subsystem participates in the Live Update process to enable drivers
+ * to preserve their PCI devices across kexec.
+ *
+ * Preserving a device requires preserving two independent sets of state: the
+ * driver's own state, which the driver preserves with no involvement from the
+ * PCI core, and the PCI core's state about the device, which the next kernel
+ * needs so that enumeration does not disturb a device that is still running.
+ * This file implements the latter.
+ *
+ * :ref:`FLB <flb>` Data
+ * =====================
+ *
+ * Userspace decides which devices are preserved, using :ref:`LUO <luo>` file
+ * preservation: a driver exposes a file that represents a single PCI device,
+ * and userspace preserves the device with
+ * ``ioctl(LIVEUPDATE_SESSION_PRESERVE_FD)`` on that file. Binding preservation
+ * to a file gives it proper lifecycle management, e.g. the preservation is
+ * undone if userspace cancels it or goes away. How a driver exposes that file
+ * is up to the driver and invisible to the PCI core (vfio-pci variant drivers,
+ * the first intended use-case, use their per-device cdev).
+ *
+ * LUO only knows that a file was preserved; it does not know that the file
+ * represents a PCI device. Drivers therefore register their
+ * struct liveupdate_file_handler with the PCI core:
+ *
+ *  * ``pci_liveupdate_register_flb(driver_file_handler)``
+ *  * ``pci_liveupdate_unregister_flb(driver_file_handler)``
+ *
+ * LUO then refcounts the PCI core's FLB against the files preserved by that
+ * handler, and that refcount drives the lifetime of struct pci_ser:
+ * pci_flb_preserve() allocates and preserves it when the first file is
+ * preserved, and pci_flb_unpreserve() frees it when the last file is
+ * unpreserved. In the next kernel, pci_flb_retrieve() hands the PCI core the
+ * struct pci_ser built by the previous kernel, whenever the PCI core asks for
+ * it (e.g. during enumeration), and pci_flb_finish() frees it once the PCI
+ * core is done with it.
+ *
+ * State handed over by the previous kernel is trusted. The PCI core validates
+ * it only far enough to detect an incompatible or corrupt hand over, and makes
+ * no attempt to defend against deliberate modification, since a previous kernel
+ * able to corrupt preserved state is able to corrupt arbitrary memory anyway.
+ *
+ * Call Flow
+ * ---------
+ *
+ * ::
+ *
+ *   # Driver initialization
+ *   pci_liveupdate_register_flb(fh)
+ *
+ *   # Userspace: ioctl(LIVEUPDATE_SESSION_PRESERVE_FD, devfd)
+ *   luo_preserve_file()
+ *     luo_flb_file_preserve()
+ *       luo_flb_file_preserve_one()      # first preserved file only
+ *         pci_flb_preserve()             # alloc and preserve struct pci_ser
+ *     fh->ops->preserve()                # driver callback
+ *       pci_liveupdate_preserve(dev)     # record this device in struct pci_ser
+ *
+ *   # Userspace: preservation cancelled or session torn down
+ *   luo_file_unpreserve_files()
+ *     luo_flb_file_unpreserve()
+ *       liveupdate_flb_put_outgoing()    # last unpreserved file only
+ *         pci_flb_unpreserve()           # free struct pci_ser
+ *
+ *   # ---------------- kexec ----------------
+ *
+ *   # New kernel: PCI enumeration
+ *   pci_setup_device()
+ *     pci_liveupdate_setup_device()
+ *       liveupdate_flb_get_incoming()
+ *         luo_flb_retrieve_one()         # first request only
+ *           pci_flb_retrieve()           # previous kernel's struct pci_ser
+ *
+ *   # Userspace: ioctl(LIVEUPDATE_SESSION_FINISH)
+ *   luo_file_finish_one()
+ *     fh->ops->finish()                  # driver callback
+ *       pci_liveupdate_finish(dev)       # release this device's pci_dev_ser
+ *     luo_flb_file_finish()
+ *       liveupdate_flb_put_incoming()    # last incoming file only
+ *         pci_flb_finish()               # free struct pci_ser
+ *
+ * Device Tracking
+ * ===============
+ *
+ * Drivers must notify the PCI core when specific devices are preserved or
+ * unpreserved with the following APIs:
+ *
+ *  * ``pci_liveupdate_preserve(pci_dev)``
+ *  * ``pci_liveupdate_unpreserve(pci_dev)``
+ *
+ * This allows the PCI core to keep its FLB data (struct pci_ser) up to date
+ * with the list of **outgoing** preserved devices for the next kernel.
+ *
+ * After kexec, whenever a device is enumerated, the PCI core will check if it
+ * is an **incoming** preserved device (i.e. preserved by the previous kernel)
+ * by checking the incoming FLB data (struct pci_ser).
+ *
+ * Drivers must notify the PCI core when an **incoming** device is done
+ * participating in the incoming Live Update with the following API:
+ *
+ *  * ``pci_liveupdate_finish(pci_dev)``
+ *
+ * The PCI core does not enforce any ordering of ``pci_liveupdate_finish()`` and
+ * ``pci_liveupdate_preserve()``, i.e., a PCI device can be **outgoing**
+ * (preserved for next kernel) and **incoming** (preserved by previous kernel)
+ * at the same time.
+ *
+ * Restrictions
+ * ============
+ *
+ * The PCI core enforces the following restrictions on which devices can be
+ * preserved. These may be relaxed in the future:
+ *
+ *  * The device cannot be a Virtual Function (VF).
+ *
+ *  * The device cannot require device-specific quirks to enable Access
+ *    Control Services (ACS).
+ *
+ * Driver Binding
+ * ==============
+ *
+ * In the outgoing kernel, the driver must ensure that it does not release a
+ * device between pci_liveupdate_preserve() and pci_liveupdate_unpreserve().
+ *
+ * In the incoming kernel, the driver must ensure that it does not release a
+ * preserved device between probe() and pci_liveupdate_finish().
+ *
+ * It is the user's responsibility to ensure that incoming preserved devices are
+ * bound to the correct driver. The PCI core does not protect against a device
+ * getting preserved by driver A in the outgoing kernel and then getting bound
+ * to driver B in the incoming kernel.
+ *
+ * PCI-to-PCI Bridges
+ * ==================
+ *
+ * Any PCI-to-PCI bridges upstream of a preserved device are automatically
+ * preserved when the device is preserved. The PCI core keeps track of the
+ * number of downstream devices that are preserved under a bridge so that the
+ * bridge is only unpreserved once all downstream devices are unpreserved.
+ *
+ * This enables the PCI core and any drivers bound to the bridge to participate
+ * in the Live Update so that preserved endpoints can continue issuing memory
+ * transactions during the Live Update.
+ *
+ * BDF Stability
+ * =============
+ *
+ * The PCI core guarantees that preserved devices can be identified by the same
+ * bus, device, and function numbers for as long as they are preserved
+ * (including across kexec). To accomplish this, the PCI core keeps the
+ * secondary and subordinate bus numbers that the previous kernel programmed
+ * into bridges, if the previous kernel preserved any device. This is true even
+ * on architectures that always assign new bus numbers during scanning. The
+ * kernel assumes the previous kernel established a sane bus topology across
+ * kexec.
+ *
+ * Bridges that do not have bus numbers are assigned new ones as usual, so
+ * hot-adding a bridge keeps working, both during and after a Live Update. The
+ * two-pass bridge scan ensures such bridges are only assigned bus numbers above
+ * those already claimed by preserved bridges.
+ *
+ * If a preserved bridge comes up without a valid bus number configuration, e.g.
+ * because it was reset during kexec, the PCI core refuses to assign it new bus
+ * numbers and does not enumerate anything below it. Assigning new bus numbers
+ * would silently change the BDF of every preserved device in its hierarchy. The
+ * PCI core also stops assigning bus numbers to the other bridges on the same
+ * bus, since the bus numbers of the failed bridge can no longer be read from
+ * hardware and handing them to another bridge would let an unrelated device
+ * inherit the BDF of a preserved device.
+ *
+ * Handling Preserved Devices
+ * ==========================
+ *
+ * The PCI core treats preserved devices differently than non-preserved devices.
+ * This section enumerates those differences.
+ *
+ *  * The PCI core adopts all ACS controls enabled on incoming preserved devices
+ *    rather than assigning new ones. This ensures that TLPs are routed the same
+ *    way after Live Update and ensures that IOMMU groups do not change. Note
+ *    that a device will use its adopted ACS controls for the lifetime of its
+ *    struct pci_dev (i.e. even after pci_liveupdate_finish()).
+ *
+ *  * The PCI core adopts ARI Forwarding Enable on all bridges with downstream
+ *    preserved devices to ensure that all preserved devices on the bridge's
+ *    secondary bus are addressable after the Live Update.
+ *
+ *  * The PCI core does not disable bus mastering on outgoing preserved devices
+ *    during kexec. This allows preserved devices to issue memory transactions
+ *    throughout the Live Update.
+ */
+
+#define pr_fmt(fmt) "PCI: liveupdate: " fmt
+
+#include <linux/io.h>
+#include <linux/kexec_handover.h>
+#include <linux/kho/abi/pci.h>
+#include <linux/kho_block.h>
+#include <linux/liveupdate.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+
+#include "liveupdate.h"
+#include "pci.h"
+
+/**
+ * struct pci_liveupdate_global - Global state for PCI Live Update support
+ * @rwsem: Reader/writer semaphore used to protect the incoming and outgoing
+ *         FLBs, and the references to them in struct pci_dev.
+ * @had_incoming: True if the previous kernel preserved at least one PCI device.
+ *                Set when the incoming FLB is retrieved and never cleared, so
+ *                it stays true after Live Update finishes.
+ */
+struct pci_liveupdate_global {
+	struct rw_semaphore rwsem;
+	bool had_incoming;
+};
+
+static struct pci_liveupdate_global pci_liveupdate = {
+	.rwsem = __RWSEM_INITIALIZER(pci_liveupdate.rwsem),
+};
+
+/**
+ * struct pci_flb_outgoing - Outgoing PCI FLB object
+ * @ser: Pointer to the preserved struct pci_ser.
+ * @block_set: The KHO block set holding the outgoing devices.
+ *
+ * This structure holds the runtime state for the outgoing PCI Live Update
+ * state. It wraps the serialized pci_ser and the block_set used to manage
+ * the serialized entries.
+ */
+struct pci_flb_outgoing {
+	struct pci_ser *ser;
+	struct kho_block_set block_set;
+};
+
+/**
+ * struct pci_flb_incoming - Incoming PCI FLB object
+ * @ser: The incoming struct pci_ser from the previous kernel.
+ * @xa: Xarray used to quickly lookup devices in @ser.
+ * @block_set: The KHO block set holding the incoming devices.
+ *
+ * This structure holds the runtime state for the incoming PCI Live Update
+ * state. It wraps the serialized pci_ser, the block_set used to restore
+ * the serialized entries, and an xarray for fast lookups.
+ */
+struct pci_flb_incoming {
+	struct pci_ser *ser;
+	struct xarray xa;
+	struct kho_block_set block_set;
+};
+
+static unsigned long pci_ser_xa_key(u32 domain, u16 bdf)
+{
+	return (unsigned long)domain << 16 | bdf;
+}
+
+static int pci_flb_preserve(struct liveupdate_flb_op_args *args)
+{
+	struct pci_flb_outgoing *outgoing __free(kfree) = NULL;
+	struct pci_ser *ser;
+
+	outgoing = kzalloc_obj(*outgoing);
+	if (!outgoing)
+		return -ENOMEM;
+
+	ser = kho_alloc_preserve(sizeof(*ser));
+	if (IS_ERR(ser))
+		return PTR_ERR(ser);
+
+	ser->version = PCI_LUO_FLB_VERSION;
+	ser->nr_devices = 0;
+	ser->devices = 0;
+
+	outgoing->ser = ser;
+	kho_block_set_init(&outgoing->block_set, sizeof(struct pci_dev_ser));
+
+	args->obj = no_free_ptr(outgoing);
+	args->data = virt_to_phys(ser);
+	pr_debug("Preserved struct pci_ser (0x%llx)\n", args->data);
+	return 0;
+}
+
+static void pci_flb_unpreserve(struct liveupdate_flb_op_args *args)
+{
+	struct pci_flb_outgoing *outgoing = args->obj;
+
+	pr_debug("Unpreserving struct pci_ser (0x%llx)\n", args->data);
+
+	WARN_ON(outgoing->ser->nr_devices);
+	kho_block_set_destroy(&outgoing->block_set);
+	kho_unpreserve_free(outgoing->ser);
+	kfree(outgoing);
+}
+
+/*
+ * Any failure here is fatal. The previous kernel handed over devices that are
+ * still performing DMA, and this kernel cannot identify them without this
+ * state. Continuing would let the PCI core reassign bus numbers and rebind
+ * drivers underneath live devices, so fail loudly instead of unwinding.
+ */
+static int pci_flb_retrieve(struct liveupdate_flb_op_args *args)
+{
+	struct pci_ser *ser = phys_to_virt(args->data);
+	struct pci_flb_incoming *incoming;
+	struct pci_dev_ser *dev_ser;
+	struct kho_block_set_it it;
+	int ret;
+
+	pr_debug("Retrieving struct pci_ser (0x%llx)\n", args->data);
+
+	if (ser->version != PCI_LUO_FLB_VERSION)
+		panic("Incoming PCI FLB version (v%d) is incompatible with this kernel (v%d)\n",
+		      ser->version, PCI_LUO_FLB_VERSION);
+
+	incoming = kzalloc_obj(*incoming);
+	if (!incoming)
+		panic("Failed to allocate struct pci_flb_incoming\n");
+
+	incoming->ser = ser;
+	xa_init(&incoming->xa);
+
+	kho_block_set_init(&incoming->block_set, sizeof(struct pci_dev_ser));
+	ret = kho_block_set_restore(&incoming->block_set, ser->devices);
+	if (ret)
+		panic("Failed to restore devices KHO block set (%d)\n", ret);
+
+	kho_block_set_it_init(&it, &incoming->block_set);
+	while ((dev_ser = kho_block_set_it_read_entry(&it))) {
+		unsigned long key;
+
+		if (!dev_ser->refcount)
+			continue;
+
+		key = pci_ser_xa_key(dev_ser->domain, dev_ser->bdf);
+		ret = xa_insert(&incoming->xa, key, dev_ser, GFP_KERNEL);
+		if (ret)
+			panic("Failed to insert PCI device %04x:%02x:%02x.%d into xarray (%d)\n",
+			      dev_ser->domain, PCI_BUS_NUM(dev_ser->bdf),
+			      PCI_SLOT(dev_ser->bdf), PCI_FUNC(dev_ser->bdf),
+			      ret);
+	}
+
+	/*
+	 * Remember that the previous kernel preserved devices for the lifetime
+	 * of this kernel, even after Live Update finishes and the incoming FLB
+	 * is freed. See pci_liveupdate_preserve_bus_numbers().
+	 */
+	if (!xa_empty(&incoming->xa))
+		pci_liveupdate.had_incoming = true;
+
+	args->obj = incoming;
+	return 0;
+}
+
+static void pci_check_all_devices_finished(struct pci_flb_incoming *incoming)
+{
+	struct pci_dev_ser *dev_ser;
+	unsigned long index;
+	u32 nr_devices;
+
+	/*
+	 * nr_devices is only decremented by pci_liveupdate_finish_device().
+	 * This runs once the last reference to the incoming FLB is dropped, so
+	 * there are no finishers left in flight.
+	 */
+	nr_devices = incoming->ser->nr_devices;
+	if (nr_devices == 0)
+		return;
+
+	/*
+	 * Report the unfinished devices from the incoming FLB rather than by
+	 * walking struct pci_dev, so that devices that never showed up after
+	 * kexec, or that were destroyed before they finished, are identified
+	 * as well.
+	 */
+	xa_for_each(&incoming->xa, index, dev_ser) {
+		if (!dev_ser->refcount)
+			continue;
+
+		pr_emerg("%04x:%02x:%02x.%d was never finished!\n",
+			 dev_ser->domain, PCI_BUS_NUM(dev_ser->bdf),
+			 PCI_SLOT(dev_ser->bdf), PCI_FUNC(dev_ser->bdf));
+	}
+
+	/*
+	 * This should only happen if a driver violated the contract to call
+	 * pci_liveupdate_finish() (something is extremely broken).
+	 */
+	panic("%u preserved device(s) were never finished!\n", nr_devices);
+}
+
+static void pci_flb_finish(struct liveupdate_flb_op_args *args)
+{
+	struct pci_flb_incoming *incoming = args->obj;
+
+	pr_debug("Finished struct pci_ser (0x%llx)\n", args->data);
+	pci_check_all_devices_finished(incoming);
+
+	xa_destroy(&incoming->xa);
+	kho_block_set_destroy(&incoming->block_set);
+	kho_restore_free(incoming->ser);
+	kfree(incoming);
+}
+
+static struct liveupdate_flb_ops pci_liveupdate_flb_ops = {
+	.preserve = pci_flb_preserve,
+	.unpreserve = pci_flb_unpreserve,
+	.retrieve = pci_flb_retrieve,
+	.finish = pci_flb_finish,
+	.owner = THIS_MODULE,
+};
+
+static struct liveupdate_flb pci_liveupdate_flb = {
+	.ops = &pci_liveupdate_flb_ops,
+	.compatible = PCI_LUO_FLB_COMPATIBLE,
+};
+
+static void pci_liveupdate_flb_put_outgoing(void)
+{
+	liveupdate_flb_put_outgoing(&pci_liveupdate_flb);
+}
+
+static struct pci_flb_outgoing *pci_liveupdate_flb_get_outgoing(void)
+{
+	struct pci_flb_outgoing *outgoing = NULL;
+	int ret;
+
+	ret = liveupdate_flb_get_outgoing(&pci_liveupdate_flb, (void **)&outgoing);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (!outgoing)
+		return ERR_PTR(-ENOENT);
+
+	return outgoing;
+}
+
+static struct pci_dev_ser *pci_flb_alloc_dev_ser(struct pci_flb_outgoing *outgoing)
+{
+	struct pci_dev_ser *dev_ser;
+	struct kho_block_set_it it;
+	u64 count = 0;
+	int err;
+
+	kho_block_set_it_init(&it, &outgoing->block_set);
+
+	/* Try to find an existing, previously unpreserved, entry. */
+	while ((dev_ser = kho_block_set_it_read_entry(&it))) {
+		if (!dev_ser->refcount)
+			return dev_ser;
+
+		count++;
+	}
+
+	/* Otherwise grow the block set and reserve a new entry. */
+	err = kho_block_set_grow(&outgoing->block_set, count + 1);
+	if (err)
+		return ERR_PTR(err);
+
+	if (!count)
+		kho_block_set_it_init(&it, &outgoing->block_set);
+
+	/* This should always succeed since kho_block_set_grow() succeeded. */
+	dev_ser = kho_block_set_it_reserve_entry(&it);
+	if (WARN_ON_ONCE(!dev_ser))
+		return ERR_PTR(-ENOSPC);
+
+	return dev_ser;
+}
+
+static int pci_liveupdate_unpreserve_device(struct pci_flb_outgoing *outgoing,
+					    struct pci_dev *dev)
+{
+	struct pci_dev_ser *dev_ser = dev->liveupdate.outgoing;
+
+	if (dev->liveupdate.frozen) {
+		pci_warn(dev, "Cannot unpreserve device after it is frozen!\n");
+		return -EINVAL;
+	}
+
+	if (!dev_ser) {
+		pci_warn(dev, "Cannot unpreserve device that is not preserved\n");
+		return -EINVAL;
+	}
+
+	if (!dev_ser->refcount) {
+		pci_WARN(dev, 1, "Preserved device has a 0 refcount!\n");
+		return -EINVAL;
+	}
+
+	if (--dev_ser->refcount)
+		return 0;
+
+	pci_info(dev, "Device will no longer be preserved across next Live Update\n");
+	outgoing->ser->nr_devices--;
+	memset(dev_ser, 0, sizeof(*dev_ser));
+	dev->liveupdate.outgoing = NULL;
+	return 0;
+}
+
+static void pci_liveupdate_unpreserve_path(struct pci_flb_outgoing *outgoing,
+					   struct pci_dev *dev,
+					   struct pci_dev *end)
+{
+	for_each_pci_dev_in_path(dev) {
+		if (dev == end)
+			break;
+
+		if (pci_liveupdate_unpreserve_device(outgoing, dev))
+			return;
+	}
+}
+
+static int pci_liveupdate_preserve_device(struct pci_flb_outgoing *outgoing,
+					  struct pci_dev *dev)
+{
+	if (dev->is_virtfn) {
+		pci_warn(dev, "Cannot preserve Virtual Functions\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Do not preserve devices that rely on device-specific ACS equivalents
+	 * (for now) since that would complicate keeping ACS constant across
+	 * Live Update.
+	 */
+	if (pci_need_dev_specific_enable_acs(dev)) {
+		pci_warn(dev, "Refusing to preserve device that relies on ACS quirks\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Endpoint devices should not be preserved more than once.
+	 * Bridges are preserved once for every downstream device that
+	 * is preserved.
+	 */
+	if (dev->liveupdate.outgoing && !dev->subordinate) {
+		pci_warn(dev, "Device is already preserved\n");
+		return -EBUSY;
+	}
+
+	if (dev->liveupdate.outgoing && !dev->liveupdate.outgoing->refcount) {
+		pci_WARN(dev, 1, "Preserved device with 0 refcount!\n");
+		return -EINVAL;
+	}
+
+	if (dev->liveupdate.frozen) {
+		pci_warn(dev, "Cannot preserve device after it is frozen!\n");
+		return -EINVAL;
+	}
+
+	if (!dev->liveupdate.outgoing) {
+		struct pci_dev_ser *dev_ser;
+
+		dev_ser = pci_flb_alloc_dev_ser(outgoing);
+		if (IS_ERR(dev_ser))
+			return PTR_ERR(dev_ser);
+
+		pci_info(dev, "Device will be preserved across next Live Update\n");
+		outgoing->ser->nr_devices++;
+		outgoing->ser->devices = kho_block_set_head_pa(&outgoing->block_set);
+
+		dev_ser->domain = pci_domain_nr(dev->bus);
+		dev_ser->bdf = pci_dev_id(dev);
+		dev->liveupdate.outgoing = dev_ser;
+	}
+
+	dev->liveupdate.outgoing->refcount++;
+	return 0;
+}
+
+/**
+ * pci_liveupdate_preserve() - Preserve a PCI device across Live Update
+ * @dev: The PCI device to preserve.
+ *
+ * pci_liveupdate_preserve() notifies the PCI core that a PCI device should be
+ * preserved across the next Live Update. Drivers are expected to call
+ * pci_liveupdate_preserve() from their struct liveupdate_file_handler
+ * preserve() callback to ensure the outgoing struct pci_ser is already set up.
+ *
+ * pci_liveupdate_preserve() automatically preserves all bridges upstream of
+ * @dev.
+ *
+ * Returns: 0 on success, <0 on failure.
+ */
+int pci_liveupdate_preserve(struct pci_dev *dev)
+{
+	struct pci_flb_outgoing *outgoing = NULL;
+	struct pci_dev *start = dev;
+	int ret = -ENODEV;
+
+	guard(rwsem_write)(&pci_liveupdate.rwsem);
+
+	outgoing = pci_liveupdate_flb_get_outgoing();
+	if (IS_ERR(outgoing))
+		return PTR_ERR(outgoing);
+
+	for_each_pci_dev_in_path(dev) {
+		ret = pci_liveupdate_preserve_device(outgoing, dev);
+		if (ret) {
+			pci_liveupdate_unpreserve_path(outgoing, start, dev);
+			break;
+		}
+	}
+
+	pci_liveupdate_flb_put_outgoing();
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_liveupdate_preserve);
+
+/**
+ * pci_liveupdate_unpreserve() - Cancel preservation of a PCI device
+ * @dev: The PCI device to unpreserve.
+ *
+ * pci_liveupdate_unpreserve() notifies the PCI core that a PCI device should no
+ * longer be preserved across the next Live Update. Drivers are expected to call
+ * pci_liveupdate_unpreserve() from their struct liveupdate_file_handler
+ * unpreserve() callback to ensure the outgoing struct pci_ser is already set
+ * up.
+ *
+ * pci_liveupdate_unpreserve() automatically unpreserves all bridges upstream of
+ * @dev.
+ */
+void pci_liveupdate_unpreserve(struct pci_dev *dev)
+{
+	struct pci_flb_outgoing *outgoing = NULL;
+
+	guard(rwsem_write)(&pci_liveupdate.rwsem);
+
+	outgoing = pci_liveupdate_flb_get_outgoing();
+	if (IS_ERR(outgoing)) {
+		pci_warn(dev, "Cannot unpreserve device without outgoing Live Update state\n");
+		return;
+	}
+
+	pci_liveupdate_unpreserve_path(outgoing, dev, /*end=*/NULL);
+	pci_liveupdate_flb_put_outgoing();
+}
+EXPORT_SYMBOL_GPL(pci_liveupdate_unpreserve);
+
+static struct pci_flb_incoming *pci_liveupdate_flb_get_incoming(void)
+{
+	struct pci_flb_incoming *incoming = NULL;
+	int ret;
+
+	ret = liveupdate_flb_get_incoming(&pci_liveupdate_flb, (void **)&incoming);
+
+	/* Live Update is not enabled. */
+	if (ret == -EOPNOTSUPP)
+		return NULL;
+
+	/* Live Update is enabled, but there is no incoming FLB data. */
+	if (ret == -ENODATA)
+		return NULL;
+
+	/*
+	 * Live Update is enabled and there is incoming FLB data, but none of it
+	 * matches pci_liveupdate_flb.compatible.
+	 */
+	if (ret == -ENOENT)
+		return NULL;
+
+	/*
+	 * There is incoming FLB data that matches pci_liveupdate_flb.compatible
+	 * but retrieve failed (pci_flb_retrieve() returned an error or LUO
+	 * failed to acquire a reference to pci_liveupdate_flb_ops.owner).
+	 */
+	if (ret)
+		panic("Failed to retrieve incoming FLB data (%d)\n", ret);
+
+	return incoming;
+}
+
+static void pci_liveupdate_flb_put_incoming(void)
+{
+	liveupdate_flb_put_incoming(&pci_liveupdate_flb);
+}
+
+/**
+ * pci_liveupdate_preserve_bus_numbers() - Determine if the PCI core should
+ *                                         preserve bus numbers when scanning
+ *                                         bridges.
+ *
+ * This function is called by the PCI core when it is scanning a bridge. It
+ * determines whether the PCI core should preserve the secondary and subordinate
+ * bus numbers that the previous kernel programmed into that bridge, rather than
+ * assigning new ones. This is necessary to keep RequesterIDs constant for
+ * preserved devices issuing memory transactions.
+ *
+ * Bus numbers are preserved everywhere, and for the lifetime of the kernel, if
+ * the previous kernel preserved any device. Bus numbers have to be preserved
+ * above a preserved device anyway, since an upstream bridge cannot expand its
+ * window. Applying the same policy everywhere matches the scope of
+ * pcibios_assign_all_busses(), and gives an answer that cannot change part way
+ * through the two passes of a bridge scan.
+ *
+ * The incoming FLB is retrieved while setting up the first device, which always
+ * happens before any bridge is scanned, so this returns the same answer for the
+ * entire enumeration.
+ *
+ * Note that this does not prevent the PCI core from assigning bus numbers to
+ * bridges that do not have any, e.g. bridges that are hot-added after the
+ * Live Update. See pci_liveupdate_refuse_bus_numbers() for the one case where
+ * the PCI core must refuse to do so.
+ *
+ * Return: True if bus numbers should be preserved, false otherwise.
+ */
+bool pci_liveupdate_preserve_bus_numbers(void)
+{
+	return pci_liveupdate.had_incoming;
+}
+
+/**
+ * pci_liveupdate_refuse_bus_numbers() - Determine if the PCI core must refuse
+ *                                       to assign bus numbers to the provided
+ *                                       bridge.
+ * @bus: The PCI bus the bus numbers would be assigned from.
+ * @dev: The PCI bridge device the bus numbers would be assigned to.
+ *
+ * This function is called by the PCI core before it assigns bus numbers to a
+ * bridge that does not have any.
+ *
+ * A bridge that was preserved by the previous kernel but came up without a
+ * valid bus number configuration, e.g. because it was reset during kexec, is
+ * left alone by the PCI core and therefore has no child bus once the first pass
+ * of the bridge scan is done.
+ *
+ * The PCI core must not assign bus numbers from @bus while such a bridge is on
+ * it, including to the failed bridge itself. Assigning new bus numbers to the
+ * failed bridge would silently change the BDF of every preserved device in its
+ * hierarchy. Its bus numbers cannot be read from hardware anymore either, so
+ * they cannot be excluded from assignment, and handing them to another bridge
+ * would let an unrelated device inherit the BDF of a preserved device.
+ *
+ * Return: True if @dev must not be assigned bus numbers, false otherwise.
+ */
+bool pci_liveupdate_refuse_bus_numbers(struct pci_bus *bus, struct pci_dev *dev)
+{
+	struct pci_dev *bridge;
+
+	for_each_pci_bridge(bridge, bus) {
+		if (!bridge->liveupdate.was_incoming || bridge->subordinate)
+			continue;
+
+		pci_err(dev, "Not assigning bus numbers, preserved bridge %s lost its bus number configuration\n",
+			pci_name(bridge));
+		return true;
+	}
+
+	return false;
+}
+
+void pci_liveupdate_setup_device(struct pci_dev *dev)
+{
+	struct pci_flb_incoming *incoming;
+	struct pci_dev_ser *dev_ser;
+	unsigned long key;
+
+	guard(rwsem_write)(&pci_liveupdate.rwsem);
+
+	incoming = pci_liveupdate_flb_get_incoming();
+	if (!incoming)
+		return;
+
+	key = pci_ser_xa_key(pci_domain_nr(dev->bus), pci_dev_id(dev));
+	dev_ser = xa_load(&incoming->xa, key);
+
+	/*
+	 * This device was not preserved across Live Update, or it was preserved
+	 * but has already been probed and gone through pci_liveupdate_finish(),
+	 * e.g. due to removing and re-adding the device. Either way, it's not
+	 * treated as incoming-preserved.
+	 */
+	if (!dev_ser || !dev_ser->refcount) {
+		pci_liveupdate_flb_put_incoming();
+		return;
+	}
+
+	pci_info(dev, "Device was preserved by previous kernel across Live Update\n");
+	dev->liveupdate.incoming = dev_ser;
+	dev->liveupdate.was_incoming = true;
+
+	pci_liveupdate_flb_put_incoming();
+}
+
+void pci_liveupdate_cleanup_device(struct pci_dev *dev)
+{
+	/*
+	 * It should be safe to READ_ONCE() outside of the rwsem during cleanup
+	 * since there should no longer be any references to @dev on the system.
+	 *
+	 * This should never happen in practice. Drivers should block removal
+	 * while a device is preserved.
+	 */
+	if (READ_ONCE(dev->liveupdate.outgoing))
+		pci_WARN(dev, 1, "Destroying outgoing-preserved device!\n");
+
+	if (READ_ONCE(dev->liveupdate.incoming))
+		pci_WARN(dev, 1, "Destroying incoming-preserved device!\n");
+}
+
+void pci_liveupdate_freeze(struct pci_dev *dev)
+{
+	guard(rwsem_write)(&pci_liveupdate.rwsem);
+	dev->liveupdate.frozen = true;
+}
+
+static int pci_liveupdate_finish_device(struct pci_ser *ser, struct pci_dev *dev)
+{
+	if (!dev->liveupdate.incoming) {
+		pci_warn(dev, "Cannot finish preserving an unpreserved device\n");
+		return -EINVAL;
+	}
+
+	if (!dev->liveupdate.incoming->refcount) {
+		pci_WARN(dev, 1, "Preserved device has a 0 refcount!\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Decrement the refcount so this device does not get treated as an
+	 * incoming device again, e.g. in case pci_liveupdate_setup_device()
+	 * gets called again because the device is hot-plugged.
+	 */
+	if (--dev->liveupdate.incoming->refcount)
+		return 0;
+
+	pci_info(dev, "Device is finished participating in Live Update\n");
+	dev->liveupdate.incoming = NULL;
+	ser->nr_devices--;
+	return 0;
+}
+
+/**
+ * pci_liveupdate_finish() - Finish the preservation of a PCI device
+ * @dev: The PCI device
+ *
+ * pci_liveupdate_finish() notifies the PCI core that a PCI device that was
+ * preserved across the previous Live Update has finished participating in Live
+ * Update. Drivers must call pci_liveupdate_finish() from their struct
+ * liveupdate_file_handler finish() callback to ensure the incoming struct
+ * pci_ser is allocated.
+ *
+ * pci_liveupdate_finish() automatically finishes all bridges upstream of @dev.
+ */
+void pci_liveupdate_finish(struct pci_dev *dev)
+{
+	struct pci_flb_incoming *incoming;
+
+	guard(rwsem_write)(&pci_liveupdate.rwsem);
+
+	incoming = pci_liveupdate_flb_get_incoming();
+	if (!incoming) {
+		pci_warn(dev, "Cannot finish preserving device without incoming FLB\n");
+		return;
+	}
+
+	for_each_pci_dev_in_path(dev) {
+		if (pci_liveupdate_finish_device(incoming->ser, dev))
+			break;
+	}
+
+	pci_liveupdate_flb_put_incoming();
+}
+EXPORT_SYMBOL_GPL(pci_liveupdate_finish);
+
+/**
+ * pci_liveupdate_adopt_acs() - Adopt ACS controls
+ * @dev: The PCI device to adopt ACS controls for
+ *
+ * For devices preserved across a Live Update, leave the ACS controls
+ * established by the previous kernel alone instead of programming new ones.
+ * The adopted controls are recorded by the pci_save_state() call in
+ * pci_bus_add_device(), so they are reapplied by pci_restore_state() if the
+ * device is subsequently reset.
+ *
+ * Return: 0 on success, or -EINVAL if the device was not preserved, requires
+ * device-specific quirks, or has nowhere to record the adopted controls.
+ */
+int pci_liveupdate_adopt_acs(struct pci_dev *dev)
+{
+	/*
+	 * Check if the device was preserved over a previous Live Update (even
+	 * if it has already gone through pci_liveupdate_finish()). This ensures
+	 * that the device continues to use the ACS controls established by the
+	 * previous kernel.
+	 */
+	if (!dev->liveupdate.was_incoming)
+		return -EINVAL;
+
+	/*
+	 * The previous kernel should not have preserved any devices that
+	 * require device-specific quirks to enable ACS, but if such a device is
+	 * detected (e.g. new device-specific ACS quirk in the current kernel),
+	 * log a big warning and fall back to the normal enable ACS path.
+	 */
+	if (pci_need_dev_specific_enable_acs(dev)) {
+		pci_warn(dev, "Device-specific quirk required to enable ACS!\n");
+		WARN_ON_ONCE(true);
+		return -EINVAL;
+	}
+
+	/*
+	 * Adopting the previous kernel's controls depends on them being
+	 * captured in the ACS save buffer, so that they are reapplied if the
+	 * device is later reset. Without that buffer, e.g. because it could
+	 * not be allocated under memory pressure, the adopted controls would
+	 * be silently lost by the first reset. Program ACS from scratch
+	 * instead, which is a better outcome than leaving ACS disabled.
+	 */
+	if (dev->acs_cap && !pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ACS)) {
+		pci_err(dev, "No ACS save buffer, not adopting ACS controls\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * pci_liveupdate_adopt_ari() - Adopt ARI configuration
+ * @dev: The PCI device to adopt ARI configuration for
+ *
+ * For devices preserved across a Live Update, read the ARI state from
+ * hardware and adopt it. This ensures the device continues to use the ARI
+ * configuration established by the previous kernel.
+ *
+ * Return: 0 on success, or -EINVAL if the device was not preserved.
+ */
+int pci_liveupdate_adopt_ari(struct pci_dev *dev)
+{
+	u16 val;
+
+	guard(rwsem_read)(&pci_liveupdate.rwsem);
+
+	if (!dev->liveupdate.incoming)
+		return -EINVAL;
+
+	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &val);
+
+	/* Safe to modify dev->ari_enabled bitfield during enumeration. */
+	dev->ari_enabled = !!(val & PCI_EXP_DEVCTL2_ARI);
+	return 0;
+}
+
+bool pci_liveupdate_is_outgoing(struct pci_dev *dev)
+{
+	guard(rwsem_read)(&pci_liveupdate.rwsem);
+	pci_WARN_ONCE(dev, !dev->liveupdate.frozen, "Preservation status is unstable!\n");
+	return dev->liveupdate.outgoing;
+}
+
+/**
+ * pci_liveupdate_is_incoming() - Check if a device is incoming-preserved
+ * @dev: The PCI device to check
+ *
+ * Check if a device was preserved across Live Update by the previous kernel,
+ * i.e. the device is incoming-preserved. Note that a device is only considered
+ * incoming-preserved prior to pci_liveupdate_finish(). It is up to drivers to
+ * synchronize usage of pci_liveupdate_is_incoming() with their own call to
+ * pci_liveupdate_finish() to avoid acting on stale data.
+ *
+ * Returns: True if the device is incoming-preserved, false otherwise.
+ */
+bool pci_liveupdate_is_incoming(struct pci_dev *dev)
+{
+	guard(rwsem_read)(&pci_liveupdate.rwsem);
+	return dev->liveupdate.incoming;
+}
+EXPORT_SYMBOL_GPL(pci_liveupdate_is_incoming);
+
+/**
+ * pci_liveupdate_register_flb() - Register a file handler with the PCI core
+ * @fh: The file handler to register.
+ *
+ * Drivers that support preserving PCI devices across Live Update must call
+ * pci_liveupdate_register_flb() to register their
+ * struct liveupdate_file_handler with the PCI core, typically at module init,
+ * and always before any file managed by @fh can be preserved.
+ *
+ * Registering links the PCI core's FLB to @fh, so that LUO allocates the PCI
+ * core's outgoing struct pci_ser (via pci_flb_preserve()) when the first file
+ * managed by any registered handler is preserved, and frees it (via
+ * pci_flb_unpreserve()) when the last such file is unpreserved.
+ *
+ * Return: 0 on success, <0 on failure.
+ */
+int pci_liveupdate_register_flb(struct liveupdate_file_handler *fh)
+{
+	pr_debug("Registering file handler \"%s\"\n", fh->compatible);
+	return liveupdate_register_flb(fh, &pci_liveupdate_flb);
+}
+EXPORT_SYMBOL_GPL(pci_liveupdate_register_flb);
+
+/**
+ * pci_liveupdate_unregister_flb() - Unregister a file handler with the PCI core
+ * @fh: The file handler to unregister.
+ */
+void pci_liveupdate_unregister_flb(struct liveupdate_file_handler *fh)
+{
+	pr_debug("Unregistering file handler \"%s\"\n", fh->compatible);
+	liveupdate_unregister_flb(fh, &pci_liveupdate_flb);
+}
+EXPORT_SYMBOL_GPL(pci_liveupdate_unregister_flb);
diff --git a/drivers/pci/liveupdate.h b/drivers/pci/liveupdate.h
new file mode 100644
index 00000000000000..333ddd8f36c514
--- /dev/null
+++ b/drivers/pci/liveupdate.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * PCI Live Update support (core API)
+ *
+ * Copyright (c) 2026, Google LLC.
+ * David Matlack <dmatlack@google.com>
+ */
+#ifndef DRIVERS_PCI_LIVEUPDATE_H
+#define DRIVERS_PCI_LIVEUPDATE_H
+
+#include <linux/pci.h>
+
+#ifdef CONFIG_PCI_LIVEUPDATE
+void pci_liveupdate_setup_device(struct pci_dev *dev);
+void pci_liveupdate_cleanup_device(struct pci_dev *dev);
+void pci_liveupdate_freeze(struct pci_dev *dev);
+bool pci_liveupdate_preserve_bus_numbers(void);
+bool pci_liveupdate_refuse_bus_numbers(struct pci_bus *bus, struct pci_dev *dev);
+int pci_liveupdate_adopt_acs(struct pci_dev *dev);
+int pci_liveupdate_adopt_ari(struct pci_dev *dev);
+bool pci_liveupdate_is_outgoing(struct pci_dev *dev);
+#else
+static inline void pci_liveupdate_setup_device(struct pci_dev *dev)
+{
+}
+
+static inline void pci_liveupdate_cleanup_device(struct pci_dev *dev)
+{
+}
+
+static inline void pci_liveupdate_freeze(struct pci_dev *dev)
+{
+}
+
+static inline bool pci_liveupdate_preserve_bus_numbers(void)
+{
+	return false;
+}
+
+static inline bool pci_liveupdate_refuse_bus_numbers(struct pci_bus *bus,
+						     struct pci_dev *dev)
+{
+	return false;
+}
+
+static inline int pci_liveupdate_adopt_acs(struct pci_dev *dev)
+{
+	return -EINVAL;
+}
+
+static inline int pci_liveupdate_adopt_ari(struct pci_dev *dev)
+{
+	return -EINVAL;
+}
+
+static inline bool pci_liveupdate_is_outgoing(struct pci_dev *dev)
+{
+	return false;
+}
+#endif
+
+#endif /* DRIVERS_PCI_LIVEUPDATE_H */
diff --git a/drivers/pci/of_property.c b/drivers/pci/of_property.c
index 75a358f73e6940..1500740cc55d25 100644
--- a/drivers/pci/of_property.c
+++ b/drivers/pci/of_property.c
@@ -95,9 +95,13 @@ static int of_pci_prop_bus_range(struct pci_dev *pdev,
 				 struct of_changeset *ocs,
 				 struct device_node *np)
 {
-	u32 bus_range[] = { pdev->subordinate->busn_res.start,
-			    pdev->subordinate->busn_res.end };
+	u32 bus_range[2];
 
+	if (!pdev->subordinate)
+		return 0;
+
+	bus_range[0] = pdev->subordinate->busn_res.start;
+	bus_range[1] = pdev->subordinate->busn_res.end;
 	return of_changeset_add_prop_u32_array(ocs, np, "bus-range", bus_range,
 					       ARRAY_SIZE(bus_range));
 }
@@ -220,6 +224,9 @@ static int of_pci_prop_intr_map(struct pci_dev *pdev, struct of_changeset *ocs,
 	int ret;
 	u8 pin;
 
+	if (!pdev->subordinate)
+		return 0;
+
 	pnode = pci_device_to_OF_node(pdev->bus->self);
 	if (!pnode)
 		pnode = pci_bus_to_OF_node(pdev->bus);
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index 9334eb314663ee..beb1fe20c9de54 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -236,9 +236,8 @@ static void pci_p2pdma_release(void *data)
 		return;
 
 	/* Flush and disable pci_alloc_p2p_mem() */
-	pdev->p2pdma = NULL;
-	if (p2pdma->pool)
-		synchronize_rcu();
+	RCU_INIT_POINTER(pdev->p2pdma, NULL);
+	synchronize_rcu();
 	xa_destroy(&p2pdma->map_types);
 
 	if (!p2pdma->pool)
@@ -440,8 +439,8 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
 		goto pgmap_free;
 	}
 
-	error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_unmap_mappings,
-					 p2p_pgmap);
+	error = devm_add_action(&pdev->dev, pci_p2pdma_unmap_mappings,
+				p2p_pgmap);
 	if (error)
 		goto pages_free;
 
@@ -451,13 +450,15 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
 			range_len(&pgmap->range), dev_to_node(&pdev->dev),
 			&pgmap->ref);
 	if (error)
-		goto pages_free;
+		goto mappings_remove;
 
 	pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx\n",
 		 pgmap->range.start, pgmap->range.end);
 
 	return 0;
 
+mappings_remove:
+	devm_remove_action(&pdev->dev, pci_p2pdma_unmap_mappings, p2p_pgmap);
 pages_free:
 	devm_memunmap_pages(&pdev->dev, pgmap);
 pgmap_free:
@@ -569,6 +570,14 @@ static const struct pci_p2pdma_whitelist_entry {
 	{PCI_VENDOR_ID_NVIDIA, 0x2f96, 0},
 	{PCI_VENDOR_ID_NVIDIA, 0x2f97, 0},
 	{PCI_VENDOR_ID_NVIDIA, 0x2f98, 0},
+	/* Zhaoxin KX-6000/KH-40000/KX-6000G/KX-7000/KH-50000 */
+	{PCI_VENDOR_ID_ZHAOXIN, 0x1003, REQ_SAME_HOST_BRIDGE},
+	{PCI_VENDOR_ID_ZHAOXIN, 0x1005, REQ_SAME_HOST_BRIDGE},
+	{PCI_VENDOR_ID_ZHAOXIN, 0x1006, REQ_SAME_HOST_BRIDGE},
+	{PCI_VENDOR_ID_ZHAOXIN, 0x1007, REQ_SAME_HOST_BRIDGE},
+	{PCI_VENDOR_ID_ZHAOXIN, 0x1008, 0},
+	/* Alibaba T-HEAD Yitian 710 CPU */
+	{PCI_VENDOR_ID_ALIBABA, 0x8000, 0},
 	{}
 };
 
@@ -708,7 +717,6 @@ calc_map_type_and_dist(struct pci_dev *provider, struct pci_dev *client,
 {
 	enum pci_p2pdma_map_type map_type = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
 	struct pci_dev *a = provider, *b = client, *bb;
-	bool acs_redirects = false;
 	struct pci_p2pdma *p2pdma;
 	struct seq_buf acs_list;
 	int acs_cnt = 0;
@@ -771,17 +779,18 @@ calc_map_type_and_dist(struct pci_dev *provider, struct pci_dev *client,
 	}
 
 	if (verbose) {
-		acs_list.buffer[acs_list.len-1] = 0; /* drop final semicolon */
+		/* Drop the final semicolon; the list is not empty here */
+		if (!seq_buf_has_overflowed(&acs_list))
+			acs_list.buffer[acs_list.len - 1] = '\0';
 		pci_warn(client, "ACS redirect is set between the client and provider (%s)\n",
 			 pci_name(provider));
 		pci_warn(client, "to disable ACS redirect for this path, add the kernel parameter: pci=disable_acs_redir=%s\n",
-			 acs_list.buffer);
+			 seq_buf_str(&acs_list));
 	}
-	acs_redirects = true;
 
 map_through_host_bridge:
 	if (!cpu_supports_p2pdma() &&
-	    !host_bridge_whitelist(provider, client, acs_redirects)) {
+	    !host_bridge_whitelist(provider, client, verbose)) {
 		if (verbose)
 			pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share an upstream bridge or whitelisted host bridge\n",
 				 pci_name(provider));
@@ -867,7 +876,13 @@ static bool pci_has_p2pmem(struct pci_dev *pdev)
 
 	rcu_read_lock();
 	p2pdma = rcu_dereference(pdev->p2pdma);
-	res = p2pdma && p2pdma->p2pmem_published;
+
+	/*
+	 * The callers hand the result to pci_alloc_p2pmem(), so only a
+	 * provider backed by a pool is of any use here. pcim_p2pdma_init()
+	 * creates providers without one.
+	 */
+	res = p2pdma && p2pdma->pool && p2pdma->p2pmem_published;
 	rcu_read_unlock();
 
 	return res;
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index e16aa59dd7ac8b..c1ac2ef025e233 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -21,6 +21,7 @@
 #include <linux/acpi.h>
 #include <linux/dma-map-ops.h>
 #include <linux/iommu.h>
+#include "liveupdate.h"
 #include "pci.h"
 #include "pcie/portdrv.h"
 
@@ -559,6 +560,7 @@ static void pci_device_shutdown(struct device *dev)
 	struct pci_dev *pci_dev = to_pci_dev(dev);
 	struct pci_driver *drv = pci_dev->driver;
 
+	pci_liveupdate_freeze(pci_dev);
 	pm_runtime_resume(dev);
 
 	if (drv && drv->shutdown)
@@ -567,11 +569,14 @@ static void pci_device_shutdown(struct device *dev)
 	/*
 	 * If this is a kexec reboot, turn off Bus Master bit on the
 	 * device to tell it to not continue to do DMA. Don't touch
-	 * devices in D3cold or unknown states.
+	 * devices being preserved for Live Update or in D3cold or
+	 * unknown states.
+	 *
 	 * If it is not a kexec reboot, firmware will hit the PCI
 	 * devices with big hammer and stop their DMA any way.
 	 */
-	if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
+	if (kexec_in_progress && !pci_liveupdate_is_outgoing(pci_dev) &&
+	    pci_dev->current_state <= PCI_D3hot)
 		pci_clear_master(pci_dev);
 }
 
diff --git a/drivers/pci/pci-label.c b/drivers/pci/pci-label.c
index 0c644651964043..08c461f1142d7c 100644
--- a/drivers/pci/pci-label.c
+++ b/drivers/pci/pci-label.c
@@ -144,7 +144,7 @@ static int dsm_label_utf16s_to_utf8s(union acpi_object *obj, char *buf)
 	int len;
 
 	len = utf16s_to_utf8s((const wchar_t *)obj->buffer.pointer,
-			      obj->buffer.length,
+			      obj->buffer.length / sizeof(wchar_t),
 			      UTF16_LITTLE_ENDIAN,
 			      buf, PAGE_SIZE - 1);
 	buf[len++] = '\n';
@@ -157,40 +157,53 @@ static int dsm_get_label(struct device *dev, char *buf,
 {
 	acpi_handle handle = ACPI_HANDLE(dev);
 	union acpi_object *obj, *tmp;
-	int len = 0;
+	int len;
 
 	if (!handle)
-		return -1;
+		return -ENODEV;
 
 	obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 0x2,
 				DSM_PCI_DEVICE_NAME, NULL);
 	if (!obj)
-		return -1;
+		return -EIO;
+
+	if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != 2) {
+		len = -EIO;
+		goto out;
+	}
 
 	tmp = obj->package.elements;
-	if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 2 &&
-	    tmp[0].type == ACPI_TYPE_INTEGER &&
-	    (tmp[1].type == ACPI_TYPE_STRING ||
-	     tmp[1].type == ACPI_TYPE_BUFFER)) {
-		/*
-		 * The second string element is optional even when
-		 * this _DSM is implemented; when not implemented,
-		 * this entry must return a null string.
-		 */
-		if (attr == ACPI_ATTR_INDEX_SHOW) {
-			len = sysfs_emit(buf, "%llu\n", tmp->integer.value);
-		} else if (attr == ACPI_ATTR_LABEL_SHOW) {
-			if (tmp[1].type == ACPI_TYPE_STRING)
-				len = sysfs_emit(buf, "%s\n",
-						 tmp[1].string.pointer);
-			else if (tmp[1].type == ACPI_TYPE_BUFFER)
-				len = dsm_label_utf16s_to_utf8s(tmp + 1, buf);
-		}
+	if (tmp[0].type != ACPI_TYPE_INTEGER) {
+		len = -EIO;
+		goto out;
+	}
+
+	if (attr == ACPI_ATTR_INDEX_SHOW) {
+		len = sysfs_emit(buf, "%llu\n", tmp[0].integer.value);
+		goto out;
 	}
 
+	/*
+	 * Per PCI Firmware r3.3, sec 4.6.7, the device name is optional
+	 * even when this _DSM is implemented. When not implemented, this
+	 * entry must return a NULL string.
+	 */
+	switch (tmp[1].type) {
+	case ACPI_TYPE_STRING:
+		len = sysfs_emit(buf, "%s\n", tmp[1].string.pointer);
+		break;
+	case ACPI_TYPE_BUFFER:
+		len = dsm_label_utf16s_to_utf8s(&tmp[1], buf);
+		break;
+	default:
+		len = -EIO;
+		break;
+	}
+
+out:
 	ACPI_FREE(obj);
 
-	return len > 0 ? len : -1;
+	return len;
 }
 
 static ssize_t label_show(struct device *dev, struct device_attribute *attr,
@@ -217,11 +230,42 @@ static umode_t acpi_attr_is_visible(struct kobject *kobj, struct attribute *a,
 				    int n)
 {
 	struct device *dev = kobj_to_dev(kobj);
+	union acpi_object *obj, *tmp;
+	umode_t mode = 0;
 
 	if (!device_has_acpi_name(dev))
 		return 0;
 
-	return a->mode;
+	/*
+	 * The bitmap from _DSM function 0 only advertises function 7,
+	 * and whether the returned object can be parsed is a separate
+	 * question. Evaluate it and expose each attribute only if the
+	 * element it exports has one of the types the read path
+	 * accepts, mirroring the checks in dsm_get_label().
+	 */
+	obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &pci_acpi_dsm_guid, 0x2,
+				DSM_PCI_DEVICE_NAME, NULL);
+	if (!obj)
+		return 0;
+
+	if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != 2)
+		goto out;
+
+	tmp = obj->package.elements;
+	if (tmp[0].type != ACPI_TYPE_INTEGER)
+		goto out;
+
+	if (a == &dev_attr_acpi_index.attr)
+		mode = a->mode;
+	else if (a == &dev_attr_label.attr &&
+		 (tmp[1].type == ACPI_TYPE_STRING ||
+		  tmp[1].type == ACPI_TYPE_BUFFER))
+		mode = a->mode;
+
+out:
+	ACPI_FREE(obj);
+
+	return mode;
 }
 
 const struct attribute_group pci_dev_acpi_attr_group = {
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index b2879a6be5f808..0ae405b86b2f80 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -35,6 +35,8 @@
 #include <linux/aer.h>
 #include <linux/bitfield.h>
 #include <linux/suspend.h>
+
+#include "liveupdate.h"
 #include "pci.h"
 
 DEFINE_MUTEX(pci_slot_mutex);
@@ -1021,6 +1023,55 @@ static void pci_std_enable_acs(struct pci_dev *dev, struct pci_acs *caps)
 		caps->ctrl |= (dev->acs_capabilities & PCI_ACS_TB);
 }
 
+/**
+ * pci_save_acs_state - save the ACS Control register
+ * @dev: the PCI device
+ *
+ * Record the ACS controls currently programmed in hardware so that
+ * pci_restore_acs_state() can reapply them after a reset.
+ */
+static void pci_save_acs_state(struct pci_dev *dev)
+{
+	struct pci_cap_saved_state *save_state;
+
+	if (!dev->acs_cap)
+		return;
+
+	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ACS);
+	if (!save_state)
+		return;
+
+	pci_read_config_word(dev, dev->acs_cap + PCI_ACS_CTRL,
+			     (u16 *)&save_state->cap.data[0]);
+}
+
+/**
+ * pci_restore_acs_state - restore the ACS Control register
+ * @dev: the PCI device
+ */
+static void pci_restore_acs_state(struct pci_dev *dev)
+{
+	struct pci_cap_saved_state *save_state = NULL;
+
+	if (dev->acs_cap && !pci_need_dev_specific_enable_acs(dev))
+		save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ACS);
+
+	/*
+	 * Devices that rely on device-specific quirks to enable an ACS
+	 * equivalent keep that configuration outside of the ACS Control
+	 * register, so there is nothing useful to restore for them. Configure
+	 * ACS from scratch instead, which also covers devices that have no
+	 * saved ACS state at all.
+	 */
+	if (!save_state) {
+		pci_enable_acs(dev);
+		return;
+	}
+
+	pci_write_config_word(dev, dev->acs_cap + PCI_ACS_CTRL,
+			      *(u16 *)&save_state->cap.data[0]);
+}
+
 /**
  * pci_enable_acs - enable ACS if hardware support it
  * @dev: the PCI device
@@ -1031,6 +1082,9 @@ void pci_enable_acs(struct pci_dev *dev)
 	bool enable_acs = false;
 	int pos;
 
+	if (!pci_liveupdate_adopt_acs(dev))
+		return;
+
 	/* If an iommu is present we start with kernel default caps */
 	if (pci_acs_enable) {
 		if (pci_dev_specific_enable_acs(dev))
@@ -1057,6 +1111,15 @@ void pci_enable_acs(struct pci_dev *dev)
 	__pci_config_acs(dev, &caps, config_acs_param, 0, 0);
 
 	pci_write_config_word(dev, pos + PCI_ACS_CTRL, caps.ctrl);
+
+	/*
+	 * pci_enable_acs() runs when a driver binds to the device, i.e. after
+	 * pci_bus_add_device() has already saved the device's state. Refresh
+	 * the saved ACS Control register so that a subsequent reset restores
+	 * the controls programmed here rather than the ones left behind by
+	 * firmware.
+	 */
+	pci_save_acs_state(dev);
 }
 
 /**
@@ -1800,6 +1863,7 @@ int pci_save_state(struct pci_dev *dev)
 	pci_save_aer_state(dev);
 	pci_save_ptm_state(dev);
 	pci_save_tph_state(dev);
+	pci_save_acs_state(dev);
 	return pci_save_vc_state(dev);
 }
 EXPORT_SYMBOL(pci_save_state);
@@ -1877,7 +1941,7 @@ void pci_restore_state(struct pci_dev *dev)
 	pci_restore_msi_state(dev);
 
 	/* Restore ACS and IOV configuration state */
-	pci_enable_acs(dev);
+	pci_restore_acs_state(dev);
 	pci_restore_iov_state(dev);
 
 	dev->state_saved = false;
@@ -3532,6 +3596,11 @@ void pci_allocate_cap_save_buffers(struct pci_dev *dev)
 	if (error)
 		pci_err(dev, "unable to allocate suspend buffer for LTR\n");
 
+	error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ACS,
+					    sizeof(u16));
+	if (error)
+		pci_err(dev, "unable to allocate suspend buffer for ACS\n");
+
 	pci_allocate_vc_save_buffers(dev);
 }
 
@@ -3556,7 +3625,7 @@ void pci_configure_ari(struct pci_dev *dev)
 	u32 cap;
 	struct pci_dev *bridge;
 
-	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
+	if (!pci_is_pcie(dev) || dev->devfn)
 		return;
 
 	bridge = dev->bus->self;
@@ -3567,6 +3636,12 @@ void pci_configure_ari(struct pci_dev *dev)
 	if (!(cap & PCI_EXP_DEVCAP2_ARI))
 		return;
 
+	if (!pci_liveupdate_adopt_ari(bridge))
+		return;
+
+	if (pcie_ari_disabled)
+		return;
+
 	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
 		pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
 					 PCI_EXP_DEVCTL2_ARI);
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index ba3c3fddddc238..b7b83bfb95a6c4 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -40,6 +40,15 @@ struct pcie_tlp_log;
  */
 #define PCIE_T_PERST_CLK_US		100
 
+/*
+ * PERST# active time.
+ *
+ * See the "Power Sequencing and Reset Signal Timings" table of the PCI Express
+ * Card Electromechanical Specification, Revision 6.0, Section 2.11.2, Symbol
+ * "T_PERST".
+ */
+#define PCIE_T_PERST_US			100
+
 /*
  * PCIe r6.0, sec 5.3.3.2.1 <PME Synchronization>
  * Recommends 1ms to 10ms timeout to check L2 ready.
@@ -711,10 +720,6 @@ struct pci_sriov {
 	u16		driver_max_VFs;	/* Max num VFs driver supports */
 	struct pci_dev	*dev;		/* Lowest numbered PF */
 	struct pci_dev	*self;		/* This PF */
-	u32		class;		/* VF device */
-	u8		hdr_type;	/* VF header type */
-	u16		subsystem_vendor; /* VF subsystem vendor */
-	u16		subsystem_device; /* VF subsystem device */
 	resource_size_t	barsz[PCI_SRIOV_NUM_BARS];	/* VF BAR size */
 	u16		vf_rebar_cap;	/* VF Resizable BAR capability offset */
 	bool		drivers_autoprobe; /* Auto probing of VFs by driver */
@@ -1095,6 +1100,7 @@ void pci_acs_init(struct pci_dev *dev);
 void pci_enable_acs(struct pci_dev *dev);
 #ifdef CONFIG_PCI_QUIRKS
 int pci_dev_specific_acs_enabled(struct pci_dev *dev, u16 acs_flags);
+bool pci_need_dev_specific_enable_acs(struct pci_dev *dev);
 int pci_dev_specific_enable_acs(struct pci_dev *dev);
 int pci_dev_specific_disable_acs_redir(struct pci_dev *dev);
 void pci_disable_broken_acs_cap(struct pci_dev *pdev);
@@ -1105,6 +1111,10 @@ static inline int pci_dev_specific_acs_enabled(struct pci_dev *dev,
 {
 	return -ENOTTY;
 }
+static inline bool pci_need_dev_specific_enable_acs(struct pci_dev *dev)
+{
+	return false;
+}
 static inline int pci_dev_specific_enable_acs(struct pci_dev *dev)
 {
 	return -ENOTTY;
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 95ac34a34bd5b1..f8e64971c6c3db 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -1504,16 +1504,23 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked
 		return -EINVAL;
 	/*
 	 * A driver requested that ASPM be disabled on this device, but
-	 * if we don't have permission to manage ASPM (e.g., on ACPI
+	 * if we might not have permission to manage ASPM (e.g., on ACPI
 	 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
-	 * the _OSC method), we can't honor that request.  Windows has
-	 * a similar mechanism using "PciASPMOptOut", which is also
-	 * ignored in this situation.
+	 * the _OSC method), previously we chose to not honor disable
+	 * request in that case. Windows has a similar mechanism using
+	 * "PciASPMOptOut", which is also ignored in this situation.
+	 *
+	 * Not honoring the requests to disable ASPM, however, led to
+	 * drivers forcing ASPM off on their own. As such changes of ASPM
+	 * state are not tracked by this service driver, the state kept here
+	 * became out of sync.
+	 *
+	 * Therefore, honor ASPM disable requests even when OS does not have
+	 * ASPM control. Plain disable for ASPM is assumed to be slightly
+	 * safer than fully managing it.
 	 */
-	if (aspm_disabled) {
-		pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
-		return -EPERM;
-	}
+	if (aspm_disabled)
+		pci_warn(pdev, "OS doesn't have ASPM control, disabling ASPM anyway\n");
 
 	if (!locked)
 		down_read(&pci_bus_sem);
@@ -1531,6 +1538,17 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked
 	return 0;
 }
 
+/**
+ * pci_disable_link_state_locked - Disable device's link state
+ * @pdev: PCI device
+ * @state: ASPM link state to disable
+ *
+ * Disable device's link state so the link will never enter specific states.
+ *
+ * Context: Caller holds pci_bus_sem read lock.
+ *
+ * Return: 0 on success, a negative errno otherwise.
+ */
 int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
 {
 	lockdep_assert_held_read(&pci_bus_sem);
@@ -1540,13 +1558,13 @@ int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
 EXPORT_SYMBOL(pci_disable_link_state_locked);
 
 /**
- * pci_disable_link_state - Disable device's link state, so the link will
- * never enter specific states.  Note that if the BIOS didn't grant ASPM
- * control to the OS, this does nothing because we can't touch the LNKCTL
- * register. Returns 0 or a negative errno.
- *
+ * pci_disable_link_state - Disable device's link state
  * @pdev: PCI device
  * @state: ASPM link state to disable
+ *
+ * Disable device's link state so the link will never enter specific states.
+ *
+ * Return: 0 or a negative errno
  */
 int pci_disable_link_state(struct pci_dev *pdev, int state)
 {
@@ -1554,7 +1572,8 @@ int pci_disable_link_state(struct pci_dev *pdev, int state)
 }
 EXPORT_SYMBOL(pci_disable_link_state);
 
-static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)
+static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked,
+				   bool force)
 {
 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
 
@@ -1571,10 +1590,25 @@ static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)
 		return -EPERM;
 	}
 
+	/*
+	 * Ensure the device is in D0 before enabling PCI-PM L1 PM Substates, per
+	 * PCIe r6.0, sec 5.5.4.
+	 */
+	if (state & PCIE_LINK_STATE_L1_SS_PCIPM) {
+		if (locked)
+			pci_set_power_state_locked(pdev, PCI_D0);
+		else
+			pci_set_power_state(pdev, PCI_D0);
+	}
+
 	if (!locked)
 		down_read(&pci_bus_sem);
 	mutex_lock(&aspm_lock);
 	link->aspm_default = pci_calc_aspm_enable_mask(state);
+
+	/* Force enable states that were previously disabled */
+	if (force)
+		link->aspm_disable &= ~link->aspm_default;
 	pcie_config_aspm_link(link, policy_to_aspm_state(link));
 
 	link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;
@@ -1587,47 +1621,79 @@ static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)
 }
 
 /**
- * pci_enable_link_state - Clear and set the default device link state so that
- * the link may be allowed to enter the specified states. Note that if the
- * BIOS didn't grant ASPM control to the OS, this does nothing because we can't
- * touch the LNKCTL register. Also note that this does not enable states
- * disabled by pci_disable_link_state(). Return 0 or a negative errno.
- *
- * Note: Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per
- * PCIe r6.0, sec 5.5.4.
- *
+ * pci_enable_link_state - Enable device's link state
  * @pdev: PCI device
  * @state: Mask of ASPM link states to enable
+ *
+ * Enable device's link state, so the link will enter the specified states.
+ * Note that if the BIOS didn't grant ASPM control to the OS, this does
+ * nothing because we can't touch the LNKCTL register. Also note that this
+ * does not enable states disabled by pci_disable_link_state(). Use
+ * pci_force_enable_link_state() for that.
+ *
+ * Note: The device will be transitioned to D0 state if the PCI-PM L1 Substates
+ * are getting enabled.
+ *
+ * Return: 0 on success, a negative errno otherwise.
  */
 int pci_enable_link_state(struct pci_dev *pdev, int state)
 {
-	return __pci_enable_link_state(pdev, state, false);
+	return __pci_enable_link_state(pdev, state, false, false);
 }
 EXPORT_SYMBOL(pci_enable_link_state);
 
 /**
- * pci_enable_link_state_locked - Clear and set the default device link state
- * so that the link may be allowed to enter the specified states. Note that if
- * the BIOS didn't grant ASPM control to the OS, this does nothing because we
- * can't touch the LNKCTL register. Also note that this does not enable states
- * disabled by pci_disable_link_state(). Return 0 or a negative errno.
- *
- * Note: Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per
- * PCIe r6.0, sec 5.5.4.
- *
+ * pci_enable_link_state_locked - Enable device's link state
  * @pdev: PCI device
  * @state: Mask of ASPM link states to enable
  *
+ * Enable device's link state, so the link will enter the specified states.
+ * Note that if the BIOS didn't grant ASPM control to the OS, this does
+ * nothing because we can't touch the LNKCTL register. Also note that this
+ * does not enable states disabled by pci_disable_link_state(). Use
+ * pci_force_enable_link_state() for that.
+ *
+ * Note: The device will be transitioned to D0 state if the PCI-PM L1 Substates
+ * are getting enabled.
+ *
  * Context: Caller holds pci_bus_sem read lock.
+ *
+ * Return: 0 on success, a negative errno otherwise.
  */
 int pci_enable_link_state_locked(struct pci_dev *pdev, int state)
 {
 	lockdep_assert_held_read(&pci_bus_sem);
 
-	return __pci_enable_link_state(pdev, state, true);
+	return __pci_enable_link_state(pdev, state, true, false);
 }
 EXPORT_SYMBOL(pci_enable_link_state_locked);
 
+/**
+ * pci_force_enable_link_state - Force enable device link state
+ * @pdev: PCI device
+ * @state: Mask of ASPM link states to enable
+ *
+ * Enable device link state, so the link will enter the specified states.
+ * Unlike pci_enable_link_state(), this also re-enables states previously
+ * disabled by pci_disable_link_state() and overrides the ASPM states disabled
+ * during init (e.g., the pre-1.1 device blacklist). The caller is therefore
+ * responsible for only enabling states the device supports, typically the ones
+ * previously reported by pcie_aspm_enabled().
+ *
+ * Note that if the BIOS didn't grant ASPM control to the OS, this does nothing
+ * because we can't touch the LNKCTL register.
+ *
+ * Note: The device will be transitioned to D0 state if the PCI-PM L1 Substates
+ * are getting enabled.
+ *
+ * Return: 0 on success, a negative errno otherwise.
+ */
+int pci_force_enable_link_state(struct pci_dev *pdev, int state)
+{
+	return __pci_enable_link_state(pdev, state, false, true);
+}
+EXPORT_SYMBOL(pci_force_enable_link_state);
+
 void pcie_aspm_remove_cap(struct pci_dev *pdev, u32 lnkcap)
 {
 	if (lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)
@@ -1683,15 +1749,17 @@ module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
 	NULL, 0644);
 
 /**
- * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
+ * pcie_aspm_enabled - Get the enabled PCIe ASPM states for a device.
  * @pdev: Target device.
  *
  * Relies on the upstream bridge's link_state being valid.  The link_state
  * is deallocated only when the last child of the bridge (i.e., @pdev or a
  * sibling) is removed, and the caller should be holding a reference to
  * @pdev, so this should be safe.
+ *
+ * Return: Enabled PCIe ASPM states. 0 if ASPM is disabled.
  */
-bool pcie_aspm_enabled(struct pci_dev *pdev)
+u32 pcie_aspm_enabled(struct pci_dev *pdev)
 {
 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
 
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 27008e2ea5afc7..fbc4d02c81e57e 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -24,6 +24,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/bitfield.h>
 #include <trace/events/pci.h>
+#include "liveupdate.h"
 #include "pci.h"
 
 static struct resource busn_resource = {
@@ -1396,6 +1397,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
 				  int max, unsigned int available_buses,
 				  int pass)
 {
+	bool preserve_bus_numbers = !pcibios_assign_all_busses() ||
+				    pci_liveupdate_preserve_bus_numbers();
 	struct pci_bus *child;
 	u32 buses;
 	u16 bctl;
@@ -1448,8 +1451,7 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
 		goto out;
 	}
 
-	if ((secondary || subordinate) &&
-	    !pcibios_assign_all_busses() && !broken) {
+	if ((secondary || subordinate) && preserve_bus_numbers && !broken) {
 		unsigned int cmax, buses;
 
 		/*
@@ -1491,8 +1493,7 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
 		 * do in the second pass.
 		 */
 		if (!pass) {
-			if (pcibios_assign_all_busses() || broken)
-
+			if (!preserve_bus_numbers || broken)
 				/*
 				 * Temporarily disable forwarding of the
 				 * configuration cycles on all bridges in
@@ -1506,6 +1507,9 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
 			goto out;
 		}
 
+		if (pci_liveupdate_refuse_bus_numbers(bus, dev))
+			goto out;
+
 		/* Clear errors */
 		pci_write_config_word(dev, PCI_STATUS, 0xffff);
 
@@ -1890,8 +1894,13 @@ static u32 pci_class(struct pci_dev *dev)
 	u32 class;
 
 #ifdef CONFIG_PCI_IOV
-	if (dev->is_virtfn)
-		return dev->physfn->sriov->class;
+	if (dev->is_virtfn) {
+		u8 rev;
+
+		if (pci_read_config_byte(dev, PCI_REVISION_ID, &rev))
+			rev = 0;
+		return (dev->physfn->class << 8) | rev;
+	}
 #endif
 	pci_read_config_dword(dev, PCI_CLASS_REVISION, &class);
 	return class;
@@ -1901,8 +1910,8 @@ static void pci_subsystem_ids(struct pci_dev *dev, u16 *vendor, u16 *device)
 {
 #ifdef CONFIG_PCI_IOV
 	if (dev->is_virtfn) {
-		*vendor = dev->physfn->sriov->subsystem_vendor;
-		*device = dev->physfn->sriov->subsystem_device;
+		*vendor = dev->physfn->subsystem_vendor;
+		pci_read_config_word(dev, PCI_SUBSYSTEM_ID, device);
 		return;
 	}
 #endif
@@ -1916,7 +1925,7 @@ static u8 pci_hdr_type(struct pci_dev *dev)
 
 #ifdef CONFIG_PCI_IOV
 	if (dev->is_virtfn)
-		return dev->physfn->sriov->hdr_type;
+		return 0;
 #endif
 	pci_read_config_byte(dev, PCI_HEADER_TYPE, &hdr_type);
 	return hdr_type;
@@ -2064,6 +2073,8 @@ int pci_setup_device(struct pci_dev *dev)
 	if (pci_early_dump)
 		early_dump_pci_device(dev);
 
+	pci_liveupdate_setup_device(dev);
+
 	/* Need to have dev->class ready */
 	dev->cfg_size = pci_cfg_space_size(dev);
 
@@ -2187,6 +2198,7 @@ int pci_setup_device(struct pci_dev *dev)
 	default:				    /* unknown header */
 		pci_err(dev, "unknown header type %02x, ignoring device\n",
 			dev->hdr_type);
+		pci_liveupdate_cleanup_device(dev);
 		pci_release_of_node(dev);
 		return -EIO;
 
@@ -2485,6 +2497,7 @@ static void pci_release_dev(struct device *dev)
 
 	pci_dev = to_pci_dev(dev);
 	pci_release_capabilities(pci_dev);
+	pci_liveupdate_cleanup_device(pci_dev);
 	pci_release_of_node(pci_dev);
 	pcibios_release_device(pci_dev);
 	pci_bus_put(pci_dev->bus);
diff --git a/drivers/pci/pwrctrl/Kconfig b/drivers/pci/pwrctrl/Kconfig
index 9eec767cda86e2..38aab596aa04b8 100644
--- a/drivers/pci/pwrctrl/Kconfig
+++ b/drivers/pci/pwrctrl/Kconfig
@@ -29,6 +29,8 @@ config PCI_PWRCTRL_TC9563
 	select PCI_PWRCTRL
 	default m if ARCH_QCOM
 	depends on I2C
+	depends on GPIO_TC9563
+	select REGMAP_I2C
 	help
 	  Say Y here to enable the PCI Power Control driver of TC9563 PCIe
 	  switch.
diff --git a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c
index fd53a7db9875d8..59bc0d77d3c444 100644
--- a/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c
+++ b/drivers/pci/pwrctrl/pci-pwrctrl-tc9563.c
@@ -4,28 +4,28 @@
  */
 
 #include <linux/array_size.h>
+#include <linux/auxiliary_bus.h>
 #include <linux/bitfield.h>
 #include <linux/bits.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
+#include <linux/idr.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/pci.h>
 #include <linux/pci-pwrctrl.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
+#include <linux/soc/qcom/tc9563.h>
 #include <linux/string.h>
 #include <linux/types.h>
-#include <linux/unaligned.h>
 
 #include "../pci.h"
 
-#define TC9563_GPIO_CONFIG		0x801208
-#define TC9563_RESET_GPIO		0x801210
-
 #define TC9563_PORT_L0S_DELAY		0x82496c
 #define TC9563_PORT_L1_DELAY		0x824970
 
@@ -57,9 +57,6 @@
 #define TC9563_POWER_CONTROL		0x82b09c
 #define TC9563_POWER_CONTROL_OVREN	0x82b2c8
 
-#define TC9563_GPIO_MASK		0xfffffff3
-#define TC9563_GPIO_DEASSERT_BITS	0xc  /* Clear to deassert GPIO */
-
 #define TC9563_TX_MARGIN_MIN_UA		400000
 
 /*
@@ -70,11 +67,6 @@
 
 #define TC9563_L0S_L1_DELAY_UNIT_NS	256  /* Each unit represents 256 ns */
 
-struct tc9563_pwrctrl_reg_setting {
-	unsigned int offset;
-	unsigned int val;
-};
-
 enum tc9563_pwrctrl_ports {
 	TC9563_USP,
 	TC9563_DSP1,
@@ -90,6 +82,7 @@ struct tc9563_pwrctrl_cfg {
 	u8 nfts[2]; /* GEN1 & GEN2 */
 	bool disable_dfe;
 	bool disable_port;
+	struct gpio_desc *reset;
 };
 
 #define TC9563_PWRCTL_MAX_SUPPLY	6
@@ -110,14 +103,14 @@ struct tc9563_pwrctrl {
 	struct tc9563_pwrctrl_cfg ep_cfg;
 	struct gpio_desc *reset_gpio;
 	struct i2c_adapter *adapter;
-	struct i2c_client *client;
+	struct regmap *regmap;
 };
 
 /*
  * downstream port power off sequence, hardcoding the address
  * as we don't know register names for these register offsets.
  */
-static const struct tc9563_pwrctrl_reg_setting common_pwroff_seq[] = {
+static const struct reg_sequence common_pwroff_seq[] = {
 	{0x82900c, 0x1},
 	{0x829010, 0x1},
 	{0x829018, 0x0},
@@ -140,7 +133,7 @@ static const struct tc9563_pwrctrl_reg_setting common_pwroff_seq[] = {
 	{0x829114, 0x1},
 };
 
-static const struct tc9563_pwrctrl_reg_setting dsp1_pwroff_seq[] = {
+static const struct reg_sequence dsp1_pwroff_seq[] = {
 	{TC9563_PORT_ACCESS_ENABLE, 0x2},
 	{TC9563_PORT_LANE_ACCESS_ENABLE, 0x3},
 	{TC9563_POWER_CONTROL, 0x014f4804},
@@ -148,7 +141,7 @@ static const struct tc9563_pwrctrl_reg_setting dsp1_pwroff_seq[] = {
 	{TC9563_PORT_ACCESS_ENABLE, 0x4},
 };
 
-static const struct tc9563_pwrctrl_reg_setting dsp2_pwroff_seq[] = {
+static const struct reg_sequence dsp2_pwroff_seq[] = {
 	{TC9563_PORT_ACCESS_ENABLE, 0x8},
 	{TC9563_PORT_LANE_ACCESS_ENABLE, 0x1},
 	{TC9563_POWER_CONTROL, 0x014f4804},
@@ -156,85 +149,13 @@ static const struct tc9563_pwrctrl_reg_setting dsp2_pwroff_seq[] = {
 	{TC9563_PORT_ACCESS_ENABLE, 0x8},
 };
 
-/*
- * Since all transfers are initiated by the probe, no locks are necessary,
- * as there are no concurrent calls.
- */
-static int tc9563_pwrctrl_i2c_write(struct i2c_client *client,
-				    u32 reg_addr, u32 reg_val)
-{
-	struct i2c_msg msg;
-	u8 msg_buf[7];
-	int ret;
-
-	msg.addr = client->addr;
-	msg.len = 7;
-	msg.flags = 0;
-
-	/* Big Endian for reg addr */
-	put_unaligned_be24(reg_addr, &msg_buf[0]);
-
-	/* Little Endian for reg val */
-	put_unaligned_le32(reg_val, &msg_buf[3]);
-
-	msg.buf = msg_buf;
-	ret = i2c_transfer(client->adapter, &msg, 1);
-	return ret == 1 ? 0 : ret;
-}
-
-static int tc9563_pwrctrl_i2c_read(struct i2c_client *client,
-				   u32 reg_addr, u32 *reg_val)
-{
-	struct i2c_msg msg[2];
-	u8 wr_data[3];
-	u32 rd_data;
-	int ret;
-
-	msg[0].addr = client->addr;
-	msg[0].len = 3;
-	msg[0].flags = 0;
-
-	/* Big Endian for reg addr */
-	put_unaligned_be24(reg_addr, &wr_data[0]);
-
-	msg[0].buf = wr_data;
-
-	msg[1].addr = client->addr;
-	msg[1].len = 4;
-	msg[1].flags = I2C_M_RD;
-
-	msg[1].buf = (u8 *)&rd_data;
-
-	ret = i2c_transfer(client->adapter, &msg[0], 2);
-	if (ret == 2) {
-		*reg_val = get_unaligned_le32(&rd_data);
-		return 0;
-	}
-
-	/* If only one message successfully completed, return -EIO */
-	return ret == 1 ? -EIO : ret;
-}
-
-static int tc9563_pwrctrl_i2c_bulk_write(struct i2c_client *client,
-				const struct tc9563_pwrctrl_reg_setting *seq,
-				int len)
-{
-	int ret, i;
-
-	for (i = 0; i < len; i++) {
-		ret = tc9563_pwrctrl_i2c_write(client, seq[i].offset, seq[i].val);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
+static DEFINE_IDA(tc9563_pwrctrl_ida);
 
 static int tc9563_pwrctrl_disable_port(struct tc9563_pwrctrl *tc9563,
 				       enum tc9563_pwrctrl_ports port)
 {
 	struct tc9563_pwrctrl_cfg *cfg = &tc9563->cfg[port];
-	const struct tc9563_pwrctrl_reg_setting *seq;
+	const struct reg_sequence *seq;
 	int ret, len;
 
 	if (!cfg->disable_port)
@@ -254,12 +175,12 @@ static int tc9563_pwrctrl_disable_port(struct tc9563_pwrctrl *tc9563,
 		return 0;
 	}
 
-	ret = tc9563_pwrctrl_i2c_bulk_write(tc9563->client, seq, len);
+	ret = regmap_multi_reg_write(tc9563->regmap, seq, len);
 	if (ret)
 		return ret;
 
-	return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, common_pwroff_seq,
-					     ARRAY_SIZE(common_pwroff_seq));
+	return regmap_multi_reg_write(tc9563->regmap, common_pwroff_seq,
+				      ARRAY_SIZE(common_pwroff_seq));
 }
 
 static int tc9563_pwrctrl_set_port_l0s_l1_entry_delay(struct tc9563_pwrctrl *tc9563,
@@ -275,14 +196,13 @@ static int tc9563_pwrctrl_set_port_l0s_l1_entry_delay(struct tc9563_pwrctrl *tc9
 	/* convert to units of 256ns */
 	units = ns / TC9563_L0S_L1_DELAY_UNIT_NS;
 
-	ret = tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_PORT_SELECT,
-				       BIT(port));
+	ret = regmap_write(tc9563->regmap, TC9563_PORT_SELECT, BIT(port));
 	if (ret)
 		return ret;
 
-	return tc9563_pwrctrl_i2c_write(tc9563->client,
-			is_l1 ? TC9563_PORT_L1_DELAY : TC9563_PORT_L0S_DELAY,
-			units);
+	return regmap_write(tc9563->regmap,
+			    is_l1 ? TC9563_PORT_L1_DELAY : TC9563_PORT_L0S_DELAY,
+			    units);
 }
 
 static int tc9563_pwrctrl_set_eth_l0s_l1_entry_delay(struct tc9563_pwrctrl *tc9563,
@@ -297,8 +217,7 @@ static int tc9563_pwrctrl_set_eth_l0s_l1_entry_delay(struct tc9563_pwrctrl *tc95
 	/* convert to units of 256ns */
 	units = ns / TC9563_L0S_L1_DELAY_UNIT_NS;
 
-	ret = tc9563_pwrctrl_i2c_read(tc9563->client, TC9563_EMBEDDED_ETH_DELAY,
-				      &rd_val);
+	ret = regmap_read(tc9563->regmap, TC9563_EMBEDDED_ETH_DELAY, &rd_val);
 	if (ret)
 		return ret;
 
@@ -309,8 +228,7 @@ static int tc9563_pwrctrl_set_eth_l0s_l1_entry_delay(struct tc9563_pwrctrl *tc95
 		rd_val = u32_replace_bits(rd_val, units,
 					  TC9563_ETH_L0S_DELAY_MASK);
 
-	return tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_EMBEDDED_ETH_DELAY,
-					rd_val);
+	return regmap_write(tc9563->regmap, TC9563_EMBEDDED_ETH_DELAY, rd_val);
 }
 
 static int tc9563_pwrctrl_set_tx_amplitude(struct tc9563_pwrctrl *tc9563,
@@ -343,14 +261,14 @@ static int tc9563_pwrctrl_set_tx_amplitude(struct tc9563_pwrctrl *tc9563,
 		return -EINVAL;
 	}
 
-	struct tc9563_pwrctrl_reg_setting tx_amp_seq[] = {
+	struct reg_sequence tx_amp_seq[] = {
 		{TC9563_PORT_ACCESS_ENABLE, port_access},
 		{TC9563_PORT_LANE_ACCESS_ENABLE, 0x3},
 		{TC9563_TX_MARGIN, amp},
 	};
 
-	return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, tx_amp_seq,
-					     ARRAY_SIZE(tx_amp_seq));
+	return regmap_multi_reg_write(tc9563->regmap, tx_amp_seq,
+				      ARRAY_SIZE(tx_amp_seq));
 }
 
 static int tc9563_pwrctrl_disable_dfe(struct tc9563_pwrctrl *tc9563,
@@ -383,7 +301,7 @@ static int tc9563_pwrctrl_disable_dfe(struct tc9563_pwrctrl *tc9563,
 		return -EINVAL;
 	}
 
-	struct tc9563_pwrctrl_reg_setting disable_dfe_seq[] = {
+	struct reg_sequence disable_dfe_seq[] = {
 		{TC9563_PORT_ACCESS_ENABLE, port_access},
 		{TC9563_PORT_LANE_ACCESS_ENABLE, lane_access},
 		{TC9563_DFE_ENABLE, 0x0},
@@ -397,15 +315,15 @@ static int tc9563_pwrctrl_disable_dfe(struct tc9563_pwrctrl *tc9563,
 		{TC9563_PHY_RATE_CHANGE_OVERRIDE, 0x0},
 	};
 
-	return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, disable_dfe_seq,
-					     ARRAY_SIZE(disable_dfe_seq));
+	return regmap_multi_reg_write(tc9563->regmap, disable_dfe_seq,
+				      ARRAY_SIZE(disable_dfe_seq));
 }
 
 static int tc9563_pwrctrl_set_nfts(struct tc9563_pwrctrl *tc9563,
 				   enum tc9563_pwrctrl_ports port)
 {
 	u8 *nfts = tc9563->cfg[port].nfts;
-	struct tc9563_pwrctrl_reg_setting nfts_seq[] = {
+	struct reg_sequence nfts_seq[] = {
 		{TC9563_NFTS_2_5_GT, nfts[0]},
 		{TC9563_NFTS_5_GT, nfts[1]},
 	};
@@ -420,28 +338,51 @@ static int tc9563_pwrctrl_set_nfts(struct tc9563_pwrctrl *tc9563,
 		return 0;
 	}
 
-	ret =  tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_PORT_SELECT,
-					BIT(port));
+	ret =  regmap_write(tc9563->regmap, TC9563_PORT_SELECT, BIT(port));
 	if (ret)
 		return ret;
 
-	return tc9563_pwrctrl_i2c_bulk_write(tc9563->client, nfts_seq,
-					     ARRAY_SIZE(nfts_seq));
+	return regmap_multi_reg_write(tc9563->regmap, nfts_seq,
+				      ARRAY_SIZE(nfts_seq));
 }
 
 static int tc9563_pwrctrl_assert_deassert_reset(struct tc9563_pwrctrl *tc9563,
 						bool deassert)
 {
-	int ret, val;
-
-	ret = tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_GPIO_CONFIG,
-				       TC9563_GPIO_MASK);
-	if (ret)
-		return ret;
-
-	val = deassert ? TC9563_GPIO_DEASSERT_BITS : 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(tc9563->cfg); i++) {
+		int err;
+
+		if (tc9563->cfg[i].reset) {
+			err = gpiod_direction_output(tc9563->cfg[i].reset,
+						     !deassert);
+			if (err)
+				return err;
+		} else {
+			/* Fallback: legacy DTS without reset-gpios */
+			switch (i) {
+			case TC9563_DSP1:
+			case TC9563_DSP2:
+				err = regmap_clear_bits(tc9563->regmap,
+							TC9563_GPIO_CONFIG,
+							BIT(i + 1));
+				if (err)
+					return err;
+
+				err = regmap_assign_bits(tc9563->regmap,
+							 TC9563_RESET_GPIO,
+							 BIT(i + 1), deassert);
+				if (err)
+					return err;
+				break;
+			default:
+				break;
+			}
+		}
+	}
 
-	return tc9563_pwrctrl_i2c_write(tc9563->client, TC9563_RESET_GPIO, val);
+	return 0;
 }
 
 static int tc9563_pwrctrl_parse_device_dt(struct device_node *node,
@@ -476,12 +417,115 @@ static int tc9563_pwrctrl_parse_device_dt(struct device_node *node,
 	return 0;
 }
 
+static int tc9563_pwrctrl_parse_reset_line(struct tc9563_pwrctrl *tc9563)
+{
+	enum tc9563_pwrctrl_ports port = TC9563_USP;
+	struct device *dev = tc9563->pwrctrl.dev;
+	struct device_node *node = dev->of_node;
+
+	for_each_child_of_node_scoped(node, child) {
+		struct tc9563_pwrctrl_cfg *cfg;
+
+		if (++port >= TC9563_MAX)
+			break;
+
+		cfg = &tc9563->cfg[port];
+		if (cfg->reset) /* Already discovered */
+			continue;
+
+		cfg->reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(child),
+						   "reset", GPIOD_ASIS,
+						   NULL);
+		if (IS_ERR(cfg->reset)) {
+			int err = PTR_ERR(cfg->reset);
+
+			cfg->reset = NULL;
+			if (err != -ENOENT)
+				return dev_err_probe(dev, err,
+						     "failed to get reset\n");
+		}
+	}
+
+	return 0;
+}
+
+static void tc9563_pwrctrl_adev_release(struct device *dev)
+{
+	struct auxiliary_device *adev = to_auxiliary_dev(dev);
+
+	ida_free(&tc9563_pwrctrl_ida, adev->id);
+	of_node_put(adev->dev.of_node);
+	kfree(adev);
+}
+
+static void tc9563_pwrctrl_adev_remove(void *data)
+{
+	struct auxiliary_device *adev = data;
+
+	auxiliary_device_delete(adev);
+	auxiliary_device_uninit(adev);
+}
+
+static int tc9563_pwrctrl_adev_add(struct device *dev, const char *name,
+				   struct device_node *of_node,
+				   void *priv_data)
+{
+	struct auxiliary_device *adev;
+	int id, ret;
+
+	adev = kzalloc_obj(*adev);
+	if (!adev)
+		return -ENOMEM;
+
+	id = ida_alloc(&tc9563_pwrctrl_ida, GFP_KERNEL);
+	if (id < 0) {
+		kfree(adev);
+		return id;
+	}
+
+	adev->id = id;
+	adev->name = name;
+	adev->dev.parent = dev;
+	adev->dev.platform_data = priv_data;
+	adev->dev.release = tc9563_pwrctrl_adev_release;
+	adev->dev.of_node = of_node_get(of_node);
+	dev_set_of_node_reused(&adev->dev);
+
+	ret = auxiliary_device_init(adev);
+	if (ret) {
+		ida_free(&tc9563_pwrctrl_ida, id);
+		of_node_put(adev->dev.of_node);
+		kfree(adev);
+		return ret;
+	}
+
+	ret = auxiliary_device_add(adev);
+	if (ret) {
+		auxiliary_device_uninit(adev);
+		return ret;
+	}
+
+	return devm_add_action_or_reset(dev, tc9563_pwrctrl_adev_remove, adev);
+}
+
+static int tc9563_pwrctrl_add_gpio_adev(struct tc9563_pwrctrl *tc9563)
+{
+	struct device *dev = tc9563->pwrctrl.dev;
+
+	if (!of_property_read_bool(dev->of_node, "gpio-controller") ||
+	    !of_property_present(dev->of_node, "#gpio-cells"))
+		return 0;
+
+	return tc9563_pwrctrl_adev_add(dev, TC9563_GPIO_DEV_NAME, dev->of_node,
+				       tc9563->regmap);
+}
+
 static int tc9563_pwrctrl_power_off(struct pci_pwrctrl *pwrctrl)
 {
 	struct tc9563_pwrctrl *tc9563 = container_of(pwrctrl,
 					    struct tc9563_pwrctrl, pwrctrl);
 
-	gpiod_set_value(tc9563->reset_gpio, 1);
+	gpiod_set_value_cansleep(tc9563->reset_gpio, 1);
 
 	regulator_bulk_disable(ARRAY_SIZE(tc9563->supplies), tc9563->supplies);
 
@@ -496,12 +540,16 @@ static int tc9563_pwrctrl_power_on(struct pci_pwrctrl *pwrctrl)
 	struct tc9563_pwrctrl_cfg *cfg;
 	int ret, i;
 
+	ret = tc9563_pwrctrl_parse_reset_line(tc9563);
+	if (ret)
+		return ret;
+
 	ret = regulator_bulk_enable(ARRAY_SIZE(tc9563->supplies),
 				    tc9563->supplies);
 	if (ret < 0)
 		return dev_err_probe(dev, ret, "cannot enable regulators\n");
 
-	gpiod_set_value(tc9563->reset_gpio, 0);
+	gpiod_set_value_cansleep(tc9563->reset_gpio, 0);
 
 	fsleep(TC9563_OSC_STAB_DELAY_US);
 
@@ -572,6 +620,13 @@ static int tc9563_pwrctrl_power_on(struct pci_pwrctrl *pwrctrl)
 	return ret;
 }
 
+static const struct regmap_config tc9563_regmap_config = {
+	.reg_bits = 24,
+	.val_bits = 32,
+	.reg_format_endian = REGMAP_ENDIAN_BIG,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+};
+
 static int tc9563_pwrctrl_probe(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
@@ -579,6 +634,7 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev)
 	enum tc9563_pwrctrl_ports port;
 	struct tc9563_pwrctrl *tc9563;
 	struct device_node *i2c_node;
+	struct i2c_client *client;
 	int ret, addr;
 
 	tc9563 = devm_kzalloc(dev, sizeof(*tc9563), GFP_KERNEL);
@@ -595,11 +651,18 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev)
 	if (!tc9563->adapter)
 		return dev_err_probe(dev, -EPROBE_DEFER, "Failed to find I2C adapter\n");
 
-	tc9563->client = i2c_new_dummy_device(tc9563->adapter, addr);
-	if (IS_ERR(tc9563->client)) {
+	client = devm_i2c_new_dummy_device(dev, tc9563->adapter, addr);
+	if (IS_ERR(client)) {
 		dev_err(dev, "Failed to create I2C client\n");
-		i2c_put_adapter(tc9563->adapter);
-		return PTR_ERR(tc9563->client);
+		ret = PTR_ERR(client);
+		goto remove_i2c;
+	}
+
+	tc9563->regmap = devm_regmap_init_i2c(client, &tc9563_regmap_config);
+	if (IS_ERR(tc9563->regmap)) {
+		ret = dev_err_probe(dev, PTR_ERR(tc9563->regmap),
+				    "Failed to allocate register map\n");
+		goto remove_i2c;
 	}
 
 	for (int i = 0; i < ARRAY_SIZE(tc9563_supply_names); i++)
@@ -664,6 +727,10 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev)
 	tc9563->pwrctrl.power_on = tc9563_pwrctrl_power_on;
 	tc9563->pwrctrl.power_off = tc9563_pwrctrl_power_off;
 
+	ret = tc9563_pwrctrl_add_gpio_adev(tc9563);
+	if (ret)
+		goto remove_i2c;
+
 	ret = devm_pci_pwrctrl_device_set_ready(dev, &tc9563->pwrctrl);
 	if (ret)
 		goto power_off;
@@ -673,7 +740,6 @@ static int tc9563_pwrctrl_probe(struct platform_device *pdev)
 power_off:
 	tc9563_pwrctrl_power_off(&tc9563->pwrctrl);
 remove_i2c:
-	i2c_unregister_device(tc9563->client);
 	i2c_put_adapter(tc9563->adapter);
 	return ret;
 }
@@ -685,7 +751,6 @@ static void tc9563_pwrctrl_remove(struct platform_device *pdev)
 					struct tc9563_pwrctrl, pwrctrl);
 
 	tc9563_pwrctrl_power_off(&tc9563->pwrctrl);
-	i2c_unregister_device(tc9563->client);
 	i2c_put_adapter(tc9563->adapter);
 }
 
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index de9bbccda21fd1..5a26e24a485e02 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -22,6 +22,7 @@
 #include <linux/isa-dma.h> /* isa_dma_bridge_buggy */
 #include <linux/init.h>
 #include <linux/iommu.h>
+#include <linux/iopoll.h>
 #include <linux/delay.h>
 #include <linux/acpi.h>
 #include <linux/dmi.h>
@@ -108,7 +109,11 @@ int pcie_failed_link_retrain(struct pci_dev *dev)
 
 	pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &oldlnkctl2);
-	if (!(lnksta & PCI_EXP_LNKSTA_DLLLA) && pcie_lbms_seen(dev, lnksta)) {
+	if (lnksta & PCI_EXP_LNKSTA_DLLLA) {
+		;
+	} else if (PCIE_LNKCTL2_TLS2SPEED(oldlnkctl2) == PCIE_SPEED_2_5GT) {
+		return ret;
+	} else if (pcie_lbms_seen(dev, lnksta)) {
 		pci_info(dev, "broken device, retraining non-functional downstream link at 2.5GT/s\n");
 		ret = pcie_set_target_speed(dev, PCIE_SPEED_2_5GT, false);
 		if (ret)
@@ -4230,6 +4235,118 @@ static int reset_hinic_vf_dev(struct pci_dev *pdev, bool probe)
 	return 0;
 }
 
+#define QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET	0x3008
+#define QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET_V	BIT(0)
+#define QUALCOMM_WLAN_MHICTRL			0x38
+#define QUALCOMM_WLAN_MHICTRL_RESET_MASK	0x2
+
+/*
+ * Qualcomm WLAN device-specific reset using SoC global reset via BAR0
+ * registers.
+ */
+static int reset_qualcomm_wlan(struct pci_dev *pdev, bool probe)
+{
+	void __iomem *bar;
+	u32 val;
+	u16 cmd;
+	int ret;
+
+	if (probe)
+		return 0;
+
+	if (pdev->current_state != PCI_D0)
+		return -EINVAL;
+
+	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+	pci_write_config_word(pdev, PCI_COMMAND, cmd | PCI_COMMAND_MEMORY);
+
+	bar = pci_iomap(pdev, 0, 0);
+	if (!bar) {
+		pci_write_config_word(pdev, PCI_COMMAND, cmd);
+		return -ENODEV;
+	}
+
+	val = ioread32(bar + QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET);
+	if (PCI_POSSIBLE_ERROR(val)) {
+		ret = -ENODEV;
+		goto out_restore;
+	}
+	val |= QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET_V;
+	iowrite32(val, bar + QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET);
+	ioread32(bar + QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET);
+
+	msleep(10);
+
+	val &= ~QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET_V;
+	iowrite32(val, bar + QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET);
+	ioread32(bar + QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET);
+
+	msleep(10);
+
+	ret = read_poll_timeout(ioread32, val,
+				!PCI_POSSIBLE_ERROR(val),
+				20 * USEC_PER_MSEC,
+				5 * USEC_PER_SEC, false,
+				bar + QUALCOMM_WLAN_PCIE_SOC_GLOBAL_RESET);
+	if (ret) {
+		pci_err(pdev, "PCIe link failed to recover after reset\n");
+		goto out_restore;
+	}
+
+	/* After SOC_GLOBAL_RESET, MHISTATUS may still have SYSERR bit set
+	 * and thus need to set MHICTRL_RESET to clear SYSERR.
+	 */
+	iowrite32(QUALCOMM_WLAN_MHICTRL_RESET_MASK, bar + QUALCOMM_WLAN_MHICTRL);
+	ioread32(bar + QUALCOMM_WLAN_MHICTRL);
+
+	msleep(10);
+
+out_restore:
+	pci_iounmap(pdev, bar);
+	pci_write_config_word(pdev, PCI_COMMAND, cmd);
+
+	return ret;
+}
+
+#define MHI_SOC_RESET_REQ_OFFSET		0xb0
+#define MHI_SOC_RESET_REQ			BIT(0)
+
+/*
+ * Qualcomm modem device-specific reset using MHI SoC reset via BAR0
+ * register.
+ */
+static int reset_qualcomm_modem(struct pci_dev *pdev, bool probe)
+{
+	void __iomem *bar;
+	u16 cmd;
+
+	if (probe)
+		return 0;
+
+	if (pdev->current_state != PCI_D0)
+		return -EINVAL;
+
+	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
+	pci_write_config_word(pdev, PCI_COMMAND, cmd | PCI_COMMAND_MEMORY);
+
+	bar = pci_iomap(pdev, 0, 0);
+	if (!bar) {
+		pci_write_config_word(pdev, PCI_COMMAND, cmd);
+		return -ENODEV;
+	}
+
+	iowrite32(MHI_SOC_RESET_REQ, bar + MHI_SOC_RESET_REQ_OFFSET);
+	ioread32(bar + MHI_SOC_RESET_REQ_OFFSET);
+
+	/* Be sure device reset has been executed */
+	msleep(2000);
+
+	pci_iounmap(pdev, bar);
+	pci_write_config_word(pdev, PCI_COMMAND, cmd);
+
+	return 0;
+}
+
 static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
 		 reset_intel_82599_sfp_virtfn },
@@ -4245,6 +4362,9 @@ static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
 		reset_chelsio_generic_dev },
 	{ PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HINIC_VF,
 		reset_hinic_vf_dev },
+	{ PCI_VENDOR_ID_QCOM, 0x0308, reset_qualcomm_modem }, /* SDX62/SDX65 modems */
+	{ PCI_VENDOR_ID_QCOM, 0x1103, reset_qualcomm_wlan },  /* WCN6855 WLAN */
+	{ PCI_VENDOR_ID_QCOM, 0x1107, reset_qualcomm_wlan },  /* WCN7850 WLAN */
 	{ 0 }
 };
 
@@ -5377,9 +5497,6 @@ static void pci_quirk_enable_intel_rp_mpc_acs(struct pci_dev *dev)
  */
 static int pci_quirk_enable_intel_pch_acs(struct pci_dev *dev)
 {
-	if (!pci_quirk_intel_pch_acs_match(dev))
-		return -ENOTTY;
-
 	if (pci_quirk_enable_intel_lpc_acs(dev)) {
 		pci_warn(dev, "Failed to enable Intel PCH ACS quirk\n");
 		return 0;
@@ -5399,9 +5516,6 @@ static int pci_quirk_enable_intel_spt_pch_acs(struct pci_dev *dev)
 	int pos;
 	u32 cap, ctrl;
 
-	if (!pci_quirk_intel_spt_pch_acs_match(dev))
-		return -ENOTTY;
-
 	pos = dev->acs_cap;
 	if (!pos)
 		return -ENOTTY;
@@ -5429,9 +5543,6 @@ static int pci_quirk_disable_intel_spt_pch_acs_redir(struct pci_dev *dev)
 	int pos;
 	u32 cap, ctrl;
 
-	if (!pci_quirk_intel_spt_pch_acs_match(dev))
-		return -ENOTTY;
-
 	pos = dev->acs_cap;
 	if (!pos)
 		return -ENOTTY;
@@ -5451,56 +5562,63 @@ static int pci_quirk_disable_intel_spt_pch_acs_redir(struct pci_dev *dev)
 static const struct pci_dev_acs_ops {
 	u16 vendor;
 	u16 device;
+	bool (*match)(struct pci_dev *dev);
 	int (*enable_acs)(struct pci_dev *dev);
 	int (*disable_acs_redir)(struct pci_dev *dev);
 } pci_dev_acs_ops[] = {
 	{ PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
+	    .match = pci_quirk_intel_pch_acs_match,
 	    .enable_acs = pci_quirk_enable_intel_pch_acs,
 	},
 	{ PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
+	    .match = pci_quirk_intel_spt_pch_acs_match,
 	    .enable_acs = pci_quirk_enable_intel_spt_pch_acs,
 	    .disable_acs_redir = pci_quirk_disable_intel_spt_pch_acs_redir,
 	},
 };
 
-int pci_dev_specific_enable_acs(struct pci_dev *dev)
+static const struct pci_dev_acs_ops *pci_dev_acs_ops_get(struct pci_dev *dev)
 {
 	const struct pci_dev_acs_ops *p;
-	int i, ret;
+	int i;
 
 	for (i = 0; i < ARRAY_SIZE(pci_dev_acs_ops); i++) {
 		p = &pci_dev_acs_ops[i];
 		if ((p->vendor == dev->vendor ||
 		     p->vendor == (u16)PCI_ANY_ID) &&
 		    (p->device == dev->device ||
-		     p->device == (u16)PCI_ANY_ID) &&
-		    p->enable_acs) {
-			ret = p->enable_acs(dev);
-			if (ret >= 0)
-				return ret;
+		     p->device == (u16)PCI_ANY_ID)) {
+			if (!p->match || p->match(dev))
+				return p;
 		}
 	}
 
+	return NULL;
+}
+
+bool pci_need_dev_specific_enable_acs(struct pci_dev *dev)
+{
+	const struct pci_dev_acs_ops *p = pci_dev_acs_ops_get(dev);
+
+	return p && p->enable_acs;
+}
+
+int pci_dev_specific_enable_acs(struct pci_dev *dev)
+{
+	const struct pci_dev_acs_ops *p = pci_dev_acs_ops_get(dev);
+
+	if (p && p->enable_acs)
+		return p->enable_acs(dev);
+
 	return -ENOTTY;
 }
 
 int pci_dev_specific_disable_acs_redir(struct pci_dev *dev)
 {
-	const struct pci_dev_acs_ops *p;
-	int i, ret;
+	const struct pci_dev_acs_ops *p = pci_dev_acs_ops_get(dev);
 
-	for (i = 0; i < ARRAY_SIZE(pci_dev_acs_ops); i++) {
-		p = &pci_dev_acs_ops[i];
-		if ((p->vendor == dev->vendor ||
-		     p->vendor == (u16)PCI_ANY_ID) &&
-		    (p->device == dev->device ||
-		     p->device == (u16)PCI_ANY_ID) &&
-		    p->disable_acs_redir) {
-			ret = p->disable_acs_redir(dev);
-			if (ret >= 0)
-				return ret;
-		}
-	}
+	if (p && p->disable_acs_redir)
+		return p->disable_acs_redir(dev);
 
 	return -ENOTTY;
 }
@@ -6263,6 +6381,10 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2303,
 			 pci_fixup_pericom_acs_store_forward);
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2303,
 			 pci_fixup_pericom_acs_store_forward);
+DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb304,
+			 pci_fixup_pericom_acs_store_forward);
+DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb304,
+			 pci_fixup_pericom_acs_store_forward);
 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb404,
 			 pci_fixup_pericom_acs_store_forward);
 DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb404,
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index e8c94aa1d3c129..ed16ef7c26fa71 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -2380,6 +2380,7 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size
 	struct resource *res = pci_resource_n(pdev, resno);
 	struct pci_dev_resource *dev_res;
 	struct pci_bus *bus = pdev->bus;
+	struct pci_dev *bridge = pci_upstream_bridge(pdev);
 	struct resource *b_win, *r;
 	LIST_HEAD(saved);
 	unsigned int i;
@@ -2397,6 +2398,8 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size
 	if (ret)
 		return ret;
 
+	down_read(&pci_bus_sem);
+
 	pci_dev_for_each_resource(pdev, r, i) {
 		if (i >= PCI_BRIDGE_RESOURCES)
 			break;
@@ -2415,13 +2418,21 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size
 
 	pci_resize_resource_set_size(pdev, resno, size);
 
-	if (!bus->self)
-		goto out;
+	if (bridge) {
+		ret = pbus_reassign_bridge_resources(bus, res, &saved);
+		if (ret)
+			goto restore;
+	} else {
+		/* No bridge window to adjust; let the core reassign the bus. */
+		pci_bus_assign_resources(bus);
 
-	down_read(&pci_bus_sem);
-	ret = pbus_reassign_bridge_resources(bus, res, &saved);
-	if (ret)
-		goto restore;
+		list_for_each_entry(dev_res, &saved, list) {
+			if (!resource_assigned(dev_res->res)) {
+				ret = -ENOSPC;
+				goto restore;
+			}
+		}
+	}
 
 out:
 	up_read(&pci_bus_sem);
diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c
index 655ffd60e62f6a..4d064b86befcc5 100644
--- a/drivers/pci/tph.c
+++ b/drivers/pci/tph.c
@@ -119,12 +119,14 @@ static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid,
 	if (!out_obj)
 		return AE_ERROR;
 
-	if (out_obj->type != ACPI_TYPE_BUFFER) {
+	if (out_obj->type != ACPI_TYPE_BUFFER ||
+	    out_obj->buffer.length < sizeof(st_out->value) ||
+	    !out_obj->buffer.pointer) {
 		ACPI_FREE(out_obj);
 		return AE_ERROR;
 	}
 
-	st_out->value = *((u64 *)(out_obj->buffer.pointer));
+	memcpy(&st_out->value, out_obj->buffer.pointer, sizeof(st_out->value));
 
 	ACPI_FREE(out_obj);
 
diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c
index fc934bccf4a640..5b6d3a4736da39 100644
--- a/drivers/perf/alibaba_uncore_drw_pmu.c
+++ b/drivers/perf/alibaba_uncore_drw_pmu.c
@@ -135,7 +135,7 @@ static ssize_t ali_drw_pmu_event_show(struct device *dev,
 
 #define ALI_DRW_PMU_ATTR(_name, _func, _config)                            \
 		(&((struct dev_ext_attribute[]) {                               \
-				{ __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
+				{ __DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_config }   \
 		})[0].attr.attr)
 
 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config)            \
diff --git a/drivers/perf/arm-cci.c b/drivers/perf/arm-cci.c
index f0ef0a679e746b..71035227f58bfc 100644
--- a/drivers/perf/arm-cci.c
+++ b/drivers/perf/arm-cci.c
@@ -132,7 +132,7 @@ static ssize_t __maybe_unused cci_pmu_event_show(struct device *dev,
 
 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config) 				\
 	&((struct dev_ext_attribute[]) {					\
-		{ __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config }	\
+		{ __DEVICE_ATTR(_name, S_IRUGO, _func, NULL), (void *)_config }	\
 	})[0].attr.attr
 
 #define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \
diff --git a/drivers/perf/arm-ccn.c b/drivers/perf/arm-ccn.c
index c40da3450c8fd4..e3411bdf5c7c52 100644
--- a/drivers/perf/arm-ccn.c
+++ b/drivers/perf/arm-ccn.c
@@ -216,7 +216,7 @@ static void arm_ccn_pmu_config_set(u64 *config, u32 node_xp, u32 type, u32 port)
 
 #define CCN_FORMAT_ATTR(_name, _config) \
 	struct dev_ext_attribute arm_ccn_pmu_format_attr_##_name = \
-			{ __ATTR(_name, S_IRUGO, device_show_string, \
+			{ __DEVICE_ATTR(_name, S_IRUGO, device_show_string, \
 			NULL), _config }
 
 static CCN_FORMAT_ATTR(node, "config:0-7");
diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c
index 33ee2be9b38642..5378fba916cf53 100644
--- a/drivers/perf/arm-cmn.c
+++ b/drivers/perf/arm-cmn.c
@@ -305,9 +305,9 @@ enum cmn_filter_select {
 	SEL_NONE,
 	SEL_OCCUP1_ID,
 	SEL_CLASS_OCCUP_ID,
-	SEL_CBUSY_SNTHROTTLE_SEL,
 	SEL_HBT_LBT_SEL,
 	SEL_SN_HOME_SEL,
+	SEL_CBUSY_SNTHROTTLE_SEL,
 	SEL_SNP_VC_SEL,
 	SEL_ENHANCED_HBT_LBT_SEL,
 	SEL_EVICT_STATE_SEL,
@@ -978,10 +978,14 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj,
 	_CMN_EVENT_HNS(_model, _name##_all, _event, _sel, 0),		\
 	_CMN_EVENT_HNS(_model, _name##_hbt, _event, _sel, 1),		\
 	_CMN_EVENT_HNS(_model, _name##_lbt, _event, _sel, 2)
-#define _CMN_EVENT_HNS_HBT2(_model, _name, _event, _fsel1, f1)		\
-	_CMN_EVENT_HNS(_model, _name##_all, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 0), \
-	_CMN_EVENT_HNS(_model, _name##_hbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 1), \
-	_CMN_EVENT_HNS(_model, _name##_lbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 2)
+#define _CMN_EVENT_HNS_EVICT(_model, _name, _event, _fsel1, f1)			\
+	_CMN_EVENT_HNS(_model, _name##_all, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 0), \
+	_CMN_EVENT_HNS(_model, _name##_eu, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 1), \
+	_CMN_EVENT_HNS(_model, _name##_en, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 2), \
+	_CMN_EVENT_HNS(_model, _name##_su, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 3), \
+	_CMN_EVENT_HNS(_model, _name##_sn, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 4), \
+	_CMN_EVENT_HNS(_model, _name##_mu, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 5), \
+	_CMN_EVENT_HNS(_model, _name##_mn, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 6)
 
 #define CMN_EVENT_HNS_OCC(_model, _name, _event)			\
 	CMN_EVENT_HN_OCC(_model, hns_##_name, CMN_TYPE_HNS, _event),	\
@@ -1014,13 +1018,9 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj,
 	_CMN_EVENT_HNS(CMNS3R2, _name##_ccglcn, _event, SEL_ENHANCED_HBT_LBT_SEL, 5), \
 	_CMN_EVENT_HNS(CMNS3R2, _name##_ccgrn, _event, SEL_ENHANCED_HBT_LBT_SEL, 6)
 #define CMN_EVENT_HNS_EVICT(_model, _name, _event)			\
-	_CMN_EVENT_HNS_HBT2(_model, _name##_all, _event, SEL_EVICT_STATE_SEL, 0), \
-	_CMN_EVENT_HNS_HBT2(_model, _name##_eu, _event, SEL_EVICT_STATE_SEL, 1), \
-	_CMN_EVENT_HNS_HBT2(_model, _name##_en, _event, SEL_EVICT_STATE_SEL, 2), \
-	_CMN_EVENT_HNS_HBT2(_model, _name##_su, _event, SEL_EVICT_STATE_SEL, 3), \
-	_CMN_EVENT_HNS_HBT2(_model, _name##_sn, _event, SEL_EVICT_STATE_SEL, 4), \
-	_CMN_EVENT_HNS_HBT2(_model, _name##_mu, _event, SEL_EVICT_STATE_SEL, 5), \
-	_CMN_EVENT_HNS_HBT2(_model, _name##_mn, _event, SEL_EVICT_STATE_SEL, 6)
+	_CMN_EVENT_HNS_EVICT(_model, _name##_all, _event, SEL_HBT_LBT_SEL, 0), \
+	_CMN_EVENT_HNS_EVICT(_model, _name##_hbt, _event, SEL_HBT_LBT_SEL, 1), \
+	_CMN_EVENT_HNS_EVICT(_model, _name##_lbt, _event, SEL_HBT_LBT_SEL, 2)
 
 #define CMN_EVENT_HNSR0_HBT(_name, _event)				\
 	_CMN_EVENT_HNS_HBT(CMN700 | CMNS3R01, _name, _event, SEL_HBT_LBT_SEL)
@@ -1860,10 +1860,12 @@ static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val,
 
 		val->dtm_count[dtm]++;
 
+		if (sel == SEL_EVICT_STATE_SEL) {
+			val->filter[dtm][sel] = CMN_EVENT_FILTER2(event) + 1;
+			sel = SEL_HBT_LBT_SEL;
+		}
 		if (sel)
 			val->filter[dtm][sel] = CMN_EVENT_FILTER(event) + 1;
-		if (sel == SEL_EVICT_STATE_SEL)
-			val->filter[dtm][SEL_HBT_LBT_SEL] = CMN_EVENT_FILTER2(event) + 1;
 
 		if (type != CMN_TYPE_WP)
 			continue;
@@ -1914,14 +1916,17 @@ static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event)
 		if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS)
 			goto done;
 
+		if (sel == SEL_EVICT_STATE_SEL) {
+			if (val->filter[dtm][sel] &&
+			    val->filter[dtm][sel] != CMN_EVENT_FILTER2(event) + 1)
+				goto done;
+			sel = SEL_HBT_LBT_SEL;
+		}
+
 		if (sel && val->filter[dtm][sel] &&
 		    val->filter[dtm][sel] != CMN_EVENT_FILTER(event) + 1)
 			goto done;
 
-		if (sel == SEL_EVICT_STATE_SEL && val->filter[dtm][SEL_HBT_LBT_SEL] &&
-		    val->filter[dtm][SEL_HBT_LBT_SEL] != CMN_EVENT_FILTER2(event) + 1)
-			goto done;
-
 		if (type != CMN_TYPE_WP)
 			continue;
 
@@ -1950,7 +1955,7 @@ static enum cmn_filter_select arm_cmn_event_filter(const struct arm_cmn *cmn,
 	for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) {
 		e = container_of(arm_cmn_event_attrs[i], typeof(*e), attr.attr);
 		if (e->model & model && e->type == type && e->eventid == eventid)
-			return e->filter[0].sel;
+			return e->filter[1].sel ?: e->filter[0].sel;
 	}
 	return SEL_NONE;
 }
@@ -2081,16 +2086,21 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event,
 static int arm_cmn_set_event_filter(struct arm_cmn_node *dn, struct perf_event *event)
 {
 	enum cmn_filter_select fsel = to_cmn_hw(event)->filter_sel;
+	bool evict_state = fsel == SEL_EVICT_STATE_SEL;
 	int ret = 0;
 
+	if (evict_state) {
+		ret = arm_cmn_set_event_sel_hi(dn, fsel, CMN_EVENT_FILTER2(event));
+		if (ret)
+			return ret;
+		fsel = SEL_HBT_LBT_SEL;
+	}
 	if (fsel)
 		ret = arm_cmn_set_event_sel_hi(dn, fsel, CMN_EVENT_FILTER(event));
 
-	if (fsel == SEL_EVICT_STATE_SEL && !ret) {
-		ret = arm_cmn_set_event_sel_hi(dn, SEL_HBT_LBT_SEL, CMN_EVENT_FILTER2(event));
-		if (ret)
-			dn->filter[fsel].count--;
-	}
+	if (ret && evict_state)
+		dn->filter[SEL_EVICT_STATE_SEL].count--;
+
 	return ret;
 }
 
diff --git a/drivers/perf/arm_brbe.c b/drivers/perf/arm_brbe.c
index ba554e0c846c4f..254be4da8ae298 100644
--- a/drivers/perf/arm_brbe.c
+++ b/drivers/perf/arm_brbe.c
@@ -604,7 +604,7 @@ static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *ent
 		return false;
 
 	brbinf = bregs.brbinf;
-	perf_clear_branch_entry_bitfields(entry);
+	*entry = (struct perf_branch_entry){ };
 	if (brbe_record_is_complete(brbinf)) {
 		entry->from = bregs.brbsrc;
 		entry->to = bregs.brbtgt;
diff --git a/drivers/perf/arm_cspmu/arm_cspmu.h b/drivers/perf/arm_cspmu/arm_cspmu.h
index c4058d602477ca..93d5b062de8b04 100644
--- a/drivers/perf/arm_cspmu/arm_cspmu.h
+++ b/drivers/perf/arm_cspmu/arm_cspmu.h
@@ -23,7 +23,7 @@
 #define ARM_CSPMU_EXT_ATTR(_name, _func, _config)			\
 	(&((struct dev_ext_attribute[]){				\
 		{							\
-			.attr = __ATTR(_name, 0444, _func, NULL),	\
+			.attr = __DEVICE_ATTR(_name, 0444, _func, NULL),\
 			.var = (void *)_config				\
 		}							\
 	})[0].attr.attr)
diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c
index bcbd19e075a579..8332eaca06dcc6 100644
--- a/drivers/perf/arm_dsu_pmu.c
+++ b/drivers/perf/arm_dsu_pmu.c
@@ -69,7 +69,7 @@
 #define DSU_EXT_ATTR(_name, _func, _config)		\
 	(&((struct dev_ext_attribute[]) {				\
 		{							\
-			.attr = __ATTR(_name, 0444, _func, NULL),	\
+			.attr = __DEVICE_ATTR(_name, 0444, _func, NULL),\
 			.var = (void *)_config				\
 		}							\
 	})[0].attr.attr)
diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c
index b70f3fedf0282b..e298d6d6371ea3 100644
--- a/drivers/perf/arm_spe_pmu.c
+++ b/drivers/perf/arm_spe_pmu.c
@@ -172,9 +172,9 @@ static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev,
 	return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap));
 }
 
-#define SPE_EXT_ATTR_ENTRY(_name, _func, _var)				\
-	&((struct dev_ext_attribute[]) {				\
-		{ __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var }	\
+#define SPE_EXT_ATTR_ENTRY(_name, _func, _var)					\
+	&((struct dev_ext_attribute[]) {					\
+		{ __DEVICE_ATTR(_name, S_IRUGO, _func, NULL), (void *)_var }	\
 	})[0].attr.attr
 
 #define SPE_CAP_EXT_ATTR_ENTRY(_name, _var)				\
diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c
index 2c8a03cc11e4d4..a36cf792a2d098 100644
--- a/drivers/perf/cxl_pmu.c
+++ b/drivers/perf/cxl_pmu.c
@@ -216,11 +216,11 @@ static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info)
 }
 
 #define CXL_PMU_FORMAT_ATTR(_name, _format)\
-	(&((struct dev_ext_attribute[]) {					\
-		{								\
-			.attr = __ATTR(_name, 0444, device_show_string, NULL),	\
-			.var = (void *)_format					\
-		}								\
+	(&((struct dev_ext_attribute[]) {						\
+		{									\
+			.attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL),	\
+			.var = (void *)_format						\
+		}									\
 		})[0].attr.attr)
 
 enum {
diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c
index dd8046265584ce..7553c587260063 100644
--- a/drivers/perf/fsl_imx8_ddr_perf.c
+++ b/drivers/perf/fsl_imx8_ddr_perf.c
@@ -214,7 +214,7 @@ static ssize_t ddr_perf_filter_cap_show(struct device *dev,
 
 #define PERF_EXT_ATTR_ENTRY(_name, _func, _var)				\
 	(&((struct dev_ext_attribute) {					\
-		__ATTR(_name, 0444, _func, NULL), (void *)_var		\
+		__DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_var	\
 	}).attr.attr)
 
 #define PERF_FILTER_EXT_ATTR_ENTRY(_name, _var)				\
diff --git a/drivers/perf/fujitsu_uncore_pmu.c b/drivers/perf/fujitsu_uncore_pmu.c
index fa64146fcac6b6..db90e181b9a871 100644
--- a/drivers/perf/fujitsu_uncore_pmu.c
+++ b/drivers/perf/fujitsu_uncore_pmu.c
@@ -291,7 +291,7 @@ static void fujitsu_uncore_event_read(struct perf_event *event)
 
 #define UNCORE_PMU_FORMAT_ATTR(_name, _config)				      \
 	(&((struct dev_ext_attribute[]) {				      \
-		{ .attr = __ATTR(_name, 0444, device_show_string, NULL),      \
+		{ .attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL), \
 		  .var = (void *)_config, }				      \
 	})[0].attr.attr)
 
diff --git a/drivers/perf/hisilicon/hisi_pcie_pmu.c b/drivers/perf/hisilicon/hisi_pcie_pmu.c
index 0f55d871c67e73..0a69d13fa4539b 100644
--- a/drivers/perf/hisilicon/hisi_pcie_pmu.c
+++ b/drivers/perf/hisilicon/hisi_pcie_pmu.c
@@ -110,7 +110,7 @@ static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attr
 
 #define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format)                              \
 	(&((struct dev_ext_attribute[]){                                       \
-		{ .attr = __ATTR(_name, 0444, device_show_string, NULL),       \
+		{ .attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL),\
 		  .var = (void *)_format }                                     \
 	})[0].attr.attr)
 
diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.h b/drivers/perf/hisilicon/hisi_uncore_pmu.h
index 3ffe6acda65391..a4cb9ed7ea204a 100644
--- a/drivers/perf/hisilicon/hisi_uncore_pmu.h
+++ b/drivers/perf/hisilicon/hisi_uncore_pmu.h
@@ -27,9 +27,9 @@
 #define HISI_MAX_COUNTERS	0x18
 #define to_hisi_pmu(p)	(container_of(p, struct hisi_pmu, pmu))
 
-#define HISI_PMU_ATTR(_name, _func, _config)				\
-	(&((struct dev_ext_attribute[]) {				\
-		{ __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
+#define HISI_PMU_ATTR(_name, _func, _config)				     \
+	(&((struct dev_ext_attribute[]) {				     \
+		{ __DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_config } \
 	})[0].attr.attr)
 
 #define HISI_PMU_FORMAT_ATTR(_name, _config)		\
diff --git a/drivers/perf/hisilicon/hns3_pmu.c b/drivers/perf/hisilicon/hns3_pmu.c
index c157f3572cae57..88a1129e9b90e6 100644
--- a/drivers/perf/hisilicon/hns3_pmu.c
+++ b/drivers/perf/hisilicon/hns3_pmu.c
@@ -405,9 +405,9 @@ static ssize_t hns3_pmu_filter_mode_show(struct device *dev,
 	return len;
 }
 
-#define HNS3_PMU_ATTR(_name, _func, _config)				\
-	(&((struct dev_ext_attribute[]) {				\
-		{ __ATTR(_name, 0444, _func, NULL), (void *)_config }	\
+#define HNS3_PMU_ATTR(_name, _func, _config)				     \
+	(&((struct dev_ext_attribute[]) {				     \
+		{ __DEVICE_ATTR(_name, 0444, _func, NULL), (void *)_config } \
 	})[0].attr.attr)
 
 #define HNS3_PMU_FORMAT_ATTR(_name, _format) \
diff --git a/drivers/perf/nvidia_t410_c2c_pmu.c b/drivers/perf/nvidia_t410_c2c_pmu.c
index bff875f4f625df..a41d5712da91b7 100644
--- a/drivers/perf/nvidia_t410_c2c_pmu.c
+++ b/drivers/perf/nvidia_t410_c2c_pmu.c
@@ -545,7 +545,7 @@ static struct attribute_group nv_c2c_pmu_peer_attr_group = {
 #define NV_C2C_PMU_EXT_ATTR(_name, _func, _config)			\
 	(&((struct dev_ext_attribute[]){				\
 		{							\
-			.attr = __ATTR(_name, 0444, _func, NULL),	\
+			.attr = __DEVICE_ATTR(_name, 0444, _func, NULL),\
 			.var = (void *)_config				\
 		}							\
 	})[0].attr.attr)
diff --git a/drivers/perf/nvidia_t410_cmem_latency_pmu.c b/drivers/perf/nvidia_t410_cmem_latency_pmu.c
index 6c8e41598ec1d8..da91a51fe29e79 100644
--- a/drivers/perf/nvidia_t410_cmem_latency_pmu.c
+++ b/drivers/perf/nvidia_t410_cmem_latency_pmu.c
@@ -435,12 +435,12 @@ static struct attribute_group cmem_lat_pmu_identifier_attr_group = {
 
 /* Format attributes. */
 
-#define NV_PMU_EXT_ATTR(_name, _func, _config)			\
-	(&((struct dev_ext_attribute[]){				\
-		{							\
-			.attr = __ATTR(_name, 0444, _func, NULL),	\
-			.var = (void *)_config				\
-		}							\
+#define NV_PMU_EXT_ATTR(_name, _func, _config)					\
+	(&((struct dev_ext_attribute[]){					\
+		{								\
+			.attr = __DEVICE_ATTR(_name, 0444, _func, NULL),	\
+			.var = (void *)_config					\
+		}								\
 	})[0].attr.attr)
 
 static struct attribute *cmem_lat_pmu_formats[] = {
diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c
index e18d0f94e8d864..d036fa05aeaba7 100644
--- a/drivers/perf/qcom_l3_pmu.c
+++ b/drivers/perf/qcom_l3_pmu.c
@@ -609,10 +609,10 @@ static void qcom_l3_cache__event_read(struct perf_event *event)
 
 /* formats */
 
-#define L3CACHE_PMU_FORMAT_ATTR(_name, _config)				      \
-	(&((struct dev_ext_attribute[]) {				      \
-		{ .attr = __ATTR(_name, 0444, device_show_string, NULL),      \
-		  .var = (void *) _config, }				      \
+#define L3CACHE_PMU_FORMAT_ATTR(_name, _config)				        \
+	(&((struct dev_ext_attribute[]) {				        \
+		{ .attr = __DEVICE_ATTR(_name, 0444, device_show_string, NULL), \
+		  .var = (void *) _config, }				        \
 	})[0].attr.attr)
 
 static struct attribute *qcom_l3_cache_pmu_formats[] = {
diff --git a/drivers/perf/starfive_starlink_pmu.c b/drivers/perf/starfive_starlink_pmu.c
index 98e5ccee2fdc06..d9843645554bad 100644
--- a/drivers/perf/starfive_starlink_pmu.c
+++ b/drivers/perf/starfive_starlink_pmu.c
@@ -51,7 +51,7 @@
 
 #define STARLINK_FORMAT_ATTR(_name, _config)				      \
 	(&((struct dev_ext_attribute[]) {				      \
-		{ .attr = __ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \
+		{ .attr = __DEVICE_ATTR(_name, 0444, starlink_pmu_sysfs_format_show, NULL), \
 		  .var = (void *)_config, }				      \
 	})[0].attr.attr)
 
diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c
index ab3623fab914c4..104cd0277b58e0 100644
--- a/drivers/perf/xgene_pmu.c
+++ b/drivers/perf/xgene_pmu.c
@@ -164,7 +164,7 @@ enum xgene_pmu_dev_type {
  */
 #define XGENE_PMU_FORMAT_ATTR(_name, _config)		\
 	(&((struct dev_ext_attribute[]) {		\
-		{ .attr = __ATTR(_name, S_IRUGO, device_show_string, NULL), \
+		{ .attr = __DEVICE_ATTR(_name, S_IRUGO, device_show_string, NULL), \
 		  .var = (void *) _config, }		\
 	})[0].attr.attr)
 
diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c
index 4156fabad742cc..f09305dd76d131 100644
--- a/drivers/phy/apple/atc.c
+++ b/drivers/phy/apple/atc.c
@@ -585,8 +585,6 @@ struct atcphy_mode_configuration {
  * @phys.dp: DisplayPort PHY instance
  * @phy_provider: PHY provider instance
  * @rcdev: Reset controller device
- * @sw: Type-C switch instance
- * @mux: Type-C mux instance
  * @lock: Mutex for synchronizing register access across PHY, Type-C switch/mux and reset controller
  */
 struct apple_atcphy {
diff --git a/drivers/phy/broadcom/phy-brcm-usb.c b/drivers/phy/broadcom/phy-brcm-usb.c
index 59d756a10d6cba..e606db489698a4 100644
--- a/drivers/phy/broadcom/phy-brcm-usb.c
+++ b/drivers/phy/broadcom/phy-brcm-usb.c
@@ -71,6 +71,8 @@ struct brcm_usb_phy_data {
 	int			wake_irq;
 	struct brcm_usb_phy	phys[BRCM_USB_PHY_ID_MAX];
 	struct notifier_block	pm_notifier;
+	bool			pm_notifier_registered;
+	bool			sysfs_group_created;
 	bool			pm_active;
 };
 
@@ -545,7 +547,11 @@ static int brcm_usb_phy_probe(struct platform_device *pdev)
 		return err;
 
 	priv->pm_notifier.notifier_call = brcm_pm_notifier;
-	register_pm_notifier(&priv->pm_notifier);
+	err = register_pm_notifier(&priv->pm_notifier);
+	if (err)
+		dev_warn(dev, "Error registering PM notifier\n");
+	else
+		priv->pm_notifier_registered = true;
 
 	mutex_init(&priv->mutex);
 
@@ -561,6 +567,8 @@ static int brcm_usb_phy_probe(struct platform_device *pdev)
 	err = sysfs_create_group(&dev->kobj, &brcm_usb_phy_group);
 	if (err)
 		dev_warn(dev, "Error creating sysfs attributes\n");
+	else
+		priv->sysfs_group_created = true;
 
 	/* Get piarbctl syscon if it exists */
 	rmap = syscon_regmap_lookup_by_phandle(dev->of_node,
@@ -581,16 +589,29 @@ static int brcm_usb_phy_probe(struct platform_device *pdev)
 	clk_disable_unprepare(priv->usb_30_clk);
 
 	phy_provider = devm_of_phy_provider_register(dev, brcm_usb_phy_xlate);
+	err = PTR_ERR_OR_ZERO(phy_provider);
+	if (err)
+		goto err_remove_debug;
+
+	return 0;
+
+err_remove_debug:
+	if (priv->sysfs_group_created)
+		sysfs_remove_group(&dev->kobj, &brcm_usb_phy_group);
+	if (priv->pm_notifier_registered)
+		unregister_pm_notifier(&priv->pm_notifier);
 
-	return PTR_ERR_OR_ZERO(phy_provider);
+	return err;
 }
 
 static void brcm_usb_phy_remove(struct platform_device *pdev)
 {
 	struct brcm_usb_phy_data *priv = dev_get_drvdata(&pdev->dev);
 
-	sysfs_remove_group(&pdev->dev.kobj, &brcm_usb_phy_group);
-	unregister_pm_notifier(&priv->pm_notifier);
+	if (priv->sysfs_group_created)
+		sysfs_remove_group(&pdev->dev.kobj, &brcm_usb_phy_group);
+	if (priv->pm_notifier_registered)
+		unregister_pm_notifier(&priv->pm_notifier);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/phy/cadence/phy-cadence-sierra.c b/drivers/phy/cadence/phy-cadence-sierra.c
index 92ab1a31646aee..dbeda7f01cbb27 100644
--- a/drivers/phy/cadence/phy-cadence-sierra.c
+++ b/drivers/phy/cadence/phy-cadence-sierra.c
@@ -698,23 +698,16 @@ static const struct phy_ops noop_ops = {
 static u8 cdns_sierra_pll_mux_get_parent(struct clk_hw *hw)
 {
 	struct cdns_sierra_pll_mux *mux = to_cdns_sierra_pll_mux(hw);
-	struct regmap_field *plllc1en_field = mux->plllc1en_field;
-	struct regmap_field *termen_field = mux->termen_field;
 	struct regmap_field *field = mux->pfdclk_sel_preg;
 	unsigned int val;
 	int index;
 
 	regmap_field_read(field, &val);
 
-	if (strstr(clk_hw_get_name(hw), clk_names[CDNS_SIERRA_PLL_CMNLC1])) {
+	if (strstr(clk_hw_get_name(hw), clk_names[CDNS_SIERRA_PLL_CMNLC1]))
 		index = clk_mux_val_to_index(hw, cdns_sierra_pll_mux_table[CMN_PLLLC1], 0, val);
-		if (index == 1) {
-			regmap_field_write(plllc1en_field, 1);
-			regmap_field_write(termen_field, 1);
-		}
-	} else {
+	else
 		index = clk_mux_val_to_index(hw, cdns_sierra_pll_mux_table[CMN_PLLLC], 0, val);
-	}
 
 	return index;
 }
diff --git a/drivers/phy/hisilicon/phy-hi3670-pcie.c b/drivers/phy/hisilicon/phy-hi3670-pcie.c
index 7396c601d8747b..0b2dc590aa6f7a 100644
--- a/drivers/phy/hisilicon/phy-hi3670-pcie.c
+++ b/drivers/phy/hisilicon/phy-hi3670-pcie.c
@@ -569,6 +569,7 @@ static int hi3670_pcie_get_resources_from_pcie(struct hi3670_pcie_phy *phy)
 	}
 
 	pcie_dev = bus_find_device_by_of_node(&platform_bus_type, pcie_port);
+	of_node_put(pcie_port);
 	if (!pcie_dev) {
 		dev_err(dev, "Didn't find pcie device\n");
 		return -ENODEV;
@@ -583,6 +584,7 @@ static int hi3670_pcie_get_resources_from_pcie(struct hi3670_pcie_phy *phy)
 	 * right regmap. So, let's use the named version.
 	 */
 	phy->apb = dev_get_regmap(pcie_dev, "kirin_pcie_apb");
+	put_device(pcie_dev);
 	if (!phy->apb) {
 		dev_err(dev, "Failed to get APB regmap\n");
 		return -ENODEV;
diff --git a/drivers/phy/mediatek/phy-mtk-dp.c b/drivers/phy/mediatek/phy-mtk-dp.c
index d7024a14433584..dbef4a4bd60a96 100644
--- a/drivers/phy/mediatek/phy-mtk-dp.c
+++ b/drivers/phy/mediatek/phy-mtk-dp.c
@@ -4,104 +4,350 @@
  *
  * Copyright (c) 2022, BayLibre Inc.
  * Copyright (c) 2022, MediaTek Inc.
+ *
+ * Major refactoring and new SoCs support
+ * Copyright (c) 2026, Collabora Ltd.
+ *                     AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
  */
 
+#include <linux/bitfield.h>
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/mfd/syscon.h>
+#include <linux/nvmem-consumer.h>
 #include <linux/of.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
-#define PHY_OFFSET			0x1000
+#define MTK_DP_PHY_MAX_LANES		4
+
+/* DP_PHYA_GLB_BIAS_GEN_0 (PHYA - Analog) */
+#define XTP_GLB_BIAS_INT_R_CTRL		GENMASK(20, 16)
+
+/* DP_PHYA_GLB_FORCE_CTRL_1 */
+#define CKM_CKTX0_EN_FORCE_MODE		BIT(10)
+
+/* DP_PHYA_GLB_DPAUX_TX */
+#define CKM_PT0_CKTX_IMPSEL		GENMASK(23, 20)
 
-#define MTK_DP_PHY_DIG_PLL_CTL_1	(PHY_OFFSET + 0x14)
+/* DP_PHYA_LAN_LANE_TX_0 */
+#define XTP_LN_TX_IMPSEL_PMOS		GENMASK(15, 12)
+#define XTP_LN_TX_IMPSEL_NMOS		GENMASK(19, 16)
+
+/* DP_PHYA_GLB_FORCE_CTRL_1 */
+#define CKM_CKTX0_EN_FORCE_MODE		BIT(10)
+
+/* DP_PHYD_PLL_CTL_1 */
 #define TPLL_SSC_EN			BIT(3)
 
-#define MTK_DP_PHY_DIG_BIT_RATE		(PHY_OFFSET + 0x3C)
-#define BIT_RATE_RBR			0
-#define BIT_RATE_HBR			1
-#define BIT_RATE_HBR2			2
-#define BIT_RATE_HBR3			3
-
-#define MTK_DP_PHY_DIG_SW_RST		(PHY_OFFSET + 0x38)
-#define DP_GLB_SW_RST_PHYD		BIT(0)
-
-#define MTK_DP_LANE0_DRIVING_PARAM_3		(PHY_OFFSET + 0x138)
-#define MTK_DP_LANE1_DRIVING_PARAM_3		(PHY_OFFSET + 0x238)
-#define MTK_DP_LANE2_DRIVING_PARAM_3		(PHY_OFFSET + 0x338)
-#define MTK_DP_LANE3_DRIVING_PARAM_3		(PHY_OFFSET + 0x438)
-#define XTP_LN_TX_LCTXC0_SW0_PRE0_DEFAULT	BIT(4)
-#define XTP_LN_TX_LCTXC0_SW0_PRE1_DEFAULT	(BIT(10) | BIT(12))
-#define XTP_LN_TX_LCTXC0_SW0_PRE2_DEFAULT	GENMASK(20, 19)
-#define XTP_LN_TX_LCTXC0_SW0_PRE3_DEFAULT	GENMASK(29, 29)
-#define DRIVING_PARAM_3_DEFAULT	(XTP_LN_TX_LCTXC0_SW0_PRE0_DEFAULT | \
-				 XTP_LN_TX_LCTXC0_SW0_PRE1_DEFAULT | \
-				 XTP_LN_TX_LCTXC0_SW0_PRE2_DEFAULT | \
-				 XTP_LN_TX_LCTXC0_SW0_PRE3_DEFAULT)
-
-#define XTP_LN_TX_LCTXC0_SW1_PRE0_DEFAULT	GENMASK(4, 3)
-#define XTP_LN_TX_LCTXC0_SW1_PRE1_DEFAULT	GENMASK(12, 9)
-#define XTP_LN_TX_LCTXC0_SW1_PRE2_DEFAULT	(BIT(18) | BIT(21))
-#define XTP_LN_TX_LCTXC0_SW2_PRE0_DEFAULT	GENMASK(29, 29)
-#define DRIVING_PARAM_4_DEFAULT	(XTP_LN_TX_LCTXC0_SW1_PRE0_DEFAULT | \
-				 XTP_LN_TX_LCTXC0_SW1_PRE1_DEFAULT | \
-				 XTP_LN_TX_LCTXC0_SW1_PRE2_DEFAULT | \
-				 XTP_LN_TX_LCTXC0_SW2_PRE0_DEFAULT)
-
-#define XTP_LN_TX_LCTXC0_SW2_PRE1_DEFAULT	(BIT(3) | BIT(5))
-#define XTP_LN_TX_LCTXC0_SW3_PRE0_DEFAULT	GENMASK(13, 12)
-#define DRIVING_PARAM_5_DEFAULT	(XTP_LN_TX_LCTXC0_SW2_PRE1_DEFAULT | \
-				 XTP_LN_TX_LCTXC0_SW3_PRE0_DEFAULT)
-
-#define XTP_LN_TX_LCTXCP1_SW0_PRE0_DEFAULT	0
-#define XTP_LN_TX_LCTXCP1_SW0_PRE1_DEFAULT	GENMASK(10, 10)
-#define XTP_LN_TX_LCTXCP1_SW0_PRE2_DEFAULT	GENMASK(19, 19)
-#define XTP_LN_TX_LCTXCP1_SW0_PRE3_DEFAULT	GENMASK(28, 28)
-#define DRIVING_PARAM_6_DEFAULT	(XTP_LN_TX_LCTXCP1_SW0_PRE0_DEFAULT | \
-				 XTP_LN_TX_LCTXCP1_SW0_PRE1_DEFAULT | \
-				 XTP_LN_TX_LCTXCP1_SW0_PRE2_DEFAULT | \
-				 XTP_LN_TX_LCTXCP1_SW0_PRE3_DEFAULT)
-
-#define XTP_LN_TX_LCTXCP1_SW1_PRE0_DEFAULT	0
-#define XTP_LN_TX_LCTXCP1_SW1_PRE1_DEFAULT	GENMASK(10, 9)
-#define XTP_LN_TX_LCTXCP1_SW1_PRE2_DEFAULT	GENMASK(19, 18)
-#define XTP_LN_TX_LCTXCP1_SW2_PRE0_DEFAULT	0
-#define DRIVING_PARAM_7_DEFAULT	(XTP_LN_TX_LCTXCP1_SW1_PRE0_DEFAULT | \
-				 XTP_LN_TX_LCTXCP1_SW1_PRE1_DEFAULT | \
-				 XTP_LN_TX_LCTXCP1_SW1_PRE2_DEFAULT | \
-				 XTP_LN_TX_LCTXCP1_SW2_PRE0_DEFAULT)
-
-#define XTP_LN_TX_LCTXCP1_SW2_PRE1_DEFAULT	GENMASK(3, 3)
-#define XTP_LN_TX_LCTXCP1_SW3_PRE0_DEFAULT	0
-#define DRIVING_PARAM_8_DEFAULT	(XTP_LN_TX_LCTXCP1_SW2_PRE1_DEFAULT | \
-				 XTP_LN_TX_LCTXCP1_SW3_PRE0_DEFAULT)
+/* DP_PHYD_BIT_RATE */
+#define PHYD_DIG_RG_BIT_RATE_V2		GENMASK(3, 0)
+#define PHYD_DIG_RG_BIT_RATE		GENMASK(1, 0)
+
+/* DP_PHYD_SW_RST */
+#define PHYD_DIG_GLB_SW_RST_B		GENMASK(7, 0)
+#  define DP_GLB_SW_RST_PHYD		BIT(0)
+#  define DP_GLB_SW_RST_TFIFO_ANA	BIT(1)
+#  define DP_GLB_SW_RST_XTAL_CLK	BIT(2)
+#  define DP_GLB_SW_RST_MAIN_LINK	BIT(3)
+
+/* DP_PHYD_AUX_RX_CTL */
+#define PHYD_DIG_DPAUX_RX_EN		BIT(0)
+#define PHYD_DIG_XTP_GLB_CKDET_EN	BIT(1)
+#define PHYD_DIG_DPAUX_RX_DEGLITCH_EN	BIT(2)
+
+#define DRIVING_PARAM_0_DEFAULT	0x0
+#define DRIVING_PARAM_1_DEFAULT	0x0
+#define DRIVING_PARAM_2_DEFAULT	0x0
+
+/* DP_PHYD_TX_CTL_0 */
+#define PHYD_TX_LN_EN			GENMASK(7, 4)
+#define PHYD_TX_LN_EN_V2		GENMASK(3, 0)
+
+/* DP_PHYD_DRIVING_FORCE */
+#define PHYD_DP_TX_FORCE_VOLT_SWING_EN	BIT(0)
+#define PHYD_DP_TX_FORCE_VOLT_SWING_VAL	GENMASK(2, 1)
+#define PHYD_DP_TX_FORCE_PRE_EMPH_VAL	GENMASK(4, 3)
+
+/*
+ * DRIVING_PARAM_X (PHYD - Digital)
+ *
+ * Driving param registers are split in three sets, all containing settings
+ * for Voltage Swing and Pre-Emphasis for each lane's differential pair.
+ *
+ * All three sets share the same layout, but for different physical signals;
+ * In particular:
+ * [0-2]: LC TX CM (Minus / Negative Edge)
+ * [3-5]: LC TX C  (Logic State Change Point)
+ * [6-8]: LC TX CP (Plus / Positive Edge)
+ *
+ * And they contain values for:
+ * [0,3,6]: Swing 0 Pre[0-3]
+ * [1,4,7]: Swing 1 Pre[0-2] and Swing 2 Pre0
+ * [2,5,8]: Swing 2 Pre1 and Swing 3 Pre0
+ */
+#define PHYD_DIG_NUM_DRV_PARA_REGS	9
+#define XTP_LN_TX_LCTXC_SW0_PRE0	GENMASK(5, 0)
+#define XTP_LN_TX_LCTXC_SW0_PRE1	GENMASK(13, 8)
+#define XTP_LN_TX_LCTXC_SW0_PRE2	GENMASK(21, 16)
+#define XTP_LN_TX_LCTXC_SW0_PRE3	GENMASK(29, 24)
+
+#define XTP_LN_TX_LCTXC_SW1_PRE0	GENMASK(5, 0)
+#define XTP_LN_TX_LCTXC_SW1_PRE1	GENMASK(13, 8)
+#define XTP_LN_TX_LCTXC_SW1_PRE2	GENMASK(21, 16)
+#define XTP_LN_TX_LCTXC_SW2_PRE0	GENMASK(29, 24)
+
+#define XTP_LN_TX_LCTXC_SW2_PRE1	GENMASK(5, 0)
+#define XTP_LN_TX_LCTXC_SW3_PRE0	GENMASK(13, 8)
+
+#define BUILD_DRIVING_PARAM_0(sw0_pre0, sw0_pre1, sw0_pre2, sw0_pre3) (	\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW0_PRE0, sw0_pre0) |		\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW0_PRE1, sw0_pre1) |		\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW0_PRE2, sw0_pre2) |		\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW0_PRE3, sw0_pre3)		\
+)
+
+#define BUILD_DRIVING_PARAM_12(sw1_pre0, sw1_pre1, sw1_pre2, sw2_pre0) (\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW1_PRE0, sw1_pre0) |		\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW1_PRE1, sw1_pre1) |		\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW1_PRE2, sw1_pre2) |		\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW2_PRE0, sw2_pre0)		\
+)
+
+#define BUILD_DRIVING_PARAM_23(sw2_pre1, sw3_pre0) (			\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW2_PRE1, sw2_pre1) |		\
+	FIELD_PREP_CONST(XTP_LN_TX_LCTXC_SW3_PRE0, sw3_pre0)		\
+)
+
+/* MT8195: Logic State Change Point (LC TX C) */
+#define MT8195_DRIVING_PARAM_3_DEFAULT	BUILD_DRIVING_PARAM_0( 16, 20, 24, 32)
+#define MT8195_DRIVING_PARAM_4_DEFAULT	BUILD_DRIVING_PARAM_12(24, 30, 36, 32)
+#define MT8195_DRIVING_PARAM_5_DEFAULT	BUILD_DRIVING_PARAM_23(40, 48)
+
+/* MT8195: Positive Edge (LC TX CP) */
+#define MT8195_DRIVING_PARAM_6_DEFAULT	BUILD_DRIVING_PARAM_0( 0, 4, 8, 16)
+#define MT8195_DRIVING_PARAM_7_DEFAULT	BUILD_DRIVING_PARAM_12(0, 6, 12, 0)
+#define MT8195_DRIVING_PARAM_8_DEFAULT	BUILD_DRIVING_PARAM_23(8, 0)
+
+/* MT8196/MT6991: Logic State Change Point (LC TX C) */
+#define MT8196_DRIVING_PARAM_3_DEFAULT	BUILD_DRIVING_PARAM_0( 10, 12, 14, 17)
+#define MT8196_DRIVING_PARAM_4_DEFAULT	BUILD_DRIVING_PARAM_12(14, 17, 18, 18)
+#define MT8196_DRIVING_PARAM_5_DEFAULT	BUILD_DRIVING_PARAM_23(21, 24)
+
+/* MT8196/MT6991: Positive Edge (LC TX CP) */
+#define MT8196_DRIVING_PARAM_6_DEFAULT	BUILD_DRIVING_PARAM_0( 0, 2, 4, 7)
+#define MT8196_DRIVING_PARAM_7_DEFAULT	BUILD_DRIVING_PARAM_12(0, 3, 6, 0)
+#define MT8196_DRIVING_PARAM_8_DEFAULT	BUILD_DRIVING_PARAM_23(3, 0)
+
+enum mtk_dp_phya_ana_glb_regidx {
+	DP_PHYA_GLB_BIAS_GEN_0,
+	DP_PHYA_GLB_BIAS_GEN_1,
+	DP_PHYA_GLB_DPAUX_TX,
+	DP_PHYA_GLB_FORCE_CTRL_0,
+	DP_PHYA_GLB_FORCE_CTRL_1,
+	DP_PHYA_GLOBAL_MAX
+};
+
+enum mtk_dp_phya_ana_lane_regidx {
+	DP_PHYA_LAN_LANE_TX_0,
+	DP_PHYA_LAN_MAX
+};
+
+enum mtk_dp_phyd_dig_lane_regidx {
+	DP_PHYD_LAN_DRIVING_FORCE,
+	DP_PHYD_LAN_DRIVING_PARAM_0,
+	DP_PHYD_LAN_MAX
+};
+
+enum mtk_dp_phyd_dig_glb_regidx {
+	DP_PHYD_PLL_CTL_0,
+	DP_PHYD_PLL_CTL_1,
+	DP_PHYD_SW_RST,
+	DP_PHYD_BIT_RATE,
+	DP_PHYD_AUX_RX_CTL,
+	DP_PHYD_TX_CTL_0,
+	DP_PHYD_GLOBAL_MAX
+};
+
+enum mtk_dp_phyd_bit_rate_regval {
+	DP_PHYD_BIT_RATE_RBR,
+	DP_PHYD_BIT_RATE_HBR,
+	DP_PHYD_BIT_RATE_HBR2,
+	DP_PHYD_BIT_RATE_HBR3,
+	DP_PHYD_BIT_RATE_MAX,
+};
+
+static const u8 mt8195_phy_ana_glb_regs[DP_PHYA_GLOBAL_MAX] = {
+	[DP_PHYA_GLB_BIAS_GEN_0] = 0x0,
+	[DP_PHYA_GLB_BIAS_GEN_1] = 0x4,
+	[DP_PHYA_GLB_DPAUX_TX] = 0x8,
+	[DP_PHYA_GLB_FORCE_CTRL_0] = 0x30,
+	[DP_PHYA_GLB_FORCE_CTRL_1] = 0x34,
+};
+
+static const u8 mt8195_phy_ana_lane_regs[DP_PHYA_LAN_MAX] = {
+	[DP_PHYA_LAN_LANE_TX_0] = 0x4,
+};
+
+static const u8 mt8195_phy_dig_lane_regs[DP_PHYD_LAN_MAX] = {
+	[DP_PHYD_LAN_DRIVING_FORCE] = 0x18,
+	[DP_PHYD_LAN_DRIVING_PARAM_0] = 0x2c,
+};
+
+static const u8 mt8196_phy_dig_lane_regs[DP_PHYD_LAN_MAX] = {
+	[DP_PHYD_LAN_DRIVING_FORCE] = 0x30,
+	[DP_PHYD_LAN_DRIVING_PARAM_0] = 0x34,
+};
+
+static const u8 mt8195_phy_dig_glb_regs[DP_PHYD_GLOBAL_MAX] = {
+	[DP_PHYD_PLL_CTL_0] = 0x10,
+	[DP_PHYD_PLL_CTL_1] = 0x14,
+	[DP_PHYD_SW_RST] = 0x38,
+	[DP_PHYD_BIT_RATE] = 0x3c,
+	[DP_PHYD_AUX_RX_CTL] = 0x40,
+	[DP_PHYD_TX_CTL_0] = 0x44,
+};
+
+static const u8 mt8196_phy_dig_glb_regs[DP_PHYD_GLOBAL_MAX] = {
+	[DP_PHYD_PLL_CTL_0] = 0x10,
+	[DP_PHYD_PLL_CTL_1] = 0x14,
+	[DP_PHYD_SW_RST] = 0x38,
+	[DP_PHYD_BIT_RATE] = 0x3c,
+	[DP_PHYD_AUX_RX_CTL] = 0x40,
+	[DP_PHYD_TX_CTL_0] = 0x74,
+};
+
+static const u8 mt8195_phy_dig_bitrate_val[DP_PHYD_BIT_RATE_MAX] = {
+	[DP_PHYD_BIT_RATE_RBR] = 0,
+	[DP_PHYD_BIT_RATE_HBR] = 1,
+	[DP_PHYD_BIT_RATE_HBR2] = 2,
+	[DP_PHYD_BIT_RATE_HBR3] = 3
+};
+
+static const u8 mt8196_edp_phy_dig_bitrate_val[DP_PHYD_BIT_RATE_MAX] = {
+	[DP_PHYD_BIT_RATE_RBR] = 1,
+	[DP_PHYD_BIT_RATE_HBR] = 4,
+	[DP_PHYD_BIT_RATE_HBR2] = 7,
+	[DP_PHYD_BIT_RATE_HBR3] = 9
+};
+
+/**
+ * struct mtk_dp_phya_imp_sel - Per-Lane Impedance Selection
+ * @pmos: Impedance selection for P-Channel MOSFET
+ * @nmos: Impedance selection for N-Channel MOSFET
+ */
+struct mtk_dp_phya_imp_sel {
+	u8 pmos : 4;
+	u8 nmos : 4;
+};
+
+/**
+ * struct mtk_dp_phy_pdata - Platform data and defaults for MediaTek DP/eDP PHY
+ * @off_ana_glb:    Base offset for dptx_phyd_sifslv_ana_glb
+ * @off_ana_lane:   Base offsets for dptx_phyd_sifslv_ana_lan (for each lane)
+ * @off_dig_glb:    Base offset for dptx_phyd_sifslv_dig_glb
+ * @off_dig_lane:   Base offsets for dptx_phyd_sifslv_dig_lan (for each lane)
+ * @regs_ana_glb:   Register (layout) offsets for ana_glb
+ * @regs_ana_lane:  Register (layout) offsets for ana_lan
+ * @regs_dig_glb:   Register (layout) offsets for dig_glb
+ * @regs_dig_lane:  Register (layout) offsets for dig_lan
+ * @mask_dig_tx_ln: Register mask for PHYD_TX_LN_EN field
+ * @val_dig_bitrate:IP Version specific register values for Bit Rate setting
+ * @ana_bias_r:     Internal resistance "R" Selection Settings (global)
+ * @ana_cktx_imp:   TX Clock Impedance Selection Settings (global)
+ * @ana_lanes_imp:  TX Impedance Selection Settings (for all lanes)
+ * @driving_params: Voltage Swing and Pre-Emphasis settings (for all lanes)
+ */
+struct mtk_dp_phy_pdata {
+	/* Register offsets */
+	u16 off_ana_glb;
+	u16 off_ana_lane[MTK_DP_PHY_MAX_LANES];
+	u16 off_dig_glb;
+	u16 off_dig_lane[MTK_DP_PHY_MAX_LANES];
+
+	/* Register maps */
+	const u8 *regs_ana_glb;
+	const u8 *regs_ana_lane;
+	const u8 *regs_dig_glb;
+	const u8 *regs_dig_lane;
+
+	/* Register masks */
+	u32 mask_dig_tx_ln;
+
+	/* IP-Version specific register value arrays */
+	const u8 *val_dig_bitrate;
+
+	/* Calibration defaults */
+	u8 ana_bias_r;
+	u8 ana_cktx_imp;
+	struct mtk_dp_phya_imp_sel ana_lanes_imp;
+	u32 driving_params[PHYD_DIG_NUM_DRV_PARA_REGS];
+};
 
 struct mtk_dp_phy {
-	struct regmap *regs;
+	struct device *dev;
+	struct regmap *regmap;
+	const struct mtk_dp_phy_pdata *pdata;
+
+	u8 ana_bias_r;
+	u8 ana_cktx_imp;
+	struct mtk_dp_phya_imp_sel ana_impsel[MTK_DP_PHY_MAX_LANES];
 };
 
+static void mtk_dp_phy_set_analog_calibration_params(struct mtk_dp_phy *dp_phy)
+{
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
+	const u8 *regs_ana_glb = pdata->regs_ana_glb;
+	const u8 *regs_ana_lane = pdata->regs_ana_lane;
+	int i;
+
+	regmap_update_bits(dp_phy->regmap,
+			   pdata->off_ana_glb + regs_ana_glb[DP_PHYA_GLB_BIAS_GEN_0],
+			   XTP_GLB_BIAS_INT_R_CTRL,
+			   FIELD_PREP(XTP_GLB_BIAS_INT_R_CTRL, dp_phy->ana_bias_r));
+
+	regmap_update_bits(dp_phy->regmap,
+			   pdata->off_ana_glb + regs_ana_glb[DP_PHYA_GLB_DPAUX_TX],
+			   CKM_PT0_CKTX_IMPSEL,
+			   FIELD_PREP(CKM_PT0_CKTX_IMPSEL, dp_phy->ana_cktx_imp));
+
+	for (i = 0; i < MTK_DP_PHY_MAX_LANES; i++) {
+		struct mtk_dp_phya_imp_sel *ana_imp = &dp_phy->ana_impsel[i];
+		u32 val = FIELD_PREP(XTP_LN_TX_IMPSEL_PMOS, ana_imp->pmos) |
+			  FIELD_PREP(XTP_LN_TX_IMPSEL_NMOS, ana_imp->nmos);
+		u32 off_ana_lane = pdata->off_ana_lane[i];
+
+		regmap_update_bits(dp_phy->regmap,
+				   off_ana_lane + regs_ana_lane[DP_PHYA_LAN_LANE_TX_0],
+				   XTP_LN_TX_IMPSEL_PMOS | XTP_LN_TX_IMPSEL_NMOS, val);
+	}
+}
+
+static void mtk_dp_phy_set_digital_drv_params(struct mtk_dp_phy *dp_phy)
+{
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
+	const u32 reg = pdata->regs_dig_lane[DP_PHYD_LAN_DRIVING_PARAM_0];
+	int i;
+
+	/*
+	 * Assume that all lanes need the same driving parameters: this
+	 * will bulk write from DRIVING_PARAM_0 to DRIVING_PARAM_8 on
+	 * all lanes (a grand total of [9 * num_lanes] 32-bit writes)
+	 */
+	for (i = 0; i < MTK_DP_PHY_MAX_LANES; i++)
+		regmap_bulk_write(dp_phy->regmap, pdata->off_dig_lane[i] + reg,
+				  pdata->driving_params,
+				  ARRAY_SIZE(pdata->driving_params));
+}
+
 static int mtk_dp_phy_init(struct phy *phy)
 {
 	struct mtk_dp_phy *dp_phy = phy_get_drvdata(phy);
-	static const u32 driving_params[] = {
-		DRIVING_PARAM_3_DEFAULT,
-		DRIVING_PARAM_4_DEFAULT,
-		DRIVING_PARAM_5_DEFAULT,
-		DRIVING_PARAM_6_DEFAULT,
-		DRIVING_PARAM_7_DEFAULT,
-		DRIVING_PARAM_8_DEFAULT
-	};
 
-	regmap_bulk_write(dp_phy->regs, MTK_DP_LANE0_DRIVING_PARAM_3,
-			  driving_params, ARRAY_SIZE(driving_params));
-	regmap_bulk_write(dp_phy->regs, MTK_DP_LANE1_DRIVING_PARAM_3,
-			  driving_params, ARRAY_SIZE(driving_params));
-	regmap_bulk_write(dp_phy->regs, MTK_DP_LANE2_DRIVING_PARAM_3,
-			  driving_params, ARRAY_SIZE(driving_params));
-	regmap_bulk_write(dp_phy->regs, MTK_DP_LANE3_DRIVING_PARAM_3,
-			  driving_params, ARRAY_SIZE(driving_params));
+	mtk_dp_phy_set_digital_drv_params(dp_phy);
+	mtk_dp_phy_set_analog_calibration_params(dp_phy);
 
 	return 0;
 }
@@ -109,9 +355,14 @@ static int mtk_dp_phy_init(struct phy *phy)
 static int mtk_dp_phy_configure(struct phy *phy, union phy_configure_opts *opts)
 {
 	struct mtk_dp_phy *dp_phy = phy_get_drvdata(phy);
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
 	u32 val;
+	int i;
 
 	if (opts->dp.set_rate) {
+		const u32 reg_bit_rate = pdata->regs_dig_glb[DP_PHYD_BIT_RATE];
+		enum mtk_dp_phyd_bit_rate_regval regval_idx;
+
 		switch (opts->dp.link_rate) {
 		default:
 			dev_err(&phy->dev,
@@ -119,84 +370,469 @@ static int mtk_dp_phy_configure(struct phy *phy, union phy_configure_opts *opts)
 				opts->dp.link_rate);
 			return -EINVAL;
 		case 1620:
-			val = BIT_RATE_RBR;
+			regval_idx = DP_PHYD_BIT_RATE_RBR;
 			break;
 		case 2700:
-			val = BIT_RATE_HBR;
+			regval_idx = DP_PHYD_BIT_RATE_HBR;
 			break;
 		case 5400:
-			val = BIT_RATE_HBR2;
+			regval_idx = DP_PHYD_BIT_RATE_HBR2;
 			break;
 		case 8100:
-			val = BIT_RATE_HBR3;
+			regval_idx = DP_PHYD_BIT_RATE_HBR3;
 			break;
 		}
-		regmap_write(dp_phy->regs, MTK_DP_PHY_DIG_BIT_RATE, val);
+		regmap_write(dp_phy->regmap, pdata->off_dig_glb + reg_bit_rate,
+			     pdata->val_dig_bitrate[regval_idx]);
+	}
+
+	if (opts->dp.set_lanes) {
+		const u32 reg_dig_tx_ctl = pdata->regs_dig_glb[DP_PHYD_TX_CTL_0];
+
+		val = 0;
+		for (i = 0; i < opts->dp.lanes; i++)
+			val |= field_prep(pdata->mask_dig_tx_ln, BIT(i));
+
+		regmap_update_bits(dp_phy->regmap, pdata->off_dig_glb + reg_dig_tx_ctl,
+				   pdata->mask_dig_tx_ln, val);
+	}
+
+	if (opts->dp.set_voltages) {
+		const u32 reg_drv_force = pdata->regs_dig_lane[DP_PHYD_LAN_DRIVING_FORCE];
+
+		if (opts->dp.lanes > 4) {
+			dev_err(&phy->dev, "Wrong lanes config %u\n", opts->dp.lanes);
+			return -EINVAL;
+		}
+
+		for (i = 0; i < opts->dp.lanes; i++) {
+			const u32 off_dig_lane = pdata->off_dig_lane[i];
+			u32 val;
+
+			val = FIELD_PREP(PHYD_DP_TX_FORCE_VOLT_SWING_VAL, opts->dp.voltage[i]);
+			val |= FIELD_PREP(PHYD_DP_TX_FORCE_PRE_EMPH_VAL, opts->dp.pre[i]);
+			val |= PHYD_DP_TX_FORCE_VOLT_SWING_EN;
+
+			regmap_update_bits(dp_phy->regmap, off_dig_lane + reg_drv_force,
+					   PHYD_DP_TX_FORCE_VOLT_SWING_EN |
+					   PHYD_DP_TX_FORCE_VOLT_SWING_VAL |
+					   PHYD_DP_TX_FORCE_PRE_EMPH_VAL,
+					   val);
+		}
 	}
 
-	regmap_update_bits(dp_phy->regs, MTK_DP_PHY_DIG_PLL_CTL_1,
+	regmap_update_bits(dp_phy->regmap,
+			   pdata->off_dig_glb + pdata->regs_dig_glb[DP_PHYD_PLL_CTL_1],
 			   TPLL_SSC_EN, opts->dp.ssc ? TPLL_SSC_EN : 0);
 
 	return 0;
 }
 
+static int mtk_dp_phy_power_on(struct phy *phy)
+{
+	struct mtk_dp_phy *dp_phy = phy_get_drvdata(phy);
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
+	const u8 *regs_dig = pdata->regs_dig_glb;
+	const u8 *regs_ana = pdata->regs_ana_glb;
+
+	/* Enable AUX Channel with RX De-Glitch and input clock detection */
+	regmap_write(dp_phy->regmap,
+		     pdata->off_dig_glb + regs_dig[DP_PHYD_AUX_RX_CTL],
+		     PHYD_DIG_DPAUX_RX_EN |
+		     PHYD_DIG_XTP_GLB_CKDET_EN |
+		     PHYD_DIG_DPAUX_RX_DEGLITCH_EN);
+
+	regmap_clear_bits(dp_phy->regmap,
+			  pdata->off_ana_glb + regs_ana[DP_PHYA_GLB_FORCE_CTRL_1],
+			  CKM_CKTX0_EN_FORCE_MODE);
+
+	return 0;
+}
+
+static int mtk_dp_phy_disable_all_lanes(struct mtk_dp_phy *dp_phy)
+{
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
+	const u8 *regs = pdata->regs_dig_glb;
+	u32 val;
+
+	/* Get mask of currently enabled lane */
+	regmap_read(dp_phy->regmap, pdata->off_dig_glb + regs[DP_PHYD_TX_CTL_0], &val);
+	val = field_get(pdata->mask_dig_tx_ln, val);
+	if (val == 0)
+		return 0;
+
+	/* Disable all lanes (needs to be done one by one, from last to first) */
+	do {
+		u32 lane_num = fls(val) - 1;
+		val &= ~BIT(lane_num);
+
+		regmap_clear_bits(dp_phy->regmap,
+				  pdata->off_dig_glb + regs[DP_PHYD_TX_CTL_0],
+				  field_prep(pdata->mask_dig_tx_ln, BIT(lane_num)));
+	} while (val);
+
+	return 0;
+}
+
+static int mtk_dp_phy_power_off(struct phy *phy)
+{
+	struct mtk_dp_phy *dp_phy = phy_get_drvdata(phy);
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
+	const u8 *regs_dig = pdata->regs_dig_glb;
+	const u8 *regs_ana = pdata->regs_ana_glb;
+	int ret;
+
+	regmap_set_bits(dp_phy->regmap,
+			pdata->off_ana_glb + regs_ana[DP_PHYA_GLB_FORCE_CTRL_1],
+			CKM_CKTX0_EN_FORCE_MODE);
+
+	/* Disable RX unconditionally */
+	regmap_write(dp_phy->regmap,
+		     pdata->off_dig_glb + regs_dig[DP_PHYD_AUX_RX_CTL], 0);
+
+	ret = mtk_dp_phy_disable_all_lanes(dp_phy);
+	if (ret) {
+		dev_err(dp_phy->dev, "Could not disable lanes for poweroff!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int mtk_dp_phy_reset(struct phy *phy)
 {
 	struct mtk_dp_phy *dp_phy = phy_get_drvdata(phy);
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
+	const u32 reg_rst = pdata->regs_dig_glb[DP_PHYD_SW_RST];
+	const u32 reg_drv_force = pdata->regs_dig_lane[DP_PHYD_LAN_DRIVING_FORCE];
+	int i, ret;
+
+	/* Clearing bits sets reset state */
+	regmap_clear_bits(dp_phy->regmap, pdata->off_dig_glb + reg_rst, DP_GLB_SW_RST_PHYD);
 
-	regmap_update_bits(dp_phy->regs, MTK_DP_PHY_DIG_SW_RST,
-			   DP_GLB_SW_RST_PHYD, 0);
+	/* PHYD needs 50uS to guarantee reset done */
 	usleep_range(50, 200);
-	regmap_update_bits(dp_phy->regs, MTK_DP_PHY_DIG_SW_RST,
-			   DP_GLB_SW_RST_PHYD, 1);
+
+	/* Setting bits means go out of reset */
+	regmap_set_bits(dp_phy->regmap, pdata->off_dig_glb + reg_rst, DP_GLB_SW_RST_PHYD);
+
+	/* Disable all lanes and continue reset even if this fails, but notify */
+	ret = mtk_dp_phy_disable_all_lanes(dp_phy);
+	if (ret)
+		dev_err(dp_phy->dev, "Could not disable lanes during reset!\n");
+
+	/* Reset Voltage Swing and Preemphasis values */
+	for (i = 0; i < MTK_DP_PHY_MAX_LANES; i++) {
+		const u32 off_dig_lane = pdata->off_dig_lane[i];
+
+		regmap_clear_bits(dp_phy->regmap, off_dig_lane + reg_drv_force,
+				   PHYD_DP_TX_FORCE_VOLT_SWING_EN |
+				   PHYD_DP_TX_FORCE_VOLT_SWING_VAL |
+				   PHYD_DP_TX_FORCE_PRE_EMPH_VAL);
+	}
 
 	return 0;
 }
 
 static const struct phy_ops mtk_dp_phy_dev_ops = {
 	.init = mtk_dp_phy_init,
+	.power_on = mtk_dp_phy_power_on,
+	.power_off = mtk_dp_phy_power_off,
 	.configure = mtk_dp_phy_configure,
 	.reset = mtk_dp_phy_reset,
 	.owner = THIS_MODULE,
 };
 
+static void mtk_dp_phy_get_default_cal_data(struct mtk_dp_phy *dp_phy)
+{
+	const struct mtk_dp_phy_pdata *pdata = dp_phy->pdata;
+	int i;
+
+	dp_phy->ana_bias_r = pdata->ana_bias_r;
+	dp_phy->ana_cktx_imp = pdata->ana_cktx_imp;
+
+	/* Copy the default lane impedance settings to all lanes */
+	for (i = 0; i < MTK_DP_PHY_MAX_LANES; i++)
+		memcpy(&dp_phy->ana_impsel[i], &pdata->ana_lanes_imp,
+		       sizeof(dp_phy->ana_impsel[0]));
+
+	return;
+}
+
+static int mtk_dp_phy_get_one_cal_para(struct device *dev, const char *name, u8 max_val)
+{
+	u8 buf_byte;
+	u16 buf;
+	int ret;
+
+	/*
+	 * All of the calibrations are always max 8 bits long, but some may
+	 * be split between two different 8-bits cells: handle this corner
+	 * case by retrying reading as u16.
+	 */
+	ret = nvmem_cell_read_u8(dev, name, &buf_byte);
+	if (ret)
+		ret = nvmem_cell_read_u16(dev, name, &buf);
+	else
+		buf = buf_byte;
+
+	if (ret)
+		return dev_err_probe(dev, ret, "Cannot get calibration data for %s\n", name);
+
+	if (buf == 0) {
+		dev_warn(dev, "No calibration for %s. Using defaults\n", name);
+		return -ENOENT;
+	}
+
+	if (buf > max_val)
+		return dev_err_probe(dev, -ERANGE, "Bad value %u retrieved for %s\n", buf, name);
+
+	return buf;
+}
+
+static int mtk_dp_phy_get_calibration_data(struct mtk_dp_phy *dp_phy)
+{
+	char mtk_dp_cal_lane_imp_name[] = "impedance-laneXM";
+	struct device *dev = dp_phy->dev;
+	int i, ret;
+
+	ret = mtk_dp_phy_get_one_cal_para(dev, "rbias-trim", FIELD_MAX(XTP_GLB_BIAS_INT_R_CTRL));
+	if (ret < 0)
+		goto end;
+	dp_phy->ana_bias_r = ret;
+
+	ret = mtk_dp_phy_get_one_cal_para(dev, "impedance-txclk", FIELD_MAX(CKM_PT0_CKTX_IMPSEL));
+	if (ret < 0)
+		goto end;
+	dp_phy->ana_cktx_imp = ret;
+
+	/* Get impedance params for each lane */
+	for (i = 0; i < MTK_DP_PHY_MAX_LANES; i++) {
+		/* P-MOSFET first */
+		snprintf(mtk_dp_cal_lane_imp_name, ARRAY_SIZE(mtk_dp_cal_lane_imp_name),
+			 "impedance-lane%dp", i);
+		ret = mtk_dp_phy_get_one_cal_para(dev, mtk_dp_cal_lane_imp_name,
+						  FIELD_MAX(XTP_LN_TX_IMPSEL_PMOS));
+		if (ret < 0)
+			goto end;
+		dp_phy->ana_impsel[i].pmos = ret;
+
+		/* ...and then N-MOSFET too */
+		snprintf(mtk_dp_cal_lane_imp_name, ARRAY_SIZE(mtk_dp_cal_lane_imp_name),
+			 "impedance-lane%dn", i);
+		ret = mtk_dp_phy_get_one_cal_para(dev, mtk_dp_cal_lane_imp_name,
+						  FIELD_MAX(XTP_LN_TX_IMPSEL_NMOS));
+		if (ret < 0)
+			goto end;
+		dp_phy->ana_impsel[i].nmos = ret;
+	}
+end:
+	if (ret < 0) {
+		/*
+		 * If any of the calibration values is missing, or if there
+		 * is no calibration at all in the eFuses, copy the default
+		 * one entirely (as partial values shall not be mixed!)
+		 */
+		if (ret == -ENOENT) {
+			dev_info(dev, "Using calibration default values\n");
+			mtk_dp_phy_get_default_cal_data(dp_phy);
+			return 0;
+		}
+		return ret;
+	};
+
+	return 0;
+}
+
+static void mtk_dp_phy_legacy_remove_lookup(void *data)
+{
+	struct phy *phy = data;
+	struct mtk_dp_phy *dp_phy = phy_get_drvdata(phy);
+
+	phy_remove_lookup(phy, "dp", dev_name(dp_phy->dev));
+}
+
+static const struct mtk_dp_phy_pdata mt8195_dp_phy_data;
+
+static int mtk_dp_phy_legacy_probe(struct platform_device *pdev, struct mtk_dp_phy *dp_phy)
+{
+	struct device *dev = &pdev->dev;
+	struct phy *phy;
+	int ret;
+
+	/*
+	 * If legacy platform driver probe, assume this is MT8195 or compatible
+	 * with a devicetree that was not migrated to the new, proper bindings.
+	 */
+	dp_phy->pdata = &mt8195_dp_phy_data;
+	dp_phy->regmap = *(struct regmap **)dev->platform_data;
+	if (!dp_phy->regmap)
+		return dev_err_probe(dev, -EINVAL, "No platform data available\n");
+
+	phy = devm_phy_create(dev, NULL, &mtk_dp_phy_dev_ops);
+	if (IS_ERR(phy))
+		return dev_err_probe(dev, PTR_ERR(phy),
+				     "Failed to create DP PHY\n");
+
+	phy_set_drvdata(phy, dp_phy);
+
+	/*
+	 * Set default calibration data before exposing the PHY.
+	 * For legacy probe, mtk_dp will set calibrations from eFuse, if found.
+	 */
+	mtk_dp_phy_get_default_cal_data(dp_phy);
+
+	ret = phy_create_lookup(phy, "dp", dev_name(dev));
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, mtk_dp_phy_legacy_remove_lookup, phy);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct regmap_config mtk_dp_phy_regmap_cfg = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.disable_locking = true,
+};
+
 static int mtk_dp_phy_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct phy_provider *provider;
 	struct mtk_dp_phy *dp_phy;
+	void __iomem *base;
 	struct phy *phy;
-	struct regmap *regs;
-
-	regs = *(struct regmap **)dev->platform_data;
-	if (!regs)
-		return dev_err_probe(dev, -EINVAL,
-				     "No data passed, requires struct regmap**\n");
+	int ret;
 
 	dp_phy = devm_kzalloc(dev, sizeof(*dp_phy), GFP_KERNEL);
 	if (!dp_phy)
 		return -ENOMEM;
 
-	dp_phy->regs = regs;
+	dp_phy->dev = dev;
+
+	/* If there's no devicetree, go for legacy pdev probe */
+	if (!dev->of_node)
+		return mtk_dp_phy_legacy_probe(pdev, dp_phy);
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	dp_phy->regmap = devm_regmap_init_mmio(dev, base, &mtk_dp_phy_regmap_cfg);
+	if (IS_ERR(dp_phy->regmap))
+		return PTR_ERR(dp_phy->regmap);
+
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
+
+	dp_phy->pdata = device_get_match_data(dev);
+
+	if (IS_REACHABLE(CONFIG_NVMEM)) {
+		ret = mtk_dp_phy_get_calibration_data(dp_phy);
+		if (ret)
+			return ret;
+	} else {
+		/* Use default calibration data */
+		mtk_dp_phy_get_default_cal_data(dp_phy);
+	}
+
 	phy = devm_phy_create(dev, NULL, &mtk_dp_phy_dev_ops);
 	if (IS_ERR(phy))
 		return dev_err_probe(dev, PTR_ERR(phy),
 				     "Failed to create DP PHY\n");
 
 	phy_set_drvdata(phy, dp_phy);
-	if (!dev->of_node)
-		phy_create_lookup(phy, "dp", dev_name(dev));
+
+	provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (IS_ERR(provider))
+		return PTR_ERR(provider);
 
 	return 0;
 }
 
+static const struct mtk_dp_phy_pdata mt8195_dp_phy_data = {
+	.off_ana_glb = 0,
+	.off_ana_lane = { 0x100, 0x200, 0x300, 0x400 },
+	.off_dig_glb = 0x1000,
+	.off_dig_lane = { 0x1100, 0x1200, 0x1300, 0x1400 },
+	.regs_ana_glb = mt8195_phy_ana_glb_regs,
+	.regs_ana_lane = mt8195_phy_ana_lane_regs,
+	.regs_dig_glb = mt8195_phy_dig_glb_regs,
+	.regs_dig_lane = mt8195_phy_dig_lane_regs,
+	.mask_dig_tx_ln = PHYD_TX_LN_EN,
+	.val_dig_bitrate = mt8195_phy_dig_bitrate_val,
+	.ana_bias_r = 15,
+	.ana_cktx_imp = 8,
+	.ana_lanes_imp = {
+		.pmos = 8,
+		.nmos = 8,
+	},
+	.driving_params = {
+		[0] = 0,
+		[1] = 0,
+		[2] = 0,
+		[3] = MT8195_DRIVING_PARAM_3_DEFAULT,
+		[4] = MT8195_DRIVING_PARAM_4_DEFAULT,
+		[5] = MT8195_DRIVING_PARAM_5_DEFAULT,
+		[6] = MT8195_DRIVING_PARAM_6_DEFAULT,
+		[7] = MT8195_DRIVING_PARAM_7_DEFAULT,
+		[8] = MT8195_DRIVING_PARAM_8_DEFAULT
+	},
+};
+
+static const struct mtk_dp_phy_pdata mt8196_edp_phy_data = {
+	.off_ana_glb = 0x400,
+	.off_ana_lane = { 0x0, 0x100, 0x200, 0x300 },
+	.off_dig_glb = 0x1400,
+	.off_dig_lane = { 0x1000, 0x1100, 0x1200, 0x1300 },
+	.regs_ana_glb = mt8195_phy_ana_glb_regs,
+	.regs_ana_lane = mt8195_phy_ana_lane_regs,
+	.regs_dig_glb = mt8196_phy_dig_glb_regs,
+	.regs_dig_lane = mt8196_phy_dig_lane_regs,
+	.mask_dig_tx_ln = PHYD_TX_LN_EN_V2,
+	.val_dig_bitrate = mt8196_edp_phy_dig_bitrate_val,
+	.ana_bias_r = 15,
+	.ana_cktx_imp = 8,
+	.ana_lanes_imp = {
+		.pmos = 8,
+		.nmos = 8,
+	},
+	.driving_params = {
+		[0] = 0,
+		[1] = 0,
+		[2] = 0,
+		[3] = MT8196_DRIVING_PARAM_3_DEFAULT,
+		[4] = MT8196_DRIVING_PARAM_4_DEFAULT,
+		[5] = MT8196_DRIVING_PARAM_5_DEFAULT,
+		[6] = MT8196_DRIVING_PARAM_6_DEFAULT,
+		[7] = MT8196_DRIVING_PARAM_7_DEFAULT,
+		[8] = MT8196_DRIVING_PARAM_8_DEFAULT
+	},
+};
+
+static const struct of_device_id mtk_dp_phy_of_match[] = {
+	{ .compatible = "mediatek,mt8195-dp-phy", .data = &mt8195_dp_phy_data },
+	{ .compatible = "mediatek,mt8196-edp-phy", .data = &mt8196_edp_phy_data },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mtk_dp_phy_of_match);
+
 static struct platform_driver mtk_dp_phy_driver = {
 	.probe = mtk_dp_phy_probe,
 	.driver = {
 		.name = "mediatek-dp-phy",
+		.of_match_table = mtk_dp_phy_of_match,
 	},
 };
 module_platform_driver(mtk_dp_phy_driver);
 
+MODULE_AUTHOR("AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>");
 MODULE_AUTHOR("Markus Schneider-Pargmann <msp@baylibre.com>");
-MODULE_DESCRIPTION("MediaTek DP PHY Driver");
+MODULE_DESCRIPTION("MediaTek DisplayPort PHY Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c
index 1426a2db984d53..a4bc1268946d83 100644
--- a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c
+++ b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.c
@@ -36,7 +36,7 @@ mtk_phy_tmds_clk_ratio(struct mtk_hdmi_phy *hdmi_phy, bool enable)
 	 * clock bit ratio 1:40, under 3.4Gbps, clock bit ratio 1:10
 	 */
 	if (enable)
-		mtk_phy_update_field(regs + HDMI20_CLK_CFG, REG_TXC_DIV, 3);
+		mtk_phy_update_field(regs + HDMI20_CLK_CFG, REG_TXC_DIV, VAL_TXC_DIV4);
 	else
 		mtk_phy_clear_bits(regs + HDMI20_CLK_CFG, REG_TXC_DIV);
 }
@@ -290,7 +290,7 @@ static int mtk_hdmi_pll_calc(struct mtk_hdmi_phy *hdmi_phy, struct clk_hw *hw,
 	posdiv2 = 1;
 
 	/* Digital clk divider, max /32 */
-	digital_div = div_u64(ns_hdmipll_ck, posdiv1 * posdiv2 * pixel_clk);
+	digital_div = div64_u64(ns_hdmipll_ck, posdiv1 * posdiv2 * pixel_clk);
 	if (!(digital_div <= 32 && digital_div >= 1))
 		return -EINVAL;
 
diff --git a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h
index e26caaf4d104ce..58800d7659ca01 100644
--- a/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h
+++ b/drivers/phy/mediatek/phy-mtk-hdmi-mt8195.h
@@ -17,6 +17,9 @@
 
 #define HDMI20_CLK_CFG 0x70
 #define REG_TXC_DIV GENMASK(31, 30)
+#define VAL_TXC_DIV2 1
+#define VAL_TXC_DIV4 2
+#define VAL_TXC_DIV8 3
 
 #define HDMI_1_CFG_0 0x00
 #define RG_HDMITX21_DRV_IBIAS_CLK GENMASK(10, 5)
diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c
index 21aaf2f76e53eb..597e6b234f5ef6 100644
--- a/drivers/phy/phy-core.c
+++ b/drivers/phy/phy-core.c
@@ -124,13 +124,13 @@ static struct phy *phy_find(struct device *dev, const char *con_id)
 	const char *dev_id = dev_name(dev);
 	struct phy_lookup *p, *pl = NULL;
 
-	mutex_lock(&phy_provider_mutex);
+	lockdep_assert_held(&phy_provider_mutex);
+
 	list_for_each_entry(p, &phys, node)
 		if (!strcmp(p->dev_id, dev_id) && !strcmp(p->con_id, con_id)) {
 			pl = p;
 			break;
 		}
-	mutex_unlock(&phy_provider_mutex);
 
 	return pl ? pl->phy : ERR_PTR(-ENODEV);
 }
@@ -606,12 +606,44 @@ int phy_validate(struct phy *phy, enum phy_mode mode, int submode,
 }
 EXPORT_SYMBOL_GPL(phy_validate);
 
+/**
+ * _of_phy_get_with_args() - lookup and obtain a reference to a phy by of_phandle_args
+ * @args: of_phandle_args to the phy
+ *
+ * Returns: the phy from the provider's of_xlate, -ENODEV if disabled,
+ * -EPROBE_DEFER if the provider is not yet registered.
+ */
+static struct phy *_of_phy_get_with_args(struct of_phandle_args *args)
+{
+	struct phy *phy;
+	struct phy_provider *phy_provider;
+
+	lockdep_assert_held(&phy_provider_mutex);
+
+	phy_provider = of_phy_provider_lookup(args->np);
+	if (IS_ERR(phy_provider) || !try_module_get(phy_provider->owner))
+		return ERR_PTR(-EPROBE_DEFER);
+
+	if (!of_device_is_available(args->np)) {
+		dev_warn(phy_provider->dev, "Requested PHY is disabled\n");
+		phy = ERR_PTR(-ENODEV);
+		goto out_put_module;
+	}
+
+	phy = phy_provider->of_xlate(phy_provider->dev, args);
+
+out_put_module:
+	module_put(phy_provider->owner);
+
+	return phy;
+}
+
 /**
  * _of_phy_get() - lookup and obtain a reference to a phy by phandle
  * @np: device_node for which to get the phy
  * @index: the index of the phy
  *
- * Returns the phy associated with the given phandle value,
+ * Returns: the phy associated with the given phandle value,
  * after getting a refcount to it or -ENODEV if there is no such phy or
  * -EPROBE_DEFER if there is a phandle to the phy, but the device is
  * not yet loaded. This function uses of_xlate call back function provided
@@ -620,10 +652,11 @@ EXPORT_SYMBOL_GPL(phy_validate);
 static struct phy *_of_phy_get(struct device_node *np, int index)
 {
 	int ret;
-	struct phy_provider *phy_provider;
-	struct phy *phy = NULL;
+	struct phy *phy;
 	struct of_phandle_args args;
 
+	lockdep_assert_held(&phy_provider_mutex);
+
 	ret = of_parse_phandle_with_args(np, "phys", "#phy-cells",
 		index, &args);
 	if (ret)
@@ -635,26 +668,8 @@ static struct phy *_of_phy_get(struct device_node *np, int index)
 		goto out_put_node;
 	}
 
-	mutex_lock(&phy_provider_mutex);
-	phy_provider = of_phy_provider_lookup(args.np);
-	if (IS_ERR(phy_provider) || !try_module_get(phy_provider->owner)) {
-		phy = ERR_PTR(-EPROBE_DEFER);
-		goto out_unlock;
-	}
-
-	if (!of_device_is_available(args.np)) {
-		dev_warn(phy_provider->dev, "Requested PHY is disabled\n");
-		phy = ERR_PTR(-ENODEV);
-		goto out_put_module;
-	}
-
-	phy = phy_provider->of_xlate(phy_provider->dev, &args);
-
-out_put_module:
-	module_put(phy_provider->owner);
+	phy = _of_phy_get_with_args(&args);
 
-out_unlock:
-	mutex_unlock(&phy_provider_mutex);
 out_put_node:
 	of_node_put(args.np);
 
@@ -666,7 +681,7 @@ static struct phy *_of_phy_get(struct device_node *np, int index)
  * @np: device_node for which to get the phy
  * @con_id: name of the phy from device's point of view
  *
- * Returns the phy driver, after getting a refcount to it; or
+ * Returns: the phy driver, after getting a refcount to it; or
  * -ENODEV if there is no such phy. The caller is responsible for
  * calling of_phy_put() to release that count.
  */
@@ -678,15 +693,21 @@ struct phy *of_phy_get(struct device_node *np, const char *con_id)
 	if (con_id)
 		index = of_property_match_string(np, "phy-names", con_id);
 
+	mutex_lock(&phy_provider_mutex);
+
 	phy = _of_phy_get(np, index);
 	if (IS_ERR(phy))
-		return phy;
+		goto out_unlock;
 
-	if (!try_module_get(phy->ops->owner))
-		return ERR_PTR(-EPROBE_DEFER);
+	if (!try_module_get(phy->ops->owner)) {
+		phy = ERR_PTR(-EPROBE_DEFER);
+		goto out_unlock;
+	}
 
 	get_device(&phy->dev);
 
+out_unlock:
+	mutex_unlock(&phy_provider_mutex);
 	return phy;
 }
 EXPORT_SYMBOL_GPL(of_phy_get);
@@ -776,7 +797,7 @@ EXPORT_SYMBOL_GPL(of_phy_simple_xlate);
  * @string: the phy name as given in the dt data or the name of the controller
  * port for non-dt case
  *
- * Returns the phy driver, after getting a refcount to it; or
+ * Returns: the phy driver, after getting a refcount to it; or
  * -ENODEV if there is no such phy.  The caller is responsible for
  * calling phy_put() to release that count.
  */
@@ -786,6 +807,7 @@ struct phy *phy_get(struct device *dev, const char *string)
 	struct phy *phy;
 	struct device_link *link;
 
+	mutex_lock(&phy_provider_mutex);
 	if (dev->of_node) {
 		if (string)
 			index = of_property_match_string(dev->of_node, "phy-names",
@@ -796,15 +818,18 @@ struct phy *phy_get(struct device *dev, const char *string)
 	} else {
 		if (string == NULL) {
 			dev_WARN(dev, "missing string\n");
-			return ERR_PTR(-EINVAL);
+			phy = ERR_PTR(-EINVAL);
+			goto out_unlock;
 		}
 		phy = phy_find(dev, string);
 	}
 	if (IS_ERR(phy))
-		return phy;
+		goto out_unlock;
 
-	if (!try_module_get(phy->ops->owner))
-		return ERR_PTR(-EPROBE_DEFER);
+	if (!try_module_get(phy->ops->owner)) {
+		phy = ERR_PTR(-EPROBE_DEFER);
+		goto out_unlock;
+	}
 
 	get_device(&phy->dev);
 
@@ -813,6 +838,8 @@ struct phy *phy_get(struct device *dev, const char *string)
 		dev_dbg(dev, "failed to create device link to %s\n",
 			dev_name(phy->dev.parent));
 
+out_unlock:
+	mutex_unlock(&phy_provider_mutex);
 	return phy;
 }
 EXPORT_SYMBOL_GPL(phy_get);
@@ -961,15 +988,17 @@ struct phy *devm_of_phy_get_by_index(struct device *dev, struct device_node *np,
 	if (!ptr)
 		return ERR_PTR(-ENOMEM);
 
+	mutex_lock(&phy_provider_mutex);
 	phy = _of_phy_get(np, index);
 	if (IS_ERR(phy)) {
 		devres_free(ptr);
-		return phy;
+		goto out_unlock;
 	}
 
 	if (!try_module_get(phy->ops->owner)) {
 		devres_free(ptr);
-		return ERR_PTR(-EPROBE_DEFER);
+		phy = ERR_PTR(-EPROBE_DEFER);
+		goto out_unlock;
 	}
 
 	get_device(&phy->dev);
@@ -982,10 +1011,77 @@ struct phy *devm_of_phy_get_by_index(struct device *dev, struct device_node *np,
 		dev_dbg(dev, "failed to create device link to %s\n",
 			dev_name(phy->dev.parent));
 
+out_unlock:
+	mutex_unlock(&phy_provider_mutex);
 	return phy;
 }
 EXPORT_SYMBOL_GPL(devm_of_phy_get_by_index);
 
+/**
+ * phy_get_by_of_node() - lookup and obtain a reference to a phy by device_node
+ * @np: node containing the phy
+ *
+ * Returns: the phy associated with the device node or ERR_PTR.
+ */
+struct phy *phy_get_by_of_node(struct device_node *np)
+{
+	struct of_phandle_args args = { .np = np, .args_count = 0 };
+	struct phy *phy;
+
+	mutex_lock(&phy_provider_mutex);
+
+	phy = _of_phy_get_with_args(&args);
+	if (IS_ERR(phy))
+		goto out_unlock;
+
+	if (!try_module_get(phy->ops->owner)) {
+		phy = ERR_PTR(-EPROBE_DEFER);
+		goto out_unlock;
+	}
+
+	get_device(&phy->dev);
+
+out_unlock:
+	mutex_unlock(&phy_provider_mutex);
+
+	return phy;
+}
+EXPORT_SYMBOL_GPL(phy_get_by_of_node);
+
+/**
+ * devm_phy_get_by_of_node() - devm managed lookup and obtain phy reference by device node
+ * @dev: device requesting the PHY
+ * @np: device_node of the PHY provider
+ *
+ * Returns: phy associated with the device_node or ERR_PTR. devres manages
+ * releasing resources.
+ */
+struct phy *devm_phy_get_by_of_node(struct device *dev, struct device_node *np)
+{
+	struct phy **ptr, *phy;
+	struct device_link *link;
+
+	ptr = devres_alloc(devm_phy_release, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
+
+	phy = phy_get_by_of_node(np);
+	if (IS_ERR(phy)) {
+		devres_free(ptr);
+		return phy;
+	}
+
+	*ptr = phy;
+	devres_add(dev, ptr);
+	link = device_link_add(dev, &phy->dev, DL_FLAG_STATELESS);
+	if (!link)
+		dev_dbg(dev, "failed to create device link to %s\n",
+			dev_name(phy->dev.parent));
+
+	return phy;
+}
+EXPORT_SYMBOL_GPL(devm_phy_get_by_of_node);
+
 /**
  * phy_create() - create a new phy
  * @dev: device that is creating the new phy
diff --git a/drivers/phy/qualcomm/Kconfig b/drivers/phy/qualcomm/Kconfig
index d910a5d1a1ac3a..d802e63ad74ae9 100644
--- a/drivers/phy/qualcomm/Kconfig
+++ b/drivers/phy/qualcomm/Kconfig
@@ -28,6 +28,21 @@ config PHY_QCOM_EDP
 	  Enable this driver to support the Qualcomm eDP PHY found in various
 	  Qualcomm chipsets.
 
+config PHY_QCOM_MIPI_CSI2
+	tristate "Qualcomm MIPI CSI2 PHY driver"
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on OF
+	depends on PM
+	depends on PM_OPP
+	depends on COMMON_CLK
+	select GENERIC_PHY
+	select GENERIC_PHY_MIPI_DPHY
+	help
+	  Enable this to support the MIPI CSI2 PHY driver found in various
+	  Qualcomm chipsets. This PHY is used to connect MIPI CSI2
+	  camera sensors to the CSI Decoder in the Qualcomm Camera Subsystem
+	  CAMSS.
+
 config PHY_QCOM_IPQ4019_USB
 	tristate "Qualcomm IPQ4019 USB PHY driver"
 	depends on OF && (ARCH_QCOM || COMPILE_TEST)
diff --git a/drivers/phy/qualcomm/Makefile b/drivers/phy/qualcomm/Makefile
index 8bf887d58ee4a6..4b5e6640a6faa0 100644
--- a/drivers/phy/qualcomm/Makefile
+++ b/drivers/phy/qualcomm/Makefile
@@ -6,6 +6,11 @@ obj-$(CONFIG_PHY_QCOM_IPQ4019_USB)	+= phy-qcom-ipq4019-usb.o
 obj-$(CONFIG_PHY_QCOM_IPQ806X_SATA)	+= phy-qcom-ipq806x-sata.o
 obj-$(CONFIG_PHY_QCOM_M31_USB)		+= phy-qcom-m31.o
 obj-$(CONFIG_PHY_QCOM_M31_EUSB)		+= phy-qcom-m31-eusb2.o
+
+phy-qcom-mipi-csi2-objs			+= phy-qcom-mipi-csi2-core.o \
+					   phy-qcom-mipi-csi2-3ph-dphy.o
+obj-$(CONFIG_PHY_QCOM_MIPI_CSI2)	+= phy-qcom-mipi-csi2.o
+
 obj-$(CONFIG_PHY_QCOM_PCIE2)		+= phy-qcom-pcie2.o
 
 obj-$(CONFIG_PHY_QCOM_QMP_COMBO)	+= phy-qcom-qmp-combo.o phy-qcom-qmp-usbc.o
diff --git a/drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c
new file mode 100644
index 00000000000000..966d79c98f9b21
--- /dev/null
+++ b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-3ph-dphy.c
@@ -0,0 +1,385 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Qualcomm MSM Camera Subsystem - CSIPHY Module 3phase v1.0
+ *
+ * Copyright (c) 2011-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2016-2026 Linaro Ltd.
+ */
+
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/time64.h>
+
+#include "phy-qcom-mipi-csi2.h"
+
+#define CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(offset, n)	((offset) + 0x4 * (n))
+#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL0_PHY_SW_RESET	BIT(0)
+#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL5_CLK_ENABLE	BIT(7)
+#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_COMMON_PWRDN_B	BIT(0)
+#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_SHOW_REV_ID	BIT(1)
+#define CSIPHY_3PH_CMN_CSI_COMMON_CTRL10_IRQ_CLEAR_CMD	BIT(0)
+#define CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(offset, n)	((offset) + 0xb0 + 0x4 * (n))
+
+#define CSIPHY_2PH_LN_CSI_2PHASE_CTRL9n(n)		((0x200 * (n)) + 0x24)
+
+/*
+ * 3 phase CSI has 19 common status regs with only 0-10 being used
+ * and 11-18 being reserved.
+ */
+#define CSI_COMMON_STATUS_NUM				11
+/*
+ * There are a number of common control registers
+ * The offset to clear the CSIPHY IRQ status starts @ 22
+ * So to clear CSI_COMMON_STATUS0 this is CSI_COMMON_CONTROL22, STATUS1 is
+ * CONTROL23 and so on
+ */
+#define CSI_CTRL_STATUS_INDEX				22
+
+/*
+ * There are 43 COMMON_CTRL registers with regs after # 33 being reserved
+ */
+#define CSI_CTRL_MAX					33
+
+#define CSIPHY_DEFAULT_PARAMS				0
+#define CSIPHY_SETTLE_CNT_LOWER_BYTE			2
+#define CSIPHY_SKEW_CAL					7
+
+/* 4nm 2PH v 2.1.2 2p5Gbps 4 lane DPHY mode */
+static const struct
+mipi_csi2phy_lane_regs lane_regs_x1e80100[] = {
+	/* Power up lanes 2ph mode */
+	{.reg_addr = 0x101c, .reg_data = 0x7a, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x1018, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+
+	{.reg_addr = 0x0094, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x00a0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0090, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0098, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0094, .reg_data = 0x07, .delay_us = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0030, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0000, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0038, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x002c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0034, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x001c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0014, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x003c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0004, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0020, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0008, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{.reg_addr = 0x0010, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0094, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x005c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0060, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0064, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL},
+
+	{.reg_addr = 0x0e94, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0ea0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e90, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e98, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e94, .reg_data = 0x07, .delay_us =  0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e30, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e28, .reg_data = 0x04, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e00, .reg_data = 0x80, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e0c, .reg_data = 0xff, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e38, .reg_data = 0x1f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e2c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e34, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e1c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e14, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e3c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e04, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e20, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0e08, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{.reg_addr = 0x0e10, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS},
+
+	{.reg_addr = 0x0494, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x04a0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0490, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0498, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0494, .reg_data = 0x07, .delay_us =  0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0430, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0400, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0438, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x042c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0434, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x041c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0414, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x043c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0404, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0420, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0408, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{.reg_addr = 0x0410, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0494, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x045c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0460, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0464, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL},
+
+	{.reg_addr = 0x0894, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x08a0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0890, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0898, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0894, .reg_data = 0x07, .delay_us =  0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0830, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0800, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0838, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x082c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0834, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x081c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0814, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x083c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0804, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0820, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0808, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{.reg_addr = 0x0810, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0894, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x085c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0860, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0864, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL},
+
+	{.reg_addr = 0x0c94, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0ca0, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c90, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c98, .reg_data = 0x08, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c94, .reg_data = 0x07, .delay_us =  0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c30, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c00, .reg_data = 0x8e, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c38, .reg_data = 0xfe, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c2c, .reg_data = 0x01, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c34, .reg_data = 0x0f, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c1c, .reg_data = 0x0a, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c14, .reg_data = 0x60, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c3c, .reg_data = 0xb8, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c04, .reg_data = 0x0c, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c20, .reg_data = 0x00, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c08, .reg_data = 0x10, .param_type = CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{.reg_addr = 0x0c10, .reg_data = 0x52, .param_type = CSIPHY_DEFAULT_PARAMS},
+	{.reg_addr = 0x0c94, .reg_data = 0xd7, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0c5c, .reg_data = 0x00, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0c60, .reg_data = 0xbd, .param_type = CSIPHY_SKEW_CAL},
+	{.reg_addr = 0x0c64, .reg_data = 0x7f, .param_type = CSIPHY_SKEW_CAL},
+};
+
+static inline const struct mipi_csi2phy_device_regs *
+csi2phy_dev_to_regs(struct mipi_csi2phy_device *csi2phy)
+{
+	return &csi2phy->soc_cfg->reg_info;
+}
+
+static void phy_qcom_mipi_csi2_hw_version_read(struct mipi_csi2phy_device *csi2phy)
+{
+	const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy);
+	u32 tmp;
+
+	writel(CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_SHOW_REV_ID, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 6));
+
+	tmp = readl_relaxed(csi2phy->base +
+			    CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 12));
+	csi2phy->hw_version = tmp;
+
+	tmp = readl_relaxed(csi2phy->base +
+			    CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 13));
+	csi2phy->hw_version |= (tmp << 8) & 0xFF00;
+
+	tmp = readl_relaxed(csi2phy->base +
+			    CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 14));
+	csi2phy->hw_version |= (tmp << 16) & 0xFF0000;
+
+	tmp = readl_relaxed(csi2phy->base +
+			    CSIPHY_3PH_CMN_CSI_COMMON_STATUSn(regs->common_regs_offset, 15));
+	csi2phy->hw_version |= (tmp << 24) & 0xFF000000;
+
+	dev_dbg_once(csi2phy->dev, "CSIPHY 3PH HW Version = 0x%08x\n", csi2phy->hw_version);
+}
+
+/*
+ * phy_qcom_mipi_csi2_reset - Perform software reset on CSIPHY module
+ * @phy_qcom_mipi_csi2: CSIPHY device
+ */
+static void phy_qcom_mipi_csi2_reset(struct mipi_csi2phy_device *csi2phy)
+{
+	const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy);
+
+	writel(CSIPHY_3PH_CMN_CSI_COMMON_CTRL0_PHY_SW_RESET,
+	       csi2phy->base + CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 0));
+	usleep_range(5000, 8000);
+	writel(0x0, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 0));
+}
+
+/*
+ * phy_qcom_mipi_csi2_settle_cnt_calc - Calculate settle count value
+ *
+ * Helper function to calculate settle count value. This is
+ * based on the CSI2 T_hs_settle parameter which in turn
+ * is calculated based on the CSI2 transmitter link frequency.
+ *
+ * Return settle count.
+ */
+static u8 phy_qcom_mipi_csi2_settle_cnt_calc(s64 link_freq, u32 timer_clk_rate)
+{
+	u32 t_hs_prepare_max_ps;
+	u32 timer_period_ps;
+	u32 t_hs_settle_ps;
+	u8 settle_cnt;
+	u32 ui_ps;
+
+	ui_ps = div64_u64(PSEC_PER_SEC, link_freq);
+	ui_ps /= 2;
+	t_hs_prepare_max_ps = 85000 + 6 * ui_ps;
+	t_hs_settle_ps = t_hs_prepare_max_ps;
+
+	timer_period_ps = div_u64(PSEC_PER_SEC, timer_clk_rate);
+
+	if ((t_hs_settle_ps / timer_period_ps) < 6)
+		return 0;
+
+	settle_cnt = t_hs_settle_ps / timer_period_ps - 6;
+
+	return settle_cnt;
+}
+
+static void
+phy_qcom_mipi_csi2_gen2_config_lanes(struct mipi_csi2phy_device *csi2phy,
+				     u8 settle_cnt)
+{
+	const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy);
+	const struct mipi_csi2phy_lane_regs *r = regs->init_seq;
+	int i, array_size = regs->lane_array_size;
+	u32 val;
+
+	for (i = 0; i < array_size; i++, r++) {
+		switch (r->param_type) {
+		case CSIPHY_SETTLE_CNT_LOWER_BYTE:
+			val = settle_cnt & 0xff;
+			break;
+		case CSIPHY_SKEW_CAL:
+			/* TODO: support application of skew from dt flag */
+			continue;
+		default:
+			val = r->reg_data;
+			break;
+		}
+		writel(val, csi2phy->base + r->reg_addr);
+		if (r->delay_us)
+			udelay(r->delay_us);
+	}
+}
+
+static int phy_qcom_mipi_csi2_lanes_enable(struct mipi_csi2phy_device *csi2phy,
+					   struct mipi_csi2phy_stream_cfg *cfg)
+{
+	const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy);
+	struct mipi_csi2phy_lanes_cfg *lane_cfg = &cfg->lane_cfg;
+	u8 settle_cnt;
+	u8 val;
+	int i;
+
+	if (cfg->link_freq <= 0)
+		return -EINVAL;
+
+	settle_cnt = phy_qcom_mipi_csi2_settle_cnt_calc(cfg->link_freq, csi2phy->timer_clk_rate);
+	if (!settle_cnt)
+		return -ENODEV;
+
+	/*
+	 * CSI_COMMON_CTRL5 is a physical lane power-up bitmap:
+	 * - Bits [0,2,4,6] → D-PHY data lanes(LN0, LN2, LN4, LN6)
+	 * - Bits [1,3,5] → C-PHY trio lanes(LN1, LN3, LN5)
+	 * - Bit [7] → D-PHY clock lane(LNCK) dedicated clock enable
+	 */
+	val = BIT(lane_cfg->clk.pos);
+	for (i = 0; i < cfg->num_data_lanes; i++)
+		val |= BIT(lane_cfg->data[i].pos * 2);
+
+	writel(val, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 5));
+
+	/* Lane configuration for polarity @ CSIPHY-base + CTRL9 */
+	for (i = 0; i < cfg->num_data_lanes; i++) {
+		if (lane_cfg->data[i].pol) {
+			u8 pos = lane_cfg->data[i].pos;
+
+			writel(BIT(2), csi2phy->base + CSIPHY_2PH_LN_CSI_2PHASE_CTRL9n(pos * 2));
+		}
+	}
+
+	if (lane_cfg->clk.pol)
+		writel(BIT(2), csi2phy->base + CSIPHY_2PH_LN_CSI_2PHASE_CTRL9n(lane_cfg->clk.pos));
+
+	val = CSIPHY_3PH_CMN_CSI_COMMON_CTRL6_COMMON_PWRDN_B;
+	writel(val, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 6));
+
+	val = 0x02;
+	writel(val, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 7));
+
+	val = 0x00;
+	writel(val, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 0));
+
+	phy_qcom_mipi_csi2_gen2_config_lanes(csi2phy, settle_cnt);
+
+	/* IRQ_MASK registers - disable all interrupts */
+	for (i = CSI_COMMON_STATUS_NUM; i < CSI_CTRL_STATUS_INDEX; i++) {
+		writel(0, csi2phy->base +
+		       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, i));
+	}
+
+	return 0;
+}
+
+static void
+phy_qcom_mipi_csi2_lanes_disable(struct mipi_csi2phy_device *csi2phy,
+				 struct mipi_csi2phy_stream_cfg *cfg)
+{
+	const struct mipi_csi2phy_device_regs *regs = csi2phy_dev_to_regs(csi2phy);
+
+	writel(0, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 5));
+
+	writel(0, csi2phy->base +
+	       CSIPHY_3PH_CMN_CSI_COMMON_CTRLn(regs->common_regs_offset, 6));
+}
+
+static const struct mipi_csi2phy_hw_ops phy_qcom_mipi_csi2_ops_3ph_1_0 = {
+	.hw_version_read = phy_qcom_mipi_csi2_hw_version_read,
+	.reset = phy_qcom_mipi_csi2_reset,
+	.lanes_enable = phy_qcom_mipi_csi2_lanes_enable,
+	.lanes_disable = phy_qcom_mipi_csi2_lanes_disable,
+};
+
+static const char * const x1e_clks[] = {
+	"core",
+	"timer",
+	"ahb"
+};
+
+static const char * const x1e_supplies[] = {
+	"vdda-0p9",
+	"vdda-1p2"
+};
+
+static struct mipi_csi2_genpd x1e_genpds[] = {
+	{ .name = "top", .scaled = false },
+	{ .name = "mmcx", .scaled = true },
+	{ .name = "mx", .scaled = true },
+};
+
+const struct mipi_csi2phy_soc_cfg mipi_csi2_dphy_4nm_x1e = {
+	.ops = &phy_qcom_mipi_csi2_ops_3ph_1_0,
+	.reg_info = {
+		.init_seq = lane_regs_x1e80100,
+		.lane_array_size = ARRAY_SIZE(lane_regs_x1e80100),
+		.common_regs_offset = 0x1000,
+	},
+	.supply_names = (const char **)x1e_supplies,
+	.num_supplies = ARRAY_SIZE(x1e_supplies),
+	.clk_names = (const char **)x1e_clks,
+	.num_clk = ARRAY_SIZE(x1e_clks),
+	.genpds = x1e_genpds,
+	.num_genpds = ARRAY_SIZE(x1e_genpds),
+};
diff --git a/drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c
new file mode 100644
index 00000000000000..c0a4fa20eaf0c8
--- /dev/null
+++ b/drivers/phy/qualcomm/phy-qcom-mipi-csi2-core.c
@@ -0,0 +1,463 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026, Linaro Ltd.
+ */
+
+#include <dt-bindings/media/video-interfaces.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pm_opp.h>
+#include <linux/phy/phy.h>
+#include <linux/phy/phy-mipi-dphy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+
+#include "phy-qcom-mipi-csi2.h"
+
+static int
+phy_qcom_mipi_csi2_set_clock_rates(struct mipi_csi2phy_device *csi2phy,
+				   s64 link_freq)
+{
+	struct device *dev = csi2phy->dev;
+	unsigned long opp_rate = link_freq / 4;
+	struct dev_pm_opp *opp;
+	long timer_rate;
+	int i, pstate;
+	int ret;
+
+	opp = dev_pm_opp_find_freq_ceil(dev, &opp_rate);
+	if (IS_ERR(opp)) {
+		dev_err(csi2phy->dev, "Couldn't find ceiling for %lld Hz\n",
+			link_freq);
+		return PTR_ERR(opp);
+	}
+
+	pstate = 0;
+	for (i = 0; i < csi2phy->pd_list->num_pds; i++) {
+		unsigned int perf;
+
+		if (!csi2phy->soc_cfg->genpds[i].scaled)
+			continue;
+
+		perf = dev_pm_opp_get_required_pstate(opp, pstate);
+		pstate += 1;
+
+		ret = dev_pm_genpd_set_performance_state(csi2phy->pd_list->pd_devs[i], perf);
+		if (ret) {
+			dev_err(csi2phy->dev, "Couldn't set perf state %u\n",
+				perf);
+			dev_pm_opp_put(opp);
+			goto unset_pstate;
+		}
+	}
+	dev_pm_opp_put(opp);
+
+	ret = dev_pm_opp_set_rate(dev, opp_rate);
+	if (ret) {
+		dev_err(csi2phy->dev, "dev_pm_opp_set_rate() fail\n");
+		goto unset_opp_rate;
+	}
+
+	timer_rate = clk_round_rate(csi2phy->timer_clk, link_freq / 4);
+	if (timer_rate <= 0) {
+		ret = -ENODEV;
+		goto unset_opp_rate;
+	}
+
+	ret = clk_set_rate(csi2phy->timer_clk, timer_rate);
+	if (ret)
+		goto unset_opp_rate;
+
+	csi2phy->timer_clk_rate = timer_rate;
+
+	return 0;
+
+unset_opp_rate:
+	dev_pm_opp_set_rate(dev, 0);
+
+unset_pstate:
+	while (i--) {
+		if (!csi2phy->soc_cfg->genpds[i].scaled)
+			continue;
+
+		dev_pm_genpd_set_performance_state(csi2phy->pd_list->pd_devs[i], 0);
+	}
+
+	return ret;
+}
+
+static int phy_qcom_mipi_csi2_configure(struct phy *phy,
+					union phy_configure_opts *opts)
+{
+	struct mipi_csi2phy_device *csi2phy = phy_get_drvdata(phy);
+	struct phy_configure_opts_mipi_dphy *dphy_cfg = &opts->mipi_dphy;
+	struct mipi_csi2phy_stream_cfg *stream_cfg = &csi2phy->stream_cfg;
+	int ret;
+
+	ret = phy_mipi_dphy_config_validate(dphy_cfg);
+	if (ret)
+		return ret;
+
+	if (dphy_cfg->lanes < 1 || dphy_cfg->lanes > CSI2_MAX_DATA_LANES)
+		return -EINVAL;
+
+	if (dphy_cfg->lanes != stream_cfg->num_data_lanes)
+		return -EINVAL;
+
+	stream_cfg->link_freq = dphy_cfg->hs_clk_rate;
+
+	return 0;
+}
+
+static int phy_qcom_mipi_csi2_set_mode(struct phy *phy,
+				       enum phy_mode mode, int submode)
+{
+	struct mipi_csi2phy_device *csi2phy = phy_get_drvdata(phy);
+
+	if (mode != PHY_MODE_MIPI_DPHY)
+		return -EOPNOTSUPP;
+
+	csi2phy->phy_mode = mode;
+	return 0;
+}
+
+static int phy_qcom_mipi_csi2_power_on(struct phy *phy)
+{
+	struct mipi_csi2phy_device *csi2phy = phy_get_drvdata(phy);
+	const struct mipi_csi2phy_hw_ops *ops = csi2phy->soc_cfg->ops;
+	int i, ret;
+
+	ret = regulator_bulk_enable(csi2phy->soc_cfg->num_supplies,
+				    csi2phy->supplies);
+	if (ret)
+		return ret;
+
+	ret = pm_runtime_resume_and_get(csi2phy->dev);
+	if (ret < 0)
+		goto disable_regulators;
+
+	ret = phy_qcom_mipi_csi2_set_clock_rates(csi2phy, csi2phy->stream_cfg.link_freq);
+	if (ret)
+		goto poweroff_phy;
+
+	ret = clk_bulk_prepare_enable(csi2phy->soc_cfg->num_clk,
+				      csi2phy->clks);
+	if (ret) {
+		dev_err(csi2phy->dev, "failed to enable clocks, %d\n", ret);
+		goto unset_rate;
+	}
+
+	ops->reset(csi2phy);
+
+	ops->hw_version_read(csi2phy);
+
+	ret = ops->lanes_enable(csi2phy, &csi2phy->stream_cfg);
+	if (ret)
+		goto unset_clocks;
+
+	return 0;
+
+unset_clocks:
+	clk_bulk_disable_unprepare(csi2phy->soc_cfg->num_clk,
+				   csi2phy->clks);
+
+unset_rate:
+	dev_pm_opp_set_rate(csi2phy->dev, 0);
+
+	for (i = 0; i < csi2phy->pd_list->num_pds; i++) {
+		if (!csi2phy->soc_cfg->genpds[i].scaled)
+			continue;
+
+		dev_pm_genpd_set_performance_state(csi2phy->pd_list->pd_devs[i], 0);
+	}
+
+poweroff_phy:
+	pm_runtime_put_sync(csi2phy->dev);
+
+disable_regulators:
+	regulator_bulk_disable(csi2phy->soc_cfg->num_supplies,
+			       csi2phy->supplies);
+
+	return ret;
+}
+
+static int phy_qcom_mipi_csi2_power_off(struct phy *phy)
+{
+	struct mipi_csi2phy_device *csi2phy = phy_get_drvdata(phy);
+	const struct mipi_csi2phy_hw_ops *ops = csi2phy->soc_cfg->ops;
+	int i;
+
+	ops->lanes_disable(csi2phy, &csi2phy->stream_cfg);
+
+	clk_bulk_disable_unprepare(csi2phy->soc_cfg->num_clk,
+				   csi2phy->clks);
+
+	dev_pm_opp_set_rate(csi2phy->dev, 0);
+
+	for (i = 0; i < csi2phy->pd_list->num_pds; i++) {
+		if (!csi2phy->soc_cfg->genpds[i].scaled)
+			continue;
+
+		dev_pm_genpd_set_performance_state(csi2phy->pd_list->pd_devs[i], 0);
+	}
+
+	pm_runtime_put_sync(csi2phy->dev);
+
+	regulator_bulk_disable(csi2phy->soc_cfg->num_supplies,
+			       csi2phy->supplies);
+
+	return 0;
+}
+
+static const struct phy_ops phy_qcom_mipi_csi2_ops = {
+	.configure	= phy_qcom_mipi_csi2_configure,
+	.set_mode	= phy_qcom_mipi_csi2_set_mode,
+	.power_on	= phy_qcom_mipi_csi2_power_on,
+	.power_off	= phy_qcom_mipi_csi2_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static int phy_qcom_mipi_csi2_attach_pm_domains(struct mipi_csi2phy_device *csi2phy)
+{
+	struct dev_pm_domain_attach_data pd_data = { 0 };
+	const char **pd_names;
+	int i;
+
+	pd_names = devm_kzalloc(csi2phy->dev,
+				sizeof(char *) * csi2phy->soc_cfg->num_genpds,
+				GFP_KERNEL);
+	if (!pd_names)
+		return -ENOMEM;
+
+	for (i = 0; i < csi2phy->soc_cfg->num_genpds; i++)
+		pd_names[i] = csi2phy->soc_cfg->genpds[i].name;
+
+	pd_data.pd_names = pd_names;
+	pd_data.num_pd_names = csi2phy->soc_cfg->num_genpds;
+
+	return devm_pm_domain_attach_list(csi2phy->dev, &pd_data,
+					  &csi2phy->pd_list);
+}
+
+static int phy_qcom_mipi_csi2_parse_routing(struct mipi_csi2phy_device *csi2phy)
+{
+	struct mipi_csi2phy_stream_cfg *stream_cfg = &csi2phy->stream_cfg;
+	u32 lane_polarities[CSI2_MAX_DATA_LANES + 1];
+	u32 data_lanes[CSI2_MAX_DATA_LANES];
+	struct device *dev = csi2phy->dev;
+	struct fwnode_handle *ep;
+	int num_polarities;
+	int num_data_lanes;
+	u32 bus_type;
+	int i, ret;
+
+	ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev), 0, 1, 0);
+	if (ep) {
+		fwnode_handle_put(ep);
+		dev_err(dev, "DPHY split mode is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev), 0, 0, 0);
+	if (!ep) {
+		dev_err(dev, "Missing port@0\n");
+		return -ENODEV;
+	}
+
+	/* bus-type is optional, D-PHY is the default */
+	bus_type = MEDIA_BUS_TYPE_CSI2_DPHY;
+	fwnode_property_read_u32(ep, "bus-type", &bus_type);
+
+	if (bus_type != MEDIA_BUS_TYPE_CSI2_DPHY) {
+		ret = -EOPNOTSUPP;
+		dev_err(dev, "Unsupported bus-type %u\n", bus_type);
+		goto out_put;
+	}
+
+	num_data_lanes = fwnode_property_count_u32(ep, "data-lanes");
+	if (num_data_lanes < 1 || num_data_lanes > CSI2_MAX_DATA_LANES) {
+		ret = -EINVAL;
+		dev_err(dev, "Invalid data-lanes count: %d\n", num_data_lanes);
+		goto out_put;
+	}
+	stream_cfg->num_data_lanes = num_data_lanes;
+
+	ret = fwnode_property_read_u32_array(ep, "data-lanes", data_lanes,
+					     stream_cfg->num_data_lanes);
+	if (ret) {
+		dev_err(dev, "Failed to read data-lanes: %d\n", ret);
+		goto out_put;
+	}
+
+	/* lane-polarities: optional, up to num_data_lanes + 1 entries */
+	memset(lane_polarities, 0x00, sizeof(lane_polarities));
+	num_polarities = fwnode_property_count_u32(ep, "lane-polarities");
+	if (num_polarities > 0) {
+		if (num_polarities != stream_cfg->num_data_lanes + 1) {
+			ret = -EINVAL;
+			dev_err(dev, "clock+data-lane %d/polarities %d mismatch\n",
+				stream_cfg->num_data_lanes + 1, num_polarities);
+			goto out_put;
+		}
+
+		ret = fwnode_property_read_u32_array(ep, "lane-polarities", lane_polarities,
+						     num_polarities);
+		if (ret) {
+			dev_err(dev, "Failed to read lane-polarities: %d\n", ret);
+			goto out_put;
+		}
+	}
+
+	csi2phy->stream_cfg.lane_cfg.clk.pos = CSI2_DEFAULT_CLK_LANE;
+	csi2phy->stream_cfg.lane_cfg.clk.pol = lane_polarities[0];
+
+	for (i = 0; i < csi2phy->stream_cfg.num_data_lanes; i++) {
+		if (data_lanes[i] < 1 || data_lanes[i] > CSI2_MAX_DATA_LANES) {
+			dev_err(dev, "Invalid lane %d\n", data_lanes[i]);
+			ret = -EINVAL;
+			goto out_put;
+		}
+
+		/* Convert data-lanes = <1 2 3 4> to bit positions */
+		csi2phy->stream_cfg.lane_cfg.data[i].pos = data_lanes[i] - 1;
+		csi2phy->stream_cfg.lane_cfg.data[i].pol = lane_polarities[i + 1];
+	}
+
+	ret = 0;
+
+out_put:
+	fwnode_handle_put(ep);
+
+	return ret;
+}
+
+static int phy_qcom_mipi_csi2_probe(struct platform_device *pdev)
+{
+	unsigned int i, num_clk, num_supplies;
+	struct mipi_csi2phy_device *csi2phy;
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	struct phy *generic_phy;
+	int ret;
+
+	csi2phy = devm_kzalloc(dev, sizeof(*csi2phy), GFP_KERNEL);
+	if (!csi2phy)
+		return -ENOMEM;
+
+	csi2phy->dev = dev;
+	dev_set_drvdata(dev, csi2phy);
+
+	csi2phy->soc_cfg = device_get_match_data(&pdev->dev);
+
+	if (!csi2phy->soc_cfg)
+		return -EINVAL;
+
+	num_clk = csi2phy->soc_cfg->num_clk;
+
+	ret = phy_qcom_mipi_csi2_parse_routing(csi2phy);
+	if (ret)
+		return ret;
+
+	ret = phy_qcom_mipi_csi2_attach_pm_domains(csi2phy);
+	if (ret < 0 || csi2phy->pd_list == NULL) {
+		if (ret == 0)
+			ret = -ENODEV;
+		return dev_err_probe(dev, ret, "Failed to attach power-domain list\n");
+	}
+
+	ret = devm_pm_runtime_enable(dev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to enable pm runtime\n");
+
+	ret = devm_clk_bulk_get_all(dev, &csi2phy->clks);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get clocks\n");
+
+	if (num_clk != ret) {
+		return dev_err_probe(dev, -ENODEV, "clock count %d expected %d\n",
+				     ret, num_clk);
+	}
+
+	for (i = 0; i < num_clk; i++) {
+		if (!csi2phy->clks[i].id)
+			return dev_err_probe(dev, -EINVAL, "Missing clock-names\n");
+
+		if (!strcmp(csi2phy->clks[i].id, "timer")) {
+			csi2phy->timer_clk = csi2phy->clks[i].clk;
+			break;
+		}
+	}
+	if (!csi2phy->timer_clk)
+		return dev_err_probe(dev, -ENODEV, "no timer clock\n");
+
+	ret = devm_pm_opp_set_clkname(dev, "core");
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to set opp clkname\n");
+
+	ret = devm_pm_opp_of_add_table(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "invalid OPP table in device tree\n");
+
+	num_supplies = csi2phy->soc_cfg->num_supplies;
+	csi2phy->supplies = devm_kzalloc(dev, sizeof(*csi2phy->supplies) * num_supplies,
+					 GFP_KERNEL);
+	if (!csi2phy->supplies)
+		return -ENOMEM;
+
+	for (i = 0; i < num_supplies; i++)
+		csi2phy->supplies[i].supply = csi2phy->soc_cfg->supply_names[i];
+
+	ret = devm_regulator_bulk_get(dev, num_supplies, csi2phy->supplies);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to get regulator supplies\n");
+
+	csi2phy->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(csi2phy->base))
+		return PTR_ERR(csi2phy->base);
+
+	generic_phy = devm_phy_create(dev, NULL, &phy_qcom_mipi_csi2_ops);
+	if (IS_ERR(generic_phy)) {
+		ret = PTR_ERR(generic_phy);
+		return dev_err_probe(dev, ret, "failed to create phy\n");
+	}
+	csi2phy->phy = generic_phy;
+
+	phy_set_drvdata(generic_phy, csi2phy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	if (!IS_ERR(phy_provider))
+		dev_dbg(dev, "Registered MIPI CSI2 PHY device\n");
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static const struct of_device_id phy_qcom_mipi_csi2_of_match_table[] = {
+	{ .compatible	= "qcom,x1e80100-csi2-phy", .data = &mipi_csi2_dphy_4nm_x1e },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, phy_qcom_mipi_csi2_of_match_table);
+
+static struct platform_driver phy_qcom_mipi_csi2_driver = {
+	.probe		= phy_qcom_mipi_csi2_probe,
+	.driver = {
+		.name	= "qcom-mipi-csi2-phy",
+		.of_match_table = phy_qcom_mipi_csi2_of_match_table,
+	},
+};
+
+module_platform_driver(phy_qcom_mipi_csi2_driver);
+
+MODULE_DESCRIPTION("Qualcomm MIPI CSI2 PHY driver");
+MODULE_AUTHOR("Bryan O'Donoghue <bod@kernel.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/qualcomm/phy-qcom-mipi-csi2.h b/drivers/phy/qualcomm/phy-qcom-mipi-csi2.h
new file mode 100644
index 00000000000000..7e55ae00737047
--- /dev/null
+++ b/drivers/phy/qualcomm/phy-qcom-mipi-csi2.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *
+ * Qualcomm MIPI CSI2 CPHY/DPHY driver
+ *
+ * Copyright (C) 2025 Linaro Ltd.
+ */
+#ifndef __PHY_QCOM_MIPI_CSI2_H__
+#define __PHY_QCOM_MIPI_CSI2_H__
+
+#include <linux/phy/phy.h>
+
+#define CSI2_MAX_DATA_LANES	4
+#define CSI2_DEFAULT_CLK_LANE	7
+
+struct mipi_csi2phy_lane {
+	u8 pos;
+	u8 pol;
+};
+
+struct mipi_csi2phy_lanes_cfg {
+	struct mipi_csi2phy_lane data[CSI2_MAX_DATA_LANES];
+	struct mipi_csi2phy_lane clk;
+};
+
+struct mipi_csi2phy_stream_cfg {
+	s64 link_freq;
+	u8 num_data_lanes;
+	struct mipi_csi2phy_lanes_cfg lane_cfg;
+};
+
+struct mipi_csi2phy_device;
+
+struct mipi_csi2phy_hw_ops {
+	void (*hw_version_read)(struct mipi_csi2phy_device *csi2phy_dev);
+	void (*reset)(struct mipi_csi2phy_device *csi2phy_dev);
+	int (*lanes_enable)(struct mipi_csi2phy_device *csi2phy_dev,
+			    struct mipi_csi2phy_stream_cfg *cfg);
+	void (*lanes_disable)(struct mipi_csi2phy_device *csi2phy_dev,
+			      struct mipi_csi2phy_stream_cfg *cfg);
+};
+
+struct mipi_csi2phy_lane_regs {
+	const s32 reg_addr;
+	const s32 reg_data;
+	const u32 delay_us;
+	const u32 param_type;
+};
+
+struct mipi_csi2phy_device_regs {
+	const struct mipi_csi2phy_lane_regs *init_seq;
+	const int lane_array_size;
+	const u32 common_regs_offset;
+};
+
+struct mipi_csi2_genpd {
+	const char *name;
+	bool scaled;
+};
+
+struct mipi_csi2phy_soc_cfg {
+	const struct mipi_csi2phy_hw_ops *ops;
+	const struct mipi_csi2phy_device_regs reg_info;
+
+	const char ** const supply_names;
+	const unsigned int num_supplies;
+
+	const char ** const clk_names;
+	const unsigned int num_clk;
+
+	const struct mipi_csi2_genpd *genpds;
+	const unsigned int num_genpds;
+};
+
+struct mipi_csi2phy_device {
+	struct device *dev;
+	u8 phy_mode;
+
+	struct phy *phy;
+	void __iomem *base;
+
+	struct clk_bulk_data *clks;
+	struct clk *timer_clk;
+	u32 timer_clk_rate;
+
+	struct regulator_bulk_data *supplies;
+	struct dev_pm_domain_list *pd_list;
+
+	const struct mipi_csi2phy_soc_cfg *soc_cfg;
+	struct mipi_csi2phy_stream_cfg stream_cfg;
+
+	u32 hw_version;
+};
+
+extern const struct mipi_csi2phy_soc_cfg mipi_csi2_dphy_4nm_x1e;
+
+#endif /* __PHY_QCOM_MIPI_CSI2_H__ */
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
index c39ced168d035e..a4f130fc33e399 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
@@ -22,6 +22,7 @@
 #include <linux/usb/typec.h>
 #include <linux/usb/typec_dp.h>
 #include <linux/usb/typec_mux.h>
+#include <linux/usb/typec_tbt.h>
 
 #include <drm/bridge/aux-bridge.h>
 
@@ -66,10 +67,14 @@
 #define SW_USB3PHY_RESET			BIT(2)
 /* mux to select USB3 PHY reset control, 0:HW control, 1: software reset */
 #define SW_USB3PHY_RESET_MUX			BIT(3)
+#define SW_USB4PHY_RESET			BIT(4)
+#define SW_USB4PHY_RESET_MUX			BIT(5)
 
 /* QPHY_V3_DP_COM_PHY_MODE_CTRL register bits */
 #define USB3_MODE				BIT(0) /* enables USB3 mode */
 #define DP_MODE					BIT(1) /* enables DP mode */
+#define USB4_MODE				BIT(2) /* mutually exclusive with the above */
+#define DP_TUNNELING_CLOCK_GEN_EN		BIT(3)
 
 /* QPHY_V3_DP_COM_TYPEC_CTRL register bits */
 #define SW_PORTSELECT_VAL			BIT(0)
@@ -82,6 +87,8 @@ enum qmpphy_mode {
 	QMPPHY_MODE_USB3DP = 0,
 	QMPPHY_MODE_DP_ONLY,
 	QMPPHY_MODE_USB3_ONLY,
+	/* USB4 QMPPHY mode refers to both USB4 and TBT3 */
+	QMPPHY_MODE_USB4,
 };
 
 /* set of registers with offsets different per-PHY */
@@ -94,6 +101,7 @@ enum qphy_reg_layout {
 	QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR,
 	QPHY_PCS_POWER_DOWN_CONTROL,
 	QPHY_PCS_CLAMP_ENABLE,
+	QPHY_PCS_USB4_CLAMP_ENABLE,
 
 	QPHY_COM_RESETSM_CNTRL,
 	QPHY_COM_C_READY_STATUS,
@@ -2489,6 +2497,8 @@ struct qmp_combo_offsets {
 	u16 dp_txb;
 	u16 dp_dp_phy;
 	u16 aon_toggle;
+	u16 usb4_serdes;
+	u16 usb4_pcs;
 };
 
 struct qmp_phy_cfg {
@@ -2536,6 +2546,18 @@ struct qmp_phy_cfg {
 	int (*calibrate_dp_phy)(struct qmp_combo *qmp);
 	void (*dp_aux_init)(struct qmp_combo *qmp);
 
+	/* USB4 specifics */
+	const struct qmp_phy_init_tbl *usb4_serdes_tbl;
+	int usb4_serdes_tbl_num;
+	const struct qmp_phy_init_tbl *usb4_serdes_tb3_ovrd_tbl;
+	int usb4_serdes_tb3_ovrd_num;
+	const struct qmp_phy_init_tbl *usb4_tx_tbl;
+	int usb4_tx_tbl_num;
+	const struct qmp_phy_init_tbl *usb4_rx_tbl;
+	int usb4_rx_tbl_num;
+	const struct qmp_phy_init_tbl *usb4_pcs_tbl;
+	int usb4_pcs_tbl_num;
+
 	/* resets to be requested */
 	const char * const *reset_list;
 	int num_resets;
@@ -2578,8 +2600,12 @@ struct qmp_combo {
 	void __iomem *dp_tx2;
 	void __iomem *dp_dp_phy;
 
+	void __iomem *usb4_serdes;
+	void __iomem *usb4_pcs;
+
 	struct clk *pipe_clk;
 	struct clk_bulk_data *clks;
+	struct clk *p2rr2p_pipe_clk;
 	int num_clks;
 	struct reset_control_bulk_data *resets;
 	struct regulator_bulk_data *vregs;
@@ -2598,6 +2624,10 @@ struct qmp_combo {
 	unsigned int dp_init_count;
 	bool dp_powered_on;
 
+	struct phy *usb4_phy;
+	enum tbt_phy_submode usb4_phy_submode;
+	unsigned int usb4_init_count;
+
 	struct clk_fixed_rate pipe_clk_fixed;
 	struct clk_hw dp_link_hw;
 	struct clk_hw dp_pixel_hw;
@@ -4006,6 +4036,9 @@ static int qmp_combo_dp_calibrate(struct phy *phy)
 static int qmp_combo_com_init(struct qmp_combo *qmp, bool force)
 {
 	const struct qmp_phy_cfg *cfg = qmp->cfg;
+	u32 dp_reset_val = SW_DPPHY_RESET_MUX | SW_DPPHY_RESET;
+	void __iomem *pcs_aon = qmp->pcs_aon;
+	void __iomem *pcs = qmp->pcs;
 	void __iomem *com = qmp->com;
 	void __iomem *aon_toggle;
 	int ret;
@@ -4019,6 +4052,16 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force)
 	else
 		aon_toggle = qmp->pcs_aon;
 
+	if (qmp->qmpphy_mode == QMPPHY_MODE_USB4) {
+		pcs = qmp->usb4_pcs;
+
+		qphy_setbits(pcs_aon, cfg->regs[QPHY_PCS_USB4_CLAMP_ENABLE], CLAMP_EN);
+
+		/* Do not disturb the DP PLL in case there's an active DP tunnel */
+		if (readl(com + QPHY_V3_DP_COM_RESET_OVRD_CTRL) & DP_TUNNELING_CLOCK_GEN_EN)
+			dp_reset_val &= ~SW_DPPHY_RESET_MUX;
+	}
+
 	ret = regulator_bulk_enable(cfg->num_vregs, qmp->vregs);
 	if (ret) {
 		dev_err(qmp->dev, "failed to enable regulators, err=%d\n", ret);
@@ -4052,8 +4095,9 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force)
 
 	/* override hardware control for reset of qmp phy */
 	qphy_setbits(com, QPHY_V3_DP_COM_RESET_OVRD_CTRL,
-			SW_DPPHY_RESET_MUX | SW_DPPHY_RESET |
-			SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET);
+			dp_reset_val |
+			SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET |
+			SW_USB4PHY_RESET_MUX | SW_USB4PHY_RESET);
 
 	/* override hardware control for reset of qmp phy */
 	if (aon_toggle && cfg->regs[QPHY_AON_TOGGLE_ENABLE]) {
@@ -4069,6 +4113,10 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force)
 	if (cfg->invert_cc_polarity)
 		val |= INVERT_CC_POLARITY;
 
+	/*
+	 * Note that in USB4 mode, the router controls pin assignments instead
+	 * and the values of the PORTSELECT registers are ignored.
+	 */
 	writel(val, com + QPHY_V3_DP_COM_TYPEC_CTRL);
 
 	switch (qmp->qmpphy_mode) {
@@ -4096,12 +4144,24 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force)
 		qphy_clrbits(com, QPHY_V3_DP_COM_RESET_OVRD_CTRL,
 				SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET);
 		break;
+	case QMPPHY_MODE_USB4:
+		writel(USB4_MODE, com + QPHY_V3_DP_COM_PHY_MODE_CTRL);
+
+		/* bring both QMP USB and QMP DP PHYs PCS block out of reset */
+		/*
+		 * TODO: disable the DP PLL to save power if there are no
+		 * active DP tunnels after router setup has concluded
+		 */
+		qphy_clrbits(com, QPHY_V3_DP_COM_RESET_OVRD_CTRL,
+			     SW_USB4PHY_RESET_MUX | SW_USB4PHY_RESET |
+			     SW_DPPHY_RESET_MUX | SW_DPPHY_RESET);
+		break;
 	}
 
 	qphy_clrbits(com, QPHY_V3_DP_COM_SWI_CTRL, 0x03);
 	qphy_clrbits(com, QPHY_V3_DP_COM_SW_RESET, SW_RESET);
 
-	qphy_setbits(qmp->pcs, cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL],
+	qphy_setbits(pcs, cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL],
 			SW_PWRDN);
 
 	return 0;
@@ -4219,6 +4279,10 @@ static int qmp_combo_usb_power_on(struct phy *phy)
 	struct qmp_combo *qmp = phy_get_drvdata(phy);
 	const struct qmp_phy_cfg *cfg = qmp->cfg;
 	void __iomem *serdes = qmp->serdes;
+	const struct qmp_phy_init_tbl *pcs_tbl;
+	const struct qmp_phy_init_tbl *rx_tbl;
+	const struct qmp_phy_init_tbl *serdes_tbl;
+	const struct qmp_phy_init_tbl *tx_tbl;
 	void __iomem *tx = qmp->tx;
 	void __iomem *rx = qmp->rx;
 	void __iomem *tx2 = qmp->tx2;
@@ -4226,23 +4290,54 @@ static int qmp_combo_usb_power_on(struct phy *phy)
 	void __iomem *pcs = qmp->pcs;
 	void __iomem *pcs_usb = qmp->pcs_usb;
 	void __iomem *status;
+	int pcs_tbl_num, rx_tbl_num, serdes_tbl_num, tx_tbl_num;
 	unsigned int val;
 	int ret;
 
-	qmp_configure(qmp->dev, serdes, cfg->serdes_tbl, cfg->serdes_tbl_num);
+	if (qmp->qmpphy_mode == QMPPHY_MODE_USB4) {
+		pcs = qmp->usb4_pcs;
+		serdes = qmp->usb4_serdes;
+		pcs_tbl = cfg->usb4_pcs_tbl;
+		pcs_tbl_num = cfg->usb4_pcs_tbl_num;
+
+		serdes_tbl = cfg->usb4_serdes_tbl;
+		serdes_tbl_num = cfg->usb4_serdes_tbl_num;
+
+		rx_tbl = cfg->usb4_rx_tbl;
+		rx_tbl_num = cfg->usb4_rx_tbl_num;
+
+		tx_tbl = cfg->usb4_tx_tbl;
+		tx_tbl_num = cfg->usb4_tx_tbl_num;
+	} else {
+		pcs_tbl = cfg->pcs_tbl;
+		pcs_tbl_num = cfg->pcs_tbl_num;
+
+		serdes_tbl = cfg->serdes_tbl;
+		serdes_tbl_num = cfg->serdes_tbl_num;
+
+		rx_tbl = cfg->rx_tbl;
+		rx_tbl_num = cfg->rx_tbl_num;
+
+		tx_tbl = cfg->tx_tbl;
+		tx_tbl_num = cfg->tx_tbl_num;
+	}
+
+	qmp_configure(qmp->dev, serdes, serdes_tbl, serdes_tbl_num);
 
 	/* Tx, Rx, and PCS configurations */
-	qmp_configure_lane(qmp->dev, tx, cfg->tx_tbl, cfg->tx_tbl_num, 1);
-	qmp_configure_lane(qmp->dev, tx2, cfg->tx_tbl, cfg->tx_tbl_num, 2);
+	qmp_configure_lane(qmp->dev, tx, tx_tbl, tx_tbl_num, 1);
+	qmp_configure_lane(qmp->dev, tx2, tx_tbl, tx_tbl_num, 2);
 
-	qmp_configure_lane(qmp->dev, rx, cfg->rx_tbl, cfg->rx_tbl_num, 1);
-	qmp_configure_lane(qmp->dev, rx2, cfg->rx_tbl, cfg->rx_tbl_num, 2);
+	qmp_configure_lane(qmp->dev, rx, rx_tbl, rx_tbl_num, 1);
+	qmp_configure_lane(qmp->dev, rx2, rx_tbl, rx_tbl_num, 2);
 
-	qmp_configure(qmp->dev, pcs, cfg->pcs_tbl, cfg->pcs_tbl_num);
+	qmp_configure(qmp->dev, pcs, pcs_tbl, pcs_tbl_num);
 	qmp_configure(qmp->dev, qmp->pcs_misc, cfg->pcs_misc_tbl, cfg->pcs_misc_tbl_num);
 
-
-	if (pcs_usb)
+	if (qmp->qmpphy_mode == QMPPHY_MODE_USB4 && qmp->usb4_phy_submode == PHY_SUBMODE_TBT3)
+		qmp_configure(qmp->dev, serdes, cfg->usb4_serdes_tb3_ovrd_tbl,
+			      cfg->usb4_serdes_tb3_ovrd_num);
+	else if (qmp->qmpphy_mode != QMPPHY_MODE_USB4 && pcs_usb)
 		qmp_configure(qmp->dev, pcs_usb, cfg->pcs_usb_tbl,
 			      cfg->pcs_usb_tbl_num);
 
@@ -4275,16 +4370,20 @@ static int qmp_combo_usb_power_off(struct phy *phy)
 {
 	struct qmp_combo *qmp = phy_get_drvdata(phy);
 	const struct qmp_phy_cfg *cfg = qmp->cfg;
+	void __iomem *pcs = qmp->pcs;
+
+	if (qmp->usb4_init_count)
+		pcs = qmp->usb4_pcs;
 
 	/* PHY reset */
-	qphy_setbits(qmp->pcs, cfg->regs[QPHY_SW_RESET], SW_RESET);
+	qphy_setbits(pcs, cfg->regs[QPHY_SW_RESET], SW_RESET);
 
 	/* stop SerDes and Phy-Coding-Sublayer */
-	qphy_clrbits(qmp->pcs, cfg->regs[QPHY_START_CTRL],
+	qphy_clrbits(pcs, cfg->regs[QPHY_START_CTRL],
 			SERDES_START | PCS_START);
 
 	/* Put PHY into POWER DOWN state: active low */
-	qphy_clrbits(qmp->pcs, cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL],
+	qphy_clrbits(pcs, cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL],
 			SW_PWRDN);
 
 	return 0;
@@ -4295,21 +4394,27 @@ static int qmp_combo_usb_init(struct phy *phy)
 	struct qmp_combo *qmp = phy_get_drvdata(phy);
 	int ret;
 
-	mutex_lock(&qmp->phy_mutex);
+	guard(mutex)(&qmp->phy_mutex);
+
+	/* USB4 mode takes precedence, do not reprogram the PHY in that case */
+	if (qmp->usb4_init_count) {
+		qmp->init_count++;
+		qmp->usb_init_count++;
+		return 0;
+	}
+
 	ret = qmp_combo_com_init(qmp, false);
 	if (ret)
-		goto out_unlock;
+		return ret;
 
 	ret = qmp_combo_usb_power_on(phy);
 	if (ret) {
 		qmp_combo_com_exit(qmp, false);
-		goto out_unlock;
+		return ret;
 	}
 
 	qmp->usb_init_count++;
 
-out_unlock:
-	mutex_unlock(&qmp->phy_mutex);
 	return ret;
 }
 
@@ -4318,20 +4423,26 @@ static int qmp_combo_usb_exit(struct phy *phy)
 	struct qmp_combo *qmp = phy_get_drvdata(phy);
 	int ret;
 
-	mutex_lock(&qmp->phy_mutex);
+	guard(mutex)(&qmp->phy_mutex);
+
+	/* USB4 mode takes precedence, do not reprogram the PHY in that case */
+	if (qmp->usb4_init_count) {
+		qmp->init_count--;
+		qmp->usb_init_count--;
+		return 0;
+	}
+
 	ret = qmp_combo_usb_power_off(phy);
 	if (ret)
-		goto out_unlock;
+		return ret;
 
 	ret = qmp_combo_com_exit(qmp, false);
 	if (ret)
-		goto out_unlock;
+		return ret;
 
 	qmp->usb_init_count--;
 
-out_unlock:
-	mutex_unlock(&qmp->phy_mutex);
-	return ret;
+	return 0;
 }
 
 static int qmp_combo_usb_set_mode(struct phy *phy, enum phy_mode mode, int submode)
@@ -4343,6 +4454,125 @@ static int qmp_combo_usb_set_mode(struct phy *phy, enum phy_mode mode, int submo
 	return 0;
 }
 
+static int qmp_combo_reconfigure_phy(struct qmp_combo *qmp, enum qmpphy_mode new_mode)
+{
+	dev_dbg(qmp->dev, "qmp_combo_reconfigure_phy: switching from qmpphy mode %d to %d\n",
+		qmp->qmpphy_mode, new_mode);
+
+	if (qmp->usb_init_count || qmp->usb4_init_count)
+		qmp_combo_usb_power_off(qmp->usb_phy);
+
+	if (qmp->dp_init_count)
+		writel(DP_PHY_PD_CTL_PSR_PWRDN, qmp->dp_dp_phy + QSERDES_DP_PHY_PD_CTL);
+
+	qmp_combo_com_exit(qmp, true);
+
+	qmp->qmpphy_mode = new_mode;
+
+	/* Now everything's powered down, power up the right PHYs */
+	qmp_combo_com_init(qmp, true);
+
+	if ((qmp->usb_init_count || qmp->usb4_init_count) &&
+	    new_mode != QMPPHY_MODE_DP_ONLY)
+		qmp_combo_usb_power_on(qmp->usb_phy);
+
+	if ((new_mode == QMPPHY_MODE_USB3DP || new_mode == QMPPHY_MODE_DP_ONLY) &&
+	    qmp->dp_init_count)
+		qmp->cfg->dp_aux_init(qmp);
+
+	return 0;
+}
+
+static int qmp_combo_usb4_init(struct phy *phy)
+{
+	struct qmp_combo *qmp = phy_get_drvdata(phy);
+	int ret;
+
+	guard(mutex)(&qmp->phy_mutex);
+
+	if (!qmp->p2rr2p_pipe_clk) {
+		dev_err(qmp->dev, "missing p2rr2p_pipe clock handle. Update your Device Tree.\n");
+		return -EINVAL;
+	}
+
+	ret = clk_prepare_enable(qmp->p2rr2p_pipe_clk);
+	if (ret) {
+		dev_err(qmp->dev, "p2rr2p_pipe enable failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = qmp_combo_com_init(qmp, false);
+	if (ret) {
+		clk_disable_unprepare(qmp->p2rr2p_pipe_clk);
+		return ret;
+	}
+
+	/* USB4 mode takes precedence to USB3(+DP), force reconfigure the PHY */
+	ret = qmp_combo_reconfigure_phy(qmp, QMPPHY_MODE_USB4);
+	if (ret) {
+		clk_disable_unprepare(qmp->p2rr2p_pipe_clk);
+		qmp_combo_com_exit(qmp, false);
+		return ret;
+	}
+
+	ret = qmp_combo_usb_power_on(phy);
+	if (ret) {
+		clk_disable_unprepare(qmp->p2rr2p_pipe_clk);
+		qmp_combo_com_exit(qmp, false);
+		return ret;
+	}
+
+	/*
+	 * Due to the SoC design, the PHY only has a single valid consumer and
+	 * preventing it from having sole ownership of the PHY's power state
+	 * makes suspending/resuming the router impossible.
+	 */
+	WARN_ON(qmp->usb4_init_count++);
+
+	return 0;
+}
+
+static int qmp_combo_usb4_exit(struct phy *phy)
+{
+	struct qmp_combo *qmp = phy_get_drvdata(phy);
+	int ret;
+
+	guard(mutex)(&qmp->phy_mutex);
+
+	ret = qmp_combo_usb_power_off(qmp->usb_phy);
+	if (ret)
+		return ret;
+
+	ret = qmp_combo_com_exit(qmp, false);
+	if (ret)
+		return ret;
+
+	/*
+	 * Mark the USB4 PHY uninitialized and wait for a mux_set event to determine the correct
+	 * setting. This will always be possible because USB4 requires Type-C.
+	 */
+	qmp->usb4_init_count--;
+
+	clk_disable_unprepare(qmp->p2rr2p_pipe_clk);
+
+	return 0;
+}
+
+static int qmp_combo_usb4_set_mode(struct phy *phy, enum phy_mode mode, int submode)
+{
+	struct qmp_combo *qmp = phy_get_drvdata(phy);
+
+	if (mode != PHY_MODE_TBT)
+		return -EINVAL;
+
+	if (submode != PHY_SUBMODE_USB4 && submode != PHY_SUBMODE_TBT3)
+		return -EINVAL;
+
+	qmp->usb4_phy_submode = submode;
+
+	return 0;
+}
+
 static const struct phy_ops qmp_combo_usb_phy_ops = {
 	.init		= qmp_combo_usb_init,
 	.exit		= qmp_combo_usb_exit,
@@ -4350,6 +4580,13 @@ static const struct phy_ops qmp_combo_usb_phy_ops = {
 	.owner		= THIS_MODULE,
 };
 
+static const struct phy_ops qmp_combo_usb4_phy_ops = {
+	.init		= qmp_combo_usb4_init,
+	.exit		= qmp_combo_usb4_exit,
+	.set_mode	= qmp_combo_usb4_set_mode,
+	.owner		= THIS_MODULE,
+};
+
 static const struct phy_ops qmp_combo_dp_phy_ops = {
 	.init		= qmp_combo_dp_init,
 	.configure	= qmp_combo_dp_configure,
@@ -4389,9 +4626,12 @@ static void qmp_combo_enable_autonomous_mode(struct qmp_combo *qmp)
 	 * Enable i/o clamp_n for autonomous mode
 	 * V6 and later versions use pcs aon clamp register
 	 */
-	if (pcs_aon)
-		qphy_clrbits(pcs_aon, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN);
-	else if (pcs_misc)
+	if (pcs_aon) {
+		if (qmp->qmpphy_mode == QMPPHY_MODE_USB4)
+			qphy_clrbits(pcs_aon, cfg->regs[QPHY_PCS_USB4_CLAMP_ENABLE], CLAMP_EN);
+		else
+			qphy_clrbits(pcs_aon, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN);
+	} else if (pcs_misc)
 		qphy_clrbits(pcs_misc, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN);
 }
 
@@ -4403,9 +4643,12 @@ static void qmp_combo_disable_autonomous_mode(struct qmp_combo *qmp)
 	void __iomem *pcs_aon = qmp->pcs_aon;
 
 	/* Disable i/o clamp_n on resume for normal mode */
-	if (pcs_aon)
-		qphy_setbits(pcs_aon, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN);
-	else if (pcs_misc)
+	if (pcs_aon) {
+		if (qmp->qmpphy_mode == QMPPHY_MODE_USB4)
+			qphy_setbits(pcs_aon, cfg->regs[QPHY_PCS_USB4_CLAMP_ENABLE], CLAMP_EN);
+		else
+			qphy_setbits(pcs_aon, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN);
+	} else if (pcs_misc)
 		qphy_setbits(pcs_misc, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN);
 
 	qphy_clrbits(pcs_usb, cfg->regs[QPHY_PCS_AUTONOMOUS_MODE_CTRL],
@@ -4779,24 +5022,29 @@ static int qmp_combo_typec_switch_set(struct typec_switch_dev *sw,
 	struct qmp_combo *qmp = typec_switch_get_drvdata(sw);
 	const struct qmp_phy_cfg *cfg = qmp->cfg;
 
+	if (qmp->qmpphy_mode == QMPPHY_MODE_USB4) {
+		/* QMPPHY has no orientation handling in USB4 mode, don't cache the setting */
+		qmp->orientation = TYPEC_ORIENTATION_NONE;
+		return 0;
+	}
+
 	if (orientation == qmp->orientation || orientation == TYPEC_ORIENTATION_NONE)
 		return 0;
 
-	mutex_lock(&qmp->phy_mutex);
+	guard(mutex)(&qmp->phy_mutex);
 	qmp->orientation = orientation;
 
 	if (qmp->init_count) {
-		if (qmp->usb_init_count)
+		if (qmp->usb_init_count || qmp->usb4_init_count)
 			qmp_combo_usb_power_off(qmp->usb_phy);
 		qmp_combo_com_exit(qmp, true);
 
 		qmp_combo_com_init(qmp, true);
-		if (qmp->usb_init_count)
+		if (qmp->usb_init_count || qmp->usb4_init_count)
 			qmp_combo_usb_power_on(qmp->usb_phy);
 		if (qmp->dp_init_count)
 			cfg->dp_aux_init(qmp);
 	}
-	mutex_unlock(&qmp->phy_mutex);
 
 	return 0;
 }
@@ -4804,7 +5052,6 @@ static int qmp_combo_typec_switch_set(struct typec_switch_dev *sw,
 static int qmp_combo_typec_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state)
 {
 	struct qmp_combo *qmp = typec_mux_get_drvdata(mux);
-	const struct qmp_phy_cfg *cfg = qmp->cfg;
 	enum qmpphy_mode new_mode;
 	unsigned int svid;
 
@@ -4815,6 +5062,29 @@ static int qmp_combo_typec_mux_set(struct typec_mux_dev *mux, struct typec_mux_s
 	else
 		svid = 0;
 
+	/*
+	 * The USB4 router driver must excercise fine-grained control over the timing of
+	 * USB4 QMPPHY mode entry/exit, which is difficult to otherwise ensure within Linux's
+	 * Type-C framework if the PHY acts as a self-decisive mode mux in parallel.
+	 *
+	 * Keep the hardware in QMPPHY_MODE_USB4 at all times between .init and .exit of the
+	 * (Linux) USB4 PHY and wait for the router driver to turn it off at its discretion.
+	 * Once that happens, fall back to the usual USB3/DP/Combo mode logic.
+	 *
+	 * After the QMPPHY has been turned off through phy_exit(usb4_phy), the next mux_set
+	 * will initialize it in the right mode.
+	 */
+	if (qmp->usb4_init_count)
+		return 0;
+
+	/*
+	 * Explicitly ignore TBT/USB4 mode requests that may come if the USB4 PHY hasn't been
+	 * initialized, either due to the USB4 drivers being disabled or due to this PHY instance
+	 * lacking USB4 support.
+	 */
+	if (svid == USB_TYPEC_TBT_SID || (!state->alt && state->mode == TYPEC_MODE_USB4))
+		return 0;
+
 	if (svid == USB_TYPEC_DP_SID) {
 		switch (state->mode) {
 		/* DP Only */
@@ -4847,41 +5117,11 @@ static int qmp_combo_typec_mux_set(struct typec_mux_dev *mux, struct typec_mux_s
 		return 0;
 	}
 
-	dev_dbg(qmp->dev, "typec_mux_set: switching from qmpphy mode %d to %d\n",
-		qmp->qmpphy_mode, new_mode);
-
-	qmp->qmpphy_mode = new_mode;
-
-	if (qmp->init_count) {
-		if (qmp->usb_init_count)
-			qmp_combo_usb_power_off(qmp->usb_phy);
-
-		if (qmp->dp_init_count)
-			writel(DP_PHY_PD_CTL_PSR_PWRDN, qmp->dp_dp_phy + QSERDES_DP_PHY_PD_CTL);
-
-		qmp_combo_com_exit(qmp, true);
-
-		/* Now everything's powered down, power up the right PHYs */
-		qmp_combo_com_init(qmp, true);
-
-		if (new_mode == QMPPHY_MODE_DP_ONLY) {
-			if (qmp->usb_init_count)
-				qmp->usb_init_count--;
-		}
-
-		if (new_mode == QMPPHY_MODE_USB3DP || new_mode == QMPPHY_MODE_USB3_ONLY) {
-			qmp_combo_usb_power_on(qmp->usb_phy);
-			if (!qmp->usb_init_count)
-				qmp->usb_init_count++;
-		}
-
-		if (new_mode == QMPPHY_MODE_DP_ONLY || new_mode == QMPPHY_MODE_USB3DP) {
-			if (qmp->dp_init_count)
-				cfg->dp_aux_init(qmp);
-		}
-	}
+	/* The mux still receives Type-C events, even if all PHYs are uninitialized */
+	if (!qmp->init_count)
+		return 0;
 
-	return 0;
+	return qmp_combo_reconfigure_phy(qmp, new_mode);
 }
 
 static void qmp_combo_typec_switch_unregister(void *data)
@@ -5088,6 +5328,9 @@ static int qmp_combo_parse_dt(struct qmp_combo *qmp)
 	if (offs->aon_toggle)
 		qmp->aon_toggle = base + offs->aon_toggle;
 
+	qmp->usb4_serdes = base + offs->usb4_serdes;
+	qmp->usb4_pcs = base + offs->usb4_pcs;
+
 	ret = qmp_combo_clk_init(qmp);
 	if (ret)
 		return ret;
@@ -5098,6 +5341,12 @@ static int qmp_combo_parse_dt(struct qmp_combo *qmp)
 				"failed to get usb3_pipe clock\n");
 	}
 
+	qmp->p2rr2p_pipe_clk = devm_clk_get_optional(dev, "p2rr2p_pipe");
+	if (IS_ERR(qmp->p2rr2p_pipe_clk)) {
+		return dev_err_probe(dev, PTR_ERR(qmp->p2rr2p_pipe_clk),
+				     "failed to get p2rr2p_pipe clock\n");
+	}
+
 	return 0;
 }
 
@@ -5113,6 +5362,8 @@ static struct phy *qmp_combo_phy_xlate(struct device *dev, const struct of_phand
 		return qmp->usb_phy;
 	case QMP_USB43DP_DP_PHY:
 		return qmp->dp_phy;
+	case QMP_USB43DP_USB4_PHY:
+		return qmp->usb4_phy ?: ERR_PTR(-EINVAL);
 	}
 
 	return ERR_PTR(-EINVAL);
@@ -5325,6 +5576,17 @@ static int qmp_combo_probe(struct platform_device *pdev)
 
 	phy_set_drvdata(qmp->dp_phy, qmp);
 
+	if (qmp->cfg->usb4_serdes_tbl) {
+		qmp->usb4_phy = devm_phy_create(dev, dev->of_node, &qmp_combo_usb4_phy_ops);
+		if (IS_ERR(qmp->usb4_phy)) {
+			ret = PTR_ERR(qmp->usb4_phy);
+			dev_err(dev, "failed to create USB4 PHY: %d\n", ret);
+			goto err_node_put;
+		}
+
+		phy_set_drvdata(qmp->usb4_phy, qmp);
+	}
+
 	if (usb_np == dev->of_node)
 		phy_provider = devm_of_phy_provider_register(dev, qmp_combo_phy_xlate);
 	else
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
index da83358c97ead1..d30194c6ed17f4 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
@@ -2417,6 +2417,50 @@ static const struct qmp_phy_init_tbl sdx65_qmp_pcie_pcs_lane1_tbl[] = {
 	QMP_PHY_INIT_CFG(QPHY_V5_20_PCS_LANE1_INSIG_MX_CTRL2, 0x00),
 };
 
+static const struct qmp_phy_init_tbl shikra_pcie_serdes_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_ENABLE1, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_EN, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER2, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x6),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESETSM_CNTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x9),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x3),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0xd),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x35),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_BUF_ENABLE, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CTRL_BY_PSM, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0x2f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_EP_DIV, 0x19),
+};
+
 static const struct qmp_phy_init_tbl sm8450_qmp_gen3_pcie_serdes_tbl[] = {
 	QMP_PHY_INIT_CFG(QSERDES_V5_COM_SYSCLK_EN_SEL, 0x08),
 	QMP_PHY_INIT_CFG(QSERDES_V5_COM_CLK_SELECT, 0x34),
@@ -4643,6 +4687,32 @@ static const struct qmp_phy_cfg sdm845_qhp_pciephy_cfg = {
 	.phy_status		= PHYSTATUS,
 };
 
+static const struct qmp_phy_cfg shikra_pciephy_cfg = {
+	.lanes			= 1,
+
+	.offsets		= &qmp_pcie_offsets_v2,
+
+	.tbls = {
+		.serdes		= shikra_pcie_serdes_tbl,
+		.serdes_num	= ARRAY_SIZE(shikra_pcie_serdes_tbl),
+		.tx		= qcs615_pcie_tx_tbl,
+		.tx_num		= ARRAY_SIZE(qcs615_pcie_tx_tbl),
+		.rx		= qcs615_pcie_rx_tbl,
+		.rx_num		= ARRAY_SIZE(qcs615_pcie_rx_tbl),
+		.pcs		= qcs615_pcie_pcs_tbl,
+		.pcs_num	= ARRAY_SIZE(qcs615_pcie_pcs_tbl),
+	},
+	.reset_list		= sdm845_pciephy_reset_l,
+	.num_resets		= ARRAY_SIZE(sdm845_pciephy_reset_l),
+	.vreg_list		= qmp_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(qmp_phy_vreg_l),
+	.regs			= pciephy_v2_regs_layout,
+
+	.pwrdn_ctrl		= SW_PWRDN | REFCLK_DRV_DSBL,
+	.phy_status		= PHYSTATUS,
+};
+
+
 static const struct qmp_phy_cfg sm8250_qmp_gen3x1_pciephy_cfg = {
 	.lanes			= 1,
 
@@ -6444,6 +6514,9 @@ static const struct of_device_id qmp_pcie_of_match_table[] = {
 	}, {
 		.compatible = "qcom,sdx65-qmp-gen4x2-pcie-phy",
 		.data = &sdx65_qmp_pciephy_cfg,
+	}, {
+		.compatible = "qcom,shikra-qmp-gen2x1-pcie-phy",
+		.data = &shikra_pciephy_cfg,
 	}, {
 		.compatible = "qcom,sm8150-qmp-gen3x1-pcie-phy",
 		.data = &sm8250_qmp_gen3x1_pciephy_cfg,
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h
index 325c127e8eb7ad..f094517ae3c184 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v8_50.h
@@ -7,7 +7,7 @@
 #define QCOM_PHY_QMP_PCS_V8_50_H_
 
 #define QPHY_V8_50_PCS_STATUS1			0x010
-#define QPHY_V8_50_PCS_START_CONTROL			0x05c
-#define QPHY_V8_50_PCS_POWER_DOWN_CONTROL			0x64
+#define QPHY_V8_50_PCS_START_CONTROL		0x05c
+#define QPHY_V8_50_PCS_POWER_DOWN_CONTROL	0x64
 
 #endif
diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
index 9ae9975d3255bd..b5ba751805aae9 100644
--- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c
+++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
@@ -27,6 +27,7 @@
 #include <linux/reset.h>
 #include <linux/string.h>
 #include <linux/usb/of.h>
+#include <linux/wait.h>
 #include <linux/workqueue.h>
 
 /******* USB2.0 Host registers (original offset is +0x200) *******/
@@ -106,6 +107,13 @@
 /*  RZ/G2L specific */
 #define USB2_LINECTRL1_USB2_IDMON	BIT(0)
 
+/*
+ * The OTG initialization is expected to finish in 20ms. Choose a large enough
+ * timeout to avoid waiters exit prematurely the waiting section under heavy
+ * CPU load.
+ */
+#define USB2_OTG_INIT_TIMEOUT		msecs_to_jiffies(120)
+
 #define NUM_OF_PHYS			4
 enum rcar_gen3_phy_index {
 	PHY_INDEX_BOTH_HC,
@@ -138,12 +146,20 @@ struct rcar_gen3_chan {
 	struct rcar_gen3_phy rphys[NUM_OF_PHYS];
 	struct regulator *vbus;
 	struct work_struct work;
+	wait_queue_head_t otg_init_done;
 	spinlock_t lock;	/* protects access to hardware and driver data structure. */
 	enum usb_dr_mode dr_mode;
 	bool extcon_host;
 	bool is_otg_channel;
 	bool uses_otg_pins;
 	bool otg_internal_reg;
+	/*
+	 * The OTG can be initialized only once and needs to release the spinlock
+	 * and wait for 20 ms due to hardware constraints. If a thread executes
+	 * PHY configuration code while the OTG PHY is waiting for the 20 ms, the
+	 * thread will have to wait for the OTG PHY initialization to complete.
+	 */
+	bool otg_initializing;
 };
 
 struct rcar_gen3_phy_drv_data {
@@ -392,26 +408,58 @@ static ssize_t role_store(struct device *dev, struct device_attribute *attr,
 	struct rcar_gen3_chan *ch = dev_get_drvdata(dev);
 	bool is_b_device;
 	enum phy_mode cur_mode, new_mode;
+	int retries = NUM_OF_PHYS;
+	unsigned long flags;
+	int ret = -EIO;
 
-	guard(spinlock_irqsave)(&ch->lock);
+	spin_lock_irqsave(&ch->lock, flags);
 
-	if (!ch->is_otg_channel || !rcar_gen3_is_any_otg_rphy_initialized(ch))
-		return -EIO;
+	if (!ch->is_otg_channel)
+		goto unlock;
+
+	while (retries-- && ch->otg_initializing) {
+		spin_unlock_irqrestore(&ch->lock, flags);
+
+		ret = wait_event_timeout(ch->otg_init_done, !ch->otg_initializing,
+					 USB2_OTG_INIT_TIMEOUT);
+		ret = ret ? 0 : -ETIMEDOUT;
+		if (ret && !retries)
+			goto exit;
+
+		spin_lock_irqsave(&ch->lock, flags);
+	}
+
+	/* If another thread started a new initialization just return -EBUSY. */
+	if (ch->otg_initializing) {
+		ret = -EBUSY;
+		goto unlock;
+	} else {
+		ret = 0;
+	}
+
+	if (!rcar_gen3_is_any_otg_rphy_initialized(ch)) {
+		ret = -EIO;
+		goto unlock;
+	}
 
-	if (sysfs_streq(buf, "host"))
+	if (sysfs_streq(buf, "host")) {
 		new_mode = PHY_MODE_USB_HOST;
-	else if (sysfs_streq(buf, "peripheral"))
+	} else if (sysfs_streq(buf, "peripheral")) {
 		new_mode = PHY_MODE_USB_DEVICE;
-	else
-		return -EINVAL;
+	} else {
+		ret = -EINVAL;
+		goto unlock;
+	}
 
 	/* is_b_device: true is B-Device. false is A-Device. */
 	is_b_device = rcar_gen3_check_id(ch);
 	cur_mode = rcar_gen3_get_phy_mode(ch);
 
 	/* If current and new mode is the same, this returns the error */
-	if (cur_mode == new_mode)
-		return -EINVAL;
+	if (cur_mode == new_mode) {
+		ret = -EINVAL;
+		goto unlock;
+	}
 
 	if (new_mode == PHY_MODE_USB_HOST) { /* And is_host must be false */
 		if (!is_b_device)	/* A-Peripheral */
@@ -425,7 +473,10 @@ static ssize_t role_store(struct device *dev, struct device_attribute *attr,
 			rcar_gen3_init_for_peri(ch);
 	}
 
-	return count;
+unlock:
+	spin_unlock_irqrestore(&ch->lock, flags);
+exit:
+	return ret ?: count;
 }
 
 static ssize_t role_show(struct device *dev, struct device_attribute *attr,
@@ -441,14 +492,11 @@ static ssize_t role_show(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(role);
 
-static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch)
+static void rcar_gen3_init_otg_phase0(struct rcar_gen3_chan *ch)
 {
 	void __iomem *usb2_base = ch->base;
 	u32 val;
 
-	if (!ch->is_otg_channel || rcar_gen3_is_any_otg_rphy_initialized(ch))
-		return;
-
 	/* Should not use functions of read-modify-write a register */
 	val = readl(usb2_base + USB2_LINECTRL1);
 	val = (val & ~USB2_LINECTRL1_DP_RPD) | USB2_LINECTRL1_DPRPD_EN |
@@ -471,7 +519,11 @@ static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch)
 			writel(val | USB2_ADPCTRL_IDPULLUP, usb2_base + USB2_ADPCTRL);
 		}
 	}
-	mdelay(20);
+}
+
+static void rcar_gen3_init_otg_phase1(struct rcar_gen3_chan *ch)
+{
+	void __iomem *usb2_base = ch->base;
 
 	writel(0xffffffff, usb2_base + USB2_OBINTSTA);
 	writel(ch->phy_data->obint_enable_bits, usb2_base + USB2_OBINTEN);
@@ -502,6 +554,7 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch)
 	void __iomem *usb2_base = ch->base;
 	struct device *dev = ch->dev;
 	irqreturn_t ret = IRQ_NONE;
+	unsigned long flags;
 	u32 status;
 
 	pm_runtime_get_noresume(dev);
@@ -509,33 +562,102 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch)
 	if (pm_runtime_suspended(dev))
 		goto rpm_put;
 
-	scoped_guard(spinlock, &ch->lock) {
-		status = readl(usb2_base + USB2_OBINTSTA);
-		if (status & ch->phy_data->obint_enable_bits) {
-			dev_vdbg(dev, "%s: %08x\n", __func__, status);
-			if (ch->phy_data->vblvl_ctrl)
-				writel(USB2_OBINTSTA_CLEAR, usb2_base + USB2_OBINTSTA);
-			else
-				writel(ch->phy_data->obint_enable_bits, usb2_base + USB2_OBINTSTA);
-			rcar_gen3_device_recognition(ch);
-			rcar_gen3_configure_vblvl_ctrl(ch);
-			ret = IRQ_HANDLED;
-		}
+	spin_lock_irqsave(&ch->lock, flags);
+
+	status = readl(usb2_base + USB2_OBINTSTA);
+	if (status & ch->phy_data->obint_enable_bits) {
+		dev_vdbg(dev, "%s: %08x\n", __func__, status);
+		if (ch->phy_data->vblvl_ctrl)
+			writel(USB2_OBINTSTA_CLEAR, usb2_base + USB2_OBINTSTA);
+		else
+			writel(ch->phy_data->obint_enable_bits, usb2_base + USB2_OBINTSTA);
+
+		ret = IRQ_HANDLED;
+
+		/* This should not happen! */
+		if (ch->otg_initializing)
+			goto unlock;
+
+		rcar_gen3_device_recognition(ch);
+		rcar_gen3_configure_vblvl_ctrl(ch);
 	}
 
+unlock:
+	spin_unlock_irqrestore(&ch->lock, flags);
 rpm_put:
 	pm_runtime_put_noidle(dev);
 	return ret;
 }
 
+static void rcar_gen3_phy_usb2_irqs_mask_all(struct rcar_gen3_chan *channel,
+					     u32 *masked_irqs_bits)
+{
+	u32 val, bitmask = USB2_INT_ENABLE_UCOM_INTEN;
+	void __iomem *usb2_base = channel->base;
+
+	for (unsigned int i = 0; i < NUM_OF_PHYS; i++)
+		bitmask |= channel->rphys[i].int_enable_bits;
+
+	val = readl(usb2_base + USB2_INT_ENABLE);
+	*masked_irqs_bits = val & bitmask;
+	val &= ~bitmask;
+	writel(val, usb2_base + USB2_INT_ENABLE);
+
+	/*
+	 * Don't report channel->phy_data->obint_enable_bits IRQs. These are
+	 * unmasked anyway in rcar_gen3_init_otg_phase1().
+	 */
+	val = readl(usb2_base + USB2_OBINTEN);
+	val &= ~channel->phy_data->obint_enable_bits;
+	writel(val, usb2_base + USB2_OBINTEN);
+}
+
+static void rcar_gen3_phy_usb2_irqs_unmask(struct rcar_gen3_chan *channel,
+					   u32 irqs_bits)
+{
+	u32 val, bitmask = USB2_INT_ENABLE_UCOM_INTEN;
+	void __iomem *usb2_base = channel->base;
+
+	for (unsigned int i = 0; i < NUM_OF_PHYS; i++)
+		bitmask |= channel->rphys[i].int_enable_bits;
+
+	val = readl(usb2_base + USB2_INT_ENABLE);
+	val &= ~bitmask;
+	val |= irqs_bits;
+	writel(val, usb2_base + USB2_INT_ENABLE);
+}
+
 static int rcar_gen3_phy_usb2_init(struct phy *p)
 {
 	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
 	struct rcar_gen3_chan *channel = rphy->ch;
 	void __iomem *usb2_base = channel->base;
+	int retries = NUM_OF_PHYS;
+	unsigned long flags;
 	u32 val;
+	int ret;
+
+	spin_lock_irqsave(&channel->lock, flags);
 
-	guard(spinlock_irqsave)(&channel->lock);
+	while (retries-- && channel->otg_initializing) {
+		spin_unlock_irqrestore(&channel->lock, flags);
+
+		ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing,
+					 USB2_OTG_INIT_TIMEOUT);
+		ret = ret ? 0 : -ETIMEDOUT;
+		if (ret && !retries)
+			return ret;
+
+		spin_lock_irqsave(&channel->lock, flags);
+	}
+
+	/* If another thread started a new initialization just return -EBUSY. */
+	if (channel->otg_initializing) {
+		ret = -EBUSY;
+		goto unlock;
+	} else {
+		ret = 0;
+	}
 
 	/* Initialize USB2 part */
 	val = readl(usb2_base + USB2_INT_ENABLE);
@@ -548,8 +670,23 @@ static int rcar_gen3_phy_usb2_init(struct phy *p)
 	}
 
 	/* Initialize otg part (only if we initialize a PHY with IRQs). */
-	if (rphy->int_enable_bits)
-		rcar_gen3_init_otg(channel);
+	if (rphy->int_enable_bits && channel->is_otg_channel &&
+	    !rcar_gen3_is_any_otg_rphy_initialized(channel)) {
+		u32 masked_irq_bits = 0;
+
+		rcar_gen3_init_otg_phase0(channel);
+		rcar_gen3_phy_usb2_irqs_mask_all(channel, &masked_irq_bits);
+		channel->otg_initializing = true;
+		spin_unlock_irqrestore(&channel->lock, flags);
+
+		fsleep(20000);
+
+		spin_lock_irqsave(&channel->lock, flags);
+		rcar_gen3_phy_usb2_irqs_unmask(channel, masked_irq_bits);
+		rcar_gen3_init_otg_phase1(channel);
+		channel->otg_initializing = false;
+		wake_up_all(&channel->otg_init_done);
+	}
 
 	if (channel->phy_data->vblvl_ctrl) {
 		/* SIDDQ mode release */
@@ -568,7 +705,10 @@ static int rcar_gen3_phy_usb2_init(struct phy *p)
 
 	rphy->initialized = true;
 
-	return 0;
+unlock:
+	spin_unlock_irqrestore(&channel->lock, flags);
+
+	return ret;
 }
 
 static int rcar_gen3_phy_usb2_exit(struct phy *p)
@@ -576,9 +716,32 @@ static int rcar_gen3_phy_usb2_exit(struct phy *p)
 	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
 	struct rcar_gen3_chan *channel = rphy->ch;
 	void __iomem *usb2_base = channel->base;
+	int retries = NUM_OF_PHYS;
+	unsigned long flags;
 	u32 val;
+	int ret;
+
+	spin_lock_irqsave(&channel->lock, flags);
+
+	while (retries-- && channel->otg_initializing) {
+		spin_unlock_irqrestore(&channel->lock, flags);
+
+		ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing,
+					 USB2_OTG_INIT_TIMEOUT);
+		ret = ret ? 0 : -ETIMEDOUT;
+		if (ret && !retries)
+			return ret;
+
+		spin_lock_irqsave(&channel->lock, flags);
+	}
 
-	guard(spinlock_irqsave)(&channel->lock);
+	/* If another thread started a new initialization just return -EBUSY. */
+	if (channel->otg_initializing) {
+		ret = -EBUSY;
+		goto unlock;
+	} else {
+		ret = 0;
+	}
 
 	rphy->initialized = false;
 
@@ -588,7 +751,9 @@ static int rcar_gen3_phy_usb2_exit(struct phy *p)
 		val &= ~USB2_INT_ENABLE_UCOM_INTEN;
 	writel(val, usb2_base + USB2_INT_ENABLE);
 
-	return 0;
+unlock:
+	spin_unlock_irqrestore(&channel->lock, flags);
+	return ret;
 }
 
 static int rcar_gen3_phy_usb2_power_on(struct phy *p)
@@ -596,8 +761,10 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p)
 	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
 	struct rcar_gen3_chan *channel = rphy->ch;
 	void __iomem *usb2_base = channel->base;
+	int retries = NUM_OF_PHYS;
+	unsigned long flags;
 	u32 val;
-	int ret = 0;
+	int ret;
 
 	if (channel->vbus && !channel->otg_internal_reg) {
 		ret = regulator_enable(channel->vbus);
@@ -605,7 +772,27 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p)
 			return ret;
 	}
 
-	guard(spinlock_irqsave)(&channel->lock);
+	spin_lock_irqsave(&channel->lock, flags);
+
+	while (retries-- && channel->otg_initializing) {
+		spin_unlock_irqrestore(&channel->lock, flags);
+
+		ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing,
+					 USB2_OTG_INIT_TIMEOUT);
+		ret = ret ? 0 : -ETIMEDOUT;
+		if (ret && !retries)
+			goto disable_regulator;
+
+		spin_lock_irqsave(&channel->lock, flags);
+	}
+
+	/* If another thread started a new initialization just return -EBUSY. */
+	if (channel->otg_initializing) {
+		ret = -EBUSY;
+		goto unlock;
+	} else {
+		ret = 0;
+	}
 
 	if (!rcar_gen3_are_all_rphys_power_off(channel))
 		goto out;
@@ -620,27 +807,59 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p)
 	/* The powered flag should be set for any other phys anyway */
 	rphy->powered = true;
 
-	return 0;
+unlock:
+	spin_unlock_irqrestore(&channel->lock, flags);
+
+disable_regulator:
+	if (ret && channel->vbus && !channel->otg_internal_reg)
+		regulator_disable(channel->vbus);
+
+	return ret;
 }
 
 static int rcar_gen3_phy_usb2_power_off(struct phy *p)
 {
 	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
 	struct rcar_gen3_chan *channel = rphy->ch;
-	int ret = 0;
+	int retries = NUM_OF_PHYS;
+	unsigned long flags;
+	int ret;
 
-	scoped_guard(spinlock_irqsave, &channel->lock) {
-		rphy->powered = false;
+	spin_lock_irqsave(&channel->lock, flags);
 
-		if (rcar_gen3_are_all_rphys_power_off(channel)) {
-			u32 val = readl(channel->base + USB2_USBCTR);
+	while (retries-- && channel->otg_initializing) {
+		spin_unlock_irqrestore(&channel->lock, flags);
 
-			val |= USB2_USBCTR_PLL_RST;
-			writel(val, channel->base + USB2_USBCTR);
-		}
+		ret = wait_event_timeout(channel->otg_init_done, !channel->otg_initializing,
+					 USB2_OTG_INIT_TIMEOUT);
+		ret = ret ? 0 : -ETIMEDOUT;
+		if (ret && !retries)
+			return ret;
+
+		spin_lock_irqsave(&channel->lock, flags);
+	}
+
+	/* If another thread started a new initialization just return -EBUSY. */
+	if (channel->otg_initializing) {
+		ret = -EBUSY;
+		goto unlock;
+	} else {
+		ret = 0;
+	}
+
+	rphy->powered = false;
+
+	if (rcar_gen3_are_all_rphys_power_off(channel)) {
+		u32 val = readl(channel->base + USB2_USBCTR);
+
+		val |= USB2_USBCTR_PLL_RST;
+		writel(val, channel->base + USB2_USBCTR);
 	}
 
-	if (channel->vbus && !channel->otg_internal_reg)
+unlock:
+	spin_unlock_irqrestore(&channel->lock, flags);
+
+	if (!ret && channel->vbus && !channel->otg_internal_reg)
 		ret = regulator_disable(channel->vbus);
 
 	return ret;
@@ -1022,6 +1241,7 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 		return ret;
 
 	spin_lock_init(&channel->lock);
+	init_waitqueue_head(&channel->otg_init_done);
 	for (i = 0; i < NUM_OF_PHYS; i++) {
 		channel->rphys[i].phy = devm_phy_create(dev, NULL,
 							channel->phy_data->phy_usb2_ops);
diff --git a/drivers/phy/samsung/phy-exynos5-usbdrd.c b/drivers/phy/samsung/phy-exynos5-usbdrd.c
index 8711a3b62c8e14..087d552eb5119c 100644
--- a/drivers/phy/samsung/phy-exynos5-usbdrd.c
+++ b/drivers/phy/samsung/phy-exynos5-usbdrd.c
@@ -471,7 +471,7 @@ struct exynos5_usbdrd_phy;
 struct exynos5_usbdrd_phy_config {
 	u32 id;
 	void (*phy_isol)(struct phy_usb_instance *inst, bool isolate);
-	void (*phy_init)(struct exynos5_usbdrd_phy *phy_drd);
+	int (*phy_init)(struct exynos5_usbdrd_phy *phy_drd);
 	unsigned int (*set_refclk)(struct phy_usb_instance *inst);
 };
 
@@ -708,7 +708,7 @@ exynos5_usbdrd_apply_phy_tunes(struct exynos5_usbdrd_phy *phy_drd,
 	}
 }
 
-static void exynos5_usbdrd_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
+static int exynos5_usbdrd_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	u32 reg;
 
@@ -721,6 +721,8 @@ static void exynos5_usbdrd_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
 	reg = readl(phy_drd->reg_phy + EXYNOS5_DRD_PHYTEST);
 	reg &= ~PHYTEST_POWERDOWN_SSP;
 	writel(reg, phy_drd->reg_phy + EXYNOS5_DRD_PHYTEST);
+
+	return 0;
 }
 
 static void
@@ -831,7 +833,7 @@ exynos5_usbdrd_usbdp_g2_v4_pma_check_pll_lock(struct exynos5_usbdrd_phy *phy_drd
 	return err;
 }
 
-static void
+static int
 exynos5_usbdrd_usbdp_g2_v4_pma_check_cdr_lock(struct exynos5_usbdrd_phy *phy_drd)
 {
 	static const unsigned int timeout_us = 40000;
@@ -857,9 +859,11 @@ exynos5_usbdrd_usbdp_g2_v4_pma_check_cdr_lock(struct exynos5_usbdrd_phy *phy_drd
 			((phy_drd->orientation == TYPEC_ORIENTATION_NORMAL)
 			 ? 0
 			 : 2), reg);
+
+	return err;
 }
 
-static void exynos5_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
+static int exynos5_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	u32 reg;
 
@@ -881,6 +885,8 @@ static void exynos5_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 	reg = readl(phy_drd->reg_phy + EXYNOS5_DRD_PHYTEST);
 	reg &= ~PHYTEST_POWERDOWN_HSP;
 	writel(reg, phy_drd->reg_phy + EXYNOS5_DRD_PHYTEST);
+
+	return 0;
 }
 
 static int exynos5_usbdrd_phy_init(struct phy *phy)
@@ -917,7 +923,9 @@ static int exynos5_usbdrd_phy_init(struct phy *phy)
 	writel(reg, phy_drd->reg_phy + EXYNOS5_DRD_PHYUTMICLKSEL);
 
 	/* UTMI or PIPE3 specific init */
-	inst->phy_cfg->phy_init(phy_drd);
+	ret = inst->phy_cfg->phy_init(phy_drd);
+	if (ret)
+		goto disable_clks;
 
 	/* reference clock settings */
 	reg = inst->phy_cfg->set_refclk(inst);
@@ -940,9 +948,10 @@ static int exynos5_usbdrd_phy_init(struct phy *phy)
 	reg &= ~PHYCLKRST_PORTRESET;
 	writel(reg, phy_drd->reg_phy + EXYNOS5_DRD_PHYCLKRST);
 
+disable_clks:
 	clk_bulk_disable_unprepare(phy_drd->drv_data->n_clks, phy_drd->clks);
 
-	return 0;
+	return ret;
 }
 
 static int exynos5_usbdrd_phy_exit(struct phy *phy)
@@ -1203,7 +1212,7 @@ static void exynos7870_usbdrd_phy_isol(struct phy_usb_instance *inst,
 			   EXYNOS7870_USB2PHY_ENABLE, val);
 }
 
-static void exynos7870_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
+static int exynos7870_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	u32 reg;
 
@@ -1291,6 +1300,8 @@ static void exynos7870_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 	if (phy_drd->drv_data->phy_tunes)
 		exynos5_usbdrd_apply_phy_tunes(phy_drd,
 					       PTS_UTMI_POSTINIT);
+
+	return 0;
 }
 
 static int exynos7870_usbdrd_phy_init(struct phy *phy)
@@ -1304,11 +1315,11 @@ static int exynos7870_usbdrd_phy_init(struct phy *phy)
 		return ret;
 
 	/* UTMI or PIPE3 specific init */
-	inst->phy_cfg->phy_init(phy_drd);
+	ret = inst->phy_cfg->phy_init(phy_drd);
 
 	clk_bulk_disable_unprepare(phy_drd->drv_data->n_clks, phy_drd->clks);
 
-	return 0;
+	return ret;
 }
 
 static int exynos7870_usbdrd_phy_exit(struct phy *phy)
@@ -1355,10 +1366,12 @@ static const struct phy_ops exynos7870_usbdrd_phy_ops = {
 	.owner		= THIS_MODULE,
 };
 
-static void exynos2200_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
+static int exynos2200_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	/* Configure non-Samsung IP PHY, responsible for UTMI */
 	phy_init(phy_drd->hs_phy);
+
+	return 0;
 }
 
 static void exynos2200_usbdrd_link_init(struct exynos5_usbdrd_phy *phy_drd)
@@ -1458,11 +1471,11 @@ static int exynos2200_usbdrd_phy_init(struct phy *phy)
 	exynos2200_usbdrd_link_attach_detach_pipe3_phy(inst);
 
 	/* UTMI or PIPE3 specific init */
-	inst->phy_cfg->phy_init(phy_drd);
+	ret = inst->phy_cfg->phy_init(phy_drd);
 
 	clk_bulk_disable_unprepare(phy_drd->drv_data->n_clks, phy_drd->clks);
 
-	return 0;
+	return ret;
 }
 
 static int exynos2200_usbdrd_phy_exit(struct phy *phy)
@@ -1516,7 +1529,7 @@ exynos5_usbdrd_usb_v3p1_pipe_override(struct exynos5_usbdrd_phy *phy_drd)
 	writel(reg, regs_base + EXYNOS850_DRD_SECPMACTL);
 }
 
-static void exynos850_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
+static int exynos850_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	void __iomem *regs_base = phy_drd->reg_phy;
 	u32 reg;
@@ -1622,6 +1635,8 @@ static void exynos850_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 
 	if (ss_ports)
 		exynos5_usbdrd_usb_v3p1_pipe_override(phy_drd);
+
+	return 0;
 }
 
 static int exynos850_usbdrd_phy_init(struct phy *phy)
@@ -1635,12 +1650,13 @@ static int exynos850_usbdrd_phy_init(struct phy *phy)
 		return ret;
 
 	/* UTMI or PIPE3 specific init */
+	ret = 0;
 	scoped_guard(mutex, &phy_drd->phy_mutex)
-		inst->phy_cfg->phy_init(phy_drd);
+		ret = inst->phy_cfg->phy_init(phy_drd);
 
 	clk_bulk_disable_unprepare(phy_drd->drv_data->n_clks, phy_drd->clks);
 
-	return 0;
+	return ret;
 }
 
 static int exynos850_usbdrd_phy_exit(struct phy *phy)
@@ -1689,11 +1705,12 @@ static const struct phy_ops exynos850_usbdrd_phy_ops = {
 	.owner		= THIS_MODULE,
 };
 
-static void exynos5_usbdrd_gs101_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
+static int exynos5_usbdrd_gs101_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	void __iomem *regs_pma = phy_drd->reg_pma;
 	void __iomem *regs_phy = phy_drd->reg_phy;
 	u32 reg;
+	int ret;
 
 	exynos5_usbdrd_usbdp_g2_v4_ctrl_pma_ready(phy_drd);
 
@@ -1715,8 +1732,11 @@ static void exynos5_usbdrd_gs101_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
 		 SECPMACTL_PMA_INIT_SW_RST);
 	writel(reg, regs_phy + EXYNOS850_DRD_SECPMACTL);
 
-	if (!exynos5_usbdrd_usbdp_g2_v4_pma_check_pll_lock(phy_drd))
-		exynos5_usbdrd_usbdp_g2_v4_pma_check_cdr_lock(phy_drd);
+	ret = exynos5_usbdrd_usbdp_g2_v4_pma_check_pll_lock(phy_drd);
+	if (ret)
+		return ret;
+
+	return exynos5_usbdrd_usbdp_g2_v4_pma_check_cdr_lock(phy_drd);
 }
 
 static int exynos5_usbdrd_gs101_phy_init(struct phy *phy)
@@ -1741,7 +1761,15 @@ static int exynos5_usbdrd_gs101_phy_init(struct phy *phy)
 	 */
 	exynos5_usbdrd_phy_isol(inst, false);
 
-	return exynos850_usbdrd_phy_init(phy);
+	ret = exynos850_usbdrd_phy_init(phy);
+	if (ret) {
+		exynos5_usbdrd_phy_isol(inst, true);
+		if (inst->phy_cfg->id == EXYNOS5_DRDPHY_UTMI)
+			regulator_bulk_disable(phy_drd->drv_data->n_regulators,
+					       phy_drd->regulators);
+	}
+
+	return ret;
 }
 
 static int exynos5_usbdrd_gs101_phy_exit(struct phy *phy)
@@ -2277,7 +2305,7 @@ exynosautov920_usb31drd_lane0_reset(struct exynos5_usbdrd_phy *phy_drd, int val)
 	writel(reg, reg_phy + EXYNOSAUTOV920_USB31DRD_PHY_RST_CTRL);
 }
 
-static void
+static int
 exynosautov920_usb31drd_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	void __iomem *reg_phy = phy_drd->reg_phy;
@@ -2335,6 +2363,8 @@ exynosautov920_usb31drd_pipe3_init(struct exynos5_usbdrd_phy *phy_drd)
 
 	/* override values for level settings */
 	exynosautov920_usb31drd_cr_write(phy_drd, 0x22, 0x00F5);
+
+	return 0;
 }
 
 static void
@@ -2353,7 +2383,7 @@ exynosautov920_usb31drd_ssphy_disable(struct exynos5_usbdrd_phy *phy_drd)
 	writel(reg, reg_phy + EXYNOSAUTOV920_USB31DRD_PHY_CONFIG7);
 }
 
-static void
+static int
 exynosautov920_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 {
 	void __iomem *reg_phy = phy_drd->reg_phy;
@@ -2457,6 +2487,8 @@ exynosautov920_usbdrd_utmi_init(struct exynos5_usbdrd_phy *phy_drd)
 	reg = readl(reg_phy + EXYNOS2200_DRD_CLKRST);
 	reg |= EXYNOS2200_CLKRST_LINK_PCLK_SEL;
 	writel(reg, reg_phy + EXYNOS2200_DRD_CLKRST);
+
+	return 0;
 }
 
 static void
@@ -2504,11 +2536,11 @@ static int exynosautov920_usbdrd_phy_init(struct phy *phy)
 	inst->phy_cfg->phy_isol(inst, false);
 
 	/* UTMI or PIPE3 specific init */
-	inst->phy_cfg->phy_init(phy_drd);
+	ret = inst->phy_cfg->phy_init(phy_drd);
 
 	clk_bulk_disable_unprepare(phy_drd->drv_data->n_clks, phy_drd->clks);
 
-	return 0;
+	return ret;
 }
 
 static int exynosautov920_usbdrd_phy_exit(struct phy *phy)
diff --git a/drivers/phy/socionext/phy-uniphier-usb3hs.c b/drivers/phy/socionext/phy-uniphier-usb3hs.c
index 8c8673df00842f..c8971eee36404a 100644
--- a/drivers/phy/socionext/phy-uniphier-usb3hs.c
+++ b/drivers/phy/socionext/phy-uniphier-usb3hs.c
@@ -104,11 +104,12 @@ static int uniphier_u3hsphy_get_nvparam(struct uniphier_u3hsphy_priv *priv,
 	struct nvmem_cell *cell;
 	u8 *buf;
 
-	cell = devm_nvmem_cell_get(priv->dev, name);
+	cell = nvmem_cell_get(priv->dev, name);
 	if (IS_ERR(cell))
 		return PTR_ERR(cell);
 
 	buf = nvmem_cell_read(cell, NULL);
+	nvmem_cell_put(cell);
 	if (IS_ERR(buf))
 		return PTR_ERR(buf);
 
diff --git a/drivers/phy/st/phy-stm32-combophy.c b/drivers/phy/st/phy-stm32-combophy.c
index 607b4d607eb5e6..ca4a472f954f28 100644
--- a/drivers/phy/st/phy-stm32-combophy.c
+++ b/drivers/phy/st/phy-stm32-combophy.c
@@ -8,6 +8,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/clk.h>
+#include <linux/interrupt.h>
 #include <linux/mfd/syscon.h>
 #include <linux/platform_device.h>
 #include <linux/phy/phy.h>
diff --git a/drivers/phy/starfive/phy-jh7110-dphy-tx.c b/drivers/phy/starfive/phy-jh7110-dphy-tx.c
index 181491a9380790..e9ffd0f811d2ba 100644
--- a/drivers/phy/starfive/phy-jh7110-dphy-tx.c
+++ b/drivers/phy/starfive/phy-jh7110-dphy-tx.c
@@ -209,7 +209,7 @@ static u32 stf_dphy_get_config_index(u32 bitrate)
 	return 0;
 }
 
-static void stf_dphy_hw_reset(struct stf_dphy *dphy, int assert)
+static int stf_dphy_hw_reset(struct stf_dphy *dphy, int assert)
 {
 	int rc;
 	u32 status = 0;
@@ -224,8 +224,12 @@ static void stf_dphy_hw_reset(struct stf_dphy *dphy, int assert)
 					       !(FIELD_GET(STF_DPHY_RGS_CDTX_PLL_UNLOCK, status)),
 					       STF_DPHY_HW_DELAY_US, STF_DPHY_HW_TIMEOUT_US);
 		if (rc)
-			dev_err(dphy->dev, "MIPI dphy-tx # PLL Locked\n");
+			dev_err(dphy->dev, "MIPI dphy-tx PLL failed to lock\n");
+
+		return rc;
 	}
+
+	return 0;
 }
 
 static int stf_dphy_configure(struct phy *phy, union phy_configure_opts *opts)
@@ -313,7 +317,9 @@ static int stf_dphy_init(struct phy *phy)
 	struct stf_dphy *dphy = phy_get_drvdata(phy);
 	int ret;
 
-	stf_dphy_hw_reset(dphy, 1);
+	ret = stf_dphy_hw_reset(dphy, 1);
+	if (ret)
+		return ret;
 
 	writel(FIELD_PREP(STF_DPHY_SCFG_PPI_C_READY_SEL, 0) |
 	       FIELD_PREP(STF_DPHY_SCFG_DSI_TXREADY_ESC_SEL, 0),
@@ -350,7 +356,9 @@ static int stf_dphy_exit(struct phy *phy)
 
 	clk_disable_unprepare(dphy->txesc_clk);
 
-	stf_dphy_hw_reset(dphy, 0);
+	ret = stf_dphy_hw_reset(dphy, 0);
+	if (ret)
+		return ret;
 
 	return 0;
 }
diff --git a/drivers/pinctrl/airoha/Kconfig b/drivers/pinctrl/airoha/Kconfig
index 22b8f850cee1ce..0f0986c81219b5 100644
--- a/drivers/pinctrl/airoha/Kconfig
+++ b/drivers/pinctrl/airoha/Kconfig
@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menu "Airoha pinctrl drivers"
-	depends on ARCH_AIROHA || COMPILE_TEST
+	depends on ARCH_AIROHA || ECONET || COMPILE_TEST
 
 config PINCTRL_AIROHA
 	tristate "Airoha pin control"
 	depends on OF
-	depends on ARCH_AIROHA || COMPILE_TEST
+	depends on ARCH_AIROHA || ECONET || COMPILE_TEST
 	select PINMUX
 	select GENERIC_PINCONF
 	select GENERIC_PINCTRL_GROUPS
@@ -17,6 +17,7 @@ config PINCTRL_AIROHA
 	imply PINCTRL_AIROHA_AN7581
 	imply PINCTRL_AIROHA_AN7583
 	imply PINCTRL_AIROHA_EN7523
+	imply PINCTRL_AIROHA_EN7528
 	help
 	  Shared pin controller and gpio driver code for different
 	  Airoha SoCs.
@@ -53,4 +54,14 @@ config PINCTRL_AIROHA_EN7523
 	  Say yes here to support pin controller and gpio driver
 	  on Airoha EN7523, EN7529, EN7562 and other compatible SoCs.
 
+config PINCTRL_AIROHA_EN7528
+	tristate "EN7528 pinctrl"
+	depends on ECONET || COMPILE_TEST
+	depends on PINCTRL_AIROHA
+	help
+	  Say yes here to support pin controller and gpio driver
+	  on EcoNet EN7528 and other compatible SoCs. Unlike the
+	  other variants, only the first sixteen GPIOs of this SoC
+	  are wired to the interrupt controller.
+
 endmenu
diff --git a/drivers/pinctrl/airoha/Makefile b/drivers/pinctrl/airoha/Makefile
index 30e4bee57a4a1e..4dc7c556ce4d51 100644
--- a/drivers/pinctrl/airoha/Makefile
+++ b/drivers/pinctrl/airoha/Makefile
@@ -8,3 +8,4 @@ obj-$(CONFIG_PINCTRL_AIROHA_AN7563)	+= pinctrl-an7563.o
 obj-$(CONFIG_PINCTRL_AIROHA_AN7581)	+= pinctrl-an7581.o
 obj-$(CONFIG_PINCTRL_AIROHA_AN7583)	+= pinctrl-an7583.o
 obj-$(CONFIG_PINCTRL_AIROHA_EN7523)	+= pinctrl-en7523.o
+obj-$(CONFIG_PINCTRL_AIROHA_EN7528)	+= pinctrl-en7528.o
diff --git a/drivers/pinctrl/airoha/airoha-common.h b/drivers/pinctrl/airoha/airoha-common.h
index c1acbfb7426e87..16f7abcdfb832d 100644
--- a/drivers/pinctrl/airoha/airoha-common.h
+++ b/drivers/pinctrl/airoha/airoha-common.h
@@ -127,6 +127,7 @@ struct airoha_pinctrl {
 
 	struct gpio_chip gpiochip;
 	struct airoha_gpiochip_regs *gpio_regs;
+	unsigned int num_irq_pins;
 };
 
 struct airoha_pinctrl_match_data {
@@ -140,6 +141,8 @@ struct airoha_pinctrl_match_data {
 	const struct airoha_pinctrl_func *funcs;
 	const unsigned int num_funcs;
 	const struct airoha_pinctrl_confs_info confs_info[AIROHA_PINCTRL_CONFS_MAX];
+	/* number of GPIOs wired to the interrupt controller */
+	const unsigned int num_irq_pins;
 };
 
 int airoha_pinctrl_probe(struct platform_device *pdev);
diff --git a/drivers/pinctrl/airoha/pinctrl-airoha.c b/drivers/pinctrl/airoha/pinctrl-airoha.c
index f505a3f69c5d17..52a768c859b71c 100644
--- a/drivers/pinctrl/airoha/pinctrl-airoha.c
+++ b/drivers/pinctrl/airoha/pinctrl-airoha.c
@@ -213,7 +213,7 @@ static void airoha_irq_unmask(struct irq_data *data)
 	u32 mask = GENMASK(2 * offset + 1, 2 * offset);
 	u32 val = BIT(2 * offset);
 
-	if (WARN_ON_ONCE(data->hwirq >= AIROHA_NUM_PINS))
+	if (WARN_ON_ONCE(data->hwirq >= pinctrl->num_irq_pins))
 		return;
 
 	gpiochip_enable_irq(gc, irqd_to_hwirq(data));
@@ -249,7 +249,7 @@ static void airoha_irq_mask(struct irq_data *data)
 	u8 index = data->hwirq / AIROHA_REG_GPIOCTRL_NUM_PIN;
 	u32 mask = GENMASK(2 * offset + 1, 2 * offset);
 
-	if (data->hwirq >= AIROHA_NUM_PINS)
+	if (data->hwirq >= pinctrl->num_irq_pins)
 		return;
 
 	regmap_clear_bits(pinctrl->regmap, gpio_regs->level[index], mask);
@@ -265,7 +265,7 @@ static void airoha_irq_ack(struct irq_data *data)
 	u8 offset = data->hwirq % AIROHA_PIN_BANK_SIZE;
 	u8 index = data->hwirq / AIROHA_PIN_BANK_SIZE;
 
-	if (data->hwirq >= AIROHA_NUM_PINS)
+	if (data->hwirq >= pinctrl->num_irq_pins)
 		return;
 
 	regmap_write(pinctrl->regmap, gpio_regs->status[index], BIT(offset));
@@ -273,7 +273,10 @@ static void airoha_irq_ack(struct irq_data *data)
 
 static int airoha_irq_type(struct irq_data *data, unsigned int type)
 {
-	if (data->hwirq >= AIROHA_NUM_PINS)
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(data);
+	struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc);
+
+	if (data->hwirq >= pinctrl->num_irq_pins)
 		return -EINVAL;
 
 	if (type == IRQ_TYPE_NONE) {
@@ -304,9 +307,11 @@ static irqreturn_t airoha_irq_handler(int irq, void *data)
 {
 	struct airoha_pinctrl *pinctrl = data;
 	bool handled = false;
+	unsigned int nbanks;
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(irq_status_regs); i++) {
+	nbanks = DIV_ROUND_UP(pinctrl->num_irq_pins, AIROHA_PIN_BANK_SIZE);
+	for (i = 0; i < nbanks; i++) {
 		struct gpio_irq_chip *girq = &pinctrl->gpiochip.irq;
 		u32 regmap;
 		unsigned long status;
@@ -340,6 +345,22 @@ static const struct irq_chip airoha_gpio_irq_chip = {
 	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
+/*
+ * Mark the GPIOs that are not wired to the interrupt controller as not
+ * valid, so that gpiod_to_irq() fails for them with -ENXIO instead of
+ * handing out an interrupt that can never fire.
+ */
+static void airoha_gpio_init_valid_mask(struct gpio_chip *gc,
+					unsigned long *valid_mask,
+					unsigned int ngpios)
+{
+	struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc);
+	unsigned int num_irq_pins = pinctrl->num_irq_pins;
+
+	if (num_irq_pins < ngpios)
+		bitmap_clear(valid_mask, num_irq_pins, ngpios - num_irq_pins);
+}
+
 static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl,
 					struct platform_device *pdev)
 {
@@ -362,6 +383,7 @@ static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl,
 
 	girq->default_type = IRQ_TYPE_NONE;
 	girq->handler = handle_bad_irq;
+	girq->init_valid_mask = airoha_gpio_init_valid_mask;
 	gpio_irq_chip_set_chip(girq, &airoha_gpio_irq_chip);
 
 	irq = platform_get_irq(pdev, 0);
@@ -848,6 +870,7 @@ int airoha_pinctrl_probe(struct platform_device *pdev)
 	pinctrl->grps = data->grps;
 	pinctrl->funcs = data->funcs;
 	pinctrl->confs_info = data->confs_info;
+	pinctrl->num_irq_pins = data->num_irq_pins;
 
 	err = pinctrl_enable(pinctrl->ctrl);
 	if (err)
diff --git a/drivers/pinctrl/airoha/pinctrl-an7563.c b/drivers/pinctrl/airoha/pinctrl-an7563.c
index 40cbbe90cc4648..f011c6c9cccec2 100644
--- a/drivers/pinctrl/airoha/pinctrl-an7563.c
+++ b/drivers/pinctrl/airoha/pinctrl-an7563.c
@@ -1069,6 +1069,7 @@ static const struct airoha_pinctrl_match_data pinctrl_match_data = {
 	.num_grps = ARRAY_SIZE(pinctrl_groups),
 	.funcs = pinctrl_funcs,
 	.num_funcs = ARRAY_SIZE(pinctrl_funcs),
+	.num_irq_pins = AIROHA_NUM_PINS,
 	.confs_info = {
 		[AIROHA_PINCTRL_CONFS_PULLUP] = {
 			.confs = pinctrl_pullup_conf,
diff --git a/drivers/pinctrl/airoha/pinctrl-an7581.c b/drivers/pinctrl/airoha/pinctrl-an7581.c
index 2fcf88106e11dc..bfb777594811be 100644
--- a/drivers/pinctrl/airoha/pinctrl-an7581.c
+++ b/drivers/pinctrl/airoha/pinctrl-an7581.c
@@ -1441,6 +1441,7 @@ static const struct airoha_pinctrl_match_data pinctrl_match_data = {
 	.num_grps = ARRAY_SIZE(pinctrl_groups),
 	.funcs = pinctrl_funcs,
 	.num_funcs = ARRAY_SIZE(pinctrl_funcs),
+	.num_irq_pins = AIROHA_NUM_PINS,
 	.confs_info = {
 		[AIROHA_PINCTRL_CONFS_PULLUP] = {
 			.confs = pinctrl_pullup_conf,
diff --git a/drivers/pinctrl/airoha/pinctrl-an7583.c b/drivers/pinctrl/airoha/pinctrl-an7583.c
index 2c3a75c35915c1..1cd0f442ddc1d5 100644
--- a/drivers/pinctrl/airoha/pinctrl-an7583.c
+++ b/drivers/pinctrl/airoha/pinctrl-an7583.c
@@ -1471,6 +1471,7 @@ static const struct airoha_pinctrl_match_data pinctrl_match_data = {
 	.num_grps = ARRAY_SIZE(pinctrl_groups),
 	.funcs = pinctrl_funcs,
 	.num_funcs = ARRAY_SIZE(pinctrl_funcs),
+	.num_irq_pins = AIROHA_NUM_PINS,
 	.confs_info = {
 		[AIROHA_PINCTRL_CONFS_PULLUP] = {
 			.confs = pinctrl_pullup_conf,
diff --git a/drivers/pinctrl/airoha/pinctrl-en7523.c b/drivers/pinctrl/airoha/pinctrl-en7523.c
index 5aa39bacf460c1..b0c5e60f0aeb92 100644
--- a/drivers/pinctrl/airoha/pinctrl-en7523.c
+++ b/drivers/pinctrl/airoha/pinctrl-en7523.c
@@ -1113,6 +1113,7 @@ static const struct airoha_pinctrl_match_data pinctrl_match_data = {
 	.num_grps = ARRAY_SIZE(pinctrl_groups),
 	.funcs = pinctrl_funcs,
 	.num_funcs = ARRAY_SIZE(pinctrl_funcs),
+	.num_irq_pins = AIROHA_NUM_PINS,
 	.confs_info = {
 		[AIROHA_PINCTRL_CONFS_PULLUP] = {
 			.confs = pinctrl_pullup_conf,
diff --git a/drivers/pinctrl/airoha/pinctrl-en7528.c b/drivers/pinctrl/airoha/pinctrl-en7528.c
new file mode 100644
index 00000000000000..f6c8f834421f74
--- /dev/null
+++ b/drivers/pinctrl/airoha/pinctrl-en7528.c
@@ -0,0 +1,1204 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Author: Benjamin Larsson <benjamin.larsson@genexis.eu>
+ *
+ * EcoNet EN7528 SoC pinctrl driver.
+ *
+ * Sources: EN7528 External Programming Guide V1.3 chapter 1.2.2/1.3.2
+ * (CHIP SCU, base 0x1fa20000), chapter 12 (GPIO controller, base
+ * 0x1fbf0200) and EN7528DU/EN7528HU datasheets chapter 4.3 (pin sharing
+ * schemes; both packages use the same GPIO mapping).
+ *
+ * Unlike the later SoCs, all IOMUX mode bits live in a single register
+ * (RG_PON_I2C_MODE at 0x15c) plus the spill-over bits in IOMUX Control 3
+ * (RG_FORCE_GPIO22_EN at 0x224). EJTAG has its own enable register at
+ * 0x1ac.
+ *
+ * Pin numbering follows the GPIO line numbering of the SoC:
+ *   pins  0-11 -> GPIO0-GPIO11 (dedicated GPIO pads)
+ *   pins 12-15 -> ZSYNC1/ZCLK1/ZMOSI1/ZMISO1 pads (GPIO12-GPIO15)
+ *   pins 16-21 -> PON I/O pads (GPIO16-GPIO21, GPIO21 = PON_RX_DIS)
+ *   pins 22-39 -> GPIO22-GPIO39 (dedicated GPIO pads)
+ *   pins 40/41 -> PCIE_RESET0/PCIE_RESET1 pads (GPIO40/GPIO41)
+ * Pins 42-49 are the standalone pads (I2C, UART, SPI). They have no
+ * GPIO function and are only referenced by the pinconf tables.
+ *
+ * Note: the RG_PON_I2C_MODE register description in the programming
+ * guide carries stale "GPIO30/GPIO31" text for the PCIE_RESET pads; the
+ * EN7528 datasheets (table 4-10) map them to GPIO40/GPIO41.
+ *
+ * The PCM SPI chip select register bit names are offset by two against
+ * the datasheet signal names: rg_gpio_pcm_spi_cs3_mode drives
+ * PCM_SPI_CS1 (spelled out in datasheet table 4-6).
+ */
+#include "airoha-common.h"
+
+/* MUX */
+#define REG_PON_I2C_MODE			0x015c
+#define SIPO_RCLK_MODE_MASK		BIT(30)
+/*
+ * A set PCIE_RESET{0,1}_GPIO bit routes the pad to its GPIO function
+ * (GPIO40/GPIO41), a cleared bit selects the PCIe reset function.
+ */
+#define GPIO_PCIE_RESET1_MASK		BIT(29)	/* GPIO41 */
+#define GPIO_PCIE_RESET0_MASK		BIT(28)	/* GPIO40 */
+#define GPIO_SPI_QUAD_MODE_MASK		BIT(27)
+#define GPIO_UART3_MODE_MASK		BIT(26)
+#define GPIO_UART2_CTS_RTS_MODE_MASK	BIT(25)
+#define GPIO_UART2_MODE_MASK		BIT(24)
+#define GPIO_SIPO_MODE_MASK		BIT(23)
+#define GPIO_PON_MODE_MASK		BIT(21)
+#define GPIO_PCM2_MODE_MASK		BIT(20)
+#define GPIO_PCM1_MODE_MASK		BIT(19)
+#define GPIO_PCM_SPI_MODE_MASK		BIT(18)	/* RG_GPIO_SPI2_MODE */
+#define GPIO_PCM_INT_MODE_MASK		BIT(17)
+#define GPIO_PCM_RESET_MODE_MASK		BIT(16)
+#define GPIO_SPI_CS1_MODE_MASK		BIT(15)
+#define GPIO_PCM_SPI_CS1_MODE_MASK	BIT(14)	/* RG ..._CS3 */
+#define GPIO_LAN3_LED1_MODE_MASK		BIT(11)
+#define GPIO_LAN3_LED0_MODE_MASK		BIT(10)
+#define GPIO_LAN2_LED1_MODE_MASK		BIT(9)
+#define GPIO_LAN2_LED0_MODE_MASK		BIT(8)
+#define GPIO_LAN1_LED1_MODE_MASK		BIT(7)
+#define GPIO_LAN1_LED0_MODE_MASK		BIT(6)
+#define GPIO_LAN0_LED1_MODE_MASK		BIT(5)
+#define GPIO_LAN0_LED0_MODE_MASK		BIT(4)
+#define PON_TOD_1PPS_MODE_MASK		BIT(2)
+#define GSW_TOD_1PPS_MODE_MASK		BIT(1)
+
+#define REG_FORCE_GPIO22_EN		0x0224
+#define GPIO_PCM_SPI_CS7_MODE_MASK	BIT(15)	/* RG ..._CS9 */
+#define GPIO_PCM_SPI_CS6_MODE_MASK	BIT(14)	/* RG ..._CS8 */
+#define GPIO_PCM_SPI_CS5_MODE_MASK	BIT(13)	/* RG ..._CS7 */
+#define GPIO_PCM_SPI_CS4_MODE_MASK	BIT(12)	/* RG ..._CS6 */
+#define GPIO_PCM_SPI_CS3_MODE_MASK	BIT(11)	/* RG ..._CS5 */
+#define GPIO_PCM_SPI_CS2_MODE_MASK	BIT(10)	/* RG ..._CS4 */
+#define GPIO_PNAND_MODE_MASK		BIT(9)	/* RG_GPIO_NFD_MODE */
+
+#define REG_CPU_EJTAG_EN			0x01ac
+#define CPU_EJTAG_EN_MASK		BIT(0)
+
+/* LED MAP */
+#define REG_LAN_LED0_MAPPING		0x01b0
+#define REG_LAN_LED1_MAPPING		0x01b4
+
+/* shared */
+#define LAN3_LED_MAPPING_MASK			GENMASK(14, 12)
+#define LAN3_PHY_LED_MAP(_n)			FIELD_PREP_CONST(LAN3_LED_MAPPING_MASK, (_n))
+
+#define LAN2_LED_MAPPING_MASK			GENMASK(10, 8)
+#define LAN2_PHY_LED_MAP(_n)			FIELD_PREP_CONST(LAN2_LED_MAPPING_MASK, (_n))
+
+#define LAN1_LED_MAPPING_MASK			GENMASK(6, 4)
+#define LAN1_PHY_LED_MAP(_n)			FIELD_PREP_CONST(LAN1_LED_MAPPING_MASK, (_n))
+
+#define LAN0_LED_MAPPING_MASK			GENMASK(2, 0)
+#define LAN0_PHY_LED_MAP(_n)			FIELD_PREP_CONST(LAN0_LED_MAPPING_MASK, (_n))
+
+/*
+ * CONF.
+ * The EN7528 documentation names the two drive strength stages E4/E8;
+ * they fill the DRIVE_E2/DRIVE_E4 pinconf slots. All conf register
+ * offsets differ from the later SoCs.
+ */
+#define REG_I2C_SDA_E2			0x0014	/* RG_I2C_SDA_E4 */
+#define REG_I2C_SDA_E4			0x0018	/* RG_I2C_SDA_E8 */
+#define REG_GPIO_L_E2			0x001c	/* RG_GPIO_L_E4 */
+#define REG_GPIO_H_E2			0x0020	/* RG_GPIO_H_E4 */
+#define REG_GPIO_L_E4			0x0024	/* RG_GPIO_L_E8 */
+#define REG_GPIO_H_E4			0x0028	/* RG_GPIO_H_E8 */
+
+#define REG_I2C_SDA_PU			0x003c
+#define REG_I2C_SDA_PD			0x0040
+#define REG_GPIO_L_PU			0x0044
+#define REG_GPIO_H_PU			0x0048
+#define REG_GPIO_L_PD			0x004c
+#define REG_GPIO_H_PD			0x0050
+
+/* standalone IO conf bit layout, common to E4/E8/PU/PD registers */
+#define SPI_MISO_CONF_MASK		BIT(13)
+#define SPI_MOSI_CONF_MASK		BIT(12)
+#define SPI_CLK_CONF_MASK		BIT(11)
+#define SPI_CS0_CONF_MASK		BIT(10)
+#define PCIE1_RESET_CONF_MASK		BIT(9)
+#define PCIE0_RESET_CONF_MASK		BIT(8)
+#define UART_RXD_CONF_MASK		BIT(3)
+#define UART_TXD_CONF_MASK		BIT(2)
+#define I2C_SCL_CONF_MASK		BIT(1)
+#define I2C_SDA_CONF_MASK		BIT(0)
+
+/*
+ * PWM MODE CONF - the flash mode registers match EN7580: EXT bit 16
+ * maps to GPIO32 and EXT bits 17+ to GPIO37+. GPIO33-GPIO36 have no
+ * flash mode configuration bit.
+ */
+#define REG_GPIO_FLASH_MODE_CFG			0x0034
+#define GPIO15_FLASH_MODE_CFG			BIT(15)
+#define GPIO14_FLASH_MODE_CFG			BIT(14)
+#define GPIO13_FLASH_MODE_CFG			BIT(13)
+#define GPIO12_FLASH_MODE_CFG			BIT(12)
+#define GPIO11_FLASH_MODE_CFG			BIT(11)
+#define GPIO10_FLASH_MODE_CFG			BIT(10)
+#define GPIO9_FLASH_MODE_CFG			BIT(9)
+#define GPIO8_FLASH_MODE_CFG			BIT(8)
+#define GPIO7_FLASH_MODE_CFG			BIT(7)
+#define GPIO6_FLASH_MODE_CFG			BIT(6)
+#define GPIO5_FLASH_MODE_CFG			BIT(5)
+#define GPIO4_FLASH_MODE_CFG			BIT(4)
+#define GPIO3_FLASH_MODE_CFG			BIT(3)
+#define GPIO2_FLASH_MODE_CFG			BIT(2)
+#define GPIO1_FLASH_MODE_CFG			BIT(1)
+#define GPIO0_FLASH_MODE_CFG			BIT(0)
+
+#define REG_GPIO_FLASH_MODE_CFG_EXT		0x0068
+#define GPIO41_FLASH_MODE_CFG			BIT(21)
+#define GPIO40_FLASH_MODE_CFG			BIT(20)
+#define GPIO39_FLASH_MODE_CFG			BIT(19)
+#define GPIO38_FLASH_MODE_CFG			BIT(18)
+#define GPIO37_FLASH_MODE_CFG			BIT(17)
+#define GPIO32_FLASH_MODE_CFG			BIT(16)	/* GPIO36 on EN7581 */
+#define GPIO31_FLASH_MODE_CFG			BIT(15)
+#define GPIO30_FLASH_MODE_CFG			BIT(14)
+#define GPIO29_FLASH_MODE_CFG			BIT(13)
+#define GPIO28_FLASH_MODE_CFG			BIT(12)
+#define GPIO27_FLASH_MODE_CFG			BIT(11)
+#define GPIO26_FLASH_MODE_CFG			BIT(10)
+#define GPIO25_FLASH_MODE_CFG			BIT(9)
+#define GPIO24_FLASH_MODE_CFG			BIT(8)
+#define GPIO23_FLASH_MODE_CFG			BIT(7)
+#define GPIO22_FLASH_MODE_CFG			BIT(6)
+#define GPIO21_FLASH_MODE_CFG			BIT(5)
+#define GPIO20_FLASH_MODE_CFG			BIT(4)
+#define GPIO19_FLASH_MODE_CFG			BIT(3)
+#define GPIO18_FLASH_MODE_CFG			BIT(2)
+#define GPIO17_FLASH_MODE_CFG			BIT(1)
+#define GPIO16_FLASH_MODE_CFG			BIT(0)
+
+static const struct pinctrl_pin_desc pinctrl_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),	/* PCM_INT */
+	PINCTRL_PIN(2, "gpio2"),	/* PCM_RESET */
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "gpio8"),
+	PINCTRL_PIN(9, "gpio9"),
+	PINCTRL_PIN(10, "gpio10"),
+	PINCTRL_PIN(11, "gpio11"),
+	PINCTRL_PIN(12, "gpio12"),	/* ZSYNC1 */
+	PINCTRL_PIN(13, "gpio13"),	/* ZCLK1 */
+	PINCTRL_PIN(14, "gpio14"),	/* ZMOSI1 */
+	PINCTRL_PIN(15, "gpio15"),	/* ZMISO1 */
+	PINCTRL_PIN(16, "gpio16"),	/* PON_TX_DIS */
+	PINCTRL_PIN(17, "gpio17"),	/* PON_TX_FAULT */
+	PINCTRL_PIN(18, "gpio18"),	/* PON_TX_SD */
+	PINCTRL_PIN(19, "gpio19"),	/* PON_RX_SD */
+	PINCTRL_PIN(20, "gpio20"),	/* PON_BURST_EN */
+	PINCTRL_PIN(21, "gpio21"),	/* PON_RX_DIS */
+	PINCTRL_PIN(22, "gpio22"),
+	PINCTRL_PIN(23, "gpio23"),
+	PINCTRL_PIN(24, "gpio24"),
+	PINCTRL_PIN(25, "gpio25"),
+	PINCTRL_PIN(26, "gpio26"),
+	PINCTRL_PIN(27, "gpio27"),
+	PINCTRL_PIN(28, "gpio28"),
+	PINCTRL_PIN(29, "gpio29"),
+	PINCTRL_PIN(30, "gpio30"),
+	PINCTRL_PIN(31, "gpio31"),
+	PINCTRL_PIN(32, "gpio32"),
+	PINCTRL_PIN(33, "gpio33"),
+	PINCTRL_PIN(34, "gpio34"),
+	PINCTRL_PIN(35, "gpio35"),
+	PINCTRL_PIN(36, "gpio36"),
+	PINCTRL_PIN(37, "gpio37"),
+	PINCTRL_PIN(38, "gpio38"),
+	PINCTRL_PIN(39, "gpio39"),
+	PINCTRL_PIN(40, "pcie_reset0"),	/* GPIO40 */
+	PINCTRL_PIN(41, "pcie_reset1"),	/* GPIO41 */
+	/* standalone pads, pinconf only */
+	PINCTRL_PIN(42, "i2c_sda"),
+	PINCTRL_PIN(43, "i2c_scl"),
+	PINCTRL_PIN(44, "uart_txd"),
+	PINCTRL_PIN(45, "uart_rxd"),
+	PINCTRL_PIN(46, "spi_cs0"),
+	PINCTRL_PIN(47, "spi_clk"),
+	PINCTRL_PIN(48, "spi_mosi"),
+	PINCTRL_PIN(49, "spi_miso"),
+};
+
+static const int pon_pins[] = { 17, 18, 19, 20 };
+static const int sipo_pins[] = { 8, 11 };
+static const int sipo_rclk_pins[] = { 8, 9, 11 };
+static const int uart2_pins[] = { 28, 29 };
+static const int uart2_cts_rts_pins[] = { 12, 13 };
+static const int uart3_pins[] = { 38, 39 };
+static const int ejtag_pins[] = { 31, 33, 35, 37, 39 };
+static const int pcm1_pins[] = { 12, 13, 14, 15 };
+static const int pcm2_pins[] = { 24, 25, 26, 27 };
+static const int spi_quad_pins[] = { 10, 23 };
+static const int spi_cs1_pins[] = { 9 };
+static const int pcm_spi_pins[] = { 4, 5, 6, 7 };
+static const int pcm_spi_int_pins[] = { 1 };
+static const int pcm_spi_rst_pins[] = { 2 };
+static const int pcm_spi_cs1_pins[] = { 3 };
+static const int pcm_spi_cs2_pins[] = { 10 };
+static const int pcm_spi_cs3_pins[] = { 23 };
+static const int pcm_spi_cs4_pins[] = { 21 };
+static const int pcm_spi_cs5_pins[] = { 9 };
+static const int pcm_spi_cs6_pins[] = { 28 };
+static const int pcm_spi_cs7_pins[] = { 29 };
+static const int pnand_pins[] = { 3, 8, 10, 11, 21, 22, 23, 24, 25, 26, 27,
+				  28, 29, 46, 47, 48, 49 };
+static const int gpio0_pins[] = { 0 };
+static const int gpio1_pins[] = { 1 };
+static const int gpio2_pins[] = { 2 };
+static const int gpio3_pins[] = { 3 };
+static const int gpio4_pins[] = { 4 };
+static const int gpio5_pins[] = { 5 };
+static const int gpio6_pins[] = { 6 };
+static const int gpio7_pins[] = { 7 };
+static const int gpio8_pins[] = { 8 };
+static const int gpio9_pins[] = { 9 };
+static const int gpio10_pins[] = { 10 };
+static const int gpio11_pins[] = { 11 };
+static const int gpio12_pins[] = { 12 };
+static const int gpio13_pins[] = { 13 };
+static const int gpio14_pins[] = { 14 };
+static const int gpio15_pins[] = { 15 };
+static const int gpio16_pins[] = { 16 };
+static const int gpio17_pins[] = { 17 };
+static const int gpio18_pins[] = { 18 };
+static const int gpio19_pins[] = { 19 };
+static const int gpio20_pins[] = { 20 };
+static const int gpio21_pins[] = { 21 };
+static const int gpio22_pins[] = { 22 };
+static const int gpio23_pins[] = { 23 };
+static const int gpio24_pins[] = { 24 };
+static const int gpio25_pins[] = { 25 };
+static const int gpio26_pins[] = { 26 };
+static const int gpio27_pins[] = { 27 };
+static const int gpio28_pins[] = { 28 };
+static const int gpio29_pins[] = { 29 };
+static const int gpio30_pins[] = { 30 };
+static const int gpio31_pins[] = { 31 };
+static const int gpio32_pins[] = { 32 };
+static const int gpio33_pins[] = { 33 };
+static const int gpio34_pins[] = { 34 };
+static const int gpio35_pins[] = { 35 };
+static const int gpio36_pins[] = { 36 };
+static const int gpio37_pins[] = { 37 };
+static const int gpio38_pins[] = { 38 };
+static const int gpio39_pins[] = { 39 };
+static const int gpio40_pins[] = { 40 };
+static const int gpio41_pins[] = { 41 };
+static const int pcie_reset0_pins[] = { 40 };
+static const int pcie_reset1_pins[] = { 41 };
+
+static const struct pingroup pinctrl_groups[] = {
+	PINCTRL_PIN_GROUP("pon", pon),
+	PINCTRL_PIN_GROUP("sipo", sipo),
+	PINCTRL_PIN_GROUP("sipo_rclk", sipo_rclk),
+	PINCTRL_PIN_GROUP("uart2", uart2),
+	PINCTRL_PIN_GROUP("uart2_cts_rts", uart2_cts_rts),
+	PINCTRL_PIN_GROUP("uart3", uart3),
+	PINCTRL_PIN_GROUP("ejtag", ejtag),
+	PINCTRL_PIN_GROUP("pcm1", pcm1),
+	PINCTRL_PIN_GROUP("pcm2", pcm2),
+	PINCTRL_PIN_GROUP("spi_quad", spi_quad),
+	PINCTRL_PIN_GROUP("spi_cs1", spi_cs1),
+	PINCTRL_PIN_GROUP("pcm_spi", pcm_spi),
+	PINCTRL_PIN_GROUP("pcm_spi_int", pcm_spi_int),
+	PINCTRL_PIN_GROUP("pcm_spi_rst", pcm_spi_rst),
+	PINCTRL_PIN_GROUP("pcm_spi_cs1", pcm_spi_cs1),
+	PINCTRL_PIN_GROUP("pcm_spi_cs2", pcm_spi_cs2),
+	PINCTRL_PIN_GROUP("pcm_spi_cs3", pcm_spi_cs3),
+	PINCTRL_PIN_GROUP("pcm_spi_cs4", pcm_spi_cs4),
+	PINCTRL_PIN_GROUP("pcm_spi_cs5", pcm_spi_cs5),
+	PINCTRL_PIN_GROUP("pcm_spi_cs6", pcm_spi_cs6),
+	PINCTRL_PIN_GROUP("pcm_spi_cs7", pcm_spi_cs7),
+	PINCTRL_PIN_GROUP("pnand", pnand),
+	PINCTRL_PIN_GROUP("gpio0", gpio0),
+	PINCTRL_PIN_GROUP("gpio1", gpio1),
+	PINCTRL_PIN_GROUP("gpio2", gpio2),
+	PINCTRL_PIN_GROUP("gpio3", gpio3),
+	PINCTRL_PIN_GROUP("gpio4", gpio4),
+	PINCTRL_PIN_GROUP("gpio5", gpio5),
+	PINCTRL_PIN_GROUP("gpio6", gpio6),
+	PINCTRL_PIN_GROUP("gpio7", gpio7),
+	PINCTRL_PIN_GROUP("gpio8", gpio8),
+	PINCTRL_PIN_GROUP("gpio9", gpio9),
+	PINCTRL_PIN_GROUP("gpio10", gpio10),
+	PINCTRL_PIN_GROUP("gpio11", gpio11),
+	PINCTRL_PIN_GROUP("gpio12", gpio12),
+	PINCTRL_PIN_GROUP("gpio13", gpio13),
+	PINCTRL_PIN_GROUP("gpio14", gpio14),
+	PINCTRL_PIN_GROUP("gpio15", gpio15),
+	PINCTRL_PIN_GROUP("gpio16", gpio16),
+	PINCTRL_PIN_GROUP("gpio17", gpio17),
+	PINCTRL_PIN_GROUP("gpio18", gpio18),
+	PINCTRL_PIN_GROUP("gpio19", gpio19),
+	PINCTRL_PIN_GROUP("gpio20", gpio20),
+	PINCTRL_PIN_GROUP("gpio21", gpio21),
+	PINCTRL_PIN_GROUP("gpio22", gpio22),
+	PINCTRL_PIN_GROUP("gpio23", gpio23),
+	PINCTRL_PIN_GROUP("gpio24", gpio24),
+	PINCTRL_PIN_GROUP("gpio25", gpio25),
+	PINCTRL_PIN_GROUP("gpio26", gpio26),
+	PINCTRL_PIN_GROUP("gpio27", gpio27),
+	PINCTRL_PIN_GROUP("gpio28", gpio28),
+	PINCTRL_PIN_GROUP("gpio29", gpio29),
+	PINCTRL_PIN_GROUP("gpio30", gpio30),
+	PINCTRL_PIN_GROUP("gpio31", gpio31),
+	PINCTRL_PIN_GROUP("gpio32", gpio32),
+	PINCTRL_PIN_GROUP("gpio33", gpio33),
+	PINCTRL_PIN_GROUP("gpio34", gpio34),
+	PINCTRL_PIN_GROUP("gpio35", gpio35),
+	PINCTRL_PIN_GROUP("gpio36", gpio36),
+	PINCTRL_PIN_GROUP("gpio37", gpio37),
+	PINCTRL_PIN_GROUP("gpio38", gpio38),
+	PINCTRL_PIN_GROUP("gpio39", gpio39),
+	PINCTRL_PIN_GROUP("gpio40", gpio40),
+	PINCTRL_PIN_GROUP("gpio41", gpio41),
+	PINCTRL_PIN_GROUP("pcie_reset0", pcie_reset0),
+	PINCTRL_PIN_GROUP("pcie_reset1", pcie_reset1),
+};
+
+static const char *const pon_groups[] = { "pon" };
+static const char *const sipo_groups[] = { "sipo", "sipo_rclk" };
+static const char *const uart_groups[] = { "uart2", "uart2_cts_rts",
+						  "uart3" };
+static const char *const jtag_groups[] = { "ejtag" };
+static const char *const pcm_groups[] = { "pcm1", "pcm2" };
+static const char *const spi_groups[] = { "spi_quad", "spi_cs1" };
+static const char *const pcm_spi_groups[] = { "pcm_spi", "pcm_spi_int",
+						     "pcm_spi_rst",
+						     "pcm_spi_cs1",
+						     "pcm_spi_cs2",
+						     "pcm_spi_cs3",
+						     "pcm_spi_cs4",
+						     "pcm_spi_cs5",
+						     "pcm_spi_cs6",
+						     "pcm_spi_cs7" };
+static const char *const pnand_groups[] = { "pnand" };
+static const char *const pcie_reset_groups[] = { "pcie_reset0",
+							"pcie_reset1" };
+static const char *const gpio_groups[] = { "gpio40", "gpio41" };
+static const char *const pwm_groups[] = { "gpio0", "gpio1",
+						 "gpio2", "gpio3",
+						 "gpio4", "gpio5",
+						 "gpio6", "gpio7",
+						 "gpio8", "gpio9",
+						 "gpio10", "gpio11",
+						 "gpio12", "gpio13",
+						 "gpio14", "gpio15",
+						 "gpio16", "gpio17",
+						 "gpio18", "gpio19",
+						 "gpio20", "gpio21",
+						 "gpio22", "gpio23",
+						 "gpio24", "gpio25",
+						 "gpio26", "gpio27",
+						 "gpio28", "gpio29",
+						 "gpio30", "gpio31",
+						 "gpio32", "gpio37",
+						 "gpio38", "gpio39",
+						 "gpio40", "gpio41" };
+static const char *const phy1_led0_groups[] = { "gpio30", "gpio32",
+						       "gpio34", "gpio36" };
+static const char *const phy2_led0_groups[] = { "gpio30", "gpio32",
+						       "gpio34", "gpio36" };
+static const char *const phy3_led0_groups[] = { "gpio30", "gpio32",
+						       "gpio34", "gpio36" };
+static const char *const phy4_led0_groups[] = { "gpio30", "gpio32",
+						       "gpio34", "gpio36" };
+static const char *const phy1_led1_groups[] = { "gpio31", "gpio33",
+						       "gpio35", "gpio37" };
+static const char *const phy2_led1_groups[] = { "gpio31", "gpio33",
+						       "gpio35", "gpio37" };
+static const char *const phy3_led1_groups[] = { "gpio31", "gpio33",
+						       "gpio35", "gpio37" };
+static const char *const phy4_led1_groups[] = { "gpio31", "gpio33",
+						       "gpio35", "gpio37" };
+
+static const struct airoha_pinctrl_func_group pon_func_group[] = {
+	{
+		.name = "pon",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PON_MODE_MASK,
+			GPIO_PON_MODE_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+static const struct airoha_pinctrl_func_group sipo_func_group[] = {
+	{
+		.name = "sipo",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_SIPO_MODE_MASK | SIPO_RCLK_MODE_MASK,
+			GPIO_SIPO_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "sipo_rclk",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_SIPO_MODE_MASK | SIPO_RCLK_MODE_MASK,
+			GPIO_SIPO_MODE_MASK | SIPO_RCLK_MODE_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+static const struct airoha_pinctrl_func_group uart_func_group[] = {
+	{
+		.name = "uart2",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_UART2_MODE_MASK |
+			GPIO_UART2_CTS_RTS_MODE_MASK,
+			GPIO_UART2_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "uart2_cts_rts",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_UART2_MODE_MASK |
+			GPIO_UART2_CTS_RTS_MODE_MASK,
+			GPIO_UART2_MODE_MASK |
+			GPIO_UART2_CTS_RTS_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "uart3",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_UART3_MODE_MASK,
+			GPIO_UART3_MODE_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+static const struct airoha_pinctrl_func_group jtag_func_group[] = {
+	{
+		.name = "ejtag",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_CPU_EJTAG_EN,
+			CPU_EJTAG_EN_MASK,
+			CPU_EJTAG_EN_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+static const struct airoha_pinctrl_func_group pcm_func_group[] = {
+	{
+		.name = "pcm1",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCM1_MODE_MASK,
+			GPIO_PCM1_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm2",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCM2_MODE_MASK,
+			GPIO_PCM2_MODE_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+static const struct airoha_pinctrl_func_group spi_func_group[] = {
+	{
+		.name = "spi_quad",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_SPI_QUAD_MODE_MASK,
+			GPIO_SPI_QUAD_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "spi_cs1",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_SPI_CS1_MODE_MASK,
+			GPIO_SPI_CS1_MODE_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+static const struct airoha_pinctrl_func_group pcm_spi_func_group[] = {
+	{
+		.name = "pcm_spi",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCM_SPI_MODE_MASK,
+			GPIO_PCM_SPI_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_int",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCM_INT_MODE_MASK,
+			GPIO_PCM_INT_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_rst",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCM_RESET_MODE_MASK,
+			GPIO_PCM_RESET_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_cs1",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCM_SPI_CS1_MODE_MASK,
+			GPIO_PCM_SPI_CS1_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_cs2",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_FORCE_GPIO22_EN,
+			GPIO_PCM_SPI_CS2_MODE_MASK,
+			GPIO_PCM_SPI_CS2_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_cs3",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_FORCE_GPIO22_EN,
+			GPIO_PCM_SPI_CS3_MODE_MASK,
+			GPIO_PCM_SPI_CS3_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_cs4",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_FORCE_GPIO22_EN,
+			GPIO_PCM_SPI_CS4_MODE_MASK,
+			GPIO_PCM_SPI_CS4_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_cs5",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_FORCE_GPIO22_EN,
+			GPIO_PCM_SPI_CS5_MODE_MASK,
+			GPIO_PCM_SPI_CS5_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_cs6",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_FORCE_GPIO22_EN,
+			GPIO_PCM_SPI_CS6_MODE_MASK,
+			GPIO_PCM_SPI_CS6_MODE_MASK
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcm_spi_cs7",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_FORCE_GPIO22_EN,
+			GPIO_PCM_SPI_CS7_MODE_MASK,
+			GPIO_PCM_SPI_CS7_MODE_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+static const struct airoha_pinctrl_func_group pnand_func_group[] = {
+	{
+		.name = "pnand",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_FORCE_GPIO22_EN,
+			GPIO_PNAND_MODE_MASK,
+			GPIO_PNAND_MODE_MASK
+		},
+		.regmap_size = 1,
+	},
+};
+
+/*
+ * On EN7528 a set PCIE_RESET{0,1}_GPIO bit routes the pad to its GPIO
+ * function, so the PCIe reset function must clear it.
+ */
+static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = {
+	{
+		.name = "pcie_reset0",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCIE_RESET0_MASK,
+			0
+		},
+		.regmap_size = 1,
+	}, {
+		.name = "pcie_reset1",
+		.regmap[0] = {
+			AIROHA_FUNC_MUX,
+			REG_PON_I2C_MODE,
+			GPIO_PCIE_RESET1_MASK,
+			0
+		},
+		.regmap_size = 1,
+	},
+};
+
+/* GPIO */
+#define AIROHA_PINCTRL_GPIO(gpio, mux_val)		\
+	{						\
+		.name = (gpio),				\
+		.regmap[0] = {				\
+			AIROHA_FUNC_MUX,		\
+			REG_PON_I2C_MODE,	\
+			(mux_val),			\
+			(mux_val)			\
+		},					\
+		.regmap_size = 1,			\
+	}						\
+
+static const struct airoha_pinctrl_func_group gpio_func_group[] = {
+	AIROHA_PINCTRL_GPIO("gpio40", GPIO_PCIE_RESET0_MASK),
+	AIROHA_PINCTRL_GPIO("gpio41", GPIO_PCIE_RESET1_MASK),
+};
+
+/* PWM - AIROHA_PINCTRL_PWM and AIROHA_PINCTRL_PWM_EXT are shared */
+#define AIROHA_PINCTRL_PWM(gpio, mux_val)		\
+	{						\
+		.name = (gpio),				\
+		.regmap[0] = {				\
+			AIROHA_FUNC_PWM_MUX,		\
+			REG_GPIO_FLASH_MODE_CFG,	\
+			(mux_val),			\
+			(mux_val)			\
+		},					\
+		.regmap_size = 1,			\
+	}						\
+
+#define AIROHA_PINCTRL_PWM_EXT(gpio, mux_val)		\
+	{						\
+		.name = (gpio),				\
+		.regmap[0] = {				\
+			AIROHA_FUNC_PWM_EXT_MUX,	\
+			REG_GPIO_FLASH_MODE_CFG_EXT,	\
+			(mux_val),			\
+			(mux_val)			\
+		},					\
+		.regmap_size = 1,			\
+	}						\
+
+/*
+ * PWM on the PCIE_RESET pads additionally requires the pad to be
+ * routed to its GPIO function via the IOMUX.
+ */
+#define AIROHA_PINCTRL_PWM_EXT_SEC(gpio, mux_val, smux_val)	\
+	{							\
+		.name = (gpio),					\
+		.regmap[0] = {					\
+			AIROHA_FUNC_PWM_EXT_MUX,		\
+			REG_GPIO_FLASH_MODE_CFG_EXT,		\
+			(mux_val),				\
+			(mux_val)				\
+		},						\
+		.regmap[1] = {					\
+			AIROHA_FUNC_MUX,			\
+			REG_PON_I2C_MODE,		\
+			(smux_val),				\
+			(smux_val)				\
+		},						\
+		.regmap_size = 2,				\
+	}
+
+static const struct airoha_pinctrl_func_group pwm_func_group[] = {
+	AIROHA_PINCTRL_PWM("gpio0", GPIO0_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio1", GPIO1_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio2", GPIO2_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio3", GPIO3_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio4", GPIO4_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio5", GPIO5_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio6", GPIO6_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio7", GPIO7_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio8", GPIO8_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio9", GPIO9_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio10", GPIO10_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio11", GPIO11_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio12", GPIO12_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio13", GPIO13_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio14", GPIO14_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM("gpio15", GPIO15_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio16", GPIO16_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio17", GPIO17_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio18", GPIO18_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio19", GPIO19_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio20", GPIO20_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio21", GPIO21_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio22", GPIO22_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio23", GPIO23_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio24", GPIO24_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio25", GPIO25_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio26", GPIO26_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio27", GPIO27_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio28", GPIO28_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio29", GPIO29_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio30", GPIO30_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio31", GPIO31_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio32", GPIO32_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio37", GPIO37_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio38", GPIO38_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT("gpio39", GPIO39_FLASH_MODE_CFG),
+	AIROHA_PINCTRL_PWM_EXT_SEC("gpio40", GPIO40_FLASH_MODE_CFG,
+				   GPIO_PCIE_RESET0_MASK),
+	AIROHA_PINCTRL_PWM_EXT_SEC("gpio41", GPIO41_FLASH_MODE_CFG,
+				   GPIO_PCIE_RESET1_MASK),
+};
+
+#define PINCTRL_PHY_LED0(gpio, mux_val, map_mask, map_val)	\
+	{								\
+		.name = (gpio),						\
+		.regmap[0] = {						\
+			AIROHA_FUNC_MUX,				\
+			REG_PON_I2C_MODE,			\
+			(mux_val),					\
+			(mux_val),					\
+		},							\
+		.regmap[1] = {						\
+			AIROHA_FUNC_MUX,				\
+			REG_LAN_LED0_MAPPING,			\
+			(map_mask),					\
+			(map_val),					\
+		},							\
+		.regmap_size = 2,					\
+	}
+
+#define PINCTRL_PHY_LED1(gpio, mux_val, map_mask, map_val)	\
+	{								\
+		.name = (gpio),						\
+		.regmap[0] = {						\
+			AIROHA_FUNC_MUX,				\
+			REG_PON_I2C_MODE,			\
+			(mux_val),					\
+			(mux_val),					\
+		},							\
+		.regmap[1] = {						\
+			AIROHA_FUNC_MUX,				\
+			REG_LAN_LED1_MAPPING,			\
+			(map_mask),					\
+			(map_val),					\
+		},							\
+		.regmap_size = 2,					\
+	}
+
+/*
+ * LED pad mapping (datasheet table 4-12):
+ * GPIO30: LAN0_LED0, GPIO32: LAN1_LED0, GPIO34: LAN2_LED0, GPIO36: LAN3_LED0
+ * GPIO31: LAN0_LED1, GPIO33: LAN1_LED1, GPIO35: LAN2_LED1, GPIO37: LAN3_LED1
+ */
+static const struct airoha_pinctrl_func_group phy1_led0_func_group[] = {
+	PINCTRL_PHY_LED0("gpio30", GPIO_LAN0_LED0_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(0)),
+	PINCTRL_PHY_LED0("gpio32", GPIO_LAN1_LED0_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(0)),
+	PINCTRL_PHY_LED0("gpio34", GPIO_LAN2_LED0_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(0)),
+	PINCTRL_PHY_LED0("gpio36", GPIO_LAN3_LED0_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(0)),
+};
+
+static const struct airoha_pinctrl_func_group phy2_led0_func_group[] = {
+	PINCTRL_PHY_LED0("gpio30", GPIO_LAN0_LED0_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(1)),
+	PINCTRL_PHY_LED0("gpio32", GPIO_LAN1_LED0_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(1)),
+	PINCTRL_PHY_LED0("gpio34", GPIO_LAN2_LED0_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(1)),
+	PINCTRL_PHY_LED0("gpio36", GPIO_LAN3_LED0_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(1)),
+};
+
+static const struct airoha_pinctrl_func_group phy3_led0_func_group[] = {
+	PINCTRL_PHY_LED0("gpio30", GPIO_LAN0_LED0_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)),
+	PINCTRL_PHY_LED0("gpio32", GPIO_LAN1_LED0_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)),
+	PINCTRL_PHY_LED0("gpio34", GPIO_LAN2_LED0_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)),
+	PINCTRL_PHY_LED0("gpio36", GPIO_LAN3_LED0_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)),
+};
+
+static const struct airoha_pinctrl_func_group phy4_led0_func_group[] = {
+	PINCTRL_PHY_LED0("gpio30", GPIO_LAN0_LED0_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)),
+	PINCTRL_PHY_LED0("gpio32", GPIO_LAN1_LED0_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)),
+	PINCTRL_PHY_LED0("gpio34", GPIO_LAN2_LED0_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)),
+	PINCTRL_PHY_LED0("gpio36", GPIO_LAN3_LED0_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)),
+};
+
+static const struct airoha_pinctrl_func_group phy1_led1_func_group[] = {
+	PINCTRL_PHY_LED1("gpio31", GPIO_LAN0_LED1_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(0)),
+	PINCTRL_PHY_LED1("gpio33", GPIO_LAN1_LED1_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(0)),
+	PINCTRL_PHY_LED1("gpio35", GPIO_LAN2_LED1_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(0)),
+	PINCTRL_PHY_LED1("gpio37", GPIO_LAN3_LED1_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(0)),
+};
+
+static const struct airoha_pinctrl_func_group phy2_led1_func_group[] = {
+	PINCTRL_PHY_LED1("gpio31", GPIO_LAN0_LED1_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(1)),
+	PINCTRL_PHY_LED1("gpio33", GPIO_LAN1_LED1_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(1)),
+	PINCTRL_PHY_LED1("gpio35", GPIO_LAN2_LED1_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(1)),
+	PINCTRL_PHY_LED1("gpio37", GPIO_LAN3_LED1_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(1)),
+};
+
+static const struct airoha_pinctrl_func_group phy3_led1_func_group[] = {
+	PINCTRL_PHY_LED1("gpio31", GPIO_LAN0_LED1_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)),
+	PINCTRL_PHY_LED1("gpio33", GPIO_LAN1_LED1_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)),
+	PINCTRL_PHY_LED1("gpio35", GPIO_LAN2_LED1_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)),
+	PINCTRL_PHY_LED1("gpio37", GPIO_LAN3_LED1_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)),
+};
+
+static const struct airoha_pinctrl_func_group phy4_led1_func_group[] = {
+	PINCTRL_PHY_LED1("gpio31", GPIO_LAN0_LED1_MODE_MASK,
+			 LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)),
+	PINCTRL_PHY_LED1("gpio33", GPIO_LAN1_LED1_MODE_MASK,
+			 LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)),
+	PINCTRL_PHY_LED1("gpio35", GPIO_LAN2_LED1_MODE_MASK,
+			 LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)),
+	PINCTRL_PHY_LED1("gpio37", GPIO_LAN3_LED1_MODE_MASK,
+			 LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)),
+};
+
+static const struct airoha_pinctrl_func pinctrl_funcs[] = {
+	PINCTRL_FUNC_DESC("pon", pon),
+	PINCTRL_FUNC_DESC("sipo", sipo),
+	PINCTRL_FUNC_DESC("uart", uart),
+	PINCTRL_FUNC_DESC("jtag", jtag),
+	PINCTRL_FUNC_DESC("pcm", pcm),
+	PINCTRL_FUNC_DESC("spi", spi),
+	PINCTRL_FUNC_DESC("pcm_spi", pcm_spi),
+	PINCTRL_FUNC_DESC("pnand", pnand),
+	PINCTRL_FUNC_DESC("pcie_reset", pcie_reset),
+	PINCTRL_FUNC_DESC("gpio", gpio),
+	PINCTRL_FUNC_DESC("pwm", pwm),
+	PINCTRL_FUNC_DESC("phy1_led0", phy1_led0),
+	PINCTRL_FUNC_DESC("phy2_led0", phy2_led0),
+	PINCTRL_FUNC_DESC("phy3_led0", phy3_led0),
+	PINCTRL_FUNC_DESC("phy4_led0", phy4_led0),
+	PINCTRL_FUNC_DESC("phy1_led1", phy1_led1),
+	PINCTRL_FUNC_DESC("phy2_led1", phy2_led1),
+	PINCTRL_FUNC_DESC("phy3_led1", phy3_led1),
+	PINCTRL_FUNC_DESC("phy4_led1", phy4_led1),
+};
+
+static const struct airoha_pinctrl_conf pinctrl_pullup_conf[] = {
+	PINCTRL_CONF_DESC(0, REG_GPIO_L_PU, BIT(0)),
+	PINCTRL_CONF_DESC(1, REG_GPIO_L_PU, BIT(1)),
+	PINCTRL_CONF_DESC(2, REG_GPIO_L_PU, BIT(2)),
+	PINCTRL_CONF_DESC(3, REG_GPIO_L_PU, BIT(3)),
+	PINCTRL_CONF_DESC(4, REG_GPIO_L_PU, BIT(4)),
+	PINCTRL_CONF_DESC(5, REG_GPIO_L_PU, BIT(5)),
+	PINCTRL_CONF_DESC(6, REG_GPIO_L_PU, BIT(6)),
+	PINCTRL_CONF_DESC(7, REG_GPIO_L_PU, BIT(7)),
+	PINCTRL_CONF_DESC(8, REG_GPIO_L_PU, BIT(8)),
+	PINCTRL_CONF_DESC(9, REG_GPIO_L_PU, BIT(9)),
+	PINCTRL_CONF_DESC(10, REG_GPIO_L_PU, BIT(10)),
+	PINCTRL_CONF_DESC(11, REG_GPIO_L_PU, BIT(11)),
+	PINCTRL_CONF_DESC(12, REG_GPIO_L_PU, BIT(12)),
+	PINCTRL_CONF_DESC(13, REG_GPIO_L_PU, BIT(13)),
+	PINCTRL_CONF_DESC(14, REG_GPIO_L_PU, BIT(14)),
+	PINCTRL_CONF_DESC(15, REG_GPIO_L_PU, BIT(15)),
+	PINCTRL_CONF_DESC(16, REG_GPIO_L_PU, BIT(16)),
+	PINCTRL_CONF_DESC(17, REG_GPIO_L_PU, BIT(17)),
+	PINCTRL_CONF_DESC(18, REG_GPIO_L_PU, BIT(18)),
+	PINCTRL_CONF_DESC(19, REG_GPIO_L_PU, BIT(19)),
+	PINCTRL_CONF_DESC(20, REG_GPIO_L_PU, BIT(20)),
+	PINCTRL_CONF_DESC(21, REG_GPIO_L_PU, BIT(21)),
+	PINCTRL_CONF_DESC(22, REG_GPIO_L_PU, BIT(22)),
+	PINCTRL_CONF_DESC(23, REG_GPIO_L_PU, BIT(23)),
+	PINCTRL_CONF_DESC(24, REG_GPIO_L_PU, BIT(24)),
+	PINCTRL_CONF_DESC(25, REG_GPIO_L_PU, BIT(25)),
+	PINCTRL_CONF_DESC(26, REG_GPIO_L_PU, BIT(26)),
+	PINCTRL_CONF_DESC(27, REG_GPIO_L_PU, BIT(27)),
+	PINCTRL_CONF_DESC(28, REG_GPIO_L_PU, BIT(28)),
+	PINCTRL_CONF_DESC(29, REG_GPIO_L_PU, BIT(29)),
+	PINCTRL_CONF_DESC(30, REG_GPIO_L_PU, BIT(30)),
+	PINCTRL_CONF_DESC(31, REG_GPIO_L_PU, BIT(31)),
+	PINCTRL_CONF_DESC(32, REG_GPIO_H_PU, BIT(0)),
+	PINCTRL_CONF_DESC(33, REG_GPIO_H_PU, BIT(1)),
+	PINCTRL_CONF_DESC(34, REG_GPIO_H_PU, BIT(2)),
+	PINCTRL_CONF_DESC(35, REG_GPIO_H_PU, BIT(3)),
+	PINCTRL_CONF_DESC(36, REG_GPIO_H_PU, BIT(4)),
+	PINCTRL_CONF_DESC(37, REG_GPIO_H_PU, BIT(5)),
+	PINCTRL_CONF_DESC(38, REG_GPIO_H_PU, BIT(6)),
+	PINCTRL_CONF_DESC(39, REG_GPIO_H_PU, BIT(7)),
+	PINCTRL_CONF_DESC(40, REG_I2C_SDA_PU, PCIE0_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(41, REG_I2C_SDA_PU, PCIE1_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(42, REG_I2C_SDA_PU, I2C_SDA_CONF_MASK),
+	PINCTRL_CONF_DESC(43, REG_I2C_SDA_PU, I2C_SCL_CONF_MASK),
+	PINCTRL_CONF_DESC(44, REG_I2C_SDA_PU, UART_TXD_CONF_MASK),
+	PINCTRL_CONF_DESC(45, REG_I2C_SDA_PU, UART_RXD_CONF_MASK),
+	PINCTRL_CONF_DESC(46, REG_I2C_SDA_PU, SPI_CS0_CONF_MASK),
+	PINCTRL_CONF_DESC(47, REG_I2C_SDA_PU, SPI_CLK_CONF_MASK),
+	PINCTRL_CONF_DESC(48, REG_I2C_SDA_PU, SPI_MOSI_CONF_MASK),
+	PINCTRL_CONF_DESC(49, REG_I2C_SDA_PU, SPI_MISO_CONF_MASK),
+};
+
+static const struct airoha_pinctrl_conf pinctrl_pulldown_conf[] = {
+	PINCTRL_CONF_DESC(0, REG_GPIO_L_PD, BIT(0)),
+	PINCTRL_CONF_DESC(1, REG_GPIO_L_PD, BIT(1)),
+	PINCTRL_CONF_DESC(2, REG_GPIO_L_PD, BIT(2)),
+	PINCTRL_CONF_DESC(3, REG_GPIO_L_PD, BIT(3)),
+	PINCTRL_CONF_DESC(4, REG_GPIO_L_PD, BIT(4)),
+	PINCTRL_CONF_DESC(5, REG_GPIO_L_PD, BIT(5)),
+	PINCTRL_CONF_DESC(6, REG_GPIO_L_PD, BIT(6)),
+	PINCTRL_CONF_DESC(7, REG_GPIO_L_PD, BIT(7)),
+	PINCTRL_CONF_DESC(8, REG_GPIO_L_PD, BIT(8)),
+	PINCTRL_CONF_DESC(9, REG_GPIO_L_PD, BIT(9)),
+	PINCTRL_CONF_DESC(10, REG_GPIO_L_PD, BIT(10)),
+	PINCTRL_CONF_DESC(11, REG_GPIO_L_PD, BIT(11)),
+	PINCTRL_CONF_DESC(12, REG_GPIO_L_PD, BIT(12)),
+	PINCTRL_CONF_DESC(13, REG_GPIO_L_PD, BIT(13)),
+	PINCTRL_CONF_DESC(14, REG_GPIO_L_PD, BIT(14)),
+	PINCTRL_CONF_DESC(15, REG_GPIO_L_PD, BIT(15)),
+	PINCTRL_CONF_DESC(16, REG_GPIO_L_PD, BIT(16)),
+	PINCTRL_CONF_DESC(17, REG_GPIO_L_PD, BIT(17)),
+	PINCTRL_CONF_DESC(18, REG_GPIO_L_PD, BIT(18)),
+	PINCTRL_CONF_DESC(19, REG_GPIO_L_PD, BIT(19)),
+	PINCTRL_CONF_DESC(20, REG_GPIO_L_PD, BIT(20)),
+	PINCTRL_CONF_DESC(21, REG_GPIO_L_PD, BIT(21)),
+	PINCTRL_CONF_DESC(22, REG_GPIO_L_PD, BIT(22)),
+	PINCTRL_CONF_DESC(23, REG_GPIO_L_PD, BIT(23)),
+	PINCTRL_CONF_DESC(24, REG_GPIO_L_PD, BIT(24)),
+	PINCTRL_CONF_DESC(25, REG_GPIO_L_PD, BIT(25)),
+	PINCTRL_CONF_DESC(26, REG_GPIO_L_PD, BIT(26)),
+	PINCTRL_CONF_DESC(27, REG_GPIO_L_PD, BIT(27)),
+	PINCTRL_CONF_DESC(28, REG_GPIO_L_PD, BIT(28)),
+	PINCTRL_CONF_DESC(29, REG_GPIO_L_PD, BIT(29)),
+	PINCTRL_CONF_DESC(30, REG_GPIO_L_PD, BIT(30)),
+	PINCTRL_CONF_DESC(31, REG_GPIO_L_PD, BIT(31)),
+	PINCTRL_CONF_DESC(32, REG_GPIO_H_PD, BIT(0)),
+	PINCTRL_CONF_DESC(33, REG_GPIO_H_PD, BIT(1)),
+	PINCTRL_CONF_DESC(34, REG_GPIO_H_PD, BIT(2)),
+	PINCTRL_CONF_DESC(35, REG_GPIO_H_PD, BIT(3)),
+	PINCTRL_CONF_DESC(36, REG_GPIO_H_PD, BIT(4)),
+	PINCTRL_CONF_DESC(37, REG_GPIO_H_PD, BIT(5)),
+	PINCTRL_CONF_DESC(38, REG_GPIO_H_PD, BIT(6)),
+	PINCTRL_CONF_DESC(39, REG_GPIO_H_PD, BIT(7)),
+	PINCTRL_CONF_DESC(40, REG_I2C_SDA_PD, PCIE0_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(41, REG_I2C_SDA_PD, PCIE1_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(42, REG_I2C_SDA_PD, I2C_SDA_CONF_MASK),
+	PINCTRL_CONF_DESC(43, REG_I2C_SDA_PD, I2C_SCL_CONF_MASK),
+	PINCTRL_CONF_DESC(44, REG_I2C_SDA_PD, UART_TXD_CONF_MASK),
+	PINCTRL_CONF_DESC(45, REG_I2C_SDA_PD, UART_RXD_CONF_MASK),
+	PINCTRL_CONF_DESC(46, REG_I2C_SDA_PD, SPI_CS0_CONF_MASK),
+	PINCTRL_CONF_DESC(47, REG_I2C_SDA_PD, SPI_CLK_CONF_MASK),
+	PINCTRL_CONF_DESC(48, REG_I2C_SDA_PD, SPI_MOSI_CONF_MASK),
+	PINCTRL_CONF_DESC(49, REG_I2C_SDA_PD, SPI_MISO_CONF_MASK),
+};
+
+static const struct airoha_pinctrl_conf pinctrl_drive_e2_conf[] = {
+	PINCTRL_CONF_DESC(0, REG_GPIO_L_E2, BIT(0)),
+	PINCTRL_CONF_DESC(1, REG_GPIO_L_E2, BIT(1)),
+	PINCTRL_CONF_DESC(2, REG_GPIO_L_E2, BIT(2)),
+	PINCTRL_CONF_DESC(3, REG_GPIO_L_E2, BIT(3)),
+	PINCTRL_CONF_DESC(4, REG_GPIO_L_E2, BIT(4)),
+	PINCTRL_CONF_DESC(5, REG_GPIO_L_E2, BIT(5)),
+	PINCTRL_CONF_DESC(6, REG_GPIO_L_E2, BIT(6)),
+	PINCTRL_CONF_DESC(7, REG_GPIO_L_E2, BIT(7)),
+	PINCTRL_CONF_DESC(8, REG_GPIO_L_E2, BIT(8)),
+	PINCTRL_CONF_DESC(9, REG_GPIO_L_E2, BIT(9)),
+	PINCTRL_CONF_DESC(10, REG_GPIO_L_E2, BIT(10)),
+	PINCTRL_CONF_DESC(11, REG_GPIO_L_E2, BIT(11)),
+	PINCTRL_CONF_DESC(12, REG_GPIO_L_E2, BIT(12)),
+	PINCTRL_CONF_DESC(13, REG_GPIO_L_E2, BIT(13)),
+	PINCTRL_CONF_DESC(14, REG_GPIO_L_E2, BIT(14)),
+	PINCTRL_CONF_DESC(15, REG_GPIO_L_E2, BIT(15)),
+	PINCTRL_CONF_DESC(16, REG_GPIO_L_E2, BIT(16)),
+	PINCTRL_CONF_DESC(17, REG_GPIO_L_E2, BIT(17)),
+	PINCTRL_CONF_DESC(18, REG_GPIO_L_E2, BIT(18)),
+	PINCTRL_CONF_DESC(19, REG_GPIO_L_E2, BIT(19)),
+	PINCTRL_CONF_DESC(20, REG_GPIO_L_E2, BIT(20)),
+	PINCTRL_CONF_DESC(21, REG_GPIO_L_E2, BIT(21)),
+	PINCTRL_CONF_DESC(22, REG_GPIO_L_E2, BIT(22)),
+	PINCTRL_CONF_DESC(23, REG_GPIO_L_E2, BIT(23)),
+	PINCTRL_CONF_DESC(24, REG_GPIO_L_E2, BIT(24)),
+	PINCTRL_CONF_DESC(25, REG_GPIO_L_E2, BIT(25)),
+	PINCTRL_CONF_DESC(26, REG_GPIO_L_E2, BIT(26)),
+	PINCTRL_CONF_DESC(27, REG_GPIO_L_E2, BIT(27)),
+	PINCTRL_CONF_DESC(28, REG_GPIO_L_E2, BIT(28)),
+	PINCTRL_CONF_DESC(29, REG_GPIO_L_E2, BIT(29)),
+	PINCTRL_CONF_DESC(30, REG_GPIO_L_E2, BIT(30)),
+	PINCTRL_CONF_DESC(31, REG_GPIO_L_E2, BIT(31)),
+	PINCTRL_CONF_DESC(32, REG_GPIO_H_E2, BIT(0)),
+	PINCTRL_CONF_DESC(33, REG_GPIO_H_E2, BIT(1)),
+	PINCTRL_CONF_DESC(34, REG_GPIO_H_E2, BIT(2)),
+	PINCTRL_CONF_DESC(35, REG_GPIO_H_E2, BIT(3)),
+	PINCTRL_CONF_DESC(36, REG_GPIO_H_E2, BIT(4)),
+	PINCTRL_CONF_DESC(37, REG_GPIO_H_E2, BIT(5)),
+	PINCTRL_CONF_DESC(38, REG_GPIO_H_E2, BIT(6)),
+	PINCTRL_CONF_DESC(39, REG_GPIO_H_E2, BIT(7)),
+	PINCTRL_CONF_DESC(40, REG_I2C_SDA_E2, PCIE0_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(41, REG_I2C_SDA_E2, PCIE1_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(42, REG_I2C_SDA_E2, I2C_SDA_CONF_MASK),
+	PINCTRL_CONF_DESC(43, REG_I2C_SDA_E2, I2C_SCL_CONF_MASK),
+	PINCTRL_CONF_DESC(44, REG_I2C_SDA_E2, UART_TXD_CONF_MASK),
+	PINCTRL_CONF_DESC(45, REG_I2C_SDA_E2, UART_RXD_CONF_MASK),
+	PINCTRL_CONF_DESC(46, REG_I2C_SDA_E2, SPI_CS0_CONF_MASK),
+	PINCTRL_CONF_DESC(47, REG_I2C_SDA_E2, SPI_CLK_CONF_MASK),
+	PINCTRL_CONF_DESC(48, REG_I2C_SDA_E2, SPI_MOSI_CONF_MASK),
+	PINCTRL_CONF_DESC(49, REG_I2C_SDA_E2, SPI_MISO_CONF_MASK),
+};
+
+static const struct airoha_pinctrl_conf pinctrl_drive_e4_conf[] = {
+	PINCTRL_CONF_DESC(0, REG_GPIO_L_E4, BIT(0)),
+	PINCTRL_CONF_DESC(1, REG_GPIO_L_E4, BIT(1)),
+	PINCTRL_CONF_DESC(2, REG_GPIO_L_E4, BIT(2)),
+	PINCTRL_CONF_DESC(3, REG_GPIO_L_E4, BIT(3)),
+	PINCTRL_CONF_DESC(4, REG_GPIO_L_E4, BIT(4)),
+	PINCTRL_CONF_DESC(5, REG_GPIO_L_E4, BIT(5)),
+	PINCTRL_CONF_DESC(6, REG_GPIO_L_E4, BIT(6)),
+	PINCTRL_CONF_DESC(7, REG_GPIO_L_E4, BIT(7)),
+	PINCTRL_CONF_DESC(8, REG_GPIO_L_E4, BIT(8)),
+	PINCTRL_CONF_DESC(9, REG_GPIO_L_E4, BIT(9)),
+	PINCTRL_CONF_DESC(10, REG_GPIO_L_E4, BIT(10)),
+	PINCTRL_CONF_DESC(11, REG_GPIO_L_E4, BIT(11)),
+	PINCTRL_CONF_DESC(12, REG_GPIO_L_E4, BIT(12)),
+	PINCTRL_CONF_DESC(13, REG_GPIO_L_E4, BIT(13)),
+	PINCTRL_CONF_DESC(14, REG_GPIO_L_E4, BIT(14)),
+	PINCTRL_CONF_DESC(15, REG_GPIO_L_E4, BIT(15)),
+	PINCTRL_CONF_DESC(16, REG_GPIO_L_E4, BIT(16)),
+	PINCTRL_CONF_DESC(17, REG_GPIO_L_E4, BIT(17)),
+	PINCTRL_CONF_DESC(18, REG_GPIO_L_E4, BIT(18)),
+	PINCTRL_CONF_DESC(19, REG_GPIO_L_E4, BIT(19)),
+	PINCTRL_CONF_DESC(20, REG_GPIO_L_E4, BIT(20)),
+	PINCTRL_CONF_DESC(21, REG_GPIO_L_E4, BIT(21)),
+	PINCTRL_CONF_DESC(22, REG_GPIO_L_E4, BIT(22)),
+	PINCTRL_CONF_DESC(23, REG_GPIO_L_E4, BIT(23)),
+	PINCTRL_CONF_DESC(24, REG_GPIO_L_E4, BIT(24)),
+	PINCTRL_CONF_DESC(25, REG_GPIO_L_E4, BIT(25)),
+	PINCTRL_CONF_DESC(26, REG_GPIO_L_E4, BIT(26)),
+	PINCTRL_CONF_DESC(27, REG_GPIO_L_E4, BIT(27)),
+	PINCTRL_CONF_DESC(28, REG_GPIO_L_E4, BIT(28)),
+	PINCTRL_CONF_DESC(29, REG_GPIO_L_E4, BIT(29)),
+	PINCTRL_CONF_DESC(30, REG_GPIO_L_E4, BIT(30)),
+	PINCTRL_CONF_DESC(31, REG_GPIO_L_E4, BIT(31)),
+	PINCTRL_CONF_DESC(32, REG_GPIO_H_E4, BIT(0)),
+	PINCTRL_CONF_DESC(33, REG_GPIO_H_E4, BIT(1)),
+	PINCTRL_CONF_DESC(34, REG_GPIO_H_E4, BIT(2)),
+	PINCTRL_CONF_DESC(35, REG_GPIO_H_E4, BIT(3)),
+	PINCTRL_CONF_DESC(36, REG_GPIO_H_E4, BIT(4)),
+	PINCTRL_CONF_DESC(37, REG_GPIO_H_E4, BIT(5)),
+	PINCTRL_CONF_DESC(38, REG_GPIO_H_E4, BIT(6)),
+	PINCTRL_CONF_DESC(39, REG_GPIO_H_E4, BIT(7)),
+	PINCTRL_CONF_DESC(40, REG_I2C_SDA_E4, PCIE0_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(41, REG_I2C_SDA_E4, PCIE1_RESET_CONF_MASK),
+	PINCTRL_CONF_DESC(42, REG_I2C_SDA_E4, I2C_SDA_CONF_MASK),
+	PINCTRL_CONF_DESC(43, REG_I2C_SDA_E4, I2C_SCL_CONF_MASK),
+	PINCTRL_CONF_DESC(44, REG_I2C_SDA_E4, UART_TXD_CONF_MASK),
+	PINCTRL_CONF_DESC(45, REG_I2C_SDA_E4, UART_RXD_CONF_MASK),
+	PINCTRL_CONF_DESC(46, REG_I2C_SDA_E4, SPI_CS0_CONF_MASK),
+	PINCTRL_CONF_DESC(47, REG_I2C_SDA_E4, SPI_CLK_CONF_MASK),
+	PINCTRL_CONF_DESC(48, REG_I2C_SDA_E4, SPI_MOSI_CONF_MASK),
+	PINCTRL_CONF_DESC(49, REG_I2C_SDA_E4, SPI_MISO_CONF_MASK),
+};
+
+static const struct airoha_pinctrl_match_data pinctrl_match_data = {
+	.chip_scu_compatible = "econet,en7528-chip-scu",
+	.pinctrl_name = KBUILD_MODNAME,
+	.pinctrl_owner = THIS_MODULE,
+	.pins = pinctrl_pins,
+	.num_pins = ARRAY_SIZE(pinctrl_pins),
+	.grps = pinctrl_groups,
+	.num_grps = ARRAY_SIZE(pinctrl_groups),
+	.funcs = pinctrl_funcs,
+	.num_funcs = ARRAY_SIZE(pinctrl_funcs),
+	/* only GPIO0-GPIO15 can raise an interrupt */
+	.num_irq_pins = 16,
+	.confs_info = {
+		[AIROHA_PINCTRL_CONFS_PULLUP] = {
+			.confs = pinctrl_pullup_conf,
+			.num_confs = ARRAY_SIZE(pinctrl_pullup_conf),
+		},
+		[AIROHA_PINCTRL_CONFS_PULLDOWN] = {
+			.confs = pinctrl_pulldown_conf,
+			.num_confs = ARRAY_SIZE(pinctrl_pulldown_conf),
+		},
+		[AIROHA_PINCTRL_CONFS_DRIVE_E2] = {
+			.confs = pinctrl_drive_e2_conf,
+			.num_confs = ARRAY_SIZE(pinctrl_drive_e2_conf),
+		},
+		[AIROHA_PINCTRL_CONFS_DRIVE_E4] = {
+			.confs = pinctrl_drive_e4_conf,
+			.num_confs = ARRAY_SIZE(pinctrl_drive_e4_conf),
+		},
+		/* no PCIe reset open drain register on EN7528 */
+	},
+};
+
+static const struct of_device_id airoha_pinctrl_of_match[] = {
+	{ .compatible = "econet,en7528-pinctrl", .data = &pinctrl_match_data },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, airoha_pinctrl_of_match);
+
+static struct platform_driver airoha_pinctrl_driver = {
+	.probe = airoha_pinctrl_probe,
+	.driver = {
+		.name = "pinctrl-airoha-en7528",
+		.of_match_table = airoha_pinctrl_of_match,
+	},
+};
+module_platform_driver(airoha_pinctrl_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Benjamin Larsson <benjamin.larsson@genexis.eu>");
+MODULE_DESCRIPTION("Pinctrl driver for EcoNet EN7528 SoC");
diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c
index 1675dd36bd5cec..ba6c71e6fc2c29 100644
--- a/drivers/pinctrl/core.c
+++ b/drivers/pinctrl/core.c
@@ -147,7 +147,7 @@ int pin_get_from_name(struct pinctrl_dev *pctldev, const char *name)
 {
 	unsigned int i, pin;
 
-	/* The pin number can be retrived from the pin controller descriptor */
+	/* The pin number can be retrieved from the pin controller descriptor */
 	for (i = 0; i < pctldev->desc->npins; i++) {
 		struct pin_desc *desc;
 
@@ -1747,7 +1747,7 @@ static int pinctrl_pins_show(struct seq_file *s, void *what)
 
 	mutex_lock(&pctldev->mutex);
 
-	/* The pin number can be retrived from the pin controller descriptor */
+	/* The pin number can be retrieved from the pin controller descriptor */
 	for (i = 0; i < pctldev->desc->npins; i++) {
 		struct pin_desc *desc;
 
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c
index 9a45b376d36f9a..f08ef52bf4f85f 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx.c
@@ -21,6 +21,7 @@
 
 #include <linux/pinctrl/machine.h>
 #include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/pinctrl/pinmux.h>
 
@@ -291,8 +292,8 @@ struct pinmux_ops imx_pmx_ops = {
 	.set_mux = imx_pmx_set,
 };
 
-static int imx_pinconf_get_mmio(struct pinctrl_dev *pctldev, unsigned pin_id,
-				unsigned long *config)
+static int imx_pinconf_get_raw_mmio(struct pinctrl_dev *pctldev,
+				    unsigned int pin_id, unsigned long *config)
 {
 	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
 	const struct imx_pinctrl_soc_info *info = ipctl->info;
@@ -312,16 +313,61 @@ static int imx_pinconf_get_mmio(struct pinctrl_dev *pctldev, unsigned pin_id,
 	return 0;
 }
 
+static int imx_pinconf_get_mmio(struct pinctrl_dev *pctldev,
+				unsigned int pin_id, unsigned long *config)
+{
+	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct imx_pinctrl_soc_info *info = ipctl->info;
+	const struct imx_pin_reg *pin_reg = &ipctl->pin_regs[pin_id];
+	enum pin_config_param param = pinconf_to_config_param(*config);
+	unsigned int mask;
+	u32 raw;
+
+	switch (param) {
+	case PIN_CONFIG_OUTPUT_ENABLE:
+		mask = info->obe_mask;
+		break;
+	case PIN_CONFIG_INPUT_ENABLE:
+		mask = info->ibe_mask;
+		break;
+	default:
+		mask = 0;
+		break;
+	}
+
+	if (!mask)
+		return -ENOTSUPP;
+	if (pin_reg->conf_reg == -1)
+		return -EINVAL;
+
+	raw = readl(ipctl->base + pin_reg->conf_reg);
+	*config = pinconf_to_config_packed(param, !!(raw & mask));
+
+	return 0;
+}
+
 static int imx_pinconf_get(struct pinctrl_dev *pctldev,
 			   unsigned pin_id, unsigned long *config)
 {
 	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
 	const struct imx_pinctrl_soc_info *info = ipctl->info;
 
+	if (info->flags & IMX_USE_SCU)
+		return -ENOTSUPP;
+
+	return imx_pinconf_get_mmio(pctldev, pin_id, config);
+}
+
+static int imx_pinconf_get_raw(struct pinctrl_dev *pctldev,
+			       unsigned int pin_id, unsigned long *config)
+{
+	struct imx_pinctrl *ipctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct imx_pinctrl_soc_info *info = ipctl->info;
+
 	if (info->flags & IMX_USE_SCU)
 		return info->imx_pinconf_get(pctldev, pin_id, config);
 	else
-		return imx_pinconf_get_mmio(pctldev, pin_id, config);
+		return imx_pinconf_get_raw_mmio(pctldev, pin_id, config);
 }
 
 static int imx_pinconf_set_mmio(struct pinctrl_dev *pctldev,
@@ -426,7 +472,7 @@ static void imx_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
 		struct imx_pin *pin = &((struct imx_pin *)(grp->data))[i];
 
 		name = pin_get_name(pctldev, pin->pin);
-		ret = imx_pinconf_get(pctldev, pin->pin, &config);
+		ret = imx_pinconf_get_raw(pctldev, pin->pin, &config);
 		if (ret)
 			return;
 		seq_printf(s, "  %s: 0x%lx\n", name, config);
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.h b/drivers/pinctrl/freescale/pinctrl-imx.h
index f65ff45b40038d..8fa7e1e2521d20 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.h
+++ b/drivers/pinctrl/freescale/pinctrl-imx.h
@@ -91,6 +91,10 @@ struct imx_pinctrl_soc_info {
 	unsigned int mux_mask;
 	u8 mux_shift;
 
+	/* OBE/IBE bits in the conf register, 0 if the pad does not have them */
+	unsigned int obe_mask;
+	unsigned int ibe_mask;
+
 	int (*gpio_set_direction)(struct pinctrl_dev *pctldev,
 				  struct pinctrl_gpio_range *range,
 				  unsigned offset,
diff --git a/drivers/pinctrl/freescale/pinctrl-vf610.c b/drivers/pinctrl/freescale/pinctrl-vf610.c
index 76a4bc0181a044..77d07761878299 100644
--- a/drivers/pinctrl/freescale/pinctrl-vf610.c
+++ b/drivers/pinctrl/freescale/pinctrl-vf610.c
@@ -319,6 +319,8 @@ static const struct imx_pinctrl_soc_info vf610_pinctrl_info = {
 	.gpio_set_direction = vf610_pmx_gpio_set_direction,
 	.mux_mask = 0x700000,
 	.mux_shift = 20,
+	.obe_mask = 0x2,
+	.ibe_mask = 0x1,
 };
 
 static const struct of_device_id vf610_pinctrl_of_match[] = {
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 261ec2ee63ef5b..1363301098d5a6 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -1702,35 +1702,42 @@ int intel_pinctrl_probe(struct platform_device *pdev,
 }
 EXPORT_SYMBOL_NS_GPL(intel_pinctrl_probe, "PINCTRL_INTEL");
 
-int intel_pinctrl_probe_by_hid(struct platform_device *pdev)
+static const struct intel_pinctrl_soc_data *
+intel_pinctrl_get_soc_data_by_hid(const struct platform_device *pdev)
 {
 	const struct intel_pinctrl_soc_data *data;
+	const struct device *dev = &pdev->dev;
+	const struct platform_device_id *id;
 
-	data = device_get_match_data(&pdev->dev);
-	if (!data)
-		return -ENODATA;
+	data = device_get_match_data(dev);
+	if (data)
+		return data;
 
-	return intel_pinctrl_probe(pdev, data);
+	id = platform_get_device_id(pdev);
+	if (!id)
+		return ERR_PTR(-ENODEV);
+
+	data = (const struct intel_pinctrl_soc_data *)id->driver_data;
+	return data ?: ERR_PTR(-ENODATA);
 }
-EXPORT_SYMBOL_NS_GPL(intel_pinctrl_probe_by_hid, "PINCTRL_INTEL");
 
-int intel_pinctrl_probe_by_uid(struct platform_device *pdev)
+int intel_pinctrl_probe_by_hid(struct platform_device *pdev)
 {
 	const struct intel_pinctrl_soc_data *data;
 
-	data = intel_pinctrl_get_soc_data(pdev);
+	data = intel_pinctrl_get_soc_data_by_hid(pdev);
 	if (IS_ERR(data))
 		return PTR_ERR(data);
 
 	return intel_pinctrl_probe(pdev, data);
 }
-EXPORT_SYMBOL_NS_GPL(intel_pinctrl_probe_by_uid, "PINCTRL_INTEL");
+EXPORT_SYMBOL_NS_GPL(intel_pinctrl_probe_by_hid, "PINCTRL_INTEL");
 
-const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(struct platform_device *pdev)
+const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(const struct platform_device *pdev)
 {
 	const struct intel_pinctrl_soc_data * const *table;
 	const struct intel_pinctrl_soc_data *data;
-	struct device *dev = &pdev->dev;
+	const struct device *dev = &pdev->dev;
 
 	table = device_get_match_data(dev);
 	if (table) {
@@ -1757,6 +1764,18 @@ const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(struct platform_
 }
 EXPORT_SYMBOL_NS_GPL(intel_pinctrl_get_soc_data, "PINCTRL_INTEL");
 
+int intel_pinctrl_probe_by_uid(struct platform_device *pdev)
+{
+	const struct intel_pinctrl_soc_data *data;
+
+	data = intel_pinctrl_get_soc_data(pdev);
+	if (IS_ERR(data))
+		return PTR_ERR(data);
+
+	return intel_pinctrl_probe(pdev, data);
+}
+EXPORT_SYMBOL_NS_GPL(intel_pinctrl_probe_by_uid, "PINCTRL_INTEL");
+
 static bool __intel_gpio_is_direct_irq(u32 value)
 {
 	return (value & PADCFG0_GPIROUTIOXAPIC) &&
diff --git a/drivers/pinctrl/intel/pinctrl-intel.h b/drivers/pinctrl/intel/pinctrl-intel.h
index b5476b9de0dbf4..b7ee7fd0c5f26b 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.h
+++ b/drivers/pinctrl/intel/pinctrl-intel.h
@@ -225,7 +225,7 @@ struct intel_pinctrl_soc_data {
 	size_t ncommunities;
 };
 
-const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(struct platform_device *pdev);
+const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(const struct platform_device *pdev);
 
 struct intel_pad_context;
 struct intel_community_context;
diff --git a/drivers/pinctrl/meson/pinctrl-meson-s4.c b/drivers/pinctrl/meson/pinctrl-meson-s4.c
index 872948699e9fee..365dafe457a9f1 100644
--- a/drivers/pinctrl/meson/pinctrl-meson-s4.c
+++ b/drivers/pinctrl/meson/pinctrl-meson-s4.c
@@ -854,7 +854,7 @@ static const char * const i2c1_groups[] = {
 static const char * const i2c2_groups[] = {
 	"i2c2_sda_d", "i2c2_scl_d",
 	"i2c2_sda_h8", "i2c2_scl_h9",
-	"i2c2_sda_h0", "i2c2_scl_h1l,"
+	"i2c2_sda_h0", "i2c2_scl_h1",
 };
 
 static const char * const i2c3_groups[] = {
diff --git a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c
index ea1026a0d22c8d..8da0c5312ce376 100644
--- a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c
+++ b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c
@@ -86,7 +86,7 @@ static struct mpfs_pinctrl_bank_voltage mpfs_pinctrl_bank_voltages[8] = {
 	{ .uv = 1800000, .val = 4 },
 	{ .uv = 2500000, .val = 6 },
 	{ .uv = 3300000, .val = 8 },
-	{ .uv = 0, .val = 0x3f }, // pin unused
+	{ .uv = 0, .val = 0xf }, // pin unused
 };
 
 static int mpfs_pinctrl_get_drive_strength_ma(u32 drive_strength)
@@ -156,10 +156,10 @@ static void mpfs_pinctrl_set_bank_voltage(struct mpfs_pinctrl *pctrl, unsigned i
 	u32 val = FIELD_PREP(MPFS_PINCTRL_BANK_VOLTAGE_MASK, bank_voltage);
 
 	if (pin < MPFS_PINCTRL_BANK2_START)
-		regmap_assign_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK4_CFG_CR,
+		regmap_update_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK4_CFG_CR,
 				   MPFS_PINCTRL_BANK_VOLTAGE_MASK, val);
 	else
-		regmap_assign_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK2_CFG_CR,
+		regmap_update_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK2_CFG_CR,
 				   MPFS_PINCTRL_BANK_VOLTAGE_MASK, val);
 }
 
@@ -612,7 +612,7 @@ static void mpfs_pinctrl_pinconf_dbg_show(struct pinctrl_dev *pctrl_dev, struct
 	reg = mpfs_pinctrl_pin_to_iocfg_reg(pin);
 	offset = mpfs_pinctrl_pin_to_iocfg_offset(pin);
 
-	seq_printf(seq, "pin: %u ", pin);
+	seq_printf(seq, " pin: %u ", pin);
 	seq_printf(seq, "reg: %x offset: %u ", reg, offset);
 
 	if (reg < 0 || offset < 0)
diff --git a/drivers/pinctrl/pinconf.c b/drivers/pinctrl/pinconf.c
index 81686844dfa5f1..ecc888b029d761 100644
--- a/drivers/pinctrl/pinconf.c
+++ b/drivers/pinctrl/pinconf.c
@@ -311,7 +311,7 @@ static int pinconf_pins_show(struct seq_file *s, void *what)
 
 	mutex_lock(&pctldev->mutex);
 
-	/* The pin number can be retrived from the pin controller descriptor */
+	/* The pin number can be retrieved from the pin controller descriptor */
 	for (i = 0; i < pctldev->desc->npins; i++) {
 		struct pin_desc *desc;
 
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index 15a398bb3be23e..65728b59099957 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -18,6 +18,7 @@
 #include <linux/errno.h>
 #include <linux/log2.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/gpio/driver.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
@@ -494,7 +495,7 @@ static void amd_gpio_irq_eoi(struct irq_data *d)
 static int amd_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	int ret = 0;
-	u32 pin_reg, pin_reg_irq_en, mask;
+	u32 pin_reg, pin_reg_irq_en, mask, reg;
 	unsigned long flags;
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct amd_gpio *gpio_dev = gpiochip_get_data(gc);
@@ -568,8 +569,9 @@ static int amd_gpio_irq_set_type(struct irq_data *d, unsigned int type)
 	pin_reg_irq_en |= mask;
 	pin_reg_irq_en &= ~BIT(INTERRUPT_MASK_OFF);
 	writel(pin_reg_irq_en, gpio_dev->base + hwirq * 4);
-	while ((readl(gpio_dev->base + hwirq * 4) & mask) != mask)
-		continue;
+	if (readl_poll_timeout_atomic(gpio_dev->base + hwirq * 4, reg,
+				      (reg & mask) == mask, 1, 1000))
+		ret = -ETIMEDOUT;
 	writel(pin_reg, gpio_dev->base + hwirq * 4);
 	raw_spin_unlock_irqrestore(&gpio_dev->lock, flags);
 
diff --git a/drivers/pinctrl/pinctrl-equilibrium.c b/drivers/pinctrl/pinctrl-equilibrium.c
index 349eb944b4ac88..8e6cd35fc80cc7 100644
--- a/drivers/pinctrl/pinctrl-equilibrium.c
+++ b/drivers/pinctrl/pinctrl-equilibrium.c
@@ -618,7 +618,7 @@ static int funcs_utils(struct device *dev, struct pinfunction *funcs,
 
 		if (of_property_read_string(np, "function", &fn_name)) {
 			/* some groups may not have function, it's OK */
-			dev_dbg(dev, "Group %s: not function binded!\n",
+			dev_dbg(dev, "Group %s: not function bound!\n",
 				(char *)prop->value);
 			continue;
 		}
diff --git a/drivers/pinctrl/pinctrl-generic.c b/drivers/pinctrl/pinctrl-generic.c
index fd6bdb74028aa5..4277c87485135c 100644
--- a/drivers/pinctrl/pinctrl-generic.c
+++ b/drivers/pinctrl/pinctrl-generic.c
@@ -3,8 +3,10 @@
 #define pr_fmt(fmt) "generic pinconfig core: " fmt
 
 #include <linux/array_size.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/slab.h>
 
@@ -196,6 +198,8 @@ static int pinctrl_generic_dt_node_to_map(struct pinctrl_dev *pctldev,
 	int ngroups = 0;
 	int ret;
 
+	guard(mutex)(&pctldev->mutex);
+
 	*maps = NULL;
 	*num_maps = 0;
 
diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c
index 4d5f85b7e6bbe2..e0eb7240a98595 100644
--- a/drivers/pinctrl/pinctrl-single.c
+++ b/drivers/pinctrl/pinctrl-single.c
@@ -1628,7 +1628,8 @@ static int pcs_irq_init_chained_handler(struct pcs_device *pcs,
 					       &pcs_irqdomain_ops,
 					       pcs_soc);
 	if (!pcs->domain) {
-		irq_set_chained_handler(pcs_soc->irq, NULL);
+		pcs_irq_free(pcs);
+		pcs_soc->irq = -1;
 		return -EINVAL;
 	}
 
diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c
index 8ce88e591f475a..59ecc295fb8ee7 100644
--- a/drivers/pinctrl/pinctrl-st.c
+++ b/drivers/pinctrl/pinctrl-st.c
@@ -1148,7 +1148,7 @@ static void st_parse_syscfgs(struct st_pinctrl *info, int bank,
 	pc->pu = st_pc_get_value(dev, regmap, bank/4, data->pu, lsb, msb);
 	pc->od = st_pc_get_value(dev, regmap, bank/4, data->od, lsb, msb);
 
-	/* retime avaiable for all pins by default */
+	/* retime available for all pins by default */
 	pc->rt_pin_mask = 0xff;
 	of_property_read_u32(np, "st,retime-pin-mask", &pc->rt_pin_mask);
 	st_pctl_dt_setup_retime(info, bank, pc);
diff --git a/drivers/pinctrl/pinctrl-sx150x.c b/drivers/pinctrl/pinctrl-sx150x.c
index 04a0e6269e5bc4..def21c25605d7e 100644
--- a/drivers/pinctrl/pinctrl-sx150x.c
+++ b/drivers/pinctrl/pinctrl-sx150x.c
@@ -17,6 +17,7 @@
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/mutex.h>
+#include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/gpio/driver.h>
diff --git a/drivers/pinctrl/pinmux.c b/drivers/pinctrl/pinmux.c
index c705bc182266c5..574fc95b6712e9 100644
--- a/drivers/pinctrl/pinmux.c
+++ b/drivers/pinctrl/pinmux.c
@@ -636,7 +636,7 @@ static int pinmux_pins_show(struct seq_file *s, void *what)
 
 	mutex_lock(&pctldev->mutex);
 
-	/* The pin number can be retrived from the pin controller descriptor */
+	/* The pin number can be retrieved from the pin controller descriptor */
 	for (i = 0; i < pctldev->desc->npins; i++) {
 		struct pin_desc *desc;
 		bool is_hog = false;
diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig
index 18db350222b997..707067b9251f04 100644
--- a/drivers/pinctrl/qcom/Kconfig
+++ b/drivers/pinctrl/qcom/Kconfig
@@ -62,6 +62,16 @@ config PINCTRL_LPASS_LPI
 	  Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SoCs.
 
+config PINCTRL_HAWI_LPASS_LPI
+	tristate "Qualcomm Technologies Inc Hawi LPASS LPI pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	depends on PINCTRL_LPASS_LPI
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
+	  (Low Power Island) found on the Qualcomm Technologies Inc Hawi
+	  platform.
+
 config PINCTRL_MILOS_LPASS_LPI
 	tristate "Qualcomm Milos LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
diff --git a/drivers/pinctrl/qcom/Kconfig.msm b/drivers/pinctrl/qcom/Kconfig.msm
index 42875457b5fc19..e45c701db627cf 100644
--- a/drivers/pinctrl/qcom/Kconfig.msm
+++ b/drivers/pinctrl/qcom/Kconfig.msm
@@ -153,6 +153,17 @@ config PINCTRL_KAANAPALI
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc Kaanapali platform.
 
+config PINCTRL_KUNO
+	tristate "Qualcomm Kuno pin controller driver"
+	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
+	  Technologies Inc Kuno platform.
+	  Say Y here to compile statically, or M here to compile it as a
+	  module. If unsure, say N.
+
 config PINCTRL_MAILI
 	tristate "Qualcomm Technologies Inc Maili pin controller driver"
 	depends on ARM64 || COMPILE_TEST
@@ -232,6 +243,14 @@ config PINCTRL_MSM8917
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found on the Qualcomm MSM8917, MSM8937 platform.
 
+config PINCTRL_MSM8952
+	tristate "Qualcomm 8952 pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm TLMM block found on the Qualcomm MSM8952 platform.
+
 config PINCTRL_MSM8953
 	tristate "Qualcomm 8953 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile
index 43ecd246afe884..a3daca77a4487f 100644
--- a/drivers/pinctrl/qcom/Makefile
+++ b/drivers/pinctrl/qcom/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_PINCTRL_APQ8084)	+= pinctrl-apq8084.o
 obj-$(CONFIG_PINCTRL_ELIZA)	+= pinctrl-eliza.o
 obj-$(CONFIG_PINCTRL_GLYMUR)	+= pinctrl-glymur.o
 obj-$(CONFIG_PINCTRL_HAWI)	+= pinctrl-hawi.o
+obj-$(CONFIG_PINCTRL_HAWI_LPASS_LPI) += pinctrl-hawi-lpass-lpi.o
 obj-$(CONFIG_PINCTRL_IPQ4019)	+= pinctrl-ipq4019.o
 obj-$(CONFIG_PINCTRL_IPQ5018)	+= pinctrl-ipq5018.o
 obj-$(CONFIG_PINCTRL_IPQ8064)	+= pinctrl-ipq8064.o
@@ -17,6 +18,7 @@ obj-$(CONFIG_PINCTRL_IPQ6018)	+= pinctrl-ipq6018.o
 obj-$(CONFIG_PINCTRL_IPQ9574)	+= pinctrl-ipq9574.o
 obj-$(CONFIG_PINCTRL_IPQ9650)	+= pinctrl-ipq9650.o
 obj-$(CONFIG_PINCTRL_KAANAPALI) += pinctrl-kaanapali.o
+obj-$(CONFIG_PINCTRL_KUNO)	+= pinctrl-kuno.o
 obj-$(CONFIG_PINCTRL_MAILI)	+= pinctrl-maili.o
 obj-$(CONFIG_PINCTRL_MSM8226)	+= pinctrl-msm8226.o
 obj-$(CONFIG_PINCTRL_MSM8660)	+= pinctrl-msm8660.o
@@ -25,6 +27,7 @@ obj-$(CONFIG_PINCTRL_MSM8X74)	+= pinctrl-msm8x74.o
 obj-$(CONFIG_PINCTRL_MSM8909)	+= pinctrl-msm8909.o
 obj-$(CONFIG_PINCTRL_MSM8916)	+= pinctrl-msm8916.o
 obj-$(CONFIG_PINCTRL_MSM8917)	+= pinctrl-msm8917.o
+obj-$(CONFIG_PINCTRL_MSM8952)	+= pinctrl-msm8952.o
 obj-$(CONFIG_PINCTRL_MSM8953)	+= pinctrl-msm8953.o
 obj-$(CONFIG_PINCTRL_MSM8976)	+= pinctrl-msm8976.o
 obj-$(CONFIG_PINCTRL_MSM8994)   += pinctrl-msm8994.o
diff --git a/drivers/pinctrl/qcom/pinctrl-apq8064.c b/drivers/pinctrl/qcom/pinctrl-apq8064.c
index a47f3d4f1ea509..acfe1c55416258 100644
--- a/drivers/pinctrl/qcom/pinctrl-apq8064.c
+++ b/drivers/pinctrl/qcom/pinctrl-apq8064.c
@@ -214,17 +214,17 @@ static const unsigned int sdc3_data_pins[] = { 95 };
 			gpio##id##_pins, 		\
 			ARRAY_SIZE(gpio##id##_pins)),	\
 		.funcs = (int[]){			\
-			APQ_MUX_gpio,			\
-			APQ_MUX_##f1,			\
-			APQ_MUX_##f2,			\
-			APQ_MUX_##f3,			\
-			APQ_MUX_##f4,			\
-			APQ_MUX_##f5,			\
-			APQ_MUX_##f6,			\
-			APQ_MUX_##f7,			\
-			APQ_MUX_##f8,			\
-			APQ_MUX_##f9,			\
-			APQ_MUX_##f10,			\
+			msm_mux_gpio,			\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9,			\
+			msm_mux_##f10,			\
 		},					\
 		.nfuncs = 11,				\
 		.ctl_reg = 0x1000 + 0x10 * id,		\
@@ -276,48 +276,48 @@ static const unsigned int sdc3_data_pins[] = { 95 };
 	}
 
 enum apq8064_functions {
-	APQ_MUX_cam_mclk,
-	APQ_MUX_codec_mic_i2s,
-	APQ_MUX_codec_spkr_i2s,
-	APQ_MUX_gp_clk_0a,
-	APQ_MUX_gp_clk_0b,
-	APQ_MUX_gp_clk_1a,
-	APQ_MUX_gp_clk_1b,
-	APQ_MUX_gp_clk_2a,
-	APQ_MUX_gp_clk_2b,
-	APQ_MUX_gpio,
-	APQ_MUX_gsbi1,
-	APQ_MUX_gsbi2,
-	APQ_MUX_gsbi3,
-	APQ_MUX_gsbi4,
-	APQ_MUX_gsbi4_cam_i2c,
-	APQ_MUX_gsbi5,
-	APQ_MUX_gsbi5_spi_cs1,
-	APQ_MUX_gsbi5_spi_cs2,
-	APQ_MUX_gsbi5_spi_cs3,
-	APQ_MUX_gsbi6,
-	APQ_MUX_gsbi6_spi_cs1,
-	APQ_MUX_gsbi6_spi_cs2,
-	APQ_MUX_gsbi6_spi_cs3,
-	APQ_MUX_gsbi7,
-	APQ_MUX_gsbi7_spi_cs1,
-	APQ_MUX_gsbi7_spi_cs2,
-	APQ_MUX_gsbi7_spi_cs3,
-	APQ_MUX_gsbi_cam_i2c,
-	APQ_MUX_hdmi,
-	APQ_MUX_mi2s,
-	APQ_MUX_riva_bt,
-	APQ_MUX_riva_fm,
-	APQ_MUX_riva_wlan,
-	APQ_MUX_sdc2,
-	APQ_MUX_sdc4,
-	APQ_MUX_slimbus,
-	APQ_MUX_spkr_i2s,
-	APQ_MUX_tsif1,
-	APQ_MUX_tsif2,
-	APQ_MUX_usb2_hsic,
-	APQ_MUX_ps_hold,
-	APQ_MUX_NA,
+	msm_mux_cam_mclk,
+	msm_mux_codec_mic_i2s,
+	msm_mux_codec_spkr_i2s,
+	msm_mux_gp_clk_0a,
+	msm_mux_gp_clk_0b,
+	msm_mux_gp_clk_1a,
+	msm_mux_gp_clk_1b,
+	msm_mux_gp_clk_2a,
+	msm_mux_gp_clk_2b,
+	msm_mux_gpio,
+	msm_mux_gsbi1,
+	msm_mux_gsbi2,
+	msm_mux_gsbi3,
+	msm_mux_gsbi4,
+	msm_mux_gsbi4_cam_i2c,
+	msm_mux_gsbi5,
+	msm_mux_gsbi5_spi_cs1,
+	msm_mux_gsbi5_spi_cs2,
+	msm_mux_gsbi5_spi_cs3,
+	msm_mux_gsbi6,
+	msm_mux_gsbi6_spi_cs1,
+	msm_mux_gsbi6_spi_cs2,
+	msm_mux_gsbi6_spi_cs3,
+	msm_mux_gsbi7,
+	msm_mux_gsbi7_spi_cs1,
+	msm_mux_gsbi7_spi_cs2,
+	msm_mux_gsbi7_spi_cs3,
+	msm_mux_gsbi_cam_i2c,
+	msm_mux_hdmi,
+	msm_mux_mi2s,
+	msm_mux_riva_bt,
+	msm_mux_riva_fm,
+	msm_mux_riva_wlan,
+	msm_mux_sdc2,
+	msm_mux_sdc4,
+	msm_mux_slimbus,
+	msm_mux_spkr_i2s,
+	msm_mux_tsif1,
+	msm_mux_tsif2,
+	msm_mux_usb2_hsic,
+	msm_mux_ps_hold,
+	msm_mux_NA,
 };
 
 static const char * const cam_mclk_groups[] = {
@@ -457,47 +457,47 @@ static const char * const usb2_hsic_groups[] = {
 };
 
 static const struct pinfunction apq8064_functions[] = {
-	APQ_PIN_FUNCTION(cam_mclk),
-	APQ_PIN_FUNCTION(codec_mic_i2s),
-	APQ_PIN_FUNCTION(codec_spkr_i2s),
-	APQ_PIN_FUNCTION(gp_clk_0a),
-	APQ_PIN_FUNCTION(gp_clk_0b),
-	APQ_PIN_FUNCTION(gp_clk_1a),
-	APQ_PIN_FUNCTION(gp_clk_1b),
-	APQ_PIN_FUNCTION(gp_clk_2a),
-	APQ_PIN_FUNCTION(gp_clk_2b),
-	APQ_PIN_FUNCTION(gpio),
-	APQ_PIN_FUNCTION(gsbi1),
-	APQ_PIN_FUNCTION(gsbi2),
-	APQ_PIN_FUNCTION(gsbi3),
-	APQ_PIN_FUNCTION(gsbi4),
-	APQ_PIN_FUNCTION(gsbi4_cam_i2c),
-	APQ_PIN_FUNCTION(gsbi5),
-	APQ_PIN_FUNCTION(gsbi5_spi_cs1),
-	APQ_PIN_FUNCTION(gsbi5_spi_cs2),
-	APQ_PIN_FUNCTION(gsbi5_spi_cs3),
-	APQ_PIN_FUNCTION(gsbi6),
-	APQ_PIN_FUNCTION(gsbi6_spi_cs1),
-	APQ_PIN_FUNCTION(gsbi6_spi_cs2),
-	APQ_PIN_FUNCTION(gsbi6_spi_cs3),
-	APQ_PIN_FUNCTION(gsbi7),
-	APQ_PIN_FUNCTION(gsbi7_spi_cs1),
-	APQ_PIN_FUNCTION(gsbi7_spi_cs2),
-	APQ_PIN_FUNCTION(gsbi7_spi_cs3),
-	APQ_PIN_FUNCTION(gsbi_cam_i2c),
-	APQ_PIN_FUNCTION(hdmi),
-	APQ_PIN_FUNCTION(mi2s),
-	APQ_PIN_FUNCTION(riva_bt),
-	APQ_PIN_FUNCTION(riva_fm),
-	APQ_PIN_FUNCTION(riva_wlan),
-	APQ_PIN_FUNCTION(sdc2),
-	APQ_PIN_FUNCTION(sdc4),
-	APQ_PIN_FUNCTION(slimbus),
-	APQ_PIN_FUNCTION(spkr_i2s),
-	APQ_PIN_FUNCTION(tsif1),
-	APQ_PIN_FUNCTION(tsif2),
-	APQ_PIN_FUNCTION(usb2_hsic),
-	APQ_PIN_FUNCTION(ps_hold),
+	MSM_PIN_FUNCTION(cam_mclk),
+	MSM_PIN_FUNCTION(codec_mic_i2s),
+	MSM_PIN_FUNCTION(codec_spkr_i2s),
+	MSM_PIN_FUNCTION(gp_clk_0a),
+	MSM_PIN_FUNCTION(gp_clk_0b),
+	MSM_PIN_FUNCTION(gp_clk_1a),
+	MSM_PIN_FUNCTION(gp_clk_1b),
+	MSM_PIN_FUNCTION(gp_clk_2a),
+	MSM_PIN_FUNCTION(gp_clk_2b),
+	MSM_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(gsbi1),
+	MSM_PIN_FUNCTION(gsbi2),
+	MSM_PIN_FUNCTION(gsbi3),
+	MSM_PIN_FUNCTION(gsbi4),
+	MSM_PIN_FUNCTION(gsbi4_cam_i2c),
+	MSM_PIN_FUNCTION(gsbi5),
+	MSM_PIN_FUNCTION(gsbi5_spi_cs1),
+	MSM_PIN_FUNCTION(gsbi5_spi_cs2),
+	MSM_PIN_FUNCTION(gsbi5_spi_cs3),
+	MSM_PIN_FUNCTION(gsbi6),
+	MSM_PIN_FUNCTION(gsbi6_spi_cs1),
+	MSM_PIN_FUNCTION(gsbi6_spi_cs2),
+	MSM_PIN_FUNCTION(gsbi6_spi_cs3),
+	MSM_PIN_FUNCTION(gsbi7),
+	MSM_PIN_FUNCTION(gsbi7_spi_cs1),
+	MSM_PIN_FUNCTION(gsbi7_spi_cs2),
+	MSM_PIN_FUNCTION(gsbi7_spi_cs3),
+	MSM_PIN_FUNCTION(gsbi_cam_i2c),
+	MSM_PIN_FUNCTION(hdmi),
+	MSM_PIN_FUNCTION(mi2s),
+	MSM_PIN_FUNCTION(riva_bt),
+	MSM_PIN_FUNCTION(riva_fm),
+	MSM_PIN_FUNCTION(riva_wlan),
+	MSM_PIN_FUNCTION(sdc2),
+	MSM_PIN_FUNCTION(sdc4),
+	MSM_PIN_FUNCTION(slimbus),
+	MSM_PIN_FUNCTION(spkr_i2s),
+	MSM_PIN_FUNCTION(tsif1),
+	MSM_PIN_FUNCTION(tsif2),
+	MSM_PIN_FUNCTION(usb2_hsic),
+	MSM_PIN_FUNCTION(ps_hold),
 };
 
 static const struct msm_pingroup apq8064_groups[] = {
diff --git a/drivers/pinctrl/qcom/pinctrl-apq8084.c b/drivers/pinctrl/qcom/pinctrl-apq8084.c
index e3c1f86aba7d06..67d06f6988eff3 100644
--- a/drivers/pinctrl/qcom/pinctrl-apq8084.c
+++ b/drivers/pinctrl/qcom/pinctrl-apq8084.c
@@ -329,14 +329,14 @@ static const unsigned int sdc2_data_pins[] = { 152 };
 			gpio##id##_pins, 		\
 			ARRAY_SIZE(gpio##id##_pins)),	\
 		.funcs = (int[]){			\
-			APQ_MUX_gpio,			\
-			APQ_MUX_##f1,			\
-			APQ_MUX_##f2,			\
-			APQ_MUX_##f3,			\
-			APQ_MUX_##f4,			\
-			APQ_MUX_##f5,			\
-			APQ_MUX_##f6,			\
-			APQ_MUX_##f7			\
+			msm_mux_gpio,			\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7			\
 		},					\
 		.nfuncs = 8,				\
 		.ctl_reg = 0x1000 + 0x10 * id,		\
@@ -386,128 +386,128 @@ static const unsigned int sdc2_data_pins[] = { 152 };
 	}
 
 enum apq8084_functions {
-	APQ_MUX_adsp_ext,
-	APQ_MUX_audio_ref,
-	APQ_MUX_blsp_i2c1,
-	APQ_MUX_blsp_i2c2,
-	APQ_MUX_blsp_i2c3,
-	APQ_MUX_blsp_i2c4,
-	APQ_MUX_blsp_i2c5,
-	APQ_MUX_blsp_i2c6,
-	APQ_MUX_blsp_i2c7,
-	APQ_MUX_blsp_i2c8,
-	APQ_MUX_blsp_i2c9,
-	APQ_MUX_blsp_i2c10,
-	APQ_MUX_blsp_i2c11,
-	APQ_MUX_blsp_i2c12,
-	APQ_MUX_blsp_spi1,
-	APQ_MUX_blsp_spi1_cs1,
-	APQ_MUX_blsp_spi1_cs2,
-	APQ_MUX_blsp_spi1_cs3,
-	APQ_MUX_blsp_spi2,
-	APQ_MUX_blsp_spi3,
-	APQ_MUX_blsp_spi3_cs1,
-	APQ_MUX_blsp_spi3_cs2,
-	APQ_MUX_blsp_spi3_cs3,
-	APQ_MUX_blsp_spi4,
-	APQ_MUX_blsp_spi5,
-	APQ_MUX_blsp_spi6,
-	APQ_MUX_blsp_spi7,
-	APQ_MUX_blsp_spi8,
-	APQ_MUX_blsp_spi9,
-	APQ_MUX_blsp_spi10,
-	APQ_MUX_blsp_spi10_cs1,
-	APQ_MUX_blsp_spi10_cs2,
-	APQ_MUX_blsp_spi10_cs3,
-	APQ_MUX_blsp_spi11,
-	APQ_MUX_blsp_spi12,
-	APQ_MUX_blsp_uart1,
-	APQ_MUX_blsp_uart2,
-	APQ_MUX_blsp_uart3,
-	APQ_MUX_blsp_uart4,
-	APQ_MUX_blsp_uart5,
-	APQ_MUX_blsp_uart6,
-	APQ_MUX_blsp_uart7,
-	APQ_MUX_blsp_uart8,
-	APQ_MUX_blsp_uart9,
-	APQ_MUX_blsp_uart10,
-	APQ_MUX_blsp_uart11,
-	APQ_MUX_blsp_uart12,
-	APQ_MUX_blsp_uim1,
-	APQ_MUX_blsp_uim2,
-	APQ_MUX_blsp_uim3,
-	APQ_MUX_blsp_uim4,
-	APQ_MUX_blsp_uim5,
-	APQ_MUX_blsp_uim6,
-	APQ_MUX_blsp_uim7,
-	APQ_MUX_blsp_uim8,
-	APQ_MUX_blsp_uim9,
-	APQ_MUX_blsp_uim10,
-	APQ_MUX_blsp_uim11,
-	APQ_MUX_blsp_uim12,
-	APQ_MUX_cam_mclk0,
-	APQ_MUX_cam_mclk1,
-	APQ_MUX_cam_mclk2,
-	APQ_MUX_cam_mclk3,
-	APQ_MUX_cci_async,
-	APQ_MUX_cci_async_in0,
-	APQ_MUX_cci_i2c0,
-	APQ_MUX_cci_i2c1,
-	APQ_MUX_cci_timer0,
-	APQ_MUX_cci_timer1,
-	APQ_MUX_cci_timer2,
-	APQ_MUX_cci_timer3,
-	APQ_MUX_cci_timer4,
-	APQ_MUX_edp_hpd,
-	APQ_MUX_gcc_gp1,
-	APQ_MUX_gcc_gp2,
-	APQ_MUX_gcc_gp3,
-	APQ_MUX_gcc_obt,
-	APQ_MUX_gcc_vtt,
-	APQ_MUX_gp_mn,
-	APQ_MUX_gp_pdm0,
-	APQ_MUX_gp_pdm1,
-	APQ_MUX_gp_pdm2,
-	APQ_MUX_gp0_clk,
-	APQ_MUX_gp1_clk,
-	APQ_MUX_gpio,
-	APQ_MUX_hdmi_cec,
-	APQ_MUX_hdmi_ddc,
-	APQ_MUX_hdmi_dtest,
-	APQ_MUX_hdmi_hpd,
-	APQ_MUX_hdmi_rcv,
-	APQ_MUX_hsic,
-	APQ_MUX_ldo_en,
-	APQ_MUX_ldo_update,
-	APQ_MUX_mdp_vsync,
-	APQ_MUX_pci_e0,
-	APQ_MUX_pci_e0_n,
-	APQ_MUX_pci_e0_rst,
-	APQ_MUX_pci_e1,
-	APQ_MUX_pci_e1_rst,
-	APQ_MUX_pci_e1_rst_n,
-	APQ_MUX_pci_e1_clkreq_n,
-	APQ_MUX_pri_mi2s,
-	APQ_MUX_qua_mi2s,
-	APQ_MUX_sata_act,
-	APQ_MUX_sata_devsleep,
-	APQ_MUX_sata_devsleep_n,
-	APQ_MUX_sd_write,
-	APQ_MUX_sdc_emmc_mode,
-	APQ_MUX_sdc3,
-	APQ_MUX_sdc4,
-	APQ_MUX_sec_mi2s,
-	APQ_MUX_slimbus,
-	APQ_MUX_spdif_tx,
-	APQ_MUX_spkr_i2s,
-	APQ_MUX_spkr_i2s_ws,
-	APQ_MUX_spss_geni,
-	APQ_MUX_ter_mi2s,
-	APQ_MUX_tsif1,
-	APQ_MUX_tsif2,
-	APQ_MUX_uim,
-	APQ_MUX_uim_batt_alarm,
-	APQ_MUX_NA,
+	msm_mux_adsp_ext,
+	msm_mux_audio_ref,
+	msm_mux_blsp_i2c1,
+	msm_mux_blsp_i2c2,
+	msm_mux_blsp_i2c3,
+	msm_mux_blsp_i2c4,
+	msm_mux_blsp_i2c5,
+	msm_mux_blsp_i2c6,
+	msm_mux_blsp_i2c7,
+	msm_mux_blsp_i2c8,
+	msm_mux_blsp_i2c9,
+	msm_mux_blsp_i2c10,
+	msm_mux_blsp_i2c11,
+	msm_mux_blsp_i2c12,
+	msm_mux_blsp_spi1,
+	msm_mux_blsp_spi1_cs1,
+	msm_mux_blsp_spi1_cs2,
+	msm_mux_blsp_spi1_cs3,
+	msm_mux_blsp_spi2,
+	msm_mux_blsp_spi3,
+	msm_mux_blsp_spi3_cs1,
+	msm_mux_blsp_spi3_cs2,
+	msm_mux_blsp_spi3_cs3,
+	msm_mux_blsp_spi4,
+	msm_mux_blsp_spi5,
+	msm_mux_blsp_spi6,
+	msm_mux_blsp_spi7,
+	msm_mux_blsp_spi8,
+	msm_mux_blsp_spi9,
+	msm_mux_blsp_spi10,
+	msm_mux_blsp_spi10_cs1,
+	msm_mux_blsp_spi10_cs2,
+	msm_mux_blsp_spi10_cs3,
+	msm_mux_blsp_spi11,
+	msm_mux_blsp_spi12,
+	msm_mux_blsp_uart1,
+	msm_mux_blsp_uart2,
+	msm_mux_blsp_uart3,
+	msm_mux_blsp_uart4,
+	msm_mux_blsp_uart5,
+	msm_mux_blsp_uart6,
+	msm_mux_blsp_uart7,
+	msm_mux_blsp_uart8,
+	msm_mux_blsp_uart9,
+	msm_mux_blsp_uart10,
+	msm_mux_blsp_uart11,
+	msm_mux_blsp_uart12,
+	msm_mux_blsp_uim1,
+	msm_mux_blsp_uim2,
+	msm_mux_blsp_uim3,
+	msm_mux_blsp_uim4,
+	msm_mux_blsp_uim5,
+	msm_mux_blsp_uim6,
+	msm_mux_blsp_uim7,
+	msm_mux_blsp_uim8,
+	msm_mux_blsp_uim9,
+	msm_mux_blsp_uim10,
+	msm_mux_blsp_uim11,
+	msm_mux_blsp_uim12,
+	msm_mux_cam_mclk0,
+	msm_mux_cam_mclk1,
+	msm_mux_cam_mclk2,
+	msm_mux_cam_mclk3,
+	msm_mux_cci_async,
+	msm_mux_cci_async_in0,
+	msm_mux_cci_i2c0,
+	msm_mux_cci_i2c1,
+	msm_mux_cci_timer0,
+	msm_mux_cci_timer1,
+	msm_mux_cci_timer2,
+	msm_mux_cci_timer3,
+	msm_mux_cci_timer4,
+	msm_mux_edp_hpd,
+	msm_mux_gcc_gp1,
+	msm_mux_gcc_gp2,
+	msm_mux_gcc_gp3,
+	msm_mux_gcc_obt,
+	msm_mux_gcc_vtt,
+	msm_mux_gp_mn,
+	msm_mux_gp_pdm0,
+	msm_mux_gp_pdm1,
+	msm_mux_gp_pdm2,
+	msm_mux_gp0_clk,
+	msm_mux_gp1_clk,
+	msm_mux_gpio,
+	msm_mux_hdmi_cec,
+	msm_mux_hdmi_ddc,
+	msm_mux_hdmi_dtest,
+	msm_mux_hdmi_hpd,
+	msm_mux_hdmi_rcv,
+	msm_mux_hsic,
+	msm_mux_ldo_en,
+	msm_mux_ldo_update,
+	msm_mux_mdp_vsync,
+	msm_mux_pci_e0,
+	msm_mux_pci_e0_n,
+	msm_mux_pci_e0_rst,
+	msm_mux_pci_e1,
+	msm_mux_pci_e1_rst,
+	msm_mux_pci_e1_rst_n,
+	msm_mux_pci_e1_clkreq_n,
+	msm_mux_pri_mi2s,
+	msm_mux_qua_mi2s,
+	msm_mux_sata_act,
+	msm_mux_sata_devsleep,
+	msm_mux_sata_devsleep_n,
+	msm_mux_sd_write,
+	msm_mux_sdc_emmc_mode,
+	msm_mux_sdc3,
+	msm_mux_sdc4,
+	msm_mux_sec_mi2s,
+	msm_mux_slimbus,
+	msm_mux_spdif_tx,
+	msm_mux_spkr_i2s,
+	msm_mux_spkr_i2s_ws,
+	msm_mux_spss_geni,
+	msm_mux_ter_mi2s,
+	msm_mux_tsif1,
+	msm_mux_tsif2,
+	msm_mux_uim,
+	msm_mux_uim_batt_alarm,
+	msm_mux_NA,
 };
 
 static const char * const gpio_groups[] = {
@@ -897,127 +897,127 @@ static const char * const uim_batt_alarm_groups[] = {
 	"gpio102"
 };
 static const struct pinfunction apq8084_functions[] = {
-	APQ_PIN_FUNCTION(adsp_ext),
-	APQ_PIN_FUNCTION(audio_ref),
-	APQ_PIN_FUNCTION(blsp_i2c1),
-	APQ_PIN_FUNCTION(blsp_i2c2),
-	APQ_PIN_FUNCTION(blsp_i2c3),
-	APQ_PIN_FUNCTION(blsp_i2c4),
-	APQ_PIN_FUNCTION(blsp_i2c5),
-	APQ_PIN_FUNCTION(blsp_i2c6),
-	APQ_PIN_FUNCTION(blsp_i2c7),
-	APQ_PIN_FUNCTION(blsp_i2c8),
-	APQ_PIN_FUNCTION(blsp_i2c9),
-	APQ_PIN_FUNCTION(blsp_i2c10),
-	APQ_PIN_FUNCTION(blsp_i2c11),
-	APQ_PIN_FUNCTION(blsp_i2c12),
-	APQ_PIN_FUNCTION(blsp_spi1),
-	APQ_PIN_FUNCTION(blsp_spi1_cs1),
-	APQ_PIN_FUNCTION(blsp_spi1_cs2),
-	APQ_PIN_FUNCTION(blsp_spi1_cs3),
-	APQ_PIN_FUNCTION(blsp_spi2),
-	APQ_PIN_FUNCTION(blsp_spi3),
-	APQ_PIN_FUNCTION(blsp_spi3_cs1),
-	APQ_PIN_FUNCTION(blsp_spi3_cs2),
-	APQ_PIN_FUNCTION(blsp_spi3_cs3),
-	APQ_PIN_FUNCTION(blsp_spi4),
-	APQ_PIN_FUNCTION(blsp_spi5),
-	APQ_PIN_FUNCTION(blsp_spi6),
-	APQ_PIN_FUNCTION(blsp_spi7),
-	APQ_PIN_FUNCTION(blsp_spi8),
-	APQ_PIN_FUNCTION(blsp_spi9),
-	APQ_PIN_FUNCTION(blsp_spi10),
-	APQ_PIN_FUNCTION(blsp_spi10_cs1),
-	APQ_PIN_FUNCTION(blsp_spi10_cs2),
-	APQ_PIN_FUNCTION(blsp_spi10_cs3),
-	APQ_PIN_FUNCTION(blsp_spi11),
-	APQ_PIN_FUNCTION(blsp_spi12),
-	APQ_PIN_FUNCTION(blsp_uart1),
-	APQ_PIN_FUNCTION(blsp_uart2),
-	APQ_PIN_FUNCTION(blsp_uart3),
-	APQ_PIN_FUNCTION(blsp_uart4),
-	APQ_PIN_FUNCTION(blsp_uart5),
-	APQ_PIN_FUNCTION(blsp_uart6),
-	APQ_PIN_FUNCTION(blsp_uart7),
-	APQ_PIN_FUNCTION(blsp_uart8),
-	APQ_PIN_FUNCTION(blsp_uart9),
-	APQ_PIN_FUNCTION(blsp_uart10),
-	APQ_PIN_FUNCTION(blsp_uart11),
-	APQ_PIN_FUNCTION(blsp_uart12),
-	APQ_PIN_FUNCTION(blsp_uim1),
-	APQ_PIN_FUNCTION(blsp_uim2),
-	APQ_PIN_FUNCTION(blsp_uim3),
-	APQ_PIN_FUNCTION(blsp_uim4),
-	APQ_PIN_FUNCTION(blsp_uim5),
-	APQ_PIN_FUNCTION(blsp_uim6),
-	APQ_PIN_FUNCTION(blsp_uim7),
-	APQ_PIN_FUNCTION(blsp_uim8),
-	APQ_PIN_FUNCTION(blsp_uim9),
-	APQ_PIN_FUNCTION(blsp_uim10),
-	APQ_PIN_FUNCTION(blsp_uim11),
-	APQ_PIN_FUNCTION(blsp_uim12),
-	APQ_PIN_FUNCTION(cam_mclk0),
-	APQ_PIN_FUNCTION(cam_mclk1),
-	APQ_PIN_FUNCTION(cam_mclk2),
-	APQ_PIN_FUNCTION(cam_mclk3),
-	APQ_PIN_FUNCTION(cci_async),
-	APQ_PIN_FUNCTION(cci_async_in0),
-	APQ_PIN_FUNCTION(cci_i2c0),
-	APQ_PIN_FUNCTION(cci_i2c1),
-	APQ_PIN_FUNCTION(cci_timer0),
-	APQ_PIN_FUNCTION(cci_timer1),
-	APQ_PIN_FUNCTION(cci_timer2),
-	APQ_PIN_FUNCTION(cci_timer3),
-	APQ_PIN_FUNCTION(cci_timer4),
-	APQ_PIN_FUNCTION(edp_hpd),
-	APQ_PIN_FUNCTION(gcc_gp1),
-	APQ_PIN_FUNCTION(gcc_gp2),
-	APQ_PIN_FUNCTION(gcc_gp3),
-	APQ_PIN_FUNCTION(gcc_obt),
-	APQ_PIN_FUNCTION(gcc_vtt),
-	APQ_PIN_FUNCTION(gp_mn),
-	APQ_PIN_FUNCTION(gp_pdm0),
-	APQ_PIN_FUNCTION(gp_pdm1),
-	APQ_PIN_FUNCTION(gp_pdm2),
-	APQ_PIN_FUNCTION(gp0_clk),
-	APQ_PIN_FUNCTION(gp1_clk),
-	APQ_PIN_FUNCTION(gpio),
-	APQ_PIN_FUNCTION(hdmi_cec),
-	APQ_PIN_FUNCTION(hdmi_ddc),
-	APQ_PIN_FUNCTION(hdmi_dtest),
-	APQ_PIN_FUNCTION(hdmi_hpd),
-	APQ_PIN_FUNCTION(hdmi_rcv),
-	APQ_PIN_FUNCTION(hsic),
-	APQ_PIN_FUNCTION(ldo_en),
-	APQ_PIN_FUNCTION(ldo_update),
-	APQ_PIN_FUNCTION(mdp_vsync),
-	APQ_PIN_FUNCTION(pci_e0),
-	APQ_PIN_FUNCTION(pci_e0_n),
-	APQ_PIN_FUNCTION(pci_e0_rst),
-	APQ_PIN_FUNCTION(pci_e1),
-	APQ_PIN_FUNCTION(pci_e1_rst),
-	APQ_PIN_FUNCTION(pci_e1_rst_n),
-	APQ_PIN_FUNCTION(pci_e1_clkreq_n),
-	APQ_PIN_FUNCTION(pri_mi2s),
-	APQ_PIN_FUNCTION(qua_mi2s),
-	APQ_PIN_FUNCTION(sata_act),
-	APQ_PIN_FUNCTION(sata_devsleep),
-	APQ_PIN_FUNCTION(sata_devsleep_n),
-	APQ_PIN_FUNCTION(sd_write),
-	APQ_PIN_FUNCTION(sdc_emmc_mode),
-	APQ_PIN_FUNCTION(sdc3),
-	APQ_PIN_FUNCTION(sdc4),
-	APQ_PIN_FUNCTION(sec_mi2s),
-	APQ_PIN_FUNCTION(slimbus),
-	APQ_PIN_FUNCTION(spdif_tx),
-	APQ_PIN_FUNCTION(spkr_i2s),
-	APQ_PIN_FUNCTION(spkr_i2s_ws),
-	APQ_PIN_FUNCTION(spss_geni),
-	APQ_PIN_FUNCTION(ter_mi2s),
-	APQ_PIN_FUNCTION(tsif1),
-	APQ_PIN_FUNCTION(tsif2),
-	APQ_PIN_FUNCTION(uim),
-	APQ_PIN_FUNCTION(uim_batt_alarm),
+	MSM_PIN_FUNCTION(adsp_ext),
+	MSM_PIN_FUNCTION(audio_ref),
+	MSM_PIN_FUNCTION(blsp_i2c1),
+	MSM_PIN_FUNCTION(blsp_i2c2),
+	MSM_PIN_FUNCTION(blsp_i2c3),
+	MSM_PIN_FUNCTION(blsp_i2c4),
+	MSM_PIN_FUNCTION(blsp_i2c5),
+	MSM_PIN_FUNCTION(blsp_i2c6),
+	MSM_PIN_FUNCTION(blsp_i2c7),
+	MSM_PIN_FUNCTION(blsp_i2c8),
+	MSM_PIN_FUNCTION(blsp_i2c9),
+	MSM_PIN_FUNCTION(blsp_i2c10),
+	MSM_PIN_FUNCTION(blsp_i2c11),
+	MSM_PIN_FUNCTION(blsp_i2c12),
+	MSM_PIN_FUNCTION(blsp_spi1),
+	MSM_PIN_FUNCTION(blsp_spi1_cs1),
+	MSM_PIN_FUNCTION(blsp_spi1_cs2),
+	MSM_PIN_FUNCTION(blsp_spi1_cs3),
+	MSM_PIN_FUNCTION(blsp_spi2),
+	MSM_PIN_FUNCTION(blsp_spi3),
+	MSM_PIN_FUNCTION(blsp_spi3_cs1),
+	MSM_PIN_FUNCTION(blsp_spi3_cs2),
+	MSM_PIN_FUNCTION(blsp_spi3_cs3),
+	MSM_PIN_FUNCTION(blsp_spi4),
+	MSM_PIN_FUNCTION(blsp_spi5),
+	MSM_PIN_FUNCTION(blsp_spi6),
+	MSM_PIN_FUNCTION(blsp_spi7),
+	MSM_PIN_FUNCTION(blsp_spi8),
+	MSM_PIN_FUNCTION(blsp_spi9),
+	MSM_PIN_FUNCTION(blsp_spi10),
+	MSM_PIN_FUNCTION(blsp_spi10_cs1),
+	MSM_PIN_FUNCTION(blsp_spi10_cs2),
+	MSM_PIN_FUNCTION(blsp_spi10_cs3),
+	MSM_PIN_FUNCTION(blsp_spi11),
+	MSM_PIN_FUNCTION(blsp_spi12),
+	MSM_PIN_FUNCTION(blsp_uart1),
+	MSM_PIN_FUNCTION(blsp_uart2),
+	MSM_PIN_FUNCTION(blsp_uart3),
+	MSM_PIN_FUNCTION(blsp_uart4),
+	MSM_PIN_FUNCTION(blsp_uart5),
+	MSM_PIN_FUNCTION(blsp_uart6),
+	MSM_PIN_FUNCTION(blsp_uart7),
+	MSM_PIN_FUNCTION(blsp_uart8),
+	MSM_PIN_FUNCTION(blsp_uart9),
+	MSM_PIN_FUNCTION(blsp_uart10),
+	MSM_PIN_FUNCTION(blsp_uart11),
+	MSM_PIN_FUNCTION(blsp_uart12),
+	MSM_PIN_FUNCTION(blsp_uim1),
+	MSM_PIN_FUNCTION(blsp_uim2),
+	MSM_PIN_FUNCTION(blsp_uim3),
+	MSM_PIN_FUNCTION(blsp_uim4),
+	MSM_PIN_FUNCTION(blsp_uim5),
+	MSM_PIN_FUNCTION(blsp_uim6),
+	MSM_PIN_FUNCTION(blsp_uim7),
+	MSM_PIN_FUNCTION(blsp_uim8),
+	MSM_PIN_FUNCTION(blsp_uim9),
+	MSM_PIN_FUNCTION(blsp_uim10),
+	MSM_PIN_FUNCTION(blsp_uim11),
+	MSM_PIN_FUNCTION(blsp_uim12),
+	MSM_PIN_FUNCTION(cam_mclk0),
+	MSM_PIN_FUNCTION(cam_mclk1),
+	MSM_PIN_FUNCTION(cam_mclk2),
+	MSM_PIN_FUNCTION(cam_mclk3),
+	MSM_PIN_FUNCTION(cci_async),
+	MSM_PIN_FUNCTION(cci_async_in0),
+	MSM_PIN_FUNCTION(cci_i2c0),
+	MSM_PIN_FUNCTION(cci_i2c1),
+	MSM_PIN_FUNCTION(cci_timer0),
+	MSM_PIN_FUNCTION(cci_timer1),
+	MSM_PIN_FUNCTION(cci_timer2),
+	MSM_PIN_FUNCTION(cci_timer3),
+	MSM_PIN_FUNCTION(cci_timer4),
+	MSM_PIN_FUNCTION(edp_hpd),
+	MSM_PIN_FUNCTION(gcc_gp1),
+	MSM_PIN_FUNCTION(gcc_gp2),
+	MSM_PIN_FUNCTION(gcc_gp3),
+	MSM_PIN_FUNCTION(gcc_obt),
+	MSM_PIN_FUNCTION(gcc_vtt),
+	MSM_PIN_FUNCTION(gp_mn),
+	MSM_PIN_FUNCTION(gp_pdm0),
+	MSM_PIN_FUNCTION(gp_pdm1),
+	MSM_PIN_FUNCTION(gp_pdm2),
+	MSM_PIN_FUNCTION(gp0_clk),
+	MSM_PIN_FUNCTION(gp1_clk),
+	MSM_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(hdmi_cec),
+	MSM_PIN_FUNCTION(hdmi_ddc),
+	MSM_PIN_FUNCTION(hdmi_dtest),
+	MSM_PIN_FUNCTION(hdmi_hpd),
+	MSM_PIN_FUNCTION(hdmi_rcv),
+	MSM_PIN_FUNCTION(hsic),
+	MSM_PIN_FUNCTION(ldo_en),
+	MSM_PIN_FUNCTION(ldo_update),
+	MSM_PIN_FUNCTION(mdp_vsync),
+	MSM_PIN_FUNCTION(pci_e0),
+	MSM_PIN_FUNCTION(pci_e0_n),
+	MSM_PIN_FUNCTION(pci_e0_rst),
+	MSM_PIN_FUNCTION(pci_e1),
+	MSM_PIN_FUNCTION(pci_e1_rst),
+	MSM_PIN_FUNCTION(pci_e1_rst_n),
+	MSM_PIN_FUNCTION(pci_e1_clkreq_n),
+	MSM_PIN_FUNCTION(pri_mi2s),
+	MSM_PIN_FUNCTION(qua_mi2s),
+	MSM_PIN_FUNCTION(sata_act),
+	MSM_PIN_FUNCTION(sata_devsleep),
+	MSM_PIN_FUNCTION(sata_devsleep_n),
+	MSM_PIN_FUNCTION(sd_write),
+	MSM_PIN_FUNCTION(sdc_emmc_mode),
+	MSM_PIN_FUNCTION(sdc3),
+	MSM_PIN_FUNCTION(sdc4),
+	MSM_PIN_FUNCTION(sec_mi2s),
+	MSM_PIN_FUNCTION(slimbus),
+	MSM_PIN_FUNCTION(spdif_tx),
+	MSM_PIN_FUNCTION(spkr_i2s),
+	MSM_PIN_FUNCTION(spkr_i2s_ws),
+	MSM_PIN_FUNCTION(spss_geni),
+	MSM_PIN_FUNCTION(ter_mi2s),
+	MSM_PIN_FUNCTION(tsif1),
+	MSM_PIN_FUNCTION(tsif2),
+	MSM_PIN_FUNCTION(uim),
+	MSM_PIN_FUNCTION(uim_batt_alarm),
 };
 
 static const struct msm_pingroup apq8084_groups[] = {
diff --git a/drivers/pinctrl/qcom/pinctrl-hawi-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-hawi-lpass-lpi.c
new file mode 100644
index 00000000000000..7036bf4c66920f
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-hawi-lpass-lpi.c
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_clock.h>
+#include <linux/pm_runtime.h>
+
+#include "pinctrl-lpass-lpi.h"
+
+enum lpass_lpi_functions {
+	LPI_MUX_dmic1_clk,
+	LPI_MUX_dmic1_data,
+	LPI_MUX_dmic2_clk,
+	LPI_MUX_dmic2_data,
+	LPI_MUX_dmic3_clk,
+	LPI_MUX_dmic3_data,
+	LPI_MUX_dmic4_clk,
+	LPI_MUX_dmic4_data,
+	LPI_MUX_ext_mclk1_a,
+	LPI_MUX_ext_mclk1_b,
+	LPI_MUX_ext_mclk1_c,
+	LPI_MUX_ext_mclk1_d,
+	LPI_MUX_ext_mclk1_e,
+	LPI_MUX_i2s0_clk,
+	LPI_MUX_i2s0_data,
+	LPI_MUX_i2s0_ws,
+	LPI_MUX_i2s1_clk,
+	LPI_MUX_i2s1_data,
+	LPI_MUX_i2s1_ws,
+	LPI_MUX_i2s2_clk,
+	LPI_MUX_i2s2_data,
+	LPI_MUX_i2s2_ws,
+	LPI_MUX_i2s3_clk,
+	LPI_MUX_i2s3_data,
+	LPI_MUX_i2s3_ws,
+	LPI_MUX_lpass_lpi_dbg_clk,
+	LPI_MUX_qca_swr_clk,
+	LPI_MUX_qca_swr_data,
+	LPI_MUX_slimbus_clk,
+	LPI_MUX_slimbus_data,
+	LPI_MUX_swr_rx_clk,
+	LPI_MUX_swr_rx_data,
+	LPI_MUX_swr_tx_clk,
+	LPI_MUX_swr_tx_clk1,
+	LPI_MUX_swr_tx_data,
+	LPI_MUX_va_i2s0_clk,
+	LPI_MUX_va_i2s0_data,
+	LPI_MUX_va_i2s0_ws,
+	LPI_MUX_wsa2_swr_clk,
+	LPI_MUX_wsa2_swr_data,
+	LPI_MUX_wsa_swr_clk,
+	LPI_MUX_wsa_swr_data,
+	LPI_MUX_gpio,
+	LPI_MUX__,
+};
+
+static const struct pinctrl_pin_desc hawi_lpi_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "gpio8"),
+	PINCTRL_PIN(9, "gpio9"),
+	PINCTRL_PIN(10, "gpio10"),
+	PINCTRL_PIN(11, "gpio11"),
+	PINCTRL_PIN(12, "gpio12"),
+	PINCTRL_PIN(13, "gpio13"),
+	PINCTRL_PIN(14, "gpio14"),
+	PINCTRL_PIN(15, "gpio15"),
+	PINCTRL_PIN(16, "gpio16"),
+	PINCTRL_PIN(17, "gpio17"),
+	PINCTRL_PIN(18, "gpio18"),
+	PINCTRL_PIN(19, "gpio19"),
+	PINCTRL_PIN(20, "gpio20"),
+	PINCTRL_PIN(21, "gpio21"),
+	PINCTRL_PIN(22, "gpio22"),
+};
+
+static const char * const gpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7",
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14",
+	"gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20", "gpio21",
+	"gpio22",
+};
+
+static const char * const swr_tx_clk_groups[] = { "gpio0" };
+static const char * const i2s0_clk_groups[] = { "gpio0" };
+static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" };
+static const char * const i2s0_ws_groups[] = { "gpio1" };
+static const char * const swr_rx_clk_groups[] = { "gpio3" };
+static const char * const i2s0_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" };
+static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5" };
+static const char * const ext_mclk1_c_groups[] = { "gpio5" };
+static const char * const dmic1_clk_groups[] = { "gpio6" };
+static const char * const va_i2s0_clk_groups[] = { "gpio6" };
+static const char * const dmic1_data_groups[] = { "gpio7" };
+static const char * const va_i2s0_ws_groups[] = { "gpio7" };
+static const char * const dmic2_clk_groups[] = { "gpio8" };
+static const char * const va_i2s0_data_groups[] = { "gpio8", "gpio9" };
+static const char * const dmic2_data_groups[] = { "gpio9" };
+static const char * const ext_mclk1_b_groups[] = { "gpio9" };
+static const char * const i2s1_clk_groups[] = { "gpio10" };
+static const char * const wsa_swr_clk_groups[] = { "gpio10" };
+static const char * const i2s1_ws_groups[] = { "gpio11" };
+static const char * const wsa_swr_data_groups[] = { "gpio11", "gpio21" };
+static const char * const dmic3_clk_groups[] = { "gpio12" };
+static const char * const i2s3_clk_groups[] = { "gpio12" };
+static const char * const dmic3_data_groups[] = { "gpio13" };
+static const char * const i2s3_ws_groups[] = { "gpio13" };
+static const char * const ext_mclk1_a_groups[] = { "gpio13" };
+static const char * const lpass_lpi_dbg_clk_groups[] = { "gpio13" };
+static const char * const swr_tx_clk1_groups[] = { "gpio14" };
+static const char * const ext_mclk1_d_groups[] = { "gpio14" };
+static const char * const i2s1_data_groups[] = { "gpio15", "gpio16", "gpio21", "gpio22" };
+static const char * const wsa2_swr_clk_groups[] = { "gpio15" };
+static const char * const wsa2_swr_data_groups[] = { "gpio16", "gpio22" };
+static const char * const dmic4_clk_groups[] = { "gpio17" };
+static const char * const i2s3_data_groups[] = { "gpio17", "gpio18" };
+static const char * const dmic4_data_groups[] = { "gpio18" };
+static const char * const i2s2_clk_groups[] = { "gpio19" };
+static const char * const slimbus_clk_groups[] = { "gpio19" };
+static const char * const qca_swr_clk_groups[] = { "gpio19" };
+static const char * const i2s2_ws_groups[] = { "gpio20" };
+static const char * const slimbus_data_groups[] = { "gpio20" };
+static const char * const qca_swr_data_groups[] = { "gpio20" };
+static const char * const i2s2_data_groups[] = { "gpio21", "gpio22" };
+static const char * const ext_mclk1_e_groups[] = { "gpio22" };
+
+static const struct lpi_pingroup hawi_groups[] = {
+	LPI_PINGROUP(0, 11, swr_tx_clk, i2s0_clk, _, _),
+	LPI_PINGROUP(1, 11, swr_tx_data, i2s0_ws, _, _),
+	LPI_PINGROUP(2, 11, swr_tx_data, i2s0_data, _, _),
+	LPI_PINGROUP(3, 11, swr_rx_clk, i2s0_data, _, _),
+	LPI_PINGROUP(4, 11, swr_rx_data, i2s0_data, _, _),
+	LPI_PINGROUP(5, 11, swr_rx_data, ext_mclk1_c, i2s0_data, _),
+	LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, va_i2s0_clk, _, _),
+	LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, va_i2s0_ws, _, _),
+	LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, va_i2s0_data, _, _),
+	LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, va_i2s0_data, ext_mclk1_b, _),
+	LPI_PINGROUP(10, 11, i2s1_clk, wsa_swr_clk, _, _),
+	LPI_PINGROUP(11, 11, i2s1_ws, wsa_swr_data, _, _),
+	LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s3_clk, _, _),
+	LPI_PINGROUP6(13, LPI_NO_SLEW, dmic3_data, i2s3_ws, ext_mclk1_a, _, lpass_lpi_dbg_clk),
+	LPI_PINGROUP(14, 11, swr_tx_data, swr_tx_clk1, ext_mclk1_d, _),
+	LPI_PINGROUP(15, 11, i2s1_data, wsa2_swr_clk, _, _),
+	LPI_PINGROUP(16, 11, i2s1_data, wsa2_swr_data, _, _),
+	LPI_PINGROUP(17, LPI_NO_SLEW, dmic4_clk, i2s3_data, _, _),
+	LPI_PINGROUP(18, LPI_NO_SLEW, dmic4_data, i2s3_data, _, _),
+	LPI_PINGROUP(19, LPI_NO_SLEW, i2s2_clk, slimbus_clk, qca_swr_clk, _),
+	LPI_PINGROUP(20, LPI_NO_SLEW, i2s2_ws, slimbus_data, qca_swr_data, _),
+	LPI_PINGROUP(21, 11, i2s2_data, _, wsa_swr_data, i2s1_data),
+	LPI_PINGROUP6(22, 11, i2s2_data, ext_mclk1_e, _, i2s1_data, wsa2_swr_data),
+};
+
+static const struct lpi_function hawi_functions[] = {
+	LPI_FUNCTION(gpio),
+	LPI_FUNCTION(dmic1_clk),
+	LPI_FUNCTION(dmic1_data),
+	LPI_FUNCTION(dmic2_clk),
+	LPI_FUNCTION(dmic2_data),
+	LPI_FUNCTION(dmic3_clk),
+	LPI_FUNCTION(dmic3_data),
+	LPI_FUNCTION(dmic4_clk),
+	LPI_FUNCTION(dmic4_data),
+	LPI_FUNCTION(ext_mclk1_a),
+	LPI_FUNCTION(ext_mclk1_b),
+	LPI_FUNCTION(ext_mclk1_c),
+	LPI_FUNCTION(ext_mclk1_d),
+	LPI_FUNCTION(ext_mclk1_e),
+	LPI_FUNCTION(i2s0_clk),
+	LPI_FUNCTION(i2s0_data),
+	LPI_FUNCTION(i2s0_ws),
+	LPI_FUNCTION(i2s1_clk),
+	LPI_FUNCTION(i2s1_data),
+	LPI_FUNCTION(i2s1_ws),
+	LPI_FUNCTION(i2s2_clk),
+	LPI_FUNCTION(i2s2_data),
+	LPI_FUNCTION(i2s2_ws),
+	LPI_FUNCTION(i2s3_clk),
+	LPI_FUNCTION(i2s3_data),
+	LPI_FUNCTION(i2s3_ws),
+	LPI_FUNCTION(lpass_lpi_dbg_clk),
+	LPI_FUNCTION(qca_swr_clk),
+	LPI_FUNCTION(qca_swr_data),
+	LPI_FUNCTION(slimbus_clk),
+	LPI_FUNCTION(slimbus_data),
+	LPI_FUNCTION(swr_rx_clk),
+	LPI_FUNCTION(swr_rx_data),
+	LPI_FUNCTION(swr_tx_clk),
+	LPI_FUNCTION(swr_tx_clk1),
+	LPI_FUNCTION(swr_tx_data),
+	LPI_FUNCTION(va_i2s0_clk),
+	LPI_FUNCTION(va_i2s0_data),
+	LPI_FUNCTION(va_i2s0_ws),
+	LPI_FUNCTION(wsa2_swr_clk),
+	LPI_FUNCTION(wsa2_swr_data),
+	LPI_FUNCTION(wsa_swr_clk),
+	LPI_FUNCTION(wsa_swr_data),
+};
+
+static const struct lpi_pinctrl_variant_data hawi_lpi_data = {
+	.pins = hawi_lpi_pins,
+	.npins = ARRAY_SIZE(hawi_lpi_pins),
+	.groups = hawi_groups,
+	.ngroups = ARRAY_SIZE(hawi_groups),
+	.functions = hawi_functions,
+	.nfunctions = ARRAY_SIZE(hawi_functions),
+	.flags = LPI_FLAG_SLEW_RATE_SAME_REG,
+};
+
+static const struct of_device_id lpi_pinctrl_of_match[] = {
+	{
+		.compatible = "qcom,hawi-lpass-lpi-pinctrl",
+		.data = &hawi_lpi_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match);
+
+static const struct dev_pm_ops lpi_pinctrl_pm_ops = {
+	RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL)
+};
+
+static struct platform_driver lpi_pinctrl_driver = {
+	.driver = {
+		.name = "qcom-hawi-lpass-lpi-pinctrl",
+		.of_match_table = lpi_pinctrl_of_match,
+		.pm = pm_ptr(&lpi_pinctrl_pm_ops),
+	},
+	.probe = lpi_pinctrl_probe,
+	.remove = lpi_pinctrl_remove,
+};
+
+module_platform_driver(lpi_pinctrl_driver);
+MODULE_DESCRIPTION("Qualcomm Hawi LPI GPIO pin control driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-hawi.c b/drivers/pinctrl/qcom/pinctrl-hawi.c
index cdde69e954a76d..3486a6d63d10e7 100644
--- a/drivers/pinctrl/qcom/pinctrl-hawi.c
+++ b/drivers/pinctrl/qcom/pinctrl-hawi.c
@@ -47,6 +47,7 @@
 		.intr_wakeup_present_bit = 6,	\
 		.intr_wakeup_enable_bit = 7,	\
 		.intr_target_bit = 8,			\
+		.intr_target_width = 4,		\
 		.intr_target_kpss_val = 3,		\
 		.intr_raw_status_bit = 4,		\
 		.intr_polarity_bit = 1,			\
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq4019.c b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
index adf7c8631f7219..ea407e426bba54 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq4019.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
@@ -221,21 +221,21 @@ DECLARE_QCA_GPIO_PINS(99);
 			gpio##id##_pins, 		\
 			ARRAY_SIZE(gpio##id##_pins)),	\
 		.funcs = (int[]){			\
-			qca_mux_gpio, /* gpio mode */	\
-			qca_mux_##f1,			\
-			qca_mux_##f2,			\
-			qca_mux_##f3,			\
-			qca_mux_##f4,			\
-			qca_mux_##f5,			\
-			qca_mux_##f6,			\
-			qca_mux_##f7,			\
-			qca_mux_##f8,			\
-			qca_mux_##f9,			\
-			qca_mux_##f10,			\
-			qca_mux_##f11,			\
-			qca_mux_##f12,			\
-			qca_mux_##f13,			\
-			qca_mux_##f14			\
+			msm_mux_gpio, /* gpio mode */	\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9,			\
+			msm_mux_##f10,			\
+			msm_mux_##f11,			\
+			msm_mux_##f12,			\
+			msm_mux_##f13,			\
+			msm_mux_##f14			\
 		},				        \
 		.nfuncs = 15,				\
 		.ctl_reg = 0x0 + 0x1000 * id,		\
@@ -260,51 +260,51 @@ DECLARE_QCA_GPIO_PINS(99);
 
 
 enum ipq4019_functions {
-	qca_mux_gpio,
-	qca_mux_aud_pin,
-	qca_mux_audio_pwm,
-	qca_mux_blsp_i2c0,
-	qca_mux_blsp_i2c1,
-	qca_mux_blsp_spi0,
-	qca_mux_blsp_spi1,
-	qca_mux_blsp_uart0,
-	qca_mux_blsp_uart1,
-	qca_mux_chip_rst,
-	qca_mux_i2s_rx,
-	qca_mux_i2s_spdif_in,
-	qca_mux_i2s_spdif_out,
-	qca_mux_i2s_td,
-	qca_mux_i2s_tx,
-	qca_mux_jtag,
-	qca_mux_led0,
-	qca_mux_led1,
-	qca_mux_led2,
-	qca_mux_led3,
-	qca_mux_led4,
-	qca_mux_led5,
-	qca_mux_led6,
-	qca_mux_led7,
-	qca_mux_led8,
-	qca_mux_led9,
-	qca_mux_led10,
-	qca_mux_led11,
-	qca_mux_mdc,
-	qca_mux_mdio,
-	qca_mux_pcie,
-	qca_mux_pmu,
-	qca_mux_prng_rosc,
-	qca_mux_qpic,
-	qca_mux_rgmii,
-	qca_mux_rmii,
-	qca_mux_sdio,
-	qca_mux_smart0,
-	qca_mux_smart1,
-	qca_mux_smart2,
-	qca_mux_smart3,
-	qca_mux_tm,
-	qca_mux_wifi0,
-	qca_mux_wifi1,
-	qca_mux_NA,
+	msm_mux_gpio,
+	msm_mux_aud_pin,
+	msm_mux_audio_pwm,
+	msm_mux_blsp_i2c0,
+	msm_mux_blsp_i2c1,
+	msm_mux_blsp_spi0,
+	msm_mux_blsp_spi1,
+	msm_mux_blsp_uart0,
+	msm_mux_blsp_uart1,
+	msm_mux_chip_rst,
+	msm_mux_i2s_rx,
+	msm_mux_i2s_spdif_in,
+	msm_mux_i2s_spdif_out,
+	msm_mux_i2s_td,
+	msm_mux_i2s_tx,
+	msm_mux_jtag,
+	msm_mux_led0,
+	msm_mux_led1,
+	msm_mux_led2,
+	msm_mux_led3,
+	msm_mux_led4,
+	msm_mux_led5,
+	msm_mux_led6,
+	msm_mux_led7,
+	msm_mux_led8,
+	msm_mux_led9,
+	msm_mux_led10,
+	msm_mux_led11,
+	msm_mux_mdc,
+	msm_mux_mdio,
+	msm_mux_pcie,
+	msm_mux_pmu,
+	msm_mux_prng_rosc,
+	msm_mux_qpic,
+	msm_mux_rgmii,
+	msm_mux_rmii,
+	msm_mux_sdio,
+	msm_mux_smart0,
+	msm_mux_smart1,
+	msm_mux_smart2,
+	msm_mux_smart3,
+	msm_mux_tm,
+	msm_mux_wifi0,
+	msm_mux_wifi1,
+	msm_mux_NA,
 };
 
 static const char * const gpio_groups[] = {
@@ -470,50 +470,50 @@ static const char * const wifi1_groups[] = {
 };
 
 static const struct pinfunction ipq4019_functions[] = {
-	QCA_PIN_FUNCTION(aud_pin),
-	QCA_PIN_FUNCTION(audio_pwm),
-	QCA_PIN_FUNCTION(blsp_i2c0),
-	QCA_PIN_FUNCTION(blsp_i2c1),
-	QCA_PIN_FUNCTION(blsp_spi0),
-	QCA_PIN_FUNCTION(blsp_spi1),
-	QCA_PIN_FUNCTION(blsp_uart0),
-	QCA_PIN_FUNCTION(blsp_uart1),
-	QCA_PIN_FUNCTION(chip_rst),
-	QCA_GPIO_PIN_FUNCTION(gpio),
-	QCA_PIN_FUNCTION(i2s_rx),
-	QCA_PIN_FUNCTION(i2s_spdif_in),
-	QCA_PIN_FUNCTION(i2s_spdif_out),
-	QCA_PIN_FUNCTION(i2s_td),
-	QCA_PIN_FUNCTION(i2s_tx),
-	QCA_PIN_FUNCTION(jtag),
-	QCA_PIN_FUNCTION(led0),
-	QCA_PIN_FUNCTION(led1),
-	QCA_PIN_FUNCTION(led2),
-	QCA_PIN_FUNCTION(led3),
-	QCA_PIN_FUNCTION(led4),
-	QCA_PIN_FUNCTION(led5),
-	QCA_PIN_FUNCTION(led6),
-	QCA_PIN_FUNCTION(led7),
-	QCA_PIN_FUNCTION(led8),
-	QCA_PIN_FUNCTION(led9),
-	QCA_PIN_FUNCTION(led10),
-	QCA_PIN_FUNCTION(led11),
-	QCA_PIN_FUNCTION(mdc),
-	QCA_PIN_FUNCTION(mdio),
-	QCA_PIN_FUNCTION(pcie),
-	QCA_PIN_FUNCTION(pmu),
-	QCA_PIN_FUNCTION(prng_rosc),
-	QCA_PIN_FUNCTION(qpic),
-	QCA_PIN_FUNCTION(rgmii),
-	QCA_PIN_FUNCTION(rmii),
-	QCA_PIN_FUNCTION(sdio),
-	QCA_PIN_FUNCTION(smart0),
-	QCA_PIN_FUNCTION(smart1),
-	QCA_PIN_FUNCTION(smart2),
-	QCA_PIN_FUNCTION(smart3),
-	QCA_PIN_FUNCTION(tm),
-	QCA_PIN_FUNCTION(wifi0),
-	QCA_PIN_FUNCTION(wifi1),
+	MSM_PIN_FUNCTION(aud_pin),
+	MSM_PIN_FUNCTION(audio_pwm),
+	MSM_PIN_FUNCTION(blsp_i2c0),
+	MSM_PIN_FUNCTION(blsp_i2c1),
+	MSM_PIN_FUNCTION(blsp_spi0),
+	MSM_PIN_FUNCTION(blsp_spi1),
+	MSM_PIN_FUNCTION(blsp_uart0),
+	MSM_PIN_FUNCTION(blsp_uart1),
+	MSM_PIN_FUNCTION(chip_rst),
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(i2s_rx),
+	MSM_PIN_FUNCTION(i2s_spdif_in),
+	MSM_PIN_FUNCTION(i2s_spdif_out),
+	MSM_PIN_FUNCTION(i2s_td),
+	MSM_PIN_FUNCTION(i2s_tx),
+	MSM_PIN_FUNCTION(jtag),
+	MSM_PIN_FUNCTION(led0),
+	MSM_PIN_FUNCTION(led1),
+	MSM_PIN_FUNCTION(led2),
+	MSM_PIN_FUNCTION(led3),
+	MSM_PIN_FUNCTION(led4),
+	MSM_PIN_FUNCTION(led5),
+	MSM_PIN_FUNCTION(led6),
+	MSM_PIN_FUNCTION(led7),
+	MSM_PIN_FUNCTION(led8),
+	MSM_PIN_FUNCTION(led9),
+	MSM_PIN_FUNCTION(led10),
+	MSM_PIN_FUNCTION(led11),
+	MSM_PIN_FUNCTION(mdc),
+	MSM_PIN_FUNCTION(mdio),
+	MSM_PIN_FUNCTION(pcie),
+	MSM_PIN_FUNCTION(pmu),
+	MSM_PIN_FUNCTION(prng_rosc),
+	MSM_PIN_FUNCTION(qpic),
+	MSM_PIN_FUNCTION(rgmii),
+	MSM_PIN_FUNCTION(rmii),
+	MSM_PIN_FUNCTION(sdio),
+	MSM_PIN_FUNCTION(smart0),
+	MSM_PIN_FUNCTION(smart1),
+	MSM_PIN_FUNCTION(smart2),
+	MSM_PIN_FUNCTION(smart3),
+	MSM_PIN_FUNCTION(tm),
+	MSM_PIN_FUNCTION(wifi0),
+	MSM_PIN_FUNCTION(wifi1),
 };
 
 static const struct msm_pingroup ipq4019_groups[] = {
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq5018.c b/drivers/pinctrl/qcom/pinctrl-ipq5018.c
index 03cfba9534f845..84392fe258c9f3 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq5018.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq5018.c
@@ -563,7 +563,7 @@ static const char * const qdss_cti_trig_out_b1_groups[] = {
 };
 
 static const char * const pwm3_groups[] = {
-	"gpio45",
+	"gpio45", "gpio13",
 };
 
 static const char * const qdss_cti_trig_in_b1_groups[] = {
@@ -689,7 +689,7 @@ static const struct msm_pingroup ipq5018_groups[] = {
 	PINGROUP(10, blsp0_spi, blsp1_uart0, led0, gcc_plltest, qdss_tracedata_a, _, _, _, _),
 	PINGROUP(11, blsp0_spi, blsp1_uart0, _, gcc_tlmm, qdss_tracedata_a, _, _, _, _),
 	PINGROUP(12, blsp0_spi, blsp0_i2c, blsp1_uart0, _, gcc_plltest, qdss_tracedata_a, _, _, _),
-	PINGROUP(13, blsp0_spi, blsp0_i2c, blsp1_uart0, _, qdss_tracedata_a, _, _, _, _),
+	PINGROUP(13, blsp0_spi, blsp0_i2c, blsp1_uart0, pwm3, qdss_tracedata_a, _, _, _, _),
 	PINGROUP(14, pcie0_clk, _, _, cri_trng0, qdss_tracedata_a, _, _, _, _),
 	PINGROUP(15, _, _, cri_trng1, qdss_tracedata_a, _, _, _, _, _),
 	PINGROUP(16, pcie0_wake, _, _, cri_trng, qdss_tracedata_a, _, _, _, _),
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq5210.c b/drivers/pinctrl/qcom/pinctrl-ipq5210.c
index 827a4ad07a6b00..d5b4eb3e734352 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq5210.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq5210.c
@@ -867,6 +867,7 @@ static const struct of_device_id ipq5210_tlmm_of_match[] = {
 	{ .compatible = "qcom,ipq5210-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, ipq5210_tlmm_of_match);
 
 static int ipq5210_tlmm_probe(struct platform_device *pdev)
 {
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq8064.c b/drivers/pinctrl/qcom/pinctrl-ipq8064.c
index 0a50486337d319..02682eb82be19a 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq8064.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq8064.c
@@ -166,17 +166,17 @@ static const unsigned int sdc3_data_pins[] = { 71 };
 			gpio##id##_pins, 		\
 			ARRAY_SIZE(gpio##id##_pins)),	\
 		.funcs = (int[]){			\
-			IPQ_MUX_gpio,			\
-			IPQ_MUX_##f1,			\
-			IPQ_MUX_##f2,			\
-			IPQ_MUX_##f3,			\
-			IPQ_MUX_##f4,			\
-			IPQ_MUX_##f5,			\
-			IPQ_MUX_##f6,			\
-			IPQ_MUX_##f7,			\
-			IPQ_MUX_##f8,			\
-			IPQ_MUX_##f9,			\
-			IPQ_MUX_##f10,			\
+			msm_mux_gpio,			\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9,			\
+			msm_mux_##f10,			\
 		},					\
 		.nfuncs = 11,				\
 		.ctl_reg = 0x1000 + 0x10 * id,		\
@@ -228,53 +228,53 @@ static const unsigned int sdc3_data_pins[] = { 71 };
 	}
 
 enum ipq8064_functions {
-	IPQ_MUX_gpio,
-	IPQ_MUX_mdio,
-	IPQ_MUX_mi2s,
-	IPQ_MUX_pdm,
-	IPQ_MUX_ssbi,
-	IPQ_MUX_spmi,
-	IPQ_MUX_audio_pcm,
-	IPQ_MUX_gsbi1,
-	IPQ_MUX_gsbi2,
-	IPQ_MUX_gsbi4,
-	IPQ_MUX_gsbi5,
-	IPQ_MUX_gsbi5_spi_cs1,
-	IPQ_MUX_gsbi5_spi_cs2,
-	IPQ_MUX_gsbi5_spi_cs3,
-	IPQ_MUX_gsbi6,
-	IPQ_MUX_gsbi7,
-	IPQ_MUX_nss_spi,
-	IPQ_MUX_sdc1,
-	IPQ_MUX_spdif,
-	IPQ_MUX_nand,
-	IPQ_MUX_tsif1,
-	IPQ_MUX_tsif2,
-	IPQ_MUX_usb_fs_n,
-	IPQ_MUX_usb_fs,
-	IPQ_MUX_usb2_hsic,
-	IPQ_MUX_rgmii2,
-	IPQ_MUX_sata,
-	IPQ_MUX_pcie1_rst,
-	IPQ_MUX_pcie1_prsnt,
-	IPQ_MUX_pcie1_pwrflt,
-	IPQ_MUX_pcie1_pwren_n,
-	IPQ_MUX_pcie1_pwren,
-	IPQ_MUX_pcie1_clk_req,
-	IPQ_MUX_pcie2_rst,
-	IPQ_MUX_pcie2_prsnt,
-	IPQ_MUX_pcie2_pwrflt,
-	IPQ_MUX_pcie2_pwren_n,
-	IPQ_MUX_pcie2_pwren,
-	IPQ_MUX_pcie2_clk_req,
-	IPQ_MUX_pcie3_rst,
-	IPQ_MUX_pcie3_prsnt,
-	IPQ_MUX_pcie3_pwrflt,
-	IPQ_MUX_pcie3_pwren_n,
-	IPQ_MUX_pcie3_pwren,
-	IPQ_MUX_pcie3_clk_req,
-	IPQ_MUX_ps_hold,
-	IPQ_MUX_NA,
+	msm_mux_gpio,
+	msm_mux_mdio,
+	msm_mux_mi2s,
+	msm_mux_pdm,
+	msm_mux_ssbi,
+	msm_mux_spmi,
+	msm_mux_audio_pcm,
+	msm_mux_gsbi1,
+	msm_mux_gsbi2,
+	msm_mux_gsbi4,
+	msm_mux_gsbi5,
+	msm_mux_gsbi5_spi_cs1,
+	msm_mux_gsbi5_spi_cs2,
+	msm_mux_gsbi5_spi_cs3,
+	msm_mux_gsbi6,
+	msm_mux_gsbi7,
+	msm_mux_nss_spi,
+	msm_mux_sdc1,
+	msm_mux_spdif,
+	msm_mux_nand,
+	msm_mux_tsif1,
+	msm_mux_tsif2,
+	msm_mux_usb_fs_n,
+	msm_mux_usb_fs,
+	msm_mux_usb2_hsic,
+	msm_mux_rgmii2,
+	msm_mux_sata,
+	msm_mux_pcie1_rst,
+	msm_mux_pcie1_prsnt,
+	msm_mux_pcie1_pwrflt,
+	msm_mux_pcie1_pwren_n,
+	msm_mux_pcie1_pwren,
+	msm_mux_pcie1_clk_req,
+	msm_mux_pcie2_rst,
+	msm_mux_pcie2_prsnt,
+	msm_mux_pcie2_pwrflt,
+	msm_mux_pcie2_pwren_n,
+	msm_mux_pcie2_pwren,
+	msm_mux_pcie2_clk_req,
+	msm_mux_pcie3_rst,
+	msm_mux_pcie3_prsnt,
+	msm_mux_pcie3_pwrflt,
+	msm_mux_pcie3_pwren_n,
+	msm_mux_pcie3_pwren,
+	msm_mux_pcie3_clk_req,
+	msm_mux_ps_hold,
+	msm_mux_NA,
 };
 
 static const char * const gpio_groups[] = {
@@ -480,52 +480,52 @@ static const char * const ps_hold_groups[] = {
 };
 
 static const struct pinfunction ipq8064_functions[] = {
-	IPQ_GPIO_PIN_FUNCTION(gpio),
-	IPQ_PIN_FUNCTION(mdio),
-	IPQ_PIN_FUNCTION(ssbi),
-	IPQ_PIN_FUNCTION(spmi),
-	IPQ_PIN_FUNCTION(mi2s),
-	IPQ_PIN_FUNCTION(pdm),
-	IPQ_PIN_FUNCTION(audio_pcm),
-	IPQ_PIN_FUNCTION(gsbi1),
-	IPQ_PIN_FUNCTION(gsbi2),
-	IPQ_PIN_FUNCTION(gsbi4),
-	IPQ_PIN_FUNCTION(gsbi5),
-	IPQ_PIN_FUNCTION(gsbi5_spi_cs1),
-	IPQ_PIN_FUNCTION(gsbi5_spi_cs2),
-	IPQ_PIN_FUNCTION(gsbi5_spi_cs3),
-	IPQ_PIN_FUNCTION(gsbi6),
-	IPQ_PIN_FUNCTION(gsbi7),
-	IPQ_PIN_FUNCTION(nss_spi),
-	IPQ_PIN_FUNCTION(sdc1),
-	IPQ_PIN_FUNCTION(spdif),
-	IPQ_PIN_FUNCTION(nand),
-	IPQ_PIN_FUNCTION(tsif1),
-	IPQ_PIN_FUNCTION(tsif2),
-	IPQ_PIN_FUNCTION(usb_fs_n),
-	IPQ_PIN_FUNCTION(usb_fs),
-	IPQ_PIN_FUNCTION(usb2_hsic),
-	IPQ_PIN_FUNCTION(rgmii2),
-	IPQ_PIN_FUNCTION(sata),
-	IPQ_GPIO_PIN_FUNCTION(pcie1_rst),
-	IPQ_PIN_FUNCTION(pcie1_prsnt),
-	IPQ_PIN_FUNCTION(pcie1_pwren_n),
-	IPQ_PIN_FUNCTION(pcie1_pwren),
-	IPQ_PIN_FUNCTION(pcie1_pwrflt),
-	IPQ_PIN_FUNCTION(pcie1_clk_req),
-	IPQ_GPIO_PIN_FUNCTION(pcie2_rst),
-	IPQ_PIN_FUNCTION(pcie2_prsnt),
-	IPQ_PIN_FUNCTION(pcie2_pwren_n),
-	IPQ_PIN_FUNCTION(pcie2_pwren),
-	IPQ_PIN_FUNCTION(pcie2_pwrflt),
-	IPQ_PIN_FUNCTION(pcie2_clk_req),
-	IPQ_GPIO_PIN_FUNCTION(pcie3_rst),
-	IPQ_PIN_FUNCTION(pcie3_prsnt),
-	IPQ_PIN_FUNCTION(pcie3_pwren_n),
-	IPQ_PIN_FUNCTION(pcie3_pwren),
-	IPQ_PIN_FUNCTION(pcie3_pwrflt),
-	IPQ_PIN_FUNCTION(pcie3_clk_req),
-	IPQ_PIN_FUNCTION(ps_hold),
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(mdio),
+	MSM_PIN_FUNCTION(ssbi),
+	MSM_PIN_FUNCTION(spmi),
+	MSM_PIN_FUNCTION(mi2s),
+	MSM_PIN_FUNCTION(pdm),
+	MSM_PIN_FUNCTION(audio_pcm),
+	MSM_PIN_FUNCTION(gsbi1),
+	MSM_PIN_FUNCTION(gsbi2),
+	MSM_PIN_FUNCTION(gsbi4),
+	MSM_PIN_FUNCTION(gsbi5),
+	MSM_PIN_FUNCTION(gsbi5_spi_cs1),
+	MSM_PIN_FUNCTION(gsbi5_spi_cs2),
+	MSM_PIN_FUNCTION(gsbi5_spi_cs3),
+	MSM_PIN_FUNCTION(gsbi6),
+	MSM_PIN_FUNCTION(gsbi7),
+	MSM_PIN_FUNCTION(nss_spi),
+	MSM_PIN_FUNCTION(sdc1),
+	MSM_PIN_FUNCTION(spdif),
+	MSM_PIN_FUNCTION(nand),
+	MSM_PIN_FUNCTION(tsif1),
+	MSM_PIN_FUNCTION(tsif2),
+	MSM_PIN_FUNCTION(usb_fs_n),
+	MSM_PIN_FUNCTION(usb_fs),
+	MSM_PIN_FUNCTION(usb2_hsic),
+	MSM_PIN_FUNCTION(rgmii2),
+	MSM_PIN_FUNCTION(sata),
+	MSM_GPIO_PIN_FUNCTION(pcie1_rst),
+	MSM_PIN_FUNCTION(pcie1_prsnt),
+	MSM_PIN_FUNCTION(pcie1_pwren_n),
+	MSM_PIN_FUNCTION(pcie1_pwren),
+	MSM_PIN_FUNCTION(pcie1_pwrflt),
+	MSM_PIN_FUNCTION(pcie1_clk_req),
+	MSM_GPIO_PIN_FUNCTION(pcie2_rst),
+	MSM_PIN_FUNCTION(pcie2_prsnt),
+	MSM_PIN_FUNCTION(pcie2_pwren_n),
+	MSM_PIN_FUNCTION(pcie2_pwren),
+	MSM_PIN_FUNCTION(pcie2_pwrflt),
+	MSM_PIN_FUNCTION(pcie2_clk_req),
+	MSM_GPIO_PIN_FUNCTION(pcie3_rst),
+	MSM_PIN_FUNCTION(pcie3_prsnt),
+	MSM_PIN_FUNCTION(pcie3_pwren_n),
+	MSM_PIN_FUNCTION(pcie3_pwren),
+	MSM_PIN_FUNCTION(pcie3_pwrflt),
+	MSM_PIN_FUNCTION(pcie3_clk_req),
+	MSM_PIN_FUNCTION(ps_hold),
 };
 
 static const struct msm_pingroup ipq8064_groups[] = {
diff --git a/drivers/pinctrl/qcom/pinctrl-kuno.c b/drivers/pinctrl/qcom/pinctrl-kuno.c
new file mode 100644
index 00000000000000..404065df2b9f81
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-kuno.c
@@ -0,0 +1,801 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-msm.h"
+
+#define REG_BASE 0x100000
+#define REG_SIZE 0x1000
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9)		\
+	{					        \
+		.grp = PINCTRL_PINGROUP("gpio" #id,	\
+			gpio##id##_pins,		\
+			ARRAY_SIZE(gpio##id##_pins)),	\
+		.funcs = (int[]){			\
+			msm_mux_gpio, /* gpio mode */	\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9			\
+		},				        \
+		.nfuncs = 10,				\
+		.ctl_reg = REG_BASE + REG_SIZE * id,			\
+		.io_reg = REG_BASE + 0x4 + REG_SIZE * id,		\
+		.intr_cfg_reg = REG_BASE + 0x8 + REG_SIZE * id,		\
+		.intr_status_reg = REG_BASE + 0xc + REG_SIZE * id,	\
+		.mux_bit = 2,			\
+		.pull_bit = 0,			\
+		.drv_bit = 6,			\
+		.oe_bit = 9,			\
+		.in_bit = 0,			\
+		.out_bit = 1,			\
+		.intr_enable_bit = 0,		\
+		.intr_status_bit = 0,		\
+		.intr_target_bit = 5,		\
+		.intr_target_kpss_val = 3,	\
+		.intr_raw_status_bit = 4,	\
+		.intr_polarity_bit = 1,		\
+		.intr_detection_bit = 2,	\
+		.intr_detection_width = 2,	\
+	}
+
+static const struct pinctrl_pin_desc kuno_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),
+	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),
+	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),
+	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),
+	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),
+	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),
+	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),
+	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),
+	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),
+	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),
+	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),
+	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),
+	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),
+	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),
+	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),
+	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),
+	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),
+	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),
+	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),
+	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),
+	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),
+	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),
+	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),
+	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),
+	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),
+	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),
+	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),
+	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),
+	PINCTRL_PIN(109, "GPIO_109"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+
+enum kuno_functions {
+	msm_mux_gpio,
+	msm_mux_audio_ref_clk,
+	msm_mux_coex_uart,
+	msm_mux_ebi2_lcd_a,
+	msm_mux_ebi2_lcd_cs,
+	msm_mux_ebi2_lcd_reset,
+	msm_mux_ebi2_lcd_te,
+	msm_mux_emac_mdc,
+	msm_mux_emac_mdio,
+	msm_mux_emac_pps_in,
+	msm_mux_emac_ptp_aux,
+	msm_mux_emac_ptp_pps,
+	msm_mux_gcc_gp1_clk,
+	msm_mux_gcc_gp2_clk,
+	msm_mux_gcc_gp3_clk,
+	msm_mux_mi2s0_data0,
+	msm_mux_mi2s0_data1,
+	msm_mux_mi2s0_sck,
+	msm_mux_mi2s0_ws,
+	msm_mux_mi2s1_data0,
+	msm_mux_mi2s1_data1,
+	msm_mux_mi2s1_sck,
+	msm_mux_mi2s1_ws,
+	msm_mux_mi2s_mclk,
+	msm_mux_nav_gpio,
+	msm_mux_pci_e_rst,
+	msm_mux_pcie_clkreq_n,
+	msm_mux_pll_bist_sync,
+	msm_mux_pll_clk_aux,
+	msm_mux_qdss_cti_trig0,
+	msm_mux_qdss_cti_trig1,
+	msm_mux_qdss_cti_trig1_mire,
+	msm_mux_qdss_gpio_traceclk,
+	msm_mux_qdss_gpio_tracectl,
+	msm_mux_qup0_se0,
+	msm_mux_qup0_se1,
+	msm_mux_qup0_se2,
+	msm_mux_qup0_se3_mira,
+	msm_mux_qup0_se3_mirb,
+	msm_mux_qup0_se4,
+	msm_mux_sdc4_clk,
+	msm_mux_sdc4_cmd,
+	msm_mux_sdc4_data,
+	msm_mux_sdc4_tb_trig,
+	msm_mux_sgmii_phy_intr,
+	msm_mux_spmi_coex_clk,
+	msm_mux_spmi_coex_data,
+	msm_mux_spmi_vgi_hwevent,
+	msm_mux_uim1_clk,
+	msm_mux_uim1_data,
+	msm_mux_uim1_present,
+	msm_mux_uim1_reset,
+	msm_mux_usb2phy_ac_en,
+	msm_mux__,
+};
+
+static const char * const gpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7",
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14",
+	"gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20", "gpio21",
+	"gpio22", "gpio23", "gpio24", "gpio25", "gpio26", "gpio27", "gpio28",
+	"gpio29", "gpio30", "gpio31", "gpio32", "gpio33", "gpio34", "gpio35",
+	"gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41", "gpio42",
+	"gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio48", "gpio49",
+	"gpio50", "gpio51", "gpio52", "gpio53", "gpio54", "gpio55", "gpio56",
+	"gpio57", "gpio58", "gpio59", "gpio60", "gpio61", "gpio62", "gpio63",
+	"gpio64", "gpio65", "gpio66", "gpio67", "gpio68", "gpio69", "gpio70",
+	"gpio71", "gpio72", "gpio73", "gpio74", "gpio75", "gpio76", "gpio77",
+	"gpio78", "gpio79", "gpio80", "gpio81", "gpio82", "gpio83", "gpio84",
+	"gpio85", "gpio86", "gpio87", "gpio88", "gpio89", "gpio90", "gpio91",
+	"gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio97", "gpio98",
+	"gpio99", "gpio100", "gpio101", "gpio102", "gpio103", "gpio104",
+	"gpio105", "gpio106", "gpio107", "gpio108", "gpio109",
+};
+
+static const char * const audio_ref_clk_groups[] = {
+	"gpio62",
+};
+
+static const char * const coex_uart_groups[] = {
+	"gpio44", "gpio45",
+};
+
+static const char * const ebi2_lcd_a_groups[] = {
+	"gpio93",
+};
+
+static const char * const ebi2_lcd_cs_groups[] = {
+	"gpio94",
+};
+
+static const char * const ebi2_lcd_reset_groups[] = {
+	"gpio89",
+};
+
+static const char * const ebi2_lcd_te_groups[] = {
+	"gpio88",
+};
+
+static const char * const emac_mdc_groups[] = {
+	"gpio98",
+};
+
+static const char * const emac_mdio_groups[] = {
+	"gpio99",
+};
+
+static const char * const emac_pps_in_groups[] = {
+	"gpio88",
+};
+
+static const char * const emac_ptp_aux_groups[] = {
+	"gpio93", "gpio94",
+};
+
+static const char * const emac_ptp_pps_groups[] = {
+	"gpio93", "gpio94",
+};
+
+static const char * const gcc_gp1_clk_groups[] = {
+	"gpio34",
+};
+
+static const char * const gcc_gp2_clk_groups[] = {
+	"gpio46",
+};
+
+static const char * const gcc_gp3_clk_groups[] = {
+	"gpio35",
+};
+
+static const char * const mi2s0_data0_groups[] = {
+	"gpio13",
+};
+
+static const char * const mi2s0_data1_groups[] = {
+	"gpio14",
+};
+
+static const char * const mi2s0_sck_groups[] = {
+	"gpio15",
+};
+
+static const char * const mi2s0_ws_groups[] = {
+	"gpio12",
+};
+
+static const char * const mi2s1_data0_groups[] = {
+	"gpio17",
+};
+
+static const char * const mi2s1_data1_groups[] = {
+	"gpio18",
+};
+
+static const char * const mi2s1_sck_groups[] = {
+	"gpio19",
+};
+
+static const char * const mi2s1_ws_groups[] = {
+	"gpio16",
+};
+
+static const char * const mi2s_mclk_groups[] = {
+	"gpio62",
+};
+
+static const char * const nav_gpio_groups[] = {
+	"gpio31", "gpio32", "gpio33",
+};
+
+static const char * const pci_e_rst_groups[] = {
+	"gpio57",
+};
+
+static const char * const pcie_clkreq_n_groups[] = {
+	"gpio56",
+};
+
+static const char * const pll_bist_sync_groups[] = {
+	"gpio47",
+};
+
+static const char * const pll_clk_aux_groups[] = {
+	"gpio54",
+};
+
+static const char * const qdss_cti_trig0_groups[] = {
+	"gpio16", "gpio17", "gpio54", "gpio55", "gpio60", "gpio84", "gpio101",
+	"gpio102",
+};
+
+static const char * const qdss_cti_trig1_groups[] = {
+	"gpio16", "gpio17", "gpio54", "gpio55", "gpio65", "gpio66", "gpio101",
+	"gpio102",
+};
+static const char * const qdss_cti_trig1_mire_groups[] = {
+	"gpio65", "gpio66",
+};
+
+static const char * const qdss_gpio_traceclk_groups[] = {
+	"gpio8",
+};
+
+static const char * const qdss_gpio_tracectl_groups[] = {
+	"gpio9",
+};
+
+static const char * const qup0_se0_groups[] = {
+	"gpio14", "gpio18", "gpio19", "gpio48", "gpio49", "gpio92", "gpio101",
+};
+
+static const char * const qup0_se1_groups[] = {
+	"gpio14", "gpio63", "gpio64", "gpio65", "gpio66", "gpio92", "gpio101",
+};
+
+static const char * const qup0_se2_groups[] = {
+	"gpio4", "gpio5", "gpio6", "gpio7", "gpio14", "gpio92", "gpio101",
+};
+
+static const char * const qup0_se3_mira_groups[] = {
+	"gpio8", "gpio9", "gpio14", "gpio16", "gpio17", "gpio92", "gpio101",
+};
+
+static const char * const qup0_se3_mirb_groups[] = {
+	"gpio12", "gpio13", "gpio14", "gpio15",
+};
+
+static const char * const qup0_se4_groups[] = {
+	"gpio10", "gpio11", "gpio14", "gpio80", "gpio81", "gpio92", "gpio101",
+};
+
+static const char * const sdc4_clk_groups[] = {
+	"gpio103",
+};
+
+static const char * const sdc4_cmd_groups[] = {
+	"gpio102",
+};
+
+static const char * const sdc4_data_groups[] = {
+	"gpio104", "gpio105", "gpio106", "gpio107",
+};
+
+static const char * const sdc4_tb_trig_groups[] = {
+	"gpio101",
+};
+
+static const char * const sgmii_phy_intr_groups[] = {
+	"gpio91",
+};
+
+static const char * const spmi_coex_clk_groups[] = {
+	"gpio76",
+};
+
+static const char * const spmi_coex_data_groups[] = {
+	"gpio75",
+};
+
+static const char * const spmi_vgi_hwevent_groups[] = {
+	"gpio78", "gpio79",
+};
+
+static const char * const uim1_clk_groups[] = {
+	"gpio3",
+};
+
+static const char * const uim1_data_groups[] = {
+	"gpio0",
+};
+
+static const char * const uim1_present_groups[] = {
+	"gpio1",
+};
+
+static const char * const uim1_reset_groups[] = {
+	"gpio2",
+};
+
+static const char * const usb2phy_ac_en_groups[] = {
+	"gpio90",
+};
+
+static const struct pinfunction kuno_functions[] = {
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(audio_ref_clk),
+	MSM_PIN_FUNCTION(coex_uart),
+	MSM_PIN_FUNCTION(ebi2_lcd_a),
+	MSM_PIN_FUNCTION(ebi2_lcd_cs),
+	MSM_PIN_FUNCTION(ebi2_lcd_reset),
+	MSM_PIN_FUNCTION(ebi2_lcd_te),
+	MSM_PIN_FUNCTION(emac_mdc),
+	MSM_PIN_FUNCTION(emac_mdio),
+	MSM_PIN_FUNCTION(emac_pps_in),
+	MSM_PIN_FUNCTION(emac_ptp_aux),
+	MSM_PIN_FUNCTION(emac_ptp_pps),
+	MSM_PIN_FUNCTION(gcc_gp1_clk),
+	MSM_PIN_FUNCTION(gcc_gp2_clk),
+	MSM_PIN_FUNCTION(gcc_gp3_clk),
+	MSM_PIN_FUNCTION(mi2s0_data0),
+	MSM_PIN_FUNCTION(mi2s0_data1),
+	MSM_PIN_FUNCTION(mi2s0_sck),
+	MSM_PIN_FUNCTION(mi2s0_ws),
+	MSM_PIN_FUNCTION(mi2s1_data0),
+	MSM_PIN_FUNCTION(mi2s1_data1),
+	MSM_PIN_FUNCTION(mi2s1_sck),
+	MSM_PIN_FUNCTION(mi2s1_ws),
+	MSM_PIN_FUNCTION(mi2s_mclk),
+	MSM_PIN_FUNCTION(nav_gpio),
+	MSM_PIN_FUNCTION(pci_e_rst),
+	MSM_PIN_FUNCTION(pcie_clkreq_n),
+	MSM_PIN_FUNCTION(pll_bist_sync),
+	MSM_PIN_FUNCTION(pll_clk_aux),
+	MSM_PIN_FUNCTION(qdss_cti_trig0),
+	MSM_PIN_FUNCTION(qdss_cti_trig1),
+	MSM_PIN_FUNCTION(qdss_cti_trig1_mire),
+	MSM_PIN_FUNCTION(qdss_gpio_traceclk),
+	MSM_PIN_FUNCTION(qdss_gpio_tracectl),
+	MSM_PIN_FUNCTION(qup0_se0),
+	MSM_PIN_FUNCTION(qup0_se1),
+	MSM_PIN_FUNCTION(qup0_se2),
+	MSM_PIN_FUNCTION(qup0_se3_mira),
+	MSM_PIN_FUNCTION(qup0_se3_mirb),
+	MSM_PIN_FUNCTION(qup0_se4),
+	MSM_PIN_FUNCTION(sdc4_clk),
+	MSM_PIN_FUNCTION(sdc4_cmd),
+	MSM_PIN_FUNCTION(sdc4_data),
+	MSM_PIN_FUNCTION(sdc4_tb_trig),
+	MSM_PIN_FUNCTION(sgmii_phy_intr),
+	MSM_PIN_FUNCTION(spmi_coex_clk),
+	MSM_PIN_FUNCTION(spmi_coex_data),
+	MSM_PIN_FUNCTION(spmi_vgi_hwevent),
+	MSM_PIN_FUNCTION(uim1_clk),
+	MSM_PIN_FUNCTION(uim1_data),
+	MSM_PIN_FUNCTION(uim1_present),
+	MSM_PIN_FUNCTION(uim1_reset),
+	MSM_PIN_FUNCTION(usb2phy_ac_en),
+};
+
+/* Every pin is maintained as a single group, and missing or non-existing pin
+ * would be maintained as dummy group to synchronize pin group index with
+ * pin descriptor registered with pinctrl core.
+ * Clients would not be able to request these dummy pin groups.
+ */
+static const struct msm_pingroup kuno_groups[] = {
+	[0] = PINGROUP(0, uim1_data, _, _, _, _, _, _, _, _),
+	[1] = PINGROUP(1, uim1_present, _, _, _, _, _, _, _, _),
+	[2] = PINGROUP(2, uim1_reset, _, _, _, _, _, _, _, _),
+	[3] = PINGROUP(3, uim1_clk, _, _, _, _, _, _, _, _),
+	[4] = PINGROUP(4, qup0_se2, _, _, _, _, _, _, _, _),
+	[5] = PINGROUP(5, qup0_se2, _, _, _, _, _, _, _, _),
+	[6] = PINGROUP(6, qup0_se2, _, _, _, _, _, _, _, _),
+	[7] = PINGROUP(7, qup0_se2, _, _, _, _, _, _, _, _),
+	[8] = PINGROUP(8, qup0_se3_mira, _, qdss_gpio_traceclk, _, _, _, _, _, _),
+	[9] = PINGROUP(9, qup0_se3_mira, _, qdss_gpio_tracectl, _, _, _, _, _, _),
+	[10] = PINGROUP(10, qup0_se4, _, _, _, _, _, _, _, _),
+	[11] = PINGROUP(11, qup0_se4, _, _, _, _, _, _, _, _),
+	[12] = PINGROUP(12, mi2s0_ws, qup0_se3_mirb, _, _, _, _, _, _, _),
+	[13] = PINGROUP(13, mi2s0_data0, qup0_se3_mirb, _, _, _, _, _, _, _),
+	[14] = PINGROUP(14, mi2s0_data1, qup0_se3_mirb, qup0_se0, qup0_se1, qup0_se2,
+			qup0_se3_mira, qup0_se4, _, _),
+	[15] = PINGROUP(15, mi2s0_sck, qup0_se3_mirb, _, _, _, _, _, _, _),
+	[16] = PINGROUP(16, mi2s1_ws, qup0_se3_mira, qdss_cti_trig1, qdss_cti_trig0,
+			_, _, _, _, _),
+	[17] = PINGROUP(17, mi2s1_data0, qup0_se3_mira, qdss_cti_trig1, qdss_cti_trig0,
+			_, _, _, _, _),
+	[18] = PINGROUP(18, mi2s1_data1, qup0_se0, _, _, _, _, _, _, _),
+	[19] = PINGROUP(19, mi2s1_sck, qup0_se0, _, _, _, _, _, _, _),
+	[20] = PINGROUP(20, _, _, _, _, _, _, _, _, _),
+	[21] = PINGROUP(21, _, _, _, _, _, _, _, _, _),
+	[22] = PINGROUP(22, _, _, _, _, _, _, _, _, _),
+	[23] = PINGROUP(23, _, _, _, _, _, _, _, _, _),
+	[24] = PINGROUP(24, _, _, _, _, _, _, _, _, _),
+	[25] = PINGROUP(25, _, _, _, _, _, _, _, _, _),
+	[26] = PINGROUP(26, _, _, _, _, _, _, _, _, _),
+	[27] = PINGROUP(27, _, _, _, _, _, _, _, _, _),
+	[28] = PINGROUP(28, _, _, _, _, _, _, _, _, _),
+	[29] = PINGROUP(29, _, _, _, _, _, _, _, _, _),
+	[30] = PINGROUP(30, _, _, _, _, _, _, _, _, _),
+	[31] = PINGROUP(31, nav_gpio, _, _, _, _, _, _, _, _),
+	[32] = PINGROUP(32, nav_gpio, _, _, _, _, _, _, _, _),
+	[33] = PINGROUP(33, nav_gpio, _, _, _, _, _, _, _, _),
+	[34] = PINGROUP(34, _, gcc_gp1_clk, _, _, _, _, _, _, _),
+	[35] = PINGROUP(35, _, _, gcc_gp3_clk, _, _, _, _, _, _),
+	[36] = PINGROUP(36, _, _, _, _, _, _, _, _, _),
+	[37] = PINGROUP(37, _, _, _, _, _, _, _, _, _),
+	[38] = PINGROUP(38, _, _, _, _, _, _, _, _, _),
+	[39] = PINGROUP(39, _, _, _, _, _, _, _, _, _),
+	[40] = PINGROUP(40, _, _, _, _, _, _, _, _, _),
+	[41] = PINGROUP(41, _, _, _, _, _, _, _, _, _),
+	[42] = PINGROUP(42, _, _, _, _, _, _, _, _, _),
+	[43] = PINGROUP(43, _, _, _, _, _, _, _, _, _),
+	[44] = PINGROUP(44, coex_uart, _, _, _, _, _, _, _, _),
+	[45] = PINGROUP(45, coex_uart, _, _, _, _, _, _, _, _),
+	[46] = PINGROUP(46, _, _, _, gcc_gp2_clk, _, _, _, _, _),
+	[47] = PINGROUP(47, _, pll_bist_sync, _, _, _, _, _, _, _),
+	[48] = PINGROUP(48, qup0_se0, _, _, _, _, _, _, _, _),
+	[49] = PINGROUP(49, qup0_se0, _, _, _, _, _, _, _, _),
+	[50] = PINGROUP(50, _, _, _, _, _, _, _, _, _),
+	[51] = PINGROUP(51, _, _, _, _, _, _, _, _, _),
+	[52] = PINGROUP(52, _, _, _, _, _, _, _, _, _),
+	[53] = PINGROUP(53, _, _, _, _, _, _, _, _, _),
+	[54] = PINGROUP(54, qdss_cti_trig1, qdss_cti_trig0, pll_clk_aux, _, _, _, _, _, _),
+	[55] = PINGROUP(55, qdss_cti_trig1, qdss_cti_trig0, _, _, _, _, _, _, _),
+	[56] = PINGROUP(56, pcie_clkreq_n, _, _, _, _, _, _, _, _),
+	[57] = PINGROUP(57, pci_e_rst, _, _, _, _, _, _, _, _),
+	[58] = PINGROUP(58, _, _, _, _, _, _, _, _, _),
+	[59] = PINGROUP(59, _, _, _, _, _, _, _, _, _),
+	[60] = PINGROUP(60, qdss_cti_trig0, _, _, _, _, _, _, _, _),
+	[61] = PINGROUP(61, _, _, _, _, _, _, _, _, _),
+	[62] = PINGROUP(62, mi2s_mclk, audio_ref_clk, _, _, _, _, _, _, _),
+	[63] = PINGROUP(63, qup0_se1, _, _, _, _, _, _, _, _),
+	[64] = PINGROUP(64, qup0_se1, _, _, _, _, _, _, _, _),
+	[65] = PINGROUP(65, qup0_se1, qdss_cti_trig1, qdss_cti_trig1_mire, _, _, _, _, _, _),
+	[66] = PINGROUP(66, qup0_se1, qdss_cti_trig1, qdss_cti_trig1_mire, _, _, _, _, _, _),
+	[67] = PINGROUP(67, _, _, _, _, _, _, _, _, _),
+	[68] = PINGROUP(68, _, _, _, _, _, _, _, _, _),
+	[69] = PINGROUP(69, _, _, _, _, _, _, _, _, _),
+	[70] = PINGROUP(70, _, _, _, _, _, _, _, _, _),
+	[71] = PINGROUP(71, _, _, _, _, _, _, _, _, _),
+	[72] = PINGROUP(72, _, _, _, _, _, _, _, _, _),
+	[73] = PINGROUP(73, _, _, _, _, _, _, _, _, _),
+	[74] = PINGROUP(74, _, _, _, _, _, _, _, _, _),
+	[75] = PINGROUP(75, spmi_coex_data, _, _, _, _, _, _, _, _),
+	[76] = PINGROUP(76, spmi_coex_clk, _, _, _, _, _, _, _, _),
+	[77] = PINGROUP(77, _, _, _, _, _, _, _, _, _),
+	[78] = PINGROUP(78, spmi_vgi_hwevent, _, _, _, _, _, _, _, _),
+	[79] = PINGROUP(79, spmi_vgi_hwevent, _, _, _, _, _, _, _, _),
+	[80] = PINGROUP(80, qup0_se4, _, _, _, _, _, _, _, _),
+	[81] = PINGROUP(81, qup0_se4, _, _, _, _, _, _, _, _),
+	[82] = PINGROUP(82, _, _, _, _, _, _, _, _, _),
+	[83] = PINGROUP(83, _, _, _, _, _, _, _, _, _),
+	[84] = PINGROUP(84, qdss_cti_trig0, _, _, _, _, _, _, _, _),
+	[85] = PINGROUP(85, _, _, _, _, _, _, _, _, _),
+	[86] = PINGROUP(86, _, _, _, _, _, _, _, _, _),
+	[87] = PINGROUP(87, _, _, _, _, _, _, _, _, _),
+	[88] = PINGROUP(88, ebi2_lcd_te, emac_pps_in, _, _, _, _, _, _, _),
+	[89] = PINGROUP(89, ebi2_lcd_reset, _, _, _, _, _, _, _, _),
+	[90] = PINGROUP(90, usb2phy_ac_en, _, _, _, _, _, _, _, _),
+	[91] = PINGROUP(91, sgmii_phy_intr, _, _, _, _, _, _, _, _),
+	[92] = PINGROUP(92, qup0_se0, qup0_se1, qup0_se2, qup0_se3_mira, qup0_se4, _, _, _, _),
+	[93] = PINGROUP(93, ebi2_lcd_a, emac_ptp_pps, emac_ptp_aux, _, _, _, _, _, _),
+	[94] = PINGROUP(94, ebi2_lcd_cs, emac_ptp_pps, emac_ptp_aux, _, _, _, _, _, _),
+	[95] = PINGROUP(95, _, _, _, _, _, _, _, _, _),
+	[96] = PINGROUP(96, _, _, _, _, _, _, _, _, _),
+	[97] = PINGROUP(97, _, _, _, _, _, _, _, _, _),
+	[98] = PINGROUP(98, emac_mdc, _, _, _, _, _, _, _, _),
+	[99] = PINGROUP(99, emac_mdio, _, _, _, _, _, _, _, _),
+	[100] = PINGROUP(100, _, _, _, _, _, _, _, _, _),
+	[101] = PINGROUP(101, sdc4_tb_trig, qdss_cti_trig1, qdss_cti_trig0, qup0_se0,
+			qup0_se1, qup0_se2, qup0_se3_mira, qup0_se4, _),
+	[102] = PINGROUP(102, sdc4_cmd, qdss_cti_trig1, qdss_cti_trig0, _, _, _, _, _, _),
+	[103] = PINGROUP(103, sdc4_clk, _, _, _, _, _, _, _, _),
+	[104] = PINGROUP(104, sdc4_data, _, _, _, _, _, _, _, _),
+	[105] = PINGROUP(105, sdc4_data, _, _, _, _, _, _, _, _),
+	[106] = PINGROUP(106, sdc4_data, _, _, _, _, _, _, _, _),
+	[107] = PINGROUP(107, sdc4_data, _, _, _, _, _, _, _, _),
+	[108] = PINGROUP(108, _, _, _, _, _, _, _, _, _),
+	[109] = PINGROUP(109, _, _, _, _, _, _, _, _, _),
+};
+
+static const struct msm_gpio_wakeirq_map kuno_pdc_map[] = {
+	{ 1, 62 }, { 2, 83 }, { 4, 59 }, { 5, 107 }, { 6, 112 }, { 7, 119 },
+	{ 8, 123 }, { 10, 52 }, { 11, 73 }, { 12, 74 }, { 13, 75 }, { 14, 76 },
+	{ 15, 77 }, { 16, 81 }, { 17, 82 }, { 18, 80 }, { 19, 100 }, { 21, 88 },
+	{ 22, 99 }, { 23, 84 }, { 24, 85 }, { 25, 97 }, { 26, 86 }, { 27, 102 },
+	{ 28, 87 }, { 35, 103 }, { 43, 90 }, { 44, 60 }, { 45, 61 }, { 46, 104 },
+	{ 47, 89 }, { 48, 105 }, { 49, 106 }, { 50, 101 }, { 52, 109 }, { 53, 79 },
+	{ 54, 110 }, { 55, 111 }, { 56, 78 }, { 57, 71 }, { 60, 113 }, { 61, 114 },
+	{ 63, 115 }, { 64, 116 }, { 65, 57 }, { 66, 117 }, { 68, 64 }, { 69, 118 },
+	{ 71, 108 }, { 73, 120 }, { 74, 121 }, { 76, 47 }, { 77, 122 }, { 78, 65 },
+	{ 79, 66 }, { 80, 124 }, { 81, 125 }, { 82, 126 }, { 83, 127 }, { 84, 93 },
+	{ 85, 128 }, { 86, 129 }, { 87, 70 }, { 88, 130 }, { 96, 68 }, { 98, 50 },
+	{ 99, 67 }, { 100, 51 }, { 101, 53 }, { 102, 54 }, { 103, 98 }, { 104, 91 },
+	{ 105, 69 }, { 106, 55 }, { 107, 56 }, { 109, 72 },
+};
+
+static const struct msm_pinctrl_soc_data kuno_tlmm = {
+	.pins = kuno_pins,
+	.npins = ARRAY_SIZE(kuno_pins),
+	.functions = kuno_functions,
+	.nfunctions = ARRAY_SIZE(kuno_functions),
+	.groups = kuno_groups,
+	.ngroups = ARRAY_SIZE(kuno_groups),
+	.ngpios = 110,
+	.wakeirq_map = kuno_pdc_map,
+	.nwakeirq_map = ARRAY_SIZE(kuno_pdc_map),
+};
+
+static int kuno_tlmm_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &kuno_tlmm);
+}
+
+static const struct of_device_id kuno_tlmm_of_match[] = {
+	{ .compatible = "qcom,kuno-tlmm", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, kuno_tlmm_of_match);
+
+static struct platform_driver kuno_tlmm_driver = {
+	.driver = {
+		.name = "kuno-tlmm",
+		.of_match_table = kuno_tlmm_of_match,
+	},
+	.probe = kuno_tlmm_probe,
+};
+
+static int __init kuno_tlmm_init(void)
+{
+	return platform_driver_register(&kuno_tlmm_driver);
+}
+arch_initcall(kuno_tlmm_init);
+
+static void __exit kuno_tlmm_exit(void)
+{
+	platform_driver_unregister(&kuno_tlmm_driver);
+}
+module_exit(kuno_tlmm_exit);
+
+MODULE_DESCRIPTION("QTI Kuno TLMM driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
index 5fd4a4eba654de..9412fd36a4aae5 100644
--- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
+++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
@@ -421,6 +421,7 @@ static void lpi_gpio_dbg_show_one(struct seq_file *s,
 	struct pinctrl_pin_desc pindesc;
 	unsigned int func;
 	int is_out;
+	int value;
 	int drive;
 	int pull;
 	u32 ctl_reg;
@@ -443,7 +444,10 @@ static void lpi_gpio_dbg_show_one(struct seq_file *s,
 	drive = FIELD_GET(LPI_GPIO_OUT_STRENGTH_MASK, ctl_reg);
 	pull = FIELD_GET(LPI_GPIO_PULL_MASK, ctl_reg);
 
-	seq_printf(s, " %-8s: %-3s %d", pindesc.name, is_out ? "out" : "in", func);
+	value = lpi_gpio_get(chip, offset);
+
+	seq_printf(s, " %-8s: %-3s", pindesc.name, is_out ? "out" : "in");
+	seq_printf(s, " %-4s func%d", str_high_low(value), func);
 	seq_printf(s, " %dmA", lpi_regval_to_drive(drive));
 	seq_printf(s, " %s", pulls[pull]);
 }
diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
index 6ba0c4eba984c9..056c4a7743893b 100644
--- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
+++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
@@ -92,6 +92,23 @@ struct pinctrl_pin_desc;
 		.pin_offset = 0,				\
 	}
 
+#define LPI_PINGROUP6(id, soff, f1, f2, f3, f4, f5)		\
+	{							\
+		.pin = id,					\
+		.slew_offset = soff,				\
+		.slew_base_spare_1 = false,			\
+		.funcs = (int[]){				\
+			LPI_MUX_gpio,				\
+			LPI_MUX_##f1,				\
+			LPI_MUX_##f2,				\
+			LPI_MUX_##f3,				\
+			LPI_MUX_##f4,				\
+			LPI_MUX_##f5,				\
+		},						\
+		.nfuncs = 6,					\
+		.pin_offset = 0,				\
+	}
+
 /*
  * Slew rate control is done in the same register as rest of the
  * pin configuration.
diff --git a/drivers/pinctrl/qcom/pinctrl-maili.c b/drivers/pinctrl/qcom/pinctrl-maili.c
index ffa084cfad6487..b9ff0f44794106 100644
--- a/drivers/pinctrl/qcom/pinctrl-maili.c
+++ b/drivers/pinctrl/qcom/pinctrl-maili.c
@@ -47,6 +47,7 @@
 		.intr_wakeup_present_bit = 6,		\
 		.intr_wakeup_enable_bit = 7,		\
 		.intr_target_bit = 8,			\
+		.intr_target_width = 4,			\
 		.intr_target_kpss_val = 3,		\
 		.intr_raw_status_bit = 4,		\
 		.intr_polarity_bit = 1,			\
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.h b/drivers/pinctrl/qcom/pinctrl-msm.h
index b94ba1a4177ede..342bf306e9be97 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.h
+++ b/drivers/pinctrl/qcom/pinctrl-msm.h
@@ -14,21 +14,6 @@ struct platform_device;
 
 struct pinctrl_pin_desc;
 
-#define APQ_PIN_FUNCTION(fname)					\
-	[APQ_MUX_##fname] = PINCTRL_PINFUNCTION(#fname,		\
-					fname##_groups,		\
-					ARRAY_SIZE(fname##_groups))
-
-#define IPQ_PIN_FUNCTION(fname)					\
-	[IPQ_MUX_##fname] = PINCTRL_PINFUNCTION(#fname,		\
-					fname##_groups,		\
-					ARRAY_SIZE(fname##_groups))
-
-#define IPQ_GPIO_PIN_FUNCTION(fname)					\
-	[IPQ_MUX_##fname] = PINCTRL_GPIO_PINFUNCTION(#fname,		\
-					fname##_groups,		\
-					ARRAY_SIZE(fname##_groups))
-
 #define MSM_PIN_FUNCTION(fname) 				\
 	[msm_mux_##fname] = PINCTRL_PINFUNCTION(#fname,		\
 					fname##_groups,		\
@@ -39,16 +24,6 @@ struct pinctrl_pin_desc;
 					fname##_groups,		\
 					ARRAY_SIZE(fname##_groups))
 
-#define QCA_PIN_FUNCTION(fname)					\
-	[qca_mux_##fname] = PINCTRL_PINFUNCTION(#fname,		\
-					fname##_groups,		\
-					ARRAY_SIZE(fname##_groups))
-
-#define QCA_GPIO_PIN_FUNCTION(fname)				\
-	[qca_mux_##fname] = PINCTRL_GPIO_PINFUNCTION(#fname,	\
-					fname##_groups,		\
-					ARRAY_SIZE(fname##_groups))
-
 /**
  * struct msm_pingroup - Qualcomm pingroup definition
  * @grp:                  Generic data of the pin group (name and pins)
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8952.c b/drivers/pinctrl/qcom/pinctrl-msm8952.c
new file mode 100644
index 00000000000000..6f83255cdc5587
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-msm8952.c
@@ -0,0 +1,1257 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2013, Sony Mobile Communications AB.
+ * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
+ * Copyright (C) 2026 Muzaffer Kadir <muzafferkadir@mainlining.org>
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-msm.h"
+
+static const struct pinctrl_pin_desc msm8952_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),	PINCTRL_PIN(109, "GPIO_109"),
+	PINCTRL_PIN(110, "GPIO_110"),	PINCTRL_PIN(111, "GPIO_111"),
+	PINCTRL_PIN(112, "GPIO_112"),	PINCTRL_PIN(113, "GPIO_113"),
+	PINCTRL_PIN(114, "GPIO_114"),	PINCTRL_PIN(115, "GPIO_115"),
+	PINCTRL_PIN(116, "GPIO_116"),	PINCTRL_PIN(117, "GPIO_117"),
+	PINCTRL_PIN(118, "GPIO_118"),	PINCTRL_PIN(119, "GPIO_119"),
+	PINCTRL_PIN(120, "GPIO_120"),	PINCTRL_PIN(121, "GPIO_121"),
+	PINCTRL_PIN(122, "GPIO_122"),	PINCTRL_PIN(123, "GPIO_123"),
+	PINCTRL_PIN(124, "GPIO_124"),	PINCTRL_PIN(125, "GPIO_125"),
+	PINCTRL_PIN(126, "GPIO_126"),	PINCTRL_PIN(127, "GPIO_127"),
+	PINCTRL_PIN(128, "GPIO_128"),	PINCTRL_PIN(129, "GPIO_129"),
+	PINCTRL_PIN(130, "GPIO_130"),	PINCTRL_PIN(131, "GPIO_131"),
+	PINCTRL_PIN(132, "GPIO_132"),	PINCTRL_PIN(133, "GPIO_133"),
+	PINCTRL_PIN(134, "SDC1_CLK"),	PINCTRL_PIN(135, "SDC1_CMD"),
+	PINCTRL_PIN(136, "SDC1_DATA"),	PINCTRL_PIN(137, "SDC1_RCLK"),
+	PINCTRL_PIN(138, "SDC2_CLK"),	PINCTRL_PIN(139, "SDC2_CMD"),
+	PINCTRL_PIN(140, "SDC2_DATA"),	PINCTRL_PIN(141, "QDSD_CLK"),
+	PINCTRL_PIN(142, "QDSD_CMD"),	PINCTRL_PIN(143, "QDSD_DATA0"),
+	PINCTRL_PIN(144, "QDSD_DATA1"), PINCTRL_PIN(145, "QDSD_DATA2"),
+	PINCTRL_PIN(146, "QDSD_DATA3"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+DECLARE_MSM_GPIO_PINS(110);
+DECLARE_MSM_GPIO_PINS(111);
+DECLARE_MSM_GPIO_PINS(112);
+DECLARE_MSM_GPIO_PINS(113);
+DECLARE_MSM_GPIO_PINS(114);
+DECLARE_MSM_GPIO_PINS(115);
+DECLARE_MSM_GPIO_PINS(116);
+DECLARE_MSM_GPIO_PINS(117);
+DECLARE_MSM_GPIO_PINS(118);
+DECLARE_MSM_GPIO_PINS(119);
+DECLARE_MSM_GPIO_PINS(120);
+DECLARE_MSM_GPIO_PINS(121);
+DECLARE_MSM_GPIO_PINS(122);
+DECLARE_MSM_GPIO_PINS(123);
+DECLARE_MSM_GPIO_PINS(124);
+DECLARE_MSM_GPIO_PINS(125);
+DECLARE_MSM_GPIO_PINS(126);
+DECLARE_MSM_GPIO_PINS(127);
+DECLARE_MSM_GPIO_PINS(128);
+DECLARE_MSM_GPIO_PINS(129);
+DECLARE_MSM_GPIO_PINS(130);
+DECLARE_MSM_GPIO_PINS(131);
+DECLARE_MSM_GPIO_PINS(132);
+DECLARE_MSM_GPIO_PINS(133);
+
+static const unsigned int sdc1_clk_pins[] = { 134 };
+static const unsigned int sdc1_cmd_pins[] = { 135 };
+static const unsigned int sdc1_data_pins[] = { 136 };
+static const unsigned int sdc1_rclk_pins[] = { 137 };
+static const unsigned int sdc2_clk_pins[] = { 138 };
+static const unsigned int sdc2_cmd_pins[] = { 139 };
+static const unsigned int sdc2_data_pins[] = { 140 };
+static const unsigned int qdsd_clk_pins[] = { 141 };
+static const unsigned int qdsd_cmd_pins[] = { 142 };
+static const unsigned int qdsd_data0_pins[] = { 143 };
+static const unsigned int qdsd_data1_pins[] = { 144 };
+static const unsigned int qdsd_data2_pins[] = { 145 };
+static const unsigned int qdsd_data3_pins[] = { 146 };
+
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13,  \
+		 f14)                                                         \
+	{                                                                     \
+		.grp = PINCTRL_PINGROUP("gpio" #id, gpio##id##_pins,          \
+					ARRAY_SIZE(gpio##id##_pins)),         \
+		.funcs = (int[]){ msm_mux_gpio, msm_mux_##f1, msm_mux_##f2,   \
+				  msm_mux_##f3, msm_mux_##f4, msm_mux_##f5,   \
+				  msm_mux_##f6, msm_mux_##f7, msm_mux_##f8,   \
+				  msm_mux_##f9, msm_mux_##f10, msm_mux_##f11, \
+				  msm_mux_##f12, msm_mux_##f13,               \
+				  msm_mux_##f14 },                            \
+		.nfuncs = 15,                                                 \
+		.ctl_reg = 0x1000 * id,                                       \
+		.io_reg = 0x4 + 0x1000 * id,                                  \
+		.intr_cfg_reg = 0x8 + 0x1000 * id,                            \
+		.intr_status_reg = 0xc + 0x1000 * id,                         \
+		.mux_bit = 2,                                                 \
+		.pull_bit = 0,                                                \
+		.drv_bit = 6,                                                 \
+		.oe_bit = 9,                                                  \
+		.in_bit = 0,                                                  \
+		.out_bit = 1,                                                 \
+		.intr_enable_bit = 0,                                         \
+		.intr_status_bit = 0,                                         \
+		.intr_target_bit = 5,                                         \
+		.intr_target_kpss_val = 4,                                    \
+		.intr_raw_status_bit = 4,                                     \
+		.intr_polarity_bit = 1,                                       \
+		.intr_detection_bit = 2,                                      \
+		.intr_detection_width = 2,                                    \
+	}
+
+#define SDC_PINGROUP(pg_name, ctl, pull, drv)                        \
+	{                                                            \
+		.grp = PINCTRL_PINGROUP(#pg_name, pg_name##_pins,    \
+					ARRAY_SIZE(pg_name##_pins)), \
+		.ctl_reg = ctl,                                      \
+		.io_reg = 0,                                         \
+		.intr_cfg_reg = 0,                                   \
+		.intr_status_reg = 0,                                \
+		.intr_target_reg = 0,                                \
+		.mux_bit = -1,                                       \
+		.pull_bit = pull,                                    \
+		.drv_bit = drv,                                      \
+		.oe_bit = -1,                                        \
+		.in_bit = -1,                                        \
+		.out_bit = -1,                                       \
+		.intr_enable_bit = -1,                               \
+		.intr_status_bit = -1,                               \
+		.intr_target_bit = -1,                               \
+		.intr_target_kpss_val = -1,                          \
+		.intr_raw_status_bit = -1,                           \
+		.intr_polarity_bit = -1,                             \
+		.intr_detection_bit = -1,                            \
+		.intr_detection_width = -1,                          \
+	}
+
+enum msm8952_functions {
+	msm_mux_blsp_uart1,
+	msm_mux_blsp_spi1,
+	msm_mux_adsp_ext,
+	msm_mux_gpio,
+	msm_mux_blsp_i2c1,
+	msm_mux_prng_rosc,
+	msm_mux_qdss_cti_trig_out_b0,
+	msm_mux_qdss_cti_trig_out_a1,
+	msm_mux_blsp_spi2,
+	msm_mux_blsp_uart2,
+	msm_mux_ldo_update,
+	msm_mux_qdss_tracedata_b,
+	msm_mux_pbs1,
+	msm_mux_pwr_modem_enabled_b,
+	msm_mux_blsp_i2c3,
+	msm_mux_gcc_gp2_clk_b,
+	msm_mux_atest_combodac_to_gpio_native,
+	msm_mux_ldo_en,
+	msm_mux_blsp_i2c2,
+	msm_mux_gcc_gp1_clk_b,
+	msm_mux_atest_gpsadc_dtest0_native,
+	msm_mux_blsp_spi3,
+	msm_mux_blsp_uart3,
+	msm_mux_sec_mi2s,
+	msm_mux_pbs0,
+	msm_mux_gcc_gp3_clk_b,
+	msm_mux_blsp_spi4,
+	msm_mux_blsp_uart4,
+	msm_mux_pbs2,
+	msm_mux_pwr_nav_enabled_b,
+	msm_mux_pwr_crypto_enabled_b,
+	msm_mux_blsp_i2c4,
+	msm_mux_blsp_spi5,
+	msm_mux_blsp_uart5,
+	msm_mux_qdss_traceclk_a,
+	msm_mux_atest_bbrx1,
+	msm_mux_qdss_tracectl_b,
+	msm_mux_qdss_cti_trig_in_b0,
+	msm_mux_blsp_i2c6,
+	msm_mux_qdss_traceclk_b,
+	msm_mux_atest_wlan0,
+	msm_mux_m_voc,
+	msm_mux_qdss_cti_trig_in_a0,
+	msm_mux_atest_bbrx0,
+	msm_mux_blsp_i2c5,
+	msm_mux_qdss_tracectl_a,
+	msm_mux_atest_gpsadc_dtest1_native,
+	msm_mux_qdss_tracedata_a,
+	msm_mux_blsp_spi6,
+	msm_mux_blsp_uart6,
+	msm_mux_atest_wlan1,
+	msm_mux_mdp_vsync,
+	msm_mux_pri_mi2s_mclk_a,
+	msm_mux_sec_mi2s_mclk_a,
+	msm_mux_cam_mclk,
+	msm_mux_cci_i2c,
+	msm_mux_pwr_modem_enabled_a,
+	msm_mux_cci_timer0,
+	msm_mux_cci_timer1,
+	msm_mux_pwr_nav_enabled_a,
+	msm_mux_pwr_crypto_enabled_a,
+	msm_mux_cci_async,
+	msm_mux_sd_write,
+	msm_mux_qdss_cti_trig_out_a0,
+	msm_mux_gcc_gp1_clk_a,
+	msm_mux_gcc_gp2_clk_a,
+	msm_mux_gcc_gp3_clk_a,
+	msm_mux_blsp6_spi,
+	msm_mux_qdss_cti_trig_in_b1,
+	msm_mux_uim_batt,
+	msm_mux_qdss_cti_trig_out_b1,
+	msm_mux_uim1_data,
+	msm_mux_uim1_clk,
+	msm_mux_uim1_reset,
+	msm_mux_uim1_present,
+	msm_mux_uim2_data,
+	msm_mux_uim2_clk,
+	msm_mux_uim2_reset,
+	msm_mux_uim2_present,
+	msm_mux_uim3_data,
+	msm_mux_usb_fs,
+	msm_mux_uim3_clk,
+	msm_mux_uim3_reset,
+	msm_mux_uim3_present,
+	msm_mux_atest_char3,
+	msm_mux_dbg_out,
+	msm_mux_bimc_dte0,
+	msm_mux_sec_mi2s_mclk_b,
+	msm_mux_pri_mi2s,
+	msm_mux_atest_char1,
+	msm_mux_ebi_cdc,
+	msm_mux_atest_char0,
+	msm_mux_cdc_pdm0,
+	msm_mux_gp_mn,
+	msm_mux_gp_pdm_0a,
+	msm_mux_gp_pdm_0b,
+	msm_mux_gp_pdm_1a,
+	msm_mux_gp_pdm_1b,
+	msm_mux_gp_pdm_2a,
+	msm_mux_gp_pdm_2b,
+	msm_mux_gp0_clk,
+	msm_mux_gp1_clk,
+	msm_mux_pri_mi2s_mclk_b,
+	msm_mux_lpass_slimbus,
+	msm_mux_lpass_slimbus0,
+	msm_mux_lpass_slimbus1,
+	msm_mux_wcss_bt,
+	msm_mux_atest_char2,
+	msm_mux_ebi_ch0,
+	msm_mux_wcss_wlan2,
+	msm_mux_wcss_wlan1,
+	msm_mux_wcss_wlan0,
+	msm_mux_wcss_wlan,
+	msm_mux_wcss_fm,
+	msm_mux_ext_lpass,
+	msm_mux_gcc_tlmm,
+	msm_mux_dmic0_clk,
+	msm_mux_dmic0_data,
+	msm_mux_qdss_cti_trig_in_a1,
+	msm_mux_blsp_spi7,
+	msm_mux_blsp_uart7,
+	msm_mux_pri_mi2s_ws,
+	msm_mux_wsa_io,
+	msm_mux_blsp_i2c7,
+	msm_mux_blsp_spi8,
+	msm_mux_blsp_uart8,
+	msm_mux_blsp_i2c8,
+	msm_mux_gcc_plltest,
+	msm_mux_nav_pps_in_a,
+	msm_mux_pa_indicator,
+	msm_mux_nav_pps_in_b,
+	msm_mux_nav_pps,
+	msm_mux_modem_tsync,
+	msm_mux_nav_tsync,
+	msm_mux_ssbi_wtr1,
+	msm_mux_gsm1_tx,
+	msm_mux_gsm0_tx,
+	msm_mux_atest_char,
+	msm_mux_atest_tsens,
+	msm_mux_bimc_dte1,
+	msm_mux_NA,
+};
+
+static const char *const gpio_groups[] = {
+	"gpio0",   "gpio1",   "gpio2",	 "gpio3",   "gpio4",   "gpio5",
+	"gpio6",   "gpio7",   "gpio8",	 "gpio9",   "gpio10",  "gpio11",
+	"gpio12",  "gpio13",  "gpio14",	 "gpio15",  "gpio16",  "gpio17",
+	"gpio18",  "gpio19",  "gpio20",	 "gpio21",  "gpio22",  "gpio23",
+	"gpio24",  "gpio25",  "gpio26",	 "gpio27",  "gpio28",  "gpio29",
+	"gpio30",  "gpio31",  "gpio32",	 "gpio33",  "gpio34",  "gpio35",
+	"gpio36",  "gpio37",  "gpio38",	 "gpio39",  "gpio40",  "gpio41",
+	"gpio42",  "gpio43",  "gpio44",	 "gpio45",  "gpio46",  "gpio47",
+	"gpio48",  "gpio49",  "gpio50",	 "gpio51",  "gpio52",  "gpio53",
+	"gpio54",  "gpio55",  "gpio56",	 "gpio57",  "gpio58",  "gpio59",
+	"gpio60",  "gpio61",  "gpio62",	 "gpio63",  "gpio64",  "gpio65",
+	"gpio66",  "gpio67",  "gpio68",	 "gpio69",  "gpio70",  "gpio71",
+	"gpio72",  "gpio73",  "gpio74",	 "gpio75",  "gpio76",  "gpio77",
+	"gpio78",  "gpio79",  "gpio80",	 "gpio81",  "gpio82",  "gpio83",
+	"gpio84",  "gpio85",  "gpio86",	 "gpio87",  "gpio88",  "gpio89",
+	"gpio90",  "gpio91",  "gpio92",	 "gpio93",  "gpio94",  "gpio95",
+	"gpio96",  "gpio97",  "gpio98",	 "gpio99",  "gpio100", "gpio101",
+	"gpio102", "gpio103", "gpio104", "gpio105", "gpio106", "gpio107",
+	"gpio108", "gpio109", "gpio110", "gpio111", "gpio112", "gpio113",
+	"gpio114", "gpio115", "gpio116", "gpio117", "gpio118", "gpio119",
+	"gpio120", "gpio121", "gpio122", "gpio123", "gpio124", "gpio125",
+	"gpio126", "gpio127", "gpio128", "gpio129", "gpio130", "gpio131",
+	"gpio132", "gpio133",
+};
+static const char *const blsp_uart1_groups[] = {
+	"gpio0",
+	"gpio1",
+	"gpio2",
+	"gpio3",
+};
+static const char *const blsp_spi1_groups[] = {
+	"gpio0",
+	"gpio1",
+	"gpio2",
+	"gpio3",
+};
+static const char *const adsp_ext_groups[] = { "gpio1" };
+static const char *const qdss_cti_trig_out_b0_groups[] = { "gpio2" };
+static const char *const qdss_cti_trig_out_a1_groups[] = { "gpio3" };
+static const char *const blsp_spi2_groups[] = {
+	"gpio4",
+	"gpio5",
+	"gpio6",
+	"gpio7",
+};
+static const char *const blsp_uart2_groups[] = {
+	"gpio4",
+	"gpio5",
+	"gpio6",
+	"gpio7",
+};
+static const char *const ldo_update_groups[] = { "gpio4" };
+static const char *const qdss_tracedata_b_groups[] = {
+	"gpio8",  "gpio9",  "gpio12", "gpio13", "gpio23", "gpio42",
+	"gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio66",
+	"gpio86", "gpio87", "gpio88", "gpio92",
+};
+static const char *const blsp_i2c1_groups[] = {
+	"gpio2",
+	"gpio3",
+};
+static const char *const prng_rosc_groups[] = {
+	"gpio2",
+};
+static const char *const pbs1_groups[] = { "gpio9" };
+static const char *const pwr_modem_enabled_b_groups[] = { "gpio9" };
+static const char *const blsp_i2c3_groups[] = {
+	"gpio10",
+	"gpio11",
+};
+static const char *const gcc_gp2_clk_b_groups[] = { "gpio10" };
+static const char *const atest_combodac_to_gpio_native_groups[] = {
+	"gpio4",  "gpio12", "gpio13", "gpio20", "gpio21", "gpio28",  "gpio29",
+	"gpio30", "gpio39", "gpio40", "gpio41", "gpio42", "gpio43",  "gpio44",
+	"gpio45", "gpio46", "gpio47", "gpio48", "gpio67", "gpio115",
+};
+static const char *const ldo_en_groups[] = { "gpio5" };
+static const char *const blsp_i2c2_groups[] = {
+	"gpio6",
+	"gpio7",
+};
+static const char *const gcc_gp1_clk_b_groups[] = { "gpio6" };
+static const char *const atest_gpsadc_dtest0_native_groups[] = { "gpio7" };
+static const char *const gp_pdm_2a_groups[] = { "gpio7" };
+static const char *const blsp_spi3_groups[] = {
+	"gpio8",
+	"gpio9",
+	"gpio10",
+	"gpio11",
+};
+static const char *const blsp_uart3_groups[] = {
+	"gpio8",
+	"gpio9",
+	"gpio10",
+	"gpio11",
+};
+static const char *const sec_mi2s_groups[] = {
+	"gpio8",
+	"gpio9",
+	"gpio12",
+	"gpio13",
+};
+static const char *const pbs0_groups[] = { "gpio8" };
+static const char *const gcc_gp3_clk_b_groups[] = { "gpio11" };
+static const char *const blsp_spi4_groups[] = {
+	"gpio12",
+	"gpio13",
+	"gpio14",
+	"gpio15",
+};
+static const char *const blsp_uart4_groups[] = {
+	"gpio12",
+	"gpio13",
+	"gpio14",
+	"gpio15",
+};
+static const char *const pbs2_groups[] = { "gpio12" };
+static const char *const pwr_nav_enabled_b_groups[] = { "gpio12" };
+static const char *const pwr_crypto_enabled_b_groups[] = { "gpio13" };
+static const char *const blsp_i2c4_groups[] = {
+	"gpio14",
+	"gpio15",
+};
+static const char *const blsp_spi5_groups[] = {
+	"gpio16",
+	"gpio17",
+	"gpio18",
+	"gpio19",
+};
+static const char *const blsp_uart5_groups[] = {
+	"gpio16",
+	"gpio17",
+	"gpio18",
+	"gpio19",
+};
+static const char *const qdss_traceclk_a_groups[] = { "gpio16" };
+static const char *const atest_bbrx1_groups[] = { "gpio16" };
+static const char *const qdss_tracectl_b_groups[] = { "gpio20" };
+static const char *const gp_pdm_1b_groups[] = { "gpio20" };
+static const char *const qdss_cti_trig_in_b0_groups[] = { "gpio21" };
+static const char *const blsp_i2c6_groups[] = {
+	"gpio22",
+	"gpio23",
+};
+static const char *const qdss_traceclk_b_groups[] = { "gpio22" };
+static const char *const atest_wlan0_groups[] = { "gpio22" };
+static const char *const m_voc_groups[] = {
+	"gpio17",
+	"gpio21",
+};
+static const char *const qdss_cti_trig_in_a0_groups[] = { "gpio17" };
+static const char *const atest_bbrx0_groups[] = { "gpio17" };
+static const char *const blsp_i2c5_groups[] = {
+	"gpio18",
+	"gpio19",
+};
+static const char *const qdss_tracectl_a_groups[] = { "gpio18" };
+static const char *const atest_gpsadc_dtest1_native_groups[] = { "gpio18" };
+static const char *const qdss_tracedata_a_groups[] = {
+	"gpio19", "gpio26", "gpio27", "gpio28", "gpio29", "gpio30",
+	"gpio31", "gpio32", "gpio33", "gpio34", "gpio35", "gpio36",
+	"gpio37", "gpio38", "gpio39", "gpio40",
+};
+static const char *const blsp_spi6_groups[] = {
+	"gpio20",
+	"gpio21",
+	"gpio22",
+	"gpio23",
+};
+static const char *const blsp_uart6_groups[] = {
+	"gpio20",
+	"gpio21",
+	"gpio22",
+	"gpio23",
+};
+static const char *const atest_wlan1_groups[] = { "gpio23" };
+static const char *const mdp_vsync_groups[] = {
+	"gpio24",
+	"gpio25",
+};
+static const char *const pri_mi2s_mclk_a_groups[] = { "gpio25" };
+static const char *const sec_mi2s_mclk_a_groups[] = { "gpio25" };
+static const char *const cam_mclk_groups[] = {
+	"gpio26",
+	"gpio27",
+	"gpio28",
+};
+static const char *const cci_i2c_groups[] = {
+	"gpio29",
+	"gpio30",
+	"gpio31",
+	"gpio32",
+};
+static const char *const pwr_modem_enabled_a_groups[] = { "gpio29" };
+static const char *const cci_timer0_groups[] = { "gpio33" };
+static const char *const cci_timer1_groups[] = { "gpio34" };
+static const char *const pwr_nav_enabled_a_groups[] = { "gpio35" };
+static const char *const pwr_crypto_enabled_a_groups[] = { "gpio36" };
+static const char *const gp_pdm_2b_groups[] = { "gpio38" };
+static const char *const cci_async_groups[] = { "gpio39" };
+static const char *const gp_mn_groups[] = { "gpio39" };
+static const char *const sd_write_groups[] = { "gpio41" };
+static const char *const qdss_cti_trig_out_a0_groups[] = { "gpio41" };
+static const char *const gcc_gp1_clk_a_groups[] = { "gpio42" };
+static const char *const gcc_gp2_clk_a_groups[] = { "gpio43" };
+static const char *const gcc_gp3_clk_a_groups[] = { "gpio44" };
+static const char *const gp_pdm_1a_groups[] = { "gpio45" };
+static const char *const gp0_clk_groups[] = { "gpio46" };
+static const char *const gp1_clk_groups[] = { "gpio47" };
+static const char *const blsp6_spi_groups[] = {
+	"gpio47",
+	"gpio48",
+};
+static const char *const gp_pdm_0b_groups[] = { "gpio48" };
+static const char *const qdss_cti_trig_in_b1_groups[] = { "gpio48" };
+static const char *const uim_batt_groups[] = { "gpio49" };
+static const char *const qdss_cti_trig_out_b1_groups[] = { "gpio50" };
+static const char *const uim1_data_groups[] = { "gpio51" };
+static const char *const uim1_clk_groups[] = { "gpio52" };
+static const char *const uim1_reset_groups[] = { "gpio53" };
+static const char *const uim1_present_groups[] = { "gpio54" };
+static const char *const uim2_data_groups[] = { "gpio55" };
+static const char *const uim2_clk_groups[] = { "gpio56" };
+static const char *const uim2_reset_groups[] = { "gpio57" };
+static const char *const uim2_present_groups[] = { "gpio58" };
+static const char *const uim3_data_groups[] = { "gpio59" };
+static const char *const usb_fs_groups[] = {
+	"gpio59",
+	"gpio60",
+};
+static const char *const uim3_clk_groups[] = { "gpio60" };
+static const char *const uim3_reset_groups[] = { "gpio61" };
+static const char *const uim3_present_groups[] = { "gpio62" };
+static const char *const atest_char3_groups[] = { "gpio63" };
+static const char *const dbg_out_groups[] = { "gpio63" };
+static const char *const bimc_dte0_groups[] = {
+	"gpio63",
+	"gpio65",
+};
+static const char *const sec_mi2s_mclk_b_groups[] = { "gpio66" };
+static const char *const gp_pdm_0a_groups[] = { "gpio66" };
+static const char *const pri_mi2s_groups[] = {
+	"gpio66",
+	"gpio88",
+	"gpio91",
+	"gpio93",
+};
+static const char *const atest_char1_groups[] = { "gpio67" };
+static const char *const ebi_cdc_groups[] = {
+	"gpio67", "gpio69", "gpio118", "gpio119", "gpio120", "gpio123",
+};
+static const char *const atest_char0_groups[] = { "gpio68" };
+static const char *const cdc_pdm0_groups[] = {
+	"gpio69", "gpio70", "gpio71", "gpio72", "gpio73", "gpio74",
+};
+static const char *const pri_mi2s_mclk_b_groups[] = { "gpio69" };
+static const char *const lpass_slimbus_groups[] = { "gpio70" };
+static const char *const lpass_slimbus0_groups[] = { "gpio71" };
+static const char *const lpass_slimbus1_groups[] = { "gpio72" };
+static const char *const wcss_bt_groups[] = {
+	"gpio75",
+	"gpio83",
+	"gpio84",
+};
+static const char *const atest_char2_groups[] = { "gpio75" };
+static const char *const ebi_ch0_groups[] = { "gpio75" };
+static const char *const wcss_wlan2_groups[] = { "gpio76" };
+static const char *const wcss_wlan1_groups[] = { "gpio77" };
+static const char *const wcss_wlan0_groups[] = { "gpio78" };
+static const char *const wcss_wlan_groups[] = {
+	"gpio79",
+	"gpio80",
+};
+static const char *const wcss_fm_groups[] = {
+	"gpio81",
+	"gpio82",
+};
+static const char *const ext_lpass_groups[] = { "gpio81" };
+static const char *const gcc_tlmm_groups[] = { "gpio87" };
+static const char *const dmic0_clk_groups[] = { "gpio89" };
+static const char *const dmic0_data_groups[] = { "gpio90" };
+static const char *const qdss_cti_trig_in_a1_groups[] = { "gpio91" };
+static const char *const blsp_spi7_groups[] = {
+	"gpio92",
+	"gpio93",
+	"gpio94",
+	"gpio95",
+};
+static const char *const blsp_uart7_groups[] = {
+	"gpio92",
+	"gpio93",
+	"gpio94",
+	"gpio95",
+};
+static const char *const pri_mi2s_ws_groups[] = { "gpio92" };
+static const char *const wsa_io_groups[] = {
+	"gpio94",
+	"gpio95",
+};
+static const char *const blsp_i2c7_groups[] = {
+	"gpio94",
+	"gpio95",
+};
+static const char *const blsp_spi8_groups[] = {
+	"gpio96",
+	"gpio97",
+	"gpio98",
+	"gpio99",
+};
+static const char *const blsp_uart8_groups[] = {
+	"gpio96",
+	"gpio97",
+	"gpio98",
+	"gpio99",
+};
+static const char *const blsp_i2c8_groups[] = {
+	"gpio98",
+	"gpio99",
+};
+static const char *const gcc_plltest_groups[] = {
+	"gpio98",
+	"gpio99",
+};
+static const char *const nav_pps_in_a_groups[] = { "gpio111" };
+static const char *const pa_indicator_groups[] = { "gpio112" };
+static const char *const nav_pps_in_b_groups[] = { "gpio113" };
+static const char *const nav_pps_groups[] = { "gpio113" };
+static const char *const modem_tsync_groups[] = { "gpio113" };
+static const char *const nav_tsync_groups[] = { "gpio113" };
+static const char *const ssbi_wtr1_groups[] = {
+	"gpio114",
+	"gpio123",
+};
+static const char *const gsm1_tx_groups[] = { "gpio115" };
+static const char *const gsm0_tx_groups[] = { "gpio117" };
+static const char *const atest_char_groups[] = { "gpio120" };
+static const char *const atest_tsens_groups[] = { "gpio120" };
+static const char *const bimc_dte1_groups[] = {
+	"gpio121",
+	"gpio122",
+};
+
+static const struct pinfunction msm8952_functions[] = {
+	MSM_PIN_FUNCTION(blsp_uart1),
+	MSM_PIN_FUNCTION(blsp_spi1),
+	MSM_PIN_FUNCTION(adsp_ext),
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(blsp_i2c1),
+	MSM_PIN_FUNCTION(prng_rosc),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_b0),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_a1),
+	MSM_PIN_FUNCTION(blsp_spi2),
+	MSM_PIN_FUNCTION(blsp_uart2),
+	MSM_PIN_FUNCTION(ldo_update),
+	MSM_PIN_FUNCTION(qdss_tracedata_b),
+	MSM_PIN_FUNCTION(pbs1),
+	MSM_PIN_FUNCTION(pwr_modem_enabled_b),
+	MSM_PIN_FUNCTION(blsp_i2c3),
+	MSM_PIN_FUNCTION(gcc_gp2_clk_b),
+	MSM_PIN_FUNCTION(atest_combodac_to_gpio_native),
+	MSM_PIN_FUNCTION(ldo_en),
+	MSM_PIN_FUNCTION(blsp_i2c2),
+	MSM_PIN_FUNCTION(gcc_gp1_clk_b),
+	MSM_PIN_FUNCTION(atest_gpsadc_dtest0_native),
+	MSM_PIN_FUNCTION(blsp_spi3),
+	MSM_PIN_FUNCTION(blsp_uart3),
+	MSM_PIN_FUNCTION(sec_mi2s),
+	MSM_PIN_FUNCTION(pbs0),
+	MSM_PIN_FUNCTION(gcc_gp3_clk_b),
+	MSM_PIN_FUNCTION(blsp_spi4),
+	MSM_PIN_FUNCTION(blsp_uart4),
+	MSM_PIN_FUNCTION(pbs2),
+	MSM_PIN_FUNCTION(pwr_nav_enabled_b),
+	MSM_PIN_FUNCTION(pwr_crypto_enabled_b),
+	MSM_PIN_FUNCTION(blsp_i2c4),
+	MSM_PIN_FUNCTION(blsp_spi5),
+	MSM_PIN_FUNCTION(blsp_uart5),
+	MSM_PIN_FUNCTION(qdss_traceclk_a),
+	MSM_PIN_FUNCTION(atest_bbrx1),
+	MSM_PIN_FUNCTION(qdss_tracectl_b),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_b0),
+	MSM_PIN_FUNCTION(blsp_i2c6),
+	MSM_PIN_FUNCTION(qdss_traceclk_b),
+	MSM_PIN_FUNCTION(atest_wlan0),
+	MSM_PIN_FUNCTION(m_voc),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_a0),
+	MSM_PIN_FUNCTION(atest_bbrx0),
+	MSM_PIN_FUNCTION(blsp_i2c5),
+	MSM_PIN_FUNCTION(qdss_tracectl_a),
+	MSM_PIN_FUNCTION(atest_gpsadc_dtest1_native),
+	MSM_PIN_FUNCTION(qdss_tracedata_a),
+	MSM_PIN_FUNCTION(blsp_spi6),
+	MSM_PIN_FUNCTION(blsp_uart6),
+	MSM_PIN_FUNCTION(atest_wlan1),
+	MSM_PIN_FUNCTION(mdp_vsync),
+	MSM_PIN_FUNCTION(pri_mi2s_mclk_a),
+	MSM_PIN_FUNCTION(sec_mi2s_mclk_a),
+	MSM_PIN_FUNCTION(cam_mclk),
+	MSM_PIN_FUNCTION(cci_i2c),
+	MSM_PIN_FUNCTION(pwr_modem_enabled_a),
+	MSM_PIN_FUNCTION(cci_timer0),
+	MSM_PIN_FUNCTION(cci_timer1),
+	MSM_PIN_FUNCTION(pwr_nav_enabled_a),
+	MSM_PIN_FUNCTION(pwr_crypto_enabled_a),
+	MSM_PIN_FUNCTION(cci_async),
+	MSM_PIN_FUNCTION(sd_write),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_a0),
+	MSM_PIN_FUNCTION(gcc_gp1_clk_a),
+	MSM_PIN_FUNCTION(gcc_gp2_clk_a),
+	MSM_PIN_FUNCTION(gcc_gp3_clk_a),
+	MSM_PIN_FUNCTION(blsp6_spi),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_b1),
+	MSM_PIN_FUNCTION(uim_batt),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_b1),
+	MSM_PIN_FUNCTION(uim1_data),
+	MSM_PIN_FUNCTION(uim1_clk),
+	MSM_PIN_FUNCTION(uim1_reset),
+	MSM_PIN_FUNCTION(uim1_present),
+	MSM_PIN_FUNCTION(uim2_data),
+	MSM_PIN_FUNCTION(uim2_clk),
+	MSM_PIN_FUNCTION(uim2_reset),
+	MSM_PIN_FUNCTION(uim2_present),
+	MSM_PIN_FUNCTION(uim3_data),
+	MSM_PIN_FUNCTION(usb_fs),
+	MSM_PIN_FUNCTION(uim3_clk),
+	MSM_PIN_FUNCTION(uim3_reset),
+	MSM_PIN_FUNCTION(uim3_present),
+	MSM_PIN_FUNCTION(atest_char3),
+	MSM_PIN_FUNCTION(dbg_out),
+	MSM_PIN_FUNCTION(bimc_dte0),
+	MSM_PIN_FUNCTION(sec_mi2s_mclk_b),
+	MSM_PIN_FUNCTION(pri_mi2s),
+	MSM_PIN_FUNCTION(atest_char1),
+	MSM_PIN_FUNCTION(ebi_cdc),
+	MSM_PIN_FUNCTION(atest_char0),
+	MSM_PIN_FUNCTION(cdc_pdm0),
+	MSM_PIN_FUNCTION(gp_mn),
+	MSM_PIN_FUNCTION(gp_pdm_0a),
+	MSM_PIN_FUNCTION(gp_pdm_0b),
+	MSM_PIN_FUNCTION(gp_pdm_1a),
+	MSM_PIN_FUNCTION(gp_pdm_1b),
+	MSM_PIN_FUNCTION(gp_pdm_2a),
+	MSM_PIN_FUNCTION(gp_pdm_2b),
+	MSM_PIN_FUNCTION(gp0_clk),
+	MSM_PIN_FUNCTION(gp1_clk),
+	MSM_PIN_FUNCTION(pri_mi2s_mclk_b),
+	MSM_PIN_FUNCTION(lpass_slimbus),
+	MSM_PIN_FUNCTION(lpass_slimbus0),
+	MSM_PIN_FUNCTION(lpass_slimbus1),
+	MSM_PIN_FUNCTION(wcss_bt),
+	MSM_PIN_FUNCTION(atest_char2),
+	MSM_PIN_FUNCTION(ebi_ch0),
+	MSM_PIN_FUNCTION(wcss_wlan2),
+	MSM_PIN_FUNCTION(wcss_wlan1),
+	MSM_PIN_FUNCTION(wcss_wlan0),
+	MSM_PIN_FUNCTION(wcss_wlan),
+	MSM_PIN_FUNCTION(wcss_fm),
+	MSM_PIN_FUNCTION(ext_lpass),
+	MSM_PIN_FUNCTION(gcc_tlmm),
+	MSM_PIN_FUNCTION(dmic0_clk),
+	MSM_PIN_FUNCTION(dmic0_data),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_a1),
+	MSM_PIN_FUNCTION(blsp_spi7),
+	MSM_PIN_FUNCTION(blsp_uart7),
+	MSM_PIN_FUNCTION(pri_mi2s_ws),
+	MSM_PIN_FUNCTION(wsa_io),
+	MSM_PIN_FUNCTION(blsp_i2c7),
+	MSM_PIN_FUNCTION(blsp_spi8),
+	MSM_PIN_FUNCTION(blsp_uart8),
+	MSM_PIN_FUNCTION(blsp_i2c8),
+	MSM_PIN_FUNCTION(gcc_plltest),
+	MSM_PIN_FUNCTION(nav_pps_in_a),
+	MSM_PIN_FUNCTION(pa_indicator),
+	MSM_PIN_FUNCTION(nav_pps_in_b),
+	MSM_PIN_FUNCTION(nav_pps),
+	MSM_PIN_FUNCTION(modem_tsync),
+	MSM_PIN_FUNCTION(nav_tsync),
+	MSM_PIN_FUNCTION(ssbi_wtr1),
+	MSM_PIN_FUNCTION(gsm1_tx),
+	MSM_PIN_FUNCTION(gsm0_tx),
+	MSM_PIN_FUNCTION(atest_char),
+	MSM_PIN_FUNCTION(atest_tsens),
+	MSM_PIN_FUNCTION(bimc_dte1),
+};
+
+static const struct msm_pingroup msm8952_groups[] = {
+	PINGROUP(0, blsp_spi1, blsp_uart1, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(1, blsp_spi1, blsp_uart1, adsp_ext, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(2, blsp_spi1, blsp_uart1, blsp_i2c1, prng_rosc, NA, NA, NA, NA,
+		 NA, NA, NA, qdss_cti_trig_out_b0, NA, NA),
+	PINGROUP(3, blsp_spi1, blsp_uart1, blsp_i2c1, NA, NA, NA, NA, NA, NA,
+		 NA, qdss_cti_trig_out_a1, NA, NA, NA),
+	PINGROUP(4, blsp_spi2, blsp_uart2, ldo_update, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(5, blsp_spi2, blsp_uart2, ldo_en, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(6, blsp_spi2, blsp_uart2, blsp_i2c2, gcc_gp1_clk_b, NA, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(7, blsp_spi2, blsp_uart2, blsp_i2c2, gp_pdm_2a, NA,
+		 atest_gpsadc_dtest0_native, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(8, blsp_spi3, blsp_uart3, sec_mi2s, pbs0, NA, NA, NA, NA, NA,
+		 NA, qdss_tracedata_b, NA, NA, NA),
+	PINGROUP(9, blsp_spi3, blsp_uart3, sec_mi2s, pbs1, pwr_modem_enabled_b,
+		 NA, NA, NA, NA, NA, NA, qdss_tracedata_b, NA, NA),
+	PINGROUP(10, blsp_spi3, blsp_uart3, blsp_i2c3, gcc_gp2_clk_b, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(11, blsp_spi3, blsp_uart3, blsp_i2c3, gcc_gp3_clk_b, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(12, blsp_spi4, blsp_uart4, sec_mi2s, pbs2, pwr_nav_enabled_b,
+		 NA, NA, NA, NA, NA, NA, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native),
+	PINGROUP(13, blsp_spi4, blsp_uart4, sec_mi2s, pwr_crypto_enabled_b, NA,
+		 NA, NA, NA, NA, NA, NA, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native),
+	PINGROUP(14, blsp_spi4, blsp_uart4, blsp_i2c4, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(15, blsp_spi4, blsp_uart4, blsp_i2c4, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(16, blsp_spi5, blsp_uart5, NA, NA, NA, NA, qdss_traceclk_a, NA,
+		 atest_bbrx1, NA, NA, NA, NA, NA),
+	PINGROUP(17, blsp_spi5, blsp_uart5, m_voc, qdss_cti_trig_in_a0, NA,
+		 atest_bbrx0, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(18, blsp_spi5, blsp_uart5, blsp_i2c5, qdss_tracectl_a, NA,
+		 atest_gpsadc_dtest1_native, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(19, blsp_spi5, blsp_uart5, blsp_i2c5, qdss_tracedata_a, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(20, blsp_spi6, blsp_uart6, gp_pdm_1b, NA, NA, NA, NA, NA,
+		 qdss_tracectl_b, NA, atest_combodac_to_gpio_native, NA, NA,
+		 NA),
+	PINGROUP(21, blsp_spi6, blsp_uart6, m_voc, NA, NA, NA, NA, NA,
+		 qdss_cti_trig_in_b0, NA, atest_combodac_to_gpio_native, NA, NA,
+		 NA),
+	PINGROUP(22, blsp_spi6, blsp_uart6, blsp_i2c6, qdss_traceclk_b, NA,
+		 atest_wlan0, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(23, blsp_spi6, blsp_uart6, blsp_i2c6, qdss_tracedata_b, NA,
+		 atest_wlan1, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(24, mdp_vsync, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(25, mdp_vsync, pri_mi2s_mclk_a, sec_mi2s_mclk_a, NA, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(26, cam_mclk, NA, NA, NA, NA, NA, qdss_tracedata_a, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(27, cam_mclk, NA, NA, NA, NA, NA, NA, NA, qdss_tracedata_a, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(28, cam_mclk, NA, NA, NA, NA, NA, qdss_tracedata_a, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA),
+	PINGROUP(29, cci_i2c, pwr_modem_enabled_a, NA, NA, NA, NA, NA,
+		 qdss_tracedata_a, NA, atest_combodac_to_gpio_native, NA, NA,
+		 NA, NA),
+	PINGROUP(30, cci_i2c, NA, NA, NA, NA, NA, NA, NA, qdss_tracedata_a, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA),
+	PINGROUP(31, cci_i2c, NA, NA, NA, NA, NA, NA, NA, qdss_tracedata_a, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(32, cci_i2c, NA, NA, NA, NA, NA, NA, NA, qdss_tracedata_a, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(33, cci_timer0, NA, NA, NA, NA, NA, NA, NA, qdss_tracedata_a,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(34, cci_timer1, NA, NA, NA, NA, NA, NA, NA, qdss_tracedata_a,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(35, pwr_nav_enabled_a, NA, NA, NA, NA, NA, NA, NA,
+		 qdss_tracedata_a, NA, NA, NA, NA, NA),
+	PINGROUP(36, pwr_crypto_enabled_a, NA, NA, NA, NA, NA, NA, NA,
+		 qdss_tracedata_a, NA, NA, NA, NA, NA),
+	PINGROUP(37, NA, NA, NA, NA, NA, qdss_tracedata_a, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(38, gp_pdm_2b, qdss_tracedata_a, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(39, cci_async, gp_mn, NA, NA, NA, NA, qdss_tracedata_a, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA),
+	PINGROUP(40, NA, NA, NA, NA, qdss_tracedata_a, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(41, sd_write, NA, NA, NA, NA, NA, NA, qdss_cti_trig_out_a0, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA),
+	PINGROUP(42, gcc_gp1_clk_a, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(43, gcc_gp2_clk_a, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(44, gcc_gp3_clk_a, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(45, gp_pdm_1a, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(46, gp0_clk, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(47, blsp6_spi, gp1_clk, qdss_tracedata_b, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(48, blsp6_spi, gp_pdm_0b, qdss_cti_trig_in_b1, NA,
+		 atest_combodac_to_gpio_native, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(49, uim_batt, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(50, qdss_cti_trig_out_b1, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(51, uim1_data, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(52, uim1_clk, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(53, uim1_reset, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(54, uim1_present, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(55, uim2_data, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(56, uim2_clk, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(57, uim2_reset, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(58, uim2_present, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(59, uim3_data, usb_fs, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(60, uim3_clk, usb_fs, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(61, uim3_reset, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(62, uim3_present, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(63, atest_char3, dbg_out, bimc_dte0, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(64, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(65, bimc_dte0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(66, sec_mi2s_mclk_b, pri_mi2s, gp_pdm_0a, qdss_tracedata_b, NA,
+		 NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(67, atest_char1, ebi_cdc, NA, atest_combodac_to_gpio_native,
+		 NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(68, atest_char0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(69, cdc_pdm0, pri_mi2s_mclk_b, ebi_cdc, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(70, lpass_slimbus, cdc_pdm0, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(71, lpass_slimbus0, cdc_pdm0, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(72, lpass_slimbus1, cdc_pdm0, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(73, cdc_pdm0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(74, cdc_pdm0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(75, wcss_bt, atest_char2, NA, ebi_ch0, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(76, wcss_wlan2, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(77, wcss_wlan1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(78, wcss_wlan0, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(79, wcss_wlan, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(80, wcss_wlan, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(81, wcss_fm, ext_lpass, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(82, wcss_fm, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(83, wcss_bt, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(84, wcss_bt, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(85, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(86, qdss_tracedata_b, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(87, qdss_tracedata_b, gcc_tlmm, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(88, pri_mi2s, NA, NA, NA, NA, NA, NA, NA, qdss_tracedata_b, NA,
+		 NA, NA, NA, NA),
+	PINGROUP(89, dmic0_clk, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(90, dmic0_data, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(91, pri_mi2s, NA, NA, NA, NA, NA, qdss_cti_trig_in_a1, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(92, blsp_spi7, blsp_uart7, pri_mi2s_ws, NA, NA, NA, NA, NA,
+		 qdss_tracedata_b, NA, NA, NA, NA, NA),
+	PINGROUP(93, blsp_spi7, blsp_uart7, pri_mi2s, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(94, wsa_io, blsp_spi7, blsp_uart7, blsp_i2c7, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA, NA),
+	PINGROUP(95, wsa_io, blsp_spi7, blsp_uart7, blsp_i2c7, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA, NA),
+	PINGROUP(96, blsp_spi8, blsp_uart8, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(97, blsp_spi8, blsp_uart8, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(98, blsp_i2c8, blsp_spi8, blsp_uart8, gcc_plltest, NA, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(99, blsp_i2c8, blsp_spi8, blsp_uart8, gcc_plltest, NA, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(100, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(101, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(102, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(103, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(104, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(105, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(106, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(107, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(108, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(109, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(110, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(111, NA, NA, nav_pps_in_a, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(112, NA, pa_indicator, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA),
+	PINGROUP(113, NA, nav_pps_in_b, nav_pps, modem_tsync, nav_tsync, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(114, NA, ssbi_wtr1, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(115, NA, gsm1_tx, NA, atest_combodac_to_gpio_native, NA, NA,
+		 NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(116, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(117, gsm0_tx, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(118, NA, ebi_cdc, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(119, NA, ebi_cdc, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(120, NA, atest_char, ebi_cdc, atest_tsens, NA, NA, NA, NA, NA,
+		 NA, NA, NA, NA, NA),
+	PINGROUP(121, NA, NA, NA, bimc_dte1, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(122, NA, NA, NA, bimc_dte1, NA, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA),
+	PINGROUP(123, NA, ssbi_wtr1, ebi_cdc, NA, NA, NA, NA, NA, NA, NA, NA,
+		 NA, NA, NA),
+	PINGROUP(124, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(125, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(126, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(127, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(128, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(129, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(130, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(131, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(132, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	PINGROUP(133, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA),
+	SDC_PINGROUP(sdc1_clk, 0x10a000, 13, 6),
+	SDC_PINGROUP(sdc1_cmd, 0x10a000, 11, 3),
+	SDC_PINGROUP(sdc1_data, 0x10a000, 9, 0),
+	SDC_PINGROUP(sdc1_rclk, 0x10a000, 15, 0),
+	SDC_PINGROUP(sdc2_clk, 0x109000, 14, 6),
+	SDC_PINGROUP(sdc2_cmd, 0x109000, 11, 3),
+	SDC_PINGROUP(sdc2_data, 0x109000, 9, 0),
+	SDC_PINGROUP(qdsd_clk, 0x19c000, 3, 0),
+	SDC_PINGROUP(qdsd_cmd, 0x19c000, 8, 5),
+	SDC_PINGROUP(qdsd_data0, 0x19c000, 13, 10),
+	SDC_PINGROUP(qdsd_data1, 0x19c000, 18, 15),
+	SDC_PINGROUP(qdsd_data2, 0x19c000, 23, 20),
+	SDC_PINGROUP(qdsd_data3, 0x19c000, 28, 25),
+};
+
+static const struct msm_gpio_wakeirq_map msm8952_mpm_map[] = {
+	{ 1, 4 },   { 5, 5 },	{ 9, 6 },   { 12, 21 }, { 13, 10 }, { 17, 12 },
+	{ 21, 13 }, { 25, 20 }, { 28, 18 }, { 31, 16 }, { 34, 15 }, { 35, 11 },
+	{ 36, 9 },  { 37, 8 },	{ 38, 3 },  { 42, 19 }, { 43, 22 }, { 44, 23 },
+	{ 45, 24 }, { 46, 25 }, { 48, 26 }, { 54, 14 }, { 58, 17 }, { 62, 53 },
+	{ 63, 30 }, { 65, 27 }, { 67, 50 }, { 70, 31 }, { 71, 32 }, { 72, 33 },
+	{ 73, 51 }, { 74, 52 }, { 81, 34 }, { 85, 35 }, { 90, 36 }, { 93, 28 },
+	{ 97, 29 },
+};
+
+static const struct msm_pinctrl_soc_data msm8952_pinctrl = {
+	.pins = msm8952_pins,
+	.npins = ARRAY_SIZE(msm8952_pins),
+	.functions = msm8952_functions,
+	.nfunctions = ARRAY_SIZE(msm8952_functions),
+	.groups = msm8952_groups,
+	.ngroups = ARRAY_SIZE(msm8952_groups),
+	.ngpios = 134,
+	.wakeirq_map = msm8952_mpm_map,
+	.nwakeirq_map = ARRAY_SIZE(msm8952_mpm_map),
+};
+
+static int msm8952_pinctrl_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &msm8952_pinctrl);
+}
+
+static const struct of_device_id msm8952_pinctrl_of_match[] = {
+	{ .compatible = "qcom,msm8952-pinctrl", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, msm8952_pinctrl_of_match);
+
+static struct platform_driver msm8952_pinctrl_driver = {
+	.driver = {
+		.name = "msm8952-pinctrl",
+		.of_match_table = msm8952_pinctrl_of_match,
+	},
+	.probe = msm8952_pinctrl_probe,
+};
+
+static int __init msm8952_pinctrl_init(void)
+{
+	return platform_driver_register(&msm8952_pinctrl_driver);
+}
+arch_initcall(msm8952_pinctrl_init);
+
+static void __exit msm8952_pinctrl_exit(void)
+{
+	platform_driver_unregister(&msm8952_pinctrl_driver);
+}
+module_exit(msm8952_pinctrl_exit);
+
+MODULE_DESCRIPTION("Qualcomm msm8952 pinctrl driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-nord.c b/drivers/pinctrl/qcom/pinctrl-nord.c
index 7c21306e77ff72..010760cf0bf5cb 100644
--- a/drivers/pinctrl/qcom/pinctrl-nord.c
+++ b/drivers/pinctrl/qcom/pinctrl-nord.c
@@ -32,6 +32,7 @@
 		.intr_wakeup_present_bit = 6,                         \
 		.intr_wakeup_enable_bit = 7,                          \
 		.intr_target_bit = 8,                                 \
+		.intr_target_width = 4,                               \
 		.intr_target_kpss_val = 3,                            \
 		.intr_raw_status_bit = 4,                             \
 		.intr_polarity_bit = 1,                               \
@@ -570,8 +571,10 @@ enum nord_functions {
 	msm_mux_qup0_se5,
 	msm_mux_qup1_se0,
 	msm_mux_qup1_se1,
-	msm_mux_qup1_se2,
-	msm_mux_qup1_se3,
+	msm_mux_qup1_se2_01,
+	msm_mux_qup1_se2_23,
+	msm_mux_qup1_se3_01,
+	msm_mux_qup1_se3_23,
 	msm_mux_qup1_se4,
 	msm_mux_qup1_se5,
 	msm_mux_qup1_se6,
@@ -1152,11 +1155,19 @@ static const char *const qup1_se1_groups[] = {
 	"gpio123", "gpio124", "gpio125", "gpio126",
 };
 
-static const char *const qup1_se2_groups[] = {
-	"gpio127", "gpio128", "gpio129", "gpio130",
+static const char *const qup1_se2_01_groups[] = {
+	"gpio127", "gpio128",
 };
 
-static const char *const qup1_se3_groups[] = {
+static const char *const qup1_se2_23_groups[] = {
+	"gpio127", "gpio128",
+};
+
+static const char *const qup1_se3_01_groups[] = {
+	"gpio129", "gpio130",
+};
+
+static const char *const qup1_se3_23_groups[] = {
 	"gpio129", "gpio130",
 };
 
@@ -1428,8 +1439,10 @@ static const struct pinfunction nord_functions[] = {
 	MSM_PIN_FUNCTION(qup0_se5),
 	MSM_PIN_FUNCTION(qup1_se0),
 	MSM_PIN_FUNCTION(qup1_se1),
-	MSM_PIN_FUNCTION(qup1_se2),
-	MSM_PIN_FUNCTION(qup1_se3),
+	MSM_PIN_FUNCTION(qup1_se2_01),
+	MSM_PIN_FUNCTION(qup1_se2_23),
+	MSM_PIN_FUNCTION(qup1_se3_01),
+	MSM_PIN_FUNCTION(qup1_se3_23),
 	MSM_PIN_FUNCTION(qup1_se4),
 	MSM_PIN_FUNCTION(qup1_se5),
 	MSM_PIN_FUNCTION(qup1_se6),
@@ -1633,13 +1646,13 @@ static const struct msm_pingroup nord_groups[] = {
 			 _, _, _, _, _, _, _),
 	[126] = PINGROUP(126, qup1_se1, qup1_se0, ccu_i2c_scl, mdp1_vsync_out,
 			 _, atest_usb20, ddr_pxi, _, _, _, _),
-	[127] = PINGROUP(127, qup1_se2, qup1_se2, _, atest_usb21, ddr_pxi,
+	[127] = PINGROUP(127, qup1_se2_23, qup1_se2_01, _, atest_usb21, ddr_pxi,
 			 _, _, _, _, _, _),
-	[128] = PINGROUP(128, qup1_se2, qup1_se2, _, atest_usb20, ddr_pxi,
+	[128] = PINGROUP(128, qup1_se2_23, qup1_se2_01, _, atest_usb20, ddr_pxi,
 			 _, _, _, _, _, _),
-	[129] = PINGROUP(129, qup1_se3, qup1_se3, ccu_i2c_sda, mdp1_vsync_out,
+	[129] = PINGROUP(129, qup1_se3_23, qup1_se3_01, ccu_i2c_sda, mdp1_vsync_out,
 			 _, atest_usb21, ddr_pxi, _, _, _, _),
-	[130] = PINGROUP(130, qup1_se3, qup1_se3, ccu_i2c_scl, mdp1_vsync_out,
+	[130] = PINGROUP(130, qup1_se3_23, qup1_se3_01, ccu_i2c_scl, mdp1_vsync_out,
 			 _, atest_usb20, ddr_pxi, _, _, _, _),
 	[131] = PINGROUP(131, qup1_se4, qup1_se6, ccu_i2c_sda, mdp1_vsync_out,
 			 _, atest_usb21, ddr_pxi, _, _, _, _),
diff --git a/drivers/pinctrl/qcom/pinctrl-qcs8300.c b/drivers/pinctrl/qcom/pinctrl-qcs8300.c
index 9a904d809e11da..fec86818c51188 100644
--- a/drivers/pinctrl/qcom/pinctrl-qcs8300.c
+++ b/drivers/pinctrl/qcom/pinctrl-qcs8300.c
@@ -45,6 +45,7 @@
 		.intr_enable_bit = 0,		\
 		.intr_status_bit = 0,		\
 		.intr_target_bit = 5,		\
+		.intr_target_width = 4,		\
 		.intr_target_kpss_val = 3,	\
 		.intr_raw_status_bit = 4,	\
 		.intr_polarity_bit = 1,		\
diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
index 922c27177b5ac1..9163bb14d70365 100644
--- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c
@@ -734,29 +734,50 @@ static int pmic_gpio_get_direction(struct gpio_chip *chip, unsigned pin)
 	    (!pad->input_enabled && !pad->output_enabled))
 		return -EINVAL;
 
-	/* Make sure the state is aligned on what pmic_gpio_get() returns */
-	return pad->input_enabled ? GPIO_LINE_DIRECTION_IN : GPIO_LINE_DIRECTION_OUT;
+	/*
+	 * An open-drain or open-source pad keeps its input buffer enabled while
+	 * driving, so the output buffer is what tells the direction.
+	 */
+	return pad->output_enabled ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
 }
 
 static int pmic_gpio_direction_input(struct gpio_chip *chip, unsigned pin)
 {
 	struct pmic_gpio_state *state = gpiochip_get_data(chip);
-	unsigned long config;
+	unsigned long configs[2];
 
-	config = pinconf_to_config_packed(PIN_CONFIG_INPUT_ENABLE, 1);
+	configs[0] = pinconf_to_config_packed(PIN_CONFIG_OUTPUT_ENABLE, 0);
+	configs[1] = pinconf_to_config_packed(PIN_CONFIG_INPUT_ENABLE, 1);
 
-	return pmic_gpio_config_set(state->ctrl, pin, &config, 1);
+	return pmic_gpio_config_set(state->ctrl, pin, configs,
+				    ARRAY_SIZE(configs));
 }
 
 static int pmic_gpio_direction_output(struct gpio_chip *chip,
 				      unsigned pin, int val)
 {
 	struct pmic_gpio_state *state = gpiochip_get_data(chip);
-	unsigned long config;
+	struct pmic_gpio_pad *pad = state->ctrl->desc->pins[pin].drv_data;
+	unsigned long configs[2];
 
-	config = pinconf_to_config_packed(PIN_CONFIG_LEVEL, val);
+	/*
+	 * An open-drain or open-source pad only ever drives one rail, so the
+	 * line can still be sampled while the pad is an output.  Keep the input
+	 * buffer enabled for those, so that pmic_gpio_get() reports what is on
+	 * the wire rather than the value last written, and disable it for a
+	 * CMOS pad, so that the pad ends up in DIGITAL_OUTPUT rather than
+	 * DIGITAL_INPUT_OUTPUT.  Program it either way, as the buffer may have
+	 * been left in the opposite state by the bootloader or by an earlier
+	 * direction change with a different buffer type.  gpiolib applies
+	 * PIN_CONFIG_DRIVE_OPEN_DRAIN before calling this, so buffer_type is
+	 * already up to date here.
+	 */
+	configs[0] = pinconf_to_config_packed(PIN_CONFIG_INPUT_ENABLE,
+					     pad->buffer_type != PMIC_GPIO_OUT_BUF_CMOS);
+	configs[1] = pinconf_to_config_packed(PIN_CONFIG_LEVEL, val);
 
-	return pmic_gpio_config_set(state->ctrl, pin, &config, 1);
+	return pmic_gpio_config_set(state->ctrl, pin, configs,
+				    ARRAY_SIZE(configs));
 }
 
 static int pmic_gpio_get(struct gpio_chip *chip, unsigned pin)
@@ -1259,6 +1280,7 @@ static const struct of_device_id pmic_gpio_of_match[] = {
 	{ .compatible = "qcom,pm8994-gpio", .data = (void *) 22 },
 	{ .compatible = "qcom,pm8998-gpio", .data = (void *) 26 },
 	{ .compatible = "qcom,pma8084-gpio", .data = (void *) 22 },
+	{ .compatible = "qcom,pmau0102-gpio", .data = (void *) 12 },
 	{ .compatible = "qcom,pmc8380-gpio", .data = (void *) 10 },
 	{ .compatible = "qcom,pmcx0102-gpio", .data = (void *)14 },
 	{ .compatible = "qcom,pmd8028-gpio", .data = (void *) 4 },
diff --git a/drivers/pinctrl/qcom/tlmm-test.c b/drivers/pinctrl/qcom/tlmm-test.c
index 13c417731be7a4..8bff92dc0d0b8f 100644
--- a/drivers/pinctrl/qcom/tlmm-test.c
+++ b/drivers/pinctrl/qcom/tlmm-test.c
@@ -587,7 +587,7 @@ static int tlmm_reg_base(struct device_node *tlmm, struct resource *res)
 		return count;
 	}
 
-	reg_names = kcalloc(count, sizeof(char *), GFP_KERNEL);
+	reg_names = kcalloc(count, sizeof(*reg_names), GFP_KERNEL);
 	if (!reg_names)
 		return -ENOMEM;
 
diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
index be52d47d77ae23..9bf3743dc1ce2e 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
@@ -69,6 +69,8 @@
 #define PIN_CFG_PVDD1833_OTH_AWO_POC	BIT(19) /* known on RZ/G3L only */
 #define PIN_CFG_PVDD1833_OTH_ISO_POC	BIT(20) /* known on RZ/G3L only */
 #define PIN_CFG_WDTOVF_N_POC		BIT(21) /* known on RZ/G3L only */
+#define PIN_CFG_IO_VMC_SD2		BIT(22) /* known on RZ/G3L only */
+#define PIN_CFG_IO_VMC_I3C		BIT(23)
 
 #define RZG2L_SINGLE_PIN		BIT_ULL(63)	/* Dedicated pin */
 #define RZG2L_VARIABLE_CFG		BIT_ULL(62)	/* Variable cfg for port pins */
@@ -180,12 +182,8 @@
 #define SMT(off)		(0x3400 + (off) * 8)
 #define SD_CH(off, ch)		((off) + (ch) * 4)
 #define ETH_POC(off, ch)	((off) + (ch) * 4)
-#define QSPI			(0x3008) /* known on RZ/{G2L,G2LC,G2UL,Five} only */
-#define OTHER_POC		(0x3028) /* known on RZ/G3L only */
 
-#define PVDD_2500		2	/* I/O domain voltage 2.5V */
-#define PVDD_1800		1	/* I/O domain voltage <= 1.8V */
-#define PVDD_3300		0	/* I/O domain voltage >= 3.3V */
+#define PVDD_MASK		0x3
 
 #define PWPR_B0WI		BIT(7)	/* Bit Write Disable */
 #define PWPR_PFCWE		BIT(6)	/* PFC Register Write Enable */
@@ -254,28 +252,57 @@ static const struct pin_config_item renesas_rzv2h_conf_items[] = {
  * struct rzg2l_register_offsets - specific register offsets
  * @pwpr: PWPR register offset
  * @sd_ch: SD_CH register offset
+ * @sd_ch2: SD_CH2_POC register offset
  * @eth_poc: ETH_POC register offset
  * @oen: OEN register offset
  * @qspi: QSPI register offset
  * @other_poc: OTHER_POC register offset
+ * @i3c_set: I3C_SET register offset
  */
 struct rzg2l_register_offsets {
 	u16 pwpr;
 	u16 sd_ch;
+	u16 sd_ch2;
 	u16 eth_poc;
 	u16 oen;
 	u16 qspi;
 	u16 other_poc;
+	u16 i3c_set;
+};
+
+/**
+ * struct rzg2l_register_masks - Masks for different RZ/G2L pinctrl functionalities
+ * @other_poc_pvdd1833_oth_awo_poc: PVDD1833_OTH_AWO_POC mask
+ * @other_poc_pvdd1833_oth_iso_poc: PVDD1833_OTH_ISO_POC mask
+ * @other_poc_wdtovf_n_poc: WDTOVF_N_POC mask
+ * @i3c_set_poc: I3C_SET_POC mask
+ */
+struct rzg2l_register_masks {
+	union {
+		/* RZ/G3L masks */
+		struct {
+			u8 other_poc_pvdd1833_oth_awo_poc;
+			u8 other_poc_pvdd1833_oth_iso_poc;
+			u8 other_poc_wdtovf_n_poc;
+		};
+
+		/* RZ/G3S masks */
+		struct {
+			u8 i3c_set_poc;
+		};
+	};
 };
 
 /**
  * enum rzg2l_iolh_index - starting indices in IOLH specific arrays
+ * @RZG2L_IOLH_IDX_NA: index N/A (to be used for error checking)
  * @RZG2L_IOLH_IDX_1V8: starting index for 1V8 power source
  * @RZG2L_IOLH_IDX_2V5: starting index for 2V5 power source
  * @RZG2L_IOLH_IDX_3V3: starting index for 3V3 power source
  * @RZG2L_IOLH_IDX_MAX: maximum index
  */
 enum rzg2l_iolh_index {
+	RZG2L_IOLH_IDX_NA = -1,
 	RZG2L_IOLH_IDX_1V8 = 0,
 	RZG2L_IOLH_IDX_2V5 = 4,
 	RZG2L_IOLH_IDX_3V3 = 8,
@@ -288,6 +315,8 @@ enum rzg2l_iolh_index {
 /**
  * struct rzg2l_hwcfg - hardware configuration data structure
  * @regs: hardware specific register offsets
+ * @masks: hardware specific masks for various functionalities available in
+ * the registers described by regs
  * @iolh_groupa_ua: IOLH group A uA specific values
  * @iolh_groupb_ua: IOLH group B uA specific values
  * @iolh_groupc_ua: IOLH group C uA specific values
@@ -301,6 +330,7 @@ enum rzg2l_iolh_index {
  */
 struct rzg2l_hwcfg {
 	const struct rzg2l_register_offsets regs;
+	const struct rzg2l_register_masks masks;
 	u16 iolh_groupa_ua[RZG2L_IOLH_IDX_MAX];
 	u16 iolh_groupb_ua[RZG2L_IOLH_IDX_MAX];
 	u16 iolh_groupc_ua[RZG2L_IOLH_IDX_MAX];
@@ -368,10 +398,12 @@ struct rzg2l_pinctrl_pin_settings {
  * @nod: NOD registers cache
  * @clone: Clone register cache
  * @sd_ch: SD_CH registers cache
+ * @sd_ch2: SD_CH2_POC registers cache
  * @eth_poc: ET_POC registers cache
  * @oen: Output Enable register cache
  * @other_poc: OTHER_POC register cache
  * @qspi: QSPI registers cache
+ * @i3c_set: I3C_SET register cache
  */
 struct rzg2l_pinctrl_reg_cache {
 	u8	*p;
@@ -386,10 +418,12 @@ struct rzg2l_pinctrl_reg_cache {
 	u32	*nod[2];
 	u32	clone;
 	u8	sd_ch[2];
+	u8	sd_ch2;
 	u8	eth_poc[2];
 	u8	oen;
 	u8	other_poc;
 	u8	qspi;
+	u8	i3c_set;
 };
 
 struct rzg2l_pinctrl {
@@ -421,7 +455,89 @@ struct rzg2l_pinctrl {
 	u32				clone_offset;
 };
 
-static const u16 available_ps[] = { 1800, 2500, 3300 };
+/**
+ * struct rzg2l_pinctrl_ps_desc - RZ/G2L power source descriptor
+ * @caps: Capabilities that applies to the power source
+ * @iolh_index: IOLH index
+ * @ps: Power source value
+ * @pwr_reg_val: Power source register value
+ */
+struct rzg2l_pinctrl_ps_desc {
+	u32 caps;
+	enum rzg2l_iolh_index iolh_index;
+	u16 ps;
+	u16 pwr_reg_val;
+};
+
+#define RZG2L_PINCTRL_PS_DESC(_ps, _pwr_reg_val, _caps, _iolh_index) \
+	{ \
+		.ps = _ps, \
+		.pwr_reg_val = _pwr_reg_val, \
+		.caps = _caps, \
+		.iolh_index = _iolh_index, \
+	}
+
+static const struct rzg2l_pinctrl_ps_desc available_ps[] = {
+	/* Ethernet I/O voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1800, 1, PIN_CFG_IO_VMC_ETH0 | PIN_CFG_IO_VMC_ETH1,
+			      RZG2L_IOLH_IDX_1V8),
+	RZG2L_PINCTRL_PS_DESC(2500, 2, PIN_CFG_IO_VMC_ETH0 | PIN_CFG_IO_VMC_ETH1,
+			      RZG2L_IOLH_IDX_2V5),
+	RZG2L_PINCTRL_PS_DESC(3300, 0, PIN_CFG_IO_VMC_ETH0 | PIN_CFG_IO_VMC_ETH1,
+			      RZG2L_IOLH_IDX_3V3),
+
+	/* SD I/O voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1800, 1, PIN_CFG_IO_VMC_SD0 | PIN_CFG_IO_VMC_SD1 |
+			      PIN_CFG_IO_VMC_SD2, RZG2L_IOLH_IDX_1V8),
+	RZG2L_PINCTRL_PS_DESC(3300, 0, PIN_CFG_IO_VMC_SD0 | PIN_CFG_IO_VMC_SD1 |
+			      PIN_CFG_IO_VMC_SD2, RZG2L_IOLH_IDX_3V3),
+
+	/* QSPI I/O voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1800, 1, PIN_CFG_IO_VMC_QSPI, RZG2L_IOLH_IDX_1V8),
+	RZG2L_PINCTRL_PS_DESC(3300, 0, PIN_CFG_IO_VMC_QSPI, RZG2L_IOLH_IDX_3V3),
+
+	/* AWO I/O voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1800, 1, PIN_CFG_PVDD1833_OTH_AWO_POC,
+			      RZG2L_IOLH_IDX_1V8),
+	RZG2L_PINCTRL_PS_DESC(3300, 0, PIN_CFG_PVDD1833_OTH_AWO_POC,
+			      RZG2L_IOLH_IDX_3V3),
+
+	/* ISO I/O voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1800, 1, PIN_CFG_PVDD1833_OTH_ISO_POC,
+			      RZG2L_IOLH_IDX_1V8),
+	RZG2L_PINCTRL_PS_DESC(3300, 0, PIN_CFG_PVDD1833_OTH_ISO_POC,
+			      RZG2L_IOLH_IDX_3V3),
+
+	/* WDTOVF I/O voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1800, 1, PIN_CFG_WDTOVF_N_POC, RZG2L_IOLH_IDX_1V8),
+	RZG2L_PINCTRL_PS_DESC(3300, 0, PIN_CFG_WDTOVF_N_POC, RZG2L_IOLH_IDX_3V3),
+
+	/* I3C I/O voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1200, 1, PIN_CFG_IO_VMC_I3C, RZG2L_IOLH_IDX_NA),
+	RZG2L_PINCTRL_PS_DESC(1800, 0, PIN_CFG_IO_VMC_I3C, RZG2L_IOLH_IDX_NA),
+
+	/* Software voltage domains */
+	RZG2L_PINCTRL_PS_DESC(1800, 0, PIN_CFG_SOFT_PS, RZG2L_IOLH_IDX_1V8),
+	RZG2L_PINCTRL_PS_DESC(2500, 0, PIN_CFG_SOFT_PS, RZG2L_IOLH_IDX_2V5),
+	RZG2L_PINCTRL_PS_DESC(3300, 0, PIN_CFG_SOFT_PS, RZG2L_IOLH_IDX_3V3),
+};
+
+#define RZG2L_PINCTRL_PS_DESC_MEMBER_TO_DESC_FUNC(_name, _desc_member, _caps)	\
+static const struct rzg2l_pinctrl_ps_desc *_name(u16 _desc_member, u32 _caps)	\
+{										\
+	for (unsigned int i = 0; i < ARRAY_SIZE(available_ps); i++) {		\
+		if (available_ps[i]._desc_member != _desc_member)		\
+			continue;						\
+										\
+		if (available_ps[i].caps & _caps)				\
+			return &available_ps[i];				\
+	}									\
+										\
+	return NULL;								\
+}
+
+RZG2L_PINCTRL_PS_DESC_MEMBER_TO_DESC_FUNC(rzg2l_ps_to_desc, ps, caps)
+RZG2L_PINCTRL_PS_DESC_MEMBER_TO_DESC_FUNC(rzg2l_pwr_reg_val_to_desc, pwr_reg_val, caps)
 
 static u64 rzg2l_pinctrl_get_variable_pin_cfg(struct rzg2l_pinctrl *pctrl,
 					      u64 pincfg,
@@ -474,20 +590,32 @@ static const u64 r9a08g046_variable_pin_cfg[] = {
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 5, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 6, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 7, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 0, RZG3L_MPXED_PIN_FUNCS(B)),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 1, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 2, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 3, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 4, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 5, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 0, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IO_VMC_SD1),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 1, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD1),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 2, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD1),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 3, RZG2L_MPXED_COMMON_PIN_FUNCS(B) | PIN_CFG_IEN |
+				    PIN_CFG_IO_VMC_SD1),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 4, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD1),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 5, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD1),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 6, RZG3L_MPXED_PIN_FUNCS_POC(B, ISO)),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 7, RZG3L_MPXED_PIN_FUNCS_POC(B, ISO)),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 0, RZG3L_MPXED_PIN_FUNCS(B)),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 1, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 2, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 3, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 4, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
-	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 5, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 0, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IO_VMC_SD2),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 1, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD2),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 2, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD2),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 3, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD2),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 4, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD2),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 5, RZG2L_MPXED_COMMON_PIN_FUNCS(B) |
+				    PIN_CFG_IEN | PIN_CFG_IO_VMC_SD2),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 0, RZG3L_MPXED_PIN_FUNCS(A) | PIN_CFG_IEN),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 1, RZG3L_MPXED_PIN_FUNCS(A)),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 2, RZG3L_MPXED_PIN_FUNCS(A)),
@@ -1047,106 +1175,125 @@ static void rzg2l_rmw_pin_config(struct rzg2l_pinctrl *pctrl, u32 offset,
 }
 
 static int rzg2l_caps_to_pwr_reg(const struct rzg2l_register_offsets *regs,
-				 u32 caps, u8 *mask)
+				 const struct rzg2l_register_masks *masks,
+				 u32 caps, u16 *offset, u8 *mask)
 {
-	if (caps & PIN_CFG_IO_VMC_SD0)
-		return SD_CH(regs->sd_ch, 0);
-	if (caps & PIN_CFG_IO_VMC_SD1)
-		return SD_CH(regs->sd_ch, 1);
-	if (caps & PIN_CFG_IO_VMC_ETH0)
-		return ETH_POC(regs->eth_poc, 0);
-	if (caps & PIN_CFG_IO_VMC_ETH1)
-		return ETH_POC(regs->eth_poc, 1);
-	if (caps & PIN_CFG_IO_VMC_QSPI)
-		return QSPI;
+	*mask = PVDD_MASK;
+
+	if (caps & PIN_CFG_IO_VMC_SD0) {
+		*offset = SD_CH(regs->sd_ch, 0);
+		return 0;
+	}
+	if (caps & PIN_CFG_IO_VMC_SD1) {
+		*offset = SD_CH(regs->sd_ch, 1);
+		return 0;
+	}
+	if (caps & PIN_CFG_IO_VMC_SD2) {
+		*offset = regs->sd_ch2;
+		return 0;
+	}
+	if (caps & PIN_CFG_IO_VMC_ETH0) {
+		*offset = ETH_POC(regs->eth_poc, 0);
+		return 0;
+	}
+	if (caps & PIN_CFG_IO_VMC_ETH1) {
+		*offset = ETH_POC(regs->eth_poc, 1);
+		return 0;
+	}
+	if (caps & PIN_CFG_IO_VMC_QSPI) {
+		*offset = regs->qspi;
+		return 0;
+	}
 	if (caps & PIN_CFG_OTHER_POC_MASK) {
+		*offset = regs->other_poc;
 		if (caps & PIN_CFG_PVDD1833_OTH_AWO_POC)
-			*mask = BIT(0);
+			*mask = masks->other_poc_pvdd1833_oth_awo_poc;
 		else if (caps & PIN_CFG_PVDD1833_OTH_ISO_POC)
-			*mask = BIT(1);
+			*mask = masks->other_poc_pvdd1833_oth_iso_poc;
 		else
-			*mask = BIT(2);
-
-		return OTHER_POC;
+			*mask = masks->other_poc_wdtovf_n_poc;
+		return 0;
+	}
+	if (caps & PIN_CFG_IO_VMC_I3C) {
+		*offset = regs->i3c_set;
+		*mask = masks->i3c_set_poc;
+		return 0;
 	}
 
 	return -EINVAL;
 }
 
+static int rzg2l_pwr_reg_val_to_ps(u16 pwr_reg_val, u32 caps)
+{
+	const struct rzg2l_pinctrl_ps_desc *desc;
+
+	desc = rzg2l_pwr_reg_val_to_desc(pwr_reg_val, caps);
+	if (!desc)
+		return -EINVAL;
+
+	return desc->ps;
+}
+
+static int rzg2l_ps_to_pwr_reg_val(u16 ps, u32 caps)
+{
+	const struct rzg2l_pinctrl_ps_desc *desc;
+
+	desc = rzg2l_ps_to_desc(ps, caps);
+	if (!desc)
+		return -EINVAL;
+
+	return desc->pwr_reg_val;
+}
+
 static int rzg2l_get_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps)
 {
 	const struct rzg2l_hwcfg *hwcfg = pctrl->data->hwcfg;
 	const struct rzg2l_register_offsets *regs = &hwcfg->regs;
-	u8 val, mask;
-	int pwr_reg;
+	const struct rzg2l_register_masks *masks = &hwcfg->masks;
+	u8 mask, val;
+	u16 offset;
+	int ret;
 
 	if (caps & PIN_CFG_SOFT_PS)
 		return pctrl->settings[pin].power_source;
 
-	pwr_reg = rzg2l_caps_to_pwr_reg(regs, caps, &mask);
-	if (pwr_reg < 0)
-		return pwr_reg;
-
-	val = readb(pctrl->base + pwr_reg);
-	if (pwr_reg == OTHER_POC)
-		val = field_get(mask, val);
-
-	switch (val) {
-	case PVDD_1800:
-		return 1800;
-	case PVDD_2500:
-		return 2500;
-	case PVDD_3300:
-		return 3300;
-	default:
-		/* Should not happen. */
-		return -EINVAL;
-	}
+	ret = rzg2l_caps_to_pwr_reg(regs, masks, caps, &offset, &mask);
+	if (ret)
+		return ret;
+
+	val = readb(pctrl->base + offset);
+
+	return rzg2l_pwr_reg_val_to_ps(field_get(mask, val), caps);
 }
 
 static int rzg2l_set_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps, u32 ps)
 {
 	const struct rzg2l_hwcfg *hwcfg = pctrl->data->hwcfg;
 	const struct rzg2l_register_offsets *regs = &hwcfg->regs;
-	u8 poc_val, val, mask;
-	int pwr_reg;
+	const struct rzg2l_register_masks *masks = &hwcfg->masks;
+	u16 offset;
+	u8 mask;
+	int ret;
 
 	if (caps & PIN_CFG_SOFT_PS) {
 		pctrl->settings[pin].power_source = ps;
 		return 0;
 	}
 
-	switch (ps) {
-	case 1800:
-		poc_val = PVDD_1800;
-		break;
-	case 2500:
-		if (!(caps & (PIN_CFG_IO_VMC_ETH0 | PIN_CFG_IO_VMC_ETH1)))
-			return -EINVAL;
-		poc_val = PVDD_2500;
-		break;
-	case 3300:
-		poc_val = PVDD_3300;
-		break;
-	default:
-		return -EINVAL;
-	}
+	ret = rzg2l_caps_to_pwr_reg(regs, masks, caps, &offset, &mask);
+	if (ret)
+		return ret;
+
+	ret = rzg2l_ps_to_pwr_reg_val(ps, caps);
+	if (ret < 0)
+		return ret;
 
-	pwr_reg = rzg2l_caps_to_pwr_reg(regs, caps, &mask);
-	if (pwr_reg < 0)
-		return pwr_reg;
+	scoped_guard(raw_spinlock_irqsave, &pctrl->lock) {
+		u8 tmp = readb(pctrl->base + offset);
 
-	if (pwr_reg == OTHER_POC) {
-		scoped_guard(raw_spinlock_irqsave, &pctrl->lock) {
-			val = readb(pctrl->base + pwr_reg);
-			if (poc_val)
-				val |= mask;
-			else
-				val &= ~mask;
-			writeb(val, pctrl->base + pwr_reg);
-		}
-	} else {
-		writeb(poc_val, pctrl->base + pwr_reg);
+		tmp &= ~mask;
+		tmp |= field_prep(mask, ret);
+		writeb(tmp, pctrl->base + offset);
 	}
 
 	pctrl->settings[pin].power_source = ps;
@@ -1154,32 +1301,11 @@ static int rzg2l_set_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps
 	return 0;
 }
 
-static bool rzg2l_ps_is_supported(u16 ps)
+static enum rzg2l_iolh_index rzg2l_ps_to_iolh_idx(u16 ps, u32 caps)
 {
-	unsigned int i;
+	const struct rzg2l_pinctrl_ps_desc *desc = rzg2l_ps_to_desc(ps, caps);
 
-	for (i = 0; i < ARRAY_SIZE(available_ps); i++) {
-		if (available_ps[i] == ps)
-			return true;
-	}
-
-	return false;
-}
-
-static enum rzg2l_iolh_index rzg2l_ps_to_iolh_idx(u16 ps)
-{
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(available_ps); i++) {
-		if (available_ps[i] == ps)
-			break;
-	}
-
-	/*
-	 * We multiply with RZG2L_IOLH_MAX_DS_ENTRIES as we have
-	 * RZG2L_IOLH_MAX_DS_ENTRIES DS values per power source
-	 */
-	return i * RZG2L_IOLH_MAX_DS_ENTRIES;
+	return desc ? desc->iolh_index : RZG2L_IOLH_IDX_NA;
 }
 
 static u16 rzg2l_iolh_val_to_ua(const struct rzg2l_hwcfg *hwcfg, u32 caps, u8 val)
@@ -1564,7 +1690,11 @@ static int rzg2l_pinctrl_pinconf_get(struct pinctrl_dev *pctldev,
 		ret = rzg2l_get_power_source(pctrl, _pin, cfg);
 		if (ret < 0)
 			return ret;
-		iolh_idx = rzg2l_ps_to_iolh_idx(ret);
+
+		iolh_idx = rzg2l_ps_to_iolh_idx(ret, cfg);
+		if (iolh_idx == RZG2L_IOLH_IDX_NA)
+			return -EINVAL;
+
 		val = rzg2l_read_pin_config(pctrl, IOLH(off), bit, IOLH_MASK);
 		arg = rzg2l_iolh_val_to_ua(hwcfg, cfg, iolh_idx + val);
 		break;
@@ -1760,8 +1890,7 @@ static int rzg2l_pinctrl_pinconf_set(struct pinctrl_dev *pctldev,
 
 	/* Apply power source. */
 	if (settings.power_source != pctrl->settings[_pin].power_source) {
-		ret = rzg2l_ps_is_supported(settings.power_source);
-		if (!ret)
+		if (!rzg2l_ps_to_desc(settings.power_source, cfg))
 			return -EINVAL;
 
 		/* Apply power source. */
@@ -1775,7 +1904,10 @@ static int rzg2l_pinctrl_pinconf_set(struct pinctrl_dev *pctldev,
 		enum rzg2l_iolh_index iolh_idx;
 		int val;
 
-		iolh_idx = rzg2l_ps_to_iolh_idx(settings.power_source);
+		iolh_idx = rzg2l_ps_to_iolh_idx(settings.power_source, cfg);
+		if (iolh_idx == RZG2L_IOLH_IDX_NA)
+			return -EINVAL;
+
 		ret = rzg2l_ds_is_supported(pctrl, cfg, iolh_idx,
 					    settings.drive_strength_ua);
 		if (!ret)
@@ -2458,6 +2590,8 @@ static const struct rzg2l_dedicated_configs rzg3s_dedicated_pins[] = {
 	{ "AUDIO_CLK1", RZG2L_SINGLE_PIN_PACK(0x2, 0, PIN_CFG_IEN) },
 	{ "AUDIO_CLK2", RZG2L_SINGLE_PIN_PACK(0x2, 1, PIN_CFG_IEN) },
 	{ "WDTOVF_PERROUT#", RZG2L_SINGLE_PIN_PACK(0x6, 0, PIN_CFG_IOLH_A | PIN_CFG_SOFT_PS) },
+	{ "I3C_SDA", RZG2L_SINGLE_PIN_PACK(0x9, 0, (PIN_CFG_IEN | PIN_CFG_IO_VMC_I3C)) },
+	{ "I3C_SCL", RZG2L_SINGLE_PIN_PACK(0x9, 1, (PIN_CFG_IEN | PIN_CFG_IO_VMC_I3C)) },
 	{ "SD0_CLK", RZG2L_SINGLE_PIN_PACK(0x10, 0, (PIN_CFG_IOLH_B | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_CMD", RZG2L_SINGLE_PIN_PACK(0x10, 1, (PIN_CFG_IOLH_B | PIN_CFG_IEN |
 						     PIN_CFG_IO_VMC_SD0)) },
@@ -2677,28 +2811,28 @@ static const struct rzg2l_dedicated_configs rzg3l_dedicated_pins[] = {
 	  (PIN_CFG_IOLH_A | PIN_CFG_PUPD | PIN_CFG_PVDD1833_OTH_AWO_POC)) },
 	{ "SCIF0_TXD", RZG2L_SINGLE_PIN_PACK(0x6, 1,
 	  (PIN_CFG_IOLH_A | PIN_CFG_PUPD | PIN_CFG_PVDD1833_OTH_AWO_POC)) },
-	{ "SD0_CLK", RZG2L_SINGLE_PIN_PACK(0x9, 0, PIN_CFG_IOLH_B) },
+	{ "SD0_CLK", RZG2L_SINGLE_PIN_PACK(0x9, 0, PIN_CFG_IOLH_B | PIN_CFG_IO_VMC_SD0) },
 	{ "SD0_CMD", RZG2L_SINGLE_PIN_PACK(0x9, 1,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
-	{ "SD0_RST#", RZG2L_SINGLE_PIN_PACK(0x9, 2, PIN_CFG_IOLH_B) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
+	{ "SD0_RST#", RZG2L_SINGLE_PIN_PACK(0x9, 2, PIN_CFG_IOLH_B | PIN_CFG_IO_VMC_SD0) },
 	{ "SD0_DS", RZG2L_SINGLE_PIN_PACK(0x9, 5,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT0", RZG2L_SINGLE_PIN_PACK(0x0a, 0,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT1", RZG2L_SINGLE_PIN_PACK(0x0a, 1,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT2", RZG2L_SINGLE_PIN_PACK(0x0a, 2,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT3", RZG2L_SINGLE_PIN_PACK(0x0a, 3,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT4", RZG2L_SINGLE_PIN_PACK(0x0a, 4,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT5", RZG2L_SINGLE_PIN_PACK(0x0a, 5,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT6", RZG2L_SINGLE_PIN_PACK(0x0a, 6,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 	{ "SD0_DAT7", RZG2L_SINGLE_PIN_PACK(0x0a, 7,
-	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD | PIN_CFG_IO_VMC_SD0)) },
 };
 
 static const u32 r9a08g046_clone_channel_data[] = {
@@ -3672,11 +3806,16 @@ static int rzg2l_pinctrl_suspend_noirq(struct device *dev)
 			cache->eth_poc[i] = readb(pctrl->base + ETH_POC(regs->eth_poc, i));
 	}
 
+	if (regs->sd_ch2)
+		cache->sd_ch2 = readb(pctrl->base + regs->sd_ch2);
+
 	if (regs->qspi)
 		cache->qspi = readb(pctrl->base + regs->qspi);
 	cache->oen = readb(pctrl->base + pctrl->data->hwcfg->regs.oen);
 	if (regs->other_poc)
 		cache->other_poc = readb(pctrl->base + regs->other_poc);
+	if (regs->i3c_set)
+		cache->i3c_set = readb(pctrl->base + regs->i3c_set);
 
 	if (pctrl->syscon) {
 		int ret;
@@ -3719,11 +3858,16 @@ static int rzg2l_pinctrl_resume_noirq(struct device *dev)
 		writeb(cache->qspi, pctrl->base + regs->qspi);
 	if (regs->other_poc)
 		writeb(cache->other_poc, pctrl->base + regs->other_poc);
+	if (regs->i3c_set)
+		writeb(cache->i3c_set, pctrl->base + regs->i3c_set);
 
 	raw_spin_lock_irqsave(&pctrl->lock, flags);
 	rzg2l_oen_write_with_pwpr(pctrl, cache->oen);
 	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
 
+	if (regs->sd_ch2)
+		writeb(cache->sd_ch2, pctrl->base + regs->sd_ch2);
+
 	for (u8 i = 0; i < 2; i++) {
 		if (regs->sd_ch)
 			writeb(cache->sd_ch[i], pctrl->base + SD_CH(regs->sd_ch, i));
@@ -3774,9 +3918,9 @@ static const struct rzg2l_hwcfg rzg2l_hwcfg = {
 	.regs = {
 		.pwpr = 0x3014,
 		.sd_ch = 0x3000,
+		.qspi = 0x3008,
 		.eth_poc = 0x300c,
 		.oen = 0x3018,
-		.qspi = QSPI,
 	},
 	.iolh_groupa_ua = {
 		/* 3v3 power source */
@@ -3791,9 +3935,15 @@ static const struct rzg2l_hwcfg rzg3l_hwcfg = {
 	.regs = {
 		.pwpr = 0x3000,
 		.sd_ch = 0x3004,
+		.sd_ch2 = 0x3024,
 		.eth_poc = 0x3010,
 		.oen = 0x3018,
-		.other_poc = OTHER_POC,
+		.other_poc = 0x3028,
+	},
+	.masks = {
+		.other_poc_pvdd1833_oth_awo_poc = BIT(0),
+		.other_poc_pvdd1833_oth_iso_poc = BIT(1),
+		.other_poc_wdtovf_n_poc = BIT(2),
 	},
 	.iolh_groupa_ua = {
 		/* 1v8 power source */
@@ -3826,8 +3976,12 @@ static const struct rzg2l_hwcfg rzg3s_hwcfg = {
 		.pwpr = 0x3000,
 		.sd_ch = 0x3004,
 		.eth_poc = 0x3010,
+		.i3c_set = 0x301c,
 		.oen = 0x3018,
 	},
+	.masks = {
+		.i3c_set_poc = BIT(2),
+	},
 	.iolh_groupa_ua = {
 		/* 1v8 power source */
 		[RZG2L_IOLH_IDX_1V8] = 2200, 4400, 9000, 10000,
diff --git a/drivers/pinctrl/renesas/pinctrl-rzt2h.c b/drivers/pinctrl/renesas/pinctrl-rzt2h.c
index eb70aee39f1ae3..4872a208e0c3db 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzt2h.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzt2h.c
@@ -94,8 +94,8 @@ struct rzt2h_pinctrl {
 	void __iomem			*base0, *base1;
 	struct device			*dev;
 	struct gpio_chip		gpio_chip;
-	struct pinctrl_gpio_range	gpio_range;
 	DECLARE_BITMAP(used_irqs, RZT2H_INTERRUPTS_NUM);
+	u8				saved_pm[RZT2H_INTERRUPTS_NUM];
 	raw_spinlock_t			lock; /* lock read/write registers */
 	struct mutex			mutex; /* serialize adding groups and functions */
 	bool				safety_port_enabled;
@@ -170,6 +170,43 @@ static int rzt2h_validate_pin(struct rzt2h_pinctrl *pctrl, unsigned int offset)
 	return (pincfg & BIT(pin)) ? 0 : -EINVAL;
 }
 
+static bool rzt2h_pin_mode_is_peripheral(struct rzt2h_pinctrl *pctrl, u8 port, u8 bit)
+{
+	return rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit);
+}
+
+static u8 rzt2h_pin_read_pfc(struct rzt2h_pinctrl *pctrl, u8 port, u8 pin)
+{
+	u64 reg64 = rzt2h_pinctrl_readq(pctrl, port, PFC(port));
+
+	return field_get(PFC_PIN_MASK(pin), reg64);
+}
+
+static bool rzt2h_pin_read_input(struct rzt2h_pinctrl *pctrl, u8 port, u8 bit)
+{
+	return rzt2h_pinctrl_readb(pctrl, port, PIN(port)) & BIT(bit);
+}
+
+static u8 rzt2h_pin_read_pm(struct rzt2h_pinctrl *pctrl, u8 port, u8 pin)
+{
+	u16 reg = rzt2h_pinctrl_readw(pctrl, port, PM(port));
+
+	return field_get(PM_PIN_MASK(pin), reg);
+}
+
+static void rzt2h_pin_write_pm(struct rzt2h_pinctrl *pctrl, u8 port, u8 pin,
+			       unsigned int pm)
+{
+	u16 reg;
+
+	guard(raw_spinlock_irqsave)(&pctrl->lock);
+
+	reg = rzt2h_pinctrl_readw(pctrl, port, PM(port));
+	reg &= ~PM_PIN_MASK(pin);
+	reg |= pm << (pin * 2);
+	rzt2h_pinctrl_writew(pctrl, port, reg, PM(port));
+}
+
 static void rzt2h_pinctrl_set_gpio_en(struct rzt2h_pinctrl *pctrl,
 				      u8 port, u8 pin, bool en)
 {
@@ -193,7 +230,7 @@ static void rzt2h_pinctrl_set_pfc_mode(struct rzt2h_pinctrl *pctrl,
 
 	reg64 = rzt2h_pinctrl_readq(pctrl, port, PFC(port));
 	/* Check if pin is already configured to the desired function */
-	if ((rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(pin)) &&
+	if (rzt2h_pin_mode_is_peripheral(pctrl, port, pin) &&
 	    field_get(PFC_PIN_MASK(pin), reg64) == func)
 		return;
 
@@ -747,15 +784,7 @@ static int rzt2h_gpio_request(struct gpio_chip *chip, unsigned int offset)
 static void rzt2h_gpio_set_direction(struct rzt2h_pinctrl *pctrl, u32 port,
 				     u8 bit, bool output)
 {
-	u16 reg;
-
-	guard(raw_spinlock_irqsave)(&pctrl->lock);
-
-	reg = rzt2h_pinctrl_readw(pctrl, port, PM(port));
-	reg &= ~PM_PIN_MASK(bit);
-
-	reg |= (output ? PM_OUTPUT : PM_INPUT) << (bit * 2);
-	rzt2h_pinctrl_writew(pctrl, port, reg, PM(port));
+	rzt2h_pin_write_pm(pctrl, port, bit, output ? PM_OUTPUT : PM_INPUT);
 }
 
 static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
@@ -763,9 +792,8 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
 	struct rzt2h_pinctrl *pctrl = gpiochip_get_data(chip);
 	u8 port = RZT2H_PIN_ID_TO_PORT(offset);
 	u8 bit = RZT2H_PIN_ID_TO_PIN(offset);
-	u64 reg64;
-	u16 reg;
 	int ret;
+	u8 pm;
 
 	ret = rzt2h_validate_pin(pctrl, offset);
 	if (ret)
@@ -773,7 +801,7 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
 
 	guard(raw_spinlock_irqsave)(&pctrl->lock);
 
-	if (rzt2h_pinctrl_readb(pctrl, port, PMC(port)) & BIT(bit)) {
+	if (rzt2h_pin_mode_is_peripheral(pctrl, port, bit)) {
 		/*
 		 * When a GPIO is being requested as an IRQ, the pinctrl
 		 * framework expects to be able to read the GPIO's direction.
@@ -783,19 +811,16 @@ static int rzt2h_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
 		 * called to enable the IRQ function.
 		 * Default to input direction for IRQ function.
 		 */
-		reg64 = rzt2h_pinctrl_readq(pctrl, port, PFC(port));
-		reg64 = (reg64 >> (bit * 8)) & PFC_MASK;
-		if (reg64 == PFC_FUNC_INTERRUPT)
+		if (rzt2h_pin_read_pfc(pctrl, port, bit) == PFC_FUNC_INTERRUPT)
 			return GPIO_LINE_DIRECTION_IN;
 
 		return -EINVAL;
 	}
 
-	reg = rzt2h_pinctrl_readw(pctrl, port, PM(port));
-	reg = (reg >> (bit * 2)) & PM_MASK;
-	if (reg & PM_OUTPUT)
+	pm = rzt2h_pin_read_pm(pctrl, port, bit);
+	if (pm & PM_OUTPUT)
 		return GPIO_LINE_DIRECTION_OUT;
-	if (reg & PM_INPUT)
+	if (pm & PM_INPUT)
 		return GPIO_LINE_DIRECTION_IN;
 
 	return -EINVAL;
@@ -825,13 +850,15 @@ static int rzt2h_gpio_get(struct gpio_chip *chip, unsigned int offset)
 	struct rzt2h_pinctrl *pctrl = gpiochip_get_data(chip);
 	u8 port = RZT2H_PIN_ID_TO_PORT(offset);
 	u8 bit = RZT2H_PIN_ID_TO_PIN(offset);
-	u16 reg;
+	u8 pm;
 
-	reg = rzt2h_pinctrl_readw(pctrl, port, PM(port));
-	reg = (reg >> (bit * 2)) & PM_MASK;
-	if (reg & PM_INPUT)
-		return !!(rzt2h_pinctrl_readb(pctrl, port, PIN(port)) & BIT(bit));
-	if (reg & PM_OUTPUT)
+	if (rzt2h_pin_mode_is_peripheral(pctrl, port, bit))
+		return rzt2h_pin_read_input(pctrl, port, bit);
+
+	pm = rzt2h_pin_read_pm(pctrl, port, bit);
+	if (pm & PM_INPUT)
+		return rzt2h_pin_read_input(pctrl, port, bit);
+	if (pm & PM_OUTPUT)
 		return !!(rzt2h_pinctrl_readb(pctrl, port, P(port)) & BIT(bit));
 
 	return -EINVAL;
@@ -1000,6 +1027,66 @@ static int rzt2h_gpio_irq_set_wake(struct irq_data *d, unsigned int on)
 	return 0;
 }
 
+static int rzt2h_gpio_irq_request_resources(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct rzt2h_pinctrl *pctrl = gpiochip_get_data(gc);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u8 port = RZT2H_PIN_ID_TO_PORT(hwirq);
+	u8 pin = RZT2H_PIN_ID_TO_PIN(hwirq);
+	u8 parent_irq, irq_idx;
+	int ret;
+
+	parent_irq = rzt2h_gpio_irq_map[hwirq];
+	if (parent_irq < RZT2H_INTERRUPTS_START)
+		return -EINVAL;
+
+	irq_idx = parent_irq - RZT2H_INTERRUPTS_START;
+	if (test_and_set_bit(irq_idx, pctrl->used_irqs))
+		return -EBUSY;
+
+	/*
+	 * rzt2h_pinctrl_set_pfc_mode() sets PM to Hi-Z before switching to the
+	 * interrupt function, losing the previous PM value.
+	 * Save it so it can be restored when the IRQ is freed.
+	 */
+	pctrl->saved_pm[irq_idx] = rzt2h_pin_read_pm(pctrl, port, pin);
+
+	rzt2h_pinctrl_set_pfc_mode(pctrl, port, pin, PFC_FUNC_INTERRUPT);
+
+	ret = gpiochip_lock_as_irq(gc, hwirq);
+	if (ret) {
+		clear_bit(irq_idx, pctrl->used_irqs);
+		rzt2h_pin_write_pm(pctrl, port, pin, pctrl->saved_pm[irq_idx]);
+		rzt2h_pinctrl_set_gpio_en(pctrl, port, pin, true);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void rzt2h_gpio_irq_release_resources(struct irq_data *d)
+{
+	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct rzt2h_pinctrl *pctrl = gpiochip_get_data(gc);
+	irq_hw_number_t hwirq = irqd_to_hwirq(d);
+	u8 port = RZT2H_PIN_ID_TO_PORT(hwirq);
+	u8 pin = RZT2H_PIN_ID_TO_PIN(hwirq);
+	u8 parent_irq, irq_idx;
+
+	parent_irq = rzt2h_gpio_irq_map[hwirq];
+	if (parent_irq < RZT2H_INTERRUPTS_START)
+		return;
+
+	irq_idx = parent_irq - RZT2H_INTERRUPTS_START;
+	if (test_and_clear_bit(irq_idx, pctrl->used_irqs)) {
+		rzt2h_pin_write_pm(pctrl, port, pin, pctrl->saved_pm[irq_idx]);
+		rzt2h_pinctrl_set_gpio_en(pctrl, port, pin, true);
+	}
+
+	gpiochip_unlock_as_irq(gc, hwirq);
+}
+
 static const struct irq_chip rzt2h_gpio_irqchip = {
 	.name = "rzt2h-gpio",
 	.irq_disable = rzt2h_gpio_irq_disable,
@@ -1009,9 +1096,10 @@ static const struct irq_chip rzt2h_gpio_irqchip = {
 	.irq_set_type = irq_chip_set_type_parent,
 	.irq_set_wake = rzt2h_gpio_irq_set_wake,
 	.irq_eoi = irq_chip_eoi_parent,
+	.irq_request_resources = rzt2h_gpio_irq_request_resources,
+	.irq_release_resources = rzt2h_gpio_irq_release_resources,
 	.irq_set_affinity = irq_chip_set_affinity_parent,
 	.flags = IRQCHIP_IMMUTABLE,
-	GPIOCHIP_IRQ_RESOURCE_HELPERS,
 };
 
 static int rzt2h_gpio_child_to_parent_hwirq(struct gpio_chip *gc,
@@ -1020,43 +1108,17 @@ static int rzt2h_gpio_child_to_parent_hwirq(struct gpio_chip *gc,
 					    unsigned int *parent,
 					    unsigned int *parent_type)
 {
-	struct rzt2h_pinctrl *pctrl = gpiochip_get_data(gc);
-	u8 port = RZT2H_PIN_ID_TO_PORT(child);
-	u8 pin = RZT2H_PIN_ID_TO_PIN(child);
-	u8 parent_irq;
+	u8 parent_irq = rzt2h_gpio_irq_map[child];
 
-	parent_irq = rzt2h_gpio_irq_map[child];
 	if (parent_irq < RZT2H_INTERRUPTS_START)
 		return -EINVAL;
 
-	if (test_and_set_bit(parent_irq - RZT2H_INTERRUPTS_START,
-			     pctrl->used_irqs))
-		return -EBUSY;
-
-	rzt2h_pinctrl_set_pfc_mode(pctrl, port, pin, PFC_FUNC_INTERRUPT);
-
 	*parent = parent_irq;
 	*parent_type = child_type;
 
 	return 0;
 }
 
-static void rzt2h_gpio_irq_domain_free(struct irq_domain *domain, unsigned int virq,
-				       unsigned int nr_irqs)
-{
-	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
-	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
-	struct rzt2h_pinctrl *pctrl = container_of(gc, struct rzt2h_pinctrl, gpio_chip);
-	irq_hw_number_t hwirq = irqd_to_hwirq(d);
-	u8 port = RZT2H_PIN_ID_TO_PORT(hwirq);
-	u8 pin = RZT2H_PIN_ID_TO_PIN(hwirq);
-
-	if (test_and_clear_bit(hwirq - RZT2H_INTERRUPTS_START, pctrl->used_irqs))
-		rzt2h_pinctrl_set_gpio_en(pctrl, port, pin, false);
-
-	irq_domain_free_irqs_common(domain, virq, nr_irqs);
-}
-
 static void rzt2h_gpio_init_irq_valid_mask(struct gpio_chip *gc,
 					   unsigned long *valid_mask,
 					   unsigned int ngpios)
@@ -1072,7 +1134,6 @@ static void rzt2h_gpio_init_irq_valid_mask(struct gpio_chip *gc,
 
 static int rzt2h_gpio_register(struct rzt2h_pinctrl *pctrl)
 {
-	struct pinctrl_gpio_range *range = &pctrl->gpio_range;
 	struct gpio_chip *chip = &pctrl->gpio_chip;
 	struct device_node *np = pctrl->dev->of_node;
 	struct irq_domain *parent_domain;
@@ -1113,6 +1174,7 @@ static int rzt2h_gpio_register(struct rzt2h_pinctrl *pctrl)
 	chip->direction_output = rzt2h_gpio_direction_output;
 	chip->get = rzt2h_gpio_get;
 	chip->set = rzt2h_gpio_set;
+	chip->set_config = gpiochip_generic_config;
 	chip->label = dev_name(dev);
 
 	if (of_property_present(np, "interrupt-controller")) {
@@ -1122,17 +1184,9 @@ static int rzt2h_gpio_register(struct rzt2h_pinctrl *pctrl)
 		girq->parent_domain = parent_domain;
 		girq->child_to_parent_hwirq = rzt2h_gpio_child_to_parent_hwirq;
 		girq->populate_parent_alloc_arg = gpiochip_populate_parent_fwspec_twocell;
-		girq->child_irq_domain_ops.free = rzt2h_gpio_irq_domain_free;
 		girq->init_valid_mask = rzt2h_gpio_init_irq_valid_mask;
 	}
 
-	range->id = 0;
-	range->pin_base = 0;
-	range->base = 0;
-	range->npins = chip->ngpio;
-	range->name = chip->label;
-	range->gc = chip;
-
 	ret = devm_gpiochip_add_data(dev, chip, pctrl);
 	if (ret)
 		return dev_err_probe(dev, ret, "gpiochip registration failed\n");
diff --git a/drivers/pinctrl/stm32/Kconfig b/drivers/pinctrl/stm32/Kconfig
index d6a17152301216..cc425bbfc996ef 100644
--- a/drivers/pinctrl/stm32/Kconfig
+++ b/drivers/pinctrl/stm32/Kconfig
@@ -55,7 +55,7 @@ config PINCTRL_STM32MP157
 config PINCTRL_STM32MP257
 	tristate "STMicroelectronics STM32MP257 pin control"
 	depends on OF && HAS_IOMEM
-	default MACH_STM32MP25 || (ARCH_STM32 && ARM64)
+	default ARCH_STM32 && ARM64
 	select PINCTRL_STM32
 
 config PINCTRL_STM32_HDP
diff --git a/drivers/pinctrl/sunplus/Kconfig b/drivers/pinctrl/sunplus/Kconfig
index 69f82590f6d245..aa9435d023d84f 100644
--- a/drivers/pinctrl/sunplus/Kconfig
+++ b/drivers/pinctrl/sunplus/Kconfig
@@ -4,8 +4,8 @@
 #
 
 config PINCTRL_SPPCTL
-	tristate "Sunplus SP7021 PinMux and GPIO driver"
-	depends on SOC_SP7021
+	bool "Sunplus SP7021 PinMux and GPIO driver"
+	depends on SOC_SP7021 || COMPILE_TEST
 	depends on OF && HAS_IOMEM
 	select GENERIC_PINCTRL_GROUPS
 	select GENERIC_PINMUX_FUNCTIONS
@@ -16,6 +16,4 @@ config PINCTRL_SPPCTL
 	help
 	  Say Y here to support Sunplus SP7021 pinmux controller.
 	  This driver requires the pinctrl framework.
-	  GPIO is provided by the same driver.
-	  To compile this driver as a module, choose M here.
-	  The module will be called sppinctrl.
+	  This driver also provides GPIO.
diff --git a/drivers/pinctrl/sunxi/Kconfig b/drivers/pinctrl/sunxi/Kconfig
index dc62eba96348ec..831bb1dc93738b 100644
--- a/drivers/pinctrl/sunxi/Kconfig
+++ b/drivers/pinctrl/sunxi/Kconfig
@@ -141,4 +141,9 @@ config PINCTRL_SUN55I_A523_R
 	default ARM64 && ARCH_SUNXI
 	select PINCTRL_SUNXI
 
+config PINCTRL_SUN60I_A733
+	bool "Support for the Allwinner A733 PIO"
+	default ARM64 && ARCH_SUNXI
+	select PINCTRL_SUNXI
+
 endif
diff --git a/drivers/pinctrl/sunxi/Makefile b/drivers/pinctrl/sunxi/Makefile
index 951b3f1e4b4f1c..3658c9d06b4980 100644
--- a/drivers/pinctrl/sunxi/Makefile
+++ b/drivers/pinctrl/sunxi/Makefile
@@ -29,5 +29,6 @@ obj-$(CONFIG_PINCTRL_SUN50I_H616)	+= pinctrl-sun50i-h616.o
 obj-$(CONFIG_PINCTRL_SUN50I_H616_R)	+= pinctrl-sun50i-h616-r.o
 obj-$(CONFIG_PINCTRL_SUN55I_A523)	+= pinctrl-sun55i-a523.o
 obj-$(CONFIG_PINCTRL_SUN55I_A523_R)	+= pinctrl-sun55i-a523-r.o
+obj-$(CONFIG_PINCTRL_SUN60I_A733)	+= pinctrl-sun60i-a733.o
 obj-$(CONFIG_PINCTRL_SUN9I_A80)		+= pinctrl-sun9i-a80.o
 obj-$(CONFIG_PINCTRL_SUN9I_A80_R)	+= pinctrl-sun9i-a80-r.o
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun20i-d1.c b/drivers/pinctrl/sunxi/pinctrl-sun20i-d1.c
index 8efe35b77af4d6..37a60e5d1163bb 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun20i-d1.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun20i-d1.c
@@ -821,7 +821,7 @@ static const struct sunxi_pinctrl_desc d1_pinctrl_data = {
 static int d1_pinctrl_probe(struct platform_device *pdev)
 {
 	return sunxi_pinctrl_init_with_flags(pdev, &d1_pinctrl_data,
-					     SUNXI_PINCTRL_NEW_REG_LAYOUT);
+					     SUNXI_PINCTRL_NCAT2_REG_LAYOUT);
 }
 
 static const struct of_device_id d1_pinctrl_match[] = {
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c
index 462aa1c4a5fa65..cffc1e53eef141 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c
@@ -26,7 +26,7 @@ static const u8 a523_r_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] =
 static struct sunxi_pinctrl_desc a523_r_pinctrl_data = {
 	.irq_banks = ARRAY_SIZE(a523_r_irq_bank_map),
 	.irq_bank_map = a523_r_irq_bank_map,
-	.io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL,
+	.io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV,
 	.pin_base = PL_BASE,
 };
 
@@ -35,11 +35,12 @@ static int a523_r_pinctrl_probe(struct platform_device *pdev)
 	return sunxi_pinctrl_dt_table_init(pdev, a523_r_nr_bank_pins,
 					   a523_r_irq_bank_muxes,
 					   &a523_r_pinctrl_data,
-					   SUNXI_PINCTRL_NEW_REG_LAYOUT);
+					   SUNXI_PINCTRL_NCAT2_REG_LAYOUT);
 }
 
 static const struct of_device_id a523_r_pinctrl_match[] = {
 	{ .compatible = "allwinner,sun55i-a523-r-pinctrl", },
+	{ .compatible = "allwinner,sun60i-a733-r-pinctrl", },
 	{}
 };
 
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c
index b6f78f1f30ac4a..001bd42afa3ef8 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c
@@ -26,7 +26,7 @@ static const u8 a523_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] =
 static struct sunxi_pinctrl_desc a523_pinctrl_data = {
 	.irq_banks = ARRAY_SIZE(a523_irq_bank_map),
 	.irq_bank_map = a523_irq_bank_map,
-	.io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL,
+	.io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV,
 };
 
 static int a523_pinctrl_probe(struct platform_device *pdev)
@@ -34,7 +34,7 @@ static int a523_pinctrl_probe(struct platform_device *pdev)
 	return sunxi_pinctrl_dt_table_init(pdev, a523_nr_bank_pins,
 					   a523_irq_bank_muxes,
 					   &a523_pinctrl_data,
-					   SUNXI_PINCTRL_NEW_REG_LAYOUT |
+					   SUNXI_PINCTRL_NCAT2_REG_LAYOUT |
 					   SUNXI_PINCTRL_ELEVEN_BANKS);
 }
 
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun60i-a733.c b/drivers/pinctrl/sunxi/pinctrl-sun60i-a733.c
new file mode 100644
index 00000000000000..ca82f08a3a65b3
--- /dev/null
+++ b/drivers/pinctrl/sunxi/pinctrl-sun60i-a733.c
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Allwinner A733 SoC pinctrl driver.
+ *
+ * Copyright (C) 2025 Arm Ltd.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-sunxi.h"
+
+static const u8 a733_nr_bank_pins[SUNXI_PINCTRL_MAX_BANKS] =
+/*	  PA  PB  PC  PD  PE  PF  PG  PH  PI  PJ  PK */
+	{  0, 11, 17, 24, 16,  7, 15, 20, 17, 28, 26 };
+
+static const u8 a733_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] =
+/*	  PA  PB  PC  PD  PE  PF  PG  PH  PI  PJ  PK */
+	{  0, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14};
+
+static struct sunxi_pinctrl_desc a733_pinctrl_data = {
+	.irq_banks = 10,
+	.irq_read_needs_mux = true,
+	.io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL,
+};
+
+static int a733_pinctrl_probe(struct platform_device *pdev)
+{
+	return sunxi_pinctrl_dt_table_init(pdev, a733_nr_bank_pins,
+					   a733_irq_bank_muxes,
+					   &a733_pinctrl_data,
+					   SUNXI_PINCTRL_NCAT3_REG_LAYOUT);
+}
+
+static const struct of_device_id a733_pinctrl_match[] = {
+	{ .compatible = "allwinner,sun60i-a733-pinctrl", },
+	{}
+};
+
+static struct platform_driver a733_pinctrl_driver = {
+	.probe	= a733_pinctrl_probe,
+	.driver	= {
+		.name		= "sun60i-a733-pinctrl",
+		.of_match_table	= a733_pinctrl_match,
+	},
+};
+builtin_platform_driver(a733_pinctrl_driver);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index 25489beeb31253..3ee5012463da37 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -65,9 +65,9 @@ static struct irq_chip sunxi_pinctrl_level_irq_chip;
  */
 static u32 sunxi_bank_offset(const struct sunxi_pinctrl *pctl, u32 pin)
 {
-	u32 offset = 0;
+	u32 offset = pctl->bank_offset;
 
-	if (pin >= PK_BASE) {
+	if (pin >= PK_BASE && (pctl->flags & SUNXI_PINCTRL_ELEVEN_BANKS)) {
 		pin -= PK_BASE;
 		offset = PIO_BANK_K_OFFSET;
 	}
@@ -102,7 +102,7 @@ static void sunxi_dlevel_reg(const struct sunxi_pinctrl *pctl,
 {
 	u32 offset = pin % PINS_PER_BANK * pctl->dlevel_field_width;
 
-	*reg   = sunxi_bank_offset(pctl, pin) + DLEVEL_REGS_OFFSET +
+	*reg   = sunxi_bank_offset(pctl, pin) + pctl->drv_regs_offset +
 		 offset / BITS_PER_TYPE(u32) * sizeof(u32);
 	*shift = offset % BITS_PER_TYPE(u32);
 	*mask  = (BIT(pctl->dlevel_field_width) - 1) << *shift;
@@ -728,6 +728,7 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl,
 {
 	unsigned short bank;
 	unsigned long flags;
+	bool inverted = false;
 	u32 val, reg;
 	int uV;
 
@@ -762,10 +763,14 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl,
 		else
 			val = 0xD; /* 3.3V */
 
-		reg = readl(pctl->membase + sunxi_grp_config_reg(pin));
+		reg = readl(pctl->membase + sunxi_grp_config_reg(pctl, pin));
 		reg &= ~IO_BIAS_MASK;
-		writel(reg | val, pctl->membase + sunxi_grp_config_reg(pin));
+		writel(reg | val, pctl->membase +
+		       sunxi_grp_config_reg(pctl, pin));
 		return 0;
+	case BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV:
+		inverted = true;
+		fallthrough;
 	case BIAS_VOLTAGE_PIO_POW_MODE_CTL:
 		val = uV > 1800000 && uV <= 2500000 ? BIT(bank) : 0;
 
@@ -780,6 +785,8 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl,
 		fallthrough;
 	case BIAS_VOLTAGE_PIO_POW_MODE_SEL:
 		val = uV <= 1800000 ? 1 : 0;
+		if (inverted)
+			val = !val;
 
 		raw_spin_lock_irqsave(&pctl->lock, flags);
 		reg = readl(pctl->membase + pctl->pow_mod_sel_offset);
@@ -837,6 +844,21 @@ static void sunxi_pmx_set(struct pinctrl_dev *pctldev,
 	writel((readl(pctl->membase + reg) & ~mask) | config << shift,
 	       pctl->membase + reg);
 
+	/*
+	 * A pin muxed to gpio_out directly through a pinmux node bypasses
+	 * sunxi_pinctrl_gpio_set() and drives whatever its output latch
+	 * holds.  Now that the pin is in output mode the data register
+	 * reads back the latch, so refresh the shadow to keep such pins
+	 * driving their pre-existing level.
+	 */
+	if (config == SUN4I_FUNC_OUTPUT) {
+		u32 *shadow = &pctl->dat_shadow[pin / PINS_PER_BANK];
+
+		sunxi_data_reg(pctl, pin, &reg, &shift, &mask);
+		*shadow = (*shadow & ~mask) |
+			  (readl(pctl->membase + reg) & mask);
+	}
+
 	raw_spin_unlock_irqrestore(&pctl->lock, flags);
 }
 
@@ -1017,21 +1039,29 @@ static int sunxi_pinctrl_gpio_set(struct gpio_chip *chip, unsigned int offset,
 				  int value)
 {
 	struct sunxi_pinctrl *pctl = gpiochip_get_data(chip);
-	u32 reg, shift, mask, val;
+	u32 *shadow = &pctl->dat_shadow[offset / PINS_PER_BANK];
+	u32 reg, shift, mask;
 	unsigned long flags;
 
 	sunxi_data_reg(pctl, offset, &reg, &shift, &mask);
 
 	raw_spin_lock_irqsave(&pctl->lock, flags);
 
-	val = readl(pctl->membase + reg);
-
+	/*
+	 * Reading the data register returns the pin level, not the output
+	 * latch, for pins muxed as inputs.  A read-modify-write based on
+	 * the register would therefore corrupt the latches of input-muxed
+	 * pins in the same bank (e.g. an emulated open-drain I2C line
+	 * released high), making them drive the wrong level once switched
+	 * to output.  Base the read-modify-write on a shadow copy of the
+	 * latches instead.
+	 */
 	if (value)
-		val |= mask;
+		*shadow |= mask;
 	else
-		val &= ~mask;
+		*shadow &= ~mask;
 
-	writel(val, pctl->membase + reg);
+	writel(*shadow, pctl->membase + reg);
 
 	raw_spin_unlock_irqrestore(&pctl->lock, flags);
 
@@ -1107,7 +1137,7 @@ static int sunxi_pinctrl_irq_request_resources(struct irq_data *d)
 	muxval = (readl(pctl->membase + reg) & mask) >> shift;
 
 	/* Change muxing to GPIO INPUT mode if at reset value */
-	if (pctl->flags & SUNXI_PINCTRL_NEW_REG_LAYOUT)
+	if (pctl->flags & SUNXI_PINCTRL_NCAT2_REG_LAYOUT)
 		disabled_mux = SUN4I_FUNC_DISABLED_NEW;
 	else
 		disabled_mux = SUN4I_FUNC_DISABLED_OLD;
@@ -1140,7 +1170,7 @@ static void sunxi_pinctrl_irq_release_resources(struct irq_data *d)
 static int sunxi_pinctrl_irq_set_type(struct irq_data *d, unsigned int type)
 {
 	struct sunxi_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	u32 reg = sunxi_irq_cfg_reg(pctl->desc, d->hwirq);
+	u32 reg = sunxi_irq_cfg_reg(pctl, d->hwirq);
 	u8 index = sunxi_irq_cfg_offset(d->hwirq);
 	unsigned long flags;
 	u32 regval;
@@ -1187,7 +1217,7 @@ static int sunxi_pinctrl_irq_set_type(struct irq_data *d, unsigned int type)
 static void sunxi_pinctrl_irq_ack(struct irq_data *d)
 {
 	struct sunxi_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	u32 status_reg = sunxi_irq_status_reg(pctl->desc, d->hwirq);
+	u32 status_reg = sunxi_irq_status_reg(pctl, d->hwirq);
 	u8 status_idx = sunxi_irq_status_offset(d->hwirq);
 
 	/* Clear the IRQ */
@@ -1197,7 +1227,7 @@ static void sunxi_pinctrl_irq_ack(struct irq_data *d)
 static void sunxi_pinctrl_irq_mask(struct irq_data *d)
 {
 	struct sunxi_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	u32 reg = sunxi_irq_ctrl_reg(pctl->desc, d->hwirq);
+	u32 reg = sunxi_irq_ctrl_reg(pctl, d->hwirq);
 	u8 idx = sunxi_irq_ctrl_offset(d->hwirq);
 	unsigned long flags;
 	u32 val;
@@ -1214,7 +1244,7 @@ static void sunxi_pinctrl_irq_mask(struct irq_data *d)
 static void sunxi_pinctrl_irq_unmask(struct irq_data *d)
 {
 	struct sunxi_pinctrl *pctl = irq_data_get_irq_chip_data(d);
-	u32 reg = sunxi_irq_ctrl_reg(pctl->desc, d->hwirq);
+	u32 reg = sunxi_irq_ctrl_reg(pctl, d->hwirq);
 	u8 idx = sunxi_irq_ctrl_offset(d->hwirq);
 	unsigned long flags;
 	u32 val;
@@ -1318,7 +1348,7 @@ static void sunxi_pinctrl_irq_handler(struct irq_desc *desc)
 
 	chained_irq_enter(chip, desc);
 
-	reg = sunxi_irq_status_reg_from_bank(pctl->desc, bank);
+	reg = sunxi_irq_status_reg_from_bank(pctl, bank);
 	val = readl(pctl->membase + reg);
 
 	if (val) {
@@ -1557,7 +1587,7 @@ static int sunxi_pinctrl_setup_debounce(struct sunxi_pinctrl *pctl,
 
 		writel(src | div << 4,
 		       pctl->membase +
-		       sunxi_irq_debounce_reg_from_bank(pctl->desc, i));
+		       sunxi_irq_debounce_reg_from_bank(pctl, i));
 	}
 
 	return 0;
@@ -1572,7 +1602,7 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev,
 	struct pinctrl_pin_desc *pins;
 	struct sunxi_pinctrl *pctl;
 	struct pinmux_ops *pmxops;
-	int i, ret, last_pin, pin_idx;
+	int i, ret, last_pin, pin_idx, nbanks;
 	struct clk *clk;
 
 	pctl = devm_kzalloc(&pdev->dev, sizeof(*pctl), GFP_KERNEL);
@@ -1589,17 +1619,27 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev,
 	pctl->dev = &pdev->dev;
 	pctl->desc = desc;
 	pctl->flags = flags;
-	if (flags & SUNXI_PINCTRL_NEW_REG_LAYOUT) {
+	if (flags & SUNXI_PINCTRL_NCAT2_REG_LAYOUT) {
 		pctl->bank_mem_size = D1_BANK_MEM_SIZE;
+		pctl->drv_regs_offset = DLEVEL_REGS_OFFSET;
 		pctl->pull_regs_offset = D1_PULL_REGS_OFFSET;
 		pctl->dlevel_field_width = D1_DLEVEL_FIELD_WIDTH;
+	} else if (flags & SUNXI_PINCTRL_NCAT3_REG_LAYOUT) {
+		pctl->bank_mem_size = A733_BANK_MEM_SIZE;
+		pctl->bank_offset = A733_BANK_OFFSET;
+		pctl->drv_regs_offset = A733_DLEVEL_REGS_OFFSET;
+		pctl->pull_regs_offset = A733_PULL_REGS_OFFSET;
+		pctl->dlevel_field_width = D1_DLEVEL_FIELD_WIDTH;
 	} else {
 		pctl->bank_mem_size = BANK_MEM_SIZE;
+		pctl->drv_regs_offset = DLEVEL_REGS_OFFSET;
 		pctl->pull_regs_offset = PULL_REGS_OFFSET;
 		pctl->dlevel_field_width = DLEVEL_FIELD_WIDTH;
 	}
 	if (flags & SUNXI_PINCTRL_ELEVEN_BANKS)
 		pctl->pow_mod_sel_offset = PIO_11B_POW_MOD_SEL_REG;
+	else if (flags & SUNXI_PINCTRL_NCAT3_REG_LAYOUT)
+		pctl->pow_mod_sel_offset = PIO_NCAT3_POW_MOD_SEL_REG;
 	else
 		pctl->pow_mod_sel_offset = PIO_POW_MOD_SEL_REG;
 
@@ -1610,6 +1650,37 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev,
 	if (!pctl->irq_array)
 		return -ENOMEM;
 
+	/*
+	 * The bus clock has to be enabled before the pinctrl device
+	 * registers, as the pin hogs claimed from there access registers.
+	 */
+	ret = of_clk_get_parent_count(node);
+	clk = devm_clk_get_enabled(&pdev->dev, ret == 1 ? NULL : "apb");
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	/*
+	 * Seed the output latch shadow from the hardware so pins the
+	 * bootloader left in output mode keep their state; see
+	 * sunxi_pinctrl_gpio_set() for why a shadow is needed.  This must
+	 * happen before the pinctrl device registers, as pin hogs can mux
+	 * pins to gpio_out and thereby update the shadow.
+	 */
+	last_pin = pctl->desc->pins[pctl->desc->npins - 1].pin.number;
+	nbanks = DIV_ROUND_UP(last_pin + 1 - pctl->desc->pin_base,
+			      PINS_PER_BANK);
+	pctl->dat_shadow = devm_kcalloc(&pdev->dev, nbanks,
+					sizeof(*pctl->dat_shadow), GFP_KERNEL);
+	if (!pctl->dat_shadow)
+		return -ENOMEM;
+
+	for (i = 0; i < nbanks; i++) {
+		u32 reg, shift, mask;
+
+		sunxi_data_reg(pctl, i * PINS_PER_BANK, &reg, &shift, &mask);
+		pctl->dat_shadow[i] = readl(pctl->membase + reg);
+	}
+
 	ret = sunxi_pinctrl_build_state(pdev);
 	if (ret) {
 		dev_err(&pdev->dev, "dt probe failed: %d\n", ret);
@@ -1665,7 +1736,6 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev,
 	if (!pctl->chip)
 		return -ENOMEM;
 
-	last_pin = pctl->desc->pins[pctl->desc->npins - 1].pin.number;
 	pctl->chip->owner = THIS_MODULE;
 	pctl->chip->request = gpiochip_generic_request;
 	pctl->chip->free = gpiochip_generic_free;
@@ -1699,13 +1769,6 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev,
 			goto gpiochip_error;
 	}
 
-	ret = of_clk_get_parent_count(node);
-	clk = devm_clk_get_enabled(&pdev->dev, ret == 1 ? NULL : "apb");
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
-		goto gpiochip_error;
-	}
-
 	pctl->irq = devm_kcalloc(&pdev->dev,
 				 pctl->desc->irq_banks,
 				 sizeof(*pctl->irq),
@@ -1745,10 +1808,10 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev,
 	for (i = 0; i < pctl->desc->irq_banks; i++) {
 		/* Mask and clear all IRQs before registering a handler */
 		writel(0, pctl->membase +
-			  sunxi_irq_ctrl_reg_from_bank(pctl->desc, i));
+			  sunxi_irq_ctrl_reg_from_bank(pctl, i));
 		writel(0xffffffff,
 		       pctl->membase +
-		       sunxi_irq_status_reg_from_bank(pctl->desc, i));
+		       sunxi_irq_status_reg_from_bank(pctl, i));
 
 		irq_set_chained_handler_and_data(pctl->irq[i],
 						 sunxi_pinctrl_irq_handler,
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.h b/drivers/pinctrl/sunxi/pinctrl-sunxi.h
index 0daf7600e2fb01..7c2f29e75519a2 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.h
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.h
@@ -53,24 +53,31 @@
 #define D1_DLEVEL_FIELD_WIDTH	4
 #define D1_PULL_REGS_OFFSET	0x24
 
+#define A733_BANK_MEM_SIZE	0x80
+#define A733_BANK_OFFSET	0x80
+#define A733_DLEVEL_REGS_OFFSET	0x20
+#define A733_PULL_REGS_OFFSET	0x30
+#define A733_IRQ_REGS_OFFSET	0x40
+
 #define PINS_PER_BANK		32
 
 #define IRQ_PER_BANK		32
 
-#define IRQ_CFG_REG		0x200
+#define IRQ_REGS_OFFSET		0x200
+#define IRQ_CFG_REG_OFFSET	0x00
 #define IRQ_CFG_IRQ_PER_REG		8
 #define IRQ_CFG_IRQ_BITS		4
 #define IRQ_CFG_IRQ_MASK		((1 << IRQ_CFG_IRQ_BITS) - 1)
-#define IRQ_CTRL_REG		0x210
+#define IRQ_CTRL_REG_OFFSET	0x10
 #define IRQ_CTRL_IRQ_PER_REG		32
 #define IRQ_CTRL_IRQ_BITS		1
 #define IRQ_CTRL_IRQ_MASK		((1 << IRQ_CTRL_IRQ_BITS) - 1)
-#define IRQ_STATUS_REG		0x214
+#define IRQ_STATUS_REG_OFFSET	0x14
 #define IRQ_STATUS_IRQ_PER_REG		32
 #define IRQ_STATUS_IRQ_BITS		1
 #define IRQ_STATUS_IRQ_MASK		((1 << IRQ_STATUS_IRQ_BITS) - 1)
 
-#define IRQ_DEBOUNCE_REG	0x218
+#define IRQ_DEBOUNCE_REG_OFFSET	0x18
 
 #define IRQ_MEM_SIZE		0x20
 
@@ -85,15 +92,18 @@
 #define IO_BIAS_MASK		GENMASK(3, 0)
 
 #define SUN4I_FUNC_INPUT	0
+#define SUN4I_FUNC_OUTPUT	1
 #define SUN4I_FUNC_IRQ		6
 #define SUN4I_FUNC_DISABLED_OLD 7
 #define SUN4I_FUNC_DISABLED_NEW 15
 
 #define SUNXI_PINCTRL_VARIANT_MASK	GENMASK(7, 0)
-#define SUNXI_PINCTRL_NEW_REG_LAYOUT	BIT(8)
+#define SUNXI_PINCTRL_NCAT2_REG_LAYOUT	BIT(8)
 #define SUNXI_PINCTRL_PORTF_SWITCH	BIT(9)
 #define SUNXI_PINCTRL_ELEVEN_BANKS	BIT(10)
+#define SUNXI_PINCTRL_NCAT3_REG_LAYOUT	BIT(11)
 
+#define PIO_NCAT3_POW_MOD_SEL_REG	0x040
 #define PIO_POW_MOD_SEL_REG		0x340
 #define PIO_11B_POW_MOD_SEL_REG		0x380
 #define PIO_POW_MOD_CTL_OFS		0x004
@@ -116,8 +126,10 @@ enum sunxi_desc_bias_voltage {
 	 * Bias voltage is set through PIO_POW_MOD_SEL_REG
 	 * and PIO_POW_MOD_CTL_REG register, as seen on
 	 * A100 and D1 SoC, for example.
+	 * Some SoCs invert the encoding for 1.8V vs. 3.3V.
 	 */
 	BIAS_VOLTAGE_PIO_POW_MODE_CTL,
+	BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV,
 };
 
 struct sunxi_desc_function {
@@ -175,9 +187,17 @@ struct sunxi_pinctrl {
 	int				*irq;
 	unsigned			*irq_array;
 	raw_spinlock_t			lock;
+	/*
+	 * Output latch shadow, one word per bank.  Seeded lockless at
+	 * probe before the pinctrl device registers, protected by @lock
+	 * afterwards.
+	 */
+	u32				*dat_shadow;
 	struct pinctrl_dev		*pctl_dev;
 	unsigned long			flags;
 	u32				bank_mem_size;
+	u32				bank_offset;
+	u32				drv_regs_offset;
 	u32				pull_regs_offset;
 	u32				dlevel_field_width;
 	u32				pow_mod_sel_offset;
@@ -226,7 +246,8 @@ struct sunxi_pinctrl {
 		.irqnum = _irq,					\
 	}
 
-static inline u32 sunxi_irq_hw_bank_num(const struct sunxi_pinctrl_desc *desc, u8 bank)
+static inline u32 sunxi_irq_hw_bank_num(const struct sunxi_pinctrl_desc *desc,
+					u8 bank)
 {
 	if (!desc->irq_bank_map)
 		return bank;
@@ -234,68 +255,81 @@ static inline u32 sunxi_irq_hw_bank_num(const struct sunxi_pinctrl_desc *desc, u
 		return desc->irq_bank_map[bank];
 }
 
-static inline u32 sunxi_irq_cfg_reg(const struct sunxi_pinctrl_desc *desc,
+static inline u32 sunxi_irq_base_reg(const struct sunxi_pinctrl *pctl, u16 bank)
+{
+	if (pctl->flags & SUNXI_PINCTRL_NCAT3_REG_LAYOUT)
+		return pctl->bank_offset + bank * pctl->bank_mem_size +
+		       A733_IRQ_REGS_OFFSET;
+
+	return IRQ_REGS_OFFSET +
+	       sunxi_irq_hw_bank_num(pctl->desc, bank) * IRQ_MEM_SIZE;
+}
+
+static inline u32 sunxi_irq_cfg_reg(const struct sunxi_pinctrl *pctl,
 				    u16 irq)
 {
 	u8 bank = irq / IRQ_PER_BANK;
 	u8 reg = (irq % IRQ_PER_BANK) / IRQ_CFG_IRQ_PER_REG * 0x04;
 
-	return IRQ_CFG_REG +
-	       sunxi_irq_hw_bank_num(desc, bank) * IRQ_MEM_SIZE + reg;
+	return sunxi_irq_base_reg(pctl, bank) + IRQ_CFG_REG_OFFSET + reg;
 }
 
 static inline u32 sunxi_irq_cfg_offset(u16 irq)
 {
 	u32 irq_num = irq % IRQ_CFG_IRQ_PER_REG;
+
 	return irq_num * IRQ_CFG_IRQ_BITS;
 }
 
-static inline u32 sunxi_irq_ctrl_reg_from_bank(const struct sunxi_pinctrl_desc *desc, u8 bank)
+static inline u32 sunxi_irq_ctrl_reg_from_bank(const struct sunxi_pinctrl *pctl,
+					       u8 bank)
 {
-	return IRQ_CTRL_REG + sunxi_irq_hw_bank_num(desc, bank) * IRQ_MEM_SIZE;
+	return sunxi_irq_base_reg(pctl, bank) + IRQ_CTRL_REG_OFFSET;
 }
 
-static inline u32 sunxi_irq_ctrl_reg(const struct sunxi_pinctrl_desc *desc,
-				     u16 irq)
+static inline u32 sunxi_irq_ctrl_reg(const struct sunxi_pinctrl *pctl, u16 irq)
 {
 	u8 bank = irq / IRQ_PER_BANK;
 
-	return sunxi_irq_ctrl_reg_from_bank(desc, bank);
+	return sunxi_irq_ctrl_reg_from_bank(pctl, bank);
 }
 
 static inline u32 sunxi_irq_ctrl_offset(u16 irq)
 {
 	u32 irq_num = irq % IRQ_CTRL_IRQ_PER_REG;
+
 	return irq_num * IRQ_CTRL_IRQ_BITS;
 }
 
-static inline u32 sunxi_irq_debounce_reg_from_bank(const struct sunxi_pinctrl_desc *desc, u8 bank)
+static inline
+u32 sunxi_irq_debounce_reg_from_bank(const struct sunxi_pinctrl *pctl, u8 bank)
 {
-	return IRQ_DEBOUNCE_REG +
-	       sunxi_irq_hw_bank_num(desc, bank) * IRQ_MEM_SIZE;
+	return sunxi_irq_base_reg(pctl, bank) + IRQ_DEBOUNCE_REG_OFFSET;
 }
 
-static inline u32 sunxi_irq_status_reg_from_bank(const struct sunxi_pinctrl_desc *desc, u8 bank)
+static inline
+u32 sunxi_irq_status_reg_from_bank(const struct sunxi_pinctrl *pctl, u8 bank)
 {
-	return IRQ_STATUS_REG +
-	       sunxi_irq_hw_bank_num(desc, bank) * IRQ_MEM_SIZE;
+	return sunxi_irq_base_reg(pctl, bank) + IRQ_STATUS_REG_OFFSET;
 }
 
-static inline u32 sunxi_irq_status_reg(const struct sunxi_pinctrl_desc *desc,
+static inline u32 sunxi_irq_status_reg(const struct sunxi_pinctrl *pctl,
 				       u16 irq)
 {
 	u8 bank = irq / IRQ_PER_BANK;
 
-	return sunxi_irq_status_reg_from_bank(desc, bank);
+	return sunxi_irq_status_reg_from_bank(pctl, bank);
 }
 
 static inline u32 sunxi_irq_status_offset(u16 irq)
 {
 	u32 irq_num = irq % IRQ_STATUS_IRQ_PER_REG;
+
 	return irq_num * IRQ_STATUS_IRQ_BITS;
 }
 
-static inline u32 sunxi_grp_config_reg(u16 pin)
+static inline u32 sunxi_grp_config_reg(const struct sunxi_pinctrl *pctl,
+				       u16 pin)
 {
 	u8 bank = pin / PINS_PER_BANK;
 
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra238.c b/drivers/pinctrl/tegra/pinctrl-tegra238.c
index ec482365f14f2c..40aba285944eb8 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra238.c
+++ b/drivers/pinctrl/tegra/pinctrl-tegra238.c
@@ -1744,57 +1744,57 @@ static const char * const tegra238_functions[] = {
 #define drive_sdmmc1_dat0_pu2		DRV_PINGROUP_ENTRY_Y(0x8034,	28,	2,	30,	2,	-1,	-1,	-1,	-1,	0)
 #define drive_ufs0_rst_n_pv1		DRV_PINGROUP_ENTRY_Y(0x11004,	12,	5,	24,	5,	-1,	-1,	-1,	-1,	0)
 #define drive_ufs0_ref_clk_pv0		DRV_PINGROUP_ENTRY_Y(0x1100c,	12,	5,	24,	5,	-1,	-1,	-1,	-1,	0)
-#define drive_batt_oc_paa4		    DRV_PINGROUP_ENTRY_Y(0x1024,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_bootv_ctl_n_paa0		DRV_PINGROUP_ENTRY_Y(0x102c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_vcomp_alert_paa2		DRV_PINGROUP_ENTRY_Y(0x105c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_hdmi_cec_pbb0		    DRV_PINGROUP_ENTRY_Y(0x1064,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_touch_clk_pdd3		DRV_PINGROUP_ENTRY_Y(0x106c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart3_rx_pcc6		    DRV_PINGROUP_ENTRY_Y(0x1074,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart3_tx_pcc5		    DRV_PINGROUP_ENTRY_Y(0x107c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_gen8_i2c_sda_pdd2		DRV_PINGROUP_ENTRY_Y(0x1084,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_gen8_i2c_scl_pdd1		DRV_PINGROUP_ENTRY_Y(0x108c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_spi2_mosi_pcc2		DRV_PINGROUP_ENTRY_Y(0x1094,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_gen2_i2c_scl_pcc7		DRV_PINGROUP_ENTRY_Y(0x109c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_spi2_cs0_pcc3		    DRV_PINGROUP_ENTRY_Y(0x10a4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_gen2_i2c_sda_pdd0		DRV_PINGROUP_ENTRY_Y(0x10ac,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_spi2_sck_pcc0		    DRV_PINGROUP_ENTRY_Y(0x10b4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_spi2_miso_pcc1		DRV_PINGROUP_ENTRY_Y(0x10bc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio49_pee2		DRV_PINGROUP_ENTRY_Y(0x10c4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio50_pee4		DRV_PINGROUP_ENTRY_Y(0x10cc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio82_pee3		DRV_PINGROUP_ENTRY_Y(0x10d4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio71_pff2		DRV_PINGROUP_ENTRY_Y(0x10dc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio76_pff7		DRV_PINGROUP_ENTRY_Y(0x10e4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio74_pff5		DRV_PINGROUP_ENTRY_Y(0x10ec,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio00_paa1		DRV_PINGROUP_ENTRY_Y(0x10f4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio19_pdd6		DRV_PINGROUP_ENTRY_Y(0x10fc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio86_phh3		DRV_PINGROUP_ENTRY_Y(0x1104,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio72_pff3		DRV_PINGROUP_ENTRY_Y(0x110c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio77_pgg0		DRV_PINGROUP_ENTRY_Y(0x1114,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio80_pff6		DRV_PINGROUP_ENTRY_Y(0x111c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio84_pgg1		DRV_PINGROUP_ENTRY_Y(0x1124,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio83_pee5		DRV_PINGROUP_ENTRY_Y(0x112c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio73_pff4		DRV_PINGROUP_ENTRY_Y(0x1134,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio70_pff1		DRV_PINGROUP_ENTRY_Y(0x113c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio04_paa5		DRV_PINGROUP_ENTRY_Y(0x1144,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio85_pgg6		DRV_PINGROUP_ENTRY_Y(0x114c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio69_pff0		DRV_PINGROUP_ENTRY_Y(0x1154,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio25_paa6		DRV_PINGROUP_ENTRY_Y(0x115c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_soc_gpio26_paa7		DRV_PINGROUP_ENTRY_Y(0x1164,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart5_tx_pgg7		    DRV_PINGROUP_ENTRY_Y(0x116c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart5_rx_phh0		    DRV_PINGROUP_ENTRY_Y(0x1174,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart2_tx_pgg2		    DRV_PINGROUP_ENTRY_Y(0x117c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart2_rx_pgg3		    DRV_PINGROUP_ENTRY_Y(0x1184,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart2_cts_pgg5		DRV_PINGROUP_ENTRY_Y(0x118c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart2_rts_pgg4		DRV_PINGROUP_ENTRY_Y(0x1194,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart5_cts_phh2		DRV_PINGROUP_ENTRY_Y(0x119c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_uart5_rts_phh1		DRV_PINGROUP_ENTRY_Y(0x11a4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_pwm7_pee1			    DRV_PINGROUP_ENTRY_Y(0x11ac,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_pwm2_pdd7			    DRV_PINGROUP_ENTRY_Y(0x11b4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_pwm3_pee0			    DRV_PINGROUP_ENTRY_Y(0x11bc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_pwm1_paa3			    DRV_PINGROUP_ENTRY_Y(0x11c4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_spi2_cs1_pcc4		    DRV_PINGROUP_ENTRY_Y(0x11cc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_dmic1_clk_pdd4		DRV_PINGROUP_ENTRY_Y(0x11d4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
-#define drive_dmic1_dat_pdd5		DRV_PINGROUP_ENTRY_Y(0x11dc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	1)
+#define drive_batt_oc_paa4		    DRV_PINGROUP_ENTRY_Y(0x1024,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_bootv_ctl_n_paa0		DRV_PINGROUP_ENTRY_Y(0x102c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_vcomp_alert_paa2		DRV_PINGROUP_ENTRY_Y(0x105c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_hdmi_cec_pbb0		    DRV_PINGROUP_ENTRY_Y(0x1064,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_touch_clk_pdd3		DRV_PINGROUP_ENTRY_Y(0x106c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart3_rx_pcc6		    DRV_PINGROUP_ENTRY_Y(0x1074,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart3_tx_pcc5		    DRV_PINGROUP_ENTRY_Y(0x107c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_gen8_i2c_sda_pdd2		DRV_PINGROUP_ENTRY_Y(0x1084,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_gen8_i2c_scl_pdd1		DRV_PINGROUP_ENTRY_Y(0x108c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_spi2_mosi_pcc2		DRV_PINGROUP_ENTRY_Y(0x1094,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_gen2_i2c_scl_pcc7		DRV_PINGROUP_ENTRY_Y(0x109c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_spi2_cs0_pcc3		    DRV_PINGROUP_ENTRY_Y(0x10a4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_gen2_i2c_sda_pdd0		DRV_PINGROUP_ENTRY_Y(0x10ac,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_spi2_sck_pcc0		    DRV_PINGROUP_ENTRY_Y(0x10b4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_spi2_miso_pcc1		DRV_PINGROUP_ENTRY_Y(0x10bc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio49_pee2		DRV_PINGROUP_ENTRY_Y(0x10c4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio50_pee4		DRV_PINGROUP_ENTRY_Y(0x10cc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio82_pee3		DRV_PINGROUP_ENTRY_Y(0x10d4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio71_pff2		DRV_PINGROUP_ENTRY_Y(0x10dc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio76_pff7		DRV_PINGROUP_ENTRY_Y(0x10e4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio74_pff5		DRV_PINGROUP_ENTRY_Y(0x10ec,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio00_paa1		DRV_PINGROUP_ENTRY_Y(0x10f4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio19_pdd6		DRV_PINGROUP_ENTRY_Y(0x10fc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio86_phh3		DRV_PINGROUP_ENTRY_Y(0x1104,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio72_pff3		DRV_PINGROUP_ENTRY_Y(0x110c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio77_pgg0		DRV_PINGROUP_ENTRY_Y(0x1114,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio80_pff6		DRV_PINGROUP_ENTRY_Y(0x111c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio84_pgg1		DRV_PINGROUP_ENTRY_Y(0x1124,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio83_pee5		DRV_PINGROUP_ENTRY_Y(0x112c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio73_pff4		DRV_PINGROUP_ENTRY_Y(0x1134,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio70_pff1		DRV_PINGROUP_ENTRY_Y(0x113c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio04_paa5		DRV_PINGROUP_ENTRY_Y(0x1144,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio85_pgg6		DRV_PINGROUP_ENTRY_Y(0x114c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio69_pff0		DRV_PINGROUP_ENTRY_Y(0x1154,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio25_paa6		DRV_PINGROUP_ENTRY_Y(0x115c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_soc_gpio26_paa7		DRV_PINGROUP_ENTRY_Y(0x1164,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart5_tx_pgg7		    DRV_PINGROUP_ENTRY_Y(0x116c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart5_rx_phh0		    DRV_PINGROUP_ENTRY_Y(0x1174,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart2_tx_pgg2		    DRV_PINGROUP_ENTRY_Y(0x117c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart2_rx_pgg3		    DRV_PINGROUP_ENTRY_Y(0x1184,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart2_cts_pgg5		DRV_PINGROUP_ENTRY_Y(0x118c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart2_rts_pgg4		DRV_PINGROUP_ENTRY_Y(0x1194,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart5_cts_phh2		DRV_PINGROUP_ENTRY_Y(0x119c,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_uart5_rts_phh1		DRV_PINGROUP_ENTRY_Y(0x11a4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_pwm7_pee1			    DRV_PINGROUP_ENTRY_Y(0x11ac,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_pwm2_pdd7			    DRV_PINGROUP_ENTRY_Y(0x11b4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_pwm3_pee0			    DRV_PINGROUP_ENTRY_Y(0x11bc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_pwm1_paa3			    DRV_PINGROUP_ENTRY_Y(0x11c4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_spi2_cs1_pcc4		    DRV_PINGROUP_ENTRY_Y(0x11cc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_dmic1_clk_pdd4		DRV_PINGROUP_ENTRY_Y(0x11d4,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
+#define drive_dmic1_dat_pdd5		DRV_PINGROUP_ENTRY_Y(0x11dc,	12,	5,	20,	5,	-1,	-1,	-1,	-1,	0)
 
 #define drive_sdmmc1_comp		DRV_PINGROUP_ENTRY_N
 
@@ -1961,57 +1961,57 @@ static const struct tegra_pingroup tegra238_groups[] = {
 };
 
 static const struct tegra_pingroup tegra238_aon_groups[] = {
-	PINGROUP(bootv_ctl_n_paa0,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1028,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio00_paa1,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10f0,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(vcomp_alert_paa2,	SOC_THERM_OC1,	RSVD1,			RSVD2,		RSVD3,		0x1058,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(pwm1_paa3,		GP_PWM1,	RSVD1,			RSVD2,		RSVD3,		0x11c0,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(batt_oc_paa4,		SOC_THERM_OC2,	RSVD1,			RSVD2,		RSVD3,		0x1020,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio04_paa5,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1140,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio25_paa6,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1158,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio26_paa7,	RSVD0,		SOC_THERM_OC3,		RSVD2,		RSVD3,		0x1160,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(hdmi_cec_pbb0,		HDMI_CEC,	RSVD1,			RSVD2,		RSVD3,		0x1060,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(spi2_sck_pcc0,		SPI2_SCK,	RSVD1,			RSVD2,		RSVD3,		0x10b0,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(spi2_miso_pcc1,	SPI2_DIN,	RSVD1,			RSVD2,		RSVD3,		0x10b8,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(spi2_mosi_pcc2,	SPI2_DOUT,	RSVD1,			RSVD2,		RSVD3,		0x1090,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(spi2_cs0_pcc3,		SPI2_CS0,	RSVD1,			RSVD2,		RSVD3,		0x10a0,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(spi2_cs1_pcc4,		SPI2_CS1,	RSVD1,			RSVD2,		RSVD3,		0x11c8,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart3_tx_pcc5,		UARTC_TXD,	RSVD1,			RSVD2,		RSVD3,		0x1078,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart3_rx_pcc6,		UARTC_RXD,	RSVD1,			RSVD2,		RSVD3,		0x1070,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(gen2_i2c_scl_pcc7,	I2C2_CLK,	RSVD1,			RSVD2,		RSVD3,		0x1098,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(gen2_i2c_sda_pdd0,	I2C2_DAT,	RSVD1,			RSVD2,		RSVD3,		0x10a8,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(gen8_i2c_scl_pdd1,	I2C8_CLK,	RSVD1,			RSVD2,		RSVD3,		0x1088,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(gen8_i2c_sda_pdd2,	I2C8_DAT,	RSVD1,			RSVD2,		RSVD3,		0x1080,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(touch_clk_pdd3,	GP_PWM4,	TOUCH_CLK,		RSVD2,		RSVD3,		0x1068,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(dmic1_clk_pdd4,	DMIC1_CLK,	RSVD1,			DMIC5_CLK,	RSVD3,		0x11d0,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(dmic1_dat_pdd5,	DMIC1_DAT,	RSVD1,			DMIC5_DAT,	RSVD3,		0x11d8,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio19_pdd6,	RSVD0,		WDT_RESET_OUTB,		RSVD2,		RSVD3,		0x10f8,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio49_pee2,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10c0,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio50_pee4,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10c8,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio82_pee3,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10d0,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio71_pff2,	PPC_MODE_1,	RSVD1,			RSVD2,		RSVD3,		0x10d8,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio76_pff7,	RSVD0,		RSVD1,			TSC_EDGE_OUT0,	TSC_EDGE_OUT0A,	0x10e0,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio74_pff5,	PPC_READY,	PPC_I2C_DAT,		RSVD2,		RSVD3,		0x10e8,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio86_phh3,	RSVD0,		SPI5_CS1,		TSC_EDGE_OUT3,	TSC_EDGE_OUT0D,	0x1100,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio72_pff3,	PPC_MODE_2,	RSVD1,			RSVD2,		RSVD3,		0x1108,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio77_pgg0,	RSVD0,		RSVD1,			TSC_EDGE_OUT1,	TSC_EDGE_OUT0B,	0x1110,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio80_pff6,	RSVD0,		PPC_RST_N,		RSVD2,		RSVD3,		0x1118,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio84_pgg1,	RSVD0,		RSVD1,			TSC_EDGE_OUT2,	TSC_EDGE_OUT0C,	0x1120,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio83_pee5,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1128,		1,	Y,	-1,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio73_pff4,	PPC_CC,		PPC_I2C_CLK,		RSVD2,		RSVD3,		0x1130,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio70_pff1,	PPC_MODE_0,	RSVD1,			RSVD2,		RSVD3,		0x1138,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio85_pgg6,	RSVD0,		SPI4_CS1,		RSVD2,		RSVD3,		0x1148,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(soc_gpio69_pff0,	PPC_INT_N,	RSVD1,			RSVD2,		RSVD3,		0x1150,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart5_tx_pgg7,		UARTE_TXD,	SPI5_SCK,		RSVD2,		RSVD3,		0x1168,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart5_rx_phh0,		UARTE_RXD,	SPI5_MISO,		RSVD2,		RSVD3,		0x1170,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart2_tx_pgg2,		UARTB_TXD,	SPI4_SCK,		RSVD2,		RSVD3,		0x1178,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart2_rx_pgg3,		UARTB_RXD,	SPI4_MISO,		RSVD2,		RSVD3,		0x1180,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart2_cts_pgg5,	UARTB_CTS,	SPI4_CS0,		RSVD2,		RSVD3,		0x1188,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart2_rts_pgg4,	UARTB_RTS,	SPI4_MOSI,		RSVD2,		RSVD3,		0x1190,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart5_cts_phh2,	UARTE_CTS,	SPI5_CS0,		RSVD2,		RSVD3,		0x1198,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(uart5_rts_phh1,	UARTE_RTS,	SPI5_MOSI,		RSVD2,		RSVD3,		0x11a0,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(pwm2_pdd7,		GP_PWM2,	LED_BLINK,		RSVD2,		RSVD3,		0x11b0,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(pwm3_pee0,		GP_PWM3,	RSVD1,			RSVD2,		RSVD3,		0x11b8,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
-	PINGROUP(pwm7_pee1,		GP_PWM7,	RSVD1,			RSVD2,		RSVD3,		0x11a8,		1,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(bootv_ctl_n_paa0,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1028,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio00_paa1,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10f0,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(vcomp_alert_paa2,	SOC_THERM_OC1,	RSVD1,			RSVD2,		RSVD3,		0x1058,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(pwm1_paa3,		GP_PWM1,	RSVD1,			RSVD2,		RSVD3,		0x11c0,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(batt_oc_paa4,		SOC_THERM_OC2,	RSVD1,			RSVD2,		RSVD3,		0x1020,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio04_paa5,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1140,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio25_paa6,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1158,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio26_paa7,	RSVD0,		SOC_THERM_OC3,		RSVD2,		RSVD3,		0x1160,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(hdmi_cec_pbb0,		HDMI_CEC,	RSVD1,			RSVD2,		RSVD3,		0x1060,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(spi2_sck_pcc0,		SPI2_SCK,	RSVD1,			RSVD2,		RSVD3,		0x10b0,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(spi2_miso_pcc1,	SPI2_DIN,	RSVD1,			RSVD2,		RSVD3,		0x10b8,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(spi2_mosi_pcc2,	SPI2_DOUT,	RSVD1,			RSVD2,		RSVD3,		0x1090,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(spi2_cs0_pcc3,		SPI2_CS0,	RSVD1,			RSVD2,		RSVD3,		0x10a0,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(spi2_cs1_pcc4,		SPI2_CS1,	RSVD1,			RSVD2,		RSVD3,		0x11c8,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart3_tx_pcc5,		UARTC_TXD,	RSVD1,			RSVD2,		RSVD3,		0x1078,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart3_rx_pcc6,		UARTC_RXD,	RSVD1,			RSVD2,		RSVD3,		0x1070,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(gen2_i2c_scl_pcc7,	I2C2_CLK,	RSVD1,			RSVD2,		RSVD3,		0x1098,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(gen2_i2c_sda_pdd0,	I2C2_DAT,	RSVD1,			RSVD2,		RSVD3,		0x10a8,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(gen8_i2c_scl_pdd1,	I2C8_CLK,	RSVD1,			RSVD2,		RSVD3,		0x1088,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(gen8_i2c_sda_pdd2,	I2C8_DAT,	RSVD1,			RSVD2,		RSVD3,		0x1080,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(touch_clk_pdd3,	GP_PWM4,	TOUCH_CLK,		RSVD2,		RSVD3,		0x1068,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(dmic1_clk_pdd4,	DMIC1_CLK,	RSVD1,			DMIC5_CLK,	RSVD3,		0x11d0,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(dmic1_dat_pdd5,	DMIC1_DAT,	RSVD1,			DMIC5_DAT,	RSVD3,		0x11d8,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio19_pdd6,	RSVD0,		WDT_RESET_OUTB,		RSVD2,		RSVD3,		0x10f8,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio49_pee2,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10c0,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio50_pee4,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10c8,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio82_pee3,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x10d0,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio71_pff2,	PPC_MODE_1,	RSVD1,			RSVD2,		RSVD3,		0x10d8,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio76_pff7,	RSVD0,		RSVD1,			TSC_EDGE_OUT0,	TSC_EDGE_OUT0A,	0x10e0,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio74_pff5,	PPC_READY,	PPC_I2C_DAT,		RSVD2,		RSVD3,		0x10e8,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio86_phh3,	RSVD0,		SPI5_CS1,		TSC_EDGE_OUT3,	TSC_EDGE_OUT0D,	0x1100,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio72_pff3,	PPC_MODE_2,	RSVD1,			RSVD2,		RSVD3,		0x1108,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio77_pgg0,	RSVD0,		RSVD1,			TSC_EDGE_OUT1,	TSC_EDGE_OUT0B,	0x1110,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio80_pff6,	RSVD0,		PPC_RST_N,		RSVD2,		RSVD3,		0x1118,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio84_pgg1,	RSVD0,		RSVD1,			TSC_EDGE_OUT2,	TSC_EDGE_OUT0C,	0x1120,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio83_pee5,	RSVD0,		RSVD1,			RSVD2,		RSVD3,		0x1128,		0,	Y,	-1,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio73_pff4,	PPC_CC,		PPC_I2C_CLK,		RSVD2,		RSVD3,		0x1130,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio70_pff1,	PPC_MODE_0,	RSVD1,			RSVD2,		RSVD3,		0x1138,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio85_pgg6,	RSVD0,		SPI4_CS1,		RSVD2,		RSVD3,		0x1148,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(soc_gpio69_pff0,	PPC_INT_N,	RSVD1,			RSVD2,		RSVD3,		0x1150,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart5_tx_pgg7,		UARTE_TXD,	SPI5_SCK,		RSVD2,		RSVD3,		0x1168,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart5_rx_phh0,		UARTE_RXD,	SPI5_MISO,		RSVD2,		RSVD3,		0x1170,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart2_tx_pgg2,		UARTB_TXD,	SPI4_SCK,		RSVD2,		RSVD3,		0x1178,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart2_rx_pgg3,		UARTB_RXD,	SPI4_MISO,		RSVD2,		RSVD3,		0x1180,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart2_cts_pgg5,	UARTB_CTS,	SPI4_CS0,		RSVD2,		RSVD3,		0x1188,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart2_rts_pgg4,	UARTB_RTS,	SPI4_MOSI,		RSVD2,		RSVD3,		0x1190,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart5_cts_phh2,	UARTE_CTS,	SPI5_CS0,		RSVD2,		RSVD3,		0x1198,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(uart5_rts_phh1,	UARTE_RTS,	SPI5_MOSI,		RSVD2,		RSVD3,		0x11a0,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(pwm2_pdd7,		GP_PWM2,	LED_BLINK,		RSVD2,		RSVD3,		0x11b0,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(pwm3_pee0,		GP_PWM3,	RSVD1,			RSVD2,		RSVD3,		0x11b8,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
+	PINGROUP(pwm7_pee1,		GP_PWM7,	RSVD1,			RSVD2,		RSVD3,		0x11a8,		0,	Y,	5,	7,	6,	8,	-1,	10,	12),
 };
 
 static const struct tegra_pinctrl_soc_data tegra238_pinctrl_aon = {
diff --git a/drivers/platform/arm64/acer-aspire1-ec.c b/drivers/platform/arm64/acer-aspire1-ec.c
index 08d0b155a197af..501c62a096f81b 100644
--- a/drivers/platform/arm64/acer-aspire1-ec.c
+++ b/drivers/platform/arm64/acer-aspire1-ec.c
@@ -514,7 +514,7 @@ static int aspire_ec_probe(struct i2c_client *client)
 					aspire_ec_irq_handler, IRQF_ONESHOT,
 					dev_name(dev), ec);
 	if (ret)
-		return dev_err_probe(dev, ret, "Failed to request irq\n");
+		return ret;
 
 	return 0;
 }
diff --git a/drivers/platform/arm64/huawei-gaokun-ec.c b/drivers/platform/arm64/huawei-gaokun-ec.c
index 80a8ba8b8dda16..3e082cc07cdd7d 100644
--- a/drivers/platform/arm64/huawei-gaokun-ec.c
+++ b/drivers/platform/arm64/huawei-gaokun-ec.c
@@ -12,6 +12,7 @@
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/input.h>
 #include <linux/notifier.h>
 #include <linux/module.h>
@@ -783,7 +784,7 @@ static int gaokun_ec_probe(struct i2c_client *client)
 					gaokun_ec_irq_handler, IRQF_ONESHOT,
 					dev_name(dev), ec);
 	if (ret)
-		return dev_err_probe(dev, ret, "Failed to request IRQ\n");
+		return ret;
 
 	ec->hwmon_dev = devm_hwmon_device_register_with_info(dev, "gaokun_ec_hwmon",
 							     ec, &gaokun_ec_hwmon_chip_info, NULL);
diff --git a/drivers/platform/arm64/lenovo-thinkpad-t14s.c b/drivers/platform/arm64/lenovo-thinkpad-t14s.c
index e7acb66b77f22e..7a875877c8f307 100644
--- a/drivers/platform/arm64/lenovo-thinkpad-t14s.c
+++ b/drivers/platform/arm64/lenovo-thinkpad-t14s.c
@@ -488,8 +488,14 @@ static irqreturn_t t14s_ec_irq_handler(int irq, void *data)
 	case T14S_EC_EVT_NONE:
 		break;
 	case T14S_EC_EVT_KEY_FN_SPACE:
+		/*
+		 * Firmware already cycles the keyboard backlight
+		 * (off / low / high). Notify userspace through the LED
+		 * class only. Emitting KEY_KBDILLUMTOGGLE as well makes
+		 * desktop environments toggle the LED a second time.
+		 */
 		t14s_kbd_bl_update(ec);
-		fallthrough;
+		break;
 	case T14S_EC_EVT_KEY_FN_F4:
 	case T14S_EC_EVT_KEY_FN_F7:
 	case T14S_EC_EVT_KEY_FN_4:
@@ -594,7 +600,7 @@ static int t14s_ec_probe(struct i2c_client *client)
 					t14s_ec_irq_handler,
 					IRQF_ONESHOT, dev_name(dev), ec);
 	if (ret < 0)
-		return dev_err_probe(dev, ret, "Failed to get IRQ\n");
+		return ret;
 
 	/*
 	 * Disable wakeup support by default, because the driver currently does
diff --git a/drivers/platform/arm64/lenovo-yoga-c630.c b/drivers/platform/arm64/lenovo-yoga-c630.c
index a8600a977fbc82..24f9463f64f10e 100644
--- a/drivers/platform/arm64/lenovo-yoga-c630.c
+++ b/drivers/platform/arm64/lenovo-yoga-c630.c
@@ -10,7 +10,7 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
-#include <linux/irqreturn.h>
+#include <linux/interrupt.h>
 #include <linux/lockdep.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
@@ -221,7 +221,7 @@ static int yoga_c630_ec_probe(struct i2c_client *client)
 					NULL, yoga_c630_ec_thread_intr,
 					IRQF_ONESHOT, "yoga_c630_ec", ec);
 	if (ret < 0)
-		return dev_err_probe(dev, ret, "unable to request irq\n");
+		return ret;
 
 	ret = yoga_c630_aux_init(dev, YOGA_C630_DEV_PSY, ec);
 	if (ret)
diff --git a/drivers/platform/chrome/cros_ec_ishtp.c b/drivers/platform/chrome/cros_ec_ishtp.c
index 3766cef81fe82e..b21a27ad376250 100644
--- a/drivers/platform/chrome/cros_ec_ishtp.c
+++ b/drivers/platform/chrome/cros_ec_ishtp.c
@@ -369,7 +369,7 @@ static void ish_event_cb(struct ishtp_cl_device *cl_device)
  * @cros_ish_cl: ISHTP client instance
  * @reset: true if called from reset handler
  *
- * This function complete the initializtion of the client.
+ * This function complete the initialization of the client.
  *
  * Return: 0 for success, negative error code for failure.
  */
diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c
index 724d1313f6b213..b81b37c7169a50 100644
--- a/drivers/platform/chrome/cros_ec_proto.c
+++ b/drivers/platform/chrome/cros_ec_proto.c
@@ -138,7 +138,8 @@ static int cros_ec_xfer_command(struct cros_ec_device *ec_dev, struct cros_ec_co
 	return ret;
 }
 
-static int cros_ec_wait_until_complete(struct cros_ec_device *ec_dev, uint32_t *result)
+static int cros_ec_wait_until_complete(struct cros_ec_device *ec_dev,
+				       struct cros_ec_command *orig_msg)
 {
 	DEFINE_RAW_FLEX(struct cros_ec_command, msg, data,
 			sizeof(struct ec_response_get_comms_status));
@@ -161,23 +162,41 @@ static int cros_ec_wait_until_complete(struct cros_ec_device *ec_dev, uint32_t *
 		if (ret < 0)
 			return ret;
 
-		*result = msg->result;
+		orig_msg->result = msg->result;
 		if (msg->result != EC_RES_SUCCESS)
 			return ret;
 
-		if (ret == 0) {
-			ret = -EPROTO;
-			break;
-		}
+		if (ret == 0)
+			return -EPROTO;
+
+		if (!(status->flags & EC_COMMS_STATUS_PROCESSING)) {
+			u32 orig_cmd, orig_outsize, orig_version;
+
+			/* If no response payload is expected, return 0. */
+			if (orig_msg->insize == 0)
+				return 0;
 
-		if (!(status->flags & EC_COMMS_STATUS_PROCESSING))
+			/*
+			 * Request the response using EC_CMD_RESEND_RESPONSE.
+			 * Restore the original message fields so it appears
+			 * to be a direct response to the original command.
+			 */
+			orig_cmd = orig_msg->command;
+			orig_outsize = orig_msg->outsize;
+			orig_version = orig_msg->version;
+
+			orig_msg->command = EC_CMD_RESEND_RESPONSE;
+			orig_msg->outsize = 0;
+			orig_msg->version = 0;
+			ret = cros_ec_xfer_command(ec_dev, orig_msg);
+			orig_msg->command = orig_cmd;
+			orig_msg->outsize = orig_outsize;
+			orig_msg->version = orig_version;
 			return ret;
+		}
 	}
 
-	if (i >= EC_COMMAND_RETRIES)
-		ret = -EAGAIN;
-
-	return ret;
+	return -EAGAIN;
 }
 
 static int cros_ec_send_command(struct cros_ec_device *ec_dev, struct cros_ec_command *msg)
@@ -185,7 +204,7 @@ static int cros_ec_send_command(struct cros_ec_device *ec_dev, struct cros_ec_co
 	int ret = cros_ec_xfer_command(ec_dev, msg);
 
 	if (msg->result == EC_RES_IN_PROGRESS)
-		ret = cros_ec_wait_until_complete(ec_dev, &msg->result);
+		ret = cros_ec_wait_until_complete(ec_dev, msg);
 
 	return ret;
 }
diff --git a/drivers/platform/chrome/cros_ec_proto_test.c b/drivers/platform/chrome/cros_ec_proto_test.c
index 3f281996a6863c..cbaaf98d6877e0 100644
--- a/drivers/platform/chrome/cros_ec_proto_test.c
+++ b/drivers/platform/chrome/cros_ec_proto_test.c
@@ -1744,7 +1744,7 @@ static void cros_ec_proto_test_cmd_xfer_protocol_v2_no_op(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, ret, -EIO);
 }
 
-static void cros_ec_proto_test_cmd_xfer_in_progress_normal(struct kunit *test)
+static void cros_ec_proto_test_cmd_xfer_in_progress_payload_0byte(struct kunit *test)
 {
 	struct cros_ec_proto_test_priv *priv = test->priv;
 	struct cros_ec_device *ec_dev = &priv->ec_dev;
@@ -1774,7 +1774,7 @@ static void cros_ec_proto_test_cmd_xfer_in_progress_normal(struct kunit *test)
 	}
 
 	ret = cros_ec_cmd_xfer(ec_dev, &msg);
-	KUNIT_EXPECT_EQ(test, ret, sizeof(struct ec_response_get_comms_status));
+	KUNIT_EXPECT_EQ(test, ret, 0);
 
 	KUNIT_EXPECT_EQ(test, msg.result, EC_RES_SUCCESS);
 
@@ -1799,6 +1799,156 @@ static void cros_ec_proto_test_cmd_xfer_in_progress_normal(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, cros_kunit_ec_pkt_xfer_mock_called, 2);
 }
 
+static void cros_ec_proto_test_cmd_xfer_in_progress_payload_4bytes(struct kunit *test)
+{
+	struct cros_ec_proto_test_priv *priv = test->priv;
+	struct cros_ec_device *ec_dev = &priv->ec_dev;
+	struct ec_xfer_mock *mock;
+	u8 resp_data[4] = {0x11, 0x22, 0x33, 0x44};
+	struct {
+		struct cros_ec_command msg;
+		u8 data[4];
+	} buf;
+	struct cros_ec_command *msg = &buf.msg;
+	int ret;
+
+	memset(&buf, 0, sizeof(buf));
+	msg->version = 1;
+	msg->command = 0x1234;
+	msg->insize = sizeof(buf.data);
+
+	ec_dev->max_response = sizeof(buf.data);
+	ec_dev->pkt_xfer = cros_kunit_ec_pkt_xfer_mock;
+
+	/* For initial command returning EC_RES_IN_PROGRESS. */
+	{
+		mock = cros_kunit_ec_xfer_mock_addx(test, 0, EC_RES_IN_PROGRESS, 0);
+		KUNIT_ASSERT_PTR_NE(test, mock, NULL);
+	}
+
+	/* For EC_CMD_GET_COMMS_STATUS status polling. */
+	{
+		struct ec_response_get_comms_status *data;
+
+		mock = cros_kunit_ec_xfer_mock_add(test, sizeof(*data));
+		KUNIT_ASSERT_PTR_NE(test, mock, NULL);
+
+		data = (struct ec_response_get_comms_status *)mock->o_data;
+		data->flags = 0;
+	}
+
+	/* For EC_CMD_RESEND_RESPONSE returning 4 bytes payload. */
+	{
+		mock = cros_kunit_ec_xfer_mock_add(test, sizeof(resp_data));
+		KUNIT_ASSERT_PTR_NE(test, mock, NULL);
+		memcpy(mock->o_data, resp_data, sizeof(resp_data));
+	}
+
+	ret = cros_ec_cmd_xfer(ec_dev, msg);
+	KUNIT_EXPECT_EQ(test, ret, 4);
+	KUNIT_EXPECT_EQ(test, msg->result, EC_RES_SUCCESS);
+	KUNIT_EXPECT_EQ(test, msg->command, 0x1234);
+	KUNIT_EXPECT_EQ(test, msg->insize, sizeof(buf.data));
+	KUNIT_EXPECT_EQ(test, msg->version, 1);
+	KUNIT_EXPECT_EQ(test, memcmp(msg->data, resp_data, sizeof(resp_data)), 0);
+
+	/* Verify mock sequence */
+	{
+		mock = cros_kunit_ec_xfer_mock_next();
+		KUNIT_EXPECT_PTR_NE(test, mock, NULL);
+
+		mock = cros_kunit_ec_xfer_mock_next();
+		KUNIT_EXPECT_PTR_NE(test, mock, NULL);
+		KUNIT_EXPECT_EQ(test, mock->msg.command, EC_CMD_GET_COMMS_STATUS);
+
+		mock = cros_kunit_ec_xfer_mock_next();
+		KUNIT_EXPECT_PTR_NE(test, mock, NULL);
+		KUNIT_EXPECT_EQ(test, mock->msg.command, EC_CMD_RESEND_RESPONSE);
+		KUNIT_EXPECT_EQ(test, mock->msg.outsize, 0);
+		KUNIT_EXPECT_EQ(test, mock->msg.version, 0);
+		KUNIT_EXPECT_EQ(test, mock->msg.insize, 4);
+	}
+
+	KUNIT_EXPECT_EQ(test, cros_kunit_ec_pkt_xfer_mock_called, 3);
+}
+
+static void cros_ec_proto_test_cmd_xfer_in_progress_payload_gt4bytes(struct kunit *test)
+{
+	struct cros_ec_proto_test_priv *priv = test->priv;
+	struct cros_ec_device *ec_dev = &priv->ec_dev;
+	struct ec_xfer_mock *mock;
+	u8 resp_data[16];
+	struct {
+		struct cros_ec_command msg;
+		u8 data[16];
+	} buf;
+	struct cros_ec_command *msg = &buf.msg;
+	int ret, i;
+
+	for (i = 0; i < sizeof(resp_data); ++i)
+		resp_data[i] = (u8)(i + 1);
+
+	memset(&buf, 0, sizeof(buf));
+	msg->version = 0;
+	msg->command = 0x5678;
+	msg->insize = sizeof(buf.data);
+
+	ec_dev->max_response = sizeof(buf.data);
+	ec_dev->pkt_xfer = cros_kunit_ec_pkt_xfer_mock;
+
+	/* For initial command returning EC_RES_IN_PROGRESS. */
+	{
+		mock = cros_kunit_ec_xfer_mock_addx(test, 0, EC_RES_IN_PROGRESS, 0);
+		KUNIT_ASSERT_PTR_NE(test, mock, NULL);
+	}
+
+	/* For EC_CMD_GET_COMMS_STATUS status polling. */
+	{
+		struct ec_response_get_comms_status *data;
+
+		mock = cros_kunit_ec_xfer_mock_add(test, sizeof(*data));
+		KUNIT_ASSERT_PTR_NE(test, mock, NULL);
+
+		data = (struct ec_response_get_comms_status *)mock->o_data;
+		data->flags = 0;
+	}
+
+	/* For EC_CMD_RESEND_RESPONSE returning 16 bytes payload. */
+	{
+		mock = cros_kunit_ec_xfer_mock_add(test, sizeof(resp_data));
+		KUNIT_ASSERT_PTR_NE(test, mock, NULL);
+		memcpy(mock->o_data, resp_data, sizeof(resp_data));
+	}
+
+	ret = cros_ec_cmd_xfer(ec_dev, msg);
+	KUNIT_EXPECT_EQ(test, ret, 16);
+	KUNIT_EXPECT_EQ(test, msg->result, EC_RES_SUCCESS);
+	KUNIT_EXPECT_EQ(test, msg->command, 0x5678);
+	KUNIT_EXPECT_EQ(test, msg->outsize, 0);
+	KUNIT_EXPECT_EQ(test, msg->insize, sizeof(buf.data));
+	KUNIT_EXPECT_EQ(test, msg->version, 0);
+	KUNIT_EXPECT_EQ(test, memcmp(msg->data, resp_data, sizeof(resp_data)), 0);
+
+	/* Verify mock sequence */
+	{
+		mock = cros_kunit_ec_xfer_mock_next();
+		KUNIT_EXPECT_PTR_NE(test, mock, NULL);
+
+		mock = cros_kunit_ec_xfer_mock_next();
+		KUNIT_EXPECT_PTR_NE(test, mock, NULL);
+		KUNIT_EXPECT_EQ(test, mock->msg.command, EC_CMD_GET_COMMS_STATUS);
+
+		mock = cros_kunit_ec_xfer_mock_next();
+		KUNIT_EXPECT_PTR_NE(test, mock, NULL);
+		KUNIT_EXPECT_EQ(test, mock->msg.command, EC_CMD_RESEND_RESPONSE);
+		KUNIT_EXPECT_EQ(test, mock->msg.outsize, 0);
+		KUNIT_EXPECT_EQ(test, mock->msg.version, 0);
+		KUNIT_EXPECT_EQ(test, mock->msg.insize, 16);
+	}
+
+	KUNIT_EXPECT_EQ(test, cros_kunit_ec_pkt_xfer_mock_called, 3);
+}
+
 static void cros_ec_proto_test_cmd_xfer_in_progress_retries_eagain(struct kunit *test)
 {
 	struct cros_ec_proto_test_priv *priv = test->priv;
@@ -2703,7 +2853,9 @@ static struct kunit_case cros_ec_proto_test_cases[] = {
 	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_protocol_v3_no_op),
 	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_protocol_v2_normal),
 	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_protocol_v2_no_op),
-	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_in_progress_normal),
+	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_in_progress_payload_0byte),
+	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_in_progress_payload_4bytes),
+	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_in_progress_payload_gt4bytes),
 	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_in_progress_retries_eagain),
 	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_in_progress_retries_status_processing),
 	KUNIT_CASE(cros_ec_proto_test_cmd_xfer_in_progress_xfer_error),
diff --git a/drivers/platform/chrome/cros_ec_rpmsg.c b/drivers/platform/chrome/cros_ec_rpmsg.c
index 09bd9e49464e62..7390c2c91ea5d7 100644
--- a/drivers/platform/chrome/cros_ec_rpmsg.c
+++ b/drivers/platform/chrome/cros_ec_rpmsg.c
@@ -20,7 +20,7 @@
 #define HOST_EVENT_MARK		2
 
 /**
- * struct cros_ec_rpmsg_response - rpmsg message format from from EC.
+ * struct cros_ec_rpmsg_response - rpmsg message format from EC.
  *
  * @type:	The type of message, should be either HOST_COMMAND_MARK or
  *		HOST_EVENT_MARK, representing that the message is a response to
diff --git a/drivers/platform/chrome/cros_usbpd_notify.c b/drivers/platform/chrome/cros_usbpd_notify.c
index d842c05f4db00b..6f5eea4938cb43 100644
--- a/drivers/platform/chrome/cros_usbpd_notify.c
+++ b/drivers/platform/chrome/cros_usbpd_notify.c
@@ -64,7 +64,7 @@ static void cros_usbpd_get_event_and_notify(struct device  *dev,
 
 	/*
 	 * We still send a 0 event out to older devices which don't
-	 * have the updated device heirarchy.
+	 * have the updated device hierarchy.
 	 */
 	if (!ec_dev) {
 		dev_dbg(dev,
@@ -115,7 +115,7 @@ static int cros_usbpd_notify_probe_acpi(struct platform_device *pdev)
 	if (!ec_dev) {
 		/*
 		 * We continue even for older devices which don't have the
-		 * correct device heirarchy, namely, GOOG0003 is a child
+		 * correct device hierarchy, namely, GOOG0003 is a child
 		 * of GOOG0004. If GOOG0003 is a child of GOOG0004 and we
 		 * can't get a pointer to the Chrome EC device, defer the
 		 * probe function.
diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c
index 3c6408581373b6..d67215df7371ed 100644
--- a/drivers/platform/mellanox/mlxbf-tmfifo.c
+++ b/drivers/platform/mellanox/mlxbf-tmfifo.c
@@ -1135,6 +1135,20 @@ static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev)
 	tm_vdev->status = 0;
 }
 
+/* Synchronize with callbacks running in the FIFO work item. */
+static void mlxbf_tmfifo_virtio_synchronize_cbs(struct virtio_device *vdev)
+{
+	struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
+	struct mlxbf_tmfifo *fifo = tm_vdev->vrings[0].fifo;
+	unsigned long flags;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(fifo->spin_lock); i++) {
+		spin_lock_irqsave(&fifo->spin_lock[i], flags);
+		spin_unlock_irqrestore(&fifo->spin_lock[i], flags);
+	}
+}
+
 /* Read the value of a configuration field. */
 static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev,
 				    unsigned int offset,
@@ -1179,6 +1193,7 @@ static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = {
 	.find_vqs = mlxbf_tmfifo_virtio_find_vqs,
 	.del_vqs = mlxbf_tmfifo_virtio_del_vqs,
 	.reset = mlxbf_tmfifo_virtio_reset,
+	.synchronize_cbs = mlxbf_tmfifo_virtio_synchronize_cbs,
 	.set_status = mlxbf_tmfifo_virtio_set_status,
 	.get_status = mlxbf_tmfifo_virtio_get_status,
 	.get = mlxbf_tmfifo_virtio_get,
@@ -1390,7 +1405,6 @@ static int mlxbf_tmfifo_probe(struct platform_device *pdev)
 				      mlxbf_tmfifo_irq_handler, 0,
 				      "tmfifo", &fifo->irq_info[i]);
 		if (rc) {
-			dev_err(dev, "devm_request_irq failed\n");
 			fifo->irq_info[i].irq = 0;
 			return rc;
 		}
diff --git a/drivers/platform/mellanox/mlxreg-hotplug.c b/drivers/platform/mellanox/mlxreg-hotplug.c
index d246772aafd65c..d6549dd2177265 100644
--- a/drivers/platform/mellanox/mlxreg-hotplug.c
+++ b/drivers/platform/mellanox/mlxreg-hotplug.c
@@ -740,10 +740,8 @@ static int mlxreg_hotplug_probe(struct platform_device *pdev)
 	err = devm_request_irq(&pdev->dev, priv->irq,
 			       mlxreg_hotplug_irq_handler, IRQF_TRIGGER_FALLING
 			       | IRQF_SHARED, "mlxreg-hotplug", priv);
-	if (err) {
-		dev_err(&pdev->dev, "Failed to request irq: %d\n", err);
+	if (err)
 		return err;
-	}
 
 	disable_irq(priv->irq);
 	spin_lock_init(&priv->lock);
diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c
index d37a7e4c6b02a6..061d4a567a0618 100644
--- a/drivers/platform/surface/surface_aggregator_registry.c
+++ b/drivers/platform/surface/surface_aggregator_registry.c
@@ -410,13 +410,14 @@ static const struct software_node *ssam_node_group_sp11[] = {
 	&ssam_node_hub_kip,
 	&ssam_node_bat_ac,
 	&ssam_node_bat_main,
+	&ssam_node_tmp_perf_profile_with_fan,
 	&ssam_node_tmp_sensors,
+	&ssam_node_fan_speed,
 	&ssam_node_hid_kip_keyboard,
 	&ssam_node_hid_kip_penstash,
 	&ssam_node_hid_kip_touchpad,
 	&ssam_node_hid_kip_fwupd,
-	&ssam_node_hid_sam_sensors,
-	&ssam_node_kip_tablet_switch,
+	&ssam_node_pos_tablet_switch,
 	NULL,
 };
 
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index 957034f39e4e7a..45f15f93afa729 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -189,6 +189,18 @@ config ACER_WMI
 	  If you have an ACPI-WMI compatible Acer/ Wistron laptop, say Y or M
 	  here.
 
+config ACER_WMI_BATTERY
+	tristate "Acer WMI Battery"
+	depends on ACPI_WMI
+	depends on ACPI_BATTERY
+	depends on DMI
+	help
+	  This is a driver for Acer laptops with battery health control. It
+	  adds charge limit control and battery temperature reporting.
+
+	  If you have an ACPI-WMI Battery compatible Acer laptop, say Y or M
+	  here.
+
 source "drivers/platform/x86/amd/Kconfig"
 
 config ADV_SWBUTTON
diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile
index 872ac3842391f7..a877acd937cd85 100644
--- a/drivers/platform/x86/Makefile
+++ b/drivers/platform/x86/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_BITLAND_MIFS_WMI)		+= bitland-mifs-wmi.o
 obj-$(CONFIG_ACERHDF)		+= acerhdf.o
 obj-$(CONFIG_ACER_WIRELESS)	+= acer-wireless.o
 obj-$(CONFIG_ACER_WMI)		+= acer-wmi.o
+obj-$(CONFIG_ACER_WMI_BATTERY)	+= acer-wmi-battery.o
 
 # AMD
 obj-y				+= amd/
diff --git a/drivers/platform/x86/acer-wmi-battery.c b/drivers/platform/x86/acer-wmi-battery.c
new file mode 100644
index 00000000000000..10a34d4ab0581e
--- /dev/null
+++ b/drivers/platform/x86/acer-wmi-battery.c
@@ -0,0 +1,450 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * acer-wmi-battery.c: Acer battery health control driver
+ *
+ * This is a driver for the WMI battery health control interface found
+ * on some Acer laptops.  This interface allows to enable/disable a
+ * battery charge limit ("health mode") and exposes the battery temperature.
+ *
+ * Based on acer-wmi-battery https://github.com/frederik-h/acer-wmi-battery/
+ *
+ * Copyright (C) 2022-2025  Frederik Harwath <frederik@harwath.name>
+ */
+
+#include <linux/acpi.h>
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/byteorder/generic.h>
+#include <linux/cleanup.h>
+#include <linux/compiler_attributes.h>
+#include <linux/dmi.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/limits.h>
+#include <linux/module.h>
+#include <linux/power_supply.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/units.h>
+#include <linux/wmi.h>
+
+#include <acpi/battery.h>
+
+#define DRIVER_NAME	"acer-wmi-battery"
+
+#define ACER_BATTERY_GUID "79772EC5-04B1-4BFD-843C-61E7F77B6CC9"
+
+/*
+ * The Acer OEM software seems to always use this battery index,
+ * so we emulate this behaviour to not confuse the underlying firmware.
+ *
+ * However this also means that we only fully support devices with a
+ * single battery for now.
+ */
+#define ACER_BATTERY_INDEX	0x1
+
+static bool force;
+module_param_unsafe(force, bool, 0);
+MODULE_PARM_DESC(force, "Enable health mode control without checking for supported devices\n");
+
+struct get_battery_health_control_status_input {
+	u8 battery_no;
+	u8 function_query;
+	u8 reserved[2];
+} __packed;
+
+struct get_battery_health_control_status_output {
+	u8 function_list;
+	u8 ret[2];
+	u8 function_status[5];
+} __packed;
+
+struct set_battery_health_control_input {
+	u8 battery_no;
+	u8 function_mask;
+	u8 function_status;
+	u8 reserved_in[5];
+} __packed;
+
+struct set_battery_health_control_output {
+	__le16 ret;
+	__le16 reserved_out;
+} __packed;
+
+enum battery_mode {
+	HEALTH_MODE = 1,
+	CALIBRATION_MODE = 2,
+};
+
+struct acer_wmi_battery_data {
+	struct acpi_battery_hook hook;
+	struct wmi_device *wdev;
+	const struct power_supply_ext *battery_ext;
+};
+
+static bool health_mode __ro_after_init;
+
+static int acer_wmi_battery_get_information(struct acer_wmi_battery_data *data,
+					    u32 index, u32 battery, u32 *result)
+{
+	u32 args[2] = { index, battery };
+	struct wmi_buffer input = { .length = sizeof(args), .data = args };
+	struct wmi_buffer output;
+	int ret;
+
+	ret = wmidev_invoke_method(data->wdev, 0, 19, &input, &output, sizeof(*result));
+	if (ret)
+		return ret;
+
+	*result = le32_to_cpup((__le32 *)output.data);
+
+	kfree(output.data);
+
+	return 0;
+}
+
+static int acer_wmi_battery_get_health_control_status(struct acer_wmi_battery_data *data,
+						      bool *health_mode)
+{
+	/*
+	 * Acer Care Center seems to always call the WMI method
+	 * with fixed parameters. This yields information about
+	 * the availability and state of both health and
+	 * calibration mode. The modes probably apply to
+	 * all batteries of the system.
+	 */
+	struct get_battery_health_control_status_input args = {
+		.battery_no = ACER_BATTERY_INDEX,
+		.function_query = 0x1,
+		.reserved = { 0x0, 0x0 },
+	};
+	struct wmi_buffer input = { .length = sizeof(args), .data = &args };
+	struct wmi_buffer output;
+	int ret;
+
+	ret = wmidev_invoke_method(data->wdev, 0, 20, &input, &output,
+				   sizeof(struct get_battery_health_control_status_output));
+	if (ret)
+		return ret;
+
+	struct get_battery_health_control_status_output *status_output __free(kfree) = output.data;
+
+	if (!(status_output->function_list & HEALTH_MODE))
+		return -EINVAL;
+
+	*health_mode = status_output->function_status[0] > 0;
+
+	return 0;
+}
+
+static int acer_wmi_battery_set_health_control(struct acer_wmi_battery_data *data,
+					       u8 function, bool function_status)
+{
+	struct set_battery_health_control_input args = {
+		.battery_no = ACER_BATTERY_INDEX,
+		.function_mask = function,
+		.function_status = function_status ? 1 : 0,
+		.reserved_in = { 0x0, 0x0, 0x0, 0x0, 0x0 },
+	};
+	struct wmi_buffer input = { .length = sizeof(args), .data = &args };
+	struct wmi_buffer output;
+	int ret;
+
+	ret = wmidev_invoke_method(data->wdev, 0, 21, &input, &output,
+				   sizeof(struct set_battery_health_control_output));
+	if (ret)
+		return ret;
+
+	struct set_battery_health_control_output *status_output __free(kfree) = output.data;
+	if (le16_to_cpu(status_output->ret) != 0)
+		return -EIO;
+
+	return 0;
+}
+
+static int acer_battery_ext_property_get(struct power_supply *psy,
+					 const struct power_supply_ext *ext,
+					 void *ext_data,
+					 enum power_supply_property psp,
+					 union power_supply_propval *val)
+{
+	struct acer_wmi_battery_data *data = ext_data;
+	bool health_mode;
+	u32 value;
+	int ret;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_CHARGE_TYPES:
+		ret = acer_wmi_battery_get_health_control_status(data, &health_mode);
+		if (ret)
+			return ret;
+
+		val->intval = health_mode ? POWER_SUPPLY_CHARGE_TYPE_LONGLIFE
+					  : POWER_SUPPLY_CHARGE_TYPE_STANDARD;
+		break;
+	case POWER_SUPPLY_PROP_TEMP:
+		ret = acer_wmi_battery_get_information(data, 0x8, ACER_BATTERY_INDEX, &value);
+		if (ret)
+			return ret;
+
+		if (value > U16_MAX)
+			return -ERANGE;
+
+		val->intval = value + ABSOLUTE_ZERO_MILLICELSIUS / MILLIDEGREE_PER_DECIDEGREE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int acer_battery_ext_property_set(struct power_supply *psy,
+					 const struct power_supply_ext *ext,
+					 void *ext_data,
+					 enum power_supply_property psp,
+					 const union power_supply_propval *val)
+{
+	struct acer_wmi_battery_data *data = ext_data;
+
+	switch (psp) {
+	case POWER_SUPPLY_PROP_CHARGE_TYPES:
+		return acer_wmi_battery_set_health_control(data, HEALTH_MODE,
+				val->intval == POWER_SUPPLY_CHARGE_TYPE_LONGLIFE);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int acer_battery_ext_property_is_writeable(struct power_supply *psy,
+						  const struct power_supply_ext *ext,
+						  void *ext_data,
+						  enum power_supply_property psp)
+{
+	switch (psp) {
+	case POWER_SUPPLY_PROP_CHARGE_TYPES:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const enum power_supply_property acer_battery_properties_v1[] = {
+	POWER_SUPPLY_PROP_TEMP,
+};
+
+static const enum power_supply_property acer_battery_properties_v2[] = {
+	POWER_SUPPLY_PROP_TEMP,
+	POWER_SUPPLY_PROP_CHARGE_TYPES,
+};
+
+static const struct power_supply_ext acer_wmi_battery_extension_v1 = {
+	.name			= DRIVER_NAME,
+	.properties		= acer_battery_properties_v1,
+	.num_properties		= ARRAY_SIZE(acer_battery_properties_v1),
+	.get_property		= acer_battery_ext_property_get,
+};
+
+static const struct power_supply_ext acer_wmi_battery_extension_v2 = {
+	.name			= DRIVER_NAME,
+	.properties		= acer_battery_properties_v2,
+	.num_properties		= ARRAY_SIZE(acer_battery_properties_v2),
+	.charge_types           = BIT(POWER_SUPPLY_CHARGE_TYPE_STANDARD) |
+				  BIT(POWER_SUPPLY_CHARGE_TYPE_LONGLIFE),
+	.get_property		= acer_battery_ext_property_get,
+	.set_property		= acer_battery_ext_property_set,
+	.property_is_writeable	= acer_battery_ext_property_is_writeable,
+};
+
+static int acer_battery_add(struct power_supply *battery, struct acpi_battery_hook *hook)
+{
+	struct acer_wmi_battery_data *data = container_of(hook, struct acer_wmi_battery_data, hook);
+
+	return power_supply_register_extension(battery, data->battery_ext,
+					       &data->wdev->dev, data);
+}
+
+static int acer_battery_remove(struct power_supply *battery, struct acpi_battery_hook *hook)
+{
+	struct acer_wmi_battery_data *data = container_of(hook, struct acer_wmi_battery_data, hook);
+
+	power_supply_unregister_extension(battery, data->battery_ext);
+
+	return 0;
+}
+
+static int acer_wmi_battery_probe(struct wmi_device *wdev, const void *context)
+{
+	struct acer_wmi_battery_data *data;
+
+	data = devm_kzalloc(&wdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	dev_set_drvdata(&wdev->dev, data);
+	data->wdev = wdev;
+	data->battery_ext = health_mode ? &acer_wmi_battery_extension_v2
+					: &acer_wmi_battery_extension_v1;
+	data->hook.name = "Acer Battery Extension";
+	data->hook.add_battery = acer_battery_add;
+	data->hook.remove_battery = acer_battery_remove;
+
+	return devm_battery_hook_register(&data->wdev->dev, &data->hook);
+}
+
+static const struct wmi_device_id acer_wmi_battery_id_table[] = {
+	{ ACER_BATTERY_GUID, NULL },
+	{ }
+};
+MODULE_DEVICE_TABLE(wmi, acer_wmi_battery_id_table);
+
+static struct wmi_driver acer_wmi_battery_driver = {
+	.driver = {
+		.name = DRIVER_NAME,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
+	.id_table = acer_wmi_battery_id_table,
+	.probe = acer_wmi_battery_probe,
+	.no_singleton = true,
+};
+
+static const struct dmi_system_id acer_wmi_battery_health_mode_table[] __initconst = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A315-510P"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A315-24PT"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A315-44P"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A315-58G"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A315-59"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A315-59"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A715-42G"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Nitro ANV15-51"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Nitro AN515-57"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Nitro AN515-58"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Nitro AN517-54"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Nitro AN517-54"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Predator PHN16-71"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Swift SF314-34"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Swift SF314-43"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Swift SFE16-44"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Swift SFG16-72"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Swift SFX14-71G"),
+		},
+	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Swift SFX16-61G"),
+		},
+	},
+	{}
+};
+
+static int __init acer_wmi_battery_module_init(void)
+{
+	if (dmi_check_system(acer_wmi_battery_health_mode_table) || force)
+		health_mode = true;
+	else
+		health_mode = false;
+
+	return wmi_driver_register(&acer_wmi_battery_driver);
+}
+
+static void __exit acer_wmi_battery_module_exit(void)
+{
+	wmi_driver_unregister(&acer_wmi_battery_driver);
+}
+
+module_init(acer_wmi_battery_module_init);
+module_exit(acer_wmi_battery_module_exit);
+
+MODULE_AUTHOR("Frederik Harwath <frederik@harwath.name>");
+MODULE_AUTHOR("Jelle van der Waa <jelle@vdwaa.nl>");
+MODULE_DESCRIPTION("Acer battery health control WMI driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c
index 61ae622c93d949..d3d741aad5568b 100644
--- a/drivers/platform/x86/acer-wmi.c
+++ b/drivers/platform/x86/acer-wmi.c
@@ -400,11 +400,12 @@ struct quirk_entry {
 	u8 mailled;
 	s8 brightness;
 	u8 bluetooth;
-	u8 turbo;
+	bool turbo;
 	u8 cpu_fans;
 	u8 gpu_fans;
-	u8 predator_v4;
-	u8 pwm;
+	bool hwmon;
+	bool platform_profile;
+	bool pwm;
 };
 
 static struct quirk_entry *quirks;
@@ -421,9 +422,11 @@ static void __init set_quirks(void)
 		interface->capability |= ACER_CAP_TURBO_OC | ACER_CAP_TURBO_LED
 					 | ACER_CAP_TURBO_FAN;
 
-	if (quirks->predator_v4)
-		interface->capability |= ACER_CAP_PLATFORM_PROFILE |
-					 ACER_CAP_HWMON;
+	if (quirks->hwmon)
+		interface->capability |= ACER_CAP_HWMON;
+
+	if (quirks->platform_profile)
+		interface->capability |= ACER_CAP_PLATFORM_PROFILE;
 
 	if (quirks->pwm)
 		interface->capability |= ACER_CAP_PWM;
@@ -451,39 +454,47 @@ static struct quirk_entry quirk_acer_aspire_1520 = {
 	.brightness = -1,
 };
 
+static struct quirk_entry quirk_acer_aspire_a315_58g = {
+	.hwmon = true,
+};
+
 static struct quirk_entry quirk_acer_travelmate_2490 = {
 	.mailled = 1,
 };
 
 static struct quirk_entry quirk_acer_nitro_an515_58 = {
-	.predator_v4 = 1,
-	.pwm = 1,
+	.hwmon = true,
+	.platform_profile = true,
+	.pwm = true,
 };
 
 static struct quirk_entry quirk_acer_predator_ph315_53 = {
-	.turbo = 1,
+	.turbo = true,
 	.cpu_fans = 1,
 	.gpu_fans = 1,
 };
 
 static struct quirk_entry quirk_acer_predator_ph16_72 = {
-	.turbo = 1,
+	.turbo = true,
 	.cpu_fans = 1,
 	.gpu_fans = 1,
-	.predator_v4 = 1,
-	.pwm = 1,
+	.hwmon = true,
+	.platform_profile = true,
+	.pwm = true,
 };
 
 static struct quirk_entry quirk_acer_predator_pt14_51 = {
-	.turbo = 1,
+	.turbo = true,
 	.cpu_fans = 1,
 	.gpu_fans = 1,
-	.predator_v4 = 1,
-	.pwm = 1,
+	.hwmon = true,
+	.platform_profile = true,
+	.pwm = true,
 };
 
 static struct quirk_entry quirk_acer_predator_v4 = {
-	.predator_v4 = 1,
+	.hwmon = true,
+	.platform_profile = true,
 };
 
 /* This AMW0 laptop has no bluetooth */
@@ -635,6 +646,15 @@ static const struct dmi_system_id acer_quirks[] __initconst = {
 		},
 		.driver_data = &quirk_acer_travelmate_2490,
 	},
+	{
+		.callback = dmi_matched,
+		.ident = "Acer Aspire A315-58G",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire A315-58G"),
+		},
+		.driver_data = &quirk_acer_aspire_a315_58g,
+	},
 	{
 		.callback = dmi_matched,
 		.ident = "Acer TravelMate 2490",
@@ -2194,7 +2214,7 @@ static const struct platform_profile_ops acer_predator_v4_platform_profile_ops =
 
 static int acer_platform_profile_setup(struct platform_device *device)
 {
-	if (quirks->predator_v4) {
+	if (quirks->platform_profile) {
 		platform_profile_device = devm_platform_profile_register(
 			&device->dev, "acer-wmi", NULL, &acer_predator_v4_platform_profile_ops);
 		if (IS_ERR(platform_profile_device))
@@ -2211,7 +2231,7 @@ static int acer_thermal_profile_change(void)
 	 * This mode key will either cycle through each mode or toggle the
 	 * most performant profile.
 	 */
-	if (quirks->predator_v4) {
+	if (quirks->platform_profile) {
 		u8 current_tp;
 		int err, tp;
 
diff --git a/drivers/platform/x86/amd/pmc/pmc-quirks.c b/drivers/platform/x86/amd/pmc/pmc-quirks.c
index 09a30f33375964..e662b123ecee5a 100644
--- a/drivers/platform/x86/amd/pmc/pmc-quirks.c
+++ b/drivers/platform/x86/amd/pmc/pmc-quirks.c
@@ -280,6 +280,15 @@ static const struct dmi_system_id fwbug_list[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "83N1"),
 		}
 	},
+	{
+		.ident = "HP Laptop 14-fq1xxx",
+		.driver_data = &quirk_s2idle_spurious_8042,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "HP"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "HP Laptop 14-fq1xxx"),
+			DMI_MATCH(DMI_BOARD_NAME, "887C"),
+		}
+	},
 	/* https://gitlab.freedesktop.org/drm/amd/-/issues/2684 */
 	{
 		.ident = "HP Laptop 15s-eq2xxx",
diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c
index 6792aa2c61874e..e384b56d67559d 100644
--- a/drivers/platform/x86/amd/pmc/pmc.c
+++ b/drivers/platform/x86/amd/pmc/pmc.c
@@ -701,19 +701,6 @@ static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg)
 	return rc;
 }
 
-static bool amd_pmc_intermediate_wakeup_need_delay(struct amd_pmc_dev *pdev)
-{
-	/*
-	 * Starting a new HW sleep cycle right after waking from one
-	 * can cause electrical problems triggering the over voltage protection.
-	 * That is avoided by delaying the next suspend a bit, see also
-	 * https://lore.kernel.org/all/20250414162446.3853194-1-superm1@kernel.org/
-	 */
-	struct smu_metrics table;
-
-	return get_metrics_table(pdev, &table) == 0 && table.s0i3_last_entry_status;
-}
-
 static bool amd_pmc_want_suspend_delay(struct amd_pmc_dev *pdev)
 {
 	/*
@@ -804,7 +791,7 @@ static void amd_pmc_s2idle_check(void)
 	struct amd_pmc_dev *pdev = &pmc;
 	int rc;
 
-	if (amd_pmc_intermediate_wakeup_need_delay(pdev) ||
+	if (!pdev->is_first_check_after_suspend ||
 	    amd_pmc_want_suspend_delay(pdev))
 		msleep(2500);
 
diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c
index b4eae65e675b81..5966190d1496ba 100644
--- a/drivers/platform/x86/amd/pmf/core.c
+++ b/drivers/platform/x86/amd/pmf/core.c
@@ -454,6 +454,7 @@ static const struct acpi_device_id amd_pmf_acpi_ids[] = {
 	{"AMDI0107", 0},
 	{"AMDI0108", 0},
 	{"AMDI0109", 0},
+	{"AMDI0112", 0},
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, amd_pmf_acpi_ids);
diff --git a/drivers/platform/x86/amd/pmf/sps.c b/drivers/platform/x86/amd/pmf/sps.c
index 0b70a5153f465e..0918fb388fde6d 100644
--- a/drivers/platform/x86/amd/pmf/sps.c
+++ b/drivers/platform/x86/amd/pmf/sps.c
@@ -248,20 +248,24 @@ static int amd_pmf_update_sps_power_limits_v2(struct amd_pmf_dev *pdev, int pwr_
 	switch (pwr_mode) {
 	case POWER_MODE_PERFORMANCE:
 		index = config_store_v2.sps_idx.power_states[src][POWER_MODE_BEST_PERFORMANCE];
-		amd_pmf_update_slider_v2(pdev, index);
 		break;
 	case POWER_MODE_BALANCED_POWER:
 		index = config_store_v2.sps_idx.power_states[src][POWER_MODE_BALANCED];
-		amd_pmf_update_slider_v2(pdev, index);
 		break;
 	case POWER_MODE_POWER_SAVER:
 		index = config_store_v2.sps_idx.power_states[src][POWER_MODE_BEST_POWER_EFFICIENCY];
-		amd_pmf_update_slider_v2(pdev, index);
 		break;
 	default:
 		return -EINVAL;
 	}
 
+	if (index >= APTS_MAX_STATES) {
+		dev_err(pdev->dev, "Invalid APTS index: %d\n", index);
+		return -EINVAL;
+	}
+
+	amd_pmf_update_slider_v2(pdev, index);
+
 	return 0;
 }
 
diff --git a/drivers/platform/x86/amd/pmf/util.c b/drivers/platform/x86/amd/pmf/util.c
index 4d1a61e4f1b90e..2befc1b14842fb 100644
--- a/drivers/platform/x86/amd/pmf/util.c
+++ b/drivers/platform/x86/amd/pmf/util.c
@@ -123,6 +123,7 @@ static const struct file_operations pmf_if_ops = {
 	.owner          = THIS_MODULE,
 	.open           = amd_pmf_open,
 	.unlocked_ioctl = amd_pmf_set_ioctl,
+	.compat_ioctl	= compat_ptr_ioctl,
 };
 
 static struct miscdevice amd_pmf_util_if = {
diff --git a/drivers/platform/x86/asus-armoury.c b/drivers/platform/x86/asus-armoury.c
index 93d9665717afa5..2d5ca75bc72742 100644
--- a/drivers/platform/x86/asus-armoury.c
+++ b/drivers/platform/x86/asus-armoury.c
@@ -515,13 +515,25 @@ static ssize_t dgpu_disable_current_value_store(struct kobject *kobj,
 	if (err)
 		return err;
 
-	if (asus_armoury.gpu_mux_dev_id) {
-		err = armoury_get_devstate(NULL, &result, asus_armoury.gpu_mux_dev_id);
-		if (err)
-			return err;
-		if (!result && disable) {
-			pr_warn("Cannot disable dGPU when the MUX is in dGPU mode\n");
-			return -EBUSY;
+	if (disable) {
+		if (asus_armoury.gpu_mux_dev_id) {
+			err = armoury_get_devstate(NULL, &result, asus_armoury.gpu_mux_dev_id);
+			if (err)
+				return err;
+			if (!result) {
+				pr_warn("Cannot disable dGPU when the MUX is in dGPU mode\n");
+				return -EBUSY;
+			}
+		}
+
+		if (armoury_has_devstate(ASUS_WMI_DEVID_DGPU_POWER_STATE)) {
+			err = armoury_get_devstate(NULL, &result, ASUS_WMI_DEVID_DGPU_POWER_STATE);
+			if (err)
+				return err;
+			if (result) {
+				pr_warn("Cannot disable dGPU when it is in use\n");
+				return -EBUSY;
+			}
 		}
 	}
 
@@ -786,6 +798,8 @@ ASUS_ATTR_GROUP_BOOL_RW(screen_auto_brightness, "screen_auto_brightness",
 			"Set the panel brightness to Off<0> or On<1>");
 ASUS_ATTR_GROUP_BOOL_RO(egpu_connected, "egpu_connected", ASUS_WMI_DEVID_EGPU_CONNECTED,
 			"Show the eGPU connection status");
+ASUS_ATTR_GROUP_BOOL_RO(dgpu_power_state, "dgpu_power_state", ASUS_WMI_DEVID_DGPU_POWER_STATE,
+			"Show the dGPU power state (0: D3 cold, 1: D0 active)");
 ASUS_ATTR_GROUP_ROG_TUNABLE(ppt_pl1_spl, ATTR_PPT_PL1_SPL, ASUS_WMI_DEVID_PPT_PL1_SPL,
 			    "Set the CPU slow package limit");
 ASUS_ATTR_GROUP_ROG_TUNABLE(ppt_pl2_sppt, ATTR_PPT_PL2_SPPT, ASUS_WMI_DEVID_PPT_PL2_SPPT,
@@ -810,6 +824,7 @@ static const struct asus_attr_group armoury_attr_groups[] = {
 	{ &egpu_connected_attr_group, ASUS_WMI_DEVID_EGPU_CONNECTED },
 	{ &egpu_enable_attr_group, ASUS_WMI_DEVID_EGPU },
 	{ &dgpu_disable_attr_group, ASUS_WMI_DEVID_DGPU },
+	{ &dgpu_power_state_attr_group, ASUS_WMI_DEVID_DGPU_POWER_STATE },
 	{ &apu_mem_attr_group, ASUS_WMI_DEVID_APU_MEM },
 
 	{ &ppt_pl1_spl_attr_group, ASUS_WMI_DEVID_PPT_PL1_SPL },
diff --git a/drivers/platform/x86/asus-armoury.h b/drivers/platform/x86/asus-armoury.h
index 0e301d663d0c2e..d605af2fdaa102 100644
--- a/drivers/platform/x86/asus-armoury.h
+++ b/drivers/platform/x86/asus-armoury.h
@@ -1056,6 +1056,20 @@ static const struct dmi_system_id power_limits[] = {
 			.requires_fan_curve = true,
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "GA401IHR"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_max = 80,
+				.ppt_pl1_spl_min = 15,
+				.ppt_pl2_sppt_max = 80,
+				.ppt_pl2_sppt_min = 15,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "GA401Q"),
@@ -1881,6 +1895,33 @@ static const struct dmi_system_id power_limits[] = {
 			},
 		},
 	},
+		{
+			.matches = {
+				DMI_MATCH(DMI_BOARD_NAME, "GX651AR"),
+			},
+			.driver_data = &(struct power_data) {
+				.ac_data = &(struct power_limits) {
+					.ppt_pl1_spl_min = 30,
+					.ppt_pl1_spl_max = 75,
+					.ppt_pl2_sppt_min = 38,
+					.ppt_pl2_sppt_max = 80,
+					.nv_dynamic_boost_min = 5,
+					.nv_dynamic_boost_max = 25,
+					.nv_temp_target_min = 75,
+					.nv_temp_target_max = 87,
+					.nv_tgp_min = 80,
+					.nv_tgp_max = 115,
+				},
+				.dc_data = &(struct power_limits) {
+					.ppt_pl1_spl_min = 30,
+					.ppt_pl1_spl_max = 75,
+					.ppt_pl2_sppt_min = 38,
+					.ppt_pl2_sppt_max = 80,
+					.nv_temp_target_min = 75,
+					.nv_temp_target_max = 87,
+				},
+			},
+		},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "GZ302EA"),
diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c
index 79a575d0f5b467..e4f20ab5250a94 100644
--- a/drivers/platform/x86/asus-laptop.c
+++ b/drivers/platform/x86/asus-laptop.c
@@ -865,7 +865,7 @@ static ssize_t infos_show(struct device *dev, struct device_attribute *attr,
 		len += sysfs_emit_at(page, len, "SFUN value         : %#x\n",
 			       (uint) temp);
 	/*
-	 * The HWRS method return informations about the hardware.
+	 * The HWRS method return information about the hardware.
 	 * 0x80 bit is for WLAN, 0x100 for Bluetooth.
 	 * 0x40 for WWAN, 0x10 for WIMAX.
 	 * The significance of others is yet to be found.
diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c
index aeb461b1644dd9..da1ddd6016c658 100644
--- a/drivers/platform/x86/asus-nb-wmi.c
+++ b/drivers/platform/x86/asus-nb-wmi.c
@@ -137,6 +137,10 @@ static struct quirk_entry quirk_asus_use_lid_flip_devid = {
 	.tablet_switch_mode = asus_wmi_lid_flip_devid,
 };
 
+static struct quirk_entry quirk_asus_proart_px13 = {
+	.tablet_switch_mode = asus_wmi_lid_flip_devid,
+};
+
 static struct quirk_entry quirk_asus_tablet_mode = {
 	.wmi_backlight_set_devstate = true,
 	.tablet_switch_mode = asus_wmi_lid_flip_rog_devid,
@@ -562,6 +566,15 @@ static const struct dmi_system_id asus_quirks[] = {
 		},
 		.driver_data = &quirk_asus_z13,
 	},
+	{
+		.callback = dmi_matched,
+		.ident = "ASUS ProArt PX13",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "HN7306WV"),
+		},
+		.driver_data = &quirk_asus_proart_px13,
+	},
 	{},
 };
 
@@ -573,7 +586,7 @@ static void asus_nb_wmi_quirks(struct asus_wmi_driver *driver)
 	driver->quirks = quirks;
 	driver->panel_power = BACKLIGHT_POWER_ON;
 
-	/* overwrite the wapf setting if the wapf paramater is specified */
+	/* overwrite the wapf setting if the wapf parameter is specified */
 	if (wapf != -1)
 		quirks->wapf = wapf;
 	else
diff --git a/drivers/platform/x86/asus-tf103c-dock.c b/drivers/platform/x86/asus-tf103c-dock.c
index 92466477de9a01..3be5b96de5484e 100644
--- a/drivers/platform/x86/asus-tf103c-dock.c
+++ b/drivers/platform/x86/asus-tf103c-dock.c
@@ -772,7 +772,7 @@ static int tf103c_dock_probe(struct i2c_client *client)
 					IRQF_TRIGGER_LOW | IRQF_ONESHOT | IRQF_NO_AUTOEN,
 					"dock_irq", dock);
 	if (ret)
-		return dev_err_probe(dev, ret, "requesting dock IRQ");
+		return ret;
 
 	dock->hpd_gpio = devm_gpiod_get(dev, "dock_hpd", GPIOD_IN);
 	if (IS_ERR(dock->hpd_gpio))
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index a65090429ca703..711f59c1aa623b 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -127,7 +127,6 @@ module_param(fnlock_default, bool, 0444);
 #define NVIDIA_TEMP_MAX		87
 
 #define ASUS_SCREENPAD_BRIGHT_MAX 255
-#define ASUS_SCREENPAD_BRIGHT_DEFAULT 60
 
 #define ASUS_MINI_LED_MODE_MASK		0x03
 /* Standard modes for devices with only on/off */
@@ -262,6 +261,7 @@ struct asus_wmi {
 	struct led_classdev lightbar_led;
 	int lightbar_led_wk;
 	struct led_classdev micmute_led;
+	struct led_classdev mute_led;
 	struct led_classdev camera_led;
 	struct workqueue_struct *led_workqueue;
 	struct work_struct tpd_led_work;
@@ -2054,6 +2054,16 @@ static int micmute_led_set(struct led_classdev *led_cdev,
 	return err < 0 ? err : 0;
 }
 
+static int mute_led_set(struct led_classdev *led_cdev,
+			enum led_brightness brightness)
+{
+	int state = brightness != LED_OFF;
+	int err;
+
+	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_MUTE_LED, state, NULL);
+	return err < 0 ? err : 0;
+}
+
 static enum led_brightness camera_led_get(struct led_classdev *led_cdev)
 {
 	struct asus_wmi *asus;
@@ -2084,10 +2094,23 @@ static void asus_wmi_led_exit(struct asus_wmi *asus)
 	led_classdev_unregister(&asus->wlan_led);
 	led_classdev_unregister(&asus->lightbar_led);
 	led_classdev_unregister(&asus->micmute_led);
+	led_classdev_unregister(&asus->mute_led);
 	led_classdev_unregister(&asus->camera_led);
 
 	if (asus->led_workqueue)
 		destroy_workqueue(asus->led_workqueue);
+
+	/*
+	 * kbd_led is registered lazily by kbd_led_work: now that the
+	 * workqueue is destroyed and asus_ref.asus is NULL, the work can
+	 * neither run nor be queued anymore, furthermore leaving it to
+	 * devres would run the unregister from devres_release_all(),
+	 * after .remove() returned and the struct asus_wmi embedding
+	 * kbd_led has been freed.
+	 */
+	if (asus->kbd_led_registered)
+		devm_led_classdev_unregister(&asus->platform_device->dev,
+					     &asus->kbd_led);
 }
 
 static int asus_wmi_led_init(struct asus_wmi *asus)
@@ -2176,7 +2199,19 @@ static int asus_wmi_led_init(struct asus_wmi *asus)
 		asus->micmute_led.default_trigger = "audio-micmute";
 
 		rv = led_classdev_register(&asus->platform_device->dev,
-						&asus->micmute_led);
+					   &asus->micmute_led);
+		if (rv)
+			goto error;
+	}
+
+	if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_MUTE_LED)) {
+		asus->mute_led.name = "platform::mute";
+		asus->mute_led.max_brightness = 1;
+		asus->mute_led.brightness_set_blocking = mute_led_set;
+		asus->mute_led.default_trigger = "audio-mute";
+
+		rv = led_classdev_register(&asus->platform_device->dev,
+					   &asus->mute_led);
 		if (rv)
 			goto error;
 	}
@@ -3610,7 +3645,7 @@ static int fan_curve_get_factory_default(struct asus_wmi *asus, u32 fan_dev)
 	err = asus_wmi_evaluate_method_buf(asus->dsts_id, fan_dev, mode, buf,
 					   FAN_CURVE_BUF_LEN);
 	if (err) {
-		pr_warn("%s (0x%08x) failed: %d\n", __func__, fan_dev, err);
+		pr_debug("%s (0x%08x) failed: %d\n", __func__, fan_dev, err);
 		return err;
 	}
 
@@ -4486,13 +4521,20 @@ static int is_display_toggle(int code)
 
 static int read_screenpad_backlight_power(struct asus_wmi *asus)
 {
-	int ret;
+	int ret, retval;
 
-	ret = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_SCREENPAD_POWER);
+	ret = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_SCREENPAD_POWER, &retval);
 	if (ret < 0)
 		return ret;
-	/* 1 == powered */
-	return ret ? BACKLIGHT_POWER_ON : BACKLIGHT_POWER_OFF;
+
+	/*
+	 * The firmware reports the panel power in the low byte of the
+	 * devstate as a raw EC status byte that is non-zero when the
+	 * panel is powered; other models report it through
+	 * ASUS_WMI_DSTS_STATUS_BIT, which lies inside the same mask.
+	 */
+	return (retval & ASUS_WMI_DSTS_BRIGHTNESS_MASK) ?
+		BACKLIGHT_POWER_ON : BACKLIGHT_POWER_OFF;
 }
 
 static int read_screenpad_brightness(struct backlight_device *bd)
@@ -4517,26 +4559,18 @@ static int read_screenpad_brightness(struct backlight_device *bd)
 
 static int update_screenpad_bl_status(struct backlight_device *bd)
 {
-	u32 ctrl_param = bd->props.brightness;
-	int err = 0;
-
-	if (bd->props.power) {
-		err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 1, NULL);
-		if (err < 0)
-			return err;
+	int err;
 
-		err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_LIGHT, ctrl_param, NULL);
-		if (err < 0)
-			return err;
-	}
+	if (backlight_is_blank(bd))
+		return asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER,
+					     0, NULL);
 
-	if (!bd->props.power) {
-		err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 0, NULL);
-		if (err < 0)
-			return err;
-	}
+	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 1, NULL);
+	if (err < 0)
+		return err;
 
-	return err;
+	return asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_LIGHT,
+				     backlight_get_brightness(bd), NULL);
 }
 
 static const struct backlight_ops asus_screenpad_bl_ops = {
@@ -4550,16 +4584,17 @@ static int asus_screenpad_init(struct asus_wmi *asus)
 	struct backlight_device *bd;
 	struct backlight_properties props;
 	int err, power;
-	int brightness = 0;
+	u32 brightness = 0;
 
-	power = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_SCREENPAD_POWER);
+	power = read_screenpad_backlight_power(asus);
 	if (power < 0)
 		return power;
 
-	if (power) {
+	if (power == BACKLIGHT_POWER_ON) {
 		err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_SCREENPAD_LIGHT, &brightness);
 		if (err < 0)
 			return err;
+		brightness &= ASUS_WMI_DSTS_BRIGHTNESS_MASK;
 	}
 
 	memset(&props, 0, sizeof(struct backlight_properties));
@@ -4574,7 +4609,6 @@ static int asus_screenpad_init(struct asus_wmi *asus)
 	}
 
 	asus->screenpad_backlight_device = bd;
-	asus->driver->screenpad_brightness = brightness;
 	bd->props.brightness = brightness;
 	bd->props.power = power;
 	backlight_update_status(bd);
@@ -5421,8 +5455,11 @@ static struct acpi_s2idle_dev_ops asus_ally_s2idle_dev_ops = {
 
 static void asus_s2idle_check_register(void)
 {
-	if (acpi_register_lps0_dev(&asus_ally_s2idle_dev_ops))
-		pr_warn("failed to register LPS0 sleep handler in asus-wmi\n");
+	int ret;
+
+	ret = acpi_register_lps0_dev(&asus_ally_s2idle_dev_ops);
+	if (ret && ret != -ENODEV)
+		pr_warn("failed to register LPS0 sleep handler: %d\n", ret);
 }
 
 static void asus_s2idle_check_unregister(void)
diff --git a/drivers/platform/x86/asus-wmi.h b/drivers/platform/x86/asus-wmi.h
index 5cd4392b964eb8..b45f81b38a4752 100644
--- a/drivers/platform/x86/asus-wmi.h
+++ b/drivers/platform/x86/asus-wmi.h
@@ -57,7 +57,6 @@ struct quirk_entry {
 struct asus_wmi_driver {
 	int			brightness;
 	int			panel_power;
-	int			screenpad_brightness;
 	int			wlan_ctrl_by_user;
 
 	const char		*name;
diff --git a/drivers/platform/x86/ayaneo-ec.c b/drivers/platform/x86/ayaneo-ec.c
index 41a24e09124869..8b2e28d523bfee 100644
--- a/drivers/platform/x86/ayaneo-ec.c
+++ b/drivers/platform/x86/ayaneo-ec.c
@@ -79,10 +79,17 @@ static const struct dmi_system_id dmi_table[] = {
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"),
-			DMI_MATCH(DMI_BOARD_NAME, "AYANEO 2"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "AYANEO 2"),
 		},
 		.driver_data = (void *)&quirk_fan,
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "AYANEO 2S"),
+		},
+		.driver_data = (void *)&quirk_charge_limit,
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_VENDOR, "AYANEO"),
diff --git a/drivers/platform/x86/bitland-mifs-wmi.c b/drivers/platform/x86/bitland-mifs-wmi.c
index 3a373184519da4..dd1648b7de7753 100644
--- a/drivers/platform/x86/bitland-mifs-wmi.c
+++ b/drivers/platform/x86/bitland-mifs-wmi.c
@@ -30,6 +30,7 @@
 #include <linux/stddef.h>
 #include <linux/string.h>
 #include <linux/sysfs.h>
+#include <linux/types.h>
 #include <linux/unaligned.h>
 #include <linux/units.h>
 #include <linux/wmi.h>
@@ -39,25 +40,30 @@
 #define BITLAND_EVENT_GUID	"46C93E13-EE9B-4262-8488-563BCA757FEF"
 
 enum bitland_mifs_operation {
-	WMI_METHOD_GET	= 250,
-	WMI_METHOD_SET	= 251,
+	WMI_METHOD_GET	= 0xFA00,
+	WMI_METHOD_SET	= 0xFB00,
 };
 
 enum bitland_mifs_function {
-	WMI_FN_SYSTEM_PER_MODE		= 8,
-	WMI_FN_GPU_MODE			= 9,
-	WMI_FN_KBD_TYPE			= 10,
-	WMI_FN_FN_LOCK			= 11,
-	WMI_FN_TP_LOCK			= 12,
-	WMI_FN_FAN_SPEEDS		= 13,
-	WMI_FN_RGB_KB_MODE		= 16,
-	WMI_FN_RGB_KB_COLOR		= 17,
-	WMI_FN_RGB_KB_BRIGHTNESS	= 18,
-	WMI_FN_SYSTEM_AC_TYPE		= 19,
-	WMI_FN_MAX_FAN_SWITCH		= 20,
-	WMI_FN_MAX_FAN_SPEED		= 21,
-	WMI_FN_CPU_THERMOMETER		= 22,
-	WMI_FN_CPU_POWER		= 23,
+	WMI_FN_SYSTEM_PER_MODE		= 0x0800,
+	WMI_FN_GPU_MODE			= 0x0900,
+	WMI_FN_KBD_TYPE			= 0x0A00,
+	WMI_FN_FN_LOCK			= 0x0B00,
+	WMI_FN_TP_LOCK			= 0x0C00,
+	WMI_FN_FAN_SPEEDS		= 0x0D00,
+	WMI_FN_RGB_KB_MODE		= 0x1000,
+	WMI_FN_RGB_KB_COLOR		= 0x1100,
+	WMI_FN_RGB_KB_BRIGHTNESS	= 0x1200,
+	WMI_FN_SYSTEM_AC_TYPE		= 0x1300,
+	WMI_FN_MAX_FAN_SWITCH		= 0x1400,
+	WMI_FN_MAX_FAN_SPEED		= 0x1500,
+	WMI_FN_CPU_THERMOMETER		= 0x1600,
+	WMI_FN_CPU_POWER		= 0x1700,
+};
+
+enum bitland_mifs_return_code {
+	WMI_RETURN_CODE_SUCCESS			= 0x8000,
+	WMI_RETURN_CODE_UNKNOWN_FUNCTION	= 0xE000,
 };
 
 enum bitland_system_ac_mode {
@@ -114,18 +120,14 @@ enum bitland_wmi_device_type {
 };
 
 struct bitland_mifs_input {
-	u8 reserved1;
-	u8 operation;
-	u8 reserved2;
-	u8 function;
+	__le16 operation;
+	__le16 function;
 	u8 payload[28];
 } __packed;
 
 struct bitland_mifs_output {
-	u8 reserved1;
-	u8 operation;
-	u8 reserved2;
-	u8 function;
+	__le16 return_code;
+	__le16 function;
 	u8 data[28];
 } __packed;
 
@@ -171,15 +173,27 @@ static int bitland_mifs_wmi_call(struct bitland_mifs_wmi_data *data,
 
 	guard(mutex)(&data->lock);
 
-	if (!output)
-		return wmidev_invoke_procedure(data->wdev, 0, 1, &in_buf);
-
-	ret = wmidev_invoke_method(data->wdev, 0, 1, &in_buf, &out_buf, sizeof(*output));
+	ret = wmidev_invoke_method(data->wdev, 0, 1, &in_buf, &out_buf,
+				   sizeof(struct bitland_mifs_output));
 	if (ret)
 		return ret;
 
-	memcpy(output, out_buf.data, sizeof(*output));
-	kfree(out_buf.data);
+	struct bitland_mifs_output *result __free(kfree) = out_buf.data;
+
+	switch (le16_to_cpu(result->return_code)) {
+	case WMI_RETURN_CODE_SUCCESS:
+		break;
+	case WMI_RETURN_CODE_UNKNOWN_FUNCTION:
+		return -EOPNOTSUPP;
+	default:
+		return -EIO;
+	}
+
+	if (result->function != input->function)
+		return -ENOMSG;
+
+	if (output)
+		memcpy(output, result, sizeof(*output));
 
 	return 0;
 }
@@ -189,10 +203,8 @@ static int laptop_profile_get(struct device *dev,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_GET,
-		.reserved2 = 0,
-		.function = WMI_FN_SYSTEM_PER_MODE,
+		.operation = cpu_to_le16(WMI_METHOD_GET),
+		.function = cpu_to_le16(WMI_FN_SYSTEM_PER_MODE),
 	};
 	struct bitland_mifs_output result;
 	int ret;
@@ -223,8 +235,8 @@ static int laptop_profile_get(struct device *dev,
 static int bitland_check_performance_capability(struct bitland_mifs_wmi_data *data)
 {
 	struct bitland_mifs_input input = {
-		.operation = WMI_METHOD_GET,
-		.function = WMI_FN_SYSTEM_AC_TYPE,
+		.operation = cpu_to_le16(WMI_METHOD_GET),
+		.function = cpu_to_le16(WMI_FN_SYSTEM_AC_TYPE),
 	};
 	struct bitland_mifs_output output;
 	int ret;
@@ -234,7 +246,11 @@ static int bitland_check_performance_capability(struct bitland_mifs_wmi_data *da
 		return -EOPNOTSUPP;
 
 	ret = bitland_mifs_wmi_call(data, &input, &output);
-	if (ret)
+	/* Not all systems support this function, do not perform further checks on them */
+	if (ret == -EOPNOTSUPP)
+		return 0;
+
+	if (ret < 0)
 		return ret;
 
 	if (output.data[0] != WMI_SYSTEM_AC_CIRCULARHOLE)
@@ -248,10 +264,8 @@ static int laptop_profile_set(struct device *dev,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_SET,
-		.reserved2 = 0,
-		.function = WMI_FN_SYSTEM_PER_MODE,
+		.operation = cpu_to_le16(WMI_METHOD_SET),
+		.function = cpu_to_le16(WMI_FN_SYSTEM_PER_MODE),
 	};
 	int ret;
 	u8 val;
@@ -344,22 +358,20 @@ static int laptop_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_GET,
-		.reserved2 = 0,
+		.operation = cpu_to_le16(WMI_METHOD_GET),
 	};
 	struct bitland_mifs_output res;
 	int ret;
 
 	switch (type) {
 	case hwmon_temp:
-		input.function = WMI_FN_CPU_THERMOMETER;
+		input.function = cpu_to_le16(WMI_FN_CPU_THERMOMETER);
 		ret = bitland_mifs_wmi_call(data, &input, &res);
 		if (!ret)
 			*val = res.data[0] * MILLIDEGREE_PER_DEGREE;
 		return ret;
 	case hwmon_fan:
-		input.function = WMI_FN_FAN_SPEEDS;
+		input.function = cpu_to_le16(WMI_FN_FAN_SPEEDS);
 		ret = bitland_mifs_wmi_call(data, &input, &res);
 		if (ret)
 			return ret;
@@ -420,10 +432,8 @@ static int laptop_kbd_led_set(struct led_classdev *led_cdev,
 	struct bitland_mifs_wmi_data *data =
 		container_of(led_cdev, struct bitland_mifs_wmi_data, kbd_led);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_SET,
-		.reserved2 = 0,
-		.function = WMI_FN_RGB_KB_BRIGHTNESS,
+		.operation = cpu_to_le16(WMI_METHOD_SET),
+		.function = cpu_to_le16(WMI_FN_RGB_KB_BRIGHTNESS),
 	};
 
 	input.payload[0] = (u8)value;
@@ -436,10 +446,8 @@ static enum led_brightness laptop_kbd_led_get(struct led_classdev *led_cdev)
 	struct bitland_mifs_wmi_data *data =
 		container_of(led_cdev, struct bitland_mifs_wmi_data, kbd_led);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_GET,
-		.reserved2 = 0,
-		.function = WMI_FN_RGB_KB_BRIGHTNESS,
+		.operation = cpu_to_le16(WMI_METHOD_GET),
+		.function = cpu_to_le16(WMI_FN_RGB_KB_BRIGHTNESS),
 	};
 	struct bitland_mifs_output res;
 	int ret;
@@ -463,10 +471,8 @@ static ssize_t gpu_mode_show(struct device *dev, struct device_attribute *attr,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_GET,
-		.reserved2 = 0,
-		.function = WMI_FN_GPU_MODE,
+		.operation = cpu_to_le16(WMI_METHOD_GET),
+		.function = cpu_to_le16(WMI_FN_GPU_MODE),
 	};
 	struct bitland_mifs_output res;
 	u8 mode_val;
@@ -488,10 +494,8 @@ static ssize_t gpu_mode_store(struct device *dev, struct device_attribute *attr,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_SET,
-		.reserved2 = 0,
-		.function = WMI_FN_GPU_MODE,
+		.operation = cpu_to_le16(WMI_METHOD_SET),
+		.function = cpu_to_le16(WMI_FN_GPU_MODE),
 	};
 	int val;
 	int ret;
@@ -521,10 +525,8 @@ static ssize_t kb_mode_show(struct device *dev, struct device_attribute *attr,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_GET,
-		.reserved2 = 0,
-		.function = WMI_FN_RGB_KB_MODE,
+		.operation = cpu_to_le16(WMI_METHOD_GET),
+		.function = cpu_to_le16(WMI_FN_RGB_KB_MODE),
 	};
 	struct bitland_mifs_output res;
 	u8 mode_val;
@@ -546,10 +548,8 @@ static ssize_t kb_mode_store(struct device *dev, struct device_attribute *attr,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_SET,
-		.reserved2 = 0,
-		.function = WMI_FN_RGB_KB_MODE,
+		.operation = cpu_to_le16(WMI_METHOD_SET),
+		.function = cpu_to_le16(WMI_FN_RGB_KB_MODE),
 	};
 	// the wmi value (0, 1, 2 or 3)
 	int val;
@@ -575,10 +575,8 @@ static ssize_t fan_boost_store(struct device *dev,
 {
 	struct bitland_mifs_wmi_data *data = dev_get_drvdata(dev);
 	struct bitland_mifs_input input = {
-		.reserved1 = 0,
-		.operation = WMI_METHOD_SET,
-		.reserved2 = 0,
-		.function = WMI_FN_MAX_FAN_SWITCH,
+		.operation = cpu_to_le16(WMI_METHOD_SET),
+		.function = cpu_to_le16(WMI_FN_MAX_FAN_SWITCH),
 	};
 	bool val;
 	int ret;
diff --git a/drivers/platform/x86/dell/alienware-wmi-wmax.c b/drivers/platform/x86/dell/alienware-wmi-wmax.c
index d1b4df91401b14..2b0ffcb56a95e1 100644
--- a/drivers/platform/x86/dell/alienware-wmi-wmax.c
+++ b/drivers/platform/x86/dell/alienware-wmi-wmax.c
@@ -121,6 +121,14 @@ static const struct dmi_system_id awcc_dmi_table[] __initconst = {
 		},
 		.driver_data = &g_series_quirks,
 	},
+	{
+		.ident = "Alienware Area-51 AAT2250",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Alienware"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Alienware Area-51 AAT2250"),
+		},
+		.driver_data = &generic_quirks,
+	},
 	{
 		.ident = "Alienware Area-51m",
 		.matches = {
diff --git a/drivers/platform/x86/dell/dell-dw5826e-reset.c b/drivers/platform/x86/dell/dell-dw5826e-reset.c
index 64e6e9b1d7a03b..c72c243b460ed3 100644
--- a/drivers/platform/x86/dell/dell-dw5826e-reset.c
+++ b/drivers/platform/x86/dell/dell-dw5826e-reset.c
@@ -6,8 +6,10 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/array_size.h>
 #include <linux/bits.h>
 #include <linux/device.h>
+#include <linux/gfp_types.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/types.h>
@@ -15,15 +17,23 @@
 
 #define PALC_DSM_FN_TRIGGER_PLDR    1
 
+static const u64 palc_dsm_revs[] = { 1, 0 };
+
 static guid_t palc_dsm_guid =
 	GUID_INIT(0x5a1a4bba, 0x8006, 0x487e, 0xbe, 0x0a, 0xac, 0xf5, 0xd8, 0xfd, 0xfe, 0x59);
 
+struct palc_data {
+	u64 dsm_rev;
+};
+
 static int trigger_palc_pldr(struct device *dev, acpi_handle handle)
 {
+	struct palc_data *data = dev_get_drvdata(dev);
 	union acpi_object *obj;
 	int ret = 0;
 
-	obj = acpi_evaluate_dsm(handle, &palc_dsm_guid, 1, PALC_DSM_FN_TRIGGER_PLDR, NULL);
+	obj = acpi_evaluate_dsm(handle, &palc_dsm_guid, data->dsm_rev,
+				PALC_DSM_FN_TRIGGER_PLDR, NULL);
 	if (!obj) {
 		dev_err(dev, "Failed to evaluate _DSM\n");
 		return -EIO;
@@ -60,16 +70,28 @@ ATTRIBUTE_GROUPS(palc);
 
 static int palc_probe(struct platform_device *pdev)
 {
+	struct palc_data *data;
 	acpi_handle handle;
+	unsigned int i;
 
 	handle = ACPI_HANDLE(&pdev->dev);
 	if (!handle)
 		return -ENODEV;
 
-	if (!acpi_check_dsm(handle, &palc_dsm_guid, 1, BIT(PALC_DSM_FN_TRIGGER_PLDR)))
-		return -ENODEV;
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(palc_dsm_revs); i++) {
+		if (acpi_check_dsm(handle, &palc_dsm_guid, palc_dsm_revs[i],
+				   BIT(PALC_DSM_FN_TRIGGER_PLDR))) {
+			data->dsm_rev = palc_dsm_revs[i];
+			platform_set_drvdata(pdev, data);
+			return 0;
+		}
+	}
 
-	return 0;
+	return -ENODEV;
 }
 
 static const struct acpi_device_id palc_acpi_ids[] = {
diff --git a/drivers/platform/x86/dell/dell-wmi-aio.c b/drivers/platform/x86/dell/dell-wmi-aio.c
index 54096495719bfa..8849ad93480ba9 100644
--- a/drivers/platform/x86/dell/dell-wmi-aio.c
+++ b/drivers/platform/x86/dell/dell-wmi-aio.c
@@ -5,14 +5,20 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/cleanup.h>
+#include <linux/compiler_attributes.h>
+#include <linux/device.h>
+#include <linux/device/driver.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/overflow.h>
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/input.h>
 #include <linux/input/sparse-keymap.h>
-#include <linux/acpi.h>
 #include <linux/string.h>
+#include <linux/wmi.h>
 
 MODULE_DESCRIPTION("WMI hotkeys driver for Dell All-In-One series");
 MODULE_LICENSE("GPL");
@@ -20,22 +26,19 @@ MODULE_LICENSE("GPL");
 #define EVENT_GUID1 "284A0E6B-380E-472A-921F-E52786257FB4"
 #define EVENT_GUID2 "02314822-307C-4F66-BF0E-48AEAEB26CC8"
 
+struct dell_wmi_aio_data {
+	struct input_dev *input_device;
+	/* Protects the input sequence */
+	struct mutex input_lock;
+};
+
 struct dell_wmi_event {
-	u16	length;
+	__le16	length;
 	/* 0x000: A hot key pressed or an event occurred
 	 * 0x00F: A sequence of hot keys are pressed */
-	u16	type;
-	u16	event[];
-};
-
-static const char *dell_wmi_aio_guids[] = {
-	EVENT_GUID1,
-	EVENT_GUID2,
-	NULL
-};
-
-MODULE_ALIAS("wmi:"EVENT_GUID1);
-MODULE_ALIAS("wmi:"EVENT_GUID2);
+	__le16	type;
+	__le16	event[];
+} __packed;
 
 static const struct key_entry dell_wmi_aio_keymap[] = {
 	{ KE_KEY, 0xc0, { KEY_VOLUMEUP } },
@@ -50,137 +53,92 @@ static const struct key_entry dell_wmi_aio_keymap[] = {
 	{ KE_END, 0 }
 };
 
-static struct input_dev *dell_wmi_aio_input_dev;
-
 /*
  * The new WMI event data format will follow the dell_wmi_event structure
  * So, we will check if the buffer matches the format
  */
-static bool dell_wmi_aio_event_check(u8 *buffer, int length)
+static bool dell_wmi_aio_event_check(const struct wmi_buffer *buffer)
 {
-	struct dell_wmi_event *event = (struct dell_wmi_event *)buffer;
+	struct dell_wmi_event *event;
+	u16 length, type;
 
-	if (event == NULL || length < 6)
+	if (buffer->length < struct_size(event, event, 1))
 		return false;
 
-	if ((event->type == 0 || event->type == 0xf) &&
-			event->length >= 2)
+	event = buffer->data;
+	length = le16_to_cpu(event->length);
+	type = le16_to_cpu(event->type);
+	if ((type == 0 || type == 0xf) && length >= 2)
 		return true;
 
 	return false;
 }
 
-static void dell_wmi_aio_notify(union acpi_object *obj, void *context)
+static void dell_wmi_aio_notify(struct wmi_device *wdev, const struct wmi_buffer *data)
 {
-	struct dell_wmi_event *event;
-
-	if (obj) {
-		unsigned int scancode = 0;
-
-		switch (obj->type) {
-		case ACPI_TYPE_INTEGER:
-			/* Most All-In-One correctly return integer scancode */
-			scancode = obj->integer.value;
-			sparse_keymap_report_event(dell_wmi_aio_input_dev,
-				scancode, 1, true);
-			break;
-		case ACPI_TYPE_BUFFER:
-			if (dell_wmi_aio_event_check(obj->buffer.pointer,
-						obj->buffer.length)) {
-				event = (struct dell_wmi_event *)
-					obj->buffer.pointer;
-				scancode = event->event[0];
-			} else {
-				/* Broken machines return the scancode in a
-				   buffer */
-				if (obj->buffer.pointer &&
-						obj->buffer.length > 0)
-					scancode = obj->buffer.pointer[0];
-			}
-			if (scancode)
-				sparse_keymap_report_event(
-					dell_wmi_aio_input_dev,
-					scancode, 1, true);
-			break;
-		}
-	}
-}
-
-static int __init dell_wmi_aio_input_setup(void)
-{
-	int err;
-
-	dell_wmi_aio_input_dev = input_allocate_device();
-
-	if (!dell_wmi_aio_input_dev)
-		return -ENOMEM;
-
-	dell_wmi_aio_input_dev->name = "Dell AIO WMI hotkeys";
-	dell_wmi_aio_input_dev->phys = "wmi/input0";
-	dell_wmi_aio_input_dev->id.bustype = BUS_HOST;
-
-	err = sparse_keymap_setup(dell_wmi_aio_input_dev,
-			dell_wmi_aio_keymap, NULL);
-	if (err) {
-		pr_err("Unable to setup input device keymap\n");
-		goto err_free_dev;
-	}
-	err = input_register_device(dell_wmi_aio_input_dev);
-	if (err) {
-		pr_info("Unable to register input device\n");
-		goto err_free_dev;
+	struct dell_wmi_aio_data *drvdata = dev_get_drvdata(&wdev->dev);
+	const struct dell_wmi_event *new_event;
+	unsigned int scancode;
+	const u8 *old_event;
+
+	if (dell_wmi_aio_event_check(data)) {
+		new_event = data->data;
+		scancode = le16_to_cpu(new_event->event[0]);
+	} else {
+		old_event = data->data;
+		scancode = old_event[0];
 	}
-	return 0;
-
-err_free_dev:
-	input_free_device(dell_wmi_aio_input_dev);
-	return err;
-}
-
-static const char *dell_wmi_aio_find(void)
-{
-	int i;
 
-	for (i = 0; dell_wmi_aio_guids[i] != NULL; i++)
-		if (wmi_has_guid(dell_wmi_aio_guids[i]))
-			return dell_wmi_aio_guids[i];
+	guard(mutex)(&drvdata->input_lock);
 
-	return NULL;
+	sparse_keymap_report_event(drvdata->input_device, scancode, 1, true);
 }
 
-static int __init dell_wmi_aio_init(void)
+static int dell_wmi_aio_probe(struct wmi_device *wdev, const void *context)
 {
-	int err;
-	const char *guid;
+	struct dell_wmi_aio_data *data;
+	int ret;
 
-	guid = dell_wmi_aio_find();
-	if (!guid) {
-		pr_warn("No known WMI GUID found\n");
-		return -ENXIO;
-	}
+	data = devm_kzalloc(&wdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
 
-	err = dell_wmi_aio_input_setup();
-	if (err)
-		return err;
+	dev_set_drvdata(&wdev->dev, data);
+	ret = devm_mutex_init(&wdev->dev, &data->input_lock);
+	if (ret < 0)
+		return ret;
 
-	err = wmi_install_notify_handler(guid, dell_wmi_aio_notify, NULL);
-	if (err) {
-		pr_err("Unable to register notify handler - %d\n", err);
-		input_unregister_device(dell_wmi_aio_input_dev);
-		return err;
-	}
+	data->input_device = devm_input_allocate_device(&wdev->dev);
+	if (!data->input_device)
+		return -ENOMEM;
 
-	return 0;
-}
+	data->input_device->name = "Dell AIO WMI hotkeys";
+	data->input_device->phys = "wmi/input0";
+	data->input_device->id.bustype = BUS_HOST;
 
-static void __exit dell_wmi_aio_exit(void)
-{
-	const char *guid;
+	ret = sparse_keymap_setup(data->input_device, dell_wmi_aio_keymap, NULL);
+	if (ret < 0)
+		return ret;
 
-	guid = dell_wmi_aio_find();
-	wmi_remove_notify_handler(guid);
-	input_unregister_device(dell_wmi_aio_input_dev);
+	return input_register_device(data->input_device);
 }
 
-module_init(dell_wmi_aio_init);
-module_exit(dell_wmi_aio_exit);
+static const struct wmi_device_id dell_wmi_aio_id_table[] = {
+	{ EVENT_GUID1, NULL },
+	{ EVENT_GUID2, NULL },
+	{ }
+};
+MODULE_DEVICE_TABLE(wmi, dell_wmi_aio_id_table);
+
+static struct wmi_driver dell_wmi_aio_driver = {
+	.driver = {
+		.name = "dell-wmi-aio",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+	},
+	.id_table = dell_wmi_aio_id_table,
+	.probe = dell_wmi_aio_probe,
+	.notify_new = dell_wmi_aio_notify,
+	.min_event_size = sizeof(u8),
+	.no_singleton = true,
+};
+module_wmi_driver(dell_wmi_aio_driver);
diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
index 309634c1cc20ee..2dab9c0624b287 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
@@ -442,7 +442,7 @@ int hp_convert_hexstr_to_str(const char *input, u32 input_len, char **str, int *
 	*len = 0;
 	*str = NULL;
 
-	new_str = kmalloc(input_len, GFP_KERNEL);
+	new_str = kzalloc(2 * DIV_ROUND_UP(input_len, 5) + 1, GFP_KERNEL);
 	if (!new_str)
 		return -ENOMEM;
 
diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c
index 615b4cf6fc4517..994de0a6d29e63 100644
--- a/drivers/platform/x86/hp/hp-wmi.c
+++ b/drivers/platform/x86/hp/hp-wmi.c
@@ -207,9 +207,8 @@ static const char * const omen_thermal_profile_boards[] = {
 	"886B", "886C", "88C8", "88CB", "88D1", "88D2", "88F4", "88F5", "88F6",
 	"88F7", "88FD", "88FE", "88FF",
 	"8900", "8901", "8902", "8912", "8917", "8918", "8949", "894A", "89EB",
-	"8A15", "8A42", "8A43",
+	"8A15", "8A18", "8A42", "8A43",
 	"8BAD",
-	"8C58",
 	"8E41",
 };
 
@@ -299,6 +298,10 @@ static const struct dmi_system_id hp_wmi_feature_boards[] __initconst = {
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8BD5") },
 		.driver_data = (void *)&victus_s_board_params,
 	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C58") },
+		.driver_data = (void *)&omen_v1_legacy_board_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8C76") },
 		.driver_data = (void *)&omen_v1_board_params,
@@ -323,10 +326,22 @@ static const struct dmi_system_id hp_wmi_feature_boards[] __initconst = {
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D26") },
 		.driver_data = (void *)&omen_v1_legacy_board_params,
 	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D3F") },
+		.driver_data = (void *)&omen_v1_legacy_board_params,
+	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D40") },
+		.driver_data = (void *)&omen_v1_no_ec_board_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") },
 		.driver_data = (void *)&omen_v1_no_ec_board_params,
 	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D42") },
+		.driver_data = (void *)&omen_v1_no_ec_board_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8D87") },
 		.driver_data = (void *)&omen_v1_no_ec_board_params,
@@ -339,10 +354,18 @@ static const struct dmi_system_id hp_wmi_feature_boards[] __initconst = {
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8DD6") },
 		.driver_data = (void *)&omen_v1_no_ec_board_params,
 	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8DE1") },
+		.driver_data = (void *)&victus_s_board_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8E35") },
 		.driver_data = (void *)&omen_v1_legacy_board_params,
 	},
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8E41") },
+		.driver_data = (void *)&omen_v1_legacy_board_params,
+	},
 	{},
 };
 
diff --git a/drivers/platform/x86/hp/hp_accel.c b/drivers/platform/x86/hp/hp_accel.c
index 39b73dc473f1ce..40e1b9fb1b6d91 100644
--- a/drivers/platform/x86/hp/hp_accel.c
+++ b/drivers/platform/x86/hp/hp_accel.c
@@ -122,7 +122,7 @@ static int lis3lv02d_acpi_read(struct lis3lv02d *lis3, int reg, u8 *ret)
 static int lis3lv02d_acpi_write(struct lis3lv02d *lis3, int reg, u8 val)
 {
 	struct acpi_device *dev = lis3->bus_priv;
-	unsigned long long ret; /* Not used when writting */
+	unsigned long long ret; /* Not used when writing */
 	union acpi_object in_obj[2];
 	struct acpi_object_list args = { 2, in_obj };
 
diff --git a/drivers/platform/x86/huawei-wmi.c b/drivers/platform/x86/huawei-wmi.c
index d6aaf14d66a5dc..96ee9a97c9b8f6 100644
--- a/drivers/platform/x86/huawei-wmi.c
+++ b/drivers/platform/x86/huawei-wmi.c
@@ -79,6 +79,7 @@ static const struct key_entry huawei_wmi_keymap[] = {
 	{ KE_KEY,    0x285, { KEY_VOLUMEDOWN } },
 	{ KE_KEY,    0x286, { KEY_VOLUMEUP } },
 	{ KE_KEY,    0x287, { KEY_MICMUTE } },
+	{ KE_KEY,    0x288, { KEY_CAMERA_ACCESS_TOGGLE } },
 	{ KE_KEY,    0x289, { KEY_WLAN } },
 	// Huawei |M| key
 	{ KE_KEY,    0x28a, { KEY_CONFIG } },
diff --git a/drivers/platform/x86/intel/bxtwc_tmu.c b/drivers/platform/x86/intel/bxtwc_tmu.c
index b3666704d85bb9..73d7f71b2fd0f5 100644
--- a/drivers/platform/x86/intel/bxtwc_tmu.c
+++ b/drivers/platform/x86/intel/bxtwc_tmu.c
@@ -64,11 +64,8 @@ static int bxt_wcove_tmu_probe(struct platform_device *pdev)
 	ret = devm_request_threaded_irq(&pdev->dev, wctmu->irq,
 					NULL, bxt_wcove_tmu_irq_handler,
 					IRQF_ONESHOT, "bxt_wcove_tmu", wctmu);
-	if (ret) {
-		dev_err(&pdev->dev, "request irq failed: %d,virq: %d\n",
-			ret, wctmu->irq);
+	if (ret)
 		return ret;
-	}
 
 	/* Unmask TMU second level Wake & System alarm */
 	regmap_update_bits(wctmu->regmap, BXTWC_MTMUIRQ_REG,
diff --git a/drivers/platform/x86/intel/bytcrc_pwrsrc.c b/drivers/platform/x86/intel/bytcrc_pwrsrc.c
index 68ac040082df8e..1f39850b80fd0f 100644
--- a/drivers/platform/x86/intel/bytcrc_pwrsrc.c
+++ b/drivers/platform/x86/intel/bytcrc_pwrsrc.c
@@ -221,7 +221,7 @@ static int crc_pwrsrc_probe(struct platform_device *pdev)
 						crc_pwrsrc_irq_handler,
 						IRQF_ONESHOT, KBUILD_MODNAME, data);
 		if (ret)
-			return dev_err_probe(dev, ret, "requesting IRQ\n");
+			return ret;
 	}
 
 	data->debug_dentry = debugfs_create_dir(KBUILD_MODNAME, NULL);
diff --git a/drivers/platform/x86/intel/crystal_cove_charger.c b/drivers/platform/x86/intel/crystal_cove_charger.c
index e4299cfa220545..425c7eb32442a9 100644
--- a/drivers/platform/x86/intel/crystal_cove_charger.c
+++ b/drivers/platform/x86/intel/crystal_cove_charger.c
@@ -135,7 +135,7 @@ static int crystal_cove_charger_probe(struct platform_device *pdev)
 					crystal_cove_charger_irq,
 					IRQF_ONESHOT, KBUILD_MODNAME, charger);
 	if (ret)
-		return dev_err_probe(&pdev->dev, ret, "requesting irq\n");
+		return ret;
 
 	return 0;
 }
diff --git a/drivers/platform/x86/intel/int0002_vgpio.c b/drivers/platform/x86/intel/int0002_vgpio.c
index 562e8802564367..96b36cfcd9e406 100644
--- a/drivers/platform/x86/intel/int0002_vgpio.c
+++ b/drivers/platform/x86/intel/int0002_vgpio.c
@@ -208,10 +208,8 @@ static int int0002_probe(struct platform_device *pdev)
 	 */
 	ret = devm_request_irq(dev, irq, int0002_irq, IRQF_SHARED, "INT0002",
 			       chip);
-	if (ret) {
-		dev_err(dev, "Error requesting IRQ %d: %d\n", irq, ret);
+	if (ret)
 		return ret;
-	}
 
 	girq = &chip->irq;
 	gpio_irq_chip_set_chip(girq, &int0002_irqchip);
diff --git a/drivers/platform/x86/intel/int3472/clk_and_regulator.c b/drivers/platform/x86/intel/int3472/clk_and_regulator.c
index 9e052b164a1ab8..f2f71142867ce5 100644
--- a/drivers/platform/x86/intel/int3472/clk_and_regulator.c
+++ b/drivers/platform/x86/intel/int3472/clk_and_regulator.c
@@ -9,6 +9,7 @@
 #include <linux/platform_data/x86/int3472.h>
 #include <linux/regulator/driver.h>
 #include <linux/slab.h>
+#include <linux/string_helpers.h>
 
 /*
  * 82c0d13a-78c5-4244-9bb1-eb8b539a8d11
diff --git a/drivers/platform/x86/intel/int3472/discrete.c b/drivers/platform/x86/intel/int3472/discrete.c
index 6c729fcfce5dbe..b48cf1b7fd5f7c 100644
--- a/drivers/platform/x86/intel/int3472/discrete.c
+++ b/drivers/platform/x86/intel/int3472/discrete.c
@@ -15,6 +15,7 @@
 #include <linux/platform_data/x86/int3472.h>
 #include <linux/platform_device.h>
 #include <linux/string_choices.h>
+#include <linux/time64.h>
 #include <linux/uuid.h>
 
 /*
@@ -143,6 +144,11 @@ static const char * const power_enable_hids_enable[] = {
 	NULL
 };
 
+static const char * const power_enable_hids_vdda[] = {
+	"INT347E", /* ov7251 */
+	NULL
+};
+
 /**
  * struct int3472_gpio_map - Map GPIOs to whatever is expected by the
  * sensor driver (as in DT bindings)
@@ -178,12 +184,12 @@ static const struct int3472_gpio_map int3472_gpio_map[] = {
 		.type_to = INT3472_GPIO_TYPE_RESET,
 		.con_id = "enable",
 	},
-	{	/* ov08x40's handshake pin needs a 45 ms delay on some HP laptops */
-		.hids = (const char * const[]) { "OVTI08F4", NULL },
-		.type_from = INT3472_GPIO_TYPE_HANDSHAKE,
-		.type_to = INT3472_GPIO_TYPE_HANDSHAKE,
-		.con_id = "dvdd",
-		.enable_time_us = 45 * USEC_PER_MSEC,
+	{	/* Sensors which expect "vdda" as con_id for power enable */
+		.hids = power_enable_hids_vdda,
+		.type_from = INT3472_GPIO_TYPE_POWER_ENABLE,
+		.type_to = INT3472_GPIO_TYPE_POWER_ENABLE,
+		.con_id = "vdda",
+		.enable_time_us = GPIO_REGULATOR_ENABLE_TIME,
 	},
 	{	/* Sensors which expect "vana" as con_id for power enable */
 		.hids = power_enable_hids_vana,
@@ -266,6 +272,10 @@ static void int3472_get_con_id_and_polarity(struct int3472_discrete_device *int3
 		*con_id = "avdd";
 		*gpio_flags = GPIO_ACTIVE_HIGH;
 		break;
+	case INT3472_GPIO_TYPE_POWER1:
+		*con_id = "dvdd";
+		*gpio_flags = GPIO_ACTIVE_HIGH;
+		break;
 	case INT3472_GPIO_TYPE_DOVDD:
 		*con_id = "dovdd";
 		*gpio_flags = GPIO_ACTIVE_HIGH;
@@ -273,8 +283,8 @@ static void int3472_get_con_id_and_polarity(struct int3472_discrete_device *int3
 	case INT3472_GPIO_TYPE_HANDSHAKE:
 		*con_id = "dvdd";
 		*gpio_flags = GPIO_ACTIVE_HIGH;
-		/* Setups using a handshake pin need 25 ms enable delay */
-		*enable_time_us = 25 * USEC_PER_MSEC;
+		/* Powering up the sensor through the handshake pin takes up to 200 ms */
+		*enable_time_us = 200 * USEC_PER_MSEC;
 		break;
 	default:
 		*con_id = "unknown";
@@ -296,6 +306,8 @@ static void int3472_get_con_id_and_polarity(struct int3472_discrete_device *int3
  * 0x00 Reset
  * 0x01 Power down
  * 0x02 Strobe
+ * 0x07 Power 0
+ * 0x08 Power 1
  * 0x0b Power enable
  * 0x0c Clock enable
  * 0x0d Privacy LED
@@ -328,8 +340,8 @@ static int skl_int3472_handle_gpio_resources(struct acpi_resource *ares,
 	u8 active_value, pin, type;
 	unsigned long gpio_flags;
 	union acpi_object *obj;
+	unsigned int obj_value;
 	struct gpio_desc *gpio;
-	const char *err_msg;
 	const char *con_id;
 	int ret;
 
@@ -344,24 +356,27 @@ static int skl_int3472_handle_gpio_resources(struct acpi_resource *ares,
 				      &int3472_gpio_guid, 0x00,
 				      int3472->ngpios + 2,
 				      NULL, ACPI_TYPE_INTEGER);
-
 	if (!obj) {
 		dev_warn(int3472->dev, "No _DSM entry for GPIO pin %u\n",
 			 agpio->pin_table[0]);
 		return 1;
 	}
 
-	type = FIELD_GET(INT3472_GPIO_DSM_TYPE, obj->integer.value);
+	obj_value = obj->integer.value;
+
+	ACPI_FREE(obj);
+
+	type = FIELD_GET(INT3472_GPIO_DSM_TYPE, obj_value);
 
 	int3472_get_con_id_and_polarity(int3472, &type, &con_id, &gpio_flags, &enable_time_us);
 
-	pin = FIELD_GET(INT3472_GPIO_DSM_PIN, obj->integer.value);
+	pin = FIELD_GET(INT3472_GPIO_DSM_PIN, obj_value);
 	/* Pin field is not really used under Windows and wraps around at 8 bits */
 	if (pin != (agpio->pin_table[0] & 0xff))
 		dev_dbg(int3472->dev, FW_BUG "%s %s pin number mismatch _DSM %d resource %d\n",
 			con_id, agpio->resource_source.string_ptr, pin, agpio->pin_table[0]);
 
-	active_value = FIELD_GET(INT3472_GPIO_DSM_SENSOR_ON_VAL, obj->integer.value);
+	active_value = FIELD_GET(INT3472_GPIO_DSM_SENSOR_ON_VAL, obj_value);
 	if (!active_value)
 		gpio_flags ^= GPIO_ACTIVE_LOW;
 
@@ -369,51 +384,57 @@ static int skl_int3472_handle_gpio_resources(struct acpi_resource *ares,
 		agpio->resource_source.string_ptr, agpio->pin_table[0],
 		str_high_low(gpio_flags == GPIO_ACTIVE_HIGH));
 
+	int3472->ngpios++;
+
 	switch (type) {
 	case INT3472_GPIO_TYPE_RESET:
 	case INT3472_GPIO_TYPE_POWERDOWN:
 	case INT3472_GPIO_TYPE_HOTPLUG_DETECT:
 		ret = skl_int3472_map_gpio_to_sensor(int3472, agpio, con_id, gpio_flags);
 		if (ret)
-			err_msg = "Failed to map GPIO pin to sensor\n";
+			return dev_err_probe(int3472->dev, ret,
+					     "Failed to map GPIO pin to sensor\n");
 
-		break;
+		return 1;
 	case INT3472_GPIO_TYPE_CLK_ENABLE:
 	case INT3472_GPIO_TYPE_PRIVACY_LED:
 	case INT3472_GPIO_TYPE_STROBE:
 	case INT3472_GPIO_TYPE_POWER_ENABLE:
+	case INT3472_GPIO_TYPE_POWER1:
 	case INT3472_GPIO_TYPE_DOVDD:
 	case INT3472_GPIO_TYPE_HANDSHAKE:
 		gpio = skl_int3472_gpiod_get_from_temp_lookup(int3472, agpio, con_id, gpio_flags);
-		if (IS_ERR(gpio)) {
-			ret = PTR_ERR(gpio);
-			err_msg = "Failed to get GPIO\n";
-			break;
-		}
+		if (IS_ERR(gpio))
+			return dev_err_probe(int3472->dev, PTR_ERR(gpio),
+					     "Failed to get GPIO\n");
 
 		switch (type) {
 		case INT3472_GPIO_TYPE_CLK_ENABLE:
 			ret = skl_int3472_register_gpio_clock(int3472, gpio);
 			if (ret)
-				err_msg = "Failed to register clock\n";
+				dev_err_probe(int3472->dev, ret,
+					      "Failed to register clock\n");
 
 			break;
 		case INT3472_GPIO_TYPE_PRIVACY_LED:
 		case INT3472_GPIO_TYPE_STROBE:
 			ret = skl_int3472_register_led(int3472, gpio, con_id);
 			if (ret)
-				err_msg = "Failed to register LED\n";
+				dev_err_probe(int3472->dev, ret,
+					      "Failed to register LED\n");
 
 			break;
 		case INT3472_GPIO_TYPE_POWER_ENABLE:
 			second_sensor = int3472->quirks.avdd_second_sensor;
 			fallthrough;
+		case INT3472_GPIO_TYPE_POWER1:
 		case INT3472_GPIO_TYPE_DOVDD:
 		case INT3472_GPIO_TYPE_HANDSHAKE:
 			ret = skl_int3472_register_regulator(int3472, gpio, enable_time_us,
 							     con_id, second_sensor);
 			if (ret)
-				err_msg = "Failed to register regulator\n";
+				dev_err_probe(int3472->dev, ret,
+					      "Failed to register regulator\n");
 
 			break;
 		default: /* Never reached */
@@ -424,23 +445,13 @@ static int skl_int3472_handle_gpio_resources(struct acpi_resource *ares,
 		if (ret)
 			gpiod_put(gpio);
 
-		break;
+		return ret < 0 ? ret : 1;
 	default:
 		dev_warn(int3472->dev,
 			 "GPIO type 0x%02x unknown; the sensor may not work\n",
 			 type);
-		ret = 1;
-		break;
+		return 1;
 	}
-
-	int3472->ngpios++;
-	ACPI_FREE(obj);
-
-	if (ret < 0)
-		return dev_err_probe(int3472->dev, ret, err_msg);
-
-	/* Tell acpi_dev_get_resources() to not make a copy of the resource */
-	return 1;
 }
 
 int int3472_discrete_parse_crs(struct int3472_discrete_device *int3472)
diff --git a/drivers/platform/x86/intel/int3472/tps68470.c b/drivers/platform/x86/intel/int3472/tps68470.c
index a77ed32abe55f1..dc777dbac61f15 100644
--- a/drivers/platform/x86/intel/int3472/tps68470.c
+++ b/drivers/platform/x86/intel/int3472/tps68470.c
@@ -161,7 +161,7 @@ static int skl_int3472_tps68470_probe(struct i2c_client *client)
 
 	regmap = devm_regmap_init_i2c(client, &tps68470_regmap_config);
 	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Failed to create regmap: %ld\n", PTR_ERR(regmap));
+		dev_err(&client->dev, "Failed to create regmap: %pe\n", regmap);
 		return PTR_ERR(regmap);
 	}
 
diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c
index 9f77c0716e5983..71be313bdf0e1c 100644
--- a/drivers/platform/x86/intel/pmc/core.c
+++ b/drivers/platform/x86/intel/pmc/core.c
@@ -1849,6 +1849,7 @@ static const struct x86_cpu_id intel_pmc_core_ids[] = {
 	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&arl_h_pmc_dev),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_pmc_dev),
 	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&ptl_pmc_dev),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_R,	&ptl_pmc_dev),
 	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,	&wcl_pmc_dev),
 	X86_MATCH_VFM(INTEL_NOVALAKE,		&nvl_s_pmc_dev),
 	X86_MATCH_VFM(INTEL_NOVALAKE_L,		&nvl_h_pmc_dev),
diff --git a/drivers/platform/x86/intel/pmt/class.c b/drivers/platform/x86/intel/pmt/class.c
index d0ab8e33c62a65..2b11c4012f6a10 100644
--- a/drivers/platform/x86/intel/pmt/class.c
+++ b/drivers/platform/x86/intel/pmt/class.c
@@ -12,6 +12,7 @@
 #include <linux/log2.h>
 #include <linux/intel_vsec.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/ioport.h>
 #include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/mm.h>
@@ -100,7 +101,7 @@ intel_pmt_read(struct file *filp, struct kobject *kobj,
 	if (count > entry->size - off)
 		count = entry->size - off;
 
-	count = pmt_telem_read_mmio(entry->ep->dev, entry->cb, entry->header.guid, buf,
+	count = pmt_telem_read_mmio(entry->dev, entry->cb, entry->header.guid, buf,
 				    entry->base, off, count);
 
 	return count;
@@ -286,8 +287,6 @@ static int pmt_resolve_access_pci(struct intel_pmt_entry *entry,
 		return -EINVAL;
 	}
 
-	entry->pcidev = pci_dev;
-
 	return 0;
 }
 
@@ -365,6 +364,7 @@ static int intel_pmt_populate_entry(struct intel_pmt_entry *entry,
 	entry->guid = header->guid;
 	entry->size = header->size;
 	entry->cb = ivdev->priv_data;
+	entry->dev = ivdev->dev;
 
 	return 0;
 }
@@ -374,7 +374,6 @@ static int intel_pmt_dev_register(struct intel_pmt_entry *entry,
 				  struct device *parent)
 {
 	struct intel_vsec_device *ivdev = dev_to_ivdev(parent);
-	struct resource res = {0};
 	struct device *dev;
 	int ret;
 
@@ -404,20 +403,32 @@ static int intel_pmt_dev_register(struct intel_pmt_entry *entry,
 	if (!entry->size)
 		return 0;
 
-	res.start = entry->base_addr;
-	res.end = res.start + entry->size - 1;
-	res.flags = IORESOURCE_MEM;
+	/*
+	 * The read_telem callback is responsible for this mapping, and may have
+	 * different requirements for use. If the callback is present do not
+	 * create the map.
+	 */
+	if (!(entry->cb && entry->cb->read_telem)) {
+		struct resource res = DEFINE_RES_MEM(entry->base_addr, entry->size);
 
-	entry->base = devm_ioremap_resource(dev, &res);
-	if (IS_ERR(entry->base)) {
-		ret = PTR_ERR(entry->base);
-		goto fail_ioremap;
+		entry->base = devm_ioremap_resource(dev, &res);
+		if (IS_ERR(entry->base)) {
+			ret = PTR_ERR(entry->base);
+			goto fail_ioremap;
+		}
 	}
 
 	sysfs_bin_attr_init(&entry->pmt_bin_attr);
 	entry->pmt_bin_attr.attr.name = ns->name;
 	entry->pmt_bin_attr.attr.mode = 0440;
-	entry->pmt_bin_attr.mmap = intel_pmt_mmap;
+
+	/*
+	 * The read_telem callback controls access to the memory area. Block
+	 * usage if the read_telem callback is present.
+	 */
+	if (!(entry->cb && entry->cb->read_telem))
+		entry->pmt_bin_attr.mmap = intel_pmt_mmap;
+
 	entry->pmt_bin_attr.read = intel_pmt_read;
 	entry->pmt_bin_attr.size = entry->size;
 
diff --git a/drivers/platform/x86/intel/pmt/class.h b/drivers/platform/x86/intel/pmt/class.h
index a0ece4fc3837d3..aeb04afb67cf69 100644
--- a/drivers/platform/x86/intel/pmt/class.h
+++ b/drivers/platform/x86/intel/pmt/class.h
@@ -20,7 +20,6 @@
 #define GET_ADDRESS(v)		((v) & GENMASK(31, 3))
 
 struct device;
-struct pci_dev;
 extern struct class intel_pmt_class;
 
 struct telem_endpoint {
@@ -42,14 +41,14 @@ struct intel_pmt_header {
 
 struct intel_pmt_entry {
 	struct telem_endpoint	*ep;
-	struct pci_dev		*pcidev;
+	struct device		*dev;
 	struct intel_pmt_header	header;
 	u32			disc_header[PMT_DISC_DWORDS];
 	struct bin_attribute	pmt_bin_attr;
 	const struct attribute_group *attr_grp;
 	struct kobject		*kobj;
 	void __iomem		*disc_table;
-	void __iomem		*base;
+	void __iomem		*base;  /* unused if cb->read_telem is in use */
 	struct pmt_callbacks	*cb;
 	unsigned long		base_addr;
 	size_t			size;
diff --git a/drivers/platform/x86/intel/pmt/crashlog.c b/drivers/platform/x86/intel/pmt/crashlog.c
index f936daf99e4d24..d7018b495217ca 100644
--- a/drivers/platform/x86/intel/pmt/crashlog.c
+++ b/drivers/platform/x86/intel/pmt/crashlog.c
@@ -11,10 +11,12 @@
 #include <linux/auxiliary_bus.h>
 #include <linux/cleanup.h>
 #include <linux/intel_vsec.h>
+#include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
+#include <linux/printk.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/overflow.h>
@@ -124,12 +126,36 @@ struct pmt_crashlog_priv {
  * I/O
  */
 
+static int pmt_crashlog_read_reg(struct intel_pmt_entry *entry, u32 *reg, u32 offset)
+{
+	int ret;
+
+	*reg = 0;
+
+	if (entry->cb && entry->cb->read_reg) {
+		ret = entry->cb->read_reg(entry->dev, entry->header.guid, reg, offset);
+		if (ret) {
+			dev_err_ratelimited(entry->dev, "failed to read reg: %d\n", ret);
+			return ret;
+		}
+	} else {
+		*reg = readl(entry->disc_table + offset);
+	}
+
+	return 0;
+}
+
 /* Read, modify, write the control register, setting or clearing @bit based on @set */
-static void pmt_crashlog_rmw(struct crashlog_entry *crashlog, u32 bit, bool set)
+static int pmt_crashlog_rmw(struct crashlog_entry *crashlog, u32 bit, bool set)
 {
 	const struct crashlog_control *control = &crashlog->info->control;
 	struct intel_pmt_entry *entry = &crashlog->entry;
-	u32 reg = readl(entry->disc_table + control->offset);
+	u32 reg;
+	int ret;
+
+	ret = pmt_crashlog_read_reg(entry, &reg, control->offset);
+	if (ret)
+		return ret;
 
 	reg &= ~control->trigger_mask;
 
@@ -138,28 +164,46 @@ static void pmt_crashlog_rmw(struct crashlog_entry *crashlog, u32 bit, bool set)
 	else
 		reg &= ~bit;
 
-	writel(reg, entry->disc_table + control->offset);
+	if (entry->cb && entry->cb->write_reg) {
+		ret = entry->cb->write_reg(entry->dev, entry->header.guid, reg, control->offset);
+		if (ret) {
+			dev_err_ratelimited(entry->dev, "failed to write reg: %d\n", ret);
+			return ret;
+		}
+	} else {
+		writel(reg, entry->disc_table + control->offset);
+	}
+
+	return 0;
 }
 
 /* Read the status register and see if the specified @bit is set */
-static bool pmt_crashlog_rc(struct crashlog_entry *crashlog, u32 bit)
+static int pmt_crashlog_rc(struct crashlog_entry *crashlog, u32 bit, bool *state)
 {
 	const struct crashlog_status *status = &crashlog->info->status;
-	u32 reg = readl(crashlog->entry.disc_table + status->offset);
+	struct intel_pmt_entry *entry = &crashlog->entry;
+	u32 reg;
+	int ret;
+
+	ret = pmt_crashlog_read_reg(entry, &reg, status->offset);
+	if (ret)
+		return ret;
+
+	*state = !!(reg & bit);
 
-	return !!(reg & bit);
+	return 0;
 }
 
-static bool pmt_crashlog_complete(struct crashlog_entry *crashlog)
+static int pmt_crashlog_complete(struct crashlog_entry *crashlog, bool *state)
 {
 	/* return current value of the crashlog complete flag */
-	return pmt_crashlog_rc(crashlog, crashlog->info->status.complete);
+	return pmt_crashlog_rc(crashlog, crashlog->info->status.complete, state);
 }
 
-static bool pmt_crashlog_disabled(struct crashlog_entry *crashlog)
+static int pmt_crashlog_disabled(struct crashlog_entry *crashlog, bool *state)
 {
 	/* return current value of the crashlog disabled flag */
-	return pmt_crashlog_rc(crashlog, crashlog->info->status.disabled);
+	return pmt_crashlog_rc(crashlog, crashlog->info->status.disabled, state);
 }
 
 static bool pmt_crashlog_supported(struct intel_pmt_entry *entry, u32 *crash_type, u32 *version)
@@ -181,50 +225,50 @@ static bool pmt_crashlog_supported(struct intel_pmt_entry *entry, u32 *crash_typ
 	return false;
 }
 
-static void pmt_crashlog_set_disable(struct crashlog_entry *crashlog,
-				     bool disable)
+static int pmt_crashlog_set_disable(struct crashlog_entry *crashlog,
+				    bool disable)
 {
-	pmt_crashlog_rmw(crashlog, crashlog->info->control.disable, disable);
+	return pmt_crashlog_rmw(crashlog, crashlog->info->control.disable, disable);
 }
 
-static void pmt_crashlog_set_clear(struct crashlog_entry *crashlog)
+static int pmt_crashlog_set_clear(struct crashlog_entry *crashlog)
 {
-	pmt_crashlog_rmw(crashlog, crashlog->info->control.clear, true);
+	return pmt_crashlog_rmw(crashlog, crashlog->info->control.clear, true);
 }
 
-static void pmt_crashlog_set_execute(struct crashlog_entry *crashlog)
+static int pmt_crashlog_set_execute(struct crashlog_entry *crashlog)
 {
-	pmt_crashlog_rmw(crashlog, crashlog->info->control.manual, true);
+	return pmt_crashlog_rmw(crashlog, crashlog->info->control.manual, true);
 }
 
-static bool pmt_crashlog_cleared(struct crashlog_entry *crashlog)
+static int pmt_crashlog_cleared(struct crashlog_entry *crashlog, bool *state)
 {
-	return pmt_crashlog_rc(crashlog, crashlog->info->status.cleared);
+	return pmt_crashlog_rc(crashlog, crashlog->info->status.cleared, state);
 }
 
-static bool pmt_crashlog_consumed(struct crashlog_entry *crashlog)
+static int pmt_crashlog_consumed(struct crashlog_entry *crashlog, bool *state)
 {
-	return pmt_crashlog_rc(crashlog, crashlog->info->status.consumed);
+	return pmt_crashlog_rc(crashlog, crashlog->info->status.consumed, state);
 }
 
-static void pmt_crashlog_set_consumed(struct crashlog_entry *crashlog)
+static int pmt_crashlog_set_consumed(struct crashlog_entry *crashlog)
 {
-	pmt_crashlog_rmw(crashlog, crashlog->info->control.consume, true);
+	return pmt_crashlog_rmw(crashlog, crashlog->info->control.consume, true);
 }
 
-static bool pmt_crashlog_error(struct crashlog_entry *crashlog)
+static int pmt_crashlog_error(struct crashlog_entry *crashlog, bool *state)
 {
-	return pmt_crashlog_rc(crashlog, crashlog->info->status.error);
+	return pmt_crashlog_rc(crashlog, crashlog->info->status.error, state);
 }
 
-static bool pmt_crashlog_rearm(struct crashlog_entry *crashlog)
+static int pmt_crashlog_rearm(struct crashlog_entry *crashlog, bool *state)
 {
-	return pmt_crashlog_rc(crashlog, crashlog->info->status.rearmed);
+	return pmt_crashlog_rc(crashlog, crashlog->info->status.rearmed, state);
 }
 
-static void pmt_crashlog_set_rearm(struct crashlog_entry *crashlog)
+static int pmt_crashlog_set_rearm(struct crashlog_entry *crashlog)
 {
-	pmt_crashlog_rmw(crashlog, crashlog->info->control.rearm, true);
+	return pmt_crashlog_rmw(crashlog, crashlog->info->control.rearm, true);
 }
 
 /*
@@ -234,7 +278,12 @@ static ssize_t
 clear_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct crashlog_entry *crashlog = dev_get_drvdata(dev);
-	bool cleared = pmt_crashlog_cleared(crashlog);
+	bool cleared;
+	int ret;
+
+	ret = pmt_crashlog_cleared(crashlog, &cleared);
+	if (ret)
+		return ret;
 
 	return sysfs_emit(buf, "%d\n", cleared);
 }
@@ -245,13 +294,13 @@ clear_store(struct device *dev, struct device_attribute *attr,
 {
 	struct crashlog_entry *crashlog;
 	bool clear;
-	int result;
+	int ret;
 
 	crashlog = dev_get_drvdata(dev);
 
-	result = kstrtobool(buf, &clear);
-	if (result)
-		return result;
+	ret = kstrtobool(buf, &clear);
+	if (ret)
+		return ret;
 
 	/* set bit only */
 	if (!clear)
@@ -259,9 +308,9 @@ clear_store(struct device *dev, struct device_attribute *attr,
 
 	guard(mutex)(&crashlog->control_mutex);
 
-	pmt_crashlog_set_clear(crashlog);
+	ret = pmt_crashlog_set_clear(crashlog);
 
-	return count;
+	return ret ?: count;
 }
 static DEVICE_ATTR_RW(clear);
 
@@ -269,7 +318,12 @@ static ssize_t
 consumed_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct crashlog_entry *crashlog = dev_get_drvdata(dev);
-	bool consumed = pmt_crashlog_consumed(crashlog);
+	bool consumed;
+	int ret;
+
+	ret = pmt_crashlog_consumed(crashlog, &consumed);
+	if (ret)
+		return ret;
 
 	return sysfs_emit(buf, "%d\n", consumed);
 }
@@ -279,14 +333,16 @@ consumed_store(struct device *dev, struct device_attribute *attr, const char *bu
 	       size_t count)
 {
 	struct crashlog_entry *crashlog;
+	bool complete;
 	bool consumed;
-	int result;
+	bool disabled;
+	int ret;
 
 	crashlog = dev_get_drvdata(dev);
 
-	result = kstrtobool(buf, &consumed);
-	if (result)
-		return result;
+	ret = kstrtobool(buf, &consumed);
+	if (ret)
+		return ret;
 
 	/* set bit only */
 	if (!consumed)
@@ -294,15 +350,21 @@ consumed_store(struct device *dev, struct device_attribute *attr, const char *bu
 
 	guard(mutex)(&crashlog->control_mutex);
 
-	if (pmt_crashlog_disabled(crashlog))
+	ret = pmt_crashlog_disabled(crashlog, &disabled);
+	if (ret)
+		return ret;
+	if (disabled)
 		return -EBUSY;
 
-	if (!pmt_crashlog_complete(crashlog))
+	ret = pmt_crashlog_complete(crashlog, &complete);
+	if (ret)
+		return ret;
+	if (!complete)
 		return -EEXIST;
 
-	pmt_crashlog_set_consumed(crashlog);
+	ret = pmt_crashlog_set_consumed(crashlog);
 
-	return count;
+	return ret ?: count;
 }
 static DEVICE_ATTR_RW(consumed);
 
@@ -310,9 +372,14 @@ static ssize_t
 enable_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct crashlog_entry *crashlog = dev_get_drvdata(dev);
-	bool enabled = !pmt_crashlog_disabled(crashlog);
+	bool disabled;
+	int ret;
+
+	ret = pmt_crashlog_disabled(crashlog, &disabled);
+	if (ret)
+		return ret;
 
-	return sprintf(buf, "%d\n", enabled);
+	return sysfs_emit(buf, "%d\n", !disabled);
 }
 
 static ssize_t
@@ -321,19 +388,19 @@ enable_store(struct device *dev, struct device_attribute *attr,
 {
 	struct crashlog_entry *crashlog;
 	bool enabled;
-	int result;
+	int ret;
 
 	crashlog = dev_get_drvdata(dev);
 
-	result = kstrtobool(buf, &enabled);
-	if (result)
-		return result;
+	ret = kstrtobool(buf, &enabled);
+	if (ret)
+		return ret;
 
 	guard(mutex)(&crashlog->control_mutex);
 
-	pmt_crashlog_set_disable(crashlog, !enabled);
+	ret = pmt_crashlog_set_disable(crashlog, !enabled);
 
-	return count;
+	return ret ?: count;
 }
 static DEVICE_ATTR_RW(enable);
 
@@ -341,7 +408,12 @@ static ssize_t
 error_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct crashlog_entry *crashlog = dev_get_drvdata(dev);
-	bool error = pmt_crashlog_error(crashlog);
+	bool error;
+	int ret;
+
+	ret = pmt_crashlog_error(crashlog, &error);
+	if (ret)
+		return ret;
 
 	return sysfs_emit(buf, "%d\n", error);
 }
@@ -351,7 +423,12 @@ static ssize_t
 rearm_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct crashlog_entry *crashlog = dev_get_drvdata(dev);
-	int rearmed = pmt_crashlog_rearm(crashlog);
+	bool rearmed;
+	int ret;
+
+	ret = pmt_crashlog_rearm(crashlog, &rearmed);
+	if (ret)
+		return ret;
 
 	return sysfs_emit(buf, "%d\n", rearmed);
 }
@@ -362,13 +439,13 @@ rearm_store(struct device *dev, struct device_attribute *attr, const char *buf,
 {
 	struct crashlog_entry *crashlog;
 	bool rearm;
-	int result;
+	int ret;
 
 	crashlog = dev_get_drvdata(dev);
 
-	result = kstrtobool(buf, &rearm);
-	if (result)
-		return result;
+	ret = kstrtobool(buf, &rearm);
+	if (ret)
+		return ret;
 
 	/* set only */
 	if (!rearm)
@@ -376,9 +453,9 @@ rearm_store(struct device *dev, struct device_attribute *attr, const char *buf,
 
 	guard(mutex)(&crashlog->control_mutex);
 
-	pmt_crashlog_set_rearm(crashlog);
+	ret = pmt_crashlog_set_rearm(crashlog);
 
-	return count;
+	return ret ?: count;
 }
 static DEVICE_ATTR_RW(rearm);
 
@@ -387,11 +464,15 @@ trigger_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct crashlog_entry *crashlog;
 	bool trigger;
+	int ret;
 
 	crashlog = dev_get_drvdata(dev);
-	trigger = pmt_crashlog_complete(crashlog);
 
-	return sprintf(buf, "%d\n", trigger);
+	ret = pmt_crashlog_complete(crashlog, &trigger);
+	if (ret)
+		return ret;
+
+	return sysfs_emit(buf, "%d\n", trigger);
 }
 
 static ssize_t
@@ -399,33 +480,41 @@ trigger_store(struct device *dev, struct device_attribute *attr,
 	      const char *buf, size_t count)
 {
 	struct crashlog_entry *crashlog;
+	bool complete;
+	bool disabled;
 	bool trigger;
-	int result;
+	int ret;
 
 	crashlog = dev_get_drvdata(dev);
 
-	result = kstrtobool(buf, &trigger);
-	if (result)
-		return result;
+	ret = kstrtobool(buf, &trigger);
+	if (ret)
+		return ret;
 
 	guard(mutex)(&crashlog->control_mutex);
 
 	/* if device is currently disabled, return busy */
-	if (pmt_crashlog_disabled(crashlog))
+	ret = pmt_crashlog_disabled(crashlog, &disabled);
+	if (ret)
+		return ret;
+	if (disabled)
 		return -EBUSY;
 
 	if (!trigger) {
-		pmt_crashlog_set_clear(crashlog);
-		return count;
+		ret = pmt_crashlog_set_clear(crashlog);
+		return ret ?: count;
 	}
 
 	/* we cannot trigger a new crash if one is still pending */
-	if (pmt_crashlog_complete(crashlog))
+	ret = pmt_crashlog_complete(crashlog, &complete);
+	if (ret)
+		return ret;
+	if (complete)
 		return -EEXIST;
 
-	pmt_crashlog_set_execute(crashlog);
+	ret = pmt_crashlog_set_execute(crashlog);
 
-	return count;
+	return ret ?: count;
 }
 static DEVICE_ATTR_RW(trigger);
 
diff --git a/drivers/platform/x86/intel/pmt/discovery.c b/drivers/platform/x86/intel/pmt/discovery.c
index c482368bfaaeff..f4203d240f5423 100644
--- a/drivers/platform/x86/intel/pmt/discovery.c
+++ b/drivers/platform/x86/intel/pmt/discovery.c
@@ -609,7 +609,7 @@ void intel_pmt_get_features(struct intel_pmt_entry *entry)
 
 	mutex_lock(&feature_list_lock);
 	list_for_each_entry(feature, &pmt_feature_list, list) {
-		if (feature->priv->parent != entry->ep->dev)
+		if (feature->priv->parent != entry->dev)
 			continue;
 
 		pmt_get_features(entry, feature);
diff --git a/drivers/platform/x86/intel/pmt/telemetry.c b/drivers/platform/x86/intel/pmt/telemetry.c
index 953f35b6daecb6..5423a4dfce088c 100644
--- a/drivers/platform/x86/intel/pmt/telemetry.c
+++ b/drivers/platform/x86/intel/pmt/telemetry.c
@@ -329,6 +329,9 @@ int pmt_telem_read32(struct telem_endpoint *ep, u32 id, u32 *data, u32 count)
 	if (!ep->present)
 		return -ENODEV;
 
+	if (!ep->base)
+		return -EIO;
+
 	offset = SAMPLE_ID_OFFSET32(id);
 	size = ep->header.size;
 
diff --git a/drivers/platform/x86/intel/punit_ipc.c b/drivers/platform/x86/intel/punit_ipc.c
index 6d770b950dfb8a..a3d2b0b64135e0 100644
--- a/drivers/platform/x86/intel/punit_ipc.c
+++ b/drivers/platform/x86/intel/punit_ipc.c
@@ -250,10 +250,8 @@ static int intel_punit_ipc_probe(struct platform_device *pdev)
 		ret = devm_request_irq(&pdev->dev, irq, intel_punit_ioc,
 				       IRQF_NO_SUSPEND, "intel_punit_ipc",
 				       punit_ipcdev);
-		if (ret) {
-			dev_err(&pdev->dev, "Failed to request irq: %d\n", irq);
+		if (ret)
 			return ret;
-		}
 		punit_ipcdev->irq = irq;
 	}
 
diff --git a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
index 1c48bf6d545794..4c04f0bc9b472a 100644
--- a/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
+++ b/drivers/platform/x86/intel/speed_select_if/isst_if_common.c
@@ -7,6 +7,7 @@
  * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
  */
 
+#include <linux/acpi.h>
 #include <linux/cpufeature.h>
 #include <linux/cpuhotplug.h>
 #include <linux/fs.h>
@@ -774,6 +775,54 @@ void isst_if_cdev_unregister(int device_type)
 }
 EXPORT_SYMBOL_GPL(isst_if_cdev_unregister);
 
+#ifdef CONFIG_ACPI
+
+static bool isst_acpi_pm_profile_server(void)
+{
+	if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
+	    acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
+		return true;
+
+	return false;
+}
+
+#else
+
+static bool isst_acpi_pm_profile_server(void)
+{
+	return false;
+}
+
+#endif
+
+static const struct x86_cpu_id isst_allowed_families[] = {
+	X86_MATCH_VENDOR_FAM(INTEL, 19, NULL),
+	{}
+};
+
+static bool isst_features_allowed(void)
+{
+	u64 data;
+
+	/* For hypervisors, explicit model addition is required */
+	if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
+		return false;
+
+	/* SST is a server only feature */
+	if (!isst_acpi_pm_profile_server())
+		return false;
+
+	/*
+	 * Family 19 and the checks above guarantee the SST enumeration
+	 * interfaces are present
+	 */
+	if (!x86_match_cpu(isst_allowed_families))
+		return false;
+
+	/* MSR_PM_LOGICAL_ID is required to get PM logical CPU ID */
+	return !rdmsrq_safe(MSR_PM_LOGICAL_ID, &data);
+}
+
 #define SST_HPM_SUPPORTED	0x01
 #define SST_MBOX_SUPPORTED	0x02
 
@@ -798,8 +847,13 @@ static int __init isst_if_common_init(void)
 	const struct x86_cpu_id *id;
 
 	id = x86_match_cpu(isst_cpu_ids);
-	if (!id)
+	if (!id) {
+		if (isst_features_allowed()) {
+			isst_hpm_support = true;
+			goto misc_reg;
+		}
 		return -ENODEV;
+	}
 
 	if (id->driver_data == SST_HPM_SUPPORTED) {
 		isst_hpm_support = true;
@@ -812,6 +866,7 @@ static int __init isst_if_common_init(void)
 			return -ENODEV;
 	}
 
+misc_reg:
 	return isst_misc_reg();
 }
 module_init(isst_if_common_init)
diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c
index 667f2c8b9594af..3bc57b553eff15 100644
--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c
+++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency.c
@@ -257,6 +257,7 @@ static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
 	X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M, NULL),
 	X86_MATCH_VFM(INTEL_PANTHERLAKE_L, NULL),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_R, NULL),
 	X86_MATCH_VFM(INTEL_WILDCATLAKE_L, NULL),
 	X86_MATCH_VFM(INTEL_NOVALAKE, NULL),
 	X86_MATCH_VFM(INTEL_NOVALAKE_L, NULL),
diff --git a/drivers/platform/x86/intel/vsec.c b/drivers/platform/x86/intel/vsec.c
index 5ab2215fdd7fa8..f86149e9ba16c8 100644
--- a/drivers/platform/x86/intel/vsec.c
+++ b/drivers/platform/x86/intel/vsec.c
@@ -51,6 +51,13 @@ struct vsec_priv {
 	unsigned long found_caps;
 };
 
+static void intel_vsec_reset_state(struct vsec_priv *priv)
+{
+	memset(priv->suppliers, 0, sizeof(priv->suppliers));
+	memset(priv->state, 0, sizeof(priv->state));
+	priv->found_caps = 0;
+}
+
 static const char *intel_vsec_name(enum intel_vsec_id id)
 {
 	switch (id) {
@@ -669,8 +676,8 @@ static int intel_vsec_pci_init(struct pci_dev *pdev)
 {
 	struct vsec_priv *priv = pci_get_drvdata(pdev);
 	const struct intel_vsec_platform_info *info = priv->info;
-	int run_once = 0;
 	bool found_any = false;
+	bool run_once = false;
 	int num_caps;
 
 	num_caps = hweight_long(info->caps);
@@ -682,7 +689,7 @@ static int intel_vsec_pci_init(struct pci_dev *pdev)
 
 		if (!run_once) {
 			intel_vsec_skip_missing_dependencies(pdev);
-			run_once = 1;
+			run_once = true;
 		}
 	}
 
@@ -857,6 +864,7 @@ static pci_ers_result_t intel_vsec_pci_slot_reset(struct pci_dev *pdev)
 {
 	struct intel_vsec_device *intel_vsec_dev;
 	pci_ers_result_t status = PCI_ERS_RESULT_DISCONNECT;
+	struct vsec_priv *priv = pci_get_drvdata(pdev);
 	unsigned long index;
 
 	dev_info(&pdev->dev, "Resetting PCI slot\n");
@@ -877,6 +885,13 @@ static pci_ers_result_t intel_vsec_pci_slot_reset(struct pci_dev *pdev)
 		devm_release_action(&pdev->dev, intel_vsec_remove_aux,
 				    &intel_vsec_dev->auxdev);
 	}
+
+	/*
+	 * intel_vsec_pci_init() reuses priv. Reset the registration state
+	 * after tearing down old aux devices or the rewalk will skip them as
+	 * already registered.
+	 */
+	intel_vsec_reset_state(priv);
 	pci_restore_state(pdev);
 	intel_vsec_pci_init(pdev);
 
diff --git a/drivers/platform/x86/intel/vsec_tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c
index 0153dd57838e70..fbe9c8af4aeea2 100644
--- a/drivers/platform/x86/intel/vsec_tpmi.c
+++ b/drivers/platform/x86/intel/vsec_tpmi.c
@@ -597,7 +597,7 @@ static const char *intel_tpmi_name(enum intel_tpmi_id id)
 	}
 }
 
-/* String Length for tpmi-"feature_name(upto 8 bytes)" */
+/* String Length for tpmi-"feature_name(up to 8 bytes)" */
 #define TPMI_FEATURE_NAME_LEN	14
 
 static int tpmi_create_device(struct intel_tpmi_info *tpmi_info,
diff --git a/drivers/platform/x86/lenovo/ideapad-laptop.c b/drivers/platform/x86/lenovo/ideapad-laptop.c
index 8213524504eeb3..207368f5d48973 100644
--- a/drivers/platform/x86/lenovo/ideapad-laptop.c
+++ b/drivers/platform/x86/lenovo/ideapad-laptop.c
@@ -1358,8 +1358,12 @@ static const struct key_entry ideapad_keymap[] = {
 	{ KE_KEY,	0x44 | IDEAPAD_WMI_KEY, { KEY_PROG1 } },
 	/* Eye */
 	{ KE_KEY,	0x45 | IDEAPAD_WMI_KEY, { KEY_PROG3 } },
+	/* Lenovo Smart Connect */
+	{ KE_KEY,	0x48 | IDEAPAD_WMI_KEY, { KEY_LINK_PHONE } },
 	/* Performance toggle also Fn+Q, handled inside ideapad_wmi_notify() */
 	{ KE_KEY,	0x3d | IDEAPAD_WMI_KEY, { KEY_PROG4 } },
+	/* Mode, opens a profile chooser; the Fn+Q button cycles the profile */
+	{ KE_KEY,	0x46 | IDEAPAD_WMI_KEY, { KEY_PROG4 } },
 	/* shift + prtsc */
 	{ KE_KEY,   0x2d | IDEAPAD_WMI_KEY, { KEY_SELECTIVE_SCREENSHOT } },
 	{ KE_KEY,   0x29 | IDEAPAD_WMI_KEY, { KEY_TOUCHPAD_TOGGLE } },
diff --git a/drivers/platform/x86/lenovo/think-lmi.c b/drivers/platform/x86/lenovo/think-lmi.c
index a0e3fa766e37b5..bd8973c1369db5 100644
--- a/drivers/platform/x86/lenovo/think-lmi.c
+++ b/drivers/platform/x86/lenovo/think-lmi.c
@@ -249,7 +249,7 @@ static int tlmi_errstr_to_err(const char *errstr)
 {
 	int i;
 
-	for (i = 0; i < sizeof(tlmi_errs)/sizeof(struct tlmi_err_codes); i++) {
+	for (i = 0; i < ARRAY_SIZE(tlmi_errs); i++) {
 		if (!strcmp(tlmi_errs[i].err_str, errstr))
 			return tlmi_errs[i].err_code;
 	}
diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c
index 1661f070c5718a..1225b01ed14d34 100644
--- a/drivers/platform/x86/lenovo/thinkpad_acpi.c
+++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c
@@ -222,8 +222,10 @@ enum tpacpi_hkey_event_t {
 	TP_HKEY_EV_LID_OPEN		= 0x5002, /* laptop lid opened */
 	TP_HKEY_EV_TABLET_TABLET	= 0x5009, /* tablet swivel up */
 	TP_HKEY_EV_TABLET_NOTEBOOK	= 0x500a, /* tablet swivel down */
-	TP_HKEY_EV_TABLET_CHANGED	= 0x60c0, /* X1 Yoga (2016):
-						   * enter/leave tablet mode
+	TP_HKEY_EV_TABLET_CHANGED	= 0x60c0, /* posture change event:
+						   * X1 Yoga (2016): enter/leave tablet mode
+						   * X1 Fold 16 Gen 1: keyboard
+						   * attachment state changed
 						   */
 	TP_HKEY_EV_PEN_INSERTED		= 0x500b, /* tablet pen inserted */
 	TP_HKEY_EV_PEN_REMOVED		= 0x500c, /* tablet pen removed */
@@ -379,6 +381,7 @@ static struct {
 	u32 kbd_lang:1;
 	u32 trackpoint_doubletap_enable:1;
 	u32 usbc_security_supported:1;
+	u32 has_keyboard_attached_on_screen:1;
 	bool usbc_security_enabled;
 	struct quirk_entry *quirks;
 } tp_features;
@@ -456,9 +459,7 @@ do {									\
 
 #ifdef CONFIG_THINKPAD_ACPI_DEBUG
 #define vdbg_printk dbg_printk
-static const char *str_supported(int is_supported);
 #else
-static inline const char *str_supported(int is_supported) { return ""; }
 #define vdbg_printk(a_dbg_level, format, arg...)	\
 	do { if (0) no_printk(format, ##arg); } while (0)
 #endif
@@ -2169,7 +2170,7 @@ static int tpacpi_hotkey_driver_mask_set(const u32 mask)
 		return 0;
 	}
 
-	mutex_lock(&hotkey_mutex);
+	guard(mutex)(&hotkey_mutex);
 
 	HOTKEY_CONFIG_CRITICAL_START
 	hotkey_driver_mask = mask;
@@ -2182,8 +2183,6 @@ static int tpacpi_hotkey_driver_mask_set(const u32 mask)
 							~hotkey_source_mask);
 	hotkey_poll_setup(true);
 
-	mutex_unlock(&hotkey_mutex);
-
 	return rc;
 }
 
@@ -2207,15 +2206,12 @@ static void tpacpi_input_send_tabletsw(void)
 {
 	int state;
 
-	if (tp_features.hotkey_tablet &&
-	    !hotkey_get_tablet_mode(&state)) {
-		mutex_lock(&tpacpi_inputdev_send_mutex);
+	if (tp_features.hotkey_tablet && !hotkey_get_tablet_mode(&state)) {
+		guard(mutex)(&tpacpi_inputdev_send_mutex);
 
 		input_report_switch(tpacpi_inputdev,
 				    SW_TABLET_MODE, !!state);
 		input_sync(tpacpi_inputdev);
-
-		mutex_unlock(&tpacpi_inputdev_send_mutex);
 	}
 }
 
@@ -2240,7 +2236,6 @@ static int get_camera_shutter(void)
 
 static bool tpacpi_input_send_key(const u32 hkey, bool *send_acpi_ev)
 {
-	bool known_ev;
 	u32 scancode;
 
 	if (tpacpi_driver_event(hkey))
@@ -2283,11 +2278,8 @@ static bool tpacpi_input_send_key(const u32 hkey, bool *send_acpi_ev)
 		scancode = hkey;
 	}
 
-	mutex_lock(&tpacpi_inputdev_send_mutex);
-	known_ev = sparse_keymap_report_event(tpacpi_inputdev, scancode, 1, true);
-	mutex_unlock(&tpacpi_inputdev_send_mutex);
-
-	return known_ev;
+	guard(mutex)(&tpacpi_inputdev_send_mutex);
+	return sparse_keymap_report_event(tpacpi_inputdev, scancode, 1, true);
 }
 
 #ifdef CONFIG_THINKPAD_ACPI_HOTKEY_POLL
@@ -2552,9 +2544,10 @@ static void hotkey_poll_setup(const bool may_warn)
 
 	lockdep_assert_held(&hotkey_mutex);
 
+	guard(mutex)(&tpacpi_inputdev->mutex);
 	if (hotkey_poll_freq > 0 &&
 	    (poll_driver_mask ||
-	     (poll_user_mask && tpacpi_inputdev->users > 0))) {
+	     (poll_user_mask && input_device_enabled(tpacpi_inputdev)))) {
 		if (!tpacpi_hotkey_task) {
 			tpacpi_hotkey_task = kthread_run(hotkey_kthread,
 					NULL, TPACPI_NVRAM_KTHREAD_NAME);
@@ -2575,9 +2568,8 @@ static void hotkey_poll_setup(const bool may_warn)
 
 static void hotkey_poll_setup_safe(const bool may_warn)
 {
-	mutex_lock(&hotkey_mutex);
+	guard(mutex)(&hotkey_mutex);
 	hotkey_poll_setup(may_warn);
-	mutex_unlock(&hotkey_mutex);
 }
 
 static void hotkey_poll_set_freq(unsigned int freq)
@@ -2893,6 +2885,63 @@ static void hotkey_tablet_mode_notify_change(void)
 			     "hotkey_tablet_mode");
 }
 
+static bool keyboard_attached_on_screen;
+static bool keyboard_attached_on_screen_initialized;
+
+static int x1_fold_keyboard_attached_on_screen_get(bool *attached)
+{
+	int state;
+
+	if (!tp_features.has_keyboard_attached_on_screen)
+		return -ENODEV;
+
+	if (!acpi_evalf(NULL, &state, "\\_SB.DEVD.GDST", "d"))
+		return -EIO;
+
+	*attached = state != 0;
+	return 0;
+}
+
+static ssize_t keyboard_attached_on_screen_show(struct device *dev,
+						struct device_attribute *attr,
+						char *buf)
+{
+	bool attached;
+	int res;
+
+	res = x1_fold_keyboard_attached_on_screen_get(&attached);
+	if (res)
+		return res;
+
+	return sysfs_emit(buf, "%d\n", attached);
+}
+
+static DEVICE_ATTR_RO(keyboard_attached_on_screen);
+
+static void keyboard_attached_on_screen_notify_change(void)
+{
+	if (tp_features.has_keyboard_attached_on_screen)
+		sysfs_notify(&tpacpi_pdev->dev.kobj, NULL,
+			     "keyboard_attached_on_screen");
+}
+
+static bool keyboard_attached_on_screen_update(void)
+{
+	bool attached;
+
+	if (x1_fold_keyboard_attached_on_screen_get(&attached))
+		return false;
+
+	if (keyboard_attached_on_screen_initialized &&
+	    keyboard_attached_on_screen == attached)
+		return false;
+
+	keyboard_attached_on_screen = attached;
+	keyboard_attached_on_screen_initialized = true;
+
+	return true;
+}
+
 /* sysfs wakeup reason (pollable) -------------------------------------- */
 static ssize_t hotkey_wakeup_reason_show(struct device *dev,
 			   struct device_attribute *attr,
@@ -3022,6 +3071,7 @@ static struct attribute *hotkey_attributes[] = {
 	&dev_attr_hotkey_adaptive_all_mask.attr,
 	&dev_attr_hotkey_recommended_mask.attr,
 	&dev_attr_hotkey_tablet_mode.attr,
+	&dev_attr_keyboard_attached_on_screen.attr,
 	&dev_attr_hotkey_radio_sw.attr,
 	&dev_attr_doubletap_enable.attr,
 #ifdef CONFIG_THINKPAD_ACPI_HOTKEY_POLL
@@ -3037,6 +3087,9 @@ static umode_t hotkey_attr_is_visible(struct kobject *kobj,
 	if (attr == &dev_attr_hotkey_tablet_mode.attr) {
 		if (!tp_features.hotkey_tablet)
 			return 0;
+	} else if (attr == &dev_attr_keyboard_attached_on_screen.attr) {
+		if (!tp_features.has_keyboard_attached_on_screen)
+			return 0;
 	} else if (attr == &dev_attr_hotkey_radio_sw.attr) {
 		if (!tp_features.hotkey_wlsw)
 			return 0;
@@ -3080,13 +3133,11 @@ static void tpacpi_send_radiosw_update(void)
 
 	/* Issue rfkill input event for WLSW switch */
 	if (!(wlsw < 0)) {
-		mutex_lock(&tpacpi_inputdev_send_mutex);
+		guard(mutex)(&tpacpi_inputdev_send_mutex);
 
 		input_report_switch(tpacpi_inputdev,
 				    SW_RFKILL_ALL, (wlsw > 0));
 		input_sync(tpacpi_inputdev);
-
-		mutex_unlock(&tpacpi_inputdev_send_mutex);
 	}
 
 	/*
@@ -3098,7 +3149,7 @@ static void tpacpi_send_radiosw_update(void)
 
 static void hotkey_exit(void)
 {
-	mutex_lock(&hotkey_mutex);
+	guard(mutex)(&hotkey_mutex);
 	hotkey_poll_stop_sync();
 	dbg_printk(TPACPI_DBG_EXIT | TPACPI_DBG_HKEY,
 		   "restoring original HKEY status and mask\n");
@@ -3108,8 +3159,6 @@ static void hotkey_exit(void)
 	      hotkey_mask_set(hotkey_orig_mask)) |
 	     hotkey_status_set(false)) != 0)
 		pr_err("failed to restore hot key mask to BIOS defaults\n");
-
-	mutex_unlock(&hotkey_mutex);
 }
 
 /*
@@ -3338,7 +3387,7 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 
 	vdbg_printk(TPACPI_DBG_INIT | TPACPI_DBG_HKEY,
 		"hotkeys are %s\n",
-		str_supported(tp_features.hotkey));
+		str_supported_unsupported(tp_features.hotkey));
 
 	if (!tp_features.hotkey)
 		return -ENODEV;
@@ -3415,7 +3464,7 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 
 	vdbg_printk(TPACPI_DBG_INIT | TPACPI_DBG_HKEY,
 		"hotkey masks are %s\n",
-		str_supported(tp_features.hotkey_mask));
+		str_supported_unsupported(tp_features.hotkey_mask));
 
 	/* Init hotkey_all_mask if not initialized yet */
 	if (!tp_features.hotkey_mask && !hotkey_all_mask &&
@@ -3426,11 +3475,11 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 	if (tp_features.hotkey_mask) {
 		/* hotkey_source_mask *must* be zero for
 		 * the first hotkey_mask_get to return hotkey_orig_mask */
-		mutex_lock(&hotkey_mutex);
-		res = hotkey_mask_get();
-		mutex_unlock(&hotkey_mutex);
-		if (res)
-			return res;
+		scoped_guard(mutex, &hotkey_mutex) {
+			res = hotkey_mask_get();
+			if (res)
+				return res;
+		}
 
 		hotkey_orig_mask = hotkey_acpi_mask;
 	} else {
@@ -3453,6 +3502,7 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 	}
 
 	tabletsw_state = hotkey_init_tablet_mode();
+	keyboard_attached_on_screen_update();
 
 	/* Set up key map */
 	keymap_id = tpacpi_check_quirks(tpacpi_keymap_qtable,
@@ -3529,11 +3579,11 @@ static int __init hotkey_init(struct ibm_init_struct *iibm)
 		hotkey_exit();
 		return res;
 	}
-	mutex_lock(&hotkey_mutex);
-	res = hotkey_mask_set(((hotkey_all_mask & ~hotkey_reserved_mask)
-			       | hotkey_driver_mask)
-			      & ~hotkey_source_mask);
-	mutex_unlock(&hotkey_mutex);
+	scoped_guard(mutex, &hotkey_mutex) {
+		res = hotkey_mask_set(((hotkey_all_mask & ~hotkey_reserved_mask)
+				       | hotkey_driver_mask)
+				      & ~hotkey_source_mask);
+	}
 	if (res < 0 && res != -ENXIO) {
 		hotkey_exit();
 		return res;
@@ -3833,6 +3883,8 @@ static bool hotkey_notify_6xxx(const u32 hkey, bool *send_acpi_ev)
 	case TP_HKEY_EV_TABLET_CHANGED:
 		tpacpi_input_send_tabletsw();
 		hotkey_tablet_mode_notify_change();
+		if (keyboard_attached_on_screen_update())
+			keyboard_attached_on_screen_notify_change();
 		*send_acpi_ev = false;
 		return true;
 
@@ -3980,15 +4032,17 @@ static void hotkey_resume(void)
 {
 	tpacpi_disable_brightness_delay();
 
-	mutex_lock(&hotkey_mutex);
-	if (hotkey_status_set(true) < 0 ||
-	    hotkey_mask_set(hotkey_acpi_mask) < 0)
-		pr_err("error while attempting to reset the event firmware interface\n");
-	mutex_unlock(&hotkey_mutex);
+	scoped_guard(mutex, &hotkey_mutex) {
+		if (hotkey_status_set(true) < 0 ||
+		    hotkey_mask_set(hotkey_acpi_mask) < 0)
+			pr_err("error while attempting to reset the event firmware interface\n");
+	}
 
 	tpacpi_send_radiosw_update();
 	tpacpi_input_send_tabletsw();
 	hotkey_tablet_mode_notify_change();
+	keyboard_attached_on_screen_update();
+	keyboard_attached_on_screen_notify_change();
 	hotkey_wakeup_reason_notify_change();
 	hotkey_wakeup_hotunplug_complete_notify_change();
 	hotkey_poll_setup_safe(false);
@@ -4287,6 +4341,17 @@ static const struct dmi_system_id fwbug_list[] __initconst = {
 	{}
 };
 
+static const struct dmi_system_id keyboard_attached_on_screen_list[] __initconst = {
+	{
+		.ident = "ThinkPad X1 Fold 16 Gen 1",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_FAMILY, "ThinkPad X1 Fold 16 Gen 1"),
+		},
+	},
+	{}
+};
+
 static const struct pci_device_id fwbug_cards_ids[] __initconst = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x24F3) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x24FD) },
@@ -4327,7 +4392,7 @@ static int __init bluetooth_init(struct ibm_init_struct *iibm)
 
 	vdbg_printk(TPACPI_DBG_INIT | TPACPI_DBG_RFKILL,
 		"bluetooth is %s, status 0x%02x\n",
-		str_supported(tp_features.bluetooth),
+		str_supported_unsupported(tp_features.bluetooth),
 		status);
 
 #ifdef CONFIG_THINKPAD_ACPI_DEBUGFACILITIES
@@ -4506,7 +4571,7 @@ static int __init wan_init(struct ibm_init_struct *iibm)
 
 	vdbg_printk(TPACPI_DBG_INIT | TPACPI_DBG_RFKILL,
 		"wan is %s, status 0x%02x\n",
-		str_supported(tp_features.wan),
+		str_supported_unsupported(tp_features.wan),
 		status);
 
 #ifdef CONFIG_THINKPAD_ACPI_DEBUGFACILITIES
@@ -4634,7 +4699,7 @@ static int __init uwb_init(struct ibm_init_struct *iibm)
 
 	vdbg_printk(TPACPI_DBG_INIT | TPACPI_DBG_RFKILL,
 		"uwb is %s, status 0x%02x\n",
-		str_supported(tp_features.uwb),
+		str_supported_unsupported(tp_features.uwb),
 		status);
 
 #ifdef CONFIG_THINKPAD_ACPI_DEBUGFACILITIES
@@ -4742,7 +4807,7 @@ static int __init video_init(struct ibm_init_struct *iibm)
 		video_supported = TPACPI_VIDEO_NEW;
 
 	vdbg_printk(TPACPI_DBG_INIT, "video is %s, mode %d\n",
-		str_supported(video_supported != TPACPI_VIDEO_NONE),
+		str_supported_unsupported(video_supported != TPACPI_VIDEO_NONE),
 		video_supported);
 
 	return (video_supported != TPACPI_VIDEO_NONE) ? 0 : -ENODEV;
@@ -5037,21 +5102,16 @@ static DEFINE_MUTEX(kbdlight_mutex);
 
 static int kbdlight_set_level(int level)
 {
-	int ret = 0;
-
 	if (!hkey_handle)
 		return -ENXIO;
 
-	mutex_lock(&kbdlight_mutex);
+	guard(mutex)(&kbdlight_mutex);
 
 	if (!acpi_evalf(hkey_handle, NULL, "MLCS", "dd", level))
-		ret = -EIO;
-	else
-		kbdlight_brightness = level;
-
-	mutex_unlock(&kbdlight_mutex);
+		return -EIO;
 
-	return ret;
+	kbdlight_brightness = level;
+	return 0;
 }
 
 static int kbdlight_get_level(void)
@@ -5340,8 +5400,8 @@ static int __init light_init(struct ibm_init_struct *iibm)
 			acpi_evalf(ec_handle, NULL, "KBLT", "qv");
 
 	vdbg_printk(TPACPI_DBG_INIT, "light is %s, light status is %s\n",
-		str_supported(tp_features.light),
-		str_supported(tp_features.light_status));
+		str_supported_unsupported(tp_features.light),
+		str_supported_unsupported(tp_features.light_status));
 
 	if (!tp_features.light)
 		return -ENODEV;
@@ -5458,7 +5518,7 @@ static int __init cmos_init(struct ibm_init_struct *iibm)
 	TPACPI_ACPIHANDLE_INIT(cmos);
 
 	vdbg_printk(TPACPI_DBG_INIT, "cmos commands are %s\n",
-		    str_supported(cmos_handle != NULL));
+		    str_supported_unsupported(cmos_handle));
 
 	return cmos_handle ? 0 : -ENODEV;
 }
@@ -5803,7 +5863,8 @@ static int __init led_init(struct ibm_init_struct *iibm)
 	}
 
 	vdbg_printk(TPACPI_DBG_INIT, "LED commands are %s, mode %d\n",
-		str_supported(led_supported), led_supported);
+		str_supported_unsupported(led_supported != TPACPI_LED_NONE),
+		led_supported);
 
 	if (led_supported == TPACPI_LED_NONE)
 		return -ENODEV;
@@ -5924,7 +5985,7 @@ static int __init beep_init(struct ibm_init_struct *iibm)
 	TPACPI_ACPIHANDLE_INIT(beep);
 
 	vdbg_printk(TPACPI_DBG_INIT, "beep is %s\n",
-		str_supported(beep_handle != NULL));
+		str_supported_unsupported(beep_handle));
 
 	quirks = tpacpi_check_quirks(beep_quirk_table,
 				     ARRAY_SIZE(beep_quirk_table));
@@ -6021,6 +6082,7 @@ static const struct tpacpi_quirk thermal_quirk_table[] __initconst = {
 	TPACPI_Q_LNV3('R', '0', 'T', true),	/* 11e Gen5 GL*/
 	TPACPI_Q_LNV3('R', '1', 'D', true),	/* 11e Gen5 GL-R*/
 	TPACPI_Q_LNV3('R', '0', 'V', true),	/* 11e Gen5 KL-Y*/
+	TPACPI_Q_LNV3('N', '4', 'D', true),	/* X9-14 Gen 1 */
 };
 
 static enum thermal_access_mode thermal_read_mode;
@@ -6374,7 +6436,7 @@ static int __init thermal_init(struct ibm_init_struct *iibm)
 	thermal_read_mode = thermal_read_mode_check();
 
 	vdbg_printk(TPACPI_DBG_INIT, "thermal is %s, mode %d\n",
-		str_supported(thermal_read_mode != TPACPI_THERMAL_NONE),
+		str_supported_unsupported(thermal_read_mode != TPACPI_THERMAL_NONE),
 		thermal_read_mode);
 
 	return thermal_read_mode != TPACPI_THERMAL_NONE ? 0 : -ENODEV;
@@ -7596,7 +7658,7 @@ static int __init volume_init(struct ibm_init_struct *iibm)
 
 	vdbg_printk(TPACPI_DBG_INIT | TPACPI_DBG_MIXER,
 			"mute is supported, volume control is %s\n",
-			str_supported(!tp_features.mixer_no_level_control));
+			str_supported_unsupported(!tp_features.mixer_no_level_control));
 
 	if (software_mute_requested && volume_set_software_mute(true) == 0) {
 		software_mute_active = true;
@@ -8845,7 +8907,9 @@ static const struct tpacpi_quirk fan_quirk_table[] __initconst = {
 	TPACPI_Q_LNV3('R', '0', 'T', TPACPI_FAN_NS),	/* 11e Gen5 GL */
 	TPACPI_Q_LNV3('R', '1', 'D', TPACPI_FAN_NS),	/* 11e Gen5 GL-R */
 	TPACPI_Q_LNV3('R', '0', 'V', TPACPI_FAN_NS),	/* 11e Gen5 KL-Y */
+	TPACPI_Q_LNV3('N', '4', 'D', TPACPI_FAN_NS),	/* X9-14 Gen 1 */
 	TPACPI_Q_LNV('H', '3', TPACPI_FAN_NS),		/* Edge E330 */
+	TPACPI_Q_LNV('J', '4', TPACPI_FAN_NS),		/* L440 / L540 */
 	TPACPI_Q_LNV3('N', '1', 'O', TPACPI_FAN_NOFAN),	/* X1 Tablet (2nd gen) */
 	TPACPI_Q_LNV3('R', '0', 'Q', TPACPI_FAN_DECRPM),/* L480 */
 	TPACPI_Q_LNV('8', 'F', TPACPI_FAN_TPR),		/* ThinkPad x120e */
@@ -8993,8 +9057,8 @@ static int __init fan_init(struct ibm_init_struct *iibm)
 
 	vdbg_printk(TPACPI_DBG_INIT | TPACPI_DBG_FAN,
 		"fan is %s, modes %d, %d\n",
-		str_supported(fan_status_access_mode != TPACPI_FAN_NONE ||
-		  fan_control_access_mode != TPACPI_FAN_WR_NONE),
+		str_supported_unsupported(fan_status_access_mode != TPACPI_FAN_NONE ||
+					  fan_control_access_mode != TPACPI_FAN_WR_NONE),
 		fan_status_access_mode, fan_control_access_mode);
 
 	/* fan control master switch */
@@ -10107,9 +10171,8 @@ static void lcdshadow_resume(void)
 	if (!lcdshadow_dev)
 		return;
 
-	mutex_lock(&lcdshadow_dev->lock);
+	guard(mutex)(&lcdshadow_dev->lock);
 	lcdshadow_set_sw_state(lcdshadow_dev, lcdshadow_dev->sw_state);
-	mutex_unlock(&lcdshadow_dev->lock);
 }
 
 static int lcdshadow_read(struct seq_file *m)
@@ -10141,9 +10204,8 @@ static int lcdshadow_write(char *buf)
 	if (state >= 2 || state < 0)
 		return -EINVAL;
 
-	mutex_lock(&lcdshadow_dev->lock);
-	res = lcdshadow_set_sw_state(lcdshadow_dev, state);
-	mutex_unlock(&lcdshadow_dev->lock);
+	scoped_guard(mutex, &lcdshadow_dev->lock)
+		res = lcdshadow_set_sw_state(lcdshadow_dev, state);
 
 	drm_privacy_screen_call_notifier_chain(lcdshadow_dev);
 
@@ -10607,26 +10669,26 @@ static const struct platform_profile_ops dytc_profile_ops = {
 static void dytc_profile_refresh(void)
 {
 	enum platform_profile_option profile;
-	int output = 0, err = 0;
+	int output = 0, err;
 	int perfmode, funcmode = 0;
 
-	mutex_lock(&dytc_mutex);
-	if (dytc_capabilities & BIT(DYTC_FC_MMC)) {
-		if (dytc_mmc_get_available)
-			err = dytc_command(DYTC_CMD_MMC_GET, &output);
-		else
-			err = dytc_cql_command(DYTC_CMD_GET, &output);
-		funcmode = DYTC_FUNCTION_MMC;
-	} else if (dytc_capabilities & BIT(DYTC_FC_PSC)) {
-		err = dytc_command(DYTC_CMD_GET, &output);
-		/* Check if we are PSC mode, or have AMT enabled */
-		funcmode = (output >> DYTC_GET_FUNCTION_BIT) & 0xF;
-	} else { /* Unknown profile mode */
-		err = -ENODEV;
+	scoped_guard(mutex, &dytc_mutex) {
+		if (dytc_capabilities & BIT(DYTC_FC_MMC)) {
+			if (dytc_mmc_get_available)
+				err = dytc_command(DYTC_CMD_MMC_GET, &output);
+			else
+				err = dytc_cql_command(DYTC_CMD_GET, &output);
+			funcmode = DYTC_FUNCTION_MMC;
+		} else if (dytc_capabilities & BIT(DYTC_FC_PSC)) {
+			err = dytc_command(DYTC_CMD_GET, &output);
+			/* Check if we are PSC mode, or have AMT enabled */
+			funcmode = (output >> DYTC_GET_FUNCTION_BIT) & 0xF;
+		} else { /* Unknown profile mode */
+			err = -ENODEV;
+		}
+		if (err)
+			return;
 	}
-	mutex_unlock(&dytc_mutex);
-	if (err)
-		return;
 
 	perfmode = (output >> DYTC_GET_MODE_BIT) & 0xF;
 	err = convert_dytc_to_profile(funcmode, perfmode, &profile);
@@ -11538,7 +11600,7 @@ static bool tpacpi_driver_event(const unsigned int hkey_event)
 		if (tp_features.kbdlight) {
 			enum led_brightness brightness;
 
-			mutex_lock(&kbdlight_mutex);
+			guard(mutex)(&kbdlight_mutex);
 
 			/*
 			 * Check the brightness actually changed, setting the brightness
@@ -11550,8 +11612,6 @@ static bool tpacpi_driver_event(const unsigned int hkey_event)
 				led_classdev_notify_brightness_hw_changed(
 					&tpacpi_led_kbdlight.led_classdev, brightness);
 			}
-
-			mutex_unlock(&kbdlight_mutex);
 		}
 		/* Key events are suppressed by default hotkey_user_mask */
 		return false;
@@ -11573,11 +11633,11 @@ static bool tpacpi_driver_event(const unsigned int hkey_event)
 			enum drm_privacy_screen_status old_hw_state;
 			bool changed;
 
-			mutex_lock(&lcdshadow_dev->lock);
-			old_hw_state = lcdshadow_dev->hw_state;
-			lcdshadow_get_hw_state(lcdshadow_dev);
-			changed = lcdshadow_dev->hw_state != old_hw_state;
-			mutex_unlock(&lcdshadow_dev->lock);
+			scoped_guard(mutex, &lcdshadow_dev->lock) {
+				old_hw_state = lcdshadow_dev->hw_state;
+				lcdshadow_get_hw_state(lcdshadow_dev);
+				changed = lcdshadow_dev->hw_state != old_hw_state;
+			}
 
 			if (changed)
 				drm_privacy_screen_call_notifier_chain(lcdshadow_dev);
@@ -11598,12 +11658,10 @@ static bool tpacpi_driver_event(const unsigned int hkey_event)
 			pr_err("Error retrieving camera shutter state after shutter event\n");
 			return true;
 		}
-		mutex_lock(&tpacpi_inputdev_send_mutex);
-
-		input_report_switch(tpacpi_inputdev, SW_CAMERA_LENS_COVER, camera_shutter_state);
-		input_sync(tpacpi_inputdev);
-
-		mutex_unlock(&tpacpi_inputdev_send_mutex);
+		scoped_guard(mutex, &tpacpi_inputdev_send_mutex) {
+			input_report_switch(tpacpi_inputdev, SW_CAMERA_LENS_COVER, camera_shutter_state);
+			input_sync(tpacpi_inputdev);
+		}
 		return true;
 	case TP_HKEY_EV_DOUBLETAP_TOGGLE:
 		/* Toggle kernel-level doubletap event filtering */
@@ -11632,15 +11690,6 @@ static struct proc_dir_entry *proc_dir;
 
 static bool force_load;
 
-#ifdef CONFIG_THINKPAD_ACPI_DEBUG
-static const char * __init str_supported(int is_supported)
-{
-	static char text_unsupported[] __initdata = "not supported";
-
-	return (is_supported) ? &text_unsupported[4] : &text_unsupported[0];
-}
-#endif /* CONFIG_THINKPAD_ACPI_DEBUG */
-
 static struct dentry *tpacpi_dbg;
 static void tpacpi_debugfs_init(void)
 {
@@ -12360,6 +12409,8 @@ static int __init thinkpad_acpi_module_init(void)
 	dmi_id = dmi_first_match(fwbug_list);
 	if (dmi_id)
 		tp_features.quirks = dmi_id->driver_data;
+	tp_features.has_keyboard_attached_on_screen =
+		dmi_check_system(keyboard_attached_on_screen_list);
 
 	/* Device initialization */
 	tpacpi_pdev = platform_device_register_simple(TPACPI_DRVR_NAME, PLATFORM_DEVID_NONE,
diff --git a/drivers/platform/x86/lenovo/yogabook.c b/drivers/platform/x86/lenovo/yogabook.c
index 1a4b2ab1f35d66..8a91d440724b3c 100644
--- a/drivers/platform/x86/lenovo/yogabook.c
+++ b/drivers/platform/x86/lenovo/yogabook.c
@@ -353,13 +353,13 @@ static int yogabook_wmi_probe(struct wmi_device *wdev, const void *context)
 		goto error_put_devs;
 	}
 
-	data->kbd_dev = get_device(acpi_get_first_physical_node(data->kbd_adev));
+	data->kbd_dev = acpi_bus_get_primary_device(data->kbd_adev);
 	if (!data->kbd_dev || !data->kbd_dev->driver) {
 		r = -EPROBE_DEFER;
 		goto error_put_devs;
 	}
 
-	data->dig_dev = get_device(acpi_get_first_physical_node(data->dig_adev));
+	data->dig_dev = acpi_bus_get_primary_device(data->dig_adev);
 	if (!data->dig_dev || !data->dig_dev->driver) {
 		r = -EPROBE_DEFER;
 		goto error_put_devs;
diff --git a/drivers/platform/x86/msi-laptop.c b/drivers/platform/x86/msi-laptop.c
index ddef6b78d2fa9f..0a0c756a55987f 100644
--- a/drivers/platform/x86/msi-laptop.c
+++ b/drivers/platform/x86/msi-laptop.c
@@ -117,7 +117,7 @@ struct quirk_entry {
 	 * Wlan/Bluetooth/3G, those netbook will load the SCM (windows app) to
 	 * disable the original Wlan/Bluetooth control by BIOS when user press
 	 * fn key, then control Wlan/Bluetooth/3G by SCM (software control by
-	 * OS). Without SCM, user cann't on/off 3G module on those 3G netbook.
+	 * OS). Without SCM, user can't on/off 3G module on those 3G netbook.
 	 * On Linux, msi-laptop driver will do the same thing to disable the
 	 * original BIOS control, then might need use HAL or other userland
 	 * application to do the software control that simulate with SCM.
diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c
index 19d194ff37ca93..2b1d3c2eb2e5fd 100644
--- a/drivers/platform/x86/panasonic-laptop.c
+++ b/drivers/platform/x86/panasonic-laptop.c
@@ -107,7 +107,7 @@
  *
  *	Jul.25, 2004	Hiroshi Miura <miura@da-cha.org>
  *		- v0.4  first post version
- *		        add function to retrive SIFR
+ *		        add function to retrieve SIFR
  *
  *	Jul.24, 2004	Hiroshi Miura <miura@da-cha.org>
  *		- v0.3  get proper status of hotkey
@@ -232,6 +232,7 @@ static const struct key_entry panasonic_keymap[] = {
 	{ KE_KEY, 41, { KEY_MACRO9 } },
 	{ KE_KEY, 42, { KEY_MACRO10 } },
 	{ KE_KEY, 43, { KEY_MACRO11 } },
+	{ KE_KEY, 48, { KEY_FN } },
 	{ KE_END, 0 }
 };
 
@@ -863,6 +864,17 @@ static void acpi_pcc_generate_keyinput(struct pcc_acpi *pcc)
 					key, 0x80, false);
 	}
 
+	/*
+	 * Let's Note models report both FN key down and key up
+	 * without the expected 0x80 key up/down bit
+	 */
+	if (key == 48) {
+		updown = 1;
+	} else if (key == 49) {
+		key = 48;
+		updown = 0;
+	}
+
 	/*
 	 * Don't report brightness key-presses if they are also reported
 	 * by the ACPI video bus.
diff --git a/drivers/platform/x86/samsung-laptop.c b/drivers/platform/x86/samsung-laptop.c
index 710f3d5bf84c1b..18631e96bb171d 100644
--- a/drivers/platform/x86/samsung-laptop.c
+++ b/drivers/platform/x86/samsung-laptop.c
@@ -295,7 +295,7 @@ static const struct sabi_config sabi_configs[] = {
  *   d0, d1, d2, d3   - data fields
  *   call             - call SABI using command and data
  *
- * This allow to call arbitrary sabi commands wihout
+ * This allow to call arbitrary sabi commands without
  * modifying the driver at all.
  * For example, setting the keyboard backlight brightness to 5
  *
diff --git a/drivers/platform/x86/serdev_helpers.h b/drivers/platform/x86/serdev_helpers.h
index 57eac75805e223..20267e1941f16a 100644
--- a/drivers/platform/x86/serdev_helpers.h
+++ b/drivers/platform/x86/serdev_helpers.h
@@ -74,8 +74,7 @@ get_serdev_controller(const char *serial_ctrl_hid,
 		return ERR_PTR(-ENODEV);
 	}
 
-	/* get_first_physical_node() returns a weak ref */
-	parent = get_device(acpi_get_first_physical_node(adev));
+	parent = acpi_bus_get_primary_device(adev);
 	acpi_dev_put(adev);
 	if (!parent) {
 		pr_err("error could not get %s/%s serial-ctrl physical node\n",
diff --git a/drivers/platform/x86/serial-multi-instantiate.c b/drivers/platform/x86/serial-multi-instantiate.c
index 3e89fcdd45f785..d535025a3f733f 100644
--- a/drivers/platform/x86/serial-multi-instantiate.c
+++ b/drivers/platform/x86/serial-multi-instantiate.c
@@ -426,6 +426,7 @@ static const struct acpi_device_id smi_acpi_ids[] = {
 	/* Non-conforming _HID for Cirrus Logic already released */
 	{ "CLSA0100", (unsigned long)&cs35l41_hda },
 	{ "CLSA0101", (unsigned long)&cs35l41_hda },
+	{ "CLSA0102", (unsigned long)&cs35l41_hda },
 	{ }
 };
 MODULE_DEVICE_TABLE(acpi, smi_acpi_ids);
diff --git a/drivers/platform/x86/topstar-laptop.c b/drivers/platform/x86/topstar-laptop.c
index 0e842c55dcc527..730a599abf0f7d 100644
--- a/drivers/platform/x86/topstar-laptop.c
+++ b/drivers/platform/x86/topstar-laptop.c
@@ -287,10 +287,14 @@ static const struct dmi_system_id topstar_dmi_ids[] = {
 
 static int topstar_acpi_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *device;
 	struct topstar_laptop *topstar;
 	int err;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	dmi_check_system(topstar_dmi_ids);
 
 	topstar = kzalloc_obj(struct topstar_laptop);
diff --git a/drivers/platform/x86/uniwill/uniwill-acpi.c b/drivers/platform/x86/uniwill/uniwill-acpi.c
index 7a2eeaec4c961c..eeead0159828bb 100644
--- a/drivers/platform/x86/uniwill/uniwill-acpi.c
+++ b/drivers/platform/x86/uniwill/uniwill-acpi.c
@@ -1705,7 +1705,7 @@ static int uniwill_kbd_led_mc_brightness_set(struct led_classdev *led_cdev,
 
 	/*
 	 * The EC interprets a RGB value of 0x000000 as a command to restore
-	 * the device-specfic default RGB value. Work around this by writing
+	 * the device-specific default RGB value. Work around this by writing
 	 * a RGB value of 0x010101 (faint white) instead.
 	 */
 	if (data->kbd_led_mc_subled_info[0].intensity == 0 &&
diff --git a/drivers/platform/x86/x86-android-tablets/core.c b/drivers/platform/x86/x86-android-tablets/core.c
index cfff7f5eac5d7d..2c262f117235b3 100644
--- a/drivers/platform/x86/x86-android-tablets/core.c
+++ b/drivers/platform/x86/x86-android-tablets/core.c
@@ -390,7 +390,6 @@ static int gpio_secondary_fwnode_init(struct device *parent,
 {
 	const struct software_node *const *swnode;
 	struct fwnode_handle *fwnode;
-	struct device *phys_dev;
 	int ret;
 
 	if (!node_group)
@@ -418,7 +417,8 @@ static int gpio_secondary_fwnode_init(struct device *parent,
 		if (WARN_ON(!fwnode))
 			return -ENOENT;
 
-		phys_dev = acpi_get_first_physical_node(to_acpi_device(dev));
+		struct device *phys_dev __free(put_device) =
+				acpi_bus_get_primary_device(to_acpi_device(dev));
 		if (!phys_dev)
 			return dev_err_probe(parent, -ENODEV,
 					     "No physical device for ACPI GPIO dev: %pfwP\n",
diff --git a/drivers/pmdomain/Kconfig b/drivers/pmdomain/Kconfig
index 23076ae90e6641..5341dff669bebc 100644
--- a/drivers/pmdomain/Kconfig
+++ b/drivers/pmdomain/Kconfig
@@ -11,6 +11,7 @@ source "drivers/pmdomain/marvell/Kconfig"
 source "drivers/pmdomain/mediatek/Kconfig"
 source "drivers/pmdomain/qcom/Kconfig"
 source "drivers/pmdomain/renesas/Kconfig"
+source "drivers/pmdomain/riscv/Kconfig"
 source "drivers/pmdomain/rockchip/Kconfig"
 source "drivers/pmdomain/samsung/Kconfig"
 source "drivers/pmdomain/st/Kconfig"
diff --git a/drivers/pmdomain/Makefile b/drivers/pmdomain/Makefile
index ebc802f13eb953..d7aee13ae57164 100644
--- a/drivers/pmdomain/Makefile
+++ b/drivers/pmdomain/Makefile
@@ -9,6 +9,7 @@ obj-y					+= marvell/
 obj-y					+= mediatek/
 obj-y					+= qcom/
 obj-y					+= renesas/
+obj-y					+= riscv/
 obj-y					+= rockchip/
 obj-y					+= samsung/
 obj-y					+= st/
diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c
index 052ec478dfcc4c..964e44c2f78f1d 100644
--- a/drivers/pmdomain/core.c
+++ b/drivers/pmdomain/core.c
@@ -10,6 +10,7 @@
 #include <linux/idr.h>
 #include <linux/kernel.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
 #include <linux/pm_runtime.h>
@@ -19,11 +20,15 @@
 #include <linux/slab.h>
 #include <linux/err.h>
 #include <linux/sched.h>
+#include <linux/smp.h>
 #include <linux/suspend.h>
 #include <linux/export.h>
 #include <linux/cpu.h>
 #include <linux/debugfs.h>
 
+#include <trace/events/ipi.h>
+#include "core.h"
+
 /* Provides a unique ID for each genpd device */
 static DEFINE_IDA(genpd_ida);
 
@@ -32,7 +37,9 @@ static const struct bus_type genpd_provider_bus_type = {
 	.name		= "genpd_provider",
 };
 
-#define GENPD_RETRY_MAX_MS	250		/* Approximate */
+#define GENPD_RETRY_MAX_MS		250		/* Approximate */
+#define GENPD_CPU_ON_POLL_PERIOD_US	100		/* 100us */
+#define GENPD_CPU_ON_TIMEOUT_US		300000		/* 300ms */
 
 #define GENPD_DEV_CALLBACK(genpd, type, callback, dev)		\
 ({								\
@@ -173,7 +180,7 @@ static const struct genpd_lock_ops genpd_raw_spin_ops = {
 #define genpd_lock_interruptible(p)	p->lock_ops->lock_interruptible(p)
 #define genpd_unlock(p)			p->lock_ops->unlock(p)
 
-#define genpd_status_on(genpd)		(genpd->status == GENPD_STATE_ON)
+#define genpd_status_on_unlocked(genpd)	(genpd->status == GENPD_STATE_ON)
 #define genpd_is_irq_safe(genpd)	(genpd->flags & GENPD_FLAG_IRQ_SAFE)
 #define genpd_is_always_on(genpd)	(genpd->flags & GENPD_FLAG_ALWAYS_ON)
 #define genpd_is_active_wakeup(genpd)	(genpd->flags & GENPD_FLAG_ACTIVE_WAKEUP)
@@ -183,6 +190,7 @@ static const struct genpd_lock_ops genpd_raw_spin_ops = {
 #define genpd_is_dev_name_fw(genpd)	(genpd->flags & GENPD_FLAG_DEV_NAME_FW)
 #define genpd_is_no_sync_state(genpd)	(genpd->flags & GENPD_FLAG_NO_SYNC_STATE)
 #define genpd_is_no_stay_on(genpd)	(genpd->flags & GENPD_FLAG_NO_STAY_ON)
+#define genpd_is_power_unknown(genpd)	(genpd->flags & GENPD_FLAG_POWER_UNKNOWN)
 
 static inline bool irq_safe_dev_in_sleep_domain(struct device *dev,
 		const struct generic_pm_domain *genpd)
@@ -276,6 +284,49 @@ static void genpd_sd_counter_inc(struct generic_pm_domain *genpd)
 	smp_mb__after_atomic();
 }
 
+/**
+ * genpd_for_each_child - Recursively iterate over all devices
+ *                        in a PM domain and its subdomains.
+ * @genpd: PM domain to iterate over.
+ * @fn: Callback function to invoke for each device.
+ * @data: Data to pass to the callback function.
+ *
+ * This function recursively walks through all devices in the given PM domain
+ * and all devices in its child PM domains (subdomains). For each device found,
+ * the callback function @fn is invoked with the device and @data as arguments.
+ *
+ * Note: this function is inteded for use by the core and governors,
+ * not for pmdomain providers.
+ *
+ * Returns: 0 on success, or the first non-zero value returned by @fn.
+ */
+int genpd_for_each_child(struct generic_pm_domain *genpd,
+			 int (*fn)(struct device *dev, void *data),
+			 void *data)
+{
+	struct pm_domain_data *pdd;
+	struct gpd_link *link;
+	int ret;
+
+	/* First, iterate over all devices in this domain */
+	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
+		ret = fn(pdd->dev, data);
+		if (ret)
+			return ret;
+	}
+
+	/* Then, recursively iterate over all child domains (subdomains) */
+	list_for_each_entry(link, &genpd->parent_links, parent_node) {
+		struct generic_pm_domain *child_pd = link->child;
+
+		ret = genpd_for_each_child(child_pd, fn, data);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 #ifdef CONFIG_DEBUG_FS
 static struct dentry *genpd_debugfs_dir;
 
@@ -771,8 +822,7 @@ EXPORT_SYMBOL_GPL(dev_pm_genpd_rpm_always_on);
  * @dev: Device to get the current power status
  *
  * This function checks whether the generic power domain associated with the
- * given device is on or not by verifying if genpd_status_on equals
- * GENPD_STATE_ON.
+ * given device is on or not by checking if genpd->status equals GENPD_STATE_ON.
  *
  * Note: this function returns the power status of the genpd at the time of the
  * call. The power status may change after due to activity from other devices
@@ -791,7 +841,7 @@ bool dev_pm_genpd_is_on(struct device *dev)
 		return false;
 
 	genpd_lock(genpd);
-	is_on = genpd_status_on(genpd);
+	is_on = genpd_status_on_unlocked(genpd);
 	genpd_unlock(genpd);
 
 	return is_on;
@@ -940,7 +990,7 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd)
  * genpd_power_off - Remove power from a given PM domain.
  * @genpd: PM domain to power down.
  * @one_dev_on: If invoked from genpd's ->runtime_suspend|resume() callback, the
- * RPM status of the releated device is in an intermediate state, not yet turned
+ * RPM status of the related device is in an intermediate state, not yet turned
  * into RPM_SUSPENDED. This means genpd_power_off() must allow one device to not
  * be RPM_SUSPENDED, while it tries to power off the PM domain.
  * @depth: nesting count for lockdep.
@@ -963,7 +1013,7 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on,
 	 * The domain was on at boot and still need to stay on.
 	 * The domain has a subdomain being powered on.
 	 */
-	if (!genpd_status_on(genpd) || genpd->prepared_count > 0 ||
+	if (!genpd_status_on_unlocked(genpd) || genpd->prepared_count > 0 ||
 	    genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd) ||
 	    genpd->stay_on || atomic_read(&genpd->sd_count) > 0)
 		return;
@@ -1027,22 +1077,86 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on,
 	}
 }
 
+static bool genpd_status_on(struct generic_pm_domain *genpd)
+{
+	bool is_on;
+
+	genpd_lock(genpd);
+	is_on = genpd_status_on_unlocked(genpd);
+	genpd_unlock(genpd);
+
+	return is_on;
+}
+
+static int genpd_wakeup_cpu(struct generic_pm_domain *genpd)
+{
+	unsigned int cpu;
+	bool is_on;
+	int ret;
+
+	/* Find the first online CPU in the genpd's cpumask. */
+	cpu = cpumask_first_and(genpd->cpus, cpu_online_mask);
+	if (cpu >= nr_cpu_ids)
+		return -EAGAIN;
+
+	genpd_unlock(genpd);
+
+	/* Send a IPI to wakeup the selected CPU. */
+	smp_send_reschedule(cpu);
+
+	/* Poll to wait for it to complete the power on sequence. */
+	ret = readx_poll_timeout_atomic(genpd_status_on, genpd, is_on, is_on,
+					GENPD_CPU_ON_POLL_PERIOD_US,
+					GENPD_CPU_ON_TIMEOUT_US);
+
+	genpd_lock(genpd);
+
+	/* Re-check the status as we have released the lock in between. */
+	if (ret || !genpd_status_on_unlocked(genpd))
+		return -EAGAIN;
+
+	return 0;
+}
+
+static bool genpd_need_alive_cpu(struct generic_pm_domain *genpd,
+				 struct device *dev)
+{
+	if (!genpd_is_cpu_domain(genpd))
+		return false;
+
+	/* This is not for CPU devices as those are managed differently. */
+	if (to_gpd_data(dev->power.subsys_data->domain_data)->cpu >= 0)
+		return false;
+
+	/*
+	 * If the current CPU doesn't belong to the genpd's cpumask, we need to
+	 * wake up one of those idle CPUs to power on the CPU domain correctly.
+	 */
+	return !cpumask_test_cpu(smp_processor_id(), genpd->cpus);
+}
+
 /**
  * genpd_power_on - Restore power to a given PM domain and its parents.
  * @genpd: PM domain to power up.
+ * @dev: The device that needs the PM domain to power on.
  * @depth: nesting count for lockdep.
  *
  * Restore power to @genpd and all of its parents so that it is possible to
  * resume a device belonging to it.
  */
-static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth)
+static int genpd_power_on(struct generic_pm_domain *genpd, struct device *dev,
+			  unsigned int depth)
 {
 	struct gpd_link *link;
 	int ret = 0;
 
-	if (genpd_status_on(genpd))
+	if (genpd_status_on_unlocked(genpd))
 		return 0;
 
+	/* Special case for a device attached to a CPU domain. */
+	if (genpd_need_alive_cpu(genpd, dev))
+		return genpd_wakeup_cpu(genpd);
+
 	/* Reflect over the entered idle-states residency for debugfs. */
 	genpd_reflect_residency(genpd);
 
@@ -1057,7 +1171,7 @@ static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth)
 		genpd_sd_counter_inc(parent);
 
 		genpd_lock_nested(parent, depth + 1);
-		ret = genpd_power_on(parent, depth + 1);
+		ret = genpd_power_on(parent, dev, depth + 1);
 		genpd_unlock(parent);
 
 		if (ret) {
@@ -1307,7 +1421,7 @@ static int genpd_runtime_resume(struct device *dev)
 
 	genpd_lock(genpd);
 	genpd_restore_performance_state(dev, gpd_data->rpm_pstate);
-	ret = genpd_power_on(genpd, 0);
+	ret = genpd_power_on(genpd, dev, 0);
 	genpd_unlock(genpd);
 
 	if (ret)
@@ -1406,7 +1520,7 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock,
 {
 	struct gpd_link *link;
 
-	if (!genpd_status_on(genpd) || genpd_is_always_on(genpd))
+	if (!genpd_status_on_unlocked(genpd) || genpd_is_always_on(genpd))
 		return;
 
 	if (genpd->suspended_count != genpd->device_count
@@ -1472,7 +1586,7 @@ static void genpd_sync_power_on(struct generic_pm_domain *genpd, bool use_lock,
 {
 	struct gpd_link *link;
 
-	if (genpd_status_on(genpd))
+	if (genpd_status_on_unlocked(genpd))
 		return;
 
 	list_for_each_entry(link, &genpd->child_links, child_node) {
@@ -2170,7 +2284,8 @@ static int genpd_add_subdomain(struct generic_pm_domain *genpd,
 	genpd_lock(subdomain);
 	genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING);
 
-	if (!genpd_status_on(genpd) && genpd_status_on(subdomain)) {
+	if (!genpd_status_on_unlocked(genpd) &&
+	    genpd_status_on_unlocked(subdomain)) {
 		ret = -EINVAL;
 		goto out;
 	}
@@ -2186,7 +2301,7 @@ static int genpd_add_subdomain(struct generic_pm_domain *genpd,
 	list_add_tail(&link->parent_node, &genpd->parent_links);
 	link->child = subdomain;
 	list_add_tail(&link->child_node, &subdomain->child_links);
-	if (genpd_status_on(subdomain))
+	if (genpd_status_on_unlocked(subdomain))
 		genpd_sd_counter_inc(genpd);
 
  out:
@@ -2246,7 +2361,7 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
 		list_del(&link->parent_node);
 		list_del(&link->child_node);
 		kfree(link);
-		if (genpd_status_on(subdomain))
+		if (genpd_status_on_unlocked(subdomain))
 			genpd_sd_counter_dec(genpd);
 
 		ret = 0;
@@ -2375,7 +2490,10 @@ static void genpd_lock_init(struct generic_pm_domain *genpd)
 #ifdef CONFIG_PM_GENERIC_DOMAINS_OF
 static void genpd_set_stay_on(struct generic_pm_domain *genpd, bool is_off)
 {
-	genpd->stay_on = !genpd_is_no_stay_on(genpd) && !is_off;
+	if (genpd_is_power_unknown(genpd))
+		genpd->stay_on = is_off;
+	else
+		genpd->stay_on = !genpd_is_no_stay_on(genpd) && !is_off;
 }
 #else
 static void genpd_set_stay_on(struct generic_pm_domain *genpd, bool is_off)
@@ -2441,7 +2559,7 @@ int pm_genpd_init(struct generic_pm_domain *genpd,
 
 	/* Always-on domains must be powered on at initialization. */
 	if ((genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd)) &&
-			!genpd_status_on(genpd)) {
+			!genpd_status_on_unlocked(genpd)) {
 		pr_err("always-on PM domain %s is not on\n", genpd->name);
 		return -EINVAL;
 	}
@@ -3410,7 +3528,7 @@ static int __genpd_dev_pm_attach(struct device *dev, struct device *base_dev,
 
 	if (power_on) {
 		genpd_lock(pd);
-		ret = genpd_power_on(pd, 0);
+		ret = genpd_power_on(pd, dev, 0);
 		genpd_unlock(pd);
 	}
 
@@ -3583,7 +3701,10 @@ static int genpd_parse_state(struct genpd_power_state *genpd_state,
 	if (!err)
 		genpd_state->residency_ns = 1000LL * residency;
 
-	of_property_read_string(state_node, "idle-state-name", &genpd_state->name);
+	err = of_property_read_string(state_node, "idle-state-name",
+				      &genpd_state->name);
+	if (err)
+		genpd_state->name = state_node->name;
 
 	genpd_state->power_on_latency_ns = 1000LL * exit_latency;
 	genpd_state->power_off_latency_ns = 1000LL * entry_latency;
@@ -3839,7 +3960,7 @@ static int genpd_summary_one(struct seq_file *s,
 
 	if (WARN_ON(genpd->status >= ARRAY_SIZE(status_lookup)))
 		goto exit;
-	if (!genpd_status_on(genpd))
+	if (!genpd_status_on_unlocked(genpd))
 		snprintf(state, sizeof(state), "%s-%u",
 			 status_lookup[genpd->status], genpd->state_idx);
 	else
diff --git a/drivers/pmdomain/core.h b/drivers/pmdomain/core.h
new file mode 100644
index 00000000000000..d1df535a834a9f
--- /dev/null
+++ b/drivers/pmdomain/core.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Internal header for PM domain core
+ *
+ * Copyright (C) 2026 Kevin Hilman <khilman@baylibre.com>, Texas Instruments
+ */
+
+#ifndef __PM_DOMAIN_CORE_H__
+#define __PM_DOMAIN_CORE_H__
+
+struct device;
+struct generic_pm_domain;
+
+int genpd_for_each_child(struct generic_pm_domain *genpd,
+			 int (*fn)(struct device *dev, void *data),
+			 void *data);
+
+#endif /* __PM_DOMAIN_CORE_H__ */
diff --git a/drivers/pmdomain/governor.c b/drivers/pmdomain/governor.c
index 96737abbb49691..38bba24ec26e3e 100644
--- a/drivers/pmdomain/governor.c
+++ b/drivers/pmdomain/governor.c
@@ -13,6 +13,8 @@
 #include <linux/cpumask.h>
 #include <linux/ktime.h>
 
+#include "core.h"
+
 static int dev_update_qos_constraint(struct device *dev, void *data)
 {
 	s64 *constraint_ns_p = data;
@@ -425,17 +427,60 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd)
 	return true;
 }
 
+/**
+ * check_device_qos_latency - Callback to check device QoS latency constraints
+ * @dev: Device to check
+ * @data: Pointer to s32 variable holding minimum latency found so far
+ *
+ * This callback checks if the device has a system-wide resume latency QoS
+ * constraint and updates the minimum latency if this device has a stricter
+ * constraint.
+ *
+ * The system-wide flag is checked first so that devices that have not opted
+ * in only incur a single lockless read.
+ *
+ * Returns: 0 to continue iteration.
+ */
+static int check_device_qos_latency(struct device *dev, void *data)
+{
+	s32 *min_dev_latency = data;
+	s32 dev_latency;
+
+	if (!(dev_pm_qos_raw_flags(dev) & PM_QOS_FLAG_LATENCY_SYS))
+		return 0;
+
+	dev_latency = dev_pm_qos_raw_resume_latency(dev);
+	if (dev_latency != PM_QOS_RESUME_LATENCY_NO_CONSTRAINT) {
+		if (dev_latency < *min_dev_latency)
+			*min_dev_latency = dev_latency;
+	}
+
+	return 0;
+}
+
 static bool cpu_system_power_down_ok(struct dev_pm_domain *pd)
 {
 	s64 constraint_ns = cpu_wakeup_latency_qos_limit() * NSEC_PER_USEC;
 	struct generic_pm_domain *genpd = pd_to_genpd(pd);
 	int state_idx = genpd->state_count - 1;
+	s32 min_dev_latency = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT;
+	s64 min_dev_latency_ns;
 
 	if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) {
 		genpd->state_idx = state_idx;
 		return true;
 	}
 
+	genpd_for_each_child(genpd, check_device_qos_latency,
+			     &min_dev_latency);
+
+	/* If device latency < CPU wakeup latency, use it instead */
+	if (min_dev_latency != PM_QOS_RESUME_LATENCY_NO_CONSTRAINT) {
+		min_dev_latency_ns = (s64)min_dev_latency * NSEC_PER_USEC;
+		if (min_dev_latency_ns < constraint_ns)
+			constraint_ns = min_dev_latency_ns;
+	}
+
 	/* Find the deepest state for the latency constraint. */
 	while (state_idx >= 0) {
 		s64 latency_ns = genpd->states[state_idx].power_off_latency_ns +
diff --git a/drivers/pmdomain/imx/gpcv2.c b/drivers/pmdomain/imx/gpcv2.c
index a829f8da5be70d..6a54ee73451464 100644
--- a/drivers/pmdomain/imx/gpcv2.c
+++ b/drivers/pmdomain/imx/gpcv2.c
@@ -317,11 +317,9 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd)
 	u32 reg_val, pgc;
 	int ret;
 
-	ret = pm_runtime_get_sync(domain->dev);
-	if (ret < 0) {
-		pm_runtime_put_noidle(domain->dev);
+	ret = pm_runtime_resume_and_get(domain->dev);
+	if (ret < 0)
 		return ret;
-	}
 
 	if (!IS_ERR(domain->regulator)) {
 		ret = regulator_enable(domain->regulator);
@@ -1397,21 +1395,13 @@ static void imx_pgc_domain_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int imx_pgc_domain_suspend(struct device *dev)
 {
-	int ret;
-
 	/*
 	 * This may look strange, but is done so the generic PM_SLEEP code
 	 * can power down our domain and more importantly power it up again
 	 * after resume, without tripping over our usage of runtime PM to
 	 * power up/down the nested domains.
 	 */
-	ret = pm_runtime_get_sync(dev);
-	if (ret < 0) {
-		pm_runtime_put_noidle(dev);
-		return ret;
-	}
-
-	return 0;
+	return pm_runtime_resume_and_get(dev);
 }
 
 static int imx_pgc_domain_resume(struct device *dev)
diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c
index 479789009c7f44..14fb969bba2876 100644
--- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c
+++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c
@@ -15,6 +15,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/clk.h>
+#include <linux/mutex.h>
 
 #include <dt-bindings/power/imx8mm-power.h>
 #include <dt-bindings/power/imx8mn-power.h>
@@ -34,6 +35,12 @@ struct imx8m_blk_ctrl {
 	struct regmap *regmap;
 	struct imx8m_blk_ctrl_domain *domains;
 	struct genpd_onecell_data onecell_data;
+	/*
+	 * Serializes the blk-ctrl reset/clock sequence across sibling domains;
+	 * their transitions interact through the shared VPUMIX bus domain,
+	 * VPU_NOC and the not-ack-verified ADB400 handshake (ERR050531).
+	 */
+	struct mutex power_lock;
 };
 
 struct imx8m_blk_ctrl_domain_data {
@@ -98,10 +105,11 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd)
 	struct imx8m_blk_ctrl *bc = domain->bc;
 	int ret;
 
+	guard(mutex)(&bc->power_lock);
+
 	/* make sure bus domain is awake */
-	ret = pm_runtime_get_sync(bc->bus_power_dev);
+	ret = pm_runtime_resume_and_get(bc->bus_power_dev);
 	if (ret < 0) {
-		pm_runtime_put_noidle(bc->bus_power_dev);
 		dev_err(bc->dev, "failed to power up bus domain\n");
 		return ret;
 	}
@@ -164,6 +172,8 @@ static int imx8m_blk_ctrl_power_off(struct generic_pm_domain *genpd)
 	const struct imx8m_blk_ctrl_domain_data *data = domain->data;
 	struct imx8m_blk_ctrl *bc = domain->bc;
 
+	guard(mutex)(&bc->power_lock);
+
 	/* put devices into reset and disable clocks */
 	if (data->mipi_phy_rst_mask)
 		regmap_clear_bits(bc->regmap, BLK_MIPI_RESET_DIV, data->mipi_phy_rst_mask);
@@ -202,6 +212,10 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev)
 
 	bc->dev = dev;
 
+	ret = devm_mutex_init(dev, &bc->power_lock);
+	if (ret)
+		return ret;
+
 	bc_data = of_device_get_match_data(dev);
 
 	base = devm_platform_ioremap_resource(pdev, 0);
@@ -382,20 +396,16 @@ static int imx8m_blk_ctrl_suspend(struct device *dev)
 	 * in the system suspend/resume paths due to the device parent/child
 	 * hierarchy.
 	 */
-	ret = pm_runtime_get_sync(bc->bus_power_dev);
-	if (ret < 0) {
-		pm_runtime_put_noidle(bc->bus_power_dev);
+	ret = pm_runtime_resume_and_get(bc->bus_power_dev);
+	if (ret < 0)
 		return ret;
-	}
 
 	for (i = 0; i < bc->onecell_data.num_domains; i++) {
 		struct imx8m_blk_ctrl_domain *domain = &bc->domains[i];
 
-		ret = pm_runtime_get_sync(domain->power_dev);
-		if (ret < 0) {
-			pm_runtime_put_noidle(domain->power_dev);
+		ret = pm_runtime_resume_and_get(domain->power_dev);
+		if (ret < 0)
 			goto out_fail;
-		}
 	}
 
 	return 0;
diff --git a/drivers/pmdomain/imx/imx93-blk-ctrl.c b/drivers/pmdomain/imx/imx93-blk-ctrl.c
index 243ce939ba6807..6d4c55a85c6457 100644
--- a/drivers/pmdomain/imx/imx93-blk-ctrl.c
+++ b/drivers/pmdomain/imx/imx93-blk-ctrl.c
@@ -152,9 +152,8 @@ static int imx93_blk_ctrl_power_on(struct generic_pm_domain *genpd)
 		return ret;
 	}
 
-	ret = pm_runtime_get_sync(bc->dev);
+	ret = pm_runtime_resume_and_get(bc->dev);
 	if (ret < 0) {
-		pm_runtime_put_noidle(bc->dev);
 		dev_err(bc->dev, "failed to power up domain\n");
 		goto disable_clk;
 	}
diff --git a/drivers/pmdomain/imx/scu-pd.c b/drivers/pmdomain/imx/scu-pd.c
index e3f9e741daf76f..05d4364f34be9d 100644
--- a/drivers/pmdomain/imx/scu-pd.c
+++ b/drivers/pmdomain/imx/scu-pd.c
@@ -474,7 +474,7 @@ static int imx_scu_init_pm_domains(struct device *dev,
 	struct genpd_onecell_data *pd_data;
 	struct imx_sc_pm_domain *sc_pd;
 	u32 count = 0;
-	int i, j;
+	int i, j, ret;
 
 	for (i = 0; i < pd_soc->num_ranges; i++)
 		count += pd_ranges[i].num;
@@ -503,7 +503,13 @@ static int imx_scu_init_pm_domains(struct device *dev,
 	pd_data->num_domains = count;
 	pd_data->xlate = imx_scu_pd_xlate;
 
-	of_genpd_add_provider_onecell(dev->of_node, pd_data);
+	ret = of_genpd_add_provider_onecell(dev->of_node, pd_data);
+	if (ret) {
+		while (count)
+			pm_genpd_remove(domains[--count]);
+
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/drivers/pmdomain/qcom/rpmhpd.c b/drivers/pmdomain/qcom/rpmhpd.c
index 90743275942db2..fa8fed5bd3e585 100644
--- a/drivers/pmdomain/qcom/rpmhpd.c
+++ b/drivers/pmdomain/qcom/rpmhpd.c
@@ -198,6 +198,11 @@ static struct rpmhpd mxc_ao = {
 	.res_name = "mxc.lvl",
 };
 
+static struct rpmhpd nmxc = {
+	.pd = { .name = "nmxc", },
+	.res_name = "nmxc.lvl",
+};
+
 static struct rpmhpd nsp = {
 	.pd = { .name = "nsp", },
 	.res_name = "nsp.lvl",
@@ -327,6 +332,7 @@ static struct rpmhpd *nord_rpmhpds[] = {
 	[RPMHPD_MX_AO] = &mx_ao,
 	[RPMHPD_MXC] = &mxc,
 	[RPMHPD_MXC_AO] = &mxc_ao,
+	[RPMHPD_NMXC] = &nmxc,
 	[RPMHPD_NSP0] = &nsp0,
 	[RPMHPD_NSP1] = &nsp1,
 	[RPMHPD_NSP2] = &nsp2,
@@ -486,6 +492,24 @@ static const struct rpmhpd_desc sm7150_desc = {
 	.num_pds = ARRAY_SIZE(sm7150_rpmhpds),
 };
 
+/* SM7250 RPMH powerdomains */
+static struct rpmhpd *sm7250_rpmhpds[] = {
+	[RPMHPD_CX] = &cx_w_mx_parent,
+	[RPMHPD_CX_AO] = &cx_ao_w_mx_parent,
+	[RPMHPD_EBI] = &ebi,
+	[RPMHPD_GFX] = &gfx,
+	[RPMHPD_LCX] = &lcx,
+	[RPMHPD_LMX] = &lmx,
+	[RPMHPD_MSS] = &mss,
+	[RPMHPD_MX] = &mx,
+	[RPMHPD_MX_AO] = &mx_ao,
+};
+
+static const struct rpmhpd_desc sm7250_desc = {
+	.rpmhpds = sm7250_rpmhpds,
+	.num_pds = ARRAY_SIZE(sm7250_rpmhpds),
+};
+
 /* SM8150 RPMH powerdomains */
 static struct rpmhpd *sm8150_rpmhpds[] = {
 	[SM8150_CX] = &cx_w_mx_parent,
@@ -681,6 +705,21 @@ static const struct rpmhpd_desc kaanapali_desc = {
 	.num_pds = ARRAY_SIZE(kaanapali_rpmhpds),
 };
 
+/* Kuno RPMH power domains */
+static struct rpmhpd *kuno_rpmhpds[] = {
+	[RPMHPD_CX] = &cx,
+	[RPMHPD_CX_AO] = &cx_ao,
+	[RPMHPD_MX] = &mx,
+	[RPMHPD_MX_AO] = &mx_ao,
+	[RPMHPD_MXC] = &mxc,
+	[RPMHPD_MXC_AO] = &mxc_ao,
+};
+
+static const struct rpmhpd_desc kuno_desc = {
+	.rpmhpds = kuno_rpmhpds,
+	.num_pds = ARRAY_SIZE(kuno_rpmhpds),
+};
+
 /* Hawi RPMH powerdomains */
 static struct rpmhpd *hawi_rpmhpds[] = {
 	[RPMHPD_CX] = &cx,
@@ -885,6 +924,7 @@ static const struct of_device_id rpmhpd_match_table[] = {
 	{ .compatible = "qcom,glymur-rpmhpd", .data = &glymur_desc },
 	{ .compatible = "qcom,hawi-rpmhpd", .data = &hawi_desc },
 	{ .compatible = "qcom,kaanapali-rpmhpd", .data = &kaanapali_desc },
+	{ .compatible = "qcom,kuno-rpmhpd", .data = &kuno_desc },
 	{ .compatible = "qcom,milos-rpmhpd", .data = &milos_desc },
 	{ .compatible = "qcom,nord-rpmhpd", .data = &nord_desc },
 	{ .compatible = "qcom,qcs615-rpmhpd", .data = &qcs615_desc },
@@ -906,6 +946,7 @@ static const struct of_device_id rpmhpd_match_table[] = {
 	{ .compatible = "qcom,sm4450-rpmhpd", .data = &sm4450_desc },
 	{ .compatible = "qcom,sm6350-rpmhpd", .data = &sm6350_desc },
 	{ .compatible = "qcom,sm7150-rpmhpd", .data = &sm7150_desc },
+	{ .compatible = "qcom,sm7250-rpmhpd", .data = &sm7250_desc },
 	{ .compatible = "qcom,sm8150-rpmhpd", .data = &sm8150_desc },
 	{ .compatible = "qcom,sm8250-rpmhpd", .data = &sm8250_desc },
 	{ .compatible = "qcom,sm8350-rpmhpd", .data = &sm8350_desc },
diff --git a/drivers/pmdomain/renesas/Kconfig b/drivers/pmdomain/renesas/Kconfig
index f2f52d3c29a083..df19b95c465596 100644
--- a/drivers/pmdomain/renesas/Kconfig
+++ b/drivers/pmdomain/renesas/Kconfig
@@ -37,6 +37,10 @@ config SYSC_R8A774A1
 	bool "System Controller support for R8A774A1 (RZ/G2M)" if COMPILE_TEST
 	select SYSC_RCAR
 
+config SYSC_R8A774A3
+	bool "System Controller support for R8A774A3 (RZ/G2M v3.0)" if COMPILE_TEST
+	select SYSC_RCAR
+
 config SYSC_R8A774B1
 	bool "System Controller support for R8A774B1 (RZ/G2N)" if COMPILE_TEST
 	select SYSC_RCAR
diff --git a/drivers/pmdomain/renesas/Makefile b/drivers/pmdomain/renesas/Makefile
index 17849aad37a5ac..8eda820478e411 100644
--- a/drivers/pmdomain/renesas/Makefile
+++ b/drivers/pmdomain/renesas/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_SYSC_R8A7743)	+= r8a7743-sysc.o
 obj-$(CONFIG_SYSC_R8A7745)	+= r8a7745-sysc.o
 obj-$(CONFIG_SYSC_R8A77470)	+= r8a77470-sysc.o
 obj-$(CONFIG_SYSC_R8A774A1)	+= r8a774a1-sysc.o
+obj-$(CONFIG_SYSC_R8A774A3)	+= r8a774a3-sysc.o
 obj-$(CONFIG_SYSC_R8A774B1)	+= r8a774b1-sysc.o
 obj-$(CONFIG_SYSC_R8A774C0)	+= r8a774c0-sysc.o
 obj-$(CONFIG_SYSC_R8A774E1)	+= r8a774e1-sysc.o
diff --git a/drivers/pmdomain/renesas/r8a774a3-sysc.c b/drivers/pmdomain/renesas/r8a774a3-sysc.c
new file mode 100644
index 00000000000000..58c98700bd5533
--- /dev/null
+++ b/drivers/pmdomain/renesas/r8a774a3-sysc.c
@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Renesas RZ/G2M v3.0 System Controller
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ *
+ * Based on Renesas R-Car M3-W System Controller
+ * Copyright (C) 2016 Glider bvba
+ */
+
+#include <linux/kernel.h>
+
+#include <dt-bindings/power/renesas,r8a774a3-sysc.h>
+
+#include "rcar-sysc.h"
+
+static const struct rcar_sysc_area r8a774a3_areas[] __initconst = {
+	{ "always-on",     0, 0, R8A774A3_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
+	{ "ca57-scu",  0x1c0, 0, R8A774A3_PD_CA57_SCU, R8A774A3_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca57-cpu0",  0x80, 0, R8A774A3_PD_CA57_CPU0, R8A774A3_PD_CA57_SCU,
+	  PD_CPU_NOCR },
+	{ "ca57-cpu1",  0x80, 1, R8A774A3_PD_CA57_CPU1, R8A774A3_PD_CA57_SCU,
+	  PD_CPU_NOCR },
+	{ "ca53-scu",  0x140, 0, R8A774A3_PD_CA53_SCU, R8A774A3_PD_ALWAYS_ON,
+	  PD_SCU },
+	{ "ca53-cpu0", 0x200, 0, R8A774A3_PD_CA53_CPU0, R8A774A3_PD_CA53_SCU,
+	  PD_CPU_NOCR },
+	{ "ca53-cpu1", 0x200, 1, R8A774A3_PD_CA53_CPU1, R8A774A3_PD_CA53_SCU,
+	  PD_CPU_NOCR },
+	{ "ca53-cpu2", 0x200, 2, R8A774A3_PD_CA53_CPU2, R8A774A3_PD_CA53_SCU,
+	  PD_CPU_NOCR },
+	{ "ca53-cpu3", 0x200, 3, R8A774A3_PD_CA53_CPU3, R8A774A3_PD_CA53_SCU,
+	  PD_CPU_NOCR },
+	{ "a3vc",      0x380, 0, R8A774A3_PD_A3VC, R8A774A3_PD_ALWAYS_ON },
+	{ "a2vc1",     0x3c0, 1, R8A774A3_PD_A2VC1, R8A774A3_PD_A3VC },
+	{ "3dg-a",     0x100, 0, R8A774A3_PD_3DG_A, R8A774A3_PD_ALWAYS_ON },
+	{ "3dg-b",     0x100, 1, R8A774A3_PD_3DG_B, R8A774A3_PD_3DG_A },
+};
+
+const struct rcar_sysc_info r8a774a3_sysc_info __initconst = {
+	.areas = r8a774a3_areas,
+	.num_areas = ARRAY_SIZE(r8a774a3_areas),
+	.extmask_offs = 0x2f8,
+	.extmask_val = BIT(0),
+};
diff --git a/drivers/pmdomain/renesas/r8a7795-sysc.c b/drivers/pmdomain/renesas/r8a7795-sysc.c
index cbe1ff0fc58393..2ed377b5884662 100644
--- a/drivers/pmdomain/renesas/r8a7795-sysc.c
+++ b/drivers/pmdomain/renesas/r8a7795-sysc.c
@@ -13,7 +13,7 @@
 
 #include "rcar-sysc.h"
 
-static struct rcar_sysc_area r8a7795_areas[] __initdata = {
+static const struct rcar_sysc_area r8a7795_areas[] __initconst = {
 	{ "always-on",	    0, 0, R8A7795_PD_ALWAYS_ON,	-1, PD_ALWAYS_ON },
 	{ "ca57-scu",	0x1c0, 0, R8A7795_PD_CA57_SCU,	R8A7795_PD_ALWAYS_ON,
 	  PD_SCU },
diff --git a/drivers/pmdomain/renesas/r8a779a0-sysc.c b/drivers/pmdomain/renesas/r8a779a0-sysc.c
index 54cdf250f7c2d1..5f84b8d56a3282 100644
--- a/drivers/pmdomain/renesas/r8a779a0-sysc.c
+++ b/drivers/pmdomain/renesas/r8a779a0-sysc.c
@@ -11,7 +11,7 @@
 
 #include "rcar-gen4-sysc.h"
 
-static struct rcar_gen4_sysc_area r8a779a0_areas[] __initdata = {
+static const struct rcar_gen4_sysc_area r8a779a0_areas[] __initconst = {
 	{ "always-on",	R8A779A0_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
 	{ "a3e0",	R8A779A0_PD_A3E0, R8A779A0_PD_ALWAYS_ON, PD_SCU },
 	{ "a3e1",	R8A779A0_PD_A3E1, R8A779A0_PD_ALWAYS_ON, PD_SCU },
diff --git a/drivers/pmdomain/renesas/r8a779f0-sysc.c b/drivers/pmdomain/renesas/r8a779f0-sysc.c
index 6ed13cd1cb249d..d22cbc33a6048c 100644
--- a/drivers/pmdomain/renesas/r8a779f0-sysc.c
+++ b/drivers/pmdomain/renesas/r8a779f0-sysc.c
@@ -11,7 +11,7 @@
 
 #include "rcar-gen4-sysc.h"
 
-static struct rcar_gen4_sysc_area r8a779f0_areas[] __initdata = {
+static const struct rcar_gen4_sysc_area r8a779f0_areas[] __initconst = {
 	{ "always-on",	R8A779F0_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
 	{ "a3e0",	R8A779F0_PD_A3E0, R8A779F0_PD_ALWAYS_ON, PD_SCU },
 	{ "a3e1",	R8A779F0_PD_A3E1, R8A779F0_PD_ALWAYS_ON, PD_SCU },
diff --git a/drivers/pmdomain/renesas/r8a779g0-sysc.c b/drivers/pmdomain/renesas/r8a779g0-sysc.c
index 249cf43af45b64..96d39a3ac46056 100644
--- a/drivers/pmdomain/renesas/r8a779g0-sysc.c
+++ b/drivers/pmdomain/renesas/r8a779g0-sysc.c
@@ -11,7 +11,7 @@
 
 #include "rcar-gen4-sysc.h"
 
-static struct rcar_gen4_sysc_area r8a779g0_areas[] __initdata = {
+static const struct rcar_gen4_sysc_area r8a779g0_areas[] __initconst = {
 	{ "always-on",	R8A779G0_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
 	{ "a3e0",	R8A779G0_PD_A3E0, R8A779G0_PD_ALWAYS_ON, PD_SCU },
 	{ "a2e0d0",	R8A779G0_PD_A2E0D0, R8A779G0_PD_A3E0, PD_SCU },
diff --git a/drivers/pmdomain/renesas/r8a779h0-sysc.c b/drivers/pmdomain/renesas/r8a779h0-sysc.c
index e13372cb80eea8..3775735854f29f 100644
--- a/drivers/pmdomain/renesas/r8a779h0-sysc.c
+++ b/drivers/pmdomain/renesas/r8a779h0-sysc.c
@@ -11,7 +11,7 @@
 
 #include "rcar-gen4-sysc.h"
 
-static struct rcar_gen4_sysc_area r8a779h0_areas[] __initdata = {
+static const struct rcar_gen4_sysc_area r8a779h0_areas[] __initconst = {
 	{ "always-on",	R8A779H0_PD_ALWAYS_ON, -1, PD_ALWAYS_ON },
 	{ "c4",		R8A779H0_PD_C4, R8A779H0_PD_ALWAYS_ON },
 	{ "a2e0d0",	R8A779H0_PD_A2E0D0, R8A779H0_PD_C4, PD_SCU },
diff --git a/drivers/pmdomain/renesas/r8a78000-mdlc.c b/drivers/pmdomain/renesas/r8a78000-mdlc.c
index 2668fc2ac9cbd3..1fd71bb3da776e 100644
--- a/drivers/pmdomain/renesas/r8a78000-mdlc.c
+++ b/drivers/pmdomain/renesas/r8a78000-mdlc.c
@@ -45,8 +45,9 @@ struct r8a78000_mdlc_priv {
 };
 
 static struct generic_pm_domain *r8a78000_genpd_always_on;
+
 static HLIST_HEAD(r8a78000_mdlc_list);
-static DEFINE_MUTEX(r8a78000_mdlc_lock);	/* protects the two above */
+static DEFINE_MUTEX(r8a78000_mdlc_lock);	/* protects the list above */
 
 static struct generic_pm_domain *r8a78000_genpd_xlate(
 			const struct of_phandle_args *spec, void *data)
@@ -191,10 +192,11 @@ static void r8a78000_genpd_del_provider(void *data)
 
 static int r8a78000_genpd_always_on_singleton(struct device *dev)
 {
+	static DEFINE_MUTEX(singleton_lock);
 	struct generic_pm_domain *genpd;
 	int ret;
 
-	guard(mutex)(&r8a78000_mdlc_lock);
+	guard(mutex)(&singleton_lock);
 
 	if (r8a78000_genpd_always_on)
 		return 0;
diff --git a/drivers/pmdomain/renesas/rcar-sysc.c b/drivers/pmdomain/renesas/rcar-sysc.c
index bd7bb9cbd9da00..3c8a44c6d772b6 100644
--- a/drivers/pmdomain/renesas/rcar-sysc.c
+++ b/drivers/pmdomain/renesas/rcar-sysc.c
@@ -284,6 +284,9 @@ static const struct of_device_id rcar_sysc_matches[] __initconst = {
 #ifdef CONFIG_SYSC_R8A774A1
 	{ .compatible = "renesas,r8a774a1-sysc", .data = &r8a774a1_sysc_info },
 #endif
+#ifdef CONFIG_SYSC_R8A774A3
+	{ .compatible = "renesas,r8a774a3-sysc", .data = &r8a774a3_sysc_info },
+#endif
 #ifdef CONFIG_SYSC_R8A774B1
 	{ .compatible = "renesas,r8a774b1-sysc", .data = &r8a774b1_sysc_info },
 #endif
diff --git a/drivers/pmdomain/renesas/rcar-sysc.h b/drivers/pmdomain/renesas/rcar-sysc.h
index 07ffce310686b2..b264e727883bb8 100644
--- a/drivers/pmdomain/renesas/rcar-sysc.h
+++ b/drivers/pmdomain/renesas/rcar-sysc.h
@@ -29,9 +29,15 @@
 
 struct rcar_sysc_area {
 	const char *name;
-	u16 chan_offs;		/* Offset of PWRSR register for this area */
-	u8 chan_bit;		/* Bit in PWR* (except for PWRUP in PWRSR) */
-	u8 isr_bit;		/* Bit in SYSCI*R */
+	u16 chan_offs;		/*
+				 * PWRSR register offset; or 0 if area is
+				 * always on
+				 */
+	u8 chan_bit;		/*
+				 * Bit in PWR* (except for PWRUP in PWRSR);
+				 * 0 if area is always on
+				 */
+	u8 isr_bit;		/* Bit in SYSCI*R; 0 if area is always on */
 	s8 parent;		/* -1 if none */
 	u8 flags;		/* See PD_* */
 };
@@ -55,6 +61,7 @@ extern const struct rcar_sysc_info r8a7743_sysc_info;
 extern const struct rcar_sysc_info r8a7745_sysc_info;
 extern const struct rcar_sysc_info r8a77470_sysc_info;
 extern const struct rcar_sysc_info r8a774a1_sysc_info;
+extern const struct rcar_sysc_info r8a774a3_sysc_info;
 extern const struct rcar_sysc_info r8a774b1_sysc_info;
 extern const struct rcar_sysc_info r8a774c0_sysc_info;
 extern const struct rcar_sysc_info r8a774e1_sysc_info;
diff --git a/drivers/pmdomain/riscv/Kconfig b/drivers/pmdomain/riscv/Kconfig
new file mode 100644
index 00000000000000..97232edc294a98
--- /dev/null
+++ b/drivers/pmdomain/riscv/Kconfig
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config RISCV_RPMI_DEVICE_POWER
+	bool "RISC-V RPMI Based Device Power driver"
+	depends on RISCV || COMPILE_TEST
+	depends on MAILBOX
+	depends on PM
+	default RISCV
+	select PM_GENERIC_DOMAINS
+	help
+	  Support for device power domains based on the device power service
+	  group defined by the RISC-V platform management interface (RPMI)
+	  specification. The power domains advertised by the platform
+	  microcontroller are registered as generic power domains, so that
+	  devices can reference them through the "power-domains" property.
diff --git a/drivers/pmdomain/riscv/Makefile b/drivers/pmdomain/riscv/Makefile
new file mode 100644
index 00000000000000..2cb0ee8ad41380
--- /dev/null
+++ b/drivers/pmdomain/riscv/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+obj-$(CONFIG_RISCV_RPMI_DEVICE_POWER)	+= riscv-rpmi-device-power.o
diff --git a/drivers/pmdomain/riscv/riscv-rpmi-device-power.c b/drivers/pmdomain/riscv/riscv-rpmi-device-power.c
new file mode 100644
index 00000000000000..c7c46717dd7329
--- /dev/null
+++ b/drivers/pmdomain/riscv/riscv-rpmi-device-power.c
@@ -0,0 +1,485 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RISC-V RPMI Based Device Power Driver through SBI MPXY
+ *
+ * Copyright (C) 2026 Shanghai StarFive Technology Co., Ltd.
+ *
+ * Implements a Device Power driver on top of SBI RPMI Message Proxy Extension (MPXY)
+ *
+ * Each SBI MPXY Device Power instance is associated, through the means of a proper DT
+ * entry description, to a specific Transport ID.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/mailbox/riscv-rpmi-message.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+
+#define RPMI_PM_DOMAIN_NAME_LEN		16
+
+/* power state for device power domains */
+#define RPMI_POWER_STATE_CONTEXT_MASK		GENMASK(16, 16)
+#define RPMI_POWER_STATE_CONTEXT_PRESERVED	0
+#define RPMI_POWER_STATE_CONTEXT_LOST		1
+#define RPMI_POWER_STATE_VALUE_MASK		GENMASK(15, 0)
+#define RPMI_POWER_STATE_VALUE_ON		0
+#define RPMI_POWER_STATE_VALUE_OFF		3
+
+#define RPMI_POWER_STATE_PARAM(context_type, state_value) \
+	((FIELD_PREP(RPMI_POWER_STATE_CONTEXT_MASK, context_type)) | \
+	(FIELD_PREP(RPMI_POWER_STATE_VALUE_MASK, state_value)))
+
+#define RPMI_POWER_STATE_GENERIC_ON  \
+	RPMI_POWER_STATE_PARAM(RPMI_POWER_STATE_CONTEXT_PRESERVED, \
+				   RPMI_POWER_STATE_VALUE_ON)
+
+#define RPMI_POWER_STATE_GENERIC_OFF  \
+	RPMI_POWER_STATE_PARAM(RPMI_POWER_STATE_CONTEXT_PRESERVED, \
+				   RPMI_POWER_STATE_VALUE_OFF)
+
+/**
+ * struct rpmi_ctx - RPMI transport context shared by all power domains
+ *
+ * @chan: mailbox channel of the underlying SBI MPXY transport
+ * @client: mailbox client bound to @chan
+ * @lock: serialises RPMI transactions on @chan
+ */
+struct rpmi_ctx {
+	struct mbox_chan *chan;
+	struct mbox_client client;
+	/* serialises RPMI transactions on @chan */
+	struct mutex lock;
+};
+
+/**
+ * struct rpmi_device_power_domain - describe one available MPXY Device Power Domain
+ *
+ * @id: the power domain ID as advertised by PuC
+ * @transition_latency: worst case transition latency of power domain from one state
+ *			to another
+ * @dev: device associated with this power domain
+ * @mpxy_ctx: RPMI transport context used to reach the PuC
+ * @genpd: generic power domain registered with the genpd core
+ * @name: device power domain name assigned by PuC
+ */
+struct rpmi_device_power_domain {
+	u32 id;
+	u32 transition_latency;
+	struct device *dev;
+	struct rpmi_ctx *mpxy_ctx;
+	struct generic_pm_domain genpd;
+	char name[RPMI_PM_DOMAIN_NAME_LEN];
+};
+
+#define to_rpmi_pd(gpd) container_of(gpd, struct rpmi_device_power_domain, genpd)
+
+/* Service: GET_POWER_DOMAINS */
+struct rpmi_pm_get_num_domain_rx {
+	__le32 status;
+	__le32 num_domains;
+};
+
+/* Service: GET_POWER_DOMAIN_ATTRS */
+struct rpmi_pm_get_domain_attrs_tx {
+	__le32 domain_id;
+};
+
+/* pm domain attributes response data */
+struct rpmi_pm_get_domain_attrs_rx {
+	__le32 status;
+	__le32 flags;
+	__le32 transition_latency;
+	char name[RPMI_PM_DOMAIN_NAME_LEN];
+};
+
+/* Service: SET_POWER_DOMAIN_STATE */
+struct rpmi_pm_set_power_state_tx {
+	__le32 domain_id;
+	__le32 power_state;
+};
+
+struct rpmi_pm_set_power_state_rx {
+	__le32 status;
+};
+
+/* Service: GET_POWER_DOMAIN_STATE */
+struct rpmi_pm_get_power_state_tx {
+	__le32 domain_id;
+};
+
+struct rpmi_pm_get_power_state_rx {
+	__le32 status;
+	__le32 power_state;
+};
+
+static int rpmi_pm_send_message(struct rpmi_ctx *mpxy_ctx,
+				struct rpmi_mbox_message *msg)
+{
+	guard(mutex)(&mpxy_ctx->lock);
+
+	return rpmi_mbox_send_message(mpxy_ctx->chan, msg);
+}
+
+static int rpmi_power_state_get(struct rpmi_device_power_domain *mpxy_pm_domain,
+				u32 domain_id, u32 *state)
+{
+	struct rpmi_pm_get_power_state_rx rx = { };
+	struct rpmi_pm_get_power_state_tx tx;
+	struct rpmi_mbox_message msg;
+	int ret;
+
+	tx.domain_id = cpu_to_le32(domain_id);
+
+	rpmi_mbox_init_send_with_response(&msg, RPMI_DP_SRV_GET_STATE,
+					  &tx, sizeof(tx), &rx, sizeof(rx));
+
+	ret = rpmi_pm_send_message(mpxy_pm_domain->mpxy_ctx, &msg);
+	if (ret)
+		return ret;
+
+	if (rx.status)
+		return rpmi_to_linux_error(le32_to_cpu(rx.status));
+
+	if (msg.data.out_response_len < sizeof(rx))
+		return -EPROTO;
+
+	*state = le32_to_cpu(rx.power_state);
+
+	return 0;
+}
+
+static int rpmi_power_state_set(struct rpmi_device_power_domain *mpxy_pm_domain,
+				u32 domain_id, u32 state)
+{
+	struct rpmi_pm_set_power_state_rx rx = { };
+	struct rpmi_pm_set_power_state_tx tx;
+	struct rpmi_mbox_message msg;
+	int ret;
+
+	tx.domain_id = cpu_to_le32(domain_id);
+	tx.power_state = cpu_to_le32(state);
+
+	rpmi_mbox_init_send_with_response(&msg, RPMI_DP_SRV_SET_STATE,
+					  &tx, sizeof(tx), &rx, sizeof(rx));
+	ret = rpmi_pm_send_message(mpxy_pm_domain->mpxy_ctx, &msg);
+	if (ret)
+		return ret;
+
+	if (rx.status)
+		return rpmi_to_linux_error(le32_to_cpu(rx.status));
+
+	if (msg.data.out_response_len < sizeof(rx))
+		return -EPROTO;
+
+	return 0;
+}
+
+static int rpmi_pd_power(struct generic_pm_domain *domain, bool power_on)
+{
+	struct rpmi_device_power_domain *mpxy_pm_domain;
+	u32 state, ret_state, domain_id;
+	int ret;
+
+	if (power_on)
+		state = RPMI_POWER_STATE_GENERIC_ON;
+	else
+		state = RPMI_POWER_STATE_GENERIC_OFF;
+
+	mpxy_pm_domain = to_rpmi_pd(domain);
+	domain_id = mpxy_pm_domain->id;
+
+	ret = rpmi_power_state_set(mpxy_pm_domain, domain_id, state);
+	if (ret)
+		return ret;
+
+	ret = rpmi_power_state_get(mpxy_pm_domain, domain_id, &ret_state);
+	if (ret)
+		return ret;
+
+	if (FIELD_GET(RPMI_POWER_STATE_VALUE_MASK, ret_state) !=
+	    FIELD_GET(RPMI_POWER_STATE_VALUE_MASK, state))
+		return -EIO;
+
+	return ret;
+}
+
+static int rpmi_pd_power_on(struct generic_pm_domain *domain)
+{
+	return rpmi_pd_power(domain, true);
+}
+
+static int rpmi_pd_power_off(struct generic_pm_domain *domain)
+{
+	return rpmi_pd_power(domain, false);
+}
+
+static int rpmi_pm_get_num_domains(struct rpmi_ctx *mpxy_ctx, u32 *domain)
+{
+	struct rpmi_pm_get_num_domain_rx rx = { };
+	struct rpmi_mbox_message msg;
+	int ret;
+
+	rpmi_mbox_init_send_with_response(&msg, RPMI_DP_SRV_GET_NUM_DOMAINS,
+					  NULL, 0, &rx, sizeof(rx));
+	ret = rpmi_pm_send_message(mpxy_ctx, &msg);
+	if (ret)
+		return ret;
+
+	if (rx.status)
+		return rpmi_to_linux_error(le32_to_cpu(rx.status));
+
+	if (msg.data.out_response_len < sizeof(rx))
+		return -EPROTO;
+
+	*domain = le32_to_cpu(rx.num_domains);
+
+	return 0;
+}
+
+/* obtain the MPXY device power domain attributes */
+static int rpmi_device_power_get_attrs(u32 domain_id,
+				       struct rpmi_device_power_domain *mpxy_pm_domain)
+{
+	struct rpmi_pm_get_domain_attrs_rx rx = { };
+	struct rpmi_pm_get_domain_attrs_tx tx;
+	struct rpmi_mbox_message msg;
+	int ret;
+
+	tx.domain_id = cpu_to_le32(domain_id);
+
+	rpmi_mbox_init_send_with_response(&msg, RPMI_DP_SRV_GET_ATTRS,
+					  &tx, sizeof(tx), &rx, sizeof(rx));
+	ret = rpmi_pm_send_message(mpxy_pm_domain->mpxy_ctx, &msg);
+	if (ret)
+		return ret;
+
+	if (rx.status)
+		return rpmi_to_linux_error(le32_to_cpu(rx.status));
+
+	if (msg.data.out_response_len < sizeof(rx))
+		return -EPROTO;
+
+	mpxy_pm_domain->transition_latency = le32_to_cpu(rx.transition_latency);
+	strscpy(mpxy_pm_domain->name, rx.name, sizeof(mpxy_pm_domain->name));
+
+	return 0;
+}
+
+static int rpmi_pm_attr_setup(struct device *dev, struct rpmi_ctx *mpxy_ctx)
+{
+	struct rpmi_mbox_message msg;
+	int ret;
+
+	/* Validate RPMI specification version */
+	rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_SPEC_VERSION);
+	ret = rpmi_pm_send_message(mpxy_ctx, &msg);
+	if (ret) {
+		dev_dbg(dev, "Failed to get spec version\n");
+		return ret;
+	}
+
+	if (msg.attr.value < RPMI_MKVER(1, 0)) {
+		dev_dbg(dev,
+			"msg protocol version mismatch, expected 0x%x, found 0x%x\n",
+			RPMI_MKVER(1, 0), msg.attr.value);
+		return -EINVAL;
+	}
+
+	/* Validate device power service group ID */
+	rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_SERVICEGROUP_ID);
+	ret = rpmi_pm_send_message(mpxy_ctx, &msg);
+	if (ret) {
+		dev_dbg(dev, "Failed to get service group ID\n");
+		return ret;
+	}
+
+	if (msg.attr.value != RPMI_SRVGRP_DEVICE_POWER) {
+		dev_dbg(dev,
+			"service group match failed, expected 0x%x, found 0x%x\n",
+			RPMI_SRVGRP_DEVICE_POWER, msg.attr.value);
+		return -EINVAL;
+	}
+
+	/* Validate device power service group version */
+	rpmi_mbox_init_get_attribute(&msg, RPMI_MBOX_ATTR_SERVICEGROUP_VERSION);
+	ret = rpmi_pm_send_message(mpxy_ctx, &msg);
+	if (ret) {
+		dev_dbg(dev, "Failed to get service group version\n");
+		return ret;
+	}
+
+	if (msg.attr.value < RPMI_MKVER(1, 0)) {
+		dev_dbg(dev,
+			"service group version failed, expected 0x%x, found 0x%x\n",
+			RPMI_MKVER(1, 0), msg.attr.value);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void rpmi_pm_domain_mbox_chan_release(void *data)
+{
+	mbox_free_channel((struct mbox_chan *)data);
+}
+
+static int rpmi_pm_domain_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct rpmi_device_power_domain *mpxy_pd;
+	struct genpd_onecell_data *mpxy_pd_data;
+	struct generic_pm_domain **domains;
+	struct device *dev = &pdev->dev;
+	struct rpmi_ctx *mpxy_ctx;
+	u32 num_domains = 0;
+	int ret;
+	u32 i;
+
+	mpxy_ctx = devm_kzalloc(&pdev->dev, sizeof(*mpxy_ctx), GFP_KERNEL);
+	if (!mpxy_ctx)
+		return -ENOMEM;
+
+	ret = devm_mutex_init(dev, &mpxy_ctx->lock);
+	if (ret)
+		return ret;
+
+	/* Setup mailbox client */
+	mpxy_ctx->client.dev		= dev;
+	mpxy_ctx->client.rx_callback	= NULL;
+	mpxy_ctx->client.tx_block	= false;
+	mpxy_ctx->client.knows_txdone	= true;
+	mpxy_ctx->client.tx_tout	= 0;
+
+	/* Request mailbox channel */
+	mpxy_ctx->chan = mbox_request_channel(&mpxy_ctx->client, 0);
+	if (IS_ERR(mpxy_ctx->chan))
+		return PTR_ERR(mpxy_ctx->chan);
+
+	ret = devm_add_action_or_reset(dev, rpmi_pm_domain_mbox_chan_release,
+				       mpxy_ctx->chan);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to add rpmi mbox channel cleanup\n");
+
+	ret = rpmi_pm_attr_setup(dev, mpxy_ctx);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to verify RPMI attribute\n");
+
+	/* Get number of device power domain */
+	ret = rpmi_pm_get_num_domains(mpxy_ctx, &num_domains);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to get number of pm domains\n");
+
+	if (!num_domains)
+		return dev_err_probe(dev, -EINVAL, "No PM domains found!\n");
+
+	dev_info(&pdev->dev, "%u MPXY PM domains are found\n", num_domains);
+
+	mpxy_pd = devm_kcalloc(&pdev->dev, num_domains, sizeof(*mpxy_pd), GFP_KERNEL);
+	if (!mpxy_pd)
+		return -ENOMEM;
+
+	domains = devm_kcalloc(&pdev->dev, num_domains, sizeof(*domains), GFP_KERNEL);
+	if (!domains)
+		return -ENOMEM;
+
+	mpxy_pd_data = devm_kzalloc(&pdev->dev, sizeof(*mpxy_pd_data), GFP_KERNEL);
+	if (!mpxy_pd_data)
+		return -ENOMEM;
+
+	for (i = 0; i < num_domains; i++, mpxy_pd++) {
+		u32 state;
+
+		mpxy_pd->dev = &pdev->dev;
+		mpxy_pd->mpxy_ctx = mpxy_ctx;
+		mpxy_pd->id = i;
+
+		ret = rpmi_device_power_get_attrs(i, mpxy_pd);
+		if (ret) {
+			dev_warn(mpxy_pd->dev,
+				 "power domain %u initialization failed\n",
+				 mpxy_pd->id);
+			domains[i] = NULL;
+			continue;
+		}
+
+		ret = rpmi_power_state_get(mpxy_pd, i, &state);
+		if (ret) {
+			dev_warn(mpxy_pd->dev,
+				 "failed to get state for power domain %u\n",
+				 mpxy_pd->id);
+			domains[i] = NULL;
+			continue;
+		}
+
+		state = FIELD_GET(RPMI_POWER_STATE_VALUE_MASK, state);
+		if (state != RPMI_POWER_STATE_VALUE_OFF &&
+		    state != RPMI_POWER_STATE_VALUE_ON) {
+			dev_warn(mpxy_pd->dev,
+				 "unsupported state 0x%x for power domain %u\n",
+				 state, mpxy_pd->id);
+			domains[i] = NULL;
+			continue;
+		}
+
+		mpxy_pd->genpd.name = mpxy_pd->name;
+		mpxy_pd->genpd.power_off = rpmi_pd_power_off;
+		mpxy_pd->genpd.power_on = rpmi_pd_power_on;
+		/*
+		 * Domain names come from the PuC and are not guaranteed to be
+		 * unique or non-empty, so let genpd derive unique device names
+		 * from them rather than using them verbatim.
+		 */
+		mpxy_pd->genpd.flags = GENPD_FLAG_DEV_NAME_FW;
+
+		ret = pm_genpd_init(&mpxy_pd->genpd, NULL,
+				    state == RPMI_POWER_STATE_VALUE_OFF);
+		if (ret) {
+			dev_warn(mpxy_pd->dev,
+				 "failed to init power domain %u\n",
+				 mpxy_pd->id);
+			domains[i] = NULL;
+			continue;
+		}
+
+		domains[i] = &mpxy_pd->genpd;
+	}
+
+	mpxy_pd_data->domains = domains;
+	mpxy_pd_data->num_domains = num_domains;
+
+	ret = of_genpd_add_provider_onecell(np, mpxy_pd_data);
+	if (ret) {
+		while (i--) {
+			if (domains[i])
+				pm_genpd_remove(domains[i]);
+		}
+
+		return dev_err_probe(dev, ret, "failed to add genpd provider\n");
+	}
+
+	return 0;
+}
+
+static const struct of_device_id rpmi_pm_domain_of_match[] = {
+	{ .compatible = "riscv,rpmi-device-power" },
+	{},
+};
+
+static struct platform_driver rpmi_pm_domain_platdrv = {
+	.driver = {
+		.name = "riscv-rpmi-device-power",
+		.of_match_table = rpmi_pm_domain_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe = rpmi_pm_domain_probe,
+};
+
+builtin_platform_driver(rpmi_pm_domain_platdrv);
+
+MODULE_AUTHOR("Joshua Yeong <joshua.yeong@starfivetech.com>");
+MODULE_DESCRIPTION("Device Power Driver based on RPMI message protocol");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pmdomain/rockchip/pm-domains.c b/drivers/pmdomain/rockchip/pm-domains.c
index ba66ae71942892..db3879846db67f 100644
--- a/drivers/pmdomain/rockchip/pm-domains.c
+++ b/drivers/pmdomain/rockchip/pm-domains.c
@@ -721,7 +721,7 @@ static int rockchip_pd_power_on(struct generic_pm_domain *domain)
 	ret = rockchip_pd_regulator_enable(pd);
 	if (ret)
 		return dev_err_probe(pd->pmu->dev, ret,
-				     "Failed to enable supply: %d\n", ret);
+				     "Failed to enable supply\n");
 
 	ret = rockchip_pd_power(pd, true);
 	if (ret)
@@ -759,7 +759,19 @@ static int rockchip_pd_attach_dev(struct generic_pm_domain *genpd,
 	}
 
 	i = 0;
-	while ((clk = of_clk_get(dev->of_node, i++)) && !IS_ERR(clk)) {
+	while (1) {
+		clk = of_clk_get(dev->of_node, i++);
+		if (IS_ERR(clk)) {
+			error = PTR_ERR(clk);
+			if (error == -ENOENT)
+				break;
+
+			dev_err(dev, "failed to get clock %d: %d\n", i - 1,
+				error);
+			pm_clk_destroy(dev);
+			return error;
+		}
+
 		dev_dbg(dev, "adding clock '%pC' to list of PM clocks\n", clk);
 		error = pm_clk_add_clk(dev, clk);
 		if (error) {
@@ -809,7 +821,7 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 		return 0;
 
 	pd_info = &pmu->info->domain_info[id];
-	if (!pd_info) {
+	if (!pd_info->pwr_mask && !pd_info->req_mask) {
 		dev_err(pmu->dev, "%pOFn: undefined domain id %d\n",
 			node, id);
 		return -EINVAL;
@@ -842,7 +854,7 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 			dev_err(pmu->dev,
 				"%pOFn: failed to get clk at index %d: %d\n",
 				node, i, error);
-			return error;
+			goto err_put_clocks;
 		}
 	}
 
@@ -1018,7 +1030,9 @@ static int rockchip_pm_add_subdomain(struct rockchip_pmu *pmu,
 				parent_domain->name, child_domain->name);
 		}
 
-		rockchip_pm_add_subdomain(pmu, np);
+		error = rockchip_pm_add_subdomain(pmu, np);
+		if (error)
+			return error;
 	}
 
 	return 0;
diff --git a/drivers/pnp/driver.c b/drivers/pnp/driver.c
index 6d58b146508136..80c2a0f15b808f 100644
--- a/drivers/pnp/driver.c
+++ b/drivers/pnp/driver.c
@@ -96,13 +96,13 @@ static int pnp_device_probe(struct device *dev)
 		if (!(pnp_drv->flags & PNP_DRIVER_RES_DO_NOT_CHANGE)) {
 			error = pnp_activate_dev(pnp_dev);
 			if (error < 0)
-				return error;
+				goto fail;
 		}
 	} else if ((pnp_drv->flags & PNP_DRIVER_RES_DISABLE)
 		   == PNP_DRIVER_RES_DISABLE) {
 		error = pnp_disable_dev(pnp_dev);
 		if (error < 0)
-			return error;
+			goto fail;
 	}
 	error = 0;
 	if (pnp_drv->probe) {
diff --git a/drivers/pnp/pnpbios/core.c b/drivers/pnp/pnpbios/core.c
index f7e86ae9f72faa..46af1f54933754 100644
--- a/drivers/pnp/pnpbios/core.c
+++ b/drivers/pnp/pnpbios/core.c
@@ -47,7 +47,7 @@
 #include <linux/delay.h>
 #include <linux/acpi.h>
 #include <linux/freezer.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/kthread.h>
 
 #include <asm/page.h>
diff --git a/drivers/power/reset/keystone-reset.c b/drivers/power/reset/keystone-reset.c
index 3c44cd6cee0aa4..491ba490b8eabd 100644
--- a/drivers/power/reset/keystone-reset.c
+++ b/drivers/power/reset/keystone-reset.c
@@ -112,8 +112,7 @@ static int rsctrl_probe(struct platform_device *pdev)
 	for (i = 0; i < WDT_MUX_NUMBER; i++) {
 		ret = of_property_read_u32_index(np, "ti,wdt-list", i, &val);
 		if (ret == -EOVERFLOW && !i) {
-			dev_err(dev, "ti,wdt-list property has to contain at"
-				"least one entry\n");
+			dev_err(dev, "ti,wdt-list property has to contain at least one entry\n");
 			return -EINVAL;
 		} else if (ret) {
 			break;
diff --git a/drivers/power/sequencing/Kconfig b/drivers/power/sequencing/Kconfig
index 226c62704d9bb5..b6a9f25fe3e34f 100644
--- a/drivers/power/sequencing/Kconfig
+++ b/drivers/power/sequencing/Kconfig
@@ -27,6 +27,15 @@ config POWER_SEQUENCING_QCOM_WCN
 	  this driver is needed for correct power control or else we'd risk not
 	  respecting the required delays between enabling Bluetooth and WLAN.
 
+config POWER_SEQUENCING_RENESAS_PWRRDY
+	tristate "Renesas Power Ready sequencing driver"
+	depends on SYSC_RZ || COMPILE_TEST
+	select AUXILIARY_BUS
+	help
+	  Say Y here to enable the power sequencing driver for the Renesas
+	  Power Ready signals. This driver handles the power ready signals
+	  required to power on the various IP's on RZ/G3L platform.
+
 config POWER_SEQUENCING_TH1520_GPU
 	tristate "T-HEAD TH1520 GPU power sequencing driver"
 	depends on ARCH_THEAD || COMPILE_TEST
@@ -47,4 +56,14 @@ config POWER_SEQUENCING_PCIE_M2
 	  connectors. This driver handles the power sequencing for the M.2
 	  connectors exposing multiple interfaces like PCIe, SATA, UART, etc...
 
+config POWER_SEQUENCING_KUNIT
+	tristate "Build power sequencing KUnit test cases"
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  Say Y here to build the module containing KUnit test cases for the
+	  power sequencing subsystem.
+
+	  If unsure, say N.
+
 endif
diff --git a/drivers/power/sequencing/Makefile b/drivers/power/sequencing/Makefile
index 0911d461829897..76a2775fab33e4 100644
--- a/drivers/power/sequencing/Makefile
+++ b/drivers/power/sequencing/Makefile
@@ -4,5 +4,7 @@ obj-$(CONFIG_POWER_SEQUENCING)		+= pwrseq-core.o
 pwrseq-core-y				:= core.o
 
 obj-$(CONFIG_POWER_SEQUENCING_QCOM_WCN)	+= pwrseq-qcom-wcn.o
+obj-$(CONFIG_POWER_SEQUENCING_RENESAS_PWRRDY) += pwrseq-renesas-pwrrdy.o
 obj-$(CONFIG_POWER_SEQUENCING_TH1520_GPU) += pwrseq-thead-gpu.o
 obj-$(CONFIG_POWER_SEQUENCING_PCIE_M2)	+= pwrseq-pcie-m2.o
+obj-$(CONFIG_POWER_SEQUENCING_KUNIT)	+= pwrseq-kunit.o
diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c
index 3076b3879af9e2..d7a545bb4ed2e2 100644
--- a/drivers/power/sequencing/core.c
+++ b/drivers/power/sequencing/core.c
@@ -183,12 +183,15 @@ static void pwrseq_unit_release(struct kref *ref)
  *               the state lock has been released. It's useful for implementing
  *               boot-up delays without blocking other users from powering up
  *               using the same power sequencer.
+ * @is_controllable: Optional callback reporting whether enabling/disabling
+ *                   this target actually controls power.
  */
 struct pwrseq_target {
 	struct list_head list;
 	const char *name;
 	struct pwrseq_unit *unit;
 	pwrseq_power_state_func post_enable;
+	pwrseq_is_controllable_func is_controllable;
 };
 
 static struct pwrseq_target *
@@ -207,6 +210,7 @@ pwrseq_target_new(const struct pwrseq_target_data *data)
 	}
 
 	target->post_enable = data->post_enable;
+	target->is_controllable = data->is_controllable;
 
 	return target;
 }
@@ -994,6 +998,51 @@ struct device *pwrseq_to_device(struct pwrseq_desc *desc)
 }
 EXPORT_SYMBOL_GPL(pwrseq_to_device);
 
+/**
+ * pwrseq_is_controllable() - Check whether the target provides a
+ *                            host-controllable power actuator.
+ * @desc: Descriptor referencing the power sequencer.
+ *
+ * Some power sequencing targets provide no host-controllable enable for their
+ * function on a given board, for instance when the enable line is not wired up
+ * and is instead hardwired to an always-on level. For such targets a call to
+ * pwrseq_power_off() is still allowed, so that the consumer can drop its vote
+ * on the (possibly shared) resources, but the host cannot gate the function
+ * on its own.
+ *
+ * Returns:
+ * True if the target provides a host-controllable power actuator, false
+ * otherwise. Also returns false if @desc is NULL.
+ */
+bool pwrseq_is_controllable(struct pwrseq_desc *desc)
+{
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_target *target;
+	struct pwrseq_unit *unit;
+
+	might_sleep();
+
+	if (!desc)
+		return false;
+
+	pwrseq = desc->pwrseq;
+	target = desc->target;
+	unit = target->unit;
+
+	guard(rwsem_read)(&pwrseq->rw_lock);
+	if (!device_is_registered(&pwrseq->dev))
+		return false;
+
+	if (!unit->enable && !unit->disable)
+		return false;
+
+	if (!target->is_controllable)
+		return true;
+
+	return target->is_controllable(pwrseq);
+}
+EXPORT_SYMBOL_GPL(pwrseq_is_controllable);
+
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 
 struct pwrseq_debugfs_count_ctx {
diff --git a/drivers/power/sequencing/pwrseq-kunit.c b/drivers/power/sequencing/pwrseq-kunit.c
new file mode 100644
index 00000000000000..c2899f3b16e645
--- /dev/null
+++ b/drivers/power/sequencing/pwrseq-kunit.c
@@ -0,0 +1,1497 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Qualcomm Technologies, Inc. and/or its subsidiaries
+ */
+
+#include <linux/err.h>
+#include <linux/fwnode.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/pwrseq/consumer.h>
+#include <linux/pwrseq/provider.h>
+
+#include <kunit/fwnode.h>
+#include <kunit/platform_device.h>
+#include <kunit/resource.h>
+#include <kunit/test.h>
+
+#define PWRSEQ_TEST_PARENT			"pwrseq-test-parent"
+#define PWRSEQ_TEST_CONSUMER			"pwrseq-test-consumer"
+
+#define PWRSEQ_SWNODE_TEST_PROVIDER		"pwrseq-swnode-test-provider"
+#define PWRSEQ_SWNODE_TEST_PROVIDER_2		"pwrseq-swnode-test-provider-2"
+#define PWRSEQ_PROBE_ORDER_TEST_CONSUMER	"pwrseq-probe-order-test-consumer"
+#define PWRSEQ_PROBE_DEFER_TEST_CONSUMER	"pwrseq-probe-defer-test-consumer"
+
+static const struct software_node pwrseq_test_provider_swnode = {
+	.name = "pwrseq-test-provider",
+};
+
+KUNIT_DEFINE_ACTION_WRAPPER(pwrseq_device_unregister_wrapper,
+			    pwrseq_device_unregister,
+			    struct pwrseq_device *);
+
+static struct pwrseq_device *
+kunit_pwrseq_device_register(struct kunit *test,
+			     const struct pwrseq_config *config)
+{
+	struct pwrseq_device *pwrseq;
+	int ret;
+
+	pwrseq = pwrseq_device_register(config);
+	if (IS_ERR(pwrseq))
+		return pwrseq;
+
+	ret = kunit_add_action_or_reset(test, pwrseq_device_unregister_wrapper,
+					pwrseq);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return pwrseq;
+}
+
+KUNIT_DEFINE_ACTION_WRAPPER(pwrseq_put_wrapper, pwrseq_put,
+			    struct pwrseq_desc *);
+
+static struct pwrseq_desc *
+kunit_pwrseq_get(struct kunit *test, struct device *dev, const char *target)
+{
+	struct pwrseq_desc *desc;
+	int ret;
+
+	desc = pwrseq_get(dev, target);
+	if (IS_ERR(desc))
+		return desc;
+
+	ret = kunit_add_action_or_reset(test, pwrseq_put_wrapper, desc);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return desc;
+}
+
+struct pwrseq_test_ctx {
+	const char *consumer_name;
+	int enable_count_a;
+	int enable_count_b;
+	int disable_count_a;
+	int disable_count_b;
+	int enable_calls_a;
+	int post_enable_calls;
+	bool enable_error_a;
+	bool enable_error_b;
+	bool post_enable_error;
+};
+
+static int pwrseq_test_no_match(struct pwrseq_device *pwrseq,
+				struct device *dev)
+{
+	return PWRSEQ_NO_MATCH;
+}
+
+static int pwrseq_test_match_by_name(struct pwrseq_device *pwrseq,
+				     struct device *dev)
+{
+	struct pwrseq_test_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	return strcmp(dev_name(dev), ctx->consumer_name) == 0 ?
+					PWRSEQ_MATCH_OK : PWRSEQ_NO_MATCH;
+}
+
+static int pwrseq_test_match_always(struct pwrseq_device *pwrseq,
+				    struct device *dev)
+{
+	return PWRSEQ_MATCH_OK;
+}
+
+static int pwrseq_test_enable_a(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_test_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	ctx->enable_calls_a++;
+
+	if (ctx->enable_error_a)
+		return -EIO;
+
+	ctx->enable_count_a++;
+
+	return 0;
+}
+
+static int pwrseq_test_disable_a(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_test_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	ctx->disable_count_a++;
+
+	return 0;
+}
+
+static int pwrseq_test_enable_b(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_test_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	if (ctx->enable_error_b)
+		return -EIO;
+
+	ctx->enable_count_b++;
+
+	return 0;
+}
+
+static int pwrseq_test_disable_b(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_test_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	ctx->disable_count_b++;
+
+	return 0;
+}
+
+static int pwrseq_test_post_enable(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_test_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	ctx->post_enable_calls++;
+
+	return ctx->post_enable_error ? -EIO : 0;
+}
+
+static int pwrseq_test_parent_probe(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static struct platform_driver pwrseq_test_parent_driver = {
+	.probe = pwrseq_test_parent_probe,
+	.driver = {
+		.name = PWRSEQ_TEST_PARENT,
+	},
+};
+
+static int pwrseq_test_parent_init(struct kunit *test)
+{
+	static const struct platform_device_info pdevinfo = {
+		.name = PWRSEQ_TEST_PARENT,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	struct platform_device *pdev;
+	bool bound;
+	int ret;
+
+	ret = kunit_platform_driver_register(test, &pwrseq_test_parent_driver);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	wait_for_device_probe();
+	scoped_guard(device, &pdev->dev)
+		bound = device_is_bound(&pdev->dev);
+
+	KUNIT_ASSERT_TRUE(test, bound);
+
+	test->priv = pdev;
+
+	return 0;
+}
+
+/*
+ * Test that registering a power sequencer whose unit dependency graph
+ * contains a two-node cycle (A -> B -> A) is rejected with -EINVAL.
+ */
+static void pwrseq_circular_deps(struct kunit *test)
+{
+	const struct pwrseq_unit_data *unit_a_deps[2] = { };
+	const struct pwrseq_unit_data *unit_b_deps[2] = { };
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct pwrseq_unit_data unit_a, unit_b;
+	struct pwrseq_target_data target;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+
+	unit_a = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.deps = unit_a_deps,
+	};
+
+	unit_b = (struct pwrseq_unit_data){
+		.name = "unit-b",
+		.deps = unit_b_deps,
+	};
+
+	unit_a_deps[0] = &unit_b;
+	unit_b_deps[0] = &unit_a;
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit_a,
+	};
+
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.match = pwrseq_test_no_match,
+		.targets = targets,
+	};
+
+	kunit_warning_suppress(test) {
+		pwrseq = kunit_pwrseq_device_register(test, &config);
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+	}
+
+	KUNIT_EXPECT_TRUE(test, IS_ERR(pwrseq));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(pwrseq), -EINVAL);
+}
+
+/*
+ * Test that a longer chain cycle (A -> B -> C -> D -> A) is also rejected
+ * with -EINVAL.
+ */
+static void pwrseq_circular_deps_chain(struct kunit *test)
+{
+	struct pwrseq_unit_data unit_a, unit_b, unit_c, unit_d;
+	const struct pwrseq_unit_data *unit_a_deps[2] = { };
+	const struct pwrseq_unit_data *unit_b_deps[2] = { };
+	const struct pwrseq_unit_data *unit_c_deps[2] = { };
+	const struct pwrseq_unit_data *unit_d_deps[2] = { };
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct pwrseq_target_data target;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+
+	unit_a = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.deps = unit_a_deps,
+	};
+	unit_b = (struct pwrseq_unit_data){
+		.name = "unit-b",
+		.deps = unit_b_deps,
+	};
+	unit_c = (struct pwrseq_unit_data){
+		.name = "unit-c",
+		.deps = unit_c_deps,
+	};
+	unit_d = (struct pwrseq_unit_data){
+		.name = "unit-d",
+		.deps = unit_d_deps,
+	};
+
+	unit_a_deps[0] = &unit_b;
+	unit_b_deps[0] = &unit_c;
+	unit_c_deps[0] = &unit_d;
+	unit_d_deps[0] = &unit_a;
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit_a,
+	};
+
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.match = pwrseq_test_no_match,
+		.targets = targets,
+	};
+
+	kunit_warning_suppress(test) {
+		pwrseq = kunit_pwrseq_device_register(test, &config);
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+	}
+
+	KUNIT_EXPECT_TRUE(test, IS_ERR(pwrseq));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(pwrseq), -EINVAL);
+}
+
+/*
+ * Test that a valid acyclic dependency graph (A -> B) registers successfully.
+ */
+static void pwrseq_register_valid(struct kunit *test)
+{
+	const struct pwrseq_unit_data *unit_a_deps[2] = { };
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct pwrseq_unit_data unit_a, unit_b;
+	struct pwrseq_target_data target;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+
+	unit_b = (struct pwrseq_unit_data){
+		.name = "unit-b",
+	};
+
+	unit_a_deps[0] = &unit_b;
+	unit_a = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.deps = unit_a_deps,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit_a,
+	};
+
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.match = pwrseq_test_no_match,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+}
+
+/*
+ * Test that registration is rejected with -EINVAL when .parent, .match or
+ * .targets is missing, or when .targets is a non-NULL but empty array.
+ */
+static void pwrseq_register_invalid_args(struct kunit *test)
+{
+	const struct pwrseq_target_data *empty_targets[1] = { };
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct pwrseq_target_data target;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+
+	unit = (struct pwrseq_unit_data){ .name = "unit-a" };
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit,
+	};
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.match = pwrseq_test_no_match,
+		.targets = targets,
+	};
+
+	config.parent = NULL;
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_EXPECT_TRUE(test, IS_ERR(pwrseq));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(pwrseq), -EINVAL);
+	config.parent = &pdev->dev;
+
+	config.match = NULL;
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_EXPECT_TRUE(test, IS_ERR(pwrseq));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(pwrseq), -EINVAL);
+	config.match = pwrseq_test_no_match;
+
+	config.targets = NULL;
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_EXPECT_TRUE(test, IS_ERR(pwrseq));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(pwrseq), -EINVAL);
+
+	config.targets = empty_targets;
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_EXPECT_TRUE(test, IS_ERR(pwrseq));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(pwrseq), -EINVAL);
+}
+
+/*
+ * Test that a target without a unit is rejected with -EINVAL.
+ */
+static void pwrseq_register_target_without_unit(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct pwrseq_target_data target;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = NULL,
+	};
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.match = pwrseq_test_no_match,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_EXPECT_TRUE(test, IS_ERR(pwrseq));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(pwrseq), -EINVAL);
+}
+
+/*
+ * Test that enabling and disabling a single-unit target increments and
+ * decrements enable_count correctly and fires the enable/disable callbacks
+ * exactly once.
+ */
+static void pwrseq_enable_disable(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_test_ctx *ctx;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+	struct pwrseq_desc *desc;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	unit = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.enable = pwrseq_test_enable_a,
+		.disable = pwrseq_test_disable_a,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit,
+	};
+
+	targets[0] = &target;
+
+	ctx->consumer_name = PWRSEQ_TEST_CONSUMER;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_by_name,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	desc = kunit_pwrseq_get(test, &pdev->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 0);
+
+	ret = pwrseq_disable(desc);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 1);
+}
+
+/*
+ * Test that two consumers sharing a common dependency unit cause it to be
+ * enabled only once and disabled only once, while enable_count tracks each
+ * consumer correctly.
+ */
+static void pwrseq_shared_deps(struct kunit *test)
+{
+	const struct pwrseq_unit_data *unit_a_deps[2] = { };
+	const struct pwrseq_unit_data *unit_b_deps[2] = { };
+	const struct pwrseq_target_data *targets[3] = { };
+	struct pwrseq_unit_data dep_unit, unit_a, unit_b;
+	struct pwrseq_target_data target_a, target_b;
+	struct platform_device *parent = test->priv;
+	struct platform_device *pdev_a, *pdev_b;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_desc *desc_a, *desc_b;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+	struct pwrseq_test_ctx *ctx;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	dep_unit = (struct pwrseq_unit_data){
+		.name = "dep",
+		.enable = pwrseq_test_enable_a,
+		.disable = pwrseq_test_disable_a,
+	};
+
+	unit_a_deps[0] = &dep_unit;
+	unit_a = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.deps = unit_a_deps,
+		.enable = pwrseq_test_enable_b,
+		.disable = pwrseq_test_disable_b,
+	};
+
+	unit_b_deps[0] = &dep_unit;
+	unit_b = (struct pwrseq_unit_data){
+		.name = "unit-b",
+		.deps = unit_b_deps,
+	};
+
+	target_a = (struct pwrseq_target_data){
+		.name = "target-a",
+		.unit = &unit_a,
+	};
+	target_b = (struct pwrseq_target_data){
+		.name = "target-b",
+		.unit = &unit_b,
+	};
+
+	targets[0] = &target_a;
+	targets[1] = &target_b;
+
+	ctx->consumer_name = PWRSEQ_TEST_CONSUMER "-a";
+
+	config = (struct pwrseq_config){
+		.parent = &parent->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_by_name,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER "-a",
+		.id = PLATFORM_DEVID_NONE,
+	};
+	pdev_a = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev_a);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER "-b",
+		.id = PLATFORM_DEVID_NONE,
+	};
+	pdev_b = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev_b);
+
+	/*
+	 * match_by_name() only matches one consumer name at a time, so
+	 * reconfigure ctx->consumer_name between the two pwrseq_get() calls
+	 * below to acquire descriptors for both consumer-a and consumer-b.
+	 */
+	desc_a = kunit_pwrseq_get(test, &pdev_a->dev, "target-a");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc_a);
+
+	ctx->consumer_name = PWRSEQ_TEST_CONSUMER "-b";
+	desc_b = kunit_pwrseq_get(test, &pdev_b->dev, "target-b");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc_b);
+
+	ret = pwrseq_enable(desc_a);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_b, 1);
+
+	ret = pwrseq_enable(desc_b);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+
+	ret = pwrseq_disable(desc_a);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 0);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_b, 1);
+
+	ret = pwrseq_disable(desc_b);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 1);
+}
+
+/*
+ * Test that calling pwrseq_enable() twice on the same descriptor has no
+ * effect.
+ */
+static void pwrseq_enable_idempotent(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_config config;
+	struct pwrseq_test_ctx *ctx;
+	struct pwrseq_desc *desc;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	unit = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.enable = pwrseq_test_enable_a,
+		.disable = pwrseq_test_disable_a,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit,
+	};
+
+	targets[0] = &target;
+
+	ctx->consumer_name = PWRSEQ_TEST_CONSUMER;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_by_name,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	desc = kunit_pwrseq_get(test, &pdev->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+
+	/* Second power_on on same descriptor must be a no-op. */
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+
+	ret = pwrseq_disable(desc);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 1);
+}
+
+/*
+ * Test that when a dependency unit's enable() callback fails, the error
+ * propagates to the caller, the top unit's enable callback is never reached,
+ * and target->post_enable() is never invoked.
+ */
+static void pwrseq_enable_enable_error(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	const struct pwrseq_unit_data *top_deps[2] = { };
+	struct pwrseq_unit_data dep_unit, top_unit;
+	struct platform_device *pdev = test->priv;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+	struct pwrseq_test_ctx *ctx;
+	struct pwrseq_desc *desc;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	dep_unit = (struct pwrseq_unit_data){
+		.name = "dep",
+		.enable = pwrseq_test_enable_a,
+	};
+
+	top_deps[0] = &dep_unit;
+	top_unit = (struct pwrseq_unit_data){
+		.name = "top",
+		.deps = top_deps,
+		.enable = pwrseq_test_enable_b,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &top_unit,
+		.post_enable = pwrseq_test_post_enable,
+	};
+
+	targets[0] = &target;
+
+	ctx->consumer_name = PWRSEQ_TEST_CONSUMER;
+	ctx->enable_error_a = true;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_by_name,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	desc = kunit_pwrseq_get(test, &pdev->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, -EIO);
+	/*
+	 * Dependency's enable() was attempted but failed, so it never
+	 * incremented.
+	 */
+	KUNIT_EXPECT_EQ(test, ctx->enable_calls_a, 1);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 0);
+	/* Target unit's enable() was never reached. */
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_b, 0);
+	/* Target's .post_enable() must not run on an unpowered sequencer. */
+	KUNIT_EXPECT_EQ(test, ctx->post_enable_calls, 0);
+}
+
+/*
+ * Test that when the top unit's own enable() callback fails after its
+ * dependency's enable() already succeeded, the dependency is rolled back.
+ */
+static void pwrseq_enable_rollback_on_top_failure(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	const struct pwrseq_unit_data *top_deps[2] = { };
+	struct pwrseq_unit_data dep_unit, top_unit;
+	struct platform_device *pdev = test->priv;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+	struct pwrseq_test_ctx *ctx;
+	struct pwrseq_desc *desc;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	dep_unit = (struct pwrseq_unit_data){
+		.name = "dep",
+		.enable = pwrseq_test_enable_a,
+		.disable = pwrseq_test_disable_a,
+	};
+
+	top_deps[0] = &dep_unit;
+	top_unit = (struct pwrseq_unit_data){
+		.name = "top",
+		.deps = top_deps,
+		.enable = pwrseq_test_enable_b,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &top_unit,
+	};
+
+	targets[0] = &target;
+
+	ctx->consumer_name = PWRSEQ_TEST_CONSUMER;
+	ctx->enable_error_b = true;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_by_name,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	desc = kunit_pwrseq_get(test, &pdev->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, -EIO);
+	/*
+	 * Dependency was enabled, then rolled back once top's own enable
+	 * failed.
+	 */
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 1);
+	/*
+	 * Target unit's own enable never incremented since it returned an
+	 * error.
+	 */
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_b, 0);
+}
+
+/*
+ * Test target->post_enable() on both the success and failure paths: it must
+ * run exactly once per pwrseq_enable() call that actually powers the target
+ * unit on, and a failing post_enable() must roll the unit back and clear
+ * powered_on so a subsequent pwrseq_enable() is called again.
+ */
+static void pwrseq_enable_post_enable(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+	struct pwrseq_test_ctx *ctx;
+	struct pwrseq_desc *desc;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	unit = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.enable = pwrseq_test_enable_a,
+		.disable = pwrseq_test_disable_a,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit,
+		.post_enable = pwrseq_test_post_enable,
+	};
+
+	targets[0] = &target;
+
+	ctx->consumer_name = PWRSEQ_TEST_CONSUMER;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_by_name,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	desc = kunit_pwrseq_get(test, &pdev->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	/* Success path: post_enable() runs once and reports success. */
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->post_enable_calls, 1);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 1);
+
+	ret = pwrseq_disable(desc);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	/* Failure path: post_enable() fails, unit is rolled back. */
+	ctx->post_enable_error = true;
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, -EIO);
+	KUNIT_EXPECT_EQ(test, ctx->post_enable_calls, 2);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 2);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 2);
+
+	/* powered_on must have been cleared. */
+	ret = pwrseq_enable(desc);
+	KUNIT_EXPECT_EQ(test, ret, -EIO);
+	KUNIT_EXPECT_EQ(test, ctx->enable_count_a, 3);
+}
+
+/*
+ * Test that pwrseq_get() returns -ENOENT when the matched provider does not
+ * have the requested target name.
+ */
+static void pwrseq_get_target_not_found(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+	struct pwrseq_desc *desc;
+
+	unit = (struct pwrseq_unit_data){
+		.name = "unit-a",
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "real-target",
+		.unit = &unit,
+	};
+
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.match = pwrseq_test_match_always,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	desc = kunit_pwrseq_get(test, &pdev->dev, "nonexistent-target");
+	KUNIT_EXPECT_TRUE(test, IS_ERR(desc));
+	KUNIT_EXPECT_EQ(test, PTR_ERR(desc), -ENOENT);
+}
+
+/*
+ * Test that pwrseq_put() on a descriptor that is still powered on disables
+ * it as part of tear-down, instead of leaking the power-on state.
+ */
+static void pwrseq_put_disables_powered_desc(struct kunit *test)
+{
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device *pdev = test->priv;
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_test_ctx *ctx;
+	struct pwrseq_config config;
+	struct pwrseq_desc *desc;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	unit = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.enable = pwrseq_test_enable_a,
+		.disable = pwrseq_test_disable_a,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit,
+	};
+
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_always,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	pdev = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev);
+
+	desc = kunit_pwrseq_get(test, &pdev->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	ret = pwrseq_enable(desc);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 0);
+
+	/*
+	 * Run the deferred pwrseq_put() action now instead of at test
+	 * teardown, so its effect can be asserted below.
+	 */
+	kunit_release_action(test, pwrseq_put_wrapper, desc);
+	KUNIT_EXPECT_EQ(test, ctx->disable_count_a, 1);
+}
+
+/*
+ * Test that pwrseq_device_unregister() on a target with an active user
+ * triggers "REMOVING POWER SEQUENCER WITH ACTIVE USERS", and that both
+ * pwrseq_enable() and pwrseq_disable() start returning -ENODEV for
+ * previously acquired descriptors afterwards.
+ */
+static void pwrseq_unregister_with_active_user(struct kunit *test)
+{
+	struct platform_device *pdev = test->priv, *cons_a, *cons_b;
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device_info pdevinfo;
+	struct pwrseq_desc *desc_a, *desc_b;
+	struct pwrseq_target_data target;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_test_ctx *ctx;
+	struct pwrseq_config config;
+	bool bound;
+	int ret;
+
+	ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ctx);
+
+	unit = (struct pwrseq_unit_data){
+		.name = "unit-a",
+		.enable = pwrseq_test_enable_a,
+		.disable = pwrseq_test_disable_a,
+	};
+
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit,
+	};
+
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = ctx,
+		.match = pwrseq_test_match_always,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = 0,
+	};
+	cons_a = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cons_a);
+
+	wait_for_device_probe();
+	scoped_guard(device, &cons_a->dev)
+		bound = device_is_bound(&cons_a->dev);
+
+	KUNIT_ASSERT_FALSE(test, bound);
+
+	pdevinfo.id = 1;
+	cons_b = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cons_b);
+
+	wait_for_device_probe();
+	scoped_guard(device, &cons_b->dev)
+		bound = device_is_bound(&cons_b->dev);
+
+	KUNIT_ASSERT_FALSE(test, bound);
+
+	desc_a = kunit_pwrseq_get(test, &cons_a->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc_a);
+	desc_b = kunit_pwrseq_get(test, &cons_b->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc_b);
+
+	ret = pwrseq_enable(desc_a);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	kunit_warning_suppress(test) {
+		kunit_release_action(test, pwrseq_device_unregister_wrapper, pwrseq);
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+	}
+
+	ret = pwrseq_enable(desc_b);
+	KUNIT_EXPECT_EQ(test, ret, -ENODEV);
+
+	ret = pwrseq_disable(desc_a);
+	KUNIT_EXPECT_EQ(test, ret, -ENODEV);
+}
+
+/*
+ * Test that pwrseq_to_device() returns the pwrseq provider's own device,
+ * and NULL for a NULL descriptor.
+ */
+static void pwrseq_to_device_test(struct kunit *test)
+{
+	struct platform_device *parent = test->priv, *cons;
+	const struct pwrseq_target_data *targets[2] = { };
+	struct platform_device_info pdevinfo;
+	struct pwrseq_target_data target;
+	struct pwrseq_unit_data unit;
+	struct pwrseq_device *pwrseq;
+	struct pwrseq_config config;
+	struct pwrseq_desc *desc;
+	struct device *dev;
+
+	unit = (struct pwrseq_unit_data){ .name = "unit-a" };
+	target = (struct pwrseq_target_data){
+		.name = "test-target",
+		.unit = &unit,
+	};
+	targets[0] = &target;
+
+	config = (struct pwrseq_config){
+		.parent = &parent->dev,
+		.match = pwrseq_test_match_always,
+		.targets = targets,
+	};
+
+	pwrseq = kunit_pwrseq_device_register(test, &config);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pwrseq);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+	};
+
+	cons = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cons);
+
+	desc = kunit_pwrseq_get(test, &cons->dev, "test-target");
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, desc);
+
+	dev = pwrseq_to_device(desc);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+	KUNIT_EXPECT_PTR_EQ(test, dev->parent, &parent->dev);
+	KUNIT_EXPECT_TRUE(test, device_is_registered(dev));
+
+	KUNIT_EXPECT_PTR_EQ(test, pwrseq_to_device(NULL), NULL);
+}
+
+static struct kunit_case pwrseq_tests[] = {
+	KUNIT_CASE(pwrseq_circular_deps),
+	KUNIT_CASE(pwrseq_circular_deps_chain),
+	KUNIT_CASE(pwrseq_register_valid),
+	KUNIT_CASE(pwrseq_register_invalid_args),
+	KUNIT_CASE(pwrseq_register_target_without_unit),
+	KUNIT_CASE(pwrseq_enable_disable),
+	KUNIT_CASE(pwrseq_shared_deps),
+	KUNIT_CASE(pwrseq_enable_idempotent),
+	KUNIT_CASE(pwrseq_enable_enable_error),
+	KUNIT_CASE(pwrseq_enable_rollback_on_top_failure),
+	KUNIT_CASE(pwrseq_enable_post_enable),
+	KUNIT_CASE(pwrseq_get_target_not_found),
+	KUNIT_CASE(pwrseq_put_disables_powered_desc),
+	KUNIT_CASE(pwrseq_unregister_with_active_user),
+	KUNIT_CASE(pwrseq_to_device_test),
+	{ }
+};
+
+static struct kunit_suite pwrseq_test_suite = {
+	.name = "pwrseq",
+	.init = pwrseq_test_parent_init,
+	.test_cases = pwrseq_tests,
+};
+
+/*
+ * Match a consumer to this test provider by resolving its "pwrseq-provider"
+ * software-node reference and comparing it to the provider's own fwnode.
+ */
+static int pwrseq_test_match_by_swnode_ref(struct pwrseq_device *pwrseq,
+					   struct device *dev)
+{
+	const struct software_node *provider_swnode;
+	struct fwnode_handle *provider_fwnode, *ref;
+	bool match;
+
+	if (!dev_fwnode(dev))
+		return PWRSEQ_NO_MATCH;
+
+	ref = fwnode_find_reference(dev_fwnode(dev), "pwrseq-provider", 0);
+	if (IS_ERR_OR_NULL(ref))
+		return PWRSEQ_NO_MATCH;
+
+	provider_swnode = pwrseq_device_get_drvdata(pwrseq);
+	provider_fwnode = software_node_fwnode(provider_swnode);
+	match = (ref == provider_fwnode);
+	fwnode_handle_put(ref);
+
+	return match ? PWRSEQ_MATCH_OK : PWRSEQ_NO_MATCH;
+}
+
+static const struct pwrseq_unit_data pwrseq_swnode_test_unit = {
+	.name = "test-target",
+};
+
+static const struct pwrseq_target_data pwrseq_swnode_test_target = {
+	.name = "test-target",
+	.unit = &pwrseq_swnode_test_unit,
+};
+
+static const struct pwrseq_target_data *pwrseq_swnode_test_targets[] = {
+	&pwrseq_swnode_test_target,
+	NULL,
+};
+
+struct pwrseq_swnode_provider_pdata {
+	const struct software_node *provider_swnode;
+};
+
+static int pwrseq_swnode_provider_probe(struct platform_device *pdev)
+{
+	const struct pwrseq_swnode_provider_pdata *pdata = dev_get_platdata(&pdev->dev);
+	struct pwrseq_config config;
+
+	config = (struct pwrseq_config){
+		.parent = &pdev->dev,
+		.drvdata = (void *)pdata->provider_swnode,
+		.match = pwrseq_test_match_by_swnode_ref,
+		.targets = pwrseq_swnode_test_targets,
+	};
+
+	return PTR_ERR_OR_ZERO(devm_pwrseq_device_register(&pdev->dev, &config));
+}
+
+static struct platform_driver pwrseq_swnode_provider_driver = {
+	.probe = pwrseq_swnode_provider_probe,
+	.driver = {
+		.name = PWRSEQ_SWNODE_TEST_PROVIDER,
+	},
+};
+
+static struct platform_driver pwrseq_swnode_provider_driver_2 = {
+	.probe = pwrseq_swnode_provider_probe,
+	.driver = {
+		.name = PWRSEQ_SWNODE_TEST_PROVIDER_2,
+	},
+};
+
+struct pwrseq_probe_order_pdata {
+	unsigned int probe_count;
+	int pwrseq_err;
+};
+
+static const struct pwrseq_probe_order_pdata pwrseq_probe_order_pdata_template;
+
+static int pwrseq_probe_order_consumer_probe(struct platform_device *pdev)
+{
+	struct pwrseq_probe_order_pdata *pdata = dev_get_platdata(&pdev->dev);
+	struct pwrseq_desc *desc;
+
+	pdata->probe_count++;
+
+	desc = devm_pwrseq_get(&pdev->dev, "test-target");
+	pdata->pwrseq_err = PTR_ERR_OR_ZERO(desc);
+	if (IS_ERR(desc))
+		return PTR_ERR(desc);
+
+	return 0;
+}
+
+static struct platform_driver pwrseq_probe_order_consumer_driver = {
+	.probe = pwrseq_probe_order_consumer_probe,
+	.driver = {
+		.name = PWRSEQ_PROBE_ORDER_TEST_CONSUMER,
+	},
+};
+
+static struct platform_driver pwrseq_probe_defer_consumer_driver = {
+	.probe = pwrseq_probe_order_consumer_probe,
+	.driver = {
+		.name = PWRSEQ_PROBE_DEFER_TEST_CONSUMER,
+	},
+};
+
+/*
+ * Verify that driver core orders the probe of a pwrseq consumer after its
+ * provider. The consumer references the provider through a software node and
+ * is registered first and we rely on devlink for ordering.
+ */
+static void pwrseq_swnode_probe_order(struct kunit *test)
+{
+	struct property_entry properties[2] = { };
+	struct pwrseq_probe_order_pdata *pdata;
+	struct platform_device_info pdevinfo;
+	struct platform_device *prvd, *cons;
+	struct fwnode_handle *fwnode;
+	bool bound = false;
+	int ret;
+
+	ret = kunit_platform_driver_register(test, &pwrseq_swnode_provider_driver);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ret = kunit_platform_driver_register(test, &pwrseq_probe_order_consumer_driver);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	fwnode = kunit_software_node_register(test, &pwrseq_test_provider_swnode);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode);
+
+	properties[0] = PROPERTY_ENTRY_REF("pwrseq-provider",
+					   &pwrseq_test_provider_swnode);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_PROBE_ORDER_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+		.data = &pwrseq_probe_order_pdata_template,
+		.size_data = sizeof(pwrseq_probe_order_pdata_template),
+		.properties = properties,
+	};
+
+	cons = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cons);
+
+	wait_for_device_probe();
+	scoped_guard(device, &cons->dev)
+		bound = device_is_bound(&cons->dev);
+
+	KUNIT_ASSERT_FALSE(test, bound);
+
+	pdata = dev_get_platdata(&cons->dev);
+	KUNIT_ASSERT_EQ(test, pdata->probe_count, 0);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_SWNODE_TEST_PROVIDER,
+		.id = PLATFORM_DEVID_NONE,
+		.swnode = &pwrseq_test_provider_swnode,
+		.data = &(const struct pwrseq_swnode_provider_pdata){
+			.provider_swnode = &pwrseq_test_provider_swnode,
+		},
+		.size_data = sizeof(struct pwrseq_swnode_provider_pdata),
+	};
+
+	prvd = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prvd);
+
+	wait_for_device_probe();
+
+	scoped_guard(device, &prvd->dev)
+		bound = device_is_bound(&prvd->dev);
+	KUNIT_ASSERT_TRUE(test, bound);
+
+	scoped_guard(device, &cons->dev)
+		bound = device_is_bound(&cons->dev);
+	KUNIT_ASSERT_TRUE(test, bound);
+
+	pdata = dev_get_platdata(&cons->dev);
+	KUNIT_EXPECT_EQ(test, pdata->probe_count, 1);
+	KUNIT_EXPECT_EQ(test, pdata->pwrseq_err, 0);
+}
+
+/*
+ * Verify that a pwrseq consumer referencing a provider whose software node is
+ * not registered yet, defers its probe instead of failing.
+ *
+ * The provider software node is deliberately left unregistered when the
+ * consumer is added. fw_devlink cannot resolve the reference, so it creates no
+ * supplier link and does not order the consumer - the consumer's probe() runs
+ * and calls pwrseq_get(), which finds no matching provider and returns
+ * -EPROBE_DEFER. Once the provider software node and device appear, the
+ * deferred consumer probes again and binds.
+ */
+static void pwrseq_swnode_probe_defer_on_unregistered(struct kunit *test)
+{
+	struct property_entry properties[2] = { };
+	struct pwrseq_probe_order_pdata *pdata;
+	struct platform_device_info pdevinfo;
+	struct platform_device *prvd, *cons;
+	struct fwnode_handle *fwnode;
+	bool bound = false;
+	int ret;
+
+	ret = kunit_platform_driver_register(test, &pwrseq_swnode_provider_driver_2);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	ret = kunit_platform_driver_register(test, &pwrseq_probe_defer_consumer_driver);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	properties[0] = PROPERTY_ENTRY_REF("pwrseq-provider",
+					   &pwrseq_test_provider_swnode);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_PROBE_DEFER_TEST_CONSUMER,
+		.id = PLATFORM_DEVID_NONE,
+		.data = &pwrseq_probe_order_pdata_template,
+		.size_data = sizeof(pwrseq_probe_order_pdata_template),
+		.properties = properties,
+	};
+
+	cons = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cons);
+
+	wait_for_device_probe();
+	scoped_guard(device, &cons->dev)
+		bound = device_is_bound(&cons->dev);
+
+	KUNIT_ASSERT_FALSE(test, bound);
+
+	pdata = dev_get_platdata(&cons->dev);
+	KUNIT_ASSERT_GT(test, pdata->probe_count, 0);
+	KUNIT_ASSERT_EQ(test, pdata->pwrseq_err, -EPROBE_DEFER);
+
+	fwnode = kunit_software_node_register(test, &pwrseq_test_provider_swnode);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode);
+
+	pdevinfo = (struct platform_device_info){
+		.name = PWRSEQ_SWNODE_TEST_PROVIDER_2,
+		.id = PLATFORM_DEVID_NONE,
+		.swnode = &pwrseq_test_provider_swnode,
+		.data = &(const struct pwrseq_swnode_provider_pdata){
+			.provider_swnode = &pwrseq_test_provider_swnode,
+		},
+		.size_data = sizeof(struct pwrseq_swnode_provider_pdata),
+	};
+
+	prvd = kunit_platform_device_register_full(test, &pdevinfo);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prvd);
+
+	wait_for_device_probe();
+
+	scoped_guard(device, &prvd->dev)
+		bound = device_is_bound(&prvd->dev);
+	KUNIT_ASSERT_TRUE(test, bound);
+
+	scoped_guard(device, &cons->dev)
+		bound = device_is_bound(&cons->dev);
+	KUNIT_ASSERT_TRUE(test, bound);
+
+	pdata = dev_get_platdata(&cons->dev);
+	KUNIT_EXPECT_EQ(test, pdata->pwrseq_err, 0);
+}
+
+static int pwrseq_swnode_test_init(struct kunit *test)
+{
+	/*
+	 * A device link teardown from an earlier test case, or from this
+	 * suite's previous module load, may still be queued on device_link_mq.
+	 * Flush it so software_node_register() below doesn't spuriously see
+	 * the about-to-be-reused node name as still taken.
+	 */
+	device_link_wait_removal();
+
+	return 0;
+}
+
+static struct kunit_case pwrseq_swnode_probe_order_tests[] = {
+	KUNIT_CASE(pwrseq_swnode_probe_order),
+	KUNIT_CASE(pwrseq_swnode_probe_defer_on_unregistered),
+	{ }
+};
+
+static struct kunit_suite pwrseq_swnode_probe_order_test_suite = {
+	.name = "pwrseq-swnode-probe-order",
+	.init = pwrseq_swnode_test_init,
+	.test_cases = pwrseq_swnode_probe_order_tests,
+};
+
+kunit_test_suites(&pwrseq_test_suite,
+		  &pwrseq_swnode_probe_order_test_suite);
+
+MODULE_DESCRIPTION("KUnit test cases for the power sequencing subsystem");
+MODULE_AUTHOR("Bartosz Golaszewski <bartosz.golaszewski@oss.qualcomm.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/sequencing/pwrseq-pcie-m2.c b/drivers/power/sequencing/pwrseq-pcie-m2.c
index 471ffe914a4e21..91916768e92209 100644
--- a/drivers/power/sequencing/pwrseq-pcie-m2.c
+++ b/drivers/power/sequencing/pwrseq-pcie-m2.c
@@ -83,6 +83,18 @@ static int pwrseq_pci_m2_e_uart_disable(struct pwrseq_device *pwrseq)
 	return gpiod_set_value_cansleep(ctx->w_disable2_gpio, 1);
 }
 
+static bool pwrseq_pci_m2_e_uart_is_controllable(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_pcie_m2_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	/*
+	 * The UART enable is driven through the W_DISABLE2# line. When it is not
+	 * wired up on this connector the enable/disable callbacks are no-ops, so
+	 * the host cannot gate the Bluetooth function on its own.
+	 */
+	return !!ctx->w_disable2_gpio;
+}
+
 static const struct pwrseq_unit_data pwrseq_pcie_m2_e_uart_unit_data = {
 	.name = "uart-enable",
 	.deps = pwrseq_pcie_m2_unit_deps,
@@ -104,6 +116,18 @@ static int pwrseq_pci_m2_e_pcie_disable(struct pwrseq_device *pwrseq)
 	return gpiod_set_value_cansleep(ctx->w_disable1_gpio, 1);
 }
 
+static bool pwrseq_pci_m2_e_pcie_is_controllable(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_pcie_m2_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	/*
+	 * The PCIe/WiFi enable is driven through the W_DISABLE1# line. When it
+	 * is not wired up on this connector the enable/disable callbacks are
+	 * no-ops, so the host cannot gate the PCIe/WiFi function on its own.
+	 */
+	return !!ctx->w_disable1_gpio;
+}
+
 static const struct pwrseq_unit_data pwrseq_pcie_m2_e_pcie_unit_data = {
 	.name = "pcie-enable",
 	.deps = pwrseq_pcie_m2_unit_deps,
@@ -132,12 +156,14 @@ static const struct pwrseq_target_data pwrseq_pcie_m2_e_uart_target_data = {
 	.name = "uart",
 	.unit = &pwrseq_pcie_m2_e_uart_unit_data,
 	.post_enable = pwrseq_pcie_m2_e_pwup_delay,
+	.is_controllable = pwrseq_pci_m2_e_uart_is_controllable,
 };
 
 static const struct pwrseq_target_data pwrseq_pcie_m2_e_pcie_target_data = {
 	.name = "pcie",
 	.unit = &pwrseq_pcie_m2_e_pcie_unit_data,
 	.post_enable = pwrseq_pcie_m2_e_pwup_delay,
+	.is_controllable = pwrseq_pci_m2_e_pcie_is_controllable,
 };
 
 static const struct pwrseq_target_data pwrseq_pcie_m2_m_pcie_target_data = {
@@ -197,6 +223,8 @@ static const struct pci_device_id pwrseq_m2_pci_ids[] = {
 	  .driver_data = (kernel_ulong_t)"qcom,wcn6855-bt" },
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x1107, PCI_VENDOR_ID_QCOM, 0x337c),
 	  .driver_data = (kernel_ulong_t)"qcom,wcn7850-bt" },
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x1107, PCI_VENDOR_ID_LENOVO, 0xe0e9),
+	  .driver_data = (kernel_ulong_t)"qcom,wcn7850-bt" },
 	{ } /* Sentinel */
 };
 
@@ -376,6 +404,21 @@ static void pwrseq_pcie_m2_remove_serdev(struct pwrseq_pcie_m2_ctx *ctx,
 	mutex_unlock(&ctx->list_lock);
 }
 
+static bool pwrseq_pcie_m2_pci_parent_matches(struct pci_dev *pdev,
+					       struct device_node *pci_parent)
+{
+	struct device *dev = pdev->dev.parent;
+
+	while (dev) {
+		if (dev->of_node == pci_parent)
+			return true;
+		if (!dev_is_pci(dev))
+			break;
+		dev = dev->parent;
+	}
+	return false;
+}
+
 static int pwrseq_pcie_m2_notify(struct notifier_block *nb, unsigned long action,
 			      void *data)
 {
@@ -390,7 +433,7 @@ static int pwrseq_pcie_m2_notify(struct notifier_block *nb, unsigned long action
 	 */
 	struct device_node *pci_parent __free(device_node) =
 			of_graph_get_remote_node(dev_of_node(ctx->dev), 0, 0);
-	if (!pci_parent || (pci_parent != pdev->dev.parent->of_node))
+	if (!pci_parent || !pwrseq_pcie_m2_pci_parent_matches(pdev, pci_parent))
 		return NOTIFY_DONE;
 
 	switch (action) {
@@ -464,7 +507,7 @@ static int pwrseq_pcie_m2_create_serdev(struct pwrseq_pcie_m2_ctx *ctx)
 
 	/* Create serdev for existing PCI devices if required */
 	for_each_pci_dev(pdev) {
-		if (!pdev->dev.parent || pci_parent != pdev->dev.parent->of_node)
+		if (!pwrseq_pcie_m2_pci_parent_matches(pdev, pci_parent))
 			continue;
 
 		if (!pci_match_id(pwrseq_m2_pci_ids, pdev))
diff --git a/drivers/power/sequencing/pwrseq-qcom-wcn.c b/drivers/power/sequencing/pwrseq-qcom-wcn.c
index d41793e1fcd94e..7f88a29b2c33bb 100644
--- a/drivers/power/sequencing/pwrseq-qcom-wcn.c
+++ b/drivers/power/sequencing/pwrseq-qcom-wcn.c
@@ -183,6 +183,19 @@ static int pwrseq_qcom_wcn_bt_disable(struct pwrseq_device *pwrseq)
 	return 0;
 }
 
+static bool pwrseq_qcom_wcn_bt_is_controllable(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_qcom_wcn_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	/*
+	 * The Bluetooth enable is driven through the BT_EN GPIO. When it is not
+	 * wired up (e.g. hardwired to an always-on pull-up), the enable/disable
+	 * callbacks are no-ops, so the host cannot gate the Bluetooth function
+	 * on its own.
+	 */
+	return !!ctx->bt_gpio;
+}
+
 static const struct pwrseq_unit_data pwrseq_qcom_wcn_bt_unit_data = {
 	.name = "bluetooth-enable",
 	.deps = pwrseq_qcom_wcn_unit_deps,
@@ -217,6 +230,19 @@ static int pwrseq_qcom_wcn_wlan_disable(struct pwrseq_device *pwrseq)
 	return 0;
 }
 
+static bool pwrseq_qcom_wcn_wlan_is_controllable(struct pwrseq_device *pwrseq)
+{
+	struct pwrseq_qcom_wcn_ctx *ctx = pwrseq_device_get_drvdata(pwrseq);
+
+	/*
+	 * The WLAN enable is driven through the WLAN_EN GPIO. When it is not
+	 * wired up (e.g. hardwired to an always-on pull-up), the enable/disable
+	 * callbacks are no-ops, so the host cannot gate the WLAN function on
+	 * its own.
+	 */
+	return !!ctx->wlan_gpio;
+}
+
 static const struct pwrseq_unit_data pwrseq_qcom_wcn_wlan_unit_data = {
 	.name = "wlan-enable",
 	.deps = pwrseq_qcom_wcn_unit_deps,
@@ -257,12 +283,14 @@ static const struct pwrseq_target_data pwrseq_qcom_wcn_bt_target_data = {
 	.name = "bluetooth",
 	.unit = &pwrseq_qcom_wcn_bt_unit_data,
 	.post_enable = pwrseq_qcom_wcn_pwup_delay,
+	.is_controllable = pwrseq_qcom_wcn_bt_is_controllable,
 };
 
 static const struct pwrseq_target_data pwrseq_qcom_wcn_wlan_target_data = {
 	.name = "wlan",
 	.unit = &pwrseq_qcom_wcn_wlan_unit_data,
 	.post_enable = pwrseq_qcom_wcn_pwup_delay,
+	.is_controllable = pwrseq_qcom_wcn_wlan_is_controllable,
 };
 
 /* There are no separate BT and WLAN enablement pins */
@@ -280,12 +308,14 @@ static const struct pwrseq_target_data pwrseq_qcom_wcn6855_bt_target_data = {
 	.name = "bluetooth",
 	.unit = &pwrseq_qcom_wcn6855_bt_unit_data,
 	.post_enable = pwrseq_qcom_wcn6855_xo_clk_deassert,
+	.is_controllable = pwrseq_qcom_wcn_bt_is_controllable,
 };
 
 static const struct pwrseq_target_data pwrseq_qcom_wcn6855_wlan_target_data = {
 	.name = "wlan",
 	.unit = &pwrseq_qcom_wcn6855_wlan_unit_data,
 	.post_enable = pwrseq_qcom_wcn6855_xo_clk_deassert,
+	.is_controllable = pwrseq_qcom_wcn_wlan_is_controllable,
 };
 
 static const struct pwrseq_target_data *pwrseq_qcom_wcn_targets[] = {
diff --git a/drivers/power/sequencing/pwrseq-renesas-pwrrdy.c b/drivers/power/sequencing/pwrseq-renesas-pwrrdy.c
new file mode 100644
index 00000000000000..743855620501de
--- /dev/null
+++ b/drivers/power/sequencing/pwrseq-renesas-pwrrdy.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Renesas RZ/G3L Power Ready driver
+ *
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pwrseq/provider.h>
+#include <linux/regmap.h>
+
+#define SYS_PWRRDY_N		0xd70
+#define SYS_PWRRDY_N_USB_MASK	BIT(0)
+#define SYS_PWRRDY_N_DSI_MASK	BIT(1)
+#define SYS_PWRRDY_N_CSI_MASK	BIT(2)
+
+static int pwrseq_rzg3l_set_pwrrdy(struct pwrseq_device *pwrseq, u32 mask,
+				   bool val)
+{
+	struct regmap *regmap = pwrseq_device_get_drvdata(pwrseq);
+
+	return regmap_assign_bits(regmap, SYS_PWRRDY_N, mask, val);
+}
+
+static int pwrseq_rzg3l_usb_pwrrdy_enable(struct pwrseq_device *pwrseq)
+{
+	return pwrseq_rzg3l_set_pwrrdy(pwrseq, SYS_PWRRDY_N_USB_MASK, 0);
+}
+
+static int pwrseq_rzg3l_usb_pwrrdy_disable(struct pwrseq_device *pwrseq)
+{
+	return pwrseq_rzg3l_set_pwrrdy(pwrseq, SYS_PWRRDY_N_USB_MASK, 1);
+}
+
+static const struct pwrseq_unit_data pwrseq_rzg3l_usb_pwrrdy_unit = {
+	.name = "usb-pwrrdy-power-sequence",
+	.enable = pwrseq_rzg3l_usb_pwrrdy_enable,
+	.disable = pwrseq_rzg3l_usb_pwrrdy_disable,
+};
+
+static int pwrseq_rzg3l_dsi_pwrrdy_enable(struct pwrseq_device *pwrseq)
+{
+	return pwrseq_rzg3l_set_pwrrdy(pwrseq, SYS_PWRRDY_N_DSI_MASK, 0);
+}
+
+static int pwrseq_rzg3l_dsi_pwrrdy_disable(struct pwrseq_device *pwrseq)
+{
+	return pwrseq_rzg3l_set_pwrrdy(pwrseq, SYS_PWRRDY_N_DSI_MASK, 1);
+}
+
+static const struct pwrseq_unit_data pwrseq_rzg3l_dsi_pwrrdy_unit = {
+	.name = "dsi-pwrrdy-sequence",
+	.enable = pwrseq_rzg3l_dsi_pwrrdy_enable,
+	.disable = pwrseq_rzg3l_dsi_pwrrdy_disable,
+};
+
+static int pwrseq_rzg3l_csi_pwrrdy_enable(struct pwrseq_device *pwrseq)
+{
+	return pwrseq_rzg3l_set_pwrrdy(pwrseq, SYS_PWRRDY_N_CSI_MASK, 0);
+}
+
+static int pwrseq_rzg3l_csi_pwrrdy_disable(struct pwrseq_device *pwrseq)
+{
+	return pwrseq_rzg3l_set_pwrrdy(pwrseq, SYS_PWRRDY_N_CSI_MASK, 1);
+}
+
+static const struct pwrseq_unit_data pwrseq_rzg3l_csi_pwrrdy_unit = {
+	.name = "csi-pwrrdy-power-sequence",
+	.enable = pwrseq_rzg3l_csi_pwrrdy_enable,
+	.disable = pwrseq_rzg3l_csi_pwrrdy_disable,
+};
+
+static const struct pwrseq_target_data pwrseq_rzg3l_usb_pwrrdy_target = {
+	.name = "usb-pwrrdy",
+	.unit = &pwrseq_rzg3l_usb_pwrrdy_unit,
+};
+
+static const struct pwrseq_target_data pwrseq_rzg3l_dsi_pwrrdy_target = {
+	.name = "dsi-pwrrdy",
+	.unit = &pwrseq_rzg3l_dsi_pwrrdy_unit,
+};
+
+static const struct pwrseq_target_data pwrseq_rzg3l_csi_pwrrdy_target = {
+	.name = "csi-pwrrdy",
+	.unit = &pwrseq_rzg3l_csi_pwrrdy_unit,
+};
+
+static const struct pwrseq_target_data *pwrseq_rzg3l_pwrrdy_targets[] = {
+	&pwrseq_rzg3l_usb_pwrrdy_target,
+	&pwrseq_rzg3l_dsi_pwrrdy_target,
+	&pwrseq_rzg3l_csi_pwrrdy_target,
+	NULL
+};
+
+static const char * const pwrseq_rzg3l_pwrrdy_compats[] = {
+	"renesas,r9a08g046-usbphy-ctrl",
+	"renesas,r9a08g046-mipi-dsi",
+	NULL
+};
+
+static int pwrseq_rzg3l_pwrrdy_match(struct pwrseq_device *pwrseq,
+				     struct device *dev)
+{
+	if (of_device_compatible_match(dev->of_node, pwrseq_rzg3l_pwrrdy_compats))
+		return PWRSEQ_MATCH_OK;
+
+	return PWRSEQ_NO_MATCH;
+}
+
+static int pwrseq_rzg3l_pwrrdy_probe(struct auxiliary_device *adev,
+				     const struct auxiliary_device_id *id)
+{
+	struct pwrseq_config config = {
+		.parent = &adev->dev,
+		.owner = THIS_MODULE,
+		.drvdata = adev->dev.platform_data,
+		.match = pwrseq_rzg3l_pwrrdy_match,
+		.targets = pwrseq_rzg3l_pwrrdy_targets,
+	};
+
+	return PTR_ERR_OR_ZERO(devm_pwrseq_device_register(&adev->dev, &config));
+}
+
+static const struct auxiliary_device_id pwrseq_rzg3l_pwrrdy_id_table[] = {
+	{ .name = "rz_sysc.pwrseq-pwrrdy" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(auxiliary, pwrseq_rzg3l_pwrrdy_id_table);
+
+static struct auxiliary_driver pwrseq_rzg3l_pwrrdy_driver = {
+	.driver = {
+		.name = "pwrseq-rzg3l-pwrrdy",
+	},
+	.probe = pwrseq_rzg3l_pwrrdy_probe,
+	.id_table = pwrseq_rzg3l_pwrrdy_id_table,
+};
+module_auxiliary_driver(pwrseq_rzg3l_pwrrdy_driver);
+
+MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
+MODULE_DESCRIPTION("Renesas RZ/G3L Power Ready Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
index 4bea6fd83c36f2..c9b86d5a21a4a0 100644
--- a/drivers/power/supply/bq24190_charger.c
+++ b/drivers/power/supply/bq24190_charger.c
@@ -10,6 +10,7 @@
 #include <linux/delay.h>
 #include <linux/devm-helpers.h>
 #include <linux/pm_runtime.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/power_supply.h>
 #include <linux/power/bq24190_charger.h>
 #include <linux/regulator/driver.h>
@@ -1294,6 +1295,58 @@ static int bq24190_charger_set_iinlimit(struct bq24190_dev_info *bdi,
 			ARRAY_SIZE(bq24190_isc_iinlim_values), val->intval);
 }
 
+static int bq24190_charger_get_batfet_ctrl(struct bq24190_dev_info *bdi,
+					   union power_supply_propval *val)
+{
+	u8 regval;
+	int ret;
+
+	ret = bq24190_read_mask(bdi, BQ24190_REG_MOC,
+				BQ24190_REG_MOC_BATFET_DISABLE_MASK,
+				BQ24190_REG_MOC_BATFET_DISABLE_SHIFT,
+				&regval);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Datasheet defines shipping mode as BATFET off _and_ the watchdog
+	 * disabled. Since the watchdog is disabled during the whole lifetime
+	 * (from probe), shipping mode is the only "off" state that can be
+	 * achievable.
+	 */
+	val->intval = regval ? POWER_SUPPLY_LOAD_SWITCH_SHIP :
+			       POWER_SUPPLY_LOAD_SWITCH_ON;
+	return 0;
+}
+
+static int
+bq24190_charger_set_batfet_ctrl(struct bq24190_dev_info *bdi,
+				const union power_supply_propval *val)
+{
+	u8 regval;
+
+	/*
+	 * Datasheet defines shipping mode as BATFET off _and_ the watchdog
+	 * disabled. Since the watchdog is disabled during the whole lifetime
+	 * (from probe), shipping mode is the only "off" state that can be
+	 * achievable.
+	 */
+	switch (val->intval) {
+	case POWER_SUPPLY_LOAD_SWITCH_ON:
+		regval = 0;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_SHIP:
+		regval = 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return bq24190_write_mask(bdi, BQ24190_REG_MOC,
+				  BQ24190_REG_MOC_BATFET_DISABLE_MASK,
+				  BQ24190_REG_MOC_BATFET_DISABLE_SHIFT, regval);
+}
+
 static int bq24190_charger_get_property(struct power_supply *psy,
 		enum power_supply_property psp, union power_supply_propval *val)
 {
@@ -1350,6 +1403,9 @@ static int bq24190_charger_get_property(struct power_supply *psy,
 		val->intval = POWER_SUPPLY_SCOPE_SYSTEM;
 		ret = 0;
 		break;
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		ret = bq24190_charger_get_batfet_ctrl(bdi, val);
+		break;
 	case POWER_SUPPLY_PROP_MODEL_NAME:
 		val->strval = bdi->model_name;
 		ret = 0;
@@ -1400,6 +1456,9 @@ static int bq24190_charger_set_property(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
 		ret = bq24190_charger_set_iinlimit(bdi, val);
 		break;
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		ret = bq24190_charger_set_batfet_ctrl(bdi, val);
+		break;
 	default:
 		ret = -EINVAL;
 	}
@@ -1420,6 +1479,7 @@ static int bq24190_charger_property_is_writeable(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT:
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE:
 	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
 		return 1;
 	default:
 		return 0;
@@ -1480,6 +1540,7 @@ static enum power_supply_property bq24190_charger_properties[] = {
 	POWER_SUPPLY_PROP_SCOPE,
 	POWER_SUPPLY_PROP_MODEL_NAME,
 	POWER_SUPPLY_PROP_MANUFACTURER,
+	POWER_SUPPLY_PROP_LOAD_SWITCH,
 };
 
 static char *bq24190_charger_supplied_to[] = {
@@ -1498,6 +1559,8 @@ static const struct power_supply_desc bq24190_charger_desc = {
 	.charge_types		= BIT(POWER_SUPPLY_CHARGE_TYPE_NONE)    |
 				  BIT(POWER_SUPPLY_CHARGE_TYPE_TRICKLE) |
 				  BIT(POWER_SUPPLY_CHARGE_TYPE_FAST),
+	.load_switches		= BIT(POWER_SUPPLY_LOAD_SWITCH_ON) |
+				  BIT(POWER_SUPPLY_LOAD_SWITCH_SHIP),
 };
 
 /* Battery power supply property routines */
@@ -2168,7 +2231,13 @@ static int bq24190_probe(struct i2c_client *client)
 	if (ret < 0)
 		goto out_charger;
 
-	enable_irq_wake(client->irq);
+	ret = devm_device_init_wakeup(dev);
+	if (ret < 0)
+		goto out_charger;
+
+	ret = devm_pm_set_wake_irq(dev, client->irq);
+	if (ret < 0)
+		goto out_charger;
 
 	pm_runtime_put_autosuspend(dev);
 
diff --git a/drivers/power/supply/bq24257_charger.c b/drivers/power/supply/bq24257_charger.c
index 944a7aa6f187d5..8eea7d0395ce02 100644
--- a/drivers/power/supply/bq24257_charger.c
+++ b/drivers/power/supply/bq24257_charger.c
@@ -295,6 +295,40 @@ static int bq24257_set_input_current_limit(struct bq24257_device *bq,
 						    BQ24257_IILIMIT_MAP_SIZE));
 }
 
+static int bq24257_get_sys_switch(struct bq24257_device *bq,
+				  union power_supply_propval *val)
+{
+	int ret;
+
+	ret = bq24257_field_read(bq, F_SYSOFF);
+	if (ret < 0)
+		return ret;
+
+	val->intval = ret ? POWER_SUPPLY_LOAD_SWITCH_OFF :
+			    POWER_SUPPLY_LOAD_SWITCH_ON;
+
+	return 0;
+}
+
+static int bq24257_set_sys_switch(struct bq24257_device *bq,
+				  const union power_supply_propval *val)
+{
+	u8 regval;
+
+	switch (val->intval) {
+	case POWER_SUPPLY_LOAD_SWITCH_ON:
+		regval = 0;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_OFF:
+		regval = 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return bq24257_field_write(bq, F_SYSOFF, regval);
+}
+
 static int bq24257_power_supply_get_property(struct power_supply *psy,
 					     enum power_supply_property psp,
 					     union power_supply_propval *val)
@@ -381,7 +415,8 @@ static int bq24257_power_supply_get_property(struct power_supply *psy,
 
 	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
 		return bq24257_get_input_current_limit(bq, val);
-
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return bq24257_get_sys_switch(bq, val);
 	default:
 		return -EINVAL;
 	}
@@ -398,6 +433,8 @@ static int bq24257_power_supply_set_property(struct power_supply *psy,
 	switch (prop) {
 	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
 		return bq24257_set_input_current_limit(bq, val);
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return bq24257_set_sys_switch(bq, val);
 	default:
 		return -EINVAL;
 	}
@@ -408,6 +445,7 @@ static int bq24257_power_supply_property_is_writeable(struct power_supply *psy,
 {
 	switch (psp) {
 	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
 		return true;
 	default:
 		return false;
@@ -740,6 +778,7 @@ static enum power_supply_property bq24257_power_supply_props[] = {
 	POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX,
 	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
 	POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,
+	POWER_SUPPLY_PROP_LOAD_SWITCH,
 };
 
 static char *bq24257_charger_supplied_to[] = {
@@ -749,6 +788,8 @@ static char *bq24257_charger_supplied_to[] = {
 static const struct power_supply_desc bq24257_power_supply_desc = {
 	.name = "bq24257-charger",
 	.type = POWER_SUPPLY_TYPE_USB,
+	.load_switches = BIT(POWER_SUPPLY_LOAD_SWITCH_ON) |
+			 BIT(POWER_SUPPLY_LOAD_SWITCH_OFF),
 	.properties = bq24257_power_supply_props,
 	.num_properties = ARRAY_SIZE(bq24257_power_supply_props),
 	.get_property = bq24257_power_supply_get_property,
diff --git a/drivers/power/supply/bq2515x_charger.c b/drivers/power/supply/bq2515x_charger.c
index 0208358ebbe4c7..c2b17562cf8bb0 100644
--- a/drivers/power/supply/bq2515x_charger.c
+++ b/drivers/power/supply/bq2515x_charger.c
@@ -420,7 +420,7 @@ static int bq2515x_get_battery_current_now(struct bq2515x_device *bq2515x)
 							BQ2515X_ICHG_DIVISOR);
 }
 
-static bool bq2515x_get_charge_disable(struct bq2515x_device *bq2515x)
+static int bq2515x_get_charge_disable(struct bq2515x_device *bq2515x)
 {
 	int ret;
 	int ce_pin;
@@ -428,6 +428,8 @@ static bool bq2515x_get_charge_disable(struct bq2515x_device *bq2515x)
 	int charger_disable;
 
 	ce_pin = gpiod_get_value_cansleep(bq2515x->ce_gpio);
+	if (ce_pin < 0)
+		return ce_pin;
 
 	ret = regmap_read(bq2515x->regmap, BQ2515X_ICCTRL2, &icctrl2);
 	if (ret)
@@ -435,10 +437,7 @@ static bool bq2515x_get_charge_disable(struct bq2515x_device *bq2515x)
 
 	charger_disable = icctrl2 & BQ2515X_CHARGER_DISABLE;
 
-	if (charger_disable || ce_pin)
-		return true;
-
-	return false;
+	return charger_disable || ce_pin;
 }
 
 static int bq2515x_set_charge_disable(struct bq2515x_device *bq2515x, int val)
@@ -615,7 +614,11 @@ static int bq2515x_charging_status(struct bq2515x_device *bq2515x,
 	else
 		status1_no_fault = false;
 
-	ce_status = (!bq2515x_get_charge_disable(bq2515x));
+	ret = bq2515x_get_charge_disable(bq2515x);
+	if (ret < 0)
+		return ret;
+
+	ce_status = !ret;
 
 	/*
 	 * If there are no faults and charging is enabled, then status is
diff --git a/drivers/power/supply/bq25630_charger.c b/drivers/power/supply/bq25630_charger.c
index 200f74f8eab618..3106da710d6b68 100644
--- a/drivers/power/supply/bq25630_charger.c
+++ b/drivers/power/supply/bq25630_charger.c
@@ -8,6 +8,7 @@
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
 
@@ -112,6 +113,12 @@
 #define BQ25630_ITERM_MAX 1000000
 #define BQ25630_ITERM_STEP 10000
 
+/* BATFET control modes. */
+#define BQ25630_BATFET_CTRL_IDLE	0x00
+#define BQ25630_BATFET_CTRL_SHUTDOWN	0x01
+#define BQ25630_BATFET_CTRL_SHIP	0x02
+#define BQ25630_BATFET_CTRL_STANDBY	0x03
+
 /* Charge types. */
 #define BQ25630_CHG_STAT_NOT_CHARGING	0x00
 #define BQ25630_CHG_STAT_TRICKLE_CHARGE 0x01
@@ -587,6 +594,71 @@ static int bq25630_read_vbus(struct bq25630_data *data, int *val)
 	return 0;
 }
 
+static int bq25630_read_batfet_ctrl(struct bq25630_data *data, int *val)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_field_read(data->regfields[BQ25630_REGF_BATFET_CTRL],
+				&regval);
+	if (ret) {
+		dev_err(data->dev, "Could not read BATFET control (%d)\n", ret);
+		return ret;
+	}
+
+	switch (regval) {
+	case BQ25630_BATFET_CTRL_IDLE:
+		*val = POWER_SUPPLY_LOAD_SWITCH_ON;
+		break;
+	case BQ25630_BATFET_CTRL_SHUTDOWN:
+		*val = POWER_SUPPLY_LOAD_SWITCH_OFF;
+		break;
+	case BQ25630_BATFET_CTRL_SHIP:
+		*val = POWER_SUPPLY_LOAD_SWITCH_SHIP;
+		break;
+	case BQ25630_BATFET_CTRL_STANDBY:
+		*val = POWER_SUPPLY_LOAD_SWITCH_STANDBY;
+		break;
+	default:
+		*val = POWER_SUPPLY_LOAD_SWITCH_UNKNOWN;
+	}
+
+	return 0;
+}
+
+static int bq25630_write_batfet_ctrl(struct bq25630_data *data, int val)
+{
+	unsigned int regval;
+	int ret;
+
+	switch (val) {
+	case POWER_SUPPLY_LOAD_SWITCH_ON:
+		regval = BQ25630_BATFET_CTRL_IDLE;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_OFF:
+		regval = BQ25630_BATFET_CTRL_SHUTDOWN;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_SHIP:
+		regval = BQ25630_BATFET_CTRL_SHIP;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_STANDBY:
+		regval = BQ25630_BATFET_CTRL_STANDBY;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = regmap_field_write(data->regfields[BQ25630_REGF_BATFET_CTRL],
+				 regval);
+	if (ret) {
+		dev_err(data->dev, "Could not write BATFET control (%d)\n",
+			ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 static int bq25630_get_status(struct bq25630_data *data, int *val)
 {
 	unsigned int regval;
@@ -845,6 +917,9 @@ static int bq25630_charger_get_property(struct power_supply *psy,
 					 BQ25630_ITERM_MIN_REGVAL,
 					 &val->intval);
 		break;
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		ret = bq25630_read_batfet_ctrl(data, &val->intval);
+		break;
 	case POWER_SUPPLY_PROP_MODEL_NAME:
 		val->strval = "BQ25630";
 		break;
@@ -913,6 +988,9 @@ static int bq25630_charger_set_property(struct power_supply *psy,
 					  BQ25630_ITERM_MIN_REGVAL,
 					  val->intval);
 		break;
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		ret = bq25630_write_batfet_ctrl(data, val->intval);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -932,6 +1010,7 @@ static int bq25630_charger_property_is_writeable(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT:
 	case POWER_SUPPLY_PROP_PRECHARGE_CURRENT:
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
 		return true;
 	default:
 		return false;
@@ -954,6 +1033,7 @@ static const enum power_supply_property bq25630_charger_properties[] = {
 	POWER_SUPPLY_PROP_USB_TYPE,
 	POWER_SUPPLY_PROP_PRECHARGE_CURRENT,
 	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
+	POWER_SUPPLY_PROP_LOAD_SWITCH,
 	POWER_SUPPLY_PROP_MODEL_NAME,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
@@ -972,6 +1052,11 @@ static const struct power_supply_desc bq25630_charger_psy_desc = {
 		     BIT(POWER_SUPPLY_USB_TYPE_DCP) |
 		     BIT(POWER_SUPPLY_USB_TYPE_CDP) |
 		     BIT(POWER_SUPPLY_USB_TYPE_C),
+	.load_switches = BIT(POWER_SUPPLY_LOAD_SWITCH_UNKNOWN) |
+			 BIT(POWER_SUPPLY_LOAD_SWITCH_ON) |
+			 BIT(POWER_SUPPLY_LOAD_SWITCH_OFF) |
+			 BIT(POWER_SUPPLY_LOAD_SWITCH_STANDBY) |
+			 BIT(POWER_SUPPLY_LOAD_SWITCH_SHIP),
 	.properties = bq25630_charger_properties,
 	.num_properties = ARRAY_SIZE(bq25630_charger_properties),
 	.get_property = bq25630_charger_get_property,
diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c
index 741a45cc5f35dd..f7510ada6b7342 100644
--- a/drivers/power/supply/bq25890_charger.c
+++ b/drivers/power/supply/bq25890_charger.c
@@ -1590,7 +1590,7 @@ static int bq25890_suspend(struct device *dev)
 	struct bq25890_device *bq = dev_get_drvdata(dev);
 
 	/*
-	 * If charger is removed, while in suspend, make sure ADC is diabled
+	 * If charger is removed, while in suspend, make sure ADC is disabled
 	 * since it consumes slightly more power.
 	 */
 	return bq25890_field_write(bq, F_CONV_RATE, 0);
diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c
index 01c9ec1116fa09..17f5b58609bebf 100644
--- a/drivers/power/supply/bq27xxx_battery.c
+++ b/drivers/power/supply/bq27xxx_battery.c
@@ -37,12 +37,14 @@
  * https://www.ti.com/product/bq27441-g1
  * https://www.ti.com/product/bq27621-g1
  * https://www.ti.com/product/bq27z561
+ * https://www.ti.com/product/bq27z746
  * https://www.ti.com/product/bq28z610
  * https://www.ti.com/product/bq34z100-g1
  * https://www.ti.com/product/bq78z100
  */
 
 #include <linux/device.h>
+#include <linux/devm-helpers.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/param.h>
@@ -501,6 +503,7 @@ static u8
 		[BQ27XXX_REG_AP] = 0x22,
 		BQ27XXX_DM_REG_ROWS,
 	},
+#define bq27z746_regs bq27z561_regs
 	bq28z610_regs[BQ27XXX_REG_MAX] = {
 		[BQ27XXX_REG_CTRL] = 0x00,
 		[BQ27XXX_REG_TEMP] = 0x06,
@@ -836,6 +839,8 @@ static enum power_supply_property bq27z561_props[] = {
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
 
+#define bq27z746_props bq27z561_props
+
 static enum power_supply_property bq28z610_props[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_PRESENT,
@@ -994,6 +999,7 @@ static struct bq27xxx_dm_reg bq27621_dm_regs[] = {
 #endif
 
 #define bq27z561_dm_regs NULL
+#define bq27z746_dm_regs NULL
 #define bq28z610_dm_regs NULL
 #define bq34z100_dm_regs NULL
 #define bq78z100_dm_regs NULL
@@ -1052,6 +1058,7 @@ static struct {
 	[BQ27441]   = BQ27XXX_DATA(bq27441,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
 	[BQ27621]   = BQ27XXX_DATA(bq27621,   0x80008000, BQ27XXX_O_UTOT | BQ27XXX_O_CFGUP | BQ27XXX_O_RAM),
 	[BQ27Z561]  = BQ27XXX_DATA(bq27z561,  0         , BQ27Z561_O_BITS),
+	[BQ27Z746]  = BQ27XXX_DATA(bq27z746,  0         , BQ27Z561_O_BITS),
 	[BQ28Z610]  = BQ27XXX_DATA(bq28z610,  0         , BQ27Z561_O_BITS),
 	[BQ34Z100]  = BQ27XXX_DATA(bq34z100,  0         , BQ27XXX_O_OTDC | BQ27XXX_O_SOC_SI | \
 							  BQ27XXX_O_HAS_CI | BQ27XXX_O_MUL_CHEM),
@@ -2234,7 +2241,6 @@ int bq27xxx_battery_setup(struct bq27xxx_device_info *di)
 	};
 	int ret;
 
-	INIT_DELAYED_WORK(&di->work, bq27xxx_battery_poll);
 	ret = devm_mutex_init(di->dev, &di->lock);
 	if (ret)
 		return ret;
@@ -2255,6 +2261,12 @@ int bq27xxx_battery_setup(struct bq27xxx_device_info *di)
 	psy_desc->get_property = bq27xxx_battery_get_property;
 	psy_desc->external_power_changed = bq27xxx_external_power_changed;
 
+	/* Cancel the poll work after the power_supply is unregistered. */
+	ret = devm_delayed_work_autocancel(di->dev, &di->work,
+					   bq27xxx_battery_poll);
+	if (ret)
+		return ret;
+
 	di->bat = devm_power_supply_register(di->dev, psy_desc, &psy_cfg);
 	if (IS_ERR(di->bat))
 		return dev_err_probe(di->dev, PTR_ERR(di->bat),
@@ -2281,8 +2293,6 @@ void bq27xxx_battery_teardown(struct bq27xxx_device_info *di)
 	mutex_lock(&di->lock);
 	di->removed = true;
 	mutex_unlock(&di->lock);
-
-	cancel_delayed_work_sync(&di->work);
 }
 EXPORT_SYMBOL_GPL(bq27xxx_battery_teardown);
 
@@ -2291,7 +2301,7 @@ static int bq27xxx_battery_suspend(struct device *dev)
 {
 	struct bq27xxx_device_info *di = dev_get_drvdata(dev);
 
-	cancel_delayed_work(&di->work);
+	cancel_delayed_work_sync(&di->work);
 	return 0;
 }
 
diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
index c4e7a9521d0869..2a6b4534cb28a5 100644
--- a/drivers/power/supply/bq27xxx_battery_i2c.c
+++ b/drivers/power/supply/bq27xxx_battery_i2c.c
@@ -253,6 +253,7 @@ static const struct i2c_device_id bq27xxx_i2c_id_table[] = {
 	{ .name = "bq27441", .driver_data = BQ27441 },
 	{ .name = "bq27621", .driver_data = BQ27621 },
 	{ .name = "bq27z561", .driver_data = BQ27Z561 },
+	{ .name = "bq27z746", .driver_data = BQ27Z746 },
 	{ .name = "bq28z610", .driver_data = BQ28Z610 },
 	{ .name = "bq34z100", .driver_data = BQ34Z100 },
 	{ .name = "bq78z100", .driver_data = BQ78Z100 },
@@ -290,6 +291,7 @@ static const struct of_device_id bq27xxx_battery_i2c_of_match_table[] = {
 	{ .compatible = "ti,bq27441" },
 	{ .compatible = "ti,bq27621" },
 	{ .compatible = "ti,bq27z561" },
+	{ .compatible = "ti,bq27z746" },
 	{ .compatible = "ti,bq28z610" },
 	{ .compatible = "ti,bq34z100" },
 	{ .compatible = "ti,bq78z100" },
diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c
index fd4cd9cd2ce2aa..8e8c5109401a4b 100644
--- a/drivers/power/supply/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
@@ -904,7 +904,7 @@ static bool cm_setup_timer(void)
 }
 
 /**
- * charger_extcon_work - enable/diable charger according to the state
+ * charger_extcon_work - enable/disable charger according to the state
  *			of charger cable
  *
  * @work: work_struct of the function charger_extcon_work.
diff --git a/drivers/power/supply/cros_charge-control.c b/drivers/power/supply/cros_charge-control.c
index e0f168624807e2..07bcc6a9aeefee 100644
--- a/drivers/power/supply/cros_charge-control.c
+++ b/drivers/power/supply/cros_charge-control.c
@@ -21,12 +21,22 @@
 
 /*
  * Semantics of data *returned* from the EC API and Linux sysfs differ
- * slightly, also the v1 API can not return any data.
- * To match the expected sysfs API, data is never read back from the EC but
- * cached in the driver.
+ * slightly:
+ *  - EC sustainer off is lower=upper=-1
+ *  - Linux "no limit" is start=0, end=100
+ * Also the v1 API can not return any data.
  *
- * Changes to the EC bypassing the driver will not be reflected in sysfs.
- * Any change to "charge_behaviour" will synchronize the EC with the driver state.
+ * While the sustainer is active the EC may report IDLE or DISCHARGE as the
+ * current charge-control mode; that is an internal hold/discharge step, not a
+ * Linux inhibit-charge / force-discharge request. Valid sustainer limits are
+ * therefore adopted as AUTO with the reported thresholds.
+ *
+ * Sysfs reads come from a driver-side cache. On probe, command versions that
+ * support GET (v2+) are initialized from the EC so firmware or firmware-setup
+ * programmed limits and modes are preserved (unlike earlier behaviour that
+ * always forced AUTO with no limits). v1 still forces a well-known EC state.
+ * Subsequent sysfs writes keep the cache and EC in sync; changes that bypass
+ * the driver are not reflected until the next probe.
  */
 
 struct cros_chctl_priv {
@@ -44,18 +54,23 @@ struct cros_chctl_priv {
 };
 
 static int cros_chctl_send_charge_control_cmd(struct cros_ec_device *cros_ec,
-					      u8 cmd_version, struct ec_params_charge_control *req)
+					      u8 cmd_version,
+					      struct ec_params_charge_control *req,
+					      struct ec_response_charge_control *resp)
 {
-	int ret;
 	static const u8 outsizes[] = {
 		[1] = offsetof(struct ec_params_charge_control, cmd),
 		[2] = sizeof(struct ec_params_charge_control),
 		[3] = sizeof(struct ec_params_charge_control),
 	};
+	size_t insize = resp ? sizeof(*resp) : 0;
+	int ret;
 
-	ret = cros_ec_cmd(cros_ec, cmd_version, EC_CMD_CHARGE_CONTROL, req,
-			  outsizes[cmd_version], NULL, 0);
+	if (resp)
+		*resp = (struct ec_response_charge_control){};
 
+	ret = cros_ec_cmd(cros_ec, cmd_version, EC_CMD_CHARGE_CONTROL, req,
+			  outsizes[cmd_version], resp, insize);
 	if (ret < 0)
 		return ret;
 
@@ -94,7 +109,120 @@ static int cros_chctl_configure_ec(struct cros_chctl_priv *priv)
 		req.sustain_soc.upper = -1;
 	}
 
-	return cros_chctl_send_charge_control_cmd(priv->cros_ec, priv->cmd_version, &req);
+	return cros_chctl_send_charge_control_cmd(priv->cros_ec, priv->cmd_version,
+						  &req, NULL);
+}
+
+static int cros_chctl_get_ec_status(struct cros_chctl_priv *priv,
+				    struct ec_response_charge_control *resp)
+{
+	struct ec_params_charge_control req = {
+		.cmd = EC_CHARGE_CONTROL_CMD_GET,
+	};
+
+	return cros_chctl_send_charge_control_cmd(priv->cros_ec, priv->cmd_version,
+						  &req, resp);
+}
+
+static void cros_chctl_set_default_state(struct cros_chctl_priv *priv)
+{
+	lockdep_assert_held(&priv->lock);
+
+	priv->current_behaviour = POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO;
+	priv->current_start_threshold = 0;
+	priv->current_end_threshold = 100;
+}
+
+static bool cros_chctl_sustainer_limits_valid(s8 lower, s8 upper)
+{
+	return lower >= 0 && upper >= 0 && lower <= 100 && upper <= 100 && lower <= upper;
+}
+
+static int cros_chctl_adopt_ec_mode(struct cros_chctl_priv *priv, u32 mode)
+{
+	lockdep_assert_held(&priv->lock);
+
+	switch (mode) {
+	case CHARGE_CONTROL_NORMAL:
+		priv->current_behaviour = POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO;
+		return 0;
+	case CHARGE_CONTROL_IDLE:
+		priv->current_behaviour = POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE;
+		return 0;
+	case CHARGE_CONTROL_DISCHARGE:
+		priv->current_behaviour = POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE;
+		return 0;
+	default:
+		dev_warn(priv->dev, "unknown charge control mode %u\n", mode);
+		return -EINVAL;
+	}
+}
+
+static int cros_chctl_init_state(struct cros_chctl_priv *priv)
+{
+	struct ec_response_charge_control resp;
+	s8 lower, upper;
+	int ret;
+
+	guard(mutex)(&priv->lock);
+
+	/* v1 cannot report current state; force a well-known EC configuration. */
+	if (priv->cmd_version < 2) {
+		cros_chctl_set_default_state(priv);
+		return cros_chctl_configure_ec(priv);
+	}
+
+	ret = cros_chctl_get_ec_status(priv, &resp);
+	if (ret < 0) {
+		dev_warn(priv->dev,
+			 "failed to read EC charge state (%pe), applying defaults\n",
+			 ERR_PTR(ret));
+		goto defaults;
+	}
+
+	lower = resp.sustain_soc.lower;
+	upper = resp.sustain_soc.upper;
+
+	/*
+	 * Valid sustainer limits mean "auto with thresholds". The EC mode may
+	 * be IDLE/DISCHARGE while the sustainer holds or bleeds SoC; do not
+	 * expose that as inhibit-charge / force-discharge.
+	 */
+	if (cros_chctl_sustainer_limits_valid(lower, upper)) {
+		priv->current_behaviour = POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO;
+		priv->current_start_threshold = lower;
+		priv->current_end_threshold = upper;
+	} else {
+		ret = cros_chctl_adopt_ec_mode(priv, resp.mode);
+		if (ret < 0)
+			goto defaults;
+
+		/*
+		 * Sustainer off is lower=upper=-1 → Linux "no limit" (0/100).
+		 * Any other non-valid pair is unexpected; remap, warn, and
+		 * push the cleaned state back to the EC.
+		 */
+		priv->current_start_threshold = 0;
+		priv->current_end_threshold = 100;
+
+		if (!(lower == -1 && upper == -1)) {
+			dev_warn(priv->dev,
+				 "invalid EC sustainer limits (%d/%d), treating as no limit\n",
+				 lower, upper);
+			return cros_chctl_configure_ec(priv);
+		}
+	}
+
+	dev_dbg(priv->dev,
+		"adopted EC charge state: behaviour=%d start=%u end=%u (ec mode=%u)\n",
+		priv->current_behaviour, priv->current_start_threshold,
+		priv->current_end_threshold, resp.mode);
+
+	return 0;
+
+defaults:
+	cros_chctl_set_default_state(priv);
+	return cros_chctl_configure_ec(priv);
 }
 
 static int cros_chctl_psy_ext_get_prop(struct power_supply *psy,
@@ -152,7 +280,6 @@ static int cros_chctl_psy_ext_set_threshold(struct cros_chctl_priv *priv,
 	return 0;
 }
 
-
 static int cros_chctl_psy_ext_set_prop(struct power_supply *psy,
 				       const struct power_supply_ext *ext,
 				       void *data,
@@ -305,13 +432,7 @@ static int cros_chctl_probe(struct platform_device *pdev)
 	priv->battery_hook.add_battery = cros_chctl_add_battery;
 	priv->battery_hook.remove_battery = cros_chctl_remove_battery;
 
-	priv->current_behaviour = POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO;
-	priv->current_start_threshold = 0;
-	priv->current_end_threshold = 100;
-
-	/* Bring EC into well-known state */
-	scoped_guard(mutex, &priv->lock)
-		ret = cros_chctl_configure_ec(priv);
+	ret = cros_chctl_init_state(priv);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/power/supply/da9030_battery.c b/drivers/power/supply/da9030_battery.c
index d25279c260308b..8c4daec0a6b67e 100644
--- a/drivers/power/supply/da9030_battery.c
+++ b/drivers/power/supply/da9030_battery.c
@@ -263,7 +263,7 @@ static void da9030_charger_check_state(struct da9030_charger *charger)
 		} else if (charger->adc.vbat_res >
 			   charger->thresholds.vbat_low) {
 			/* we are charging and passed LOW_THRESH,
-			   so upate DA9030 VBAT threshold
+			   so update DA9030 VBAT threshold
 			 */
 			da903x_write(charger->master, DA9030_VBATMON,
 				     charger->thresholds.vbat_low);
diff --git a/drivers/power/supply/ds2780_battery.c b/drivers/power/supply/ds2780_battery.c
index 5b57bbba79d449..ef2446386c2af6 100644
--- a/drivers/power/supply/ds2780_battery.c
+++ b/drivers/power/supply/ds2780_battery.c
@@ -304,7 +304,7 @@ static int ds2780_get_capacity(struct ds2780_device_info *dev_info,
 		return ret;
 
 	*capacity = raw;
-	return raw;
+	return 0;
 }
 
 static int ds2780_get_status(struct ds2780_device_info *dev_info, int *status)
diff --git a/drivers/power/supply/ltc4162-l-charger.c b/drivers/power/supply/ltc4162-l-charger.c
index 5c09e0368e6f02..01b0511da9aeec 100644
--- a/drivers/power/supply/ltc4162-l-charger.c
+++ b/drivers/power/supply/ltc4162-l-charger.c
@@ -755,6 +755,41 @@ static int ltc4162l_set_term_current(struct ltc4162l_info *info,
 				  BIT(2), BIT(2));
 }
 
+static int ltc4162l_get_ship_mode(struct ltc4162l_info *info,
+				  union power_supply_propval *val)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_read(info->regmap, LTC4162L_ARM_SHIP_MODE, &regval);
+	if (ret < 0)
+		return ret;
+
+	val->intval = regval == LTC4162L_ARM_SHIP_MODE_MAGIC ?
+			      POWER_SUPPLY_LOAD_SWITCH_SHIP :
+			      POWER_SUPPLY_LOAD_SWITCH_ON;
+
+	return 0;
+}
+
+static int ltc4162l_set_ship_mode(struct ltc4162l_info *info, int value)
+{
+	unsigned int regval;
+
+	switch (value) {
+	case POWER_SUPPLY_LOAD_SWITCH_ON:
+		regval = 0;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_SHIP:
+		regval = LTC4162L_ARM_SHIP_MODE_MAGIC;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_write(info->regmap, LTC4162L_ARM_SHIP_MODE, regval);
+}
+
 /* Custom properties */
 static const char * const ltc4162l_charge_status_name[] = {
 	"ilim_reg_active", /* 32 */
@@ -890,15 +925,15 @@ static ssize_t arm_ship_mode_show(struct device *dev,
 {
 	struct power_supply *psy = to_power_supply(dev);
 	struct ltc4162l_info *info = power_supply_get_drvdata(psy);
-	unsigned int regval;
+	union power_supply_propval val;
 	int ret;
 
-	ret = regmap_read(info->regmap, LTC4162L_ARM_SHIP_MODE, &regval);
-	if (ret)
+	ret = ltc4162l_get_ship_mode(info, &val);
+	if (ret < 0)
 		return ret;
 
 	return sysfs_emit(buf, "%u\n",
-		regval == LTC4162L_ARM_SHIP_MODE_MAGIC ? 1 : 0);
+			  val.intval == POWER_SUPPLY_LOAD_SWITCH_SHIP ? 1 : 0);
 }
 
 static ssize_t arm_ship_mode_store(struct device *dev,
@@ -915,8 +950,9 @@ static ssize_t arm_ship_mode_store(struct device *dev,
 	if (ret < 0)
 		return ret;
 
-	ret = regmap_write(info->regmap, LTC4162L_ARM_SHIP_MODE,
-				value ? LTC4162L_ARM_SHIP_MODE_MAGIC : 0);
+	ret = ltc4162l_set_ship_mode(info,
+				     value ? POWER_SUPPLY_LOAD_SWITCH_SHIP :
+					     POWER_SUPPLY_LOAD_SWITCH_ON);
 	if (ret < 0)
 		return ret;
 
@@ -981,6 +1017,8 @@ static int ltc4162l_get_property(struct power_supply *psy,
 		return chip_info->get_die_temp(info, val);
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
 		return ltc4162l_get_term_current(info, val);
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return ltc4162l_get_ship_mode(info, val);
 	default:
 		return -EINVAL;
 	}
@@ -1003,6 +1041,8 @@ static int ltc4162l_set_property(struct power_supply *psy,
 		return ltc4162l_set_iin_limit(info, val->intval);
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
 		return ltc4162l_set_term_current(info, val->intval);
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return ltc4162l_set_ship_mode(info, val->intval);
 	default:
 		return -EINVAL;
 	}
@@ -1016,6 +1056,7 @@ static int ltc4162l_property_is_writeable(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX:
 	case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT:
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
 		return 1;
 	default:
 		return 0;
@@ -1037,10 +1078,13 @@ static enum power_supply_property ltc4162l_properties[] = {
 	POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT,
 	POWER_SUPPLY_PROP_TEMP,
 	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
+	POWER_SUPPLY_PROP_LOAD_SWITCH,
 };
 
 static const struct power_supply_desc ltc4162l_desc = {
 	.type		= POWER_SUPPLY_TYPE_MAINS,
+	.load_switches	= BIT(POWER_SUPPLY_LOAD_SWITCH_ON) |
+			  BIT(POWER_SUPPLY_LOAD_SWITCH_SHIP),
 	.properties	= ltc4162l_properties,
 	.num_properties	= ARRAY_SIZE(ltc4162l_properties),
 	.get_property	= ltc4162l_get_property,
diff --git a/drivers/power/supply/max17042_battery.c b/drivers/power/supply/max17042_battery.c
index cbf3ac9c3e145e..7b0b0c88c44391 100644
--- a/drivers/power/supply/max17042_battery.c
+++ b/drivers/power/supply/max17042_battery.c
@@ -1080,7 +1080,7 @@ static int max17042_init_chip(struct max17042_chip *chip)
 		/* update capacity params */
 		max17042_update_capacity_regs(chip);
 
-		/* delay must be atleast 350mS to allow VFSOC
+		/* delay must be at least 350mS to allow VFSOC
 		 * to be calculated from the new configuration
 		 */
 		msleep(350);
@@ -1233,7 +1233,7 @@ static int max17042_parse_dt(struct max17042_chip *chip)
 	if (of_property_read_u32(np, "maxim,over-volt", &prop))
 		chip->vmax = INT_MAX;
 	else
-		chip->vmin = prop;
+		chip->vmax = prop;
 
 	return 0;
 }
diff --git a/drivers/power/supply/mm8013.c b/drivers/power/supply/mm8013.c
index 819667a27cad47..c38f48338d42ad 100644
--- a/drivers/power/supply/mm8013.c
+++ b/drivers/power/supply/mm8013.c
@@ -293,6 +293,7 @@ static const struct of_device_id mm8013_match_table[] = {
 	{ .compatible = "mitsumi,mm8013" },
 	{}
 };
+MODULE_DEVICE_TABLE(of, mm8013_match_table);
 
 static struct i2c_driver mm8013_i2c_driver = {
 	.probe = mm8013_probe,
diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c
index 9d6b24856c8b0f..b0f524d152cf6d 100644
--- a/drivers/power/supply/power_supply_sysfs.c
+++ b/drivers/power/supply/power_supply_sysfs.c
@@ -152,6 +152,14 @@ static const char * const POWER_SUPPLY_CHARGE_BEHAVIOUR_TEXT[] = {
 	[POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE]		= "force-discharge",
 };
 
+static const char *const POWER_SUPPLY_LOAD_SWITCH_TEXT[] = {
+	[POWER_SUPPLY_LOAD_SWITCH_UNKNOWN] = "Unknown",
+	[POWER_SUPPLY_LOAD_SWITCH_ON]	   = "On",
+	[POWER_SUPPLY_LOAD_SWITCH_OFF]	   = "Off",
+	[POWER_SUPPLY_LOAD_SWITCH_STANDBY] = "Standby",
+	[POWER_SUPPLY_LOAD_SWITCH_SHIP]	   = "Ship",
+};
+
 static struct power_supply_attr power_supply_attrs[] __ro_after_init = {
 	/* Properties of type `int' */
 	POWER_SUPPLY_ENUM_ATTR(STATUS),
@@ -231,6 +239,7 @@ static struct power_supply_attr power_supply_attrs[] __ro_after_init = {
 	POWER_SUPPLY_ATTR(MANUFACTURE_DAY),
 	POWER_SUPPLY_ATTR(INTERNAL_RESISTANCE),
 	POWER_SUPPLY_ATTR(STATE_OF_HEALTH),
+	POWER_SUPPLY_ENUM_ATTR(LOAD_SWITCH),
 	/* Properties of type `const char *' */
 	POWER_SUPPLY_ATTR(MODEL_NAME),
 	POWER_SUPPLY_ATTR(MANUFACTURER),
@@ -400,6 +409,14 @@ static ssize_t power_supply_format_property(struct device *dev,
 		ret = power_supply_show_charge_types(dev, psy,
 						     value.intval, buf);
 		break;
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		if (uevent) /* no possible values in uevents */
+			goto default_format;
+		ret = power_supply_show_enum_with_available(
+			dev, POWER_SUPPLY_LOAD_SWITCH_TEXT,
+			ARRAY_SIZE(POWER_SUPPLY_LOAD_SWITCH_TEXT),
+			psy->desc->load_switches, value.intval, buf);
+		break;
 	case POWER_SUPPLY_PROP_MODEL_NAME ... POWER_SUPPLY_PROP_SERIAL_NUMBER:
 		ret = sysfs_emit(buf, "%s\n", value.strval);
 		break;
diff --git a/drivers/power/supply/qcom_smbx.c b/drivers/power/supply/qcom_smbx.c
index 67fdb433533883..f5557b8cc92aee 100644
--- a/drivers/power/supply/qcom_smbx.c
+++ b/drivers/power/supply/qcom_smbx.c
@@ -352,6 +352,9 @@
 
 /* pmi8998 registers represent current in increments of 1/40th of an amp */
 #define CURRENT_SCALE_FACTOR				25000
+#define SMB2_FLOAT_VOLTAGE_MIN_UV			3487500
+#define SMB2_FLOAT_VOLTAGE_MAX_UV			4920000
+#define SMB2_FLOAT_VOLTAGE_STEP_UV			7500
 /* clang-format on */
 
 enum charger_status {
@@ -378,6 +381,7 @@ struct smb_init_register {
  * @base:		Base address for smb registers
  * @regmap:		Register map
  * @batt_info:		Battery data from DT
+ * @initial_charge_enable: Charging enable state before hardware setup
  * @status_change_work: Worker to handle plug/unplug events
  * @cable_irq:		USB plugin IRQ
  * @wakeup_enabled:	If the cable IRQ will cause a wakeup
@@ -391,6 +395,7 @@ struct smb_chip {
 	unsigned int base;
 	struct regmap *regmap;
 	struct power_supply_battery_info *batt_info;
+	u8 initial_charge_enable;
 
 	struct delayed_work status_change_work;
 	int cable_irq;
@@ -632,26 +637,20 @@ static int smb_get_prop_health(struct smb_chip *chip, int *val)
 		return rc;
 	}
 
-	switch (stat) {
-	case CHARGER_ERROR_STATUS_BAT_OV_BIT:
+	if (stat & CHARGER_ERROR_STATUS_BAT_OV_BIT)
 		*val = POWER_SUPPLY_HEALTH_OVERVOLTAGE;
-		return 0;
-	case BAT_TEMP_STATUS_TOO_COLD_BIT:
+	else if (stat & BAT_TEMP_STATUS_TOO_COLD_BIT)
 		*val = POWER_SUPPLY_HEALTH_COLD;
-		return 0;
-	case BAT_TEMP_STATUS_TOO_HOT_BIT:
+	else if (stat & BAT_TEMP_STATUS_TOO_HOT_BIT)
 		*val = POWER_SUPPLY_HEALTH_OVERHEAT;
-		return 0;
-	case BAT_TEMP_STATUS_COLD_SOFT_LIMIT_BIT:
+	else if (stat & BAT_TEMP_STATUS_COLD_SOFT_LIMIT_BIT)
 		*val = POWER_SUPPLY_HEALTH_COOL;
-		return 0;
-	case BAT_TEMP_STATUS_HOT_SOFT_LIMIT_BIT:
+	else if (stat & BAT_TEMP_STATUS_HOT_SOFT_LIMIT_BIT)
 		*val = POWER_SUPPLY_HEALTH_WARM;
-		return 0;
-	default:
+	else
 		*val = POWER_SUPPLY_HEALTH_GOOD;
-		return 0;
-	}
+
+	return 0;
 }
 
 static int smb_get_property(struct power_supply *psy,
@@ -723,16 +722,22 @@ static irqreturn_t smb_handle_batt_overvoltage(int irq, void *data)
 {
 	struct smb_chip *chip = data;
 	unsigned int status;
+	int rc;
 
-	regmap_read(chip->regmap, chip->base + BATTERY_CHARGER_STATUS_2,
-		    &status);
+	rc = regmap_read(chip->regmap,
+			 chip->base + BATTERY_CHARGER_STATUS_2, &status);
+	if (rc < 0) {
+		dev_err(chip->dev, "Couldn't read charger status: %d\n", rc);
+		return IRQ_HANDLED;
+	}
 
 	if (status & CHARGER_ERROR_STATUS_BAT_OV_BIT) {
 		/* The hardware stops charging automatically */
 		dev_err(chip->dev, "battery overvoltage detected\n");
-		power_supply_changed(chip->chg_psy);
 	}
 
+	power_supply_changed(chip->chg_psy);
+
 	return IRQ_HANDLED;
 }
 
@@ -764,7 +769,7 @@ static irqreturn_t smb_handle_wdog_bark(int irq, void *data)
 
 	power_supply_changed(chip->chg_psy);
 
-	rc = regmap_write(chip->regmap, BARK_BITE_WDOG_PET,
+	rc = regmap_write(chip->regmap, chip->base + BARK_BITE_WDOG_PET,
 			  BARK_BITE_WDOG_PET_BIT);
 	if (rc < 0)
 		dev_err(chip->dev, "Couldn't pet the dog rc=%d\n", rc);
@@ -909,6 +914,19 @@ static int smb_init_hw(struct smb_chip *chip)
 	return 0;
 }
 
+static void smb_restore_charge_enable(void *data)
+{
+	struct smb_chip *chip = data;
+	int rc;
+
+	rc = regmap_update_bits(chip->regmap,
+				chip->base + CHARGING_ENABLE_CMD,
+				CHARGING_ENABLE_CMD_BIT,
+				chip->initial_charge_enable);
+	if (rc < 0)
+		dev_err(chip->dev, "Couldn't restore charging state: %d\n", rc);
+}
+
 static int smb_init_irq(struct smb_chip *chip, int *irq, const char *name,
 			 irqreturn_t (*handler)(int irq, void *data))
 {
@@ -935,6 +953,9 @@ static int smb_probe(struct platform_device *pdev)
 	struct power_supply_config supply_config = {};
 	struct power_supply_desc *desc;
 	struct smb_chip *chip;
+	unsigned int charge_enable;
+	unsigned int float_voltage_sel;
+	int float_voltage_uv;
 	int rc, irq;
 
 	chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL);
@@ -965,6 +986,18 @@ static int smb_probe(struct platform_device *pdev)
 				     "Couldn't get usbin_i IIO channel\n");
 	}
 
+	rc = regmap_read(chip->regmap, chip->base + CHARGING_ENABLE_CMD,
+			 &charge_enable);
+	if (rc < 0)
+		return dev_err_probe(chip->dev, rc,
+				     "Couldn't read charging state\n");
+
+	chip->initial_charge_enable = charge_enable & CHARGING_ENABLE_CMD_BIT;
+	rc = devm_add_action_or_reset(chip->dev, smb_restore_charge_enable, chip);
+	if (rc)
+		return dev_err_probe(chip->dev, rc,
+				     "Couldn't register charging state rollback\n");
+
 	rc = smb_init_hw(chip);
 	if (rc < 0)
 		return rc;
@@ -999,12 +1032,35 @@ static int smb_probe(struct platform_device *pdev)
 		return dev_err_probe(chip->dev, rc,
 				     "Failed to init status change work\n");
 
-	rc = (chip->batt_info->voltage_max_design_uv - 3487500) / 7500 + 1;
+	if (power_supply_battery_info_has_prop(chip->batt_info,
+					       POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX)) {
+		float_voltage_uv =
+			chip->batt_info->constant_charge_voltage_max_uv;
+	} else if (power_supply_battery_info_has_prop(chip->batt_info,
+					      POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN)) {
+		float_voltage_uv = chip->batt_info->voltage_max_design_uv;
+	} else {
+		dev_warn(chip->dev, "No battery float voltage; preserving hardware setting\n");
+		goto skip_float_voltage;
+	}
+
+	if (float_voltage_uv < SMB2_FLOAT_VOLTAGE_MIN_UV ||
+	    float_voltage_uv > SMB2_FLOAT_VOLTAGE_MAX_UV)
+		return dev_err_probe(chip->dev, -EINVAL,
+				     "float voltage %d uV outside %d-%d uV\n",
+				     float_voltage_uv,
+				     SMB2_FLOAT_VOLTAGE_MIN_UV,
+				     SMB2_FLOAT_VOLTAGE_MAX_UV);
+
+	float_voltage_sel =
+		(float_voltage_uv - SMB2_FLOAT_VOLTAGE_MIN_UV) /
+		SMB2_FLOAT_VOLTAGE_STEP_UV;
 	rc = regmap_update_bits(chip->regmap, chip->base + FLOAT_VOLTAGE_CFG,
-				FLOAT_VOLTAGE_SETTING_MASK, rc);
+				FLOAT_VOLTAGE_SETTING_MASK, float_voltage_sel);
 	if (rc < 0)
 		return dev_err_probe(chip->dev, rc, "Couldn't set vbat max\n");
 
+skip_float_voltage:
 	rc = smb_init_irq(chip, &irq, "bat-ov", smb_handle_batt_overvoltage);
 	if (rc < 0)
 		return rc;
@@ -1028,6 +1084,8 @@ static int smb_probe(struct platform_device *pdev)
 	if (rc < 0)
 		return dev_err_probe(chip->dev, rc, "Couldn't set wake irq\n");
 
+	devm_remove_action(chip->dev, smb_restore_charge_enable, chip);
+
 	platform_set_drvdata(pdev, chip);
 
 	/* Initialise charger state */
diff --git a/drivers/power/supply/rk817_charger.c b/drivers/power/supply/rk817_charger.c
index 5558b182a1a617..cf2ba6c31623e3 100644
--- a/drivers/power/supply/rk817_charger.c
+++ b/drivers/power/supply/rk817_charger.c
@@ -11,7 +11,7 @@
 #include <linux/unaligned.h>
 #include <linux/devm-helpers.h>
 #include <linux/mfd/rk808.h>
-#include <linux/irq.h>
+#include <linux/interrupt.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/power_supply.h>
diff --git a/drivers/power/supply/rt9467-charger.c b/drivers/power/supply/rt9467-charger.c
index a9a83b263e1878..f0d10f44b10902 100644
--- a/drivers/power/supply/rt9467-charger.c
+++ b/drivers/power/supply/rt9467-charger.c
@@ -633,6 +633,39 @@ static int rt9467_psy_set_ieoc(struct rt9467_chg_data *data, int microamp)
 	return ret;
 }
 
+static int rt9467_set_batfet(struct rt9467_chg_data *data, int val)
+{
+	unsigned int regval;
+
+	switch (val) {
+	case POWER_SUPPLY_LOAD_SWITCH_ON:
+		regval = 0;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_OFF:
+		regval = 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_field_write(data->rm_field[F_SHIP_MODE], regval);
+}
+
+static int rt9467_get_batfet(struct rt9467_chg_data *data, int *val)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_field_read(data->rm_field[F_SHIP_MODE], &regval);
+	if (ret < 0)
+		return ret;
+
+	*val = regval ? POWER_SUPPLY_LOAD_SWITCH_OFF :
+			POWER_SUPPLY_LOAD_SWITCH_ON;
+
+	return 0;
+}
+
 static const enum power_supply_property rt9467_chg_properties[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_ONLINE,
@@ -648,6 +681,7 @@ static const enum power_supply_property rt9467_chg_properties[] = {
 	POWER_SUPPLY_PROP_USB_TYPE,
 	POWER_SUPPLY_PROP_PRECHARGE_CURRENT,
 	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
+	POWER_SUPPLY_PROP_LOAD_SWITCH,
 };
 
 static int rt9467_psy_get_property(struct power_supply *psy,
@@ -711,6 +745,8 @@ static int rt9467_psy_get_property(struct power_supply *psy,
 		val->intval = data->ieoc_ua;
 		mutex_unlock(&data->ichg_ieoc_lock);
 		return 0;
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return rt9467_get_batfet(data, &val->intval);
 	default:
 		return -ENODATA;
 	}
@@ -747,6 +783,8 @@ static int rt9467_psy_set_property(struct power_supply *psy,
 						    RT9467_RANGE_IPREC, val->intval);
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
 		return rt9467_psy_set_ieoc(data, val->intval);
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return rt9467_set_batfet(data, val->intval);
 	default:
 		return -EINVAL;
 	}
@@ -764,6 +802,7 @@ static int rt9467_chg_prop_is_writeable(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT:
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
 	case POWER_SUPPLY_PROP_PRECHARGE_CURRENT:
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
 		return 1;
 	default:
 		return 0;
@@ -777,6 +816,8 @@ static const struct power_supply_desc rt9467_chg_psy_desc = {
 		     BIT(POWER_SUPPLY_USB_TYPE_CDP) |
 		     BIT(POWER_SUPPLY_USB_TYPE_DCP) |
 		     BIT(POWER_SUPPLY_USB_TYPE_UNKNOWN),
+	.load_switches = BIT(POWER_SUPPLY_LOAD_SWITCH_ON) |
+			 BIT(POWER_SUPPLY_LOAD_SWITCH_OFF),
 	.properties = rt9467_chg_properties,
 	.num_properties = ARRAY_SIZE(rt9467_chg_properties),
 	.property_is_writeable = rt9467_chg_prop_is_writeable,
@@ -793,14 +834,14 @@ static ssize_t sysoff_enable_show(struct device *dev,
 				  struct device_attribute *attr, char *buf)
 {
 	struct rt9467_chg_data *data = psy_device_to_chip(dev);
-	unsigned int sysoff_enable;
+	int val;
 	int ret;
 
-	ret = regmap_field_read(data->rm_field[F_SHIP_MODE], &sysoff_enable);
-	if (ret)
+	ret = rt9467_get_batfet(data, &val);
+	if (ret < 0)
 		return ret;
 
-	return sysfs_emit(buf, "%d\n", sysoff_enable);
+	return sysfs_emit(buf, "%d\n", val == POWER_SUPPLY_LOAD_SWITCH_OFF);
 }
 
 static ssize_t sysoff_enable_store(struct device *dev,
@@ -815,8 +856,9 @@ static ssize_t sysoff_enable_store(struct device *dev,
 	if (ret)
 		return ret;
 
-	ret = regmap_field_write(data->rm_field[F_SHIP_MODE], !!tmp);
-	if (ret)
+	ret = rt9467_set_batfet(data, tmp ? POWER_SUPPLY_LOAD_SWITCH_OFF :
+					    POWER_SUPPLY_LOAD_SWITCH_ON);
+	if (ret < 0)
 		return ret;
 
 	return count;
diff --git a/drivers/power/supply/rt9471.c b/drivers/power/supply/rt9471.c
index b67f5abaa9938b..b16d39a774dd4d 100644
--- a/drivers/power/supply/rt9471.c
+++ b/drivers/power/supply/rt9471.c
@@ -265,6 +265,39 @@ static int rt9471_get_ieoc(struct rt9471_chip *chip, int *microamp)
 	return rt9471_get_value_by_field_range(chip, F_IEOC_CHG, RT9471_RANGE_IEOC, microamp);
 }
 
+static int rt9471_set_batfet(struct rt9471_chip *chip, int val)
+{
+	unsigned int regval;
+
+	switch (val) {
+	case POWER_SUPPLY_LOAD_SWITCH_ON:
+		regval = 0;
+		break;
+	case POWER_SUPPLY_LOAD_SWITCH_OFF:
+		regval = 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_field_write(chip->rm_fields[F_BATFET_DIS], regval);
+}
+
+static int rt9471_get_batfet(struct rt9471_chip *chip, int *val)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_field_read(chip->rm_fields[F_BATFET_DIS], &regval);
+	if (ret < 0)
+		return ret;
+
+	*val = regval ? POWER_SUPPLY_LOAD_SWITCH_OFF :
+			POWER_SUPPLY_LOAD_SWITCH_ON;
+
+	return 0;
+}
+
 static int rt9471_get_status(struct rt9471_chip *chip, int *status)
 {
 	unsigned int ic_stat;
@@ -342,6 +375,7 @@ static enum power_supply_property rt9471_charger_properties[] = {
 	POWER_SUPPLY_PROP_USB_TYPE,
 	POWER_SUPPLY_PROP_PRECHARGE_CURRENT,
 	POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT,
+	POWER_SUPPLY_PROP_LOAD_SWITCH,
 	POWER_SUPPLY_PROP_MODEL_NAME,
 	POWER_SUPPLY_PROP_MANUFACTURER,
 };
@@ -358,6 +392,7 @@ static int rt9471_charger_property_is_writeable(struct power_supply *psy,
 	case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT:
 	case POWER_SUPPLY_PROP_PRECHARGE_CURRENT:
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
 		return 1;
 	default:
 		return 0;
@@ -393,6 +428,8 @@ static int rt9471_charger_set_property(struct power_supply *psy,
 			chip, F_IPRE_CHG, RT9471_RANGE_IPRE, val->intval);
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
 		return rt9471_set_ieoc(chip, val->intval);
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return rt9471_set_batfet(chip, val->intval);
 	default:
 		return -EINVAL;
 	}
@@ -439,6 +476,8 @@ static int rt9471_charger_get_property(struct power_supply *psy,
 			chip, F_IPRE_CHG, RT9471_RANGE_IPRE, &val->intval);
 	case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT:
 		return rt9471_get_ieoc(chip, &val->intval);
+	case POWER_SUPPLY_PROP_LOAD_SWITCH:
+		return rt9471_get_batfet(chip, &val->intval);
 	case POWER_SUPPLY_PROP_MODEL_NAME:
 		val->strval = rt9471_model;
 		return 0;
@@ -649,14 +688,14 @@ static ssize_t sysoff_enable_show(struct device *dev,
 				  struct device_attribute *attr, char *buf)
 {
 	struct rt9471_chip *chip = psy_device_to_chip(dev);
-	unsigned int sysoff_enable;
+	unsigned int val;
 	int ret;
 
-	ret = regmap_field_read(chip->rm_fields[F_BATFET_DIS], &sysoff_enable);
-	if (ret)
+	ret = rt9471_get_batfet(chip, &val);
+	if (ret < 0)
 		return ret;
 
-	return sysfs_emit(buf, "%d\n", sysoff_enable);
+	return sysfs_emit(buf, "%d\n", val == POWER_SUPPLY_LOAD_SWITCH_OFF);
 }
 
 static ssize_t sysoff_enable_store(struct device *dev,
@@ -671,8 +710,9 @@ static ssize_t sysoff_enable_store(struct device *dev,
 	if (ret)
 		return ret;
 
-	ret = regmap_field_write(chip->rm_fields[F_BATFET_DIS], !!tmp);
-	if (ret)
+	ret = rt9471_set_batfet(chip, tmp ? POWER_SUPPLY_LOAD_SWITCH_OFF :
+					    POWER_SUPPLY_LOAD_SWITCH_ON);
+	if (ret < 0)
 		return ret;
 
 	return count;
@@ -744,6 +784,8 @@ static int rt9471_register_psy(struct rt9471_chip *chip)
 			  BIT(POWER_SUPPLY_USB_TYPE_DCP) |
 			  BIT(POWER_SUPPLY_USB_TYPE_APPLE_BRICK_ID) |
 			  BIT(POWER_SUPPLY_USB_TYPE_UNKNOWN);
+	desc->load_switches = BIT(POWER_SUPPLY_LOAD_SWITCH_ON) |
+			      BIT(POWER_SUPPLY_LOAD_SWITCH_OFF);
 	desc->properties = rt9471_charger_properties;
 	desc->num_properties = ARRAY_SIZE(rt9471_charger_properties);
 	desc->get_property = rt9471_charger_get_property;
diff --git a/drivers/power/supply/s2mu005-battery.c b/drivers/power/supply/s2mu005-battery.c
index 53de6605b49222..a76154e5a09176 100644
--- a/drivers/power/supply/s2mu005-battery.c
+++ b/drivers/power/supply/s2mu005-battery.c
@@ -262,7 +262,7 @@ static int s2mu005_fg_i2c_probe(struct i2c_client *client)
 
 	ret = devm_mutex_init(dev, &priv->monout_mutex);
 	if (ret)
-		dev_err_probe(dev, ret, "failed to initialize MONOUT mutex\n");
+		return dev_err_probe(dev, ret, "failed to initialize MONOUT mutex\n");
 
 	psy_desc = device_get_match_data(dev);
 
@@ -277,7 +277,7 @@ static int s2mu005_fg_i2c_probe(struct i2c_client *client)
 					s2mu005_handle_irq, IRQF_ONESHOT,
 					psy_desc->name, priv);
 	if (ret)
-		dev_err_probe(dev, ret, "failed to request IRQ\n");
+		return dev_err_probe(dev, ret, "failed to request IRQ\n");
 
 	return 0;
 }
diff --git a/drivers/power/supply/sbs-charger.c b/drivers/power/supply/sbs-charger.c
index e7b4e10dcfd5f1..bc40bbb429e686 100644
--- a/drivers/power/supply/sbs-charger.c
+++ b/drivers/power/supply/sbs-charger.c
@@ -2,10 +2,10 @@
 /*
  * Copyright (c) 2016, Prodys S.L.
  *
- * This adds support for sbs-charger compilant chips as defined here:
+ * This adds support for sbs-charger compliant chips as defined here:
  * http://sbs-forum.org/specs/sbc110.pdf
  *
- * Implemetation based on sbs-battery.c
+ * Implementation based on sbs-battery.c
  */
 
 #include <linux/init.h>
diff --git a/drivers/power/supply/smb347-charger.c b/drivers/power/supply/smb347-charger.c
index c1b67dac8ef6f9..72dfa15b3d8060 100644
--- a/drivers/power/supply/smb347-charger.c
+++ b/drivers/power/supply/smb347-charger.c
@@ -149,7 +149,7 @@
  * @pre_charge_current: current (in uA) to use in pre-charging phase
  * @termination_current: current (in uA) used to determine when the
  *			 charging cycle terminates
- * @pre_to_fast_voltage: voltage (in uV) treshold used for transitioning to
+ * @pre_to_fast_voltage: voltage (in uV) threshold used for transitioning to
  *			 pre-charge to fast charge mode
  * @mains_current_limit: maximum input current drawn from AC/DC input (in uA)
  * @usb_hc_current_limit: maximum input high current (in uA) drawn from USB
@@ -302,7 +302,7 @@ static int current_to_hw(const unsigned int *tbl, size_t size, unsigned int val)
  *
  * Function checks whether any power source is connected to the charger and
  * updates internal state accordingly. If there is a change to previous state
- * function returns %1, otherwise %0 and negative errno in case of errror.
+ * function returns %1, otherwise %0 and negative errno in case of error.
  */
 static int smb347_update_ps_status(struct smb347_charger *smb)
 {
diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c
index a34543e664460b..d6686d03056a6b 100644
--- a/drivers/powercap/intel_rapl_msr.c
+++ b/drivers/powercap/intel_rapl_msr.c
@@ -456,6 +456,7 @@ static const struct x86_cpu_id rapl_ids[]  = {
 	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,		&rapl_defaults_spr_server),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M,		&rapl_defaults_core),
 	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,		&rapl_defaults_core_pl4_pmu),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_R,		&rapl_defaults_core_pl4_pmu),
 	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,		&rapl_defaults_core_pl4_pmu),
 	X86_MATCH_VFM(INTEL_NOVALAKE,			&rapl_defaults_core_pl4),
 	X86_MATCH_VFM(INTEL_NOVALAKE_L,			&rapl_defaults_core_pl4),
@@ -582,8 +583,8 @@ static int intel_rapl_msr_init(void)
 	rapl_msr_platdev = platform_device_register_data(NULL, "intel_rapl_msr", 0, def,
 							 sizeof(*def));
 	if (IS_ERR(rapl_msr_platdev))
-		pr_debug("intel_rapl_msr device register failed, ret:%ld\n",
-			 PTR_ERR(rapl_msr_platdev));
+		pr_debug("intel_rapl_msr device register failed, ret:%pe\n",
+			 rapl_msr_platdev);
 
 	return 0;
 }
diff --git a/drivers/ptp/ptp_idt82p33.c b/drivers/ptp/ptp_idt82p33.c
index f01c50dfa44e8f..ffc2ec1fc4bedb 100644
--- a/drivers/ptp/ptp_idt82p33.c
+++ b/drivers/ptp/ptp_idt82p33.c
@@ -918,6 +918,13 @@ static void idt82p33_ptp_clock_unregister_all(struct idt82p33 *idt82p33)
 	struct idt82p33_channel *channel;
 	u8 i;
 
+	mutex_lock(idt82p33->lock);
+	idt82p33->stopping = true;
+	idt82p33->extts_mask = 0;
+	mutex_unlock(idt82p33->lock);
+
+	cancel_delayed_work_sync(&idt82p33->extts_work);
+
 	for (i = 0; i < MAX_PHC_PLL; i++) {
 		channel = &idt82p33->channel[i];
 		cancel_delayed_work_sync(&channel->adjtime_work);
@@ -937,6 +944,10 @@ static int idt82p33_enable(struct ptp_clock_info *ptp,
 	int err = -EOPNOTSUPP;
 
 	mutex_lock(idt82p33->lock);
+	if (idt82p33->stopping) {
+		err = -ENODEV;
+		goto out;
+	}
 
 	switch (rq->type) {
 	case PTP_CLK_REQ_PEROUT:
@@ -958,6 +969,7 @@ static int idt82p33_enable(struct ptp_clock_info *ptp,
 		break;
 	}
 
+out:
 	mutex_unlock(idt82p33->lock);
 
 	if (err)
@@ -1044,11 +1056,14 @@ static int idt82p33_adjtime(struct ptp_clock_info *ptp, s64 delta_ns)
 		return -EBUSY;
 
 	mutex_lock(idt82p33->lock);
+	if (idt82p33->stopping) {
+		err = -ENODEV;
+		goto out;
+	}
 
 	if (abs(delta_ns) < phase_snap_threshold) {
 		err = idt82p33_start_ddco(channel, delta_ns);
-		mutex_unlock(idt82p33->lock);
-		return err;
+		goto out;
 	}
 
 	/* Use more accurate internal 1pps triggered write first */
@@ -1056,6 +1071,7 @@ static int idt82p33_adjtime(struct ptp_clock_info *ptp, s64 delta_ns)
 	if (err && delta_ns > IMMEDIATE_SNAP_THRESHOLD_NS)
 		err = _idt82p33_adjtime_immediate(channel, delta_ns);
 
+out:
 	mutex_unlock(idt82p33->lock);
 
 	if (err)
@@ -1342,6 +1358,8 @@ static void idt82p33_extts_check(struct work_struct *work)
 		return;
 
 	mutex_lock(idt82p33->lock);
+	if (idt82p33->stopping)
+		goto out;
 
 	for (i = 0; i < MAX_PHC_PLL; i++) {
 		mask = 1 << i;
@@ -1367,6 +1385,7 @@ static void idt82p33_extts_check(struct work_struct *work)
 		schedule_delayed_work(&idt82p33->extts_work,
 				      msecs_to_jiffies(EXTTS_PERIOD_MS));
 
+out:
 	mutex_unlock(idt82p33->lock);
 }
 
@@ -1442,8 +1461,6 @@ static void idt82p33_remove(struct platform_device *pdev)
 {
 	struct idt82p33 *idt82p33 = platform_get_drvdata(pdev);
 
-	cancel_delayed_work_sync(&idt82p33->extts_work);
-
 	idt82p33_ptp_clock_unregister_all(idt82p33);
 }
 
diff --git a/drivers/ptp/ptp_idt82p33.h b/drivers/ptp/ptp_idt82p33.h
index 6a63c14b6966b6..b88c921adaa914 100644
--- a/drivers/ptp/ptp_idt82p33.h
+++ b/drivers/ptp/ptp_idt82p33.h
@@ -91,6 +91,7 @@ struct idt82p33 {
 	u8			extts_mask;
 	bool			extts_single_shot;
 	struct delayed_work	extts_work;
+	bool			stopping;
 	/* Remember the ptp channel to report extts */
 	struct idt82p33_channel	*event_channel[MAX_PHC_PLL];
 	/* Mutex to protect operations from being interrupted */
diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c
index 3d26ec1f7b9ec2..2802989e849470 100644
--- a/drivers/ptp/ptp_ocp.c
+++ b/drivers/ptp/ptp_ocp.c
@@ -20,10 +20,14 @@
 #include <linux/spi/altera.h>
 #include <net/devlink.h>
 #include <linux/i2c.h>
+#include <linux/iopoll.h>
 #include <linux/mtd/mtd.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/crc16.h>
 #include <linux/dpll.h>
+#include <linux/unaligned.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
 
 #define PCI_DEVICE_ID_META_TIMECARD		0x0400
 
@@ -85,6 +89,8 @@ struct ptp_ocp_adva_info {
 	u8				signals_nr;
 	u8				freq_in_nr;
 	const struct ocp_attr_group	*attr_groups;
+	/* x1: TAP CPLD present */
+	bool				has_cpld;
 };
 
 #define OCP_CTRL_ENABLE		BIT(0)
@@ -163,7 +169,8 @@ struct gpio_reg {
 	u32	gpio1;
 	u32	__pad0;
 	u32	gpio2;
-	u32	__pad1;
+	/* adva_x1: I2C bus ownership register; reserved on other variants */
+	u32	i2c_bus_ctrl;
 };
 
 struct irig_master_reg {
@@ -416,6 +423,31 @@ struct ptp_ocp {
 	dpll_tracker tracker;
 	int signals_nr;
 	int freq_in_nr;
+	/* adva_x1 CPLD I2C (internal use only) */
+	/* serialises CPLD operations */
+	struct mutex		cpld_lock;
+	/* guards cpld_i2c_adap_nr against the bus notifier */
+	spinlock_t		cpld_adap_lock;
+	/* I2C adapter nr; -1 if absent.  Under cpld_adap_lock */
+	int			cpld_i2c_adap_nr;
+	/* claimed adapter; valid under cpld_lock */
+	struct i2c_adapter	*cpld_adap;
+	/* scratch, write half then read half; valid under cpld_lock */
+	u8			*cpld_buf;
+	/* Lattice device ID; 0 if unread */
+	u32			cpld_id;
+	/* USERCODE of the image in the part; valid once cpld_usercode_ok */
+	u32			cpld_usercode;
+	/* cpld_usercode has been read since the last flash */
+	bool			cpld_usercode_ok;
+	/* one-shot ID read finished, successfully or not; under cpld_lock */
+	bool			cpld_id_tried;
+	/* failed ID read attempts so far; under cpld_lock */
+	unsigned int		cpld_id_attempts;
+	/* x1 TAP CPLD present */
+	bool			has_cpld;
+	/* EN_CFG_TP issued but not yet REFRESH'd */
+	bool			cpld_in_config_mode;
 };
 
 #define OCP_REQ_TIMESTAMP	BIT(0)
@@ -452,6 +484,13 @@ static int ptp_ocp_adva_board_init(struct ptp_ocp *bp, struct ocp_resource *r);
 static const struct ocp_sma_op ocp_adva_sma_op;
 static const struct ocp_sma_op ocp_adva_x1_sma_op;
 
+/* Flash component naming the CPLD image, as reported by ->info_get(). */
+#define ADVA_CPLD_COMPONENT	"fw.cpld"
+
+static int adva_x1_cpld_flash(struct ptp_ocp *bp, struct devlink *devlink,
+			      const struct firmware *fw,
+			      struct netlink_ext_ack *extack);
+
 static const struct ocp_attr_group fb_timecard_groups[];
 
 static const struct ocp_attr_group art_timecard_groups[];
@@ -1273,6 +1312,7 @@ static struct ocp_resource ocp_adva_x1_resource[] = {
 			.signals_nr   = 4,
 			.freq_in_nr   = 4,
 			.attr_groups  = adva_timecard_x1_groups,
+			.has_cpld     = true,
 		},
 	},
 	{ }
@@ -2133,6 +2173,21 @@ ptp_ocp_devlink_flash_update(struct devlink *devlink,
 	const char *msg;
 	int err;
 
+	if (params->component) {
+		if (!bp->has_cpld ||
+		    strcmp(params->component, ADVA_CPLD_COMPONENT)) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "unsupported flash component");
+			return -EOPNOTSUPP;
+		}
+
+		err = adva_x1_cpld_flash(bp, devlink, params->fw, extack);
+		msg = err ? "Flash error" : "Flash complete";
+		devlink_flash_update_status_notify(devlink, msg,
+						   ADVA_CPLD_COMPONENT, 0, 0);
+		return err;
+	}
+
 	dev = ptp_ocp_find_flash(bp);
 	if (!dev) {
 		dev_err(&bp->pdev->dev, "Can't find Flash SPI adapter\n");
@@ -2155,10 +2210,13 @@ static int
 ptp_ocp_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req,
 			 struct netlink_ext_ack *extack)
 {
+	enum devlink_info_version_type ver_type =
+					DEVLINK_INFO_VERSION_TYPE_COMPONENT;
 	struct ptp_ocp *bp = devlink_priv(devlink);
 	const char *fw_image;
 	char buf[32];
 	int err;
+	u32 id;
 
 	fw_image = bp->fw_loader ? "loader" : "fw";
 	sprintf(buf, "%d.%d", bp->fw_tag, bp->fw_version);
@@ -2166,6 +2224,39 @@ ptp_ocp_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req,
 	if (err)
 		return err;
 
+	if (bp->has_cpld) {
+		/* Read by the worker, not here: this command is unprivileged
+		 * and reading the ID claims the I2C bus.  0 means unread -
+		 * skip it rather than fail, devlink discards the whole reply
+		 * on error.
+		 */
+		id = READ_ONCE(bp->cpld_id);
+		if (id) {
+			sprintf(buf, "0x%08x", id);
+			err = devlink_info_version_fixed_put(req, "cpld.id",
+							     buf);
+			if (err)
+				return err;
+		}
+
+		/* The flashable component.  Naming it here is what lets
+		 * "devlink dev flash ... component fw.cpld" through, as the
+		 * core matches the name against the versions reported here,
+		 * so it is reported for every board that has the part and not
+		 * only once its USERCODE has been read: a part left holding a
+		 * bad image answers neither, and gating the component on the
+		 * read would make that state unrecoverable.
+		 */
+		if (smp_load_acquire(&bp->cpld_usercode_ok))
+			sprintf(buf, "0x%08x", READ_ONCE(bp->cpld_usercode));
+		else
+			strscpy(buf, "unknown", sizeof(buf));
+		err = devlink_info_version_running_put_ext(req, "fw.cpld", buf,
+							   ver_type);
+		if (err)
+			return err;
+	}
+
 	if (!bp->has_eeprom_data) {
 		ptp_ocp_read_eeprom(bp);
 		if (!bp->has_eeprom_data)
@@ -3199,6 +3290,8 @@ ptp_ocp_adva_board_init(struct ptp_ocp *bp, struct ocp_resource *r)
 		return err;
 	ptp_ocp_sma_init(bp);
 
+	bp->has_cpld = info->has_cpld;
+
 	return ptp_ocp_init_clock(bp, &info->servo);
 }
 
@@ -4226,6 +4319,794 @@ static const struct ocp_attr_group art_timecard_groups[] = {
 	{ },
 };
 
+/*
+ * adva_x1 TAP CPLD (Lattice LCMXO3LF-2100) helpers.
+ *
+ * The card has two I2C controllers; Linux registers only 0x00150000.
+ * The i2c_bus_ctrl handshake re-routes what each is wired to:
+ *
+ *   grant 0:  0x150000 -> EEPROMs     0x120000 -> TMC or M.2, MB's choice
+ *   grant 1:  0x150000 -> TMC bus     0x120000 -> M.2
+ *
+ * The PCA9548 at 0x74 and the CPLD at 0x40 behind its channel 0 are on the
+ * TMC bus, reachable only while the grant is held.  The EEPROMs are not
+ * behind the adapter for that window, so a CPLD operation holds cpld_lock
+ * and the adapter lock to keep the EEPROM and nvmem paths off it.
+ */
+
+#define ADVA_MUX_ADDR     0x74
+#define ADVA_CPLD_ADDR    0x40
+#define ADVA_MUX_CHANNEL  0
+
+/* Longest command or reply; bounds each half of the per-claim scratch. */
+#define ADVA_CPLD_XFER_MAX 32
+
+/* i2c_bus_ctrl carries a host-written request half and a firmware-written
+ * grant half.  MBLAZE_GRANTED is what a granted register reads while our
+ * request still stands; the release write clears the request half, so from
+ * then on only the grant field can be tested.  Measured on an ADVA
+ * TimeCard X1: the register reads 0x00000012 once the firmware has handed
+ * the segment back, about 670 ms after the release write, so the low half
+ * is not zero then and only the grant field is a valid test.
+ */
+#define MBLAZE_REQUEST    0x0000aaaaU
+#define MBLAZE_GRANTED    0x5555aaaaU
+#define MBLAZE_RELEASE    0x55550000U
+#define MBLAZE_GRANT_MASK 0xffff0000U
+#define MBLAZE_GRANT      0x55550000U
+#define MBLAZE_RETRIES    200
+#define MBLAZE_RETRY_US   10000
+
+/* Lattice LCMXO3LF ISC command codes */
+#define CPLD_CMD_READ_ID      0xE0000000UL
+/* The USERCODE lives in the configuration flash that ERASE wipes, and the
+ * page writes carry it: after programming an image whose JEDEC UH field is
+ * 0x00000004 the part reads back 0x00000004, not an erased value.  So there
+ * is no separate ISC_PROGRAM_USERCODE step here.
+ */
+#define CPLD_CMD_READ_USERCODE 0xC0000000UL
+#define CPLD_CMD_READ_STATUS  0x3C000000UL
+#define CPLD_CMD_EN_CFG_TP    0x74   /* enable config, transparent mode */
+#define CPLD_CMD_DIS_CFG      0x26
+#define CPLD_CMD_ERASE        0x0E
+#define CPLD_CMD_RESET_ADDR   0x46
+#define CPLD_CMD_WRITE_PAGE   0x70
+#define CPLD_CMD_SET_DONE     0x5E
+#define CPLD_CMD_REFRESH      0x79
+#define CPLD_PAGE_SIZE        16
+#define CPLD_POLL_US          10000  /* status poll interval while busy */
+/* Bounds how long a claim can hold the i2c root lock, not image validity:
+ * far above any bitstream this part takes.
+ */
+#define CPLD_MAX_IMAGE_SZ     (256 * 1024)
+#define CPLD_EXIT_CFG_TRIES   3
+#define CPLD_ERASE_MS         15000  /* config sector erase, datasheet max */
+#define CPLD_ID_MAX_ATTEMPTS  10     /* one per sync_work tick */
+
+/* Status register bit positions (Lattice LCMXO3LF datasheet) */
+#define CPLD_STATUS_DONE   BIT(8)
+#define CPLD_STATUS_ENAB   BIT(9)
+#define CPLD_STATUS_BUSY   BIT(12)
+#define CPLD_STATUS_FAILED BIT(13)
+#define CPLD_STATUS_ERR    GENMASK(25, 23)
+
+/*
+ * Issue one I2C transaction on the TMC bus: @cmd if not negative, then
+ * @wlen bytes of @wdata (NULL sends zeros), then @rlen bytes read back.
+ * The message is built in the scratch buffer from adva_x1_bus_claim(), so
+ * a run of transfers costs one allocation, not one each.  I2C_M_DMA_SAFE
+ * is not set: the two halves are not separately aligned, so let the core
+ * bounce.  Caller holds the claim, hence __i2c_transfer().
+ */
+static int adva_x1_i2c_xfer(struct ptp_ocp *bp, u8 addr, int cmd,
+			    const void *wdata, u8 wlen,
+			    void *rdata, u8 rlen)
+{
+	u8 *wbuf, *rbuf;
+	struct i2c_adapter *adap;
+	struct i2c_msg msgs[2];
+	int nmsgs = 0, ret;
+	u16 hdr = cmd >= 0;
+
+	lockdep_assert_held(&bp->cpld_lock);
+
+	adap = bp->cpld_adap;
+	if (!adap || !bp->cpld_buf)
+		return -ENODEV;
+
+	if (hdr + wlen > ADVA_CPLD_XFER_MAX || rlen > ADVA_CPLD_XFER_MAX)
+		return -EINVAL;
+
+	wbuf = bp->cpld_buf;
+	rbuf = bp->cpld_buf + ADVA_CPLD_XFER_MAX;
+
+	if (hdr + wlen) {
+		if (hdr)
+			wbuf[0] = cmd;
+		if (wdata)
+			memcpy(wbuf + hdr, wdata, wlen);
+		else
+			memset(wbuf + hdr, 0, wlen);
+		msgs[nmsgs++] = (struct i2c_msg){
+			.addr  = addr,
+			.flags = 0,
+			.len   = hdr + wlen,
+			.buf   = wbuf,
+		};
+	}
+	if (rlen) {
+		msgs[nmsgs++] = (struct i2c_msg){
+			.addr  = addr,
+			.flags = I2C_M_RD,
+			.len   = rlen,
+			.buf   = rbuf,
+		};
+	}
+
+	ret = __i2c_transfer(adap, msgs, nmsgs);
+	if (ret != nmsgs)
+		return (ret < 0) ? ret : -EIO;
+
+	if (rdata && rlen)
+		memcpy(rdata, rbuf, rlen);
+
+	return 0;
+}
+
+/* Hand the segment back, and wait for the firmware to drop the grant before
+ * the caller unlocks the adapter: a transfer queued behind us would
+ * otherwise be clocked out while the controller is still on the TMC bus,
+ * which is what holding the adapter lock across the grant exists to stop.
+ */
+static int adva_x1_mblaze_release(struct ptp_ocp *bp)
+{
+	u32 val = 0;
+	int i;
+
+	if (!bp->pps_select)
+		return 0;
+
+	iowrite32(MBLAZE_RELEASE, &bp->pps_select->i2c_bus_ctrl);
+	/* Flush the posted write, as the acquire path does. */
+	ioread32(&bp->pps_select->i2c_bus_ctrl);
+
+	for (i = 0; i < MBLAZE_RETRIES; i++) {
+		val = ioread32(&bp->pps_select->i2c_bus_ctrl);
+		if ((val & MBLAZE_GRANT_MASK) != MBLAZE_GRANT)
+			return 0;
+		usleep_range(MBLAZE_RETRY_US, MBLAZE_RETRY_US + 1000);
+	}
+
+	dev_err(&bp->pdev->dev,
+		"TMC bus still granted after release, i2c_bus_ctrl 0x%08x\n",
+		val);
+
+	return -ETIMEDOUT;
+}
+
+/* Acquire the shared I2C bus from the MicroBlaze firmware.  Returns with no
+ * request outstanding on failure, so the firmware is never left granting a
+ * segment to a host that has given up waiting for it.
+ */
+static int adva_x1_mblaze_acquire(struct ptp_ocp *bp)
+{
+	u32 val;
+	int i;
+
+	if (!bp->pps_select)
+		return -ENODEV;
+
+	/* Drop a request left by a caller that died mid-sequence; cpld_lock
+	 * keeps live ones out.  The read back only flushes the posted write.
+	 */
+	iowrite32(0, &bp->pps_select->i2c_bus_ctrl);
+	ioread32(&bp->pps_select->i2c_bus_ctrl);
+
+	iowrite32(MBLAZE_REQUEST, &bp->pps_select->i2c_bus_ctrl);
+	for (i = 0; i < MBLAZE_RETRIES; i++) {
+		usleep_range(MBLAZE_RETRY_US, MBLAZE_RETRY_US + 1000);
+		val = ioread32(&bp->pps_select->i2c_bus_ctrl);
+		if (val == MBLAZE_GRANTED)
+			return 0;
+	}
+
+	/* Drop the request we gave up on.  Any error from the hand-back is
+	 * subsumed by the -ETIMEDOUT we are already returning.
+	 */
+	adva_x1_mblaze_release(bp);
+	return -ETIMEDOUT;
+}
+
+/* Route the host controller back to the EEPROMs and release the adapter.
+ * Safe after a failed claim: it also clears a request that was never granted.
+ *
+ * The adapter is unlocked even when the hand-back timed out: keeping the
+ * root lock would stall every other user of the controller for good, with
+ * no way to recover it.  The error is returned instead, so the operation
+ * that held the claim fails rather than reporting success over a bus that
+ * may still be routed to the TMC segment.
+ */
+static int adva_x1_bus_release(struct ptp_ocp *bp)
+{
+	struct i2c_adapter *adap = bp->cpld_adap;
+	int err;
+
+	if (!adap)
+		return 0;
+
+	err = adva_x1_mblaze_release(bp);
+	bp->cpld_adap = NULL;
+	kfree(bp->cpld_buf);
+	bp->cpld_buf = NULL;
+	i2c_unlock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
+	i2c_put_adapter(adap);
+
+	return err;
+}
+
+static int adva_x1_i2c_adap_match(struct device *dev, const void *data)
+{
+	return !!i2c_verify_adapter(dev);
+}
+
+/*
+ * Cache the adapter the CPLD hangs off.  ptp_ocp_i2c_notifier_call() misses
+ * it when the i2c controller driver is already loaded: the adapter is then
+ * added from inside platform_device_register(), before bp->i2c_ctrl is set.
+ * Only direct children are searched, so a mux adapter cannot match.
+ *
+ * Called from the worker rather than once from probe: device_add() notifies
+ * before it links the device to its parent, so with asynchronous probing
+ * both the notifier and a single lookup here can miss the same adapter.
+ * Retrying costs a short klist walk per tick until one of them succeeds.
+ */
+static void adva_x1_cache_i2c_adap(struct ptp_ocp *bp)
+{
+	struct device *child;
+
+	if (!bp->has_cpld || !bp->i2c_ctrl ||
+	    READ_ONCE(bp->cpld_i2c_adap_nr) >= 0)
+		return;
+
+	child = device_find_child(&bp->i2c_ctrl->dev, NULL,
+				  adva_x1_i2c_adap_match);
+	if (!child)
+		return;
+
+	/* Re-test under the lock so this does not overwrite a number the
+	 * notifier published meanwhile.  It cannot tell that apart from a
+	 * number the notifier has just invalidated, so an adapter removed in
+	 * this window can still be cached; adva_x1_bus_claim() rejects it on
+	 * the parent check and forgets it, and the next tick looks again.
+	 */
+	scoped_guard(spinlock, &bp->cpld_adap_lock) {
+		if (bp->cpld_i2c_adap_nr < 0)
+			bp->cpld_i2c_adap_nr = i2c_verify_adapter(child)->nr;
+	}
+
+	put_device(child);
+}
+
+/*
+ * Claim the TMC bus.  The adapter lock is held across the handshake to keep
+ * the EEPROM, nvmem and at24 paths off the controller while it is routed
+ * away from the EEPROMs; an EEPROM read blocks until the claim is dropped.
+ */
+static int adva_x1_bus_claim(struct ptp_ocp *bp)
+{
+	struct i2c_adapter *adap;
+	int ret;
+
+	lockdep_assert_held(&bp->cpld_lock);
+
+	adap = i2c_get_adapter(READ_ONCE(bp->cpld_i2c_adap_nr));
+	if (!adap)
+		return -ENODEV;
+
+	/* The number is freed before the notifier clears it, so it can
+	 * already be another adapter's.  Check this is still ours, and drop
+	 * a number that is not: otherwise every later claim fails the same
+	 * way, where forgetting it lets the worker look the adapter up again.
+	 */
+	if (!bp->i2c_ctrl || adap->dev.parent != &bp->i2c_ctrl->dev) {
+		scoped_guard(spinlock, &bp->cpld_adap_lock) {
+			if (bp->cpld_i2c_adap_nr == adap->nr)
+				bp->cpld_i2c_adap_nr = -1;
+		}
+		i2c_put_adapter(adap);
+		return -ENODEV;
+	}
+
+	/* One scratch buffer per claim rather than per transfer: a claim can
+	 * cover many transfers.
+	 */
+	bp->cpld_buf = kzalloc(2 * ADVA_CPLD_XFER_MAX, GFP_KERNEL);
+	if (!bp->cpld_buf) {
+		i2c_put_adapter(adap);
+		return -ENOMEM;
+	}
+
+	i2c_lock_bus(adap, I2C_LOCK_ROOT_ADAPTER);
+	bp->cpld_adap = adap;
+
+	ret = adva_x1_mblaze_acquire(bp);
+	if (ret)
+		adva_x1_bus_release(bp);	/* keeps the acquire error */
+
+	return ret;
+}
+
+/* Select a mux channel, or deselect all with ch < 0 - the power-on state.
+ * The mux is on the TMC bus, so what it is left set to never affects the
+ * EEPROM paths.
+ */
+static int adva_x1_mux_select(struct ptp_ocp *bp, int ch)
+{
+	u8 val = (ch >= 0) ? BIT(ch) : 0;
+
+	return adva_x1_i2c_xfer(bp, ADVA_MUX_ADDR, val, NULL, 0, NULL, 0);
+}
+
+/*
+ * Argument bytes that follow an ISC opcode.  Returns NULL with @nargs set
+ * when the arguments are all zero: adva_x1_i2c_xfer() zeroes the buffer.
+ *
+ * EN_CFG_TP, DIS_CFG and REFRESH take two operand bytes here where the SPI
+ * drivers in drivers/fpga send three.  These are the frames the part on this
+ * board answers: the sequence below has programmed and refreshed it
+ * successfully over I2C, so the counts are kept as validated rather than
+ * aligned with the SPI framing.
+ */
+static const u8 *adva_x1_cpld_args(u8 cmd, u8 *nargs)
+{
+	static const u8 en_cfg_tp[] = { 0x08, 0x00 };
+	/* cfg sector only */
+	static const u8 erase_cfg[] = { 0x04, 0x00, 0x00 };
+
+	switch (cmd) {
+	case CPLD_CMD_EN_CFG_TP:
+		*nargs = sizeof(en_cfg_tp);
+		return en_cfg_tp;
+	case CPLD_CMD_ERASE:
+		*nargs = sizeof(erase_cfg);
+		return erase_cfg;
+	case CPLD_CMD_RESET_ADDR:
+	case CPLD_CMD_SET_DONE:
+		*nargs = 3;
+		return NULL;
+	case CPLD_CMD_DIS_CFG:
+	case CPLD_CMD_REFRESH:
+		*nargs = 2;
+		return NULL;
+	default:
+		*nargs = 0;
+		return NULL;
+	}
+}
+
+/* Send an ISC command with the fixed arguments that belong to it. */
+static int adva_x1_cpld_write(struct ptp_ocp *bp, u8 cmd)
+{
+	const u8 *args;
+	u8 nargs;
+
+	args = adva_x1_cpld_args(cmd, &nargs);
+
+	return adva_x1_i2c_xfer(bp, ADVA_CPLD_ADDR, cmd, args, nargs, NULL, 0);
+}
+
+/*
+ * Send a 4-byte command and read back without an intermediate STOP: two
+ * messages in one transfer is the Lattice write -> repeated START -> read,
+ * so no protocol-mangling flag is needed.
+ */
+static int adva_x1_cpld_cmd_read(struct ptp_ocp *bp,
+				 u32 cmd_be, u8 *out, u8 out_len)
+{
+	__be32 cmd = cpu_to_be32(cmd_be);
+
+	return adva_x1_i2c_xfer(bp, ADVA_CPLD_ADDR, -1, &cmd, 4, out, out_len);
+}
+
+static int adva_x1_cpld_read_status(struct ptp_ocp *bp, u32 *status)
+{
+	u8 buf[4];
+	int ret;
+
+	ret = adva_x1_cpld_cmd_read(bp, CPLD_CMD_READ_STATUS, buf, 4);
+	if (ret)
+		return ret;
+	*status = get_unaligned_be32(buf);
+	return 0;
+}
+
+/* Poll the status register until the CPLD goes idle, or @max_ms elapses.
+ * The deadline is on wall time, so the I2C transactions count against it,
+ * and the status is read once more after it expires before giving up.
+ */
+static int adva_x1_cpld_wait_ready(struct ptp_ocp *bp, unsigned int max_ms)
+{
+	u32 status = 0;
+	int err = 0, ret;
+
+	ret = read_poll_timeout(adva_x1_cpld_read_status, err,
+				err || (status & CPLD_STATUS_FAILED) ||
+				!(status & CPLD_STATUS_BUSY),
+				CPLD_POLL_US, max_ms * USEC_PER_MSEC, false,
+				bp, &status);
+	if (ret)
+		return ret;
+	/* Keep the transport errno so it reaches userspace as-is; -EIO is
+	 * reserved for the CPLD itself reporting FAILED.
+	 */
+	if (err)
+		return err;
+	if (status & CPLD_STATUS_FAILED)
+		return -EIO;
+
+	return 0;
+}
+
+/* Wait for BUSY to clear.  Unlike adva_x1_cpld_wait_ready() a latched FAILED
+ * status is not an error here: this is used on the exit path, where an ISC
+ * command issued while the part is still erasing or programming may not
+ * latch, so the operation has to be waited out whatever its outcome.
+ */
+static int adva_x1_cpld_wait_idle(struct ptp_ocp *bp, unsigned int max_ms)
+{
+	u32 status = 0;
+	int err = 0, ret;
+
+	ret = read_poll_timeout(adva_x1_cpld_read_status, err,
+				err || !(status & CPLD_STATUS_BUSY),
+				CPLD_POLL_US, max_ms * USEC_PER_MSEC, false,
+				bp, &status);
+	if (ret)
+		return ret;
+
+	return err;
+}
+
+/* Leave transparent configuration mode.
+ *
+ * A DIS_CFG issued while the part is still busy may not latch, so the wait
+ * has to succeed before the write is believed - an ACKed write after a
+ * timed-out wait says nothing about whether the part left the mode.  Retry
+ * a few times rather than leaving it latched on one bad attempt; there is
+ * no other caller that would try again.
+ */
+static void adva_x1_cpld_exit_config(struct ptp_ocp *bp)
+{
+	int err, i;
+
+	if (!bp->cpld_in_config_mode)
+		return;
+
+	for (i = 0; i < CPLD_EXIT_CFG_TRIES; i++) {
+		err = adva_x1_cpld_wait_idle(bp, CPLD_ERASE_MS);
+		if (!err)
+			err = adva_x1_cpld_write(bp, CPLD_CMD_DIS_CFG);
+		if (!err) {
+			bp->cpld_in_config_mode = false;
+			return;
+		}
+	}
+
+	dev_err(&bp->pdev->dev, "CPLD left in configuration mode: %d\n", err);
+}
+
+/*
+ * Read the Lattice device ID and the image USERCODE under one bus claim.
+ * Done once, off the unprivileged devlink path, which reports the cached
+ * values.  -EBUSY means cpld_lock is held, so the caller can retry rather
+ * than wait behind a long CPLD operation.
+ */
+static int adva_x1_cpld_read_id(struct ptp_ocp *bp)
+{
+	u8 data[4];
+	int err, ret;
+
+	if (!mutex_trylock(&bp->cpld_lock))
+		return -EBUSY;
+
+	ret = adva_x1_bus_claim(bp);
+	if (ret)
+		goto out;
+	/* A select reported as failed may still have been ACKed, so the
+	 * deselect runs either way rather than handing the mux back with a
+	 * channel possibly left open.
+	 */
+	ret = adva_x1_mux_select(bp, ADVA_MUX_CHANNEL);
+	if (ret)
+		goto deselect;
+	ret = adva_x1_cpld_cmd_read(bp, CPLD_CMD_READ_ID, data, 4);
+	if (ret)
+		goto deselect;
+	/* Publish the ID before the USERCODE is attempted: a part that does
+	 * not answer 0xC0 must not cost us the device ID as well.
+	 */
+	WRITE_ONCE(bp->cpld_id, get_unaligned_be32(data));
+
+	ret = adva_x1_cpld_cmd_read(bp, CPLD_CMD_READ_USERCODE, data, 4);
+	if (ret)
+		goto deselect;
+	WRITE_ONCE(bp->cpld_usercode, get_unaligned_be32(data));
+	/* Pairs with the acquire in ptp_ocp_devlink_info_get(): the value has
+	 * to be visible before the flag that declares it valid.
+	 */
+	smp_store_release(&bp->cpld_usercode_ok, true);
+deselect:
+	err = adva_x1_mux_select(bp, -1);
+	if (!ret)
+		ret = err;
+	err = adva_x1_bus_release(bp);
+	if (!ret)
+		ret = err;
+out:
+	/* Under cpld_lock, so an operation that resets this bookkeeping
+	 * while holding the lock cannot have it overwritten afterwards by a
+	 * worker that had already finished reading.
+	 */
+	if (!ret || ++bp->cpld_id_attempts >= CPLD_ID_MAX_ATTEMPTS)
+		bp->cpld_id_tried = true;
+	mutex_unlock(&bp->cpld_lock);
+	if (ret)
+		dev_dbg(&bp->pdev->dev,
+			"CPLD device ID unavailable: %d\n", ret);
+
+	return ret;
+}
+
+/* cpld_status - show the flags set in the TAP CPLD status register. */
+static ssize_t
+cpld_status_show(struct device *dev, struct device_attribute *attr,
+		 char *buf)
+{
+	struct ptp_ocp *bp = dev_get_drvdata(dev);
+	int count = 0;
+	u32 st = 0;
+	int err, ret;
+
+	/* A CPLD operation can hold cpld_lock a long time; stay killable. */
+	ret = mutex_lock_interruptible(&bp->cpld_lock);
+	if (ret)
+		return ret;
+
+	ret = adva_x1_bus_claim(bp);
+	if (ret)
+		goto out;
+	ret = adva_x1_mux_select(bp, ADVA_MUX_CHANNEL);
+	if (ret)
+		goto deselect;
+	ret = adva_x1_cpld_read_status(bp, &st);
+deselect:
+	err = adva_x1_mux_select(bp, -1);
+	if (!ret)
+		ret = err;
+	err = adva_x1_bus_release(bp);
+	if (!ret)
+		ret = err;
+out:
+	mutex_unlock(&bp->cpld_lock);
+	if (ret)
+		return ret;
+
+	/* Flags: the names of the bits that are set, nothing when none is,
+	 * rather than a value per bit in one file.
+	 */
+	if (st & CPLD_STATUS_DONE)
+		count += sysfs_emit_at(buf, count, "done");
+	if (st & CPLD_STATUS_BUSY)
+		count += sysfs_emit_at(buf, count, "%sbusy", count ? " " : "");
+	if (st & CPLD_STATUS_FAILED)
+		count += sysfs_emit_at(buf, count, "%sfailed",
+				       count ? " " : "");
+
+	return count + sysfs_emit_at(buf, count, "\n");
+}
+static DEVICE_ATTR_ADMIN_RO(cpld_status);
+
+/*
+ * Program the CPLD configuration flash from @fw and activate it.  Selected
+ * by the "fw.cpld" flash component; the SPI flash keeps the default path.
+ * Runs under cpld_lock with the i2c adapter lock held, so an EEPROM read
+ * blocks for as long as programming takes.
+ */
+static int adva_x1_cpld_flash(struct ptp_ocp *bp, struct devlink *devlink,
+			      const struct firmware *fw,
+			      struct netlink_ext_ack *extack)
+{
+	size_t offset;
+	int err, ret;
+	u32 st;
+
+	if (!fw->size || fw->size % CPLD_PAGE_SIZE) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "image must be a whole number of 16-byte pages");
+		return -EINVAL;
+	}
+	if (fw->size > CPLD_MAX_IMAGE_SZ) {
+		NL_SET_ERR_MSG_MOD(extack, "image too large for this part");
+		return -EINVAL;
+	}
+
+	mutex_lock(&bp->cpld_lock);
+
+	err = adva_x1_bus_claim(bp);
+	if (err)
+		goto unlock;
+	/* A select reported as failed may still have been ACKed, so hand the
+	 * mux back deselected either way.
+	 */
+	err = adva_x1_mux_select(bp, ADVA_MUX_CHANNEL);
+	if (err)
+		goto deselect;
+
+	/* A previous flash may have failed to leave configuration mode.  Now
+	 * that the bus is claimed again, retry the exit before re-entering;
+	 * a no-op when the flag is already clear.
+	 */
+	adva_x1_cpld_exit_config(bp);
+
+	/* Set before EN_CFG_TP, not after: the CPLD may have entered config
+	 * mode even if the write errors or the wait times out, and only this
+	 * makes the exit path send DIS_CFG.  A stray DIS_CFG is harmless;
+	 * leaving config mode enabled is not.
+	 */
+	bp->cpld_in_config_mode = true;
+
+	err = adva_x1_cpld_write(bp, CPLD_CMD_EN_CFG_TP);
+	if (!err)
+		err = adva_x1_cpld_wait_ready(bp, 5000);
+	if (err)
+		goto exit_config;
+
+	/* Confirm the part really entered configuration mode.  An enable
+	 * frame that did not latch leaves an idle part reporting neither
+	 * BUSY nor FAILED, the erase and page writes are ignored, and DONE
+	 * is still set from the old image - so every later check passes and
+	 * the update would be reported successful with the flash untouched.
+	 */
+	err = adva_x1_cpld_read_status(bp, &st);
+	if (err)
+		goto exit_config;
+	if (!(st & CPLD_STATUS_ENAB)) {
+		dev_err(&bp->pdev->dev,
+			"CPLD did not enter configuration mode, status 0x%08x\n",
+			st);
+		NL_SET_ERR_MSG_MOD(extack,
+				   "CPLD did not enter configuration mode");
+		err = -EIO;
+		goto exit_config;
+	}
+
+	devlink_flash_update_status_notify(devlink, "Erasing",
+					   ADVA_CPLD_COMPONENT, 0, 0);
+	err = adva_x1_cpld_write(bp, CPLD_CMD_ERASE);
+	if (!err)
+		err = adva_x1_cpld_wait_ready(bp, CPLD_ERASE_MS);
+	if (err)
+		goto exit_config;
+
+	/* The old image is gone from here on, so stop reporting its
+	 * identity even if the rest of the sequence fails.  Written under
+	 * cpld_lock, which adva_x1_cpld_read_id() also holds across its own
+	 * bookkeeping, so the worker cannot resurrect any of it.
+	 */
+	WRITE_ONCE(bp->cpld_id, 0);
+	WRITE_ONCE(bp->cpld_usercode_ok, false);
+	bp->cpld_id_tried = false;
+	bp->cpld_id_attempts = 0;
+
+	err = adva_x1_cpld_write(bp, CPLD_CMD_RESET_ADDR);
+	if (err)
+		goto exit_config;
+
+	for (offset = 0; offset < fw->size; offset += CPLD_PAGE_SIZE) {
+		u8 args[3 + CPLD_PAGE_SIZE] = { 0x00, 0x00, 0x01 };
+
+		/* The loop holds cpld_lock and the i2c root lock for the
+		 * whole image, so give a dying task a way out.  The part is
+		 * left unconfigured, which is recoverable: fw.cpld stays
+		 * advertised so the image can be written again.
+		 */
+		if (fatal_signal_pending(current)) {
+			err = -EINTR;
+			goto exit_config;
+		}
+
+		memcpy(&args[3], fw->data + offset, CPLD_PAGE_SIZE);
+		err = adva_x1_i2c_xfer(bp, ADVA_CPLD_ADDR, CPLD_CMD_WRITE_PAGE,
+				       args, sizeof(args), NULL, 0);
+		if (!err)
+			err = adva_x1_cpld_wait_ready(bp, 100);
+		if (err)
+			goto exit_config;
+
+		if (!(offset % (CPLD_PAGE_SIZE * 64)))
+			devlink_flash_update_status_notify(devlink,
+							   "Programming",
+							   ADVA_CPLD_COMPONENT,
+							   offset, fw->size);
+	}
+	devlink_flash_update_status_notify(devlink, "Programming",
+					   ADVA_CPLD_COMPONENT,
+					   fw->size, fw->size);
+
+	err = adva_x1_cpld_write(bp, CPLD_CMD_SET_DONE);
+	if (!err)
+		err = adva_x1_cpld_wait_ready(bp, 1000);
+	if (err)
+		goto exit_config;
+
+	err = adva_x1_cpld_read_status(bp, &st);
+	if (err)
+		goto exit_config;
+	if (!(st & CPLD_STATUS_DONE)) {
+		dev_err(&bp->pdev->dev,
+			"CPLD SET_DONE left status 0x%08x\n", st);
+		NL_SET_ERR_MSG_MOD(extack, "CPLD did not accept the image");
+		err = -EIO;
+		goto exit_config;
+	}
+
+	devlink_flash_update_status_notify(devlink, "Activating",
+					   ADVA_CPLD_COMPONENT, 0, 0);
+	err = adva_x1_cpld_write(bp, CPLD_CMD_REFRESH);
+	if (err)
+		goto exit_config;
+
+	/* REFRESH reboots the CPLD out of configuration mode, so the exit
+	 * path must not send DIS_CFG afterwards even if a check below fails.
+	 */
+	bp->cpld_in_config_mode = false;
+
+	/* The new image is already running, so a segment that is not back
+	 * yet must not be reported as a failed update: retry the reselect
+	 * rather than sampling the mux once at a fixed delay.
+	 */
+	msleep(1500);
+	err = read_poll_timeout(adva_x1_mux_select, err, !err, CPLD_POLL_US,
+				3000 * USEC_PER_MSEC, false,
+				bp, ADVA_MUX_CHANNEL);
+	if (err)
+		goto deselect;
+
+	err = adva_x1_cpld_wait_ready(bp, 3000);
+	if (err)
+		goto deselect;
+
+	/* Require DONE set, not busy and no error code, as machxo2-spi.c does
+	 * after a refresh: without it a CRC or preamble error reads back as a
+	 * successful update.
+	 */
+	err = adva_x1_cpld_read_status(bp, &st);
+	if (err)
+		goto deselect;
+	if (!(st & CPLD_STATUS_DONE) || (st & CPLD_STATUS_BUSY) ||
+	    (st & CPLD_STATUS_ERR)) {
+		dev_err(&bp->pdev->dev,
+			"CPLD refresh left status 0x%08x\n", st);
+		NL_SET_ERR_MSG_MOD(extack, "CPLD did not come back configured");
+		err = -EIO;
+		goto deselect;
+	}
+
+exit_config:
+	adva_x1_cpld_exit_config(bp);
+deselect:
+	ret = adva_x1_mux_select(bp, -1);
+	if (!err)
+		err = ret;
+	ret = adva_x1_bus_release(bp);
+	if (!err)
+		err = ret;
+unlock:
+	mutex_unlock(&bp->cpld_lock);
+
+	return err;
+}
+
 static struct attribute *adva_timecard_attrs[] = {
 	&dev_attr_serialnum.attr,
 	&dev_attr_gnss_sync.attr,
@@ -4274,6 +5155,7 @@ static struct attribute *adva_timecard_x1_attrs[] = {
 	&dev_attr_ts_window_adjust.attr,
 	&dev_attr_utc_tai_offset.attr,
 	&dev_attr_tod_correction.attr,
+	&dev_attr_cpld_status.attr,
 	NULL,
 };
 
@@ -4904,6 +5786,7 @@ ptp_ocp_detach(struct ptp_ocp *bp)
 		clk_hw_unregister_fixed_rate(bp->i2c_clk);
 	if (bp->n_irqs)
 		pci_free_irq_vectors(bp->pdev);
+	mutex_destroy(&bp->cpld_lock);
 	device_unregister(&bp->dev);
 }
 
@@ -5052,6 +5935,17 @@ ptp_ocp_sync_work(struct work_struct *work)
 
 	bp->sync = sync;
 
+	/* Resolve the adapter here rather than once in probe, where it can
+	 * race the adapter's own registration, and read the ID as soon as it
+	 * turns up.  A claim can fail transiently - the firmware may not
+	 * grant the segment straight after power-up - so adva_x1_cpld_read_id()
+	 * retries a bounded number of times before giving up.
+	 */
+	adva_x1_cache_i2c_adap(bp);
+	if (bp->has_cpld && !READ_ONCE(bp->cpld_id_tried) &&
+	    READ_ONCE(bp->cpld_i2c_adap_nr) >= 0)
+		adva_x1_cpld_read_id(bp);
+
 	queue_delayed_work(system_power_efficient_wq, &bp->sync_work, HZ);
 }
 
@@ -5080,6 +5974,15 @@ ptp_ocp_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (err)
 		goto out_disable;
 
+	/* Before the first error path that calls ptp_ocp_detach(), so
+	 * mutex_destroy() runs on an initialised mutex, and before
+	 * ptp_ocp_register_resources(), so this sentinel cannot overwrite an
+	 * adapter number the i2c notifier stores during registration.
+	 */
+	mutex_init(&bp->cpld_lock);
+	spin_lock_init(&bp->cpld_adap_lock);
+	bp->cpld_i2c_adap_nr = -1;
+
 	INIT_DELAYED_WORK(&bp->sync_work, ptp_ocp_sync_work);
 
 	/* compat mode.
@@ -5119,12 +6022,12 @@ ptp_ocp_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (IS_ERR(bp->dpll)) {
 		err = PTR_ERR(bp->dpll);
 		dev_err(&pdev->dev, "dpll_device_alloc failed\n");
-		goto out;
+		goto out_devlink;
 	}
 
 	err = dpll_device_register(bp->dpll, DPLL_TYPE_PPS, &dpll_ops, bp);
 	if (err)
-		goto out;
+		goto out_dpll_put;
 
 	for (i = 0; i < OCP_SMA_NUM; i++) {
 		bp->sma[i].dpll_pin = dpll_pin_get(clkid, i, THIS_MODULE,
@@ -5150,7 +6053,16 @@ ptp_ocp_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		dpll_pin_unregister(bp->dpll, bp->sma[i].dpll_pin, &dpll_pins_ops, &bp->sma[i]);
 		dpll_pin_put(bp->sma[i].dpll_pin, &bp->sma[i].tracker);
 	}
+	/* dpll_device_register() takes a second reference and stores bp; only
+	 * dpll_device_unregister() drops it.  Without this the device stays
+	 * visible with a priv pointer into storage devlink_free() will release.
+	 */
+	dpll_device_unregister(bp->dpll, &dpll_ops, bp);
+out_dpll_put:
 	dpll_device_put(bp->dpll, &bp->tracker);
+out_devlink:
+	/* detach() destroys cpld_lock; no devlink op may still reach it */
+	devlink_unregister(devlink);
 out:
 	ptp_ocp_detach(bp);
 out_disable:
@@ -5196,6 +6108,7 @@ ptp_ocp_i2c_notifier_call(struct notifier_block *nb,
 			  unsigned long action, void *data)
 {
 	struct device *dev, *child = data;
+	struct i2c_adapter *adap;
 	struct ptp_ocp *bp;
 	bool add;
 
@@ -5219,11 +6132,25 @@ ptp_ocp_i2c_notifier_call(struct notifier_block *nb,
 
 found:
 	bp = dev_get_drvdata(dev);
-	if (add)
+	adap = i2c_verify_adapter(child);
+	if (add) {
 		ptp_ocp_symlink(bp, child, "i2c");
-	else
+		/* Only the card's own controller: a mux child added on it
+		 * reaches ptp_ocp through the same walk.  Numbers are
+		 * recycled, so the claim re-checks the parent itself.
+		 */
+		if (bp->i2c_ctrl && adap->dev.parent == &bp->i2c_ctrl->dev) {
+			scoped_guard(spinlock, &bp->cpld_adap_lock)
+				bp->cpld_i2c_adap_nr = adap->nr;
+		}
+	} else {
+		/* invalidate before free, but only for the cached adapter */
+		scoped_guard(spinlock, &bp->cpld_adap_lock) {
+			if (bp->cpld_i2c_adap_nr == adap->nr)
+				bp->cpld_i2c_adap_nr = -1;
+		}
 		sysfs_remove_link(&bp->dev.kobj, "i2c");
-
+	}
 	return 0;
 }
 
diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c
index 2a050ff4608bf1..4c6d716e5fde1d 100644
--- a/drivers/pwm/core.c
+++ b/drivers/pwm/core.c
@@ -2389,6 +2389,7 @@ static const struct file_operations pwm_cdev_fileops = {
 	.release = pwm_cdev_release,
 	.owner = THIS_MODULE,
 	.unlocked_ioctl = pwm_cdev_ioctl,
+	.compat_ioctl = compat_ptr_ioctl,
 };
 
 static dev_t pwm_devt;
diff --git a/drivers/pwm/pwm-brcmstb.c b/drivers/pwm/pwm-brcmstb.c
index 790ef7ffbbe571..78378787e4e930 100644
--- a/drivers/pwm/pwm-brcmstb.c
+++ b/drivers/pwm/pwm-brcmstb.c
@@ -101,7 +101,7 @@ static int brcmstb_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
 	u32 value;
 
 	/*
-	 * If asking for a duty_ns equal to period_ns, we need to substract
+	 * If asking for a duty_ns equal to period_ns, we need to subtract
 	 * the period value by 1 to make it shorter than the "on" time and
 	 * produce a flat 100% duty cycle signal, and max out the "on" time
 	 */
diff --git a/drivers/pwm/pwm-ipq.c b/drivers/pwm/pwm-ipq.c
index 6b7aec566f7eb9..3e1231b212b10f 100644
--- a/drivers/pwm/pwm-ipq.c
+++ b/drivers/pwm/pwm-ipq.c
@@ -186,7 +186,7 @@ static int ipq_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
 	state->period = DIV64_U64_ROUND_UP(effective_div * NSEC_PER_SEC,
 					   ipq_chip->clk_rate);
 
-	hi_div = hi_dur * (pre_div + 1);
+	hi_div = (u64)hi_dur * (pre_div + 1);
 	state->duty_cycle = DIV64_U64_ROUND_UP(hi_div * NSEC_PER_SEC,
 					       ipq_chip->clk_rate);
 
diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c
index 13e5e138c8e9cd..6c6caee65b8f13 100644
--- a/drivers/pwm/pwm-iqs620a.c
+++ b/drivers/pwm/pwm-iqs620a.c
@@ -173,18 +173,6 @@ static const struct pwm_ops iqs620_pwm_ops = {
 	.get_state = iqs620_pwm_get_state,
 };
 
-static void iqs620_pwm_notifier_unregister(void *context)
-{
-	struct iqs620_pwm_private *iqs620_pwm = context;
-	int ret;
-
-	ret = blocking_notifier_chain_unregister(&iqs620_pwm->iqs62x->nh,
-						 &iqs620_pwm->notifier);
-	if (ret)
-		dev_err(iqs620_pwm->dev,
-			"Failed to unregister notifier: %d\n", ret);
-}
-
 static int iqs620_pwm_probe(struct platform_device *pdev)
 {
 	struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent);
@@ -218,19 +206,14 @@ static int iqs620_pwm_probe(struct platform_device *pdev)
 	mutex_init(&iqs620_pwm->lock);
 
 	iqs620_pwm->notifier.notifier_call = iqs620_pwm_notifier;
-	ret = blocking_notifier_chain_register(&iqs620_pwm->iqs62x->nh,
-					       &iqs620_pwm->notifier);
+	ret = devm_blocking_notifier_chain_register(&pdev->dev,
+						    &iqs620_pwm->iqs62x->nh,
+						    &iqs620_pwm->notifier);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret);
 		return ret;
 	}
 
-	ret = devm_add_action_or_reset(&pdev->dev,
-				       iqs620_pwm_notifier_unregister,
-				       iqs620_pwm);
-	if (ret)
-		return ret;
-
 	ret = devm_pwmchip_add(&pdev->dev, chip);
 	if (ret)
 		dev_err(&pdev->dev, "Failed to add device: %d\n", ret);
diff --git a/drivers/pwm/pwm-loongson.c b/drivers/pwm/pwm-loongson.c
index f2fb35b7af2b3e..98c809bde2b30d 100644
--- a/drivers/pwm/pwm-loongson.c
+++ b/drivers/pwm/pwm-loongson.c
@@ -99,6 +99,7 @@ static void pwm_loongson_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 	struct pwm_loongson_ddata *ddata = to_pwm_loongson_ddata(chip);
 
 	val = pwm_loongson_readl(ddata, LOONGSON_PWM_REG_CTRL);
+	val |= LOONGSON_PWM_CTRL_REG_RST;
 	val &= ~LOONGSON_PWM_CTRL_REG_EN;
 	pwm_loongson_writel(ddata, val, LOONGSON_PWM_REG_CTRL);
 }
@@ -109,6 +110,7 @@ static int pwm_loongson_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	struct pwm_loongson_ddata *ddata = to_pwm_loongson_ddata(chip);
 
 	val = pwm_loongson_readl(ddata, LOONGSON_PWM_REG_CTRL);
+	val &= ~LOONGSON_PWM_CTRL_REG_RST;
 	val |= LOONGSON_PWM_CTRL_REG_EN;
 	pwm_loongson_writel(ddata, val, LOONGSON_PWM_REG_CTRL);
 
diff --git a/drivers/pwm/pwm-lp3943.c b/drivers/pwm/pwm-lp3943.c
index 10537e74be2887..e42af9007546e9 100644
--- a/drivers/pwm/pwm-lp3943.c
+++ b/drivers/pwm/pwm-lp3943.c
@@ -38,6 +38,9 @@ lp3943_pwm_request_map(struct lp3943_pwm *lp3943_pwm, int hwpwm)
 	struct lp3943_pwm_map *pwm_map = &lp3943_pwm->pwm_map[hwpwm];
 	int i, offset;
 
+	if (!pdata->pwms[hwpwm])
+		return ERR_PTR(-ENODEV);
+
 	pwm_map->output = pdata->pwms[hwpwm]->output;
 	pwm_map->num_outputs = pdata->pwms[hwpwm]->num_outputs;
 
diff --git a/drivers/pwm/pwm-mtk-disp.c b/drivers/pwm/pwm-mtk-disp.c
index b5409e15dac866..56d10cde5b4603 100644
--- a/drivers/pwm/pwm-mtk-disp.c
+++ b/drivers/pwm/pwm-mtk-disp.c
@@ -210,9 +210,9 @@ static int mtk_disp_pwm_get_state(struct pwm_chip *chip,
 	period = FIELD_GET(PWM_PERIOD_MASK, con1);
 	/*
 	 * period has 12 bits, clk_div 11 and NSEC_PER_SEC has 30,
-	 * so period * (clk_div + 1) * NSEC_PER_SEC doesn't overflow.
+	 * so (period + 1) * (clk_div + 1) * NSEC_PER_SEC doesn't overflow.
 	 */
-	state->period = DIV64_U64_ROUND_UP(period * (clk_div + 1) * NSEC_PER_SEC, rate);
+	state->period = DIV64_U64_ROUND_UP((period + 1) * (clk_div + 1) * NSEC_PER_SEC, rate);
 	high_width = FIELD_GET(PWM_HIGH_WIDTH_MASK, con1);
 	state->duty_cycle = DIV64_U64_ROUND_UP(high_width * (clk_div + 1) * NSEC_PER_SEC,
 					       rate);
diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c
index a2792b80364371..3374ed76f8d39d 100644
--- a/drivers/pwm/pwm-pca9685.c
+++ b/drivers/pwm/pwm-pca9685.c
@@ -54,7 +54,7 @@
 /*
  * The time value of one counter tick. Note that NSEC_PER_SEC is an integer
  * multiple of PCA9685_OSC_CLOCK_HZ, so there is no rounding involved and we're
- * not loosing precision due to the early division.
+ * not losing precision due to the early division.
  */
 #define PCA9685_QUANTUM_NS(_prescale)	((NSEC_PER_SEC / PCA9685_OSC_CLOCK_HZ) * (_prescale + 1))
 
diff --git a/drivers/pwm/pwm-renesas-tpu.c b/drivers/pwm/pwm-renesas-tpu.c
index 140fd0bcb93f32..e35f2e14777dd3 100644
--- a/drivers/pwm/pwm-renesas-tpu.c
+++ b/drivers/pwm/pwm-renesas-tpu.c
@@ -149,10 +149,14 @@ static int tpu_pwm_timer_start(struct tpu_pwm_device *tpd)
 
 	if (!tpd->timer_on) {
 		/* Wake up device and enable clock. */
-		pm_runtime_get_sync(&tpd->tpu->pdev->dev);
+		ret = pm_runtime_resume_and_get(&tpd->tpu->pdev->dev);
+		if (ret < 0)
+			return ret;
+
 		ret = clk_prepare_enable(tpd->tpu->clk);
 		if (ret) {
 			dev_err(&tpd->tpu->pdev->dev, "cannot enable clock\n");
+			pm_runtime_put(&tpd->tpu->pdev->dev);
 			return ret;
 		}
 		tpd->timer_on = true;
@@ -382,15 +386,21 @@ static int tpu_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 	return 0;
 }
 
-static void tpu_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
+static int tpu_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
 	struct tpu_device *tpu = to_tpu_device(chip);
 	struct tpu_pwm_device *tpd = &tpu->tpd[pwm->hwpwm];
+	int ret;
 
 	/* The timer must be running to modify the pin output configuration. */
-	tpu_pwm_timer_start(tpd);
+	ret = tpu_pwm_timer_start(tpd);
+	if (ret < 0)
+		return ret;
+
 	tpu_pwm_set_pin(tpd, TPU_PIN_INACTIVE);
 	tpu_pwm_timer_stop(tpd);
+
+	return 0;
 }
 
 static int tpu_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
@@ -401,7 +411,10 @@ static int tpu_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 
 	if (state->polarity != pwm->state.polarity) {
 		if (enabled) {
-			tpu_pwm_disable(chip, pwm);
+			err = tpu_pwm_disable(chip, pwm);
+			if (err)
+				return err;
+
 			enabled = false;
 		}
 
@@ -412,7 +425,7 @@ static int tpu_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
 
 	if (!state->enabled) {
 		if (enabled)
-			tpu_pwm_disable(chip, pwm);
+			return tpu_pwm_disable(chip, pwm);
 
 		return 0;
 	}
diff --git a/drivers/pwm/pwm-tiehrpwm.c b/drivers/pwm/pwm-tiehrpwm.c
index 3a53489f0b9825..e3407d0a1f7fe8 100644
--- a/drivers/pwm/pwm-tiehrpwm.c
+++ b/drivers/pwm/pwm-tiehrpwm.c
@@ -358,6 +358,9 @@ static void ehrpwm_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 
 	/* Disable clock on PWM disable */
 	pm_runtime_put_sync(pwmchip_parent(chip));
+
+	/* A disabled channel must not constrain the sibling's period */
+	pc->period_cycles[pwm->hwpwm] = 0;
 }
 
 static void ehrpwm_pwm_free(struct pwm_chip *chip, struct pwm_device *pwm)
diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c
index ad82c2108a567d..40d2953a9a7f9e 100644
--- a/drivers/rapidio/devices/rio_mport_cdev.c
+++ b/drivers/rapidio/devices/rio_mport_cdev.c
@@ -555,7 +555,7 @@ static void dma_req_free(struct kref *ref)
 			refcount);
 	struct mport_cdev_priv *priv = req->priv;
 
-	dma_unmap_sg(req->dmach->device->dev,
+	dma_unmap_sg(dmaengine_get_dma_device(req->dmach),
 		     req->sgt.sgl, req->sgt.nents, req->dir);
 	sg_free_table(&req->sgt);
 	if (req->page_list) {
@@ -916,7 +916,7 @@ rio_dma_transfer(struct file *filp, u32 transfer_mode,
 				xfer->offset, xfer->length);
 	}
 
-	nents = dma_map_sg(chan->device->dev,
+	nents = dma_map_sg(dmaengine_get_dma_device(chan),
 			   req->sgt.sgl, req->sgt.nents, dir);
 	if (nents == 0) {
 		rmcd_error("Failed to map SG list");
@@ -2167,11 +2167,12 @@ static void mport_mm_open(struct vm_area_struct *vma)
 static void mport_mm_close(struct vm_area_struct *vma)
 {
 	struct rio_mport_mapping *map = vma->vm_private_data;
+	struct mport_dev *md = map->md;
 
 	rmcd_debug(MMAP, "%pad", &map->phys_addr);
-	mutex_lock(&map->md->buf_mutex);
+	mutex_lock(&md->buf_mutex);
 	kref_put(&map->ref, mport_release_mapping);
-	mutex_unlock(&map->md->buf_mutex);
+	mutex_unlock(&md->buf_mutex);
 }
 
 static const struct vm_operations_struct vm_ops = {
diff --git a/drivers/regulator/88pm886-regulator.c b/drivers/regulator/88pm886-regulator.c
index 7328cd1cf2654c..1961a796898266 100644
--- a/drivers/regulator/88pm886-regulator.c
+++ b/drivers/regulator/88pm886-regulator.c
@@ -337,6 +337,25 @@ static struct regulator_desc pm886_regulators[] = {
 	},
 };
 
+static const struct linear_range pm886_vbus_volt_ranges[] = {
+	REGULATOR_LINEAR_RANGE(3750000, 0, 7, 250000)
+};
+
+static struct regulator_desc pm886_vbus_regulator = {
+	.name = "vbus",
+	.regulators_node = "regulators",
+	.of_match = "vbus",
+	.ops = &pm886_buck_ops,
+	.type = REGULATOR_VOLTAGE,
+	.n_voltages = 8,
+	.linear_ranges = pm886_vbus_volt_ranges,
+	.n_linear_ranges = ARRAY_SIZE(pm886_vbus_volt_ranges),
+	.vsel_reg = PM886_REG_BOOST_CONFIG1,
+	.vsel_mask = PM886_REG_BOOST_MASK,
+	.enable_reg = PM886_REG_BATTERY_CONFIG1,
+	.enable_mask = PM886_REG_VBUS_EN,
+};
+
 static int pm886_regulator_probe(struct platform_device *pdev)
 {
 	struct pm886_chip *chip = dev_get_drvdata(pdev->dev.parent);
@@ -369,6 +388,13 @@ static int pm886_regulator_probe(struct platform_device *pdev)
 					"Failed to register %s\n", rdesc->name);
 	}
 
+	rcfg.regmap = chip->regmap_battery;
+	rdesc = &pm886_vbus_regulator;
+	rdev = devm_regulator_register(dev, rdesc, &rcfg);
+	if (IS_ERR(rdev))
+		return dev_err_probe(dev, PTR_ERR(rdev),
+				      "Failed to register %s\n", rdesc->name);
+
 	return 0;
 }
 
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index 89789ac7a786c8..be6b72daa76f01 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -241,12 +241,12 @@ config REGULATOR_BD71815
 	  will be called bd71815-regulator.
 
 config REGULATOR_BD71828
-	tristate "ROHM BD71828, BD72720 and BD73900 Power Regulators"
+	tristate "ROHM BD71828, BD72720 and BD73[8/9]00 Power Regulators"
 	depends on MFD_ROHM_BD71828
 	select REGULATOR_ROHM
 	help
 	  This driver supports voltage regulators on ROHM BD71828,
-	  BD71879, BD72720 and BD73900 PMICs. This will enable
+	  BD71879, BD72720 and BD73[8/9]00 PMICs. This will enable
 	  support for the software controllable buck and LDO regulators.
 
 	  This driver can also be built as a module. If so, the module
@@ -864,12 +864,23 @@ config REGULATOR_MP8859
 	  module. The module will be named "mp8859".
 
 config REGULATOR_MP886X
-	tristate "MPS MP8869 regulator driver"
+	tristate "MPS MP8864/MP8867/MP8869 regulator driver"
 	depends on I2C && OF
 	select REGMAP_I2C
 	help
 	  This driver supports the MP8869 voltage regulator.
 
+config REGULATOR_MPQ4210
+	tristate "MPS MPQ4210 regulator driver"
+	depends on I2C && OF
+	select REGMAP_I2C
+	help
+	  This driver supports the MPQ4210 synchronous buck-boost controller.
+	  It supports get/set voltage, enable/disable and ramp rate selection
+	  through the regulator interface.
+	  Say M here if you want to include support for the regulator as a
+	  module. The module will be named "mpq4210".
+
 config REGULATOR_MPQ7920
 	tristate "Monolithic MPQ7920 PMIC"
 	depends on I2C && OF
@@ -1020,6 +1031,14 @@ config REGULATOR_MTK_DVFSRC
 	  of Mediatek. It allows for voting on regulator state
 	  between multiple users.
 
+config REGULATOR_NEX10000UB
+	tristate "Nexperia NEX10000UB Dual Output Power regulators"
+	depends on I2C && GPIOLIB
+	select REGMAP_I2C
+	help
+	  This driver supports the NEX10000UB single inductor - dual output
+	  power supply specifically designed for display panels.
+
 config REGULATOR_PALMAS
 	tristate "TI Palmas PMIC Regulators"
 	depends on MFD_PALMAS
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index 5a764cec8df834..6aabcc54382a2e 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -104,6 +104,7 @@ obj-$(CONFIG_REGULATOR_MCP16502) += mcp16502.o
 obj-$(CONFIG_REGULATOR_MP5416) += mp5416.o
 obj-$(CONFIG_REGULATOR_MP8859) += mp8859.o
 obj-$(CONFIG_REGULATOR_MP886X) += mp886x.o
+obj-$(CONFIG_REGULATOR_MPQ4210) += mpq4210.o
 obj-$(CONFIG_REGULATOR_MPQ7920) += mpq7920.o
 obj-$(CONFIG_REGULATOR_MT6311) += mt6311-regulator.o
 obj-$(CONFIG_REGULATOR_MT6315) += mt6315-regulator.o
@@ -120,6 +121,7 @@ obj-$(CONFIG_REGULATOR_MT6370) += mt6370-regulator.o
 obj-$(CONFIG_REGULATOR_MT6380)	+= mt6380-regulator.o
 obj-$(CONFIG_REGULATOR_MT6397)	+= mt6397-regulator.o
 obj-$(CONFIG_REGULATOR_MTK_DVFSRC) += mtk-dvfsrc-regulator.o
+obj-$(CONFIG_REGULATOR_NEX10000UB) += nex10000ub-regulator.o
 obj-$(CONFIG_REGULATOR_QCOM_LABIBB) += qcom-labibb-regulator.o
 obj-$(CONFIG_REGULATOR_QCOM_PM8008) += qcom-pm8008-regulator.o
 obj-$(CONFIG_REGULATOR_QCOM_REFGEN) += qcom-refgen-regulator.o
diff --git a/drivers/regulator/ab8500-ext.c b/drivers/regulator/ab8500-ext.c
index b9955aa4e0d1b0..c0657362a102f5 100644
--- a/drivers/regulator/ab8500-ext.c
+++ b/drivers/regulator/ab8500-ext.c
@@ -88,7 +88,7 @@ static struct regulator_init_data ab8500_ext_regulators[] = {
  * @update_reg: register to control on/off
  * @update_mask: mask to enable/disable and set mode of regulator
  * @update_val: bits holding the regulator current mode
- * @update_val_hp: bits to set EN pin active (LPn pin deactive)
+ * @update_val_hp: bits to set EN pin active (LPn pin deactivate)
  *                 normally this means high power mode
  * @update_val_lp: bits to set EN pin active and LPn pin active
  *                 normally this means low power mode
diff --git a/drivers/regulator/ab8500.c b/drivers/regulator/ab8500.c
index 3705c98b071395..3c1962ad85d497 100644
--- a/drivers/regulator/ab8500.c
+++ b/drivers/regulator/ab8500.c
@@ -9,15 +9,18 @@
  * AB8500 peripheral regulators
  *
  * AB8500 supports the following regulators:
- *   VAUX1/2/3, VINTCORE, VTVOUT, VUSB, VAUDIO, VAMIC1/2, VDMIC, VANA
+ *   VSMPS1/2/3, VARM, VAPE, VMOD, VAUX1/2/3, VINTCORE, VTVOUT,
+ *   VUSB, VAUDIO, VAMIC1/2, VDMIC, VANA
  *
  * AB8505 supports the following regulators:
- *   VAUX1/2/3/4/5/6, VINTCORE, VADC, VUSB, VAUDIO, VAMIC1/2, VDMIC, VANA
+ *   VSMPSA/B/C/M, VSAFE, VARM, VAUX1/2/3/4/5/6, VINTCORE,
+ *   VADC, VUSB, VAUDIO, VAMIC1/2, VDMIC, VANA
  */
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/err.h>
+#include <linux/mutex.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/abx500.h>
 #include <linux/mfd/abx500/ab8500.h>
@@ -39,6 +42,12 @@ enum ab8500_regulator_id {
 	AB8500_LDO_ANAMIC2,
 	AB8500_LDO_DMIC,
 	AB8500_LDO_ANA,
+	AB8500_BUCK_SMPS1,
+	AB8500_BUCK_SMPS2,
+	AB8500_BUCK_SMPS3,
+	AB8500_BUCK_ARM,
+	AB8500_BUCK_APE,
+	AB8500_BUCK_MOD,
 	AB8500_NUM_REGULATORS,
 };
 
@@ -57,6 +66,12 @@ enum ab8505_regulator_id {
 	AB8505_LDO_ANAMIC2,
 	AB8505_LDO_AUX8,
 	AB8505_LDO_ANA,
+	AB8505_BUCK_SMPSA,
+	AB8505_BUCK_SMPSB,
+	AB8505_BUCK_SAFE,
+	AB8505_BUCK_ARM,
+	AB8505_BUCK_SMPSC,
+	AB8505_BUCK_SMPSM,
 	AB8505_NUM_REGULATORS,
 };
 
@@ -162,6 +177,7 @@ struct ab8500_shared_mode {
  * @update_bank: bank to control on/off
  * @update_reg: register to control on/off
  * @update_mask: mask to enable/disable and set mode of regulator
+ * @enable_mask: optional mask for an enable bit separate from the mode bit
  * @update_val: bits holding the regulator current mode
  * @update_val_idle: bits to enable the regulator in idle (low power) mode
  * @update_val_normal: bits to enable the regulator in normal (high power) mode
@@ -171,8 +187,17 @@ struct ab8500_shared_mode {
  * @mode_val_idle: mode setting for low power
  * @mode_val_normal: mode setting for normal power
  * @voltage_bank: bank to control regulator voltage
- * @voltage_reg: register to control regulator voltage
+ * @voltage_reg: first register containing a selectable regulator voltage
  * @voltage_mask: mask to control regulator voltage
+ * @expand_register: additional register used to select an extra voltage
+ * @voltage_ctrl_bank: bank containing the voltage selector control
+ * @voltage_ctrl_reg: register containing the voltage selector control
+ * @voltage_ctrl_mask: mask selecting one of the first voltage registers
+ * @voltage_ext_ctrl_bank: bank containing the extended selector control
+ * @voltage_ext_ctrl_reg: register containing the extended selector control
+ * @voltage_ext_ctrl_mask: mask selecting one of the extended voltage registers
+ * @voltage_ext_reg: first extended voltage register
+ * @voltage_ext_regs: number of extended voltage registers
  */
 struct ab8500_regulator_info {
 	struct device		*dev;
@@ -182,6 +207,7 @@ struct ab8500_regulator_info {
 	u8 update_bank;
 	u8 update_reg;
 	u8 update_mask;
+	u8 enable_mask;
 	u8 update_val;
 	u8 update_val_idle;
 	u8 update_val_normal;
@@ -193,6 +219,20 @@ struct ab8500_regulator_info {
 	u8 voltage_bank;
 	u8 voltage_reg;
 	u8 voltage_mask;
+	struct {
+		u8 voltage_limit;
+		u8 voltage_bank;
+		u8 voltage_reg;
+		u8 voltage_mask;
+	} expand_register;
+	u8 voltage_ctrl_bank;
+	u8 voltage_ctrl_reg;
+	u8 voltage_ctrl_mask;
+	u8 voltage_ext_ctrl_bank;
+	u8 voltage_ext_ctrl_reg;
+	u8 voltage_ext_ctrl_mask;
+	u8 voltage_ext_reg;
+	u8 voltage_ext_regs;
 };
 
 /* voltage tables for the vauxn/vintcore supplies */
@@ -226,6 +266,18 @@ static const unsigned int ldo_vaux3_voltages[] = {
 	2910000,
 };
 
+static const unsigned int ldo_vaux3_ab8505_voltages[] = {
+	1200000,
+	1500000,
+	1800000,
+	2100000,
+	2500000,
+	2750000,
+	2790000,
+	2910000,
+	3050000,
+};
+
 static const unsigned int ldo_vaux56_voltages[] = {
 	1800000,
 	1050000,
@@ -237,14 +289,13 @@ static const unsigned int ldo_vaux56_voltages[] = {
 	2790000,
 };
 
-static const unsigned int ldo_vintcore_voltages[] = {
-	1200000,
-	1225000,
-	1250000,
-	1275000,
-	1300000,
-	1325000,
-	1350000,
+static const struct linear_range ldo_vintcore_ranges[] = {
+	REGULATOR_LINEAR_RANGE(1200000, 0, 6, 25000),
+};
+
+static const struct linear_range ldo_vintcore_ab8505_ranges[] = {
+	REGULATOR_LINEAR_RANGE(1200000, 0, 6, 25000),
+	REGULATOR_LINEAR_RANGE(1350000, 7, 7, 0),
 };
 
 static const unsigned int fixed_1200000_voltage[] = {
@@ -264,25 +315,54 @@ static const unsigned int fixed_2050000_voltage[] = {
 };
 
 static const unsigned int ldo_vana_voltages[] = {
+	1200000,
 	1050000,
 	1075000,
 	1100000,
 	1125000,
 	1150000,
 	1175000,
-	1200000,
 	1225000,
 };
 
-static const unsigned int ldo_vaudio_voltages[] = {
-	2000000,
-	2100000,
-	2200000,
-	2300000,
-	2400000,
-	2500000,
-	2600000,
-	2600000,	/* Duplicated in Vaudio and IsoUicc Control register. */
+static const struct linear_range ldo_vaudio_ranges[] = {
+	REGULATOR_LINEAR_RANGE(2000000, 0, 6, 100000),
+	/* Duplicated in Vaudio and IsoUicc Control register. */
+	REGULATOR_LINEAR_RANGE(2600000, 7, 7, 0),
+};
+
+/*
+ * AB8505 buck ranges except VARM are selected by OTP.  The supported
+ * platforms use the AB8500-compatible profiles for VSMPSA/B and the low
+ * profiles for VSAFE, VSMPSC and VSMPSM.
+ */
+static const struct linear_range buck_low_voltages[] = {
+	REGULATOR_LINEAR_RANGE(700000, 0, 53, 12500),
+	REGULATOR_LINEAR_RANGE(1362500, 54, 63, 0),
+};
+
+/* VSMPS3 and VSAFE have a 7-bit selector, but the same low range. */
+static const struct linear_range buck_low_7bit_voltages[] = {
+	REGULATOR_LINEAR_RANGE(700000, 0, 53, 12500),
+	REGULATOR_LINEAR_RANGE(1362500, 54, 127, 0),
+};
+
+/* AB8505 VARM uses a separate 0.6 V to 1.39375 V selector range. */
+static const struct linear_range ab8505_buck_arm_voltages[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0, 127, 6250),
+};
+
+/* VSMPS1 and the VSMPSA AB8500-compatible profile clamp to this range. */
+static const struct linear_range buck_smps1_voltages[] = {
+	REGULATOR_LINEAR_RANGE(1100000, 0, 32, 0),
+	REGULATOR_LINEAR_RANGE(1112500, 33, 48, 12500),
+	REGULATOR_LINEAR_RANGE(1300000, 49, 63, 0),
+};
+
+/* VSMPS2 and the VSMPSB AB8500-compatible profile clamp to this range. */
+static const struct linear_range buck_smps2_voltages[] = {
+	REGULATOR_LINEAR_RANGE(1800000, 0, 57, 0),
+	REGULATOR_LINEAR_RANGE(1812500, 58, 63, 12500),
 };
 
 static DEFINE_MUTEX(shared_mode_mutex);
@@ -343,7 +423,7 @@ static int ab8500_regulator_disable(struct regulator_dev *rdev)
 	return ret;
 }
 
-static int ab8500_regulator_is_enabled(struct regulator_dev *rdev)
+static int ab8500_regulator_get_enable_value(struct regulator_dev *rdev)
 {
 	int ret;
 	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
@@ -368,10 +448,65 @@ static int ab8500_regulator_is_enabled(struct regulator_dev *rdev)
 		info->desc.name, info->update_bank, info->update_reg,
 		info->update_mask, regval);
 
-	if (regval & info->update_mask)
-		return 1;
+	return regval & info->update_mask;
+}
+
+static int ab8500_regulator_is_enabled(struct regulator_dev *rdev)
+{
+	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
+	u8 enable_mask;
+	int ret;
+
+	ret = ab8500_regulator_get_enable_value(rdev);
+	if (ret < 0)
+		return ret;
+
+	enable_mask = info->enable_mask ? info->enable_mask : info->update_mask;
+
+	return !!(ret & enable_mask);
+}
+
+static int ab8500_buck_enable(struct regulator_dev *rdev)
+{
+	int ret;
+
+	/* Keep an OTP-selected hardware or low-power mode intact. */
+	ret = ab8500_regulator_is_enabled(rdev);
+	if (ret)
+		return ret < 0 ? ret : 0;
+
+	return ab8500_regulator_enable(rdev);
+}
+
+static int ab8500_buck_init(struct regulator_dev *rdev,
+			    struct regulator_config *config)
+{
+	struct ab8500_regulator_info *info = config->driver_data;
+	int ret;
+
+	ret = ab8500_regulator_get_enable_value(rdev);
+	if (ret <= 0)
+		return ret;
+
+	/* Report forced LP accurately; HP and hardware control are normal mode. */
+	if (ret == info->update_val_idle)
+		info->update_val = info->update_val_idle;
 	else
-		return 0;
+		info->update_val = info->update_val_normal;
+
+	/*
+	 * The SMPS enable state is selected by OTP.  An enabled rail may
+	 * supply discrete board components which are not represented as
+	 * regulator consumers, so keep it out of the unused-regulator sweep.
+	 */
+	rdev->constraints->boot_on = true;
+	rdev->constraints->always_on = true;
+	rdev->constraints->valid_ops_mask &= ~REGULATOR_CHANGE_STATUS;
+
+	dev_dbg(config->dev, "%s: preserving OTP-enabled state\n",
+		info->desc.name);
+
+	return 0;
 }
 
 static unsigned int ab8500_regulator_get_optimum_mode(
@@ -398,7 +533,7 @@ static unsigned int ab8500_regulator_get_optimum_mode(
 static int ab8500_regulator_set_mode(struct regulator_dev *rdev,
 				     unsigned int mode)
 {
-	int ret = 0;
+	int enabled, ret = 0;
 	u8 bank, reg, mask, val;
 	bool lp_mode_req = false;
 	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
@@ -408,6 +543,8 @@ static int ab8500_regulator_set_mode(struct regulator_dev *rdev,
 		return -EINVAL;
 	}
 
+	guard(mutex)(&shared_mode_mutex);
+
 	if (info->mode_mask) {
 		bank = info->mode_bank;
 		reg = info->mode_reg;
@@ -418,9 +555,6 @@ static int ab8500_regulator_set_mode(struct regulator_dev *rdev,
 		mask = info->update_mask;
 	}
 
-	if (info->shared_mode)
-		mutex_lock(&shared_mode_mutex);
-
 	switch (mode) {
 	case REGULATOR_MODE_NORMAL:
 		if (info->shared_mode)
@@ -439,7 +573,7 @@ static int ab8500_regulator_set_mode(struct regulator_dev *rdev,
 			if (!shared_regulator->shared_mode->lp_mode_req) {
 				/* Other regulator prevent LP mode */
 				info->shared_mode->lp_mode_req = true;
-				goto out_unlock;
+				return 0;
 			}
 
 			lp_mode_req = true;
@@ -451,17 +585,24 @@ static int ab8500_regulator_set_mode(struct regulator_dev *rdev,
 			val = info->update_val_idle;
 		break;
 	default:
-		ret = -EINVAL;
-		goto out_unlock;
+		return -EINVAL;
+	}
+
+	if (info->mode_mask) {
+		enabled = 1;
+	} else {
+		enabled = ab8500_regulator_is_enabled(rdev);
+		if (enabled < 0)
+			return enabled;
 	}
 
-	if (info->mode_mask || ab8500_regulator_is_enabled(rdev)) {
+	if (enabled) {
 		ret = abx500_mask_and_set_register_interruptible(info->dev,
 			bank, reg, mask, val);
 		if (ret < 0) {
 			dev_err(rdev_get_dev(rdev),
 				"couldn't set regulator mode\n");
-			goto out_unlock;
+			return ret;
 		}
 
 		dev_vdbg(rdev_get_dev(rdev),
@@ -477,10 +618,6 @@ static int ab8500_regulator_set_mode(struct regulator_dev *rdev,
 	if (info->shared_mode)
 		info->shared_mode->lp_mode_req = lp_mode_req;
 
-out_unlock:
-	if (info->shared_mode)
-		mutex_unlock(&shared_mode_mutex);
-
 	return ret;
 }
 
@@ -530,11 +667,67 @@ static unsigned int ab8500_regulator_get_mode(struct regulator_dev *rdev)
 	return ret;
 }
 
+static int ab8500_regulator_get_voltage_reg(struct regulator_dev *rdev,
+					    u8 *voltage_reg)
+{
+	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
+	u8 regval;
+	unsigned int selector;
+	int ret;
+
+	if (info->voltage_ext_ctrl_mask) {
+		ret = abx500_get_register_interruptible(info->dev,
+							info->voltage_ext_ctrl_bank,
+							info->voltage_ext_ctrl_reg, &regval);
+		if (ret < 0)
+			return ret;
+
+		selector = (regval & info->voltage_ext_ctrl_mask) >>
+			(ffs(info->voltage_ext_ctrl_mask) - 1);
+		if (selector) {
+			selector = min_t(unsigned int, selector,
+					 info->voltage_ext_regs);
+			*voltage_reg = info->voltage_ext_reg + selector - 1;
+			return 0;
+		}
+	}
+
+	if (!info->voltage_ctrl_mask) {
+		*voltage_reg = info->voltage_reg;
+		return 0;
+	}
+
+	ret = abx500_get_register_interruptible(info->dev,
+						info->voltage_ctrl_bank,
+						info->voltage_ctrl_reg, &regval);
+	if (ret < 0)
+		return ret;
+
+	/* The three hardware selector layouts all use consecutive registers. */
+	switch (info->voltage_ctrl_mask) {
+	case 0x0c:
+		selector = min((unsigned int)((regval & 0x0c) >> 2), 2U);
+		break;
+	case 0x24:
+		selector = regval & BIT(5) ? 2 : !!(regval & BIT(2));
+		break;
+	case 0x04:
+		selector = !!(regval & BIT(2));
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	*voltage_reg = info->voltage_reg + selector;
+
+	return 0;
+}
+
 static int ab8500_regulator_get_voltage_sel(struct regulator_dev *rdev)
 {
 	int ret, voltage_shift;
 	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
-	u8 regval;
+	u8 regval, voltage_reg;
 
 	if (info == NULL) {
 		dev_err(rdev_get_dev(rdev), "regulator info null pointer\n");
@@ -543,8 +736,15 @@ static int ab8500_regulator_get_voltage_sel(struct regulator_dev *rdev)
 
 	voltage_shift = ffs(info->voltage_mask) - 1;
 
+	ret = ab8500_regulator_get_voltage_reg(rdev, &voltage_reg);
+	if (ret < 0) {
+		dev_err(rdev_get_dev(rdev),
+			"couldn't read voltage selector control\n");
+		return ret;
+	}
+
 	ret = abx500_get_register_interruptible(info->dev,
-			info->voltage_bank, info->voltage_reg, &regval);
+			info->voltage_bank, voltage_reg, &regval);
 	if (ret < 0) {
 		dev_err(rdev_get_dev(rdev),
 			"couldn't read voltage reg for regulator\n");
@@ -555,7 +755,7 @@ static int ab8500_regulator_get_voltage_sel(struct regulator_dev *rdev)
 		"%s-get_voltage (bank, reg, mask, shift, value): "
 		"0x%x, 0x%x, 0x%x, 0x%x, 0x%x\n",
 		info->desc.name, info->voltage_bank,
-		info->voltage_reg, info->voltage_mask,
+		voltage_reg, info->voltage_mask,
 		voltage_shift, regval);
 
 	return (regval & info->voltage_mask) >> voltage_shift;
@@ -566,7 +766,7 @@ static int ab8500_regulator_set_voltage_sel(struct regulator_dev *rdev,
 {
 	int ret, voltage_shift;
 	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
-	u8 regval;
+	u8 regval, voltage_reg;
 
 	if (info == NULL) {
 		dev_err(rdev_get_dev(rdev), "regulator info null pointer\n");
@@ -575,10 +775,17 @@ static int ab8500_regulator_set_voltage_sel(struct regulator_dev *rdev,
 
 	voltage_shift = ffs(info->voltage_mask) - 1;
 
+	ret = ab8500_regulator_get_voltage_reg(rdev, &voltage_reg);
+	if (ret < 0) {
+		dev_err(rdev_get_dev(rdev),
+			"couldn't read voltage selector control\n");
+		return ret;
+	}
+
 	/* set the registers for the request */
 	regval = (u8)selector << voltage_shift;
 	ret = abx500_mask_and_set_register_interruptible(info->dev,
-			info->voltage_bank, info->voltage_reg,
+			info->voltage_bank, voltage_reg,
 			info->voltage_mask, regval);
 	if (ret < 0)
 		dev_err(rdev_get_dev(rdev),
@@ -587,12 +794,70 @@ static int ab8500_regulator_set_voltage_sel(struct regulator_dev *rdev,
 	dev_vdbg(rdev_get_dev(rdev),
 		"%s-set_voltage (bank, reg, mask, value): 0x%x, 0x%x, 0x%x,"
 		" 0x%x\n",
-		info->desc.name, info->voltage_bank, info->voltage_reg,
+		info->desc.name, info->voltage_bank, voltage_reg,
 		info->voltage_mask, regval);
 
 	return ret;
 }
 
+static int ab8500_regulator_get_voltage_sel_expand(struct regulator_dev *rdev)
+{
+	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
+	u8 regval;
+	int ret;
+
+	if (!info)
+		return -EINVAL;
+
+	ret = abx500_get_register_interruptible(info->dev,
+			info->expand_register.voltage_bank,
+			info->expand_register.voltage_reg, &regval);
+	if (ret < 0) {
+		dev_err(rdev_get_dev(rdev),
+			"couldn't read voltage expand reg for regulator\n");
+		return ret;
+	}
+
+	if (regval & info->expand_register.voltage_mask)
+		return info->expand_register.voltage_limit;
+
+	return ab8500_regulator_get_voltage_sel(rdev);
+}
+
+static int ab8500_regulator_set_voltage_sel_expand(struct regulator_dev *rdev,
+						   unsigned int selector)
+{
+	struct ab8500_regulator_info *info = rdev_get_drvdata(rdev);
+	u8 regval;
+	int ret;
+
+	if (!info)
+		return -EINVAL;
+
+	if (selector > info->expand_register.voltage_limit)
+		return -EINVAL;
+
+	if (selector < info->expand_register.voltage_limit) {
+		ret = ab8500_regulator_set_voltage_sel(rdev, selector);
+		if (ret < 0)
+			return ret;
+
+		regval = 0;
+	} else {
+		regval = info->expand_register.voltage_mask;
+	}
+
+	ret = abx500_mask_and_set_register_interruptible(info->dev,
+			info->expand_register.voltage_bank,
+			info->expand_register.voltage_reg,
+			info->expand_register.voltage_mask, regval);
+	if (ret < 0)
+		dev_err(rdev_get_dev(rdev),
+			"couldn't set voltage expand reg for regulator\n");
+
+	return ret;
+}
+
 static const struct regulator_ops ab8500_regulator_volt_mode_ops = {
 	.enable			= ab8500_regulator_enable,
 	.disable		= ab8500_regulator_disable,
@@ -605,13 +870,59 @@ static const struct regulator_ops ab8500_regulator_volt_mode_ops = {
 	.list_voltage		= regulator_list_voltage_table,
 };
 
-static const struct regulator_ops ab8500_regulator_volt_ops = {
+static const struct regulator_ops ab8500_regulator_volt_mode_expand_ops = {
+	.enable			= ab8500_regulator_enable,
+	.disable		= ab8500_regulator_disable,
+	.is_enabled		= ab8500_regulator_is_enabled,
+	.get_optimum_mode	= ab8500_regulator_get_optimum_mode,
+	.set_mode		= ab8500_regulator_set_mode,
+	.get_mode		= ab8500_regulator_get_mode,
+	.get_voltage_sel	= ab8500_regulator_get_voltage_sel_expand,
+	.set_voltage_sel	= ab8500_regulator_set_voltage_sel_expand,
+	.list_voltage		= regulator_list_voltage_table,
+};
+
+static const struct regulator_ops ab8500_regulator_linear_range_volt_mode_ops = {
+	.enable			= ab8500_regulator_enable,
+	.disable		= ab8500_regulator_disable,
+	.is_enabled		= ab8500_regulator_is_enabled,
+	.get_optimum_mode	= ab8500_regulator_get_optimum_mode,
+	.set_mode		= ab8500_regulator_set_mode,
+	.get_mode		= ab8500_regulator_get_mode,
+	.get_voltage_sel	= ab8500_regulator_get_voltage_sel,
+	.set_voltage_sel	= ab8500_regulator_set_voltage_sel,
+	.list_voltage		= regulator_list_voltage_linear_range,
+	.map_voltage		= regulator_map_voltage_linear_range,
+};
+
+static const struct regulator_ops ab8500_regulator_linear_range_volt_ops = {
 	.enable		= ab8500_regulator_enable,
 	.disable	= ab8500_regulator_disable,
 	.is_enabled	= ab8500_regulator_is_enabled,
 	.get_voltage_sel = ab8500_regulator_get_voltage_sel,
 	.set_voltage_sel = ab8500_regulator_set_voltage_sel,
-	.list_voltage	= regulator_list_voltage_table,
+	.list_voltage	= regulator_list_voltage_linear_range,
+	.map_voltage	= regulator_map_voltage_linear_range,
+};
+
+static const struct regulator_ops ab8500_buck_ops = {
+	.enable			= ab8500_buck_enable,
+	.disable		= ab8500_regulator_disable,
+	.is_enabled		= ab8500_regulator_is_enabled,
+	.get_optimum_mode	= ab8500_regulator_get_optimum_mode,
+	.set_mode		= ab8500_regulator_set_mode,
+	.get_mode		= ab8500_regulator_get_mode,
+	.get_voltage_sel	= ab8500_regulator_get_voltage_sel,
+	.set_voltage_sel	= ab8500_regulator_set_voltage_sel,
+	.list_voltage		= regulator_list_voltage_linear_range,
+	.map_voltage		= regulator_map_voltage_linear_range,
+};
+
+static const struct regulator_ops ab8500_buck_voltage_ops = {
+	.get_voltage_sel	= ab8500_regulator_get_voltage_sel,
+	.set_voltage_sel	= ab8500_regulator_set_voltage_sel,
+	.list_voltage		= regulator_list_voltage_linear_range,
+	.map_voltage		= regulator_map_voltage_linear_range,
 };
 
 static const struct regulator_ops ab8500_regulator_mode_ops = {
@@ -721,18 +1032,20 @@ static struct ab8500_regulator_info
 	[AB8500_LDO_INTCORE] = {
 		.desc = {
 			.name		= "LDO-INTCORE",
-			.ops		= &ab8500_regulator_volt_mode_ops,
+			.ops		= &ab8500_regulator_linear_range_volt_mode_ops,
 			.type		= REGULATOR_VOLTAGE,
 			.id		= AB8500_LDO_INTCORE,
 			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vintcore_voltages),
-			.volt_table	= ldo_vintcore_voltages,
+			.n_voltages	= 7,
+			.linear_ranges	= ldo_vintcore_ranges,
+			.n_linear_ranges = ARRAY_SIZE(ldo_vintcore_ranges),
 			.enable_time	= 750,
 		},
 		.load_lp_uA		= 5000,
 		.update_bank		= 0x03,
 		.update_reg		= 0x80,
 		.update_mask		= 0x44,
+		.enable_mask		= 0x04,
 		.update_val		= 0x44,
 		.update_val_idle	= 0x44,
 		.update_val_normal	= 0x04,
@@ -761,6 +1074,7 @@ static struct ab8500_regulator_info
 		.update_bank		= 0x03,
 		.update_reg		= 0x80,
 		.update_mask		= 0x82,
+		.enable_mask		= 0x02,
 		.update_val		= 0x02,
 		.update_val_idle	= 0x82,
 		.update_val_normal	= 0x02,
@@ -852,6 +1166,140 @@ static struct ab8500_regulator_info
 		.update_val_idle	= 0x0c,
 		.update_val_normal	= 0x04,
 	},
+
+	/* Buck converters */
+	[AB8500_BUCK_SMPS1] = {
+		.desc = {
+			.name		= "BUCK-SMPS1",
+			.ops		= &ab8500_buck_ops,
+			.init_cb	= ab8500_buck_init,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8500_BUCK_SMPS1,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_smps1_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_smps1_voltages),
+		},
+		.load_lp_uA		= 20000,
+		.update_bank		= 0x04,
+		.update_reg		= 0x03,
+		.update_mask		= 0x03,
+		.update_val		= 0x01,
+		.update_val_idle	= 0x03,
+		.update_val_normal	= 0x01,
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x13,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x03,
+		.voltage_ctrl_mask	= 0x0c,
+	},
+	[AB8500_BUCK_SMPS2] = {
+		.desc = {
+			.name		= "BUCK-SMPS2",
+			.ops		= &ab8500_buck_ops,
+			.init_cb	= ab8500_buck_init,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8500_BUCK_SMPS2,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_smps2_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_smps2_voltages),
+		},
+		.load_lp_uA		= 20000,
+		.update_bank		= 0x04,
+		.update_reg		= 0x04,
+		.update_mask		= 0x03,
+		.update_val		= 0x01,
+		.update_val_idle	= 0x03,
+		.update_val_normal	= 0x01,
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x17,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x04,
+		.voltage_ctrl_mask	= 0x0c,
+	},
+	[AB8500_BUCK_SMPS3] = {
+		.desc = {
+			.name		= "BUCK-SMPS3",
+			.ops		= &ab8500_buck_ops,
+			.init_cb	= ab8500_buck_init,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8500_BUCK_SMPS3,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 128,
+			.linear_ranges	= buck_low_7bit_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_low_7bit_voltages),
+		},
+		.load_lp_uA		= 50000,
+		.update_bank		= 0x04,
+		.update_reg		= 0x05,
+		.update_mask		= 0x03,
+		.update_val		= 0x01,
+		.update_val_idle	= 0x03,
+		.update_val_normal	= 0x01,
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x1b,
+		.voltage_mask		= 0x7f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x05,
+		.voltage_ctrl_mask	= 0x0c,
+	},
+	[AB8500_BUCK_ARM] = {
+		.desc = {
+			.name		= "BUCK-ARM",
+			.ops		= &ab8500_buck_voltage_ops,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8500_BUCK_ARM,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_low_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_low_voltages),
+		},
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x0b,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x00,
+		.voltage_ctrl_mask	= 0x0c,
+	},
+	[AB8500_BUCK_APE] = {
+		.desc = {
+			.name		= "BUCK-APE",
+			.ops		= &ab8500_buck_voltage_ops,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8500_BUCK_APE,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_low_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_low_voltages),
+		},
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x0e,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x02,
+		.voltage_ctrl_mask	= 0x24,
+	},
+	[AB8500_BUCK_MOD] = {
+		.desc = {
+			.name		= "BUCK-MOD",
+			.ops		= &ab8500_buck_voltage_ops,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8500_BUCK_MOD,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_low_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_low_voltages),
+		},
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x41,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x40,
+		.voltage_ctrl_mask	= 0x04,
+	},
 };
 
 /* AB8505 regulator information */
@@ -908,12 +1356,12 @@ static struct ab8500_regulator_info
 	[AB8505_LDO_AUX3] = {
 		.desc = {
 			.name		= "LDO-AUX3",
-			.ops		= &ab8500_regulator_volt_mode_ops,
+			.ops		= &ab8500_regulator_volt_mode_expand_ops,
 			.type		= REGULATOR_VOLTAGE,
 			.id		= AB8505_LDO_AUX3,
 			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vaux3_voltages),
-			.volt_table	= ldo_vaux3_voltages,
+			.n_voltages	= ARRAY_SIZE(ldo_vaux3_ab8505_voltages),
+			.volt_table	= ldo_vaux3_ab8505_voltages,
 		},
 		.load_lp_uA		= 5000,
 		.update_bank		= 0x04,
@@ -925,6 +1373,12 @@ static struct ab8500_regulator_info
 		.voltage_bank		= 0x04,
 		.voltage_reg		= 0x21,
 		.voltage_mask		= 0x07,
+		.expand_register = {
+			.voltage_limit	= 8,
+			.voltage_bank	= 0x04,
+			.voltage_reg	= 0x01,
+			.voltage_mask	= 0x10,
+		},
 	},
 	[AB8505_LDO_AUX4] = {
 		.desc = {
@@ -964,6 +1418,7 @@ static struct ab8500_regulator_info
 		.update_bank		= 0x01,
 		.update_reg		= 0x55,
 		.update_mask		= 0x18,
+		.enable_mask		= 0x10,
 		.update_val		= 0x10,
 		.update_val_idle	= 0x18,
 		.update_val_normal	= 0x10,
@@ -986,6 +1441,7 @@ static struct ab8500_regulator_info
 		.update_bank		= 0x01,
 		.update_reg		= 0x56,
 		.update_mask		= 0x18,
+		.enable_mask		= 0x10,
 		.update_val		= 0x10,
 		.update_val_idle	= 0x18,
 		.update_val_normal	= 0x10,
@@ -996,17 +1452,19 @@ static struct ab8500_regulator_info
 	[AB8505_LDO_INTCORE] = {
 		.desc = {
 			.name		= "LDO-INTCORE",
-			.ops		= &ab8500_regulator_volt_mode_ops,
+			.ops		= &ab8500_regulator_linear_range_volt_mode_ops,
 			.type		= REGULATOR_VOLTAGE,
 			.id		= AB8505_LDO_INTCORE,
 			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vintcore_voltages),
-			.volt_table	= ldo_vintcore_voltages,
+			.n_voltages	= 8,
+			.linear_ranges	= ldo_vintcore_ab8505_ranges,
+			.n_linear_ranges = ARRAY_SIZE(ldo_vintcore_ab8505_ranges),
 		},
 		.load_lp_uA		= 5000,
 		.update_bank		= 0x03,
 		.update_reg		= 0x80,
 		.update_mask		= 0x44,
+		.enable_mask		= 0x04,
 		.update_val		= 0x04,
 		.update_val_idle	= 0x44,
 		.update_val_normal	= 0x04,
@@ -1035,6 +1493,7 @@ static struct ab8500_regulator_info
 		.update_bank		= 0x03,
 		.update_reg		= 0x80,
 		.update_mask		= 0x82,
+		.enable_mask		= 0x02,
 		.update_val		= 0x02,
 		.update_val_idle	= 0x82,
 		.update_val_normal	= 0x02,
@@ -1042,12 +1501,13 @@ static struct ab8500_regulator_info
 	[AB8505_LDO_AUDIO] = {
 		.desc = {
 			.name		= "LDO-AUDIO",
-			.ops		= &ab8500_regulator_volt_ops,
+			.ops		= &ab8500_regulator_linear_range_volt_ops,
 			.type		= REGULATOR_VOLTAGE,
 			.id		= AB8505_LDO_AUDIO,
 			.owner		= THIS_MODULE,
-			.n_voltages	= ARRAY_SIZE(ldo_vaudio_voltages),
-			.volt_table	= ldo_vaudio_voltages,
+			.n_voltages	= 8,
+			.linear_ranges	= ldo_vaudio_ranges,
+			.n_linear_ranges = ARRAY_SIZE(ldo_vaudio_ranges),
 		},
 		.update_bank		= 0x03,
 		.update_reg		= 0x83,
@@ -1138,6 +1598,155 @@ static struct ab8500_regulator_info
 		.voltage_reg		= 0x29,
 		.voltage_mask		= 0x7,
 	},
+
+	/* Buck converters */
+	[AB8505_BUCK_SMPSA] = {
+		.desc = {
+			.name		= "BUCK-SMPSA",
+			.ops		= &ab8500_buck_ops,
+			.init_cb	= ab8500_buck_init,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8505_BUCK_SMPSA,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_smps1_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_smps1_voltages),
+		},
+		.load_lp_uA		= 20000,
+		.update_bank		= 0x04,
+		.update_reg		= 0x03,
+		.update_mask		= 0x03,
+		.update_val		= 0x01,
+		.update_val_idle	= 0x03,
+		.update_val_normal	= 0x01,
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x13,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x03,
+		.voltage_ctrl_mask	= 0x0c,
+	},
+	[AB8505_BUCK_SMPSB] = {
+		.desc = {
+			.name		= "BUCK-SMPSB",
+			.ops		= &ab8500_buck_ops,
+			.init_cb	= ab8500_buck_init,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8505_BUCK_SMPSB,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_smps2_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_smps2_voltages),
+		},
+		.load_lp_uA		= 20000,
+		.update_bank		= 0x04,
+		.update_reg		= 0x04,
+		.update_mask		= 0x03,
+		.update_val		= 0x01,
+		.update_val_idle	= 0x03,
+		.update_val_normal	= 0x01,
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x17,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x04,
+		.voltage_ctrl_mask	= 0x0c,
+	},
+	[AB8505_BUCK_SAFE] = {
+		.desc = {
+			.name		= "BUCK-SAFE",
+			.ops		= &ab8500_buck_ops,
+			.init_cb	= ab8500_buck_init,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8505_BUCK_SAFE,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 128,
+			.linear_ranges	= buck_low_7bit_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_low_7bit_voltages),
+		},
+		.load_lp_uA		= 50000,
+		.update_bank		= 0x04,
+		.update_reg		= 0x05,
+		.update_mask		= 0x03,
+		.update_val		= 0x01,
+		.update_val_idle	= 0x03,
+		.update_val_normal	= 0x01,
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x1b,
+		.voltage_mask		= 0x7f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x05,
+		.voltage_ctrl_mask	= 0x0c,
+	},
+	[AB8505_BUCK_ARM] = {
+		.desc = {
+			.name		= "BUCK-ARM",
+			.ops		= &ab8500_buck_voltage_ops,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8505_BUCK_ARM,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 128,
+			.linear_ranges	= ab8505_buck_arm_voltages,
+			.n_linear_ranges = ARRAY_SIZE(ab8505_buck_arm_voltages),
+		},
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x0b,
+		.voltage_mask		= 0x7f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x00,
+		.voltage_ctrl_mask	= 0x0c,
+		.voltage_ext_ctrl_bank = 0x04,
+		.voltage_ext_ctrl_reg	= 0x28,
+		.voltage_ext_ctrl_mask = 0x07,
+		.voltage_ext_reg	= 0x24,
+		.voltage_ext_regs	= 4,
+	},
+	[AB8505_BUCK_SMPSC] = {
+		.desc = {
+			.name		= "BUCK-SMPSC",
+			.ops		= &ab8500_buck_voltage_ops,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8505_BUCK_SMPSC,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_low_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_low_voltages),
+		},
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x0e,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x02,
+		.voltage_ctrl_mask	= 0x24,
+		.voltage_ext_ctrl_bank = 0x04,
+		.voltage_ext_ctrl_reg	= 0x2a,
+		.voltage_ext_ctrl_mask = 0x03,
+		.voltage_ext_reg	= 0x2b,
+		.voltage_ext_regs	= 2,
+	},
+	[AB8505_BUCK_SMPSM] = {
+		.desc = {
+			.name		= "BUCK-SMPSM",
+			.ops		= &ab8500_buck_voltage_ops,
+			.type		= REGULATOR_VOLTAGE,
+			.id		= AB8505_BUCK_SMPSM,
+			.owner		= THIS_MODULE,
+			.n_voltages	= 64,
+			.linear_ranges	= buck_low_voltages,
+			.n_linear_ranges = ARRAY_SIZE(buck_low_voltages),
+		},
+		.voltage_bank		= 0x04,
+		.voltage_reg		= 0x41,
+		.voltage_mask		= 0x3f,
+		.voltage_ctrl_bank	= 0x04,
+		.voltage_ctrl_reg	= 0x40,
+		.voltage_ctrl_mask	= 0x04,
+		.voltage_ext_ctrl_bank = 0x04,
+		.voltage_ext_ctrl_reg	= 0x47,
+		.voltage_ext_ctrl_mask = 0x03,
+		.voltage_ext_reg	= 0x45,
+		.voltage_ext_regs	= 2,
+	},
 };
 
 static struct ab8500_shared_mode ldo_anamic1_shared = {
@@ -1613,6 +2222,12 @@ static struct of_regulator_match ab8500_regulator_match[] = {
 	{ .name	= "ab8500_ldo_anamic2", .driver_data = (void *) AB8500_LDO_ANAMIC2, },
 	{ .name	= "ab8500_ldo_dmic",    .driver_data = (void *) AB8500_LDO_DMIC, },
 	{ .name	= "ab8500_ldo_ana",     .driver_data = (void *) AB8500_LDO_ANA, },
+	{ .name = "ab8500_buck_smps1",  .driver_data = (void *)AB8500_BUCK_SMPS1, },
+	{ .name = "ab8500_buck_smps2",  .driver_data = (void *)AB8500_BUCK_SMPS2, },
+	{ .name = "ab8500_buck_smps3",  .driver_data = (void *)AB8500_BUCK_SMPS3, },
+	{ .name = "ab8500_buck_arm",    .driver_data = (void *)AB8500_BUCK_ARM, },
+	{ .name = "ab8500_buck_ape",    .driver_data = (void *)AB8500_BUCK_APE, },
+	{ .name = "ab8500_buck_mod",    .driver_data = (void *)AB8500_BUCK_MOD, },
 };
 
 static struct of_regulator_match ab8505_regulator_match[] = {
@@ -1629,6 +2244,12 @@ static struct of_regulator_match ab8505_regulator_match[] = {
 	{ .name	= "ab8500_ldo_anamic2", .driver_data = (void *) AB8505_LDO_ANAMIC2, },
 	{ .name	= "ab8500_ldo_aux8",    .driver_data = (void *) AB8505_LDO_AUX8, },
 	{ .name	= "ab8500_ldo_ana",     .driver_data = (void *) AB8505_LDO_ANA, },
+	{ .name = "ab8505_buck_smpsa",  .driver_data = (void *)AB8505_BUCK_SMPSA, },
+	{ .name = "ab8505_buck_smpsb",  .driver_data = (void *)AB8505_BUCK_SMPSB, },
+	{ .name = "ab8505_buck_safe",   .driver_data = (void *)AB8505_BUCK_SAFE, },
+	{ .name = "ab8505_buck_arm",    .driver_data = (void *)AB8505_BUCK_ARM, },
+	{ .name = "ab8505_buck_smpsc",  .driver_data = (void *)AB8505_BUCK_SMPSC, },
+	{ .name = "ab8505_buck_smpsm",  .driver_data = (void *)AB8505_BUCK_SMPSM, },
 };
 
 static struct {
@@ -1677,9 +2298,14 @@ static int ab8500_regulator_register(struct platform_device *pdev,
 	config.driver_data = info;
 	config.of_node = np;
 
-	/* fix for hardware before ab8500v2.0 */
-	if (is_ab8500_1p1_or_earlier(ab8500)) {
-		if (info->desc.id == AB8500_LDO_AUX3) {
+	/* Handle the different VAUX3 implementations in early AB8500 cuts. */
+	if (info->desc.id == AB8500_LDO_AUX3) {
+		if (is_ab8500_1p0_or_earlier(ab8500)) {
+			info->desc.ops = &ab8500_regulator_mode_ops;
+			info->desc.n_voltages = 1;
+			info->desc.volt_table = fixed_1200000_voltage;
+			info->voltage_mask = 0;
+		} else if (is_ab8500_1p1_or_earlier(ab8500)) {
 			info->desc.n_voltages =
 				ARRAY_SIZE(ldo_vauxn_voltages);
 			info->desc.volt_table = ldo_vauxn_voltages;
diff --git a/drivers/regulator/axp20x-regulator.c b/drivers/regulator/axp20x-regulator.c
index da891415efc0b3..ec6ec071ffdeef 100644
--- a/drivers/regulator/axp20x-regulator.c
+++ b/drivers/regulator/axp20x-regulator.c
@@ -1652,7 +1652,7 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 		    (regulators == axp803_regulators && i == AXP803_DC1SW) ||
 		    (regulators == axp809_regulators && i == AXP809_DC1SW) ||
 		    (regulators == axp15060_regulators && i == AXP15060_SW)) {
-			new_desc = devm_kzalloc(&pdev->dev, sizeof(*desc),
+			new_desc = devm_kzalloc(&pdev->dev, sizeof(*new_desc),
 						GFP_KERNEL);
 			if (!new_desc)
 				return -ENOMEM;
@@ -1665,7 +1665,7 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 		if ((regulators == axp22x_regulators && i == AXP22X_DC5LDO) ||
 		    (regulators == axp809_regulators && i == AXP809_DC5LDO) ||
 		    (regulators == axp15060_regulators && i == AXP15060_CPUSLDO)) {
-			new_desc = devm_kzalloc(&pdev->dev, sizeof(*desc),
+			new_desc = devm_kzalloc(&pdev->dev, sizeof(*new_desc),
 						GFP_KERNEL);
 			if (!new_desc)
 				return -ENOMEM;
@@ -1677,7 +1677,7 @@ static int axp20x_regulator_probe(struct platform_device *pdev)
 
 
 		if (regulators == axp15060_regulators && i == AXP15060_RTC_LDO) {
-			new_desc = devm_kzalloc(&pdev->dev, sizeof(*desc),
+			new_desc = devm_kzalloc(&pdev->dev, sizeof(*new_desc),
 						GFP_KERNEL);
 			if (!new_desc)
 				return -ENOMEM;
diff --git a/drivers/regulator/bd71828-regulator.c b/drivers/regulator/bd71828-regulator.c
index 2ced81df0c0204..dea551cf658ab2 100644
--- a/drivers/regulator/bd71828-regulator.c
+++ b/drivers/regulator/bd71828-regulator.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 // Copyright (C) 2019 ROHM Semiconductors
 // bd71828-regulator.c ROHM BD71828GW-DS1 regulator driver
-//
 
 #include <linux/cleanup.h>
 #include <linux/delay.h>
@@ -10,6 +9,7 @@
 #include <linux/kernel.h>
 #include <linux/mfd/rohm-bd71828.h>
 #include <linux/mfd/rohm-bd72720.h>
+#include <linux/mfd/rohm-bd73800.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -31,6 +31,13 @@ struct bd71828_regulator_data {
 	int reg_init_amnt;
 };
 
+/*
+ * LDO1 and LDO3 node names are used also in the driver to find OTP setting for
+ * the used voltage range.
+ */
+#define LDO1_NODE_NAME "ldo1"
+#define LDO3_NODE_NAME "ldo3"
+
 static const struct reg_init bd71828_buck1_inits[] = {
 	/*
 	 * DVS Buck voltages can be changed by register values or via GPIO.
@@ -172,6 +179,55 @@ static const struct linear_range bd72720_ldo6_volts[] = {
 	REGULATOR_LINEAR_RANGE(1800000, 0x79, 0x7f, 0),
 };
 
+static const struct linear_range bd73800_buck12348_volts[] = {
+	REGULATOR_LINEAR_RANGE(500000, 0x00, 0x50, 10000),
+	REGULATOR_LINEAR_RANGE(1300000, 0x51, 0xff, 0),
+};
+
+static const unsigned int bd73800_buck5_volt_range_sel[] = {
+	0x0, 0x0, 0x1, 0x1,
+};
+
+static const struct linear_range bd73800_buck5_volts[] = {
+	/* range selector 0 */
+	REGULATOR_LINEAR_RANGE(500000, 0x00, 0x50, 10000),
+	REGULATOR_LINEAR_RANGE(1300000, 0x51, 0x7f, 0),
+	/* range selector 1 */
+	REGULATOR_LINEAR_RANGE(300000, 0x00, 0x46, 10000),
+	REGULATOR_LINEAR_RANGE(1000000, 0x47, 0x7f, 0),
+};
+
+/*
+ * BUCK5 has two different pickable ranges, each having voltages matching
+ * selectors 0x0 to 0x7f. This gives 0x7f + 1 different voltages for each of
+ * the ranges.
+ */
+#define BD73800_NUM_BUCK5_VOLTS ((0x7f + 1) * 2)
+
+static const struct linear_range bd73800_buck67_volts[] = {
+	REGULATOR_LINEAR_RANGE(1500000, 0x00, 0xc8, 10000),
+	REGULATOR_LINEAR_RANGE(3500000, 0xc9, 0xff, 0),
+};
+
+/*
+ * On the BD73800 the LDO1 and LDO3 support two different voltage areas.
+ * Whether to use 'high' or 'low' ranges is configured by OTP. There seems
+ * to be no register to read the configured area so it must be given via
+ * device-tree properties.
+ */
+static const struct linear_range bd73800_ldo13_low_volts[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0x00, 0x78, 10000),
+	REGULATOR_LINEAR_RANGE(1800000, 0x79, 0xff, 0),
+};
+
+static const struct linear_range bd73800_ldo13_high_volts[] = {
+	REGULATOR_LINEAR_RANGE(750000, 0x00, 0xff, 10000),
+};
+
+static const struct linear_range bd73800_ldo24_volts[] = {
+	REGULATOR_LINEAR_RANGE(750000, 0x00, 0xff, 10000),
+};
+
 static const unsigned int bd71828_ramp_delay[] = { 2500, 5000, 10000, 20000 };
 
 /*
@@ -225,6 +281,36 @@ static int bd71828_ldo6_parse_dt(struct device_node *np,
 	return 0;
 }
 
+/* Operations for all other BUCKs but BUCK 5 */
+static const struct regulator_ops bd73800_buck_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_ramp_delay = regulator_set_ramp_delay_regmap,
+};
+
+static const struct regulator_ops bd73800_buck5_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_pickable_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_pickable_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_pickable_regmap,
+	.set_ramp_delay = regulator_set_ramp_delay_regmap,
+};
+
+static const struct regulator_ops bd73800_ldo_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+};
+
 static const struct regulator_ops bd71828_buck_ops = {
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
@@ -1565,6 +1651,412 @@ static const struct bd71828_regulator_data bd72720_rdata[] = {
 	},
 };
 
+static const struct bd71828_regulator_data bd73800_rdata[] = {
+	{
+		.desc = {
+			.name = "buck1",
+			.of_match = of_match_ptr("buck1"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK1,
+			.ops = &bd73800_buck_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck12348_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck12348_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_BUCK1_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK1_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK1_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK1_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_reg = BD73800_REG_BUCK1_VOLT_IDLE,
+			.idle_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_reg = BD73800_REG_BUCK1_VOLT_SUSP,
+			.suspend_mask = BD73800_MASK_VOLT,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "buck2",
+			.of_match = of_match_ptr("buck2"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK2,
+			.ops = &bd73800_buck_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck12348_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck12348_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_BUCK2_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK2_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK2_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK2_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_reg = BD73800_REG_BUCK2_VOLT_IDLE,
+			.idle_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_reg = BD73800_REG_BUCK2_VOLT_SUSP,
+			.suspend_mask = BD73800_MASK_VOLT,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "buck3",
+			.of_match = of_match_ptr("buck3"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK3,
+			.ops = &bd73800_buck_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck12348_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck12348_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_BUCK3_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK3_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK3_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK3_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_reg = BD73800_REG_BUCK3_VOLT_IDLE,
+			.idle_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_reg = BD73800_REG_BUCK3_VOLT_SUSP,
+			.suspend_mask = BD73800_MASK_VOLT,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+
+	}, {
+		.desc = {
+			.name = "buck4",
+			.of_match = of_match_ptr("buck4"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK4,
+			.ops = &bd73800_buck_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck12348_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck12348_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_BUCK4_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK4_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK4_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK4_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_reg = BD73800_REG_BUCK4_VOLT_IDLE,
+			.idle_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_reg = BD73800_REG_BUCK4_VOLT_SUSP,
+			.suspend_mask = BD73800_MASK_VOLT,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "buck5",
+			.of_match = of_match_ptr("buck5"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK5,
+			.ops = &bd73800_buck5_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck5_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck5_volts),
+			.linear_range_selectors_bitfield =
+						bd73800_buck5_volt_range_sel,
+			.n_voltages = BD73800_NUM_BUCK5_VOLTS,
+			.enable_reg = BD73800_REG_BUCK5_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK5_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_BUCK5_VOLT,
+			.vsel_range_reg = BD73800_REG_BUCK5_MODE,
+			.vsel_range_mask = BD73800_BUCK5_RANGE_MASK,
+			.range_applied_by_vsel = true,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK5_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+		},
+		.dvs = {
+			/*
+			 * We don't support setting the IDLE / SUSPEND voltages
+			 * for the BUCK 5 for now. Reason is that the pickable
+			 * range selector bit is common for all states (RUN,
+			 * SUSPEND and IDLE), and toggling it for one impacts
+			 * also others.
+			 * Furthermore, writing the BD73800 range selector does
+			 * not impact the RUN voltage until a new voltage value
+			 * is also written to the vsel register. Hence the
+			 * voltage change can be done without any 'intermediate
+			 * voltages', even when the range is changed when the
+			 * BUCK is enabled.
+			 * It, however, is not documented how the IDLE / SUSPEND
+			 * states work in this regard. I expect the full
+			 * combination of the range selector and vsel is taken
+			 * into account at the state change, no matter when they
+			 * have been written to.
+			 */
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK5_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+
+	}, {
+		.desc = {
+			.name = "buck6",
+			.of_match = of_match_ptr("buck6"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK6,
+			.ops = &bd73800_buck_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck67_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck67_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_BUCK6_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK6_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK6_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK6_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_reg = BD73800_REG_BUCK6_VOLT_IDLE,
+			.idle_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_reg = BD73800_REG_BUCK6_VOLT_SUSP,
+			.suspend_mask = BD73800_MASK_VOLT,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "buck7",
+			.of_match = of_match_ptr("buck7"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK7,
+			.ops = &bd73800_buck_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck67_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck67_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_BUCK7_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK7_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK7_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK7_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_reg = BD73800_REG_BUCK7_VOLT_IDLE,
+			.idle_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_reg = BD73800_REG_BUCK7_VOLT_SUSP,
+			.suspend_mask = BD73800_MASK_VOLT,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "buck8",
+			.of_match = of_match_ptr("buck8"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_BUCK8,
+			.ops = &bd73800_buck_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_buck12348_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_buck12348_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_BUCK8_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_BUCK8_VOLT_RUN,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.ramp_delay_table = bd71828_ramp_delay,
+			.n_ramp_values = ARRAY_SIZE(bd71828_ramp_delay),
+			.ramp_reg = BD73800_REG_BUCK8_MODE,
+			.ramp_mask = BD73800_MASK_RAMP_DELAY,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_BUCK8_VOLT_RUN,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_reg = BD73800_REG_BUCK8_VOLT_IDLE,
+			.idle_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_reg = BD73800_REG_BUCK8_VOLT_SUSP,
+			.suspend_mask = BD73800_MASK_VOLT,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "ldo1",
+			.of_match = of_match_ptr(LDO1_NODE_NAME),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_LDO1,
+			.ops = &bd73800_ldo_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_ldo13_low_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_ldo13_low_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_LDO1_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_LDO1_VOLT,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			/*
+			 * LDOs support only single voltage for all states.
+			 * Voltage can be individually enabled for each state
+			 * though. Allow setting enable/disable config by
+			 * setting the <state>_on_mask for all supported states.
+			 */
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_LDO1_VOLT,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "ldo2",
+			.of_match = of_match_ptr("ldo2"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_LDO2,
+			.ops = &bd73800_ldo_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_ldo24_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_ldo24_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_LDO2_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_LDO2_VOLT,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_LDO2_VOLT,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "ldo3",
+			.of_match = of_match_ptr(LDO3_NODE_NAME),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_LDO3,
+			.ops = &bd73800_ldo_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_ldo13_low_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_ldo13_low_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_LDO3_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_LDO3_VOLT,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_LDO3_VOLT,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	}, {
+		.desc = {
+			.name = "ldo4",
+			.of_match = of_match_ptr("ldo4"),
+			.regulators_node = of_match_ptr("regulators"),
+			.id = BD73800_LDO4,
+			.ops = &bd73800_ldo_ops,
+			.type = REGULATOR_VOLTAGE,
+			.linear_ranges = bd73800_ldo24_volts,
+			.n_linear_ranges = ARRAY_SIZE(bd73800_ldo24_volts),
+			.n_voltages = BD73800_NUM_VOLTS,
+			.enable_reg = BD73800_REG_LDO4_ON,
+			.enable_mask = BD73800_MASK_RUN_EN,
+			.vsel_reg = BD73800_REG_LDO4_VOLT,
+			.vsel_mask = BD73800_MASK_VOLT,
+			.owner = THIS_MODULE,
+			.of_parse_cb = buck_set_hw_dvs_levels,
+		},
+		.dvs = {
+			.level_map = ROHM_DVS_LEVEL_RUN | ROHM_DVS_LEVEL_IDLE |
+				     ROHM_DVS_LEVEL_SUSPEND,
+			.run_reg = BD73800_REG_LDO4_VOLT,
+			.run_mask = BD73800_MASK_VOLT,
+			.idle_on_mask = BD73800_MASK_IDLE_EN,
+			.suspend_on_mask = BD73800_MASK_SUSP_EN,
+		},
+	},
+};
+
 static int bd72720_buck10_ldon_head_mode(struct device *dev,
 					 struct device_node *npreg,
 					 struct regmap *regmap,
@@ -1619,6 +2111,52 @@ static int bd72720_dt_parse(struct device *dev,
 	return bd72720_buck10_ldon_head_mode(dev, nproot, regmap, buck10_desc);
 }
 
+static bool bd73800_use_ldo_high_range(struct device_node *nproot,
+				       const char *ldo_node)
+{
+	struct device_node *np __free(device_node) =
+		of_get_child_by_name(nproot, ldo_node);
+
+	if (!np)
+		return false;
+
+	return of_property_read_bool(np, "rohm,ldo-range-high");
+}
+
+static int bd73800_check_ldo_otp_options(struct device *dev,
+					 struct bd71828_regulator_data *d,
+					 unsigned int num_reg_data)
+{
+	struct device_node *nproot __free(device_node) =
+		of_get_child_by_name(dev->parent->of_node, "regulators");
+
+	/*
+	 * The code assumes regulator IDs to start from 0 and to match the
+	 * indexing of regulator data arrays. WARN if someone changes this.
+	 */
+	WARN_ON(d[BD73800_LDO1].desc.id != BD73800_LDO1);
+	WARN_ON(d[BD73800_LDO3].desc.id != BD73800_LDO3);
+
+	if (!nproot) {
+		dev_err(dev, "failed to find regulators node\n");
+		return -ENODEV;
+	}
+
+	if (bd73800_use_ldo_high_range(nproot, LDO1_NODE_NAME)) {
+		d[BD73800_LDO1].desc.linear_ranges = bd73800_ldo13_high_volts;
+		d[BD73800_LDO1].desc.n_linear_ranges =
+					ARRAY_SIZE(bd73800_ldo13_high_volts);
+	}
+
+	if (bd73800_use_ldo_high_range(nproot, LDO3_NODE_NAME)) {
+		d[BD73800_LDO3].desc.linear_ranges = bd73800_ldo13_high_volts;
+		d[BD73800_LDO3].desc.n_linear_ranges =
+					ARRAY_SIZE(bd73800_ldo13_high_volts);
+	}
+
+	return 0;
+}
+
 static int bd71828_probe(struct platform_device *pdev)
 {
 	int i, j, ret, num_regulators;
@@ -1656,6 +2194,23 @@ static int bd71828_probe(struct platform_device *pdev)
 		num_regulators = ARRAY_SIZE(bd71828_rdata);
 
 		break;
+
+	case ROHM_CHIP_TYPE_BD73800:
+	{
+		rdata = devm_kmemdup(&pdev->dev, bd73800_rdata,
+				     sizeof(bd73800_rdata), GFP_KERNEL);
+		if (!rdata)
+			return -ENOMEM;
+
+		num_regulators = ARRAY_SIZE(bd73800_rdata);
+
+		ret = bd73800_check_ldo_otp_options(&pdev->dev, rdata,
+						    num_regulators);
+		if (ret)
+			return ret;
+
+		break;
+	}
 	default:
 		return dev_err_probe(&pdev->dev, -EINVAL,
 				     "Unsupported device\n");
@@ -1692,6 +2247,7 @@ static int bd71828_probe(struct platform_device *pdev)
 static const struct platform_device_id bd71828_pmic_id[] = {
 	{ .name = "bd71828-pmic", .driver_data = ROHM_CHIP_TYPE_BD71828 },
 	{ .name = "bd72720-pmic", .driver_data = ROHM_CHIP_TYPE_BD72720 },
+	{ .name = "bd73800-pmic", .driver_data = ROHM_CHIP_TYPE_BD73800 },
 	{ }
 };
 MODULE_DEVICE_TABLE(platform, bd71828_pmic_id);
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index 6a4008f387b5e7..14422cfe80e2d3 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -462,6 +462,7 @@ int regulator_check_voltage(struct regulator_dev *rdev,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(regulator_check_voltage);
 
 /* return 0 if the state is valid */
 static int regulator_check_states(suspend_state_t state)
@@ -502,6 +503,7 @@ int regulator_check_consumers(struct regulator_dev *rdev,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(regulator_check_consumers);
 
 /* current constraint check */
 static int regulator_check_current_limit(struct regulator_dev *rdev,
@@ -2239,6 +2241,7 @@ static int regulator_resolve_supply(struct regulator_dev *rdev)
 	if (r == rdev) {
 		dev_err(dev, "Supply for %s (%s) resolved to itself\n",
 			rdev->desc->name, rdev->supply_name);
+		put_device(&rdev->dev);
 		if (!have_full_constraints()) {
 			ret = -EINVAL;
 			goto out;
@@ -3029,7 +3032,7 @@ static int _regulator_do_enable(struct regulator_dev *rdev)
 	 */
 	trace_regulator_enable_delay(rdev_get_name(rdev));
 
-	/* If poll_enabled_time is set, poll upto the delay calculated
+	/* If poll_enabled_time is set, poll up to the delay calculated
 	 * above, delaying poll_enabled_time uS to check if the regulator
 	 * actually got enabled.
 	 * If the regulator isn't enabled after our delay helper has expired,
@@ -4468,6 +4471,7 @@ int regulator_do_balance_voltage(struct regulator_dev *rdev,
 out:
 	return ret;
 }
+EXPORT_SYMBOL_GPL(regulator_do_balance_voltage);
 
 static int regulator_balance_voltage(struct regulator_dev *rdev,
 				     suspend_state_t state)
@@ -5797,6 +5801,7 @@ int regulator_coupler_register(struct regulator_coupler *coupler)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(regulator_coupler_register);
 
 static struct regulator_coupler *
 regulator_find_coupler(struct regulator_dev *rdev)
@@ -5861,6 +5866,7 @@ static void regulator_resolve_coupling(struct regulator_dev *rdev)
 		if (c_rdev->coupling_desc.coupler != coupler) {
 			rdev_err(rdev, "coupler mismatch with %s\n",
 				 rdev_get_name(c_rdev));
+			put_device(&c_rdev->dev);
 			return;
 		}
 
@@ -5907,6 +5913,8 @@ static void regulator_remove_coupling(struct regulator_dev *rdev)
 
 		c_desc->coupled_rdevs[i] = NULL;
 		c_desc->n_resolved--;
+
+		put_device(&c_rdev->dev);
 	}
 
 	if (coupler && coupler->detach_regulator) {
diff --git a/drivers/regulator/fixed.c b/drivers/regulator/fixed.c
index 417d282dd4ed70..f797cb504b9aea 100644
--- a/drivers/regulator/fixed.c
+++ b/drivers/regulator/fixed.c
@@ -15,6 +15,7 @@
  */
 
 #include <linux/err.h>
+#include <linux/interrupt.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
@@ -28,6 +29,7 @@
 #include <linux/regulator/of_regulator.h>
 #include <linux/regulator/machine.h>
 #include <linux/clk.h>
+#include <linux/string.h>
 
 struct fixed_voltage_data {
 	struct regulator_desc desc;
@@ -225,6 +227,9 @@ static int reg_fixed_voltage_probe(struct platform_device *pdev)
 	enum gpiod_flags gflags;
 	int ret;
 
+	if (!pdev->dev.of_node && strcmp(pdev->name, "reg-fixed-voltage"))
+		return -ENODEV;
+
 	drvdata = devm_kzalloc(&pdev->dev, sizeof(struct fixed_voltage_data),
 			       GFP_KERNEL);
 	if (!drvdata)
diff --git a/drivers/regulator/fp9931.c b/drivers/regulator/fp9931.c
index ff743a8b0dfe80..48c3b21ebafcd0 100644
--- a/drivers/regulator/fp9931.c
+++ b/drivers/regulator/fp9931.c
@@ -7,6 +7,7 @@
 #include <linux/completion.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/hwmon.h>
diff --git a/drivers/regulator/mp886x.c b/drivers/regulator/mp886x.c
index e0b62bc02a1e11..e483e9d7dcf22e 100644
--- a/drivers/regulator/mp886x.c
+++ b/drivers/regulator/mp886x.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 //
-// MP8867/MP8869 regulator driver
+// MP8864/MP8867/MP8869 regulator driver
 //
 // Copyright (C) 2020 Synaptics Incorporated
 //
@@ -229,6 +229,23 @@ static const struct regulator_ops mp8867_regulator_ops = {
 	.set_ramp_delay = regulator_set_ramp_delay_regmap,
 };
 
+static const struct mp886x_cfg_info mp8864_ci = {
+	.rops = &mp8867_regulator_ops,
+	.slew_rates = {
+		64000,
+		32000,
+		16000,
+		8000,
+		4000,
+		2000,
+		1000,
+		500,
+	},
+	.switch_freq = { 600000, 850000, 1100000, 1600000 },
+	.fs_reg = MP886X_SYSCNTLREG1,
+	.fs_shift = 1,
+};
+
 static const struct mp886x_cfg_info mp8867_ci = {
 	.rops = &mp8867_regulator_ops,
 	.slew_rates = {
@@ -256,6 +273,7 @@ static int mp886x_regulator_register(struct mp886x_device_info *di,
 {
 	struct regulator_desc *rdesc = &di->desc;
 	struct regulator_dev *rdev;
+	int sel;
 
 	rdesc->name = "mp886x-reg";
 	rdesc->supply_name = "vin";
@@ -267,7 +285,7 @@ static int mp886x_regulator_register(struct mp886x_device_info *di,
 	rdesc->min_uV = 600000;
 	rdesc->uV_step = 10000;
 	rdesc->vsel_reg = MP886X_VSEL;
-	rdesc->vsel_mask = 0x3f;
+	rdesc->vsel_mask = 0x7f;
 	rdesc->ramp_reg = MP886X_SYSCNTLREG1;
 	rdesc->ramp_mask = MP886X_SLEW_MASK;
 	rdesc->ramp_delay_table = di->ci->slew_rates;
@@ -277,7 +295,12 @@ static int mp886x_regulator_register(struct mp886x_device_info *di,
 	rdev = devm_regulator_register(di->dev, &di->desc, config);
 	if (IS_ERR(rdev))
 		return PTR_ERR(rdev);
-	di->sel = rdesc->ops->get_voltage_sel(rdev);
+
+	sel = rdesc->ops->get_voltage_sel(rdev);
+	if (sel < 0)
+		return sel;
+	di->sel = sel;
+
 	return 0;
 }
 
@@ -311,6 +334,9 @@ static int mp886x_i2c_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
+	if (di->r[1] == 0)
+		return -EINVAL;
+
 	di->en_gpio = devm_gpiod_get(dev, "enable", GPIOD_OUT_HIGH);
 	if (IS_ERR(di->en_gpio))
 		return PTR_ERR(di->en_gpio);
@@ -341,6 +367,7 @@ static int mp886x_i2c_probe(struct i2c_client *client)
 }
 
 static const struct of_device_id mp886x_dt_ids[] = {
+	{ .compatible = "mps,mp8864", .data = &mp8864_ci },
 	{ .compatible = "mps,mp8867", .data = &mp8867_ci },
 	{ .compatible = "mps,mp8869", .data = &mp8869_ci },
 	{ }
@@ -348,6 +375,7 @@ static const struct of_device_id mp886x_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, mp886x_dt_ids);
 
 static const struct i2c_device_id mp886x_id[] = {
+	{ .name = "mp8864", .driver_data = (kernel_ulong_t)&mp8864_ci },
 	{ .name = "mp8867", .driver_data = (kernel_ulong_t)&mp8867_ci },
 	{ .name = "mp8869", .driver_data = (kernel_ulong_t)&mp8869_ci },
 	{ }
diff --git a/drivers/regulator/mpq4210.c b/drivers/regulator/mpq4210.c
new file mode 100644
index 00000000000000..293e7b209a987a
--- /dev/null
+++ b/drivers/regulator/mpq4210.c
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Monolithic Power Systems MPQ4210 buck-boost regulator
+ *
+ * Copyright (c) 2026 Vaisala Oyj
+ */
+
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/math.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/of_regulator.h>
+
+#define MPQ4210_REF_LSB			0x00
+#define MPQ4210_REF_LSB_MASK		GENMASK(2, 0)
+#define MPQ4210_REF_LSB_BITS		3
+#define MPQ4210_REF_MSB			0x01
+#define MPQ4210_CONTROL1		0x02
+#define MPQ4210_CONTROL1_SR		GENMASK(7, 6)
+#define MPQ4210_CONTROL1_RESERVED	BIT(2)
+#define MPQ4210_CONTROL1_GO		BIT(1)
+#define MPQ4210_CONTROL1_ENPWR		BIT(0)
+#define MPQ4210_INT_MASK		0x06
+
+/*
+ * The feedback reference is an 11 bit value with a 1mV step. The datasheet
+ * specifies no reference below 0.3V, so those selectors are not offered.
+ */
+#define MPQ4210_REF_MIN			0x12c
+#define MPQ4210_REF_MAX			0x7ff
+#define MPQ4210_REF_STEP_UV		1000
+
+#define MPQ4210_ENPWR_DELAY_MS		200
+
+/* Feedback reference slew rate per MPQ4210_CONTROL1_SR value, in uV/us. */
+static const unsigned int mpq4210_ref_slew_rate[] = { 38, 50, 75, 150 };
+
+static const struct regmap_config mpq4210_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = MPQ4210_INT_MASK,
+};
+
+/* Scale a feedback reference value by the output voltage divider ratio. */
+static unsigned int mpq4210_scale(unsigned int val, u32 r1, u32 r2)
+{
+	u64 tmp = (u64)val * (r1 + r2);
+
+	do_div(tmp, r2);
+
+	return tmp;
+}
+
+static int mpq4210_set_voltage_sel(struct regulator_dev *rdev, unsigned int sel)
+{
+	int ret;
+
+	ret = regmap_write(rdev->regmap, MPQ4210_REF_LSB,
+			   sel & MPQ4210_REF_LSB_MASK);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(rdev->regmap, MPQ4210_REF_MSB,
+			   sel >> MPQ4210_REF_LSB_BITS);
+	if (ret)
+		return ret;
+
+	return regmap_set_bits(rdev->regmap, MPQ4210_CONTROL1,
+			       MPQ4210_CONTROL1_GO);
+}
+
+static int mpq4210_get_voltage_sel(struct regulator_dev *rdev)
+{
+	unsigned int lsb, msb;
+	int ret;
+
+	ret = regmap_read(rdev->regmap, MPQ4210_REF_MSB, &msb);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(rdev->regmap, MPQ4210_REF_LSB, &lsb);
+	if (ret)
+		return ret;
+
+	return (msb << MPQ4210_REF_LSB_BITS) | (lsb & MPQ4210_REF_LSB_MASK);
+}
+
+static int mpq4210_enable(struct regulator_dev *rdev)
+{
+	int ret;
+
+	ret = regmap_set_bits(rdev->regmap, MPQ4210_CONTROL1,
+			      MPQ4210_CONTROL1_GO);
+	if (ret)
+		return ret;
+
+	/* The reference has to settle before power switching may start. */
+	msleep(MPQ4210_ENPWR_DELAY_MS);
+
+	return regmap_set_bits(rdev->regmap, MPQ4210_CONTROL1,
+			       MPQ4210_CONTROL1_ENPWR);
+}
+
+static const struct regulator_ops mpq4210_regulator_ops = {
+	.set_voltage_sel = mpq4210_set_voltage_sel,
+	.get_voltage_sel = mpq4210_get_voltage_sel,
+	.list_voltage = regulator_list_voltage_linear,
+	.set_ramp_delay = regulator_set_ramp_delay_regmap,
+	.enable = mpq4210_enable,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+};
+
+static const struct regulator_desc mpq4210_regulator = {
+	.name = "mpq4210",
+	.type = REGULATOR_VOLTAGE,
+	.owner = THIS_MODULE,
+	.ops = &mpq4210_regulator_ops,
+	.n_voltages = MPQ4210_REF_MAX + 1,
+	.linear_min_sel = MPQ4210_REF_MIN,
+	.enable_reg = MPQ4210_CONTROL1,
+	.enable_mask = MPQ4210_CONTROL1_ENPWR,
+	.ramp_reg = MPQ4210_CONTROL1,
+	.ramp_mask = MPQ4210_CONTROL1_SR,
+	.n_ramp_values = ARRAY_SIZE(mpq4210_ref_slew_rate),
+};
+
+static int mpq4210_i2c_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct regulator_config config = { };
+	struct regulator_desc *desc;
+	struct regulator_dev *rdev;
+	struct gpio_desc *enable;
+	struct regmap *regmap;
+	unsigned int *slew;
+	unsigned int i, val;
+	u32 r[2];
+	int ret;
+
+	ret = of_property_read_u32_array(dev->of_node,
+					 "mps,fb-voltage-divider-ohms",
+					 r, ARRAY_SIZE(r));
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to read mps,fb-voltage-divider-ohms\n");
+
+	if (!r[1])
+		return dev_err_probe(dev, -EINVAL,
+				     "feedback divider R2 must not be zero\n");
+
+	/* The controller does not answer on the bus while EN is deasserted. */
+	enable = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_HIGH);
+	if (IS_ERR(enable))
+		return dev_err_probe(dev, PTR_ERR(enable),
+				     "failed to get enable GPIO\n");
+
+	regmap = devm_regmap_init_i2c(client, &mpq4210_regmap_config);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap),
+				     "failed to init regmap\n");
+
+	desc = devm_kmemdup(dev, &mpq4210_regulator, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	desc->min_uV = mpq4210_scale(MPQ4210_REF_MIN * MPQ4210_REF_STEP_UV,
+				     r[0], r[1]);
+	desc->uV_step = mpq4210_scale(MPQ4210_REF_STEP_UV, r[0], r[1]);
+
+	slew = devm_kcalloc(dev, ARRAY_SIZE(mpq4210_ref_slew_rate),
+			    sizeof(*slew), GFP_KERNEL);
+	if (!slew)
+		return -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(mpq4210_ref_slew_rate); i++)
+		slew[i] = mpq4210_scale(mpq4210_ref_slew_rate[i], r[0], r[1]);
+
+	desc->ramp_delay_table = slew;
+
+	ret = regmap_read(regmap, MPQ4210_CONTROL1, &val);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to read control 1\n");
+
+	/* The core overrides this if the board sets regulator-ramp-delay. */
+	desc->ramp_delay = slew[FIELD_GET(MPQ4210_CONTROL1_SR, val)];
+
+	/*
+	 * Documented as reserved, but the datasheet requires it to be set
+	 * before the controller starts up.
+	 */
+	ret = regmap_set_bits(regmap, MPQ4210_CONTROL1,
+			      MPQ4210_CONTROL1_RESERVED);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to write control 1\n");
+
+	config.dev = dev;
+	config.regmap = regmap;
+	config.of_node = dev->of_node;
+	config.init_data = of_get_regulator_init_data(dev, dev->of_node, desc);
+	if (!config.init_data)
+		return -ENOMEM;
+
+	rdev = devm_regulator_register(dev, desc, &config);
+	if (IS_ERR(rdev))
+		return dev_err_probe(dev, PTR_ERR(rdev),
+				     "failed to register regulator\n");
+
+	return 0;
+}
+
+static const struct of_device_id mpq4210_of_match[] = {
+	{ .compatible = "mps,mpq4210" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mpq4210_of_match);
+
+static const struct i2c_device_id mpq4210_i2c_id[] = {
+	{ .name = "mpq4210" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, mpq4210_i2c_id);
+
+static struct i2c_driver mpq4210_regulator_driver = {
+	.driver = {
+		.name = "mpq4210",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.of_match_table = mpq4210_of_match,
+	},
+	.probe = mpq4210_i2c_probe,
+	.id_table = mpq4210_i2c_id,
+};
+
+module_i2c_driver(mpq4210_regulator_driver);
+
+MODULE_DESCRIPTION("Monolithic Power Systems MPQ4210 voltage regulator driver");
+MODULE_AUTHOR("Tapio Reijonen <tapio.reijonen@vaisala.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/mt6360-regulator.c b/drivers/regulator/mt6360-regulator.c
index ed25e9603b2b4c..daef54ea4e6e8a 100644
--- a/drivers/regulator/mt6360-regulator.c
+++ b/drivers/regulator/mt6360-regulator.c
@@ -5,6 +5,7 @@
 // Author: Gene Chen <gene_chen@richtek.com>
 
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
diff --git a/drivers/regulator/nex10000ub-regulator.c b/drivers/regulator/nex10000ub-regulator.c
new file mode 100644
index 00000000000000..f44b8b7906b922
--- /dev/null
+++ b/drivers/regulator/nex10000ub-regulator.c
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Nexperia NEX10000UB Regulator driver
+ *
+ * Copyright (C) 2026 Linaro Limited.
+ */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+
+#define NEX10000UB_REG_VPOS		0x00
+#define NEX10000UB_REG_VNEG		0x01
+#define NEX10000UB_REG_ID		0x02
+
+#define NEX10000UB_VOUT_MASK		0x1F
+#define NEX10000UB_VOUT_N_VOLTAGE	0x15
+#define NEX10000UB_VOUT_VMIN		4000000
+#define NEX10000UB_VOUT_VMAX		6000000
+#define NEX10000UB_VOUT_STEP		100000
+
+#define NEX10000UB_REGULATOR_ID_VPOS	0
+#define NEX10000UB_REGULATOR_ID_VNEG	1
+#define NEX10000UB_MAX_REGULATORS	2
+
+static const struct regulator_ops nex10000ub_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear,
+	.map_voltage = regulator_map_voltage_linear,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+};
+
+static int nex10000ub_of_parse_cb(struct device_node *np,
+				  const struct regulator_desc *desc,
+				  struct regulator_config *config)
+{
+	struct gpio_desc *ena_gpiod;
+
+	ena_gpiod = fwnode_gpiod_get_index(of_fwnode_handle(np), "enable", 0,
+					   GPIOD_OUT_LOW, desc->name);
+	if (IS_ERR(ena_gpiod))
+		return PTR_ERR(ena_gpiod);
+
+	config->ena_gpiod = ena_gpiod;
+
+	return 0;
+}
+
+#define NEX10000UB_REGULATOR_DESC(_id, _name)			\
+	[NEX10000UB_REGULATOR_ID_##_id] = {			\
+		.name = "nex10000ub-"#_name,			\
+		.supply_name = "vin",				\
+		.id = NEX10000UB_REGULATOR_ID_##_id,		\
+		.of_match = of_match_ptr(#_name),		\
+		.of_parse_cb = nex10000ub_of_parse_cb,		\
+		.ops = &nex10000ub_regulator_ops,		\
+		.n_voltages = NEX10000UB_VOUT_N_VOLTAGE,	\
+		.min_uV = NEX10000UB_VOUT_VMIN,			\
+		.uV_step = NEX10000UB_VOUT_STEP,		\
+		.enable_time = 2000,				\
+		.vsel_mask = NEX10000UB_VOUT_MASK,		\
+		.vsel_reg = NEX10000UB_REG_##_id,		\
+		.type = REGULATOR_VOLTAGE,			\
+		.owner = THIS_MODULE,				\
+	}
+
+static const struct regulator_desc nex10000_regs_desc[NEX10000UB_MAX_REGULATORS] = {
+	NEX10000UB_REGULATOR_DESC(VPOS, vpos),
+	NEX10000UB_REGULATOR_DESC(VNEG, vneg),
+};
+
+static const struct regmap_config nex10000ub_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 8,
+	.max_register	= NEX10000UB_REG_ID,
+	.cache_type	= REGCACHE_NONE,
+};
+
+static int nex10000ub_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct regmap *map;
+	int id;
+	int ret;
+
+	map = devm_regmap_init_i2c(client, &nex10000ub_regmap_config);
+	if (IS_ERR(map)) {
+		ret = PTR_ERR(map);
+		dev_err(dev, "regmap init failed: %d\n", ret);
+		return ret;
+	}
+
+	for (id = 0; id < NEX10000UB_MAX_REGULATORS; ++id) {
+		struct regulator_config config = { };
+		struct regulator_dev *rdev;
+
+		config.regmap = map;
+		config.dev = dev;
+
+		rdev = devm_regulator_register(dev, &nex10000_regs_desc[id],
+					       &config);
+		if (IS_ERR(rdev)) {
+			ret = PTR_ERR(rdev);
+			dev_err(dev, "regulator %s register failed: %d\n",
+				nex10000_regs_desc[id].name, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static const struct i2c_device_id nex10000ub_id[] = {
+	{ .name = "nex10000ub" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, nex10000ub_id);
+
+static const struct of_device_id __maybe_unused nex10000ub_of_match[] = {
+	{ .compatible = "nexperia,nex10000ub" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, nex10000ub_of_match);
+
+static struct i2c_driver nex10000ub_i2c_driver = {
+	.driver = {
+		.name = "nex10000ub",
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
+		.of_match_table = of_match_ptr(nex10000ub_of_match),
+	},
+	.probe = nex10000ub_probe,
+	.id_table = nex10000ub_id,
+};
+
+module_i2c_driver(nex10000ub_i2c_driver);
+
+MODULE_DESCRIPTION("NEX10000UB regulator driver");
+MODULE_AUTHOR("Neil Armstrong <neil.armstrong@linaro.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/pca9450-regulator.c b/drivers/regulator/pca9450-regulator.c
index c41db70fa05291..613bce411f3198 100644
--- a/drivers/regulator/pca9450-regulator.c
+++ b/drivers/regulator/pca9450-regulator.c
@@ -77,10 +77,56 @@ static const unsigned int pca9450_dvs_buck_ramp_table[] = {
 	25000, 12500, 6250, 3125
 };
 
+static int pca9450_regulator_is_enabled(struct regulator_dev *rdev)
+{
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(rdev->regmap, rdev->desc->enable_reg, &val);
+	if (ret)
+		return ret;
+
+	/* All valid nonzero ENMODE values enable the regulator in RUN mode. */
+	return !!(val & rdev->desc->enable_mask);
+}
+
+static int pca9450_set_suspend_disable(struct regulator_dev *rdev,
+				       unsigned int suspend_val)
+{
+	int ret;
+
+	/* Do not enable a runtime-disabled regulator when returning to RUN. */
+	ret = pca9450_regulator_is_enabled(rdev);
+	if (ret < 0)
+		return ret;
+	if (!ret)
+		return 0;
+
+	return regmap_update_bits(rdev->regmap, rdev->desc->enable_reg,
+				  rdev->desc->enable_mask,
+				  suspend_val);
+}
+
+/*
+ * ENMODE = 10b keeps the regulator enabled in RUN mode and disables it
+ * while PMIC_STBY_REQ is asserted.
+ */
+static int pca9450_buck_set_suspend_disable(struct regulator_dev *rdev)
+{
+	return pca9450_set_suspend_disable(rdev,
+					   BUCK_ENMODE_ONREQ_STBYREQ);
+}
+
+static int pca9450_ldo_set_suspend_disable(struct regulator_dev *rdev)
+{
+	return pca9450_set_suspend_disable(rdev,
+					   LDO_ENMODE_ONREQ_STBYREQ);
+}
+
 static const struct regulator_ops pca9450_dvs_buck_regulator_ops = {
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
-	.is_enabled = regulator_is_enabled_regmap,
+	.is_enabled = pca9450_regulator_is_enabled,
 	.list_voltage = regulator_list_voltage_linear_range,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
@@ -88,18 +134,20 @@ static const struct regulator_ops pca9450_dvs_buck_regulator_ops = {
 	.set_ramp_delay	= regulator_set_ramp_delay_regmap,
 	.set_mode = pca9450_buck_set_mode,
 	.get_mode = pca9450_buck_get_mode,
+	.set_suspend_disable = pca9450_buck_set_suspend_disable,
 };
 
 static const struct regulator_ops pca9450_buck_regulator_ops = {
 	.enable = regulator_enable_regmap,
 	.disable = regulator_disable_regmap,
-	.is_enabled = regulator_is_enabled_regmap,
+	.is_enabled = pca9450_regulator_is_enabled,
 	.list_voltage = regulator_list_voltage_linear_range,
 	.set_voltage_sel = regulator_set_voltage_sel_regmap,
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
 	.set_voltage_time_sel = regulator_set_voltage_time_sel,
 	.set_mode = pca9450_buck_set_mode,
 	.get_mode = pca9450_buck_get_mode,
+	.set_suspend_disable = pca9450_buck_set_suspend_disable,
 };
 
 static const struct regulator_ops pca9450_ldo_regulator_ops = {
@@ -111,6 +159,16 @@ static const struct regulator_ops pca9450_ldo_regulator_ops = {
 	.get_voltage_sel = regulator_get_voltage_sel_regmap,
 };
 
+static const struct regulator_ops pca9450_ldo_suspend_regulator_ops = {
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.list_voltage = regulator_list_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_suspend_disable = pca9450_ldo_set_suspend_disable,
+};
+
 static unsigned int pca9450_ldo5_get_reg_voltage_sel(struct regulator_dev *rdev)
 {
 	struct pca9450 *pca9450 = rdev_get_drvdata(rdev);
@@ -164,6 +222,7 @@ static const struct regulator_ops pca9450_ldo5_regulator_ops = {
 	.list_voltage = regulator_list_voltage_linear_range,
 	.set_voltage_sel = pca9450_ldo5_set_voltage_sel_regmap,
 	.get_voltage_sel = pca9450_ldo5_get_voltage_sel_regmap,
+	.set_suspend_disable = pca9450_ldo_set_suspend_disable,
 };
 
 /*
@@ -575,7 +634,7 @@ static struct pca9450_regulator_desc pca9450a_regulators[] = {
 			.of_match = of_match_ptr("LDO3"),
 			.regulators_node = of_match_ptr("regulators"),
 			.id = PCA9450_LDO3,
-			.ops = &pca9450_ldo_regulator_ops,
+			.ops = &pca9450_ldo_suspend_regulator_ops,
 			.type = REGULATOR_VOLTAGE,
 			.n_voltages = PCA9450_LDO3_VOLTAGE_NUM,
 			.linear_ranges = pca9450_ldo34_volts,
@@ -594,7 +653,7 @@ static struct pca9450_regulator_desc pca9450a_regulators[] = {
 			.of_match = of_match_ptr("LDO4"),
 			.regulators_node = of_match_ptr("regulators"),
 			.id = PCA9450_LDO4,
-			.ops = &pca9450_ldo_regulator_ops,
+			.ops = &pca9450_ldo_suspend_regulator_ops,
 			.type = REGULATOR_VOLTAGE,
 			.n_voltages = PCA9450_LDO4_VOLTAGE_NUM,
 			.linear_ranges = pca9450_ldo34_volts,
@@ -621,7 +680,7 @@ static struct pca9450_regulator_desc pca9450a_regulators[] = {
 			.vsel_reg = PCA9450_REG_LDO5CTRL_H,
 			.vsel_mask = LDO5HOUT_MASK,
 			.enable_reg = PCA9450_REG_LDO5CTRL_L,
-			.enable_mask = LDO5H_EN_MASK,
+			.enable_mask = LDO5L_EN_MASK,
 			.owner = THIS_MODULE,
 		},
 	},
@@ -820,7 +879,7 @@ static struct pca9450_regulator_desc pca9450bc_regulators[] = {
 			.of_match = of_match_ptr("LDO3"),
 			.regulators_node = of_match_ptr("regulators"),
 			.id = PCA9450_LDO3,
-			.ops = &pca9450_ldo_regulator_ops,
+			.ops = &pca9450_ldo_suspend_regulator_ops,
 			.type = REGULATOR_VOLTAGE,
 			.n_voltages = PCA9450_LDO3_VOLTAGE_NUM,
 			.linear_ranges = pca9450_ldo34_volts,
@@ -839,7 +898,7 @@ static struct pca9450_regulator_desc pca9450bc_regulators[] = {
 			.of_match = of_match_ptr("LDO4"),
 			.regulators_node = of_match_ptr("regulators"),
 			.id = PCA9450_LDO4,
-			.ops = &pca9450_ldo_regulator_ops,
+			.ops = &pca9450_ldo_suspend_regulator_ops,
 			.type = REGULATOR_VOLTAGE,
 			.n_voltages = PCA9450_LDO4_VOLTAGE_NUM,
 			.linear_ranges = pca9450_ldo34_volts,
@@ -866,7 +925,7 @@ static struct pca9450_regulator_desc pca9450bc_regulators[] = {
 			.vsel_reg = PCA9450_REG_LDO5CTRL_H,
 			.vsel_mask = LDO5HOUT_MASK,
 			.enable_reg = PCA9450_REG_LDO5CTRL_L,
-			.enable_mask = LDO5H_EN_MASK,
+			.enable_mask = LDO5L_EN_MASK,
 			.owner = THIS_MODULE,
 		},
 	},
@@ -1040,7 +1099,7 @@ static struct pca9450_regulator_desc pca9451a_regulators[] = {
 			.of_match = of_match_ptr("LDO3"),
 			.regulators_node = of_match_ptr("regulators"),
 			.id = PCA9450_LDO3,
-			.ops = &pca9450_ldo_regulator_ops,
+			.ops = &pca9450_ldo_suspend_regulator_ops,
 			.type = REGULATOR_VOLTAGE,
 			.n_voltages = PCA9450_LDO3_VOLTAGE_NUM,
 			.linear_ranges = pca9450_ldo34_volts,
@@ -1059,7 +1118,7 @@ static struct pca9450_regulator_desc pca9451a_regulators[] = {
 			.of_match = of_match_ptr("LDO4"),
 			.regulators_node = of_match_ptr("regulators"),
 			.id = PCA9450_LDO4,
-			.ops = &pca9450_ldo_regulator_ops,
+			.ops = &pca9450_ldo_suspend_regulator_ops,
 			.type = REGULATOR_VOLTAGE,
 			.n_voltages = PCA9450_LDO4_VOLTAGE_NUM,
 			.linear_ranges = pca9450_ldo34_volts,
@@ -1086,7 +1145,7 @@ static struct pca9450_regulator_desc pca9451a_regulators[] = {
 			.vsel_reg = PCA9450_REG_LDO5CTRL_H,
 			.vsel_mask = LDO5HOUT_MASK,
 			.enable_reg = PCA9450_REG_LDO5CTRL_L,
-			.enable_mask = LDO5H_EN_MASK,
+			.enable_mask = LDO5L_EN_MASK,
 			.owner = THIS_MODULE,
 		},
 	},
diff --git a/drivers/regulator/pf1550-regulator.c b/drivers/regulator/pf1550-regulator.c
index ceee553a84b237..84f8b187a91bf5 100644
--- a/drivers/regulator/pf1550-regulator.c
+++ b/drivers/regulator/pf1550-regulator.c
@@ -49,17 +49,26 @@ static const int pf1550_ldo13_volts[] = {
 static int pf1550_set_ramp_delay(struct regulator_dev *rdev, int ramp_delay)
 {
 	int id = rdev_get_id(rdev);
-	unsigned int ramp_bits = 0;
+	unsigned int ramp_bits;
 	int ret;
 
 	if (id > PF1550_VREFDDR)
 		return -EACCES;
 
-	if (ramp_delay < 0 || ramp_delay > 6250)
+	switch (ramp_delay) {
+	case 0:
+		/* Ramp control is disabled, so use the fastest rate. */
+		ramp_bits = 0;
+		break;
+	case 1 ... 3125:
+		ramp_bits = 1;
+		break;
+	case 3126 ... 6250:
+		ramp_bits = 0;
+		break;
+	default:
 		return -EINVAL;
-
-	ramp_delay = 6250 / ramp_delay;
-	ramp_bits = ramp_delay >> 1;
+	}
 
 	ret = regmap_update_bits(rdev->regmap, rdev->desc->vsel_reg + 4, 0x10,
 				 ramp_bits << 4);
diff --git a/drivers/regulator/pv88080-regulator.c b/drivers/regulator/pv88080-regulator.c
index b96afbf11ff343..1f2c405fd2bdec 100644
--- a/drivers/regulator/pv88080-regulator.c
+++ b/drivers/regulator/pv88080-regulator.c
@@ -432,7 +432,7 @@ static int pv88080_i2c_probe(struct i2c_client *i2c)
 	config.dev = chip->dev;
 	config.regmap = chip->regmap;
 
-	/* Registeration for BUCK1, 2, 3 */
+	/* Registration for BUCK1, 2, 3 */
 	for (i = 0; i < PV88080_MAX_REGULATORS-1; i++) {
 		if (init_data)
 			config.init_data = &init_data[i];
@@ -498,7 +498,7 @@ static int pv88080_i2c_probe(struct i2c_client *i2c)
 	pv88080_regulator_info[PV88080_ID_HVBUCK].desc.vsel_mask
 		= regmap_config->hvbuck_vsel_mask;
 
-	/* Registeration for HVBUCK */
+	/* Registration for HVBUCK */
 	if (init_data)
 		config.init_data = &init_data[PV88080_ID_HVBUCK];
 
diff --git a/drivers/regulator/qcom-labibb-regulator.c b/drivers/regulator/qcom-labibb-regulator.c
index 1b14015caca635..4e089a75cd652f 100644
--- a/drivers/regulator/qcom-labibb-regulator.c
+++ b/drivers/regulator/qcom-labibb-regulator.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 // Copyright (c) 2020, The Linux Foundation. All rights reserved.
 
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/of_irq.h>
 #include <linux/of.h>
diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c
index 25c14de3cd8b5e..4c90d206e63bc1 100644
--- a/drivers/regulator/qcom-rpmh-regulator.c
+++ b/drivers/regulator/qcom-rpmh-regulator.c
@@ -400,7 +400,7 @@ static unsigned int rpmh_regulator_vrm_get_mode(struct regulator_dev *rdev)
  * This function is used in the regulator_ops for VRM type RPMh regulator
  * devices.
  *
- * Return: 0 on success, or a negative error number on failure
+ * Return: the regulator mode to use for the load
  */
 static unsigned int rpmh_regulator_vrm_get_optimum_mode(
 	struct regulator_dev *rdev, int input_uV, int output_uV, int load_uA)
@@ -1589,31 +1589,34 @@ static const struct rpmh_vreg_init_data pm6150l_vreg_data[] = {
 };
 
 static const struct rpmh_vreg_init_data pm6350_vreg_data[] = {
-	RPMH_VREG("smps1",  SMPS, 1,  &pmic5_ftsmps510, NULL),
-	RPMH_VREG("smps2",  SMPS, 2,  &pmic5_hfsmps510, NULL),
-	/* smps3 - smps5 not configured */
-	RPMH_VREG("ldo1",   LDO,  1,  &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo2",   LDO,  2,  &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo3",   LDO,  3,  &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo4",   LDO,  4,  &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo5",   LDO,  5,  &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo6",   LDO,  6,  &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo7",   LDO,  7,  &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo8",   LDO,  8,  &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo9",   LDO,  9,  &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo10",  LDO,  10, &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo11",  LDO,  11, &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo12",  LDO,  12, &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo13",  LDO,  13, &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo14",  LDO,  14, &pmic5_pldo,      NULL),
-	RPMH_VREG("ldo15",  LDO,  15, &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo16",  LDO,  16, &pmic5_nldo,      NULL),
-	/* ldo17 not configured */
-	RPMH_VREG("ldo18",  LDO,  18, &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo19",  LDO,  19, &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo20",  LDO,  20, &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo21",  LDO,  21, &pmic5_nldo,      NULL),
-	RPMH_VREG("ldo22",  LDO,  22, &pmic5_nldo,      NULL),
+	RPMH_VREG("smps1",  SMPS, 1,  &pmic5_ftsmps510, "vdd-s1"),
+	RPMH_VREG("smps2",  SMPS, 2,  &pmic5_hfsmps510, "vdd-s2"),
+	RPMH_VREG("smps3",  SMPS, 3,  &pmic5_ftsmps510, "vdd-s3"),
+	RPMH_VREG("smps4",  SMPS, 4,  &pmic5_ftsmps510, "vdd-s4"),
+	RPMH_VREG("smps5",  SMPS, 5,  &pmic5_ftsmps510, "vdd-s5"),
+	RPMH_VREG("smps6",  SMPS, 6,  &pmic5_hfsmps510, "vdd-s6"),
+	RPMH_VREG("ldo1",   LDO,  1,  &pmic5_nldo,      "vdd-l1"),
+	RPMH_VREG("ldo2",   LDO,  2,  &pmic5_pldo,      "vdd-l2-l5-l7"),
+	RPMH_VREG("ldo3",   LDO,  3,  &pmic5_pldo,      "vdd-l3-l6-l8-l9-l10"),
+	RPMH_VREG("ldo4",   LDO,  4,  &pmic5_nldo,      "vdd-l4"),
+	RPMH_VREG("ldo5",   LDO,  5,  &pmic5_pldo,      "vdd-l2-l5-l7"),
+	RPMH_VREG("ldo6",   LDO,  6,  &pmic5_pldo,      "vdd-l3-l6-l8-l9-l10"),
+	RPMH_VREG("ldo7",   LDO,  7,  &pmic5_pldo,      "vdd-l2-l5-l7"),
+	RPMH_VREG("ldo8",   LDO,  8,  &pmic5_pldo,      "vdd-l3-l6-l8-l9-l10"),
+	RPMH_VREG("ldo9",   LDO,  9,  &pmic5_pldo,      "vdd-l3-l6-l8-l9-l10"),
+	RPMH_VREG("ldo10",  LDO,  10, &pmic5_pldo,      "vdd-l3-l6-l8-l9-l10"),
+	RPMH_VREG("ldo11",  LDO,  11, &pmic5_pldo,      "vdd-l11"),
+	RPMH_VREG("ldo12",  LDO,  12, &pmic5_pldo,      "vdd-l12-l14"),
+	RPMH_VREG("ldo13",  LDO,  13, &pmic5_nldo,      "vdd-l13"),
+	RPMH_VREG("ldo14",  LDO,  14, &pmic5_pldo,      "vdd-l12-l14"),
+	RPMH_VREG("ldo15",  LDO,  15, &pmic5_nldo,      "vdd-l15-l19-l22"),
+	RPMH_VREG("ldo16",  LDO,  16, &pmic5_nldo,      "vdd-l16"),
+	RPMH_VREG("ldo17",  LDO,  17, &pmic5_nldo,      "vdd-l17"),
+	RPMH_VREG("ldo18",  LDO,  18, &pmic5_nldo,      "vdd-l18-l20-l21"),
+	RPMH_VREG("ldo19",  LDO,  19, &pmic5_nldo,      "vdd-l15-l19-l22"),
+	RPMH_VREG("ldo20",  LDO,  20, &pmic5_nldo,      "vdd-l18-l20-l21"),
+	RPMH_VREG("ldo21",  LDO,  21, &pmic5_nldo,      "vdd-l18-l20-l21"),
+	RPMH_VREG("ldo22",  LDO,  22, &pmic5_nldo,      "vdd-l15-l19-l22"),
 };
 
 static const struct rpmh_vreg_init_data pmcx0102_vreg_data[] = {
diff --git a/drivers/regulator/rt6190-regulator.c b/drivers/regulator/rt6190-regulator.c
index f2cd9540038d91..21ced348a2cfed 100644
--- a/drivers/regulator/rt6190-regulator.c
+++ b/drivers/regulator/rt6190-regulator.c
@@ -108,19 +108,27 @@ static int rt6190_out_enable(struct regulator_dev *rdev)
 	ret = regmap_raw_read(regmap, RT6190_REG_OUTV, out_cfg,
 			      sizeof(out_cfg));
 	if (ret)
-		return ret;
+		goto err_pm_put;
 
 	ret = regulator_enable_regmap(rdev);
 	if (ret)
-		return ret;
+		goto err_pm_put;
 
 	ret = regmap_raw_write(regmap, RT6190_REG_OUTV, out_cfg,
 			       sizeof(out_cfg));
 	if (ret)
-		return ret;
+		goto err_pm_put;
+
+	ret = regmap_update_bits(regmap, RT6190_REG_SET5, RT6190_ENGCP_MASK,
+				 RT6190_ENGCP_MASK);
+	if (ret)
+		goto err_pm_put;
+
+	return 0;
 
-	return regmap_update_bits(regmap, RT6190_REG_SET5, RT6190_ENGCP_MASK,
-				  RT6190_ENGCP_MASK);
+err_pm_put:
+	pm_runtime_put(data->dev);
+	return ret;
 }
 
 static int rt6190_out_disable(struct regulator_dev *rdev)
diff --git a/drivers/regulator/rtq2208-regulator.c b/drivers/regulator/rtq2208-regulator.c
index b4eb5d3eb5bdef..bd48c97b8ada27 100644
--- a/drivers/regulator/rtq2208-regulator.c
+++ b/drivers/regulator/rtq2208-regulator.c
@@ -5,6 +5,7 @@
 #include <linux/util_macros.h>
 #include <linux/module.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/regmap.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/machine.h>
diff --git a/drivers/regulator/tps6105x-regulator.c b/drivers/regulator/tps6105x-regulator.c
index a09c6ae6a0cea2..7f99f3e90bec34 100644
--- a/drivers/regulator/tps6105x-regulator.c
+++ b/drivers/regulator/tps6105x-regulator.c
@@ -65,7 +65,7 @@ static int tps6105x_regulator_probe(struct platform_device *pdev)
 	/* This instance is not set for regulator mode so bail out */
 	if (pdata->mode != TPS6105X_MODE_VOLTAGE) {
 		dev_info(&pdev->dev,
-			"chip not in voltage mode mode, exit probe\n");
+			"chip not in voltage mode, exit probe\n");
 		return 0;
 	}
 
diff --git a/drivers/regulator/tps65185.c b/drivers/regulator/tps65185.c
index 6a3130177e7a3f..8780b73c7aa879 100644
--- a/drivers/regulator/tps65185.c
+++ b/drivers/regulator/tps65185.c
@@ -8,6 +8,7 @@
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/hwmon.h>
@@ -392,8 +393,7 @@ static int tps65185_probe(struct i2c_client *client)
 					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
 					"PGOOD", data);
 	if (ret)
-		return dev_err_probe(&client->dev, ret,
-				     "failed to request power good irq\n");
+		return ret;
 
 	if (client->irq) {
 		ret = devm_request_threaded_irq(&client->dev, client->irq,
@@ -401,8 +401,7 @@ static int tps65185_probe(struct i2c_client *client)
 						IRQF_TRIGGER_LOW | IRQF_ONESHOT,
 						"tps65185", data);
 		if (ret)
-			return dev_err_probe(&client->dev, ret,
-					     "failed to request irq\n");
+			return ret;
 	}
 
 	ret = regmap_update_bits(data->regmap, TPS65185_REG_INT_EN2, BIT(0), BIT(0));
diff --git a/drivers/regulator/tps65219-regulator.c b/drivers/regulator/tps65219-regulator.c
index a667c3f44bb794..c2b474dfedac97 100644
--- a/drivers/regulator/tps65219-regulator.c
+++ b/drivers/regulator/tps65219-regulator.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
diff --git a/drivers/regulator/tps6594-regulator.c b/drivers/regulator/tps6594-regulator.c
index 31a5218d551059..a54d88c58e7bfa 100644
--- a/drivers/regulator/tps6594-regulator.c
+++ b/drivers/regulator/tps6594-regulator.c
@@ -7,6 +7,7 @@
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index 5b56b2dcc725fc..70dec8c3fe4ecb 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -83,9 +83,7 @@ config OMAP_REMOTEPROC
 	select OMAP2PLUS_MBOX
 	help
 	  Say y here to support OMAP's remote processors (dual M3
-	  and DSP on OMAP4) via the remote processor framework.
-
-	  Currently only supported on OMAP4.
+	  and DSP) via the remote processor framework.
 
 	  Usually you want to say Y here, in order to enable multimedia
 	  use-cases to run on your platform (multimedia codecs are
@@ -108,7 +106,7 @@ config OMAP_REMOTEPROC_WATCHDOG
 
 config WKUP_M3_RPROC
 	tristate "AMx3xx Wakeup M3 remoteproc support"
-	depends on SOC_AM33XX || SOC_AM43XX
+	depends on SOC_AM33XX || SOC_AM43XX || COMPILE_TEST
 	help
 	  Say y here to support Wakeup M3 remote processor on TI AM33xx
 	  and AM43xx family of SoCs.
@@ -120,7 +118,7 @@ config WKUP_M3_RPROC
 
 config DA8XX_REMOTEPROC
 	tristate "DA8xx/OMAP-L13x remoteproc support"
-	depends on ARCH_DAVINCI_DA8XX
+	depends on ARCH_DAVINCI_DA8XX || COMPILE_TEST
 	depends on DMA_CMA
 	help
 	  Say y here to support DA8xx/OMAP-L13x remote processors via the
@@ -141,7 +139,7 @@ config DA8XX_REMOTEPROC
 
 config KEYSTONE_REMOTEPROC
 	tristate "Keystone Remoteproc support"
-	depends on ARCH_KEYSTONE
+	depends on ARCH_KEYSTONE || COMPILE_TEST
 	help
 	  Say Y here here to support Keystone remote processors (DSP)
 	  via the remote processor framework.
@@ -238,6 +236,7 @@ config QCOM_Q6V5_PAS
 	select QCOM_PIL_INFO
 	select QCOM_MDT_LOADER
 	select QCOM_Q6V5_COMMON
+	select QCOM_QMI_TMD if NET
 	select QCOM_RPROC_COMMON
 	select QCOM_SCM
 	select QCOM_PAS
diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
index fd60c67ba8a9f7..96d619daf3f5b1 100644
--- a/drivers/remoteproc/imx_dsp_rproc.c
+++ b/drivers/remoteproc/imx_dsp_rproc.c
@@ -726,14 +726,17 @@ static int imx_dsp_rproc_prepare(struct rproc *rproc)
 	struct device *dev = rproc->dev.parent;
 	int ret;
 
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
+		return ret;
+
 	ret = imx_dsp_rproc_add_carveout(priv);
 	if (ret) {
 		dev_err(dev, "failed on imx_dsp_rproc_add_carveout\n");
+		pm_runtime_put_sync(dev);
 		return ret;
 	}
 
-	pm_runtime_get_sync(dev);
-
 	return 0;
 }
 
diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
index 745ce52cd82267..581bb07f5893fb 100644
--- a/drivers/remoteproc/imx_rproc.c
+++ b/drivers/remoteproc/imx_rproc.c
@@ -24,7 +24,6 @@
 #include <linux/regmap.h>
 #include <linux/remoteproc.h>
 #include <linux/scmi_imx_protocol.h>
-#include <linux/workqueue.h>
 
 #include "imx_rproc.h"
 #include "remoteproc_internal.h"
@@ -115,8 +114,6 @@ struct imx_rproc {
 	struct mbox_client		cl;
 	struct mbox_chan		*tx_ch;
 	struct mbox_chan		*rx_ch;
-	struct work_struct		rproc_work;
-	struct workqueue_struct		*workqueue;
 	void __iomem			*rsc_table;
 	struct imx_sc_ipc		*ipc_handle;
 	struct notifier_block		rproc_nb;
@@ -540,6 +537,7 @@ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da,
 	/* parse address translation table */
 	for (i = 0; i < dcfg->att_size; i++) {
 		const struct imx_rproc_att *att = &dcfg->att[i];
+		u64 offset;
 
 		/*
 		 * Ignore entries not belong to current core:
@@ -552,9 +550,11 @@ static int imx_rproc_da_to_sys(struct imx_rproc *priv, u64 da,
 				continue;
 		}
 
-		if (da >= att->da && da + len < att->da + att->size) {
-			unsigned int offset = da - att->da;
+		if (da < att->da)
+			continue;
 
+		offset = da - att->da;
+		if (offset <= att->size && len <= att->size - offset) {
 			*sys = att->sa + offset;
 			if (is_iomem)
 				*is_iomem = att->flags & ATT_IOMEM;
@@ -585,9 +585,14 @@ static void *imx_rproc_da_to_va(struct rproc *rproc, u64 da, size_t len, bool *i
 		return NULL;
 
 	for (i = 0; i < IMX_RPROC_MEM_MAX; i++) {
-		if (sys >= priv->mem[i].sys_addr && sys + len <
-		    priv->mem[i].sys_addr +  priv->mem[i].size) {
-			unsigned int offset = sys - priv->mem[i].sys_addr;
+		u64 offset;
+
+		if (sys < priv->mem[i].sys_addr)
+			continue;
+
+		offset = sys - priv->mem[i].sys_addr;
+		if (offset <= priv->mem[i].size &&
+		    len <= priv->mem[i].size - offset) {
 			/* __force to make sparse happy with type conversion */
 			va = (__force void *)(priv->mem[i].cpu_addr + offset);
 			break;
@@ -860,21 +865,11 @@ static int imx_rproc_notified_idr_cb(int id, void *ptr, void *data)
 	return 0;
 }
 
-static void imx_rproc_vq_work(struct work_struct *work)
-{
-	struct imx_rproc *priv = container_of(work, struct imx_rproc,
-					      rproc_work);
-	struct rproc *rproc = priv->rproc;
-
-	idr_for_each(&rproc->notifyids, imx_rproc_notified_idr_cb, rproc);
-}
-
 static void imx_rproc_rx_callback(struct mbox_client *cl, void *msg)
 {
 	struct rproc *rproc = dev_get_drvdata(cl->dev);
-	struct imx_rproc *priv = rproc->priv;
 
-	queue_work(priv->workqueue, &priv->rproc_work);
+	idr_for_each(&rproc->notifyids, imx_rproc_notified_idr_cb, rproc);
 }
 
 static int imx_rproc_xtr_mbox_init(struct rproc *rproc, bool tx_block)
@@ -1239,13 +1234,6 @@ static int imx_rproc_sys_off_handler(struct sys_off_data *data)
 	return NOTIFY_DONE;
 }
 
-static void imx_rproc_destroy_workqueue(void *data)
-{
-	struct workqueue_struct *workqueue = data;
-
-	destroy_workqueue(workqueue);
-}
-
 static int imx_rproc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -1273,17 +1261,6 @@ static int imx_rproc_probe(struct platform_device *pdev)
 		priv->ops = dcfg->ops;
 
 	dev_set_drvdata(dev, rproc);
-	priv->workqueue = create_workqueue(dev_name(dev));
-	if (!priv->workqueue) {
-		dev_err(dev, "cannot create workqueue\n");
-		return -ENOMEM;
-	}
-
-	ret = devm_add_action_or_reset(dev, imx_rproc_destroy_workqueue, priv->workqueue);
-	if (ret)
-		return dev_err_probe(dev, ret, "Failed to add devm destroy workqueue action\n");
-
-	INIT_WORK(&priv->rproc_work, imx_rproc_vq_work);
 
 	ret = imx_rproc_xtr_mbox_init(rproc, true);
 	if (ret)
diff --git a/drivers/remoteproc/omap_remoteproc.c b/drivers/remoteproc/omap_remoteproc.c
index 6ed0f28edac9c8..1e96506ce7ca79 100644
--- a/drivers/remoteproc/omap_remoteproc.c
+++ b/drivers/remoteproc/omap_remoteproc.c
@@ -499,7 +499,7 @@ static void omap_rproc_mbox_callback(struct mbox_client *client, void *data)
 						client);
 	struct device *dev = oproc->rproc->dev.parent;
 	const char *name = oproc->rproc->name;
-	u32 msg = (u32)data;
+	mbox_msg_t msg = omap_mbox_from_message(data);
 
 	dev_dbg(dev, "mbox msg: 0x%x\n", msg);
 
@@ -550,7 +550,7 @@ static void omap_rproc_kick(struct rproc *rproc, int vqid)
 	}
 
 	/* send the index of the triggered virtqueue in the mailbox payload */
-	ret = mbox_send_message(oproc->mbox, (void *)vqid);
+	ret = mbox_send_message(oproc->mbox, omap_mbox_to_message(vqid));
 	if (ret < 0)
 		dev_err(dev, "failed to send mailbox message, status = %d\n",
 			ret);
@@ -628,7 +628,7 @@ static int omap_rproc_start(struct rproc *rproc)
 	 * Note that the reply will _not_ arrive immediately: this message
 	 * will wait in the mailbox fifo until the remote processor is booted.
 	 */
-	ret = mbox_send_message(oproc->mbox, (void *)RP_MBOX_ECHO_REQUEST);
+	ret = mbox_send_message(oproc->mbox, omap_mbox_to_message(RP_MBOX_ECHO_REQUEST));
 	if (ret < 0) {
 		dev_err(dev, "mbox_send_message failed: %d\n", ret);
 		goto put_mbox;
@@ -777,13 +777,13 @@ static int _omap_rproc_suspend(struct rproc *rproc, bool auto_suspend)
 	struct omap_rproc *oproc = rproc->priv;
 	unsigned long to = msecs_to_jiffies(DEF_SUSPEND_TIMEOUT);
 	unsigned long ta = jiffies + to;
-	u32 suspend_msg = auto_suspend ?
+	mbox_msg_t suspend_msg = auto_suspend ?
 				RP_MBOX_SUSPEND_AUTO : RP_MBOX_SUSPEND_SYSTEM;
 	int ret;
 
 	reinit_completion(&oproc->pm_comp);
 	oproc->suspend_acked = false;
-	ret = mbox_send_message(oproc->mbox, (void *)suspend_msg);
+	ret = mbox_send_message(oproc->mbox, omap_mbox_to_message(suspend_msg));
 	if (ret < 0) {
 		dev_err(dev, "PM mbox_send_message failed: %d\n", ret);
 		return ret;
@@ -1208,7 +1208,7 @@ static int omap_rproc_of_get_internal_memories(struct platform_device *pdev,
 		oproc->mem[i].dev_addr = data->mems[i].dev_addr;
 		oproc->mem[i].size = resource_size(res);
 
-		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%x va %p da 0x%x\n",
+		dev_dbg(dev, "memory %8s: bus addr %pa size 0x%zx va %p da 0x%x\n",
 			data->mems[i].name, &oproc->mem[i].bus_addr,
 			oproc->mem[i].size, oproc->mem[i].cpu_addr,
 			oproc->mem[i].dev_addr);
diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c
index 39654206781d22..46d9169a37f6f6 100644
--- a/drivers/remoteproc/qcom_q6v5_adsp.c
+++ b/drivers/remoteproc/qcom_q6v5_adsp.c
@@ -104,6 +104,7 @@ struct qcom_adsp {
 	struct completion stop_done;
 
 	phys_addr_t mem_phys;
+	unsigned long iova;
 	phys_addr_t mem_reloc;
 	void __iomem *mem_region;
 	size_t mem_size;
@@ -333,7 +334,7 @@ static void adsp_unmap_carveout(struct rproc *rproc)
 	struct qcom_adsp *adsp = rproc->priv;
 
 	if (adsp->has_iommu)
-		iommu_unmap(rproc->domain, adsp->mem_phys, adsp->mem_size);
+		iommu_unmap(rproc->domain, adsp->iova, adsp->mem_size);
 }
 
 static int adsp_map_carveout(struct rproc *rproc)
@@ -341,7 +342,6 @@ static int adsp_map_carveout(struct rproc *rproc)
 	struct qcom_adsp *adsp = rproc->priv;
 	struct of_phandle_args args;
 	long long sid;
-	unsigned long iova;
 	int ret;
 
 	if (!adsp->has_iommu)
@@ -358,9 +358,9 @@ static int adsp_map_carveout(struct rproc *rproc)
 	of_node_put(args.np);
 
 	/* Add SID configuration for ADSP Firmware to SMMU */
-	iova =  adsp->mem_phys | (sid << 32);
+	adsp->iova = adsp->mem_phys | (sid << 32);
 
-	ret = iommu_map(rproc->domain, iova, adsp->mem_phys,
+	ret = iommu_map(rproc->domain, adsp->iova, adsp->mem_phys,
 			adsp->mem_size,	IOMMU_READ | IOMMU_WRITE,
 			GFP_KERNEL);
 	if (ret) {
@@ -430,7 +430,7 @@ static int adsp_start(struct rproc *rproc)
 		goto disable_adsp_clks;
 	}
 
-	ret = qcom_q6v5_wait_for_start(&adsp->q6v5, msecs_to_jiffies(5 * HZ));
+	ret = qcom_q6v5_wait_for_start(&adsp->q6v5, msecs_to_jiffies(5000));
 	if (ret == -ETIMEDOUT) {
 		dev_err(adsp->dev, "start timed out\n");
 		goto disable_adsp_clks;
diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c
index bef198b9ee6367..2f71ed2feff696 100644
--- a/drivers/remoteproc/qcom_q6v5_mss.c
+++ b/drivers/remoteproc/qcom_q6v5_mss.c
@@ -2079,7 +2079,16 @@ static int q6v5_probe(struct platform_device *pdev)
 	if (!desc)
 		return -EINVAL;
 
-	if (desc->need_mem_protection && !qcom_pas_is_available())
+	/*
+	 * Memory protection is done through qcom_scm_assign_mem(), which needs
+	 * SCM but not PAS. Only the memory setup path issues PAS calls, so
+	 * requiring PAS for every need_mem_protection platform prevents the
+	 * modem from probing at all on TZ firmware that offers no PAS.
+	 */
+	if (desc->need_mem_protection && !qcom_scm_is_available())
+		return -EPROBE_DEFER;
+
+	if (desc->need_pas_mem_setup && !qcom_pas_is_available())
 		return -EPROBE_DEFER;
 
 	mba_image = desc->hexagon_mba_image;
diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c
index ca8e61254c449d..2e1e39826ffa0f 100644
--- a/drivers/remoteproc/qcom_q6v5_pas.c
+++ b/drivers/remoteproc/qcom_q6v5_pas.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2016 Linaro Ltd
  * Copyright (C) 2014 Sony Mobile Communications AB
  * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/clk.h>
@@ -26,8 +27,11 @@
 #include <linux/regulator/consumer.h>
 #include <linux/remoteproc.h>
 #include <linux/soc/qcom/mdt_loader.h>
+#include <linux/soc/qcom/qmi_tmd.h>
 #include <linux/soc/qcom/smem.h>
 #include <linux/soc/qcom/smem_state.h>
+#include <dt-bindings/power/qcom,rpmhpd.h>
+#include <dt-bindings/thermal/qcom,pas.h>
 
 #include "qcom_common.h"
 #include "qcom_pil_info.h"
@@ -38,6 +42,16 @@
 
 #define MAX_ASSIGN_COUNT 3
 
+/**
+ * struct tmd_name - TMD device name to cooling-device index mapping
+ * @name: TMD device name
+ * @id: Cooling-device index used as #cooling-cells cell 0 in DT
+ */
+struct tmd_name {
+	const char *name;
+	int id;
+};
+
 struct qcom_pas_data {
 	int crash_reason_smem;
 	const char *firmware_name;
@@ -51,6 +65,8 @@ struct qcom_pas_data {
 	bool decrypt_shutdown;
 
 	char **proxy_pd_names;
+	const unsigned int *proxy_pd_performance_states;
+	unsigned int num_proxy_pd_performance_states;
 
 	const char *load_state;
 	const char *ssr_name;
@@ -58,6 +74,10 @@ struct qcom_pas_data {
 	int ssctl_id;
 	unsigned int smem_host_id;
 
+	unsigned int tmd_instance_id;
+	const struct tmd_name *tmd_name;
+	int num_tmd;
+
 	int region_assign_idx;
 	int region_assign_count;
 	bool region_assign_shared;
@@ -79,6 +99,7 @@ struct qcom_pas {
 	struct regulator *px_supply;
 
 	struct device *proxy_pds[3];
+	const unsigned int *proxy_pd_performance_states;
 
 	int proxy_pd_count;
 
@@ -123,6 +144,8 @@ struct qcom_pas {
 
 	struct qcom_pas_context *pas_ctx;
 	struct qcom_pas_context *dtb_pas_ctx;
+
+	struct qmi_tmd_client *tmd_inst;
 };
 
 static void qcom_pas_segment_dump(struct rproc *rproc,
@@ -167,7 +190,12 @@ static int qcom_pas_pds_enable(struct qcom_pas *pas, struct device **pds,
 	int i;
 
 	for (i = 0; i < pd_count; i++) {
-		dev_pm_genpd_set_performance_state(pds[i], INT_MAX);
+		unsigned int state = INT_MAX;
+
+		if (pas->proxy_pd_performance_states)
+			state = pas->proxy_pd_performance_states[i];
+
+		dev_pm_genpd_set_performance_state(pds[i], state);
 		ret = pm_runtime_get_sync(pds[i]);
 		if (ret < 0) {
 			pm_runtime_put_noidle(pds[i]);
@@ -405,6 +433,7 @@ static int qcom_pas_stop(struct rproc *rproc)
 {
 	struct qcom_pas *pas = rproc->priv;
 	int handover;
+	int dtb_ret;
 	int ret;
 
 	ret = qcom_q6v5_request_stop(&pas->q6v5, pas->sysmon);
@@ -419,9 +448,12 @@ static int qcom_pas_stop(struct rproc *rproc)
 		dev_err(pas->dev, "failed to shutdown: %d\n", ret);
 
 	if (pas->dtb_pas_id) {
-		ret = qcom_pas_shutdown(pas->dtb_pas_id);
-		if (ret)
-			dev_err(pas->dev, "failed to shutdown dtb: %d\n", ret);
+		dtb_ret = qcom_pas_shutdown(pas->dtb_pas_id);
+		if (dtb_ret)
+			dev_err(pas->dev, "failed to shutdown dtb: %d\n", dtb_ret);
+
+		if (!ret && dtb_ret)
+			ret = dtb_ret;
 
 		qcom_pas_unmap_carveout(rproc, pas->dtb_mem_phys, pas->dtb_mem_size);
 	}
@@ -524,7 +556,6 @@ static int qcom_pas_attach(struct rproc *rproc)
 	int ret;
 
 	pas->q6v5.handover_issued = true;
-	enable_irq(pas->q6v5.handover_irq);
 
 	pas->q6v5.running = true;
 	ret = irq_get_irqchip_state(pas->q6v5.fatal_irq,
@@ -570,7 +601,6 @@ static int qcom_pas_attach(struct rproc *rproc)
 	pas->rproc->state = RPROC_OFFLINE;
 	ret = -EINVAL;
 disable_running:
-	disable_irq(pas->q6v5.handover_irq);
 	pas->q6v5.running = false;
 
 	return ret;
@@ -816,6 +846,64 @@ static void qcom_pas_unassign_memory_region(struct qcom_pas *pas)
 	}
 }
 
+static int qcom_pas_setup_tmd(struct qcom_pas *pas, const struct qcom_pas_data *desc)
+{
+	struct qmi_tmd_client *tmd_inst;
+	const struct tmd_name *tmd;
+	const char **tmd_names;
+	int i, ret;
+
+	if (!device_property_present(pas->dev, "#cooling-cells"))
+		return 0;
+
+	if (!desc->tmd_name || desc->num_tmd == 0)
+		return 0;
+
+	tmd_names = devm_kcalloc(pas->dev, desc->num_tmd,
+				 sizeof(*tmd_names), GFP_KERNEL);
+	if (!tmd_names)
+		return -ENOMEM;
+
+	for (i = 0; i < desc->num_tmd; i++) {
+		tmd = &desc->tmd_name[i];
+
+		if (tmd->id >= desc->num_tmd) {
+			dev_err(pas->dev, "Invalid TMD id %d for '%s'\n",
+				tmd->id, tmd->name);
+			return -EINVAL;
+		}
+
+		if (tmd_names[tmd->id]) {
+			dev_err(pas->dev, "Duplicate TMD id %d for '%s'\n",
+				tmd->id, tmd->name);
+			return -EINVAL;
+		}
+
+		tmd_names[tmd->id] = tmd->name;
+	}
+
+	for (i = 0; i < desc->num_tmd; i++) {
+		if (!tmd_names[i]) {
+			dev_err(pas->dev, "Missing TMD mapping for id %d\n", i);
+			return -EINVAL;
+		}
+	}
+
+	tmd_inst = qmi_tmd_init(pas->dev, desc->tmd_instance_id, tmd_names,
+				desc->num_tmd);
+	if (IS_ERR(tmd_inst)) {
+		ret = PTR_ERR(tmd_inst);
+		if (ret == -ENODEV)
+			return 0;
+
+		return ret;
+	}
+
+	pas->tmd_inst = tmd_inst;
+
+	return 0;
+}
+
 static int qcom_pas_probe(struct platform_device *pdev)
 {
 	const struct qcom_pas_data *desc;
@@ -871,6 +959,7 @@ static int qcom_pas_probe(struct platform_device *pdev)
 	pas->info_name = desc->sysmon_name;
 	pas->smem_host_id = desc->smem_host_id;
 	pas->decrypt_shutdown = desc->decrypt_shutdown;
+	pas->proxy_pd_performance_states = desc->proxy_pd_performance_states;
 	pas->region_assign_idx = desc->region_assign_idx;
 	pas->region_assign_count = min_t(int, MAX_ASSIGN_COUNT, desc->region_assign_count);
 	pas->region_assign_vmid = desc->region_assign_vmid;
@@ -906,6 +995,15 @@ static int qcom_pas_probe(struct platform_device *pdev)
 		goto unassign_mem;
 	pas->proxy_pd_count = ret;
 
+	if (desc->proxy_pd_performance_states &&
+	    desc->num_proxy_pd_performance_states != pas->proxy_pd_count) {
+		dev_err(&pdev->dev,
+			"proxy_pd_performance_states count %u != pd count %d\n",
+			desc->num_proxy_pd_performance_states, pas->proxy_pd_count);
+		ret = -EINVAL;
+		goto detach_proxy_pds;
+	}
+
 	ret = qcom_q6v5_init(&pas->q6v5, pdev, rproc, desc->crash_reason_smem,
 			     desc->load_state, qcom_pas_handover);
 	if (ret)
@@ -929,16 +1027,24 @@ static int qcom_pas_probe(struct platform_device *pdev)
 	if (desc->early_boot)
 		pas->rproc->state = RPROC_DETACHED;
 
-	ret = rproc_add(rproc);
+	ret = qcom_pas_setup_tmd(pas, desc);
 	if (ret)
 		goto remove_ssr_sysmon;
 
+	ret = rproc_add(rproc);
+	if (ret)
+		goto remove_setup_tmd;
+
 	node = of_get_compatible_child(pdev->dev.of_node, "qcom,bam-dmux");
 	pas->bam_dmux = of_platform_device_create(node, NULL, &pdev->dev);
 	of_node_put(node);
 
 	return 0;
 
+remove_setup_tmd:
+	if (pas->tmd_inst)
+		qmi_tmd_exit(pas->tmd_inst);
+
 remove_ssr_sysmon:
 	qcom_remove_ssr_subdev(rproc, &pas->ssr_subdev);
 	qcom_remove_sysmon_subdev(pas->sysmon);
@@ -966,6 +1072,9 @@ static void qcom_pas_remove(struct platform_device *pdev)
 
 	rproc_del(pas->rproc);
 
+	if (pas->tmd_inst)
+		qmi_tmd_exit(pas->tmd_inst);
+
 	qcom_q6v5_deinit(&pas->q6v5);
 	qcom_pas_unassign_memory_region(pas);
 	qcom_remove_glink_subdev(pas->rproc, &pas->glink_subdev);
@@ -977,6 +1086,15 @@ static void qcom_pas_remove(struct platform_device *pdev)
 	device_init_wakeup(pas->dev, false);
 }
 
+static const struct tmd_name cdsp_tmd_name[] = {
+	{ .name = "cdsp_sw", .id = QCOM_TMD_CDSP_SW },
+};
+
+static const struct tmd_name modem_tmd_name[] = {
+	{ .name = "pa", .id = QCOM_TMD_PA },
+	{ .name = "modem", .id = QCOM_TMD_MODEM },
+};
+
 static const struct qcom_pas_data adsp_resource_init = {
 	.crash_reason_smem = 423,
 	.firmware_name = "adsp.mdt",
@@ -1134,6 +1252,9 @@ static const struct qcom_pas_data sa8775p_cdsp0_resource = {
 	.ssr_name = "cdsp",
 	.sysmon_name = "cdsp",
 	.ssctl_id = 0x17,
+	.tmd_instance_id = 0x43,
+	.tmd_name = cdsp_tmd_name,
+	.num_tmd = ARRAY_SIZE(cdsp_tmd_name),
 };
 
 static const struct qcom_pas_data sa8775p_cdsp1_resource = {
@@ -1152,6 +1273,9 @@ static const struct qcom_pas_data sa8775p_cdsp1_resource = {
 	.ssr_name = "cdsp1",
 	.sysmon_name = "cdsp1",
 	.ssctl_id = 0x20,
+	.tmd_instance_id = 0x44,
+	.tmd_name = cdsp_tmd_name,
+	.num_tmd = ARRAY_SIZE(cdsp_tmd_name),
 };
 
 static const struct qcom_pas_data sdm845_cdsp_resource_init = {
@@ -1179,6 +1303,9 @@ static const struct qcom_pas_data sm6350_cdsp_resource = {
 	.ssr_name = "cdsp",
 	.sysmon_name = "cdsp",
 	.ssctl_id = 0x17,
+	.tmd_instance_id = 0x43,
+	.tmd_name = cdsp_tmd_name,
+	.num_tmd = ARRAY_SIZE(cdsp_tmd_name),
 };
 
 static const struct qcom_pas_data sm8150_cdsp_resource = {
@@ -1194,6 +1321,9 @@ static const struct qcom_pas_data sm8150_cdsp_resource = {
 	.ssr_name = "cdsp",
 	.sysmon_name = "cdsp",
 	.ssctl_id = 0x17,
+	.tmd_instance_id = 0x43,
+	.tmd_name = cdsp_tmd_name,
+	.num_tmd = ARRAY_SIZE(cdsp_tmd_name),
 };
 
 static const struct qcom_pas_data sm8250_cdsp_resource = {
@@ -1278,6 +1408,9 @@ static const struct qcom_pas_data x1e80100_cdsp_resource = {
 	.ssr_name = "cdsp",
 	.sysmon_name = "cdsp",
 	.ssctl_id = 0x17,
+	.tmd_instance_id = 0x43,
+	.tmd_name = cdsp_tmd_name,
+	.num_tmd = ARRAY_SIZE(cdsp_tmd_name),
 };
 
 static const struct qcom_pas_data sm8350_cdsp_resource = {
@@ -1346,6 +1479,9 @@ static const struct qcom_pas_data mpss_resource_init = {
 	.ssr_name = "mpss",
 	.sysmon_name = "modem",
 	.ssctl_id = 0x12,
+	.tmd_instance_id = 0x0,
+	.tmd_name = modem_tmd_name,
+	.num_tmd = ARRAY_SIZE(modem_tmd_name),
 };
 
 static const struct qcom_pas_data sc8180x_mpss_resource = {
@@ -1796,6 +1932,33 @@ static const struct qcom_pas_data glymur_soccp_resource = {
 	.needs_tzmem = true,
 };
 
+static const struct qcom_pas_data hawi_cdsp_resource = {
+	.crash_reason_smem = 601,
+	.firmware_name = "cdsp.mdt",
+	.dtb_firmware_name = "cdsp_dtb.mdt",
+	.pas_id = 18,
+	.dtb_pas_id = 0x25,
+	.minidump_id = 7,
+	.auto_boot = true,
+	.proxy_pd_names = (char*[]){
+		"cx",
+		"mxc",
+		"nsp",
+		NULL
+	},
+	.proxy_pd_performance_states = (const unsigned int[]){
+		RPMH_REGULATOR_LEVEL_TURBO,
+		RPMH_REGULATOR_LEVEL_TURBO,
+		RPMH_REGULATOR_LEVEL_NOM,
+	},
+	.num_proxy_pd_performance_states = 3,
+	.load_state = "cdsp",
+	.ssr_name = "cdsp",
+	.sysmon_name = "cdsp",
+	.ssctl_id = 0x17,
+	.smem_host_id = 5,
+};
+
 static const struct qcom_pas_data eliza_cdsp_resource = {
 	.crash_reason_smem = 601,
 	.firmware_name = "cdsp.mbn",
@@ -1825,6 +1988,7 @@ static const struct of_device_id qcom_pas_of_match[] = {
 	{ .compatible = "qcom,eliza-adsp-pas", .data = &sm8550_adsp_resource },
 	{ .compatible = "qcom,eliza-cdsp-pas", .data = &eliza_cdsp_resource },
 	{ .compatible = "qcom,glymur-soccp-pas", .data = &glymur_soccp_resource },
+	{ .compatible = "qcom,hawi-cdsp-pas", .data = &hawi_cdsp_resource },
 	{ .compatible = "qcom,kaanapali-soccp-pas", .data = &kaanapali_soccp_resource },
 	{ .compatible = "qcom,milos-adsp-pas", .data = &sm8550_adsp_resource },
 	{ .compatible = "qcom,milos-cdsp-pas", .data = &milos_cdsp_resource },
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index 1ed40671484918..4bf69003b7da2f 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -574,6 +574,9 @@ static int rproc_handle_trace(struct rproc *rproc, void *ptr,
 	/* create the debugfs entry */
 	trace->tfile = rproc_create_trace_file(name, rproc, trace);
 
+	/* keep the name for the diagnostic in rproc_trace_read() */
+	strscpy(trace->trace_mem.name, name, sizeof(trace->trace_mem.name));
+
 	list_add_tail(&trace->node, &rproc->traces);
 
 	rproc->num_traces++;
@@ -2410,6 +2413,7 @@ static void rproc_type_release(struct device *dev)
 
 	dev_info(&rproc->dev, "releasing %s\n", rproc->name);
 
+	cleanup_srcu_struct(&rproc->vq_srcu);
 	idr_destroy(&rproc->notifyids);
 
 	if (rproc->index >= 0)
@@ -2507,6 +2511,11 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 	if (!rproc)
 		return NULL;
 
+	if (init_srcu_struct(&rproc->vq_srcu)) {
+		kfree(rproc);
+		return NULL;
+	}
+
 	rproc->priv = &rproc[1];
 	rproc->auto_boot = true;
 	rproc->elf_class = ELFCLASSNONE;
@@ -2571,6 +2580,9 @@ EXPORT_SYMBOL(rproc_alloc);
  *
  * If no one holds any reference to rproc anymore, then its refcount would
  * now drop to zero, and it would be freed.
+ *
+ * Context: Any context, but the last reference must not be dropped from
+ * atomic context.
  */
 void rproc_free(struct rproc *rproc)
 {
@@ -2586,6 +2598,9 @@ EXPORT_SYMBOL(rproc_free);
  *
  * If no one holds any reference to rproc anymore, then its refcount would
  * now drop to zero, and it would be freed.
+ *
+ * Context: Any context, but the last reference must not be dropped from
+ * atomic context.
  */
 void rproc_put(struct rproc *rproc)
 {
diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c
index b86c1d09c70c0f..e8e7b6a3769b82 100644
--- a/drivers/remoteproc/remoteproc_debugfs.c
+++ b/drivers/remoteproc/remoteproc_debugfs.c
@@ -347,8 +347,8 @@ static int rproc_rsc_table_show(struct seq_file *seq, void *p)
 			}
 			break;
 		default:
-			seq_printf(seq, "Unknown resource type found: %d [hdr: %pK]\n",
-				   hdr->type, hdr);
+			seq_printf(seq, "Unknown resource type found: %d\n",
+				   hdr->type);
 			break;
 		}
 	}
@@ -367,7 +367,7 @@ static int rproc_carveouts_show(struct seq_file *seq, void *p)
 	list_for_each_entry(carveout, &rproc->carveouts, node) {
 		seq_puts(seq, "Carveout memory entry:\n");
 		seq_printf(seq, "\tName: %s\n", carveout->name);
-		seq_printf(seq, "\tVirtual address: %pK\n", carveout->va);
+		seq_printf(seq, "\tVirtual address: %p\n", carveout->va);
 		seq_printf(seq, "\tDMA address: %pad\n", &carveout->dma);
 		seq_printf(seq, "\tDevice address: 0x%x\n", carveout->da);
 		seq_printf(seq, "\tLength: 0x%zx Bytes\n\n", carveout->len);
diff --git a/drivers/remoteproc/remoteproc_elf_loader.c b/drivers/remoteproc/remoteproc_elf_loader.c
index 94177e4160473c..da3cddbe7d4cb3 100644
--- a/drivers/remoteproc/remoteproc_elf_loader.c
+++ b/drivers/remoteproc/remoteproc_elf_loader.c
@@ -46,8 +46,9 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw)
 	struct elf32_hdr *ehdr;
 	u32 elf_shdr_get_size;
 	u64 phoff, shoff;
+	size_t shend;
 	char class;
-	u16 phnum;
+	u16 phnum, shnum, shstrndx;
 
 	if (!fw) {
 		dev_err(dev, "failed to load %s\n", name);
@@ -90,9 +91,27 @@ int rproc_elf_sanity_check(struct rproc *rproc, const struct firmware *fw)
 	phoff = elf_hdr_get_e_phoff(class, fw->data);
 	shoff = elf_hdr_get_e_shoff(class, fw->data);
 	phnum =  elf_hdr_get_e_phnum(class, fw->data);
+	shnum = elf_hdr_get_e_shnum(class, fw->data);
+	shstrndx = elf_hdr_get_e_shstrndx(class, fw->data);
 	elf_shdr_get_size = elf_size_of_shdr(class);
 
-	if (fw->size < shoff + elf_shdr_get_size) {
+	/* keeps shoff in size_t range for the two bounds below */
+	if (shoff > fw->size) {
+		dev_err(dev, "Section header table is out of bounds\n");
+		return -EINVAL;
+	}
+
+	if (shnum) {
+		shend = size_add(size_mul(elf_shdr_get_size, shnum), shoff);
+		if (shend > fw->size) {
+			dev_err(dev, "Section headers are out of bounds\n");
+			return -EINVAL;
+		}
+	}
+
+	/* find_table() reads the header at shstrndx even with no sections */
+	shend = size_add(size_mul(elf_shdr_get_size, (size_t)shstrndx + 1), shoff);
+	if (shend > fw->size) {
 		dev_err(dev, "Image is too small\n");
 		return -EINVAL;
 	}
diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c
index 925b0cdbe57788..c37736ff4acc63 100644
--- a/drivers/remoteproc/remoteproc_sysfs.c
+++ b/drivers/remoteproc/remoteproc_sysfs.c
@@ -76,7 +76,7 @@ static const char * const rproc_coredump_str[] = {
 	[RPROC_COREDUMP_INLINE]		= "inline",
 };
 
-/* Expose the current coredump configuration via debugfs */
+/* Expose the current coredump configuration via sysfs */
 static ssize_t coredump_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
 {
@@ -87,14 +87,15 @@ static ssize_t coredump_show(struct device *dev,
 
 /*
  * By writing to the 'coredump' sysfs entry, we control the behavior of the
- * coredump mechanism dynamically. The default value of this entry is "default".
+ * coredump mechanism dynamically. The default value of this entry is
+ * "disabled".
  *
  * The 'coredump' sysfs entry supports these commands:
  *
  * disabled:	This is the default coredump mechanism. Recovery will proceed
  *		without collecting any dump.
  *
- * default:	When the remoteproc crashes the entire coredump will be
+ * enabled:	When the remoteproc crashes the entire coredump will be
  *		copied to a separate buffer and exposed to userspace.
  *
  * inline:	The coredump will not be copied to a separate buffer and the
diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c
index d5e9ff045a28aa..7fefb4bd7adcb6 100644
--- a/drivers/remoteproc/remoteproc_virtio.c
+++ b/drivers/remoteproc/remoteproc_virtio.c
@@ -23,6 +23,7 @@
 #include <linux/err.h>
 #include <linux/kref.h>
 #include <linux/slab.h>
+#include <linux/srcu.h>
 
 #include "remoteproc_internal.h"
 
@@ -88,15 +89,23 @@ static bool rproc_virtio_notify(struct virtqueue *vq)
  */
 irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int notifyid)
 {
+	irqreturn_t ret = IRQ_NONE;
 	struct rproc_vring *rvring;
+	struct virtqueue *vq;
+	int idx;
 
 	dev_dbg(&rproc->dev, "vq index %d is interrupted\n", notifyid);
 
+	idx = srcu_read_lock(&rproc->vq_srcu);
+
 	rvring = idr_find(&rproc->notifyids, notifyid);
-	if (!rvring || !rvring->vq)
-		return IRQ_NONE;
+	vq = rvring ? READ_ONCE(rvring->vq) : NULL;
+	if (vq)
+		ret = vring_interrupt(0, vq);
 
-	return vring_interrupt(0, rvring->vq);
+	srcu_read_unlock(&rproc->vq_srcu, idx);
+
+	return ret;
 }
 EXPORT_SYMBOL(rproc_vq_interrupt);
 
@@ -153,7 +162,7 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev,
 
 	vq->num_max = num;
 
-	rvring->vq = vq;
+	WRITE_ONCE(rvring->vq, vq);
 	vq->priv = rvring;
 
 	/* Update vring in resource table */
@@ -165,14 +174,20 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev,
 
 static void __rproc_virtio_del_vqs(struct virtio_device *vdev)
 {
+	struct rproc *rproc = vdev_to_rproc(vdev);
 	struct virtqueue *vq, *n;
 	struct rproc_vring *rvring;
 
-	list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
+	list_for_each_entry(vq, &vdev->vqs, list) {
 		rvring = vq->priv;
-		rvring->vq = NULL;
-		vring_del_virtqueue(vq);
+		WRITE_ONCE(rvring->vq, NULL);
 	}
+
+	/* Synchronize with rproc_vq_interrupt() callers that found a queue. */
+	synchronize_srcu(&rproc->vq_srcu);
+
+	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
+		vring_del_virtqueue(vq);
 }
 
 static void rproc_virtio_del_vqs(struct virtio_device *vdev)
@@ -242,6 +257,13 @@ static void rproc_virtio_reset(struct virtio_device *vdev)
 	dev_dbg(&vdev->dev, "reset !\n");
 }
 
+static void rproc_virtio_synchronize_cbs(struct virtio_device *vdev)
+{
+	struct rproc *rproc = vdev_to_rproc(vdev);
+
+	synchronize_srcu(&rproc->vq_srcu);
+}
+
 /* provide the vdev features as retrieved from the firmware */
 static u64 rproc_virtio_get_features(struct virtio_device *vdev)
 {
@@ -330,6 +352,7 @@ static const struct virtio_config_ops rproc_virtio_config_ops = {
 	.find_vqs	= rproc_virtio_find_vqs,
 	.del_vqs	= rproc_virtio_del_vqs,
 	.reset		= rproc_virtio_reset,
+	.synchronize_cbs = rproc_virtio_synchronize_cbs,
 	.set_status	= rproc_virtio_set_status,
 	.get_status	= rproc_virtio_get_status,
 	.get		= rproc_virtio_get,
diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c
index 1fe4cdc0a13a10..a47e040c6db89d 100644
--- a/drivers/remoteproc/stm32_rproc.c
+++ b/drivers/remoteproc/stm32_rproc.c
@@ -675,7 +675,7 @@ static int stm32_rproc_parse_dt(struct platform_device *pdev,
 	int err, irq;
 
 	irq = platform_get_irq_optional(pdev, 0);
-	if (irq == -EPROBE_DEFER)
+	if (irq < 0 && irq != -ENXIO)
 		return irq;
 
 	if (irq > 0) {
diff --git a/drivers/remoteproc/ti_k3_common.c b/drivers/remoteproc/ti_k3_common.c
index 3cb8ae5d72f674..4b6da3363f6c2b 100644
--- a/drivers/remoteproc/ti_k3_common.c
+++ b/drivers/remoteproc/ti_k3_common.c
@@ -54,7 +54,7 @@ void k3_rproc_mbox_callback(struct mbox_client *client, void *data)
 	struct k3_rproc *kproc = container_of(client, struct k3_rproc, client);
 	struct device *dev = kproc->rproc->dev.parent;
 	struct rproc *rproc = kproc->rproc;
-	u32 msg = (u32)(uintptr_t)(data);
+	mbox_msg_t msg = omap_mbox_from_message(data);
 
 	dev_dbg(dev, "mbox msg: 0x%x\n", msg);
 
@@ -94,15 +94,9 @@ void k3_rproc_kick(struct rproc *rproc, int vqid)
 {
 	struct k3_rproc *kproc = rproc->priv;
 	struct device *dev = kproc->dev;
-	u32 msg = (u32)vqid;
 	int ret;
 
-	/*
-	 * Send the index of the triggered virtqueue in the mailbox payload.
-	 * NOTE: msg is cast to uintptr_t to prevent compiler warnings when
-	 * void* is 64bit. It is safely cast back to u32 in the mailbox driver.
-	 */
-	ret = mbox_send_message(kproc->mbox, (void *)(uintptr_t)msg);
+	ret = mbox_send_message(kproc->mbox, omap_mbox_to_message(vqid));
 	if (ret < 0)
 		dev_err(dev, "failed to send mailbox message, status = %d\n",
 			ret);
diff --git a/drivers/remoteproc/ti_k3_common.h b/drivers/remoteproc/ti_k3_common.h
index aee3c28dbe510d..c8430e42225fec 100644
--- a/drivers/remoteproc/ti_k3_common.h
+++ b/drivers/remoteproc/ti_k3_common.h
@@ -21,6 +21,10 @@
 #ifndef REMOTEPROC_TI_K3_COMMON_H
 #define REMOTEPROC_TI_K3_COMMON_H
 
+#include <linux/mailbox_client.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
 #define KEYSTONE_RPROC_LOCAL_ADDRESS_MASK	(SZ_16M - 1)
 
 /**
diff --git a/drivers/remoteproc/ti_k3_dsp_remoteproc.c b/drivers/remoteproc/ti_k3_dsp_remoteproc.c
index d6ceea6dc920ed..6582c7404e54ca 100644
--- a/drivers/remoteproc/ti_k3_dsp_remoteproc.c
+++ b/drivers/remoteproc/ti_k3_dsp_remoteproc.c
@@ -7,11 +7,8 @@
  */
 
 #include <linux/io.h>
-#include <linux/mailbox_client.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_reserved_mem.h>
-#include <linux/omap-mailbox.h>
 #include <linux/platform_device.h>
 #include <linux/remoteproc.h>
 #include <linux/reset.h>
@@ -154,6 +151,10 @@ static int k3_dsp_rproc_probe(struct platform_device *pdev)
 		dev_info(dev, "configured DSP for remoteproc mode\n");
 	}
 
+	ret = dma_coerce_mask_and_coherent(&rproc->dev, DMA_BIT_MASK(48));
+	if (ret)
+		dev_warn(dev, "Failed to set DMA mask (%d)\n", ret);
+
 	ret = devm_rproc_add(dev, rproc);
 	if (ret)
 		return dev_err_probe(dev, ret, "failed to add register device with remoteproc core\n");
diff --git a/drivers/remoteproc/ti_k3_m4_remoteproc.c b/drivers/remoteproc/ti_k3_m4_remoteproc.c
index 3a11fd24eb52b5..4c46f4a256a92b 100644
--- a/drivers/remoteproc/ti_k3_m4_remoteproc.c
+++ b/drivers/remoteproc/ti_k3_m4_remoteproc.c
@@ -7,10 +7,8 @@
  */
 
 #include <linux/io.h>
-#include <linux/mailbox_client.h>
 #include <linux/module.h>
-#include <linux/of_address.h>
-#include <linux/of_reserved_mem.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/remoteproc.h>
 #include <linux/reset.h>
diff --git a/drivers/remoteproc/ti_k3_r5_remoteproc.c b/drivers/remoteproc/ti_k3_r5_remoteproc.c
index b1d04d082e44cd..d7d9756afed0e9 100644
--- a/drivers/remoteproc/ti_k3_r5_remoteproc.c
+++ b/drivers/remoteproc/ti_k3_r5_remoteproc.c
@@ -6,17 +6,12 @@
  *	Suman Anna <s-anna@ti.com>
  */
 
-#include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/mailbox_client.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/of_reserved_mem.h>
 #include <linux/of_platform.h>
-#include <linux/omap-mailbox.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/remoteproc.h>
@@ -1047,6 +1042,10 @@ static int k3_r5_cluster_rproc_init(struct platform_device *pdev)
 			goto out;
 		}
 
+		ret = dma_coerce_mask_and_coherent(&rproc->dev, DMA_BIT_MASK(48));
+		if (ret)
+			dev_warn(dev, "Failed to set DMA mask (%d)\n", ret);
+
 		/* K3 R5s have a Region Address Translator (RAT) but no MMU */
 		rproc->has_iommu = false;
 		/* error recovery is not supported at present */
diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c
index c685bb5fa62c34..63062128843095 100644
--- a/drivers/remoteproc/xlnx_r5_remoteproc.c
+++ b/drivers/remoteproc/xlnx_r5_remoteproc.c
@@ -318,6 +318,8 @@ static struct mbox_info *zynqmp_r5_setup_mbox(struct device *cdev)
 	if (!ipi)
 		return NULL;
 
+	INIT_WORK(&ipi->mbox_work, handle_event_notified);
+
 	mbox_cl = &ipi->mbox_cl;
 	mbox_cl->rx_callback = zynqmp_r5_mb_rx_cb;
 	mbox_cl->tx_block = false;
@@ -329,6 +331,7 @@ static struct mbox_info *zynqmp_r5_setup_mbox(struct device *cdev)
 	ipi->tx_chan = mbox_request_channel_byname(mbox_cl, "tx");
 	if (IS_ERR(ipi->tx_chan)) {
 		ipi->tx_chan = NULL;
+		cancel_work_sync(&ipi->mbox_work);
 		kfree(ipi);
 		dev_warn(cdev, "mbox tx channel request failed\n");
 		return NULL;
@@ -339,13 +342,12 @@ static struct mbox_info *zynqmp_r5_setup_mbox(struct device *cdev)
 		mbox_free_channel(ipi->tx_chan);
 		ipi->rx_chan = NULL;
 		ipi->tx_chan = NULL;
+		cancel_work_sync(&ipi->mbox_work);
 		kfree(ipi);
 		dev_warn(cdev, "mbox rx channel request failed\n");
 		return NULL;
 	}
 
-	INIT_WORK(&ipi->mbox_work, handle_event_notified);
-
 	return ipi;
 }
 
@@ -364,6 +366,8 @@ static void zynqmp_r5_free_mbox(struct mbox_info *ipi)
 		ipi->rx_chan = NULL;
 	}
 
+	cancel_work_sync(&ipi->mbox_work);
+
 	kfree(ipi);
 }
 
@@ -381,7 +385,10 @@ static void zynqmp_r5_rproc_kick(struct rproc *rproc, int vqid)
 	int ret;
 
 	ipi = r5_core->ipi;
-	if (!ipi)
+	if (!ipi || !ipi->tx_chan)
+		return;
+
+	if (mbox_chan_tx_slots_available(ipi->tx_chan) == 0)
 		return;
 
 	mb_msg = (struct zynqmp_ipi_message *)ipi->tx_mc_buf;
diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c
index 9d223057953abc..f2c651e9f6ea19 100644
--- a/drivers/resctrl/mpam_resctrl.c
+++ b/drivers/resctrl/mpam_resctrl.c
@@ -148,6 +148,11 @@ bool resctrl_arch_get_cdp_enabled(enum resctrl_res_level rid)
 	return mpam_resctrl_controls[rid].cdp_enabled;
 }
 
+u32 resctrl_arch_preconvert_bw(const struct rdt_resource *r, u32 val)
+{
+	return val;
+}
+
 /**
  * resctrl_reset_task_closids() - Reset the PARTID/PMG values for all tasks.
  *
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index 3168ab1eea88df..eda9c698a95f3e 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -73,6 +73,16 @@ config RESET_BRCMSTB_RESCAL
 	  This enables the RESCAL reset controller for SATA, PCIe0, or PCIe1 on
 	  BCM7216 or the BCM2712.
 
+config RESET_DR1V90
+	tristate "Anlogic DR1V90 reset controller"
+	depends on ARCH_ANLOGIC || COMPILE_TEST
+	depends on ANLOGIC_DR1V90_CRU
+	select AUXILIARY_BUS
+	default ARCH_ANLOGIC
+	help
+	  This enables the reset controller driver for Anlogic DR1V90 SoCs
+	  provided by the CRU unit.
+
 config RESET_EIC7700
 	bool "Reset controller driver for ESWIN SoCs"
 	depends on ARCH_ESWIN || COMPILE_TEST
diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile
index 45ad170624624a..489fafd6bbe3fe 100644
--- a/drivers/reset/Makefile
+++ b/drivers/reset/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_RESET_BCM6345) += reset-bcm6345.o
 obj-$(CONFIG_RESET_BERLIN) += reset-berlin.o
 obj-$(CONFIG_RESET_BRCMSTB) += reset-brcmstb.o
 obj-$(CONFIG_RESET_BRCMSTB_RESCAL) += reset-brcmstb-rescal.o
+obj-$(CONFIG_RESET_DR1V90) += reset-dr1v90.o
 obj-$(CONFIG_RESET_EIC7700) += reset-eic7700.o
 obj-$(CONFIG_RESET_EIC7700_HSP) += reset-eic7700-hsp.o
 obj-$(CONFIG_RESET_EYEQ) += reset-eyeq.o
diff --git a/drivers/reset/reset-dr1v90.c b/drivers/reset/reset-dr1v90.c
new file mode 100644
index 00000000000000..daa99c797b1af2
--- /dev/null
+++ b/drivers/reset/reset-dr1v90.c
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026 Junhui Liu <junhui.liu@pigmoral.tech>
+ */
+
+#include <linux/auxiliary_bus.h>
+#include <linux/cleanup.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/reset-controller.h>
+
+#include <dt-bindings/reset/anlogic,dr1v90-cru.h>
+
+struct dr1v90_reset_map {
+	u32 offset;
+	u32 bit;
+};
+
+struct dr1v90_reset_controller {
+	struct reset_controller_dev rcdev;
+	void __iomem *base;
+	spinlock_t lock; /* protect register read-modify-write */
+};
+
+static inline struct dr1v90_reset_controller *
+to_dr1v90_reset_controller(struct reset_controller_dev *rcdev)
+{
+	return container_of(rcdev, struct dr1v90_reset_controller, rcdev);
+}
+
+static const struct dr1v90_reset_map dr1v90_resets[] = {
+	[RESET_OCM]		= { 0x74, BIT(4)},
+	[RESET_QSPI]		= { 0x74, BIT(5)},
+	[RESET_SMC]		= { 0x74, BIT(6)},
+	[RESET_WDT]		= { 0x74, BIT(7)},
+	[RESET_DMAC_AXI]	= { 0x74, BIT(8)},
+	[RESET_DMAC_AHB]	= { 0x74, BIT(9)},
+	[RESET_NPU]		= { 0x74, BIT(12)},
+	[RESET_JPU]		= { 0x74, BIT(13)},
+	[RESET_DDRBUS]		= { 0x74, BIT(14)},
+	[RESET_NIC_HP0]		= { 0x78, BIT(0)},
+	[RESET_NIC_HP1]		= { 0x78, BIT(1)},
+	[RESET_NIC_GP0M]	= { 0x78, BIT(4)},
+	[RESET_NIC_GP1M]	= { 0x78, BIT(5)},
+	[RESET_GPIO]		= { 0x78, BIT(8)},
+	[RESET_IPC]		= { 0x78, BIT(12)},
+	[RESET_USB0]		= { 0x7C, BIT(0)},
+	[RESET_USB1]		= { 0x7C, BIT(1)},
+	[RESET_GBE0]		= { 0x7C, BIT(4)},
+	[RESET_GBE1]		= { 0x7C, BIT(5)},
+	[RESET_SDIO0]		= { 0x7C, BIT(8)},
+	[RESET_SDIO1]		= { 0x7C, BIT(9)},
+	[RESET_UART0]		= { 0x7C, BIT(12)},
+	[RESET_UART1]		= { 0x7C, BIT(13)},
+	[RESET_SPI0]		= { 0x7C, BIT(16)},
+	[RESET_SPI1]		= { 0x7C, BIT(17)},
+	[RESET_CAN0]		= { 0x7C, BIT(20)},
+	[RESET_CAN1]		= { 0x7C, BIT(21)},
+	[RESET_TTC0]		= { 0x7C, BIT(24)},
+	[RESET_TTC1]		= { 0x7C, BIT(25)},
+	[RESET_I2C0]		= { 0x7C, BIT(28)},
+	[RESET_I2C1]		= { 0x7C, BIT(29)}
+};
+
+static int dr1v90_reset_control_update(struct reset_controller_dev *rcdev,
+				       unsigned long id, bool assert)
+{
+	struct dr1v90_reset_controller *rstc = to_dr1v90_reset_controller(rcdev);
+	u32 offset = dr1v90_resets[id].offset;
+	u32 bit = dr1v90_resets[id].bit;
+	u32 reg;
+
+	guard(spinlock_irqsave)(&rstc->lock);
+
+	reg = readl(rstc->base + offset);
+	if (assert)
+		reg &= ~bit;
+	else
+		reg |= bit;
+	writel(reg, rstc->base + offset);
+
+	return 0;
+}
+
+static int dr1v90_reset_control_assert(struct reset_controller_dev *rcdev,
+				       unsigned long id)
+{
+	return dr1v90_reset_control_update(rcdev, id, true);
+}
+
+static int dr1v90_reset_control_deassert(struct reset_controller_dev *rcdev,
+					 unsigned long id)
+{
+	return dr1v90_reset_control_update(rcdev, id, false);
+}
+
+static const struct reset_control_ops dr1v90_reset_control_ops = {
+	.assert = dr1v90_reset_control_assert,
+	.deassert = dr1v90_reset_control_deassert,
+};
+
+static int dr1v90_reset_probe(struct auxiliary_device *adev,
+			      const struct auxiliary_device_id *id)
+{
+	struct dr1v90_reset_controller *rstc;
+	struct device *dev = &adev->dev;
+
+	rstc = devm_kzalloc(dev, sizeof(*rstc), GFP_KERNEL);
+	if (!rstc)
+		return -ENOMEM;
+
+	spin_lock_init(&rstc->lock);
+
+	rstc->base = dev->platform_data;
+	rstc->rcdev.dev = dev;
+	rstc->rcdev.nr_resets = ARRAY_SIZE(dr1v90_resets);
+	rstc->rcdev.of_node = dev->parent->of_node;
+	rstc->rcdev.ops = &dr1v90_reset_control_ops;
+	rstc->rcdev.owner = THIS_MODULE;
+
+	return devm_reset_controller_register(dev, &rstc->rcdev);
+}
+
+static const struct auxiliary_device_id dr1v90_reset_ids[] = {
+	{
+		.name = "anlogic_dr1_cru.reset"
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(auxiliary, dr1v90_reset_ids);
+
+static struct auxiliary_driver dr1v90_reset_driver = {
+	.probe = dr1v90_reset_probe,
+	.id_table = dr1v90_reset_ids,
+};
+module_auxiliary_driver(dr1v90_reset_driver);
+
+MODULE_AUTHOR("Junhui Liu <junhui.liu@pigmoral.tech>");
+MODULE_DESCRIPTION("Anlogic DR1V90 reset controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c
index 1b8bb05924af73..40ec93fedde9e2 100644
--- a/drivers/rpmsg/virtio_rpmsg_bus.c
+++ b/drivers/rpmsg/virtio_rpmsg_bus.c
@@ -20,6 +20,7 @@
 #include <linux/rpmsg.h>
 #include <linux/rpmsg/byteorder.h>
 #include <linux/rpmsg/ns.h>
+#include <linux/rpmsg/virtio_rpmsg.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
@@ -35,13 +36,15 @@
  * @vdev:	the virtio device
  * @rvq:	rx virtqueue
  * @svq:	tx virtqueue
- * @rbufs:	kernel address of rx buffers
- * @sbufs:	kernel address of tx buffers
- * @num_bufs:	total number of buffers for rx and tx
- * @buf_size:   size of one rx or tx buffer
- * @last_sbuf:	index of last tx buffer used
+ * @rx_bufs:	kernel address of rx buffers
+ * @tx_bufs:	kernel address of tx buffers
+ * @num_rx_buf: total number of rx buffers
+ * @num_tx_buf: total number of tx buffers
+ * @rx_buf_size: size of one rx buffer
+ * @tx_buf_size: size of one tx buffer
+ * @last_tx_buf: index of last tx buffer used
  * @bufs_dma:	dma base addr of the buffers
- * @tx_lock:	protects svq and sbufs, to allow concurrent senders.
+ * @tx_lock:	protects svq and tx_bufs, to allow concurrent senders.
  *		sending a message might require waking up a dozing remote
  *		processor, which involves sleeping, hence the mutex.
  * @endpoints:	idr of local endpoints, allows fast retrieval
@@ -55,10 +58,12 @@
 struct virtproc_info {
 	struct virtio_device *vdev;
 	struct virtqueue *rvq, *svq;
-	void *rbufs, *sbufs;
-	unsigned int num_bufs;
-	unsigned int buf_size;
-	int last_sbuf;
+	void *rx_bufs, *tx_bufs;
+	unsigned int num_rx_buf;
+	unsigned int num_tx_buf;
+	unsigned int rx_buf_size;
+	unsigned int tx_buf_size;
+	int last_tx_buf;
 	dma_addr_t bufs_dma;
 	struct mutex tx_lock;
 	struct idr endpoints;
@@ -66,9 +71,6 @@ struct virtproc_info {
 	wait_queue_head_t sendq;
 };
 
-/* The feature bitmap for virtio rpmsg */
-#define VIRTIO_RPMSG_F_NS	0 /* RP supports name service notifications */
-
 /**
  * struct rpmsg_hdr - common header for all rpmsg messages
  * @src: source address
@@ -110,7 +112,7 @@ struct virtio_rpmsg_channel {
 /*
  * We're allocating buffers of 512 bytes each for communications. The
  * number of buffers will be computed from the number of buffers supported
- * by the vring, upto a maximum of 512 buffers (256 in each direction).
+ * by the vring, up to a maximum of 256 in each direction.
  *
  * Each buffer will have 16 bytes for the msg header and 496 bytes for
  * the payload.
@@ -125,8 +127,8 @@ struct virtio_rpmsg_channel {
  * can change this without changing anything in the firmware of the remote
  * processor.
  */
-#define MAX_RPMSG_NUM_BUFS	(512)
-#define MAX_RPMSG_BUF_SIZE	(512)
+#define MAX_RPMSG_NUM_BUFS	(256)
+#define DEFAULT_RPMSG_BUF_SIZE	(512)
 
 /*
  * Local addresses are dynamically allocated on-demand.
@@ -439,12 +441,9 @@ static void *get_a_tx_buf(struct virtproc_info *vrp)
 
 	mutex_lock(&vrp->tx_lock);
 
-	/*
-	 * either pick the next unused tx buffer
-	 * (half of our buffers are used for sending messages)
-	 */
-	if (vrp->last_sbuf < vrp->num_bufs / 2)
-		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;
+	/* either pick the next unused tx buffer */
+	if (vrp->last_tx_buf < vrp->num_tx_buf)
+		ret = vrp->tx_bufs + vrp->tx_buf_size * vrp->last_tx_buf++;
 	/* or recycle a used one */
 	else
 		ret = virtqueue_get_buf(vrp->svq, &len);
@@ -514,7 +513,7 @@ static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev,
 	 * messaging), or to improve the buffer allocator, to support
 	 * variable-length buffer sizes.
 	 */
-	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {
+	if (len > vrp->tx_buf_size - sizeof(struct rpmsg_hdr)) {
 		dev_err(dev, "message is too big (%d)\n", len);
 		return -EMSGSIZE;
 	}
@@ -630,11 +629,10 @@ static __poll_t virtio_rpmsg_poll(struct rpmsg_endpoint *ept, struct file *filp,
 
 	/*
 	 * check for a free buffer, either:
-	 * - we haven't used all of the available transmit buffers (half of the
-	 *   allocated buffers are used for transmit, hence num_bufs / 2), or,
+	 * - we haven't used all of the available transmit buffers or,
 	 * - we ask the virtqueue if there's a buffer available
 	 */
-	if (vrp->last_sbuf < vrp->num_bufs / 2 ||
+	if (vrp->last_tx_buf < vrp->num_tx_buf ||
 	    !virtqueue_enable_cb(vrp->svq))
 		mask |= EPOLLOUT;
 
@@ -648,7 +646,7 @@ static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept)
 	struct rpmsg_device *rpdev = ept->rpdev;
 	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev);
 
-	return vch->vrp->buf_size - sizeof(struct rpmsg_hdr);
+	return vch->vrp->tx_buf_size - sizeof(struct rpmsg_hdr);
 }
 
 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
@@ -674,7 +672,7 @@ static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
 	 * We currently use fixed-sized buffers, so trivially sanitize
 	 * the reported payload length.
 	 */
-	if (len > vrp->buf_size ||
+	if (len > vrp->rx_buf_size ||
 	    msg_len > (len - sizeof(struct rpmsg_hdr))) {
 		dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg_len);
 		return -EINVAL;
@@ -707,7 +705,7 @@ static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
 		dev_warn_ratelimited(dev, "msg received with no recipient\n");
 
 	/* publish the real size of the buffer */
-	rpmsg_sg_init(&sg, msg, vrp->buf_size);
+	rpmsg_sg_init(&sg, msg, vrp->rx_buf_size);
 
 	/* add the buffer back to the remote processor's virtqueue */
 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);
@@ -811,6 +809,65 @@ static void rpmsg_virtio_del_ctrl_dev(struct rpmsg_device *rpdev_ctrl)
 	device_unregister(&rpdev_ctrl->dev);
 }
 
+static int rpmsg_virtio_get_buf_size(struct virtproc_info *vrp)
+{
+	struct virtio_device *vdev;
+	u8 version;
+	u16 size;
+
+	vdev = vrp->vdev;
+
+	/*
+	 * If VIRTIO_RPMSG_F_BUFSZ feature is supported, then configure buf
+	 * size from virtio device config space from the resource table.
+	 * If the feature is not supported, then assign default buf size.
+	 */
+	if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_BUFSZ)) {
+		virtio_cread(vdev, struct virtio_rpmsg_config,
+			     version, &version);
+
+		/* for now we support only v1 */
+		if (version != RPMSG_VDEV_CONFIG_V1) {
+			dev_err(&vdev->dev,
+				"unsupported vdev config version %u\n", version);
+			return -EINVAL;
+		}
+
+		/* size of the config space must match */
+		virtio_cread(vdev, struct virtio_rpmsg_config,
+			     size, &size);
+		if (size != sizeof(struct virtio_rpmsg_config)) {
+			dev_err(&vdev->dev, "invalid size of vdev config %u\n",
+				size);
+			return -EINVAL;
+		}
+
+		/* note: tx and rx are defined from remote view */
+		virtio_cread(vdev, struct virtio_rpmsg_config,
+			     txbuf_size, &vrp->rx_buf_size);
+		virtio_cread(vdev, struct virtio_rpmsg_config,
+			     rxbuf_size, &vrp->tx_buf_size);
+
+		/* The buffers must hold at least the rpmsg header */
+		if (vrp->rx_buf_size < sizeof(struct rpmsg_hdr) ||
+		    vrp->tx_buf_size < sizeof(struct rpmsg_hdr)) {
+			dev_err(&vdev->dev,
+				"bad vdev config: rx buf sz = %u, tx buf sz = %u\n",
+				vrp->rx_buf_size, vrp->tx_buf_size);
+			return -EINVAL;
+		}
+
+		dev_dbg(&vdev->dev,
+			"vdev config: ver=%u, rx sz = 0x%x, tx sz = 0x%x\n",
+			version, vrp->rx_buf_size, vrp->tx_buf_size);
+	} else {
+		vrp->rx_buf_size = DEFAULT_RPMSG_BUF_SIZE;
+		vrp->tx_buf_size = DEFAULT_RPMSG_BUF_SIZE;
+	}
+
+	return 0;
+}
+
 static int rpmsg_probe(struct virtio_device *vdev)
 {
 	struct virtqueue_info vqs_info[] = {
@@ -845,19 +902,23 @@ static int rpmsg_probe(struct virtio_device *vdev)
 	vrp->rvq = vqs[0];
 	vrp->svq = vqs[1];
 
-	/* we expect symmetric tx/rx vrings */
-	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=
-		virtqueue_get_vring_size(vrp->svq));
-
 	/* we need less buffers if vrings are small */
-	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)
-		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;
+	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS)
+		vrp->num_rx_buf = virtqueue_get_vring_size(vrp->rvq);
+	else
+		vrp->num_rx_buf = MAX_RPMSG_NUM_BUFS;
+
+	if (virtqueue_get_vring_size(vrp->svq) < MAX_RPMSG_NUM_BUFS)
+		vrp->num_tx_buf = virtqueue_get_vring_size(vrp->svq);
 	else
-		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;
+		vrp->num_tx_buf = MAX_RPMSG_NUM_BUFS;
 
-	vrp->buf_size = MAX_RPMSG_BUF_SIZE;
+	err = rpmsg_virtio_get_buf_size(vrp);
+	if (err)
+		goto vqs_del;
 
-	total_buf_space = vrp->num_bufs * vrp->buf_size;
+	total_buf_space = (vrp->num_rx_buf * vrp->rx_buf_size) +
+			  (vrp->num_tx_buf * vrp->tx_buf_size);
 
 	/* allocate coherent memory for the buffers */
 	bufs_va = dma_alloc_coherent(vdev->dev.parent,
@@ -871,18 +932,17 @@ static int rpmsg_probe(struct virtio_device *vdev)
 	dev_dbg(&vdev->dev, "buffers: va %p, dma %pad\n",
 		bufs_va, &vrp->bufs_dma);
 
-	/* half of the buffers is dedicated for RX */
-	vrp->rbufs = bufs_va;
+	/* first part of the buffers is dedicated for RX */
+	vrp->rx_bufs = bufs_va;
 
-	/* and half is dedicated for TX */
-	vrp->sbufs = bufs_va + total_buf_space / 2;
+	vrp->tx_bufs = bufs_va + (vrp->num_rx_buf * vrp->rx_buf_size);
 
 	/* set up the receive buffers */
-	for (i = 0; i < vrp->num_bufs / 2; i++) {
+	for (i = 0; i < vrp->num_rx_buf; i++) {
 		struct scatterlist sg;
-		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;
+		void *cpu_addr = vrp->rx_bufs + i * vrp->rx_buf_size;
 
-		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);
+		rpmsg_sg_init(&sg, cpu_addr, vrp->rx_buf_size);
 
 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
 					  GFP_KERNEL);
@@ -965,7 +1025,8 @@ static int rpmsg_remove_device(struct device *dev, void *data)
 static void rpmsg_remove(struct virtio_device *vdev)
 {
 	struct virtproc_info *vrp = vdev->priv;
-	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;
+	size_t total_buf_space = (vrp->num_rx_buf * vrp->rx_buf_size) +
+				 (vrp->num_tx_buf * vrp->tx_buf_size);
 	int ret;
 
 	virtio_reset_device(vdev);
@@ -979,7 +1040,7 @@ static void rpmsg_remove(struct virtio_device *vdev)
 	vdev->config->del_vqs(vrp->vdev);
 
 	dma_free_coherent(vdev->dev.parent, total_buf_space,
-			  vrp->rbufs, vrp->bufs_dma);
+			  vrp->rx_bufs, vrp->bufs_dma);
 
 	kfree(vrp);
 }
@@ -991,6 +1052,7 @@ static struct virtio_device_id id_table[] = {
 
 static unsigned int features[] = {
 	VIRTIO_RPMSG_F_NS,
+	VIRTIO_RPMSG_F_BUFSZ,
 };
 
 static struct virtio_driver virtio_ipc_driver = {
diff --git a/drivers/rtc/dev.c b/drivers/rtc/dev.c
index 8ba7c25d2565ef..be173c9511faca 100644
--- a/drivers/rtc/dev.c
+++ b/drivers/rtc/dev.c
@@ -216,7 +216,7 @@ static long rtc_dev_ioctl(struct file *file,
 	struct rtc_device *rtc = file->private_data;
 	const struct rtc_class_ops *ops = rtc->ops;
 	struct rtc_time tm;
-	struct rtc_wkalrm alarm;
+	struct rtc_wkalrm alarm = { 0 };
 	struct rtc_param param;
 	void __user *uarg = (void __user *)arg;
 
diff --git a/drivers/rtc/rtc-ac100.c b/drivers/rtc/rtc-ac100.c
index bba7115ff3ad0f..c579c4bf5578d4 100644
--- a/drivers/rtc/rtc-ac100.c
+++ b/drivers/rtc/rtc-ac100.c
@@ -317,10 +317,12 @@ static int ac100_rtc_register_clks(struct ac100_rtc_dev *chip)
 	if (!chip->clk_data)
 		return -ENOMEM;
 
-	chip->rtc_32k_clk = clk_hw_register_fixed_rate(chip->dev,
-						       AC100_RTC_32K_NAME,
-						       NULL, 0,
-						       AC100_RTC_32K_RATE);
+	chip->clk_data->num = AC100_CLKOUT_NUM;
+
+	chip->rtc_32k_clk = devm_clk_hw_register_fixed_rate(chip->dev,
+							    AC100_RTC_32K_NAME,
+							    NULL, 0,
+							    AC100_RTC_32K_RATE);
 	if (IS_ERR(chip->rtc_32k_clk)) {
 		ret = PTR_ERR(chip->rtc_32k_clk);
 		dev_err(chip->dev, "Failed to register RTC-32k clock: %d\n",
@@ -354,29 +356,14 @@ static int ac100_rtc_register_clks(struct ac100_rtc_dev *chip)
 		if (ret) {
 			dev_err(chip->dev, "Failed to register clk '%s': %d\n",
 				init.name, ret);
-			goto err_unregister_rtc_32k;
+			return ret;
 		}
 
 		chip->clk_data->hws[i] = &clk->hw;
 	}
 
-	chip->clk_data->num = i;
-	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, chip->clk_data);
-	if (ret)
-		goto err_unregister_rtc_32k;
-
-	return 0;
-
-err_unregister_rtc_32k:
-	clk_unregister_fixed_rate(chip->rtc_32k_clk->clk);
-
-	return ret;
-}
-
-static void ac100_rtc_unregister_clks(struct ac100_rtc_dev *chip)
-{
-	of_clk_del_provider(chip->dev->of_node);
-	clk_unregister_fixed_rate(chip->rtc_32k_clk->clk);
+	return devm_of_clk_add_hw_provider(chip->dev, of_clk_hw_onecell_get,
+					   chip->clk_data);
 }
 
 /*
@@ -614,13 +601,6 @@ static int ac100_rtc_probe(struct platform_device *pdev)
 	return devm_rtc_register_device(chip->rtc);
 }
 
-static void ac100_rtc_remove(struct platform_device *pdev)
-{
-	struct ac100_rtc_dev *chip = platform_get_drvdata(pdev);
-
-	ac100_rtc_unregister_clks(chip);
-}
-
 static const struct of_device_id ac100_rtc_match[] = {
 	{ .compatible = "x-powers,ac100-rtc" },
 	{ },
@@ -629,7 +609,6 @@ MODULE_DEVICE_TABLE(of, ac100_rtc_match);
 
 static struct platform_driver ac100_rtc_driver = {
 	.probe		= ac100_rtc_probe,
-	.remove		= ac100_rtc_remove,
 	.driver		= {
 		.name		= "ac100-rtc",
 		.of_match_table	= of_match_ptr(ac100_rtc_match),
diff --git a/drivers/rtc/rtc-bd70528.c b/drivers/rtc/rtc-bd70528.c
index 482810b61495dc..fd415e327ea615 100644
--- a/drivers/rtc/rtc-bd70528.c
+++ b/drivers/rtc/rtc-bd70528.c
@@ -8,6 +8,7 @@
 #include <linux/mfd/rohm-bd71815.h>
 #include <linux/mfd/rohm-bd71828.h>
 #include <linux/mfd/rohm-bd72720.h>
+#include <linux/mfd/rohm-bd73800.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
@@ -284,6 +285,12 @@ static int bd70528_probe(struct platform_device *pdev)
 		bd_rtc->bd718xx_alm_block_start = BD72720_REG_RTC_ALM_START;
 		hour_reg = BD72720_REG_RTC_HOUR;
 		break;
+	case ROHM_CHIP_TYPE_BD73800:
+		bd_rtc->reg_time_start = BD73800_REG_RTC_START;
+		bd_rtc->bd718xx_alm_block_start = BD73800_REG_RTC_ALM_START;
+		hour_reg = BD73800_REG_RTC_HOUR;
+		break;
+
 	default:
 		dev_err(&pdev->dev, "Unknown chip\n");
 		return -ENOENT;
@@ -344,6 +351,7 @@ static const struct platform_device_id bd718x7_rtc_id[] = {
 	{ .name = "bd71828-rtc", .driver_data = ROHM_CHIP_TYPE_BD71828 },
 	{ .name = "bd71815-rtc", .driver_data = ROHM_CHIP_TYPE_BD71815 },
 	{ .name = "bd72720-rtc", .driver_data = ROHM_CHIP_TYPE_BD72720 },
+	{ .name = "bd73800-rtc", .driver_data = ROHM_CHIP_TYPE_BD73800 },
 	{ }
 };
 MODULE_DEVICE_TABLE(platform, bd718x7_rtc_id);
diff --git a/drivers/rtc/rtc-efi.c b/drivers/rtc/rtc-efi.c
index b4f44999ef0f4b..a8c18ebc559967 100644
--- a/drivers/rtc/rtc-efi.c
+++ b/drivers/rtc/rtc-efi.c
@@ -52,6 +52,7 @@ compute_wday(efi_time_t *eft, int yday)
 static void
 convert_to_efi_time(struct rtc_time *wtime, efi_time_t *eft)
 {
+	memset(eft, 0, sizeof(*eft));
 	eft->year	= wtime->tm_year + 1900;
 	eft->month	= wtime->tm_mon + 1;
 	eft->day	= wtime->tm_mday;
@@ -112,6 +113,35 @@ convert_from_efi_time(efi_time_t *eft, struct rtc_time *wtime)
 	return true;
 }
 
+static int efi_read_alarm(struct device *dev, struct rtc_wkalrm *wkalrm)
+{
+	efi_time_t eft;
+	efi_status_t status;
+
+	memset(&eft, 0, sizeof(eft));
+	status = efi.get_wakeup_time((efi_bool_t *)&wkalrm->enabled,
+				     (efi_bool_t *)&wkalrm->pending, &eft);
+	if (status != EFI_SUCCESS)
+		return -EINVAL;
+
+	if (!convert_from_efi_time(&eft, &wkalrm->time))
+		return -EIO;
+
+	return rtc_valid_tm(&wkalrm->time);
+}
+
+static int efi_set_alarm(struct device *dev, struct rtc_wkalrm *wkalrm)
+{
+	efi_time_t eft;
+	efi_status_t status;
+
+	convert_to_efi_time(&wkalrm->time, &eft);
+
+	status = efi.set_wakeup_time((efi_bool_t)!!wkalrm->enabled, &eft);
+
+	return status == EFI_SUCCESS ? 0 : -EINVAL;
+}
+
 static int efi_read_time(struct device *dev, struct rtc_time *tm)
 {
 	efi_status_t status;
@@ -146,13 +176,21 @@ static int efi_set_time(struct device *dev, struct rtc_time *tm)
 
 static int efi_procfs(struct device *dev, struct seq_file *seq)
 {
-	efi_time_t        eft;
+	efi_time_t        eft, alm;
 	efi_time_cap_t    cap;
+	efi_bool_t        enabled, pending;
+	struct rtc_device *rtc = dev_get_drvdata(dev);
 
 	memset(&eft, 0, sizeof(eft));
+	memset(&alm, 0, sizeof(alm));
 	memset(&cap, 0, sizeof(cap));
 
 	efi.get_time(&eft, &cap);
+	if (test_bit(RTC_FEATURE_ALARM, rtc->features) &&
+	    efi.get_wakeup_time(&enabled, &pending, &alm) != EFI_SUCCESS) {
+		enabled = 0;
+		pending = 0;
+	}
 
 	seq_printf(seq,
 		   "Time\t\t: %u:%u:%u.%09u\n"
@@ -168,6 +206,25 @@ static int efi_procfs(struct device *dev, struct seq_file *seq)
 		/* XXX fixme: convert to string? */
 		seq_printf(seq, "Timezone\t: %u\n", eft.timezone);
 
+	if (test_bit(RTC_FEATURE_ALARM, rtc->features)) {
+		seq_printf(seq,
+			   "Alarm Time\t: %u:%u:%u.%09u\n"
+			   "Alarm Date\t: %u-%u-%u\n"
+			   "Alarm Daylight\t: %u\n"
+			   "Enabled\t\t: %s\n"
+			   "Pending\t\t: %s\n",
+			   alm.hour, alm.minute, alm.second, alm.nanosecond,
+			   alm.year, alm.month, alm.day,
+			   alm.daylight,
+			   enabled == 1 ? "yes" : "no",
+			   pending == 1 ? "yes" : "no");
+
+		if (alm.timezone == EFI_UNSPECIFIED_TIMEZONE)
+			seq_puts(seq, "Alarm Timezone\t: unspecified\n");
+		else
+			seq_printf(seq, "Alarm Timezone\t: %d\n", alm.timezone);
+	}
+
 	/*
 	 * now prints the capabilities
 	 */
@@ -183,6 +240,8 @@ static int efi_procfs(struct device *dev, struct seq_file *seq)
 static const struct rtc_class_ops efi_rtc_ops = {
 	.read_time	= efi_read_time,
 	.set_time	= efi_set_time,
+	.read_alarm	= efi_read_alarm,
+	.set_alarm	= efi_set_alarm,
 	.proc		= efi_procfs,
 };
 
@@ -191,6 +250,7 @@ static int __init efi_rtc_probe(struct platform_device *dev)
 	struct rtc_device *rtc;
 	efi_time_t eft;
 	efi_time_cap_t cap;
+	efi_bool_t enabled, pending;
 
 	/* First check if the RTC is usable */
 	if (efi.get_time(&eft, &cap) != EFI_SUCCESS)
@@ -203,7 +263,23 @@ static int __init efi_rtc_probe(struct platform_device *dev)
 	platform_set_drvdata(dev, rtc);
 
 	rtc->ops = &efi_rtc_ops;
-	clear_bit(RTC_FEATURE_ALARM, rtc->features);
+	clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features);
+
+	/*
+	 * The EFI_RT_SUPPORTED_WAKEUP_SERVICES bit defaults to enabled
+	 * and only gets cleared when the RT_PROP table explicitly says
+	 * wakeup is unsupported. Many platforms lack an RT_PROP table
+	 * even though they don't implement the wakeup runtime service,
+	 * so probe by actually calling GetWakeupTime() to avoid exposing
+	 * a broken alarm to userspace.
+	 */
+	if (efi_rt_services_supported(EFI_RT_SUPPORTED_WAKEUP_SERVICES) &&
+	    efi.get_wakeup_time(&enabled, &pending, &eft) == EFI_SUCCESS) {
+		set_bit(RTC_FEATURE_ALARM, rtc->features);
+		set_bit(RTC_FEATURE_ALARM_WAKEUP_ONLY, rtc->features);
+	} else {
+		clear_bit(RTC_FEATURE_ALARM, rtc->features);
+	}
 
 	device_init_wakeup(&dev->dev, true);
 
diff --git a/drivers/rtc/rtc-mpfs.c b/drivers/rtc/rtc-mpfs.c
index ece6de4a6adbd1..60596b0ea9d809 100644
--- a/drivers/rtc/rtc-mpfs.c
+++ b/drivers/rtc/rtc-mpfs.c
@@ -256,8 +256,12 @@ static int mpfs_rtc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	clk = devm_clk_get(&pdev->dev, "rtcref");
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
 	/* prescaler hardware adds 1 to reg value */
-	prescaler = clk_get_rate(devm_clk_get(&pdev->dev, "rtcref")) - 1;
+	prescaler = clk_get_rate(clk) - 1;
 	if (prescaler > MAX_PRESCALER_COUNT) {
 		dev_dbg(&pdev->dev, "invalid prescaler %lu\n", prescaler);
 		return -EINVAL;
diff --git a/drivers/rtc/rtc-spear.c b/drivers/rtc/rtc-spear.c
index 959acff8faff0f..d12bccb3cf5ecb 100644
--- a/drivers/rtc/rtc-spear.c
+++ b/drivers/rtc/rtc-spear.c
@@ -363,6 +363,12 @@ static int spear_rtc_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
+	config->ioaddr = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(config->ioaddr))
+		return PTR_ERR(config->ioaddr);
+
+	spin_lock_init(&config->lock);
+
 	status = devm_request_irq(&pdev->dev, irq, spear_rtc_irq, 0, pdev->name,
 			config);
 	if (status) {
@@ -371,10 +377,6 @@ static int spear_rtc_probe(struct platform_device *pdev)
 		return status;
 	}
 
-	config->ioaddr = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(config->ioaddr))
-		return PTR_ERR(config->ioaddr);
-
 	config->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(config->clk))
 		return PTR_ERR(config->clk);
@@ -383,7 +385,6 @@ static int spear_rtc_probe(struct platform_device *pdev)
 	if (status < 0)
 		return status;
 
-	spin_lock_init(&config->lock);
 	platform_set_drvdata(pdev, config);
 
 	config->rtc->ops = &spear_rtc_ops;
diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c
index 736459477c1964..121cb9ae5c678a 100644
--- a/drivers/s390/block/dasd_3990_erp.c
+++ b/drivers/s390/block/dasd_3990_erp.c
@@ -103,7 +103,7 @@ dasd_3990_erp_int_req(struct dasd_ccw_req * erp)
 
 	/* first time set initial retry counter and erp_function */
 	/* and retry once without blocking queue		 */
-	/* (this enables easier enqueing of the cqr)		 */
+	/* (this enables easier enqueuing of the cqr)		 */
 	if (erp->function != dasd_3990_erp_int_req) {
 
 		erp->retries = 256;
@@ -302,7 +302,7 @@ dasd_3990_erp_action_4(struct dasd_ccw_req * erp, char *sense)
 
 	/* first time set initial retry counter and erp_function    */
 	/* and retry once without waiting for state change pending  */
-	/* interrupt (this enables easier enqueing of the cqr)	    */
+	/* interrupt (this enables easier enqueuing of the cqr)	    */
 	if (erp->function != dasd_3990_erp_action_4) {
 
 		DBF_DEV_EVENT(DBF_INFO, device, "%s",
@@ -1078,7 +1078,7 @@ dasd_3990_erp_bus_out(struct dasd_ccw_req * erp)
 
 	/* first time set initial retry counter and erp_function */
 	/* and retry once without blocking queue		 */
-	/* (this enables easier enqueing of the cqr)		 */
+	/* (this enables easier enqueuing of the cqr)		 */
 	if (erp->function != dasd_3990_erp_bus_out) {
 		erp->retries = 256;
 		erp->function = dasd_3990_erp_bus_out;
diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c
index 8d976dd58a6c44..9d1b11020b4e82 100644
--- a/drivers/s390/block/dasd_eckd.c
+++ b/drivers/s390/block/dasd_eckd.c
@@ -2060,7 +2060,7 @@ dasd_eckd_psf_ssc(struct dasd_device *device, int enable_pav,
 }
 
 /*
- * Valide storage server of current device.
+ * Valid storage server of current device.
  */
 static int dasd_eckd_validate_server(struct dasd_device *device,
 				     unsigned long flags)
@@ -5672,7 +5672,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_raw(struct dasd_device *startdev,
 	char *dst;
 
 	/*
-	 * raw track access needs to be mutiple of 64k and on 64k boundary
+	 * raw track access needs to be multiple of 64k and on 64k boundary
 	 * For read requests we can fix an incorrect alignment by padding
 	 * the request with dummy pages.
 	 */
diff --git a/drivers/s390/char/keyboard.c b/drivers/s390/char/keyboard.c
index ecd9ace2b3aa8d..63bcb52d6e87c8 100644
--- a/drivers/s390/char/keyboard.c
+++ b/drivers/s390/char/keyboard.c
@@ -81,7 +81,8 @@ kbd_alloc(void) {
 	kbd = kzalloc_obj(struct kbd_data);
 	if (!kbd)
 		goto out;
-	kbd->key_maps = kzalloc(sizeof(ebc_key_maps), GFP_KERNEL);
+	kbd->key_maps = kcalloc(ARRAY_SIZE(ebc_key_maps),
+				sizeof(*kbd->key_maps), GFP_KERNEL);
 	if (!kbd->key_maps)
 		goto out_kbd;
 	for (i = 0; i < ARRAY_SIZE(ebc_key_maps); i++) {
@@ -93,7 +94,8 @@ kbd_alloc(void) {
 				goto out_maps;
 		}
 	}
-	kbd->func_table = kzalloc(sizeof(ebc_func_table), GFP_KERNEL);
+	kbd->func_table = kcalloc(ARRAY_SIZE(ebc_func_table),
+				  sizeof(*kbd->func_table), GFP_KERNEL);
 	if (!kbd->func_table)
 		goto out_maps;
 	for (i = 0; i < ARRAY_SIZE(ebc_func_table); i++) {
diff --git a/drivers/s390/char/raw3270.c b/drivers/s390/char/raw3270.c
index aa9c4d81225cbe..6861bf27d15fc5 100644
--- a/drivers/s390/char/raw3270.c
+++ b/drivers/s390/char/raw3270.c
@@ -420,7 +420,7 @@ struct raw3270_ua {	/* Query Reply structure for Usable Area */
 		char  flags0;
 		char  flags1;
 		short w;	/* Width of usable area */
-		short h;	/* Heigth of usavle area */
+		short h;	/* Height of usavle area */
 		char  units;	/* 0x00:in; 0x01:mm */
 		int   xr;
 		int   yr;
diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c
index 4e2bf8186a5dca..b95af0974624d0 100644
--- a/drivers/s390/char/sclp_vt220.c
+++ b/drivers/s390/char/sclp_vt220.c
@@ -27,6 +27,7 @@
 #include <linux/init.h>
 #include <linux/reboot.h>
 #include <linux/slab.h>
+#include <linux/workqueue.h>
 
 #include <linux/uaccess.h>
 #include "sclp.h"
@@ -52,6 +53,17 @@ struct sclp_vt220_sccb {
 	struct evbuf_header evbuf;
 };
 
+struct sclp_vt220_evbuf {
+	struct evbuf_header header;
+	char type;
+	char data[];
+} __packed;
+
+struct sclp_vt220_resize_data_t {
+	u16 rows;
+	u16 cols;
+} __packed;
+
 #define SCLP_VT220_MAX_CHARS_PER_BUFFER	(PAGE_SIZE - \
 					 sizeof(struct sclp_vt220_request) - \
 					 sizeof(struct sclp_vt220_sccb))
@@ -89,6 +101,15 @@ static int __initdata sclp_vt220_init_count;
  * another buffer */
 static int sclp_vt220_flush_later;
 
+/* Work struct required for scheduling resize */
+static struct work_struct sclp_vt220_resize_work;
+
+/* Current vt220 terminal winsize */
+static struct winsize sclp_vt220_winsize = {
+	.ws_row = 24,
+	.ws_col = 80,
+};
+
 static void sclp_vt220_receiver_fn(struct evbuf_header *evbuf);
 static int __sclp_vt220_emit(struct sclp_vt220_request *request);
 static void sclp_vt220_emit_current(void);
@@ -468,6 +489,7 @@ sclp_vt220_write(struct tty_struct *tty, const u8 *buf, size_t count)
 #define SCLP_VT220_SESSION_ENDED	0x01
 #define	SCLP_VT220_SESSION_STARTED	0x80
 #define SCLP_VT220_SESSION_DATA		0x00
+#define SCLP_VT220_SESSION_RESIZE	0x08
 
 #ifdef CONFIG_MAGIC_SYSRQ
 
@@ -516,30 +538,65 @@ static void sclp_vt220_handle_input(const char *buffer, unsigned int count)
 
 #endif
 
+static void sclp_vt220_resize(struct work_struct *work)
+{
+	struct tty_struct *tty;
+	struct winsize ws;
+
+	spin_lock_irq(&sclp_vt220_lock);
+	ws = sclp_vt220_winsize;
+	spin_unlock_irq(&sclp_vt220_lock);
+
+	tty = tty_port_tty_get(&sclp_vt220_port);
+	if (!tty)
+		return;
+
+	tty_do_resize(tty, &ws);
+	tty_kref_put(tty);
+}
+
+static void sclp_vt220_resize_sched(void *buffer, unsigned int count)
+{
+	struct sclp_vt220_resize_data_t *data = buffer;
+	unsigned long flags;
+
+	if (count < sizeof(*data))
+		return;
+
+	spin_lock_irqsave(&sclp_vt220_lock, flags);
+	sclp_vt220_winsize.ws_row = data->rows;
+	sclp_vt220_winsize.ws_col = data->cols;
+	schedule_work(&sclp_vt220_resize_work);
+	spin_unlock_irqrestore(&sclp_vt220_lock, flags);
+}
+
 /*
  * Called by the SCLP to report incoming event buffers.
  */
-static void
-sclp_vt220_receiver_fn(struct evbuf_header *evbuf)
+static void sclp_vt220_receiver_fn(struct evbuf_header *evbuf)
 {
-	char *buffer;
+	struct sclp_vt220_evbuf *buffer;
 	unsigned int count;
 
-	buffer = (char *) ((addr_t) evbuf + sizeof(struct evbuf_header));
-	count = evbuf->length - sizeof(struct evbuf_header);
+	if (evbuf->length < offsetof(struct sclp_vt220_evbuf, data))
+		return;
+
+	buffer = (struct sclp_vt220_evbuf *)evbuf;
+	count = evbuf->length - offsetof(struct sclp_vt220_evbuf, data);
 
-	switch (*buffer) {
+	switch (buffer->type) {
 	case SCLP_VT220_SESSION_ENDED:
 	case SCLP_VT220_SESSION_STARTED:
 		sclp_vt220_reset_session();
 		break;
 	case SCLP_VT220_SESSION_DATA:
 		/* Send input to line discipline */
-		buffer++;
-		count--;
-		sclp_vt220_handle_input(buffer, count);
+		sclp_vt220_handle_input(buffer->data, count);
 		tty_flip_buffer_push(&sclp_vt220_port);
 		break;
+	case SCLP_VT220_SESSION_RESIZE:
+		sclp_vt220_resize_sched(buffer->data, count);
+		break;
 	}
 }
 
@@ -551,10 +608,12 @@ sclp_vt220_open(struct tty_struct *tty, struct file *filp)
 {
 	if (tty->count == 1) {
 		tty_port_tty_set(&sclp_vt220_port, tty);
-		if (!tty->winsize.ws_row && !tty->winsize.ws_col) {
-			tty->winsize.ws_row = 24;
-			tty->winsize.ws_col = 80;
-		}
+
+		mutex_lock(&tty->winsize_mutex);
+		spin_lock_irq(&sclp_vt220_lock);
+		tty->winsize = sclp_vt220_winsize;
+		spin_unlock_irq(&sclp_vt220_lock);
+		mutex_unlock(&tty->winsize_mutex);
 	}
 	return 0;
 }
@@ -747,6 +806,8 @@ static int __init sclp_vt220_tty_init(void)
 	rc = tty_register_driver(driver);
 	if (rc)
 		goto out_init;
+
+	INIT_WORK(&sclp_vt220_resize_work, sclp_vt220_resize);
 	rc = sclp_register(&sclp_vt220_register_input);
 	if (rc)
 		goto out_reg;
diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c
index 383e7e2bd69f4d..005735a899203c 100644
--- a/drivers/s390/char/vmlogrdr.c
+++ b/drivers/s390/char/vmlogrdr.c
@@ -444,7 +444,7 @@ static int vmlogrdr_receive_data(struct vmlogrdr_priv_t *priv)
 		spin_unlock_bh(&priv->priv_lock);
 		/* An rc of 5 indicates that the record was bigger than
 		 * the buffer, which is OK for us. A 9 indicates that the
-		 * record was purged befor we could receive it.
+		 * record was purged before we could receive it.
 		 */
 		if (rc == 5)
 			rc = 0;
diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c
index c890f21a82ce37..eaf0527bff6cc5 100644
--- a/drivers/s390/cio/chp.c
+++ b/drivers/s390/cio/chp.c
@@ -78,6 +78,9 @@ u8 chp_get_sch_opm(struct subchannel *sch)
 	int opm;
 	int i;
 
+	if (!sch->schib.pmcw.dnv)
+		return 0;
+
 	opm = 0;
 	chp_id_init(&chpid);
 	for (i = 0; i < 8; i++) {
diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c
index 9689f722c863c1..d98f629e892957 100644
--- a/drivers/s390/cio/chsc.c
+++ b/drivers/s390/cio/chsc.c
@@ -931,12 +931,12 @@ static int cub_alloc(struct channel_subsystem *css)
 	int i;
 
 	for (i = 0; i < CSS_NUM_CUB_PAGES; i++) {
-		css->cub[i] = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+		css->cub[i] = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 		if (!css->cub[i])
 			return -ENOMEM;
 	}
 	for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) {
-		css->ecub[i] = (void *)get_zeroed_page(GFP_KERNEL);
+		css->ecub[i] = kzalloc(PAGE_SIZE, GFP_KERNEL);
 		if (!css->ecub[i])
 			return -ENOMEM;
 	}
@@ -949,11 +949,11 @@ static void cub_free(struct channel_subsystem *css)
 	int i;
 
 	for (i = 0; i < CSS_NUM_CUB_PAGES; i++) {
-		free_page((unsigned long)css->cub[i]);
+		kfree(css->cub[i]);
 		css->cub[i] = NULL;
 	}
 	for (i = 0; i < CSS_NUM_ECUB_PAGES; i++) {
-		free_page((unsigned long)css->ecub[i]);
+		kfree(css->ecub[i]);
 		css->ecub[i] = NULL;
 	}
 }
@@ -1142,8 +1142,8 @@ int __init chsc_init(void)
 {
 	int ret;
 
-	sei_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
-	chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	sei_page = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
+	chsc_page = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!sei_page || !chsc_page) {
 		ret = -ENOMEM;
 		goto out_err;
@@ -1153,16 +1153,16 @@ int __init chsc_init(void)
 		goto out_err;
 	return ret;
 out_err:
-	free_page((unsigned long)chsc_page);
-	free_page((unsigned long)sei_page);
+	kfree(chsc_page);
+	kfree(sei_page);
 	return ret;
 }
 
 void __init chsc_init_cleanup(void)
 {
 	crw_unregister_handler(CRW_RSC_CSS);
-	free_page((unsigned long)chsc_page);
-	free_page((unsigned long)sei_page);
+	kfree(chsc_page);
+	kfree(sei_page);
 }
 
 int __chsc_enable_facility(struct chsc_sda_area *sda_area, int operation_code)
diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c
index b6cb8bb8bcc4b2..225aaac7dd309f 100644
--- a/drivers/s390/cio/chsc_sch.c
+++ b/drivers/s390/cio/chsc_sch.c
@@ -15,6 +15,7 @@
 #include <linux/uaccess.h>
 #include <linux/miscdevice.h>
 #include <linux/kernel_stat.h>
+#include <linux/cleanup.h>
 
 #include <asm/cio.h>
 #include <asm/chsc.h>
@@ -292,7 +293,7 @@ static int chsc_ioctl_start(void __user *user_area)
 	if (!css_general_characteristics.dynio)
 		/* It makes no sense to try. */
 		return -EOPNOTSUPP;
-	chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL);
+	chsc_area = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL);
 	if (!chsc_area)
 		return -ENOMEM;
 	request = kzalloc_obj(*request);
@@ -321,7 +322,7 @@ static int chsc_ioctl_start(void __user *user_area)
 	snprintf(dbf, sizeof(dbf), "ret:%d", ret);
 	CHSC_LOG(0, dbf);
 	kfree(request);
-	free_page((unsigned long)chsc_area);
+	kfree(chsc_area);
 	return ret;
 }
 
@@ -340,7 +341,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area)
 		ret = -ENOMEM;
 		goto out_unlock;
 	}
-	on_close_chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL);
+	on_close_chsc_area = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL);
 	if (!on_close_chsc_area) {
 		ret = -ENOMEM;
 		goto out_free_request;
@@ -353,7 +354,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area)
 	goto out_unlock;
 
 out_free_chsc:
-	free_page((unsigned long)on_close_chsc_area);
+	kfree(on_close_chsc_area);
 	on_close_chsc_area = NULL;
 out_free_request:
 	kfree(on_close_request);
@@ -375,7 +376,7 @@ static int chsc_ioctl_on_close_remove(void)
 		ret = -ENOENT;
 		goto out_unlock;
 	}
-	free_page((unsigned long)on_close_chsc_area);
+	kfree(on_close_chsc_area);
 	on_close_chsc_area = NULL;
 	kfree(on_close_request);
 	on_close_request = NULL;
@@ -389,39 +390,29 @@ static int chsc_ioctl_on_close_remove(void)
 
 static int chsc_ioctl_start_sync(void __user *user_area)
 {
-	struct chsc_sync_area *chsc_area;
-	int ret, ccode;
+	struct chsc_sync_area *chsc_area __free(kfree) = NULL;
+	int ccode;
 
-	chsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	chsc_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!chsc_area)
 		return -ENOMEM;
-	if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) {
-		ret = -EFAULT;
-		goto out_free;
-	}
-	if (chsc_area->header.code & 0x4000) {
-		ret = -EINVAL;
-		goto out_free;
-	}
+	if (copy_from_user(chsc_area, user_area, PAGE_SIZE))
+		return -EFAULT;
+	if (chsc_area->header.code & 0x4000)
+		return -EINVAL;
 	chsc_log_command(chsc_area);
 	ccode = chsc(chsc_area);
-	if (ccode != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
+	if (ccode != 0)
+		return -EIO;
 	if (copy_to_user(user_area, chsc_area, PAGE_SIZE))
-		ret = -EFAULT;
-	else
-		ret = 0;
-out_free:
-	free_page((unsigned long)chsc_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static int chsc_ioctl_info_channel_path(void __user *user_cd)
 {
-	struct chsc_chp_cd *cd;
-	int ret, ccode;
+	struct chsc_chp_cd *cd __free(kfree) = NULL;
+	int ccode;
 	struct {
 		struct chsc_header request;
 		u32 : 2;
@@ -436,20 +427,16 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd)
 		u32 : 32;
 		struct chsc_header response;
 		u8 data[PAGE_SIZE - 20];
-	} __attribute__ ((packed)) *scpcd_area;
+	} __attribute__ ((packed)) *scpcd_area __free(kfree) = NULL;
 
-	scpcd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	scpcd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!scpcd_area)
 		return -ENOMEM;
 	cd = kzalloc_obj(*cd);
-	if (!cd) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
-	if (copy_from_user(cd, user_cd, sizeof(*cd))) {
-		ret = -EFAULT;
-		goto out_free;
-	}
+	if (!cd)
+		return -ENOMEM;
+	if (copy_from_user(cd, user_cd, sizeof(*cd)))
+		return -EFAULT;
 	scpcd_area->request.length = 0x0010;
 	scpcd_area->request.code = 0x0028;
 	scpcd_area->m = cd->m;
@@ -459,31 +446,23 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd)
 	scpcd_area->last_chpid = cd->chpid.id;
 
 	ccode = chsc(scpcd_area);
-	if (ccode != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
+	if (ccode != 0)
+		return -EIO;
 	if (scpcd_area->response.code != 0x0001) {
-		ret = -EIO;
 		CHSC_MSG(0, "scpcd: response code=%x\n",
 			 scpcd_area->response.code);
-		goto out_free;
+		return -EIO;
 	}
 	memcpy(&cd->cpcb, &scpcd_area->response, scpcd_area->response.length);
 	if (copy_to_user(user_cd, cd, sizeof(*cd)))
-		ret = -EFAULT;
-	else
-		ret = 0;
-out_free:
-	kfree(cd);
-	free_page((unsigned long)scpcd_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static int chsc_ioctl_info_cu(void __user *user_cd)
 {
-	struct chsc_cu_cd *cd;
-	int ret, ccode;
+	struct chsc_cu_cd *cd __free(kfree) = NULL;
+	int ccode;
 	struct {
 		struct chsc_header request;
 		u32 : 2;
@@ -498,20 +477,16 @@ static int chsc_ioctl_info_cu(void __user *user_cd)
 		u32 : 32;
 		struct chsc_header response;
 		u8 data[PAGE_SIZE - 20];
-	} __attribute__ ((packed)) *scucd_area;
+	} __attribute__ ((packed)) *scucd_area __free(kfree) = NULL;
 
-	scucd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	scucd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!scucd_area)
 		return -ENOMEM;
 	cd = kzalloc_obj(*cd);
-	if (!cd) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
-	if (copy_from_user(cd, user_cd, sizeof(*cd))) {
-		ret = -EFAULT;
-		goto out_free;
-	}
+	if (!cd)
+		return -ENOMEM;
+	if (copy_from_user(cd, user_cd, sizeof(*cd)))
+		return -EFAULT;
 	scucd_area->request.length = 0x0010;
 	scucd_area->request.code = 0x0026;
 	scucd_area->m = cd->m;
@@ -521,31 +496,23 @@ static int chsc_ioctl_info_cu(void __user *user_cd)
 	scucd_area->last_cun = cd->cun;
 
 	ccode = chsc(scucd_area);
-	if (ccode != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
+	if (ccode != 0)
+		return -EIO;
 	if (scucd_area->response.code != 0x0001) {
-		ret = -EIO;
 		CHSC_MSG(0, "scucd: response code=%x\n",
 			 scucd_area->response.code);
-		goto out_free;
+		return -EIO;
 	}
 	memcpy(&cd->cucb, &scucd_area->response, scucd_area->response.length);
 	if (copy_to_user(user_cd, cd, sizeof(*cd)))
-		ret = -EFAULT;
-	else
-		ret = 0;
-out_free:
-	kfree(cd);
-	free_page((unsigned long)scucd_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static int chsc_ioctl_info_sch_cu(void __user *user_cud)
 {
-	struct chsc_sch_cud *cud;
-	int ret, ccode;
+	struct chsc_sch_cud *cud __free(kfree) = NULL;
+	int ccode;
 	struct {
 		struct chsc_header request;
 		u32 : 2;
@@ -561,20 +528,16 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud)
 		u32 : 32;
 		struct chsc_header response;
 		u8 data[PAGE_SIZE - 20];
-	} __attribute__ ((packed)) *sscud_area;
+	} __attribute__ ((packed)) *sscud_area __free(kfree) = NULL;
 
-	sscud_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	sscud_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!sscud_area)
 		return -ENOMEM;
 	cud = kzalloc_obj(*cud);
-	if (!cud) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
-	if (copy_from_user(cud, user_cud, sizeof(*cud))) {
-		ret = -EFAULT;
-		goto out_free;
-	}
+	if (!cud)
+		return -ENOMEM;
+	if (copy_from_user(cud, user_cud, sizeof(*cud)))
+		return -EFAULT;
 	sscud_area->request.length = 0x0010;
 	sscud_area->request.code = 0x0006;
 	sscud_area->m = cud->schid.m;
@@ -585,31 +548,23 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud)
 	sscud_area->last_sch = cud->schid.sch_no;
 
 	ccode = chsc(sscud_area);
-	if (ccode != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
+	if (ccode != 0)
+		return -EIO;
 	if (sscud_area->response.code != 0x0001) {
-		ret = -EIO;
 		CHSC_MSG(0, "sscud: response code=%x\n",
 			 sscud_area->response.code);
-		goto out_free;
+		return -EIO;
 	}
 	memcpy(&cud->scub, &sscud_area->response, sscud_area->response.length);
 	if (copy_to_user(user_cud, cud, sizeof(*cud)))
-		ret = -EFAULT;
-	else
-		ret = 0;
-out_free:
-	kfree(cud);
-	free_page((unsigned long)sscud_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static int chsc_ioctl_conf_info(void __user *user_ci)
 {
-	struct chsc_conf_info *ci;
-	int ret, ccode;
+	struct chsc_conf_info *ci __free(kfree) = NULL;
+	int ccode;
 	struct {
 		struct chsc_header request;
 		u32 : 2;
@@ -623,20 +578,16 @@ static int chsc_ioctl_conf_info(void __user *user_ci)
 		u64 : 64;
 		struct chsc_header response;
 		u8 data[PAGE_SIZE - 20];
-	} __attribute__ ((packed)) *sci_area;
+	} __attribute__ ((packed)) *sci_area __free(kfree) = NULL;
 
-	sci_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	sci_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!sci_area)
 		return -ENOMEM;
 	ci = kzalloc_obj(*ci);
-	if (!ci) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
-	if (copy_from_user(ci, user_ci, sizeof(*ci))) {
-		ret = -EFAULT;
-		goto out_free;
-	}
+	if (!ci)
+		return -ENOMEM;
+	if (copy_from_user(ci, user_ci, sizeof(*ci)))
+		return -EFAULT;
 	sci_area->request.length = 0x0010;
 	sci_area->request.code = 0x0012;
 	sci_area->m = ci->id.m;
@@ -645,31 +596,23 @@ static int chsc_ioctl_conf_info(void __user *user_ci)
 	sci_area->ssid = ci->id.ssid;
 
 	ccode = chsc(sci_area);
-	if (ccode != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
+	if (ccode != 0)
+		return -EIO;
 	if (sci_area->response.code != 0x0001) {
-		ret = -EIO;
 		CHSC_MSG(0, "sci: response code=%x\n",
 			 sci_area->response.code);
-		goto out_free;
+		return -EIO;
 	}
 	memcpy(&ci->scid, &sci_area->response, sci_area->response.length);
 	if (copy_to_user(user_ci, ci, sizeof(*ci)))
-		ret = -EFAULT;
-	else
-		ret = 0;
-out_free:
-	kfree(ci);
-	free_page((unsigned long)sci_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static int chsc_ioctl_conf_comp_list(void __user *user_ccl)
 {
-	struct chsc_comp_list *ccl;
-	int ret, ccode;
+	struct chsc_comp_list *ccl __free(kfree) = NULL;
+	int ccode;
 	struct {
 		struct chsc_header request;
 		u32 ctype : 8;
@@ -681,7 +624,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl)
 		u64 : 64;
 		struct chsc_header response;
 		u8 data[PAGE_SIZE - 36];
-	} __attribute__ ((packed)) *sccl_area;
+	} __attribute__ ((packed)) *sccl_area __free(kfree) = NULL;
 	struct {
 		u32 m : 1;
 		u32 : 31;
@@ -696,18 +639,14 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl)
 		u32 res;
 	} __attribute__ ((packed)) *cssids_parm;
 
-	sccl_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	sccl_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!sccl_area)
 		return -ENOMEM;
 	ccl = kzalloc_obj(*ccl);
-	if (!ccl) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
-	if (copy_from_user(ccl, user_ccl, sizeof(*ccl))) {
-		ret = -EFAULT;
-		goto out_free;
-	}
+	if (!ccl)
+		return -ENOMEM;
+	if (copy_from_user(ccl, user_ccl, sizeof(*ccl)))
+		return -EFAULT;
 	sccl_area->request.length = 0x0020;
 	sccl_area->request.code = 0x0030;
 	sccl_area->fmt = ccl->req.fmt;
@@ -728,61 +667,46 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl)
 		break;
 	}
 	ccode = chsc(sccl_area);
-	if (ccode != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
+	if (ccode != 0)
+		return -EIO;
 	if (sccl_area->response.code != 0x0001) {
-		ret = -EIO;
 		CHSC_MSG(0, "sccl: response code=%x\n",
 			 sccl_area->response.code);
-		goto out_free;
+		return -EIO;
 	}
 	memcpy(&ccl->sccl, &sccl_area->response, sccl_area->response.length);
 	if (copy_to_user(user_ccl, ccl, sizeof(*ccl)))
-		ret = -EFAULT;
-	else
-		ret = 0;
-out_free:
-	kfree(ccl);
-	free_page((unsigned long)sccl_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static int chsc_ioctl_chpd(void __user *user_chpd)
 {
-	struct chsc_scpd *scpd_area;
-	struct chsc_cpd_info *chpd;
+	struct chsc_scpd *scpd_area __free(kfree) = NULL;
+	struct chsc_cpd_info *chpd __free(kfree) = NULL;
 	int ret;
 
 	chpd = kzalloc_obj(*chpd);
-	scpd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
-	if (!scpd_area || !chpd) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
-	if (copy_from_user(chpd, user_chpd, sizeof(*chpd))) {
-		ret = -EFAULT;
-		goto out_free;
-	}
+	scpd_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
+	if (!scpd_area || !chpd)
+		return -ENOMEM;
+	if (copy_from_user(chpd, user_chpd, sizeof(*chpd)))
+		return -EFAULT;
 	ret = chsc_determine_channel_path_desc(chpd->chpid, chpd->fmt,
 					       chpd->rfmt, chpd->c, chpd->m,
 					       scpd_area);
 	if (ret)
-		goto out_free;
+		return ret;
 	memcpy(&chpd->chpdb, &scpd_area->response, scpd_area->response.length);
 	if (copy_to_user(user_chpd, chpd, sizeof(*chpd)))
-		ret = -EFAULT;
-out_free:
-	kfree(chpd);
-	free_page((unsigned long)scpd_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static int chsc_ioctl_dcal(void __user *user_dcal)
 {
-	struct chsc_dcal *dcal;
-	int ret, ccode;
+	struct chsc_dcal *dcal __free(kfree) = NULL;
+	int ccode;
 	struct {
 		struct chsc_header request;
 		u32 atype : 8;
@@ -794,20 +718,16 @@ static int chsc_ioctl_dcal(void __user *user_dcal)
 		u32 res1[2];
 		struct chsc_header response;
 		u8 data[PAGE_SIZE - 36];
-	} __attribute__ ((packed)) *sdcal_area;
+	} __attribute__ ((packed)) *sdcal_area __free(kfree) = NULL;
 
-	sdcal_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	sdcal_area = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!sdcal_area)
 		return -ENOMEM;
 	dcal = kzalloc_obj(*dcal);
-	if (!dcal) {
-		ret = -ENOMEM;
-		goto out_free;
-	}
-	if (copy_from_user(dcal, user_dcal, sizeof(*dcal))) {
-		ret = -EFAULT;
-		goto out_free;
-	}
+	if (!dcal)
+		return -ENOMEM;
+	if (copy_from_user(dcal, user_dcal, sizeof(*dcal)))
+		return -EFAULT;
 	sdcal_area->request.length = 0x0020;
 	sdcal_area->request.code = 0x0034;
 	sdcal_area->atype = dcal->req.atype;
@@ -816,26 +736,18 @@ static int chsc_ioctl_dcal(void __user *user_dcal)
 	       sizeof(sdcal_area->list_parm));
 
 	ccode = chsc(sdcal_area);
-	if (ccode != 0) {
-		ret = -EIO;
-		goto out_free;
-	}
+	if (ccode != 0)
+		return -EIO;
 	if (sdcal_area->response.code != 0x0001) {
-		ret = -EIO;
 		CHSC_MSG(0, "sdcal: response code=%x\n",
 			 sdcal_area->response.code);
-		goto out_free;
+		return -EIO;
 	}
 	memcpy(&dcal->sdcal, &sdcal_area->response,
 	       sdcal_area->response.length);
 	if (copy_to_user(user_dcal, dcal, sizeof(*dcal)))
-		ret = -EFAULT;
-	else
-		ret = 0;
-out_free:
-	kfree(dcal);
-	free_page((unsigned long)sdcal_area);
-	return ret;
+		return -EFAULT;
+	return 0;
 }
 
 static long chsc_ioctl(struct file *filp, unsigned int cmd,
@@ -904,7 +816,7 @@ static int chsc_release(struct inode *inode, struct file *filp)
 	}
 	snprintf(dbf, sizeof(dbf), "relret:%d", ret);
 	CHSC_LOG(0, dbf);
-	free_page((unsigned long)on_close_chsc_area);
+	kfree(on_close_chsc_area);
 	on_close_chsc_area = NULL;
 	kfree(on_close_request);
 	on_close_request = NULL;
diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c
index 70dc8cc765948a..e1c62eb60cca70 100644
--- a/drivers/s390/cio/cio.c
+++ b/drivers/s390/cio/cio.c
@@ -453,7 +453,8 @@ EXPORT_SYMBOL_GPL(cio_commit_config);
 /**
  * cio_update_schib - Perform stsch and update schib if subchannel is valid.
  * @sch: subchannel on which to perform stsch
- * Return zero on success, -ENODEV otherwise.
+ * Return zero on success, -ENODEV if the subchannel is not operational,
+ * -EACCES if the subchannel has no valid device.
  */
 int cio_update_schib(struct subchannel *sch)
 {
@@ -462,10 +463,12 @@ int cio_update_schib(struct subchannel *sch)
 	if (stsch(sch->schid, &schib))
 		return -ENODEV;
 
-	memcpy(&sch->schib, &schib, sizeof(schib));
-
-	if (!css_sch_is_valid(&schib))
+	if (!css_sch_is_valid(&schib)) {
+		memset(&sch->schib, 0, sizeof(sch->schib));
 		return -EACCES;
+	}
+
+	memcpy(&sch->schib, &schib, sizeof(schib));
 
 	return 0;
 }
diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h
index bad142c536e1e6..6d28a62bc67a37 100644
--- a/drivers/s390/cio/cio.h
+++ b/drivers/s390/cio/cio.h
@@ -7,6 +7,7 @@
 #include <linux/mod_devicetable.h>
 #include <asm/chpid.h>
 #include <asm/cio.h>
+#include <asm/dma-types.h>
 #include <asm/fcx.h>
 #include <asm/schid.h>
 #include <asm/tpi.h>
@@ -49,7 +50,7 @@ struct pmcw {
 
 /* Target SCHIB configuration. */
 struct schib_config {
-	u64 mba;
+	dma64_t mba;
 	u32 intparm;
 	u16 mbi;
 	u32 isc:3;
@@ -66,7 +67,7 @@ struct schib_config {
 struct schib {
 	struct pmcw pmcw;	 /* path management control word */
 	union scsw scsw;	 /* subchannel status word */
-	__u64 mba;               /* measurement block address */
+	dma64_t mba;		 /* measurement block address */
 	__u8 mda[4];		 /* model dependent area */
 } __attribute__ ((packed,aligned(4)));
 
diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c
index 92ab3d546fe47c..02b0feb9d1fd03 100644
--- a/drivers/s390/cio/cmf.c
+++ b/drivers/s390/cio/cmf.c
@@ -183,7 +183,7 @@ static int set_schib(struct ccw_device *cdev, u32 mme, int mbfc,
 	sch->config.mbfc = mbfc;
 	/* address can be either a block address or a block index */
 	if (mbfc)
-		sch->config.mba = address;
+		sch->config.mba = address ? virt_to_dma64((void *)address) : 0;
 	else
 		sch->config.mbi = address;
 
@@ -482,6 +482,9 @@ static int alloc_cmb(struct ccw_device *cdev)
 	ssize_t size;
 	struct cmb_data *cmb_data;
 
+	if (cmb_area.num_channels <= 0)
+		return -ENOMEM;
+
 	/* Allocate private cmb_data. */
 	cmb_data = kzalloc_obj(struct cmb_data);
 	if (!cmb_data)
@@ -501,12 +504,12 @@ static int alloc_cmb(struct ccw_device *cdev)
 		WARN_ON(!list_empty(&cmb_area.list));
 
 		spin_unlock(&cmb_area.lock);
-		mem = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
+		mem = kmalloc(size, GFP_KERNEL);
 		spin_lock(&cmb_area.lock);
 
 		if (cmb_area.mem) {
 			/* ok, another thread was faster */
-			free_pages((unsigned long)mem, get_order(size));
+			kfree(mem);
 		} else if (!mem) {
 			/* no luck */
 			ret = -ENOMEM;
@@ -547,10 +550,8 @@ static void free_cmb(struct ccw_device *cdev)
 	list_del_init(&priv->cmb_list);
 
 	if (list_empty(&cmb_area.list)) {
-		ssize_t size;
-		size = sizeof(struct cmb) * cmb_area.num_channels;
 		cmf_activate(NULL, CMF_OFF);
-		free_pages((unsigned long)cmb_area.mem, get_order(size));
+		kfree(cmb_area.mem);
 		cmb_area.mem = NULL;
 	}
 	spin_unlock_irq(cdev->ccwlock);
diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
index fb591118ecb2ae..68dd4a62975d22 100644
--- a/drivers/s390/cio/device.c
+++ b/drivers/s390/cio/device.c
@@ -922,7 +922,7 @@ static int ccw_device_move_to_sch(struct ccw_device *cdev,
 
 	if (!sch_is_pseudo_sch(old_sch)) {
 		spin_lock_irq(&old_sch->lock);
-		old_enabled = old_sch->schib.pmcw.ena;
+		old_enabled = old_sch->schib.pmcw.dnv && old_sch->schib.pmcw.ena;
 		rc = 0;
 		if (old_enabled)
 			rc = cio_disable_subchannel(old_sch);
@@ -941,7 +941,7 @@ static int ccw_device_move_to_sch(struct ccw_device *cdev,
 		CIO_MSG_EVENT(0, "device_move(0.%x.%04x,0.%x.%04x)=%d\n",
 			      cdev->private->dev_id.ssid,
 			      cdev->private->dev_id.devno, sch->schid.ssid,
-			      sch->schib.pmcw.dev, rc);
+			      sch->schid.sch_no, rc);
 		if (old_enabled) {
 			/* Try to re-enable the old subchannel. */
 			spin_lock_irq(&old_sch->lock);
@@ -1207,7 +1207,7 @@ static void io_subchannel_quiesce(struct subchannel *sch)
 	cdev = sch_get_cdev(sch);
 	if (cio_is_console(sch->schid))
 		goto out_unlock;
-	if (!sch->schib.pmcw.ena)
+	if (!sch->schib.pmcw.dnv || !sch->schib.pmcw.ena)
 		goto out_unlock;
 	ret = cio_disable_subchannel(sch);
 	if (ret != -EBUSY)
@@ -1254,7 +1254,8 @@ static int recovery_check(struct device *dev, void *data)
 	switch (cdev->private->state) {
 	case DEV_STATE_ONLINE:
 		sch = to_subchannel(cdev->dev.parent);
-		if ((sch->schib.pmcw.pam & sch->opm) == sch->vpm)
+		if (sch->schib.pmcw.dnv &&
+		    (sch->schib.pmcw.pam & sch->opm) == sch->vpm)
 			break;
 		fallthrough;
 	case DEV_STATE_DISCONNECTED:
diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c
index ab419d40a8a7aa..b5686c25c83c7f 100644
--- a/drivers/s390/cio/device_fsm.c
+++ b/drivers/s390/cio/device_fsm.c
@@ -170,6 +170,9 @@ __recover_lost_chpids(struct subchannel *sch, int old_lpm)
 	int mask, i;
 	struct chp_id chpid;
 
+	if (!sch->schib.pmcw.dnv)
+		return;
+
 	chp_id_init(&chpid);
 	for (i = 0; i<8; i++) {
 		mask = 0x80 >> i;
diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c
index 61c07b4a0fe892..1f7e83fd50872b 100644
--- a/drivers/s390/cio/device_ops.c
+++ b/drivers/s390/cio/device_ops.c
@@ -142,6 +142,8 @@ int ccw_device_clear(struct ccw_device *cdev, unsigned long intparm)
 	if (!cdev || !cdev->dev.parent)
 		return -ENODEV;
 	sch = to_subchannel(cdev->dev.parent);
+	if (!sch->schib.pmcw.dnv)
+		return -ENODEV;
 	if (!sch->schib.pmcw.ena)
 		return -EINVAL;
 	if (cdev->private->state == DEV_STATE_NOT_OPER)
@@ -198,6 +200,8 @@ int ccw_device_start_timeout_key(struct ccw_device *cdev, struct ccw1 *cpa,
 	if (!cdev || !cdev->dev.parent)
 		return -ENODEV;
 	sch = to_subchannel(cdev->dev.parent);
+	if (!sch->schib.pmcw.dnv)
+		return -ENODEV;
 	if (!sch->schib.pmcw.ena)
 		return -EINVAL;
 	if (cdev->private->state == DEV_STATE_NOT_OPER)
@@ -379,6 +383,8 @@ int ccw_device_halt(struct ccw_device *cdev, unsigned long intparm)
 	if (!cdev || !cdev->dev.parent)
 		return -ENODEV;
 	sch = to_subchannel(cdev->dev.parent);
+	if (!sch->schib.pmcw.dnv)
+		return -ENODEV;
 	if (!sch->schib.pmcw.ena)
 		return -EINVAL;
 	if (cdev->private->state == DEV_STATE_NOT_OPER)
@@ -413,6 +419,8 @@ int ccw_device_resume(struct ccw_device *cdev)
 	if (!cdev || !cdev->dev.parent)
 		return -ENODEV;
 	sch = to_subchannel(cdev->dev.parent);
+	if (!sch->schib.pmcw.dnv)
+		return -ENODEV;
 	if (!sch->schib.pmcw.ena)
 		return -EINVAL;
 	if (cdev->private->state == DEV_STATE_NOT_OPER)
@@ -482,6 +490,8 @@ struct channel_path_desc_fmt0 *ccw_device_get_chp_desc(struct ccw_device *cdev,
 	struct chp_id chpid;
 
 	sch = to_subchannel(cdev->dev.parent);
+	if (!sch->schib.pmcw.dnv)
+		return NULL;
 	chp_id_init(&chpid);
 	chpid.id = sch->schib.pmcw.chpid[chp_idx];
 	return chp_get_chp_desc(chpid);
@@ -502,9 +512,13 @@ u8 *ccw_device_get_util_str(struct ccw_device *cdev, int chp_idx)
 	struct chp_id chpid;
 	u8 *util_str;
 
+	if (!sch->schib.pmcw.dnv)
+		return NULL;
 	chp_id_init(&chpid);
 	chpid.id = sch->schib.pmcw.chpid[chp_idx];
 	chp = chpid_to_chp(chpid);
+	if (!chp)
+		return NULL;
 
 	util_str = kmalloc(sizeof(chp->desc_fmt3.util_str), GFP_KERNEL);
 	if (!util_str)
@@ -548,6 +562,8 @@ int ccw_device_tm_start_timeout_key(struct ccw_device *cdev, struct tcw *tcw,
 	int rc;
 
 	sch = to_subchannel(cdev->dev.parent);
+	if (!sch->schib.pmcw.dnv)
+		return -ENODEV;
 	if (!sch->schib.pmcw.ena)
 		return -EINVAL;
 	if (cdev->private->state == DEV_STATE_VERIFY) {
@@ -652,6 +668,9 @@ int ccw_device_get_mdc(struct ccw_device *cdev, u8 mask)
 	struct chp_id chpid;
 	int mdc = 0, i;
 
+	if (!sch->schib.pmcw.dnv)
+		return 0;
+
 	/* Adjust requested path mask to excluded varied off paths. */
 	if (mask)
 		mask &= sch->lpm;
@@ -694,6 +713,8 @@ int ccw_device_tm_intrg(struct ccw_device *cdev)
 {
 	struct subchannel *sch = to_subchannel(cdev->dev.parent);
 
+	if (!sch->schib.pmcw.dnv)
+		return -ENODEV;
 	if (!sch->schib.pmcw.ena)
 		return -EINVAL;
 	if (cdev->private->state != DEV_STATE_ONLINE)
@@ -786,6 +807,8 @@ int ccw_device_get_chpid(struct ccw_device *cdev, int chp_idx, u8 *chpid)
 
 	if ((chp_idx < 0) || (chp_idx > 7))
 		return -EINVAL;
+	if (!sch->schib.pmcw.dnv)
+		return -ENODEV;
 	mask = 0x80 >> chp_idx;
 	if (!(sch->schib.pmcw.pim & mask))
 		return -ENODEV;
diff --git a/drivers/s390/cio/qdio.h b/drivers/s390/cio/qdio.h
index 4bd4c00c9c0ca6..dff5f53a8795df 100644
--- a/drivers/s390/cio/qdio.h
+++ b/drivers/s390/cio/qdio.h
@@ -244,7 +244,7 @@ struct qdio_irq {
 	int perf_stat_enabled;
 
 	struct qdr *qdr;
-	unsigned long chsc_page;
+	void *chsc_page;
 
 	struct qdio_q *input_qs[QDIO_MAX_QUEUES_PER_IRQ];
 	struct qdio_q *output_qs[QDIO_MAX_QUEUES_PER_IRQ];
diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c
index c1e09fa34e7745..889f63f94a4a78 100644
--- a/drivers/s390/cio/qdio_main.c
+++ b/drivers/s390/cio/qdio_main.c
@@ -12,11 +12,11 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
-#include <linux/kmemleak.h>
 #include <linux/delay.h>
 #include <linux/gfp.h>
 #include <linux/io.h>
 #include <linux/atomic.h>
+#include <linux/slab.h>
 #include <asm/debug.h>
 #include <asm/qdio.h>
 #include <asm/asm.h>
@@ -933,10 +933,10 @@ int qdio_free(struct ccw_device *cdev)
 	mutex_unlock(&irq_ptr->setup_mutex);
 
 	qdio_free_queues(irq_ptr);
-	free_page((unsigned long) irq_ptr->qdr);
-	free_page(irq_ptr->chsc_page);
+	kfree(irq_ptr->qdr);
+	kfree(irq_ptr->chsc_page);
 	kfree(irq_ptr->ccw);
-	free_page((unsigned long) irq_ptr);
+	kfree(irq_ptr);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(qdio_free);
@@ -961,7 +961,7 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 	    no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)
 		return -EINVAL;
 
-	irq_ptr = (void *) get_zeroed_page(GFP_KERNEL);
+	irq_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!irq_ptr)
 		return -ENOMEM;
 
@@ -969,9 +969,6 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 	if (!irq_ptr->ccw)
 		goto err_ccw;
 
-	/* kmemleak doesn't scan the page-allocated irq_ptr: */
-	kmemleak_not_leak(irq_ptr->ccw);
-
 	irq_ptr->cdev = cdev;
 	mutex_init(&irq_ptr->setup_mutex);
 	if (qdio_allocate_dbf(irq_ptr))
@@ -986,12 +983,12 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 	 * qdio_establish. In case of low memory and swap on a zfcp disk
 	 * we may not be able to allocate memory otherwise.
 	 */
-	irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
+	irq_ptr->chsc_page = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!irq_ptr->chsc_page)
 		goto err_chsc;
 
 	/* qdr is used in ccw1.cda which is u32 */
-	irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	irq_ptr->qdr = kzalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!irq_ptr->qdr)
 		goto err_qdr;
 
@@ -1004,14 +1001,14 @@ int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
 	return 0;
 
 err_queues:
-	free_page((unsigned long) irq_ptr->qdr);
+	kfree(irq_ptr->qdr);
 err_qdr:
-	free_page(irq_ptr->chsc_page);
+	kfree(irq_ptr->chsc_page);
 err_chsc:
 err_dbf:
 	kfree(irq_ptr->ccw);
 err_ccw:
-	free_page((unsigned long) irq_ptr);
+	kfree(irq_ptr);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(qdio_allocate);
diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c
index ea09aadaae4eca..e1c5c6a8b3b270 100644
--- a/drivers/s390/cio/qdio_setup.c
+++ b/drivers/s390/cio/qdio_setup.c
@@ -35,7 +35,7 @@ void qdio_free_buffers(struct qdio_buffer **buf, unsigned int count)
 	int pos;
 
 	for (pos = 0; pos < count; pos += QBUFF_PER_PAGE)
-		free_page((unsigned long) buf[pos]);
+		kfree(buf[pos]);
 }
 EXPORT_SYMBOL_GPL(qdio_free_buffers);
 
@@ -49,7 +49,7 @@ int qdio_alloc_buffers(struct qdio_buffer **buf, unsigned int count)
 	int pos;
 
 	for (pos = 0; pos < count; pos += QBUFF_PER_PAGE) {
-		buf[pos] = (void *) get_zeroed_page(GFP_KERNEL);
+		buf[pos] = kzalloc(PAGE_SIZE, GFP_KERNEL);
 		if (!buf[pos]) {
 			qdio_free_buffers(buf, count);
 			return -ENOMEM;
@@ -83,7 +83,7 @@ static void __qdio_free_queues(struct qdio_q **queues, unsigned int count)
 
 	for (i = 0; i < count; i++) {
 		q = queues[i];
-		free_page((unsigned long)q->sl_page);
+		kfree(q->sl_page);
 		kmem_cache_free(qdio_q_cache, q);
 	}
 }
@@ -109,7 +109,7 @@ static int __qdio_allocate_qs(struct qdio_q **irq_ptr_qs, int nr_queues)
 			return -ENOMEM;
 		}
 
-		q->sl_page = (void *)__get_free_page(GFP_KERNEL);
+		q->sl_page = kmalloc(PAGE_SIZE, GFP_KERNEL);
 		if (!q->sl_page) {
 			kmem_cache_free(qdio_q_cache, q);
 			__qdio_free_queues(irq_ptr_qs, i);
@@ -249,11 +249,11 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr,
 
 	DBF_EVENT("getssqd:%4x", schid->sch_no);
 	if (!irq_ptr) {
-		ssqd = (struct chsc_ssqd_area *)__get_free_page(GFP_KERNEL);
+		ssqd = kmalloc(PAGE_SIZE, GFP_KERNEL);
 		if (!ssqd)
 			return -ENOMEM;
 	} else {
-		ssqd = (struct chsc_ssqd_area *)irq_ptr->chsc_page;
+		ssqd = irq_ptr->chsc_page;
 	}
 
 	rc = chsc_ssqd(*schid, ssqd);
@@ -270,7 +270,7 @@ int qdio_setup_get_ssqd(struct qdio_irq *irq_ptr,
 
 out:
 	if (!irq_ptr)
-		free_page((unsigned long)ssqd);
+		kfree(ssqd);
 
 	return rc;
 }
diff --git a/drivers/s390/cio/qdio_thinint.c b/drivers/s390/cio/qdio_thinint.c
index e167aa75c3dff7..a2ff51887537fe 100644
--- a/drivers/s390/cio/qdio_thinint.c
+++ b/drivers/s390/cio/qdio_thinint.c
@@ -136,7 +136,7 @@ static struct airq_struct tiqdio_airq = {
 
 static int set_subchannel_ind(struct qdio_irq *irq_ptr, int reset)
 {
-	struct chsc_scssc_area *scssc = (void *)irq_ptr->chsc_page;
+	struct chsc_scssc_area *scssc = irq_ptr->chsc_page;
 	dma64_t summary_indicator_addr, subchannel_indicator_addr;
 	int rc;
 
diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c
index 171212a6d2d9cd..b79fe023ab8d18 100644
--- a/drivers/s390/cio/scm.c
+++ b/drivers/s390/cio/scm.c
@@ -13,6 +13,7 @@
 #include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/err.h>
+#include <linux/cleanup.h>
 #include <asm/eadm.h>
 #include "chsc.h"
 
@@ -224,12 +225,12 @@ static int scm_add(struct chsc_scm_info *scm_info, size_t num)
 
 int scm_update_information(void)
 {
-	struct chsc_scm_info *scm_info;
+	struct chsc_scm_info *scm_info __free(kfree) = NULL;
 	u64 token = 0;
 	size_t num;
 	int ret;
 
-	scm_info = (void *)__get_free_page(GFP_KERNEL | GFP_DMA);
+	scm_info = kmalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA);
 	if (!scm_info)
 		return -ENOMEM;
 
@@ -250,8 +251,6 @@ int scm_update_information(void)
 		token = scm_info->restok;
 	} while (token);
 
-	free_page((unsigned long)scm_info);
-
 	return ret;
 }
 
diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c
index 5fd94e9d5c6181..9a000b0231d60e 100644
--- a/drivers/s390/cio/vfio_ccw_fsm.c
+++ b/drivers/s390/cio/vfio_ccw_fsm.c
@@ -399,7 +399,7 @@ static void fsm_close(struct vfio_ccw_private *private,
 
 	spin_lock_irq(&sch->lock);
 
-	if (!sch->schib.pmcw.ena)
+	if (!sch->schib.pmcw.dnv || !sch->schib.pmcw.ena)
 		goto err_unlock;
 
 	ret = cio_disable_subchannel(sch);
diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index 940c0ff668beef..4db878c18f41c1 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -277,9 +277,9 @@ static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
 {
 	if (!q)
 		return;
-	if (q->saved_isc != VFIO_AP_ISC_INVALID &&
-	    !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
-		kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
+	if (q->saved_isc != VFIO_AP_ISC_INVALID) {
+		if (!WARN_ON(!q->matrix_mdev) && q->matrix_mdev->kvm)
+			kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
 		q->saved_isc = VFIO_AP_ISC_INVALID;
 	}
 	if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {
@@ -1935,6 +1935,8 @@ static int apq_status_check(int apqn, struct ap_queue_status *status)
 		 * a value indicating a reset needs to be performed again.
 		 */
 		return -EAGAIN;
+	case AP_RESPONSE_Q_NOT_AVAIL:
+		return -ENODEV;
 	default:
 		WARN(true,
 		     "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n",
@@ -1961,6 +1963,10 @@ static void apq_reset_check(struct work_struct *reset_work)
 		ret = apq_status_check(q->apqn, &status);
 		if (ret == -EIO)
 			return;
+		if (ret == -ENODEV) {
+			vfio_ap_free_aqic_resources(q);
+			return;
+		}
 		if (ret == -EBUSY) {
 			pr_notice_ratelimited(WAIT_MSG, elapsed,
 					      AP_QID_CARD(q->apqn),
@@ -2004,6 +2010,7 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q)
 		break;
 	case AP_RESPONSE_DECONFIGURED:
 	case AP_RESPONSE_CHECKSTOPPED:
+	case AP_RESPONSE_Q_NOT_AVAIL:
 		vfio_ap_free_aqic_resources(q);
 		break;
 	default:
@@ -2528,12 +2535,15 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev)
 	/*
 	 * If the queue is not in the host's AP configuration, then resetting
 	 * it will fail with response code 01, (APQN not valid); so, let's make
-	 * sure it is in the host's config.
+	 * sure it is in the host's config. If it is not, free the KVM GISC
+	 * resources.
 	 */
 	if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) &&
 	    test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) {
 		vfio_ap_mdev_reset_queue(q);
 		flush_work(&q->reset_work);
+	} else {
+		vfio_ap_free_aqic_resources(q);
 	}
 
 done:
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index ec6c0e10211997..adfe15f5cb4298 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -95,11 +95,11 @@ static void zfcp_erp_action_dismiss_port(struct zfcp_port *port)
 	if (atomic_read(&port->status) & ZFCP_STATUS_COMMON_ERP_INUSE)
 		zfcp_erp_action_dismiss(&port->erp_action);
 	else {
-		spin_lock(port->adapter->scsi_host->host_lock);
+		spin_lock(&port->adapter->scsi_host->host_lock);
 		__shost_for_each_device(sdev, port->adapter->scsi_host)
 			if (sdev_to_zfcp(sdev)->port == port)
 				zfcp_erp_action_dismiss_lun(sdev);
-		spin_unlock(port->adapter->scsi_host->host_lock);
+		spin_unlock(&port->adapter->scsi_host->host_lock);
 	}
 }
 
@@ -670,11 +670,11 @@ static void _zfcp_erp_lun_reopen_all(struct zfcp_port *port, int clear,
 {
 	struct scsi_device *sdev;
 
-	spin_lock(port->adapter->scsi_host->host_lock);
+	spin_lock(&port->adapter->scsi_host->host_lock);
 	__shost_for_each_device(sdev, port->adapter->scsi_host)
 		if (sdev_to_zfcp(sdev)->port == port)
 			_zfcp_erp_lun_reopen(sdev, clear, dbftag, 0);
-	spin_unlock(port->adapter->scsi_host->host_lock);
+	spin_unlock(&port->adapter->scsi_host->host_lock);
 }
 
 static void zfcp_erp_strategy_followup_failed(struct zfcp_erp_action *act)
@@ -1429,7 +1429,7 @@ static void zfcp_erp_try_rport_unblock(struct zfcp_port *port)
 		write_unlock_irqrestore(&adapter->erp_lock, flags);
 		return;
 	}
-	spin_lock(shost->host_lock);
+	spin_lock(&shost->host_lock);
 	__shost_for_each_device(sdev, shost) {
 		struct zfcp_scsi_dev *zsdev = sdev_to_zfcp(sdev);
 		int lun_status;
@@ -1451,7 +1451,7 @@ static void zfcp_erp_try_rport_unblock(struct zfcp_port *port)
 			 * or meanwhile blocked [new LUN recovery triggered]
 			 */
 			zfcp_dbf_rec_run_lvl(4, "ertru_l", &zsdev->erp_action);
-			spin_unlock(shost->host_lock);
+			spin_unlock(&shost->host_lock);
 			write_unlock_irqrestore(&adapter->erp_lock, flags);
 			return;
 		}
@@ -1460,7 +1460,7 @@ static void zfcp_erp_try_rport_unblock(struct zfcp_port *port)
 	 * and no ERP of severity >= port was meanwhile triggered elsewhere
 	 */
 	zfcp_scsi_schedule_rport_register(port);
-	spin_unlock(shost->host_lock);
+	spin_unlock(&shost->host_lock);
 	write_unlock_irqrestore(&adapter->erp_lock, flags);
 }
 
@@ -1713,10 +1713,10 @@ void zfcp_erp_set_adapter_status(struct zfcp_adapter *adapter, u32 mask)
 	if (adapter->scsi_host == NULL)
 		return;
 
-	spin_lock_irqsave(adapter->scsi_host->host_lock, flags);
+	spin_lock_irqsave(&adapter->scsi_host->host_lock, flags);
 	__shost_for_each_device(sdev, adapter->scsi_host)
 		atomic_or(common_mask, &sdev_to_zfcp(sdev)->status);
-	spin_unlock_irqrestore(adapter->scsi_host->host_lock, flags);
+	spin_unlock_irqrestore(&adapter->scsi_host->host_lock, flags);
 }
 
 /**
@@ -1757,13 +1757,13 @@ void zfcp_erp_clear_adapter_status(struct zfcp_adapter *adapter, u32 mask)
 	if (adapter->scsi_host == NULL)
 		return;
 
-	spin_lock_irqsave(adapter->scsi_host->host_lock, flags);
+	spin_lock_irqsave(&adapter->scsi_host->host_lock, flags);
 	__shost_for_each_device(sdev, adapter->scsi_host) {
 		atomic_andnot(common_mask, &sdev_to_zfcp(sdev)->status);
 		if (clear_counter)
 			atomic_set(&sdev_to_zfcp(sdev)->erp_counter, 0);
 	}
-	spin_unlock_irqrestore(adapter->scsi_host->host_lock, flags);
+	spin_unlock_irqrestore(&adapter->scsi_host->host_lock, flags);
 }
 
 /**
@@ -1784,12 +1784,12 @@ void zfcp_erp_set_port_status(struct zfcp_port *port, u32 mask)
 	if (!common_mask)
 		return;
 
-	spin_lock_irqsave(port->adapter->scsi_host->host_lock, flags);
+	spin_lock_irqsave(&port->adapter->scsi_host->host_lock, flags);
 	__shost_for_each_device(sdev, port->adapter->scsi_host)
 		if (sdev_to_zfcp(sdev)->port == port)
 			atomic_or(common_mask,
 					&sdev_to_zfcp(sdev)->status);
-	spin_unlock_irqrestore(port->adapter->scsi_host->host_lock, flags);
+	spin_unlock_irqrestore(&port->adapter->scsi_host->host_lock, flags);
 }
 
 /**
@@ -1814,7 +1814,7 @@ void zfcp_erp_clear_port_status(struct zfcp_port *port, u32 mask)
 	if (clear_counter)
 		atomic_set(&port->erp_counter, 0);
 
-	spin_lock_irqsave(port->adapter->scsi_host->host_lock, flags);
+	spin_lock_irqsave(&port->adapter->scsi_host->host_lock, flags);
 	__shost_for_each_device(sdev, port->adapter->scsi_host)
 		if (sdev_to_zfcp(sdev)->port == port) {
 			atomic_andnot(common_mask,
@@ -1822,7 +1822,7 @@ void zfcp_erp_clear_port_status(struct zfcp_port *port, u32 mask)
 			if (clear_counter)
 				atomic_set(&sdev_to_zfcp(sdev)->erp_counter, 0);
 		}
-	spin_unlock_irqrestore(port->adapter->scsi_host->host_lock, flags);
+	spin_unlock_irqrestore(&port->adapter->scsi_host->host_lock, flags);
 }
 
 /**
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index 0eb9bc9d03e440..cafdeaf2919501 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -176,7 +176,7 @@ extern void zfcp_scsi_schedule_rport_register(struct zfcp_port *);
 extern void zfcp_scsi_schedule_rport_block(struct zfcp_port *);
 extern void zfcp_scsi_schedule_rports_block(struct zfcp_adapter *);
 extern void zfcp_scsi_set_prot(struct zfcp_adapter *);
-extern void zfcp_scsi_dif_sense_error(struct scsi_cmnd *, int);
+extern void zfcp_scsi_dif_sense_error(struct scsi_cmnd *, u16);
 extern void zfcp_scsi_shost_update_config_data(
 	struct zfcp_adapter *const adapter,
 	const struct fsf_qtcb_bottom_config *const bottom,
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index d407f16e77efc1..1ea6ab720fc0ba 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -2485,13 +2485,16 @@ static void zfcp_fsf_fcp_cmnd_handler(struct zfcp_fsf_req *req)
 		set_host_byte(scpnt, DID_ERROR);
 		goto skip_fsfstatus;
 	case FSF_BLOCK_GUARD_CHECK_FAILURE:
-		zfcp_scsi_dif_sense_error(scpnt, 0x1);
+		zfcp_scsi_dif_sense_error(scpnt,
+				LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 		goto skip_fsfstatus;
 	case FSF_APP_TAG_CHECK_FAILURE:
-		zfcp_scsi_dif_sense_error(scpnt, 0x2);
+		zfcp_scsi_dif_sense_error(scpnt,
+				LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED);
 		goto skip_fsfstatus;
 	case FSF_REF_TAG_CHECK_FAILURE:
-		zfcp_scsi_dif_sense_error(scpnt, 0x3);
+		zfcp_scsi_dif_sense_error(scpnt,
+				LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 		goto skip_fsfstatus;
 	}
 	BUILD_BUG_ON(sizeof(struct fcp_resp_with_ext) > FSF_FCP_RSP_SIZE);
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index 3590113c2b744a..ea4b968352f4d3 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -849,14 +849,14 @@ void zfcp_scsi_set_prot(struct zfcp_adapter *adapter)
 /**
  * zfcp_scsi_dif_sense_error - Report DIF/DIX error as driver sense error
  * @scmd: The SCSI command to report the error for
- * @ascq: The ASCQ to put in the sense buffer
+ * @sense_code: The combined ASC and ASCQ to put in the sense buffer
  *
  * See the error handling in sd_done for the sense codes used here.
  * Set DID_SOFT_ERROR to retry the request, if possible.
  */
-void zfcp_scsi_dif_sense_error(struct scsi_cmnd *scmd, int ascq)
+void zfcp_scsi_dif_sense_error(struct scsi_cmnd *scmd, u16 sense_code)
 {
-	scsi_build_sense(scmd, 1, ILLEGAL_REQUEST, 0x10, ascq);
+	scsi_set_sense(scmd, 1, ILLEGAL_REQUEST, sense_code);
 	set_host_byte(scmd, DID_SOFT_ERROR);
 }
 
diff --git a/drivers/s390/scsi/zfcp_sysfs.c b/drivers/s390/scsi/zfcp_sysfs.c
index 4f23d585d06206..b1ed0cb30a7242 100644
--- a/drivers/s390/scsi/zfcp_sysfs.c
+++ b/drivers/s390/scsi/zfcp_sysfs.c
@@ -270,7 +270,7 @@ static bool zfcp_sysfs_port_in_use(struct zfcp_port *const port)
 	if (atomic_read(&port->units) > 0)
 		goto unlock_port_units_mutex; /* zfcp_unit(s) under port */
 
-	spin_lock_irqsave(adapter->scsi_host->host_lock, flags);
+	spin_lock_irqsave(&adapter->scsi_host->host_lock, flags);
 	__shost_for_each_device(sdev, adapter->scsi_host) {
 		const struct zfcp_scsi_dev *zsdev = sdev_to_zfcp(sdev);
 
@@ -288,7 +288,7 @@ static bool zfcp_sysfs_port_in_use(struct zfcp_port *const port)
 	in_use = false;
 
 unlock_host_lock:
-	spin_unlock_irqrestore(adapter->scsi_host->host_lock, flags);
+	spin_unlock_irqrestore(&adapter->scsi_host->host_lock, flags);
 unlock_port_units_mutex:
 	mutex_unlock(&zfcp_sysfs_port_units_mutex);
 	return in_use;
diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c
index bab6cad3fd5ce0..552d779980129b 100644
--- a/drivers/s390/virtio/virtio_ccw.c
+++ b/drivers/s390/virtio/virtio_ccw.c
@@ -1062,7 +1062,7 @@ static void virtio_ccw_synchronize_cbs(struct virtio_device *vdev)
 	struct virtio_ccw_device *vcdev = to_vc_device(vdev);
 	struct airq_info *info = vcdev->airq_info;
 
-	if (info) {
+	if (vcdev->is_thinint && info) {
 		/*
 		 * This device uses adapter interrupts: synchronize with
 		 * vring_interrupt() called by virtio_airq_handler()
@@ -1204,13 +1204,11 @@ static void virtio_ccw_int_handler(struct ccw_device *cdev,
 			vcdev->err = -EIO;
 	}
 	virtio_ccw_check_activity(vcdev, activity);
-#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
 	/*
 	 * Paired with virtio_ccw_synchronize_cbs() and interrupts are
 	 * disabled here.
 	 */
 	read_lock(&vcdev->irq_lock);
-#endif
 	for_each_set_bit(i, indicators(vcdev),
 			 sizeof(*indicators(vcdev)) * BITS_PER_BYTE) {
 		/* The bit clear must happen before the vring kick. */
@@ -1219,9 +1217,7 @@ static void virtio_ccw_int_handler(struct ccw_device *cdev,
 		vq = virtio_ccw_vq_by_ind(vcdev, i);
 		vring_interrupt(0, vq);
 	}
-#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
 	read_unlock(&vcdev->irq_lock);
-#endif
 	if (test_bit(0, indicators2(vcdev))) {
 		virtio_config_changed(&vcdev->vdev);
 		clear_bit(0, indicators2(vcdev));
diff --git a/drivers/scsi/3w-9xxx.c b/drivers/scsi/3w-9xxx.c
index 9b93a2440af8f6..e6694f242ebed6 100644
--- a/drivers/scsi/3w-9xxx.c
+++ b/drivers/scsi/3w-9xxx.c
@@ -160,7 +160,7 @@ static ssize_t twa_show_stats(struct device *dev,
 	unsigned long flags = 0;
 	ssize_t len;
 
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 	len = sysfs_emit(buf, "3w-9xxx Driver version: %s\n"
 			 "Current commands posted:   %4d\n"
 			 "Max commands posted:       %4d\n"
@@ -183,7 +183,7 @@ static ssize_t twa_show_stats(struct device *dev,
 			 tw_dev->max_sector_count,
 			 tw_dev->num_resets,
 			 tw_dev->aen_count);
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 	return len;
 } /* End twa_show_stats() */
 
@@ -694,7 +694,7 @@ static long twa_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long
 	/* See which ioctl we are doing */
 	switch (cmd) {
 	case TW_IOCTL_FIRMWARE_PASS_THROUGH:
-		spin_lock_irqsave(tw_dev->host->host_lock, flags);
+		spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 		twa_get_request_id(tw_dev, &request_id);
 
 		/* Flag internal command */
@@ -712,7 +712,7 @@ static long twa_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long
 
 		/* Now post the command packet to the controller */
 		twa_post_command_packet(tw_dev, request_id, 1);
-		spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+		spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 		timeout = TW_IOCTL_CHRDEV_TIMEOUT*HZ;
 
@@ -734,11 +734,11 @@ static long twa_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long
 		memcpy(&(tw_ioctl->firmware_command), tw_dev->command_packet_virt[request_id], sizeof(TW_Command_Full));
 
 		/* Now complete the io */
-		spin_lock_irqsave(tw_dev->host->host_lock, flags);
+		spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 		tw_dev->posted_request_count--;
 		tw_dev->state[request_id] = TW_S_COMPLETED;
 		twa_free_request_id(tw_dev, request_id);
-		spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+		spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 		break;
 	case TW_IOCTL_GET_COMPATIBILITY_INFO:
 		tw_ioctl->driver_command.status = 0;
@@ -1223,7 +1223,7 @@ static irqreturn_t twa_interrupt(int irq, void *dev_instance)
 	int handled = 0;
 
 	/* Get the per adapter lock */
-	spin_lock(tw_dev->host->host_lock);
+	spin_lock(&tw_dev->host->host_lock);
 
 	/* Read the registers */
 	status_reg_value = readl(TW_STATUS_REG_ADDR(tw_dev));
@@ -1372,7 +1372,7 @@ static irqreturn_t twa_interrupt(int irq, void *dev_instance)
 	}
 
 twa_interrupt_bail:
-	spin_unlock(tw_dev->host->host_lock);
+	spin_unlock(&tw_dev->host->host_lock);
 	return IRQ_RETVAL(handled);
 } /* End twa_interrupt() */
 
@@ -1585,7 +1585,7 @@ static int twa_reset_device_extension(TW_Device_Extension *tw_dev)
 	set_bit(TW_IN_RESET, &tw_dev->flags);
 	TW_DISABLE_INTERRUPTS(tw_dev);
 	TW_MASK_COMMAND_INTERRUPT(tw_dev);
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 
 	/* Abort all requests that are in progress */
 	for (i = 0; i < TW_Q_LENGTH; i++) {
@@ -1616,7 +1616,7 @@ static int twa_reset_device_extension(TW_Device_Extension *tw_dev)
 	tw_dev->pending_tail = TW_Q_START;
 	tw_dev->reset_print = 0;
 
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 	if (twa_reset_sequence(tw_dev, 1))
 		goto out;
diff --git a/drivers/scsi/3w-sas.c b/drivers/scsi/3w-sas.c
index 52dc1aa639f7e6..060f97805319fb 100644
--- a/drivers/scsi/3w-sas.c
+++ b/drivers/scsi/3w-sas.c
@@ -108,9 +108,9 @@ static ssize_t twl_sysfs_aen_read(struct file *filp, struct kobject *kobj,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 	ret = memory_read_from_buffer(outbuf, count, &offset, tw_dev->event_queue[0], sizeof(TW_Event) * TW_Q_LENGTH);
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 	return ret;
 } /* End twl_sysfs_aen_read() */
@@ -139,9 +139,9 @@ static ssize_t twl_sysfs_compat_info(struct file *filp, struct kobject *kobj,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 	ret = memory_read_from_buffer(outbuf, count, &offset, &tw_dev->tw_compat_info, sizeof(TW_Compatibility_Info));
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 	return ret;
 } /* End twl_sysfs_compat_info() */
@@ -165,7 +165,7 @@ static ssize_t twl_show_stats(struct device *dev,
 	unsigned long flags = 0;
 	ssize_t len;
 
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 	len = sysfs_emit(buf, "3w-sas Driver version: %s\n"
 			 "Current commands posted:   %4d\n"
 			 "Max commands posted:       %4d\n"
@@ -184,7 +184,7 @@ static ssize_t twl_show_stats(struct device *dev,
 			 tw_dev->max_sector_count,
 			 tw_dev->num_resets,
 			 tw_dev->aen_count);
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 	return len;
 } /* End twl_show_stats() */
 
@@ -763,7 +763,7 @@ static long twl_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long
 	/* See which ioctl we are doing */
 	switch (cmd) {
 	case TW_IOCTL_FIRMWARE_PASS_THROUGH:
-		spin_lock_irqsave(tw_dev->host->host_lock, flags);
+		spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 		twl_get_request_id(tw_dev, &request_id);
 
 		/* Flag internal command */
@@ -781,7 +781,7 @@ static long twl_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long
 
 		/* Now post the command packet to the controller */
 		twl_post_command_packet(tw_dev, request_id);
-		spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+		spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 		timeout = TW_IOCTL_CHRDEV_TIMEOUT*HZ;
 
@@ -803,11 +803,11 @@ static long twl_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long
 		memcpy(&(tw_ioctl->firmware_command), tw_dev->command_packet_virt[request_id], sizeof(TW_Command_Full));
 
 		/* Now complete the io */
-		spin_lock_irqsave(tw_dev->host->host_lock, flags);
+		spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 		tw_dev->posted_request_count--;
 		tw_dev->state[request_id] = TW_S_COMPLETED;
 		twl_free_request_id(tw_dev, request_id);
-		spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+		spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 		break;
 	default:
 		retval = -ENOTTY;
@@ -1125,7 +1125,7 @@ static irqreturn_t twl_interrupt(int irq, void *dev_instance)
 	struct scsi_cmnd *cmd;
 	TW_Command_Full *full_command_packet;
 
-	spin_lock(tw_dev->host->host_lock);
+	spin_lock(&tw_dev->host->host_lock);
 
 	/* Read host interrupt status */
 	reg = readl(TWL_HISTAT_REG_ADDR(tw_dev));
@@ -1229,7 +1229,7 @@ static irqreturn_t twl_interrupt(int irq, void *dev_instance)
 	}
 
 twl_interrupt_bail:
-	spin_unlock(tw_dev->host->host_lock);
+	spin_unlock(&tw_dev->host->host_lock);
 	return IRQ_RETVAL(handled);
 } /* End twl_interrupt() */
 
@@ -1360,7 +1360,7 @@ static int twl_reset_device_extension(TW_Device_Extension *tw_dev, int ioctl_res
 	TWL_MASK_INTERRUPTS(tw_dev);
 	TWL_CLEAR_DB_INTERRUPT(tw_dev);
 
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 
 	/* Abort all requests that are in progress */
 	for (i = 0; i < TW_Q_LENGTH; i++) {
@@ -1386,7 +1386,7 @@ static int twl_reset_device_extension(TW_Device_Extension *tw_dev, int ioctl_res
 	tw_dev->free_tail = TW_Q_START;
 	tw_dev->posted_request_count = 0;
 
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 	if (twl_reset_sequence(tw_dev, 1))
 		goto out;
diff --git a/drivers/scsi/3w-xxxx.c b/drivers/scsi/3w-xxxx.c
index c68678fa72c1e5..11326465abb647 100644
--- a/drivers/scsi/3w-xxxx.c
+++ b/drivers/scsi/3w-xxxx.c
@@ -495,7 +495,7 @@ static ssize_t tw_show_stats(struct device *dev, struct device_attribute *attr,
 	unsigned long flags = 0;
 	ssize_t len;
 
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 	len = sysfs_emit(buf, "3w-xxxx Driver version: %s\n"
 			 "Current commands posted:   %4d\n"
 			 "Max commands posted:       %4d\n"
@@ -518,7 +518,7 @@ static ssize_t tw_show_stats(struct device *dev, struct device_attribute *attr,
 			 tw_dev->max_sector_count,
 			 tw_dev->num_resets,
 			 tw_dev->aen_count);
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 	return len;
 } /* End tw_show_stats() */
 
@@ -935,7 +935,7 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a
 			dprintk(KERN_WARNING "3w-xxxx: tw_chrdev_ioctl(): caught TW_AEN_LISTEN.\n");
 			memset(tw_ioctl->data_buffer, 0, data_buffer_length);
 
-			spin_lock_irqsave(tw_dev->host->host_lock, flags);
+			spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 			if (tw_dev->aen_head == tw_dev->aen_tail) {
 				tw_aen_code = TW_AEN_QUEUE_EMPTY;
 			} else {
@@ -946,12 +946,12 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a
 					tw_dev->aen_head = tw_dev->aen_head + 1;
 				}
 			}
-			spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+			spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 			memcpy(tw_ioctl->data_buffer, &tw_aen_code, sizeof(tw_aen_code));
 			break;
 		case TW_CMD_PACKET_WITH_DATA:
 			dprintk(KERN_WARNING "3w-xxxx: tw_chrdev_ioctl(): caught TW_CMD_PACKET_WITH_DATA.\n");
-			spin_lock_irqsave(tw_dev->host->host_lock, flags);
+			spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 
 			tw_state_request_start(tw_dev, &request_id);
 
@@ -983,7 +983,7 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a
 
 			/* Now post the command packet to the controller */
 			tw_post_command_packet(tw_dev, request_id);
-			spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+			spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 			timeout = TW_IOCTL_CHRDEV_TIMEOUT*HZ;
 
@@ -1005,11 +1005,11 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a
 			memcpy(&(tw_ioctl->firmware_command), tw_dev->command_packet_virtual_address[request_id], sizeof(TW_Command));
 
 			/* Now complete the io */
-			spin_lock_irqsave(tw_dev->host->host_lock, flags);
+			spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 			tw_dev->posted_request_count--;
 			tw_dev->state[request_id] = TW_S_COMPLETED;
 			tw_state_request_finish(tw_dev, request_id);
-			spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+			spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 			break;
 		default:
 			retval = -ENOTTY;
@@ -1296,7 +1296,7 @@ static int tw_reset_device_extension(TW_Device_Extension *tw_dev)
 	set_bit(TW_IN_RESET, &tw_dev->flags);
 	TW_DISABLE_INTERRUPTS(tw_dev);
 	TW_MASK_COMMAND_INTERRUPT(tw_dev);
-	spin_lock_irqsave(tw_dev->host->host_lock, flags);
+	spin_lock_irqsave(&tw_dev->host->host_lock, flags);
 
 	/* Abort all requests that are in progress */
 	for (i=0;i<TW_Q_LENGTH;i++) {
@@ -1325,7 +1325,7 @@ static int tw_reset_device_extension(TW_Device_Extension *tw_dev)
 	tw_dev->pending_tail = TW_Q_START;
 	tw_dev->reset_print = 0;
 
-	spin_unlock_irqrestore(tw_dev->host->host_lock, flags);
+	spin_unlock_irqrestore(&tw_dev->host->host_lock, flags);
 
 	if (tw_reset_sequence(tw_dev)) {
 		printk(KERN_WARNING "3w-xxxx: scsi%d: Reset sequence failed.\n", tw_dev->host->host_no);
@@ -1977,7 +1977,8 @@ static enum scsi_qc_status tw_scsi_queue_lck(struct scsi_cmnd *SCpnt)
 		printk(KERN_NOTICE "3w-xxxx: scsi%d: Unknown scsi opcode: 0x%x\n", tw_dev->host->host_no, *command);
 		tw_dev->state[request_id] = TW_S_COMPLETED;
 		tw_state_request_finish(tw_dev, request_id);
-		scsi_build_sense(SCpnt, 1, ILLEGAL_REQUEST, 0x20, 0);
+		scsi_set_sense(SCpnt, 1, ILLEGAL_REQUEST,
+			       INVALID_COMMAND_OP_CODE);
 		done(SCpnt);
 		retval = 0;
 	}
@@ -2005,7 +2006,7 @@ static irqreturn_t tw_interrupt(int irq, void *dev_instance)
 	int handled = 0;
 
 	/* Get the host lock for io completions */
-	spin_lock(tw_dev->host->host_lock);
+	spin_lock(&tw_dev->host->host_lock);
 
 	/* Read the registers */
 	status_reg_value = inl(TW_STATUS_REG_ADDR(tw_dev));
@@ -2185,7 +2186,7 @@ static irqreturn_t tw_interrupt(int irq, void *dev_instance)
 	}
 
 tw_interrupt_bail:
-	spin_unlock(tw_dev->host->host_lock);
+	spin_unlock(&tw_dev->host->host_lock);
 	return IRQ_RETVAL(handled);
 } /* End tw_interrupt() */
 
diff --git a/drivers/scsi/53c700.c b/drivers/scsi/53c700.c
index c78f74b8f45c88..eb3a69dff1e726 100644
--- a/drivers/scsi/53c700.c
+++ b/drivers/scsi/53c700.c
@@ -1499,7 +1499,7 @@ NCR_700_intr(int irq, void *dev_id)
 	 * lock to enter the done routines.  When that happens, we
 	 * need to ensure that for this driver, the host lock and the
 	 * queue lock point to the same thing. */
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	if((istat = NCR_700_readb(host, ISTAT_REG))
 	      & (SCSI_INT_PENDING | DMA_INT_PENDING)) {
 		__u32 dsps;
@@ -1747,7 +1747,7 @@ NCR_700_intr(int irq, void *dev_id)
 		}
 	}
  out_unlock:
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 	return IRQ_RETVAL(handled);
 }
 
@@ -1948,27 +1948,27 @@ NCR_700_host_reset(struct scsi_cmnd * SCp)
 	/* In theory, eh_complete should always be null because the
 	 * eh is single threaded, but just in case we're handling a
 	 * reset via sg or something */
-	spin_lock_irq(SCp->device->host->host_lock);
+	spin_lock_irq(&SCp->device->host->host_lock);
 	while (hostdata->eh_complete != NULL) {
-		spin_unlock_irq(SCp->device->host->host_lock);
+		spin_unlock_irq(&SCp->device->host->host_lock);
 		msleep_interruptible(100);
-		spin_lock_irq(SCp->device->host->host_lock);
+		spin_lock_irq(&SCp->device->host->host_lock);
 	}
 
 	hostdata->eh_complete = &complete;
 	NCR_700_internal_bus_reset(SCp->device->host);
 	NCR_700_chip_reset(SCp->device->host);
 
-	spin_unlock_irq(SCp->device->host->host_lock);
+	spin_unlock_irq(&SCp->device->host->host_lock);
 	wait_for_completion(&complete);
-	spin_lock_irq(SCp->device->host->host_lock);
+	spin_lock_irq(&SCp->device->host->host_lock);
 
 	hostdata->eh_complete = NULL;
 	/* Revalidate the transport parameters of the failing device */
 	if(hostdata->fast)
 		spi_schedule_dv_device(SCp->device);
 
-	spin_unlock_irq(SCp->device->host->host_lock);
+	spin_unlock_irq(&SCp->device->host->host_lock);
 	return SUCCESS;
 }
 
diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c
index 5304d2febd63e5..27b446c0c5d0f0 100644
--- a/drivers/scsi/BusLogic.c
+++ b/drivers/scsi/BusLogic.c
@@ -2744,7 +2744,7 @@ static irqreturn_t blogic_inthandler(int irq_ch, void *devid)
 	/*
 	   Acquire exclusive access to Host Adapter.
 	 */
-	spin_lock_irqsave(adapter->scsi_host->host_lock, processor_flag);
+	spin_lock_irqsave(&adapter->scsi_host->host_lock, processor_flag);
 	/*
 	   Handle Interrupts appropriately for each Host Adapter type.
 	 */
@@ -2812,7 +2812,7 @@ static irqreturn_t blogic_inthandler(int irq_ch, void *devid)
 	/*
 	   Release exclusive access to Host Adapter.
 	 */
-	spin_unlock_irqrestore(adapter->scsi_host->host_lock, processor_flag);
+	spin_unlock_irqrestore(&adapter->scsi_host->host_lock, processor_flag);
 	return IRQ_HANDLED;
 }
 
@@ -2864,12 +2864,12 @@ static int blogic_hostreset(struct scsi_cmnd *SCpnt)
 	struct blogic_tgt_stats *stats = &adapter->tgt_stats[id];
 	int rc;
 
-	spin_lock_irq(SCpnt->device->host->host_lock);
+	spin_lock_irq(&SCpnt->device->host->host_lock);
 
 	blogic_inc_count(&stats->adapter_reset_req);
 
 	rc = blogic_resetadapter(adapter, false);
-	spin_unlock_irq(SCpnt->device->host->host_lock);
+	spin_unlock_irq(&SCpnt->device->host->host_lock);
 	return rc;
 }
 
@@ -2915,9 +2915,9 @@ static enum scsi_qc_status blogic_qcmd_lck(struct scsi_cmnd *command)
 	 */
 	ccb = blogic_alloc_ccb(adapter);
 	if (ccb == NULL) {
-		spin_unlock_irq(adapter->scsi_host->host_lock);
+		spin_unlock_irq(&adapter->scsi_host->host_lock);
 		blogic_delay(1);
-		spin_lock_irq(adapter->scsi_host->host_lock);
+		spin_lock_irq(&adapter->scsi_host->host_lock);
 		ccb = blogic_alloc_ccb(adapter);
 		if (ccb == NULL) {
 			command->result = DID_ERROR << 16;
@@ -3062,10 +3062,10 @@ static enum scsi_qc_status blogic_qcmd_lck(struct scsi_cmnd *command)
 		   be initiated soon.
 		 */
 		if (!blogic_write_outbox(adapter, BLOGIC_MBOX_START, ccb)) {
-			spin_unlock_irq(adapter->scsi_host->host_lock);
+			spin_unlock_irq(&adapter->scsi_host->host_lock);
 			blogic_warn("Unable to write Outgoing Mailbox - Pausing for 1 second\n", adapter);
 			blogic_delay(1);
-			spin_lock_irq(adapter->scsi_host->host_lock);
+			spin_lock_irq(&adapter->scsi_host->host_lock);
 			if (!blogic_write_outbox(adapter, BLOGIC_MBOX_START,
 						ccb)) {
 				blogic_warn("Still unable to write Outgoing Mailbox - Host Adapter Dead?\n", adapter);
@@ -3212,9 +3212,9 @@ static int blogic_resetadapter(struct blogic_adapter *adapter, bool hard_reset)
 	 */
 
 	if (hard_reset) {
-		spin_unlock_irq(adapter->scsi_host->host_lock);
+		spin_unlock_irq(&adapter->scsi_host->host_lock);
 		blogic_delay(adapter->bus_settle_time);
-		spin_lock_irq(adapter->scsi_host->host_lock);
+		spin_lock_irq(&adapter->scsi_host->host_lock);
 	}
 
 	for (tgt_id = 0; tgt_id < adapter->maxdev; tgt_id++) {
diff --git a/drivers/scsi/a100u2w.c b/drivers/scsi/a100u2w.c
index 4365b896f5c47b..7e6776325d9b85 100644
--- a/drivers/scsi/a100u2w.c
+++ b/drivers/scsi/a100u2w.c
@@ -1058,9 +1058,9 @@ static irqreturn_t inia100_intr(int irqno, void *devid)
 	unsigned long flags;
 	irqreturn_t res;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	res = orc_interrupt(host);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	return res;
 }
diff --git a/drivers/scsi/a2091.c b/drivers/scsi/a2091.c
index f81e53b53e203e..223065529654fd 100644
--- a/drivers/scsi/a2091.c
+++ b/drivers/scsi/a2091.c
@@ -39,9 +39,9 @@ static irqreturn_t a2091_intr(int irq, void *data)
 	if (!(status & (ISTR_INT_F | ISTR_INT_P)) || !(status & ISTR_INTS))
 		return IRQ_NONE;
 
-	spin_lock_irqsave(instance->host_lock, flags);
+	spin_lock_irqsave(&instance->host_lock, flags);
 	wd33c93_intr(instance);
-	spin_unlock_irqrestore(instance->host_lock, flags);
+	spin_unlock_irqrestore(&instance->host_lock, flags);
 	return IRQ_HANDLED;
 }
 
diff --git a/drivers/scsi/a3000.c b/drivers/scsi/a3000.c
index bf054dd7682bf1..dd0924572c0e1a 100644
--- a/drivers/scsi/a3000.c
+++ b/drivers/scsi/a3000.c
@@ -41,9 +41,9 @@ static irqreturn_t a3000_intr(int irq, void *data)
 	if (!(status & ISTR_INT_P))
 		return IRQ_NONE;
 	if (status & ISTR_INTS) {
-		spin_lock_irqsave(instance->host_lock, flags);
+		spin_lock_irqsave(&instance->host_lock, flags);
 		wd33c93_intr(instance);
-		spin_unlock_irqrestore(instance->host_lock, flags);
+		spin_unlock_irqrestore(&instance->host_lock, flags);
 		return IRQ_HANDLED;
 	}
 	pr_warn("Non-serviced A3000 SCSI-interrupt? ISTR = %02x\n", status);
diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c
index c4485629f79237..347d480a39b08c 100644
--- a/drivers/scsi/aacraid/commsup.c
+++ b/drivers/scsi/aacraid/commsup.c
@@ -1487,7 +1487,7 @@ static int _aac_reset_adapter(struct aac_dev *aac, int forced, u8 reset_type)
 	 */
 	aac_adapter_disable_int(aac);
 	if (aac->thread && aac->thread->pid != current->pid) {
-		spin_unlock_irq(host->host_lock);
+		spin_unlock_irq(&host->host_lock);
 		kthread_stop(aac->thread);
 		aac->thread = NULL;
 		jafo = 1;
@@ -1612,7 +1612,7 @@ static int _aac_reset_adapter(struct aac_dev *aac, int forced, u8 reset_type)
 	}
 
 	if (jafo) {
-		spin_lock_irq(host->host_lock);
+		spin_lock_irq(&host->host_lock);
 	}
 	return retval;
 }
@@ -1644,11 +1644,11 @@ int aac_reset_adapter(struct aac_dev *aac, int forced, u8 reset_type)
 	/* Quiesce build, flush cache, write through mode */
 	if (forced < 2)
 		aac_send_shutdown(aac);
-	spin_lock_irqsave(host->host_lock, flagv);
+	spin_lock_irqsave(&host->host_lock, flagv);
 	bled = forced ? forced :
 			(aac_check_reset != 0 && aac_check_reset != 1);
 	retval = _aac_reset_adapter(aac, bled, reset_type);
-	spin_unlock_irqrestore(host->host_lock, flagv);
+	spin_unlock_irqrestore(&host->host_lock, flagv);
 
 	unblock_retval = scsi_host_unblock(host, SDEV_RUNNING);
 	if (!retval)
diff --git a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
index 5cdbf2bdb13df4..e6fa05ffc02402 100644
--- a/drivers/scsi/advansys.c
+++ b/drivers/scsi/advansys.c
@@ -7072,9 +7072,9 @@ static int advansys_reset(struct scsi_cmnd *scp)
 			ret = FAILED;
 			break;
 		}
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		AdvISR(adv_dvc);
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 	}
 
 	ASC_DBG(1, "ret %d\n", ret);
@@ -7138,7 +7138,7 @@ static irqreturn_t advansys_interrupt(int irq, void *dev_id)
 	unsigned long flags;
 
 	ASC_DBG(2, "boardp 0x%p\n", boardp);
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (ASC_NARROW_BOARD(boardp)) {
 		if (AscIsIntPending(shost->io_port)) {
 			result = IRQ_HANDLED;
@@ -7153,7 +7153,7 @@ static irqreturn_t advansys_interrupt(int irq, void *dev_id)
 			ASC_STATS(shost, interrupt);
 		}
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	ASC_DBG(1, "end\n");
 	return result;
diff --git a/drivers/scsi/aha1542.c b/drivers/scsi/aha1542.c
index 93dab19c1cb954..7ea38d5090720e 100644
--- a/drivers/scsi/aha1542.c
+++ b/drivers/scsi/aha1542.c
@@ -306,7 +306,7 @@ static irqreturn_t aha1542_interrupt(int irq, void *dev_id)
 #endif
 	number_serviced = 0;
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 	while (1) {
 		flag = inb(INTRFLAGS(sh->io_port));
 
@@ -339,7 +339,7 @@ static irqreturn_t aha1542_interrupt(int irq, void *dev_id)
 		} while (mbi != aha1542->aha1542_last_mbi_used);
 
 		if (mb[mbi].status == 0) {
-			spin_unlock_irqrestore(sh->host_lock, flags);
+			spin_unlock_irqrestore(&sh->host_lock, flags);
 			/* Hmm, no mail.  Must have read it the last time around */
 			if (!number_serviced)
 				shost_printk(KERN_WARNING, sh, "interrupt received, but no mail.\n");
@@ -367,7 +367,7 @@ static irqreturn_t aha1542_interrupt(int irq, void *dev_id)
 		tmp_cmd = aha1542->int_cmds[mbo];
 
 		if (!tmp_cmd) {
-			spin_unlock_irqrestore(sh->host_lock, flags);
+			spin_unlock_irqrestore(&sh->host_lock, flags);
 			shost_printk(KERN_WARNING, sh, "Unexpected interrupt\n");
 			shost_printk(KERN_WARNING, sh, "tarstat=%x, hastat=%x idlun=%x ccb#=%d\n", ccb[mbo].tarstat,
 			       ccb[mbo].hastat, ccb[mbo].idlun, mbo);
@@ -461,7 +461,7 @@ static enum scsi_qc_status aha1542_queuecommand(struct Scsi_Host *sh,
 	 * is how the host adapter will scan for them
 	 */
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 	mbo = aha1542->aha1542_last_mbo_used + 1;
 	if (mbo >= AHA1542_MAILBOXES)
 		mbo = 0;
@@ -518,7 +518,7 @@ static enum scsi_qc_status aha1542_queuecommand(struct Scsi_Host *sh,
 #endif
 	mb[mbo].status = 1;
 	aha1542_outb(cmd->device->host->io_port, CMD_START_SCSI);
-	spin_unlock_irqrestore(sh->host_lock, flags);
+	spin_unlock_irqrestore(&sh->host_lock, flags);
 
 	return 0;
 }
@@ -873,7 +873,7 @@ static int aha1542_dev_reset(struct scsi_cmnd *cmd)
 	int mbo;
 	struct ccb *ccb = aha1542->ccb;
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 	mbo = aha1542->aha1542_last_mbo_used + 1;
 	if (mbo >= AHA1542_MAILBOXES)
 		mbo = 0;
@@ -913,7 +913,7 @@ static int aha1542_dev_reset(struct scsi_cmnd *cmd)
 	 * target
 	 */
 	aha1542_outb(sh->io_port, CMD_START_SCSI);
-	spin_unlock_irqrestore(sh->host_lock, flags);
+	spin_unlock_irqrestore(&sh->host_lock, flags);
 
 	scmd_printk(KERN_WARNING, cmd,
 		"Trying device reset for target\n");
@@ -928,7 +928,7 @@ static int aha1542_reset(struct scsi_cmnd *cmd, u8 reset_cmd)
 	unsigned long flags;
 	int i;
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 	/*
 	 * This does a scsi reset for all devices on the bus.
 	 * In principle, we could also reset the 1542 - should
@@ -939,7 +939,7 @@ static int aha1542_reset(struct scsi_cmnd *cmd, u8 reset_cmd)
 
 	if (!wait_mask(STATUS(cmd->device->host->io_port),
 	     STATMASK, IDLE, STST | DIAGF | INVDCMD | DF | CDF, 0)) {
-		spin_unlock_irqrestore(sh->host_lock, flags);
+		spin_unlock_irqrestore(&sh->host_lock, flags);
 		return FAILED;
 	}
 
@@ -978,7 +978,7 @@ static int aha1542_reset(struct scsi_cmnd *cmd, u8 reset_cmd)
 		}
 	}
 
-	spin_unlock_irqrestore(sh->host_lock, flags);
+	spin_unlock_irqrestore(&sh->host_lock, flags);
 	return SUCCESS;
 }
 
diff --git a/drivers/scsi/aha1740.c b/drivers/scsi/aha1740.c
index c435769359f224..b01701d139b90b 100644
--- a/drivers/scsi/aha1740.c
+++ b/drivers/scsi/aha1740.c
@@ -225,7 +225,7 @@ static irqreturn_t aha1740_intr_handle(int irq, void *dev_id)
 	
 	if (!host)
 		panic("aha1740.c: Irq from unknown host!\n");
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	base = host->io_port;
 	number_serviced = 0;
 	edev = HOSTDATA(host)->edev;
@@ -315,7 +315,7 @@ static irqreturn_t aha1740_intr_handle(int irq, void *dev_id)
 		number_serviced++;
 	}
 
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 	return IRQ_RETVAL(handled);
 }
 
@@ -353,7 +353,7 @@ static enum scsi_qc_status aha1740_queuecommand_lck(struct scsi_cmnd *SCpnt)
 #endif
 
 	/* locate an available ecb */
-	spin_lock_irqsave(SCpnt->device->host->host_lock, flags);
+	spin_lock_irqsave(&SCpnt->device->host->host_lock, flags);
 	ecbno = host->last_ecb_used + 1; /* An optimization */
 	if (ecbno >= AHA1740_ECBS)
 		ecbno = 0;
@@ -372,7 +372,7 @@ static enum scsi_qc_status aha1740_queuecommand_lck(struct scsi_cmnd *SCpnt)
 						    doubles as reserved flag */
 
 	host->last_ecb_used = ecbno;    
-	spin_unlock_irqrestore(SCpnt->device->host->host_lock, flags);
+	spin_unlock_irqrestore(&SCpnt->device->host->host_lock, flags);
 
 #ifdef DEBUG
 	printk("Sending command (%d %x)...", ecbno, done);
@@ -465,7 +465,7 @@ static enum scsi_qc_status aha1740_queuecommand_lck(struct scsi_cmnd *SCpnt)
 		unsigned int base = SCpnt->device->host->io_port;
 		DEB(printk("aha1740[%d] critical section\n",ecbno));
 
-		spin_lock_irqsave(SCpnt->device->host->host_lock, flags);
+		spin_lock_irqsave(&SCpnt->device->host->host_lock, flags);
 		for (loopcnt = 0; ; loopcnt++) {
 			if (inb(G2STAT(base)) & G2STAT_MBXOUT) break;
 			if (loopcnt == LOOPCNT_WARN) {
@@ -485,7 +485,7 @@ static enum scsi_qc_status aha1740_queuecommand_lck(struct scsi_cmnd *SCpnt)
 				panic("aha1740.c: attn wait failed!\n");
 		}
 		outb(ATTN_START | (target & 7), ATTN(base)); /* Start it up */
-		spin_unlock_irqrestore(SCpnt->device->host->host_lock, flags);
+		spin_unlock_irqrestore(&SCpnt->device->host->host_lock, flags);
 		DEB(printk("aha1740[%d] request queued.\n",ecbno));
 	} else
 		printk(KERN_ALERT "aha1740_queuecommand: done can't be NULL\n");
diff --git a/drivers/scsi/arm/eesox.c b/drivers/scsi/arm/eesox.c
index 99be9da8757f46..0cb1308eda4e66 100644
--- a/drivers/scsi/arm/eesox.c
+++ b/drivers/scsi/arm/eesox.c
@@ -124,14 +124,14 @@ eesoxscsi_terminator_ctl(struct Scsi_Host *host, int on_off)
 	struct eesoxscsi_info *info = (struct eesoxscsi_info *)host->hostdata;
 	unsigned long flags;
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	if (on_off)
 		info->control |= EESOX_TERM_ENABLE;
 	else
 		info->control &= ~EESOX_TERM_ENABLE;
 
 	writeb(info->control, info->ctl_port);
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 }
 
 /* Prototype: void eesoxscsi_intr(irq, *dev_id, *regs)
@@ -457,14 +457,14 @@ static ssize_t eesoxscsi_store_term(struct device *dev, struct device_attribute
 	unsigned long flags;
 
 	if (len > 1) {
-		spin_lock_irqsave(host->host_lock, flags);
+		spin_lock_irqsave(&host->host_lock, flags);
 		if (buf[0] != '0') {
 			info->control |= EESOX_TERM_ENABLE;
 		} else {
 			info->control &= ~EESOX_TERM_ENABLE;
 		}
 		writeb(info->control, info->ctl_port);
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 	}
 
 	return len;
diff --git a/drivers/scsi/arm/fas216.c b/drivers/scsi/arm/fas216.c
index fccfacaaf1d52e..d0752c0bf42542 100644
--- a/drivers/scsi/arm/fas216.c
+++ b/drivers/scsi/arm/fas216.c
@@ -2295,7 +2295,7 @@ static enum scsi_qc_status fas216_noqueue_command_lck(struct scsi_cmnd *SCpnt)
 	 * However, we must re-enable interrupts, or else we'll be
 	 * waiting forever.
 	 */
-	spin_unlock_irq(info->host->host_lock);
+	spin_unlock_irq(&info->host->host_lock);
 
 	while (!info->internal_done) {
 		/*
@@ -2307,13 +2307,13 @@ static enum scsi_qc_status fas216_noqueue_command_lck(struct scsi_cmnd *SCpnt)
 		 * to be some time (eg, disconnected).
 		 */
 		if (fas216_readb(info, REG_STAT) & STAT_INT) {
-			spin_lock_irq(info->host->host_lock);
+			spin_lock_irq(&info->host->host_lock);
 			fas216_intr(info);
-			spin_unlock_irq(info->host->host_lock);
+			spin_unlock_irq(&info->host->host_lock);
 		}
 	}
 
-	spin_lock_irq(info->host->host_lock);
+	spin_lock_irq(&info->host->host_lock);
 
 	scsi_done(SCpnt);
 
@@ -2662,7 +2662,7 @@ int fas216_eh_host_reset(struct scsi_cmnd *SCpnt)
 {
 	FAS216_Info *info = (FAS216_Info *)SCpnt->device->host->hostdata;
 
-	spin_lock_irq(info->host->host_lock);
+	spin_lock_irq(&info->host->host_lock);
 
 	fas216_checkmagic(info);
 
@@ -2679,9 +2679,9 @@ int fas216_eh_host_reset(struct scsi_cmnd *SCpnt)
 	 * IRQs if we sleep, but we must relock and disable
 	 * IRQs after the sleep.
 	 */
-	spin_unlock_irq(info->host->host_lock);
+	spin_unlock_irq(&info->host->host_lock);
 	msleep(50 * 1000/100);
-	spin_lock_irq(info->host->host_lock);
+	spin_lock_irq(&info->host->host_lock);
 
 	/*
 	 * Release the SCSI reset.
@@ -2690,7 +2690,7 @@ int fas216_eh_host_reset(struct scsi_cmnd *SCpnt)
 
 	fas216_init_chip(info);
 
-	spin_unlock_irq(info->host->host_lock);
+	spin_unlock_irq(&info->host->host_lock);
 	return SUCCESS;
 }
 
@@ -2917,9 +2917,9 @@ int fas216_add(struct Scsi_Host *host, struct device *dev)
 	/*
 	 * scsi standard says wait 250ms
 	 */
-	spin_unlock_irq(info->host->host_lock);
+	spin_unlock_irq(&info->host->host_lock);
 	msleep(100*1000/100);
-	spin_lock_irq(info->host->host_lock);
+	spin_lock_irq(&info->host->host_lock);
 
 	fas216_writeb(info, REG_CNTL1, info->scsi.cfg[0]);
 	fas216_readb(info, REG_INST);
diff --git a/drivers/scsi/atp870u.c b/drivers/scsi/atp870u.c
index 67459d81f479e4..c608d22ad2c606 100644
--- a/drivers/scsi/atp870u.c
+++ b/drivers/scsi/atp870u.c
@@ -225,7 +225,7 @@ static irqreturn_t atp870u_intr_handle(int irq, void *dev_id)
 			/*
 			 *	Issue more commands
 			 */
-			spin_lock_irqsave(dev->host->host_lock, flags);
+			spin_lock_irqsave(&dev->host->host_lock, flags);
 			if (((dev->quhd[c] != dev->quend[c]) ||
 			     (dev->last_cmd[c] != 0xff)) &&
 			    (dev->in_snd[c] == 0)) {
@@ -234,7 +234,7 @@ static irqreturn_t atp870u_intr_handle(int irq, void *dev_id)
 #endif
 				send_s870(dev,c);
 			}
-			spin_unlock_irqrestore(dev->host->host_lock, flags);
+			spin_unlock_irqrestore(&dev->host->host_lock, flags);
 			/*
 			 *	Done
 			 */
@@ -511,7 +511,7 @@ static irqreturn_t atp870u_intr_handle(int irq, void *dev_id)
 			 */
 			scsi_dma_unmap(workreq);
 
-			spin_lock_irqsave(dev->host->host_lock, flags);
+			spin_lock_irqsave(&dev->host->host_lock, flags);
 			scsi_done(workreq);
 #ifdef ED_DBGP
 			   printk("workreq->scsi_done\n");
@@ -521,7 +521,7 @@ static irqreturn_t atp870u_intr_handle(int irq, void *dev_id)
 			 */
 			dev->id[c][target_id].curr_req = NULL;
 			dev->working[c]--;
-			spin_unlock_irqrestore(dev->host->host_lock, flags);
+			spin_unlock_irqrestore(&dev->host->host_lock, flags);
 			/*
 			 *      Take it back wide
 			 */
@@ -533,7 +533,7 @@ static irqreturn_t atp870u_intr_handle(int irq, void *dev_id)
 			/*
 			 *	If there is stuff to send and nothing going then send it
 			 */
-			spin_lock_irqsave(dev->host->host_lock, flags);
+			spin_lock_irqsave(&dev->host->host_lock, flags);
 			if (((dev->last_cmd[c] != 0xff) ||
 			     (dev->quhd[c] != dev->quend[c])) &&
 			    (dev->in_snd[c] == 0)) {
@@ -542,7 +542,7 @@ static irqreturn_t atp870u_intr_handle(int irq, void *dev_id)
 #endif
 			   send_s870(dev,c);
 			}
-			spin_unlock_irqrestore(dev->host->host_lock, flags);
+			spin_unlock_irqrestore(&dev->host->host_lock, flags);
 			dev->in_int[c] = 0;
 			return IRQ_HANDLED;
 		}
diff --git a/drivers/scsi/bfa/bfad_bsg.c b/drivers/scsi/bfa/bfad_bsg.c
index 292bc9aa43f1f8..143a3913c65048 100644
--- a/drivers/scsi/bfa/bfad_bsg.c
+++ b/drivers/scsi/bfa/bfad_bsg.c
@@ -2552,7 +2552,7 @@ static void bfad_reset_sdev_bflags(struct bfad_im_port_s *im_port,
 	struct scsi_device *sdev;
 	unsigned long flags;
 
-	spin_lock_irqsave(im_port->shost->host_lock, flags);
+	spin_lock_irqsave(&im_port->shost->host_lock, flags);
 	list_for_each_entry(itnim, &im_port->itnim_mapped_list, list_entry) {
 		sdev = __scsi_device_lookup(im_port->shost, itnim->channel,
 					    itnim->scsi_tgt_id, 0);
@@ -2563,7 +2563,7 @@ static void bfad_reset_sdev_bflags(struct bfad_im_port_s *im_port,
 				sdev->sdev_bflags &= ~scan_flags;
 		}
 	}
-	spin_unlock_irqrestore(im_port->shost->host_lock, flags);
+	spin_unlock_irqrestore(&im_port->shost->host_lock, flags);
 }
 
 /* Function to reset the LUN SCAN mode */
diff --git a/drivers/scsi/ch.c b/drivers/scsi/ch.c
index 4010fdbf813cc7..87e51e50a7415a 100644
--- a/drivers/scsi/ch.c
+++ b/drivers/scsi/ch.c
@@ -123,38 +123,32 @@ static DEFINE_IDR(ch_index_idr);
 static DEFINE_SPINLOCK(ch_index_lock);
 
 static const struct {
-	unsigned char  sense;
-	unsigned char  asc;
-	unsigned char  ascq;
-	int	       errno;
+	u8	sense_key;
+	u16	sense_code;
+	int	errno;
 } ch_err[] = {
 /* Just filled in what looks right. Hav'nt checked any standard paper for
    these errno assignments, so they may be wrong... */
 	{
-		.sense  = ILLEGAL_REQUEST,
-		.asc    = 0x21,
-		.ascq   = 0x01,
-		.errno  = EBADSLT, /* Invalid element address */
+		.sense_key	= ILLEGAL_REQUEST,
+		.sense_code	= INVALID_ELEMENT_ADDRESS,
+		.errno		= EBADSLT,
 	},{
-		.sense  = ILLEGAL_REQUEST,
-		.asc    = 0x28,
-		.ascq   = 0x01,
-		.errno  = EBADE,   /* Import or export element accessed */
+		.sense_key	= ILLEGAL_REQUEST,
+		.sense_code	= IMPORT_OR_EXPORT_ELEMENT_ACCESSED,
+		.errno		= EBADE,
 	},{
-		.sense  = ILLEGAL_REQUEST,
-		.asc    = 0x3B,
-		.ascq   = 0x0D,
-		.errno  = EXFULL,  /* Medium destination element full */
+		.sense_key	= ILLEGAL_REQUEST,
+		.sense_code	= MEDIUM_DESTINATION_ELEMENT_FULL,
+		.errno		= EXFULL,
 	},{
-		.sense  = ILLEGAL_REQUEST,
-		.asc    = 0x3B,
-		.ascq   = 0x0E,
-		.errno  = EBADE,   /* Medium source element empty */
+		.sense_key	= ILLEGAL_REQUEST,
+		.sense_code	= MEDIUM_SOURCE_ELEMENT_EMPTY,
+		.errno		= EBADE,
 	},{
-		.sense  = ILLEGAL_REQUEST,
-		.asc    = 0x20,
-		.ascq   = 0x00,
-		.errno  = EBADRQC, /* Invalid command operation code */
+		.sense_key	= ILLEGAL_REQUEST,
+		.sense_code	= INVALID_COMMAND_OP_CODE,
+		.errno		= EBADRQC,
 	},{
 	        /* end of list */
 	}
@@ -167,12 +161,10 @@ static int ch_find_errno(struct scsi_sense_hdr *sshdr)
 	int i,errno = 0;
 
 	/* Check to see if additional sense information is available */
-	if (scsi_sense_valid(sshdr) &&
-	    sshdr->asc != 0) {
+	if (scsi_sense_valid(sshdr) && scsi_sense_asc(sshdr) != 0) {
 		for (i = 0; ch_err[i].errno != 0; i++) {
-			if (ch_err[i].sense == sshdr->sense_key &&
-			    ch_err[i].asc   == sshdr->asc &&
-			    ch_err[i].ascq  == sshdr->ascq) {
+			if (ch_err[i].sense_key == sshdr->sense_key &&
+			    ch_err[i].sense_code == sshdr->sense_code) {
 				errno = -ch_err[i].errno;
 				break;
 			}
@@ -191,9 +183,8 @@ ch_do_scsi(scsi_changer *ch, unsigned char *cmd, int cmd_len,
 	struct scsi_sense_hdr sshdr;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = 3,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
diff --git a/drivers/scsi/constants.c b/drivers/scsi/constants.c
index 340785536998d2..767631835c815a 100644
--- a/drivers/scsi/constants.c
+++ b/drivers/scsi/constants.c
@@ -292,7 +292,7 @@ bool scsi_opcode_sa_name(int opcode, int service_action,
 }
 
 struct error_info {
-	unsigned short code12;	/* 0x0302 looks better than 0x03,0x02 */
+	unsigned short code;	/* 0x0302 looks better than 0x03,0x02 */
 	unsigned short size;
 };
 
@@ -318,20 +318,46 @@ static const char *additional_text =
 	;
 
 struct error_info2 {
-	unsigned char code1, code2_min, code2_max;
-	const char * str;
-	const char * fmt;
+	u8		asc;
+	u8		ascq_min;
+	u8		ascq_max;
+	const char	*str;
+	const char	*fmt;
 };
 
 static const struct error_info2 additional2[] =
 {
-	{0x40, 0x00, 0x7f, "Ram failure", ""},
-	{0x40, 0x80, 0xff, "Diagnostic failure on component", ""},
-	{0x41, 0x00, 0xff, "Data path failure", ""},
-	{0x42, 0x00, 0xff, "Power-on or self-test failure", ""},
-	{0x4D, 0x00, 0xff, "Tagged overlapped commands", "task tag "},
-	{0x70, 0x00, 0xff, "Decompression exception", "short algorithm id of "},
-	{0, 0, 0, NULL, NULL}
+	{
+		ASC_RAM_FAILURE,
+		0x00, 0x7f,
+		"Ram failure", ""
+	},
+	{
+		ASC_RAM_FAILURE,
+		0x80, 0xff,
+		"Diagnostic failure on component", ""
+	},
+	{
+		ASC_DATA_PATH_FAILURE,
+		0x00, 0xff,
+		"Data path failure", ""
+	},
+	{
+		ASC_POWER_ON_OR_SELFTEST_FAILURE,
+		0x00, 0xff,
+		"Power-on or self-test failure", ""
+	},
+	{
+		ASC_TAGGED_OVERLAPPED_COMMANDS,
+		0x00, 0xff,
+		"Tagged overlapped commands", "task tag "
+	},
+	{
+		ASC_DECOMPRESSION_EXCEPTION_SHORT_ALGORITHM_ID,
+		0x00, 0xff,
+		"Decompression exception", "short algorithm id of "
+	},
+	{ 0, 0, 0, NULL, NULL }
 };
 
 /* description of the sense key values */
@@ -375,22 +401,21 @@ EXPORT_SYMBOL(scsi_sense_key_string);
  * This string may contain a "%x" and should be printed with ascq as arg.
  */
 const char *
-scsi_extd_sense_format(unsigned char asc, unsigned char ascq, const char **fmt)
+scsi_extd_sense_format(const struct scsi_sense_hdr *sshdr, const char **fmt)
 {
-	int i;
-	unsigned short code = ((asc << 8) | ascq);
 	unsigned offset = 0;
+	int i;
 
 	*fmt = NULL;
 	for (i = 0; i < ARRAY_SIZE(additional); i++) {
-		if (additional[i].code12 == code)
+		if (additional[i].code == sshdr->sense_code)
 			return additional_text + offset;
 		offset += additional[i].size;
 	}
 	for (i = 0; additional2[i].fmt; i++) {
-		if (additional2[i].code1 == asc &&
-		    ascq >= additional2[i].code2_min &&
-		    ascq <= additional2[i].code2_max) {
+		if (additional2[i].asc == scsi_sense_asc(sshdr) &&
+		    scsi_sense_ascq(sshdr) >= additional2[i].ascq_min &&
+		    scsi_sense_ascq(sshdr) <= additional2[i].ascq_max) {
 			*fmt = additional2[i].fmt;
 			return additional2[i].str;
 		}
diff --git a/drivers/scsi/csiostor/csio_attr.c b/drivers/scsi/csiostor/csio_attr.c
index 200e50089711e5..9f021a7f78bd2f 100644
--- a/drivers/scsi/csiostor/csio_attr.c
+++ b/drivers/scsi/csiostor/csio_attr.c
@@ -84,9 +84,9 @@ csio_reg_rnode(struct csio_rnode *rn)
 
 	ln->num_reg_rnodes++;
 	rport = rn->rport;
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	*((struct csio_rnode **)rport->dd_data) = rn;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	sp = &rn->rn_sparm;
 	rport->maxframe_size = ntohs(sp->csp.sp_bb_data);
diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c
index b1de615cf31626..0114e70e988442 100644
--- a/drivers/scsi/csiostor/csio_scsi.c
+++ b/drivers/scsi/csiostor/csio_scsi.c
@@ -2256,14 +2256,14 @@ csio_scan_finished(struct Scsi_Host *shost, unsigned long time)
 	struct csio_lnode *ln = shost_priv(shost);
 	int rv = 1;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	if (!ln->hwp || csio_list_deleted(&ln->sm.sm_list))
 		goto out;
 
 	rv = csio_scan_done(ln, jiffies, time, csio_max_scan_tmo * HZ,
 			    csio_delta_scan_tmo * HZ);
 out:
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	return rv;
 }
diff --git a/drivers/scsi/dc395x.c b/drivers/scsi/dc395x.c
index ba490174a417d9..10e2078f4a0fde 100644
--- a/drivers/scsi/dc395x.c
+++ b/drivers/scsi/dc395x.c
@@ -91,8 +91,8 @@
 #endif
 
 
-#define DC395x_LOCK_IO(dev,flags)		spin_lock_irqsave(((struct Scsi_Host *)dev)->host_lock, flags)
-#define DC395x_UNLOCK_IO(dev,flags)		spin_unlock_irqrestore(((struct Scsi_Host *)dev)->host_lock, flags)
+#define DC395x_LOCK_IO(dev,flags)		spin_lock_irqsave(&((struct Scsi_Host *)dev)->host_lock, flags)
+#define DC395x_UNLOCK_IO(dev,flags)		spin_unlock_irqrestore(&((struct Scsi_Host *)dev)->host_lock, flags)
 
 #define DC395x_read8(acb,address)		(u8)(inb(acb->io_port_base + (address)))
 #define DC395x_read16(acb,address)		(u16)(inw(acb->io_port_base + (address)))
@@ -1016,9 +1016,9 @@ static int dc395x_eh_bus_reset(struct scsi_cmnd *cmd)
 {
 	int rc;
 
-	spin_lock_irq(cmd->device->host->host_lock);
+	spin_lock_irq(&cmd->device->host->host_lock);
 	rc = __dc395x_eh_bus_reset(cmd);
-	spin_unlock_irq(cmd->device->host->host_lock);
+	spin_unlock_irq(&cmd->device->host->host_lock);
 
 	return rc;
 }
diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index d81ec47a71402f..87ac0a6d63e54a 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -433,61 +433,43 @@ static enum scsi_disposition alua_check_sense(struct scsi_device *sdev,
 {
 	switch (sense_hdr->sense_key) {
 	case NOT_READY:
-		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x0a) {
-			/*
-			 * LUN Not Accessible - ALUA state transition
-			 */
+		if (sense_hdr->sense_code ==
+		    LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION) {
 			alua_handle_state_transition(sdev);
 			return NEEDS_RETRY;
 		}
 		break;
 	case UNIT_ATTENTION:
-		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x0a) {
-			/*
-			 * LUN Not Accessible - ALUA state transition
-			 */
+		if (sense_hdr->sense_code ==
+		    LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION) {
 			alua_handle_state_transition(sdev);
 			return NEEDS_RETRY;
 		}
-		if (sense_hdr->asc == 0x29 && sense_hdr->ascq == 0x00) {
+		if (sense_hdr->sense_code ==
+		    POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED) {
 			/*
-			 * Power On, Reset, or Bus Device Reset.
 			 * Might have obscured a state transition,
 			 * so schedule a recheck.
 			 */
 			alua_check(sdev, true);
 			return ADD_TO_MLQUEUE;
 		}
-		if (sense_hdr->asc == 0x29 && sense_hdr->ascq == 0x04)
-			/*
-			 * Device internal reset
-			 */
+		if (sense_hdr->sense_code == DEVICE_INTERNAL_RESET)
 			return ADD_TO_MLQUEUE;
-		if (sense_hdr->asc == 0x2a && sense_hdr->ascq == 0x01)
-			/*
-			 * Mode Parameters Changed
-			 */
+		if (sense_hdr->sense_code == MODE_PARAMETERS_CHANGED)
 			return ADD_TO_MLQUEUE;
-		if (sense_hdr->asc == 0x2a && sense_hdr->ascq == 0x06) {
-			/*
-			 * ALUA state changed
-			 */
+		if (sense_hdr->sense_code == ASYMMETRIC_ACCESS_STATE_CHANGED) {
 			alua_check(sdev, true);
 			return ADD_TO_MLQUEUE;
 		}
-		if (sense_hdr->asc == 0x2a && sense_hdr->ascq == 0x07) {
-			/*
-			 * Implicit ALUA state transition failed
-			 */
+		if (sense_hdr->sense_code ==
+		    IMPLICIT_ASYMMETRIC_ACCESS_STATE_TRANSITION_FAILED) {
 			alua_check(sdev, true);
 			return ADD_TO_MLQUEUE;
 		}
-		if (sense_hdr->asc == 0x3f && sense_hdr->ascq == 0x03)
-			/*
-			 * Inquiry data has changed
-			 */
+		if (sense_hdr->sense_code == INQUIRY_DATA_HAS_CHANGED)
 			return ADD_TO_MLQUEUE;
-		if (sense_hdr->asc == 0x3f && sense_hdr->ascq == 0x0e)
+		if (sense_hdr->sense_code == REPORTED_LUNS_DATA_HAS_CHANGED)
 			/*
 			 * REPORTED_LUNS_DATA_HAS_CHANGED is reported
 			 * when switching controllers on targets like
@@ -517,12 +499,12 @@ static int alua_tur(struct scsi_device *sdev)
 				      ALUA_FAILOVER_RETRIES, &sense_hdr);
 	if ((sense_hdr.sense_key == NOT_READY ||
 	     sense_hdr.sense_key == UNIT_ATTENTION) &&
-	    sense_hdr.asc == 0x04 && sense_hdr.ascq == 0x0a)
+	    sense_hdr.sense_code ==
+	    LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION)
 		return SCSI_DH_RETRY;
-	else if (retval)
+	if (retval)
 		return SCSI_DH_IO;
-	else
-		return SCSI_DH_OK;
+	return SCSI_DH_OK;
 }
 
 /*
@@ -619,7 +601,8 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_port_group *pg)
 		 * transition. So set the state to 'transitioning' directly.
 		 */
 		if (sense_hdr.sense_key == NOT_READY &&
-		    sense_hdr.asc == 0x04 && sense_hdr.ascq == 0x0a) {
+		    sense_hdr.sense_code ==
+		    LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION) {
 			transitioning_sense = true;
 			goto skip_rtpg;
 		}
diff --git a/drivers/scsi/device_handler/scsi_dh_emc.c b/drivers/scsi/device_handler/scsi_dh_emc.c
index ff41b51ef46272..4b0ea6045ba397 100644
--- a/drivers/scsi/device_handler/scsi_dh_emc.c
+++ b/drivers/scsi/device_handler/scsi_dh_emc.c
@@ -105,10 +105,10 @@ static int trespass_endio(struct scsi_device *sdev,
 	sdev_printk(KERN_ERR, sdev, "%s: Found valid sense data 0x%2x, "
 		    "0x%2x, 0x%2x while sending CLARiiON trespass "
 		    "command.\n", CLARIION_NAME, sshdr->sense_key,
-		    sshdr->asc, sshdr->ascq);
+		    scsi_sense_asc(sshdr), scsi_sense_ascq(sshdr));
 
-	if (sshdr->sense_key == 0x05 && sshdr->asc == 0x04 &&
-	    sshdr->ascq == 0x00) {
+	if (sshdr->sense_key == ILLEGAL_REQUEST &&
+	    sshdr->sense_code == LU_NOT_READY) {
 		/*
 		 * Array based copy in progress -- do not send
 		 * mode_select or copy will be aborted mid-stream.
@@ -117,18 +117,18 @@ static int trespass_endio(struct scsi_device *sdev,
 			    "progress while sending CLARiiON trespass "
 			    "command.\n", CLARIION_NAME);
 		err = SCSI_DH_DEV_TEMP_BUSY;
-	} else if (sshdr->sense_key == 0x02 && sshdr->asc == 0x04 &&
-		   sshdr->ascq == 0x03) {
-		/*
-		 * LUN Not Ready - Manual Intervention Required
-		 * indicates in-progress ucode upgrade (NDU).
-		 */
+	} else if (sshdr->sense_key == NOT_READY &&
+		   sshdr->sense_code ==
+		   LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED) {
+		/* Indicates in-progress ucode upgrade (NDU). */
 		sdev_printk(KERN_INFO, sdev, "%s: Detected in-progress "
 			    "ucode upgrade NDU operation while sending "
 			    "CLARiiON trespass command.\n", CLARIION_NAME);
 		err = SCSI_DH_DEV_TEMP_BUSY;
-	} else
+	} else {
 		err = SCSI_DH_DEV_FAILED;
+	}
+
 	return err;
 }
 
@@ -288,7 +288,8 @@ static enum scsi_disposition clariion_check_sense(struct scsi_device *sdev,
 {
 	switch (sense_hdr->sense_key) {
 	case NOT_READY:
-		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x03)
+		if (sense_hdr->sense_code ==
+		    LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED)
 			/*
 			 * LUN Not Ready - Manual Intervention Required
 			 * indicates this is a passive path.
@@ -305,7 +306,7 @@ static enum scsi_disposition clariion_check_sense(struct scsi_device *sdev,
 			return SUCCESS;
 		break;
 	case ILLEGAL_REQUEST:
-		if (sense_hdr->asc == 0x25 && sense_hdr->ascq == 0x01)
+		if (sense_hdr->sense_code == scsi_sense_code(0x25, 0x01))
 			/*
 			 * An array based copy is in progress. Do not
 			 * fail the path, do not bypass to another PG,
@@ -319,7 +320,8 @@ static enum scsi_disposition clariion_check_sense(struct scsi_device *sdev,
 			return SUCCESS;
 		break;
 	case UNIT_ATTENTION:
-		if (sense_hdr->asc == 0x29 && sense_hdr->ascq == 0x00)
+		if (sense_hdr->sense_code ==
+		    POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED)
 			/*
 			 * Unit Attention Code. This is the first IO
 			 * to the new path, so just retry.
diff --git a/drivers/scsi/device_handler/scsi_dh_hp_sw.c b/drivers/scsi/device_handler/scsi_dh_hp_sw.c
index 6e8849d7f0a32b..d8653452ed2b57 100644
--- a/drivers/scsi/device_handler/scsi_dh_hp_sw.c
+++ b/drivers/scsi/device_handler/scsi_dh_hp_sw.c
@@ -47,7 +47,8 @@ static int tur_done(struct scsi_device *sdev, struct hp_sw_dh_data *h,
 
 	switch (sshdr->sense_key) {
 	case NOT_READY:
-		if (sshdr->asc == 0x04 && sshdr->ascq == 2) {
+		if (sshdr->sense_code ==
+		    LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED) {
 			/*
 			 * LUN not ready - Initialization command required
 			 *
@@ -61,8 +62,8 @@ static int tur_done(struct scsi_device *sdev, struct hp_sw_dh_data *h,
 	default:
 		sdev_printk(KERN_WARNING, sdev,
 			   "%s: sending tur failed, sense %x/%x/%x\n",
-			   HP_SW_NAME, sshdr->sense_key, sshdr->asc,
-			   sshdr->ascq);
+			    HP_SW_NAME, sshdr->sense_key,
+			    scsi_sense_asc(sshdr), scsi_sense_ascq(sshdr));
 		break;
 	}
 	return ret;
@@ -84,9 +85,8 @@ static int hp_sw_tur(struct scsi_device *sdev, struct hp_sw_dh_data *h)
 				REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = SCMD_FAILURE_NO_LIMIT,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -133,14 +133,9 @@ static int hp_sw_start_stop(struct hp_sw_dh_data *h)
 				REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER;
 	struct scsi_failure failure_defs[] = {
 		{
-			/*
-			 * LUN not ready - manual intervention required
-			 *
-			 * Switch-over in progress, retry.
-			 */
-			.sense = NOT_READY,
-			.asc = 0x04,
-			.ascq = 0x03,
+			/* Switch-over in progress, retry. */
+			.sense_key = NOT_READY,
+			.sense_code = LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED,
 			.allowed = HP_SW_RETRIES,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -167,7 +162,8 @@ static int hp_sw_start_stop(struct hp_sw_dh_data *h)
 
 	switch (sshdr.sense_key) {
 	case NOT_READY:
-		if (sshdr.asc == 0x04 && sshdr.ascq == 3) {
+		if (sshdr.sense_code ==
+		    LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED) {
 			rc = SCSI_DH_RETRY;
 			break;
 		}
@@ -176,7 +172,8 @@ static int hp_sw_start_stop(struct hp_sw_dh_data *h)
 		sdev_printk(KERN_WARNING, sdev,
 			    "%s: sending start_stop_unit failed, "
 			    "sense %x/%x/%x\n", HP_SW_NAME,
-			    sshdr.sense_key, sshdr.asc, sshdr.ascq);
+			    sshdr.sense_key, scsi_sense_asc(&sshdr),
+			    scsi_sense_ascq(&sshdr));
 		rc = SCSI_DH_IO;
 	}
 
diff --git a/drivers/scsi/device_handler/scsi_dh_rdac.c b/drivers/scsi/device_handler/scsi_dh_rdac.c
index 88c8e36b221e2e..742534c9938654 100644
--- a/drivers/scsi/device_handler/scsi_dh_rdac.c
+++ b/drivers/scsi/device_handler/scsi_dh_rdac.c
@@ -493,7 +493,8 @@ static int mode_select_handle_sense(struct scsi_device *sdev,
 	RDAC_LOG(RDAC_LOG_FAILOVER, sdev, "array %s, ctlr %d, "
 		"MODE_SELECT returned with sense %02x/%02x/%02x",
 		(char *) h->ctlr->array_name, h->ctlr->index,
-		sense_hdr->sense_key, sense_hdr->asc, sense_hdr->ascq);
+		 sense_hdr->sense_key, scsi_sense_asc(sense_hdr),
+		 scsi_sense_ascq(sense_hdr));
 
 	return SCSI_DH_IO;
 }
@@ -514,35 +515,29 @@ static void send_mode_select(struct work_struct *work)
 				REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = NO_SENSE,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = NO_SENSE,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = ABORTED_COMMAND,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = ABORTED_COMMAND,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
-		/* LUN Not Ready and is in the Process of Becoming Ready */
 		{
-			.sense = NOT_READY,
-			.asc = 0x04,
-			.ascq = 0x01,
+			.sense_key = NOT_READY,
+			.sense_code = LU_IS_IN_PROCESS_OF_BECOMING_READY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
-		/* Command Lock contention */
+		/* Command Lock contention (not a standard sense code) */
 		{
-			.sense = ILLEGAL_REQUEST,
-			.asc = 0x91,
-			.ascq = 0x36,
+			.sense_key = ILLEGAL_REQUEST,
+			.sense_code = scsi_sense_code(0x91, 0x36),
 			.allowed = SCMD_FAILURE_NO_LIMIT,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -674,33 +669,34 @@ static enum scsi_disposition rdac_check_sense(struct scsi_device *sdev,
 {
 	struct rdac_dh_data *h = sdev->handler_data;
 
-	RDAC_LOG(RDAC_LOG_SENSE, sdev, "array %s, ctlr %d, "
-			"I/O returned with sense %02x/%02x/%02x",
-			(char *) h->ctlr->array_name, h->ctlr->index,
-			sense_hdr->sense_key, sense_hdr->asc, sense_hdr->ascq);
+	RDAC_LOG(RDAC_LOG_SENSE, sdev,
+		 "array %s, ctlr %d, I/O returned with sense %02x/%02x/%02x",
+		 (char *) h->ctlr->array_name, h->ctlr->index,
+		 sense_hdr->sense_key, scsi_sense_asc(sense_hdr),
+		 scsi_sense_ascq(sense_hdr));
 
 	switch (sense_hdr->sense_key) {
 	case NOT_READY:
-		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x01)
+		if (sense_hdr->sense_code == LU_IS_IN_PROCESS_OF_BECOMING_READY)
 			/* LUN Not Ready - Logical Unit Not Ready and is in
 			* the process of becoming ready
 			* Just retry.
 			*/
 			return ADD_TO_MLQUEUE;
-		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0x81)
+		if (sense_hdr->sense_code == scsi_sense_code(0x04, 0x81))
 			/* LUN Not Ready - Storage firmware incompatible
 			 * Manual code synchonisation required.
 			 *
 			 * Nothing we can do here. Try to bypass the path.
 			 */
 			return SUCCESS;
-		if (sense_hdr->asc == 0x04 && sense_hdr->ascq == 0xA1)
+		if (sense_hdr->sense_code == scsi_sense_code(0x04, 0xA1))
 			/* LUN Not Ready - Quiescense in progress
 			 *
 			 * Just retry and wait.
 			 */
 			return ADD_TO_MLQUEUE;
-		if (sense_hdr->asc == 0xA1  && sense_hdr->ascq == 0x02)
+		if (sense_hdr->sense_code == scsi_sense_code(0xA1, 0x02))
 			/* LUN Not Ready - Quiescense in progress
 			 * or has been achieved
 			 * Just retry.
@@ -708,7 +704,7 @@ static enum scsi_disposition rdac_check_sense(struct scsi_device *sdev,
 			return ADD_TO_MLQUEUE;
 		break;
 	case ILLEGAL_REQUEST:
-		if (sense_hdr->asc == 0x94 && sense_hdr->ascq == 0x01) {
+		if (sense_hdr->sense_code == scsi_sense_code(0x94, 0x01)) {
 			/* Invalid Request - Current Logical Unit Ownership.
 			 * Controller is not the current owner of the LUN,
 			 * Fail the path, so that the other path be used.
@@ -718,12 +714,13 @@ static enum scsi_disposition rdac_check_sense(struct scsi_device *sdev,
 		}
 		break;
 	case UNIT_ATTENTION:
-		if (sense_hdr->asc == 0x29 && sense_hdr->ascq == 0x00)
+		if (sense_hdr->sense_code ==
+		    POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED)
 			/*
 			 * Power On, Reset, or Bus Device Reset, just retry.
 			 */
 			return ADD_TO_MLQUEUE;
-		if (sense_hdr->asc == 0x8b && sense_hdr->ascq == 0x02)
+		if (sense_hdr->sense_code == scsi_sense_code(0x8b, 0x02))
 			/*
 			 * Quiescence in progress , just retry.
 			 */
diff --git a/drivers/scsi/esp_scsi.c b/drivers/scsi/esp_scsi.c
index 897a01d3e30387..9993b0ae9c4f27 100644
--- a/drivers/scsi/esp_scsi.c
+++ b/drivers/scsi/esp_scsi.c
@@ -2174,7 +2174,7 @@ irqreturn_t scsi_esp_intr(int irq, void *dev_id)
 	unsigned long flags;
 	irqreturn_t ret;
 
-	spin_lock_irqsave(esp->host->host_lock, flags);
+	spin_lock_irqsave(&esp->host->host_lock, flags);
 	ret = IRQ_NONE;
 	if (esp->ops->irq_pending(esp)) {
 		ret = IRQ_HANDLED;
@@ -2194,7 +2194,7 @@ irqreturn_t scsi_esp_intr(int irq, void *dev_id)
 				break;
 		}
 	}
-	spin_unlock_irqrestore(esp->host->host_lock, flags);
+	spin_unlock_irqrestore(&esp->host->host_lock, flags);
 
 	return ret;
 }
@@ -2497,7 +2497,7 @@ static int esp_eh_abort_handler(struct scsi_cmnd *cmd)
 	/* XXX This helps a lot with debugging but might be a bit
 	 * XXX much for the final driver.
 	 */
-	spin_lock_irqsave(esp->host->host_lock, flags);
+	spin_lock_irqsave(&esp->host->host_lock, flags);
 	shost_printk(KERN_ERR, esp->host, "Aborting command [%p:%02x]\n",
 		     cmd, cmd->cmnd[0]);
 	ent = esp->active_cmd;
@@ -2514,9 +2514,9 @@ static int esp_eh_abort_handler(struct scsi_cmnd *cmd)
 			     ent->cmd, ent->cmd->cmnd[0]);
 	}
 	esp_dump_cmd_log(esp);
-	spin_unlock_irqrestore(esp->host->host_lock, flags);
+	spin_unlock_irqrestore(&esp->host->host_lock, flags);
 
-	spin_lock_irqsave(esp->host->host_lock, flags);
+	spin_lock_irqsave(&esp->host->host_lock, flags);
 
 	ent = NULL;
 	list_for_each_entry(tmp, &esp->queued_cmds, list) {
@@ -2579,12 +2579,12 @@ static int esp_eh_abort_handler(struct scsi_cmnd *cmd)
 		goto out_failure;
 	}
 
-	spin_unlock_irqrestore(esp->host->host_lock, flags);
+	spin_unlock_irqrestore(&esp->host->host_lock, flags);
 
 	if (!wait_for_completion_timeout(&eh_done, 5 * HZ)) {
-		spin_lock_irqsave(esp->host->host_lock, flags);
+		spin_lock_irqsave(&esp->host->host_lock, flags);
 		ent->eh_done = NULL;
-		spin_unlock_irqrestore(esp->host->host_lock, flags);
+		spin_unlock_irqrestore(&esp->host->host_lock, flags);
 
 		return FAILED;
 	}
@@ -2592,7 +2592,7 @@ static int esp_eh_abort_handler(struct scsi_cmnd *cmd)
 	return SUCCESS;
 
 out_success:
-	spin_unlock_irqrestore(esp->host->host_lock, flags);
+	spin_unlock_irqrestore(&esp->host->host_lock, flags);
 	return SUCCESS;
 
 out_failure:
@@ -2600,7 +2600,7 @@ static int esp_eh_abort_handler(struct scsi_cmnd *cmd)
 	 * XXX since we know which target/lun in particular is
 	 * XXX causing trouble.
 	 */
-	spin_unlock_irqrestore(esp->host->host_lock, flags);
+	spin_unlock_irqrestore(&esp->host->host_lock, flags);
 	return FAILED;
 }
 
@@ -2612,7 +2612,7 @@ static int esp_eh_bus_reset_handler(struct scsi_cmnd *cmd)
 
 	init_completion(&eh_reset);
 
-	spin_lock_irqsave(esp->host->host_lock, flags);
+	spin_lock_irqsave(&esp->host->host_lock, flags);
 
 	esp->eh_reset = &eh_reset;
 
@@ -2624,14 +2624,14 @@ static int esp_eh_bus_reset_handler(struct scsi_cmnd *cmd)
 	esp->flags |= ESP_FLAG_RESETTING;
 	scsi_esp_cmd(esp, ESP_CMD_RS);
 
-	spin_unlock_irqrestore(esp->host->host_lock, flags);
+	spin_unlock_irqrestore(&esp->host->host_lock, flags);
 
 	ssleep(esp_bus_reset_settle);
 
 	if (!wait_for_completion_timeout(&eh_reset, 5 * HZ)) {
-		spin_lock_irqsave(esp->host->host_lock, flags);
+		spin_lock_irqsave(&esp->host->host_lock, flags);
 		esp->eh_reset = NULL;
-		spin_unlock_irqrestore(esp->host->host_lock, flags);
+		spin_unlock_irqrestore(&esp->host->host_lock, flags);
 
 		return FAILED;
 	}
@@ -2645,10 +2645,10 @@ static int esp_eh_host_reset_handler(struct scsi_cmnd *cmd)
 	struct esp *esp = shost_priv(cmd->device->host);
 	unsigned long flags;
 
-	spin_lock_irqsave(esp->host->host_lock, flags);
+	spin_lock_irqsave(&esp->host->host_lock, flags);
 	esp_bootup_reset(esp);
 	esp_reset_cleanup(esp);
-	spin_unlock_irqrestore(esp->host->host_lock, flags);
+	spin_unlock_irqrestore(&esp->host->host_lock, flags);
 
 	ssleep(esp_bus_reset_settle);
 
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 438ac7c3a9e3fd..a2cf65fac29704 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -2705,7 +2705,7 @@ static void fcoe_vport_remove(struct fc_lport *lport)
 	fc_host = shost_to_fc_host(shost);
 
 	/* Loop through all the vports and mark them for deletion */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
 		if (vport->flags & (FC_VPORT_DEL | FC_VPORT_CREATING)) {
 			continue;
@@ -2715,7 +2715,7 @@ static void fcoe_vport_remove(struct fc_lport *lport)
 				   &vport->vport_delete_work);
 		}
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	flush_workqueue(fc_host_work_q(shost));
 }
diff --git a/drivers/scsi/fdomain.c b/drivers/scsi/fdomain.c
index 22fbb0222f07de..b5766f752cdd87 100644
--- a/drivers/scsi/fdomain.c
+++ b/drivers/scsi/fdomain.c
@@ -273,7 +273,7 @@ static void fdomain_work(struct work_struct *work)
 	int status;
 	int done = 0;
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 
 	if (scsi_pointer->phase & in_arbitration) {
 		status = inb(fd->base + REG_ASTAT);
@@ -380,7 +380,7 @@ static void fdomain_work(struct work_struct *work)
 			     fd->base + REG_ICTL);
 	}
 out:
-	spin_unlock_irqrestore(sh->host_lock, flags);
+	spin_unlock_irqrestore(&sh->host_lock, flags);
 }
 
 static irqreturn_t fdomain_irq(int irq, void *dev_id)
@@ -416,7 +416,7 @@ static enum scsi_qc_status fdomain_queue(struct Scsi_Host *sh,
 	scsi_pointer->phase		= in_arbitration;
 	scsi_set_resid(cmd, scsi_bufflen(cmd));
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 
 	fd->cur_cmd = cmd;
 
@@ -431,7 +431,7 @@ static enum scsi_qc_status fdomain_queue(struct Scsi_Host *sh,
 	/* Start arbitration */
 	outb(ACTL_ARB | ACTL_IRQEN | PARITY_MASK, fd->base + REG_ACTL);
 
-	spin_unlock_irqrestore(sh->host_lock, flags);
+	spin_unlock_irqrestore(&sh->host_lock, flags);
 
 	return 0;
 }
@@ -445,7 +445,7 @@ static int fdomain_abort(struct scsi_cmnd *cmd)
 	if (!fd->cur_cmd)
 		return FAILED;
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 
 	fdomain_make_bus_idle(fd);
 	fdomain_scsi_pointer(fd->cur_cmd)->phase |= aborted;
@@ -453,7 +453,7 @@ static int fdomain_abort(struct scsi_cmnd *cmd)
 	/* Aborts are not done well. . . */
 	set_host_byte(fd->cur_cmd, DID_ABORT);
 	fdomain_finish_cmd(fd);
-	spin_unlock_irqrestore(sh->host_lock, flags);
+	spin_unlock_irqrestore(&sh->host_lock, flags);
 	return SUCCESS;
 }
 
@@ -463,9 +463,9 @@ static int fdomain_host_reset(struct scsi_cmnd *cmd)
 	struct fdomain *fd = shost_priv(sh);
 	unsigned long flags;
 
-	spin_lock_irqsave(sh->host_lock, flags);
+	spin_lock_irqsave(&sh->host_lock, flags);
 	fdomain_reset(fd->base);
-	spin_unlock_irqrestore(sh->host_lock, flags);
+	spin_unlock_irqrestore(&sh->host_lock, flags);
 	return SUCCESS;
 }
 
diff --git a/drivers/scsi/gvp11.c b/drivers/scsi/gvp11.c
index 79bd64e12adc2a..a29457f5adfbe0 100644
--- a/drivers/scsi/gvp11.c
+++ b/drivers/scsi/gvp11.c
@@ -42,9 +42,9 @@ static irqreturn_t gvp11_intr(int irq, void *data)
 	if (!(status & GVP11_DMAC_INT_PENDING))
 		return IRQ_NONE;
 
-	spin_lock_irqsave(instance->host_lock, flags);
+	spin_lock_irqsave(&instance->host_lock, flags);
 	wd33c93_intr(instance);
-	spin_unlock_irqrestore(instance->host_lock, flags);
+	spin_unlock_irqrestore(&instance->host_lock, flags);
 	return IRQ_HANDLED;
 }
 
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index e72055c1ebbb57..e8361ba655644c 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -73,7 +73,7 @@ static struct class shost_class = {
  *	transition is illegal.
  **/
 int scsi_host_set_state(struct Scsi_Host *shost, enum scsi_host_state state)
-	__must_hold(shost->host_lock)
+	__must_hold(&shost->host_lock)
 {
 	enum scsi_host_state oldstate = READ_ONCE(shost->shost_state);
 
@@ -168,14 +168,14 @@ void scsi_remove_host(struct Scsi_Host *shost)
 	unsigned long flags;
 
 	mutex_lock(&shost->scan_mutex);
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (scsi_host_set_state(shost, SHOST_CANCEL))
 		if (scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY)) {
-			spin_unlock_irqrestore(shost->host_lock, flags);
+			spin_unlock_irqrestore(&shost->host_lock, flags);
 			mutex_unlock(&shost->scan_mutex);
 			return;
 		}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	scsi_autopm_get_host(shost);
 	flush_workqueue(shost->tmf_work_q);
@@ -193,10 +193,10 @@ void scsi_remove_host(struct Scsi_Host *shost)
 	kref_put(&shost->tagset_refcnt, scsi_mq_free_tags);
 	wait_for_completion(&shost->tagset_freed);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (scsi_host_set_state(shost, SHOST_DEL))
 		BUG_ON(scsi_host_set_state(shost, SHOST_DEL_RECOVERY));
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	transport_unregister_device(&shost->shost_gendev);
 	device_unregister(&shost->shost_dev);
@@ -277,7 +277,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev,
 	if (error)
 		goto out_disable_runtime_pm;
 
-	scoped_guard(spinlock_irq, shost->host_lock)
+	scoped_guard(spinlock_irq, &shost->host_lock)
 		scsi_host_set_state(shost, SHOST_RUNNING);
 	get_device(shost->shost_gendev.parent);
 
@@ -414,8 +414,7 @@ struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int priv
 	if (!shost)
 		return NULL;
 
-	shost->host_lock = &shost->default_lock;
-	scoped_guard(spinlock_init, shost->host_lock)
+	scoped_guard(spinlock_init, &shost->host_lock)
 		shost->shost_state = SHOST_CREATED;
 	INIT_LIST_HEAD(&shost->__devices);
 	INIT_LIST_HEAD(&shost->__targets);
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index 8edad1830abe4d..795e8c7592af8d 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -353,14 +353,13 @@ static inline bool hpsa_is_cmd_idle(struct CommandList *c)
 
 /* extract sense key, asc, and ascq from sense data.  -1 means invalid. */
 static void decode_sense_data(const u8 *sense_data, int sense_data_len,
-			u8 *sense_key, u8 *asc, u8 *ascq)
+			      u8 *sense_key, u16 *sense_code)
 {
 	struct scsi_sense_hdr sshdr;
 	bool rc;
 
 	*sense_key = -1;
-	*asc = -1;
-	*ascq = -1;
+	*sense_code = -1;
 
 	if (sense_data_len < 1)
 		return;
@@ -368,15 +367,15 @@ static void decode_sense_data(const u8 *sense_data, int sense_data_len,
 	rc = scsi_normalize_sense(sense_data, sense_data_len, &sshdr);
 	if (rc) {
 		*sense_key = sshdr.sense_key;
-		*asc = sshdr.asc;
-		*ascq = sshdr.ascq;
+		*sense_code = sshdr.sense_code;
 	}
 }
 
 static int check_for_unit_attention(struct ctlr_info *h,
 	struct CommandList *c)
 {
-	u8 sense_key, asc, ascq;
+	u8 sense_key;
+	u16 sense_code;
 	int sense_len;
 
 	if (c->err_info->SenseLen > sizeof(c->err_info->SenseInfo))
@@ -385,34 +384,35 @@ static int check_for_unit_attention(struct ctlr_info *h,
 		sense_len = c->err_info->SenseLen;
 
 	decode_sense_data(c->err_info->SenseInfo, sense_len,
-				&sense_key, &asc, &ascq);
-	if (sense_key != UNIT_ATTENTION || asc == 0xff)
+			  &sense_key, &sense_code);
+	if (sense_key != UNIT_ATTENTION ||
+	    scsi_sense_code_asc(sense_code) == 0xff)
 		return 0;
 
-	switch (asc) {
-	case STATE_CHANGED:
+	switch (scsi_sense_code_asc(sense_code)) {
+	case ASC_PARAMETERS_CHANGED:
 		dev_warn(&h->pdev->dev,
 			"%s: a state change detected, command retried\n",
 			h->devname);
 		break;
-	case LUN_FAILED:
+	case ASC_LU_HAS_NOT_SELF_CONFIGURED_YET:
 		dev_warn(&h->pdev->dev,
 			"%s: LUN failure detected\n", h->devname);
 		break;
-	case REPORT_LUNS_CHANGED:
+	case ASC_TARGET_OPERATING_CONDITIONS_HAVE_CHANGED:
 		dev_warn(&h->pdev->dev,
 			"%s: report LUN data changed\n", h->devname);
 	/*
-	 * Note: this REPORT_LUNS_CHANGED condition only occurs on the external
-	 * target (array) devices.
+	 * Note: this ASC_TARGET_OPERATING_CONDITIONS_HAVE_CHANGED condition
+	 * only occurs on the external target (array) devices.
 	 */
 		break;
-	case POWER_OR_RESET:
+	case ASC_POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED:
 		dev_warn(&h->pdev->dev,
 			"%s: a power on or device reset detected\n",
 			h->devname);
 		break;
-	case UNIT_ATTENTION_CLEARED:
+	case ASC_COMMANDS_CLEARED_BY_ANOTHER_INITIATOR:
 		dev_warn(&h->pdev->dev,
 			"%s: unit attention cleared by another initiator\n",
 			h->devname);
@@ -2562,10 +2562,8 @@ static void complete_scsi_command(struct CommandList *cp)
 	struct ErrorInfo *ei;
 	struct hpsa_scsi_dev_t *dev;
 	struct io_accel2_cmd *c2;
-
 	u8 sense_key;
-	u8 asc;      /* additional sense code */
-	u8 ascq;     /* additional sense code qualifier */
+	u16 sense_code;
 	unsigned long sense_data_size;
 
 	ei = cp->err_info;
@@ -2666,18 +2664,19 @@ static void complete_scsi_command(struct CommandList *cp)
 		memcpy(cmd->sense_buffer, ei->SenseInfo, sense_data_size);
 		if (ei->ScsiStatus)
 			decode_sense_data(ei->SenseInfo, sense_data_size,
-				&sense_key, &asc, &ascq);
+				&sense_key, &sense_code);
 		if (ei->ScsiStatus == SAM_STAT_CHECK_CONDITION) {
 			switch (sense_key) {
 			case ABORTED_COMMAND:
 				cmd->result |= DID_SOFT_ERROR << 16;
 				break;
 			case UNIT_ATTENTION:
-				if (asc == 0x3F && ascq == 0x0E)
+				if (sense_code ==
+				    REPORTED_LUNS_DATA_HAS_CHANGED)
 					h->drv_req_rescan = 1;
 				break;
 			case ILLEGAL_REQUEST:
-				if (asc == 0x25 && ascq == 0x00) {
+				if (sense_code == LU_NOT_SUPPORTED) {
 					dev->removed = 1;
 					cmd->result = DID_NO_CONNECT << 16;
 				}
@@ -2693,7 +2692,9 @@ static void complete_scsi_command(struct CommandList *cp)
 				"Sense: 0x%x, ASC: 0x%x, ASCQ: 0x%x, "
 				"Returning result: 0x%x\n",
 				cp, ei->ScsiStatus,
-				sense_key, asc, ascq,
+				sense_key,
+				scsi_sense_code_asc(sense_code),
+				scsi_sense_code_ascq(sense_code),
 				cmd->result);
 		} else {  /* scsi status is zero??? How??? */
 			dev_warn(&h->pdev->dev, "cp %p SCSI status was 0. "
@@ -2919,7 +2920,8 @@ static void hpsa_scsi_interpret_error(struct ctlr_info *h,
 {
 	const struct ErrorInfo *ei = cp->err_info;
 	struct device *d = &cp->h->pdev->dev;
-	u8 sense_key, asc, ascq;
+	u8 sense_key;
+	u16 sense_code;
 	int sense_len;
 
 	switch (ei->CommandStatus) {
@@ -2928,12 +2930,13 @@ static void hpsa_scsi_interpret_error(struct ctlr_info *h,
 			sense_len = sizeof(ei->SenseInfo);
 		else
 			sense_len = ei->SenseLen;
-		decode_sense_data(ei->SenseInfo, sense_len,
-					&sense_key, &asc, &ascq);
+		decode_sense_data(ei->SenseInfo, sense_len, &sense_key,
+				  &sense_code);
 		hpsa_print_cmd(h, "SCSI status", cp);
 		if (ei->ScsiStatus == SAM_STAT_CHECK_CONDITION)
 			dev_warn(d, "SCSI Status = 02, Sense key = 0x%02x, ASC = 0x%02x, ASCQ = 0x%02x\n",
-				sense_key, asc, ascq);
+				sense_key, scsi_sense_code_asc(sense_code),
+				 scsi_sense_code_ascq(sense_code));
 		else
 			dev_warn(d, "SCSI Status = 0x%02x\n", ei->ScsiStatus);
 		if (ei->ScsiStatus == 0)
@@ -3868,12 +3871,10 @@ static unsigned char hpsa_volume_offline(struct ctlr_info *h,
 {
 	struct CommandList *c;
 	unsigned char *sense;
-	u8 sense_key, asc, ascq;
+	u8 sense_key;
+	u16 sense_code;
 	int sense_len;
 	int rc, ldstat = 0;
-#define ASC_LUN_NOT_READY 0x04
-#define ASCQ_LUN_NOT_READY_FORMAT_IN_PROGRESS 0x04
-#define ASCQ_LUN_NOT_READY_INITIALIZING_CMD_REQ 0x02
 
 	c = cmd_alloc(h);
 
@@ -3889,7 +3890,7 @@ static unsigned char hpsa_volume_offline(struct ctlr_info *h,
 		sense_len = sizeof(c->err_info->SenseInfo);
 	else
 		sense_len = c->err_info->SenseLen;
-	decode_sense_data(sense, sense_len, &sense_key, &asc, &ascq);
+	decode_sense_data(sense, sense_len, &sense_key, &sense_code);
 	cmd_free(h, c);
 
 	/* Determine the reason for not ready state */
@@ -3912,8 +3913,8 @@ static unsigned char hpsa_volume_offline(struct ctlr_info *h,
 		/* If VPD status page isn't available,
 		 * use ASC/ASCQ to determine state
 		 */
-		if ((ascq == ASCQ_LUN_NOT_READY_FORMAT_IN_PROGRESS) ||
-			(ascq == ASCQ_LUN_NOT_READY_INITIALIZING_CMD_REQ))
+		if (sense_code == LU_NOT_READY_FORMAT_IN_PROGRESS ||
+		    sense_code == LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED)
 			return ldstat;
 		break;
 	default:
diff --git a/drivers/scsi/hpsa_cmd.h b/drivers/scsi/hpsa_cmd.h
index ba6a3aa8d95405..0dbfbec37ce23e 100644
--- a/drivers/scsi/hpsa_cmd.h
+++ b/drivers/scsi/hpsa_cmd.h
@@ -63,29 +63,6 @@
 #define CISS_TMF_WRONG_LUN	0x09
 #define CISS_TMF_OVERLAPPED_TAG 0x0a
 
-/* Unit Attentions ASC's as defined for the MSA2012sa */
-#define POWER_OR_RESET			0x29
-#define STATE_CHANGED			0x2a
-#define UNIT_ATTENTION_CLEARED		0x2f
-#define LUN_FAILED			0x3e
-#define REPORT_LUNS_CHANGED		0x3f
-
-/* Unit Attentions ASCQ's as defined for the MSA2012sa */
-
-	/* These ASCQ's defined for ASC = POWER_OR_RESET */
-#define POWER_ON_RESET			0x00
-#define POWER_ON_REBOOT			0x01
-#define SCSI_BUS_RESET			0x02
-#define MSA_TARGET_RESET		0x03
-#define CONTROLLER_FAILOVER		0x04
-#define TRANSCEIVER_SE			0x05
-#define TRANSCEIVER_LVD			0x06
-
-	/* These ASCQ's defined for ASC = STATE_CHANGED */
-#define RESERVATION_PREEMPTED		0x03
-#define ASYM_ACCESS_CHANGED		0x06
-#define LUN_CAPACITY_CHANGED		0x09
-
 /* transfer direction */
 #define XFER_NONE               0x00
 #define XFER_WRITE              0x01
diff --git a/drivers/scsi/hptiop.c b/drivers/scsi/hptiop.c
index 7083c14c53028f..a3afb3446bbba2 100644
--- a/drivers/scsi/hptiop.c
+++ b/drivers/scsi/hptiop.c
@@ -385,9 +385,9 @@ static int iop_send_sync_msg(struct hptiop_hba *hba, u32 msg, u32 millisec)
 	hba->ops->post_msg(hba, msg);
 
 	for (i = 0; i < millisec; i++) {
-		spin_lock_irq(hba->host->host_lock);
+		spin_lock_irq(&hba->host->host_lock);
 		hba->ops->iop_intr(hba);
-		spin_unlock_irq(hba->host->host_lock);
+		spin_unlock_irq(&hba->host->host_lock);
 		if (hba->msg_done)
 			break;
 		msleep(1);
@@ -836,9 +836,9 @@ static irqreturn_t hptiop_intr(int irq, void *dev_id)
 	int  handled;
 	unsigned long flags;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	handled = hba->ops->iop_intr(hba);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return handled;
 }
diff --git a/drivers/scsi/ibmvscsi/ibmvfc-core.c b/drivers/scsi/ibmvscsi/ibmvfc-core.c
index 3534ac45e9b87a..aa7ae81df41b15 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc-core.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc-core.c
@@ -974,14 +974,14 @@ static int ibmvfc_reenable_crq_queue(struct ibmvfc_host *vhost)
 	if (rc)
 		dev_err(vhost->dev, "Error enabling adapter (rc=%d)\n", rc);
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	spin_lock(vhost->crq.q_lock);
 	vhost->do_enquiry = 1;
 	vhost->using_channels = 0;
 	vhost->do_scsi_login = 0;
 	vhost->do_nvme_login = 0;
 	spin_unlock(vhost->crq.q_lock);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	ibmvfc_reg_sub_crqs(vhost, &vhost->scsi_scrqs);
 	ibmvfc_reg_sub_crqs(vhost, &vhost->nvme_scrqs);
@@ -1013,7 +1013,7 @@ static int ibmvfc_reset_crq(struct ibmvfc_host *vhost)
 		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
 	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	spin_lock(vhost->crq.q_lock);
 	vhost->state = IBMVFC_NO_CRQ;
 	vhost->logged_in = 0;
@@ -1037,7 +1037,7 @@ static int ibmvfc_reset_crq(struct ibmvfc_host *vhost)
 		dev_warn(vhost->dev, "Couldn't register crq (rc=%d)\n", rc);
 
 	spin_unlock(vhost->crq.q_lock);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	ibmvfc_reg_sub_crqs(vhost, &vhost->scsi_scrqs);
 	ibmvfc_reg_sub_crqs(vhost, &vhost->nvme_scrqs);
@@ -1250,9 +1250,9 @@ static void ibmvfc_reset_host(struct ibmvfc_host *vhost)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	__ibmvfc_reset_host(vhost);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 }
 
 /**
@@ -1318,9 +1318,9 @@ static struct ibmvfc_target *ibmvfc_get_target(struct scsi_target *starget)
 	struct ibmvfc_target *tgt;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	tgt = __ibmvfc_get_target(starget);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return tgt;
 }
 
@@ -1336,7 +1336,7 @@ static void ibmvfc_get_host_speed(struct Scsi_Host *shost)
 	struct ibmvfc_host *vhost = shost_priv(shost);
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (vhost->state == IBMVFC_ACTIVE) {
 		switch (be64_to_cpu(vhost->login_buf->resp.link_speed) / 100) {
 		case 1:
@@ -1365,7 +1365,7 @@ static void ibmvfc_get_host_speed(struct Scsi_Host *shost)
 		}
 	} else
 		fc_host_speed(shost) = FC_PORTSPEED_UNKNOWN;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -1380,7 +1380,7 @@ static void ibmvfc_get_host_port_state(struct Scsi_Host *shost)
 	struct ibmvfc_host *vhost = shost_priv(shost);
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	switch (vhost->state) {
 	case IBMVFC_INITIALIZING:
 	case IBMVFC_ACTIVE:
@@ -1404,7 +1404,7 @@ static void ibmvfc_get_host_port_state(struct Scsi_Host *shost)
 		fc_host_port_state(shost) = FC_PORTSTATE_UNKNOWN;
 		break;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -1642,9 +1642,9 @@ static void ibmvfc_locked_done(struct ibmvfc_event *evt)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(evt->vhost->host->host_lock, flags);
+	spin_lock_irqsave(&evt->vhost->host->host_lock, flags);
 	evt->_done(evt);
-	spin_unlock_irqrestore(evt->vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&evt->vhost->host->host_lock, flags);
 }
 
 /**
@@ -1905,7 +1905,7 @@ static void ibmvfc_scsi_relogin(struct scsi_device *sdev)
 	struct ibmvfc_target *tgt;
 	unsigned long flags;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	list_for_each_entry(tgt, &vhost->scsi_scrqs.targets, queue) {
 		if (rport == tgt->rport) {
 			ibmvfc_del_tgt(tgt);
@@ -1914,7 +1914,7 @@ static void ibmvfc_scsi_relogin(struct scsi_device *sdev)
 	}
 
 	ibmvfc_reinit_host(vhost);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 }
 
 /**
@@ -2147,17 +2147,17 @@ static int ibmvfc_bsg_timeout(struct bsg_job *job)
 	int rc;
 
 	ENTER;
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	if (vhost->aborting_passthru || vhost->state != IBMVFC_ACTIVE) {
 		__ibmvfc_reset_host(vhost);
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return 0;
 	}
 
 	vhost->aborting_passthru = 1;
 	evt = ibmvfc_get_reserved_event(&vhost->crq);
 	if (!evt) {
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return -ENOMEM;
 	}
 
@@ -2181,7 +2181,7 @@ static int ibmvfc_bsg_timeout(struct bsg_job *job)
 		dev_info(vhost->dev, "Cancelling passthru command to port id 0x%lx\n",
 			 port_id);
 
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	LEAVE;
 	return rc;
@@ -2205,7 +2205,7 @@ static int ibmvfc_bsg_plogi(struct ibmvfc_host *vhost, unsigned int port_id)
 	int rc = 0, issue_login = 1;
 
 	ENTER;
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	list_for_each_entry(tgt, &vhost->scsi_scrqs.targets, queue) {
 		if (tgt->scsi_id == port_id) {
 			issue_login = 0;
@@ -2234,7 +2234,7 @@ static int ibmvfc_bsg_plogi(struct ibmvfc_host *vhost, unsigned int port_id)
 	init_completion(&evt->comp);
 
 	rc = ibmvfc_send_event(evt, vhost, default_timeout);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	if (rc)
 		return -EIO;
@@ -2244,10 +2244,10 @@ static int ibmvfc_bsg_plogi(struct ibmvfc_host *vhost, unsigned int port_id)
 	if (rsp_iu.plogi.common.status)
 		rc = -EIO;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_free_event(evt);
 unlock_out:
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	LEAVE;
 	return rc;
 }
@@ -2332,17 +2332,17 @@ static int ibmvfc_bsg_request(struct bsg_job *job)
 	if (issue_login)
 		rc = ibmvfc_bsg_plogi(vhost, port_id);
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 
 	if (unlikely(rc || (rport && (rc = fc_remote_port_chkready(rport)))) ||
 	    unlikely((rc = ibmvfc_host_chkready(vhost)))) {
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		goto out;
 	}
 
 	evt = ibmvfc_get_reserved_event(&vhost->crq);
 	if (!evt) {
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		rc = -ENOMEM;
 		goto out;
 	}
@@ -2374,7 +2374,7 @@ static int ibmvfc_bsg_request(struct bsg_job *job)
 	evt->sync_iu = &rsp_iu;
 	init_completion(&evt->comp);
 	rc = ibmvfc_send_event(evt, vhost, 0);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	if (rc) {
 		rc = -EIO;
@@ -2388,9 +2388,9 @@ static int ibmvfc_bsg_request(struct bsg_job *job)
 	else
 		bsg_reply->reply_payload_rcv_len = rsp_len;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_free_event(evt);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	bsg_reply->result = rc;
 	bsg_job_done(job, bsg_reply->result,
 		       bsg_reply->reply_payload_rcv_len);
@@ -2427,7 +2427,7 @@ static int ibmvfc_reset_device(struct scsi_device *sdev, int type, char *desc)
 	unsigned long flags;
 	int rsp_code = 0;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	if (vhost->state == IBMVFC_ACTIVE) {
 		if (vhost->using_channels)
 			evt = ibmvfc_get_event(&vhost->scsi_scrqs.scrqs[0]);
@@ -2435,7 +2435,7 @@ static int ibmvfc_reset_device(struct scsi_device *sdev, int type, char *desc)
 			evt = ibmvfc_get_event(&vhost->crq);
 
 		if (!evt) {
-			spin_unlock_irqrestore(vhost->host->host_lock, flags);
+			spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 			return -ENOMEM;
 		}
 
@@ -2452,7 +2452,7 @@ static int ibmvfc_reset_device(struct scsi_device *sdev, int type, char *desc)
 		init_completion(&evt->comp);
 		rsp_rc = ibmvfc_send_event(evt, vhost, default_timeout);
 	}
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	if (rsp_rc != 0) {
 		sdev_printk(KERN_ERR, sdev, "Failed to send %s reset event. rc=%d\n",
@@ -2479,9 +2479,9 @@ static int ibmvfc_reset_device(struct scsi_device *sdev, int type, char *desc)
 	} else
 		sdev_printk(KERN_INFO, sdev, "%s reset successful\n", desc);
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_free_event(evt);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	return rsp_rc;
 }
 
@@ -2583,7 +2583,7 @@ static int ibmvfc_wait_for_ops(struct ibmvfc_host *vhost, void *device,
 
 	do {
 		wait = 0;
-		spin_lock_irqsave(vhost->host->host_lock, flags);
+		spin_lock_irqsave(&vhost->host->host_lock, flags);
 		for (q_index = 0; q_index < q_size; q_index++) {
 			spin_lock(&queues[q_index].l_lock);
 			for (i = 0; i < queues[q_index].evt_pool.size; i++) {
@@ -2597,14 +2597,14 @@ static int ibmvfc_wait_for_ops(struct ibmvfc_host *vhost, void *device,
 			}
 			spin_unlock(&queues[q_index].l_lock);
 		}
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 		if (wait) {
 			timeout = wait_for_completion_timeout(&comp, timeout);
 
 			if (!timeout) {
 				wait = 0;
-				spin_lock_irqsave(vhost->host->host_lock, flags);
+				spin_lock_irqsave(&vhost->host->host_lock, flags);
 				for (q_index = 0; q_index < q_size; q_index++) {
 					spin_lock(&queues[q_index].l_lock);
 					for (i = 0; i < queues[q_index].evt_pool.size; i++) {
@@ -2618,7 +2618,7 @@ static int ibmvfc_wait_for_ops(struct ibmvfc_host *vhost, void *device,
 					}
 					spin_unlock(&queues[q_index].l_lock);
 				}
-				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				if (wait)
 					dev_err(vhost->dev, "Timed out waiting for aborted commands\n");
 				LEAVE;
@@ -2684,7 +2684,7 @@ static int ibmvfc_cancel_all_mq(struct scsi_device *sdev, int type)
 	u16 status;
 
 	ENTER;
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	num_hwq = vhost->scsi_scrqs.active_queues;
 	for (i = 0; i < num_hwq; i++) {
 		spin_lock(queues[i].q_lock);
@@ -2702,7 +2702,7 @@ static int ibmvfc_cancel_all_mq(struct scsi_device *sdev, int type)
 			evt = ibmvfc_init_tmf(&queues[i], sdev, type);
 			if (!evt) {
 				spin_unlock(queues[i].q_lock);
-				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				return -ENOMEM;
 			}
 			evt->sync_iu = &queues[i].cancel_rsp;
@@ -2712,7 +2712,7 @@ static int ibmvfc_cancel_all_mq(struct scsi_device *sdev, int type)
 
 		spin_unlock(queues[i].q_lock);
 	}
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	if (list_empty(&cancelq)) {
 		if (vhost->log_level > IBMVFC_DEFAULT_LOG_LEVEL)
@@ -2763,7 +2763,7 @@ static int ibmvfc_cancel_all_sq(struct scsi_device *sdev, int type)
 
 	ENTER;
 	found_evt = NULL;
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	spin_lock(&vhost->crq.l_lock);
 	list_for_each_entry(evt, &vhost->crq.sent, queue_list) {
 		if (evt->cmnd && evt->cmnd->device == sdev) {
@@ -2776,7 +2776,7 @@ static int ibmvfc_cancel_all_sq(struct scsi_device *sdev, int type)
 	if (!found_evt) {
 		if (vhost->log_level > IBMVFC_DEFAULT_LOG_LEVEL)
 			sdev_printk(KERN_INFO, sdev, "No events found to cancel\n");
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return 0;
 	}
 
@@ -2786,7 +2786,7 @@ static int ibmvfc_cancel_all_sq(struct scsi_device *sdev, int type)
 		rsp_rc = ibmvfc_send_event(evt, vhost, default_timeout);
 	}
 
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	if (rsp_rc != 0) {
 		sdev_printk(KERN_ERR, sdev, "Failed to send cancel event. rc=%d\n", rsp_rc);
@@ -2800,9 +2800,9 @@ static int ibmvfc_cancel_all_sq(struct scsi_device *sdev, int type)
 
 	wait_for_completion(&evt->comp);
 	status = be16_to_cpu(rsp.mad_common.status);
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_free_event(evt);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	if (status != IBMVFC_MAD_SUCCESS) {
 		sdev_printk(KERN_WARNING, sdev, "Cancel failed with rc=%x\n", status);
@@ -2899,7 +2899,7 @@ static int ibmvfc_abort_task_set(struct scsi_device *sdev)
 	int rsp_code = 0;
 
 	found_evt = NULL;
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	spin_lock(&vhost->crq.l_lock);
 	list_for_each_entry(evt, &vhost->crq.sent, queue_list) {
 		if (evt->cmnd && evt->cmnd->device == sdev) {
@@ -2912,14 +2912,14 @@ static int ibmvfc_abort_task_set(struct scsi_device *sdev)
 	if (!found_evt) {
 		if (vhost->log_level > IBMVFC_DEFAULT_LOG_LEVEL)
 			sdev_printk(KERN_INFO, sdev, "No events found to abort\n");
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return 0;
 	}
 
 	if (vhost->state == IBMVFC_ACTIVE) {
 		evt = ibmvfc_get_event(&vhost->crq);
 		if (!evt) {
-			spin_unlock_irqrestore(vhost->host->host_lock, flags);
+			spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 			return -ENOMEM;
 		}
 		ibmvfc_init_event(evt, ibmvfc_sync_completion, IBMVFC_CMD_FORMAT);
@@ -2938,7 +2938,7 @@ static int ibmvfc_abort_task_set(struct scsi_device *sdev)
 		rsp_rc = ibmvfc_send_event(evt, vhost, default_timeout);
 	}
 
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	if (rsp_rc != 0) {
 		sdev_printk(KERN_ERR, sdev, "Failed to send abort. rc=%d\n", rsp_rc);
@@ -2967,9 +2967,9 @@ static int ibmvfc_abort_task_set(struct scsi_device *sdev)
 
 			rc = ibmvfc_wait_for_ops(vhost, evt, ibmvfc_match_evt);
 			if (rc != SUCCESS) {
-				spin_lock_irqsave(vhost->host->host_lock, flags);
+				spin_lock_irqsave(&vhost->host->host_lock, flags);
 				ibmvfc_hard_reset_host(vhost);
-				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				rsp_rc = 0;
 			}
 
@@ -2994,9 +2994,9 @@ static int ibmvfc_abort_task_set(struct scsi_device *sdev)
 		sdev_printk(KERN_INFO, sdev, "Abort successful\n");
 
 out:
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_free_event(evt);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	return rsp_rc;
 }
 
@@ -3176,7 +3176,7 @@ static void ibmvfc_terminate_rport_io(struct fc_rport *rport)
 	if (rc == FAILED)
 		ibmvfc_issue_fc_host_lip(shost);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	found = 0;
 	list_for_each_entry(tgt, &vhost->scsi_scrqs.targets, queue) {
 		if (tgt->scsi_id == rport->port_id) {
@@ -3195,7 +3195,7 @@ static void ibmvfc_terminate_rport_io(struct fc_rport *rport)
 		ibmvfc_reinit_host(vhost);
 	}
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	LEAVE;
 }
 
@@ -3468,7 +3468,7 @@ static int ibmvfc_scan_finished(struct Scsi_Host *shost, unsigned long time)
 	struct ibmvfc_host *vhost = shost_priv(shost);
 	int done = 0;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (!vhost->scan_timeout)
 		done = 1;
 	else if (time >= (vhost->scan_timeout * HZ)) {
@@ -3481,7 +3481,7 @@ static int ibmvfc_scan_finished(struct Scsi_Host *shost, unsigned long time)
 		vhost->scan_timeout = init_timeout;
 		done = 1;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return done;
 }
 
@@ -3505,9 +3505,9 @@ static int ibmvfc_sdev_init(struct scsi_device *sdev)
 	if (!rport || fc_remote_port_chkready(rport))
 		return -ENXIO;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	sdev->hostdata = (void *)(unsigned long)vhost->task_set++;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return 0;
 }
 
@@ -3527,9 +3527,9 @@ static int ibmvfc_target_alloc(struct scsi_target *starget)
 	struct ibmvfc_host *vhost = shost_priv(shost);
 	unsigned long flags = 0;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	starget->hostdata = (void *)(unsigned long)vhost->task_set++;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return 0;
 }
 
@@ -3550,12 +3550,12 @@ static int ibmvfc_sdev_configure(struct scsi_device *sdev,
 	struct Scsi_Host *shost = sdev->host;
 	unsigned long flags = 0;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (sdev->type == TYPE_DISK) {
 		sdev->allow_restart = 1;
 		blk_queue_rq_timeout(sdev->request_queue, 120 * HZ);
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return 0;
 }
 
@@ -3646,9 +3646,9 @@ static ssize_t ibmvfc_show_log_level(struct device *dev,
 	unsigned long flags = 0;
 	int len;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	len = sysfs_emit(buf, "%d\n", vhost->log_level);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return len;
 }
 
@@ -3670,9 +3670,9 @@ static ssize_t ibmvfc_store_log_level(struct device *dev,
 	struct ibmvfc_host *vhost = shost_priv(shost);
 	unsigned long flags = 0;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	vhost->log_level = simple_strtoul(buf, NULL, 10);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return strlen(buf);
 }
 
@@ -3685,9 +3685,9 @@ static ssize_t ibmvfc_show_scsi_channels(struct device *dev,
 	unsigned long flags = 0;
 	int len;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	len = sysfs_emit(buf, "%d\n", scsi->desired_queues);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return len;
 }
 
@@ -3701,11 +3701,11 @@ static ssize_t ibmvfc_store_scsi_channels(struct device *dev,
 	unsigned long flags = 0;
 	unsigned int channels;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	channels = simple_strtoul(buf, NULL, 10);
 	scsi->desired_queues = min(channels, shost->nr_hw_queues);
 	ibmvfc_hard_reset_host(vhost);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return strlen(buf);
 }
 
@@ -3751,9 +3751,9 @@ static ssize_t ibmvfc_read_trace(struct file *filp, struct kobject *kobj,
 		count = size;
 	}
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	memcpy(buf, &src[off], count);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return count;
 }
 
@@ -3864,10 +3864,10 @@ static irqreturn_t ibmvfc_interrupt(int irq, void *dev_instance)
 	struct ibmvfc_host *vhost = (struct ibmvfc_host *)dev_instance;
 	unsigned long flags;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	vio_disable_interrupts(to_vio_dev(vhost->dev));
 	tasklet_schedule(&vhost->tasklet);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	return IRQ_HANDLED;
 }
 
@@ -3889,7 +3889,7 @@ static void ibmvfc_tasklet(void *data)
 	int done = 0;
 	LIST_HEAD(evt_doneq);
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	spin_lock(vhost->crq.q_lock);
 	while (!done) {
 		/* Pull all the valid messages off the async CRQ */
@@ -3922,7 +3922,7 @@ static void ibmvfc_tasklet(void *data)
 	}
 
 	spin_unlock(vhost->crq.q_lock);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	list_for_each_entry_safe(evt, temp, &evt_doneq, queue_list) {
 		timer_delete(&evt->timer);
@@ -4776,12 +4776,12 @@ static void ibmvfc_adisc_timeout(struct timer_list *t)
 	int rc;
 
 	tgt_dbg(tgt, "ADISC timeout\n");
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	if (vhost->abort_threads >= disc_threads ||
 	    tgt->action != IBMVFC_TGT_ACTION_INIT_WAIT ||
 	    vhost->state != IBMVFC_INITIALIZING ||
 	    vhost->action != IBMVFC_HOST_ACTION_QUERY_TGTS) {
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return;
 	}
 
@@ -4793,7 +4793,7 @@ static void ibmvfc_adisc_timeout(struct timer_list *t)
 		vhost->abort_threads--;
 		kref_put(&tgt->kref, ibmvfc_release_tgt);
 		__ibmvfc_reset_host(vhost);
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return;
 	}
 	ibmvfc_init_event(evt, ibmvfc_tgt_adisc_cancel_done, IBMVFC_MAD_FORMAT);
@@ -4821,7 +4821,7 @@ static void ibmvfc_adisc_timeout(struct timer_list *t)
 		__ibmvfc_reset_host(vhost);
 	} else
 		tgt_dbg(tgt, "Attempting to cancel ADISC\n");
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 }
 
 /**
@@ -5010,7 +5010,7 @@ static int ibmvfc_alloc_target(struct ibmvfc_host *vhost,
 		channels = &vhost->nvme_scrqs;
 
 	/* Look to see if we already have a target allocated for this SCSI ID or WWPN */
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	list_for_each_entry(tgt, &channels->targets, queue) {
 		if (tgt->wwpn == wwpn) {
 			wtgt = tgt;
@@ -5062,7 +5062,7 @@ static int ibmvfc_alloc_target(struct ibmvfc_host *vhost,
 			ibmvfc_init_tgt(tgt, ibmvfc_tgt_implicit_logout);
 		goto unlock_out;
 	}
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	tgt = mempool_alloc(vhost->tgt_pool, GFP_NOIO);
 	memset(tgt, 0, sizeof(*tgt));
@@ -5074,12 +5074,12 @@ static int ibmvfc_alloc_target(struct ibmvfc_host *vhost,
 	timer_setup(&tgt->timer, ibmvfc_adisc_timeout, 0);
 	kref_init(&tgt->kref);
 	ibmvfc_init_tgt(tgt, ibmvfc_tgt_implicit_logout);
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	tgt->cancel_key = vhost->task_set++;
 	list_add_tail(&tgt->queue, &channels->targets);
 
 unlock_out:
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	return 0;
 }
 
@@ -5853,9 +5853,9 @@ static int ibmvfc_work_to_do(struct ibmvfc_host *vhost)
 	unsigned long flags;
 	int rc;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	rc = __ibmvfc_work_to_do(vhost);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	return rc;
 }
 
@@ -5890,14 +5890,14 @@ static void ibmvfc_tgt_add_nvme_rport(struct ibmvfc_target *tgt)
 
 	tgt_dbg(tgt, "Adding NVMe rport\n");
 	ibmvfc_nvme_register_remoteport(tgt);
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	rport = tgt->nvme_remote_port;
 
 	if (rport && tgt->action == IBMVFC_TGT_ACTION_DEL_RPORT) {
 		tgt_dbg(tgt, "Deleting NVMe rport\n");
 		list_del(&tgt->queue);
 		ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_DELETED_RPORT);
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		ibmvfc_nvme_unregister_remoteport(tgt);
 		timer_delete_sync(&tgt->timer);
 		kref_put(&tgt->kref, ibmvfc_release_tgt);
@@ -5906,11 +5906,11 @@ static void ibmvfc_tgt_add_nvme_rport(struct ibmvfc_target *tgt)
 		tgt_dbg(tgt, "Deleting NVMe rport with outstanding I/O\n");
 		ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_LOGOUT_DELETED_RPORT);
 		tgt->init_retries = 0;
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		ibmvfc_nvme_unregister_remoteport(tgt);
 		return;
 	} else if (rport && tgt->action == IBMVFC_TGT_ACTION_DELETED_RPORT) {
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return;
 	}
 
@@ -5919,7 +5919,7 @@ static void ibmvfc_tgt_add_nvme_rport(struct ibmvfc_target *tgt)
 		tgt->target_id = tgt->nvme_remote_port->port_id;
 	} else
 		tgt_dbg(tgt, "NVMe rport add failed\n");
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 }
 
 /**
@@ -5935,13 +5935,13 @@ static void ibmvfc_tgt_add_rport(struct ibmvfc_target *tgt)
 
 	tgt_dbg(tgt, "Adding rport\n");
 	rport = fc_remote_port_add(vhost->host, 0, &tgt->ids);
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 
 	if (rport && tgt->action == IBMVFC_TGT_ACTION_DEL_RPORT) {
 		tgt_dbg(tgt, "Deleting rport\n");
 		list_del(&tgt->queue);
 		ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_DELETED_RPORT);
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		fc_remote_port_delete(rport);
 		timer_delete_sync(&tgt->timer);
 		kref_put(&tgt->kref, ibmvfc_release_tgt);
@@ -5951,11 +5951,11 @@ static void ibmvfc_tgt_add_rport(struct ibmvfc_target *tgt)
 		ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_LOGOUT_DELETED_RPORT);
 		tgt->rport = NULL;
 		tgt->init_retries = 0;
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		fc_remote_port_delete(rport);
 		return;
 	} else if (rport && tgt->action == IBMVFC_TGT_ACTION_DELETED_RPORT) {
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		return;
 	}
 
@@ -5973,7 +5973,7 @@ static void ibmvfc_tgt_add_rport(struct ibmvfc_target *tgt)
 			rport->supported_classes |= FC_COS_CLASS3;
 	} else
 		tgt_dbg(tgt, "rport add failed\n");
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 }
 
 /**
@@ -5991,7 +5991,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 	int rc;
 
 	ibmvfc_log_ae(vhost, vhost->events_to_log);
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	vhost->events_to_log = 0;
 	switch (vhost->action) {
 	case IBMVFC_HOST_ACTION_NONE:
@@ -6000,12 +6000,12 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 		break;
 	case IBMVFC_HOST_ACTION_RESET:
 		list_splice_init(&vhost->purge, &purge);
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		ibmvfc_complete_purge(&purge);
 		ibmvfc_nvme_unregister(vhost);
 		rc = ibmvfc_reset_crq(vhost);
 
-		spin_lock_irqsave(vhost->host->host_lock, flags);
+		spin_lock_irqsave(&vhost->host->host_lock, flags);
 		if (!rc || rc == H_CLOSED)
 			vio_enable_interrupts(to_vio_dev(vhost->dev));
 		if (vhost->action == IBMVFC_HOST_ACTION_RESET) {
@@ -6026,11 +6026,11 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 		break;
 	case IBMVFC_HOST_ACTION_REENABLE:
 		list_splice_init(&vhost->purge, &purge);
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		ibmvfc_complete_purge(&purge);
 		rc = ibmvfc_reenable_crq_queue(vhost);
 
-		spin_lock_irqsave(vhost->host->host_lock, flags);
+		spin_lock_irqsave(&vhost->host->host_lock, flags);
 		if (vhost->action == IBMVFC_HOST_ACTION_REENABLE) {
 			/*
 			 * The only action we could have changed to would have
@@ -6052,7 +6052,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 		BUG_ON(vhost->state != IBMVFC_INITIALIZING);
 		if (vhost->delay_init) {
 			vhost->delay_init = 0;
-			spin_unlock_irqrestore(vhost->host->host_lock, flags);
+			spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 			ssleep(15);
 			return;
 		} else
@@ -6098,7 +6098,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 		}
 
 		if (ibmvfc_dev_logo_to_do(vhost)) {
-			spin_unlock_irqrestore(vhost->host->host_lock, flags);
+			spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 			return;
 		}
 
@@ -6109,7 +6109,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 				tgt->rport = NULL;
 				list_del(&tgt->queue);
 				ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_DELETED_RPORT);
-				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				if (rport)
 					fc_remote_port_delete(rport);
 				timer_delete_sync(&tgt->timer);
@@ -6133,7 +6133,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 				 */
 				if (rport && rport->fast_io_fail_tmo == -1)
 					tgt->move_login = 1;
-				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				if (rport)
 					fc_remote_port_delete(rport);
 				return;
@@ -6146,7 +6146,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 				nvme_rport = tgt->nvme_remote_port;
 				list_del(&tgt->queue);
 				ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_DELETED_RPORT);
-				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				if (nvme_rport)
 					ibmvfc_nvme_unregister_remoteport(tgt);
 				timer_delete_sync(&tgt->timer);
@@ -6157,7 +6157,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 				nvme_rport = tgt->nvme_remote_port;
 				ibmvfc_set_tgt_action(tgt, IBMVFC_TGT_ACTION_LOGOUT_DELETED_RPORT);
 				tgt->init_retries = 0;
-				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				if (nvme_rport)
 					ibmvfc_nvme_unregister_remoteport(tgt);
 				return;
@@ -6170,14 +6170,14 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 					vhost->reinit = 0;
 					scsi_block_requests(vhost->host);
 					ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_QUERY);
-					spin_unlock_irqrestore(vhost->host->host_lock, flags);
+					spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				} else {
 					ibmvfc_set_host_state(vhost, IBMVFC_ACTIVE);
 					ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_NONE);
 					wake_up(&vhost->init_wait_q);
 					schedule_work(&vhost->rport_add_work_q);
 					vhost->init_retries = 0;
-					spin_unlock_irqrestore(vhost->host->host_lock, flags);
+					spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 					scsi_unblock_requests(vhost->host);
 				}
 
@@ -6188,7 +6188,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 			}
 		} else {
 			ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_NONE);
-			spin_unlock_irqrestore(vhost->host->host_lock, flags);
+			spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 			scsi_unblock_requests(vhost->host);
 			wake_up(&vhost->init_wait_q);
 			return;
@@ -6196,9 +6196,9 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 		break;
 	case IBMVFC_HOST_ACTION_ALLOC_TGTS:
 		ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_TGT_INIT);
-		spin_unlock_irqrestore(vhost->host->host_lock, flags);
+		spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 		ibmvfc_alloc_targets(vhost);
-		spin_lock_irqsave(vhost->host->host_lock, flags);
+		spin_lock_irqsave(&vhost->host->host_lock, flags);
 		break;
 	case IBMVFC_HOST_ACTION_TGT_INIT:
 		list_for_each_entry(tgt, &vhost->scsi_scrqs.targets, queue) {
@@ -6221,7 +6221,7 @@ static void ibmvfc_do_work(struct ibmvfc_host *vhost)
 		break;
 	}
 
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 }
 
 /**
@@ -6754,7 +6754,7 @@ static void ibmvfc_rport_add_thread(struct work_struct *work)
 	int did_work;
 
 	ENTER;
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	do {
 		did_work = 0;
 		if (vhost->state != IBMVFC_ACTIVE)
@@ -6767,19 +6767,19 @@ static void ibmvfc_rport_add_thread(struct work_struct *work)
 				kref_get(&tgt->kref);
 				rport = tgt->rport;
 				if (!rport) {
-					spin_unlock_irqrestore(vhost->host->host_lock, flags);
+					spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 					ibmvfc_tgt_add_rport(tgt);
 				} else if (get_device(&rport->dev)) {
-					spin_unlock_irqrestore(vhost->host->host_lock, flags);
+					spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 					tgt_dbg(tgt, "Setting rport roles\n");
 					fc_remote_port_rolechg(rport, tgt->ids.roles);
 					put_device(&rport->dev);
 				} else {
-					spin_unlock_irqrestore(vhost->host->host_lock, flags);
+					spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 				}
 
 				kref_put(&tgt->kref, ibmvfc_release_tgt);
-				spin_lock_irqsave(vhost->host->host_lock, flags);
+				spin_lock_irqsave(&vhost->host->host_lock, flags);
 				break;
 			}
 		}
@@ -6791,16 +6791,16 @@ static void ibmvfc_rport_add_thread(struct work_struct *work)
 				kref_get(&tgt->kref);
 				nvme_rport = tgt->nvme_remote_port;
 				if (!nvme_rport) {
-					spin_unlock_irqrestore(vhost->host->host_lock, flags);
+					spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 					ibmvfc_tgt_add_nvme_rport(tgt);
 				} else {
-					spin_unlock_irqrestore(vhost->host->host_lock, flags);
+					spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 					if (IS_ENABLED(CONFIG_NVME_FC))
 						nvme_fc_rescan_remoteport(nvme_rport);
 				}
 
 				kref_put(&tgt->kref, ibmvfc_release_tgt);
-				spin_lock_irqsave(vhost->host->host_lock, flags);
+				spin_lock_irqsave(&vhost->host->host_lock, flags);
 				break;
 			}
 		}
@@ -6808,7 +6808,7 @@ static void ibmvfc_rport_add_thread(struct work_struct *work)
 
 	if (vhost->state == IBMVFC_ACTIVE)
 		vhost->scan_complete = 1;
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	LEAVE;
 }
 
@@ -6946,9 +6946,9 @@ static void ibmvfc_remove(struct vio_dev *vdev)
 	ENTER;
 	ibmvfc_remove_trace_file(&vhost->host->shost_dev.kobj, &ibmvfc_trace_attr);
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_link_down(vhost, IBMVFC_HOST_OFFLINE);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	ibmvfc_wait_while_resetting(vhost);
 	kthread_stop(vhost->work_thread);
@@ -6956,10 +6956,10 @@ static void ibmvfc_remove(struct vio_dev *vdev)
 	fc_remove_host(vhost->host);
 	scsi_remove_host(vhost->host);
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_purge_requests(vhost, DID_ERROR);
 	list_splice_init(&vhost->purge, &purge);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	ibmvfc_complete_purge(&purge);
 	ibmvfc_release_sub_crqs(vhost);
 	ibmvfc_release_crq_queue(vhost);
@@ -6986,10 +6986,10 @@ static int ibmvfc_resume(struct device *dev)
 	struct ibmvfc_host *vhost = dev_get_drvdata(dev);
 	struct vio_dev *vdev = to_vio_dev(dev);
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	vio_disable_interrupts(vdev);
 	tasklet_schedule(&vhost->tasklet);
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	return 0;
 }
 
diff --git a/drivers/scsi/ibmvscsi/ibmvfc-nvme.c b/drivers/scsi/ibmvscsi/ibmvfc-nvme.c
index 54f80b6265d455..9ac2671a3567db 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc-nvme.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc-nvme.c
@@ -208,7 +208,7 @@ static void ibmvfc_nvme_ls_abort(struct nvme_fc_local_port *lport,
 	if (!vhost->logged_in || !evt)
 		return;
 
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	kref_get(&tgt->kref);
 	ibmvfc_init_event(evt, ibmvfc_sync_nvme_completion, IBMVFC_MAD_FORMAT);
 	ibmvfc_init_ls_abort(evt, ls_abort);
@@ -217,14 +217,14 @@ static void ibmvfc_nvme_ls_abort(struct nvme_fc_local_port *lport,
 	if (ibmvfc_send_event(evt, vhost, default_timeout))
 		goto out;
 
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 
 	wait_for_completion(&evt->comp);
 	status = be16_to_cpu(rsp.mad_common.status);
-	spin_lock_irqsave(vhost->host->host_lock, flags);
+	spin_lock_irqsave(&vhost->host->host_lock, flags);
 	ibmvfc_free_event(evt);
 out:
-	spin_unlock_irqrestore(vhost->host->host_lock, flags);
+	spin_unlock_irqrestore(&vhost->host->host_lock, flags);
 	ibmvfc_dbg(vhost, "ls_abort: cancel failed with rc=%x\n", status);
 	kref_put(&tgt->kref, ibmvfc_release_tgt);
 }
diff --git a/drivers/scsi/ibmvscsi/ibmvscsi.c b/drivers/scsi/ibmvscsi/ibmvscsi.c
index 609bda730b3a89..4ecd3db088751d 100644
--- a/drivers/scsi/ibmvscsi/ibmvscsi.c
+++ b/drivers/scsi/ibmvscsi/ibmvscsi.c
@@ -785,13 +785,13 @@ static void purge_requests(struct ibmvscsi_host_data *hostdata, int error_code)
 	struct srp_event_struct *evt;
 	unsigned long flags;
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	while (!list_empty(&hostdata->sent)) {
 		evt = list_first_entry(&hostdata->sent, struct srp_event_struct, list);
 		list_del(&evt->list);
 		timer_delete(&evt->timer);
 
-		spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+		spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 		if (evt->cmnd) {
 			evt->cmnd->result = (error_code << 16);
 			unmap_cmd_data(&evt->iu.srp.cmd, evt,
@@ -802,9 +802,9 @@ static void purge_requests(struct ibmvscsi_host_data *hostdata, int error_code)
 			   evt->iu.srp.login_req.opcode != SRP_LOGIN_REQ)
 			evt->done(evt);
 		free_event_struct(&evt->hostdata->pool, evt);
-		spin_lock_irqsave(hostdata->host->host_lock, flags);
+		spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	}
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 }
 
 /**
@@ -818,9 +818,9 @@ static void ibmvscsi_set_request_limit(struct ibmvscsi_host_data *hostdata, int
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	atomic_set(&hostdata->request_limit, limit);
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 }
 
 /**
@@ -1221,9 +1221,9 @@ static int send_srp_login(struct ibmvscsi_host_data *hostdata)
 	 */
 	ibmvscsi_set_request_limit(hostdata, 0);
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	rc = ibmvscsi_send_srp_event(evt_struct, hostdata, login_timeout * 2);
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 	dev_info(hostdata->dev, "sent SRP login\n");
 	return rc;
 };
@@ -1317,10 +1317,10 @@ static void send_mad_capabilities(struct ibmvscsi_host_data *hostdata)
 		req->common.length = cpu_to_be16(sizeof(hostdata->caps) -
 						sizeof(hostdata->caps.reserve));
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	if (ibmvscsi_send_srp_event(evt_struct, hostdata, info_timeout * 2))
 		dev_err(hostdata->dev, "couldn't send CAPABILITIES_REQ!\n");
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 };
 
 /**
@@ -1373,9 +1373,9 @@ static int enable_fast_fail(struct ibmvscsi_host_data *hostdata)
 	fast_fail_mad->common.type = cpu_to_be32(VIOSRP_ENABLE_FAST_FAIL);
 	fast_fail_mad->common.length = cpu_to_be16(sizeof(*fast_fail_mad));
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	rc = ibmvscsi_send_srp_event(evt_struct, hostdata, info_timeout * 2);
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 	return rc;
 }
 
@@ -1454,10 +1454,10 @@ static void send_mad_adapter_info(struct ibmvscsi_host_data *hostdata)
 	req->common.length = cpu_to_be16(sizeof(hostdata->madapter_info));
 	req->buffer = cpu_to_be64(hostdata->adapter_info_addr);
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	if (ibmvscsi_send_srp_event(evt_struct, hostdata, info_timeout * 2))
 		dev_err(hostdata->dev, "couldn't send ADAPTER_INFO_REQ!\n");
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 };
 
 /*
@@ -1502,7 +1502,7 @@ static int ibmvscsi_eh_abort_handler(struct scsi_cmnd *cmd)
 	/* First, find this command in our sent list so we can figure
 	 * out the correct tag
 	 */
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	wait_switch = jiffies + (init_timeout * HZ);
 	do {
 		found_evt = NULL;
@@ -1514,13 +1514,13 @@ static int ibmvscsi_eh_abort_handler(struct scsi_cmnd *cmd)
 		}
 
 		if (!found_evt) {
-			spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+			spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 			return SUCCESS;
 		}
 
 		evt = get_event_struct(&hostdata->pool);
 		if (evt == NULL) {
-			spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+			spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 			sdev_printk(KERN_ERR, cmd->device,
 				"failed to allocate abort event\n");
 			return FAILED;
@@ -1548,12 +1548,12 @@ static int ibmvscsi_eh_abort_handler(struct scsi_cmnd *cmd)
 		if (rsp_rc != SCSI_MLQUEUE_HOST_BUSY)
 			break;
 
-		spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+		spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 		msleep(10);
-		spin_lock_irqsave(hostdata->host->host_lock, flags);
+		spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	} while (time_before(jiffies, wait_switch));
 
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 
 	if (rsp_rc != 0) {
 		sdev_printk(KERN_ERR, cmd->device,
@@ -1592,7 +1592,7 @@ static int ibmvscsi_eh_abort_handler(struct scsi_cmnd *cmd)
 	 * The event is no longer in our list.  Make sure it didn't
 	 * complete while we were aborting
 	 */
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	found_evt = NULL;
 	list_for_each_entry(tmp_evt, &hostdata->sent, list) {
 		if (tmp_evt->cmnd == cmd) {
@@ -1602,7 +1602,7 @@ static int ibmvscsi_eh_abort_handler(struct scsi_cmnd *cmd)
 	}
 
 	if (found_evt == NULL) {
-		spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+		spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 		sdev_printk(KERN_INFO, cmd->device, "aborted task tag 0x%llx completed\n",
 			    tsk_mgmt->task_tag);
 		return SUCCESS;
@@ -1616,7 +1616,7 @@ static int ibmvscsi_eh_abort_handler(struct scsi_cmnd *cmd)
 	unmap_cmd_data(&found_evt->iu.srp.cmd, found_evt,
 		       found_evt->hostdata->dev);
 	free_event_struct(&found_evt->hostdata->pool, found_evt);
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 	atomic_inc(&hostdata->request_limit);
 	return SUCCESS;
 }
@@ -1638,12 +1638,12 @@ static int ibmvscsi_eh_device_reset_handler(struct scsi_cmnd *cmd)
 	u16 lun = lun_from_dev(cmd->device);
 	unsigned long wait_switch = 0;
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	wait_switch = jiffies + (init_timeout * HZ);
 	do {
 		evt = get_event_struct(&hostdata->pool);
 		if (evt == NULL) {
-			spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+			spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 			sdev_printk(KERN_ERR, cmd->device,
 				"failed to allocate reset event\n");
 			return FAILED;
@@ -1670,12 +1670,12 @@ static int ibmvscsi_eh_device_reset_handler(struct scsi_cmnd *cmd)
 		if (rsp_rc != SCSI_MLQUEUE_HOST_BUSY)
 			break;
 
-		spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+		spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 		msleep(10);
-		spin_lock_irqsave(hostdata->host->host_lock, flags);
+		spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	} while (time_before(jiffies, wait_switch));
 
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 
 	if (rsp_rc != 0) {
 		sdev_printk(KERN_ERR, cmd->device,
@@ -1712,7 +1712,7 @@ static int ibmvscsi_eh_device_reset_handler(struct scsi_cmnd *cmd)
 	/* We need to find all commands for this LUN that have not yet been
 	 * responded to, and fail them with DID_RESET
 	 */
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	list_for_each_entry_safe(tmp_evt, pos, &hostdata->sent, list) {
 		if ((tmp_evt->cmnd) && (tmp_evt->cmnd->device == cmd->device)) {
 			if (tmp_evt->cmnd)
@@ -1729,7 +1729,7 @@ static int ibmvscsi_eh_device_reset_handler(struct scsi_cmnd *cmd)
 				tmp_evt->done(tmp_evt);
 		}
 	}
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 	return SUCCESS;
 }
 
@@ -1855,10 +1855,10 @@ static void ibmvscsi_handle_crq(struct viosrp_crq *crq,
 	 * Lock the host_lock before messing with these structures, since we
 	 * are running in a task context
 	 */
-	spin_lock_irqsave(evt_struct->hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&evt_struct->hostdata->host->host_lock, flags);
 	list_del(&evt_struct->list);
 	free_event_struct(&evt_struct->hostdata->pool, evt_struct);
-	spin_unlock_irqrestore(evt_struct->hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&evt_struct->hostdata->host->host_lock, flags);
 }
 
 /**
@@ -1876,12 +1876,12 @@ static int ibmvscsi_sdev_configure(struct scsi_device *sdev,
 	struct Scsi_Host *shost = sdev->host;
 	unsigned long lock_flags = 0;
 
-	spin_lock_irqsave(shost->host_lock, lock_flags);
+	spin_lock_irqsave(&shost->host_lock, lock_flags);
 	if (sdev->type == TYPE_DISK) {
 		sdev->allow_restart = 1;
 		blk_queue_rq_timeout(sdev->request_queue, 120 * HZ);
 	}
-	spin_unlock_irqrestore(shost->host_lock, lock_flags);
+	spin_unlock_irqrestore(&shost->host_lock, lock_flags);
 	return 0;
 }
 
@@ -2131,35 +2131,35 @@ static void ibmvscsi_do_work(struct ibmvscsi_host_data *hostdata)
 	int rc;
 	char *action = "reset";
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	switch (hostdata->action) {
 	case IBMVSCSI_HOST_ACTION_UNBLOCK:
 		rc = 0;
 		break;
 	case IBMVSCSI_HOST_ACTION_RESET:
-		spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+		spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 		rc = ibmvscsi_reset_crq_queue(&hostdata->queue, hostdata);
-		spin_lock_irqsave(hostdata->host->host_lock, flags);
+		spin_lock_irqsave(&hostdata->host->host_lock, flags);
 		if (!rc)
 			rc = ibmvscsi_send_crq(hostdata, 0xC001000000000000LL, 0);
 		vio_enable_interrupts(to_vio_dev(hostdata->dev));
 		break;
 	case IBMVSCSI_HOST_ACTION_REENABLE:
 		action = "enable";
-		spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+		spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 		rc = ibmvscsi_reenable_crq_queue(&hostdata->queue, hostdata);
-		spin_lock_irqsave(hostdata->host->host_lock, flags);
+		spin_lock_irqsave(&hostdata->host->host_lock, flags);
 		if (!rc)
 			rc = ibmvscsi_send_crq(hostdata, 0xC001000000000000LL, 0);
 		break;
 	case IBMVSCSI_HOST_ACTION_NONE:
 	default:
-		spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+		spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 		return;
 	}
 
 	hostdata->action = IBMVSCSI_HOST_ACTION_NONE;
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 
 	if (rc) {
 		ibmvscsi_set_request_limit(hostdata, -1);
@@ -2191,9 +2191,9 @@ static int ibmvscsi_work_to_do(struct ibmvscsi_host_data *hostdata)
 	unsigned long flags;
 	int rc;
 
-	spin_lock_irqsave(hostdata->host->host_lock, flags);
+	spin_lock_irqsave(&hostdata->host->host_lock, flags);
 	rc = __ibmvscsi_work_to_do(hostdata);
-	spin_unlock_irqrestore(hostdata->host->host_lock, flags);
+	spin_unlock_irqrestore(&hostdata->host->host_lock, flags);
 
 	return rc;
 }
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index 61f68280076543..e2af90b6da39b8 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -3757,9 +3757,8 @@ static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd)
 		dev_err(&vscsi->dev, "srp_transfer_data failed: %d\n", rc);
 		se_cmd->scsi_sense_length = 18;
 		memset(se_cmd->sense_buffer, 0, se_cmd->scsi_sense_length);
-		/* Logical Unit Communication Time-out asc/ascq = 0x0801 */
-		scsi_build_sense_buffer(0, se_cmd->sense_buffer, MEDIUM_ERROR,
-					0x08, 0x01);
+		scsi_set_sense_buffer(0, se_cmd->sense_buffer, MEDIUM_ERROR,
+				      LU_COMMUNICATION_TIMEOUT);
 	}
 
 	srp_build_response(vscsi, cmd, &len);
diff --git a/drivers/scsi/imm.c b/drivers/scsi/imm.c
index 0535252e77e3b4..680399bc866dd1 100644
--- a/drivers/scsi/imm.c
+++ b/drivers/scsi/imm.c
@@ -781,10 +781,10 @@ static void imm_interrupt(struct work_struct *work)
 
 	imm_pb_dismiss(dev);
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	dev->cur_cmd = NULL;
 	scsi_done(cmd);
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 	return;
 }
 
diff --git a/drivers/scsi/initio.c b/drivers/scsi/initio.c
index 06fbe85dccfa63..da2670cd7edf34 100644
--- a/drivers/scsi/initio.c
+++ b/drivers/scsi/initio.c
@@ -2507,9 +2507,9 @@ static irqreturn_t i91u_intr(int irqno, void *dev_id)
 	unsigned long flags;
 	int r;
 	
-	spin_lock_irqsave(dev->host_lock, flags);
+	spin_lock_irqsave(&dev->host_lock, flags);
 	r = initio_isr((struct initio_host *)dev->hostdata);
-	spin_unlock_irqrestore(dev->host_lock, flags);
+	spin_unlock_irqrestore(&dev->host_lock, flags);
 	if (r)
 		return IRQ_HANDLED;
 	else
@@ -2635,9 +2635,9 @@ static int i91u_bus_reset(struct scsi_cmnd * cmnd)
 
 	host = (struct initio_host *) cmnd->device->host->hostdata;
 
-	spin_lock_irq(cmnd->device->host->host_lock);
+	spin_lock_irq(&cmnd->device->host->host_lock);
 	initio_reset_scsi(host, 0);
-	spin_unlock_irq(cmnd->device->host->host_lock);
+	spin_unlock_irq(&cmnd->device->host->host_lock);
 
 	return SUCCESS;
 }
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index d207e5e81afe0a..1c74eec3a0c020 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -612,9 +612,9 @@ static void ipr_lock_and_done(struct ipr_cmnd *ipr_cmd)
 	unsigned long lock_flags;
 	struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	ipr_cmd->done(ipr_cmd);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 }
 
 /**
@@ -1022,9 +1022,9 @@ static void ipr_send_blocking_cmd(struct ipr_cmnd *ipr_cmd,
 	init_completion(&ipr_cmd->completion);
 	ipr_do_req(ipr_cmd, ipr_internal_cmd_done, timeout_func, timeout);
 
-	spin_unlock_irq(ioa_cfg->host->host_lock);
+	spin_unlock_irq(&ioa_cfg->host->host_lock);
 	wait_for_completion(&ipr_cmd->completion);
-	spin_lock_irq(ioa_cfg->host->host_lock);
+	spin_lock_irq(&ioa_cfg->host->host_lock);
 }
 
 static int ipr_get_hrrq_index(struct ipr_ioa_cfg *ioa_cfg)
@@ -2594,7 +2594,7 @@ static void ipr_timeout(struct timer_list *t)
 	struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
 
 	ENTER;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	ioa_cfg->errors_logged++;
 	dev_err(&ioa_cfg->pdev->dev,
@@ -2606,7 +2606,7 @@ static void ipr_timeout(struct timer_list *t)
 	if (!ioa_cfg->in_reset_reload || ioa_cfg->reset_cmd == ipr_cmd)
 		ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	LEAVE;
 }
 
@@ -2627,7 +2627,7 @@ static void ipr_oper_timeout(struct timer_list *t)
 	struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
 
 	ENTER;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	ioa_cfg->errors_logged++;
 	dev_err(&ioa_cfg->pdev->dev,
@@ -2642,7 +2642,7 @@ static void ipr_oper_timeout(struct timer_list *t)
 		ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE);
 	}
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	LEAVE;
 }
 
@@ -2893,7 +2893,7 @@ static int ipr_sdt_copy(struct ipr_ioa_cfg *ioa_cfg,
 	       (ioa_dump->hdr.len + bytes_copied) < max_dump_size) {
 		if (ioa_dump->page_offset >= PAGE_SIZE ||
 		    ioa_dump->page_offset == 0) {
-			page = (__be32 *)__get_free_page(GFP_ATOMIC);
+			page = kmalloc(PAGE_SIZE, GFP_NOIO);
 
 			if (!page) {
 				ipr_trace;
@@ -2910,7 +2910,7 @@ static int ipr_sdt_copy(struct ipr_ioa_cfg *ioa_cfg,
 		rem_page_len = PAGE_SIZE - ioa_dump->page_offset;
 		cur_len = min(rem_len, rem_page_len);
 
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 		if (ioa_cfg->sdt_state == ABORT_DUMP) {
 			rc = -EIO;
 		} else {
@@ -2919,7 +2919,7 @@ static int ipr_sdt_copy(struct ipr_ioa_cfg *ioa_cfg,
 							&page[ioa_dump->page_offset / 4],
 							(cur_len / sizeof(u32)));
 		}
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 		if (!rc) {
 			ioa_dump->page_offset += cur_len;
@@ -3060,17 +3060,17 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump)
 
 	ENTER;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	if (ioa_cfg->sdt_state != READ_DUMP) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return;
 	}
 
 	if (ioa_cfg->sis64) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		ssleep(IPR_DUMP_DELAY_SECONDS);
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	}
 
 	start_addr = readl(ioa_cfg->ioa_mailbox);
@@ -3078,7 +3078,7 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump)
 	if (!ioa_cfg->sis64 && !ipr_sdt_is_fmt2(start_addr)) {
 		dev_err(&ioa_cfg->pdev->dev,
 			"Invalid dump table format: %lx\n", start_addr);
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return;
 	}
 
@@ -3135,7 +3135,7 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump)
 			rc, be32_to_cpu(sdt->hdr.state));
 		driver_dump->hdr.status = IPR_DUMP_STATUS_FAILED;
 		ioa_cfg->sdt_state = DUMP_OBTAINED;
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return;
 	}
 
@@ -3151,7 +3151,7 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump)
 	else
 		dump->driver_dump.hdr.len += max_num_entries * sizeof(struct ipr_sdt_entry);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	for (i = 0; i < num_entries; i++) {
 		if (ioa_dump->hdr.len > max_dump_size) {
@@ -3220,13 +3220,13 @@ static void ipr_release_dump(struct kref *kref)
 	int i;
 
 	ENTER;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	ioa_cfg->dump = NULL;
 	ioa_cfg->sdt_state = INACTIVE;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	for (i = 0; i < dump->ioa_dump.next_page_index; i++)
-		free_page((unsigned long) dump->ioa_dump.ioa_data[i]);
+		kfree(dump->ioa_dump.ioa_data[i]);
 
 	vfree(dump->ioa_dump.ioa_data);
 	kfree(dump);
@@ -3244,13 +3244,13 @@ static void ipr_add_remove_thread(struct work_struct *work)
 	int did_work;
 
 	ENTER;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 restart:
 	do {
 		did_work = 0;
 		if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds) {
-			spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+			spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 			return;
 		}
 
@@ -3263,10 +3263,10 @@ static void ipr_add_remove_thread(struct work_struct *work)
 						list_move_tail(&res->queue, &ioa_cfg->free_res_q);
 					else
 						res->del_from_ml = 0;
-					spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+					spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 					scsi_remove_device(sdev);
 					scsi_device_put(sdev);
-					spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+					spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 				}
 				break;
 			}
@@ -3279,15 +3279,15 @@ static void ipr_add_remove_thread(struct work_struct *work)
 			target = res->target;
 			lun = res->lun;
 			res->add_to_ml = 0;
-			spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+			spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 			scsi_add_device(ioa_cfg->host, bus, target, lun);
-			spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+			spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 			goto restart;
 		}
 	}
 
 	ioa_cfg->scan_done = 1;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	kobject_uevent(&ioa_cfg->host->shost_dev.kobj, KOBJ_CHANGE);
 	LEAVE;
 }
@@ -3311,44 +3311,44 @@ static void ipr_worker_thread(struct work_struct *work)
 		container_of(work, struct ipr_ioa_cfg, work_q);
 
 	ENTER;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	if (ioa_cfg->sdt_state == READ_DUMP) {
 		dump = ioa_cfg->dump;
 		if (!dump) {
-			spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+			spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 			return;
 		}
 		kref_get(&dump->kref);
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		ipr_get_ioa_dump(ioa_cfg, dump);
 		kref_put(&dump->kref, ipr_release_dump);
 
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 		if (ioa_cfg->sdt_state == DUMP_OBTAINED && !ioa_cfg->dump_timeout)
 			ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE);
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return;
 	}
 
 	if (ioa_cfg->scsi_unblock) {
 		ioa_cfg->scsi_unblock = 0;
 		ioa_cfg->scsi_blocked = 0;
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		scsi_unblock_requests(ioa_cfg->host);
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 		if (ioa_cfg->scsi_blocked)
 			scsi_block_requests(ioa_cfg->host);
 	}
 
 	if (!ioa_cfg->scan_enabled) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return;
 	}
 
 	schedule_work(&ioa_cfg->scsi_add_work_q);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	LEAVE;
 }
 
@@ -3375,10 +3375,10 @@ static ssize_t ipr_read_trace(struct file *filp, struct kobject *kobj,
 	unsigned long lock_flags = 0;
 	ssize_t ret;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	ret = memory_read_from_buffer(buf, count, &off, ioa_cfg->trace,
 				IPR_TRACE_SIZE);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	return ret;
 }
@@ -3411,12 +3411,12 @@ static ssize_t ipr_show_fw_version(struct device *dev,
 	unsigned long lock_flags = 0;
 	int len;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	len = snprintf(buf, PAGE_SIZE, "%02X%02X%02X%02X\n",
 		       ucode_vpd->major_release, ucode_vpd->card_type,
 		       ucode_vpd->minor_release[0],
 		       ucode_vpd->minor_release[1]);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -3445,9 +3445,9 @@ static ssize_t ipr_show_log_level(struct device *dev,
 	unsigned long lock_flags = 0;
 	int len;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	len = snprintf(buf, PAGE_SIZE, "%d\n", ioa_cfg->log_level);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -3469,9 +3469,9 @@ static ssize_t ipr_store_log_level(struct device *dev,
 	struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata;
 	unsigned long lock_flags = 0;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	ioa_cfg->log_level = simple_strtoul(buf, NULL, 10);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return strlen(buf);
 }
 
@@ -3509,31 +3509,31 @@ static ssize_t ipr_store_diagnostics(struct device *dev,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	while (ioa_cfg->in_reset_reload) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	}
 
 	ioa_cfg->errors_logged = 0;
 	ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NORMAL);
 
 	if (ioa_cfg->in_reset_reload) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
 
 		/* Wait for a second for any errors to be logged */
 		msleep(1000);
 	} else {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return -EIO;
 	}
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	if (ioa_cfg->in_reset_reload || ioa_cfg->errors_logged)
 		rc = -EIO;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	return rc;
 }
@@ -3563,12 +3563,12 @@ static ssize_t ipr_show_adapter_state(struct device *dev,
 	unsigned long lock_flags = 0;
 	int len;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead)
 		len = snprintf(buf, PAGE_SIZE, "offline\n");
 	else
 		len = snprintf(buf, PAGE_SIZE, "online\n");
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -3596,7 +3596,7 @@ static ssize_t ipr_store_adapter_state(struct device *dev,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead &&
 	    !strncmp(buf, "online", 6)) {
 		for (i = 0; i < ioa_cfg->hrrq_num; i++) {
@@ -3609,7 +3609,7 @@ static ssize_t ipr_store_adapter_state(struct device *dev,
 		ioa_cfg->in_ioa_bringdown = 0;
 		ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE);
 	}
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
 
 	return result;
@@ -3648,10 +3648,10 @@ static ssize_t ipr_store_reset_adapter(struct device *dev,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	if (!ioa_cfg->in_reset_reload)
 		ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NORMAL);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
 
 	return result;
@@ -3683,9 +3683,9 @@ static ssize_t ipr_show_iopoll_weight(struct device *dev,
 	unsigned long lock_flags = 0;
 	int len;
 
-	spin_lock_irqsave(shost->host_lock, lock_flags);
+	spin_lock_irqsave(&shost->host_lock, lock_flags);
 	len = snprintf(buf, PAGE_SIZE, "%d\n", ioa_cfg->iopoll_weight);
-	spin_unlock_irqrestore(shost->host_lock, lock_flags);
+	spin_unlock_irqrestore(&shost->host_lock, lock_flags);
 
 	return len;
 }
@@ -3732,7 +3732,7 @@ static ssize_t ipr_store_iopoll_weight(struct device *dev,
 			irq_poll_disable(&ioa_cfg->hrrq[i].iopoll);
 	}
 
-	spin_lock_irqsave(shost->host_lock, lock_flags);
+	spin_lock_irqsave(&shost->host_lock, lock_flags);
 	ioa_cfg->iopoll_weight = user_iopoll_weight;
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
 		for (i = 1; i < ioa_cfg->hrrq_num; i++) {
@@ -3740,7 +3740,7 @@ static ssize_t ipr_store_iopoll_weight(struct device *dev,
 					ioa_cfg->iopoll_weight, ipr_iopoll);
 		}
 	}
-	spin_unlock_irqrestore(shost->host_lock, lock_flags);
+	spin_unlock_irqrestore(&shost->host_lock, lock_flags);
 
 	return strlen(buf);
 }
@@ -3939,15 +3939,15 @@ static int ipr_update_ioa_ucode(struct ipr_ioa_cfg *ioa_cfg,
 {
 	unsigned long lock_flags;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	while (ioa_cfg->in_reset_reload) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	}
 
 	if (ioa_cfg->ucode_sglist) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		dev_err(&ioa_cfg->pdev->dev,
 			"Microcode download already in progress\n");
 		return -EIO;
@@ -3958,7 +3958,7 @@ static int ipr_update_ioa_ucode(struct ipr_ioa_cfg *ioa_cfg,
 					DMA_TO_DEVICE);
 
 	if (!sglist->num_dma_sg) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		dev_err(&ioa_cfg->pdev->dev,
 			"Failed to map microcode download buffer!\n");
 		return -EIO;
@@ -3966,12 +3966,12 @@ static int ipr_update_ioa_ucode(struct ipr_ioa_cfg *ioa_cfg,
 
 	ioa_cfg->ucode_sglist = sglist;
 	ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NORMAL);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	ioa_cfg->ucode_sglist = NULL;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return 0;
 }
 
@@ -4072,9 +4072,9 @@ static ssize_t ipr_show_fw_type(struct device *dev,
 	unsigned long lock_flags = 0;
 	int len;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	len = snprintf(buf, PAGE_SIZE, "%d\n", ioa_cfg->sis64);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -4097,16 +4097,16 @@ static ssize_t ipr_read_async_err_log(struct file *filep, struct kobject *kobj,
 	unsigned long lock_flags = 0;
 	int ret;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	hostrcb = list_first_entry_or_null(&ioa_cfg->hostrcb_report_q,
 					struct ipr_hostrcb, queue);
 	if (!hostrcb) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return 0;
 	}
 	ret = memory_read_from_buffer(buf, count, &off, &hostrcb->hcam,
 				sizeof(hostrcb->hcam));
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return ret;
 }
 
@@ -4120,17 +4120,17 @@ static ssize_t ipr_next_async_err_log(struct file *filep, struct kobject *kobj,
 	struct ipr_hostrcb *hostrcb;
 	unsigned long lock_flags = 0;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	hostrcb = list_first_entry_or_null(&ioa_cfg->hostrcb_report_q,
 					struct ipr_hostrcb, queue);
 	if (!hostrcb) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return count;
 	}
 
 	/* Reclaim hostrcb before exit */
 	list_move_tail(&hostrcb->queue, &ioa_cfg->hostrcb_free_q);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return count;
 }
 
@@ -4187,15 +4187,15 @@ static ssize_t ipr_read_dump(struct file *filp, struct kobject *kobj,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EACCES;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	dump = ioa_cfg->dump;
 
 	if (ioa_cfg->sdt_state != DUMP_OBTAINED || !dump) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return 0;
 	}
 	kref_get(&dump->kref);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	if (off > dump->driver_dump.hdr.len) {
 		kref_put(&dump->kref, ipr_release_dump);
@@ -4298,10 +4298,10 @@ static int ipr_alloc_dump(struct ipr_ioa_cfg *ioa_cfg)
 	kref_init(&dump->kref);
 	dump->ioa_cfg = ioa_cfg;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	if (INACTIVE != ioa_cfg->sdt_state) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		vfree(dump->ioa_dump.ioa_data);
 		kfree(dump);
 		return 0;
@@ -4313,7 +4313,7 @@ static int ipr_alloc_dump(struct ipr_ioa_cfg *ioa_cfg)
 		ioa_cfg->dump_taken = 1;
 		schedule_work(&ioa_cfg->work_q);
 	}
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	return 0;
 }
@@ -4332,15 +4332,15 @@ static int ipr_free_dump(struct ipr_ioa_cfg *ioa_cfg)
 
 	ENTER;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	dump = ioa_cfg->dump;
 	if (!dump) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return 0;
 	}
 
 	ioa_cfg->dump = NULL;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	kref_put(&dump->kref, ipr_release_dump);
 
@@ -4429,11 +4429,11 @@ static ssize_t ipr_show_adapter_handle(struct device *dev, struct device_attribu
 	unsigned long lock_flags = 0;
 	ssize_t len = -ENXIO;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = (struct ipr_resource_entry *)sdev->hostdata;
 	if (res)
 		len = snprintf(buf, PAGE_SIZE, "%08X\n", res->res_handle);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -4464,7 +4464,7 @@ static ssize_t ipr_show_resource_path(struct device *dev, struct device_attribut
 	ssize_t len = -ENXIO;
 	char buffer[IPR_MAX_RES_PATH_LENGTH];
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = (struct ipr_resource_entry *)sdev->hostdata;
 	if (res && ioa_cfg->sis64)
 		len = snprintf(buf, PAGE_SIZE, "%s\n",
@@ -4474,7 +4474,7 @@ static ssize_t ipr_show_resource_path(struct device *dev, struct device_attribut
 		len = snprintf(buf, PAGE_SIZE, "%d:%d:%d:%d\n", ioa_cfg->host->host_no,
 			       res->bus, res->target, res->lun);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -4503,14 +4503,14 @@ static ssize_t ipr_show_device_id(struct device *dev, struct device_attribute *a
 	unsigned long lock_flags = 0;
 	ssize_t len = -ENXIO;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = (struct ipr_resource_entry *)sdev->hostdata;
 	if (res && ioa_cfg->sis64)
 		len = snprintf(buf, PAGE_SIZE, "0x%llx\n", be64_to_cpu(res->dev_id));
 	else if (res)
 		len = snprintf(buf, PAGE_SIZE, "0x%llx\n", res->lun_wwn);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -4539,13 +4539,13 @@ static ssize_t ipr_show_resource_type(struct device *dev, struct device_attribut
 	unsigned long lock_flags = 0;
 	ssize_t len = -ENXIO;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = (struct ipr_resource_entry *)sdev->hostdata;
 
 	if (res)
 		len = snprintf(buf, PAGE_SIZE, "%x\n", res->type);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -4575,13 +4575,13 @@ static ssize_t ipr_show_raw_mode(struct device *dev,
 	unsigned long lock_flags = 0;
 	ssize_t len;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = (struct ipr_resource_entry *)sdev->hostdata;
 	if (res)
 		len = snprintf(buf, PAGE_SIZE, "%d\n", res->raw_mode);
 	else
 		len = -ENXIO;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -4605,7 +4605,7 @@ static ssize_t ipr_store_raw_mode(struct device *dev,
 	unsigned long lock_flags = 0;
 	ssize_t len;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = (struct ipr_resource_entry *)sdev->hostdata;
 	if (res) {
 		if (ipr_is_af_dasd_device(res)) {
@@ -4618,7 +4618,7 @@ static ssize_t ipr_store_raw_mode(struct device *dev,
 			len = -EINVAL;
 	} else
 		len = -ENXIO;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return len;
 }
 
@@ -4759,13 +4759,13 @@ static void ipr_sdev_destroy(struct scsi_device *sdev)
 
 	ioa_cfg = (struct ipr_ioa_cfg *) sdev->host->hostdata;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = (struct ipr_resource_entry *) sdev->hostdata;
 	if (res) {
 		sdev->hostdata = NULL;
 		res->sdev = NULL;
 	}
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 }
 
 /**
@@ -4786,7 +4786,7 @@ static int ipr_sdev_configure(struct scsi_device *sdev,
 	unsigned long lock_flags = 0;
 	char buffer[IPR_MAX_RES_PATH_LENGTH];
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	res = sdev->hostdata;
 	if (res) {
 		if (ipr_is_af_dasd_device(res))
@@ -4802,7 +4802,7 @@ static int ipr_sdev_configure(struct scsi_device *sdev,
 					     IPR_VSET_RW_TIMEOUT);
 			lim->max_hw_sectors = IPR_VSET_MAX_SECTORS;
 		}
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 		if (ioa_cfg->sis64)
 			sdev_printk(KERN_INFO, sdev, "Resource path: %s\n",
@@ -4810,7 +4810,7 @@ static int ipr_sdev_configure(struct scsi_device *sdev,
 				res->res_path, buffer, sizeof(buffer)));
 		return 0;
 	}
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return 0;
 }
 
@@ -4835,7 +4835,7 @@ static int ipr_sdev_init(struct scsi_device *sdev)
 
 	sdev->hostdata = NULL;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	res = ipr_find_sdev(sdev);
 	if (res) {
@@ -4849,12 +4849,12 @@ static int ipr_sdev_init(struct scsi_device *sdev)
 		if (ipr_is_gata(res)) {
 			sdev_printk(KERN_ERR, sdev, "SATA devices are no longer "
 				"supported by this driver. Skipping device.\n");
-			spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+			spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 			return -ENXIO;
 		}
 	}
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	return rc;
 }
@@ -4970,7 +4970,7 @@ static int ipr_eh_host_reset(struct scsi_cmnd *cmd)
 
 	ENTER;
 	ioa_cfg = (struct ipr_ioa_cfg *) cmd->device->host->hostdata;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	if (!ioa_cfg->in_reset_reload && !ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) {
 		ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_ABBREV);
@@ -4981,9 +4981,9 @@ static int ipr_eh_host_reset(struct scsi_cmnd *cmd)
 			ioa_cfg->sdt_state = GET_DUMP;
 	}
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	/* If we got hit with a host reset while we were already resetting
 	 the adapter for some reason, and the reset failed. */
@@ -4992,7 +4992,7 @@ static int ipr_eh_host_reset(struct scsi_cmnd *cmd)
 		rc = FAILED;
 	}
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	LEAVE;
 	return rc;
 }
@@ -5092,9 +5092,9 @@ static int ipr_eh_dev_reset(struct scsi_cmnd *cmd)
 	if (!res)
 		return FAILED;
 
-	spin_lock_irq(cmd->device->host->host_lock);
+	spin_lock_irq(&cmd->device->host->host_lock);
 	rc = __ipr_eh_dev_reset(cmd);
-	spin_unlock_irq(cmd->device->host->host_lock);
+	spin_unlock_irq(&cmd->device->host->host_lock);
 
 	if (rc == SUCCESS)
 		rc = ipr_wait_for_ops(ioa_cfg, cmd->device, ipr_match_lun);
@@ -5158,9 +5158,9 @@ static void ipr_abort_timeout(struct timer_list *t)
 	unsigned long lock_flags = 0;
 
 	ENTER;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	if (ipr_cmd->completion.done || ioa_cfg->in_reset_reload) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		return;
 	}
 
@@ -5175,7 +5175,7 @@ static void ipr_abort_timeout(struct timer_list *t)
 	cmd_pkt->cdb[2] = IPR_RESET_TYPE_SELECT | IPR_BUS_RESET;
 
 	ipr_do_req(reset_cmd, ipr_bus_reset_done, ipr_timeout, IPR_DEVICE_RESET_TIMEOUT);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	LEAVE;
 }
 
@@ -5281,12 +5281,12 @@ static int ipr_scan_finished(struct Scsi_Host *shost, unsigned long elapsed_time
 	struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *) shost->hostdata;
 	int rc = 0;
 
-	spin_lock_irqsave(shost->host_lock, lock_flags);
+	spin_lock_irqsave(&shost->host_lock, lock_flags);
 	if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead || ioa_cfg->scan_done)
 		rc = 1;
 	if ((elapsed_time/HZ) > (ioa_cfg->transop_timeout * 2))
 		rc = 1;
-	spin_unlock_irqrestore(shost->host_lock, lock_flags);
+	spin_unlock_irqrestore(&shost->host_lock, lock_flags);
 	return rc;
 }
 
@@ -5307,9 +5307,9 @@ static int ipr_eh_abort(struct scsi_cmnd *scsi_cmd)
 
 	ioa_cfg = (struct ipr_ioa_cfg *) scsi_cmd->device->host->hostdata;
 
-	spin_lock_irqsave(scsi_cmd->device->host->host_lock, flags);
+	spin_lock_irqsave(&scsi_cmd->device->host->host_lock, flags);
 	rc = ipr_cancel_op(scsi_cmd);
-	spin_unlock_irqrestore(scsi_cmd->device->host->host_lock, flags);
+	spin_unlock_irqrestore(&scsi_cmd->device->host->host_lock, flags);
 
 	if (rc == SUCCESS)
 		rc = ipr_wait_for_ops(ioa_cfg, scsi_cmd->device, ipr_match_lun);
@@ -6222,11 +6222,11 @@ static void ipr_scsi_done(struct ipr_cmnd *ipr_cmd)
 		list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q);
 		spin_unlock_irqrestore(ipr_cmd->hrrq->lock, lock_flags);
 	} else {
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 		spin_lock(&ipr_cmd->hrrq->_lock);
 		ipr_erp_start(ioa_cfg, ipr_cmd);
 		spin_unlock(&ipr_cmd->hrrq->_lock);
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	}
 }
 
@@ -6383,9 +6383,9 @@ static const char *ipr_ioa_info(struct Scsi_Host *host)
 
 	ioa_cfg = (struct ipr_ioa_cfg *) host->hostdata;
 
-	spin_lock_irqsave(host->host_lock, lock_flags);
+	spin_lock_irqsave(&host->host_lock, lock_flags);
 	sprintf(buffer, "IBM %X Storage Adapter", ioa_cfg->type);
-	spin_unlock_irqrestore(host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&host->host_lock, lock_flags);
 
 	return buffer;
 }
@@ -7480,14 +7480,14 @@ static void ipr_reset_timer_done(struct timer_list *t)
 	struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
 	unsigned long lock_flags = 0;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	if (ioa_cfg->reset_cmd == ipr_cmd) {
 		list_del(&ipr_cmd->queue);
 		ipr_cmd->done(ipr_cmd);
 	}
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 }
 
 /**
@@ -8023,10 +8023,10 @@ static void ipr_reset_reset_work(struct work_struct *work)
 	msleep(jiffies_to_msecs(IPR_PCI_RESET_TIMEOUT));
 	pci_set_pcie_reset_state(pdev, pcie_deassert_reset);
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	if (ioa_cfg->reset_cmd == ipr_cmd)
 		ipr_reset_ioa_job(ipr_cmd);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	LEAVE;
 }
 
@@ -8588,10 +8588,10 @@ static pci_ers_result_t ipr_pci_mmio_enabled(struct pci_dev *pdev)
 	unsigned long flags = 0;
 	struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev);
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, flags);
 	if (!ioa_cfg->probe_done)
 		pci_save_state(pdev);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, flags);
 	return PCI_ERS_RESULT_NEED_RESET;
 }
 
@@ -8608,10 +8608,10 @@ static void ipr_pci_frozen(struct pci_dev *pdev)
 	unsigned long flags = 0;
 	struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev);
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, flags);
 	if (ioa_cfg->probe_done)
 		_ipr_initiate_ioa_reset(ioa_cfg, ipr_reset_freeze, IPR_SHUTDOWN_NONE);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, flags);
 }
 
 /**
@@ -8627,7 +8627,7 @@ static pci_ers_result_t ipr_pci_slot_reset(struct pci_dev *pdev)
 	unsigned long flags = 0;
 	struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev);
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, flags);
 	if (ioa_cfg->probe_done) {
 		if (ioa_cfg->needs_warm_reset)
 			ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE);
@@ -8636,7 +8636,7 @@ static pci_ers_result_t ipr_pci_slot_reset(struct pci_dev *pdev)
 						IPR_SHUTDOWN_NONE);
 	} else
 		wake_up_all(&ioa_cfg->eeh_wait_q);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, flags);
 	return PCI_ERS_RESULT_RECOVERED;
 }
 
@@ -8653,7 +8653,7 @@ static void ipr_pci_perm_failure(struct pci_dev *pdev)
 	struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev);
 	int i;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, flags);
 	if (ioa_cfg->probe_done) {
 		if (ioa_cfg->sdt_state == WAIT_FOR_DUMP)
 			ioa_cfg->sdt_state = ABORT_DUMP;
@@ -8668,7 +8668,7 @@ static void ipr_pci_perm_failure(struct pci_dev *pdev)
 		ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE);
 	} else
 		wake_up_all(&ioa_cfg->eeh_wait_q);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, flags);
 }
 
 /**
@@ -8712,7 +8712,7 @@ static void ipr_probe_ioa_part2(struct ipr_ioa_cfg *ioa_cfg)
 	unsigned long host_lock_flags = 0;
 
 	ENTER;
-	spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, host_lock_flags);
 	dev_dbg(&ioa_cfg->pdev->dev, "ioa_cfg adx: 0x%p\n", ioa_cfg);
 	ioa_cfg->probe_done = 1;
 	if (ioa_cfg->needs_hard_reset) {
@@ -8721,7 +8721,7 @@ static void ipr_probe_ioa_part2(struct ipr_ioa_cfg *ioa_cfg)
 	} else
 		_ipr_initiate_ioa_reset(ioa_cfg, ipr_reset_enable_ioa,
 					IPR_SHUTDOWN_NONE);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, host_lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, host_lock_flags);
 
 	LEAVE;
 }
@@ -9196,7 +9196,7 @@ static void ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg,
 		INIT_LIST_HEAD(&ioa_cfg->hrrq[i].hrrq_pending_q);
 		spin_lock_init(&ioa_cfg->hrrq[i]._lock);
 		if (i == 0)
-			ioa_cfg->hrrq[i].lock = ioa_cfg->host->host_lock;
+			ioa_cfg->hrrq[i].lock = &ioa_cfg->host->host_lock;
 		else
 			ioa_cfg->hrrq[i].lock = &ioa_cfg->hrrq[i]._lock;
 	}
@@ -9291,12 +9291,12 @@ static irqreturn_t ipr_test_intr(int irq, void *devp)
 	unsigned long lock_flags = 0;
 
 	dev_info(&ioa_cfg->pdev->dev, "Received IRQ : %d\n", irq);
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 
 	ioa_cfg->msi_received = 1;
 	wake_up(&ioa_cfg->msi_wait_q);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	return IRQ_HANDLED;
 }
 
@@ -9320,13 +9320,13 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev)
 
 	ENTER;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	init_waitqueue_head(&ioa_cfg->msi_wait_q);
 	ioa_cfg->msi_received = 0;
 	ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER);
 	writel(IPR_PCII_IO_DEBUG_ACKNOWLEDGE, ioa_cfg->regs.clr_interrupt_mask_reg32);
 	readl(ioa_cfg->regs.sense_interrupt_mask_reg);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	rc = request_irq(irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg);
 	if (rc) {
@@ -9338,7 +9338,7 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev)
 	writel(IPR_PCII_IO_DEBUG_ACKNOWLEDGE, ioa_cfg->regs.sense_interrupt_reg32);
 	readl(ioa_cfg->regs.sense_interrupt_reg);
 	wait_event_timeout(ioa_cfg->msi_wait_q, ioa_cfg->msi_received, HZ);
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER);
 
 	if (!ioa_cfg->msi_received) {
@@ -9348,7 +9348,7 @@ static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev)
 	} else if (ipr_debug)
 		dev_info(&pdev->dev, "MSI test succeeded.\n");
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	free_irq(irq, ioa_cfg);
 
@@ -9573,9 +9573,9 @@ static int ipr_probe_ioa(struct pci_dev *pdev,
 	if (interrupts & IPR_PCII_IOA_UNIT_CHECKED)
 		ioa_cfg->ioa_unit_checked = 1;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 
 	if (pdev->msi_enabled || pdev->msix_enabled) {
 		name_msi_vectors(ioa_cfg);
@@ -9680,11 +9680,11 @@ static void __ipr_remove(struct pci_dev *pdev)
 	unsigned long driver_lock_flags;
 	ENTER;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, host_lock_flags);
 	while (ioa_cfg->in_reset_reload) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, host_lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, host_lock_flags);
 		wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
-		spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, host_lock_flags);
 	}
 
 	for (i = 0; i < ioa_cfg->hrrq_num; i++) {
@@ -9695,13 +9695,13 @@ static void __ipr_remove(struct pci_dev *pdev)
 	wmb();
 	ipr_initiate_ioa_bringdown(ioa_cfg, IPR_SHUTDOWN_NORMAL);
 
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, host_lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, host_lock_flags);
 	wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
 	flush_work(&ioa_cfg->work_q);
 	if (ioa_cfg->reset_work_q)
 		flush_workqueue(ioa_cfg->reset_work_q);
 	INIT_LIST_HEAD(&ioa_cfg->used_res_q);
-	spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, host_lock_flags);
 
 	spin_lock_irqsave(&ipr_driver_lock, driver_lock_flags);
 	list_del(&ioa_cfg->queue);
@@ -9709,7 +9709,7 @@ static void __ipr_remove(struct pci_dev *pdev)
 
 	if (ioa_cfg->sdt_state == ABORT_DUMP)
 		ioa_cfg->sdt_state = WAIT_FOR_DUMP;
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, host_lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, host_lock_flags);
 
 	ipr_free_all_resources(ioa_cfg);
 
@@ -9807,10 +9807,10 @@ static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
 		__ipr_remove(pdev);
 		return rc;
 	}
-	spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, flags);
 	ioa_cfg->scan_enabled = 1;
 	schedule_work(&ioa_cfg->work_q);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, flags);
 
 	ioa_cfg->iopoll_weight = ioa_cfg->chip_cfg->iopoll_weight;
 
@@ -9843,7 +9843,7 @@ static void ipr_shutdown(struct pci_dev *pdev)
 	enum ipr_shutdown_type shutdown_type = IPR_SHUTDOWN_NORMAL;
 	int i;
 
-	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
 		ioa_cfg->iopoll_weight = 0;
 		for (i = 1; i < ioa_cfg->hrrq_num; i++)
@@ -9851,16 +9851,16 @@ static void ipr_shutdown(struct pci_dev *pdev)
 	}
 
 	while (ioa_cfg->in_reset_reload) {
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 		wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
-		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, lock_flags);
 	}
 
 	if (ipr_fast_reboot && system_state == SYSTEM_RESTART && ioa_cfg->sis64)
 		shutdown_type = IPR_SHUTDOWN_QUIESCE;
 
 	ipr_initiate_ioa_bringdown(ioa_cfg, shutdown_type);
-	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&ioa_cfg->host->host_lock, lock_flags);
 	wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload);
 	if (ipr_fast_reboot && system_state == SYSTEM_RESTART && ioa_cfg->sis64) {
 		ipr_free_irqs(ioa_cfg);
@@ -10030,10 +10030,10 @@ static int ipr_halt(struct notifier_block *nb, ulong event, void *buf)
 	spin_lock_irqsave(&ipr_driver_lock, driver_lock_flags);
 
 	list_for_each_entry(ioa_cfg, &ipr_ioa_head, queue) {
-		spin_lock_irqsave(ioa_cfg->host->host_lock, flags);
+		spin_lock_irqsave(&ioa_cfg->host->host_lock, flags);
 		if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds ||
 		    (ipr_fast_reboot && event == SYS_RESTART && ioa_cfg->sis64)) {
-			spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+			spin_unlock_irqrestore(&ioa_cfg->host->host_lock, flags);
 			continue;
 		}
 
@@ -10044,7 +10044,7 @@ static int ipr_halt(struct notifier_block *nb, ulong event, void *buf)
 		ipr_cmd->ioarcb.cmd_pkt.cdb[1] = IPR_SHUTDOWN_PREPARE_FOR_NORMAL;
 
 		ipr_do_req(ipr_cmd, ipr_halt_done, ipr_timeout, IPR_DEVICE_RESET_TIMEOUT);
-		spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags);
+		spin_unlock_irqrestore(&ioa_cfg->host->host_lock, flags);
 	}
 	spin_unlock_irqrestore(&ipr_driver_lock, driver_lock_flags);
 
diff --git a/drivers/scsi/ips.c b/drivers/scsi/ips.c
index 41ed73966a48c2..45c5b9acb664c6 100644
--- a/drivers/scsi/ips.c
+++ b/drivers/scsi/ips.c
@@ -794,7 +794,7 @@ int ips_eh_abort(struct scsi_cmnd *SC)
 	if (!ha->active)
 		return (FAILED);
 
-	spin_lock(host->host_lock);
+	spin_lock(&host->host_lock);
 
 	/* See if the command is on the copp queue */
 	item = ha->copp_waitlist.head;
@@ -815,7 +815,7 @@ int ips_eh_abort(struct scsi_cmnd *SC)
 		ret = (FAILED);
 	}
 
-	spin_unlock(host->host_lock);
+	spin_unlock(&host->host_lock);
 	return ret;
 }
 
@@ -999,9 +999,9 @@ static int ips_eh_reset(struct scsi_cmnd *SC)
 {
 	int rc;
 
-	spin_lock_irq(SC->device->host->host_lock);
+	spin_lock_irq(&SC->device->host->host_lock);
 	rc = __ips_eh_reset(SC);
-	spin_unlock_irq(SC->device->host->host_lock);
+	spin_unlock_irq(&SC->device->host->host_lock);
 
 	return rc;
 }
@@ -1221,16 +1221,16 @@ do_ipsintr(int irq, void *dev_id)
 		return IRQ_HANDLED;
 	}
 
-	spin_lock(host->host_lock);
+	spin_lock(&host->host_lock);
 
 	if (!ha->active) {
-		spin_unlock(host->host_lock);
+		spin_unlock(&host->host_lock);
 		return IRQ_HANDLED;
 	}
 
 	irqstatus = (*ha->func.intr) (ha);
 
-	spin_unlock(host->host_lock);
+	spin_unlock(&host->host_lock);
 
 	/* start the next command */
 	ips_next(ha, IPS_INTR_ON);
@@ -2524,7 +2524,7 @@ ips_next(ips_ha_t * ha, int intr)
 	 * this command won't time out
 	 */
 	if (intr == IPS_INTR_ON)
-		spin_lock(host->host_lock);
+		spin_lock(&host->host_lock);
 
 	if ((ha->subsys->param[3] & 0x300000)
 	    && (ha->scb_activelist.count == 0)) {
@@ -2548,14 +2548,14 @@ ips_next(ips_ha_t * ha, int intr)
 		item = ips_removeq_copp_head(&ha->copp_waitlist);
 		ha->num_ioctl++;
 		if (intr == IPS_INTR_ON)
-			spin_unlock(host->host_lock);
+			spin_unlock(&host->host_lock);
 		scb->scsi_cmd = item->scsi_cmd;
 		kfree(item);
 
 		ret = ips_make_passthru(ha, scb->scsi_cmd, scb, intr);
 
 		if (intr == IPS_INTR_ON)
-			spin_lock(host->host_lock);
+			spin_lock(&host->host_lock);
 		switch (ret) {
 		case IPS_FAILURE:
 			if (scb->scsi_cmd) {
@@ -2625,7 +2625,7 @@ ips_next(ips_ha_t * ha, int intr)
 		SC = ips_removeq_wait(&ha->scb_waitlist, q);
 
 		if (intr == IPS_INTR_ON)
-			spin_unlock(host->host_lock);	/* Unlock HA after command is taken off queue */
+			spin_unlock(&host->host_lock);	/* Unlock HA after command is taken off queue */
 
 		SC->result = DID_OK;
 		SC->host_scribble = NULL;
@@ -2682,7 +2682,7 @@ ips_next(ips_ha_t * ha, int intr)
 			scb->dcdb.transfer_length = 0;
 		}
 		if (intr == IPS_INTR_ON)
-			spin_lock(host->host_lock);
+			spin_lock(&host->host_lock);
 
 		ret = ips_send_cmd(ha, scb);
 
@@ -2721,7 +2721,7 @@ ips_next(ips_ha_t * ha, int intr)
 	}			/* end while */
 
 	if (intr == IPS_INTR_ON)
-		spin_unlock(host->host_lock);
+		spin_unlock(&host->host_lock);
 }
 
 /****************************************************************************/
diff --git a/drivers/scsi/leapraid/leapraid_func.h b/drivers/scsi/leapraid/leapraid_func.h
index 4c0b9ca728d87f..a16390a71662af 100644
--- a/drivers/scsi/leapraid/leapraid_func.h
+++ b/drivers/scsi/leapraid/leapraid_func.h
@@ -113,7 +113,6 @@
 #define DESC_FORMAT_THRESHOLD                   0x72
 #define SENSE_KEY_MASK                          0x0F
 #define SCSI_SENSE_RESPONSE_CODE_MASK           0x7F
-#define ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED       0x5D
 #define LEAPRAID_LARGE_DISK_THRESHOLD           0x200000UL
 #define LEAPRAID_LARGE_DISK_HEADS               255
 #define LEAPRAID_LARGE_DISK_SECTORS             63
@@ -126,12 +125,6 @@
 #define LEAPRAID_SCSI_HOST_SHIFT                16
 #define LEAPRAID_SCSI_DRIVER_SHIFT              24
 
-/* SCSI ASC/ASCQ definitions. */
-#define LEAPRAID_SCSI_ASCQ_DEFAULT              0x00
-#define LEAPRAID_SCSI_ASC_POWER_ON_RESET        0x29
-#define LEAPRAID_SCSI_ASC_INVALID_CMD_CODE      0x20
-#define LEAPRAID_SCSI_ASCQ_POWER_ON_RESET       0x07
-
 /* VPD Page 0x89 (ATA Information). */
 #define LEAPRAID_VPD_PAGE_ATA_INFO      0x89
 #define LEAPRAID_VPD_PG89_MAX_LEN       255
@@ -554,7 +547,6 @@ struct leapraid_fw_evt_work {
 /**
  * struct leapraid_fw_evt_struct - Firmware event handling structure
  *
- * @fw_evt_name: Name of the firmware event.
  * @fw_evt_thread: Workqueue used for processing firmware events.
  * @fw_evt_lock: Spinlock protecting access to the firmware event list.
  * @fw_evt_list: Linked list of pending firmware events.
@@ -565,7 +557,6 @@ struct leapraid_fw_evt_work {
  */
 struct leapraid_fw_evt_struct {
 	u32 leapraid_evt_masks[4];
-	char fw_evt_name[48];
 	struct workqueue_struct *fw_evt_thread;
 	spinlock_t fw_evt_lock; /* protects firmware event */
 	struct list_head fw_evt_list;
diff --git a/drivers/scsi/leapraid/leapraid_os.c b/drivers/scsi/leapraid/leapraid_os.c
index ee3242779dfdff..da597eb99377eb 100644
--- a/drivers/scsi/leapraid/leapraid_os.c
+++ b/drivers/scsi/leapraid/leapraid_os.c
@@ -382,9 +382,8 @@ static void leapraid_handle_data_underrun(
 	} else if (!xfer_cnt && scmd->cmnd[0] == REPORT_LUNS) {
 		scsiio_rep->scsi_state = LEAPRAID_SCSI_STATE_AUTOSENSE_VALID;
 		scsiio_rep->scsi_status = SAM_STAT_CHECK_CONDITION;
-		scsi_build_sense(scmd, 0, ILLEGAL_REQUEST,
-				 LEAPRAID_SCSI_ASC_INVALID_CMD_CODE,
-				 LEAPRAID_SCSI_ASCQ_DEFAULT);
+		scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+			       INVALID_COMMAND_OP_CODE);
 	}
 }
 
@@ -515,8 +514,9 @@ static void leapraid_scsiio_done_dispatch(
 					 &sshdr))
 			dev_warn(&adapter->pdev->dev,
 				 "Sense: key=0x%x asc=0x%x ascq=0x%x\n",
-				 sshdr.sense_key, sshdr.asc,
-				 sshdr.ascq);
+				 sshdr.sense_key,
+				 scsi_sense_asc(&sshdr),
+				 scsi_sense_ascq(&sshdr));
 		else
 			dev_warn(&adapter->pdev->dev,
 				 "Sense: Invalid sense data\n");
@@ -810,9 +810,8 @@ static bool leapraid_should_queuecommand(struct leapraid_adapter *adapter,
 	if (sdev_priv->block &&
 	    scsi_get_host_state(scmd->device->host) == SHOST_RECOVERY &&
 	    scmd->cmnd[0] == TEST_UNIT_READY) {
-		scsi_build_sense(scmd, 0, UNIT_ATTENTION,
-				 LEAPRAID_SCSI_ASC_POWER_ON_RESET,
-				 LEAPRAID_SCSI_ASCQ_POWER_ON_RESET);
+		scsi_set_sense(scmd, 0, UNIT_ATTENTION,
+			       I_T_NEXUS_LOSS_OCCURRED);
 		goto scsiio_done;
 	}
 
@@ -2054,12 +2053,10 @@ static int leapraid_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	shost->transportt = leapraid_transport_template;
 	shost->unique_id = adapter->adapter_attr.id;
 
-	snprintf(adapter->fw_evt_s.fw_evt_name,
-		 sizeof(adapter->fw_evt_s.fw_evt_name),
-		 "fw_event_%s%d", LEAPRAID_DRIVER_NAME,
-		 adapter->adapter_attr.id);
 	adapter->fw_evt_s.fw_evt_thread =
-		alloc_ordered_workqueue(adapter->fw_evt_s.fw_evt_name, 0);
+		alloc_ordered_workqueue("fw_event_%s%d", 0,
+					LEAPRAID_DRIVER_NAME,
+					adapter->adapter_attr.id);
 	if (!adapter->fw_evt_s.fw_evt_thread) {
 		dev_err(&adapter->pdev->dev,
 			"%s: Failed to create fw event workqueue\n", __func__);
diff --git a/drivers/scsi/libfc/fc_fcp.c b/drivers/scsi/libfc/fc_fcp.c
index a5139e43ca4ce6..60a675aa5a99cb 100644
--- a/drivers/scsi/libfc/fc_fcp.c
+++ b/drivers/scsi/libfc/fc_fcp.c
@@ -358,7 +358,7 @@ static void fc_fcp_can_queue_ramp_up(struct fc_lport *lport)
 	unsigned long flags;
 	int can_queue;
 
-	spin_lock_irqsave(lport->host->host_lock, flags);
+	spin_lock_irqsave(&lport->host->host_lock, flags);
 
 	if (si->last_can_queue_ramp_up_time &&
 	    (time_before(jiffies, si->last_can_queue_ramp_up_time +
@@ -381,7 +381,7 @@ static void fc_fcp_can_queue_ramp_up(struct fc_lport *lport)
 		     "can_queue to %d.\n", can_queue);
 
 unlock:
-	spin_unlock_irqrestore(lport->host->host_lock, flags);
+	spin_unlock_irqrestore(&lport->host->host_lock, flags);
 }
 
 /**
@@ -401,7 +401,7 @@ static bool fc_fcp_can_queue_ramp_down(struct fc_lport *lport)
 	int can_queue;
 	bool changed = false;
 
-	spin_lock_irqsave(lport->host->host_lock, flags);
+	spin_lock_irqsave(&lport->host->host_lock, flags);
 
 	if (si->last_can_queue_ramp_down_time &&
 	    (time_before(jiffies, si->last_can_queue_ramp_down_time +
@@ -418,7 +418,7 @@ static bool fc_fcp_can_queue_ramp_down(struct fc_lport *lport)
 	changed = true;
 
 unlock:
-	spin_unlock_irqrestore(lport->host->host_lock, flags);
+	spin_unlock_irqrestore(&lport->host->host_lock, flags);
 	return changed;
 }
 
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 5cbc51899de00c..7637aacd603758 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -892,7 +892,8 @@ static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 
 		ascq = session->tt->check_protection(task, &sector);
 		if (ascq) {
-			scsi_build_sense(sc, 1, ILLEGAL_REQUEST, 0x10, ascq);
+			scsi_set_sense(sc, 1, ILLEGAL_REQUEST,
+				scsi_sense_code(ASC_ID_CRC_OR_ECC_ERROR, ascq));
 			scsi_set_sense_information(sc->sense_buffer,
 						   SCSI_SENSE_BUFFERSIZE,
 						   sector);
diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c
index 7223bb18b04809..d35f93451ee9bb 100644
--- a/drivers/scsi/libiscsi_tcp.c
+++ b/drivers/scsi/libiscsi_tcp.c
@@ -480,6 +480,9 @@ static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task)
 	int datasn = be32_to_cpu(rhdr->datasn);
 	unsigned total_in_length = task->sc->sdb.length;
 
+	if (task->sc->sc_data_direction != DMA_FROM_DEVICE)
+		return ISCSI_ERR_PROTO;
+
 	/*
 	 * lib iscsi will update this in the completion handling if there
 	 * is status.
diff --git a/drivers/scsi/libsas/sas_scsi_host.c b/drivers/scsi/libsas/sas_scsi_host.c
index 97ac3db5dccb3f..76517c8f0684de 100644
--- a/drivers/scsi/libsas/sas_scsi_host.c
+++ b/drivers/scsi/libsas/sas_scsi_host.c
@@ -436,13 +436,13 @@ int sas_eh_abort_handler(struct scsi_cmnd *cmd)
 	if (!i->dft->lldd_abort_task)
 		return FAILED;
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	/* We cannot do async aborts for SATA devices */
 	if (dev_is_sata(dev) && !host->host_eh_scheduled) {
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 		return FAILED;
 	}
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 
 	if (task)
 		res = i->dft->lldd_abort_task(task);
@@ -714,9 +714,9 @@ void sas_scsi_recover_host(struct Scsi_Host *shost)
 retry:
 	tries++;
 	retry = true;
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	list_splice_init(&shost->eh_cmd_q, &eh_work_q);
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	pr_notice("Enter %s busy: %d failed: %d\n",
 		  __func__, scsi_host_busy(shost), shost->host_failed);
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 52fc5058976d99..6f6394a0047c28 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -7917,15 +7917,15 @@ lpfc_els_flush_rscn(struct lpfc_vport *vport)
 	struct lpfc_hba  *phba = vport->phba;
 	int i;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	if (vport->fc_rscn_flush) {
 		/* Another thread is walking fc_rscn_id_list on this vport */
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		return;
 	}
 	/* Indicate we are walking lpfc_els_flush_rscn on this vport */
 	vport->fc_rscn_flush = 1;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	for (i = 0; i < vport->fc_rscn_id_cnt; i++) {
 		lpfc_in_buf_free(phba, vport->fc_rscn_id_list[i]);
@@ -7933,9 +7933,9 @@ lpfc_els_flush_rscn(struct lpfc_vport *vport)
 	}
 	clear_bit(FC_RSCN_MODE, &vport->fc_flag);
 	clear_bit(FC_RSCN_DISCOVERY, &vport->fc_flag);
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	vport->fc_rscn_id_cnt = 0;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	lpfc_can_disctmo(vport);
 	/* Indicate we are done walking this fc_rscn_id_list */
 	vport->fc_rscn_flush = 0;
@@ -7972,15 +7972,15 @@ lpfc_rscn_payload_check(struct lpfc_vport *vport, uint32_t did)
 	if (test_bit(FC_RSCN_DISCOVERY, &vport->fc_flag))
 		return did;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	if (vport->fc_rscn_flush) {
 		/* Another thread is walking fc_rscn_id_list on this vport */
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		return 0;
 	}
 	/* Indicate we are walking fc_rscn_id_list on this vport */
 	vport->fc_rscn_flush = 1;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	for (i = 0; i < vport->fc_rscn_id_cnt; i++) {
 		lp = vport->fc_rscn_id_list[i]->virt;
 		payload_len = be32_to_cpu(*lp++ & ~ELS_CMD_MASK);
@@ -8234,10 +8234,10 @@ lpfc_els_rcv_rscn(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb,
 		}
 	}
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	if (vport->fc_rscn_flush) {
 		/* Another thread is walking fc_rscn_id_list on this vport */
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		set_bit(FC_RSCN_DISCOVERY, &vport->fc_flag);
 		/* Send back ACC */
 		lpfc_els_rsp_acc(vport, ELS_CMD_ACC, cmdiocb, ndlp, NULL);
@@ -8245,7 +8245,7 @@ lpfc_els_rcv_rscn(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb,
 	}
 	/* Indicate we are walking fc_rscn_id_list on this vport */
 	vport->fc_rscn_flush = 1;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	/* Get the array count after successfully have the token */
 	rscn_cnt = vport->fc_rscn_id_cnt;
 	/* If we are already processing an RSCN, save the received
@@ -8538,14 +8538,14 @@ lpfc_els_rcv_flogi(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb,
 	 * The vport state should go to LPFC_FLOGI only
 	 * AFTER we issue a FLOGI, not receive one.
 	 */
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	fc_flag = vport->fc_flag;
 	port_state = vport->port_state;
 	/* Acking an unsol FLOGI.  Count 1 for link bounce
 	 * work-around.
 	 */
 	vport->rcv_flogi_cnt++;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	set_bit(FC_PT2PT, &vport->fc_flag);
 	clear_bit(FC_FABRIC, &vport->fc_flag);
 	clear_bit(FC_PUBLIC_LOOP, &vport->fc_flag);
@@ -11190,9 +11190,9 @@ lpfc_cmpl_reg_new_vport(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 			break;
 		}
 	} else {
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		vport->vpi_state |= LPFC_VPI_REGISTERED;
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		if (vport == phba->pport) {
 			if (phba->sli_rev < LPFC_SLI_REV4)
 				lpfc_issue_fabric_reglogin(vport);
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index c9b02d2c63054b..4c673dffa671bf 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -1340,9 +1340,9 @@ lpfc_linkdown(struct lpfc_hba *phba)
 		}
 		clear_bit(FC_PT2PT, &phba->pport->fc_flag);
 		clear_bit(FC_PT2PT_PLOGI, &phba->pport->fc_flag);
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		phba->pport->rcv_flogi_cnt = 0;
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 	}
 	return 0;
 }
@@ -1406,9 +1406,9 @@ lpfc_linkup_port(struct lpfc_vport *vport)
 	}
 	set_bit(FC_NDISC_ACTIVE, &vport->fc_flag);
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	vport->fc_ns_retry = 0;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	lpfc_setup_fdmi_mask(vport);
 
 	lpfc_linkup_cleanup_nodes(vport);
@@ -1437,9 +1437,9 @@ lpfc_linkup(struct lpfc_hba *phba)
 	 * absorbed without an ACQE. No lock here - in worker thread
 	 * and discovery is synchronized.
 	 */
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	phba->pport->rcv_flogi_cnt = 0;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	return 0;
 }
@@ -3400,9 +3400,9 @@ lpfc_mbx_cmpl_reg_vfi(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 	set_bit(FC_VFI_REGISTERED, &vport->fc_flag);
 	clear_bit(FC_VPORT_NEEDS_REG_VPI, &vport->fc_flag);
 	clear_bit(FC_VPORT_NEEDS_INIT_VPI, &vport->fc_flag);
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	vport->vpi_state |= LPFC_VPI_REGISTERED;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	/* In case SLI4 FC loopback test, we are ready */
 	if ((phba->sli_rev == LPFC_SLI_REV4) &&
@@ -3943,9 +3943,9 @@ lpfc_mbx_cmpl_unreg_vpi(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 	}
 
 	set_bit(FC_VPORT_NEEDS_REG_VPI, &vport->fc_flag);
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	vport->vpi_state &= ~LPFC_VPI_REGISTERED;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	mempool_free(pmb, phba->mbox_mem_pool);
 	lpfc_cleanup_vports_rrqs(vport, NULL);
 	/*
@@ -4010,9 +4010,9 @@ lpfc_mbx_cmpl_reg_vpi(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 	}
 
 	clear_bit(FC_VPORT_NEEDS_REG_VPI, &vport->fc_flag);
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	vport->vpi_state |= LPFC_VPI_REGISTERED;
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	vport->num_disc_nodes = 0;
 	/* go thru NPR list and issue ELS PLOGIs */
 	if (atomic_read(&vport->fc_npr_cnt))
@@ -5620,9 +5620,9 @@ lpfc_findnode_did(struct lpfc_vport *vport, uint32_t did)
 	struct lpfc_nodelist *ndlp;
 	unsigned long iflags;
 
-	spin_lock_irqsave(shost->host_lock, iflags);
+	spin_lock_irqsave(&shost->host_lock, iflags);
 	ndlp = __lpfc_findnode_did(vport, did);
-	spin_unlock_irqrestore(shost->host_lock, iflags);
+	spin_unlock_irqrestore(&shost->host_lock, iflags);
 	return ndlp;
 }
 
@@ -6426,9 +6426,9 @@ lpfc_findnode_wwpn(struct lpfc_vport *vport, struct lpfc_name *wwpn)
 	struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
 	struct lpfc_nodelist *ndlp;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	ndlp = __lpfc_find_node(vport, lpfc_filter_by_wwpn, wwpn);
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	return ndlp;
 }
 
@@ -6444,9 +6444,9 @@ lpfc_findnode_rpi(struct lpfc_vport *vport, uint16_t rpi)
 	struct lpfc_nodelist *ndlp;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	ndlp = __lpfc_findnode_rpi(vport, rpi);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return ndlp;
 }
 
@@ -6786,9 +6786,9 @@ lpfc_unregister_fcf_prep(struct lpfc_hba *phba)
 				lpfc_sli4_unreg_all_rpis(vports[i]);
 			lpfc_mbx_unreg_vpi(vports[i]);
 			shost = lpfc_shost_from_vport(vports[i]);
-			spin_lock_irq(shost->host_lock);
+			spin_lock_irq(&shost->host_lock);
 			vports[i]->vpi_state &= ~LPFC_VPI_REGISTERED;
-			spin_unlock_irq(shost->host_lock);
+			spin_unlock_irq(&shost->host_lock);
 			set_bit(FC_VPORT_NEEDS_INIT_VPI, &vports[i]->fc_flag);
 		}
 	lpfc_destroy_vport_work_array(phba, vports);
@@ -6801,9 +6801,9 @@ lpfc_unregister_fcf_prep(struct lpfc_hba *phba)
 			lpfc_sli4_unreg_all_rpis(phba->pport);
 		lpfc_mbx_unreg_vpi(phba->pport);
 		shost = lpfc_shost_from_vport(phba->pport);
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		phba->pport->vpi_state &= ~LPFC_VPI_REGISTERED;
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		set_bit(FC_VPORT_NEEDS_INIT_VPI, &phba->pport->fc_flag);
 	}
 
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 23355f12fbffa4..6460127bcc7b0f 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -3833,9 +3833,9 @@ lpfc_offline_prep(struct lpfc_hba *phba, int mbx_action)
 			if (test_bit(FC_UNLOADING, &vports[i]->load_flag))
 				continue;
 			shost = lpfc_shost_from_vport(vports[i]);
-			spin_lock_irq(shost->host_lock);
+			spin_lock_irq(&shost->host_lock);
 			vports[i]->vpi_state &= ~LPFC_VPI_REGISTERED;
-			spin_unlock_irq(shost->host_lock);
+			spin_unlock_irq(&shost->host_lock);
 			set_bit(FC_VPORT_NEEDS_REG_VPI, &vports[i]->fc_flag);
 			clear_bit(FC_VFI_REGISTERED, &vports[i]->fc_flag);
 
@@ -3926,9 +3926,9 @@ lpfc_offline(struct lpfc_hba *phba)
 	if (vports != NULL)
 		for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) {
 			shost = lpfc_shost_from_vport(vports[i]);
-			spin_lock_irq(shost->host_lock);
+			spin_lock_irq(&shost->host_lock);
 			vports[i]->work_port_events = 0;
-			spin_unlock_irq(shost->host_lock);
+			spin_unlock_irq(&shost->host_lock);
 			set_bit(FC_OFFLINE_MODE, &vports[i]->fc_flag);
 		}
 	lpfc_destroy_vport_work_array(phba, vports);
@@ -4930,7 +4930,7 @@ int lpfc_scan_finished(struct Scsi_Host *shost, unsigned long time)
 	struct lpfc_hba   *phba = vport->phba;
 	int stat = 0;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 
 	if (test_bit(FC_UNLOADING, &vport->load_flag)) {
 		stat = 1;
@@ -4965,7 +4965,7 @@ int lpfc_scan_finished(struct Scsi_Host *shost, unsigned long time)
 	stat = 1;
 
 finished:
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 	return stat;
 }
 
@@ -9226,9 +9226,9 @@ lpfc_post_init_setup(struct lpfc_hba *phba)
 	lpfc_host_attrib_init(shost);
 
 	if (phba->cfg_poll & DISABLE_FCP_RING_INT) {
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		lpfc_poll_start_timer(phba);
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 	}
 
 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 8a795c65e3c3e8..8dd876a1a78d3d 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -2848,7 +2848,8 @@ lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
 	}
 out:
 	if (err_type == BGS_GUARD_ERR_MASK) {
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x1);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 		phba->bg_guard_err_cnt++;
 		lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
@@ -2857,7 +2858,8 @@ lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
 				sum, guard_tag);
 
 	} else if (err_type == BGS_REFTAG_ERR_MASK) {
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x3);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 
 		phba->bg_reftag_err_cnt++;
@@ -2867,7 +2869,8 @@ lpfc_calc_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd)
 				ref_tag, start_ref_tag);
 
 	} else if (err_type == BGS_APPTAG_ERR_MASK) {
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x2);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 
 		phba->bg_apptag_err_cnt++;
@@ -2970,7 +2973,8 @@ lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd,
 
 	if (lpfc_bgs_get_guard_err(bgstat)) {
 		ret = 1;
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x1);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 		phba->bg_guard_err_cnt++;
 		lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
@@ -2983,7 +2987,8 @@ lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd,
 
 	if (lpfc_bgs_get_reftag_err(bgstat)) {
 		ret = 1;
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x3);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 		phba->bg_reftag_err_cnt++;
 		lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
@@ -2996,7 +3001,8 @@ lpfc_parse_bg_err(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd,
 
 	if (lpfc_bgs_get_apptag_err(bgstat)) {
 		ret = 1;
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x2);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 		phba->bg_apptag_err_cnt++;
 		lpfc_printf_log(phba, KERN_WARNING, LOG_FCP | LOG_BG,
@@ -4272,7 +4278,7 @@ lpfc_fcp_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
 	if (vport->cfg_max_scsicmpl_time &&
 	    time_after(jiffies, lpfc_cmd->start_time +
 	    msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		if (ndlp) {
 			if (ndlp->cmd_qdepth >
 				atomic_read(&ndlp->cmd_pending) &&
@@ -4285,7 +4291,7 @@ lpfc_fcp_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
 
 			ndlp->last_change_time = jiffies;
 		}
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 	}
 	lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
 
@@ -4552,7 +4558,7 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 	if (vport->cfg_max_scsicmpl_time &&
 	   time_after(jiffies, lpfc_cmd->start_time +
 		msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) {
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		if (pnode) {
 			if (pnode->cmd_qdepth >
 				atomic_read(&pnode->cmd_pending) &&
@@ -4565,7 +4571,7 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 
 			pnode->last_change_time = jiffies;
 		}
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 	}
 	lpfc_scsi_unprep_dma_buf(phba, lpfc_cmd);
 
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index cfa4371169f13d..cd285e87c278c2 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -2886,9 +2886,9 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 		!test_bit(FC_UNLOADING, &phba->pport->load_flag) &&
 		!pmb->u.mb.mbxStatus) {
 		shost = lpfc_shost_from_vport(vport);
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		vport->vpi_state |= LPFC_VPI_REGISTERED;
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		clear_bit(FC_VPORT_NEEDS_REG_VPI, &vport->fc_flag);
 	}
 
diff --git a/drivers/scsi/mac53c94.c b/drivers/scsi/mac53c94.c
index de2bd860b9d7cf..613bb204b6afa0 100644
--- a/drivers/scsi/mac53c94.c
+++ b/drivers/scsi/mac53c94.c
@@ -107,7 +107,7 @@ static int mac53c94_host_reset(struct scsi_cmnd *cmd)
 	struct dbdma_regs __iomem *dma = state->dma;
 	unsigned long flags;
 
-	spin_lock_irqsave(cmd->device->host->host_lock, flags);
+	spin_lock_irqsave(&cmd->device->host->host_lock, flags);
 
 	writel((RUN|PAUSE|FLUSH|WAKE) << 16, &dma->control);
 	writeb(CMD_SCSI_RESET, &regs->command);	/* assert RST */
@@ -117,7 +117,7 @@ static int mac53c94_host_reset(struct scsi_cmnd *cmd)
 	mac53c94_init(state);
 	writeb(CMD_NOP, &regs->command);
 
-	spin_unlock_irqrestore(cmd->device->host->host_lock, flags);
+	spin_unlock_irqrestore(&cmd->device->host->host_lock, flags);
 	return SUCCESS;
 }
 
@@ -182,9 +182,9 @@ static irqreturn_t do_mac53c94_interrupt(int irq, void *dev_id)
 	unsigned long flags;
 	struct Scsi_Host *dev = ((struct fsc_state *) dev_id)->current_req->device->host;
 	
-	spin_lock_irqsave(dev->host_lock, flags);
+	spin_lock_irqsave(&dev->host_lock, flags);
 	mac53c94_interrupt(irq, dev_id);
-	spin_unlock_irqrestore(dev->host_lock, flags);
+	spin_unlock_irqrestore(&dev->host_lock, flags);
 	return IRQ_HANDLED;
 }
 
diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c
index 9476a0d2c72de3..a4079866f747ef 100644
--- a/drivers/scsi/megaraid.c
+++ b/drivers/scsi/megaraid.c
@@ -1592,9 +1592,11 @@ mega_cmd_done(adapter_t *adapter, u8 completed[], int nstatus, int status)
 						epthru->reqsensearea, 14);
 
 					cmd->result = SAM_STAT_CHECK_CONDITION;
-				} else
-					scsi_build_sense(cmd, 0,
-							 ABORTED_COMMAND, 0, 0);
+				} else {
+					scsi_set_sense(cmd, 0,
+						 ABORTED_COMMAND,
+						 NO_ADDITIONAL_SENSE_INFORMATION);
+				}
 			}
 			break;
 
diff --git a/drivers/scsi/megaraid/mega_common.h b/drivers/scsi/megaraid/mega_common.h
index 2ad0aa2f837d10..8e58599d5685f8 100644
--- a/drivers/scsi/megaraid/mega_common.h
+++ b/drivers/scsi/megaraid/mega_common.h
@@ -37,8 +37,6 @@
 #define PCI_CONF_AMISIG64		0xa4
 
 #define MEGA_SCSI_INQ_EVPD		1
-#define MEGA_INVALID_FIELD_IN_CDB	0x24
-
 
 /**
  * scb_t - scsi command control block
diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c
index ce89032a5a7441..b7319282e3c5b7 100644
--- a/drivers/scsi/megaraid/megaraid_mbox.c
+++ b/drivers/scsi/megaraid/megaraid_mbox.c
@@ -1577,8 +1577,8 @@ megaraid_mbox_build_cmd(adapter_t *adapter, struct scsi_cmnd *scp,
 			}
 
 			if (scp->cmnd[1] & MEGA_SCSI_INQ_EVPD) {
-				scsi_build_sense(scp, 0, ILLEGAL_REQUEST,
-						 MEGA_INVALID_FIELD_IN_CDB, 0);
+				scsi_set_sense(scp, 0, ILLEGAL_REQUEST,
+					       INVALID_FIELD_IN_CDB);
 				return NULL;
 			}
 
@@ -2311,9 +2311,11 @@ megaraid_mbox_dpc(unsigned long devp)
 						epthru->reqsensearea, 14);
 
 					scp->result = SAM_STAT_CHECK_CONDITION;
-				} else
-					scsi_build_sense(scp, 0,
-							 ABORTED_COMMAND, 0, 0);
+				} else {
+					scsi_set_sense(scp, 0,
+						ABORTED_COMMAND,
+						NO_ADDITIONAL_SENSE_INFORMATION);
+				}
 			}
 			break;
 
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index b95f187297aed8..1064c5baae8e49 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -2274,11 +2274,11 @@ megasas_check_and_restore_queue_depth(struct megasas_instance *instance)
 	    && atomic_read(&instance->fw_outstanding) <
 	    instance->throttlequeuedepth + 1) {
 
-		spin_lock_irqsave(instance->host->host_lock, flags);
+		spin_lock_irqsave(&instance->host->host_lock, flags);
 		instance->flag &= ~MEGASAS_FW_BUSY;
 
 		instance->host->can_queue = instance->cur_can_queue;
-		spin_unlock_irqrestore(instance->host->host_lock, flags);
+		spin_unlock_irqrestore(&instance->host->host_lock, flags);
 	}
 }
 
@@ -2955,13 +2955,13 @@ static enum scsi_timeout_action megasas_reset_timer(struct scsi_cmnd *scmd)
 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
 	if (!(instance->flag & MEGASAS_FW_BUSY)) {
 		/* FW is busy, throttle IO */
-		spin_lock_irqsave(instance->host->host_lock, flags);
+		spin_lock_irqsave(&instance->host->host_lock, flags);
 
 		instance->host->can_queue = instance->throttlequeuedepth;
 		instance->last_time = jiffies;
 		instance->flag |= MEGASAS_FW_BUSY;
 
-		spin_unlock_irqrestore(instance->host->host_lock, flags);
+		spin_unlock_irqrestore(&instance->host->host_lock, flags);
 	}
 	return SCSI_EH_RESET_TIMER;
 }
@@ -3727,7 +3727,7 @@ megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd,
 		if ((opcode == MR_DCMD_LD_MAP_GET_INFO)
 			&& (cmd->frame->dcmd.mbox.b[1] == 1)) {
 			fusion->fast_path_io = 0;
-			spin_lock_irqsave(instance->host->host_lock, flags);
+			spin_lock_irqsave(&instance->host->host_lock, flags);
 			status = cmd->frame->hdr.cmd_status;
 			instance->map_update_cmd = NULL;
 			if (status != MFI_STAT_OK) {
@@ -3737,7 +3737,7 @@ megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd,
 				else {
 					megasas_return_cmd(instance, cmd);
 					spin_unlock_irqrestore(
-						instance->host->host_lock,
+						&instance->host->host_lock,
 						flags);
 					break;
 				}
@@ -3762,7 +3762,7 @@ megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd,
 				megasas_set_ld_removed_by_fw(instance);
 
 			megasas_sync_map_info(instance);
-			spin_unlock_irqrestore(instance->host->host_lock,
+			spin_unlock_irqrestore(&instance->host->host_lock,
 					       flags);
 
 			break;
@@ -3778,7 +3778,7 @@ megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd,
 		if ((opcode == MR_DCMD_SYSTEM_PD_MAP_GET_INFO) &&
 			(cmd->frame->dcmd.mbox.b[0] == 1)) {
 
-			spin_lock_irqsave(instance->host->host_lock, flags);
+			spin_lock_irqsave(&instance->host->host_lock, flags);
 			status = cmd->frame->hdr.cmd_status;
 			instance->jbod_seq_cmd = NULL;
 			megasas_return_cmd(instance, cmd);
@@ -3791,7 +3791,7 @@ megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd,
 			} else
 				instance->use_seqnum_jbod_fp = false;
 
-			spin_unlock_irqrestore(instance->host->host_lock, flags);
+			spin_unlock_irqrestore(&instance->host->host_lock, flags);
 			break;
 		}
 
@@ -7886,7 +7886,9 @@ megasas_resume(struct device *dev)
 			goto fail_init_mfi;
 	}
 
-	if (megasas_get_ctrl_info(instance) != DCMD_SUCCESS)
+	scoped_guard(mutex, &instance->reset_mutex)
+		rval = megasas_get_ctrl_info(instance);
+	if (rval != DCMD_SUCCESS)
 		goto fail_init_mfi;
 
 	tasklet_init(&instance->isr_tasklet, instance->instancet->tasklet,
diff --git a/drivers/scsi/mesh.c b/drivers/scsi/mesh.c
index dc1402b321da49..1883ba08af7db0 100644
--- a/drivers/scsi/mesh.c
+++ b/drivers/scsi/mesh.c
@@ -1016,9 +1016,9 @@ static irqreturn_t do_mesh_interrupt(int irq, void *dev_id)
 	struct mesh_state *ms = dev_id;
 	struct Scsi_Host *dev = ms->host;
 	
-	spin_lock_irqsave(dev->host_lock, flags);
+	spin_lock_irqsave(&dev->host_lock, flags);
 	mesh_interrupt(ms);
-	spin_unlock_irqrestore(dev->host_lock, flags);
+	spin_unlock_irqrestore(&dev->host_lock, flags);
 	return IRQ_HANDLED;
 }
 
@@ -1708,7 +1708,7 @@ static int mesh_host_reset(struct scsi_cmnd *cmd)
 
 	printk(KERN_DEBUG "mesh_host_reset\n");
 
-	spin_lock_irqsave(ms->host->host_lock, flags);
+	spin_lock_irqsave(&ms->host->host_lock, flags);
 
 	if (ms->dma_started)
 		halt_dma(ms);
@@ -1734,7 +1734,7 @@ static int mesh_host_reset(struct scsi_cmnd *cmd)
 	/* Complete pending commands */
 	handle_reset(ms);
 	
-	spin_unlock_irqrestore(ms->host->host_lock, flags);
+	spin_unlock_irqrestore(&ms->host->host_lock, flags);
 	return SUCCESS;
 }
 
@@ -1771,14 +1771,14 @@ static int mesh_suspend(struct macio_dev *mdev, pm_message_t mesg)
 		return 0;
 
 	scsi_block_requests(ms->host);
-	spin_lock_irqsave(ms->host->host_lock, flags);
+	spin_lock_irqsave(&ms->host->host_lock, flags);
 	while(ms->phase != idle) {
-		spin_unlock_irqrestore(ms->host->host_lock, flags);
+		spin_unlock_irqrestore(&ms->host->host_lock, flags);
 		msleep(10);
-		spin_lock_irqsave(ms->host->host_lock, flags);
+		spin_lock_irqsave(&ms->host->host_lock, flags);
 	}
 	ms->phase = sleeping;
-	spin_unlock_irqrestore(ms->host->host_lock, flags);
+	spin_unlock_irqrestore(&ms->host->host_lock, flags);
 	disable_irq(ms->meshintr);
 	set_mesh_power(ms, 0);
 
@@ -1795,9 +1795,9 @@ static int mesh_resume(struct macio_dev *mdev)
 
 	set_mesh_power(ms, 1);
 	mesh_init(ms);
-	spin_lock_irqsave(ms->host->host_lock, flags);
+	spin_lock_irqsave(&ms->host->host_lock, flags);
 	mesh_start(ms);
-	spin_unlock_irqrestore(ms->host->host_lock, flags);
+	spin_unlock_irqrestore(&ms->host->host_lock, flags);
 	enable_irq(ms->meshintr);
 	scsi_unblock_requests(ms->host);
 
@@ -1818,7 +1818,7 @@ static int mesh_shutdown(struct macio_dev *mdev)
 	unsigned long flags;
 
        	printk(KERN_INFO "resetting MESH scsi bus(es)\n");
-	spin_lock_irqsave(ms->host->host_lock, flags);
+	spin_lock_irqsave(&ms->host->host_lock, flags);
        	mr = ms->mesh;
 	out_8(&mr->intr_mask, 0);
 	out_8(&mr->interrupt, INT_ERROR | INT_EXCEPTION | INT_CMDDONE);
@@ -1826,7 +1826,7 @@ static int mesh_shutdown(struct macio_dev *mdev)
 	mesh_flush_io(mr);
 	udelay(30);
 	out_8(&mr->bus_status1, 0);
-	spin_unlock_irqrestore(ms->host->host_lock, flags);
+	spin_unlock_irqrestore(&ms->host->host_lock, flags);
 
 	return 0;
 }
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h b/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h
index 33dd303c97bb55..7cf16a5c15b720 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h
@@ -72,6 +72,12 @@
 #define MPI3_SECURITY_PGAD_SLOT_GROUP_SHIFT		(8)
 #define MPI3_SECURITY_PGAD_SLOT_MASK                    (0x000000ff)
 #define MPI3_INSTANCE_PGAD_INSTANCE_MASK                (0x0000ffff)
+#define MPI3_INSTANCE_PGAD_INSTANCE_SHIFT               (0)
+#define MPI3_INTERFACE_PGAD_INTERFACE_MASK              (0x0000000f)
+#define MPI3_INTERFACE_PGAD_INTERFACE_SHIFT             (0)
+#define MPI3_INTERFACE_PGAD_INTERFACE_MPI               (0)
+#define MPI3_INTERFACE_PGAD_INTERFACE_NVME_VD           (1)
+#define MPI3_INTERFACE_PGAD_INTERFACE_NVME_PD           (2)
 struct mpi3_config_request {
 	__le16             host_tag;
 	u8                 ioc_use_only02;
@@ -492,10 +498,31 @@ struct mpi3_man10_istwi_ctrlr_entry {
 };
 
 #define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_MASK         (0x000c)
-#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_100K         (0x0000)
-#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_400K         (0x0004)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_100_KHZ		(0x0000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_400_KHZ		(0x0004)
 #define MPI3_MAN10_ISTWI_CTRLR_FLAGS_TARGET_ENABLED          (0x0002)
 #define MPI3_MAN10_ISTWI_CTRLR_FLAGS_INITIATOR_ENABLED         (0x0001)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I2C_GLITCH_FLTR_MASK	(0xc000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I2C_GLITCH_FLTR_SHIFT      (14)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I2C_GLITCH_FLTR_50_NS      (0x0000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I2C_GLITCH_FLTR_10_NS      (0x4000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I2C_GLITCH_FLTR_5_NS       (0x8000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I2C_GLITCH_FLTR_0_NS       (0xc000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_TYPE_MASK              (0x3000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_TYPE_SHIFT             (12)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_TYPE_I2C               (0x0000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_TYPE_I3C               (0x1000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_TYPE_AUTO              (0x2000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I3C_MAX_DATA_RATE_MASK     (0x0e00)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I3C_MAX_DATA_RATE_SHIFT    (9)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I3C_MAX_DATA_RATE_12_5_MHZ (0x0000)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I3C_MAX_DATA_RATE_8_MHZ    (0x0200)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I3C_MAX_DATA_RATE_6_MHZ    (0x0400)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I3C_MAX_DATA_RATE_4_MHZ    (0x0600)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_I3C_MAX_DATA_RATE_2_MHZ    (0x0800)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_MASK             (0x000c)
+#define MPI3_MAN10_ISTWI_CTRLR_FLAGS_BUS_SPEED_SHIFT            (0)
+
 #ifndef MPI3_MAN10_ISTWI_CTRLR_MAX
 #define MPI3_MAN10_ISTWI_CTRLR_MAX          (1)
 #endif
@@ -1027,6 +1054,16 @@ struct mpi3_io_unit_page5 {
 #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SWITCH_ATTACHED       (0x02)
 #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_DIRECT_AND_EXPANDER   (0x03)
 #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_DIRECT_AND_SWITCH     (0x03)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_HDD_SPINDOWN_MASK     (0xc000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_HDD_SPINDOWN_NONE     (0x0000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_HDD_SPINDOWN_ALL      (0x4000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_HDD_SPINDOWN_FILTERED (0x8000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_HDD_SPINDOWN_RESERVED (0xc000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SYNC_CACHE_MASK       (0x3000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SYNC_CACHE_ALL        (0x0000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SYNC_CACHE_FILTERED   (0x1000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SYNC_CACHE_NONE       (0x2000)
+#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SYNC_CACHE_RESERVED   (0x3000)
 #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SATA_HDD_MASK         (0x0300)
 #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SATA_HDD_SHIFT        (8)
 #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SAS_HDD_MASK          (0x00c0)
@@ -1069,7 +1106,8 @@ struct mpi3_io_unit_page8 {
 	struct mpi3_config_page_header         header;
 	u8                                 sb_mode;
 	u8                                 sb_state;
-	__le16                             reserved0a;
+	u8                                 flags;
+	u8				   reserved0b;
 	u8                                 num_slots;
 	u8                                 slots_available;
 	u8                                 current_key_encryption_algo;
@@ -1088,6 +1126,8 @@ struct mpi3_io_unit_page8 {
 #define MPI3_IOUNIT8_SBSTATE_SVN_UPDATE_PENDING   (0x04)
 #define MPI3_IOUNIT8_SBSTATE_KEY_UPDATE_PENDING   (0x02)
 #define MPI3_IOUNIT8_SBSTATE_SECURE_BOOT_ENABLED  (0x01)
+#define MPI3_IOUNIT8_FLAGS_FWQR_CAPABLE			(0x80)
+#define MPI3_IOUNIT8_FLAGS_FWQR_SECURED			(0x40)
 #define MPI3_IOUNIT8_SBMODE_CURRENT_KEY_IOUNIT17	(0x10)
 #define MPI3_IOUNIT8_SBMODE_HARD_SECURE_RECERTIFIED	(0x08)
 struct mpi3_io_unit_page9 {
@@ -1174,10 +1214,16 @@ struct mpi3_io_unit_page12 {
 #define MPI3_IOUNIT12_FLAGS_NUMPASSES_32           (0x00000200)
 #define MPI3_IOUNIT12_FLAGS_NUMPASSES_64           (0x00000300)
 #define MPI3_IOUNIT12_FLAGS_PASSPERIOD_MASK        (0x00000003)
+#define MPI3_IOUNIT12_FLAGS_PASSPERIOD_SHIFT       (0)
 #define MPI3_IOUNIT12_FLAGS_PASSPERIOD_DISABLED    (0x00000000)
 #define MPI3_IOUNIT12_FLAGS_PASSPERIOD_500US       (0x00000001)
 #define MPI3_IOUNIT12_FLAGS_PASSPERIOD_1MS         (0x00000002)
 #define MPI3_IOUNIT12_FLAGS_PASSPERIOD_2MS         (0x00000003)
+#define MPI3_IOUNIT12_FLAGS_INTERFACE_MASK         (0x0000000c)
+#define MPI3_IOUNIT12_FLAGS_INTERFACE_SHIFT        (2)
+#define MPI3_IOUNIT12_FLAGS_INTERFACE_MPI          (0x00000000)
+#define MPI3_IOUNIT12_FLAGS_INTERFACE_NVME_VD      (0x00000004)
+#define MPI3_IOUNIT12_FLAGS_INTERFACE_NVME_PD      (0x00000008)
 #ifndef MPI3_IOUNIT13_FUNC_MAX
 #define MPI3_IOUNIT13_FUNC_MAX                                     (1)
 #endif
@@ -1238,6 +1284,7 @@ struct mpi3_io_unit_page15 {
 #define MPI3_IOUNIT15_PAGEVERSION                                   (0x00)
 #define MPI3_IOUNIT15_FLAGS_EPRINIT_INITREQUIRED                    (0x04)
 #define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_MASK                         (0x03)
+#define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_SHIFT                        (0)
 #define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_NOT_SUPPORTED                (0x00)
 #define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_WITHOUT_POWER_BRAKE_GPIO     (0x01)
 #define MPI3_IOUNIT15_FLAGS_EPRSUPPORT_WITH_POWER_BRAKE_GPIO        (0x02)
@@ -1255,6 +1302,9 @@ struct mpi3_io_unit_page17 {
 	__le32                             current_key[];
 };
 #define MPI3_IOUNIT17_PAGEVERSION		(0x00)
+#define MPI3_IOUNIT17_FLAGS_KEYROOT_MASK	(0x01)
+#define MPI3_IOUNIT17_FLAGS_KEYROOT_HW		(0x00)
+#define MPI3_IOUNIT17_FLAGS_KEYROOT_FW		(0x01)
 struct mpi3_io_unit_page18 {
 	struct mpi3_config_page_header		header;
 	u8					flags;
@@ -1640,11 +1690,28 @@ struct mpi3_security_page3 {
 };
 
 #define MPI3_SECURITY3_PAGEVERSION               (0x00)
-#define MPI3_SECURITY3_FLAGS_TYPE_MASK           (0x0f)
+#define MPI3_SECURITY3_FLAGS_TYPE_MASK           (0x1f)
 #define MPI3_SECURITY3_FLAGS_TYPE_SHIFT          (0)
 #define MPI3_SECURITY3_FLAGS_TYPE_NOT_VALID      (0)
 #define MPI3_SECURITY3_FLAGS_TYPE_MLDSA_PRIVATE  (1)
 #define MPI3_SECURITY3_FLAGS_TYPE_MLDSA_PUBLIC   (2)
+union mpi3_security_digest {
+	__le32                             dword[16];
+	__le16                             word[32];
+	u8                                 byte[64];
+};
+struct mpi3_security_page4 {
+	struct mpi3_config_page_header         header;
+	__le32                             reserved08[2];
+	union mpi3_security_mac               mac;
+	union mpi3_security_nonce             nonce;
+	u8                                 num_digests;
+	u8                                 hash_algorithm;
+	__le16                             reserved92;
+	__le32                             reserved94[3];
+	union mpi3_security_digest		digest[];
+};
+#define MPI3_SECURITY4_PAGEVERSION               (0x00)
 struct mpi3_security_page10 {
 	struct mpi3_config_page_header         header;
 	__le32                             reserved08[2];
@@ -2074,7 +2141,7 @@ struct mpi3_sas_phy3_phy_event_config {
 #define MPI3_SASPHY3_EVENT_CODE_LCCONN_TIME                 (0xd5)
 #define MPI3_SASPHY3_EVENT_CODE_SSP_TX_START_TRANSMIT       (0xd6)
 #define MPI3_SASPHY3_EVENT_CODE_SATA_TX_START               (0xd7)
-#define MPI3_SASPHY3_EVENT_CODE_SMP_TX_START_TRANSMT        (0xd8)
+#define MPI3_SASPHY3_EVENT_CODE_SMP_TX_START_TRANSMIT       (0xd8)
 #define MPI3_SASPHY3_EVENT_CODE_TX_SMP_BREAK_CONN           (0xd9)
 #define MPI3_SASPHY3_EVENT_CODE_SSP_RX_START_RECEIVE        (0xda)
 #define MPI3_SASPHY3_EVENT_CODE_SATA_RX_START_RECEIVE       (0xdb)
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_image.h b/drivers/scsi/mpi3mr/mpi/mpi30_image.h
index 62ddf094d46c95..5fa09fa79358e4 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_image.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_image.h
@@ -18,7 +18,7 @@ struct mpi3_hash_exclusion_format {
 	__le32                     size;
 };
 
-#define MPI3_IMAGE_HASH_EXCUSION_NUM                           (4)
+#define MPI3_IMAGE_HASH_EXCLUSION_NUM                           (4)
 struct mpi3_component_image_header {
 	__le32                            signature0;
 	__le32                            load_address;
@@ -42,7 +42,7 @@ struct mpi3_component_image_header {
 	union mpi3_version_union             rmc_interface_version;
 	union mpi3_version_union             etp_interface_version;
 	struct mpi3_comp_image_version        component_image_version;
-	struct mpi3_hash_exclusion_format     hash_exclusion[MPI3_IMAGE_HASH_EXCUSION_NUM];
+	struct mpi3_hash_exclusion_format     hash_exclusion[MPI3_IMAGE_HASH_EXCLUSION_NUM];
 	__le32                            next_image_header_offset;
 	union mpi3_version_union             security_version;
 	__le32                            reserved84[31];
@@ -347,7 +347,8 @@ struct mpi3_encrypted_hash_entry {
 struct mpi3_encrypted_hash_data {
 	u8                                  image_version;
 	u8                                  num_hash;
-	__le16                              reserved02;
+	u8                                  fw_num_hash;
+	u8                                  reserved03;
 	__le32                              reserved04;
 	struct mpi3_encrypted_hash_entry        encrypted_hash_entry[MPI3_ENCRYPTED_HASH_ENTRY_MAX];
 };
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
index 68efa0d5134596..aa42fba7f930f5 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h
@@ -428,10 +428,10 @@ struct mpi3_event_data_sas_discovery {
 #define MPI3_EVENT_SAS_DISC_FLAGS_IN_PROGRESS                   (0x01)
 #define MPI3_EVENT_SAS_DISC_RC_STARTED                          (0x01)
 #define MPI3_EVENT_SAS_DISC_RC_COMPLETED                        (0x02)
-#define MPI3_SAS_DISC_STATUS_MAX_ENCLOSURES_EXCEED            (0x80000000)
-#define MPI3_SAS_DISC_STATUS_MAX_EXPANDERS_EXCEED             (0x40000000)
-#define MPI3_SAS_DISC_STATUS_MAX_DEVICES_EXCEED               (0x20000000)
-#define MPI3_SAS_DISC_STATUS_MAX_TOPO_PHYS_EXCEED             (0x10000000)
+#define MPI3_SAS_DISC_STATUS_MAX_ENCLOSURES_EXCEEDED            (0x80000000)
+#define MPI3_SAS_DISC_STATUS_MAX_EXPANDERS_EXCEEDED             (0x40000000)
+#define MPI3_SAS_DISC_STATUS_MAX_DEVICES_EXCEEDED               (0x20000000)
+#define MPI3_SAS_DISC_STATUS_MAX_TOPO_PHYS_EXCEEDED             (0x10000000)
 #define MPI3_SAS_DISC_STATUS_INVALID_CEI                      (0x00010000)
 #define MPI3_SAS_DISC_STATUS_FECEI_MISMATCH                   (0x00008000)
 #define MPI3_SAS_DISC_STATUS_MULTIPLE_DEVICES_IN_SLOT         (0x00004000)
@@ -965,7 +965,7 @@ struct mpi3_ci_download_reply {
 	u8                                 flags;
 	u8                                 cache_dirty;
 	u8                                 pending_count;
-	u8                                 reserved13;
+	u8                                 additional_flags;
 };
 
 #define MPI3_CI_DOWNLOAD_FLAGS_DOWNLOAD_IN_PROGRESS                  (0x80)
@@ -979,6 +979,11 @@ struct mpi3_ci_download_reply {
 #define MPI3_CI_DOWNLOAD_FLAGS_ACTIVATION_STATUS_ONLINE_PENDING      (0x04)
 #define MPI3_CI_DOWNLOAD_FLAGS_ACTIVATION_STATUS_OFFLINE_PENDING     (0x06)
 #define MPI3_CI_DOWNLOAD_FLAGS_COMPATIBLE                            (0x01)
+#define MPI3_CI_DOWNLOAD_ADDITIONALFLAGS_REDUNDANCYRESTORATION_MASK       (0x03)
+#define MPI3_CI_DOWNLOAD_ADDITIONALFLAGS_REDUNDANCYRESTORATION_SHIFT      (0)
+#define MPI3_CI_DOWNLOAD_ADDITIONALFLAGS_REDUNDANCYRESTORATION_NONE       (0x00)
+#define MPI3_CI_DOWNLOAD_ADDITIONALFLAGS_REDUNDANCYRESTORATION_PRIMARY    (0x01)
+#define MPI3_CI_DOWNLOAD_ADDITIONALFLAGS_REDUNDANCYRESTORATION_SECONDARY  (0x02)
 struct mpi3_ci_upload_request {
 	__le16                             host_tag;
 	u8                                 ioc_use_only02;
diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_transport.h b/drivers/scsi/mpi3mr/mpi/mpi30_transport.h
index 290a1f5c29244c..794ecc77894568 100644
--- a/drivers/scsi/mpi3mr/mpi/mpi30_transport.h
+++ b/drivers/scsi/mpi3mr/mpi/mpi30_transport.h
@@ -18,7 +18,7 @@ union mpi3_version_union {
 
 #define MPI3_VERSION_MAJOR                                              (3)
 #define MPI3_VERSION_MINOR                                              (0)
-#define MPI3_VERSION_UNIT                                               (39)
+#define MPI3_VERSION_UNIT                                               (41)
 #define MPI3_VERSION_DEV                                                (0)
 #define MPI3_DEVHANDLE_INVALID                                          (0xffff)
 struct mpi3_sysif_oper_queue_indexes {
diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h
index c25525fe0671a8..d6e16707fd97c8 100644
--- a/drivers/scsi/mpi3mr/mpi3mr.h
+++ b/drivers/scsi/mpi3mr/mpi3mr.h
@@ -56,8 +56,8 @@ extern struct list_head mrioc_list;
 extern int prot_mask;
 extern atomic64_t event_counter;
 
-#define MPI3MR_DRIVER_VERSION	"8.17.0.3.50"
-#define MPI3MR_DRIVER_RELDATE	"09-January-2026"
+#define MPI3MR_DRIVER_VERSION	"8.18.0.8.50"
+#define MPI3MR_DRIVER_RELDATE	"26-June-2026"
 
 #define MPI3MR_DRIVER_NAME	"mpi3mr"
 #define MPI3MR_DRIVER_LICENSE	"GPL"
@@ -125,6 +125,7 @@ extern atomic64_t event_counter;
 #define MPI3MR_RESETTM_TIMEOUT			60
 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT	5
 #define MPI3MR_TSUPDATE_INTERVAL		900
+#define MPI3MR_EARLY_TSUPDATE_SECONDS		60
 #define MPI3MR_DEFAULT_SHUTDOWN_TIME		120
 #define	MPI3MR_RAID_ERRREC_RESET_TIMEOUT	180
 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT	180
@@ -168,6 +169,7 @@ extern atomic64_t event_counter;
 
 #define MPI3MR_DEFAULT_MDTS	(128 * 1024)
 #define MPI3MR_DEFAULT_PGSZEXP         (12)
+#define MPI3MR_MAX_PGSZEXP             (27)
 
 /* Command retry count definitions */
 #define MPI3MR_DEV_RMHS_RETRY_COUNT 3
@@ -177,7 +179,7 @@ extern atomic64_t event_counter;
 #define MPI3MR_DEFAULT_SDEV_QD	32
 
 /* Definitions for Threaded IRQ poll*/
-#define MPI3MR_IRQ_POLL_SLEEP			20
+#define MPI3MR_IRQ_POLL_SLEEP			2
 #define MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT		8
 
 /* Definitions for the controller security status*/
@@ -1118,6 +1120,7 @@ struct scmd_priv {
  * @evtack_cmds_bitmap: Event Ack bitmap
  * @delayed_evtack_cmds_list: Delayed event acknowledgment list
  * @ts_update_counter: Timestamp update counter
+ * @early_ts_sync_done: Early (1 min) timestamp sync completed after load
  * @ts_update_interval: Timestamp update interval
  * @reset_in_progress: Reset in progress flag
  * @unrecoverable: Controller unrecoverable flag
@@ -1318,6 +1321,7 @@ struct mpi3mr_ioc {
 	struct list_head delayed_evtack_cmds_list;
 
 	u16 ts_update_counter;
+	u8 early_ts_sync_done;
 	u16 ts_update_interval;
 	u8 reset_in_progress;
 	u8 unrecoverable;
@@ -1410,6 +1414,9 @@ struct mpi3mr_ioc {
 	struct dma_pool *trace_buf_pool;
 	struct segments *trace_buf;
 	u8 invalid_io_comp;
+	bool is_unload;
+	bool skip_dev_shutdown_on_unload;
+
 
 };
 
diff --git a/drivers/scsi/mpi3mr/mpi3mr_app.c b/drivers/scsi/mpi3mr/mpi3mr_app.c
index 1353a8ff9c85d5..5184d9d516a882 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_app.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_app.c
@@ -141,25 +141,27 @@ void mpi3mr_alloc_diag_bufs(struct mpi3mr_ioc *mrioc)
 		trace_min_size = fw_min_size = MPI3MR_DEFAULT_HDB_MIN_SZ;
 
 	} else {
-		trace_size = driver_pg1.host_diag_trace_max_size * 1024;
-		trace_dec_size = driver_pg1.host_diag_trace_decrement_size
+		trace_size = le16_to_cpu(driver_pg1.host_diag_trace_max_size) * 1024;
+		trace_dec_size = le16_to_cpu(driver_pg1.host_diag_trace_decrement_size)
 			 * 1024;
-		trace_min_size = driver_pg1.host_diag_trace_min_size * 1024;
-		fw_size = driver_pg1.host_diag_fw_max_size * 1024;
-		fw_dec_size = driver_pg1.host_diag_fw_decrement_size * 1024;
-		fw_min_size = driver_pg1.host_diag_fw_min_size * 1024;
+		trace_min_size = le16_to_cpu(driver_pg1.host_diag_trace_min_size) * 1024;
+		fw_size = le16_to_cpu(driver_pg1.host_diag_fw_max_size) * 1024;
+		fw_dec_size = le16_to_cpu(driver_pg1.host_diag_fw_decrement_size) * 1024;
+		fw_min_size = le16_to_cpu(driver_pg1.host_diag_fw_min_size) * 1024;
 		dprint_init(mrioc,
 		    "%s:trace diag buffer sizes read from driver\n"
 		    "page1: maximum size = %dKB, decrement size = %dKB\n"
-		    ", minimum size = %dKB\n", __func__, driver_pg1.host_diag_trace_max_size,
-		    driver_pg1.host_diag_trace_decrement_size,
-		    driver_pg1.host_diag_trace_min_size);
+		    ", minimum size = %dKB\n", __func__,
+		    le16_to_cpu(driver_pg1.host_diag_trace_max_size),
+		    le16_to_cpu(driver_pg1.host_diag_trace_decrement_size),
+		    le16_to_cpu(driver_pg1.host_diag_trace_min_size));
 		dprint_init(mrioc,
 		    "%s:firmware diag buffer sizes read from driver\n"
 		    "page1: maximum size = %dKB, decrement size = %dKB\n"
-		    ", minimum size = %dKB\n", __func__, driver_pg1.host_diag_fw_max_size,
-		    driver_pg1.host_diag_fw_decrement_size,
-		    driver_pg1.host_diag_fw_min_size);
+		    ", minimum size = %dKB\n", __func__,
+		    le16_to_cpu(driver_pg1.host_diag_fw_max_size),
+		    le16_to_cpu(driver_pg1.host_diag_fw_decrement_size),
+		    le16_to_cpu(driver_pg1.host_diag_fw_min_size));
 		if ((trace_size == 0) && (fw_size == 0))
 			return;
 	}
@@ -179,6 +181,12 @@ void mpi3mr_alloc_diag_bufs(struct mpi3mr_ioc *mrioc)
 		    mpi3mr_alloc_trace_buffer(mrioc, trace_size)) {
 
 			retry = true;
+
+			if (!trace_dec_size || trace_dec_size > trace_size) {
+				retry = false;
+				goto retry_fw;
+			}
+
 			trace_size -= trace_dec_size;
 			dprint_init(mrioc, "trace diag buffer allocation failed\n"
 			"retrying smaller size %dKB\n", trace_size / 1024);
@@ -211,11 +219,13 @@ void mpi3mr_alloc_diag_bufs(struct mpi3mr_ioc *mrioc)
 			diag_buffer->size = fw_size;
 		} else {
 			retry = true;
-			fw_size -= fw_dec_size;
-			dprint_init(mrioc, "%s:trace diag buffer allocation failed,\n"
-					"retrying smaller size %dKB\n",
-					__func__, fw_size / 1024);
-			goto retry_fw;
+			if (fw_dec_size && fw_dec_size <= fw_size) {
+				fw_size -= fw_dec_size;
+				dprint_init(mrioc, "%s:trace diag buffer allocation failed,\n"
+						"retrying smaller size %dKB\n",
+						__func__, fw_size / 1024);
+				goto retry_fw;
+			}
 		}
 	}
 }
@@ -1244,7 +1254,7 @@ static long mpi3mr_bsg_query_hdb(struct mpi3mr_ioc *mrioc,
 
 	length = (sizeof(*hbd_status) + ((MPI3MR_MAX_NUM_HDB - 1) *
 		    sizeof(*hbd_status_entry)));
-	hbd_status = kmalloc(length, GFP_KERNEL);
+	hbd_status = kzalloc(length, GFP_KERNEL);
 	if (!hbd_status)
 		return -ENOMEM;
 	hbd_status_entry = &hbd_status->entry[0];
@@ -1466,7 +1476,8 @@ static long mpi3mr_bsg_pel_enable(struct mpi3mr_ioc *mrioc,
 static long mpi3mr_get_all_tgt_info(struct mpi3mr_ioc *mrioc,
 	struct bsg_job *job)
 {
-	u16 num_devices = 0, i = 0, size;
+	u16 num_devices = 0, i = 0;
+	size_t size;
 	unsigned long flags;
 	struct mpi3mr_tgt_dev *tgtdev;
 	struct mpi3mr_device_map_info *devmap_info = NULL;
@@ -1492,8 +1503,8 @@ static long mpi3mr_get_all_tgt_info(struct mpi3mr_ioc *mrioc,
 		return 0;
 	}
 
-	kern_entrylen = num_devices * sizeof(*devmap_info);
-	size = sizeof(u64) + kern_entrylen;
+	kern_entrylen = (uint32_t)num_devices * sizeof(*devmap_info);
+	size = sizeof(u64) + (size_t)kern_entrylen;
 	alltgt_info = kzalloc(size, GFP_KERNEL);
 	if (!alltgt_info)
 		return -ENOMEM;
@@ -2384,7 +2395,7 @@ static long mpi3mr_bsg_process_mpt_cmds(struct bsg_job *job)
 	long rval = -EINVAL;
 	struct mpi3mr_ioc *mrioc = NULL;
 	u8 *mpi_req = NULL, *sense_buff_k = NULL;
-	u8 mpi_msg_size = 0;
+	u32 mpi_msg_size = 0;
 	struct mpi3mr_bsg_packet *bsg_req = NULL;
 	struct mpi3mr_bsg_mptcmd *karg;
 	struct mpi3mr_buf_entry *buf_entries = NULL;
@@ -2538,7 +2549,15 @@ static long mpi3mr_bsg_process_mpt_cmds(struct bsg_job *job)
 				rval = -EINVAL;
 				goto out;
 			}
-			memcpy(mpi_req, sgl_iter, buf_entries->buf_len);
+			if (sgl_iter + mpi_msg_size >
+			    dout_buf + job->request_payload.payload_len) {
+				dprint_bsg_err(mrioc, "%s: MPI request buf exceeds dout_buf\n",
+					       __func__);
+				mutex_unlock(&mrioc->bsg_cmds.mutex);
+				rval = -EINVAL;
+				goto out;
+			}
+			memcpy(mpi_req, sgl_iter, mpi_msg_size);
 			break;
 		default:
 			invalid_be = 1;
@@ -2737,10 +2756,12 @@ static long mpi3mr_bsg_process_mpt_cmds(struct bsg_job *job)
 	}
 	if (block_io) {
 		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle);
-		if (tgtdev && tgtdev->starget && tgtdev->starget->hostdata) {
-			stgt_priv = (struct mpi3mr_stgt_priv_data *)
-			    tgtdev->starget->hostdata;
-			atomic_inc(&stgt_priv->block_io);
+		if (tgtdev) {
+			if (tgtdev->starget && tgtdev->starget->hostdata) {
+				stgt_priv = (struct mpi3mr_stgt_priv_data *)
+				    tgtdev->starget->hostdata;
+				atomic_inc(&stgt_priv->block_io);
+			}
 			mpi3mr_tgtdev_put(tgtdev);
 		}
 	}
@@ -2774,6 +2795,8 @@ static long mpi3mr_bsg_process_mpt_cmds(struct bsg_job *job)
 		dprint_bsg_err(mrioc,
 		    "%s: posting bsg request is failed\n", __func__);
 		rval = -EAGAIN;
+		if (block_io && stgt_priv)
+			atomic_dec(&stgt_priv->block_io);
 		goto out_unlock;
 	}
 	wait_for_completion_timeout(&mrioc->bsg_cmds.done,
@@ -2935,7 +2958,8 @@ static long mpi3mr_bsg_process_mpt_cmds(struct bsg_job *job)
 void mpi3mr_app_save_logdata_th(struct mpi3mr_ioc *mrioc, char *event_data,
 	u16 event_data_size)
 {
-	u32 index = mrioc->logdata_buf_idx, sz;
+	u32 index = mrioc->logdata_buf_idx;
+	size_t entry_payload_len, sz;
 	struct mpi3mr_logdata_entry *entry;
 
 	if (!(mrioc->logdata_buf))
@@ -2944,7 +2968,12 @@ void mpi3mr_app_save_logdata_th(struct mpi3mr_ioc *mrioc, char *event_data,
 	entry = (struct mpi3mr_logdata_entry *)
 		(mrioc->logdata_buf + (index * mrioc->logdata_entry_sz));
 	entry->valid_entry = 1;
-	sz = min(mrioc->logdata_entry_sz, event_data_size);
+	if (mrioc->logdata_entry_sz > MPI3MR_BSG_LOGDATA_ENTRY_HEADER_SZ)
+		entry_payload_len = (size_t)mrioc->logdata_entry_sz -
+		    MPI3MR_BSG_LOGDATA_ENTRY_HEADER_SZ;
+	else
+		entry_payload_len = 0;
+	sz = min_t(size_t, entry_payload_len, event_data_size);
 	memcpy(entry->data, event_data, sz);
 	mrioc->logdata_buf_idx =
 		((++index) % MPI3MR_BSG_LOGDATA_MAX_ENTRIES);
diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c
index 681868716ebdbf..f0d3cd398dd00c 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_fw.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c
@@ -124,11 +124,16 @@ void mpi3mr_build_zero_len_sge(void *paddr)
 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 	dma_addr_t phys_addr)
 {
+	u64 offset;
+
 	if (!phys_addr)
 		return NULL;
 
+	offset = phys_addr - mrioc->reply_buf_dma;
+
 	if ((phys_addr < mrioc->reply_buf_dma) ||
-	    (phys_addr > mrioc->reply_buf_dma_max_address))
+	    (phys_addr > mrioc->reply_buf_dma_max_address - mrioc->reply_sz) ||
+	    do_div(offset, mrioc->reply_sz))
 		return NULL;
 
 	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
@@ -137,9 +142,19 @@ void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 	dma_addr_t phys_addr)
 {
+	u64 offset;
+
 	if (!phys_addr)
 		return NULL;
 
+	offset = phys_addr - mrioc->sense_buf_dma;
+
+	if (phys_addr < mrioc->sense_buf_dma ||
+	    (phys_addr > mrioc->sense_buf_dma +
+	     (mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ) - MPI3MR_SENSE_BUF_SZ) ||
+	    do_div(offset, MPI3MR_SENSE_BUF_SZ))
+		return NULL;
+
 	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 }
 
@@ -404,7 +419,8 @@ static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 				scsi_normalize_sense(sense_buf, sense_count,
 				    &sshdr);
 				mpi3mr_scsisense_trigger(mrioc, sshdr.sense_key,
-				    sshdr.asc, sshdr.ascq);
+						scsi_sense_asc(&sshdr),
+						scsi_sense_ascq(&sshdr));
 			}
 		}
 		mpi3mr_reply_trigger(mrioc, masked_ioc_status, ioc_loginfo);
@@ -474,6 +490,12 @@ int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 		return 0;
 	}
 
+	/*
+	 * Ensure that the descriptor payload is read only after
+	 * the phase bit check is complete.
+	 */
+	dma_rmb();
+
 	do {
 		if (mrioc->unrecoverable || mrioc->io_admin_reset_sync)
 			break;
@@ -494,6 +516,13 @@ int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 		if ((le16_to_cpu(reply_desc->reply_flags) &
 		    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 			break;
+
+		/*
+		 * Ensure that the descriptor payload is read only after
+		 * the phase bit check is complete.
+		 */
+		dma_rmb();
+
 		if (threshold_comps == MPI3MR_THRESHOLD_REPLY_COUNT) {
 			writel(admin_reply_ci,
 			    &mrioc->sysif_regs->admin_reply_queue_ci);
@@ -554,6 +583,9 @@ int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 	struct mpi3_default_reply_descriptor *reply_desc;
 	u16 req_q_idx = 0, reply_qidx, threshold_comps = 0;
 
+	if (!op_reply_q)
+		return 0;
+
 	reply_qidx = op_reply_q->qid - 1;
 
 	if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
@@ -565,15 +597,33 @@ int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 	reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 	if ((le16_to_cpu(reply_desc->reply_flags) &
 	    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
+		/* Recheck under in_use before releasing, to avoid a reclaim race */
+		dma_rmb();
+		if ((le16_to_cpu(reply_desc->reply_flags) &
+		    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) == exp_phase)
+			goto process_desc;
 		atomic_dec(&op_reply_q->in_use);
 		return 0;
 	}
+process_desc:
+	/*
+	 * Ensure that the descriptor payload is read only after
+	 * the phase bit check is complete.
+	 */
+	dma_rmb();
 
 	do {
 		if (mrioc->unrecoverable || mrioc->io_admin_reset_sync)
 			break;
 
 		req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
+
+		if (unlikely(req_q_idx >= mrioc->num_op_req_q)) {
+			ioc_err(mrioc, "Invalid request queue id %d, skipping reply\n",
+			    req_q_idx + 1);
+			goto next_reply;
+		}
+
 		op_req_q = &mrioc->req_qinfo[req_q_idx];
 
 		WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
@@ -582,8 +632,9 @@ int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 
 		if (reply_dma)
 			mpi3mr_repost_reply_buf(mrioc, reply_dma);
-		num_op_reply++;
 		threshold_comps++;
+next_reply:
+		num_op_reply++;
 
 		if (++reply_ci == op_reply_q->num_replies) {
 			reply_ci = 0;
@@ -593,8 +644,19 @@ int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 		reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 
 		if ((le16_to_cpu(reply_desc->reply_flags) &
-		    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
+		    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
+			dma_rmb();
+			if ((le16_to_cpu(reply_desc->reply_flags) &
+			    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) == exp_phase)
+				goto reply_ready;
 			break;
+		}
+reply_ready:
+		/*
+		 * Ensure that the descriptor payload is read only after
+		 * the phase bit check is complete.
+		 */
+		dma_rmb();
 #ifndef CONFIG_PREEMPT_RT
 		/*
 		 * Exit completion loop to avoid CPU lockup
@@ -656,6 +718,7 @@ static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 {
 	struct mpi3mr_intr_info *intr_info = privdata;
 	struct mpi3mr_ioc *mrioc;
+	struct op_reply_qinfo *op_reply_q;
 	u16 midx;
 	u32 num_admin_replies = 0, num_op_reply = 0;
 
@@ -671,9 +734,9 @@ static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 
 	if (!midx)
 		num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
-	if (intr_info->op_reply_q)
-		num_op_reply = mpi3mr_process_op_reply_q(mrioc,
-		    intr_info->op_reply_q);
+	op_reply_q = READ_ONCE(intr_info->op_reply_q);
+	if (op_reply_q)
+		num_op_reply = mpi3mr_process_op_reply_q(mrioc, op_reply_q);
 
 	if (num_admin_replies || num_op_reply)
 		return IRQ_HANDLED;
@@ -686,6 +749,7 @@ static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 {
 	struct mpi3mr_intr_info *intr_info = privdata;
+	struct op_reply_qinfo *op_reply_q;
 	int ret;
 
 	if (!intr_info)
@@ -698,11 +762,12 @@ static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 	 * If more IOs are expected, schedule IRQ polling thread.
 	 * Otherwise exit from ISR.
 	 */
-	if ((threaded_isr_poll == false) || !intr_info->op_reply_q)
+	op_reply_q = READ_ONCE(intr_info->op_reply_q);
+	if ((threaded_isr_poll == false) || !op_reply_q)
 		return ret;
 
-	if (!intr_info->op_reply_q->enable_irq_poll ||
-	    !atomic_read(&intr_info->op_reply_q->pend_ios))
+	if (!op_reply_q->enable_irq_poll ||
+	    !atomic_read(&op_reply_q->pend_ios))
 		return ret;
 
 	disable_irq_nosync(intr_info->os_irq);
@@ -724,12 +789,19 @@ static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 {
 	struct mpi3mr_intr_info *intr_info = privdata;
 	struct mpi3mr_ioc *mrioc;
+	struct op_reply_qinfo *op_reply_q;
 	u16 midx;
 	u32 num_op_reply = 0;
 
-	if (!intr_info || !intr_info->op_reply_q)
+	if (!intr_info)
 		return IRQ_NONE;
 
+	op_reply_q = READ_ONCE(intr_info->op_reply_q);
+	if (!op_reply_q) {
+		enable_irq(intr_info->os_irq);
+		return IRQ_HANDLED;
+	}
+
 	mrioc = intr_info->mrioc;
 	midx = intr_info->msix_index;
 
@@ -738,19 +810,23 @@ static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 		if (!mrioc->intr_enabled || mrioc->unrecoverable)
 			break;
 
+		op_reply_q = READ_ONCE(intr_info->op_reply_q);
+		if (!op_reply_q)
+			break;
+
 		if (!midx)
 			mpi3mr_process_admin_reply_q(mrioc);
-		if (intr_info->op_reply_q)
-			num_op_reply +=
-			    mpi3mr_process_op_reply_q(mrioc,
-				intr_info->op_reply_q);
+		num_op_reply +=
+		    mpi3mr_process_op_reply_q(mrioc, op_reply_q);
+		if (!atomic_read(&op_reply_q->pend_ios))
+			break;
 
-		usleep_range(MPI3MR_IRQ_POLL_SLEEP, MPI3MR_IRQ_POLL_SLEEP + 1);
+		usleep_range(MPI3MR_IRQ_POLL_SLEEP, 10 * MPI3MR_IRQ_POLL_SLEEP);
 
-	} while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
-	    (num_op_reply < mrioc->max_host_ios));
+	} while (num_op_reply < mrioc->max_host_ios);
 
-	intr_info->op_reply_q->enable_irq_poll = false;
+	if (op_reply_q)
+		op_reply_q->enable_irq_poll = false;
 	enable_irq(intr_info->os_irq);
 
 	return IRQ_HANDLED;
@@ -1934,10 +2010,6 @@ static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
 	int size;
 	struct segments *segments;
 
-	segments = mrioc->req_qinfo[q_idx].q_segments;
-	if (!segments)
-		return;
-
 	if (mrioc->enable_segqueue) {
 		size = MPI3MR_OP_REQ_Q_SEG_SIZE;
 		if (mrioc->req_qinfo[q_idx].q_segment_list) {
@@ -1951,6 +2023,10 @@ static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
 		size = mrioc->req_qinfo[q_idx].segment_qd *
 		    mrioc->facts.op_req_sz;
 
+	segments = mrioc->req_qinfo[q_idx].q_segments;
+	if (!segments)
+		return;
+
 	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
 		if (!segments[j].segment)
 			continue;
@@ -1977,10 +2053,17 @@ static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
 	u16 j;
 	int size;
 	struct segments *segments;
+	u16 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(q_idx, mrioc->op_reply_q_offset);
 
-	segments = mrioc->op_reply_qinfo[q_idx].q_segments;
-	if (!segments)
-		return;
+	/*
+	 * Stop the ISR/poll thread from picking up this queue before its
+	 * segments are freed below, and wait for any in-flight handler
+	 * that already has the old pointer to finish using it.
+	 */
+	if (midx < mrioc->intr_info_count) {
+		WRITE_ONCE(mrioc->intr_info[midx].op_reply_q, NULL);
+		synchronize_irq(pci_irq_vector(mrioc->pdev, midx));
+	}
 
 	if (mrioc->enable_segqueue) {
 		size = MPI3MR_OP_REP_Q_SEG_SIZE;
@@ -1995,6 +2078,10 @@ static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
 		size = mrioc->op_reply_qinfo[q_idx].segment_qd *
 		    mrioc->op_reply_desc_sz;
 
+	segments = mrioc->op_reply_qinfo[q_idx].q_segments;
+	if (!segments)
+		return;
+
 	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
 		if (!segments[j].segment)
 			continue;
@@ -2461,7 +2548,7 @@ static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
 {
 	int retval = 0;
-	u16 num_queues = 0, i = 0, msix_count_op_q = 1;
+	u16 num_queues = 0, i = 0, j = 0, msix_count_op_q = 1;
 	u32 ioc_status;
 	enum mpi3mr_iocstate ioc_state;
 
@@ -2513,6 +2600,13 @@ static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
 		}
 	}
 
+	if (i < num_queues) {
+		for (j = i; j < num_queues; j++) {
+			mpi3mr_free_op_req_q_segments(mrioc, j);
+			mpi3mr_free_op_reply_q_segments(mrioc, j);
+		}
+	}
+
 	if (i == 0) {
 		/* Not even one queue is created successfully*/
 		retval = -1;
@@ -2534,11 +2628,19 @@ static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
 
 	return retval;
 out_failed:
-	kfree(mrioc->req_qinfo);
-	mrioc->req_qinfo = NULL;
+	if (mrioc->req_qinfo) {
+		for (j = 0; j < i; j++) {
+			mpi3mr_free_op_req_q_segments(mrioc, j);
+			mpi3mr_free_op_reply_q_segments(mrioc, j);
+		}
+		kfree(mrioc->req_qinfo);
+		mrioc->req_qinfo = NULL;
+	}
+	mrioc->num_op_req_q = 0;
 
 	kfree(mrioc->op_reply_qinfo);
 	mrioc->op_reply_qinfo = NULL;
+	mrioc->num_op_reply_q = 0;
 
 	return retval;
 }
@@ -2582,7 +2684,8 @@ int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
 	if (mpi3mr_check_req_qfull(op_req_q)) {
 		midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
 		    reply_qidx, mrioc->op_reply_q_offset);
-		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);
+		mpi3mr_process_op_reply_q(mrioc,
+		    READ_ONCE(mrioc->intr_info[midx].op_reply_q));
 
 		if (mpi3mr_check_req_qfull(op_req_q)) {
 
@@ -2875,8 +2978,9 @@ static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
  * @work: work struct
  *
  * Watch dog work periodically executed (1 second interval) to
- * monitor firmware fault and to issue periodic timer sync to
- * the firmware.
+ * monitor firmware fault and perform timestamp synchronization
+ * to firmware, with an early sync 1 minute after load followed
+ * by periodic updates at ts_update_interval seconds (default 15 minutes).
  *
  * Return: Nothing.
  */
@@ -2922,11 +3026,23 @@ static void mpi3mr_watchdog_work(struct work_struct *work)
 	}
 
 	if (!(mrioc->facts.ioc_capabilities &
-		MPI3_IOCFACTS_CAPABILITY_NON_SUPERVISOR_IOC) &&
-		(mrioc->ts_update_counter++ >= mrioc->ts_update_interval)) {
+		MPI3_IOCFACTS_CAPABILITY_NON_SUPERVISOR_IOC)) {
+		if (!mrioc->early_ts_sync_done) {
+			/*
+			 * Send time sync 1 min after load
+			 */
+			if (mrioc->ts_update_counter++ >=
+					MPI3MR_EARLY_TSUPDATE_SECONDS) {
+				mrioc->early_ts_sync_done = 1;
+				mrioc->ts_update_counter = 0;
+				mpi3mr_sync_timestamp(mrioc);
+			}
+		} else if (mrioc->ts_update_counter++ >=
+				mrioc->ts_update_interval) {
+			mrioc->ts_update_counter = 0;
+			mpi3mr_sync_timestamp(mrioc);
+		}
 
-		mrioc->ts_update_counter = 0;
-		mpi3mr_sync_timestamp(mrioc);
 	}
 
 	if ((mrioc->prepare_for_reset) &&
@@ -4112,26 +4228,35 @@ static int mpi3mr_repost_diag_bufs(struct mpi3mr_ioc *mrioc)
 }
 
 /**
- * mpi3mr_read_tsu_interval - Update time stamp interval
+ * mpi3mr_read_driver_page1 - Read Driver Page 1 parameters
  * @mrioc: Adapter instance reference
  *
- * Update time stamp interval if its defined in driver page 1,
- * otherwise use default value.
+ * Reads and caches Driver Page 1 parameters such as
+ * timestamp update interval and driver behavior flags.
  *
  * Return: Nothing
  */
 static void
-mpi3mr_read_tsu_interval(struct mpi3mr_ioc *mrioc)
+mpi3mr_read_driver_page1(struct mpi3mr_ioc *mrioc)
 {
 	struct mpi3_driver_page1 driver_pg1;
 	u16 pg_sz = sizeof(driver_pg1);
 	int retval = 0;
 
 	mrioc->ts_update_interval = MPI3MR_TSUPDATE_INTERVAL;
+	mrioc->skip_dev_shutdown_on_unload = 0;
 
 	retval = mpi3mr_cfg_get_driver_pg1(mrioc, &driver_pg1, pg_sz);
-	if (!retval && driver_pg1.time_stamp_update)
+
+	if (retval)
+		return;
+
+	if (driver_pg1.time_stamp_update)
 		mrioc->ts_update_interval = (driver_pg1.time_stamp_update * 60);
+
+	mrioc->skip_dev_shutdown_on_unload =
+		(le32_to_cpu(driver_pg1.flags) &
+		 MPI3_DRIVER1_FLAGS_DEVICE_SHUTDOWN_ON_UNLOAD_DISABLE) ? 1 : 0;
 }
 
 /**
@@ -4437,7 +4562,7 @@ int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc)
 		goto out_failed_noretry;
 	}
 
-	mpi3mr_read_tsu_interval(mrioc);
+	mpi3mr_read_driver_page1(mrioc);
 	mpi3mr_print_ioc_info(mrioc);
 
 	dprint_init(mrioc, "allocating host diag buffers\n");
@@ -4609,7 +4734,7 @@ int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume)
 		goto out_failed_noretry;
 	}
 
-	mpi3mr_read_tsu_interval(mrioc);
+	mpi3mr_read_driver_page1(mrioc);
 	mpi3mr_print_ioc_info(mrioc);
 
 	if (is_resume) {
@@ -5094,8 +5219,16 @@ static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
 		return;
 	}
 
-	shutdown_action = MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL |
-	    MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
+	shutdown_action = MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
+
+	if (!(mrioc->is_unload && mrioc->skip_dev_shutdown_on_unload))
+		shutdown_action |=
+			MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
+	else
+		ioc_info(mrioc,
+		    "The shutdown request is issued without the device shutdown bit set\n"
+		    "as indicated by the controller configuration\n");
+
 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 	ioc_config |= shutdown_action;
 
diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c
index 0f738044871883..6b0156e54d4543 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_os.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_os.c
@@ -233,10 +233,11 @@ static void mpi3mr_fwevt_add_to_list(struct mpi3mr_ioc *mrioc,
 {
 	unsigned long flags;
 
-	if (!mrioc->fwevt_worker_thread)
-		return;
-
 	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+	if (!mrioc->fwevt_worker_thread) {
+		spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+		return;
+	}
 	/* get fwevt reference count while adding it to fwevt_list */
 	mpi3mr_fwevt_get(fwevt);
 	INIT_LIST_HEAD(&fwevt->list);
@@ -282,32 +283,6 @@ void mpi3mr_hdb_trigger_data_event(struct mpi3mr_ioc *mrioc,
 	mpi3mr_fwevt_add_to_list(mrioc, fwevt);
 }
 
-/**
- * mpi3mr_fwevt_del_from_list - Delete firmware event from list
- * @mrioc: Adapter instance reference
- * @fwevt: Firmware event reference
- *
- * Delete the given firmware event from the firmware event list.
- *
- * Return: Nothing.
- */
-static void mpi3mr_fwevt_del_from_list(struct mpi3mr_ioc *mrioc,
-	struct mpi3mr_fwevt *fwevt)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
-	if (!list_empty(&fwevt->list)) {
-		list_del_init(&fwevt->list);
-		/*
-		 * Put fwevt reference count after
-		 * removing it from fwevt_list
-		 */
-		mpi3mr_fwevt_put(fwevt);
-	}
-	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
-}
-
 /**
  * mpi3mr_dequeue_fwevt - Dequeue firmware event from the list
  * @mrioc: Adapter instance reference
@@ -327,11 +302,7 @@ static struct mpi3mr_fwevt *mpi3mr_dequeue_fwevt(
 		fwevt = list_first_entry(&mrioc->fwevt_list,
 		    struct mpi3mr_fwevt, list);
 		list_del_init(&fwevt->list);
-		/*
-		 * Put fwevt reference count after
-		 * removing it from fwevt_list
-		 */
-		mpi3mr_fwevt_put(fwevt);
+
 	}
 	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 
@@ -365,6 +336,11 @@ static void mpi3mr_cancel_work(struct mpi3mr_fwevt *fwevt)
 		 */
 		mpi3mr_fwevt_put(fwevt);
 	}
+
+	/*
+	 * Drop the reference count that was acquired by the caller.
+	 */
+	mpi3mr_fwevt_put(fwevt);
 }
 
 /**
@@ -379,17 +355,44 @@ static void mpi3mr_cancel_work(struct mpi3mr_fwevt *fwevt)
 void mpi3mr_cleanup_fwevt_list(struct mpi3mr_ioc *mrioc)
 {
 	struct mpi3mr_fwevt *fwevt = NULL;
+	unsigned long flags;
 
+	/*
+	 * Safely read current_event under lock to prevent TOCTOU race
+	 * with the firmware event worker thread.
+	 */
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 	if ((list_empty(&mrioc->fwevt_list) && !mrioc->current_event) ||
-	    !mrioc->fwevt_worker_thread)
+	    !mrioc->fwevt_worker_thread) {
+		spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 		return;
+	}
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 
 	while ((fwevt = mpi3mr_dequeue_fwevt(mrioc)))
 		mpi3mr_cancel_work(fwevt);
 
-	if (mrioc->current_event) {
-		fwevt = mrioc->current_event;
+	/*
+	 * Safely read current_event under lock to prevent TOCTOU race
+	 * with the firmware event worker thread.
+	 */
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+	fwevt = mrioc->current_event;
+	if (fwevt) {
+		/*
+		 * Take a reference to ensure the event is not freed by the
+		 * worker thread while we are evaluating or cancelling it.
+		 */
+		mpi3mr_fwevt_get(fwevt);
+	}
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+
+	if (fwevt) {
+		bool pending_at_sml;
+
 		/*
+		 * Read pending_at_sml under lock to avoid a stale value.
+		 *
 		 * Don't call cancel_work_sync() API for the
 		 * fwevt work if the controller reset is
 		 * get called as part of processing the
@@ -397,8 +400,13 @@ void mpi3mr_cleanup_fwevt_list(struct mpi3mr_ioc *mrioc)
 		 * waiting for device add/remove APIs to complete.
 		 * Otherwise we will see deadlock.
 		 */
-		if (current_work() == &fwevt->work || fwevt->pending_at_sml) {
+		spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+		pending_at_sml = fwevt->pending_at_sml;
+		spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+
+		if (current_work() == &fwevt->work || pending_at_sml) {
 			fwevt->discard = 1;
+			mpi3mr_fwevt_put(fwevt);
 			return;
 		}
 
@@ -912,6 +920,8 @@ void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc,
 	struct mpi3mr_tgt_dev *tgtdev)
 {
 	struct mpi3mr_stgt_priv_data *tgt_priv;
+	unsigned long flags;
+	bool discard = false;
 
 	ioc_info(mrioc, "%s :Removing handle(0x%04x), wwid(0x%016llx)\n",
 	    __func__, tgtdev->dev_handle, (unsigned long long)tgtdev->wwid);
@@ -924,17 +934,27 @@ void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc,
 	if (!mrioc->sas_transport_enabled || (tgtdev->dev_type !=
 	    MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl) {
 		if (tgtdev->starget) {
+			spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+			if (mrioc->stop_drv_processing ||
+			    mrioc->reset_in_progress) {
+				spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+				return;
+			}
 			if (mrioc->current_event)
 				mrioc->current_event->pending_at_sml = 1;
+			spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 			scsi_remove_target(&tgtdev->starget->dev);
 			tgtdev->host_exposed = 0;
+			spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 			if (mrioc->current_event) {
 				mrioc->current_event->pending_at_sml = 0;
-				if (mrioc->current_event->discard) {
-					mpi3mr_print_device_event_notice(mrioc,
-					    false);
-					return;
-				}
+				discard = mrioc->current_event->discard;
+			}
+			spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+			if (discard) {
+				mpi3mr_print_device_event_notice(mrioc,
+				    false);
+				return;
 			}
 		}
 	} else
@@ -962,6 +982,8 @@ static int mpi3mr_report_tgtdev_to_host(struct mpi3mr_ioc *mrioc,
 {
 	int retval = 0;
 	struct mpi3mr_tgt_dev *tgtdev;
+	unsigned long flags;
+	bool discard = false;
 
 	if (mrioc->reset_in_progress || mrioc->pci_err_recovery)
 		return -1;
@@ -978,19 +1000,29 @@ static int mpi3mr_report_tgtdev_to_host(struct mpi3mr_ioc *mrioc,
 	if (!mrioc->sas_transport_enabled || (tgtdev->dev_type !=
 	    MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl){
 		tgtdev->host_exposed = 1;
+		spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+		if (mrioc->stop_drv_processing || mrioc->reset_in_progress) {
+			spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+			tgtdev->host_exposed = 0;
+			goto out;
+		}
 		if (mrioc->current_event)
 			mrioc->current_event->pending_at_sml = 1;
+		spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 		scsi_scan_target(&mrioc->shost->shost_gendev,
 		    mrioc->scsi_device_channel, tgtdev->perst_id,
 		    SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
 		if (!tgtdev->starget)
 			tgtdev->host_exposed = 0;
+		spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 		if (mrioc->current_event) {
 			mrioc->current_event->pending_at_sml = 0;
-			if (mrioc->current_event->discard) {
-				mpi3mr_print_device_event_notice(mrioc, true);
-				goto out;
-			}
+			discard = mrioc->current_event->discard;
+		}
+		spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+		if (discard) {
+			mpi3mr_print_device_event_notice(mrioc, true);
+			goto out;
 		}
 		dprint_event_bh(mrioc,
 		    "exposed target device with handle(0x%04x), perst_id(%d)\n",
@@ -1363,24 +1395,32 @@ static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc,
 		struct mpi3_device0_pcie_format *pcieinf =
 		    &dev_pg0->device_specific.pcie_format;
 		u16 dev_info = le16_to_cpu(pcieinf->device_info);
+		u8 pgsz = MPI3MR_DEFAULT_PGSZEXP;
+		u8 reset_to = MPI3MR_INTADMCMD_TIMEOUT;
+		u8 abort_to = MPI3MR_INTADMCMD_TIMEOUT;
+		u8 fw_pgsz = READ_ONCE(pcieinf->page_size);
 
 		tgtdev->dev_spec.pcie_inf.dev_info = dev_info;
 		tgtdev->dev_spec.pcie_inf.capb =
 		    le32_to_cpu(pcieinf->capabilities);
 		tgtdev->dev_spec.pcie_inf.mdts = MPI3MR_DEFAULT_MDTS;
-		/* 2^12 = 4096 */
-		tgtdev->dev_spec.pcie_inf.pgsz = 12;
+		/* Validate firmware page size to prevent undefined shift behavior */
+		if (fw_pgsz >= MPI3MR_DEFAULT_PGSZEXP && fw_pgsz <= MPI3MR_MAX_PGSZEXP)
+			pgsz = fw_pgsz;
+
 		if (dev_pg0->access_status == MPI3_DEVICE0_ASTATUS_NO_ERRORS) {
 			tgtdev->dev_spec.pcie_inf.mdts =
 			    le32_to_cpu(pcieinf->maximum_data_transfer_size);
-			tgtdev->dev_spec.pcie_inf.pgsz = pcieinf->page_size;
-			tgtdev->dev_spec.pcie_inf.reset_to =
-			    max_t(u8, pcieinf->controller_reset_to,
-			     MPI3MR_INTADMCMD_TIMEOUT);
-			tgtdev->dev_spec.pcie_inf.abort_to =
-			    max_t(u8, pcieinf->nvme_abort_to,
+			reset_to = max_t(u8, pcieinf->controller_reset_to,
+			    MPI3MR_INTADMCMD_TIMEOUT);
+			abort_to = max_t(u8, pcieinf->nvme_abort_to,
 			    MPI3MR_INTADMCMD_TIMEOUT);
 		}
+
+		tgtdev->dev_spec.pcie_inf.pgsz = pgsz;
+		tgtdev->dev_spec.pcie_inf.reset_to = reset_to;
+		tgtdev->dev_spec.pcie_inf.abort_to = abort_to;
+
 		if (tgtdev->dev_spec.pcie_inf.mdts > (1024 * 1024))
 			tgtdev->dev_spec.pcie_inf.mdts = (1024 * 1024);
 		if (((dev_info & MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) !=
@@ -2150,13 +2190,25 @@ static void mpi3mr_fwevt_bh(struct mpi3mr_ioc *mrioc,
 	u16 perst_id, handle, dev_info;
 	struct mpi3_device0_sas_sata_format *sasinf = NULL;
 	unsigned int timeout;
+	unsigned long flags;
 
-	mpi3mr_fwevt_del_from_list(mrioc, fwevt);
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+	if (!list_empty(&fwevt->list)) {
+		list_del_init(&fwevt->list);
+		/*
+		 * Put fwevt reference count after
+		 * removing it from fwevt_list
+		 */
+		mpi3mr_fwevt_put(fwevt);
+	}
 	mrioc->current_event = fwevt;
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 
-	if (mrioc->stop_drv_processing) {
-		dprint_event_bh(mrioc, "ignoring event(0x%02x) in the bottom half handler\n"
-				"due to stop_drv_processing\n", fwevt->event_id);
+	if (mrioc->stop_drv_processing || mrioc->pci_err_recovery) {
+		dprint_event_bh(mrioc,
+				"ignoring event(0x%02x) in the bottom half handler\n"
+				"due to stop_drv_processing or pci_err_recovery\n",
+				fwevt->event_id);
 		goto out;
 	}
 
@@ -2285,9 +2337,12 @@ static void mpi3mr_fwevt_bh(struct mpi3mr_ioc *mrioc,
 		mpi3mr_process_event_ack(mrioc, fwevt->event_id,
 		    fwevt->evt_ctx);
 out:
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+	mrioc->current_event = NULL;
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+
 	/* Put fwevt reference count to neutralize kref_init increment */
 	mpi3mr_fwevt_put(fwevt);
-	mrioc->current_event = NULL;
 }
 
 /**
@@ -2426,7 +2481,8 @@ static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_ioc *mrioc,
 		ioc_info(mrioc,
 		    "%s :dev removal handshake completed successfully: handle(0x%04x)\n",
 		    __func__, drv_cmd->dev_handle);
-		clear_bit(drv_cmd->dev_handle, mrioc->removepend_bitmap);
+		if (drv_cmd->dev_handle < mrioc->facts.max_devhandle)
+			clear_bit(drv_cmd->dev_handle, mrioc->removepend_bitmap);
 	}
 
 	if (!list_empty(&mrioc->delayed_rmhs_list)) {
@@ -2540,12 +2596,29 @@ static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_ioc *mrioc, u16 handle,
 	struct mpi3mr_tgt_dev *tgtdev = NULL;
 	unsigned long flags;
 
+	if (handle >= mrioc->facts.max_devhandle) {
+		ioc_err(mrioc, "dev_remove_hs: handle(0x%04x) >= max_devhandle(0x%04x)\n",
+			handle, mrioc->facts.max_devhandle);
+		if (drv_cmd) {
+			cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
+			drv_cmd->state = MPI3MR_CMD_NOTUSED;
+			drv_cmd->callback = NULL;
+			drv_cmd->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
+			drv_cmd->retry_count = 0;
+			clear_bit(cmd_idx, mrioc->devrem_bitmap);
+		}
+		return;
+	}
+
 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
 	tgtdev = __mpi3mr_get_tgtdev_by_handle(mrioc, handle);
 	if (tgtdev && (iou_rc == MPI3_CTRL_OP_REMOVE_DEVICE))
 		tgtdev->state = MPI3MR_DEV_REMOVE_HS_STARTED;
 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 
+	if (tgtdev)
+		mpi3mr_tgtdev_put(tgtdev);
+
 	if (drv_cmd)
 		goto issue_cmd;
 	do {
@@ -2768,12 +2841,28 @@ static void mpi3mr_pcietopochg_evt_th(struct mpi3mr_ioc *mrioc,
 	struct mpi3_event_data_pcie_topology_change_list *topo_evt =
 	    (struct mpi3_event_data_pcie_topology_change_list *)event_reply->event_data;
 	int i;
-	u16 handle;
-	u8 reason_code;
+	u16 handle, avail_len;
+	u8 reason_code, max_entries, num_entries;
 	struct mpi3mr_tgt_dev *tgtdev = NULL;
 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
 
-	for (i = 0; i < topo_evt->num_entries; i++) {
+	avail_len = event_reply->event_data_length * 4;
+	if (avail_len < offsetof(struct mpi3_event_data_pcie_topology_change_list, port_entry)) {
+		ioc_err(mrioc, "PCIe topology event: event data too small (%u bytes)\n",
+			avail_len);
+		return;
+	}
+	max_entries = (avail_len -
+		      offsetof(struct mpi3_event_data_pcie_topology_change_list, port_entry)) /
+		      sizeof(struct mpi3_event_pcie_topo_port_entry);
+	num_entries = topo_evt->num_entries;
+	if (num_entries > max_entries) {
+		ioc_err(mrioc, "PCIe topology event: num_entries(%d) exceeds max(%d)\n",
+			num_entries, max_entries);
+		return;
+	}
+
+	for (i = 0; i < num_entries; i++) {
 		handle = le16_to_cpu(topo_evt->port_entry[i].attached_dev_handle);
 		if (!handle)
 			continue;
@@ -2834,12 +2923,28 @@ static void mpi3mr_sastopochg_evt_th(struct mpi3mr_ioc *mrioc,
 	struct mpi3_event_data_sas_topology_change_list *topo_evt =
 	    (struct mpi3_event_data_sas_topology_change_list *)event_reply->event_data;
 	int i;
-	u16 handle;
-	u8 reason_code;
+	u16 handle, avail_len;
+	u8 reason_code, max_entries, num_entries;
 	struct mpi3mr_tgt_dev *tgtdev = NULL;
 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
 
-	for (i = 0; i < topo_evt->num_entries; i++) {
+	avail_len = event_reply->event_data_length * 4;
+	if (avail_len < offsetof(struct mpi3_event_data_sas_topology_change_list, phy_entry)) {
+		ioc_err(mrioc, "SAS topology event: event data too small (%u bytes)\n",
+			avail_len);
+		return;
+	}
+	max_entries = (avail_len -
+		      offsetof(struct mpi3_event_data_sas_topology_change_list, phy_entry)) /
+		      sizeof(struct mpi3_event_sas_topo_phy_entry);
+	num_entries = topo_evt->num_entries;
+	if (num_entries > max_entries) {
+		ioc_err(mrioc, "SAS topology event: num_entries(%d) exceeds max(%d)\n",
+			num_entries, max_entries);
+		return;
+	}
+
+	for (i = 0; i < num_entries; i++) {
 		handle = le16_to_cpu(topo_evt->phy_entry[i].attached_dev_handle);
 		if (!handle)
 			continue;
@@ -3109,7 +3214,7 @@ void mpi3mr_add_event_wait_for_device_refresh(struct mpi3mr_ioc *mrioc)
 void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc,
 	struct mpi3_event_notification_reply *event_reply)
 {
-	u16 evt_type, sz;
+	u16 evt_type, sz, avail_reply_room;
 	struct mpi3mr_fwevt *fwevt = NULL;
 	bool ack_req = 0, process_evt_bh = 0;
 
@@ -3170,7 +3275,12 @@ void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc,
 	case MPI3_EVENT_DEVICE_INFO_CHANGED:
 	case MPI3_EVENT_LOG_DATA:
 
-		sz = event_reply->event_data_length * 4;
+		if (mrioc->reply_sz > offsetof(struct mpi3_event_notification_reply, event_data))
+			avail_reply_room = mrioc->reply_sz -
+			    offsetof(struct mpi3_event_notification_reply, event_data);
+		else
+			avail_reply_room = 0;
+		sz = min_t(u16, event_reply->event_data_length * 4, avail_reply_room);
 		mpi3mr_app_save_logdata_th(mrioc,
 			(char *)event_reply->event_data, sz);
 		break;
@@ -3204,7 +3314,12 @@ void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc,
 		dprint_event_th(mrioc,
 		    "scheduling bottom half handler for event(0x%02x) - (0x%08x), ack_required=%d\n",
 		    evt_type, le32_to_cpu(event_reply->event_context), ack_req);
-		sz = event_reply->event_data_length * 4;
+		if (mrioc->reply_sz > offsetof(struct mpi3_event_notification_reply, event_data))
+			avail_reply_room = mrioc->reply_sz -
+			    offsetof(struct mpi3_event_notification_reply, event_data);
+		else
+			avail_reply_room = 0;
+		sz = min_t(u16, event_reply->event_data_length * 4, avail_reply_room);
 		fwevt = mpi3mr_alloc_fwevt(sz);
 		if (!fwevt) {
 			dprint_event_th(mrioc,
@@ -3329,8 +3444,7 @@ static void mpi3mr_setup_eedp(struct mpi3mr_ioc *mrioc,
  * @desc: Sense type
  * @buf: Sense buffer to populate
  * @key: Sense key
- * @asc: Additional sense code
- * @ascq: Additional sense code qualifier
+ * @code: Additional sense code and its qualifier
  *
  * Maps the given sense information into either descriptor or
  * fixed format sense data.
@@ -3338,20 +3452,20 @@ static void mpi3mr_setup_eedp(struct mpi3mr_ioc *mrioc,
  * Return: Nothing
  */
 static inline void mpi3mr_build_sense_buffer(int desc, u8 *buf, u8 key,
-	u8 asc, u8 ascq)
+					     u16 code)
 {
 	if (desc) {
 		buf[0] = 0x72;	/* descriptor, current */
 		buf[1] = key;
-		buf[2] = asc;
-		buf[3] = ascq;
+		buf[2] = scsi_sense_code_asc(code);
+		buf[3] = scsi_sense_code_ascq(code);
 		buf[7] = 0;
 	} else {
 		buf[0] = 0x70;	/* fixed, current */
 		buf[2] = key;
 		buf[7] = 0xa;
-		buf[12] = asc;
-		buf[13] = ascq;
+		buf[12] = scsi_sense_code_asc(code);
+		buf[13] = scsi_sense_code_ascq(code);
 	}
 }
 
@@ -3368,25 +3482,25 @@ static inline void mpi3mr_build_sense_buffer(int desc, u8 *buf, u8 key,
 static void mpi3mr_map_eedp_error(struct scsi_cmnd *scmd,
 	u16 ioc_status)
 {
-	u8 ascq = 0;
+	u16 sense_code;
 
 	switch (ioc_status) {
 	case MPI3_IOCSTATUS_EEDP_GUARD_ERROR:
-		ascq = 0x01;
+		sense_code = LOGICAL_BLOCK_GUARD_CHECK_FAILED;
 		break;
 	case MPI3_IOCSTATUS_EEDP_APP_TAG_ERROR:
-		ascq = 0x02;
+		sense_code = LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED;
 		break;
 	case MPI3_IOCSTATUS_EEDP_REF_TAG_ERROR:
-		ascq = 0x03;
+		sense_code = LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED;
 		break;
 	default:
-		ascq = 0x00;
+		sense_code = ID_CRC_OR_ECC_ERROR;
 		break;
 	}
 
 	mpi3mr_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
-	    0x10, ascq);
+				  sense_code);
 	scmd->result = (DID_ABORT << 16) | SAM_STAT_CHECK_CONDITION;
 }
 
@@ -3447,8 +3561,12 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
 		scsi_reply = mpi3mr_get_reply_virt_addr(mrioc,
 		    *reply_dma);
 		if (!scsi_reply) {
-			panic("%s: scsi_reply is NULL, this shouldn't happen\n",
-			    mrioc->name);
+			ioc_err(mrioc, "scsi_reply is NULL, invalid reply_frame_address\n");
+			/*
+			 * Do not let the caller repost an address that
+			 * failed virt-addr lookup back to the hardware.
+			 */
+			*reply_dma = 0;
 			goto out;
 		}
 		host_tag = le16_to_cpu(scsi_reply->host_tag);
@@ -3471,7 +3589,8 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
 		if (sense_buf) {
 			scsi_normalize_sense(sense_buf, sense_count, &sshdr);
 			mpi3mr_scsisense_trigger(mrioc, sshdr.sense_key,
-			    sshdr.asc, sshdr.ascq);
+						 scsi_sense_asc(&sshdr),
+						 scsi_sense_ascq(&sshdr));
 		}
 		mpi3mr_reply_trigger(mrioc, ioc_status, ioc_loginfo);
 		break;
@@ -3658,7 +3777,8 @@ void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
 			ioc_info(mrioc,
 			    "%s :sense_count 0x%x, sense_key 0x%x ASC 0x%x, ASCQ 0x%x\n",
 			    __func__, sense_count, sshdr.sense_key,
-			    sshdr.asc, sshdr.ascq);
+			    scsi_sense_asc(&sshdr),
+			    scsi_sense_ascq(&sshdr));
 		}
 	}
 out_success:
@@ -3943,7 +4063,7 @@ inline void mpi3mr_poll_pend_io_completions(struct mpi3mr_ioc *mrioc)
 
 	for (i = mrioc->op_reply_q_offset; i < num_of_reply_queues; i++)
 		mpi3mr_process_op_reply_q(mrioc,
-		    mrioc->intr_info[i].op_reply_q);
+		    READ_ONCE(mrioc->intr_info[i].op_reply_q));
 }
 
 /**
@@ -5065,8 +5185,8 @@ static bool mpi3mr_check_return_unmap(struct mpi3mr_ioc *mrioc,
 		    __func__, param_len);
 		scsi_print_command(scmd);
 		scmd->result = SAM_STAT_CHECK_CONDITION;
-		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
-		    0x1A, 0);
+		scsi_set_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
+				      PARAMETER_LIST_LENGTH_ERROR);
 		scsi_done(scmd);
 		return true;
 	}
@@ -5076,8 +5196,8 @@ static bool mpi3mr_check_return_unmap(struct mpi3mr_ioc *mrioc,
 		    __func__, param_len, scsi_bufflen(scmd));
 		scsi_print_command(scmd);
 		scmd->result = SAM_STAT_CHECK_CONDITION;
-		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
-		    0x1A, 0);
+		scsi_set_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
+				      PARAMETER_LIST_LENGTH_ERROR);
 		scsi_done(scmd);
 		return true;
 	}
@@ -5085,8 +5205,8 @@ static bool mpi3mr_check_return_unmap(struct mpi3mr_ioc *mrioc,
 	if (!buf) {
 		scsi_print_command(scmd);
 		scmd->result = SAM_STAT_CHECK_CONDITION;
-		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
-		    0x55, 0x03);
+		scsi_set_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
+				      INSUFFICIENT_RESOURCES);
 		scsi_done(scmd);
 		return true;
 	}
@@ -5099,8 +5219,8 @@ static bool mpi3mr_check_return_unmap(struct mpi3mr_ioc *mrioc,
 		    __func__, desc_len);
 		scsi_print_command(scmd);
 		scmd->result = SAM_STAT_CHECK_CONDITION;
-		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
-		    0x26, 0);
+		scsi_set_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
+				      INVALID_FIELD_IN_PARAMETER_LIST);
 		scsi_done(scmd);
 		kfree(buf);
 		return true;
@@ -5200,7 +5320,8 @@ static enum scsi_qc_status mpi3mr_qcmd(struct Scsi_Host *shost,
 	if (scsi_get_host_state(scmd->device->host) == SHOST_RECOVERY &&
 		scmd->cmnd[0] == TEST_UNIT_READY &&
 		(stgt_priv_data->dev_removed || (dev_handle == MPI3MR_INVALID_DEV_HANDLE))) {
-		scsi_build_sense(scmd, 0, UNIT_ATTENTION, 0x29, 0x07);
+		scsi_set_sense(scmd, 0, UNIT_ATTENTION,
+			       I_T_NEXUS_LOSS_OCCURRED);
 		scsi_done(scmd);
 		goto out;
 	}
@@ -5690,6 +5811,8 @@ static void mpi3mr_remove(struct pci_dev *pdev)
 		return;
 
 	mrioc = shost_priv(shost);
+	mrioc->is_unload = true;
+
 	while (mrioc->reset_in_progress || mrioc->is_driver_loading)
 		ssleep(1);
 
@@ -5822,6 +5945,9 @@ mpi3mr_suspend(struct device *dev)
 		ssleep(1);
 	mrioc->stop_drv_processing = 1;
 	mpi3mr_cleanup_fwevt_list(mrioc);
+	/* Flush any pending discarded event before unmapping PCI resources below. */
+	if (mrioc->fwevt_worker_thread)
+		flush_workqueue(mrioc->fwevt_worker_thread);
 	scsi_block_requests(shost);
 	mpi3mr_stop_watchdog(mrioc);
 	mpi3mr_cleanup_ioc(mrioc);
diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c
index 232af978d73729..f1e34329f5a2b5 100644
--- a/drivers/scsi/mpi3mr/mpi3mr_transport.c
+++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c
@@ -1090,6 +1090,13 @@ void mpi3mr_update_links(struct mpi3mr_ioc *mrioc,
 		return;
 	}
 
+	if (phy_number >= mr_sas_node->num_phys) {
+		spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
+		ioc_err(mrioc, "%s: phy_number(%d) >= num_phys(%d)\n",
+			__func__, phy_number, mr_sas_node->num_phys);
+		return;
+	}
+
 	mr_sas_phy = &mr_sas_node->phy[phy_number];
 	mr_sas_phy->attached_handle = handle;
 	spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
@@ -1216,13 +1223,14 @@ void mpi3mr_sas_host_add(struct mpi3mr_ioc *mrioc)
 	}
 	num_phys = sas_io_unit_pg0->num_phys;
 	kfree(sas_io_unit_pg0);
+	sas_io_unit_pg0 = NULL;
 
 	mrioc->sas_hba.host_node = 1;
 	INIT_LIST_HEAD(&mrioc->sas_hba.sas_port_list);
 	mrioc->sas_hba.parent_dev = &mrioc->shost->shost_gendev;
 	mrioc->sas_hba.phy = kzalloc_objs(struct mpi3mr_sas_phy, num_phys);
 	if (!mrioc->sas_hba.phy)
-		return;
+		goto out;
 
 	mrioc->sas_hba.num_phys = num_phys;
 
@@ -1230,12 +1238,12 @@ void mpi3mr_sas_host_add(struct mpi3mr_ioc *mrioc)
 	    (num_phys * sizeof(struct mpi3_sas_io_unit0_phy_data));
 	sas_io_unit_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_io_unit_pg0)
-		return;
+		goto out_free_phy;
 
 	if (mpi3mr_cfg_get_sas_io_unit_pg0(mrioc, sas_io_unit_pg0, sz)) {
 		ioc_err(mrioc, "failure at %s:%d/%s()!\n",
 		    __FILE__, __LINE__, __func__);
-		goto out;
+		goto out_free_phy;
 	}
 
 	mrioc->sas_hba.handle = 0;
@@ -1249,12 +1257,12 @@ void mpi3mr_sas_host_add(struct mpi3mr_ioc *mrioc)
 		    MPI3_SAS_PHY_PGAD_FORM_PHY_NUMBER, i)) {
 			ioc_err(mrioc, "failure at %s:%d/%s()!\n",
 			    __FILE__, __LINE__, __func__);
-			goto out;
+			goto out_free_phy;
 		}
 		if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
 			ioc_err(mrioc, "failure at %s:%d/%s()!\n",
 			    __FILE__, __LINE__, __func__);
-			goto out;
+			goto out_free_phy;
 		}
 
 		if (!mrioc->sas_hba.handle)
@@ -1264,26 +1272,27 @@ void mpi3mr_sas_host_add(struct mpi3mr_ioc *mrioc)
 
 		if (!(mpi3mr_get_hba_port_by_id(mrioc, port_id)))
 			if (!mpi3mr_alloc_hba_port(mrioc, port_id))
-				goto out;
+				goto out_free_phy;
 
 		mrioc->sas_hba.phy[i].handle = mrioc->sas_hba.handle;
 		mrioc->sas_hba.phy[i].phy_id = i;
 		mrioc->sas_hba.phy[i].hba_port =
 		    mpi3mr_get_hba_port_by_id(mrioc, port_id);
-		mpi3mr_add_host_phy(mrioc, &mrioc->sas_hba.phy[i],
-		    phy_pg0, mrioc->sas_hba.parent_dev);
+		if (mpi3mr_add_host_phy(mrioc, &mrioc->sas_hba.phy[i],
+		    phy_pg0, mrioc->sas_hba.parent_dev))
+			goto out_free_phy;
 	}
 	if ((mpi3mr_cfg_get_dev_pg0(mrioc, &ioc_status, &dev_pg0,
 	    sizeof(dev_pg0), MPI3_DEVICE_PGAD_FORM_HANDLE,
 	    mrioc->sas_hba.handle))) {
 		ioc_err(mrioc, "%s: device page0 read failed\n", __func__);
-		goto out;
+		goto out_free_phy;
 	}
 	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
 		ioc_err(mrioc, "device page read failed for handle(0x%04x), with ioc_status(0x%04x) failure at %s:%d/%s()!\n",
 		    mrioc->sas_hba.handle, ioc_status, __FILE__, __LINE__,
 		    __func__);
-		goto out;
+		goto out_free_phy;
 	}
 	mrioc->sas_hba.enclosure_handle =
 	    le16_to_cpu(dev_pg0.enclosure_handle);
@@ -1306,6 +1315,17 @@ void mpi3mr_sas_host_add(struct mpi3mr_ioc *mrioc)
 				le64_to_cpu(encl_pg0.enclosure_logical_id);
 	}
 
+	goto out;
+
+out_free_phy:
+	for (i = 0; i < mrioc->sas_hba.num_phys; i++) {
+		if (mrioc->sas_hba.phy[i].phy)
+			sas_phy_delete(mrioc->sas_hba.phy[i].phy);
+	}
+	kfree(mrioc->sas_hba.phy);
+	mrioc->sas_hba.phy = NULL;
+	mrioc->sas_hba.num_phys = 0;
+
 out:
 	kfree(sas_io_unit_pg0);
 }
@@ -1330,6 +1350,7 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc,
 	struct mpi3mr_sas_phy *mr_sas_phy, *next;
 	struct mpi3mr_sas_port *mr_sas_port;
 	unsigned long flags;
+	bool discard = false;
 	struct mpi3mr_sas_node *mr_sas_node;
 	struct sas_rphy *rphy;
 	struct mpi3mr_tgt_dev *tgtdev = NULL;
@@ -1429,7 +1450,7 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc,
 
 	port = sas_port_alloc_num(mr_sas_node->parent_dev);
 	if (!port) {
-		ioc_err(mrioc, "failure at %s:%d/%s()!\n",
+		ioc_err(mrioc, "failure at %s:%d/%s() (sas_port_alloc)!\n",
 		    __FILE__, __LINE__, __func__);
 		goto out_fail;
 	}
@@ -1456,19 +1477,51 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc,
 	mr_sas_port->port = port;
 	if (mr_sas_port->remote_identify.device_type == SAS_END_DEVICE) {
 		rphy = sas_end_device_alloc(port);
+		if (!rphy) {
+			ioc_err(mrioc, "failure at %s:%d/%s() (sas_end_device_alloc)!\n",
+			    __FILE__, __LINE__, __func__);
+			sas_port_delete(port);
+			goto out_fail;
+		}
 		tgtdev->dev_spec.sas_sata_inf.rphy = rphy;
 	} else {
 		rphy = sas_expander_alloc(port,
 		    mr_sas_port->remote_identify.device_type);
+		if (!rphy) {
+			ioc_err(mrioc, "failure at %s:%d/%s() (sas_expander_alloc)!\n",
+			    __FILE__, __LINE__, __func__);
+			sas_port_delete(port);
+			goto out_fail;
+		}
 	}
 	rphy->identify = mr_sas_port->remote_identify;
 
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+	if (mrioc->stop_drv_processing || mrioc->reset_in_progress) {
+		spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+		sas_rphy_free(rphy);
+		sas_port_delete(port);
+		goto out_fail;
+	}
 	if (mrioc->current_event)
 		mrioc->current_event->pending_at_sml = 1;
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 
 	if ((sas_rphy_add(rphy))) {
 		ioc_err(mrioc, "failure at %s:%d/%s()!\n",
 		    __FILE__, __LINE__, __func__);
+		spin_lock_irqsave(&mrioc->fwevt_lock, flags);
+		if (mrioc->current_event) {
+			mrioc->current_event->pending_at_sml = 0;
+			discard = mrioc->current_event->discard;
+		}
+		spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+		if (discard)
+			mpi3mr_print_device_event_notice(mrioc, true);
+		sas_rphy_unlink(rphy);
+		sas_rphy_free(rphy);
+		sas_port_delete(port);
+		goto out_fail;
 	}
 	if (mr_sas_port->remote_identify.device_type == SAS_END_DEVICE) {
 		tgtdev->dev_spec.sas_sata_inf.pend_sas_rphy_add = 0;
@@ -1486,11 +1539,14 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc,
 	list_add_tail(&mr_sas_port->port_list, &mr_sas_node->sas_port_list);
 	spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
 
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 	if (mrioc->current_event) {
 		mrioc->current_event->pending_at_sml = 0;
-		if (mrioc->current_event->discard)
-			mpi3mr_print_device_event_notice(mrioc, true);
+		discard = mrioc->current_event->discard;
 	}
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+	if (discard)
+		mpi3mr_print_device_event_notice(mrioc, true);
 
 	/* fill in report manufacture */
 	if (mr_sas_port->remote_identify.device_type ==
@@ -1504,11 +1560,17 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc,
 	return mr_sas_port;
 
  out_fail:
+	if (tgtdev) {
+		tgtdev->dev_spec.sas_sata_inf.pend_sas_rphy_add = 0;
+		tgtdev->dev_spec.sas_sata_inf.rphy = NULL;
+		mpi3mr_tgtdev_put(tgtdev);
+	}
+
 	list_for_each_entry_safe(mr_sas_phy, next, &mr_sas_port->phy_list,
-	    port_siblings)
+				 port_siblings) {
+		mr_sas_phy->phy_belongs_to_port = 0;
 		list_del(&mr_sas_phy->port_siblings);
-	if (tgtdev)
-		mpi3mr_tgtdev_put(tgtdev);
+	}
 	kfree(mr_sas_port);
 	return NULL;
 }
@@ -1530,6 +1592,7 @@ static void mpi3mr_sas_port_remove(struct mpi3mr_ioc *mrioc, u64 sas_address,
 {
 	int i;
 	unsigned long flags;
+	bool discard = false;
 	struct mpi3mr_sas_port *mr_sas_port, *next;
 	struct mpi3mr_sas_node *mr_sas_node;
 	u8 found = 0;
@@ -1586,8 +1649,10 @@ static void mpi3mr_sas_port_remove(struct mpi3mr_ioc *mrioc, u64 sas_address,
 
 	spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
 
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 	if (mrioc->current_event)
 		mrioc->current_event->pending_at_sml = 1;
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 
 	list_for_each_entry_safe(mr_sas_phy, next_phy,
 	    &mr_sas_port->phy_list, port_siblings) {
@@ -1609,11 +1674,14 @@ static void mpi3mr_sas_port_remove(struct mpi3mr_ioc *mrioc, u64 sas_address,
 	ioc_info(mrioc, "%s: removed sas_address(0x%016llx)\n",
 	    __func__, (unsigned long long)sas_address);
 
+	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 	if (mrioc->current_event) {
 		mrioc->current_event->pending_at_sml = 0;
-		if (mrioc->current_event->discard)
-			mpi3mr_print_device_event_notice(mrioc, false);
+		discard = mrioc->current_event->discard;
 	}
+	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
+	if (discard)
+		mpi3mr_print_device_event_notice(mrioc, false);
 
 	kfree(mr_sas_port);
 }
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 0e12009a87f62d..085d4672db8545 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -5401,23 +5401,23 @@ _scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
 static void
 _scsih_eedp_error_handling(struct scsi_cmnd *scmd, u16 ioc_status)
 {
-	u8 ascq;
+	u16 sense_code;
 
 	switch (ioc_status) {
 	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
-		ascq = 0x01;
+		sense_code = LOGICAL_BLOCK_GUARD_CHECK_FAILED;
 		break;
 	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
-		ascq = 0x02;
+		sense_code = LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED;
 		break;
 	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
-		ascq = 0x03;
+		sense_code = LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED;
 		break;
 	default:
-		ascq = 0x00;
+		sense_code = ID_CRC_OR_ECC_ERROR;
 		break;
 	}
-	scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x10, ascq);
+	scsi_set_sense(scmd, 0, ILLEGAL_REQUEST, sense_code);
 	set_host_byte(scmd, DID_ABORT);
 }
 
@@ -5474,7 +5474,8 @@ static enum scsi_qc_status scsih_qcmd(struct Scsi_Host *shost,
 	if (handle == MPT3SAS_INVALID_DEVICE_HANDLE || sas_device_priv_data->block) {
 		if (scsi_get_host_state(scmd->device->host) == SHOST_RECOVERY &&
 		    scmd->cmnd[0] == TEST_UNIT_READY) {
-			scsi_build_sense(scmd, 0, UNIT_ATTENTION, 0x29, 0x07);
+			scsi_set_sense(scmd, 0, UNIT_ATTENTION,
+				       I_T_NEXUS_LOSS_OCCURRED);
 			scsi_done(scmd);
 			return 0;
 		}
@@ -5986,7 +5987,7 @@ _scsih_smart_predicted_fault(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 	event_data = (Mpi2EventDataSasDeviceStatusChange_t *)
 	    event_reply->EventData;
 	event_data->ReasonCode = MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA;
-	event_data->ASC = 0x5D;
+	event_data->ASC = ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED;
 	event_data->DevHandle = cpu_to_le16(handle);
 	event_data->SASAddress = cpu_to_le64(sas_target_priv_data->sas_address);
 	mpt3sas_ctl_add_to_event_log(ioc, event_reply);
@@ -6109,8 +6110,7 @@ _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
 		    le32_to_cpu(mpi_reply->SenseCount));
 		memcpy(scmd->sense_buffer, sense_data, sz);
 		_scsih_normalize_sense(scmd->sense_buffer, &data);
-		/* failure prediction threshold exceeded */
-		if (data.asc == 0x5D)
+		if (data.asc == ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED)
 			_scsih_smart_predicted_fault(ioc,
 			    le16_to_cpu(mpi_reply->DevHandle));
 		mpt3sas_trigger_scsi(ioc, data.skey, data.asc, data.ascq);
@@ -6199,8 +6199,8 @@ _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
 		else if (!xfer_cnt && scmd->cmnd[0] == REPORT_LUNS) {
 			mpi_reply->SCSIState = MPI2_SCSI_STATE_AUTOSENSE_VALID;
 			mpi_reply->SCSIStatus = SAM_STAT_CHECK_CONDITION;
-			scsi_build_sense(scmd, 0, ILLEGAL_REQUEST,
-					 0x20, 0);
+			scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+				       INVALID_COMMAND_OP_CODE);
 		}
 		break;
 
@@ -7691,38 +7691,41 @@ _scsih_determine_disposition(struct MPT3SAS_ADAPTER *ioc,
 
 	if (check_sense) {
 		_scsih_normalize_sense(transfer_packet->sense, &sense_info);
-		if (sense_info.skey == UNIT_ATTENTION)
+		if (sense_info.skey == UNIT_ATTENTION) {
 			rc = DEVICE_RETRY_UA;
-		else if (sense_info.skey == NOT_READY) {
-			/* medium isn't present */
-			if (sense_info.asc == 0x3a)
+		} else if (sense_info.skey == NOT_READY) {
+			if (sense_info.asc == ASC_MEDIUM_NOT_PRESENT) {
 				rc = DEVICE_READY;
-			/* LOGICAL UNIT NOT READY */
-			else if (sense_info.asc == 0x04) {
-				if (sense_info.ascq == 0x03 ||
-				   sense_info.ascq == 0x0b ||
-				   sense_info.ascq == 0x0c) {
+			} else if (sense_info.asc == ASC_LU_NOT_READY) {
+				u16 scode = scsi_sense_code(sense_info.asc,
+							    sense_info.ascq);
+
+				if (scode ==
+				    LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED ||
+				    scode ==
+				    LU_NOT_ACCESSIBLE_TARGET_PORT_IN_STANDBY_STATE ||
+				    scode ==
+				    LU_NOT_ACCESSIBLE_TARGET_PORT_IN_UNAVAILABLE_STATE)
 					rc = DEVICE_ERROR;
-				} else
+				else
 					rc = DEVICE_START_UNIT;
-			}
-			/* LOGICAL UNIT HAS NOT SELF-CONFIGURED YET */
-			else if (sense_info.asc == 0x3e && !sense_info.ascq)
+			} else if (sense_info.asc ==
+				   ASC_LU_HAS_NOT_SELF_CONFIGURED_YET &&
+				   !sense_info.ascq) {
 				rc = DEVICE_START_UNIT;
+			}
 		} else if (sense_info.skey == ILLEGAL_REQUEST &&
-		    transfer_packet->cdb[0] == REPORT_LUNS) {
+			   transfer_packet->cdb[0] == REPORT_LUNS) {
 			rc = DEVICE_READY;
 		} else if (sense_info.skey == MEDIUM_ERROR) {
-
-			/* medium is corrupt, lets add the device so
-			 * users can collect some info as needed
+			/*
+			 * medium is corrupt, lets add the device so users can
+			 * collect some info as needed
 			 */
-
-			if (sense_info.asc == 0x31)
+			if (sense_info.asc == ASC_MEDIUM_FORMAT_CORRUPTED)
 				rc = DEVICE_READY;
 		} else if (sense_info.skey == HARDWARE_ERROR) {
-			/* Defect List Error, still add the device */
-			if (sense_info.asc == 0x19)
+			if (sense_info.asc == ASC_DEFECT_LIST_ERROR)
 				rc = DEVICE_READY;
 		}
 	}
diff --git a/drivers/scsi/mvumi.c b/drivers/scsi/mvumi.c
index e70d336b4ab391..91c2db697c6913 100644
--- a/drivers/scsi/mvumi.c
+++ b/drivers/scsi/mvumi.c
@@ -722,9 +722,9 @@ static int mvumi_issue_blocked_cmd(struct mvumi_hba *mhba,
 		return -1;
 	}
 	atomic_inc(&cmd->sync_cmd);
-	spin_lock_irqsave(mhba->shost->host_lock, flags);
+	spin_lock_irqsave(&mhba->shost->host_lock, flags);
 	mhba->instancet->fire_cmd(mhba, cmd);
-	spin_unlock_irqrestore(mhba->shost->host_lock, flags);
+	spin_unlock_irqrestore(&mhba->shost->host_lock, flags);
 
 	wait_event_timeout(mhba->int_cmd_wait_q,
 		(cmd->cmd_status != REQ_STATUS_PENDING),
@@ -732,7 +732,7 @@ static int mvumi_issue_blocked_cmd(struct mvumi_hba *mhba,
 
 	/* command timeout */
 	if (atomic_read(&cmd->sync_cmd)) {
-		spin_lock_irqsave(mhba->shost->host_lock, flags);
+		spin_lock_irqsave(&mhba->shost->host_lock, flags);
 		atomic_dec(&cmd->sync_cmd);
 		if (mhba->tag_cmd[cmd->frame->tag]) {
 			mhba->tag_cmd[cmd->frame->tag] = NULL;
@@ -747,7 +747,7 @@ static int mvumi_issue_blocked_cmd(struct mvumi_hba *mhba,
 		} else
 			atomic_dec(&mhba->fw_outstanding);
 
-		spin_unlock_irqrestore(mhba->shost->host_lock, flags);
+		spin_unlock_irqrestore(&mhba->shost->host_lock, flags);
 	}
 	return 0;
 }
@@ -1791,9 +1791,9 @@ static irqreturn_t mvumi_isr_handler(int irq, void *devp)
 	struct mvumi_hba *mhba = (struct mvumi_hba *) devp;
 	unsigned long flags;
 
-	spin_lock_irqsave(mhba->shost->host_lock, flags);
+	spin_lock_irqsave(&mhba->shost->host_lock, flags);
 	if (unlikely(mhba->instancet->clear_intr(mhba) || !mhba->global_isr)) {
-		spin_unlock_irqrestore(mhba->shost->host_lock, flags);
+		spin_unlock_irqrestore(&mhba->shost->host_lock, flags);
 		return IRQ_NONE;
 	}
 
@@ -1814,7 +1814,7 @@ static irqreturn_t mvumi_isr_handler(int irq, void *devp)
 	mhba->isr_status = 0;
 	if (mhba->fw_state == FW_STATE_STARTED)
 		mvumi_handle_clob(mhba);
-	spin_unlock_irqrestore(mhba->shost->host_lock, flags);
+	spin_unlock_irqrestore(&mhba->shost->host_lock, flags);
 	return IRQ_HANDLED;
 }
 
@@ -2067,7 +2067,7 @@ static unsigned char mvumi_build_frame(struct mvumi_hba *mhba,
 	return 0;
 
 error:
-	scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x24, 0);
+	scsi_set_sense(scmd, 0, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 	return -1;
 }
 
@@ -2083,13 +2083,13 @@ static enum scsi_qc_status mvumi_queue_command(struct Scsi_Host *shost,
 	struct mvumi_hba *mhba;
 	unsigned long irq_flags;
 
-	spin_lock_irqsave(shost->host_lock, irq_flags);
+	spin_lock_irqsave(&shost->host_lock, irq_flags);
 
 	mhba = (struct mvumi_hba *) shost->hostdata;
 	scmd->result = 0;
 	cmd = mvumi_get_cmd(mhba);
 	if (unlikely(!cmd)) {
-		spin_unlock_irqrestore(shost->host_lock, irq_flags);
+		spin_unlock_irqrestore(&shost->host_lock, irq_flags);
 		return SCSI_MLQUEUE_HOST_BUSY;
 	}
 
@@ -2099,13 +2099,13 @@ static enum scsi_qc_status mvumi_queue_command(struct Scsi_Host *shost,
 	cmd->scmd = scmd;
 	mvumi_priv(scmd)->cmd_priv = cmd;
 	mhba->instancet->fire_cmd(mhba, cmd);
-	spin_unlock_irqrestore(shost->host_lock, irq_flags);
+	spin_unlock_irqrestore(&shost->host_lock, irq_flags);
 	return 0;
 
 out_return_cmd:
 	mvumi_return_cmd(mhba, cmd);
 	scsi_done(scmd);
-	spin_unlock_irqrestore(shost->host_lock, irq_flags);
+	spin_unlock_irqrestore(&shost->host_lock, irq_flags);
 	return 0;
 }
 
@@ -2116,7 +2116,7 @@ static enum scsi_timeout_action mvumi_timed_out(struct scsi_cmnd *scmd)
 	struct mvumi_hba *mhba = shost_priv(host);
 	unsigned long flags;
 
-	spin_lock_irqsave(mhba->shost->host_lock, flags);
+	spin_lock_irqsave(&mhba->shost->host_lock, flags);
 
 	if (mhba->tag_cmd[cmd->frame->tag]) {
 		mhba->tag_cmd[cmd->frame->tag] = NULL;
@@ -2135,7 +2135,7 @@ static enum scsi_timeout_action mvumi_timed_out(struct scsi_cmnd *scmd)
 			     scmd->sc_data_direction);
 	}
 	mvumi_return_cmd(mhba, cmd);
-	spin_unlock_irqrestore(mhba->shost->host_lock, flags);
+	spin_unlock_irqrestore(&mhba->shost->host_lock, flags);
 
 	return SCSI_EH_NOT_HANDLED;
 }
diff --git a/drivers/scsi/myrb.c b/drivers/scsi/myrb.c
index 3678b66310ed0e..95df6e5345f7aa 100644
--- a/drivers/scsi/myrb.c
+++ b/drivers/scsi/myrb.c
@@ -314,17 +314,19 @@ static void myrb_get_event(struct myrb_hba *cb, unsigned int event)
 		scsi_normalize_sense(ev_buf->sense, 32, &sshdr);
 
 		if (sshdr.sense_key == VENDOR_SPECIFIC &&
-		    sshdr.asc == 0x80 &&
-		    sshdr.ascq < ARRAY_SIZE(myrb_event_msg))
+		    scsi_sense_asc(&sshdr) == 0x80 &&
+		    scsi_sense_ascq(&sshdr) < ARRAY_SIZE(myrb_event_msg))
 			shost_printk(KERN_CRIT, cb->host,
 				     "Physical drive %d:%d: %s\n",
 				     ev_buf->channel, ev_buf->target,
-				     myrb_event_msg[sshdr.ascq]);
+				     myrb_event_msg[scsi_sense_ascq(&sshdr)]);
 		else
 			shost_printk(KERN_CRIT, cb->host,
 				     "Physical drive %d:%d: Sense: %X/%02X/%02X\n",
 				     ev_buf->channel, ev_buf->target,
-				     sshdr.sense_key, sshdr.asc, sshdr.ascq);
+				     sshdr.sense_key,
+				     scsi_sense_asc(&sshdr),
+				     scsi_sense_ascq(&sshdr));
 	}
 
 	dma_free_coherent(&cb->pdev->dev, sizeof(struct myrb_log_entry),
@@ -1398,7 +1400,7 @@ myrb_mode_sense(struct myrb_hba *cb, struct scsi_cmnd *scmd,
 static void myrb_request_sense(struct myrb_hba *cb,
 		struct scsi_cmnd *scmd)
 {
-	scsi_build_sense(scmd, 0, NO_SENSE, 0, 0);
+	scsi_set_sense(scmd, 0, NO_SENSE, NO_ADDITIONAL_SENSE_INFORMATION);
 	scsi_sg_copy_from_buffer(scmd, scmd->sense_buffer,
 				 SCSI_SENSE_BUFFERSIZE);
 }
@@ -1446,8 +1448,8 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 		return 0;
 	case INQUIRY:
 		if (scmd->cmnd[1] & 1) {
-			/* Illegal request, invalid field in CDB */
-			scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x24, 0);
+			scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+				       INVALID_FIELD_IN_CDB);
 		} else {
 			myrb_inquiry(cb, scmd);
 			scmd->result = (DID_OK << 16);
@@ -1461,8 +1463,8 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 	case MODE_SENSE:
 		if ((scmd->cmnd[2] & 0x3F) != 0x3F &&
 		    (scmd->cmnd[2] & 0x3F) != 0x08) {
-			/* Illegal request, invalid field in CDB */
-			scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x24, 0);
+			scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+				       INVALID_FIELD_IN_CDB);
 		} else {
 			myrb_mode_sense(cb, scmd, ldev_info);
 			scmd->result = (DID_OK << 16);
@@ -1472,15 +1474,15 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 	case READ_CAPACITY:
 		if ((scmd->cmnd[1] & 1) ||
 		    (scmd->cmnd[8] & 1)) {
-			/* Illegal request, invalid field in CDB */
-			scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x24, 0);
+			scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+				       INVALID_FIELD_IN_CDB);
 			scsi_done(scmd);
 			return 0;
 		}
 		lba = get_unaligned_be32(&scmd->cmnd[2]);
 		if (lba) {
-			/* Illegal request, invalid field in CDB */
-			scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x24, 0);
+			scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+				       INVALID_FIELD_IN_CDB);
 			scsi_done(scmd);
 			return 0;
 		}
@@ -1493,8 +1495,8 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 		return 0;
 	case SEND_DIAGNOSTIC:
 		if (scmd->cmnd[1] != 0x04) {
-			/* Illegal request, invalid field in CDB */
-			scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x24, 0);
+			scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+				       INVALID_FIELD_IN_CDB);
 		} else {
 			/* Assume good status */
 			scmd->result = (DID_OK << 16);
@@ -1503,8 +1505,8 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 		return 0;
 	case READ_6:
 		if (ldev_info->state == MYRB_DEVICE_WO) {
-			/* Data protect, attempt to read invalid data */
-			scsi_build_sense(scmd, 0, DATA_PROTECT, 0x21, 0x06);
+			scsi_set_sense(scmd, 0, DATA_PROTECT,
+				       ATTEMPT_TO_READ_INVALID_DATA);
 			scsi_done(scmd);
 			return 0;
 		}
@@ -1517,8 +1519,8 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 		break;
 	case READ_10:
 		if (ldev_info->state == MYRB_DEVICE_WO) {
-			/* Data protect, attempt to read invalid data */
-			scsi_build_sense(scmd, 0, DATA_PROTECT, 0x21, 0x06);
+			scsi_set_sense(scmd, 0, DATA_PROTECT,
+				       ATTEMPT_TO_READ_INVALID_DATA);
 			scsi_done(scmd);
 			return 0;
 		}
@@ -1531,8 +1533,8 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 		break;
 	case READ_12:
 		if (ldev_info->state == MYRB_DEVICE_WO) {
-			/* Data protect, attempt to read invalid data */
-			scsi_build_sense(scmd, 0, DATA_PROTECT, 0x21, 0x06);
+			scsi_set_sense(scmd, 0, DATA_PROTECT,
+				       ATTEMPT_TO_READ_INVALID_DATA);
 			scsi_done(scmd);
 			return 0;
 		}
@@ -1544,8 +1546,8 @@ static enum scsi_qc_status myrb_ldev_queuecommand(struct Scsi_Host *shost,
 		block_cnt = get_unaligned_be32(&scmd->cmnd[6]);
 		break;
 	default:
-		/* Illegal request, invalid opcode */
-		scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x20, 0);
+		scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+			       INVALID_COMMAND_OP_CODE);
 		scsi_done(scmd);
 		return 0;
 	}
@@ -2328,20 +2330,19 @@ static void myrb_handle_scsi(struct myrb_hba *cb, struct myrb_cmdblk *cmd_blk,
 		dev_dbg(&scmd->device->sdev_gendev,
 			"Bad Data Encountered\n");
 		if (scmd->sc_data_direction == DMA_FROM_DEVICE)
-			/* Unrecovered read error */
-			scsi_build_sense(scmd, 0, MEDIUM_ERROR, 0x11, 0);
+			scsi_set_sense(scmd, 0, MEDIUM_ERROR,
+					 UNRECOVERED_READ_ERROR);
 		else
-			/* Write error */
-			scsi_build_sense(scmd, 0, MEDIUM_ERROR, 0x0C, 0);
+			scsi_set_sense(scmd, 0, MEDIUM_ERROR, SSC_WRITE_ERROR);
 		break;
 	case MYRB_STATUS_IRRECOVERABLE_DATA_ERROR:
 		scmd_printk(KERN_ERR, scmd, "Irrecoverable Data Error\n");
 		if (scmd->sc_data_direction == DMA_FROM_DEVICE)
-			/* Unrecovered read error, auto-reallocation failed */
-			scsi_build_sense(scmd, 0, MEDIUM_ERROR, 0x11, 0x04);
+			scsi_set_sense(scmd, 0, MEDIUM_ERROR,
+				UNRECOVERED_READ_ERROR_AUTO_REALLOCATE_FAILED);
 		else
-			/* Write error, auto-reallocation failed */
-			scsi_build_sense(scmd, 0, MEDIUM_ERROR, 0x0C, 0x02);
+			scsi_set_sense(scmd, 0, MEDIUM_ERROR,
+				       WRITE_ERROR_AUTO_REALLOCATION_FAILED);
 		break;
 	case MYRB_STATUS_LDRV_NONEXISTENT_OR_OFFLINE:
 		dev_dbg(&scmd->device->sdev_gendev,
@@ -2351,8 +2352,8 @@ static void myrb_handle_scsi(struct myrb_hba *cb, struct myrb_cmdblk *cmd_blk,
 	case MYRB_STATUS_ACCESS_BEYOND_END_OF_LDRV:
 		dev_dbg(&scmd->device->sdev_gendev,
 			    "Attempt to Access Beyond End of Logical Drive");
-		/* Logical block address out of range */
-		scsi_build_sense(scmd, 0, NOT_READY, 0x21, 0);
+		scsi_set_sense(scmd, 0, NOT_READY,
+			       LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		break;
 	case MYRB_STATUS_DEVICE_NONRESPONSIVE:
 		dev_dbg(&scmd->device->sdev_gendev, "Device nonresponsive\n");
diff --git a/drivers/scsi/myrs.c b/drivers/scsi/myrs.c
index afd68225221a64..1192b01fab8ec6 100644
--- a/drivers/scsi/myrs.c
+++ b/drivers/scsi/myrs.c
@@ -816,7 +816,7 @@ static void myrs_log_event(struct myrs_hba *cs, struct myrs_event *ev)
 	struct Scsi_Host *shost = cs->host;
 	struct scsi_device *sdev;
 	struct scsi_sense_hdr sshdr = {0};
-	unsigned char sense_info[4];
+	unsigned char asc, sense_info[4];
 	unsigned char cmd_specific[4];
 
 	if (ev->ev_code == 0x1C) {
@@ -829,9 +829,11 @@ static void myrs_log_event(struct myrs_hba *cs, struct myrs_event *ev)
 			memcpy(cmd_specific, &ev->sense_data[7], 4);
 		}
 	}
+	asc = scsi_sense_asc(&sshdr);
 	if (sshdr.sense_key == VENDOR_SPECIFIC &&
-	    (sshdr.asc == 0x80 || sshdr.asc == 0x81))
-		ev->ev_code = ((sshdr.asc - 0x80) << 8 | sshdr.ascq);
+	    (asc == 0x80 || asc == 0x81))
+		ev->ev_code = scsi_sense_code(asc - 0x80,
+					      scsi_sense_ascq(&sshdr));
 	while (true) {
 		ev_code = myrs_ev_list[ev_idx].ev_code;
 		if (ev_code == ev->ev_code || ev_code == 0)
@@ -891,8 +893,9 @@ static void myrs_log_event(struct myrs_hba *cs, struct myrs_event *ev)
 	case 'S':
 		if (sshdr.sense_key == NO_SENSE ||
 		    (sshdr.sense_key == NOT_READY &&
-		     sshdr.asc == 0x04 && (sshdr.ascq == 0x01 ||
-					    sshdr.ascq == 0x02)))
+		     (sshdr.sense_code == LU_IS_IN_PROCESS_OF_BECOMING_READY ||
+		      sshdr.sense_code ==
+		      LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED)))
 			break;
 		shost_printk(KERN_INFO, shost,
 			     "event %d: Physical Device %d:%d %s\n",
@@ -900,7 +903,8 @@ static void myrs_log_event(struct myrs_hba *cs, struct myrs_event *ev)
 		shost_printk(KERN_INFO, shost,
 			     "Physical Device %d:%d Sense Key = %X, ASC = %02X, ASCQ = %02X\n",
 			     ev->channel, ev->target,
-			     sshdr.sense_key, sshdr.asc, sshdr.ascq);
+			     sshdr.sense_key, scsi_sense_asc(&sshdr),
+			     scsi_sense_ascq(&sshdr));
 		shost_printk(KERN_INFO, shost,
 			     "Physical Device %d:%d Sense Information = %02X%02X%02X%02X %02X%02X%02X%02X\n",
 			     ev->channel, ev->target,
@@ -1603,7 +1607,8 @@ static enum scsi_qc_status myrs_queuecommand(struct Scsi_Host *shost,
 
 	switch (scmd->cmnd[0]) {
 	case REPORT_LUNS:
-		scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x20, 0x0);
+		scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+			       INVALID_COMMAND_OP_CODE);
 		scsi_done(scmd);
 		return 0;
 	case MODE_SENSE:
@@ -1612,8 +1617,8 @@ static enum scsi_qc_status myrs_queuecommand(struct Scsi_Host *shost,
 
 			if ((scmd->cmnd[2] & 0x3F) != 0x3F &&
 			    (scmd->cmnd[2] & 0x3F) != 0x08) {
-				/* Illegal request, invalid field in CDB */
-				scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x24, 0);
+				scsi_set_sense(scmd, 0, ILLEGAL_REQUEST,
+					       INVALID_FIELD_IN_CDB);
 			} else {
 				myrs_mode_sense(cs, scmd, ldev_info);
 				scmd->result = (DID_OK << 16);
diff --git a/drivers/scsi/nsp32.c b/drivers/scsi/nsp32.c
index 9c9281222a0acf..98550c73d8a0c6 100644
--- a/drivers/scsi/nsp32.c
+++ b/drivers/scsi/nsp32.c
@@ -1166,7 +1166,7 @@ static irqreturn_t do_nsp32_isr(int irq, void *dev_id)
 	int handled = 0;
 	struct Scsi_Host *host = data->Host;
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 
 	/*
 	 * IRQ check, then enable IRQ mask
@@ -1433,7 +1433,7 @@ static irqreturn_t do_nsp32_isr(int irq, void *dev_id)
 	nsp32_write2(base, IRQ_CONTROL, 0);
 
  out2:
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 
 	nsp32_dbg(NSP32_DEBUG_INTR, "exit");
 
@@ -2886,14 +2886,14 @@ static int nsp32_eh_host_reset(struct scsi_cmnd *SCpnt)
 	nsp32_msg(KERN_INFO, "Host Reset");
 	nsp32_dbg(NSP32_DEBUG_BUSRESET, "SCpnt=0x%x", SCpnt);
 
-	spin_lock_irq(SCpnt->device->host->host_lock);
+	spin_lock_irq(&SCpnt->device->host->host_lock);
 
 	nsp32hw_init(data);
 	nsp32_write2(base, IRQ_CONTROL, IRQ_CONTROL_ALL_IRQ_MASK);
 	nsp32_do_bus_reset(data);
 	nsp32_write2(base, IRQ_CONTROL, 0);
 
-	spin_unlock_irq(SCpnt->device->host->host_lock);
+	spin_unlock_irq(&SCpnt->device->host->host_lock);
 	return SUCCESS;	/* Host reset is succeeded at any time. */
 }
 
diff --git a/drivers/scsi/pcmcia/sym53c500_cs.c b/drivers/scsi/pcmcia/sym53c500_cs.c
index 1530c1ad5d36a7..aeae5c16e406fe 100644
--- a/drivers/scsi/pcmcia/sym53c500_cs.c
+++ b/drivers/scsi/pcmcia/sym53c500_cs.c
@@ -360,7 +360,7 @@ SYM53C500_intr(int irq, void *dev_id)
 	struct sym53c500_cmd_priv *scp = scsi_cmd_priv(curSC);
 	int fast_pio = data->fast_pio;
 
-	spin_lock_irqsave(dev->host_lock, flags);
+	spin_lock_irqsave(&dev->host_lock, flags);
 
 	VDEB(printk("SYM53C500_intr called\n"));
 
@@ -494,7 +494,7 @@ SYM53C500_intr(int irq, void *dev_id)
 		break;
 	}
 out:
-	spin_unlock_irqrestore(dev->host_lock, flags);
+	spin_unlock_irqrestore(&dev->host_lock, flags);
 	return IRQ_HANDLED;
 
 idle_out:
@@ -588,9 +588,9 @@ SYM53C500_host_reset(struct scsi_cmnd *SCpnt)
 	int port_base = SCpnt->device->host->io_port;
 
 	DEB(printk("SYM53C500_host_reset called\n"));
-	spin_lock_irq(SCpnt->device->host->host_lock);
+	spin_lock_irq(&SCpnt->device->host->host_lock);
 	SYM53C500_int_host_reset(port_base);
-	spin_unlock_irq(SCpnt->device->host->host_lock);
+	spin_unlock_irq(&SCpnt->device->host->host_lock);
 
 	return SUCCESS;
 }
diff --git a/drivers/scsi/pm8001/pm8001_defs.h b/drivers/scsi/pm8001/pm8001_defs.h
index 4e19d61dffbbb3..a9605ca317f341 100644
--- a/drivers/scsi/pm8001/pm8001_defs.h
+++ b/drivers/scsi/pm8001/pm8001_defs.h
@@ -41,6 +41,104 @@
 #ifndef _PM8001_DEFS_H_
 #define _PM8001_DEFS_H_
 
+#include <scsi/sas.h>
+
+#ifdef __LITTLE_ENDIAN_BITFIELD
+struct sas_identify_frame_local {
+	/* Byte 0 */
+	u8  frame_type:4;
+	u8  dev_type:3;
+	u8  _un0:1;
+
+	/* Byte 1 */
+	u8  _un1;
+
+	/* Byte 2 */
+	union {
+		struct {
+			u8  _un20:1;
+			u8  smp_iport:1;
+			u8  stp_iport:1;
+			u8  ssp_iport:1;
+			u8  _un247:4;
+		};
+		u8 initiator_bits;
+	};
+
+	/* Byte 3 */
+	union {
+		struct {
+			u8  _un30:1;
+			u8 smp_tport:1;
+			u8 stp_tport:1;
+			u8 ssp_tport:1;
+			u8 _un347:4;
+		};
+		u8 target_bits;
+	};
+
+	/* Byte 4 - 11 */
+	u8 _un4_11[8];
+
+	/* Byte 12 - 19 */
+	u8 sas_addr[SAS_ADDR_SIZE];
+
+	/* Byte 20 */
+	u8 phy_id;
+
+	u8 _un21_27[7];
+
+} __packed;
+
+#elif defined(__BIG_ENDIAN_BITFIELD)
+struct sas_identify_frame_local {
+	/* Byte 0 */
+	u8  _un0:1;
+	u8  dev_type:3;
+	u8  frame_type:4;
+
+	/* Byte 1 */
+	u8  _un1;
+
+	/* Byte 2 */
+	union {
+		struct {
+			u8  _un247:4;
+			u8  ssp_iport:1;
+			u8  stp_iport:1;
+			u8  smp_iport:1;
+			u8  _un20:1;
+		};
+		u8 initiator_bits;
+	};
+
+	/* Byte 3 */
+	union {
+		struct {
+			u8 _un347:4;
+			u8 ssp_tport:1;
+			u8 stp_tport:1;
+			u8 smp_tport:1;
+			u8 _un30:1;
+		};
+		u8 target_bits;
+	};
+
+	/* Byte 4 - 11 */
+	u8 _un4_11[8];
+
+	/* Byte 12 - 19 */
+	u8 sas_addr[SAS_ADDR_SIZE];
+
+	/* Byte 20 */
+	u8 phy_id;
+
+	u8 _un21_27[7];
+} __packed;
+#else
+#error "Bitfield order not defined!"
+#endif
+
 enum chip_flavors {
 	chip_8001,
 	chip_8008,
diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c
index fff8d877abb9ee..55a01e533fd198 100644
--- a/drivers/scsi/pm8001/pm8001_hwi.c
+++ b/drivers/scsi/pm8001/pm8001_hwi.c
@@ -3208,7 +3208,7 @@ hw_event_sata_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb)
 	phy->sas_phy.oob_mode = SATA_OOB_MODE;
 	sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE, GFP_ATOMIC);
 	spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags);
-	memcpy(phy->frame_rcvd, ((u8 *)&pPayload->sata_fis - 4),
+	memcpy(phy->frame_rcvd, &pPayload->sata_fis,
 		sizeof(struct dev_to_host_fis));
 	phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
 	phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
diff --git a/drivers/scsi/pm8001/pm8001_hwi.h b/drivers/scsi/pm8001/pm8001_hwi.h
index f1ce8df082b0c6..a21b1b37a8ea52 100644
--- a/drivers/scsi/pm8001/pm8001_hwi.h
+++ b/drivers/scsi/pm8001/pm8001_hwi.h
@@ -42,6 +42,7 @@
 
 #include <linux/types.h>
 #include <scsi/libsas.h>
+#include "pm8001_defs.h"
 
 
 /* for Request Opcode of IOMB */
@@ -229,8 +230,9 @@ struct hw_event_resp {
 	__le32	lr_evt_status_phyid_portid;
 	__le32	evt_param;
 	__le32	npip_portstate;
-	struct sas_identify_frame	sas_identify;
+	struct sas_identify_frame_local	sas_identify;
 	struct dev_to_host_fis	sata_fis;
+	u32	_r_a;
 } __attribute__((packed, aligned(4)));
 
 
diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c
index 2c0fa7ab33d224..659dbd603a0333 100644
--- a/drivers/scsi/pm8001/pm80xx_hwi.c
+++ b/drivers/scsi/pm8001/pm80xx_hwi.c
@@ -3296,7 +3296,7 @@ hw_event_sata_phy_up(struct pm8001_hba_info *pm8001_ha, void *piomb)
 	phy->sas_phy.oob_mode = SATA_OOB_MODE;
 	sas_notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE, GFP_ATOMIC);
 	spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags);
-	memcpy(phy->frame_rcvd, ((u8 *)&pPayload->sata_fis - 4),
+	memcpy(phy->frame_rcvd, &pPayload->sata_fis,
 		sizeof(struct dev_to_host_fis));
 	phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
 	phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
diff --git a/drivers/scsi/pm8001/pm80xx_hwi.h b/drivers/scsi/pm8001/pm80xx_hwi.h
index d8a63b7fed6a04..d6796ba40df96b 100644
--- a/drivers/scsi/pm8001/pm80xx_hwi.h
+++ b/drivers/scsi/pm8001/pm80xx_hwi.h
@@ -43,6 +43,7 @@
 
 #include <linux/types.h>
 #include <scsi/libsas.h>
+#include "pm8001_defs.h"
 
 /* for Request Opcode of IOMB */
 #define OPC_INB_ECHO				1	/* 0x000 */
@@ -236,102 +237,6 @@
 /* Port recovery timeout, 10000 ms for PM8006 controller */
 #define CHIP_8006_PORT_RECOVERY_TIMEOUT 0x640000
 
-#ifdef __LITTLE_ENDIAN_BITFIELD
-struct sas_identify_frame_local {
-	/* Byte 0 */
-	u8  frame_type:4;
-	u8  dev_type:3;
-	u8  _un0:1;
-
-	/* Byte 1 */
-	u8  _un1;
-
-	/* Byte 2 */
-	union {
-		struct {
-			u8  _un20:1;
-			u8  smp_iport:1;
-			u8  stp_iport:1;
-			u8  ssp_iport:1;
-			u8  _un247:4;
-		};
-		u8 initiator_bits;
-	};
-
-	/* Byte 3 */
-	union {
-		struct {
-			u8  _un30:1;
-			u8 smp_tport:1;
-			u8 stp_tport:1;
-			u8 ssp_tport:1;
-			u8 _un347:4;
-		};
-		u8 target_bits;
-	};
-
-	/* Byte 4 - 11 */
-	u8 _un4_11[8];
-
-	/* Byte 12 - 19 */
-	u8 sas_addr[SAS_ADDR_SIZE];
-
-	/* Byte 20 */
-	u8 phy_id;
-
-	u8 _un21_27[7];
-
-} __packed;
-
-#elif defined(__BIG_ENDIAN_BITFIELD)
-struct sas_identify_frame_local {
-	/* Byte 0 */
-	u8  _un0:1;
-	u8  dev_type:3;
-	u8  frame_type:4;
-
-	/* Byte 1 */
-	u8  _un1;
-
-	/* Byte 2 */
-	union {
-		struct {
-			u8  _un247:4;
-			u8  ssp_iport:1;
-			u8  stp_iport:1;
-			u8  smp_iport:1;
-			u8  _un20:1;
-		};
-		u8 initiator_bits;
-	};
-
-	/* Byte 3 */
-	union {
-		struct {
-			u8 _un347:4;
-			u8 ssp_tport:1;
-			u8 stp_tport:1;
-			u8 smp_tport:1;
-			u8 _un30:1;
-		};
-		u8 target_bits;
-	};
-
-	/* Byte 4 - 11 */
-	u8 _un4_11[8];
-
-	/* Byte 12 - 19 */
-	u8 sas_addr[SAS_ADDR_SIZE];
-
-	/* Byte 20 */
-	u8 phy_id;
-
-	u8 _un21_27[7];
-} __packed;
-#else
-#error "Bitfield order not defined!"
-#endif
-
 struct mpi_msg_hdr {
 	__le32	header;	/* Bits [11:0] - Message operation code */
 	/* Bits [15:12] - Message Category */
@@ -427,8 +332,9 @@ struct hw_event_resp {
 	__le32	lr_status_evt_portid;
 	__le32	evt_param;
 	__le32	phyid_npip_portstate;
-	struct sas_identify_frame	sas_identify;
+	struct sas_identify_frame_local	sas_identify;
 	struct dev_to_host_fis	sata_fis;
+	u32	_r_a;
 } __attribute__((packed, aligned(4)));
 
 /*
diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
index 942a9939320498..b68f6460207c79 100644
--- a/drivers/scsi/pmcraid.c
+++ b/drivers/scsi/pmcraid.c
@@ -497,10 +497,10 @@ static void pmcraid_clr_trans_op(
 	if (pinstance->reset_cmd != NULL) {
 		timer_delete(&pinstance->reset_cmd->timer);
 		spin_lock_irqsave(
-			pinstance->host->host_lock, lock_flags);
+			&pinstance->host->host_lock, lock_flags);
 		pinstance->reset_cmd->cmd_done(pinstance->reset_cmd);
 		spin_unlock_irqrestore(
-			pinstance->host->host_lock, lock_flags);
+			&pinstance->host->host_lock, lock_flags);
 	}
 }
 
@@ -562,9 +562,9 @@ static void pmcraid_bist_done(struct timer_list *t)
 	} else {
 		cmd->time_left = 0;
 		pmcraid_info("BIST is complete, proceeding with reset\n");
-		spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+		spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 		pmcraid_ioa_reset(cmd);
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 	}
 }
 
@@ -613,9 +613,9 @@ static void pmcraid_reset_alert_done(struct timer_list *t)
 	if (((status & INTRS_CRITICAL_OP_IN_PROGRESS) == 0) ||
 	    cmd->time_left <= 0) {
 		pmcraid_info("critical op is reset proceeding with reset\n");
-		spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+		spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 		pmcraid_ioa_reset(cmd);
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 	} else {
 		pmcraid_info("critical op is not yet reset waiting again\n");
 		/* restart timer if some more time is available to wait */
@@ -699,7 +699,7 @@ static void pmcraid_timeout_handler(struct timer_list *t)
 	 * reset is in progress. Otherwise fail this command and get a free
 	 * command block to restart the reset sequence.
 	 */
-	spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+	spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 	if (!pinstance->ioa_reset_in_progress) {
 		pinstance->ioa_reset_attempts = 0;
 		cmd = pmcraid_get_free_cmd(pinstance);
@@ -708,7 +708,7 @@ static void pmcraid_timeout_handler(struct timer_list *t)
 		 * Some other command's timeout handler can do the reset job
 		 */
 		if (cmd == NULL) {
-			spin_unlock_irqrestore(pinstance->host->host_lock,
+			spin_unlock_irqrestore(&pinstance->host->host_lock,
 					       lock_flags);
 			pmcraid_err("no free cmnd block for timeout handler\n");
 			return;
@@ -745,7 +745,7 @@ static void pmcraid_timeout_handler(struct timer_list *t)
 	pinstance->ioa_state = IOA_STATE_IN_RESET_ALERT;
 	scsi_block_requests(pinstance->host);
 	pmcraid_reset_alert(cmd);
-	spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 }
 
 /**
@@ -921,9 +921,9 @@ static void pmcraid_ioa_shutdown_done(struct pmcraid_cmd *cmd)
 	struct pmcraid_instance *pinstance = cmd->drv_inst;
 	unsigned long lock_flags;
 
-	spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+	spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 	pmcraid_ioa_reset(cmd);
-	spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 }
 
 /**
@@ -982,10 +982,10 @@ static void pmcraid_get_fwversion_done(struct pmcraid_cmd *cmd)
 	 */
 	if (ioasc) {
 		pmcraid_err("IOA Inquiry failed with %x\n", ioasc);
-		spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+		spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 		pinstance->ioa_state = IOA_STATE_IN_RESET_ALERT;
 		pmcraid_reset_alert(cmd);
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 	} else  {
 		pmcraid_querycfg(cmd);
 	}
@@ -1596,11 +1596,11 @@ static void pmcraid_handle_config_change(struct pmcraid_instance *pinstance)
 			spin_unlock_irqrestore(&pinstance->resource_lock,
 						lock_flags);
 			pmcraid_err("too many resources attached\n");
-			spin_lock_irqsave(pinstance->host->host_lock,
+			spin_lock_irqsave(&pinstance->host->host_lock,
 					  host_lock_flags);
 			pmcraid_send_hcam(pinstance,
 					  PMCRAID_HCAM_CODE_CONFIG_CHANGE);
-			spin_unlock_irqrestore(pinstance->host->host_lock,
+			spin_unlock_irqrestore(&pinstance->host->host_lock,
 					       host_lock_flags);
 			return;
 		}
@@ -1760,9 +1760,9 @@ static void pmcraid_process_ccn(struct pmcraid_cmd *cmd)
 	} else if (ioasc) {
 		dev_info(&pinstance->pdev->dev,
 			"Host RCB (CCN) failed with IOASC: 0x%08X\n", ioasc);
-		spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+		spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 		pmcraid_send_hcam(pinstance, PMCRAID_HCAM_CODE_CONFIG_CHANGE);
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 	} else {
 		pmcraid_handle_config_change(pinstance);
 	}
@@ -1800,10 +1800,10 @@ static void pmcraid_process_ldn(struct pmcraid_cmd *cmd)
 	} else if (!ioasc) {
 		pmcraid_handle_error_log(pinstance);
 		if (fd_ioasc == PMCRAID_IOASC_NR_IOA_RESET_REQUIRED) {
-			spin_lock_irqsave(pinstance->host->host_lock,
+			spin_lock_irqsave(&pinstance->host->host_lock,
 					  lock_flags);
 			pmcraid_initiate_reset(pinstance);
-			spin_unlock_irqrestore(pinstance->host->host_lock,
+			spin_unlock_irqrestore(&pinstance->host->host_lock,
 					       lock_flags);
 			return;
 		}
@@ -2322,17 +2322,17 @@ static int pmcraid_reset_reload(
 	unsigned long lock_flags;
 	int reset = 1;
 
-	spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+	spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 
 	if (pinstance->ioa_reset_in_progress) {
 		pmcraid_info("reset_reload: reset is already in progress\n");
 
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 
 		wait_event(pinstance->reset_wait_q,
 			   !pinstance->ioa_reset_in_progress);
 
-		spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+		spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 
 		if (pinstance->ioa_state == IOA_STATE_DEAD) {
 			pmcraid_info("reset_reload: IOA is dead\n");
@@ -2361,7 +2361,7 @@ static int pmcraid_reset_reload(
 	pinstance->force_ioa_reset = reset;
 	pmcraid_info("reset_reload: initiating reset\n");
 	pmcraid_ioa_reset(reset_cmd);
-	spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 	pmcraid_info("reset_reload: waiting for reset to complete\n");
 	wait_event(pinstance->reset_wait_q,
 		   !pinstance->ioa_reset_in_progress);
@@ -2371,7 +2371,7 @@ static int pmcraid_reset_reload(
 	return pinstance->ioa_state != target_state;
 
 out_unlock:
-	spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 	return reset;
 }
 
@@ -2719,10 +2719,10 @@ static int pmcraid_reset_device(
 	 * This will force the mid-layer to call _eh_bus/host reset, which
 	 * will then go to sleep and wait for the reset to complete
 	 */
-	spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+	spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 	if (pinstance->ioa_reset_in_progress ||
 	    pinstance->ioa_state == IOA_STATE_DEAD) {
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 		return FAILED;
 	}
 
@@ -2736,7 +2736,7 @@ static int pmcraid_reset_device(
 	cmd = pmcraid_get_free_cmd(pinstance);
 
 	if (cmd == NULL) {
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 		pmcraid_err("%s: no cmd blocks are available\n", __func__);
 		return FAILED;
 	}
@@ -2765,7 +2765,7 @@ static int pmcraid_reset_device(
 			 timeout,
 			 pmcraid_timeout_handler);
 
-	spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+	spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 
 	/* RESET_DEVICE command completes after all pending IOARCBs are
 	 * completed. Once this command is completed, pmcraind_internal_done
@@ -2960,11 +2960,11 @@ static int pmcraid_eh_abort_handler(struct scsi_cmnd *scsi_cmd)
 	 * pmcraid_eh_host_reset which will then go to sleep and wait for the
 	 * reset to complete
 	 */
-	spin_lock_irqsave(pinstance->host->host_lock, host_lock_flags);
+	spin_lock_irqsave(&pinstance->host->host_lock, host_lock_flags);
 
 	if (pinstance->ioa_reset_in_progress ||
 	    pinstance->ioa_state == IOA_STATE_DEAD) {
-		spin_unlock_irqrestore(pinstance->host->host_lock,
+		spin_unlock_irqrestore(&pinstance->host->host_lock,
 				       host_lock_flags);
 		return rc;
 	}
@@ -2992,7 +2992,7 @@ static int pmcraid_eh_abort_handler(struct scsi_cmnd *scsi_cmd)
 	if (cmd_found)
 		cancel_cmd = pmcraid_abort_cmd(cmd);
 
-	spin_unlock_irqrestore(pinstance->host->host_lock,
+	spin_unlock_irqrestore(&pinstance->host->host_lock,
 			       host_lock_flags);
 
 	if (cancel_cmd) {
@@ -3720,11 +3720,11 @@ static irqreturn_t pmcraid_isr_msix(int irq, void *dev_id)
 
 				pmcraid_err("ISR: error interrupts: %x \
 					initiating reset\n", intrs_val);
-				spin_lock_irqsave(pinstance->host->host_lock,
+				spin_lock_irqsave(&pinstance->host->host_lock,
 					lock_flags);
 				pmcraid_initiate_reset(pinstance);
 				spin_unlock_irqrestore(
-					pinstance->host->host_lock,
+					&pinstance->host->host_lock,
 					lock_flags);
 			}
 			/* If interrupt was as part of the ioa initialization,
@@ -3798,9 +3798,9 @@ static irqreturn_t pmcraid_isr(int irq, void *dev_id)
 			    intrs);
 		intrs = ioread32(
 				pinstance->int_regs.ioa_host_interrupt_clr_reg);
-		spin_lock_irqsave(pinstance->host->host_lock, lock_flags);
+		spin_lock_irqsave(&pinstance->host->host_lock, lock_flags);
 		pmcraid_initiate_reset(pinstance);
-		spin_unlock_irqrestore(pinstance->host->host_lock, lock_flags);
+		spin_unlock_irqrestore(&pinstance->host->host_lock, lock_flags);
 	} else {
 		/* If interrupt was as part of the ioa initialization,
 		 * clear. Delete the timer and wakeup the
@@ -3859,11 +3859,11 @@ static void pmcraid_worker_function(struct work_struct *workp)
 			/* host_lock must be held before calling
 			 * scsi_device_get
 			 */
-			spin_lock_irqsave(pinstance->host->host_lock,
+			spin_lock_irqsave(&pinstance->host->host_lock,
 					  host_lock_flags);
 			if (!scsi_device_get(sdev)) {
 				spin_unlock_irqrestore(
-						pinstance->host->host_lock,
+						&pinstance->host->host_lock,
 						host_lock_flags);
 				pmcraid_info("deleting %x from midlayer\n",
 					     res->cfg_entry.resource_address);
@@ -3879,7 +3879,7 @@ static void pmcraid_worker_function(struct work_struct *workp)
 				res->change_detected = 0;
 			} else {
 				spin_unlock_irqrestore(
-						pinstance->host->host_lock,
+						&pinstance->host->host_lock,
 						host_lock_flags);
 			}
 		}
@@ -3986,10 +3986,10 @@ static void pmcraid_tasklet_function(unsigned long instance)
 		atomic_dec(&pinstance->outstanding_cmds);
 
 		if (cmd->cmd_done == pmcraid_ioa_reset) {
-			spin_lock_irqsave(pinstance->host->host_lock,
+			spin_lock_irqsave(&pinstance->host->host_lock,
 					  host_lock_flags);
 			cmd->cmd_done(cmd);
-			spin_unlock_irqrestore(pinstance->host->host_lock,
+			spin_unlock_irqrestore(&pinstance->host->host_lock,
 					       host_lock_flags);
 		} else if (cmd->cmd_done != NULL) {
 			cmd->cmd_done(cmd);
@@ -4880,9 +4880,9 @@ static void pmcraid_complete_ioa_reset(struct pmcraid_cmd *cmd)
 	struct pmcraid_instance *pinstance = cmd->drv_inst;
 	unsigned long flags;
 
-	spin_lock_irqsave(pinstance->host->host_lock, flags);
+	spin_lock_irqsave(&pinstance->host->host_lock, flags);
 	pmcraid_ioa_reset(cmd);
-	spin_unlock_irqrestore(pinstance->host->host_lock, flags);
+	spin_unlock_irqrestore(&pinstance->host->host_lock, flags);
 	scsi_unblock_requests(pinstance->host);
 	schedule_work(&pinstance->worker_q);
 }
diff --git a/drivers/scsi/ps3rom.c b/drivers/scsi/ps3rom.c
index a9c727d229313a..c63c581b493c34 100644
--- a/drivers/scsi/ps3rom.c
+++ b/drivers/scsi/ps3rom.c
@@ -233,7 +233,8 @@ static enum scsi_qc_status ps3rom_queuecommand_lck(struct scsi_cmnd *cmd)
 	}
 
 	if (res) {
-		scsi_build_sense(cmd, 0, ILLEGAL_REQUEST, 0, 0);
+		scsi_set_sense(cmd, 0, ILLEGAL_REQUEST,
+			       NO_ADDITIONAL_SENSE_INFORMATION);
 		cmd->result = res;
 		priv->curr_cmd = NULL;
 		scsi_done(cmd);
@@ -316,7 +317,7 @@ static irqreturn_t ps3rom_interrupt(int irq, void *data)
 		goto done;
 	}
 
-	scsi_build_sense(cmd, 0, sense_key, asc, ascq);
+	scsi_set_sense(cmd, 0, sense_key, scsi_sense_code(asc, ascq));
 
 done:
 	priv->curr_cmd = NULL;
diff --git a/drivers/scsi/qla1280.c b/drivers/scsi/qla1280.c
index cdd6fe002c324c..348c59a99cc563 100644
--- a/drivers/scsi/qla1280.c
+++ b/drivers/scsi/qla1280.c
@@ -741,9 +741,9 @@ _qla1280_wait_for_single_command(struct scsi_qla_host *ha, struct srb *sp,
 	int	status = FAILED;
 	struct scsi_cmnd *cmd = sp->cmd;
 
-	spin_unlock_irq(ha->host->host_lock);
+	spin_unlock_irq(&ha->host->host_lock);
 	wait_for_completion_timeout(wait, 4*HZ);
-	spin_lock_irq(ha->host->host_lock);
+	spin_lock_irq(&ha->host->host_lock);
 	sp->wait = NULL;
 	if(CMD_HANDLE(cmd) == COMPLETED_HANDLE) {
 		status = SUCCESS;
@@ -953,9 +953,9 @@ qla1280_eh_abort(struct scsi_cmnd * cmd)
 {
 	int rc;
 
-	spin_lock_irq(cmd->device->host->host_lock);
+	spin_lock_irq(&cmd->device->host->host_lock);
 	rc = qla1280_error_action(cmd, ABORT_COMMAND);
-	spin_unlock_irq(cmd->device->host->host_lock);
+	spin_unlock_irq(&cmd->device->host->host_lock);
 
 	return rc;
 }
@@ -969,9 +969,9 @@ qla1280_eh_device_reset(struct scsi_cmnd *cmd)
 {
 	int rc;
 
-	spin_lock_irq(cmd->device->host->host_lock);
+	spin_lock_irq(&cmd->device->host->host_lock);
 	rc = qla1280_error_action(cmd, DEVICE_RESET);
-	spin_unlock_irq(cmd->device->host->host_lock);
+	spin_unlock_irq(&cmd->device->host->host_lock);
 
 	return rc;
 }
@@ -985,9 +985,9 @@ qla1280_eh_bus_reset(struct scsi_cmnd *cmd)
 {
 	int rc;
 
-	spin_lock_irq(cmd->device->host->host_lock);
+	spin_lock_irq(&cmd->device->host->host_lock);
 	rc = qla1280_error_action(cmd, BUS_RESET);
-	spin_unlock_irq(cmd->device->host->host_lock);
+	spin_unlock_irq(&cmd->device->host->host_lock);
 
 	return rc;
 }
@@ -1003,7 +1003,7 @@ qla1280_eh_adapter_reset(struct scsi_cmnd *cmd)
 	struct Scsi_Host *shost = cmd->device->host;
 	struct scsi_qla_host *ha = (struct scsi_qla_host *)shost->hostdata;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	if (qla1280_verbose) {
 		printk(KERN_INFO
 		       "scsi(%ld): Issued ADAPTER RESET\n",
@@ -1019,7 +1019,7 @@ qla1280_eh_adapter_reset(struct scsi_cmnd *cmd)
 
 	ha->flags.reset_active = 0;
 
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	return rc;
 }
@@ -1080,7 +1080,7 @@ qla1280_intr_handler(int irq, void *dev_id)
 	ENTER_INTR ("qla1280_intr_handler");
 	ha = (struct scsi_qla_host *)dev_id;
 
-	spin_lock(ha->host->host_lock);
+	spin_lock(&ha->host->host_lock);
 
 	ha->isr_count++;
 	reg = ha->iobase;
@@ -1096,7 +1096,7 @@ qla1280_intr_handler(int irq, void *dev_id)
 	if (!list_empty(&ha->done_q))
 		qla1280_done(ha);
 
-	spin_unlock(ha->host->host_lock);
+	spin_unlock(&ha->host->host_lock);
 
 	qla1280_enable_intrs(ha);
 
@@ -1214,11 +1214,11 @@ qla1280_sdev_configure(struct scsi_device *device, struct queue_limits *lim)
 			nv->bus[bus].target[target].ppr_1x160.flags.enable_ppr = 0;
 	}
 
-	spin_lock_irqsave(ha->host->host_lock, flags);
+	spin_lock_irqsave(&ha->host->host_lock, flags);
 	if (nv->bus[bus].target[target].parameter.enable_sync)
 		status = qla1280_set_target_parameters(ha, bus, target);
 	qla1280_get_target_parameters(ha, device);
-	spin_unlock_irqrestore(ha->host->host_lock, flags);
+	spin_unlock_irqrestore(&ha->host->host_lock, flags);
 	return status;
 }
 
@@ -1435,7 +1435,7 @@ qla1280_initialize_adapter(struct scsi_qla_host *ha)
 	 * needs to be able to drop the lock unconditionally to wait
 	 * for completion.
 	 */
-	spin_lock_irqsave(ha->host->host_lock, flags);
+	spin_lock_irqsave(&ha->host->host_lock, flags);
 
 	status = qla1280_load_firmware(ha);
 	if (status) {
@@ -1467,7 +1467,7 @@ qla1280_initialize_adapter(struct scsi_qla_host *ha)
 
 	ha->flags.online = 1;
  out:
-	spin_unlock_irqrestore(ha->host->host_lock, flags);
+	spin_unlock_irqrestore(&ha->host->host_lock, flags);
 
 	if (status)
 		dprintk(2, "qla1280_initialize_adapter: **** FAILED ****\n");
@@ -1496,7 +1496,7 @@ qla1280_request_firmware(struct scsi_qla_host *ha)
 	int index;
 	char *fwname;
 
-	spin_unlock_irq(ha->host->host_lock);
+	spin_unlock_irq(&ha->host->host_lock);
 	mutex_lock(&qla1280_firmware_mutex);
 
 	index = ql1280_board_tbl[ha->devnum].fw_index;
@@ -1529,7 +1529,7 @@ qla1280_request_firmware(struct scsi_qla_host *ha)
 	ha->fwver3 = fw->data[2];
  unlock:
 	mutex_unlock(&qla1280_firmware_mutex);
-	spin_lock_irq(ha->host->host_lock);
+	spin_lock_irq(&ha->host->host_lock);
 	return fw;
 }
 
@@ -2451,14 +2451,14 @@ qla1280_mailbox_command(struct scsi_qla_host *ha, uint8_t mr, uint16_t *mb)
 	timer_setup(&ha->mailbox_timer, qla1280_mailbox_timeout, 0);
 	mod_timer(&ha->mailbox_timer, jiffies + 20 * HZ);
 
-	spin_unlock_irq(ha->host->host_lock);
+	spin_unlock_irq(&ha->host->host_lock);
 	WRT_REG_WORD(&reg->host_cmd, HC_SET_HOST_INT);
 	qla1280_debounce_register(&reg->istatus);
 
 	wait_for_completion(&wait);
 	timer_delete_sync(&ha->mailbox_timer);
 
-	spin_lock_irq(ha->host->host_lock);
+	spin_lock_irq(&ha->host->host_lock);
 
 	ha->mailbox_wait = NULL;
 
@@ -2560,9 +2560,9 @@ qla1280_bus_reset(struct scsi_qla_host *ha, int bus)
 			ha->bus_settings[bus].scsi_bus_dead = 1;
 		ha->bus_settings[bus].failed_reset_count++;
 	} else {
-		spin_unlock_irq(ha->host->host_lock);
+		spin_unlock_irq(&ha->host->host_lock);
 		ssleep(reset_delay);
-		spin_lock_irq(ha->host->host_lock);
+		spin_lock_irq(&ha->host->host_lock);
 
 		ha->bus_settings[bus].scsi_bus_dead = 0;
 		ha->bus_settings[bus].failed_reset_count = 0;
diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
index a4ca22024edebe..781149497ff807 100644
--- a/drivers/scsi/qla2xxx/qla_attr.c
+++ b/drivers/scsi/qla2xxx/qla_attr.c
@@ -2814,13 +2814,13 @@ qla2x00_dev_loss_tmo_callbk(struct fc_rport *rport)
 	 * Transport has effectively 'deleted' the rport, clear
 	 * all local references.
 	 */
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	/* Confirm port has not reappeared before clearing pointers. */
 	if (rport->port_state != FC_PORTSTATE_ONLINE) {
 		fcport->rport = NULL;
 		*((fc_port_t **)rport->dd_data) = NULL;
 	}
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 
 	if (test_bit(ABORT_ISP_ACTIVE, &fcport->vha->dpc_flags))
 		return;
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index bb0cc71de37b8c..ccb039d97353b2 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -6247,9 +6247,9 @@ qla2x00_reg_remote_port(scsi_qla_host_t *vha, fc_port_t *fcport)
 		return;
 	}
 
-	spin_lock_irqsave(fcport->vha->host->host_lock, flags);
+	spin_lock_irqsave(&fcport->vha->host->host_lock, flags);
 	*((fc_port_t **)rport->dd_data) = fcport;
-	spin_unlock_irqrestore(fcport->vha->host->host_lock, flags);
+	spin_unlock_irqrestore(&fcport->vha->host->host_lock, flags);
 	fcport->dev_loss_tmo = rport->dev_loss_tmo;
 
 	rport->supported_classes = fcport->supported_classes;
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 63b70d4abbf02f..89c2af3fee4f2d 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -3263,21 +3263,24 @@ qla2x00_handle_dif_error(srb_t *sp, void *pkt)
 
 	/* check guard */
 	if (e_guard != a_guard) {
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x1);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 		return 1;
 	}
 
 	/* check ref tag */
 	if (e_ref_tag != a_ref_tag) {
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x3);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 		return 1;
 	}
 
 	/* check appl tag */
 	if (e_app_tag != a_app_tag) {
-		scsi_build_sense(cmd, 1, ILLEGAL_REQUEST, 0x10, 0x2);
+		scsi_set_sense(cmd, 1, ILLEGAL_REQUEST,
+			       LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED);
 		set_host_byte(cmd, DID_ABORT);
 		return 1;
 	}
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index 53a505df8da0a2..431a59d499b351 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -2243,18 +2243,20 @@ void qlt_free_ul_mcmd(struct qla_hw_data *ha, struct qla_tgt_mgmt_cmd *mcmd)
  * ha->hardware_lock supposed to be held on entry. Might drop it, then
  * reacquire
  */
-void qlt_send_resp_ctio(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
-    uint8_t scsi_status, uint8_t sense_key, uint8_t asc, uint8_t ascq)
+static void qlt_send_resp_ctio(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
+    uint8_t scsi_status, uint8_t sense_key, uint16_t sense_code)
 {
 	struct atio_from_isp *atio = &cmd->atio;
 	struct ctio7_to_24xx *ctio;
 	uint16_t temp;
 	struct scsi_qla_host *vha = cmd->vha;
+	uint8_t asc = scsi_sense_code_asc(sense_code);
+	uint8_t ascq = scsi_sense_code_ascq(sense_code);
 
 	ql_dbg(ql_dbg_tgt_dif, vha, 0x3066,
 	    "Sending response CTIO7 (vha=%p, atio=%p, scsi_status=%02x, "
 	    "sense_key=%02x, asc=%02x, ascq=%02x",
-	    vha, atio, scsi_status, sense_key, asc, ascq);
+	       vha, atio, scsi_status, sense_key, asc, ascq);
 
 	ctio = (struct ctio7_to_24xx *)qla2x00_alloc_iocbs(vha, NULL);
 	if (!ctio) {
@@ -2677,44 +2679,47 @@ static void qlt_print_dif_err(struct qla_tgt_prm *prm)
 {
 	struct qla_tgt_cmd *cmd;
 	struct scsi_qla_host *vha;
+	u16 scode;
 
 	/* asc 0x10=dif error */
-	if (prm->sense_buffer && (prm->sense_buffer[12] == 0x10)) {
-		cmd = prm->cmd;
-		vha = cmd->vha;
-		/* ASCQ */
-		switch (prm->sense_buffer[13]) {
-		case 1:
-			ql_dbg(ql_dbg_tgt_dif, vha, 0xe00b,
-			    "BE detected Guard TAG ERR: lba[0x%llx|%lld] len[0x%x] "
-			    "se_cmd=%p tag[%x]",
-			    cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
-			    cmd->atio.u.isp24.exchange_addr);
-			break;
-		case 2:
-			ql_dbg(ql_dbg_tgt_dif, vha, 0xe00c,
-			    "BE detected APP TAG ERR: lba[0x%llx|%lld] len[0x%x] "
-			    "se_cmd=%p tag[%x]",
-			    cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
-			    cmd->atio.u.isp24.exchange_addr);
-			break;
-		case 3:
-			ql_dbg(ql_dbg_tgt_dif, vha, 0xe00f,
-			    "BE detected REF TAG ERR: lba[0x%llx|%lld] len[0x%x] "
-			    "se_cmd=%p tag[%x]",
-			    cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
-			    cmd->atio.u.isp24.exchange_addr);
-			break;
-		default:
-			ql_dbg(ql_dbg_tgt_dif, vha, 0xe010,
-			    "BE detected Dif ERR: lba[%llx|%lld] len[%x] "
-			    "se_cmd=%p tag[%x]",
-			    cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
-			    cmd->atio.u.isp24.exchange_addr);
-			break;
-		}
-		ql_dump_buffer(ql_dbg_tgt_dif, vha, 0xe011, cmd->cdb, 16);
+	if (!prm->sense_buffer ||
+	    prm->sense_buffer[12] != ASC_ID_CRC_OR_ECC_ERROR)
+		return;
+
+	cmd = prm->cmd;
+	vha = cmd->vha;
+	scode = scsi_sense_code(prm->sense_buffer[12], prm->sense_buffer[13]);
+	switch (scode) {
+	case LOGICAL_BLOCK_GUARD_CHECK_FAILED:
+		ql_dbg(ql_dbg_tgt_dif, vha, 0xe00b,
+		       "BE detected Guard TAG ERR: lba[0x%llx|%lld] len[0x%x] "
+		       "se_cmd=%p tag[%x]",
+		       cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
+		       cmd->atio.u.isp24.exchange_addr);
+		break;
+	case LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED:
+		ql_dbg(ql_dbg_tgt_dif, vha, 0xe00c,
+		       "BE detected APP TAG ERR: lba[0x%llx|%lld] len[0x%x] "
+		       "se_cmd=%p tag[%x]",
+		       cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
+		       cmd->atio.u.isp24.exchange_addr);
+		break;
+	case LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED:
+		ql_dbg(ql_dbg_tgt_dif, vha, 0xe00f,
+		       "BE detected REF TAG ERR: lba[0x%llx|%lld] len[0x%x] "
+		       "se_cmd=%p tag[%x]",
+		       cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
+		       cmd->atio.u.isp24.exchange_addr);
+		break;
+	default:
+		ql_dbg(ql_dbg_tgt_dif, vha, 0xe010,
+		       "BE detected Dif ERR: lba[%llx|%lld] len[%x] "
+		       "se_cmd=%p tag[%x]",
+		       cmd->lba, cmd->lba, cmd->num_blks, &cmd->se_cmd,
+		       cmd->atio.u.isp24.exchange_addr);
+		break;
 	}
+	ql_dump_buffer(ql_dbg_tgt_dif, vha, 0xe011, cmd->cdb, 16);
 }
 
 /*
@@ -3467,7 +3472,8 @@ qlt_handle_dif_error(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 	uint8_t		*ap = &sts->actual_dif[0];
 	uint8_t		*ep = &sts->expected_dif[0];
 	uint64_t	lba = cmd->se_cmd.t_task_lba;
-	uint8_t scsi_status, sense_key, asc, ascq;
+	uint8_t scsi_status, sense_key;
+	uint16_t sense_code = 0;
 	struct scsi_qla_host *vha = cmd->vha;
 
 	cmd->trc_flags |= TRC_DIF_ERR;
@@ -3483,7 +3489,7 @@ qlt_handle_dif_error(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 	ql_dbg(ql_dbg_tgt_dif, vha, 0xf075,
 	    "%s: aborted %d state %d\n", __func__, cmd->aborted, cmd->state);
 
-	scsi_status = sense_key = asc = ascq = 0;
+	scsi_status = sense_key = 0;
 
 	/* check appl tag */
 	if (cmd->e_app_tag != cmd->a_app_tag) {
@@ -3497,8 +3503,7 @@ qlt_handle_dif_error(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 		cmd->dif_err_code = DIF_ERR_APP;
 		scsi_status = SAM_STAT_CHECK_CONDITION;
 		sense_key = ABORTED_COMMAND;
-		asc = 0x10;
-		ascq = 0x2;
+		sense_code = LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED;
 	}
 
 	/* check ref tag */
@@ -3513,8 +3518,7 @@ qlt_handle_dif_error(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 		cmd->dif_err_code = DIF_ERR_REF;
 		scsi_status = SAM_STAT_CHECK_CONDITION;
 		sense_key = ABORTED_COMMAND;
-		asc = 0x10;
-		ascq = 0x3;
+		sense_code = LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED;
 		goto out;
 	}
 
@@ -3530,8 +3534,7 @@ qlt_handle_dif_error(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 		cmd->dif_err_code = DIF_ERR_GRD;
 		scsi_status = SAM_STAT_CHECK_CONDITION;
 		sense_key = ABORTED_COMMAND;
-		asc = 0x10;
-		ascq = 0x1;
+		sense_code = LOGICAL_BLOCK_GUARD_CHECK_FAILED;
 	}
 out:
 	switch (cmd->state) {
@@ -3547,8 +3550,8 @@ qlt_handle_dif_error(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 			break;
 		}
 
-		qlt_send_resp_ctio(qpair, cmd, scsi_status, sense_key, asc,
-		    ascq);
+		qlt_send_resp_ctio(qpair, cmd, scsi_status, sense_key,
+				   sense_code);
 		/* assume scsi status gets out on the wire.
 		 * Will not wait for completion.
 		 */
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index 61072fb41b29f8..71bc1affb1bc91 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -1178,7 +1178,4 @@ extern irqreturn_t qla83xx_msix_atio_q(int, void *);
 extern void qlt_logo_completion_handler(fc_port_t *, int);
 extern void qlt_do_generation_tick(struct scsi_qla_host *, int *);
 
-void qlt_send_resp_ctio(struct qla_qpair *, struct qla_tgt_cmd *, uint8_t,
-    uint8_t, uint8_t, uint8_t);
-
 #endif /* __QLA_TARGET_H */
diff --git a/drivers/scsi/qlogicfas408.c b/drivers/scsi/qlogicfas408.c
index d36293bc271731..af6f6eb516363e 100644
--- a/drivers/scsi/qlogicfas408.c
+++ b/drivers/scsi/qlogicfas408.c
@@ -454,9 +454,9 @@ irqreturn_t qlogicfas408_ihandl(int irq, void *dev_id)
 	unsigned long flags;
 	struct Scsi_Host *host = dev_id;
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	ql_ihandl(dev_id);
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 	return IRQ_HANDLED;
 }
 
@@ -536,9 +536,9 @@ int qlogicfas408_host_reset(struct scsi_cmnd *cmd)
 
 	priv->qabort = 2;
 
-	spin_lock_irqsave(cmd->device->host->host_lock, flags);
+	spin_lock_irqsave(&cmd->device->host->host_lock, flags);
 	ql_zap(priv);
-	spin_unlock_irqrestore(cmd->device->host->host_lock, flags);
+	spin_unlock_irqrestore(&cmd->device->host->host_lock, flags);
 
 	return SUCCESS;
 }
diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c
index ea0a2b5a0a4200..09891606183569 100644
--- a/drivers/scsi/qlogicpti.c
+++ b/drivers/scsi/qlogicpti.c
@@ -333,7 +333,7 @@ static int qlogicpti_reset_hardware(struct Scsi_Host *host)
 
 	risc_code_addr = 0x1000;	/* all load addresses are at 0x1000 */
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 
 	sbus_writew(HCCTRL_PAUSE, qpti->qregs + HCCTRL);
 
@@ -382,7 +382,7 @@ static int qlogicpti_reset_hardware(struct Scsi_Host *host)
 	if (qlogicpti_mbox_command(qpti, param, 1)) {
 		printk(KERN_EMERG "qlogicpti%d: Cannot execute ISP firmware.\n",
 		       qpti->qpti_id);
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 		return 1;
 	}
 
@@ -393,7 +393,7 @@ static int qlogicpti_reset_hardware(struct Scsi_Host *host)
 	   (param[0] != MBOX_COMMAND_COMPLETE)) {
 		printk(KERN_EMERG "qlogicpti%d: Cannot set initiator SCSI ID.\n",
 		       qpti->qpti_id);
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 		return 1;
 	}
 
@@ -408,7 +408,7 @@ static int qlogicpti_reset_hardware(struct Scsi_Host *host)
 	if (qlogicpti_mbox_command(qpti, param, 1)) {
 		printk(KERN_EMERG "qlogicpti%d: Cannot init response queue.\n",
 		       qpti->qpti_id);
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 		return 1;
 	}
 
@@ -420,7 +420,7 @@ static int qlogicpti_reset_hardware(struct Scsi_Host *host)
 	if (qlogicpti_mbox_command(qpti, param, 1)) {
 		printk(KERN_EMERG "qlogicpti%d: Cannot init request queue.\n",
 		       qpti->qpti_id);
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 		return 1;
 	}
 
@@ -466,7 +466,7 @@ static int qlogicpti_reset_hardware(struct Scsi_Host *host)
 	qlogicpti_mbox_command(qpti, param, 0);
 	qpti->send_marker = 1;
 
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 	return 0;
 }
 
@@ -501,7 +501,7 @@ static int qlogicpti_load_firmware(struct qlogicpti *qpti)
 	risc_code_addr = 0x1000;	/* all f/w modules load at 0x1000 */
 	risc_code_length = fw->size / 2;
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 
 	/* Verify the checksum twice, one before loading it, and once
 	 * afterwards via the mailbox commands.
@@ -642,7 +642,7 @@ static int qlogicpti_load_firmware(struct qlogicpti *qpti)
 	}
 
 out:
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 outfirm:
 	release_firmware(fw);
 	return err;
@@ -1207,7 +1207,7 @@ static irqreturn_t qpti_intr(int irq, void *dev_id)
 	unsigned long flags;
 	struct scsi_cmnd *dq;
 
-	spin_lock_irqsave(qpti->qhost->host_lock, flags);
+	spin_lock_irqsave(&qpti->qhost->host_lock, flags);
 	dq = qlogicpti_intr_handler(qpti);
 
 	if (dq != NULL) {
@@ -1219,7 +1219,7 @@ static irqreturn_t qpti_intr(int irq, void *dev_id)
 			dq = next;
 		} while (dq != NULL);
 	}
-	spin_unlock_irqrestore(qpti->qhost->host_lock, flags);
+	spin_unlock_irqrestore(&qpti->qhost->host_lock, flags);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index f285521d9de6d5..05232ab6366feb 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -606,7 +606,8 @@ int scsi_report_opcode(struct scsi_device *sdev, unsigned char *buffer,
 		return result;
 	if (result && scsi_sense_valid(&sshdr) &&
 	    sshdr.sense_key == ILLEGAL_REQUEST &&
-	    (sshdr.asc == 0x20 || sshdr.asc == 0x24) && sshdr.ascq == 0x00)
+	    (sshdr.sense_code == INVALID_COMMAND_OP_CODE ||
+	     sshdr.sense_code == INVALID_FIELD_IN_CDB))
 		return -EINVAL;
 
 	if ((buffer[1] & 3) == 3) /* Command supported */
@@ -842,7 +843,7 @@ struct scsi_device *__scsi_iterate_devices(struct Scsi_Host *shost,
 	struct scsi_device *next = NULL;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	while (list->next != &shost->__devices) {
 		next = list_entry(list->next, struct scsi_device, siblings);
 		/*
@@ -854,7 +855,7 @@ struct scsi_device *__scsi_iterate_devices(struct Scsi_Host *shost,
 		next = NULL;
 		list = list->next;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (prev)
 		scsi_device_put(prev);
@@ -961,11 +962,11 @@ struct scsi_device *scsi_device_lookup_by_target(struct scsi_target *starget,
 	struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	sdev = __scsi_device_lookup_by_target(starget, lun);
 	if (sdev && scsi_device_get(sdev))
 		sdev = NULL;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	return sdev;
 }
@@ -1021,11 +1022,11 @@ struct scsi_device *scsi_device_lookup(struct Scsi_Host *shost,
 	struct scsi_device *sdev;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	sdev = __scsi_device_lookup(shost, channel, id, lun);
 	if (sdev && scsi_device_get(sdev))
 		sdev = NULL;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	return sdev;
 }
diff --git a/drivers/scsi/scsi_common.c b/drivers/scsi/scsi_common.c
index e1a2a62b6910fe..94b16033624724 100644
--- a/drivers/scsi/scsi_common.c
+++ b/drivers/scsi/scsi_common.c
@@ -191,6 +191,8 @@ EXPORT_SYMBOL(int_to_scsilun);
 bool scsi_normalize_sense(const u8 *sense_buffer, int sb_len,
 			  struct scsi_sense_hdr *sshdr)
 {
+	u8 asc = 0, ascq = 0;
+
 	memset(sshdr, 0, sizeof(struct scsi_sense_hdr));
 
 	if (!sense_buffer || !sb_len)
@@ -208,9 +210,9 @@ bool scsi_normalize_sense(const u8 *sense_buffer, int sb_len,
 		if (sb_len > 1)
 			sshdr->sense_key = (sense_buffer[1] & 0xf);
 		if (sb_len > 2)
-			sshdr->asc = sense_buffer[2];
+			asc = sense_buffer[2];
 		if (sb_len > 3)
-			sshdr->ascq = sense_buffer[3];
+			ascq = sense_buffer[3];
 		if (sb_len > 7)
 			sshdr->additional_length = sense_buffer[7];
 	} else {
@@ -222,12 +224,14 @@ bool scsi_normalize_sense(const u8 *sense_buffer, int sb_len,
 		if (sb_len > 7) {
 			sb_len = min(sb_len, sense_buffer[7] + 8);
 			if (sb_len > 12)
-				sshdr->asc = sense_buffer[12];
+				asc = sense_buffer[12];
 			if (sb_len > 13)
-				sshdr->ascq = sense_buffer[13];
+				ascq = sense_buffer[13];
 		}
 	}
 
+	sshdr->sense_code = scsi_sense_code(asc, ascq);
+
 	return true;
 }
 EXPORT_SYMBOL(scsi_normalize_sense);
@@ -272,17 +276,19 @@ const u8 * scsi_sense_desc_find(const u8 * sense_buffer, int sb_len,
 EXPORT_SYMBOL(scsi_sense_desc_find);
 
 /**
- * scsi_build_sense_buffer - build sense data in a buffer
+ * scsi_set_sense_buffer - build sense data in a buffer
  * @desc:	Sense format (non-zero == descriptor format,
  *              0 == fixed format)
  * @buf:	Where to build sense data
  * @key:	Sense key
- * @asc:	Additional sense code
- * @ascq:	Additional sense code qualifier
+ * @code:	Additional sense code and sense code qualifier
  *
  **/
-void scsi_build_sense_buffer(int desc, u8 *buf, u8 key, u8 asc, u8 ascq)
+void scsi_set_sense_buffer(int desc, u8 *buf, u8 key, u16 code)
 {
+	u8 asc = scsi_sense_code_asc(code);
+	u8 ascq = scsi_sense_code_ascq(code);
+
 	if (desc) {
 		buf[0] = 0x72;	/* descriptor, current */
 		buf[1] = key;
@@ -297,7 +303,7 @@ void scsi_build_sense_buffer(int desc, u8 *buf, u8 key, u8 asc, u8 ascq)
 		buf[13] = ascq;
 	}
 }
-EXPORT_SYMBOL(scsi_build_sense_buffer);
+EXPORT_SYMBOL(scsi_set_sense_buffer);
 
 /**
  * scsi_set_sense_information - set the information field in a
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 3346562275f801..6941809dfdb750 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -44,6 +44,7 @@
 #include <linux/debugfs.h>
 #include <linux/async.h>
 #include <linux/cleanup.h>
+#include <linux/parser.h>
 
 #include <net/checksum.h>
 
@@ -67,9 +68,6 @@ static const char *sdebug_version_date = "20210520";
 
 #define MY_NAME "scsi_debug"
 
-/* Additional Sense Code Qualifier (ASCQ) */
-#define ACK_NAK_TO 0x3
-
 /* Default values for driver parameters */
 #define DEF_NUM_HOST   1
 #define DEF_NUM_TGTS   1
@@ -79,7 +77,7 @@ static const char *sdebug_version_date = "20210520";
  */
 #define DEF_ATO 1
 #define DEF_CDB_LEN 10
-#define DEF_JDELAY   1		/* if > 0 unit is a jiffy */
+#define DEF_JDELAY   0		/* if > 0 unit is a jiffy */
 #define DEF_DEV_SIZE_PRE_INIT   0
 #define DEF_DEV_SIZE_MB   8
 #define DEF_ZBC_DEV_SIZE_MB   128
@@ -106,6 +104,7 @@ static const char *sdebug_version_date = "20210520";
 #define DEF_PTYPE   TYPE_DISK
 #define DEF_RANDOM false
 #define DEF_REMOVABLE false
+#define DEF_CLUSTERING true
 #define DEF_SCSI_LEVEL   7    /* INQUIRY, byte2 [6->SPC-4; 7->SPC-5] */
 #define DEF_SECTOR_SIZE 512
 #define DEF_UNMAP_ALIGNMENT 0
@@ -217,8 +216,7 @@ struct tape_block {
  * /sys/class/scsi_device/<h:c:t:l>/device/queue_depth
  * but cannot exceed SDEBUG_CANQUEUE .
  */
-#define SDEBUG_CANQUEUE_WORDS  3	/* a WORD is bits in a long */
-#define SDEBUG_CANQUEUE  (SDEBUG_CANQUEUE_WORDS * BITS_PER_LONG)
+#define SDEBUG_CANQUEUE  (4096)
 #define DEF_CMD_PER_LUN  SDEBUG_CANQUEUE
 
 /* UA - Unit Attention; SA - Service Action; SSU - Start Stop Unit */
@@ -850,7 +848,7 @@ static int sdebug_num_hosts;
 static int sdebug_add_host = DEF_NUM_HOST;  /* in sysfs this is relative */
 static int sdebug_ato = DEF_ATO;
 static int sdebug_cdb_len = DEF_CDB_LEN;
-static int sdebug_jdelay = DEF_JDELAY;	/* if > 0 then unit is jiffies */
+static int sdebug_jdelay = DEF_JDELAY;
 static int sdebug_dev_size_mb = DEF_DEV_SIZE_PRE_INIT;
 static int sdebug_dif = DEF_DIF;
 static int sdebug_dix = DEF_DIX;
@@ -899,7 +897,7 @@ static int sdebug_uuid_ctl = DEF_UUID_CTL;
 static bool sdebug_random = DEF_RANDOM;
 static bool sdebug_per_host_store = DEF_PER_HOST_STORE;
 static bool sdebug_removable = DEF_REMOVABLE;
-static bool sdebug_clustering;
+static bool sdebug_clustering = DEF_CLUSTERING;
 static bool sdebug_host_lock = DEF_HOST_LOCK;
 static bool sdebug_strict = DEF_STRICT;
 static bool sdebug_no_rwlock;
@@ -1168,6 +1166,95 @@ static const struct file_operations sdebug_error_fops = {
 	.release = single_release,
 };
 
+enum corrupt_options {
+	Opt_lba			= (1u << 0),
+	Opt_num			= (1u << 1),
+	Opt_bit_errors		= (1u << 2),
+	Opt_reftag_adjust	= (1u << 3),
+
+	Opt_invalid,
+};
+
+static const match_table_t corrupt_tokens = {
+	{ Opt_lba,			"lba=%u"		},
+	{ Opt_num,			"num=%u"		},
+	{ Opt_bit_errors,		"bit_errors=%u"		},
+	{ Opt_reftag_adjust,		"reftag_adjust=%d"	},
+	{ Opt_invalid,			NULL,			},
+};
+
+static int corrupt_lbas(struct sdebug_dev_info *devip, u64 lba, u32 num,
+		u32 nr_bit_errors, s32 reftag_adjust);
+static ssize_t sdebug_corrupt_write(struct file *file, const char __user *ubuf,
+		size_t count, loff_t *ppos)
+{
+	struct scsi_device *sdev = file->f_inode->i_private;
+	struct sdebug_dev_info *devip = sdev->hostdata;
+	substring_t args[MAX_OPT_ARGS];
+	char *buf, *options, *p;
+	int error = 0;
+	u64 lba = 0;
+	u32 num = 1;
+	u32 nr_bit_errors = 0;
+	s32 reftag_adjust = 0;
+
+	buf = memdup_user_nul(ubuf, count);
+	if (IS_ERR(buf))
+		return PTR_ERR(buf);
+
+	options = buf;
+	while ((p = strsep(&options, ",\n")) != NULL) {
+		if (!*p)
+			continue;
+		switch (match_token(p, corrupt_tokens, args)) {
+		case Opt_lba:
+			error = match_u64(args, &lba);
+			break;
+		case Opt_num:
+			error = match_uint(args, &num);
+			break;
+		case Opt_bit_errors:
+			error = match_uint(args, &nr_bit_errors);
+			break;
+		case Opt_reftag_adjust:
+			error = match_int(args, &reftag_adjust);
+			break;
+		default:
+			pr_warn("unknown parameter or missing value '%s'\n", p);
+			error = -EINVAL;
+		}
+		if (error)
+			goto out_free_buf;
+	}
+
+	if (num == 0) {
+		pr_warn("invalid number of logical blocks: %u\n", num);
+		error = -EINVAL;
+		goto out_free_buf;
+	}
+
+	if (reftag_adjust &&
+	    (!sdebug_dix ||
+	     (sdebug_dif != T10_PI_TYPE1_PROTECTION &&
+	      sdebug_dif != T10_PI_TYPE2_PROTECTION))) {
+		pr_warn("reftag adjust requires type 1 or type 2 PI with DIX\n");
+		error = -EINVAL;
+		goto out_free_buf;
+	}
+
+	error = corrupt_lbas(devip, lba, num, nr_bit_errors, reftag_adjust);
+
+out_free_buf:
+	kfree(buf);
+	if (error)
+		return error;
+	return count;
+}
+
+static const struct file_operations sdebug_corrupt_fops = {
+	.write		= sdebug_corrupt_write,
+};
+
 static int sdebug_target_reset_fail_show(struct seq_file *m, void *p)
 {
 	struct scsi_target *starget = (struct scsi_target *)m->private;
@@ -1320,9 +1407,13 @@ static void mk_sense_invalid_fld(struct scsi_cmnd *scp,
 		sdev_printk(KERN_ERR, scp->device, "sense_buffer is NULL\n");
 		return;
 	}
-	asc = c_d ? INVALID_FIELD_IN_CDB : INVALID_FIELD_IN_PARAM_LIST;
+	if (c_d)
+		asc = ASC_INVALID_FIELD_IN_CDB;
+	else
+		asc = ASC_INVALID_FIELD_IN_PARAMETER_LIST;
 	memset(sbuff, 0, SCSI_SENSE_BUFFERSIZE);
-	scsi_build_sense(scp, sdebug_dsense, ILLEGAL_REQUEST, asc, 0);
+	scsi_set_sense(scp, sdebug_dsense, ILLEGAL_REQUEST,
+		       scsi_sense_code(asc, 0));
 	memset(sks, 0, sizeof(sks));
 	sks[0] = 0x80;
 	if (c_d)
@@ -1346,7 +1437,7 @@ static void mk_sense_invalid_fld(struct scsi_cmnd *scp,
 			    my_name, asc, c_d ? 'C' : 'D', in_byte, in_bit);
 }
 
-static void mk_sense_buffer(struct scsi_cmnd *scp, int key, int asc, int asq)
+static void mk_sense_buffer(struct scsi_cmnd *scp, u8 key, u16 code)
 {
 	if (!scp->sense_buffer) {
 		sdev_printk(KERN_ERR, scp->device, "sense_buffer is NULL\n");
@@ -1354,16 +1445,16 @@ static void mk_sense_buffer(struct scsi_cmnd *scp, int key, int asc, int asq)
 	}
 	memset(scp->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
 
-	scsi_build_sense(scp, sdebug_dsense, key, asc, asq);
+	scsi_set_sense(scp, sdebug_dsense, key, code);
 
 	if (sdebug_verbose)
 		sdev_printk(KERN_INFO, scp->device,
-			    "%s:  [sense_key,asc,ascq]: [0x%x,0x%x,0x%x]\n",
-			    my_name, key, asc, asq);
+			    "%s:  [sense_key,sense_code]: [0x%02x,0x%04x]\n",
+			    my_name, key, code);
 }
 
 /* Sense data that has information fields for tapes */
-static void mk_sense_info_tape(struct scsi_cmnd *scp, int key, int asc, int asq,
+static void mk_sense_info_tape(struct scsi_cmnd *scp, u8 key, u16 code,
 			unsigned int information, unsigned char tape_flags)
 {
 	if (!scp->sense_buffer) {
@@ -1372,7 +1463,7 @@ static void mk_sense_info_tape(struct scsi_cmnd *scp, int key, int asc, int asq,
 	}
 	memset(scp->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
 
-	scsi_build_sense(scp, /* sdebug_dsense */ 0, key, asc, asq);
+	scsi_set_sense(scp, /* sdebug_dsense */ 0, key, code);
 	/* only fixed format so far */
 
 	scp->sense_buffer[0] |= 0x80; /* valid */
@@ -1381,13 +1472,13 @@ static void mk_sense_info_tape(struct scsi_cmnd *scp, int key, int asc, int asq,
 
 	if (sdebug_verbose)
 		sdev_printk(KERN_INFO, scp->device,
-			    "%s:  [sense_key,asc,ascq]: [0x%x,0x%x,0x%x]\n",
-			    my_name, key, asc, asq);
+			    "%s:  [sense_key,sense_code]: [0x%02x,0x%04x]\n",
+			    my_name, key, code);
 }
 
 static void mk_sense_invalid_opcode(struct scsi_cmnd *scp)
 {
-	mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_OPCODE, 0);
+	mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_COMMAND_OP_CODE);
 }
 
 static int scsi_debug_ioctl(struct scsi_device *dev, unsigned int cmd,
@@ -1484,46 +1575,43 @@ static int make_ua(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 
 		switch (k) {
 		case SDEBUG_UA_POR:
-			mk_sense_buffer(scp, UNIT_ATTENTION, UA_RESET_ASC,
-					POWER_ON_RESET_ASCQ);
+			mk_sense_buffer(scp, UNIT_ATTENTION,
+				POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED);
 			if (sdebug_verbose)
 				cp = "power on reset";
 			break;
 		case SDEBUG_UA_POOCCUR:
-			mk_sense_buffer(scp, UNIT_ATTENTION, UA_RESET_ASC,
-					POWER_ON_OCCURRED_ASCQ);
+			mk_sense_buffer(scp, UNIT_ATTENTION, POWER_ON_OCCURRED);
 			if (sdebug_verbose)
 				cp = "power on occurred";
 			break;
 		case SDEBUG_UA_BUS_RESET:
-			mk_sense_buffer(scp, UNIT_ATTENTION, UA_RESET_ASC,
-					BUS_RESET_ASCQ);
+			mk_sense_buffer(scp, UNIT_ATTENTION,
+					SCSI_BUS_RESET_OCCURRED);
 			if (sdebug_verbose)
 				cp = "bus reset";
 			break;
 		case SDEBUG_UA_MODE_CHANGED:
-			mk_sense_buffer(scp, UNIT_ATTENTION, UA_CHANGED_ASC,
-					MODE_CHANGED_ASCQ);
+			mk_sense_buffer(scp, UNIT_ATTENTION,
+					MODE_PARAMETERS_CHANGED);
 			if (sdebug_verbose)
 				cp = "mode parameters changed";
 			break;
 		case SDEBUG_UA_CAPACITY_CHANGED:
-			mk_sense_buffer(scp, UNIT_ATTENTION, UA_CHANGED_ASC,
-					CAPACITY_CHANGED_ASCQ);
+			mk_sense_buffer(scp, UNIT_ATTENTION,
+					CAPACITY_DATA_HAS_CHANGED);
 			if (sdebug_verbose)
 				cp = "capacity data changed";
 			break;
 		case SDEBUG_UA_MICROCODE_CHANGED:
 			mk_sense_buffer(scp, UNIT_ATTENTION,
-					TARGET_CHANGED_ASC,
-					MICROCODE_CHANGED_ASCQ);
+					MICROCODE_HAS_BEEN_CHANGED);
 			if (sdebug_verbose)
 				cp = "microcode has been changed";
 			break;
 		case SDEBUG_UA_MICROCODE_CHANGED_WO_RESET:
 			mk_sense_buffer(scp, UNIT_ATTENTION,
-					TARGET_CHANGED_ASC,
-					MICROCODE_CHANGED_WO_RESET_ASCQ);
+				MICROCODE_HAS_BEEN_CHANGED_WITHOUT_RESET);
 			if (sdebug_verbose)
 				cp = "microcode has been changed without reset";
 			break;
@@ -1539,14 +1627,13 @@ static int make_ua(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 			if (sdebug_scsi_level >= 6)	/* SPC-4 and above */
 				clear_luns_changed_on_target(devip);
 			mk_sense_buffer(scp, UNIT_ATTENTION,
-					TARGET_CHANGED_ASC,
-					LUNS_CHANGED_ASCQ);
+					REPORTED_LUNS_DATA_HAS_CHANGED);
 			if (sdebug_verbose)
 				cp = "reported luns data has changed";
 			break;
 		case SDEBUG_UA_NOT_READY_TO_READY:
-			mk_sense_buffer(scp, UNIT_ATTENTION, UA_READY_ASC,
-					0);
+			mk_sense_buffer(scp, UNIT_ATTENTION,
+				NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED);
 			if (sdebug_verbose)
 				cp = "not ready to ready transition/media change";
 			break;
@@ -2184,14 +2271,14 @@ static int resp_requests(struct scsi_cmnd *scp,
 		if (dsense) {
 			arr[0] = 0x72;
 			arr[1] = NOT_READY;
-			arr[2] = LOGICAL_UNIT_NOT_READY;
+			arr[2] = ASC_LU_NOT_READY;
 			arr[3] = (stopped_state == 2) ? 0x1 : 0x2;
 			len = 8;
 		} else {
 			arr[0] = 0x70;
 			arr[2] = NOT_READY;		/* NO_SENSE in sense_key */
 			arr[7] = 0xa;			/* 18 byte sense buffer */
-			arr[12] = LOGICAL_UNIT_NOT_READY;
+			arr[12] = ASC_LU_NOT_READY;
 			arr[13] = (stopped_state == 2) ? 0x1 : 0x2;
 		}
 	} else if ((iec_m_pg[2] & 0x4) && (6 == (iec_m_pg[3] & 0xf))) {
@@ -2199,14 +2286,14 @@ static int resp_requests(struct scsi_cmnd *scp,
 		if (dsense) {
 			arr[0] = 0x72;
 			arr[1] = 0x0;		/* NO_SENSE in sense_key */
-			arr[2] = THRESHOLD_EXCEEDED;
+			arr[2] = ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED;
 			arr[3] = 0xff;		/* Failure prediction(false) */
 			len = 8;
 		} else {
 			arr[0] = 0x70;
 			arr[2] = 0x0;		/* NO_SENSE in sense_key */
 			arr[7] = 0xa;   	/* 18 byte sense buffer */
-			arr[12] = THRESHOLD_EXCEEDED;
+			arr[12] = ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED;
 			arr[13] = 0xff;		/* Failure prediction(false) */
 		}
 	} else {	/* nothing to report */
@@ -2457,8 +2544,7 @@ static int resp_rsup_opcodes(struct scsi_cmnd *scp,
 		a_len = alloc_len;
 	arr = kzalloc((a_len < 256) ? 320 : a_len + 64, GFP_ATOMIC);
 	if (NULL == arr) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
-				INSUFF_RES_ASCQ);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFFICIENT_RESOURCES);
 		return check_condition_result;
 	}
 	switch (reporting_opts) {
@@ -2924,8 +3010,9 @@ static int resp_mode_sense(struct scsi_cmnd *scp,
 	else
 		bd_len = 0;
 	alloc_len = msense_6 ? cmd[4] : get_unaligned_be16(cmd + 7);
-	if (0x3 == pcontrol) {  /* Saving values not supported */
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, SAVING_PARAMS_UNSUP, 0);
+	if (0x3 == pcontrol) {
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				SAVING_PARAMETERS_NOT_SUPPORTED);
 		return check_condition_result;
 	}
 	target_dev_id = ((devip->sdbg_host->shost->host_no + 1) * 2000) +
@@ -3174,7 +3261,7 @@ static int resp_mode_select(struct scsi_cmnd *scp,
 		       (arr[off + 1] + 2);
 	if ((pg_len + off) > param_len) {
 		mk_sense_buffer(scp, ILLEGAL_REQUEST,
-				PARAMETER_LIST_LENGTH_ERR, 0);
+				PARAMETER_LIST_LENGTH_ERROR);
 		return check_condition_result;
 	}
 	switch (mpage) {
@@ -3256,7 +3343,7 @@ static int resp_ie_l_pg(unsigned char *arr)
 
 	memcpy(arr, ie_l_pg, sizeof(ie_l_pg));
 	if (iec_m_pg[2] & 0x4) {	/* TEST bit set */
-		arr[4] = THRESHOLD_EXCEEDED;
+		arr[4] = ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED;
 		arr[5] = 0xff;
 	}
 	return sizeof(ie_l_pg);
@@ -3413,7 +3500,8 @@ static int resp_locate(struct scsi_cmnd *scp,
 			break;
 	if (i < pos) {
 		devip->tape_location[partition] = i;
-		mk_sense_buffer(scp, BLANK_CHECK, 0x05, 0);
+		mk_sense_buffer(scp, BLANK_CHECK,
+				LU_DOES_NOT_RESPOND_TO_SELECTION);
 		return check_condition_result;
 	}
 	devip->tape_location[partition] = pos;
@@ -3438,8 +3526,9 @@ static int resp_write_filemarks(struct scsi_cmnd *scp,
 	for (i = 0, pos = devip->tape_location[partition]; i < count; i++, pos++) {
 		if (pos >= devip->tape_eop[partition] - 1) { /* don't overwrite EOD */
 			devip->tape_location[partition] = devip->tape_eop[partition] - 1;
-			mk_sense_info_tape(scp, VOLUME_OVERFLOW, NO_ADDITIONAL_SENSE,
-					EOP_EOM_DETECTED_ASCQ, count, SENSE_FLAG_EOM);
+			mk_sense_info_tape(scp, VOLUME_OVERFLOW,
+					   END_OF_PARTITION_MEDIUM_DETECTED,
+					   count, SENSE_FLAG_EOM);
 			return check_condition_result;
 		}
 		(devip->tape_blocks[partition] + pos)->fl_size = data;
@@ -3540,31 +3629,28 @@ static int resp_space(struct scsi_cmnd *scp,
 
 is_fm:
 	devip->tape_location[partition] = pos;
-	mk_sense_info_tape(scp, NO_SENSE, NO_ADDITIONAL_SENSE,
-			FILEMARK_DETECTED_ASCQ, count - i,
-			SENSE_FLAG_FILEMARK);
+	mk_sense_info_tape(scp, NO_SENSE, FILEMARK_DETECTED,
+			   count - i, SENSE_FLAG_FILEMARK);
 	return check_condition_result;
 
 is_eod:
 	devip->tape_location[partition] = pos;
-	mk_sense_info_tape(scp, BLANK_CHECK, NO_ADDITIONAL_SENSE,
-			EOD_DETECTED_ASCQ, count - i,
-			0);
+	mk_sense_info_tape(scp, BLANK_CHECK, END_OF_DATA_DETECTED,
+			   count - i, 0);
 	return check_condition_result;
 
 is_bop:
 	devip->tape_location[partition] = 0;
-	mk_sense_info_tape(scp, NO_SENSE, NO_ADDITIONAL_SENSE,
-			BEGINNING_OF_P_M_DETECTED_ASCQ, count - i,
-			SENSE_FLAG_EOM);
+	mk_sense_info_tape(scp, NO_SENSE,
+			   BEGINNING_OF_PARTITION_MEDIUM_DETECTED,
+			   count - i, SENSE_FLAG_EOM);
 	devip->tape_location[partition] = 0;
 	return check_condition_result;
 
 is_eop:
 	devip->tape_location[partition] = devip->tape_eop[partition] - 1;
-	mk_sense_info_tape(scp, MEDIUM_ERROR, NO_ADDITIONAL_SENSE,
-			EOP_EOM_DETECTED_ASCQ, (unsigned int)i,
-			SENSE_FLAG_EOM);
+	mk_sense_info_tape(scp, MEDIUM_ERROR, END_OF_PARTITION_MEDIUM_DETECTED,
+			   (unsigned int)i, SENSE_FLAG_EOM);
 	return check_condition_result;
 }
 
@@ -3852,8 +3938,7 @@ static int check_zbc_access_params(struct scsi_cmnd *scp,
 		/* For host-managed, reads cannot cross zone types boundaries */
 		if (zsp->z_type != zsp_end->z_type) {
 			mk_sense_buffer(scp, ILLEGAL_REQUEST,
-					LBA_OUT_OF_RANGE,
-					READ_INVDATA_ASCQ);
+					ATTEMPT_TO_READ_INVALID_DATA);
 			return check_condition_result;
 		}
 		return 0;
@@ -3861,8 +3946,8 @@ static int check_zbc_access_params(struct scsi_cmnd *scp,
 
 	/* Writing into a gap zone is not allowed */
 	if (zbc_zone_is_gap(zsp)) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE,
-				ATTEMPT_ACCESS_GAP);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				ATTEMPT_TO_ACCESS_GAP_ZONE);
 		return check_condition_result;
 	}
 
@@ -3870,8 +3955,7 @@ static int check_zbc_access_params(struct scsi_cmnd *scp,
 	if (zbc_zone_is_conv(zsp)) {
 		if (!zbc_zone_is_conv(zsp_end)) {
 			mk_sense_buffer(scp, ILLEGAL_REQUEST,
-					LBA_OUT_OF_RANGE,
-					WRITE_BOUNDARY_ASCQ);
+					WRITE_BOUNDARY_VIOLATION);
 			return check_condition_result;
 		}
 		return 0;
@@ -3881,21 +3965,19 @@ static int check_zbc_access_params(struct scsi_cmnd *scp,
 		/* Writes cannot cross sequential zone boundaries */
 		if (zsp_end != zsp) {
 			mk_sense_buffer(scp, ILLEGAL_REQUEST,
-					LBA_OUT_OF_RANGE,
-					WRITE_BOUNDARY_ASCQ);
+					WRITE_BOUNDARY_VIOLATION);
 			return check_condition_result;
 		}
 		/* Cannot write full zones */
 		if (zsp->z_cond == ZC5_FULL) {
 			mk_sense_buffer(scp, ILLEGAL_REQUEST,
-					INVALID_FIELD_IN_CDB, 0);
+					INVALID_FIELD_IN_CDB);
 			return check_condition_result;
 		}
 		/* Writes must be aligned to the zone WP */
 		if (lba != zsp->z_wp) {
 			mk_sense_buffer(scp, ILLEGAL_REQUEST,
-					LBA_OUT_OF_RANGE,
-					UNALIGNED_WRITE_ASCQ);
+					UNALIGNED_WRITE_COMMAND);
 			return check_condition_result;
 		}
 	}
@@ -3905,8 +3987,7 @@ static int check_zbc_access_params(struct scsi_cmnd *scp,
 		if (devip->max_open &&
 		    devip->nr_exp_open >= devip->max_open) {
 			mk_sense_buffer(scp, DATA_PROTECT,
-					INSUFF_RES_ASC,
-					INSUFF_ZONE_ASCQ);
+					INSUFFICIENT_ZONE_RESOURCES);
 			return check_condition_result;
 		}
 		zbc_open_zone(devip, zsp, false);
@@ -3923,17 +4004,18 @@ static inline int check_device_access_params
 	struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata;
 
 	if (lba + num > sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		return check_condition_result;
 	}
 	/* transfer length excessive (tie in to block limits VPD page) */
 	if (num > sdebug_store_sectors) {
 		/* needs work to find which cdb byte 'num' comes from */
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		return check_condition_result;
 	}
 	if (write && unlikely(sdebug_wp)) {
-		mk_sense_buffer(scp, DATA_PROTECT, WRITE_PROTECTED, 0x2);
+		mk_sense_buffer(scp, DATA_PROTECT, LU_SOFTWARE_WRITE_PROTECTED);
 		return check_condition_result;
 	}
 	if (sdebug_dev_is_zoned(devip))
@@ -4442,17 +4524,17 @@ static int resp_read_tape(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	     i++, pos++, blp++) {
 		devip->tape_location[partition] = pos + 1;
 		if (IS_TAPE_BLOCK_FM(blp->fl_size)) {
-			mk_sense_info_tape(scp, NO_SENSE, NO_ADDITIONAL_SENSE,
-					FILEMARK_DETECTED_ASCQ, fixed ? num - i : size,
-					SENSE_FLAG_FILEMARK);
+			mk_sense_info_tape(scp, NO_SENSE, FILEMARK_DETECTED,
+					   fixed ? num - i : size,
+					   SENSE_FLAG_FILEMARK);
 			scsi_set_resid(scp, (num - i) * size);
 			return check_condition_result;
 		}
 		/* Assume no REW */
 		if (IS_TAPE_BLOCK_EOD(blp->fl_size)) {
-			mk_sense_info_tape(scp, BLANK_CHECK, NO_ADDITIONAL_SENSE,
-					EOD_DETECTED_ASCQ, fixed ? num - i : size,
-					0);
+			mk_sense_info_tape(scp, BLANK_CHECK,
+					   END_OF_DATA_DETECTED,
+					   fixed ? num - i : size, 0);
 			devip->tape_location[partition] = pos;
 			scsi_set_resid(scp, (num - i) * size);
 			return check_condition_result;
@@ -4464,9 +4546,9 @@ static int resp_read_tape(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		if (fixed) {
 			if (blp->fl_size != devip->tape_blksize) {
 				scsi_set_resid(scp, (num - i) * size);
-				mk_sense_info_tape(scp, NO_SENSE, NO_ADDITIONAL_SENSE,
-						0, num - i,
-						SENSE_FLAG_ILI);
+				mk_sense_info_tape(scp, NO_SENSE,
+						NO_ADDITIONAL_SENSE_INFORMATION,
+						num - i, SENSE_FLAG_ILI);
 				return check_condition_result;
 			}
 		} else {
@@ -4474,18 +4556,19 @@ static int resp_read_tape(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 				if (blp->fl_size < size)
 					scsi_set_resid(scp, size - blp->fl_size);
 				if (!sili) {
-					mk_sense_info_tape(scp, NO_SENSE, NO_ADDITIONAL_SENSE,
-							0, size - blp->fl_size,
-							SENSE_FLAG_ILI);
+					mk_sense_info_tape(scp, NO_SENSE,
+						NO_ADDITIONAL_SENSE_INFORMATION,
+						size - blp->fl_size,
+						SENSE_FLAG_ILI);
 					return check_condition_result;
 				}
 			}
 		}
 	}
 	if (pos >= devip->tape_eop[partition]) {
-		mk_sense_info_tape(scp, NO_SENSE, NO_ADDITIONAL_SENSE,
-				EOP_EOM_DETECTED_ASCQ, fixed ? num - i : size,
-				SENSE_FLAG_EOM);
+		mk_sense_info_tape(scp, NO_SENSE,
+				   END_OF_PARTITION_MEDIUM_DETECTED,
+				   fixed ? num - i : size, SENSE_FLAG_EOM);
 		devip->tape_location[partition] = pos - 1;
 		return check_condition_result;
 	}
@@ -4572,8 +4655,7 @@ static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	if (unlikely((SDEBUG_OPT_MEDIUM_ERR & sdebug_opts) &&
 		     (lba <= (sdebug_medium_error_start + sdebug_medium_error_count - 1)) &&
 		     ((lba + num) > sdebug_medium_error_start))) {
-		/* claim unrecoverable read error */
-		mk_sense_buffer(scp, MEDIUM_ERROR, UNRECOVERED_READ_ERR, 0);
+		mk_sense_buffer(scp, MEDIUM_ERROR, UNRECOVERED_READ_ERROR);
 		/* set info field and valid bit for fixed descriptor */
 		if (0x70 == (scp->sense_buffer[0] & 0x7f)) {
 			scp->sense_buffer[0] |= 0x80;	/* Valid bit */
@@ -4597,22 +4679,26 @@ static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		case 1: /* Guard tag error */
 			if (cmd[1] >> 5 != 3) { /* RDPROTECT != 3 */
 				sdeb_meta_read_unlock(sip);
-				mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+				mk_sense_buffer(scp, ABORTED_COMMAND,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 				return check_condition_result;
 			} else if (scp->prot_flags & SCSI_PROT_GUARD_CHECK) {
 				sdeb_meta_read_unlock(sip);
-				mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+				mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 				return illegal_condition_result;
 			}
 			break;
 		case 3: /* Reference tag error */
 			if (cmd[1] >> 5 != 3) { /* RDPROTECT != 3 */
 				sdeb_meta_read_unlock(sip);
-				mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 3);
+				mk_sense_buffer(scp, ABORTED_COMMAND,
+				      LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 				return check_condition_result;
 			} else if (scp->prot_flags & SCSI_PROT_REF_CHECK) {
 				sdeb_meta_read_unlock(sip);
-				mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 3);
+				mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				      LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 				return illegal_condition_result;
 			}
 			break;
@@ -4630,16 +4716,18 @@ static int resp_read_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	if (unlikely((sdebug_opts & SDEBUG_OPT_RECOV_DIF_DIX) &&
 		     atomic_read(&sdeb_inject_pending))) {
 		if (sdebug_opts & SDEBUG_OPT_RECOVERED_ERR) {
-			mk_sense_buffer(scp, RECOVERED_ERROR, THRESHOLD_EXCEEDED, 0);
+			mk_sense_buffer(scp, RECOVERED_ERROR,
+					FAILURE_PREDICTION_THRESHOLD_EXCEEDED);
 			atomic_set(&sdeb_inject_pending, 0);
 			return check_condition_result;
 		} else if (sdebug_opts & SDEBUG_OPT_DIF_ERR) {
-			/* Logical block guard check failed */
-			mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+			mk_sense_buffer(scp, ABORTED_COMMAND,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 			atomic_set(&sdeb_inject_pending, 0);
 			return illegal_condition_result;
 		} else if (SDEBUG_OPT_DIX_ERR & sdebug_opts) {
-			mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 			atomic_set(&sdeb_inject_pending, 0);
 			return illegal_condition_result;
 		}
@@ -4846,22 +4934,96 @@ static int resp_write_tape(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	blp->fl_size = TAPE_BLOCK_EOD_FLAG;
 	if (pos >= devip->tape_eop[partition] - 1) {
 		mk_sense_info_tape(scp, VOLUME_OVERFLOW,
-				NO_ADDITIONAL_SENSE, EOP_EOM_DETECTED_ASCQ,
-				fixed ? num - i : transfer,
-				SENSE_FLAG_EOM);
+				   END_OF_PARTITION_MEDIUM_DETECTED,
+				   fixed ? num - i : transfer, SENSE_FLAG_EOM);
 		return check_condition_result;
 	}
 	if (ew) { /* early warning */
 		mk_sense_info_tape(scp, NO_SENSE,
-				NO_ADDITIONAL_SENSE, EOP_EOM_DETECTED_ASCQ,
-				fixed ? num - i : transfer,
-				SENSE_FLAG_EOM);
+				   END_OF_PARTITION_MEDIUM_DETECTED,
+				   fixed ? num - i : transfer, SENSE_FLAG_EOM);
 		return check_condition_result;
 	}
 
 	return 0;
 }
 
+static int corrupt_lbas(struct sdebug_dev_info *devip, u64 lba, u32 num,
+		u32 nr_bit_errors, s32 reftag_adjust)
+{
+	struct sdeb_store_info *sip = devip2sip(devip, false);
+	bool meta_data_locked = false;
+	u32 block, num_mapped, b, i;
+	int error = 0;
+
+	if (sdebug_dev_is_zoned(devip) ||
+	    sdebug_dix ||
+	    scsi_debug_lbp())  {
+		sdeb_meta_write_lock(sip);
+		meta_data_locked = true;
+	}
+
+	if (!sip) {
+		pr_err("can't corrupt with fake_rw\n");
+		error = -EINVAL;
+		goto out_unlock;
+	}
+
+	if (num > sdebug_capacity || lba > sdebug_capacity - num) {
+		pr_err("logical blocks out of bounds: %llu:%u", lba, num);
+		error = -EINVAL;
+		goto out_unlock;
+	}
+
+	if (scsi_debug_lbp() &&
+	    (!map_state(sip, lba, &num_mapped) || num > num_mapped)) {
+		pr_err("can't modify unmapped logical blocks: %llu:%u",
+			lba, num);
+		error = -EINVAL;
+		goto out_unlock;
+	}
+
+	/*
+	 * For each possible bit error, flip the same bit in each corrupted
+	 * block to simulate repeated patterns like errors in cables.
+	 */
+	sdeb_data_lock(sip, false);
+	for (b = 0; b < nr_bit_errors; b++) {
+		unsigned int bit;
+
+		bit = get_random_u32_below(sdebug_sector_size * BITS_PER_BYTE);
+		pr_info("flipping bit %u in LBAs %llu:%llu\n",
+				bit, lba, lba + num - 1);
+
+		div_u64_rem(lba, sdebug_store_sectors, &block);
+		for (i = 0; i < num; i++) {
+			u8 *p = sip->storep + (block * sdebug_sector_size);
+
+			sdeb_data_sector_lock(sip, true);
+			p[bit / BITS_PER_BYTE] ^= (1 << (bit % BITS_PER_BYTE));
+			sdeb_data_sector_unlock(sip, true);
+			if (++block >= sdebug_store_sectors)
+				block = 0;
+		}
+	}
+
+	if (reftag_adjust) {
+		pr_info("adjusting reftag by %d sectors for LBAs %llu:%llu\n",
+			reftag_adjust, lba, lba + num - 1);
+		for (i = 0; i < num; i++) {
+			struct t10_pi_tuple *sdt = dif_store(sip, lba + i);
+
+			be32_add_cpu(&sdt->ref_tag, reftag_adjust);
+		}
+	}
+	sdeb_data_unlock(sip, false);
+
+out_unlock:
+	if (meta_data_locked)
+		sdeb_meta_write_unlock(sip);
+	return error;
+}
+
 static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 {
 	bool check_prot;
@@ -4877,8 +5039,7 @@ static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	if (unlikely(sdebug_opts & SDEBUG_OPT_UNALIGNED_WRITE &&
 		     atomic_read(&sdeb_inject_pending))) {
 		atomic_set(&sdeb_inject_pending, 0);
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE,
-				UNALIGNED_WRITE_ASCQ);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, UNALIGNED_WRITE_COMMAND);
 		return check_condition_result;
 	}
 
@@ -4959,22 +5120,26 @@ static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		case 1: /* Guard tag error */
 			if (scp->prot_flags & SCSI_PROT_GUARD_CHECK) {
 				sdeb_meta_write_unlock(sip);
-				mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+				mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 				return illegal_condition_result;
 			} else if (scp->cmnd[1] >> 5 != 3) { /* WRPROTECT != 3 */
 				sdeb_meta_write_unlock(sip);
-				mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+				mk_sense_buffer(scp, ABORTED_COMMAND,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 				return check_condition_result;
 			}
 			break;
 		case 3: /* Reference tag error */
 			if (scp->prot_flags & SCSI_PROT_REF_CHECK) {
 				sdeb_meta_write_unlock(sip);
-				mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 3);
+				mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 				return illegal_condition_result;
 			} else if (scp->cmnd[1] >> 5 != 3) { /* WRPROTECT != 3 */
 				sdeb_meta_write_unlock(sip);
-				mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 3);
+				mk_sense_buffer(scp, ABORTED_COMMAND,
+					LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED);
 				return check_condition_result;
 			}
 			break;
@@ -5002,16 +5167,18 @@ static int resp_write_dt0(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	if (unlikely((sdebug_opts & SDEBUG_OPT_RECOV_DIF_DIX) &&
 		     atomic_read(&sdeb_inject_pending))) {
 		if (sdebug_opts & SDEBUG_OPT_RECOVERED_ERR) {
-			mk_sense_buffer(scp, RECOVERED_ERROR, THRESHOLD_EXCEEDED, 0);
+			mk_sense_buffer(scp, RECOVERED_ERROR,
+					FAILURE_PREDICTION_THRESHOLD_EXCEEDED);
 			atomic_set(&sdeb_inject_pending, 0);
 			return check_condition_result;
 		} else if (sdebug_opts & SDEBUG_OPT_DIF_ERR) {
-			/* Logical block guard check failed */
-			mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+			mk_sense_buffer(scp, ABORTED_COMMAND,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 			atomic_set(&sdeb_inject_pending, 0);
 			return illegal_condition_result;
 		} else if (sdebug_opts & SDEBUG_OPT_DIX_ERR) {
-			mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+			mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 			atomic_set(&sdeb_inject_pending, 0);
 			return illegal_condition_result;
 		}
@@ -5074,7 +5241,7 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 		if (sdebug_verbose)
 			sdev_printk(KERN_INFO, scp->device,
 				"%s: LB Data Offset field bad\n", my_name);
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		return illegal_condition_result;
 	}
 	lbdof_blen = lbdof * lb_size;
@@ -5082,7 +5249,7 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 		if (sdebug_verbose)
 			sdev_printk(KERN_INFO, scp->device,
 				"%s: LBA range descriptors don't fit\n", my_name);
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		return illegal_condition_result;
 	}
 	lrdp = kzalloc(lbdof_blen, GFP_ATOMIC | __GFP_NOWARN);
@@ -5123,8 +5290,7 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 				sdev_printk(KERN_INFO, scp->device,
 				    "%s: sum of blocks > data provided\n",
 				    my_name);
-			mk_sense_buffer(scp, ILLEGAL_REQUEST, WRITE_ERROR_ASC,
-					0);
+			mk_sense_buffer(scp, ILLEGAL_REQUEST, SSC_WRITE_ERROR);
 			ret = illegal_condition_result;
 			goto err_out_unlock;
 		}
@@ -5135,8 +5301,9 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 							 ei_lba);
 
 			if (prot_ret) {
-				mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10,
-						prot_ret);
+				mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					scsi_sense_code(ASC_ID_CRC_OR_ECC_ERROR,
+							prot_ret));
 				ret = illegal_condition_result;
 				goto err_out_unlock;
 			}
@@ -5163,18 +5330,20 @@ static int resp_write_scat(struct scsi_cmnd *scp,
 		if (unlikely((sdebug_opts & SDEBUG_OPT_RECOV_DIF_DIX) &&
 			     atomic_read(&sdeb_inject_pending))) {
 			if (sdebug_opts & SDEBUG_OPT_RECOVERED_ERR) {
-				mk_sense_buffer(scp, RECOVERED_ERROR, THRESHOLD_EXCEEDED, 0);
+				mk_sense_buffer(scp, RECOVERED_ERROR,
+					FAILURE_PREDICTION_THRESHOLD_EXCEEDED);
 				atomic_set(&sdeb_inject_pending, 0);
 				ret = check_condition_result;
 				goto err_out_unlock;
 			} else if (sdebug_opts & SDEBUG_OPT_DIF_ERR) {
-				/* Logical block guard check failed */
-				mk_sense_buffer(scp, ABORTED_COMMAND, 0x10, 1);
+				mk_sense_buffer(scp, ABORTED_COMMAND,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 				atomic_set(&sdeb_inject_pending, 0);
 				ret = illegal_condition_result;
 				goto err_out_unlock;
 			} else if (sdebug_opts & SDEBUG_OPT_DIX_ERR) {
-				mk_sense_buffer(scp, ILLEGAL_REQUEST, 0x10, 1);
+				mk_sense_buffer(scp, ILLEGAL_REQUEST,
+					LOGICAL_BLOCK_GUARD_CHECK_FAILED);
 				atomic_set(&sdeb_inject_pending, 0);
 				ret = illegal_condition_result;
 				goto err_out_unlock;
@@ -5393,8 +5562,7 @@ static int resp_comp_write(struct scsi_cmnd *scp,
 	dnum = 2 * num;
 	arr = kcalloc(lb_size, dnum, GFP_ATOMIC);
 	if (NULL == arr) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
-				INSUFF_RES_ASCQ);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFFICIENT_RESOURCES);
 		return check_condition_result;
 	}
 
@@ -5410,7 +5578,8 @@ static int resp_comp_write(struct scsi_cmnd *scp,
 	sdeb_data_write_lock(sip);
 	sdeb_meta_write_lock(sip);
 	if (!comp_write_worker(sip, lba, num, arr, false)) {
-		mk_sense_buffer(scp, MISCOMPARE, MISCOMPARE_VERIFY_ASC, 0);
+		mk_sense_buffer(scp, MISCOMPARE,
+				MISCOMPARE_DURING_VERIFY_OPERATION);
 		retval = check_condition_result;
 		goto cleanup_unlock;
 	}
@@ -5453,8 +5622,7 @@ static int resp_unmap(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 
 	buf = kzalloc(scsi_bufflen(scp), GFP_ATOMIC);
 	if (!buf) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
-				INSUFF_RES_ASCQ);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFFICIENT_RESOURCES);
 		return check_condition_result;
 	}
 
@@ -5592,7 +5760,8 @@ static int resp_sync_cache(struct scsi_cmnd *scp,
 		num_blocks = get_unaligned_be32(cmd + 10);
 	}
 	if (lba + num_blocks > sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		return check_condition_result;
 	}
 	if (!write_since_sync || (cmd[1] & 0x2))
@@ -5630,7 +5799,8 @@ static int resp_pre_fetch(struct scsi_cmnd *scp,
 		nblks = get_unaligned_be32(cmd + 10);
 	}
 	if (lba + nblks > sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		return check_condition_result;
 	}
 	if (!fsp)
@@ -5801,8 +5971,7 @@ static int resp_verify(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 
 	arr = kcalloc(lb_size, vnum, GFP_ATOMIC | __GFP_NOWARN);
 	if (!arr) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
-				INSUFF_RES_ASCQ);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFFICIENT_RESOURCES);
 		return check_condition_result;
 	}
 	/* Not changing store, so only need read access */
@@ -5823,7 +5992,8 @@ static int resp_verify(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	}
 	ret = 0;
 	if (!comp_write_worker(sip, lba, vnum, arr, true)) {
-		mk_sense_buffer(scp, MISCOMPARE, MISCOMPARE_VERIFY_ASC, 0);
+		mk_sense_buffer(scp, MISCOMPARE,
+				MISCOMPARE_DURING_VERIFY_OPERATION);
 		ret = check_condition_result;
 		goto cleanup;
 	}
@@ -5862,7 +6032,8 @@ static int resp_report_zones(struct scsi_cmnd *scp,
 	partial = cmd[14] & 0x80;
 
 	if (zs_lba >= sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		return check_condition_result;
 	}
 
@@ -5873,8 +6044,8 @@ static int resp_report_zones(struct scsi_cmnd *scp,
 
 	arr = kzalloc(arr_len, GFP_ATOMIC | __GFP_NOWARN);
 	if (!arr) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC,
-				INSUFF_RES_ASCQ);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				INSUFFICIENT_RESOURCES);
 		return check_condition_result;
 	}
 
@@ -5939,8 +6110,7 @@ static int resp_report_zones(struct scsi_cmnd *scp,
 				continue;
 			break;
 		default:
-			mk_sense_buffer(scp, ILLEGAL_REQUEST,
-					INVALID_FIELD_IN_CDB, 0);
+			mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 			ret = check_condition_result;
 			goto fini;
 		}
@@ -6005,13 +6175,13 @@ static int resp_atomic_write(struct scsi_cmnd *scp,
 	if (sdebug_atomic_wr_align &&
 	    do_div(lba_tmp, sdebug_atomic_wr_align)) {
 		/* Does not meet alignment requirement */
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		return check_condition_result;
 	}
 
 	if (sdebug_atomic_wr_gran && len % sdebug_atomic_wr_gran) {
 		/* Does not meet alignment requirement */
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		return check_condition_result;
 	}
 
@@ -6072,8 +6242,8 @@ static int resp_open_zone(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		/* Check if all closed zones can be open */
 		if (devip->max_open &&
 		    devip->nr_exp_open + devip->nr_closed > devip->max_open) {
-			mk_sense_buffer(scp, DATA_PROTECT, INSUFF_RES_ASC,
-					INSUFF_ZONE_ASCQ);
+			mk_sense_buffer(scp, DATA_PROTECT,
+					INSUFFICIENT_ZONE_RESOURCES);
 			res = check_condition_result;
 			goto fini;
 		}
@@ -6085,19 +6255,20 @@ static int resp_open_zone(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 	/* Open the specified zone */
 	z_id = get_unaligned_be64(cmd + 2);
 	if (z_id >= sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		res = check_condition_result;
 		goto fini;
 	}
 
 	zsp = zbc_zone(devip, z_id);
 	if (z_id != zsp->z_start) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
 	if (zbc_zone_is_conv(zsp)) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
@@ -6107,8 +6278,7 @@ static int resp_open_zone(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 		goto fini;
 
 	if (devip->max_open && devip->nr_exp_open >= devip->max_open) {
-		mk_sense_buffer(scp, DATA_PROTECT, INSUFF_RES_ASC,
-				INSUFF_ZONE_ASCQ);
+		mk_sense_buffer(scp, DATA_PROTECT, INSUFFICIENT_ZONE_RESOURCES);
 		res = check_condition_result;
 		goto fini;
 	}
@@ -6152,19 +6322,20 @@ static int resp_close_zone(struct scsi_cmnd *scp,
 	/* Close specified zone */
 	z_id = get_unaligned_be64(cmd + 2);
 	if (z_id >= sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		res = check_condition_result;
 		goto fini;
 	}
 
 	zsp = zbc_zone(devip, z_id);
 	if (z_id != zsp->z_start) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
 	if (zbc_zone_is_conv(zsp)) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
@@ -6224,19 +6395,20 @@ static int resp_finish_zone(struct scsi_cmnd *scp,
 	/* Finish the specified zone */
 	z_id = get_unaligned_be64(cmd + 2);
 	if (z_id >= sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		res = check_condition_result;
 		goto fini;
 	}
 
 	zsp = zbc_zone(devip, z_id);
 	if (z_id != zsp->z_start) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
 	if (zbc_zone_is_conv(zsp)) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
@@ -6303,19 +6475,20 @@ static int resp_rwp_zone(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 
 	z_id = get_unaligned_be64(cmd + 2);
 	if (z_id >= sdebug_capacity) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, LBA_OUT_OF_RANGE, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST,
+				LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE);
 		res = check_condition_result;
 		goto fini;
 	}
 
 	zsp = zbc_zone(devip, z_id);
 	if (z_id != zsp->z_start) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
 	if (zbc_zone_is_conv(zsp)) {
-		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB, 0);
+		mk_sense_buffer(scp, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 		res = check_condition_result;
 		goto fini;
 	}
@@ -6628,6 +6801,8 @@ static int scsi_debug_sdev_configure(struct scsi_device *sdp,
 	if (IS_ERR_OR_NULL(dentry))
 		pr_info("failed to create error file for device %s\n",
 			dev_name(&sdp->sdev_gendev));
+	debugfs_create_file("corrupt", 0600, devip->debugfs_entry, sdp,
+				&sdebug_corrupt_fops);
 
 	return 0;
 }
@@ -7186,7 +7361,7 @@ static int schedule_resp(struct scsi_cmnd *cmnd, struct sdebug_dev_info *devip,
 		cmnd->result = scsi_result;
 	if (cmnd->result == 0 && unlikely(sdebug_opts & SDEBUG_OPT_TRANSPORT_ERR)) {
 		if (atomic_read(&sdeb_inject_pending)) {
-			mk_sense_buffer(cmnd, ABORTED_COMMAND, TRANSPORT_PROBLEM, ACK_NAK_TO);
+			mk_sense_buffer(cmnd, ABORTED_COMMAND, ACK_NAK_TIMEOUT);
 			atomic_set(&sdeb_inject_pending, 0);
 			cmnd->result = check_condition_result;
 		}
@@ -7373,8 +7548,8 @@ MODULE_VERSION(SDEBUG_VERSION);
 MODULE_PARM_DESC(add_host, "add n hosts, in sysfs if negative remove host(s) (def=1)");
 MODULE_PARM_DESC(ato, "application tag ownership: 0=disk 1=host (def=1)");
 MODULE_PARM_DESC(cdb_len, "suggest CDB lengths to drivers (def=10)");
-MODULE_PARM_DESC(clustering, "when set enables larger transfers (def=0)");
-MODULE_PARM_DESC(delay, "response delay (def=1 jiffy); 0:imm, -1,-2:tiny");
+MODULE_PARM_DESC(clustering, "when set enables larger transfers (def=1)");
+MODULE_PARM_DESC(delay, "response delay (def=0 jiffy); 0:imm, -1,-2:tiny");
 MODULE_PARM_DESC(dev_size_mb, "size in MiB of ram shared by devs(def=8)");
 MODULE_PARM_DESC(dif, "data integrity field type: 0-3 (def=0)");
 MODULE_PARM_DESC(dix, "data integrity extensions mask (def=0)");
@@ -8979,7 +9154,8 @@ static int resp_not_ready(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 				return 0;
 			}
 		}
-		mk_sense_buffer(scp, NOT_READY, LOGICAL_UNIT_NOT_READY, 0x1);
+		mk_sense_buffer(scp, NOT_READY,
+				LU_IS_IN_PROCESS_OF_BECOMING_READY);
 		if (sdebug_verbose)
 			sdev_printk(KERN_INFO, sdp,
 				    "%s: Not ready: in process of becoming ready\n", my_name);
@@ -8997,7 +9173,8 @@ static int resp_not_ready(struct scsi_cmnd *scp, struct sdebug_dev_info *devip)
 			return check_condition_result;
 		}
 	}
-	mk_sense_buffer(scp, NOT_READY, LOGICAL_UNIT_NOT_READY, 0x2);
+	mk_sense_buffer(scp, NOT_READY,
+			LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED);
 	if (sdebug_verbose)
 		sdev_printk(KERN_INFO, sdp, "%s: Not ready: initializing command required\n",
 			    my_name);
@@ -9194,7 +9371,8 @@ static int sdebug_fail_cmd(struct scsi_cmnd *cmnd, int *retval,
 out_handle:
 	if (err->cnt < 0)
 		err->cnt++;
-	mk_sense_buffer(cmnd, err->sense_key, err->asc, err->asq);
+	mk_sense_buffer(cmnd, err->sense_key,
+			scsi_sense_code(err->asc, err->asq));
 	result = err->status_byte | err->host_byte << 16 | err->driver_byte << 24;
 	*info = *err;
 	*retval = schedule_resp(cmnd, devip, result, NULL, 0, 0);
diff --git a/drivers/scsi/scsi_debugfs.c b/drivers/scsi/scsi_debugfs.c
index eb52e39f37c926..54d8824250b1a1 100644
--- a/drivers/scsi/scsi_debugfs.c
+++ b/drivers/scsi/scsi_debugfs.c
@@ -38,7 +38,7 @@ static const char *scsi_cmd_list_info(struct scsi_cmnd *cmd)
 	struct Scsi_Host *shost = cmd->device->host;
 	struct scsi_cmnd *cmd2;
 
-	guard(spinlock_irq)(shost->host_lock);
+	guard(spinlock_irq)(&shost->host_lock);
 
 	list_for_each_entry(cmd2, &shost->eh_abort_list, eh_entry)
 		if (cmd == cmd2)
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index 15ffbe93ac7227..cfc259fd01415c 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -161,7 +161,7 @@ static struct {
 	{"DGC", "DISK", NULL, BLIST_SPARSELUN},	/* EMC CLARiiON, no storage on LUN 0 */
 	{"EMC",  "Invista", "*", BLIST_SPARSELUN | BLIST_LARGELUN},
 	{"EMC", "SYMMETRIX", NULL, BLIST_SPARSELUN | BLIST_LARGELUN |
-	 BLIST_REPORTLUN2 | BLIST_RETRY_ITF},
+	 BLIST_REPORTLUN2 | BLIST_RETRY_ITF | BLIST_SKIP_IO_HINTS},
 	{"EMULEX", "MD21/S2     ESDI", NULL, BLIST_SINGLELUN},
 	{"easyRAID", "16P", NULL, BLIST_NOREPORTLUN},
 	{"easyRAID", "X6P", NULL, BLIST_NOREPORTLUN},
@@ -691,7 +691,8 @@ static ssize_t proc_scsi_devinfo_write(struct file *file,
 
 	if (!buf || length>PAGE_SIZE)
 		return -EINVAL;
-	if (!(buffer = (char *) __get_free_page(GFP_KERNEL)))
+	buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buffer)
 		return -ENOMEM;
 	if (copy_from_user(buffer, buf, length)) {
 		err =-EFAULT;
@@ -708,7 +709,7 @@ static ssize_t proc_scsi_devinfo_write(struct file *file,
 	scsi_dev_info_list_add_str(buffer);
 
 out:
-	free_page((unsigned long)buffer);
+	kfree(buffer);
 	return err;
 }
 
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index b729407348f765..2e0b1909dc69a8 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -63,7 +63,7 @@ static enum scsi_disposition scsi_try_to_abort_cmd(const struct scsi_host_templa
 
 void scsi_eh_wakeup(struct Scsi_Host *shost, unsigned int busy)
 {
-	lockdep_assert_held(shost->host_lock);
+	lockdep_assert_held(&shost->host_lock);
 
 	if (busy == shost->host_failed) {
 		trace_scsi_eh_wakeup(shost);
@@ -88,9 +88,9 @@ void scsi_rcu_eh_wakeup(struct work_struct *work)
 
 	busy = scsi_host_busy(shost);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	scsi_eh_wakeup(shost, busy);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -103,7 +103,7 @@ void scsi_schedule_eh(struct Scsi_Host *shost)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	if (scsi_host_set_state(shost, SHOST_RECOVERY) == 0 ||
 	    scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY) == 0) {
@@ -111,7 +111,7 @@ void scsi_schedule_eh(struct Scsi_Host *shost)
 		queue_work(shost->tmf_work_q, &shost->eh_work);
 	}
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 EXPORT_SYMBOL_GPL(scsi_schedule_eh);
 
@@ -201,7 +201,7 @@ scmd_eh_abort_handler(struct work_struct *work)
 		goto out;
 	}
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_del_init(&scmd->eh_entry);
 
 	/*
@@ -213,7 +213,7 @@ scmd_eh_abort_handler(struct work_struct *work)
 		if (shost->eh_deadline != -1)
 			shost->last_reset = 0;
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (!scsi_noretry_cmd(scmd) &&
 	    scsi_cmd_retry_allowed(scmd) &&
@@ -231,9 +231,9 @@ scmd_eh_abort_handler(struct work_struct *work)
 	return;
 
 out:
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_del_init(&scmd->eh_entry);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	scsi_eh_scmd_add(scmd);
 }
@@ -267,12 +267,12 @@ scsi_abort_command(struct scsi_cmnd *scmd)
 		return FAILED;
 	}
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (shost->eh_deadline != -1 && !shost->last_reset)
 		shost->last_reset = jiffies;
 	BUG_ON(!list_empty(&scmd->eh_entry));
 	list_add_tail(&scmd->eh_entry, &shost->eh_abort_list);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	scmd->eh_eflags |= SCSI_EH_ABORT_SCHEDULED;
 	SCSI_LOG_ERROR_RECOVERY(3,
@@ -305,9 +305,9 @@ static void scsi_eh_inc_host_failed(struct rcu_head *head)
 	unsigned int busy;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	shost->host_failed++;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	/*
 	 * The counting of busy requests needs to occur after adding to
 	 * host_failed or after the lock acquire for adding to host_failed
@@ -315,9 +315,9 @@ static void scsi_eh_inc_host_failed(struct rcu_head *head)
 	 */
 	busy = scsi_host_busy(shost);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	scsi_eh_wakeup(shost, busy);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -333,7 +333,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)
 	WARN_ON_ONCE(!shost->ehandler);
 	WARN_ON_ONCE(!test_bit(SCMD_STATE_INFLIGHT, &scmd->state));
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (scsi_host_set_state(shost, SHOST_RECOVERY)) {
 		ret = scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY);
 		WARN_ON_ONCE(ret);
@@ -343,7 +343,7 @@ void scsi_eh_scmd_add(struct scsi_cmnd *scmd)
 
 	scsi_eh_reset(scmd);
 	list_add_tail(&scmd->eh_entry, &shost->eh_cmd_q);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	/*
 	 * Ensure that all tasks observe the host state change before the
 	 * host_failed change.
@@ -490,24 +490,27 @@ static void scsi_report_sense(struct scsi_device *sdev,
 	enum scsi_device_event evt_type = SDEV_EVT_MAXBITS;	/* i.e. none */
 
 	if (sshdr->sense_key == UNIT_ATTENTION) {
-		if (sshdr->asc == 0x3f && sshdr->ascq == 0x03) {
+		if (sshdr->sense_code == INQUIRY_DATA_HAS_CHANGED) {
 			evt_type = SDEV_EVT_INQUIRY_CHANGE_REPORTED;
 			sdev_printk(KERN_WARNING, sdev,
 				    "Inquiry data has changed");
-		} else if (sshdr->asc == 0x3f && sshdr->ascq == 0x0e) {
+		} else if (sshdr->sense_code == REPORTED_LUNS_DATA_HAS_CHANGED) {
 			evt_type = SDEV_EVT_LUN_CHANGE_REPORTED;
 			scsi_report_lun_change(sdev);
 			sdev_printk(KERN_WARNING, sdev,
 				    "LUN assignments on this target have "
 				    "changed. The Linux SCSI layer does not "
 				    "automatically remap LUN assignments.\n");
-		} else if (sshdr->asc == 0x3f)
+		} else if (scsi_sense_asc(sshdr) ==
+			   ASC_TARGET_OPERATING_CONDITIONS_HAVE_CHANGED) {
 			sdev_printk(KERN_WARNING, sdev,
 				    "Operating parameters on this target have "
 				    "changed. The Linux SCSI layer does not "
 				    "automatically adjust these parameters.\n");
+		}
 
-		if (sshdr->asc == 0x38 && sshdr->ascq == 0x07) {
+		if (sshdr->sense_code ==
+		    THIN_PROVISIONING_SOFT_THRESHOLD_REACHED) {
 			evt_type = SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED;
 			sdev_printk(KERN_WARNING, sdev,
 				    "Warning! Received an indication that the "
@@ -515,7 +518,8 @@ static void scsi_report_sense(struct scsi_device *sdev,
 				    "threshold.\n");
 		}
 
-		if (sshdr->asc == 0x29) {
+		if (scsi_sense_asc(sshdr) ==
+		    ASC_POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED) {
 			evt_type = SDEV_EVT_POWER_ON_RESET_OCCURRED;
 			/*
 			 * Do not print message if it is an expected side-effect
@@ -526,21 +530,22 @@ static void scsi_report_sense(struct scsi_device *sdev,
 					    "Power-on or device reset occurred\n");
 		}
 
-		if (sshdr->asc == 0x2a && sshdr->ascq == 0x01) {
+		if (sshdr->sense_code == MODE_PARAMETERS_CHANGED) {
 			evt_type = SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED;
 			sdev_printk(KERN_WARNING, sdev,
 				    "Mode parameters changed");
-		} else if (sshdr->asc == 0x2a && sshdr->ascq == 0x06) {
+		} else if (sshdr->sense_code == ASYMMETRIC_ACCESS_STATE_CHANGED) {
 			evt_type = SDEV_EVT_ALUA_STATE_CHANGE_REPORTED;
 			sdev_printk(KERN_WARNING, sdev,
 				    "Asymmetric access state changed");
-		} else if (sshdr->asc == 0x2a && sshdr->ascq == 0x09) {
+		} else if (sshdr->sense_code == CAPACITY_DATA_HAS_CHANGED) {
 			evt_type = SDEV_EVT_CAPACITY_CHANGE_REPORTED;
 			sdev_printk(KERN_WARNING, sdev,
 				    "Capacity data has changed");
-		} else if (sshdr->asc == 0x2a)
+		} else if (scsi_sense_asc(sshdr) == ASC_PARAMETERS_CHANGED) {
 			sdev_printk(KERN_WARNING, sdev,
 				    "Parameters changed");
+		}
 	}
 
 	if (evt_type != SDEV_EVT_MAXBITS) {
@@ -582,9 +587,11 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		 * that all ULDs interested in these can see that those have
 		 * happened, even if someone else gets the sense data.
 		 */
-		if (sshdr.asc == 0x28)
+		if (scsi_sense_asc(&sshdr) ==
+		    ASC_NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED)
 			atomic_inc(&sdev->ua_new_media_ctr);
-		else if (sshdr.asc == 0x29)
+		else if (scsi_sense_asc(&sshdr) ==
+			 ASC_POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED)
 			atomic_inc(&sdev->ua_por_ctr);
 	}
 
@@ -636,7 +643,8 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		return /* soft_error */ SUCCESS;
 
 	case ABORTED_COMMAND:
-		if (sshdr.asc == 0x10) /* DIF */
+		if (scsi_sense_asc(&sshdr) == ASC_ID_CRC_OR_ECC_ERROR)
+			/* DIF */
 			return SUCCESS;
 
 		/*
@@ -647,18 +655,20 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		 * COMMAND TIMEOUT DURING PROCESSING DUE TO ERROR RECOVERY
 		 * additional sense code qualifiers.
 		 */
-		if (sshdr.asc == 0x2e &&
-		    sshdr.ascq >= 0x01 && sshdr.ascq <= 0x03) {
+		if (sshdr.sense_code >= COMMAND_TIMEOUT_BEFORE_PROCESSING &&
+		    sshdr.sense_code <=
+		    COMMAND_TIMEOUT_DURING_PROCESSING_DUE_TO_ERROR_RECOVERY) {
 			set_scsi_ml_byte(scmd, SCSIML_STAT_DL_TIMEOUT);
 			req->cmd_flags |= REQ_FAILFAST_DEV;
 			req->rq_flags |= RQF_QUIET;
 			return SUCCESS;
 		}
 
-		if (sshdr.asc == 0x44 && sdev->sdev_bflags & BLIST_RETRY_ITF)
+		if ((sdev->sdev_bflags & BLIST_RETRY_ITF) &&
+		    scsi_sense_asc(&sshdr) == ASC_INTERNAL_TARGET_FAILURE)
 			return ADD_TO_MLQUEUE;
-		if (sshdr.asc == 0xc1 && sshdr.ascq == 0x01 &&
-		    sdev->sdev_bflags & BLIST_RETRY_ASC_C1)
+		if ((sdev->sdev_bflags & BLIST_RETRY_ASC_C1) &&
+		    sshdr.sense_code == scsi_sense_code(0xc1, 0x01))
 			return ADD_TO_MLQUEUE;
 
 		return NEEDS_RETRY;
@@ -672,12 +682,13 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		 */
 		if (scmd->device->expecting_cc_ua) {
 			/*
-			 * Because some device does not queue unit
-			 * attentions correctly, we carefully check
-			 * additional sense code and qualifier so as
-			 * not to squash media change unit attention.
+			 * Because some devices do not queue unit attentions
+			 * correctly, we carefully check additional sense code
+			 * and qualifier so as not to squash media change unit
+			 * attention.
 			 */
-			if (sshdr.asc != 0x28 || sshdr.ascq != 0x00) {
+			if (sshdr.sense_code !=
+			    NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED) {
 				scmd->device->expecting_cc_ua = 0;
 				return NEEDS_RETRY;
 			}
@@ -688,21 +699,23 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		 * REPORTED LUNS DATA HAS CHANGED.
 		 */
 		if (scmd->device->sdev_target->expecting_lun_change &&
-		    sshdr.asc == 0x3f && sshdr.ascq == 0x0e)
+		    sshdr.sense_code == REPORTED_LUNS_DATA_HAS_CHANGED)
 			return NEEDS_RETRY;
 		/*
 		 * if the device is in the process of becoming ready, we
 		 * should retry.
 		 */
-		if ((sshdr.asc == 0x04) &&
-		    (sshdr.ascq == 0x01 || sshdr.ascq == 0x0a))
+		if (sshdr.sense_code == LU_IS_IN_PROCESS_OF_BECOMING_READY ||
+		    sshdr.sense_code ==
+		    LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION)
 			return NEEDS_RETRY;
 		/*
 		 * if the device is not started, we need to wake
 		 * the error handler to start the motor
 		 */
 		if (scmd->device->allow_restart &&
-		    (sshdr.asc == 0x04) && (sshdr.ascq == 0x02))
+		    sshdr.sense_code ==
+		    LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED)
 			return FAILED;
 		/*
 		 * Pass the UA upwards for a determination in the completion
@@ -712,7 +725,7 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 
 		/* these are not supported */
 	case DATA_PROTECT:
-		if (sshdr.asc == 0x27 && sshdr.ascq == 0x07) {
+		if (sshdr.sense_code == SPACE_ALLOCATION_FAILED_WRITE_PROTECT) {
 			/* Thin provisioning hard threshold reached */
 			set_scsi_ml_byte(scmd, SCSIML_STAT_NOSPC);
 			return SUCCESS;
@@ -726,11 +739,14 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		return SUCCESS;
 
 	case MEDIUM_ERROR:
-		if (sshdr.asc == 0x11 || /* UNRECOVERED READ ERR */
-		    sshdr.asc == 0x13 || /* AMNF DATA FIELD */
-		    sshdr.asc == 0x14) { /* RECORD NOT FOUND */
+		switch (scsi_sense_asc(&sshdr)) {
+		case ASC_UNRECOVERED_READ_ERROR:
+		case ASC_ADDRESS_MARK_NOT_FOUND_FOR_DATA_FIELD:
+		case ASC_RECORDED_ENTITY_NOT_FOUND:
 			set_scsi_ml_byte(scmd, SCSIML_STAT_MED_ERROR);
 			return SUCCESS;
+		default:
+			break;
 		}
 		return NEEDS_RETRY;
 
@@ -742,13 +758,17 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		fallthrough;
 
 	case ILLEGAL_REQUEST:
-		if (sshdr.asc == 0x20 || /* Invalid command operation code */
-		    sshdr.asc == 0x21 || /* Logical block address out of range */
-		    sshdr.asc == 0x22 || /* Invalid function */
-		    sshdr.asc == 0x24 || /* Invalid field in cdb */
-		    sshdr.asc == 0x26 || /* Parameter value invalid */
-		    sshdr.asc == 0x27) { /* Write protected */
+		switch (scsi_sense_asc(&sshdr)) {
+		case ASC_INVALID_COMMAND_OP_CODE:
+		case ASC_LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE:
+		case ASC_ILLEGAL_FUNCTION:
+		case ASC_INVALID_FIELD_IN_CDB:
+		case ASC_INVALID_FIELD_IN_PARAMETER_LIST:
+		case ASC_WRITE_PROTECTED:
 			set_scsi_ml_byte(scmd, SCSIML_STAT_TGT_FAILURE);
+			break;
+		default:
+			break;
 		}
 		return SUCCESS;
 
@@ -760,7 +780,7 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd)
 		 * that the command was in fact aborted because it exceeded its
 		 * duration limit. Never retry these commands.
 		 */
-		if (sshdr.asc == 0x55 && sshdr.ascq == 0x0a) {
+		if (sshdr.sense_code == DATA_CURRENTLY_UNAVAILABLE) {
 			set_scsi_ml_byte(scmd, SCSIML_STAT_DL_TIMEOUT);
 			req->cmd_flags |= REQ_FAILFAST_DEV;
 			req->rq_flags |= RQF_QUIET;
@@ -940,9 +960,9 @@ static enum scsi_disposition scsi_try_host_reset(struct scsi_cmnd *scmd)
 	if (rtn == SUCCESS) {
 		if (!hostt->skip_settle_delay)
 			ssleep(HOST_RESET_SETTLE_TIME);
-		spin_lock_irqsave(host->host_lock, flags);
+		spin_lock_irqsave(&host->host_lock, flags);
 		scsi_report_bus_reset(host, scmd_channel(scmd));
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 	}
 
 	return rtn;
@@ -970,9 +990,9 @@ static enum scsi_disposition scsi_try_bus_reset(struct scsi_cmnd *scmd)
 	if (rtn == SUCCESS) {
 		if (!hostt->skip_settle_delay)
 			ssleep(BUS_RESET_SETTLE_TIME);
-		spin_lock_irqsave(host->host_lock, flags);
+		spin_lock_irqsave(&host->host_lock, flags);
 		scsi_report_bus_reset(host, scmd_channel(scmd));
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 	}
 
 	return rtn;
@@ -1006,10 +1026,10 @@ static enum scsi_disposition scsi_try_target_reset(struct scsi_cmnd *scmd)
 
 	rtn = hostt->eh_target_reset_handler(scmd);
 	if (rtn == SUCCESS) {
-		spin_lock_irqsave(host->host_lock, flags);
+		spin_lock_irqsave(&host->host_lock, flags);
 		__starget_for_each_device(scsi_target(scmd->device), NULL,
 					  __scsi_report_device_reset);
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 	}
 
 	return rtn;
@@ -2215,11 +2235,11 @@ static void scsi_restart_operations(struct Scsi_Host *shost)
 	SCSI_LOG_ERROR_RECOVERY(3,
 		shost_printk(KERN_INFO, shost, "waking up host to restart\n"));
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (scsi_host_set_state(shost, SHOST_RUNNING))
 		if (scsi_host_set_state(shost, SHOST_CANCEL))
 			BUG_ON(scsi_host_set_state(shost, SHOST_DEL));
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	wake_up(&shost->host_wait);
 
@@ -2239,11 +2259,11 @@ static void scsi_restart_operations(struct Scsi_Host *shost)
 	 * recovery or scsi_device_unbusy() will wake us again when these
 	 * pending commands complete.
 	 */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (shost->host_eh_scheduled)
 		if (scsi_host_set_state(shost, SHOST_RECOVERY))
 			WARN_ON(scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY));
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -2335,19 +2355,19 @@ static void scsi_unjam_host(struct Scsi_Host *shost)
 	LIST_HEAD(eh_work_q);
 	LIST_HEAD(eh_done_q);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_splice_init(&shost->eh_cmd_q, &eh_work_q);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	SCSI_LOG_ERROR_RECOVERY(1, scsi_eh_prt_fail_stats(shost, &eh_work_q));
 
 	if (!scsi_eh_get_sense(&eh_work_q, &eh_done_q))
 		scsi_eh_ready_devs(shost, &eh_work_q, &eh_done_q);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (shost->eh_deadline != -1)
 		shost->last_reset = 0;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	scsi_eh_flush_done_q(&eh_done_q);
 }
 
@@ -2551,9 +2571,9 @@ scsi_ioctl_reset(struct scsi_device *dev, int __user *arg)
 
 	scmd->sc_data_direction		= DMA_BIDIRECTIONAL;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	shost->tmf_in_progress = 1;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	switch (val & ~SG_SCSI_RESET_NO_ESCALATE) {
 	case SG_SCSI_RESET_NOTHING:
@@ -2586,9 +2606,9 @@ scsi_ioctl_reset(struct scsi_device *dev, int __user *arg)
 
 	error = (rtn == SUCCESS) ? 0 : -EIO;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	shost->tmf_in_progress = 0;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	/*
 	 * be sure to wake up anyone who was sleeping or had their queue
diff --git a/drivers/scsi/scsi_ioctl.c b/drivers/scsi/scsi_ioctl.c
index c14f81403a09e6..122403ec3ab8d9 100644
--- a/drivers/scsi/scsi_ioctl.c
+++ b/drivers/scsi/scsi_ioctl.c
@@ -96,7 +96,8 @@ static int ioctl_internal_command(struct scsi_device *sdev, char *cmd,
 					    "ioctl_internal_command: "
 					    "ILLEGAL REQUEST "
 					    "asc=0x%x ascq=0x%x\n",
-					    sshdr.asc, sshdr.ascq);
+					    scsi_sense_asc(&sshdr),
+					    scsi_sense_ascq(&sshdr));
 			break;
 		case NOT_READY:	/* This happens if there is no disc in drive */
 			if (sdev->removable)
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index af27fd3df8d46b..db64470f522949 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -243,23 +243,23 @@ static int scsi_check_passthrough(struct scsi_cmnd *scmd,
 			continue;
 
 		if (status_byte(failure->result) != SAM_STAT_CHECK_CONDITION ||
-		    failure->sense == SCMD_FAILURE_SENSE_ANY)
+		    failure->sense_key == SCMD_FAILURE_SENSE_KEY_ANY)
 			goto maybe_retry;
 
 		if (!scsi_command_normalize_sense(scmd, &sshdr))
 			return 0;
 
-		if (failure->sense != sshdr.sense_key)
+		if (failure->sense_key != sshdr.sense_key)
 			continue;
 
-		if (failure->asc == SCMD_FAILURE_ASC_ANY)
+		if (scsi_failure_asc(failure) == SCMD_FAILURE_ASC_ANY)
 			goto maybe_retry;
 
-		if (failure->asc != sshdr.asc)
+		if (scsi_failure_asc(failure) != scsi_sense_asc(&sshdr))
 			continue;
 
-		if (failure->ascq == SCMD_FAILURE_ASCQ_ANY ||
-		    failure->ascq == sshdr.ascq)
+		if (scsi_failure_ascq(failure) == SCMD_FAILURE_ASCQ_ANY ||
+		    scsi_failure_ascq(failure) == scsi_sense_ascq(&sshdr))
 			goto maybe_retry;
 	}
 
@@ -388,10 +388,10 @@ static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
 
 		unsigned int busy = scsi_host_busy(shost);
 
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		if (shost->host_failed || shost->host_eh_scheduled)
 			scsi_eh_wakeup(shost, busy);
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 	}
 	rcu_read_unlock();
 }
@@ -436,9 +436,9 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev)
 	struct scsi_target *starget = scsi_target(current_sdev);
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	starget->starget_sdev_user = NULL;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	/*
 	 * Call blk_run_queue for all LUNs on the target, starting with
@@ -449,11 +449,11 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev)
 	blk_mq_run_hw_queues(current_sdev->request_queue,
 			     shost->queuecommand_may_block);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (!starget->starget_sdev_user)
 		__starget_for_each_device(starget, current_sdev,
 					  scsi_kick_sdev_queue);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 static inline bool scsi_device_is_busy(struct scsi_device *sdev)
@@ -491,7 +491,7 @@ static void scsi_starved_list_run(struct Scsi_Host *shost)
 	struct scsi_device *sdev;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_splice_init(&shost->starved_list, &starved_list);
 
 	while (!list_empty(&starved_list)) {
@@ -532,16 +532,16 @@ static void scsi_starved_list_run(struct Scsi_Host *shost)
 		slq = sdev->request_queue;
 		if (!blk_get_queue(slq))
 			continue;
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 
 		blk_mq_run_hw_queues(slq, false);
 		blk_put_queue(slq);
 
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 	}
 	/* put any unprocessed entries back */
 	list_splice(&starved_list, &shost->starved_list);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -578,7 +578,7 @@ void scsi_run_host_queues(struct Scsi_Host *shost)
 	struct scsi_device *sdev, *prev = NULL;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	__shost_for_each_device(sdev, shost) {
 		/*
 		 * Only skip devices so deep into removal they will never need
@@ -589,7 +589,7 @@ void scsi_run_host_queues(struct Scsi_Host *shost)
 		if (sdev->sdev_state == SDEV_DEL ||
 		    !get_device(&sdev->sdev_gendev))
 			continue;
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 
 		if (prev)
 			put_device(&prev->sdev_gendev);
@@ -597,9 +597,9 @@ void scsi_run_host_queues(struct Scsi_Host *shost)
 
 		prev = sdev;
 
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	if (prev)
 		put_device(&prev->sdev_gendev);
 }
@@ -839,6 +839,8 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)
 		 */
 		action = ACTION_RETRY;
 	} else if (sense_valid && sense_current) {
+		u8 asc = scsi_sense_asc(&sshdr);
+
 		switch (sshdr.sense_key) {
 		case UNIT_ATTENTION:
 			if (cmd->device->removable) {
@@ -865,64 +867,72 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)
 			 * where READ CAPACITY failed, we may have
 			 * read past the end of the disk.
 			 */
-			if ((cmd->device->use_10_for_rw &&
-			    sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
+			if (cmd->device->use_10_for_rw &&
+			    sshdr.sense_code == INVALID_COMMAND_OP_CODE &&
 			    (cmd->cmnd[0] == READ_10 ||
 			     cmd->cmnd[0] == WRITE_10)) {
 				/* This will issue a new 6-byte command. */
 				cmd->device->use_10_for_rw = 0;
 				action = ACTION_REPREP;
-			} else if (sshdr.asc == 0x10) /* DIX */ {
+				break;
+			}
+			if (asc == ASC_ID_CRC_OR_ECC_ERROR) {
+				/* DIX */
 				action = ACTION_FAIL;
 				blk_stat = BLK_STS_PROTECTION;
-			/* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
-			} else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {
+				break;
+			}
+			if (asc == ASC_INVALID_COMMAND_OP_CODE ||
+			    asc == ASC_INVALID_FIELD_IN_CDB) {
 				action = ACTION_FAIL;
 				blk_stat = BLK_STS_TARGET;
-			} else
-				action = ACTION_FAIL;
+				break;
+			}
+			action = ACTION_FAIL;
 			break;
 		case ABORTED_COMMAND:
 			action = ACTION_FAIL;
-			if (sshdr.asc == 0x10) /* DIF */
+			if (asc == ASC_ID_CRC_OR_ECC_ERROR) /* DIF */
 				blk_stat = BLK_STS_PROTECTION;
 			break;
 		case NOT_READY:
 			/* If the device is in the process of becoming
 			 * ready, or has a temporary blockage, retry.
 			 */
-			if (sshdr.asc == 0x04) {
-				switch (sshdr.ascq) {
-				case 0x01: /* becoming ready */
-				case 0x04: /* format in progress */
-				case 0x05: /* rebuild in progress */
-				case 0x06: /* recalculation in progress */
-				case 0x07: /* operation in progress */
-				case 0x08: /* Long write in progress */
-				case 0x09: /* self test in progress */
-				case 0x11: /* notify (enable spinup) required */
-				case 0x14: /* space allocation in progress */
-				case 0x1a: /* start stop unit in progress */
-				case 0x1b: /* sanitize in progress */
-				case 0x1d: /* configuration in progress */
-					action = ACTION_DELAYED_RETRY;
-					break;
-				case 0x0a: /* ALUA state transition */
-					action = ACTION_DELAYED_REPREP;
-					break;
-				/*
-				 * Depopulation might take many hours,
-				 * thus it is not worthwhile to retry.
-				 */
-				case 0x24: /* depopulation in progress */
-				case 0x25: /* depopulation restore in progress */
-					fallthrough;
-				default:
-					action = ACTION_FAIL;
-					break;
-				}
-			} else
+			if (asc != ASC_LU_NOT_READY) {
 				action = ACTION_FAIL;
+				break;
+			}
+
+			switch (sshdr.sense_code) {
+			case LU_IS_IN_PROCESS_OF_BECOMING_READY:
+			case LU_NOT_READY_FORMAT_IN_PROGRESS:
+			case LU_NOT_READY_REBUILD_IN_PROGRESS:
+			case LU_NOT_READY_RECALCULATION_IN_PROGRESS:
+			case LU_NOT_READY_OP_IN_PROGRESS:
+			case LU_NOT_READY_LONG_WRITE_IN_PROGRESS:
+			case LU_NOT_READY_SELFTEST_IN_PROGRESS:
+			case LU_NOT_READY_NOTIFY_REQUIRED:
+			case LU_NOT_READY_SPACE_ALLOCATION_IN_PROGRESS:
+			case LU_NOT_READY_START_STOP_UNIT_COMMAND_IN_PROGRESS:
+			case LU_NOT_READY_SANITIZE_IN_PROGRESS:
+			case LU_NOT_READY_CONFIG_IN_PROGRESS:
+				action = ACTION_DELAYED_RETRY;
+				break;
+			case LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION:
+				action = ACTION_DELAYED_REPREP;
+				break;
+			/*
+			 * Depopulation might take many hours, thus it is not
+			 * worthwhile to retry.
+			 */
+			case DEPOPULATION_IN_PROGRESS:
+			case DEPOPULATION_RESTORATION_IN_PROGRESS:
+				fallthrough;
+			default:
+				action = ACTION_FAIL;
+				break;
+			}
 			break;
 		case VOLUME_OVERFLOW:
 			/* See SSC3rXX or current. */
@@ -930,11 +940,14 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)
 			break;
 		case DATA_PROTECT:
 			action = ACTION_FAIL;
-			if ((sshdr.asc == 0x0C && sshdr.ascq == 0x12) ||
-			    (sshdr.asc == 0x55 &&
-			     (sshdr.ascq == 0x0E || sshdr.ascq == 0x0F))) {
-				/* Insufficient zone resources */
+			switch (sshdr.sense_code) {
+			case WRITE_ERROR_INSUFFICIENT_ZONE_RESOURCES:
+			case INSUFFICIENT_ZONE_RESOURCES:
+			case INSUFFICIENT_ZONE_RESOURCES_TO_COMPLETE_WRITE:
 				blk_stat = BLK_STS_ZONE_OPEN_RESOURCE;
+				break;
+			default:
+				break;
 			}
 			break;
 		case COMPLETED:
@@ -1041,7 +1054,7 @@ static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result,
 		 * skip print since caller wants ATA registers. Only occurs
 		 * on SCSI ATA PASS_THROUGH commands when CK_COND=1
 		 */
-		if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
+		if (sshdr.sense_code == ATA_PASS_THROUGH_INFORMATION_AVAILABLE)
 			do_print = false;
 		else if (req->rq_flags & RQF_QUIET)
 			do_print = false;
@@ -1433,14 +1446,14 @@ static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
 	unsigned int busy;
 
 	if (starget->single_lun) {
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		if (starget->starget_sdev_user &&
 		    starget->starget_sdev_user != sdev) {
-			spin_unlock_irq(shost->host_lock);
+			spin_unlock_irq(&shost->host_lock);
 			return 0;
 		}
 		starget->starget_sdev_user = sdev;
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 	}
 
 	if (starget->can_queue <= 0)
@@ -1467,9 +1480,9 @@ static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
 	return 1;
 
 starved:
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	list_move_tail(&sdev->starved_entry, &shost->starved_list);
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 out_dec:
 	if (starget->can_queue > 0)
 		atomic_dec(&starget->target_busy);
@@ -1506,10 +1519,10 @@ static inline int scsi_host_queue_ready(struct request_queue *q,
 
 	/* We're OK to process the command, so we can't be starved */
 	if (!list_empty(&sdev->starved_entry)) {
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		if (!list_empty(&sdev->starved_entry))
 			list_del_init(&sdev->starved_entry);
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 	}
 
 	__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
@@ -1517,10 +1530,10 @@ static inline int scsi_host_queue_ready(struct request_queue *q,
 	return 1;
 
 starved:
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	if (list_empty(&sdev->starved_entry))
 		list_add_tail(&sdev->starved_entry, &shost->starved_list);
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 out_dec:
 	scsi_dec_host_busy(shost, cmd);
 	return 0;
@@ -2372,9 +2385,8 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, int subpage,
 	struct scsi_sense_hdr my_sshdr;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = retries,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -2432,8 +2444,8 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, int subpage,
 
 	if (!scsi_status_is_good(result)) {
 		if (scsi_sense_valid(sshdr)) {
-			if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
-			    (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
+			if (sshdr->sense_key == ILLEGAL_REQUEST &&
+			    sshdr->sense_code == INVALID_COMMAND_OP_CODE) {
 				/*
 				 * Invalid command operation code: retry using
 				 * MODE SENSE(6) if this was a MODE SENSE(10)
@@ -3573,21 +3585,20 @@ int scsi_vpd_tpg_id(struct scsi_device *sdev, int *rel_id)
 EXPORT_SYMBOL(scsi_vpd_tpg_id);
 
 /**
- * scsi_build_sense - build sense data for a command
+ * scsi_set_sense - build sense data for a command
  * @scmd:	scsi command for which the sense should be formatted
  * @desc:	Sense format (non-zero == descriptor format,
  *              0 == fixed format)
  * @key:	Sense key
- * @asc:	Additional sense code
- * @ascq:	Additional sense code qualifier
+ * @code:	Additional sense code and sense code qualifier
  *
  **/
-void scsi_build_sense(struct scsi_cmnd *scmd, int desc, u8 key, u8 asc, u8 ascq)
+void scsi_set_sense(struct scsi_cmnd *scmd, int desc, u8 key, u16 code)
 {
-	scsi_build_sense_buffer(desc, scmd->sense_buffer, key, asc, ascq);
+	scsi_set_sense_buffer(desc, scmd->sense_buffer, key, code);
 	scmd->result = SAM_STAT_CHECK_CONDITION;
 }
-EXPORT_SYMBOL_GPL(scsi_build_sense);
+EXPORT_SYMBOL_GPL(scsi_set_sense);
 
 #ifdef CONFIG_SCSI_LIB_KUNIT_TEST
 #include "scsi_lib_test.c"
diff --git a/drivers/scsi/scsi_lib_test.c b/drivers/scsi/scsi_lib_test.c
index ae8af0e0047a85..dfd8eefe57e11d 100644
--- a/drivers/scsi/scsi_lib_test.c
+++ b/drivers/scsi/scsi_lib_test.c
@@ -17,42 +17,43 @@ static void scsi_lib_test_multiple_sense(struct kunit *test)
 {
 	struct scsi_failure multiple_sense_failure_defs[] = {
 		{
-			.sense = DATA_PROTECT,
-			.asc = 0x1,
-			.ascq = 0x1,
+			.sense_key = DATA_PROTECT,
+			.sense_code =
+				scsi_sense_code(ASC_NO_INDEX_SECTOR_SIGNAL,
+						0x1),
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x11,
-			.ascq = 0x0,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = UNRECOVERED_READ_ERROR,
 			.allowed = SCSI_LIB_TEST_MAX_ALLOWED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = NOT_READY,
-			.asc = 0x11,
-			.ascq = 0x22,
+			.sense_key = NOT_READY,
+			.sense_code =
+				scsi_sense_code(ASC_UNRECOVERED_READ_ERROR,
+						0x22),
 			.allowed = SCSI_LIB_TEST_MAX_ALLOWED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = ABORTED_COMMAND,
-			.asc = 0x11,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = ABORTED_COMMAND,
+			.sense_code =
+				scsi_sense_code(ASC_UNRECOVERED_READ_ERROR,
+						SCMD_FAILURE_ASCQ_ANY),
 			.allowed = SCSI_LIB_TEST_MAX_ALLOWED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = HARDWARE_ERROR,
-			.asc = SCMD_FAILURE_ASC_ANY,
+			.sense_key = HARDWARE_ERROR,
+			.sense_code = scsi_sense_code(SCMD_FAILURE_ASC_ANY, 0),
 			.allowed = SCSI_LIB_TEST_MAX_ALLOWED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = ILLEGAL_REQUEST,
-			.asc = 0x91,
-			.ascq = 0x36,
+			.sense_key = ILLEGAL_REQUEST,
+			.sense_code = scsi_sense_code(0x91, 0x36),
 			.allowed = SCSI_LIB_TEST_MAX_ALLOWED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -72,32 +73,36 @@ static void scsi_lib_test_multiple_sense(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, &failures));
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, NULL));
 	/* Command failed but caller did not pass in a failures array */
-	scsi_build_sense(&sc, 0, ILLEGAL_REQUEST, 0x91, 0x36);
+	scsi_set_sense(&sc, 0, ILLEGAL_REQUEST, scsi_sense_code(0x91, 0x36));
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, NULL));
 	/* Match end of array */
-	scsi_build_sense(&sc, 0, ILLEGAL_REQUEST, 0x91, 0x36);
+	scsi_set_sense(&sc, 0, ILLEGAL_REQUEST, scsi_sense_code(0x91, 0x36));
 	KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc, &failures));
 	/* Basic match in array */
-	scsi_build_sense(&sc, 0, UNIT_ATTENTION, 0x11, 0x0);
+	scsi_set_sense(&sc, 0, UNIT_ATTENTION, UNRECOVERED_READ_ERROR);
 	KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc, &failures));
 	/* No matching sense entry */
-	scsi_build_sense(&sc, 0, MISCOMPARE, 0x11, 0x11);
+	scsi_set_sense(&sc, 0, MISCOMPARE, READ_ERROR_LOSS_OF_STREAMING);
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, &failures));
 	/* Match using SCMD_FAILURE_ASCQ_ANY */
-	scsi_build_sense(&sc, 0, ABORTED_COMMAND, 0x11, 0x22);
+	scsi_set_sense(&sc, 0, ABORTED_COMMAND,
+		       scsi_sense_code(ASC_UNRECOVERED_READ_ERROR, 0x22));
 	KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc, &failures));
 	/* Fail to match */
-	scsi_build_sense(&sc, 0, ABORTED_COMMAND, 0x22, 0x22);
+	scsi_set_sense(&sc, 0, ABORTED_COMMAND,
+		       scsi_sense_code(ASC_ILLEGAL_FUNCTION, 0x22));
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, &failures));
 	/* Match using SCMD_FAILURE_ASC_ANY */
-	scsi_build_sense(&sc, 0, HARDWARE_ERROR, 0x11, 0x22);
+	scsi_set_sense(&sc, 0, HARDWARE_ERROR,
+		       scsi_sense_code(ASC_UNRECOVERED_READ_ERROR, 0x22));
 	KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc, &failures));
 	/* No matching status entry */
 	sc.result = SAM_STAT_RESERVATION_CONFLICT;
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, &failures));
 
 	/* Test hitting allowed limit */
-	scsi_build_sense(&sc, 0, NOT_READY, 0x11, 0x22);
+	scsi_set_sense(&sc, 0, NOT_READY,
+		       scsi_sense_code(ASC_UNRECOVERED_READ_ERROR, 0x22));
 	for (i = 0; i < SCSI_LIB_TEST_MAX_ALLOWED; i++)
 		KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc,
 				&failures));
@@ -108,7 +113,8 @@ static void scsi_lib_test_multiple_sense(struct kunit *test)
 	scsi_failures_reset_retries(&failures);
 
 	/* Test no retries allowed */
-	scsi_build_sense(&sc, 0, DATA_PROTECT, 0x1, 0x1);
+	scsi_set_sense(&sc, 0, DATA_PROTECT,
+		       scsi_sense_code(ASC_NO_INDEX_SECTOR_SIGNAL, 0x1));
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, &failures));
 }
 
@@ -116,8 +122,9 @@ static void scsi_lib_test_any_sense(struct kunit *test)
 {
 	struct scsi_failure any_sense_failure_defs[] = {
 		{
-			.result = SCMD_FAILURE_SENSE_ANY,
+			.sense_key = SCMD_FAILURE_SENSE_KEY_ANY,
 			.allowed = SCSI_LIB_TEST_MAX_ALLOWED,
+			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{}
 	};
@@ -129,9 +136,10 @@ static void scsi_lib_test_any_sense(struct kunit *test)
 		.sense_buffer = sense,
 	};
 
-	/* Match using SCMD_FAILURE_SENSE_ANY */
+	/* Match using SCMD_FAILURE_SENSE_KEY_ANY */
 	failures.failure_definitions = any_sense_failure_defs;
-	scsi_build_sense(&sc, 0, MEDIUM_ERROR, 0x11, 0x22);
+	scsi_set_sense(&sc, 0, MEDIUM_ERROR,
+		       scsi_sense_code(ASC_UNRECOVERED_READ_ERROR, 0x22));
 	KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc, &failures));
 }
 
@@ -215,14 +223,13 @@ static void scsi_lib_test_total_allowed(struct kunit *test)
 {
 	struct scsi_failure total_allowed_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		/* Fail all CCs except the UA above */
 		{
-			.sense = SCMD_FAILURE_SENSE_ANY,
+			.sense_key = SCMD_FAILURE_SENSE_KEY_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		/* Retry any other errors not listed above */
@@ -245,7 +252,8 @@ static void scsi_lib_test_total_allowed(struct kunit *test)
 	scsi_failures_reset_retries(&failures);
 	failures.total_allowed = SCSI_LIB_TEST_TOTAL_MAX_ALLOWED;
 
-	scsi_build_sense(&sc, 0, UNIT_ATTENTION, 0x28, 0x0);
+	scsi_set_sense(&sc, 0, UNIT_ATTENTION,
+		       NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED);
 	for (i = 0; i < SCSI_LIB_TEST_TOTAL_MAX_ALLOWED; i++)
 		/* Retry since we under the total_allowed limit */
 		KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc,
@@ -259,13 +267,15 @@ static void scsi_lib_test_mixed_total(struct kunit *test)
 {
 	struct scsi_failure mixed_total_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x28,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code =
+				NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code =
+				POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
@@ -291,7 +301,8 @@ static void scsi_lib_test_mixed_total(struct kunit *test)
 	scsi_failures_reset_retries(&failures);
 	failures.total_allowed = SCSI_LIB_TEST_TOTAL_MAX_ALLOWED;
 
-	scsi_build_sense(&sc, 0, UNIT_ATTENTION, 0x28, 0x0);
+	scsi_set_sense(&sc, 0, UNIT_ATTENTION,
+		       NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED);
 	for (i = 0; i < SCSI_LIB_TEST_TOTAL_MAX_ALLOWED; i++)
 		/* Retry since we under the total_allowed limit */
 		KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc,
@@ -300,7 +311,8 @@ static void scsi_lib_test_mixed_total(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, &failures));
 
 	scsi_failures_reset_retries(&failures);
-	scsi_build_sense(&sc, 0, UNIT_ATTENTION, 0x28, 0x0);
+	scsi_set_sense(&sc, 0, UNIT_ATTENTION,
+		       NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED);
 	for (i = 0; i < SCSI_LIB_TEST_TOTAL_MAX_ALLOWED; i++)
 		/* Retry since we under the total_allowed limit */
 		KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc,
@@ -308,7 +320,8 @@ static void scsi_lib_test_mixed_total(struct kunit *test)
 	sc.result = DID_TIME_OUT << 16;
 	/* Retry because this failure has a per failure limit */
 	KUNIT_EXPECT_EQ(test, -EAGAIN, scsi_check_passthrough(&sc, &failures));
-	scsi_build_sense(&sc, 0, UNIT_ATTENTION, 0x29, 0x0);
+	scsi_set_sense(&sc, 0, UNIT_ATTENTION,
+		       POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED);
 	/* total_allowed is now hit so no more retries */
 	KUNIT_EXPECT_EQ(test, 0, scsi_check_passthrough(&sc, &failures));
 }
diff --git a/drivers/scsi/scsi_logging.c b/drivers/scsi/scsi_logging.c
index 3cd0d30740855f..0a9a2f7c1966e0 100644
--- a/drivers/scsi/scsi_logging.c
+++ b/drivers/scsi/scsi_logging.c
@@ -238,29 +238,32 @@ EXPORT_SYMBOL(scsi_print_command);
 
 static size_t
 scsi_format_extd_sense(char *buffer, size_t buf_len,
-		       unsigned char asc, unsigned char ascq)
+		       const struct scsi_sense_hdr *sshdr)
 {
 	size_t off = 0;
 	const char *extd_sense_fmt = NULL;
-	const char *extd_sense_str = scsi_extd_sense_format(asc, ascq,
-							    &extd_sense_fmt);
+	const char *extd_sense_str =
+		scsi_extd_sense_format(sshdr, &extd_sense_fmt);
 
 	if (extd_sense_str) {
 		off = scnprintf(buffer, buf_len, "Add. Sense: %s",
 				extd_sense_str);
 		if (extd_sense_fmt)
 			off += scnprintf(buffer + off, buf_len - off,
-					 "(%s%x)", extd_sense_fmt, ascq);
+					 "(%s%x)", extd_sense_fmt,
+					 scsi_sense_ascq(sshdr));
 	} else {
-		if (asc >= 0x80)
+		if (scsi_sense_asc(sshdr) >= 0x80)
 			off = scnprintf(buffer, buf_len, "<<vendor>>");
 		off += scnprintf(buffer + off, buf_len - off,
-				 "ASC=0x%x ", asc);
-		if (ascq >= 0x80)
+				 "ASC=0x%x ",
+				 scsi_sense_asc(sshdr));
+		if (scsi_sense_ascq(sshdr) >= 0x80)
 			off += scnprintf(buffer + off, buf_len - off,
 					 "<<vendor>>");
 		off += scnprintf(buffer + off, buf_len - off,
-				 "ASCQ=0x%x ", ascq);
+				 "ASCQ=0x%x ",
+				 scsi_sense_ascq(sshdr));
 	}
 	return off;
 }
@@ -333,8 +336,7 @@ scsi_log_print_sense_hdr(const struct scsi_device *sdev, const char *name,
 	if (!logbuf)
 		return;
 	off = sdev_format_header(logbuf, logbuf_len, name, tag);
-	off += scsi_format_extd_sense(logbuf + off, logbuf_len - off,
-				      sshdr->asc, sshdr->ascq);
+	off += scsi_format_extd_sense(logbuf + off, logbuf_len - off, sshdr);
 	dev_printk(KERN_INFO, &sdev->sdev_gendev, "%s", logbuf);
 	scsi_log_release_buffer(logbuf);
 }
diff --git a/drivers/scsi/scsi_proc.c b/drivers/scsi/scsi_proc.c
index 1799dcae775c01..fdc355d783da36 100644
--- a/drivers/scsi/scsi_proc.c
+++ b/drivers/scsi/scsi_proc.c
@@ -28,6 +28,7 @@
 #include <linux/mutex.h>
 #include <linux/gfp.h>
 #include <linux/uaccess.h>
+#include <linux/slab.h>
 
 #include <scsi/scsi.h>
 #include <scsi/scsi_device.h>
@@ -74,7 +75,7 @@ static ssize_t proc_scsi_host_write(struct file *file, const char __user *buf,
 	if (!shost->hostt->write_info)
 		return -EINVAL;
 
-	page = (char *)__get_free_page(GFP_KERNEL);
+	page = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (page) {
 		ret = -EFAULT;
 		if (copy_from_user(page, buf, count))
@@ -82,7 +83,7 @@ static ssize_t proc_scsi_host_write(struct file *file, const char __user *buf,
 		ret = shost->hostt->write_info(shost, page, count);
 	}
 out:
-	free_page((unsigned long)page);
+	kfree(page);
 	return ret;
 }
 
@@ -412,7 +413,7 @@ static ssize_t proc_scsi_write(struct file *file, const char __user *buf,
 	if (!buf || length > PAGE_SIZE)
 		return -EINVAL;
 
-	buffer = (char *)__get_free_page(GFP_KERNEL);
+	buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buffer)
 		return -ENOMEM;
 
@@ -467,7 +468,7 @@ static ssize_t proc_scsi_write(struct file *file, const char __user *buf,
 		err = length;
 
  out:
-	free_page((unsigned long)buffer);
+	kfree(buffer);
 	return err;
 }
 
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index 0f0f243c25615b..8aae12fd38f3a6 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -397,11 +397,11 @@ static void scsi_target_destroy(struct scsi_target *starget)
 	BUG_ON(starget->state == STARGET_DEL);
 	starget->state = STARGET_DEL;
 	transport_destroy_device(dev);
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (shost->hostt->target_destroy)
 		shost->hostt->target_destroy(starget);
 	list_del_init(&starget->siblings);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	put_device(dev);
 }
 
@@ -524,14 +524,14 @@ static struct scsi_target *scsi_alloc_target(struct device *parent,
 	starget->scsi_level = SCSI_2;
 	starget->max_target_blocked = SCSI_DEFAULT_TARGET_BLOCKED;
  retry:
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	found_target = __scsi_find_target(parent, channel, id);
 	if (found_target)
 		goto found;
 
 	list_add_tail(&starget->siblings, &shost->__targets);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	/* allocate and add */
 	transport_setup_device(dev);
 	if (shost->hostt->target_alloc) {
@@ -558,7 +558,7 @@ static struct scsi_target *scsi_alloc_target(struct device *parent,
 	 */
 	ref_got = kref_get_unless_zero(&found_target->reap_ref);
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	if (ref_got) {
 		put_device(dev);
 		return found_target;
@@ -652,19 +652,18 @@ static int scsi_probe_lun(struct scsi_device *sdev, unsigned char *inq_result,
 	int pass, count, result, resid;
 	struct scsi_failure failure_defs[] = {
 		/*
-		 * not-ready to ready transition [asc/ascq=0x28/0x0] or
-		 * power-on, reset [asc/ascq=0x29/0x0], continue. INQUIRY
-		 * should not yield UNIT_ATTENTION but many buggy devices do
-		 * so anyway.
+		 * not-ready to ready transition or power-on, reset, continue.
+		 * INQUIRY should not yield UNIT_ATTENTION but many buggy
+		 * devices do so anyway.
 		 */
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x28,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
@@ -1457,14 +1456,13 @@ static int scsi_report_lun_scan(struct Scsi_Host *shost,
 	struct scsi_device *sdev;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		/* Fail all CCs except the UA above */
 		{
-			.sense = SCMD_FAILURE_SENSE_ANY,
+			.sense_key = SCMD_FAILURE_SENSE_KEY_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		/* Retry any other errors not listed above */
@@ -2106,16 +2104,16 @@ void scsi_forget_host(struct Scsi_Host *shost)
 	unsigned long flags;
 
  restart:
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_for_each_entry(sdev, &shost->__devices, siblings) {
 		if (scsi_device_is_pseudo_dev(sdev) ||
 		    sdev->sdev_state == SDEV_DEL)
 			continue;
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		__scsi_remove_device(sdev);
 		goto restart;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	/*
 	 * Remove the pseudo device last since it may be needed during removal
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index 9480432f650b75..fe54792ed69e4f 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -214,7 +214,7 @@ store_shost_state(struct device *dev, struct device_attribute *attr,
 	if (!state)
 		return -EINVAL;
 
-	scoped_guard(spinlock_irq, shost->host_lock)
+	scoped_guard(spinlock_irq, &shost->host_lock)
 		if (scsi_host_set_state(shost, state))
 			return -EINVAL;
 	return count;
@@ -352,7 +352,7 @@ store_shost_eh_deadline(struct device *dev, struct device_attribute *attr,
 			return -EINVAL;
 	}
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (scsi_host_in_recovery(shost))
 		ret = -EBUSY;
 	else {
@@ -363,7 +363,7 @@ store_shost_eh_deadline(struct device *dev, struct device_attribute *attr,
 
 		ret = count;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	return ret;
 }
@@ -459,11 +459,11 @@ static void scsi_device_dev_release(struct device *dev)
 
 	parent = sdev->sdev_gendev.parent;
 
-	spin_lock_irqsave(sdev->host->host_lock, flags);
+	spin_lock_irqsave(&sdev->host->host_lock, flags);
 	list_del(&sdev->siblings);
 	list_del(&sdev->same_target_siblings);
 	list_del(&sdev->starved_entry);
-	spin_unlock_irqrestore(sdev->host->host_lock, flags);
+	spin_unlock_irqrestore(&sdev->host->host_lock, flags);
 
 	cancel_work_sync(&sdev->event_work);
 
@@ -1547,7 +1547,7 @@ static void __scsi_remove_target(struct scsi_target *starget)
 	unsigned long flags;
 	struct scsi_device *sdev;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
  restart:
 	list_for_each_entry(sdev, &shost->__devices, siblings) {
 		/*
@@ -1563,13 +1563,13 @@ static void __scsi_remove_target(struct scsi_target *starget)
 		    sdev->sdev_state == SDEV_CANCEL ||
 		    !get_device(&sdev->sdev_gendev))
 			continue;
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		scsi_remove_device(sdev);
 		put_device(&sdev->sdev_gendev);
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		goto restart;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -1587,7 +1587,7 @@ void scsi_remove_target(struct device *dev)
 	unsigned long flags;
 
 restart:
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_for_each_entry(starget, &shost->__targets, siblings) {
 		if (starget->state == STARGET_DEL ||
 		    starget->state == STARGET_REMOVE ||
@@ -1599,13 +1599,13 @@ void scsi_remove_target(struct device *dev)
 				starget->state = STARGET_CREATED_REMOVE;
 			else
 				starget->state = STARGET_REMOVE;
-			spin_unlock_irqrestore(shost->host_lock, flags);
+			spin_unlock_irqrestore(&shost->host_lock, flags);
 			__scsi_remove_target(starget);
 			scsi_target_reap(starget);
 			goto restart;
 		}
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 EXPORT_SYMBOL(scsi_remove_target);
 
@@ -1710,10 +1710,10 @@ void scsi_sysfs_device_initialize(struct scsi_device *sdev)
 		sdev->lun_in_cdb = 1;
 
 	transport_setup_device(&sdev->sdev_gendev);
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_add_tail(&sdev->same_target_siblings, &starget->devices);
 	list_add_tail(&sdev->siblings, &shost->__devices);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	/*
 	 * device can now only be removed via __scsi_remove_device() so hold
 	 * the target.  Target will be held in CREATED state until something
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index 173ed6373f04ba..be0efffab2fcc0 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -645,19 +645,19 @@ fc_find_rport_by_wwpn(struct Scsi_Host *shost, u64 wwpn)
 	struct fc_rport *rport;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	list_for_each_entry(rport, &fc_host_rports(shost), peers) {
 		if (rport->port_state != FC_PORTSTATE_ONLINE)
 			continue;
 
 		if (rport->port_name == wwpn) {
-			spin_unlock_irqrestore(shost->host_lock, flags);
+			spin_unlock_irqrestore(&shost->host_lock, flags);
 			return rport;
 		}
 	}
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return NULL;
 }
 EXPORT_SYMBOL(fc_find_rport_by_wwpn);
@@ -1718,13 +1718,13 @@ store_fc_vport_delete(struct device *dev, struct device_attribute *attr,
 	struct Scsi_Host *shost = vport_to_shost(vport);
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (vport->flags & (FC_VPORT_DEL | FC_VPORT_CREATING)) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		return -EBUSY;
 	}
 	vport->flags |= FC_VPORT_DELETING;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	fc_queue_work(shost, &vport->vport_delete_work);
 	return count;
@@ -2052,7 +2052,7 @@ store_fc_private_host_tgtid_bind_type(struct device *dev,
 
 	/* if changing bind type, purge all unused consistent bindings */
 	if (val != fc_host_tgtid_bind_type(shost)) {
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		while (!list_empty(&fc_host_rport_bindings(shost))) {
 			get_list_head_entry(rport,
 				&fc_host_rport_bindings(shost), peers);
@@ -2060,7 +2060,7 @@ store_fc_private_host_tgtid_bind_type(struct device *dev,
 			rport->port_state = FC_PORTSTATE_DELETED;
 			fc_queue_work(shost, &rport->rport_delete_work);
 		}
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 	}
 
 	fc_host_tgtid_bind_type(shost) = val;
@@ -2107,10 +2107,10 @@ store_fc_private_host_dev_loss_tmo(struct device *dev,
 		return rc;
 
 	fc_host_dev_loss_tmo(shost) = val;
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_for_each_entry(rport, &fc_host->rports, peers)
 		fc_rport_set_dev_loss_tmo(rport, val);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	return count;
 }
 
@@ -2417,7 +2417,7 @@ store_fc_host_vport_delete(struct device *dev, struct device_attribute *attr,
 	if (stat)
 		return stat;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	match = 0;
 	/* we only allow support on Channel 0 !!! */
 	list_for_each_entry(vport, &fc_host->vports, peers) {
@@ -2430,7 +2430,7 @@ store_fc_host_vport_delete(struct device *dev, struct device_attribute *attr,
 			break;
 		}
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (!match)
 		return -ENODEV;
@@ -2590,7 +2590,7 @@ fc_user_scan_tgt(struct Scsi_Host *shost, uint channel, uint id, u64 lun)
 	struct fc_rport *rport;
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	list_for_each_entry(rport, &fc_host_rports(shost), peers) {
 		if (rport->scsi_target_id == -1)
@@ -2602,14 +2602,14 @@ fc_user_scan_tgt(struct Scsi_Host *shost, uint channel, uint id, u64 lun)
 
 		if ((channel == rport->channel) &&
 		    (id == rport->scsi_target_id)) {
-			spin_unlock_irqrestore(shost->host_lock, flags);
+			spin_unlock_irqrestore(&shost->host_lock, flags);
 			scsi_scan_target(&rport->dev, channel, id, lun,
 					 SCSI_SCAN_MANUAL);
 			return;
 		}
 	}
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /*
@@ -2922,7 +2922,7 @@ fc_remove_host(struct Scsi_Host *shost)
 	struct fc_host_attrs *fc_host = shost_to_fc_host(shost);
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	/* Remove any vports */
 	list_for_each_entry_safe(vport, next_vport, &fc_host->vports, peers) {
@@ -2945,7 +2945,7 @@ fc_remove_host(struct Scsi_Host *shost)
 		fc_queue_work(shost, &rport->rport_delete_work);
 	}
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	/* flush all scan work items */
 	scsi_flush_work(shost);
@@ -3021,18 +3021,18 @@ fc_rport_final_delete(struct work_struct *work)
 	 * only when rmmod'ing the LLDD and we're asking for
 	 * immediate termination of the rports
 	 */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (rport->flags & FC_RPORT_DEVLOSS_PENDING) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		if (!cancel_delayed_work(&rport->fail_io_work))
 			fc_flush_devloss(shost, rport);
 		if (!cancel_delayed_work(&rport->dev_loss_work))
 			fc_flush_devloss(shost, rport);
 		cancel_work_sync(&rport->scan_work);
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	/* Delete SCSI target and sdevs */
 	if (rport->scsi_target_id != -1)
@@ -3045,13 +3045,13 @@ fc_rport_final_delete(struct work_struct *work)
 	 * Avoid this call if we already called it when we preserved the
 	 * rport for the binding.
 	 */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (!(rport->flags & FC_RPORT_DEVLOSS_CALLBK_DONE) &&
 	    (i->f->dev_loss_tmo_callbk)) {
 		rport->flags |= FC_RPORT_DEVLOSS_CALLBK_DONE;
 		do_callback = 1;
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (do_callback)
 		i->f->dev_loss_tmo_callbk(rport);
@@ -3123,7 +3123,7 @@ fc_remote_port_create(struct Scsi_Host *shost, int channel,
 	INIT_WORK(&rport->stgt_delete_work, fc_starget_delete);
 	INIT_WORK(&rport->rport_delete_work, fc_rport_final_delete);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	rport->number = fc_host->next_rport_number++;
 	if ((rport->roles & FC_PORT_ROLE_FCP_TARGET) ||
@@ -3134,7 +3134,7 @@ fc_remote_port_create(struct Scsi_Host *shost, int channel,
 	list_add_tail(&rport->peers, &fc_host->rports);
 	scsi_host_get(shost);			/* for fc_host->rport list */
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	rport->devloss_work_q = alloc_workqueue("fc_dl_%d_%d", WQ_PERCPU, 0,
 						shost->host_no, rport->number);
@@ -3144,10 +3144,10 @@ fc_remote_port_create(struct Scsi_Host *shost, int channel,
  * Note that we have not yet called device_initialize() / get_device()
  * Cannot reclaim incremented rport->number because we released host_lock
  */
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		list_del(&rport->peers);
 		scsi_host_put(shost);			/* for fc_host->rport list */
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		kfree(rport);
 		return NULL;
 	}
@@ -3181,10 +3181,10 @@ fc_remote_port_create(struct Scsi_Host *shost, int channel,
 
 delete_rport:
 	transport_destroy_device(dev);
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_del(&rport->peers);
 	scsi_host_put(shost);			/* for fc_host->rport list */
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	put_device(dev->parent);
 	kfree(rport);
 	return NULL;
@@ -3246,7 +3246,7 @@ fc_remote_port_add(struct Scsi_Host *shost, int channel,
 	 * deleted, but we've held off the real delete while the target
 	 * is in a "blocked" state.
 	 */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	list_for_each_entry(rport, &fc_host->rports, peers) {
 
@@ -3281,7 +3281,7 @@ fc_remote_port_add(struct Scsi_Host *shost, int channel,
 				rport->port_state = FC_PORTSTATE_ONLINE;
 				rport->roles = ids->roles;
 
-				spin_unlock_irqrestore(shost->host_lock, flags);
+				spin_unlock_irqrestore(&shost->host_lock, flags);
 
 				if (fci->f->dd_fcrport_size)
 					memset(rport->dd_data, 0,
@@ -3318,24 +3318,24 @@ fc_remote_port_add(struct Scsi_Host *shost, int channel,
 				if (!cancel_delayed_work(&rport->dev_loss_work))
 					fc_flush_devloss(shost, rport);
 
-				spin_lock_irqsave(shost->host_lock, flags);
+				spin_lock_irqsave(&shost->host_lock, flags);
 
 				rport->flags &= ~(FC_RPORT_FAST_FAIL_TIMEDOUT |
 						  FC_RPORT_DEVLOSS_PENDING |
 						  FC_RPORT_DEVLOSS_CALLBK_DONE);
 
-				spin_unlock_irqrestore(shost->host_lock, flags);
+				spin_unlock_irqrestore(&shost->host_lock, flags);
 
 				/* if target, initiate a scan */
 				if (rport->scsi_target_id != -1) {
 					scsi_target_unblock(&rport->dev,
 							    SDEV_RUNNING);
-					spin_lock_irqsave(shost->host_lock,
+					spin_lock_irqsave(&shost->host_lock,
 							  flags);
 					rport->flags |= FC_RPORT_SCAN_PENDING;
 					scsi_queue_work(shost,
 							&rport->scan_work);
-					spin_unlock_irqrestore(shost->host_lock,
+					spin_unlock_irqrestore(&shost->host_lock,
 							flags);
 				}
 
@@ -3394,14 +3394,14 @@ fc_remote_port_add(struct Scsi_Host *shost, int channel,
 			if (fci->f->dd_fcrport_size)
 				memset(rport->dd_data, 0,
 						fci->f->dd_fcrport_size);
-			spin_unlock_irqrestore(shost->host_lock, flags);
+			spin_unlock_irqrestore(&shost->host_lock, flags);
 
 			fc_remote_port_rolechg(rport, ids->roles);
 			return rport;
 		}
 	}
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	/* No consistent binding found - create new remote port entry */
 	rport = fc_remote_port_create(shost, channel, ids);
@@ -3478,11 +3478,11 @@ fc_remote_port_delete(struct fc_rport  *rport)
 	 * there's still a scan pending.
 	 */
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	if ((rport->port_state != FC_PORTSTATE_ONLINE) &&
 		(rport->port_state != FC_PORTSTATE_MARGINAL)) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		return;
 	}
 
@@ -3503,7 +3503,7 @@ fc_remote_port_delete(struct fc_rport  *rport)
 
 	rport->flags |= FC_RPORT_DEVLOSS_PENDING;
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	scsi_block_targets(shost, &rport->dev);
 
@@ -3547,7 +3547,7 @@ fc_remote_port_rolechg(struct fc_rport  *rport, u32 roles)
 	unsigned long flags;
 	int create = 0;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (roles & FC_PORT_ROLE_FCP_TARGET) {
 		if (rport->scsi_target_id == -1) {
 			rport->scsi_target_id = fc_host->next_target_id++;
@@ -3558,7 +3558,7 @@ fc_remote_port_rolechg(struct fc_rport  *rport, u32 roles)
 
 	rport->roles = roles;
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (create) {
 		/*
@@ -3578,21 +3578,21 @@ fc_remote_port_rolechg(struct fc_rport  *rport, u32 roles)
 		if (!cancel_delayed_work(&rport->dev_loss_work))
 			fc_flush_devloss(shost, rport);
 
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		rport->flags &= ~(FC_RPORT_FAST_FAIL_TIMEDOUT |
 				  FC_RPORT_DEVLOSS_PENDING |
 				  FC_RPORT_DEVLOSS_CALLBK_DONE);
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 
 		/* ensure any stgt delete functions are done */
 		fc_flush_work(shost);
 
 		scsi_target_unblock(&rport->dev, SDEV_RUNNING);
 		/* initiate a scan of the target */
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		rport->flags |= FC_RPORT_SCAN_PENDING;
 		scsi_queue_work(shost, &rport->scan_work);
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 	}
 }
 EXPORT_SYMBOL(fc_remote_port_rolechg);
@@ -3615,7 +3615,7 @@ fc_timeout_deleted_rport(struct work_struct *work)
 	unsigned long flags;
 	int do_callback = 0;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	rport->flags &= ~FC_RPORT_DEVLOSS_PENDING;
 
@@ -3631,7 +3631,7 @@ fc_timeout_deleted_rport(struct work_struct *work)
 		dev_printk(KERN_ERR, &rport->dev,
 			"blocked FC remote port time out: no longer"
 			" a FCP target, removing starget\n");
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		scsi_target_unblock(&rport->dev, SDEV_TRANSPORT_OFFLINE);
 		fc_queue_work(shost, &rport->stgt_delete_work);
 		return;
@@ -3639,7 +3639,7 @@ fc_timeout_deleted_rport(struct work_struct *work)
 
 	/* NOOP state - we're flushing workq's */
 	if (rport->port_state != FC_PORTSTATE_BLOCKED) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		dev_printk(KERN_ERR, &rport->dev,
 			"blocked FC remote port time out: leaving"
 			" rport%s alone\n",
@@ -3656,7 +3656,7 @@ fc_timeout_deleted_rport(struct work_struct *work)
 			" rport%s\n",
 			(rport->scsi_target_id != -1) ?  " and starget" : "");
 		fc_queue_work(shost, &rport->rport_delete_work);
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		return;
 	}
 
@@ -3686,10 +3686,10 @@ fc_timeout_deleted_rport(struct work_struct *work)
 	 * item to teardown the starget. (FCOE libFC folks prefer this
 	 * and to have the rport_port_id still set when it's done).
 	 */
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	fc_terminate_rport_io(rport);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	if (rport->port_state == FC_PORTSTATE_NOTPRESENT) {	/* still missing */
 
@@ -3722,7 +3722,7 @@ fc_timeout_deleted_rport(struct work_struct *work)
 		do_callback = 1;
 	}
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	/*
 	 * Notify the driver that the rport is now dead. The LLDD will
@@ -3777,9 +3777,9 @@ fc_scsi_scan_rport(struct work_struct *work)
 				 SCSI_SCAN_RESCAN);
 	}
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	rport->flags &= ~FC_RPORT_SCAN_PENDING;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 }
 
 /**
@@ -3801,14 +3801,14 @@ int fc_block_rport(struct fc_rport *rport)
 	struct Scsi_Host *shost = rport_to_shost(rport);
 	unsigned long flags;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	while (rport->port_state == FC_PORTSTATE_BLOCKED &&
 	       !(rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT)) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		msleep(1000);
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT)
 		return FAST_IO_FAIL;
@@ -3916,10 +3916,10 @@ fc_vport_setup(struct Scsi_Host *shost, int channel, struct device *pdev,
 	vport->flags = FC_VPORT_CREATING;
 	INIT_WORK(&vport->vport_delete_work, fc_vport_sched_delete);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 
 	if (fc_host->npiv_vports_inuse >= fc_host->max_npiv_vports) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		kfree(vport);
 		return -ENOSPC;
 	}
@@ -3928,7 +3928,7 @@ fc_vport_setup(struct Scsi_Host *shost, int channel, struct device *pdev,
 	list_add_tail(&vport->peers, &fc_host->vports);
 	scsi_host_get(shost);			/* for fc_host->vport list */
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	dev = &vport->dev;
 	device_initialize(dev);			/* takes self reference */
@@ -3965,9 +3965,9 @@ fc_vport_setup(struct Scsi_Host *shost, int channel, struct device *pdev,
 				"%s, err=%d\n",
 				__func__, dev_name(dev), error);
 	}
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	vport->flags &= ~FC_VPORT_CREATING;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	dev_printk(KERN_NOTICE, pdev,
 			"%s created via shost%d channel %d\n", dev_name(dev),
@@ -3982,11 +3982,11 @@ fc_vport_setup(struct Scsi_Host *shost, int channel, struct device *pdev,
 	device_del(dev);
 delete_vport:
 	transport_destroy_device(dev);
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	list_del(&vport->peers);
 	scsi_host_put(shost);			/* for fc_host->vport list */
 	fc_host->npiv_vports_inuse--;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	put_device(dev->parent);
 	kfree(vport);
 
@@ -4041,7 +4041,7 @@ fc_vport_terminate(struct fc_vport *vport)
 	else
 		stat = -ENOENT;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	vport->flags &= ~FC_VPORT_DELETING;
 	if (!stat) {
 		vport->flags |= FC_VPORT_DELETED;
@@ -4049,7 +4049,7 @@ fc_vport_terminate(struct fc_vport *vport)
 		fc_host->npiv_vports_inuse--;
 		scsi_host_put(shost);		/* for fc_host->vport list */
 	}
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (stat)
 		return stat;
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index d689b9ed08a6c4..aaa47552a215ce 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -223,8 +223,8 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
 
 /*
  * Set shost->opt_sectors from the DMA optimal mapping size, but only
- * when dma_opt_mapping_size() is strictly less than dma_max_mapping_size(),
- * indicating a genuine optimization hint from an IOMMU or DMA backend.
+ * when dma_max_opt_mapping_size() is strictly less than
+ * dma_max_mapping_size(), indicating a genuine optimization hint.
  * When the two are equal (e.g. IOMMU disabled / passthrough), no real
  * hint exists, so leave opt_sectors at 0 to avoid bogus optimal_io_size
  * values that break filesystem geometry (e.g. mkfs.xfs stripe alignment).
@@ -232,16 +232,16 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
 static void sas_dma_setup_opt_sectors(struct Scsi_Host *shost)
 {
 	struct device *dma_dev = shost->dma_dev;
-	size_t opt = dma_opt_mapping_size(dma_dev);
+	size_t max_opt = dma_max_opt_mapping_size(dma_dev);
 	size_t max = dma_max_mapping_size(dma_dev);
 	unsigned int opt_sectors;
 
-	/* opt >= max means no real hint was provided by the DMA layer */
-	if (opt >= max)
+	/* max_opt >= max means no real hint was provided by the DMA layer */
+	if (max_opt >= max)
 		return;
 
 	/* Clamp to max_sectors to avoid overflow in sector arithmetic */
-	opt_sectors = min_t(unsigned int, opt >> SECTOR_SHIFT,
+	opt_sectors = min_t(unsigned int, max_opt >> SECTOR_SHIFT,
 			    shost->max_sectors);
 
 	/* Guard against zero before rounddown_pow_of_two() */
diff --git a/drivers/scsi/scsi_transport_spi.c b/drivers/scsi/scsi_transport_spi.c
index 3e3da8c2ff26f5..bd049d549244ae 100644
--- a/drivers/scsi/scsi_transport_spi.c
+++ b/drivers/scsi/scsi_transport_spi.c
@@ -112,9 +112,8 @@ static int spi_execute(struct scsi_device *sdev, const void *cmd,
 			REQ_FAILFAST_DRIVER;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = DV_RETRIES,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -680,10 +679,9 @@ spi_dv_device_echo_buffer(struct scsi_device *sdev, u8 *buffer,
 		if (result || !scsi_device_online(sdev)) {
 
 			scsi_device_set_state(sdev, SDEV_QUIESCE);
-			if (result > 0 && scsi_sense_valid(&sshdr)
-			    && sshdr.sense_key == ILLEGAL_REQUEST
-			    /* INVALID FIELD IN CDB */
-			    && sshdr.asc == 0x24 && sshdr.ascq == 0x00)
+			if (result > 0 && scsi_sense_valid(&sshdr) &&
+			    sshdr.sense_key == ILLEGAL_REQUEST &&
+			    sshdr.sense_code == INVALID_FIELD_IN_CDB)
 				/* This would mean that the drive lied
 				 * to us about supporting an echo
 				 * buffer (unfortunately some Western
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index a1b21ea14e5491..b0d2542fd31720 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -1786,8 +1786,7 @@ static int media_not_present(struct scsi_disk *sdkp,
 	switch (sshdr->sense_key) {
 	case UNIT_ATTENTION:
 	case NOT_READY:
-		/* medium not present */
-		if (sshdr->asc == 0x3A) {
+		if (scsi_sense_asc(sshdr) == ASC_MEDIUM_NOT_PRESENT) {
 			set_media_not_present(sdkp);
 			return 1;
 		}
@@ -1911,12 +1910,14 @@ static int sd_sync_cache(struct scsi_disk *sdkp)
 
 		if (scsi_status_is_check_condition(res) &&
 		    scsi_sense_valid(&sshdr)) {
+			u8 asc = scsi_sense_asc(&sshdr);
+
 			sd_print_sense_hdr(sdkp, &sshdr);
 
 			/* we need to evaluate the error return  */
-			if (sshdr.asc == 0x3a ||	/* medium not present */
-			    sshdr.asc == 0x20 ||	/* invalid command */
-			    (sshdr.asc == 0x74 && sshdr.ascq == 0x71))	/* drive is password locked */
+			if (asc == ASC_MEDIUM_NOT_PRESENT ||
+			    asc == ASC_INVALID_COMMAND_OP_CODE ||
+			    sshdr.sense_code == LU_ACCESS_NOT_AUTHORIZED)
 				/* this is no error here */
 				return 0;
 
@@ -1926,8 +1927,8 @@ static int sd_sync_cache(struct scsi_disk *sdkp)
 			 * this is called during shutdown or suspend so just
 			 * return success so those operations can proceed.
 			 */
-			if ((sshdr.asc == 0x04 && sshdr.ascq == 0x04) ||
-			    sshdr.sense_key == ILLEGAL_REQUEST)
+			if (sshdr.sense_key == ILLEGAL_REQUEST ||
+			    sshdr.sense_code == LU_NOT_READY_FORMAT_IN_PROGRESS)
 				return 0;
 		}
 
@@ -2015,7 +2016,9 @@ static int sd_scsi_to_pr_err(struct scsi_sense_hdr *sshdr, int result)
 			return PR_STS_IOERR;
 
 		if (sshdr->sense_key == ILLEGAL_REQUEST &&
-		    (sshdr->asc == 0x26 || sshdr->asc == 0x24))
+		    (scsi_sense_asc(sshdr) ==
+				ASC_INVALID_FIELD_IN_PARAMETER_LIST ||
+		     scsi_sense_asc(sshdr) == ASC_INVALID_FIELD_IN_CDB))
 			return -EINVAL;
 
 		fallthrough;
@@ -2033,9 +2036,8 @@ static int sd_pr_in_command(struct block_device *bdev, u8 sa,
 	u8 cmd[10] = { PERSISTENT_RESERVE_IN, sa };
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = 5,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -2145,9 +2147,8 @@ static int sd_pr_out_command(struct block_device *bdev, u8 sa, u64 key,
 	struct scsi_sense_hdr sshdr;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = 5,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -2440,16 +2441,17 @@ static int sd_done(struct scsi_cmnd *SCpnt)
 		memset(SCpnt->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
 		break;
 	case ABORTED_COMMAND:
-		if (sshdr.asc == 0x10)  /* DIF: Target detected corruption */
+		if (scsi_sense_asc(&sshdr) == ASC_ID_CRC_OR_ECC_ERROR) /* DIF */
 			good_bytes = sd_completed_bytes(SCpnt);
 		break;
 	case ILLEGAL_REQUEST:
-		switch (sshdr.asc) {
-		case 0x10:	/* DIX: Host detected corruption */
+		switch (scsi_sense_asc(&sshdr)) {
+		case ASC_ID_CRC_OR_ECC_ERROR:
+			/* DIX */
 			good_bytes = sd_completed_bytes(SCpnt);
 			break;
-		case 0x20:	/* INVALID COMMAND OPCODE */
-		case 0x24:	/* INVALID FIELD IN CDB */
+		case ASC_INVALID_COMMAND_OP_CODE:
+		case ASC_INVALID_FIELD_IN_CDB:
 			switch (SCpnt->cmnd[0]) {
 			case UNMAP:
 				sd_disable_discard(sdkp);
@@ -2494,15 +2496,15 @@ sd_spinup_disk(struct scsi_disk *sdkp)
 	struct scsi_failure failure_defs[] = {
 		/* Do not retry Medium Not Present */
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x3A,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code =
+				MEDIUM_NOT_PRESENT | SCMD_FAILURE_ASCQ_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = NOT_READY,
-			.asc = 0x3A,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = NOT_READY,
+			.sense_code =
+				MEDIUM_NOT_PRESENT | SCMD_FAILURE_ASCQ_ANY,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		/* Retry when scsi_status_is_good would return false 3 times */
@@ -2566,18 +2568,18 @@ sd_spinup_disk(struct scsi_disk *sdkp)
 			break;
 
 		if (sense_valid && sshdr.sense_key == NOT_READY) {
-			if (sshdr.asc == 4 && sshdr.ascq == 3)
-				break;	/* manual intervention required */
-			if (sshdr.asc == 4 && sshdr.ascq == 0xb)
-				break;	/* standby */
-			if (sshdr.asc == 4 && sshdr.ascq == 0xc)
-				break;	/* unavailable */
-			if (sshdr.asc == 4 && sshdr.ascq == 0x1b)
-				break;	/* sanitize in progress */
-			if (sshdr.asc == 4 && sshdr.ascq == 0x24)
-				break;	/* depopulation in progress */
-			if (sshdr.asc == 4 && sshdr.ascq == 0x25)
-				break;	/* depopulation restoration in progress */
+			switch (sshdr.sense_code) {
+			case LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED:
+			case LU_NOT_ACCESSIBLE_TARGET_PORT_IN_STANDBY_STATE:
+			case LU_NOT_ACCESSIBLE_TARGET_PORT_IN_UNAVAILABLE_STATE:
+			case LU_NOT_READY_SANITIZE_IN_PROGRESS:
+			case DEPOPULATION_IN_PROGRESS:
+			case DEPOPULATION_RESTORATION_IN_PROGRESS:
+				goto out;
+			default:
+				break;
+			}
+
 			/*
 			 * Issue command to spin up drive when not ready
 			 */
@@ -2608,8 +2610,9 @@ sd_spinup_disk(struct scsi_disk *sdkp)
 		 * occur here.  It's characteristic of these devices.
 		 */
 		} else if (sense_valid &&
-				sshdr.sense_key == UNIT_ATTENTION &&
-				sshdr.asc == 0x28) {
+			   sshdr.sense_key == UNIT_ATTENTION &&
+			   scsi_sense_asc(&sshdr) ==
+			   ASC_NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED) {
 			if (!spintime) {
 				spintime_expire = jiffies + 5 * HZ;
 				spintime = 1;
@@ -2628,6 +2631,7 @@ sd_spinup_disk(struct scsi_disk *sdkp)
 				
 	} while (spintime && time_before_eq(jiffies, spintime_expire));
 
+out:
 	if (spintime) {
 		if (scsi_status_is_good(the_result))
 			printk(KERN_CONT "ready\n");
@@ -2753,15 +2757,14 @@ static int read_capacity_16(struct scsi_disk *sdkp, struct scsi_device *sdp,
 			sense_valid = scsi_sense_valid(&sshdr);
 			if (sense_valid &&
 			    sshdr.sense_key == ILLEGAL_REQUEST &&
-			    (sshdr.asc == 0x20 || sshdr.asc == 0x24) &&
-			    sshdr.ascq == 0x00)
-				/* Invalid Command Operation Code or
-				 * Invalid Field in CDB, just retry
-				 * silently with RC10 */
+			    (sshdr.sense_code == INVALID_COMMAND_OP_CODE ||
+			     sshdr.sense_code == INVALID_FIELD_IN_CDB))
+				/* Just retry silently with RC10 */
 				return -EINVAL;
 			if (sense_valid &&
 			    sshdr.sense_key == UNIT_ATTENTION &&
-			    sshdr.asc == 0x29 && sshdr.ascq == 0x00)
+			    sshdr.sense_code ==
+			    POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED)
 				/* Device reset might occur several times,
 				 * give it one more chance */
 				if (--reset_retries > 0)
@@ -2817,19 +2820,19 @@ static int read_capacity_10(struct scsi_disk *sdkp, struct scsi_device *sdp,
 	struct scsi_failure failure_defs[] = {
 		/* Do not retry Medium Not Present */
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x3A,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = MEDIUM_NOT_PRESENT,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = NOT_READY,
-			.asc = 0x3A,
+			.sense_key = NOT_READY,
+			.sense_code = MEDIUM_NOT_PRESENT,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		 /* Device reset might occur several times so retry a lot */
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED,
 			.allowed = READ_CAPACITY_RETRIES_ON_RESET,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -3258,8 +3261,7 @@ sd_read_cache_type(struct scsi_disk *sdkp, unsigned char *buffer)
 bad_sense:
 	if (res == -EIO && scsi_sense_valid(&sshdr) &&
 	    sshdr.sense_key == ILLEGAL_REQUEST &&
-	    sshdr.asc == 0x24 && sshdr.ascq == 0x0)
-		/* Invalid field in CDB */
+	    sshdr.sense_code == INVALID_FIELD_IN_CDB)
 		sd_first_printk(KERN_NOTICE, sdkp, "Cache data unavailable\n");
 	else
 		sd_first_printk(KERN_ERR, sdkp,
@@ -4159,23 +4161,20 @@ static int sd_start_stop_device(struct scsi_disk *sdkp, int start)
 	struct scsi_failure failure_defs[] = {
 		{
 			/* Power on, reset, or bus device reset occurred */
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
-			.ascq = 0,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
 			/* Power on occurred */
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
-			.ascq = 1,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = POWER_ON_OCCURRED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
 			/* SCSI bus reset */
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
-			.ascq = 2,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = SCSI_BUS_RESET_OCCURRED,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{}
@@ -4207,8 +4206,7 @@ static int sd_start_stop_device(struct scsi_disk *sdkp, int start)
 		sd_print_result(sdkp, "Start/Stop Unit failed", res);
 		if (res > 0 && scsi_sense_valid(&sshdr)) {
 			sd_print_sense_hdr(sdkp, &sshdr);
-			/* 0x3a is medium not present */
-			if (sshdr.asc == 0x3a)
+			if (scsi_sense_asc(&sshdr) == ASC_MEDIUM_NOT_PRESENT)
 				res = 0;
 		}
 	}
diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c
index 56e455fb5addd3..8cbf171e6fabf1 100644
--- a/drivers/scsi/sd_zbc.c
+++ b/drivers/scsi/sd_zbc.c
@@ -369,7 +369,7 @@ unsigned int sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes,
 	if (op_is_zone_mgmt(req_op(rq)) &&
 	    result &&
 	    sshdr->sense_key == ILLEGAL_REQUEST &&
-	    sshdr->asc == 0x24) {
+	    scsi_sense_asc(sshdr) == ASC_INVALID_FIELD_IN_CDB) {
 		/*
 		 * INVALID FIELD IN CDB error: a zone management command was
 		 * attempted on a conventional zone. Nothing to worry about,
@@ -589,7 +589,7 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp)
 int sd_zbc_read_zones(struct scsi_disk *sdkp, struct queue_limits *lim,
 		u8 buf[SD_BUF_SIZE])
 {
-	unsigned int nr_zones;
+	u64 nr_zones;
 	u32 zone_blocks = 0;
 	int ret;
 
@@ -621,6 +621,12 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, struct queue_limits *lim,
 		goto err;
 
 	nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks);
+	if (nr_zones > INT_MAX) {
+		sd_printk(KERN_ERR, sdkp, "Too many zones (%llu)\n",
+			  nr_zones);
+		ret = -EINVAL;
+		goto err;
+	}
 	sdkp->early_zone_info.nr_zones = nr_zones;
 	sdkp->early_zone_info.zone_blocks = zone_blocks;
 
diff --git a/drivers/scsi/sense_codes.h b/drivers/scsi/sense_codes.h
index 805d4c13d070c5..c2eb1a9ec4c061 100644
--- a/drivers/scsi/sense_codes.h
+++ b/drivers/scsi/sense_codes.h
@@ -4,876 +4,1642 @@
  * http://www.t10.org/lists/asc-num.txt [most recent: 20200817]
  */
 
-SENSE_CODE(0x0000, "No additional sense information")
-SENSE_CODE(0x0001, "Filemark detected")
-SENSE_CODE(0x0002, "End-of-partition/medium detected")
-SENSE_CODE(0x0003, "Setmark detected")
-SENSE_CODE(0x0004, "Beginning-of-partition/medium detected")
-SENSE_CODE(0x0005, "End-of-data detected")
-SENSE_CODE(0x0006, "I/O process terminated")
-SENSE_CODE(0x0007, "Programmable early warning detected")
-SENSE_CODE(0x0011, "Audio play operation in progress")
-SENSE_CODE(0x0012, "Audio play operation paused")
-SENSE_CODE(0x0013, "Audio play operation successfully completed")
-SENSE_CODE(0x0014, "Audio play operation stopped due to error")
-SENSE_CODE(0x0015, "No current audio status to return")
-SENSE_CODE(0x0016, "Operation in progress")
-SENSE_CODE(0x0017, "Cleaning requested")
-SENSE_CODE(0x0018, "Erase operation in progress")
-SENSE_CODE(0x0019, "Locate operation in progress")
-SENSE_CODE(0x001A, "Rewind operation in progress")
-SENSE_CODE(0x001B, "Set capacity operation in progress")
-SENSE_CODE(0x001C, "Verify operation in progress")
-SENSE_CODE(0x001D, "ATA pass through information available")
-SENSE_CODE(0x001E, "Conflicting SA creation request")
-SENSE_CODE(0x001F, "Logical unit transitioning to another power condition")
-SENSE_CODE(0x0020, "Extended copy information available")
-SENSE_CODE(0x0021, "Atomic command aborted due to ACA")
-SENSE_CODE(0x0022, "Deferred microcode is pending")
-
-SENSE_CODE(0x0100, "No index/sector signal")
-
-SENSE_CODE(0x0200, "No seek complete")
-
-SENSE_CODE(0x0300, "Peripheral device write fault")
-SENSE_CODE(0x0301, "No write current")
-SENSE_CODE(0x0302, "Excessive write errors")
-
-SENSE_CODE(0x0400, "Logical unit not ready, cause not reportable")
-SENSE_CODE(0x0401, "Logical unit is in process of becoming ready")
-SENSE_CODE(0x0402, "Logical unit not ready, initializing command required")
-SENSE_CODE(0x0403, "Logical unit not ready, manual intervention required")
-SENSE_CODE(0x0404, "Logical unit not ready, format in progress")
-SENSE_CODE(0x0405, "Logical unit not ready, rebuild in progress")
-SENSE_CODE(0x0406, "Logical unit not ready, recalculation in progress")
-SENSE_CODE(0x0407, "Logical unit not ready, operation in progress")
-SENSE_CODE(0x0408, "Logical unit not ready, long write in progress")
-SENSE_CODE(0x0409, "Logical unit not ready, self-test in progress")
-SENSE_CODE(0x040A, "Logical unit not accessible, asymmetric access state transition")
-SENSE_CODE(0x040B, "Logical unit not accessible, target port in standby state")
-SENSE_CODE(0x040C, "Logical unit not accessible, target port in unavailable state")
-SENSE_CODE(0x040D, "Logical unit not ready, structure check required")
-SENSE_CODE(0x040E, "Logical unit not ready, security session in progress")
-SENSE_CODE(0x0410, "Logical unit not ready, auxiliary memory not accessible")
-SENSE_CODE(0x0411, "Logical unit not ready, notify (enable spinup) required")
-SENSE_CODE(0x0412, "Logical unit not ready, offline")
-SENSE_CODE(0x0413, "Logical unit not ready, SA creation in progress")
-SENSE_CODE(0x0414, "Logical unit not ready, space allocation in progress")
-SENSE_CODE(0x0415, "Logical unit not ready, robotics disabled")
-SENSE_CODE(0x0416, "Logical unit not ready, configuration required")
-SENSE_CODE(0x0417, "Logical unit not ready, calibration required")
-SENSE_CODE(0x0418, "Logical unit not ready, a door is open")
-SENSE_CODE(0x0419, "Logical unit not ready, operating in sequential mode")
-SENSE_CODE(0x041A, "Logical unit not ready, start stop unit command in progress")
-SENSE_CODE(0x041B, "Logical unit not ready, sanitize in progress")
-SENSE_CODE(0x041C, "Logical unit not ready, additional power use not yet granted")
-SENSE_CODE(0x041D, "Logical unit not ready, configuration in progress")
-SENSE_CODE(0x041E, "Logical unit not ready, microcode activation required")
-SENSE_CODE(0x041F, "Logical unit not ready, microcode download required")
-SENSE_CODE(0x0420, "Logical unit not ready, logical unit reset required")
-SENSE_CODE(0x0421, "Logical unit not ready, hard reset required")
-SENSE_CODE(0x0422, "Logical unit not ready, power cycle required")
-SENSE_CODE(0x0423, "Logical unit not ready, affiliation required")
-SENSE_CODE(0x0424, "Depopulation in progress")
-SENSE_CODE(0x0425, "Depopulation restoration in progress")
-
-SENSE_CODE(0x0500, "Logical unit does not respond to selection")
-
-SENSE_CODE(0x0600, "No reference position found")
-
-SENSE_CODE(0x0700, "Multiple peripheral devices selected")
-
-SENSE_CODE(0x0800, "Logical unit communication failure")
-SENSE_CODE(0x0801, "Logical unit communication time-out")
-SENSE_CODE(0x0802, "Logical unit communication parity error")
-SENSE_CODE(0x0803, "Logical unit communication CRC error (Ultra-DMA/32)")
-SENSE_CODE(0x0804, "Unreachable copy target")
-
-SENSE_CODE(0x0900, "Track following error")
-SENSE_CODE(0x0901, "Tracking servo failure")
-SENSE_CODE(0x0902, "Focus servo failure")
-SENSE_CODE(0x0903, "Spindle servo failure")
-SENSE_CODE(0x0904, "Head select fault")
-SENSE_CODE(0x0905, "Vibration induced tracking error")
-
-SENSE_CODE(0x0A00, "Error log overflow")
-
-SENSE_CODE(0x0B00, "Warning")
-SENSE_CODE(0x0B01, "Warning - specified temperature exceeded")
-SENSE_CODE(0x0B02, "Warning - enclosure degraded")
-SENSE_CODE(0x0B03, "Warning - background self-test failed")
-SENSE_CODE(0x0B04, "Warning - background pre-scan detected medium error")
-SENSE_CODE(0x0B05, "Warning - background medium scan detected medium error")
-SENSE_CODE(0x0B06, "Warning - non-volatile cache now volatile")
-SENSE_CODE(0x0B07, "Warning - degraded power to non-volatile cache")
-SENSE_CODE(0x0B08, "Warning - power loss expected")
-SENSE_CODE(0x0B09, "Warning - device statistics notification active")
-SENSE_CODE(0x0B0A, "Warning - high critical temperature limit exceeded")
-SENSE_CODE(0x0B0B, "Warning - low critical temperature limit exceeded")
-SENSE_CODE(0x0B0C, "Warning - high operating temperature limit exceeded")
-SENSE_CODE(0x0B0D, "Warning - low operating temperature limit exceeded")
-SENSE_CODE(0x0B0E, "Warning - high critical humidity limit exceeded")
-SENSE_CODE(0x0B0F, "Warning - low critical humidity limit exceeded")
-SENSE_CODE(0x0B10, "Warning - high operating humidity limit exceeded")
-SENSE_CODE(0x0B11, "Warning - low operating humidity limit exceeded")
-SENSE_CODE(0x0B12, "Warning - microcode security at risk")
-SENSE_CODE(0x0B13, "Warning - microcode digital signature validation failure")
-SENSE_CODE(0x0B14, "Warning - physical element status change")
-
-SENSE_CODE(0x0C00, "Write error")
-SENSE_CODE(0x0C01, "Write error - recovered with auto reallocation")
-SENSE_CODE(0x0C02, "Write error - auto reallocation failed")
-SENSE_CODE(0x0C03, "Write error - recommend reassignment")
-SENSE_CODE(0x0C04, "Compression check miscompare error")
-SENSE_CODE(0x0C05, "Data expansion occurred during compression")
-SENSE_CODE(0x0C06, "Block not compressible")
-SENSE_CODE(0x0C07, "Write error - recovery needed")
-SENSE_CODE(0x0C08, "Write error - recovery failed")
-SENSE_CODE(0x0C09, "Write error - loss of streaming")
-SENSE_CODE(0x0C0A, "Write error - padding blocks added")
-SENSE_CODE(0x0C0B, "Auxiliary memory write error")
-SENSE_CODE(0x0C0C, "Write error - unexpected unsolicited data")
-SENSE_CODE(0x0C0D, "Write error - not enough unsolicited data")
-SENSE_CODE(0x0C0E, "Multiple write errors")
-SENSE_CODE(0x0C0F, "Defects in error window")
-SENSE_CODE(0x0C10, "Incomplete multiple atomic write operations")
-SENSE_CODE(0x0C11, "Write error - recovery scan needed")
-SENSE_CODE(0x0C12, "Write error - insufficient zone resources")
-
-SENSE_CODE(0x0D00, "Error detected by third party temporary initiator")
-SENSE_CODE(0x0D01, "Third party device failure")
-SENSE_CODE(0x0D02, "Copy target device not reachable")
-SENSE_CODE(0x0D03, "Incorrect copy target device type")
-SENSE_CODE(0x0D04, "Copy target device data underrun")
-SENSE_CODE(0x0D05, "Copy target device data overrun")
-
-SENSE_CODE(0x0E00, "Invalid information unit")
-SENSE_CODE(0x0E01, "Information unit too short")
-SENSE_CODE(0x0E02, "Information unit too long")
-SENSE_CODE(0x0E03, "Invalid field in command information unit")
-
-SENSE_CODE(0x1000, "Id CRC or ECC error")
-SENSE_CODE(0x1001, "Logical block guard check failed")
-SENSE_CODE(0x1002, "Logical block application tag check failed")
-SENSE_CODE(0x1003, "Logical block reference tag check failed")
-SENSE_CODE(0x1004, "Logical block protection error on recover buffered data")
-SENSE_CODE(0x1005, "Logical block protection method error")
-
-SENSE_CODE(0x1100, "Unrecovered read error")
-SENSE_CODE(0x1101, "Read retries exhausted")
-SENSE_CODE(0x1102, "Error too long to correct")
-SENSE_CODE(0x1103, "Multiple read errors")
-SENSE_CODE(0x1104, "Unrecovered read error - auto reallocate failed")
-SENSE_CODE(0x1105, "L-EC uncorrectable error")
-SENSE_CODE(0x1106, "CIRC unrecovered error")
-SENSE_CODE(0x1107, "Data re-synchronization error")
-SENSE_CODE(0x1108, "Incomplete block read")
-SENSE_CODE(0x1109, "No gap found")
-SENSE_CODE(0x110A, "Miscorrected error")
-SENSE_CODE(0x110B, "Unrecovered read error - recommend reassignment")
-SENSE_CODE(0x110C, "Unrecovered read error - recommend rewrite the data")
-SENSE_CODE(0x110D, "De-compression CRC error")
-SENSE_CODE(0x110E, "Cannot decompress using declared algorithm")
-SENSE_CODE(0x110F, "Error reading UPC/EAN number")
-SENSE_CODE(0x1110, "Error reading ISRC number")
-SENSE_CODE(0x1111, "Read error - loss of streaming")
-SENSE_CODE(0x1112, "Auxiliary memory read error")
-SENSE_CODE(0x1113, "Read error - failed retransmission request")
-SENSE_CODE(0x1114, "Read error - lba marked bad by application client")
-SENSE_CODE(0x1115, "Write after sanitize required")
-
-SENSE_CODE(0x1200, "Address mark not found for id field")
-
-SENSE_CODE(0x1300, "Address mark not found for data field")
-
-SENSE_CODE(0x1400, "Recorded entity not found")
-SENSE_CODE(0x1401, "Record not found")
-SENSE_CODE(0x1402, "Filemark or setmark not found")
-SENSE_CODE(0x1403, "End-of-data not found")
-SENSE_CODE(0x1404, "Block sequence error")
-SENSE_CODE(0x1405, "Record not found - recommend reassignment")
-SENSE_CODE(0x1406, "Record not found - data auto-reallocated")
-SENSE_CODE(0x1407, "Locate operation failure")
-
-SENSE_CODE(0x1500, "Random positioning error")
-SENSE_CODE(0x1501, "Mechanical positioning error")
-SENSE_CODE(0x1502, "Positioning error detected by read of medium")
-
-SENSE_CODE(0x1600, "Data synchronization mark error")
-SENSE_CODE(0x1601, "Data sync error - data rewritten")
-SENSE_CODE(0x1602, "Data sync error - recommend rewrite")
-SENSE_CODE(0x1603, "Data sync error - data auto-reallocated")
-SENSE_CODE(0x1604, "Data sync error - recommend reassignment")
-
-SENSE_CODE(0x1700, "Recovered data with no error correction applied")
-SENSE_CODE(0x1701, "Recovered data with retries")
-SENSE_CODE(0x1702, "Recovered data with positive head offset")
-SENSE_CODE(0x1703, "Recovered data with negative head offset")
-SENSE_CODE(0x1704, "Recovered data with retries and/or circ applied")
-SENSE_CODE(0x1705, "Recovered data using previous sector id")
-SENSE_CODE(0x1706, "Recovered data without ECC - data auto-reallocated")
-SENSE_CODE(0x1707, "Recovered data without ECC - recommend reassignment")
-SENSE_CODE(0x1708, "Recovered data without ECC - recommend rewrite")
-SENSE_CODE(0x1709, "Recovered data without ECC - data rewritten")
-
-SENSE_CODE(0x1800, "Recovered data with error correction applied")
-SENSE_CODE(0x1801, "Recovered data with error corr. & retries applied")
-SENSE_CODE(0x1802, "Recovered data - data auto-reallocated")
-SENSE_CODE(0x1803, "Recovered data with CIRC")
-SENSE_CODE(0x1804, "Recovered data with L-EC")
-SENSE_CODE(0x1805, "Recovered data - recommend reassignment")
-SENSE_CODE(0x1806, "Recovered data - recommend rewrite")
-SENSE_CODE(0x1807, "Recovered data with ECC - data rewritten")
-SENSE_CODE(0x1808, "Recovered data with linking")
-
-SENSE_CODE(0x1900, "Defect list error")
-SENSE_CODE(0x1901, "Defect list not available")
-SENSE_CODE(0x1902, "Defect list error in primary list")
-SENSE_CODE(0x1903, "Defect list error in grown list")
-
-SENSE_CODE(0x1A00, "Parameter list length error")
-
-SENSE_CODE(0x1B00, "Synchronous data transfer error")
-
-SENSE_CODE(0x1C00, "Defect list not found")
-SENSE_CODE(0x1C01, "Primary defect list not found")
-SENSE_CODE(0x1C02, "Grown defect list not found")
-
-SENSE_CODE(0x1D00, "Miscompare during verify operation")
-SENSE_CODE(0x1D01, "Miscompare verify of unmapped LBA")
-
-SENSE_CODE(0x1E00, "Recovered id with ECC correction")
-
-SENSE_CODE(0x1F00, "Partial defect list transfer")
-
-SENSE_CODE(0x2000, "Invalid command operation code")
-SENSE_CODE(0x2001, "Access denied - initiator pending-enrolled")
-SENSE_CODE(0x2002, "Access denied - no access rights")
-SENSE_CODE(0x2003, "Access denied - invalid mgmt id key")
-SENSE_CODE(0x2004, "Illegal command while in write capable state")
-SENSE_CODE(0x2005, "Obsolete")
-SENSE_CODE(0x2006, "Illegal command while in explicit address mode")
-SENSE_CODE(0x2007, "Illegal command while in implicit address mode")
-SENSE_CODE(0x2008, "Access denied - enrollment conflict")
-SENSE_CODE(0x2009, "Access denied - invalid LU identifier")
-SENSE_CODE(0x200A, "Access denied - invalid proxy token")
-SENSE_CODE(0x200B, "Access denied - ACL LUN conflict")
-SENSE_CODE(0x200C, "Illegal command when not in append-only mode")
-SENSE_CODE(0x200D, "Not an administrative logical unit")
-SENSE_CODE(0x200E, "Not a subsidiary logical unit")
-SENSE_CODE(0x200F, "Not a conglomerate logical unit")
-
-SENSE_CODE(0x2100, "Logical block address out of range")
-SENSE_CODE(0x2101, "Invalid element address")
-SENSE_CODE(0x2102, "Invalid address for write")
-SENSE_CODE(0x2103, "Invalid write crossing layer jump")
-SENSE_CODE(0x2104, "Unaligned write command")
-SENSE_CODE(0x2105, "Write boundary violation")
-SENSE_CODE(0x2106, "Attempt to read invalid data")
-SENSE_CODE(0x2107, "Read boundary violation")
-SENSE_CODE(0x2108, "Misaligned write command")
-SENSE_CODE(0x2109, "Attempt to access gap zone")
-
-SENSE_CODE(0x2200, "Illegal function (use 20 00, 24 00, or 26 00)")
-
-SENSE_CODE(0x2300, "Invalid token operation, cause not reportable")
-SENSE_CODE(0x2301, "Invalid token operation, unsupported token type")
-SENSE_CODE(0x2302, "Invalid token operation, remote token usage not supported")
-SENSE_CODE(0x2303, "Invalid token operation, remote rod token creation not supported")
-SENSE_CODE(0x2304, "Invalid token operation, token unknown")
-SENSE_CODE(0x2305, "Invalid token operation, token corrupt")
-SENSE_CODE(0x2306, "Invalid token operation, token revoked")
-SENSE_CODE(0x2307, "Invalid token operation, token expired")
-SENSE_CODE(0x2308, "Invalid token operation, token cancelled")
-SENSE_CODE(0x2309, "Invalid token operation, token deleted")
-SENSE_CODE(0x230A, "Invalid token operation, invalid token length")
-
-SENSE_CODE(0x2400, "Invalid field in cdb")
-SENSE_CODE(0x2401, "CDB decryption error")
-SENSE_CODE(0x2402, "Obsolete")
-SENSE_CODE(0x2403, "Obsolete")
-SENSE_CODE(0x2404, "Security audit value frozen")
-SENSE_CODE(0x2405, "Security working key frozen")
-SENSE_CODE(0x2406, "Nonce not unique")
-SENSE_CODE(0x2407, "Nonce timestamp out of range")
-SENSE_CODE(0x2408, "Invalid XCDB")
-SENSE_CODE(0x2409, "Invalid fast format")
-
-SENSE_CODE(0x2500, "Logical unit not supported")
-
-SENSE_CODE(0x2600, "Invalid field in parameter list")
-SENSE_CODE(0x2601, "Parameter not supported")
-SENSE_CODE(0x2602, "Parameter value invalid")
-SENSE_CODE(0x2603, "Threshold parameters not supported")
-SENSE_CODE(0x2604, "Invalid release of persistent reservation")
-SENSE_CODE(0x2605, "Data decryption error")
-SENSE_CODE(0x2606, "Too many target descriptors")
-SENSE_CODE(0x2607, "Unsupported target descriptor type code")
-SENSE_CODE(0x2608, "Too many segment descriptors")
-SENSE_CODE(0x2609, "Unsupported segment descriptor type code")
-SENSE_CODE(0x260A, "Unexpected inexact segment")
-SENSE_CODE(0x260B, "Inline data length exceeded")
-SENSE_CODE(0x260C, "Invalid operation for copy source or destination")
-SENSE_CODE(0x260D, "Copy segment granularity violation")
-SENSE_CODE(0x260E, "Invalid parameter while port is enabled")
-SENSE_CODE(0x260F, "Invalid data-out buffer integrity check value")
-SENSE_CODE(0x2610, "Data decryption key fail limit reached")
-SENSE_CODE(0x2611, "Incomplete key-associated data set")
-SENSE_CODE(0x2612, "Vendor specific key reference not found")
-SENSE_CODE(0x2613, "Application tag mode page is invalid")
-SENSE_CODE(0x2614, "Tape stream mirroring prevented")
-SENSE_CODE(0x2615, "Copy source or copy destination not authorized")
-SENSE_CODE(0x2616, "Fast copy not possible")
-
-SENSE_CODE(0x2700, "Write protected")
-SENSE_CODE(0x2701, "Hardware write protected")
-SENSE_CODE(0x2702, "Logical unit software write protected")
-SENSE_CODE(0x2703, "Associated write protect")
-SENSE_CODE(0x2704, "Persistent write protect")
-SENSE_CODE(0x2705, "Permanent write protect")
-SENSE_CODE(0x2706, "Conditional write protect")
-SENSE_CODE(0x2707, "Space allocation failed write protect")
-SENSE_CODE(0x2708, "Zone is read only")
-
-SENSE_CODE(0x2800, "Not ready to ready change, medium may have changed")
-SENSE_CODE(0x2801, "Import or export element accessed")
-SENSE_CODE(0x2802, "Format-layer may have changed")
-SENSE_CODE(0x2803, "Import/export element accessed, medium changed")
-
-SENSE_CODE(0x2900, "Power on, reset, or bus device reset occurred")
-SENSE_CODE(0x2901, "Power on occurred")
-SENSE_CODE(0x2902, "Scsi bus reset occurred")
-SENSE_CODE(0x2903, "Bus device reset function occurred")
-SENSE_CODE(0x2904, "Device internal reset")
-SENSE_CODE(0x2905, "Transceiver mode changed to single-ended")
-SENSE_CODE(0x2906, "Transceiver mode changed to lvd")
-SENSE_CODE(0x2907, "I_T nexus loss occurred")
-
-SENSE_CODE(0x2A00, "Parameters changed")
-SENSE_CODE(0x2A01, "Mode parameters changed")
-SENSE_CODE(0x2A02, "Log parameters changed")
-SENSE_CODE(0x2A03, "Reservations preempted")
-SENSE_CODE(0x2A04, "Reservations released")
-SENSE_CODE(0x2A05, "Registrations preempted")
-SENSE_CODE(0x2A06, "Asymmetric access state changed")
-SENSE_CODE(0x2A07, "Implicit asymmetric access state transition failed")
-SENSE_CODE(0x2A08, "Priority changed")
-SENSE_CODE(0x2A09, "Capacity data has changed")
-SENSE_CODE(0x2A0A, "Error history I_T nexus cleared")
-SENSE_CODE(0x2A0B, "Error history snapshot released")
-SENSE_CODE(0x2A0C, "Error recovery attributes have changed")
-SENSE_CODE(0x2A0D, "Data encryption capabilities changed")
-SENSE_CODE(0x2A10, "Timestamp changed")
-SENSE_CODE(0x2A11, "Data encryption parameters changed by another i_t nexus")
-SENSE_CODE(0x2A12, "Data encryption parameters changed by vendor specific event")
-SENSE_CODE(0x2A13, "Data encryption key instance counter has changed")
-SENSE_CODE(0x2A14, "SA creation capabilities data has changed")
-SENSE_CODE(0x2A15, "Medium removal prevention preempted")
-SENSE_CODE(0x2A16, "Zone reset write pointer recommended")
-
-SENSE_CODE(0x2B00, "Copy cannot execute since host cannot disconnect")
-
-SENSE_CODE(0x2C00, "Command sequence error")
-SENSE_CODE(0x2C01, "Too many windows specified")
-SENSE_CODE(0x2C02, "Invalid combination of windows specified")
-SENSE_CODE(0x2C03, "Current program area is not empty")
-SENSE_CODE(0x2C04, "Current program area is empty")
-SENSE_CODE(0x2C05, "Illegal power condition request")
-SENSE_CODE(0x2C06, "Persistent prevent conflict")
-SENSE_CODE(0x2C07, "Previous busy status")
-SENSE_CODE(0x2C08, "Previous task set full status")
-SENSE_CODE(0x2C09, "Previous reservation conflict status")
-SENSE_CODE(0x2C0A, "Partition or collection contains user objects")
-SENSE_CODE(0x2C0B, "Not reserved")
-SENSE_CODE(0x2C0C, "Orwrite generation does not match")
-SENSE_CODE(0x2C0D, "Reset write pointer not allowed")
-SENSE_CODE(0x2C0E, "Zone is offline")
-SENSE_CODE(0x2C0F, "Stream not open")
-SENSE_CODE(0x2C10, "Unwritten data in zone")
-SENSE_CODE(0x2C11, "Descriptor format sense data required")
-SENSE_CODE(0x2C12, "Zone is inactive")
-SENSE_CODE(0x2C13, "Well known logical unit access required")
-
-SENSE_CODE(0x2D00, "Overwrite error on update in place")
-
-SENSE_CODE(0x2E00, "Insufficient time for operation")
-SENSE_CODE(0x2E01, "Command timeout before processing")
-SENSE_CODE(0x2E02, "Command timeout during processing")
-SENSE_CODE(0x2E03, "Command timeout during processing due to error recovery")
-
-SENSE_CODE(0x2F00, "Commands cleared by another initiator")
-SENSE_CODE(0x2F01, "Commands cleared by power loss notification")
-SENSE_CODE(0x2F02, "Commands cleared by device server")
-SENSE_CODE(0x2F03, "Some commands cleared by queuing layer event")
-
-SENSE_CODE(0x3000, "Incompatible medium installed")
-SENSE_CODE(0x3001, "Cannot read medium - unknown format")
-SENSE_CODE(0x3002, "Cannot read medium - incompatible format")
-SENSE_CODE(0x3003, "Cleaning cartridge installed")
-SENSE_CODE(0x3004, "Cannot write medium - unknown format")
-SENSE_CODE(0x3005, "Cannot write medium - incompatible format")
-SENSE_CODE(0x3006, "Cannot format medium - incompatible medium")
-SENSE_CODE(0x3007, "Cleaning failure")
-SENSE_CODE(0x3008, "Cannot write - application code mismatch")
-SENSE_CODE(0x3009, "Current session not fixated for append")
-SENSE_CODE(0x300A, "Cleaning request rejected")
-SENSE_CODE(0x300C, "WORM medium - overwrite attempted")
-SENSE_CODE(0x300D, "WORM medium - integrity check")
-SENSE_CODE(0x3010, "Medium not formatted")
-SENSE_CODE(0x3011, "Incompatible volume type")
-SENSE_CODE(0x3012, "Incompatible volume qualifier")
-SENSE_CODE(0x3013, "Cleaning volume expired")
-
-SENSE_CODE(0x3100, "Medium format corrupted")
-SENSE_CODE(0x3101, "Format command failed")
-SENSE_CODE(0x3102, "Zoned formatting failed due to spare linking")
-SENSE_CODE(0x3103, "Sanitize command failed")
-SENSE_CODE(0x3104, "Depopulation failed")
-SENSE_CODE(0x3105, "Depopulation restoration failed")
-
-SENSE_CODE(0x3200, "No defect spare location available")
-SENSE_CODE(0x3201, "Defect list update failure")
-
-SENSE_CODE(0x3300, "Tape length error")
-
-SENSE_CODE(0x3400, "Enclosure failure")
-
-SENSE_CODE(0x3500, "Enclosure services failure")
-SENSE_CODE(0x3501, "Unsupported enclosure function")
-SENSE_CODE(0x3502, "Enclosure services unavailable")
-SENSE_CODE(0x3503, "Enclosure services transfer failure")
-SENSE_CODE(0x3504, "Enclosure services transfer refused")
-SENSE_CODE(0x3505, "Enclosure services checksum error")
-
-SENSE_CODE(0x3600, "Ribbon, ink, or toner failure")
-
-SENSE_CODE(0x3700, "Rounded parameter")
-
-SENSE_CODE(0x3800, "Event status notification")
-SENSE_CODE(0x3802, "Esn - power management class event")
-SENSE_CODE(0x3804, "Esn - media class event")
-SENSE_CODE(0x3806, "Esn - device busy class event")
-SENSE_CODE(0x3807, "Thin Provisioning soft threshold reached")
-SENSE_CODE(0x3808, "Depopulation interrupted")
-
-SENSE_CODE(0x3900, "Saving parameters not supported")
-
-SENSE_CODE(0x3A00, "Medium not present")
-SENSE_CODE(0x3A01, "Medium not present - tray closed")
-SENSE_CODE(0x3A02, "Medium not present - tray open")
-SENSE_CODE(0x3A03, "Medium not present - loadable")
-SENSE_CODE(0x3A04, "Medium not present - medium auxiliary memory accessible")
-
-SENSE_CODE(0x3B00, "Sequential positioning error")
-SENSE_CODE(0x3B01, "Tape position error at beginning-of-medium")
-SENSE_CODE(0x3B02, "Tape position error at end-of-medium")
-SENSE_CODE(0x3B03, "Tape or electronic vertical forms unit not ready")
-SENSE_CODE(0x3B04, "Slew failure")
-SENSE_CODE(0x3B05, "Paper jam")
-SENSE_CODE(0x3B06, "Failed to sense top-of-form")
-SENSE_CODE(0x3B07, "Failed to sense bottom-of-form")
-SENSE_CODE(0x3B08, "Reposition error")
-SENSE_CODE(0x3B09, "Read past end of medium")
-SENSE_CODE(0x3B0A, "Read past beginning of medium")
-SENSE_CODE(0x3B0B, "Position past end of medium")
-SENSE_CODE(0x3B0C, "Position past beginning of medium")
-SENSE_CODE(0x3B0D, "Medium destination element full")
-SENSE_CODE(0x3B0E, "Medium source element empty")
-SENSE_CODE(0x3B0F, "End of medium reached")
-SENSE_CODE(0x3B11, "Medium magazine not accessible")
-SENSE_CODE(0x3B12, "Medium magazine removed")
-SENSE_CODE(0x3B13, "Medium magazine inserted")
-SENSE_CODE(0x3B14, "Medium magazine locked")
-SENSE_CODE(0x3B15, "Medium magazine unlocked")
-SENSE_CODE(0x3B16, "Mechanical positioning or changer error")
-SENSE_CODE(0x3B17, "Read past end of user object")
-SENSE_CODE(0x3B18, "Element disabled")
-SENSE_CODE(0x3B19, "Element enabled")
-SENSE_CODE(0x3B1A, "Data transfer device removed")
-SENSE_CODE(0x3B1B, "Data transfer device inserted")
-SENSE_CODE(0x3B1C, "Too many logical objects on partition to support operation")
-SENSE_CODE(0x3B20, "Element static information changed")
-
-SENSE_CODE(0x3D00, "Invalid bits in identify message")
-
-SENSE_CODE(0x3E00, "Logical unit has not self-configured yet")
-SENSE_CODE(0x3E01, "Logical unit failure")
-SENSE_CODE(0x3E02, "Timeout on logical unit")
-SENSE_CODE(0x3E03, "Logical unit failed self-test")
-SENSE_CODE(0x3E04, "Logical unit unable to update self-test log")
-
-SENSE_CODE(0x3F00, "Target operating conditions have changed")
-SENSE_CODE(0x3F01, "Microcode has been changed")
-SENSE_CODE(0x3F02, "Changed operating definition")
-SENSE_CODE(0x3F03, "Inquiry data has changed")
-SENSE_CODE(0x3F04, "Component device attached")
-SENSE_CODE(0x3F05, "Device identifier changed")
-SENSE_CODE(0x3F06, "Redundancy group created or modified")
-SENSE_CODE(0x3F07, "Redundancy group deleted")
-SENSE_CODE(0x3F08, "Spare created or modified")
-SENSE_CODE(0x3F09, "Spare deleted")
-SENSE_CODE(0x3F0A, "Volume set created or modified")
-SENSE_CODE(0x3F0B, "Volume set deleted")
-SENSE_CODE(0x3F0C, "Volume set deassigned")
-SENSE_CODE(0x3F0D, "Volume set reassigned")
-SENSE_CODE(0x3F0E, "Reported luns data has changed")
-SENSE_CODE(0x3F0F, "Echo buffer overwritten")
-SENSE_CODE(0x3F10, "Medium loadable")
-SENSE_CODE(0x3F11, "Medium auxiliary memory accessible")
-SENSE_CODE(0x3F12, "iSCSI IP address added")
-SENSE_CODE(0x3F13, "iSCSI IP address removed")
-SENSE_CODE(0x3F14, "iSCSI IP address changed")
-SENSE_CODE(0x3F15, "Inspect referrals sense descriptors")
-SENSE_CODE(0x3F16, "Microcode has been changed without reset")
-SENSE_CODE(0x3F17, "Zone transition to full")
-SENSE_CODE(0x3F18, "Bind completed")
-SENSE_CODE(0x3F19, "Bind redirected")
-SENSE_CODE(0x3F1A, "Subsidiary binding changed")
+SENSE_CODE(NO_ADDITIONAL_SENSE_INFORMATION,
+	   "No additional sense information")
+SENSE_CODE(FILEMARK_DETECTED,
+	   "Filemark detected")
+SENSE_CODE(END_OF_PARTITION_MEDIUM_DETECTED,
+	   "End-of-partition/medium detected")
+SENSE_CODE(SETMARK_DETECTED,
+	   "Setmark detected")
+SENSE_CODE(BEGINNING_OF_PARTITION_MEDIUM_DETECTED,
+	   "Beginning-of-partition/medium detected")
+SENSE_CODE(END_OF_DATA_DETECTED,
+	   "End-of-data detected")
+SENSE_CODE(IO_PROCESS_TERMINATED,
+	   "I/O process terminated")
+SENSE_CODE(PROGRAMMABLE_EARLY_WARNING_DETECTED,
+	   "Programmable early warning detected")
+SENSE_CODE(AUDIO_PLAY_OP_IN_PROGRESS,
+	   "Audio play operation in progress")
+SENSE_CODE(AUDIO_PLAY_OP_PAUSED,
+	   "Audio play operation paused")
+SENSE_CODE(AUDIO_PLAY_OP_SUCCESSFULLY_COMPLETED,
+	   "Audio play operation successfully completed")
+SENSE_CODE(AUDIO_PLAY_OP_STOPPED_DUE_TO_ERROR,
+	   "Audio play operation stopped due to error")
+SENSE_CODE(NO_CURRENT_AUDIO_STATUS_TO_RETURN,
+	   "No current audio status to return")
+SENSE_CODE(OP_IN_PROGRESS,
+	   "Operation in progress")
+SENSE_CODE(CLEANING_REQUESTED,
+	   "Cleaning requested")
+SENSE_CODE(ERASE_OP_IN_PROGRESS,
+	   "Erase operation in progress")
+SENSE_CODE(LOCATE_OP_IN_PROGRESS,
+	   "Locate operation in progress")
+SENSE_CODE(REWIND_OP_IN_PROGRESS,
+	   "Rewind operation in progress")
+SENSE_CODE(SET_CAPACITY_OP_IN_PROGRESS,
+	   "Set capacity operation in progress")
+SENSE_CODE(VERIFY_OP_IN_PROGRESS,
+	   "Verify operation in progress")
+SENSE_CODE(ATA_PASS_THROUGH_INFORMATION_AVAILABLE,
+	   "ATA pass through information available")
+SENSE_CODE(CONFLICTING_SA_CREATION_REQUEST,
+	   "Conflicting SA creation request")
+SENSE_CODE(LU_TRANSITIONING_TO_ANOTHER_POWER_CONDITION,
+	   "Logical unit transitioning to another power condition")
+SENSE_CODE(EXTENDED_COPY_INFORMATION_AVAILABLE,
+	   "Extended copy information available")
+SENSE_CODE(ATOMIC_COMMAND_ABORTED_DUE_TO_ACA,
+	   "Atomic command aborted due to ACA")
+SENSE_CODE(DEFERRED_MICROCODE_IS_PENDING,
+	   "Deferred microcode is pending")
+SENSE_CODE(OVERLAPPING_ATOMIC_COMMAND_IN_PROGRESS,
+	   "Overlapping atomic command in progress")
+
+SENSE_CODE(NO_INDEX_SECTOR_SIGNAL,
+	   "No index/sector signal")
+
+SENSE_CODE(NO_SEEK_COMPLETE,
+	   "No seek complete")
+
+SENSE_CODE(PERIPHERAL_DEVICE_WRITE_FAULT,
+	   "Peripheral device write fault")
+SENSE_CODE(NO_WRITE_CURRENT,
+	   "No write current")
+SENSE_CODE(EXCESSIVE_WRITE_ERRORS,
+	   "Excessive write errors")
+
+SENSE_CODE(LU_NOT_READY,
+	   "Logical unit not ready, cause not reportable")
+SENSE_CODE(LU_IS_IN_PROCESS_OF_BECOMING_READY,
+	   "Logical unit is in process of becoming ready")
+SENSE_CODE(LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED,
+	   "Logical unit not ready, initializing command required")
+SENSE_CODE(LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED,
+	   "Logical unit not ready, manual intervention required")
+SENSE_CODE(LU_NOT_READY_FORMAT_IN_PROGRESS,
+	   "Logical unit not ready, format in progress")
+SENSE_CODE(LU_NOT_READY_REBUILD_IN_PROGRESS,
+	   "Logical unit not ready, rebuild in progress")
+SENSE_CODE(LU_NOT_READY_RECALCULATION_IN_PROGRESS,
+	   "Logical unit not ready, recalculation in progress")
+SENSE_CODE(LU_NOT_READY_OP_IN_PROGRESS,
+	   "Logical unit not ready, operation in progress")
+SENSE_CODE(LU_NOT_READY_LONG_WRITE_IN_PROGRESS,
+	   "Logical unit not ready, long write in progress")
+SENSE_CODE(LU_NOT_READY_SELFTEST_IN_PROGRESS,
+	   "Logical unit not ready, self-test in progress")
+SENSE_CODE(LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION,
+	   "Logical unit not accessible, asymmetric access state transition")
+SENSE_CODE(LU_NOT_ACCESSIBLE_TARGET_PORT_IN_STANDBY_STATE,
+	   "Logical unit not accessible, target port in standby state")
+SENSE_CODE(LU_NOT_ACCESSIBLE_TARGET_PORT_IN_UNAVAILABLE_STATE,
+	   "Logical unit not accessible, target port in unavailable state")
+SENSE_CODE(LU_NOT_READY_STRUCTURE_CHECK_REQUIRED,
+	   "Logical unit not ready, structure check required")
+SENSE_CODE(LU_NOT_READY_SECURITY_SESSION_IN_PROGRESS,
+	   "Logical unit not ready, security session in progress")
+SENSE_CODE(LU_NOT_READY_AUXILIARY_MEMORY_NOT_ACCESSIBLE,
+	   "Logical unit not ready, auxiliary memory not accessible")
+SENSE_CODE(LU_NOT_READY_NOTIFY_REQUIRED,
+	   "Logical unit not ready, notify (enable spinup) required")
+SENSE_CODE(LU_NOT_READY_OFFLINE,
+	   "Logical unit not ready, offline")
+SENSE_CODE(LU_NOT_READY_SA_CREATION_IN_PROGRESS,
+	   "Logical unit not ready, SA creation in progress")
+SENSE_CODE(LU_NOT_READY_SPACE_ALLOCATION_IN_PROGRESS,
+	   "Logical unit not ready, space allocation in progress")
+SENSE_CODE(LU_NOT_READY_ROBOTICS_DISABLED,
+	   "Logical unit not ready, robotics disabled")
+SENSE_CODE(LU_NOT_READY_CONFIG_REQUIRED,
+	   "Logical unit not ready, configuration required")
+SENSE_CODE(LU_NOT_READY_CALIBRATION_REQUIRED,
+	   "Logical unit not ready, calibration required")
+SENSE_CODE(LU_NOT_READY_A_DOOR_IS_OPEN,
+	   "Logical unit not ready, a door is open")
+SENSE_CODE(LU_NOT_READY_OPERATING_IN_SEQUENTIAL_MODE,
+	   "Logical unit not ready, operating in sequential mode")
+SENSE_CODE(LU_NOT_READY_START_STOP_UNIT_COMMAND_IN_PROGRESS,
+	   "Logical unit not ready, start stop unit command in progress")
+SENSE_CODE(LU_NOT_READY_SANITIZE_IN_PROGRESS,
+	   "Logical unit not ready, sanitize in progress")
+SENSE_CODE(LU_NOT_READY_ADDITIONAL_POWER_USE_NOT_YET_GRANTED,
+	   "Logical unit not ready, additional power use not yet granted")
+SENSE_CODE(LU_NOT_READY_CONFIG_IN_PROGRESS,
+	   "Logical unit not ready, configuration in progress")
+SENSE_CODE(LU_NOT_READY_MICROCODE_ACTIVATION_REQUIRED,
+	   "Logical unit not ready, microcode activation required")
+SENSE_CODE(LU_NOT_READY_MICROCODE_DOWNLOAD_REQUIRED,
+	   "Logical unit not ready, microcode download required")
+SENSE_CODE(LU_NOT_READY_LU_RESET_REQUIRED,
+	   "Logical unit not ready, logical unit reset required")
+SENSE_CODE(LU_NOT_READY_HARD_RESET_REQUIRED,
+	   "Logical unit not ready, hard reset required")
+SENSE_CODE(LU_NOT_READY_POWER_CYCLE_REQUIRED,
+	   "Logical unit not ready, power cycle required")
+SENSE_CODE(LU_NOT_READY_AFFILIATION_REQUIRED,
+	   "Logical unit not ready, affiliation required")
+SENSE_CODE(DEPOPULATION_IN_PROGRESS,
+	   "Depopulation in progress")
+SENSE_CODE(DEPOPULATION_RESTORATION_IN_PROGRESS,
+	   "Depopulation restoration in progress")
+
+SENSE_CODE(LU_DOES_NOT_RESPOND_TO_SELECTION,
+	   "Logical unit does not respond to selection")
+
+SENSE_CODE(NO_REFERENCE_POSITION_FOUND,
+	   "No reference position found")
+
+SENSE_CODE(MULTIPLE_PERIPHERAL_DEVICES_SELECTED,
+	   "Multiple peripheral devices selected")
+
+SENSE_CODE(LU_COMMUNICATION_FAILURE,
+	   "Logical unit communication failure")
+SENSE_CODE(LU_COMMUNICATION_TIMEOUT,
+	   "Logical unit communication time-out")
+SENSE_CODE(LU_COMMUNICATION_PARITY_ERROR,
+	   "Logical unit communication parity error")
+SENSE_CODE(LU_COMMUNICATION_CRC_ERROR,
+	   "Logical unit communication CRC error (Ultra-DMA/32)")
+SENSE_CODE(UNREACHABLE_COPY_TARGET,
+	   "Unreachable copy target")
+
+SENSE_CODE(TRACK_FOLLOWING_ERROR,
+	   "Track following error")
+SENSE_CODE(TRACKING_SERVO_FAILURE,
+	   "Tracking servo failure")
+SENSE_CODE(FOCUS_SERVO_FAILURE,
+	   "Focus servo failure")
+SENSE_CODE(SPINDLE_SERVO_FAILURE,
+	   "Spindle servo failure")
+SENSE_CODE(HEAD_SELECT_FAULT,
+	   "Head select fault")
+SENSE_CODE(VIBRATION_INDUCED_TRACKING_ERROR,
+	   "Vibration induced tracking error")
+
+SENSE_CODE(ERROR_LOG_OVERFLOW,
+	   "Error log overflow")
+
+SENSE_CODE(SSC_WARNING,
+	   "Warning")
+SENSE_CODE(WARNING_SPECIFIED_TEMPERATURE_EXCEEDED,
+	   "Warning - specified temperature exceeded")
+SENSE_CODE(WARNING_ENCLOSURE_DEGRADED,
+	   "Warning - enclosure degraded")
+SENSE_CODE(WARNING_BACKGROUND_SELFTEST_FAILED,
+	   "Warning - background self-test failed")
+SENSE_CODE(WARNING_BACKGROUND_PRESCAN_DETECTED_MEDIUM_ERROR,
+	   "Warning - background pre-scan detected medium error")
+SENSE_CODE(WARNING_BACKGROUND_MEDIUM_SCAN_DETECTED_MEDIUM_ERROR,
+	   "Warning - background medium scan detected medium error")
+SENSE_CODE(WARNING_NONVOLATILE_CACHE_NOW_VOLATILE,
+	   "Warning - non-volatile cache now volatile")
+SENSE_CODE(WARNING_DEGRADED_POWER_TO_NONVOLATILE_CACHE,
+	   "Warning - degraded power to non-volatile cache")
+SENSE_CODE(WARNING_POWER_LOSS_EXPECTED,
+	   "Warning - power loss expected")
+SENSE_CODE(WARNING_DEVICE_STATISTICS_NOTIFICATION_ACTIVE,
+	   "Warning - device statistics notification active")
+SENSE_CODE(WARNING_HIGH_CRITICAL_TEMPERATURE_LIMIT_EXCEEDED,
+	   "Warning - high critical temperature limit exceeded")
+SENSE_CODE(WARNING_LOW_CRITICAL_TEMPERATURE_LIMIT_EXCEEDED,
+	   "Warning - low critical temperature limit exceeded")
+SENSE_CODE(WARNING_HIGH_OPERATING_TEMPERATURE_LIMIT_EXCEEDED,
+	   "Warning - high operating temperature limit exceeded")
+SENSE_CODE(WARNING_LOW_OPERATING_TEMPERATURE_LIMIT_EXCEEDED,
+	   "Warning - low operating temperature limit exceeded")
+SENSE_CODE(WARNING_HIGH_CRITICAL_HUMIDITY_LIMIT_EXCEEDED,
+	   "Warning - high critical humidity limit exceeded")
+SENSE_CODE(WARNING_LOW_CRITICAL_HUMIDITY_LIMIT_EXCEEDED,
+	   "Warning - low critical humidity limit exceeded")
+SENSE_CODE(WARNING_HIGH_OPERATING_HUMIDITY_LIMIT_EXCEEDED,
+	   "Warning - high operating humidity limit exceeded")
+SENSE_CODE(WARNING_LOW_OPERATING_HUMIDITY_LIMIT_EXCEEDED,
+	   "Warning - low operating humidity limit exceeded")
+SENSE_CODE(WARNING_MICROCODE_SECURITY_AT_RISK,
+	   "Warning - microcode security at risk")
+SENSE_CODE(WARNING_MICROCODE_DIGITAL_SIGNATURE_VALIDATION_FAILURE,
+	   "Warning - microcode digital signature validation failure")
+SENSE_CODE(WARNING_PHYSICAL_ELEMENT_STATUS_CHANGE,
+	   "Warning - physical element status change")
+
+SENSE_CODE(SSC_WRITE_ERROR,
+	   "Write error")
+SENSE_CODE(WRITE_ERROR_RECOVERED_WITH_AUTO_REALLOCATION,
+	   "Write error - recovered with auto reallocation")
+SENSE_CODE(WRITE_ERROR_AUTO_REALLOCATION_FAILED,
+	   "Write error - auto reallocation failed")
+SENSE_CODE(WRITE_ERROR_RECOMMEND_REASSIGNMENT,
+	   "Write error - recommend reassignment")
+SENSE_CODE(COMPRESSION_CHECK_MISCOMPARE_ERROR,
+	   "Compression check miscompare error")
+SENSE_CODE(DATA_EXPANSION_OCCURRED_DURING_COMPRESSION,
+	   "Data expansion occurred during compression")
+SENSE_CODE(BLOCK_NOT_COMPRESSIBLE,
+	   "Block not compressible")
+SENSE_CODE(WRITE_ERROR_RECOVERY_NEEDED,
+	   "Write error - recovery needed")
+SENSE_CODE(WRITE_ERROR_RECOVERY_FAILED,
+	   "Write error - recovery failed")
+SENSE_CODE(WRITE_ERROR_LOSS_OF_STREAMING,
+	   "Write error - loss of streaming")
+SENSE_CODE(WRITE_ERROR_PADDING_BLOCKS_ADDED,
+	   "Write error - padding blocks added")
+SENSE_CODE(AUXILIARY_MEMORY_WRITE_ERROR,
+	   "Auxiliary memory write error")
+SENSE_CODE(WRITE_ERROR_UNEXPECTED_UNSOLICITED_DATA,
+	   "Write error - unexpected unsolicited data")
+SENSE_CODE(WRITE_ERROR_NOT_ENOUGH_UNSOLICITED_DATA,
+	   "Write error - not enough unsolicited data")
+SENSE_CODE(MULTIPLE_WRITE_ERRORS,
+	   "Multiple write errors")
+SENSE_CODE(DEFECTS_IN_ERROR_WINDOW,
+	   "Defects in error window")
+SENSE_CODE(INCOMPLETE_MULTIPLE_ATOMIC_WRITE_OPERATIONS,
+	   "Incomplete multiple atomic write operations")
+SENSE_CODE(WRITE_ERROR_RECOVERY_SCAN_NEEDED,
+	   "Write error - recovery scan needed")
+SENSE_CODE(WRITE_ERROR_INSUFFICIENT_ZONE_RESOURCES,
+	   "Write error - insufficient zone resources")
+
+SENSE_CODE(ERROR_DETECTED_BY_THIRD_PARTY_TEMPORARY_INITIATOR,
+	   "Error detected by third party temporary initiator")
+SENSE_CODE(THIRD_PARTY_DEVICE_FAILURE,
+	   "Third party device failure")
+SENSE_CODE(COPY_TARGET_DEVICE_NOT_REACHABLE,
+	   "Copy target device not reachable")
+SENSE_CODE(INCORRECT_COPY_TARGET_DEVICE_TYPE,
+	   "Incorrect copy target device type")
+SENSE_CODE(COPY_TARGET_DEVICE_DATA_UNDERRUN,
+	   "Copy target device data underrun")
+SENSE_CODE(COPY_TARGET_DEVICE_DATA_OVERRUN,
+	   "Copy target device data overrun")
+
+SENSE_CODE(INVALID_INFORMATION_UNIT,
+	   "Invalid information unit")
+SENSE_CODE(INFORMATION_UNIT_TOO_SHORT,
+	   "Information unit too short")
+SENSE_CODE(INFORMATION_UNIT_TOO_LONG,
+	   "Information unit too long")
+SENSE_CODE(INVALID_FIELD_IN_COMMAND_INFORMATION_UNIT,
+	   "Invalid field in command information unit")
+
+SENSE_CODE(ID_CRC_OR_ECC_ERROR,
+	   "Id CRC or ECC error")
+SENSE_CODE(LOGICAL_BLOCK_GUARD_CHECK_FAILED,
+	   "Logical block guard check failed")
+SENSE_CODE(LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED,
+	   "Logical block application tag check failed")
+SENSE_CODE(LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED,
+	   "Logical block reference tag check failed")
+SENSE_CODE(LOGICAL_BLOCK_PROTECTION_ERROR_ON_RECOVER_BUFFERED_DATA,
+	   "Logical block protection error on recover buffered data")
+SENSE_CODE(LOGICAL_BLOCK_PROTECTION_METHOD_ERROR,
+	   "Logical block protection method error")
+
+SENSE_CODE(UNRECOVERED_READ_ERROR,
+	   "Unrecovered read error")
+SENSE_CODE(READ_RETRIES_EXHAUSTED,
+	   "Read retries exhausted")
+SENSE_CODE(ERROR_TOO_LONG_TO_CORRECT,
+	   "Error too long to correct")
+SENSE_CODE(MULTIPLE_READ_ERRORS,
+	   "Multiple read errors")
+SENSE_CODE(UNRECOVERED_READ_ERROR_AUTO_REALLOCATE_FAILED,
+	   "Unrecovered read error - auto reallocate failed")
+SENSE_CODE(LEC_UNCORRECTABLE_ERROR,
+	   "L-EC uncorrectable error")
+SENSE_CODE(CIRC_UNRECOVERED_ERROR,
+	   "CIRC unrecovered error")
+SENSE_CODE(DATA_RESYNCHRONIZATION_ERROR,
+	   "Data re-synchronization error")
+SENSE_CODE(INCOMPLETE_BLOCK_READ,
+	   "Incomplete block read")
+SENSE_CODE(NO_GAP_FOUND,
+	   "No gap found")
+SENSE_CODE(MISCORRECTED_ERROR,
+	   "Miscorrected error")
+SENSE_CODE(UNRECOVERED_READ_ERROR_RECOMMEND_REASSIGNMENT,
+	   "Unrecovered read error - recommend reassignment")
+SENSE_CODE(UNRECOVERED_READ_ERROR_RECOMMEND_REWRITE_THE_DATA,
+	   "Unrecovered read error - recommend rewrite the data")
+SENSE_CODE(DECOMPRESSION_CRC_ERROR,
+	   "De-compression CRC error")
+SENSE_CODE(CANNOT_DECOMPRESS_USING_DECLARED_ALGORITHM,
+	   "Cannot decompress using declared algorithm")
+SENSE_CODE(ERROR_READING_UPC_EAN_NUMBER,
+	   "Error reading UPC/EAN number")
+SENSE_CODE(ERROR_READING_ISRC_NUMBER,
+	   "Error reading ISRC number")
+SENSE_CODE(READ_ERROR_LOSS_OF_STREAMING,
+	   "Read error - loss of streaming")
+SENSE_CODE(AUXILIARY_MEMORY_READ_ERROR,
+	   "Auxiliary memory read error")
+SENSE_CODE(READ_ERROR_FAILED_RETRANSMISSION_REQUEST,
+	   "Read error - failed retransmission request")
+SENSE_CODE(READ_ERROR_LBA_MARKED_BAD_BY_APPLICATION_CLIENT,
+	   "Read error - lba marked bad by application client")
+SENSE_CODE(WRITE_AFTER_SANITIZE_REQUIRED,
+	   "Write after sanitize required")
+
+SENSE_CODE(ADDRESS_MARK_NOT_FOUND_FOR_ID_FIELD,
+	   "Address mark not found for id field")
+
+SENSE_CODE(ADDRESS_MARK_NOT_FOUND_FOR_DATA_FIELD,
+	   "Address mark not found for data field")
+
+SENSE_CODE(RECORDED_ENTITY_NOT_FOUND,
+	   "Recorded entity not found")
+SENSE_CODE(RECORD_NOT_FOUND,
+	   "Record not found")
+SENSE_CODE(FILEMARK_OR_SETMARK_NOT_FOUND,
+	   "Filemark or setmark not found")
+SENSE_CODE(END_OF_DATA_NOT_FOUND,
+	   "End-of-data not found")
+SENSE_CODE(BLOCK_SEQUENCE_ERROR,
+	   "Block sequence error")
+SENSE_CODE(RECORD_NOT_FOUND_RECOMMEND_REASSIGNMENT,
+	   "Record not found - recommend reassignment")
+SENSE_CODE(RECORD_NOT_FOUND_DATA_AUTO_REALLOCATED,
+	   "Record not found - data auto-reallocated")
+SENSE_CODE(LOCATE_OPERATION_FAILURE,
+	   "Locate operation failure")
+
+SENSE_CODE(RANDOM_POSITIONING_ERROR,
+	   "Random positioning error")
+SENSE_CODE(MECHANICAL_POSITIONING_ERROR,
+	   "Mechanical positioning error")
+SENSE_CODE(POSITIONING_ERROR_DETECTED_BY_READ_OF_MEDIUM,
+	   "Positioning error detected by read of medium")
+
+SENSE_CODE(DATA_SYNCHRONIZATION_MARK_ERROR,
+	   "Data synchronization mark error")
+SENSE_CODE(DATA_SYNC_ERROR_DATA_REWRITTEN,
+	   "Data sync error - data rewritten")
+SENSE_CODE(DATA_SYNC_ERROR_RECOMMEND_REWRITE,
+	   "Data sync error - recommend rewrite")
+SENSE_CODE(DATA_SYNC_ERROR_DATA_AUTO_REALLOCATED,
+	   "Data sync error - data auto-reallocated")
+SENSE_CODE(DATA_SYNC_ERROR_RECOMMEND_REASSIGNMENT,
+	   "Data sync error - recommend reassignment")
+
+SENSE_CODE(RECOVERED_DATA_WITH_NO_ERROR_CORRECTION_APPLIED,
+	   "Recovered data with no error correction applied")
+SENSE_CODE(RECOVERED_DATA_WITH_RETRIES,
+	   "Recovered data with retries")
+SENSE_CODE(RECOVERED_DATA_WITH_POSITIVE_HEAD_OFFSET,
+	   "Recovered data with positive head offset")
+SENSE_CODE(RECOVERED_DATA_WITH_NEGATIVE_HEAD_OFFSET,
+	   "Recovered data with negative head offset")
+SENSE_CODE(RECOVERED_DATA_WITH_RETRIES_AND_OR_CIRC_APPLIED,
+	   "Recovered data with retries and/or circ applied")
+SENSE_CODE(RECOVERED_DATA_USING_PREVIOUS_SECTOR_ID,
+	   "Recovered data using previous sector id")
+SENSE_CODE(RECOVERED_DATA_WITHOUT_ECC_DATA_AUTO_REALLOCATED,
+	   "Recovered data without ECC - data auto-reallocated")
+SENSE_CODE(RECOVERED_DATA_WITHOUT_ECC_RECOMMEND_REASSIGNMENT,
+	   "Recovered data without ECC - recommend reassignment")
+SENSE_CODE(RECOVERED_DATA_WITHOUT_ECC_RECOMMEND_REWRITE,
+	   "Recovered data without ECC - recommend rewrite")
+SENSE_CODE(RECOVERED_DATA_WITHOUT_ECC_DATA_REWRITTEN,
+	   "Recovered data without ECC - data rewritten")
+
+SENSE_CODE(RECOVERED_DATA_WITH_ERROR_CORRECTION_APPLIED,
+	   "Recovered data with error correction applied")
+SENSE_CODE(RECOVERED_DATA_WITH_ERROR_CORRECTION_RETRIES_APPLIED,
+	   "Recovered data with error corr. & retries applied")
+SENSE_CODE(RECOVERED_DATA_DATA_AUTO_REALLOCATED,
+	   "Recovered data - data auto-reallocated")
+SENSE_CODE(RECOVERED_DATA_WITH_CIRC,
+	   "Recovered data with CIRC")
+SENSE_CODE(RECOVERED_DATA_WITH_LEC,
+	   "Recovered data with L-EC")
+SENSE_CODE(RECOVERED_DATA_RECOMMEND_REASSIGNMENT,
+	   "Recovered data - recommend reassignment")
+SENSE_CODE(RECOVERED_DATA_RECOMMEND_REWRITE,
+	   "Recovered data - recommend rewrite")
+SENSE_CODE(RECOVERED_DATA_WITH_ECC_DATA_REWRITTEN,
+	   "Recovered data with ECC - data rewritten")
+SENSE_CODE(RECOVERED_DATA_WITH_LINKING,
+	   "Recovered data with linking")
+
+SENSE_CODE(DEFECT_LIST_ERROR,
+	   "Defect list error")
+SENSE_CODE(DEFECT_LIST_NOT_AVAILABLE,
+	   "Defect list not available")
+SENSE_CODE(DEFECT_LIST_ERROR_IN_PRIMARY_LIST,
+	   "Defect list error in primary list")
+SENSE_CODE(DEFECT_LIST_ERROR_IN_GROWN_LIST,
+	   "Defect list error in grown list")
+
+SENSE_CODE(PARAMETER_LIST_LENGTH_ERROR,
+	   "Parameter list length error")
+
+SENSE_CODE(SYNCHRONOUS_DATA_TRANSFER_ERROR,
+	   "Synchronous data transfer error")
+
+SENSE_CODE(DEFECT_LIST_NOT_FOUND,
+	   "Defect list not found")
+SENSE_CODE(PRIMARY_DEFECT_LIST_NOT_FOUND,
+	   "Primary defect list not found")
+SENSE_CODE(GROWN_DEFECT_LIST_NOT_FOUND,
+	   "Grown defect list not found")
+
+SENSE_CODE(MISCOMPARE_DURING_VERIFY_OPERATION,
+	   "Miscompare during verify operation")
+SENSE_CODE(MISCOMPARE_VERIFY_OF_UNMAPPED_LBA,
+	   "Miscompare verify of unmapped LBA")
+
+SENSE_CODE(RECOVERED_ID_WITH_ECC_CORRECTION,
+	   "Recovered id with ECC correction")
+
+SENSE_CODE(PARTIAL_DEFECT_LIST_TRANSFER,
+	   "Partial defect list transfer")
+
+SENSE_CODE(INVALID_COMMAND_OP_CODE,
+	   "Invalid command operation code")
+SENSE_CODE(ACCESS_DENIED_INITIATOR_PENDING_ENROLLED,
+	   "Access denied - initiator pending-enrolled")
+SENSE_CODE(ACCESS_DENIED_NO_ACCESS_RIGHTS,
+	   "Access denied - no access rights")
+SENSE_CODE(ACCESS_DENIED_INVALID_MGMT_ID_KEY,
+	   "Access denied - invalid mgmt id key")
+SENSE_CODE(ILLEGAL_COMMAND_WHILE_IN_WRITE_CAPABLE_STATE,
+	   "Illegal command while in write capable state")
+SENSE_CODE(0x2005,
+	   "Obsolete")
+SENSE_CODE(ILLEGAL_COMMAND_WHILE_IN_EXPLICIT_ADDRESS_MODE,
+	   "Illegal command while in explicit address mode")
+SENSE_CODE(ILLEGAL_COMMAND_WHILE_IN_IMPLICIT_ADDRESS_MODE,
+	   "Illegal command while in implicit address mode")
+SENSE_CODE(ACCESS_DENIED_ENROLLMENT_CONFLICT,
+	   "Access denied - enrollment conflict")
+SENSE_CODE(ACCESS_DENIED_INVALID_LU_IDENTIFIER,
+	   "Access denied - invalid LU identifier")
+SENSE_CODE(ACCESS_DENIED_INVALID_PROXY_TOKEN,
+	   "Access denied - invalid proxy token")
+SENSE_CODE(ACCESS_DENIED_ACL_LUN_CONFLICT,
+	   "Access denied - ACL LUN conflict")
+SENSE_CODE(ILLEGAL_COMMAND_WHEN_NOT_IN_APPEND_ONLY_MODE,
+	   "Illegal command when not in append-only mode")
+SENSE_CODE(NOT_AN_ADMINISTRATIVE_LOGICAL_UNIT,
+	   "Not an administrative logical unit")
+SENSE_CODE(NOT_A_SUBSIDIARY_LOGICAL_UNIT,
+	   "Not a subsidiary logical unit")
+SENSE_CODE(NOT_A_CONGLOMERATE_LOGICAL_UNIT,
+	   "Not a conglomerate logical unit")
+
+SENSE_CODE(LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE,
+	   "Logical block address out of range")
+SENSE_CODE(INVALID_ELEMENT_ADDRESS,
+	   "Invalid element address")
+SENSE_CODE(INVALID_ADDRESS_FOR_WRITE,
+	   "Invalid address for write")
+SENSE_CODE(INVALID_WRITE_CROSSING_LAYER_JUMP,
+	   "Invalid write crossing layer jump")
+SENSE_CODE(UNALIGNED_WRITE_COMMAND,
+	   "Unaligned write command")
+SENSE_CODE(WRITE_BOUNDARY_VIOLATION,
+	   "Write boundary violation")
+SENSE_CODE(ATTEMPT_TO_READ_INVALID_DATA,
+	   "Attempt to read invalid data")
+SENSE_CODE(READ_BOUNDARY_VIOLATION,
+	   "Read boundary violation")
+SENSE_CODE(MISALIGNED_WRITE_COMMAND,
+	   "Misaligned write command")
+SENSE_CODE(ATTEMPT_TO_ACCESS_GAP_ZONE,
+	   "Attempt to access gap zone")
+
+SENSE_CODE(ILLEGAL_FUNCTION,
+	   "Illegal function (use 20 00, 24 00, or 26 00)")
+
+SENSE_CODE(INVALID_TOKEN_OP,
+	   "Invalid token operation, cause not reportable")
+SENSE_CODE(INVALID_TOKEN_OP_UNSUPPORTED_TOKEN_TYPE,
+	   "Invalid token operation, unsupported token type")
+SENSE_CODE(INVALID_TOKEN_OP_REMOTE_TOKEN_USAGE_NOT_SUPPORTED,
+	   "Invalid token operation, remote token usage not supported")
+SENSE_CODE(INVALID_TOKEN_OP_REMOTE_ROD_TOKEN_CREATION_NOT_SUPPORTED,
+	   "Invalid token operation, remote rod token creation not supported")
+SENSE_CODE(INVALID_TOKEN_OP_TOKEN_UNKNOWN,
+	   "Invalid token operation, token unknown")
+SENSE_CODE(INVALID_TOKEN_OP_TOKEN_CORRUPT,
+	   "Invalid token operation, token corrupt")
+SENSE_CODE(INVALID_TOKEN_OP_TOKEN_REVOKED,
+	   "Invalid token operation, token revoked")
+SENSE_CODE(INVALID_TOKEN_OP_TOKEN_EXPIRED,
+	   "Invalid token operation, token expired")
+SENSE_CODE(INVALID_TOKEN_OP_TOKEN_CANCELLED,
+	   "Invalid token operation, token cancelled")
+SENSE_CODE(INVALID_TOKEN_OP_TOKEN_DELETED,
+	   "Invalid token operation, token deleted")
+SENSE_CODE(INVALID_TOKEN_OP_INVALID_TOKEN_LENGTH,
+	   "Invalid token operation, invalid token length")
+
+SENSE_CODE(INVALID_FIELD_IN_CDB,
+	   "Invalid field in cdb")
+SENSE_CODE(CDB_DECRYPTION_ERROR,
+	   "CDB decryption error")
+SENSE_CODE(0x2402,
+	   "Obsolete")
+SENSE_CODE(0x2403,
+	   "Obsolete")
+SENSE_CODE(SECURITY_AUDIT_VALUE_FROZEN,
+	   "Security audit value frozen")
+SENSE_CODE(SECURITY_WORKING_KEY_FROZEN,
+	   "Security working key frozen")
+SENSE_CODE(NONCE_NOT_UNIQUE,
+	   "Nonce not unique")
+SENSE_CODE(NONCE_TIMESTAMP_OUT_OF_RANGE,
+	   "Nonce timestamp out of range")
+SENSE_CODE(INVALID_XCDB,
+	   "Invalid XCDB")
+SENSE_CODE(INVALID_FAST_FORMAT,
+	   "Invalid fast format")
+
+SENSE_CODE(LU_NOT_SUPPORTED,
+	   "Logical unit not supported")
+
+SENSE_CODE(INVALID_FIELD_IN_PARAMETER_LIST,
+	   "Invalid field in parameter list")
+SENSE_CODE(PARAMETER_NOT_SUPPORTED,
+	   "Parameter not supported")
+SENSE_CODE(PARAMETER_VALUE_INVALID,
+	   "Parameter value invalid")
+SENSE_CODE(THRESHOLD_PARAMETERS_NOT_SUPPORTED,
+	   "Threshold parameters not supported")
+SENSE_CODE(INVALID_RELEASE_OF_PERSISTENT_RESERVATION,
+	   "Invalid release of persistent reservation")
+SENSE_CODE(DATA_DECRYPTION_ERROR,
+	   "Data decryption error")
+SENSE_CODE(TOO_MANY_TARGET_DESCRIPTORS,
+	   "Too many target descriptors")
+SENSE_CODE(UNSUPPORTED_TARGET_DESCRIPTOR_TYPE_CODE,
+	   "Unsupported target descriptor type code")
+SENSE_CODE(TOO_MANY_SEGMENT_DESCRIPTORS,
+	   "Too many segment descriptors")
+SENSE_CODE(UNSUPPORTED_SEGMENT_DESCRIPTOR_TYPE_CODE,
+	   "Unsupported segment descriptor type code")
+SENSE_CODE(UNEXPECTED_INEXACT_SEGMENT,
+	   "Unexpected inexact segment")
+SENSE_CODE(INLINE_DATA_LENGTH_EXCEEDED,
+	   "Inline data length exceeded")
+SENSE_CODE(INVALID_OP_FOR_COPY_SOURCE_OR_DESTINATION,
+	   "Invalid operation for copy source or destination")
+SENSE_CODE(COPY_SEGMENT_GRANULARITY_VIOLATION,
+	   "Copy segment granularity violation")
+SENSE_CODE(INVALID_PARAMETER_WHILE_PORT_IS_ENABLED,
+	   "Invalid parameter while port is enabled")
+SENSE_CODE(INVALID_DATA_OUT_BUFFER_INTEGRITY_CHECK_VALUE,
+	   "Invalid data-out buffer integrity check value")
+SENSE_CODE(DATA_DECRYPTION_KEY_FAIL_LIMIT_REACHED,
+	   "Data decryption key fail limit reached")
+SENSE_CODE(INCOMPLETE_KEY_ASSOCIATED_DATA_SET,
+	   "Incomplete key-associated data set")
+SENSE_CODE(VENDOR_SPECIFIC_KEY_REFERENCE_NOT_FOUND,
+	   "Vendor specific key reference not found")
+SENSE_CODE(APPLICATION_TAG_MODE_PAGE_IS_INVALID,
+	   "Application tag mode page is invalid")
+SENSE_CODE(TAPE_STREAM_MIRRORING_PREVENTED,
+	   "Tape stream mirroring prevented")
+SENSE_CODE(COPY_SOURCE_OR_COPY_DESTINATION_NOT_AUTHORIZED,
+	   "Copy source or copy destination not authorized")
+SENSE_CODE(FAST_COPY_NOT_POSSIBLE,
+	   "Fast copy not possible")
+
+SENSE_CODE(WRITE_PROTECTED,
+	   "Write protected")
+SENSE_CODE(HW_WRITE_PROTECTED,
+	   "Hardware write protected")
+SENSE_CODE(LU_SOFTWARE_WRITE_PROTECTED,
+	   "Logical unit software write protected")
+SENSE_CODE(ASSOCIATED_WRITE_PROTECT,
+	   "Associated write protect")
+SENSE_CODE(PERSISTENT_WRITE_PROTECT,
+	   "Persistent write protect")
+SENSE_CODE(PERMANENT_WRITE_PROTECT,
+	   "Permanent write protect")
+SENSE_CODE(CONDITIONAL_WRITE_PROTECT,
+	   "Conditional write protect")
+SENSE_CODE(SPACE_ALLOCATION_FAILED_WRITE_PROTECT,
+	   "Space allocation failed write protect")
+SENSE_CODE(ZONE_IS_READ_ONLY,
+	   "Zone is read only")
+
+SENSE_CODE(NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED,
+	   "Not ready to ready change, medium may have changed")
+SENSE_CODE(IMPORT_OR_EXPORT_ELEMENT_ACCESSED,
+	   "Import or export element accessed")
+SENSE_CODE(FORMAT_LAYER_MAY_HAVE_CHANGED,
+	   "Format-layer may have changed")
+SENSE_CODE(IMPORT_EXPORT_ELEMENT_ACCESSED_MEDIUM_CHANGED,
+	   "Import/export element accessed, medium changed")
+
+SENSE_CODE(POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED,
+	   "Power on, reset, or bus device reset occurred")
+SENSE_CODE(POWER_ON_OCCURRED,
+	   "Power on occurred")
+SENSE_CODE(SCSI_BUS_RESET_OCCURRED,
+	   "Scsi bus reset occurred")
+SENSE_CODE(BUS_DEVICE_RESET_FUNCTION_OCCURRED,
+	   "Bus device reset function occurred")
+SENSE_CODE(DEVICE_INTERNAL_RESET,
+	   "Device internal reset")
+SENSE_CODE(TRANSCEIVER_MODE_CHANGED_TO_SINGLE_ENDED,
+	   "Transceiver mode changed to single-ended")
+SENSE_CODE(TRANSCEIVER_MODE_CHANGED_TO_LVD,
+	   "Transceiver mode changed to lvd")
+SENSE_CODE(I_T_NEXUS_LOSS_OCCURRED,
+	   "I_T nexus loss occurred")
+
+SENSE_CODE(PARAMETERS_CHANGED,
+	   "Parameters changed")
+SENSE_CODE(MODE_PARAMETERS_CHANGED,
+	   "Mode parameters changed")
+SENSE_CODE(LOG_PARAMETERS_CHANGED,
+	   "Log parameters changed")
+SENSE_CODE(RESERVATIONS_PREEMPTED,
+	   "Reservations preempted")
+SENSE_CODE(RESERVATIONS_RELEASED,
+	   "Reservations released")
+SENSE_CODE(REGISTRATIONS_PREEMPTED,
+	   "Registrations preempted")
+SENSE_CODE(ASYMMETRIC_ACCESS_STATE_CHANGED,
+	   "Asymmetric access state changed")
+SENSE_CODE(IMPLICIT_ASYMMETRIC_ACCESS_STATE_TRANSITION_FAILED,
+	   "Implicit asymmetric access state transition failed")
+SENSE_CODE(PRIORITY_CHANGED,
+	   "Priority changed")
+SENSE_CODE(CAPACITY_DATA_HAS_CHANGED,
+	   "Capacity data has changed")
+SENSE_CODE(ERROR_HISTORY_I_T_NEXUS_CLEARED,
+	   "Error history I_T nexus cleared")
+SENSE_CODE(ERROR_HISTORY_SNAPSHOT_RELEASED,
+	   "Error history snapshot released")
+SENSE_CODE(ERROR_RECOVERY_ATTRIBUTES_HAVE_CHANGED,
+	   "Error recovery attributes have changed")
+SENSE_CODE(DATA_ENCRYPTION_CAPABILITIES_CHANGED,
+	   "Data encryption capabilities changed")
+SENSE_CODE(TIMESTAMP_CHANGED,
+	   "Timestamp changed")
+SENSE_CODE(DATA_ENCRYPTION_PARAMS_CHANGED_BY_ANOTHER_I_T_NEXUS,
+	   "Data encryption parameters changed by another i_t nexus")
+SENSE_CODE(DATA_ENCRYPTION_PARAMS_CHANGED_BY_VENDOR_SPECIFIC_EVENT,
+	   "Data encryption parameters changed by vendor specific event")
+SENSE_CODE(DATA_ENCRYPTION_KEY_INSTANCE_COUNTER_HAS_CHANGED,
+	   "Data encryption key instance counter has changed")
+SENSE_CODE(SA_CREATION_CAPABILITIES_DATA_HAS_CHANGED,
+	   "SA creation capabilities data has changed")
+SENSE_CODE(MEDIUM_REMOVAL_PREVENTION_PREEMPTED,
+	   "Medium removal prevention preempted")
+SENSE_CODE(ZONE_RESET_WRITE_POINTER_RECOMMENDED,
+	   "Zone reset write pointer recommended")
+
+SENSE_CODE(COPY_CANNOT_EXECUTE_SINCE_HOST_CANNOT_DISCONNECT,
+	   "Copy cannot execute since host cannot disconnect")
+
+SENSE_CODE(COMMAND_SEQUENCE_ERROR,
+	   "Command sequence error")
+SENSE_CODE(TOO_MANY_WINDOWS_SPECIFIED,
+	   "Too many windows specified")
+SENSE_CODE(INVALID_COMBINATION_OF_WINDOWS_SPECIFIED,
+	   "Invalid combination of windows specified")
+SENSE_CODE(CURRENT_PROGRAM_AREA_IS_NOT_EMPTY,
+	   "Current program area is not empty")
+SENSE_CODE(CURRENT_PROGRAM_AREA_IS_EMPTY,
+	   "Current program area is empty")
+SENSE_CODE(ILLEGAL_POWER_CONDITION_REQUEST,
+	   "Illegal power condition request")
+SENSE_CODE(PERSISTENT_PREVENT_CONFLICT,
+	   "Persistent prevent conflict")
+SENSE_CODE(PREVIOUS_BUSY_STATUS,
+	   "Previous busy status")
+SENSE_CODE(PREVIOUS_TASK_SET_FULL_STATUS,
+	   "Previous task set full status")
+SENSE_CODE(PREVIOUS_RESERVATION_CONFLICT_STATUS,
+	   "Previous reservation conflict status")
+SENSE_CODE(PARTITION_OR_COLLECTION_CONTAINS_USER_OBJECTS,
+	   "Partition or collection contains user objects")
+SENSE_CODE(NOT_RESERVED,
+	   "Not reserved")
+SENSE_CODE(ORWRITE_GENERATION_DOES_NOT_MATCH,
+	   "Orwrite generation does not match")
+SENSE_CODE(RESET_WRITE_POINTER_NOT_ALLOWED,
+	   "Reset write pointer not allowed")
+SENSE_CODE(ZONE_IS_OFFLINE,
+	   "Zone is offline")
+SENSE_CODE(STREAM_NOT_OPEN,
+	   "Stream not open")
+SENSE_CODE(UNWRITTEN_DATA_IN_ZONE,
+	   "Unwritten data in zone")
+SENSE_CODE(DESCRIPTOR_FORMAT_SENSE_DATA_REQUIRED,
+	   "Descriptor format sense data required")
+SENSE_CODE(ZONE_IS_INACTIVE,
+	   "Zone is inactive")
+SENSE_CODE(WELL_KNOWN_LU_ACCESS_REQUIRED,
+	   "Well known logical unit access required")
+
+SENSE_CODE(OVERWRITE_ERROR_ON_UPDATE_IN_PLACE,
+	   "Overwrite error on update in place")
+
+SENSE_CODE(INSUFFICIENT_TIME_FOR_OPERATION,
+	   "Insufficient time for operation")
+SENSE_CODE(COMMAND_TIMEOUT_BEFORE_PROCESSING,
+	   "Command timeout before processing")
+SENSE_CODE(COMMAND_TIMEOUT_DURING_PROCESSING,
+	   "Command timeout during processing")
+SENSE_CODE(COMMAND_TIMEOUT_DURING_PROCESSING_DUE_TO_ERROR_RECOVERY,
+	   "Command timeout during processing due to error recovery")
+
+SENSE_CODE(COMMANDS_CLEARED_BY_ANOTHER_INITIATOR,
+	   "Commands cleared by another initiator")
+SENSE_CODE(COMMANDS_CLEARED_BY_POWER_LOSS_NOTIFICATION,
+	   "Commands cleared by power loss notification")
+SENSE_CODE(COMMANDS_CLEARED_BY_DEVICE_SERVER,
+	   "Commands cleared by device server")
+SENSE_CODE(SOME_COMMANDS_CLEARED_BY_QUEUING_LAYER_EVENT,
+	   "Some commands cleared by queuing layer event")
+
+SENSE_CODE(INCOMPATIBLE_MEDIUM_INSTALLED,
+	   "Incompatible medium installed")
+SENSE_CODE(CANNOT_READ_MEDIUM_UNKNOWN_FORMAT,
+	   "Cannot read medium - unknown format")
+SENSE_CODE(CANNOT_READ_MEDIUM_INCOMPATIBLE_FORMAT,
+	   "Cannot read medium - incompatible format")
+SENSE_CODE(CLEANING_CARTRIDGE_INSTALLED,
+	   "Cleaning cartridge installed")
+SENSE_CODE(CANNOT_WRITE_MEDIUM_UNKNOWN_FORMAT,
+	   "Cannot write medium - unknown format")
+SENSE_CODE(CANNOT_WRITE_MEDIUM_INCOMPATIBLE_FORMAT,
+	   "Cannot write medium - incompatible format")
+SENSE_CODE(CANNOT_FORMAT_MEDIUM_INCOMPATIBLE_MEDIUM,
+	   "Cannot format medium - incompatible medium")
+SENSE_CODE(CLEANING_FAILURE,
+	   "Cleaning failure")
+SENSE_CODE(CANNOT_WRITE_APPLICATION_CODE_MISMATCH,
+	   "Cannot write - application code mismatch")
+SENSE_CODE(CURRENT_SESSION_NOT_FIXATED_FOR_APPEND,
+	   "Current session not fixated for append")
+SENSE_CODE(CLEANING_REQUEST_REJECTED,
+	   "Cleaning request rejected")
+SENSE_CODE(WORM_MEDIUM_OVERWRITE_ATTEMPTED,
+	   "WORM medium - overwrite attempted")
+SENSE_CODE(WORM_MEDIUM_INTEGRITY_CHECK,
+	   "WORM medium - integrity check")
+SENSE_CODE(MEDIUM_NOT_FORMATTED,
+	   "Medium not formatted")
+SENSE_CODE(INCOMPATIBLE_VOLUME_TYPE,
+	   "Incompatible volume type")
+SENSE_CODE(INCOMPATIBLE_VOLUME_QUALIFIER,
+	   "Incompatible volume qualifier")
+SENSE_CODE(CLEANING_VOLUME_EXPIRED,
+	   "Cleaning volume expired")
+
+SENSE_CODE(MEDIUM_FORMAT_CORRUPTED,
+	   "Medium format corrupted")
+SENSE_CODE(FORMAT_COMMAND_FAILED,
+	   "Format command failed")
+SENSE_CODE(ZONED_FORMATTING_FAILED_DUE_TO_SPARE_LINKING,
+	   "Zoned formatting failed due to spare linking")
+SENSE_CODE(SANITIZE_COMMAND_FAILED,
+	   "Sanitize command failed")
+SENSE_CODE(DEPOPULATION_FAILED,
+	   "Depopulation failed")
+SENSE_CODE(DEPOPULATION_RESTORATION_FAILED,
+	   "Depopulation restoration failed")
+
+SENSE_CODE(NO_DEFECT_SPARE_LOCATION_AVAILABLE,
+	   "No defect spare location available")
+SENSE_CODE(DEFECT_LIST_UPDATE_FAILURE,
+	   "Defect list update failure")
+
+SENSE_CODE(TAPE_LENGTH_ERROR,
+	   "Tape length error")
+
+SENSE_CODE(ENCLOSURE_FAILURE,
+	   "Enclosure failure")
+
+SENSE_CODE(ENCLOSURE_SERVICES_FAILURE,
+	   "Enclosure services failure")
+SENSE_CODE(UNSUPPORTED_ENCLOSURE_FUNCTION,
+	   "Unsupported enclosure function")
+SENSE_CODE(ENCLOSURE_SERVICES_UNAVAILABLE,
+	   "Enclosure services unavailable")
+SENSE_CODE(ENCLOSURE_SERVICES_TRANSFER_FAILURE,
+	   "Enclosure services transfer failure")
+SENSE_CODE(ENCLOSURE_SERVICES_TRANSFER_REFUSED,
+	   "Enclosure services transfer refused")
+SENSE_CODE(ENCLOSURE_SERVICES_CHECKSUM_ERROR,
+	   "Enclosure services checksum error")
+
+SENSE_CODE(RIBBON_INK_OR_TONER_FAILURE,
+	   "Ribbon, ink, or toner failure")
+
+SENSE_CODE(ROUNDED_PARAMETER,
+	   "Rounded parameter")
+
+SENSE_CODE(EVENT_STATUS_NOTIFICATION,
+	   "Event status notification")
+SENSE_CODE(ESN_POWER_MANAGEMENT_CLASS_EVENT,
+	   "Esn - power management class event")
+SENSE_CODE(ESN_MEDIA_CLASS_EVENT,
+	   "Esn - media class event")
+SENSE_CODE(ESN_DEVICE_BUSY_CLASS_EVENT,
+	   "Esn - device busy class event")
+SENSE_CODE(THIN_PROVISIONING_SOFT_THRESHOLD_REACHED,
+	   "Thin Provisioning soft threshold reached")
+SENSE_CODE(DEPOPULATION_INTERRUPTED,
+	   "Depopulation interrupted")
+SENSE_CODE(DEPOPULATION_RESTORATION_INTERRUPTED,
+	   "Depopulation restoration interrupted")
+
+SENSE_CODE(SAVING_PARAMETERS_NOT_SUPPORTED,
+	   "Saving parameters not supported")
+
+SENSE_CODE(MEDIUM_NOT_PRESENT,
+	   "Medium not present")
+SENSE_CODE(MEDIUM_NOT_PRESENT_TRAY_CLOSED,
+	   "Medium not present - tray closed")
+SENSE_CODE(MEDIUM_NOT_PRESENT_TRAY_OPEN,
+	   "Medium not present - tray open")
+SENSE_CODE(MEDIUM_NOT_PRESENT_LOADABLE,
+	   "Medium not present - loadable")
+SENSE_CODE(MEDIUM_NOT_PRESENT_MEDIUM_AUXILIARY_MEMORY_ACCESSIBLE,
+	   "Medium not present - medium auxiliary memory accessible")
+
+SENSE_CODE(SEQUENTIAL_POSITIONING_ERROR,
+	   "Sequential positioning error")
+SENSE_CODE(TAPE_POSITION_ERROR_AT_BEGINNING_OF_MEDIUM,
+	   "Tape position error at beginning-of-medium")
+SENSE_CODE(TAPE_POSITION_ERROR_AT_END_OF_MEDIUM,
+	   "Tape position error at end-of-medium")
+SENSE_CODE(TAPE_OR_ELECTRONIC_VERTICAL_FORMS_UNIT_NOT_READY,
+	   "Tape or electronic vertical forms unit not ready")
+SENSE_CODE(SLEW_FAILURE,
+	   "Slew failure")
+SENSE_CODE(PAPER_JAM,
+	   "Paper jam")
+SENSE_CODE(FAILED_TO_SENSE_TOP_OF_FORM,
+	   "Failed to sense top-of-form")
+SENSE_CODE(FAILED_TO_SENSE_BOTTOM_OF_FORM,
+	   "Failed to sense bottom-of-form")
+SENSE_CODE(REPOSITION_ERROR,
+	   "Reposition error")
+SENSE_CODE(READ_PAST_END_OF_MEDIUM,
+	   "Read past end of medium")
+SENSE_CODE(READ_PAST_BEGINNING_OF_MEDIUM,
+	   "Read past beginning of medium")
+SENSE_CODE(POSITION_PAST_END_OF_MEDIUM,
+	   "Position past end of medium")
+SENSE_CODE(POSITION_PAST_BEGINNING_OF_MEDIUM,
+	   "Position past beginning of medium")
+SENSE_CODE(MEDIUM_DESTINATION_ELEMENT_FULL,
+	   "Medium destination element full")
+SENSE_CODE(MEDIUM_SOURCE_ELEMENT_EMPTY,
+	   "Medium source element empty")
+SENSE_CODE(END_OF_MEDIUM_REACHED,
+	   "End of medium reached")
+SENSE_CODE(MEDIUM_MAGAZINE_NOT_ACCESSIBLE,
+	   "Medium magazine not accessible")
+SENSE_CODE(MEDIUM_MAGAZINE_REMOVED,
+	   "Medium magazine removed")
+SENSE_CODE(MEDIUM_MAGAZINE_INSERTED,
+	   "Medium magazine inserted")
+SENSE_CODE(MEDIUM_MAGAZINE_LOCKED,
+	   "Medium magazine locked")
+SENSE_CODE(MEDIUM_MAGAZINE_UNLOCKED,
+	   "Medium magazine unlocked")
+SENSE_CODE(MECHANICAL_POSITIONING_OR_CHANGER_ERROR,
+	   "Mechanical positioning or changer error")
+SENSE_CODE(READ_PAST_END_OF_USER_OBJECT,
+	   "Read past end of user object")
+SENSE_CODE(ELEMENT_DISABLED,
+	   "Element disabled")
+SENSE_CODE(ELEMENT_ENABLED,
+	   "Element enabled")
+SENSE_CODE(DATA_TRANSFER_DEVICE_REMOVED,
+	   "Data transfer device removed")
+SENSE_CODE(DATA_TRANSFER_DEVICE_INSERTED,
+	   "Data transfer device inserted")
+SENSE_CODE(TOO_MANY_LOGICAL_OBJECTS_ON_PARTITION_TO_SUPPORT_OP,
+	   "Too many logical objects on partition to support operation")
+SENSE_CODE(ELEMENT_STATIC_INFORMATION_CHANGED,
+	   "Element static information changed")
+
+SENSE_CODE(INVALID_BITS_IN_IDENTIFY_MESSAGE,
+	   "Invalid bits in identify message")
+
+SENSE_CODE(LU_HAS_NOT_SELF_CONFIGURED_YET,
+	   "Logical unit has not self-configured yet")
+SENSE_CODE(LU_FAILURE,
+	   "Logical unit failure")
+SENSE_CODE(TIMEOUT_ON_LOGICAL_UNIT,
+	   "Timeout on logical unit")
+SENSE_CODE(LU_FAILED_SELF_TEST,
+	   "Logical unit failed self-test")
+SENSE_CODE(LU_UNABLE_TO_UPDATE_SELFTEST_LOG,
+	   "Logical unit unable to update self-test log")
+
+SENSE_CODE(TARGET_OPERATING_CONDITIONS_HAVE_CHANGED,
+	   "Target operating conditions have changed")
+SENSE_CODE(MICROCODE_HAS_BEEN_CHANGED,
+	   "Microcode has been changed")
+SENSE_CODE(CHANGED_OPERATING_DEFINITION,
+	   "Changed operating definition")
+SENSE_CODE(INQUIRY_DATA_HAS_CHANGED,
+	   "Inquiry data has changed")
+SENSE_CODE(COMPONENT_DEVICE_ATTACHED,
+	   "Component device attached")
+SENSE_CODE(DEVICE_IDENTIFIER_CHANGED,
+	   "Device identifier changed")
+SENSE_CODE(REDUNDANCY_GROUP_CREATED_OR_MODIFIED,
+	   "Redundancy group created or modified")
+SENSE_CODE(REDUNDANCY_GROUP_DELETED,
+	   "Redundancy group deleted")
+SENSE_CODE(SPARE_CREATED_OR_MODIFIED,
+	   "Spare created or modified")
+SENSE_CODE(SPARE_DELETED,
+	   "Spare deleted")
+SENSE_CODE(VOLUME_SET_CREATED_OR_MODIFIED,
+	   "Volume set created or modified")
+SENSE_CODE(VOLUME_SET_DELETED,
+	   "Volume set deleted")
+SENSE_CODE(VOLUME_SET_DEASSIGNED,
+	   "Volume set deassigned")
+SENSE_CODE(VOLUME_SET_REASSIGNED,
+	   "Volume set reassigned")
+SENSE_CODE(REPORTED_LUNS_DATA_HAS_CHANGED,
+	   "Reported luns data has changed")
+SENSE_CODE(ECHO_BUFFER_OVERWRITTEN,
+	   "Echo buffer overwritten")
+SENSE_CODE(MEDIUM_LOADABLE,
+	   "Medium loadable")
+SENSE_CODE(MEDIUM_AUXILIARY_MEMORY_ACCESSIBLE,
+	   "Medium auxiliary memory accessible")
+SENSE_CODE(iSCSI_IP_ADDRESS_ADDED,
+	   "iSCSI IP address added")
+SENSE_CODE(iSCSI_IP_ADDRESS_REMOVED,
+	   "iSCSI IP address removed")
+SENSE_CODE(iSCSI_IP_ADDRESS_CHANGED,
+	   "iSCSI IP address changed")
+SENSE_CODE(INSPECT_REFERRALS_SENSE_DESCRIPTORS,
+	   "Inspect referrals sense descriptors")
+SENSE_CODE(MICROCODE_HAS_BEEN_CHANGED_WITHOUT_RESET,
+	   "Microcode has been changed without reset")
+SENSE_CODE(ZONE_TRANSITION_TO_FULL,
+	   "Zone transition to full")
+SENSE_CODE(BIND_COMPLETED,
+	   "Bind completed")
+SENSE_CODE(BIND_REDIRECTED,
+	   "Bind redirected")
+SENSE_CODE(SUBSIDIARY_BINDING_CHANGED,
+	   "Subsidiary binding changed")
 
 /*
- *	SENSE_CODE(0x40NN, "Ram failure")
- *	SENSE_CODE(0x40NN, "Diagnostic failure on component nn")
- *	SENSE_CODE(0x41NN, "Data path failure")
- *	SENSE_CODE(0x42NN, "Power-on or self-test failure")
+ * SENSE_CODE(0x40NN, "Ram failure")
+ * SENSE_CODE(0x40NN, "Diagnostic failure on component nn")
+ *
+ * SENSE_CODE(0x41NN, "Data path failure")
+ *
+ * SENSE_CODE(0x42NN, "Power-on or self-test failure")
  */
-SENSE_CODE(0x4300, "Message error")
-
-SENSE_CODE(0x4400, "Internal target failure")
-SENSE_CODE(0x4401, "Persistent reservation information lost")
-SENSE_CODE(0x4471, "ATA device failed set features")
-
-SENSE_CODE(0x4500, "Select or reselect failure")
-
-SENSE_CODE(0x4600, "Unsuccessful soft reset")
-
-SENSE_CODE(0x4700, "Scsi parity error")
-SENSE_CODE(0x4701, "Data phase CRC error detected")
-SENSE_CODE(0x4702, "Scsi parity error detected during st data phase")
-SENSE_CODE(0x4703, "Information unit iuCRC error detected")
-SENSE_CODE(0x4704, "Asynchronous information protection error detected")
-SENSE_CODE(0x4705, "Protocol service CRC error")
-SENSE_CODE(0x4706, "Phy test function in progress")
-SENSE_CODE(0x477f, "Some commands cleared by iSCSI Protocol event")
-
-SENSE_CODE(0x4800, "Initiator detected error message received")
-
-SENSE_CODE(0x4900, "Invalid message error")
-
-SENSE_CODE(0x4A00, "Command phase error")
-
-SENSE_CODE(0x4B00, "Data phase error")
-SENSE_CODE(0x4B01, "Invalid target port transfer tag received")
-SENSE_CODE(0x4B02, "Too much write data")
-SENSE_CODE(0x4B03, "Ack/nak timeout")
-SENSE_CODE(0x4B04, "Nak received")
-SENSE_CODE(0x4B05, "Data offset error")
-SENSE_CODE(0x4B06, "Initiator response timeout")
-SENSE_CODE(0x4B07, "Connection lost")
-SENSE_CODE(0x4B08, "Data-in buffer overflow - data buffer size")
-SENSE_CODE(0x4B09, "Data-in buffer overflow - data buffer descriptor area")
-SENSE_CODE(0x4B0A, "Data-in buffer error")
-SENSE_CODE(0x4B0B, "Data-out buffer overflow - data buffer size")
-SENSE_CODE(0x4B0C, "Data-out buffer overflow - data buffer descriptor area")
-SENSE_CODE(0x4B0D, "Data-out buffer error")
-SENSE_CODE(0x4B0E, "PCIe fabric error")
-SENSE_CODE(0x4B0F, "PCIe completion timeout")
-SENSE_CODE(0x4B10, "PCIe completer abort")
-SENSE_CODE(0x4B11, "PCIe poisoned tlp received")
-SENSE_CODE(0x4B12, "PCIe eCRC check failed")
-SENSE_CODE(0x4B13, "PCIe unsupported request")
-SENSE_CODE(0x4B14, "PCIe acs violation")
-SENSE_CODE(0x4B15, "PCIe tlp prefix blocked")
-
-SENSE_CODE(0x4C00, "Logical unit failed self-configuration")
+
+SENSE_CODE(MESSAGE_ERROR,
+	   "Message error")
+
+SENSE_CODE(INTERNAL_TARGET_FAILURE,
+	   "Internal target failure")
+SENSE_CODE(PERSISTENT_RESERVATION_INFORMATION_LOST,
+	   "Persistent reservation information lost")
+SENSE_CODE(ATA_DEVICE_FAILED_SET_FEATURES,
+	   "ATA device failed set features")
+
+SENSE_CODE(SELECT_OR_RESELECT_FAILURE,
+	   "Select or reselect failure")
+
+SENSE_CODE(UNSUCCESSFUL_SOFT_RESET,
+	   "Unsuccessful soft reset")
+
+SENSE_CODE(SCSI_PARITY_ERROR,
+	   "Scsi parity error")
+SENSE_CODE(DATA_PHASE_CRC_ERROR_DETECTED,
+	   "Data phase CRC error detected")
+SENSE_CODE(SCSI_PARITY_ERROR_DETECTED_DURING_ST_DATA_PHASE,
+	   "Scsi parity error detected during st data phase")
+SENSE_CODE(INFORMATION_UNIT_iuCRC_ERROR_DETECTED,
+	   "Information unit iuCRC error detected")
+SENSE_CODE(ASYNCHRONOUS_INFORMATION_PROTECTION_ERROR_DETECTED,
+	   "Asynchronous information protection error detected")
+SENSE_CODE(PROTOCOL_SERVICE_CRC_ERROR,
+	   "Protocol service CRC error")
+SENSE_CODE(PHY_TEST_FUNCTION_IN_PROGRESS,
+	   "Phy test function in progress")
+SENSE_CODE(SOME_COMMANDS_CLEARED_BY_ISCSI_PROTOCOL_EVENT,
+	   "Some commands cleared by iSCSI Protocol event")
+
+SENSE_CODE(INITIATOR_DETECTED_ERROR_MESSAGE_RECEIVED,
+	   "Initiator detected error message received")
+
+SENSE_CODE(INVALID_MESSAGE_ERROR,
+	   "Invalid message error")
+
+SENSE_CODE(COMMAND_PHASE_ERROR,
+	   "Command phase error")
+
+SENSE_CODE(DATA_PHASE_ERROR,
+	   "Data phase error")
+SENSE_CODE(INVALID_TARGET_PORT_TRANSFER_TAG_RECEIVED,
+	   "Invalid target port transfer tag received")
+SENSE_CODE(TOO_MUCH_WRITE_DATA,
+	   "Too much write data")
+SENSE_CODE(ACK_NAK_TIMEOUT,
+	   "Ack/nak timeout")
+SENSE_CODE(NAK_RECEIVED,
+	   "Nak received")
+SENSE_CODE(DATA_OFFSET_ERROR,
+	   "Data offset error")
+SENSE_CODE(INITIATOR_RESPONSE_TIMEOUT,
+	   "Initiator response timeout")
+SENSE_CODE(CONNECTION_LOST,
+	   "Connection lost")
+SENSE_CODE(DATA_IN_BUFFER_OVERFLOW_BUFFER_SIZE,
+	   "Data-in buffer overflow - data buffer size")
+SENSE_CODE(DATA_IN_BUFFER_OVERFLOW_BUFFER_DESCRIPTOR_AREA,
+	   "Data-in buffer overflow - data buffer descriptor area")
+SENSE_CODE(DATA_IN_BUFFER_ERROR,
+	   "Data-in buffer error")
+SENSE_CODE(DATA_OUT_BUFFER_OVERFLOW_BUFFER_SIZE,
+	   "Data-out buffer overflow - data buffer size")
+SENSE_CODE(DATA_OUT_BUFFER_OVERFLOW_BUFFER_DESCRIPTOR_AREA,
+	   "Data-out buffer overflow - data buffer descriptor area")
+SENSE_CODE(DATA_OUT_BUFFER_ERROR,
+	   "Data-out buffer error")
+SENSE_CODE(PCIE_FABRIC_ERROR,
+	   "PCIe fabric error")
+SENSE_CODE(PCIE_COMPLETION_TIMEOUT,
+	   "PCIe completion timeout")
+SENSE_CODE(PCIE_COMPLETER_ABORT,
+	   "PCIe completer abort")
+SENSE_CODE(PCIE_POISONED_TLP_RECEIVED,
+	   "PCIe poisoned tlp received")
+SENSE_CODE(PCIE_ECRC_CHECK_FAILED,
+	   "PCIe eCRC check failed")
+SENSE_CODE(PCIE_UNSUPPORTED_REQUEST,
+	   "PCIe unsupported request")
+SENSE_CODE(PCIE_ACS_VIOLATION,
+	   "PCIe acs violation")
+SENSE_CODE(PCIE_TLP_PREFIX_BLOCKED,
+	   "PCIe tlp prefix blocked")
+
+SENSE_CODE(LU_FAILED_SELF_CONFIGURATION,
+	   "Logical unit failed self-configuration")
+
 /*
- *	SENSE_CODE(0x4DNN, "Tagged overlapped commands (nn = queue tag)")
+ * SENSE_CODE(0x4DNN, "Tagged overlapped commands (nn = queue tag)")
  */
-SENSE_CODE(0x4E00, "Overlapped commands attempted")
-
-SENSE_CODE(0x5000, "Write append error")
-SENSE_CODE(0x5001, "Write append position error")
-SENSE_CODE(0x5002, "Position error related to timing")
-
-SENSE_CODE(0x5100, "Erase failure")
-SENSE_CODE(0x5101, "Erase failure - incomplete erase operation detected")
-
-SENSE_CODE(0x5200, "Cartridge fault")
-
-SENSE_CODE(0x5300, "Media load or eject failed")
-SENSE_CODE(0x5301, "Unload tape failure")
-SENSE_CODE(0x5302, "Medium removal prevented")
-SENSE_CODE(0x5303, "Medium removal prevented by data transfer element")
-SENSE_CODE(0x5304, "Medium thread or unthread failure")
-SENSE_CODE(0x5305, "Volume identifier invalid")
-SENSE_CODE(0x5306, "Volume identifier missing")
-SENSE_CODE(0x5307, "Duplicate volume identifier")
-SENSE_CODE(0x5308, "Element status unknown")
-SENSE_CODE(0x5309, "Data transfer device error - load failed")
-SENSE_CODE(0x530a, "Data transfer device error - unload failed")
-SENSE_CODE(0x530b, "Data transfer device error - unload missing")
-SENSE_CODE(0x530c, "Data transfer device error - eject failed")
-SENSE_CODE(0x530d, "Data transfer device error - library communication failed")
-
-SENSE_CODE(0x5400, "Scsi to host system interface failure")
-
-SENSE_CODE(0x5500, "System resource failure")
-SENSE_CODE(0x5501, "System buffer full")
-SENSE_CODE(0x5502, "Insufficient reservation resources")
-SENSE_CODE(0x5503, "Insufficient resources")
-SENSE_CODE(0x5504, "Insufficient registration resources")
-SENSE_CODE(0x5505, "Insufficient access control resources")
-SENSE_CODE(0x5506, "Auxiliary memory out of space")
-SENSE_CODE(0x5507, "Quota error")
-SENSE_CODE(0x5508, "Maximum number of supplemental decryption keys exceeded")
-SENSE_CODE(0x5509, "Medium auxiliary memory not accessible")
-SENSE_CODE(0x550A, "Data currently unavailable")
-SENSE_CODE(0x550B, "Insufficient power for operation")
-SENSE_CODE(0x550C, "Insufficient resources to create rod")
-SENSE_CODE(0x550D, "Insufficient resources to create rod token")
-SENSE_CODE(0x550E, "Insufficient zone resources")
-SENSE_CODE(0x550F, "Insufficient zone resources to complete write")
-SENSE_CODE(0x5510, "Maximum number of streams open")
-SENSE_CODE(0x5511, "Insufficient resources to bind")
-
-SENSE_CODE(0x5700, "Unable to recover table-of-contents")
-
-SENSE_CODE(0x5800, "Generation does not exist")
-
-SENSE_CODE(0x5900, "Updated block read")
-
-SENSE_CODE(0x5A00, "Operator request or state change input")
-SENSE_CODE(0x5A01, "Operator medium removal request")
-SENSE_CODE(0x5A02, "Operator selected write protect")
-SENSE_CODE(0x5A03, "Operator selected write permit")
-
-SENSE_CODE(0x5B00, "Log exception")
-SENSE_CODE(0x5B01, "Threshold condition met")
-SENSE_CODE(0x5B02, "Log counter at maximum")
-SENSE_CODE(0x5B03, "Log list codes exhausted")
-
-SENSE_CODE(0x5C00, "Rpl status change")
-SENSE_CODE(0x5C01, "Spindles synchronized")
-SENSE_CODE(0x5C02, "Spindles not synchronized")
-
-SENSE_CODE(0x5D00, "Failure prediction threshold exceeded")
-SENSE_CODE(0x5D01, "Media failure prediction threshold exceeded")
-SENSE_CODE(0x5D02, "Logical unit failure prediction threshold exceeded")
-SENSE_CODE(0x5D03, "Spare area exhaustion prediction threshold exceeded")
-SENSE_CODE(0x5D10, "Hardware impending failure general hard drive failure")
-SENSE_CODE(0x5D11, "Hardware impending failure drive error rate too high")
-SENSE_CODE(0x5D12, "Hardware impending failure data error rate too high")
-SENSE_CODE(0x5D13, "Hardware impending failure seek error rate too high")
-SENSE_CODE(0x5D14, "Hardware impending failure too many block reassigns")
-SENSE_CODE(0x5D15, "Hardware impending failure access times too high")
-SENSE_CODE(0x5D16, "Hardware impending failure start unit times too high")
-SENSE_CODE(0x5D17, "Hardware impending failure channel parametrics")
-SENSE_CODE(0x5D18, "Hardware impending failure controller detected")
-SENSE_CODE(0x5D19, "Hardware impending failure throughput performance")
-SENSE_CODE(0x5D1A, "Hardware impending failure seek time performance")
-SENSE_CODE(0x5D1B, "Hardware impending failure spin-up retry count")
-SENSE_CODE(0x5D1C, "Hardware impending failure drive calibration retry count")
-SENSE_CODE(0x5D20, "Controller impending failure general hard drive failure")
-SENSE_CODE(0x5D21, "Controller impending failure drive error rate too high")
-SENSE_CODE(0x5D22, "Controller impending failure data error rate too high")
-SENSE_CODE(0x5D23, "Controller impending failure seek error rate too high")
-SENSE_CODE(0x5D24, "Controller impending failure too many block reassigns")
-SENSE_CODE(0x5D25, "Controller impending failure access times too high")
-SENSE_CODE(0x5D26, "Controller impending failure start unit times too high")
-SENSE_CODE(0x5D27, "Controller impending failure channel parametrics")
-SENSE_CODE(0x5D28, "Controller impending failure controller detected")
-SENSE_CODE(0x5D29, "Controller impending failure throughput performance")
-SENSE_CODE(0x5D2A, "Controller impending failure seek time performance")
-SENSE_CODE(0x5D2B, "Controller impending failure spin-up retry count")
-SENSE_CODE(0x5D2C, "Controller impending failure drive calibration retry count")
-SENSE_CODE(0x5D30, "Data channel impending failure general hard drive failure")
-SENSE_CODE(0x5D31, "Data channel impending failure drive error rate too high")
-SENSE_CODE(0x5D32, "Data channel impending failure data error rate too high")
-SENSE_CODE(0x5D33, "Data channel impending failure seek error rate too high")
-SENSE_CODE(0x5D34, "Data channel impending failure too many block reassigns")
-SENSE_CODE(0x5D35, "Data channel impending failure access times too high")
-SENSE_CODE(0x5D36, "Data channel impending failure start unit times too high")
-SENSE_CODE(0x5D37, "Data channel impending failure channel parametrics")
-SENSE_CODE(0x5D38, "Data channel impending failure controller detected")
-SENSE_CODE(0x5D39, "Data channel impending failure throughput performance")
-SENSE_CODE(0x5D3A, "Data channel impending failure seek time performance")
-SENSE_CODE(0x5D3B, "Data channel impending failure spin-up retry count")
-SENSE_CODE(0x5D3C, "Data channel impending failure drive calibration retry count")
-SENSE_CODE(0x5D40, "Servo impending failure general hard drive failure")
-SENSE_CODE(0x5D41, "Servo impending failure drive error rate too high")
-SENSE_CODE(0x5D42, "Servo impending failure data error rate too high")
-SENSE_CODE(0x5D43, "Servo impending failure seek error rate too high")
-SENSE_CODE(0x5D44, "Servo impending failure too many block reassigns")
-SENSE_CODE(0x5D45, "Servo impending failure access times too high")
-SENSE_CODE(0x5D46, "Servo impending failure start unit times too high")
-SENSE_CODE(0x5D47, "Servo impending failure channel parametrics")
-SENSE_CODE(0x5D48, "Servo impending failure controller detected")
-SENSE_CODE(0x5D49, "Servo impending failure throughput performance")
-SENSE_CODE(0x5D4A, "Servo impending failure seek time performance")
-SENSE_CODE(0x5D4B, "Servo impending failure spin-up retry count")
-SENSE_CODE(0x5D4C, "Servo impending failure drive calibration retry count")
-SENSE_CODE(0x5D50, "Spindle impending failure general hard drive failure")
-SENSE_CODE(0x5D51, "Spindle impending failure drive error rate too high")
-SENSE_CODE(0x5D52, "Spindle impending failure data error rate too high")
-SENSE_CODE(0x5D53, "Spindle impending failure seek error rate too high")
-SENSE_CODE(0x5D54, "Spindle impending failure too many block reassigns")
-SENSE_CODE(0x5D55, "Spindle impending failure access times too high")
-SENSE_CODE(0x5D56, "Spindle impending failure start unit times too high")
-SENSE_CODE(0x5D57, "Spindle impending failure channel parametrics")
-SENSE_CODE(0x5D58, "Spindle impending failure controller detected")
-SENSE_CODE(0x5D59, "Spindle impending failure throughput performance")
-SENSE_CODE(0x5D5A, "Spindle impending failure seek time performance")
-SENSE_CODE(0x5D5B, "Spindle impending failure spin-up retry count")
-SENSE_CODE(0x5D5C, "Spindle impending failure drive calibration retry count")
-SENSE_CODE(0x5D60, "Firmware impending failure general hard drive failure")
-SENSE_CODE(0x5D61, "Firmware impending failure drive error rate too high")
-SENSE_CODE(0x5D62, "Firmware impending failure data error rate too high")
-SENSE_CODE(0x5D63, "Firmware impending failure seek error rate too high")
-SENSE_CODE(0x5D64, "Firmware impending failure too many block reassigns")
-SENSE_CODE(0x5D65, "Firmware impending failure access times too high")
-SENSE_CODE(0x5D66, "Firmware impending failure start unit times too high")
-SENSE_CODE(0x5D67, "Firmware impending failure channel parametrics")
-SENSE_CODE(0x5D68, "Firmware impending failure controller detected")
-SENSE_CODE(0x5D69, "Firmware impending failure throughput performance")
-SENSE_CODE(0x5D6A, "Firmware impending failure seek time performance")
-SENSE_CODE(0x5D6B, "Firmware impending failure spin-up retry count")
-SENSE_CODE(0x5D6C, "Firmware impending failure drive calibration retry count")
-SENSE_CODE(0x5D73, "Media impending failure endurance limit met")
-SENSE_CODE(0x5DFF, "Failure prediction threshold exceeded (false)")
-
-SENSE_CODE(0x5E00, "Low power condition on")
-SENSE_CODE(0x5E01, "Idle condition activated by timer")
-SENSE_CODE(0x5E02, "Standby condition activated by timer")
-SENSE_CODE(0x5E03, "Idle condition activated by command")
-SENSE_CODE(0x5E04, "Standby condition activated by command")
-SENSE_CODE(0x5E05, "Idle_b condition activated by timer")
-SENSE_CODE(0x5E06, "Idle_b condition activated by command")
-SENSE_CODE(0x5E07, "Idle_c condition activated by timer")
-SENSE_CODE(0x5E08, "Idle_c condition activated by command")
-SENSE_CODE(0x5E09, "Standby_y condition activated by timer")
-SENSE_CODE(0x5E0A, "Standby_y condition activated by command")
-SENSE_CODE(0x5E41, "Power state change to active")
-SENSE_CODE(0x5E42, "Power state change to idle")
-SENSE_CODE(0x5E43, "Power state change to standby")
-SENSE_CODE(0x5E45, "Power state change to sleep")
-SENSE_CODE(0x5E47, "Power state change to device control")
-
-SENSE_CODE(0x6000, "Lamp failure")
-
-SENSE_CODE(0x6100, "Video acquisition error")
-SENSE_CODE(0x6101, "Unable to acquire video")
-SENSE_CODE(0x6102, "Out of focus")
-
-SENSE_CODE(0x6200, "Scan head positioning error")
-
-SENSE_CODE(0x6300, "End of user area encountered on this track")
-SENSE_CODE(0x6301, "Packet does not fit in available space")
-
-SENSE_CODE(0x6400, "Illegal mode for this track")
-SENSE_CODE(0x6401, "Invalid packet size")
-
-SENSE_CODE(0x6500, "Voltage fault")
-
-SENSE_CODE(0x6600, "Automatic document feeder cover up")
-SENSE_CODE(0x6601, "Automatic document feeder lift up")
-SENSE_CODE(0x6602, "Document jam in automatic document feeder")
-SENSE_CODE(0x6603, "Document miss feed automatic in document feeder")
-
-SENSE_CODE(0x6700, "Configuration failure")
-SENSE_CODE(0x6701, "Configuration of incapable logical units failed")
-SENSE_CODE(0x6702, "Add logical unit failed")
-SENSE_CODE(0x6703, "Modification of logical unit failed")
-SENSE_CODE(0x6704, "Exchange of logical unit failed")
-SENSE_CODE(0x6705, "Remove of logical unit failed")
-SENSE_CODE(0x6706, "Attachment of logical unit failed")
-SENSE_CODE(0x6707, "Creation of logical unit failed")
-SENSE_CODE(0x6708, "Assign failure occurred")
-SENSE_CODE(0x6709, "Multiply assigned logical unit")
-SENSE_CODE(0x670A, "Set target port groups command failed")
-SENSE_CODE(0x670B, "ATA device feature not enabled")
-SENSE_CODE(0x670C, "Command rejected")
-SENSE_CODE(0x670D, "Explicit bind not allowed")
-
-SENSE_CODE(0x6800, "Logical unit not configured")
-SENSE_CODE(0x6801, "Subsidiary logical unit not configured")
-
-SENSE_CODE(0x6900, "Data loss on logical unit")
-SENSE_CODE(0x6901, "Multiple logical unit failures")
-SENSE_CODE(0x6902, "Parity/data mismatch")
-
-SENSE_CODE(0x6A00, "Informational, refer to log")
-
-SENSE_CODE(0x6B00, "State change has occurred")
-SENSE_CODE(0x6B01, "Redundancy level got better")
-SENSE_CODE(0x6B02, "Redundancy level got worse")
-
-SENSE_CODE(0x6C00, "Rebuild failure occurred")
-
-SENSE_CODE(0x6D00, "Recalculate failure occurred")
-
-SENSE_CODE(0x6E00, "Command to logical unit failed")
-
-SENSE_CODE(0x6F00, "Copy protection key exchange failure - authentication failure")
-SENSE_CODE(0x6F01, "Copy protection key exchange failure - key not present")
-SENSE_CODE(0x6F02, "Copy protection key exchange failure - key not established")
-SENSE_CODE(0x6F03, "Read of scrambled sector without authentication")
-SENSE_CODE(0x6F04, "Media region code is mismatched to logical unit region")
-SENSE_CODE(0x6F05, "Drive region must be permanent/region reset count error")
-SENSE_CODE(0x6F06, "Insufficient block count for binding nonce recording")
-SENSE_CODE(0x6F07, "Conflict in binding nonce recording")
-SENSE_CODE(0x6F08, "Insufficient permission")
-SENSE_CODE(0x6F09, "Invalid drive-host pairing server")
-SENSE_CODE(0x6F0A, "Drive-host pairing suspended")
+
+SENSE_CODE(OVERLAPPED_COMMANDS_ATTEMPTED,
+	   "Overlapped commands attempted")
+
+SENSE_CODE(WRITE_APPEND_ERROR,
+	   "Write append error")
+SENSE_CODE(WRITE_APPEND_POSITION_ERROR,
+	   "Write append position error")
+SENSE_CODE(POSITION_ERROR_RELATED_TO_TIMING,
+	   "Position error related to timing")
+
+SENSE_CODE(ERASE_FAILURE,
+	   "Erase failure")
+SENSE_CODE(ERASE_FAILURE_INCOMPLETE_ERASE_OP_DETECTED,
+	   "Erase failure - incomplete erase operation detected")
+
+SENSE_CODE(CARTRIDGE_FAULT,
+	   "Cartridge fault")
+
+SENSE_CODE(MEDIA_LOAD_OR_EJECT_FAILED,
+	   "Media load or eject failed")
+SENSE_CODE(UNLOAD_TAPE_FAILURE,
+	   "Unload tape failure")
+SENSE_CODE(MEDIUM_REMOVAL_PREVENTED,
+	   "Medium removal prevented")
+SENSE_CODE(MEDIUM_REMOVAL_PREVENTED_BY_DATA_TRANSFER_ELEMENT,
+	   "Medium removal prevented by data transfer element")
+SENSE_CODE(MEDIUM_THREAD_OR_UNTHREAD_FAILURE,
+	   "Medium thread or unthread failure")
+SENSE_CODE(VOLUME_IDENTIFIER_INVALID,
+	   "Volume identifier invalid")
+SENSE_CODE(VOLUME_IDENTIFIER_MISSING,
+	   "Volume identifier missing")
+SENSE_CODE(DUPLICATE_VOLUME_IDENTIFIER,
+	   "Duplicate volume identifier")
+SENSE_CODE(ELEMENT_STATUS_UNKNOWN,
+	   "Element status unknown")
+SENSE_CODE(DATA_TRANSFER_DEVICE_ERR_LOAD_FAILED,
+	   "Data transfer device error - load failed")
+SENSE_CODE(DATA_TRANSFER_DEVICE_ERR_UNLOAD_FAILED,
+	   "Data transfer device error - unload failed")
+SENSE_CODE(DATA_TRANSFER_DEVICE_ERR_UNLOAD_MISSING,
+	   "Data transfer device error - unload missing")
+SENSE_CODE(DATA_TRANSFER_DEVICE_ERR_EJECT_FAILED,
+	   "Data transfer device error - eject failed")
+SENSE_CODE(DATA_TRANSFER_DEVICE_ERR_LIBRARY_COMMUNICATION_FAILED,
+	   "Data transfer device error - library communication failed")
+
+SENSE_CODE(SCSI_TO_HOST_SYSTEM_INTERFACE_FAILURE,
+	   "Scsi to host system interface failure")
+
+SENSE_CODE(SYSTEM_RESOURCE_FAILURE,
+	   "System resource failure")
+SENSE_CODE(SYSTEM_BUFFER_FULL,
+	   "System buffer full")
+SENSE_CODE(INSUFFICIENT_RESERVATION_RESOURCES,
+	   "Insufficient reservation resources")
+SENSE_CODE(INSUFFICIENT_RESOURCES,
+	   "Insufficient resources")
+SENSE_CODE(INSUFFICIENT_REGISTRATION_RESOURCES,
+	   "Insufficient registration resources")
+SENSE_CODE(INSUFFICIENT_ACCESS_CONTROL_RESOURCES,
+	   "Insufficient access control resources")
+SENSE_CODE(AUXILIARY_MEMORY_OUT_OF_SPACE,
+	   "Auxiliary memory out of space")
+SENSE_CODE(QUOTA_ERROR,
+	   "Quota error")
+SENSE_CODE(MAXIMUM_NUMBER_OF_SUPPLEMENTAL_DECRYPTION_KEYS_EXCEEDED,
+	   "Maximum number of supplemental decryption keys exceeded")
+SENSE_CODE(MEDIUM_AUXILIARY_MEMORY_NOT_ACCESSIBLE,
+	   "Medium auxiliary memory not accessible")
+SENSE_CODE(DATA_CURRENTLY_UNAVAILABLE,
+	   "Data currently unavailable")
+SENSE_CODE(INSUFFICIENT_POWER_FOR_OPERATION,
+	   "Insufficient power for operation")
+SENSE_CODE(INSUFFICIENT_RESOURCES_TO_CREATE_ROD,
+	   "Insufficient resources to create rod")
+SENSE_CODE(INSUFFICIENT_RESOURCES_TO_CREATE_ROD_TOKEN,
+	   "Insufficient resources to create rod token")
+SENSE_CODE(INSUFFICIENT_ZONE_RESOURCES,
+	   "Insufficient zone resources")
+SENSE_CODE(INSUFFICIENT_ZONE_RESOURCES_TO_COMPLETE_WRITE,
+	   "Insufficient zone resources to complete write")
+SENSE_CODE(MAXIMUM_NUMBER_OF_STREAMS_OPEN,
+	   "Maximum number of streams open")
+SENSE_CODE(INSUFFICIENT_RESOURCES_TO_BIND,
+	   "Insufficient resources to bind")
+
+SENSE_CODE(UNABLE_TO_RECOVER_TABLE_OF_CONTENTS,
+	   "Unable to recover table-of-contents")
+
+SENSE_CODE(GENERATION_DOES_NOT_EXIST,
+	   "Generation does not exist")
+
+SENSE_CODE(UPDATED_BLOCK_READ,
+	   "Updated block read")
+
+SENSE_CODE(OPERATOR_REQUEST_OR_STATE_CHANGE_INPUT,
+	   "Operator request or state change input")
+SENSE_CODE(OPERATOR_MEDIUM_REMOVAL_REQUEST,
+	   "Operator medium removal request")
+SENSE_CODE(OPERATOR_SELECTED_WRITE_PROTECT,
+	   "Operator selected write protect")
+SENSE_CODE(OPERATOR_SELECTED_WRITE_PERMIT,
+	   "Operator selected write permit")
+
+SENSE_CODE(LOG_EXCEPTION,
+	   "Log exception")
+SENSE_CODE(THRESHOLD_CONDITION_MET,
+	   "Threshold condition met")
+SENSE_CODE(LOG_COUNTER_AT_MAXIMUM,
+	   "Log counter at maximum")
+SENSE_CODE(LOG_LIST_CODES_EXHAUSTED,
+	   "Log list codes exhausted")
+
+SENSE_CODE(RPL_STATUS_CHANGE,
+	   "Rpl status change")
+SENSE_CODE(SPINDLES_SYNCHRONIZED,
+	   "Spindles synchronized")
+SENSE_CODE(SPINDLES_NOT_SYNCHRONIZED,
+	   "Spindles not synchronized")
+
+SENSE_CODE(FAILURE_PREDICTION_THRESHOLD_EXCEEDED,
+	   "Failure prediction threshold exceeded")
+SENSE_CODE(MEDIA_FAILURE_PREDICTION_THRESHOLD_EXCEEDED,
+	   "Media failure prediction threshold exceeded")
+SENSE_CODE(LU_FAILURE_PREDICTION_THRESHOLD_EXCEEDED,
+	   "Logical unit failure prediction threshold exceeded")
+SENSE_CODE(SPARE_AREA_EXHAUSTION_PREDICTION_THRESHOLD_EXCEEDED,
+	   "Spare area exhaustion prediction threshold exceeded")
+SENSE_CODE(HW_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE,
+	   "Hardware impending failure general hard drive failure")
+SENSE_CODE(HW_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH,
+	   "Hardware impending failure drive error rate too high")
+SENSE_CODE(HW_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH,
+	   "Hardware impending failure data error rate too high")
+SENSE_CODE(HW_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH,
+	   "Hardware impending failure seek error rate too high")
+SENSE_CODE(HW_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS,
+	   "Hardware impending failure too many block reassigns")
+SENSE_CODE(HW_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH,
+	   "Hardware impending failure access times too high")
+SENSE_CODE(HW_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH,
+	   "Hardware impending failure start unit times too high")
+SENSE_CODE(HW_IMPENDING_FAILURE_CHANNEL_PARAMETRICS,
+	   "Hardware impending failure channel parametrics")
+SENSE_CODE(HW_IMPENDING_FAILURE_CONTROLLER_DETECTED,
+	   "Hardware impending failure controller detected")
+SENSE_CODE(HW_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE,
+	   "Hardware impending failure throughput performance")
+SENSE_CODE(HW_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE,
+	   "Hardware impending failure seek time performance")
+SENSE_CODE(HW_IMPENDING_FAILURE_SPINUP_RETRY_COUNT,
+	   "Hardware impending failure spin-up retry count")
+SENSE_CODE(HW_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT,
+	   "Hardware impending failure drive calibration retry count")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE,
+	   "Controller impending failure general hard drive failure")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH,
+	   "Controller impending failure drive error rate too high")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH,
+	   "Controller impending failure data error rate too high")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH,
+	   "Controller impending failure seek error rate too high")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS,
+	   "Controller impending failure too many block reassigns")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH,
+	   "Controller impending failure access times too high")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH,
+	   "Controller impending failure start unit times too high")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_CHANNEL_PARAMETRICS,
+	   "Controller impending failure channel parametrics")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_CONTROLLER_DETECTED,
+	   "Controller impending failure controller detected")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE,
+	   "Controller impending failure throughput performance")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE,
+	   "Controller impending failure seek time performance")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_SPINUP_RETRY_COUNT,
+	   "Controller impending failure spin-up retry count")
+SENSE_CODE(CONTROLLER_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT,
+	   "Controller impending failure drive calibration retry count")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE,
+	   "Data channel impending failure general hard drive failure")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH,
+	   "Data channel impending failure drive error rate too high")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH,
+	   "Data channel impending failure data error rate too high")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH,
+	   "Data channel impending failure seek error rate too high")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS,
+	   "Data channel impending failure too many block reassigns")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH,
+	   "Data channel impending failure access times too high")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH,
+	   "Data channel impending failure start unit times too high")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_CHANNEL_PARAMETRICS,
+	   "Data channel impending failure channel parametrics")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_CONTROLLER_DETECTED,
+	   "Data channel impending failure controller detected")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE,
+	   "Data channel impending failure throughput performance")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE,
+	   "Data channel impending failure seek time performance")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_SPINUP_RETRY_COUNT,
+	   "Data channel impending failure spin-up retry count")
+SENSE_CODE(DATA_CHANNEL_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT,
+	   "Data channel impending failure drive calibration retry count")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE,
+	   "Servo impending failure general hard drive failure")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH,
+	   "Servo impending failure drive error rate too high")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH,
+	   "Servo impending failure data error rate too high")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH,
+	   "Servo impending failure seek error rate too high")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS,
+	   "Servo impending failure too many block reassigns")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH,
+	   "Servo impending failure access times too high")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH,
+	   "Servo impending failure start unit times too high")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_CHANNEL_PARAMETRICS,
+	   "Servo impending failure channel parametrics")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_CONTROLLER_DETECTED,
+	   "Servo impending failure controller detected")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE,
+	   "Servo impending failure throughput performance")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE,
+	   "Servo impending failure seek time performance")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_SPINUP_RETRY_COUNT,
+	   "Servo impending failure spin-up retry count")
+SENSE_CODE(SERVO_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT,
+	   "Servo impending failure drive calibration retry count")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE,
+	   "Spindle impending failure general hard drive failure")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH,
+	   "Spindle impending failure drive error rate too high")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH,
+	   "Spindle impending failure data error rate too high")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH,
+	   "Spindle impending failure seek error rate too high")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS,
+	   "Spindle impending failure too many block reassigns")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH,
+	   "Spindle impending failure access times too high")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH,
+	   "Spindle impending failure start unit times too high")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_CHANNEL_PARAMETRICS,
+	   "Spindle impending failure channel parametrics")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_CONTROLLER_DETECTED,
+	   "Spindle impending failure controller detected")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE,
+	   "Spindle impending failure throughput performance")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE,
+	   "Spindle impending failure seek time performance")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_SPINUP_RETRY_COUNT,
+	   "Spindle impending failure spin-up retry count")
+SENSE_CODE(SPINDLE_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT,
+	   "Spindle impending failure drive calibration retry count")
+SENSE_CODE(FW_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE,
+	   "Firmware impending failure general hard drive failure")
+SENSE_CODE(FW_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH,
+	   "Firmware impending failure drive error rate too high")
+SENSE_CODE(FW_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH,
+	   "Firmware impending failure data error rate too high")
+SENSE_CODE(FW_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH,
+	   "Firmware impending failure seek error rate too high")
+SENSE_CODE(FW_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS,
+	   "Firmware impending failure too many block reassigns")
+SENSE_CODE(FW_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH,
+	   "Firmware impending failure access times too high")
+SENSE_CODE(FW_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH,
+	   "Firmware impending failure start unit times too high")
+SENSE_CODE(FW_IMPENDING_FAILURE_CHANNEL_PARAMETRICS,
+	   "Firmware impending failure channel parametrics")
+SENSE_CODE(FW_IMPENDING_FAILURE_CONTROLLER_DETECTED,
+	   "Firmware impending failure controller detected")
+SENSE_CODE(FW_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE,
+	   "Firmware impending failure throughput performance")
+SENSE_CODE(FW_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE,
+	   "Firmware impending failure seek time performance")
+SENSE_CODE(FW_IMPENDING_FAILURE_SPINUP_RETRY_COUNT,
+	   "Firmware impending failure spin-up retry count")
+SENSE_CODE(FW_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT,
+	   "Firmware impending failure drive calibration retry count")
+SENSE_CODE(MEDIA_IMPENDING_FAILURE_ENDURANCE_LIMIT_MET,
+	   "Media impending failure endurance limit met")
+SENSE_CODE(FAILURE_PREDICTION_THRESHOLD_EXCEEDED_FALSE,
+	   "Failure prediction threshold exceeded (false)")
+
+SENSE_CODE(LOW_POWER_CONDITION_ON,
+	   "Low power condition on")
+SENSE_CODE(IDLE_CONDITION_ACTIVATED_BY_TIMER,
+	   "Idle condition activated by timer")
+SENSE_CODE(STANDBY_CONDITION_ACTIVATED_BY_TIMER,
+	   "Standby condition activated by timer")
+SENSE_CODE(IDLE_CONDITION_ACTIVATED_BY_COMMAND,
+	   "Idle condition activated by command")
+SENSE_CODE(STANDBY_CONDITION_ACTIVATED_BY_COMMAND,
+	   "Standby condition activated by command")
+SENSE_CODE(IDLE_B_CONDITION_ACTIVATED_BY_TIMER,
+	   "Idle_b condition activated by timer")
+SENSE_CODE(IDLE_B_CONDITION_ACTIVATED_BY_COMMAND,
+	   "Idle_b condition activated by command")
+SENSE_CODE(IDLE_C_CONDITION_ACTIVATED_BY_TIMER,
+	   "Idle_c condition activated by timer")
+SENSE_CODE(IDLE_C_CONDITION_ACTIVATED_BY_COMMAND,
+	   "Idle_c condition activated by command")
+SENSE_CODE(STANDBY_Y_CONDITION_ACTIVATED_BY_TIMER,
+	   "Standby_y condition activated by timer")
+SENSE_CODE(STANDBY_Y_CONDITION_ACTIVATED_BY_COMMAND,
+	   "Standby_y condition activated by command")
+SENSE_CODE(POWER_STATE_CHANGE_TO_ACTIVE,
+	   "Power state change to active")
+SENSE_CODE(POWER_STATE_CHANGE_TO_IDLE,
+	   "Power state change to idle")
+SENSE_CODE(POWER_STATE_CHANGE_TO_STANDBY,
+	   "Power state change to standby")
+SENSE_CODE(POWER_STATE_CHANGE_TO_SLEEP,
+	   "Power state change to sleep")
+SENSE_CODE(POWER_STATE_CHANGE_TO_DEVICE_CONTROL,
+	   "Power state change to device control")
+
+SENSE_CODE(LAMP_FAILURE,
+	   "Lamp failure")
+
+SENSE_CODE(VIDEO_ACQUISITION_ERROR,
+	   "Video acquisition error")
+SENSE_CODE(UNABLE_TO_ACQUIRE_VIDEO,
+	   "Unable to acquire video")
+SENSE_CODE(OUT_OF_FOCUS,
+	   "Out of focus")
+
+SENSE_CODE(SCAN_HEAD_POSITIONING_ERROR,
+	   "Scan head positioning error")
+
+SENSE_CODE(END_OF_USER_AREA_ENCOUNTERED_ON_THIS_TRACK,
+	   "End of user area encountered on this track")
+SENSE_CODE(PACKET_DOES_NOT_FIT_IN_AVAILABLE_SPACE,
+	   "Packet does not fit in available space")
+
+SENSE_CODE(ILLEGAL_MODE_FOR_THIS_TRACK,
+	   "Illegal mode for this track")
+SENSE_CODE(INVALID_PACKET_SIZE,
+	   "Invalid packet size")
+
+SENSE_CODE(VOLTAGE_FAULT,
+	   "Voltage fault")
+
+SENSE_CODE(AUTOMATIC_DOCUMENT_FEEDER_COVER_UP,
+	   "Automatic document feeder cover up")
+SENSE_CODE(AUTOMATIC_DOCUMENT_FEEDER_LIFT_UP,
+	   "Automatic document feeder lift up")
+SENSE_CODE(DOCUMENT_JAM_IN_AUTOMATIC_DOCUMENT_FEEDER,
+	   "Document jam in automatic document feeder")
+SENSE_CODE(DOCUMENT_MISS_FEED_AUTOMATIC_IN_DOCUMENT_FEEDER,
+	   "Document miss feed automatic in document feeder")
+
+SENSE_CODE(CONFIG_FAILURE,
+	   "Configuration failure")
+SENSE_CODE(CONFIG_OF_INCAPABLE_LOGICAL_UNITS_FAILED,
+	   "Configuration of incapable logical units failed")
+SENSE_CODE(ADD_LU_FAILED,
+	   "Add logical unit failed")
+SENSE_CODE(MODIFICATION_OF_LU_FAILED,
+	   "Modification of logical unit failed")
+SENSE_CODE(EXCHANGE_OF_LU_FAILED,
+	   "Exchange of logical unit failed")
+SENSE_CODE(REMOVE_OF_LU_FAILED,
+	   "Remove of logical unit failed")
+SENSE_CODE(ATTACHMENT_OF_LU_FAILED,
+	   "Attachment of logical unit failed")
+SENSE_CODE(CREATION_OF_LU_FAILED,
+	   "Creation of logical unit failed")
+SENSE_CODE(ASSIGN_FAILURE_OCCURRED,
+	   "Assign failure occurred")
+SENSE_CODE(MULTIPLY_ASSIGNED_LOGICAL_UNIT,
+	   "Multiply assigned logical unit")
+SENSE_CODE(SET_TARGET_PORT_GROUPS_COMMAND_FAILED,
+	   "Set target port groups command failed")
+SENSE_CODE(ATA_DEVICE_FEATURE_NOT_ENABLED,
+	   "ATA device feature not enabled")
+SENSE_CODE(COMMAND_REJECTED,
+	   "Command rejected")
+SENSE_CODE(EXPLICIT_BIND_NOT_ALLOWED,
+	   "Explicit bind not allowed")
+
+SENSE_CODE(LU_NOT_CONFIGURED,
+	   "Logical unit not configured")
+SENSE_CODE(SUBSIDIARY_LU_NOT_CONFIGURED,
+	   "Subsidiary logical unit not configured")
+
+SENSE_CODE(DATA_LOSS_ON_LOGICAL_UNIT,
+	   "Data loss on logical unit")
+SENSE_CODE(MULTIPLE_LU_FAILURES,
+	   "Multiple logical unit failures")
+SENSE_CODE(PARITY_DATA_MISMATCH,
+	   "Parity/data mismatch")
+
+SENSE_CODE(INFORMATIONAL_REFER_TO_LOG,
+	   "Informational, refer to log")
+
+SENSE_CODE(STATE_CHANGE_HAS_OCCURRED,
+	   "State change has occurred")
+SENSE_CODE(REDUNDANCY_LEVEL_GOT_BETTER,
+	   "Redundancy level got better")
+SENSE_CODE(REDUNDANCY_LEVEL_GOT_WORSE,
+	   "Redundancy level got worse")
+
+SENSE_CODE(REBUILD_FAILURE_OCCURRED,
+	   "Rebuild failure occurred")
+
+SENSE_CODE(RECALCULATE_FAILURE_OCCURRED,
+	   "Recalculate failure occurred")
+
+SENSE_CODE(COMMAND_TO_LU_FAILED,
+	   "Command to logical unit failed")
+
+SENSE_CODE(COPY_PROTECTION_KEY_EXCHANGE_FAILURE_AUTHENTICATION_FAILURE,
+	   "Copy protection key exchange failure - authentication failure")
+SENSE_CODE(COPY_PROTECTION_KEY_EXCHANGE_FAILURE_KEY_NOT_PRESENT,
+	   "Copy protection key exchange failure - key not present")
+SENSE_CODE(COPY_PROTECTION_KEY_EXCHANGE_FAILURE_KEY_NOT_ESTABLISHED,
+	   "Copy protection key exchange failure - key not established")
+SENSE_CODE(READ_OF_SCRAMBLED_SECTOR_WITHOUT_AUTHENTICATION,
+	   "Read of scrambled sector without authentication")
+SENSE_CODE(MEDIA_REGION_CODE_IS_MISMATCHED_TO_LU_REGION,
+	   "Media region code is mismatched to logical unit region")
+SENSE_CODE(DRIVE_REGION_MUST_BE_PERMANENT_REGION_RESET_COUNT_ERROR,
+	   "Drive region must be permanent/region reset count error")
+SENSE_CODE(INSUFFICIENT_BLOCK_COUNT_FOR_BINDING_NONCE_RECORDING,
+	   "Insufficient block count for binding nonce recording")
+SENSE_CODE(CONFLICT_IN_BINDING_NONCE_RECORDING,
+	   "Conflict in binding nonce recording")
+SENSE_CODE(INSUFFICIENT_PERMISSION,
+	   "Insufficient permission")
+SENSE_CODE(INVALID_DRIVE_HOST_PAIRING_SERVER,
+	   "Invalid drive-host pairing server")
+SENSE_CODE(DRIVE_HOST_PAIRING_SUSPENDED,
+	   "Drive-host pairing suspended")
 
 /*
- *	SENSE_CODE(0x70NN, "Decompression exception short algorithm id of nn")
+ * SENSE_CODE(0x70NN, "Decompression exception short algorithm id of nn")
  */
-SENSE_CODE(0x7100, "Decompression exception long algorithm id")
-
-SENSE_CODE(0x7200, "Session fixation error")
-SENSE_CODE(0x7201, "Session fixation error writing lead-in")
-SENSE_CODE(0x7202, "Session fixation error writing lead-out")
-SENSE_CODE(0x7203, "Session fixation error - incomplete track in session")
-SENSE_CODE(0x7204, "Empty or partially written reserved track")
-SENSE_CODE(0x7205, "No more track reservations allowed")
-SENSE_CODE(0x7206, "RMZ extension is not allowed")
-SENSE_CODE(0x7207, "No more test zone extensions are allowed")
-
-SENSE_CODE(0x7300, "Cd control error")
-SENSE_CODE(0x7301, "Power calibration area almost full")
-SENSE_CODE(0x7302, "Power calibration area is full")
-SENSE_CODE(0x7303, "Power calibration area error")
-SENSE_CODE(0x7304, "Program memory area update failure")
-SENSE_CODE(0x7305, "Program memory area is full")
-SENSE_CODE(0x7306, "RMA/PMA is almost full")
-SENSE_CODE(0x7310, "Current power calibration area almost full")
-SENSE_CODE(0x7311, "Current power calibration area is full")
-SENSE_CODE(0x7317, "RDZ is full")
-
-SENSE_CODE(0x7400, "Security error")
-SENSE_CODE(0x7401, "Unable to decrypt data")
-SENSE_CODE(0x7402, "Unencrypted data encountered while decrypting")
-SENSE_CODE(0x7403, "Incorrect data encryption key")
-SENSE_CODE(0x7404, "Cryptographic integrity validation failed")
-SENSE_CODE(0x7405, "Error decrypting data")
-SENSE_CODE(0x7406, "Unknown signature verification key")
-SENSE_CODE(0x7407, "Encryption parameters not useable")
-SENSE_CODE(0x7408, "Digital signature validation failure")
-SENSE_CODE(0x7409, "Encryption mode mismatch on read")
-SENSE_CODE(0x740A, "Encrypted block not raw read enabled")
-SENSE_CODE(0x740B, "Incorrect Encryption parameters")
-SENSE_CODE(0x740C, "Unable to decrypt parameter list")
-SENSE_CODE(0x740D, "Encryption algorithm disabled")
-SENSE_CODE(0x7410, "SA creation parameter value invalid")
-SENSE_CODE(0x7411, "SA creation parameter value rejected")
-SENSE_CODE(0x7412, "Invalid SA usage")
-SENSE_CODE(0x7421, "Data Encryption configuration prevented")
-SENSE_CODE(0x7430, "SA creation parameter not supported")
-SENSE_CODE(0x7440, "Authentication failed")
-SENSE_CODE(0x7461, "External data encryption key manager access error")
-SENSE_CODE(0x7462, "External data encryption key manager error")
-SENSE_CODE(0x7463, "External data encryption key not found")
-SENSE_CODE(0x7464, "External data encryption request not authorized")
-SENSE_CODE(0x746E, "External data encryption control timeout")
-SENSE_CODE(0x746F, "External data encryption control error")
-SENSE_CODE(0x7471, "Logical unit access not authorized")
-SENSE_CODE(0x7479, "Security conflict in translated device")
+
+SENSE_CODE(DECOMPRESSION_EXCEPTION_LONG_ALGORITHM_ID,
+	   "Decompression exception long algorithm id")
+
+SENSE_CODE(SESSION_FIXATION_ERROR,
+	   "Session fixation error")
+SENSE_CODE(SESSION_FIXATION_ERROR_WRITING_LEAD_IN,
+	   "Session fixation error writing lead-in")
+SENSE_CODE(SESSION_FIXATION_ERROR_WRITING_LEAD_OUT,
+	   "Session fixation error writing lead-out")
+SENSE_CODE(SESSION_FIXATION_ERROR_INCOMPLETE_TRACK_IN_SESSION,
+	   "Session fixation error - incomplete track in session")
+SENSE_CODE(EMPTY_OR_PARTIALLY_WRITTEN_RESERVED_TRACK,
+	   "Empty or partially written reserved track")
+SENSE_CODE(NO_MORE_TRACK_RESERVATIONS_ALLOWED,
+	   "No more track reservations allowed")
+SENSE_CODE(RMZ_EXTENSION_IS_NOT_ALLOWED,
+	   "RMZ extension is not allowed")
+SENSE_CODE(NO_MORE_TEST_ZONE_EXTENSIONS_ARE_ALLOWED,
+	   "No more test zone extensions are allowed")
+
+SENSE_CODE(CD_CONTROL_ERROR,
+	   "Cd control error")
+SENSE_CODE(POWER_CALIBRATION_AREA_ALMOST_FULL,
+	   "Power calibration area almost full")
+SENSE_CODE(POWER_CALIBRATION_AREA_IS_FULL,
+	   "Power calibration area is full")
+SENSE_CODE(POWER_CALIBRATION_AREA_ERROR,
+	   "Power calibration area error")
+SENSE_CODE(PROGRAM_MEMORY_AREA_UPDATE_FAILURE,
+	   "Program memory area update failure")
+SENSE_CODE(PROGRAM_MEMORY_AREA_IS_FULL,
+	   "Program memory area is full")
+SENSE_CODE(RMA_PMA_IS_ALMOST_FULL,
+	   "RMA/PMA is almost full")
+SENSE_CODE(CURRENT_POWER_CALIBRATION_AREA_ALMOST_FULL,
+	   "Current power calibration area almost full")
+SENSE_CODE(CURRENT_POWER_CALIBRATION_AREA_IS_FULL,
+	   "Current power calibration area is full")
+SENSE_CODE(RDZ_IS_FULL,
+	   "RDZ is full")
+
+SENSE_CODE(SECURITY_ERROR,
+	   "Security error")
+SENSE_CODE(UNABLE_TO_DECRYPT_DATA,
+	   "Unable to decrypt data")
+SENSE_CODE(UNENCRYPTED_DATA_ENCOUNTERED_WHILE_DECRYPTING,
+	   "Unencrypted data encountered while decrypting")
+SENSE_CODE(INCORRECT_DATA_ENCRYPTION_KEY,
+	   "Incorrect data encryption key")
+SENSE_CODE(CRYPTOGRAPHIC_INTEGRITY_VALIDATION_FAILED,
+	   "Cryptographic integrity validation failed")
+SENSE_CODE(ERROR_DECRYPTING_DATA,
+	   "Error decrypting data")
+SENSE_CODE(UNKNOWN_SIGNATURE_VERIFICATION_KEY,
+	   "Unknown signature verification key")
+SENSE_CODE(ENCRYPTION_PARAMS_NOT_USEABLE,
+	   "Encryption parameters not useable")
+SENSE_CODE(DIGITAL_SIGNATURE_VALIDATION_FAILURE,
+	   "Digital signature validation failure")
+SENSE_CODE(ENCRYPTION_MODE_MISMATCH_ON_READ,
+	   "Encryption mode mismatch on read")
+SENSE_CODE(ENCRYPTED_BLOCK_NOT_RAW_READ_ENABLED,
+	   "Encrypted block not raw read enabled")
+SENSE_CODE(INCORRECT_ENCRYPTION_PARAMS,
+	   "Incorrect Encryption parameters")
+SENSE_CODE(UNABLE_TO_DECRYPT_PARAMETER_LIST,
+	   "Unable to decrypt parameter list")
+SENSE_CODE(ENCRYPTION_ALGORITHM_DISABLED,
+	   "Encryption algorithm disabled")
+SENSE_CODE(SA_CREATION_PARAMETER_VALUE_INVALID,
+	   "SA creation parameter value invalid")
+SENSE_CODE(SA_CREATION_PARAMETER_VALUE_REJECTED,
+	   "SA creation parameter value rejected")
+SENSE_CODE(INVALID_SA_USAGE,
+	   "Invalid SA usage")
+SENSE_CODE(DATA_ENCRYPTION_CONFIG_PREVENTED,
+	   "Data Encryption configuration prevented")
+SENSE_CODE(SA_CREATION_PARAMETER_NOT_SUPPORTED,
+	   "SA creation parameter not supported")
+SENSE_CODE(AUTHENTICATION_FAILED,
+	   "Authentication failed")
+SENSE_CODE(EXTERNAL_DATA_ENCRYPTION_KEY_MANAGER_ACCESS_ERROR,
+	   "External data encryption key manager access error")
+SENSE_CODE(EXTERNAL_DATA_ENCRYPTION_KEY_MANAGER_ERROR,
+	   "External data encryption key manager error")
+SENSE_CODE(EXTERNAL_DATA_ENCRYPTION_KEY_NOT_FOUND,
+	   "External data encryption key not found")
+SENSE_CODE(EXTERNAL_DATA_ENCRYPTION_REQUEST_NOT_AUTHORIZED,
+	   "External data encryption request not authorized")
+SENSE_CODE(EXTERNAL_DATA_ENCRYPTION_CONTROL_TIMEOUT,
+	   "External data encryption control timeout")
+SENSE_CODE(EXTERNAL_DATA_ENCRYPTION_CONTROL_ERROR,
+	   "External data encryption control error")
+SENSE_CODE(LU_ACCESS_NOT_AUTHORIZED,
+	   "Logical unit access not authorized")
+SENSE_CODE(SECURITY_CONFLICT_IN_TRANSLATED_DEVICE,
+	   "Security conflict in translated device")
diff --git a/drivers/scsi/ses.c b/drivers/scsi/ses.c
index 4c348645b04ee4..eefeb8b9a75cad 100644
--- a/drivers/scsi/ses.c
+++ b/drivers/scsi/ses.c
@@ -72,6 +72,9 @@ static void init_device_slot_control(unsigned char *dest_desc,
 	dest_desc[3] &= 0x3c;
 }
 
+#define POWER_ON_RESET_ANY						\
+	scsi_sense_code(ASC_POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED, \
+			SCMD_FAILURE_ASCQ_ANY)
 
 static int ses_recv_diag(struct scsi_device *sdev, int page_code,
 			 void *buf, int bufflen)
@@ -88,16 +91,14 @@ static int ses_recv_diag(struct scsi_device *sdev, int page_code,
 	unsigned char recv_page_code;
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = POWER_ON_RESET_ANY,
 			.allowed = SES_RETRIES,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = NOT_READY,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = NOT_READY,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = SES_RETRIES,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
@@ -145,16 +146,14 @@ static int ses_send_diag(struct scsi_device *sdev, int page_code,
 	};
 	struct scsi_failure failure_defs[] = {
 		{
-			.sense = UNIT_ATTENTION,
-			.asc = 0x29,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = UNIT_ATTENTION,
+			.sense_code = POWER_ON_RESET_ANY,
 			.allowed = SES_RETRIES,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
 		{
-			.sense = NOT_READY,
-			.asc = SCMD_FAILURE_ASC_ANY,
-			.ascq = SCMD_FAILURE_ASCQ_ANY,
+			.sense_key = NOT_READY,
+			.sense_code = SCMD_FAILURE_SENSE_CODE_ANY,
 			.allowed = SES_RETRIES,
 			.result = SAM_STAT_CHECK_CONDITION,
 		},
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index 5408f002e6c01f..3f9e08725602ca 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -1212,85 +1212,72 @@ sg_fasync(int fd, struct file *filp, int mode)
 	return fasync_helper(fd, filp, mode, &sfp->async_qp);
 }
 
-static vm_fault_t
-sg_vma_fault(struct vm_fault *vmf)
+static int sg_discontig_init(void *vm_private_data, void **private)
 {
-	struct vm_area_struct *vma = vmf->vma;
-	Sg_fd *sfp;
-	unsigned long offset, len, sa;
-	Sg_scatter_hold *rsv_schp;
-	int k, length;
-
-	if ((NULL == vma) || (!(sfp = (Sg_fd *) vma->vm_private_data)))
-		return VM_FAULT_SIGBUS;
-	rsv_schp = &sfp->reserve;
-	offset = vmf->pgoff << PAGE_SHIFT;
-	if (offset >= rsv_schp->bufflen)
-		return VM_FAULT_SIGBUS;
-	SCSI_LOG_TIMEOUT(3, sg_printk(KERN_INFO, sfp->parentdp,
-				      "sg_vma_fault: offset=%lu, scatg=%d\n",
-				      offset, rsv_schp->k_use_sg));
-	sa = vma->vm_start;
-	length = 1 << (PAGE_SHIFT + rsv_schp->page_order);
-	for (k = 0; k < rsv_schp->k_use_sg && sa < vma->vm_end; k++) {
-		len = vma->vm_end - sa;
-		len = (len < length) ? len : length;
-		if (offset < len) {
-			struct page *page = rsv_schp->pages[k] + (offset >> PAGE_SHIFT);
-			get_page(page);	/* increment page count */
-			vmf->page = page;
-			return 0; /* success */
-		}
-		sa += len;
-		offset -= len;
+	const unsigned long req_sz = (unsigned long)*private;
+	Sg_fd *sfp = vm_private_data;
+	Sg_scatter_hold *rsv_schp = &sfp->reserve;
+	int err = 0;
+
+	mutex_lock(&sfp->f_mutex);
+	if (req_sz > rsv_schp->bufflen) {
+		err = -ENOMEM;	/* cannot map more than reserved buffer */
+		goto out;
+	}
+	sfp->mmap_called = 1; /* Prevents changes to buffer size. */
+out:
+	mutex_unlock(&sfp->f_mutex);
+	return err;
+}
+
+static int
+sg_discontig_get(struct discontig_kernel_page_state *state)
+{
+	Sg_fd *sfp = state->vm_private_data;
+	Sg_scatter_hold *rsv_schp = &sfp->reserve;
+	const unsigned int order = rsv_schp->page_order;
+	const pgoff_t nr_pages = state->nr_pages_mapped;
+
+	if (nr_pages >= (rsv_schp->bufflen >> PAGE_SHIFT)) {
+		discontig_kernel_map_abort(state);
+		return 0;
 	}
 
-	return VM_FAULT_SIGBUS;
+	SCSI_LOG_TIMEOUT(3, sg_printk(KERN_INFO, sfp->parentdp,
+				      "%s: offset=%lu, scatg=%d\n", __func__,
+				      nr_pages << PAGE_SHIFT, rsv_schp->k_use_sg));
+
+	discontig_kernel_map_page(state, rsv_schp->pages[nr_pages >> order]);
+	return 0;
 }
 
-static const struct vm_operations_struct sg_mmap_vm_ops = {
-	.fault = sg_vma_fault,
+static const struct discontig_kernel_page_ops sg_discontig_ops = {
+	.init = sg_discontig_init,
+	.get = sg_discontig_get,
 };
 
 static int
-sg_mmap(struct file *filp, struct vm_area_struct *vma)
+sg_mmap_prepare(struct vm_area_desc *desc)
 {
-	Sg_fd *sfp;
-	unsigned long req_sz, len, sa;
-	Sg_scatter_hold *rsv_schp;
-	int k, length;
-	int ret = 0;
+	Sg_fd *sfp = desc->file->private_data;
+	const unsigned long req_sz = vma_desc_size(desc);
 
-	if ((!filp) || (!vma) || (!(sfp = (Sg_fd *) filp->private_data)))
+	if (!sfp)
 		return -ENXIO;
-	req_sz = vma->vm_end - vma->vm_start;
+
 	SCSI_LOG_TIMEOUT(3, sg_printk(KERN_INFO, sfp->parentdp,
 				      "sg_mmap starting, vm_start=%p, len=%d\n",
-				      (void *) vma->vm_start, (int) req_sz));
-	if (vma->vm_pgoff)
+				      (void *) desc->start, (int) req_sz));
+
+	if (desc->pgoff)
 		return -EINVAL;	/* want no offset */
-	rsv_schp = &sfp->reserve;
-	mutex_lock(&sfp->f_mutex);
-	if (req_sz > rsv_schp->bufflen) {
-		ret = -ENOMEM;	/* cannot map more than reserved buffer */
-		goto out;
-	}
 
-	sa = vma->vm_start;
-	length = 1 << (PAGE_SHIFT + rsv_schp->page_order);
-	for (k = 0; k < rsv_schp->k_use_sg && sa < vma->vm_end; k++) {
-		len = vma->vm_end - sa;
-		len = (len < length) ? len : length;
-		sa += len;
-	}
+	vma_desc_set_flags(desc, VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT);
+	desc->private_data = sfp;
 
-	sfp->mmap_called = 1;
-	vm_flags_set(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP);
-	vma->vm_private_data = sfp;
-	vma->vm_ops = &sg_mmap_vm_ops;
-out:
-	mutex_unlock(&sfp->f_mutex);
-	return ret;
+	mmap_action_map_discontig_kernel_pages(desc, (void *)req_sz,
+					       &sg_discontig_ops);
+	return 0;
 }
 
 static void
@@ -1415,7 +1402,7 @@ static const struct file_operations sg_fops = {
 	.unlocked_ioctl = sg_ioctl,
 	.compat_ioctl = compat_ptr_ioctl,
 	.open = sg_open,
-	.mmap = sg_mmap,
+	.mmap_prepare = sg_mmap_prepare,
 	.release = sg_release,
 	.fasync = sg_fasync,
 };
diff --git a/drivers/scsi/sgiwd93.c b/drivers/scsi/sgiwd93.c
index 6594661db5f407..455208776239c4 100644
--- a/drivers/scsi/sgiwd93.c
+++ b/drivers/scsi/sgiwd93.c
@@ -59,9 +59,9 @@ static irqreturn_t sgiwd93_intr(int irq, void *dev_id)
 	struct Scsi_Host * host = dev_id;
 	unsigned long flags;
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	wd33c93_intr(host);
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
index 140e8a6addbe31..d8d0fe915805a1 100644
--- a/drivers/scsi/smartpqi/smartpqi_init.c
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -3249,15 +3249,15 @@ static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
 			sense_data_length = sizeof(error_info->data);
 
 		if (scsi_status == SAM_STAT_CHECK_CONDITION &&
-			scsi_normalize_sense(error_info->data,
-				sense_data_length, &sshdr) &&
-				sshdr.sense_key == HARDWARE_ERROR &&
-				sshdr.asc == 0x3e) {
+		    scsi_normalize_sense(error_info->data,
+					 sense_data_length, &sshdr) &&
+		    sshdr.sense_key == HARDWARE_ERROR &&
+		    scsi_sense_asc(&sshdr) == ASC_LU_HAS_NOT_SELF_CONFIGURED_YET) {
 			struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);
 			struct pqi_scsi_dev *device = scmd->device->hostdata;
 
-			switch (sshdr.ascq) {
-			case 0x1: /* LOGICAL UNIT FAILURE */
+			switch (sshdr.sense_code) {
+			case LU_FAILURE:
 				if (printk_ratelimit())
 					scmd_printk(KERN_ERR, scmd, "received 'logical unit failure' from controller for scsi %d:%d:%d:%d\n",
 						ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
@@ -3265,10 +3265,15 @@ static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
 				host_byte = DID_NO_CONNECT;
 				break;
 
-			default: /* See http://www.t10.org/lists/asc-num.htm#ASC_3E */
+			default:
+				/* include/scsi/scsi_sense.h */
 				if (printk_ratelimit())
-					scmd_printk(KERN_ERR, scmd, "received unhandled error %d from controller for scsi %d:%d:%d:%d\n",
-						sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);
+					scmd_printk(KERN_ERR, scmd,
+						"received unhandled error %d from controller for scsi %d:%d:%d:%d\n",
+						scsi_sense_ascq(&sshdr),
+						ctrl_info->scsi_host->host_no,
+						device->bus, device->target,
+						device->lun);
 				break;
 			}
 		}
@@ -3286,11 +3291,11 @@ static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
 	    sense_data_length &&
 	    scsi_normalize_sense(error_info->data, sense_data_length, &sshdr) &&
 	    sshdr.sense_key == ILLEGAL_REQUEST &&
-	    sshdr.asc == 0x26 &&
-	    sshdr.ascq == 0x0) {
+	    sshdr.sense_code == INVALID_FIELD_IN_PARAMETER_LIST) {
 		host_byte = DID_NO_CONNECT;
 		pqi_take_device_offline(scmd->device, "AIO");
-		scsi_build_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR, 0x3e, 0x1);
+		scsi_set_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR,
+				      LU_FAILURE);
 	}
 
 	scmd->result = scsi_status;
@@ -3380,7 +3385,7 @@ static void pqi_process_aio_io_error(struct pqi_io_request *io_request)
 	}
 
 	if (device_offline && sense_data_length == 0)
-		scsi_build_sense(scmd, 0, HARDWARE_ERROR, 0x3e, 0x1);
+		scsi_set_sense(scmd, 0, HARDWARE_ERROR, LU_FAILURE);
 
 	scmd->result = scsi_status;
 	set_host_byte(scmd, host_byte);
diff --git a/drivers/scsi/snic/snic_disc.c b/drivers/scsi/snic/snic_disc.c
index 71d34a702f2abb..8c262b4bbb831a 100644
--- a/drivers/scsi/snic/snic_disc.c
+++ b/drivers/scsi/snic/snic_disc.c
@@ -160,9 +160,9 @@ snic_scsi_scan_tgt(struct work_struct *work)
 			 SCAN_WILD_CARD,
 			 SCSI_SCAN_RESCAN);
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	tgt->flags &= ~SNIC_TGT_SCAN_PENDING;
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 } /* end of snic_scsi_scan_tgt */
 
 /*
@@ -287,11 +287,11 @@ snic_tgt_create(struct snic *snic, struct snic_tgt_id *tgtid)
 		break;
 	}
 
-	spin_lock_irqsave(snic->shost->host_lock, flags);
+	spin_lock_irqsave(&snic->shost->host_lock, flags);
 	list_add_tail(&tgt->list, &snic->disc.tgt_list);
 	tgt->scsi_tgt_id = snic->disc.nxt_tgt_id++;
 	tgt->state = SNIC_TGT_STAT_ONLINE;
-	spin_unlock_irqrestore(snic->shost->host_lock, flags);
+	spin_unlock_irqrestore(&snic->shost->host_lock, flags);
 
 	SNIC_HOST_INFO(snic->shost,
 		       "Tgt %d, type = %s detected. Adding..\n",
@@ -304,9 +304,9 @@ snic_tgt_create(struct snic *snic, struct snic_tgt_id *tgtid)
 			      ret);
 
 		put_device(&snic->shost->shost_gendev);
-		spin_lock_irqsave(snic->shost->host_lock, flags);
+		spin_lock_irqsave(&snic->shost->host_lock, flags);
 		list_del(&tgt->list);
-		spin_unlock_irqrestore(snic->shost->host_lock, flags);
+		spin_unlock_irqrestore(&snic->shost->host_lock, flags);
 		put_device(&tgt->dev);
 		tgt = NULL;
 
@@ -537,7 +537,7 @@ snic_tgt_del_all(struct snic *snic)
 	scsi_flush_work(snic->shost);
 
 	mutex_lock(&snic->disc.mutex);
-	spin_lock_irqsave(snic->shost->host_lock, flags);
+	spin_lock_irqsave(&snic->shost->host_lock, flags);
 
 	list_for_each_safe(cur, nxt, &snic->disc.tgt_list) {
 		tgt = list_entry(cur, struct snic_tgt, list);
@@ -547,7 +547,7 @@ snic_tgt_del_all(struct snic *snic)
 		queue_work(snic_glob->event_q, &tgt->del_work);
 		tgt = NULL;
 	}
-	spin_unlock_irqrestore(snic->shost->host_lock, flags);
+	spin_unlock_irqrestore(&snic->shost->host_lock, flags);
 	mutex_unlock(&snic->disc.mutex);
 
 	flush_workqueue(snic_glob->event_q);
diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c
index c36c54ecd354b4..4f2e8d1be6da87 100644
--- a/drivers/scsi/sr.c
+++ b/drivers/scsi/sr.c
@@ -253,7 +253,8 @@ static unsigned int sr_check_events(struct cdrom_device_info *cdi,
 	 * (ASC 0x3a).
 	 */
 	cd->media_present = scsi_status_is_good(ret) ||
-		(scsi_sense_valid(&sshdr) && sshdr.asc != 0x3a);
+		(scsi_sense_valid(&sshdr) &&
+		 scsi_sense_asc(&sshdr) != ASC_MEDIUM_NOT_PRESENT);
 
 	if (last_present != cd->media_present)
 		cd->device->changed = 1;
diff --git a/drivers/scsi/sr_ioctl.c b/drivers/scsi/sr_ioctl.c
index 089653018d32c1..e4ec0f63cdccbf 100644
--- a/drivers/scsi/sr_ioctl.c
+++ b/drivers/scsi/sr_ioctl.c
@@ -226,9 +226,8 @@ int sr_do_ioctl(Scsi_CD *cd, struct packet_command *cgc)
 			err = -ENOMEDIUM;
 			break;
 		case NOT_READY:	/* This happens if there is no disc in drive */
-			if (sshdr->asc == 0x04 &&
-			    sshdr->ascq == 0x01) {
-				/* sense: Logical unit is in process of becoming ready */
+			if (sshdr->sense_code ==
+			    LU_IS_IN_PROCESS_OF_BECOMING_READY) {
 				if (!cgc->quiet)
 					sr_printk(KERN_INFO, cd,
 						  "CDROM not ready yet.\n");
@@ -250,9 +249,7 @@ int sr_do_ioctl(Scsi_CD *cd, struct packet_command *cgc)
 			break;
 		case ILLEGAL_REQUEST:
 			err = -EIO;
-			if (sshdr->asc == 0x20 &&
-			    sshdr->ascq == 0x00)
-				/* sense: Invalid command operation code */
+			if (sshdr->sense_code == INVALID_COMMAND_OP_CODE)
 				err = -EDRIVE_CANT_DO_THIS;
 			break;
 		default:
@@ -303,9 +300,8 @@ int sr_drive_status(struct cdrom_device_info *cdi, int slot)
 	if (!scsi_test_unit_ready(cd->device, SR_TIMEOUT, MAX_RETRIES, &sshdr))
 		return CDS_DISC_OK;
 
-	/* SK/ASC/ASCQ of 2/4/1 means "unit is becoming ready" */
-	if (scsi_sense_valid(&sshdr) && sshdr.sense_key == NOT_READY
-			&& sshdr.asc == 0x04 && sshdr.ascq == 0x01)
+	if (scsi_sense_valid(&sshdr) && sshdr.sense_key == NOT_READY &&
+	    sshdr.sense_code == LU_IS_IN_PROCESS_OF_BECOMING_READY)
 		return CDS_DRIVE_NOT_READY;
 
 	if (!cdrom_get_media_event(cdi, &med)) {
@@ -318,18 +314,18 @@ int sr_drive_status(struct cdrom_device_info *cdi, int slot)
 	}
 
 	/*
-	 * SK/ASC/ASCQ of 2/4/2 means "initialization required"
 	 * Using CD_TRAY_OPEN results in an START_STOP_UNIT to close
 	 * the tray, which resolves the initialization requirement.
 	 */
-	if (scsi_sense_valid(&sshdr) && sshdr.sense_key == NOT_READY
-			&& sshdr.asc == 0x04 && sshdr.ascq == 0x02)
+	if (scsi_sense_valid(&sshdr) && sshdr.sense_key == NOT_READY &&
+	    sshdr.sense_code == LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED)
 		return CDS_TRAY_OPEN;
 
 	/*
 	 * 0x04 is format in progress .. but there must be a disc present!
 	 */
-	if (sshdr.sense_key == NOT_READY && sshdr.asc == 0x04)
+	if (sshdr.sense_key == NOT_READY &&
+	    scsi_sense_asc(&sshdr) == ASC_LU_NOT_READY)
 		return CDS_DISC_OK;
 
 	/*
@@ -338,13 +334,10 @@ int sr_drive_status(struct cdrom_device_info *cdi, int slot)
 	 * any other way to detect this...
 	 */
 	if (scsi_sense_valid(&sshdr) &&
-	    /* 0x3a is medium not present */
-	    sshdr.asc == 0x3a)
+	    scsi_sense_asc(&sshdr) == ASC_MEDIUM_NOT_PRESENT)
 		return CDS_NO_DISC;
-	else
-		return CDS_TRAY_OPEN;
 
-	return CDS_DRIVE_NOT_READY;
+	return CDS_TRAY_OPEN;
 }
 
 int sr_disk_status(struct cdrom_device_info *cdi)
diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
index f1c3c49466379e..1ca9753cb734bd 100644
--- a/drivers/scsi/st.c
+++ b/drivers/scsi/st.c
@@ -421,11 +421,13 @@ static int st_chk_result(struct scsi_tape *STp, struct st_request * SRpnt)
 					       STp->cln_sense_mask) != 0);
 	}
 	if (cmdstatp->have_sense &&
-	    cmdstatp->sense_hdr.asc == 0 && cmdstatp->sense_hdr.ascq == 0x17)
-		STp->cleaning_req = 1; /* ASC and ASCQ => cleaning requested */
+	    cmdstatp->sense_hdr.sense_code == CLEANING_REQUESTED)
+		STp->cleaning_req = 1;
 	if (cmdstatp->have_sense && scode == UNIT_ATTENTION &&
-		cmdstatp->sense_hdr.asc == 0x29 && !STp->pos_unknown) {
-		STp->pos_unknown = 1; /* ASC => power on / reset */
+	    scsi_sense_asc(&cmdstatp->sense_hdr) ==
+	    ASC_POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED &&
+	    !STp->pos_unknown) {
+		STp->pos_unknown = 1;
 		st_printk(KERN_WARNING, STp, "Power on/reset recognized.");
 	}
 
@@ -1003,7 +1005,8 @@ static int test_ready(struct scsi_tape *STp, int do_wait)
 			scode = cmdstatp->sense_hdr.sense_key;
 
 			if (scode == UNIT_ATTENTION) { /* New media? */
-				if (cmdstatp->sense_hdr.asc == 0x28) { /* New media */
+				if (scsi_sense_asc(&cmdstatp->sense_hdr) ==
+				    ASC_NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED) {
 					new_session = 1;
 					DEBC_printk(STp, "New tape session.");
 				}
@@ -1026,14 +1029,13 @@ static int test_ready(struct scsi_tape *STp, int do_wait)
 					waits++;
 					continue;
 				}
-				else {
-					if ((STp->device)->scsi_level >= SCSI_2 &&
-					    cmdstatp->sense_hdr.asc == 0x3a)	/* Check ASC */
-						retval = CHKRES_NO_TAPE;
-					else
-						retval = CHKRES_NOT_READY;
-					break;
-				}
+				if ((STp->device)->scsi_level >= SCSI_2 &&
+				    scsi_sense_asc(&cmdstatp->sense_hdr) ==
+				    ASC_MEDIUM_NOT_PRESENT)
+					retval = CHKRES_NO_TAPE;
+				else
+					retval = CHKRES_NOT_READY;
+				break;
 			}
 		}
 
@@ -3105,9 +3107,10 @@ static int st_int_ioctl(struct scsi_tape *STp, unsigned int cmd_in, unsigned lon
 			   cmd_in == MTSETDRVBUFFER ||
 			   cmd_in == SET_DENS_AND_BLK) {
 			if (cmdstatp->sense_hdr.sense_key == ILLEGAL_REQUEST &&
-				cmdstatp->sense_hdr.asc == 0x24 &&
-				(STp->device)->scsi_level <= SCSI_2 &&
-				!(STp->use_pf & PF_TESTED)) {
+			    scsi_sense_asc(&cmdstatp->sense_hdr) ==
+			    ASC_INVALID_FIELD_IN_CDB &&
+			    (STp->device)->scsi_level <= SCSI_2 &&
+			    !(STp->use_pf & PF_TESTED)) {
 				/* Try the other possible state of Page Format if not
 				   already tried */
 				STp->use_pf = (STp->use_pf ^ USE_PF) | PF_TESTED;
diff --git a/drivers/scsi/stex.c b/drivers/scsi/stex.c
index 6aeeb338633dff..d4899816f7dc8d 100644
--- a/drivers/scsi/stex.c
+++ b/drivers/scsi/stex.c
@@ -401,8 +401,7 @@ static struct status_msg *stex_get_status(struct st_hba *hba)
 static void stex_invalid_field(struct scsi_cmnd *cmd,
 			       void (*done)(struct scsi_cmnd *))
 {
-	/* "Invalid field in cdb" */
-	scsi_build_sense(cmd, 0, ILLEGAL_REQUEST, 0x24, 0x0);
+	scsi_set_sense(cmd, 0, ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB);
 	done(cmd);
 }
 
@@ -568,7 +567,7 @@ static void return_abnormal_state(struct st_hba *hba, int status)
 	unsigned long flags;
 	u16 tag;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	for (tag = 0; tag < hba->host->can_queue; tag++) {
 		ccb = &hba->ccb[tag];
 		if (ccb->req == NULL)
@@ -581,7 +580,7 @@ static void return_abnormal_state(struct st_hba *hba, int status)
 			ccb->cmd = NULL;
 		}
 	}
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 }
 static int
 stex_sdev_configure(struct scsi_device *sdev, struct queue_limits *lim)
@@ -887,7 +886,7 @@ static irqreturn_t stex_intr(int irq, void *__hba)
 	u32 data;
 	unsigned long flags;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 
 	data = readl(base + ODBL);
 
@@ -896,14 +895,14 @@ static irqreturn_t stex_intr(int irq, void *__hba)
 		writel(data, base + ODBL);
 		readl(base + ODBL); /* flush */
 		stex_mu_intr(hba, data);
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		if (unlikely(data & MU_OUTBOUND_DOORBELL_REQUEST_RESET &&
 			hba->cardtype == st_shasta))
 			queue_work(hba->work_q, &hba->reset_work);
 		return IRQ_HANDLED;
 	}
 
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return IRQ_NONE;
 }
@@ -988,7 +987,7 @@ static irqreturn_t stex_ss_intr(int irq, void *__hba)
 	u32 data;
 	unsigned long flags;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 
 	if (hba->cardtype == st_yel) {
 		data = readl(base + YI2H_INT);
@@ -996,7 +995,7 @@ static irqreturn_t stex_ss_intr(int irq, void *__hba)
 			/* clear the interrupt */
 			writel(data, base + YI2H_INT_C);
 			stex_ss_mu_intr(hba);
-			spin_unlock_irqrestore(hba->host->host_lock, flags);
+			spin_unlock_irqrestore(&hba->host->host_lock, flags);
 			if (unlikely(data & SS_I2H_REQUEST_RESET))
 				queue_work(hba->work_q, &hba->reset_work);
 			return IRQ_HANDLED;
@@ -1010,14 +1009,14 @@ static irqreturn_t stex_ss_intr(int irq, void *__hba)
 				writel((1 << 22), base + YH2I_INT);
 			}
 			stex_ss_mu_intr(hba);
-			spin_unlock_irqrestore(hba->host->host_lock, flags);
+			spin_unlock_irqrestore(&hba->host->host_lock, flags);
 			if (unlikely(data & SS_I2H_REQUEST_RESET))
 				queue_work(hba->work_q, &hba->reset_work);
 			return IRQ_HANDLED;
 		}
 	}
 
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return IRQ_NONE;
 }
@@ -1227,7 +1226,7 @@ static int stex_handshake(struct st_hba *hba)
 		err = stex_ss_handshake(hba);
 	else
 		err = stex_common_handshake(hba);
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	mu_status = hba->mu_status;
 	if (err == 0) {
 		hba->req_head = 0;
@@ -1240,7 +1239,7 @@ static int stex_handshake(struct st_hba *hba)
 		hba->mu_status = MU_STATE_FAILED;
 	if (mu_status == MU_STATE_RESETTING)
 		wake_up_all(&hba->reset_waitq);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	return err;
 }
 
@@ -1257,7 +1256,7 @@ static int stex_abort(struct scsi_cmnd *cmd)
 	scmd_printk(KERN_INFO, cmd, "aborting command\n");
 
 	base = hba->mmio_base;
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 	if (tag < host->can_queue &&
 		hba->ccb[tag].req && hba->ccb[tag].cmd == cmd)
 		hba->wait_ccb = &hba->ccb[tag];
@@ -1301,7 +1300,7 @@ static int stex_abort(struct scsi_cmnd *cmd)
 	hba->wait_ccb = NULL;
 	result = FAILED;
 out:
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 	return result;
 }
 
@@ -1364,13 +1363,13 @@ static int stex_yos_reset(struct st_hba *hba)
 		msleep(1);
 	}
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (ret == -1)
 		hba->mu_status = MU_STATE_FAILED;
 	else
 		hba->mu_status = MU_STATE_STARTED;
 	wake_up_all(&hba->reset_waitq);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return ret;
 }
@@ -1393,29 +1392,29 @@ static int stex_do_reset(struct st_hba *hba)
 	unsigned long flags;
 	unsigned int mu_status = MU_STATE_RESETTING;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (hba->mu_status == MU_STATE_STARTING) {
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		printk(KERN_INFO DRV_NAME "(%s): request reset during init\n",
 			pci_name(hba->pdev));
 		return 0;
 	}
 	while (hba->mu_status == MU_STATE_RESETTING) {
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		wait_event_timeout(hba->reset_waitq,
 				   hba->mu_status != MU_STATE_RESETTING,
 				   MU_MAX_DELAY * HZ);
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		mu_status = hba->mu_status;
 	}
 
 	if (mu_status != MU_STATE_RESETTING) {
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		return (mu_status == MU_STATE_STARTED) ? 0 : -1;
 	}
 
 	hba->mu_status = MU_STATE_RESETTING;
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	if (hba->cardtype == st_yosemite)
 		return stex_yos_reset(hba);
@@ -1858,12 +1857,12 @@ static void stex_hba_stop(struct st_hba *hba, int st_sleep_mic)
 	unsigned long before;
 	u16 tag = 0;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 
 	if ((hba->cardtype == st_yel || hba->cardtype == st_P3) &&
 		hba->supports_pm == 1) {
 		if (st_sleep_mic == ST_NOTHANDLED) {
-			spin_unlock_irqrestore(hba->host->host_lock, flags);
+			spin_unlock_irqrestore(&hba->host->host_lock, flags);
 			return;
 		}
 	}
@@ -1899,7 +1898,7 @@ static void stex_hba_stop(struct st_hba *hba, int st_sleep_mic)
 	hba->ccb[tag].sense_buffer = NULL;
 	hba->ccb[tag].req_type = PASSTHRU_REQ_TYPE;
 	hba->send(hba, req, tag);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	before = jiffies;
 	while (hba->ccb[tag].req_type & PASSTHRU_REQ_TYPE) {
 		if (time_after(jiffies, before + ST_INTERNAL_TIMEOUT * HZ)) {
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index ef3cd3bc7390e5..625db97d1a03e7 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -1016,7 +1016,7 @@ static int storvsc_channel_init(struct hv_device *device, bool is_fc)
 static void storvsc_handle_error(struct vmscsi_request *vm_srb,
 				struct scsi_cmnd *scmnd,
 				struct Scsi_Host *host,
-				u8 asc, u8 ascq)
+				struct scsi_sense_hdr *sshdr)
 {
 	struct storvsc_scan_work *wrk;
 	void (*process_err_fn)(struct work_struct *work);
@@ -1033,7 +1033,7 @@ static void storvsc_handle_error(struct vmscsi_request *vm_srb,
 	case SRB_STATUS_DATA_OVERRUN:
 		if (vm_srb->srb_status & SRB_STATUS_AUTOSENSE_VALID) {
 			/* Check for capacity change */
-			if ((asc == 0x2a) && (ascq == 0x9)) {
+			if (sshdr->sense_code == CAPACITY_DATA_HAS_CHANGED) {
 				process_err_fn = storvsc_device_scan;
 				/* Retry the I/O that triggered this. */
 				set_host_byte(scmnd, DID_REQUEUE);
@@ -1049,8 +1049,12 @@ static void storvsc_handle_error(struct vmscsi_request *vm_srb,
 			 * want scsi_report_sense() to output a message
 			 * that a sysadmin wouldn't know what to do with.
 			 */
-			if ((asc == 0x3f) && (ascq != 0x03) &&
-					(ascq != 0x0e)) {
+			if (scsi_sense_asc(sshdr) ==
+			    ASC_TARGET_OPERATING_CONDITIONS_HAVE_CHANGED &&
+			    sshdr->sense_code !=
+			    INQUIRY_DATA_HAS_CHANGED &&
+			    sshdr->sense_code !=
+			    REPORTED_LUNS_DATA_HAS_CHANGED) {
 				process_err_fn = storvsc_device_scan;
 				set_host_byte(scmnd, DID_REQUEUE);
 				goto do_work;
@@ -1141,8 +1145,7 @@ static void storvsc_command_completion(struct storvsc_cmd_request *cmd_request,
 	}
 
 	if (vm_srb->srb_status != SRB_STATUS_SUCCESS) {
-		storvsc_handle_error(vm_srb, scmnd, host, sense_hdr.asc,
-					 sense_hdr.ascq);
+		storvsc_handle_error(vm_srb, scmnd, host, &sense_hdr);
 		/*
 		 * The Windows driver set data_transfer_length on
 		 * SRB_STATUS_DATA_OVERRUN. On other errors, this value
diff --git a/drivers/scsi/sym53c8xx_2/sym_glue.c b/drivers/scsi/sym53c8xx_2/sym_glue.c
index 27e22acaf1a7e7..e3783c015b02d7 100644
--- a/drivers/scsi/sym53c8xx_2/sym_glue.c
+++ b/drivers/scsi/sym53c8xx_2/sym_glue.c
@@ -531,9 +531,9 @@ static irqreturn_t sym53c8xx_intr(int irq, void *dev_id)
 
 	if (DEBUG_FLAGS & DEBUG_TINY) printf_debug ("[");
 
-	spin_lock(shost->host_lock);
+	spin_lock(&shost->host_lock);
 	result = sym_interrupt(shost);
-	spin_unlock(shost->host_lock);
+	spin_unlock(&shost->host_lock);
 
 	if (DEBUG_FLAGS & DEBUG_TINY) printf_debug ("]\n");
 
@@ -548,9 +548,9 @@ static void sym53c8xx_timer(struct timer_list *t)
 	struct sym_hcb *np = timer_container_of(np, t, s.timer);
 	unsigned long flags;
 
-	spin_lock_irqsave(np->s.host->host_lock, flags);
+	spin_lock_irqsave(&np->s.host->host_lock, flags);
 	sym_timer(np);
-	spin_unlock_irqrestore(np->s.host->host_lock, flags);
+	spin_unlock_irqrestore(&np->s.host->host_lock, flags);
 }
 
 
@@ -587,7 +587,7 @@ static int sym53c8xx_eh_abort_handler(struct scsi_cmnd *cmd)
 	if (pci_channel_offline(pdev))
 		return SCSI_FAILED;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	/* This one is queued in some place -> to wait for completion */
 	FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) {
 		struct sym_ccb *cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
@@ -605,13 +605,13 @@ static int sym53c8xx_eh_abort_handler(struct scsi_cmnd *cmd)
 	if (cmd_queued) {
 		init_completion(&eh_done);
 		ucmd->eh_done = &eh_done;
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		if (!wait_for_completion_timeout(&eh_done, 5*HZ)) {
 			ucmd->eh_done = NULL;
 			sts = -2;
 		}
 	} else {
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 	}
 
 	dev_warn(&cmd->device->sdev_gendev, "ABORT operation %s.\n",
@@ -639,7 +639,7 @@ static int sym53c8xx_eh_target_reset_handler(struct scsi_cmnd *cmd)
 	if (pci_channel_offline(pdev))
 		return SCSI_FAILED;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	sts = sym_reset_scsi_target(np, starget->id);
 	if (!sts) {
 		FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) {
@@ -655,15 +655,15 @@ static int sym53c8xx_eh_target_reset_handler(struct scsi_cmnd *cmd)
 			ucmd = SYM_UCMD_PTR(cmd);
 			init_completion(&eh_done);
 			ucmd->eh_done = &eh_done;
-			spin_unlock_irq(shost->host_lock);
+			spin_unlock_irq(&shost->host_lock);
 			if (!wait_for_completion_timeout(&eh_done, 5*HZ)) {
 				ucmd->eh_done = NULL;
 				sts = -2;
 			}
-			spin_lock_irq(shost->host_lock);
+			spin_lock_irq(&shost->host_lock);
 		}
 	}
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	starget_printk(KERN_WARNING, starget, "TARGET RESET operation %s.\n",
 			sts==0 ? "complete" :sts==-2 ? "timed-out" : "failed");
@@ -685,9 +685,9 @@ static int sym53c8xx_eh_bus_reset_handler(struct scsi_cmnd *cmd)
 	if (pci_channel_offline(pdev))
 		return SCSI_FAILED;
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	sym_reset_scsi_bus(np, 1);
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 
 	dev_warn(&cmd->device->sdev_gendev, "BUS RESET operation complete.\n");
 	return SCSI_SUCCESS;
@@ -713,21 +713,21 @@ static int sym53c8xx_eh_host_reset_handler(struct scsi_cmnd *cmd)
 #define WAIT_FOR_PCI_RECOVERY	35
 	if (pci_channel_offline(pdev)) {
 		init_completion(&eh_done);
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		/* Make sure we didn't race */
 		if (pci_channel_offline(pdev)) {
 			BUG_ON(sym_data->io_reset);
 			sym_data->io_reset = &eh_done;
 			finished_reset = 0;
 		}
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 		if (!finished_reset)
 			finished_reset = wait_for_completion_timeout
 						(sym_data->io_reset,
 						WAIT_FOR_PCI_RECOVERY*HZ);
-		spin_lock_irq(shost->host_lock);
+		spin_lock_irq(&shost->host_lock);
 		sym_data->io_reset = NULL;
-		spin_unlock_irq(shost->host_lock);
+		spin_unlock_irq(&shost->host_lock);
 	}
 
 	if (finished_reset) {
@@ -776,7 +776,7 @@ static int sym53c8xx_sdev_init(struct scsi_device *sdev)
 	if (sdev->id >= SYM_CONF_MAX_TARGET || sdev->lun >= SYM_CONF_MAX_LUN)
 		return -ENXIO;
 
-	spin_lock_irqsave(np->s.host->host_lock, flags);
+	spin_lock_irqsave(&np->s.host->host_lock, flags);
 
 	/*
 	 * Fail the device init if the device is flagged NOSCAN at BOOT in
@@ -817,7 +817,7 @@ static int sym53c8xx_sdev_init(struct scsi_device *sdev)
 
 	error = 0;
 out:
-	spin_unlock_irqrestore(np->s.host->host_lock, flags);
+	spin_unlock_irqrestore(&np->s.host->host_lock, flags);
 
 	return error;
 }
@@ -873,7 +873,7 @@ static void sym53c8xx_sdev_destroy(struct scsi_device *sdev)
 	if (!lp)
 		return;
 
-	spin_lock_irqsave(np->s.host->host_lock, flags);
+	spin_lock_irqsave(&np->s.host->host_lock, flags);
 
 	if (lp->busy_itlq || lp->busy_itl) {
 		/*
@@ -896,7 +896,7 @@ static void sym53c8xx_sdev_destroy(struct scsi_device *sdev)
 		tp->starget	     = NULL;
 	}
 
-	spin_unlock_irqrestore(np->s.host->host_lock, flags);
+	spin_unlock_irqrestore(&np->s.host->host_lock, flags);
 }
 
 /*
@@ -1191,9 +1191,9 @@ printk("sym_user_command: data=%ld\n", uc->data);
 	else {
 		unsigned long flags;
 
-		spin_lock_irqsave(shost->host_lock, flags);
+		spin_lock_irqsave(&shost->host_lock, flags);
 		sym_exec_user_command(np, uc);
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 	}
 	return length;
 }
@@ -1371,7 +1371,7 @@ static struct Scsi_Host *sym_attach(const struct scsi_host_template *tpnt, int u
 	 *  After SCSI devices have been opened, we cannot
 	 *  reset the bus safely, so we do it here.
 	 */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (sym_reset_scsi_bus(np, 0))
 		goto reset_failed;
 
@@ -1407,14 +1407,14 @@ static struct Scsi_Host *sym_attach(const struct scsi_host_template *tpnt, int u
 	if (pdev->device == PCI_DEVICE_ID_NCR_53C896 && pdev->revision < 2)
 		shost->dma_boundary = 0xFFFFFF;
 
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	return shost;
 
  reset_failed:
 	printf_err("%s: FATAL ERROR: CHECK SCSI BUS - CABLES, "
 		   "TERMINATION, DEVICE POWER etc.!\n", sym_name(np));
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
  attach_failed:
 	printf_info("sym%d: giving up ...\n", unit);
 	if (np)
@@ -1903,10 +1903,10 @@ static void sym2_io_resume(struct pci_dev *pdev)
 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
 	struct sym_data *sym_data = shost_priv(shost);
 
-	spin_lock_irq(shost->host_lock);
+	spin_lock_irq(&shost->host_lock);
 	if (sym_data->io_reset)
 		complete(sym_data->io_reset);
-	spin_unlock_irq(shost->host_lock);
+	spin_unlock_irq(&shost->host_lock);
 }
 
 static void sym2_get_signalling(struct Scsi_Host *shost)
diff --git a/drivers/scsi/sym53c8xx_2/sym_hipd.h b/drivers/scsi/sym53c8xx_2/sym_hipd.h
index 9231a289906436..aa365e8ba66fc1 100644
--- a/drivers/scsi/sym53c8xx_2/sym_hipd.h
+++ b/drivers/scsi/sym53c8xx_2/sym_hipd.h
@@ -1110,9 +1110,9 @@ sym_build_sge(struct sym_hcb *np, struct sym_tblmove *data, u64 badd, int len)
  */
 
 #define sym_get_mem_cluster()	\
-	(void *) __get_free_pages(GFP_ATOMIC, SYM_MEM_PAGE_ORDER)
+	kmalloc(PAGE_SIZE << SYM_MEM_PAGE_ORDER, GFP_ATOMIC)
 #define sym_free_mem_cluster(p)	\
-	free_pages((unsigned long)p, SYM_MEM_PAGE_ORDER)
+	kfree(p)
 
 /*
  *  Link between free memory chunks of a given size.
diff --git a/drivers/scsi/wd33c93.c b/drivers/scsi/wd33c93.c
index 1e49d0402f0b54..2a698b93faf8f1 100644
--- a/drivers/scsi/wd33c93.c
+++ b/drivers/scsi/wd33c93.c
@@ -1515,7 +1515,7 @@ wd33c93_host_reset(struct scsi_cmnd * SCpnt)
 	int i;
 
 	instance = SCpnt->device->host;
-	spin_lock_irq(instance->host_lock);
+	spin_lock_irq(&instance->host_lock);
 	hostdata = (struct WD33C93_hostdata *) instance->hostdata;
 
 	printk("scsi%d: reset. ", instance->host_no);
@@ -1541,7 +1541,7 @@ wd33c93_host_reset(struct scsi_cmnd * SCpnt)
 	reset_wd33c93(instance);
 	SCpnt->result = DID_RESET << 16;
 	enable_irq(instance->irq);
-	spin_unlock_irq(instance->host_lock);
+	spin_unlock_irq(&instance->host_lock);
 	return SUCCESS;
 }
 
diff --git a/drivers/scsi/wd719x.c b/drivers/scsi/wd719x.c
index 830d40f57f6a47..85cf6c9d2ec909 100644
--- a/drivers/scsi/wd719x.c
+++ b/drivers/scsi/wd719x.c
@@ -270,11 +270,11 @@ static enum scsi_qc_status wd719x_queuecommand(struct Scsi_Host *sh,
 		scb->data_p = 0;
 	}
 
-	spin_lock_irqsave(wd->sh->host_lock, flags);
+	spin_lock_irqsave(&wd->sh->host_lock, flags);
 
 	/* check if the Command register is free */
 	if (wd719x_readb(wd, WD719X_AMR_COMMAND) != WD719X_CMD_READY) {
-		spin_unlock_irqrestore(wd->sh->host_lock, flags);
+		spin_unlock_irqrestore(&wd->sh->host_lock, flags);
 		return SCSI_MLQUEUE_HOST_BUSY;
 	}
 
@@ -285,7 +285,7 @@ static enum scsi_qc_status wd719x_queuecommand(struct Scsi_Host *sh,
 	/* send SCB opcode */
 	wd719x_writeb(wd, WD719X_AMR_COMMAND, WD719X_CMD_PROCESS_SCB);
 
-	spin_unlock_irqrestore(wd->sh->host_lock, flags);
+	spin_unlock_irqrestore(&wd->sh->host_lock, flags);
 	return 0;
 
 out_unmap_sense:
@@ -473,12 +473,12 @@ static int wd719x_abort(struct scsi_cmnd *cmd)
 
 	action = WD719X_CMD_ABORT;
 
-	spin_lock_irqsave(wd->sh->host_lock, flags);
+	spin_lock_irqsave(&wd->sh->host_lock, flags);
 	result = wd719x_direct_cmd(wd, action, cmd->device->id,
 				   cmd->device->lun, scsi_cmd_to_rq(cmd)->tag,
 				   scb->phys, 0);
 	wd719x_finish_cmd(scb, DID_ABORT);
-	spin_unlock_irqrestore(wd->sh->host_lock, flags);
+	spin_unlock_irqrestore(&wd->sh->host_lock, flags);
 	if (result)
 		return FAILED;
 
@@ -495,7 +495,7 @@ static int wd719x_reset(struct scsi_cmnd *cmd, u8 opcode, u8 device)
 	dev_info(&wd->pdev->dev, "%s reset requested\n",
 		 (opcode == WD719X_CMD_BUSRESET) ? "bus" : "device");
 
-	spin_lock_irqsave(wd->sh->host_lock, flags);
+	spin_lock_irqsave(&wd->sh->host_lock, flags);
 	result = wd719x_direct_cmd(wd, opcode, device, 0, 0, 0,
 				   WD719X_WAIT_FOR_SCSI_RESET);
 	/* flush all SCBs (or all for a device if dev_reset) */
@@ -504,7 +504,7 @@ static int wd719x_reset(struct scsi_cmnd *cmd, u8 opcode, u8 device)
 		    scb->cmd->device->id == device)
 			wd719x_finish_cmd(scb, DID_RESET);
 	}
-	spin_unlock_irqrestore(wd->sh->host_lock, flags);
+	spin_unlock_irqrestore(&wd->sh->host_lock, flags);
 	if (result)
 		return FAILED;
 
@@ -528,7 +528,7 @@ static int wd719x_host_reset(struct scsi_cmnd *cmd)
 	unsigned long flags;
 
 	dev_info(&wd->pdev->dev, "host reset requested\n");
-	spin_lock_irqsave(wd->sh->host_lock, flags);
+	spin_lock_irqsave(&wd->sh->host_lock, flags);
 	/* stop the RISC */
 	if (wd719x_direct_cmd(wd, WD719X_CMD_SLEEP, 0, 0, 0, 0,
 			      WD719X_WAIT_FOR_RISC))
@@ -539,7 +539,7 @@ static int wd719x_host_reset(struct scsi_cmnd *cmd)
 	/* flush all SCBs */
 	list_for_each_entry_safe(scb, tmp, &wd->active_scbs, list)
 		wd719x_finish_cmd(scb, DID_RESET);
-	spin_unlock_irqrestore(wd->sh->host_lock, flags);
+	spin_unlock_irqrestore(&wd->sh->host_lock, flags);
 
 	/* Try to reinit the RISC */
 	return wd719x_chip_init(wd) == 0 ? SUCCESS : FAILED;
@@ -663,7 +663,7 @@ static irqreturn_t wd719x_interrupt(int irq, void *dev_id)
 	unsigned long flags;
 	u32 SCB_out;
 
-	spin_lock_irqsave(wd->sh->host_lock, flags);
+	spin_lock_irqsave(&wd->sh->host_lock, flags);
 	/* read SCB pointer back from card */
 	SCB_out = wd719x_readl(wd, WD719X_AMR_SCB_OUT);
 	/* read all status info at once */
@@ -671,7 +671,7 @@ static irqreturn_t wd719x_interrupt(int irq, void *dev_id)
 
 	switch (regs.bytes.INT) {
 	case WD719X_INT_NONE:
-		spin_unlock_irqrestore(wd->sh->host_lock, flags);
+		spin_unlock_irqrestore(&wd->sh->host_lock, flags);
 		return IRQ_NONE;
 	case WD719X_INT_LINKNOSTATUS:
 		dev_err(&wd->pdev->dev, "linked command completed with no status\n");
@@ -708,7 +708,7 @@ static irqreturn_t wd719x_interrupt(int irq, void *dev_id)
 	}
 	/* clear interrupt so another can happen */
 	wd719x_writeb(wd, WD719X_AMR_INT_STATUS, WD719X_INT_NONE);
-	spin_unlock_irqrestore(wd->sh->host_lock, flags);
+	spin_unlock_irqrestore(&wd->sh->host_lock, flags);
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/scsi/xen-scsifront.c b/drivers/scsi/xen-scsifront.c
index 989bcaee42caf6..2bff5e35822f1f 100644
--- a/drivers/scsi/xen-scsifront.c
+++ b/drivers/scsi/xen-scsifront.c
@@ -417,13 +417,13 @@ static int scsifront_cmd_done(struct vscsifrnt_info *info,
 	int more_to_do;
 	unsigned long flags;
 
-	spin_lock_irqsave(info->host->host_lock, flags);
+	spin_lock_irqsave(&info->host->host_lock, flags);
 
 	more_to_do = scsifront_ring_drain(info, eoiflag);
 
 	info->wait_ring_available = 0;
 
-	spin_unlock_irqrestore(info->host->host_lock, flags);
+	spin_unlock_irqrestore(&info->host->host_lock, flags);
 
 	wake_up(&info->wq_sync);
 
@@ -619,9 +619,9 @@ static enum scsi_qc_status scsifront_queuecommand(struct Scsi_Host *shost,
 	shadow->sc  = sc;
 	shadow->act = VSCSIIF_ACT_SCSI_CDB;
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	if (scsifront_enter(info)) {
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		return SCSI_MLQUEUE_HOST_BUSY;
 	}
 
@@ -629,7 +629,7 @@ static enum scsi_qc_status scsifront_queuecommand(struct Scsi_Host *shost,
 	if (err < 0) {
 		pr_debug("%s: err %d\n", __func__, err);
 		scsifront_return(info);
-		spin_unlock_irqrestore(shost->host_lock, flags);
+		spin_unlock_irqrestore(&shost->host_lock, flags);
 		if (err == -ENOMEM)
 			return SCSI_MLQUEUE_HOST_BUSY;
 		sc->result = DID_ERROR << 16;
@@ -643,13 +643,13 @@ static enum scsi_qc_status scsifront_queuecommand(struct Scsi_Host *shost,
 	}
 
 	scsifront_return(info);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	return 0;
 
 busy:
 	scsifront_return(info);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 	pr_debug("%s: busy\n", __func__);
 	return SCSI_MLQUEUE_HOST_BUSY;
 }
@@ -679,7 +679,7 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act)
 	shadow->ref_rqid = s->rqid;
 	init_waitqueue_head(&shadow->wq_reset);
 
-	spin_lock_irq(host->host_lock);
+	spin_lock_irq(&host->host_lock);
 
 	for (;;) {
 		if (scsifront_enter(info))
@@ -692,15 +692,15 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act)
 		if (err)
 			goto fail;
 		info->wait_ring_available = 1;
-		spin_unlock_irq(host->host_lock);
+		spin_unlock_irq(&host->host_lock);
 		err = wait_event_interruptible(info->wq_sync,
 					       !info->wait_ring_available);
-		spin_lock_irq(host->host_lock);
+		spin_lock_irq(&host->host_lock);
 	}
 
-	spin_unlock_irq(host->host_lock);
+	spin_unlock_irq(&host->host_lock);
 	err = wait_event_interruptible(shadow->wq_reset, shadow->wait_reset);
-	spin_lock_irq(host->host_lock);
+	spin_lock_irq(&host->host_lock);
 
 	if (!err) {
 		err = shadow->rslt_reset;
@@ -714,11 +714,11 @@ static int scsifront_action_handler(struct scsi_cmnd *sc, uint8_t act)
 	}
 
 	scsifront_return(info);
-	spin_unlock_irq(host->host_lock);
+	spin_unlock_irq(&host->host_lock);
 	return err;
 
 fail:
-	spin_unlock_irq(host->host_lock);
+	spin_unlock_irq(&host->host_lock);
 	kfree(shadow);
 	return FAILED;
 }
@@ -953,12 +953,12 @@ static int scsifront_resume(struct xenbus_device *dev)
 	struct Scsi_Host *host = info->host;
 	int err;
 
-	spin_lock_irq(host->host_lock);
+	spin_lock_irq(&host->host_lock);
 
 	/* Finish all still pending commands. */
 	scsifront_finish_all(info);
 
-	spin_unlock_irq(host->host_lock);
+	spin_unlock_irq(&host->host_lock);
 
 	/* Reconnect to dom0. */
 	scsifront_free_ring(info);
@@ -981,18 +981,18 @@ static int scsifront_suspend(struct xenbus_device *dev)
 	int err = 0;
 
 	/* No new commands for the backend. */
-	spin_lock_irq(host->host_lock);
+	spin_lock_irq(&host->host_lock);
 	info->pause = 1;
 	while (info->callers && !err) {
 		info->waiting_pause = 1;
 		info->wait_ring_available = 0;
-		spin_unlock_irq(host->host_lock);
+		spin_unlock_irq(&host->host_lock);
 		wake_up(&info->wq_sync);
 		err = wait_event_interruptible(info->wq_pause,
 					       !info->waiting_pause);
-		spin_lock_irq(host->host_lock);
+		spin_lock_irq(&host->host_lock);
 	}
-	spin_unlock_irq(host->host_lock);
+	spin_unlock_irq(&host->host_lock);
 	return err;
 }
 
diff --git a/drivers/scsi/zorro7xx.c b/drivers/scsi/zorro7xx.c
index 21c769dc1ecbda..89475632675c0f 100644
--- a/drivers/scsi/zorro7xx.c
+++ b/drivers/scsi/zorro7xx.c
@@ -35,38 +35,45 @@ static struct scsi_host_template zorro7xx_scsi_driver_template = {
 	.module		= THIS_MODULE,
 };
 
-static struct zorro_driver_data {
+struct zorro_driver_data {
 	const char *name;
 	unsigned long offset;
 	int absolute;	/* offset is absolute address */
-} zorro7xx_driver_data[] = {
-	{ .name = "PowerUP 603e+", .offset = 0xf40000, .absolute = 1 },
-	{ .name = "WarpEngine 40xx", .offset = 0x40000 },
-	{ .name = "A4091", .offset = 0x800000 },
-	{ .name = "GForce 040/060", .offset = 0x40000 },
-	{ 0 }
+};
+
+static const struct zorro_driver_data zorro7xx_blizzard_603e_plus_data = {
+	.name = "PowerUP 603e+", .offset = 0xf40000, .absolute = 1
+};
+static const struct zorro_driver_data zorro7xx_warp_engine_40xx_data = {
+	.name = "WarpEngine 40xx", .offset = 0x40000
+};
+static const struct zorro_driver_data zorro7xx_a4091_data = {
+	.name = "A4091", .offset = 0x800000
+};
+static const struct zorro_driver_data zorro7xx_gforce_040_060_data = {
+	.name = "GForce 040/060", .offset = 0x40000
 };
 
 static struct zorro_device_id zorro7xx_zorro_tbl[] = {
 	{
 		.id = ZORRO_PROD_PHASE5_BLIZZARD_603E_PLUS,
-		.driver_data_ptr = &zorro7xx_driver_data[0],
+		.driver_data_ptr = &zorro7xx_blizzard_603e_plus_data,
 	},
 	{
 		.id = ZORRO_PROD_MACROSYSTEMS_WARP_ENGINE_40xx,
-		.driver_data_ptr = &zorro7xx_driver_data[1],
+		.driver_data_ptr = &zorro7xx_warp_engine_40xx_data,
 	},
 	{
 		.id = ZORRO_PROD_CBM_A4091_1,
-		.driver_data_ptr = &zorro7xx_driver_data[2],
+		.driver_data_ptr = &zorro7xx_a4091_data,
 	},
 	{
 		.id = ZORRO_PROD_CBM_A4091_2,
-		.driver_data_ptr = &zorro7xx_driver_data[2],
+		.driver_data_ptr = &zorro7xx_a4091_data,
 	},
 	{
 		.id = ZORRO_PROD_GVP_GFORCE_040_060,
-		.driver_data_ptr = &zorro7xx_driver_data[3],
+		.driver_data_ptr = &zorro7xx_gforce_040_060_data,
 	},
 	{ }
 };
diff --git a/drivers/slimbus/messaging.c b/drivers/slimbus/messaging.c
index e2dbe4a66b703a..ee127ef42046bf 100644
--- a/drivers/slimbus/messaging.c
+++ b/drivers/slimbus/messaging.c
@@ -139,7 +139,7 @@ int slim_do_transfer(struct slim_controller *ctrl, struct slim_msg_txn *txn)
 	if (need_tid) {
 		ret = slim_alloc_txn_tid(ctrl, txn);
 		if (ret)
-			return ret;
+			goto slim_xfer_err;
 
 		if (!txn->msg->comp)
 			txn->comp = &done;
diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c
index 6e89415712ce1a..eb983b6d1815fa 100644
--- a/drivers/slimbus/qcom-ngd-ctrl.c
+++ b/drivers/slimbus/qcom-ngd-ctrl.c
@@ -2,6 +2,7 @@
 // Copyright (c) 2011-2017, The Linux Foundation. All rights reserved.
 // Copyright (c) 2018, Linaro Limited
 
+#include <linux/bitfield.h>
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -87,6 +88,10 @@
 #define SLIM_ROOT_FREQ	24576000
 #define LADDR_RETRY	5
 
+#define SLIM_CHAN_CTRL_CMD		GENMASK(7, 6)
+#define SLIM_CHAN_CTRL_LADDR		GENMASK(4, 0)
+#define SLIM_MSG_HDR_LEN		4
+
 /* Per spec.max 40 bytes per received message */
 #define SLIM_MSGQ_BUF_LEN	40
 #define QCOM_SLIM_NGD_DESC_NUM	32
@@ -1086,6 +1091,78 @@ static int qcom_slim_ngd_enable_stream(struct slim_stream_runtime *rt)
 	return ret;
 }
 
+static int qcom_slim_ngd_disable_stream(struct slim_stream_runtime *rt)
+{
+	struct slim_device *sdev = rt->dev;
+	struct slim_controller *ctrl = sdev->ctrl;
+	struct slim_msg_txn txn = { 0 };
+	struct slim_val_inf msg = { 0 };
+	u8 wbuf[SLIM_MSGQ_BUF_LEN];
+	u8 rbuf[SLIM_MSGQ_BUF_LEN];
+	int i, ret;
+
+	txn.mt = SLIM_MSG_MT_DEST_REFERRED_USER;
+	txn.dt = SLIM_MSG_DEST_LOGICALADDR;
+	txn.la = SLIM_LA_MGR;
+	txn.ec = 0;
+	txn.msg = &msg;
+	txn.msg->num_bytes = 0;
+	txn.msg->wbuf = wbuf;
+	txn.msg->rbuf = rbuf;
+
+	for (i = 0; i < rt->num_ports; i++) {
+		struct slim_port *port = &rt->ports[i];
+
+		if (txn.msg->num_bytes == 0) {
+			wbuf[txn.msg->num_bytes++] =
+				FIELD_PREP(SLIM_CHAN_CTRL_CMD, SLIM_CH_REMOVE) |
+				FIELD_PREP(SLIM_CHAN_CTRL_LADDR, sdev->laddr);
+
+			ret = slim_alloc_txn_tid(ctrl, &txn);
+			if (ret) {
+				dev_err(&sdev->dev, "Fail to allocate TID ret:%d\n", ret);
+				return ret;
+			}
+			wbuf[txn.msg->num_bytes++] = txn.tid;
+		}
+		wbuf[txn.msg->num_bytes++] = port->ch.id;
+	}
+
+	txn.mc = SLIM_USR_MC_CHAN_CTRL;
+	txn.rl = txn.msg->num_bytes + SLIM_MSG_HDR_LEN;
+	ret = qcom_slim_ngd_xfer_msg_sync(ctrl, &txn);
+	if (ret) {
+		slim_free_txn_tid(ctrl, &txn);
+		/* Controller restarting, the channels are gone anyway */
+		if (ret == -EREMOTEIO)
+			return 0;
+		dev_err(&sdev->dev, "TX timed out: MC: 0x%x, mt: 0x%x, laddr: 0x%x, ret: %d\n",
+			txn.mc, txn.mt, sdev->laddr, ret);
+		return ret;
+	}
+
+	txn.mc = SLIM_USR_MC_RECONFIG_NOW;
+	txn.msg->num_bytes = 2;
+	wbuf[1] = sdev->laddr;
+	txn.rl = txn.msg->num_bytes + SLIM_MSG_HDR_LEN;
+
+	ret = slim_alloc_txn_tid(ctrl, &txn);
+	if (ret) {
+		dev_err(&sdev->dev, "Fail to allocate TID ret:%d\n", ret);
+		return ret;
+	}
+
+	wbuf[0] = txn.tid;
+	ret = qcom_slim_ngd_xfer_msg_sync(ctrl, &txn);
+	if (ret) {
+		slim_free_txn_tid(ctrl, &txn);
+		dev_err(&sdev->dev, "TX timed out: MC: 0x%x, mt: 0x%x, laddr: 0x%x, ret: %d\n",
+			txn.mc, txn.mt, sdev->laddr, ret);
+	}
+
+	return ret;
+}
+
 static int qcom_slim_ngd_get_laddr(struct slim_controller *ctrl,
 				   struct slim_eaddr *ea, u8 *laddr)
 {
@@ -1539,7 +1616,7 @@ static int of_qcom_slim_ngd_register(struct device *parent,
 			kfree(ngd);
 			return ret;
 		}
-		platform_device_set_of_node(ngd->pdev, node);
+		platform_device_set_fwnode(ngd->pdev, of_fwnode_handle(node));
 		ctrl->ngd = ngd;
 
 		ret = platform_device_add(ngd->pdev);
@@ -1614,7 +1691,7 @@ static int qcom_slim_ngd_ctrl_probe(struct platform_device *pdev)
 			       IRQF_TRIGGER_HIGH | IRQF_NO_AUTOEN,
 			       "slim-ngd", ctrl);
 	if (ret)
-		return dev_err_probe(&pdev->dev, ret, "request IRQ failed\n");
+		return ret;
 
 	ctrl->dev = dev;
 	ctrl->framer.rootfreq = SLIM_ROOT_FREQ >> 3;
@@ -1625,6 +1702,7 @@ static int qcom_slim_ngd_ctrl_probe(struct platform_device *pdev)
 	ctrl->ctrl.clkgear = SLIM_MAX_CLK_GEAR;
 	ctrl->ctrl.get_laddr = qcom_slim_ngd_get_laddr;
 	ctrl->ctrl.enable_stream = qcom_slim_ngd_enable_stream;
+	ctrl->ctrl.disable_stream = qcom_slim_ngd_disable_stream;
 	ctrl->ctrl.xfer_msg = qcom_slim_ngd_xfer_msg;
 	ctrl->ctrl.wakeup = NULL;
 	ctrl->state = QCOM_SLIM_NGD_CTRL_DOWN;
diff --git a/drivers/slimbus/slimbus.h b/drivers/slimbus/slimbus.h
index 00a7f112574b53..8e34f30601d309 100644
--- a/drivers/slimbus/slimbus.h
+++ b/drivers/slimbus/slimbus.h
@@ -316,6 +316,18 @@ enum slim_transport_protocol {
 	SLIM_PROTO_EXT_HALF_DUP,
 };
 
+/**
+ * enum slim_ch_control: Channel control.
+ * @SLIM_CH_ACTIVATE: Schedules channel or group of channels in the TDM frame.
+ * @SLIM_CH_SUSPEND: Keeps the TDM schedule but halts data transfer.
+ * @SLIM_CH_REMOVE: Drops the channel or group from the TDM frame.
+ */
+enum slim_ch_control {
+	SLIM_CH_ACTIVATE,
+	SLIM_CH_SUSPEND,
+	SLIM_CH_REMOVE,
+};
+
 /**
  * struct slim_stream_runtime  - SLIMbus stream runtime instance
  *
@@ -325,7 +337,7 @@ enum slim_transport_protocol {
  * @prot: Transport protocol used in this stream
  * @rate: Data rate of samples *
  * @bps: bits per sample
- * @ratem: rate multipler which is super frame rate/data rate
+ * @ratem: rate multiplier which is super frame rate/data rate
  * @num_ports: number of ports
  * @ports: pointer to instance of ports
  * @node: list head for stream associated with slim device.
diff --git a/drivers/soc/apple/rtkit-crashlog.c b/drivers/soc/apple/rtkit-crashlog.c
index 8319e365110b87..195b7869ce5846 100644
--- a/drivers/soc/apple/rtkit-crashlog.c
+++ b/drivers/soc/apple/rtkit-crashlog.c
@@ -13,7 +13,8 @@
 #define APPLE_RTKIT_CRASHLOG_VERSION FOURCC('C', 'v', 'e', 'r')
 #define APPLE_RTKIT_CRASHLOG_MBOX FOURCC('C', 'm', 'b', 'x')
 #define APPLE_RTKIT_CRASHLOG_TIME FOURCC('C', 't', 'i', 'm')
-#define APPLE_RTKIT_CRASHLOG_REGS FOURCC('C', 'r', 'g', '8')
+#define APPLE_RTKIT_CRASHLOG_AARCH32 FOURCC('C', 'r', 'g', 'A')
+#define APPLE_RTKIT_CRASHLOG_AARCH64 FOURCC('C', 'r', 'g', '8')
 
 /* For COMPILE_TEST on non-ARM64 architectures */
 #ifndef PSR_MODE_EL0t
@@ -25,6 +26,21 @@
 #define PSR_MODE_MASK	0x0000000f
 #endif
 
+/*
+ * AArch32 mode definitions: APPLE_RTKIT depends on APPLE_MAILBOX, which
+ * depends on 64BIT, so there is no need to care about COMPILE_TEST on ARM.
+ */
+#define USR_MODE        0x00000010
+#define FIQ_MODE        0x00000011
+#define IRQ_MODE        0x00000012
+#define SVC_MODE        0x00000013
+#define MON_MODE        0x00000016
+#define ABT_MODE        0x00000017
+#define HYP_MODE        0x0000001a
+#define UND_MODE        0x0000001b
+#define SYSTEM_MODE     0x0000001f
+#define MODE_MASK_32    0x0000001f
+
 struct apple_rtkit_crashlog_header {
 	u32 fourcc;
 	u32 version;
@@ -42,7 +58,7 @@ struct apple_rtkit_crashlog_mbox_entry {
 };
 static_assert(sizeof(struct apple_rtkit_crashlog_mbox_entry) == 0x18);
 
-struct apple_rtkit_crashlog_regs {
+struct apple_rtkit_crashlog_aarch64 {
 	u32 unk_0;
 	u32 unk_4;
 	u64 regs[31];
@@ -58,7 +74,21 @@ struct apple_rtkit_crashlog_regs {
 	u64 esr;
 	u64 unk_Z;
 } __packed;
-static_assert(sizeof(struct apple_rtkit_crashlog_regs) == 0x350);
+static_assert(sizeof(struct apple_rtkit_crashlog_aarch64) == 0x350);
+
+struct apple_rtkit_crashlog_aarch32 {
+	u32 unk_0;
+	u32 unk_4;
+	u32 regs[16];
+	u32 psr;
+	u32 unk_x1;
+	u8 stack[0x100];
+	u32 unk_x2;
+	u32 unk_x3;
+	u32 unk_x4;
+	u32 unk_x5;
+} __packed;
+static_assert(sizeof(struct apple_rtkit_crashlog_aarch32) == 0x160);
 
 static void apple_rtkit_crashlog_dump_str(struct apple_rtkit *rtk, u8 *bfr,
 					  size_t size)
@@ -123,21 +153,84 @@ static void apple_rtkit_crashlog_dump_mailbox(struct apple_rtkit *rtk, u8 *bfr,
 	}
 }
 
-static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr,
-					   size_t size)
+static void apple_rtkit_crashlog_dump_aarch32(struct apple_rtkit *rtk, u8 *bfr,
+					      size_t size)
 {
-	struct apple_rtkit_crashlog_regs *regs;
+	struct apple_rtkit_crashlog_aarch32 *state;
+	const char *mode;
+	int i;
+
+	if (size < sizeof(*state)) {
+		dev_warn(rtk->dev, "RTKit: AArch32 section too small: 0x%zx", size);
+		return;
+	}
+
+	state = (struct apple_rtkit_crashlog_aarch32 *)bfr;
+
+	switch (state->psr & MODE_MASK_32) {
+	case USR_MODE:
+		mode = "User";
+		break;
+	case FIQ_MODE:
+		mode = "FIQ";
+		break;
+	case IRQ_MODE:
+		mode = "IRQ";
+		break;
+	case SVC_MODE:
+		mode = "Supervisor";
+		break;
+	case MON_MODE:
+		mode = "Monitor";
+		break;
+	case ABT_MODE:
+		mode = "Abort";
+		break;
+	case HYP_MODE:
+		mode = "Hypervisor";
+		break;
+	case UND_MODE:
+		mode = "Undefined";
+		break;
+	case SYSTEM_MODE:
+		mode = "System";
+		break;
+	default:
+		mode = "Unknown";
+		break;
+	}
+
+	dev_warn(rtk->dev, "RTKit: Exception dump:");
+	dev_warn(rtk->dev, "  == Exception taken from %s mode ==", mode);
+	dev_warn(rtk->dev, "  PSR    = 0x%x", state->psr);
+	dev_warn(rtk->dev, "\n");
+
+	for (i = 0; i < 16; i += 4)
+		dev_warn(rtk->dev,
+				 "  r%02d-r%02d = %08x %08x %08x %08x\n",
+				 i, i + 3,
+				 state->regs[i], state->regs[i + 1],
+				 state->regs[i + 2], state->regs[i + 3]);
+
+	dev_warn(rtk->dev, "\n");
+}
+
+static void apple_rtkit_crashlog_dump_aarch64(struct apple_rtkit *rtk, u8 *bfr,
+					      size_t size)
+{
+	struct apple_rtkit_crashlog_aarch64 *state;
 	const char *el;
 	int i;
 
-	if (size < sizeof(*regs)) {
-		dev_warn(rtk->dev, "RTKit: Regs section too small: 0x%zx", size);
+	if (size < sizeof(*state)) {
+		dev_warn(rtk->dev, "RTKit: AArch64 section too small: 0x%zx",
+			 size);
 		return;
 	}
 
-	regs = (struct apple_rtkit_crashlog_regs *)bfr;
+	state = (struct apple_rtkit_crashlog_aarch64 *)bfr;
 
-	switch (regs->psr & PSR_MODE_MASK) {
+	switch (state->psr & PSR_MODE_MASK) {
 	case PSR_MODE_EL0t:
 		el = "EL0t";
 		break;
@@ -160,11 +253,11 @@ static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr,
 
 	dev_warn(rtk->dev, "RTKit: Exception dump:");
 	dev_warn(rtk->dev, "  == Exception taken from %s ==", el);
-	dev_warn(rtk->dev, "  PSR    = 0x%llx", regs->psr);
-	dev_warn(rtk->dev, "  PC     = 0x%llx\n", regs->pc);
-	dev_warn(rtk->dev, "  ESR    = 0x%llx\n", regs->esr);
-	dev_warn(rtk->dev, "  FAR    = 0x%llx\n", regs->far);
-	dev_warn(rtk->dev, "  SP     = 0x%llx\n", regs->sp);
+	dev_warn(rtk->dev, "  PSR    = 0x%llx", state->psr);
+	dev_warn(rtk->dev, "  PC     = 0x%llx\n", state->pc);
+	dev_warn(rtk->dev, "  ESR    = 0x%llx\n", state->esr);
+	dev_warn(rtk->dev, "  FAR    = 0x%llx\n", state->far);
+	dev_warn(rtk->dev, "  SP     = 0x%llx\n", state->sp);
 	dev_warn(rtk->dev, "\n");
 
 	for (i = 0; i < 31; i += 4) {
@@ -172,12 +265,12 @@ static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr,
 			dev_warn(rtk->dev,
 					 "  x%02d-x%02d = %016llx %016llx %016llx %016llx\n",
 					 i, i + 3,
-					 regs->regs[i], regs->regs[i + 1],
-					 regs->regs[i + 2], regs->regs[i + 3]);
+					 state->regs[i], state->regs[i + 1],
+					 state->regs[i + 2], state->regs[i + 3]);
 		else
 			dev_warn(rtk->dev,
 					 "  x%02d-x%02d = %016llx %016llx %016llx\n", i, i + 3,
-					 regs->regs[i], regs->regs[i + 1], regs->regs[i + 2]);
+					 state->regs[i], state->regs[i + 1], state->regs[i + 2]);
 	}
 
 	dev_warn(rtk->dev, "\n");
@@ -229,9 +322,13 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size)
 			apple_rtkit_crashlog_dump_time(rtk, bfr + offset + 16,
 						       section_size);
 			break;
-		case APPLE_RTKIT_CRASHLOG_REGS:
-			apple_rtkit_crashlog_dump_regs(rtk, bfr + offset + 16,
-						       section_size);
+		case APPLE_RTKIT_CRASHLOG_AARCH32:
+			apple_rtkit_crashlog_dump_aarch32(rtk, bfr + offset + 16,
+							  section_size);
+			break;
+		case APPLE_RTKIT_CRASHLOG_AARCH64:
+			apple_rtkit_crashlog_dump_aarch64(rtk, bfr + offset + 16,
+							  section_size);
 			break;
 		default:
 			dev_warn(rtk->dev,
diff --git a/drivers/soc/atmel/soc.c b/drivers/soc/atmel/soc.c
index d89f9dcade7d26..7e60c5681ad0d8 100644
--- a/drivers/soc/atmel/soc.c
+++ b/drivers/soc/atmel/soc.c
@@ -404,9 +404,7 @@ static const struct of_device_id at91_soc_allowed_list[] __initconst = {
 
 static int __init atmel_soc_device_init(void)
 {
-	struct device_node *np __free(device_node) = of_find_node_by_path("/");
-
-	if (!of_match_node(at91_soc_allowed_list, np))
+	if (!of_machine_device_match(at91_soc_allowed_list))
 		return 0;
 
 	at91_soc_init(socs);
diff --git a/drivers/soc/fsl/qe/gpio.c b/drivers/soc/fsl/qe/gpio.c
index 5cd59cfbf22fa6..1b5a1b134596e8 100644
--- a/drivers/soc/fsl/qe/gpio.c
+++ b/drivers/soc/fsl/qe/gpio.c
@@ -32,8 +32,13 @@ struct qe_gpio_chip {
 	/* shadowed data register to clear/set bits safely */
 	u32 cpdata;
 
-	/* saved_regs used to restore dedicated functions */
-	struct qe_pio_regs saved_regs;
+	/* saved regs used to restore dedicated functions */
+	u32 saved_cpodr;
+	u32 saved_cpdata;
+	u32 saved_cpdir1;
+	u32 saved_cpdir2;
+	u32 saved_cppar1;
+	u32 saved_cppar2;
 };
 
 static void qe_gpio_save_regs(struct qe_gpio_chip *qe_gc)
@@ -41,12 +46,12 @@ static void qe_gpio_save_regs(struct qe_gpio_chip *qe_gc)
 	struct qe_pio_regs __iomem *regs = qe_gc->regs;
 
 	qe_gc->cpdata = ioread32be(&regs->cpdata);
-	qe_gc->saved_regs.cpdata = qe_gc->cpdata;
-	qe_gc->saved_regs.cpdir1 = ioread32be(&regs->cpdir1);
-	qe_gc->saved_regs.cpdir2 = ioread32be(&regs->cpdir2);
-	qe_gc->saved_regs.cppar1 = ioread32be(&regs->cppar1);
-	qe_gc->saved_regs.cppar2 = ioread32be(&regs->cppar2);
-	qe_gc->saved_regs.cpodr = ioread32be(&regs->cpodr);
+	qe_gc->saved_cpdata = qe_gc->cpdata;
+	qe_gc->saved_cpdir1 = ioread32be(&regs->cpdir1);
+	qe_gc->saved_cpdir2 = ioread32be(&regs->cpdir2);
+	qe_gc->saved_cppar1 = ioread32be(&regs->cppar1);
+	qe_gc->saved_cppar2 = ioread32be(&regs->cppar2);
+	qe_gc->saved_cpodr = ioread32be(&regs->cpodr);
 }
 
 static int qe_gpio_get(struct gpio_chip *gc, unsigned int gpio)
@@ -196,18 +201,17 @@ struct qe_pin {
 /**
  * qe_pin_request - Request a QE pin
  * @dev:	device to get the pin from
- * @index:	index of the pin in the device tree
+ * @gpiod:	GPIO descriptor associated with this pin
  * Context:	non-atomic
  *
  * This function return qe_pin so that you could use it with the rest of
  * the QE Pin Multiplexing API.
  */
-struct qe_pin *qe_pin_request(struct device *dev, int index)
+struct qe_pin *qe_pin_request(struct device *dev, struct gpio_desc *gpiod)
 {
+	struct gpio_device *gdev;
 	struct qe_pin *qe_pin;
 	struct gpio_chip *gc;
-	struct gpio_desc *gpiod;
-	int gpio_num;
 	int err;
 
 	qe_pin = kzalloc_obj(*qe_pin);
@@ -216,34 +220,20 @@ struct qe_pin *qe_pin_request(struct device *dev, int index)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	/*
-	 * Request gpio as nonexclusive as it was likely reserved by the
-	 * caller, and we are not planning on controlling it, we only need
-	 * the descriptor to the to the gpio chip structure.
-	 */
-	gpiod = gpiod_get_index(dev, NULL, index,
-			        GPIOD_ASIS | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
-	err = PTR_ERR_OR_ZERO(gpiod);
-	if (err)
-		goto err0;
-
-	gc = gpiod_to_chip(gpiod);
-	gpio_num = desc_to_gpio(gpiod);
-	/* We no longer need this descriptor */
-	gpiod_put(gpiod);
-
-	if (WARN_ON(!gc)) {
+	gdev = gpiod_to_gpio_device(gpiod);
+	if (WARN_ON(!gdev)) {
 		err = -ENODEV;
 		goto err0;
 	}
 
+	gc = gpio_device_get_chip(gdev);
 	qe_pin->controller = gpiochip_get_data(gc);
 	/*
 	 * FIXME: this gets the local offset on the gpio_chip so that the driver
 	 * can manipulate pin control settings through its custom API. The real
 	 * solution is to create a real pin control driver for this.
 	 */
-	qe_pin->num = gpio_num - gc->base;
+	qe_pin->num = desc_to_gpio(gpiod) - gc->base;
 
 	if (!fwnode_device_is_compatible(gc->fwnode, "fsl,mpc8323-qe-pario-bank")) {
 		dev_dbg(dev, "%s: tried to get a non-qe pin\n", __func__);
@@ -284,7 +274,6 @@ void qe_pin_set_dedicated(struct qe_pin *qe_pin)
 {
 	struct qe_gpio_chip *qe_gc = qe_pin->controller;
 	struct qe_pio_regs __iomem *regs = qe_gc->regs;
-	struct qe_pio_regs *sregs = &qe_gc->saved_regs;
 	int pin = qe_pin->num;
 	u32 mask1 = 1 << (QE_PIO_PINS - (pin + 1));
 	u32 mask2 = 0x3 << (QE_PIO_PINS - (pin % (QE_PIO_PINS / 2) + 1) * 2);
@@ -294,24 +283,20 @@ void qe_pin_set_dedicated(struct qe_pin *qe_pin)
 	spin_lock_irqsave(&qe_gc->lock, flags);
 
 	if (second_reg) {
-		qe_clrsetbits_be32(&regs->cpdir2, mask2,
-				   sregs->cpdir2 & mask2);
-		qe_clrsetbits_be32(&regs->cppar2, mask2,
-				   sregs->cppar2 & mask2);
+		qe_clrsetbits_be32(&regs->cpdir2, mask2, qe_gc->saved_cpdir2 & mask2);
+		qe_clrsetbits_be32(&regs->cppar2, mask2, qe_gc->saved_cppar2 & mask2);
 	} else {
-		qe_clrsetbits_be32(&regs->cpdir1, mask2,
-				   sregs->cpdir1 & mask2);
-		qe_clrsetbits_be32(&regs->cppar1, mask2,
-				   sregs->cppar1 & mask2);
+		qe_clrsetbits_be32(&regs->cpdir1, mask2, qe_gc->saved_cpdir1 & mask2);
+		qe_clrsetbits_be32(&regs->cppar1, mask2, qe_gc->saved_cppar1 & mask2);
 	}
 
-	if (sregs->cpdata & mask1)
+	if (qe_gc->saved_cpdata & mask1)
 		qe_gc->cpdata |= mask1;
 	else
 		qe_gc->cpdata &= ~mask1;
 
 	iowrite32be(qe_gc->cpdata, &regs->cpdata);
-	qe_clrsetbits_be32(&regs->cpodr, mask1, sregs->cpodr & mask1);
+	qe_clrsetbits_be32(&regs->cpodr, mask1, qe_gc->saved_cpodr & mask1);
 	spin_unlock_irqrestore(&qe_gc->lock, flags);
 }
 EXPORT_SYMBOL(qe_pin_set_dedicated);
diff --git a/drivers/soc/fsl/qe/qe_ports_ic.c b/drivers/soc/fsl/qe/qe_ports_ic.c
index 7375f92f528b92..fb3b92039547a6 100644
--- a/drivers/soc/fsl/qe/qe_ports_ic.c
+++ b/drivers/soc/fsl/qe/qe_ports_ic.c
@@ -140,7 +140,6 @@ static int qepic_probe(struct platform_device *pdev)
 	};
 	struct irq_domain_info d_info = {
 		.fwnode = of_fwnode_handle(pdev->dev.of_node),
-		.domain_flags = IRQ_DOMAIN_FLAG_DESTROY_GC,
 		.size = 32,
 		.hwirq_max = 32,
 		.ops = &irq_generic_chip_ops,
diff --git a/drivers/soc/fsl/qe/tsa.c b/drivers/soc/fsl/qe/tsa.c
index 4a88e54d25b949..94691369353e1c 100644
--- a/drivers/soc/fsl/qe/tsa.c
+++ b/drivers/soc/fsl/qe/tsa.c
@@ -865,12 +865,12 @@ static int tsa_of_parse_tdms(struct tsa *tsa, struct device_node *np)
 			clk_put(tsa->tdm[i].l1rclk_clk);
 		}
 		if (tsa->tdm[i].l1tsync_clk) {
-			clk_disable_unprepare(tsa->tdm[i].l1rsync_clk);
-			clk_put(tsa->tdm[i].l1rsync_clk);
+			clk_disable_unprepare(tsa->tdm[i].l1tsync_clk);
+			clk_put(tsa->tdm[i].l1tsync_clk);
 		}
 		if (tsa->tdm[i].l1tclk_clk) {
-			clk_disable_unprepare(tsa->tdm[i].l1rclk_clk);
-			clk_put(tsa->tdm[i].l1rclk_clk);
+			clk_disable_unprepare(tsa->tdm[i].l1tclk_clk);
+			clk_put(tsa->tdm[i].l1tclk_clk);
 		}
 	}
 	return ret;
diff --git a/drivers/soc/mediatek/mt8167-mmsys.h b/drivers/soc/mediatek/mt8167-mmsys.h
index 001379373507a6..858779ace8951f 100644
--- a/drivers/soc/mediatek/mt8167-mmsys.h
+++ b/drivers/soc/mediatek/mt8167-mmsys.h
@@ -5,35 +5,200 @@
 
 #define MT8167_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN	0x030
 #define MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN	0x038
+#define MT8167_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN	0x03c
 #define MT8167_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN	0x058
+#define MT8167_DISP_REG_CONFIG_DISP_WDMA0_SEL_IN	0x05c
+#define MT8167_DISP_REG_CONFIG_DISP_UFOE_SEL_IN		0x060
 #define MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN		0x064
+#define MT8167_DISP_REG_CONFIG_DISP_DPI0_SEL_IN		0x068
 #define MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN	0x06c
+#define MT8167_DISP_REG_CONFIG_DISP_RDMA1_SOUT_SEL_IN	0x070
+#define MT8167_DISP_REG_CONFIG_DISP_DPI1_SEL_IN		0x074
+
+#define MT8167_OVL0_MOUT_EN_COLOR0			0x1
+#define MT8167_OVL0_MOUT_EN_WDMA0			0x2
+#define MT8167_OVL0_MOUT_EN_MASK			0x3
 
 #define MT8167_DITHER_MOUT_EN_RDMA0			0x1
+#define MT8167_DITHER_MOUT_EN_UFOE			0x2
+#define MT8167_DITHER_MOUT_EN_WDMA0			0x4
 #define MT8167_DITHER_MOUT_EN_MASK			0x7
 
-#define MT8167_RDMA0_SOUT_DSI0				0x2
-#define MT8167_RDMA0_SOUT_MASK				0x3
+#define MT8167_UFOE_MOUT_EN_DSI0			0x1
+#define MT8167_UFOE_MOUT_EN_DPI0			0x2
+/*
+ * For some reason Android sources and the data sheet don't agree here,
+ * the data sheet has UFOE_MOUT_EN_WDMA0 as 0x4 and does not mention DPI1.
+ */
+#define MT8167_UFOE_MOUT_EN_DPI1			0x4
+#define MT8167_UFOE_MOUT_EN_WDMA0			0x8
+#define MT8167_UFOE_MOUT_EN_MASK			0xF
+
+#define MT8167_COLOR0_SEL_IN_RDMA0			0x0
+#define MT8167_COLOR0_SEL_IN_OVL0			0x1
+#define MT8167_COLOR0_SEL_IN_MASK			0x1
+
+#define MT8167_WDMA0_SEL_IN_OVL0			0x0
+#define MT8167_WDMA0_SEL_IN_DITHER			0x1
+#define MT8167_WDMA0_SEL_IN_UFOE			0x2
+#define MT8167_WDMA0_SEL_IN_MASK			0x3
+
+#define MT8167_UFOE_SEL_IN_RDMA0			0x0
+#define MT8167_UFOE_SEL_IN_DITHER			0x1
+#define MT8167_UFOE_SEL_IN_MASK				0x1
 
+#define MT8167_DSI0_SEL_IN_UFOE				0x0
 #define MT8167_DSI0_SEL_IN_RDMA0			0x1
+#define MT8167_DSI0_SEL_IN_RDMA1			0x2
 #define MT8167_DSI0_SEL_IN_MASK				0x3
 
+#define MT8167_DPI0_SEL_IN_UFOE				0x0
+#define MT8167_DPI0_SEL_IN_RDMA0			0x1
+#define MT8167_DPI0_SEL_IN_RDMA1			0x2
+#define MT8167_DPI0_SEL_IN_MASK				0x3
+
+#define MT8167_DPI1_SEL_IN_UFOE				0x0
+#define MT8167_DPI1_SEL_IN_RDMA0			0x1
+#define MT8167_DPI1_SEL_IN_RDMA1			0x2
+#define MT8167_DPI1_SEL_IN_MASK				0x3
+
+#define MT8167_RDMA0_SOUT_UFOE				0x0
+#define MT8167_RDMA0_SOUT_COLOR0			0x1
+#define MT8167_RDMA0_SOUT_DSI0				0x2
+#define MT8167_RDMA0_SOUT_DPI0				0x3
+/* DPI1 is only defined in Android sources, not in the data sheet */
+#define MT8167_RDMA0_SOUT_DPI1				0x4
+#define MT8167_RDMA0_SOUT_MASK				0x7
+
+#define MT8167_RDMA1_SOUT_DSI0				0x0
+#define MT8167_RDMA1_SOUT_DPI0				0x1
+/* DPI1 is only defined in Android sources, not in the data sheet */
+#define MT8167_RDMA1_SOUT_DPI1				0x2
+#define MT8167_RDMA1_SOUT_MASK				0x3
+
 static const struct mtk_mmsys_routes mt8167_mmsys_routing_table[] = {
+	/* OVL0<->COLOR0 */
 	MMSYS_ROUTE(OVL, 0, COLOR, 0,
-		    MT8167_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, OVL0_MOUT_EN_COLOR0,
-		    OVL0_MOUT_EN_COLOR0),
+		    MT8167_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, MT8167_OVL0_MOUT_EN_MASK,
+		    MT8167_OVL0_MOUT_EN_COLOR0),
+	MMSYS_ROUTE(OVL, 0, COLOR, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, MT8167_COLOR0_SEL_IN_MASK,
+		    MT8167_COLOR0_SEL_IN_OVL0),
+
+	/* OVL0<->WDMA0 */
+	MMSYS_ROUTE(OVL, 0, WDMA, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, MT8167_OVL0_MOUT_EN_MASK,
+		    MT8167_OVL0_MOUT_EN_WDMA0),
+	MMSYS_ROUTE(OVL, 0, WDMA, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_WDMA0_SEL_IN, MT8167_COLOR0_SEL_IN_MASK,
+		    MT8167_WDMA0_SEL_IN_OVL0),
+
+	/* DITHER0 outputs */
 	MMSYS_ROUTE(DITHER, 0, RDMA, 0,
 		    MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_MASK,
 		    MT8167_DITHER_MOUT_EN_RDMA0),
-	MMSYS_ROUTE(OVL, 0, COLOR, 0,
-		    MT8167_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, COLOR0_SEL_IN_OVL0,
-		    COLOR0_SEL_IN_OVL0),
-	MMSYS_ROUTE(RDMA, 0, DSI, 0,
+	MMSYS_ROUTE(DITHER, 0, UFOE, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_MASK,
+		    MT8167_DITHER_MOUT_EN_UFOE),
+	MMSYS_ROUTE(DITHER, 0, WDMA, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_MASK,
+		    MT8167_DITHER_MOUT_EN_WDMA0),
+
+	/* UFOE<->DSI0 */
+	MMSYS_ROUTE(UFOE, 0, DSI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN, MT8167_UFOE_MOUT_EN_MASK,
+		    MT8167_UFOE_MOUT_EN_DSI0),
+	MMSYS_ROUTE(UFOE, 0, DSI, 0,
 		    MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_MASK,
-		    MT8167_DSI0_SEL_IN_RDMA0),
+		    MT8167_DSI0_SEL_IN_UFOE),
+
+	/* UFOE<->DPI0 */
+	MMSYS_ROUTE(UFOE, 0, DPI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN, MT8167_UFOE_MOUT_EN_MASK,
+		    MT8167_UFOE_MOUT_EN_DPI0),
+	MMSYS_ROUTE(UFOE, 0, DPI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DPI0_SEL_IN, MT8167_DPI0_SEL_IN_MASK,
+		    MT8167_DPI0_SEL_IN_UFOE),
+
+	/* UFOE<->DPI1 */
+	MMSYS_ROUTE(UFOE, 0, DPI, 1,
+		    MT8167_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN, MT8167_UFOE_MOUT_EN_MASK,
+		    MT8167_UFOE_MOUT_EN_DPI1),
+	MMSYS_ROUTE(UFOE, 0, DPI, 1,
+		    MT8167_DISP_REG_CONFIG_DISP_DPI1_SEL_IN, MT8167_DPI1_SEL_IN_MASK,
+		    MT8167_DPI1_SEL_IN_UFOE),
+
+	/* UFOE<->WDMA0 */
+	MMSYS_ROUTE(UFOE, 0, WDMA, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_UFOE_MOUT_EN, MT8167_UFOE_MOUT_EN_MASK,
+		    MT8167_UFOE_MOUT_EN_WDMA0),
+	MMSYS_ROUTE(UFOE, 0, WDMA, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_WDMA0_SEL_IN, MT8167_WDMA0_SEL_IN_MASK,
+		    MT8167_WDMA0_SEL_IN_UFOE),
+
+	/* RDMA0<->UFOE */
+	MMSYS_ROUTE(RDMA, 0, UFOE, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK,
+		    MT8167_RDMA0_SOUT_UFOE),
+	MMSYS_ROUTE(RDMA, 0, UFOE, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_UFOE_SEL_IN, MT8167_UFOE_SEL_IN_MASK,
+		    MT8167_UFOE_SEL_IN_RDMA0),
+
+	/* RDMA0->COLOR0 */
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK,
+		    MT8167_RDMA0_SOUT_COLOR0),
+	MMSYS_ROUTE(RDMA, 0, COLOR, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, MT8167_COLOR0_SEL_IN_MASK,
+		    MT8167_COLOR0_SEL_IN_RDMA0),
+
+	/* RDMA0<->DSI0 */
 	MMSYS_ROUTE(RDMA, 0, DSI, 0,
 		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK,
 		    MT8167_RDMA0_SOUT_DSI0),
+	MMSYS_ROUTE(RDMA, 0, DSI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_MASK,
+		    MT8167_DSI0_SEL_IN_RDMA0),
+
+	/* RDMA0<->DPI0 */
+	MMSYS_ROUTE(RDMA, 0, DPI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK,
+		    MT8167_RDMA0_SOUT_DPI0),
+	MMSYS_ROUTE(RDMA, 0, DPI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DPI0_SEL_IN, MT8167_DPI0_SEL_IN_MASK,
+		    MT8167_DPI0_SEL_IN_RDMA0),
+
+	/* RDMA0<->DPI1 */
+	MMSYS_ROUTE(RDMA, 0, DPI, 1,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK,
+		    MT8167_RDMA0_SOUT_DPI1),
+	MMSYS_ROUTE(RDMA, 0, DPI, 1,
+		    MT8167_DISP_REG_CONFIG_DISP_DPI1_SEL_IN, MT8167_DPI1_SEL_IN_MASK,
+		    MT8167_DPI1_SEL_IN_RDMA0),
+
+	/* RDMA1<->DSI0 */
+	MMSYS_ROUTE(RDMA, 1, DSI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA1_SOUT_SEL_IN, MT8167_RDMA1_SOUT_MASK,
+		    MT8167_RDMA1_SOUT_DSI0),
+	MMSYS_ROUTE(RDMA, 1, DSI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_MASK,
+		    MT8167_DSI0_SEL_IN_RDMA1),
+
+	/* RDMA1<->DPI0 */
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA1_SOUT_SEL_IN, MT8167_RDMA1_SOUT_MASK,
+		    MT8167_RDMA1_SOUT_DPI0),
+	MMSYS_ROUTE(RDMA, 1, DPI, 0,
+		    MT8167_DISP_REG_CONFIG_DISP_DPI0_SEL_IN, MT8167_DPI0_SEL_IN_MASK,
+		    MT8167_DPI0_SEL_IN_RDMA1),
+
+	/* RDMA1<->DPI1 */
+	MMSYS_ROUTE(RDMA, 1, DPI, 1,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA1_SOUT_SEL_IN, MT8167_RDMA1_SOUT_MASK,
+		    MT8167_RDMA1_SOUT_DPI1),
+	MMSYS_ROUTE(RDMA, 1, DPI, 1,
+		    MT8167_DISP_REG_CONFIG_DISP_DPI1_SEL_IN, MT8167_DPI1_SEL_IN_MASK,
+		    MT8167_DPI1_SEL_IN_RDMA1),
 };
 
 #endif /* __SOC_MEDIATEK_MT8167_MMSYS_H */
diff --git a/drivers/soc/mediatek/mtk-regulator-coupler.c b/drivers/soc/mediatek/mtk-regulator-coupler.c
index 0b6a2884145e37..1109e3d9a319be 100644
--- a/drivers/soc/mediatek/mtk-regulator-coupler.c
+++ b/drivers/soc/mediatek/mtk-regulator-coupler.c
@@ -148,6 +148,7 @@ static int mediatek_regulator_coupler_init(void)
 	if (!of_machine_is_compatible("mediatek,mt8183") &&
 	    !of_machine_is_compatible("mediatek,mt8186") &&
 	    !of_machine_is_compatible("mediatek,mt8188") &&
+	    !of_machine_is_compatible("mediatek,mt8189") &&
 	    !of_machine_is_compatible("mediatek,mt8192"))
 		return 0;
 
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index 2b524154d9fb33..535c8619197b3e 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -39,7 +39,6 @@ config QCOM_COMMAND_DB
 
 config QCOM_GENI_SE
 	tristate "Qualcomm GENI Serial Engine Driver"
-	depends on ARM64 || COMPILE_TEST
 	default ARCH_QCOM
 	help
 	  This driver is used to manage Generic Interface (GENI) firmware based
@@ -319,6 +318,18 @@ config QCOM_QMI_HELPERS
 	tristate
 	depends on NET
 
+config QCOM_QMI_TMD
+	tristate "Qualcomm remote subsystem TMD" if COMPILE_TEST
+	depends on NET
+	depends on ARCH_QCOM || COMPILE_TEST
+	select QRTR
+	select QCOM_QMI_HELPERS
+	help
+	  This enables Qualcomm Messaging Interface (QMI) based Thermal Mitigation
+	  Device (TMD) support for Qualcomm remote subsystems. It manages
+	  TMD messaging and handles QMI communication with remote processors
+	  to exchange mitigation state and apply thermal mitigation requests.
+
 config QCOM_UBWC_CONFIG
 	tristate
 	depends on QCOM_SMEM
diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile
index 798643be3590cc..bddd6f0a583673 100644
--- a/drivers/soc/qcom/Makefile
+++ b/drivers/soc/qcom/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_QCOM_PMIC_GLINK)	+= pmic_glink.o
 obj-$(CONFIG_QCOM_PMIC_GLINK)	+= pmic_glink_altmode.o
 obj-$(CONFIG_QCOM_PMIC_PDCHARGER_ULOG)	+= pmic_pdcharger_ulog.o
 CFLAGS_pmic_pdcharger_ulog.o	:=  -I$(src)
+obj-$(CONFIG_QCOM_QMI_TMD) += qmi_tmd.o
 obj-$(CONFIG_QCOM_QMI_HELPERS)	+= qmi_helpers.o
 qmi_helpers-y	+= qmi_encdec.o qmi_interface.o
 obj-$(CONFIG_QCOM_RAMP_CTRL)	+= ramp_controller.o
diff --git a/drivers/soc/qcom/apr.c b/drivers/soc/qcom/apr.c
index ea7f83916d8d8d..2d5645ff991971 100644
--- a/drivers/soc/qcom/apr.c
+++ b/drivers/soc/qcom/apr.c
@@ -338,29 +338,6 @@ static void apr_rxwq(struct work_struct *work)
 	}
 }
 
-static int apr_device_match(struct device *dev, const struct device_driver *drv)
-{
-	struct apr_device *adev = to_apr_device(dev);
-	const struct apr_driver *adrv = to_apr_driver(drv);
-	const struct apr_device_id *id = adrv->id_table;
-
-	/* Attempt an OF style match first */
-	if (of_driver_match_device(dev, drv))
-		return 1;
-
-	if (!id)
-		return 0;
-
-	while (id->domain_id != 0 || id->svc_id != 0) {
-		if (id->domain_id == adev->domain_id &&
-		    id->svc_id == adev->svc.id)
-			return 1;
-		id++;
-	}
-
-	return 0;
-}
-
 static int apr_device_probe(struct device *dev)
 {
 	struct apr_device *adev = to_apr_device(dev);
@@ -401,7 +378,7 @@ static int apr_uevent(const struct device *dev, struct kobj_uevent_env *env)
 
 const struct bus_type aprbus = {
 	.name		= "aprbus",
-	.match		= apr_device_match,
+	.match		= of_driver_match_device,
 	.probe		= apr_device_probe,
 	.uevent		= apr_uevent,
 	.remove		= apr_device_remove,
@@ -434,31 +411,35 @@ static int apr_add_device(struct device *dev, struct device_node *np,
 	if (np)
 		snprintf(adev->name, APR_NAME_SIZE, "%pOFn", np);
 
+	adev->dev.bus = &aprbus;
+	adev->dev.parent = dev;
+	adev->dev.of_node = np;
+	adev->dev.release = apr_dev_release;
+	adev->dev.driver = NULL;
+	device_initialize(&adev->dev);
+
 	switch (apr->type) {
 	case PR_TYPE_APR:
-		dev_set_name(&adev->dev, "aprsvc:%s:%x:%x", adev->name,
-			     domain_id, svc_id);
+		ret = dev_set_name(&adev->dev, "aprsvc:%s:%x:%x", adev->name,
+				   domain_id, svc_id);
 		break;
 	case PR_TYPE_GPR:
-		dev_set_name(&adev->dev, "gprsvc:%s:%x:%x", adev->name,
-			     domain_id, svc_id);
+		ret = dev_set_name(&adev->dev, "gprsvc:%s:%x:%x", adev->name,
+				   domain_id, svc_id);
 		break;
 	default:
+		ret = -EINVAL;
 		break;
 	}
-
-	adev->dev.bus = &aprbus;
-	adev->dev.parent = dev;
-	adev->dev.of_node = np;
-	adev->dev.release = apr_dev_release;
-	adev->dev.driver = NULL;
+	if (ret)
+		goto out_put_device;
 
 	spin_lock(&apr->svcs_lock);
 	ret = idr_alloc(&apr->svcs_idr, svc, svc_id, svc_id + 1, GFP_ATOMIC);
 	spin_unlock(&apr->svcs_lock);
 	if (ret < 0) {
 		dev_err(dev, "idr_alloc failed: %d\n", ret);
-		goto out;
+		goto out_put_device;
 	}
 
 	/* Protection domain is optional, it does not exist on older platforms */
@@ -466,18 +447,26 @@ static int apr_add_device(struct device *dev, struct device_node *np,
 					    1, &adev->service_path);
 	if (ret < 0 && ret != -EINVAL) {
 		dev_err(dev, "Failed to read second value of qcom,protection-domain\n");
-		goto out;
+		goto out_remove_idr;
 	}
 
 	dev_info(dev, "Adding APR/GPR dev: %s\n", dev_name(&adev->dev));
 
-	ret = device_register(&adev->dev);
+	ret = device_add(&adev->dev);
 	if (ret) {
-		dev_err(dev, "device_register failed: %d\n", ret);
-		put_device(&adev->dev);
+		dev_err(dev, "device_add failed: %d\n", ret);
+		goto out_remove_idr;
 	}
 
-out:
+	return 0;
+
+out_remove_idr:
+	spin_lock(&apr->svcs_lock);
+	idr_remove(&apr->svcs_idr, svc_id);
+	spin_unlock(&apr->svcs_lock);
+	flush_workqueue(apr->rxwq);
+out_put_device:
+	put_device(&adev->dev);
 	return ret;
 }
 
diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
index 733999867bbf6e..d7a42a8040a163 100644
--- a/drivers/soc/qcom/llcc-qcom.c
+++ b/drivers/soc/qcom/llcc-qcom.c
@@ -626,11 +626,11 @@ static const struct llcc_slice_config glymur_data[] = {
 
 static const struct llcc_slice_config ipq5424_data[] =  {
 	{
-		.usecase_id = LLCC_CPUSS,
+		.usecase_id = LLCC_PPE_RXDESC,
 		.slice_id = 1,
-		.max_cap = 768,
-		.priority = 1,
-		.bonus_ways = 0xFFFF,
+		.max_cap = 128,
+		.priority = 3,
+		.bonus_ways = 0x0FFF,
 		.retain_on_pc = true,
 		.activate_on_init = true,
 		.write_scid_cacheable_en = true,
@@ -642,12 +642,52 @@ static const struct llcc_slice_config ipq5424_data[] =  {
 		.vict_prio = true,
 	},
 	{
-		.usecase_id = LLCC_VIDSC0,
+		.usecase_id = LLCC_CPUSS,
 		.slice_id = 2,
-		.max_cap = 256,
-		.priority = 2,
+		.max_cap = 768,
+		.priority = 1,
 		.fixed_size = true,
-		.bonus_ways = 0xF000,
+		.bonus_ways = 0x0FFF,
+		.retain_on_pc = true,
+		.activate_on_init = true,
+		.write_scid_cacheable_en = true,
+		.stale_en = true,
+		.stale_cap_en = true,
+	},
+	{
+		.usecase_id = LLCC_PPE_RXFILL,
+		.slice_id = 5,
+		.max_cap = 128,
+		.priority = 3,
+		.bonus_ways = 0x0FFF,
+		.retain_on_pc = true,
+		.activate_on_init = true,
+		.write_scid_cacheable_en = true,
+		.stale_en = true,
+		.stale_cap_en = true,
+		.alloc_oneway_en = true,
+		.ovcap_en = true,
+		.ovcap_prio = true,
+		.vict_prio = true,
+	},
+	{
+		.usecase_id = LLCC_WLAN_5G,
+		.slice_id = 6,
+		.max_cap = 128,
+		.priority = 3,
+		.bonus_ways = 0xC000,
+		.retain_on_pc = true,
+		.activate_on_init = true,
+		.write_scid_cacheable_en = true,
+		.stale_en = true,
+		.stale_cap_en = true,
+	},
+	{
+		.usecase_id = LLCC_WLAN_6G,
+		.slice_id = 7,
+		.max_cap = 128,
+		.priority = 3,
+		.bonus_ways = 0x3000,
 		.retain_on_pc = true,
 		.activate_on_init = true,
 		.write_scid_cacheable_en = true,
@@ -1014,6 +1054,128 @@ static const struct llcc_slice_config kaanapali_data[] = {
 	   },
 };
 
+static const struct llcc_slice_config nord_data[] = {
+	{
+		.usecase_id = LLCC_CPUSS,
+		.slice_id = 1,
+		.max_cap = 3072,
+		.priority = 0,
+		.bonus_ways = 0xff,
+		.activate_on_init = true,
+		.write_scid_en = true,
+		.write_scid_cacheable_en = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_CMPT,
+		.slice_id = 34,
+		.max_cap = 0,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_GPUHTW,
+		.slice_id = 11,
+		.max_cap = 512,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_GPU,
+		.slice_id = 12,
+		.max_cap = 2048,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.write_scid_en = true,
+		.write_scid_cacheable_en = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_MMUHWT,
+		.slice_id = 13,
+		.max_cap = 1024,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.activate_on_init = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_CAMFW,
+		.slice_id = 20,
+		.max_cap = 512,
+		.priority = 2,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_ECC,
+		.slice_id = 26,
+		.max_cap = 1536,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.activate_on_init = true,
+		.write_scid_en = true,
+		.write_scid_cacheable_en = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_WRCACHE,
+		.slice_id = 31,
+		.max_cap = 2048,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.activate_on_init = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_LCPDARE,
+		.slice_id = 30,
+		.max_cap = 512,
+		.priority = 3,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.activate_on_init = true,
+		.alloc_oneway_en = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_COMPUTE1,
+		.slice_id = 22,
+		.max_cap = 0,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_PCIE_TCU,
+		.slice_id = 15,
+		.max_cap = 0,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.activate_on_init = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_GPU_LITTLE,
+		.slice_id = 9,
+		.max_cap = 2048,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.write_scid_en = true,
+		.write_scid_cacheable_en = true,
+		.vict_prio = true,
+	}, {
+		.usecase_id = LLCC_GPUHTW_LITTLE,
+		.slice_id = 14,
+		.max_cap = 512,
+		.priority = 1,
+		.fixed_size = true,
+		.bonus_ways = 0xff,
+		.vict_prio = true,
+	},
+};
+
 static const struct llcc_slice_config sa8775p_data[] =  {
 	{
 		.usecase_id = LLCC_CPUSS,
@@ -4444,6 +4606,17 @@ static const struct qcom_llcc_config glymur_cfg[] = {
 	},
 };
 
+static const struct qcom_llcc_config nord_cfg[] = {
+	{
+		.sct_data	= nord_data,
+		.size		= ARRAY_SIZE(nord_data),
+		.reg_offset	= llcc_v6_reg_offset,
+		.edac_reg_offset = &llcc_v6_edac_reg_offset,
+		.num_banks	= 16,
+		.no_edac	= true,
+	},
+};
+
 static const struct qcom_llcc_config qcs615_cfg[] = {
 	{
 		.sct_data	= qcs615_data,
@@ -4711,6 +4884,11 @@ static const struct qcom_sct_config glymur_cfgs = {
 	.num_config	= ARRAY_SIZE(glymur_cfg),
 };
 
+static const struct qcom_sct_config nord_cfgs = {
+	.llcc_config	= nord_cfg,
+	.num_config	= ARRAY_SIZE(nord_cfg),
+};
+
 static const struct qcom_sct_config qcs615_cfgs = {
 	.llcc_config	= qcs615_cfg,
 	.num_config	= ARRAY_SIZE(qcs615_cfg),
@@ -5617,6 +5795,10 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 	}
 
 	drv_data->ecc_irq = platform_get_irq_optional(pdev, 0);
+	if (drv_data->ecc_irq < 0 && drv_data->ecc_irq != -ENXIO)
+		return dev_err_probe(&pdev->dev, drv_data->ecc_irq,
+				     "failed to get IRQ resource\n");
+
 	drv_data->edac_reg_offset = cfg->edac_reg_offset;
 	drv_data->ecc_irq_configured = cfg->irq_configured;
 	drv_data->dev = dev;
@@ -5649,6 +5831,7 @@ static const struct of_device_id qcom_llcc_of_match[] = {
 	{ .compatible = "qcom,hawi-llcc", .data = &hawi_sct_cfgs },
 	{ .compatible = "qcom,ipq5424-llcc", .data = &ipq5424_cfgs},
 	{ .compatible = "qcom,kaanapali-llcc", .data = &kaanapali_cfgs},
+	{ .compatible = "qcom,nord-llcc", .data = &nord_cfgs},
 	{ .compatible = "qcom,qcs615-llcc", .data = &qcs615_cfgs},
 	{ .compatible = "qcom,qcs8300-llcc", .data = &qcs8300_cfgs},
 	{ .compatible = "qcom,qdu1000-llcc", .data = &qdu1000_cfgs},
diff --git a/drivers/soc/qcom/pmic_glink.c b/drivers/soc/qcom/pmic_glink.c
index 3042261578aad8..84fd0c82c0c4db 100644
--- a/drivers/soc/qcom/pmic_glink.c
+++ b/drivers/soc/qcom/pmic_glink.c
@@ -250,7 +250,8 @@ static int pmic_glink_rpmsg_probe(struct rpmsg_device *rpdev)
 	guard(mutex)(&__pmic_glink_lock);
 	pg = __pmic_glink;
 	if (!pg)
-		return dev_err_probe(&rpdev->dev, -ENODEV, "no pmic_glink device to attach to\n");
+		return dev_err_probe(&rpdev->dev, -EPROBE_DEFER,
+				     "no pmic_glink device to attach to\n");
 
 	dev_set_drvdata(&rpdev->dev, pg);
 	pg->pdr_available = rpdev->id.driver_data;
diff --git a/drivers/soc/qcom/pmic_glink_altmode.c b/drivers/soc/qcom/pmic_glink_altmode.c
index 13c434f8d03a64..465e5d2316a048 100644
--- a/drivers/soc/qcom/pmic_glink_altmode.c
+++ b/drivers/soc/qcom/pmic_glink_altmode.c
@@ -451,7 +451,7 @@ static void pmic_glink_altmode_sc8180xp_notify(struct pmic_glink_altmode *altmod
 	alt_port->mode = mode;
 	alt_port->hpd_state = hpd_state;
 	alt_port->hpd_irq = hpd_irq;
-	schedule_work(&alt_port->work);
+	queue_work(system_freezable_wq, &alt_port->work);
 }
 
 #define SC8280XP_DPAM_MASK	0x3f
@@ -502,7 +502,7 @@ static void pmic_glink_altmode_sc8280xp_notify(struct pmic_glink_altmode *altmod
 		alt_port->tbt_data = *tbt;
 	}
 
-	schedule_work(&alt_port->work);
+	queue_work(system_freezable_wq, &alt_port->work);
 }
 
 static void pmic_glink_altmode_callback(const void *data, size_t len, void *priv)
diff --git a/drivers/soc/qcom/pmic_pdcharger_ulog.c b/drivers/soc/qcom/pmic_pdcharger_ulog.c
index 39f412bbf2c1e6..06cd9c4411c2fb 100644
--- a/drivers/soc/qcom/pmic_pdcharger_ulog.c
+++ b/drivers/soc/qcom/pmic_pdcharger_ulog.c
@@ -143,7 +143,7 @@ static void pmic_pdcharger_ulog_rpmsg_remove(struct rpmsg_device *rpdev)
 {
 	struct pmic_pdcharger_ulog *pg = dev_get_drvdata(&rpdev->dev);
 
-	cancel_delayed_work_sync(&pg->ulog_work);
+	disable_delayed_work_sync(&pg->ulog_work);
 }
 
 static const struct rpmsg_device_id pmic_pdcharger_ulog_rpmsg_id_match[] = {
diff --git a/drivers/soc/qcom/qcom-geni-se.c b/drivers/soc/qcom/qcom-geni-se.c
index 873bfbd6b2b7b8..7ad002b111c83e 100644
--- a/drivers/soc/qcom/qcom-geni-se.c
+++ b/drivers/soc/qcom/qcom-geni-se.c
@@ -42,7 +42,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(geni_se_regs);
  *
  * GENI based QUP is a highly-flexible and programmable module for supporting
  * a wide range of serial interfaces like UART, SPI, I2C, I3C, etc. A single
- * QUP module can provide upto 8 serial interfaces, using its internal
+ * QUP module can provide up to 8 serial interfaces, using its internal
  * serial engines. The actual configuration is determined by the target
  * platform configuration. The protocol supported by each interface is
  * determined by the firmware loaded to the serial engine. Each SE consists
@@ -101,22 +101,26 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(geni_se_regs);
  * @base:		Base address of this instance of QUP wrapper core
  * @clks:		Handle to the primary & optional secondary AHB clocks
  * @num_clks:		Count of clocks
+ * @prog_ram_depth:	Available CFG RAM depth for instance without SE_HW_PARAM_2
  */
 struct geni_wrapper {
 	struct device *dev;
 	void __iomem *base;
 	struct clk_bulk_data clks[MAX_CLKS];
 	unsigned int num_clks;
+	unsigned int prog_ram_depth;
 };
 
 /**
  * struct geni_se_desc - Data structure to represent the QUP Wrapper resources
  * @clks:		Name of the primary & optional secondary AHB clocks
  * @num_clks:		Count of clock names
+ * @prog_ram_depth:	Available CFG RAM depth for instance without SE_HW_PARAM_2
  */
 struct geni_se_desc {
 	unsigned int num_clks;
 	const char * const *clks;
+	unsigned int prog_ram_depth;
 };
 
 static const char * const icc_path_names[] = {"qup-core", "qup-config",
@@ -1230,25 +1234,29 @@ EXPORT_SYMBOL_GPL(geni_se_resources_init);
  * @se: Pointer to the serial engine structure.
  * @fw: Pointer to the firmware image.
  * @protocol: Expected serial engine protocol type.
+ * @fw_size_out: Non-NULL output parameter; receives the rounded, validated
+ *               firmware word count on success.
  *
  * Identifies the appropriate firmware image or configuration required for a
  * specific communication protocol instance running on a Qualcomm GENI
  * controller. Validates the firmware size against the hardware PROG_RAM_DEPTH
  * read from SE_HW_PARAM_2.
  *
- * Return: pointer to a valid 'struct se_fw_hdr' if found, or NULL otherwise.
+ * Return: pointer to a valid 'const struct se_fw_hdr' if found, or NULL otherwise.
  */
-static struct se_fw_hdr *geni_find_protocol_fw(struct geni_se *se, const struct firmware *fw,
-					       enum geni_se_protocol_type protocol)
+static const struct se_fw_hdr *geni_find_protocol_fw(struct geni_se *se, const struct firmware *fw,
+						     enum geni_se_protocol_type protocol,
+						     u32 *fw_size_out)
 {
+	struct geni_wrapper *wrapper = se->wrapper;
 	struct device *dev = se->dev;
 	const struct elf32_hdr *ehdr;
 	const struct elf32_phdr *phdrs;
 	const struct elf32_phdr	*phdr;
-	struct se_fw_hdr *sefw;
+	const struct se_fw_hdr *sefw;
 	u32 fw_end, cfg_idx_end, cfg_val_end;
 	u32 prog_ram_depth;
-	u16 fw_size;
+	u32 fw_size;
 	int i;
 
 	if (!fw || fw->size < sizeof(struct elf32_hdr))
@@ -1287,27 +1295,29 @@ static struct se_fw_hdr *geni_find_protocol_fw(struct geni_se *se, const struct
 		if (phdr->p_filesz < sizeof(struct se_fw_hdr))
 			continue;
 
-		sefw = (struct se_fw_hdr *)(fw->data + phdr->p_offset);
+		sefw = (const struct se_fw_hdr *)(fw->data + phdr->p_offset);
 		fw_size = le16_to_cpu(sefw->fw_size_in_items);
-		fw_end = le16_to_cpu(sefw->fw_offset) + fw_size * sizeof(u32);
-		cfg_idx_end = le16_to_cpu(sefw->cfg_idx_offset) +
-			      le16_to_cpu(sefw->cfg_size_in_items) * sizeof(u8);
-		cfg_val_end = le16_to_cpu(sefw->cfg_val_offset) +
-			      le16_to_cpu(sefw->cfg_size_in_items) * sizeof(u32);
 
 		if (le32_to_cpu(sefw->magic) != SE_MAGIC_NUM || le32_to_cpu(sefw->version) != 1)
 			continue;
 
-		if (le32_to_cpu(sefw->serial_protocol) != protocol)
+		if (le16_to_cpu(sefw->serial_protocol) != protocol)
 			continue;
 
-		if (fw_size % 2 != 0) {
+		if (fw_size % 2 != 0)
 			fw_size++;
-			sefw->fw_size_in_items = cpu_to_le16(fw_size);
-		}
 
-		prog_ram_depth = FIELD_GET(PROG_RAM_DEPTH_MSK,
-					   readl_relaxed(se->base + SE_HW_PARAM_2));
+		fw_end = le16_to_cpu(sefw->fw_offset) + fw_size * sizeof(u32);
+		cfg_idx_end = le16_to_cpu(sefw->cfg_idx_offset) +
+			      le16_to_cpu(sefw->cfg_size_in_items) * sizeof(u8);
+		cfg_val_end = le16_to_cpu(sefw->cfg_val_offset) +
+			      le16_to_cpu(sefw->cfg_size_in_items) * sizeof(u32);
+
+		if (wrapper->prog_ram_depth)
+			prog_ram_depth = wrapper->prog_ram_depth;
+		else
+			prog_ram_depth = FIELD_GET(PROG_RAM_DEPTH_MSK,
+					readl_relaxed(se->base + SE_HW_PARAM_2));
 		if (fw_size >= prog_ram_depth) {
 			dev_err(dev, "Firmware size (%u) exceeds RAM size (%u)\n",
 				fw_size, prog_ram_depth);
@@ -1320,6 +1330,7 @@ static struct se_fw_hdr *geni_find_protocol_fw(struct geni_se *se, const struct
 			continue;
 		}
 
+		*fw_size_out = fw_size;
 		return sefw;
 	}
 
@@ -1430,17 +1441,17 @@ static int geni_load_se_fw(struct geni_se *se, const struct firmware *fw,
 {
 	const u32 *fw_data, *cfg_val_arr;
 	const u8 *cfg_idx_arr;
-	u32 i, reg_value;
+	u32 i, reg_value, fw_size_in_items;
 	int ret;
-	struct se_fw_hdr *hdr;
+	const struct se_fw_hdr *hdr;
 
-	hdr = geni_find_protocol_fw(se, fw, protocol);
+	hdr = geni_find_protocol_fw(se, fw, protocol, &fw_size_in_items);
 	if (!hdr)
 		return -EINVAL;
 
-	fw_data = (const u32 *)((u8 *)hdr + le16_to_cpu(hdr->fw_offset));
+	fw_data = (const u32 *)((const u8 *)hdr + le16_to_cpu(hdr->fw_offset));
 	cfg_idx_arr = (const u8 *)hdr + le16_to_cpu(hdr->cfg_idx_offset);
-	cfg_val_arr = (const u32 *)((u8 *)hdr + le16_to_cpu(hdr->cfg_val_offset));
+	cfg_val_arr = (const u32 *)((const u8 *)hdr + le16_to_cpu(hdr->cfg_val_offset));
 
 	ret = geni_icc_set_bw(se);
 	if (ret)
@@ -1511,7 +1522,7 @@ static int geni_load_se_fw(struct geni_se *se, const struct firmware *fw,
 
 	/* Program RAM address space. */
 	memcpy_toio(se->base + SE_GENI_CFG_RAMN, fw_data,
-		    le16_to_cpu(hdr->fw_size_in_items) * sizeof(u32));
+		    fw_size_in_items * sizeof(u32));
 
 	/* Put default values on GENI's output pads. */
 	writel_relaxed(0x1, se->base + GENI_FORCE_DEFAULT_REG);
@@ -1617,6 +1628,7 @@ static int geni_se_probe(struct platform_device *pdev)
 		return PTR_ERR(wrapper->base);
 
 	desc = device_get_match_data(&pdev->dev);
+	wrapper->prog_ram_depth = desc->prog_ram_depth;
 
 	if (!has_acpi_companion(&pdev->dev) && desc->num_clks) {
 		int i;
@@ -1669,6 +1681,7 @@ static const char * const i2c_master_hub_clks[] = {
 static const struct geni_se_desc i2c_master_hub_desc = {
 	.clks = i2c_master_hub_clks,
 	.num_clks = ARRAY_SIZE(i2c_master_hub_clks),
+	.prog_ram_depth = 1020,
 };
 
 static const struct of_device_id geni_se_dt_match[] = {
diff --git a/drivers/soc/qcom/qcom_aoss.c b/drivers/soc/qcom/qcom_aoss.c
index 259c41f0c34ef5..a6b673cd76786b 100644
--- a/drivers/soc/qcom/qcom_aoss.c
+++ b/drivers/soc/qcom/qcom_aoss.c
@@ -602,10 +602,8 @@ static int qmp_probe(struct platform_device *pdev)
 	irq = platform_get_irq(pdev, 0);
 	ret = devm_request_irq(&pdev->dev, irq, qmp_intr, 0,
 			       "aoss-qmp", qmp);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to request interrupt\n");
+	if (ret < 0)
 		goto err_free_mbox;
-	}
 
 	ret = qmp_open(qmp);
 	if (ret < 0)
diff --git a/drivers/soc/qcom/qcom_stats.c b/drivers/soc/qcom/qcom_stats.c
index 06923408bfce57..3422d6df7b51f2 100644
--- a/drivers/soc/qcom/qcom_stats.c
+++ b/drivers/soc/qcom/qcom_stats.c
@@ -371,6 +371,14 @@ static const struct stats_config rpm_data_dba0 = {
 	.subsystem_stats_in_smem = false,
 };
 
+static const struct stats_config rpm_data_shikra = {
+	.stats_offset = 0,
+	.num_records = 2,
+	.appended_stats_avail = true,
+	.dynamic_offset = true,
+	.subsystem_stats_in_smem = true,
+};
+
 static const struct stats_config rpmh_data_sdm845 = {
 	.stats_offset = 0x48,
 	.num_records = 2,
@@ -396,6 +404,7 @@ static const struct of_device_id qcom_stats_table[] = {
 	{ .compatible = "qcom,rpm-stats", .data = &rpm_data },
 	{ .compatible = "qcom,rpmh-stats", .data = &rpmh_data },
 	{ .compatible = "qcom,sdm845-rpmh-stats", .data = &rpmh_data_sdm845 },
+	{ .compatible = "qcom,shikra-rpm-stats", .data = &rpm_data_shikra },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, qcom_stats_table);
diff --git a/drivers/soc/qcom/qmi_tmd.c b/drivers/soc/qcom/qmi_tmd.c
new file mode 100644
index 00000000000000..54e5ce8dc462ac
--- /dev/null
+++ b/drivers/soc/qcom/qmi_tmd.c
@@ -0,0 +1,595 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2025, Linaro Limited
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ * QMI Thermal Mitigation Device (TMD).
+ * Provides cooling device support for remote subsystems
+ * running the TMD service via QMI.
+ */
+#include <linux/cleanup.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/net.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/soc/qcom/qmi.h>
+#include <linux/soc/qcom/qmi_tmd.h>
+#include <linux/thermal.h>
+
+#define QMI_TMD_SERVICE_VERS_V01 0x01
+
+#define QMI_TMD_SET_LEVEL_REQ 0x0021
+#define QMI_TMD_GET_DEV_LIST_REQ 0x0020
+
+#define QMI_TMD_DEV_ID_LEN_MAX 32
+#define QMI_TMD_DEV_LIST_MAX 32
+#define QMI_TMD_RESP_TIMEOUT	msecs_to_jiffies(100)
+#define TMD_GET_LEVEL_REQ_MAX_LEN 36
+#define TMD_SET_LEVEL_REQ_MAX_LEN 40
+
+#define TMD_GET_DEV_LIST_REQ_MAX_LEN 0
+#define TMD_GET_DEV_LIST_RESP_MAX_LEN 1099
+
+struct tmd_dev_id {
+	char mitigation_dev_id[QMI_TMD_DEV_ID_LEN_MAX + 1];
+};
+
+static const struct qmi_elem_info tmd_dev_id_ei[] = {
+	{
+		.data_type = QMI_STRING,
+		.elem_len = QMI_TMD_DEV_ID_LEN_MAX + 1,
+		.elem_size = sizeof(char),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0,
+		.offset = offsetof(struct tmd_dev_id,
+				   mitigation_dev_id),
+	},
+	{
+		.data_type = QMI_EOTI,
+		.array_type = NO_ARRAY,
+		.tlv_type = QMI_COMMON_TLV_TYPE,
+	},
+};
+
+struct tmd_dev_list {
+	struct tmd_dev_id mitigation_dev_id;
+	u8 max_mitigation_level;
+};
+
+static const struct qmi_elem_info tmd_dev_list_ei[] = {
+	{
+		.data_type = QMI_STRUCT,
+		.elem_len = 1,
+		.elem_size = sizeof(struct tmd_dev_id),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0,
+		.offset = offsetof(struct tmd_dev_list,
+				   mitigation_dev_id),
+		.ei_array = tmd_dev_id_ei,
+	},
+	{
+		.data_type = QMI_UNSIGNED_1_BYTE,
+		.elem_len = 1,
+		.elem_size = sizeof(uint8_t),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0,
+		.offset = offsetof(struct tmd_dev_list,
+				   max_mitigation_level),
+	},
+	{
+		.data_type = QMI_EOTI,
+		.array_type = NO_ARRAY,
+		.tlv_type = QMI_COMMON_TLV_TYPE,
+	},
+};
+
+struct tmd_get_dev_list_req {
+	char placeholder;
+};
+
+static const struct qmi_elem_info tmd_get_dev_list_req_ei[] = {
+	{
+		.data_type = QMI_EOTI,
+		.array_type = NO_ARRAY,
+		.tlv_type = QMI_COMMON_TLV_TYPE,
+	},
+};
+
+struct tmd_get_dev_list_resp {
+	struct qmi_response_type_v01 resp;
+	u8 mitigation_device_list_valid;
+	u32 mitigation_device_list_len;
+	struct tmd_dev_list
+		mitigation_device_list[QMI_TMD_DEV_LIST_MAX];
+};
+
+static const struct qmi_elem_info tmd_get_dev_list_resp_ei[] = {
+	{
+		.data_type = QMI_STRUCT,
+		.elem_len = 1,
+		.elem_size = sizeof(struct qmi_response_type_v01),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0x02,
+		.offset = offsetof(struct tmd_get_dev_list_resp,
+				   resp),
+		.ei_array = qmi_response_type_v01_ei,
+	},
+	{
+		.data_type = QMI_OPT_FLAG,
+		.elem_len = 1,
+		.elem_size = sizeof(uint8_t),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0x10,
+		.offset = offsetof(struct tmd_get_dev_list_resp,
+				   mitigation_device_list_valid),
+	},
+	{
+		.data_type = QMI_DATA_LEN,
+		.elem_len = 1,
+		.elem_size = sizeof(uint8_t),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0x10,
+		.offset = offsetof(struct tmd_get_dev_list_resp,
+				   mitigation_device_list_len),
+	},
+	{
+		.data_type = QMI_STRUCT,
+		.elem_len = QMI_TMD_DEV_LIST_MAX,
+		.elem_size = sizeof(struct tmd_dev_list),
+		.array_type = VAR_LEN_ARRAY,
+		.tlv_type = 0x10,
+		.offset = offsetof(struct tmd_get_dev_list_resp,
+				   mitigation_device_list),
+		.ei_array = tmd_dev_list_ei,
+	},
+	{
+		.data_type = QMI_EOTI,
+		.array_type = NO_ARRAY,
+		.tlv_type = QMI_COMMON_TLV_TYPE,
+	},
+};
+
+struct tmd_set_level_req {
+	struct tmd_dev_id mitigation_dev_id;
+	u8 mitigation_level;
+};
+
+static const struct qmi_elem_info tmd_set_level_req_ei[] = {
+	{
+		.data_type = QMI_STRUCT,
+		.elem_len = 1,
+		.elem_size = sizeof(struct tmd_dev_id),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0x01,
+		.offset = offsetof(struct tmd_set_level_req,
+				   mitigation_dev_id),
+		.ei_array = tmd_dev_id_ei,
+	},
+	{
+		.data_type = QMI_UNSIGNED_1_BYTE,
+		.elem_len = 1,
+		.elem_size = sizeof(uint8_t),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0x02,
+		.offset = offsetof(struct tmd_set_level_req,
+				   mitigation_level),
+	},
+	{
+		.data_type = QMI_EOTI,
+		.array_type = NO_ARRAY,
+		.tlv_type = QMI_COMMON_TLV_TYPE,
+	},
+};
+
+struct tmd_set_level_resp {
+	struct qmi_response_type_v01 resp;
+};
+
+static const struct qmi_elem_info tmd_set_level_resp_ei[] = {
+	{
+		.data_type = QMI_STRUCT,
+		.elem_len = 1,
+		.elem_size = sizeof(struct qmi_response_type_v01),
+		.array_type = NO_ARRAY,
+		.tlv_type = 0x02,
+		.offset = offsetof(struct tmd_set_level_resp, resp),
+		.ei_array = qmi_response_type_v01_ei,
+	},
+	{
+		.data_type = QMI_EOTI,
+		.array_type = NO_ARRAY,
+		.tlv_type = QMI_COMMON_TLV_TYPE,
+	},
+};
+
+/**
+ * struct qmi_tmd - A TMD cooling device
+ * @name:	The name of this tmd shared by the remote subsystem
+ * @cdev:	Thermal cooling device handle
+ * @cur_state:	The current mitigation state
+ * @max_state:	The maximum state
+ * @qmi_tmd_cli:	Parent QMI TMD client
+ */
+struct qmi_tmd {
+	const char *name;
+	struct thermal_cooling_device *cdev;
+	unsigned int cur_state;
+	unsigned int max_state;
+	struct qmi_tmd_client *qmi_tmd_cli;
+};
+
+/**
+ * struct qmi_tmd_client - QMI TMD client state
+ * @dev:		Device associated with this instance
+ * @handle:		QMI connection handle
+ * @mutex:		Serializes QMI request/response sequences (qmi_txn_init,
+ *			qmi_send_request) during DSP subsystem restart and
+ *			protects @connection_active and @exiting
+ * @connection_active:	Whether or not we're connected to the QMI TMD service
+ * @exiting:		Whether or not teardown has started
+ * @svc_arrive_work:	Work item for initialising when the TMD service starts
+ * @num_tmds:		Number of tmds described in the device tree
+ * @tmds:		An array of tmd structures
+ */
+struct qmi_tmd_client {
+	struct device *dev;
+	struct qmi_handle handle;
+	/* protects QMI transactions, connection_active and exiting */
+	struct mutex mutex;
+	bool connection_active;
+	bool exiting;
+	struct work_struct svc_arrive_work;
+	int num_tmds;
+	struct qmi_tmd tmds[] __counted_by(num_tmds);
+};
+
+/* Notify the remote subsystem of the requested cooling state */
+static int qmi_tmd_send_state_request(struct qmi_tmd *tmd, int state)
+{
+	struct tmd_set_level_resp resp = { 0 };
+	struct tmd_set_level_req req = { 0 };
+	struct qmi_tmd_client *qmi_tmd_cli = tmd->qmi_tmd_cli;
+	struct qmi_txn txn;
+	int ret = 0;
+
+	if (!qmi_tmd_cli->connection_active)
+		return -ENOTCONN;
+
+	strscpy(req.mitigation_dev_id.mitigation_dev_id, tmd->name,
+		QMI_TMD_DEV_ID_LEN_MAX + 1);
+	req.mitigation_level = state;
+
+	ret = qmi_txn_init(&qmi_tmd_cli->handle, &txn,
+			   tmd_set_level_resp_ei, &resp);
+	if (ret < 0) {
+		dev_err(qmi_tmd_cli->dev, "qmi set state %d txn init failed for %s ret %d\n",
+			state, tmd->name, ret);
+		return ret;
+	}
+
+	ret = qmi_send_request(&qmi_tmd_cli->handle, NULL, &txn,
+			       QMI_TMD_SET_LEVEL_REQ,
+			       TMD_SET_LEVEL_REQ_MAX_LEN,
+			       tmd_set_level_req_ei, &req);
+	if (ret < 0) {
+		dev_err(qmi_tmd_cli->dev, "qmi set state %d txn send failed for %s ret %d\n",
+			state, tmd->name, ret);
+		qmi_txn_cancel(&txn);
+		return ret;
+	}
+
+	ret = qmi_txn_wait(&txn, QMI_TMD_RESP_TIMEOUT);
+	if (ret < 0) {
+		dev_err(qmi_tmd_cli->dev, "qmi set state %d txn wait failed for %s ret %d\n",
+			state, tmd->name, ret);
+		return ret;
+	}
+
+	if (resp.resp.result != QMI_RESULT_SUCCESS_V01) {
+		dev_err(qmi_tmd_cli->dev,
+			"qmi set state %d failed for %s result %#x error %#x\n",
+			state, tmd->name,
+			resp.resp.result, resp.resp.error);
+		return -EREMOTEIO;
+	}
+
+	dev_dbg(qmi_tmd_cli->dev, "Requested state %d/%d for %s\n", state,
+		tmd->max_state, tmd->name);
+
+	tmd->cur_state = state;
+
+	return 0;
+}
+
+static int qmi_tmd_get_max_state(struct thermal_cooling_device *cdev,
+				 unsigned long *state)
+{
+	struct qmi_tmd *tmd = cdev->devdata;
+
+	*state = tmd->max_state;
+
+	return 0;
+}
+
+static int qmi_tmd_get_cur_state(struct thermal_cooling_device *cdev,
+				 unsigned long *state)
+{
+	struct qmi_tmd *tmd = cdev->devdata;
+
+	/* cur_state is protected by thermal core's cdev->lock */
+	*state = tmd->cur_state;
+
+	return 0;
+}
+
+static int qmi_tmd_set_cur_state(struct thermal_cooling_device *cdev,
+				 unsigned long state)
+{
+	struct qmi_tmd *tmd = cdev->devdata;
+
+	if (state > tmd->max_state)
+		return -EINVAL;
+
+	/* cur_state is protected by thermal core's cdev->lock */
+	if (tmd->cur_state == state)
+		return 0;
+
+	guard(mutex)(&tmd->qmi_tmd_cli->mutex);
+
+	return qmi_tmd_send_state_request(tmd, state);
+}
+
+static const struct thermal_cooling_device_ops qmi_tmd_cooling_ops = {
+	.get_max_state = qmi_tmd_get_max_state,
+	.get_cur_state = qmi_tmd_get_cur_state,
+	.set_cur_state = qmi_tmd_set_cur_state,
+};
+
+static int qmi_tmd_register(struct qmi_tmd_client *qmi_tmd_cli,
+			    const char *label, u8 max_state)
+{
+	struct device *dev = qmi_tmd_cli->dev;
+	struct qmi_tmd *tmd;
+	int index;
+
+	for (index = 0; index < qmi_tmd_cli->num_tmds; index++) {
+		tmd = &qmi_tmd_cli->tmds[index];
+
+		if (!strncasecmp(tmd->name, label,
+				 QMI_TMD_DEV_ID_LEN_MAX + 1))
+			goto found;
+	}
+
+	dev_dbg(qmi_tmd_cli->dev,
+		"TMD '%s' available in firmware but not specified in DT\n",
+		label);
+	return 0;
+
+found:
+	tmd->max_state = max_state;
+
+	/*
+	 * If the cooling device already exists then the QMI service went away and
+	 * came back. So just make sure the current cooling device state is
+	 * reflected on the remote side and then return.
+	 */
+	if (tmd->cdev) {
+		guard(mutex)(&qmi_tmd_cli->mutex);
+
+		return qmi_tmd_send_state_request(tmd, tmd->cur_state);
+	}
+
+	tmd->cdev = thermal_of_cooling_device_register(dev->of_node, index,
+						       label, tmd, &qmi_tmd_cooling_ops);
+	if (IS_ERR(tmd->cdev)) {
+		int ret = PTR_ERR(tmd->cdev);
+
+		tmd->cdev = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
+static void qmi_tmd_unregister(struct qmi_tmd_client *qmi_tmd_cli)
+{
+	struct qmi_tmd *tmd;
+	int index;
+
+	for (index = 0; index < qmi_tmd_cli->num_tmds; index++) {
+		tmd = &qmi_tmd_cli->tmds[index];
+
+		if (!tmd->cdev)
+			continue;
+
+		thermal_cooling_device_unregister(tmd->cdev);
+		tmd->cdev = NULL;
+	}
+}
+
+static void qmi_tmd_svc_arrive(struct work_struct *work)
+{
+	struct qmi_tmd_client *qmi_tmd_cli =
+		container_of(work, struct qmi_tmd_client, svc_arrive_work);
+
+	struct tmd_get_dev_list_req req = { 0 };
+	struct tmd_get_dev_list_resp *resp __free(kfree) = NULL;
+	int ret, i;
+	struct qmi_txn txn;
+
+	resp = kzalloc_obj(*resp, GFP_KERNEL);
+	if (!resp) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	scoped_guard(mutex, &qmi_tmd_cli->mutex) {
+		if (qmi_tmd_cli->exiting)
+			return;
+
+		ret = qmi_txn_init(&qmi_tmd_cli->handle, &txn,
+				   tmd_get_dev_list_resp_ei, resp);
+		if (ret < 0)
+			goto out;
+
+		ret = qmi_send_request(&qmi_tmd_cli->handle, NULL, &txn,
+				       QMI_TMD_GET_DEV_LIST_REQ,
+				       TMD_GET_DEV_LIST_REQ_MAX_LEN,
+				       tmd_get_dev_list_req_ei, &req);
+		if (ret < 0) {
+			qmi_txn_cancel(&txn);
+			goto out;
+		}
+
+		ret = qmi_txn_wait(&txn, QMI_TMD_RESP_TIMEOUT);
+		if (ret < 0)
+			goto out;
+
+		if (resp->resp.result != QMI_RESULT_SUCCESS_V01) {
+			ret = -EPROTO;
+			goto out;
+		}
+
+		qmi_tmd_cli->connection_active = true;
+	}
+
+	for (i = 0; i < resp->mitigation_device_list_len; i++) {
+		struct tmd_dev_list *device =
+			&resp->mitigation_device_list[i];
+
+		ret = qmi_tmd_register(qmi_tmd_cli,
+				       device->mitigation_dev_id.mitigation_dev_id,
+				       device->max_mitigation_level);
+		if (ret)
+			break;
+	}
+
+out:
+	if (ret)
+		dev_err(qmi_tmd_cli->dev, "Failed to initialize TMD service: %d\n", ret);
+}
+
+static void qmi_tmd_del_server(struct qmi_handle *qmi, struct qmi_service *service)
+{
+	struct qmi_tmd_client *qmi_tmd_cli =
+		container_of(qmi, struct qmi_tmd_client, handle);
+
+	scoped_guard(mutex, &qmi_tmd_cli->mutex) {
+		qmi_tmd_cli->connection_active = false;
+	}
+}
+
+static int qmi_tmd_new_server(struct qmi_handle *qmi, struct qmi_service *service)
+{
+	struct sockaddr_qrtr sq = { AF_QIPCRTR, service->node, service->port };
+	struct qmi_tmd_client *qmi_tmd_cli;
+	int ret;
+
+	qmi_tmd_cli = container_of(qmi, struct qmi_tmd_client, handle);
+
+	scoped_guard(mutex, &qmi_tmd_cli->mutex) {
+		if (qmi_tmd_cli->exiting)
+			return 0;
+
+		ret = kernel_connect(qmi->sock, (struct sockaddr_unsized *)&sq,
+				     sizeof(sq), 0);
+	}
+
+	if (ret < 0) {
+		dev_err(qmi_tmd_cli->dev, "QMI connect failed for node %u port %u: %d\n",
+			service->node, service->port, ret);
+		return ret;
+	}
+
+	queue_work(system_highpri_wq, &qmi_tmd_cli->svc_arrive_work);
+
+	return 0;
+}
+
+static const struct qmi_ops qmi_tmd_ops = {
+	.new_server = qmi_tmd_new_server,
+	.del_server = qmi_tmd_del_server,
+};
+
+/**
+ * qmi_tmd_init() - Initialize QMI TMD instance
+ * @dev: Device pointer
+ * @instance_id: QMI service instance ID for the remote subsystem
+ * @tmd_names: Array of TMD names
+ * @num_tmds: Number of TMD names
+ *
+ * Context: Must be called from probe context.
+ *
+ * Return: Pointer to qmi_tmd_client on success, ERR_PTR on failure
+ */
+struct qmi_tmd_client *qmi_tmd_init(struct device *dev,
+				    unsigned int instance_id,
+				    const char * const *tmd_names,
+				    int num_tmds)
+{
+	struct qmi_tmd_client *qmi_tmd_cli;
+	int ret, i;
+
+	if (!dev || !tmd_names || num_tmds <= 0)
+		return ERR_PTR(-EINVAL);
+
+	qmi_tmd_cli = devm_kzalloc(dev, struct_size(qmi_tmd_cli, tmds, num_tmds), GFP_KERNEL);
+	if (!qmi_tmd_cli)
+		return ERR_PTR(-ENOMEM);
+
+	qmi_tmd_cli->dev = dev;
+	qmi_tmd_cli->num_tmds = num_tmds;
+	mutex_init(&qmi_tmd_cli->mutex);
+	INIT_WORK(&qmi_tmd_cli->svc_arrive_work, qmi_tmd_svc_arrive);
+
+	for (i = 0; i < num_tmds; i++) {
+		qmi_tmd_cli->tmds[i].name = tmd_names[i];
+		qmi_tmd_cli->tmds[i].qmi_tmd_cli = qmi_tmd_cli;
+	}
+
+	ret = qmi_handle_init(&qmi_tmd_cli->handle,
+			      TMD_GET_DEV_LIST_RESP_MAX_LEN,
+			      &qmi_tmd_ops, NULL);
+	if (ret < 0)
+		return ERR_PTR(dev_err_probe(dev, ret, "QMI handle init failed\n"));
+
+	ret = qmi_add_lookup(&qmi_tmd_cli->handle, QMI_SERVICE_ID_TMD,
+			     QMI_TMD_SERVICE_VERS_V01, instance_id);
+	if (ret < 0) {
+		dev_err_probe(dev, ret, "QMI add lookup failed\n");
+		goto err_release_handle;
+	}
+
+	return qmi_tmd_cli;
+
+err_release_handle:
+	qmi_handle_release(&qmi_tmd_cli->handle);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(qmi_tmd_init);
+
+/**
+ * qmi_tmd_exit() - Deinitialize QMI TMD instance
+ * @qmi_tmd_cli: QMI TMD client to deinitialize
+ */
+void qmi_tmd_exit(struct qmi_tmd_client *qmi_tmd_cli)
+{
+	if (!qmi_tmd_cli)
+		return;
+
+	scoped_guard(mutex, &qmi_tmd_cli->mutex) {
+		qmi_tmd_cli->exiting = true;
+		qmi_tmd_cli->connection_active = false;
+	}
+
+	qmi_handle_release(&qmi_tmd_cli->handle);
+	cancel_work_sync(&qmi_tmd_cli->svc_arrive_work);
+	qmi_tmd_unregister(qmi_tmd_cli);
+}
+EXPORT_SYMBOL_GPL(qmi_tmd_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Qualcomm QMI Thermal Mitigation support");
diff --git a/drivers/soc/qcom/rpmh-internal.h b/drivers/soc/qcom/rpmh-internal.h
index e3cf1beff80388..39441a25af9ba9 100644
--- a/drivers/soc/qcom/rpmh-internal.h
+++ b/drivers/soc/qcom/rpmh-internal.h
@@ -20,7 +20,7 @@
 struct rsc_drv;
 
 /**
- * struct tcs_group: group of Trigger Command Sets (TCS) to send state requests
+ * struct tcs_group - group of Trigger Command Sets (TCS) to send state requests
  * to the controller
  *
  * @drv:       The controller.
@@ -53,7 +53,7 @@ struct tcs_group {
 };
 
 /**
- * struct rpmh_request: the message to be sent to rpmh-rsc
+ * struct rpmh_request - the message to be sent to rpmh-rsc
  *
  * @msg: the request
  * @cmd: the payload that will be part of the @msg
@@ -70,7 +70,7 @@ struct rpmh_request {
 };
 
 /**
- * struct rpmh_ctrlr: our representation of the controller
+ * struct rpmh_ctrlr - our representation of the controller
  *
  * @cache: the list of cached requests
  * @cache_lock: synchronize access to the cache data
@@ -84,13 +84,19 @@ struct rpmh_ctrlr {
 	struct list_head batch_cache;
 };
 
+/**
+ * struct rsc_ver - Hardware version of the RSC controller.
+ *
+ * @major: Major version number.
+ * @minor: Minor version number.
+ */
 struct rsc_ver {
 	u32 major;
 	u32 minor;
 };
 
 /**
- * struct rsc_drv: the Direct Resource Voter (DRV) of the
+ * struct rsc_drv - the Direct Resource Voter (DRV) of the
  * Resource State Coordinator controller (RSC)
  *
  * @name:               Controller identifier.
@@ -116,6 +122,8 @@ struct rsc_ver {
  *                      slot
  * @client:             Handle to the DRV's client.
  * @dev:                RSC device.
+ * @ver:                HW version of the RSC controller.
+ * @regs:               Pointer to the register offset table for this version.
  */
 struct rsc_drv {
 	const char *name;
diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c
index f881af35ecfc51..c4542318eac47b 100644
--- a/drivers/soc/qcom/rpmh-rsc.c
+++ b/drivers/soc/qcom/rpmh-rsc.c
@@ -842,7 +842,7 @@ void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv)
 }
 
 /**
- * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
+ * rpmh_rsc_cpu_pm_callback() - CPU PM notifier to flush sleep/wake TCS data.
  * @nfb:    Pointer to the notifier block in struct rsc_drv.
  * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
  * @v:      Unused
@@ -923,7 +923,7 @@ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
 }
 
 /**
- * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy.
+ * rpmh_rsc_pd_callback() - PM domain callback to flush sleep/wake TCS data.
  * @nfb:    Pointer to the genpd notifier block in struct rsc_drv.
  * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON.
  * @v:      Unused
diff --git a/drivers/soc/qcom/rpmh.c b/drivers/soc/qcom/rpmh.c
index f881c4c757ec83..360242a315e37e 100644
--- a/drivers/soc/qcom/rpmh.c
+++ b/drivers/soc/qcom/rpmh.c
@@ -41,7 +41,7 @@
 #define ctrlr_to_drv(ctrlr) container_of(ctrlr, struct rsc_drv, client)
 
 /**
- * struct cache_req: the request object for caching
+ * struct cache_req - the request object for caching
  *
  * @addr: the address of the resource
  * @sleep_val: the sleep vote
@@ -56,7 +56,7 @@ struct cache_req {
 };
 
 /**
- * struct batch_cache_req - An entry in our batch catch
+ * struct batch_cache_req - An entry in our batch cache
  *
  * @list: linked list obj
  * @count: number of messages
@@ -76,6 +76,14 @@ static struct rpmh_ctrlr *get_rpmh_ctrlr(const struct device *dev)
 	return &drv->client;
 }
 
+/**
+ * rpmh_tx_done() - Signal the completion of a RPMH transfer.
+ * @msg: The request that was previously sent.
+ *
+ * Called by the RSC driver when an active-only transfer is complete.
+ * Signals any blocking waiter and frees the message if it was dynamically
+ * allocated.
+ */
 void rpmh_tx_done(const struct tcs_request *msg)
 {
 	struct rpmh_request *rpm_msg = container_of(msg, struct rpmh_request,
@@ -157,7 +165,7 @@ static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
 }
 
 /**
- * __rpmh_write: Cache and send the RPMH request
+ * __rpmh_write() - Cache and send the RPMH request.
  *
  * @dev: The device making the request
  * @state: Active/Sleep request type
@@ -166,6 +174,8 @@ static struct cache_req *cache_rpm_request(struct rpmh_ctrlr *ctrlr,
  * Cache the RPMH request and send if the state is ACTIVE_ONLY.
  * SLEEP/WAKE_ONLY requests are not sent to the controller at
  * this time. Use rpmh_flush() to send them to the controller.
+ *
+ * Return: 0 on success, negative error code on failure.
  */
 static int __rpmh_write(const struct device *dev, enum rpmh_state state,
 			struct rpmh_request *rpm_msg)
@@ -248,7 +258,7 @@ int rpmh_read(const struct device *dev, struct tcs_cmd *cmd)
 EXPORT_SYMBOL_GPL(rpmh_read);
 
 /**
- * rpmh_write_async: Write a set of RPMH commands
+ * rpmh_write_async() - Write a set of RPMH commands.
  *
  * @dev: The device making the request
  * @state: Active/sleep set
@@ -257,6 +267,8 @@ EXPORT_SYMBOL_GPL(rpmh_read);
  *
  * Write a set of RPMH commands, the order of commands is maintained
  * and will be sent as a single shot.
+ *
+ * Return: 0 on success, negative error code on failure.
  */
 int rpmh_write_async(const struct device *dev, enum rpmh_state state,
 		     const struct tcs_cmd *cmd, u32 n)
@@ -280,7 +292,7 @@ int rpmh_write_async(const struct device *dev, enum rpmh_state state,
 EXPORT_SYMBOL_GPL(rpmh_write_async);
 
 /**
- * rpmh_write: Write a set of RPMH commands and block until response
+ * rpmh_write() - Write a set of RPMH commands and block until response
  *
  * @dev: The device making the request
  * @state: Active/sleep set
@@ -288,6 +300,8 @@ EXPORT_SYMBOL_GPL(rpmh_write_async);
  * @n: The number of elements in @cmd
  *
  * May sleep. Do not call from atomic contexts.
+ *
+ * Return: 0 on success, negative error code on failure.
  */
 int rpmh_write(const struct device *dev, enum rpmh_state state,
 	       const struct tcs_cmd *cmd, u32 n)
@@ -342,7 +356,7 @@ static int flush_batch(struct rpmh_ctrlr *ctrlr)
 }
 
 /**
- * rpmh_write_batch: Write multiple sets of RPMH commands and wait for the
+ * rpmh_write_batch() - Write multiple sets of RPMH commands and wait for the
  * batch to finish.
  *
  * @dev: the device making the request
@@ -350,13 +364,15 @@ static int flush_batch(struct rpmh_ctrlr *ctrlr)
  * @cmd: The payload data
  * @n: The array of count of elements in each batch, 0 terminated.
  *
- * Write a request to the RSC controller without caching. If the request
- * state is ACTIVE, then the requests are treated as completion request
- * and sent to the controller immediately. The function waits until all the
- * commands are complete. If the request was to SLEEP or WAKE_ONLY, then the
- * request is sent as fire-n-forget and no ack is expected.
+ * If the request state is ACTIVE_ONLY, the requests are sent to the
+ * controller immediately and the function waits until all commands are
+ * complete. If the state is SLEEP or WAKE_ONLY, the requests are cached
+ * and the function returns immediately; they are sent to the controller
+ * later by rpmh_flush().
  *
  * May sleep. Do not call from atomic contexts for ACTIVE_ONLY requests.
+ *
+ * Return: 0 on success, negative error code on failure.
  */
 int rpmh_write_batch(const struct device *dev, enum rpmh_state state,
 		     const struct tcs_cmd *cmd, u32 *n)
@@ -519,7 +535,7 @@ int rpmh_flush(struct rpmh_ctrlr *ctrlr)
 }
 
 /**
- * rpmh_invalidate: Invalidate sleep and wake sets in batch_cache
+ * rpmh_invalidate() - Invalidate sleep and wake sets in batch_cache
  *
  * @dev: The device making the request
  *
diff --git a/drivers/soc/qcom/smem_dramc.c b/drivers/soc/qcom/smem_dramc.c
index 63ebe7f6db5905..497513c953084d 100644
--- a/drivers/soc/qcom/smem_dramc.c
+++ b/drivers/soc/qcom/smem_dramc.c
@@ -20,12 +20,22 @@
 #define MAX_DDR_FREQ_NUM_V5		14
 
 #define MAX_CHAN_NUM			8
+#define MAX_CHAN_NUM_V7_GLYMUR		16
 #define MAX_RANK_NUM			2
 
 #define DDR_HBB_MIN			13
 #define DDR_HBB_MAX			19
 
 #define MAX_SHUB_ENTRIES		8
+#define MAX_SHUB_ENTRIES_V7_GLYMUR	10
+#define MAX_DDR_REGIONS_V7_GLYMUR	4
+
+/*
+ * Glymur's DRAM information entry has 256 bytes of trailing zeroes
+ * after its DDR details, four region slots, 10-entry SHUB frequency
+ * plan, and v6 misc information.
+ */
+#define DDR_V7_GLYMUR_RESERVED_SIZE	256
 
 static struct smem_dram *__dram;
 
@@ -39,6 +49,7 @@ enum ddr_info_version {
 	INFO_V6, /* INFO_V6 seems to only have shipped with 6 DDR regions, unlike V7 */
 	INFO_V7,
 	INFO_V7_WITH_6_REGIONS,
+	INFO_V7_GLYMUR,
 };
 
 struct smem_dram {
@@ -188,6 +199,37 @@ struct ddr_details_v7 {
 	struct ddr_regions_v5 ddr_regions;
 };
 
+struct ddr_regions_v7_glymur {
+	__le32 ddr_region_num;
+	__le64 ddr_rank0_size;
+	__le64 ddr_rank1_size;
+	__le64 ddr_cs0_start_addr;
+	__le64 ddr_cs1_start_addr;
+	__le32 highest_bank_addr_bit;
+	struct ddr_region_v5 ddr_region[MAX_DDR_REGIONS_V7_GLYMUR];
+};
+
+struct ddr_details_v7_glymur {
+	u8 manufacturer_id;
+	u8 device_type;
+	struct ddr_part_details ddr_params[MAX_CHAN_NUM_V7_GLYMUR];
+	struct ddr_freq_plan_v5 ddr_freq_tbl;
+	u8 num_channels;
+	u8 sct_config;
+	struct ddr_regions_v7_glymur ddr_regions;
+};
+
+struct shub_freq_plan_entry_v7_glymur {
+	u8 num_shub_freqs;
+	struct shub_freq_table shub_freq[MAX_SHUB_ENTRIES_V7_GLYMUR];
+};
+
+struct ddr_v7_glymur_tail {
+	struct shub_freq_plan_entry_v7_glymur shub_freq_plan;
+	struct ddr_misc_info_v6 misc_info;
+	u8 reserved[DDR_V7_GLYMUR_RESERVED_SIZE];
+};
+
 /**
  * qcom_smem_dram_get_hbb(): Get the Highest bank address bit
  *
@@ -219,7 +261,7 @@ static void smem_dram_parse_v3_data(struct smem_dram *dram, void *data)
 
 		if (freq_entry->freq_khz && freq_entry->enabled) {
 			u32 freq_khz = le32_to_cpu(freq_entry->freq_khz);
-			dram->frequencies[dram->num_frequencies++] = 1000 * freq_khz;
+			dram->frequencies[dram->num_frequencies++] = 1000UL * freq_khz;
 		}
 	}
 }
@@ -231,8 +273,11 @@ static void smem_dram_parse_v3_14freqs_data(struct smem_dram *dram, void *data)
 	for (int i = 0; i < MAX_DDR_FREQ_NUM_V3 + 1; i++) {
 		struct ddr_freq_table *freq_entry = &details->ddr_freq_tbl.ddr_freq[i];
 
-		if (freq_entry->freq_khz && freq_entry->enabled)
-			dram->frequencies[dram->num_frequencies++] = 1000 * freq_entry->freq_khz;
+		if (freq_entry->freq_khz && freq_entry->enabled) {
+			u32 freq_khz = le32_to_cpu(freq_entry->freq_khz);
+
+			dram->frequencies[dram->num_frequencies++] = 1000UL * freq_khz;
+		}
 	}
 }
 
@@ -248,7 +293,7 @@ static void smem_dram_parse_v4_data(struct smem_dram *dram, void *data)
 
 		if (freq_entry->freq_khz && freq_entry->enabled) {
 			u32 freq_khz = le32_to_cpu(freq_entry->freq_khz);
-			dram->frequencies[dram->num_frequencies++] = 1000 * freq_khz;
+			dram->frequencies[dram->num_frequencies++] = 1000UL * freq_khz;
 		}
 	}
 }
@@ -265,7 +310,7 @@ static void smem_dram_parse_v5_data(struct smem_dram *dram, void *data)
 
 		if (freq_entry->freq_khz && freq_entry->enabled) {
 			u32 freq_khz = le32_to_cpu(freq_entry->freq_khz);
-			dram->frequencies[dram->num_frequencies++] = 1000 * freq_khz;
+			dram->frequencies[dram->num_frequencies++] = 1000UL * freq_khz;
 		}
 	}
 }
@@ -282,6 +327,23 @@ static void smem_dram_parse_v7_data(struct smem_dram *dram, void *data)
 
 		if (freq_entry->freq_khz && freq_entry->enabled) {
 			u32 freq_khz = le32_to_cpu(freq_entry->freq_khz);
+			dram->frequencies[dram->num_frequencies++] = 1000UL * freq_khz;
+		}
+	}
+}
+
+static void smem_dram_parse_v7_glymur_data(struct smem_dram *dram, void *data)
+{
+	struct ddr_details_v7_glymur *details = data;
+
+	dram->hbb = le32_to_cpu(details->ddr_regions.highest_bank_addr_bit);
+
+	for (int i = 0; i < MAX_DDR_FREQ_NUM_V5; i++) {
+		struct ddr_freq_table *freq_entry = &details->ddr_freq_tbl.ddr_freq[i];
+
+		if (freq_entry->freq_khz && freq_entry->enabled) {
+			u32 freq_khz = le32_to_cpu(freq_entry->freq_khz);
+
 			dram->frequencies[dram->num_frequencies++] = 1000 * freq_khz;
 		}
 	}
@@ -341,6 +403,10 @@ static int smem_dram_infer_struct_version(size_t size)
 		    sizeof(struct shub_freq_plan_entry))
 		return INFO_V7_WITH_6_REGIONS;
 
+	if (size == sizeof(struct ddr_details_v7_glymur) +
+		    sizeof(struct ddr_v7_glymur_tail))
+		return INFO_V7_GLYMUR;
+
 	return INFO_UNKNOWN;
 }
 
@@ -382,7 +448,7 @@ struct dentry *smem_dram_parse(struct qcom_smem *smem, struct device *dev)
 		return ERR_PTR(-ENODATA);
 
 	ver = smem_dram_infer_struct_version(actual_size);
-	if (ver < 0) {
+	if ((int)ver < 0) {
 		/* Some SoCs don't provide data that's useful for us */
 		return ERR_PTR(-ENODATA);
 	} else if (ver == INFO_UNKNOWN) {
@@ -415,6 +481,9 @@ struct dentry *smem_dram_parse(struct qcom_smem *smem, struct device *dev)
 	case INFO_V7_WITH_6_REGIONS:
 		smem_dram_parse_v7_data(dram, data);
 		break;
+	case INFO_V7_GLYMUR:
+		smem_dram_parse_v7_glymur_data(dram, data);
+		break;
 	default:
 		return ERR_PTR(-EINVAL);
 	}
diff --git a/drivers/soc/qcom/smp2p.c b/drivers/soc/qcom/smp2p.c
index af0ceeaf6e0730..01b8580c9f2712 100644
--- a/drivers/soc/qcom/smp2p.c
+++ b/drivers/soc/qcom/smp2p.c
@@ -102,7 +102,7 @@ struct smp2p_entry {
 	struct qcom_smp2p *smp2p;
 
 	const char *name;
-	u32 *value;
+	u32 __iomem *value;
 	u32 last_value;
 
 	struct irq_domain *domain;
@@ -276,7 +276,7 @@ static void qcom_smp2p_start_in(struct qcom_smp2p *smp2p)
 		list_for_each_entry(entry, &smp2p->inbound, node) {
 			memcpy(buf, in->entries[i].name, sizeof(buf));
 			if (!strcmp(buf, entry->name)) {
-				entry->value = &in->entries[i].value;
+				entry->value = (u32 __iomem *)&in->entries[i].value;
 				entry->last_value = readl(entry->value);
 				break;
 			}
@@ -302,7 +302,7 @@ static void qcom_smp2p_notify_in(struct qcom_smp2p *smp2p)
 		list_for_each_entry(entry, &smp2p->inbound, node) {
 			memcpy(buf, in->entries[i].name, sizeof(buf));
 			if (!strcmp(buf, entry->name)) {
-				entry->value = &in->entries[i].value;
+				entry->value = (u32 __iomem *)&in->entries[i].value;
 				break;
 			}
 		}
@@ -411,15 +411,9 @@ static int smp2p_set_irq_type(struct irq_data *irqd, unsigned int type)
 	if (!(type & IRQ_TYPE_EDGE_BOTH))
 		return -EINVAL;
 
-	if (type & IRQ_TYPE_EDGE_RISING)
-		set_bit(irq, entry->irq_rising);
-	else
-		clear_bit(irq, entry->irq_rising);
+	assign_bit(irq, entry->irq_rising, type & IRQ_TYPE_EDGE_RISING);
 
-	if (type & IRQ_TYPE_EDGE_FALLING)
-		set_bit(irq, entry->irq_falling);
-	else
-		clear_bit(irq, entry->irq_falling);
+	assign_bit(irq, entry->irq_falling, type & IRQ_TYPE_EDGE_FALLING);
 
 	return 0;
 }
@@ -528,7 +522,7 @@ static int qcom_smp2p_outbound_entry(struct qcom_smp2p *smp2p,
 	memcpy(out->entries[out->valid_entries].name, buf, SMP2P_MAX_ENTRY_NAME);
 
 	/* Make the logical entry reference the physical value */
-	entry->value = &out->entries[out->valid_entries].value;
+	entry->value = (u32 __iomem *)&out->entries[out->valid_entries].value;
 
 	out->valid_entries++;
 
@@ -720,10 +714,8 @@ static int qcom_smp2p_probe(struct platform_device *pdev)
 					NULL, qcom_smp2p_intr,
 					IRQF_ONESHOT,
 					NULL, (void *)smp2p);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to request interrupt\n");
+	if (ret)
 		goto unwind_interfaces;
-	}
 
 	/*
 	 * Treat smp2p interrupt as wakeup source, but keep it disabled
diff --git a/drivers/soc/qcom/smsm.c b/drivers/soc/qcom/smsm.c
index 021e9d1f61dc52..bb487bf6bf037f 100644
--- a/drivers/soc/qcom/smsm.c
+++ b/drivers/soc/qcom/smsm.c
@@ -82,8 +82,8 @@ struct qcom_smsm {
 	u32 num_hosts;
 	u32 num_entries;
 
-	u32 *local_state;
-	u32 *subscription;
+	u32 __iomem *local_state;
+	u32 __iomem *subscription;
 	struct qcom_smem_state *state;
 
 	spinlock_t lock;
@@ -115,8 +115,8 @@ struct smsm_entry {
 	DECLARE_BITMAP(irq_falling, 32);
 	unsigned long last_value;
 
-	u32 *remote_state;
-	u32 *subscription;
+	u32 __iomem *remote_state;
+	u32 __iomem *subscription;
 };
 
 /**
@@ -277,10 +277,8 @@ static void smsm_unmask_irq(struct irq_data *irqd)
 	u32 val;
 
 	/* Make sure our last cached state is up-to-date */
-	if (readl(entry->remote_state) & BIT(irq))
-		set_bit(irq, &entry->last_value);
-	else
-		clear_bit(irq, &entry->last_value);
+	assign_bit(irq, &entry->last_value,
+		   readl(entry->remote_state) & BIT(irq));
 
 	set_bit(irq, entry->irq_enabled);
 
@@ -304,15 +302,9 @@ static int smsm_set_irq_type(struct irq_data *irqd, unsigned int type)
 	if (!(type & IRQ_TYPE_EDGE_BOTH))
 		return -EINVAL;
 
-	if (type & IRQ_TYPE_EDGE_RISING)
-		set_bit(irq, entry->irq_rising);
-	else
-		clear_bit(irq, entry->irq_rising);
+	assign_bit(irq, entry->irq_rising, type & IRQ_TYPE_EDGE_RISING);
 
-	if (type & IRQ_TYPE_EDGE_FALLING)
-		set_bit(irq, entry->irq_falling);
-	else
-		clear_bit(irq, entry->irq_falling);
+	assign_bit(irq, entry->irq_falling, type & IRQ_TYPE_EDGE_FALLING);
 
 	return 0;
 }
@@ -451,10 +443,8 @@ static int smsm_inbound_entry(struct qcom_smsm *smsm,
 					NULL, smsm_intr,
 					IRQF_ONESHOT,
 					"smsm", (void *)entry);
-	if (ret) {
-		dev_err(smsm->dev, "failed to request interrupt\n");
+	if (ret)
 		return ret;
-	}
 
 	entry->domain = irq_domain_create_linear(of_fwnode_handle(node), 32, &smsm_irq_ops, entry);
 	if (!entry->domain) {
@@ -513,9 +503,9 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 	struct device_node *node;
 	struct smsm_entry *entry;
 	struct qcom_smsm *smsm;
-	u32 *intr_mask;
+	u32 __iomem *intr_mask;
 	size_t size;
-	u32 *states;
+	u32 __iomem *states;
 	u32 id;
 	int ret;
 
@@ -579,7 +569,7 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 		goto out_put;
 	}
 
-	states = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SHARED_STATE, NULL);
+	states = (u32 __iomem *)qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SHARED_STATE, NULL);
 	if (IS_ERR(states)) {
 		dev_err(&pdev->dev, "Unable to acquire shared state entry\n");
 		ret = PTR_ERR(states);
@@ -594,7 +584,7 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 		goto out_put;
 	}
 
-	intr_mask = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, NULL);
+	intr_mask = (u32 __iomem *)qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, NULL);
 	if (IS_ERR(intr_mask)) {
 		dev_err(&pdev->dev, "unable to acquire shared memory interrupt mask\n");
 		ret = PTR_ERR(intr_mask);
diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
index 1836e226a2945d..38efc5db7fa834 100644
--- a/drivers/soc/qcom/socinfo.c
+++ b/drivers/soc/qcom/socinfo.c
@@ -450,6 +450,7 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(IPQ8172) },
 	{ qcom_board_id(IPQ8173) },
 	{ qcom_board_id(IPQ8174) },
+	{ qcom_board_id(SM7250) },
 	{ qcom_board_id(IPQ6018) },
 	{ qcom_board_id(IPQ6028) },
 	{ qcom_board_id(SDM429W) },
@@ -541,8 +542,10 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(QCS9075) },
 	{ qcom_board_id(QCS615) },
 	{ qcom_board_id(SA8797P) },
+	{ qcom_board_id(SM8975) },
 	{ qcom_board_id(CQ7790M) },
 	{ qcom_board_id(CQ7790S) },
+	{ qcom_board_id(SM8950) },
 	{ qcom_board_id(CQ2390M) },
 	{ qcom_board_id(CQ2390S) },
 	{ qcom_board_id(IQ2390S) },
diff --git a/drivers/soc/qcom/ubwc_config.c b/drivers/soc/qcom/ubwc_config.c
index 23901a4c51f730..a4d8769f855059 100644
--- a/drivers/soc/qcom/ubwc_config.c
+++ b/drivers/soc/qcom/ubwc_config.c
@@ -105,6 +105,13 @@ static const struct qcom_ubwc_cfg_data milos_data = {
 	.highest_bank_bit = 15,
 };
 
+static const struct qcom_ubwc_cfg_data ubwc_7_0_hbb16 = {
+	.ubwc_enc_version = UBWC_7_0,
+	.flags = UBWC_SWIZZLE_ENABLE_LVL2 |
+		 UBWC_SWIZZLE_ENABLE_LVL3,
+	.highest_bank_bit = 16,
+};
+
 static const struct of_device_id qcom_ubwc_configs[] __maybe_unused = {
 	{ .compatible = "qcom,apq8016", .data = &no_ubwc_data },
 	{ .compatible = "qcom,apq8026", .data = &no_ubwc_data },
@@ -112,8 +119,11 @@ static const struct of_device_id qcom_ubwc_configs[] __maybe_unused = {
 	{ .compatible = "qcom,apq8096", .data = &ubwc_1_0_hbb15 },
 	{ .compatible = "qcom,eliza", .data = &ubwc_5_0_hbb15 },
 	{ .compatible = "qcom,glymur", .data = &glymur_data},
+	{ .compatible = "qcom,hawi", .data = &ubwc_7_0_hbb16 },
 	{ .compatible = "qcom,kaanapali", .data = &ubwc_6_0_hbb16 },
+	{ .compatible = "qcom,kalambo", .data = &glymur_data },
 	{ .compatible = "qcom,mahua", .data = &glymur_data },
+	{ .compatible = "qcom,maili", .data = &ubwc_7_0_hbb16, },
 	{ .compatible = "qcom,milos", .data = &milos_data },
 	{ .compatible = "qcom,msm8226", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8916", .data = &no_ubwc_data },
@@ -121,12 +131,14 @@ static const struct of_device_id qcom_ubwc_configs[] __maybe_unused = {
 	{ .compatible = "qcom,msm8937", .data = &ubwc_1_0_hbb14 },
 	{ .compatible = "qcom,msm8929", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8939", .data = &no_ubwc_data },
+	{ .compatible = "qcom,msm8952", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8953", .data = &ubwc_1_0_hbb14 },
 	{ .compatible = "qcom,msm8956", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8974", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8976", .data = &no_ubwc_data },
 	{ .compatible = "qcom,msm8996", .data = &ubwc_1_0_hbb15 },
 	{ .compatible = "qcom,msm8998", .data = &ubwc_1_0_hbb15 },
+	{ .compatible = "qcom,nord", .data = &glymur_data },
 	{ .compatible = "qcom,qcm2290", .data = &ubwc_0_0_hbb15, },
 	{ .compatible = "qcom,qcm6490", .data = &ubwc_3_1_hbb14, },
 	{ .compatible = "qcom,qcs8300", .data = &ubwc_4_0_hbb16, },
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index eb8ac0d47d9d42..a63df28df3b5e7 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -227,6 +227,14 @@ config ARCH_R8A774A1
 	help
 	  This enables support for the Renesas RZ/G2M SoC.
 
+config ARCH_R8A774A3
+	bool "ARM64 Platform support for R8A774A3 (RZ/G2M v3.0)"
+	default y if ARCH_RENESAS
+	select ARCH_RCAR_GEN3
+	select SYSC_R8A774A3
+	help
+	  This enables support for the Renesas RZ/G2M SoC (version 3.0).
+
 config ARCH_R8A774B1
 	bool "ARM64 Platform support for R8A774B1 (RZ/G2N)"
 	default y if ARCH_RENESAS
@@ -410,6 +418,7 @@ config ARCH_R9A09G011
 config ARCH_R9A09G047
 	bool "ARM64 Platform support for R9A09G047 (RZ/G3E)"
 	default y if ARCH_RENESAS
+	select RENESAS_RZV2H_ICU
 	select SYS_R9A09G047
 	help
 	  This enables support for the Renesas RZ/G3E SoC variants.
@@ -417,6 +426,7 @@ config ARCH_R9A09G047
 config ARCH_R9A09G056
 	bool "ARM64 Platform support for R9A09G056 (RZ/V2N)"
 	default y if ARCH_RENESAS
+	select RENESAS_RZV2H_ICU
 	select SYS_R9A09G056
 	help
 	  This enables support for the Renesas RZ/V2N SoC variants.
@@ -468,11 +478,12 @@ endif # RISCV
 config RCAR_MFIS
 	tristate "Renesas R-Car MFIS driver"
 	depends on ARCH_RENESAS || COMPILE_TEST
-	depends on MAILBOX
+	depends on MAILBOX && HWSPINLOCK
 	help
-	  Select this option to enable the Renesas R-Car MFIS core driver for
-	  the MFIS device found on SoCs like R-Car. On families like Gen5, this
-	  is needed to communicate with the SCP.
+	  Select this option to enable support for the Renesas R-Car
+	  Multifunctional Interface (MFIS) device found on SoCs like R-Car.
+	  On families like R-Car Gen5, this is needed to communicate with the
+	  SCP.
 
 config PWC_RZV2M
 	bool "Renesas RZ/V2M PWC support" if COMPILE_TEST
@@ -486,6 +497,7 @@ config RZN1_IRQMUX
 config SYSC_RZ
 	bool "System controller for RZ SoCs" if COMPILE_TEST
 	select MFD_SYSCON
+	select AUXILIARY_BUS
 
 config SYSC_R9A08G045
 	bool "Renesas System controller support for R9A08G045 (RZ/G3S)" if COMPILE_TEST
@@ -494,6 +506,8 @@ config SYSC_R9A08G045
 config SYSC_R9A08G046
 	bool "Renesas System controller support for R9A08G046 (RZ/G3L)" if COMPILE_TEST
 	select SYSC_RZ
+	select POWER_SEQUENCING
+	select POWER_SEQUENCING_RENESAS_PWRRDY
 
 config SYS_R9A09G047
 	bool "Renesas System controller support for R9A09G047 (RZ/G3E)" if COMPILE_TEST
diff --git a/drivers/soc/renesas/r9a08g046-sysc.c b/drivers/soc/renesas/r9a08g046-sysc.c
index 90db9d38353955..cd129c72746105 100644
--- a/drivers/soc/renesas/r9a08g046-sysc.c
+++ b/drivers/soc/renesas/r9a08g046-sysc.c
@@ -76,6 +76,7 @@ static const struct rz_sysc_soc_id_init_data rzg3l_sysc_soc_id_init_data __initc
 	.devid_offset = 0xa04,
 	.revision_mask = GENMASK(31, 28),
 	.specific_id_mask = GENMASK(27, 0),
+	.pwrrdy_pwrseq = true,
 };
 
 const struct rz_sysc_init_data rzg3l_sysc_init_data __initconst = {
diff --git a/drivers/soc/renesas/rcar-mfis.c b/drivers/soc/renesas/rcar-mfis.c
index 3435c3e1619814..d5bbc3077373b4 100644
--- a/drivers/soc/renesas/rcar-mfis.c
+++ b/drivers/soc/renesas/rcar-mfis.c
@@ -8,6 +8,7 @@
  */
 #include <dt-bindings/soc/renesas,r8a78000-mfis.h>
 #include <linux/device.h>
+#include <linux/hwspinlock.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
@@ -19,12 +20,18 @@
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 
+/* FIXME: hwspinlock core refactoring to not need internal header is on-going */
+#include "../../hwspinlock/hwspinlock_internal.h"
+
+#define MFISLCKR0	0xc0
+#define MFISLCKR8	0x0724
 #define MFISWPCNTR	0x0900
 #define MFISWACNTR	0x0904
 
 #define MFIS_X5H_IICR(i) ((i) * 0x1000 + 0x00)
 #define MFIS_X5H_EICR(i) ((i) * 0x1000 + 0x04)
 
+#define MFIS_NUM_LOCKS 64
 #define MFIS_UNPROTECT_KEY 0xACCE0000
 
 struct mfis_priv;
@@ -42,6 +49,7 @@ struct mfis_info {
 	unsigned int mb_tx_uses_eicr:1;
 	unsigned int mb_channels_are_unidir:1;
 	u32 (*mb_calc_reg)(u32 chan_num, bool tx_uses_eicr, bool is_only_rx);
+	int hwsp_base_id;
 };
 
 struct mfis_chan_priv {
@@ -50,7 +58,7 @@ struct mfis_chan_priv {
 };
 
 struct mfis_priv {
-	spinlock_t unprotect_lock; /* guards access to the unprotection reg */
+	raw_spinlock_t unprotect_lock; /* guards access to unprotection reg */
 	struct device *dev;
 	struct mfis_reg common_reg;
 	struct mfis_reg mbox_reg;
@@ -59,6 +67,9 @@ struct mfis_priv {
 	/* mailbox private data */
 	struct mbox_controller mbox;
 	struct mfis_chan_priv *chan_privs;
+
+	/* hwspinlock private data */
+	struct hwspinlock_device bank; /* flex array inside, must be last! */
 };
 
 static u32 mfis_read(struct mfis_reg *mreg, unsigned int reg)
@@ -80,12 +91,44 @@ static void mfis_write(struct mfis_reg *mreg, u32 reg, u32 val)
 	unprotect_code = (MFIS_UNPROTECT_KEY & ~unprotect_mask) |
 			 ((mreg->start + reg) & unprotect_mask);
 
-	spin_lock_irqsave(&priv->unprotect_lock, flags);
+	raw_spin_lock_irqsave(&priv->unprotect_lock, flags);
 	iowrite32(unprotect_code, priv->common_reg.base + MFISWACNTR);
 	iowrite32(val, mreg->base + reg);
-	spin_unlock_irqrestore(&priv->unprotect_lock, flags);
+	raw_spin_unlock_irqrestore(&priv->unprotect_lock, flags);
 }
 
+/********************************************************
+ *			HW Spinlocks			*
+ ********************************************************/
+
+#define MFISLCKR8_CH_OFS (MFISLCKR8 - 8 * sizeof(u32))
+
+static int rcar_mfis_hwsp_trylock(struct hwspinlock *lock)
+{
+	struct mfis_priv *priv = lock->priv;
+	int id = hwlock_to_id(lock) - lock->bank->base_id;
+	u32 val, reg;
+
+	reg = id * sizeof(u32) + (id < 8 ? MFISLCKR0 : MFISLCKR8_CH_OFS);
+	val = mfis_read(&priv->common_reg, reg);
+	return !val;
+}
+
+static void rcar_mfis_hwsp_unlock(struct hwspinlock *lock)
+{
+	struct mfis_priv *priv = lock->priv;
+	int id = hwlock_to_id(lock) - lock->bank->base_id;
+	u32 reg;
+
+	reg = id * sizeof(u32) + (id < 8 ? MFISLCKR0 : MFISLCKR8_CH_OFS);
+	mfis_write(&priv->common_reg, reg, 0);
+}
+
+static const struct hwspinlock_ops rcar_mfis_hwsp_ops = {
+	.trylock	= rcar_mfis_hwsp_trylock,
+	.unlock		= rcar_mfis_hwsp_unlock,
+};
+
 /********************************************************
  *			Mailbox				*
  ********************************************************/
@@ -314,7 +357,7 @@ static int mfis_probe(struct platform_device *pdev)
 	struct mfis_priv *priv;
 	int ret;
 
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	priv = devm_kzalloc(dev, struct_size(priv, bank.lock, MFIS_NUM_LOCKS), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
@@ -323,7 +366,7 @@ static int mfis_probe(struct platform_device *pdev)
 	if (!priv->info)
 		return -ENOENT;
 
-	spin_lock_init(&priv->unprotect_lock);
+	raw_spin_lock_init(&priv->unprotect_lock);
 
 	ret = mfis_reg_probe(pdev, priv, &priv->common_reg, "common", true);
 	if (ret)
@@ -333,6 +376,14 @@ static int mfis_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	for (unsigned int ch = 0; ch < MFIS_NUM_LOCKS; ch++)
+		priv->bank.lock[ch].priv = priv;
+
+	ret = devm_hwspin_lock_register(dev, &priv->bank, &rcar_mfis_hwsp_ops,
+					priv->info->hwsp_base_id, MFIS_NUM_LOCKS);
+	if (ret)
+		return ret;
+
 	return mfis_mb_probe(priv);
 }
 
@@ -357,6 +408,7 @@ static const struct mfis_info mfis_info_r8a78000_scp = {
 	.mb_tx_uses_eicr = true,
 	.mb_channels_are_unidir = true,
 	.mb_calc_reg = mfis_mb_r8a78000_calc_reg,
+	.hwsp_base_id = MFIS_NUM_LOCKS,
 };
 
 static const struct of_device_id mfis_mfd_of_match[] = {
diff --git a/drivers/soc/renesas/rcar-rst.c b/drivers/soc/renesas/rcar-rst.c
index 0541990901fcbd..26f976880d3ff3 100644
--- a/drivers/soc/renesas/rcar-rst.c
+++ b/drivers/soc/renesas/rcar-rst.c
@@ -93,6 +93,7 @@ static const struct of_device_id rcar_rst_matches[] __initconst = {
 	{ .compatible = "renesas,r8a77470-rst", .data = &rcar_rst_gen2 },
 	/* RZ/G2 is handled like R-Car Gen3 */
 	{ .compatible = "renesas,r8a774a1-rst", .data = &rcar_rst_gen3 },
+	{ .compatible = "renesas,r8a774a3-rst", .data = &rcar_rst_gen3 },
 	{ .compatible = "renesas,r8a774b1-rst", .data = &rcar_rst_gen3 },
 	{ .compatible = "renesas,r8a774c0-rst", .data = &rcar_rst_gen3 },
 	{ .compatible = "renesas,r8a774e1-rst", .data = &rcar_rst_gen3 },
diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c
index 18acca73064c19..5a5d05013eb797 100644
--- a/drivers/soc/renesas/renesas-soc.c
+++ b/drivers/soc/renesas/renesas-soc.c
@@ -312,6 +312,9 @@ static const struct of_device_id renesas_socs[] __initconst __maybe_unused = {
 #ifdef CONFIG_ARCH_R8A774A1
 	{ .compatible = "renesas,r8a774a1",	.data = &soc_rz_g2m },
 #endif
+#ifdef CONFIG_ARCH_R8A774A3
+	{ .compatible = "renesas,r8a774a3",	.data = &soc_rz_g2m },
+#endif
 #ifdef CONFIG_ARCH_R8A774B1
 	{ .compatible = "renesas,r8a774b1",	.data = &soc_rz_g2n },
 #endif
diff --git a/drivers/soc/renesas/rz-sysc.c b/drivers/soc/renesas/rz-sysc.c
index 161e8c38eea69a..0e92c415d4befc 100644
--- a/drivers/soc/renesas/rz-sysc.c
+++ b/drivers/soc/renesas/rz-sysc.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2024 Renesas Electronics Corp.
  */
 
+#include <linux/auxiliary_bus.h>
 #include <linux/bitfield.h>
 #include <linux/cleanup.h>
 #include <linux/io.h>
@@ -149,6 +150,10 @@ static int rz_sysc_probe(struct platform_device *pdev)
 	if (IS_ERR(regmap))
 		return PTR_ERR(regmap);
 
+	if (data->soc_id_init_data->pwrrdy_pwrseq &&
+	    !devm_auxiliary_device_create(dev, "pwrseq-pwrrdy", regmap))
+		return -ENODEV;
+
 	return of_syscon_register_regmap(dev->of_node, regmap);
 }
 
diff --git a/drivers/soc/renesas/rz-sysc.h b/drivers/soc/renesas/rz-sysc.h
index 921ee0d26c470c..e55f3258d703f9 100644
--- a/drivers/soc/renesas/rz-sysc.h
+++ b/drivers/soc/renesas/rz-sysc.h
@@ -20,6 +20,7 @@
  * @revision_mask: SYSC SoC ID revision mask
  * @specific_id_mask: SYSC SoC ID specific ID mask
  * @print_id: print SoC-specific extended device identification
+ * @pwrrdy_pwrseq: has pwrrdy register controlled through power sequencer
  */
 struct rz_sysc_soc_id_init_data {
 	const char * const family;
@@ -29,6 +30,7 @@ struct rz_sysc_soc_id_init_data {
 	u32 specific_id_mask;
 	void (*print_id)(struct device *dev, void __iomem *sysc_base,
 			 struct soc_device_attribute *soc_dev_attr);
+	bool pwrrdy_pwrseq;
 };
 
 /**
diff --git a/drivers/soc/samsung/exynos-chipid.c b/drivers/soc/samsung/exynos-chipid.c
index 6ef9751e2509c9..506a0bcdfe30ab 100644
--- a/drivers/soc/samsung/exynos-chipid.c
+++ b/drivers/soc/samsung/exynos-chipid.c
@@ -64,6 +64,7 @@ static const struct exynos_soc_id {
 	{ "EXYNOS8890", 0xE8890000 },
 	/* Compatible with: samsung,exynos850-chipid */
 	{ "EXYNOS2200", 0xE9925000 },
+	{ "EXYNOS5515", 0xE5515000 },
 	{ "EXYNOS7885", 0xE7885000 },
 	{ "EXYNOS850", 0xE3830000 },
 	{ "EXYNOS8895", 0xE8895000 },
diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig
index 8b6fdb9ad1ea3d..318e7b60434249 100644
--- a/drivers/soc/tegra/Kconfig
+++ b/drivers/soc/tegra/Kconfig
@@ -102,7 +102,6 @@ config ARCH_TEGRA_210_SOC
 config ARCH_TEGRA_186_SOC
 	bool "NVIDIA Tegra186 SoC"
 	default ARCH_TEGRA
-	depends on !CPU_BIG_ENDIAN
 	select PINCTRL_TEGRA186
 	select MAILBOX
 	select SOC_TEGRA_PMC
@@ -117,7 +116,6 @@ config ARCH_TEGRA_186_SOC
 config ARCH_TEGRA_194_SOC
 	bool "NVIDIA Tegra194 SoC"
 	default ARCH_TEGRA
-	depends on !CPU_BIG_ENDIAN
 	select MAILBOX
 	select PINCTRL_TEGRA194
 	select SOC_TEGRA_PMC
@@ -127,7 +125,6 @@ config ARCH_TEGRA_194_SOC
 config ARCH_TEGRA_234_SOC
 	bool "NVIDIA Tegra234 SoC"
 	default ARCH_TEGRA
-	depends on !CPU_BIG_ENDIAN
 	select MAILBOX
 	select PINCTRL_TEGRA234
 	select SOC_TEGRA_PMC
@@ -137,7 +134,6 @@ config ARCH_TEGRA_234_SOC
 config ARCH_TEGRA_238_SOC
 	bool "NVIDIA Tegra238 SoC"
 	default ARCH_TEGRA
-	depends on !CPU_BIG_ENDIAN
 	select MAILBOX
 	select SOC_TEGRA_PMC
 	help
@@ -152,7 +148,6 @@ config ARCH_TEGRA_241_SOC
 config ARCH_TEGRA_264_SOC
 	bool "NVIDIA Tegra264 SoC"
 	default ARCH_TEGRA
-	depends on !CPU_BIG_ENDIAN
 	select MAILBOX
 	select SOC_TEGRA_PMC
 	help
diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c
index 39654964c35c3e..b8c8ccf80056cb 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra.c
@@ -65,6 +65,9 @@ static struct tegra_fuse *fuse = &(struct tegra_fuse) {
 };
 
 static const struct of_device_id tegra_fuse_match[] = {
+#ifdef CONFIG_ARCH_TEGRA_264_SOC
+	{ .compatible = "nvidia,tegra264-efuse", .data = &tegra264_fuse_soc },
+#endif
 #ifdef CONFIG_ARCH_TEGRA_234_SOC
 	{ .compatible = "nvidia,tegra234-efuse", .data = &tegra234_fuse_soc },
 #endif
@@ -175,6 +178,11 @@ static int tegra_fuse_probe(struct platform_device *pdev)
 		case TEGRA241:
 			fuse->soc = &tegra241_fuse_soc;
 			break;
+#endif
+#if defined(CONFIG_ARCH_TEGRA_264_SOC)
+		case TEGRA264:
+			fuse->soc = &tegra264_fuse_soc;
+			break;
 #endif
 		default:
 			return dev_err_probe(&pdev->dev, -EINVAL, "Unsupported SoC: %02x\n", chip);
@@ -413,7 +421,8 @@ const struct attribute_group tegra_soc_attr_group = {
 
 #if IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
     IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC) || \
-    IS_ENABLED(CONFIG_ARCH_TEGRA_241_SOC)
+    IS_ENABLED(CONFIG_ARCH_TEGRA_241_SOC) || \
+    IS_ENABLED(CONFIG_ARCH_TEGRA_264_SOC)
 static ssize_t platform_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
 {
diff --git a/drivers/soc/tegra/fuse/fuse-tegra30.c b/drivers/soc/tegra/fuse/fuse-tegra30.c
index 2fad8b13a3b961..623848116b693b 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra30.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra30.c
@@ -39,7 +39,8 @@
     defined(CONFIG_ARCH_TEGRA_186_SOC) || \
     defined(CONFIG_ARCH_TEGRA_194_SOC) || \
     defined(CONFIG_ARCH_TEGRA_234_SOC) || \
-    defined(CONFIG_ARCH_TEGRA_241_SOC)
+    defined(CONFIG_ARCH_TEGRA_241_SOC) || \
+    defined(CONFIG_ARCH_TEGRA_264_SOC)
 static u32 tegra30_fuse_read_early(struct tegra_fuse *fuse, unsigned int offset)
 {
 	if (WARN_ON(!fuse->base))
@@ -802,3 +803,168 @@ const struct tegra_fuse_soc tegra241_fuse_soc = {
 	.soc_attr_group = &tegra194_soc_attr_group,
 };
 #endif
+
+#ifdef CONFIG_ARCH_TEGRA_264_SOC
+static const struct nvmem_keepout tegra264_fuse_keepouts[] = {
+	{ .start = 0x0019c, .end = 0x001a0 },
+	{ .start = 0x00308, .end = 0x00310 },
+	{ .start = 0x0032c, .end = 0x00334 },
+	{ .start = 0x00350, .end = 0x00354 },
+	{ .start = 0x00494, .end = 0x0049c },
+	{ .start = 0x0078c, .end = 0x00790 },
+	{ .start = 0x007a0, .end = 0x007a8 },
+	{ .start = 0x007dc, .end = 0x007e4 },
+	{ .start = 0x008d8, .end = 0x008dc },
+	{ .start = 0x0096c, .end = 0x00970 },
+	{ .start = 0x00974, .end = 0x0097c },
+	{ .start = 0x009f4, .end = 0x009f8 },
+	{ .start = 0x00a14, .end = 0x00a20 },
+	{ .start = 0x00a44, .end = 0x00a4c },
+	{ .start = 0x00a50, .end = 0x00a58 },
+	{ .start = 0x00a5c, .end = 0x00a64 },
+	{ .start = 0x00a68, .end = 0x00a70 },
+	{ .start = 0x00acc, .end = 0x00ad0 },
+	{ .start = 0x00b0c, .end = 0x00b18 },
+	{ .start = 0x00c80, .end = 0x00c8c },
+	{ .start = 0x00dac, .end = 0x00db4 },
+	{ .start = 0x00db8, .end = 0x00dbc },
+	{ .start = 0x00e0c, .end = 0x00e10 },
+	{ .start = 0x00fd0, .end = 0x01f00 },
+	{ .start = 0x01f1c, .end = 0x10064 },
+	{ .start = 0x10084, .end = 0x100a0 },
+	{ .start = 0x100a4, .end = 0x1019c },
+	{ .start = 0x101a0, .end = 0x101cc },
+	{ .start = 0x101d0, .end = 0x10308 },
+	{ .start = 0x10310, .end = 0x1055c },
+	{ .start = 0x1057c, .end = 0x106b0 },
+	{ .start = 0x106b4, .end = 0x11008 },
+	{ .start = 0x1100c, .end = 0x11018 },
+	{ .start = 0x11020, .end = 0x110b8 },
+	{ .start = 0x110c4, .end = 0x110c8 },
+	{ .start = 0x110e8, .end = 0x110ec },
+	{ .start = 0x110f0, .end = 0x11124 },
+	{ .start = 0x11128, .end = 0x11168 },
+	{ .start = 0x1116c, .end = 0x111b8 },
+	{ .start = 0x111bc, .end = 0x111e4 },
+	{ .start = 0x111e8, .end = 0x111ec },
+	{ .start = 0x111f0, .end = 0x1121c },
+	{ .start = 0x11250, .end = 0x1133c },
+	{ .start = 0x11340, .end = 0x113c8 },
+	{ .start = 0x113d0, .end = 0x11420 },
+	{ .start = 0x11430, .end = 0x11440 },
+	{ .start = 0x11444, .end = 0x11468 },
+	{ .start = 0x1146c, .end = 0x114a8 },
+	{ .start = 0x114ac, .end = 0x115cc },
+	{ .start = 0x115d0, .end = 0x115dc },
+	{ .start = 0x115e0, .end = 0x116b4 },
+	{ .start = 0x116bc, .end = 0x11740 },
+	{ .start = 0x11744, .end = 0x117b8 },
+	{ .start = 0x117bc, .end = 0x11810 },
+	{ .start = 0x11814, .end = 0x11850 },
+	{ .start = 0x11858, .end = 0x118a0 },
+	{ .start = 0x118ac, .end = 0x11b48 },
+	{ .start = 0x11b50, .end = 0x11b5c },
+	{ .start = 0x11b60, .end = 0x11bf8 },
+	{ .start = 0x11bfc, .end = 0x11c18 },
+	{ .start = 0x11c40, .end = 0x12000 },
+	{ .start = 0x12004, .end = 0x12010 },
+	{ .start = 0x12014, .end = 0x120fc },
+	{ .start = 0x1211c, .end = 0x12120 },
+	{ .start = 0x12124, .end = 0x12144 },
+	{ .start = 0x1214c, .end = 0x12300 },
+	{ .start = 0x12308, .end = 0x1236c },
+	{ .start = 0x12370, .end = 0x126ac },
+	{ .start = 0x126b0, .end = 0x12c60 },
+	{ .start = 0x12c64, .end = 0x12d94 },
+	{ .start = 0x12da4, .end = 0x13004 },
+	{ .start = 0x13008, .end = 0x1300c },
+	{ .start = 0x13010, .end = 0x13014 },
+	{ .start = 0x13018, .end = 0x13020 },
+	{ .start = 0x13064, .end = 0x13084 },
+	{ .start = 0x130a0, .end = 0x130a4 },
+	{ .start = 0x130b8, .end = 0x130c4 },
+	{ .start = 0x130c8, .end = 0x130e8 },
+	{ .start = 0x130ec, .end = 0x130f0 },
+	{ .start = 0x130fc, .end = 0x1311c },
+	{ .start = 0x13120, .end = 0x13128 },
+	{ .start = 0x13144, .end = 0x1314c },
+	{ .start = 0x13168, .end = 0x1316c },
+	{ .start = 0x1319c, .end = 0x131a0 },
+	{ .start = 0x131ac, .end = 0x131c0 },
+	{ .start = 0x131cc, .end = 0x131d0 },
+	{ .start = 0x131e4, .end = 0x131e8 },
+	{ .start = 0x131ec, .end = 0x131f0 },
+	{ .start = 0x1321c, .end = 0x13250 },
+	{ .start = 0x132f0, .end = 0x13310 },
+	{ .start = 0x1332c, .end = 0x13334 },
+	{ .start = 0x1333c, .end = 0x13344 },
+	{ .start = 0x13350, .end = 0x13354 },
+	{ .start = 0x1336c, .end = 0x13370 },
+	{ .start = 0x133ac, .end = 0x133b0 },
+	{ .start = 0x133c8, .end = 0x133d0 },
+	{ .start = 0x13420, .end = 0x13430 },
+	{ .start = 0x13440, .end = 0x13444 },
+	{ .start = 0x13468, .end = 0x13470 },
+	{ .start = 0x13494, .end = 0x1349c },
+	{ .start = 0x134a8, .end = 0x134ac },
+	{ .start = 0x13544, .end = 0x1354c },
+	{ .start = 0x1355c, .end = 0x1357c },
+	{ .start = 0x135cc, .end = 0x135e0 },
+	{ .start = 0x135e4, .end = 0x13608 },
+	{ .start = 0x1360c, .end = 0x13620 },
+	{ .start = 0x136ac, .end = 0x136bc },
+	{ .start = 0x136c0, .end = 0x136c8 },
+	{ .start = 0x13714, .end = 0x13718 },
+	{ .start = 0x13724, .end = 0x13728 },
+	{ .start = 0x1372c, .end = 0x13730 },
+	{ .start = 0x13734, .end = 0x1373c },
+	{ .start = 0x13740, .end = 0x13744 },
+	{ .start = 0x13754, .end = 0x13758 },
+	{ .start = 0x13760, .end = 0x13784 },
+	{ .start = 0x1378c, .end = 0x13790 },
+	{ .start = 0x137a0, .end = 0x137a8 },
+	{ .start = 0x137b8, .end = 0x137bc },
+	{ .start = 0x137dc, .end = 0x137e4 },
+	{ .start = 0x137ec, .end = 0x137f0 },
+	{ .start = 0x13800, .end = 0x13818 },
+	{ .start = 0x13820, .end = 0x13828 },
+	{ .start = 0x13830, .end = 0x13858 },
+	{ .start = 0x138a0, .end = 0x138b0 },
+	{ .start = 0x138d0, .end = 0x138dc },
+	{ .start = 0x1396c, .end = 0x13970 },
+	{ .start = 0x13974, .end = 0x1397c },
+	{ .start = 0x139f4, .end = 0x139f8 },
+	{ .start = 0x13a14, .end = 0x13a20 },
+	{ .start = 0x13a44, .end = 0x13a4c },
+	{ .start = 0x13a50, .end = 0x13a58 },
+	{ .start = 0x13a5c, .end = 0x13a64 },
+	{ .start = 0x13a68, .end = 0x13a70 },
+	{ .start = 0x13acc, .end = 0x13ad8 },
+	{ .start = 0x13b0c, .end = 0x13b18 },
+	{ .start = 0x13b48, .end = 0x13b50 },
+	{ .start = 0x13b5c, .end = 0x13b60 },
+	{ .start = 0x13bf8, .end = 0x13bfc },
+	{ .start = 0x13c18, .end = 0x13c40 },
+	{ .start = 0x13c60, .end = 0x13c64 },
+	{ .start = 0x13c74, .end = 0x13c8c },
+	{ .start = 0x13d94, .end = 0x13da4 },
+	{ .start = 0x13dac, .end = 0x13db4 },
+	{ .start = 0x13db8, .end = 0x13dbc },
+	{ .start = 0x13df4, .end = 0x13e04 },
+	{ .start = 0x13e0c, .end = 0x13e10 },
+};
+
+static const struct tegra_fuse_info tegra264_fuse_info = {
+	.read = tegra30_fuse_read,
+	.size = 0x13eb8,
+};
+
+const struct tegra_fuse_soc tegra264_fuse_soc = {
+	.init = tegra30_fuse_init,
+	.info = &tegra264_fuse_info,
+	.keepouts = tegra264_fuse_keepouts,
+	.num_keepouts = ARRAY_SIZE(tegra264_fuse_keepouts),
+	.soc_attr_group = &tegra194_soc_attr_group,
+	.clk_suspend_on = false,
+};
+#endif
diff --git a/drivers/soc/tegra/fuse/fuse.h b/drivers/soc/tegra/fuse/fuse.h
index 9fee6ad6ad9e98..0fb97508db0a56 100644
--- a/drivers/soc/tegra/fuse/fuse.h
+++ b/drivers/soc/tegra/fuse/fuse.h
@@ -125,7 +125,8 @@ extern const struct tegra_fuse_soc tegra186_fuse_soc;
 
 #if IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
     IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC) || \
-    IS_ENABLED(CONFIG_ARCH_TEGRA_241_SOC)
+    IS_ENABLED(CONFIG_ARCH_TEGRA_241_SOC) || \
+    IS_ENABLED(CONFIG_ARCH_TEGRA_264_SOC)
 extern const struct attribute_group tegra194_soc_attr_group;
 #endif
 
@@ -141,4 +142,8 @@ extern const struct tegra_fuse_soc tegra234_fuse_soc;
 extern const struct tegra_fuse_soc tegra241_fuse_soc;
 #endif
 
+#ifdef CONFIG_ARCH_TEGRA_264_SOC
+extern const struct tegra_fuse_soc tegra264_fuse_soc;
+#endif
+
 #endif
diff --git a/drivers/soc/tegra/fuse/tegra-apbmisc.c b/drivers/soc/tegra/fuse/tegra-apbmisc.c
index 6af69ccaa295c5..f86c88fab16862 100644
--- a/drivers/soc/tegra/fuse/tegra-apbmisc.c
+++ b/drivers/soc/tegra/fuse/tegra-apbmisc.c
@@ -309,7 +309,7 @@ void tegra_acpi_init_apbmisc(void)
 
 	ret = acpi_dev_get_memory_resources(adev, &resource_list);
 	if (ret < 0) {
-		pr_err("failed to get APBMISC memory resources");
+		pr_err("failed to get APBMISC memory resources\n");
 		goto out_put_acpi_dev;
 	}
 
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index 41ed716d585731..4aece33707086d 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -11,6 +11,7 @@
 
 #define pr_fmt(fmt) "tegra-pmc: " fmt
 
+#include <linux/acpi.h>
 #include <linux/arm-smccc.h>
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
@@ -2980,6 +2981,18 @@ static void tegra_pmc_reset_suspend_mode(void *data)
 	pmc->suspend_mode = TEGRA_SUSPEND_NOT_READY;
 }
 
+static int tegra_pmc_acpi_probe(struct platform_device *pdev, struct tegra_pmc *pmc)
+{
+	pmc->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(pmc->base))
+		return PTR_ERR(pmc->base);
+
+	tegra_pmc_reset_sysfs_init(pmc);
+	platform_set_drvdata(pdev, pmc);
+
+	return 0;
+}
+
 static int tegra_pmc_probe(struct platform_device *pdev)
 {
 	struct tegra_pmc *pmc;
@@ -3004,6 +3017,9 @@ static int tegra_pmc_probe(struct platform_device *pdev)
 	mutex_init(&pmc->powergates_lock);
 	pmc->dev = &pdev->dev;
 
+	if (is_acpi_node(dev_fwnode(&pdev->dev)))
+		return tegra_pmc_acpi_probe(pdev, pmc);
+
 	err = tegra_pmc_parse_dt(pmc, pdev->dev.of_node);
 	if (err < 0)
 		return dev_err_probe(&pdev->dev, err, "failed to parse DT\n");
@@ -3151,6 +3167,7 @@ static int tegra_pmc_probe(struct platform_device *pdev)
 
 #if defined(CONFIG_ARM)
 	mutex_lock(&early_pmc->powergates_lock);
+	early_pmc->rate = pmc->rate;
 	iounmap(early_pmc->base);
 	early_pmc->base = pmc->base;
 	mutex_unlock(&early_pmc->powergates_lock);
@@ -4820,6 +4837,81 @@ static const struct tegra_pmc_soc tegra264_pmc_soc = {
 	.max_wake_vectors = 4,
 };
 
+static const char * const tegra410_reset_sources[] = {
+	"SYS_RESET_N",		/* 0x0 */
+	"CSDC_RTC_XTAL",
+	"VREFRO_POWER_BAD",
+	"FMON_32K",
+	"FMON_OSC",
+	"POD_RTC",
+	"POD_IO",
+	"POD_PLUS_IO_SPLL",
+	"POD_PLUS_IO_VMON",	/* 0x8 */
+	"POD_PLUS_SOC",
+	"VMON_PLUS_UV",
+	"VMON_PLUS_OV",
+	"FUSECRC_FAULT",
+	"OSC_FAULT",
+	"BPMP_BOOT_FAULT",
+	"SCPM_BPMP_CORE_CLK",
+	"SCPM_PSC_SE_CLK",	/* 0x10 */
+	"VMON_SOC_MIN",
+	"VMON_SOC_MAX",
+	"NVJTAG_SEL_MONITOR",
+	"L0_RST_REQ_N",
+	"NV_THERM_FAULT",
+	"PSC_SW",
+	"POD_C2C_LPI_0",
+	"POD_C2C_LPI_1",	/* 0x18 */
+	"BPMP_FMON",
+	"FMON_SPLL_OUT",
+	"L1_RST_REQ_N",
+	"OCP_RECOVERY",
+	"AO_WDT_POR",
+	"BPMP_WDT_POR",
+	"RAS_WDT_POR",
+	"TOP_0_WDT_POR",	/* 0x20 */
+	"TOP_1_WDT_POR",
+	"TOP_2_WDT_POR",
+	"PSC_WDT_POR",
+	"OOBHUB_WDT_POR",
+	"MSS_SEQ_WDT_POR",
+	"SW_MAIN",
+	"L0L1_RST_OUT_N",
+	"HSM",			/* 0x28 */
+	"CSITE_SW",
+	"AO_WDT_DBG",
+	"BPMP_WDT_DBG",
+	"RAS_WDT_DBG",
+	"TOP_0_WDT_DBG",
+	"TOP_1_WDT_DBG",
+	"TOP_2_WDT_DBG",
+	"PSC_WDT_DBG",		/* 0x30 */
+	"TSC_0_WDT_DBG",
+	"TSC_1_WDT_DBG",
+	"OOBHUB_WDT_DBG",
+	"MSS_SEQ_WDT_DBG",
+	"L2_RST_REQ_N",
+	"L2_RST_OUT_N",
+	"SC7"
+};
+
+static const struct tegra_pmc_regs tegra410_pmc_regs = {
+	.rst_status = 0x8,
+	.rst_source_shift = 0x2,
+	.rst_source_mask = 0xfc,
+	.rst_level_shift = 0x0,
+	.rst_level_mask = 0x3,
+};
+
+static const struct tegra_pmc_soc tegra410_pmc_soc = {
+	.regs = &tegra410_pmc_regs,
+	.reset_sources = tegra410_reset_sources,
+	.num_reset_sources = ARRAY_SIZE(tegra410_reset_sources),
+	.reset_levels = tegra186_reset_levels,
+	.num_reset_levels = ARRAY_SIZE(tegra186_reset_levels),
+};
+
 static const struct of_device_id tegra_pmc_match[] = {
 	{ .compatible = "nvidia,tegra264-pmc", .data = &tegra264_pmc_soc },
 	{ .compatible = "nvidia,tegra238-pmc", .data = &tegra238_pmc_soc },
@@ -4835,6 +4927,12 @@ static const struct of_device_id tegra_pmc_match[] = {
 	{ }
 };
 
+static const struct acpi_device_id tegra_pmc_acpi_match[] = {
+	{ .id = "NVDA2016", .driver_data = (kernel_ulong_t)&tegra410_pmc_soc },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, tegra_pmc_acpi_match);
+
 static void tegra_pmc_sync_state(struct device *dev)
 {
 	struct tegra_pmc *pmc = dev_get_drvdata(dev);
@@ -4886,6 +4984,7 @@ static struct platform_driver tegra_pmc_driver = {
 		.name = "tegra-pmc",
 		.suppress_bind_attrs = true,
 		.of_match_table = tegra_pmc_match,
+		.acpi_match_table = tegra_pmc_acpi_match,
 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_ARM)
 		.pm = &tegra_pmc_pm_ops,
 #endif
diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c
index 3e4041454f6943..6186454dff517d 100644
--- a/drivers/soc/ti/knav_qmss_queue.c
+++ b/drivers/soc/ti/knav_qmss_queue.c
@@ -509,7 +509,7 @@ static int knav_queue_flush(struct knav_queue *qh)
 /**
  * knav_queue_open()	- open a hardware queue
  * @name:		- name to give the queue handle
- * @id:			- desired queue number if any or specifes the type
+ * @id:			- desired queue number if any or specifies the type
  *			  of queue
  * @flags:		- the following flags are applicable to queues:
  *	KNAV_QUEUE_SHARED - allow the queue to be shared. Queues are
@@ -805,7 +805,7 @@ void *knav_pool_create(const char *name,
 	}
 
 	/* Region maintains a sorted (by region offset) list of pools
-	 * use the first free slot which is large enough to accomodate
+	 * use the first free slot which is large enough to accommodate
 	 * the request
 	 */
 	last_offset = 0;
diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c
index 54c796afb89efd..82224346eb2967 100644
--- a/drivers/soc/xilinx/zynqmp_power.c
+++ b/drivers/soc/xilinx/zynqmp_power.c
@@ -9,6 +9,7 @@
  *  Rajan Vaja <rajan.vaja@xilinx.com>
  */
 
+#include <linux/interrupt.h>
 #include <linux/mailbox_client.h>
 #include <linux/module.h>
 #include <linux/of.h>
diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c
index 12087cbd87b7bc..aeaae5a57c89d8 100644
--- a/drivers/soundwire/bus.c
+++ b/drivers/soundwire/bus.c
@@ -958,8 +958,31 @@ static void sdw_modify_slave_status(struct sdw_slave *slave,
 	mutex_unlock(&bus->bus_lock);
 }
 
+static enum sdw_clk_stop_mode sdw_get_clk_stop_mode(struct sdw_slave *slave)
+{
+	enum sdw_clk_stop_mode mode;
+
+	mode = slave->prop.clk_stop_mode1 ? SDW_CLK_STOP_MODE1 : SDW_CLK_STOP_MODE0;
+
+	mutex_lock(&slave->sdw_dev_lock);
+
+	/*
+	 * Query for clock stop mode if Slave implements
+	 * ops->get_clk_stop_mode, else read from property.
+	 */
+	if (slave->probed) {
+		struct sdw_driver *drv = drv_to_sdw_driver(slave->dev.driver);
+
+		if (drv->ops && drv->ops->get_clk_stop_mode)
+			mode = drv->ops->get_clk_stop_mode(slave);
+	}
+
+	mutex_unlock(&slave->sdw_dev_lock);
+
+	return mode;
+}
+
 static int sdw_slave_clk_stop_callback(struct sdw_slave *slave,
-				       enum sdw_clk_stop_mode mode,
 				       enum sdw_clk_stop_type type)
 {
 	int ret = 0;
@@ -971,7 +994,7 @@ static int sdw_slave_clk_stop_callback(struct sdw_slave *slave,
 		struct sdw_driver *drv = drv_to_sdw_driver(dev->driver);
 
 		if (drv->ops && drv->ops->clk_stop)
-			ret = drv->ops->clk_stop(slave, mode, type);
+			ret = drv->ops->clk_stop(slave, slave->clk_stop_mode, type);
 	}
 
 	mutex_unlock(&slave->sdw_dev_lock);
@@ -980,7 +1003,6 @@ static int sdw_slave_clk_stop_callback(struct sdw_slave *slave,
 }
 
 static int sdw_slave_clk_stop_prepare(struct sdw_slave *slave,
-				      enum sdw_clk_stop_mode mode,
 				      bool prepare)
 {
 	bool wake_en;
@@ -992,7 +1014,7 @@ static int sdw_slave_clk_stop_prepare(struct sdw_slave *slave,
 	if (prepare) {
 		val = SDW_SCP_SYSTEMCTRL_CLK_STP_PREP;
 
-		if (mode == SDW_CLK_STOP_MODE1)
+		if (slave->clk_stop_mode == SDW_CLK_STOP_MODE1)
 			val |= SDW_SCP_SYSTEMCTRL_CLK_STP_MODE1;
 
 		if (wake_en)
@@ -1080,9 +1102,9 @@ int sdw_bus_prep_clk_stop(struct sdw_bus *bus)
 		/* Identify if Slave(s) are available on Bus */
 		is_slave = true;
 
-		ret = sdw_slave_clk_stop_callback(slave,
-						  SDW_CLK_STOP_MODE0,
-						  SDW_CLK_PRE_PREPARE);
+		slave->clk_stop_mode = sdw_get_clk_stop_mode(slave);
+
+		ret = sdw_slave_clk_stop_callback(slave, SDW_CLK_PRE_PREPARE);
 		if (ret < 0 && ret != -ENODATA) {
 			dev_err(&slave->dev, "clock stop pre-prepare cb failed:%d\n", ret);
 			return ret;
@@ -1092,9 +1114,7 @@ int sdw_bus_prep_clk_stop(struct sdw_bus *bus)
 		if (!slave->prop.simple_clk_stop_capable) {
 			simple_clk_stop = false;
 
-			ret = sdw_slave_clk_stop_prepare(slave,
-							 SDW_CLK_STOP_MODE0,
-							 true);
+			ret = sdw_slave_clk_stop_prepare(slave, true);
 			if (ret < 0 && ret != -ENODATA) {
 				dev_err(&slave->dev, "clock stop prepare failed:%d\n", ret);
 				return ret;
@@ -1132,9 +1152,7 @@ int sdw_bus_prep_clk_stop(struct sdw_bus *bus)
 		    slave->status != SDW_SLAVE_ALERT)
 			continue;
 
-		ret = sdw_slave_clk_stop_callback(slave,
-						  SDW_CLK_STOP_MODE0,
-						  SDW_CLK_POST_PREPARE);
+		ret = sdw_slave_clk_stop_callback(slave, SDW_CLK_POST_PREPARE);
 
 		if (ret < 0 && ret != -ENODATA) {
 			dev_err(&slave->dev, "clock stop post-prepare cb failed:%d\n", ret);
@@ -1206,8 +1224,7 @@ int sdw_bus_exit_clk_stop(struct sdw_bus *bus)
 		/* Identify if Slave(s) are available on Bus */
 		is_slave = true;
 
-		ret = sdw_slave_clk_stop_callback(slave, SDW_CLK_STOP_MODE0,
-						  SDW_CLK_PRE_DEPREPARE);
+		ret = sdw_slave_clk_stop_callback(slave, SDW_CLK_PRE_DEPREPARE);
 		if (ret < 0)
 			dev_warn(&slave->dev, "clock stop pre-deprepare cb failed:%d\n", ret);
 
@@ -1215,9 +1232,7 @@ int sdw_bus_exit_clk_stop(struct sdw_bus *bus)
 		if (!slave->prop.simple_clk_stop_capable) {
 			simple_clk_stop = false;
 
-			ret = sdw_slave_clk_stop_prepare(slave, SDW_CLK_STOP_MODE0,
-							 false);
-
+			ret = sdw_slave_clk_stop_prepare(slave, false);
 			if (ret < 0)
 				dev_warn(&slave->dev, "clock stop deprepare failed:%d\n", ret);
 		}
@@ -1245,8 +1260,7 @@ int sdw_bus_exit_clk_stop(struct sdw_bus *bus)
 		    slave->status != SDW_SLAVE_ALERT)
 			continue;
 
-		ret = sdw_slave_clk_stop_callback(slave, SDW_CLK_STOP_MODE0,
-						  SDW_CLK_POST_DEPREPARE);
+		ret = sdw_slave_clk_stop_callback(slave, SDW_CLK_POST_DEPREPARE);
 		if (ret < 0)
 			dev_warn(&slave->dev, "clock stop post-deprepare cb failed:%d\n", ret);
 	}
diff --git a/drivers/soundwire/bus_type.c b/drivers/soundwire/bus_type.c
index d61a97c5b41ef7..fea15107cd9b9c 100644
--- a/drivers/soundwire/bus_type.c
+++ b/drivers/soundwire/bus_type.c
@@ -71,6 +71,13 @@ int sdw_slave_uevent(const struct device *dev, struct kobj_uevent_env *env)
 	return 0;
 }
 
+static void sdw_slave_ida_free(void *data)
+{
+	struct sdw_slave *slave = data;
+
+	ida_free(&slave->bus->slave_ida, slave->index);
+}
+
 static int sdw_bus_probe(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
@@ -104,15 +111,16 @@ static int sdw_bus_probe(struct device *dev)
 		return ret;
 	}
 	slave->index = ret;
+	ret = devm_add_action_or_reset(dev, sdw_slave_ida_free, slave);
+	if (ret)
+		return ret;
 
 	/* Create IRQ mapping now so the driver can get it in probe() */
 	sdw_irq_create_mapping(slave);
 
 	ret = drv->probe(slave, id);
-	if (ret) {
-		ida_free(&slave->bus->slave_ida, slave->index);
+	if (ret)
 		return ret;
-	}
 
 	mutex_lock(&slave->sdw_dev_lock);
 
@@ -170,8 +178,6 @@ static void sdw_bus_remove(struct device *dev)
 
 	if (drv->remove)
 		drv->remove(slave);
-
-	ida_free(&slave->bus->slave_ida, slave->index);
 }
 
 static void sdw_bus_shutdown(struct device *dev)
diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c
index e690237fe98192..a0a63b76793e29 100644
--- a/drivers/soundwire/cadence_master.c
+++ b/drivers/soundwire/cadence_master.c
@@ -1701,6 +1701,13 @@ int sdw_cdns_clock_stop(struct sdw_cdns *cdns, bool block_wake)
 		return 0;
 	}
 
+	/*
+	 * wait for any in-flight peripheral event handling to complete before stopping the clock.
+	 * No need to disable peripheral interrupts before canceling the work, as the peripheral
+	 * interrupts are already masked before the work is scheduled.
+	 */
+	cancel_work_sync(&cdns->work);
+
 	/*
 	 * Before entering clock stop we mask the Slave
 	 * interrupts. This helps avoid having to deal with e.g. a
diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c
index 768255dd12db6d..197b7540689208 100644
--- a/drivers/soundwire/dmi-quirks.c
+++ b/drivers/soundwire/dmi-quirks.c
@@ -200,6 +200,13 @@ static const struct dmi_system_id adr_remap_quirk_table[] = {
 		},
 		.driver_data = (void *)ghost_realtek,
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUS"),
+			DMI_MATCH(DMI_BOARD_NAME, "GX651AX"),
+		},
+		.driver_data = (void *)ghost_realtek,
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "ASUS"),
@@ -214,6 +221,13 @@ static const struct dmi_system_id adr_remap_quirk_table[] = {
 		},
 		.driver_data = (void *)ghost_realtek,
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "83QM"),
+		},
+		.driver_data = (void *)ghost_realtek,
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
@@ -228,6 +242,13 @@ static const struct dmi_system_id adr_remap_quirk_table[] = {
 		},
 		.driver_data = (void *)ghost_realtek,
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "83U9"),
+		},
+		.driver_data = (void *)ghost_realtek,
+	},
 	{}
 };
 
diff --git a/drivers/soundwire/intel.h b/drivers/soundwire/intel.h
index 86abc465260faf..7a2e7e73ad632a 100644
--- a/drivers/soundwire/intel.h
+++ b/drivers/soundwire/intel.h
@@ -188,10 +188,10 @@ static inline int sdw_intel_start_bus_after_clock_stop(struct sdw_intel *sdw)
 	return -ENOTSUPP;
 }
 
-static inline int sdw_intel_stop_bus(struct sdw_intel *sdw, bool clock_stop)
+static inline int sdw_intel_stop_bus(struct sdw_intel *sdw, bool clock_stop, bool wake_enable)
 {
 	if (SDW_INTEL_CHECK_OPS(sdw, stop_bus))
-		return SDW_INTEL_OPS(sdw, stop_bus)(sdw, clock_stop);
+		return SDW_INTEL_OPS(sdw, stop_bus)(sdw, clock_stop, wake_enable);
 	return -ENOTSUPP;
 }
 
@@ -261,7 +261,7 @@ int intel_start_bus(struct sdw_intel *sdw);
 int intel_start_bus_after_reset(struct sdw_intel *sdw);
 void intel_check_clock_stop(struct sdw_intel *sdw);
 int intel_start_bus_after_clock_stop(struct sdw_intel *sdw);
-int intel_stop_bus(struct sdw_intel *sdw, bool clock_stop);
+int intel_stop_bus(struct sdw_intel *sdw, bool clock_stop, bool wake_enable);
 
 /* common bank switch routines */
 int intel_pre_bank_switch(struct sdw_intel *sdw);
diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c
index a8407560bf4fda..901a71262094f9 100644
--- a/drivers/soundwire/intel_auxdevice.c
+++ b/drivers/soundwire/intel_auxdevice.c
@@ -74,6 +74,8 @@ static struct wake_capable_part wake_capable_list[] = {
 	{0x025d, 0x721},
 	{0x025d, 0x722},
 	{0x04b3, 0x9356},
+	{0x0102, 0x5572},
+	{0x0102, 0x5682},
 };
 
 static bool is_wake_capable(struct sdw_slave *slave)
@@ -159,7 +161,7 @@ static int sdw_master_read_intel_prop(struct sdw_bus *bus)
 	struct sdw_intel_prop *intel_prop;
 	struct fwnode_handle *link;
 	char name[32];
-	u32 quirk_mask;
+	u32 quirk_mask = 0;
 
 	/* Find master handle */
 	snprintf(name, sizeof(name),
@@ -670,7 +672,8 @@ static int __maybe_unused intel_suspend(struct device *dev)
 		return 0;
 	}
 
-	ret = sdw_intel_stop_bus(sdw, false);
+	/* Perform clock stop with proper mode and keep the bus unwakeable in system suspend. */
+	ret = sdw_intel_stop_bus(sdw, true, false);
 	if (ret < 0) {
 		dev_err(dev, "%s: cannot stop bus: %d\n", __func__, ret);
 		return ret;
@@ -696,14 +699,14 @@ static int __maybe_unused intel_suspend_runtime(struct device *dev)
 	clock_stop_quirks = sdw->link_res->clock_stop_quirks;
 
 	if (clock_stop_quirks & SDW_INTEL_CLK_STOP_TEARDOWN) {
-		ret = sdw_intel_stop_bus(sdw, false);
+		ret = sdw_intel_stop_bus(sdw, false, false);
 		if (ret < 0) {
 			dev_err(dev, "%s: cannot stop bus during teardown: %d\n",
 				__func__, ret);
 			return ret;
 		}
 	} else if (clock_stop_quirks & SDW_INTEL_CLK_STOP_BUS_RESET || !clock_stop_quirks) {
-		ret = sdw_intel_stop_bus(sdw, true);
+		ret = sdw_intel_stop_bus(sdw, true, true);
 		if (ret < 0) {
 			dev_err(dev, "%s: cannot stop bus during clock_stop: %d\n",
 				__func__, ret);
diff --git a/drivers/soundwire/intel_bus_common.c b/drivers/soundwire/intel_bus_common.c
index ad1f8ebdbfc9b1..fa35eb2fc76328 100644
--- a/drivers/soundwire/intel_bus_common.c
+++ b/drivers/soundwire/intel_bus_common.c
@@ -202,11 +202,11 @@ int intel_start_bus_after_clock_stop(struct sdw_intel *sdw)
 	return 0;
 }
 
-int intel_stop_bus(struct sdw_intel *sdw, bool clock_stop)
+int intel_stop_bus(struct sdw_intel *sdw, bool clock_stop, bool wake_enable)
 {
 	struct device *dev = sdw->cdns.dev;
 	struct sdw_cdns *cdns = &sdw->cdns;
-	bool wake_enable = false;
+	bool en = false;
 	int ret;
 
 	cancel_delayed_work_sync(&cdns->attach_dwork);
@@ -216,7 +216,7 @@ int intel_stop_bus(struct sdw_intel *sdw, bool clock_stop)
 		if (ret < 0)
 			dev_err(dev, "%s: cannot stop clock: %d\n", __func__, ret);
 		else
-			wake_enable = true;
+			en = wake_enable;
 	}
 
 	ret = sdw_cdns_enable_interrupt(cdns, false);
@@ -231,7 +231,7 @@ int intel_stop_bus(struct sdw_intel *sdw, bool clock_stop)
 		return ret;
 	}
 
-	sdw_intel_shim_wake(sdw, wake_enable);
+	sdw_intel_shim_wake(sdw, en);
 
 	return 0;
 }
diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c
index 55678a30cd4a1e..35ffffd541bd10 100644
--- a/drivers/soundwire/qcom.c
+++ b/drivers/soundwire/qcom.c
@@ -221,6 +221,9 @@ struct qcom_swrm_ctrl {
 	u32 slave_status;
 	u32 wr_fifo_depth;
 	bool clock_stop_not_supported;
+	/* Per-Slave SCP_ADDRPAGE1/2 shadow; -1 = unknown. */
+	s16 page1_cache[SDW_MAX_DEVICES + 1];
+	s16 page2_cache[SDW_MAX_DEVICES + 1];
 };
 
 struct qcom_swrm_data {
@@ -630,6 +633,10 @@ static void qcom_swrm_set_slave_dev_num(struct sdw_bus *bus,
 		mutex_lock(&bus->bus_lock);
 		set_bit(devnum, bus->assigned);
 		mutex_unlock(&bus->bus_lock);
+
+		/* Re-attach resets SCP_ADDRPAGE1/2 to defaults; invalidate. */
+		ctrl->page1_cache[devnum] = -1;
+		ctrl->page2_cache[devnum] = -1;
 	}
 }
 
@@ -976,17 +983,25 @@ static enum sdw_command_response qcom_swrm_xfer_msg(struct sdw_bus *bus,
 	int ret, i, len;
 
 	if (msg->page) {
-		ret = qcom_swrm_cmd_fifo_wr_cmd(ctrl, msg->addr_page1,
-						msg->dev_num,
-						SDW_SCP_ADDRPAGE1);
-		if (ret)
-			return ret;
+		if (ctrl->page1_cache[msg->dev_num] != msg->addr_page1) {
+			ret = qcom_swrm_cmd_fifo_wr_cmd(ctrl, msg->addr_page1,
+							msg->dev_num,
+							SDW_SCP_ADDRPAGE1);
+			if (ret)
+				return ret;
 
-		ret = qcom_swrm_cmd_fifo_wr_cmd(ctrl, msg->addr_page2,
-						msg->dev_num,
-						SDW_SCP_ADDRPAGE2);
-		if (ret)
-			return ret;
+			ctrl->page1_cache[msg->dev_num] = msg->addr_page1;
+		}
+
+		if (ctrl->page2_cache[msg->dev_num] != msg->addr_page2) {
+			ret = qcom_swrm_cmd_fifo_wr_cmd(ctrl, msg->addr_page2,
+							msg->dev_num,
+							SDW_SCP_ADDRPAGE2);
+			if (ret)
+				return ret;
+
+			ctrl->page2_cache[msg->dev_num] = msg->addr_page2;
+		}
 	}
 
 	if (msg->flags == SDW_MSG_FLAG_READ) {
@@ -1561,6 +1576,9 @@ static int qcom_swrm_probe(struct platform_device *pdev)
 	if (!ctrl)
 		return -ENOMEM;
 
+	memset(ctrl->page1_cache, 0xff, sizeof(ctrl->page1_cache));
+	memset(ctrl->page2_cache, 0xff, sizeof(ctrl->page2_cache));
+
 	data = of_device_get_match_data(dev);
 	ctrl->max_reg = data->max_reg;
 	ctrl->reg_layout = data->reg_layout;
diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c
index e0c49cbcb1bab4..a7d2938b031d0f 100644
--- a/drivers/soundwire/slave.c
+++ b/drivers/soundwire/slave.c
@@ -243,17 +243,27 @@ int sdw_of_find_slaves(struct sdw_bus *bus)
 		const char *compat = NULL;
 		struct sdw_slave_id id;
 		const __be32 *addr;
+		struct property *prop;
+		bool matched = false;
 
-		ret = of_property_read_string(node, "compatible", &compat);
-		if (ret)
-			continue;
-
-		ret = sscanf(compat, "sdw%01x%04hx%04hx%02hhx", &sdw_version,
-			     &id.mfg_id, &id.part_id, &id.class_id);
+		/*
+		 * Iterate the compatible list so DT nodes can carry a
+		 * vendor-variant compatible ahead of the SoundWire class-ID
+		 * compatible (e.g. "vendor,part", "sdwVMMMPPPPXX").
+		 */
+		of_property_for_each_string(node, "compatible", prop, compat) {
+			ret = sscanf(compat, "sdw%01x%04hx%04hx%02hhx",
+				     &sdw_version, &id.mfg_id, &id.part_id,
+				     &id.class_id);
+			if (ret == 4) {
+				matched = true;
+				break;
+			}
+		}
 
-		if (ret != 4) {
-			dev_err(dev, "Invalid compatible string found %s\n",
-				compat);
+		if (!matched) {
+			dev_err(dev, "%pOFn: no SoundWire class-ID compatible\n",
+				node);
 			continue;
 		}
 
diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c
index f106a877d7b2ee..0b00fcdcab2d6d 100644
--- a/drivers/soundwire/stream.c
+++ b/drivers/soundwire/stream.c
@@ -1571,7 +1571,7 @@ int sdw_prepare_stream(struct sdw_stream_runtime *stream)
 	if (stream->state != SDW_STREAM_CONFIGURED &&
 	    stream->state != SDW_STREAM_DEPREPARED &&
 	    stream->state != SDW_STREAM_DISABLED) {
-		pr_err("%s: %s: inconsistent state state %d\n",
+		pr_err("%s: %s: inconsistent state %d\n",
 		       __func__, stream->name, stream->state);
 		ret = -EINVAL;
 		goto state_err;
@@ -1655,7 +1655,7 @@ int sdw_enable_stream(struct sdw_stream_runtime *stream)
 
 	if (stream->state != SDW_STREAM_PREPARED &&
 	    stream->state != SDW_STREAM_DISABLED) {
-		pr_err("%s: %s: inconsistent state state %d\n",
+		pr_err("%s: %s: inconsistent state %d\n",
 		       __func__, stream->name, stream->state);
 		ret = -EINVAL;
 		goto state_err;
@@ -1742,7 +1742,7 @@ int sdw_disable_stream(struct sdw_stream_runtime *stream)
 	}
 
 	if (stream->state != SDW_STREAM_ENABLED) {
-		pr_err("%s: %s: inconsistent state state %d\n",
+		pr_err("%s: %s: inconsistent state %d\n",
 		       __func__, stream->name, stream->state);
 		ret = -EINVAL;
 		goto state_err;
@@ -1848,7 +1848,7 @@ int sdw_deprepare_stream(struct sdw_stream_runtime *stream)
 
 	if (stream->state != SDW_STREAM_PREPARED &&
 	    stream->state != SDW_STREAM_DISABLED) {
-		pr_err("%s: %s: inconsistent state state %d\n",
+		pr_err("%s: %s: inconsistent state %d\n",
 		       __func__, stream->name, stream->state);
 		ret = -EINVAL;
 		goto state_err;
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index a0f37e2df71658..357cebef991737 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -63,7 +63,6 @@ comment "SPI Master Controller Drivers"
 config SPI_AIROHA_SNFI
 	tristate "Airoha SPI NAND Flash Interface"
 	depends on ARCH_AIROHA || COMPILE_TEST
-	depends on SPI_MASTER
 	select REGMAP_MMIO
 	help
 	  This enables support for SPI-NAND mode on the Airoha NAND
@@ -92,6 +91,14 @@ config SPI_ALTERA_DFL
 	  Altera SPI master controller.  The SPI master is connected
 	  to a SPI slave to Avalon bridge in a Intel MAX BMC.
 
+config SPI_AMD
+	tristate "AMD SPI controller"
+	depends on PCI
+	depends on X86 || COMPILE_TEST
+	depends on SPI_MEM
+	help
+	  Enables SPI controller driver for AMD SoC.
+
 config SPI_AMLOGIC_SPIFC_A1
 	tristate "Amlogic A1 SPIFC controller"
 	depends on ARCH_MESON || COMPILE_TEST
@@ -136,6 +143,13 @@ config SPI_AR934X
 	  This enables support for the SPI controller present on the
 	  Qualcomm Atheros AR934X/QCA95XX SoCs.
 
+config SPI_ARMADA_3700
+	tristate "Marvell Armada 3700 SPI Controller"
+	depends on (ARCH_MVEBU && OF) || COMPILE_TEST
+	help
+	  This enables support for the SPI controller present on the
+	  Marvell Armada 3700 SoCs.
+
 config SPI_ATCSPI200
 	tristate "Andes ATCSPI200 SPI controller"
 	depends on ARCH_ANDES || COMPILE_TEST
@@ -153,13 +167,6 @@ config SPI_ATH79
 	  This enables support for the SPI controller present on the
 	  Atheros AR71XX/AR724X/AR913X SoCs.
 
-config SPI_ARMADA_3700
-	tristate "Marvell Armada 3700 SPI Controller"
-	depends on (ARCH_MVEBU && OF) || COMPILE_TEST
-	help
-	  This enables support for the SPI controller present on the
-	  Marvell Armada 3700 SoCs.
-
 config SPI_ASPEED_SMC
 	tristate "Aspeed flash controllers in SPI mode"
 	depends on ARCH_ASPEED || COMPILE_TEST
@@ -333,7 +340,7 @@ config SPI_CADENCE_XSPI
 
 config SPI_CH341
 	tristate "CH341 USB2SPI adapter"
-	depends on SPI_MASTER && USB
+	depends on USB
 	help
 	  Enables the SPI controller on the CH341a USB to serial chip
 
@@ -388,14 +395,14 @@ config SPI_DW_MMIO
 endif
 
 config SPI_DLN2
-       tristate "Diolan DLN-2 USB SPI adapter"
-       depends on MFD_DLN2
-       help
-	 If you say yes to this option, support will be included for Diolan
-	 DLN2, a USB to SPI interface.
+	tristate "Diolan DLN-2 USB SPI adapter"
+	depends on MFD_DLN2
+	help
+	  If you say yes to this option, support will be included for Diolan
+	  DLN2, a USB to SPI interface.
 
-	 This driver can also be built as a module.  If so, the module
-	 will be called spi-dln2.
+	  This driver can also be built as a module.  If so, the module
+	  will be called spi-dln2.
 
 config SPI_EP93XX
 	tristate "Cirrus Logic EP93xx SPI controller"
@@ -420,6 +427,22 @@ config SPI_FSI
 	  This enables support for the driver for FSI bus attached SPI
 	  controllers.
 
+config SPI_FSL_DSPI
+	tristate "Freescale DSPI controller"
+	select REGMAP_MMIO
+	depends on ARCH_MXC || ARCH_NXP || M5441x || COMPILE_TEST
+	help
+	  This enables support for the Freescale DSPI controller in master
+	  mode. S32, VF610, LS1021A and ColdFire platforms uses the controller.
+
+config SPI_FSL_ESPI
+	tristate "Freescale eSPI controller"
+	depends on FSL_SOC
+	help
+	  This enables using the Freescale eSPI controllers in master mode.
+	  From MPC8536, 85xx platform uses the controller, and all P10xx,
+	  P20xx, P30xx,P40xx, P50xx uses this controller.
+
 config SPI_FSL_LPSPI
 	tristate "Freescale i.MX LPSPI controller"
 	depends on ARCH_MXC || COMPILE_TEST
@@ -438,6 +461,26 @@ config SPI_FSL_QUADSPI
 	  This controller does not support generic SPI messages. It only
 	  supports the high-level SPI memory interface.
 
+config SPI_FSL_LIB
+	tristate
+	depends on OF
+
+config SPI_FSL_CPM
+	tristate
+	depends on FSL_SOC
+
+config SPI_FSL_SPI
+	tristate "Freescale SPI controller and Aeroflex Gaisler GRLIB SPI controller"
+	depends on OF
+	select SPI_FSL_LIB
+	select SPI_FSL_CPM if FSL_SOC
+	help
+	  This enables using the Freescale SPI controllers in master mode.
+	  MPC83xx platform uses the controller in cpu mode or CPM/QE mode.
+	  MPC8569 uses the controller in QE mode, MPC8610 in cpu mode.
+	  This also enables using the Aeroflex Gaisler GRLIB SPI controller in
+	  master mode.
+
 config SPI_GXP
 	tristate "GXP SPI driver"
 	depends on ARCH_HPE || COMPILE_TEST
@@ -463,27 +506,6 @@ config SPI_HISI_SFC_V3XX
 	  This enables support for HiSilicon v3xx SPI NOR flash controller
 	  found in hi16xx chipsets.
 
-config SPI_NXP_FLEXSPI
-	tristate "NXP Flex SPI controller"
-	depends on ARCH_LAYERSCAPE || ARCH_MXC || COMPILE_TEST
-	depends on HAS_IOMEM
-	help
-	  This enables support for the Flex SPI controller in master mode.
-	  Up to four slave devices can be connected on two buses with two
-	  chipselects each.
-	  This controller does not support generic SPI messages and only
-	  supports the high-level SPI memory interface.
-
-config SPI_NXP_XSPI
-        tristate "NXP xSPI controller"
-	depends on ARCH_MXC || COMPILE_TEST
-	depends on HAS_IOMEM
-	help
-	  This enables support for the xSPI controller. Up to two devices
-	  can be connected to one host.
-	  This controller does not support generic SPI messages and only
-	  supports the high-level SPI memory interface.
-
 config SPI_GPIO
 	tristate "GPIO-based bitbanging SPI Master"
 	depends on GPIOLIB || COMPILE_TEST
@@ -581,6 +603,25 @@ config SPI_KSPI2
 	  This driver can also be built as a module. If so, the module
 	  will be called spi-kspi2.
 
+config SPI_LANTIQ_SSC
+	tristate "Lantiq SSC SPI controller"
+	depends on LANTIQ || X86 || COMPILE_TEST
+	help
+	  This driver supports the Lantiq SSC SPI controller in master
+	  mode. This controller is found on Intel (former Lantiq) SoCs like
+	  the Danube, Falcon, xRX200, xRX300, Lightning Mountain.
+
+config SPI_LJCA
+	tristate "Intel La Jolla Cove Adapter SPI support"
+	depends on USB_LJCA
+	default USB_LJCA
+	help
+	  Select this option to enable SPI driver for the Intel
+	  La Jolla Cove Adapter (LJCA) board.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called spi-ljca.
+
 config SPI_LM70_LLP
 	tristate "Parallel port adapter for LM70 eval board (DEVELOPMENT)"
 	depends on PARPORT
@@ -626,74 +667,6 @@ config SPI_LP8841_RTC
 	  Say N here unless you plan to run the kernel on an ICP DAS
 	  LP-8x4x industrial computer.
 
-config SPI_MPC52xx
-	tristate "Freescale MPC52xx SPI (non-PSC) controller support"
-	depends on PPC_MPC52xx
-	help
-	  This drivers supports the MPC52xx SPI controller in master SPI
-	  mode.
-
-config SPI_MPC52xx_PSC
-	tristate "Freescale MPC52xx PSC SPI controller"
-	depends on PPC_MPC52xx
-	help
-	  This enables using the Freescale MPC52xx Programmable Serial
-	  Controller in master SPI mode.
-
-config SPI_MPC512x_PSC
-	tristate "Freescale MPC512x PSC SPI controller"
-	depends on PPC_MPC512x
-	help
-	  This enables using the Freescale MPC5121 Programmable Serial
-	  Controller in SPI master mode.
-
-config SPI_FSL_LIB
-	tristate
-	depends on OF
-
-config SPI_FSL_CPM
-	tristate
-	depends on FSL_SOC
-
-config SPI_FSL_SPI
-	tristate "Freescale SPI controller and Aeroflex Gaisler GRLIB SPI controller"
-	depends on OF
-	select SPI_FSL_LIB
-	select SPI_FSL_CPM if FSL_SOC
-	help
-	  This enables using the Freescale SPI controllers in master mode.
-	  MPC83xx platform uses the controller in cpu mode or CPM/QE mode.
-	  MPC8569 uses the controller in QE mode, MPC8610 in cpu mode.
-	  This also enables using the Aeroflex Gaisler GRLIB SPI controller in
-	  master mode.
-
-config SPI_FSL_DSPI
-	tristate "Freescale DSPI controller"
-	select REGMAP_MMIO
-	depends on ARCH_MXC || ARCH_NXP || M5441x || COMPILE_TEST
-	help
-	  This enables support for the Freescale DSPI controller in master
-	  mode. S32, VF610, LS1021A and ColdFire platforms uses the controller.
-
-config SPI_FSL_ESPI
-	tristate "Freescale eSPI controller"
-	depends on FSL_SOC
-	help
-	  This enables using the Freescale eSPI controllers in master mode.
-	  From MPC8536, 85xx platform uses the controller, and all P10xx,
-	  P20xx, P30xx,P40xx, P50xx uses this controller.
-
-config SPI_LJCA
-	tristate "Intel La Jolla Cove Adapter SPI support"
-	depends on USB_LJCA
-	default USB_LJCA
-	help
-	  Select this option to enable SPI driver for the Intel
-	  La Jolla Cove Adapter (LJCA) board.
-
-	  This driver can also be built as a module. If so, the module
-	  will be called spi-ljca.
-
 config SPI_MA35D1_QSPI
 	tristate "Nuvoton MA35D1 QSPI controller"
 	depends on ARCH_MA35 || COMPILE_TEST
@@ -722,7 +695,6 @@ config SPI_MESON_SPIFC
 
 config SPI_MICROCHIP_CORE_QSPI
 	tristate "Microchip FPGA QSPI controllers"
-	depends on SPI_MASTER
 	help
 	  This enables the QSPI driver for Microchip FPGA QSPI controllers.
 	  Say Y or M here if you want to use the QSPI controllers on
@@ -731,13 +703,33 @@ config SPI_MICROCHIP_CORE_QSPI
 
 config SPI_MICROCHIP_CORE_SPI
 	tristate "Microchip FPGA CoreSPI controller"
-	depends on SPI_MASTER
 	help
 	  This enables the SPI driver for Microchip FPGA CoreSPI controller.
 	  Say Y or M here if you want to use the "soft" controllers on
 	  PolarFire SoC.
 	  If built as a module, it will be called spi-microchip-core-spi.
 
+config SPI_MPC52xx
+	tristate "Freescale MPC52xx SPI (non-PSC) controller support"
+	depends on PPC_MPC52xx
+	help
+	  This drivers supports the MPC52xx SPI controller in master SPI
+	  mode.
+
+config SPI_MPC52xx_PSC
+	tristate "Freescale MPC52xx PSC SPI controller"
+	depends on PPC_MPC52xx
+	help
+	  This enables using the Freescale MPC52xx Programmable Serial
+	  Controller in master SPI mode.
+
+config SPI_MPC512x_PSC
+	tristate "Freescale MPC512x PSC SPI controller"
+	depends on PPC_MPC512x
+	help
+	  This enables using the Freescale MPC5121 Programmable Serial
+	  Controller in SPI master mode.
+
 config SPI_MT65XX
 	tristate "MediaTek SPI controller"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
@@ -773,6 +765,19 @@ config SPI_MTK_SNFI
 	  is implemented as a SPI-MEM controller with pipelined ECC
 	  capability.
 
+config SPI_MXIC
+	tristate "Macronix MX25F0A SPI controller"
+	imply MTD_NAND_ECC_MXIC
+	help
+	  This selects the Macronix MX25F0A SPI controller driver.
+
+config SPI_MXS
+	tristate "Freescale MXS SPI controller"
+	depends on ARCH_MXS
+	select STMP_DEVICE
+	help
+	  SPI driver for Freescale MXS devices.
+
 config SPI_WPCM_FIU
 	tristate "Nuvoton WPCM450 Flash Interface Unit"
 	depends on ARCH_NPCM || COMPILE_TEST
@@ -801,13 +806,26 @@ config SPI_NPCM_PSPI
 	  This driver provides support for Nuvoton NPCM BMC
 	  Peripheral SPI controller in master mode.
 
-config SPI_LANTIQ_SSC
-	tristate "Lantiq SSC SPI controller"
-	depends on LANTIQ || X86 || COMPILE_TEST
+config SPI_NXP_FLEXSPI
+	tristate "NXP Flex SPI controller"
+	depends on ARCH_LAYERSCAPE || ARCH_MXC || COMPILE_TEST
+	depends on HAS_IOMEM
 	help
-	  This driver supports the Lantiq SSC SPI controller in master
-	  mode. This controller is found on Intel (former Lantiq) SoCs like
-	  the Danube, Falcon, xRX200, xRX300, Lightning Mountain.
+	  This enables support for the Flex SPI controller in master mode.
+	  Up to four slave devices can be connected on two buses with two
+	  chipselects each.
+	  This controller does not support generic SPI messages and only
+	  supports the high-level SPI memory interface.
+
+config SPI_NXP_XSPI
+	tristate "NXP xSPI controller"
+	depends on ARCH_MXC || COMPILE_TEST
+	depends on HAS_IOMEM
+	help
+	  This enables support for the xSPI controller. Up to two devices
+	  can be connected to one host.
+	  This controller does not support generic SPI messages and only
+	  supports the high-level SPI memory interface.
 
 config SPI_OC_TINY
 	tristate "OpenCores tiny SPI"
@@ -838,14 +856,6 @@ config SPI_OMAP24XX
 	  SPI master controller for OMAP24XX and later Multichannel SPI
 	  (McSPI) modules.
 
-config SPI_TI_QSPI
-	tristate "DRA7xxx QSPI controller support"
-	depends on ARCH_OMAP2PLUS || COMPILE_TEST
-	help
-	  QSPI master controller for DRA7xxx used for flash devices.
-	  This device supports single, dual and quad read support, while
-	  it only supports single write mode.
-
 config SPI_ORION
 	tristate "Orion SPI master"
 	depends on PLAT_ORION || ARCH_MVEBU || COMPILE_TEST
@@ -887,7 +897,6 @@ config SPI_PL022
 
 config SPI_POLARFIRE_SOC
 	tristate "Microchip FPGA SPI controllers"
-	depends on SPI_MASTER
 	depends on ARCH_MICROCHIP || COMPILE_TEST
 	help
 	  This enables the SPI driver for Microchip FPGA SPI controllers.
@@ -950,7 +959,7 @@ config SPI_ROCKCHIP_SFC
 
 config SPI_RB4XX
 	tristate "Mikrotik RB4XX SPI master"
-	depends on SPI_MASTER && (ATH79 || COMPILE_TEST)
+	depends on ATH79 || COMPILE_TEST
 	depends on OF
 	help
 	  SPI controller driver for the Mikrotik RB4xx series boards.
@@ -1035,12 +1044,14 @@ config SPI_SC18IS602
 	help
 	  SPI driver for NXP SC18IS602/602B/603 I2C to SPI bridge.
 
-config SPI_SH_MSIOF
-	tristate "SuperH MSIOF SPI controller"
-	depends on HAVE_CLK
-	depends on ARCH_SHMOBILE || ARCH_RENESAS || COMPILE_TEST
+config SPI_SG2044_NOR
+	tristate "SG2044 SPI NOR Controller"
+	depends on ARCH_SOPHGO || COMPILE_TEST
 	help
-	  SPI driver for SuperH and SH Mobile MSIOF blocks.
+	  This enables support for the SG2044 SPI NOR controller,
+	  which supports Dual/Quad read and write operations while
+	  also supporting 3Byte address devices and 4Byte address
+	  devices.
 
 config SPI_SH
 	tristate "SuperH SPI controller"
@@ -1048,6 +1059,19 @@ config SPI_SH
 	help
 	  SPI driver for SuperH SPI blocks.
 
+config SPI_SH_HSPI
+	tristate "SuperH HSPI controller"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	help
+	  SPI driver for SuperH HSPI blocks.
+
+config SPI_SH_MSIOF
+	tristate "SuperH MSIOF SPI controller"
+	depends on HAVE_CLK
+	depends on ARCH_SHMOBILE || ARCH_RENESAS || COMPILE_TEST
+	help
+	  SPI driver for SuperH and SH Mobile MSIOF blocks.
+
 config SPI_SH_SCI
 	tristate "SuperH SCI SPI controller"
 	depends on SUPERH
@@ -1055,28 +1079,12 @@ config SPI_SH_SCI
 	help
 	  SPI driver for SuperH SCI blocks.
 
-config SPI_SH_HSPI
-	tristate "SuperH HSPI controller"
-	depends on ARCH_RENESAS || COMPILE_TEST
-	help
-	  SPI driver for SuperH HSPI blocks.
-
 config SPI_SIFIVE
 	tristate "SiFive SPI controller"
 	depends on HAS_IOMEM
 	help
 	  This exposes the SPI controller IP from SiFive.
 
-config SPI_SLAVE_MT27XX
-	tristate "MediaTek SPI slave device"
-	depends on ARCH_MEDIATEK || COMPILE_TEST
-	depends on SPI_SLAVE
-	help
-	  This selects the MediaTek(R) SPI slave device driver.
-	  If you want to use MediaTek(R) SPI slave interface,
-	  say Y or M here.If you are not sure, say N.
-	  SPI slave drivers for Mediatek MT27XX series ARM SoCs.
-
 config SPI_SN_F_OSPI
 	tristate "Socionext F_OSPI SPI flash controller"
 	depends on OF && HAS_IOMEM
@@ -1086,15 +1094,6 @@ config SPI_SN_F_OSPI
 	  for connecting an SPI Flash memory over up to 8-bit wide bus.
 	  It supports indirect access mode only.
 
-config SPI_SG2044_NOR
-	tristate "SG2044 SPI NOR Controller"
-	depends on ARCH_SOPHGO || COMPILE_TEST
-	help
-	  This enables support for the SG2044 SPI NOR controller,
-	  which supports Dual/Quad read and write operations while
-	  also supporting 3Byte address devices and 4Byte address
-	  devices.
-
 config SPI_SPACEMIT_K1
 	tristate "K1 SPI Controller"
 	depends on ARCH_SPACEMIT || COMPILE_TEST
@@ -1189,20 +1188,6 @@ config SPI_SYNQUACER
 
 	  It also supports the new dual-bit and quad-bit SPI protocol.
 
-config SPI_MXIC
-	tristate "Macronix MX25F0A SPI controller"
-	depends on SPI_MASTER
-	imply MTD_NAND_ECC_MXIC
-	help
-	  This selects the Macronix MX25F0A SPI controller driver.
-
-config SPI_MXS
-	tristate "Freescale MXS SPI controller"
-	depends on ARCH_MXS
-	select STMP_DEVICE
-	help
-	  SPI driver for Freescale MXS devices.
-
 config SPI_TEGRA210_QUAD
 	tristate "NVIDIA Tegra QSPI Controller"
 	depends on ARCH_TEGRA || COMPILE_TEST
@@ -1244,6 +1229,23 @@ config SPI_THUNDERX
 	  SPI host driver for the hardware found on Cavium ThunderX
 	  SOCs.
 
+config SPI_TI_QSPI
+	tristate "DRA7xxx QSPI controller support"
+	depends on ARCH_OMAP2PLUS || COMPILE_TEST
+	help
+	  QSPI master controller for DRA7xxx used for flash devices.
+	  This device supports single, dual and quad read support, while
+	  it only supports single write mode.
+
+config SPI_TLE62X0
+	tristate "Infineon TLE62X0 (for power switching)"
+	depends on SYSFS
+	help
+	  SPI driver for Infineon TLE62X0 series line driver chips,
+	  such as the TLE6220, TLE6230 and TLE6240.  This provides a
+	  sysfs interface, with each line presented as a kind of GPIO
+	  exposing both switch control and diagnostic feedback.
+
 config SPI_TOPCLIFF_PCH
 	tristate "Intel EG20T PCH/LAPIS Semicon IOH(ML7213/ML7223/ML7831) SPI"
 	depends on PCI && (X86_32 || MIPS || COMPILE_TEST)
@@ -1270,7 +1272,7 @@ config SPI_UNIPHIER
 
 config SPI_VIRTIO
 	tristate "Virtio SPI Controller"
-	depends on SPI_MASTER && VIRTIO
+	depends on VIRTIO
 	help
 	  If you say yes to this option, support will be included for the virtio
 	  SPI controller driver. The hardware can be emulated by any device model
@@ -1336,14 +1338,6 @@ config SPI_ZYNQMP_GQSPI
 	  Enables Xilinx GQSPI controller driver for Zynq UltraScale+ MPSoC.
 	  This controller only supports SPI memory interface.
 
-config SPI_AMD
-	tristate "AMD SPI controller"
-	depends on PCI
-	depends on SPI_MASTER || X86 || COMPILE_TEST
-	depends on SPI_MEM
-	help
-	  Enables SPI controller driver for AMD SoC.
-
 #
 # Add new SPI master controllers in alphabetical order above this line
 #
@@ -1392,15 +1386,6 @@ config SPI_LOOPBACK_TEST
 	  primarily used for development of spi_master drivers
 	  and to detect regressions
 
-config SPI_TLE62X0
-	tristate "Infineon TLE62X0 (for power switching)"
-	depends on SYSFS
-	help
-	  SPI driver for Infineon TLE62X0 series line driver chips,
-	  such as the TLE6220, TLE6230 and TLE6240.  This provides a
-	  sysfs interface, with each line presented as a kind of GPIO
-	  exposing both switch control and diagnostic feedback.
-
 #
 # Add new SPI protocol masters in alphabetical order above this line
 #
@@ -1431,6 +1416,15 @@ config SPI_SLAVE_SYSTEM_CONTROL
 	  SPI slave handler to allow remote control of system reboot, power
 	  off, halt, and suspend.
 
+config SPI_SLAVE_MT27XX
+	tristate "MediaTek SPI slave device"
+	depends on ARCH_MEDIATEK || COMPILE_TEST
+	help
+	  This selects the MediaTek(R) SPI slave device driver.
+	  If you want to use MediaTek(R) SPI slave interface,
+	  say Y or M here.If you are not sure, say N.
+	  SPI slave drivers for Mediatek MT27XX series ARM SoCs.
+
 endif # SPI_SLAVE
 
 config SPI_DYNAMIC
diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c
index 9049a87e9d0f2d..1b00f34e9e931a 100644
--- a/drivers/spi/spi-amlogic-spisg.c
+++ b/drivers/spi/spi-amlogic-spisg.c
@@ -37,6 +37,12 @@
 #define CFG_HW_POS			BIT(6)
 /* start on vsync falling */
 #define CFG_HW_NEG			BIT(7)
+#define CFG_WORD_GAP			GENMASK(9, 8)
+#define CFG_MO_IDLE_OUTPUT		GENMASK(11, 10)
+/* cs hold time in pclk */
+#define CFG_CS_HOLD			GENMASK(26, 12)
+/* high 4 bits of cs setup time in sclk */
+#define CFG_CS_SETUP_EXTEND		GENMASK(30, 27)
 
 #define SPISG_REG_CFG_START		0x08
 #define CFG_BLOCK_NUM			GENMASK(19, 0)
@@ -95,7 +101,7 @@
 
 #define SPISG_MAX_REG			0x40
 
-#define SPISG_BLOCK_MAX			0x100000
+#define SPISG_BLOCK_MAX			0xFFFFF
 
 #define SPISG_OP_MODE_WRITE_CMD		0
 #define SPISG_OP_MODE_READ_STS		1
@@ -143,6 +149,13 @@ struct spisg_descriptor_extra {
 	int				rx_ccsg_len;
 };
 
+struct aml_spisg_data {
+	bool				mo_idle_output_ctrl;
+	bool				word_gap_ctrl;
+	bool				cs_hold_ctrl;
+	bool				cs_setup_extend_ctrl;
+};
+
 struct spisg_device {
 	struct spi_controller		*controller;
 	struct platform_device		*pdev;
@@ -152,6 +165,7 @@ struct spisg_device {
 	struct clk			*sclk;
 	struct clk_div_table		*tbl;
 	struct completion		completion;
+	const struct aml_spisg_data	*data;
 	u32				status;
 	u32				speed_hz;
 	u32				effective_speed_hz;
@@ -175,7 +189,7 @@ static int spi_delay_to_sclk(u32 slck_speed_hz, struct spi_delay *delay)
 	if (ns < 0)
 		return 0;
 
-	return DIV_ROUND_UP_ULL(slck_speed_hz * ns, NSEC_PER_SEC);
+	return DIV_ROUND_UP_ULL((u64)slck_speed_hz * ns, NSEC_PER_SEC);
 }
 
 static inline u32 aml_spisg_sem_down_read(struct spisg_device *spisg)
@@ -424,7 +438,8 @@ static void aml_spisg_setup_null_desc(struct spisg_device *spisg,
 static void aml_spisg_pending(struct spisg_device *spisg,
 			      dma_addr_t desc_paddr,
 			      bool trig,
-			      bool irq_en)
+			      bool irq_en,
+			      u32 delay)
 {
 	u32 desc_l, desc_h, cfg_spi, irq_enable;
 
@@ -437,6 +452,13 @@ static void aml_spisg_pending(struct spisg_device *spisg,
 #endif
 
 	cfg_spi = spisg->cfg_spi;
+
+	if (spisg->data && spisg->data->word_gap_ctrl) {
+		if (delay > 3)
+			delay = 3;
+		cfg_spi |=  FIELD_PREP(CFG_WORD_GAP, delay);
+	}
+
 	if (trig)
 		cfg_spi |= CFG_HW_POS;
 	else
@@ -483,13 +505,16 @@ static int aml_spisg_transfer_one_message(struct spi_controller *ctlr,
 {
 	struct spisg_device *spisg = spi_controller_get_devdata(ctlr);
 	struct device *dev = &spisg->pdev->dev;
+	const struct aml_spisg_data *data = spisg->data;
 	unsigned long long ms = 0;
 	struct spi_transfer *xfer;
 	struct spisg_descriptor *descs, *desc;
 	struct spisg_descriptor_extra *exdescs, *exdesc;
 	dma_addr_t descs_paddr;
 	int desc_num = 1, descs_len;
+	bool last_xfer_keep_ss = false;
 	u32 cs_hold_in_sclk = 0;
+	u32 val, delay = 0;
 	int ret = -EIO;
 
 	if (!aml_spisg_sem_down_read(spisg)) {
@@ -523,15 +548,27 @@ static int aml_spisg_transfer_one_message(struct spi_controller *ctlr,
 			goto end;
 		}
 
-		/* calculate cs-setup delay with the first xfer speed */
-		if (list_is_first(&xfer->transfer_list, &msg->transfers))
-			desc->cfg_bus |= FIELD_PREP(CFG_CS_SETUP,
-				spi_delay_to_sclk(xfer->effective_speed_hz, &msg->spi->cs_setup));
+		/* calculate cs-setup delay with the first xfer speed  and word dealy*/
+		if (list_is_first(&xfer->transfer_list, &msg->transfers)) {
+			val = spi_delay_to_sclk(xfer->effective_speed_hz, &msg->spi->cs_setup);
+			if (data && data->cs_setup_extend_ctrl) {
+				val = min_t(u32, 0xFF, val);
+				desc->cfg_bus |= FIELD_PREP(CFG_CS_SETUP, val & 0xF);
+				FIELD_MODIFY(CFG_CS_SETUP_EXTEND, &spisg->cfg_spi, val >> 4);
+			} else {
+				val = min_t(u32, 0xF, val);
+				desc->cfg_bus |= FIELD_PREP(CFG_CS_SETUP, val);
+			}
+
+			delay = spi_delay_to_sclk(xfer->effective_speed_hz, &msg->spi->word_delay);
+		}
 
 		/* calculate cs-hold delay with the last xfer speed */
-		if (list_is_last(&xfer->transfer_list, &msg->transfers))
+		if (list_is_last(&xfer->transfer_list, &msg->transfers)) {
 			cs_hold_in_sclk =
 				spi_delay_to_sclk(xfer->effective_speed_hz, &msg->spi->cs_hold);
+			last_xfer_keep_ss = xfer->cs_change;
+		}
 
 		desc++;
 		exdesc++;
@@ -539,13 +576,23 @@ static int aml_spisg_transfer_one_message(struct spi_controller *ctlr,
 				       xfer->effective_speed_hz);
 	}
 
-	if (cs_hold_in_sclk)
+	if (data && data->cs_hold_ctrl) {
+		cs_hold_in_sclk = cs_hold_in_sclk ? : 1;
+		val = cs_hold_in_sclk * (FIELD_GET(CFG_CLK_DIV, spisg->cfg_bus) + 1);
+		val = min_t(u32, 0x7FFF, val);
+		FIELD_MODIFY(CFG_CS_HOLD, &spisg->cfg_spi, val);
+		desc--;
+	} else if (cs_hold_in_sclk) {
 		/* additional null-descriptor to achieve the cs-hold delay */
 		aml_spisg_setup_null_desc(spisg, desc, cs_hold_in_sclk);
-	else
 		desc--;
+		desc->cfg_bus |= FIELD_PREP(CFG_KEEP_SS, 1);
+		desc++;
+	} else {
+		desc--;
+	}
 
-	desc->cfg_bus |= FIELD_PREP(CFG_KEEP_SS, 0);
+	FIELD_MODIFY(CFG_KEEP_SS, &desc->cfg_bus, last_xfer_keep_ss);
 	desc->cfg_start |= FIELD_PREP(CFG_EOC, 1);
 
 	/* some tolerances */
@@ -562,13 +609,16 @@ static int aml_spisg_transfer_one_message(struct spi_controller *ctlr,
 	}
 
 	reinit_completion(&spisg->completion);
-	aml_spisg_pending(spisg, descs_paddr, false, true);
+	aml_spisg_pending(spisg, descs_paddr, false, true, delay);
 	if (wait_for_completion_timeout(&spisg->completion,
 					spi_controller_is_target(spisg->controller) ?
-					MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(ms)))
+					MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(ms))) {
 		ret = spisg->status ? -EIO : 0;
-	else
+	} else {
+		/* stop transfer */
+		regmap_write(spisg->map, SPISG_REG_DESC_LIST_H, 0);
 		ret = -ETIMEDOUT;
+	}
 
 	dma_unmap_single(dev, descs_paddr, descs_len, DMA_TO_DEVICE);
 end:
@@ -704,7 +754,9 @@ static int aml_spisg_clk_init(struct spisg_device *spisg, void __iomem *base)
 		return PTR_ERR(spisg->sclk);
 	}
 
-	clk_prepare_enable(spisg->sclk);
+	ret = clk_prepare_enable(spisg->sclk);
+	if (ret)
+		return ret;
 
 	return 0;
 }
@@ -715,6 +767,7 @@ static int aml_spisg_probe(struct platform_device *pdev)
 	struct spisg_device *spisg;
 	struct device *dev = &pdev->dev;
 	void __iomem *base;
+	u32 val = 0;
 	int ret, irq;
 
 	const struct regmap_config aml_regmap_config = {
@@ -733,6 +786,7 @@ static int aml_spisg_probe(struct platform_device *pdev)
 
 	spisg = spi_controller_get_devdata(ctlr);
 	spisg->controller = ctlr;
+	spisg->data = (struct aml_spisg_data *)of_device_get_match_data(dev);
 
 	spisg->pdev = pdev;
 	platform_set_drvdata(pdev, spisg);
@@ -763,6 +817,14 @@ static int aml_spisg_probe(struct platform_device *pdev)
 
 	spisg->cfg_spi = FIELD_PREP(CFG_SFLASH_WP, 1) |
 			 FIELD_PREP(CFG_SFLASH_HD, 1);
+
+	if (spisg->data && spisg->data->mo_idle_output_ctrl) {
+		if (!of_property_read_u32(dev->of_node, "amlogic,mo-idle-output", &val))
+			spisg->cfg_spi |=  FIELD_PREP(CFG_MO_IDLE_OUTPUT, val);
+		else
+			spisg->cfg_spi |=  FIELD_PREP(CFG_MO_IDLE_OUTPUT, 0);
+	}
+
 	if (spi_controller_is_target(ctlr)) {
 		spisg->cfg_spi |= FIELD_PREP(CFG_SLAVE_EN, 1);
 		spisg->cfg_bus = FIELD_PREP(CFG_TX_TUNING, 0xf);
@@ -841,14 +903,30 @@ static int spisg_suspend_runtime(struct device *dev)
 static int spisg_resume_runtime(struct device *dev)
 {
 	struct spisg_device *spisg = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(spisg->core);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(spisg->sclk);
+	if (ret) {
+		clk_disable_unprepare(spisg->core);
+		return ret;
+	}
 
-	clk_prepare_enable(spisg->core);
-	clk_prepare_enable(spisg->sclk);
 	pinctrl_pm_select_default_state(&spisg->pdev->dev);
 
 	return 0;
 }
 
+static const struct aml_spisg_data a9_spisg_data = {
+	.mo_idle_output_ctrl = true,
+	.word_gap_ctrl = true,
+	.cs_hold_ctrl = true,
+	.cs_setup_extend_ctrl = true,
+};
+
 static const struct dev_pm_ops amlogic_spisg_pm_ops = {
 	.runtime_suspend	= spisg_suspend_runtime,
 	.runtime_resume		= spisg_resume_runtime,
@@ -858,6 +936,10 @@ static const struct of_device_id amlogic_spisg_of_match[] = {
 	{
 		.compatible = "amlogic,a4-spisg",
 	},
+	{
+		.compatible = "amlogic,a9-spisg",
+		.data = &a9_spisg_data,
+	},
 
 	{ /* sentinel */ }
 };
diff --git a/drivers/spi/spi-ar934x.c b/drivers/spi/spi-ar934x.c
index 2210186feab8f1..01e51a30591dec 100644
--- a/drivers/spi/spi-ar934x.c
+++ b/drivers/spi/spi-ar934x.c
@@ -197,23 +197,13 @@ static int ar934x_spi_probe(struct platform_device *pdev)
 				   SPI_BPW_MASK(16) | SPI_BPW_MASK(8);
 	ctlr->num_chipselect = 3;
 
-	dev_set_drvdata(&pdev->dev, ctlr);
-
 	sp = spi_controller_get_devdata(ctlr);
 	sp->base = base;
 	sp->clk = clk;
 	sp->clk_freq = clk_get_rate(clk);
 	sp->ctlr = ctlr;
 
-	return spi_register_controller(ctlr);
-}
-
-static void ar934x_spi_remove(struct platform_device *pdev)
-{
-	struct spi_controller *ctlr;
-
-	ctlr = dev_get_drvdata(&pdev->dev);
-	spi_unregister_controller(ctlr);
+	return devm_spi_register_controller(&pdev->dev, ctlr);
 }
 
 static struct platform_driver ar934x_spi_driver = {
@@ -222,7 +212,6 @@ static struct platform_driver ar934x_spi_driver = {
 		.of_match_table = ar934x_spi_match,
 	},
 	.probe = ar934x_spi_probe,
-	.remove = ar934x_spi_remove,
 };
 
 module_platform_driver(ar934x_spi_driver);
diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c
index c8012c82c3a788..e91e84fdbe8f00 100644
--- a/drivers/spi/spi-atmel.c
+++ b/drivers/spi/spi-atmel.c
@@ -624,7 +624,7 @@ static int atmel_spi_configure_dma(struct spi_controller *host,
 	if (IS_ENABLED(CONFIG_SOC_SAM_V4_V5)) {
 		as->addr_tx_bbuf = dma_alloc_coherent(dev, SPI_MAX_DMA_XFER,
 						      &as->dma_addr_tx_bbuf,
-						      GFP_KERNEL | GFP_DMA);
+						      GFP_KERNEL);
 		if (!as->addr_tx_bbuf) {
 			err = -ENOMEM;
 			goto err_release_dma;
@@ -632,7 +632,7 @@ static int atmel_spi_configure_dma(struct spi_controller *host,
 
 		as->addr_rx_bbuf = dma_alloc_coherent(dev, SPI_MAX_DMA_XFER,
 						      &as->dma_addr_rx_bbuf,
-						      GFP_KERNEL | GFP_DMA);
+						      GFP_KERNEL);
 		if (!as->addr_rx_bbuf) {
 			err = -ENOMEM;
 			goto err_release_dma;
diff --git a/drivers/spi/spi-bcm2835.c b/drivers/spi/spi-bcm2835.c
index 8f8715809c7ce4..23606967f7d1f4 100644
--- a/drivers/spi/spi-bcm2835.c
+++ b/drivers/spi/spi-bcm2835.c
@@ -1297,7 +1297,9 @@ static int bcm2835_spi_setup(struct spi_device *spi)
 	}
 
 	for (i = 0; i < ARRAY_SIZE(pinctrl_compats); i++) {
-		if (of_find_compatible_node(NULL, NULL, pinctrl_compats[i]))
+		struct device_node *np __free(device_node) =
+			of_find_compatible_node(NULL, NULL, pinctrl_compats[i]);
+		if (np)
 			break;
 	}
 
diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c
index 1d8e257bad3021..3f349c1db5815b 100644
--- a/drivers/spi/spi-bcm63xx.c
+++ b/drivers/spi/spi-bcm63xx.c
@@ -413,7 +413,7 @@ static irqreturn_t bcm63xx_spi_interrupt(int irq, void *dev_id)
 	struct bcm63xx_spi *bs = spi_controller_get_devdata(host);
 	u8 intr;
 
-	/* Read interupts and clear them immediately */
+	/* Read interrupts and clear them immediately */
 	intr = bcm_spi_readb(bs, SPI_INT_STATUS);
 	bcm_spi_writeb(bs, SPI_INTR_CLEAR_ALL, SPI_INT_STATUS);
 	bcm_spi_writeb(bs, 0, SPI_INT_MASK);
diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
index 206d3f9dd83d44..ed6e688cfafb5e 100644
--- a/drivers/spi/spi-dw-core.c
+++ b/drivers/spi/spi-dw-core.c
@@ -275,7 +275,7 @@ static irqreturn_t dw_spi_irq(int irq, void *dev_id)
 {
 	struct spi_controller *ctlr = dev_id;
 	struct dw_spi *dws = spi_controller_get_devdata(ctlr);
-	u16 irq_status = dw_readl(dws, DW_SPI_ISR) & DW_SPI_INT_MASK;
+	u16 irq_status = dw_readl(dws, DW_SPI_ISR);
 
 	if (!irq_status)
 		return IRQ_NONE;
@@ -1104,12 +1104,13 @@ static int dw_spi_setup(struct spi_device *spi)
 		if (!chip)
 			return -ENOMEM;
 		spi_set_ctldata(spi, chip);
-		/* Get specific / default rx-sample-delay */
-		if (device_property_read_u32(&spi->dev,
-					     "rx-sample-delay-ns",
-					     &rx_sample_dly_ns) != 0)
-			/* Use default controller value */
-			rx_sample_dly_ns = dws->def_rx_sample_dly_ns;
+		/*
+		 * Use the per-device value the core parsed from the peripheral
+		 * node, and fall back to the controller-wide default when the
+		 * device does not ask for a delay of its own.
+		 */
+		rx_sample_dly_ns = spi->rx_sample_delay_ns ?:
+				   dws->def_rx_sample_dly_ns;
 		chip->rx_sample_dly = DIV_ROUND_CLOSEST(rx_sample_dly_ns,
 							NSEC_PER_SEC /
 							dws->max_freq);
diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c
index 57358851029ba5..d2c2090442f840 100644
--- a/drivers/spi/spi-fsl-qspi.c
+++ b/drivers/spi/spi-fsl-qspi.c
@@ -289,6 +289,7 @@ struct fsl_qspi {
 	struct pm_qos_request pm_qos_req;
 	struct device *dev;
 	int selected;
+	u32 selected_freq;
 	u32 memmap_phy;
 };
 
@@ -551,7 +552,8 @@ static void fsl_qspi_select_mem(struct fsl_qspi *q, struct spi_device *spi,
 	unsigned long rate = op->max_freq;
 	int ret;
 
-	if (q->selected == spi_get_chipselect(spi, 0))
+	if (q->selected == spi_get_chipselect(spi, 0) &&
+	    q->selected_freq == op->max_freq)
 		return;
 
 	if (needs_4x_clock(q))
@@ -571,6 +573,7 @@ static void fsl_qspi_select_mem(struct fsl_qspi *q, struct spi_device *spi,
 	}
 
 	q->selected = spi_get_chipselect(spi, 0);
+	q->selected_freq = op->max_freq;
 
 	fsl_qspi_invalidate(q);
 }
diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c
index 6c57f8309a3b2c..bced6279a06242 100644
--- a/drivers/spi/spi-geni-qcom.c
+++ b/drivers/spi/spi-geni-qcom.c
@@ -12,8 +12,10 @@
 #include <linux/dma/qcom-gpi-dma.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/log2.h>
 #include <linux/module.h>
+#include <linux/panic_notifier.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
 #include <linux/pm_runtime.h>
@@ -115,6 +117,7 @@ struct spi_geni_master {
 	struct dma_chan *rx;
 	int cur_xfer_mode;
 	const struct geni_spi_desc *dev_data;
+	struct notifier_block panic_nb;
 };
 
 static void spi_slv_setup(struct spi_geni_master *mas)
@@ -363,8 +366,8 @@ static int geni_spi_set_clock_and_bw(struct geni_se *se,
 	return 0;
 }
 
-static int setup_fifo_params(struct spi_device *spi_slv,
-					struct spi_controller *spi)
+static void setup_spi_params(struct spi_device *spi_slv,
+			     struct spi_controller *spi)
 {
 	struct spi_geni_master *mas = spi_controller_get_devdata(spi);
 	struct geni_se *se = &mas->se;
@@ -390,8 +393,6 @@ static int setup_fifo_params(struct spi_device *spi_slv,
 
 	trace_geni_spi_setup_params(mas->dev, chipselect, spi_slv->mode,
 				    mode_changed, cs_changed);
-
-	return 0;
 }
 
 static void
@@ -554,17 +555,14 @@ static int spi_geni_prepare_message(struct spi_controller *spi,
 				    struct spi_message *spi_msg)
 {
 	struct spi_geni_master *mas = spi_controller_get_devdata(spi);
-	int ret;
 
 	switch (mas->cur_xfer_mode) {
 	case GENI_SE_FIFO:
 	case GENI_SE_DMA:
 		if (spi_geni_is_abort_still_pending(mas))
 			return -EBUSY;
-		ret = setup_fifo_params(spi_msg->spi, spi);
-		if (ret)
-			dev_err(mas->dev, "Couldn't select mode %d\n", ret);
-		return ret;
+		setup_spi_params(spi_msg->spi, spi);
+		return 0;
 
 	case GENI_GPI_DMA:
 		/* nothing to do for GPI DMA */
@@ -700,7 +698,7 @@ static int spi_geni_init(struct spi_geni_master *mas)
 	case 0:
 		mas->cur_xfer_mode = GENI_SE_FIFO;
 		geni_se_select_mode(se, GENI_SE_FIFO);
-		/* setup_fifo_params assumes that these registers start with a zero value */
+		/* setup_spi_params assumes that these registers start with a zero value */
 		writel(0, se->base + SE_SPI_LOOPBACK);
 		writel(0, se->base + SE_SPI_DEMUX_SEL);
 		writel(0, se->base + SE_SPI_CPHA);
@@ -1066,6 +1064,69 @@ static int spi_geni_target_abort(struct spi_controller *spi)
 	return 0;
 }
 
+static void spi_geni_shutdown(struct platform_device *pdev)
+{
+	struct spi_controller *spi = platform_get_drvdata(pdev);
+
+	spi_controller_suspend(spi);
+}
+
+static int spi_geni_panic_notifier(struct notifier_block *nb,
+				   unsigned long action, void *data)
+{
+	struct spi_geni_master *mas = container_of(nb, struct spi_geni_master, panic_nb);
+	struct spi_controller *spi = dev_get_drvdata(mas->dev);
+	struct geni_se *se = &mas->se;
+	u32 val;
+
+	if (!pm_runtime_active(mas->dev))
+		return NOTIFY_OK;
+
+	if (mas->cur_xfer_mode == GENI_GPI_DMA) {
+		dmaengine_terminate_async(mas->tx);
+		dmaengine_terminate_async(mas->rx);
+		return NOTIFY_OK;
+	}
+
+	if (!(readl_relaxed(se->base + SE_GENI_STATUS) & M_GENI_CMD_ACTIVE))
+		return NOTIFY_OK;
+
+	if (!spi->target) {
+		geni_se_cancel_m_cmd(se);
+		if (!readl_poll_timeout_atomic(se->base + SE_GENI_M_IRQ_STATUS, val,
+					       val & M_CMD_CANCEL_EN, 10, 50000)) {
+			writel_relaxed(M_CMD_CANCEL_EN, se->base + SE_GENI_M_IRQ_CLEAR);
+			return NOTIFY_OK;
+		}
+	}
+
+	geni_se_abort_m_cmd(se);
+	if (!readl_poll_timeout_atomic(se->base + SE_GENI_M_IRQ_STATUS, val,
+				       val & M_CMD_ABORT_EN, 10, 50000))
+		writel_relaxed(M_CMD_ABORT_EN, se->base + SE_GENI_M_IRQ_CLEAR);
+
+	if (mas->cur_xfer_mode == GENI_SE_DMA) {
+		writel_relaxed(1, se->base + SE_DMA_TX_FSM_RST);
+		readl_poll_timeout_atomic(se->base + SE_DMA_TX_IRQ_STAT, val,
+					  val & TX_RESET_DONE, 10, 50000);
+		writel_relaxed(val, se->base + SE_DMA_TX_IRQ_CLR);
+
+		writel_relaxed(1, se->base + SE_DMA_RX_FSM_RST);
+		readl_poll_timeout_atomic(se->base + SE_DMA_RX_IRQ_STAT, val,
+					  val & RX_RESET_DONE, 10, 50000);
+		writel_relaxed(val, se->base + SE_DMA_RX_IRQ_CLR);
+	}
+
+	return NOTIFY_OK;
+}
+
+static void spi_geni_unregister_notifiers(void *data)
+{
+	struct spi_geni_master *mas = data;
+
+	atomic_notifier_chain_unregister(&panic_notifier_list, &mas->panic_nb);
+}
+
 static int spi_geni_probe(struct platform_device *pdev)
 {
 	int ret, irq;
@@ -1154,6 +1215,15 @@ static int spi_geni_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	mas->panic_nb.notifier_call = spi_geni_panic_notifier;
+	ret = atomic_notifier_chain_register(&panic_notifier_list, &mas->panic_nb);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(dev, spi_geni_unregister_notifiers, mas);
+	if (ret)
+		return ret;
+
 	return devm_spi_register_controller(dev, spi);
 }
 
@@ -1241,7 +1311,8 @@ static const struct of_device_id spi_geni_dt_match[] = {
 MODULE_DEVICE_TABLE(of, spi_geni_dt_match);
 
 static struct platform_driver spi_geni_driver = {
-	.probe  = spi_geni_probe,
+	.probe    = spi_geni_probe,
+	.shutdown = spi_geni_shutdown,
 	.driver = {
 		.name = "geni_spi",
 		.pm = pm_ptr(&spi_geni_pm_ops),
diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c
index 79a6c1a60b0a05..6f9686df0486e6 100644
--- a/drivers/spi/spi-imx.c
+++ b/drivers/spi/spi-imx.c
@@ -673,7 +673,7 @@ static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,
 	 * propagate into the hardware. It takes exactly one tick of the
 	 * SCLK clock, but we will wait two SCLK clock just to be sure. The
 	 * effect of the delay it takes for the hardware to apply changes
-	 * is noticable if the SCLK clock run very slow. In such a case, if
+	 * is noticeable if the SCLK clock run very slow. In such a case, if
 	 * the polarity of SCLK should be inverted, the GPIO ChipSelect might
 	 * be asserted before the SCLK polarity changes, which would disrupt
 	 * the SPI communication as the device on the other end would consider
diff --git a/drivers/spi/spi-ingenic.c b/drivers/spi/spi-ingenic.c
index adcf85bccbcc01..7ce4fe965cac10 100644
--- a/drivers/spi/spi-ingenic.c
+++ b/drivers/spi/spi-ingenic.c
@@ -354,8 +354,11 @@ static int spi_ingenic_request_dma(struct spi_controller *ctlr,
 	ctlr->dma_tx = chan;
 
 	chan = dma_request_chan(dev, "rx");
-	if (IS_ERR(chan))
+	if (IS_ERR(chan)) {
+		dma_release_channel(ctlr->dma_tx);
+		ctlr->dma_tx = NULL;
 		return PTR_ERR(chan);
+	}
 	ctlr->dma_rx = chan;
 
 	ctlr->can_dma = spi_ingenic_can_dma;
diff --git a/drivers/spi/spi-ma35d1-qspi.c b/drivers/spi/spi-ma35d1-qspi.c
index 7f938d0c6f1fc4..30e3239f777d6f 100644
--- a/drivers/spi/spi-ma35d1-qspi.c
+++ b/drivers/spi/spi-ma35d1-qspi.c
@@ -633,18 +633,23 @@ static int nuvoton_qspi_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, ret, "failed to deassert reset\n");
 
 	ret = device_property_read_u32(dev, "num-cs", &num_cs);
-	if (ret && ret != -EINVAL)
-		return dev_err_probe(dev, ret, "failed to read num-cs\n");
+	if (ret && ret != -EINVAL) {
+		ret = dev_err_probe(dev, ret, "failed to read num-cs\n");
+		goto err_assert;
+	}
 
-	if (!num_cs || num_cs > NUVOTON_QSPI_MAX_NUM_CS)
-		return dev_err_probe(dev, -EINVAL, "invalid num-cs %u\n",
+	if (!num_cs || num_cs > NUVOTON_QSPI_MAX_NUM_CS) {
+		ret = dev_err_probe(dev, -EINVAL, "invalid num-cs %u\n",
 				     num_cs);
+		goto err_assert;
+	}
 
 	ctlr->num_chipselect = num_cs;
 	ctlr->max_transfer_size = nuvoton_qspi_max_transfer_size;
 	ctlr->max_message_size = nuvoton_qspi_max_message_size;
 	ctlr->mem_ops = &nuvoton_qspi_mem_ops;
 	ctlr->mem_caps = &nuvoton_qspi_mem_caps;
+	ctlr->dtr_caps = true;
 	ctlr->set_cs = nuvoton_qspi_set_cs;
 	ctlr->transfer_one = nuvoton_qspi_transfer_one;
 	ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
@@ -655,14 +660,20 @@ static int nuvoton_qspi_probe(struct platform_device *pdev)
 
 	ret = nuvoton_qspi_hw_init(qspi);
 	if (ret)
-		return ret;
+		goto err_assert;
 
 	ret = devm_spi_register_controller(dev, ctlr);
-	if (ret)
-		return dev_err_probe(dev, ret,
-				     "failed to register spi controller\n");
+	if (ret) {
+		ret = dev_err_probe(dev, ret,
+				    "failed to register spi controller\n");
+		goto err_assert;
+	}
 
 	return 0;
+
+err_assert:
+	reset_control_assert(rst);
+	return ret;
 }
 
 static const struct of_device_id nuvoton_qspi_of_match[] = {
diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c
index 5f973ebfb8b6eb..b94680b60af958 100644
--- a/drivers/spi/spi-mem.c
+++ b/drivers/spi/spi-mem.c
@@ -46,9 +46,9 @@ int spi_controller_dma_map_mem_op_data(struct spi_controller *ctlr,
 		return -EINVAL;
 
 	if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx)
-		dmadev = ctlr->dma_tx->device->dev;
+		dmadev = dmaengine_get_dma_device(ctlr->dma_tx);
 	else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx)
-		dmadev = ctlr->dma_rx->device->dev;
+		dmadev = dmaengine_get_dma_device(ctlr->dma_rx);
 	else
 		dmadev = ctlr->dev.parent;
 
@@ -92,9 +92,9 @@ void spi_controller_dma_unmap_mem_op_data(struct spi_controller *ctlr,
 		return;
 
 	if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx)
-		dmadev = ctlr->dma_tx->device->dev;
+		dmadev = dmaengine_get_dma_device(ctlr->dma_tx);
 	else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx)
-		dmadev = ctlr->dma_rx->device->dev;
+		dmadev = dmaengine_get_dma_device(ctlr->dma_rx);
 	else
 		dmadev = ctlr->dev.parent;
 
@@ -172,7 +172,7 @@ bool spi_mem_default_supports_op(struct spi_mem *mem,
 		op->cmd.dtr || op->addr.dtr || op->dummy.dtr || op->data.dtr;
 
 	if (op_is_dtr) {
-		if (!spi_mem_controller_is_capable(ctlr, dtr))
+		if (!spi_mem_controller_is_capable(ctlr, dtr) && !ctlr->dtr_caps)
 			return false;
 
 		if (op->data.swap16 && !spi_mem_controller_is_capable(ctlr, swap16))
@@ -461,6 +461,7 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 	xfers[xferpos].len = op->cmd.nbytes;
 	xfers[xferpos].tx_nbits = op->cmd.buswidth;
 	xfers[xferpos].speed_hz = op->max_freq;
+	xfers[xferpos].dtr_mode = op->cmd.dtr;
 	spi_message_add_tail(&xfers[xferpos], &msg);
 	xferpos++;
 	totalxferlen++;
@@ -476,6 +477,7 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 		xfers[xferpos].len = op->addr.nbytes;
 		xfers[xferpos].tx_nbits = op->addr.buswidth;
 		xfers[xferpos].speed_hz = op->max_freq;
+		xfers[xferpos].dtr_mode = op->addr.dtr;
 		spi_message_add_tail(&xfers[xferpos], &msg);
 		xferpos++;
 		totalxferlen += op->addr.nbytes;
@@ -488,6 +490,7 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 		xfers[xferpos].tx_nbits = op->dummy.buswidth;
 		xfers[xferpos].dummy_data = 1;
 		xfers[xferpos].speed_hz = op->max_freq;
+		xfers[xferpos].dtr_mode = op->dummy.dtr;
 		spi_message_add_tail(&xfers[xferpos], &msg);
 		xferpos++;
 		totalxferlen += op->dummy.nbytes;
@@ -504,6 +507,7 @@ int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 
 		xfers[xferpos].len = op->data.nbytes;
 		xfers[xferpos].speed_hz = op->max_freq;
+		xfers[xferpos].dtr_mode = op->data.dtr;
 		spi_message_add_tail(&xfers[xferpos], &msg);
 		xferpos++;
 		totalxferlen += op->data.nbytes;
diff --git a/drivers/spi/spi-mtk-nor.c b/drivers/spi/spi-mtk-nor.c
index 63f5139176c8d0..6b6e55d68048b1 100644
--- a/drivers/spi/spi-mtk-nor.c
+++ b/drivers/spi/spi-mtk-nor.c
@@ -929,6 +929,7 @@ static int mtk_nor_probe(struct platform_device *pdev)
 	return 0;
 
 err_probe:
+	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_set_suspended(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
diff --git a/drivers/spi/spi-mxic.c b/drivers/spi/spi-mxic.c
index e3c85e5191e369..481edd4b35cbf9 100644
--- a/drivers/spi/spi-mxic.c
+++ b/drivers/spi/spi-mxic.c
@@ -744,7 +744,13 @@ static int mxic_spi_runtime_resume(struct device *dev)
 		return ret;
 	}
 
-	return mxic_spi_clk_enable(mxic);
+	ret = mxic_spi_clk_enable(mxic);
+	if (ret) {
+		clk_disable_unprepare(mxic->ps_clk);
+		return ret;
+	}
+
+	return 0;
 }
 
 static const struct dev_pm_ops mxic_spi_dev_pm_ops = {
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index a3355147f5de51..ace6886a75c3e9 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -1244,7 +1244,7 @@ static int omap2_mcspi_transfer_one(struct spi_controller *ctlr,
 		omap2_mcspi_set_fifo(spi, t, 0);
 
 out:
-	/* Restore defaults if they were overriden */
+	/* Restore defaults if they were overridden */
 	if (par_override) {
 		par_override = 0;
 		status = omap2_mcspi_setup_transfer(spi, NULL);
@@ -1595,7 +1595,6 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 
 err_disable_rpm:
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
-	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 err_release_dma:
 	omap2_mcspi_release_dma(ctlr);
@@ -1613,7 +1612,6 @@ static void omap2_mcspi_remove(struct platform_device *pdev)
 	omap2_mcspi_release_dma(ctlr);
 
 	pm_runtime_dont_use_autosuspend(mcspi->dev);
-	pm_runtime_put_sync(mcspi->dev);
 	pm_runtime_disable(&pdev->dev);
 }
 
diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c
index 26a9f268b2d303..b8882229c05493 100644
--- a/drivers/spi/spi-orion.c
+++ b/drivers/spi/spi-orion.c
@@ -370,7 +370,16 @@ static inline int orion_spi_wait_till_ready(struct orion_spi *orion_spi)
 		if (readl(spi_reg(orion_spi, ORION_SPI_INT_CAUSE_REG)))
 			return 1;
 
+		/*
+		 * This is a polled, byte-at-a-time transfer loop. Each
+		 * iteration busy-waits in a tight udelay() loop, which can
+		 * starve other time-sensitive peripherals (e.g. SATA) of CPU
+		 * time and interfere with them if they run on this SoC.
+		 * Yield to the scheduler between polls so pending IRQs can
+		 * be serviced.
+		 */
 		udelay(1);
+		cond_resched();
 	}
 
 	return -1;
diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c
index 972128271e4bfa..8363bc873b03a3 100644
--- a/drivers/spi/spi-pic32.c
+++ b/drivers/spi/spi-pic32.c
@@ -53,7 +53,7 @@ struct pic32_spi_regs {
 #define  TX_FIFO_ALL_EMPTY	0 /* completely empty */
 #define  TX_FIFO_EMPTY		1 /* empty */
 #define  TX_FIFO_HALF_EMPTY	2 /* empty by half or more */
-#define  TX_FIFO_NOT_FULL	3 /* atleast one empty */
+#define  TX_FIFO_NOT_FULL	3 /* at least one empty */
 
 #define CTRL_MSTEN	BIT(5) /* enable master mode */
 #define CTRL_CKP	BIT(6) /* active low */
diff --git a/drivers/spi/spi-qpic-snand.c b/drivers/spi/spi-qpic-snand.c
index 61b1f2eb19ce5a..00072c44db7a96 100644
--- a/drivers/spi/spi-qpic-snand.c
+++ b/drivers/spi/spi-qpic-snand.c
@@ -63,7 +63,6 @@
 #define ACC_FEATURE			0xe
 #define BAD_BLOCK_MARKER_SIZE		0x2
 #define OOB_BUF_SIZE			128
-#define ecceng_to_qspi(eng)		container_of(eng, struct qpic_spi_nand, ecc_eng)
 
 struct snandc_read_status {
 	__le32 snandc_flash;
@@ -98,7 +97,6 @@ struct qpic_ecc {
 	u32 cfg1;
 	u32 cfg0_raw;
 	u32 cfg1_raw;
-	u32 ecc_buf_cfg;
 	u32 ecc_bch_cfg;
 	bool bch_enabled;
 };
@@ -168,8 +166,9 @@ static void qcom_spi_set_read_loc_last(struct qcom_nand_controller *snandc,
 static struct qcom_nand_controller *nand_to_qcom_snand(struct nand_device *nand)
 {
 	struct nand_ecc_engine *eng = nand->ecc.engine;
-	struct qpic_spi_nand *qspi = ecceng_to_qspi(eng);
+	struct qpic_spi_nand *qspi;
 
+	qspi = container_of(eng, struct qpic_spi_nand, ecc_eng);
 	return qspi->snandc;
 }
 
@@ -343,6 +342,15 @@ static int qcom_spi_ecc_init_ctx_pipelined(struct nand_device *nand)
 	ecc_cfg->cw_size = ecc_cfg->cw_data + ecc_cfg->bytes;
 	bad_block_byte = mtd->writesize - ecc_cfg->cw_size * (cwperpage - 1) + 1;
 
+	if ((ecc_cfg->cw_size * cwperpage) > (mtd->writesize + mtd->oobsize)) {
+		dev_err(snandc->dev,
+			"Current ECC settings require %d bytes, but the flash only has %d+%d bytes.\n",
+			(ecc_cfg->cw_size * cwperpage), mtd->writesize,
+			mtd->oobsize);
+		ret = -EINVAL;
+		goto err_free_ecc_cfg;
+	}
+
 	mtd_set_ooblayout(mtd, &qcom_spi_ooblayout);
 
 	/*
@@ -398,8 +406,6 @@ static int qcom_spi_ecc_init_ctx_pipelined(struct nand_device *nand)
 			       FIELD_PREP(ECC_MODE_MASK, ecc_cfg->ecc_mode) |
 			       FIELD_PREP(ECC_PARITY_SIZE_BYTES_BCH_MASK, ecc_cfg->ecc_bytes_hw);
 
-	ecc_cfg->ecc_buf_cfg = FIELD_PREP(NUM_STEPS_MASK, 0x203);
-
 	conf->step_size = ecc_cfg->step_size;
 	conf->strength = ecc_cfg->strength;
 
@@ -411,6 +417,8 @@ static int qcom_spi_ecc_init_ctx_pipelined(struct nand_device *nand)
 	dev_dbg(snandc->dev, "ECC strength: %u bits per %u bytes\n",
 		ecc_cfg->strength, ecc_cfg->step_size);
 
+	snandc->qspi->ecc = ecc_cfg;
+
 	return 0;
 
 err_free_ecc_cfg:
@@ -427,6 +435,7 @@ static void qcom_spi_ecc_cleanup_ctx_pipelined(struct nand_device *nand)
 
 	kfree(snandc->qspi->oob_buf);
 	snandc->qspi->oob_buf = NULL;
+	snandc->qspi->ecc = NULL;
 	kfree(ecc_cfg);
 }
 
@@ -434,9 +443,7 @@ static int qcom_spi_ecc_prepare_io_req_pipelined(struct nand_device *nand,
 						 struct nand_page_io_req *req)
 {
 	struct qcom_nand_controller *snandc = nand_to_qcom_snand(nand);
-	struct qpic_ecc *ecc_cfg = nand_to_ecc_ctx(nand);
 
-	snandc->qspi->ecc = ecc_cfg;
 	snandc->qspi->raw_rw = false;
 	snandc->qspi->oob_rw = false;
 	snandc->qspi->page_rw = false;
@@ -500,18 +507,28 @@ static void qcom_spi_set_read_loc(struct qcom_nand_controller *snandc, int cw, i
 						   read_size, is_last_read_loc);
 }
 
+static void qcom_spi_config_page_read(struct qcom_nand_controller *snandc)
+{
+	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
+	qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0);
+	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_clr,
+			   NAND_ERASED_CW_DETECT_CFG, 1, 0);
+	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_set,
+			   NAND_ERASED_CW_DETECT_CFG, 1,
+			   NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
+}
+
 static void
 qcom_spi_config_cw_read(struct qcom_nand_controller *snandc, bool use_ecc, int cw)
 {
-	__le32 *reg = &snandc->regs->read_location0;
-	int num_cw = snandc->qspi->num_cw;
-
-	qcom_write_reg_dma(snandc, reg, NAND_READ_LOCATION_0, 4, NAND_BAM_NEXT_SGL);
-	if (cw == (num_cw - 1)) {
-		reg = &snandc->regs->read_location_last0;
-		qcom_write_reg_dma(snandc, reg, NAND_READ_LOCATION_LAST_CW_0, 4,
+	if (cw == (snandc->qspi->num_cw - 1))
+		qcom_write_reg_dma(snandc, &snandc->regs->read_location_last0,
+				   NAND_READ_LOCATION_LAST_CW_0, 4,
+				   NAND_BAM_NEXT_SGL);
+	else
+		qcom_write_reg_dma(snandc, &snandc->regs->read_location0,
+				   NAND_READ_LOCATION_0, 4,
 				   NAND_BAM_NEXT_SGL);
-	}
 
 	qcom_write_reg_dma(snandc, &snandc->regs->cmd, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
 	qcom_write_reg_dma(snandc, &snandc->regs->exec, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
@@ -563,13 +580,7 @@ static void qcom_spi_config_single_cw_page_read(struct qcom_nand_controller *sna
 	__le32 *reg = &snandc->regs->read_location0;
 	int num_cw = snandc->qspi->num_cw;
 
-	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_clr,
-			   NAND_ERASED_CW_DETECT_CFG, 1, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_set,
-			   NAND_ERASED_CW_DETECT_CFG, 1,
-			   NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
+	qcom_spi_config_page_read(snandc);
 
 	if (cw == (num_cw - 1)) {
 		reg = &snandc->regs->read_location_last0;
@@ -762,16 +773,7 @@ static int qcom_spi_read_cw_raw(struct qcom_nand_controller *snandc, u8 *data_bu
 	snandc->regs->exec = cpu_to_le32(1);
 
 	qcom_spi_set_read_loc(snandc, raw_cw, 0, 0, ecc_cfg->cw_size, 1);
-
-	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->ecc_buf_cfg, NAND_EBI2_ECC_BUF_CFG, 1, 0);
-
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_clr,
-			   NAND_ERASED_CW_DETECT_CFG, 1, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_set,
-			   NAND_ERASED_CW_DETECT_CFG, 1,
-			   NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
+	qcom_spi_config_page_read(snandc);
 
 	data_size1 = mtd->writesize - ecc_cfg->cw_size * (num_cw - 1);
 	oob_size1 = ecc_cfg->bbm_size;
@@ -876,14 +878,7 @@ static int qcom_spi_read_page_ecc(struct qcom_nand_controller *snandc,
 	snandc->regs->exec = cpu_to_le32(1);
 
 	qcom_clear_bam_transaction(snandc);
-
-	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_clr,
-			   NAND_ERASED_CW_DETECT_CFG, 1, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_set,
-			   NAND_ERASED_CW_DETECT_CFG, 1,
-			   NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
+	qcom_spi_config_page_read(snandc);
 
 	for (i = 0; i < num_cw; i++) {
 		int data_size, oob_size;
@@ -964,13 +959,7 @@ static int qcom_spi_read_page_oob(struct qcom_nand_controller *snandc,
 	snandc->regs->ecc_bch_cfg = cpu_to_le32(ecc_bch_cfg);
 	snandc->regs->exec = cpu_to_le32(1);
 
-	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_clr,
-			   NAND_ERASED_CW_DETECT_CFG, 1, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->erased_cw_detect_cfg_set,
-			   NAND_ERASED_CW_DETECT_CFG, 1,
-			   NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
+	qcom_spi_config_page_read(snandc);
 
 	for (i = 0; i < num_cw; i++) {
 		int data_size, oob_size;
@@ -1104,8 +1093,6 @@ static void qcom_spi_config_page_write(struct qcom_nand_controller *snandc)
 {
 	qcom_write_reg_dma(snandc, &snandc->regs->addr0, NAND_ADDR0, 2, 0);
 	qcom_write_reg_dma(snandc, &snandc->regs->cfg0, NAND_DEV0_CFG0, 3, 0);
-	qcom_write_reg_dma(snandc, &snandc->regs->ecc_buf_cfg, NAND_EBI2_ECC_BUF_CFG,
-			   1, NAND_BAM_NEXT_SGL);
 }
 
 static void qcom_spi_config_cw_write(struct qcom_nand_controller *snandc)
@@ -1208,13 +1195,12 @@ static int qcom_spi_program_ecc(struct qcom_nand_controller *snandc,
 	u8 *data_buf = NULL, *oob_buf = NULL;
 	int i, ret;
 	int num_cw = snandc->qspi->num_cw;
-	u32 cfg0, cfg1, ecc_bch_cfg, ecc_buf_cfg;
+	u32 cfg0, cfg1, ecc_bch_cfg;
 
 	cfg0 = (ecc_cfg->cfg0 & ~CW_PER_PAGE_MASK) |
 	       FIELD_PREP(CW_PER_PAGE_MASK, num_cw - 1);
 	cfg1 = ecc_cfg->cfg1;
 	ecc_bch_cfg = ecc_cfg->ecc_bch_cfg;
-	ecc_buf_cfg = ecc_cfg->ecc_buf_cfg;
 
 	if (snandc->qspi->data_buf)
 		data_buf = snandc->qspi->data_buf;
@@ -1232,7 +1218,6 @@ static int qcom_spi_program_ecc(struct qcom_nand_controller *snandc,
 	snandc->regs->cfg0 = cpu_to_le32(cfg0);
 	snandc->regs->cfg1 = cpu_to_le32(cfg1);
 	snandc->regs->ecc_bch_cfg = cpu_to_le32(ecc_bch_cfg);
-	snandc->regs->ecc_buf_cfg = cpu_to_le32(ecc_buf_cfg);
 	snandc->regs->exec = cpu_to_le32(1);
 
 	qcom_spi_config_page_write(snandc);
@@ -1285,13 +1270,12 @@ static int qcom_spi_program_oob(struct qcom_nand_controller *snandc,
 	u8 *oob_buf = NULL;
 	int ret, col, data_size, oob_size;
 	int num_cw = snandc->qspi->num_cw;
-	u32 cfg0, cfg1, ecc_bch_cfg, ecc_buf_cfg;
+	u32 cfg0, cfg1, ecc_bch_cfg;
 
 	cfg0 = (ecc_cfg->cfg0 & ~CW_PER_PAGE_MASK) |
 	       FIELD_PREP(CW_PER_PAGE_MASK, 0);
 	cfg1 = ecc_cfg->cfg1;
 	ecc_bch_cfg = ecc_cfg->ecc_bch_cfg;
-	ecc_buf_cfg = ecc_cfg->ecc_buf_cfg;
 
 	col = ecc_cfg->cw_size * (num_cw - 1);
 
@@ -1307,7 +1291,6 @@ static int qcom_spi_program_oob(struct qcom_nand_controller *snandc,
 	snandc->regs->cfg0 = cpu_to_le32(cfg0);
 	snandc->regs->cfg1 = cpu_to_le32(cfg1);
 	snandc->regs->ecc_bch_cfg = cpu_to_le32(ecc_bch_cfg);
-	snandc->regs->ecc_buf_cfg = cpu_to_le32(ecc_buf_cfg);
 	snandc->regs->exec = cpu_to_le32(1);
 
 	/* calculate the data and oob size for the last codeword/step */
@@ -1485,7 +1468,7 @@ static int qcom_spi_io_op(struct qcom_nand_controller *snandc, const struct spi_
 
 	if (copy_ftr) {
 		qcom_nandc_dev_to_mem(snandc, true);
-		val = le32_to_cpu(*(__le32 *)snandc->reg_read_buf);
+		val = le32_to_cpu(*snandc->reg_read_buf);
 		val >>= 8;
 		memcpy(op->data.buf.in, &val, snandc->buf_count);
 
@@ -1583,15 +1566,9 @@ static int qcom_spi_probe(struct platform_device *pdev)
 	struct spi_controller *ctlr;
 	struct qcom_nand_controller *snandc;
 	struct qpic_spi_nand *qspi;
-	struct qpic_ecc *ecc;
 	struct resource *res;
-	const void *dev_data;
 	int ret;
 
-	ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
-	if (!ecc)
-		return -ENOMEM;
-
 	qspi = devm_kzalloc(dev, sizeof(*qspi), GFP_KERNEL);
 	if (!qspi)
 		return -ENOMEM;
@@ -1611,16 +1588,13 @@ static int qcom_spi_probe(struct platform_device *pdev)
 	snandc->dev = dev;
 	snandc->qspi = qspi;
 	snandc->qspi->ctlr = ctlr;
-	snandc->qspi->ecc = ecc;
 
-	dev_data = of_device_get_match_data(dev);
-	if (!dev_data) {
+	snandc->props = of_device_get_match_data(dev);
+	if (!snandc->props) {
 		dev_err(&pdev->dev, "failed to get device data\n");
 		return -ENODEV;
 	}
 
-	snandc->props = dev_data;
-
 	snandc->core_clk = devm_clk_get_enabled(dev, "core");
 	if (IS_ERR(snandc->core_clk))
 		return PTR_ERR(snandc->core_clk);
diff --git a/drivers/spi/spi-realtek-rtl.c b/drivers/spi/spi-realtek-rtl.c
index e1c40ff2e49d38..7b250b447abfdb 100644
--- a/drivers/spi/spi-realtek-rtl.c
+++ b/drivers/spi/spi-realtek-rtl.c
@@ -18,7 +18,6 @@ struct rtspi {
 #define RTL_SPI_SFCSR_CSB0		BIT(31)
 #define RTL_SPI_SFCSR_CSB1		BIT(30)
 #define RTL_SPI_SFCSR_RDY		BIT(27)
-#define RTL_SPI_SFCSR_CS		BIT(24)
 #define RTL_SPI_SFCSR_LEN_MASK		~(0x03 << 28)
 #define RTL_SPI_SFCSR_LEN1		(0x00 << 28)
 #define RTL_SPI_SFCSR_LEN4		(0x03 << 28)
@@ -29,17 +28,31 @@ struct rtspi {
 #define REG(x)		(rtspi->base + x)
 
 
-static void rt_set_cs(struct spi_device *spi, bool active)
+static void rt_set_cs(struct spi_device *spi, bool level)
 {
 	struct rtspi *rtspi = spi_controller_get_devdata(spi->controller);
-	u32 value;
+	unsigned int cs = spi_get_chipselect(spi, 0);
+	u32 cs_mask, value;
+
+	switch (cs) {
+	case 0:
+		cs_mask = RTL_SPI_SFCSR_CSB0;
+		break;
+	case 1:
+		cs_mask = RTL_SPI_SFCSR_CSB1;
+		break;
+	default:
+		return;
+	}
 
-	/* CS0 bit is active low */
 	value = __raw_readl(REG(RTL_SPI_SFCSR));
-	if (active)
-		value |= RTL_SPI_SFCSR_CSB0;
+
+	/* CSBx is active low */
+	if (level)
+		value |= cs_mask;
 	else
-		value &= ~RTL_SPI_SFCSR_CSB0;
+		value &= ~cs_mask;
+
 	__raw_writel(value, REG(RTL_SPI_SFCSR));
 }
 
@@ -138,11 +151,9 @@ static void init_hw(struct rtspi *rtspi)
 	value |= RTL_SPI_SFCR_RBO | RTL_SPI_SFCR_WBO;
 	__raw_writel(value, REG(RTL_SPI_SFCR));
 
-	value = __raw_readl(REG(RTL_SPI_SFCSR));
-	/* Permanently disable CS1, since it's never used */
-	value |= RTL_SPI_SFCSR_CSB1;
-	/* Select CS0 for use */
-	value &= RTL_SPI_SFCSR_CS;
+	/* CHIP_SEL is only used in MMIO mode; CSB0/CSB1 are active low. */
+	value = 0;
+	value |= RTL_SPI_SFCSR_CSB0 | RTL_SPI_SFCSR_CSB1;
 	__raw_writel(value, REG(RTL_SPI_SFCSR));
 }
 
@@ -171,6 +182,7 @@ static int realtek_rtl_spi_probe(struct platform_device *pdev)
 	ctrl->flags = SPI_CONTROLLER_HALF_DUPLEX;
 	ctrl->set_cs = rt_set_cs;
 	ctrl->transfer_one = transfer_one;
+	ctrl->num_chipselect = 2;
 
 	err = devm_spi_register_controller(&pdev->dev, ctrl);
 	if (err) {
diff --git a/drivers/spi/spi-rockchip-sfc.c b/drivers/spi/spi-rockchip-sfc.c
index 662a994da60bec..a85a260ba95f75 100644
--- a/drivers/spi/spi-rockchip-sfc.c
+++ b/drivers/spi/spi-rockchip-sfc.c
@@ -719,10 +719,12 @@ static int rockchip_sfc_probe(struct platform_device *pdev)
 
 	return 0;
 err_register:
-	dma_unmap_single(dev, sfc->dma_buffer, sfc->max_iosize,
-			 DMA_BIDIRECTIONAL);
+	if (sfc->use_dma)
+		dma_unmap_single(dev, sfc->dma_buffer, sfc->max_iosize,
+				 DMA_BIDIRECTIONAL);
 err_dma_map:
-	free_pages((unsigned long)sfc->buffer, get_order(sfc->max_iosize));
+	if (sfc->use_dma)
+		free_pages((unsigned long)sfc->buffer, get_order(sfc->max_iosize));
 err_dma:
 	pm_runtime_get_sync(dev);
 	pm_runtime_put_noidle(dev);
@@ -743,9 +745,16 @@ static void rockchip_sfc_remove(struct platform_device *pdev)
 	struct spi_controller *host = sfc->host;
 
 	spi_unregister_controller(host);
-	dma_unmap_single(&pdev->dev, sfc->dma_buffer, sfc->max_iosize,
-			 DMA_BIDIRECTIONAL);
-	free_pages((unsigned long)sfc->buffer, get_order(sfc->max_iosize));
+	pm_runtime_get_sync(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_set_suspended(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
+	if (sfc->use_dma) {
+		dma_unmap_single(&pdev->dev, sfc->dma_buffer, sfc->max_iosize,
+				 DMA_BIDIRECTIONAL);
+		free_pages((unsigned long)sfc->buffer, get_order(sfc->max_iosize));
+	}
 
 	clk_disable_unprepare(sfc->clk);
 	clk_disable_unprepare(sfc->hclk);
diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c
index 38df676774ee1a..c61edd1c1353b1 100644
--- a/drivers/spi/spi-rspi.c
+++ b/drivers/spi/spi-rspi.c
@@ -22,7 +22,6 @@
 #include <linux/of.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
-#include <linux/sh_dma.h>
 #include <linux/spi/spi.h>
 #include <linux/spinlock.h>
 
@@ -1091,24 +1090,15 @@ static irqreturn_t rspi_irq_tx(int irq, void *_sr)
 
 static struct dma_chan *rspi_request_dma_chan(struct device *dev,
 					      enum dma_transfer_direction dir,
-					      unsigned int id,
 					      dma_addr_t port_addr)
 {
-	dma_cap_mask_t mask;
 	struct dma_chan *chan;
 	struct dma_slave_config cfg;
 	int ret;
 
-	dma_cap_zero(mask);
-	dma_cap_set(DMA_SLAVE, mask);
-
-	chan = dma_request_slave_channel_compat(mask, shdma_chan_filter,
-				(void *)(unsigned long)id, dev,
-				dir == DMA_MEM_TO_DEV ? "tx" : "rx");
-	if (!chan) {
-		dev_warn(dev, "dma_request_slave_channel_compat failed\n");
-		return NULL;
-	}
+	chan = dma_request_chan(dev, dir == DMA_MEM_TO_DEV ? "tx" : "rx");
+	if (IS_ERR(chan))
+		return chan;
 
 	memset(&cfg, 0, sizeof(cfg));
 	cfg.dst_addr = port_addr + RSPI_SPDR;
@@ -1121,7 +1111,7 @@ static struct dma_chan *rspi_request_dma_chan(struct device *dev,
 	if (ret) {
 		dev_warn(dev, "dmaengine_slave_config failed %d\n", ret);
 		dma_release_channel(chan);
-		return NULL;
+		return ERR_PTR(ret);
 	}
 
 	return chan;
@@ -1130,29 +1120,23 @@ static struct dma_chan *rspi_request_dma_chan(struct device *dev,
 static int rspi_request_dma(struct device *dev, struct spi_controller *ctlr,
 			    const struct resource *res)
 {
-	unsigned int dma_tx_id, dma_rx_id;
+	struct dma_chan *chan;
 
-	if (dev->of_node) {
-		/* In the OF case we will get the slave IDs from the DT */
-		dma_tx_id = 0;
-		dma_rx_id = 0;
-	} else {
-		/* The driver assumes no error. */
+	if (!dev->of_node)
 		return 0;
-	}
 
-	ctlr->dma_tx = rspi_request_dma_chan(dev, DMA_MEM_TO_DEV, dma_tx_id,
-					     res->start);
-	if (!ctlr->dma_tx)
-		return -ENODEV;
+	chan = rspi_request_dma_chan(dev, DMA_MEM_TO_DEV, res->start);
+	if (IS_ERR(chan))
+		return PTR_ERR(chan);
+	ctlr->dma_tx = chan;
 
-	ctlr->dma_rx = rspi_request_dma_chan(dev, DMA_DEV_TO_MEM, dma_rx_id,
-					     res->start);
-	if (!ctlr->dma_rx) {
+	chan = rspi_request_dma_chan(dev, DMA_DEV_TO_MEM, res->start);
+	if (IS_ERR(chan)) {
 		dma_release_channel(ctlr->dma_tx);
 		ctlr->dma_tx = NULL;
-		return -ENODEV;
+		return PTR_ERR(chan);
 	}
+	ctlr->dma_rx = chan;
 
 	ctlr->can_dma = rspi_can_dma;
 	dev_info(dev, "DMA available");
@@ -1355,6 +1339,11 @@ static int rspi_probe(struct platform_device *pdev)
 	}
 
 	ret = rspi_request_dma(&pdev->dev, ctlr, res);
+	if (ret == -EPROBE_DEFER) {
+		dev_err_probe(&pdev->dev, ret,
+			      "failed to request DMA channels\n");
+		goto error2;
+	}
 	if (ret < 0)
 		dev_warn(&pdev->dev, "DMA not available, using PIO\n");
 
diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c
index 8d4120cf9df11e..3546414dfc70f7 100644
--- a/drivers/spi/spi-s3c64xx.c
+++ b/drivers/spi/spi-s3c64xx.c
@@ -661,7 +661,7 @@ static int s3c64xx_wait_for_pio(struct s3c64xx_spi_driver_data *sdd,
 	 * loops = length / max fifo size (calculated by using the
 	 * fifo mask).
 	 * For any size less than the fifo size the below code is
-	 * executed atleast once.
+	 * executed at least once.
 	 */
 	loops = xfer->len / sdd->fifo_depth;
 	buf = xfer->rx_buf;
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index 1aeab7ec0bc8d0..90a671e9de70eb 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -453,6 +453,7 @@ static int sh_msiof_spi_setup(struct spi_device *spi)
 	struct sh_msiof_spi_priv *p =
 		spi_controller_get_devdata(spi->controller);
 	u32 clr, set, tmp;
+	int ret;
 
 	if (spi_get_csgpiod(spi, 0) || spi_controller_is_target(p->ctlr))
 		return 0;
@@ -468,7 +469,9 @@ static int sh_msiof_spi_setup(struct spi_device *spi)
 		clr |= SIMDR1_SYNCAC;
 	else
 		set |= SIMDR1_SYNCAC;
-	pm_runtime_get_sync(&p->pdev->dev);
+	ret = pm_runtime_resume_and_get(&p->pdev->dev);
+	if (ret < 0)
+		return ret;
 	tmp = sh_msiof_read(p, SITMDR1) & ~clr;
 	sh_msiof_write(p, SITMDR1, tmp | set | SIMDR1_TRMD | SITMDR1_PCON);
 	tmp = sh_msiof_read(p, SIRMDR1) & ~clr;
@@ -1013,6 +1016,7 @@ static const struct of_device_id sh_msiof_match[] __maybe_unused = {
 	{ .compatible = "renesas,msiof-r8a779a0",  .data = &rcar_gen3_data },
 	{ .compatible = "renesas,msiof-r8a779f0",  .data = &rcar_gen3_data },
 	{ .compatible = "renesas,rcar-gen4-msiof", .data = &rcar_gen4_data },
+	{ .compatible = "renesas,rcar-gen5-msiof", .data = &rcar_gen4_data },
 	{ .compatible = "renesas,sh-msiof",        .data = &sh_data }, /* Deprecated */
 	{ /* sentinel */ }
 };
diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c
index be88e62075af6c..5f5c43ae5096a4 100644
--- a/drivers/spi/spi-stm32.c
+++ b/drivers/spi/spi-stm32.c
@@ -515,7 +515,7 @@ static int stm32h7_spi_get_bpw_mask(struct stm32_spi *spi)
 
 	/*
 	 * The most significant bit at DSIZE bit field is reserved when the
-	 * maximum data size of periperal instances is limited to 16-bit
+	 * maximum data size of peripheral instances is limited to 16-bit
 	 */
 	stm32_spi_set_bits(spi, STM32H7_SPI_CFG1, STM32H7_SPI_CFG1_DSIZE);
 
diff --git a/drivers/spi/spi-sun6i.c b/drivers/spi/spi-sun6i.c
index 4631e9c7ca1dd1..6b7a05dd0de8a5 100644
--- a/drivers/spi/spi-sun6i.c
+++ b/drivers/spi/spi-sun6i.c
@@ -386,7 +386,7 @@ static int sun6i_spi_transfer_one(struct spi_controller *host,
 		 * SPI_CLK = MOD_CLK / (2 ^ cdr)
 		 * Or we can use CDR2, which is calculated with the formula:
 		 * SPI_CLK = MOD_CLK / (2 * (cdr + 1))
-		 * Wether we use the former or the latter is set through the
+		 * Whether we use the former or the latter is set through the
 		 * DRS bit.
 		 *
 		 * First try CDR2, and if we can't reach the expected
diff --git a/drivers/spi/spi-sunplus-sp7021.c b/drivers/spi/spi-sunplus-sp7021.c
index f16fcda187dd98..54431c363bbc9e 100644
--- a/drivers/spi/spi-sunplus-sp7021.c
+++ b/drivers/spi/spi-sunplus-sp7021.c
@@ -337,10 +337,15 @@ static int sp7021_spi_host_transfer_one(struct spi_controller *ctlr, struct spi_
 				      SP7021_SPI_START_FD;
 		writel(reg_temp, pspim->m_base + SP7021_SPI_STATUS_REG);
 
-		if (!wait_for_completion_interruptible_timeout(&pspim->isr_done, timeout)) {
-			dev_err(&spi->dev, "wait_for_completion err\n");
-			mutex_unlock(&pspim->buf_lock);
-			return -ETIMEDOUT;
+		{
+			long ret = wait_for_completion_interruptible_timeout(
+						&pspim->isr_done, timeout);
+			if (ret <= 0) {
+				dev_err(&spi->dev, ret == 0 ? "SPI transfer timeout\n"
+							    : "SPI transfer interrupted\n");
+				mutex_unlock(&pspim->buf_lock);
+				return ret == 0 ? -ETIMEDOUT : -EINTR;
+			}
 		}
 
 		reg_temp = readl(pspim->m_base + SP7021_SPI_STATUS_REG);
diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c
index 8ede864c3d3c26..d8cca3aa276b29 100644
--- a/drivers/spi/spi-tegra210-quad.c
+++ b/drivers/spi/spi-tegra210-quad.c
@@ -191,6 +191,15 @@ struct tegra_qspi {
 	void __iomem				*base;
 	phys_addr_t				phys;
 	unsigned int				irq;
+	struct work_struct			irq_work;
+	struct workqueue_struct			*wq;
+	/*
+	 * Set by tegra_qspi_handle_timeout() while it drains the bottom
+	 * half so tegra_qspi_isr() suppresses new queue_work() calls
+	 * that would otherwise race the recovery path or the caller's
+	 * cleanup of curr_xfer.
+	 */
+	bool					recovery_in_progress;
 
 	u32					cur_speed;
 	unsigned int				cur_pos;
@@ -205,6 +214,18 @@ struct tegra_qspi {
 	unsigned int				dma_buf_size;
 	unsigned int				max_buf_size;
 	bool					is_curr_dma_xfer;
+	/*
+	 * Cached "this PIO chunk completes the whole transfer" decision,
+	 * computed by tegra_qspi_start_cpu_based_transfer() before it
+	 * unmasks the IRQ. Used by the hard IRQ small-PIO fastpath in
+	 * place of dereferencing curr_xfer->len, so the ISR cannot touch
+	 * the spi_transfer object even on a late IRQ that races with the
+	 * synchronous teardown path. Multi-chunk PIO transfers always go
+	 * through the workqueue (this flag is only set on the final
+	 * chunk), so the fastpath cannot recurse into
+	 * tegra_qspi_start_cpu_based_transfer() from hard IRQ context.
+	 */
+	bool					is_last_pio_chunk;
 
 	struct completion			rx_dma_complete;
 	struct completion			tx_dma_complete;
@@ -212,6 +233,7 @@ struct tegra_qspi {
 	u32					tx_status;
 	u32					rx_status;
 	u32					status_reg;
+	u32					trans_status;
 	bool					is_packed;
 	bool					use_dma;
 
@@ -622,6 +644,17 @@ static int tegra_qspi_start_dma_based_transfer(struct tegra_qspi *tqspi, struct
 	val = QSPI_DMA_BLK_SET(tqspi->curr_dma_words - 1);
 	tegra_qspi_writel(tqspi, val, QSPI_DMA_BLK);
 
+	/*
+	 * Reset the cached transfer status before unmasking the IRQ for
+	 * this chunk. The cache must represent only the IRQ for THIS
+	 * chunk; a stale RDY from the previous chunk of a multi-chunk
+	 * transfer would otherwise mislead tegra_qspi_handle_timeout()
+	 * into a false-positive recovery while the new chunk is still in
+	 * flight. Pairs with smp_load_acquire() in
+	 * tegra_qspi_handle_timeout(). The new chunk's IRQ cannot fire
+	 * until QSPI_DMA_CTL is written below.
+	 */
+	smp_store_release(&tqspi->trans_status, 0);
 	tegra_qspi_unmask_irq(tqspi);
 
 	if (tqspi->is_packed)
@@ -713,7 +746,13 @@ static int tegra_qspi_start_dma_based_transfer(struct tegra_qspi *tqspi, struct
 
 	tegra_qspi_writel(tqspi, tqspi->command1_reg, QSPI_COMMAND1);
 
-	tqspi->is_curr_dma_xfer = true;
+	/*
+	 * WRITE_ONCE() pairs with READ_ONCE() in tegra_qspi_isr() and
+	 * tegra_qspi_work_handler(); the flag is read lock-free across
+	 * the hard-IRQ / process-context boundary so the annotation
+	 * prevents compiler tearing and silences KCSAN.
+	 */
+	WRITE_ONCE(tqspi->is_curr_dma_xfer, true);
 	tqspi->dma_control_reg = val;
 	val |= QSPI_DMA_EN;
 	tegra_qspi_writel(tqspi, val, QSPI_DMA_CTL);
@@ -734,9 +773,33 @@ static int tegra_qspi_start_cpu_based_transfer(struct tegra_qspi *qspi, struct s
 	val = QSPI_DMA_BLK_SET(cur_words - 1);
 	tegra_qspi_writel(qspi, val, QSPI_DMA_BLK);
 
+	/*
+	 * Snapshot whether this PIO chunk completes the whole transfer
+	 * before unmasking the IRQ, so the hard IRQ small-PIO fastpath
+	 * can decide whether to drain inline without dereferencing the
+	 * spi_transfer object. cur_pos / curr_dma_words / bytes_per_word
+	 * are stable here: they are written by
+	 * tegra_qspi_calculate_curr_xfer_param() earlier in this code
+	 * path. The IRQ cannot fire until the QSPI_COMMAND1 write below
+	 * kicks the transfer off, so this store happens-before any ISR
+	 * that observes the unmask.
+	 */
+	WRITE_ONCE(qspi->is_last_pio_chunk,
+		   qspi->cur_pos + qspi->curr_dma_words * qspi->bytes_per_word >= t->len);
+
+	/*
+	 * Reset the cached transfer status before unmasking the IRQ for
+	 * this chunk so the cache represents only the IRQ for THIS chunk;
+	 * a stale RDY from the previous chunk would otherwise mislead
+	 * tegra_qspi_handle_timeout() into a false-positive recovery
+	 * while the new chunk is still in flight. Pairs with
+	 * smp_load_acquire() in tegra_qspi_handle_timeout(). The new
+	 * chunk's IRQ cannot fire until QSPI_COMMAND1 is written below.
+	 */
+	smp_store_release(&qspi->trans_status, 0);
 	tegra_qspi_unmask_irq(qspi);
 
-	qspi->is_curr_dma_xfer = false;
+	WRITE_ONCE(qspi->is_curr_dma_xfer, false);
 	val = qspi->command1_reg;
 	val |= QSPI_PIO;
 	tegra_qspi_writel(qspi, val, QSPI_COMMAND1);
@@ -859,6 +922,13 @@ static u32 tegra_qspi_setup_transfer_one(struct spi_device *spi, struct spi_tran
 	tqspi->cur_rx_pos = 0;
 	tqspi->cur_tx_pos = 0;
 	tqspi->curr_xfer = t;
+	/*
+	 * Pairs with smp_load_acquire() in tegra_qspi_handle_timeout().
+	 * Clearing the cached trans_status before unmasking the IRQ for
+	 * the new transfer prevents a stale RDY bit from the previous
+	 * transfer fooling the timeout handler into a false recovery.
+	 */
+	smp_store_release(&tqspi->trans_status, 0);
 	spin_unlock_irqrestore(&tqspi->lock, flags);
 
 	if (is_first_of_msg) {
@@ -1065,40 +1135,206 @@ static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi);
  * tegra_qspi_handle_timeout - Handle transfer timeout with hardware check
  * @tqspi: QSPI controller instance
  *
- * When a timeout occurs but hardware has completed the transfer (interrupt
- * was lost or delayed), manually trigger transfer completion processing.
- * This avoids failing transfers that actually succeeded.
+ * When wait_for_completion_timeout() expires the hardware may still have
+ * finished the current chunk. Drain the pending bottom half and, if the
+ * whole transfer really did complete during the drain, consume the
+ * completion and report success.
+ *
+ * When the bottom half advanced the transfer by only one chunk of a
+ * multi-chunk DMA/PIO transfer without signalling xfer_completion, a
+ * fallback that ran handle_{cpu,dma}_based_xfer() here would race with
+ * the DMA engine already moving the next chunk into the client buffer
+ * (spi_finalize_current_message() would then release the buffer while
+ * the controller is still writing memory). Fake completion is therefore
+ * only attempted when the current chunk is the last chunk of the
+ * transfer; multi-chunk continuation timeouts return -ETIMEDOUT and
+ * let the caller reset the controller.
  *
- * Returns: 0 if transfer was completed, -ETIMEDOUT if real timeout
+ * Returns: 0 if the transfer completed, -ETIMEDOUT otherwise.
  */
 static int tegra_qspi_handle_timeout(struct tegra_qspi *tqspi)
 {
+	struct spi_transfer *t;
+	unsigned long flags;
+	bool is_last_chunk;
+	bool lost_irq_snapshot = false;
 	irqreturn_t ret;
-	u32 status;
+	int retval;
+	u32 status, refreshed;
+	u32 lost_fifo_status = 0;
+	u32 lost_tx_status = 0;
+	u32 lost_rx_status = 0;
 
-	/* Check if hardware actually completed the transfer */
-	status = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS);
-	if (!(status & QSPI_RDY))
-		return -ETIMEDOUT;
+	/*
+	 * Snapshot both the ISR cache and (if the cache is empty) the
+	 * live status registers BEFORE entering recovery. The recovery
+	 * path calls tegra_qspi_mask_clear_irq() below, which performs
+	 * W1Cs on QSPI_TRANS_STATUS and on the QSPI_FIFO_STATUS error
+	 * bits: a lost-IRQ recovery must capture the current FIFO error
+	 * state before the mask erases it.
+	 *
+	 * Cache-live-cache retry: if the initial cache load returns zero
+	 * we fall back to a live QSPI_TRANS_STATUS read, and if that also
+	 * returns zero we retry the cache once more. That closes the
+	 * interleaving where an ISR on another CPU publishes trans_status
+	 * with release semantics and then W1Cs the hardware between our
+	 * cache load and our live load: the second cache load observes
+	 * the now-visible release and we correctly classify the transfer
+	 * as complete rather than reporting a false timeout.
+	 *
+	 * The trans_status cache is reset to zero in
+	 * tegra_qspi_start_{cpu,dma}_based_transfer() before unmasking
+	 * the IRQ for every chunk, so a stale RDY from the previous
+	 * chunk of a multi-chunk transfer cannot survive into this
+	 * check.
+	 */
+	status = smp_load_acquire(&tqspi->trans_status);
+	if (!status) {
+		status = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS);
+		if (!status) {
+			/* Retry cache; pairs with release in ISR post-store. */
+			status = smp_load_acquire(&tqspi->trans_status);
+		} else {
+			/*
+			 * Live register shows RDY but the ISR cache is
+			 * empty: either the ISR ran and cleared HW between
+			 * our two loads (the cache retry above would have
+			 * observed it, so we would not be here), or the IRQ
+			 * was genuinely lost. Snapshot the live FIFO error
+			 * status now so tegra_qspi_mask_clear_irq() below
+			 * does not W1C it away before the manual handler
+			 * downstream can see it.
+			 */
+			lost_fifo_status = tegra_qspi_readl(tqspi,
+							    QSPI_FIFO_STATUS);
+			lost_tx_status = lost_fifo_status &
+					 (QSPI_TX_FIFO_UNF | QSPI_TX_FIFO_OVF);
+			lost_rx_status = lost_fifo_status &
+					 (QSPI_RX_FIFO_OVF | QSPI_RX_FIFO_UNF);
+			lost_irq_snapshot = true;
+		}
+	}
 
 	/*
-	 * Hardware completed but interrupt was lost/delayed. Manually
-	 * process the completion by calling the appropriate handler.
+	 * Enter recovery unconditionally. Every expired
+	 * wait_for_completion_timeout() must serialise against a delayed
+	 * ISR or worker before the caller runs dma_stop() +
+	 * device_reset() + curr_xfer clear: publishing
+	 * recovery_in_progress under tqspi->lock, masking the controller
+	 * IRQ, calling synchronize_irq() to drain any in-flight ISR
+	 * (including the small-PIO hard-IRQ fastpath), and finally
+	 * cancel_work_sync() to drain the workqueue gives us that
+	 * serialisation regardless of whether the hardware finished. A
+	 * genuine hardware timeout still ends up as -ETIMEDOUT further
+	 * down, but only after ISR and workqueue activity are quiesced.
+	 *
+	 * cancel_work_sync() cancels a pending worker without executing
+	 * it and waits for a currently running one to finish; the
+	 * recovery_in_progress guard checked inside tegra_qspi_isr()
+	 * under tqspi->lock is atomic with its queue_work() and small-PIO
+	 * fastpath dispatch decisions, so no new bottom-half work is
+	 * enqueued once we publish the flag.
+	 *
+	 * tegra_qspi_mask_clear_irq() is idempotent: its read-modify-write
+	 * of QSPI_INTR_MASK and W1C of QSPI_TRANS_STATUS / FIFO error
+	 * status all tolerate a double-write, so it is safe whether or
+	 * not the ISR has already run for this transfer.
 	 */
+	spin_lock_irqsave(&tqspi->lock, flags);
+	WRITE_ONCE(tqspi->recovery_in_progress, true);
+	spin_unlock_irqrestore(&tqspi->lock, flags);
+
+	tegra_qspi_mask_clear_irq(tqspi);
+	synchronize_irq(tqspi->irq);
+	cancel_work_sync(&tqspi->irq_work);
+
+	if (try_wait_for_completion(&tqspi->xfer_completion)) {
+		retval = 0;
+		goto out;
+	}
+
+	/*
+	 * Re-check the cache after the drain: the worker we just drained
+	 * may have published a completion status the entry snapshot did
+	 * not observe (for example the ISR fired on another CPU after we
+	 * loaded the cache but before we masked).
+	 */
+	refreshed = smp_load_acquire(&tqspi->trans_status);
+	if (refreshed)
+		status = refreshed;
+
+	if (!(status & QSPI_RDY)) {
+		retval = -ETIMEDOUT;
+		goto out;
+	}
+
+	/*
+	 * If the ISR never ran (lost IRQ path) publish the FIFO error
+	 * snapshot we captured before mask_clear_irq() so the manual
+	 * handler downstream has fresh error state rather than stale
+	 * fields from a previous chunk's ISR.
+	 */
+	if (lost_irq_snapshot) {
+		WRITE_ONCE(tqspi->status_reg, lost_fifo_status);
+		WRITE_ONCE(tqspi->tx_status, lost_tx_status);
+		WRITE_ONCE(tqspi->rx_status, lost_rx_status);
+	}
+
+	/*
+	 * The bottom half did not signal full completion. Either the work
+	 * ran and advanced the transfer by one chunk (possibly arming the
+	 * next chunk of a multi-chunk transfer), or it was cancelled
+	 * before it could run, or the current chunk really did not
+	 * complete. Only fake completion when the current chunk is the
+	 * last chunk of the transfer; otherwise the DMA engine may still
+	 * be moving the next chunk into memory, and returning 0 here would
+	 * let seq_xfer clear curr_xfer and finalise the message while the
+	 * hardware is still writing.
+	 *
+	 * The last-chunk arithmetic mirrors tegra_qspi_start_cpu_based_
+	 * transfer(), which uses cur_pos + curr_dma_words * bytes_per_word
+	 * >= t->len to set is_last_pio_chunk before arming the IRQ.
+	 */
+	spin_lock_irqsave(&tqspi->lock, flags);
+	t = tqspi->curr_xfer;
+	if (!t) {
+		/* CPU-path handler already cleared curr_xfer */
+		spin_unlock_irqrestore(&tqspi->lock, flags);
+		retval = 0;
+		goto out;
+	}
+	is_last_chunk = (tqspi->cur_pos +
+			 tqspi->curr_dma_words * tqspi->bytes_per_word) >= t->len;
+	spin_unlock_irqrestore(&tqspi->lock, flags);
+
+	if (!is_last_chunk) {
+		retval = -ETIMEDOUT;
+		goto out;
+	}
+
 	dev_warn_ratelimited(tqspi->dev,
 			     "QSPI interrupt timeout, but transfer complete\n");
 
-	/* Clear the transfer status */
-	status = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS);
-	tegra_qspi_writel(tqspi, status, QSPI_TRANS_STATUS);
-
-	/* Manually trigger completion handler */
-	if (!tqspi->is_curr_dma_xfer)
+	if (!READ_ONCE(tqspi->is_curr_dma_xfer))
 		ret = handle_cpu_based_xfer(tqspi);
 	else
 		ret = handle_dma_based_xfer(tqspi);
 
-	return (ret == IRQ_HANDLED) ? 0 : -EIO;
+	retval = (ret == IRQ_HANDLED) ? 0 : -EIO;
+
+out:
+	/*
+	 * The drained bottom half may have unmasked the controller IRQ
+	 * to arm the next chunk of a multi-chunk transfer. Re-mask and
+	 * synchronize before clearing recovery_in_progress so that no
+	 * lingering ISR can queue fresh work behind the caller's back
+	 * (the caller's dma_stop() + device_reset() + curr_xfer clear
+	 * runs immediately after we return on the error path).
+	 */
+	tegra_qspi_mask_clear_irq(tqspi);
+	synchronize_irq(tqspi->irq);
+	WRITE_ONCE(tqspi->recovery_in_progress, false);
+	return retval;
 }
 
 static u32 tegra_qspi_cmd_config(bool is_ddr, u8 bus_width, u8 len)
@@ -1232,9 +1468,9 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi,
 
 			if (ret == 0) {
 				/*
-				 * Check if hardware completed the transfer
-				 * even though interrupt was lost or delayed.
-				 * If so, process the completion and continue.
+				 * Check if hardware completed the transfer even though
+				 * workqueue was delayed. If so, process completion and
+				 * continue.
 				 */
 				ret = tegra_qspi_handle_timeout(tqspi);
 				if (ret < 0) {
@@ -1351,8 +1587,8 @@ static int tegra_qspi_non_combined_seq_xfer(struct tegra_qspi *tqspi,
 		if (ret == 0) {
 			/*
 			 * Check if hardware completed the transfer even though
-			 * interrupt was lost or delayed. If so, process the
-			 * completion and continue.
+			 * workqueue was delayed. If so, process completion and
+			 * continue.
 			 */
 			ret = tegra_qspi_handle_timeout(tqspi);
 			if (ret < 0) {
@@ -1506,6 +1742,19 @@ static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi)
 	long wait_status;
 	int num_errors = 0;
 
+	/*
+	 * Snapshot curr_xfer under the lock before the (potentially long)
+	 * DMA waits below. The timeout path can clear tqspi->curr_xfer
+	 * concurrently; using the local copy keeps the subsequent dma_unmap
+	 * and FIFO-drain steps consistent with the transfer that actually
+	 * started, and lets us bail safely if cleanup already happened.
+	 */
+	spin_lock_irqsave(&tqspi->lock, flags);
+	t = tqspi->curr_xfer;
+	spin_unlock_irqrestore(&tqspi->lock, flags);
+	if (!t)
+		return IRQ_HANDLED;
+
 	if (tqspi->cur_direction & DATA_DIR_TX) {
 		if (tqspi->tx_status) {
 			if (tqspi->tx_dma_chan)
@@ -1539,12 +1788,6 @@ static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi)
 	}
 
 	spin_lock_irqsave(&tqspi->lock, flags);
-	t = tqspi->curr_xfer;
-
-	if (!t) {
-		spin_unlock_irqrestore(&tqspi->lock, flags);
-		return IRQ_HANDLED;
-	}
 
 	if (num_errors) {
 		tegra_qspi_dma_unmap_xfer(tqspi, t);
@@ -1581,46 +1824,41 @@ static irqreturn_t handle_dma_based_xfer(struct tegra_qspi *tqspi)
 	return IRQ_HANDLED;
 }
 
-static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data)
+/**
+ * tegra_qspi_work_handler - Workqueue handler for interrupt bottom-half
+ * @work: work_struct embedded in tegra_qspi
+ *
+ * Runs in process context and can sleep (needed for DMA completion waits).
+ * Runs on any CPU in the WQ_UNBOUND pool, so the bottom half can migrate off
+ * the interrupt-taking CPU that the previous threaded IRQ pinned to
+ * (irq_thread() calls set_cpus_allowed_ptr() with the IRQ affinity mask).
+ *
+ * The hard IRQ handler has already:
+ * - Verified this is our interrupt (QSPI_RDY was set)
+ * - Cached FIFO status in tqspi->status_reg
+ * - Parsed tx_status / rx_status from FIFO status
+ * - Masked further interrupts
+ */
+static void tegra_qspi_work_handler(struct work_struct *work)
 {
-	struct tegra_qspi *tqspi = context_data;
+	struct tegra_qspi *tqspi = container_of(work, struct tegra_qspi, irq_work);
 	unsigned long flags;
-	u32 status;
 
-	/*
-	 * Read transfer status to check if interrupt was triggered by transfer
-	 * completion
-	 */
-	status = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS);
+	spin_lock_irqsave(&tqspi->lock, flags);
 
 	/*
-	 * Occasionally the IRQ thread takes a long time to wake up (usually
-	 * when the CPU that it's running on is excessively busy) and we have
-	 * already reached the timeout before and cleaned up the timed out
-	 * transfer. Avoid any processing in that case and bail out early.
-	 *
-	 * If no transfer is in progress, check if this was a real interrupt
-	 * that the timeout handler already processed, or a spurious one.
+	 * tegra_qspi_handle_timeout() sets recovery_in_progress under
+	 * tqspi->lock and then calls cancel_work_sync(), so any running
+	 * worker is drained and tegra_qspi_isr() cannot enqueue a new
+	 * one while recovery runs. The curr_xfer NULL check catches the
+	 * case where the timeout path already tore the transfer down
+	 * before this work got a chance to run.
 	 */
-	spin_lock_irqsave(&tqspi->lock, flags);
 	if (!tqspi->curr_xfer) {
 		spin_unlock_irqrestore(&tqspi->lock, flags);
-		/* Spurious interrupt - transfer not ready */
-		if (!(status & QSPI_RDY))
-			return IRQ_NONE;
-		/* Real interrupt, already handled by timeout path */
-		return IRQ_HANDLED;
+		return;
 	}
 
-	tqspi->status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
-
-	if (tqspi->cur_direction & DATA_DIR_TX)
-		tqspi->tx_status = tqspi->status_reg & (QSPI_TX_FIFO_UNF | QSPI_TX_FIFO_OVF);
-
-	if (tqspi->cur_direction & DATA_DIR_RX)
-		tqspi->rx_status = tqspi->status_reg & (QSPI_RX_FIFO_OVF | QSPI_RX_FIFO_UNF);
-
-	tegra_qspi_mask_clear_irq(tqspi);
 	spin_unlock_irqrestore(&tqspi->lock, flags);
 
 	/*
@@ -1629,10 +1867,127 @@ static irqreturn_t tegra_qspi_isr_thread(int irq, void *context_data)
 	 * DMA handler also needs to sleep in wait_for_completion_*(), which
 	 * cannot be done while holding spinlock.
 	 */
-	if (!tqspi->is_curr_dma_xfer)
+	if (!READ_ONCE(tqspi->is_curr_dma_xfer))
+		handle_cpu_based_xfer(tqspi);
+	else
+		handle_dma_based_xfer(tqspi);
+}
+
+/**
+ * tegra_qspi_isr - Hard IRQ handler
+ * @irq: IRQ number
+ * @context_data: QSPI controller instance
+ *
+ * Runs in hard IRQ context with minimal latency. Cannot sleep.
+ *
+ * Tegra QSPI uses a dedicated, non-shared GIC SPI line on every SoC that
+ * uses this driver. The handler always returns IRQ_HANDLED and always
+ * acknowledges/re-masks the controller IRQ, so the level-triggered line
+ * cannot stay asserted and trip the kernel spurious-IRQ detector into
+ * disabling the line. On a stray IRQ where curr_xfer is NULL (e.g. the
+ * timeout path has already torn the transfer down) the FIFO/status
+ * processing and bottom-half scheduling are skipped because there is no
+ * transfer to drive forward.
+ *
+ * Return: IRQ_HANDLED.
+ */
+static irqreturn_t tegra_qspi_isr(int irq, void *context_data)
+{
+	struct tegra_qspi *tqspi = context_data;
+	u32 status_reg, trans_status;
+	u32 tx_status = 0, rx_status = 0;
+
+	if (!READ_ONCE(tqspi->curr_xfer)) {
+		tegra_qspi_mask_clear_irq(tqspi);
+		return IRQ_HANDLED;
+	}
+
+	spin_lock(&tqspi->lock);
+	status_reg = tegra_qspi_readl(tqspi, QSPI_FIFO_STATUS);
+	trans_status = tegra_qspi_readl(tqspi, QSPI_TRANS_STATUS);
+
+	if (tqspi->cur_direction & DATA_DIR_TX) {
+		tx_status = status_reg & (QSPI_TX_FIFO_UNF | QSPI_TX_FIFO_OVF);
+		WRITE_ONCE(tqspi->tx_status, tx_status);
+	}
+
+	if (tqspi->cur_direction & DATA_DIR_RX) {
+		rx_status = status_reg & (QSPI_RX_FIFO_OVF | QSPI_RX_FIFO_UNF);
+		WRITE_ONCE(tqspi->rx_status, rx_status);
+	}
+
+	WRITE_ONCE(tqspi->status_reg, status_reg);
+	/*
+	 * Publish trans_status with release semantics before we clear
+	 * the hardware status in tegra_qspi_mask_clear_irq() below. That
+	 * ordering matters for the lock-free cache read in
+	 * tegra_qspi_handle_timeout(): if the timeout path sees the
+	 * released trans_status it also observes the matching status_reg
+	 * / tx_status / rx_status; if it does not yet see the released
+	 * value it falls back to a live QSPI_TRANS_STATUS read, and that
+	 * live read still returns QSPI_RDY because we have not cleared
+	 * the register yet. Reversing this order would open a window
+	 * where the cache is still zero but the hardware bit has already
+	 * been cleared, making the fallback report a false timeout.
+	 */
+	smp_store_release(&tqspi->trans_status, trans_status);
+
+	tegra_qspi_mask_clear_irq(tqspi);
+
+	/*
+	 * If tegra_qspi_handle_timeout() is draining the bottom half,
+	 * skip queueing new work. The flag is set under tqspi->lock and
+	 * queue_work() below happens while we still hold the lock, so
+	 * the guard is atomic with the queue decision. Any ISR that had
+	 * already passed this check is drained by the synchronize_irq()
+	 * call that tegra_qspi_handle_timeout() issues after publishing
+	 * the flag.
+	 */
+	if (READ_ONCE(tqspi->recovery_in_progress)) {
+		spin_unlock(&tqspi->lock);
+		return IRQ_HANDLED;
+	}
+
+	/*
+	 * Small-PIO fastpath: drain the FIFO inline only when this chunk
+	 * completes the entire outstanding transfer and no error bit was
+	 * latched, to avoid workqueue scheduling latency for TPM-style
+	 * short reads.
+	 *
+	 * The "last chunk" decision is computed and cached as a scalar by
+	 * tegra_qspi_start_cpu_based_transfer() before it unmasks the IRQ,
+	 * so the hard-IRQ fastpath never dereferences the spi_transfer
+	 * pointer here. That keeps the ISR safe against any teardown race
+	 * where the synchronous path could clear curr_xfer concurrently.
+	 *
+	 * The fastpath dispatch decision is made while still holding
+	 * tqspi->lock, so the recovery_in_progress guard above covers it
+	 * atomically with queue_work() below: an ISR that reaches the
+	 * fastpath cannot race a tegra_qspi_handle_timeout() that
+	 * subsequently observes recovery_in_progress == true, because
+	 * that path calls synchronize_irq() before proceeding. We drop
+	 * the lock before calling handle_cpu_based_xfer() so it can take
+	 * tqspi->lock internally without deadlocking.
+	 *
+	 * Multi-chunk PIO continuation stays on the workqueue so that
+	 * tegra_qspi_start_cpu_based_transfer() can re-arm the IRQ from
+	 * process context. DMA transfers also stay on the workqueue
+	 * because their completion path sleeps on the DMA engine.
+	 * tegra_qspi_handle_error() -> device_reset() can sleep, so the
+	 * fastpath only runs when both status words are clean.
+	 */
+	if (!READ_ONCE(tqspi->is_curr_dma_xfer) &&
+	    READ_ONCE(tqspi->is_last_pio_chunk) &&
+	    !tx_status && !rx_status) {
+		spin_unlock(&tqspi->lock);
 		return handle_cpu_based_xfer(tqspi);
+	}
+
+	queue_work(tqspi->wq, &tqspi->irq_work);
 
-	return handle_dma_based_xfer(tqspi);
+	spin_unlock(&tqspi->lock);
+
+	return IRQ_HANDLED;
 }
 
 static struct tegra_qspi_soc_data tegra210_qspi_soc_data = {
@@ -1800,12 +2155,21 @@ static int tegra_qspi_probe(struct platform_device *pdev)
 
 	pm_runtime_put_autosuspend(&pdev->dev);
 
-	ret = request_threaded_irq(tqspi->irq, NULL,
-				   tegra_qspi_isr_thread, IRQF_ONESHOT,
-				   dev_name(&pdev->dev), tqspi);
+	tqspi->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_UNBOUND, 0,
+				    dev_name(&pdev->dev));
+	if (!tqspi->wq) {
+		dev_err(&pdev->dev, "failed to allocate workqueue\n");
+		ret = -ENOMEM;
+		goto exit_pm_disable;
+	}
+
+	INIT_WORK(&tqspi->irq_work, tegra_qspi_work_handler);
+
+	ret = request_irq(tqspi->irq, tegra_qspi_isr, 0,
+			  dev_name(&pdev->dev), tqspi);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", tqspi->irq, ret);
-		goto exit_pm_disable;
+		goto exit_destroy_wq;
 	}
 
 	ret = spi_register_controller(host);
@@ -1817,7 +2181,9 @@ static int tegra_qspi_probe(struct platform_device *pdev)
 	return 0;
 
 exit_free_irq:
-	free_irq(qspi_irq, tqspi);
+	free_irq(tqspi->irq, tqspi);
+exit_destroy_wq:
+	destroy_workqueue(tqspi->wq);
 exit_pm_disable:
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_force_suspend(&pdev->dev);
@@ -1830,8 +2196,15 @@ static void tegra_qspi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct tegra_qspi *tqspi = spi_controller_get_devdata(host);
 
+	/*
+	 * Tear down in reverse order of probe() so that the controller stops
+	 * accepting transfers before the IRQ is released, no new work can be
+	 * queued after the IRQ is freed, and any work already queued is
+	 * drained while the clocks are still running.
+	 */
 	spi_unregister_controller(host);
 	free_irq(tqspi->irq, tqspi);
+	destroy_workqueue(tqspi->wq);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_force_suspend(&pdev->dev);
 	tegra_qspi_deinit_dma(tqspi);
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index 34b154922ff24d..fb0d940883fb93 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -855,7 +855,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	qspi->rx_bb_addr = dma_alloc_coherent(qspi->dev,
 					      QSPI_DMA_BUFFER_SIZE,
 					      &qspi->rx_bb_dma_addr,
-					      GFP_KERNEL | GFP_DMA);
+					      GFP_KERNEL);
 	if (!qspi->rx_bb_addr) {
 		dev_err(qspi->dev,
 			"dma_alloc_coherent failed, using PIO mode\n");
diff --git a/drivers/spi/spi-virtio.c b/drivers/spi/spi-virtio.c
index 2256dfec5407da..26f9711c99b39a 100644
--- a/drivers/spi/spi-virtio.c
+++ b/drivers/spi/spi-virtio.c
@@ -31,10 +31,6 @@ struct virtio_spi_priv {
 	struct virtio_device *vdev;
 	/* Pointer to the virtqueue */
 	struct virtqueue *vq;
-	/* Copy of config space mode_func_supported */
-	u32 mode_func_supported;
-	/* Copy of config space max_freq_hz */
-	u32 max_freq_hz;
 };
 
 static void virtio_spi_msg_done(struct virtqueue *vq)
@@ -162,13 +158,11 @@ static int virtio_spi_transfer_one(struct spi_controller *ctrl,
 	if (!spi_req)
 		return -ENOMEM;
 
-	init_completion(&spi_req->completion);
-
 	th = &spi_req->transfer_head;
 
 	/* Fill struct spi_transfer_head */
 	th->chip_select_id = spi_get_chipselect(spi, 0);
-	th->bits_per_word = spi->bits_per_word;
+	th->bits_per_word = xfer->bits_per_word;
 	th->cs_change = xfer->cs_change;
 	th->tx_nbits = xfer->tx_nbits;
 	th->rx_nbits = xfer->rx_nbits;
@@ -258,7 +252,7 @@ static int virtio_spi_transfer_one(struct spi_controller *ctrl,
 static void virtio_spi_read_config(struct virtio_device *vdev)
 {
 	struct spi_controller *ctrl = dev_get_drvdata(&vdev->dev);
-	struct virtio_spi_priv *priv = vdev->priv;
+	u32 mode_func_supported;
 	u8 cs_max_number;
 	u8 tx_nbits_supported;
 	u8 rx_nbits_supported;
@@ -268,18 +262,18 @@ static void virtio_spi_read_config(struct virtio_device *vdev)
 	ctrl->num_chipselect = cs_max_number;
 
 	/* Set the mode bits which are understood by this driver */
-	priv->mode_func_supported =
+	mode_func_supported =
 		virtio_cread32(vdev, offsetof(struct virtio_spi_config,
 					      mode_func_supported));
-	ctrl->mode_bits = priv->mode_func_supported &
+	ctrl->mode_bits = mode_func_supported &
 			  (VIRTIO_SPI_CS_HIGH | VIRTIO_SPI_MODE_LSB_FIRST);
-	if (priv->mode_func_supported & VIRTIO_SPI_MF_SUPPORT_CPHA_1)
+	if (mode_func_supported & VIRTIO_SPI_MF_SUPPORT_CPHA_1)
 		ctrl->mode_bits |= VIRTIO_SPI_CPHA;
-	if (priv->mode_func_supported & VIRTIO_SPI_MF_SUPPORT_CPOL_1)
+	if (mode_func_supported & VIRTIO_SPI_MF_SUPPORT_CPOL_1)
 		ctrl->mode_bits |= VIRTIO_SPI_CPOL;
-	if (priv->mode_func_supported & VIRTIO_SPI_MF_SUPPORT_LSB_FIRST)
+	if (mode_func_supported & VIRTIO_SPI_MF_SUPPORT_LSB_FIRST)
 		ctrl->mode_bits |= SPI_LSB_FIRST;
-	if (priv->mode_func_supported & VIRTIO_SPI_MF_SUPPORT_LOOPBACK)
+	if (mode_func_supported & VIRTIO_SPI_MF_SUPPORT_LOOPBACK)
 		ctrl->mode_bits |= SPI_LOOP;
 	tx_nbits_supported =
 		virtio_cread8(vdev, offsetof(struct virtio_spi_config,
@@ -304,7 +298,7 @@ static void virtio_spi_read_config(struct virtio_device *vdev)
 		virtio_cread32(vdev, offsetof(struct virtio_spi_config,
 					      bits_per_word_mask));
 
-	priv->max_freq_hz =
+	ctrl->max_speed_hz =
 		virtio_cread32(vdev, offsetof(struct virtio_spi_config,
 					      max_freq_hz));
 }
diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c
index dc1e968659e1b4..df4ea99a5427d5 100644
--- a/drivers/spi/spi-xilinx.c
+++ b/drivers/spi/spi-xilinx.c
@@ -460,7 +460,7 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 	host->num_chipselect = num_cs;
 
 	/*
-	 * Detect endianess on the IP via loop bit in CR. Detection
+	 * Detect endianness on the IP via loop bit in CR. Detection
 	 * must be done before reset is sent because incorrect reset
 	 * value generates error interrupt.
 	 * Setup little endian helper functions first and try to use them
diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c
index 4d55090fa44337..15e9d3ef8839d6 100644
--- a/drivers/spi/spi-zynqmp-gqspi.c
+++ b/drivers/spi/spi-zynqmp-gqspi.c
@@ -1373,11 +1373,45 @@ static void zynqmp_qspi_remove(struct platform_device *pdev)
 	clk_disable_unprepare(xqspi->pclk);
 }
 
+static void zynqmp_qspi_shutdown(struct platform_device *pdev)
+{
+	struct zynqmp_qspi *xqspi = platform_get_drvdata(pdev);
+	int ret;
+
+	/*
+	 * Stop the queue and reject any later transfer first, so the write
+	 * below cannot cut into a message that is still being executed.
+	 * Unlike ->suspend this cannot abort on error: a controller left
+	 * mastering the bus is worse than a truncated transfer.
+	 */
+	ret = spi_controller_suspend(xqspi->ctlr);
+	if (ret)
+		dev_warn(&pdev->dev, "could not stop the queue: %d\n", ret);
+
+	/*
+	 * Only a runtime suspended controller can be left alone: its clocks
+	 * are gated, so it cannot be mastering the bus, and its registers
+	 * must not be accessed either.  Any other answer means it may be
+	 * running and has to be stopped.  In particular, on a kernel built
+	 * without runtime PM this returns -EINVAL, and there the clocks
+	 * enabled in probe() are never gated at all.
+	 */
+	ret = pm_runtime_get_if_in_use(&pdev->dev);
+	if (!ret)
+		return;
+
+	zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0);
+
+	if (ret > 0)
+		pm_runtime_put_noidle(&pdev->dev);
+}
+
 MODULE_DEVICE_TABLE(of, zynqmp_qspi_of_match);
 
 static struct platform_driver zynqmp_qspi_driver = {
 	.probe = zynqmp_qspi_probe,
 	.remove = zynqmp_qspi_remove,
+	.shutdown = zynqmp_qspi_shutdown,
 	.driver = {
 		.name = "zynqmp-qspi",
 		.of_match_table = zynqmp_qspi_of_match,
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 5b5b3bc5f0d8cf..015d93680e5d0b 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -929,6 +929,15 @@ static int __spi_add_device(struct spi_device *spi, struct spi_device *parent)
 		}
 	}
 
+	/*
+	 * Peripheral properties the core handles on behalf of controller
+	 * drivers are parsed here, rather than in the firmware specific
+	 * instantiation paths, so that device tree, ACPI and software nodes
+	 * are covered alike, and early enough for ->setup() to act on them.
+	 */
+	device_property_read_u32(&spi->dev, "rx-sample-delay-ns",
+				 &spi->rx_sample_delay_ns);
+
 	/*
 	 * Drivers may modify this initial i/o setup, but will
 	 * normally rely on the device being setup.  Devices
@@ -1456,14 +1465,14 @@ int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
 		return 0;
 
 	if (ctlr->dma_tx)
-		tx_dev = ctlr->dma_tx->device->dev;
+		tx_dev = dmaengine_get_dma_device(ctlr->dma_tx);
 	else if (ctlr->dma_map_dev)
 		tx_dev = ctlr->dma_map_dev;
 	else
 		tx_dev = ctlr->dev.parent;
 
 	if (ctlr->dma_rx)
-		rx_dev = ctlr->dma_rx->device->dev;
+		rx_dev = dmaengine_get_dma_device(ctlr->dma_rx);
 	else if (ctlr->dma_map_dev)
 		rx_dev = ctlr->dma_map_dev;
 	else
@@ -2893,7 +2902,7 @@ static void of_register_spi_devices(struct spi_controller *ctlr) { }
  *
  * This may only be called from main SPI device's probe routine.
  *
- * Return: 0 on success; negative errno on failure
+ * Return: the new device on success; an ERR_PTR() on failure
  */
 struct spi_device *spi_new_ancillary_device(struct spi_device *spi,
 					     u8 chip_select)
diff --git a/drivers/staging/axis-fifo/axis-fifo.c b/drivers/staging/axis-fifo/axis-fifo.c
index 3d358f9193523c..7bb35e11d1dd39 100644
--- a/drivers/staging/axis-fifo/axis-fifo.c
+++ b/drivers/staging/axis-fifo/axis-fifo.c
@@ -212,9 +212,9 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf,
 			       size_t len, loff_t *off)
 {
 	struct axis_fifo *fifo = f->private_data;
-	unsigned int words_to_write;
+	size_t words_to_write;
 	u32 *txbuf;
-	int ret;
+	ssize_t ret;
 
 	words_to_write = len / sizeof(u32);
 
@@ -256,7 +256,7 @@ static ssize_t axis_fifo_write(struct file *f, const char __user *buf,
 		goto end_unlock;
 	}
 
-	for (int i = 0; i < words_to_write; ++i)
+	for (size_t i = 0; i < words_to_write; ++i)
 		iowrite32(txbuf[i], fifo->base_addr + XLLF_TDFD_OFFSET);
 
 	iowrite32(len, fifo->base_addr + XLLF_TLR_OFFSET);
diff --git a/drivers/staging/fbtft/fb_bd663474.c b/drivers/staging/fbtft/fb_bd663474.c
index 1629c2c440a97d..c34f859043ef66 100644
--- a/drivers/staging/fbtft/fb_bd663474.c
+++ b/drivers/staging/fbtft/fb_bd663474.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+
 /*
- * FB driver for the uPD161704 LCD Controller
+ * FB driver for the BD663474 LCD Controller
  *
  * Copyright (C) 2014 Seong-Woo Kim
  *
@@ -166,6 +166,6 @@ MODULE_ALIAS("platform:" DRVNAME);
 MODULE_ALIAS("spi:bd663474");
 MODULE_ALIAS("platform:bd663474");
 
-MODULE_DESCRIPTION("FB driver for the uPD161704 LCD Controller");
+MODULE_DESCRIPTION("FB driver for the BD663474 LCD Controller");
 MODULE_AUTHOR("Seong-Woo Kim");
 MODULE_LICENSE("GPL");
diff --git a/drivers/staging/greybus/loopback.c b/drivers/staging/greybus/loopback.c
index 2a623fc2e99b10..fe48a687377a7f 100644
--- a/drivers/staging/greybus/loopback.c
+++ b/drivers/staging/greybus/loopback.c
@@ -562,8 +562,7 @@ static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len)
 	return retval;
 }
 
-static int gb_loopback_async_transfer_complete(
-				struct gb_loopback_async_operation *op_async)
+static int gb_loopback_async_transfer_complete(struct gb_loopback_async_operation *op_async)
 {
 	struct gb_loopback *gb;
 	struct gb_operation *operation;
@@ -653,7 +652,7 @@ static int gb_loopback_request_handler(struct gb_operation *operation)
 		}
 
 		if (!gb_operation_response_alloc(operation,
-				len + sizeof(*response), GFP_KERNEL)) {
+						 len + sizeof(*response), GFP_KERNEL)) {
 			dev_err(dev, "error allocating response\n");
 			return -ENOMEM;
 		}
diff --git a/drivers/staging/greybus/sdio.c b/drivers/staging/greybus/sdio.c
index 3952f3d225dbe2..642d785fe918fc 100644
--- a/drivers/staging/greybus/sdio.c
+++ b/drivers/staging/greybus/sdio.c
@@ -767,17 +767,15 @@ static int gb_sdio_probe(struct gbphy_device *gbphy_dev,
 	struct gb_sdio_host *host;
 	int ret = 0;
 
-	mmc = mmc_alloc_host(sizeof(*host), &gbphy_dev->dev);
+	mmc = devm_mmc_alloc_host(&gbphy_dev->dev, sizeof(*host));
 	if (!mmc)
 		return -ENOMEM;
 
 	connection = gb_connection_create(gbphy_dev->bundle,
 					  le16_to_cpu(gbphy_dev->cport_desc->id),
 					  gb_sdio_request_handler);
-	if (IS_ERR(connection)) {
-		ret = PTR_ERR(connection);
-		goto exit_mmc_free;
-	}
+	if (IS_ERR(connection))
+		return PTR_ERR(connection);
 
 	host = mmc_priv(mmc);
 	host->mmc = mmc;
@@ -835,8 +833,6 @@ static int gb_sdio_probe(struct gbphy_device *gbphy_dev,
 	gb_connection_disable(connection);
 exit_connection_destroy:
 	gb_connection_destroy(connection);
-exit_mmc_free:
-	mmc_free_host(mmc);
 
 	return ret;
 }
@@ -863,7 +859,6 @@ static void gb_sdio_remove(struct gbphy_device *gbphy_dev)
 	mmc_remove_host(mmc);
 	gb_connection_disable(connection);
 	gb_connection_destroy(connection);
-	mmc_free_host(mmc);
 }
 
 static const struct gbphy_device_id gb_sdio_id_table[] = {
diff --git a/drivers/staging/iio/Documentation/sysfs-bus-iio-adc-ad7280a b/drivers/staging/iio/Documentation/sysfs-bus-iio-adc-ad7280a
index 863d3856718aa7..d19db00acd1de2 100644
--- a/drivers/staging/iio/Documentation/sysfs-bus-iio-adc-ad7280a
+++ b/drivers/staging/iio/Documentation/sysfs-bus-iio-adc-ad7280a
@@ -4,7 +4,7 @@ Contact:	linux-iio@vger.kernel.org
 Description:
 		Writing 1 enables the cell balance output switch corresponding
 		to input Y. Writing 0 disables it. If the inY-inZ_balance_timer
-		is set to a none zero value, the corresponding switch will
+		is set to a non-zero value, the corresponding switch will
 		enable for the programmed amount of time, before it
 		automatically disables.
 
diff --git a/drivers/staging/iio/Kconfig b/drivers/staging/iio/Kconfig
index a60631c1f449bb..bf6addf2974f33 100644
--- a/drivers/staging/iio/Kconfig
+++ b/drivers/staging/iio/Kconfig
@@ -5,7 +5,6 @@
 menu "IIO staging drivers"
 	depends on IIO
 
-source "drivers/staging/iio/accel/Kconfig"
 source "drivers/staging/iio/adc/Kconfig"
 source "drivers/staging/iio/addac/Kconfig"
 source "drivers/staging/iio/frequency/Kconfig"
diff --git a/drivers/staging/iio/Makefile b/drivers/staging/iio/Makefile
index 62858353539326..739e4cbf9a7afa 100644
--- a/drivers/staging/iio/Makefile
+++ b/drivers/staging/iio/Makefile
@@ -3,7 +3,6 @@
 # Makefile for the industrial I/O core.
 #
 
-obj-y += accel/
 obj-y += adc/
 obj-y += addac/
 obj-y += frequency/
diff --git a/drivers/staging/iio/accel/Kconfig b/drivers/staging/iio/accel/Kconfig
deleted file mode 100644
index cee51f64bc4bb1..00000000000000
--- a/drivers/staging/iio/accel/Kconfig
+++ /dev/null
@@ -1,19 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Accelerometer drivers
-#
-menu "Accelerometers"
-
-config ADIS16203
-	tristate "Analog Devices ADIS16203 Programmable 360 Degrees Inclinometer"
-	depends on SPI
-	select IIO_ADIS_LIB
-	select IIO_ADIS_LIB_BUFFER if IIO_BUFFER
-	help
-	  Say Y here to build support for Analog Devices adis16203 Programmable
-	  360 Degrees Inclinometer.
-
-	  To compile this driver as a module, say M here: the module will be
-	  called adis16203.
-
-endmenu
diff --git a/drivers/staging/iio/accel/Makefile b/drivers/staging/iio/accel/Makefile
deleted file mode 100644
index acac7bc9b9c0b2..00000000000000
--- a/drivers/staging/iio/accel/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Makefile for industrial I/O accelerometer drivers
-#
-
-obj-$(CONFIG_ADIS16203) += adis16203.o
diff --git a/drivers/staging/iio/accel/adis16203.c b/drivers/staging/iio/accel/adis16203.c
deleted file mode 100644
index 830ff38fda92e4..00000000000000
--- a/drivers/staging/iio/accel/adis16203.c
+++ /dev/null
@@ -1,315 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * ADIS16203 Programmable 360 Degrees Inclinometer
- *
- * Copyright 2010 Analog Devices Inc.
- */
-
-#include <linux/device.h>
-
-#include <linux/iio/iio.h>
-#include <linux/iio/imu/adis.h>
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/spi/spi.h>
-
-#define ADIS16203_STARTUP_DELAY 220 /* ms */
-
-/* Flash memory write count */
-#define ADIS16203_FLASH_CNT      0x00
-
-/* Output, power supply */
-#define ADIS16203_SUPPLY_OUT     0x02
-
-/* Output, auxiliary ADC input */
-#define ADIS16203_AUX_ADC        0x08
-
-/* Output, temperature */
-#define ADIS16203_TEMP_OUT       0x0A
-
-/* Output, x-axis inclination */
-#define ADIS16203_XINCL_OUT      0x0C
-
-/* Output, y-axis inclination */
-#define ADIS16203_YINCL_OUT      0x0E
-
-/* Incline null calibration */
-#define ADIS16203_INCL_NULL      0x18
-
-/* Alarm 1 amplitude threshold */
-#define ADIS16203_ALM_MAG1       0x20
-
-/* Alarm 2 amplitude threshold */
-#define ADIS16203_ALM_MAG2       0x22
-
-/* Alarm 1, sample period */
-#define ADIS16203_ALM_SMPL1      0x24
-
-/* Alarm 2, sample period */
-#define ADIS16203_ALM_SMPL2      0x26
-
-/* Alarm control */
-#define ADIS16203_ALM_CTRL       0x28
-
-/* Auxiliary DAC data */
-#define ADIS16203_AUX_DAC        0x30
-
-/* General-purpose digital input/output control */
-#define ADIS16203_GPIO_CTRL      0x32
-
-/* Miscellaneous control */
-#define ADIS16203_MSC_CTRL       0x34
-
-/* Internal sample period (rate) control */
-#define ADIS16203_SMPL_PRD       0x36
-
-/* Operation, filter configuration */
-#define ADIS16203_AVG_CNT        0x38
-
-/* Operation, sleep mode control */
-#define ADIS16203_SLP_CNT        0x3A
-
-/* Diagnostics, system status register */
-#define ADIS16203_DIAG_STAT      0x3C
-
-/* Operation, system command register */
-#define ADIS16203_GLOB_CMD       0x3E
-
-/* MSC_CTRL */
-
-/* Self-test at power-on: 1 = disabled, 0 = enabled */
-#define ADIS16203_MSC_CTRL_PWRUP_SELF_TEST      BIT(10)
-
-/* Reverses rotation of both inclination outputs */
-#define ADIS16203_MSC_CTRL_REVERSE_ROT_EN       BIT(9)
-
-/* Self-test enable */
-#define ADIS16203_MSC_CTRL_SELF_TEST_EN         BIT(8)
-
-/* Data-ready enable: 1 = enabled, 0 = disabled */
-#define ADIS16203_MSC_CTRL_DATA_RDY_EN          BIT(2)
-
-/* Data-ready polarity: 1 = active high, 0 = active low */
-#define ADIS16203_MSC_CTRL_ACTIVE_HIGH          BIT(1)
-
-/* Data-ready line selection: 1 = DIO1, 0 = DIO0 */
-#define ADIS16203_MSC_CTRL_DATA_RDY_DIO1        BIT(0)
-
-/* DIAG_STAT */
-
-/* Alarm 2 status: 1 = alarm active, 0 = alarm inactive */
-#define ADIS16203_DIAG_STAT_ALARM2        BIT(9)
-
-/* Alarm 1 status: 1 = alarm active, 0 = alarm inactive */
-#define ADIS16203_DIAG_STAT_ALARM1        BIT(8)
-
-/* Self-test diagnostic error flag */
-#define ADIS16203_DIAG_STAT_SELFTEST_FAIL_BIT 5
-
-/* SPI communications failure */
-#define ADIS16203_DIAG_STAT_SPI_FAIL_BIT      3
-
-/* Flash update failure */
-#define ADIS16203_DIAG_STAT_FLASH_UPT_BIT     2
-
-/* Power supply above 3.625 V */
-#define ADIS16203_DIAG_STAT_POWER_HIGH_BIT    1
-
-/* Power supply below 2.975 V */
-#define ADIS16203_DIAG_STAT_POWER_LOW_BIT     0
-
-/* GLOB_CMD */
-
-#define ADIS16203_GLOB_CMD_SW_RESET     BIT(7)
-#define ADIS16203_GLOB_CMD_CLEAR_STAT   BIT(4)
-#define ADIS16203_GLOB_CMD_FACTORY_CAL  BIT(1)
-
-#define ADIS16203_ERROR_ACTIVE          BIT(14)
-
-enum adis16203_scan {
-	 ADIS16203_SCAN_INCLI_X,
-	 ADIS16203_SCAN_INCLI_Y,
-	 ADIS16203_SCAN_SUPPLY,
-	 ADIS16203_SCAN_AUX_ADC,
-	 ADIS16203_SCAN_TEMP,
-};
-
-#define DRIVER_NAME		"adis16203"
-
-static const u8 adis16203_addresses[] = {
-	[ADIS16203_SCAN_INCLI_X] = ADIS16203_INCL_NULL,
-};
-
-static int adis16203_write_raw(struct iio_dev *indio_dev,
-			       struct iio_chan_spec const *chan,
-			       int val,
-			       int val2,
-			       long mask)
-{
-	struct adis *st = iio_priv(indio_dev);
-	/* currently only one writable parameter which keeps this simple */
-	u8 addr = adis16203_addresses[chan->scan_index];
-
-	return adis_write_reg_16(st, addr, val & 0x3FFF);
-}
-
-static int adis16203_read_raw(struct iio_dev *indio_dev,
-			      struct iio_chan_spec const *chan,
-			      int *val, int *val2,
-			      long mask)
-{
-	struct adis *st = iio_priv(indio_dev);
-	int ret;
-	u8 addr;
-	s16 val16;
-
-	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		return adis_single_conversion(indio_dev, chan,
-				ADIS16203_ERROR_ACTIVE, val);
-	case IIO_CHAN_INFO_SCALE:
-		switch (chan->type) {
-		case IIO_VOLTAGE:
-			if (chan->channel == 0) {
-				*val = 1;
-				*val2 = 220000; /* 1.22 mV */
-			} else {
-				*val = 0;
-				*val2 = 610000; /* 0.61 mV */
-			}
-			return IIO_VAL_INT_PLUS_MICRO;
-		case IIO_TEMP:
-			*val = -470; /* -0.47 C */
-			*val2 = 0;
-			return IIO_VAL_INT_PLUS_MICRO;
-		case IIO_INCLI:
-			*val = 0;
-			*val2 = 25000; /* 0.025 degree */
-			return IIO_VAL_INT_PLUS_MICRO;
-		default:
-			return -EINVAL;
-		}
-	case IIO_CHAN_INFO_OFFSET:
-		*val = 25000 / -470 - 1278; /* 25 C = 1278 */
-		return IIO_VAL_INT;
-	case IIO_CHAN_INFO_CALIBBIAS:
-		addr = adis16203_addresses[chan->scan_index];
-		ret = adis_read_reg_16(st, addr, &val16);
-		if (ret)
-			return ret;
-		*val = sign_extend32(val16, 13);
-		return IIO_VAL_INT;
-	default:
-		return -EINVAL;
-	}
-}
-
-static const struct iio_chan_spec adis16203_channels[] = {
-	ADIS_SUPPLY_CHAN(ADIS16203_SUPPLY_OUT, ADIS16203_SCAN_SUPPLY, 0, 12),
-	ADIS_AUX_ADC_CHAN(ADIS16203_AUX_ADC, ADIS16203_SCAN_AUX_ADC, 0, 12),
-	ADIS_INCLI_CHAN(X, ADIS16203_XINCL_OUT, ADIS16203_SCAN_INCLI_X,
-			BIT(IIO_CHAN_INFO_CALIBBIAS), 0, 14),
-	/* Fixme: Not what it appears to be - see data sheet */
-	ADIS_INCLI_CHAN(Y, ADIS16203_YINCL_OUT, ADIS16203_SCAN_INCLI_Y,
-			0, 0, 14),
-	ADIS_TEMP_CHAN(ADIS16203_TEMP_OUT, ADIS16203_SCAN_TEMP, 0, 12),
-	IIO_CHAN_SOFT_TIMESTAMP(5),
-};
-
-static const struct iio_info adis16203_info = {
-	.read_raw = adis16203_read_raw,
-	.write_raw = adis16203_write_raw,
-	.update_scan_mode = adis_update_scan_mode,
-};
-
-static const char * const adis16203_status_error_msgs[] = {
-	[ADIS16203_DIAG_STAT_SELFTEST_FAIL_BIT] = "Self test failure",
-	[ADIS16203_DIAG_STAT_SPI_FAIL_BIT] = "SPI failure",
-	[ADIS16203_DIAG_STAT_FLASH_UPT_BIT] = "Flash update failed",
-	[ADIS16203_DIAG_STAT_POWER_HIGH_BIT] = "Power supply above 3.625V",
-	[ADIS16203_DIAG_STAT_POWER_LOW_BIT] = "Power supply below 2.975V",
-};
-
-static const struct adis_timeout adis16203_timeouts = {
-	.reset_ms = ADIS16203_STARTUP_DELAY,
-	.sw_reset_ms = ADIS16203_STARTUP_DELAY,
-	.self_test_ms = ADIS16203_STARTUP_DELAY
-};
-
-static const struct adis_data adis16203_data = {
-	.read_delay = 20,
-	.msc_ctrl_reg = ADIS16203_MSC_CTRL,
-	.glob_cmd_reg = ADIS16203_GLOB_CMD,
-	.diag_stat_reg = ADIS16203_DIAG_STAT,
-
-	.self_test_mask = ADIS16203_MSC_CTRL_SELF_TEST_EN,
-	.self_test_reg = ADIS16203_MSC_CTRL,
-	.self_test_no_autoclear = true,
-	.timeouts = &adis16203_timeouts,
-
-	.status_error_msgs = adis16203_status_error_msgs,
-	.status_error_mask = BIT(ADIS16203_DIAG_STAT_SELFTEST_FAIL_BIT) |
-		BIT(ADIS16203_DIAG_STAT_SPI_FAIL_BIT) |
-		BIT(ADIS16203_DIAG_STAT_FLASH_UPT_BIT) |
-		BIT(ADIS16203_DIAG_STAT_POWER_HIGH_BIT) |
-		BIT(ADIS16203_DIAG_STAT_POWER_LOW_BIT),
-};
-
-static int adis16203_probe(struct spi_device *spi)
-{
-	int ret;
-	struct iio_dev *indio_dev;
-	struct adis *st;
-
-	/* setup the industrialio driver allocated elements */
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev)
-		return -ENOMEM;
-	st = iio_priv(indio_dev);
-	/* this is only used for removal purposes */
-	spi_set_drvdata(spi, indio_dev);
-
-	indio_dev->name = spi->dev.driver->name;
-	indio_dev->channels = adis16203_channels;
-	indio_dev->num_channels = ARRAY_SIZE(adis16203_channels);
-	indio_dev->info = &adis16203_info;
-	indio_dev->modes = INDIO_DIRECT_MODE;
-
-	ret = adis_init(st, indio_dev, spi, &adis16203_data);
-	if (ret)
-		return ret;
-
-	ret = devm_adis_setup_buffer_and_trigger(st, indio_dev, NULL);
-	if (ret)
-		return ret;
-
-	/* Get the device into a sane initial state */
-	ret = __adis_initial_startup(st);
-	if (ret)
-		return ret;
-
-	return devm_iio_device_register(&spi->dev, indio_dev);
-}
-
-static const struct of_device_id adis16203_of_match[] = {
-	{ .compatible = "adi,adis16203" },
-	{ }
-};
-
-MODULE_DEVICE_TABLE(of, adis16203_of_match);
-
-static struct spi_driver adis16203_driver = {
-	.driver = {
-		.name = "adis16203",
-		.of_match_table = adis16203_of_match,
-	},
-	.probe = adis16203_probe,
-};
-module_spi_driver(adis16203_driver);
-
-MODULE_AUTHOR("Barry Song <21cnbao@gmail.com>");
-MODULE_DESCRIPTION("Analog Devices ADIS16203 Programmable 360 Degrees Inclinometer");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("spi:adis16203");
-MODULE_IMPORT_NS("IIO_ADISLIB");
diff --git a/drivers/staging/iio/adc/ad7816.c b/drivers/staging/iio/adc/ad7816.c
index 30644d2d7c549a..2a15d7598abb75 100644
--- a/drivers/staging/iio/adc/ad7816.c
+++ b/drivers/staging/iio/adc/ad7816.c
@@ -5,19 +5,21 @@
  * Copyright 2010 Analog Devices Inc.
  */
 
-#include <linux/interrupt.h>
-#include <linux/gpio/consumer.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
 #include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
 #include <linux/list.h>
-#include <linux/spi/spi.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/spi/spi.h>
+#include <linux/sysfs.h>
 
+#include <linux/iio/events.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
-#include <linux/iio/events.h>
 
 /*
  * AD7816 config masks
@@ -47,6 +49,7 @@ struct ad7816_chip_info {
 	struct gpio_desc *rdwr_pin;
 	struct gpio_desc *convert_pin;
 	struct gpio_desc *busy_pin;
+	struct mutex lock; /* protect device state during SPI transfers */
 	u8  oti_data[AD7816_CS_MAX + 1];
 	u8  channel_id;	/* 0 always be temperature */
 	u8  mode;
@@ -67,9 +70,11 @@ static int ad7816_spi_read(struct ad7816_chip_info *chip, u16 *data)
 	int ret;
 	__be16 buf;
 
+	guard(mutex)(&chip->lock);
+
 	gpiod_set_value(chip->rdwr_pin, 1);
 	gpiod_set_value(chip->rdwr_pin, 0);
-	ret = spi_write(spi_dev, &chip->channel_id, sizeof(chip->channel_id));
+	ret = spi_write_then_read(spi_dev, &chip->channel_id, sizeof(chip->channel_id), NULL, 0);
 	if (ret < 0) {
 		dev_err(&spi_dev->dev, "SPI channel setting error\n");
 		return ret;
@@ -91,7 +96,7 @@ static int ad7816_spi_read(struct ad7816_chip_info *chip, u16 *data)
 
 	gpiod_set_value(chip->rdwr_pin, 0);
 	gpiod_set_value(chip->rdwr_pin, 1);
-	ret = spi_read(spi_dev, &buf, sizeof(*data));
+	ret = spi_write_then_read(spi_dev, NULL, 0, &buf, sizeof(buf));
 	if (ret < 0) {
 		dev_err(&spi_dev->dev, "SPI data read error\n");
 		return ret;
@@ -107,9 +112,11 @@ static int ad7816_spi_write(struct ad7816_chip_info *chip, u8 data)
 	struct spi_device *spi_dev = chip->spi_dev;
 	int ret;
 
+	guard(mutex)(&chip->lock);
+
 	gpiod_set_value(chip->rdwr_pin, 1);
 	gpiod_set_value(chip->rdwr_pin, 0);
-	ret = spi_write(spi_dev, &data, sizeof(data));
+	ret = spi_write_then_read(spi_dev, &data, sizeof(data), NULL, 0);
 	if (ret < 0)
 		dev_err(&spi_dev->dev, "SPI oti data write error\n");
 
@@ -136,6 +143,8 @@ static ssize_t ad7816_store_mode(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct ad7816_chip_info *chip = iio_priv(indio_dev);
 
+	guard(mutex)(&chip->lock);
+
 	if (strcmp(buf, "full") == 0) {
 		gpiod_set_value(chip->rdwr_pin, 1);
 		chip->mode = AD7816_FULL;
@@ -200,6 +209,8 @@ static ssize_t ad7816_store_channel(struct device *dev,
 		return -EINVAL;
 	}
 
+	guard(mutex)(&chip->lock);
+
 	chip->channel_id = data;
 
 	return len;
@@ -360,6 +371,10 @@ static int ad7816_probe(struct spi_device *spi_dev)
 		return -ENOMEM;
 	chip = iio_priv(indio_dev);
 
+	ret = devm_mutex_init(&spi_dev->dev, &chip->lock);
+	if (ret)
+		return ret;
+
 	chip->spi_dev = spi_dev;
 	for (i = 0; i <= AD7816_CS_MAX; i++)
 		chip->oti_data[i] = 203;
diff --git a/drivers/staging/iio/frequency/ad9832.c b/drivers/staging/iio/frequency/ad9832.c
index a78249812b2628..35f823f24f9b8d 100644
--- a/drivers/staging/iio/frequency/ad9832.c
+++ b/drivers/staging/iio/frequency/ad9832.c
@@ -118,6 +118,12 @@ static unsigned long ad9832_calc_freqreg(unsigned long mclk, unsigned long fout)
 {
 	u64 freqreg = (u64)fout << AD9832_FREQ_BITS;
 
+	/*
+	 * mclk is an unsigned long, which triggers a Coccinelle false positive
+	 * warning about using do_div() for 64-by-32 division. However, mclk
+	 * for this hardware will always fit within 32 bits, so do_div() is
+	 * safe to use here.
+	 */
 	do_div(freqreg, mclk);
 	return freqreg;
 }
diff --git a/drivers/staging/iio/frequency/ad9834.c b/drivers/staging/iio/frequency/ad9834.c
index 0d20f796e0f093..4cebb9d0d9241b 100644
--- a/drivers/staging/iio/frequency/ad9834.c
+++ b/drivers/staging/iio/frequency/ad9834.c
@@ -103,6 +103,12 @@ static unsigned int ad9834_calc_freqreg(unsigned long mclk, unsigned long fout)
 {
 	unsigned long long freqreg = (u64)fout * (u64)BIT(AD9834_FREQ_BITS);
 
+	/*
+	 * mclk is an unsigned long, which triggers a Coccinelle false positive
+	 * warning about using a do_div() for 64-by-32 division. However, mclk
+	 * for this hardware will always fit within 32 bits, so do_div() is
+	 * safe to use here.
+	 */
 	do_div(freqreg, mclk);
 	return freqreg;
 }
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
index f9cc5f45fe000c..38f69582c174a6 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
@@ -520,6 +520,7 @@ static int gc2235_startup(struct v4l2_subdev *sd)
 }
 
 static int gc2235_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
index 2c41c496daa636..e3137fbc58022a 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
@@ -623,6 +623,7 @@ static int ov2722_startup(struct v4l2_subdev *sd)
 }
 
 static int ov2722_set_fmt(struct v4l2_subdev *sd,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
index 6cd500d9fd26d9..52c71cd86decc0 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
@@ -3745,14 +3745,16 @@ static int atomisp_set_sensor_crop_and_fmt(struct atomisp_device *isp,
 	sel.r.left = ((input->native_rect.width - sel.r.width) / 2) & ~1;
 	sel.r.top = ((input->native_rect.height - sel.r.height) / 2) & ~1;
 
-	ret = v4l2_subdev_call(input->sensor, pad, set_selection, sd_state, &sel);
+	ret = v4l2_subdev_call(input->sensor, pad, set_selection, NULL,
+			       sd_state, &sel);
 	if (ret)
 		dev_err(isp->dev, "Error setting crop to (%d,%d)/%ux%u: %d\n",
 			sel.r.left, sel.r.top, sel.r.width, sel.r.height, ret);
 
 set_fmt:
 	if (ret == 0) {
-		ret = v4l2_subdev_call(input->sensor, pad, set_fmt, sd_state, &format);
+		ret = v4l2_subdev_call(input->sensor, pad, set_fmt, NULL,
+				       sd_state, &format);
 		dev_dbg(isp->dev, "Set sensor format ret: %d size %dx%d\n",
 			ret, format.format.width, format.format.height);
 	}
@@ -3765,13 +3767,16 @@ static int atomisp_set_sensor_crop_and_fmt(struct atomisp_device *isp,
 		sd_state = v4l2_subdev_lock_and_get_active_state(input->sensor_isp);
 
 		format.pad = SENSOR_ISP_PAD_SINK;
-		ret = v4l2_subdev_call(input->sensor_isp, pad, set_fmt, sd_state, &format);
+		ret = v4l2_subdev_call(input->sensor_isp, pad, set_fmt, NULL,
+				       sd_state, &format);
 		dev_dbg(isp->dev, "Set sensor ISP sink format ret: %d size %dx%d\n",
 			ret, format.format.width, format.format.height);
 
 		if (ret == 0) {
 			format.pad = SENSOR_ISP_PAD_SOURCE;
-			ret = v4l2_subdev_call(input->sensor_isp, pad, set_fmt, sd_state, &format);
+			ret = v4l2_subdev_call(input->sensor_isp, pad,
+					       set_fmt, NULL, sd_state,
+					       &format);
 			dev_dbg(isp->dev, "Set sensor ISP source format ret: %d size %dx%d\n",
 				ret, format.format.width, format.format.height);
 		}
@@ -3783,7 +3788,8 @@ static int atomisp_set_sensor_crop_and_fmt(struct atomisp_device *isp,
 	/* Propagate new fmt to CSI port */
 	if (ret == 0 && which == V4L2_SUBDEV_FORMAT_ACTIVE) {
 		format.pad = CSI2_PAD_SINK;
-		ret = v4l2_subdev_call(input->csi_port, pad, set_fmt, NULL, &format);
+		ret = v4l2_subdev_call(input->csi_port, pad, set_fmt, NULL,
+				       NULL, &format);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2.c b/drivers/staging/media/atomisp/pci/atomisp_csi2.c
index 95b9113d75e90f..71df4ef629c2a3 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_csi2.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_csi2.c
@@ -123,6 +123,7 @@ int atomisp_csi2_set_ffmt(struct v4l2_subdev *sd,
  * return -EINVAL or zero on success
  */
 static int csi2_set_format(struct v4l2_subdev *sd,
+			   const struct v4l2_subdev_client_info *ci,
 			   struct v4l2_subdev_state *sd_state,
 			   struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
index 9de9cd884d99ff..bdd2d9b18da25f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
@@ -259,6 +259,7 @@ static void isp_get_fmt_rect(struct v4l2_subdev *sd,
 }
 
 static int isp_subdev_get_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -452,6 +453,7 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
 }
 
 static int isp_subdev_set_selection(struct v4l2_subdev *sd,
+				    const struct v4l2_subdev_client_info *ci,
 				    struct v4l2_subdev_state *sd_state,
 				    struct v4l2_subdev_selection *sel)
 {
@@ -555,6 +557,7 @@ static int isp_subdev_get_format(struct v4l2_subdev *sd,
  * to the format type.
  */
 static int isp_subdev_set_format(struct v4l2_subdev *sd,
+				 const struct v4l2_subdev_client_info *ci,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
index 8122303974092c..476fe143a5914f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
@@ -913,7 +913,7 @@ static void atomisp_init_sensor(struct atomisp_input_subdev *input)
 
 	sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	sel.target = V4L2_SEL_TGT_NATIVE_SIZE;
-	err = v4l2_subdev_call(input->sensor, pad, get_selection,
+	err = v4l2_subdev_call(input->sensor, pad, get_selection, NULL,
 			       act_sd_state, &sel);
 	if (err)
 		goto unlock_act_sd_state;
@@ -922,7 +922,7 @@ static void atomisp_init_sensor(struct atomisp_input_subdev *input)
 
 	sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	sel.target = V4L2_SEL_TGT_CROP_DEFAULT;
-	err = v4l2_subdev_call(input->sensor, pad, get_selection,
+	err = v4l2_subdev_call(input->sensor, pad, get_selection, NULL,
 			       act_sd_state, &sel);
 	if (err)
 		goto unlock_act_sd_state;
@@ -968,7 +968,7 @@ static void atomisp_init_sensor(struct atomisp_input_subdev *input)
 	if (!input->sensor->state_lock)
 		v4l2_subdev_lock_state(input->try_sd_state);
 
-	err = v4l2_subdev_call(input->sensor, pad, set_selection,
+	err = v4l2_subdev_call(input->sensor, pad, set_selection, NULL,
 			       input->try_sd_state, &sel);
 
 	if (!input->sensor->state_lock)
@@ -980,7 +980,7 @@ static void atomisp_init_sensor(struct atomisp_input_subdev *input)
 	sel.which = V4L2_SUBDEV_FORMAT_ACTIVE;
 	sel.target = V4L2_SEL_TGT_CROP;
 	sel.r = input->native_rect;
-	err = v4l2_subdev_call(input->sensor, pad, set_selection,
+	err = v4l2_subdev_call(input->sensor, pad, set_selection, NULL,
 			       act_sd_state, &sel);
 	if (err)
 		goto unlock_act_sd_state;
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
index b8206d2f3d3106..97e63082d2542c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h
@@ -102,8 +102,8 @@ struct ia_css_3a_grid_info {
  *  awb_lg_*: Thresholds to check the saturated bayer pixels for AWB.
  *    Condition of effective pixel for AWB level gate check:
  *      bayer(sensor) <= awb_lg_high_raw &&
- *      bayer(when AWB statisitcs is calculated) >= awb_lg_low &&
- *      bayer(when AWB statisitcs is calculated) <= awb_lg_high
+ *      bayer(when AWB statistics is calculated) >= awb_lg_low &&
+ *      bayer(when AWB statistics is calculated) <= awb_lg_high
  *  af_fir*: Coefficients of high pass filter to calculate AF statistics.
  *
  *  ISP block: S3A1(ae_y_* for AE/AF, awb_lg_* for AWB)
diff --git a/drivers/staging/media/av7110/av7110.c b/drivers/staging/media/av7110/av7110.c
index 06c43b5f8a87ab..432404eceaea94 100644
--- a/drivers/staging/media/av7110/av7110.c
+++ b/drivers/staging/media/av7110/av7110.c
@@ -2347,7 +2347,7 @@ static int av7110_attach(struct saa7146_dev *dev,
 
 		/* RESET SAA7146 */
 		saa7146_write(dev, MC1, MASK_31);
-		/* autodetection success seems to be time-dependend after reset */
+		/* autodetection success seems to be time-dependent after reset */
 
 		/* Fix VSYNC level */
 		saa7146_setgpio(dev, 3, SAA7146_GPIO_OUTLO);
diff --git a/drivers/staging/media/av7110/av7110_ir.c b/drivers/staging/media/av7110/av7110_ir.c
index 9b7bf1857868d7..d0948a80d3875e 100644
--- a/drivers/staging/media/av7110/av7110_ir.c
+++ b/drivers/staging/media/av7110/av7110_ir.c
@@ -154,5 +154,3 @@ void av7110_ir_exit(struct av7110 *av7110)
 	rc_free_device(av7110->ir.rcdev);
 }
 
-//MODULE_AUTHOR("Holger Waechtler <holger@convergence.de>, Oliver Endriss <o.endriss@gmx.de>");
-//MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/av7110/sp8870.c b/drivers/staging/media/av7110/sp8870.c
index 29fb4934c039c6..aecdbd87976fd2 100644
--- a/drivers/staging/media/av7110/sp8870.c
+++ b/drivers/staging/media/av7110/sp8870.c
@@ -92,7 +92,7 @@ static int sp8870_readreg(struct sp8870_state *state, u16 reg)
 		return -1;
 	}
 
-	return (b1[0] << 8 | b1[1]);
+	return b1[0] << 8 | b1[1];
 }
 
 static int sp8870_firmware_upload(struct sp8870_state *state, const struct firmware *fw)
@@ -167,7 +167,7 @@ static void sp8870_microcontroller_start(struct sp8870_state *state)
 
 static int sp8870_read_data_valid_signal(struct sp8870_state *state)
 {
-	return (sp8870_readreg(state, 0x0D02) > 0);
+	return sp8870_readreg(state, 0x0D02) > 0;
 }
 
 static int configure_reg0xc05(struct dtv_frontend_properties *p, u16 *reg0xc05)
@@ -315,7 +315,6 @@ static int sp8870_init(struct dvb_frontend *fe)
 	sp8870_wake_up(state);
 	if (state->initialised)
 		return 0;
-	state->initialised = 1;
 
 	dprintk("initialising frontend...\n");
 
@@ -353,6 +352,8 @@ static int sp8870_init(struct dvb_frontend *fe)
 	sp8870_writereg(state, 0x0D00, 0x010);
 	sp8870_writereg(state, 0x0D01, 0x000);
 
+	state->initialised = 1;
+
 	return 0;
 }
 
@@ -401,7 +402,7 @@ static int sp8870_read_ber(struct dvb_frontend *fe, u32 *ber)
 	if (ret < 0)
 		return -EIO;
 
-	tmp = ret << 6;
+	tmp |= ret << 6;
 	if (tmp >= 0x3FFF0)
 		tmp = ~0;
 
diff --git a/drivers/staging/media/imx/imx-ic-prp.c b/drivers/staging/media/imx/imx-ic-prp.c
index 2b80d54006b396..b2c648149964c6 100644
--- a/drivers/staging/media/imx/imx-ic-prp.c
+++ b/drivers/staging/media/imx/imx-ic-prp.c
@@ -11,7 +11,6 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
-#include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/timer.h>
@@ -151,6 +150,7 @@ static int prp_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int prp_set_fmt(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *sd_state,
 		       struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/staging/media/imx/imx-ic-prpencvf.c b/drivers/staging/media/imx/imx-ic-prpencvf.c
index 77360bfe081a42..3430ddeac35696 100644
--- a/drivers/staging/media/imx/imx-ic-prpencvf.c
+++ b/drivers/staging/media/imx/imx-ic-prpencvf.c
@@ -9,9 +9,8 @@
  * Copyright (c) 2012-2017 Mentor Graphics Inc.
  */
 #include <linux/delay.h>
-#include <linux/interrupt.h>
 #include <linux/module.h>
-#include <linux/sched.h>
+#include <linux/interrupt.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/timer.h>
@@ -919,6 +918,7 @@ static void prp_try_fmt(struct prp_priv *priv,
 }
 
 static int prp_set_fmt(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *sd_state,
 		       struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/staging/media/imx/imx-media-capture.c b/drivers/staging/media/imx/imx-media-capture.c
index bfd71d25facce2..9216a221b72f4f 100644
--- a/drivers/staging/media/imx/imx-media-capture.c
+++ b/drivers/staging/media/imx/imx-media-capture.c
@@ -5,7 +5,6 @@
  * Copyright (c) 2012-2016 Mentor Graphics Inc.
  */
 #include <linux/delay.h>
-#include <linux/fs.h>
 #include <linux/module.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
diff --git a/drivers/staging/media/imx/imx-media-csc-scaler.c b/drivers/staging/media/imx/imx-media-csc-scaler.c
index 00fcdd4d048730..6f4c0b09bb3640 100644
--- a/drivers/staging/media/imx/imx-media-csc-scaler.c
+++ b/drivers/staging/media/imx/imx-media-csc-scaler.c
@@ -7,8 +7,6 @@
  */
 #include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/fs.h>
-#include <linux/sched.h>
 #include <linux/slab.h>
 #include <video/imx-ipu-v3.h>
 #include <video/imx-ipu-image-convert.h>
@@ -147,6 +145,7 @@ static void device_run(void *_ctx)
 	v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
 	v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_ERROR);
 	v4l2_m2m_job_finish(priv->m2m_dev, ctx->fh.m2m_ctx);
+	kfree(run);
 }
 
 /*
diff --git a/drivers/staging/media/imx/imx-media-csi.c b/drivers/staging/media/imx/imx-media-csi.c
index ef22a083f8eb1e..43c637b333f2f6 100644
--- a/drivers/staging/media/imx/imx-media-csi.c
+++ b/drivers/staging/media/imx/imx-media-csi.c
@@ -1525,6 +1525,7 @@ static void csi_try_fmt(struct csi_priv *priv,
 }
 
 static int csi_set_fmt(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *sd_state,
 		       struct v4l2_subdev_format *sdformat)
 {
@@ -1593,6 +1594,7 @@ static int csi_set_fmt(struct v4l2_subdev *sd,
 }
 
 static int csi_get_selection(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
@@ -1657,6 +1659,7 @@ static int csi_set_scale(u32 *compose, u32 crop, u32 flags)
 }
 
 static int csi_set_selection(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *sd_state,
 			     struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/staging/media/imx/imx-media-dev.c b/drivers/staging/media/imx/imx-media-dev.c
index a08389b99d14ac..a8f39319bd8a25 100644
--- a/drivers/staging/media/imx/imx-media-dev.c
+++ b/drivers/staging/media/imx/imx-media-dev.c
@@ -4,7 +4,6 @@
  *
  * Copyright (c) 2016-2019 Mentor Graphics Inc.
  */
-#include <linux/fs.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <media/v4l2-async.h>
diff --git a/drivers/staging/media/imx/imx-media-utils.c b/drivers/staging/media/imx/imx-media-utils.c
index f119477cac6b16..7bdcb88a686b1c 100644
--- a/drivers/staging/media/imx/imx-media-utils.c
+++ b/drivers/staging/media/imx/imx-media-utils.c
@@ -588,7 +588,7 @@ int imx_media_alloc_dma_buf(struct device *dev,
 
 	buf->len = PAGE_ALIGN(size);
 	buf->virt = dma_alloc_coherent(dev, buf->len, &buf->phys,
-				       GFP_DMA | GFP_KERNEL);
+				       GFP_KERNEL);
 	if (!buf->virt)
 		return -ENOMEM;
 
diff --git a/drivers/staging/media/imx/imx-media-vdic.c b/drivers/staging/media/imx/imx-media-vdic.c
index 58f1112e28e5c0..8dcbb6fe764910 100644
--- a/drivers/staging/media/imx/imx-media-vdic.c
+++ b/drivers/staging/media/imx/imx-media-vdic.c
@@ -566,6 +566,7 @@ static void vdic_try_fmt(struct vdic_priv *priv,
 }
 
 static int vdic_set_fmt(struct v4l2_subdev *sd,
+			const struct v4l2_subdev_client_info *ci,
 			struct v4l2_subdev_state *sd_state,
 			struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/staging/media/imx/imx6-mipi-csi2.c b/drivers/staging/media/imx/imx6-mipi-csi2.c
index 211f67fb92b5b1..f6cbf0310e89ca 100644
--- a/drivers/staging/media/imx/imx6-mipi-csi2.c
+++ b/drivers/staging/media/imx/imx6-mipi-csi2.c
@@ -525,6 +525,7 @@ static int csi2_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int csi2_set_fmt(struct v4l2_subdev *sd,
+			const struct v4l2_subdev_client_info *ci,
 			struct v4l2_subdev_state *sd_state,
 			struct v4l2_subdev_format *sdformat)
 {
diff --git a/drivers/staging/media/ipu3/ipu3-css.c b/drivers/staging/media/ipu3/ipu3-css.c
index 8063401246fb43..b2c8fb7d6813b3 100644
--- a/drivers/staging/media/ipu3/ipu3-css.c
+++ b/drivers/staging/media/ipu3/ipu3-css.c
@@ -290,7 +290,6 @@ int imgu_css_set_powerup(struct device *dev, void __iomem *base,
 
 void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
 {
-	dev_dbg(dev, "%s\n", __func__);
 	/* wait for cio idle signal */
 	if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
 			 IMGU_CIO_GATE_BURST_MASK, 0))
diff --git a/drivers/staging/media/ipu3/ipu3-v4l2.c b/drivers/staging/media/ipu3/ipu3-v4l2.c
index 2f6041d342f429..61fdefe78c3d36 100644
--- a/drivers/staging/media/ipu3/ipu3-v4l2.c
+++ b/drivers/staging/media/ipu3/ipu3-v4l2.c
@@ -145,6 +145,7 @@ static int imgu_subdev_get_fmt(struct v4l2_subdev *sd,
 }
 
 static int imgu_subdev_set_fmt(struct v4l2_subdev *sd,
+			       const struct v4l2_subdev_client_info *ci,
 			       struct v4l2_subdev_state *sd_state,
 			       struct v4l2_subdev_format *fmt)
 {
@@ -212,6 +213,7 @@ imgu_subdev_get_compose(struct imgu_v4l2_subdev *sd,
 }
 
 static int imgu_subdev_get_selection(struct v4l2_subdev *sd,
+				     const struct v4l2_subdev_client_info *ci,
 				     struct v4l2_subdev_state *sd_state,
 				     struct v4l2_subdev_selection *sel)
 {
@@ -236,6 +238,7 @@ static int imgu_subdev_get_selection(struct v4l2_subdev *sd,
 }
 
 static int imgu_subdev_set_selection(struct v4l2_subdev *sd,
+				     const struct v4l2_subdev_client_info *ci,
 				     struct v4l2_subdev_state *sd_state,
 				     struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/staging/media/ipu7/TODO b/drivers/staging/media/ipu7/TODO
index dc27bb6463da35..57cca4e4033fb2 100644
--- a/drivers/staging/media/ipu7/TODO
+++ b/drivers/staging/media/ipu7/TODO
@@ -1,26 +1,2 @@
-This is a list of things that need to be done to get this driver out of the
-staging directory.
-
-- ABI headers cleanup
-  Cleanup the firmware ABI headers
-
-- Add metadata capture support
-  The IPU7 hardware should support metadata capture, but it is not
-  fully verified with IPU7 firmware ABI so far, need to add the metadata
-  capture support.
-
-- Refine CSI2 PHY code
-  Refine the ipu7-isys-csi2-phy.c, move the hardware specific variant
-  into structure, clarify and explain the PHY registers to make it more
-  readable.
-
-- Work with the common IPU module
-  Sakari commented much of the driver code is the same than the IPU6 driver.
-  IPU7 driver is expected to work with the common IPU module in future.
-
-- Register definition cleanup
-  Add IPU7 prefix for IPU7 specific registers and related macros. Some
-  ISYS IO sub-blocks register definitions are offset values from specific
-  sub-block base, but it is not clear and well suited for driver to use,
-  need to update the register definitions to make it more clear and
-  readable.
+This driver will be removed soon in favour of supporting IPU7 and later
+in the ipu6 driver.
diff --git a/drivers/staging/media/ipu7/ipu7-isys-csi2.c b/drivers/staging/media/ipu7/ipu7-isys-csi2.c
index f34eabfe8a985f..fcc7efcf69c34a 100644
--- a/drivers/staging/media/ipu7/ipu7-isys-csi2.c
+++ b/drivers/staging/media/ipu7/ipu7-isys-csi2.c
@@ -190,6 +190,7 @@ static int ipu7_isys_csi2_enable_stream(struct ipu7_isys_csi2 *csi2)
 }
 
 static int ipu7_isys_csi2_set_sel(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *state,
 				  struct v4l2_subdev_selection *sel)
 {
@@ -241,6 +242,7 @@ static int ipu7_isys_csi2_set_sel(struct v4l2_subdev *sd,
 }
 
 static int ipu7_isys_csi2_get_sel(struct v4l2_subdev *sd,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *state,
 				  struct v4l2_subdev_selection *sel)
 {
diff --git a/drivers/staging/media/ipu7/ipu7-isys-subdev.c b/drivers/staging/media/ipu7/ipu7-isys-subdev.c
index 67a776033d5b70..2cb0521b424028 100644
--- a/drivers/staging/media/ipu7/ipu7-isys-subdev.c
+++ b/drivers/staging/media/ipu7/ipu7-isys-subdev.c
@@ -99,6 +99,7 @@ u32 ipu7_isys_convert_bayer_order(u32 code, int x, int y)
 }
 
 int ipu7_isys_subdev_set_fmt(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *format)
 {
diff --git a/drivers/staging/media/ipu7/ipu7-isys-subdev.h b/drivers/staging/media/ipu7/ipu7-isys-subdev.h
index faa50031cf245e..c3585d1c412876 100644
--- a/drivers/staging/media/ipu7/ipu7-isys-subdev.h
+++ b/drivers/staging/media/ipu7/ipu7-isys-subdev.h
@@ -31,6 +31,7 @@ bool ipu7_isys_is_bayer_format(u32 code);
 u32 ipu7_isys_convert_bayer_order(u32 code, int x, int y);
 
 int ipu7_isys_subdev_set_fmt(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_format *format);
 int ipu7_isys_subdev_enum_mbus_code(struct v4l2_subdev *sd,
diff --git a/drivers/staging/media/ipu7/ipu7-isys.c b/drivers/staging/media/ipu7/ipu7-isys.c
index 601e5a79ef8ece..a54393fbb9db63 100644
--- a/drivers/staging/media/ipu7/ipu7-isys.c
+++ b/drivers/staging/media/ipu7/ipu7-isys.c
@@ -590,6 +590,7 @@ static void isys_remove(struct auxiliary_device *auxdev)
 
 	isys_notifier_cleanup(isys);
 	isys_unregister_devices(isys);
+	ipu7_fw_isys_release(isys);
 
 	cpu_latency_qos_remove_request(&isys->pm_qos);
 
diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c
index 48a35bda42370d..88e362c7ae350a 100644
--- a/drivers/staging/media/ipu7/ipu7.c
+++ b/drivers/staging/media/ipu7/ipu7.c
@@ -39,6 +39,10 @@
 #define IPU_PCI_BAR		0
 #define IPU_PCI_PBBAR		4
 
+static int force_probe = !IS_BUILTIN(CONFIG_VIDEO_INTEL_IPU6_IPU7);
+module_param(force_probe, int, 0644);
+MODULE_PARM_DESC(force_probe, "Probe ipu7 and ipu7.5 with this driver instead of ipu6");
+
 static const unsigned int ipu7_csi_offsets[] = {
 	IPU_CSI_PORT_A_ADDR_OFFSET,
 	IPU_CSI_PORT_B_ADDR_OFFSET,
@@ -2414,6 +2418,9 @@ static int ipu7_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	u32 is_es;
 	int ret;
 
+	if (!force_probe)
+		return -ENODEV;
+
 	if (!fwnode || fwnode_property_read_u32(fwnode, "is_es", &is_es))
 		is_es = 0;
 
diff --git a/drivers/staging/media/max96712/max96712.c b/drivers/staging/media/max96712/max96712.c
index 0751b2e0489589..94ae304ac85f9a 100644
--- a/drivers/staging/media/max96712/max96712.c
+++ b/drivers/staging/media/max96712/max96712.c
@@ -264,7 +264,6 @@ static const struct v4l2_subdev_internal_ops max96712_internal_ops = {
 
 static const struct v4l2_subdev_pad_ops max96712_pad_ops = {
 	.get_fmt = v4l2_subdev_get_fmt,
-	.set_fmt = v4l2_subdev_get_fmt,
 };
 
 static const struct v4l2_subdev_ops max96712_subdev_ops = {
diff --git a/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c b/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c
index 46a334b602f192..3f376af6c22822 100644
--- a/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c
+++ b/drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c
@@ -313,6 +313,7 @@ static int sun6i_isp_proc_get_fmt(struct v4l2_subdev *subdev,
 }
 
 static int sun6i_isp_proc_set_fmt(struct v4l2_subdev *subdev,
+				  const struct v4l2_subdev_client_info *ci,
 				  struct v4l2_subdev_state *state,
 				  struct v4l2_subdev_format *format)
 {
diff --git a/drivers/staging/media/tegra-video/csi.c b/drivers/staging/media/tegra-video/csi.c
index 41d57dac61a71d..5a5fca378ae64a 100644
--- a/drivers/staging/media/tegra-video/csi.c
+++ b/drivers/staging/media/tegra-video/csi.c
@@ -170,6 +170,7 @@ static int csi_enum_frameintervals(struct v4l2_subdev *subdev,
 }
 
 static int csi_set_format(struct v4l2_subdev *subdev,
+			  const struct v4l2_subdev_client_info *ci,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
 {
diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c
index 01622013c10924..6a5f66da348b2e 100644
--- a/drivers/staging/media/tegra-video/vi.c
+++ b/drivers/staging/media/tegra-video/vi.c
@@ -477,7 +477,7 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan,
 	ret = v4l2_subdev_call(subdev, pad, enum_frame_size, sd_state, &fse);
 	if (ret) {
 		if (!v4l2_subdev_has_op(subdev, pad, get_selection) ||
-		    v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel)) {
+		    v4l2_subdev_call(subdev, pad, get_selection, NULL, NULL, &sdsel)) {
 			try_crop->width = 0;
 			try_crop->height = 0;
 		} else {
@@ -489,7 +489,7 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan,
 		try_crop->height = fse.max_height;
 	}
 
-	ret = v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &fmt);
+	ret = v4l2_subdev_call(subdev, pad, set_fmt, NULL, sd_state, &fmt);
 	if (ret < 0)
 		goto out_free;
 
@@ -543,7 +543,7 @@ static int tegra_channel_set_format(struct file *file, void *fh,
 	fmt.pad = 0;
 	v4l2_fill_mbus_format(&fmt.format, pix, fmtinfo->code);
 	subdev = tegra_channel_get_remote_source_subdev(chan);
-	ret = v4l2_subdev_call(subdev, pad, set_fmt, NULL, &fmt);
+	ret = v4l2_subdev_call(subdev, pad, set_fmt, NULL, NULL, &fmt);
 	if (ret < 0)
 		return ret;
 
@@ -626,7 +626,7 @@ static int tegra_channel_g_selection(struct file *file, void *priv,
 	 * Try the get selection operation and fallback to get format if not
 	 * implemented.
 	 */
-	ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel);
+	ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, NULL, &sdsel);
 	if (!ret)
 		sel->r = sdsel.r;
 	if (ret != -ENOIOCTLCMD)
@@ -667,7 +667,7 @@ static int tegra_channel_s_selection(struct file *file, void *fh,
 	if (vb2_is_busy(&chan->queue))
 		return -EBUSY;
 
-	ret = v4l2_subdev_call(subdev, pad, set_selection, NULL, &sdsel);
+	ret = v4l2_subdev_call(subdev, pad, set_selection, NULL, NULL, &sdsel);
 	if (!ret) {
 		sel->r = sdsel.r;
 		/*
@@ -1247,17 +1247,54 @@ static int tegra_vi_tpg_channels_alloc(struct tegra_vi *vi)
 	return 0;
 }
 
+static int tegra_vi_port_channel_alloc(struct tegra_vi *vi,
+				       struct device_node *port)
+{
+	struct v4l2_fwnode_endpoint v4l2_ep = { .bus_type = 0 };
+	struct device_node *parent;
+	struct device_node *ep;
+	unsigned int port_num;
+	unsigned int lanes;
+	int ret;
+
+	if (!of_node_name_eq(port, "port"))
+		return 0;
+
+	if (of_property_read_u32(port, "reg", &port_num) < 0)
+		return 0;
+
+	if (port_num > vi->soc->vi_max_channels) {
+		dev_err(vi->dev, "invalid port num %d for %pOF\n",
+			port_num, port);
+		return -EINVAL;
+	}
+
+	ep = of_get_child_by_name(port, "endpoint");
+	if (!ep)
+		return 0;
+
+	parent = of_graph_get_remote_port_parent(ep);
+	of_node_put(ep);
+	if (!parent)
+		return 0;
+
+	ep = of_graph_get_endpoint_by_regs(parent, 0, 0);
+	of_node_put(parent);
+	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), &v4l2_ep);
+	of_node_put(ep);
+	if (ret)
+		return 0;
+
+	lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes;
+
+	return tegra_vi_channel_alloc(vi, port_num, port, lanes);
+}
+
 static int tegra_vi_channels_alloc(struct tegra_vi *vi)
 {
 	struct device_node *node = vi->dev->of_node;
-	struct device_node *ep = NULL;
 	struct device_node *ports;
-	struct device_node *port = NULL;
-	unsigned int port_num;
-	struct device_node *parent;
-	struct v4l2_fwnode_endpoint v4l2_ep = { .bus_type = 0 };
-	unsigned int lanes;
-	int err;
+	struct device_node *port;
 	int ret = 0;
 
 	ports = of_get_child_by_name(node, "ports");
@@ -1265,46 +1302,15 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi)
 		return dev_err_probe(vi->dev, -ENODEV, "%pOF: missing 'ports' node\n", node);
 
 	for_each_child_of_node(ports, port) {
-		if (!of_node_name_eq(port, "port"))
-			continue;
-
-		err = of_property_read_u32(port, "reg", &port_num);
-		if (err < 0)
-			continue;
-
-		if (port_num > vi->soc->vi_max_channels) {
-			dev_err(vi->dev, "invalid port num %d for %pOF\n",
-				port_num, port);
-			ret = -EINVAL;
-			goto cleanup;
+		ret = tegra_vi_port_channel_alloc(vi, port);
+		if (ret) {
+			of_node_put(port);
+			break;
 		}
-
-		ep = of_get_child_by_name(port, "endpoint");
-		if (!ep)
-			continue;
-
-		parent = of_graph_get_remote_port_parent(ep);
-		of_node_put(ep);
-		if (!parent)
-			continue;
-
-		ep = of_graph_get_endpoint_by_regs(parent, 0, 0);
-		of_node_put(parent);
-		err = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep),
-						 &v4l2_ep);
-		of_node_put(ep);
-		if (err)
-			continue;
-
-		lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes;
-		ret = tegra_vi_channel_alloc(vi, port_num, port, lanes);
-		if (ret < 0)
-			goto cleanup;
 	}
 
-cleanup:
-	of_node_put(port);
 	of_node_put(ports);
+
 	return ret;
 }
 
diff --git a/drivers/staging/most/video/video.c b/drivers/staging/most/video/video.c
index 88892b5db28be3..3a0445ff62f48b 100644
--- a/drivers/staging/most/video/video.c
+++ b/drivers/staging/most/video/video.c
@@ -439,7 +439,6 @@ static int comp_register_videodev(struct most_video_dev *mdev)
 	mdev->vdev->release = video_device_release;
 
 	return 0;
-
 }
 
 static void comp_unregister_videodev(struct most_video_dev *mdev)
diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c
index cc20c1e6791fc5..5e536827f87ad3 100644
--- a/drivers/staging/octeon/ethernet-tx.c
+++ b/drivers/staging/octeon/ethernet-tx.c
@@ -481,7 +481,7 @@ netdev_tx_t cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev)
 			cvmx_fau_fetch_and_add32(FAU_TOTAL_TX_TO_CLEAN, 1);
 	}
 
-	if (total_to_clean & 0x3ff) {
+	if ((total_to_clean & 0x3ff) == 0) {
 		/*
 		 * Schedule the cleanup tasklet every 1024 packets for
 		 * the pathological case of high traffic on one port
diff --git a/drivers/staging/rtl8723bs/core/rtw_ap.c b/drivers/staging/rtl8723bs/core/rtw_ap.c
index 4728f62a15578c..9face4c8ded215 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ap.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ap.c
@@ -1068,7 +1068,7 @@ int rtw_acl_add_sta(struct adapter *padapter, u8 *addr)
 	struct __queue	*pacl_node_q = &pacl_list->acl_node_q;
 
 	if ((NUM_ACL - 1) < pacl_list->num)
-		return (-1);
+		return -1;
 
 	spin_lock_bh(&pacl_node_q->lock);
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c
index ce3dfa1fee263a..59f13f6b85ade9 100644
--- a/drivers/staging/rtl8723bs/core/rtw_cmd.c
+++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c
@@ -1109,25 +1109,26 @@ u8 rtw_dynamic_chk_wk_cmd(struct adapter *padapter)
 static void collect_traffic_statistics(struct adapter *padapter)
 {
 	struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter);
+	struct rtw_traffic_statistics *ts = &pdvobjpriv->traffic_stat;
 
 	/*  Tx */
-	pdvobjpriv->traffic_stat.tx_bytes = padapter->xmitpriv.tx_bytes;
-	pdvobjpriv->traffic_stat.tx_pkts = padapter->xmitpriv.tx_pkts;
-	pdvobjpriv->traffic_stat.tx_drop = padapter->xmitpriv.tx_drop;
+	ts->tx_bytes = padapter->xmitpriv.tx_bytes;
+	ts->tx_pkts = padapter->xmitpriv.tx_pkts;
+	ts->tx_drop = padapter->xmitpriv.tx_drop;
 
 	/*  Rx */
-	pdvobjpriv->traffic_stat.rx_bytes = padapter->recvpriv.rx_bytes;
-	pdvobjpriv->traffic_stat.rx_pkts = padapter->recvpriv.rx_pkts;
-	pdvobjpriv->traffic_stat.rx_drop = padapter->recvpriv.rx_drop;
+	ts->rx_bytes = padapter->recvpriv.rx_bytes;
+	ts->rx_pkts = padapter->recvpriv.rx_pkts;
+	ts->rx_drop = padapter->recvpriv.rx_drop;
 
 	/*  Calculate throughput in last interval */
-	pdvobjpriv->traffic_stat.cur_tx_bytes = pdvobjpriv->traffic_stat.tx_bytes - pdvobjpriv->traffic_stat.last_tx_bytes;
-	pdvobjpriv->traffic_stat.cur_rx_bytes = pdvobjpriv->traffic_stat.rx_bytes - pdvobjpriv->traffic_stat.last_rx_bytes;
-	pdvobjpriv->traffic_stat.last_tx_bytes = pdvobjpriv->traffic_stat.tx_bytes;
-	pdvobjpriv->traffic_stat.last_rx_bytes = pdvobjpriv->traffic_stat.rx_bytes;
+	ts->cur_tx_bytes = ts->tx_bytes - ts->last_tx_bytes;
+	ts->cur_rx_bytes = ts->rx_bytes - ts->last_rx_bytes;
+	ts->last_tx_bytes = ts->tx_bytes;
+	ts->last_rx_bytes = ts->rx_bytes;
 
-	pdvobjpriv->traffic_stat.cur_tx_tp = (u32)(pdvobjpriv->traffic_stat.cur_tx_bytes * 8 / 2 / 1024 / 1024);
-	pdvobjpriv->traffic_stat.cur_rx_tp = (u32)(pdvobjpriv->traffic_stat.cur_rx_bytes * 8 / 2 / 1024 / 1024);
+	ts->cur_tx_tp = (u32)(ts->cur_tx_bytes * 8 / 2 / 1024 / 1024);
+	ts->cur_rx_tp = (u32)(ts->cur_rx_bytes * 8 / 2 / 1024 / 1024);
 }
 
 bool traffic_status_watchdog(struct adapter *padapter, bool from_timer)
diff --git a/drivers/staging/rtl8723bs/core/rtw_efuse.c b/drivers/staging/rtl8723bs/core/rtw_efuse.c
index 168ecb7a998fb9..b2d22307b115e0 100644
--- a/drivers/staging/rtl8723bs/core/rtw_efuse.c
+++ b/drivers/staging/rtl8723bs/core/rtw_efuse.c
@@ -41,48 +41,48 @@ rtw_efuse_calculate_word_counts(u8 word_en)
  * 09/23/2008	MHC		Copy from WMAC.
  *
  */
-u8 rtw_efuse_read_1_byte(struct adapter *Adapter, u16 Address)
+u8 rtw_efuse_read_1_byte(struct adapter *adapter, u16 address)
 {
-	u8 Bytetemp = {0x00};
+	u8 byte_temp = {0x00};
 	u8 temp = {0x00};
 	u32 k = 0;
-	u16 contentLen = 0;
+	u16 content_len = 0;
 
-	Hal_GetEfuseDefinition(Adapter, EFUSE_WIFI, TYPE_EFUSE_REAL_CONTENT_LEN, &contentLen);
+	Hal_GetEfuseDefinition(adapter, EFUSE_WIFI, TYPE_EFUSE_REAL_CONTENT_LEN, &content_len);
 
-	if (Address < contentLen) {/* E-fuse 512Byte */
+	if (address < content_len) {/* E-fuse 512Byte */
 		/* Write E-fuse Register address bit0~7 */
-		temp = Address & 0xFF;
-		rtw_write8(Adapter, EFUSE_CTRL + 1, temp);
-		Bytetemp = rtw_read8(Adapter, EFUSE_CTRL + 2);
+		temp = address & 0xFF;
+		rtw_write8(adapter, EFUSE_CTRL + 1, temp);
+		byte_temp = rtw_read8(adapter, EFUSE_CTRL + 2);
 		/* Write E-fuse Register address bit8~9 */
-		temp = ((Address >> 8) & 0x03) | (Bytetemp & 0xFC);
-		rtw_write8(Adapter, EFUSE_CTRL + 2, temp);
+		temp = ((address >> 8) & 0x03) | (byte_temp & 0xFC);
+		rtw_write8(adapter, EFUSE_CTRL + 2, temp);
 
 		/* Write 0x30[31]= 0 */
-		Bytetemp = rtw_read8(Adapter, EFUSE_CTRL + 3);
-		temp = Bytetemp & 0x7F;
-		rtw_write8(Adapter, EFUSE_CTRL + 3, temp);
+		byte_temp = rtw_read8(adapter, EFUSE_CTRL + 3);
+		temp = byte_temp & 0x7F;
+		rtw_write8(adapter, EFUSE_CTRL + 3, temp);
 
 		/* Wait Write-ready (0x30[31]= 1) */
-		Bytetemp = rtw_read8(Adapter, EFUSE_CTRL + 3);
-		while (!(Bytetemp & 0x80)) {
-			Bytetemp = rtw_read8(Adapter, EFUSE_CTRL + 3);
+		byte_temp = rtw_read8(adapter, EFUSE_CTRL + 3);
+		while (!(byte_temp & 0x80)) {
+			byte_temp = rtw_read8(adapter, EFUSE_CTRL + 3);
 			k++;
 			if (k == 1000)
 				break;
 		}
-		return rtw_read8(Adapter, EFUSE_CTRL);
+		return rtw_read8(adapter, EFUSE_CTRL);
 	} else {
 		return 0xFF;
 	}
 } /* rtw_efuse_read_1_byte */
 
-/*  11/16/2008 MH Read one byte from real Efuse. */
+/*  11/16/2008 MH Read one byte from real efuse. */
 u8 rtw_efuse_one_byte_read(struct adapter *padapter, u16 addr, u8 *data)
 {
 	u32 tmpidx = 0;
-	u8 bResult;
+	u8 b_result;
 	u8 readbyte;
 
 	/*  <20130121, Kordan> For SMIC EFUSE specificatoin. */
@@ -107,19 +107,19 @@ u8 rtw_efuse_one_byte_read(struct adapter *padapter, u16 addr, u8 *data)
 	}
 	if (tmpidx < 100) {
 		*data = rtw_read8(padapter, EFUSE_CTRL);
-		bResult = true;
+		b_result = true;
 	} else {
 		*data = 0xff;
-		bResult = false;
+		b_result = false;
 	}
 
-	return bResult;
+	return b_result;
 }
 
 /*-----------------------------------------------------------------------------
- * Function:	Efuse_ReadAllMap
+ * Function:	efuse_read_all_map
  *
- * Overview:	Read All Efuse content
+ * Overview:	Read All efuse content
  *
  * Input:       NONE
  *
@@ -132,67 +132,19 @@ u8 rtw_efuse_one_byte_read(struct adapter *padapter, u16 addr, u8 *data)
  * 11/11/2008	MHC		Create Version 0.
  *
  */
-static void Efuse_ReadAllMap(struct adapter *padapter, u8 efuseType, u8 *Efuse)
+static void efuse_read_all_map(struct adapter *padapter, u8 efuse_type, u8 *efuse)
 {
-	u16 mapLen = 0;
+	u16 map_len = 0;
 
 	Hal_EfusePowerSwitch(padapter, true);
 
-	Hal_GetEfuseDefinition(padapter, efuseType, TYPE_EFUSE_MAP_LEN, &mapLen);
+	Hal_GetEfuseDefinition(padapter, efuse_type, TYPE_EFUSE_MAP_LEN, &map_len);
 
-	Hal_ReadEFuse(padapter, efuseType, 0, mapLen, Efuse);
+	Hal_ReadEFuse(padapter, efuse_type, 0, map_len, efuse);
 
 	Hal_EfusePowerSwitch(padapter, false);
 }
 
-/*-----------------------------------------------------------------------------
- * Function:	efuse_ShadowRead1Byte
- *		efuse_ShadowRead2Byte
- *		efuse_ShadowRead4Byte
- *
- * Overview:	Read from efuse init map by one/two/four bytes !!!!!
- *
- * Input:       NONE
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Revised History:
- * When			Who		Remark
- * 11/12/2008	MHC		Create Version 0.
- *
- */
-static void efuse_ShadowRead1Byte(struct adapter *padapter, u16 Offset, u8 *Value)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-
-	*Value = pEEPROM->efuse_eeprom_data[Offset];
-
-}	/*  EFUSE_ShadowRead1Byte */
-
-/* Read Two Bytes */
-static void efuse_ShadowRead2Byte(struct adapter *padapter, u16 Offset, u16 *Value)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-
-	*Value = pEEPROM->efuse_eeprom_data[Offset];
-	*Value |= pEEPROM->efuse_eeprom_data[Offset + 1] << 8;
-
-}	/*  EFUSE_ShadowRead2Byte */
-
-/* Read Four Bytes */
-static void efuse_ShadowRead4Byte(struct adapter *padapter, u16 Offset, u32 *Value)
-{
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-
-	*Value = pEEPROM->efuse_eeprom_data[Offset];
-	*Value |= pEEPROM->efuse_eeprom_data[Offset + 1] << 8;
-	*Value |= pEEPROM->efuse_eeprom_data[Offset + 2] << 16;
-	*Value |= pEEPROM->efuse_eeprom_data[Offset + 3] << 24;
-
-}	/*  efuse_ShadowRead4Byte */
-
 /*-----------------------------------------------------------------------------
  * Function:	rtw_efuse_shadow_map_update
  *
@@ -209,45 +161,18 @@ static void efuse_ShadowRead4Byte(struct adapter *padapter, u16 Offset, u32 *Val
  * 11/13/2008	MHC		Create Version 0.
  *
  */
-void rtw_efuse_shadow_map_update(struct adapter *padapter, u8 efuseType)
+void rtw_efuse_shadow_map_update(struct adapter *padapter, u8 efuse_type)
 {
-	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-	u16 mapLen = 0;
+	struct eeprom_priv *eeprom = GET_EEPROM_EFUSE_PRIV(padapter);
+	u16 map_len = 0;
 
-	Hal_GetEfuseDefinition(padapter, efuseType, TYPE_EFUSE_MAP_LEN, &mapLen);
+	Hal_GetEfuseDefinition(padapter, efuse_type, TYPE_EFUSE_MAP_LEN, &map_len);
 
-	if (pEEPROM->bautoload_fail_flag)
-		memset(pEEPROM->efuse_eeprom_data, 0xFF, mapLen);
+	if (eeprom->bautoload_fail_flag)
+		memset(eeprom->efuse_eeprom_data, 0xFF, map_len);
 	else
-		Efuse_ReadAllMap(padapter, efuseType, pEEPROM->efuse_eeprom_data);
+		efuse_read_all_map(padapter, efuse_type, eeprom->efuse_eeprom_data);
 
 	/* PlatformMoveMemory((void *)&pHalData->EfuseMap[EFUSE_MODIFY_MAP][0], */
-	/* void *)&pHalData->EfuseMap[EFUSE_INIT_MAP][0], mapLen); */
+	/* void *)&pHalData->EfuseMap[EFUSE_INIT_MAP][0], map_len); */
 } /*  rtw_efuse_shadow_map_update */
-
-/*-----------------------------------------------------------------------------
- * Function:	rtw_efuse_shadow_read
- *
- * Overview:	Read from efuse init map !!!!!
- *
- * Input:       NONE
- *
- * Output:      NONE
- *
- * Return:      NONE
- *
- * Revised History:
- * When			Who		Remark
- * 11/12/2008	MHC		Create Version 0.
- *
- */
-void rtw_efuse_shadow_read(struct adapter *padapter, u8 Type, u16 Offset, u32 *Value)
-{
-	if (Type == 1)
-		efuse_ShadowRead1Byte(padapter, Offset, (u8 *)Value);
-	else if (Type == 2)
-		efuse_ShadowRead2Byte(padapter, Offset, (u16 *)Value);
-	else if (Type == 4)
-		efuse_ShadowRead4Byte(padapter, Offset, (u32 *)Value);
-
-} /* rtw_efuse_shadow_read*/
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
index d18768a51b199a..e22e33c364bb4e 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
@@ -27,77 +27,77 @@ static void _rtw_set_scan_deny_timer_hdl(struct timer_list *t)
 	rtw_clear_scan_deny(adapter);
 }
 
-static void rtw_init_mlme_timer(struct adapter *padapter)
+static void rtw_init_mlme_timer(struct adapter *adapter)
 {
-	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct	mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	timer_setup(&pmlmepriv->assoc_timer, _rtw_join_timeout_handler, 0);
-	timer_setup(&pmlmepriv->scan_to_timer, rtw_scan_timeout_handler, 0);
-	timer_setup(&pmlmepriv->dynamic_chk_timer,
+	timer_setup(&mlme_priv->assoc_timer, _rtw_join_timeout_handler, 0);
+	timer_setup(&mlme_priv->scan_to_timer, rtw_scan_timeout_handler, 0);
+	timer_setup(&mlme_priv->dynamic_chk_timer,
 		    _dynamic_check_timer_handler, 0);
-	timer_setup(&pmlmepriv->set_scan_deny_timer,
+	timer_setup(&mlme_priv->set_scan_deny_timer,
 		    _rtw_set_scan_deny_timer_hdl, 0);
 }
 
-int rtw_init_mlme_priv(struct adapter *padapter)
+int rtw_init_mlme_priv(struct adapter *adapter)
 {
 	int i;
-	u8 *pbuf;
+	u8 *buf;
 	struct wlan_network *pnetwork;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 	int res = _SUCCESS;
 
-	pmlmepriv->nic_hdl = (u8 *)padapter;
+	mlme_priv->nic_hdl = (u8 *)adapter;
 
-	pmlmepriv->pscanned = NULL;
-	pmlmepriv->fw_state = WIFI_STATION_STATE; /*  Must sync with rtw_wdev_alloc() */
-	pmlmepriv->cur_network.network.infrastructure_mode = NL80211_IFTYPE_UNSPECIFIED;
-	pmlmepriv->scan_mode = SCAN_ACTIVE;/*  1: active, 0: passive. Maybe someday we should rename this variable to "active_mode" (Jeff) */
+	mlme_priv->pscanned = NULL;
+	mlme_priv->fw_state = WIFI_STATION_STATE; /*  Must sync with rtw_wdev_alloc() */
+	mlme_priv->cur_network.network.infrastructure_mode = NL80211_IFTYPE_UNSPECIFIED;
+	mlme_priv->scan_mode = SCAN_ACTIVE;/*  1: active, 0: passive. Maybe someday we should rename this variable to "active_mode" (Jeff) */
 
-	spin_lock_init(&pmlmepriv->lock);
-	INIT_LIST_HEAD(&pmlmepriv->free_bss_pool.queue);
-	spin_lock_init(&pmlmepriv->free_bss_pool.lock);
-	INIT_LIST_HEAD(&pmlmepriv->scanned_queue.queue);
-	spin_lock_init(&pmlmepriv->scanned_queue.lock);
+	spin_lock_init(&mlme_priv->lock);
+	INIT_LIST_HEAD(&mlme_priv->free_bss_pool.queue);
+	spin_lock_init(&mlme_priv->free_bss_pool.lock);
+	INIT_LIST_HEAD(&mlme_priv->scanned_queue.queue);
+	spin_lock_init(&mlme_priv->scanned_queue.lock);
 
-	memset(&pmlmepriv->assoc_ssid, 0, sizeof(struct ndis_802_11_ssid));
+	memset(&mlme_priv->assoc_ssid, 0, sizeof(struct ndis_802_11_ssid));
 
-	pbuf = vzalloc(array_size(MAX_BSS_CNT, sizeof(struct wlan_network)));
+	buf = vzalloc(array_size(MAX_BSS_CNT, sizeof(struct wlan_network)));
 
-	if (!pbuf) {
+	if (!buf) {
 		res = _FAIL;
 		goto exit;
 	}
-	pmlmepriv->free_bss_buf = pbuf;
+	mlme_priv->free_bss_buf = buf;
 
-	pnetwork = (struct wlan_network *)pbuf;
+	pnetwork = (struct wlan_network *)buf;
 
 	for (i = 0; i < MAX_BSS_CNT; i++) {
 		INIT_LIST_HEAD(&pnetwork->list);
 
-		list_add_tail(&pnetwork->list, &pmlmepriv->free_bss_pool.queue);
+		list_add_tail(&pnetwork->list, &mlme_priv->free_bss_pool.queue);
 
 		pnetwork++;
 	}
 
 	/* allocate DMA-able/Non-Page memory for cmd_buf and rsp_buf */
 
-	rtw_clear_scan_deny(padapter);
+	rtw_clear_scan_deny(adapter);
 
 	#define RTW_ROAM_SCAN_RESULT_EXP_MS 5000
 	#define RTW_ROAM_RSSI_DIFF_TH 10
 	#define RTW_ROAM_SCAN_INTERVAL_MS 10000
 
-	pmlmepriv->roam_flags = 0
+	mlme_priv->roam_flags = 0
 		| RTW_ROAM_ON_EXPIRED
 		| RTW_ROAM_ON_RESUME
 		;
 
-	pmlmepriv->roam_scanr_exp_ms = RTW_ROAM_SCAN_RESULT_EXP_MS;
-	pmlmepriv->roam_rssi_diff_th = RTW_ROAM_RSSI_DIFF_TH;
-	pmlmepriv->roam_scan_int_ms = RTW_ROAM_SCAN_INTERVAL_MS;
+	mlme_priv->roam_scanr_exp_ms = RTW_ROAM_SCAN_RESULT_EXP_MS;
+	mlme_priv->roam_rssi_diff_th = RTW_ROAM_RSSI_DIFF_TH;
+	mlme_priv->roam_scan_int_ms = RTW_ROAM_SCAN_INTERVAL_MS;
 
-	rtw_init_mlme_timer(padapter);
+	rtw_init_mlme_timer(adapter);
 
 exit:
 
@@ -113,34 +113,34 @@ static void rtw_free_mlme_ie_data(u8 **ppie, u32 *plen)
 	}
 }
 
-void rtw_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv)
+void rtw_free_mlme_priv_ie_data(struct mlme_priv *mlme_priv)
 {
-	rtw_buf_free(&pmlmepriv->assoc_req, &pmlmepriv->assoc_req_len);
-	rtw_buf_free(&pmlmepriv->assoc_rsp, &pmlmepriv->assoc_rsp_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->wps_beacon_ie, &pmlmepriv->wps_beacon_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->wps_probe_req_ie, &pmlmepriv->wps_probe_req_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->wps_probe_resp_ie, &pmlmepriv->wps_probe_resp_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->wps_assoc_resp_ie, &pmlmepriv->wps_assoc_resp_ie_len);
+	rtw_buf_free(&mlme_priv->assoc_req, &mlme_priv->assoc_req_len);
+	rtw_buf_free(&mlme_priv->assoc_rsp, &mlme_priv->assoc_rsp_len);
+	rtw_free_mlme_ie_data(&mlme_priv->wps_beacon_ie, &mlme_priv->wps_beacon_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->wps_probe_req_ie, &mlme_priv->wps_probe_req_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->wps_probe_resp_ie, &mlme_priv->wps_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->wps_assoc_resp_ie, &mlme_priv->wps_assoc_resp_ie_len);
 
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_beacon_ie, &pmlmepriv->p2p_beacon_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_probe_req_ie, &pmlmepriv->p2p_probe_req_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_probe_resp_ie, &pmlmepriv->p2p_probe_resp_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_go_probe_resp_ie, &pmlmepriv->p2p_go_probe_resp_ie_len);
-	rtw_free_mlme_ie_data(&pmlmepriv->p2p_assoc_req_ie, &pmlmepriv->p2p_assoc_req_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->p2p_beacon_ie, &mlme_priv->p2p_beacon_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->p2p_probe_req_ie, &mlme_priv->p2p_probe_req_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->p2p_probe_resp_ie, &mlme_priv->p2p_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->p2p_go_probe_resp_ie, &mlme_priv->p2p_go_probe_resp_ie_len);
+	rtw_free_mlme_ie_data(&mlme_priv->p2p_assoc_req_ie, &mlme_priv->p2p_assoc_req_ie_len);
 }
 
-void _rtw_free_mlme_priv(struct mlme_priv *pmlmepriv)
+void _rtw_free_mlme_priv(struct mlme_priv *mlme_priv)
 {
-	if (pmlmepriv) {
-		rtw_free_mlme_priv_ie_data(pmlmepriv);
-		vfree(pmlmepriv->free_bss_buf);
+	if (mlme_priv) {
+		rtw_free_mlme_priv_ie_data(mlme_priv);
+		vfree(mlme_priv->free_bss_buf);
 	}
 }
 
-struct wlan_network *rtw_alloc_network(struct mlme_priv *pmlmepriv)
+struct wlan_network *rtw_alloc_network(struct mlme_priv *mlme_priv)
 {
 	struct wlan_network *pnetwork;
-	struct __queue *free_queue = &pmlmepriv->free_bss_pool;
+	struct __queue *free_queue = &mlme_priv->free_bss_pool;
 	struct list_head *plist = NULL;
 
 	spin_lock_bh(&free_queue->lock);
@@ -167,11 +167,11 @@ struct wlan_network *rtw_alloc_network(struct mlme_priv *pmlmepriv)
 	return pnetwork;
 }
 
-void _rtw_free_network(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork, u8 isfreeall)
+void _rtw_free_network(struct mlme_priv *mlme_priv, struct wlan_network *pnetwork, u8 isfreeall)
 {
 	unsigned int delta_time;
 	u32 lifetime = SCANQUEUE_LIFETIME;
-	struct __queue *free_queue = &pmlmepriv->free_bss_pool;
+	struct __queue *free_queue = &mlme_priv->free_bss_pool;
 
 	if (!pnetwork)
 		return;
@@ -179,8 +179,8 @@ void _rtw_free_network(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwor
 	if (pnetwork->fixed)
 		return;
 
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE))
+	if (check_fwstate(mlme_priv, WIFI_ADHOC_MASTER_STATE) ||
+	    check_fwstate(mlme_priv, WIFI_ADHOC_STATE))
 		lifetime = 1;
 
 	if (!isfreeall) {
@@ -198,9 +198,9 @@ void _rtw_free_network(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwor
 	spin_unlock_bh(&free_queue->lock);
 }
 
-void _rtw_free_network_nolock(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork)
+void _rtw_free_network_nolock(struct mlme_priv *mlme_priv, struct wlan_network *pnetwork)
 {
-	struct __queue *free_queue = &pmlmepriv->free_bss_pool;
+	struct __queue *free_queue = &mlme_priv->free_bss_pool;
 
 	if (!pnetwork)
 		return;
@@ -243,12 +243,12 @@ struct wlan_network *_rtw_find_network(struct __queue *scanned_queue, u8 *addr)
 	return pnetwork;
 }
 
-void rtw_free_network_queue(struct adapter *padapter, u8 isfreeall)
+void rtw_free_network_queue(struct adapter *adapter, u8 isfreeall)
 {
 	struct list_head *phead, *plist, *tmp;
 	struct wlan_network *pnetwork;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct __queue *scanned_queue = &pmlmepriv->scanned_queue;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct __queue *scanned_queue = &mlme_priv->scanned_queue;
 
 	spin_lock_bh(&scanned_queue->lock);
 
@@ -256,31 +256,31 @@ void rtw_free_network_queue(struct adapter *padapter, u8 isfreeall)
 	list_for_each_safe(plist, tmp, phead) {
 		pnetwork = list_entry(plist, struct wlan_network, list);
 
-		_rtw_free_network(pmlmepriv, pnetwork, isfreeall);
+		_rtw_free_network(mlme_priv, pnetwork, isfreeall);
 	}
 
 	spin_unlock_bh(&scanned_queue->lock);
 }
 
-bool rtw_if_up(struct adapter *padapter)
+bool rtw_if_up(struct adapter *adapter)
 {
-	if (padapter->driver_stopped || padapter->bSurpriseRemoved ||
-	    !check_fwstate(&padapter->mlmepriv, _FW_LINKED))
+	if (adapter->driver_stopped || adapter->bSurpriseRemoved ||
+	    !check_fwstate(&adapter->mlmepriv, _FW_LINKED))
 		return false;
 
 	return true;
 }
 
-void rtw_generate_random_ibss(u8 *pibss)
+void rtw_generate_random_ibss(u8 *ibss)
 {
 	unsigned long curtime = jiffies;
 
-	pibss[0] = 0x02;  /* in ad-hoc mode bit1 must set to 1 */
-	pibss[1] = 0x11;
-	pibss[2] = 0x87;
-	pibss[3] = (u8)(curtime & 0xff) ;/* p[0]; */
-	pibss[4] = (u8)((curtime >> 8) & 0xff) ;/* p[1]; */
-	pibss[5] = (u8)((curtime >> 16) & 0xff) ;/* p[2]; */
+	ibss[0] = 0x02;  /* in ad-hoc mode bit1 must set to 1 */
+	ibss[1] = 0x11;
+	ibss[2] = 0x87;
+	ibss[3] = (u8)(curtime & 0xff) ;/* p[0]; */
+	ibss[4] = (u8)((curtime >> 8) & 0xff) ;/* p[1]; */
+	ibss[5] = (u8)((curtime >> 16) & 0xff) ;/* p[2]; */
 }
 
 u8 *rtw_get_capability_from_ie(u8 *ie)
@@ -302,16 +302,16 @@ u8 *rtw_get_beacon_interval_from_ie(u8 *ie)
 	return ie + 8;
 }
 
-void rtw_free_mlme_priv(struct mlme_priv *pmlmepriv)
+void rtw_free_mlme_priv(struct mlme_priv *mlme_priv)
 {
-	_rtw_free_mlme_priv(pmlmepriv);
+	_rtw_free_mlme_priv(mlme_priv);
 }
 
-void rtw_free_network_nolock(struct adapter *padapter, struct wlan_network *pnetwork);
-void rtw_free_network_nolock(struct adapter *padapter, struct wlan_network *pnetwork)
+void rtw_free_network_nolock(struct adapter *adapter, struct wlan_network *pnetwork);
+void rtw_free_network_nolock(struct adapter *adapter, struct wlan_network *pnetwork)
 {
-	_rtw_free_network_nolock(&padapter->mlmepriv, pnetwork);
-	rtw_cfg80211_unlink_bss(padapter, pnetwork);
+	_rtw_free_network_nolock(&adapter->mlmepriv, pnetwork);
+	rtw_cfg80211_unlink_bss(adapter, pnetwork);
 }
 
 /*
@@ -406,7 +406,7 @@ struct wlan_network *rtw_get_oldest_wlan_network(struct __queue *scanned_queue)
 }
 
 void update_network(struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src,
-		    struct adapter *padapter, bool update_ie)
+		    struct adapter *adapter, bool update_ie)
 {
 	long rssi_ori = dst->rssi;
 
@@ -417,10 +417,10 @@ void update_network(struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src,
 	long rssi_final;
 
 	/* The rule below is 1/5 for sample value, 4/5 for history value */
-	if (check_fwstate(&padapter->mlmepriv, _FW_LINKED) && is_same_network(&padapter->mlmepriv.cur_network.network, src, 0)) {
+	if (check_fwstate(&adapter->mlmepriv, _FW_LINKED) && is_same_network(&adapter->mlmepriv.cur_network.network, src, 0)) {
 		/* Take the recvpriv's value for the connected AP*/
-		ss_final = padapter->recvpriv.signal_strength;
-		sq_final = padapter->recvpriv.signal_qual;
+		ss_final = adapter->recvpriv.signal_strength;
+		sq_final = adapter->recvpriv.signal_qual;
 		/* the rssi value here is undecorated, and will be used for antenna diversity */
 		if (sq_smp != 101) /* from the right channel */
 			rssi_final = (src->rssi + dst->rssi * 4) / 5;
@@ -428,8 +428,10 @@ void update_network(struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src,
 			rssi_final = rssi_ori;
 	} else {
 		if (sq_smp != 101) { /* from the right channel */
-			ss_final = ((u32)(src->phy_info.signal_strength) + (u32)(dst->phy_info.signal_strength) * 4) / 5;
-			sq_final = ((u32)(src->phy_info.signal_quality) + (u32)(dst->phy_info.signal_quality) * 4) / 5;
+			ss_final = ((u32)(src->phy_info.signal_strength) +
+				    (u32)(dst->phy_info.signal_strength) * 4) / 5;
+			sq_final = ((u32)(src->phy_info.signal_quality) +
+				    (u32)(dst->phy_info.signal_quality) * 4) / 5;
 			rssi_final = (src->rssi + dst->rssi * 4) / 5;
 		} else {
 			/* bss info not receiving from the right channel, use the original RX signal infos */
@@ -452,25 +454,25 @@ void update_network(struct wlan_bssid_ex *dst, struct wlan_bssid_ex *src,
 
 static void update_current_network(struct adapter *adapter, struct wlan_bssid_ex *pnetwork)
 {
-	struct	mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct	mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	if (check_fwstate(pmlmepriv, _FW_LINKED) && (is_same_network(&pmlmepriv->cur_network.network, pnetwork, 0))) {
-		update_network(&pmlmepriv->cur_network.network, pnetwork, adapter, true);
-		if (pmlmepriv->cur_network.network.ie_length < sizeof(struct ndis_802_11_fix_ie))
+	if (check_fwstate(mlme_priv, _FW_LINKED) && (is_same_network(&mlme_priv->cur_network.network, pnetwork, 0))) {
+		update_network(&mlme_priv->cur_network.network, pnetwork, adapter, true);
+		if (mlme_priv->cur_network.network.ie_length < sizeof(struct ndis_802_11_fix_ie))
 			return;
 
-		rtw_update_protection(adapter, (pmlmepriv->cur_network.network.ies) + sizeof(struct ndis_802_11_fix_ie),
-								pmlmepriv->cur_network.network.ie_length - sizeof(struct ndis_802_11_fix_ie));
+		rtw_update_protection(adapter, (mlme_priv->cur_network.network.ies) + sizeof(struct ndis_802_11_fix_ie),
+								mlme_priv->cur_network.network.ie_length - sizeof(struct ndis_802_11_fix_ie));
 	}
 }
 
-/* Caller must hold pmlmepriv->lock first. */
+/* Caller must hold mlme_priv->lock first. */
 void rtw_update_scanned_network(struct adapter *adapter, struct wlan_bssid_ex *target)
 {
 	struct list_head *plist, *phead;
 	u32 bssid_ex_sz;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct __queue *queue = &pmlmepriv->scanned_queue;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct __queue *queue = &mlme_priv->scanned_queue;
 	struct wlan_network *pnetwork = NULL;
 	struct wlan_network *oldest = NULL;
 	int target_find = 0;
@@ -499,7 +501,7 @@ void rtw_update_scanned_network(struct adapter *adapter, struct wlan_bssid_ex *t
 	 * with this beacon's information
 	 */
 	if (!target_find) {
-		if (list_empty(&pmlmepriv->free_bss_pool.queue)) {
+		if (list_empty(&mlme_priv->free_bss_pool.queue)) {
 			/* If there are no more slots, expire the oldest */
 			/* list_del_init(&oldest->list); */
 			pnetwork = oldest;
@@ -521,7 +523,7 @@ void rtw_update_scanned_network(struct adapter *adapter, struct wlan_bssid_ex *t
 		} else {
 			/* Otherwise just pull from the free list */
 
-			pnetwork = rtw_alloc_network(pmlmepriv); /*  will update scan_time */
+			pnetwork = rtw_alloc_network(mlme_priv); /*  will update scan_time */
 
 			if (!pnetwork)
 				goto exit;
@@ -583,7 +585,7 @@ static void rtw_add_network(struct adapter *adapter, struct wlan_bssid_ex *pnetw
 static bool rtw_is_desired_network(struct adapter *adapter, struct wlan_network *pnetwork)
 {
 	struct security_priv *psecuritypriv = &adapter->securitypriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 	u32 desired_encmode;
 	u32 privacy;
 	uint wps_ielen;
@@ -592,7 +594,7 @@ static bool rtw_is_desired_network(struct adapter *adapter, struct wlan_network
 	desired_encmode = psecuritypriv->ndisencryptstatus;
 	privacy = pnetwork->network.privacy;
 
-	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+	if (check_fwstate(mlme_priv, WIFI_UNDER_WPS)) {
 		if (pnetwork->network.ie_length < _FIXED_IE_LENGTH_)
 			return false;
 		if (rtw_get_wps_ie(pnetwork->network.ies + _FIXED_IE_LENGTH_, pnetwork->network.ie_length - _FIXED_IE_LENGTH_, NULL, &wps_ielen))
@@ -624,47 +626,47 @@ static bool rtw_is_desired_network(struct adapter *adapter, struct wlan_network
 	if ((desired_encmode != Ndis802_11EncryptionDisabled) && (privacy == 0))
 		bselected = false;
 
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-		if (pnetwork->network.infrastructure_mode != pmlmepriv->cur_network.network.infrastructure_mode)
+	if (check_fwstate(mlme_priv, WIFI_ADHOC_STATE)) {
+		if (pnetwork->network.infrastructure_mode != mlme_priv->cur_network.network.infrastructure_mode)
 			bselected = false;
 	}
 
 	return bselected;
 }
 
-void rtw_survey_event_callback(struct adapter *adapter, u8 *pbuf)
+void rtw_survey_event_callback(struct adapter *adapter, u8 *buf)
 {
 	u32 len;
 	struct wlan_bssid_ex *pnetwork;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	pnetwork = (struct wlan_bssid_ex *)pbuf;
+	pnetwork = (struct wlan_bssid_ex *)buf;
 
 	len = get_wlan_bssid_ex_sz(pnetwork);
 	if (len > (sizeof(struct wlan_bssid_ex)))
 		return;
 
-	spin_lock_bh(&pmlmepriv->lock);
+	spin_lock_bh(&mlme_priv->lock);
 
 	/*  update IBSS_network 's timestamp */
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
-		if (!memcmp(&pmlmepriv->cur_network.network.mac_address, pnetwork->mac_address, ETH_ALEN)) {
+	if (check_fwstate(mlme_priv, WIFI_ADHOC_MASTER_STATE)) {
+		if (!memcmp(&mlme_priv->cur_network.network.mac_address, pnetwork->mac_address, ETH_ALEN)) {
 			struct wlan_network *ibss_wlan = NULL;
 
-			memcpy(pmlmepriv->cur_network.network.ies, pnetwork->ies, 8);
-			spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-			ibss_wlan = rtw_find_network(&pmlmepriv->scanned_queue,  pnetwork->mac_address);
+			memcpy(mlme_priv->cur_network.network.ies, pnetwork->ies, 8);
+			spin_lock_bh(&mlme_priv->scanned_queue.lock);
+			ibss_wlan = rtw_find_network(&mlme_priv->scanned_queue,  pnetwork->mac_address);
 			if (ibss_wlan) {
 				memcpy(ibss_wlan->network.ies, pnetwork->ies, 8);
-				spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
+				spin_unlock_bh(&mlme_priv->scanned_queue.lock);
 				goto exit;
 			}
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
+			spin_unlock_bh(&mlme_priv->scanned_queue.lock);
 		}
 	}
 
-	/*  lock pmlmepriv->lock when you accessing network_q */
-	if (!check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
+	/*  lock mlme_priv->lock when you accessing network_q */
+	if (!check_fwstate(mlme_priv, _FW_UNDER_LINKING)) {
 		if (pnetwork->ssid.ssid[0] == 0)
 			pnetwork->ssid.ssid_length = 0;
 		rtw_add_network(adapter, pnetwork);
@@ -672,55 +674,55 @@ void rtw_survey_event_callback(struct adapter *adapter, u8 *pbuf)
 
 exit:
 
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_unlock_bh(&mlme_priv->lock);
 }
 
-void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf)
+void rtw_surveydone_event_callback(struct adapter *adapter, u8 *buf)
 {
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	spin_lock_bh(&pmlmepriv->lock);
-	if (pmlmepriv->wps_probe_req_ie) {
-		pmlmepriv->wps_probe_req_ie_len = 0;
-		kfree(pmlmepriv->wps_probe_req_ie);
-		pmlmepriv->wps_probe_req_ie = NULL;
+	spin_lock_bh(&mlme_priv->lock);
+	if (mlme_priv->wps_probe_req_ie) {
+		mlme_priv->wps_probe_req_ie_len = 0;
+		kfree(mlme_priv->wps_probe_req_ie);
+		mlme_priv->wps_probe_req_ie = NULL;
 	}
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)) {
-		spin_unlock_bh(&pmlmepriv->lock);
-		timer_delete_sync(&pmlmepriv->scan_to_timer);
-		spin_lock_bh(&pmlmepriv->lock);
-		_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
+	if (check_fwstate(mlme_priv, _FW_UNDER_SURVEY)) {
+		spin_unlock_bh(&mlme_priv->lock);
+		timer_delete_sync(&mlme_priv->scan_to_timer);
+		spin_lock_bh(&mlme_priv->lock);
+		_clr_fwstate_(mlme_priv, _FW_UNDER_SURVEY);
 	}
 
 	rtw_set_signal_stat_timer(&adapter->recvpriv);
 
-	if (pmlmepriv->to_join) {
-		if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-			if (!check_fwstate(pmlmepriv, _FW_LINKED)) {
-				set_fwstate(pmlmepriv, _FW_UNDER_LINKING);
+	if (mlme_priv->to_join) {
+		if (check_fwstate(mlme_priv, WIFI_ADHOC_STATE)) {
+			if (!check_fwstate(mlme_priv, _FW_LINKED)) {
+				set_fwstate(mlme_priv, _FW_UNDER_LINKING);
 
-				if (rtw_select_and_join_from_scanned_queue(pmlmepriv) == _SUCCESS) {
-					_set_timer(&pmlmepriv->assoc_timer, MAX_JOIN_TIMEOUT);
+				if (rtw_select_and_join_from_scanned_queue(mlme_priv) == _SUCCESS) {
+					_set_timer(&mlme_priv->assoc_timer, MAX_JOIN_TIMEOUT);
 				} else {
 					u8 ret = _SUCCESS;
-					struct wlan_bssid_ex *pdev_network =
+					struct wlan_bssid_ex *dev_network =
 						&adapter->registrypriv.dev_network;
 
-					u8 *pibss = adapter->registrypriv.dev_network.mac_address;
+					u8 *ibss = adapter->registrypriv.dev_network.mac_address;
 
-					/* pmlmepriv->fw_state ^= _FW_UNDER_SURVEY;because don't set assoc_timer */
-					_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
+					/* mlme_priv->fw_state ^= _FW_UNDER_SURVEY;because don't set assoc_timer */
+					_clr_fwstate_(mlme_priv, _FW_UNDER_SURVEY);
 
-					memcpy(&pdev_network->ssid, &pmlmepriv->assoc_ssid,
+					memcpy(&dev_network->ssid, &mlme_priv->assoc_ssid,
 					       sizeof(struct ndis_802_11_ssid));
 
 					rtw_update_registrypriv_dev_network(adapter);
-					rtw_generate_random_ibss(pibss);
+					rtw_generate_random_ibss(ibss);
 
-					pmlmepriv->fw_state = WIFI_ADHOC_MASTER_STATE;
+					mlme_priv->fw_state = WIFI_ADHOC_MASTER_STATE;
 
-					pmlmepriv->to_join = false;
+					mlme_priv->to_join = false;
 
 					ret = rtw_createbss_cmd(adapter);
 					if (ret != _SUCCESS)
@@ -730,37 +732,37 @@ void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf)
 		} else {
 			int s_ret;
 
-			set_fwstate(pmlmepriv, _FW_UNDER_LINKING);
-			pmlmepriv->to_join = false;
-			s_ret = rtw_select_and_join_from_scanned_queue(pmlmepriv);
+			set_fwstate(mlme_priv, _FW_UNDER_LINKING);
+			mlme_priv->to_join = false;
+			s_ret = rtw_select_and_join_from_scanned_queue(mlme_priv);
 			if (s_ret == _SUCCESS) {
-				_set_timer(&pmlmepriv->assoc_timer, MAX_JOIN_TIMEOUT);
+				_set_timer(&mlme_priv->assoc_timer, MAX_JOIN_TIMEOUT);
 			} else if (s_ret == 2) {/* there is no need to wait for join */
-				_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
+				_clr_fwstate_(mlme_priv, _FW_UNDER_LINKING);
 				rtw_indicate_connect(adapter);
 			} else {
 				if (rtw_to_roam(adapter) != 0) {
 					if (rtw_dec_to_roam(adapter) == 0 ||
-					    rtw_sitesurvey_cmd(adapter, &pmlmepriv->assoc_ssid,
+					    rtw_sitesurvey_cmd(adapter, &mlme_priv->assoc_ssid,
 							       1, NULL, 0) != _SUCCESS) {
 						rtw_set_to_roam(adapter, 0);
 						rtw_free_assoc_resources(adapter, 1);
 						rtw_indicate_disconnect(adapter);
 					} else {
-						pmlmepriv->to_join = true;
+						mlme_priv->to_join = true;
 					}
 				} else {
 					rtw_indicate_disconnect(adapter);
 				}
-				_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
+				_clr_fwstate_(mlme_priv, _FW_UNDER_LINKING);
 			}
 		}
 	} else {
 		if (rtw_chk_roam_flags(adapter, RTW_ROAM_ACTIVE)) {
-			if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) &&
-			    check_fwstate(pmlmepriv, _FW_LINKED)) {
-				if (rtw_select_roaming_candidate(pmlmepriv) == _SUCCESS) {
-					receive_disconnect(adapter, pmlmepriv->cur_network.network.mac_address
+			if (check_fwstate(mlme_priv, WIFI_STATION_STATE) &&
+			    check_fwstate(mlme_priv, _FW_LINKED)) {
+				if (rtw_select_roaming_candidate(mlme_priv) == _SUCCESS) {
+					receive_disconnect(adapter, mlme_priv->cur_network.network.mac_address
 						, WLAN_REASON_ACTIVE_ROAM);
 				}
 			}
@@ -768,7 +770,7 @@ void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf)
 	}
 
 unlock:
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_unlock_bh(&mlme_priv->lock);
 
 	rtw_os_xmit_schedule(adapter);
 
@@ -777,18 +779,18 @@ void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf)
 	rtw_indicate_scan_done(adapter, false);
 }
 
-void rtw_dummy_event_callback(struct adapter *adapter, u8 *pbuf)
+void rtw_dummy_event_callback(struct adapter *adapter, u8 *buf)
 {
 }
 
-void rtw_fwdbg_event_callback(struct adapter *adapter, u8 *pbuf)
+void rtw_fwdbg_event_callback(struct adapter *adapter, u8 *buf)
 {
 }
 
-static void free_scanqueue(struct mlme_priv *pmlmepriv)
+static void free_scanqueue(struct mlme_priv *mlme_priv)
 {
-	struct __queue *free_queue = &pmlmepriv->free_bss_pool;
-	struct __queue *scan_queue = &pmlmepriv->scanned_queue;
+	struct __queue *free_queue = &mlme_priv->free_bss_pool;
+	struct __queue *scan_queue = &mlme_priv->scanned_queue;
 	struct list_head *plist, *phead, *ptemp;
 
 	spin_lock_bh(&scan_queue->lock);
@@ -811,34 +813,34 @@ static void free_scanqueue(struct mlme_priv *pmlmepriv)
 static void find_network(struct adapter *adapter)
 {
 	struct wlan_network *pwlan = NULL;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_network *tgt_network = &pmlmepriv->cur_network;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct wlan_network *tgt_network = &mlme_priv->cur_network;
 
-	pwlan = rtw_find_network(&pmlmepriv->scanned_queue, tgt_network->network.mac_address);
+	pwlan = rtw_find_network(&mlme_priv->scanned_queue, tgt_network->network.mac_address);
 	if (!pwlan)
 		return;
 
 	pwlan->fixed = false;
 
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) &&
+	if (check_fwstate(mlme_priv, WIFI_ADHOC_MASTER_STATE) &&
 	    (adapter->stapriv.asoc_sta_count == 1))
 		rtw_free_network_nolock(adapter, pwlan);
 }
 
-/* rtw_free_assoc_resources: the caller has to lock pmlmepriv->lock */
+/* rtw_free_assoc_resources: the caller has to lock mlme_priv->lock */
 void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue)
 {
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_network *tgt_network = &pmlmepriv->cur_network;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct wlan_network *tgt_network = &mlme_priv->cur_network;
 
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_AP_STATE)) {
+	if (check_fwstate(mlme_priv, WIFI_STATION_STATE | WIFI_AP_STATE)) {
 		struct sta_info *psta;
 
 		psta = rtw_get_stainfo(&adapter->stapriv, tgt_network->network.mac_address);
 		rtw_free_stainfo(adapter, psta);
 	}
 
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE | WIFI_AP_STATE)) {
+	if (check_fwstate(mlme_priv, WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE | WIFI_AP_STATE)) {
 		struct sta_info *psta;
 
 		rtw_free_all_stainfo(adapter);
@@ -855,82 +857,82 @@ void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue)
 		adapter->securitypriv.key_mask = 0;
 }
 
-/* rtw_indicate_connect: the caller has to lock pmlmepriv->lock */
-void rtw_indicate_connect(struct adapter *padapter)
+/* rtw_indicate_connect: the caller has to lock mlme_priv->lock */
+void rtw_indicate_connect(struct adapter *adapter)
 {
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	pmlmepriv->to_join = false;
+	mlme_priv->to_join = false;
 
-	if (!check_fwstate(&padapter->mlmepriv, _FW_LINKED)) {
-		set_fwstate(pmlmepriv, _FW_LINKED);
+	if (!check_fwstate(&adapter->mlmepriv, _FW_LINKED)) {
+		set_fwstate(mlme_priv, _FW_LINKED);
 
-		if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-		    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE))
-			rtw_cfg80211_ibss_indicate_connect(padapter);
+		if (check_fwstate(mlme_priv, WIFI_ADHOC_MASTER_STATE) ||
+		    check_fwstate(mlme_priv, WIFI_ADHOC_STATE))
+			rtw_cfg80211_ibss_indicate_connect(adapter);
 		else
-			rtw_cfg80211_indicate_connect(padapter);
+			rtw_cfg80211_indicate_connect(adapter);
 
-		netif_carrier_on(padapter->pnetdev);
+		netif_carrier_on(adapter->pnetdev);
 
-		if (padapter->pid[2] != 0)
-			rtw_signal_process(padapter->pid[2], SIGALRM);
+		if (adapter->pid[2] != 0)
+			rtw_signal_process(adapter->pid[2], SIGALRM);
 	}
 
-	rtw_set_to_roam(padapter, 0);
-	rtw_set_scan_deny(padapter, 3000);
+	rtw_set_to_roam(adapter, 0);
+	rtw_set_scan_deny(adapter, 3000);
 }
 
-/* rtw_indicate_disconnect: the caller has to lock pmlmepriv->lock */
-void rtw_indicate_disconnect(struct adapter *padapter)
+/* rtw_indicate_disconnect: the caller has to lock mlme_priv->lock */
+void rtw_indicate_disconnect(struct adapter *adapter)
 {
-	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct	mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING | WIFI_UNDER_WPS);
+	_clr_fwstate_(mlme_priv, _FW_UNDER_LINKING | WIFI_UNDER_WPS);
 
-	if (rtw_to_roam(padapter) > 0)
-		_clr_fwstate_(pmlmepriv, _FW_LINKED);
+	if (rtw_to_roam(adapter) > 0)
+		_clr_fwstate_(mlme_priv, _FW_LINKED);
 
-	if (check_fwstate(&padapter->mlmepriv, _FW_LINKED) || rtw_to_roam(padapter) <= 0) {
+	if (check_fwstate(&adapter->mlmepriv, _FW_LINKED) || rtw_to_roam(adapter) <= 0) {
 		/*  Do it first for tx broadcast pkt after disconnection issue! */
-		netif_carrier_off(padapter->pnetdev);
+		netif_carrier_off(adapter->pnetdev);
 
-		rtw_cfg80211_indicate_disconnect(padapter);
+		rtw_cfg80211_indicate_disconnect(adapter);
 
-		rtw_reset_securitypriv_cmd(padapter);
+		rtw_reset_securitypriv_cmd(adapter);
 
 		/* set ips_deny_time to avoid enter IPS before LPS leave */
-		rtw_set_ips_deny(padapter, 3000);
+		rtw_set_ips_deny(adapter, 3000);
 
-		_clr_fwstate_(pmlmepriv, _FW_LINKED);
+		_clr_fwstate_(mlme_priv, _FW_LINKED);
 
-		rtw_clear_scan_deny(padapter);
+		rtw_clear_scan_deny(adapter);
 	}
 
-	rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_DISCONNECT, 1);
+	rtw_lps_ctrl_wk_cmd(adapter, LPS_CTRL_DISCONNECT, 1);
 }
 
-inline void rtw_indicate_scan_done(struct adapter *padapter, bool aborted)
+inline void rtw_indicate_scan_done(struct adapter *adapter, bool aborted)
 {
-	rtw_cfg80211_indicate_scan_done(padapter, aborted);
+	rtw_cfg80211_indicate_scan_done(adapter, aborted);
 
-	if ((!adapter_to_pwrctl(padapter)->bInSuspend) &&
-	    (!check_fwstate(&padapter->mlmepriv,
+	if ((!adapter_to_pwrctl(adapter)->bInSuspend) &&
+	    (!check_fwstate(&adapter->mlmepriv,
 			    WIFI_ASOC_STATE | WIFI_UNDER_LINKING))) {
-		rtw_set_ips_deny(padapter, 0);
-		_set_timer(&padapter->mlmepriv.dynamic_chk_timer, 1);
+		rtw_set_ips_deny(adapter, 0);
+		_set_timer(&adapter->mlmepriv.dynamic_chk_timer, 1);
 	}
 }
 
 void rtw_scan_abort(struct adapter *adapter)
 {
 	unsigned long start;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
 
 	start = jiffies;
 	pmlmeext->scan_abort = true;
-	while (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)
+	while (check_fwstate(mlme_priv, _FW_UNDER_SURVEY)
 	       && jiffies_to_msecs(start) <= 200) {
 		if (adapter->driver_stopped || adapter->bSurpriseRemoved)
 			break;
@@ -938,19 +940,19 @@ void rtw_scan_abort(struct adapter *adapter)
 		msleep(20);
 	}
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
+	if (check_fwstate(mlme_priv, _FW_UNDER_SURVEY))
 		rtw_indicate_scan_done(adapter, true);
 
 	pmlmeext->scan_abort = false;
 }
 
-static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *padapter, struct wlan_network *pnetwork)
+static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *adapter, struct wlan_network *pnetwork)
 {
 	int i;
 	struct sta_info *bmc_sta, *psta = NULL;
 	struct recv_reorder_ctrl *preorder_ctrl;
-	struct sta_priv *pstapriv = &padapter->stapriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct sta_priv *pstapriv = &adapter->stapriv;
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
 
 	psta = rtw_get_stainfo(pstapriv, pnetwork->network.mac_address);
 	if (!psta)
@@ -960,27 +962,27 @@ static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *padapter, str
 
 		psta->aid = pnetwork->join_res;
 
-		update_sta_info(padapter, psta);
+		update_sta_info(adapter, psta);
 
 		/* update station supportRate */
 		psta->bssratelen = rtw_get_rateset_len(pnetwork->network.supported_rates);
 		memcpy(psta->bssrateset, pnetwork->network.supported_rates, psta->bssratelen);
-		rtw_hal_update_sta_rate_mask(padapter, psta);
+		rtw_hal_update_sta_rate_mask(adapter, psta);
 
 		psta->wireless_mode = pmlmeext->cur_wireless_mode;
-		psta->raid = networktype_to_raid_ex(padapter, psta);
+		psta->raid = networktype_to_raid_ex(adapter, psta);
 
 		/* sta mode */
-		rtw_hal_set_odm_var(padapter, HAL_ODM_STA_INFO, psta, true);
+		rtw_hal_set_odm_var(adapter, HAL_ODM_STA_INFO, psta, true);
 
 		/* security related */
-		if (padapter->securitypriv.dot11_auth_algrthm == dot11_auth_algrthm_8021x) {
-			padapter->securitypriv.binstallGrpkey = false;
-			padapter->securitypriv.busetkipkey = false;
-			padapter->securitypriv.bgrpkey_handshake = false;
+		if (adapter->securitypriv.dot11_auth_algrthm == dot11_auth_algrthm_8021x) {
+			adapter->securitypriv.binstallGrpkey = false;
+			adapter->securitypriv.busetkipkey = false;
+			adapter->securitypriv.bgrpkey_handshake = false;
 
 			psta->ieee8021x_blocked = true;
-			psta->dot118021XPrivacy = padapter->securitypriv.dot11_privacy_algrthm;
+			psta->dot118021XPrivacy = adapter->securitypriv.dot11_privacy_algrthm;
 
 			memset((u8 *)&psta->dot118021x_UncstKey, 0, sizeof(union Keytype));
 
@@ -995,9 +997,9 @@ static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *padapter, str
 
 		/* When doing the WPS, the wps_ie_len won't equal to 0 */
 		/* And the Wi-Fi driver shouldn't allow the data packet to be transmitted. */
-		if (padapter->securitypriv.wps_ie_len != 0) {
+		if (adapter->securitypriv.wps_ie_len != 0) {
 			psta->ieee8021x_blocked = true;
-			padapter->securitypriv.wps_ie_len = 0;
+			adapter->securitypriv.wps_ie_len = 0;
 		}
 
 		/* for A-MPDU Rx reordering buffer control for bmc_sta & sta_info */
@@ -1011,7 +1013,7 @@ static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *padapter, str
 			preorder_ctrl->wsize_b = 64;/* max_ampdu_sz;ex. 32(kbytes) -> wsize_b =32 */
 		}
 
-		bmc_sta = rtw_get_bcmc_stainfo(padapter);
+		bmc_sta = rtw_get_bcmc_stainfo(adapter);
 		if (bmc_sta) {
 			for (i = 0; i < 16 ; i++) {
 				preorder_ctrl = &bmc_sta->recvreorder_ctrl[i];
@@ -1028,11 +1030,11 @@ static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *padapter, str
 
 /* pnetwork : returns from rtw_joinbss_event_callback */
 /* ptarget_wlan: found from scanned_queue */
-static void rtw_joinbss_update_network(struct adapter *padapter, struct wlan_network *ptarget_wlan,
+static void rtw_joinbss_update_network(struct adapter *adapter, struct wlan_network *ptarget_wlan,
 				       struct wlan_network *pnetwork)
 {
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct wlan_network *cur_network = &mlme_priv->cur_network;
 
 	/*  why not use ptarget_wlan?? */
 	memcpy(&cur_network->network, &pnetwork->network, pnetwork->network.length);
@@ -1042,48 +1044,48 @@ static void rtw_joinbss_update_network(struct adapter *padapter, struct wlan_net
 
 	cur_network->aid = pnetwork->join_res;
 
-	rtw_set_signal_stat_timer(&padapter->recvpriv);
+	rtw_set_signal_stat_timer(&adapter->recvpriv);
 
-	padapter->recvpriv.signal_strength = ptarget_wlan->network.phy_info.signal_strength;
-	padapter->recvpriv.signal_qual = ptarget_wlan->network.phy_info.signal_quality;
+	adapter->recvpriv.signal_strength = ptarget_wlan->network.phy_info.signal_strength;
+	adapter->recvpriv.signal_qual = ptarget_wlan->network.phy_info.signal_quality;
 	/* the ptarget_wlan->network.rssi is raw data, we use ptarget_wlan->network.phy_info.signal_strength instead (has scaled) */
-	padapter->recvpriv.rssi = translate_percentage_to_dbm(ptarget_wlan->network.phy_info.signal_strength);
+	adapter->recvpriv.rssi = translate_percentage_to_dbm(ptarget_wlan->network.phy_info.signal_strength);
 
-	rtw_set_signal_stat_timer(&padapter->recvpriv);
+	rtw_set_signal_stat_timer(&adapter->recvpriv);
 
 	/* update fw_state will clr _FW_UNDER_LINKING here indirectly */
 	switch (pnetwork->network.infrastructure_mode) {
 	case NL80211_IFTYPE_STATION:
 
-			if (pmlmepriv->fw_state & WIFI_UNDER_WPS)
-				pmlmepriv->fw_state = WIFI_STATION_STATE | WIFI_UNDER_WPS;
+			if (mlme_priv->fw_state & WIFI_UNDER_WPS)
+				mlme_priv->fw_state = WIFI_STATION_STATE | WIFI_UNDER_WPS;
 			else
-				pmlmepriv->fw_state = WIFI_STATION_STATE;
+				mlme_priv->fw_state = WIFI_STATION_STATE;
 
 			break;
 	case NL80211_IFTYPE_ADHOC:
-			pmlmepriv->fw_state = WIFI_ADHOC_STATE;
+			mlme_priv->fw_state = WIFI_ADHOC_STATE;
 			break;
 	default:
-			pmlmepriv->fw_state = WIFI_NULL_STATE;
+			mlme_priv->fw_state = WIFI_NULL_STATE;
 			break;
 	}
 
 	if (cur_network->network.ie_length < sizeof(struct ndis_802_11_fix_ie))
 		return;
 
-	rtw_update_protection(padapter, (cur_network->network.ies) + sizeof(struct ndis_802_11_fix_ie),
+	rtw_update_protection(adapter, (cur_network->network.ies) + sizeof(struct ndis_802_11_fix_ie),
 									(cur_network->network.ie_length - sizeof(struct ndis_802_11_fix_ie)));
 
-	rtw_update_ht_cap(padapter, cur_network->network.ies, cur_network->network.ie_length, (u8) cur_network->network.configuration.ds_config);
+	rtw_update_ht_cap(adapter, cur_network->network.ies, cur_network->network.ie_length, (u8) cur_network->network.configuration.ds_config);
 }
 
-static struct rt_pmkid_list backupPMKIDList[NUM_PMKID_CACHE];
+static struct rt_pmkid_list backup_pmkid_list[NUM_PMKID_CACHE];
 void rtw_reset_securitypriv(struct adapter *adapter)
 {
-	u8 backupPMKIDIndex = 0;
-	u8 backupTKIPCountermeasure = 0x00;
-	u32 backupTKIPcountermeasure_time = 0;
+	u8 backup_pmkid_index = 0;
+	u8 backup_tkip_countermeasure = 0x00;
+	u32 backup_tkip_countermeasure_time = 0;
 	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
 
 	spin_lock_bh(&adapter->security_key_mutex);
@@ -1096,10 +1098,10 @@ void rtw_reset_securitypriv(struct adapter *adapter)
 		/*  Backup the btkip_countermeasure information. */
 		/*  When the countermeasure is trigger, the driver have to disconnect with AP for 60 seconds. */
 
-		memcpy(&backupPMKIDList[0], &adapter->securitypriv.PMKIDList[0], sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE);
-		backupPMKIDIndex = adapter->securitypriv.PMKIDIndex;
-		backupTKIPCountermeasure = adapter->securitypriv.btkip_countermeasure;
-		backupTKIPcountermeasure_time = adapter->securitypriv.btkip_countermeasure_time;
+		memcpy(&backup_pmkid_list[0], &adapter->securitypriv.PMKIDList[0], sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE);
+		backup_pmkid_index = adapter->securitypriv.PMKIDIndex;
+		backup_tkip_countermeasure = adapter->securitypriv.btkip_countermeasure;
+		backup_tkip_countermeasure_time = adapter->securitypriv.btkip_countermeasure_time;
 
 		/* reset RX BIP packet number */
 		pmlmeext->mgnt_80211w_IPN_rx = 0;
@@ -1108,10 +1110,10 @@ void rtw_reset_securitypriv(struct adapter *adapter)
 
 		/*  Added by Albert 2009/02/18 */
 		/*  Restore the PMK information to securitypriv structure for the following connection. */
-		memcpy(&adapter->securitypriv.PMKIDList[0], &backupPMKIDList[0], sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE);
-		adapter->securitypriv.PMKIDIndex = backupPMKIDIndex;
-		adapter->securitypriv.btkip_countermeasure = backupTKIPCountermeasure;
-		adapter->securitypriv.btkip_countermeasure_time = backupTKIPcountermeasure_time;
+		memcpy(&adapter->securitypriv.PMKIDList[0], &backup_pmkid_list[0], sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE);
+		adapter->securitypriv.PMKIDIndex = backup_pmkid_index;
+		adapter->securitypriv.btkip_countermeasure = backup_tkip_countermeasure;
+		adapter->securitypriv.btkip_countermeasure_time = backup_tkip_countermeasure_time;
 
 		adapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen;
 		adapter->securitypriv.ndisencryptstatus = Ndis802_11WEPDisabled;
@@ -1143,13 +1145,13 @@ void rtw_reset_securitypriv(struct adapter *adapter)
 /* if join_res > 0, for (fw_state ==WIFI_ADHOC_STATE), we only check if "ptarget_wlan" exist. */
 /* if join_res > 0, update "cur_network->network" from "pnetwork->network" if (ptarget_wlan != NULL). */
 /*  */
-void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf)
+void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *buf)
 {
 	struct sta_info *ptarget_sta = NULL, *pcur_sta = NULL;
 	struct sta_priv *pstapriv = &adapter->stapriv;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct wlan_network *pnetwork = (struct wlan_network *)pbuf;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct wlan_network *pnetwork = (struct wlan_network *)buf;
+	struct wlan_network *cur_network = &mlme_priv->cur_network;
 	struct wlan_network *pcur_wlan = NULL, *ptarget_wlan = NULL;
 	unsigned int the_same_macaddr = false;
 
@@ -1159,43 +1161,43 @@ void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf)
 	if (pnetwork->network.length > sizeof(struct wlan_bssid_ex))
 		return;
 
-	spin_lock_bh(&pmlmepriv->lock);
+	spin_lock_bh(&mlme_priv->lock);
 
-	pmlmepriv->link_detect_info.traffic_transition_count = 0;
-	pmlmepriv->link_detect_info.low_power_transition_count = 0;
+	mlme_priv->link_detect_info.traffic_transition_count = 0;
+	mlme_priv->link_detect_info.low_power_transition_count = 0;
 
 	if (pnetwork->join_res == -4) {
 		rtw_reset_securitypriv(adapter);
-		_set_timer(&pmlmepriv->assoc_timer, 1);
+		_set_timer(&mlme_priv->assoc_timer, 1);
 
-		if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING))
-			_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
+		if (check_fwstate(mlme_priv, _FW_UNDER_LINKING))
+			_clr_fwstate_(mlme_priv, _FW_UNDER_LINKING);
 
-		spin_unlock_bh(&pmlmepriv->lock);
+		spin_unlock_bh(&mlme_priv->lock);
 		return;
 	}
 
 	if (pnetwork->join_res <= 0) { /* if join_res < 0 (join fails), then try again */
-		_set_timer(&pmlmepriv->assoc_timer, 1);
-		_clr_fwstate_(pmlmepriv, _FW_UNDER_LINKING);
-		spin_unlock_bh(&pmlmepriv->lock);
+		_set_timer(&mlme_priv->assoc_timer, 1);
+		_clr_fwstate_(mlme_priv, _FW_UNDER_LINKING);
+		spin_unlock_bh(&mlme_priv->lock);
 		return;
 	}
 
-	spin_lock_bh(&pmlmepriv->scanned_queue.lock);
+	spin_lock_bh(&mlme_priv->scanned_queue.lock);
 
-	if (!check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
-		spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-		spin_unlock_bh(&pmlmepriv->lock);
+	if (!check_fwstate(mlme_priv, _FW_UNDER_LINKING)) {
+		spin_unlock_bh(&mlme_priv->scanned_queue.lock);
+		spin_unlock_bh(&mlme_priv->lock);
 		return;
 	}
 
 	/* s1. find ptarget_wlan */
-	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
+	if (check_fwstate(mlme_priv, _FW_LINKED)) {
 		if (the_same_macaddr) {
-			ptarget_wlan = rtw_find_network(&pmlmepriv->scanned_queue, cur_network->network.mac_address);
+			ptarget_wlan = rtw_find_network(&mlme_priv->scanned_queue, cur_network->network.mac_address);
 		} else {
-			pcur_wlan = rtw_find_network(&pmlmepriv->scanned_queue, cur_network->network.mac_address);
+			pcur_wlan = rtw_find_network(&mlme_priv->scanned_queue, cur_network->network.mac_address);
 			if (pcur_wlan)
 				pcur_wlan->fixed = false;
 
@@ -1203,15 +1205,15 @@ void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf)
 			if (pcur_sta)
 				rtw_free_stainfo(adapter, pcur_sta);
 
-			ptarget_wlan = rtw_find_network(&pmlmepriv->scanned_queue, pnetwork->network.mac_address);
-			if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
+			ptarget_wlan = rtw_find_network(&mlme_priv->scanned_queue, pnetwork->network.mac_address);
+			if (check_fwstate(mlme_priv, WIFI_STATION_STATE)) {
 				if (ptarget_wlan)
 					ptarget_wlan->fixed = true;
 			}
 		}
 	} else {
-		ptarget_wlan = _rtw_find_same_network(&pmlmepriv->scanned_queue, pnetwork);
-		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
+		ptarget_wlan = _rtw_find_same_network(&mlme_priv->scanned_queue, pnetwork);
+		if (check_fwstate(mlme_priv, WIFI_STATION_STATE)) {
 			if (ptarget_wlan)
 				ptarget_wlan->fixed = true;
 		}
@@ -1221,39 +1223,39 @@ void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf)
 	if (!ptarget_wlan) {
 		netdev_dbg(adapter->pnetdev,
 			   "Can't find ptarget_wlan when joinbss_event callback\n");
-		spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-		spin_unlock_bh(&pmlmepriv->lock);
+		spin_unlock_bh(&mlme_priv->scanned_queue.lock);
+		spin_unlock_bh(&mlme_priv->lock);
 		return;
 	}
 
 	rtw_joinbss_update_network(adapter, ptarget_wlan, pnetwork);
 
 	/* s3. find ptarget_sta & update ptarget_sta after update cur_network only for station mode */
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
+	if (check_fwstate(mlme_priv, WIFI_STATION_STATE)) {
 		ptarget_sta = rtw_joinbss_update_stainfo(adapter, pnetwork);
 		if (!ptarget_sta) {
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
-			spin_unlock_bh(&pmlmepriv->lock);
+			spin_unlock_bh(&mlme_priv->scanned_queue.lock);
+			spin_unlock_bh(&mlme_priv->lock);
 			return;
 		}
 	}
 
 	/* s4. indicate connect */
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-		pmlmepriv->cur_network_scanned = ptarget_wlan;
+	if (check_fwstate(mlme_priv, WIFI_STATION_STATE)) {
+		mlme_priv->cur_network_scanned = ptarget_wlan;
 		rtw_indicate_connect(adapter);
 	}
 
-	spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
+	spin_unlock_bh(&mlme_priv->scanned_queue.lock);
 
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_unlock_bh(&mlme_priv->lock);
 	/* s5. Cancel assoc_timer */
-	timer_delete_sync(&pmlmepriv->assoc_timer);
+	timer_delete_sync(&mlme_priv->assoc_timer);
 }
 
-void rtw_joinbss_event_callback(struct adapter *adapter, u8 *pbuf)
+void rtw_joinbss_event_callback(struct adapter *adapter, u8 *buf)
 {
-	struct wlan_network *pnetwork = (struct wlan_network *)pbuf;
+	struct wlan_network *pnetwork = (struct wlan_network *)buf;
 
 	mlmeext_joinbss_event_callback(adapter, pnetwork->join_res);
 
@@ -1272,18 +1274,18 @@ void rtw_sta_media_status_rpt(struct adapter *adapter, struct sta_info *psta, u3
 	rtw_hal_set_hwreg(adapter, HW_VAR_H2C_MEDIA_STATUS_RPT, (u8 *)&media_status_rpt);
 }
 
-void rtw_stassoc_event_callback(struct adapter *adapter, u8 *pbuf)
+void rtw_stassoc_event_callback(struct adapter *adapter, u8 *buf)
 {
 	struct sta_info *psta;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct stassoc_event *pstassoc = (struct stassoc_event *)pbuf;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct stassoc_event *pstassoc = (struct stassoc_event *)buf;
+	struct wlan_network *cur_network = &mlme_priv->cur_network;
 	struct wlan_network *ptarget_wlan = NULL;
 
 	if (!rtw_access_ctrl(adapter, pstassoc->macaddr))
 		return;
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+	if (check_fwstate(mlme_priv, WIFI_AP_STATE)) {
 		psta = rtw_get_stainfo(&adapter->stapriv, pstassoc->macaddr);
 		if (psta) {
 			u8 *passoc_req = NULL;
@@ -1342,37 +1344,37 @@ void rtw_stassoc_event_callback(struct adapter *adapter, u8 *pbuf)
 
 	psta->ieee8021x_blocked = false;
 
-	spin_lock_bh(&pmlmepriv->lock);
+	spin_lock_bh(&mlme_priv->lock);
 
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
+	if (check_fwstate(mlme_priv, WIFI_ADHOC_MASTER_STATE) ||
+	    check_fwstate(mlme_priv, WIFI_ADHOC_STATE)) {
 		if (adapter->stapriv.asoc_sta_count == 2) {
-			spin_lock_bh(&pmlmepriv->scanned_queue.lock);
-			ptarget_wlan = rtw_find_network(&pmlmepriv->scanned_queue, cur_network->network.mac_address);
-			pmlmepriv->cur_network_scanned = ptarget_wlan;
+			spin_lock_bh(&mlme_priv->scanned_queue.lock);
+			ptarget_wlan = rtw_find_network(&mlme_priv->scanned_queue, cur_network->network.mac_address);
+			mlme_priv->cur_network_scanned = ptarget_wlan;
 			if (ptarget_wlan)
 				ptarget_wlan->fixed = true;
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
+			spin_unlock_bh(&mlme_priv->scanned_queue.lock);
 			/*  a sta + bc/mc_stainfo (not Ibss_stainfo) */
 			rtw_indicate_connect(adapter);
 		}
 	}
 
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_unlock_bh(&mlme_priv->lock);
 
 	mlmeext_sta_add_event_callback(adapter, psta);
 }
 
-void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
+void rtw_stadel_event_callback(struct adapter *adapter, u8 *buf)
 {
 	int mac_id = (-1);
 	struct sta_info *psta;
 	struct wlan_network *pwlan = NULL;
-	struct wlan_bssid_ex    *pdev_network = NULL;
-	u8 *pibss = NULL;
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
-	struct stadel_event *pstadel = (struct stadel_event *)pbuf;
-	struct wlan_network *tgt_network = &pmlmepriv->cur_network;
+	struct wlan_bssid_ex    *dev_network = NULL;
+	u8 *ibss = NULL;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct stadel_event *pstadel = (struct stadel_event *)buf;
+	struct wlan_network *tgt_network = &mlme_priv->cur_network;
 	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
@@ -1390,15 +1392,15 @@ void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
 		rtw_hal_set_hwreg(adapter, HW_VAR_H2C_MEDIA_STATUS_RPT, (u8 *)&media_status);
 	}
 
-	/* if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) */
+	/* if (check_fwstate(mlme_priv, WIFI_AP_STATE)) */
 	if ((pmlmeinfo->state & 0x03) == WIFI_FW_AP_STATE)
 		return;
 
 	mlmeext_sta_del_event_callback(adapter);
 
-	spin_lock_bh(&pmlmepriv->lock);
+	spin_lock_bh(&mlme_priv->lock);
 
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
+	if (check_fwstate(mlme_priv, WIFI_STATION_STATE)) {
 		u16 reason = *((unsigned short *)(pstadel->rsvd));
 		bool roam = false;
 		struct wlan_network *roam_target = NULL;
@@ -1409,7 +1411,7 @@ void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
 			roam = true;
 		} else if (reason == WLAN_REASON_ACTIVE_ROAM && rtw_chk_roam_flags(adapter, RTW_ROAM_ACTIVE)) {
 			roam = true;
-			roam_target = pmlmepriv->roam_network;
+			roam_target = mlme_priv->roam_network;
 		}
 
 		if (roam) {
@@ -1426,48 +1428,48 @@ void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
 		rtw_free_assoc_resources(adapter, 1);
 		rtw_indicate_disconnect(adapter);
 
-		spin_lock_bh(&pmlmepriv->scanned_queue.lock);
+		spin_lock_bh(&mlme_priv->scanned_queue.lock);
 		/*  remove the network entry in scanned_queue */
-		pwlan = rtw_find_network(&pmlmepriv->scanned_queue, tgt_network->network.mac_address);
+		pwlan = rtw_find_network(&mlme_priv->scanned_queue, tgt_network->network.mac_address);
 		if (pwlan) {
 			pwlan->fixed = false;
 			rtw_free_network_nolock(adapter, pwlan);
 		}
-		spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
+		spin_unlock_bh(&mlme_priv->scanned_queue.lock);
 
 		_rtw_roaming(adapter, roam_target);
 	}
 
-	if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) ||
-	    check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
+	if (check_fwstate(mlme_priv, WIFI_ADHOC_MASTER_STATE) ||
+	    check_fwstate(mlme_priv, WIFI_ADHOC_STATE)) {
 		rtw_free_stainfo(adapter, psta);
 
 		if (adapter->stapriv.asoc_sta_count == 1) {/* a sta + bc/mc_stainfo (not Ibss_stainfo) */
 			u8 ret = _SUCCESS;
 
-			spin_lock_bh(&pmlmepriv->scanned_queue.lock);
+			spin_lock_bh(&mlme_priv->scanned_queue.lock);
 			/* free old ibss network */
-			pwlan = rtw_find_network(&pmlmepriv->scanned_queue, tgt_network->network.mac_address);
+			pwlan = rtw_find_network(&mlme_priv->scanned_queue, tgt_network->network.mac_address);
 			if (pwlan) {
 				pwlan->fixed = false;
 				rtw_free_network_nolock(adapter, pwlan);
 			}
-			spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
+			spin_unlock_bh(&mlme_priv->scanned_queue.lock);
 			/* re-create ibss */
-			pdev_network = &adapter->registrypriv.dev_network;
-			pibss = adapter->registrypriv.dev_network.mac_address;
+			dev_network = &adapter->registrypriv.dev_network;
+			ibss = adapter->registrypriv.dev_network.mac_address;
 
-			memcpy(pdev_network, &tgt_network->network, get_wlan_bssid_ex_sz(&tgt_network->network));
+			memcpy(dev_network, &tgt_network->network, get_wlan_bssid_ex_sz(&tgt_network->network));
 
-			memcpy(&pdev_network->ssid, &pmlmepriv->assoc_ssid, sizeof(struct ndis_802_11_ssid));
+			memcpy(&dev_network->ssid, &mlme_priv->assoc_ssid, sizeof(struct ndis_802_11_ssid));
 
 			rtw_update_registrypriv_dev_network(adapter);
 
-			rtw_generate_random_ibss(pibss);
+			rtw_generate_random_ibss(ibss);
 
-			if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
-				set_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
-				_clr_fwstate_(pmlmepriv, WIFI_ADHOC_STATE);
+			if (check_fwstate(mlme_priv, WIFI_ADHOC_STATE)) {
+				set_fwstate(mlme_priv, WIFI_ADHOC_MASTER_STATE);
+				_clr_fwstate_(mlme_priv, WIFI_ADHOC_STATE);
 			}
 
 			ret = rtw_createbss_cmd(adapter);
@@ -1477,21 +1479,21 @@ void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
 	}
 
 unlock:
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_unlock_bh(&mlme_priv->lock);
 }
 
-void rtw_cpwm_event_callback(struct adapter *padapter, u8 *pbuf)
+void rtw_cpwm_event_callback(struct adapter *adapter, u8 *buf)
 {
 	struct reportpwrstate_parm *preportpwrstate;
 
-	preportpwrstate = (struct reportpwrstate_parm *)pbuf;
-	preportpwrstate->state |= (u8)(adapter_to_pwrctl(padapter)->cpwm_tog + 0x80);
-	cpwm_int_hdl(padapter, preportpwrstate);
+	preportpwrstate = (struct reportpwrstate_parm *)buf;
+	preportpwrstate->state |= (u8)(adapter_to_pwrctl(adapter)->cpwm_tog + 0x80);
+	cpwm_int_hdl(adapter, preportpwrstate);
 }
 
-void rtw_wmm_event_callback(struct adapter *padapter, u8 *pbuf)
+void rtw_wmm_event_callback(struct adapter *adapter, u8 *buf)
 {
-	WMMOnAssocRsp(padapter);
+	WMMOnAssocRsp(adapter);
 }
 
 /*
@@ -1502,12 +1504,12 @@ void _rtw_join_timeout_handler(struct timer_list *t)
 {
 	struct adapter *adapter = timer_container_of(adapter, t,
 						     mlmepriv.assoc_timer);
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
 	if (adapter->driver_stopped || adapter->bSurpriseRemoved)
 		return;
 
-	spin_lock_bh(&pmlmepriv->lock);
+	spin_lock_bh(&mlme_priv->lock);
 
 	if (rtw_to_roam(adapter) > 0) { /* join timeout caused by roaming */
 		while (1) {
@@ -1528,13 +1530,13 @@ void _rtw_join_timeout_handler(struct timer_list *t)
 
 	} else {
 		rtw_indicate_disconnect(adapter);
-		free_scanqueue(pmlmepriv);/*  */
+		free_scanqueue(mlme_priv);/*  */
 
 		/* indicate disconnect for the case that join_timeout and check_fwstate != FW_LINKED */
 		rtw_cfg80211_indicate_disconnect(adapter);
 	}
 
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_unlock_bh(&mlme_priv->lock);
 }
 
 /*
@@ -1545,13 +1547,13 @@ void rtw_scan_timeout_handler(struct timer_list *t)
 {
 	struct adapter *adapter = timer_container_of(adapter, t,
 						     mlmepriv.scan_to_timer);
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	spin_lock_bh(&pmlmepriv->lock);
+	spin_lock_bh(&mlme_priv->lock);
 
-	_clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY);
+	_clr_fwstate_(mlme_priv, _FW_UNDER_SURVEY);
 
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_unlock_bh(&mlme_priv->lock);
 
 	rtw_indicate_scan_done(adapter, true);
 }
@@ -1575,23 +1577,23 @@ void rtw_mlme_reset_auto_scan_int(struct adapter *adapter)
 	}
 }
 
-static void rtw_auto_scan_handler(struct adapter *padapter)
+static void rtw_auto_scan_handler(struct adapter *adapter)
 {
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	rtw_mlme_reset_auto_scan_int(padapter);
+	rtw_mlme_reset_auto_scan_int(adapter);
 
-	if (pmlmepriv->auto_scan_int_ms != 0 &&
-	    jiffies_to_msecs(jiffies - pmlmepriv->scan_start_time) > pmlmepriv->auto_scan_int_ms) {
-		if (!padapter->registrypriv.wifi_spec) {
-			if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY | _FW_UNDER_LINKING))
+	if (mlme_priv->auto_scan_int_ms != 0 &&
+	    jiffies_to_msecs(jiffies - mlme_priv->scan_start_time) > mlme_priv->auto_scan_int_ms) {
+		if (!adapter->registrypriv.wifi_spec) {
+			if (check_fwstate(mlme_priv, _FW_UNDER_SURVEY | _FW_UNDER_LINKING))
 				goto exit;
 
-			if (pmlmepriv->link_detect_info.busy_traffic)
+			if (mlme_priv->link_detect_info.busy_traffic)
 				goto exit;
 		}
 
-		rtw_set_802_11_bssid_list_scan(padapter, NULL, 0);
+		rtw_set_802_11_bssid_list_scan(adapter, NULL, 0);
 	}
 
 exit:
@@ -1624,7 +1626,7 @@ void rtw_dynamic_check_timer_handler(struct adapter *adapter)
 			/* rtw_lps_ctrl_wk_cmd(adapter, LPS_CTRL_ENTER, 1); */
 			rtw_hal_dm_watchdog_in_lps(adapter);
 		} else {
-			/* call rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_LEAVE, 1) in traffic_status_watchdog() */
+			/* call rtw_lps_ctrl_wk_cmd(adapter, LPS_CTRL_LEAVE, 1) in traffic_status_watchdog() */
 		}
 
 	} else {
@@ -1786,34 +1788,34 @@ static int rtw_check_join_candidate(struct mlme_priv *mlme
  * Calling context:
  * The caller of the sub-routine will be in critical section...
  * The caller must hold the following spinlock
- * pmlmepriv->lock
+ * mlme_priv->lock
  */
 
-int rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv)
+int rtw_select_and_join_from_scanned_queue(struct mlme_priv *mlme_priv)
 {
 	int ret;
 	struct list_head *phead;
 	struct adapter *adapter;
-	struct __queue *queue = &pmlmepriv->scanned_queue;
+	struct __queue *queue = &mlme_priv->scanned_queue;
 	struct wlan_network *pnetwork = NULL;
 	struct wlan_network *candidate = NULL;
 
-	adapter = (struct adapter *)pmlmepriv->nic_hdl;
+	adapter = (struct adapter *)mlme_priv->nic_hdl;
 
-	spin_lock_bh(&pmlmepriv->scanned_queue.lock);
+	spin_lock_bh(&mlme_priv->scanned_queue.lock);
 
-	if (pmlmepriv->roam_network) {
-		candidate = pmlmepriv->roam_network;
-		pmlmepriv->roam_network = NULL;
+	if (mlme_priv->roam_network) {
+		candidate = mlme_priv->roam_network;
+		mlme_priv->roam_network = NULL;
 		goto candidate_exist;
 	}
 
 	phead = get_list_head(queue);
-	list_for_each(pmlmepriv->pscanned, phead) {
-		pnetwork = list_entry(pmlmepriv->pscanned,
+	list_for_each(mlme_priv->pscanned, phead) {
+		pnetwork = list_entry(mlme_priv->pscanned,
 				      struct wlan_network, list);
 
-		rtw_check_join_candidate(pmlmepriv, &candidate, pnetwork);
+		rtw_check_join_candidate(mlme_priv, &candidate, pnetwork);
 	}
 
 	if (!candidate) {
@@ -1826,17 +1828,17 @@ int rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv)
 candidate_exist:
 
 	/*  check for situation of  _FW_LINKED */
-	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
+	if (check_fwstate(mlme_priv, _FW_LINKED)) {
 		rtw_disassoc_cmd(adapter, 0, true);
 		rtw_indicate_disconnect(adapter);
 		rtw_free_assoc_resources(adapter, 0);
 	}
 
-	set_fwstate(pmlmepriv, _FW_UNDER_LINKING);
+	set_fwstate(mlme_priv, _FW_UNDER_LINKING);
 	ret = rtw_joinbss_cmd(adapter, candidate);
 
 exit:
-	spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
+	spin_unlock_bh(&mlme_priv->scanned_queue.lock);
 	return ret;
 }
 
@@ -1921,14 +1923,14 @@ signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecurityp
 		break;
 	default:
 		res = _FAIL;
-		kfree(psetkeyparm);
+		kfree_sensitive(psetkeyparm);
 		goto exit;
 	}
 
 	if (enqueue) {
 		pcmd = kzalloc_obj(*pcmd);
 		if (!pcmd) {
-			kfree(psetkeyparm);
+			kfree_sensitive(psetkeyparm);
 			res = _FAIL;  /* try again */
 			goto exit;
 		}
@@ -1944,7 +1946,7 @@ signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecurityp
 		res = rtw_enqueue_cmd(pcmdpriv, pcmd);
 	} else {
 		setkey_hdl(adapter, (u8 *)psetkeyparm);
-		kfree(psetkeyparm);
+		kfree_sensitive(psetkeyparm);
 	}
 exit:
 	return res;
@@ -2014,7 +2016,7 @@ static int SecIsInPMKIDList(struct adapter *Adapter, u8 *bssid)
 /*  13th element in the array is the IE length */
 /*  */
 
-static int rtw_append_pmkid(struct adapter *Adapter, int iEntry, u8 *ie, uint ie_len)
+static int rtw_append_pmkid(struct adapter *Adapter, int pmkid_idx, u8 *ie, uint ie_len)
 {
 	struct security_priv *psecuritypriv = &Adapter->securitypriv;
 
@@ -2024,7 +2026,7 @@ static int rtw_append_pmkid(struct adapter *Adapter, int iEntry, u8 *ie, uint ie
 		ie_len++;
 		ie[ie_len] = 0;	/* PMKID count = 0x0100 */
 		ie_len++;
-		memcpy(&ie[ie_len], &psecuritypriv->PMKIDList[iEntry].PMKID, 16);
+		memcpy(&ie[ie_len], &psecuritypriv->PMKIDList[pmkid_idx].PMKID, 16);
 		ie_len += 16;
 		ie[13] += 18;/* PMKID length = 2+16 */
 	}
@@ -2068,9 +2070,9 @@ signed int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, u
 {
 	u8 authmode = 0x0;
 	uint ielength;
-	int iEntry;
+	int pmkid_idx;
 
-	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 	struct security_priv *psecuritypriv = &adapter->securitypriv;
 	uint ndisauthmode = psecuritypriv->ndisauthtype;
 
@@ -2082,7 +2084,7 @@ signed int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, u
 	if ((ndisauthmode == Ndis802_11AuthModeWPA2) || (ndisauthmode == Ndis802_11AuthModeWPA2PSK))
 		authmode = WLAN_EID_RSN;
 
-	if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) {
+	if (check_fwstate(mlme_priv, WIFI_UNDER_WPS)) {
 		memcpy(out_ie + ielength, psecuritypriv->wps_ie, psecuritypriv->wps_ie_len);
 
 		ielength += psecuritypriv->wps_ie_len;
@@ -2093,175 +2095,175 @@ signed int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, u
 		rtw_report_sec_ie(adapter, authmode, psecuritypriv->supplicant_ie);
 	}
 
-	iEntry = SecIsInPMKIDList(adapter, pmlmepriv->assoc_bssid);
-	if (iEntry < 0)
+	pmkid_idx = SecIsInPMKIDList(adapter, mlme_priv->assoc_bssid);
+	if (pmkid_idx < 0)
 		return ielength;
 
 	if (authmode == WLAN_EID_RSN)
-		ielength = rtw_append_pmkid(adapter, iEntry, out_ie, ielength);
+		ielength = rtw_append_pmkid(adapter, pmkid_idx, out_ie, ielength);
 
 	return ielength;
 }
 
 void rtw_init_registrypriv_dev_network(struct adapter *adapter)
 {
-	struct registry_priv *pregistrypriv = &adapter->registrypriv;
+	struct registry_priv *registry_priv = &adapter->registrypriv;
 	struct eeprom_priv *peepriv = &adapter->eeprompriv;
-	struct wlan_bssid_ex *pdev_network = &pregistrypriv->dev_network;
+	struct wlan_bssid_ex *dev_network = &registry_priv->dev_network;
 	u8 *myhwaddr = myid(peepriv);
 
-	memcpy(pdev_network->mac_address, myhwaddr, ETH_ALEN);
+	memcpy(dev_network->mac_address, myhwaddr, ETH_ALEN);
 
-	memcpy(&pdev_network->ssid, &pregistrypriv->ssid, sizeof(struct ndis_802_11_ssid));
+	memcpy(&dev_network->ssid, &registry_priv->ssid, sizeof(struct ndis_802_11_ssid));
 
-	pdev_network->configuration.length = sizeof(struct ndis_802_11_conf);
-	pdev_network->configuration.beacon_period = 100;
+	dev_network->configuration.length = sizeof(struct ndis_802_11_conf);
+	dev_network->configuration.beacon_period = 100;
 }
 
 void rtw_update_registrypriv_dev_network(struct adapter *adapter)
 {
 	int sz = 0;
-	struct registry_priv *pregistrypriv = &adapter->registrypriv;
-	struct wlan_bssid_ex *pdev_network = &pregistrypriv->dev_network;
+	struct registry_priv *registry_priv = &adapter->registrypriv;
+	struct wlan_bssid_ex *dev_network = &registry_priv->dev_network;
 	struct security_priv *psecuritypriv = &adapter->securitypriv;
 	struct wlan_network *cur_network = &adapter->mlmepriv.cur_network;
 
-	pdev_network->privacy = (psecuritypriv->dot11_privacy_algrthm > 0 ? 1 : 0) ; /*  adhoc no 802.1x */
+	dev_network->privacy = (psecuritypriv->dot11_privacy_algrthm > 0 ? 1 : 0) ; /*  adhoc no 802.1x */
 
-	pdev_network->rssi = 0;
+	dev_network->rssi = 0;
 
-	pdev_network->configuration.ds_config = (pregistrypriv->channel);
+	dev_network->configuration.ds_config = (registry_priv->channel);
 
 	if (cur_network->network.infrastructure_mode == NL80211_IFTYPE_ADHOC)
-		pdev_network->configuration.atim_window = (0);
+		dev_network->configuration.atim_window = (0);
 
-	pdev_network->infrastructure_mode = (cur_network->network.infrastructure_mode);
+	dev_network->infrastructure_mode = (cur_network->network.infrastructure_mode);
 
 	/*  1. Supported rates */
 	/*  2. IE */
 
-	/* rtw_set_supported_rate(pdev_network->supported_rates, pregistrypriv->wireless_mode) ;  will be called in rtw_generate_ie */
-	sz = rtw_generate_ie(pregistrypriv);
+	/* rtw_set_supported_rate(dev_network->supported_rates, registry_priv->wireless_mode) ;  will be called in rtw_generate_ie */
+	sz = rtw_generate_ie(registry_priv);
 
-	pdev_network->ie_length = sz;
+	dev_network->ie_length = sz;
 
-	pdev_network->length = get_wlan_bssid_ex_sz((struct wlan_bssid_ex *)pdev_network);
+	dev_network->length = get_wlan_bssid_ex_sz((struct wlan_bssid_ex *)dev_network);
 
 	/* notes: translate ie_length & length after assign the length to cmdsz in createbss_cmd(); */
-	/* pdev_network->ie_length = cpu_to_le32(sz); */
+	/* dev_network->ie_length = cpu_to_le32(sz); */
 }
 
 /* the function is at passive_level */
-void rtw_joinbss_reset(struct adapter *padapter)
+void rtw_joinbss_reset(struct adapter *adapter)
 {
 	u8 threshold;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
+	struct ht_priv *ht_priv = &mlme_priv->htpriv;
 
 	/* todo: if you want to do something io/reg/hw setting before join_bss, please add code here */
 
-	pmlmepriv->num_FortyMHzIntolerant = 0;
+	mlme_priv->num_FortyMHzIntolerant = 0;
 
-	pmlmepriv->num_sta_no_ht = 0;
+	mlme_priv->num_sta_no_ht = 0;
 
-	phtpriv->ampdu_enable = false;/* reset to disabled */
+	ht_priv->ampdu_enable = false;/* reset to disabled */
 
 	/*  TH = 1 => means that invalidate usb rx aggregation */
 	/*  TH = 0 => means that validate usb rx aggregation, use init value. */
-	if (phtpriv->ht_option) {
-		if (padapter->registrypriv.wifi_spec == 1)
+	if (ht_priv->ht_option) {
+		if (adapter->registrypriv.wifi_spec == 1)
 			threshold = 1;
 		else
 			threshold = 0;
-		rtw_hal_set_hwreg(padapter, HW_VAR_RXDMA_AGG_PG_TH, (u8 *)(&threshold));
+		rtw_hal_set_hwreg(adapter, HW_VAR_RXDMA_AGG_PG_TH, (u8 *)(&threshold));
 	} else {
 		threshold = 1;
-		rtw_hal_set_hwreg(padapter, HW_VAR_RXDMA_AGG_PG_TH, (u8 *)(&threshold));
+		rtw_hal_set_hwreg(adapter, HW_VAR_RXDMA_AGG_PG_TH, (u8 *)(&threshold));
 	}
 }
 
-void rtw_ht_use_default_setting(struct adapter *padapter)
+void rtw_ht_use_default_setting(struct adapter *adapter)
 {
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	bool bHwLDPCSupport = false, bHwSTBCSupport = false;
-	bool bHwSupportBeamformer = false, bHwSupportBeamformee = false;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct ht_priv *ht_priv = &mlme_priv->htpriv;
+	struct registry_priv *registry_priv = &adapter->registrypriv;
+	bool hw_ldpc_support = false, hw_stbc_support = false;
+	bool hw_support_beamformer = false, hw_support_beamformee = false;
 
-	if (pregistrypriv->wifi_spec)
-		phtpriv->bss_coexist = 1;
+	if (registry_priv->wifi_spec)
+		ht_priv->bss_coexist = 1;
 	else
-		phtpriv->bss_coexist = 0;
+		ht_priv->bss_coexist = 0;
 
-	phtpriv->sgi_40m = TEST_FLAG(pregistrypriv->short_gi, BIT(1)) ? true : false;
-	phtpriv->sgi_20m = TEST_FLAG(pregistrypriv->short_gi, BIT(0)) ? true : false;
+	ht_priv->sgi_40m = TEST_FLAG(registry_priv->short_gi, BIT(1)) ? true : false;
+	ht_priv->sgi_20m = TEST_FLAG(registry_priv->short_gi, BIT(0)) ? true : false;
 
 	/*  LDPC support */
-	rtw_hal_get_def_var(padapter, HAL_DEF_RX_LDPC, (u8 *)&bHwLDPCSupport);
-	CLEAR_FLAGS(phtpriv->ldpc_cap);
-	if (bHwLDPCSupport) {
-		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT(4)))
-			SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_RX);
+	rtw_hal_get_def_var(adapter, HAL_DEF_RX_LDPC, (u8 *)&hw_ldpc_support);
+	CLEAR_FLAGS(ht_priv->ldpc_cap);
+	if (hw_ldpc_support) {
+		if (TEST_FLAG(registry_priv->ldpc_cap, BIT(4)))
+			SET_FLAG(ht_priv->ldpc_cap, LDPC_HT_ENABLE_RX);
 	}
-	rtw_hal_get_def_var(padapter, HAL_DEF_TX_LDPC, (u8 *)&bHwLDPCSupport);
-	if (bHwLDPCSupport) {
-		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT(5)))
-			SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_TX);
+	rtw_hal_get_def_var(adapter, HAL_DEF_TX_LDPC, (u8 *)&hw_ldpc_support);
+	if (hw_ldpc_support) {
+		if (TEST_FLAG(registry_priv->ldpc_cap, BIT(5)))
+			SET_FLAG(ht_priv->ldpc_cap, LDPC_HT_ENABLE_TX);
 	}
 
 	/*  STBC */
-	rtw_hal_get_def_var(padapter, HAL_DEF_TX_STBC, (u8 *)&bHwSTBCSupport);
-	CLEAR_FLAGS(phtpriv->stbc_cap);
-	if (bHwSTBCSupport) {
-		if (TEST_FLAG(pregistrypriv->stbc_cap, BIT(5)))
-			SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX);
+	rtw_hal_get_def_var(adapter, HAL_DEF_TX_STBC, (u8 *)&hw_stbc_support);
+	CLEAR_FLAGS(ht_priv->stbc_cap);
+	if (hw_stbc_support) {
+		if (TEST_FLAG(registry_priv->stbc_cap, BIT(5)))
+			SET_FLAG(ht_priv->stbc_cap, STBC_HT_ENABLE_TX);
 	}
-	rtw_hal_get_def_var(padapter, HAL_DEF_RX_STBC, (u8 *)&bHwSTBCSupport);
-	if (bHwSTBCSupport) {
-		if (TEST_FLAG(pregistrypriv->stbc_cap, BIT(4)))
-			SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX);
+	rtw_hal_get_def_var(adapter, HAL_DEF_RX_STBC, (u8 *)&hw_stbc_support);
+	if (hw_stbc_support) {
+		if (TEST_FLAG(registry_priv->stbc_cap, BIT(4)))
+			SET_FLAG(ht_priv->stbc_cap, STBC_HT_ENABLE_RX);
 	}
 
 	/*  Beamforming setting */
-	rtw_hal_get_def_var(padapter, HAL_DEF_EXPLICIT_BEAMFORMER, (u8 *)&bHwSupportBeamformer);
-	rtw_hal_get_def_var(padapter, HAL_DEF_EXPLICIT_BEAMFORMEE, (u8 *)&bHwSupportBeamformee);
-	CLEAR_FLAGS(phtpriv->beamform_cap);
-	if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(4)) && bHwSupportBeamformer)
-		SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
+	rtw_hal_get_def_var(adapter, HAL_DEF_EXPLICIT_BEAMFORMER, (u8 *)&hw_support_beamformer);
+	rtw_hal_get_def_var(adapter, HAL_DEF_EXPLICIT_BEAMFORMEE, (u8 *)&hw_support_beamformee);
+	CLEAR_FLAGS(ht_priv->beamform_cap);
+	if (TEST_FLAG(registry_priv->beamform_cap, BIT(4)) && hw_support_beamformer)
+		SET_FLAG(ht_priv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
 
-	if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(5)) && bHwSupportBeamformee)
-		SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
+	if (TEST_FLAG(registry_priv->beamform_cap, BIT(5)) && hw_support_beamformee)
+		SET_FLAG(ht_priv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
 }
 
-void rtw_build_wmm_ie_ht(struct adapter *padapter, u8 *out_ie, uint *pout_len)
+void rtw_build_wmm_ie_ht(struct adapter *adapter, u8 *out_ie, uint *pout_len)
 {
 	unsigned char WMM_IE[] = {0x00, 0x50, 0xf2, 0x02, 0x00, 0x01, 0x00};
 	int out_len;
 
-	if (padapter->mlmepriv.qospriv.qos_option == 0) {
+	if (adapter->mlmepriv.qospriv.qos_option == 0) {
 		out_len = *pout_len;
 		rtw_set_ie(out_ie + out_len, WLAN_EID_VENDOR_SPECIFIC,
 			   _WMM_IE_Length_, WMM_IE, pout_len);
 
-		padapter->mlmepriv.qospriv.qos_option = 1;
+		adapter->mlmepriv.qospriv.qos_option = 1;
 	}
 }
 
 /* the function is >= passive_level */
-unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_ie, uint in_len, uint *pout_len, u8 channel)
+unsigned int rtw_restructure_ht_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_len, uint *pout_len, u8 channel)
 {
 	u32 ielen, out_len;
 	enum ieee80211_max_ampdu_length_exp max_rx_ampdu_factor;
 	unsigned char *p;
 	struct ieee80211_ht_cap ht_capie;
 	u8 cbw40_enable = 0, stbc_rx_enable = 0, operation_bw = 0;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct registry_priv *registry_priv = &adapter->registrypriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct ht_priv *ht_priv = &mlme_priv->htpriv;
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
 
-	phtpriv->ht_option = false;
+	ht_priv->ht_option = false;
 
 	out_len = *pout_len;
 
@@ -2269,14 +2271,14 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 
 	ht_capie.cap_info = cpu_to_le16(IEEE80211_HT_CAP_DSSSCCK40);
 
-	if (phtpriv->sgi_20m)
+	if (ht_priv->sgi_20m)
 		ht_capie.cap_info |= cpu_to_le16(IEEE80211_HT_CAP_SGI_20);
 
 	/* Get HT BW */
 	if (!in_ie) {
 		/* TDLS: TODO 20/40 issue */
-		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
-			operation_bw = padapter->mlmeextpriv.cur_bwmode;
+		if (check_fwstate(mlme_priv, WIFI_STATION_STATE)) {
+			operation_bw = adapter->mlmeextpriv.cur_bwmode;
 			if (operation_bw > CHANNEL_WIDTH_40)
 				operation_bw = CHANNEL_WIDTH_40;
 		} else {
@@ -2305,24 +2307,24 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 	}
 
 	/* to disable 40M Hz support while gd_bw_40MHz_en = 0 */
-	if ((pregistrypriv->bw_mode & 0x0f) > 0)
+	if ((registry_priv->bw_mode & 0x0f) > 0)
 		cbw40_enable = 1;
 
 	if ((cbw40_enable == 1) && (operation_bw == CHANNEL_WIDTH_40)) {
 		ht_capie.cap_info |= cpu_to_le16(IEEE80211_HT_CAP_SUP_WIDTH);
-		if (phtpriv->sgi_40m)
+		if (ht_priv->sgi_40m)
 			ht_capie.cap_info |= cpu_to_le16(IEEE80211_HT_CAP_SGI_40);
 	}
 
-	if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX))
+	if (TEST_FLAG(ht_priv->stbc_cap, STBC_HT_ENABLE_TX))
 		ht_capie.cap_info |= cpu_to_le16(IEEE80211_HT_CAP_TX_STBC);
 
 	/* todo: disable SM power save mode */
 	ht_capie.cap_info |= cpu_to_le16(IEEE80211_HT_CAP_SM_PS);
 
-	if (TEST_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX)) {
-		if ((channel <= 14 && pregistrypriv->rx_stbc == 0x1) ||	/* enable for 2.4GHz */
-			(pregistrypriv->wifi_spec == 1))
+	if (TEST_FLAG(ht_priv->stbc_cap, STBC_HT_ENABLE_RX)) {
+		if ((channel <= 14 && registry_priv->rx_stbc == 0x1) ||	/* enable for 2.4GHz */
+			(registry_priv->wifi_spec == 1))
 			stbc_rx_enable = 1;
 	}
 
@@ -2338,20 +2340,20 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 	{
 		u32 rx_packet_offset, max_recvbuf_sz;
 
-		rtw_hal_get_def_var(padapter, HAL_DEF_RX_PACKET_OFFSET, &rx_packet_offset);
-		rtw_hal_get_def_var(padapter, HAL_DEF_MAX_RECVBUF_SZ, &max_recvbuf_sz);
+		rtw_hal_get_def_var(adapter, HAL_DEF_RX_PACKET_OFFSET, &rx_packet_offset);
+		rtw_hal_get_def_var(adapter, HAL_DEF_MAX_RECVBUF_SZ, &max_recvbuf_sz);
 	}
 
-	if (padapter->driver_rx_ampdu_factor != 0xFF)
+	if (adapter->driver_rx_ampdu_factor != 0xFF)
 		max_rx_ampdu_factor =
-		  (enum ieee80211_max_ampdu_length_exp)padapter->driver_rx_ampdu_factor;
+		  (enum ieee80211_max_ampdu_length_exp)adapter->driver_rx_ampdu_factor;
 	else
-		rtw_hal_get_def_var(padapter, HW_VAR_MAX_RX_AMPDU_FACTOR,
+		rtw_hal_get_def_var(adapter, HW_VAR_MAX_RX_AMPDU_FACTOR,
 				    &max_rx_ampdu_factor);
 
 	ht_capie.ampdu_params_info = (max_rx_ampdu_factor & 0x03);
 
-	if (padapter->securitypriv.dot11_privacy_algrthm == _AES_)
+	if (adapter->securitypriv.dot11_privacy_algrthm == _AES_)
 		ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & (0x07 << 2));
 	else
 		ht_capie.ampdu_params_info |= (IEEE80211_HT_CAP_AMPDU_DENSITY & 0x00);
@@ -2359,7 +2361,7 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 	rtw_set_ie(out_ie + out_len, WLAN_EID_HT_CAPABILITY,
 		   sizeof(struct ieee80211_ht_cap), (unsigned char *)&ht_capie, pout_len);
 
-	phtpriv->ht_option = true;
+	ht_priv->ht_option = true;
 
 	if (in_ie) {
 		p = rtw_get_ie(in_ie, WLAN_EID_HT_OPERATION, &ielen, in_len);
@@ -2369,31 +2371,31 @@ unsigned int rtw_restructure_ht_ie(struct adapter *padapter, u8 *in_ie, u8 *out_
 		}
 	}
 
-	return phtpriv->ht_option;
+	return ht_priv->ht_option;
 }
 
 /* the function is > passive_level (in critical_section) */
-void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len, u8 channel)
+void rtw_update_ht_cap(struct adapter *adapter, u8 *pie, uint ie_len, u8 channel)
 {
 	u8 *p, max_ampdu_sz;
 	int len;
 	struct ieee80211_ht_cap *pht_capie;
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
-	struct registry_priv *pregistrypriv = &padapter->registrypriv;
-	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct ht_priv *ht_priv = &mlme_priv->htpriv;
+	struct registry_priv *registry_priv = &adapter->registrypriv;
+	struct mlme_ext_priv *pmlmeext = &adapter->mlmeextpriv;
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	u8 cbw40_enable = 0;
 
-	if (!phtpriv->ht_option)
+	if (!ht_priv->ht_option)
 		return;
 
 	if ((!pmlmeinfo->HT_info_enable) || (!pmlmeinfo->HT_caps_enable))
 		return;
 
 	/* maybe needs check if ap supports rx ampdu. */
-	if (!(phtpriv->ampdu_enable) && pregistrypriv->ampdu_enable == 1)
-		phtpriv->ampdu_enable = true;
+	if (!(ht_priv->ampdu_enable) && registry_priv->ampdu_enable == 1)
+		ht_priv->ampdu_enable = true;
 
 	/* check Max Rx A-MPDU Size */
 	len = 0;
@@ -2403,7 +2405,7 @@ void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len, u8 channe
 		max_ampdu_sz = (pht_capie->ampdu_params_info & IEEE80211_HT_CAP_AMPDU_FACTOR);
 		max_ampdu_sz = 1 << (max_ampdu_sz + 3); /*  max_ampdu_sz (kbytes); */
 
-		phtpriv->rx_ampdu_maxlen = max_ampdu_sz;
+		ht_priv->rx_ampdu_maxlen = max_ampdu_sz;
 	}
 
 	len = 0;
@@ -2412,7 +2414,7 @@ void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len, u8 channe
 		/* todo: */
 	}
 
-	if ((pregistrypriv->bw_mode & 0x0f) > 0)
+	if ((registry_priv->bw_mode & 0x0f) > 0)
 		cbw40_enable = 1;
 
 	/* update cur_bwmode & cur_ch_offset */
@@ -2458,21 +2460,21 @@ void rtw_update_ht_cap(struct adapter *padapter, u8 *pie, uint ie_len, u8 channe
 	pmlmeinfo->HT_protection = pmlmeinfo->HT_info.infos[1] & 0x3;
 }
 
-void rtw_issue_addbareq_cmd(struct adapter *padapter, struct xmit_frame *pxmitframe)
+void rtw_issue_addbareq_cmd(struct adapter *adapter, struct xmit_frame *pxmitframe)
 {
 	u8 issued;
 	int priority;
 	struct sta_info *psta;
-	struct ht_priv *phtpriv;
+	struct ht_priv *ht_priv;
 	struct pkt_attrib *pattrib = &pxmitframe->attrib;
 	s32 bmcst = is_multicast_ether_addr(pattrib->ra);
 
-	if (bmcst || (padapter->mlmepriv.link_detect_info.num_tx_ok_in_period < 100))
+	if (bmcst || (adapter->mlmepriv.link_detect_info.num_tx_ok_in_period < 100))
 		return;
 
 	priority = pattrib->priority;
 
-	psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
+	psta = rtw_get_stainfo(&adapter->stapriv, pattrib->ra);
 	if (pattrib->psta != psta)
 		return;
 
@@ -2482,26 +2484,26 @@ void rtw_issue_addbareq_cmd(struct adapter *padapter, struct xmit_frame *pxmitfr
 	if (!(psta->state & _FW_LINKED))
 		return;
 
-	phtpriv = &psta->htpriv;
+	ht_priv = &psta->htpriv;
 
-	if (phtpriv->ht_option && phtpriv->ampdu_enable) {
-		issued = (phtpriv->agg_enable_bitmap >> priority) & 0x1;
-		issued |= (phtpriv->candidate_tid_bitmap >> priority) & 0x1;
+	if (ht_priv->ht_option && ht_priv->ampdu_enable) {
+		issued = (ht_priv->agg_enable_bitmap >> priority) & 0x1;
+		issued |= (ht_priv->candidate_tid_bitmap >> priority) & 0x1;
 
 		if (issued == 0) {
 			psta->htpriv.candidate_tid_bitmap |= BIT((u8)priority);
-			rtw_addbareq_cmd(padapter, (u8) priority, pattrib->ra);
+			rtw_addbareq_cmd(adapter, (u8) priority, pattrib->ra);
 		}
 	}
 }
 
-void rtw_append_exented_cap(struct adapter *padapter, u8 *out_ie, uint *pout_len)
+void rtw_append_exented_cap(struct adapter *adapter, u8 *out_ie, uint *pout_len)
 {
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct ht_priv *ht_priv = &mlme_priv->htpriv;
 	u8 cap_content[8] = {0};
 
-	if (phtpriv->bss_coexist)
+	if (ht_priv->bss_coexist)
 		SET_EXT_CAPABILITY_ELE_BSS_COEXIST(cap_content, 1);
 
 	rtw_set_ie(out_ie + *pout_len, WLAN_EID_EXT_CAPABILITY, 8, cap_content, pout_len);
@@ -2525,43 +2527,43 @@ inline u8 rtw_to_roam(struct adapter *adapter)
 	return adapter->mlmepriv.to_roam;
 }
 
-void rtw_roaming(struct adapter *padapter, struct wlan_network *tgt_network)
+void rtw_roaming(struct adapter *adapter, struct wlan_network *tgt_network)
 {
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
 
-	spin_lock_bh(&pmlmepriv->lock);
-	_rtw_roaming(padapter, tgt_network);
-	spin_unlock_bh(&pmlmepriv->lock);
+	spin_lock_bh(&mlme_priv->lock);
+	_rtw_roaming(adapter, tgt_network);
+	spin_unlock_bh(&mlme_priv->lock);
 }
 
-void _rtw_roaming(struct adapter *padapter, struct wlan_network *tgt_network)
+void _rtw_roaming(struct adapter *adapter, struct wlan_network *tgt_network)
 {
-	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network *cur_network = &pmlmepriv->cur_network;
+	struct mlme_priv *mlme_priv = &adapter->mlmepriv;
+	struct wlan_network *cur_network = &mlme_priv->cur_network;
 
-	if (rtw_to_roam(padapter) > 0) {
-		memcpy(&pmlmepriv->assoc_ssid, &cur_network->network.ssid, sizeof(struct ndis_802_11_ssid));
+	if (rtw_to_roam(adapter) > 0) {
+		memcpy(&mlme_priv->assoc_ssid, &cur_network->network.ssid, sizeof(struct ndis_802_11_ssid));
 
-		pmlmepriv->assoc_by_bssid = false;
+		mlme_priv->assoc_by_bssid = false;
 
-		while (rtw_do_join(padapter) != _SUCCESS) {
-			rtw_dec_to_roam(padapter);
-			if (rtw_to_roam(padapter) <= 0) {
-				rtw_indicate_disconnect(padapter);
+		while (rtw_do_join(adapter) != _SUCCESS) {
+			rtw_dec_to_roam(adapter);
+			if (rtw_to_roam(adapter) <= 0) {
+				rtw_indicate_disconnect(adapter);
 				break;
 			}
 		}
 	}
 }
 
-bool rtw_linked_check(struct adapter *padapter)
+bool rtw_linked_check(struct adapter *adapter)
 {
-	if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) ||
-	    check_fwstate(&padapter->mlmepriv, WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)) {
-		if (padapter->stapriv.asoc_sta_count > 2)
+	if (check_fwstate(&adapter->mlmepriv, WIFI_AP_STATE) ||
+	    check_fwstate(&adapter->mlmepriv, WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)) {
+		if (adapter->stapriv.asoc_sta_count > 2)
 			return true;
 	} else {	/* Station mode */
-		if (check_fwstate(&padapter->mlmepriv, _FW_LINKED))
+		if (check_fwstate(&adapter->mlmepriv, _FW_LINKED))
 			return true;
 	}
 	return false;
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
index e965133d94abba..c39fb142302edf 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
@@ -1971,9 +1971,8 @@ void dump_mgntframe(struct adapter *padapter, struct xmit_frame *pmgntframe)
 	rtw_hal_mgnt_xmit(padapter, pmgntframe);
 }
 
-s32 dump_mgntframe_and_wait(struct adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms)
+void dump_mgntframe_and_wait(struct adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms)
 {
-	s32 ret = _FAIL;
 	unsigned long irqL;
 	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
 	struct xmit_buf *pxmitbuf = pmgntframe->pxmitbuf;
@@ -1983,22 +1982,18 @@ s32 dump_mgntframe_and_wait(struct adapter *padapter, struct xmit_frame *pmgntfr
 		padapter->driver_stopped) {
 		rtw_free_xmitbuf(&padapter->xmitpriv, pmgntframe->pxmitbuf);
 		rtw_free_xmitframe(&padapter->xmitpriv, pmgntframe);
-		return ret;
+		return;
 	}
 
 	rtw_sctx_init(&sctx, timeout_ms);
 	pxmitbuf->sctx = &sctx;
 
-	ret = rtw_hal_mgnt_xmit(padapter, pmgntframe);
-
-	if (ret == _SUCCESS)
-		ret = rtw_sctx_wait(&sctx);
+	if (!rtw_hal_mgnt_xmit(padapter, pmgntframe))
+		rtw_sctx_wait(&sctx);
 
 	spin_lock_irqsave(&pxmitpriv->lock_sctx, irqL);
 	pxmitbuf->sctx = NULL;
 	spin_unlock_irqrestore(&pxmitpriv->lock_sctx, irqL);
-
-	return ret;
 }
 
 s32 dump_mgntframe_and_wait_ack(struct adapter *padapter, struct xmit_frame *pmgntframe)
@@ -2406,11 +2401,9 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 	dump_mgntframe(padapter, pmgntframe);
 }
 
-static int _issue_probereq(struct adapter *padapter,
-			   struct ndis_802_11_ssid *pssid,
-			   u8 *da, u8 ch, bool append_wps, bool wait_ack)
+static void issue_probereq(struct adapter *padapter, struct ndis_802_11_ssid *pssid, u8 *da,
+			   bool append_wps)
 {
-	int ret = _FAIL;
 	struct xmit_frame		*pmgntframe;
 	struct pkt_attrib		*pattrib;
 	unsigned char *pframe;
@@ -2425,7 +2418,7 @@ static int _issue_probereq(struct adapter *padapter,
 
 	pmgntframe = alloc_mgtxmitframe(pxmitpriv);
 	if (!pmgntframe)
-		goto exit;
+		return;
 
 	/* update attribute */
 	pattrib = &pmgntframe->attrib;
@@ -2474,9 +2467,6 @@ static int _issue_probereq(struct adapter *padapter,
 		pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES, bssrate_len, bssrate, &(pattrib->pktlen));
 	}
 
-	if (ch)
-		pframe = rtw_set_ie(pframe, WLAN_EID_DS_PARAMS, 1, &ch, &pattrib->pktlen);
-
 	if (append_wps) {
 		/* add wps_ie for wps2.0 */
 		if (pmlmepriv->wps_probe_req_ie_len > 0 && pmlmepriv->wps_probe_req_ie) {
@@ -2488,51 +2478,7 @@ static int _issue_probereq(struct adapter *padapter,
 
 	pattrib->last_txcmdsz = pattrib->pktlen;
 
-	if (wait_ack) {
-		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
-	} else {
-		dump_mgntframe(padapter, pmgntframe);
-		ret = _SUCCESS;
-	}
-
-exit:
-	return ret;
-}
-
-inline void issue_probereq(struct adapter *padapter, struct ndis_802_11_ssid *pssid, u8 *da)
-{
-	_issue_probereq(padapter, pssid, da, 0, 1, false);
-}
-
-int issue_probereq_ex(struct adapter *padapter, struct ndis_802_11_ssid *pssid, u8 *da, u8 ch, bool append_wps,
-	int try_cnt, int wait_ms)
-{
-	int ret;
-	int i = 0;
-
-	do {
-		ret = _issue_probereq(padapter, pssid, da, ch, append_wps,
-				      wait_ms > 0);
-
-		i++;
-
-		if (padapter->driver_stopped || padapter->bSurpriseRemoved)
-			break;
-
-		if (i < try_cnt && wait_ms > 0 && ret == _FAIL)
-			msleep(wait_ms);
-
-	} while ((i < try_cnt) && ((ret == _FAIL) || (wait_ms == 0)));
-
-	if (ret != _FAIL) {
-		ret = _SUCCESS;
-		#ifndef DBG_XMIT_ACK
-		goto exit;
-		#endif
-	}
-
-exit:
-	return ret;
+	dump_mgntframe(padapter, pmgntframe);
 }
 
 /*  if psta == NULL, indicate we are station(client) now... */
@@ -3771,20 +3717,25 @@ void site_survey(struct adapter *padapter)
 
 					/* IOT issue, When wifi_spec is not set, send one probe req without WPS IE. */
 					if (padapter->registrypriv.wifi_spec)
-						issue_probereq(padapter, &(pmlmeext->sitesurvey_res.ssid[i]), NULL);
+						issue_probereq(padapter,
+							       &pmlmeext->sitesurvey_res.ssid[i],
+							       NULL, true);
 					else
-						issue_probereq_ex(padapter, &(pmlmeext->sitesurvey_res.ssid[i]), NULL, 0, 0, 0, 0);
+						issue_probereq(padapter,
+							       &pmlmeext->sitesurvey_res.ssid[i],
+							       NULL, false);
 
-					issue_probereq(padapter, &(pmlmeext->sitesurvey_res.ssid[i]), NULL);
+					issue_probereq(padapter, &pmlmeext->sitesurvey_res.ssid[i],
+						       NULL, true);
 				}
 
 				if (pmlmeext->sitesurvey_res.scan_mode == SCAN_ACTIVE) {
 					/* IOT issue, When wifi_spec is not set, send one probe req without WPS IE. */
 					if (padapter->registrypriv.wifi_spec)
-						issue_probereq(padapter, NULL, NULL);
+						issue_probereq(padapter, NULL, NULL, true);
 					else
-						issue_probereq_ex(padapter, NULL, NULL, 0, 0, 0, 0);
-					issue_probereq(padapter, NULL, NULL);
+						issue_probereq(padapter, NULL, NULL, false);
+					issue_probereq(padapter, NULL, NULL, true);
 				}
 			}
 		}
@@ -4902,18 +4853,18 @@ void linked_status_chk(struct adapter *padapter)
 			{
 				if (rx_chk != _SUCCESS) {
 					if (pmlmeext->retry == 0) {
-						issue_probereq_ex(padapter,
-								  &pmlmeinfo->network.ssid,
-								  pmlmeinfo->network.mac_address,
-								  0, 0, 0, 0);
-						issue_probereq_ex(padapter,
-								  &pmlmeinfo->network.ssid,
-								  pmlmeinfo->network.mac_address,
-								  0, 0, 0, 0);
-						issue_probereq_ex(padapter,
-								  &pmlmeinfo->network.ssid,
-								  pmlmeinfo->network.mac_address,
-								  0, 0, 0, 0);
+						issue_probereq(padapter,
+							       &pmlmeinfo->network.ssid,
+							       pmlmeinfo->network.mac_address,
+							       false);
+						issue_probereq(padapter,
+							       &pmlmeinfo->network.ssid,
+							       pmlmeinfo->network.mac_address,
+							       false);
+						issue_probereq(padapter,
+							       &pmlmeinfo->network.ssid,
+							       pmlmeinfo->network.mac_address,
+							       false);
 					}
 				}
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c
index 7568fc514d7ce9..3c82012fdc6c31 100644
--- a/drivers/staging/rtl8723bs/core/rtw_recv.c
+++ b/drivers/staging/rtl8723bs/core/rtw_recv.c
@@ -9,6 +9,7 @@
 #include <rtw_recv.h>
 #include <net/cfg80211.h>
 #include <linux/unaligned.h>
+#include <linux/align.h>
 
 static u8 SNAP_ETH_TYPE_IPX[2] = {0x81, 0x37};
 static u8 SNAP_ETH_TYPE_APPLETALK_AARP[2] = {0x80, 0xf3};
@@ -29,7 +30,7 @@ signed int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *pada
 {
 	signed int i;
 	union recv_frame *precvframe;
-	signed int	res = _SUCCESS;
+	signed int res = _SUCCESS;
 
 	spin_lock_init(&precvpriv->lock);
 
@@ -52,7 +53,7 @@ signed int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *pada
 		goto exit;
 	}
 
-	precvpriv->precv_frame_buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(precvpriv->pallocated_frame_buf), RXFRAME_ALIGN_SZ);
+	precvpriv->precv_frame_buf = PTR_ALIGN(precvpriv->pallocated_frame_buf, RXFRAME_ALIGN_SZ);
 	/* precvpriv->precv_frame_buf = precvpriv->pallocated_frame_buf + RXFRAME_ALIGN_SZ - */
 	/* ((SIZE_PTR) (precvpriv->pallocated_frame_buf) &(RXFRAME_ALIGN_SZ-1)); */
 
@@ -89,7 +90,7 @@ void _rtw_free_recv_priv(struct recv_priv *precvpriv)
 {
 	signed int i;
 	union recv_frame *precvframe;
-	struct adapter	*padapter = precvpriv->adapter;
+	struct adapter *padapter = precvpriv->adapter;
 
 	rtw_free_uc_swdec_pending_queue(padapter);
 
@@ -111,8 +112,8 @@ void _rtw_free_recv_priv(struct recv_priv *precvpriv)
 
 union recv_frame *_rtw_alloc_recvframe(struct __queue *pfree_recv_queue)
 {
-	union recv_frame  *precvframe;
-	struct list_head	*plist, *phead;
+	union recv_frame *precvframe;
+	struct list_head *plist, *phead;
 	struct adapter *padapter;
 	struct recv_priv *precvpriv;
 
@@ -138,7 +139,7 @@ union recv_frame *_rtw_alloc_recvframe(struct __queue *pfree_recv_queue)
 
 union recv_frame *rtw_alloc_recvframe(struct __queue *pfree_recv_queue)
 {
-	union recv_frame  *precvframe;
+	union recv_frame *precvframe;
 
 	spin_lock_bh(&pfree_recv_queue->lock);
 
@@ -213,10 +214,10 @@ signed int rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue *q
  *
  */
 
-void rtw_free_recvframe_queue(struct __queue *pframequeue,  struct __queue *pfree_recv_queue)
+void rtw_free_recvframe_queue(struct __queue *pframequeue, struct __queue *pfree_recv_queue)
 {
-	union	recv_frame	*precvframe;
-	struct list_head	*plist, *phead;
+	union recv_frame *precvframe;
+	struct list_head *plist, *phead;
 
 	spin_lock(&pframequeue->lock);
 
@@ -273,7 +274,7 @@ signed int rtw_enqueue_recvbuf(struct recv_buf *precvbuf, struct __queue *queue)
 struct recv_buf *rtw_dequeue_recvbuf(struct __queue *queue)
 {
 	struct recv_buf *precvbuf;
-	struct list_head	*plist, *phead;
+	struct list_head *plist, *phead;
 
 	spin_lock_bh(&queue->lock);
 
@@ -298,7 +299,7 @@ static void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup)
 {
 	enum nl80211_key_type key_type = 0;
 	union iwreq_data wrqu;
-	struct iw_michaelmicfailure    ev;
+	struct iw_michaelmicfailure ev;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 	unsigned long cur_time = 0;
@@ -340,7 +341,7 @@ static void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup)
 
 static signed int recvframe_chkmic(struct adapter *adapter,  union recv_frame *precvframe)
 {
-	signed int	i, res = _SUCCESS;
+	signed int i, res = _SUCCESS;
 	u32 datalen;
 	u8 miccode[8];
 	u8 bmic_err = false, brpt_micerror = true;
@@ -977,7 +978,7 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 		}
 
 		if ((psta->state & WIFI_SLEEP_STATE) && (pstapriv->sta_dz_bitmap & BIT(psta->aid))) {
-			struct list_head	*xmitframe_plist, *xmitframe_phead;
+			struct list_head *xmitframe_plist, *xmitframe_phead;
 			struct xmit_frame *pxmitframe = NULL;
 			struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
 
@@ -1045,12 +1046,12 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 static union recv_frame *recvframe_defrag(struct adapter *adapter,
 					  struct __queue *defrag_q)
 {
-	struct list_head	 *plist, *phead;
-	u8  wlanhdr_offset;
+	struct list_head *plist, *phead;
+	u8 wlanhdr_offset;
 	u8 curfragnum;
 	struct recv_frame_hdr *pfhdr, *pnfhdr;
 	union recv_frame *prframe, *pnextrframe;
-	struct __queue	*pfree_recv_queue;
+	struct __queue *pfree_recv_queue;
 
 	curfragnum = 0;
 	pfree_recv_queue = &adapter->recvpriv.free_recv_queue;
@@ -1262,7 +1263,7 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f
 	u8 *psa, *pda, *pbssid;
 	struct sta_info *psta = NULL;
 	u8 *ptr = precv_frame->u.hdr.rx_data;
-	struct rx_pkt_attrib	*pattrib = &precv_frame->u.hdr.attrib;
+	struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
 	struct security_priv *psecuritypriv = &adapter->securitypriv;
 	signed int ret = _SUCCESS;
 
@@ -1509,13 +1510,13 @@ static signed int validate_recv_frame(struct adapter *adapter, union recv_frame
 /* remove the wlanhdr and add the eth_hdr */
 static signed int wlanhdr_to_ethhdr(union recv_frame *precvframe)
 {
-	signed int	rmv_len;
+	signed int rmv_len;
 	u16 eth_type, len;
 	u8 bsnaphdr;
 	u8 *psnap_type;
-	struct ieee80211_snap_hdr	*psnap;
+	struct ieee80211_snap_hdr *psnap;
 	__be16 be_tmp;
-	struct adapter			*adapter = precvframe->u.hdr.adapter;
+	struct adapter *adapter = precvframe->u.hdr.adapter;
 	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
 	u8 *ptr = precvframe->u.hdr.rx_data; /*  point to frame_ctrl field */
 	struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;
@@ -1523,7 +1524,7 @@ static signed int wlanhdr_to_ethhdr(union recv_frame *precvframe)
 	if (pattrib->encrypt)
 		recvframe_pull_tail(precvframe, pattrib->icv_len);
 
-	psnap = (struct ieee80211_snap_hdr	*)(ptr + pattrib->hdrlen + pattrib->iv_len);
+	psnap = (struct ieee80211_snap_hdr *)(ptr + pattrib->hdrlen + pattrib->iv_len);
 	psnap_type = ptr + pattrib->hdrlen + pattrib->iv_len + SNAP_SIZE;
 	/* convert hdr + possible LLC headers into Ethernet header */
 	/* eth_type = (psnap_type[0] << 8) | psnap_type[1]; */
@@ -1664,7 +1665,7 @@ static void rtw_recv_indicate_pkt(struct adapter *padapter, struct sk_buff *pkt,
 
 static int amsdu_to_msdu(struct adapter *padapter, union recv_frame *prframe)
 {
-	int	a_len, padding_len;
+	int a_len, padding_len;
 	u16 subframe_len;
 	u8 nr_subframes, i;
 	u8 *pdata;
@@ -1844,7 +1845,7 @@ static int rtw_recv_indicatepkt(struct adapter *padapter, union recv_frame *prec
 
 static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reorder_ctrl *preorder_ctrl, int bforced)
 {
-	struct list_head	*phead, *plist;
+	struct list_head *phead, *plist;
 	union recv_frame *prframe;
 	struct rx_pkt_attrib *pattrib;
 	/* u8 index = 0; */
@@ -1855,7 +1856,7 @@ static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reor
 	/* spin_lock_irqsave(&ppending_recvframe_queue->lock, irql); */
 	/* spin_lock(&ppending_recvframe_queue->lock); */
 
-	phead =		get_list_head(ppending_recvframe_queue);
+	phead =	get_list_head(ppending_recvframe_queue);
 	plist = get_next(phead);
 
 	/*  Handling some condition for forced indicate case. */
diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index 2217cffac5f276..3de102a27517d4 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -765,8 +765,8 @@ static void construct_ctr_preload(u8 *ctr_preload,
 	ctr_preload[15] = (unsigned char)(c % 256);
 }
 
-static signed int aes_cipher(u8 *key, uint	hdrlen,
-			u8 *pframe, uint plen)
+static signed int aes_cipher(u8 *key, uint hdrlen,
+			     u8 *pframe, uint plen)
 {
 	uint	qc_exists, a4_exists, i, j, payload_remainder,
 		num_blocks, payload_index;
diff --git a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
index 8c1f4670679b7e..6d49dff616c2b7 100644
--- a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
+++ b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
@@ -180,14 +180,14 @@ void set_mcs_rate_by_mask(u8 *mcs_set, u32 mask)
 	mcs_set[3] &= mcs_rate_4r;
 }
 
-void update_basic_rate_table(struct adapter *Adapter, u8 *mBratesOS)
+void update_basic_rate_table(struct adapter *Adapter, u8 *basic_rates)
 {
 	u8 i;
 	u8 rate;
 
 	/*  1M, 2M, 5.5M, 11M, 6M, 12M, 24M are mandatory. */
 	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
-		rate = mBratesOS[i] & 0x7f;
+		rate = basic_rates[i] & 0x7f;
 		switch (rate) {
 		case IEEE80211_CCK_RATE_1MB:
 		case IEEE80211_CCK_RATE_2MB:
@@ -196,7 +196,7 @@ void update_basic_rate_table(struct adapter *Adapter, u8 *mBratesOS)
 		case IEEE80211_OFDM_RATE_6MB:
 		case IEEE80211_OFDM_RATE_12MB:
 		case IEEE80211_OFDM_RATE_24MB:
-			mBratesOS[i] |= IEEE80211_BASIC_RATE_MASK;
+			basic_rates[i] |= IEEE80211_BASIC_RATE_MASK;
 			break;
 		}
 	}
diff --git a/drivers/staging/rtl8723bs/core/rtw_xmit.c b/drivers/staging/rtl8723bs/core/rtw_xmit.c
index 81d9e713fc12bd..95350fd6a94fc4 100644
--- a/drivers/staging/rtl8723bs/core/rtw_xmit.c
+++ b/drivers/staging/rtl8723bs/core/rtw_xmit.c
@@ -7,6 +7,7 @@
 #include <drv_types.h>
 #include <linux/delay.h>
 #include <linux/if_ether.h>
+#include <linux/align.h>
 
 static u8 P802_1H_OUI[P80211_OUI_LEN] = { 0x00, 0x00, 0xf8 };
 static u8 RFC1042_OUI[P80211_OUI_LEN] = { 0x00, 0x00, 0x00 };
@@ -80,7 +81,7 @@ static int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf
 		if (!pxmitbuf->pallocated_buf)
 			return -ENOMEM;
 
-		pxmitbuf->pbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitbuf->pallocated_buf), XMITBUF_ALIGN_SZ);
+		pxmitbuf->pbuf = PTR_ALIGN(pxmitbuf->pallocated_buf, XMITBUF_ALIGN_SZ);
 	}
 
 	return 0;
@@ -126,7 +127,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		pxmitpriv->pxmit_frame_buf = NULL;
 		return -ENOMEM;
 	}
-	pxmitpriv->pxmit_frame_buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_frame_buf), 4);
+	pxmitpriv->pxmit_frame_buf = PTR_ALIGN(pxmitpriv->pallocated_frame_buf, 4);
 
 	pxframe = (struct xmit_frame *)pxmitpriv->pxmit_frame_buf;
 
@@ -162,7 +163,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 	if (!pxmitpriv->pallocated_xmitbuf)
 		return -ENOMEM;
 
-	pxmitpriv->pxmitbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_xmitbuf), 4);
+	pxmitpriv->pxmitbuf = PTR_ALIGN(pxmitpriv->pallocated_xmitbuf, 4);
 
 	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf;
 
@@ -207,7 +208,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		pxmitpriv->xframe_ext = NULL;
 		return -ENOMEM;
 	}
-	pxmitpriv->xframe_ext = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->xframe_ext_alloc_addr), 4);
+	pxmitpriv->xframe_ext = PTR_ALIGN(pxmitpriv->xframe_ext_alloc_addr, 4);
 	pxframe = (struct xmit_frame *)pxmitpriv->xframe_ext;
 
 	for (i = 0; i < NR_XMIT_EXTBUFF; i++) {
@@ -239,7 +240,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 	if (!pxmitpriv->pallocated_xmit_extbuf)
 		return -ENOMEM;
 
-	pxmitpriv->pxmit_extbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_xmit_extbuf), 4);
+	pxmitpriv->pxmit_extbuf = PTR_ALIGN(pxmitpriv->pallocated_xmit_extbuf, 4);
 
 	pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmit_extbuf;
 
@@ -1188,7 +1189,7 @@ int rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct
 }
 
 /* broadcast or multicast management pkt use BIP, unicast management pkt use CCMP encryption */
-s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct xmit_frame *pxmitframe)
+int rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct xmit_frame *pxmitframe)
 {
 	u8 *pframe, *mem_start = NULL, *tmp_buf = NULL;
 	u8 subtype;
@@ -1211,7 +1212,7 @@ s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, s
 	BIP_AAD = kzalloc(ori_len, GFP_ATOMIC);
 
 	if (!BIP_AAD)
-		return _FAIL;
+		return -ENOMEM;
 
 	tmp_buf = BIP_AAD;
 	subtype = GetFrameSubType(pframe); /* bit(7)~bit(2) */
@@ -1342,12 +1343,12 @@ s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, s
 xmitframe_coalesce_success:
 	spin_unlock_bh(&padapter->security_key_mutex);
 	kfree(BIP_AAD);
-	return _SUCCESS;
+	return 0;
 
 xmitframe_coalesce_fail:
 	spin_unlock_bh(&padapter->security_key_mutex);
 	kfree(BIP_AAD);
-	return _FAIL;
+	return -EINVAL;
 }
 
 /* Logical Link Control(LLC) SubNetwork Attachment Point(SNAP) header
@@ -1534,13 +1535,13 @@ struct xmit_buf *rtw_alloc_xmitbuf_ext(struct xmit_priv *pxmitpriv)
 	return pxmitbuf;
 }
 
-s32 rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
+void rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
 {
 	unsigned long irqL;
 	struct __queue *pfree_queue = &pxmitpriv->free_xmit_extbuf_queue;
 
 	if (!pxmitbuf)
-		return _FAIL;
+		return;
 
 	spin_lock_irqsave(&pfree_queue->lock, irqL);
 
@@ -1550,8 +1551,6 @@ s32 rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
 	pxmitpriv->free_xmit_extbuf_cnt++;
 
 	spin_unlock_irqrestore(&pfree_queue->lock, irqL);
-
-	return _SUCCESS;
 }
 
 struct xmit_buf *rtw_alloc_xmitbuf(struct xmit_priv *pxmitpriv)
@@ -1595,13 +1594,13 @@ struct xmit_buf *rtw_alloc_xmitbuf(struct xmit_priv *pxmitpriv)
 	return pxmitbuf;
 }
 
-s32 rtw_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
+void rtw_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
 {
 	unsigned long irqL;
 	struct __queue *pfree_xmitbuf_queue = &pxmitpriv->free_xmitbuf_queue;
 
 	if (!pxmitbuf)
-		return _FAIL;
+		return;
 
 	if (pxmitbuf->sctx)
 		rtw_sctx_done_err(&pxmitbuf->sctx, RTW_SCTX_DONE_BUF_FREE);
@@ -1620,7 +1619,6 @@ s32 rtw_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf)
 		pxmitpriv->free_xmitbuf_cnt++;
 		spin_unlock_irqrestore(&pfree_xmitbuf_queue->lock, irqL);
 	}
-	return _SUCCESS;
 }
 
 static void rtw_init_xmitframe(struct xmit_frame *pxframe)
@@ -1717,7 +1715,7 @@ struct xmit_frame *rtw_alloc_xmitframe_once(struct xmit_priv *pxmitpriv)
 	if (!alloc_addr)
 		goto exit;
 
-	pxframe = (struct xmit_frame *)N_BYTE_ALIGMENT((SIZE_PTR)(alloc_addr), 4);
+	pxframe = (struct xmit_frame *)PTR_ALIGN(alloc_addr, 4);
 	pxframe->alloc_addr = alloc_addr;
 
 	pxframe->padapter = pxmitpriv->adapter;
@@ -1734,14 +1732,14 @@ struct xmit_frame *rtw_alloc_xmitframe_once(struct xmit_priv *pxmitpriv)
 	return pxframe;
 }
 
-s32 rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitframe)
+void rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitframe)
 {
 	struct __queue *queue = NULL;
 	struct adapter *padapter = pxmitpriv->adapter;
 	struct sk_buff *pndis_pkt = NULL;
 
 	if (!pxmitframe)
-		return _SUCCESS;
+		return;
 
 	if (pxmitframe->pkt) {
 		pndis_pkt = pxmitframe->pkt;
@@ -1750,7 +1748,11 @@ s32 rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitfram
 
 	if (pxmitframe->alloc_addr) {
 		kfree(pxmitframe->alloc_addr);
-		goto check_pkt_complete;
+
+		if (pndis_pkt)
+			rtw_os_pkt_complete(padapter, pndis_pkt);
+
+		return;
 	}
 
 	if (pxmitframe->ext_tag == 0)
@@ -1769,12 +1771,8 @@ s32 rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitfram
 
 	spin_unlock_bh(&queue->lock);
 
-check_pkt_complete:
-
 	if (pndis_pkt)
 		rtw_os_pkt_complete(padapter, pndis_pkt);
-
-	return _SUCCESS;
 }
 
 void rtw_free_xmitframe_queue(struct xmit_priv *pxmitpriv, struct __queue *pframequeue)
@@ -2150,7 +2148,7 @@ static void dequeue_xmitframes_to_sleeping_queue(struct adapter *padapter,
 
 		ret = xmitframe_enqueue_for_sleeping_sta(padapter, pxmitframe);
 
-		if (true == ret) {
+		if (ret) {
 			ptxservq = rtw_get_sta_pending(padapter, psta, pattrib->priority, (u8 *)(&ac_index));
 
 			ptxservq->qcnt--;
diff --git a/drivers/staging/rtl8723bs/hal/HalPhyRf.c b/drivers/staging/rtl8723bs/hal/HalPhyRf.c
index bf44c1050be57e..16a47197037e78 100644
--- a/drivers/staging/rtl8723bs/hal/HalPhyRf.c
+++ b/drivers/staging/rtl8723bs/hal/HalPhyRf.c
@@ -117,9 +117,8 @@ void ODM_TXPowerTrackingCallback_ThermalMeter(struct adapter *Adapter)
 	}
 
 	/* Calculate Average ThermalValue after average enough times */
-	if (ThermalValue_AVG_count) {
+	if (ThermalValue_AVG_count)
 		ThermalValue = (u8)(ThermalValue_AVG / ThermalValue_AVG_count);
-	}
 
 	/* 4 5. Calculate delta, delta_LCK */
 	/* delta" here is used to determine whether thermal value changes or not. */
@@ -226,8 +225,6 @@ void ODM_TXPowerTrackingCallback_ThermalMeter(struct adapter *Adapter)
 		}
 		if (pDM_Odm->RFCalibrateInfo.CCK_index > c.SwingTableSize_CCK - 1)
 			pDM_Odm->RFCalibrateInfo.CCK_index = c.SwingTableSize_CCK - 1;
-		/* else if (pDM_Odm->RFCalibrateInfo.CCK_index < 0) */
-			/* pDM_Odm->RFCalibrateInfo.CCK_index = 0; */
 	} else {
 			for (p = RF_PATH_A; p < c.RfPathCount; p++)
 				pDM_Odm->RFCalibrateInfo.PowerIndexOffset[p] = 0;
diff --git a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
index 6c5f56d5a1f498..ae01560dc8dd98 100644
--- a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
+++ b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
@@ -359,9 +359,6 @@ static u8 phy_PathA_IQK_8723B(
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_RCK_OS, bRFRegOffsetMask, 0x18000);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G1, bRFRegOffsetMask, 0x0003f);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G2, bRFRegOffsetMask, 0xc7f87);
-	/*	disable path B PA in TXIQK mode */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xed, bRFRegOffsetMask, 0x00020); */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x43, bRFRegOffsetMask, 0x40ec1); */
 
 	/* 1 Tx IQK */
 	/* IQK setting */
@@ -372,7 +369,6 @@ static u8 phy_PathA_IQK_8723B(
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_Tone_A, bMaskDWord, 0x38008c1c);
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x8214010a); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x821303ea);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x28110000);
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_B, bMaskDWord, 0x82110000);
@@ -400,7 +396,6 @@ static u8 phy_PathA_IQK_8723B(
 	PHY_SetBBReg(pDM_Odm->Adapter, rIQK_AGC_Pts, bMaskDWord, 0xf8000000);
 
 	/*  delay x ms */
-	/* PlatformStallExecution(IQK_DELAY_TIME_8723B*1000); */
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
@@ -458,7 +453,6 @@ static u8 phy_PathA_RxIQK8723B(
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G1, bRFRegOffsetMask, 0x0001f);
 	/* LNA2 off, PA on for Dcut */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G2, bRFRegOffsetMask, 0xf7fb7);
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x0); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x808000);
 
 	/* IQK setting */
@@ -471,7 +465,6 @@ static u8 phy_PathA_RxIQK8723B(
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x82160c1f); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x82130ff0);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x28110000);
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_B, bMaskDWord, 0x82110000);
@@ -499,7 +492,6 @@ static u8 phy_PathA_RxIQK8723B(
 	PHY_SetBBReg(pDM_Odm->Adapter, rIQK_AGC_Pts, bMaskDWord, 0xf8000000);
 
 	/*  delay x ms */
-	/* PlatformStallExecution(IQK_DELAY_TIME_8723B*1000); */
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
@@ -542,7 +534,6 @@ static u8 phy_PathA_RxIQK8723B(
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G1, bRFRegOffsetMask, 0x0001f);
 	/* LAN2 on, PA off for Dcut */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G2, bRFRegOffsetMask, 0xf7d77);
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x0); */
 
 	/* PA, PAD setting */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xdf, bRFRegOffsetMask, 0xf80);
@@ -558,7 +549,6 @@ static u8 phy_PathA_RxIQK8723B(
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x82110000);
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x281604c2); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x2813001f);
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_B, bMaskDWord, 0x82110000);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_B, bMaskDWord, 0x28110000);
@@ -585,7 +575,6 @@ static u8 phy_PathA_RxIQK8723B(
 	PHY_SetBBReg(pDM_Odm->Adapter, rIQK_AGC_Pts, bMaskDWord, 0xf8000000);
 
 	/*  delay x ms */
-	/* PlatformStallExecution(IQK_DELAY_TIME_8723B*1000); */
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
@@ -637,11 +626,6 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
     /* leave IQK mode */
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
 
-	/*	in TXIQK mode */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x1); */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_RCK_OS, bRFRegOffsetMask, 0x20000); */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G1, bRFRegOffsetMask, 0x0003f); */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G2, bRFRegOffsetMask, 0xc7f87); */
 	/*	enable path B PA in TXIQK mode */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xed, 0x20, 0x1);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x43, bRFRegOffsetMask, 0x30fc1);
@@ -656,7 +640,6 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x82140114); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x821303ea);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x28110000);
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_B, bMaskDWord, 0x82110000);
@@ -670,7 +653,6 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
 
 	/* switch to path B */
 	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, bRFRegOffsetMask, 0xeffe0); */
 
 	/* GNT_BT = 0 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00000800);
@@ -680,7 +662,6 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
 	PHY_SetBBReg(pDM_Odm->Adapter, rIQK_AGC_Pts, bMaskDWord, 0xf8000000);
 
 	/*  delay x ms */
-	/* PlatformStallExecution(IQK_DELAY_TIME_88E*1000); */
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
@@ -748,7 +729,6 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x82160c1f); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x82130ff0);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x28110000);
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_B, bMaskDWord, 0x82110000);
@@ -762,7 +742,6 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 
 	/* switch to path B */
 	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, bRFRegOffsetMask, 0xeffe0); */
 
 	/* GNT_BT = 0 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00000800);
@@ -772,7 +751,6 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	PHY_SetBBReg(pDM_Odm->Adapter, rIQK_AGC_Pts, bMaskDWord, 0xf8000000);
 
 	/*  delay x ms */
-	/* PlatformStallExecution(IQK_DELAY_TIME_88E*1000); */
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
@@ -815,16 +793,11 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_RCK_OS, bRFRegOffsetMask, 0x18000);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G1, bRFRegOffsetMask, 0x0001f);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_TXPA_G2, bRFRegOffsetMask, 0xf7d77);
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x0); */
 
 	/* open PA S1 & close SMIXER */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xed, 0x20, 0x1);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x43, bRFRegOffsetMask, 0x30ebd);
 
-	/* PA, PAD setting */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xdf, bRFRegOffsetMask, 0xf80); */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x56, bRFRegOffsetMask, 0x51000); */
-
 	/* IQK setting */
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK, bMaskDWord, 0x01004800);
 
@@ -835,7 +808,6 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_Tone_B, bMaskDWord, 0x38008c1c);
 
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_A, bMaskDWord, 0x82110000);
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x281604c2); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_A, bMaskDWord, 0x2813001f);
 	PHY_SetBBReg(pDM_Odm->Adapter, rTx_IQK_PI_B, bMaskDWord, 0x82110000);
 	PHY_SetBBReg(pDM_Odm->Adapter, rRx_IQK_PI_B, bMaskDWord, 0x28110000);
@@ -848,7 +820,6 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 
 	/* switch to path B */
 	PHY_SetBBReg(pDM_Odm->Adapter, rS0S1_PathSwitch, bMaskDWord, 0x00000280);
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, bRFRegOffsetMask, 0xeffe0); */
 
 	/* GNT_BT = 0 */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, 0x00000800);
@@ -858,7 +829,6 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	PHY_SetBBReg(pDM_Odm->Adapter, rIQK_AGC_Pts, bMaskDWord, 0xf8000000);
 
 	/*  delay x ms */
-	/* PlatformStallExecution(IQK_DELAY_TIME_88E*1000); */
 	mdelay(IQK_DELAY_TIME_8723B);
 
 	/* restore Ant Path */
@@ -873,11 +843,6 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 	regEAC = PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_After_IQK_A_2, bMaskDWord);
 	regEA4 = PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_Before_IQK_A_2, bMaskDWord);
 
-	/*	PA/PAD controlled by 0x0 */
-	/* leave IQK mode */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, 0xffffff00, 0x00000000); */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_B, 0xdf, bRFRegOffsetMask, 0x180); */
-
 	/* Allen 20131125 */
 	tmp = (regEAC & 0x03FF0000)>>16;
 	if ((tmp & 0x200) > 0)
@@ -948,7 +913,6 @@ static void _PHY_PathAFillIQKMatrix8723B(
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S1][IDX_0xCA0][KEY] = rOFDM0_RxIQExtAnta;
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S1][IDX_0xCA0][VAL] = 0xfffffff & PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_RxIQExtAnta, bMaskDWord);
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S1][IDX_0xC14][KEY] = rOFDM0_XARxIQImbalance;
-/* 			pRFCalibrateInfo->RxIQC_8723B[PATH_S1][IDX_0xC14][VAL] = PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_XARxIQImbalance, bMaskDWord); */
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S1][IDX_0xC14][VAL] = 0x40000100;
 			return;
 		}
@@ -1008,8 +972,6 @@ static void _PHY_PathBFillIQKMatrix8723B(
 
 		/* 2 Tx IQC */
 		PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XDTxAFE, 0xF0000000, ((TX1_C&0x3C0)>>6));
-/* 		pRFCalibrateInfo->TxIQC_8723B[PATH_S0][IDX_0xC9C][KEY] = rOFDM0_XDTxAFE; */
-/* 		pRFCalibrateInfo->TxIQC_8723B[PATH_S0][IDX_0xC9C][VAL] = PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_XDTxAFE, bMaskDWord); */
 		pRFCalibrateInfo->TxIQC_8723B[PATH_S0][IDX_0xC94][KEY] = rOFDM0_XCTxAFE;
 		pRFCalibrateInfo->TxIQC_8723B[PATH_S0][IDX_0xC94][VAL] = PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_XDTxAFE, bMaskDWord);
 
@@ -1023,7 +985,6 @@ static void _PHY_PathBFillIQKMatrix8723B(
 
 		if (bTxOnly) {
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xC14][KEY] = rOFDM0_XARxIQImbalance;
-/* 			pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xC14][VAL] = PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_XARxIQImbalance, bMaskDWord); */
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xC14][VAL] = 0x40000100;
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xCA0][KEY] = rOFDM0_RxIQExtAnta;
 			pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xCA0][VAL] = 0x0fffffff & PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_RxIQExtAnta, bMaskDWord);
@@ -1039,7 +1000,6 @@ static void _PHY_PathBFillIQKMatrix8723B(
 		pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xC14][VAL] = PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_XBRxIQImbalance, bMaskDWord);
 
 		reg = (result[final_candidate][7] >> 6) & 0xF;
-/* 		PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_AGCRSSITable, 0x0000F000, reg); */
 		pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xCA0][KEY] = rOFDM0_RxIQExtAnta;
 		pRFCalibrateInfo->RxIQC_8723B[PATH_S0][IDX_0xCA0][VAL] = (reg << 28)|(PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_RxIQExtAnta, bMaskDWord)&0x0fffffff);
 	}
@@ -1324,8 +1284,6 @@ static void phy_IQCalibrate_8723B(
 	/*  Note: IQ calibration must be performed after loading */
 	/*		PHY_REG.txt , and radio_a, radio_b.txt */
 
-	/* u32 bbvalue; */
-
 	if (t == 0) {
 
 		/*  Save ADDA parameters, turn Path A ADDA on */
@@ -1338,25 +1296,15 @@ static void phy_IQCalibrate_8723B(
 
 /* no serial mode */
 
-	/* save RF path for 8723B */
-/* 	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord); */
-/* 	Path_SEL_RF = PHY_QueryRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, 0xfffff); */
-
 	/* MAC settings */
 	_PHY_MACSettingCalibration8723B(padapter, IQK_MAC_REG, pDM_Odm->RFCalibrateInfo.IQK_MAC_backup);
 
 	/* BB setting */
-	/* PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_RFMOD, BIT24, 0x00); */
 	PHY_SetBBReg(pDM_Odm->Adapter, rCCK0_AFESetting, 0x0f000000, 0xf);
 	PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_TRxPathEnable, bMaskDWord, 0x03a05600);
 	PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_TRMuxPar, bMaskDWord, 0x000800e4);
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_XCD_RFInterfaceSW, bMaskDWord, 0x22204000);
 
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_XAB_RFInterfaceSW, BIT10, 0x01); */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_XAB_RFInterfaceSW, BIT26, 0x01); */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_XA_RFInterfaceOE, BIT10, 0x00); */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_XB_RFInterfaceOE, BIT10, 0x00); */
-
 /* RX IQ calibration setting for 8723B D cut large current issue when leaving IPS */
 
 	PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_IQK, bMaskH3Bytes, 0x000000);
@@ -1388,8 +1336,6 @@ static void phy_IQCalibrate_8723B(
 	for (i = 0 ; i < retryCount ; i++) {
 		PathAOK = phy_PathA_RxIQK8723B(padapter, is2T, RF_Path);
 		if (PathAOK == 0x03) {
-/* 				result[t][0] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_Before_IQK_A, bMaskDWord)&0x3FF0000)>>16; */
-/* 				result[t][1] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_After_IQK_A, bMaskDWord)&0x3FF0000)>>16; */
 				result[t][2] = (PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_Before_IQK_A_2, bMaskDWord)&0x3FF0000)>>16;
 				result[t][3] = (PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_After_IQK_A_2, bMaskDWord)&0x3FF0000)>>16;
 			break;
@@ -1417,8 +1363,6 @@ static void phy_IQCalibrate_8723B(
 		for (i = 0 ; i < retryCount ; i++) {
 			PathBOK = phy_PathB_RxIQK8723B(padapter, is2T);
 			if (PathBOK == 0x03) {
-/* 				result[t][0] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_Before_IQK_A, bMaskDWord)&0x3FF0000)>>16; */
-/* 				result[t][1] = (PHY_QueryBBReg(pDM_Odm->Adapter, rTx_Power_After_IQK_A, bMaskDWord)&0x3FF0000)>>16; */
 				result[t][6] = (PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_Before_IQK_A_2, bMaskDWord)&0x3FF0000)>>16;
 				result[t][7] = (PHY_QueryBBReg(pDM_Odm->Adapter, rRx_Power_After_IQK_A_2, bMaskDWord)&0x3FF0000)>>16;
 				break;
@@ -1440,10 +1384,6 @@ static void phy_IQCalibrate_8723B(
 
 		_PHY_ReloadADDARegisters8723B(padapter, IQK_BB_REG_92C, pDM_Odm->RFCalibrateInfo.IQK_BB_backup, IQK_BB_REG_NUM);
 
-		/* Reload RF path */
-/* 		PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB); */
-/* 		PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, 0xfffff, Path_SEL_RF); */
-
 		/* Allen initial gain 0xc50 */
 		/*  Restore RX initial gain */
 		PHY_SetBBReg(pDM_Odm->Adapter, 0xc50, bMaskByte0, 0x50);
@@ -1553,7 +1493,6 @@ void PHY_IQCalibrate_8723B(
 		rOFDM0_XDTxAFE,
 		rOFDM0_RxIQExtAnta
 	};
-/* 	u32 		Path_SEL_BB = 0; */
 	u32		GNT_BT_default;
 
 	if (!ODM_CheckPowerStatus(padapter))
@@ -1618,13 +1557,6 @@ void PHY_IQCalibrate_8723B(
 
 	/* save default GNT_BT */
 	GNT_BT_default = PHY_QueryBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord);
-	/*  Save RF Path */
-/* 	Path_SEL_BB = PHY_QueryBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord); */
-/* 	Path_SEL_RF = PHY_QueryRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, 0xfffff); */
-
-    /* set GNT_BT = 0, pause BT traffic */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, BIT12, 0x0); */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, BIT11, 0x1); */
 
 	for (i = 0; i < 8; i++) {
 		result[0][i] = 0;
@@ -1716,9 +1648,6 @@ void PHY_IQCalibrate_8723B(
 
 	/* restore GNT_BT */
 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, bMaskDWord, GNT_BT_default);
-	/*  Restore RF Path */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, 0x948, bMaskDWord, Path_SEL_BB); */
-/* 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xb0, 0xfffff, Path_SEL_RF); */
 
 	/* Resotr RX mode table parameter */
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_WE_LUT, 0x80000, 0x1);
@@ -1728,10 +1657,6 @@ void PHY_IQCalibrate_8723B(
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0xed, 0x20, 0x1);
 	PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, 0x43, bRFRegOffsetMask, 0x300bd);
 
-	/* set GNT_BT = HW control */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, BIT12, 0x0); */
-/* 	PHY_SetBBReg(pDM_Odm->Adapter, 0x764, BIT11, 0x0); */
-
 	if (Is2ant) {
 		if (RF_Path == 0x0)	/* S1 */
 			ODM_SetIQCbyRFpath(pDM_Odm, 0);
diff --git a/drivers/staging/rtl8723bs/hal/hal_com.c b/drivers/staging/rtl8723bs/hal/hal_com.c
index 2c856a2721b248..656c75ded16fda 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com.c
@@ -247,13 +247,13 @@ u8 HwRateToMRate(u8 rate)
 	return ret_rate;
 }
 
-void HalSetBrateCfg(struct adapter *Adapter, u8 *mBratesOS, u16 *pBrateCfg)
+void HalSetBrateCfg(struct adapter *Adapter, u8 *basic_rates, u16 *pBrateCfg)
 {
 	u8 i, is_brate, brate;
 
 	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
-		is_brate = mBratesOS[i] & IEEE80211_BASIC_RATE_MASK;
-		brate = mBratesOS[i] & 0x7f;
+		is_brate = basic_rates[i] & IEEE80211_BASIC_RATE_MASK;
+		brate = basic_rates[i] & 0x7f;
 
 		if (is_brate) {
 			switch (brate) {
diff --git a/drivers/staging/rtl8723bs/hal/hal_intf.c b/drivers/staging/rtl8723bs/hal/hal_intf.c
index 2caade1a8bdd1d..1443875d13694d 100644
--- a/drivers/staging/rtl8723bs/hal/hal_intf.c
+++ b/drivers/staging/rtl8723bs/hal/hal_intf.c
@@ -115,8 +115,10 @@ s32	rtw_hal_xmit(struct adapter *padapter, struct xmit_frame *pxmitframe)
 /*
  * [IMPORTANT] This function would be run in interrupt context.
  */
-s32	rtw_hal_mgnt_xmit(struct adapter *padapter, struct xmit_frame *pmgntframe)
+int rtw_hal_mgnt_xmit(struct adapter *padapter, struct xmit_frame *pmgntframe)
 {
+	int ret;
+
 	update_mgntframe_attrib_addr(padapter, pmgntframe);
 	/* pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET; */
 	/* pwlanhdr = (struct rtw_ieee80211_hdr *)pframe; */
@@ -130,7 +132,10 @@ s32	rtw_hal_mgnt_xmit(struct adapter *padapter, struct xmit_frame *pmgntframe)
 			pmgntframe->attrib.encrypt = _AES_;
 			pmgntframe->attrib.bswenc = true;
 		}
-		rtw_mgmt_xmitframe_coalesce(padapter, pmgntframe->pkt, pmgntframe);
+		ret = rtw_mgmt_xmitframe_coalesce(padapter, pmgntframe->pkt, pmgntframe);
+
+		if (ret)
+			return ret;
 	}
 
 	return rtl8723bs_mgnt_xmit(padapter, pmgntframe);
diff --git a/drivers/staging/rtl8723bs/hal/odm.h b/drivers/staging/rtl8723bs/hal/odm.h
index a20b48660e6316..b123df98fa22e0 100644
--- a/drivers/staging/rtl8723bs/hal/odm.h
+++ b/drivers/staging/rtl8723bs/hal/odm.h
@@ -382,7 +382,6 @@ enum { /* _ODM_Support_Ability_Definition */
 /* 	ODM_CMNINFO_INTERFACE */
 enum { /* tag_ODM_Support_Interface_Definition */
 	ODM_ITRF_SDIO	=	0x4,
-	ODM_ITRF_ALL	=	0x7,
 };
 
 /*  ODM_CMNINFO_IC_TYPE */
@@ -936,10 +935,6 @@ struct dm_odm_t { /* DM_Out_Source_Dynamic_Mechanism_Structure */
 	struct timer_list FastAntTrainingTimer;
 
 	/*  ODM relative workitem. */
-
-	#if (BEAMFORMING_SUPPORT == 1)
-	RT_BEAMFORMING_INFO BeamformingInfo;
-	#endif
 };
 
 enum ODM_BB_Config_Type {
@@ -1068,10 +1063,6 @@ u32 ODM_Get_Rate_Bitmap(
 	u8 rssi_level
 );
 
-#if (BEAMFORMING_SUPPORT == 1)
-BEAMFORMING_CAP Beamforming_GetEntryBeamCapByMacId(PMGNT_INFO pMgntInfo, u8 MacId);
-#endif
-
 void odm_TXPowerTrackingInit(struct dm_odm_t *pDM_Odm);
 
 void ODM_DMInit(struct dm_odm_t *pDM_Odm);
diff --git a/drivers/staging/rtl8723bs/hal/odm_DIG.h b/drivers/staging/rtl8723bs/hal/odm_DIG.h
index 5517ef79bdf587..1482290b263f5c 100644
--- a/drivers/staging/rtl8723bs/hal/odm_DIG.h
+++ b/drivers/staging/rtl8723bs/hal/odm_DIG.h
@@ -65,7 +65,6 @@ struct  false_ALARM_STATISTICS {
 	u32 Cnt_Crc8_fail;
 	u32 Cnt_Mcs_fail;
 	u32 Cnt_Ofdm_fail;
-	u32 Cnt_Ofdm_fail_pre; /* For RTL8881A */
 	u32 Cnt_Cck_fail;
 	u32 Cnt_all;
 	u32 Cnt_Fast_Fsync;
diff --git a/drivers/staging/rtl8723bs/hal/odm_DynamicTxPower.h b/drivers/staging/rtl8723bs/hal/odm_DynamicTxPower.h
index e2d244324ebe9a..4431a3ddf17593 100644
--- a/drivers/staging/rtl8723bs/hal/odm_DynamicTxPower.h
+++ b/drivers/staging/rtl8723bs/hal/odm_DynamicTxPower.h
@@ -11,7 +11,6 @@
 #define		TX_POWER_NEAR_FIELD_THRESH_LVL2	74
 #define		TX_POWER_NEAR_FIELD_THRESH_LVL1	67
 #define		TX_POWER_NEAR_FIELD_THRESH_AP		0x3F
-#define		TX_POWER_NEAR_FIELD_THRESH_8812	60
 
 #define		TxHighPwrLevel_Normal		0
 #define		TxHighPwrLevel_Level1		1
diff --git a/drivers/staging/rtl8723bs/hal/odm_precomp.h b/drivers/staging/rtl8723bs/hal/odm_precomp.h
index 2987857a8761fa..d4390fd203f135 100644
--- a/drivers/staging/rtl8723bs/hal/odm_precomp.h
+++ b/drivers/staging/rtl8723bs/hal/odm_precomp.h
@@ -19,7 +19,6 @@
 	/* include <drv_types.h> */
 	/* include <rtw_byteorder.h> */
 	/* include <hal_intf.h> */
-#define BEAMFORMING_SUPPORT 0
 
 /* 2 Hardware Parameter Files */
 
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
index 6a4a13544ca111..1389e5598d3ee5 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
@@ -868,8 +868,6 @@ void _InitBurstPktLen_8723BS(struct adapter *Adapter)
 	rtw_write8(Adapter, REG_MAX_AGGR_NUM_8723B, 0x1F);
 	rtw_write8(Adapter, REG_PIFS_8723B, 0x00);
 	rtw_write8(Adapter, REG_FWHW_TXQ_CTRL_8723B, rtw_read8(Adapter, REG_FWHW_TXQ_CTRL)&(~BIT(7)));
-	if (pHalData->AMPDUBurstMode)
-		rtw_write8(Adapter, REG_AMPDU_BURST_MODE_8723B,  0x5F);
 	rtw_write8(Adapter, REG_AMPDU_MAX_TIME_8723B, 0x70);
 
 	/*  ARFB table 9 for 11ac 5G 2SS */
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c b/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
index fac1270853de6a..e4f047b1482791 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
@@ -7,7 +7,7 @@
 /******************************************************************************
  *
  *
- * Module:	rtl8192c_rf6052.c	(Source C File)
+ * Module:	rtl8723b_rf6052.c	(Source C File)
  *
  * Note:	Provide RF 6052 series relative API.
  *
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
index e943c304950e7c..706d58741dcede 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
@@ -7,6 +7,7 @@
 
 #include <drv_types.h>
 #include <rtl8723b_hal.h>
+#include <linux/align.h>
 
 static void initrecvbuf(struct recv_buf *precvbuf, struct adapter *padapter)
 {
@@ -386,7 +387,7 @@ s32 rtl8723bs_init_recv_priv(struct adapter *padapter)
 		goto exit;
 	}
 
-	precvpriv->precv_buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(precvpriv->pallocated_recv_buf), 4);
+	precvpriv->precv_buf = PTR_ALIGN(precvpriv->pallocated_recv_buf, 4);
 
 	/*  init each recv buffer */
 	precvbuf = (struct recv_buf *)precvpriv->precv_buf;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
index 7f55448d544e83..0441e84ea87dac 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
@@ -525,9 +525,6 @@ s32	rtl8723bs_hal_xmitframe_enqueue(
 s32 rtl8723bs_init_xmit_priv(struct adapter *padapter)
 {
 	struct xmit_priv *xmitpriv = &padapter->xmitpriv;
-	struct hal_com_data *phal;
-
-	phal = GET_HAL_DATA(padapter);
 
 	init_completion(&xmitpriv->SdioXmitStart);
 	init_completion(&xmitpriv->SdioXmitTerminate);
diff --git a/drivers/staging/rtl8723bs/hal/sdio_halinit.c b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
index 57d2925642e786..001b4b32c02e09 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_halinit.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
@@ -538,26 +538,6 @@ static void _RfPowerSave(struct adapter *padapter)
 /* YJ, TODO */
 }
 
-/*  */
-/*  2010/08/09 MH Add for power down check. */
-/*  */
-static bool HalDetectPwrDownMode(struct adapter *Adapter)
-{
-	u8 tmpvalue;
-	struct hal_com_data *pHalData = GET_HAL_DATA(Adapter);
-	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(Adapter);
-
-	rtw_efuse_shadow_read(Adapter, 1, 0x7B/*EEPROM_RF_OPT3_92C*/, (u32 *)&tmpvalue);
-
-	/*  2010/08/25 MH INF priority > PDN Efuse value. */
-	if (tmpvalue & BIT(4) && pwrctrlpriv->reg_pdnmode)
-		pHalData->pwrdown = true;
-	else
-		pHalData->pwrdown = false;
-
-	return pHalData->pwrdown;
-}	/*  HalDetectPwrDownMode */
-
 u32 rtl8723bs_hal_init(struct adapter *padapter)
 {
 	s32 ret;
@@ -632,10 +612,6 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 	if (pwrctrlpriv->reg_rfoff)
 		pwrctrlpriv->rf_pwrstate = rf_off;
 
-	/*  2010/08/09 MH We need to check if we need to turnon or off RF after detecting */
-	/*  HW GPIO pin. Before PHY_RFConfig8192C. */
-	HalDetectPwrDownMode(padapter);
-
 	/*  Save target channel */
 	/*  <Roger_Notes> Current Channel will be updated again later. */
 	pHalData->CurrentChannel = 6;
diff --git a/drivers/staging/rtl8723bs/include/basic_types.h b/drivers/staging/rtl8723bs/include/basic_types.h
index b6b1afaa9ab843..aaa1e60f429518 100644
--- a/drivers/staging/rtl8723bs/include/basic_types.h
+++ b/drivers/staging/rtl8723bs/include/basic_types.h
@@ -14,7 +14,6 @@
 #include <linux/stddef.h>
 
 #define SIZE_PTR __kernel_size_t
-#define SSIZE_PTR __kernel_ssize_t
 
 /* port from fw by thomas */
 /*  TODO: Belows are Sync from SD7-Driver. It is necessary to check correctness */
@@ -163,9 +162,6 @@
 		); \
 }
 
-/*  Get the N-bytes alignent offset from the current length */
-#define N_BYTE_ALIGMENT(__Value, __Aligment) ((__Aligment == 1) ? (__Value) : (((__Value + __Aligment - 1) / __Aligment) * __Aligment))
-
 #define TEST_FLAG(__Flag, __testFlag)		(((__Flag) & (__testFlag)) != 0)
 #define SET_FLAG(__Flag, __setFlag)			((__Flag) |= __setFlag)
 #define CLEAR_FLAG(__Flag, __clearFlag)		((__Flag) &= ~(__clearFlag))
diff --git a/drivers/staging/rtl8723bs/include/cmd_osdep.h b/drivers/staging/rtl8723bs/include/cmd_osdep.h
index 91d933d71945de..a7b526a658a97e 100644
--- a/drivers/staging/rtl8723bs/include/cmd_osdep.h
+++ b/drivers/staging/rtl8723bs/include/cmd_osdep.h
@@ -9,8 +9,8 @@
 
 int rtw_init_cmd_priv(struct	cmd_priv *pcmdpriv);
 int rtw_init_evt_priv(struct evt_priv *pevtpriv);
-extern void _rtw_free_evt_priv(struct	evt_priv *pevtpriv);
-extern void _rtw_free_cmd_priv(struct	cmd_priv *pcmdpriv);
-extern struct	cmd_obj	*_rtw_dequeue_cmd(struct __queue *queue);
+void _rtw_free_evt_priv(struct	evt_priv *pevtpriv);
+void _rtw_free_cmd_priv(struct	cmd_priv *pcmdpriv);
+struct	cmd_obj	*_rtw_dequeue_cmd(struct __queue *queue);
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/hal_com.h b/drivers/staging/rtl8723bs/include/hal_com.h
index 7bbf62cfd82579..17bce0f88bac15 100644
--- a/drivers/staging/rtl8723bs/include/hal_com.h
+++ b/drivers/staging/rtl8723bs/include/hal_com.h
@@ -112,7 +112,7 @@ u8 HwRateToMRate(u8 rate);
 
 void HalSetBrateCfg(
 	struct adapter *Adapter,
-	u8 *mBratesOS,
+	u8 *basic_rates,
 	u16	*pBrateCfg);
 
 bool
diff --git a/drivers/staging/rtl8723bs/include/hal_data.h b/drivers/staging/rtl8723bs/include/hal_data.h
index f67ca496902a56..614fd885451b1a 100644
--- a/drivers/staging/rtl8723bs/include/hal_data.h
+++ b/drivers/staging/rtl8723bs/include/hal_data.h
@@ -12,17 +12,6 @@
 
 #include <hal_sdio.h>
 
-enum rt_ampdu_burst {
-	RT_AMPDU_BURST_NONE	= 0,
-	RT_AMPDU_BURST_92D	= 1,
-	RT_AMPDU_BURST_88E	= 2,
-	RT_AMPDU_BURST_8812_4	= 3,
-	RT_AMPDU_BURST_8812_8	= 4,
-	RT_AMPDU_BURST_8812_12	= 5,
-	RT_AMPDU_BURST_8812_15	= 6,
-	RT_AMPDU_BURST_8723B	= 7,
-};
-
 #define CHANNEL_MAX_NUMBER		(14)	/*  14 is the max channel number */
 #define CHANNEL_MAX_NUMBER_2G		14
 #define MAX_PG_GROUP			13
@@ -234,17 +223,12 @@ struct hal_com_data {
 
 	u8 u1ForcedIgiLb;			/*  forced IGI lower bound */
 
-	/*  2010/08/09 MH Add CU power down mode. */
-	bool		pwrdown;
-
 	u8 OutEpQueueSel;
 	u8 OutEpNumber;
 
 	/*  Auto FSM to Turn On, include clock, isolation, power control for MAC only */
 	u8 bMacPwrCtrlOn;
 
-	enum rt_ampdu_burst	AMPDUBurstMode; /* 92C maybe not use, but for compile successfully */
-
 	u32 		sdio_himr;
 	u32 		sdio_hisr;
 
diff --git a/drivers/staging/rtl8723bs/include/hal_intf.h b/drivers/staging/rtl8723bs/include/hal_intf.h
index a7ae8c006f0c21..7b63116525cab4 100644
--- a/drivers/staging/rtl8723bs/include/hal_intf.h
+++ b/drivers/staging/rtl8723bs/include/hal_intf.h
@@ -191,7 +191,7 @@ u8 rtw_hal_check_ips_status(struct adapter *padapter);
 
 int rtw_hal_xmitframe_enqueue(struct adapter *padapter, struct xmit_frame *pxmitframe);
 s32	rtw_hal_xmit(struct adapter *padapter, struct xmit_frame *pxmitframe);
-s32	rtw_hal_mgnt_xmit(struct adapter *padapter, struct xmit_frame *pmgntframe);
+int rtw_hal_mgnt_xmit(struct adapter *padapter, struct xmit_frame *pmgntframe);
 
 s32	rtw_hal_init_xmit_priv(struct adapter *padapter);
 void rtw_hal_free_xmit_priv(struct adapter *padapter);
diff --git a/drivers/staging/rtl8723bs/include/hal_pg.h b/drivers/staging/rtl8723bs/include/hal_pg.h
index 32e66fe451b0ca..6f8c8307eaa789 100644
--- a/drivers/staging/rtl8723bs/include/hal_pg.h
+++ b/drivers/staging/rtl8723bs/include/hal_pg.h
@@ -38,7 +38,7 @@
 
 #define	EEPROM_RF_BOARD_OPTION_8723B		0xC1
 #define	EEPROM_RF_BT_SETTING_8723B		0xC3
-#define EEPROM_DEFAULT_DIFF			0XFE
+#define EEPROM_DEFAULT_DIFF			-2
 
 /* RTL8723BS */
 #define	EEPROM_MAC_ADDR_8723BS			0x11A
diff --git a/drivers/staging/rtl8723bs/include/hal_phy.h b/drivers/staging/rtl8723bs/include/hal_phy.h
index abc0f27fdaa466..ec9bc47816db05 100644
--- a/drivers/staging/rtl8723bs/include/hal_phy.h
+++ b/drivers/staging/rtl8723bs/include/hal_phy.h
@@ -17,16 +17,6 @@
 
 
 /*--------------------------Define Parameters-------------------------------*/
-enum {
-	RF_TYPE_MIN = 0,	/*  0 */
-	RF_8225 = 1,		/*  1 11b/g RF for verification only */
-	RF_8256 = 2,		/*  2 11b/g/n */
-	RF_8258 = 3,		/*  3 11a/b/g/n RF */
-	RF_6052 = 4,		/*  4 11b/g/n RF */
-	RF_PSEUDO_11N = 5,	/*  5, It is a temporality RF. */
-	RF_TYPE_MAX
-};
-
 enum rf_path {
 	RF_PATH_A = 0,
 	RF_PATH_B,
@@ -34,12 +24,6 @@ enum rf_path {
 };
 
 #define	TX_1S			0
-#define	TX_2S			1
-#define	TX_3S			2
-#define	TX_4S			3
-
-#define	RF_PATH_MAX_92C_88E		2
-#define	RF_PATH_MAX_90_8812		4	/* Max RF number 90 support */
 
 enum wireless_mode {
 	WIRELESS_MODE_UNKNOWN = 0x00,
diff --git a/drivers/staging/rtl8723bs/include/osdep_service.h b/drivers/staging/rtl8723bs/include/osdep_service.h
index 2f5011a8210cd9..e2eebd46cb5c76 100644
--- a/drivers/staging/rtl8723bs/include/osdep_service.h
+++ b/drivers/staging/rtl8723bs/include/osdep_service.h
@@ -15,7 +15,7 @@
 #include <osdep_service_linux.h>
 
 
-extern int RTW_STATUS_CODE(int error_code);
+int RTW_STATUS_CODE(int error_code);
 
 int _rtw_netif_rx(struct net_device *ndev, struct sk_buff *skb);
 
@@ -29,7 +29,7 @@ static inline void flush_signals_thread(void)
 
 #define _RND(sz, r) ((((sz)+((r)-1))/(r))*(r))
 
-extern void rtw_free_netdev(struct net_device *netdev);
+void rtw_free_netdev(struct net_device *netdev);
 
 /* Macros for handling unaligned memory accesses */
 
diff --git a/drivers/staging/rtl8723bs/include/osdep_service_linux.h b/drivers/staging/rtl8723bs/include/osdep_service_linux.h
index c8274da940ff44..cc5eaf6a2c61b0 100644
--- a/drivers/staging/rtl8723bs/include/osdep_service_linux.h
+++ b/drivers/staging/rtl8723bs/include/osdep_service_linux.h
@@ -113,6 +113,6 @@ static inline struct adapter *rtw_netdev_priv(struct net_device *netdev)
 }
 
 struct net_device *rtw_alloc_etherdev_with_old_priv(int sizeof_priv, void *old_priv);
-extern struct net_device *rtw_alloc_etherdev(int sizeof_priv);
+struct net_device *rtw_alloc_etherdev(int sizeof_priv);
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtl8723b_hal.h b/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
index c442f01aa58532..497b1cbc54c82a 100644
--- a/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
+++ b/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
@@ -193,7 +193,7 @@ enum { /* tag_Package_Definition */
 	PACKAGE_TFBGA79
 };
 
-/*  rtl8723a_hal_init.c */
+/*  rtl8723b_hal_init.c */
 s32 rtl8723b_FirmwareDownload(struct adapter *padapter, bool  bUsedWoWLANFw);
 void rtl8723b_FirmwareSelfReset(struct adapter *padapter);
 void rtl8723b_InitializeFirmwareVars(struct adapter *padapter);
diff --git a/drivers/staging/rtl8723bs/include/rtw_cmd.h b/drivers/staging/rtl8723bs/include/rtw_cmd.h
index 2e791da7e81534..58a5ec39ce7640 100644
--- a/drivers/staging/rtl8723bs/include/rtw_cmd.h
+++ b/drivers/staging/rtl8723bs/include/rtw_cmd.h
@@ -107,16 +107,16 @@ struct c2h_evt_hdr_88xx {
 #define c2h_evt_valid(c2h_evt) ((c2h_evt)->id || (c2h_evt)->plen)
 
 int rtw_enqueue_cmd(struct cmd_priv *pcmdpriv, struct cmd_obj *obj);
-extern struct cmd_obj *rtw_dequeue_cmd(struct cmd_priv *pcmdpriv);
-extern void rtw_free_cmd_obj(struct cmd_obj *pcmd);
+struct cmd_obj *rtw_dequeue_cmd(struct cmd_priv *pcmdpriv);
+void rtw_free_cmd_obj(struct cmd_obj *pcmd);
 
 void rtw_stop_cmd_thread(struct adapter *adapter);
 int rtw_cmd_thread(void *context);
 
-extern void rtw_free_cmd_priv(struct cmd_priv *pcmdpriv);
+void rtw_free_cmd_priv(struct cmd_priv *pcmdpriv);
 
-extern void rtw_free_evt_priv(struct evt_priv *pevtpriv);
-extern void rtw_evt_notify_isr(struct evt_priv *pevtpriv);
+void rtw_free_evt_priv(struct evt_priv *pevtpriv);
+void rtw_evt_notify_isr(struct evt_priv *pevtpriv);
 
 enum {
 	NONE_WK_CID,
@@ -573,50 +573,50 @@ struct RunInThread_param {
 #define H2C_RESERVED			0x07
 
 u8 rtw_sitesurvey_cmd(struct adapter  *padapter, struct ndis_802_11_ssid *ssid, int ssid_num, struct rtw_ieee80211_channel *ch, int ch_num);
-extern u8 rtw_createbss_cmd(struct adapter  *padapter);
+u8 rtw_createbss_cmd(struct adapter  *padapter);
 int rtw_startbss_cmd(struct adapter  *padapter, int flags);
 
 struct sta_info;
-extern u8 rtw_setstakey_cmd(struct adapter  *padapter, struct sta_info *sta, u8 unicast_key, bool enqueue);
-extern u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 enqueue);
+u8 rtw_setstakey_cmd(struct adapter  *padapter, struct sta_info *sta, u8 unicast_key, bool enqueue);
+u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 enqueue);
 
-extern u8 rtw_joinbss_cmd(struct adapter *padapter, struct wlan_network *pnetwork);
+u8 rtw_joinbss_cmd(struct adapter *padapter, struct wlan_network *pnetwork);
 u8 rtw_disassoc_cmd(struct adapter *padapter, u32 deauth_timeout_ms, bool enqueue);
-extern u8 rtw_setopmode_cmd(struct adapter  *padapter, enum nl80211_iftype networktype, bool enqueue);
-extern u8 rtw_setrfintfs_cmd(struct adapter  *padapter, u8 mode);
+u8 rtw_setopmode_cmd(struct adapter  *padapter, enum nl80211_iftype networktype, bool enqueue);
+u8 rtw_setrfintfs_cmd(struct adapter  *padapter, u8 mode);
 
-extern u8 rtw_gettssi_cmd(struct adapter  *padapter, u8 offset, u8 *pval);
-extern u8 rtw_setfwdig_cmd(struct adapter *padapter, u8 type);
-extern u8 rtw_setfwra_cmd(struct adapter *padapter, u8 type);
+u8 rtw_gettssi_cmd(struct adapter  *padapter, u8 offset, u8 *pval);
+u8 rtw_setfwdig_cmd(struct adapter *padapter, u8 type);
+u8 rtw_setfwra_cmd(struct adapter *padapter, u8 type);
 
-extern u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr);
-extern u8 rtw_reset_securitypriv_cmd(struct adapter *padapter);
-extern u8 rtw_free_assoc_resources_cmd(struct adapter *padapter);
-extern u8 rtw_dynamic_chk_wk_cmd(struct adapter *adapter);
+u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr);
+u8 rtw_reset_securitypriv_cmd(struct adapter *padapter);
+u8 rtw_free_assoc_resources_cmd(struct adapter *padapter);
+u8 rtw_dynamic_chk_wk_cmd(struct adapter *adapter);
 
 u8 rtw_lps_ctrl_wk_cmd(struct adapter *padapter, u8 lps_ctrl_type, u8 enqueue);
 u8 rtw_dm_in_lps_wk_cmd(struct adapter *padapter);
 
 u8 rtw_dm_ra_mask_wk_cmd(struct adapter *padapter, u8 *psta);
 
-extern u8 rtw_ps_cmd(struct adapter *padapter);
+u8 rtw_ps_cmd(struct adapter *padapter);
 
 u8 rtw_chk_hi_queue_cmd(struct adapter *padapter);
 
-extern u8 rtw_c2h_packet_wk_cmd(struct adapter *padapter, u8 *pbuf, u16 length);
-extern u8 rtw_c2h_wk_cmd(struct adapter *padapter, u8 *c2h_evt);
+u8 rtw_c2h_packet_wk_cmd(struct adapter *padapter, u8 *pbuf, u16 length);
+u8 rtw_c2h_wk_cmd(struct adapter *padapter, u8 *c2h_evt);
 
 u8 rtw_drvextra_cmd_hdl(struct adapter *padapter, unsigned char *pbuf);
 
-extern void rtw_survey_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
-extern void rtw_disassoc_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
-extern void rtw_joinbss_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
-extern void rtw_createbss_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
-extern void rtw_getbbrfreg_cmdrsp_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
+void rtw_survey_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
+void rtw_disassoc_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
+void rtw_joinbss_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
+void rtw_createbss_cmd_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
+void rtw_getbbrfreg_cmdrsp_callback(struct adapter  *padapter, struct cmd_obj *pcmd);
 
-extern void rtw_setstaKey_cmdrsp_callback(struct adapter  *padapter,  struct cmd_obj *pcmd);
-extern void rtw_setassocsta_cmdrsp_callback(struct adapter  *padapter,  struct cmd_obj *pcmd);
-extern void rtw_getrttbl_cmdrsp_callback(struct adapter  *padapter,  struct cmd_obj *pcmd);
+void rtw_setstaKey_cmdrsp_callback(struct adapter  *padapter,  struct cmd_obj *pcmd);
+void rtw_setassocsta_cmdrsp_callback(struct adapter  *padapter,  struct cmd_obj *pcmd);
+void rtw_getrttbl_cmdrsp_callback(struct adapter  *padapter,  struct cmd_obj *pcmd);
 
 
 struct _cmd_callback {
diff --git a/drivers/staging/rtl8723bs/include/rtw_efuse.h b/drivers/staging/rtl8723bs/include/rtw_efuse.h
index 808ba94a599879..1e5d61f4587b05 100644
--- a/drivers/staging/rtl8723bs/include/rtw_efuse.h
+++ b/drivers/staging/rtl8723bs/include/rtw_efuse.h
@@ -71,8 +71,7 @@ struct efuse_hal {
 u8 rtw_efuse_calculate_word_counts(u8 word_en);
 u8 rtw_efuse_one_byte_read(struct adapter *padapter, u16 addr, u8 *data);
 
-u8 rtw_efuse_read_1_byte(struct adapter *padapter, u16 Address);
-void rtw_efuse_shadow_map_update(struct adapter *padapter, u8 efuseType);
-void rtw_efuse_shadow_read(struct adapter *padapter, u8 Type, u16 Offset, u32 *Value);
+u8 rtw_efuse_read_1_byte(struct adapter *padapter, u16 address);
+void rtw_efuse_shadow_map_update(struct adapter *padapter, u8 efuse_type);
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtw_io.h b/drivers/staging/rtl8723bs/include/rtw_io.h
index 3c99a2bc19bd0c..55bedd91299399 100644
--- a/drivers/staging/rtl8723bs/include/rtw_io.h
+++ b/drivers/staging/rtl8723bs/include/rtw_io.h
@@ -43,15 +43,15 @@ struct io_priv {
 
 };
 
-extern u8 rtw_read8(struct adapter *adapter, u32 addr);
-extern u16 rtw_read16(struct adapter *adapter, u32 addr);
-extern u32 rtw_read32(struct adapter *adapter, u32 addr);
+u8 rtw_read8(struct adapter *adapter, u32 addr);
+u16 rtw_read16(struct adapter *adapter, u32 addr);
+u32 rtw_read32(struct adapter *adapter, u32 addr);
 
-extern int rtw_write8(struct adapter *adapter, u32 addr, u8 val);
-extern int rtw_write16(struct adapter *adapter, u32 addr, u16 val);
-extern int rtw_write32(struct adapter *adapter, u32 addr, u32 val);
+int rtw_write8(struct adapter *adapter, u32 addr, u8 val);
+int rtw_write16(struct adapter *adapter, u32 addr, u16 val);
+int rtw_write32(struct adapter *adapter, u32 addr, u32 val);
 
-extern u32 rtw_write_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *pmem);
+u32 rtw_write_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *pmem);
 
 int rtw_init_io_priv(struct adapter *padapter, void (*set_intf_ops)(struct adapter *padapter, struct _io_ops *pops));
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h b/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
index 768e4868ecfd0d..020b143f36be0e 100644
--- a/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
+++ b/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
@@ -7,8 +7,6 @@
 #ifndef __RTW_IOCTL_SET_H_
 #define __RTW_IOCTL_SET_H_
 
-typedef u8 NDIS_802_11_PMKID_VALUE[16];
-
 u8 rtw_set_802_11_authentication_mode(struct adapter *pdapter, enum ndis_802_11_authentication_mode authmode);
 u8 rtw_set_802_11_add_wep(struct adapter *padapter, struct ndis_802_11_wep *wep);
 u8 rtw_set_802_11_disassociate(struct adapter *padapter);
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme.h b/drivers/staging/rtl8723bs/include/rtw_mlme.h
index 6faa13535b62ec..7709d5fd77a3de 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme.h
@@ -249,32 +249,32 @@ struct hostapd_priv {
 	struct adapter *padapter;
 };
 
-extern int hostapd_mode_init(struct adapter *padapter);
-extern void hostapd_mode_unload(struct adapter *padapter);
+int hostapd_mode_init(struct adapter *padapter);
+void hostapd_mode_unload(struct adapter *padapter);
 
-extern void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf);
-extern void rtw_survey_event_callback(struct adapter *adapter, u8 *pbuf);
-extern void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf);
-extern void rtw_joinbss_event_callback(struct adapter *adapter, u8 *pbuf);
-extern void rtw_stassoc_event_callback(struct adapter *adapter, u8 *pbuf);
-extern void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf);
-extern void rtw_cpwm_event_callback(struct adapter *adapter, u8 *pbuf);
-extern void rtw_wmm_event_callback(struct adapter *padapter, u8 *pbuf);
+void rtw_joinbss_event_prehandle(struct adapter *adapter, u8 *pbuf);
+void rtw_survey_event_callback(struct adapter *adapter, u8 *pbuf);
+void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf);
+void rtw_joinbss_event_callback(struct adapter *adapter, u8 *pbuf);
+void rtw_stassoc_event_callback(struct adapter *adapter, u8 *pbuf);
+void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf);
+void rtw_cpwm_event_callback(struct adapter *adapter, u8 *pbuf);
+void rtw_wmm_event_callback(struct adapter *padapter, u8 *pbuf);
 
-extern void rtw_join_timeout_handler(struct timer_list *t);
-extern void _rtw_scan_timeout_handler(struct timer_list *t);
+void rtw_join_timeout_handler(struct timer_list *t);
+void _rtw_scan_timeout_handler(struct timer_list *t);
 
 int event_thread(void *context);
 
-extern void rtw_free_network_queue(struct adapter *adapter, u8 isfreeall);
-extern int rtw_init_mlme_priv(struct adapter *adapter);/*  (struct mlme_priv *pmlmepriv); */
+void rtw_free_network_queue(struct adapter *adapter, u8 isfreeall);
+int rtw_init_mlme_priv(struct adapter *adapter);/*  (struct mlme_priv *pmlmepriv); */
 
-extern void rtw_free_mlme_priv(struct mlme_priv *pmlmepriv);
+void rtw_free_mlme_priv(struct mlme_priv *pmlmepriv);
 
 
-extern signed int rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv);
-extern signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecuritypriv, signed int keyid, u8 set_tx, bool enqueue);
-extern signed int rtw_set_auth(struct adapter *adapter, struct security_priv *psecuritypriv);
+signed int rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv);
+signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecuritypriv, signed int keyid, u8 set_tx, bool enqueue);
+signed int rtw_set_auth(struct adapter *adapter, struct security_priv *psecuritypriv);
 
 static inline u8 *get_bssid(struct mlme_priv *pmlmepriv)
 {	/* if sta_mode:pmlmepriv->cur_network.network.mac_address => bssid */
@@ -318,48 +318,48 @@ static inline void _clr_fwstate_(struct mlme_priv *pmlmepriv, signed int state)
 		pmlmepriv->bScanInProcess = false;
 }
 
-extern u16 rtw_get_capability(struct wlan_bssid_ex *bss);
-extern void rtw_update_scanned_network(struct adapter *adapter, struct wlan_bssid_ex *target);
-extern void rtw_disconnect_hdl_under_linked(struct adapter *adapter, struct sta_info *psta, u8 free_assoc);
-extern void rtw_generate_random_ibss(u8 *pibss);
-extern struct wlan_network *rtw_find_network(struct __queue *scanned_queue, u8 *addr);
-extern struct wlan_network *rtw_get_oldest_wlan_network(struct __queue *scanned_queue);
+u16 rtw_get_capability(struct wlan_bssid_ex *bss);
+void rtw_update_scanned_network(struct adapter *adapter, struct wlan_bssid_ex *target);
+void rtw_disconnect_hdl_under_linked(struct adapter *adapter, struct sta_info *psta, u8 free_assoc);
+void rtw_generate_random_ibss(u8 *pibss);
+struct wlan_network *rtw_find_network(struct __queue *scanned_queue, u8 *addr);
+struct wlan_network *rtw_get_oldest_wlan_network(struct __queue *scanned_queue);
 struct wlan_network *_rtw_find_same_network(struct __queue *scanned_queue, struct wlan_network *network);
 
-extern void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue);
-extern void rtw_indicate_disconnect(struct adapter *adapter);
-extern void rtw_indicate_connect(struct adapter *adapter);
+void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue);
+void rtw_indicate_disconnect(struct adapter *adapter);
+void rtw_indicate_connect(struct adapter *adapter);
 void rtw_indicate_scan_done(struct adapter *padapter, bool aborted);
 void rtw_scan_abort(struct adapter *adapter);
 
-extern int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_len);
-extern int rtw_restruct_wmm_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_len, uint initial_out_len);
-extern void rtw_init_registrypriv_dev_network(struct adapter *adapter);
+int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_len);
+int rtw_restruct_wmm_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_len, uint initial_out_len);
+void rtw_init_registrypriv_dev_network(struct adapter *adapter);
 
-extern void rtw_update_registrypriv_dev_network(struct adapter *adapter);
+void rtw_update_registrypriv_dev_network(struct adapter *adapter);
 
-extern void _rtw_join_timeout_handler(struct timer_list *t);
-extern void rtw_scan_timeout_handler(struct timer_list *t);
+void _rtw_join_timeout_handler(struct timer_list *t);
+void rtw_scan_timeout_handler(struct timer_list *t);
 
-extern void rtw_dynamic_check_timer_handler(struct adapter *adapter);
+void rtw_dynamic_check_timer_handler(struct adapter *adapter);
 bool rtw_is_scan_deny(struct adapter *adapter);
 void rtw_clear_scan_deny(struct adapter *adapter);
 void rtw_set_scan_deny(struct adapter *adapter, u32 ms);
 
 void rtw_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv);
 
-extern void _rtw_free_mlme_priv(struct mlme_priv *pmlmepriv);
+void _rtw_free_mlme_priv(struct mlme_priv *pmlmepriv);
 
 /* extern struct wlan_network* _rtw_dequeue_network(struct __queue *queue); */
 
-extern struct wlan_network *rtw_alloc_network(struct mlme_priv *pmlmepriv);
+struct wlan_network *rtw_alloc_network(struct mlme_priv *pmlmepriv);
 
 
-extern void _rtw_free_network(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork, u8 isfreeall);
-extern void _rtw_free_network_nolock(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork);
+void _rtw_free_network(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork, u8 isfreeall);
+void _rtw_free_network_nolock(struct mlme_priv *pmlmepriv, struct wlan_network *pnetwork);
 
 
-extern struct wlan_network *_rtw_find_network(struct __queue *scanned_queue, u8 *addr);
+struct wlan_network *_rtw_find_network(struct __queue *scanned_queue, u8 *addr);
 
 bool rtw_if_up(struct adapter *padapter);
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
index 179fae4d8fcc31..d486cd462509e9 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
@@ -207,22 +207,22 @@ enum {
 };
 
 struct mlme_handler {
-	unsigned int   num;
+	unsigned int num;
 	char *str;
 	unsigned int (*func)(struct adapter *padapter, union recv_frame *precv_frame);
 };
 
 struct action_handler {
-	unsigned int   num;
+	unsigned int num;
 	char *str;
 	unsigned int (*func)(struct adapter *padapter, union recv_frame *precv_frame);
 };
 
 struct	ss_res {
-	int	state;
-	int	bss_cnt;
-	int	channel_idx;
-	int	scan_mode;
+	int state;
+	int bss_cnt;
+	int channel_idx;
+	int scan_mode;
 	u8 ssid_num;
 	u8 ch_num;
 	struct ndis_802_11_ssid ssid[RTW_SSID_SCAN_AMOUNT];
@@ -253,7 +253,7 @@ struct FW_Sta_Info {
 	u32 status;
 	u32 rx_pkt;
 	u32 retry;
-	NDIS_802_11_RATES_EX  SupportedRates;
+	u8  SupportedRates[NDIS_802_11_LENGTH_RATES_EX];
 };
 
 /*
@@ -422,7 +422,7 @@ void init_mlme_default_rate_set(struct adapter *padapter);
 void init_mlme_ext_priv(struct adapter *padapter);
 void init_hw_mlme_ext(struct adapter *padapter);
 void free_mlme_ext_priv(struct mlme_ext_priv *pmlmeext);
-extern struct xmit_frame *alloc_mgtxmitframe(struct xmit_priv *pxmitpriv);
+struct xmit_frame *alloc_mgtxmitframe(struct xmit_priv *pxmitpriv);
 
 /* void fill_fwpriv(struct adapter *padapter, struct fw_priv *pfwpriv); */
 
@@ -430,7 +430,7 @@ u8 networktype_to_raid_ex(struct adapter *adapter, struct sta_info *psta);
 
 void get_rate_set(struct adapter *padapter, unsigned char *pbssrate, int *bssrate_len);
 void set_mcs_rate_by_mask(u8 *mcs_set, u32 mask);
-void update_basic_rate_table(struct adapter *padapter, u8 *mBratesOS);
+void update_basic_rate_table(struct adapter *padapter, u8 *basic_rates);
 void update_basic_rate_table_soft_ap(u8 *bssrateset, u32 bssratelen);
 
 void save_dm_func_flag(struct adapter *padapter);
@@ -516,8 +516,8 @@ s16 rtw_camid_search(struct adapter *adapter, u8 *addr, s16 kid);
 s16 rtw_camid_alloc(struct adapter *adapter, struct sta_info *sta, u8 kid);
 void rtw_camid_free(struct adapter *adapter, u8 cam_id);
 
-extern void rtw_alloc_macid(struct adapter *padapter, struct sta_info *psta);
-extern void rtw_release_macid(struct adapter *padapter, struct sta_info *psta);
+void rtw_alloc_macid(struct adapter *padapter, struct sta_info *psta);
+void rtw_release_macid(struct adapter *padapter, struct sta_info *psta);
 
 void report_join_res(struct adapter *padapter, int res);
 void report_survey_event(struct adapter *padapter, union recv_frame *precv_frame);
@@ -527,13 +527,13 @@ void report_add_sta_event(struct adapter *padapter, unsigned char *MacAddr, int
 void report_wmm_edca_update(struct adapter *padapter);
 
 u8 chk_bmc_sleepq_cmd(struct adapter *padapter);
-extern u8 set_tx_beacon_cmd(struct adapter *padapter);
+u8 set_tx_beacon_cmd(struct adapter *padapter);
 unsigned int setup_beacon_frame(struct adapter *padapter, unsigned char *beacon_frame);
 void update_mgnt_tx_rate(struct adapter *padapter, u8 rate);
 void update_mgntframe_attrib(struct adapter *padapter, struct pkt_attrib *pattrib);
 void update_mgntframe_attrib_addr(struct adapter *padapter, struct xmit_frame *pmgntframe);
 void dump_mgntframe(struct adapter *padapter, struct xmit_frame *pmgntframe);
-s32 dump_mgntframe_and_wait(struct adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms);
+void dump_mgntframe_and_wait(struct adapter *padapter, struct xmit_frame *pmgntframe, int timeout_ms);
 s32 dump_mgntframe_and_wait_ack(struct adapter *padapter, struct xmit_frame *pmgntframe);
 
 void issue_beacon(struct adapter *padapter, int timeout_ms);
@@ -541,8 +541,6 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 void issue_assocreq(struct adapter *padapter);
 void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_info *pstat, int pkt_type);
 void issue_auth(struct adapter *padapter, struct sta_info *psta, unsigned short status);
-void issue_probereq(struct adapter *padapter, struct ndis_802_11_ssid *pssid, u8 *da);
-s32 issue_probereq_ex(struct adapter *padapter, struct ndis_802_11_ssid *pssid, u8 *da, u8 ch, bool append_wps, int try_cnt, int wait_ms);
 int issue_nulldata(struct adapter *padapter, unsigned char *da, unsigned int power_mode, int try_cnt, int wait_ms);
 s32 issue_nulldata_in_interrupt(struct adapter *padapter, u8 *da);
 int issue_qos_nulldata(struct adapter *padapter, unsigned char *da, u16 tid, int try_cnt, int wait_ms);
@@ -606,12 +604,12 @@ void sa_query_timer_hdl(struct timer_list *t);
 		_set_timer(&(mlmeext)->sa_query_timer, (ms)); \
 	} while (0)
 
-extern void process_addba_req(struct adapter *padapter, u8 *paddba_req, u8 *addr);
+void process_addba_req(struct adapter *padapter, u8 *paddba_req, u8 *addr);
 
-extern void update_TSF(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len);
-extern void correct_TSF(struct adapter *padapter, struct mlme_ext_priv *pmlmeext);
-extern void adaptive_early_32k(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len);
-extern bool traffic_status_watchdog(struct adapter *padapter, bool from_timer);
+void update_TSF(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len);
+void correct_TSF(struct adapter *padapter, struct mlme_ext_priv *pmlmeext);
+void adaptive_early_32k(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len);
+bool traffic_status_watchdog(struct adapter *padapter, bool from_timer);
 
 int rtw_chk_start_clnt_join(struct adapter *padapter, u8 *ch, u8 *bw, u8 *offset);
 
diff --git a/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h b/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
index 182b8c88ccc834..49f19b6ee329fc 100644
--- a/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
+++ b/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
@@ -212,20 +212,20 @@ struct pwrctrl_priv {
 		_set_timer(&(pwrctl)->pwr_state_check_timer, (ms)); \
 	} while (0)
 
-extern void rtw_init_pwrctrl_priv(struct adapter *adapter);
-extern void rtw_free_pwrctrl_priv(struct adapter *adapter);
+void rtw_init_pwrctrl_priv(struct adapter *adapter);
+void rtw_free_pwrctrl_priv(struct adapter *adapter);
 
 s32 rtw_register_task_alive(struct adapter *adapter, u32 task);
 void rtw_unregister_task_alive(struct adapter *adapter, u32 task);
-extern s32 rtw_register_tx_alive(struct adapter *padapter);
-extern void rtw_unregister_tx_alive(struct adapter *padapter);
-extern s32 rtw_register_cmd_alive(struct adapter *padapter);
-extern void rtw_unregister_cmd_alive(struct adapter *padapter);
-extern void cpwm_int_hdl(struct adapter *padapter, struct reportpwrstate_parm *preportpwrstate);
-extern void LPS_Leave_check(struct adapter *padapter);
-
-extern void LeaveAllPowerSaveMode(struct adapter *Adapter);
-extern void LeaveAllPowerSaveModeDirect(struct adapter *Adapter);
+s32 rtw_register_tx_alive(struct adapter *padapter);
+void rtw_unregister_tx_alive(struct adapter *padapter);
+s32 rtw_register_cmd_alive(struct adapter *padapter);
+void rtw_unregister_cmd_alive(struct adapter *padapter);
+void cpwm_int_hdl(struct adapter *padapter, struct reportpwrstate_parm *preportpwrstate);
+void LPS_Leave_check(struct adapter *padapter);
+
+void LeaveAllPowerSaveMode(struct adapter *Adapter);
+void LeaveAllPowerSaveModeDirect(struct adapter *Adapter);
 void _ips_enter(struct adapter *padapter);
 void ips_enter(struct adapter *padapter);
 int _ips_leave(struct adapter *padapter);
diff --git a/drivers/staging/rtl8723bs/include/rtw_recv.h b/drivers/staging/rtl8723bs/include/rtw_recv.h
index 9c32d480a20a1c..5c197bd6915fcd 100644
--- a/drivers/staging/rtl8723bs/include/rtw_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtw_recv.h
@@ -70,26 +70,26 @@ struct phy_info {
 	u8 rx_pwd_ba11;
 
 	u8 SignalQuality;	 /*  in 0-100 index. */
-	s8		rx_mimo_signal_quality[4];	/* per-path's EVM */
+	s8 rx_mimo_signal_quality[4];	/* per-path's EVM */
 	u8 RxMIMOEVMdbm[4];		/* per-path's EVM dbm */
 
 	u8 rx_mimo_signal_strength[4];/*  in 0~100 index */
 
-	u16 	Cfo_short[4];			/*  per-path's Cfo_short */
-	u16 	Cfo_tail[4];			/*  per-path's Cfo_tail */
+	u16 Cfo_short[4];			/*  per-path's Cfo_short */
+	u16 Cfo_tail[4];			/*  per-path's Cfo_tail */
 
-	s8		RxPower; /*  in dBm Translate from PWdB */
-	s8		RecvSignalPower;/*  Real power in dBm for this packet, no beautification and aggregation. Keep this raw info to be used for the other procedures. */
+	s8 RxPower; /*  in dBm Translate from PWdB */
+	s8 RecvSignalPower;/*  Real power in dBm for this packet, no beautification and aggregation. Keep this raw info to be used for the other procedures. */
 	u8 bt_rx_rssi_percentage;
 	u8 SignalStrength; /*  in 0-100 index. */
 
-	s8		RxPwr[4];				/* per-path's pwdb */
+	s8 RxPwr[4];				/* per-path's pwdb */
 	u8 RxSNR[4];				/* per-path's SNR */
 	u8 BandWidth;
 	u8 btCoexPwrAdjust;
 };
 
-struct rx_pkt_attrib	{
+struct rx_pkt_attrib {
 	u16 pkt_len;
 	u8 physt;
 	u8 drvinfo_sz;
@@ -170,22 +170,22 @@ accesser of recv_priv: rtw_recv_entry(dispatch / passive level); recv_thread(pas
 using enter_critical section to protect
 */
 struct recv_priv {
-	spinlock_t	lock;
-	struct __queue	free_recv_queue;
-	struct __queue	recv_pending_queue;
-	struct __queue	uc_swdec_pending_queue;
+	spinlock_t lock;
+	struct __queue free_recv_queue;
+	struct __queue recv_pending_queue;
+	struct __queue uc_swdec_pending_queue;
 	u8 *pallocated_frame_buf;
 	u8 *precv_frame_buf;
 	uint free_recvframe_cnt;
-	struct adapter	*adapter;
+	struct adapter *adapter;
 	u32 bIsAnyNonBEPkts;
-	u64	rx_bytes;
-	u64	rx_pkts;
-	u64	rx_drop;
-	uint  rx_icv_err;
-	uint  rx_largepacket_crcerr;
-	uint  rx_smallpacket_crcerr;
-	uint  rx_middlepacket_crcerr;
+	u64 rx_bytes;
+	u64 rx_pkts;
+	u64 rx_drop;
+	uint rx_icv_err;
+	uint rx_largepacket_crcerr;
+	uint rx_smallpacket_crcerr;
+	uint rx_middlepacket_crcerr;
 
 	struct tasklet_struct irq_prepare_beacon_tasklet;
 	struct tasklet_struct recv_tasklet;
@@ -194,10 +194,10 @@ struct recv_priv {
 
 	u8 *pallocated_recv_buf;
 	u8 *precv_buf;    /*  4 alignment */
-	struct __queue	free_recv_buf_queue;
+	struct __queue free_recv_buf_queue;
 	u32 free_recv_buf_queue_cnt;
 
-	struct __queue	recv_buf_pending_queue;
+	struct __queue recv_buf_pending_queue;
 
 	/* For display the phy information */
 	u8 is_signal_dbg;	/*  for debug */
@@ -212,7 +212,6 @@ struct recv_priv {
 	/* s8 RxRssi[2]; */
 	/* int FalseAlmCnt_all; */
 
-
 	struct timer_list signal_stat_timer;
 	u32 signal_stat_sampling_interval;
 	/* u32 signal_stat_converging_constant; */
@@ -224,13 +223,13 @@ struct recv_priv {
 
 struct sta_recv_priv {
 
-	spinlock_t	lock;
-	signed int	option;
+	spinlock_t lock;
+	signed int option;
 
 	/* struct __queue	blk_strms[MAX_RX_NUMBLKS]; */
 	struct __queue defrag_q;	 /* keeping the fragment frame until defrag */
 
-	struct	stainfo_rxcache rxcache;
+	struct stainfo_rxcache rxcache;
 
 	/* uint	sta_rx_bytes; */
 	/* uint	sta_rx_pkts; */
@@ -238,7 +237,6 @@ struct sta_recv_priv {
 
 };
 
-
 struct recv_buf {
 	struct list_head list;
 
@@ -257,7 +255,7 @@ struct recv_buf {
 	u8 *ptail;
 	u8 *pend;
 
-	struct sk_buff	*pskb;
+	struct sk_buff *pskb;
 	u8 reuse;
 };
 
@@ -278,11 +276,11 @@ struct recv_buf {
 
 */
 struct recv_frame_hdr {
-	struct list_head	list;
-	struct sk_buff	 *pkt;
-	struct sk_buff	 *pkt_newalloc;
+	struct list_head list;
+	struct sk_buff *pkt;
+	struct sk_buff *pkt_newalloc;
 
-	struct adapter  *adapter;
+	struct adapter *adapter;
 
 	u8 fragcnt;
 
@@ -290,7 +288,7 @@ struct recv_frame_hdr {
 
 	struct rx_pkt_attrib attrib;
 
-	uint  len;
+	uint len;
 	u8 *rx_head;
 	u8 *rx_data;
 	u8 *rx_tail;
@@ -306,7 +304,6 @@ struct recv_frame_hdr {
 	struct recv_reorder_ctrl *preorder_ctrl;
 };
 
-
 union recv_frame {
 	union{
 		struct list_head list;
@@ -328,7 +325,7 @@ enum {
 
 extern union recv_frame *_rtw_alloc_recvframe(struct __queue *pfree_recv_queue);  /* get a free recv_frame from pfree_recv_queue */
 extern union recv_frame *rtw_alloc_recvframe(struct __queue *pfree_recv_queue);  /* get a free recv_frame from pfree_recv_queue */
-extern int	 rtw_free_recvframe(union recv_frame *precvframe, struct __queue *pfree_recv_queue);
+extern int rtw_free_recvframe(union recv_frame *precvframe, struct __queue *pfree_recv_queue);
 
 extern int _rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue *queue);
 extern int rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue *queue);
@@ -438,7 +435,7 @@ static inline signed int get_recvframe_len(union recv_frame *precvframe)
 
 static inline s32 translate_percentage_to_dbm(u32 SignalStrengthIndex)
 {
-	s32	SignalPower; /*  in dBm. */
+	s32 SignalPower; /*  in dBm. */
 
 	/*  Translate to dBm (x = 0.5y-95). */
 	SignalPower = (s32)((SignalStrengthIndex + 1) >> 1);
@@ -452,6 +449,6 @@ struct sta_info;
 
 extern void _rtw_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv);
 
-extern void  mgt_dispatcher(struct adapter *padapter, union recv_frame *precv_frame);
+extern void mgt_dispatcher(struct adapter *padapter, union recv_frame *precv_frame);
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtw_xmit.h b/drivers/staging/rtl8723bs/include/rtw_xmit.h
index 917e846e9ce2b8..873807fcd0dd0f 100644
--- a/drivers/staging/rtl8723bs/include/rtw_xmit.h
+++ b/drivers/staging/rtl8723bs/include/rtw_xmit.h
@@ -403,10 +403,10 @@ extern struct xmit_frame *__rtw_alloc_cmdxmitframe(struct xmit_priv *pxmitpriv,
 #define rtw_alloc_bcnxmitframe(p) __rtw_alloc_cmdxmitframe(p, CMDBUF_BEACON)
 
 extern struct xmit_buf *rtw_alloc_xmitbuf_ext(struct xmit_priv *pxmitpriv);
-extern s32 rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
+void rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
 
 extern struct xmit_buf *rtw_alloc_xmitbuf(struct xmit_priv *pxmitpriv);
-extern s32 rtw_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
+void rtw_free_xmitbuf(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf);
 
 void rtw_count_tx_stats(struct adapter *padapter, struct xmit_frame *pxmitframe, int sz);
 extern void rtw_update_protection(struct adapter *padapter, u8 *ie, uint ie_len);
@@ -416,7 +416,7 @@ extern s32 rtw_put_snap(u8 *data, u16 h_proto);
 extern struct xmit_frame *rtw_alloc_xmitframe(struct xmit_priv *pxmitpriv);
 struct xmit_frame *rtw_alloc_xmitframe_ext(struct xmit_priv *pxmitpriv);
 struct xmit_frame *rtw_alloc_xmitframe_once(struct xmit_priv *pxmitpriv);
-extern s32 rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitframe);
+void rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitframe);
 extern void rtw_free_xmitframe_queue(struct xmit_priv *pxmitpriv, struct __queue *pframequeue);
 struct tx_servq *rtw_get_sta_pending(struct adapter *padapter, struct sta_info *psta, signed int up, u8 *ac);
 int rtw_xmitframe_enqueue(struct adapter *padapter, struct xmit_frame *pxmitframe);
@@ -424,7 +424,7 @@ int rtw_xmitframe_enqueue(struct adapter *padapter, struct xmit_frame *pxmitfram
 extern u32 rtw_calculate_wlan_pkt_size_by_attribue(struct pkt_attrib *pattrib);
 #define rtw_wlan_pkt_size(f) rtw_calculate_wlan_pkt_size_by_attribue(&f->attrib)
 extern int rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct xmit_frame *pxmitframe);
-extern s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct xmit_frame *pxmitframe);
+extern int rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct xmit_frame *pxmitframe);
 s32 _rtw_init_hw_txqueue(struct hw_txqueue *phw_txqueue, u8 ac_tag);
 void _rtw_init_sta_xmit_priv(struct sta_xmit_priv *psta_xmitpriv);
 
diff --git a/drivers/staging/rtl8723bs/include/sdio_ops.h b/drivers/staging/rtl8723bs/include/sdio_ops.h
index 252b0e1646e51a..f939263495df41 100644
--- a/drivers/staging/rtl8723bs/include/sdio_ops.h
+++ b/drivers/staging/rtl8723bs/include/sdio_ops.h
@@ -7,27 +7,26 @@
 #ifndef __SDIO_OPS_H__
 #define __SDIO_OPS_H__
 
-
 #include <sdio_ops_linux.h>
 
-extern void sdio_set_intf_ops(struct adapter *padapter, struct _io_ops *pops);
+void sdio_set_intf_ops(struct adapter *padapter, struct _io_ops *pops);
 
 /* extern void sdio_func1cmd52_read(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *rmem); */
 /* extern void sdio_func1cmd52_write(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *wmem); */
-extern u8 SdioLocalCmd52Read1Byte(struct adapter *padapter, u32 addr);
-extern void SdioLocalCmd52Write1Byte(struct adapter *padapter, u32 addr, u8 v);
-extern s32 sdio_local_read(struct adapter *padapter, u32 addr, u32 cnt, u8 *pbuf);
-extern s32 sdio_local_write(struct adapter *padapter, u32 addr, u32 cnt, u8 *pbuf);
+u8 SdioLocalCmd52Read1Byte(struct adapter *padapter, u32 addr);
+void SdioLocalCmd52Write1Byte(struct adapter *padapter, u32 addr, u8 v);
+s32 sdio_local_read(struct adapter *padapter, u32 addr, u32 cnt, u8 *pbuf);
+s32 sdio_local_write(struct adapter *padapter, u32 addr, u32 cnt, u8 *pbuf);
 
 u32 _sdio_read32(struct adapter *padapter, u32 addr);
 s32 _sdio_write32(struct adapter *padapter, u32 addr, u32 val);
 
-extern void sd_int_hdl(struct adapter *padapter);
-extern u8 CheckIPSStatus(struct adapter *padapter);
+void sd_int_hdl(struct adapter *padapter);
+u8 CheckIPSStatus(struct adapter *padapter);
 
-extern void InitInterrupt8723BSdio(struct adapter *padapter);
+void InitInterrupt8723BSdio(struct adapter *padapter);
 void rtw_sdio_enable_interrupt(struct adapter *padapter);
 void rtw_sdio_disable_interrupt(struct adapter *padapter);
-extern u8 HalQueryTxBufferStatus8723BSdio(struct adapter *padapter);
-extern void HalQueryTxOQTBufferStatus8723BSdio(struct adapter *padapter);
+u8 HalQueryTxBufferStatus8723BSdio(struct adapter *padapter);
+void HalQueryTxOQTBufferStatus8723BSdio(struct adapter *padapter);
 #endif /*  !__SDIO_OPS_H__ */
diff --git a/drivers/staging/rtl8723bs/include/sta_info.h b/drivers/staging/rtl8723bs/include/sta_info.h
index 0f53e77c110c5e..bc78c32192c053 100644
--- a/drivers/staging/rtl8723bs/include/sta_info.h
+++ b/drivers/staging/rtl8723bs/include/sta_info.h
@@ -28,50 +28,50 @@ struct wlan_acl_pool {
 	int mode;
 	int num;
 	struct rtw_wlan_acl_node aclnode[NUM_ACL];
-	struct __queue	acl_node_q;
+	struct __queue acl_node_q;
 };
 
 struct rssi_sta {
-	s32	UndecoratedSmoothedPWDB;
-	s32	UndecoratedSmoothedCCK;
-	s32	UndecoratedSmoothedOFDM;
-	u64	PacketMap;
+	s32 UndecoratedSmoothedPWDB;
+	s32 UndecoratedSmoothedCCK;
+	s32 UndecoratedSmoothedOFDM;
+	u64 PacketMap;
 	u8 ValidBit;
 };
 
-struct	stainfo_stats	{
+struct stainfo_stats {
 
 	u64 rx_mgnt_pkts;
-		u64 rx_beacon_pkts;
-		u64 rx_probereq_pkts;
-		u64 rx_probersp_pkts;
-		u64 rx_probersp_bm_pkts;
-		u64 rx_probersp_uo_pkts;
+	u64 rx_beacon_pkts;
+	u64 rx_probereq_pkts;
+	u64 rx_probersp_pkts;
+	u64 rx_probersp_bm_pkts;
+	u64 rx_probersp_uo_pkts;
 	u64 rx_ctrl_pkts;
 	u64 rx_data_pkts;
 
-	u64	last_rx_mgnt_pkts;
-		u64 last_rx_beacon_pkts;
-		u64 last_rx_probereq_pkts;
-		u64 last_rx_probersp_pkts;
-		u64 last_rx_probersp_bm_pkts;
-		u64 last_rx_probersp_uo_pkts;
-	u64	last_rx_ctrl_pkts;
-	u64	last_rx_data_pkts;
-
-	u64	rx_bytes;
-	u64	rx_drops;
-
-	u64	tx_pkts;
-	u64	tx_bytes;
-	u64  tx_drops;
+	u64 last_rx_mgnt_pkts;
+	u64 last_rx_beacon_pkts;
+	u64 last_rx_probereq_pkts;
+	u64 last_rx_probersp_pkts;
+	u64 last_rx_probersp_bm_pkts;
+	u64 last_rx_probersp_uo_pkts;
+	u64 last_rx_ctrl_pkts;
+	u64 last_rx_data_pkts;
+
+	u64 rx_bytes;
+	u64 rx_drops;
+
+	u64 tx_pkts;
+	u64 tx_bytes;
+	u64 tx_drops;
 };
 
 struct sta_info {
 
-	spinlock_t	lock;
-	struct list_head	list; /* free_sta_queue */
-	struct list_head	hash_list; /* sta_hash */
+	spinlock_t lock;
+	struct list_head list; /* free_sta_queue */
+	struct list_head hash_list; /* sta_hash */
 	struct adapter *padapter;
 
 	struct sta_xmit_priv sta_xmitpriv;
@@ -87,19 +87,18 @@ struct sta_info {
 	u8 hwaddr[ETH_ALEN];
 
 	bool ieee8021x_blocked;
-	uint	dot118021XPrivacy; /* aes, tkip... */
-	union Keytype	dot11tkiptxmickey;
-	union Keytype	dot11tkiprxmickey;
-	union Keytype	dot118021x_UncstKey;
-	union pn48		dot11txpn;			/*  PN48 used for Unicast xmit */
-	union pn48		dot11wtxpn;			/*  PN48 used for Unicast mgmt xmit. */
-	union pn48		dot11rxpn;			/*  PN48 used for Unicast recv. */
-
+	uint dot118021XPrivacy; /* aes, tkip... */
+	union Keytype dot11tkiptxmickey;
+	union Keytype dot11tkiprxmickey;
+	union Keytype dot118021x_UncstKey;
+	union pn48 dot11txpn;			/*  PN48 used for Unicast xmit */
+	union pn48 dot11wtxpn;			/*  PN48 used for Unicast mgmt xmit. */
+	union pn48 dot11rxpn;			/*  PN48 used for Unicast recv. */
 
 	u8 bssrateset[16];
 	u32 bssratelen;
-	s32  rssi;
-	s32	signal_quality;
+	s32 rssi;
+	s32 signal_quality;
 
 	u8 cts2self;
 	u8 rtsen;
@@ -125,7 +124,6 @@ struct sta_info {
 	/* unsigned char 	ampdu_txen_bitmap; */
 	u16 BA_starting_seqctrl[16];
 
-
 	struct ht_priv htpriv;
 
 	/* Notes: */
@@ -186,7 +184,7 @@ struct sta_info {
 	u32 assoc_req_len;
 
 	/* for DM */
-	struct rssi_sta	 rssi_stat;
+	struct rssi_sta	rssi_stat;
 
 	/* ODM_STA_INFO_T */
 	/*  ================ODM Relative Info ======================= */
@@ -254,14 +252,14 @@ struct sta_info {
 
 #define STA_PKTS_FMT "(m:%llu, c:%llu, d:%llu)"
 
-struct	sta_priv {
+struct sta_priv {
 
 	u8 *pallocated_stainfo_buf;
 	u8 *pstainfo_buf;
-	struct __queue	free_sta_queue;
+	struct __queue free_sta_queue;
 
 	spinlock_t sta_hash_lock;
-	struct list_head   sta_hash[NUM_STA];
+	struct list_head sta_hash[NUM_STA];
 	int asoc_sta_count;
 	struct __queue sleep_q;
 	struct __queue wakeup_q;
@@ -319,7 +317,7 @@ extern u32 _rtw_free_sta_priv(struct sta_priv *pstapriv);
 int rtw_stainfo_offset(struct sta_priv *stapriv, struct sta_info *sta);
 struct sta_info *rtw_get_stainfo_by_offset(struct sta_priv *stapriv, int offset);
 
-extern struct sta_info *rtw_alloc_stainfo(struct	sta_priv *pstapriv, u8 *hwaddr);
+extern struct sta_info *rtw_alloc_stainfo(struct sta_priv *pstapriv, u8 *hwaddr);
 extern u32 rtw_free_stainfo(struct adapter *padapter, struct sta_info *psta);
 extern void rtw_free_all_stainfo(struct adapter *padapter);
 extern struct sta_info *rtw_get_stainfo(struct sta_priv *pstapriv, u8 *hwaddr);
diff --git a/drivers/staging/rtl8723bs/include/wlan_bssdef.h b/drivers/staging/rtl8723bs/include/wlan_bssdef.h
index f4c1456f147626..9137057726ef81 100644
--- a/drivers/staging/rtl8723bs/include/wlan_bssdef.h
+++ b/drivers/staging/rtl8723bs/include/wlan_bssdef.h
@@ -12,12 +12,8 @@
 
 
 #define NDIS_802_11_LENGTH_SSID         32
-#define NDIS_802_11_LENGTH_RATES        8
 #define NDIS_802_11_LENGTH_RATES_EX     16
 
-typedef unsigned char   NDIS_802_11_RATES[NDIS_802_11_LENGTH_RATES];        /*  Set of 8 data rates */
-typedef unsigned char   NDIS_802_11_RATES_EX[NDIS_802_11_LENGTH_RATES_EX];  /*  Set of 16 data rates */
-
 struct ndis_802_11_ssid {
 	u32 ssid_length;
 	u8 ssid[32];
@@ -50,7 +46,7 @@ struct ndis_80211_var_ie {
  * ETH_ALEN + 2 +
  * sizeof (struct ndis_802_11_ssid) + sizeof (u32) +
  * sizeof (long) +
- * sizeof (struct ndis_802_11_conf) + sizeof (NDIS_802_11_RATES_EX) + ie_length
+ * sizeof (struct ndis_802_11_conf) + NDIS_802_11_LENGTH_RATES_EX + ie_length
  *
  * Except for ie_length, all other fields are fixed length. Therefore, we can
  * define a macro to present the partial sum.
@@ -142,7 +138,7 @@ struct wlan_bssid_ex {
 	long rssi;/* in dBM, raw data , get from PHY) */
 	struct ndis_802_11_conf configuration;
 	enum nl80211_iftype infrastructure_mode;
-	NDIS_802_11_RATES_EX supported_rates;
+	u8 supported_rates[NDIS_802_11_LENGTH_RATES_EX];
 	struct wlan_phy_info phy_info;
 	u32 ie_length;
 	u8 ies[MAX_IE_SZ];	/* timestamp, beacon interval, and capability information) */
diff --git a/drivers/staging/rtl8723bs/include/xmit_osdep.h b/drivers/staging/rtl8723bs/include/xmit_osdep.h
index ee6616880d415a..85c42fa06dbdcf 100644
--- a/drivers/staging/rtl8723bs/include/xmit_osdep.h
+++ b/drivers/staging/rtl8723bs/include/xmit_osdep.h
@@ -25,19 +25,19 @@ struct sta_xmit_priv;
 struct xmit_frame;
 struct xmit_buf;
 
-extern void _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev);
-extern netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev);
+void _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev);
+netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev);
 
 void rtw_os_xmit_schedule(struct adapter *padapter);
 
 void rtw_os_xmit_resource_free(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 free_sz, u8 flag);
 
-extern unsigned int rtw_remainder_len(struct pkt_file *pfile);
-extern void _rtw_open_pktfile(struct sk_buff *pkt, struct pkt_file *pfile);
+unsigned int rtw_remainder_len(struct pkt_file *pfile);
+void _rtw_open_pktfile(struct sk_buff *pkt, struct pkt_file *pfile);
 int _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, unsigned int rlen);
-extern signed int rtw_endofpktfile(struct pkt_file *pfile);
+signed int rtw_endofpktfile(struct pkt_file *pfile);
 
-extern void rtw_os_pkt_complete(struct adapter *padapter, struct sk_buff *pkt);
-extern void rtw_os_xmit_complete(struct adapter *padapter, struct xmit_frame *pxframe);
+void rtw_os_pkt_complete(struct adapter *padapter, struct sk_buff *pkt);
+void rtw_os_xmit_complete(struct adapter *padapter, struct xmit_frame *pxframe);
 
 #endif /* __XMIT_OSDEP_H_ */
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
index 3468d4114f6044..50842808a85780 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
@@ -216,7 +216,7 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl
 
 	struct wireless_dev *wdev = padapter->rtw_wdev;
 	struct wiphy *wiphy = wdev->wiphy;
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	bssinf_len = pnetwork->network.ie_length + sizeof(struct ieee80211_hdr_3addr);
 	if (bssinf_len > MAX_BSSINFO_LEN)
@@ -281,7 +281,7 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl
 	pbuf = buf;
 
 	pwlanhdr = (struct ieee80211_hdr *)pbuf;
-	fctrl = &(pwlanhdr->frame_control);
+	fctrl = &pwlanhdr->frame_control;
 	*(fctrl) = 0;
 
 	SetSeqNum(pwlanhdr, 0/*pmlmeext->mgnt_seq*/);
@@ -291,7 +291,7 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl
 		eth_broadcast_addr(pwlanhdr->addr1);
 		SetFrameSubType(pbuf, WIFI_BEACON);
 	} else {
-		memcpy(pwlanhdr->addr1, myid(&(padapter->eeprompriv)), ETH_ALEN);
+		memcpy(pwlanhdr->addr1, myid(&padapter->eeprompriv), ETH_ALEN);
 		SetFrameSubType(pbuf, WIFI_PROBERSP);
 	}
 
@@ -328,7 +328,7 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl
  */
 int rtw_cfg80211_check_bss(struct adapter *padapter)
 {
-	struct wlan_bssid_ex  *pnetwork = &(padapter->mlmeextpriv.mlmext_info.network);
+	struct wlan_bssid_ex  *pnetwork = &padapter->mlmeextpriv.mlmext_info.network;
 	struct cfg80211_bss *bss = NULL;
 	struct ieee80211_channel *notify_channel = NULL;
 	u32 freq;
@@ -352,7 +352,7 @@ int rtw_cfg80211_check_bss(struct adapter *padapter)
 void rtw_cfg80211_ibss_indicate_connect(struct adapter *padapter)
 {
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network  *cur_network = &(pmlmepriv->cur_network);
+	struct wlan_network  *cur_network = &pmlmepriv->cur_network;
 	struct wireless_dev *pwdev = padapter->rtw_wdev;
 	struct wiphy *wiphy = pwdev->wiphy;
 	int freq = (int)cur_network->network.configuration.ds_config;
@@ -362,7 +362,7 @@ void rtw_cfg80211_ibss_indicate_connect(struct adapter *padapter)
 		return;
 
 	if (!rtw_cfg80211_check_bss(padapter)) {
-		struct wlan_bssid_ex  *pnetwork = &(padapter->mlmeextpriv.mlmext_info.network);
+		struct wlan_bssid_ex  *pnetwork = &padapter->mlmeextpriv.mlmext_info.network;
 		struct wlan_network *scanned = pmlmepriv->cur_network_scanned;
 
 		if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
@@ -394,7 +394,7 @@ void rtw_cfg80211_ibss_indicate_connect(struct adapter *padapter)
 void rtw_cfg80211_indicate_connect(struct adapter *padapter)
 {
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct wlan_network  *cur_network = &(pmlmepriv->cur_network);
+	struct wlan_network  *cur_network = &pmlmepriv->cur_network;
 	struct wireless_dev *pwdev = padapter->rtw_wdev;
 
 	if (pwdev->iftype != NL80211_IFTYPE_STATION
@@ -407,7 +407,7 @@ void rtw_cfg80211_indicate_connect(struct adapter *padapter)
 		return;
 
 	{
-		struct wlan_bssid_ex  *pnetwork = &(padapter->mlmeextpriv.mlmext_info.network);
+		struct wlan_bssid_ex  *pnetwork = &padapter->mlmeextpriv.mlmext_info.network;
 		struct wlan_network *scanned = pmlmepriv->cur_network_scanned;
 
 		if (!scanned) {
@@ -493,7 +493,7 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 	struct sta_info *psta = NULL, *pbcmc_sta = NULL;
 	struct adapter *padapter = rtw_netdev_priv(dev);
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
-	struct security_priv *psecuritypriv =  &(padapter->securitypriv);
+	struct security_priv *psecuritypriv =  &padapter->securitypriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	char *grpkey = padapter->securitypriv.dot118021XGrpKey[param->u.crypt.idx].skey;
 	char *txkey = padapter->securitypriv.dot118021XGrptxmickey[param->u.crypt.idx].skey;
@@ -552,7 +552,7 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 			psecuritypriv->dot11PrivacyKeyIndex = wep_key_idx;
 		}
 
-		memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), param->u.crypt.key, wep_key_len);
+		memcpy(&psecuritypriv->dot11DefKey[wep_key_idx].skey[0], param->u.crypt.key, wep_key_len);
 
 		psecuritypriv->dot11DefKeylen[wep_key_idx] = wep_key_len;
 
@@ -578,8 +578,8 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 				memcpy(grpkey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
 
 				/* set mic key */
-				memcpy(txkey, &(param->u.crypt.key[16]), 8);
-				memcpy(rxkey, &(param->u.crypt.key[24]), 8);
+				memcpy(txkey, &param->u.crypt.key[16], 8);
+				memcpy(rxkey, &param->u.crypt.key[24], 8);
 
 				psecuritypriv->busetkipkey = true;
 
@@ -622,8 +622,8 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 					psta->dot118021XPrivacy = _TKIP_;
 
 					/* set mic key */
-					memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
-					memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+					memcpy(psta->dot11tkiptxmickey.skey, &param->u.crypt.key[16], 8);
+					memcpy(psta->dot11tkiprxmickey.skey, &param->u.crypt.key[24], 8);
 
 					psecuritypriv->busetkipkey = true;
 
@@ -652,8 +652,8 @@ static int rtw_cfg80211_ap_set_encryption(struct net_device *dev, struct ieee_pa
 					memcpy(grpkey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
 
 					/* set mic key */
-					memcpy(txkey, &(param->u.crypt.key[16]), 8);
-					memcpy(rxkey, &(param->u.crypt.key[24]), 8);
+					memcpy(txkey, &param->u.crypt.key[16], 8);
+					memcpy(rxkey, &param->u.crypt.key[24], 8);
 
 					psecuritypriv->busetkipkey = true;
 
@@ -748,7 +748,7 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 			psecuritypriv->dot11PrivacyKeyIndex = wep_key_idx;
 		}
 
-		memcpy(&(psecuritypriv->dot11DefKey[wep_key_idx].skey[0]), param->u.crypt.key, wep_key_len);
+		memcpy(&psecuritypriv->dot11DefKey[wep_key_idx].skey[0], param->u.crypt.key, wep_key_len);
 
 		psecuritypriv->dot11DefKeylen[wep_key_idx] = wep_key_len;
 
@@ -778,8 +778,8 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 					memcpy(psta->dot118021x_UncstKey.skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
 
 					if (strcmp(param->u.crypt.alg, "TKIP") == 0) { /* set mic key */
-						memcpy(psta->dot11tkiptxmickey.skey, &(param->u.crypt.key[16]), 8);
-						memcpy(psta->dot11tkiprxmickey.skey, &(param->u.crypt.key[24]), 8);
+						memcpy(psta->dot11tkiptxmickey.skey, &param->u.crypt.key[16], 8);
+						memcpy(psta->dot11tkiprxmickey.skey, &param->u.crypt.key[24], 8);
 
 						padapter->securitypriv.busetkipkey = false;
 						/* _set_timer(&padapter->securitypriv.tkip_timer, 50); */
@@ -789,8 +789,8 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 				} else { /* group key */
 					if (strcmp(param->u.crypt.alg, "TKIP") == 0 || strcmp(param->u.crypt.alg, "CCMP") == 0) {
 						memcpy(padapter->securitypriv.dot118021XGrpKey[param->u.crypt.idx].skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len));
-						memcpy(padapter->securitypriv.dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8);
-						memcpy(padapter->securitypriv.dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8);
+						memcpy(padapter->securitypriv.dot118021XGrptxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[16], 8);
+						memcpy(padapter->securitypriv.dot118021XGrprxmickey[param->u.crypt.idx].skey, &param->u.crypt.key[24], 8);
 						padapter->securitypriv.binstallGrpkey = true;
 
 						padapter->securitypriv.dot118021XGrpKeyid = param->u.crypt.idx;
@@ -825,7 +825,8 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 }
 
 static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-				int link_id, u8 key_index, bool pairwise,
+				int link_id, u8 key_index,
+				enum nl80211_key_type type,
 				const u8 *mac_addr, struct key_params *params)
 {
 	char *alg_name;
@@ -904,7 +905,8 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 }
 
 static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-				int link_id, u8 key_index, bool pairwise,
+				int link_id, u8 key_index,
+				enum nl80211_key_type type,
 				const u8 *mac_addr, void *cookie,
 				void (*callback)(void *cookie,
 						 struct key_params*))
@@ -913,7 +915,8 @@ static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 }
 
 static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-				int link_id, u8 key_index, bool pairwise,
+				int link_id, u8 key_index,
+				enum nl80211_key_type type,
 				const u8 *mac_addr)
 {
 	struct adapter *padapter = rtw_netdev_priv(wdev->netdev);
@@ -981,7 +984,7 @@ static int cfg80211_rtw_get_station(struct wiphy *wiphy,
 	/* for infra./P2PClient mode */
 	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)
 		&& check_fwstate(pmlmepriv, _FW_LINKED)) {
-		struct wlan_network  *cur_network = &(pmlmepriv->cur_network);
+		struct wlan_network  *cur_network = &pmlmepriv->cur_network;
 
 		if (memcmp((u8 *)mac, cur_network->network.mac_address, ETH_ALEN)) {
 			ret = -ENOENT;
@@ -1022,7 +1025,7 @@ static int cfg80211_rtw_change_iface(struct wiphy *wiphy,
 	enum nl80211_iftype old_type;
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct wireless_dev *rtw_wdev = padapter->rtw_wdev;
-	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 	int ret = 0;
 
 	if (adapter_to_dvobj(padapter)->processing_dev_remove) {
@@ -1111,11 +1114,11 @@ void rtw_cfg80211_unlink_bss(struct adapter *padapter, struct wlan_network *pnet
 void rtw_cfg80211_surveydone_event_callback(struct adapter *padapter)
 {
 	struct list_head					*plist, *phead;
-	struct	mlme_priv *pmlmepriv = &(padapter->mlmepriv);
-	struct __queue *queue	= &(pmlmepriv->scanned_queue);
+	struct	mlme_priv *pmlmepriv = &padapter->mlmepriv;
+	struct __queue *queue	= &pmlmepriv->scanned_queue;
 	struct	wlan_network	*pnetwork = NULL;
 
-	spin_lock_bh(&(pmlmepriv->scanned_queue.lock));
+	spin_lock_bh(&pmlmepriv->scanned_queue.lock);
 
 	phead = get_list_head(queue);
 	list_for_each(plist, phead)
@@ -1124,13 +1127,13 @@ void rtw_cfg80211_surveydone_event_callback(struct adapter *padapter)
 
 		/* report network only if the current channel set contains the channel to which this network belongs */
 		if (rtw_ch_set_search_ch(padapter->mlmeextpriv.channel_set, pnetwork->network.configuration.ds_config) >= 0
-			&& true == rtw_validate_ssid(&(pnetwork->network.ssid))) {
+			&& true == rtw_validate_ssid(&pnetwork->network.ssid)) {
 			/* ev =translate_scan(padapter, a, pnetwork, ev, stop); */
 			rtw_cfg80211_inform_bss(padapter, pnetwork);
 		}
 	}
 
-	spin_unlock_bh(&(pmlmepriv->scanned_queue.lock));
+	spin_unlock_bh(&pmlmepriv->scanned_queue.lock);
 }
 
 static int rtw_cfg80211_set_probe_req_wpsp2pie(struct adapter *padapter, char *buf, int len)
@@ -1138,7 +1141,7 @@ static int rtw_cfg80211_set_probe_req_wpsp2pie(struct adapter *padapter, char *b
 	int ret = 0;
 	unsigned int wps_ielen = 0;
 	u8 *wps_ie;
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	if (len > 0) {
 		wps_ie = rtw_get_wps_ie(buf, len, NULL, &wps_ielen);
@@ -1830,27 +1833,26 @@ static int cfg80211_rtw_set_pmksa(struct wiphy *wiphy,
 				  struct net_device *ndev,
 				  struct cfg80211_pmksa *pmksa)
 {
-	u8 index, blInserted = false;
+	bool inserted = false;
+	u8 index;
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 
 	if (is_zero_ether_addr((u8 *)pmksa->bssid))
 		return -EINVAL;
 
-	blInserted = false;
-
 	/* overwrite PMKID */
 	for (index = 0 ; index < NUM_PMKID_CACHE; index++) {
 		if (!memcmp(psecuritypriv->PMKIDList[index].Bssid, (u8 *)pmksa->bssid, ETH_ALEN)) {
 			memcpy(psecuritypriv->PMKIDList[index].PMKID, (u8 *)pmksa->pmkid, WLAN_PMKID_LEN);
 			psecuritypriv->PMKIDList[index].bUsed = true;
 			psecuritypriv->PMKIDIndex = index + 1;
-			blInserted = true;
+			inserted = true;
 			break;
 		}
 	}
 
-	if (!blInserted) {
+	if (!inserted) {
 		memcpy(psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].Bssid, (u8 *)pmksa->bssid, ETH_ALEN);
 		memcpy(psecuritypriv->PMKIDList[psecuritypriv->PMKIDIndex].PMKID, (u8 *)pmksa->pmkid, WLAN_PMKID_LEN);
 
@@ -1867,7 +1869,8 @@ static int cfg80211_rtw_del_pmksa(struct wiphy *wiphy,
 				  struct net_device *ndev,
 				  struct cfg80211_pmksa *pmksa)
 {
-	u8 index, bMatched = false;
+	bool matched = false;
+	u8 index;
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 
@@ -1880,12 +1883,12 @@ static int cfg80211_rtw_del_pmksa(struct wiphy *wiphy,
 			eth_zero_addr(psecuritypriv->PMKIDList[index].Bssid);
 			memset(psecuritypriv->PMKIDList[index].PMKID, 0x00, WLAN_PMKID_LEN);
 			psecuritypriv->PMKIDList[index].bUsed = false;
-			bMatched = true;
+			matched = true;
 			break;
 		}
 	}
 
-	if (!bMatched)
+	if (!matched)
 		return -EINVAL;
 
 	return 0;
@@ -2058,8 +2061,8 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc
 		/* u8 category, action, OUI_Subtype, dialogToken = 0; */
 		/* unsigned char *frame_body; */
 		struct ieee80211_hdr *pwlanhdr;
-		struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
-		struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+		struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+		struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 		u8 *buf = skb->data;
 		u32 len = skb->len;
 		u8 category, action;
@@ -2244,7 +2247,7 @@ static int rtw_add_beacon(struct adapter *adapter, const u8 *head, size_t head_l
 	int ret = 0;
 	u8 *pbuf = NULL;
 	unsigned int len, wps_ielen = 0;
-	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &adapter->mlmepriv;
 
 	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return -EINVAL;
@@ -2336,7 +2339,7 @@ static int cfg80211_rtw_del_station(struct wiphy *wiphy,
 	u8 updated = false;
 	struct sta_info *psta = NULL;
 	struct adapter *padapter = rtw_netdev_priv(wdev->netdev);
-	struct mlme_priv *pmlmepriv = &(padapter->mlmepriv);
+	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	const u8 *mac = params->mac;
 
@@ -2461,8 +2464,8 @@ static int _cfg80211_rtw_mgmt_tx(struct adapter *padapter, u8 tx_ch, const u8 *b
 	int ret = _FAIL;
 	bool __maybe_unused ack = true;
 	struct ieee80211_hdr *pwlanhdr;
-	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
-	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 
 	rtw_set_scan_deny(padapter, 1000);
 
diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
index c61d3d12df7fc9..6ef3a719853388 100644
--- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
+++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
@@ -122,8 +122,10 @@ static u32 sdio_init(struct dvobj_priv *dvobj)
 		goto release;
 
 	err = sdio_set_block_size(func, 512);
-	if (err)
+	if (err) {
+		sdio_disable_func(func);
 		goto release;
+	}
 
 	psdio_data->block_transfer_len = 512;
 	psdio_data->tx_block_mode = 1;
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index 8b93bfeb217bb6..1ea3dd8b8f28ca 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -13,7 +13,7 @@ static int g_hwcursor = 1;
 static int g_noaccel __ro_after_init;
 static int g_nomtrr __ro_after_init;
 static const char *g_fbmode[] = {NULL, NULL};
-static const char *g_def_fbmode = "1024x768-32@60";
+static const char * const g_def_fbmode = "1024x768-32@60";
 static char *g_settings;
 static int g_dualview __ro_after_init;
 static char *g_option;
@@ -384,7 +384,7 @@ static inline unsigned int chan_to_field(unsigned int chan,
 	return chan << bf->offset;
 }
 
-static int __maybe_unused lynxfb_suspend(struct device *dev)
+static int lynxfb_suspend(struct device *dev)
 {
 	struct fb_info *info;
 	struct sm750_dev *sm750_dev;
@@ -405,7 +405,7 @@ static int __maybe_unused lynxfb_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused lynxfb_resume(struct device *dev)
+static int lynxfb_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct fb_info *info;
@@ -479,7 +479,7 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var,
 
 	/* check if current fb's video memory big enough to hold the onscreen*/
 	request = var->xres_virtual * (var->bits_per_pixel >> 3);
-	/* defaulty crtc->channel go with par->index */
+	/* default crtc->channel go with par->index */
 
 	request = ALIGN(request, crtc->line_pad);
 	request = request * var->yres_virtual;
@@ -1094,14 +1094,14 @@ static const struct pci_device_id smi_pci_table[] = {
 
 MODULE_DEVICE_TABLE(pci, smi_pci_table);
 
-static SIMPLE_DEV_PM_OPS(lynxfb_pm_ops, lynxfb_suspend, lynxfb_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(lynxfb_pm_ops, lynxfb_suspend, lynxfb_resume);
 
 static struct pci_driver lynxfb_driver = {
 	.name =		"sm750fb",
 	.id_table =	smi_pci_table,
 	.probe =	lynxfb_pci_probe,
 	.remove =	lynxfb_pci_remove,
-	.driver.pm =	&lynxfb_pm_ops,
+	.driver.pm =	pm_sleep_ptr(&lynxfb_pm_ops),
 };
 
 static int __init lynxfb_init(void)
diff --git a/drivers/staging/sm750fb/sm750_accel.h b/drivers/staging/sm750fb/sm750_accel.h
index efceefaaafb0c1..6db54c6526dc95 100644
--- a/drivers/staging/sm750fb/sm750_accel.h
+++ b/drivers/staging/sm750fb/sm750_accel.h
@@ -195,7 +195,8 @@ int sm750_hw_fillrect(struct lynx_accel *accel,
 		      u32 color, u32 rop);
 
 /**
- * sm750_hm_copyarea
+ * sm750_hw_copyarea
+ * @accel: Acceleration device data
  * @source_base: Address of source: offset in frame buffer
  * @source_pitch: Pitch value of source surface in BYTE
  * @sx: Starting x coordinate of source surface
@@ -219,6 +220,7 @@ int sm750_hw_copyarea(struct lynx_accel *accel,
 
 /**
  * sm750_hw_imageblit
+ * @accel: Acceleration device data
  * @src_buf: pointer to start of source buffer in system memory
  * @start_bit: Mono data can start at any bit in a byte, this value should be
  *>-----      0 to 7
diff --git a/drivers/staging/vme_user/vme_tsi148.c b/drivers/staging/vme_user/vme_tsi148.c
index c695ad9b4ca265..9447c00bc013d7 100644
--- a/drivers/staging/vme_user/vme_tsi148.c
+++ b/drivers/staging/vme_user/vme_tsi148.c
@@ -26,6 +26,7 @@
 #include <linux/io.h>
 #include <linux/uaccess.h>
 #include <linux/byteorder/generic.h>
+#include <linux/string_choices.h>
 
 #include "vme.h"
 #include "vme_bridge.h"
@@ -2458,7 +2459,7 @@ static int tsi148_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 			 geoid);
 
 	dev_info(&pdev->dev, "VME Write and flush and error check is %s\n",
-		 err_chk ? "enabled" : "disabled");
+		 str_enabled_disabled(err_chk));
 
 	retval = tsi148_crcsr_init(tsi148_bridge, pdev);
 	if (retval) {
diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c
index 62ced9f5102ff0..ab9824a4852f1e 100644
--- a/drivers/target/target_core_file.c
+++ b/drivers/target/target_core_file.c
@@ -516,7 +516,7 @@ fd_do_prot_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb)
 	void *buf;
 	int rc;
 
-	buf = (void *)__get_free_page(GFP_KERNEL);
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf) {
 		pr_err("Unable to allocate FILEIO prot buf\n");
 		return -ENOMEM;
@@ -524,7 +524,7 @@ fd_do_prot_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb)
 
 	rc = fd_do_prot_fill(cmd->se_dev, lba, nolb, buf, PAGE_SIZE);
 
-	free_page((unsigned long)buf);
+	kfree(buf);
 
 	return rc;
 }
diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c
index 9ed2818c3028d5..fd1b82fc729023 100644
--- a/drivers/target/target_core_pscsi.c
+++ b/drivers/target/target_core_pscsi.c
@@ -348,7 +348,7 @@ static struct se_device *pscsi_alloc_device(struct se_hba *hba,
  * Called with struct Scsi_Host->host_lock called.
  */
 static int pscsi_create_type_disk(struct se_device *dev, struct scsi_device *sd)
-	__releases(sh->host_lock)
+	__releases(&sh->host_lock)
 {
 	struct pscsi_hba_virt *phv = dev->se_hba->hba_ptr;
 	struct pscsi_dev_virt *pdv = PSCSI_DEV(dev);
@@ -359,10 +359,10 @@ static int pscsi_create_type_disk(struct se_device *dev, struct scsi_device *sd)
 	if (scsi_device_get(sd)) {
 		pr_err("scsi_device_get() failed for %d:%d:%d:%llu\n",
 			sh->host_no, sd->channel, sd->id, sd->lun);
-		spin_unlock_irq(sh->host_lock);
+		spin_unlock_irq(&sh->host_lock);
 		return -EIO;
 	}
-	spin_unlock_irq(sh->host_lock);
+	spin_unlock_irq(&sh->host_lock);
 	/*
 	 * Claim exclusive struct block_device access to struct scsi_device
 	 * for TYPE_DISK and TYPE_ZBC using supplied udev_path
@@ -393,7 +393,7 @@ static int pscsi_create_type_disk(struct se_device *dev, struct scsi_device *sd)
  * Called with struct Scsi_Host->host_lock called.
  */
 static int pscsi_create_type_nondisk(struct se_device *dev, struct scsi_device *sd)
-	__releases(sh->host_lock)
+	__releases(&sh->host_lock)
 {
 	struct pscsi_hba_virt *phv = dev->se_hba->hba_ptr;
 	struct Scsi_Host *sh = sd->host;
@@ -402,10 +402,10 @@ static int pscsi_create_type_nondisk(struct se_device *dev, struct scsi_device *
 	if (scsi_device_get(sd)) {
 		pr_err("scsi_device_get() failed for %d:%d:%d:%llu\n",
 			sh->host_no, sd->channel, sd->id, sd->lun);
-		spin_unlock_irq(sh->host_lock);
+		spin_unlock_irq(&sh->host_lock);
 		return -EIO;
 	}
-	spin_unlock_irq(sh->host_lock);
+	spin_unlock_irq(&sh->host_lock);
 
 	ret = pscsi_add_device_to_list(dev, sd);
 	if (ret) {
@@ -491,7 +491,7 @@ static int pscsi_configure_device(struct se_device *dev)
 		}
 	}
 
-	spin_lock_irq(sh->host_lock);
+	spin_lock_irq(&sh->host_lock);
 	list_for_each_entry(sd, &sh->__devices, siblings) {
 		if ((pdv->pdv_channel_id != sd->channel) ||
 		    (pdv->pdv_target_id != sd->id) ||
@@ -524,7 +524,7 @@ static int pscsi_configure_device(struct se_device *dev)
 		}
 		return 0;
 	}
-	spin_unlock_irq(sh->host_lock);
+	spin_unlock_irq(&sh->host_lock);
 
 	pr_err("pSCSI: Unable to locate %d:%d:%d:%d\n", sh->host_no,
 		pdv->pdv_channel_id,  pdv->pdv_target_id, pdv->pdv_lun_id);
diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c
index f964a531e053ad..b70e81af867e22 100644
--- a/drivers/target/target_core_spc.c
+++ b/drivers/target/target_core_spc.c
@@ -1226,9 +1226,9 @@ static sense_reason_t spc_emulate_request_sense(struct se_cmd *cmd)
 {
 	unsigned char *cdb = cmd->t_task_cdb;
 	unsigned char *rbuf;
-	u8 ua_asc = 0, ua_ascq = 0;
 	unsigned char buf[SE_SENSE_BUF];
 	bool desc_format = target_sense_desc_format(cmd->se_dev);
+	u16 sense_code = NO_ADDITIONAL_SENSE_INFORMATION;
 
 	memset(buf, 0, SE_SENSE_BUF);
 
@@ -1242,11 +1242,12 @@ static sense_reason_t spc_emulate_request_sense(struct se_cmd *cmd)
 	if (!rbuf)
 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
 
-	if (!core_scsi3_ua_clear_for_request_sense(cmd, &ua_asc, &ua_ascq))
-		scsi_build_sense_buffer(desc_format, buf, UNIT_ATTENTION,
-					ua_asc, ua_ascq);
+	if (!core_scsi3_ua_clear_for_request_sense(cmd, &sense_code))
+		scsi_set_sense_buffer(desc_format, buf, UNIT_ATTENTION,
+				      sense_code);
 	else
-		scsi_build_sense_buffer(desc_format, buf, NO_SENSE, 0x0, 0x0);
+		scsi_set_sense_buffer(desc_format, buf, NO_SENSE,
+				      NO_ADDITIONAL_SENSE_INFORMATION);
 
 	memcpy(rbuf, buf, min_t(u32, sizeof(buf), cmd->data_length));
 	transport_kunmap_data_sg(cmd);
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index dcfe945949167d..ba208def8cb599 100644
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -3555,7 +3555,8 @@ static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason)
 	const struct sense_detail *sd;
 	u8 *buffer = cmd->sense_buffer;
 	int r = (__force int)reason;
-	u8 key, asc, ascq;
+	u16 code;
+	u8 key;
 	bool desc_format = target_sense_desc_format(cmd->se_dev);
 
 	if (r < ARRAY_SIZE(sense_detail_table) && sense_detail_table[r].key)
@@ -3566,21 +3567,19 @@ static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason)
 
 	key = sd->key;
 	if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) {
-		if (!core_scsi3_ua_for_check_condition(cmd, &key, &asc,
-						       &ascq)) {
+		if (!core_scsi3_ua_for_check_condition(cmd, &key, &code)) {
 			cmd->scsi_status = SAM_STAT_BUSY;
 			return;
 		}
 	} else {
 		WARN_ON_ONCE(sd->asc == 0);
-		asc = sd->asc;
-		ascq = sd->ascq;
+		code = scsi_sense_code(sd->asc, sd->ascq);
 	}
 
 	cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
 	cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
 	cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER;
-	scsi_build_sense_buffer(desc_format, buffer, key, asc, ascq);
+	scsi_set_sense_buffer(desc_format, buffer, key, code);
 	if (sd->add_sense_info)
 		WARN_ON_ONCE(scsi_set_sense_information(buffer,
 							cmd->scsi_sense_length,
diff --git a/drivers/target/target_core_ua.c b/drivers/target/target_core_ua.c
index 4276690fb6cbd9..ea97b05d47ab7a 100644
--- a/drivers/target/target_core_ua.c
+++ b/drivers/target/target_core_ua.c
@@ -187,11 +187,10 @@ void core_scsi3_ua_release_all(
 
 /*
  * Dequeue a unit attention from the unit attention list. This function
- * returns true if the dequeuing succeeded and if *@key, *@asc and *@ascq have
+ * returns true if the dequeuing succeeded and if *@key and *@code have
  * been set.
  */
-bool core_scsi3_ua_for_check_condition(struct se_cmd *cmd, u8 *key, u8 *asc,
-				       u8 *ascq)
+bool core_scsi3_ua_for_check_condition(struct se_cmd *cmd, u8 *key, u16 *code)
 {
 	struct se_device *dev = cmd->se_dev;
 	struct se_dev_entry *deve;
@@ -214,8 +213,7 @@ bool core_scsi3_ua_for_check_condition(struct se_cmd *cmd, u8 *key, u8 *asc,
 	if (!deve) {
 		rcu_read_unlock();
 		*key = ILLEGAL_REQUEST;
-		*asc = 0x25; /* LOGICAL UNIT NOT SUPPORTED */
-		*ascq = 0;
+		*code = LU_NOT_SUPPORTED;
 		return true;
 	}
 	*key = UNIT_ATTENTION;
@@ -232,8 +230,7 @@ bool core_scsi3_ua_for_check_condition(struct se_cmd *cmd, u8 *key, u8 *asc,
 		 * clearing it.
 		 */
 		if (!dev_ua_intlck_clear) {
-			*asc = ua->ua_asc;
-			*ascq = ua->ua_ascq;
+			*code = scsi_sense_code(ua->ua_asc, ua->ua_ascq);
 			break;
 		}
 		/*
@@ -242,8 +239,7 @@ bool core_scsi3_ua_for_check_condition(struct se_cmd *cmd, u8 *key, u8 *asc,
 		 * (head of the list) in the outgoing CHECK_CONDITION + sense.
 		 */
 		if (head) {
-			*asc = ua->ua_asc;
-			*ascq = ua->ua_ascq;
+			*code = scsi_sense_code(ua->ua_asc, ua->ua_ascq);
 			head = 0;
 		}
 		list_del(&ua->ua_nacl_list);
@@ -258,15 +254,13 @@ bool core_scsi3_ua_for_check_condition(struct se_cmd *cmd, u8 *key, u8 *asc,
 		nacl->se_tpg->se_tpg_tfo->fabric_name,
 		dev_ua_intlck_clear ? "Releasing" : "Reporting",
 		dev->dev_attrib.emulate_ua_intlck_ctrl,
-		cmd->orig_fe_lun, cmd->t_task_cdb[0], *asc, *ascq);
+		cmd->orig_fe_lun, cmd->t_task_cdb[0],
+		scsi_sense_code_asc(*code), scsi_sense_code_ascq(*code));
 
 	return head == 0;
 }
 
-int core_scsi3_ua_clear_for_request_sense(
-	struct se_cmd *cmd,
-	u8 *asc,
-	u8 *ascq)
+int core_scsi3_ua_clear_for_request_sense(struct se_cmd *cmd, u16 *code)
 {
 	struct se_dev_entry *deve;
 	struct se_session *sess = cmd->se_sess;
@@ -304,8 +298,7 @@ int core_scsi3_ua_clear_for_request_sense(
 	spin_lock(&deve->ua_lock);
 	list_for_each_entry_safe(ua, ua_p, &deve->ua_list, ua_nacl_list) {
 		if (head) {
-			*asc = ua->ua_asc;
-			*ascq = ua->ua_ascq;
+			*code = scsi_sense_code(ua->ua_asc, ua->ua_ascq);
 			head = 0;
 		}
 		list_del(&ua->ua_nacl_list);
@@ -317,7 +310,8 @@ int core_scsi3_ua_clear_for_request_sense(
 	pr_debug("[%s]: Released UNIT ATTENTION condition, mapped"
 		" LUN: %llu, got REQUEST_SENSE reported ASC: 0x%02x,"
 		" ASCQ: 0x%02x\n", nacl->se_tpg->se_tpg_tfo->fabric_name,
-		cmd->orig_fe_lun, *asc, *ascq);
+		cmd->orig_fe_lun, scsi_sense_code_asc(*code),
+		scsi_sense_code_ascq(*code));
 
 	return (head) ? -EPERM : 0;
 }
diff --git a/drivers/target/target_core_ua.h b/drivers/target/target_core_ua.h
index 76487c9be09015..6b636c13fee22c 100644
--- a/drivers/target/target_core_ua.h
+++ b/drivers/target/target_core_ua.h
@@ -37,9 +37,8 @@ extern sense_reason_t target_scsi3_ua_check(struct se_cmd *);
 extern int core_scsi3_ua_allocate(struct se_dev_entry *, u8, u8);
 extern void target_ua_allocate_lun(struct se_node_acl *, u32, u8, u8);
 extern void core_scsi3_ua_release_all(struct se_dev_entry *);
-extern bool core_scsi3_ua_for_check_condition(struct se_cmd *, u8 *, u8 *,
-					      u8 *);
-extern int core_scsi3_ua_clear_for_request_sense(struct se_cmd *,
-						u8 *, u8 *);
+extern bool core_scsi3_ua_for_check_condition(struct se_cmd *cmd, u8 *key,
+					      u16 *code);
+extern int core_scsi3_ua_clear_for_request_sense(struct se_cmd *cmd, u16 *code);
 
 #endif /* TARGET_CORE_UA_H */
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index edc2afd5f4ee2c..425ad1246a7595 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -3006,10 +3006,7 @@ static ssize_t tcmu_tmr_notification_store(struct config_item *item,
 	if (val > 1)
 		return -EINVAL;
 
-	if (val)
-		set_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
-	else
-		clear_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
+	assign_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags, val);
 	return count;
 }
 CONFIGFS_ATTR(tcmu_, tmr_notification);
diff --git a/drivers/tee/optee/Kconfig b/drivers/tee/optee/Kconfig
index 50d2051f7f20b7..891dac63cab85a 100644
--- a/drivers/tee/optee/Kconfig
+++ b/drivers/tee/optee/Kconfig
@@ -5,6 +5,7 @@ config OPTEE
 	depends on HAVE_ARM_SMCCC
 	depends on MMU
 	depends on RPMB || !RPMB
+	depends on ARM_FFA_TRANSPORT || !ARM_FFA_TRANSPORT
 	help
 	  This implements the OP-TEE Trusted Execution Environment (TEE)
 	  driver.
diff --git a/drivers/tee/optee/Makefile b/drivers/tee/optee/Makefile
index ad7049c1c10721..183cdde1ac045c 100644
--- a/drivers/tee/optee/Makefile
+++ b/drivers/tee/optee/Makefile
@@ -7,8 +7,8 @@ optee-objs += rpc.o
 optee-objs += protmem.o
 optee-objs += supp.o
 optee-objs += device.o
-optee-objs += smc_abi.o
-optee-objs += ffa_abi.o
+optee-$(CONFIG_HAVE_ARM_SMCCC) += smc_abi.o
+optee-$(CONFIG_ARM_FFA_TRANSPORT) += ffa_abi.o
 
 # for tracing framework to find optee_trace.h
 CFLAGS_smc_abi.o := -I$(src)
diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c
index 633715b98625cd..723b929d02bd3d 100644
--- a/drivers/tee/optee/ffa_abi.c
+++ b/drivers/tee/optee/ffa_abi.c
@@ -1123,14 +1123,6 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 
 	optee_set_dev_group(optee);
 
-	rc = tee_device_register(optee->teedev);
-	if (rc)
-		goto err_unreg_supp_teedev;
-
-	rc = tee_device_register(optee->supp_teedev);
-	if (rc)
-		goto err_unreg_supp_teedev;
-
 	rc = rhashtable_init(&optee->ffa.global_ids, &shm_rhash_params);
 	if (rc)
 		goto err_unreg_supp_teedev;
@@ -1159,6 +1151,14 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 	if (optee_ffa_protmem_pool_init(optee, sec_caps))
 		pr_info("Protected memory service not available\n");
 
+	rc = tee_device_register(optee->teedev);
+	if (rc)
+		goto err_unregister_devices;
+
+	rc = tee_device_register(optee->supp_teedev);
+	if (rc)
+		goto err_unregister_devices;
+
 	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
 	if (rc)
 		goto err_unregister_devices;
@@ -1212,14 +1212,10 @@ static struct ffa_driver optee_ffa_driver = {
 
 int optee_ffa_abi_register(void)
 {
-	if (IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT))
-		return ffa_register(&optee_ffa_driver);
-	else
-		return -EOPNOTSUPP;
+	return ffa_register(&optee_ffa_driver);
 }
 
 void optee_ffa_abi_unregister(void)
 {
-	if (IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT))
-		ffa_unregister(&optee_ffa_driver);
+	ffa_unregister(&optee_ffa_driver);
 }
diff --git a/drivers/tee/optee/notif.c b/drivers/tee/optee/notif.c
index 6e85f2f5c516f8..68014222d7be72 100644
--- a/drivers/tee/optee/notif.c
+++ b/drivers/tee/optee/notif.c
@@ -5,7 +5,6 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/arm-smccc.h>
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h
index aefe1e6f568915..02d6f79df407bc 100644
--- a/drivers/tee/optee/optee_private.h
+++ b/drivers/tee/optee/optee_private.h
@@ -6,7 +6,6 @@
 #ifndef OPTEE_PRIVATE_H
 #define OPTEE_PRIVATE_H
 
-#include <linux/arm-smccc.h>
 #include <linux/notifier.h>
 #include <linux/rhashtable.h>
 #include <linux/rpmb.h>
@@ -15,6 +14,10 @@
 #include <linux/types.h>
 #include "optee_msg.h"
 
+#ifdef CONFIG_HAVE_ARM_SMCCC
+#include <linux/arm-smccc.h>
+#endif
+
 #define DRIVER_NAME "optee"
 
 #define OPTEE_MAX_ARG_SIZE	1024
@@ -42,10 +45,12 @@
  */
 #define OPTEE_DEFAULT_MAX_NOTIF_VALUE	255
 
+#ifdef CONFIG_HAVE_ARM_SMCCC
 typedef void (optee_invoke_fn)(unsigned long, unsigned long, unsigned long,
 				unsigned long, unsigned long, unsigned long,
 				unsigned long, unsigned long,
 				struct arm_smccc_res *);
+#endif
 
 /**
  * struct optee_call_waiter - TEE entry may need to wait for a free TEE thread
@@ -119,6 +124,7 @@ struct optee_supp {
 	struct completion reqs_c;
 };
 
+#ifdef CONFIG_HAVE_ARM_SMCCC
 /**
  * struct optee_pcpu - per cpu notif private struct passed to work functions
  * @optee:	optee device reference
@@ -149,7 +155,9 @@ struct optee_smc {
 	struct work_struct notif_pcpu_work;
 	unsigned int notif_cpuhp_state;
 };
+#endif
 
+#if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT)
 /**
  * struct optee_ffa -  FFA communication struct
  * @ffa_dev:		FFA device, contains the destination id, the id of
@@ -170,6 +178,7 @@ struct optee_ffa {
 	struct workqueue_struct *notif_wq;
 	struct work_struct notif_work;
 };
+#endif
 
 struct optee;
 
@@ -257,8 +266,12 @@ struct optee {
 	const struct optee_ops *ops;
 	struct tee_context *ctx;
 	union {
+#ifdef CONFIG_HAVE_ARM_SMCCC
 		struct optee_smc smc;
+#endif
+#if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT)
 		struct optee_ffa ffa;
+#endif
 	};
 	struct optee_shm_arg_cache shm_arg_cache;
 	struct optee_call_queue call_queue;
@@ -290,6 +303,7 @@ struct optee_context_data {
 	struct list_head sess_list;
 };
 
+#ifdef CONFIG_HAVE_ARM_SMCCC
 struct optee_rpc_param {
 	u32	a0;
 	u32	a1;
@@ -300,6 +314,7 @@ struct optee_rpc_param {
 	u32	a6;
 	u32	a7;
 };
+#endif
 
 /* Holds context that is preserved during one STD call */
 struct optee_call_ctx {
@@ -422,9 +437,31 @@ static inline void reg_pair_from_64(u32 *reg0, u32 *reg1, u64 val)
 }
 
 /* Registration of the ABIs */
+#ifdef CONFIG_HAVE_ARM_SMCCC
 int optee_smc_abi_register(void);
 void optee_smc_abi_unregister(void);
+#else
+static inline int optee_smc_abi_register(void)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void optee_smc_abi_unregister(void)
+{
+}
+#endif
+#if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT)
 int optee_ffa_abi_register(void);
 void optee_ffa_abi_unregister(void);
+#else
+static inline int optee_ffa_abi_register(void)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void optee_ffa_abi_unregister(void)
+{
+}
+#endif
 
 #endif /*OPTEE_PRIVATE_H*/
diff --git a/drivers/tee/optee/smc_abi.c b/drivers/tee/optee/smc_abi.c
index b8a2bdac3208f9..51624443359bfb 100644
--- a/drivers/tee/optee/smc_abi.c
+++ b/drivers/tee/optee/smc_abi.c
@@ -1849,14 +1849,6 @@ static int optee_probe(struct platform_device *pdev)
 
 	optee_set_dev_group(optee);
 
-	rc = tee_device_register(optee->teedev);
-	if (rc)
-		goto err_unreg_supp_teedev;
-
-	rc = tee_device_register(optee->supp_teedev);
-	if (rc)
-		goto err_unreg_supp_teedev;
-
 	optee_cq_init(&optee->call_queue, thread_count);
 	optee_supp_init(&optee->supp);
 	optee->smc.memremaped_shm = memremaped_shm;
@@ -1916,6 +1908,14 @@ static int optee_probe(struct platform_device *pdev)
 	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
 		pr_info("dynamic shared memory is enabled\n");
 
+	rc = tee_device_register(optee->teedev);
+	if (rc)
+		goto err_disable_shm_cache;
+
+	rc = tee_device_register(optee->supp_teedev);
+	if (rc)
+		goto err_disable_shm_cache;
+
 	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
 	if (rc)
 		goto err_disable_shm_cache;
@@ -1942,7 +1942,6 @@ static int optee_probe(struct platform_device *pdev)
 	optee_shm_arg_cache_uninit(optee);
 	optee_supp_uninit(&optee->supp);
 	mutex_destroy(&optee->call_queue.mutex);
-err_unreg_supp_teedev:
 	tee_device_unregister(optee->supp_teedev);
 err_unreg_teedev:
 	tee_device_unregister(optee->teedev);
diff --git a/drivers/tee/qcomtee/async.c b/drivers/tee/qcomtee/async.c
index 31bff4309e6721..5849e51612033b 100644
--- a/drivers/tee/qcomtee/async.c
+++ b/drivers/tee/qcomtee/async.c
@@ -97,10 +97,10 @@ static void qcomtee_get_async_buffer(struct qcomtee_object_invoke_ctx *oic,
 /**
  * async_release() - Process QTEE async release requests.
  * @oic: context used for the current invocation.
- * @msg: async message for object release.
+ * @async_msg: async message for object release.
  * @size: size of the async buffer available.
  *
- * Return: Size of the outbound buffer used when processing @msg.
+ * Return: Size of the outbound buffer used when processing @async_msg.
  */
 static size_t async_release(struct qcomtee_object_invoke_ctx *oic,
 			    struct qcomtee_async_msg_hdr *async_msg,
diff --git a/drivers/tee/qcomtee/qcomtee_msg.h b/drivers/tee/qcomtee/qcomtee_msg.h
index 878f70178a5b6f..5d7b21fdd36880 100644
--- a/drivers/tee/qcomtee/qcomtee_msg.h
+++ b/drivers/tee/qcomtee/qcomtee_msg.h
@@ -112,7 +112,7 @@ union qcomtee_msg_arg {
 
 /**
  * struct qcomtee_msg_object_invoke - Direct object invocation message.
- * @ctx: object ID hosted in QTEE.
+ * @cxt: object ID hosted in QTEE.
  * @op: operation for the object.
  * @counts: number of different types of arguments in @args.
  * @args: array of arguments.
diff --git a/drivers/tee/qcomtee/qcomtee_object.h b/drivers/tee/qcomtee/qcomtee_object.h
index 8b4401ecad48c2..d5de02dcef3b96 100644
--- a/drivers/tee/qcomtee/qcomtee_object.h
+++ b/drivers/tee/qcomtee/qcomtee_object.h
@@ -74,6 +74,7 @@ enum qcomtee_object_type {
  * @QCOMTEE_ARG_TYPE_OO: output object (OO).
  * @QCOMTEE_ARG_TYPE_IB: input buffer (IB).
  * @QCOMTEE_ARG_TYPE_IO: input object (IO).
+ * @QCOMTEE_ARG_TYPE_NR: number of argument types.
  *
  * Use the invalid type to specify the end of the argument array.
  */
diff --git a/drivers/thermal/airoha_thermal.c b/drivers/thermal/airoha_thermal.c
index 77b3a1af271fdb..ea3e2649c280c9 100644
--- a/drivers/thermal/airoha_thermal.c
+++ b/drivers/thermal/airoha_thermal.c
@@ -528,7 +528,7 @@ static int en7581_thermal_probe(struct platform_device *pdev,
 	struct device_node *chip_scu_np;
 	struct device *dev = &pdev->dev;
 	void __iomem *base;
-	int i, irq, ret;
+	int i;
 
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
@@ -567,16 +567,6 @@ static int en7581_thermal_probe(struct platform_device *pdev,
 	of_address_to_resource(chip_scu_np, 0, &priv->scu_adc_res);
 	of_node_put(chip_scu_np);
 
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
-					en7581_thermal_irq, IRQF_ONESHOT,
-					pdev->name, priv);
-	if (ret)
-		return ret;
-
 	en7581_thermal_setup_monitor(priv);
 	en7581_thermal_setup_adc_val(dev, priv);
 
@@ -586,6 +576,18 @@ static int en7581_thermal_probe(struct platform_device *pdev,
 static int en7581_thermal_post_probe(struct platform_device *pdev)
 {
 	struct airoha_thermal_priv *priv = platform_get_drvdata(pdev);
+	int irq, ret;
+
+	/* After zone registration: the handler uses priv->tz */
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+					en7581_thermal_irq, IRQF_ONESHOT,
+					pdev->name, priv);
+	if (ret)
+		return ret;
 
 	/* Enable LOW and HIGH interrupt (if supported) */
 	regmap_write(priv->map, EN7581_TEMPMONINT,
diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c
index 768859a7aed059..d75b42fb955edf 100644
--- a/drivers/thermal/cpufreq_cooling.c
+++ b/drivers/thermal/cpufreq_cooling.c
@@ -488,12 +488,12 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev,
 	frequency = get_state_freq(cpufreq_cdev, state);
 
 	ret = freq_qos_update_request(&cpufreq_cdev->qos_req, frequency);
-	if (ret >= 0) {
-		cpufreq_cdev->cpufreq_state = state;
-		ret = 0;
-	}
+	if (ret < 0)
+		return ret;
+
+	cpufreq_cdev->cpufreq_state = state;
 
-	return ret;
+	return 0;
 }
 
 /**
@@ -661,8 +661,8 @@ of_cpufreq_cooling_register(struct cpufreq_policy *policy)
 
 		cdev = __cpufreq_cooling_register(np, policy, em);
 		if (IS_ERR(cdev)) {
-			pr_err("cpufreq_cooling: cpu%d failed to register as cooling device: %ld\n",
-			       policy->cpu, PTR_ERR(cdev));
+			pr_err("cpufreq_cooling: cpu%d failed to register as cooling device: %pe\n",
+			       policy->cpu, cdev);
 			cdev = NULL;
 		}
 	}
diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c
index 0330a811283233..9ada52cbf7dd73 100644
--- a/drivers/thermal/devfreq_cooling.c
+++ b/drivers/thermal/devfreq_cooling.c
@@ -156,8 +156,8 @@ static unsigned long get_voltage(struct devfreq *df, unsigned long freq)
 		opp = dev_pm_opp_find_freq_exact(dev, freq, false);
 
 	if (IS_ERR(opp)) {
-		dev_err_ratelimited(dev, "Failed to find OPP for frequency %lu: %ld\n",
-				    freq, PTR_ERR(opp));
+		dev_err_ratelimited(dev, "Failed to find OPP for frequency %lu: %pe\n",
+				    freq, opp);
 		return 0;
 	}
 
diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c
index 37f2e22a999e64..b5c254187628bd 100644
--- a/drivers/thermal/gov_power_allocator.c
+++ b/drivers/thermal/gov_power_allocator.c
@@ -660,10 +660,15 @@ static void power_allocator_update_tz(struct thermal_zone_device *tz,
 				      enum thermal_notify_event reason)
 {
 	struct power_allocator_params *params = tz->governor_data;
-	const struct thermal_trip_desc *td = trip_to_trip_desc(params->trip_max);
+	const struct thermal_trip_desc *td;
 	struct thermal_instance *instance;
 	int num_actors = 0;
 
+	if (!params->trip_max)
+		return;
+
+	td = trip_to_trip_desc(params->trip_max);
+
 	switch (reason) {
 	case THERMAL_TZ_BIND_CDEV:
 	case THERMAL_TZ_UNBIND_CDEV:
diff --git a/drivers/thermal/gov_step_wise.c b/drivers/thermal/gov_step_wise.c
index ea277c466d8d2e..4fa4377f0d4286 100644
--- a/drivers/thermal/gov_step_wise.c
+++ b/drivers/thermal/gov_step_wise.c
@@ -65,14 +65,12 @@ static unsigned long get_target_state(struct thermal_instance *instance,
 				     min(instance->lower + 1, instance->upper),
 				     instance->upper);
 	} else if (trend == THERMAL_TREND_DROPPING) {
-		if (cur_state <= instance->lower)
-			return THERMAL_NO_TARGET;
-
 		/*
-		 * If 'throttle' is false, no mitigation is necessary, so
-		 * request the lower state for this instance.
+		 * If 'throttle' is false, no mitigation is necessary and
+		 * passive polling is already deactivated, so clear this
+		 * instance state by returning THERMAL_NO_TARGET.
 		 */
-		return instance->lower;
+		return THERMAL_NO_TARGET;
 	}
 
 	return instance->target;
diff --git a/drivers/thermal/imx91_thermal.c b/drivers/thermal/imx91_thermal.c
index 274eee3031429e..196ee20fce1e3e 100644
--- a/drivers/thermal/imx91_thermal.c
+++ b/drivers/thermal/imx91_thermal.c
@@ -225,7 +225,7 @@ static int imx91_tmu_change_mode(struct thermal_zone_device *tz, enum thermal_de
 	return 0;
 }
 
-static struct thermal_zone_device_ops tmu_tz_ops = {
+static const struct thermal_zone_device_ops tmu_tz_ops = {
 	.get_temp = imx91_tmu_get_temp,
 	.change_mode = imx91_tmu_change_mode,
 	.set_trips = imx91_tmu_set_trips,
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
index f80dbe2ca7e477..850bfe7c1ede05 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.c
@@ -179,17 +179,21 @@ static int proc_thermal_get_zone_temp(struct thermal_zone_device *zone,
 {
 	int cpu;
 	int curr_temp, ret;
-
-	*temp = 0;
+	bool temp_valid = false;
 
 	for_each_online_cpu(cpu) {
 		ret = intel_tcc_get_temp(cpu, &curr_temp, false);
 		if (ret < 0)
 			return ret;
-		if (!*temp || curr_temp > *temp)
+		if (!temp_valid || curr_temp > *temp) {
 			*temp = curr_temp;
+			temp_valid = true;
+		}
 	}
 
+	if (!temp_valid)
+		return -ENODATA;
+
 	*temp *= 1000;
 
 	return 0;
@@ -291,10 +295,8 @@ int proc_thermal_add(struct device *dev, struct proc_thermal_device *proc_priv)
 	}
 
 	proc_priv->int340x_zone = int340x_thermal_zone_add(adev, get_temp);
-	if (IS_ERR(proc_priv->int340x_zone)) {
+	if (IS_ERR(proc_priv->int340x_zone))
 		return PTR_ERR(proc_priv->int340x_zone);
-	} else
-		ret = 0;
 
 	ret = acpi_install_notify_handler(adev->handle, ACPI_DEVICE_NOTIFY,
 					  proc_thermal_notify,
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c
index 91f291627132aa..e161b95828e878 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_soc_slider.c
@@ -66,15 +66,16 @@ static int slider_def_balance_set(const char *arg, const struct kernel_param *kp
 	guard(mutex)(&slider_param_lock);
 
 	ret = kstrtou8(arg, 16, &slider_val);
-	if (!ret) {
-		if (slider_val <= slider_values[SOC_POWER_SLIDER_PERFORMANCE] ||
-		    slider_val >= slider_values[SOC_POWER_SLIDER_POWERSAVE])
-			return -EINVAL;
+	if (ret)
+		return ret;
 
-		slider_balanced_param = slider_val;
-	}
+	if (slider_val <= slider_values[SOC_POWER_SLIDER_PERFORMANCE] ||
+	    slider_val >= slider_values[SOC_POWER_SLIDER_POWERSAVE])
+		return -EINVAL;
 
-	return ret;
+	slider_balanced_param = slider_val;
+
+	return 0;
 }
 
 static int slider_def_balance_get(char *buf, const struct kernel_param *kp)
@@ -101,14 +102,15 @@ static int slider_def_offset_set(const char *arg, const struct kernel_param *kp)
 	guard(mutex)(&slider_param_lock);
 
 	ret = kstrtou8(arg, 16, &offset);
-	if (!ret) {
-		if (offset > SOC_SLIDER_VALUE_MAXIMUM)
-			return -EINVAL;
+	if (ret)
+		return ret;
 
-		slider_offset = offset;
-	}
+	if (offset > SOC_SLIDER_VALUE_MAXIMUM)
+		return -EINVAL;
 
-	return ret;
+	slider_offset = offset;
+
+	return 0;
 }
 
 static int slider_def_offset_get(char *buf, const struct kernel_param *kp)
diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c
index bd7fd98dc310e4..1ffcb0a699e57d 100644
--- a/drivers/thermal/intel/intel_powerclamp.c
+++ b/drivers/thermal/intel/intel_powerclamp.c
@@ -94,7 +94,7 @@ static int duration_set(const char *arg, const struct kernel_param *kp)
 	}
 
 	mutex_lock(&powerclamp_lock);
-	duration = clamp(new_duration, 6ul, 25ul) * 1000;
+	duration = new_duration * 1000;
 	mutex_unlock(&powerclamp_lock);
 exit:
 
@@ -103,13 +103,9 @@ static int duration_set(const char *arg, const struct kernel_param *kp)
 
 static int duration_get(char *buf, const struct kernel_param *kp)
 {
-	int ret;
-
-	mutex_lock(&powerclamp_lock);
-	ret = sysfs_emit(buf, "%d\n", duration / 1000);
-	mutex_unlock(&powerclamp_lock);
+	guard(mutex)(&powerclamp_lock);
 
-	return ret;
+	return sysfs_emit(buf, "%u\n", duration / 1000);
 }
 
 static const struct kernel_param_ops duration_ops = {
@@ -143,12 +139,9 @@ static int allocate_copy_idle_injection_mask(const struct cpumask *copy_mask)
 }
 
 /* Return true if the cpumask and idle percent combination is invalid */
-static bool check_invalid(cpumask_var_t mask, u8 idle)
+static bool check_invalid(const struct cpumask *mask, u8 idle)
 {
-	if (cpumask_equal(cpu_present_mask, mask) && idle > MAX_ALL_CPU_IDLE)
-		return true;
-
-	return false;
+	return cpumask_equal(cpu_present_mask, mask) && idle > MAX_ALL_CPU_IDLE;
 }
 
 static int cpumask_set(const char *arg, const struct kernel_param *kp)
@@ -214,41 +207,32 @@ MODULE_PARM_DESC(cpumask, "Mask of CPUs to use for idle injection.");
 static int max_idle_set(const char *arg, const struct kernel_param *kp)
 {
 	u8 new_max_idle;
-	int ret = 0;
+	int ret;
 
-	mutex_lock(&powerclamp_lock);
+	guard(mutex)(&powerclamp_lock);
 
 	/* Can't set mask when cooling device is in use */
-	if (powerclamp_data.clamping) {
-		ret = -EAGAIN;
-		goto skip_limit_set;
-	}
+	if (powerclamp_data.clamping)
+		return -EAGAIN;
 
 	ret = kstrtou8(arg, 10, &new_max_idle);
 	if (ret)
-		goto skip_limit_set;
+		return ret;
 
-	if (new_max_idle > MAX_TARGET_RATIO) {
-		ret = -EINVAL;
-		goto skip_limit_set;
-	}
+	if (new_max_idle > MAX_TARGET_RATIO)
+		return -EINVAL;
 
 	if (!cpumask_available(idle_injection_cpu_mask)) {
 		ret = allocate_copy_idle_injection_mask(cpu_present_mask);
 		if (ret)
-			goto skip_limit_set;
+			return ret;
 	}
 
-	if (check_invalid(idle_injection_cpu_mask, new_max_idle)) {
-		ret = -EINVAL;
-		goto skip_limit_set;
-	}
+	if (check_invalid(idle_injection_cpu_mask, new_max_idle))
+		return -EINVAL;
 
 	max_idle = new_max_idle;
 
-skip_limit_set:
-	mutex_unlock(&powerclamp_lock);
-
 	return ret;
 }
 
@@ -289,9 +273,10 @@ static int window_size_set(const char *arg, const struct kernel_param *kp)
 		pr_err("Out of recommended window size %lu, between 2-10\n",
 			new_window_size);
 		ret = -EINVAL;
+		goto exit_win;
 	}
 
-	window_size = clamp(new_window_size, 2ul, 10ul);
+	window_size = new_window_size;
 	smp_mb();
 
 exit_win:
@@ -536,23 +521,17 @@ static struct idle_inject_device *ii_dev;
  */
 static bool idle_inject_update(void)
 {
-	bool update = false;
-
 	/* We can't sleep in this callback */
 	if (!mutex_trylock(&powerclamp_lock))
 		return true;
 
 	if (!(powerclamp_data.count % powerclamp_data.window_size_now)) {
+		unsigned int runtime;
 
 		should_skip = powerclamp_adjust_controls(powerclamp_data.target_ratio,
 							 powerclamp_data.guard,
 							 powerclamp_data.window_size_now);
-		update = true;
-	}
-
-	if (update) {
-		unsigned int runtime = get_run_time();
-
+		runtime = get_run_time();
 		idle_inject_set_duration(ii_dev, runtime, duration);
 	}
 
@@ -560,10 +539,7 @@ static bool idle_inject_update(void)
 
 	mutex_unlock(&powerclamp_lock);
 
-	if (should_skip)
-		return false;
-
-	return true;
+	return !should_skip;
 }
 
 /* This function starts idle injection by calling idle_inject_start() */
diff --git a/drivers/thermal/intel/intel_tcc_cooling.c b/drivers/thermal/intel/intel_tcc_cooling.c
index 52ea4f7fc9f917..75f6fbec25302e 100644
--- a/drivers/thermal/intel/intel_tcc_cooling.c
+++ b/drivers/thermal/intel/intel_tcc_cooling.c
@@ -69,6 +69,7 @@ static const struct x86_cpu_id tcc_ids[] __initconst = {
 	X86_MATCH_VFM(INTEL_ARROWLAKE_U, NULL),
 	X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL),
 	X86_MATCH_VFM(INTEL_PANTHERLAKE_L, NULL),
+	X86_MATCH_VFM(INTEL_PANTHERLAKE_R, NULL),
 	X86_MATCH_VFM(INTEL_WILDCATLAKE_L, NULL),
 	X86_MATCH_VFM(INTEL_NOVALAKE, NULL),
 	X86_MATCH_VFM(INTEL_NOVALAKE_L, NULL),
diff --git a/drivers/thermal/khadas_mcu_fan.c b/drivers/thermal/khadas_mcu_fan.c
index 21b3d0a71bd0df..1da639d0549717 100644
--- a/drivers/thermal/khadas_mcu_fan.c
+++ b/drivers/thermal/khadas_mcu_fan.c
@@ -13,34 +13,33 @@
 #include <linux/regmap.h>
 #include <linux/sysfs.h>
 #include <linux/thermal.h>
+#include <linux/regulator/consumer.h>
+#include <linux/minmax.h>
 
-#define MAX_LEVEL 3
+#define MAX_LEVEL 5
 
 struct khadas_mcu_fan_ctx {
 	struct khadas_mcu *mcu;
+	unsigned int fan_reg;
 	unsigned int level;
+	const unsigned int *levels;
+	unsigned int nlevels;
 	struct thermal_cooling_device *cdev;
+	struct regulator *power;
 };
 
 static int khadas_mcu_fan_set_level(struct khadas_mcu_fan_ctx *ctx,
 				    unsigned int level)
 {
-	int ret;
-
-	ret = regmap_write(ctx->mcu->regmap, KHADAS_MCU_CMD_FAN_STATUS_CTRL_REG,
-			   level);
-	if (ret)
-		return ret;
-
-	ctx->level = level;
-
-	return 0;
+	return regmap_write(ctx->mcu->regmap, ctx->fan_reg, level);
 }
 
 static int khadas_mcu_fan_get_max_state(struct thermal_cooling_device *cdev,
 					unsigned long *state)
 {
-	*state = MAX_LEVEL;
+	struct khadas_mcu_fan_ctx *ctx = cdev->devdata;
+
+	*state = min_t(unsigned int, MAX_LEVEL, ctx->nlevels - 1);
 
 	return 0;
 }
@@ -60,14 +59,21 @@ khadas_mcu_fan_set_cur_state(struct thermal_cooling_device *cdev,
 			     unsigned long state)
 {
 	struct khadas_mcu_fan_ctx *ctx = cdev->devdata;
+	int ret;
 
-	if (state > MAX_LEVEL)
+	if (state > MAX_LEVEL || state >= ctx->nlevels)
 		return -EINVAL;
 
 	if (state == ctx->level)
 		return 0;
 
-	return khadas_mcu_fan_set_level(ctx, state);
+	ret = khadas_mcu_fan_set_level(ctx, ctx->levels[state]);
+	if (ret)
+		return ret;
+
+	ctx->level = state;
+
+	return 0;
 }
 
 static const struct thermal_cooling_device_ops khadas_mcu_fan_cooling_ops = {
@@ -76,22 +82,71 @@ static const struct thermal_cooling_device_ops khadas_mcu_fan_cooling_ops = {
 	.set_cur_state = khadas_mcu_fan_set_cur_state,
 };
 
+static void khadas_mcu_fan_regulator_disable(void *data)
+{
+	struct regulator *power = data;
+
+	regulator_disable(power);
+}
+
+static const unsigned int khadas_mcu_fan_levels[] = { 0, 1, 2, 3 };
+
+static const unsigned int khadas_mcu_vim4_fan_levels[] = { 0, 30, 40, 55, 75, 100 };
+
 static int khadas_mcu_fan_probe(struct platform_device *pdev)
 {
+	const struct platform_device_id *id = platform_get_device_id(pdev);
 	struct khadas_mcu *mcu = dev_get_drvdata(pdev->dev.parent);
 	struct thermal_cooling_device *cdev;
 	struct device *dev = &pdev->dev;
 	struct khadas_mcu_fan_ctx *ctx;
 	int ret;
 
+
+	if (!dev->of_node)
+		dev->of_node = of_node_get(dev->parent->of_node);
+
+	if (!id)
+		return -EINVAL;
+
 	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return -ENOMEM;
+
 	ctx->mcu = mcu;
+	switch (id->driver_data) {
+	case KHADAS_MCU_GENERIC:
+		ctx->fan_reg = KHADAS_MCU_CMD_FAN_STATUS_CTRL_REG;
+		ctx->levels = khadas_mcu_fan_levels;
+		ctx->nlevels = ARRAY_SIZE(khadas_mcu_fan_levels);
+		break;
+	case KHADAS_MCU_VIM4:
+		ctx->fan_reg = KHADAS_MCU_VIM4_FAN_CTRL_REG;
+		ctx->levels = khadas_mcu_vim4_fan_levels;
+		ctx->nlevels = ARRAY_SIZE(khadas_mcu_vim4_fan_levels);
+		break;
+	default:
+		return -ENODEV;
+	}
+
+	ctx->power = devm_regulator_get(dev, "fan");
+	if (IS_ERR(ctx->power))
+		return PTR_ERR(ctx->power);
+
+	ret = regulator_enable(ctx->power);
+	if (ret) {
+		dev_err(dev, "Failed to enable fan power supply: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_add_action_or_reset(dev, khadas_mcu_fan_regulator_disable, ctx->power);
+	if (ret)
+		return ret;
+
 	platform_set_drvdata(pdev, ctx);
 
-	cdev = devm_thermal_of_child_cooling_device_register(dev->parent,
-							     dev->parent->of_node,
+	cdev = devm_thermal_of_child_cooling_device_register(dev,
+							     dev->of_node,
 							     "khadas-mcu-fan", ctx,
 							     &khadas_mcu_fan_cooling_ops);
 	if (IS_ERR(cdev)) {
@@ -109,21 +164,24 @@ static void khadas_mcu_fan_shutdown(struct platform_device *pdev)
 {
 	struct khadas_mcu_fan_ctx *ctx = platform_get_drvdata(pdev);
 
-	khadas_mcu_fan_set_level(ctx, 0);
+	khadas_mcu_fan_set_level(ctx, ctx->levels[0]);
 }
 
 #ifdef CONFIG_PM_SLEEP
 static int khadas_mcu_fan_suspend(struct device *dev)
 {
 	struct khadas_mcu_fan_ctx *ctx = dev_get_drvdata(dev);
-	unsigned int level_save = ctx->level;
 	int ret;
 
-	ret = khadas_mcu_fan_set_level(ctx, 0);
+	ret = khadas_mcu_fan_set_level(ctx, ctx->levels[0]);
 	if (ret)
 		return ret;
 
-	ctx->level = level_save;
+	ret = regulator_disable(ctx->power);
+	if (ret) {
+		khadas_mcu_fan_set_level(ctx, ctx->levels[ctx->level]);
+		return ret;
+	}
 
 	return 0;
 }
@@ -131,8 +189,13 @@ static int khadas_mcu_fan_suspend(struct device *dev)
 static int khadas_mcu_fan_resume(struct device *dev)
 {
 	struct khadas_mcu_fan_ctx *ctx = dev_get_drvdata(dev);
+	int ret;
+
+	ret = regulator_enable(ctx->power);
+	if (ret)
+		return ret;
 
-	return khadas_mcu_fan_set_level(ctx, ctx->level);
+	return khadas_mcu_fan_set_level(ctx, ctx->levels[ctx->level]);
 }
 #endif
 
@@ -140,7 +203,8 @@ static SIMPLE_DEV_PM_OPS(khadas_mcu_fan_pm, khadas_mcu_fan_suspend,
 			 khadas_mcu_fan_resume);
 
 static const struct platform_device_id khadas_mcu_fan_id_table[] = {
-	{ .name = "khadas-mcu-fan-ctrl", },
+	{ .name = "khadas-mcu-fan-ctrl", .driver_data = KHADAS_MCU_GENERIC },
+	{ .name = "khadas-mcu-vim4-fan", .driver_data = KHADAS_MCU_VIM4 },
 	{},
 };
 MODULE_DEVICE_TABLE(platform, khadas_mcu_fan_id_table);
diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
index af72db6299cdf4..4554a6fa55d113 100644
--- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
+++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
@@ -680,8 +680,8 @@ static int adc_tm5_register_tzd(struct adc_tm5_chip *adc_tm)
 				continue;
 			}
 
-			dev_err(adc_tm->dev, "Error registering TZ zone for channel %d: %ld\n",
-				adc_tm->channels[i].channel, PTR_ERR(tzd));
+			dev_err(adc_tm->dev, "Error registering TZ zone for channel %d: %pe\n",
+				adc_tm->channels[i].channel, tzd);
 			return PTR_ERR(tzd);
 		}
 		adc_tm->channels[i].tzd = tzd;
diff --git a/drivers/thermal/qcom/tsens-v2.c b/drivers/thermal/qcom/tsens-v2.c
index 2ee117aa91baea..843dd29fad2e38 100644
--- a/drivers/thermal/qcom/tsens-v2.c
+++ b/drivers/thermal/qcom/tsens-v2.c
@@ -54,8 +54,8 @@ static struct tsens_features tsens_v2_feat = {
 	.adc		= 0,
 	.srot_split	= 1,
 	.max_sensors	= 16,
-	.trip_min_temp	= -40000,
-	.trip_max_temp	= 120000,
+	.trip_min_temp	= -204000,
+	.trip_max_temp	= 204000,
 };
 
 static struct tsens_features ipq8074_feat = {
diff --git a/drivers/thermal/renesas/rzg3e_thermal.c b/drivers/thermal/renesas/rzg3e_thermal.c
index c44f5b8858d02a..a1ceff79e83e6d 100644
--- a/drivers/thermal/renesas/rzg3e_thermal.c
+++ b/drivers/thermal/renesas/rzg3e_thermal.c
@@ -93,6 +93,7 @@ struct rzg3e_thermal_info {
  * @info: chip type specific information
  * @trmval0: calibration value 0 (b)
  * @trmval1: calibration value 1 (c)
+ * @sier: cached interrupt enable register
  * @lock: protects hardware access during conversions
  */
 struct rzg3e_thermal_priv {
@@ -103,6 +104,7 @@ struct rzg3e_thermal_priv {
 	const struct rzg3e_thermal_info *info;
 	u16 trmval0;
 	u16 trmval1;
+	u32 sier;
 	struct mutex lock;
 };
 
@@ -148,12 +150,16 @@ static int rzg3e_thermal_power_on(struct rzg3e_thermal_priv *priv)
 		return ret;
 	}
 
+	/* Restore interrupt enable state */
+	writel(priv->sier, priv->base + TSU_SIER);
+
 	return 0;
 }
 
 static void rzg3e_thermal_power_off(struct rzg3e_thermal_priv *priv)
 {
-	/* Disable all interrupts */
+	/* Save and disable all interrupts */
+	priv->sier = readl(priv->base + TSU_SIER);
 	writel(0, priv->base + TSU_SIER);
 
 	/* Clear pending interrupts */
@@ -194,7 +200,12 @@ static u16 rzg3e_thermal_temp_to_code(struct rzg3e_thermal_priv *priv, int temp_
 	s64 numerator, denominator;
 	s64 code;
 
-	numerator = (temp_mc - info->temp_d_mc) * (priv->trmval1 - priv->trmval0);
+	/*
+	 * Perform the arithmetic in 64 bits so that it cannot overflow for
+	 * -INT_MAX/INT_MAX values passed from the thermal core or when
+	 * userspace writes arbitrary trip point temperatures.
+	 */
+	numerator = ((s64)temp_mc - info->temp_d_mc) * (priv->trmval1 - priv->trmval0);
 	denominator = info->temp_e_mc - info->temp_d_mc;
 
 	code = div64_s64(numerator, denominator) + priv->trmval0;
@@ -214,6 +225,16 @@ static int rzg3e_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 
 	guard(mutex)(&priv->lock);
 
+	/* Make sure a previous conversion is not in progress */
+	ret = readl_poll_timeout(priv->base + TSU_SSR, status,
+				 !(status & TSU_SSR_CONV),
+				 TSU_POLL_DELAY_US,
+				 USEC_PER_MSEC);
+	if (ret) {
+		dev_err(priv->dev, "Timeout waiting for conversion\n");
+		goto out;
+	}
+
 	/* Clear any previous conversion status */
 	writel(TSU_SICR_ADCLR, priv->base + TSU_SICR);
 
@@ -241,7 +262,6 @@ static int rzg3e_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 		*temp, *temp / 1000, abs(*temp) % 1000, code);
 
 out:
-	pm_runtime_mark_last_busy(priv->dev);
 	pm_runtime_put_autosuspend(priv->dev);
 	return ret;
 }
@@ -254,10 +274,6 @@ static int rzg3e_thermal_set_trips(struct thermal_zone_device *tz,
 	u32 val;
 	int ret;
 
-	/* Hardware requires low < high */
-	if (low >= high)
-		return -EINVAL;
-
 	ret = pm_runtime_resume_and_get(priv->dev);
 	if (ret < 0)
 		return ret;
@@ -293,11 +309,9 @@ static int rzg3e_thermal_set_trips(struct thermal_zone_device *tz,
 	/* Enable comparison with "out of range" mode (CMPCOND=0) */
 	writel(TSU_CMSR_CMPEN, priv->base + TSU_CMSR);
 
-	/* Unmask compare IRQ and start a conversion to evaluate window */
+	/* Unmask compare IRQ */
 	writel(TSU_SIER_CMPIE, priv->base + TSU_SIER);
-	writel(TSU_STRGR_ADST, priv->base + TSU_STRGR);
 
-	pm_runtime_mark_last_busy(priv->dev);
 	pm_runtime_put_autosuspend(priv->dev);
 
 	return 0;
@@ -469,7 +483,6 @@ static int rzg3e_thermal_probe(struct platform_device *pdev)
 	if (ret)
 		dev_warn(dev, "Failed to add hwmon sysfs attributes\n");
 
-	pm_runtime_mark_last_busy(dev);
 	pm_runtime_put_autosuspend(dev);
 
 	dev_info(dev, "RZ/G3E thermal sensor registered\n");
diff --git a/drivers/thermal/renesas/rzg3s_thermal.c b/drivers/thermal/renesas/rzg3s_thermal.c
index 7ced8f76a0ec39..a8716da7fcd2f4 100644
--- a/drivers/thermal/renesas/rzg3s_thermal.c
+++ b/drivers/thermal/renesas/rzg3s_thermal.c
@@ -12,6 +12,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/property.h>
 #include <linux/reset.h>
 #include <linux/thermal.h>
 #include <linux/units.h>
@@ -27,10 +28,6 @@
 
 #define TSU_READ_STEPS		8
 
-/* Default calibration values, if FUSE values are missing. */
-#define SW_CALIB0_VAL		1297
-#define SW_CALIB1_VAL		751
-
 #define MCELSIUS(temp)		((temp) * MILLIDEGREE_PER_DEGREE)
 
 /**
@@ -55,6 +52,16 @@ struct rzg3s_thermal_priv {
 	u16 calib1;
 };
 
+/**
+ * struct rzg3s_thermal_info - RZ/G3S thermal information
+ * @calib0: calibration value if FUSE value is missing
+ * @calib1: calibration value if FUSE value is missing
+ */
+struct rzg3s_thermal_info {
+	u16 calib0;
+	u16 calib1;
+};
+
 static int rzg3s_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	struct rzg3s_thermal_priv *priv = thermal_zone_device_priv(tz);
@@ -149,7 +156,8 @@ static const struct thermal_zone_device_ops rzg3s_tz_of_ops = {
 	.change_mode = rzg3s_thermal_change_mode,
 };
 
-static int rzg3s_thermal_read_calib(struct rzg3s_thermal_priv *priv)
+static int rzg3s_thermal_read_calib(struct rzg3s_thermal_priv *priv,
+				    const struct rzg3s_thermal_info *info)
 {
 	struct device *dev = priv->dev;
 	u32 val;
@@ -163,13 +171,13 @@ static int rzg3s_thermal_read_calib(struct rzg3s_thermal_priv *priv)
 	if (val & OTPTSUTRIM_EN_MASK)
 		priv->calib0 = FIELD_GET(OTPTSUTRIM_MASK, val);
 	else
-		priv->calib0 = SW_CALIB0_VAL;
+		priv->calib0 = info->calib0;
 
 	val = readl(priv->base + OTPTSUTRIM_REG(1));
 	if (val & OTPTSUTRIM_EN_MASK)
 		priv->calib1 = FIELD_GET(OTPTSUTRIM_MASK, val);
 	else
-		priv->calib1 = SW_CALIB1_VAL;
+		priv->calib1 = info->calib1;
 
 	pm_runtime_put_autosuspend(dev);
 
@@ -178,6 +186,7 @@ static int rzg3s_thermal_read_calib(struct rzg3s_thermal_priv *priv)
 
 static int rzg3s_thermal_probe(struct platform_device *pdev)
 {
+	const struct rzg3s_thermal_info *info = device_get_match_data(&pdev->dev);
 	struct rzg3s_thermal_priv *priv;
 	struct device *dev = &pdev->dev;
 	int ret;
@@ -208,7 +217,7 @@ static int rzg3s_thermal_probe(struct platform_device *pdev)
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to enable runtime PM!\n");
 
-	ret = rzg3s_thermal_read_calib(priv);
+	ret = rzg3s_thermal_read_calib(priv, info);
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to read calibration data!\n");
 
@@ -251,8 +260,19 @@ static const struct dev_pm_ops rzg3s_thermal_pm_ops = {
 	SYSTEM_SLEEP_PM_OPS(rzg3s_thermal_suspend, rzg3s_thermal_resume)
 };
 
+static const struct rzg3s_thermal_info rzg3s_info = {
+	.calib0 = 1297,
+	.calib1 = 751,
+};
+
+static const struct rzg3s_thermal_info rzg3l_info = {
+	.calib0 = 1322,
+	.calib1 = 772,
+};
+
 static const struct of_device_id rzg3s_thermal_dt_ids[] = {
-	{ .compatible = "renesas,r9a08g045-tsu" },
+	{ .compatible = "renesas,r9a08g045-tsu", .data = &rzg3s_info },
+	{ .compatible = "renesas,r9a08g046-tsu", .data = &rzg3l_info },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, rzg3s_thermal_dt_ids);
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 82e2f0d8a26dee..e8d4da0aeac38e 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -1005,7 +1005,8 @@ thermal_cooling_device_alloc(const char *type, const struct thermal_cooling_devi
 	return ERR_PTR(ret);
 }
 
-int thermal_cooling_device_add(struct thermal_cooling_device *cdev, void *devdata)
+int thermal_cooling_device_add(struct thermal_cooling_device *cdev,
+			       struct device *parent, void *devdata)
 {
 	unsigned long current_state;
 	int ret;
@@ -1013,6 +1014,7 @@ int thermal_cooling_device_add(struct thermal_cooling_device *cdev, void *devdat
 	mutex_init(&cdev->lock);
 	INIT_LIST_HEAD(&cdev->thermal_instances);
 	cdev->updated = false;
+	cdev->device.parent = parent;
 	cdev->device.class = &thermal_class;
 	cdev->device.release = thermal_cdev_release;
 	device_initialize(&cdev->device);
@@ -1062,21 +1064,23 @@ int thermal_cooling_device_add(struct thermal_cooling_device *cdev, void *devdat
 }
 
 /**
- * thermal_cooling_device_register() - register a new thermal cooling device
+ * thermal_cooling_device_create() - register a new thermal cooling device
+ * @parent:	parent device (optional).
  * @type:	the thermal cooling device type.
  * @devdata:	device private data.
  * @ops:	standard thermal cooling devices callbacks.
  *
- * This interface function adds a new thermal cooling device (fan/processor/...)
- * to /sys/class/thermal/ folder as cooling_device[0-*]. It tries to bind itself
- * to all the thermal zone devices registered at the same time.
+ * Allocate and register a new thermal cooling device under the given parent (if
+ * not NULL) and with the given type, device data, and operations.  During the
+ * registration, it will be matched against all of the registered thermal zones
+ * and it will be bound to the matching ones.
  *
- * Return: a pointer to the created struct thermal_cooling_device or an
- * ERR_PTR. Caller must check return value with IS_ERR*() helpers.
+ * Return: A pointer to the created struct thermal_cooling_device or an ERR_PTR.
+ * Callers must use IS_ERR*() helpers to check the return value.
  */
-struct thermal_cooling_device *
-thermal_cooling_device_register(const char *type, void *devdata,
-				const struct thermal_cooling_device_ops *ops)
+struct thermal_cooling_device *thermal_cooling_device_create(
+			struct device *parent, const char *type, void *devdata,
+			const struct thermal_cooling_device_ops *ops)
 {
 	struct thermal_cooling_device *cdev;
 	int ret;
@@ -1085,13 +1089,13 @@ thermal_cooling_device_register(const char *type, void *devdata,
 	if (IS_ERR(cdev))
 		return cdev;
 
-	ret = thermal_cooling_device_add(cdev, devdata);
+	ret = thermal_cooling_device_add(cdev, parent, devdata);
 	if (ret)
 		return ERR_PTR(ret);
 
 	return cdev;
 }
-EXPORT_SYMBOL_GPL(thermal_cooling_device_register);
+EXPORT_SYMBOL_GPL(thermal_cooling_device_create);
 
 static void thermal_cooling_device_release(void *data)
 {
@@ -1662,7 +1666,7 @@ EXPORT_SYMBOL_GPL(thermal_zone_device_unregister);
  *
  * Return: On success returns a reference to an unique thermal zone with
  * matching name equals to @name, an ERR_PTR otherwise (-EINVAL for invalid
- * paramenters, -ENODEV for not found and -EEXIST for multiple matches).
+ * parameters, -ENODEV for not found and -EEXIST for multiple matches).
  */
 struct thermal_zone_device *thermal_zone_get_zone_by_name(const char *name)
 {
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index e98b0aa5aacc4f..7ef7c6cab4374f 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -270,7 +270,8 @@ void thermal_governor_update_tz(struct thermal_zone_device *tz,
 struct thermal_cooling_device *
 thermal_cooling_device_alloc(const char *type, const struct thermal_cooling_device_ops *ops);
 
-int thermal_cooling_device_add(struct thermal_cooling_device *cdev, void *devdata);
+int thermal_cooling_device_add(struct thermal_cooling_device *cdev,
+			       struct device *parent, void *devdata);
 
 /* Helpers */
 #define for_each_trip_desc(__tz, __td)	\
diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c
index 0217a49b08ae33..9e96fb1b033e94 100644
--- a/drivers/thermal/thermal_of.c
+++ b/drivers/thermal/thermal_of.c
@@ -63,22 +63,23 @@ static int thermal_of_get_trip_type(struct device_node *np,
 static int thermal_of_populate_trip(struct device_node *np,
 				    struct thermal_trip *trip)
 {
-	int prop;
+	u32 hysteresis;
+	s32 temperature;
 	int ret;
 
-	ret = of_property_read_u32(np, "temperature", &prop);
+	ret = of_property_read_s32(np, "temperature", &temperature);
 	if (ret < 0) {
 		pr_err("missing temperature property\n");
 		return ret;
 	}
-	trip->temperature = prop;
+	trip->temperature = temperature;
 
-	ret = of_property_read_u32(np, "hysteresis", &prop);
+	ret = of_property_read_u32(np, "hysteresis", &hysteresis);
 	if (ret < 0) {
 		pr_err("missing hysteresis property\n");
 		return ret;
 	}
-	trip->hysteresis = prop;
+	trip->hysteresis = hysteresis;
 
 	ret = thermal_of_get_trip_type(np, &trip->type);
 	if (ret < 0) {
@@ -349,10 +350,10 @@ static bool thermal_of_should_bind(struct thermal_zone_device *tz,
 }
 
 /**
- * thermal_of_zone_unregister - Cleanup the specific allocated ressources
+ * thermal_of_zone_unregister - Cleanup the specific allocated resources
  *
  * This function disables the thermal zone and frees the different
- * ressources allocated specific to the thermal OF.
+ * resources allocated specific to the thermal OF.
  *
  * @tz: a pointer to the thermal zone structure
  */
@@ -561,7 +562,7 @@ thermal_of_cooling_device_register(struct device_node *np, u32 cdev_id,
 	cdev->np = np;
 	cdev->cdev_id = cdev_id;
 
-	ret = thermal_cooling_device_add(cdev, devdata);
+	ret = thermal_cooling_device_add(cdev, NULL, devdata);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.c b/drivers/thermal/ti-soc-thermal/ti-bandgap.c
index ba43399d0b384c..5c028d25a5654e 100644
--- a/drivers/thermal/ti-soc-thermal/ti-bandgap.c
+++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.c
@@ -332,7 +332,7 @@ static inline int ti_bandgap_validate(struct ti_bandgap *bgp, int id)
  * ti_bandgap_read_counter() - read the sensor counter
  * @bgp: pointer to bandgap instance
  * @id: sensor id
- * @interval: resulting update interval in miliseconds
+ * @interval: resulting update interval in milliseconds
  */
 static void ti_bandgap_read_counter(struct ti_bandgap *bgp, int id,
 				    int *interval)
@@ -352,7 +352,7 @@ static void ti_bandgap_read_counter(struct ti_bandgap *bgp, int id,
  * ti_bandgap_read_counter_delay() - read the sensor counter delay
  * @bgp: pointer to bandgap instance
  * @id: sensor id
- * @interval: resulting update interval in miliseconds
+ * @interval: resulting update interval in milliseconds
  */
 static void ti_bandgap_read_counter_delay(struct ti_bandgap *bgp, int id,
 					  int *interval)
@@ -394,7 +394,7 @@ static void ti_bandgap_read_counter_delay(struct ti_bandgap *bgp, int id,
  * ti_bandgap_read_update_interval() - read the sensor update interval
  * @bgp: pointer to bandgap instance
  * @id: sensor id
- * @interval: resulting update interval in miliseconds
+ * @interval: resulting update interval in milliseconds
  *
  * Return: 0 on success or the proper error code
  */
@@ -427,7 +427,7 @@ int ti_bandgap_read_update_interval(struct ti_bandgap *bgp, int id,
  * ti_bandgap_write_counter_delay() - set the counter_delay
  * @bgp: pointer to bandgap instance
  * @id: sensor id
- * @interval: desired update interval in miliseconds
+ * @interval: desired update interval in milliseconds
  *
  * Return: 0 on success or the proper error code
  */
@@ -471,7 +471,7 @@ static int ti_bandgap_write_counter_delay(struct ti_bandgap *bgp, int id,
  * ti_bandgap_write_counter() - set the bandgap sensor counter
  * @bgp: pointer to bandgap instance
  * @id: sensor id
- * @interval: desired update interval in miliseconds
+ * @interval: desired update interval in milliseconds
  */
 static void ti_bandgap_write_counter(struct ti_bandgap *bgp, int id,
 				     u32 interval)
@@ -486,7 +486,7 @@ static void ti_bandgap_write_counter(struct ti_bandgap *bgp, int id,
  * ti_bandgap_write_update_interval() - set the update interval
  * @bgp: pointer to bandgap instance
  * @id: sensor id
- * @interval: desired update interval in miliseconds
+ * @interval: desired update interval in milliseconds
  *
  * Return: 0 on success or the proper error code
  */
diff --git a/drivers/thunderbolt/acpi.c b/drivers/thunderbolt/acpi.c
index 53546bc477a5b4..be91bc6049008d 100644
--- a/drivers/thunderbolt/acpi.c
+++ b/drivers/thunderbolt/acpi.c
@@ -17,8 +17,8 @@ static acpi_status tb_acpi_add_link(acpi_handle handle, u32 level, void *data,
 	struct acpi_device *adev = acpi_fetch_acpi_dev(handle);
 	struct fwnode_handle *fwnode;
 	struct tb_nhi *nhi = data;
+	struct device *dev = NULL;
 	struct pci_dev *pdev;
-	struct device *dev;
 
 	if (!adev)
 		return AE_OK;
@@ -37,7 +37,7 @@ static acpi_status tb_acpi_add_link(acpi_handle handle, u32 level, void *data,
 	 * USB3 ports might not even have a physical device yet if xHCI driver
 	 * isn't bound yet.
 	 */
-	dev = acpi_get_first_physical_node(adev);
+	dev = acpi_bus_get_primary_device(adev);
 	if (!dev || !dev_is_pci(dev))
 		goto out_put;
 
@@ -74,6 +74,7 @@ static acpi_status tb_acpi_add_link(acpi_handle handle, u32 level, void *data,
 	}
 
 out_put:
+	put_device(dev);
 	fwnode_handle_put(fwnode);
 	return AE_OK;
 }
diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c
index cd47b627f97b27..965988b18608f4 100644
--- a/drivers/thunderbolt/ctl.c
+++ b/drivers/thunderbolt/ctl.c
@@ -73,7 +73,6 @@ struct tb_ctl {
 #define tb_ctl_dbg_once(ctl, format, arg...) \
 	dev_dbg_once((ctl)->nhi->dev, format, ## arg)
 
-static DECLARE_WAIT_QUEUE_HEAD(tb_cfg_request_cancel_queue);
 /* Serializes access to request kref_get/put */
 static DEFINE_MUTEX(tb_cfg_request_lock);
 
@@ -133,41 +132,42 @@ void tb_cfg_request_put(struct tb_cfg_request *req)
 static int tb_cfg_request_enqueue(struct tb_ctl *ctl,
 				  struct tb_cfg_request *req)
 {
+	tb_cfg_request_get(req);
+
 	WARN_ON(test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags));
 	WARN_ON(req->ctl);
 
-	mutex_lock(&ctl->request_queue_lock);
+	guard(mutex)(&ctl->request_queue_lock);
 	if (!ctl->running) {
-		mutex_unlock(&ctl->request_queue_lock);
+		tb_cfg_request_put(req);
 		return -ENOTCONN;
 	}
 	req->ctl = ctl;
 	list_add_tail(&req->list, &ctl->request_queue);
 	set_bit(TB_CFG_REQUEST_ACTIVE, &req->flags);
-	mutex_unlock(&ctl->request_queue_lock);
 	return 0;
 }
 
-static void tb_cfg_request_dequeue(struct tb_cfg_request *req)
+static bool tb_cfg_request_is_active(struct tb_cfg_request *req)
 {
-	struct tb_ctl *ctl = req->ctl;
-
-	mutex_lock(&ctl->request_queue_lock);
-	if (!test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags)) {
-		mutex_unlock(&ctl->request_queue_lock);
-		return;
-	}
+	return test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags);
+}
 
-	list_del(&req->list);
-	clear_bit(TB_CFG_REQUEST_ACTIVE, &req->flags);
-	if (test_bit(TB_CFG_REQUEST_CANCELED, &req->flags))
-		wake_up(&tb_cfg_request_cancel_queue);
-	mutex_unlock(&ctl->request_queue_lock);
+static bool tb_cfg_request_is_canceled(struct tb_cfg_request *req)
+{
+	return test_bit(TB_CFG_REQUEST_CANCELED, &req->flags);
 }
 
-static bool tb_cfg_request_is_active(struct tb_cfg_request *req)
+static void tb_cfg_request_dequeue(struct tb_cfg_request *req)
 {
-	return test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags);
+	struct tb_ctl *ctl = req->ctl;
+
+	guard(mutex)(&ctl->request_queue_lock);
+	if (tb_cfg_request_is_active(req)) {
+		list_del(&req->list);
+		clear_bit(TB_CFG_REQUEST_ACTIVE, &req->flags);
+		tb_cfg_request_put(req);
+	}
 }
 
 static struct tb_cfg_request *
@@ -178,7 +178,7 @@ tb_cfg_request_find(struct tb_ctl *ctl, struct ctl_pkg *pkg)
 	mutex_lock(&pkg->ctl->request_queue_lock);
 	list_for_each_entry(iter, &pkg->ctl->request_queue, list) {
 		tb_cfg_request_get(iter);
-		if (iter->match(iter, pkg)) {
+		if (!tb_cfg_request_is_canceled(iter) && iter->match(iter, pkg)) {
 			req = iter;
 			break;
 		}
@@ -512,8 +512,11 @@ static void tb_ctl_rx_callback(struct tb_ring *ring, struct ring_frame *frame,
 	trace_tb_rx(pkg->ctl->index, frame->eof, pkg->buffer, frame->size, !req);
 
 	if (req) {
-		if (req->copy(req, pkg))
-			schedule_work(&req->work);
+		scoped_guard(mutex, &pkg->ctl->request_queue_lock) {
+			if (!tb_cfg_request_is_canceled(req) &&
+			    req->copy(req, pkg))
+				schedule_work(&req->work);
+		}
 		tb_cfg_request_put(req);
 	}
 
@@ -525,11 +528,10 @@ static void tb_cfg_request_work(struct work_struct *work)
 {
 	struct tb_cfg_request *req = container_of(work, typeof(*req), work);
 
-	if (!test_bit(TB_CFG_REQUEST_CANCELED, &req->flags))
+	if (!tb_cfg_request_is_canceled(req))
 		req->callback(req->callback_data);
 
 	tb_cfg_request_dequeue(req);
-	tb_cfg_request_put(req);
 }
 
 /**
@@ -555,10 +557,9 @@ int tb_cfg_request(struct tb_ctl *ctl, struct tb_cfg_request *req,
 	INIT_WORK(&req->work, tb_cfg_request_work);
 	INIT_LIST_HEAD(&req->list);
 
-	tb_cfg_request_get(req);
 	ret = tb_cfg_request_enqueue(ctl, req);
 	if (ret)
-		goto err_put;
+		return ret;
 
 	ret = tb_ctl_tx(ctl, req->request, req->request_size,
 			req->request_type);
@@ -572,9 +573,6 @@ int tb_cfg_request(struct tb_ctl *ctl, struct tb_cfg_request *req,
 
 err_dequeue:
 	tb_cfg_request_dequeue(req);
-err_put:
-	tb_cfg_request_put(req);
-
 	return ret;
 }
 
@@ -588,9 +586,10 @@ int tb_cfg_request(struct tb_ctl *ctl, struct tb_cfg_request *req,
  */
 void tb_cfg_request_cancel(struct tb_cfg_request *req, int err)
 {
-	set_bit(TB_CFG_REQUEST_CANCELED, &req->flags);
-	schedule_work(&req->work);
-	wait_event(tb_cfg_request_cancel_queue, !tb_cfg_request_is_active(req));
+	scoped_guard(mutex, &req->ctl->request_queue_lock)
+		set_bit(TB_CFG_REQUEST_CANCELED, &req->flags);
+	cancel_work_sync(&req->work);
+	tb_cfg_request_dequeue(req);
 	req->result.err = err;
 }
 
diff --git a/drivers/thunderbolt/debugfs.c b/drivers/thunderbolt/debugfs.c
index 6e9080e7bcec8e..ccefc5dd1ded7a 100644
--- a/drivers/thunderbolt/debugfs.c
+++ b/drivers/thunderbolt/debugfs.c
@@ -37,6 +37,9 @@
 
 #define COUNTER_SET_LEN		3
 
+#define HW_MARGINING_DWORDS		3
+#define SW_MARGINING_DWORDS		4
+
 /*
  * USB4 spec doesn't specify dwell range, the range of 100 ms to 500 ms
  * probed to give good results.
@@ -494,7 +497,7 @@ struct tb_margining {
 	unsigned int gen;
 	bool asym_rx;
 	u32 caps[3];
-	u32 results[3];
+	u32 results[5];
 	enum usb4_margining_lane lanes;
 	unsigned int min_ber_level;
 	unsigned int max_ber_level;
@@ -612,6 +615,11 @@ supports_optional_voltage_offset_range(const struct tb_margining *margining)
 	return margining->caps[0] & USB4_MARGIN_CAP_0_OPT_VOLTAGE_SUPPORT;
 }
 
+static bool supports_extended_error_counter(const struct tb_margining *margining)
+{
+	return margining->caps[0] & USB4_MARGIN_CAP_0_EXT_ERR_COUNTER;
+}
+
 static ssize_t
 margining_ber_level_write(struct file *file, const char __user *user_buf,
 			   size_t count, loff_t *ppos)
@@ -701,6 +709,8 @@ static int margining_caps_show(struct seq_file *s, void *not_used)
 		ber_level_show(s, margining->max_ber_level);
 	} else {
 		seq_puts(s, "# hardware margining: no\n");
+		seq_printf(s, "# extended error counter: %s\n",
+			   str_yes_no(supports_extended_error_counter(margining)));
 	}
 
 	seq_printf(s, "# all lanes simultaneously: %s\n",
@@ -1150,36 +1160,91 @@ static int margining_mode_show(struct seq_file *s, void *not_used)
 }
 DEBUGFS_ATTR_RW(margining_mode);
 
+static u32 margining_get_lane_error(const u32 *results, unsigned int lane, bool extended)
+{
+	u32 error_count;
+
+	switch (lane) {
+	case USB4_MARGINING_LANE_RX0:
+		error_count = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK, results[0]);
+		break;
+	case USB4_MARGINING_LANE_RX1:
+		error_count = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK, results[0]);
+		break;
+	case USB4_MARGINING_LANE_RX2:
+		error_count = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK, results[0]);
+		break;
+	default:
+		return 0;
+	}
+
+	if (extended)
+		error_count |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK, results[1 + lane]);
+
+	return error_count;
+}
+
 static int margining_run_sw(struct tb_margining *margining,
 			    struct usb4_port_margining_params *params)
 {
+	bool extended_err = supports_extended_error_counter(margining);
 	u32 nsamples = margining->dwell_time / DWELL_SAMPLE_INTERVAL;
 	int ret, i;
+	u32 dwords;
+
+	if (params->error_counter != USB4_MARGIN_SW_ERROR_COUNTER_START)
+		goto out_stop_or_clear;
+
+	if (extended_err)
+		dwords = SW_MARGINING_DWORDS;
+	else
+		dwords = 1;
 
 	ret = usb4_port_sw_margin(margining->port, margining->target, margining->index,
 				  params, margining->results);
 	if (ret)
-		goto out_stop;
+		return ret;
 
 	for (i = 0; i <= nsamples; i++) {
 		u32 errors = 0;
 
 		ret = usb4_port_sw_margin_errors(margining->port, margining->target,
-						 margining->index, &margining->results[1]);
-		if (ret)
+						 margining->index, &margining->results[1],
+						 dwords);
+		if (ret) {
+			tb_port_warn(margining->port, "failed to read margining error counters\n");
 			break;
+		}
 
-		if (margining->lanes == USB4_MARGINING_LANE_RX0)
-			errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK,
-					   margining->results[1]);
-		else if (margining->lanes == USB4_MARGINING_LANE_RX1)
-			errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK,
-					   margining->results[1]);
-		else if (margining->lanes == USB4_MARGINING_LANE_RX2)
-			errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK,
-					   margining->results[1]);
-		else if (margining->lanes == USB4_MARGINING_LANE_ALL)
-			errors = margining->results[1];
+		if (margining->lanes == USB4_MARGINING_LANE_ALL) {
+			if (extended_err) {
+				/* Extended counters with all lanes: check each active lane */
+				int width = tb_port_get_link_width(margining->port);
+				unsigned int lane, max_lane = 2;
+
+				if (width == TB_LINK_WIDTH_ASYM_TX) {
+					max_lane = 1;
+				} else if ((width == TB_LINK_WIDTH_ASYM_RX) && margining->asym_rx) {
+					max_lane = 3;
+				} else if (width < 0) {
+					tb_port_warn(margining->port, "failed to read link width\n");
+					break;
+				}
+
+				for (lane = 0; lane < max_lane; lane++) {
+					errors = margining_get_lane_error(&margining->results[1],
+									  lane, extended_err);
+					if (errors)
+						break;
+				}
+			} else {
+				/* Standard counters: all lane errors fit in results[1]*/
+				errors = margining->results[1];
+			}
+		} else {
+			errors = margining_get_lane_error(&margining->results[1],
+							  margining->lanes, extended_err);
+		}
 
 		/* Any errors stop the test */
 		if (errors)
@@ -1187,15 +1252,15 @@ static int margining_run_sw(struct tb_margining *margining,
 
 		fsleep(DWELL_SAMPLE_INTERVAL * USEC_PER_MSEC);
 	}
+	params->error_counter = USB4_MARGIN_SW_ERROR_COUNTER_STOP;
 
-out_stop:
+out_stop_or_clear:
 	/*
-	 * Stop the counters but don't clear them to allow the
+	 * Stop the counters or clear them as per the
 	 * different error counter configurations.
 	 */
-	margining_modify_error_counter(margining, margining->lanes,
-				       USB4_MARGIN_SW_ERROR_COUNTER_STOP);
-	return ret;
+	return margining_modify_error_counter(margining, margining->lanes,
+					      params->error_counter);
 }
 
 static int validate_margining(struct tb_margining *margining)
@@ -1272,7 +1337,7 @@ static int margining_run_write(void *data, u64 val)
 
 	if (margining->software) {
 		struct usb4_port_margining_params params = {
-			.error_counter = USB4_MARGIN_SW_ERROR_COUNTER_CLEAR,
+			.error_counter = margining->error_counter,
 			.lanes = margining->lanes,
 			.time = margining->time,
 			.voltage_time_offset = margining->voltage_time_offset,
@@ -1303,7 +1368,7 @@ static int margining_run_write(void *data, u64 val)
 			    margining->lanes);
 
 		ret = usb4_port_hw_margin(port, margining->target, margining->index, &params,
-					  margining->results, ARRAY_SIZE(margining->results));
+					  margining->results, HW_MARGINING_DWORDS);
 	}
 
 	if (down_sw)
@@ -1425,7 +1490,7 @@ static int margining_results_show(struct seq_file *s, void *not_used)
 	seq_printf(s, "0x%08x\n", margining->results[0]);
 	/* Only the hardware margining has two result dwords */
 	if (!margining->software) {
-		for (int i = 1; i < ARRAY_SIZE(margining->results); i++)
+		for (int i = 1; i < HW_MARGINING_DWORDS; i++)
 			seq_printf(s, "0x%08x\n", margining->results[i]);
 
 		if (margining->lanes == USB4_MARGINING_LANE_ALL) {
@@ -1444,18 +1509,30 @@ static int margining_results_show(struct seq_file *s, void *not_used)
 		u32 lane_errors, result;
 
 		seq_printf(s, "0x%08x\n", margining->results[1]);
+		if (supports_extended_error_counter(margining)) {
+			seq_printf(s, "0x%08x\n", margining->results[2]);
+			seq_printf(s, "0x%08x\n", margining->results[3]);
+			seq_printf(s, "0x%08x\n", margining->results[4]);
+		}
+
 
 		result = FIELD_GET(USB4_MARGIN_SW_LANES_MASK, margining->results[0]);
 		if (result == USB4_MARGINING_LANE_RX0 ||
 		    result == USB4_MARGINING_LANE_ALL) {
 			lane_errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK,
 						margining->results[1]);
+			if (supports_extended_error_counter(margining))
+				lane_errors |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK,
+							  margining->results[2]);
 			seq_printf(s, "# lane 0 errors: %u\n", lane_errors);
 		}
 		if (result == USB4_MARGINING_LANE_RX1 ||
 		    result == USB4_MARGINING_LANE_ALL) {
 			lane_errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK,
 						margining->results[1]);
+			if (supports_extended_error_counter(margining))
+				lane_errors |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK,
+							  margining->results[3]);
 			seq_printf(s, "# lane 1 errors: %u\n", lane_errors);
 		}
 		if (margining->asym_rx &&
@@ -1463,6 +1540,9 @@ static int margining_results_show(struct seq_file *s, void *not_used)
 		     result == USB4_MARGINING_LANE_ALL)) {
 			lane_errors = FIELD_GET(USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK,
 						margining->results[1]);
+			if (supports_extended_error_counter(margining))
+				lane_errors |= FIELD_PREP(USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK,
+							  margining->results[4]);
 			seq_printf(s, "# lane 2 errors: %u\n", lane_errors);
 		}
 	}
diff --git a/drivers/thunderbolt/dma_test.c b/drivers/thunderbolt/dma_test.c
index 519c67678b08b7..bcecb0edcb81a6 100644
--- a/drivers/thunderbolt/dma_test.c
+++ b/drivers/thunderbolt/dma_test.c
@@ -215,11 +215,6 @@ static void dma_test_stop_rings(struct dma_test *dt)
 {
 	int ret;
 
-	if (dt->rx_ring)
-		tb_ring_stop(dt->rx_ring);
-	if (dt->tx_ring)
-		tb_ring_stop(dt->tx_ring);
-
 	ret = tb_xdomain_disable_paths(dt->xd, dt->tx_hopid,
 				       dt->tx_ring ? dt->tx_ring->hop : -1,
 				       dt->rx_hopid,
@@ -227,6 +222,11 @@ static void dma_test_stop_rings(struct dma_test *dt)
 	if (ret)
 		dev_warn(&dt->svc->dev, "failed to disable DMA paths\n");
 
+	if (dt->rx_ring)
+		tb_ring_stop(dt->rx_ring);
+	if (dt->tx_ring)
+		tb_ring_stop(dt->tx_ring);
+
 	dma_test_free_rings(dt);
 }
 
diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c
index 12c88509a54fa8..24611f05b3cda9 100644
--- a/drivers/thunderbolt/domain.c
+++ b/drivers/thunderbolt/domain.c
@@ -788,21 +788,6 @@ int tb_domain_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 			transmit_ring, receive_path, receive_ring);
 }
 
-static void tb_domain_reset_interface(struct tb *tb)
-{
-	struct tb_nhi *nhi = tb->nhi;
-
-	if (!nhi->ops->reset_interface)
-		return;
-
-	guard(mutex)(&tb->lock);
-
-	/* The reset clears the ring state so stop the control channel */
-	tb_ctl_stop(tb->ctl);
-	nhi->ops->reset_interface(nhi);
-	tb_ctl_start(tb->ctl);
-}
-
 /**
  * tb_domain_disconnect_xdomain_paths() - Disable DMA paths for XDomain
  * @tb: Domain disabling the DMA paths
@@ -825,20 +810,11 @@ int tb_domain_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 				       int transmit_path, int transmit_ring,
 				       int receive_path, int receive_ring)
 {
-	int ret;
-
 	if (!tb->cm_ops->disconnect_xdomain_paths)
 		return -ENOTSUPP;
 
-	ret = tb->cm_ops->disconnect_xdomain_paths(tb, xd, transmit_path,
+	return tb->cm_ops->disconnect_xdomain_paths(tb, xd, transmit_path,
 			transmit_ring, receive_path, receive_ring);
-	if (ret)
-		return ret;
-
-	if (tb->nhi->quirks & QUIRK_RESET_DMA_ON_TEARDOWN)
-		tb_domain_reset_interface(tb);
-
-	return 0;
 }
 
 static int disconnect_xdomain(struct device *dev, void *data)
diff --git a/drivers/thunderbolt/eeprom.c b/drivers/thunderbolt/eeprom.c
index 2a13fa6888ba13..2acfe83928b6d8 100644
--- a/drivers/thunderbolt/eeprom.c
+++ b/drivers/thunderbolt/eeprom.c
@@ -324,7 +324,7 @@ int tb_drom_read_uid_only(struct tb_switch *sw, u64 *uid)
 }
 
 static int tb_drom_parse_entry_generic(struct tb_switch *sw,
-		struct tb_drom_entry_header *header)
+		const struct tb_drom_entry_header *header)
 {
 	const struct tb_drom_entry_generic *entry =
 		(const struct tb_drom_entry_generic *)header;
@@ -360,7 +360,7 @@ static int tb_drom_parse_entry_generic(struct tb_switch *sw,
 }
 
 static int tb_drom_parse_entry_port(struct tb_switch *sw,
-				    struct tb_drom_entry_header *header)
+				    const struct tb_drom_entry_header *header)
 {
 	struct tb_port *port;
 	int res;
@@ -386,11 +386,13 @@ static int tb_drom_parse_entry_port(struct tb_switch *sw,
 	type &= 0xffffff;
 
 	if (type == TB_TYPE_PORT) {
-		struct tb_drom_entry_port *entry = (void *) header;
+		const struct tb_drom_entry_port *entry =
+			(const struct tb_drom_entry_port *)header;
+
 		if (header->len != sizeof(*entry)) {
 			tb_sw_warn(sw,
 				"port entry has size %#x (expected %#zx)\n",
-				header->len, sizeof(struct tb_drom_entry_port));
+				header->len, sizeof(*entry));
 			return -EIO;
 		}
 		port->link_nr = entry->link_nr;
@@ -421,9 +423,16 @@ static int tb_drom_parse_entries(struct tb_switch *sw, size_t header_size)
 	int res;
 
 	while (pos < drom_size) {
-		struct tb_drom_entry_header *entry = (void *) (sw->drom + pos);
-		if (pos + 1 == drom_size || pos + entry->len > drom_size
-				|| !entry->len) {
+		const struct tb_drom_entry_header *entry;
+
+		if (drom_size - pos < sizeof(*entry)) {
+			tb_sw_warn(sw, "DROM buffer overrun\n");
+			return -EIO;
+		}
+
+		entry = (const struct tb_drom_entry_header *)(sw->drom + pos);
+		if (entry->len < sizeof(*entry) ||
+		    entry->len > drom_size - pos) {
 			tb_sw_warn(sw, "DROM buffer overrun\n");
 			return -EIO;
 		}
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index 5809809f64d42b..e99a3fcc4a2991 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -15,6 +15,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/iommu.h>
+#include <linux/lockdep.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/property.h>
@@ -40,6 +41,7 @@ static bool host_reset = true;
 module_param(host_reset, bool, 0444);
 MODULE_PARM_DESC(host_reset, "reset USB4 host router (default: true)");
 
+/* Returns absolute bit number of the ring in the interrupt registers */
 static int ring_interrupt_index(const struct tb_ring *ring)
 {
 	int bit = ring->hop;
@@ -48,24 +50,41 @@ static int ring_interrupt_index(const struct tb_ring *ring)
 	return bit;
 }
 
-static void nhi_mask_interrupt(struct tb_nhi *nhi, int mask, int ring)
+static void nhi_mask_interrupt(struct tb_nhi *nhi, u32 mask, int reg_index)
 {
-	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT) {
-		u32 val;
+	int offset = reg_index * 4;
+	u32 val;
 
-		val = ioread32(nhi->iobase + REG_RING_INTERRUPT_BASE + ring);
-		iowrite32(val & ~mask, nhi->iobase + REG_RING_INTERRUPT_BASE + ring);
-	} else {
-		iowrite32(mask, nhi->iobase + REG_RING_INTERRUPT_MASK_CLEAR_BASE + ring);
-	}
+	/* Use shadow copy instead of reading the register */
+	val = nhi->interrupt_mask[reg_index] & ~mask;
+	nhi->interrupt_mask[reg_index] = val;
+
+	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT)
+		iowrite32(val, nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
+	else
+		iowrite32(mask, nhi->iobase + REG_RING_INTERRUPT_MASK_CLEAR_BASE + offset);
+}
+
+static void nhi_unmask_interrupt(struct tb_nhi *nhi, u32 mask, int reg_index)
+{
+	int offset = reg_index * 4;
+	u32 val;
+
+	/* Use shadow copy instead of reading the register */
+	val = nhi->interrupt_mask[reg_index] | mask;
+	nhi->interrupt_mask[reg_index] = val;
+
+	iowrite32(val, nhi->iobase + REG_RING_INTERRUPT_BASE + offset);
 }
 
-static void nhi_clear_interrupt(struct tb_nhi *nhi, int ring)
+static void nhi_clear_interrupt(struct tb_nhi *nhi, int reg_index)
 {
+	int offset = reg_index * 4;
+
 	if (nhi->quirks & QUIRK_AUTO_CLEAR_INT)
-		ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + ring);
+		ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + offset);
 	else
-		iowrite32(~0, nhi->iobase + REG_RING_INT_CLEAR + ring);
+		iowrite32(~0, nhi->iobase + REG_RING_INT_CLEAR + offset);
 }
 
 /*
@@ -75,22 +94,17 @@ static void nhi_clear_interrupt(struct tb_nhi *nhi, int ring)
  */
 static void ring_interrupt_active(struct tb_ring *ring, bool active)
 {
-	int index = ring_interrupt_index(ring) / 32 * 4;
-	int reg = REG_RING_INTERRUPT_BASE + index;
-	int interrupt_bit = ring_interrupt_index(ring) & 31;
-	int mask = 1 << interrupt_bit;
+	int interrupt_index = ring_interrupt_index(ring);
+	int reg_index = interrupt_index / 32;
+	int reg = REG_RING_INTERRUPT_BASE + reg_index * 4;
+	int interrupt_bit = interrupt_index % 32;
+	u32 mask = BIT(interrupt_bit);
 	u32 old, new;
 
 	if (ring->irq > 0) {
 		u32 step, shift, ivr, misc, itr;
 		void __iomem *ivr_base;
 		int auto_clear_bit;
-		int index;
-
-		if (ring->is_tx)
-			index = ring->hop;
-		else
-			index = ring->hop + ring->nhi->hop_count;
 
 		/*
 		 * Intel routers support a bit that isn't part of
@@ -113,8 +127,8 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 				  ring->nhi->iobase + REG_DMA_MISC);
 
 		ivr_base = ring->nhi->iobase + REG_INT_VEC_ALLOC_BASE;
-		step = index / REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
-		shift = index % REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
+		step = interrupt_index / REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
+		shift = interrupt_index % REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
 		ivr = ioread32(ivr_base + step);
 		ivr &= ~(REG_INT_VEC_ALLOC_MASK << shift);
 		if (active)
@@ -128,7 +142,7 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 			  ring->vector * 4);
 	}
 
-	old = ioread32(ring->nhi->iobase + reg);
+	old = ring->nhi->interrupt_mask[reg_index];
 	if (active)
 		new = old | mask;
 	else
@@ -138,15 +152,24 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 		"%s interrupt at register %#x bit %d (%#x -> %#x)\n",
 		active ? "enabling" : "disabling", reg, interrupt_bit, old, new);
 
-	if (new == old)
-		dev_WARN(ring->nhi->dev, "interrupt for %s %d is already %s\n",
-			 RING_TYPE(ring), ring->hop,
-			 str_enabled_disabled(active));
+	if (new == old) {
+		/*
+		 * Rings that are polled mask the interrupt while the
+		 * completions are being advanced (see __ring_interrupt())
+		 * so for those it can already be disabled by the time
+		 * the ring is stopped.
+		 */
+		if (active || !ring->start_poll)
+			dev_WARN(ring->nhi->dev,
+				 "interrupt for %s %d is already %s\n",
+				 RING_TYPE(ring), ring->hop,
+				 str_enabled_disabled(active));
+	}
 
 	if (active)
-		iowrite32(new, ring->nhi->iobase + reg);
+		nhi_unmask_interrupt(ring->nhi, mask, reg_index);
 	else
-		nhi_mask_interrupt(ring->nhi, mask, index);
+		nhi_mask_interrupt(ring->nhi, mask, reg_index);
 }
 
 /*
@@ -159,11 +182,11 @@ void nhi_disable_interrupts(struct tb_nhi *nhi)
 	int i = 0;
 	/* disable interrupts */
 	for (i = 0; i < RING_INTERRUPT_REG_COUNT(nhi); i++)
-		nhi_mask_interrupt(nhi, ~0, 4 * i);
+		nhi_mask_interrupt(nhi, ~0, i);
 
 	/* clear interrupt status bits */
 	for (i = 0; i < RING_NOTIFY_REG_COUNT(nhi); i++)
-		nhi_clear_interrupt(nhi, 4 * i);
+		nhi_clear_interrupt(nhi, i);
 }
 
 /* ring helper methods */
@@ -226,12 +249,37 @@ static bool ring_empty(struct tb_ring *ring)
 	return ring->head == ring->tail;
 }
 
+static void __ring_notify(struct tb_ring *ring)
+{
+	lockdep_assert_held(&ring->lock);
+
+	if (ring->notify_pending) {
+		/*
+		 * The doorbell carries the absolute index of the head
+		 * so a single write covers all the descriptors posted
+		 * since the previous one.
+		 */
+		if (ring->is_tx)
+			ring_iowrite_prod(ring, ring->head);
+		else
+			ring_iowrite_cons(ring, ring->head);
+	}
+	ring->notify_pending = false;
+}
+
 /*
  * ring_write_descriptors() - post frames from ring->queue to the controller
+ * @ring: Ring to post the frames to
+ * @notify: Notify the controller about the posted descriptors
+ *
+ * Unless @notify is %true the controller is not notified about the posted
+ * descriptors and the caller is expected to call tb_ring_notify() once it
+ * is done queuing frames. This allows batching of frames before
+ * updating the producer/consumer indices.
  *
  * ring->lock is held.
  */
-static void ring_write_descriptors(struct tb_ring *ring)
+static void ring_write_descriptors(struct tb_ring *ring, bool notify)
 {
 	struct ring_frame *frame, *n;
 	struct ring_desc *descriptor;
@@ -255,11 +303,11 @@ static void ring_write_descriptors(struct tb_ring *ring)
 			descriptor->sof = frame->sof;
 		}
 		ring->head = (ring->head + 1) % ring->size;
-		if (ring->is_tx)
-			ring_iowrite_prod(ring, ring->head);
-		else
-			ring_iowrite_cons(ring, ring->head);
+		ring->notify_pending = true;
 	}
+
+	if (notify)
+		__ring_notify(ring);
 }
 
 /*
@@ -304,7 +352,7 @@ static void ring_work(struct work_struct *work)
 		}
 		ring->tail = (ring->tail + 1) % ring->size;
 	}
-	ring_write_descriptors(ring);
+	ring_write_descriptors(ring, true);
 
 invoke_callback:
 	/* allow callbacks to schedule new work */
@@ -323,7 +371,7 @@ static void ring_work(struct work_struct *work)
 	wake_up(&ring->wait);
 }
 
-int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
+int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame, bool more)
 {
 	unsigned long flags;
 	int ret = 0;
@@ -331,7 +379,7 @@ int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
 	spin_lock_irqsave(&ring->lock, flags);
 	if (ring->running) {
 		list_add_tail(&frame->list, &ring->queue);
-		ring_write_descriptors(ring);
+		ring_write_descriptors(ring, !more);
 	} else {
 		ret = -ESHUTDOWN;
 	}
@@ -340,6 +388,22 @@ int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
 }
 EXPORT_SYMBOL_GPL(__tb_ring_enqueue);
 
+/**
+ * tb_ring_notify() - Notify the controller about the queued frames
+ * @ring: Ring to notify
+ *
+ * Notifies the controller about frames that were enqueued using
+ * tb_ring_tx_more() or tb_ring_rx_more(). Does nothing if there are no
+ * such frames pending.
+ */
+void tb_ring_notify(struct tb_ring *ring)
+{
+	guard(spinlock_irqsave)(&ring->lock);
+	if (ring->running)
+		__ring_notify(ring);
+}
+EXPORT_SYMBOL_GPL(tb_ring_notify);
+
 /**
  * tb_ring_poll() - Poll one completed frame from the ring
  * @ring: Ring to poll
@@ -376,6 +440,13 @@ struct ring_frame *tb_ring_poll(struct tb_ring *ring)
 		}
 
 		ring->tail = (ring->tail + 1) % ring->size;
+
+		/*
+		 * There is one more slot available so we can push next
+		 * descriptor to the ring. This is needed when the ring
+		 * is created with %RING_FLAG_NO_INTERRUPT.
+		 */
+		ring_write_descriptors(ring, true);
 	}
 
 unlock:
@@ -384,19 +455,35 @@ struct ring_frame *tb_ring_poll(struct tb_ring *ring)
 }
 EXPORT_SYMBOL_GPL(tb_ring_poll);
 
+/**
+ * tb_ring_poll_pending() - Does the ring have completed frame
+ * @ring: Ring to check
+ *
+ * Can be used to check whether there is a completed frame in the ring
+ * that next call to tb_ring_poll() returns.
+ *
+ * Return: %true if a completed frame is waiting, %false otherwise.
+ */
+bool tb_ring_poll_pending(struct tb_ring *ring)
+{
+	guard(spinlock_irqsave)(&ring->lock);
+
+	if (!ring->running || ring_empty(ring))
+		return false;
+	return !!(ring->descriptors[ring->tail].flags & RING_DESC_COMPLETED);
+}
+EXPORT_SYMBOL_GPL(tb_ring_poll_pending);
+
 static void __ring_interrupt_mask(struct tb_ring *ring, bool mask)
 {
-	int idx = ring_interrupt_index(ring);
-	int reg = REG_RING_INTERRUPT_BASE + idx / 32 * 4;
-	int bit = idx % 32;
-	u32 val;
+	int interrupt_index = ring_interrupt_index(ring);
+	int reg_index = interrupt_index / 32;
+	int interrupt_bit = interrupt_index % 32;
 
-	val = ioread32(ring->nhi->iobase + reg);
 	if (mask)
-		val &= ~BIT(bit);
+		nhi_mask_interrupt(ring->nhi, BIT(interrupt_bit), reg_index);
 	else
-		val |= BIT(bit);
-	iowrite32(val, ring->nhi->iobase + reg);
+		nhi_unmask_interrupt(ring->nhi, BIT(interrupt_bit), reg_index);
 }
 
 /* Both @nhi->lock and @ring->lock should be held */
@@ -560,6 +647,8 @@ static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size,
 	INIT_LIST_HEAD(&ring->in_flight);
 	INIT_WORK(&ring->work, ring_work);
 	init_waitqueue_head(&ring->wait);
+	lockdep_register_key(&ring->lock_key);
+	lockdep_init_map(&ring->work.lockdep_map, "ring.work", &ring->lock_key, 0);
 
 	ring->nhi = nhi;
 	ring->hop = hop;
@@ -599,6 +688,7 @@ static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size,
 			  ring->size * sizeof(*ring->descriptors),
 			  ring->descriptors, ring->descriptors_dma);
 err_free_ring:
+	lockdep_unregister_key(&ring->lock_key);
 	kfree(ring);
 
 	return NULL;
@@ -784,6 +874,7 @@ void tb_ring_stop(struct tb_ring *ring)
 	ring_iowrite32desc(ring, 0, 12);
 	ring->head = 0;
 	ring->tail = 0;
+	ring->notify_pending = false;
 	ring->running = false;
 
 err:
@@ -848,6 +939,7 @@ void tb_ring_free(struct tb_ring *ring)
 	 * to finish before freeing the ring.
 	 */
 	flush_work(&ring->work);
+	lockdep_unregister_key(&ring->lock_key);
 	kfree(ring);
 }
 EXPORT_SYMBOL_GPL(tb_ring_free);
@@ -1175,52 +1267,42 @@ static void nhi_reset(struct tb_nhi *nhi)
 	dev_warn(nhi->dev, "timeout resetting host router\n");
 }
 
-/**
- * nhi_reset_interface() - Reset the host interface
- * @nhi: Host interface to reset
- *
- * Brings the registers in the memory BAR back to their default state and
- * clears the End-to-End Flow Control state. The caller is responsible for
- * stopping the control channel over the reset because it clears the ring
- * state as well.
- */
-void nhi_reset_interface(struct tb_nhi *nhi)
-{
-	u32 val;
-
-	val = ioread32(nhi->iobase + REG_CAPS);
-	/* Only v1 host interfaces implement the reset */
-	if (FIELD_GET(REG_CAPS_VERSION_MASK, val) >= REG_CAPS_VERSION_2)
-		return;
-
-	dev_dbg(nhi->dev, "issuing host interface reset\n");
-
-	iowrite32(REG_HOST_INTERFACE_RESET_RST,
-		  nhi->iobase + REG_HOST_INTERFACE_RESET);
-	/* Wait for tHIReset (10 ms) to complete */
-	usleep_range(10000, 20000);
-}
-
 static struct tb *nhi_select_cm(struct tb_nhi *nhi)
 {
+	bool linked = false;
 	struct tb *tb;
 
 	/*
 	 * USB4 case is simple. If we got control of any of the
 	 * capabilities, we use software CM.
 	 */
-	if (tb_acpi_is_native())
-		return tb_probe(nhi);
+	if (!tb_acpi_is_native()) {
+		/*
+		 * Either firmware based CM is running (we did not get
+		 * control from the firmware) or this is pre-USB4 PC so
+		 * try first firmware CM and then fallback to software CM.
+		 */
+		tb = icm_probe(nhi);
+		if (tb)
+			return tb;
+	}
 
-	/*
-	 * Either firmware based CM is running (we did not get control
-	 * from the firmware) or this is pre-USB4 PC so try first
-	 * firmware CM and then fallback to software CM.
-	 */
-	tb = icm_probe(nhi);
+	tb = tb_probe(nhi);
 	if (!tb)
-		tb = tb_probe(nhi);
+		return NULL;
 
+	if (nhi->ops->add_links)
+		linked = nhi->ops->add_links(nhi);
+	if (!linked)
+		linked = tb_acpi_add_links(nhi);
+	/*
+	 * Device links are needed to make sure we establish tunnels
+	 * before protocol stacks are resumed so complain here if we
+	 * found them missing.
+	 */
+	if (!linked)
+		dev_warn(nhi->dev,
+			 "device links to tunneled native ports are missing!\n");
 	return tb;
 }
 
@@ -1243,7 +1325,10 @@ int nhi_probe(struct tb_nhi *nhi)
 				     sizeof(*nhi->tx_rings), GFP_KERNEL);
 	nhi->rx_rings = devm_kcalloc(dev, nhi->hop_count,
 				     sizeof(*nhi->rx_rings), GFP_KERNEL);
-	if (!nhi->tx_rings || !nhi->rx_rings)
+	nhi->interrupt_mask = devm_kcalloc(dev, RING_INTERRUPT_REG_COUNT(nhi),
+					   sizeof(*nhi->interrupt_mask),
+					   GFP_KERNEL);
+	if (!nhi->tx_rings || !nhi->rx_rings || !nhi->interrupt_mask)
 		return -ENOMEM;
 
 	nhi_reset(nhi);
diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h
index f72d6b27450189..88af9ab9fde7af 100644
--- a/drivers/thunderbolt/nhi.h
+++ b/drivers/thunderbolt/nhi.h
@@ -36,13 +36,12 @@ irqreturn_t nhi_msi(int irq, void *data);
 irqreturn_t ring_msix(int irq, void *data);
 int nhi_probe(struct tb_nhi *nhi);
 void nhi_shutdown(struct tb_nhi *nhi);
-void nhi_reset_interface(struct tb_nhi *nhi);
-
 extern const struct dev_pm_ops nhi_pm_ops;
 
 /**
  * struct tb_nhi_ops - NHI specific optional operations
  * @init: NHI specific initialization
+ * @add_links: Setup device links for tunneling native protocols
  * @suspend_noirq: NHI specific suspend_noirq hook
  * @resume_noirq: NHI specific resume_noirq hook
  * @runtime_suspend: NHI specific runtime_suspend hook
@@ -54,10 +53,10 @@ extern const struct dev_pm_ops nhi_pm_ops;
  * @release_ring_irq: NHI specific interrupt release hook
  * @is_present: Whether the device is currently present on the parent bus
  * @init_interrupts: NHI specific interrupt initialization hook
- * @reset_interface: Resets the host interface
  */
 struct tb_nhi_ops {
 	int (*init)(struct tb_nhi *nhi);
+	bool (*add_links)(struct tb_nhi *nhi);
 	int (*suspend_noirq)(struct tb_nhi *nhi, bool wakeup);
 	int (*resume_noirq)(struct tb_nhi *nhi);
 	int (*runtime_suspend)(struct tb_nhi *nhi);
@@ -69,7 +68,6 @@ struct tb_nhi_ops {
 	void (*release_ring_irq)(struct tb_ring *ring);
 	bool (*is_present)(struct tb_nhi *nhi);
 	int (*init_interrupts)(struct tb_nhi *nhi);
-	void (*reset_interface)(struct tb_nhi *nhi);
 };
 
 /*
@@ -120,24 +118,11 @@ struct tb_nhi_ops {
 #define PCI_DEVICE_ID_INTEL_PTL_P_NHI0			0xe433
 #define PCI_DEVICE_ID_INTEL_PTL_P_NHI1			0xe434
 
-#define PCI_DEVICE_ID_AMD_1AH_M60H_NHI0			0x1120
-#define PCI_DEVICE_ID_AMD_1AH_M60H_NHI1			0x1121
-#define PCI_DEVICE_ID_AMD_1AH_M68H_NHI0			0x113b
-#define PCI_DEVICE_ID_AMD_1AH_M68H_NHI1			0x113c
-#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI0			0x1155
-#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI1			0x1158
-#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI2			0x1159
-#define PCI_DEVICE_ID_AMD_1AH_M24H_NHI0			0x151c
-#define PCI_DEVICE_ID_AMD_1AH_M24H_NHI1			0x151d
-#define PCI_DEVICE_ID_AMD_1AH_M70H_NHI0			0x158d
-#define PCI_DEVICE_ID_AMD_1AH_M70H_NHI1			0x158e
-
 #define PCI_CLASS_SERIAL_USB_USB4			0x0c0340
 
 /* Host interface quirks */
-#define QUIRK_AUTO_CLEAR_INT				BIT(0)
-#define QUIRK_E2E					BIT(1)
-#define QUIRK_RESET_DMA_ON_TEARDOWN			BIT(2)
+#define QUIRK_AUTO_CLEAR_INT	BIT(0)
+#define QUIRK_E2E		BIT(1)
 
 /*
  * Minimal number of vectors when we use MSI-X. Two for control channel
diff --git a/drivers/thunderbolt/nhi_regs.h b/drivers/thunderbolt/nhi_regs.h
index 99df60b6db36a7..d6a197fabc74a2 100644
--- a/drivers/thunderbolt/nhi_regs.h
+++ b/drivers/thunderbolt/nhi_regs.h
@@ -115,10 +115,6 @@ struct ring_desc {
 #define REG_CAPS_VERSION_MASK		GENMASK(23, 16)
 #define REG_CAPS_VERSION_2		0x40
 
-/* Host Interface Reset - resets TX/RX rings and E2E flow control counters */
-#define REG_HOST_INTERFACE_RESET	0x39858
-#define REG_HOST_INTERFACE_RESET_RST	BIT(0)
-
 #define REG_DMA_MISC			0x39864
 #define REG_DMA_MISC_INT_AUTO_CLEAR     BIT(2)
 #define REG_DMA_MISC_DISABLE_AUTO_CLEAR	BIT(17)
diff --git a/drivers/thunderbolt/path.c b/drivers/thunderbolt/path.c
index b2c322e76b8ad7..05249eed3f644c 100644
--- a/drivers/thunderbolt/path.c
+++ b/drivers/thunderbolt/path.c
@@ -398,7 +398,7 @@ static int __tb_path_deactivate_hop(struct tb_port *port, int hop_index,
 		return ret;
 
 	/* Wait until it is drained */
-	timeout = ktime_add_ms(ktime_get(), 500);
+	timeout = ktime_add_ms(ktime_get(), port->pp_timeout_msec);
 	do {
 		ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
 		if (ret)
diff --git a/drivers/thunderbolt/pci.c b/drivers/thunderbolt/pci.c
index 99333729f3c2ac..4408229d8ac7b6 100644
--- a/drivers/thunderbolt/pci.c
+++ b/drivers/thunderbolt/pci.c
@@ -18,6 +18,7 @@
 #include <linux/property.h>
 #include <linux/string_helpers.h>
 #include <linux/suspend.h>
+#include <linux/platform_data/x86/apple.h>
 
 #include "nhi.h"
 #include "nhi_regs.h"
@@ -62,27 +63,6 @@ static void nhi_pci_check_quirks(struct tb_nhi_pci *nhi_pci)
 			nhi->quirks |= QUIRK_E2E;
 			break;
 		}
-	} else if (pdev->vendor == PCI_VENDOR_ID_AMD) {
-		switch (pdev->device) {
-		case PCI_DEVICE_ID_AMD_1AH_M60H_NHI0:
-		case PCI_DEVICE_ID_AMD_1AH_M60H_NHI1:
-		case PCI_DEVICE_ID_AMD_1AH_M68H_NHI0:
-		case PCI_DEVICE_ID_AMD_1AH_M68H_NHI1:
-		case PCI_DEVICE_ID_AMD_1AH_M80H_NHI0:
-		case PCI_DEVICE_ID_AMD_1AH_M80H_NHI1:
-		case PCI_DEVICE_ID_AMD_1AH_M80H_NHI2:
-		case PCI_DEVICE_ID_AMD_1AH_M24H_NHI0:
-		case PCI_DEVICE_ID_AMD_1AH_M24H_NHI1:
-		case PCI_DEVICE_ID_AMD_1AH_M70H_NHI0:
-		case PCI_DEVICE_ID_AMD_1AH_M70H_NHI1:
-			/*
-			 * These AMD hosts may hang the Tx ring when the
-			 * DMA paths are torn down so they need the host
-			 * interface reset after each teardown.
-			 */
-			nhi->quirks |= QUIRK_RESET_DMA_ON_TEARDOWN;
-			break;
-		}
 	}
 }
 
@@ -129,6 +109,82 @@ static void nhi_pci_check_iommu(struct tb_nhi_pci *nhi_pci)
 		str_enabled_disabled(port_ok));
 }
 
+static bool add_link(struct tb_nhi *nhi, struct pci_dev *pdev)
+{
+	const struct device_link *link;
+
+	link = device_link_add(&pdev->dev, nhi->dev,
+			       DL_FLAG_AUTOREMOVE_SUPPLIER |
+			       DL_FLAG_PM_RUNTIME);
+	if (!link) {
+		dev_warn(nhi->dev, "device link creation from %s failed\n",
+			 dev_name(&pdev->dev));
+		return false;
+	}
+
+	dev_dbg(nhi->dev, "created link from %s\n", dev_name(&pdev->dev));
+	return true;
+}
+
+/*
+ * During suspend the Thunderbolt controller is reset and all PCIe
+ * tunnels are lost. The NHI driver will try to reestablish all tunnels
+ * during resume. This adds device links between the tunneled PCIe
+ * downstream ports and the NHI so that the device core will make sure
+ * NHI is resumed first before the rest.
+ */
+static bool nhi_pci_add_links(struct tb_nhi *nhi)
+{
+	struct pci_dev *nhi_pdev = to_pci_dev(nhi->dev);
+	struct pci_dev *upstream, *pdev;
+	bool ret;
+
+	if (!x86_apple_machine)
+		return false;
+
+	switch (nhi_pdev->device) {
+	case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:
+	case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:
+	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
+	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
+	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
+	case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
+		break;
+	default:
+		return false;
+	}
+
+	upstream = pci_upstream_bridge(nhi_pdev);
+	while (upstream) {
+		if (!pci_is_pcie(upstream))
+			return false;
+		if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)
+			break;
+		upstream = pci_upstream_bridge(upstream);
+	}
+
+	if (!upstream)
+		return false;
+
+	/*
+	 * For each hotplug downstream port, create add device link back
+	 * to NHI so that PCIe tunnels can be re-established after
+	 * sleep.
+	 */
+	ret = false;
+	for_each_pci_bridge(pdev, upstream->subordinate) {
+		if (!pci_is_pcie(pdev))
+			continue;
+		if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM ||
+		    !pdev->is_pciehp)
+			continue;
+
+		ret |= add_link(nhi, pdev);
+	}
+
+	return ret;
+}
+
 static int nhi_pci_init_msi(struct tb_nhi *nhi)
 {
 	struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi);
@@ -272,6 +328,7 @@ static bool nhi_pci_is_present(struct tb_nhi *nhi)
 }
 
 static const struct tb_nhi_ops pci_nhi_default_ops = {
+	.add_links = nhi_pci_add_links,
 	.pre_nvm_auth = nhi_pci_start_dma_port,
 	.post_nvm_auth = nhi_pci_complete_dma_port,
 	.request_ring_irq = nhi_pci_ring_request_msix,
@@ -279,7 +336,6 @@ static const struct tb_nhi_ops pci_nhi_default_ops = {
 	.shutdown = nhi_pci_release_irq,
 	.is_present = nhi_pci_is_present,
 	.init_interrupts = nhi_pci_init_msi,
-	.reset_interface = nhi_reset_interface,
 };
 
 /* Ice Lake specific NHI operations */
@@ -443,6 +499,40 @@ static int icl_nhi_resume(struct tb_nhi *nhi)
 	return 0;
 }
 
+static bool icl_nhi_add_links(struct tb_nhi *nhi)
+{
+	struct pci_dev *nhi_pdev = to_pci_dev(nhi->dev);
+	struct pci_bus *bus = nhi_pdev->bus;
+	struct pci_dev *pdev;
+	bool ret;
+
+	if (!x86_apple_machine)
+		return false;
+
+	/*
+	 * On integrated controllers the tunneled PCIe root ports are
+	 * directly under the host bridge.
+	 */
+	if (!pci_is_root_bus(bus))
+		return false;
+
+	ret = false;
+	for_each_pci_bridge(pdev, bus) {
+		switch (nhi_pdev->device) {
+		case PCI_DEVICE_ID_INTEL_ICL_NHI0:
+			if (pdev->device == 0x8a1d || pdev->device == 0x8a1f)
+				ret |= add_link(nhi, pdev);
+			break;
+		case PCI_DEVICE_ID_INTEL_ICL_NHI1:
+			if (pdev->device == 0x8a21 || pdev->device == 0x8a23)
+				ret |= add_link(nhi, pdev);
+			break;
+		}
+	}
+
+	return ret;
+}
+
 static void icl_nhi_shutdown(struct tb_nhi *nhi)
 {
 	nhi_pci_release_irq(nhi);
@@ -452,6 +542,7 @@ static void icl_nhi_shutdown(struct tb_nhi *nhi)
 
 static const struct tb_nhi_ops icl_nhi_ops = {
 	.init = icl_nhi_resume,
+	.add_links = icl_nhi_add_links,
 	.suspend_noirq = icl_nhi_suspend_noirq,
 	.resume_noirq = icl_nhi_resume,
 	.runtime_suspend = icl_nhi_suspend,
@@ -463,7 +554,6 @@ static const struct tb_nhi_ops icl_nhi_ops = {
 	.release_ring_irq = nhi_pci_ring_release_msix,
 	.is_present = nhi_pci_is_present,
 	.init_interrupts = nhi_pci_init_msi,
-	.reset_interface = nhi_reset_interface,
 };
 
 static int nhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
diff --git a/drivers/thunderbolt/quirks.c b/drivers/thunderbolt/quirks.c
index 9f7914ac2f48cf..2722918e0e0950 100644
--- a/drivers/thunderbolt/quirks.c
+++ b/drivers/thunderbolt/quirks.c
@@ -52,6 +52,19 @@ static void quirk_block_rpm_in_redrive(struct tb_switch *sw)
 	tb_sw_dbg(sw, "preventing runtime PM in DP redrive mode\n");
 }
 
+static void quirk_stuck_pending(struct tb_switch *sw)
+{
+	struct tb_port *port;
+
+	tb_switch_for_each_port(sw, port) {
+		if (!tb_port_is_nhi(port))
+			continue;
+
+		port->pp_timeout_msec = 0;
+		tb_port_dbg(port, "pending bit does not clear, reading it once\n");
+	}
+}
+
 struct tb_quirk {
 	u16 hw_vendor_id;
 	u16 hw_device_id;
@@ -99,6 +112,9 @@ static const struct tb_quirk tb_quirks[] = {
 		  quirk_usb3_maximum_bandwidth },
 	{ 0x8087, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HUB_40G_BRIDGE, 0x0000, 0x0000,
 		  quirk_usb3_maximum_bandwidth },
+	/* CLx is unstable on the Anker Prime TB5 dock (DROM 01ea:83b5) */
+	{ 0x8087, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HUB_80G_BRIDGE, 0x01ea, 0x83b5,
+		  quirk_clx_disable },
 	/*
 	 * Block Runtime PM in DP redrive mode for Intel Barlow Ridge host
 	 * controllers.
@@ -114,6 +130,11 @@ static const struct tb_quirk tb_quirks[] = {
 	{ 0x0438, 0x0209, 0x0000, 0x0000, quirk_clx_disable },
 	{ 0x0438, 0x020a, 0x0000, 0x0000, quirk_clx_disable },
 	{ 0x0438, 0x020b, 0x0000, 0x0000, quirk_clx_disable },
+	/*
+	 * ASMedia ASM4242 never clears the Pending Packets bit of its host
+	 * interface adapter.
+	 */
+	{ 0x174c, 0x2428, 0x0000, 0x0000, quirk_stuck_pending },
 };
 
 /**
diff --git a/drivers/thunderbolt/sb_regs.h b/drivers/thunderbolt/sb_regs.h
index 5391502a4b8736..b801f3a694be9c 100644
--- a/drivers/thunderbolt/sb_regs.h
+++ b/drivers/thunderbolt/sb_regs.h
@@ -59,6 +59,7 @@ enum usb4_sb_opcode {
 #define USB4_MARGIN_CAP_0_MAX_VOLTAGE_OFFSET_MASK GENMASK(18, 13)
 #define USB4_MARGIN_CAP_0_OPT_VOLTAGE_SUPPORT	BIT(19)
 #define USB4_MARGIN_CAP_0_VOLT_STEPS_OPT_MASK	GENMASK(26, 20)
+#define USB4_MARGIN_CAP_0_EXT_ERR_COUNTER	BIT(28)
 #define USB4_MARGIN_CAP_1_MAX_VOLT_OFS_OPT_MASK GENMASK(7, 0)
 #define USB4_MARGIN_CAP_1_TIME_DESTR		BIT(8)
 #define USB4_MARGIN_CAP_1_TIME_INDP_MASK	GENMASK(10, 9)
@@ -108,5 +109,6 @@ enum usb4_sb_opcode {
 #define USB4_MARGIN_SW_ERR_COUNTER_LANE_0_MASK	GENMASK(3, 0)
 #define USB4_MARGIN_SW_ERR_COUNTER_LANE_1_MASK	GENMASK(7, 4)
 #define USB4_MARGIN_SW_ERR_COUNTER_LANE_2_MASK	GENMASK(11, 8)
+#define USB4_MARGIN_SW_EXT_ERR_COUNTER_MASK	GENMASK(15, 4)
 
 #endif
diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c
index 25c259dd07602d..4f9a57b77bfa28 100644
--- a/drivers/thunderbolt/stream.c
+++ b/drivers/thunderbolt/stream.c
@@ -131,14 +131,20 @@ struct tbstream_ring {
  * @ring_size: Size of the rings
  * @throttling: Interrupt throttling rate in ns
  * @busy_poll: Instead of interrupts, busy poll the rings
+ * @rx_pending: Receive ring has completions that need to be advanced
  * @users: Number of times @cdev has been opened
- * @closed: CLOSE packet was received
+ * @close_received: CLOSE packet appeared in the RX ring
+ * @closed: CLOSE packet was handled in the read side
  * @removed: Userspace removed the ConfigFS group underneath.
  * @wait: Waitqueue for open, read and write
  * @lock: Lock protecting this structure
  * @tx_ring: Transmit ring
  * @rx_ring: Receive ring
  * @list: Stream devices are linked through this
+ *
+ * @close_received is used on write side to notify the writer that the
+ * other side sent CLOSE. @closed on the other hand is used on the read
+ * side to make read(2) return EOF to the caller.
  */
 struct tbstream_dev {
 	struct config_group group;
@@ -151,7 +157,9 @@ struct tbstream_dev {
 	unsigned int ring_size;
 	unsigned int throttling;
 	bool busy_poll;
+	bool rx_pending;
 	int users;
+	bool close_received;
 	bool closed;
 	bool removed;
 	wait_queue_head_t wait;
@@ -278,6 +286,14 @@ static inline bool tbstream_ring_available(const struct tbstream_ring *ring)
 	return ring->prod > ring->cons;
 }
 
+static void tbstream_ring_poll(struct tbstream_ring *ring)
+{
+	struct ring_frame *frame;
+
+	while ((frame = tb_ring_poll(ring->ring)))
+		frame->callback(ring->ring, frame, false);
+}
+
 static inline struct tb_xdomain *tbstream_dev_xdomain(struct tbstream_dev *sdev)
 {
 	if (sdev->stream)
@@ -338,9 +354,19 @@ static inline bool tbstream_dev_removed(const struct tbstream_dev *sdev)
 	return sdev->removed;
 }
 
+static inline bool tbstream_dev_close_received(const struct tbstream_dev *sdev)
+{
+	return READ_ONCE(sdev->close_received);
+}
+
 static inline bool tbstream_dev_closed(const struct tbstream_dev *sdev)
 {
-	return sdev->closed;
+	return READ_ONCE(sdev->closed);
+}
+
+static inline bool tbstream_dev_rx_pending(const struct tbstream_dev *sdev)
+{
+	return READ_ONCE(sdev->rx_pending);
 }
 
 static void
@@ -349,6 +375,7 @@ tbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame,
 {
 	struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);
 	struct tbstream_dev *sdev = sf->sdev;
+	__poll_t mask;
 
 	if (canceled)
 		return;
@@ -356,12 +383,17 @@ tbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame,
 	sf->completed = true;
 	sdev->rx_ring.prod++;
 
-	if (sf->frame.flags & RING_DESC_CRC_ERROR)
-		pr_warn("RX CRC error\n");
-	else if (sf->frame.flags & RING_DESC_BUFFER_OVERRUN)
-		pr_warn("RX buffer overrun\n");
-	else
-		wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);
+	mask = EPOLLIN | EPOLLRDNORM;
+	/*
+	 * The CLOSE packet does not have a payload so the flags do not
+	 * matter. The read_iter() deals with the flags.
+	 */
+	if (sf->frame.eof == TBSTREAM_CLOSE) {
+		WRITE_ONCE(sdev->close_received, true);
+		mask |= EPOLLHUP;
+	}
+
+	wake_up_interruptible_poll(&sdev->wait, mask);
 }
 
 static struct tbstream_frame *
@@ -538,38 +570,27 @@ tbstream_dev_send_data(struct tbstream_dev *sdev, struct iov_iter *from,
 	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
 }
 
-static void
-tbstream_dev_poll_ring(struct tbstream_dev *sdev, struct tbstream_ring *ring)
-{
-	struct ring_frame *frame;
-
-	if (!sdev->busy_poll)
-		return;
-
-	while ((frame = tb_ring_poll(ring->ring)))
-		frame->callback(ring->ring, frame, false);
-}
-
 static int tbstream_dev_send_close(struct tbstream_dev *sdev)
 {
 	struct tbstream_frame *sf;
+	ktime_t timeout;
 
-	if (sdev->busy_poll) {
+	/*
+	 * Wait for the ring to have available slots before we send the
+	 * CLOSE packet.
+	 */
+	timeout = ktime_add_ms(ktime_get(), 500);
+	do {
+		if (tbstream_ring_available(&sdev->tx_ring))
+			break;
 		/*
-		 * When busy polling it's the write(2) path that
-		 * advances the completions so it is possible that the
-		 * ring is full at this point. Advance the ring here so
-		 * that there is room for the CLOSE packet to be sent.
+		 * For busy polling we need to advance the ring here as
+		 * well to make the slots available.
 		 */
-		ktime_t timeout = ktime_add_ms(ktime_get(), 500);
-
-		do {
-			if (tbstream_ring_available(&sdev->tx_ring))
-				break;
-			tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
-			fsleep(15);
-		} while (ktime_before(ktime_get(), timeout));
-	}
+		if (sdev->busy_poll)
+			tbstream_ring_poll(&sdev->tx_ring);
+		fsleep(15);
+	} while (ktime_before(ktime_get(), timeout));
 
 	sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_CLOSE, NULL, SZ_256);
 	if (IS_ERR(sf))
@@ -577,16 +598,45 @@ static int tbstream_dev_send_close(struct tbstream_dev *sdev)
 	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
 }
 
+static void tbstream_dev_start_poll(void *data)
+{
+	struct tbstream_dev *sdev = data;
+
+	WRITE_ONCE(sdev->rx_pending, true);
+	wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);
+}
+
+/* sdev->lock must be held */
+static void tbstream_dev_advance_rx(struct tbstream_dev *sdev)
+{
+	/*
+	 * Clear before running the completions so that an interrupt
+	 * that arrives while we are doing that is not missed.
+	 */
+	WRITE_ONCE(sdev->rx_pending, false);
+	tbstream_ring_poll(&sdev->rx_ring);
+}
+
+/* sdev->lock must be held */
+static void tbstream_dev_complete_rx(struct tbstream_dev *sdev)
+{
+	if (!sdev->busy_poll)
+		tb_ring_poll_complete(sdev->rx_ring.ring);
+}
+
 static int tbstream_dev_start(struct tbstream_dev *sdev)
 {
 	struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
 	unsigned int flags = RING_FLAG_FRAME | RING_FLAG_E2E;
+	void (*start_poll)(void *) = NULL;
 	u16 sof_mask, eof_mask;
 	struct tb_ring *ring;
 	int ret, e2e_tx_hop;
 
 	if (sdev->busy_poll)
 		flags |= RING_FLAG_NO_INTERRUPT;
+	else
+		start_poll = tbstream_dev_start_poll;
 
 	ring = tb_ring_alloc_tx(xd->tb->nhi, -1, sdev->ring_size, flags);
 	if (!ring)
@@ -602,7 +652,8 @@ static int tbstream_dev_start(struct tbstream_dev *sdev)
 	eof_mask = BIT(TBSTREAM_DATA) | BIT(TBSTREAM_CLOSE);
 
 	ring = tb_ring_alloc_rx(xd->tb->nhi, -1, sdev->ring_size, flags,
-				e2e_tx_hop, sof_mask, eof_mask, NULL, NULL);
+				e2e_tx_hop, sof_mask, eof_mask, start_poll,
+				sdev);
 	if (!ring) {
 		ret = -ENOMEM;
 		goto err_free_tx_buffers;
@@ -619,6 +670,8 @@ static int tbstream_dev_start(struct tbstream_dev *sdev)
 	tb_ring_throttling(sdev->tx_ring.ring, sdev->throttling);
 	tb_ring_throttling(sdev->rx_ring.ring, sdev->throttling);
 
+	sdev->rx_pending = false;
+
 	tb_ring_start(sdev->tx_ring.ring);
 	tb_ring_start(sdev->rx_ring.ring);
 
@@ -665,19 +718,16 @@ static void tbstream_dev_stop(struct tbstream_dev *sdev)
 		do {
 			if (tbstream_dev_tx_drained(sdev))
 				break;
-			tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
+			tbstream_ring_poll(&sdev->tx_ring);
 			fsleep(15);
 		} while (ktime_before(ktime_get(), timeout));
-
-		tb_ring_stop(sdev->tx_ring.ring);
-		tb_ring_stop(sdev->rx_ring.ring);
 	} else {
 		tb_ring_flush(sdev->tx_ring.ring, 500);
-		tb_ring_stop(sdev->tx_ring.ring);
-		tb_ring_flush(sdev->rx_ring.ring, 500);
-		tb_ring_stop(sdev->rx_ring.ring);
 	}
 
+	tb_ring_stop(sdev->tx_ring.ring);
+	tb_ring_stop(sdev->rx_ring.ring);
+
 	xd = tbstream_dev_xdomain(sdev);
 	if (xd) {
 		tb_xdomain_disable_paths(xd, sdev->out_hopid,
@@ -707,6 +757,45 @@ static int tbstream_dev_lock(struct tbstream_dev *sdev, bool nowait)
 	return 0;
 }
 
+/* Must not be called with @sdev->lock held */
+static int tbstream_dev_busy_poll_wait(struct tbstream_dev *sdev,
+				       struct tbstream_ring *ring)
+{
+	for (;;) {
+		if (signal_pending(current))
+			return -ERESTARTSYS;
+		if (tb_ring_poll_pending(ring->ring))
+			return 0;
+		/*
+		 * For TX ring we need to check the RX side too because
+		 * it might have received CLOSE packet.
+		 */
+		if (ring == &sdev->tx_ring &&
+		    tb_ring_poll_pending(sdev->rx_ring.ring))
+			return 0;
+		if (tbstream_dev_valid(sdev) != 0 ||
+		    tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev))
+			return 0;
+		cond_resched();
+	}
+}
+
+static bool
+tbstream_dev_has_event(struct tbstream_dev *sdev, struct tbstream_ring *ring)
+{
+	if (tbstream_dev_valid(sdev) != 0)
+		return true;
+	if (tbstream_dev_closed(sdev))
+		return true;
+	if (tbstream_dev_removed(sdev))
+		return true;
+	if (tbstream_dev_rx_pending(sdev))
+		return true;
+	if (tbstream_ring_available(ring))
+		return true;
+	return tb_ring_poll_pending(sdev->rx_ring.ring);
+}
+
 static ssize_t
 tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 {
@@ -725,8 +814,8 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 		return ret;
 
 	for (;;) {
-		/* When busy polling, advance any completions manually */
-		tbstream_dev_poll_ring(sdev, &sdev->rx_ring);
+		/* Advance RX completions */
+		tbstream_dev_advance_rx(sdev);
 
 		ret = tbstream_dev_valid(sdev);
 		if (ret) {
@@ -742,21 +831,20 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 		if (tbstream_ring_available(&sdev->rx_ring))
 			break;
 
+		/* Polled all we could. Re-enable the interrupt now. */
+		tbstream_dev_complete_rx(sdev);
 		mutex_unlock(&sdev->lock);
 
 		if (nowait)
 			return -EAGAIN;
 
 		if (sdev->busy_poll) {
-			if (signal_pending(current))
-				return -ERESTARTSYS;
-			cond_resched();
+			ret = tbstream_dev_busy_poll_wait(sdev, &sdev->rx_ring);
+			if (ret)
+				return ret;
 		} else {
 			ret = wait_event_interruptible(sdev->wait,
-					tbstream_ring_available(&sdev->rx_ring) ||
-					tbstream_dev_valid(sdev) != 0 ||
-					tbstream_dev_closed(sdev) ||
-					tbstream_dev_removed(sdev));
+				tbstream_dev_has_event(sdev, &sdev->rx_ring));
 			if (ret)
 				return ret;
 		}
@@ -783,7 +871,20 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 		if (sf->frame.eof == TBSTREAM_CLOSE) {
 			if (!nbytes) {
 				tbstream_dev_consume_rx(sdev);
-				sdev->closed = true;
+				WRITE_ONCE(sdev->closed, true);
+			}
+			break;
+		} else if (sf->frame.flags &
+			   (RING_DESC_CRC_ERROR | RING_DESC_BUFFER_OVERRUN)) {
+			/*
+			 * If something was already read return that now
+			 * and next read will report the error.
+			 */
+			if (!nbytes) {
+				pr_warn("corrupted frame received, flags %#x\n",
+					sf->frame.flags);
+				tbstream_dev_consume_rx(sdev);
+				ret = -EIO;
 			}
 			break;
 		}
@@ -819,6 +920,25 @@ tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
 	return nbytes;
 }
 
+static void tbstream_dev_advance_both(struct tbstream_dev *sdev)
+{
+	/*
+	 * When busy polling, advance TX completions manually.
+	 *
+	 * We also need to advance the RX side in both modes to be able
+	 * to receive CLOSE packet from the other peer if there is no
+	 * reader.
+	 */
+	if (sdev->busy_poll) {
+		tbstream_ring_poll(&sdev->tx_ring);
+		tbstream_dev_advance_rx(sdev);
+	} else if (tbstream_dev_rx_pending(sdev) ||
+		   tb_ring_poll_pending(sdev->rx_ring.ring)) {
+		tbstream_dev_advance_rx(sdev);
+		tbstream_dev_complete_rx(sdev);
+	}
+}
+
 static ssize_t
 tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 {
@@ -837,7 +957,8 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 		return ret;
 
 	for (;;) {
-		tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
+		/* Advance TX (and RX) completions */
+		tbstream_dev_advance_both(sdev);
 
 		ret = tbstream_dev_valid(sdev);
 		if (ret) {
@@ -845,7 +966,12 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 			return ret;
 		}
 
-		if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) {
+		if (tbstream_dev_close_received(sdev)) {
+			mutex_unlock(&sdev->lock);
+			return -EPIPE;
+		}
+
+		if (tbstream_dev_removed(sdev)) {
 			mutex_unlock(&sdev->lock);
 			return -ENXIO;
 		}
@@ -859,15 +985,13 @@ tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
 			return -EAGAIN;
 
 		if (sdev->busy_poll) {
-			if (signal_pending(current))
-				return -ERESTARTSYS;
-			cond_resched();
+			ret = tbstream_dev_busy_poll_wait(sdev, &sdev->tx_ring);
+			if (ret)
+				return ret;
 		} else {
 			ret = wait_event_interruptible(sdev->wait,
-					tbstream_ring_available(&sdev->tx_ring) ||
-					tbstream_dev_valid(sdev) != 0 ||
-					tbstream_dev_closed(sdev) ||
-					tbstream_dev_removed(sdev));
+				tbstream_dev_has_event(sdev, &sdev->tx_ring) ||
+				tbstream_dev_close_received(sdev));
 			if (ret)
 				return ret;
 		}
@@ -917,14 +1041,28 @@ tbstream_dev_fops_poll(struct file *file, struct poll_table_struct *wait)
 
 	poll_wait(file, &sdev->wait, wait);
 	guard(mutex)(&sdev->lock);
-	if (tbstream_dev_valid(sdev) != 0) {
-		mask |= EPOLLHUP | EPOLLERR;
+	if (tbstream_dev_valid(sdev) != 0)
+		return EPOLLHUP | EPOLLERR;
+
+	/*
+	 * The RX completions are only advanced from here and from
+	 * read(2) so do that now, otherwise we would never report
+	 * anything to be available.
+	 */
+	tbstream_dev_advance_rx(sdev);
+
+	if (tbstream_dev_close_received(sdev))
+		mask |= EPOLLHUP;
+	if (tbstream_ring_available(&sdev->tx_ring))
+		mask |= EPOLLOUT | EPOLLWRNORM;
+	if (tbstream_ring_available(&sdev->rx_ring)) {
+		mask |= EPOLLIN | EPOLLRDNORM;
 	} else {
-		if (tbstream_ring_available(&sdev->tx_ring))
-			mask |= EPOLLOUT | EPOLLWRNORM;
-		if (tbstream_ring_available(&sdev->rx_ring))
+		tbstream_dev_complete_rx(sdev);
+		if (tb_ring_poll_pending(sdev->rx_ring.ring))
 			mask |= EPOLLIN | EPOLLRDNORM;
 	}
+
 	return mask;
 }
 
@@ -974,10 +1112,15 @@ static int tbstream_dev_fops_open(struct inode *inode, struct file *file)
 			sdev->users--;
 			goto err_unlock;
 		}
+
+		sdev->close_received = false;
 		sdev->closed = false;
 	}
 
 	mutex_unlock(&sdev->lock);
+
+	/* Stream handles IOCB_NOWAIT just fine */
+	file->f_mode |= FMODE_NOWAIT;
 	return 0;
 
 err_unlock:
@@ -994,12 +1137,19 @@ static int tbstream_dev_fops_release(struct inode *inode, struct file *file)
 
 	mutex_lock(&sdev->lock);
 	if (--sdev->users == 0) {
+		/*
+		 * Advance now in case there is CLOSE waiting in the RX
+		 * ring.
+		 */
+		tbstream_dev_advance_both(sdev);
 		/*
 		 * Send CLOSE tunneled packet to notify the other end
-		 * that we are closing the file. We do this twice if the
-		 * first one fails.
+		 * that we are closing the file, if it is not closed
+		 * already.
 		 */
-		tbstream_dev_send_close(sdev);
+		if (!tbstream_dev_close_received(sdev) &&
+		    tbstream_dev_valid(sdev) == 0)
+			tbstream_dev_send_close(sdev);
 		tbstream_dev_stop(sdev);
 	}
 	mutex_unlock(&sdev->lock);
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index 404c0693df50e7..d22d8db8f89064 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -19,6 +19,9 @@
 
 #include "tb.h"
 
+/* How long a hop is given to drain when a path is deactivated */
+#define TB_PORT_PENDING_TIMEOUT		500 /* ms */
+
 /* Switch NVM support */
 
 struct nvm_auth_status {
@@ -712,6 +715,7 @@ static int tb_init_port(struct tb_port *port)
 	int cap;
 
 	INIT_LIST_HEAD(&port->list);
+	port->pp_timeout_msec = TB_PORT_PENDING_TIMEOUT;
 
 	/* Control adapter does not have configuration space */
 	if (!port->port)
@@ -774,6 +778,7 @@ static int tb_port_alloc_hopid(struct tb_port *port, bool in, int min_hopid,
 {
 	int port_max_hopid;
 	struct ida *ida;
+	int ret;
 
 	if (in) {
 		port_max_hopid = port->config.max_in_hop_id;
@@ -793,7 +798,11 @@ static int tb_port_alloc_hopid(struct tb_port *port, bool in, int min_hopid,
 	if (max_hopid < 0 || max_hopid > port_max_hopid)
 		max_hopid = port_max_hopid;
 
-	return ida_alloc_range(ida, min_hopid, max_hopid, GFP_KERNEL);
+	ret = ida_alloc_range(ida, min_hopid, max_hopid, GFP_KERNEL);
+	if (ret >= 0)
+		tb_switch_get(port->sw);
+
+	return ret;
 }
 
 /**
@@ -832,6 +841,7 @@ int tb_port_alloc_out_hopid(struct tb_port *port, int min_hopid, int max_hopid)
 void tb_port_release_in_hopid(struct tb_port *port, int hopid)
 {
 	ida_free(&port->in_hopids, hopid);
+	tb_switch_put(port->sw);
 }
 
 /**
@@ -842,6 +852,7 @@ void tb_port_release_in_hopid(struct tb_port *port, int hopid)
 void tb_port_release_out_hopid(struct tb_port *port, int hopid)
 {
 	ida_free(&port->out_hopids, hopid);
+	tb_switch_put(port->sw);
 }
 
 static inline bool tb_switch_is_reachable(const struct tb_switch *parent,
diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c
index 47753a5c0f2ebd..4da608ccbccbf8 100644
--- a/drivers/thunderbolt/tb.c
+++ b/drivers/thunderbolt/tb.c
@@ -10,7 +10,6 @@
 #include <linux/errno.h>
 #include <linux/delay.h>
 #include <linux/pm_runtime.h>
-#include <linux/platform_data/x86/apple.h>
 
 #include "tb.h"
 #include "tb_regs.h"
@@ -89,6 +88,7 @@ static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port,
 				       const char *reason);
 static void tb_queue_dp_bandwidth_request(struct tb *tb, u64 route, u8 port,
 					  int retry, unsigned long delay);
+static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data);
 
 static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)
 {
@@ -385,7 +385,8 @@ static void tb_switch_discover_tunnels(struct tb_switch *sw,
 
 		switch (port->config.type) {
 		case TB_TYPE_DP_HDMI_IN:
-			tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids);
+			tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids,
+						       tb_dp_tunnel_active, tb);
 			tb_increase_tmu_accuracy(tunnel);
 			break;
 
@@ -1910,6 +1911,18 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data)
 	struct tb *tb = data;
 
 	mutex_lock(&tb->lock);
+
+	/*
+	 * If the DPRX read was canceled the tunnel is already being torn
+	 * down by whoever canceled it. Do not touch the adapters here
+	 * because the routers may be gone by now.
+	 */
+	if (tunnel->dprx_canceled) {
+		tb_tunnel_dbg(tunnel, "DPRX read canceled, not activating\n");
+		mutex_unlock(&tb->lock);
+		return;
+	}
+
 	if (tb_tunnel_is_active(tunnel)) {
 		int consumed_up, consumed_down, ret;
 
@@ -1964,8 +1977,6 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data)
 		tb_dp_resource_unavailable(tb, in, "DPRX negotiation failed");
 	}
 	mutex_unlock(&tb->lock);
-
-	tb_domain_put(tb);
 }
 
 static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in,
@@ -2026,8 +2037,7 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in,
 	       available_up, available_down);
 
 	tunnel = tb_tunnel_alloc_dp(tb, in, out, link_nr, available_up,
-				    available_down, tb_dp_tunnel_active,
-				    tb_domain_get(tb));
+				    available_down, tb_dp_tunnel_active, tb);
 	if (!tunnel) {
 		tb_port_dbg(out, "could not allocate DP tunnel\n");
 		goto err_reclaim_usb;
@@ -2048,7 +2058,6 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in,
 	tb_tunnel_put(tunnel);
 err_reclaim_usb:
 	tb_reclaim_usb3_bandwidth(tb, in, out);
-	tb_domain_put(tb);
 err_detach_group:
 	tb_detach_bandwidth_group(in);
 err_dealloc_dp:
@@ -2950,11 +2959,12 @@ static void tb_stop(struct tb *tb)
 	/* tunnels are only present after everything has been initialized */
 	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
 		/*
-		 * DMA tunnels require the driver to be functional so we
-		 * tear them down. Other protocol tunnels can be left
-		 * intact.
+		 * DMA tunnels and DP tunnels which are not yet active require
+		 * the driver to be functional so we tear them down.
+		 * Other protocol tunnels can be left intact.
 		 */
-		if (tb_tunnel_is_dma(tunnel))
+		if (tb_tunnel_is_dma(tunnel) ||
+		    (tb_tunnel_is_dp(tunnel) && !tb_tunnel_is_active(tunnel)))
 			tb_tunnel_deactivate(tunnel);
 		tb_tunnel_put(tunnel);
 	}
@@ -3274,11 +3284,11 @@ static void tb_remove_work(struct work_struct *work)
 	struct tb *tb = tcm_to_tb(tcm);
 
 	mutex_lock(&tb->lock);
-	if (tb->root_switch)
+	if (tb->root_switch) {
 		tb_free_unplugged_children(tb->root_switch);
+		tb_free_unplugged_xdomains(tb->root_switch);
+	}
 	mutex_unlock(&tb->lock);
-
-	tb_free_unplugged_xdomains(tb->root_switch);
 }
 
 static int tb_runtime_resume(struct tb *tb)
@@ -3323,75 +3333,6 @@ static const struct tb_cm_ops tb_cm_ops = {
 	.disconnect_xdomain_paths = tb_disconnect_xdomain_paths,
 };
 
-/*
- * During suspend the Thunderbolt controller is reset and all PCIe
- * tunnels are lost. The NHI driver will try to reestablish all tunnels
- * during resume. This adds device links between the tunneled PCIe
- * downstream ports and the NHI so that the device core will make sure
- * NHI is resumed first before the rest.
- */
-static bool tb_apple_add_links(struct tb_nhi *nhi)
-{
-	struct pci_dev *nhi_pdev = to_pci_dev(nhi->dev);
-	struct pci_dev *upstream, *pdev;
-	bool ret;
-
-	if (!x86_apple_machine)
-		return false;
-
-	switch (nhi_pdev->device) {
-	case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:
-	case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:
-	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
-	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
-		break;
-	default:
-		return false;
-	}
-
-	upstream = pci_upstream_bridge(nhi_pdev);
-	while (upstream) {
-		if (!pci_is_pcie(upstream))
-			return false;
-		if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)
-			break;
-		upstream = pci_upstream_bridge(upstream);
-	}
-
-	if (!upstream)
-		return false;
-
-	/*
-	 * For each hotplug downstream port, create add device link
-	 * back to NHI so that PCIe tunnels can be re-established after
-	 * sleep.
-	 */
-	ret = false;
-	for_each_pci_bridge(pdev, upstream->subordinate) {
-		const struct device_link *link;
-
-		if (!pci_is_pcie(pdev))
-			continue;
-		if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM ||
-		    !pdev->is_pciehp)
-			continue;
-
-		link = device_link_add(&pdev->dev, nhi->dev,
-				       DL_FLAG_AUTOREMOVE_SUPPLIER |
-				       DL_FLAG_PM_RUNTIME);
-		if (link) {
-			dev_dbg(nhi->dev, "created link from %s\n",
-				dev_name(&pdev->dev));
-			ret = true;
-		} else {
-			dev_warn(nhi->dev, "device link creation from %s failed\n",
-				 dev_name(&pdev->dev));
-		}
-	}
-
-	return ret;
-}
-
 struct tb *tb_probe(struct tb_nhi *nhi)
 {
 	struct tb_cm *tcm;
@@ -3416,13 +3357,5 @@ struct tb *tb_probe(struct tb_nhi *nhi)
 
 	tb_dbg(tb, "using software connection manager\n");
 
-	/*
-	 * Device links are needed to make sure we establish tunnels
-	 * before the PCIe/USB stack is resumed so complain here if we
-	 * found them missing.
-	 */
-	if (!tb_apple_add_links(nhi) && !tb_acpi_add_links(nhi))
-		tb_warn(tb, "device links to tunneled native ports are missing!\n");
-
 	return tb;
 }
diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h
index 4373336d9425e3..e86845762e226c 100644
--- a/drivers/thunderbolt/tb.h
+++ b/drivers/thunderbolt/tb.h
@@ -273,6 +273,8 @@ struct tb_bandwidth_group {
  * @max_bw: Maximum possible bandwidth through this adapter if set to
  *	    non-zero.
  * @redrive: For DP IN, if true the adapter is in redrive mode.
+ * @pp_timeout_msec: How long a hop of this adapter is given to drain when a
+ *		     path is deactivated. %0 means a single read.
  *
  * In USB4 terminology this structure represents an adapter (protocol or
  * lane adapter).
@@ -302,6 +304,7 @@ struct tb_port {
 	struct list_head group_list;
 	unsigned int max_bw;
 	bool redrive;
+	unsigned int pp_timeout_msec;
 };
 
 /**
@@ -1439,7 +1442,7 @@ int usb4_port_sw_margin(struct tb_port *port, enum usb4_sb_target target,
 			u8 index, const struct usb4_port_margining_params *params,
 			u32 *results);
 int usb4_port_sw_margin_errors(struct tb_port *port, enum usb4_sb_target target,
-			       u8 index, u32 *errors);
+			       u8 index, u32 *errors, size_t dwords);
 
 int usb4_port_retimer_set_inbound_sbtx(struct tb_port *port, u8 index);
 int usb4_port_retimer_unset_inbound_sbtx(struct tb_port *port, u8 index);
diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c
index 05652ee82fbf88..c8c648f31107ff 100644
--- a/drivers/thunderbolt/test.c
+++ b/drivers/thunderbolt/test.c
@@ -33,6 +33,18 @@ static void kunit_ida_init(struct kunit *test, struct ida *ida)
 	kunit_alloc_resource(test, __ida_init, __ida_destroy, GFP_KERNEL, ida);
 }
 
+static void tb_test_switch_release(struct device *dev)
+{
+	/* The memory is owned by KUnit, nothing to do here */
+}
+
+static void tb_test_switch_put(void *data)
+{
+	struct tb_switch *sw = data;
+
+	put_device(&sw->dev);
+}
+
 static struct tb_switch *alloc_switch(struct kunit *test, u64 route,
 				      u8 upstream_port, u8 max_port_number)
 {
@@ -44,6 +56,15 @@ static struct tb_switch *alloc_switch(struct kunit *test, u64 route,
 	if (!sw)
 		return NULL;
 
+	/*
+	 * HopID allocations take a reference to their routers and those devices
+	 * have to be initialized for that to work.
+	 */
+	sw->dev.release = tb_test_switch_release;
+	device_initialize(&sw->dev);
+	if (kunit_add_action_or_reset(test, tb_test_switch_put, sw))
+		return NULL;
+
 	sw->config.upstream_port_number = upstream_port;
 	sw->config.depth = tb_route_length(route);
 	sw->config.route_hi = upper_32_bits(route);
@@ -1386,6 +1407,10 @@ static void tb_test_tunnel_pcie(struct kunit *test)
 	tb_tunnel_put(tunnel1);
 }
 
+static void tb_test_dp_tunnel_active(struct tb_tunnel *tunnel, void *data)
+{
+}
+
 static void tb_test_tunnel_dp(struct kunit *test)
 {
 	struct tb_switch *host, *dev;
@@ -1406,7 +1431,8 @@ static void tb_test_tunnel_dp(struct kunit *test)
 	in = &host->ports[5];
 	out = &dev->ports[13];
 
-	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+				    tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, tunnel);
 	KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP);
 	KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in);
@@ -1452,7 +1478,8 @@ static void tb_test_tunnel_dp_chain(struct kunit *test)
 	in = &host->ports[5];
 	out = &dev4->ports[14];
 
-	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+				    tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, tunnel);
 	KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP);
 	KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in);
@@ -1502,7 +1529,8 @@ static void tb_test_tunnel_dp_tree(struct kunit *test)
 	in = &dev2->ports[13];
 	out = &dev5->ports[13];
 
-	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+				    tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, tunnel);
 	KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP);
 	KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in);
@@ -1567,7 +1595,8 @@ static void tb_test_tunnel_dp_max_length(struct kunit *test)
 	in = &dev6->ports[13];
 	out = &dev12->ports[13];
 
-	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+				    tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, tunnel);
 	KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP);
 	KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in);
@@ -1637,7 +1666,8 @@ static void tb_test_tunnel_3dp(struct kunit *test)
 	out2 = &dev5->ports[13];
 	out3 = &dev4->ports[14];
 
-	tunnel1 = tb_tunnel_alloc_dp(NULL, in1, out1, 1, 0, 0, NULL, NULL);
+	tunnel1 = tb_tunnel_alloc_dp(NULL, in1, out1, 1, 0, 0,
+				     tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_TRUE(test, tunnel1 != NULL);
 	KUNIT_EXPECT_EQ(test, tunnel1->type, TB_TUNNEL_DP);
 	KUNIT_EXPECT_PTR_EQ(test, tunnel1->src_port, in1);
@@ -1645,7 +1675,8 @@ static void tb_test_tunnel_3dp(struct kunit *test)
 	KUNIT_ASSERT_EQ(test, tunnel1->npaths, 3);
 	KUNIT_ASSERT_EQ(test, tunnel1->paths[0]->path_length, 3);
 
-	tunnel2 = tb_tunnel_alloc_dp(NULL, in2, out2, 1, 0, 0, NULL, NULL);
+	tunnel2 = tb_tunnel_alloc_dp(NULL, in2, out2, 1, 0, 0,
+				     tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_TRUE(test, tunnel2 != NULL);
 	KUNIT_EXPECT_EQ(test, tunnel2->type, TB_TUNNEL_DP);
 	KUNIT_EXPECT_PTR_EQ(test, tunnel2->src_port, in2);
@@ -1653,7 +1684,8 @@ static void tb_test_tunnel_3dp(struct kunit *test)
 	KUNIT_ASSERT_EQ(test, tunnel2->npaths, 3);
 	KUNIT_ASSERT_EQ(test, tunnel2->paths[0]->path_length, 4);
 
-	tunnel3 = tb_tunnel_alloc_dp(NULL, in3, out3, 1, 0, 0, NULL, NULL);
+	tunnel3 = tb_tunnel_alloc_dp(NULL, in3, out3, 1, 0, 0,
+				     tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_TRUE(test, tunnel3 != NULL);
 	KUNIT_EXPECT_EQ(test, tunnel3->type, TB_TUNNEL_DP);
 	KUNIT_EXPECT_PTR_EQ(test, tunnel3->src_port, in3);
@@ -1751,7 +1783,8 @@ static void tb_test_tunnel_port_on_path(struct kunit *test)
 	in = &dev2->ports[13];
 	out = &dev5->ports[13];
 
-	dp_tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	dp_tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+				       tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, dp_tunnel);
 
 	KUNIT_EXPECT_TRUE(test, tb_tunnel_port_on_path(dp_tunnel, in));
@@ -2183,7 +2216,8 @@ static void tb_test_credit_alloc_dp(struct kunit *test)
 	in = &host->ports[5];
 	out = &dev->ports[14];
 
-	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+				    tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, tunnel);
 	KUNIT_ASSERT_EQ(test, tunnel->npaths, (size_t)3);
 
@@ -2419,7 +2453,8 @@ static struct tb_tunnel *TB_TEST_DP_TUNNEL1(struct kunit *test,
 
 	in = &host->ports[5];
 	out = &dev->ports[13];
-	dp_tunnel1 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	dp_tunnel1 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+					tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, dp_tunnel1);
 	KUNIT_ASSERT_EQ(test, dp_tunnel1->npaths, (size_t)3);
 
@@ -2456,7 +2491,8 @@ static struct tb_tunnel *TB_TEST_DP_TUNNEL2(struct kunit *test,
 
 	in = &host->ports[6];
 	out = &dev->ports[14];
-	dp_tunnel2 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL);
+	dp_tunnel2 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0,
+					tb_test_dp_tunnel_active, NULL);
 	KUNIT_ASSERT_NOT_NULL(test, dp_tunnel2);
 	KUNIT_ASSERT_EQ(test, dp_tunnel2->npaths, (size_t)3);
 
diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c
index 7e8284575dffb1..ffd2d04b3bfa32 100644
--- a/drivers/thunderbolt/tunnel.c
+++ b/drivers/thunderbolt/tunnel.c
@@ -510,6 +510,7 @@ struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down,
 		goto err_deactivate;
 	}
 
+	tb_tunnel_set_active(tunnel, true);
 	tb_tunnel_dbg(tunnel, "discovered\n");
 	return tunnel;
 
@@ -1093,8 +1094,14 @@ static void tb_dp_dprx_work(struct work_struct *work)
 	struct tb_tunnel *tunnel = container_of(work, typeof(*tunnel), dprx_work.work);
 	struct tb *tb = tunnel->tb;
 
+	/*
+	 * The DPRX read can be canceled while this work is waiting for
+	 * tb->lock. Check the flag only once it is held: while the lock is
+	 * held the tunnel cannot be torn down under us and the adapters are
+	 * safe to access.
+	 */
+	mutex_lock(&tb->lock);
 	if (!tunnel->dprx_canceled) {
-		mutex_lock(&tb->lock);
 		if (tb_dp_is_usb4(tunnel->src_port->sw) &&
 		    tb_dp_wait_dprx(tunnel, TB_DPRX_WAIT_TIMEOUT)) {
 			if (ktime_before(ktime_get(), tunnel->dprx_timeout)) {
@@ -1106,41 +1113,42 @@ static void tb_dp_dprx_work(struct work_struct *work)
 		} else {
 			tb_tunnel_set_active(tunnel, true);
 		}
-		mutex_unlock(&tb->lock);
 	}
+	mutex_unlock(&tb->lock);
 
-	if (tunnel->callback)
-		tunnel->callback(tunnel, tunnel->callback_data);
+	tunnel->callback(tunnel, tunnel->callback_data);
 	tb_tunnel_put(tunnel);
+	tb_domain_put(tb);
 }
 
 static int tb_dp_dprx_start(struct tb_tunnel *tunnel)
 {
 	/*
-	 * Bump up the reference to keep the tunnel around. It will be
-	 * dropped in tb_dp_dprx_stop() once the tunnel is deactivated.
+	 * Bump up the references to keep the tunnel and the domain around
+	 * until the work has run or has been canceled.
 	 */
 	tb_tunnel_get(tunnel);
+	tb_domain_get(tunnel->tb);
 
 	tunnel->dprx_started = true;
+	tunnel->dprx_canceled = false;
+	tunnel->dprx_timeout = dprx_timeout_to_ktime(dprx_timeout);
+	queue_delayed_work(tunnel->tb->wq, &tunnel->dprx_work, 0);
 
-	if (tunnel->callback) {
-		tunnel->dprx_timeout = dprx_timeout_to_ktime(dprx_timeout);
-		queue_delayed_work(tunnel->tb->wq, &tunnel->dprx_work, 0);
-		return -EINPROGRESS;
-	}
-
-	return tb_dp_is_usb4(tunnel->src_port->sw) ?
-		tb_dp_wait_dprx(tunnel, dprx_timeout) : 0;
+	return -EINPROGRESS;
 }
 
 static void tb_dp_dprx_stop(struct tb_tunnel *tunnel)
 {
+	struct tb *tb = tunnel->tb;
+
 	if (tunnel->dprx_started) {
 		tunnel->dprx_started = false;
 		tunnel->dprx_canceled = true;
-		if (cancel_delayed_work(&tunnel->dprx_work))
+		if (cancel_delayed_work(&tunnel->dprx_work)) {
 			tb_tunnel_put(tunnel);
+			tb_domain_put(tb);
+		}
 	}
 }
 
@@ -1582,20 +1590,28 @@ static void tb_dp_dump(struct tb_tunnel *tunnel)
  * @tb: Pointer to the domain structure
  * @in: DP in adapter
  * @alloc_hopid: Allocate HopIDs from visited ports
+ * @callback: Callback that is called when the DP tunnel is fully
+ *	      activated (or there is an error)
+ * @callback_data: Data for @callback
  *
  * If @in adapter is active, follows the tunnel to the DP out adapter
  * and back. Returns the discovered tunnel or %NULL if there was no
- * tunnel.
+ * tunnel. See tb_tunnel_alloc_dp() for @callback.
  *
  * Return: Pointer to &struct tb_tunnel or %NULL if no tunnel found.
  */
 struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in,
-					bool alloc_hopid)
+					bool alloc_hopid,
+					void (*callback)(struct tb_tunnel *, void *),
+					void *callback_data)
 {
 	struct tb_tunnel *tunnel;
 	struct tb_port *port;
 	struct tb_path *path;
 
+	if (WARN_ON(!callback))
+		return NULL;
+
 	if (!tb_dp_port_is_enabled(in))
 		return NULL;
 
@@ -1611,6 +1627,9 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in,
 	tunnel->alloc_bandwidth = tb_dp_alloc_bandwidth;
 	tunnel->consumed_bandwidth = tb_dp_consumed_bandwidth;
 	tunnel->src_port = in;
+	tunnel->callback = callback;
+	tunnel->callback_data = callback_data;
+	INIT_DELAYED_WORK(&tunnel->dprx_work, tb_dp_dprx_work);
 
 	path = tb_path_discover(in, TB_DP_VIDEO_HOPID, NULL, -1,
 				&tunnel->dst_port, "Video", alloc_hopid);
@@ -1656,6 +1675,7 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in,
 
 	tb_dp_dump(tunnel);
 
+	tb_tunnel_set_active(tunnel, true);
 	tb_tunnel_dbg(tunnel, "discovered\n");
 	return tunnel;
 
@@ -1677,16 +1697,16 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in,
  *	    %0 if no available bandwidth.
  * @max_down: Maximum available downstream bandwidth for the DP tunnel.
  *	      %0 if no available bandwidth.
- * @callback: Optional callback that is called when the DP tunnel is
- *	      fully activated (or there is an error)
- * @callback_data: Optional data for @callback
+ * @callback: Callback that is called when the DP tunnel is fully
+ *	      activated (or there is an error)
+ * @callback_data: Data for @callback
  *
  * Allocates a tunnel between @in and @out that is capable of tunneling
- * Display Port traffic. If @callback is not %NULL it will be called
- * after tb_tunnel_activate() once the tunnel has been fully activated.
- * It can call tb_tunnel_is_active() to check if activation was
- * successful (or if it returns %false there was some sort of issue).
- * The @callback is called without @tb->lock held.
+ * Display Port traffic. The @callback is called after tb_tunnel_activate()
+ * once the tunnel has been fully activated. It can call
+ * tb_tunnel_is_active() to check if activation was successful (or if it
+ * returns %false there was some sort of issue). The @callback is called
+ * without @tb->lock held.
  *
  * Return: Pointer to @struct tb_tunnel or %NULL in case of failure.
  */
@@ -1701,7 +1721,7 @@ struct tb_tunnel *tb_tunnel_alloc_dp(struct tb *tb, struct tb_port *in,
 	struct tb_path *path;
 	bool pm_support;
 
-	if (WARN_ON(!in->cap_adap || !out->cap_adap))
+	if (WARN_ON(!in->cap_adap || !out->cap_adap || !callback))
 		return NULL;
 
 	tunnel = tb_tunnel_alloc(tb, 3, TB_TUNNEL_DP);
@@ -2288,6 +2308,7 @@ struct tb_tunnel *tb_tunnel_discover_usb3(struct tb *tb, struct tb_port *down,
 			tb_usb3_reclaim_available_bandwidth;
 	}
 
+	tb_tunnel_set_active(tunnel, true);
 	tb_tunnel_dbg(tunnel, "discovered\n");
 	return tunnel;
 
diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h
index 4878763a82b318..7d1d255ab5a7de 100644
--- a/drivers/thunderbolt/tunnel.h
+++ b/drivers/thunderbolt/tunnel.h
@@ -66,8 +66,8 @@ enum tb_tunnel_state {
  * @dprx_canceled: Was DPRX capabilities read poll canceled
  * @dprx_timeout: If set DPRX capabilities read poll work will timeout after this passes
  * @dprx_work: Worker that is scheduled to poll completion of DPRX capabilities read
- * @callback: Optional callback called when DP tunnel is fully activated
- * @callback_data: Optional data for @callback
+ * @callback: Callback called when DP tunnel is fully activated
+ * @callback_data: Data for @callback
  * @paths: All paths required by the tunnel
  */
 struct tb_tunnel {
@@ -117,7 +117,9 @@ struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up,
 bool tb_tunnel_reserved_pci(struct tb_port *port, int *reserved_up,
 			    int *reserved_down);
 struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in,
-					bool alloc_hopid);
+					bool alloc_hopid,
+					void (*callback)(struct tb_tunnel *, void *),
+					void *callback_data);
 struct tb_tunnel *tb_tunnel_alloc_dp(struct tb *tb, struct tb_port *in,
 				     struct tb_port *out, int link_nr,
 				     int max_up, int max_down,
diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c
index 5fd4fe070f2584..c9b3931c45ede2 100644
--- a/drivers/thunderbolt/usb4.c
+++ b/drivers/thunderbolt/usb4.c
@@ -1826,23 +1826,27 @@ int usb4_port_sw_margin(struct tb_port *port, enum usb4_sb_target target,
  * @target: Sideband target
  * @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
  * @errors: Error metadata is copied here.
+ * @dwords: Number of dwords to read error counter values
  *
  * This reads back the software margining error counters from the port.
  *
  * Return: %0 on success, negative errno otherwise.
  */
 int usb4_port_sw_margin_errors(struct tb_port *port, enum usb4_sb_target target,
-			       u8 index, u32 *errors)
+			       u8 index, u32 *errors, size_t dwords)
 {
 	int ret;
+	u8 reg;
 
 	ret = usb4_port_sb_op(port, target, index,
 			      USB4_SB_OPCODE_READ_SW_MARGIN_ERR, 150);
 	if (ret)
 		return ret;
 
-	return usb4_port_sb_read(port, target, index, USB4_SB_METADATA, errors,
-				 sizeof(*errors));
+	reg = (dwords > 1) ? USB4_SB_DATA : USB4_SB_METADATA;
+
+	return usb4_port_sb_read(port, target, index, reg, errors,
+				 sizeof(*errors) * dwords);
 }
 
 static inline int usb4_port_retimer_op(struct tb_port *port, u8 index,
diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c
index c179bd751fe423..b187a60663f513 100644
--- a/drivers/thunderbolt/xdomain.c
+++ b/drivers/thunderbolt/xdomain.c
@@ -377,6 +377,10 @@ static int tb_xdp_properties_request(struct tb_ctl *ctl, u64 route,
 
 		len += sizeof(res->hdr.xd_hdr) / 4;
 		len -= sizeof(*res) / 4;
+		if (len > TB_XDP_PROPERTIES_MAX_DATA_LENGTH) {
+			ret = -EINVAL;
+			goto err;
+		}
 
 		if (res->offset != req.offset) {
 			ret = -EINVAL;
@@ -1814,7 +1818,6 @@ static void tb_xdomain_state_work(struct work_struct *work)
 			tb_xdomain_failed(xd);
 		} else {
 			xd->state = XDOMAIN_STATE_ENUMERATED;
-			tb_xdomain_queue_properties_changed(xd);
 		}
 		break;
 
diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig
index 5d6b0dd4d42345..2ed4f1e0e9efa5 100644
--- a/drivers/tty/Kconfig
+++ b/drivers/tty/Kconfig
@@ -208,16 +208,6 @@ config SERIAL_NONSTANDARD
 
 	  Most people can say N here.
 
-config MOXA_INTELLIO
-	tristate "Moxa Intellio support"
-	depends on SERIAL_NONSTANDARD && PCI
-	select FW_LOADER
-	help
-	  Say Y here if you have a Moxa Intellio multiport serial card.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called moxa.
-
 config MOXA_SMARTIO
 	tristate "Moxa SmartIO support v. 2.0"
 	depends on SERIAL_NONSTANDARD && PCI && HAS_IOPORT
diff --git a/drivers/tty/Makefile b/drivers/tty/Makefile
index c6a063a541fdc6..43aa3f93b09a23 100644
--- a/drivers/tty/Makefile
+++ b/drivers/tty/Makefile
@@ -17,7 +17,6 @@ obj-$(CONFIG_SERIAL_DEV_BUS)	+= serdev/
 
 # tty drivers
 obj-$(CONFIG_AMIGA_BUILTIN_SERIAL) += amiserial.o
-obj-$(CONFIG_MOXA_INTELLIO)	+= moxa.o
 obj-$(CONFIG_MOXA_SMARTIO)	+= mxser.o
 obj-$(CONFIG_NOZOMI)		+= nozomi.o
 obj-$(CONFIG_NULL_TTY)	        += ttynull.o
diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c
index 28af0fd981818b..60f6060b893dcf 100644
--- a/drivers/tty/amiserial.c
+++ b/drivers/tty/amiserial.c
@@ -1138,11 +1138,6 @@ static int rs_ioctl(struct tty_struct *tty,
 				local_irq_save(flags);
 				cnow = info->icount; /* atomic copy */
 				local_irq_restore(flags);
-				if (cnow.rng == cprev.rng && cnow.dsr == cprev.dsr && 
-				    cnow.dcd == cprev.dcd && cnow.cts == cprev.cts) {
-					ret = -EIO; /* no change => error */
-					break;
-				}
 				if ( ((arg & TIOCM_RNG) && (cnow.rng != cprev.rng)) ||
 				     ((arg & TIOCM_DSR) && (cnow.dsr != cprev.dsr)) ||
 				     ((arg & TIOCM_CD)  && (cnow.dcd != cprev.dcd)) ||
@@ -1150,6 +1145,10 @@ static int rs_ioctl(struct tty_struct *tty,
 					ret = 0;
 					break;
 				}
+				if (tty_io_error(tty)) {
+					ret = -EIO;
+					break;
+				}
 				schedule();
 				/* see if a signal did it */
 				if (signal_pending(current)) {
diff --git a/drivers/tty/hvc/hvcs.c b/drivers/tty/hvc/hvcs.c
index d32b1e3c50bfde..b94d59fa36274d 100644
--- a/drivers/tty/hvc/hvcs.c
+++ b/drivers/tty/hvc/hvcs.c
@@ -1470,7 +1470,7 @@ static int hvcs_initialize(void)
 		goto register_fail;
 	}
 
-	hvcs_pi_buff = (unsigned long *) __get_free_page(GFP_KERNEL);
+	hvcs_pi_buff = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!hvcs_pi_buff) {
 		rc = -ENOMEM;
 		goto buff_alloc_fail;
@@ -1486,7 +1486,7 @@ static int hvcs_initialize(void)
 	return 0;
 
 kthread_fail:
-	free_page((unsigned long)hvcs_pi_buff);
+	kfree(hvcs_pi_buff);
 buff_alloc_fail:
 	tty_unregister_driver(hvcs_tty_driver);
 register_fail:
@@ -1528,7 +1528,7 @@ static void __exit hvcs_module_exit(void)
 	kthread_stop(hvcs_task);
 
 	spin_lock(&hvcs_pi_lock);
-	free_page((unsigned long)hvcs_pi_buff);
+	kfree(hvcs_pi_buff);
 	hvcs_pi_buff = NULL;
 	spin_unlock(&hvcs_pi_lock);
 
diff --git a/drivers/tty/moxa.c b/drivers/tty/moxa.c
deleted file mode 100644
index 40a1c614e23b1e..00000000000000
--- a/drivers/tty/moxa.c
+++ /dev/null
@@ -1,2136 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*****************************************************************************/
-/*
- *           moxa.c  -- MOXA Intellio family multiport serial driver.
- *
- *      Copyright (C) 1999-2000  Moxa Technologies (support@moxa.com).
- *      Copyright (c) 2007 Jiri Slaby <jirislaby@gmail.com>
- *
- *      This code is loosely based on the Linux serial driver, written by
- *      Linus Torvalds, Theodore T'so and others.
- */
-
-/*
- *    MOXA Intellio Series Driver
- *      for             : LINUX
- *      date            : 1999/1/7
- *      version         : 5.1
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/ioport.h>
-#include <linux/errno.h>
-#include <linux/firmware.h>
-#include <linux/signal.h>
-#include <linux/sched.h>
-#include <linux/timer.h>
-#include <linux/interrupt.h>
-#include <linux/tty.h>
-#include <linux/tty_flip.h>
-#include <linux/major.h>
-#include <linux/string.h>
-#include <linux/fcntl.h>
-#include <linux/ptrace.h>
-#include <linux/serial.h>
-#include <linux/tty_driver.h>
-#include <linux/delay.h>
-#include <linux/pci.h>
-#include <linux/init.h>
-#include <linux/bitops.h>
-#include <linux/slab.h>
-#include <linux/ratelimit.h>
-
-#include <asm/io.h>
-
-/*
- *    System Configuration
- */
-
-#define Magic_code	0x404
-
-/*
- *    for C218 BIOS initialization
- */
-#define C218_ConfBase	0x800
-#define C218_status	(C218_ConfBase + 0)	/* BIOS running status    */
-#define C218_diag	(C218_ConfBase + 2)	/* diagnostic status      */
-#define C218_key	(C218_ConfBase + 4)	/* WORD (0x218 for C218) */
-#define C218DLoad_len	(C218_ConfBase + 6)	/* WORD           */
-#define C218check_sum	(C218_ConfBase + 8)	/* BYTE           */
-#define C218chksum_ok	(C218_ConfBase + 0x0a)	/* BYTE (1:ok)            */
-#define C218_TestRx	(C218_ConfBase + 0x10)	/* 8 bytes for 8 ports    */
-#define C218_TestTx	(C218_ConfBase + 0x18)	/* 8 bytes for 8 ports    */
-#define C218_RXerr	(C218_ConfBase + 0x20)	/* 8 bytes for 8 ports    */
-#define C218_ErrFlag	(C218_ConfBase + 0x28)	/* 8 bytes for 8 ports    */
-
-#define C218_LoadBuf	0x0F00
-#define C218_KeyCode	0x218
-#define CP204J_KeyCode	0x204
-
-/*
- *    for C320 BIOS initialization
- */
-#define C320_ConfBase	0x800
-#define C320_LoadBuf	0x0f00
-#define STS_init	0x05	/* for C320_status        */
-
-#define C320_status	C320_ConfBase + 0	/* BIOS running status    */
-#define C320_diag	C320_ConfBase + 2	/* diagnostic status      */
-#define C320_key	C320_ConfBase + 4	/* WORD (0320H for C320) */
-#define C320DLoad_len	C320_ConfBase + 6	/* WORD           */
-#define C320check_sum	C320_ConfBase + 8	/* WORD           */
-#define C320chksum_ok	C320_ConfBase + 0x0a	/* WORD (1:ok)            */
-#define C320bapi_len	C320_ConfBase + 0x0c	/* WORD           */
-#define C320UART_no	C320_ConfBase + 0x0e	/* WORD           */
-
-#define C320_KeyCode	0x320
-
-#define FixPage_addr	0x0000	/* starting addr of static page  */
-#define DynPage_addr	0x2000	/* starting addr of dynamic page */
-#define C218_start	0x3000	/* starting addr of C218 BIOS prg */
-#define Control_reg	0x1ff0	/* select page and reset control */
-#define HW_reset	0x80
-
-/*
- *    Function Codes
- */
-#define FC_CardReset	0x80
-#define FC_ChannelReset 1	/* C320 firmware not supported */
-#define FC_EnableCH	2
-#define FC_DisableCH	3
-#define FC_SetParam	4
-#define FC_SetMode	5
-#define FC_SetRate	6
-#define FC_LineControl	7
-#define FC_LineStatus	8
-#define FC_XmitControl	9
-#define FC_FlushQueue	10
-#define FC_SendBreak	11
-#define FC_StopBreak	12
-#define FC_LoopbackON	13
-#define FC_LoopbackOFF	14
-#define FC_ClrIrqTable	15
-#define FC_SendXon	16
-#define FC_SetTermIrq	17	/* C320 firmware not supported */
-#define FC_SetCntIrq	18	/* C320 firmware not supported */
-#define FC_SetBreakIrq	19
-#define FC_SetLineIrq	20
-#define FC_SetFlowCtl	21
-#define FC_GenIrq	22
-#define FC_InCD180	23
-#define FC_OutCD180	24
-#define FC_InUARTreg	23
-#define FC_OutUARTreg	24
-#define FC_SetXonXoff	25
-#define FC_OutCD180CCR	26
-#define FC_ExtIQueue	27
-#define FC_ExtOQueue	28
-#define FC_ClrLineIrq	29
-#define FC_HWFlowCtl	30
-#define FC_GetClockRate 35
-#define FC_SetBaud	36
-#define FC_SetDataMode  41
-#define FC_GetCCSR      43
-#define FC_GetDataError 45
-#define FC_RxControl	50
-#define FC_ImmSend	51
-#define FC_SetXonState	52
-#define FC_SetXoffState	53
-#define FC_SetRxFIFOTrig 54
-#define FC_SetTxFIFOCnt 55
-#define FC_UnixRate	56
-#define FC_UnixResetTimer 57
-
-#define	RxFIFOTrig1	0
-#define	RxFIFOTrig4	1
-#define	RxFIFOTrig8	2
-#define	RxFIFOTrig14	3
-
-/*
- *    Dual-Ported RAM
- */
-#define DRAM_global	0
-#define INT_data	(DRAM_global + 0)
-#define Config_base	(DRAM_global + 0x108)
-
-#define IRQindex	(INT_data + 0)
-#define IRQpending	(INT_data + 4)
-#define IRQtable	(INT_data + 8)
-
-/*
- *    Interrupt Status
- */
-#define IntrRx		0x01	/* receiver data O.K.             */
-#define IntrTx		0x02	/* transmit buffer empty  */
-#define IntrFunc	0x04	/* function complete              */
-#define IntrBreak	0x08	/* received break         */
-#define IntrLine	0x10	/* line status change
-				   for transmitter                */
-#define IntrIntr	0x20	/* received INTR code             */
-#define IntrQuit	0x40	/* received QUIT code             */
-#define IntrEOF		0x80	/* received EOF code              */
-
-#define IntrRxTrigger	0x100	/* rx data count reach trigger value */
-#define IntrTxTrigger	0x200	/* tx data count below trigger value */
-
-#define Magic_no	(Config_base + 0)
-#define Card_model_no	(Config_base + 2)
-#define Total_ports	(Config_base + 4)
-#define Module_cnt	(Config_base + 8)
-#define Module_no	(Config_base + 10)
-#define Timer_10ms	(Config_base + 14)
-#define Disable_IRQ	(Config_base + 20)
-#define TMS320_PORT1	(Config_base + 22)
-#define TMS320_PORT2	(Config_base + 24)
-#define TMS320_CLOCK	(Config_base + 26)
-
-/*
- *    DATA BUFFER in DRAM
- */
-#define Extern_table	0x400	/* Base address of the external table
-				   (24 words *    64) total 3K bytes
-				   (24 words * 128) total 6K bytes */
-#define Extern_size	0x60	/* 96 bytes                       */
-#define RXrptr		0x00	/* read pointer for RX buffer     */
-#define RXwptr		0x02	/* write pointer for RX buffer    */
-#define TXrptr		0x04	/* read pointer for TX buffer     */
-#define TXwptr		0x06	/* write pointer for TX buffer    */
-#define HostStat	0x08	/* IRQ flag and general flag      */
-#define FlagStat	0x0A
-#define FlowControl	0x0C	/* B7 B6 B5 B4 B3 B2 B1 B0              */
-				/*  x  x  x  x  |  |  |  |            */
-				/*              |  |  |  + CTS flow   */
-				/*              |  |  +--- RTS flow   */
-				/*              |  +------ TX Xon/Xoff */
-				/*              +--------- RX Xon/Xoff */
-#define Break_cnt	0x0E	/* received break count   */
-#define CD180TXirq	0x10	/* if non-0: enable TX irq        */
-#define RX_mask		0x12
-#define TX_mask		0x14
-#define Ofs_rxb		0x16
-#define Ofs_txb		0x18
-#define Page_rxb	0x1A
-#define Page_txb	0x1C
-#define EndPage_rxb	0x1E
-#define EndPage_txb	0x20
-#define Data_error	0x22
-#define RxTrigger	0x28
-#define TxTrigger	0x2a
-
-#define rRXwptr		0x34
-#define Low_water	0x36
-
-#define FuncCode	0x40
-#define FuncArg		0x42
-#define FuncArg1	0x44
-
-#define C218rx_size	0x2000	/* 8K bytes */
-#define C218tx_size	0x8000	/* 32K bytes */
-
-#define C218rx_mask	(C218rx_size - 1)
-#define C218tx_mask	(C218tx_size - 1)
-
-#define C320p8rx_size	0x2000
-#define C320p8tx_size	0x8000
-#define C320p8rx_mask	(C320p8rx_size - 1)
-#define C320p8tx_mask	(C320p8tx_size - 1)
-
-#define C320p16rx_size	0x2000
-#define C320p16tx_size	0x4000
-#define C320p16rx_mask	(C320p16rx_size - 1)
-#define C320p16tx_mask	(C320p16tx_size - 1)
-
-#define C320p24rx_size	0x2000
-#define C320p24tx_size	0x2000
-#define C320p24rx_mask	(C320p24rx_size - 1)
-#define C320p24tx_mask	(C320p24tx_size - 1)
-
-#define C320p32rx_size	0x1000
-#define C320p32tx_size	0x1000
-#define C320p32rx_mask	(C320p32rx_size - 1)
-#define C320p32tx_mask	(C320p32tx_size - 1)
-
-#define Page_size	0x2000U
-#define Page_mask	(Page_size - 1)
-#define C218rx_spage	3
-#define C218tx_spage	4
-#define C218rx_pageno	1
-#define C218tx_pageno	4
-#define C218buf_pageno	5
-
-#define C320p8rx_spage	3
-#define C320p8tx_spage	4
-#define C320p8rx_pgno	1
-#define C320p8tx_pgno	4
-#define C320p8buf_pgno	5
-
-#define C320p16rx_spage 3
-#define C320p16tx_spage 4
-#define C320p16rx_pgno	1
-#define C320p16tx_pgno	2
-#define C320p16buf_pgno 3
-
-#define C320p24rx_spage 3
-#define C320p24tx_spage 4
-#define C320p24rx_pgno	1
-#define C320p24tx_pgno	1
-#define C320p24buf_pgno 2
-
-#define C320p32rx_spage 3
-#define C320p32tx_ofs	C320p32rx_size
-#define C320p32tx_spage 3
-#define C320p32buf_pgno 1
-
-/*
- *    Host Status
- */
-#define WakeupRx	0x01
-#define WakeupTx	0x02
-#define WakeupBreak	0x08
-#define WakeupLine	0x10
-#define WakeupIntr	0x20
-#define WakeupQuit	0x40
-#define WakeupEOF	0x80	/* used in VTIME control */
-#define WakeupRxTrigger	0x100
-#define WakeupTxTrigger	0x200
-/*
- *    Flag status
- */
-#define Rx_over		0x01
-#define Xoff_state	0x02
-#define Tx_flowOff	0x04
-#define Tx_enable	0x08
-#define CTS_state	0x10
-#define DSR_state	0x20
-#define DCD_state	0x80
-/*
- *    FlowControl
- */
-#define CTS_FlowCtl	1
-#define RTS_FlowCtl	2
-#define Tx_FlowCtl	4
-#define Rx_FlowCtl	8
-#define IXM_IXANY	0x10
-
-#define LowWater	128
-
-#define DTR_ON		1
-#define RTS_ON		2
-#define CTS_ON		1
-#define DSR_ON		2
-#define DCD_ON		8
-
-/* mode definition */
-#define	MX_CS8		0x03
-#define	MX_CS7		0x02
-#define	MX_CS6		0x01
-#define	MX_CS5		0x00
-
-#define	MX_STOP1	0x00
-#define	MX_STOP15	0x04
-#define	MX_STOP2	0x08
-
-#define	MX_PARNONE	0x00
-#define	MX_PAREVEN	0x40
-#define	MX_PARODD	0xC0
-#define	MX_PARMARK	0xA0
-#define	MX_PARSPACE	0x20
-
-#define MOXA_FW_HDRLEN		32
-
-#define MOXAMAJOR		172
-
-#define MAX_BOARDS		4	/* Don't change this value */
-#define MAX_PORTS_PER_BOARD	32	/* Don't change this value */
-#define MAX_PORTS		(MAX_BOARDS * MAX_PORTS_PER_BOARD)
-
-#define MOXA_IS_320(brd)	((brd)->boardType == MOXA_BOARD_C320_PCI)
-
-enum {
-	MOXA_BOARD_C218_PCI = 1,
-	MOXA_BOARD_C320_PCI,
-	MOXA_BOARD_CP204J,
-};
-
-static char *moxa_brdname[] =
-{
-	"C218 Turbo PCI series",
-	"C320 Turbo PCI series",
-	"CP-204J series",
-};
-
-static const struct pci_device_id moxa_pcibrds[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_MOXA, PCI_DEVICE_ID_MOXA_C218),
-		.driver_data = MOXA_BOARD_C218_PCI },
-	{ PCI_DEVICE(PCI_VENDOR_ID_MOXA, PCI_DEVICE_ID_MOXA_C320),
-		.driver_data = MOXA_BOARD_C320_PCI },
-	{ PCI_DEVICE(PCI_VENDOR_ID_MOXA, PCI_DEVICE_ID_MOXA_CP204J),
-		.driver_data = MOXA_BOARD_CP204J },
-	{ 0 }
-};
-MODULE_DEVICE_TABLE(pci, moxa_pcibrds);
-
-struct moxa_port;
-
-static struct moxa_board_conf {
-	int boardType;
-	int numPorts;
-
-	unsigned int ready;
-
-	struct moxa_port *ports;
-
-	void __iomem *basemem;
-	void __iomem *intNdx;
-	void __iomem *intPend;
-	void __iomem *intTable;
-} moxa_boards[MAX_BOARDS];
-
-struct moxa_port {
-	struct tty_port port;
-	struct moxa_board_conf *board;
-	void __iomem *tableAddr;
-
-	int type;
-	int cflag;
-	unsigned long statusflags;
-
-	u8 DCDState;		/* Protected by the port lock */
-	u8 lineCtrl;
-	u8 lowChkFlag;
-};
-
-/* statusflags */
-#define TXSTOPPED	1
-#define LOWWAIT 	2
-#define EMPTYWAIT	3
-
-
-#define WAKEUP_CHARS		256
-
-static int ttymajor = MOXAMAJOR;
-static unsigned int moxaFuncTout = HZ / 2;
-static unsigned int moxaLowWaterChk;
-static DEFINE_MUTEX(moxa_openlock);
-static DEFINE_SPINLOCK(moxa_lock);
-
-MODULE_AUTHOR("William Chen");
-MODULE_DESCRIPTION("MOXA Intellio Family Multiport Board Device Driver");
-MODULE_LICENSE("GPL");
-MODULE_FIRMWARE("c218tunx.cod");
-MODULE_FIRMWARE("cp204unx.cod");
-MODULE_FIRMWARE("c320tunx.cod");
-
-module_param(ttymajor, int, 0);
-
-/*
- * static functions:
- */
-static int moxa_open(struct tty_struct *, struct file *);
-static void moxa_close(struct tty_struct *, struct file *);
-static ssize_t moxa_write(struct tty_struct *, const u8 *, size_t);
-static unsigned int moxa_write_room(struct tty_struct *);
-static void moxa_flush_buffer(struct tty_struct *);
-static unsigned int moxa_chars_in_buffer(struct tty_struct *);
-static void moxa_set_termios(struct tty_struct *, const struct ktermios *);
-static void moxa_stop(struct tty_struct *);
-static void moxa_start(struct tty_struct *);
-static void moxa_hangup(struct tty_struct *);
-static int moxa_tiocmget(struct tty_struct *tty);
-static int moxa_tiocmset(struct tty_struct *tty,
-			 unsigned int set, unsigned int clear);
-static void moxa_poll(struct timer_list *);
-static void moxa_set_tty_param(struct tty_struct *, const struct ktermios *);
-static void moxa_shutdown(struct tty_port *);
-static bool moxa_carrier_raised(struct tty_port *);
-static void moxa_dtr_rts(struct tty_port *, bool);
-/*
- * moxa board interface functions:
- */
-static void MoxaPortEnable(struct moxa_port *);
-static void MoxaPortDisable(struct moxa_port *);
-static int MoxaPortSetTermio(struct moxa_port *, struct ktermios *, speed_t);
-static int MoxaPortGetLineOut(struct moxa_port *, bool *, bool *);
-static void MoxaPortLineCtrl(struct moxa_port *, bool, bool);
-static void MoxaPortFlowCtrl(struct moxa_port *, int, int, int, int, int);
-static int MoxaPortLineStatus(struct moxa_port *);
-static void MoxaPortFlushData(struct moxa_port *, int);
-static ssize_t MoxaPortWriteData(struct tty_struct *, const u8 *, size_t);
-static int MoxaPortReadData(struct moxa_port *);
-static unsigned int MoxaPortTxQueue(struct moxa_port *);
-static int MoxaPortRxQueue(struct moxa_port *);
-static unsigned int MoxaPortTxFree(struct moxa_port *);
-static void MoxaPortTxDisable(struct moxa_port *);
-static void MoxaPortTxEnable(struct moxa_port *);
-static int moxa_get_serial_info(struct tty_struct *, struct serial_struct *);
-static int moxa_set_serial_info(struct tty_struct *, struct serial_struct *);
-static void MoxaSetFifo(struct moxa_port *port, int enable);
-
-/*
- * I/O functions
- */
-
-static DEFINE_SPINLOCK(moxafunc_lock);
-
-static void moxa_wait_finish(void __iomem *ofsAddr)
-{
-	unsigned long end = jiffies + moxaFuncTout;
-
-	while (readw(ofsAddr + FuncCode) != 0)
-		if (time_after(jiffies, end))
-			return;
-	if (readw(ofsAddr + FuncCode) != 0)
-		printk_ratelimited(KERN_WARNING "moxa function expired\n");
-}
-
-static void moxafunc(void __iomem *ofsAddr, u16 cmd, u16 arg)
-{
-	guard(spinlock_irqsave)(&moxafunc_lock);
-	writew(arg, ofsAddr + FuncArg);
-	writew(cmd, ofsAddr + FuncCode);
-	moxa_wait_finish(ofsAddr);
-}
-
-static int moxafuncret(void __iomem *ofsAddr, u16 cmd, u16 arg)
-{
-	guard(spinlock_irqsave)(&moxafunc_lock);
-	writew(arg, ofsAddr + FuncArg);
-	writew(cmd, ofsAddr + FuncCode);
-	moxa_wait_finish(ofsAddr);
-
-	return readw(ofsAddr + FuncArg);
-}
-
-static void moxa_low_water_check(void __iomem *ofsAddr)
-{
-	u16 rptr, wptr, mask, len;
-
-	if (readb(ofsAddr + FlagStat) & Xoff_state) {
-		rptr = readw(ofsAddr + RXrptr);
-		wptr = readw(ofsAddr + RXwptr);
-		mask = readw(ofsAddr + RX_mask);
-		len = (wptr - rptr) & mask;
-		if (len <= Low_water)
-			moxafunc(ofsAddr, FC_SendXon, 0);
-	}
-}
-
-/*
- * TTY operations
- */
-
-static int moxa_break_ctl(struct tty_struct *tty, int state)
-{
-	struct moxa_port *port = tty->driver_data;
-
-	moxafunc(port->tableAddr, state ? FC_SendBreak : FC_StopBreak,
-			Magic_code);
-	return 0;
-}
-
-static const struct tty_operations moxa_ops = {
-	.open = moxa_open,
-	.close = moxa_close,
-	.write = moxa_write,
-	.write_room = moxa_write_room,
-	.flush_buffer = moxa_flush_buffer,
-	.chars_in_buffer = moxa_chars_in_buffer,
-	.set_termios = moxa_set_termios,
-	.stop = moxa_stop,
-	.start = moxa_start,
-	.hangup = moxa_hangup,
-	.break_ctl = moxa_break_ctl,
-	.tiocmget = moxa_tiocmget,
-	.tiocmset = moxa_tiocmset,
-	.set_serial = moxa_set_serial_info,
-	.get_serial = moxa_get_serial_info,
-};
-
-static const struct tty_port_operations moxa_port_ops = {
-	.carrier_raised = moxa_carrier_raised,
-	.dtr_rts = moxa_dtr_rts,
-	.shutdown = moxa_shutdown,
-};
-
-static struct tty_driver *moxaDriver;
-static DEFINE_TIMER(moxaTimer, moxa_poll);
-
-/*
- * HW init
- */
-
-static int moxa_check_fw_model(struct moxa_board_conf *brd, u8 model)
-{
-	switch (brd->boardType) {
-	case MOXA_BOARD_C218_PCI:
-		if (model != 1)
-			goto err;
-		break;
-	case MOXA_BOARD_CP204J:
-		if (model != 3)
-			goto err;
-		break;
-	default:
-		if (model != 2)
-			goto err;
-		break;
-	}
-	return 0;
-err:
-	return -EINVAL;
-}
-
-static int moxa_check_fw(const void *ptr)
-{
-	const __le16 *lptr = ptr;
-
-	if (*lptr != cpu_to_le16(0x7980))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int moxa_load_bios(struct moxa_board_conf *brd, const u8 *buf,
-		size_t len)
-{
-	void __iomem *baseAddr = brd->basemem;
-	u16 tmp;
-
-	writeb(HW_reset, baseAddr + Control_reg);	/* reset */
-	msleep(10);
-	memset_io(baseAddr, 0, 4096);
-	memcpy_toio(baseAddr, buf, len);	/* download BIOS */
-	writeb(0, baseAddr + Control_reg);	/* restart */
-
-	msleep(2000);
-
-	switch (brd->boardType) {
-	case MOXA_BOARD_C218_PCI:
-		tmp = readw(baseAddr + C218_key);
-		if (tmp != C218_KeyCode)
-			goto err;
-		break;
-	case MOXA_BOARD_CP204J:
-		tmp = readw(baseAddr + C218_key);
-		if (tmp != CP204J_KeyCode)
-			goto err;
-		break;
-	default:
-		tmp = readw(baseAddr + C320_key);
-		if (tmp != C320_KeyCode)
-			goto err;
-		tmp = readw(baseAddr + C320_status);
-		if (tmp != STS_init) {
-			printk(KERN_ERR "MOXA: bios upload failed -- CPU/Basic "
-					"module not found\n");
-			return -EIO;
-		}
-		break;
-	}
-
-	return 0;
-err:
-	printk(KERN_ERR "MOXA: bios upload failed -- board not found\n");
-	return -EIO;
-}
-
-static int moxa_load_320b(struct moxa_board_conf *brd, const u8 *ptr,
-		size_t len)
-{
-	void __iomem *baseAddr = brd->basemem;
-
-	if (len < 7168) {
-		printk(KERN_ERR "MOXA: invalid 320 bios -- too short\n");
-		return -EINVAL;
-	}
-
-	writew(len - 7168 - 2, baseAddr + C320bapi_len);
-	writeb(1, baseAddr + Control_reg);	/* Select Page 1 */
-	memcpy_toio(baseAddr + DynPage_addr, ptr, 7168);
-	writeb(2, baseAddr + Control_reg);	/* Select Page 2 */
-	memcpy_toio(baseAddr + DynPage_addr, ptr + 7168, len - 7168);
-
-	return 0;
-}
-
-static int moxa_real_load_code(struct moxa_board_conf *brd, const void *ptr,
-		size_t len)
-{
-	void __iomem *baseAddr = brd->basemem;
-	const __le16 *uptr = ptr;
-	size_t wlen, len2, j;
-	unsigned long key, loadbuf, loadlen, checksum, checksum_ok;
-	unsigned int i, retry;
-	u16 usum, keycode;
-
-	keycode = (brd->boardType == MOXA_BOARD_CP204J) ? CP204J_KeyCode :
-				C218_KeyCode;
-
-	switch (brd->boardType) {
-	case MOXA_BOARD_CP204J:
-	case MOXA_BOARD_C218_PCI:
-		key = C218_key;
-		loadbuf = C218_LoadBuf;
-		loadlen = C218DLoad_len;
-		checksum = C218check_sum;
-		checksum_ok = C218chksum_ok;
-		break;
-	default:
-		key = C320_key;
-		keycode = C320_KeyCode;
-		loadbuf = C320_LoadBuf;
-		loadlen = C320DLoad_len;
-		checksum = C320check_sum;
-		checksum_ok = C320chksum_ok;
-		break;
-	}
-
-	usum = 0;
-	wlen = len >> 1;
-	for (i = 0; i < wlen; i++)
-		usum += le16_to_cpu(uptr[i]);
-	retry = 0;
-	do {
-		wlen = len >> 1;
-		j = 0;
-		while (wlen) {
-			len2 = (wlen > 2048) ? 2048 : wlen;
-			wlen -= len2;
-			memcpy_toio(baseAddr + loadbuf, ptr + j, len2 << 1);
-			j += len2 << 1;
-
-			writew(len2, baseAddr + loadlen);
-			writew(0, baseAddr + key);
-			for (i = 0; i < 100; i++) {
-				if (readw(baseAddr + key) == keycode)
-					break;
-				msleep(10);
-			}
-			if (readw(baseAddr + key) != keycode)
-				return -EIO;
-		}
-		writew(0, baseAddr + loadlen);
-		writew(usum, baseAddr + checksum);
-		writew(0, baseAddr + key);
-		for (i = 0; i < 100; i++) {
-			if (readw(baseAddr + key) == keycode)
-				break;
-			msleep(10);
-		}
-		retry++;
-	} while ((readb(baseAddr + checksum_ok) != 1) && (retry < 3));
-	if (readb(baseAddr + checksum_ok) != 1)
-		return -EIO;
-
-	writew(0, baseAddr + key);
-	for (i = 0; i < 600; i++) {
-		if (readw(baseAddr + Magic_no) == Magic_code)
-			break;
-		msleep(10);
-	}
-	if (readw(baseAddr + Magic_no) != Magic_code)
-		return -EIO;
-
-	if (MOXA_IS_320(brd)) {
-		writew(0x3800, baseAddr + TMS320_PORT1);
-		writew(0x3900, baseAddr + TMS320_PORT2);
-		writew(28499, baseAddr + TMS320_CLOCK);
-	}
-	writew(1, baseAddr + Disable_IRQ);
-	writew(0, baseAddr + Magic_no);
-	for (i = 0; i < 500; i++) {
-		if (readw(baseAddr + Magic_no) == Magic_code)
-			break;
-		msleep(10);
-	}
-	if (readw(baseAddr + Magic_no) != Magic_code)
-		return -EIO;
-
-	if (MOXA_IS_320(brd)) {
-		j = readw(baseAddr + Module_cnt);
-		if (j <= 0)
-			return -EIO;
-		brd->numPorts = j * 8;
-		writew(j, baseAddr + Module_no);
-		writew(0, baseAddr + Magic_no);
-		for (i = 0; i < 600; i++) {
-			if (readw(baseAddr + Magic_no) == Magic_code)
-				break;
-			msleep(10);
-		}
-		if (readw(baseAddr + Magic_no) != Magic_code)
-			return -EIO;
-	}
-	brd->intNdx = baseAddr + IRQindex;
-	brd->intPend = baseAddr + IRQpending;
-	brd->intTable = baseAddr + IRQtable;
-
-	return 0;
-}
-
-static int moxa_load_code(struct moxa_board_conf *brd, const void *ptr,
-		size_t len)
-{
-	void __iomem *ofsAddr, *baseAddr = brd->basemem;
-	struct moxa_port *port;
-	int retval, i;
-
-	if (len % 2) {
-		printk(KERN_ERR "MOXA: bios length is not even\n");
-		return -EINVAL;
-	}
-
-	retval = moxa_real_load_code(brd, ptr, len); /* may change numPorts */
-	if (retval)
-		return retval;
-
-	switch (brd->boardType) {
-	case MOXA_BOARD_C218_PCI:
-	case MOXA_BOARD_CP204J:
-		port = brd->ports;
-		for (i = 0; i < brd->numPorts; i++, port++) {
-			port->board = brd;
-			port->DCDState = 0;
-			port->tableAddr = baseAddr + Extern_table +
-					Extern_size * i;
-			ofsAddr = port->tableAddr;
-			writew(C218rx_mask, ofsAddr + RX_mask);
-			writew(C218tx_mask, ofsAddr + TX_mask);
-			writew(C218rx_spage + i * C218buf_pageno, ofsAddr + Page_rxb);
-			writew(readw(ofsAddr + Page_rxb) + C218rx_pageno, ofsAddr + EndPage_rxb);
-
-			writew(C218tx_spage + i * C218buf_pageno, ofsAddr + Page_txb);
-			writew(readw(ofsAddr + Page_txb) + C218tx_pageno, ofsAddr + EndPage_txb);
-
-		}
-		break;
-	default:
-		port = brd->ports;
-		for (i = 0; i < brd->numPorts; i++, port++) {
-			port->board = brd;
-			port->DCDState = 0;
-			port->tableAddr = baseAddr + Extern_table +
-					Extern_size * i;
-			ofsAddr = port->tableAddr;
-			switch (brd->numPorts) {
-			case 8:
-				writew(C320p8rx_mask, ofsAddr + RX_mask);
-				writew(C320p8tx_mask, ofsAddr + TX_mask);
-				writew(C320p8rx_spage + i * C320p8buf_pgno, ofsAddr + Page_rxb);
-				writew(readw(ofsAddr + Page_rxb) + C320p8rx_pgno, ofsAddr + EndPage_rxb);
-				writew(C320p8tx_spage + i * C320p8buf_pgno, ofsAddr + Page_txb);
-				writew(readw(ofsAddr + Page_txb) + C320p8tx_pgno, ofsAddr + EndPage_txb);
-
-				break;
-			case 16:
-				writew(C320p16rx_mask, ofsAddr + RX_mask);
-				writew(C320p16tx_mask, ofsAddr + TX_mask);
-				writew(C320p16rx_spage + i * C320p16buf_pgno, ofsAddr + Page_rxb);
-				writew(readw(ofsAddr + Page_rxb) + C320p16rx_pgno, ofsAddr + EndPage_rxb);
-				writew(C320p16tx_spage + i * C320p16buf_pgno, ofsAddr + Page_txb);
-				writew(readw(ofsAddr + Page_txb) + C320p16tx_pgno, ofsAddr + EndPage_txb);
-				break;
-
-			case 24:
-				writew(C320p24rx_mask, ofsAddr + RX_mask);
-				writew(C320p24tx_mask, ofsAddr + TX_mask);
-				writew(C320p24rx_spage + i * C320p24buf_pgno, ofsAddr + Page_rxb);
-				writew(readw(ofsAddr + Page_rxb) + C320p24rx_pgno, ofsAddr + EndPage_rxb);
-				writew(C320p24tx_spage + i * C320p24buf_pgno, ofsAddr + Page_txb);
-				writew(readw(ofsAddr + Page_txb), ofsAddr + EndPage_txb);
-				break;
-			case 32:
-				writew(C320p32rx_mask, ofsAddr + RX_mask);
-				writew(C320p32tx_mask, ofsAddr + TX_mask);
-				writew(C320p32tx_ofs, ofsAddr + Ofs_txb);
-				writew(C320p32rx_spage + i * C320p32buf_pgno, ofsAddr + Page_rxb);
-				writew(readb(ofsAddr + Page_rxb), ofsAddr + EndPage_rxb);
-				writew(C320p32tx_spage + i * C320p32buf_pgno, ofsAddr + Page_txb);
-				writew(readw(ofsAddr + Page_txb), ofsAddr + EndPage_txb);
-				break;
-			}
-		}
-		break;
-	}
-	return 0;
-}
-
-static int moxa_load_fw(struct moxa_board_conf *brd, const struct firmware *fw)
-{
-	const void *ptr = fw->data;
-	char rsn[64];
-	u16 lens[5];
-	size_t len;
-	unsigned int a, lenp, lencnt;
-	int ret = -EINVAL;
-	struct {
-		__le32 magic;	/* 0x34303430 */
-		u8 reserved1[2];
-		u8 type;	/* UNIX = 3 */
-		u8 model;	/* C218T=1, C320T=2, CP204=3 */
-		u8 reserved2[8];
-		__le16 len[5];
-	} const *hdr = ptr;
-
-	BUILD_BUG_ON(ARRAY_SIZE(hdr->len) != ARRAY_SIZE(lens));
-
-	if (fw->size < MOXA_FW_HDRLEN) {
-		strcpy(rsn, "too short (even header won't fit)");
-		goto err;
-	}
-	if (hdr->magic != cpu_to_le32(0x30343034)) {
-		sprintf(rsn, "bad magic: %.8x", le32_to_cpu(hdr->magic));
-		goto err;
-	}
-	if (hdr->type != 3) {
-		sprintf(rsn, "not for linux, type is %u", hdr->type);
-		goto err;
-	}
-	if (moxa_check_fw_model(brd, hdr->model)) {
-		sprintf(rsn, "not for this card, model is %u", hdr->model);
-		goto err;
-	}
-
-	len = MOXA_FW_HDRLEN;
-	lencnt = hdr->model == 2 ? 5 : 3;
-	for (a = 0; a < ARRAY_SIZE(lens); a++) {
-		lens[a] = le16_to_cpu(hdr->len[a]);
-		if (lens[a] && len + lens[a] <= fw->size &&
-				moxa_check_fw(&fw->data[len]))
-			printk(KERN_WARNING "MOXA firmware: unexpected input "
-				"at offset %u, but going on\n", (u32)len);
-		if (!lens[a] && a < lencnt) {
-			sprintf(rsn, "too few entries in fw file");
-			goto err;
-		}
-		len += lens[a];
-	}
-
-	if (len != fw->size) {
-		sprintf(rsn, "bad length: %u (should be %u)", (u32)fw->size,
-				(u32)len);
-		goto err;
-	}
-
-	ptr += MOXA_FW_HDRLEN;
-	lenp = 0; /* bios */
-
-	strcpy(rsn, "read above");
-
-	ret = moxa_load_bios(brd, ptr, lens[lenp]);
-	if (ret)
-		goto err;
-
-	/* we skip the tty section (lens[1]), since we don't need it */
-	ptr += lens[lenp] + lens[lenp + 1];
-	lenp += 2; /* comm */
-
-	if (hdr->model == 2) {
-		ret = moxa_load_320b(brd, ptr, lens[lenp]);
-		if (ret)
-			goto err;
-		/* skip another tty */
-		ptr += lens[lenp] + lens[lenp + 1];
-		lenp += 2;
-	}
-
-	ret = moxa_load_code(brd, ptr, lens[lenp]);
-	if (ret)
-		goto err;
-
-	return 0;
-err:
-	printk(KERN_ERR "firmware failed to load, reason: %s\n", rsn);
-	return ret;
-}
-
-static int moxa_init_board(struct moxa_board_conf *brd, struct device *dev)
-{
-	const struct firmware *fw;
-	const char *file;
-	struct moxa_port *p;
-	unsigned int i, first_idx;
-	int ret;
-
-	brd->ports = kzalloc_objs(*brd->ports, MAX_PORTS_PER_BOARD);
-	if (brd->ports == NULL) {
-		printk(KERN_ERR "cannot allocate memory for ports\n");
-		ret = -ENOMEM;
-		goto err;
-	}
-
-	for (i = 0, p = brd->ports; i < MAX_PORTS_PER_BOARD; i++, p++) {
-		tty_port_init(&p->port);
-		p->port.ops = &moxa_port_ops;
-		p->type = PORT_16550A;
-		p->cflag = B9600 | CS8 | CREAD | CLOCAL | HUPCL;
-	}
-
-	switch (brd->boardType) {
-	case MOXA_BOARD_C218_PCI:
-		file = "c218tunx.cod";
-		break;
-	case MOXA_BOARD_CP204J:
-		file = "cp204unx.cod";
-		break;
-	default:
-		file = "c320tunx.cod";
-		break;
-	}
-
-	ret = request_firmware(&fw, file, dev);
-	if (ret) {
-		printk(KERN_ERR "MOXA: request_firmware failed. Make sure "
-				"you've placed '%s' file into your firmware "
-				"loader directory (e.g. /lib/firmware)\n",
-				file);
-		goto err_free;
-	}
-
-	ret = moxa_load_fw(brd, fw);
-
-	release_firmware(fw);
-
-	if (ret)
-		goto err_free;
-
-	scoped_guard(spinlock_bh, &moxa_lock) {
-		brd->ready = 1;
-		if (!timer_pending(&moxaTimer))
-			mod_timer(&moxaTimer, jiffies + HZ / 50);
-	}
-
-	first_idx = (brd - moxa_boards) * MAX_PORTS_PER_BOARD;
-	for (i = 0; i < brd->numPorts; i++)
-		tty_port_register_device(&brd->ports[i].port, moxaDriver,
-				first_idx + i, dev);
-
-	return 0;
-err_free:
-	for (i = 0; i < MAX_PORTS_PER_BOARD; i++)
-		tty_port_destroy(&brd->ports[i].port);
-	kfree(brd->ports);
-err:
-	return ret;
-}
-
-static void moxa_board_deinit(struct moxa_board_conf *brd)
-{
-	unsigned int a, opened, first_idx;
-
-	scoped_guard(mutex, &moxa_openlock) {
-		scoped_guard(spinlock_bh, &moxa_lock)
-			brd->ready = 0;
-
-		/* pci hot-un-plug support */
-		for (a = 0; a < brd->numPorts; a++)
-			if (tty_port_initialized(&brd->ports[a].port))
-				tty_port_tty_hangup(&brd->ports[a].port, false);
-
-		for (a = 0; a < MAX_PORTS_PER_BOARD; a++)
-			tty_port_destroy(&brd->ports[a].port);
-
-		while (1) {
-			opened = 0;
-			for (a = 0; a < brd->numPorts; a++)
-				if (tty_port_initialized(&brd->ports[a].port))
-					opened++;
-			if (!opened)
-				break;
-			mutex_unlock(&moxa_openlock);
-			msleep(50);
-			mutex_lock(&moxa_openlock);
-		}
-	}
-
-	first_idx = (brd - moxa_boards) * MAX_PORTS_PER_BOARD;
-	for (a = 0; a < brd->numPorts; a++)
-		tty_unregister_device(moxaDriver, first_idx + a);
-
-	iounmap(brd->basemem);
-	brd->basemem = NULL;
-	kfree(brd->ports);
-}
-
-static int moxa_pci_probe(struct pci_dev *pdev,
-		const struct pci_device_id *ent)
-{
-	struct moxa_board_conf *board;
-	unsigned int i;
-	int board_type = ent->driver_data;
-	int retval;
-
-	retval = pci_enable_device(pdev);
-	if (retval) {
-		dev_err(&pdev->dev, "can't enable pci device\n");
-		goto err;
-	}
-
-	for (i = 0; i < MAX_BOARDS; i++)
-		if (moxa_boards[i].basemem == NULL)
-			break;
-
-	retval = -ENODEV;
-	if (i >= MAX_BOARDS) {
-		dev_warn(&pdev->dev, "more than %u MOXA Intellio family boards "
-				"found. Board is ignored.\n", MAX_BOARDS);
-		goto err;
-	}
-
-	board = &moxa_boards[i];
-
-	retval = pci_request_region(pdev, 2, "moxa-base");
-	if (retval) {
-		dev_err(&pdev->dev, "can't request pci region 2\n");
-		goto err;
-	}
-
-	board->basemem = ioremap(pci_resource_start(pdev, 2), 0x4000);
-	if (board->basemem == NULL) {
-		dev_err(&pdev->dev, "can't remap io space 2\n");
-		retval = -ENOMEM;
-		goto err_reg;
-	}
-
-	board->boardType = board_type;
-	switch (board_type) {
-	case MOXA_BOARD_C218_PCI:
-		board->numPorts = 8;
-		break;
-
-	case MOXA_BOARD_CP204J:
-		board->numPorts = 4;
-		break;
-	default:
-		board->numPorts = 0;
-		break;
-	}
-
-	retval = moxa_init_board(board, &pdev->dev);
-	if (retval)
-		goto err_base;
-
-	pci_set_drvdata(pdev, board);
-
-	dev_info(&pdev->dev, "board '%s' ready (%u ports, firmware loaded)\n",
-			moxa_brdname[board_type - 1], board->numPorts);
-
-	return 0;
-err_base:
-	iounmap(board->basemem);
-	board->basemem = NULL;
-err_reg:
-	pci_release_region(pdev, 2);
-err:
-	return retval;
-}
-
-static void moxa_pci_remove(struct pci_dev *pdev)
-{
-	struct moxa_board_conf *brd = pci_get_drvdata(pdev);
-
-	moxa_board_deinit(brd);
-
-	pci_release_region(pdev, 2);
-}
-
-static struct pci_driver moxa_pci_driver = {
-	.name = "moxa",
-	.id_table = moxa_pcibrds,
-	.probe = moxa_pci_probe,
-	.remove = moxa_pci_remove
-};
-
-static int __init moxa_init(void)
-{
-	int retval = 0;
-
-	moxaDriver = tty_alloc_driver(MAX_PORTS,
-			TTY_DRIVER_REAL_RAW |
-			TTY_DRIVER_DYNAMIC_DEV);
-	if (IS_ERR(moxaDriver))
-		return PTR_ERR(moxaDriver);
-
-	moxaDriver->name = "ttyMX";
-	moxaDriver->major = ttymajor;
-	moxaDriver->minor_start = 0;
-	moxaDriver->type = TTY_DRIVER_TYPE_SERIAL;
-	moxaDriver->subtype = SERIAL_TYPE_NORMAL;
-	moxaDriver->init_termios = tty_std_termios;
-	moxaDriver->init_termios.c_cflag = B9600 | CS8 | CREAD | CLOCAL | HUPCL;
-	moxaDriver->init_termios.c_ispeed = 9600;
-	moxaDriver->init_termios.c_ospeed = 9600;
-	tty_set_operations(moxaDriver, &moxa_ops);
-
-	if (tty_register_driver(moxaDriver)) {
-		printk(KERN_ERR "can't register MOXA Smartio tty driver!\n");
-		tty_driver_kref_put(moxaDriver);
-		return -1;
-	}
-
-	retval = pci_register_driver(&moxa_pci_driver);
-	if (retval)
-		printk(KERN_ERR "Can't register MOXA pci driver!\n");
-
-	return retval;
-}
-
-static void __exit moxa_exit(void)
-{
-	pci_unregister_driver(&moxa_pci_driver);
-
-	timer_delete_sync(&moxaTimer);
-
-	tty_unregister_driver(moxaDriver);
-	tty_driver_kref_put(moxaDriver);
-}
-
-module_init(moxa_init);
-module_exit(moxa_exit);
-
-static void moxa_shutdown(struct tty_port *port)
-{
-	struct moxa_port *ch = container_of(port, struct moxa_port, port);
-        MoxaPortDisable(ch);
-	MoxaPortFlushData(ch, 2);
-}
-
-static bool moxa_carrier_raised(struct tty_port *port)
-{
-	struct moxa_port *ch = container_of(port, struct moxa_port, port);
-
-	guard(spinlock_irq)(&port->lock);
-	return ch->DCDState;
-}
-
-static void moxa_dtr_rts(struct tty_port *port, bool active)
-{
-	struct moxa_port *ch = container_of(port, struct moxa_port, port);
-	MoxaPortLineCtrl(ch, active, active);
-}
-
-
-static int moxa_open(struct tty_struct *tty, struct file *filp)
-{
-	struct moxa_board_conf *brd;
-	struct moxa_port *ch;
-	int port = tty->index;
-
-	scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &moxa_openlock) {
-		brd = &moxa_boards[port / MAX_PORTS_PER_BOARD];
-		if (!brd->ready)
-			return -ENODEV;
-
-		if (port % MAX_PORTS_PER_BOARD >= brd->numPorts)
-			return -ENODEV;
-
-		ch = &brd->ports[port % MAX_PORTS_PER_BOARD];
-		ch->port.count++;
-		tty->driver_data = ch;
-		tty_port_tty_set(&ch->port, tty);
-
-		guard(mutex)(&ch->port.mutex);
-		if (!tty_port_initialized(&ch->port)) {
-			ch->statusflags = 0;
-			moxa_set_tty_param(tty, &tty->termios);
-			MoxaPortLineCtrl(ch, true, true);
-			MoxaPortEnable(ch);
-			MoxaSetFifo(ch, ch->type == PORT_16550A);
-			tty_port_set_initialized(&ch->port, true);
-		}
-	}
-
-	return tty_port_block_til_ready(&ch->port, tty, filp);
-}
-
-static void moxa_close(struct tty_struct *tty, struct file *filp)
-{
-	struct moxa_port *ch = tty->driver_data;
-	ch->cflag = tty->termios.c_cflag;
-	tty_port_close(&ch->port, tty, filp);
-}
-
-static ssize_t moxa_write(struct tty_struct *tty, const u8 *buf, size_t count)
-{
-	struct moxa_port *ch = tty->driver_data;
-	int len;
-
-	if (ch == NULL)
-		return 0;
-
-	scoped_guard(spinlock_irqsave, &moxa_lock)
-		len = MoxaPortWriteData(tty, buf, count);
-
-	set_bit(LOWWAIT, &ch->statusflags);
-	return len;
-}
-
-static unsigned int moxa_write_room(struct tty_struct *tty)
-{
-	struct moxa_port *ch;
-
-	if (tty->flow.stopped)
-		return 0;
-	ch = tty->driver_data;
-	if (ch == NULL)
-		return 0;
-	return MoxaPortTxFree(ch);
-}
-
-static void moxa_flush_buffer(struct tty_struct *tty)
-{
-	struct moxa_port *ch = tty->driver_data;
-
-	if (ch == NULL)
-		return;
-	MoxaPortFlushData(ch, 1);
-	tty_wakeup(tty);
-}
-
-static unsigned int moxa_chars_in_buffer(struct tty_struct *tty)
-{
-	struct moxa_port *ch = tty->driver_data;
-	unsigned int chars;
-
-	chars = MoxaPortTxQueue(ch);
-	if (chars)
-		/*
-		 * Make it possible to wakeup anything waiting for output
-		 * in tty_ioctl.c, etc.
-		 */
-        	set_bit(EMPTYWAIT, &ch->statusflags);
-	return chars;
-}
-
-static int moxa_tiocmget(struct tty_struct *tty)
-{
-	struct moxa_port *ch = tty->driver_data;
-	bool dtr_active, rts_active;
-	int flag = 0;
-	int status;
-
-	MoxaPortGetLineOut(ch, &dtr_active, &rts_active);
-	if (dtr_active)
-		flag |= TIOCM_DTR;
-	if (rts_active)
-		flag |= TIOCM_RTS;
-	status = MoxaPortLineStatus(ch);
-	if (status & 1)
-		flag |= TIOCM_CTS;
-	if (status & 2)
-		flag |= TIOCM_DSR;
-	if (status & 4)
-		flag |= TIOCM_CD;
-	return flag;
-}
-
-static int moxa_tiocmset(struct tty_struct *tty,
-			 unsigned int set, unsigned int clear)
-{
-	bool dtr_active, rts_active;
-	struct moxa_port *ch;
-
-	guard(mutex)(&moxa_openlock);
-	ch = tty->driver_data;
-	if (!ch)
-		return -EINVAL;
-
-	MoxaPortGetLineOut(ch, &dtr_active, &rts_active);
-	if (set & TIOCM_RTS)
-		rts_active = true;
-	if (set & TIOCM_DTR)
-		dtr_active = true;
-	if (clear & TIOCM_RTS)
-		rts_active = false;
-	if (clear & TIOCM_DTR)
-		dtr_active = false;
-	MoxaPortLineCtrl(ch, dtr_active, rts_active);
-
-	return 0;
-}
-
-static void moxa_set_termios(struct tty_struct *tty,
-		             const struct ktermios *old_termios)
-{
-	struct moxa_port *ch = tty->driver_data;
-
-	if (ch == NULL)
-		return;
-	moxa_set_tty_param(tty, old_termios);
-	if (!(old_termios->c_cflag & CLOCAL) && C_CLOCAL(tty))
-		wake_up_interruptible(&ch->port.open_wait);
-}
-
-static void moxa_stop(struct tty_struct *tty)
-{
-	struct moxa_port *ch = tty->driver_data;
-
-	if (ch == NULL)
-		return;
-	MoxaPortTxDisable(ch);
-	set_bit(TXSTOPPED, &ch->statusflags);
-}
-
-
-static void moxa_start(struct tty_struct *tty)
-{
-	struct moxa_port *ch = tty->driver_data;
-
-	if (ch == NULL)
-		return;
-
-	if (!test_bit(TXSTOPPED, &ch->statusflags))
-		return;
-
-	MoxaPortTxEnable(ch);
-	clear_bit(TXSTOPPED, &ch->statusflags);
-}
-
-static void moxa_hangup(struct tty_struct *tty)
-{
-	struct moxa_port *ch = tty->driver_data;
-	tty_port_hangup(&ch->port);
-}
-
-static void moxa_new_dcdstate(struct moxa_port *p, u8 dcd)
-{
-	dcd = !!dcd;
-
-	scoped_guard(spinlock_irqsave, &p->port.lock) {
-		if (dcd == p->DCDState)
-			return;
-
-		p->DCDState = dcd;
-	}
-
-	if (!dcd)
-		tty_port_tty_hangup(&p->port, true);
-}
-
-static int moxa_poll_port(struct moxa_port *p, unsigned int handle,
-		u16 __iomem *ip)
-{
-	struct tty_struct *tty = tty_port_tty_get(&p->port);
-	bool inited = tty_port_initialized(&p->port);
-	void __iomem *ofsAddr;
-	u16 intr;
-
-	if (tty) {
-		if (test_bit(EMPTYWAIT, &p->statusflags) &&
-				MoxaPortTxQueue(p) == 0) {
-			clear_bit(EMPTYWAIT, &p->statusflags);
-			tty_wakeup(tty);
-		}
-		if (test_bit(LOWWAIT, &p->statusflags) && !tty->flow.stopped &&
-				MoxaPortTxQueue(p) <= WAKEUP_CHARS) {
-			clear_bit(LOWWAIT, &p->statusflags);
-			tty_wakeup(tty);
-		}
-
-		if (inited && !tty_throttled(tty) &&
-				MoxaPortRxQueue(p) > 0) { /* RX */
-			MoxaPortReadData(p);
-			tty_flip_buffer_push(&p->port);
-		}
-	} else {
-		clear_bit(EMPTYWAIT, &p->statusflags);
-		MoxaPortFlushData(p, 0); /* flush RX */
-	}
-
-	if (!handle) /* nothing else to do */
-		goto put;
-
-	intr = readw(ip); /* port irq status */
-	if (intr == 0)
-		goto put;
-
-	writew(0, ip); /* ACK port */
-	ofsAddr = p->tableAddr;
-	if (intr & IntrTx) /* disable tx intr */
-		writew(readw(ofsAddr + HostStat) & ~WakeupTx,
-				ofsAddr + HostStat);
-
-	if (!inited)
-		goto put;
-
-	if (tty && (intr & IntrBreak) && !I_IGNBRK(tty)) { /* BREAK */
-		tty_insert_flip_char(&p->port, 0, TTY_BREAK);
-		tty_flip_buffer_push(&p->port);
-	}
-
-	if (intr & IntrLine)
-		moxa_new_dcdstate(p, readb(ofsAddr + FlagStat) & DCD_state);
-put:
-	tty_kref_put(tty);
-
-	return 0;
-}
-
-static void moxa_poll(struct timer_list *unused)
-{
-	struct moxa_board_conf *brd;
-	u16 __iomem *ip;
-	unsigned int card, port, served = 0;
-
-	guard(spinlock)(&moxa_lock);
-	for (card = 0; card < MAX_BOARDS; card++) {
-		brd = &moxa_boards[card];
-		if (!brd->ready)
-			continue;
-
-		served++;
-
-		ip = NULL;
-		if (readb(brd->intPend) == 0xff)
-			ip = brd->intTable + readb(brd->intNdx);
-
-		for (port = 0; port < brd->numPorts; port++)
-			moxa_poll_port(&brd->ports[port], !!ip, ip + port);
-
-		if (ip)
-			writeb(0, brd->intPend); /* ACK */
-
-		if (moxaLowWaterChk) {
-			struct moxa_port *p = brd->ports;
-			for (port = 0; port < brd->numPorts; port++, p++)
-				if (p->lowChkFlag) {
-					p->lowChkFlag = 0;
-					moxa_low_water_check(p->tableAddr);
-				}
-		}
-	}
-	moxaLowWaterChk = 0;
-
-	if (served)
-		mod_timer(&moxaTimer, jiffies + HZ / 50);
-}
-
-/******************************************************************************/
-
-static void moxa_set_tty_param(struct tty_struct *tty,
-			       const struct ktermios *old_termios)
-{
-	register struct ktermios *ts = &tty->termios;
-	struct moxa_port *ch = tty->driver_data;
-	int rts, cts, txflow, rxflow, xany, baud;
-
-	rts = cts = txflow = rxflow = xany = 0;
-	if (ts->c_cflag & CRTSCTS)
-		rts = cts = 1;
-	if (ts->c_iflag & IXON)
-		txflow = 1;
-	if (ts->c_iflag & IXOFF)
-		rxflow = 1;
-	if (ts->c_iflag & IXANY)
-		xany = 1;
-
-	MoxaPortFlowCtrl(ch, rts, cts, txflow, rxflow, xany);
-	baud = MoxaPortSetTermio(ch, ts, tty_get_baud_rate(tty));
-	if (baud == -1)
-		baud = tty_termios_baud_rate(old_termios);
-	/* Not put the baud rate into the termios data */
-	tty_encode_baud_rate(tty, baud, baud);
-}
-
-/*****************************************************************************
- *	Driver level functions: 					     *
- *****************************************************************************/
-
-static void MoxaPortFlushData(struct moxa_port *port, int mode)
-{
-	void __iomem *ofsAddr;
-	if (mode < 0 || mode > 2)
-		return;
-	ofsAddr = port->tableAddr;
-	moxafunc(ofsAddr, FC_FlushQueue, mode);
-	if (mode != 1) {
-		port->lowChkFlag = 0;
-		moxa_low_water_check(ofsAddr);
-	}
-}
-
-/*
- *    Moxa Port Number Description:
- *
- *      MOXA serial driver supports up to 4 MOXA-C218/C320 boards. And,
- *      the port number using in MOXA driver functions will be 0 to 31 for
- *      first MOXA board, 32 to 63 for second, 64 to 95 for third and 96
- *      to 127 for fourth. For example, if you setup three MOXA boards,
- *      first board is C218, second board is C320-16 and third board is
- *      C320-32. The port number of first board (C218 - 8 ports) is from
- *      0 to 7. The port number of second board (C320 - 16 ports) is form
- *      32 to 47. The port number of third board (C320 - 32 ports) is from
- *      64 to 95. And those port numbers form 8 to 31, 48 to 63 and 96 to
- *      127 will be invalid.
- *
- *
- *      Moxa Functions Description:
- *
- *      Function 1:     Driver initialization routine, this routine must be
- *                      called when initialized driver.
- *      Syntax:
- *      void MoxaDriverInit();
- *
- *
- *      Function 2:     Moxa driver private IOCTL command processing.
- *      Syntax:
- *      int  MoxaDriverIoctl(unsigned int cmd, unsigned long arg, int port);
- *
- *           unsigned int cmd   : IOCTL command
- *           unsigned long arg  : IOCTL argument
- *           int port           : port number (0 - 127)
- *
- *           return:    0  (OK)
- *                      -EINVAL
- *                      -ENOIOCTLCMD
- *
- *
- *      Function 6:     Enable this port to start Tx/Rx data.
- *      Syntax:
- *      void MoxaPortEnable(int port);
- *           int port           : port number (0 - 127)
- *
- *
- *      Function 7:     Disable this port
- *      Syntax:
- *      void MoxaPortDisable(int port);
- *           int port           : port number (0 - 127)
- *
- *
- *      Function 10:    Setting baud rate of this port.
- *      Syntax:
- *      speed_t MoxaPortSetBaud(int port, speed_t baud);
- *           int port           : port number (0 - 127)
- *           long baud          : baud rate (50 - 115200)
- *
- *           return:    0       : this port is invalid or baud < 50
- *                      50 - 115200 : the real baud rate set to the port, if
- *                                    the argument baud is large than maximun
- *                                    available baud rate, the real setting
- *                                    baud rate will be the maximun baud rate.
- *
- *
- *      Function 12:    Configure the port.
- *      Syntax:
- *      int  MoxaPortSetTermio(int port, struct ktermios *termio, speed_t baud);
- *           int port           : port number (0 - 127)
- *           struct ktermios * termio : termio structure pointer
- *	     speed_t baud	: baud rate
- *
- *           return:    -1      : this port is invalid or termio == NULL
- *                      0       : setting O.K.
- *
- *
- *      Function 13:    Get the DTR/RTS state of this port.
- *      Syntax:
- *      int  MoxaPortGetLineOut(int port, bool *dtrState, bool *rtsState);
- *           int port           : port number (0 - 127)
- *           bool * dtr_active  : pointer to bool to receive the current DTR
- *                                state. (if NULL, this function will not
- *                                write to this address)
- *           bool * rts_active  : pointer to bool to receive the current RTS
- *                                state. (if NULL, this function will not
- *                                write to this address)
- *
- *           return:    -1      : this port is invalid
- *                      0       : O.K.
- *
- *
- *      Function 14:    Setting the DTR/RTS output state of this port.
- *      Syntax:
- *      void MoxaPortLineCtrl(int port, bool dtrState, bool rtsState);
- *           int port           : port number (0 - 127)
- *           bool dtr_active    : DTR output state
- *           bool rts_active    : RTS output state
- *
- *
- *      Function 15:    Setting the flow control of this port.
- *      Syntax:
- *      void MoxaPortFlowCtrl(int port, int rtsFlow, int ctsFlow, int rxFlow,
- *                            int txFlow,int xany);
- *           int port           : port number (0 - 127)
- *           int rtsFlow        : H/W RTS flow control (0: no, 1: yes)
- *           int ctsFlow        : H/W CTS flow control (0: no, 1: yes)
- *           int rxFlow         : S/W Rx XON/XOFF flow control (0: no, 1: yes)
- *           int txFlow         : S/W Tx XON/XOFF flow control (0: no, 1: yes)
- *           int xany           : S/W XANY flow control (0: no, 1: yes)
- *
- *
- *      Function 16:    Get ths line status of this port
- *      Syntax:
- *      int  MoxaPortLineStatus(int port);
- *           int port           : port number (0 - 127)
- *
- *           return:    Bit 0 - CTS state (0: off, 1: on)
- *                      Bit 1 - DSR state (0: off, 1: on)
- *                      Bit 2 - DCD state (0: off, 1: on)
- *
- *
- *      Function 19:    Flush the Rx/Tx buffer data of this port.
- *      Syntax:
- *      void MoxaPortFlushData(int port, int mode);
- *           int port           : port number (0 - 127)
- *           int mode    
- *                      0       : flush the Rx buffer 
- *                      1       : flush the Tx buffer 
- *                      2       : flush the Rx and Tx buffer 
- *
- *
- *      Function 20:    Write data.
- *      Syntax:
- *      ssize_t  MoxaPortWriteData(int port, u8 *buffer, size_t length);
- *           int port           : port number (0 - 127)
- *           u8 *buffer         : pointer to write data buffer.
- *           size_t length      : write data length
- *
- *           return:    0 - length      : real write data length
- *
- *
- *      Function 21:    Read data.
- *      Syntax:
- *      int  MoxaPortReadData(int port, struct tty_struct *tty);
- *           int port           : port number (0 - 127)
- *	     struct tty_struct *tty : tty for data
- *
- *           return:    0 - length      : real read data length
- *
- *
- *      Function 24:    Get the Tx buffer current queued data bytes
- *      Syntax:
- *      int  MoxaPortTxQueue(int port);
- *           int port           : port number (0 - 127)
- *
- *           return:    ..      : Tx buffer current queued data bytes
- *
- *
- *      Function 25:    Get the Tx buffer current free space
- *      Syntax:
- *      int  MoxaPortTxFree(int port);
- *           int port           : port number (0 - 127)
- *
- *           return:    ..      : Tx buffer current free space
- *
- *
- *      Function 26:    Get the Rx buffer current queued data bytes
- *      Syntax:
- *      int  MoxaPortRxQueue(int port);
- *           int port           : port number (0 - 127)
- *
- *           return:    ..      : Rx buffer current queued data bytes
- *
- *
- *      Function 28:    Disable port data transmission.
- *      Syntax:
- *      void MoxaPortTxDisable(int port);
- *           int port           : port number (0 - 127)
- *
- *
- *      Function 29:    Enable port data transmission.
- *      Syntax:
- *      void MoxaPortTxEnable(int port);
- *           int port           : port number (0 - 127)
- *
- *
- *      Function 31:    Get the received BREAK signal count and reset it.
- *      Syntax:
- *      int  MoxaPortResetBrkCnt(int port);
- *           int port           : port number (0 - 127)
- *
- *           return:    0 - ..  : BREAK signal count
- *
- *
- */
-
-static void MoxaPortEnable(struct moxa_port *port)
-{
-	void __iomem *ofsAddr;
-	u16 lowwater = 512;
-
-	ofsAddr = port->tableAddr;
-	writew(lowwater, ofsAddr + Low_water);
-	if (MOXA_IS_320(port->board))
-		moxafunc(ofsAddr, FC_SetBreakIrq, 0);
-	else
-		writew(readw(ofsAddr + HostStat) | WakeupBreak,
-				ofsAddr + HostStat);
-
-	moxafunc(ofsAddr, FC_SetLineIrq, Magic_code);
-	moxafunc(ofsAddr, FC_FlushQueue, 2);
-
-	moxafunc(ofsAddr, FC_EnableCH, Magic_code);
-	MoxaPortLineStatus(port);
-}
-
-static void MoxaPortDisable(struct moxa_port *port)
-{
-	void __iomem *ofsAddr = port->tableAddr;
-
-	moxafunc(ofsAddr, FC_SetFlowCtl, 0);	/* disable flow control */
-	moxafunc(ofsAddr, FC_ClrLineIrq, Magic_code);
-	writew(0, ofsAddr + HostStat);
-	moxafunc(ofsAddr, FC_DisableCH, Magic_code);
-}
-
-static speed_t MoxaPortSetBaud(struct moxa_port *port, speed_t baud)
-{
-	void __iomem *ofsAddr = port->tableAddr;
-	unsigned int clock, val;
-	speed_t max;
-
-	max = MOXA_IS_320(port->board) ? 460800 : 921600;
-	if (baud < 50)
-		return 0;
-	if (baud > max)
-		baud = max;
-	clock = 921600;
-	val = clock / baud;
-	moxafunc(ofsAddr, FC_SetBaud, val);
-	baud = clock / val;
-	return baud;
-}
-
-static int MoxaPortSetTermio(struct moxa_port *port, struct ktermios *termio,
-		speed_t baud)
-{
-	void __iomem *ofsAddr;
-	tcflag_t mode = 0;
-
-	ofsAddr = port->tableAddr;
-
-	mode = termio->c_cflag & CSIZE;
-	if (mode == CS5)
-		mode = MX_CS5;
-	else if (mode == CS6)
-		mode = MX_CS6;
-	else if (mode == CS7)
-		mode = MX_CS7;
-	else if (mode == CS8)
-		mode = MX_CS8;
-
-	if (termio->c_cflag & CSTOPB) {
-		if (mode == MX_CS5)
-			mode |= MX_STOP15;
-		else
-			mode |= MX_STOP2;
-	} else
-		mode |= MX_STOP1;
-
-	if (termio->c_cflag & PARENB) {
-		if (termio->c_cflag & PARODD) {
-			if (termio->c_cflag & CMSPAR)
-				mode |= MX_PARMARK;
-			else
-				mode |= MX_PARODD;
-		} else {
-			if (termio->c_cflag & CMSPAR)
-				mode |= MX_PARSPACE;
-			else
-				mode |= MX_PAREVEN;
-		}
-	} else
-		mode |= MX_PARNONE;
-
-	moxafunc(ofsAddr, FC_SetDataMode, (u16)mode);
-
-	if (MOXA_IS_320(port->board) && baud >= 921600)
-		return -1;
-
-	baud = MoxaPortSetBaud(port, baud);
-
-	if (termio->c_iflag & (IXON | IXOFF | IXANY)) {
-		guard(spinlock_irq)(&moxafunc_lock);
-		writeb(termio->c_cc[VSTART], ofsAddr + FuncArg);
-		writeb(termio->c_cc[VSTOP], ofsAddr + FuncArg1);
-		writeb(FC_SetXonXoff, ofsAddr + FuncCode);
-		moxa_wait_finish(ofsAddr);
-	}
-	return baud;
-}
-
-static int MoxaPortGetLineOut(struct moxa_port *port, bool *dtr_active,
-		bool *rts_active)
-{
-	if (dtr_active)
-		*dtr_active = port->lineCtrl & DTR_ON;
-	if (rts_active)
-		*rts_active = port->lineCtrl & RTS_ON;
-
-	return 0;
-}
-
-static void MoxaPortLineCtrl(struct moxa_port *port, bool dtr_active, bool rts_active)
-{
-	u8 mode = 0;
-
-	if (dtr_active)
-		mode |= DTR_ON;
-	if (rts_active)
-		mode |= RTS_ON;
-	port->lineCtrl = mode;
-	moxafunc(port->tableAddr, FC_LineControl, mode);
-}
-
-static void MoxaPortFlowCtrl(struct moxa_port *port, int rts, int cts,
-		int txflow, int rxflow, int txany)
-{
-	int mode = 0;
-
-	if (rts)
-		mode |= RTS_FlowCtl;
-	if (cts)
-		mode |= CTS_FlowCtl;
-	if (txflow)
-		mode |= Tx_FlowCtl;
-	if (rxflow)
-		mode |= Rx_FlowCtl;
-	if (txany)
-		mode |= IXM_IXANY;
-	moxafunc(port->tableAddr, FC_SetFlowCtl, mode);
-}
-
-static int MoxaPortLineStatus(struct moxa_port *port)
-{
-	void __iomem *ofsAddr;
-	int val;
-
-	ofsAddr = port->tableAddr;
-	if (MOXA_IS_320(port->board))
-		val = moxafuncret(ofsAddr, FC_LineStatus, 0);
-	else
-		val = readw(ofsAddr + FlagStat) >> 4;
-	val &= 0x0B;
-	if (val & 8)
-		val |= 4;
-	moxa_new_dcdstate(port, val & 8);
-	val &= 7;
-	return val;
-}
-
-static ssize_t MoxaPortWriteData(struct tty_struct *tty, const u8 *buffer,
-				 size_t len)
-{
-	struct moxa_port *port = tty->driver_data;
-	void __iomem *baseAddr, *ofsAddr, *ofs;
-	size_t c, total;
-	u16 head, tail, tx_mask, spage, epage;
-	u16 pageno, pageofs, bufhead;
-
-	ofsAddr = port->tableAddr;
-	baseAddr = port->board->basemem;
-	tx_mask = readw(ofsAddr + TX_mask);
-	spage = readw(ofsAddr + Page_txb);
-	epage = readw(ofsAddr + EndPage_txb);
-	tail = readw(ofsAddr + TXwptr);
-	head = readw(ofsAddr + TXrptr);
-	c = (head > tail) ? (head - tail - 1) : (head - tail + tx_mask);
-	if (c > len)
-		c = len;
-	total = c;
-	if (spage == epage) {
-		bufhead = readw(ofsAddr + Ofs_txb);
-		writew(spage, baseAddr + Control_reg);
-		while (c > 0) {
-			if (head > tail)
-				len = head - tail - 1;
-			else
-				len = tx_mask + 1 - tail;
-			len = (c > len) ? len : c;
-			ofs = baseAddr + DynPage_addr + bufhead + tail;
-			memcpy_toio(ofs, buffer, len);
-			buffer += len;
-			tail = (tail + len) & tx_mask;
-			c -= len;
-		}
-	} else {
-		pageno = spage + (tail >> 13);
-		pageofs = tail & Page_mask;
-		while (c > 0) {
-			len = Page_size - pageofs;
-			if (len > c)
-				len = c;
-			writeb(pageno, baseAddr + Control_reg);
-			ofs = baseAddr + DynPage_addr + pageofs;
-			memcpy_toio(ofs, buffer, len);
-			buffer += len;
-			if (++pageno == epage)
-				pageno = spage;
-			pageofs = 0;
-			c -= len;
-		}
-		tail = (tail + total) & tx_mask;
-	}
-	writew(tail, ofsAddr + TXwptr);
-	writeb(1, ofsAddr + CD180TXirq);	/* start to send */
-	return total;
-}
-
-static int MoxaPortReadData(struct moxa_port *port)
-{
-	void __iomem *baseAddr, *ofsAddr, *ofs;
-	u8 *dst;
-	unsigned int count, len, total;
-	u16 tail, rx_mask, spage, epage;
-	u16 pageno, pageofs, bufhead, head;
-
-	ofsAddr = port->tableAddr;
-	baseAddr = port->board->basemem;
-	head = readw(ofsAddr + RXrptr);
-	tail = readw(ofsAddr + RXwptr);
-	rx_mask = readw(ofsAddr + RX_mask);
-	spage = readw(ofsAddr + Page_rxb);
-	epage = readw(ofsAddr + EndPage_rxb);
-	count = (tail >= head) ? (tail - head) : (tail - head + rx_mask + 1);
-	if (count == 0)
-		return 0;
-
-	total = count;
-	if (spage == epage) {
-		bufhead = readw(ofsAddr + Ofs_rxb);
-		writew(spage, baseAddr + Control_reg);
-		while (count > 0) {
-			ofs = baseAddr + DynPage_addr + bufhead + head;
-			len = (tail >= head) ? (tail - head) :
-					(rx_mask + 1 - head);
-			len = tty_prepare_flip_string(&port->port, &dst,
-					min(len, count));
-			memcpy_fromio(dst, ofs, len);
-			head = (head + len) & rx_mask;
-			count -= len;
-		}
-	} else {
-		pageno = spage + (head >> 13);
-		pageofs = head & Page_mask;
-		while (count > 0) {
-			writew(pageno, baseAddr + Control_reg);
-			ofs = baseAddr + DynPage_addr + pageofs;
-			len = tty_prepare_flip_string(&port->port, &dst,
-					min(Page_size - pageofs, count));
-			memcpy_fromio(dst, ofs, len);
-
-			count -= len;
-			pageofs = (pageofs + len) & Page_mask;
-			if (pageofs == 0 && ++pageno == epage)
-				pageno = spage;
-		}
-		head = (head + total) & rx_mask;
-	}
-	writew(head, ofsAddr + RXrptr);
-	if (readb(ofsAddr + FlagStat) & Xoff_state) {
-		moxaLowWaterChk = 1;
-		port->lowChkFlag = 1;
-	}
-	return total;
-}
-
-
-static unsigned int MoxaPortTxQueue(struct moxa_port *port)
-{
-	void __iomem *ofsAddr = port->tableAddr;
-	u16 rptr, wptr, mask;
-
-	rptr = readw(ofsAddr + TXrptr);
-	wptr = readw(ofsAddr + TXwptr);
-	mask = readw(ofsAddr + TX_mask);
-	return (wptr - rptr) & mask;
-}
-
-static unsigned int MoxaPortTxFree(struct moxa_port *port)
-{
-	void __iomem *ofsAddr = port->tableAddr;
-	u16 rptr, wptr, mask;
-
-	rptr = readw(ofsAddr + TXrptr);
-	wptr = readw(ofsAddr + TXwptr);
-	mask = readw(ofsAddr + TX_mask);
-	return mask - ((wptr - rptr) & mask);
-}
-
-static int MoxaPortRxQueue(struct moxa_port *port)
-{
-	void __iomem *ofsAddr = port->tableAddr;
-	u16 rptr, wptr, mask;
-
-	rptr = readw(ofsAddr + RXrptr);
-	wptr = readw(ofsAddr + RXwptr);
-	mask = readw(ofsAddr + RX_mask);
-	return (wptr - rptr) & mask;
-}
-
-static void MoxaPortTxDisable(struct moxa_port *port)
-{
-	moxafunc(port->tableAddr, FC_SetXoffState, Magic_code);
-}
-
-static void MoxaPortTxEnable(struct moxa_port *port)
-{
-	moxafunc(port->tableAddr, FC_SetXonState, Magic_code);
-}
-
-static int moxa_get_serial_info(struct tty_struct *tty,
-		struct serial_struct *ss)
-{
-	struct moxa_port *info = tty->driver_data;
-
-	if (!info)
-		return -ENODEV;
-	guard(mutex)(&info->port.mutex);
-	ss->type = info->type;
-	ss->line = info->port.tty->index;
-	ss->flags = info->port.flags;
-	ss->baud_base = 921600;
-	ss->close_delay = jiffies_to_msecs(info->port.close_delay) / 10;
-
-	return 0;
-}
-
-
-static int moxa_set_serial_info(struct tty_struct *tty,
-		struct serial_struct *ss)
-{
-	struct moxa_port *info = tty->driver_data;
-	unsigned int close_delay;
-
-	if (!info)
-		return -ENODEV;
-
-	close_delay = msecs_to_jiffies(ss->close_delay * 10);
-
-	guard(mutex)(&info->port.mutex);
-	if (!capable(CAP_SYS_ADMIN)) {
-		if (close_delay != info->port.close_delay ||
-		    ss->type != info->type ||
-		    ((ss->flags & ~ASYNC_USR_MASK) !=
-		     (info->port.flags & ~ASYNC_USR_MASK))) {
-			return -EPERM;
-		}
-	} else {
-		info->port.close_delay = close_delay;
-
-		MoxaSetFifo(info, ss->type == PORT_16550A);
-
-		info->type = ss->type;
-	}
-
-	return 0;
-}
-
-
-
-/*****************************************************************************
- *	Static local functions: 					     *
- *****************************************************************************/
-
-static void MoxaSetFifo(struct moxa_port *port, int enable)
-{
-	void __iomem *ofsAddr = port->tableAddr;
-
-	if (!enable) {
-		moxafunc(ofsAddr, FC_SetRxFIFOTrig, 0);
-		moxafunc(ofsAddr, FC_SetTxFIFOCnt, 1);
-	} else {
-		moxafunc(ofsAddr, FC_SetRxFIFOTrig, 3);
-		moxafunc(ofsAddr, FC_SetTxFIFOCnt, 16);
-	}
-}
diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c
index c13e050de83b12..e55779ff0af2bb 100644
--- a/drivers/tty/n_gsm.c
+++ b/drivers/tty/n_gsm.c
@@ -454,14 +454,13 @@ static const u8 gsm_fcs8[256] = {
 
 static void gsm_dlci_close(struct gsm_dlci *dlci);
 static int gsmld_output(struct gsm_mux *gsm, u8 *data, int len);
-static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk);
+static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk, bool wait);
 static struct gsm_msg *gsm_data_alloc(struct gsm_mux *gsm, u8 addr, int len,
 								u8 ctrl);
 static int gsm_send_packet(struct gsm_mux *gsm, struct gsm_msg *msg);
 static struct gsm_dlci *gsm_dlci_alloc(struct gsm_mux *gsm, int addr);
 static void gsmld_write_trigger(struct gsm_mux *gsm);
 static void gsmld_write_task(struct work_struct *work);
-static int gsm_modem_send_initial_msc(struct gsm_dlci *dlci);
 
 /**
  *	gsm_fcs_add	-	update FCS
@@ -2174,7 +2173,7 @@ static void gsm_dlci_open(struct gsm_dlci *dlci)
 		pr_debug("DLCI %d goes open.\n", dlci->addr);
 	/* Send current modem state */
 	if (dlci->addr) {
-		gsm_modem_send_initial_msc(dlci);
+		gsm_modem_update(dlci, 0, false);
 	} else {
 		/* Start keep-alive control */
 		gsm->ka_num = 0;
@@ -4136,9 +4135,10 @@ static void gsm_modem_upd_via_data(struct gsm_dlci *dlci, u8 brk)
  *	gsm_modem_upd_via_msc	-	send modem bits via control frame
  *	@dlci: channel
  *	@brk: break signal
+ *	@wait: wait for the MSC command
  */
 
-static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk)
+static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk, bool wait)
 {
 	u8 modembits[3];
 	struct gsm_control *ctrl;
@@ -4155,41 +4155,22 @@ static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk)
 		modembits[2] = (brk << 4) | 2 | EA; /* Length, Break, EA */
 		len++;
 	}
+	if (!wait)
+		return gsm_control_command(dlci->gsm, CMD_MSC, modembits, len);
 	ctrl = gsm_control_send(dlci->gsm, CMD_MSC, modembits, len);
 	if (ctrl == NULL)
 		return -ENOMEM;
 	return gsm_control_wait(dlci->gsm, ctrl);
 }
 
-/**
- * gsm_modem_send_initial_msc - Send initial modem status message
- *
- * @dlci: channel
- *
- * Send an initial MSC message after DLCI open to set the initial
- * modem status lines. This is only done for basic mode.
- * Does not wait for a response as we cannot block the input queue
- * processing.
- */
-static int gsm_modem_send_initial_msc(struct gsm_dlci *dlci)
-{
-	u8 modembits[2];
-
-	if (dlci->adaption != 1 || dlci->gsm->encoding != GSM_BASIC_OPT)
-		return 0;
-
-	modembits[0] = (dlci->addr << 2) | 2 | EA; /* DLCI, Valid, EA */
-	modembits[1] = (gsm_encode_modem(dlci) << 1) | EA;
-	return gsm_control_command(dlci->gsm, CMD_MSC, (const u8 *)&modembits, 2);
-}
-
 /**
  *	gsm_modem_update	-	send modem status line state
  *	@dlci: channel
  *	@brk: break signal
+ *	@wait: wait for the MSC command
  */
 
-static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk)
+static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk, bool wait)
 {
 	if (dlci->gsm->dead)
 		return -EL2HLT;
@@ -4199,7 +4180,7 @@ static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk)
 		return 0;
 	} else if (dlci->gsm->encoding == GSM_BASIC_OPT) {
 		/* Send as MSC control message. */
-		return gsm_modem_upd_via_msc(dlci, brk);
+		return gsm_modem_upd_via_msc(dlci, brk, wait);
 	}
 
 	/* Modem status lines are not supported. */
@@ -4265,7 +4246,7 @@ static void gsm_dtr_rts(struct tty_port *port, bool active)
 		modem_tx &= ~(TIOCM_DTR | TIOCM_RTS);
 	if (modem_tx != dlci->modem_tx) {
 		dlci->modem_tx = modem_tx;
-		gsm_modem_update(dlci, 0);
+		gsm_modem_update(dlci, 0, true);
 	}
 }
 
@@ -4471,7 +4452,7 @@ static int gsmtty_tiocmset(struct tty_struct *tty,
 
 	if (modem_tx != dlci->modem_tx) {
 		dlci->modem_tx = modem_tx;
-		return gsm_modem_update(dlci, 0);
+		return gsm_modem_update(dlci, 0, true);
 	}
 	return 0;
 }
@@ -4553,7 +4534,7 @@ static void gsmtty_throttle(struct tty_struct *tty)
 		dlci->modem_tx &= ~TIOCM_RTS;
 	dlci->throttled = true;
 	/* Send an MSC with RTS cleared */
-	gsm_modem_update(dlci, 0);
+	gsm_modem_update(dlci, 0, true);
 }
 
 static void gsmtty_unthrottle(struct tty_struct *tty)
@@ -4565,7 +4546,7 @@ static void gsmtty_unthrottle(struct tty_struct *tty)
 		dlci->modem_tx |= TIOCM_RTS;
 	dlci->throttled = false;
 	/* Send an MSC with RTS set */
-	gsm_modem_update(dlci, 0);
+	gsm_modem_update(dlci, 0, true);
 }
 
 static int gsmtty_break_ctl(struct tty_struct *tty, int state)
@@ -4583,7 +4564,7 @@ static int gsmtty_break_ctl(struct tty_struct *tty, int state)
 		if (encode > 0x0F)
 			encode = 0x0F;	/* Best effort */
 	}
-	return gsm_modem_update(dlci, encode);
+	return gsm_modem_update(dlci, encode, true);
 }
 
 static void gsmtty_cleanup(struct tty_struct *tty)
diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c
index cc7f7091ed9a20..9a37008c4e91dd 100644
--- a/drivers/tty/pty.c
+++ b/drivers/tty/pty.c
@@ -138,10 +138,7 @@ static int pty_set_lock(struct tty_struct *tty, int __user *arg)
 
 	if (get_user(val, arg))
 		return -EFAULT;
-	if (val)
-		set_bit(TTY_PTY_LOCK, &tty->flags);
-	else
-		clear_bit(TTY_PTY_LOCK, &tty->flags);
+	assign_bit(TTY_PTY_LOCK, &tty->flags, val);
 	return 0;
 }
 
diff --git a/drivers/tty/serdev/serdev-ttyport.c b/drivers/tty/serdev/serdev-ttyport.c
index bab1b143b8a68d..b6638f9f40d2d0 100644
--- a/drivers/tty/serdev/serdev-ttyport.c
+++ b/drivers/tty/serdev/serdev-ttyport.c
@@ -137,6 +137,7 @@ static int ttyport_open(struct serdev_controller *ctrl)
 err_unlock:
 	tty_unlock(tty);
 	tty_release_struct(tty, serport->tty_idx);
+	serport->tty = NULL;
 
 	return ret;
 }
@@ -154,6 +155,7 @@ static void ttyport_close(struct serdev_controller *ctrl)
 	tty_unlock(tty);
 
 	tty_release_struct(tty, serport->tty_idx);
+	serport->tty = NULL;
 }
 
 static unsigned int ttyport_set_baudrate(struct serdev_controller *ctrl, unsigned int speed)
diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h
index 9337fec9394edf..21ed9c41b7516f 100644
--- a/drivers/tty/serial/8250/8250.h
+++ b/drivers/tty/serial/8250/8250.h
@@ -73,6 +73,12 @@ struct serial8250_config {
 	unsigned int	flags;
 };
 
+enum uart_port_type {
+	/* Internal 8250 only */
+	UART_PORT_AIROHA		= 124,
+	UART_PORT_AIROHA_HS		= 125,
+};
+
 #define UART_CAP_FIFO	BIT(8)	/* UART has FIFO */
 #define UART_CAP_EFR	BIT(9)	/* UART has EFR */
 #define UART_CAP_SLEEP	BIT(10)	/* UART has IER sleep */
@@ -180,6 +186,7 @@ void serial8250_clear_and_reinit_fifos(struct uart_8250_port *p);
 void serial8250_fifo_wait_for_lsr_thre(struct uart_8250_port *up,
 				       struct nbcon_write_context *wctxt,
 				       unsigned int count);
+void serial8250_wait_for_xmitr(struct uart_8250_port *up, int bits);
 
 void serial8250_rpm_get(struct uart_8250_port *p);
 void serial8250_rpm_put(struct uart_8250_port *p);
@@ -316,6 +323,14 @@ static inline int serial8250_pnp_init(void) { return 0; }
 static inline void serial8250_pnp_exit(void) { }
 #endif
 
+#ifdef CONFIG_SERIAL_8250_HUB6
+int serial8250_hub6_init(void);
+void serial8250_hub6_exit(void);
+#else
+static inline int serial8250_hub6_init(void) { return 0; }
+static inline void serial8250_hub6_exit(void) { }
+#endif
+
 #ifdef CONFIG_SERIAL_8250_RSA
 void univ8250_rsa_support(struct uart_ops *ops, const struct uart_ops *core_ops);
 void rsa_enable(struct uart_8250_port *up);
@@ -336,7 +351,7 @@ int fintek_8250_probe(struct uart_8250_port *uart);
 static inline int fintek_8250_probe(struct uart_8250_port *uart) { return 0; }
 #endif
 
-#if IS_REACHABLE(CONFIG_SERIAL_8250_HUB6)
+#if IS_ENABLED(CONFIG_SERIAL_8250_HUB6)
 bool hub6_match_port(const struct uart_port *port1, const struct uart_port *port2);
 #else
 static inline bool hub6_match_port(const struct uart_port *port1, const struct uart_port *port2)
diff --git a/drivers/tty/serial/8250/8250_airoha.c b/drivers/tty/serial/8250/8250_airoha.c
new file mode 100644
index 00000000000000..fe72a39561c289
--- /dev/null
+++ b/drivers/tty/serial/8250/8250_airoha.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Airoha UART driver
+ *
+ * Copyright (c) 2025 Genexis Sweden AB
+ * Author: Benjamin Larsson <benjamin.larsson@genexis.eu>
+ *	   Christian Marangi <ansuelsmth@gmail.com>
+ */
+
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/errno.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/platform_device.h>
+#include <linux/serial_8250.h>
+#include <linux/serial_core.h>
+#include <linux/serial_reg.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include "8250.h"
+
+#define UART_AIROHA_XINCLKDR	10
+#define UART_AIROHA_XYD		11
+#define   UART_AIROHA_XYD_X	GENMASK(31, 16)
+#define   UART_AIROHA_XYD_Y	GENMASK(15, 0)
+
+struct airoha_8250_priv {
+	int line;
+};
+
+#define UART_BRD_20M		0x0001
+
+#define XINDIV_CLOCK		(20 * HZ_PER_MHZ)
+#define XYD_Y			65000
+
+static const unsigned int airoha_clk_divs[] = { 2, 4, 10 };
+
+static unsigned int airoha_get_divisor(struct uart_port *port,
+				       unsigned int baud,
+				       unsigned int *frac)
+{
+	/* Hardware always uses BRDIV = 1. */
+	*frac = 0;
+
+	return UART_BRD_20M;
+}
+
+/*
+ * Airoha UART baud rate calculation logic
+ *
+ * crystal_clock = 20 MHz (fixed frequency)
+ * xindiv_clock = crystal_clock / clock_div
+ * (x/y) = XYD, 32 bit register with 16 bits of x and then 16 bits of y
+ * clock_div = XINCLK_DIVCNT (default set to 10 (0x4)),
+ *           - 3 bit register [ 1, 2, 4, 8, 10, 12, 16, 20 ]
+ *
+ * baud_rate = ((xindiv_clock) * (x/y)) / ([BRDH,BRDL] * 16)
+ *
+ * Selecting divider needs to fulfill
+ * 1.8432 MHz <= xindiv_clk <= APB clock / 2
+ * The clocks are unknown but a divider of value 1 did not result in a valid
+ * waveform.
+ *
+ * XYD_y seems to need to be larger then XYD_x for proper waveform generation.
+ * Setting [BRDH,BRDL] to [0,1] and XYD_y to 65000 gives even values
+ * for usual baud rates.
+ */
+static void airoha_set_divisor(struct uart_port *port, unsigned int baud,
+			       unsigned int quot, unsigned int quot_frac)
+{
+	struct uart_8250_port *up = up_to_u8250p(port);
+	u32 xindiv_clk;
+	u64 xyd_x, nom;
+	int div_bit;
+
+	/* Set baud rate calculation defaults (BRDIV [BRDH,BRDL] to 1) */
+	serial8250_do_set_divisor(port, baud, UART_BRD_20M);
+
+	/*
+	 * Calculate XYD_x and XINCLKDR register by searching
+	 * through a table of crystal_clock divisors.
+	 */
+	nom = (u64)baud * XYD_Y;
+	for (div_bit = ARRAY_SIZE(airoha_clk_divs) - 1; div_bit >= 0; div_bit--) {
+		unsigned int div = airoha_clk_divs[div_bit];
+
+		xindiv_clk = XINDIV_CLOCK / div;
+		xyd_x = div_u64(nom * 16, xindiv_clk);
+
+		/* For the HSUART xyd_x needs to be scaled by a factor of 2 */
+		if (port->type == UART_PORT_AIROHA_HS)
+			xyd_x /= 2;
+
+		if (xyd_x < XYD_Y)
+			break;
+	}
+
+	/* Couldn't find a valid xyd_x */
+	if (div_bit < 0) {
+		dev_err(port->dev, "failed to find suitable clock divisor for baud %u\n",
+			baud);
+		return;
+	}
+
+	serial_port_out(port, UART_AIROHA_XINCLKDR, BIT(div_bit));
+	serial_port_out(port, UART_AIROHA_XYD,
+			FIELD_PREP(UART_AIROHA_XYD_X, xyd_x) |
+			FIELD_PREP(UART_AIROHA_XYD_Y, XYD_Y));
+
+	/* Restore normal register access. */
+	serial_port_out(port, UART_LCR, up->lcr);
+}
+
+static int airoha_8250_probe(struct platform_device *pdev)
+{
+	struct uart_8250_port uart = { };
+	struct device *dev = &pdev->dev;
+	struct airoha_8250_priv *priv;
+	struct resource *res;
+	int ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return dev_err_probe(dev, -EINVAL, "invalid address\n");
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	uart.port.dev = dev;
+	if (device_is_compatible(dev, "airoha,an7581-hsuart"))
+		uart.port.type = UART_PORT_AIROHA_HS;
+	else
+		uart.port.type = UART_PORT_AIROHA;
+	uart.port.flags = UPF_BOOT_AUTOCONF | UPF_FIXED_PORT |
+			  UPF_FIXED_TYPE | UPF_IOREMAP;
+	uart.port.set_divisor = airoha_set_divisor;
+	uart.port.get_divisor = airoha_get_divisor;
+	uart.port.mapbase = res->start;
+	uart.port.mapsize = resource_size(res);
+
+	ret = uart_read_and_validate_port_properties(&uart.port);
+	if (ret)
+		return ret;
+
+	ret = serial8250_register_8250_port(&uart);
+	if (ret < 0)
+		return ret;
+
+	priv->line = ret;
+	platform_set_drvdata(pdev, priv);
+
+	return 0;
+}
+
+static void airoha_8250_remove(struct platform_device *ofdev)
+{
+	struct airoha_8250_priv *priv = platform_get_drvdata(ofdev);
+
+	serial8250_unregister_port(priv->line);
+}
+
+static const struct of_device_id airoha_8250_dt_ids[] = {
+	{ .compatible = "airoha,en7523-uart" },
+	{ .compatible = "airoha,an7581-hsuart" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, airoha_8250_dt_ids);
+
+static struct platform_driver airoha_8250_driver = {
+	.driver = {
+		.name = "8250_airoha",
+		.of_match_table = airoha_8250_dt_ids,
+	},
+	.probe = airoha_8250_probe,
+	.remove = airoha_8250_remove,
+};
+module_platform_driver(airoha_8250_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Airoha UART driver");
+MODULE_AUTHOR("Benjamin Larsson <benjamin.larsson@genexis.eu>");
+MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>");
diff --git a/drivers/tty/serial/8250/8250_bcm7271.c b/drivers/tty/serial/8250/8250_bcm7271.c
index cb7d594d263e95..5f1140f0515044 100644
--- a/drivers/tty/serial/8250/8250_bcm7271.c
+++ b/drivers/tty/serial/8250/8250_bcm7271.c
@@ -1121,8 +1121,8 @@ static void brcmuart_remove(struct platform_device *pdev)
 	struct brcmuart_priv *priv = platform_get_drvdata(pdev);
 
 	debugfs_remove_recursive(priv->debugfs_dir);
-	hrtimer_cancel(&priv->hrt);
 	serial8250_unregister_port(priv->line);
+	hrtimer_cancel(&priv->hrt);
 	brcmuart_free_bufs(&pdev->dev, priv);
 	if (priv->dma_enabled)
 		brcmuart_arbitration(priv, 0);
diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c
index b875d394796ffa..11424fef7c08b9 100644
--- a/drivers/tty/serial/8250/8250_core.c
+++ b/drivers/tty/serial/8250/8250_core.c
@@ -673,6 +673,29 @@ EXPORT_SYMBOL(serial8250_resume_port);
  */
 static DEFINE_MUTEX(serial_mutex);
 
+/**
+ * serial8250_match_port - are the two ports equivalent?
+ * @port1: first port
+ * @port2: second port
+ *
+ * This utility function can be used to determine whether two uart_port
+ * structures describe the same port.
+ */
+static bool serial8250_match_port(const struct uart_port *port1,
+				  const struct uart_port *port2)
+{
+	if (port1->iotype != port2->iotype)
+		return false;
+	else if (port1->iotype == UPIO_PORT)
+		return port1->iobase == port2->iobase;
+	else if (port1->iotype == UPIO_HUB6)
+		return hub6_match_port(port1, port2);
+	else if (uart_iotype_mmio(port1->iotype))
+		return port1->mapbase == port2->mapbase;
+	else
+		return false;
+}
+
 static struct uart_8250_port *serial8250_find_match_or_unused(const struct uart_port *port)
 {
 	int i;
@@ -681,7 +704,7 @@ static struct uart_8250_port *serial8250_find_match_or_unused(const struct uart_
 	 * First, find a port entry which matches.
 	 */
 	for (i = 0; i < nr_uarts; i++)
-		if (uart_match_port(&serial8250_ports[i].port, port))
+		if (serial8250_match_port(&serial8250_ports[i].port, port))
 			return &serial8250_ports[i];
 
 	/* try line number first if still available */
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index 51d026f20825a0..0ab55823037074 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -51,6 +51,7 @@
 #define DW_UART_QUIRK_APMC0D08		BIT(4)
 #define DW_UART_QUIRK_CPR_VALUE		BIT(5)
 #define DW_UART_QUIRK_IER_KICK		BIT(6)
+#define DW_UART_QUIRK_SKIP_EMPTY_FIFO_READ	BIT(7)
 
 /*
  * Number of consecutive IIR_NO_INT interrupts required to trigger interrupt
@@ -436,11 +437,18 @@ static int dw8250_handle_irq(struct uart_port *p)
 	 * This problem has only been observed so far when not in DMA mode
 	 * so we limit the workaround only to non-DMA mode.
 	 */
-	if (!up->dma && rx_timeout) {
+	if (!(quirks & DW_UART_QUIRK_SKIP_EMPTY_FIFO_READ) && !up->dma && rx_timeout) {
 		status = serial_lsr_in(up);
 
+		/*
+		 * Do the bogus read from Shadow RBR (SRBR) if provided.
+		 * Both RBR and SRBR are supported ways to workaround
+		 * this problem. But read RBR can cause hardware error
+		 * (PSLVERR) if hardware choose to implement in this way
+		 * (implement REG_TIMEOUT_WIDTH to 0), whereas SRBR won't.
+		 */
 		if (!(status & (UART_LSR_DR | UART_LSR_BI)))
-			serial_port_in(p, UART_RX);
+			serial_port_in(p, d->data.srbr);
 	}
 
 	/* Manually stop the Rx DMA transfer when acting as flow controller */
@@ -888,6 +896,11 @@ static const struct dw8250_platform_data dw8250_ultrarisc_dp1000_data = {
 	.quirks = DW_UART_QUIRK_CPR_VALUE,
 };
 
+static const struct dw8250_platform_data dw8250_tda54 = {
+	.usr_reg = DW_UART_USR,
+	.quirks = DW_UART_QUIRK_SKIP_EMPTY_FIFO_READ,
+};
+
 static const struct of_device_id dw8250_of_match[] = {
 	{ .compatible = "snps,dw-apb-uart", .data = &dw8250_dw_apb },
 	{ .compatible = "cavium,octeon-3860-uart", .data = &dw8250_octeon_3860_data },
@@ -895,6 +908,7 @@ static const struct of_device_id dw8250_of_match[] = {
 	{ .compatible = "renesas,rzn1-uart", .data = &dw8250_renesas_rzn1_data },
 	{ .compatible = "sophgo,sg2044-uart", .data = &dw8250_skip_set_rate_data },
 	{ .compatible = "starfive,jh7100-uart", .data = &dw8250_skip_set_rate_data },
+	{ .compatible = "ti,tda54-uart", .data = &dw8250_tda54 },
 	{ .compatible = "ultrarisc,dp1000-uart", .data = &dw8250_ultrarisc_dp1000_data },
 	{ /* Sentinel */ }
 };
@@ -910,6 +924,7 @@ static const struct acpi_device_id dw8250_acpi_match[] = {
 	{ "8086228A", (kernel_ulong_t)&dw8250_dw_apb },
 	{ "AMD0020", (kernel_ulong_t)&dw8250_dw_apb },
 	{ "AMDI0020", (kernel_ulong_t)&dw8250_dw_apb },
+	{ "AMDI0021", (kernel_ulong_t)&dw8250_dw_apb },
 	{ "AMDI0022", (kernel_ulong_t)&dw8250_dw_apb },
 	{ "APMC0D08", (kernel_ulong_t)&dw8250_apmc0d08 },
 	{ "BRCM2032", (kernel_ulong_t)&dw8250_dw_apb },
diff --git a/drivers/tty/serial/8250/8250_dwlib.c b/drivers/tty/serial/8250/8250_dwlib.c
index 8859e66d2d71cd..9bb02a4ab11f64 100644
--- a/drivers/tty/serial/8250/8250_dwlib.c
+++ b/drivers/tty/serial/8250/8250_dwlib.c
@@ -247,5 +247,10 @@ void dw8250_setup_port(struct uart_port *p)
 
 	if (reg & DW_UART_CPR_SIR_MODE)
 		up->capabilities |= UART_CAP_IRDA;
+
+	if (reg & DW_UART_CPR_SHADOW)
+		pd->srbr = DW_UART_SRBR_0;
+	else
+		pd->srbr = UART_RX;
 }
 EXPORT_SYMBOL_GPL(dw8250_setup_port);
diff --git a/drivers/tty/serial/8250/8250_dwlib.h b/drivers/tty/serial/8250/8250_dwlib.h
index 1fe52332e774b2..ee7a07fac0f62e 100644
--- a/drivers/tty/serial/8250/8250_dwlib.h
+++ b/drivers/tty/serial/8250/8250_dwlib.h
@@ -13,6 +13,7 @@
 #include "8250.h"
 
 /* Offsets for the DesignWare specific registers */
+#define DW_UART_SRBR_0	0x0c /* Shadow Receive Buffer Register */
 #define DW_UART_USR	0x1f /* UART Status Register */
 #define DW_UART_DMASA	0xa8 /* DMA Software Ack */
 #define DW_UART_TCR	0xac /* Transceiver Control Register (RS485) */
@@ -90,6 +91,10 @@ struct dw8250_port_data {
 
 	/* RS485 variables */
 	bool			hw_rs485_support;
+
+	/* Register offsets */
+	/* Shadow RBR (if not defined RBR will be used) */
+	int		srbr;
 };
 
 void dw8250_do_set_termios(struct uart_port *p, struct ktermios *termios, const struct ktermios *old);
diff --git a/drivers/tty/serial/8250/8250_hub6.c b/drivers/tty/serial/8250/8250_hub6.c
index b6767633c966cc..4d4c54c951cb9b 100644
--- a/drivers/tty/serial/8250/8250_hub6.c
+++ b/drivers/tty/serial/8250/8250_hub6.c
@@ -49,13 +49,12 @@ bool hub6_match_port(const struct uart_port *port1, const struct uart_port *port
 }
 EXPORT_SYMBOL_GPL(hub6_match_port);
 
-static int __init hub6_init(void)
+int __init serial8250_hub6_init(void)
 {
 	return platform_device_register(&hub6_device);
 }
 
-module_init(hub6_init);
-
-MODULE_AUTHOR("Russell King");
-MODULE_DESCRIPTION("8250 serial probe module for Hub6 cards");
-MODULE_LICENSE("GPL");
+void __init_or_module serial8250_hub6_exit(void)
+{
+	platform_device_unregister(&hub6_device);
+}
diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c
index 82656645b8a64b..c02ecb96e8b6fb 100644
--- a/drivers/tty/serial/8250/8250_mid.c
+++ b/drivers/tty/serial/8250/8250_mid.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/rational.h>
@@ -209,6 +210,7 @@ static void mid8250_set_termios(struct uart_port *p, struct ktermios *termios,
 				const struct ktermios *old)
 {
 	unsigned int baud = tty_termios_baud_rate(termios);
+	struct uart_8250_port *up = up_to_u8250p(p);
 	struct mid8250 *mid = p->private_data;
 	unsigned short ps = 16;
 	unsigned long fuart = baud * ps;
@@ -231,11 +233,16 @@ static void mid8250_set_termios(struct uart_port *p, struct ktermios *termios,
 	}
 
 	rational_best_approximation(fuart, mid->board->freq, w, w, &mul, &div);
-	p->uartclk = fuart * 16 / ps;		/* core uses ps = 16 always */
 
-	writel(ps, p->membase + INTEL_MID_UART_PS);		/* set PS */
-	writel(mul, p->membase + INTEL_MID_UART_MUL);		/* set MUL */
-	writel(div, p->membase + INTEL_MID_UART_DIV);
+	scoped_guard(spinlock_irq, &p->lock) {
+		p->uartclk = fuart * 16 / ps;	/* core uses ps = 16 always */
+
+		serial8250_wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
+
+		writel(ps, p->membase + INTEL_MID_UART_PS);	/* set PS */
+		writel(mul, p->membase + INTEL_MID_UART_MUL);	/* set MUL */
+		writel(div, p->membase + INTEL_MID_UART_DIV);
+	}
 
 	serial8250_do_set_termios(p, termios, old);
 }
diff --git a/drivers/tty/serial/8250/8250_mxpcie.c b/drivers/tty/serial/8250/8250_mxpcie.c
index ce873fbd62e930..19f1f51eec4779 100644
--- a/drivers/tty/serial/8250/8250_mxpcie.c
+++ b/drivers/tty/serial/8250/8250_mxpcie.c
@@ -109,6 +109,7 @@ struct mxpcie8250_port {
 struct mxpcie8250 {
 	unsigned int supp_rs;
 	unsigned int num_ports;
+	unsigned int nr;	/* ports actually registered */
 	void __iomem *bar1_base; /* UART registers (MMIO) */
 	void __iomem *bar2_base; /* UIR / GPIO / CPLD (IO) */
 	struct mxpcie8250_port port[] __counted_by(num_ports);
@@ -217,8 +218,6 @@ static void mxpcie8250_set_termios(struct uart_port *port,
 				   const struct ktermios *old)
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
-	struct tty_struct *tty = port->state->port.tty;
-	unsigned int cflag = tty->termios.c_cflag;
 	u8 efr, val;
 
 	serial8250_do_set_termios(port, new, old);
@@ -228,23 +227,25 @@ static void mxpcie8250_set_termios(struct uart_port *port,
 	efr = serial_in(up, MOXA_PUART_EFR);
 	efr &= ~(MOXA_PUART_EFR_AUTO_RTS | MOXA_PUART_EFR_AUTO_CTS);
 
-	if (cflag & CRTSCTS) {
+	if (new->c_cflag & CRTSCTS) {
 		efr |= (MOXA_PUART_EFR_AUTO_RTS | MOXA_PUART_EFR_AUTO_CTS);
 		up->port.status |= (UPSTAT_AUTORTS | UPSTAT_AUTOCTS);
 	}
 	/* Set on-chip software flow control character */
-	serial_out(up, MOXA_PUART_XON1, START_CHAR(tty));
-	serial_out(up, MOXA_PUART_XON2, START_CHAR(tty));
-	serial_out(up, MOXA_PUART_XOFF1, STOP_CHAR(tty));
-	serial_out(up, MOXA_PUART_XOFF2, STOP_CHAR(tty));
+	serial_out(up, MOXA_PUART_XON1, new->c_cc[VSTART]);
+	serial_out(up, MOXA_PUART_XON2, new->c_cc[VSTART]);
+	serial_out(up, MOXA_PUART_XOFF1, new->c_cc[VSTOP]);
+	serial_out(up, MOXA_PUART_XOFF2, new->c_cc[VSTOP]);
 
-	val = I_IXON(tty) ? MOXA_PUART_EFR_RX_FLOW_XON1_XOFF1 : MOXA_PUART_EFR_RX_FLOW_DISABLED;
+	val = (new->c_iflag & IXON) ? MOXA_PUART_EFR_RX_FLOW_XON1_XOFF1 :
+				      MOXA_PUART_EFR_RX_FLOW_DISABLED;
 	FIELD_MODIFY(MOXA_PUART_EFR_RX_FLOW_MASK, &efr, val);
 
-	val = I_IXOFF(tty) ? MOXA_PUART_EFR_TX_FLOW_XON1_XOFF1 : MOXA_PUART_EFR_TX_FLOW_DISABLED;
+	val = (new->c_iflag & IXOFF) ? MOXA_PUART_EFR_TX_FLOW_XON1_XOFF1 :
+				       MOXA_PUART_EFR_TX_FLOW_DISABLED;
 	FIELD_MODIFY(MOXA_PUART_EFR_TX_FLOW_MASK, &efr, val);
 
-	if (I_IXOFF(tty))
+	if (new->c_iflag & IXOFF)
 		up->port.status |= UPSTAT_AUTOXOFF;
 
 	serial_out(up, MOXA_PUART_EFR, efr);
@@ -517,6 +518,7 @@ static int mxpcie8250_probe(struct pci_dev *pdev, const struct pci_device_id *id
 	struct mxpcie8250 *priv;
 	unsigned short device = pdev->device;
 	unsigned int num_ports;
+	unsigned int i;
 	int ret;
 
 	ret = pcim_enable_device(pdev);
@@ -542,6 +544,8 @@ static int mxpcie8250_probe(struct pci_dev *pdev, const struct pci_device_id *id
 
 	mxpcie8250_init_board(pdev, priv);
 
+	pci_set_drvdata(pdev, priv);
+
 	up.port.dev = dev;
 	up.port.irq = pdev->irq;
 	up.port.uartclk = MOXA_PUART_BASE_BAUD * 16;
@@ -562,7 +566,7 @@ static int mxpcie8250_probe(struct pci_dev *pdev, const struct pci_device_id *id
 	up.port.handle_irq = mxpcie8250_handle_irq;
 	up.port.break_ctl = mxpcie8250_break_ctl;
 
-	for (unsigned int i = 0; i < num_ports; i++) {
+	for (i = 0; i < num_ports; i++) {
 		mxpcie8250_setup_port(pdev, priv, &up, i);
 
 		dev_dbg(dev, "Setup PCI port: port %lx, irq %d, type %d\n",
@@ -578,7 +582,7 @@ static int mxpcie8250_probe(struct pci_dev *pdev, const struct pci_device_id *id
 		}
 		priv->port[i].rx_trig_level = MOXA_PUART_RX_TRIG_DEFAULT;
 	}
-	pci_set_drvdata(pdev, priv);
+	priv->nr = i;
 
 	return 0;
 }
@@ -587,7 +591,7 @@ static void mxpcie8250_remove(struct pci_dev *pdev)
 {
 	struct mxpcie8250 *priv = pci_get_drvdata(pdev);
 
-	for (unsigned int i = 0; i < priv->num_ports; i++)
+	for (unsigned int i = 0; i < priv->nr; i++)
 		serial8250_unregister_port(priv->port[i].line);
 }
 
diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c
index 3c7775df27effd..ceecb39fb82da2 100644
--- a/drivers/tty/serial/8250/8250_omap.c
+++ b/drivers/tty/serial/8250/8250_omap.c
@@ -711,12 +711,6 @@ static int omap_8250_startup(struct uart_port *port)
 	struct uart_8250_dma *dma = &priv->omap8250_dma;
 	int ret;
 
-	if (priv->wakeirq) {
-		ret = dev_pm_set_dedicated_wake_irq(port->dev, priv->wakeirq);
-		if (ret)
-			return ret;
-	}
-
 #ifdef CONFIG_PM
 	up->capabilities |= UART_CAP_RPM;
 #endif
@@ -787,7 +781,6 @@ static void omap_8250_shutdown(struct uart_port *port)
 	}
 
 	disable_irq_nosync(port->irq);
-	dev_pm_clear_wake_irq(port->dev);
 
 	serial8250_release_dma(up);
 	up->dma = NULL;
@@ -1588,6 +1581,12 @@ static int omap8250_probe(struct platform_device *pdev)
 
 	priv->wakeirq = irq_of_parse_and_map(np, 1);
 
+	if (priv->wakeirq) {
+		ret = dev_pm_set_dedicated_wake_irq(&pdev->dev, priv->wakeirq);
+		if (ret)
+			goto err;
+	}
+
 	ret = serial8250_register_8250_port(&up);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "unable to register 8250 port\n");
@@ -1603,6 +1602,7 @@ static int omap8250_probe(struct platform_device *pdev)
 
 	return 0;
 err:
+	dev_pm_clear_wake_irq(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_put_sync(&pdev->dev);
 	flush_work(&priv->qos_work);
@@ -1625,6 +1625,7 @@ static void omap8250_remove(struct platform_device *pdev)
 	omap_8250_shutdown(&up->port);
 	serial8250_unregister_port(priv->line);
 	priv->line = -ENODEV;
+	dev_pm_clear_wake_irq(&pdev->dev);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_put_sync(&pdev->dev);
 	flush_work(&priv->qos_work);
diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
index c24a5989698262..6e53d6d72a8e20 100644
--- a/drivers/tty/serial/8250/8250_pci.c
+++ b/drivers/tty/serial/8250/8250_pci.c
@@ -3894,7 +3894,7 @@ static const struct pci_device_id blacklist[] = {
 	{ PCI_VDEVICE(ACCESSIO, PCI_ANY_ID), .driver_data = REPORT_8250_CONFIG(PERICOM), },
 
 	/* Moxa devices */
-	{ PCI_VDEVICE(MOXA, PCI_ANY_ID), REPORT_8250_CONFIG(MOXA), },
+	{ PCI_VDEVICE(MOXA, PCI_ANY_ID), .driver_data = REPORT_8250_CONFIG(MOXA), },
 
 	/* End of the black list */
 	{ }
diff --git a/drivers/tty/serial/8250/8250_platform.c b/drivers/tty/serial/8250/8250_platform.c
index af946d12e76483..0570366a71d022 100644
--- a/drivers/tty/serial/8250/8250_platform.c
+++ b/drivers/tty/serial/8250/8250_platform.c
@@ -319,10 +319,14 @@ static int __init serial8250_init(void)
 	if (ret)
 		goto unreg_uart_drv;
 
+	ret = serial8250_hub6_init();
+	if (ret)
+		goto unreg_pnp;
+
 	serial8250_isa_devs = platform_device_alloc("serial8250", PLAT8250_DEV_LEGACY);
 	if (!serial8250_isa_devs) {
 		ret = -ENOMEM;
-		goto unreg_pnp;
+		goto unreg_hub6;
 	}
 
 	ret = platform_device_add(serial8250_isa_devs);
@@ -338,6 +342,8 @@ static int __init serial8250_init(void)
 	platform_device_del(serial8250_isa_devs);
 put_dev:
 	platform_device_put(serial8250_isa_devs);
+unreg_hub6:
+	serial8250_hub6_exit();
 unreg_pnp:
 	serial8250_pnp_exit();
 unreg_uart_drv:
@@ -365,6 +371,7 @@ static void __exit serial8250_exit(void)
 	platform_driver_unregister(&serial8250_isa_driver);
 	platform_device_unregister(isa_dev);
 
+	serial8250_hub6_exit();
 	serial8250_pnp_exit();
 
 #ifdef CONFIG_SPARC
diff --git a/drivers/tty/serial/8250/8250_pnp.c b/drivers/tty/serial/8250/8250_pnp.c
index 6bfdeff5fe222a..7decf62400abdc 100644
--- a/drivers/tty/serial/8250/8250_pnp.c
+++ b/drivers/tty/serial/8250/8250_pnp.c
@@ -526,12 +526,12 @@ static struct pnp_driver serial_pnp_driver = {
 	.id_table	= pnp_dev_table,
 };
 
-int serial8250_pnp_init(void)
+int __init serial8250_pnp_init(void)
 {
 	return pnp_register_driver(&serial_pnp_driver);
 }
 
-void serial8250_pnp_exit(void)
+void __init_or_module serial8250_pnp_exit(void)
 {
 	pnp_unregister_driver(&serial_pnp_driver);
 }
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index 38fa45e74a37a2..45dc9ab5bfd870 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -319,6 +319,22 @@ static const struct serial8250_config uart_config[] = {
 		.rxtrig_bytes	= {15, 31, 63, 111},
 		.flags		= UART_CAP_FIFO,
 	},
+	[UART_PORT_AIROHA] = {
+		.name		= "Airoha UART",
+		.fifo_size	= 8,
+		.tx_loadsz	= 1,
+		.fcr		= UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01 | UART_FCR_CLEAR_RCVR,
+		.rxtrig_bytes	= {1, 4, 4, 4},
+		.flags		= UART_CAP_FIFO,
+	},
+	[UART_PORT_AIROHA_HS] = {
+		.name		= "Airoha HSUART",
+		.fifo_size	= 128,
+		.tx_loadsz	= 128,
+		.fcr		= UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_01 | UART_FCR_CLEAR_RCVR,
+		.rxtrig_bytes	= {1, 4},
+		.flags		= UART_CAP_FIFO,
+	},
 };
 
 /* Uart divisor latch read */
@@ -526,7 +542,6 @@ void serial8250_rpm_put(struct uart_8250_port *p)
 {
 	if (!(p->capabilities & UART_CAP_RPM))
 		return;
-	pm_runtime_mark_last_busy(p->port.dev);
 	pm_runtime_put_autosuspend(p->port.dev);
 }
 EXPORT_SYMBOL_GPL(serial8250_rpm_put);
@@ -667,7 +682,6 @@ static void serial8250_rpm_put_tx(struct uart_8250_port *p)
 	rpm_active = xchg(&p->rpm_tx_active, 0);
 	if (!rpm_active)
 		return;
-	pm_runtime_mark_last_busy(p->port.dev);
 	pm_runtime_put_autosuspend(p->port.dev);
 }
 
@@ -2024,7 +2038,7 @@ static bool wait_for_lsr(struct uart_8250_port *up, int bits)
 }
 
 /* Wait for transmitter and holding register to empty with timeout */
-static void wait_for_xmitr(struct uart_8250_port *up, int bits)
+void serial8250_wait_for_xmitr(struct uart_8250_port *up, int bits)
 {
 	unsigned int tmout;
 	bool tx_ready;
@@ -2052,6 +2066,7 @@ static void wait_for_xmitr(struct uart_8250_port *up, int bits)
 		}
 	}
 }
+EXPORT_SYMBOL_NS_GPL(serial8250_wait_for_xmitr, "SERIAL_8250");
 
 #ifdef CONFIG_CONSOLE_POLL
 /*
@@ -2098,7 +2113,7 @@ static void serial8250_put_poll_char(struct uart_port *port,
 	ier = serial_port_in(port, UART_IER);
 	__serial8250_clear_IER(up);
 
-	wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
+	serial8250_wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
 	/*
 	 *	Send the character out.
 	 */
@@ -2108,7 +2123,7 @@ static void serial8250_put_poll_char(struct uart_port *port,
 	 *	Finally, wait for transmitter to become empty
 	 *	and restore the IER
 	 */
-	wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
+	serial8250_wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
 	serial_port_out(port, UART_IER, ier);
 }
 
@@ -2223,7 +2238,7 @@ static void serial8250_THRE_test(struct uart_port *port)
 	 * Synchronize UART_IER access against the console.
 	 */
 	scoped_guard(uart_port_lock_irqsave, port) {
-		wait_for_xmitr(up, UART_LSR_THRE);
+		serial8250_wait_for_xmitr(up, UART_LSR_THRE);
 		serial_port_out_sync(port, UART_IER, UART_IER_THRI);
 		udelay(1); /* allow THRE to set */
 		iir_noint1 = serial_port_in(port, UART_IIR) & UART_IIR_NO_INT;
@@ -3293,7 +3308,7 @@ static void serial8250_console_wait_putchar(struct uart_port *port, unsigned cha
 {
 	struct uart_8250_port *up = up_to_u8250p(port);
 
-	wait_for_xmitr(up, UART_LSR_THRE);
+	serial8250_wait_for_xmitr(up, UART_LSR_THRE);
 	serial8250_console_putchar(port, ch);
 }
 
@@ -3504,7 +3519,7 @@ void serial8250_console_write(struct uart_8250_port *up,
 	 *	Finally, wait for transmitter to become empty
 	 *	and restore the IER
 	 */
-	wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
+	serial8250_wait_for_xmitr(up, UART_LSR_BOTH_EMPTY);
 
 	if (em485) {
 		mdelay(port->rs485.delay_rts_after_send);
@@ -3597,7 +3612,7 @@ int serial8250_console_setup(struct uart_port *port, char *options, bool probe)
 
 	up->console_line_ended = true;
 	up->console_msr_work_allow = true;
-	init_irq_work(&up->console_msr_work, console_msr_handler);
+	up->console_msr_work = IRQ_WORK_INIT_LAZY(console_msr_handler);
 
 	if (options)
 		uart_parse_options(options, &baud, &parity, &bits, &flow);
diff --git a/drivers/tty/serial/8250/8250_uniphier.c b/drivers/tty/serial/8250/8250_uniphier.c
index e3db60bf50c937..7d558b750f8363 100644
--- a/drivers/tty/serial/8250/8250_uniphier.c
+++ b/drivers/tty/serial/8250/8250_uniphier.c
@@ -165,10 +165,8 @@ static int uniphier_uart_probe(struct platform_device *pdev)
 	int ret;
 
 	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!regs) {
-		dev_err(dev, "failed to get memory resource\n");
-		return -EINVAL;
-	}
+	if (!regs)
+		return dev_err_probe(dev, -EINVAL, "failed to get memory resource\n");
 
 	membase = devm_ioremap(dev, regs->start, resource_size(regs));
 	if (!membase)
@@ -180,15 +178,10 @@ static int uniphier_uart_probe(struct platform_device *pdev)
 
 	memset(&up, 0, sizeof(up));
 
-	priv->clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(priv->clk)) {
-		dev_err(dev, "failed to get clock\n");
-		return PTR_ERR(priv->clk);
-	}
-
-	ret = clk_prepare_enable(priv->clk);
-	if (ret)
-		return ret;
+	priv->clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(priv->clk))
+		return dev_err_probe(dev, PTR_ERR(priv->clk),
+				     "failed to get and enable clock\n");
 
 	up.port.uartclk = clk_get_rate(priv->clk);
 
@@ -220,11 +213,9 @@ static int uniphier_uart_probe(struct platform_device *pdev)
 	up.dl_write = uniphier_serial_dl_write;
 
 	ret = serial8250_register_8250_port(&up);
-	if (ret < 0) {
-		dev_err(dev, "failed to register 8250 port\n");
-		clk_disable_unprepare(priv->clk);
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to register 8250 port\n");
+
 	priv->line = ret;
 
 	platform_set_drvdata(pdev, priv);
@@ -237,7 +228,6 @@ static void uniphier_uart_remove(struct platform_device *pdev)
 	struct uniphier8250_priv *priv = platform_get_drvdata(pdev);
 
 	serial8250_unregister_port(priv->line);
-	clk_disable_unprepare(priv->clk);
 }
 
 static int __maybe_unused uniphier_uart_suspend(struct device *dev)
diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig
index fa6f33303cb085..82e761b8298ab7 100644
--- a/drivers/tty/serial/8250/Kconfig
+++ b/drivers/tty/serial/8250/Kconfig
@@ -304,8 +304,8 @@ config SERIAL_8250_EXAR_ST16C554
 	  will be called 8250_exar_st16c554.
 
 config SERIAL_8250_HUB6
-	tristate "Support Hub6 cards"
-	depends on SERIAL_8250 != n && ISA && SERIAL_8250_MANY_PORTS
+	bool "Support Hub6 cards"
+	depends on SERIAL_8250_MANY_PORTS && ISA
 	help
 	  Say Y here if you have a HUB6 serial board.
 
@@ -347,6 +347,17 @@ config SERIAL_8250_ACORN
 	  system, say Y to this option.  The driver can handle 1, 2, or 3 port
 	  cards.  If unsure, say N.
 
+config SERIAL_8250_AIROHA
+	tristate "Airoha UART support"
+	depends on ARCH_AIROHA || COMPILE_TEST
+	depends on SERIAL_8250
+	help
+	  Selecting this option enables an Airoha SoC specific baud rate
+	  calculation routine on an otherwise 16550 compatible UART hardware.
+
+	  If you have an Airoha based board and want to use the serial port,
+	  say Y to this option. If unsure, say N.
+
 config SERIAL_8250_BCM2835AUX
 	tristate "BCM2835 auxiliar mini UART support"
 	depends on ARCH_BCM2835 || COMPILE_TEST
diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile
index 46adadcfe072ca..5ff2ba414e13c3 100644
--- a/drivers/tty/serial/8250/Makefile
+++ b/drivers/tty/serial/8250/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_SERIAL_8250)		+= 8250.o
 8250-y					:= 8250_core.o
 8250-y					+= 8250_platform.o
 8250-$(CONFIG_SERIAL_8250_PNP)		+= 8250_pnp.o
+8250-$(CONFIG_SERIAL_8250_HUB6)		+= 8250_hub6.o
 
 obj-$(CONFIG_SERIAL_8250)		+= 8250_base.o
 8250_base-y				:= 8250_port.o
@@ -20,6 +21,7 @@ obj-$(CONFIG_SERIAL_8250_CONSOLE)	+= 8250_early.o
 
 obj-$(CONFIG_SERIAL_8250_ACCENT)	+= 8250_accent.o
 obj-$(CONFIG_SERIAL_8250_ACORN)		+= 8250_acorn.o
+obj-$(CONFIG_SERIAL_8250_AIROHA)	+= 8250_airoha.o
 obj-$(CONFIG_SERIAL_8250_ASPEED_VUART)	+= 8250_aspeed_vuart.o
 obj-$(CONFIG_SERIAL_8250_BCM2835AUX)	+= 8250_bcm2835aux.o
 obj-$(CONFIG_SERIAL_8250_BCM7271)	+= 8250_bcm7271.o
@@ -35,7 +37,6 @@ obj-$(CONFIG_SERIAL_8250_EXAR_ST16C554)	+= 8250_exar_st16c554.o
 obj-$(CONFIG_SERIAL_8250_FOURPORT)	+= 8250_fourport.o
 obj-$(CONFIG_SERIAL_8250_FSL)		+= 8250_fsl.o
 obj-$(CONFIG_SERIAL_8250_HP300)		+= 8250_hp300.o
-obj-$(CONFIG_SERIAL_8250_HUB6)		+= 8250_hub6.o
 obj-$(CONFIG_SERIAL_8250_INGENIC)	+= 8250_ingenic.o
 obj-$(CONFIG_SERIAL_8250_IOC3)		+= 8250_ioc3.o
 obj-$(CONFIG_SERIAL_8250_KEBA)		+= 8250_keba.o
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index cf7dba473b2036..bf680d22199b14 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -904,6 +904,7 @@ config SERIAL_JSM
 config SERIAL_MSM
 	tristate "MSM on-chip serial port support"
 	depends on ARCH_QCOM || COMPILE_TEST
+	select PM_OPP
 	select SERIAL_CORE
 
 config SERIAL_MSM_CONSOLE
diff --git a/drivers/tty/serial/arc_uart.c b/drivers/tty/serial/arc_uart.c
index 5c4895d154c00e..3988e5958896c3 100644
--- a/drivers/tty/serial/arc_uart.c
+++ b/drivers/tty/serial/arc_uart.c
@@ -304,7 +304,7 @@ static unsigned int arc_serial_get_mctrl(struct uart_port *port)
 {
 	/*
 	 * Pretend we have a Modem status reg and following bits are
-	 *  always set, to satify the serial core state machine
+	 *  always set, to satisfy the serial core state machine
 	 *  (DSR) Data Set Ready
 	 *  (CTS) Clear To Send
 	 *  (CAR) Carrier Detect
diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c
index 5d8c1cfc1c6063..c3880259e3bce7 100644
--- a/drivers/tty/serial/atmel_serial.c
+++ b/drivers/tty/serial/atmel_serial.c
@@ -2723,7 +2723,7 @@ static bool atmel_serial_clk_will_stop(void)
 #endif
 }
 
-static int __maybe_unused atmel_serial_suspend(struct device *dev)
+static int atmel_serial_suspend(struct device *dev)
 {
 	struct uart_port *port = dev_get_drvdata(dev);
 	struct atmel_uart_port *atmel_port = to_atmel_uart_port(port);
@@ -2765,7 +2765,7 @@ static int __maybe_unused atmel_serial_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused atmel_serial_resume(struct device *dev)
+static int atmel_serial_resume(struct device *dev)
 {
 	struct uart_port *port = dev_get_drvdata(dev);
 	struct atmel_uart_port *atmel_port = to_atmel_uart_port(port);
@@ -2996,7 +2996,7 @@ static void atmel_serial_remove(struct platform_device *pdev)
 	pdev->dev.of_node = NULL;
 }
 
-static SIMPLE_DEV_PM_OPS(atmel_serial_pm_ops, atmel_serial_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(atmel_serial_pm_ops, atmel_serial_suspend,
 			 atmel_serial_resume);
 
 static struct platform_driver atmel_serial_driver = {
@@ -3005,7 +3005,7 @@ static struct platform_driver atmel_serial_driver = {
 	.driver		= {
 		.name			= "atmel_usart_serial",
 		.of_match_table		= of_match_ptr(atmel_serial_dt_ids),
-		.pm			= pm_ptr(&atmel_serial_pm_ops),
+		.pm			= pm_sleep_ptr(&atmel_serial_pm_ops),
 	},
 };
 
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index b7919c05f0fb80..c8575c9652035e 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -491,7 +491,7 @@ static void lpuart_dma_tx(struct lpuart_port *sport)
 	sport->dma_tx_nents = kfifo_dma_out_prepare(&tport->xmit_fifo, sgl,
 			ARRAY_SIZE(sport->tx_sgl), sport->dma_tx_bytes);
 
-	ret = dma_map_sg(chan->device->dev, sgl, sport->dma_tx_nents,
+	ret = dma_map_sg(dmaengine_get_dma_device(chan), sgl, sport->dma_tx_nents,
 			 DMA_TO_DEVICE);
 	if (!ret) {
 		dev_err(dev, "DMA mapping error for TX.\n");
@@ -502,7 +502,7 @@ static void lpuart_dma_tx(struct lpuart_port *sport)
 					ret, DMA_MEM_TO_DEV,
 					DMA_PREP_INTERRUPT);
 	if (!sport->dma_tx_desc) {
-		dma_unmap_sg(chan->device->dev, sgl, sport->dma_tx_nents,
+		dma_unmap_sg(dmaengine_get_dma_device(chan), sgl, sport->dma_tx_nents,
 			      DMA_TO_DEVICE);
 		dev_err(dev, "Cannot prepare TX slave DMA!\n");
 		return;
@@ -535,7 +535,7 @@ static void lpuart_dma_tx_complete(void *arg)
 		return;
 	}
 
-	dma_unmap_sg(chan->device->dev, sgl, sport->dma_tx_nents,
+	dma_unmap_sg(dmaengine_get_dma_device(chan), sgl, sport->dma_tx_nents,
 		     DMA_TO_DEVICE);
 
 	uart_xmit_advance(&sport->port, sport->dma_tx_bytes);
@@ -606,7 +606,7 @@ static void lpuart_flush_buffer(struct uart_port *port)
 
 	if (sport->lpuart_dma_tx_use) {
 		if (sport->dma_tx_in_progress) {
-			dma_unmap_sg(chan->device->dev, &sport->tx_sgl[0],
+			dma_unmap_sg(dmaengine_get_dma_device(chan), &sport->tx_sgl[0],
 				sport->dma_tx_nents, DMA_TO_DEVICE);
 			sport->dma_tx_in_progress = false;
 		}
@@ -1176,7 +1176,7 @@ static void lpuart_copy_rx_to_tty(struct lpuart_port *sport)
 	}
 
 	/* CPU claims ownership of RX DMA buffer */
-	dma_sync_sg_for_cpu(chan->device->dev, &sport->rx_sgl, 1,
+	dma_sync_sg_for_cpu(dmaengine_get_dma_device(chan), &sport->rx_sgl, 1,
 			    DMA_FROM_DEVICE);
 
 	/*
@@ -1237,7 +1237,7 @@ static void lpuart_copy_rx_to_tty(struct lpuart_port *sport)
 	sport->last_residue = state.residue;
 
 exit:
-	dma_sync_sg_for_device(chan->device->dev, &sport->rx_sgl, 1,
+	dma_sync_sg_for_device(dmaengine_get_dma_device(chan), &sport->rx_sgl, 1,
 			       DMA_FROM_DEVICE);
 
 	uart_port_unlock_irqrestore(&sport->port, flags);
@@ -1374,7 +1374,7 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
 		return -ENOMEM;
 
 	sg_init_one(&sport->rx_sgl, ring->buf, sport->rx_dma_rng_buf_len);
-	nent = dma_map_sg(chan->device->dev, &sport->rx_sgl, 1,
+	nent = dma_map_sg(dmaengine_get_dma_device(chan), &sport->rx_sgl, 1,
 			  DMA_FROM_DEVICE);
 
 	if (!nent) {
@@ -1430,7 +1430,7 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
 	return 0;
 
 err_unmap_sg:
-	dma_unmap_sg(chan->device->dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
+	dma_unmap_sg(dmaengine_get_dma_device(chan), &sport->rx_sgl, 1, DMA_FROM_DEVICE);
 err_free_buf:
 	kfree(ring->buf);
 	ring->buf = NULL;
@@ -1447,7 +1447,7 @@ static void lpuart_dma_rx_free(struct uart_port *port)
 	if (!sport->dma_idle_int)
 		timer_delete_sync(&sport->lpuart_timer);
 
-	dma_unmap_sg(chan->device->dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
+	dma_unmap_sg(dmaengine_get_dma_device(chan), &sport->rx_sgl, 1, DMA_FROM_DEVICE);
 	kfree(sport->rx_ring.buf);
 	sport->rx_ring.tail = 0;
 	sport->rx_ring.head = 0;
diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c
index 09648d64389796..f3fba4ec5a2968 100644
--- a/drivers/tty/serial/kgdboc.c
+++ b/drivers/tty/serial/kgdboc.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Based on the same principle as kgdboe using the NETPOLL api, this
- * driver uses a console polling api to implement a gdb serial inteface
+ * driver uses a console polling api to implement a gdb serial interface
  * which is multiplexed on a console port.
  *
  * Maintainer: Jason Wessel <jason.wessel@windriver.com>
@@ -188,6 +188,11 @@ static void cleanup_kgdboc(void)
 
 	kgdboc_unregister_kbd();
 	kgdb_unregister_io_module(&kgdboc_io_ops);
+
+	if (kgdb_tty_driver) {
+		tty_driver_kref_put(kgdb_tty_driver);
+		kgdb_tty_driver = NULL;
+	}
 }
 
 static int configure_kgdboc(void)
@@ -254,6 +259,10 @@ static int configure_kgdboc(void)
 
 noconfig:
 	kgdboc_unregister_kbd();
+	if (kgdb_tty_driver) {
+		tty_driver_kref_put(kgdb_tty_driver);
+		kgdb_tty_driver = NULL;
+	}
 	configured = 0;
 
 	return err;
diff --git a/drivers/tty/serial/ma35d1_serial.c b/drivers/tty/serial/ma35d1_serial.c
index 920fe7ff5083b7..0809655f439596 100644
--- a/drivers/tty/serial/ma35d1_serial.c
+++ b/drivers/tty/serial/ma35d1_serial.c
@@ -728,7 +728,7 @@ static int ma35d1serial_probe(struct platform_device *pdev)
 
 	ret = clk_prepare_enable(up->clk);
 	if (ret)
-		goto err_iounmap;
+		goto err_put_clk;
 
 	if (up->port.line != 0)
 		up->port.uartclk = clk_get_rate(up->clk);
@@ -755,6 +755,9 @@ static int ma35d1serial_probe(struct platform_device *pdev)
 err_clk_disable:
 	clk_disable_unprepare(up->clk);
 
+err_put_clk:
+	clk_put(up->clk);
+
 err_iounmap:
 	iounmap(up->port.membase);
 	return ret;
@@ -770,6 +773,7 @@ static void ma35d1serial_remove(struct platform_device *dev)
 
 	uart_remove_one_port(&ma35d1serial_reg, port);
 	clk_disable_unprepare(up->clk);
+	clk_put(up->clk);
 }
 
 static int ma35d1serial_suspend(struct platform_device *dev, pm_message_t state)
diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c
index 3633723acef8d7..4a6f526340b8d7 100644
--- a/drivers/tty/serial/qcom_geni_serial.c
+++ b/drivers/tty/serial/qcom_geni_serial.c
@@ -20,7 +20,6 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/panic_notifier.h>
-#include <linux/pm_domain.h>
 #include <linux/pm_opp.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
@@ -171,7 +170,6 @@ static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport);
 static int qcom_geni_serial_port_setup(struct uart_port *uport);
 static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport);
 static void qcom_geni_serial_resume_tx(struct uart_port *uport);
-static void qcom_geni_serial_poll_rx_fifo_locked(struct uart_port *uport);
 
 static inline struct qcom_geni_serial_port *to_dev_port(struct uart_port *uport)
 {
@@ -289,6 +287,10 @@ static struct qcom_geni_serial_port *get_port_from_line(int line, bool console,
 	} else {
 		int max_alias_num = of_alias_get_highest_id("serial");
 
+		port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+		if (!port)
+			return ERR_PTR(-ENOMEM);
+
 		if (line < 0 || line >= nr_ports)
 			line = ida_alloc_range(&port_ida, max_alias_num + 1,
 					       nr_ports - 1, GFP_KERNEL);
@@ -299,10 +301,6 @@ static struct qcom_geni_serial_port *get_port_from_line(int line, bool console,
 		if (line < 0)
 			return ERR_PTR(-ENXIO);
 
-		port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
-		if (!port)
-			return ERR_PTR(-ENOMEM);
-
 		port->uport.iotype = UPIO_MEM;
 		port->uport.ops = &qcom_geni_uart_pops;
 		port->uport.flags = UPF_BOOT_AUTOCONF;
@@ -467,6 +465,102 @@ static int qcom_geni_serial_poll_init(struct uart_port *uport)
 #endif
 
 #ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
+static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
+{
+	u32 i;
+	unsigned char buf[sizeof(u32)];
+	struct tty_port *tport;
+	struct qcom_geni_serial_port *port = to_dev_port(uport);
+
+	tport = &uport->state->port;
+	for (i = 0; i < bytes; ) {
+		int c;
+		int chunk = min_t(int, bytes - i, BYTES_PER_FIFO_WORD);
+
+		ioread32_rep(uport->membase + SE_GENI_RX_FIFOn, buf, 1);
+		i += chunk;
+		if (drop)
+			continue;
+
+		for (c = 0; c < chunk; c++) {
+			int sysrq;
+
+			uport->icount.rx++;
+			if (port->brk && buf[c] == 0) {
+				port->brk = false;
+				if (uart_handle_break(uport))
+					continue;
+			}
+
+			sysrq = uart_prepare_sysrq_char(uport, buf[c]);
+
+			if (!sysrq)
+				tty_insert_flip_char(tport, buf[c], TTY_NORMAL);
+		}
+	}
+	if (!drop)
+		tty_flip_buffer_push(tport);
+}
+#else
+static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
+{
+
+}
+#endif
+
+static void qcom_geni_serial_handle_rx_fifo(struct uart_port *uport, bool drop)
+{
+	u32 status;
+	u32 word_cnt;
+	u32 last_word_byte_cnt;
+	u32 last_word_partial;
+	u32 total_bytes;
+
+	status = readl(uport->membase +	SE_GENI_RX_FIFO_STATUS);
+	word_cnt = status & RX_FIFO_WC_MSK;
+	last_word_partial = status & RX_LAST;
+	last_word_byte_cnt = (status & RX_LAST_BYTE_VALID_MSK) >>
+						RX_LAST_BYTE_VALID_SHFT;
+
+	if (!word_cnt)
+		return;
+	total_bytes = BYTES_PER_FIFO_WORD * (word_cnt - 1);
+	if (last_word_partial && last_word_byte_cnt)
+		total_bytes += last_word_byte_cnt;
+	else
+		total_bytes += BYTES_PER_FIFO_WORD;
+	handle_rx_console(uport, total_bytes, drop);
+}
+
+#ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
+/* Caller holds the UART port lock. */
+static void qcom_geni_serial_poll_rx_fifo_locked(struct uart_port *uport)
+{
+	struct qcom_geni_serial_port *port = to_dev_port(uport);
+	struct tty_port *tport = &uport->state->port;
+	u32 s_irq_status;
+	bool drop_rx = false;
+
+	s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
+	writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
+
+	if (s_irq_status & S_RX_FIFO_WR_ERR_EN) {
+		uport->icount.overrun++;
+		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
+	}
+
+	if (s_irq_status & (S_GP_IRQ_0_EN | S_GP_IRQ_1_EN)) {
+		if (s_irq_status & S_GP_IRQ_0_EN)
+			uport->icount.parity++;
+		drop_rx = true;
+	} else if (s_irq_status & (S_GP_IRQ_2_EN | S_GP_IRQ_3_EN)) {
+		uport->icount.brk++;
+		port->brk = true;
+	}
+
+	qcom_geni_serial_handle_rx_fifo(uport, drop_rx);
+}
+
 static void qcom_geni_serial_drain_fifo(struct uart_port *uport)
 {
 	struct qcom_geni_serial_port *port = to_dev_port(uport);
@@ -676,47 +770,6 @@ static void qcom_geni_serial_console_device_unlock(struct console *co,
 #endif
 }
 
-static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
-{
-	u32 i;
-	unsigned char buf[sizeof(u32)];
-	struct tty_port *tport;
-	struct qcom_geni_serial_port *port = to_dev_port(uport);
-
-	tport = &uport->state->port;
-	for (i = 0; i < bytes; ) {
-		int c;
-		int chunk = min_t(int, bytes - i, BYTES_PER_FIFO_WORD);
-
-		ioread32_rep(uport->membase + SE_GENI_RX_FIFOn, buf, 1);
-		i += chunk;
-		if (drop)
-			continue;
-
-		for (c = 0; c < chunk; c++) {
-			int sysrq;
-
-			uport->icount.rx++;
-			if (port->brk && buf[c] == 0) {
-				port->brk = false;
-				if (uart_handle_break(uport))
-					continue;
-			}
-
-			sysrq = uart_prepare_sysrq_char(uport, buf[c]);
-
-			if (!sysrq)
-				tty_insert_flip_char(tport, buf[c], TTY_NORMAL);
-		}
-	}
-	if (!drop)
-		tty_flip_buffer_push(tport);
-}
-#else
-static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
-{
-
-}
 #endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */
 
 static void handle_rx_uart(struct uart_port *uport, u32 bytes)
@@ -890,58 +943,6 @@ static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport)
 	port->tx_queued = 0;
 }
 
-static void qcom_geni_serial_handle_rx_fifo(struct uart_port *uport, bool drop)
-{
-	u32 status;
-	u32 word_cnt;
-	u32 last_word_byte_cnt;
-	u32 last_word_partial;
-	u32 total_bytes;
-
-	status = readl(uport->membase +	SE_GENI_RX_FIFO_STATUS);
-	word_cnt = status & RX_FIFO_WC_MSK;
-	last_word_partial = status & RX_LAST;
-	last_word_byte_cnt = (status & RX_LAST_BYTE_VALID_MSK) >>
-						RX_LAST_BYTE_VALID_SHFT;
-
-	if (!word_cnt)
-		return;
-	total_bytes = BYTES_PER_FIFO_WORD * (word_cnt - 1);
-	if (last_word_partial && last_word_byte_cnt)
-		total_bytes += last_word_byte_cnt;
-	else
-		total_bytes += BYTES_PER_FIFO_WORD;
-	handle_rx_console(uport, total_bytes, drop);
-}
-
-/* Caller holds the UART port lock. */
-static void qcom_geni_serial_poll_rx_fifo_locked(struct uart_port *uport)
-{
-	struct qcom_geni_serial_port *port = to_dev_port(uport);
-	struct tty_port *tport = &uport->state->port;
-	u32 s_irq_status;
-	bool drop_rx = false;
-
-	s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
-	writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
-
-	if (s_irq_status & S_RX_FIFO_WR_ERR_EN) {
-		uport->icount.overrun++;
-		tty_insert_flip_char(tport, 0, TTY_OVERRUN);
-	}
-
-	if (s_irq_status & (S_GP_IRQ_0_EN | S_GP_IRQ_1_EN)) {
-		if (s_irq_status & S_GP_IRQ_0_EN)
-			uport->icount.parity++;
-		drop_rx = true;
-	} else if (s_irq_status & (S_GP_IRQ_2_EN | S_GP_IRQ_3_EN)) {
-		uport->icount.brk++;
-		port->brk = true;
-	}
-
-	qcom_geni_serial_handle_rx_fifo(uport, drop_rx);
-}
-
 static void qcom_geni_serial_stop_rx_fifo(struct uart_port *uport)
 {
 	u32 irq_en;
@@ -1291,7 +1292,6 @@ static irqreturn_t qcom_geni_serial_isr(int isr, void *dev)
 static int setup_fifos(struct qcom_geni_serial_port *port)
 {
 	struct uart_port *uport;
-	u32 old_rx_fifo_depth = port->rx_fifo_depth;
 
 	uport = &port->uport;
 	port->tx_fifo_depth = geni_se_get_tx_fifo_depth(&port->se);
@@ -1300,19 +1300,6 @@ static int setup_fifos(struct qcom_geni_serial_port *port)
 	uport->fifosize =
 		(port->tx_fifo_depth * port->tx_fifo_width) / BITS_PER_BYTE;
 
-	if (port->rx_buf && (old_rx_fifo_depth != port->rx_fifo_depth) && port->rx_fifo_depth) {
-		/*
-		 * Use krealloc rather than krealloc_array because rx_buf is
-		 * accessed as 1 byte entries as well as 4 byte entries so it's
-		 * not necessarily an array.
-		 */
-		port->rx_buf = devm_krealloc(uport->dev, port->rx_buf,
-					     port->rx_fifo_depth * sizeof(u32),
-					     GFP_KERNEL);
-		if (!port->rx_buf)
-			return -ENOMEM;
-	}
-
 	return 0;
 }
 
@@ -1482,9 +1469,8 @@ static int geni_serial_set_rate(struct geni_se *se, unsigned long baud)
 	 * Bump up BW vote on CPU and CORE path as driver supports FIFO mode
 	 * only.
 	 */
-	avg_bw_core = (baud > 115200) ? Bps_to_icc(CORE_2X_50_MHZ)
-						: GENI_DEFAULT_BW;
-	port->se.icc_paths[GENI_TO_CORE].avg_bw = avg_bw_core;
+	avg_bw_core = (baud > 115200) ? CORE_2X_50_MHZ : CORE_2X_19_2_MHZ;
+	port->se.icc_paths[GENI_TO_CORE].avg_bw = Bps_to_icc(avg_bw_core);
 	port->se.icc_paths[CPU_TO_GENI].avg_bw = Bps_to_icc(baud);
 	geni_icc_set_bw(&port->se);
 
@@ -1625,16 +1611,37 @@ static int qcom_geni_console_setup(struct console *co, char *options)
 	if (unlikely(!uport->membase))
 		return -ENXIO;
 
+	ret = pm_runtime_resume_and_get(uport->dev);
+	if (ret < 0)
+		return ret;
+
 	if (!port->setup) {
 		ret = qcom_geni_serial_port_setup(uport);
-		if (ret)
+		if (ret) {
+			pm_runtime_put_sync(uport->dev);
 			return ret;
+		}
 	}
 
 	if (options)
 		uart_parse_options(options, &baud, &parity, &bits, &flow);
 
-	return uart_set_options(uport, co, baud, parity, bits, flow);
+	ret = uart_set_options(uport, co, baud, parity, bits, flow);
+	if (ret)
+		pm_runtime_put_sync(uport->dev);
+
+	return ret;
+}
+
+static int qcom_geni_console_exit(struct console *co)
+{
+	struct qcom_geni_serial_port *port;
+
+	port = get_port_from_line(co->index, true, NULL);
+	if (IS_ERR(port))
+		return PTR_ERR(port);
+
+	return pm_runtime_put_sync(port->uport.dev);
 }
 
 static void qcom_geni_serial_earlycon_write(struct console *con,
@@ -1751,6 +1758,7 @@ static struct console cons_ops = {
 	.device_unlock = qcom_geni_serial_console_device_unlock,
 	.device = uart_console_device,
 	.setup = qcom_geni_console_setup,
+	.exit = qcom_geni_console_exit,
 	.flags = CON_PRINTBUFFER | CON_NBCON,
 	.index = -1,
 	.data = &qcom_geni_console_driver,
@@ -1897,7 +1905,7 @@ static int qcom_geni_serial_probe(struct platform_device *pdev)
 
 	ret = port->dev_data->resources_init(&port->se);
 	if (ret)
-		return ret;
+		goto error;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
@@ -1947,8 +1955,13 @@ static int qcom_geni_serial_probe(struct platform_device *pdev)
 	uport->irq = irq;
 	uport->has_sysrq = IS_ENABLED(CONFIG_SERIAL_QCOM_GENI_CONSOLE);
 
-	if (!data->console)
+	if (!data->console) {
 		port->wakeup_irq = platform_get_irq_optional(pdev, 1);
+		if (port->wakeup_irq < 0 && port->wakeup_irq != -ENXIO) {
+			ret = port->wakeup_irq;
+			goto error;
+		}
+	}
 
 	if (of_property_read_bool(pdev->dev.of_node, "rx-tx-swap"))
 		port->rx_tx_swap = true;
@@ -1979,7 +1992,6 @@ static int qcom_geni_serial_probe(struct platform_device *pdev)
 						port->wakeup_irq);
 		if (ret) {
 			device_init_wakeup(&pdev->dev, false);
-			ida_free(&port_ida, uport->line);
 			goto error;
 		}
 	}
@@ -2003,12 +2015,13 @@ static int qcom_geni_serial_probe(struct platform_device *pdev)
 	return 0;
 
 error:
+	if (!data->console)
+		ida_free(&port_ida, uport->line);
 	if (port->rx_dma_addr) {
 		dma_unmap_single(pdev->dev.parent, port->rx_dma_addr,
 				 DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
 		port->rx_dma_addr = 0;
 	}
-	dev_pm_domain_detach_list(port->se.pd_list);
 	return ret;
 }
 
@@ -2024,7 +2037,8 @@ static void qcom_geni_serial_remove(struct platform_device *pdev)
 	irq_work_sync(&port->tx_kick);
 	dev_pm_clear_wake_irq(&pdev->dev);
 	device_init_wakeup(&pdev->dev, false);
-	ida_free(&port_ida, uport->line);
+	if (!port->dev_data->console)
+		ida_free(&port_ida, uport->line);
 	uart_remove_one_port(drv, &port->uport);
 
 	if (port->rx_dma_addr) {
@@ -2032,8 +2046,6 @@ static void qcom_geni_serial_remove(struct platform_device *pdev)
 				 DMA_RX_BUF_SIZE, DMA_FROM_DEVICE);
 		port->rx_dma_addr = 0;
 	}
-
-	dev_pm_domain_detach_list(port->se.pd_list);
 }
 
 static int __maybe_unused qcom_geni_serial_runtime_suspend(struct device *dev)
@@ -2099,9 +2111,11 @@ static int qcom_geni_serial_resume(struct device *dev)
 	struct uart_port *uport = &port->uport;
 	struct qcom_geni_private_data *private_data = uport->private_data;
 
-	ret = pm_runtime_force_resume(dev);
-	if (ret)
-		return ret;
+	if (console_suspend_enabled || !uart_console(uport)) {
+		ret = pm_runtime_force_resume(dev);
+		if (ret)
+			return ret;
+	}
 
 	ret = uart_resume_port(private_data->drv, uport);
 	if (uart_console(uport)) {
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index 95774b0f1484b2..319a4b427f3a37 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -33,7 +33,6 @@
 #include <linux/uaccess.h>
 
 #include "serial_base.h"
-#include "8250/8250.h" /* For hub6_match_port() */
 
 /*
  * This is used to lock changes in serial line configuration.
@@ -247,29 +246,29 @@ static int uart_alloc_xmit_buf(struct tty_port *port)
 	struct uart_state *state = container_of(port, struct uart_state, port);
 	struct uart_port *uport;
 	unsigned long flags;
-	unsigned long page;
+	u8 *buf;
 
 	/*
 	 * Initialise and allocate the transmit and temporary
 	 * buffer.
 	 */
-	page = get_zeroed_page(GFP_KERNEL);
-	if (!page)
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
 		return -ENOMEM;
 
 	uport = uart_port_ref_lock(state, &flags);
 	if (!state->port.xmit_buf) {
-		state->port.xmit_buf = (unsigned char *)page;
+		state->port.xmit_buf = buf;
 		kfifo_init(&state->port.xmit_fifo, state->port.xmit_buf,
 				PAGE_SIZE);
 		uart_port_unlock_deref(uport, flags);
 	} else {
 		uart_port_unlock_deref(uport, flags);
 		/*
-		 * Do not free() the page under the port lock, see
+		 * Do not free() the buffer under the port lock, see
 		 * uart_free_xmit_buf().
 		 */
-		free_page(page);
+		kfree(buf);
 	}
 
 	return 0;
@@ -280,10 +279,10 @@ static void uart_free_xmit_buf(struct tty_port *port)
 	struct uart_state *state = container_of(port, struct uart_state, port);
 	struct uart_port *uport;
 	unsigned long flags;
-	char *xmit_buf;
+	u8 *xmit_buf;
 
 	/*
-	 * Do not free() the transmit buffer page under the port lock since
+	 * Do not free() the transmit buffer under the port lock since
 	 * this can create various circular locking scenarios. For instance,
 	 * console driver may need to allocate/free a debug object, which
 	 * can end up in printk() recursion.
@@ -294,7 +293,7 @@ static void uart_free_xmit_buf(struct tty_port *port)
 	INIT_KFIFO(port->xmit_fifo);
 	uart_port_unlock_deref(uport, flags);
 
-	free_page((unsigned long)xmit_buf);
+	kfree(xmit_buf);
 }
 
 /*
@@ -896,7 +895,7 @@ static int uart_set_info(struct tty_struct *tty, struct tty_port *port,
 	upf_t old_flags, new_flags;
 	int retval;
 
-	if (!uport)
+	if (!uport || tty_io_error(tty))
 		return -EIO;
 
 	new_port = new_info->port;
@@ -1119,7 +1118,7 @@ static int uart_break_ctl(struct tty_struct *tty, int break_state)
 	guard(mutex)(&port->mutex);
 
 	uport = uart_port_check(state);
-	if (!uport)
+	if (!uport || tty_io_error(tty))
 		return -EIO;
 
 	if (uport->type != PORT_UNKNOWN && uport->ops->break_ctl)
@@ -1144,7 +1143,7 @@ static int uart_do_autoconfig(struct tty_struct *tty, struct uart_state *state)
 	 */
 	scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &port->mutex) {
 		uport = uart_port_check(state);
-		if (!uport)
+		if (!uport || tty_io_error(tty))
 			return -EIO;
 
 		if (tty_port_users(port) != 1)
@@ -1199,7 +1198,7 @@ static void uart_enable_ms(struct uart_port *uport)
  * FIXME: This wants extracting into a common all driver implementation
  * of TIOCMWAIT using tty_port.
  */
-static int uart_wait_modem_status(struct uart_state *state, unsigned long arg)
+static int uart_wait_modem_status(struct tty_struct *tty, struct uart_state *state, unsigned long arg)
 {
 	struct uart_port *uport;
 	struct tty_port *port = &state->port;
@@ -1213,18 +1212,29 @@ static int uart_wait_modem_status(struct uart_state *state, unsigned long arg)
 	uport = uart_port_ref(state);
 	if (!uport)
 		return -EIO;
-	scoped_guard(uart_port_lock_irq, uport) {
-		memcpy(&cprev, &uport->icount, sizeof(struct uart_icount));
-		uart_enable_ms(uport);
+
+	mutex_lock(&port->mutex);
+	if (tty_io_error(tty)) {
+		mutex_unlock(&port->mutex);
+		ret = -EIO;
+		goto out_deref;
 	}
 
+	uart_port_lock_irq(uport);
+	memcpy(&cprev, &uport->icount, sizeof(struct uart_icount));
+	uart_enable_ms(uport);
+	uart_port_unlock_irq(uport);
+
+	mutex_unlock(&port->mutex);
+
 	add_wait_queue(&port->delta_msr_wait, &wait);
 	for (;;) {
-		scoped_guard(uart_port_lock_irq, uport)
-			memcpy(&cnow, &uport->icount, sizeof(struct uart_icount));
-
 		set_current_state(TASK_INTERRUPTIBLE);
 
+		uart_port_lock_irq(uport);
+		memcpy(&cnow, &uport->icount, sizeof(struct uart_icount));
+		uart_port_unlock_irq(uport);
+
 		if (((arg & TIOCM_RNG) && (cnow.rng != cprev.rng)) ||
 		    ((arg & TIOCM_DSR) && (cnow.dsr != cprev.dsr)) ||
 		    ((arg & TIOCM_CD)  && (cnow.dcd != cprev.dcd)) ||
@@ -1233,6 +1243,11 @@ static int uart_wait_modem_status(struct uart_state *state, unsigned long arg)
 			break;
 		}
 
+		if (tty_io_error(tty)) {
+			ret = -EIO;
+			break;
+		}
+
 		schedule();
 
 		/* see if a signal did it */
@@ -1245,6 +1260,7 @@ static int uart_wait_modem_status(struct uart_state *state, unsigned long arg)
 	}
 	__set_current_state(TASK_RUNNING);
 	remove_wait_queue(&port->delta_msr_wait, &wait);
+out_deref:
 	uart_port_deref(uport);
 
 	return ret;
@@ -1567,7 +1583,7 @@ uart_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg)
 
 	/* This should only be used when the hardware is present. */
 	if (cmd == TIOCMIWAIT)
-		return uart_wait_modem_status(state, arg);
+		return uart_wait_modem_status(tty, state, arg);
 
 	/* rs485_config requires more locking than others */
 	if (cmd == TIOCSRS485)
@@ -1623,14 +1639,13 @@ static void uart_set_ldisc(struct tty_struct *tty)
 {
 	struct uart_state *state = tty->driver_data;
 	struct uart_port *uport;
-	struct tty_port *port = &state->port;
-
-	if (!tty_port_initialized(port))
-		return;
 
 	guard(mutex)(&state->port.mutex);
 	uport = uart_port_check(state);
-	if (uport && uport->ops->set_ldisc)
+	if (!uport || tty_io_error(tty))
+		return;
+
+	if (uport->ops->set_ldisc)
 		uport->ops->set_ldisc(uport, &tty->termios);
 }
 
@@ -1646,7 +1661,7 @@ static void uart_set_termios(struct tty_struct *tty,
 	guard(mutex)(&state->port.mutex);
 
 	uport = uart_port_check(state);
-	if (!uport)
+	if (!uport || tty_io_error(tty))
 		return;
 
 	/*
@@ -1798,7 +1813,14 @@ static void uart_wait_until_sent(struct tty_struct *tty, int timeout)
 	 * 'timeout' / 'expire' give us the maximum amount of time
 	 * we wait.
 	 */
-	while (!port->ops->tx_empty(port)) {
+	for (;;) {
+		mutex_lock(&state->port.mutex);
+		if (tty_io_error(tty) || port->ops->tx_empty(port)) {
+			mutex_unlock(&state->port.mutex);
+			break;
+		}
+		mutex_unlock(&state->port.mutex);
+
 		msleep_interruptible(jiffies_to_msecs(char_time));
 		if (signal_pending(current))
 			break;
@@ -2110,7 +2132,8 @@ EXPORT_SYMBOL_GPL(uart_console_write);
 
 /**
  * uart_parse_earlycon - Parse earlycon options
- * @p:	     ptr to 2nd field (ie., just beyond '<name>,')
+ * @p:	     ptr to 2nd field (ie., just beyond '<name>,'); %NULL if
+ *	     no console options were supplied
  * @iotype:  ptr for decoded iotype (out)
  * @addr:    ptr for decoded mapbase/iobase (out)
  * @options: ptr for <options> field; %NULL if not present (out)
@@ -2130,6 +2153,9 @@ EXPORT_SYMBOL_GPL(uart_console_write);
 int uart_parse_earlycon(char *p, enum uart_iotype *iotype,
 			resource_size_t *addr, char **options)
 {
+	if (!p)
+		return -EINVAL;
+
 	if (strncmp(p, "mmio,", 5) == 0) {
 		*iotype = UPIO_MEM;
 		p += 5;
@@ -3233,32 +3259,6 @@ static void serial_core_remove_one_port(struct uart_driver *drv,
 	state->uart_port = NULL;
 }
 
-/**
- * uart_match_port - are the two ports equivalent?
- * @port1: first port
- * @port2: second port
- *
- * This utility function can be used to determine whether two uart_port
- * structures describe the same port.
- */
-bool uart_match_port(const struct uart_port *port1,
-		const struct uart_port *port2)
-{
-	if (port1->iotype != port2->iotype)
-		return false;
-	else if (port1->iotype == UPIO_PORT)
-		return port1->iobase == port2->iobase;
-	else if (port1->iotype == UPIO_HUB6)
-		return hub6_match_port(port1, port2);
-	else if (uart_iotype_mmio(port1->iotype))
-		return port1->mapbase == port2->mapbase;
-	else if (port1->iotype == UPIO_BUS)
-		return true;
-	else
-		return false;
-}
-EXPORT_SYMBOL(uart_match_port);
-
 static struct serial_ctrl_device *
 serial_core_get_ctrl_dev(struct serial_port_device *port_dev)
 {
@@ -3358,6 +3358,7 @@ int serial_core_register_port(struct uart_driver *drv, struct uart_port *port)
 
 err_unregister_port_dev:
 	serial_base_port_device_remove(port->port_dev);
+	port->port_dev = NULL;
 
 err_unregister_ctrl_dev:
 	serial_base_ctrl_device_remove(new_ctrl_dev);
@@ -3372,12 +3373,24 @@ int serial_core_register_port(struct uart_driver *drv, struct uart_port *port)
 void serial_core_unregister_port(struct uart_driver *drv, struct uart_port *port)
 {
 	struct device *phys_dev = port->dev;
-	struct serial_port_device *port_dev = port->port_dev;
-	struct serial_ctrl_device *ctrl_dev = serial_core_get_ctrl_dev(port_dev);
+	struct serial_port_device *port_dev;
+	struct serial_ctrl_device *ctrl_dev;
 	int ctrl_id = port->ctrl_id;
 
 	guard(mutex)(&port_mutex);
 
+	/*
+	 * A NULL port device means there is no registered port device to
+	 * remove: serial_core_remove_one_port() clears port_dev on
+	 * teardown, and it is never set if registration failed before
+	 * serial_core_port_device_add().
+	 */
+	port_dev = port->port_dev;
+	if (!port_dev)
+		return;
+
+	ctrl_dev = serial_core_get_ctrl_dev(port_dev);
+
 	port->flags |= UPF_DEAD;
 
 	serial_core_remove_one_port(drv, port);
diff --git a/drivers/tty/serial/serial_txx9.c b/drivers/tty/serial/serial_txx9.c
index 4ae9a45c8e3a32..5b574ed276b26a 100644
--- a/drivers/tty/serial/serial_txx9.c
+++ b/drivers/tty/serial/serial_txx9.c
@@ -965,7 +965,12 @@ static int serial_txx9_register_port(struct uart_port *port)
 	mutex_lock(&serial_txx9_mutex);
 	for (i = 0; i < UART_NR; i++) {
 		uart = &serial_txx9_ports[i];
-		if (uart_match_port(uart, port)) {
+		/*
+		 * serial_txx9_register_port() is only called from
+		 * pciserial_txx9_init_one(). So .iotype is UPIO_PORT and
+		 * comparing .iobase serves to distinguish ports.
+		 */
+		if (uart->iobase == port->iobase) {
 			uart_remove_one_port(&serial_txx9_reg, uart);
 			break;
 		}
@@ -1009,7 +1014,8 @@ static void serial_txx9_unregister_port(int line)
 	struct uart_port *uart = &serial_txx9_ports[line];
 
 	mutex_lock(&serial_txx9_mutex);
-	uart_remove_one_port(&serial_txx9_reg, uart);
+	if (uart->iobase || uart->mapbase)
+		uart_remove_one_port(&serial_txx9_reg, uart);
 	uart->flags = 0;
 	uart->type = PORT_UNKNOWN;
 	uart->iobase = 0;
@@ -1019,52 +1025,6 @@ static void serial_txx9_unregister_port(int line)
 	mutex_unlock(&serial_txx9_mutex);
 }
 
-/*
- * Register a set of serial devices attached to a platform device.
- */
-static int serial_txx9_probe(struct platform_device *dev)
-{
-	struct uart_port *p = dev_get_platdata(&dev->dev);
-	struct uart_port port;
-	int ret, i;
-
-	memset(&port, 0, sizeof(struct uart_port));
-	for (i = 0; p && p->uartclk != 0; p++, i++) {
-		port.iobase	= p->iobase;
-		port.membase	= p->membase;
-		port.irq	= p->irq;
-		port.uartclk	= p->uartclk;
-		port.iotype	= p->iotype;
-		port.flags	= p->flags;
-		port.mapbase	= p->mapbase;
-		port.dev	= &dev->dev;
-		port.has_sysrq	= IS_ENABLED(CONFIG_SERIAL_TXX9_CONSOLE);
-		ret = serial_txx9_register_port(&port);
-		if (ret < 0) {
-			dev_err(&dev->dev, "unable to register port at index %d "
-				"(IO%lx MEM%llx IRQ%d): %d\n", i,
-				p->iobase, (unsigned long long)p->mapbase,
-				p->irq, ret);
-		}
-	}
-	return 0;
-}
-
-/*
- * Remove serial ports registered against a platform device.
- */
-static void serial_txx9_remove(struct platform_device *dev)
-{
-	int i;
-
-	for (i = 0; i < UART_NR; i++) {
-		struct uart_port *up = &serial_txx9_ports[i];
-
-		if (up->dev == &dev->dev)
-			serial_txx9_unregister_port(i);
-	}
-}
-
 #ifdef CONFIG_PM
 static int serial_txx9_suspend(struct platform_device *dev, pm_message_t state)
 {
@@ -1096,8 +1056,6 @@ static int serial_txx9_resume(struct platform_device *dev)
 #endif
 
 static struct platform_driver serial_txx9_plat_driver = {
-	.probe		= serial_txx9_probe,
-	.remove		= serial_txx9_remove,
 #ifdef CONFIG_PM
 	.suspend	= serial_txx9_suspend,
 	.resume		= serial_txx9_resume,
@@ -1251,9 +1209,7 @@ static void __exit serial_txx9_exit(void)
 	platform_driver_unregister(&serial_txx9_plat_driver);
 	platform_device_unregister(serial_txx9_plat_devs);
 	for (i = 0; i < UART_NR; i++) {
-		struct uart_port *up = &serial_txx9_ports[i];
-		if (up->iobase || up->mapbase)
-			uart_remove_one_port(&serial_txx9_reg, up);
+		serial_txx9_unregister_port(i);
 	}
 
 	uart_unregister_driver(&serial_txx9_reg);
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index 92193eb26abf5e..50ae9aae6b6146 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -3937,12 +3937,12 @@ static int sci_probe(struct platform_device *dev)
 	if (sp->port.fifosize > 1) {
 		ret = device_create_file(&dev->dev, &dev_attr_rx_fifo_trigger);
 		if (ret)
-			return ret;
+			goto err_remove_port;
 
 		ret = device_create_file(&dev->dev, &dev_attr_rx_fifo_timeout);
 		if (ret) {
 			device_remove_file(&dev->dev, &dev_attr_rx_fifo_trigger);
-			return ret;
+			goto err_remove_port;
 		}
 	}
 
@@ -3952,6 +3952,10 @@ static int sci_probe(struct platform_device *dev)
 
 	sci_ports_in_use |= BIT(dev_id);
 	return 0;
+
+err_remove_port:
+	uart_remove_one_port(&sci_uart_driver, &sp->port);
+	return ret;
 }
 
 static int sci_suspend(struct device *dev)
diff --git a/drivers/tty/serial/sifive.c b/drivers/tty/serial/sifive.c
index 110d676131922a..5d76f047362609 100644
--- a/drivers/tty/serial/sifive.c
+++ b/drivers/tty/serial/sifive.c
@@ -999,7 +999,7 @@ static int sifive_serial_probe(struct platform_device *pdev)
 	}
 
 #ifdef CONFIG_SERIAL_SIFIVE_CONSOLE
-	if (id > SIFIVE_SERIAL_MAX_PORTS) {
+	if (id >= SIFIVE_SERIAL_MAX_PORTS) {
 		dev_err(&pdev->dev, "too many UARTs (%d)\n", id);
 		return -EINVAL;
 	}
diff --git a/drivers/tty/serial/vt8500_serial.c b/drivers/tty/serial/vt8500_serial.c
index 78a1c1eea11bbf..fc6fdc751b1675 100644
--- a/drivers/tty/serial/vt8500_serial.c
+++ b/drivers/tty/serial/vt8500_serial.c
@@ -656,6 +656,7 @@ static int vt8500_serial_probe(struct platform_device *pdev)
 	ret = clk_prepare_enable(vt8500_port->clk);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to enable clock\n");
+		clk_put(vt8500_port->clk);
 		return ret;
 	}
 
diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c
index c2e4b31b699aae..f4571d05b49626 100644
--- a/drivers/tty/sysrq.c
+++ b/drivers/tty/sysrq.c
@@ -911,10 +911,7 @@ static bool sysrq_handle_keypress(struct sysrq_state *sysrq,
 		 * keyboard state so we can release keys that have been
 		 * pressed before entering SysRq mode.
 		 */
-		if (value)
-			set_bit(code, sysrq->key_down);
-		else
-			clear_bit(code, sysrq->key_down);
+		assign_bit(code, sysrq->key_down, value);
 
 		if (was_active)
 			schedule_work(&sysrq->reinject_work);
diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
index 48569035da56e5..1c30faae9ec13e 100644
--- a/drivers/tty/tty_io.c
+++ b/drivers/tty/tty_io.c
@@ -620,6 +620,8 @@ static void __tty_hangup(struct tty_struct *tty, int exit_session)
 
 	tty_ldisc_hangup(tty, cons_filp != NULL);
 
+	wake_up_interruptible(&tty->break_wait);
+
 	spin_lock_irq(&tty->ctrl.lock);
 	clear_bit(TTY_THROTTLED, &tty->flags);
 	clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags);
@@ -2431,6 +2433,7 @@ static int tiocgetd(struct tty_struct *tty, int __user *p)
  * send_break - performed time break
  * @tty: device to break on
  * @duration: timeout in mS
+ * @file: file object
  *
  * Perform a timed break on hardware that lacks its own driver level timed
  * break functionality.
@@ -2438,8 +2441,9 @@ static int tiocgetd(struct tty_struct *tty, int __user *p)
  * Locking:
  *	@tty->atomic_write_lock serializes
  */
-static int send_break(struct tty_struct *tty, unsigned int duration)
+static int send_break(struct file *file, struct tty_struct *tty, unsigned int duration)
 {
+	long timeout;
 	int retval;
 
 	if (tty->ops->break_ctl == NULL)
@@ -2453,13 +2457,26 @@ static int send_break(struct tty_struct *tty, unsigned int duration)
 		return -EINTR;
 
 	retval = tty->ops->break_ctl(tty, -1);
-	if (!retval) {
-		msleep_interruptible(duration);
-		retval = tty->ops->break_ctl(tty, 0);
-	} else if (retval == -EOPNOTSUPP) {
-		/* some drivers can tell only dynamically */
-		retval = 0;
+	if (retval) {
+		if (retval == -EOPNOTSUPP) {
+			/* some drivers can tell only dynamically */
+			retval = 0;
+		}
+		goto out_unlock;
+	}
+
+	timeout = msecs_to_jiffies(duration);
+	timeout = wait_event_interruptible_timeout(tty->break_wait,
+						   tty_hung_up_p(file),
+						   timeout);
+	/* return early on hangup only */
+	if (timeout > 0) {
+		retval = -EIO;
+		goto out_unlock;
 	}
+
+	retval = tty->ops->break_ctl(tty, 0);
+out_unlock:
 	tty_write_unlock(tty);
 
 	if (signal_pending(current))
@@ -2727,10 +2744,10 @@ long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		 * This is used by the tcdrain() termios function.
 		 */
 		if (!arg)
-			return send_break(tty, 250);
+			return send_break(file, tty, 250);
 		return 0;
 	case TCSBRKP:	/* support for POSIX tcsendbreak() */
-		return send_break(tty, arg ? arg*100 : 250);
+		return send_break(file, tty, arg ? arg * 100 : 250);
 
 	case TIOCMGET:
 		return tty_tiocmget(tty, p);
@@ -3090,6 +3107,7 @@ struct tty_struct *alloc_tty_struct(struct tty_driver *driver, int idx)
 	init_ldsem(&tty->ldisc_sem);
 	init_waitqueue_head(&tty->write_wait);
 	init_waitqueue_head(&tty->read_wait);
+	init_waitqueue_head(&tty->break_wait);
 	INIT_WORK(&tty->hangup_work, do_tty_hangup);
 	mutex_init(&tty->atomic_write_lock);
 	spin_lock_init(&tty->ctrl.lock);
diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c
index 54359310e293cd..4ca4796d130977 100644
--- a/drivers/tty/tty_port.c
+++ b/drivers/tty/tty_port.c
@@ -240,13 +240,13 @@ EXPORT_SYMBOL_GPL(tty_port_unregister_device);
 
 int tty_port_alloc_xmit_buf(struct tty_port *port)
 {
-	/* We may sleep in get_zeroed_page() */
+	/* We may sleep in kzalloc() */
 	guard(mutex)(&port->buf_mutex);
 
 	if (port->xmit_buf)
 		return 0;
 
-	port->xmit_buf = (u8 *)get_zeroed_page(GFP_KERNEL);
+	port->xmit_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (port->xmit_buf == NULL)
 		return -ENOMEM;
 
@@ -259,7 +259,7 @@ EXPORT_SYMBOL(tty_port_alloc_xmit_buf);
 void tty_port_free_xmit_buf(struct tty_port *port)
 {
 	guard(mutex)(&port->buf_mutex);
-	free_page((unsigned long)port->xmit_buf);
+	kfree(port->xmit_buf);
 	port->xmit_buf = NULL;
 	INIT_KFIFO(port->xmit_fifo);
 }
@@ -288,7 +288,7 @@ static void tty_port_destructor(struct kref *kref)
 	/* check if last port ref was dropped before tty release */
 	if (WARN_ON(port->itty))
 		return;
-	free_page((unsigned long)port->xmit_buf);
+	kfree(port->xmit_buf);
 	tty_port_destroy(port);
 	if (port->ops && port->ops->destruct)
 		port->ops->destruct(port);
@@ -340,19 +340,9 @@ void tty_port_tty_set(struct tty_port *port, struct tty_struct *tty)
 }
 EXPORT_SYMBOL(tty_port_tty_set);
 
-/**
- * tty_port_shutdown - internal helper to shutdown the device
- * @port: tty port to be shut down
- * @tty: the associated tty
- *
- * It is used by tty_port_hangup() and tty_port_close(). Its task is to
- * shutdown the device if it was initialized (note consoles remain
- * functioning). It lowers DTR/RTS (if @tty has HUPCL set) and invokes
- * @port->ops->shutdown().
- */
-static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty)
+static void tty_port_shutdown_locked(struct tty_port *port, struct tty_struct *tty)
 {
-	guard(mutex)(&port->mutex);
+	lockdep_assert_held(&port->mutex);
 
 	if (port->console)
 		return;
@@ -372,6 +362,23 @@ static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty)
 		port->ops->shutdown(port);
 }
 
+/**
+ * tty_port_shutdown - internal helper to shutdown the device
+ * @port: tty port to be shut down
+ * @tty: the associated tty
+ *
+ * It is used by tty_port_hangup() and tty_port_close(). Its task is to
+ * shutdown the device if it was initialized (note consoles remain
+ * functioning). It lowers DTR/RTS (if @tty has HUPCL set) and invokes
+ * @port->ops->shutdown().
+ */
+static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty)
+{
+	guard(mutex)(&port->mutex);
+
+	tty_port_shutdown_locked(port, tty);
+}
+
 /**
  * tty_port_hangup		-	hangup helper
  * @port: tty port
@@ -385,36 +392,58 @@ void tty_port_hangup(struct tty_port *port)
 {
 	struct tty_struct *tty;
 
-	scoped_guard(spinlock_irqsave, &port->lock) {
-		port->count = 0;
-		tty = port->tty;
-		if (tty)
-			set_bit(TTY_IO_ERROR, &tty->flags);
-		port->tty = NULL;
-	}
+	scoped_guard(mutex, &port->mutex) {
+		scoped_guard(spinlock_irqsave, &port->lock) {
+			port->count = 0;
+			tty = port->tty;
+			if (tty)
+				set_bit(TTY_IO_ERROR, &tty->flags);
+			port->tty = NULL;
+		}
 
-	tty_port_set_active(port, false);
-	tty_port_shutdown(port, tty);
+		tty_port_set_active(port, false);
+		tty_port_shutdown_locked(port, tty);
+	}
 	tty_kref_put(tty);
 	wake_up_interruptible(&port->open_wait);
 	wake_up_interruptible(&port->delta_msr_wait);
 }
 EXPORT_SYMBOL(tty_port_hangup);
 
-void __tty_port_tty_hangup(struct tty_port *port, bool check_clocal, bool async)
+/**
+ * tty_port_tty_hangup - helper to hang up a tty asynchronously
+ * @port: tty port
+ * @check_clocal: hang only ttys with %CLOCAL unset?
+ */
+void tty_port_tty_hangup(struct tty_port *port, bool check_clocal)
 {
 	scoped_guard(tty_port_tty, port) {
 		struct tty_struct *tty = scoped_tty();
 
-		if (!check_clocal || !C_CLOCAL(tty)) {
-			if (async)
-				tty_hangup(tty);
-			else
-				tty_vhangup(tty);
-		}
+		if (!check_clocal || !C_CLOCAL(tty))
+			tty_hangup(tty);
+	}
+}
+EXPORT_SYMBOL_GPL(tty_port_tty_hangup);
+
+/**
+ * tty_port_tty_vhangup - helper to hang up a tty synchronously
+ * @port: tty port
+ */
+void tty_port_tty_vhangup(struct tty_port *port)
+{
+	struct tty_struct *tty;
+
+	mutex_lock(&port->mutex);
+	tty = tty_port_tty_get(port);
+	mutex_unlock(&port->mutex);
+
+	if (tty) {
+		tty_vhangup(tty);
+		tty_kref_put(tty);
 	}
 }
-EXPORT_SYMBOL_GPL(__tty_port_tty_hangup);
+EXPORT_SYMBOL_GPL(tty_port_tty_vhangup);
 
 /**
  * tty_port_tty_wakeup - helper to wake up a tty
@@ -767,8 +796,10 @@ int tty_port_open(struct tty_port *port, struct tty_struct *tty,
 		clear_bit(TTY_IO_ERROR, &tty->flags);
 		if (port->ops->activate) {
 			int retval = port->ops->activate(port, tty);
-			if (retval)
+			if (retval) {
+				set_bit(TTY_IO_ERROR, &tty->flags);
 				return retval;
+			}
 		}
 		tty_port_set_initialized(port, true);
 	}
diff --git a/drivers/tty/vcc.c b/drivers/tty/vcc.c
index 27a55465bf5ec5..9adb533f7034b9 100644
--- a/drivers/tty/vcc.c
+++ b/drivers/tty/vcc.c
@@ -492,7 +492,7 @@ static ssize_t domain_show(struct device *dev,
 
 static int vcc_send_ctl(struct vcc_port *port, int ctl)
 {
-	struct vio_vcc pkt;
+	struct vio_vcc pkt = {};
 	int rv;
 
 	pkt.tag.type = VIO_TYPE_CTRL;
@@ -986,7 +986,8 @@ static void vcc_cleanup(struct tty_struct *tty)
 
 	port = vcc_get(tty->index, true);
 	if (port) {
-		port->tty = NULL;
+		scoped_guard(spinlock_irqsave, &port->lock)
+			port->tty = NULL;
 
 		if (port->removed) {
 			vcc_table_remove(tty->index);
diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c
index c41d850b29c6e6..ef4a19aeb3ce88 100644
--- a/drivers/tty/vt/keyboard.c
+++ b/drivers/tty/vt/keyboard.c
@@ -2002,7 +2002,9 @@ static int vt_kdskbent(unsigned char kbdmode, unsigned char idx,
 		return 0;
 #endif
 
-	unsigned short __free(kfree) *new_map = kmalloc(sizeof(plain_map), GFP_KERNEL);
+	unsigned short __free(kfree) *new_map =
+		kmalloc_array(ARRAY_SIZE(plain_map), sizeof(*new_map),
+			      GFP_KERNEL);
 	if (!new_map)
 		return -ENOMEM;
 
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index 57edf37495a81f..51f8fd103dcfdc 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -1134,6 +1134,8 @@ int vc_allocate(unsigned int currcons)	/* return 0 on success */
 	return 0;
 err_free:
 	visual_deinit(vc);
+	if (*vc->uni_pagedict_loc)
+		con_free_unimap(vc);
 	kfree(vc);
 	vc_cons[currcons].d = NULL;
 	return err;
diff --git a/drivers/ufs/Kconfig b/drivers/ufs/Kconfig
index f662e7ce71f1bd..b62c00e7ff063b 100644
--- a/drivers/ufs/Kconfig
+++ b/drivers/ufs/Kconfig
@@ -7,6 +7,7 @@ menuconfig SCSI_UFSHCD
 	tristate "Universal Flash Storage Controller"
 	depends on SCSI && SCSI_DMA
 	depends on RPMB || !RPMB
+	select CRYPTO_LIB_BLAKE2B if RPMB
 	select PM_DEVFREQ
 	select DEVFREQ_GOV_SIMPLE_ONDEMAND
 	select NLS
diff --git a/drivers/ufs/core/ufs-debugfs.c b/drivers/ufs/core/ufs-debugfs.c
index ed1de70e2ec1cd..9f83628f3e2cfd 100644
--- a/drivers/ufs/core/ufs-debugfs.c
+++ b/drivers/ufs/core/ufs-debugfs.c
@@ -177,7 +177,7 @@ static ssize_t ufs_saved_err_write(struct file *file, const char __user *buf,
 	if (ret < 0)
 		return ret;
 
-	spin_lock_irq(hba->host->host_lock);
+	spin_lock_irq(&hba->host->host_lock);
 	if (strcmp(attr->name, "saved_err") == 0) {
 		hba->saved_err = val;
 	} else if (strcmp(attr->name, "saved_uic_err") == 0) {
@@ -187,7 +187,7 @@ static ssize_t ufs_saved_err_write(struct file *file, const char __user *buf,
 	}
 	if (ret == 0)
 		ufshcd_schedule_eh_work(hba);
-	spin_unlock_irq(hba->host->host_lock);
+	spin_unlock_irq(&hba->host->host_lock);
 
 	return ret < 0 ? ret : count;
 }
diff --git a/drivers/ufs/core/ufs-rpmb.c b/drivers/ufs/core/ufs-rpmb.c
index aa925cbb07e869..50ef63afd52832 100644
--- a/drivers/ufs/core/ufs-rpmb.c
+++ b/drivers/ufs/core/ufs-rpmb.c
@@ -10,6 +10,7 @@
  *	Can Guo <can.guo@oss.qualcomm.com>
  */
 
+#include <crypto/blake2b.h>
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/kernel.h>
@@ -21,6 +22,8 @@
 #include <linux/unaligned.h>
 #include "ufshcd-priv.h"
 
+#define UFS_RPMB_ID_LEN			16	/* Match eMMC CID Length */
+#define UFS_RPMB_UA_RETRIES		3	/* Retries for the power-on UNIT ATTENTION */
 #define UFS_RPMB_SEC_PROTOCOL		0xEC	/* JEDEC UFS application */
 #define UFS_RPMB_SEC_PROTOCOL_ID	0x01	/* JEDEC UFS RPMB protocol ID, CDB byte3 */
 
@@ -40,6 +43,22 @@ struct ufs_rpmb_dev {
 static int ufs_sec_submit(struct ufs_hba *hba, u16 spsp, void *buffer, size_t len, bool send)
 {
 	struct scsi_device *sdev = hba->ufs_rpmb_wlun;
+	struct scsi_failure failure_defs[] = {
+		{
+			.sense_key = UNIT_ATTENTION,
+			.sense_code =
+				POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED,
+			.allowed = UFS_RPMB_UA_RETRIES,
+			.result = SAM_STAT_CHECK_CONDITION,
+		},
+		{}
+	};
+	struct scsi_failures failures = {
+		.failure_definitions = failure_defs,
+	};
+	const struct scsi_exec_args exec_args = {
+		.failures = &failures,
+	};
 	u8 cdb[12] = { };
 
 	cdb[0] = send ? SECURITY_PROTOCOL_OUT : SECURITY_PROTOCOL_IN;
@@ -48,7 +67,8 @@ static int ufs_sec_submit(struct ufs_hba *hba, u16 spsp, void *buffer, size_t le
 	put_unaligned_be32(len, &cdb[6]);
 
 	return scsi_execute_cmd(sdev, cdb, send ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN,
-				buffer, len, /*timeout=*/30 * HZ, 0, NULL);
+				buffer, len, /*timeout=*/30 * HZ, /*retries=*/0,
+				&exec_args);
 }
 
 /* UFS RPMB route frames implementation */
@@ -139,6 +159,7 @@ static void ufs_rpmb_device_release(struct device *dev)
 int ufs_rpmb_probe(struct ufs_hba *hba)
 {
 	struct ufs_rpmb_dev *ufs_rpmb, *it, *tmp;
+	u8 dev_id[UFS_RPMB_ID_LEN];
 	struct rpmb_dev *rdev;
 	char *cid = NULL;
 	int region;
@@ -197,8 +218,10 @@ int ufs_rpmb_probe(struct ufs_hba *hba)
 			goto err_out;
 		}
 
-		descr.dev_id = cid;
-		descr.dev_id_len = strlen(cid);
+		blake2b(NULL, 0, cid, strlen(cid), dev_id, UFS_RPMB_ID_LEN);
+
+		descr.dev_id = dev_id;
+		descr.dev_id_len = UFS_RPMB_ID_LEN;
 		descr.capacity = cap;
 
 		/* Register RPMB device */
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index 63e670d1a9d9e1..cf49ad0258e140 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -150,12 +150,12 @@ static inline ssize_t ufs_sysfs_pm_lvl_store(struct device *dev,
 	     !(dev_info->wspecversion >= 0x310)))
 		return -EINVAL;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (rpm)
 		hba->rpm_lvl = value;
 	else
 		hba->spm_lvl = value;
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	return count;
 }
 
@@ -787,7 +787,7 @@ static ssize_t monitor_enable_store(struct device *dev,
 		return -EINVAL;
 
 	value = !!value;
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (value == hba->monitor.enabled)
 		goto out_unlock;
 
@@ -799,7 +799,7 @@ static ssize_t monitor_enable_store(struct device *dev,
 	}
 
 out_unlock:
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	return count;
 }
 
@@ -821,11 +821,11 @@ static ssize_t monitor_chunk_size_store(struct device *dev,
 	if (kstrtoul(buf, 0, &value))
 		return -EINVAL;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	/* Only allow chunk size change when monitor is disabled */
 	if (!hba->monitor.enabled)
 		hba->monitor.chunk_size = value;
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	return count;
 }
 
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index 2ba244cf40ac7d..eb86db492f2252 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -1273,10 +1273,10 @@ static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
 	unsigned long flags;
 	u32 pending = 0;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	__shost_for_each_device(sdev, hba->host)
 		pending += scsi_device_busy(sdev);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return pending;
 }
@@ -1681,11 +1681,6 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev,
 	if (!scaling->window_start_t)
 		goto start_window;
 
-	/*
-	 * If current frequency is 0, then the ondemand governor considers
-	 * there's no initial frequency set. And it always requests to set
-	 * to max. frequency.
-	 */
 	if (hba->use_pm_opp) {
 		stat->current_frequency = hba->clk_scaling.target_freq;
 	} else {
@@ -1716,6 +1711,26 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev,
 	return 0;
 }
 
+static int ufshcd_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	if (!ufshcd_is_clkscaling_supported(hba))
+		return -EINVAL;
+
+	if (hba->use_pm_opp) {
+		*freq = hba->clk_scaling.target_freq;
+	} else {
+		struct ufs_clk_info *clki;
+
+		clki = list_first_entry(&hba->clk_list_head,
+					struct ufs_clk_info, list);
+		*freq = clki->curr_freq;
+	}
+
+	return 0;
+}
+
 static int ufshcd_devfreq_init(struct ufs_hba *hba)
 {
 	struct list_head *clk_list = &hba->clk_list_head;
@@ -1727,12 +1742,21 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba)
 	if (list_empty(clk_list))
 		return 0;
 
+	clki = list_first_entry(clk_list, struct ufs_clk_info, list);
+
 	if (!hba->use_pm_opp) {
-		clki = list_first_entry(clk_list, struct ufs_clk_info, list);
 		dev_pm_opp_add(hba->dev, clki->min_freq, 0);
 		dev_pm_opp_add(hba->dev, clki->max_freq, 0);
 	}
 
+	/*
+	 * ufshcd_init_clocks() has already set the clocks to the highest
+	 * frequency, and nothing has changed them since. Save that frequency,
+	 * so that devfreq and the clock scaling code know where we start.
+	 */
+	hba->clk_scaling.target_freq = clki->max_freq;
+	hba->vps->devfreq_profile.initial_freq = clki->max_freq;
+
 	ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
 					 &hba->vps->ondemand_data);
 	devfreq = devfreq_add_device(hba->dev,
@@ -2061,7 +2085,7 @@ static void ufshcd_gate_work(struct work_struct *work)
 			return;
 	}
 
-	scoped_guard(spinlock_irqsave, hba->host->host_lock) {
+	scoped_guard(spinlock_irqsave, &hba->host->host_lock) {
 		if (ufshcd_is_ufs_dev_busy(hba) ||
 		    hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
 			return;
@@ -2118,7 +2142,7 @@ static void __ufshcd_release(struct ufs_hba *hba)
 	    hba->clk_gating.state == CLKS_OFF)
 		return;
 
-	scoped_guard(spinlock_irqsave, hba->host->host_lock) {
+	scoped_guard(spinlock_irqsave, &hba->host->host_lock) {
 		if (ufshcd_has_pending_tasks(hba) ||
 		    hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
 			return;
@@ -2343,10 +2367,10 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 	int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
 	unsigned long flags;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
 		hba->monitor.busy_start_ts[dir] = ktime_get();
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 }
 
 static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
@@ -2356,7 +2380,7 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 	int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
 	unsigned long flags;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
 		struct ufs_hba_monitor *m = &hba->monitor;
 		ktime_t now, inc, lat;
@@ -2379,7 +2403,7 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd)
 		/* Push forward the busy start of monitor */
 		m->busy_start_ts[dir] = now;
 	}
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 }
 
 /* Returns %true for SCSI commands and %false for device management commands. */
@@ -2636,9 +2660,9 @@ ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 		}
 	}
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	hba->active_uic_cmd = NULL;
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return ret;
 }
@@ -2688,9 +2712,9 @@ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 	mutex_lock(&hba->uic_cmd_mutex);
 	ufshcd_add_delay_before_dme_cmd(hba);
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	ret = __ufshcd_send_uic_cmd(hba, uic_cmd);
 	if (!ret)
@@ -3162,9 +3186,9 @@ static enum scsi_qc_status ufshcd_queuecommand(struct Scsi_Host *host,
 	if (ufs_trigger_eh(hba)) {
 		unsigned long flags;
 
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		ufshcd_schedule_eh_work(hba);
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	}
 
 	return err;
@@ -4498,14 +4522,14 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
 	mutex_lock(&hba->uic_cmd_mutex);
 	ufshcd_add_delay_before_dme_cmd(hba);
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (ufshcd_is_link_broken(hba)) {
 		ret = -ENOLINK;
 		goto out_unlock;
 	}
 	hba->uic_async_done = &uic_async_done;
 	ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	ret = __ufshcd_send_uic_cmd(hba, cmd);
 	if (ret) {
 		dev_err(hba->dev,
@@ -4549,7 +4573,7 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
 		ufshcd_print_evt_hist(hba);
 	}
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	hba->active_uic_cmd = NULL;
 	hba->uic_async_done = NULL;
 	if (ret && !hba->pm_op_in_progress) {
@@ -4557,7 +4581,7 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
 		ufshcd_schedule_eh_work(hba);
 	}
 out_unlock:
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	mutex_unlock(&hba->uic_cmd_mutex);
 
 	return ret;
@@ -4652,21 +4676,21 @@ int ufshcd_link_recovery(struct ufs_hba *hba)
 	int ret;
 	unsigned long flags;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	hba->ufshcd_state = UFSHCD_STATE_RESET;
 	ufshcd_set_eh_in_progress(hba);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	/* Reset the attached device */
 	ufshcd_device_reset(hba);
 
 	ret = ufshcd_host_reset_and_restore(hba);
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (ret)
 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
 	ufshcd_clear_eh_in_progress(hba);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	if (ret)
 		dev_err(hba->dev, "%s: link recovery failed, err %d",
@@ -5667,19 +5691,19 @@ static void ufshcd_sdev_destroy(struct scsi_device *sdev)
 
 	/* Drop the reference as it won't be needed anymore */
 	if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		hba->ufs_device_wlun = NULL;
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	} else if (hba->ufs_device_wlun) {
 		struct device *supplier = NULL;
 
 		/* Ensure UFS Device WLUN exists and does not disappear */
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		if (hba->ufs_device_wlun) {
 			supplier = &hba->ufs_device_wlun->sdev_gendev;
 			get_device(supplier);
 		}
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 		if (supplier) {
 			/*
@@ -5872,7 +5896,7 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
 	irqreturn_t retval = IRQ_NONE;
 	struct uic_command *cmd;
 
-	guard(spinlock_irqsave)(hba->host->host_lock);
+	guard(spinlock_irqsave)(&hba->host->host_lock);
 	cmd = hba->active_uic_cmd;
 	if (!cmd) {
 		dev_err(hba->dev,
@@ -6669,7 +6693,7 @@ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
 	unsigned long flags;
 	bool err_handling = true;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	/*
 	 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
 	 * device fatal error and/or DL NAC & REPLAY timeout errors.
@@ -6688,9 +6712,9 @@ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
 		/*
 		 * wait for 50ms to see if we can get any other errors or not.
 		 */
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		msleep(50);
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 
 		/*
 		 * now check if we have got any other severe errors other than
@@ -6708,9 +6732,9 @@ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
 		 *   - If we get response then clear the DL NAC error bit.
 		 */
 
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		err = ufshcd_verify_dev_init(hba);
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 
 		if (err)
 			goto out;
@@ -6724,7 +6748,7 @@ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
 			err_handling = false;
 	}
 out:
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	return err_handling;
 }
 
@@ -6737,7 +6761,7 @@ static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
 
 void ufshcd_schedule_eh_work(struct ufs_hba *hba)
 {
-	lockdep_assert_held(hba->host->host_lock);
+	lockdep_assert_held(&hba->host->host_lock);
 
 	/* handle fatal errors only when link is not in error state */
 	if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
@@ -6752,10 +6776,10 @@ void ufshcd_schedule_eh_work(struct ufs_hba *hba)
 
 void ufshcd_force_error_recovery(struct ufs_hba *hba)
 {
-	spin_lock_irq(hba->host->host_lock);
+	spin_lock_irq(&hba->host->host_lock);
 	hba->force_reset = true;
 	ufshcd_schedule_eh_work(hba);
-	spin_unlock_irq(hba->host->host_lock);
+	spin_unlock_irq(&hba->host->host_lock);
 }
 EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery);
 
@@ -6817,11 +6841,17 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
 	}
 	/* Wait for ongoing ufshcd_queuecommand() calls to finish. */
 	blk_mq_quiesce_tagset(&hba->host->tag_set);
+	/*
+	 * Internal commands are submitted on the pseudo SCSI device. Let them
+	 * through so that the error handler can recover the link.
+	 */
+	blk_mq_unquiesce_queue(hba->host->pseudo_sdev->request_queue);
 	cancel_work_sync(&hba->eeh_work);
 }
 
 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
 {
+	blk_mq_quiesce_queue_nowait(hba->host->pseudo_sdev->request_queue);
 	blk_mq_unquiesce_tagset(&hba->host->tag_set);
 	ufshcd_release(hba);
 	if (ufshcd_is_clkscaling_supported(hba))
@@ -6996,25 +7026,25 @@ static void ufshcd_err_handler(struct work_struct *work)
 	}
 
 	down(&hba->host_sem);
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (ufshcd_err_handling_should_stop(hba)) {
 		if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
 			hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		up(&hba->host_sem);
 		return;
 	}
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	ufshcd_err_handling_prepare(hba);
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	ufshcd_set_eh_in_progress(hba);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	/* Complete requests that have door-bell cleared by h/w */
 	ufshcd_complete_requests(hba, false);
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 again:
 	needs_restore = false;
 	needs_reset = false;
@@ -7032,10 +7062,10 @@ static void ufshcd_err_handler(struct work_struct *work)
 	    !hba->force_reset) {
 		bool ret;
 
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		/* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
 		ret = ufshcd_quirk_dl_nac_errors(hba);
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		if (!ret && ufshcd_err_handling_should_stop(hba))
 			goto skip_err_handling;
 	}
@@ -7045,14 +7075,14 @@ static void ufshcd_err_handler(struct work_struct *work)
 	     (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
 		bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
 
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		ufshcd_print_host_state(hba);
 		ufshcd_print_pwr_info(hba);
 		ufshcd_print_tx_eq_params(hba);
 		ufshcd_print_evt_hist(hba);
 		ufshcd_print_tmrs(hba, hba->outstanding_tasks);
 		ufshcd_print_trs_all(hba, pr_prdt);
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 	}
 
 	/*
@@ -7077,21 +7107,21 @@ static void ufshcd_err_handler(struct work_struct *work)
 		hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
 		if (!hba->saved_uic_err)
 			hba->saved_err &= ~UIC_ERROR;
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		if (ufshcd_is_pwr_mode_restore_needed(hba))
 			needs_restore = true;
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		if (!hba->saved_err && !needs_restore)
 			goto skip_err_handling;
 	}
 
 	hba->silence_err_logs = true;
 	/* release lock as clear command might sleep */
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	needs_reset = ufshcd_abort_all(hba);
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	hba->silence_err_logs = false;
 	if (needs_reset)
 		goto do_reset;
@@ -7101,7 +7131,7 @@ static void ufshcd_err_handler(struct work_struct *work)
 	 * now it is safe to retore power mode.
 	 */
 	if (needs_restore) {
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		/*
 		 * Hold the scaling lock just in case dev cmds
 		 * are sent via bsg and/or sysfs.
@@ -7116,7 +7146,7 @@ static void ufshcd_err_handler(struct work_struct *work)
 		}
 		ufshcd_print_pwr_info(hba);
 		up_write(&hba->clk_scaling_lock);
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 	}
 
 do_reset:
@@ -7125,14 +7155,14 @@ static void ufshcd_err_handler(struct work_struct *work)
 		int err;
 
 		hba->force_reset = false;
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		err = ufshcd_reset_and_restore(hba);
 		if (err)
 			dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
 					__func__, err);
 		else
 			ufshcd_recover_pm_error(hba);
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 	}
 
 skip_err_handling:
@@ -7151,7 +7181,7 @@ static void ufshcd_err_handler(struct work_struct *work)
 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
 	}
 	ufshcd_clear_eh_in_progress(hba);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	ufshcd_err_handling_unprepare(hba);
 	up(&hba->host_sem);
 
@@ -7274,7 +7304,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
 	bool queue_eh_work = false;
 	irqreturn_t retval = IRQ_NONE;
 
-	guard(spinlock_irqsave)(hba->host->host_lock);
+	guard(spinlock_irqsave)(&hba->host->host_lock);
 	hba->errors |= UFSHCD_ERROR_MASK & intr_status;
 
 	if (hba->errors & INT_FATAL_ERRORS) {
@@ -7352,7 +7382,7 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
 	irqreturn_t ret = IRQ_NONE;
 	int tag;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
 	issued = hba->outstanding_tasks & ~pending;
 	for_each_set_bit(tag, &issued, hba->nutmrs) {
@@ -7362,7 +7392,7 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
 		complete(c);
 		ret = IRQ_HANDLED;
 	}
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return ret;
 }
@@ -7532,7 +7562,7 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
 	req->end_io_data = &wait;
 	ufshcd_hold(hba);
 
-	spin_lock_irqsave(host->host_lock, flags);
+	spin_lock_irqsave(&host->host_lock, flags);
 
 	task_tag = req->tag;
 	hba->tmf_rqs[req->tag] = req;
@@ -7543,7 +7573,7 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
 
 	__set_bit(task_tag, &hba->outstanding_tasks);
 
-	spin_unlock_irqrestore(host->host_lock, flags);
+	spin_unlock_irqrestore(&host->host_lock, flags);
 
 	/* send command to the controller */
 	ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
@@ -7568,10 +7598,10 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
 		ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
 	}
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	hba->tmf_rqs[req->tag] = NULL;
 	__clear_bit(task_tag, &hba->outstanding_tasks);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	ufshcd_release(hba);
 	blk_mq_free_request(req);
@@ -8119,10 +8149,10 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
 	if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
 		ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
 
-		spin_lock_irqsave(host->host_lock, flags);
+		spin_lock_irqsave(&host->host_lock, flags);
 		hba->force_reset = true;
 		ufshcd_schedule_eh_work(hba);
-		spin_unlock_irqrestore(host->host_lock, flags);
+		spin_unlock_irqrestore(&host->host_lock, flags);
 		goto release;
 	}
 
@@ -8180,12 +8210,12 @@ static void ufshcd_process_probe_result(struct ufs_hba *hba,
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (ret)
 		hba->ufshcd_state = UFSHCD_STATE_ERROR;
 	else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
 		hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	trace_ufshcd_init(hba, ret,
 			  ktime_to_us(ktime_sub(ktime_get(), probe_start)),
@@ -8254,7 +8284,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba)
 	unsigned long flags;
 	int retries = MAX_HOST_RESET_RETRIES;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	do {
 		/*
 		 * This is a fresh start, cache and clear saved error first,
@@ -8266,14 +8296,14 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba)
 		hba->saved_uic_err = 0;
 		hba->force_reset = false;
 		hba->ufshcd_state = UFSHCD_STATE_RESET;
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 		/* Reset the attached device */
 		ufshcd_device_reset(hba);
 
 		err = ufshcd_host_reset_and_restore(hba);
 
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		if (err)
 			continue;
 		/* Do not exit unless operational or dead */
@@ -8293,7 +8323,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba)
 		hba->saved_err |= saved_err;
 		hba->saved_uic_err |= saved_uic_err;
 	}
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return err;
 }
@@ -8325,18 +8355,18 @@ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
 		return err;
 	}
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	hba->force_reset = true;
 	ufshcd_schedule_eh_work(hba);
 	dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	flush_work(&hba->eh_work);
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
 		err = FAILED;
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	return err;
 }
@@ -9612,6 +9642,7 @@ static struct ufs_hba_variant_params ufs_hba_vps = {
 	.devfreq_profile.polling_ms	= 100,
 	.devfreq_profile.target		= ufshcd_devfreq_target,
 	.devfreq_profile.get_dev_status	= ufshcd_devfreq_get_dev_status,
+	.devfreq_profile.get_cur_freq	= ufshcd_devfreq_get_cur_freq,
 	.ondemand_data.upthreshold	= 70,
 	.ondemand_data.downdifferential	= 5,
 };
@@ -10081,13 +10112,13 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
 	unsigned long flags;
 	int ret;
 
-	spin_lock_irqsave(hba->host->host_lock, flags);
+	spin_lock_irqsave(&hba->host->host_lock, flags);
 	sdp = hba->ufs_device_wlun;
 	if (sdp && scsi_device_online(sdp))
 		ret = scsi_device_get(sdp);
 	else
 		ret = -ENODEV;
-	spin_unlock_irqrestore(hba->host->host_lock, flags);
+	spin_unlock_irqrestore(&hba->host->host_lock, flags);
 
 	if (ret)
 		return ret;
diff --git a/drivers/ufs/host/ufs-amd-versal2.c b/drivers/ufs/host/ufs-amd-versal2.c
index 2154d628681716..dff0c2c9548655 100644
--- a/drivers/ufs/host/ufs-amd-versal2.c
+++ b/drivers/ufs/host/ufs-amd-versal2.c
@@ -225,8 +225,6 @@ static int ufs_versal2_setup_phy(struct ufs_hba *hba)
 static int ufs_versal2_phy_init(struct ufs_hba *hba)
 {
 	struct ufs_versal2_host *host = ufshcd_get_variant(hba);
-	u32 time_left;
-	bool is_ready;
 	int ret;
 	static const struct ufshcd_dme_attr_val rmmi_attrs[] = {
 		{ UIC_ARG_MIB(CBREFCLKCTRL2), CBREFREFCLK_GATE_OVR_EN, DME_LOCAL },
@@ -235,23 +233,15 @@ static int ufs_versal2_phy_init(struct ufs_hba *hba)
 		{ UIC_ARG_MIB(VS_MPHYCFGUPDT), 1, DME_LOCAL }
 	};
 
-	/* Wait for Tx/Rx config_rdy */
-	time_left = TIMEOUT_MICROSEC;
-	do {
-		time_left--;
-		ret = zynqmp_pm_is_mphy_tx_rx_config_ready(&is_ready);
-		if (ret)
-			return ret;
-
-		if (!is_ready)
-			break;
-
-		usleep_range(1, 5);
-	} while (time_left);
-
-	if (!time_left) {
+	/*
+	 * Wait for Tx/Rx config_rdy. The poll loop lives in the firmware
+	 * backend (EEMI today, SCMI in future) so this driver stays
+	 * backend-agnostic; the timeout budget stays here with the consumer.
+	 */
+	ret = zynqmp_pm_wait_mphy_tx_rx_config_ready(TIMEOUT_MICROSEC);
+	if (ret) {
 		dev_err(hba->dev, "Tx/Rx configuration signal busy.\n");
-		return -ETIMEDOUT;
+		return ret;
 	}
 
 	ret = ufshcd_dwc_dme_set_attrs(hba, rmmi_attrs, ARRAY_SIZE(rmmi_attrs));
@@ -264,23 +254,11 @@ static int ufs_versal2_phy_init(struct ufs_hba *hba)
 		return ret;
 	}
 
-	/* Wait for SRAM init done */
-	time_left = TIMEOUT_MICROSEC;
-	do {
-		time_left--;
-		ret = zynqmp_pm_is_sram_init_done(&is_ready);
-		if (ret)
-			return ret;
-
-		if (is_ready)
-			break;
-
-		usleep_range(1, 5);
-	} while (time_left);
-
-	if (!time_left) {
+	/* Wait for SRAM init done (poll handled by the firmware backend). */
+	ret = zynqmp_pm_wait_sram_init_done(TIMEOUT_MICROSEC);
+	if (ret) {
 		dev_err(hba->dev, "SRAM initialization failed.\n");
-		return -ETIMEDOUT;
+		return ret;
 	}
 
 	ret = ufs_versal2_setup_phy(hba);
diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c
index 814c1b7343b9e3..541bbaf92c21cb 100644
--- a/drivers/ufs/host/ufs-mediatek.c
+++ b/drivers/ufs/host/ufs-mediatek.c
@@ -10,6 +10,7 @@
 #include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
@@ -1124,9 +1125,9 @@ static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba)
 				ah_ms = ah_timer * scale_us[ah_scale] / 1000;
 		}
 
-		spin_lock_irqsave(hba->host->host_lock, flags);
+		spin_lock_irqsave(&hba->host->host_lock, flags);
 		hba->clk_gating.delay_ms = max(ah_ms, 10U);
-		spin_unlock_irqrestore(hba->host->host_lock, flags);
+		spin_unlock_irqrestore(&hba->host->host_lock, flags);
 	}
 }
 
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index 954be9343f42ac..7e6cb6dc42d1c6 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -10,6 +10,7 @@
 #include <linux/devfreq.h>
 #include <linux/gpio/consumer.h>
 #include <linux/interconnect.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/phy/phy.h>
@@ -347,7 +348,9 @@ static void ufs_qcom_disable_lane_clks(struct ufs_qcom_host *host)
 	if (!host->is_lane_clks_enabled)
 		return;
 
-	clk_bulk_disable_unprepare(host->num_clks, host->clks);
+	clk_disable_unprepare(host->rx_lane1_sync_clk);
+	clk_disable_unprepare(host->rx_lane0_sync_clk);
+	clk_disable_unprepare(host->tx_lane0_sync_clk);
 
 	host->is_lane_clks_enabled = false;
 }
@@ -356,28 +359,57 @@ static int ufs_qcom_enable_lane_clks(struct ufs_qcom_host *host)
 {
 	int err;
 
-	err = clk_bulk_prepare_enable(host->num_clks, host->clks);
+	if (host->is_lane_clks_enabled)
+		return 0;
+
+	err = clk_prepare_enable(host->tx_lane0_sync_clk);
 	if (err)
-		return err;
+		goto out;
+
+	err = clk_prepare_enable(host->rx_lane0_sync_clk);
+	if (err)
+		goto out_disable_tx_lane0;
+
+	err = clk_prepare_enable(host->rx_lane1_sync_clk);
+	if (err)
+		goto out_disable_rx_lane0;
 
 	host->is_lane_clks_enabled = true;
 
 	return 0;
+
+out_disable_rx_lane0:
+	clk_disable_unprepare(host->rx_lane0_sync_clk);
+out_disable_tx_lane0:
+	clk_disable_unprepare(host->tx_lane0_sync_clk);
+out:
+	return err;
 }
 
 static int ufs_qcom_init_lane_clks(struct ufs_qcom_host *host)
 {
-	int err;
 	struct device *dev = host->hba->dev;
 
 	if (has_acpi_companion(dev))
 		return 0;
 
-	err = devm_clk_bulk_get_all(dev, &host->clks);
-	if (err <= 0)
-		return err;
+	host->tx_lane0_sync_clk = devm_clk_get(dev, "tx_lane0_sync_clk");
+	if (IS_ERR(host->tx_lane0_sync_clk))
+		return dev_err_probe(dev, PTR_ERR(host->tx_lane0_sync_clk),
+				     "failed to get tx_lane0_sync_clk\n");
+
+	host->rx_lane0_sync_clk = devm_clk_get(dev, "rx_lane0_sync_clk");
+	if (IS_ERR(host->rx_lane0_sync_clk))
+		return dev_err_probe(dev, PTR_ERR(host->rx_lane0_sync_clk),
+				     "failed to get rx_lane0_sync_clk\n");
 
-	host->num_clks = err;
+	/* In case of single lane per direction, don't read lane1 clocks */
+	if (host->hba->lanes_per_direction > 1) {
+		host->rx_lane1_sync_clk = devm_clk_get(dev, "rx_lane1_sync_clk");
+		if (IS_ERR(host->rx_lane1_sync_clk))
+			return dev_err_probe(dev, PTR_ERR(host->rx_lane1_sync_clk),
+					     "failed to get rx_lane1_sync_clk\n");
+	}
 
 	return 0;
 }
diff --git a/drivers/ufs/host/ufs-qcom.h b/drivers/ufs/host/ufs-qcom.h
index a5ad5ce44a1921..944b2e532a0962 100644
--- a/drivers/ufs/host/ufs-qcom.h
+++ b/drivers/ufs/host/ufs-qcom.h
@@ -330,8 +330,9 @@ struct ufs_qcom_host {
 	struct phy *generic_phy;
 	struct ufs_hba *hba;
 	struct ufs_pa_layer_attr dev_req_params;
-	struct clk_bulk_data *clks;
-	u32 num_clks;
+	struct clk *tx_lane0_sync_clk;
+	struct clk *rx_lane0_sync_clk;
+	struct clk *rx_lane1_sync_clk;
 	bool is_lane_clks_enabled;
 
 	struct icc_path *icc_ddr;
diff --git a/drivers/ufs/host/ufs-sprd.c b/drivers/ufs/host/ufs-sprd.c
index a5e8c591beaddc..6862250c542e06 100644
--- a/drivers/ufs/host/ufs-sprd.c
+++ b/drivers/ufs/host/ufs-sprd.c
@@ -181,9 +181,9 @@ static int ufs_sprd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
 
 	if (status == PRE_CHANGE) {
 		if (ufshcd_is_auto_hibern8_supported(hba)) {
-			spin_lock_irqsave(hba->host->host_lock, flags);
+			spin_lock_irqsave(&hba->host->host_lock, flags);
 			ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER);
-			spin_unlock_irqrestore(hba->host->host_lock, flags);
+			spin_unlock_irqrestore(&hba->host->host_lock, flags);
 		}
 	}
 
diff --git a/drivers/ufs/host/ufshcd-pltfrm.c b/drivers/ufs/host/ufshcd-pltfrm.c
index 5ac7afe759346d..169dbc1a75fe57 100644
--- a/drivers/ufs/host/ufshcd-pltfrm.c
+++ b/drivers/ufs/host/ufshcd-pltfrm.c
@@ -206,7 +206,11 @@ static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba)
 		dev_dbg(hba->dev,
 			"%s: failed to read lanes-per-direction, ret=%d\n",
 			__func__, ret);
-		hba->lanes_per_direction = UFSHCD_DEFAULT_LANES_PER_DIRECTION;
+		/* Old R-Car S4 DTBs lack "lanes-per-direction = <1>" */
+		if (of_device_is_compatible(dev->of_node, "renesas,r8a779f0-ufs"))
+			hba->lanes_per_direction = 1;
+		else
+			hba->lanes_per_direction = UFSHCD_DEFAULT_LANES_PER_DIRECTION;
 	}
 }
 
diff --git a/drivers/usb/atm/cxacru.c b/drivers/usb/atm/cxacru.c
index 636f7886fc2601..464510a44c9e9b 100644
--- a/drivers/usb/atm/cxacru.c
+++ b/drivers/usb/atm/cxacru.c
@@ -942,7 +942,7 @@ static int cxacru_fw(struct usb_device *usb_dev, enum cxacru_fw_request fw,
 	int offd, offb;
 	const int stride = CMD_PACKET_SIZE - 8;
 
-	buf = (u8 *) __get_free_page(GFP_KERNEL);
+	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
@@ -978,7 +978,7 @@ static int cxacru_fw(struct usb_device *usb_dev, enum cxacru_fw_request fw,
 	ret = 0;
 
 cleanup:
-	free_page((unsigned long) buf);
+	kfree(buf);
 	return ret;
 }
 
@@ -993,7 +993,7 @@ static int cxacru_find_firmware(struct cxacru_data *instance,
 	sprintf(buf, "cxacru-%s.bin", phase);
 	usb_dbg(usbatm, "cxacru_find_firmware: looking for %s\n", buf);
 
-	if (request_firmware(fw_p, buf, dev)) {
+	if (request_firmware_direct(fw_p, buf, dev)) {
 		usb_dbg(usbatm, "no stage %s firmware found\n", phase);
 		return -ENOENT;
 	}
@@ -1146,13 +1146,13 @@ static int cxacru_bind(struct usbatm_data *usbatm_instance,
 
 	mutex_init(&instance->adsl_state_serialize);
 
-	instance->rcv_buf = (u8 *) __get_free_page(GFP_KERNEL);
+	instance->rcv_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!instance->rcv_buf) {
 		usb_dbg(usbatm_instance, "cxacru_bind: no memory for rcv_buf\n");
 		ret = -ENOMEM;
 		goto fail;
 	}
-	instance->snd_buf = (u8 *) __get_free_page(GFP_KERNEL);
+	instance->snd_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!instance->snd_buf) {
 		usb_dbg(usbatm_instance, "cxacru_bind: no memory for snd_buf\n");
 		ret = -ENOMEM;
@@ -1220,8 +1220,8 @@ static int cxacru_bind(struct usbatm_data *usbatm_instance,
 	return 0;
 
  fail:
-	free_page((unsigned long) instance->snd_buf);
-	free_page((unsigned long) instance->rcv_buf);
+	kfree(instance->snd_buf);
+	kfree(instance->rcv_buf);
 	usb_free_urb(instance->snd_urb);
 	usb_free_urb(instance->rcv_urb);
 	kfree(instance);
@@ -1254,8 +1254,8 @@ static void cxacru_unbind(struct usbatm_data *usbatm_instance,
 	usb_free_urb(instance->snd_urb);
 	usb_free_urb(instance->rcv_urb);
 
-	free_page((unsigned long) instance->snd_buf);
-	free_page((unsigned long) instance->rcv_buf);
+	kfree(instance->snd_buf);
+	kfree(instance->rcv_buf);
 
 	kfree(instance);
 
diff --git a/drivers/usb/atm/speedtch.c b/drivers/usb/atm/speedtch.c
index 9d28c652bcd806..000f8eb24ead3a 100644
--- a/drivers/usb/atm/speedtch.c
+++ b/drivers/usb/atm/speedtch.c
@@ -241,7 +241,7 @@ static int speedtch_upload_firmware(struct speedtch_instance_data *instance,
 
 	usb_dbg(usbatm, "%s entered\n", __func__);
 
-	buffer = (unsigned char *)__get_free_page(GFP_KERNEL);
+	buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!buffer) {
 		ret = -ENOMEM;
 		usb_dbg(usbatm, "%s: no memory for buffer!\n", __func__);
@@ -340,7 +340,7 @@ static int speedtch_upload_firmware(struct speedtch_instance_data *instance,
 	ret = 0;
 
 out_free:
-	free_page((unsigned long)buffer);
+	kfree(buffer);
 out:
 	return ret;
 }
@@ -357,15 +357,15 @@ static int speedtch_find_firmware(struct usbatm_data *usbatm, struct usb_interfa
 	sprintf(buf, "speedtch-%d.bin.%x.%02x", phase, major_revision, minor_revision);
 	usb_dbg(usbatm, "%s: looking for %s\n", __func__, buf);
 
-	if (request_firmware(fw_p, buf, dev)) {
+	if (request_firmware_direct(fw_p, buf, dev)) {
 		sprintf(buf, "speedtch-%d.bin.%x", phase, major_revision);
 		usb_dbg(usbatm, "%s: looking for %s\n", __func__, buf);
 
-		if (request_firmware(fw_p, buf, dev)) {
+		if (request_firmware_direct(fw_p, buf, dev)) {
 			sprintf(buf, "speedtch-%d.bin", phase);
 			usb_dbg(usbatm, "%s: looking for %s\n", __func__, buf);
 
-			if (request_firmware(fw_p, buf, dev)) {
+			if (request_firmware_direct(fw_p, buf, dev)) {
 				usb_err(usbatm, "%s: no stage %d firmware found!\n", __func__, phase);
 				return -ENOENT;
 			}
diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c
index 42311c1bfada1d..f4dfd013170832 100644
--- a/drivers/usb/cdns3/cdns3-gadget.c
+++ b/drivers/usb/cdns3/cdns3-gadget.c
@@ -3269,6 +3269,10 @@ static void cdns3_gadget_exit(struct cdns *cdns)
 	usb_del_gadget(&priv_dev->gadget);
 	devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
 
+	/* The works can still be queued until the IRQ is freed. */
+	cancel_work_sync(&priv_dev->pending_status_wq);
+	cancel_work_sync(&priv_dev->aligned_buf_wq);
+
 	cdns3_free_all_eps(priv_dev);
 
 	while (!list_empty(&priv_dev->aligned_buf_list)) {
@@ -3376,7 +3380,7 @@ static int cdns3_gadget_start(struct cdns *cdns)
 
 	/* allocate memory for setup packet buffer */
 	priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
-						 &priv_dev->setup_dma, GFP_DMA);
+						 &priv_dev->setup_dma, GFP_KERNEL);
 	if (!priv_dev->setup_buf) {
 		ret = -ENOMEM;
 		goto err2;
diff --git a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c
index 504bdf13ea8042..87122fb5bdb82d 100644
--- a/drivers/usb/cdns3/core.c
+++ b/drivers/usb/cdns3/core.c
@@ -449,10 +449,8 @@ int cdns_init(struct cdns *cdns)
 						IRQF_SHARED,
 						dev_name(cdns->dev), cdns);
 
-		if (ret) {
-			dev_err(cdns->dev, "couldn't register wakeup irq handler\n");
+		if (ret)
 			goto role_switch_unregister;
-		}
 	}
 
 	ret = cdns_drd_init(cdns);
diff --git a/drivers/usb/cdns3/drd.c b/drivers/usb/cdns3/drd.c
index d2bb682e4552c3..42f13757d904ab 100644
--- a/drivers/usb/cdns3/drd.c
+++ b/drivers/usb/cdns3/drd.c
@@ -509,10 +509,8 @@ int cdns_drd_init(struct cdns *cdns)
 					cdns_drd_thread_irq,
 					IRQF_SHARED,
 					dev_name(cdns->dev), cdns);
-	if (ret) {
-		dev_err(cdns->dev, "couldn't get otg_irq\n");
+	if (ret)
 		return ret;
-	}
 
 	state = readl(&cdns->otg_regs->sts);
 	if (OTGSTS_OTG_NRDY(state)) {
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index 282314eea7fc1a..edbf9df9b8f289 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -525,11 +525,15 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
 		data->supports_runtime_pm = true;
 
 	data->wakeup_irq = platform_get_irq_optional(pdev, 1);
+	if (data->wakeup_irq < 0 && data->wakeup_irq != -ENXIO) {
+		ret = data->wakeup_irq;
+		goto phy_shutdown;
+	}
 	if (data->wakeup_irq > 0) {
 		irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s:wakeup", pdata.name);
 		if (!irq_name) {
 			ret = dev_err_probe(dev, -ENOMEM, "failed to create irq_name\n");
-			goto err_clk;
+			goto phy_shutdown;
 		}
 
 		ret = devm_request_threaded_irq(dev, data->wakeup_irq,
@@ -537,7 +541,7 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
 						IRQF_ONESHOT | IRQF_NO_AUTOEN,
 						irq_name, data);
 		if (ret)
-			goto err_clk;
+			goto phy_shutdown;
 	}
 
 	ret = imx_usbmisc_init(data->usbmisc_data);
diff --git a/drivers/usb/chipidea/ci_hdrc_tegra.c b/drivers/usb/chipidea/ci_hdrc_tegra.c
index 372788f0f97014..26225b034d3b21 100644
--- a/drivers/usb/chipidea/ci_hdrc_tegra.c
+++ b/drivers/usb/chipidea/ci_hdrc_tegra.c
@@ -306,8 +306,10 @@ static int tegra_usb_probe(struct platform_device *pdev)
 
 	pm_runtime_enable(&pdev->dev);
 	err = pm_runtime_resume_and_get(&pdev->dev);
-	if (err)
+	if (err) {
+		pm_runtime_disable(&pdev->dev);
 		return err;
+	}
 
 	if (device_property_present(&pdev->dev, "nvidia,needs-double-reset"))
 		usb->needs_double_reset = true;
diff --git a/drivers/usb/chipidea/otg_fsm.c b/drivers/usb/chipidea/otg_fsm.c
index 929536dc96ecb9..642729893cc208 100644
--- a/drivers/usb/chipidea/otg_fsm.c
+++ b/drivers/usb/chipidea/otg_fsm.c
@@ -705,7 +705,7 @@ static void ci_otg_fsm_event(struct ci_hdrc *ci)
 		if ((intr_sts & USBi_PCI) && !port_conn) {
 			fsm->b_conn = 0;
 
-			/* if gadget driver is binded */
+			/* if gadget driver is bound */
 			if (ci->driver) {
 				/* A device to be peripheral mode */
 				ci->gadget.is_a_peripheral = 1;
diff --git a/drivers/usb/chipidea/udc.c b/drivers/usb/chipidea/udc.c
index d52f894898934e..c29b2d0dcc7c88 100644
--- a/drivers/usb/chipidea/udc.c
+++ b/drivers/usb/chipidea/udc.c
@@ -1862,7 +1862,7 @@ static int ci_udc_get_frame(struct usb_gadget *_gadget)
 }
 
 /*
- * ci_hdrc_gadget_connect: caller makes sure gadget driver is binded
+ * ci_hdrc_gadget_connect: caller makes sure gadget driver is bound
  */
 static void ci_hdrc_gadget_connect(struct usb_gadget *_gadget, int is_active)
 {
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index 7bc5329fa3ed39..21417b4c51541a 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -331,7 +331,6 @@ static void acm_process_notification(struct acm *acm, unsigned char *buf)
 
 		spin_lock_irqsave(&acm->read_lock, flags);
 		acm->ctrlin = newctrl;
-		acm->oldcount = acm->iocount;
 
 		if (difference & USB_CDC_SERIAL_STATE_DSR)
 			acm->iocount.dsr++;
@@ -1027,11 +1026,16 @@ static int wait_serial_change(struct acm *acm, unsigned long arg)
 	DECLARE_WAITQUEUE(wait, current);
 	struct async_icount old, new;
 
-	do {
+	spin_lock_irq(&acm->read_lock);
+	old = acm->iocount;
+	spin_unlock_irq(&acm->read_lock);
+
+	add_wait_queue(&acm->wioctl, &wait);
+	for (;;) {
+		set_current_state(TASK_INTERRUPTIBLE);
+
 		spin_lock_irq(&acm->read_lock);
-		old = acm->oldcount;
 		new = acm->iocount;
-		acm->oldcount = new;
 		spin_unlock_irq(&acm->read_lock);
 
 		if ((arg & TIOCM_DSR) &&
@@ -1044,22 +1048,20 @@ static int wait_serial_change(struct acm *acm, unsigned long arg)
 			old.rng != new.rng)
 			break;
 
-		add_wait_queue(&acm->wioctl, &wait);
-		set_current_state(TASK_INTERRUPTIBLE);
-		schedule();
-		remove_wait_queue(&acm->wioctl, &wait);
 		if (acm->disconnected) {
-			if (arg & TIOCM_CD)
-				break;
-			else
-				rv = -ENODEV;
-		} else {
-			if (signal_pending(current))
-				rv = -ERESTARTSYS;
+			rv = -ENODEV;
+			break;
 		}
-	} while (!rv);
 
-	
+		schedule();
+
+		if (signal_pending(current)) {
+			rv = -ERESTARTSYS;
+			break;
+		}
+	}
+	__set_current_state(TASK_RUNNING);
+	remove_wait_queue(&acm->wioctl, &wait);
 
 	return rv;
 }
diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h
index 01f448a783c031..d245e60eb9f50f 100644
--- a/drivers/usb/class/cdc-acm.h
+++ b/drivers/usb/class/cdc-acm.h
@@ -90,7 +90,6 @@ struct acm {
 	unsigned int ctrlin;				/* input control lines (DCD, DSR, RI, break, overruns) */
 	unsigned int ctrlout;				/* output control lines (DTR, RTS) */
 	struct async_icount iocount;			/* counters for control line changes */
-	struct async_icount oldcount;			/* for comparison of counter */
 	wait_queue_head_t wioctl;			/* for ioctl */
 	unsigned int writesize;				/* max packet size for the output bulk endpoint */
 	unsigned int readsize,ctrlsize;			/* buffer sizes for freeing */
diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c
index 7e43429e996e7a..dec4baca1c53d2 100644
--- a/drivers/usb/common/ulpi.c
+++ b/drivers/usb/common/ulpi.c
@@ -147,15 +147,17 @@ static const struct device_type ulpi_dev_type = {
  * __ulpi_register_driver - register a driver with the ULPI bus
  * @drv: driver being registered
  * @module: ends up being THIS_MODULE
+ * @mod_name: ends up being KBUILD_MODNAME
  *
  * Registers a driver with the ULPI bus.
  */
-int __ulpi_register_driver(struct ulpi_driver *drv, struct module *module)
+int __ulpi_register_driver(struct ulpi_driver *drv, struct module *module, const char *mod_name)
 {
 	if (!drv->probe)
 		return -EINVAL;
 
 	drv->driver.owner = module;
+	drv->driver.mod_name = mod_name;
 	drv->driver.bus = &ulpi_bus;
 
 	return driver_register(&drv->driver);
diff --git a/drivers/usb/common/usb-conn-gpio.c b/drivers/usb/common/usb-conn-gpio.c
index 421c3af38e0697..f6db03d89efdee 100644
--- a/drivers/usb/common/usb-conn-gpio.c
+++ b/drivers/usb/common/usb-conn-gpio.c
@@ -251,10 +251,8 @@ static int usb_conn_probe(struct platform_device *pdev)
 		ret = devm_request_threaded_irq(dev, info->id_irq, NULL,
 						usb_conn_isr, USB_CONN_IRQF,
 						pdev->name, info);
-		if (ret < 0) {
-			dev_err(dev, "failed to request ID IRQ\n");
+		if (ret < 0)
 			goto put_role_sw;
-		}
 	}
 
 	if (info->vbus_gpiod) {
@@ -268,10 +266,8 @@ static int usb_conn_probe(struct platform_device *pdev)
 		ret = devm_request_threaded_irq(dev, info->vbus_irq, NULL,
 						usb_conn_isr, USB_CONN_IRQF,
 						pdev->name, info);
-		if (ret < 0) {
-			dev_err(dev, "failed to request VBUS IRQ\n");
+		if (ret < 0)
 			goto put_role_sw;
-		}
 	}
 
 	platform_set_drvdata(pdev, info);
diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c
index 7f33fe5ba03bde..3c3bbaf89609aa 100644
--- a/drivers/usb/core/driver.c
+++ b/drivers/usb/core/driver.c
@@ -984,9 +984,10 @@ bool is_usb_device_driver(const struct device_driver *drv)
 }
 
 /**
- * usb_register_device_driver - register a USB device (not interface) driver
+ * __usb_register_device_driver - register a USB device (not interface) driver
  * @new_udriver: USB operations for the device driver
  * @owner: module owner of this driver.
+ * @mod_name: module name string
  *
  * Registers a USB device driver with the USB core.  The list of
  * unattached devices will be rescanned whenever a new driver is
@@ -994,8 +995,8 @@ bool is_usb_device_driver(const struct device_driver *drv)
  *
  * Return: A negative error code on failure and 0 on success.
  */
-int usb_register_device_driver(struct usb_device_driver *new_udriver,
-		struct module *owner)
+int __usb_register_device_driver(struct usb_device_driver *new_udriver,
+		struct module *owner, const char *mod_name)
 {
 	int retval = 0;
 
@@ -1007,6 +1008,7 @@ int usb_register_device_driver(struct usb_device_driver *new_udriver,
 	new_udriver->driver.probe = usb_probe_device;
 	new_udriver->driver.remove = usb_unbind_device;
 	new_udriver->driver.owner = owner;
+	new_udriver->driver.mod_name = mod_name;
 	new_udriver->driver.dev_groups = new_udriver->dev_groups;
 
 	retval = driver_register(&new_udriver->driver);
@@ -1027,7 +1029,7 @@ int usb_register_device_driver(struct usb_device_driver *new_udriver,
 
 	return retval;
 }
-EXPORT_SYMBOL_GPL(usb_register_device_driver);
+EXPORT_SYMBOL_GPL(__usb_register_device_driver);
 
 /**
  * usb_deregister_device_driver - unregister a USB device (not interface) driver
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index 8e8a5f59b319b3..a865a15ea38634 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -18,6 +18,7 @@
 #include <linux/completion.h>
 #include <linux/utsname.h>
 #include <linux/mm.h>
+#include <linux/highmem.h>
 #include <asm/io.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index 3345b3298daf35..199d3f535a8915 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -73,8 +73,11 @@
 /*
  * Give SS hubs 200ms time after wake to train downstream links before
  * assuming no port activity and allowing hub to runtime suspend back.
+ * Root hubs have no upstream hub whose wake propagation needs to be
+ * accounted for, so they need less time, use 120ms for them.
  */
 #define USB_SS_PORT_U0_WAKE_TIME	200  /* ms */
+#define USB_SS_RH_PORT_U0_WAKE_TIME     120 /* ms */
 
 /* Protect struct usb_device->state and ->children members
  * Note: Both are also protected by ->dev.sem, except that ->state can
@@ -1358,7 +1361,9 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type)
 
 		queue_delayed_work(system_power_efficient_wq,
 				   &hub->post_resume_work,
-				   msecs_to_jiffies(USB_SS_PORT_U0_WAKE_TIME));
+				   msecs_to_jiffies(hdev->parent ?
+				USB_SS_PORT_U0_WAKE_TIME :
+				USB_SS_RH_PORT_U0_WAKE_TIME));
 		return;
 	}
 
@@ -2411,7 +2416,7 @@ static void announce_device_ids(struct usb_device *udev)
 	u16 bcdDevice = le16_to_cpu(udev->descriptor.bcdDevice);
 
 	dev_info(&udev->dev,
-		"New USB device found, idVendor=%04x, idProduct=%04x, bcdDevice=%2x.%02x\n",
+		"New USB device found: idVendor=%04x, idProduct=%04x, bcdDevice=%x.%02x\n",
 		le16_to_cpu(udev->descriptor.idVendor),
 		le16_to_cpu(udev->descriptor.idProduct),
 		bcdDevice >> 8, bcdDevice & 0xff);
diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c
index c06b44ca507b04..19e2c21420b84c 100644
--- a/drivers/usb/core/urb.c
+++ b/drivers/usb/core/urb.c
@@ -951,8 +951,8 @@ EXPORT_SYMBOL_GPL(usb_wait_anchor_empty_timeout);
  * usb_get_from_anchor - get an anchor's oldest urb
  * @anchor: the anchor whose urb you want
  *
- * This will take the oldest urb from an anchor,
- * unanchor and return it
+ * This will take the oldest urb from an anchor, unanchor and return it with
+ * a reference held.
  *
  * Return: The oldest urb from @anchor, or %NULL if @anchor has no
  * urbs associated with it.
diff --git a/drivers/usb/core/usb.c b/drivers/usb/core/usb.c
index df166cafe10619..2559a180fdeec6 100644
--- a/drivers/usb/core/usb.c
+++ b/drivers/usb/core/usb.c
@@ -1241,7 +1241,7 @@ static int __init usb_init(void)
 	retval = usb_hub_init();
 	if (retval)
 		goto hub_init_failed;
-	retval = usb_register_device_driver(&usb_generic_driver, THIS_MODULE);
+	retval = usb_register_device_driver(&usb_generic_driver);
 	if (!retval)
 		goto out;
 
diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c
index f375c5185bfe22..6a0135e06b007b 100644
--- a/drivers/usb/dwc2/core.c
+++ b/drivers/usb/dwc2/core.c
@@ -173,7 +173,7 @@ static void dwc2_restore_essential_regs(struct dwc2_hsotg *hsotg, int rmode,
 	/* Umnask global Interrupt in GAHBCFG and restore it */
 	dwc2_writel(hsotg, gr->gahbcfg | GAHBCFG_GLBL_INTR_EN, GAHBCFG);
 
-	/* Clear all pending interupts */
+	/* Clear all pending interrupts */
 	dwc2_writel(hsotg, 0xffffffff, GINTSTS);
 
 	/* Unmask restore done interrupt */
diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h
index 767251aa1aa3f8..015a5fb7842d57 100644
--- a/drivers/usb/dwc2/core.h
+++ b/drivers/usb/dwc2/core.h
@@ -334,7 +334,7 @@ enum dwc2_ep0_state {
  *                           (default when phy_type is Full Speed)
  * @oc_disable:		Flag to disable overcurrent condition.
  *			0 - Allow overcurrent condition to get detected
- *			1 - Disable overcurrent condtion to get detected
+ *			1 - Disable overcurrent condition to get detected
  * @ts_dline:           Enable Term Select Dline pulsing
  *                       0 - No (default)
  *                       1 - Yes
diff --git a/drivers/usb/dwc2/debugfs.c b/drivers/usb/dwc2/debugfs.c
index 3116ac72747f78..2ecbf6523aaae9 100644
--- a/drivers/usb/dwc2/debugfs.c
+++ b/drivers/usb/dwc2/debugfs.c
@@ -9,6 +9,7 @@
 #include <linux/spinlock.h>
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
+#include <linux/slab.h>
 #include <linux/uaccess.h>
 
 #include "core.h"
@@ -787,8 +788,7 @@ int dwc2_debugfs_init(struct dwc2_hsotg *hsotg)
 	/* Add gadget debugfs nodes */
 	dwc2_hsotg_create_debug(hsotg);
 
-	hsotg->regset = devm_kzalloc(hsotg->dev, sizeof(*hsotg->regset),
-								GFP_KERNEL);
+	hsotg->regset = kzalloc_obj(*hsotg->regset, GFP_KERNEL);
 	if (!hsotg->regset) {
 		ret = -ENOMEM;
 		goto err;
@@ -810,4 +810,6 @@ void dwc2_debugfs_exit(struct dwc2_hsotg *hsotg)
 {
 	debugfs_remove_recursive(hsotg->debug_root);
 	hsotg->debug_root = NULL;
+	kfree(hsotg->regset);
+	hsotg->regset = NULL;
 }
diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c
index 102169b404196c..6c8a9f05fff006 100644
--- a/drivers/usb/dwc2/gadget.c
+++ b/drivers/usb/dwc2/gadget.c
@@ -658,7 +658,7 @@ static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg,
 }
 
 /**
- * get_ep_limit - get the maximum data legnth for this endpoint
+ * get_ep_limit - get the maximum data length for this endpoint
  * @hs_ep: The endpoint
  *
  * Return the maximum data that can be queued in one go on a given endpoint
@@ -1780,6 +1780,7 @@ static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg,
 	struct dwc2_hsotg_ep *ep;
 	int ret;
 	bool halted;
+	u32 otgctl;
 	u32 recip;
 	u32 wValue;
 	u32 wIndex;
@@ -1809,6 +1810,36 @@ static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg,
 
 			hsotg->test_mode = wIndex >> 8;
 			break;
+		case USB_DEVICE_B_HNP_ENABLE:
+			if (!hsotg->params.otg_caps.hnp_support)
+				return -ENOENT;
+			if (!set)
+				return -EINVAL;
+
+			otgctl = dwc2_readl(hsotg, GOTGCTL);
+			otgctl |= GOTGCTL_DEVHNPEN;
+			dwc2_writel(hsotg, otgctl, GOTGCTL);
+			hsotg->gadget.b_hnp_enable = 1;
+			dev_dbg(hsotg->dev, "HNP enabled\n");
+			break;
+		case USB_DEVICE_A_HNP_SUPPORT:
+			if (!hsotg->params.otg_caps.hnp_support)
+				return -ENOENT;
+			if (!set)
+				return -EINVAL;
+
+			hsotg->gadget.a_hnp_support = 1;
+			dev_dbg(hsotg->dev, "a_hnp_support set\n");
+			break;
+		case USB_DEVICE_A_ALT_HNP_SUPPORT:
+			if (!hsotg->params.otg_caps.hnp_support)
+				return -ENOENT;
+			if (!set)
+				return -EINVAL;
+
+			hsotg->gadget.a_alt_hnp_support = 1;
+			dev_dbg(hsotg->dev, "a_alt_hnp_support set\n");
+			break;
 		default:
 			return -ENOENT;
 		}
@@ -2267,6 +2298,7 @@ static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size)
 	int to_read;
 	int max_req;
 	int read_ptr;
+	u32 drain;
 
 	if (!hs_req) {
 		u32 epctl = dwc2_readl(hsotg, DOEPCTL(ep_idx));
@@ -2286,30 +2318,46 @@ static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size)
 	to_read = size;
 	read_ptr = hs_req->req.actual;
 	max_req = hs_req->req.length - read_ptr;
+	drain = 0;
 
 	dev_dbg(hsotg->dev, "%s: read %d/%d, done %d/%d\n",
 		__func__, to_read, max_req, read_ptr, hs_req->req.length);
 
 	if (to_read > max_req) {
 		/*
-		 * more data appeared than we where willing
-		 * to deal with in this request.
+		 * More data appeared than we were willing to deal with in
+		 * this request.  Keep only what fits in the request buffer
+		 * and discard the rest from the FIFO, instead of overwriting
+		 * bytes past the request buffer (a 64-byte packet into a
+		 * request with one byte left used to over-write 63 bytes past
+		 * its end).
 		 */
-
-		/* currently we don't deal this */
-		WARN_ON_ONCE(1);
+		dev_dbg(hsotg->dev, "%s: packet %d > request space %d, dropping %d\n",
+			__func__, to_read, max_req, to_read - max_req);
+		drain = DIV_ROUND_UP(to_read - max_req, 4);
+		to_read = max_req;
 	}
 
 	hs_ep->total_data += to_read;
 	hs_req->req.actual += to_read;
-	to_read = DIV_ROUND_UP(to_read, 4);
 
 	/*
-	 * note, we might over-write the buffer end by 3 bytes depending on
-	 * alignment of the data.
+	 * Copy word-at-a-time so the request buffer is exactly filled and
+	 * never over-run by the 4-byte rounding of the previous FIFO bulk
+	 * read (which could also emit up to 3 bytes past the buffer end).
 	 */
-	dwc2_readl_rep(hsotg, EPFIFO(ep_idx),
-		       hs_req->req.buf + read_ptr, to_read);
+	while (to_read > 0) {
+		u32 word = dwc2_readl(hsotg, EPFIFO(ep_idx));
+		unsigned int chunk = min_t(unsigned int, to_read, 4);
+
+		memcpy(hs_req->req.buf + read_ptr, &word, chunk);
+		read_ptr += chunk;
+		to_read -= chunk;
+	}
+
+	/* drain the discarded bytes so the next FIFO event is a fresh packet */
+	while (drain-- > 0)
+		(void)dwc2_readl(hsotg, EPFIFO(ep_idx));
 }
 
 /**
@@ -3322,6 +3370,13 @@ void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg)
 	hsotg->connected = 0;
 	hsotg->test_mode = 0;
 
+	if (hsotg->params.otg_caps.hnp_support) {
+		hsotg->gadget.b_hnp_enable = 0;
+		hsotg->gadget.a_hnp_support = 0;
+		hsotg->gadget.a_alt_hnp_support = 0;
+		dwc2_clear_bit(hsotg, GOTGCTL, GOTGCTL_DEVHNPEN);
+	}
+
 	/* all endpoints should be shutdown */
 	for (ep = 0; ep < hsotg->num_of_eps; ep++) {
 		if (hsotg->eps_in[ep])
@@ -3416,9 +3471,18 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
 	usbcfg &= ~GUSBCFG_TOUTCAL_MASK;
 	usbcfg |= GUSBCFG_TOUTCAL(7);
 
-	/* remove the HNP/SRP and set the PHY */
+	/*
+	 * Configure HNP/SRP capability from params (same idea as
+	 * dwc2_gusbcfg_init() for host). Unconditionally clearing these
+	 * bits leaves an HNP-capable OTG gadget unable to negotiate.
+	 */
 	usbcfg &= ~(GUSBCFG_SRPCAP | GUSBCFG_HNPCAP);
-        dwc2_writel(hsotg, usbcfg, GUSBCFG);
+	if (hsotg->params.otg_caps.hnp_support &&
+	    hsotg->params.otg_caps.srp_support)
+		usbcfg |= GUSBCFG_HNPCAP;
+	if (hsotg->params.otg_caps.srp_support)
+		usbcfg |= GUSBCFG_SRPCAP;
+	dwc2_writel(hsotg, usbcfg, GUSBCFG);
 
 	dwc2_phy_init(hsotg, true);
 
@@ -5053,10 +5117,8 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg)
 
 	ret = devm_request_irq(hsotg->dev, hsotg->irq, dwc2_hsotg_irq,
 			       IRQF_SHARED, dev_name(hsotg->dev), hsotg);
-	if (ret < 0) {
-		dev_err(dev, "cannot claim IRQ for gadget\n");
+	if (ret < 0)
 		return ret;
-	}
 
 	/* hsotg->num_of_eps holds number of EPs other than ep0 */
 
@@ -5501,7 +5563,7 @@ int dwc2_gadget_exit_hibernation(struct dwc2_hsotg *hsotg,
 	dwc2_hib_restore_common(hsotg, rem_wakeup, 0);
 
 	if (!reset) {
-		/* Clear all pending interupts */
+		/* Clear all pending interrupts */
 		dwc2_writel(hsotg, 0xffffffff, GINTSTS);
 	}
 
@@ -5550,7 +5612,7 @@ int dwc2_gadget_exit_hibernation(struct dwc2_hsotg *hsotg,
 	}
 	/* Wait for interrupts which must be cleared */
 	mdelay(2);
-	/* Clear all pending interupts */
+	/* Clear all pending interrupts */
 	dwc2_writel(hsotg, 0xffffffff, GINTSTS);
 
 	/* Restore global registers */
diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
index 2414291aa9087d..d5275cd658d4c6 100644
--- a/drivers/usb/dwc2/hcd.c
+++ b/drivers/usb/dwc2/hcd.c
@@ -5082,14 +5082,13 @@ static void dwc2_hcd_free(struct dwc2_hsotg *hsotg)
 	}
 
 	cancel_work_sync(&hsotg->phy_reset_work);
-
-	timer_delete(&hsotg->wkp_timer);
 }
 
 static void dwc2_hcd_release(struct dwc2_hsotg *hsotg)
 {
 	/* Turn off all host-specific interrupts */
 	dwc2_disable_host_interrupts(hsotg);
+	timer_shutdown_sync(&hsotg->wkp_timer);
 
 	dwc2_hcd_free(hsotg);
 }
@@ -5656,7 +5655,7 @@ int dwc2_host_exit_hibernation(struct dwc2_hsotg *hsotg, int rem_wakeup,
 	 */
 	mdelay(100);
 
-	/* Clear all pending interupts */
+	/* Clear all pending interrupts */
 	dwc2_writel(hsotg, 0xffffffff, GINTSTS);
 
 	/* De-assert Restore */
@@ -5726,7 +5725,7 @@ int dwc2_host_exit_hibernation(struct dwc2_hsotg *hsotg, int rem_wakeup,
 
 	hprt0 = dwc2_readl(hsotg, HPRT0);
 
-	/* Clear all pending interupts */
+	/* Clear all pending interrupts */
 	dwc2_writel(hsotg, 0xffffffff, GINTSTS);
 
 	/* Restore global registers */
diff --git a/drivers/usb/dwc2/hcd_ddma.c b/drivers/usb/dwc2/hcd_ddma.c
index 994a78ad084b1c..703cda5a589534 100644
--- a/drivers/usb/dwc2/hcd_ddma.c
+++ b/drivers/usb/dwc2/hcd_ddma.c
@@ -963,7 +963,7 @@ static void dwc2_complete_isoc_xfer_ddma(struct dwc2_hsotg *hsotg,
 		 * whether some of the descriptors (frames) succeeded or not.
 		 * Pass error code to completion routine as well, to update
 		 * urb->status, some of class drivers might use it to stop
-		 * queing transfer requests.
+		 * queuing transfer requests.
 		 */
 		int err = halt_status == DWC2_HC_XFER_AHB_ERR ?
 			  -EIO : -EOVERFLOW;
diff --git a/drivers/usb/dwc3/debugfs.c b/drivers/usb/dwc3/debugfs.c
index 65d909e4b78542..01ebd73198e037 100644
--- a/drivers/usb/dwc3/debugfs.c
+++ b/drivers/usb/dwc3/debugfs.c
@@ -367,14 +367,10 @@ static ssize_t dwc3_lsp_write(struct file *file, const char __user *ubuf,
 	struct seq_file		*s = file->private_data;
 	struct dwc3		*dwc = s->private;
 	unsigned long		flags;
-	char			buf[32] = { 0 };
 	u32			sel;
 	int			ret;
 
-	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	ret = kstrtouint(buf, 0, &sel);
+	ret = kstrtou32_from_user(ubuf, count, 0, &sel);
 	if (ret)
 		return ret;
 
diff --git a/drivers/usb/dwc3/dwc3-am62.c b/drivers/usb/dwc3/dwc3-am62.c
index 632634d6e81e10..d03b3950ce046a 100644
--- a/drivers/usb/dwc3/dwc3-am62.c
+++ b/drivers/usb/dwc3/dwc3-am62.c
@@ -299,6 +299,7 @@ static int dwc3_ti_probe(struct platform_device *pdev)
 
 err_pm_disable:
 	clk_disable_unprepare(am62->usb2_refclk);
+	pm_runtime_put_noidle(dev);
 	pm_runtime_disable(dev);
 	pm_runtime_set_suspended(dev);
 	return ret;
diff --git a/drivers/usb/dwc3/dwc3-google.c b/drivers/usb/dwc3/dwc3-google.c
index a01ca23cb6a8ad..ec53e658760e88 100644
--- a/drivers/usb/dwc3/dwc3-google.c
+++ b/drivers/usb/dwc3/dwc3-google.c
@@ -235,10 +235,8 @@ static int dwc3_google_request_irq(struct dwc3_google *google, struct platform_d
 					dwc3_google_resume_irq,
 					IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
 					req_name, google);
-	if (ret < 0) {
-		dev_err(google->dev, "failed to request irq %s\n", req_name);
+	if (ret < 0)
 		return ret;
-	}
 
 	return irq;
 }
diff --git a/drivers/usb/dwc3/dwc3-imx.c b/drivers/usb/dwc3/dwc3-imx.c
index 973a486b544d5e..d967acd43d44e5 100644
--- a/drivers/usb/dwc3/dwc3-imx.c
+++ b/drivers/usb/dwc3/dwc3-imx.c
@@ -250,7 +250,7 @@ static int dwc3_imx_probe(struct platform_device *pdev)
 					IRQF_ONESHOT | IRQF_NO_AUTOEN,
 					irq_name, dwc_imx);
 	if (ret)
-		return dev_err_probe(dev, ret, "failed to request IRQ #%d\n", irq);
+		return ret;
 
 	ret = device_add_software_node(dev, &dwc3_imx_swnode);
 	if (ret)
diff --git a/drivers/usb/dwc3/dwc3-imx8mp.c b/drivers/usb/dwc3/dwc3-imx8mp.c
index 1cf96540b66e4e..ee92a5efaa6e84 100644
--- a/drivers/usb/dwc3/dwc3-imx8mp.c
+++ b/drivers/usb/dwc3/dwc3-imx8mp.c
@@ -273,10 +273,8 @@ static int dwc3_imx8mp_probe(struct platform_device *pdev)
 
 	err = devm_request_threaded_irq(dev, irq, NULL, dwc3_imx8mp_interrupt,
 					IRQF_ONESHOT, dev_name(dev), dwc3_imx);
-	if (err) {
-		dev_err(dev, "failed to request IRQ #%d --> %d\n", irq, err);
+	if (err)
 		goto put_dwc3;
-	}
 
 	device_set_wakeup_capable(dev, true);
 	pm_runtime_put(dev);
diff --git a/drivers/usb/dwc3/dwc3-keystone.c b/drivers/usb/dwc3/dwc3-keystone.c
index 7ee1610162b94f..8ad302abe883c2 100644
--- a/drivers/usb/dwc3/dwc3-keystone.c
+++ b/drivers/usb/dwc3/dwc3-keystone.c
@@ -144,11 +144,8 @@ static int kdwc3_probe(struct platform_device *pdev)
 
 	error = devm_request_irq(dev, irq, dwc3_keystone_interrupt, IRQF_SHARED,
 			dev_name(dev), kdwc);
-	if (error) {
-		dev_err(dev, "failed to request IRQ #%d --> %d\n",
-				irq, error);
+	if (error)
 		goto err_irq;
-	}
 
 	kdwc3_enable_irqs(kdwc);
 
diff --git a/drivers/usb/dwc3/dwc3-omap.c b/drivers/usb/dwc3/dwc3-omap.c
index fe74d11bb629c9..3bff775fbe4e21 100644
--- a/drivers/usb/dwc3/dwc3-omap.c
+++ b/drivers/usb/dwc3/dwc3-omap.c
@@ -518,11 +518,8 @@ static int dwc3_omap_probe(struct platform_device *pdev)
 	ret = devm_request_threaded_irq(dev, omap->irq, dwc3_omap_interrupt,
 					dwc3_omap_interrupt_thread, IRQF_SHARED,
 					"dwc3-omap", omap);
-	if (ret) {
-		dev_err(dev, "failed to request IRQ #%d --> %d\n",
-			omap->irq, ret);
+	if (ret)
 		goto err2;
-	}
 	dwc3_omap_enable_irqs(omap);
 	return 0;
 
diff --git a/drivers/usb/fotg210/Kconfig b/drivers/usb/fotg210/Kconfig
index 87a16258274eed..b7f0b78be7a36b 100644
--- a/drivers/usb/fotg210/Kconfig
+++ b/drivers/usb/fotg210/Kconfig
@@ -16,6 +16,11 @@ if USB_FOTG210
 config USB_FOTG210_HCD
 	bool "Faraday FOTG210 USB Host Controller support"
 	depends on USB=y || USB=USB_FOTG210
+	depends on USB_EHCI_HCD=y || USB_EHCI_HCD=USB_FOTG210
+	select USB_EHCI_DEVIANT_PORT_STATUS_REG
+	select USB_EHCI_PORT_RESET_HOOKS
+	select USB_EHCI_PORT_SPEED_HOOK
+	select USB_EHCI_ROOT_HUB_TT
 	help
 	  Faraday FOTG210 is an OTG controller which can be configured as
 	  an USB2.0 host. It is designed to meet USB2.0 EHCI specification
diff --git a/drivers/usb/fotg210/fotg210-core.c b/drivers/usb/fotg210/fotg210-core.c
index 7fb4d4715e9f7f..7de49b7eb3aa68 100644
--- a/drivers/usb/fotg210/fotg210-core.c
+++ b/drivers/usb/fotg210/fotg210-core.c
@@ -1,37 +1,50 @@
 // SPDX-License-Identifier: GPL-2.0+
 /*
- * Central probing code for the FOTG210 dual role driver
- * We register one driver for the hardware and then we decide
- * whether to proceed with probing the host or the peripheral
- * driver.
+ * Central probing code for the FOTG210 dual-role controller.
+ *
+ * The role is selected once at probe time.  The driver does not attempt to
+ * switch between the host and peripheral blocks while it is running.
  */
-#include <linux/bitops.h>
+#include <linux/bits.h>
 #include <linux/clk.h>
+#include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/iopoll.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 #include <linux/regmap.h>
+#include <linux/reset.h>
 #include <linux/string_choices.h>
 #include <linux/usb.h>
+#include <linux/usb/hcd.h>
 #include <linux/usb/otg.h>
 
 #include "fotg210.h"
 
-/* Role Register 0x80 */
-#define FOTG210_RR			0x80
-#define FOTG210_RR_ID			BIT(21) /* 1 = B-device, 0 = A-device */
-#define FOTG210_RR_CROLE		BIT(20) /* 1 = device, 0 = host */
+/* OTG control/status register. */
+#define FOTG210_OTGCSR			0x80
+#define FOTG210_OTGCSR_ID		BIT(21)
+#define FOTG210_OTGCSR_CROLE		BIT(20)
+#define FOTG210_OTGCSR_A_VBUS_VLD	BIT(19)
+#define FOTG210_OTGCSR_A_BUS_DROP	BIT(5)
+#define FOTG210_OTGCSR_A_BUS_REQ	BIT(4)
+#define FOTG210_OTGCSR_B_HNP_EN		BIT(1)
 
-/*
- * Gemini-specific initialization function, only executed on the
- * Gemini SoC using the global misc control register.
- *
- * The gemini USB blocks are connected to either Mini-A (host mode) or
- * Mini-B (peripheral mode) plugs. There is no role switch support on the
- * Gemini SoC, just either-or.
- */
+/* OTG interrupt status and enable registers. */
+#define FOTG210_OTGISR			0x84
+#define FOTG210_OTGIEN			0x88
+
+/* Global interrupt mask register; set bits mask the corresponding source. */
+#define FOTG210_GINTM			0xc4
+#define FOTG210_GINTM_INT_POLARITY	BIT(3)
+#define FOTG210_GINTM_MHC_INT		BIT(2)
+#define FOTG210_GINTM_MOTG_INT		BIT(1)
+#define FOTG210_GINTM_MDEV_INT		BIT(0)
+
+/* Gemini global miscellaneous-control register. */
 #define GEMINI_GLOBAL_MISC_CTRL		0x30
 #define GEMINI_MISC_USB0_WAKEUP		BIT(14)
 #define GEMINI_MISC_USB1_WAKEUP		BIT(15)
@@ -40,94 +53,133 @@
 #define GEMINI_MISC_USB0_MINI_B		BIT(29)
 #define GEMINI_MISC_USB1_MINI_B		BIT(30)
 
-static int fotg210_gemini_init(struct fotg210 *fotg, struct resource *res,
-			       enum usb_dr_mode mode)
+static int fotg210_gemini_init(struct fotg210 *fotg)
 {
 	struct device *dev = fotg->dev;
 	struct device_node *np = dev->of_node;
-	struct regmap *map;
-	bool wakeup;
+	bool wakeup = of_property_read_bool(np, "wakeup-source");
 	u32 mask, val;
 	int ret;
 
-	map = syscon_regmap_lookup_by_phandle(np, "syscon");
-	if (IS_ERR(map))
-		return dev_err_probe(dev, PTR_ERR(map), "no syscon\n");
-	fotg->map = map;
-	wakeup = of_property_read_bool(np, "wakeup-source");
-
-	/*
-	 * Figure out if this is USB0 or USB1 by simply checking the
-	 * physical base address.
-	 */
-	mask = 0;
-	if (res->start == 0x69000000) {
+	fotg->map = syscon_regmap_lookup_by_phandle(np, "syscon");
+	if (IS_ERR(fotg->map))
+		return dev_err_probe(dev, PTR_ERR(fotg->map), "no syscon\n");
+
+	if (fotg->res->start == 0x69000000) {
 		fotg->port = GEMINI_PORT_1;
 		mask = GEMINI_MISC_USB1_VBUS_ON | GEMINI_MISC_USB1_MINI_B |
-			GEMINI_MISC_USB1_WAKEUP;
-		if (mode == USB_DR_MODE_HOST)
-			val = GEMINI_MISC_USB1_VBUS_ON;
-		else
-			val = GEMINI_MISC_USB1_MINI_B;
+		       GEMINI_MISC_USB1_WAKEUP;
+		val = fotg->mode == USB_DR_MODE_HOST ?
+		      GEMINI_MISC_USB1_VBUS_ON : GEMINI_MISC_USB1_MINI_B;
 		if (wakeup)
 			val |= GEMINI_MISC_USB1_WAKEUP;
 	} else {
 		fotg->port = GEMINI_PORT_0;
 		mask = GEMINI_MISC_USB0_VBUS_ON | GEMINI_MISC_USB0_MINI_B |
-			GEMINI_MISC_USB0_WAKEUP;
-		if (mode == USB_DR_MODE_HOST)
-			val = GEMINI_MISC_USB0_VBUS_ON;
-		else
-			val = GEMINI_MISC_USB0_MINI_B;
+		       GEMINI_MISC_USB0_WAKEUP;
+		val = fotg->mode == USB_DR_MODE_HOST ?
+		      GEMINI_MISC_USB0_VBUS_ON : GEMINI_MISC_USB0_MINI_B;
 		if (wakeup)
 			val |= GEMINI_MISC_USB0_WAKEUP;
 	}
 
-	ret = regmap_update_bits(map, GEMINI_GLOBAL_MISC_CTRL, mask, val);
-	if (ret) {
-		dev_err(dev, "failed to initialize Gemini PHY\n");
-		return ret;
-	}
+	ret = regmap_update_bits(fotg->map, GEMINI_GLOBAL_MISC_CTRL,
+				 mask, val);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to initialize Gemini PHY\n");
 
 	dev_info(dev, "initialized Gemini PHY in %s mode\n",
-		 (mode == USB_DR_MODE_HOST) ? "host" : "gadget");
+		 fotg->mode == USB_DR_MODE_HOST ? "host" : "gadget");
 	return 0;
 }
 
 /**
- * fotg210_vbus() - Called by gadget driver to enable/disable VBUS
- * @fotg: pointer to a private fotg210 object
- * @enable: true to enable VBUS, false to disable VBUS
+ * fotg210_vbus() - enable or disable the A-device VBUS supply
+ * @fotg: controller state
+ * @enable: whether to drive VBUS
  */
 void fotg210_vbus(struct fotg210 *fotg, bool enable)
 {
-	u32 mask;
+	u32 mask = 0;
 	u32 val;
 	int ret;
 
+	val = readl(fotg->base + FOTG210_OTGCSR);
+	if (enable) {
+		val &= ~FOTG210_OTGCSR_A_BUS_DROP;
+		val |= FOTG210_OTGCSR_A_BUS_REQ;
+	} else {
+		val &= ~FOTG210_OTGCSR_A_BUS_REQ;
+		val |= FOTG210_OTGCSR_A_BUS_DROP;
+	}
+	writel(val, fotg->base + FOTG210_OTGCSR);
+
 	switch (fotg->port) {
 	case GEMINI_PORT_0:
 		mask = GEMINI_MISC_USB0_VBUS_ON;
-		val = enable ? GEMINI_MISC_USB0_VBUS_ON : 0;
 		break;
 	case GEMINI_PORT_1:
 		mask = GEMINI_MISC_USB1_VBUS_ON;
-		val = enable ? GEMINI_MISC_USB1_VBUS_ON : 0;
 		break;
-	default:
-		return;
+	case GEMINI_PORT_NONE:
+		break;
+	}
+
+	if (mask) {
+		ret = regmap_update_bits(fotg->map, GEMINI_GLOBAL_MISC_CTRL,
+					 mask, enable ? mask : 0);
+		if (ret) {
+			dev_err(fotg->dev, "failed to %s VBUS\n",
+				str_enable_disable(enable));
+			return;
+		}
 	}
-	ret = regmap_update_bits(fotg->map, GEMINI_GLOBAL_MISC_CTRL, mask, val);
+
+	ret = readl_poll_timeout(fotg->base + FOTG210_OTGCSR, val,
+				 enable == !!(val & FOTG210_OTGCSR_A_VBUS_VLD),
+				 1000, 500000);
 	if (ret)
-		dev_err(fotg->dev, "failed to %s VBUS\n",
-			str_enable_disable(enable));
-	dev_info(fotg->dev, "%s: %s VBUS\n", __func__, str_enable_disable(enable));
+		dev_warn(fotg->dev, "timeout waiting for VBUS to %s\n",
+			 str_enable_disable(enable));
+}
+
+static void fotg210_init_host(struct fotg210 *fotg)
+{
+	u32 val;
+
+	/* This driver keeps a fixed A-host role and does not negotiate HNP. */
+	val = readl(fotg->base + FOTG210_OTGCSR);
+	val &= ~FOTG210_OTGCSR_B_HNP_EN;
+	writel(val, fotg->base + FOTG210_OTGCSR);
+
+	/* Mask peripheral and OTG sources, enable the host source, active high. */
+	writel(FOTG210_GINTM_MDEV_INT | FOTG210_GINTM_MOTG_INT |
+	       FOTG210_GINTM_INT_POLARITY, fotg->base + FOTG210_GINTM);
+	writel(0, fotg->base + FOTG210_OTGIEN);
+	writel(readl(fotg->base + FOTG210_OTGISR),
+	       fotg->base + FOTG210_OTGISR);
+
+	/* Vendor trees cycle the A bus before starting the host controller. */
+	fotg210_vbus(fotg, false);
+	usleep_range(10000, 12000);
+	fotg210_vbus(fotg, true);
+	usleep_range(10000, 12000);
+}
+
+static void fotg210_init_peripheral(struct fotg210 *fotg)
+{
+	/* Mask host and OTG sources, enable the device source, active high. */
+	writel(FOTG210_GINTM_MHC_INT | FOTG210_GINTM_MOTG_INT |
+	       FOTG210_GINTM_INT_POLARITY, fotg->base + FOTG210_GINTM);
+	writel(0, fotg->base + FOTG210_OTGIEN);
+	writel(readl(fotg->base + FOTG210_OTGISR),
+	       fotg->base + FOTG210_OTGISR);
 }
 
 static int fotg210_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	enum usb_dr_mode mode;
+	struct reset_control *rst;
 	struct fotg210 *fotg;
 	u32 val;
 	int ret;
@@ -141,62 +193,111 @@ static int fotg210_probe(struct platform_device *pdev)
 	if (IS_ERR(fotg->base))
 		return PTR_ERR(fotg->base);
 
-	fotg->pclk = devm_clk_get_optional_enabled(dev, "PCLK");
+	fotg->pclk = devm_clk_get_enabled(dev, "PCLK");
 	if (IS_ERR(fotg->pclk))
-		return PTR_ERR(fotg->pclk);
+		return dev_err_probe(dev, PTR_ERR(fotg->pclk),
+				     "failed to enable PCLK\n");
 
-	mode = usb_get_dr_mode(dev);
+	rst = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(rst))
+		return dev_err_probe(dev, PTR_ERR(rst),
+				     "failed to get reset\n");
+	ret = reset_control_reset(rst);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to reset controller\n");
+
+	val = readl(fotg->base + FOTG210_OTGCSR);
+	fotg->mode = usb_get_dr_mode(dev);
+	/* Keep the historical fixed-host default for old device trees. */
+	if (fotg->mode == USB_DR_MODE_UNKNOWN)
+		fotg->mode = USB_DR_MODE_HOST;
+	if (fotg->mode != USB_DR_MODE_HOST &&
+	    fotg->mode != USB_DR_MODE_PERIPHERAL)
+		return dev_err_probe(dev, -EINVAL,
+				     "dr_mode must select host or peripheral\n");
 
 	if (of_device_is_compatible(dev->of_node, "cortina,gemini-usb")) {
-		ret = fotg210_gemini_init(fotg, fotg->res, mode);
+		fotg->is_fotg210 = true;
+		ret = fotg210_gemini_init(fotg);
 		if (ret)
 			return ret;
+	} else if (of_device_is_compatible(dev->of_node, "faraday,fotg210")) {
+		fotg->is_fotg210 = true;
 	}
 
-	val = readl(fotg->base + FOTG210_RR);
-	if (mode == USB_DR_MODE_PERIPHERAL) {
-		if (!(val & FOTG210_RR_CROLE))
-			dev_err(dev, "block not in device role\n");
-		ret = fotg210_udc_probe(pdev, fotg);
-	} else {
-		if (val & FOTG210_RR_CROLE)
-			dev_err(dev, "block not in host role\n");
-		ret = fotg210_hcd_probe(pdev, fotg);
+	if (fotg->mode == USB_DR_MODE_PERIPHERAL) {
+		if (!(val & FOTG210_OTGCSR_CROLE))
+			dev_warn(dev, "controller does not report device role\n");
+		if (!(val & FOTG210_OTGCSR_ID))
+			dev_warn(dev, "controller does not report B-device role\n");
+		fotg210_init_peripheral(fotg);
+		return fotg210_udc_probe(pdev, fotg);
 	}
 
+	if (val & FOTG210_OTGCSR_CROLE)
+		dev_warn(dev, "controller does not report host role\n");
+	if (val & FOTG210_OTGCSR_ID)
+		dev_warn(dev, "controller does not report A-device role\n");
+	fotg210_init_host(fotg);
+	ret = fotg210_hcd_probe(pdev, fotg);
+	if (ret)
+		fotg210_vbus(fotg, false);
+
 	return ret;
 }
 
 static void fotg210_remove(struct platform_device *pdev)
 {
-	struct device *dev = &pdev->dev;
-	enum usb_dr_mode mode;
-
-	mode = usb_get_dr_mode(dev);
-
-	if (mode == USB_DR_MODE_PERIPHERAL)
+	/* The subdrivers own drvdata, so recover the fixed role from firmware. */
+	if (usb_get_dr_mode(&pdev->dev) == USB_DR_MODE_PERIPHERAL)
 		fotg210_udc_remove(pdev);
 	else
 		fotg210_hcd_remove(pdev);
 }
 
-#ifdef CONFIG_OF
+static void fotg210_shutdown(struct platform_device *pdev)
+{
+	if (IS_ENABLED(CONFIG_USB_FOTG210_HCD) &&
+	    usb_get_dr_mode(&pdev->dev) != USB_DR_MODE_PERIPHERAL)
+		usb_hcd_platform_shutdown(pdev);
+}
+
+static int fotg210_suspend(struct device *dev)
+{
+	if (usb_get_dr_mode(dev) == USB_DR_MODE_PERIPHERAL)
+		return 0;
+
+	return fotg210_hcd_suspend(dev);
+}
+
+static int fotg210_resume(struct device *dev)
+{
+	if (usb_get_dr_mode(dev) == USB_DR_MODE_PERIPHERAL)
+		return 0;
+
+	return fotg210_hcd_resume(dev);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(fotg210_pm_ops, fotg210_suspend,
+				fotg210_resume);
+
 static const struct of_device_id fotg210_of_match[] = {
 	{ .compatible = "faraday,fotg200" },
 	{ .compatible = "faraday,fotg210" },
-	/* TODO: can we also handle FUSB220? */
 	{},
 };
 MODULE_DEVICE_TABLE(of, fotg210_of_match);
-#endif
 
 static struct platform_driver fotg210_driver = {
 	.driver = {
-		.name   = "fotg210",
-		.of_match_table = of_match_ptr(fotg210_of_match),
+		.name = "fotg210",
+		.of_match_table = fotg210_of_match,
+		.pm = pm_sleep_ptr(&fotg210_pm_ops),
 	},
-	.probe  = fotg210_probe,
+	.probe = fotg210_probe,
 	.remove = fotg210_remove,
+	.shutdown = fotg210_shutdown,
 };
 
 static int __init fotg210_init(void)
diff --git a/drivers/usb/fotg210/fotg210-hcd.c b/drivers/usb/fotg210/fotg210-hcd.c
index 956be5b5651027..0a36820f189be8 100644
--- a/drivers/usb/fotg210/fotg210-hcd.c
+++ b/drivers/usb/fotg210/fotg210-hcd.c
@@ -1,5547 +1,127 @@
 // SPDX-License-Identifier: GPL-2.0+
-/* Faraday FOTG210 EHCI-like driver
- *
- * Copyright (c) 2013 Faraday Technology Corporation
- *
- * Author: Yuan-Hsin Chen <yhchen@faraday-tech.com>
- *	   Feng-Hsin Chiang <john453@faraday-tech.com>
- *	   Po-Yu Chuang <ratbert.chuang@gmail.com>
- *
- * Most of code borrowed from the Linux-3.7 EHCI driver
- */
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/device.h>
-#include <linux/dmapool.h>
-#include <linux/kernel.h>
+/* Faraday FOTG210 EHCI driver */
+#include <linux/bits.h>
 #include <linux/delay.h>
-#include <linux/ioport.h>
-#include <linux/sched.h>
-#include <linux/vmalloc.h>
+#include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/init.h>
-#include <linux/hrtimer.h>
-#include <linux/list.h>
 #include <linux/interrupt.h>
-#include <linux/usb.h>
-#include <linux/usb/hcd.h>
-#include <linux/moduleparam.h>
-#include <linux/dma-mapping.h>
-#include <linux/debugfs.h>
-#include <linux/slab.h>
-#include <linux/uaccess.h>
-#include <linux/platform_device.h>
 #include <linux/io.h>
-#include <linux/iopoll.h>
-
-#include <asm/byteorder.h>
-#include <asm/irq.h>
-#include <linux/unaligned.h>
-
-#include "fotg210.h"
-
-static const char hcd_name[] = "fotg210_hcd";
-
-#undef FOTG210_URB_TRACE
-#define FOTG210_STATS
-
-/* magic numbers that can affect system performance */
-#define FOTG210_TUNE_CERR	3 /* 0-3 qtd retries; 0 == don't stop */
-#define FOTG210_TUNE_RL_HS	4 /* nak throttle; see 4.9 */
-#define FOTG210_TUNE_RL_TT	0
-#define FOTG210_TUNE_MULT_HS	1 /* 1-3 transactions/uframe; 4.10.3 */
-#define FOTG210_TUNE_MULT_TT	1
-
-/* Some drivers think it's safe to schedule isochronous transfers more than 256
- * ms into the future (partly as a result of an old bug in the scheduling
- * code).  In an attempt to avoid trouble, we will use a minimum scheduling
- * length of 512 frames instead of 256.
- */
-#define FOTG210_TUNE_FLS 1 /* (medium) 512-frame schedule */
-
-/* Initial IRQ latency:  faster than hw default */
-static int log2_irq_thresh; /* 0 to 6 */
-module_param(log2_irq_thresh, int, S_IRUGO);
-MODULE_PARM_DESC(log2_irq_thresh, "log2 IRQ latency, 1-64 microframes");
-
-/* initial park setting:  slower than hw default */
-static unsigned park;
-module_param(park, uint, S_IRUGO);
-MODULE_PARM_DESC(park, "park setting; 1-3 back-to-back async packets");
-
-/* for link power management(LPM) feature */
-static unsigned int hird;
-module_param(hird, int, S_IRUGO);
-MODULE_PARM_DESC(hird, "host initiated resume duration, +1 for each 75us");
-
-#define INTR_MASK (STS_IAA | STS_FATAL | STS_PCD | STS_ERR | STS_INT)
-
-#include "fotg210-hcd.h"
-
-#define fotg210_dbg(fotg210, fmt, args...) \
-	dev_dbg(fotg210_to_hcd(fotg210)->self.controller, fmt, ## args)
-#define fotg210_err(fotg210, fmt, args...) \
-	dev_err(fotg210_to_hcd(fotg210)->self.controller, fmt, ## args)
-#define fotg210_info(fotg210, fmt, args...) \
-	dev_info(fotg210_to_hcd(fotg210)->self.controller, fmt, ## args)
-#define fotg210_warn(fotg210, fmt, args...) \
-	dev_warn(fotg210_to_hcd(fotg210)->self.controller, fmt, ## args)
-
-/* check the values in the HCSPARAMS register (host controller _Structural_
- * parameters) see EHCI spec, Table 2-4 for each value
- */
-static void dbg_hcs_params(struct fotg210_hcd *fotg210, char *label)
-{
-	u32 params = fotg210_readl(fotg210, &fotg210->caps->hcs_params);
-
-	fotg210_dbg(fotg210, "%s hcs_params 0x%x ports=%d\n", label, params,
-			HCS_N_PORTS(params));
-}
-
-/* check the values in the HCCPARAMS register (host controller _Capability_
- * parameters) see EHCI Spec, Table 2-5 for each value
- */
-static void dbg_hcc_params(struct fotg210_hcd *fotg210, char *label)
-{
-	u32 params = fotg210_readl(fotg210, &fotg210->caps->hcc_params);
-
-	fotg210_dbg(fotg210, "%s hcc_params %04x uframes %s%s\n", label,
-			params,
-			HCC_PGM_FRAMELISTLEN(params) ? "256/512/1024" : "1024",
-			HCC_CANPARK(params) ? " park" : "");
-}
-
-static void __maybe_unused
-dbg_qtd(const char *label, struct fotg210_hcd *fotg210, struct fotg210_qtd *qtd)
-{
-	fotg210_dbg(fotg210, "%s td %p n%08x %08x t%08x p0=%08x\n", label, qtd,
-			hc32_to_cpup(fotg210, &qtd->hw_next),
-			hc32_to_cpup(fotg210, &qtd->hw_alt_next),
-			hc32_to_cpup(fotg210, &qtd->hw_token),
-			hc32_to_cpup(fotg210, &qtd->hw_buf[0]));
-	if (qtd->hw_buf[1])
-		fotg210_dbg(fotg210, "  p1=%08x p2=%08x p3=%08x p4=%08x\n",
-				hc32_to_cpup(fotg210, &qtd->hw_buf[1]),
-				hc32_to_cpup(fotg210, &qtd->hw_buf[2]),
-				hc32_to_cpup(fotg210, &qtd->hw_buf[3]),
-				hc32_to_cpup(fotg210, &qtd->hw_buf[4]));
-}
-
-static void __maybe_unused
-dbg_qh(const char *label, struct fotg210_hcd *fotg210, struct fotg210_qh *qh)
-{
-	struct fotg210_qh_hw *hw = qh->hw;
-
-	fotg210_dbg(fotg210, "%s qh %p n%08x info %x %x qtd %x\n", label, qh,
-			hw->hw_next, hw->hw_info1, hw->hw_info2,
-			hw->hw_current);
-
-	dbg_qtd("overlay", fotg210, (struct fotg210_qtd *) &hw->hw_qtd_next);
-}
-
-static void __maybe_unused
-dbg_itd(const char *label, struct fotg210_hcd *fotg210, struct fotg210_itd *itd)
-{
-	fotg210_dbg(fotg210, "%s[%d] itd %p, next %08x, urb %p\n", label,
-			itd->frame, itd, hc32_to_cpu(fotg210, itd->hw_next),
-			itd->urb);
-
-	fotg210_dbg(fotg210,
-			"  trans: %08x %08x %08x %08x %08x %08x %08x %08x\n",
-			hc32_to_cpu(fotg210, itd->hw_transaction[0]),
-			hc32_to_cpu(fotg210, itd->hw_transaction[1]),
-			hc32_to_cpu(fotg210, itd->hw_transaction[2]),
-			hc32_to_cpu(fotg210, itd->hw_transaction[3]),
-			hc32_to_cpu(fotg210, itd->hw_transaction[4]),
-			hc32_to_cpu(fotg210, itd->hw_transaction[5]),
-			hc32_to_cpu(fotg210, itd->hw_transaction[6]),
-			hc32_to_cpu(fotg210, itd->hw_transaction[7]));
-
-	fotg210_dbg(fotg210,
-			"  buf:   %08x %08x %08x %08x %08x %08x %08x\n",
-			hc32_to_cpu(fotg210, itd->hw_bufp[0]),
-			hc32_to_cpu(fotg210, itd->hw_bufp[1]),
-			hc32_to_cpu(fotg210, itd->hw_bufp[2]),
-			hc32_to_cpu(fotg210, itd->hw_bufp[3]),
-			hc32_to_cpu(fotg210, itd->hw_bufp[4]),
-			hc32_to_cpu(fotg210, itd->hw_bufp[5]),
-			hc32_to_cpu(fotg210, itd->hw_bufp[6]));
-
-	fotg210_dbg(fotg210, "  index: %d %d %d %d %d %d %d %d\n",
-			itd->index[0], itd->index[1], itd->index[2],
-			itd->index[3], itd->index[4], itd->index[5],
-			itd->index[6], itd->index[7]);
-}
-
-static int __maybe_unused
-dbg_status_buf(char *buf, unsigned len, const char *label, u32 status)
-{
-	return scnprintf(buf, len, "%s%sstatus %04x%s%s%s%s%s%s%s%s%s%s",
-			label, label[0] ? " " : "", status,
-			(status & STS_ASS) ? " Async" : "",
-			(status & STS_PSS) ? " Periodic" : "",
-			(status & STS_RECL) ? " Recl" : "",
-			(status & STS_HALT) ? " Halt" : "",
-			(status & STS_IAA) ? " IAA" : "",
-			(status & STS_FATAL) ? " FATAL" : "",
-			(status & STS_FLR) ? " FLR" : "",
-			(status & STS_PCD) ? " PCD" : "",
-			(status & STS_ERR) ? " ERR" : "",
-			(status & STS_INT) ? " INT" : "");
-}
-
-static int __maybe_unused
-dbg_intr_buf(char *buf, unsigned len, const char *label, u32 enable)
-{
-	return scnprintf(buf, len, "%s%sintrenable %02x%s%s%s%s%s%s",
-			label, label[0] ? " " : "", enable,
-			(enable & STS_IAA) ? " IAA" : "",
-			(enable & STS_FATAL) ? " FATAL" : "",
-			(enable & STS_FLR) ? " FLR" : "",
-			(enable & STS_PCD) ? " PCD" : "",
-			(enable & STS_ERR) ? " ERR" : "",
-			(enable & STS_INT) ? " INT" : "");
-}
-
-static const char *const fls_strings[] = { "1024", "512", "256", "??" };
-
-static int dbg_command_buf(char *buf, unsigned len, const char *label,
-		u32 command)
-{
-	return scnprintf(buf, len,
-			"%s%scommand %07x %s=%d ithresh=%d%s%s%s period=%s%s %s",
-			label, label[0] ? " " : "", command,
-			(command & CMD_PARK) ? " park" : "(park)",
-			CMD_PARK_CNT(command),
-			(command >> 16) & 0x3f,
-			(command & CMD_IAAD) ? " IAAD" : "",
-			(command & CMD_ASE) ? " Async" : "",
-			(command & CMD_PSE) ? " Periodic" : "",
-			fls_strings[(command >> 2) & 0x3],
-			(command & CMD_RESET) ? " Reset" : "",
-			(command & CMD_RUN) ? "RUN" : "HALT");
-}
-
-static char *dbg_port_buf(char *buf, unsigned len, const char *label, int port,
-		u32 status)
-{
-	char *sig;
-
-	/* signaling state */
-	switch (status & (3 << 10)) {
-	case 0 << 10:
-		sig = "se0";
-		break;
-	case 1 << 10:
-		sig = "k";
-		break; /* low speed */
-	case 2 << 10:
-		sig = "j";
-		break;
-	default:
-		sig = "?";
-		break;
-	}
-
-	scnprintf(buf, len, "%s%sport:%d status %06x %d sig=%s%s%s%s%s%s%s%s",
-			label, label[0] ? " " : "", port, status,
-			status >> 25, /*device address */
-			sig,
-			(status & PORT_RESET) ? " RESET" : "",
-			(status & PORT_SUSPEND) ? " SUSPEND" : "",
-			(status & PORT_RESUME) ? " RESUME" : "",
-			(status & PORT_PEC) ? " PEC" : "",
-			(status & PORT_PE) ? " PE" : "",
-			(status & PORT_CSC) ? " CSC" : "",
-			(status & PORT_CONNECT) ? " CONNECT" : "");
-
-	return buf;
-}
-
-/* functions have the "wrong" filename when they're output... */
-#define dbg_status(fotg210, label, status) {			\
-	char _buf[80];						\
-	dbg_status_buf(_buf, sizeof(_buf), label, status);	\
-	fotg210_dbg(fotg210, "%s\n", _buf);			\
-}
-
-#define dbg_cmd(fotg210, label, command) {			\
-	char _buf[80];						\
-	dbg_command_buf(_buf, sizeof(_buf), label, command);	\
-	fotg210_dbg(fotg210, "%s\n", _buf);			\
-}
-
-#define dbg_port(fotg210, label, port, status) {			       \
-	char _buf[80];							       \
-	fotg210_dbg(fotg210, "%s\n",					       \
-			dbg_port_buf(_buf, sizeof(_buf), label, port, status));\
-}
-
-/* troubleshooting help: expose state in debugfs */
-static int debug_async_open(struct inode *, struct file *);
-static int debug_periodic_open(struct inode *, struct file *);
-static int debug_registers_open(struct inode *, struct file *);
-static int debug_async_open(struct inode *, struct file *);
-
-static ssize_t debug_output(struct file*, char __user*, size_t, loff_t*);
-static int debug_close(struct inode *, struct file *);
-
-static const struct file_operations debug_async_fops = {
-	.owner		= THIS_MODULE,
-	.open		= debug_async_open,
-	.read		= debug_output,
-	.release	= debug_close,
-	.llseek		= default_llseek,
-};
-static const struct file_operations debug_periodic_fops = {
-	.owner		= THIS_MODULE,
-	.open		= debug_periodic_open,
-	.read		= debug_output,
-	.release	= debug_close,
-	.llseek		= default_llseek,
-};
-static const struct file_operations debug_registers_fops = {
-	.owner		= THIS_MODULE,
-	.open		= debug_registers_open,
-	.read		= debug_output,
-	.release	= debug_close,
-	.llseek		= default_llseek,
-};
-
-static struct dentry *fotg210_debug_root;
-
-struct debug_buffer {
-	ssize_t (*fill_func)(struct debug_buffer *);	/* fill method */
-	struct usb_bus *bus;
-	struct mutex mutex;	/* protect filling of buffer */
-	size_t count;		/* number of characters filled into buffer */
-	char *output_buf;
-	size_t alloc_size;
-};
-
-static inline char speed_char(u32 scratch)
-{
-	switch (scratch & (3 << 12)) {
-	case QH_FULL_SPEED:
-		return 'f';
-
-	case QH_LOW_SPEED:
-		return 'l';
-
-	case QH_HIGH_SPEED:
-		return 'h';
-
-	default:
-		return '?';
-	}
-}
-
-static inline char token_mark(struct fotg210_hcd *fotg210, __hc32 token)
-{
-	__u32 v = hc32_to_cpu(fotg210, token);
-
-	if (v & QTD_STS_ACTIVE)
-		return '*';
-	if (v & QTD_STS_HALT)
-		return '-';
-	if (!IS_SHORT_READ(v))
-		return ' ';
-	/* tries to advance through hw_alt_next */
-	return '/';
-}
-
-static void qh_lines(struct fotg210_hcd *fotg210, struct fotg210_qh *qh,
-		char **nextp, unsigned *sizep)
-{
-	u32 scratch;
-	u32 hw_curr;
-	struct fotg210_qtd *td;
-	unsigned temp;
-	unsigned size = *sizep;
-	char *next = *nextp;
-	char mark;
-	__le32 list_end = FOTG210_LIST_END(fotg210);
-	struct fotg210_qh_hw *hw = qh->hw;
-
-	if (hw->hw_qtd_next == list_end) /* NEC does this */
-		mark = '@';
-	else
-		mark = token_mark(fotg210, hw->hw_token);
-	if (mark == '/') { /* qh_alt_next controls qh advance? */
-		if ((hw->hw_alt_next & QTD_MASK(fotg210)) ==
-		    fotg210->async->hw->hw_alt_next)
-			mark = '#'; /* blocked */
-		else if (hw->hw_alt_next == list_end)
-			mark = '.'; /* use hw_qtd_next */
-		/* else alt_next points to some other qtd */
-	}
-	scratch = hc32_to_cpup(fotg210, &hw->hw_info1);
-	hw_curr = (mark == '*') ? hc32_to_cpup(fotg210, &hw->hw_current) : 0;
-	temp = scnprintf(next, size,
-			"qh/%p dev%d %cs ep%d %08x %08x(%08x%c %s nak%d)",
-			qh, scratch & 0x007f,
-			speed_char(scratch),
-			(scratch >> 8) & 0x000f,
-			scratch, hc32_to_cpup(fotg210, &hw->hw_info2),
-			hc32_to_cpup(fotg210, &hw->hw_token), mark,
-			(cpu_to_hc32(fotg210, QTD_TOGGLE) & hw->hw_token)
-				? "data1" : "data0",
-			(hc32_to_cpup(fotg210, &hw->hw_alt_next) >> 1) & 0x0f);
-	size -= temp;
-	next += temp;
-
-	/* hc may be modifying the list as we read it ... */
-	list_for_each_entry(td, &qh->qtd_list, qtd_list) {
-		scratch = hc32_to_cpup(fotg210, &td->hw_token);
-		mark = ' ';
-		if (hw_curr == td->qtd_dma)
-			mark = '*';
-		else if (hw->hw_qtd_next == cpu_to_hc32(fotg210, td->qtd_dma))
-			mark = '+';
-		else if (QTD_LENGTH(scratch)) {
-			if (td->hw_alt_next == fotg210->async->hw->hw_alt_next)
-				mark = '#';
-			else if (td->hw_alt_next != list_end)
-				mark = '/';
-		}
-		temp = scnprintf(next, size,
-				 "\n\t%p%c%s len=%d %08x urb %p",
-				 td, mark, ({ char *tmp;
-				switch ((scratch>>8)&0x03) {
-				case 0:
-					tmp = "out";
-					break;
-				case 1:
-					tmp = "in";
-					break;
-				case 2:
-					tmp = "setup";
-					break;
-				default:
-					tmp = "?";
-					break;
-				 } tmp; }),
-				(scratch >> 16) & 0x7fff,
-				scratch,
-				td->urb);
-		size -= temp;
-		next += temp;
-	}
-
-	temp = scnprintf(next, size, "\n");
-
-	size -= temp;
-	next += temp;
-
-	*sizep = size;
-	*nextp = next;
-}
-
-static ssize_t fill_async_buffer(struct debug_buffer *buf)
-{
-	struct usb_hcd *hcd;
-	struct fotg210_hcd *fotg210;
-	unsigned long flags;
-	unsigned temp, size;
-	char *next;
-	struct fotg210_qh *qh;
-
-	hcd = bus_to_hcd(buf->bus);
-	fotg210 = hcd_to_fotg210(hcd);
-	next = buf->output_buf;
-	size = buf->alloc_size;
-
-	*next = 0;
-
-	/* dumps a snapshot of the async schedule.
-	 * usually empty except for long-term bulk reads, or head.
-	 * one QH per line, and TDs we know about
-	 */
-	spin_lock_irqsave(&fotg210->lock, flags);
-	for (qh = fotg210->async->qh_next.qh; size > 0 && qh;
-			qh = qh->qh_next.qh)
-		qh_lines(fotg210, qh, &next, &size);
-	if (fotg210->async_unlink && size > 0) {
-		temp = scnprintf(next, size, "\nunlink =\n");
-		size -= temp;
-		next += temp;
-
-		for (qh = fotg210->async_unlink; size > 0 && qh;
-				qh = qh->unlink_next)
-			qh_lines(fotg210, qh, &next, &size);
-	}
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-
-	return strlen(buf->output_buf);
-}
-
-/* count tds, get ep direction */
-static unsigned output_buf_tds_dir(char *buf, struct fotg210_hcd *fotg210,
-		struct fotg210_qh_hw *hw, struct fotg210_qh *qh, unsigned size)
-{
-	u32 scratch = hc32_to_cpup(fotg210, &hw->hw_info1);
-	struct fotg210_qtd *qtd;
-	char *type = "";
-	unsigned temp = 0;
-
-	/* count tds, get ep direction */
-	list_for_each_entry(qtd, &qh->qtd_list, qtd_list) {
-		temp++;
-		switch ((hc32_to_cpu(fotg210, qtd->hw_token) >> 8) & 0x03) {
-		case 0:
-			type = "out";
-			continue;
-		case 1:
-			type = "in";
-			continue;
-		}
-	}
-
-	return scnprintf(buf, size, "(%c%d ep%d%s [%d/%d] q%d p%d)",
-			speed_char(scratch), scratch & 0x007f,
-			(scratch >> 8) & 0x000f, type, qh->usecs,
-			qh->c_usecs, temp, (scratch >> 16) & 0x7ff);
-}
-
-#define DBG_SCHED_LIMIT 64
-static ssize_t fill_periodic_buffer(struct debug_buffer *buf)
-{
-	struct usb_hcd *hcd;
-	struct fotg210_hcd *fotg210;
-	unsigned long flags;
-	union fotg210_shadow p, *seen;
-	unsigned temp, size, seen_count;
-	char *next;
-	unsigned i;
-	__hc32 tag;
-
-	seen = kmalloc_objs(*seen, DBG_SCHED_LIMIT, GFP_ATOMIC);
-	if (!seen)
-		return 0;
-
-	seen_count = 0;
-
-	hcd = bus_to_hcd(buf->bus);
-	fotg210 = hcd_to_fotg210(hcd);
-	next = buf->output_buf;
-	size = buf->alloc_size;
-
-	temp = scnprintf(next, size, "size = %d\n", fotg210->periodic_size);
-	size -= temp;
-	next += temp;
-
-	/* dump a snapshot of the periodic schedule.
-	 * iso changes, interrupt usually doesn't.
-	 */
-	spin_lock_irqsave(&fotg210->lock, flags);
-	for (i = 0; i < fotg210->periodic_size; i++) {
-		p = fotg210->pshadow[i];
-		if (likely(!p.ptr))
-			continue;
-
-		tag = Q_NEXT_TYPE(fotg210, fotg210->periodic[i]);
-
-		temp = scnprintf(next, size, "%4d: ", i);
-		size -= temp;
-		next += temp;
-
-		do {
-			struct fotg210_qh_hw *hw;
-
-			switch (hc32_to_cpu(fotg210, tag)) {
-			case Q_TYPE_QH:
-				hw = p.qh->hw;
-				temp = scnprintf(next, size, " qh%d-%04x/%p",
-						p.qh->period,
-						hc32_to_cpup(fotg210,
-							&hw->hw_info2)
-							/* uframe masks */
-							& (QH_CMASK | QH_SMASK),
-						p.qh);
-				size -= temp;
-				next += temp;
-				/* don't repeat what follows this qh */
-				for (temp = 0; temp < seen_count; temp++) {
-					if (seen[temp].ptr != p.ptr)
-						continue;
-					if (p.qh->qh_next.ptr) {
-						temp = scnprintf(next, size,
-								" ...");
-						size -= temp;
-						next += temp;
-					}
-					break;
-				}
-				/* show more info the first time around */
-				if (temp == seen_count) {
-					temp = output_buf_tds_dir(next,
-							fotg210, hw,
-							p.qh, size);
-
-					if (seen_count < DBG_SCHED_LIMIT)
-						seen[seen_count++].qh = p.qh;
-				} else
-					temp = 0;
-				tag = Q_NEXT_TYPE(fotg210, hw->hw_next);
-				p = p.qh->qh_next;
-				break;
-			case Q_TYPE_FSTN:
-				temp = scnprintf(next, size,
-						" fstn-%8x/%p",
-						p.fstn->hw_prev, p.fstn);
-				tag = Q_NEXT_TYPE(fotg210, p.fstn->hw_next);
-				p = p.fstn->fstn_next;
-				break;
-			case Q_TYPE_ITD:
-				temp = scnprintf(next, size,
-						" itd/%p", p.itd);
-				tag = Q_NEXT_TYPE(fotg210, p.itd->hw_next);
-				p = p.itd->itd_next;
-				break;
-			}
-			size -= temp;
-			next += temp;
-		} while (p.ptr);
-
-		temp = scnprintf(next, size, "\n");
-		size -= temp;
-		next += temp;
-	}
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	kfree(seen);
-
-	return buf->alloc_size - size;
-}
-#undef DBG_SCHED_LIMIT
-
-static const char *rh_state_string(struct fotg210_hcd *fotg210)
-{
-	switch (fotg210->rh_state) {
-	case FOTG210_RH_HALTED:
-		return "halted";
-	case FOTG210_RH_SUSPENDED:
-		return "suspended";
-	case FOTG210_RH_RUNNING:
-		return "running";
-	case FOTG210_RH_STOPPING:
-		return "stopping";
-	}
-	return "?";
-}
-
-static ssize_t fill_registers_buffer(struct debug_buffer *buf)
-{
-	struct usb_hcd *hcd;
-	struct fotg210_hcd *fotg210;
-	unsigned long flags;
-	unsigned temp, size, i;
-	char *next, scratch[80];
-	static const char fmt[] = "%*s\n";
-	static const char label[] = "";
-
-	hcd = bus_to_hcd(buf->bus);
-	fotg210 = hcd_to_fotg210(hcd);
-	next = buf->output_buf;
-	size = buf->alloc_size;
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-
-	if (!HCD_HW_ACCESSIBLE(hcd)) {
-		size = scnprintf(next, size,
-				"bus %s, device %s\n"
-				"%s\n"
-				"SUSPENDED(no register access)\n",
-				hcd->self.controller->bus->name,
-				dev_name(hcd->self.controller),
-				hcd->product_desc);
-		goto done;
-	}
-
-	/* Capability Registers */
-	i = HC_VERSION(fotg210, fotg210_readl(fotg210,
-			&fotg210->caps->hc_capbase));
-	temp = scnprintf(next, size,
-			"bus %s, device %s\n"
-			"%s\n"
-			"EHCI %x.%02x, rh state %s\n",
-			hcd->self.controller->bus->name,
-			dev_name(hcd->self.controller),
-			hcd->product_desc,
-			i >> 8, i & 0x0ff, rh_state_string(fotg210));
-	size -= temp;
-	next += temp;
-
-	/* FIXME interpret both types of params */
-	i = fotg210_readl(fotg210, &fotg210->caps->hcs_params);
-	temp = scnprintf(next, size, "structural params 0x%08x\n", i);
-	size -= temp;
-	next += temp;
-
-	i = fotg210_readl(fotg210, &fotg210->caps->hcc_params);
-	temp = scnprintf(next, size, "capability params 0x%08x\n", i);
-	size -= temp;
-	next += temp;
-
-	/* Operational Registers */
-	temp = dbg_status_buf(scratch, sizeof(scratch), label,
-			fotg210_readl(fotg210, &fotg210->regs->status));
-	temp = scnprintf(next, size, fmt, temp, scratch);
-	size -= temp;
-	next += temp;
-
-	temp = dbg_command_buf(scratch, sizeof(scratch), label,
-			fotg210_readl(fotg210, &fotg210->regs->command));
-	temp = scnprintf(next, size, fmt, temp, scratch);
-	size -= temp;
-	next += temp;
-
-	temp = dbg_intr_buf(scratch, sizeof(scratch), label,
-			fotg210_readl(fotg210, &fotg210->regs->intr_enable));
-	temp = scnprintf(next, size, fmt, temp, scratch);
-	size -= temp;
-	next += temp;
-
-	temp = scnprintf(next, size, "uframe %04x\n",
-			fotg210_read_frame_index(fotg210));
-	size -= temp;
-	next += temp;
-
-	if (fotg210->async_unlink) {
-		temp = scnprintf(next, size, "async unlink qh %p\n",
-				fotg210->async_unlink);
-		size -= temp;
-		next += temp;
-	}
-
-#ifdef FOTG210_STATS
-	temp = scnprintf(next, size,
-			"irq normal %ld err %ld iaa %ld(lost %ld)\n",
-			fotg210->stats.normal, fotg210->stats.error,
-			fotg210->stats.iaa, fotg210->stats.lost_iaa);
-	size -= temp;
-	next += temp;
-
-	temp = scnprintf(next, size, "complete %ld unlink %ld\n",
-			fotg210->stats.complete, fotg210->stats.unlink);
-	size -= temp;
-	next += temp;
-#endif
-
-done:
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-
-	return buf->alloc_size - size;
-}
-
-static struct debug_buffer
-*alloc_buffer(struct usb_bus *bus, ssize_t (*fill_func)(struct debug_buffer *))
-{
-	struct debug_buffer *buf;
-
-	buf = kzalloc_obj(struct debug_buffer);
-
-	if (buf) {
-		buf->bus = bus;
-		buf->fill_func = fill_func;
-		mutex_init(&buf->mutex);
-		buf->alloc_size = PAGE_SIZE;
-	}
-
-	return buf;
-}
-
-static int fill_buffer(struct debug_buffer *buf)
-{
-	int ret = 0;
-
-	if (!buf->output_buf)
-		buf->output_buf = vmalloc(buf->alloc_size);
-
-	if (!buf->output_buf) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	ret = buf->fill_func(buf);
-
-	if (ret >= 0) {
-		buf->count = ret;
-		ret = 0;
-	}
-
-out:
-	return ret;
-}
-
-static ssize_t debug_output(struct file *file, char __user *user_buf,
-		size_t len, loff_t *offset)
-{
-	struct debug_buffer *buf = file->private_data;
-	int ret = 0;
-
-	mutex_lock(&buf->mutex);
-	if (buf->count == 0) {
-		ret = fill_buffer(buf);
-		if (ret != 0) {
-			mutex_unlock(&buf->mutex);
-			goto out;
-		}
-	}
-	mutex_unlock(&buf->mutex);
-
-	ret = simple_read_from_buffer(user_buf, len, offset,
-			buf->output_buf, buf->count);
-
-out:
-	return ret;
-
-}
-
-static int debug_close(struct inode *inode, struct file *file)
-{
-	struct debug_buffer *buf = file->private_data;
-
-	if (buf) {
-		vfree(buf->output_buf);
-		kfree(buf);
-	}
-
-	return 0;
-}
-static int debug_async_open(struct inode *inode, struct file *file)
-{
-	file->private_data = alloc_buffer(inode->i_private, fill_async_buffer);
-
-	return file->private_data ? 0 : -ENOMEM;
-}
-
-static int debug_periodic_open(struct inode *inode, struct file *file)
-{
-	struct debug_buffer *buf;
-
-	buf = alloc_buffer(inode->i_private, fill_periodic_buffer);
-	if (!buf)
-		return -ENOMEM;
-
-	buf->alloc_size = (sizeof(void *) == 4 ? 6 : 8)*PAGE_SIZE;
-	file->private_data = buf;
-	return 0;
-}
-
-static int debug_registers_open(struct inode *inode, struct file *file)
-{
-	file->private_data = alloc_buffer(inode->i_private,
-			fill_registers_buffer);
-
-	return file->private_data ? 0 : -ENOMEM;
-}
-
-static inline void create_debug_files(struct fotg210_hcd *fotg210)
-{
-	struct usb_bus *bus = &fotg210_to_hcd(fotg210)->self;
-	struct dentry *root;
-
-	root = debugfs_create_dir(bus->bus_name, fotg210_debug_root);
-
-	debugfs_create_file("async", S_IRUGO, root, bus, &debug_async_fops);
-	debugfs_create_file("periodic", S_IRUGO, root, bus,
-			    &debug_periodic_fops);
-	debugfs_create_file("registers", S_IRUGO, root, bus,
-			    &debug_registers_fops);
-}
-
-static inline void remove_debug_files(struct fotg210_hcd *fotg210)
-{
-	struct usb_bus *bus = &fotg210_to_hcd(fotg210)->self;
-
-	debugfs_lookup_and_remove(bus->bus_name, fotg210_debug_root);
-}
-
-/* handshake - spin reading hc until handshake completes or fails
- * @ptr: address of hc register to be read
- * @mask: bits to look at in result of read
- * @done: value of those bits when handshake succeeds
- * @usec: timeout in microseconds
- *
- * Returns negative errno, or zero on success
- *
- * Success happens when the "mask" bits have the specified value (hardware
- * handshake done).  There are two failure modes:  "usec" have passed (major
- * hardware flakeout), or the register reads as all-ones (hardware removed).
- *
- * That last failure should_only happen in cases like physical cardbus eject
- * before driver shutdown. But it also seems to be caused by bugs in cardbus
- * bridge shutdown:  shutting down the bridge before the devices using it.
- */
-static int handshake(struct fotg210_hcd *fotg210, void __iomem *ptr,
-		u32 mask, u32 done, int usec)
-{
-	u32 result;
-	int ret;
-
-	ret = readl_poll_timeout_atomic(ptr, result,
-					((result & mask) == done ||
-					 result == U32_MAX), 1, usec);
-	if (result == U32_MAX)		/* card removed */
-		return -ENODEV;
-
-	return ret;
-}
-
-/* Force HC to halt state from unknown (EHCI spec section 2.3).
- * Must be called with interrupts enabled and the lock not held.
- */
-static int fotg210_halt(struct fotg210_hcd *fotg210)
-{
-	u32 temp;
-
-	spin_lock_irq(&fotg210->lock);
-
-	/* disable any irqs left enabled by previous code */
-	fotg210_writel(fotg210, 0, &fotg210->regs->intr_enable);
-
-	/*
-	 * This routine gets called during probe before fotg210->command
-	 * has been initialized, so we can't rely on its value.
-	 */
-	fotg210->command &= ~CMD_RUN;
-	temp = fotg210_readl(fotg210, &fotg210->regs->command);
-	temp &= ~(CMD_RUN | CMD_IAAD);
-	fotg210_writel(fotg210, temp, &fotg210->regs->command);
-
-	spin_unlock_irq(&fotg210->lock);
-	synchronize_irq(fotg210_to_hcd(fotg210)->irq);
-
-	return handshake(fotg210, &fotg210->regs->status,
-			STS_HALT, STS_HALT, 16 * 125);
-}
-
-/* Reset a non-running (STS_HALT == 1) controller.
- * Must be called with interrupts enabled and the lock not held.
- */
-static int fotg210_reset(struct fotg210_hcd *fotg210)
-{
-	int retval;
-	u32 command = fotg210_readl(fotg210, &fotg210->regs->command);
-
-	/* If the EHCI debug controller is active, special care must be
-	 * taken before and after a host controller reset
-	 */
-	if (fotg210->debug && !dbgp_reset_prep(fotg210_to_hcd(fotg210)))
-		fotg210->debug = NULL;
-
-	command |= CMD_RESET;
-	dbg_cmd(fotg210, "reset", command);
-	fotg210_writel(fotg210, command, &fotg210->regs->command);
-	fotg210->rh_state = FOTG210_RH_HALTED;
-	fotg210->next_statechange = jiffies;
-	retval = handshake(fotg210, &fotg210->regs->command,
-			CMD_RESET, 0, 250 * 1000);
-
-	if (retval)
-		return retval;
-
-	if (fotg210->debug)
-		dbgp_external_startup(fotg210_to_hcd(fotg210));
-
-	fotg210->port_c_suspend = fotg210->suspended_ports =
-			fotg210->resuming_ports = 0;
-	return retval;
-}
-
-/* Idle the controller (turn off the schedules).
- * Must be called with interrupts enabled and the lock not held.
- */
-static void fotg210_quiesce(struct fotg210_hcd *fotg210)
-{
-	u32 temp;
-
-	if (fotg210->rh_state != FOTG210_RH_RUNNING)
-		return;
-
-	/* wait for any schedule enables/disables to take effect */
-	temp = (fotg210->command << 10) & (STS_ASS | STS_PSS);
-	handshake(fotg210, &fotg210->regs->status, STS_ASS | STS_PSS, temp,
-			16 * 125);
-
-	/* then disable anything that's still active */
-	spin_lock_irq(&fotg210->lock);
-	fotg210->command &= ~(CMD_ASE | CMD_PSE);
-	fotg210_writel(fotg210, fotg210->command, &fotg210->regs->command);
-	spin_unlock_irq(&fotg210->lock);
-
-	/* hardware can take 16 microframes to turn off ... */
-	handshake(fotg210, &fotg210->regs->status, STS_ASS | STS_PSS, 0,
-			16 * 125);
-}
-
-static void end_unlink_async(struct fotg210_hcd *fotg210);
-static void unlink_empty_async(struct fotg210_hcd *fotg210);
-static void fotg210_work(struct fotg210_hcd *fotg210);
-static void start_unlink_intr(struct fotg210_hcd *fotg210,
-			      struct fotg210_qh *qh);
-static void end_unlink_intr(struct fotg210_hcd *fotg210, struct fotg210_qh *qh);
-
-/* Set a bit in the USBCMD register */
-static void fotg210_set_command_bit(struct fotg210_hcd *fotg210, u32 bit)
-{
-	fotg210->command |= bit;
-	fotg210_writel(fotg210, fotg210->command, &fotg210->regs->command);
-
-	/* unblock posted write */
-	fotg210_readl(fotg210, &fotg210->regs->command);
-}
-
-/* Clear a bit in the USBCMD register */
-static void fotg210_clear_command_bit(struct fotg210_hcd *fotg210, u32 bit)
-{
-	fotg210->command &= ~bit;
-	fotg210_writel(fotg210, fotg210->command, &fotg210->regs->command);
-
-	/* unblock posted write */
-	fotg210_readl(fotg210, &fotg210->regs->command);
-}
-
-/* EHCI timer support...  Now using hrtimers.
- *
- * Lots of different events are triggered from fotg210->hrtimer.  Whenever
- * the timer routine runs, it checks each possible event; events that are
- * currently enabled and whose expiration time has passed get handled.
- * The set of enabled events is stored as a collection of bitflags in
- * fotg210->enabled_hrtimer_events, and they are numbered in order of
- * increasing delay values (ranging between 1 ms and 100 ms).
- *
- * Rather than implementing a sorted list or tree of all pending events,
- * we keep track only of the lowest-numbered pending event, in
- * fotg210->next_hrtimer_event.  Whenever fotg210->hrtimer gets restarted, its
- * expiration time is set to the timeout value for this event.
- *
- * As a result, events might not get handled right away; the actual delay
- * could be anywhere up to twice the requested delay.  This doesn't
- * matter, because none of the events are especially time-critical.  The
- * ones that matter most all have a delay of 1 ms, so they will be
- * handled after 2 ms at most, which is okay.  In addition to this, we
- * allow for an expiration range of 1 ms.
- */
-
-/* Delay lengths for the hrtimer event types.
- * Keep this list sorted by delay length, in the same order as
- * the event types indexed by enum fotg210_hrtimer_event in fotg210.h.
- */
-static unsigned event_delays_ns[] = {
-	1 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_POLL_ASS */
-	1 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_POLL_PSS */
-	1 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_POLL_DEAD */
-	1125 * NSEC_PER_USEC,	/* FOTG210_HRTIMER_UNLINK_INTR */
-	2 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_FREE_ITDS */
-	6 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_ASYNC_UNLINKS */
-	10 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_IAA_WATCHDOG */
-	10 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_DISABLE_PERIODIC */
-	15 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_DISABLE_ASYNC */
-	100 * NSEC_PER_MSEC,	/* FOTG210_HRTIMER_IO_WATCHDOG */
-};
-
-/* Enable a pending hrtimer event */
-static void fotg210_enable_event(struct fotg210_hcd *fotg210, unsigned event,
-		bool resched)
-{
-	ktime_t *timeout = &fotg210->hr_timeouts[event];
-
-	if (resched)
-		*timeout = ktime_add(ktime_get(), event_delays_ns[event]);
-	fotg210->enabled_hrtimer_events |= (1 << event);
-
-	/* Track only the lowest-numbered pending event */
-	if (event < fotg210->next_hrtimer_event) {
-		fotg210->next_hrtimer_event = event;
-		hrtimer_start_range_ns(&fotg210->hrtimer, *timeout,
-				NSEC_PER_MSEC, HRTIMER_MODE_ABS);
-	}
-}
-
-
-/* Poll the STS_ASS status bit; see when it agrees with CMD_ASE */
-static void fotg210_poll_ASS(struct fotg210_hcd *fotg210)
-{
-	unsigned actual, want;
-
-	/* Don't enable anything if the controller isn't running (e.g., died) */
-	if (fotg210->rh_state != FOTG210_RH_RUNNING)
-		return;
-
-	want = (fotg210->command & CMD_ASE) ? STS_ASS : 0;
-	actual = fotg210_readl(fotg210, &fotg210->regs->status) & STS_ASS;
-
-	if (want != actual) {
-
-		/* Poll again later, but give up after about 20 ms */
-		if (fotg210->ASS_poll_count++ < 20) {
-			fotg210_enable_event(fotg210, FOTG210_HRTIMER_POLL_ASS,
-					true);
-			return;
-		}
-		fotg210_dbg(fotg210, "Waited too long for the async schedule status (%x/%x), giving up\n",
-				want, actual);
-	}
-	fotg210->ASS_poll_count = 0;
-
-	/* The status is up-to-date; restart or stop the schedule as needed */
-	if (want == 0) {	/* Stopped */
-		if (fotg210->async_count > 0)
-			fotg210_set_command_bit(fotg210, CMD_ASE);
-
-	} else {		/* Running */
-		if (fotg210->async_count == 0) {
-
-			/* Turn off the schedule after a while */
-			fotg210_enable_event(fotg210,
-					FOTG210_HRTIMER_DISABLE_ASYNC,
-					true);
-		}
-	}
-}
-
-/* Turn off the async schedule after a brief delay */
-static void fotg210_disable_ASE(struct fotg210_hcd *fotg210)
-{
-	fotg210_clear_command_bit(fotg210, CMD_ASE);
-}
-
-
-/* Poll the STS_PSS status bit; see when it agrees with CMD_PSE */
-static void fotg210_poll_PSS(struct fotg210_hcd *fotg210)
-{
-	unsigned actual, want;
-
-	/* Don't do anything if the controller isn't running (e.g., died) */
-	if (fotg210->rh_state != FOTG210_RH_RUNNING)
-		return;
-
-	want = (fotg210->command & CMD_PSE) ? STS_PSS : 0;
-	actual = fotg210_readl(fotg210, &fotg210->regs->status) & STS_PSS;
-
-	if (want != actual) {
-
-		/* Poll again later, but give up after about 20 ms */
-		if (fotg210->PSS_poll_count++ < 20) {
-			fotg210_enable_event(fotg210, FOTG210_HRTIMER_POLL_PSS,
-					true);
-			return;
-		}
-		fotg210_dbg(fotg210, "Waited too long for the periodic schedule status (%x/%x), giving up\n",
-				want, actual);
-	}
-	fotg210->PSS_poll_count = 0;
-
-	/* The status is up-to-date; restart or stop the schedule as needed */
-	if (want == 0) {	/* Stopped */
-		if (fotg210->periodic_count > 0)
-			fotg210_set_command_bit(fotg210, CMD_PSE);
-
-	} else {		/* Running */
-		if (fotg210->periodic_count == 0) {
-
-			/* Turn off the schedule after a while */
-			fotg210_enable_event(fotg210,
-					FOTG210_HRTIMER_DISABLE_PERIODIC,
-					true);
-		}
-	}
-}
-
-/* Turn off the periodic schedule after a brief delay */
-static void fotg210_disable_PSE(struct fotg210_hcd *fotg210)
-{
-	fotg210_clear_command_bit(fotg210, CMD_PSE);
-}
-
-
-/* Poll the STS_HALT status bit; see when a dead controller stops */
-static void fotg210_handle_controller_death(struct fotg210_hcd *fotg210)
-{
-	if (!(fotg210_readl(fotg210, &fotg210->regs->status) & STS_HALT)) {
-
-		/* Give up after a few milliseconds */
-		if (fotg210->died_poll_count++ < 5) {
-			/* Try again later */
-			fotg210_enable_event(fotg210,
-					FOTG210_HRTIMER_POLL_DEAD, true);
-			return;
-		}
-		fotg210_warn(fotg210, "Waited too long for the controller to stop, giving up\n");
-	}
-
-	/* Clean up the mess */
-	fotg210->rh_state = FOTG210_RH_HALTED;
-	fotg210_writel(fotg210, 0, &fotg210->regs->intr_enable);
-	fotg210_work(fotg210);
-	end_unlink_async(fotg210);
-
-	/* Not in process context, so don't try to reset the controller */
-}
-
-
-/* Handle unlinked interrupt QHs once they are gone from the hardware */
-static void fotg210_handle_intr_unlinks(struct fotg210_hcd *fotg210)
-{
-	bool stopped = (fotg210->rh_state < FOTG210_RH_RUNNING);
-
-	/*
-	 * Process all the QHs on the intr_unlink list that were added
-	 * before the current unlink cycle began.  The list is in
-	 * temporal order, so stop when we reach the first entry in the
-	 * current cycle.  But if the root hub isn't running then
-	 * process all the QHs on the list.
-	 */
-	fotg210->intr_unlinking = true;
-	while (fotg210->intr_unlink) {
-		struct fotg210_qh *qh = fotg210->intr_unlink;
-
-		if (!stopped && qh->unlink_cycle == fotg210->intr_unlink_cycle)
-			break;
-		fotg210->intr_unlink = qh->unlink_next;
-		qh->unlink_next = NULL;
-		end_unlink_intr(fotg210, qh);
-	}
-
-	/* Handle remaining entries later */
-	if (fotg210->intr_unlink) {
-		fotg210_enable_event(fotg210, FOTG210_HRTIMER_UNLINK_INTR,
-				true);
-		++fotg210->intr_unlink_cycle;
-	}
-	fotg210->intr_unlinking = false;
-}
-
-
-/* Start another free-iTDs/siTDs cycle */
-static void start_free_itds(struct fotg210_hcd *fotg210)
-{
-	if (!(fotg210->enabled_hrtimer_events &
-			BIT(FOTG210_HRTIMER_FREE_ITDS))) {
-		fotg210->last_itd_to_free = list_entry(
-				fotg210->cached_itd_list.prev,
-				struct fotg210_itd, itd_list);
-		fotg210_enable_event(fotg210, FOTG210_HRTIMER_FREE_ITDS, true);
-	}
-}
-
-/* Wait for controller to stop using old iTDs and siTDs */
-static void end_free_itds(struct fotg210_hcd *fotg210)
-{
-	struct fotg210_itd *itd, *n;
-
-	if (fotg210->rh_state < FOTG210_RH_RUNNING)
-		fotg210->last_itd_to_free = NULL;
-
-	list_for_each_entry_safe(itd, n, &fotg210->cached_itd_list, itd_list) {
-		list_del(&itd->itd_list);
-		dma_pool_free(fotg210->itd_pool, itd, itd->itd_dma);
-		if (itd == fotg210->last_itd_to_free)
-			break;
-	}
-
-	if (!list_empty(&fotg210->cached_itd_list))
-		start_free_itds(fotg210);
-}
-
-
-/* Handle lost (or very late) IAA interrupts */
-static void fotg210_iaa_watchdog(struct fotg210_hcd *fotg210)
-{
-	if (fotg210->rh_state != FOTG210_RH_RUNNING)
-		return;
-
-	/*
-	 * Lost IAA irqs wedge things badly; seen first with a vt8235.
-	 * So we need this watchdog, but must protect it against both
-	 * (a) SMP races against real IAA firing and retriggering, and
-	 * (b) clean HC shutdown, when IAA watchdog was pending.
-	 */
-	if (fotg210->async_iaa) {
-		u32 cmd, status;
-
-		/* If we get here, IAA is *REALLY* late.  It's barely
-		 * conceivable that the system is so busy that CMD_IAAD
-		 * is still legitimately set, so let's be sure it's
-		 * clear before we read STS_IAA.  (The HC should clear
-		 * CMD_IAAD when it sets STS_IAA.)
-		 */
-		cmd = fotg210_readl(fotg210, &fotg210->regs->command);
-
-		/*
-		 * If IAA is set here it either legitimately triggered
-		 * after the watchdog timer expired (_way_ late, so we'll
-		 * still count it as lost) ... or a silicon erratum:
-		 * - VIA seems to set IAA without triggering the IRQ;
-		 * - IAAD potentially cleared without setting IAA.
-		 */
-		status = fotg210_readl(fotg210, &fotg210->regs->status);
-		if ((status & STS_IAA) || !(cmd & CMD_IAAD)) {
-			INCR(fotg210->stats.lost_iaa);
-			fotg210_writel(fotg210, STS_IAA,
-					&fotg210->regs->status);
-		}
-
-		fotg210_dbg(fotg210, "IAA watchdog: status %x cmd %x\n",
-				status, cmd);
-		end_unlink_async(fotg210);
-	}
-}
-
-
-/* Enable the I/O watchdog, if appropriate */
-static void turn_on_io_watchdog(struct fotg210_hcd *fotg210)
-{
-	/* Not needed if the controller isn't running or it's already enabled */
-	if (fotg210->rh_state != FOTG210_RH_RUNNING ||
-			(fotg210->enabled_hrtimer_events &
-			BIT(FOTG210_HRTIMER_IO_WATCHDOG)))
-		return;
-
-	/*
-	 * Isochronous transfers always need the watchdog.
-	 * For other sorts we use it only if the flag is set.
-	 */
-	if (fotg210->isoc_count > 0 || (fotg210->need_io_watchdog &&
-			fotg210->async_count + fotg210->intr_count > 0))
-		fotg210_enable_event(fotg210, FOTG210_HRTIMER_IO_WATCHDOG,
-				true);
-}
-
-
-/* Handler functions for the hrtimer event types.
- * Keep this array in the same order as the event types indexed by
- * enum fotg210_hrtimer_event in fotg210.h.
- */
-static void (*event_handlers[])(struct fotg210_hcd *) = {
-	fotg210_poll_ASS,			/* FOTG210_HRTIMER_POLL_ASS */
-	fotg210_poll_PSS,			/* FOTG210_HRTIMER_POLL_PSS */
-	fotg210_handle_controller_death,	/* FOTG210_HRTIMER_POLL_DEAD */
-	fotg210_handle_intr_unlinks,	/* FOTG210_HRTIMER_UNLINK_INTR */
-	end_free_itds,			/* FOTG210_HRTIMER_FREE_ITDS */
-	unlink_empty_async,		/* FOTG210_HRTIMER_ASYNC_UNLINKS */
-	fotg210_iaa_watchdog,		/* FOTG210_HRTIMER_IAA_WATCHDOG */
-	fotg210_disable_PSE,		/* FOTG210_HRTIMER_DISABLE_PERIODIC */
-	fotg210_disable_ASE,		/* FOTG210_HRTIMER_DISABLE_ASYNC */
-	fotg210_work,			/* FOTG210_HRTIMER_IO_WATCHDOG */
-};
-
-static enum hrtimer_restart fotg210_hrtimer_func(struct hrtimer *t)
-{
-	struct fotg210_hcd *fotg210 =
-			container_of(t, struct fotg210_hcd, hrtimer);
-	ktime_t now;
-	unsigned long events;
-	unsigned long flags;
-	unsigned e;
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-
-	events = fotg210->enabled_hrtimer_events;
-	fotg210->enabled_hrtimer_events = 0;
-	fotg210->next_hrtimer_event = FOTG210_HRTIMER_NO_EVENT;
-
-	/*
-	 * Check each pending event.  If its time has expired, handle
-	 * the event; otherwise re-enable it.
-	 */
-	now = ktime_get();
-	for_each_set_bit(e, &events, FOTG210_HRTIMER_NUM_EVENTS) {
-		if (ktime_compare(now, fotg210->hr_timeouts[e]) >= 0)
-			event_handlers[e](fotg210);
-		else
-			fotg210_enable_event(fotg210, e, false);
-	}
-
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	return HRTIMER_NORESTART;
-}
-
-#define fotg210_bus_suspend NULL
-#define fotg210_bus_resume NULL
-
-static int check_reset_complete(struct fotg210_hcd *fotg210, int index,
-		u32 __iomem *status_reg, int port_status)
-{
-	if (!(port_status & PORT_CONNECT))
-		return port_status;
-
-	/* if reset finished and it's still not enabled -- handoff */
-	if (!(port_status & PORT_PE))
-		/* with integrated TT, there's nobody to hand it to! */
-		fotg210_dbg(fotg210, "Failed to enable port %d on root hub TT\n",
-				index + 1);
-	else
-		fotg210_dbg(fotg210, "port %d reset complete, port enabled\n",
-				index + 1);
-
-	return port_status;
-}
-
-
-/* build "status change" packet (one or two bytes) from HC registers */
-
-static int fotg210_hub_status_data(struct usb_hcd *hcd, char *buf)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	u32 temp, status;
-	u32 mask;
-	int retval = 1;
-	unsigned long flags;
-
-	/* init status to no-changes */
-	buf[0] = 0;
-
-	/* Inform the core about resumes-in-progress by returning
-	 * a non-zero value even if there are no status changes.
-	 */
-	status = fotg210->resuming_ports;
-
-	mask = PORT_CSC | PORT_PEC;
-	/* PORT_RESUME from hardware ~= PORT_STAT_C_SUSPEND */
-
-	/* no hub change reports (bit 0) for now (power, ...) */
-
-	/* port N changes (bit N)? */
-	spin_lock_irqsave(&fotg210->lock, flags);
-
-	temp = fotg210_readl(fotg210, &fotg210->regs->port_status);
-
-	/*
-	 * Return status information even for ports with OWNER set.
-	 * Otherwise hub_wq wouldn't see the disconnect event when a
-	 * high-speed device is switched over to the companion
-	 * controller by the user.
-	 */
-
-	if ((temp & mask) != 0 || test_bit(0, &fotg210->port_c_suspend) ||
-			(fotg210->reset_done[0] &&
-			time_after_eq(jiffies, fotg210->reset_done[0]))) {
-		buf[0] |= 1 << 1;
-		status = STS_PCD;
-	}
-	/* FIXME autosuspend idle root hubs */
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	return status ? retval : 0;
-}
-
-static void fotg210_hub_descriptor(struct fotg210_hcd *fotg210,
-		struct usb_hub_descriptor *desc)
-{
-	int ports = HCS_N_PORTS(fotg210->hcs_params);
-	u16 temp;
-
-	desc->bDescriptorType = USB_DT_HUB;
-	desc->bPwrOn2PwrGood = 10;	/* fotg210 1.0, 2.3.9 says 20ms max */
-	desc->bHubContrCurrent = 0;
-
-	desc->bNbrPorts = ports;
-	temp = 1 + (ports / 8);
-	desc->bDescLength = 7 + 2 * temp;
-
-	/* two bitmaps:  ports removable, and usb 1.0 legacy PortPwrCtrlMask */
-	memset(&desc->u.hs.DeviceRemovable[0], 0, temp);
-	memset(&desc->u.hs.DeviceRemovable[temp], 0xff, temp);
-
-	temp = HUB_CHAR_INDV_PORT_OCPM;	/* per-port overcurrent reporting */
-	temp |= HUB_CHAR_NO_LPSM;	/* no power switching */
-	desc->wHubCharacteristics = cpu_to_le16(temp);
-}
-
-static int fotg210_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
-		u16 wIndex, char *buf, u16 wLength)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	int ports = HCS_N_PORTS(fotg210->hcs_params);
-	u32 __iomem *status_reg = &fotg210->regs->port_status;
-	u32 temp, temp1, status;
-	unsigned long flags;
-	int retval = 0;
-	unsigned selector;
-
-	/*
-	 * FIXME:  support SetPortFeatures USB_PORT_FEAT_INDICATOR.
-	 * HCS_INDICATOR may say we can change LEDs to off/amber/green.
-	 * (track current state ourselves) ... blink for diagnostics,
-	 * power, "this is the one", etc.  EHCI spec supports this.
-	 */
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-	switch (typeReq) {
-	case ClearHubFeature:
-		switch (wValue) {
-		case C_HUB_LOCAL_POWER:
-		case C_HUB_OVER_CURRENT:
-			/* no hub-wide feature/status flags */
-			break;
-		default:
-			goto error;
-		}
-		break;
-	case ClearPortFeature:
-		if (!wIndex || wIndex > ports)
-			goto error;
-		wIndex--;
-		temp = fotg210_readl(fotg210, status_reg);
-		temp &= ~PORT_RWC_BITS;
-
-		/*
-		 * Even if OWNER is set, so the port is owned by the
-		 * companion controller, hub_wq needs to be able to clear
-		 * the port-change status bits (especially
-		 * USB_PORT_STAT_C_CONNECTION).
-		 */
-
-		switch (wValue) {
-		case USB_PORT_FEAT_ENABLE:
-			fotg210_writel(fotg210, temp & ~PORT_PE, status_reg);
-			break;
-		case USB_PORT_FEAT_C_ENABLE:
-			fotg210_writel(fotg210, temp | PORT_PEC, status_reg);
-			break;
-		case USB_PORT_FEAT_SUSPEND:
-			if (temp & PORT_RESET)
-				goto error;
-			if (!(temp & PORT_SUSPEND))
-				break;
-			if ((temp & PORT_PE) == 0)
-				goto error;
-
-			/* resume signaling for 20 msec */
-			fotg210_writel(fotg210, temp | PORT_RESUME, status_reg);
-			fotg210->reset_done[wIndex] = jiffies
-					+ msecs_to_jiffies(USB_RESUME_TIMEOUT);
-			break;
-		case USB_PORT_FEAT_C_SUSPEND:
-			clear_bit(wIndex, &fotg210->port_c_suspend);
-			break;
-		case USB_PORT_FEAT_C_CONNECTION:
-			fotg210_writel(fotg210, temp | PORT_CSC, status_reg);
-			break;
-		case USB_PORT_FEAT_C_OVER_CURRENT:
-			fotg210_writel(fotg210, temp | OTGISR_OVC,
-					&fotg210->regs->otgisr);
-			break;
-		case USB_PORT_FEAT_C_RESET:
-			/* GetPortStatus clears reset */
-			break;
-		default:
-			goto error;
-		}
-		fotg210_readl(fotg210, &fotg210->regs->command);
-		break;
-	case GetHubDescriptor:
-		fotg210_hub_descriptor(fotg210, (struct usb_hub_descriptor *)
-				buf);
-		break;
-	case GetHubStatus:
-		/* no hub-wide feature/status flags */
-		memset(buf, 0, 4);
-		/*cpu_to_le32s ((u32 *) buf); */
-		break;
-	case GetPortStatus:
-		if (!wIndex || wIndex > ports)
-			goto error;
-		wIndex--;
-		status = 0;
-		temp = fotg210_readl(fotg210, status_reg);
-
-		/* wPortChange bits */
-		if (temp & PORT_CSC)
-			status |= USB_PORT_STAT_C_CONNECTION << 16;
-		if (temp & PORT_PEC)
-			status |= USB_PORT_STAT_C_ENABLE << 16;
-
-		temp1 = fotg210_readl(fotg210, &fotg210->regs->otgisr);
-		if (temp1 & OTGISR_OVC)
-			status |= USB_PORT_STAT_C_OVERCURRENT << 16;
-
-		/* whoever resumes must GetPortStatus to complete it!! */
-		if (temp & PORT_RESUME) {
-
-			/* Remote Wakeup received? */
-			if (!fotg210->reset_done[wIndex]) {
-				/* resume signaling for 20 msec */
-				fotg210->reset_done[wIndex] = jiffies
-						+ msecs_to_jiffies(20);
-				/* check the port again */
-				mod_timer(&fotg210_to_hcd(fotg210)->rh_timer,
-						fotg210->reset_done[wIndex]);
-			}
-
-			/* resume completed? */
-			else if (time_after_eq(jiffies,
-					fotg210->reset_done[wIndex])) {
-				clear_bit(wIndex, &fotg210->suspended_ports);
-				set_bit(wIndex, &fotg210->port_c_suspend);
-				fotg210->reset_done[wIndex] = 0;
-
-				/* stop resume signaling */
-				temp = fotg210_readl(fotg210, status_reg);
-				fotg210_writel(fotg210, temp &
-						~(PORT_RWC_BITS | PORT_RESUME),
-						status_reg);
-				clear_bit(wIndex, &fotg210->resuming_ports);
-				retval = handshake(fotg210, status_reg,
-						PORT_RESUME, 0, 2000);/* 2ms */
-				if (retval != 0) {
-					fotg210_err(fotg210,
-							"port %d resume error %d\n",
-							wIndex + 1, retval);
-					goto error;
-				}
-				temp &= ~(PORT_SUSPEND|PORT_RESUME|(3<<10));
-			}
-		}
-
-		/* whoever resets must GetPortStatus to complete it!! */
-		if ((temp & PORT_RESET) && time_after_eq(jiffies,
-				fotg210->reset_done[wIndex])) {
-			status |= USB_PORT_STAT_C_RESET << 16;
-			fotg210->reset_done[wIndex] = 0;
-			clear_bit(wIndex, &fotg210->resuming_ports);
-
-			/* force reset to complete */
-			fotg210_writel(fotg210,
-					temp & ~(PORT_RWC_BITS | PORT_RESET),
-					status_reg);
-			/* REVISIT:  some hardware needs 550+ usec to clear
-			 * this bit; seems too long to spin routinely...
-			 */
-			retval = handshake(fotg210, status_reg,
-					PORT_RESET, 0, 1000);
-			if (retval != 0) {
-				fotg210_err(fotg210, "port %d reset error %d\n",
-						wIndex + 1, retval);
-				goto error;
-			}
-
-			/* see what we found out */
-			temp = check_reset_complete(fotg210, wIndex, status_reg,
-					fotg210_readl(fotg210, status_reg));
-
-			/* restart schedule */
-			fotg210->command |= CMD_RUN;
-			fotg210_writel(fotg210, fotg210->command, &fotg210->regs->command);
-		}
-
-		if (!(temp & (PORT_RESUME|PORT_RESET))) {
-			fotg210->reset_done[wIndex] = 0;
-			clear_bit(wIndex, &fotg210->resuming_ports);
-		}
-
-		/* transfer dedicated ports to the companion hc */
-		if ((temp & PORT_CONNECT) &&
-				test_bit(wIndex, &fotg210->companion_ports)) {
-			temp &= ~PORT_RWC_BITS;
-			fotg210_writel(fotg210, temp, status_reg);
-			fotg210_dbg(fotg210, "port %d --> companion\n",
-					wIndex + 1);
-			temp = fotg210_readl(fotg210, status_reg);
-		}
-
-		/*
-		 * Even if OWNER is set, there's no harm letting hub_wq
-		 * see the wPortStatus values (they should all be 0 except
-		 * for PORT_POWER anyway).
-		 */
-
-		if (temp & PORT_CONNECT) {
-			status |= USB_PORT_STAT_CONNECTION;
-			status |= fotg210_port_speed(fotg210, temp);
-		}
-		if (temp & PORT_PE)
-			status |= USB_PORT_STAT_ENABLE;
-
-		/* maybe the port was unsuspended without our knowledge */
-		if (temp & (PORT_SUSPEND|PORT_RESUME)) {
-			status |= USB_PORT_STAT_SUSPEND;
-		} else if (test_bit(wIndex, &fotg210->suspended_ports)) {
-			clear_bit(wIndex, &fotg210->suspended_ports);
-			clear_bit(wIndex, &fotg210->resuming_ports);
-			fotg210->reset_done[wIndex] = 0;
-			if (temp & PORT_PE)
-				set_bit(wIndex, &fotg210->port_c_suspend);
-		}
-
-		temp1 = fotg210_readl(fotg210, &fotg210->regs->otgisr);
-		if (temp1 & OTGISR_OVC)
-			status |= USB_PORT_STAT_OVERCURRENT;
-		if (temp & PORT_RESET)
-			status |= USB_PORT_STAT_RESET;
-		if (test_bit(wIndex, &fotg210->port_c_suspend))
-			status |= USB_PORT_STAT_C_SUSPEND << 16;
-
-		if (status & ~0xffff)	/* only if wPortChange is interesting */
-			dbg_port(fotg210, "GetStatus", wIndex + 1, temp);
-		put_unaligned_le32(status, buf);
-		break;
-	case SetHubFeature:
-		switch (wValue) {
-		case C_HUB_LOCAL_POWER:
-		case C_HUB_OVER_CURRENT:
-			/* no hub-wide feature/status flags */
-			break;
-		default:
-			goto error;
-		}
-		break;
-	case SetPortFeature:
-		selector = wIndex >> 8;
-		wIndex &= 0xff;
-
-		if (!wIndex || wIndex > ports)
-			goto error;
-		wIndex--;
-		temp = fotg210_readl(fotg210, status_reg);
-		temp &= ~PORT_RWC_BITS;
-		switch (wValue) {
-		case USB_PORT_FEAT_SUSPEND:
-			if ((temp & PORT_PE) == 0
-					|| (temp & PORT_RESET) != 0)
-				goto error;
-
-			/* After above check the port must be connected.
-			 * Set appropriate bit thus could put phy into low power
-			 * mode if we have hostpc feature
-			 */
-			fotg210_writel(fotg210, temp | PORT_SUSPEND,
-					status_reg);
-			set_bit(wIndex, &fotg210->suspended_ports);
-			break;
-		case USB_PORT_FEAT_RESET:
-			if (temp & PORT_RESUME)
-				goto error;
-			/* line status bits may report this as low speed,
-			 * which can be fine if this root hub has a
-			 * transaction translator built in.
-			 */
-			fotg210_dbg(fotg210, "port %d reset\n", wIndex + 1);
-			temp |= PORT_RESET;
-			temp &= ~PORT_PE;
-
-			/*
-			 * caller must wait, then call GetPortStatus
-			 * usb 2.0 spec says 50 ms resets on root
-			 */
-			fotg210->reset_done[wIndex] = jiffies
-					+ msecs_to_jiffies(50);
-			fotg210_writel(fotg210, temp, status_reg);
-			break;
-
-		/* For downstream facing ports (these):  one hub port is put
-		 * into test mode according to USB2 11.24.2.13, then the hub
-		 * must be reset (which for root hub now means rmmod+modprobe,
-		 * or else system reboot).  See EHCI 2.3.9 and 4.14 for info
-		 * about the EHCI-specific stuff.
-		 */
-		case USB_PORT_FEAT_TEST:
-			if (!selector || selector > 5)
-				goto error;
-			spin_unlock_irqrestore(&fotg210->lock, flags);
-			fotg210_quiesce(fotg210);
-			spin_lock_irqsave(&fotg210->lock, flags);
-
-			/* Put all enabled ports into suspend */
-			temp = fotg210_readl(fotg210, status_reg) &
-				~PORT_RWC_BITS;
-			if (temp & PORT_PE)
-				fotg210_writel(fotg210, temp | PORT_SUSPEND,
-						status_reg);
-
-			spin_unlock_irqrestore(&fotg210->lock, flags);
-			fotg210_halt(fotg210);
-			spin_lock_irqsave(&fotg210->lock, flags);
-
-			temp = fotg210_readl(fotg210, status_reg);
-			temp |= selector << 16;
-			fotg210_writel(fotg210, temp, status_reg);
-			break;
-
-		default:
-			goto error;
-		}
-		fotg210_readl(fotg210, &fotg210->regs->command);
-		break;
-
-	default:
-error:
-		/* "stall" on error */
-		retval = -EPIPE;
-	}
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	return retval;
-}
-
-static void __maybe_unused fotg210_relinquish_port(struct usb_hcd *hcd,
-		int portnum)
-{
-	return;
-}
-
-static int __maybe_unused fotg210_port_handed_over(struct usb_hcd *hcd,
-		int portnum)
-{
-	return 0;
-}
-
-/* There's basically three types of memory:
- *	- data used only by the HCD ... kmalloc is fine
- *	- async and periodic schedules, shared by HC and HCD ... these
- *	  need to use dma_pool or dma_alloc_coherent
- *	- driver buffers, read/written by HC ... single shot DMA mapped
- *
- * There's also "register" data (e.g. PCI or SOC), which is memory mapped.
- * No memory seen by this driver is pageable.
- */
-
-/* Allocate the key transfer structures from the previously allocated pool */
-static inline void fotg210_qtd_init(struct fotg210_hcd *fotg210,
-		struct fotg210_qtd *qtd, dma_addr_t dma)
-{
-	memset(qtd, 0, sizeof(*qtd));
-	qtd->qtd_dma = dma;
-	qtd->hw_token = cpu_to_hc32(fotg210, QTD_STS_HALT);
-	qtd->hw_next = FOTG210_LIST_END(fotg210);
-	qtd->hw_alt_next = FOTG210_LIST_END(fotg210);
-	INIT_LIST_HEAD(&qtd->qtd_list);
-}
-
-static struct fotg210_qtd *fotg210_qtd_alloc(struct fotg210_hcd *fotg210,
-		gfp_t flags)
-{
-	struct fotg210_qtd *qtd;
-	dma_addr_t dma;
-
-	qtd = dma_pool_alloc(fotg210->qtd_pool, flags, &dma);
-	if (qtd != NULL)
-		fotg210_qtd_init(fotg210, qtd, dma);
-
-	return qtd;
-}
-
-static inline void fotg210_qtd_free(struct fotg210_hcd *fotg210,
-		struct fotg210_qtd *qtd)
-{
-	dma_pool_free(fotg210->qtd_pool, qtd, qtd->qtd_dma);
-}
-
-
-static void qh_destroy(struct fotg210_hcd *fotg210, struct fotg210_qh *qh)
-{
-	/* clean qtds first, and know this is not linked */
-	if (!list_empty(&qh->qtd_list) || qh->qh_next.ptr) {
-		fotg210_dbg(fotg210, "unused qh not empty!\n");
-		BUG();
-	}
-	if (qh->dummy)
-		fotg210_qtd_free(fotg210, qh->dummy);
-	dma_pool_free(fotg210->qh_pool, qh->hw, qh->qh_dma);
-	kfree(qh);
-}
-
-static struct fotg210_qh *fotg210_qh_alloc(struct fotg210_hcd *fotg210,
-		gfp_t flags)
-{
-	struct fotg210_qh *qh;
-	dma_addr_t dma;
-
-	qh = kzalloc_obj(*qh, GFP_ATOMIC);
-	if (!qh)
-		goto done;
-	qh->hw = (struct fotg210_qh_hw *)
-		dma_pool_zalloc(fotg210->qh_pool, flags, &dma);
-	if (!qh->hw)
-		goto fail;
-	qh->qh_dma = dma;
-	INIT_LIST_HEAD(&qh->qtd_list);
-
-	/* dummy td enables safe urb queuing */
-	qh->dummy = fotg210_qtd_alloc(fotg210, flags);
-	if (qh->dummy == NULL) {
-		fotg210_dbg(fotg210, "no dummy td\n");
-		goto fail1;
-	}
-done:
-	return qh;
-fail1:
-	dma_pool_free(fotg210->qh_pool, qh->hw, qh->qh_dma);
-fail:
-	kfree(qh);
-	return NULL;
-}
-
-/* The queue heads and transfer descriptors are managed from pools tied
- * to each of the "per device" structures.
- * This is the initialisation and cleanup code.
- */
-
-static void fotg210_mem_cleanup(struct fotg210_hcd *fotg210)
-{
-	if (fotg210->async)
-		qh_destroy(fotg210, fotg210->async);
-	fotg210->async = NULL;
-
-	if (fotg210->dummy)
-		qh_destroy(fotg210, fotg210->dummy);
-	fotg210->dummy = NULL;
-
-	/* DMA consistent memory and pools */
-	dma_pool_destroy(fotg210->qtd_pool);
-	fotg210->qtd_pool = NULL;
-
-	dma_pool_destroy(fotg210->qh_pool);
-	fotg210->qh_pool = NULL;
-
-	dma_pool_destroy(fotg210->itd_pool);
-	fotg210->itd_pool = NULL;
-
-	if (fotg210->periodic)
-		dma_free_coherent(fotg210_to_hcd(fotg210)->self.controller,
-				fotg210->periodic_size * sizeof(u32),
-				fotg210->periodic, fotg210->periodic_dma);
-	fotg210->periodic = NULL;
-
-	/* shadow periodic table */
-	kfree(fotg210->pshadow);
-	fotg210->pshadow = NULL;
-}
-
-/* remember to add cleanup code (above) if you add anything here */
-static int fotg210_mem_init(struct fotg210_hcd *fotg210, gfp_t flags)
-{
-	int i;
-
-	/* QTDs for control/bulk/intr transfers */
-	fotg210->qtd_pool = dma_pool_create("fotg210_qtd",
-			fotg210_to_hcd(fotg210)->self.controller,
-			sizeof(struct fotg210_qtd),
-			32 /* byte alignment (for hw parts) */,
-			4096 /* can't cross 4K */);
-	if (!fotg210->qtd_pool)
-		goto fail;
-
-	/* QHs for control/bulk/intr transfers */
-	fotg210->qh_pool = dma_pool_create("fotg210_qh",
-			fotg210_to_hcd(fotg210)->self.controller,
-			sizeof(struct fotg210_qh_hw),
-			32 /* byte alignment (for hw parts) */,
-			4096 /* can't cross 4K */);
-	if (!fotg210->qh_pool)
-		goto fail;
-
-	fotg210->async = fotg210_qh_alloc(fotg210, flags);
-	if (!fotg210->async)
-		goto fail;
-
-	/* ITD for high speed ISO transfers */
-	fotg210->itd_pool = dma_pool_create("fotg210_itd",
-			fotg210_to_hcd(fotg210)->self.controller,
-			sizeof(struct fotg210_itd),
-			64 /* byte alignment (for hw parts) */,
-			4096 /* can't cross 4K */);
-	if (!fotg210->itd_pool)
-		goto fail;
-
-	/* Hardware periodic table */
-	fotg210->periodic =
-		dma_alloc_coherent(fotg210_to_hcd(fotg210)->self.controller,
-				fotg210->periodic_size * sizeof(__le32),
-				&fotg210->periodic_dma, 0);
-	if (fotg210->periodic == NULL)
-		goto fail;
-
-	for (i = 0; i < fotg210->periodic_size; i++)
-		fotg210->periodic[i] = FOTG210_LIST_END(fotg210);
-
-	/* software shadow of hardware table */
-	fotg210->pshadow = kcalloc(fotg210->periodic_size, sizeof(void *),
-			flags);
-	if (fotg210->pshadow != NULL)
-		return 0;
-
-fail:
-	fotg210_dbg(fotg210, "couldn't init memory\n");
-	fotg210_mem_cleanup(fotg210);
-	return -ENOMEM;
-}
-/* EHCI hardware queue manipulation ... the core.  QH/QTD manipulation.
- *
- * Control, bulk, and interrupt traffic all use "qh" lists.  They list "qtd"
- * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned
- * buffers needed for the larger number).  We use one QH per endpoint, queue
- * multiple urbs (all three types) per endpoint.  URBs may need several qtds.
- *
- * ISO traffic uses "ISO TD" (itd) records, and (along with
- * interrupts) needs careful scheduling.  Performance improvements can be
- * an ongoing challenge.  That's in "ehci-sched.c".
- *
- * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs,
- * or otherwise through transaction translators (TTs) in USB 2.0 hubs using
- * (b) special fields in qh entries or (c) split iso entries.  TTs will
- * buffer low/full speed data so the host collects it at high speed.
- */
-
-/* fill a qtd, returning how much of the buffer we were able to queue up */
-static int qtd_fill(struct fotg210_hcd *fotg210, struct fotg210_qtd *qtd,
-		dma_addr_t buf, size_t len, int token, int maxpacket)
-{
-	int i, count;
-	u64 addr = buf;
-
-	/* one buffer entry per 4K ... first might be short or unaligned */
-	qtd->hw_buf[0] = cpu_to_hc32(fotg210, (u32)addr);
-	qtd->hw_buf_hi[0] = cpu_to_hc32(fotg210, (u32)(addr >> 32));
-	count = 0x1000 - (buf & 0x0fff);	/* rest of that page */
-	if (likely(len < count))		/* ... iff needed */
-		count = len;
-	else {
-		buf +=  0x1000;
-		buf &= ~0x0fff;
-
-		/* per-qtd limit: from 16K to 20K (best alignment) */
-		for (i = 1; count < len && i < 5; i++) {
-			addr = buf;
-			qtd->hw_buf[i] = cpu_to_hc32(fotg210, (u32)addr);
-			qtd->hw_buf_hi[i] = cpu_to_hc32(fotg210,
-					(u32)(addr >> 32));
-			buf += 0x1000;
-			if ((count + 0x1000) < len)
-				count += 0x1000;
-			else
-				count = len;
-		}
-
-		/* short packets may only terminate transfers */
-		if (count != len)
-			count -= (count % maxpacket);
-	}
-	qtd->hw_token = cpu_to_hc32(fotg210, (count << 16) | token);
-	qtd->length = count;
-
-	return count;
-}
-
-static inline void qh_update(struct fotg210_hcd *fotg210,
-		struct fotg210_qh *qh, struct fotg210_qtd *qtd)
-{
-	struct fotg210_qh_hw *hw = qh->hw;
-
-	/* writes to an active overlay are unsafe */
-	BUG_ON(qh->qh_state != QH_STATE_IDLE);
-
-	hw->hw_qtd_next = QTD_NEXT(fotg210, qtd->qtd_dma);
-	hw->hw_alt_next = FOTG210_LIST_END(fotg210);
-
-	/* Except for control endpoints, we make hardware maintain data
-	 * toggle (like OHCI) ... here (re)initialize the toggle in the QH,
-	 * and set the pseudo-toggle in udev. Only usb_clear_halt() will
-	 * ever clear it.
-	 */
-	if (!(hw->hw_info1 & cpu_to_hc32(fotg210, QH_TOGGLE_CTL))) {
-		unsigned is_out, epnum;
-
-		is_out = qh->is_out;
-		epnum = (hc32_to_cpup(fotg210, &hw->hw_info1) >> 8) & 0x0f;
-		if (unlikely(!usb_gettoggle(qh->dev, epnum, is_out))) {
-			hw->hw_token &= ~cpu_to_hc32(fotg210, QTD_TOGGLE);
-			usb_settoggle(qh->dev, epnum, is_out, 1);
-		}
-	}
-
-	hw->hw_token &= cpu_to_hc32(fotg210, QTD_TOGGLE | QTD_STS_PING);
-}
-
-/* if it weren't for a common silicon quirk (writing the dummy into the qh
- * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault
- * recovery (including urb dequeue) would need software changes to a QH...
- */
-static void qh_refresh(struct fotg210_hcd *fotg210, struct fotg210_qh *qh)
-{
-	struct fotg210_qtd *qtd;
-
-	if (list_empty(&qh->qtd_list))
-		qtd = qh->dummy;
-	else {
-		qtd = list_entry(qh->qtd_list.next,
-				struct fotg210_qtd, qtd_list);
-		/*
-		 * first qtd may already be partially processed.
-		 * If we come here during unlink, the QH overlay region
-		 * might have reference to the just unlinked qtd. The
-		 * qtd is updated in qh_completions(). Update the QH
-		 * overlay here.
-		 */
-		if (cpu_to_hc32(fotg210, qtd->qtd_dma) == qh->hw->hw_current) {
-			qh->hw->hw_qtd_next = qtd->hw_next;
-			qtd = NULL;
-		}
-	}
-
-	if (qtd)
-		qh_update(fotg210, qh, qtd);
-}
-
-static void qh_link_async(struct fotg210_hcd *fotg210, struct fotg210_qh *qh);
-
-static void fotg210_clear_tt_buffer_complete(struct usb_hcd *hcd,
-		struct usb_host_endpoint *ep)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	struct fotg210_qh *qh = ep->hcpriv;
-	unsigned long flags;
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-	qh->clearing_tt = 0;
-	if (qh->qh_state == QH_STATE_IDLE && !list_empty(&qh->qtd_list)
-			&& fotg210->rh_state == FOTG210_RH_RUNNING)
-		qh_link_async(fotg210, qh);
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-}
-
-static void fotg210_clear_tt_buffer(struct fotg210_hcd *fotg210,
-		struct fotg210_qh *qh, struct urb *urb, u32 token)
-{
-
-	/* If an async split transaction gets an error or is unlinked,
-	 * the TT buffer may be left in an indeterminate state.  We
-	 * have to clear the TT buffer.
-	 *
-	 * Note: this routine is never called for Isochronous transfers.
-	 */
-	if (urb->dev->tt && !usb_pipeint(urb->pipe) && !qh->clearing_tt) {
-		struct usb_device *tt = urb->dev->tt->hub;
-
-		dev_dbg(&tt->dev,
-				"clear tt buffer port %d, a%d ep%d t%08x\n",
-				urb->dev->ttport, urb->dev->devnum,
-				usb_pipeendpoint(urb->pipe), token);
-
-		if (urb->dev->tt->hub !=
-				fotg210_to_hcd(fotg210)->self.root_hub) {
-			if (usb_hub_clear_tt_buffer(urb) == 0)
-				qh->clearing_tt = 1;
-		}
-	}
-}
-
-static int qtd_copy_status(struct fotg210_hcd *fotg210, struct urb *urb,
-		size_t length, u32 token)
-{
-	int status = -EINPROGRESS;
-
-	/* count IN/OUT bytes, not SETUP (even short packets) */
-	if (likely(QTD_PID(token) != 2))
-		urb->actual_length += length - QTD_LENGTH(token);
-
-	/* don't modify error codes */
-	if (unlikely(urb->unlinked))
-		return status;
-
-	/* force cleanup after short read; not always an error */
-	if (unlikely(IS_SHORT_READ(token)))
-		status = -EREMOTEIO;
-
-	/* serious "can't proceed" faults reported by the hardware */
-	if (token & QTD_STS_HALT) {
-		if (token & QTD_STS_BABBLE) {
-			/* FIXME "must" disable babbling device's port too */
-			status = -EOVERFLOW;
-		/* CERR nonzero + halt --> stall */
-		} else if (QTD_CERR(token)) {
-			status = -EPIPE;
-
-		/* In theory, more than one of the following bits can be set
-		 * since they are sticky and the transaction is retried.
-		 * Which to test first is rather arbitrary.
-		 */
-		} else if (token & QTD_STS_MMF) {
-			/* fs/ls interrupt xfer missed the complete-split */
-			status = -EPROTO;
-		} else if (token & QTD_STS_DBE) {
-			status = (QTD_PID(token) == 1) /* IN ? */
-				? -ENOSR  /* hc couldn't read data */
-				: -ECOMM; /* hc couldn't write data */
-		} else if (token & QTD_STS_XACT) {
-			/* timeout, bad CRC, wrong PID, etc */
-			fotg210_dbg(fotg210, "devpath %s ep%d%s 3strikes\n",
-					urb->dev->devpath,
-					usb_pipeendpoint(urb->pipe),
-					usb_pipein(urb->pipe) ? "in" : "out");
-			status = -EPROTO;
-		} else {	/* unknown */
-			status = -EPROTO;
-		}
-
-		fotg210_dbg(fotg210,
-				"dev%d ep%d%s qtd token %08x --> status %d\n",
-				usb_pipedevice(urb->pipe),
-				usb_pipeendpoint(urb->pipe),
-				usb_pipein(urb->pipe) ? "in" : "out",
-				token, status);
-	}
-
-	return status;
-}
-
-static void fotg210_urb_done(struct fotg210_hcd *fotg210, struct urb *urb,
-		int status)
-__releases(fotg210->lock)
-__acquires(fotg210->lock)
-{
-	if (likely(urb->hcpriv != NULL)) {
-		struct fotg210_qh *qh = (struct fotg210_qh *) urb->hcpriv;
-
-		/* S-mask in a QH means it's an interrupt urb */
-		if ((qh->hw->hw_info2 & cpu_to_hc32(fotg210, QH_SMASK)) != 0) {
-
-			/* ... update hc-wide periodic stats (for usbfs) */
-			fotg210_to_hcd(fotg210)->self.bandwidth_int_reqs--;
-		}
-	}
-
-	if (unlikely(urb->unlinked)) {
-		INCR(fotg210->stats.unlink);
-	} else {
-		/* report non-error and short read status as zero */
-		if (status == -EINPROGRESS || status == -EREMOTEIO)
-			status = 0;
-		INCR(fotg210->stats.complete);
-	}
-
-#ifdef FOTG210_URB_TRACE
-	fotg210_dbg(fotg210,
-			"%s %s urb %p ep%d%s status %d len %d/%d\n",
-			__func__, urb->dev->devpath, urb,
-			usb_pipeendpoint(urb->pipe),
-			usb_pipein(urb->pipe) ? "in" : "out",
-			status,
-			urb->actual_length, urb->transfer_buffer_length);
-#endif
-
-	/* complete() can reenter this HCD */
-	usb_hcd_unlink_urb_from_ep(fotg210_to_hcd(fotg210), urb);
-	spin_unlock(&fotg210->lock);
-	usb_hcd_giveback_urb(fotg210_to_hcd(fotg210), urb, status);
-	spin_lock(&fotg210->lock);
-}
-
-static int qh_schedule(struct fotg210_hcd *fotg210, struct fotg210_qh *qh);
-
-/* Process and free completed qtds for a qh, returning URBs to drivers.
- * Chases up to qh->hw_current.  Returns number of completions called,
- * indicating how much "real" work we did.
- */
-static unsigned qh_completions(struct fotg210_hcd *fotg210,
-		struct fotg210_qh *qh)
-{
-	struct fotg210_qtd *last, *end = qh->dummy;
-	struct fotg210_qtd *qtd, *tmp;
-	int last_status;
-	int stopped;
-	unsigned count = 0;
-	u8 state;
-	struct fotg210_qh_hw *hw = qh->hw;
-
-	if (unlikely(list_empty(&qh->qtd_list)))
-		return count;
-
-	/* completions (or tasks on other cpus) must never clobber HALT
-	 * till we've gone through and cleaned everything up, even when
-	 * they add urbs to this qh's queue or mark them for unlinking.
-	 *
-	 * NOTE:  unlinking expects to be done in queue order.
-	 *
-	 * It's a bug for qh->qh_state to be anything other than
-	 * QH_STATE_IDLE, unless our caller is scan_async() or
-	 * scan_intr().
-	 */
-	state = qh->qh_state;
-	qh->qh_state = QH_STATE_COMPLETING;
-	stopped = (state == QH_STATE_IDLE);
-
-rescan:
-	last = NULL;
-	last_status = -EINPROGRESS;
-	qh->needs_rescan = 0;
-
-	/* remove de-activated QTDs from front of queue.
-	 * after faults (including short reads), cleanup this urb
-	 * then let the queue advance.
-	 * if queue is stopped, handles unlinks.
-	 */
-	list_for_each_entry_safe(qtd, tmp, &qh->qtd_list, qtd_list) {
-		struct urb *urb;
-		u32 token = 0;
-
-		urb = qtd->urb;
-
-		/* clean up any state from previous QTD ...*/
-		if (last) {
-			if (likely(last->urb != urb)) {
-				fotg210_urb_done(fotg210, last->urb,
-						last_status);
-				count++;
-				last_status = -EINPROGRESS;
-			}
-			fotg210_qtd_free(fotg210, last);
-			last = NULL;
-		}
-
-		/* ignore urbs submitted during completions we reported */
-		if (qtd == end)
-			break;
-
-		/* hardware copies qtd out of qh overlay */
-		rmb();
-		token = hc32_to_cpu(fotg210, qtd->hw_token);
-
-		/* always clean up qtds the hc de-activated */
-retry_xacterr:
-		if ((token & QTD_STS_ACTIVE) == 0) {
-
-			/* Report Data Buffer Error: non-fatal but useful */
-			if (token & QTD_STS_DBE)
-				fotg210_dbg(fotg210,
-					"detected DataBufferErr for urb %p ep%d%s len %d, qtd %p [qh %p]\n",
-					urb, usb_endpoint_num(&urb->ep->desc),
-					usb_endpoint_dir_in(&urb->ep->desc)
-						? "in" : "out",
-					urb->transfer_buffer_length, qtd, qh);
-
-			/* on STALL, error, and short reads this urb must
-			 * complete and all its qtds must be recycled.
-			 */
-			if ((token & QTD_STS_HALT) != 0) {
-
-				/* retry transaction errors until we
-				 * reach the software xacterr limit
-				 */
-				if ((token & QTD_STS_XACT) &&
-						QTD_CERR(token) == 0 &&
-						++qh->xacterrs < QH_XACTERR_MAX &&
-						!urb->unlinked) {
-					fotg210_dbg(fotg210,
-						"detected XactErr len %zu/%zu retry %d\n",
-						qtd->length - QTD_LENGTH(token),
-						qtd->length,
-						qh->xacterrs);
-
-					/* reset the token in the qtd and the
-					 * qh overlay (which still contains
-					 * the qtd) so that we pick up from
-					 * where we left off
-					 */
-					token &= ~QTD_STS_HALT;
-					token |= QTD_STS_ACTIVE |
-						 (FOTG210_TUNE_CERR << 10);
-					qtd->hw_token = cpu_to_hc32(fotg210,
-							token);
-					wmb();
-					hw->hw_token = cpu_to_hc32(fotg210,
-							token);
-					goto retry_xacterr;
-				}
-				stopped = 1;
-
-			/* magic dummy for some short reads; qh won't advance.
-			 * that silicon quirk can kick in with this dummy too.
-			 *
-			 * other short reads won't stop the queue, including
-			 * control transfers (status stage handles that) or
-			 * most other single-qtd reads ... the queue stops if
-			 * URB_SHORT_NOT_OK was set so the driver submitting
-			 * the urbs could clean it up.
-			 */
-			} else if (IS_SHORT_READ(token) &&
-					!(qtd->hw_alt_next &
-					FOTG210_LIST_END(fotg210))) {
-				stopped = 1;
-			}
-
-		/* stop scanning when we reach qtds the hc is using */
-		} else if (likely(!stopped
-				&& fotg210->rh_state >= FOTG210_RH_RUNNING)) {
-			break;
-
-		/* scan the whole queue for unlinks whenever it stops */
-		} else {
-			stopped = 1;
-
-			/* cancel everything if we halt, suspend, etc */
-			if (fotg210->rh_state < FOTG210_RH_RUNNING)
-				last_status = -ESHUTDOWN;
-
-			/* this qtd is active; skip it unless a previous qtd
-			 * for its urb faulted, or its urb was canceled.
-			 */
-			else if (last_status == -EINPROGRESS && !urb->unlinked)
-				continue;
-
-			/* qh unlinked; token in overlay may be most current */
-			if (state == QH_STATE_IDLE &&
-					cpu_to_hc32(fotg210, qtd->qtd_dma)
-					== hw->hw_current) {
-				token = hc32_to_cpu(fotg210, hw->hw_token);
-
-				/* An unlink may leave an incomplete
-				 * async transaction in the TT buffer.
-				 * We have to clear it.
-				 */
-				fotg210_clear_tt_buffer(fotg210, qh, urb,
-						token);
-			}
-		}
-
-		/* unless we already know the urb's status, collect qtd status
-		 * and update count of bytes transferred.  in common short read
-		 * cases with only one data qtd (including control transfers),
-		 * queue processing won't halt.  but with two or more qtds (for
-		 * example, with a 32 KB transfer), when the first qtd gets a
-		 * short read the second must be removed by hand.
-		 */
-		if (last_status == -EINPROGRESS) {
-			last_status = qtd_copy_status(fotg210, urb,
-					qtd->length, token);
-			if (last_status == -EREMOTEIO &&
-					(qtd->hw_alt_next &
-					FOTG210_LIST_END(fotg210)))
-				last_status = -EINPROGRESS;
-
-			/* As part of low/full-speed endpoint-halt processing
-			 * we must clear the TT buffer (11.17.5).
-			 */
-			if (unlikely(last_status != -EINPROGRESS &&
-					last_status != -EREMOTEIO)) {
-				/* The TT's in some hubs malfunction when they
-				 * receive this request following a STALL (they
-				 * stop sending isochronous packets).  Since a
-				 * STALL can't leave the TT buffer in a busy
-				 * state (if you believe Figures 11-48 - 11-51
-				 * in the USB 2.0 spec), we won't clear the TT
-				 * buffer in this case.  Strictly speaking this
-				 * is a violation of the spec.
-				 */
-				if (last_status != -EPIPE)
-					fotg210_clear_tt_buffer(fotg210, qh,
-							urb, token);
-			}
-		}
-
-		/* if we're removing something not at the queue head,
-		 * patch the hardware queue pointer.
-		 */
-		if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {
-			last = list_entry(qtd->qtd_list.prev,
-					struct fotg210_qtd, qtd_list);
-			last->hw_next = qtd->hw_next;
-		}
-
-		/* remove qtd; it's recycled after possible urb completion */
-		list_del(&qtd->qtd_list);
-		last = qtd;
-
-		/* reinit the xacterr counter for the next qtd */
-		qh->xacterrs = 0;
-	}
-
-	/* last urb's completion might still need calling */
-	if (likely(last != NULL)) {
-		fotg210_urb_done(fotg210, last->urb, last_status);
-		count++;
-		fotg210_qtd_free(fotg210, last);
-	}
-
-	/* Do we need to rescan for URBs dequeued during a giveback? */
-	if (unlikely(qh->needs_rescan)) {
-		/* If the QH is already unlinked, do the rescan now. */
-		if (state == QH_STATE_IDLE)
-			goto rescan;
-
-		/* Otherwise we have to wait until the QH is fully unlinked.
-		 * Our caller will start an unlink if qh->needs_rescan is
-		 * set.  But if an unlink has already started, nothing needs
-		 * to be done.
-		 */
-		if (state != QH_STATE_LINKED)
-			qh->needs_rescan = 0;
-	}
-
-	/* restore original state; caller must unlink or relink */
-	qh->qh_state = state;
-
-	/* be sure the hardware's done with the qh before refreshing
-	 * it after fault cleanup, or recovering from silicon wrongly
-	 * overlaying the dummy qtd (which reduces DMA chatter).
-	 */
-	if (stopped != 0 || hw->hw_qtd_next == FOTG210_LIST_END(fotg210)) {
-		switch (state) {
-		case QH_STATE_IDLE:
-			qh_refresh(fotg210, qh);
-			break;
-		case QH_STATE_LINKED:
-			/* We won't refresh a QH that's linked (after the HC
-			 * stopped the queue).  That avoids a race:
-			 *  - HC reads first part of QH;
-			 *  - CPU updates that first part and the token;
-			 *  - HC reads rest of that QH, including token
-			 * Result:  HC gets an inconsistent image, and then
-			 * DMAs to/from the wrong memory (corrupting it).
-			 *
-			 * That should be rare for interrupt transfers,
-			 * except maybe high bandwidth ...
-			 */
-
-			/* Tell the caller to start an unlink */
-			qh->needs_rescan = 1;
-			break;
-		/* otherwise, unlink already started */
-		}
-	}
-
-	return count;
-}
-
-/* reverse of qh_urb_transaction:  free a list of TDs.
- * used for cleanup after errors, before HC sees an URB's TDs.
- */
-static void qtd_list_free(struct fotg210_hcd *fotg210, struct urb *urb,
-		struct list_head *head)
-{
-	struct fotg210_qtd *qtd, *temp;
-
-	list_for_each_entry_safe(qtd, temp, head, qtd_list) {
-		list_del(&qtd->qtd_list);
-		fotg210_qtd_free(fotg210, qtd);
-	}
-}
-
-/* create a list of filled qtds for this URB; won't link into qh.
- */
-static struct list_head *qh_urb_transaction(struct fotg210_hcd *fotg210,
-		struct urb *urb, struct list_head *head, gfp_t flags)
-{
-	struct fotg210_qtd *qtd, *qtd_prev;
-	dma_addr_t buf;
-	int len, this_sg_len, maxpacket;
-	int is_input;
-	u32 token;
-	int i;
-	struct scatterlist *sg;
-
-	/*
-	 * URBs map to sequences of QTDs:  one logical transaction
-	 */
-	qtd = fotg210_qtd_alloc(fotg210, flags);
-	if (unlikely(!qtd))
-		return NULL;
-	list_add_tail(&qtd->qtd_list, head);
-	qtd->urb = urb;
-
-	token = QTD_STS_ACTIVE;
-	token |= (FOTG210_TUNE_CERR << 10);
-	/* for split transactions, SplitXState initialized to zero */
-
-	len = urb->transfer_buffer_length;
-	is_input = usb_pipein(urb->pipe);
-	if (usb_pipecontrol(urb->pipe)) {
-		/* SETUP pid */
-		qtd_fill(fotg210, qtd, urb->setup_dma,
-				sizeof(struct usb_ctrlrequest),
-				token | (2 /* "setup" */ << 8), 8);
-
-		/* ... and always at least one more pid */
-		token ^= QTD_TOGGLE;
-		qtd_prev = qtd;
-		qtd = fotg210_qtd_alloc(fotg210, flags);
-		if (unlikely(!qtd))
-			goto cleanup;
-		qtd->urb = urb;
-		qtd_prev->hw_next = QTD_NEXT(fotg210, qtd->qtd_dma);
-		list_add_tail(&qtd->qtd_list, head);
-
-		/* for zero length DATA stages, STATUS is always IN */
-		if (len == 0)
-			token |= (1 /* "in" */ << 8);
-	}
-
-	/*
-	 * data transfer stage:  buffer setup
-	 */
-	i = urb->num_mapped_sgs;
-	if (len > 0 && i > 0) {
-		sg = urb->sg;
-		buf = sg_dma_address(sg);
-
-		/* urb->transfer_buffer_length may be smaller than the
-		 * size of the scatterlist (or vice versa)
-		 */
-		this_sg_len = min_t(int, sg_dma_len(sg), len);
-	} else {
-		sg = NULL;
-		buf = urb->transfer_dma;
-		this_sg_len = len;
-	}
-
-	if (is_input)
-		token |= (1 /* "in" */ << 8);
-	/* else it's already initted to "out" pid (0 << 8) */
-
-	maxpacket = usb_maxpacket(urb->dev, urb->pipe);
-
-	/*
-	 * buffer gets wrapped in one or more qtds;
-	 * last one may be "short" (including zero len)
-	 * and may serve as a control status ack
-	 */
-	for (;;) {
-		int this_qtd_len;
-
-		this_qtd_len = qtd_fill(fotg210, qtd, buf, this_sg_len, token,
-				maxpacket);
-		this_sg_len -= this_qtd_len;
-		len -= this_qtd_len;
-		buf += this_qtd_len;
-
-		/*
-		 * short reads advance to a "magic" dummy instead of the next
-		 * qtd ... that forces the queue to stop, for manual cleanup.
-		 * (this will usually be overridden later.)
-		 */
-		if (is_input)
-			qtd->hw_alt_next = fotg210->async->hw->hw_alt_next;
-
-		/* qh makes control packets use qtd toggle; maybe switch it */
-		if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0)
-			token ^= QTD_TOGGLE;
-
-		if (likely(this_sg_len <= 0)) {
-			if (--i <= 0 || len <= 0)
-				break;
-			sg = sg_next(sg);
-			buf = sg_dma_address(sg);
-			this_sg_len = min_t(int, sg_dma_len(sg), len);
-		}
-
-		qtd_prev = qtd;
-		qtd = fotg210_qtd_alloc(fotg210, flags);
-		if (unlikely(!qtd))
-			goto cleanup;
-		qtd->urb = urb;
-		qtd_prev->hw_next = QTD_NEXT(fotg210, qtd->qtd_dma);
-		list_add_tail(&qtd->qtd_list, head);
-	}
-
-	/*
-	 * unless the caller requires manual cleanup after short reads,
-	 * have the alt_next mechanism keep the queue running after the
-	 * last data qtd (the only one, for control and most other cases).
-	 */
-	if (likely((urb->transfer_flags & URB_SHORT_NOT_OK) == 0 ||
-			usb_pipecontrol(urb->pipe)))
-		qtd->hw_alt_next = FOTG210_LIST_END(fotg210);
-
-	/*
-	 * control requests may need a terminating data "status" ack;
-	 * other OUT ones may need a terminating short packet
-	 * (zero length).
-	 */
-	if (likely(urb->transfer_buffer_length != 0)) {
-		int one_more = 0;
-
-		if (usb_pipecontrol(urb->pipe)) {
-			one_more = 1;
-			token ^= 0x0100;	/* "in" <--> "out"  */
-			token |= QTD_TOGGLE;	/* force DATA1 */
-		} else if (usb_pipeout(urb->pipe)
-				&& (urb->transfer_flags & URB_ZERO_PACKET)
-				&& !(urb->transfer_buffer_length % maxpacket)) {
-			one_more = 1;
-		}
-		if (one_more) {
-			qtd_prev = qtd;
-			qtd = fotg210_qtd_alloc(fotg210, flags);
-			if (unlikely(!qtd))
-				goto cleanup;
-			qtd->urb = urb;
-			qtd_prev->hw_next = QTD_NEXT(fotg210, qtd->qtd_dma);
-			list_add_tail(&qtd->qtd_list, head);
-
-			/* never any data in such packets */
-			qtd_fill(fotg210, qtd, 0, 0, token, 0);
-		}
-	}
-
-	/* by default, enable interrupt on urb completion */
-	if (likely(!(urb->transfer_flags & URB_NO_INTERRUPT)))
-		qtd->hw_token |= cpu_to_hc32(fotg210, QTD_IOC);
-	return head;
-
-cleanup:
-	qtd_list_free(fotg210, urb, head);
-	return NULL;
-}
-
-/* Would be best to create all qh's from config descriptors,
- * when each interface/altsetting is established.  Unlink
- * any previous qh and cancel its urbs first; endpoints are
- * implicitly reset then (data toggle too).
- * That'd mean updating how usbcore talks to HCDs. (2.7?)
- */
-
-
-/* Each QH holds a qtd list; a QH is used for everything except iso.
- *
- * For interrupt urbs, the scheduler must set the microframe scheduling
- * mask(s) each time the QH gets scheduled.  For highspeed, that's
- * just one microframe in the s-mask.  For split interrupt transactions
- * there are additional complications: c-mask, maybe FSTNs.
- */
-static struct fotg210_qh *qh_make(struct fotg210_hcd *fotg210, struct urb *urb,
-		gfp_t flags)
-{
-	struct fotg210_qh *qh = fotg210_qh_alloc(fotg210, flags);
-	struct usb_host_endpoint *ep;
-	u32 info1 = 0, info2 = 0;
-	int is_input, type;
-	int maxp = 0;
-	int mult;
-	struct usb_tt *tt = urb->dev->tt;
-	struct fotg210_qh_hw *hw;
-
-	if (!qh)
-		return qh;
-
-	/*
-	 * init endpoint/device data for this QH
-	 */
-	info1 |= usb_pipeendpoint(urb->pipe) << 8;
-	info1 |= usb_pipedevice(urb->pipe) << 0;
-
-	is_input = usb_pipein(urb->pipe);
-	type = usb_pipetype(urb->pipe);
-	ep = usb_pipe_endpoint(urb->dev, urb->pipe);
-	maxp = usb_endpoint_maxp(&ep->desc);
-	mult = usb_endpoint_maxp_mult(&ep->desc);
-
-	/* 1024 byte maxpacket is a hardware ceiling.  High bandwidth
-	 * acts like up to 3KB, but is built from smaller packets.
-	 */
-	if (maxp > 1024) {
-		fotg210_dbg(fotg210, "bogus qh maxpacket %d\n", maxp);
-		goto done;
-	}
-
-	/* Compute interrupt scheduling parameters just once, and save.
-	 * - allowing for high bandwidth, how many nsec/uframe are used?
-	 * - split transactions need a second CSPLIT uframe; same question
-	 * - splits also need a schedule gap (for full/low speed I/O)
-	 * - qh has a polling interval
-	 *
-	 * For control/bulk requests, the HC or TT handles these.
-	 */
-	if (type == PIPE_INTERRUPT) {
-		qh->usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
-				is_input, 0, mult * maxp));
-		qh->start = NO_FRAME;
-
-		if (urb->dev->speed == USB_SPEED_HIGH) {
-			qh->c_usecs = 0;
-			qh->gap_uf = 0;
-
-			qh->period = urb->interval >> 3;
-			if (qh->period == 0 && urb->interval != 1) {
-				/* NOTE interval 2 or 4 uframes could work.
-				 * But interval 1 scheduling is simpler, and
-				 * includes high bandwidth.
-				 */
-				urb->interval = 1;
-			} else if (qh->period > fotg210->periodic_size) {
-				qh->period = fotg210->periodic_size;
-				urb->interval = qh->period << 3;
-			}
-		} else {
-			int think_time;
-
-			/* gap is f(FS/LS transfer times) */
-			qh->gap_uf = 1 + usb_calc_bus_time(urb->dev->speed,
-					is_input, 0, maxp) / (125 * 1000);
-
-			/* FIXME this just approximates SPLIT/CSPLIT times */
-			if (is_input) {		/* SPLIT, gap, CSPLIT+DATA */
-				qh->c_usecs = qh->usecs + HS_USECS(0);
-				qh->usecs = HS_USECS(1);
-			} else {		/* SPLIT+DATA, gap, CSPLIT */
-				qh->usecs += HS_USECS(1);
-				qh->c_usecs = HS_USECS(0);
-			}
-
-			think_time = tt ? tt->think_time : 0;
-			qh->tt_usecs = NS_TO_US(think_time +
-					usb_calc_bus_time(urb->dev->speed,
-					is_input, 0, maxp));
-			qh->period = urb->interval;
-			if (qh->period > fotg210->periodic_size) {
-				qh->period = fotg210->periodic_size;
-				urb->interval = qh->period;
-			}
-		}
-	}
-
-	/* support for tt scheduling, and access to toggles */
-	qh->dev = urb->dev;
-
-	/* using TT? */
-	switch (urb->dev->speed) {
-	case USB_SPEED_LOW:
-		info1 |= QH_LOW_SPEED;
-		fallthrough;
-
-	case USB_SPEED_FULL:
-		/* EPS 0 means "full" */
-		if (type != PIPE_INTERRUPT)
-			info1 |= (FOTG210_TUNE_RL_TT << 28);
-		if (type == PIPE_CONTROL) {
-			info1 |= QH_CONTROL_EP;		/* for TT */
-			info1 |= QH_TOGGLE_CTL;		/* toggle from qtd */
-		}
-		info1 |= maxp << 16;
-
-		info2 |= (FOTG210_TUNE_MULT_TT << 30);
-
-		/* Some Freescale processors have an erratum in which the
-		 * port number in the queue head was 0..N-1 instead of 1..N.
-		 */
-		if (fotg210_has_fsl_portno_bug(fotg210))
-			info2 |= (urb->dev->ttport-1) << 23;
-		else
-			info2 |= urb->dev->ttport << 23;
-
-		/* set the address of the TT; for TDI's integrated
-		 * root hub tt, leave it zeroed.
-		 */
-		if (tt && tt->hub != fotg210_to_hcd(fotg210)->self.root_hub)
-			info2 |= tt->hub->devnum << 16;
-
-		/* NOTE:  if (PIPE_INTERRUPT) { scheduler sets c-mask } */
-
-		break;
-
-	case USB_SPEED_HIGH:		/* no TT involved */
-		info1 |= QH_HIGH_SPEED;
-		if (type == PIPE_CONTROL) {
-			info1 |= (FOTG210_TUNE_RL_HS << 28);
-			info1 |= 64 << 16;	/* usb2 fixed maxpacket */
-			info1 |= QH_TOGGLE_CTL;	/* toggle from qtd */
-			info2 |= (FOTG210_TUNE_MULT_HS << 30);
-		} else if (type == PIPE_BULK) {
-			info1 |= (FOTG210_TUNE_RL_HS << 28);
-			/* The USB spec says that high speed bulk endpoints
-			 * always use 512 byte maxpacket.  But some device
-			 * vendors decided to ignore that, and MSFT is happy
-			 * to help them do so.  So now people expect to use
-			 * such nonconformant devices with Linux too; sigh.
-			 */
-			info1 |= maxp << 16;
-			info2 |= (FOTG210_TUNE_MULT_HS << 30);
-		} else {		/* PIPE_INTERRUPT */
-			info1 |= maxp << 16;
-			info2 |= mult << 30;
-		}
-		break;
-	default:
-		fotg210_dbg(fotg210, "bogus dev %p speed %d\n", urb->dev,
-				urb->dev->speed);
-done:
-		qh_destroy(fotg210, qh);
-		return NULL;
-	}
-
-	/* NOTE:  if (PIPE_INTERRUPT) { scheduler sets s-mask } */
-
-	/* init as live, toggle clear, advance to dummy */
-	qh->qh_state = QH_STATE_IDLE;
-	hw = qh->hw;
-	hw->hw_info1 = cpu_to_hc32(fotg210, info1);
-	hw->hw_info2 = cpu_to_hc32(fotg210, info2);
-	qh->is_out = !is_input;
-	usb_settoggle(urb->dev, usb_pipeendpoint(urb->pipe), !is_input, 1);
-	qh_refresh(fotg210, qh);
-	return qh;
-}
-
-static void enable_async(struct fotg210_hcd *fotg210)
-{
-	if (fotg210->async_count++)
-		return;
-
-	/* Stop waiting to turn off the async schedule */
-	fotg210->enabled_hrtimer_events &= ~BIT(FOTG210_HRTIMER_DISABLE_ASYNC);
-
-	/* Don't start the schedule until ASS is 0 */
-	fotg210_poll_ASS(fotg210);
-	turn_on_io_watchdog(fotg210);
-}
-
-static void disable_async(struct fotg210_hcd *fotg210)
-{
-	if (--fotg210->async_count)
-		return;
-
-	/* The async schedule and async_unlink list are supposed to be empty */
-	WARN_ON(fotg210->async->qh_next.qh || fotg210->async_unlink);
-
-	/* Don't turn off the schedule until ASS is 1 */
-	fotg210_poll_ASS(fotg210);
-}
-
-/* move qh (and its qtds) onto async queue; maybe enable queue.  */
-
-static void qh_link_async(struct fotg210_hcd *fotg210, struct fotg210_qh *qh)
-{
-	__hc32 dma = QH_NEXT(fotg210, qh->qh_dma);
-	struct fotg210_qh *head;
-
-	/* Don't link a QH if there's a Clear-TT-Buffer pending */
-	if (unlikely(qh->clearing_tt))
-		return;
-
-	WARN_ON(qh->qh_state != QH_STATE_IDLE);
-
-	/* clear halt and/or toggle; and maybe recover from silicon quirk */
-	qh_refresh(fotg210, qh);
-
-	/* splice right after start */
-	head = fotg210->async;
-	qh->qh_next = head->qh_next;
-	qh->hw->hw_next = head->hw->hw_next;
-	wmb();
-
-	head->qh_next.qh = qh;
-	head->hw->hw_next = dma;
-
-	qh->xacterrs = 0;
-	qh->qh_state = QH_STATE_LINKED;
-	/* qtd completions reported later by interrupt */
-
-	enable_async(fotg210);
-}
-
-/* For control/bulk/interrupt, return QH with these TDs appended.
- * Allocates and initializes the QH if necessary.
- * Returns null if it can't allocate a QH it needs to.
- * If the QH has TDs (urbs) already, that's great.
- */
-static struct fotg210_qh *qh_append_tds(struct fotg210_hcd *fotg210,
-		struct urb *urb, struct list_head *qtd_list,
-		int epnum, void **ptr)
-{
-	struct fotg210_qh *qh = NULL;
-	__hc32 qh_addr_mask = cpu_to_hc32(fotg210, 0x7f);
-
-	qh = (struct fotg210_qh *) *ptr;
-	if (unlikely(qh == NULL)) {
-		/* can't sleep here, we have fotg210->lock... */
-		qh = qh_make(fotg210, urb, GFP_ATOMIC);
-		*ptr = qh;
-	}
-	if (likely(qh != NULL)) {
-		struct fotg210_qtd *qtd;
-
-		if (unlikely(list_empty(qtd_list)))
-			qtd = NULL;
-		else
-			qtd = list_entry(qtd_list->next, struct fotg210_qtd,
-					qtd_list);
-
-		/* control qh may need patching ... */
-		if (unlikely(epnum == 0)) {
-			/* usb_reset_device() briefly reverts to address 0 */
-			if (usb_pipedevice(urb->pipe) == 0)
-				qh->hw->hw_info1 &= ~qh_addr_mask;
-		}
-
-		/* just one way to queue requests: swap with the dummy qtd.
-		 * only hc or qh_refresh() ever modify the overlay.
-		 */
-		if (likely(qtd != NULL)) {
-			struct fotg210_qtd *dummy;
-			dma_addr_t dma;
-			__hc32 token;
-
-			/* to avoid racing the HC, use the dummy td instead of
-			 * the first td of our list (becomes new dummy).  both
-			 * tds stay deactivated until we're done, when the
-			 * HC is allowed to fetch the old dummy (4.10.2).
-			 */
-			token = qtd->hw_token;
-			qtd->hw_token = HALT_BIT(fotg210);
-
-			dummy = qh->dummy;
-
-			dma = dummy->qtd_dma;
-			*dummy = *qtd;
-			dummy->qtd_dma = dma;
-
-			list_del(&qtd->qtd_list);
-			list_add(&dummy->qtd_list, qtd_list);
-			list_splice_tail(qtd_list, &qh->qtd_list);
-
-			fotg210_qtd_init(fotg210, qtd, qtd->qtd_dma);
-			qh->dummy = qtd;
-
-			/* hc must see the new dummy at list end */
-			dma = qtd->qtd_dma;
-			qtd = list_entry(qh->qtd_list.prev,
-					struct fotg210_qtd, qtd_list);
-			qtd->hw_next = QTD_NEXT(fotg210, dma);
-
-			/* let the hc process these next qtds */
-			wmb();
-			dummy->hw_token = token;
-
-			urb->hcpriv = qh;
-		}
-	}
-	return qh;
-}
-
-static int submit_async(struct fotg210_hcd *fotg210, struct urb *urb,
-		struct list_head *qtd_list, gfp_t mem_flags)
-{
-	int epnum;
-	unsigned long flags;
-	struct fotg210_qh *qh = NULL;
-	int rc;
-
-	epnum = urb->ep->desc.bEndpointAddress;
-
-#ifdef FOTG210_URB_TRACE
-	{
-		struct fotg210_qtd *qtd;
-
-		qtd = list_entry(qtd_list->next, struct fotg210_qtd, qtd_list);
-		fotg210_dbg(fotg210,
-				"%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
-				__func__, urb->dev->devpath, urb,
-				epnum & 0x0f, (epnum & USB_DIR_IN)
-					? "in" : "out",
-				urb->transfer_buffer_length,
-				qtd, urb->ep->hcpriv);
-	}
-#endif
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-	if (unlikely(!HCD_HW_ACCESSIBLE(fotg210_to_hcd(fotg210)))) {
-		rc = -ESHUTDOWN;
-		goto done;
-	}
-	rc = usb_hcd_link_urb_to_ep(fotg210_to_hcd(fotg210), urb);
-	if (unlikely(rc))
-		goto done;
-
-	qh = qh_append_tds(fotg210, urb, qtd_list, epnum, &urb->ep->hcpriv);
-	if (unlikely(qh == NULL)) {
-		usb_hcd_unlink_urb_from_ep(fotg210_to_hcd(fotg210), urb);
-		rc = -ENOMEM;
-		goto done;
-	}
-
-	/* Control/bulk operations through TTs don't need scheduling,
-	 * the HC and TT handle it when the TT has a buffer ready.
-	 */
-	if (likely(qh->qh_state == QH_STATE_IDLE))
-		qh_link_async(fotg210, qh);
-done:
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	if (unlikely(qh == NULL))
-		qtd_list_free(fotg210, urb, qtd_list);
-	return rc;
-}
-
-static void single_unlink_async(struct fotg210_hcd *fotg210,
-		struct fotg210_qh *qh)
-{
-	struct fotg210_qh *prev;
-
-	/* Add to the end of the list of QHs waiting for the next IAAD */
-	qh->qh_state = QH_STATE_UNLINK;
-	if (fotg210->async_unlink)
-		fotg210->async_unlink_last->unlink_next = qh;
-	else
-		fotg210->async_unlink = qh;
-	fotg210->async_unlink_last = qh;
-
-	/* Unlink it from the schedule */
-	prev = fotg210->async;
-	while (prev->qh_next.qh != qh)
-		prev = prev->qh_next.qh;
-
-	prev->hw->hw_next = qh->hw->hw_next;
-	prev->qh_next = qh->qh_next;
-	if (fotg210->qh_scan_next == qh)
-		fotg210->qh_scan_next = qh->qh_next.qh;
-}
-
-static void start_iaa_cycle(struct fotg210_hcd *fotg210, bool nested)
-{
-	/*
-	 * Do nothing if an IAA cycle is already running or
-	 * if one will be started shortly.
-	 */
-	if (fotg210->async_iaa || fotg210->async_unlinking)
-		return;
-
-	/* Do all the waiting QHs at once */
-	fotg210->async_iaa = fotg210->async_unlink;
-	fotg210->async_unlink = NULL;
-
-	/* If the controller isn't running, we don't have to wait for it */
-	if (unlikely(fotg210->rh_state < FOTG210_RH_RUNNING)) {
-		if (!nested)		/* Avoid recursion */
-			end_unlink_async(fotg210);
-
-	/* Otherwise start a new IAA cycle */
-	} else if (likely(fotg210->rh_state == FOTG210_RH_RUNNING)) {
-		/* Make sure the unlinks are all visible to the hardware */
-		wmb();
-
-		fotg210_writel(fotg210, fotg210->command | CMD_IAAD,
-				&fotg210->regs->command);
-		fotg210_readl(fotg210, &fotg210->regs->command);
-		fotg210_enable_event(fotg210, FOTG210_HRTIMER_IAA_WATCHDOG,
-				true);
-	}
-}
-
-/* the async qh for the qtds being unlinked are now gone from the HC */
-
-static void end_unlink_async(struct fotg210_hcd *fotg210)
-{
-	struct fotg210_qh *qh;
-
-	/* Process the idle QHs */
-restart:
-	fotg210->async_unlinking = true;
-	while (fotg210->async_iaa) {
-		qh = fotg210->async_iaa;
-		fotg210->async_iaa = qh->unlink_next;
-		qh->unlink_next = NULL;
-
-		qh->qh_state = QH_STATE_IDLE;
-		qh->qh_next.qh = NULL;
-
-		qh_completions(fotg210, qh);
-		if (!list_empty(&qh->qtd_list) &&
-				fotg210->rh_state == FOTG210_RH_RUNNING)
-			qh_link_async(fotg210, qh);
-		disable_async(fotg210);
-	}
-	fotg210->async_unlinking = false;
-
-	/* Start a new IAA cycle if any QHs are waiting for it */
-	if (fotg210->async_unlink) {
-		start_iaa_cycle(fotg210, true);
-		if (unlikely(fotg210->rh_state < FOTG210_RH_RUNNING))
-			goto restart;
-	}
-}
-
-static void unlink_empty_async(struct fotg210_hcd *fotg210)
-{
-	struct fotg210_qh *qh, *next;
-	bool stopped = (fotg210->rh_state < FOTG210_RH_RUNNING);
-	bool check_unlinks_later = false;
-
-	/* Unlink all the async QHs that have been empty for a timer cycle */
-	next = fotg210->async->qh_next.qh;
-	while (next) {
-		qh = next;
-		next = qh->qh_next.qh;
-
-		if (list_empty(&qh->qtd_list) &&
-				qh->qh_state == QH_STATE_LINKED) {
-			if (!stopped && qh->unlink_cycle ==
-					fotg210->async_unlink_cycle)
-				check_unlinks_later = true;
-			else
-				single_unlink_async(fotg210, qh);
-		}
-	}
-
-	/* Start a new IAA cycle if any QHs are waiting for it */
-	if (fotg210->async_unlink)
-		start_iaa_cycle(fotg210, false);
-
-	/* QHs that haven't been empty for long enough will be handled later */
-	if (check_unlinks_later) {
-		fotg210_enable_event(fotg210, FOTG210_HRTIMER_ASYNC_UNLINKS,
-				true);
-		++fotg210->async_unlink_cycle;
-	}
-}
-
-/* makes sure the async qh will become idle */
-/* caller must own fotg210->lock */
-
-static void start_unlink_async(struct fotg210_hcd *fotg210,
-		struct fotg210_qh *qh)
-{
-	/*
-	 * If the QH isn't linked then there's nothing we can do
-	 * unless we were called during a giveback, in which case
-	 * qh_completions() has to deal with it.
-	 */
-	if (qh->qh_state != QH_STATE_LINKED) {
-		if (qh->qh_state == QH_STATE_COMPLETING)
-			qh->needs_rescan = 1;
-		return;
-	}
-
-	single_unlink_async(fotg210, qh);
-	start_iaa_cycle(fotg210, false);
-}
-
-static void scan_async(struct fotg210_hcd *fotg210)
-{
-	struct fotg210_qh *qh;
-	bool check_unlinks_later = false;
-
-	fotg210->qh_scan_next = fotg210->async->qh_next.qh;
-	while (fotg210->qh_scan_next) {
-		qh = fotg210->qh_scan_next;
-		fotg210->qh_scan_next = qh->qh_next.qh;
-rescan:
-		/* clean any finished work for this qh */
-		if (!list_empty(&qh->qtd_list)) {
-			int temp;
-
-			/*
-			 * Unlinks could happen here; completion reporting
-			 * drops the lock.  That's why fotg210->qh_scan_next
-			 * always holds the next qh to scan; if the next qh
-			 * gets unlinked then fotg210->qh_scan_next is adjusted
-			 * in single_unlink_async().
-			 */
-			temp = qh_completions(fotg210, qh);
-			if (qh->needs_rescan) {
-				start_unlink_async(fotg210, qh);
-			} else if (list_empty(&qh->qtd_list)
-					&& qh->qh_state == QH_STATE_LINKED) {
-				qh->unlink_cycle = fotg210->async_unlink_cycle;
-				check_unlinks_later = true;
-			} else if (temp != 0)
-				goto rescan;
-		}
-	}
-
-	/*
-	 * Unlink empty entries, reducing DMA usage as well
-	 * as HCD schedule-scanning costs.  Delay for any qh
-	 * we just scanned, there's a not-unusual case that it
-	 * doesn't stay idle for long.
-	 */
-	if (check_unlinks_later && fotg210->rh_state == FOTG210_RH_RUNNING &&
-			!(fotg210->enabled_hrtimer_events &
-			BIT(FOTG210_HRTIMER_ASYNC_UNLINKS))) {
-		fotg210_enable_event(fotg210,
-				FOTG210_HRTIMER_ASYNC_UNLINKS, true);
-		++fotg210->async_unlink_cycle;
-	}
-}
-/* EHCI scheduled transaction support:  interrupt, iso, split iso
- * These are called "periodic" transactions in the EHCI spec.
- *
- * Note that for interrupt transfers, the QH/QTD manipulation is shared
- * with the "asynchronous" transaction support (control/bulk transfers).
- * The only real difference is in how interrupt transfers are scheduled.
- *
- * For ISO, we make an "iso_stream" head to serve the same role as a QH.
- * It keeps track of every ITD (or SITD) that's linked, and holds enough
- * pre-calculated schedule data to make appending to the queue be quick.
- */
-static int fotg210_get_frame(struct usb_hcd *hcd);
-
-/* periodic_next_shadow - return "next" pointer on shadow list
- * @periodic: host pointer to qh/itd
- * @tag: hardware tag for type of this record
- */
-static union fotg210_shadow *periodic_next_shadow(struct fotg210_hcd *fotg210,
-		union fotg210_shadow *periodic, __hc32 tag)
-{
-	switch (hc32_to_cpu(fotg210, tag)) {
-	case Q_TYPE_QH:
-		return &periodic->qh->qh_next;
-	case Q_TYPE_FSTN:
-		return &periodic->fstn->fstn_next;
-	default:
-		return &periodic->itd->itd_next;
-	}
-}
-
-static __hc32 *shadow_next_periodic(struct fotg210_hcd *fotg210,
-		union fotg210_shadow *periodic, __hc32 tag)
-{
-	switch (hc32_to_cpu(fotg210, tag)) {
-	/* our fotg210_shadow.qh is actually software part */
-	case Q_TYPE_QH:
-		return &periodic->qh->hw->hw_next;
-	/* others are hw parts */
-	default:
-		return periodic->hw_next;
-	}
-}
-
-/* caller must hold fotg210->lock */
-static void periodic_unlink(struct fotg210_hcd *fotg210, unsigned frame,
-		void *ptr)
-{
-	union fotg210_shadow *prev_p = &fotg210->pshadow[frame];
-	__hc32 *hw_p = &fotg210->periodic[frame];
-	union fotg210_shadow here = *prev_p;
-
-	/* find predecessor of "ptr"; hw and shadow lists are in sync */
-	while (here.ptr && here.ptr != ptr) {
-		prev_p = periodic_next_shadow(fotg210, prev_p,
-				Q_NEXT_TYPE(fotg210, *hw_p));
-		hw_p = shadow_next_periodic(fotg210, &here,
-				Q_NEXT_TYPE(fotg210, *hw_p));
-		here = *prev_p;
-	}
-	/* an interrupt entry (at list end) could have been shared */
-	if (!here.ptr)
-		return;
-
-	/* update shadow and hardware lists ... the old "next" pointers
-	 * from ptr may still be in use, the caller updates them.
-	 */
-	*prev_p = *periodic_next_shadow(fotg210, &here,
-			Q_NEXT_TYPE(fotg210, *hw_p));
-
-	*hw_p = *shadow_next_periodic(fotg210, &here,
-			Q_NEXT_TYPE(fotg210, *hw_p));
-}
-
-/* how many of the uframe's 125 usecs are allocated? */
-static unsigned short periodic_usecs(struct fotg210_hcd *fotg210,
-		unsigned frame, unsigned uframe)
-{
-	__hc32 *hw_p = &fotg210->periodic[frame];
-	union fotg210_shadow *q = &fotg210->pshadow[frame];
-	unsigned usecs = 0;
-	struct fotg210_qh_hw *hw;
-
-	while (q->ptr) {
-		switch (hc32_to_cpu(fotg210, Q_NEXT_TYPE(fotg210, *hw_p))) {
-		case Q_TYPE_QH:
-			hw = q->qh->hw;
-			/* is it in the S-mask? */
-			if (hw->hw_info2 & cpu_to_hc32(fotg210, 1 << uframe))
-				usecs += q->qh->usecs;
-			/* ... or C-mask? */
-			if (hw->hw_info2 & cpu_to_hc32(fotg210,
-					1 << (8 + uframe)))
-				usecs += q->qh->c_usecs;
-			hw_p = &hw->hw_next;
-			q = &q->qh->qh_next;
-			break;
-		/* case Q_TYPE_FSTN: */
-		default:
-			/* for "save place" FSTNs, count the relevant INTR
-			 * bandwidth from the previous frame
-			 */
-			if (q->fstn->hw_prev != FOTG210_LIST_END(fotg210))
-				fotg210_dbg(fotg210, "ignoring FSTN cost ...\n");
-
-			hw_p = &q->fstn->hw_next;
-			q = &q->fstn->fstn_next;
-			break;
-		case Q_TYPE_ITD:
-			if (q->itd->hw_transaction[uframe])
-				usecs += q->itd->stream->usecs;
-			hw_p = &q->itd->hw_next;
-			q = &q->itd->itd_next;
-			break;
-		}
-	}
-	if (usecs > fotg210->uframe_periodic_max)
-		fotg210_err(fotg210, "uframe %d sched overrun: %d usecs\n",
-				frame * 8 + uframe, usecs);
-	return usecs;
-}
-
-static int same_tt(struct usb_device *dev1, struct usb_device *dev2)
-{
-	if (!dev1->tt || !dev2->tt)
-		return 0;
-	if (dev1->tt != dev2->tt)
-		return 0;
-	if (dev1->tt->multi)
-		return dev1->ttport == dev2->ttport;
-	else
-		return 1;
-}
-
-/* return true iff the device's transaction translator is available
- * for a periodic transfer starting at the specified frame, using
- * all the uframes in the mask.
- */
-static int tt_no_collision(struct fotg210_hcd *fotg210, unsigned period,
-		struct usb_device *dev, unsigned frame, u32 uf_mask)
-{
-	if (period == 0)	/* error */
-		return 0;
-
-	/* note bandwidth wastage:  split never follows csplit
-	 * (different dev or endpoint) until the next uframe.
-	 * calling convention doesn't make that distinction.
-	 */
-	for (; frame < fotg210->periodic_size; frame += period) {
-		union fotg210_shadow here;
-		__hc32 type;
-		struct fotg210_qh_hw *hw;
-
-		here = fotg210->pshadow[frame];
-		type = Q_NEXT_TYPE(fotg210, fotg210->periodic[frame]);
-		while (here.ptr) {
-			switch (hc32_to_cpu(fotg210, type)) {
-			case Q_TYPE_ITD:
-				type = Q_NEXT_TYPE(fotg210, here.itd->hw_next);
-				here = here.itd->itd_next;
-				continue;
-			case Q_TYPE_QH:
-				hw = here.qh->hw;
-				if (same_tt(dev, here.qh->dev)) {
-					u32 mask;
-
-					mask = hc32_to_cpu(fotg210,
-							hw->hw_info2);
-					/* "knows" no gap is needed */
-					mask |= mask >> 8;
-					if (mask & uf_mask)
-						break;
-				}
-				type = Q_NEXT_TYPE(fotg210, hw->hw_next);
-				here = here.qh->qh_next;
-				continue;
-			/* case Q_TYPE_FSTN: */
-			default:
-				fotg210_dbg(fotg210,
-						"periodic frame %d bogus type %d\n",
-						frame, type);
-			}
-
-			/* collision or error */
-			return 0;
-		}
-	}
-
-	/* no collision */
-	return 1;
-}
-
-static void enable_periodic(struct fotg210_hcd *fotg210)
-{
-	if (fotg210->periodic_count++)
-		return;
-
-	/* Stop waiting to turn off the periodic schedule */
-	fotg210->enabled_hrtimer_events &=
-		~BIT(FOTG210_HRTIMER_DISABLE_PERIODIC);
-
-	/* Don't start the schedule until PSS is 0 */
-	fotg210_poll_PSS(fotg210);
-	turn_on_io_watchdog(fotg210);
-}
-
-static void disable_periodic(struct fotg210_hcd *fotg210)
-{
-	if (--fotg210->periodic_count)
-		return;
-
-	/* Don't turn off the schedule until PSS is 1 */
-	fotg210_poll_PSS(fotg210);
-}
-
-/* periodic schedule slots have iso tds (normal or split) first, then a
- * sparse tree for active interrupt transfers.
- *
- * this just links in a qh; caller guarantees uframe masks are set right.
- * no FSTN support (yet; fotg210 0.96+)
- */
-static void qh_link_periodic(struct fotg210_hcd *fotg210, struct fotg210_qh *qh)
-{
-	unsigned i;
-	unsigned period = qh->period;
-
-	dev_dbg(&qh->dev->dev,
-			"link qh%d-%04x/%p start %d [%d/%d us]\n", period,
-			hc32_to_cpup(fotg210, &qh->hw->hw_info2) &
-			(QH_CMASK | QH_SMASK), qh, qh->start, qh->usecs,
-			qh->c_usecs);
-
-	/* high bandwidth, or otherwise every microframe */
-	if (period == 0)
-		period = 1;
-
-	for (i = qh->start; i < fotg210->periodic_size; i += period) {
-		union fotg210_shadow *prev = &fotg210->pshadow[i];
-		__hc32 *hw_p = &fotg210->periodic[i];
-		union fotg210_shadow here = *prev;
-		__hc32 type = 0;
-
-		/* skip the iso nodes at list head */
-		while (here.ptr) {
-			type = Q_NEXT_TYPE(fotg210, *hw_p);
-			if (type == cpu_to_hc32(fotg210, Q_TYPE_QH))
-				break;
-			prev = periodic_next_shadow(fotg210, prev, type);
-			hw_p = shadow_next_periodic(fotg210, &here, type);
-			here = *prev;
-		}
-
-		/* sorting each branch by period (slow-->fast)
-		 * enables sharing interior tree nodes
-		 */
-		while (here.ptr && qh != here.qh) {
-			if (qh->period > here.qh->period)
-				break;
-			prev = &here.qh->qh_next;
-			hw_p = &here.qh->hw->hw_next;
-			here = *prev;
-		}
-		/* link in this qh, unless some earlier pass did that */
-		if (qh != here.qh) {
-			qh->qh_next = here;
-			if (here.qh)
-				qh->hw->hw_next = *hw_p;
-			wmb();
-			prev->qh = qh;
-			*hw_p = QH_NEXT(fotg210, qh->qh_dma);
-		}
-	}
-	qh->qh_state = QH_STATE_LINKED;
-	qh->xacterrs = 0;
-
-	/* update per-qh bandwidth for usbfs */
-	fotg210_to_hcd(fotg210)->self.bandwidth_allocated += qh->period
-		? ((qh->usecs + qh->c_usecs) / qh->period)
-		: (qh->usecs * 8);
-
-	list_add(&qh->intr_node, &fotg210->intr_qh_list);
-
-	/* maybe enable periodic schedule processing */
-	++fotg210->intr_count;
-	enable_periodic(fotg210);
-}
-
-static void qh_unlink_periodic(struct fotg210_hcd *fotg210,
-		struct fotg210_qh *qh)
-{
-	unsigned i;
-	unsigned period;
-
-	/*
-	 * If qh is for a low/full-speed device, simply unlinking it
-	 * could interfere with an ongoing split transaction.  To unlink
-	 * it safely would require setting the QH_INACTIVATE bit and
-	 * waiting at least one frame, as described in EHCI 4.12.2.5.
-	 *
-	 * We won't bother with any of this.  Instead, we assume that the
-	 * only reason for unlinking an interrupt QH while the current URB
-	 * is still active is to dequeue all the URBs (flush the whole
-	 * endpoint queue).
-	 *
-	 * If rebalancing the periodic schedule is ever implemented, this
-	 * approach will no longer be valid.
-	 */
-
-	/* high bandwidth, or otherwise part of every microframe */
-	period = qh->period;
-	if (!period)
-		period = 1;
-
-	for (i = qh->start; i < fotg210->periodic_size; i += period)
-		periodic_unlink(fotg210, i, qh);
-
-	/* update per-qh bandwidth for usbfs */
-	fotg210_to_hcd(fotg210)->self.bandwidth_allocated -= qh->period
-		? ((qh->usecs + qh->c_usecs) / qh->period)
-		: (qh->usecs * 8);
-
-	dev_dbg(&qh->dev->dev,
-			"unlink qh%d-%04x/%p start %d [%d/%d us]\n",
-			qh->period, hc32_to_cpup(fotg210, &qh->hw->hw_info2) &
-			(QH_CMASK | QH_SMASK), qh, qh->start, qh->usecs,
-			qh->c_usecs);
-
-	/* qh->qh_next still "live" to HC */
-	qh->qh_state = QH_STATE_UNLINK;
-	qh->qh_next.ptr = NULL;
-
-	if (fotg210->qh_scan_next == qh)
-		fotg210->qh_scan_next = list_entry(qh->intr_node.next,
-				struct fotg210_qh, intr_node);
-	list_del(&qh->intr_node);
-}
-
-static void start_unlink_intr(struct fotg210_hcd *fotg210,
-		struct fotg210_qh *qh)
-{
-	/* If the QH isn't linked then there's nothing we can do
-	 * unless we were called during a giveback, in which case
-	 * qh_completions() has to deal with it.
-	 */
-	if (qh->qh_state != QH_STATE_LINKED) {
-		if (qh->qh_state == QH_STATE_COMPLETING)
-			qh->needs_rescan = 1;
-		return;
-	}
-
-	qh_unlink_periodic(fotg210, qh);
-
-	/* Make sure the unlinks are visible before starting the timer */
-	wmb();
-
-	/*
-	 * The EHCI spec doesn't say how long it takes the controller to
-	 * stop accessing an unlinked interrupt QH.  The timer delay is
-	 * 9 uframes; presumably that will be long enough.
-	 */
-	qh->unlink_cycle = fotg210->intr_unlink_cycle;
-
-	/* New entries go at the end of the intr_unlink list */
-	if (fotg210->intr_unlink)
-		fotg210->intr_unlink_last->unlink_next = qh;
-	else
-		fotg210->intr_unlink = qh;
-	fotg210->intr_unlink_last = qh;
-
-	if (fotg210->intr_unlinking)
-		;	/* Avoid recursive calls */
-	else if (fotg210->rh_state < FOTG210_RH_RUNNING)
-		fotg210_handle_intr_unlinks(fotg210);
-	else if (fotg210->intr_unlink == qh) {
-		fotg210_enable_event(fotg210, FOTG210_HRTIMER_UNLINK_INTR,
-				true);
-		++fotg210->intr_unlink_cycle;
-	}
-}
-
-static void end_unlink_intr(struct fotg210_hcd *fotg210, struct fotg210_qh *qh)
-{
-	struct fotg210_qh_hw *hw = qh->hw;
-	int rc;
-
-	qh->qh_state = QH_STATE_IDLE;
-	hw->hw_next = FOTG210_LIST_END(fotg210);
-
-	qh_completions(fotg210, qh);
-
-	/* reschedule QH iff another request is queued */
-	if (!list_empty(&qh->qtd_list) &&
-			fotg210->rh_state == FOTG210_RH_RUNNING) {
-		rc = qh_schedule(fotg210, qh);
-
-		/* An error here likely indicates handshake failure
-		 * or no space left in the schedule.  Neither fault
-		 * should happen often ...
-		 *
-		 * FIXME kill the now-dysfunctional queued urbs
-		 */
-		if (rc != 0)
-			fotg210_err(fotg210, "can't reschedule qh %p, err %d\n",
-					qh, rc);
-	}
-
-	/* maybe turn off periodic schedule */
-	--fotg210->intr_count;
-	disable_periodic(fotg210);
-}
-
-static int check_period(struct fotg210_hcd *fotg210, unsigned frame,
-		unsigned uframe, unsigned period, unsigned usecs)
-{
-	int claimed;
-
-	/* complete split running into next frame?
-	 * given FSTN support, we could sometimes check...
-	 */
-	if (uframe >= 8)
-		return 0;
-
-	/* convert "usecs we need" to "max already claimed" */
-	usecs = fotg210->uframe_periodic_max - usecs;
-
-	/* we "know" 2 and 4 uframe intervals were rejected; so
-	 * for period 0, check _every_ microframe in the schedule.
-	 */
-	if (unlikely(period == 0)) {
-		do {
-			for (uframe = 0; uframe < 7; uframe++) {
-				claimed = periodic_usecs(fotg210, frame,
-						uframe);
-				if (claimed > usecs)
-					return 0;
-			}
-		} while ((frame += 1) < fotg210->periodic_size);
-
-	/* just check the specified uframe, at that period */
-	} else {
-		do {
-			claimed = periodic_usecs(fotg210, frame, uframe);
-			if (claimed > usecs)
-				return 0;
-		} while ((frame += period) < fotg210->periodic_size);
-	}
-
-	/* success! */
-	return 1;
-}
-
-static int check_intr_schedule(struct fotg210_hcd *fotg210, unsigned frame,
-		unsigned uframe, const struct fotg210_qh *qh, __hc32 *c_maskp)
-{
-	int retval = -ENOSPC;
-	u8 mask = 0;
-
-	if (qh->c_usecs && uframe >= 6)		/* FSTN territory? */
-		goto done;
-
-	if (!check_period(fotg210, frame, uframe, qh->period, qh->usecs))
-		goto done;
-	if (!qh->c_usecs) {
-		retval = 0;
-		*c_maskp = 0;
-		goto done;
-	}
-
-	/* Make sure this tt's buffer is also available for CSPLITs.
-	 * We pessimize a bit; probably the typical full speed case
-	 * doesn't need the second CSPLIT.
-	 *
-	 * NOTE:  both SPLIT and CSPLIT could be checked in just
-	 * one smart pass...
-	 */
-	mask = 0x03 << (uframe + qh->gap_uf);
-	*c_maskp = cpu_to_hc32(fotg210, mask << 8);
-
-	mask |= 1 << uframe;
-	if (tt_no_collision(fotg210, qh->period, qh->dev, frame, mask)) {
-		if (!check_period(fotg210, frame, uframe + qh->gap_uf + 1,
-				qh->period, qh->c_usecs))
-			goto done;
-		if (!check_period(fotg210, frame, uframe + qh->gap_uf,
-				qh->period, qh->c_usecs))
-			goto done;
-		retval = 0;
-	}
-done:
-	return retval;
-}
-
-/* "first fit" scheduling policy used the first time through,
- * or when the previous schedule slot can't be re-used.
- */
-static int qh_schedule(struct fotg210_hcd *fotg210, struct fotg210_qh *qh)
-{
-	int status;
-	unsigned uframe;
-	__hc32 c_mask;
-	unsigned frame;	/* 0..(qh->period - 1), or NO_FRAME */
-	struct fotg210_qh_hw *hw = qh->hw;
-
-	qh_refresh(fotg210, qh);
-	hw->hw_next = FOTG210_LIST_END(fotg210);
-	frame = qh->start;
-
-	/* reuse the previous schedule slots, if we can */
-	if (frame < qh->period) {
-		uframe = ffs(hc32_to_cpup(fotg210, &hw->hw_info2) & QH_SMASK);
-		status = check_intr_schedule(fotg210, frame, --uframe,
-				qh, &c_mask);
-	} else {
-		uframe = 0;
-		c_mask = 0;
-		status = -ENOSPC;
-	}
-
-	/* else scan the schedule to find a group of slots such that all
-	 * uframes have enough periodic bandwidth available.
-	 */
-	if (status) {
-		/* "normal" case, uframing flexible except with splits */
-		if (qh->period) {
-			int i;
-
-			for (i = qh->period; status && i > 0; --i) {
-				frame = ++fotg210->random_frame % qh->period;
-				for (uframe = 0; uframe < 8; uframe++) {
-					status = check_intr_schedule(fotg210,
-							frame, uframe, qh,
-							&c_mask);
-					if (status == 0)
-						break;
-				}
-			}
-
-		/* qh->period == 0 means every uframe */
-		} else {
-			frame = 0;
-			status = check_intr_schedule(fotg210, 0, 0, qh,
-					&c_mask);
-		}
-		if (status)
-			goto done;
-		qh->start = frame;
-
-		/* reset S-frame and (maybe) C-frame masks */
-		hw->hw_info2 &= cpu_to_hc32(fotg210, ~(QH_CMASK | QH_SMASK));
-		hw->hw_info2 |= qh->period
-			? cpu_to_hc32(fotg210, 1 << uframe)
-			: cpu_to_hc32(fotg210, QH_SMASK);
-		hw->hw_info2 |= c_mask;
-	} else
-		fotg210_dbg(fotg210, "reused qh %p schedule\n", qh);
-
-	/* stuff into the periodic schedule */
-	qh_link_periodic(fotg210, qh);
-done:
-	return status;
-}
-
-static int intr_submit(struct fotg210_hcd *fotg210, struct urb *urb,
-		struct list_head *qtd_list, gfp_t mem_flags)
-{
-	unsigned epnum;
-	unsigned long flags;
-	struct fotg210_qh *qh;
-	int status;
-	struct list_head empty;
-
-	/* get endpoint and transfer/schedule data */
-	epnum = urb->ep->desc.bEndpointAddress;
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-
-	if (unlikely(!HCD_HW_ACCESSIBLE(fotg210_to_hcd(fotg210)))) {
-		status = -ESHUTDOWN;
-		goto done_not_linked;
-	}
-	status = usb_hcd_link_urb_to_ep(fotg210_to_hcd(fotg210), urb);
-	if (unlikely(status))
-		goto done_not_linked;
-
-	/* get qh and force any scheduling errors */
-	INIT_LIST_HEAD(&empty);
-	qh = qh_append_tds(fotg210, urb, &empty, epnum, &urb->ep->hcpriv);
-	if (qh == NULL) {
-		status = -ENOMEM;
-		goto done;
-	}
-	if (qh->qh_state == QH_STATE_IDLE) {
-		status = qh_schedule(fotg210, qh);
-		if (status)
-			goto done;
-	}
-
-	/* then queue the urb's tds to the qh */
-	qh = qh_append_tds(fotg210, urb, qtd_list, epnum, &urb->ep->hcpriv);
-	BUG_ON(qh == NULL);
-
-	/* ... update usbfs periodic stats */
-	fotg210_to_hcd(fotg210)->self.bandwidth_int_reqs++;
-
-done:
-	if (unlikely(status))
-		usb_hcd_unlink_urb_from_ep(fotg210_to_hcd(fotg210), urb);
-done_not_linked:
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	if (status)
-		qtd_list_free(fotg210, urb, qtd_list);
-
-	return status;
-}
-
-static void scan_intr(struct fotg210_hcd *fotg210)
-{
-	struct fotg210_qh *qh;
-
-	list_for_each_entry_safe(qh, fotg210->qh_scan_next,
-			&fotg210->intr_qh_list, intr_node) {
-rescan:
-		/* clean any finished work for this qh */
-		if (!list_empty(&qh->qtd_list)) {
-			int temp;
-
-			/*
-			 * Unlinks could happen here; completion reporting
-			 * drops the lock.  That's why fotg210->qh_scan_next
-			 * always holds the next qh to scan; if the next qh
-			 * gets unlinked then fotg210->qh_scan_next is adjusted
-			 * in qh_unlink_periodic().
-			 */
-			temp = qh_completions(fotg210, qh);
-			if (unlikely(qh->needs_rescan ||
-					(list_empty(&qh->qtd_list) &&
-					qh->qh_state == QH_STATE_LINKED)))
-				start_unlink_intr(fotg210, qh);
-			else if (temp != 0)
-				goto rescan;
-		}
-	}
-}
-
-/* fotg210_iso_stream ops work with both ITD and SITD */
-
-static struct fotg210_iso_stream *iso_stream_alloc(gfp_t mem_flags)
-{
-	struct fotg210_iso_stream *stream;
-
-	stream = kzalloc_obj(*stream, mem_flags);
-	if (likely(stream != NULL)) {
-		INIT_LIST_HEAD(&stream->td_list);
-		INIT_LIST_HEAD(&stream->free_list);
-		stream->next_uframe = -1;
-	}
-	return stream;
-}
-
-static void iso_stream_init(struct fotg210_hcd *fotg210,
-		struct fotg210_iso_stream *stream, struct usb_device *dev,
-		int pipe, unsigned interval)
-{
-	u32 buf1;
-	unsigned epnum, maxp;
-	int is_input;
-	long bandwidth;
-	unsigned multi;
-	struct usb_host_endpoint *ep;
-
-	/*
-	 * this might be a "high bandwidth" highspeed endpoint,
-	 * as encoded in the ep descriptor's wMaxPacket field
-	 */
-	epnum = usb_pipeendpoint(pipe);
-	is_input = usb_pipein(pipe) ? USB_DIR_IN : 0;
-	ep = usb_pipe_endpoint(dev, pipe);
-	maxp = usb_endpoint_maxp(&ep->desc);
-	if (is_input)
-		buf1 = (1 << 11);
-	else
-		buf1 = 0;
-
-	multi = usb_endpoint_maxp_mult(&ep->desc);
-	buf1 |= maxp;
-	maxp *= multi;
-
-	stream->buf0 = cpu_to_hc32(fotg210, (epnum << 8) | dev->devnum);
-	stream->buf1 = cpu_to_hc32(fotg210, buf1);
-	stream->buf2 = cpu_to_hc32(fotg210, multi);
-
-	/* usbfs wants to report the average usecs per frame tied up
-	 * when transfers on this endpoint are scheduled ...
-	 */
-	if (dev->speed == USB_SPEED_FULL) {
-		interval <<= 3;
-		stream->usecs = NS_TO_US(usb_calc_bus_time(dev->speed,
-				is_input, 1, maxp));
-		stream->usecs /= 8;
-	} else {
-		stream->highspeed = 1;
-		stream->usecs = HS_USECS_ISO(maxp);
-	}
-	bandwidth = stream->usecs * 8;
-	bandwidth /= interval;
-
-	stream->bandwidth = bandwidth;
-	stream->udev = dev;
-	stream->bEndpointAddress = is_input | epnum;
-	stream->interval = interval;
-	stream->maxp = maxp;
-}
-
-static struct fotg210_iso_stream *iso_stream_find(struct fotg210_hcd *fotg210,
-		struct urb *urb)
-{
-	unsigned epnum;
-	struct fotg210_iso_stream *stream;
-	struct usb_host_endpoint *ep;
-	unsigned long flags;
-
-	epnum = usb_pipeendpoint(urb->pipe);
-	if (usb_pipein(urb->pipe))
-		ep = urb->dev->ep_in[epnum];
-	else
-		ep = urb->dev->ep_out[epnum];
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-	stream = ep->hcpriv;
-
-	if (unlikely(stream == NULL)) {
-		stream = iso_stream_alloc(GFP_ATOMIC);
-		if (likely(stream != NULL)) {
-			ep->hcpriv = stream;
-			stream->ep = ep;
-			iso_stream_init(fotg210, stream, urb->dev, urb->pipe,
-					urb->interval);
-		}
-
-	/* if dev->ep[epnum] is a QH, hw is set */
-	} else if (unlikely(stream->hw != NULL)) {
-		fotg210_dbg(fotg210, "dev %s ep%d%s, not iso??\n",
-				urb->dev->devpath, epnum,
-				usb_pipein(urb->pipe) ? "in" : "out");
-		stream = NULL;
-	}
-
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	return stream;
-}
-
-/* fotg210_iso_sched ops can be ITD-only or SITD-only */
-
-static struct fotg210_iso_sched *iso_sched_alloc(unsigned packets,
-		gfp_t mem_flags)
-{
-	struct fotg210_iso_sched *iso_sched;
-
-	iso_sched = kzalloc_flex(*iso_sched, packet, packets, mem_flags);
-	if (likely(iso_sched != NULL))
-		INIT_LIST_HEAD(&iso_sched->td_list);
-
-	return iso_sched;
-}
-
-static inline void itd_sched_init(struct fotg210_hcd *fotg210,
-		struct fotg210_iso_sched *iso_sched,
-		struct fotg210_iso_stream *stream, struct urb *urb)
-{
-	unsigned i;
-	dma_addr_t dma = urb->transfer_dma;
-
-	/* how many uframes are needed for these transfers */
-	iso_sched->span = urb->number_of_packets * stream->interval;
-
-	/* figure out per-uframe itd fields that we'll need later
-	 * when we fit new itds into the schedule.
-	 */
-	for (i = 0; i < urb->number_of_packets; i++) {
-		struct fotg210_iso_packet *uframe = &iso_sched->packet[i];
-		unsigned length;
-		dma_addr_t buf;
-		u32 trans;
-
-		length = urb->iso_frame_desc[i].length;
-		buf = dma + urb->iso_frame_desc[i].offset;
-
-		trans = FOTG210_ISOC_ACTIVE;
-		trans |= buf & 0x0fff;
-		if (unlikely(((i + 1) == urb->number_of_packets))
-				&& !(urb->transfer_flags & URB_NO_INTERRUPT))
-			trans |= FOTG210_ITD_IOC;
-		trans |= length << 16;
-		uframe->transaction = cpu_to_hc32(fotg210, trans);
-
-		/* might need to cross a buffer page within a uframe */
-		uframe->bufp = (buf & ~(u64)0x0fff);
-		buf += length;
-		if (unlikely((uframe->bufp != (buf & ~(u64)0x0fff))))
-			uframe->cross = 1;
-	}
-}
-
-static void iso_sched_free(struct fotg210_iso_stream *stream,
-		struct fotg210_iso_sched *iso_sched)
-{
-	if (!iso_sched)
-		return;
-	/* caller must hold fotg210->lock!*/
-	list_splice(&iso_sched->td_list, &stream->free_list);
-	kfree(iso_sched);
-}
-
-static int itd_urb_transaction(struct fotg210_iso_stream *stream,
-		struct fotg210_hcd *fotg210, struct urb *urb, gfp_t mem_flags)
-{
-	struct fotg210_itd *itd;
-	dma_addr_t itd_dma;
-	int i;
-	unsigned num_itds;
-	struct fotg210_iso_sched *sched;
-	unsigned long flags;
-
-	sched = iso_sched_alloc(urb->number_of_packets, mem_flags);
-	if (unlikely(sched == NULL))
-		return -ENOMEM;
-
-	itd_sched_init(fotg210, sched, stream, urb);
-
-	if (urb->interval < 8)
-		num_itds = 1 + (sched->span + 7) / 8;
-	else
-		num_itds = urb->number_of_packets;
-
-	/* allocate/init ITDs */
-	spin_lock_irqsave(&fotg210->lock, flags);
-	for (i = 0; i < num_itds; i++) {
-
-		/*
-		 * Use iTDs from the free list, but not iTDs that may
-		 * still be in use by the hardware.
-		 */
-		if (likely(!list_empty(&stream->free_list))) {
-			itd = list_first_entry(&stream->free_list,
-					struct fotg210_itd, itd_list);
-			if (itd->frame == fotg210->now_frame)
-				goto alloc_itd;
-			list_del(&itd->itd_list);
-			itd_dma = itd->itd_dma;
-		} else {
-alloc_itd:
-			spin_unlock_irqrestore(&fotg210->lock, flags);
-			itd = dma_pool_alloc(fotg210->itd_pool, mem_flags,
-					&itd_dma);
-			spin_lock_irqsave(&fotg210->lock, flags);
-			if (!itd) {
-				iso_sched_free(stream, sched);
-				spin_unlock_irqrestore(&fotg210->lock, flags);
-				return -ENOMEM;
-			}
-		}
-
-		memset(itd, 0, sizeof(*itd));
-		itd->itd_dma = itd_dma;
-		list_add(&itd->itd_list, &sched->td_list);
-	}
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-
-	/* temporarily store schedule info in hcpriv */
-	urb->hcpriv = sched;
-	urb->error_count = 0;
-	return 0;
-}
-
-static inline int itd_slot_ok(struct fotg210_hcd *fotg210, u32 mod, u32 uframe,
-		u8 usecs, u32 period)
-{
-	uframe %= period;
-	do {
-		/* can't commit more than uframe_periodic_max usec */
-		if (periodic_usecs(fotg210, uframe >> 3, uframe & 0x7)
-				> (fotg210->uframe_periodic_max - usecs))
-			return 0;
-
-		/* we know urb->interval is 2^N uframes */
-		uframe += period;
-	} while (uframe < mod);
-	return 1;
-}
-
-/* This scheduler plans almost as far into the future as it has actual
- * periodic schedule slots.  (Affected by TUNE_FLS, which defaults to
- * "as small as possible" to be cache-friendlier.)  That limits the size
- * transfers you can stream reliably; avoid more than 64 msec per urb.
- * Also avoid queue depths of less than fotg210's worst irq latency (affected
- * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
- * and other factors); or more than about 230 msec total (for portability,
- * given FOTG210_TUNE_FLS and the slop).  Or, write a smarter scheduler!
- */
-
-#define SCHEDULE_SLOP 80 /* microframes */
-
-static int iso_stream_schedule(struct fotg210_hcd *fotg210, struct urb *urb,
-		struct fotg210_iso_stream *stream)
-{
-	u32 now, next, start, period, span;
-	int status;
-	unsigned mod = fotg210->periodic_size << 3;
-	struct fotg210_iso_sched *sched = urb->hcpriv;
-
-	period = urb->interval;
-	span = sched->span;
-
-	if (span > mod - SCHEDULE_SLOP) {
-		fotg210_dbg(fotg210, "iso request %p too long\n", urb);
-		status = -EFBIG;
-		goto fail;
-	}
-
-	now = fotg210_read_frame_index(fotg210) & (mod - 1);
-
-	/* Typical case: reuse current schedule, stream is still active.
-	 * Hopefully there are no gaps from the host falling behind
-	 * (irq delays etc), but if there are we'll take the next
-	 * slot in the schedule, implicitly assuming URB_ISO_ASAP.
-	 */
-	if (likely(!list_empty(&stream->td_list))) {
-		u32 excess;
-
-		/* For high speed devices, allow scheduling within the
-		 * isochronous scheduling threshold.  For full speed devices
-		 * and Intel PCI-based controllers, don't (work around for
-		 * Intel ICH9 bug).
-		 */
-		if (!stream->highspeed && fotg210->fs_i_thresh)
-			next = now + fotg210->i_thresh;
-		else
-			next = now;
-
-		/* Fell behind (by up to twice the slop amount)?
-		 * We decide based on the time of the last currently-scheduled
-		 * slot, not the time of the next available slot.
-		 */
-		excess = (stream->next_uframe - period - next) & (mod - 1);
-		if (excess >= mod - 2 * SCHEDULE_SLOP)
-			start = next + excess - mod + period *
-					DIV_ROUND_UP(mod - excess, period);
-		else
-			start = next + excess + period;
-		if (start - now >= mod) {
-			fotg210_dbg(fotg210, "request %p would overflow (%d+%d >= %d)\n",
-					urb, start - now - period, period,
-					mod);
-			status = -EFBIG;
-			goto fail;
-		}
-	}
-
-	/* need to schedule; when's the next (u)frame we could start?
-	 * this is bigger than fotg210->i_thresh allows; scheduling itself
-	 * isn't free, the slop should handle reasonably slow cpus.  it
-	 * can also help high bandwidth if the dma and irq loads don't
-	 * jump until after the queue is primed.
-	 */
-	else {
-		int done = 0;
-
-		start = SCHEDULE_SLOP + (now & ~0x07);
-
-		/* NOTE:  assumes URB_ISO_ASAP, to limit complexity/bugs */
-
-		/* find a uframe slot with enough bandwidth.
-		 * Early uframes are more precious because full-speed
-		 * iso IN transfers can't use late uframes,
-		 * and therefore they should be allocated last.
-		 */
-		next = start;
-		start += period;
-		do {
-			start--;
-			/* check schedule: enough space? */
-			if (itd_slot_ok(fotg210, mod, start,
-					stream->usecs, period))
-				done = 1;
-		} while (start > next && !done);
-
-		/* no room in the schedule */
-		if (!done) {
-			fotg210_dbg(fotg210, "iso resched full %p (now %d max %d)\n",
-					urb, now, now + mod);
-			status = -ENOSPC;
-			goto fail;
-		}
-	}
-
-	/* Tried to schedule too far into the future? */
-	if (unlikely(start - now + span - period >=
-			mod - 2 * SCHEDULE_SLOP)) {
-		fotg210_dbg(fotg210, "request %p would overflow (%d+%d >= %d)\n",
-				urb, start - now, span - period,
-				mod - 2 * SCHEDULE_SLOP);
-		status = -EFBIG;
-		goto fail;
-	}
-
-	stream->next_uframe = start & (mod - 1);
-
-	/* report high speed start in uframes; full speed, in frames */
-	urb->start_frame = stream->next_uframe;
-	if (!stream->highspeed)
-		urb->start_frame >>= 3;
-
-	/* Make sure scan_isoc() sees these */
-	if (fotg210->isoc_count == 0)
-		fotg210->next_frame = now >> 3;
-	return 0;
-
-fail:
-	return status;
-}
-
-static inline void itd_init(struct fotg210_hcd *fotg210,
-		struct fotg210_iso_stream *stream, struct fotg210_itd *itd)
-{
-	int i;
-
-	/* it's been recently zeroed */
-	itd->hw_next = FOTG210_LIST_END(fotg210);
-	itd->hw_bufp[0] = stream->buf0;
-	itd->hw_bufp[1] = stream->buf1;
-	itd->hw_bufp[2] = stream->buf2;
-
-	for (i = 0; i < 8; i++)
-		itd->index[i] = -1;
-
-	/* All other fields are filled when scheduling */
-}
-
-static inline void itd_patch(struct fotg210_hcd *fotg210,
-		struct fotg210_itd *itd, struct fotg210_iso_sched *iso_sched,
-		unsigned index, u16 uframe)
-{
-	struct fotg210_iso_packet *uf = &iso_sched->packet[index];
-	unsigned pg = itd->pg;
-
-	uframe &= 0x07;
-	itd->index[uframe] = index;
-
-	itd->hw_transaction[uframe] = uf->transaction;
-	itd->hw_transaction[uframe] |= cpu_to_hc32(fotg210, pg << 12);
-	itd->hw_bufp[pg] |= cpu_to_hc32(fotg210, uf->bufp & ~(u32)0);
-	itd->hw_bufp_hi[pg] |= cpu_to_hc32(fotg210, (u32)(uf->bufp >> 32));
-
-	/* iso_frame_desc[].offset must be strictly increasing */
-	if (unlikely(uf->cross)) {
-		u64 bufp = uf->bufp + 4096;
-
-		itd->pg = ++pg;
-		itd->hw_bufp[pg] |= cpu_to_hc32(fotg210, bufp & ~(u32)0);
-		itd->hw_bufp_hi[pg] |= cpu_to_hc32(fotg210, (u32)(bufp >> 32));
-	}
-}
-
-static inline void itd_link(struct fotg210_hcd *fotg210, unsigned frame,
-		struct fotg210_itd *itd)
-{
-	union fotg210_shadow *prev = &fotg210->pshadow[frame];
-	__hc32 *hw_p = &fotg210->periodic[frame];
-	union fotg210_shadow here = *prev;
-	__hc32 type = 0;
-
-	/* skip any iso nodes which might belong to previous microframes */
-	while (here.ptr) {
-		type = Q_NEXT_TYPE(fotg210, *hw_p);
-		if (type == cpu_to_hc32(fotg210, Q_TYPE_QH))
-			break;
-		prev = periodic_next_shadow(fotg210, prev, type);
-		hw_p = shadow_next_periodic(fotg210, &here, type);
-		here = *prev;
-	}
-
-	itd->itd_next = here;
-	itd->hw_next = *hw_p;
-	prev->itd = itd;
-	itd->frame = frame;
-	wmb();
-	*hw_p = cpu_to_hc32(fotg210, itd->itd_dma | Q_TYPE_ITD);
-}
-
-/* fit urb's itds into the selected schedule slot; activate as needed */
-static void itd_link_urb(struct fotg210_hcd *fotg210, struct urb *urb,
-		unsigned mod, struct fotg210_iso_stream *stream)
-{
-	int packet;
-	unsigned next_uframe, uframe, frame;
-	struct fotg210_iso_sched *iso_sched = urb->hcpriv;
-	struct fotg210_itd *itd;
-
-	next_uframe = stream->next_uframe & (mod - 1);
-
-	if (unlikely(list_empty(&stream->td_list))) {
-		fotg210_to_hcd(fotg210)->self.bandwidth_allocated
-				+= stream->bandwidth;
-		fotg210_dbg(fotg210,
-			"schedule devp %s ep%d%s-iso period %d start %d.%d\n",
-			urb->dev->devpath, stream->bEndpointAddress & 0x0f,
-			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
-			urb->interval,
-			next_uframe >> 3, next_uframe & 0x7);
-	}
-
-	/* fill iTDs uframe by uframe */
-	for (packet = 0, itd = NULL; packet < urb->number_of_packets;) {
-		if (itd == NULL) {
-			/* ASSERT:  we have all necessary itds */
-
-			/* ASSERT:  no itds for this endpoint in this uframe */
-
-			itd = list_entry(iso_sched->td_list.next,
-					struct fotg210_itd, itd_list);
-			list_move_tail(&itd->itd_list, &stream->td_list);
-			itd->stream = stream;
-			itd->urb = urb;
-			itd_init(fotg210, stream, itd);
-		}
-
-		uframe = next_uframe & 0x07;
-		frame = next_uframe >> 3;
-
-		itd_patch(fotg210, itd, iso_sched, packet, uframe);
-
-		next_uframe += stream->interval;
-		next_uframe &= mod - 1;
-		packet++;
-
-		/* link completed itds into the schedule */
-		if (((next_uframe >> 3) != frame)
-				|| packet == urb->number_of_packets) {
-			itd_link(fotg210, frame & (fotg210->periodic_size - 1),
-					itd);
-			itd = NULL;
-		}
-	}
-	stream->next_uframe = next_uframe;
-
-	/* don't need that schedule data any more */
-	iso_sched_free(stream, iso_sched);
-	urb->hcpriv = NULL;
-
-	++fotg210->isoc_count;
-	enable_periodic(fotg210);
-}
-
-#define ISO_ERRS (FOTG210_ISOC_BUF_ERR | FOTG210_ISOC_BABBLE |\
-		FOTG210_ISOC_XACTERR)
-
-/* Process and recycle a completed ITD.  Return true iff its urb completed,
- * and hence its completion callback probably added things to the hardware
- * schedule.
- *
- * Note that we carefully avoid recycling this descriptor until after any
- * completion callback runs, so that it won't be reused quickly.  That is,
- * assuming (a) no more than two urbs per frame on this endpoint, and also
- * (b) only this endpoint's completions submit URBs.  It seems some silicon
- * corrupts things if you reuse completed descriptors very quickly...
- */
-static bool itd_complete(struct fotg210_hcd *fotg210, struct fotg210_itd *itd)
-{
-	struct urb *urb = itd->urb;
-	struct usb_iso_packet_descriptor *desc;
-	u32 t;
-	unsigned uframe;
-	int urb_index = -1;
-	struct fotg210_iso_stream *stream = itd->stream;
-	struct usb_device *dev;
-	bool retval = false;
-
-	/* for each uframe with a packet */
-	for (uframe = 0; uframe < 8; uframe++) {
-		if (likely(itd->index[uframe] == -1))
-			continue;
-		urb_index = itd->index[uframe];
-		desc = &urb->iso_frame_desc[urb_index];
-
-		t = hc32_to_cpup(fotg210, &itd->hw_transaction[uframe]);
-		itd->hw_transaction[uframe] = 0;
-
-		/* report transfer status */
-		if (unlikely(t & ISO_ERRS)) {
-			urb->error_count++;
-			if (t & FOTG210_ISOC_BUF_ERR)
-				desc->status = usb_pipein(urb->pipe)
-					? -ENOSR  /* hc couldn't read */
-					: -ECOMM; /* hc couldn't write */
-			else if (t & FOTG210_ISOC_BABBLE)
-				desc->status = -EOVERFLOW;
-			else /* (t & FOTG210_ISOC_XACTERR) */
-				desc->status = -EPROTO;
-
-			/* HC need not update length with this error */
-			if (!(t & FOTG210_ISOC_BABBLE)) {
-				desc->actual_length = FOTG210_ITD_LENGTH(t);
-				urb->actual_length += desc->actual_length;
-			}
-		} else if (likely((t & FOTG210_ISOC_ACTIVE) == 0)) {
-			desc->status = 0;
-			desc->actual_length = FOTG210_ITD_LENGTH(t);
-			urb->actual_length += desc->actual_length;
-		} else {
-			/* URB was too late */
-			desc->status = -EXDEV;
-		}
-	}
-
-	/* handle completion now? */
-	if (likely((urb_index + 1) != urb->number_of_packets))
-		goto done;
-
-	/* ASSERT: it's really the last itd for this urb
-	 * list_for_each_entry (itd, &stream->td_list, itd_list)
-	 *	BUG_ON (itd->urb == urb);
-	 */
-
-	/* give urb back to the driver; completion often (re)submits */
-	dev = urb->dev;
-	fotg210_urb_done(fotg210, urb, 0);
-	retval = true;
-	urb = NULL;
-
-	--fotg210->isoc_count;
-	disable_periodic(fotg210);
-
-	if (unlikely(list_is_singular(&stream->td_list))) {
-		fotg210_to_hcd(fotg210)->self.bandwidth_allocated
-				-= stream->bandwidth;
-		fotg210_dbg(fotg210,
-			"deschedule devp %s ep%d%s-iso\n",
-			dev->devpath, stream->bEndpointAddress & 0x0f,
-			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
-	}
+#include <linux/ioport.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/usb.h>
+#include <linux/usb/hcd.h>
 
-done:
-	itd->urb = NULL;
+#include "../host/ehci.h"
+#include "fotg210.h"
 
-	/* Add to the end of the free list for later reuse */
-	list_move_tail(&itd->itd_list, &stream->free_list);
+#define FOTG210_PORTSC			0x30
+#define FOTG210_MISC			0x40
+#define FOTG200_MISC_SPEED_MASK		GENMASK(10, 9)
+#define FOTG200_MISC_SPEED_SHIFT	9
+#define FOTG210_MISC_EOF1_MASK		GENMASK(3, 2)
+#define FOTG210_MISC_EOF1_720_21000_4000	(3 << 2)
+#define FOTG210_MISC_AS_SLP_MASK	GENMASK(1, 0)
+#define FOTG210_MISC_AS_SLP_10US	BIT(0)
+#define FOTG210_OTGCSR			0x80
+#define FOTG210_OTGCSR_SPEED_MASK	GENMASK(23, 22)
+#define FOTG210_OTGCSR_SPEED_SHIFT	22
+
+struct fotg210_hcd {
+	struct fotg210 *fotg;
+};
 
-	/* Recycle the iTDs when the pipeline is empty (ep no longer in use) */
-	if (list_empty(&stream->td_list)) {
-		list_splice_tail_init(&stream->free_list,
-				&fotg210->cached_itd_list);
-		start_free_itds(fotg210);
-	}
+#define hcd_to_fotg210_priv(hcd) \
+	((struct fotg210_hcd *)hcd_to_ehci(hcd)->priv)
 
-	return retval;
-}
+static struct hc_driver __read_mostly fotg210_hc_driver;
 
-static int itd_submit(struct fotg210_hcd *fotg210, struct urb *urb,
-		gfp_t mem_flags)
+static unsigned int fotg210_get_port_speed(struct ehci_hcd *ehci,
+					   unsigned int port)
 {
-	int status = -EINVAL;
-	unsigned long flags;
-	struct fotg210_iso_stream *stream;
+	struct fotg210 *fotg = hcd_to_fotg210_priv(ehci_to_hcd(ehci))->fotg;
+	u32 speed;
 
-	/* Get iso_stream head */
-	stream = iso_stream_find(fotg210, urb);
-	if (unlikely(stream == NULL)) {
-		fotg210_dbg(fotg210, "can't get iso stream\n");
-		return -ENOMEM;
-	}
-	if (unlikely(urb->interval != stream->interval &&
-			fotg210_port_speed(fotg210, 0) ==
-			USB_PORT_STAT_HIGH_SPEED)) {
-		fotg210_dbg(fotg210, "can't change iso interval %d --> %d\n",
-				stream->interval, urb->interval);
-		goto done;
-	}
-
-#ifdef FOTG210_URB_TRACE
-	fotg210_dbg(fotg210,
-			"%s %s urb %p ep%d%s len %d, %d pkts %d uframes[%p]\n",
-			__func__, urb->dev->devpath, urb,
-			usb_pipeendpoint(urb->pipe),
-			usb_pipein(urb->pipe) ? "in" : "out",
-			urb->transfer_buffer_length,
-			urb->number_of_packets, urb->interval,
-			stream);
-#endif
-
-	/* allocate ITDs w/o locking anything */
-	status = itd_urb_transaction(stream, fotg210, urb, mem_flags);
-	if (unlikely(status < 0)) {
-		fotg210_dbg(fotg210, "can't init itds\n");
-		goto done;
-	}
-
-	/* schedule ... need to lock */
-	spin_lock_irqsave(&fotg210->lock, flags);
-	if (unlikely(!HCD_HW_ACCESSIBLE(fotg210_to_hcd(fotg210)))) {
-		status = -ESHUTDOWN;
-		goto done_not_linked;
-	}
-	status = usb_hcd_link_urb_to_ep(fotg210_to_hcd(fotg210), urb);
-	if (unlikely(status))
-		goto done_not_linked;
-	status = iso_stream_schedule(fotg210, urb, stream);
-	if (likely(status == 0))
-		itd_link_urb(fotg210, urb, fotg210->periodic_size << 3, stream);
+	if (fotg->is_fotg210)
+		speed = (readl(fotg->base + FOTG210_OTGCSR) &
+			 FOTG210_OTGCSR_SPEED_MASK) >> FOTG210_OTGCSR_SPEED_SHIFT;
 	else
-		usb_hcd_unlink_urb_from_ep(fotg210_to_hcd(fotg210), urb);
-done_not_linked:
-	if (status < 0) {
-		iso_sched_free(stream, urb->hcpriv);
-		urb->hcpriv = NULL;
-	}
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-done:
-	return status;
-}
-
-static inline int scan_frame_queue(struct fotg210_hcd *fotg210, unsigned frame,
-		unsigned now_frame, bool live)
-{
-	unsigned uf;
-	bool modified;
-	union fotg210_shadow q, *q_p;
-	__hc32 type, *hw_p;
+		speed = (readl(fotg->base + FOTG210_MISC) &
+			 FOTG200_MISC_SPEED_MASK) >> FOTG200_MISC_SPEED_SHIFT;
 
-	/* scan each element in frame's queue for completions */
-	q_p = &fotg210->pshadow[frame];
-	hw_p = &fotg210->periodic[frame];
-	q.ptr = q_p->ptr;
-	type = Q_NEXT_TYPE(fotg210, *hw_p);
-	modified = false;
-
-	while (q.ptr) {
-		switch (hc32_to_cpu(fotg210, type)) {
-		case Q_TYPE_ITD:
-			/* If this ITD is still active, leave it for
-			 * later processing ... check the next entry.
-			 * No need to check for activity unless the
-			 * frame is current.
-			 */
-			if (frame == now_frame && live) {
-				rmb();
-				for (uf = 0; uf < 8; uf++) {
-					if (q.itd->hw_transaction[uf] &
-							ITD_ACTIVE(fotg210))
-						break;
-				}
-				if (uf < 8) {
-					q_p = &q.itd->itd_next;
-					hw_p = &q.itd->hw_next;
-					type = Q_NEXT_TYPE(fotg210,
-							q.itd->hw_next);
-					q = *q_p;
-					break;
-				}
-			}
-
-			/* Take finished ITDs out of the schedule
-			 * and process them:  recycle, maybe report
-			 * URB completion.  HC won't cache the
-			 * pointer for much longer, if at all.
-			 */
-			*q_p = q.itd->itd_next;
-			*hw_p = q.itd->hw_next;
-			type = Q_NEXT_TYPE(fotg210, q.itd->hw_next);
-			wmb();
-			modified = itd_complete(fotg210, q.itd);
-			q = *q_p;
-			break;
-		default:
-			fotg210_dbg(fotg210, "corrupt type %d frame %d shadow %p\n",
-					type, frame, q.ptr);
-			fallthrough;
-		case Q_TYPE_QH:
-		case Q_TYPE_FSTN:
-			/* End of the iTDs and siTDs */
-			q.ptr = NULL;
-			break;
-		}
-
-		/* assume completion callbacks modify the queue */
-		if (unlikely(modified && fotg210->isoc_count > 0))
-			return -EINVAL;
+	switch (speed) {
+	case 0:
+		return 0;
+	case 1:
+		return USB_PORT_STAT_LOW_SPEED;
+	case 2:
+	default:
+		return USB_PORT_STAT_HIGH_SPEED;
 	}
-	return 0;
 }
 
-static void scan_isoc(struct fotg210_hcd *fotg210)
+static int fotg210_pre_port_reset(struct ehci_hcd *ehci, unsigned int port)
 {
-	unsigned uf, now_frame, frame, ret;
-	unsigned fmask = fotg210->periodic_size - 1;
-	bool live;
-
-	/*
-	 * When running, scan from last scan point up to "now"
-	 * else clean up by scanning everything that's left.
-	 * Touches as few pages as possible:  cache-friendly.
-	 */
-	if (fotg210->rh_state >= FOTG210_RH_RUNNING) {
-		uf = fotg210_read_frame_index(fotg210);
-		now_frame = (uf >> 3) & fmask;
-		live = true;
-	} else  {
-		now_frame = (fotg210->next_frame - 1) & fmask;
-		live = false;
-	}
-	fotg210->now_frame = now_frame;
+	u32 command;
 
-	frame = fotg210->next_frame;
-	for (;;) {
-		ret = 1;
-		while (ret != 0)
-			ret = scan_frame_queue(fotg210, frame,
-					now_frame, live);
+	/* FOTG210 requires Run/Stop to be clear while PORTSC.Reset is set. */
+	command = ehci_readl(ehci, &ehci->regs->command);
+	ehci_writel(ehci, command & ~CMD_RUN, &ehci->regs->command);
 
-		/* Stop when we have reached the current frame */
-		if (frame == now_frame)
-			break;
-		frame = (frame + 1) & fmask;
-	}
-	fotg210->next_frame = now_frame;
+	return ehci_handshake(ehci, &ehci->regs->status,
+			      STS_HALT, STS_HALT, 16 * 125);
 }
 
-/* Display / Set uframe_periodic_max
- */
-static ssize_t uframe_periodic_max_show(struct device *dev,
-		struct device_attribute *attr, char *buf)
+static int fotg210_post_port_reset(struct ehci_hcd *ehci, unsigned int port)
 {
-	struct fotg210_hcd *fotg210;
+	ehci_writel(ehci, ehci->command | CMD_RUN, &ehci->regs->command);
 
-	fotg210 = hcd_to_fotg210(bus_to_hcd(dev_get_drvdata(dev)));
-	return sysfs_emit(buf, "%d\n", fotg210->uframe_periodic_max);
+	return ehci_handshake(ehci, &ehci->regs->status, STS_HALT, 0,
+			      16 * 125);
 }
 
-static ssize_t uframe_periodic_max_store(struct device *dev,
-		struct device_attribute *attr, const char *buf, size_t count)
+static int fotg210_hcd_reset(struct usb_hcd *hcd)
 {
-	struct fotg210_hcd *fotg210;
-	unsigned uframe_periodic_max;
-	unsigned frame, uframe;
-	unsigned short allocated_max;
-	unsigned long flags;
-	ssize_t ret;
+	struct fotg210 *fotg = hcd_to_fotg210_priv(hcd)->fotg;
+	struct ehci_hcd *ehci = hcd_to_ehci(hcd);
+	u32 val;
+	int ret;
 
-	fotg210 = hcd_to_fotg210(bus_to_hcd(dev_get_drvdata(dev)));
+	/* FOTG210 embeds an EHCI core but moves and omits some registers. */
+	ehci->port_status = hcd->regs + FOTG210_PORTSC;
+	ehci->get_port_speed = fotg210_get_port_speed;
+	ehci->pre_port_reset = fotg210_pre_port_reset;
+	ehci->post_port_reset = fotg210_post_port_reset;
+	ehci->no_configured_flag = 1;
+	ehci->no_tdi_mode = 1;
+	ehci->no_fsls_isoc = 1;
+	hcd->has_tt = 1;
 
-	ret = kstrtouint(buf, 0, &uframe_periodic_max);
+	ret = ehci_setup(hcd);
 	if (ret)
 		return ret;
 
-	if (uframe_periodic_max < 100 || uframe_periodic_max >= 125) {
-		fotg210_info(fotg210, "rejecting invalid request for uframe_periodic_max=%u\n",
-				uframe_periodic_max);
-		return -EINVAL;
-	}
-
-	ret = -EINVAL;
-
-	/*
-	 * lock, so that our checking does not race with possible periodic
-	 * bandwidth allocation through submitting new urbs.
-	 */
-	spin_lock_irqsave(&fotg210->lock, flags);
-
-	/*
-	 * for request to decrease max periodic bandwidth, we have to check
-	 * every microframe in the schedule to see whether the decrease is
-	 * possible.
-	 */
-	if (uframe_periodic_max < fotg210->uframe_periodic_max) {
-		allocated_max = 0;
-
-		for (frame = 0; frame < fotg210->periodic_size; ++frame)
-			for (uframe = 0; uframe < 7; ++uframe)
-				allocated_max = max(allocated_max,
-						periodic_usecs(fotg210, frame,
-						uframe));
-
-		if (allocated_max > uframe_periodic_max) {
-			fotg210_info(fotg210,
-					"cannot decrease uframe_periodic_max because periodic bandwidth is already allocated (%u > %u)\n",
-					allocated_max, uframe_periodic_max);
-			goto out_unlock;
-		}
-	}
-
-	/* increasing is always ok */
-
-	fotg210_info(fotg210,
-			"setting max periodic bandwidth to %u%% (== %u usec/uframe)\n",
-			100 * uframe_periodic_max/125, uframe_periodic_max);
-
-	if (uframe_periodic_max != 100)
-		fotg210_warn(fotg210, "max periodic bandwidth set is non-standard\n");
-
-	fotg210->uframe_periodic_max = uframe_periodic_max;
-	ret = count;
-
-out_unlock:
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	return ret;
-}
-
-static DEVICE_ATTR_RW(uframe_periodic_max);
-
-static inline int create_sysfs_files(struct fotg210_hcd *fotg210)
-{
-	struct device *controller = fotg210_to_hcd(fotg210)->self.controller;
-
-	return device_create_file(controller, &dev_attr_uframe_periodic_max);
-}
-
-static inline void remove_sysfs_files(struct fotg210_hcd *fotg210)
-{
-	struct device *controller = fotg210_to_hcd(fotg210)->self.controller;
-
-	device_remove_file(controller, &dev_attr_uframe_periodic_max);
-}
-/* On some systems, leaving remote wakeup enabled prevents system shutdown.
- * The firmware seems to think that powering off is a wakeup event!
- * This routine turns off remote wakeup and everything else, on all ports.
- */
-static void fotg210_turn_off_all_ports(struct fotg210_hcd *fotg210)
-{
-	u32 __iomem *status_reg = &fotg210->regs->port_status;
-
-	fotg210_writel(fotg210, PORT_RWC_BITS, status_reg);
-}
-
-/* Halt HC, turn off all ports, and let the BIOS use the companion controllers.
- * Must be called with interrupts enabled and the lock not held.
- */
-static void fotg210_silence_controller(struct fotg210_hcd *fotg210)
-{
-	fotg210_halt(fotg210);
-
-	spin_lock_irq(&fotg210->lock);
-	fotg210->rh_state = FOTG210_RH_HALTED;
-	fotg210_turn_off_all_ports(fotg210);
-	spin_unlock_irq(&fotg210->lock);
-}
-
-/* fotg210_shutdown kick in for silicon on any bus (not just pci, etc).
- * This forcibly disables dma and IRQs, helping kexec and other cases
- * where the next system software may expect clean state.
- */
-static void fotg210_shutdown(struct usb_hcd *hcd)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-
-	spin_lock_irq(&fotg210->lock);
-	fotg210->shutdown = true;
-	fotg210->rh_state = FOTG210_RH_STOPPING;
-	fotg210->enabled_hrtimer_events = 0;
-	spin_unlock_irq(&fotg210->lock);
-
-	fotg210_silence_controller(fotg210);
-
-	hrtimer_cancel(&fotg210->hrtimer);
-}
-
-/* fotg210_work is called from some interrupts, timers, and so on.
- * it calls driver completion functions, after dropping fotg210->lock.
- */
-static void fotg210_work(struct fotg210_hcd *fotg210)
-{
-	/* another CPU may drop fotg210->lock during a schedule scan while
-	 * it reports urb completions.  this flag guards against bogus
-	 * attempts at re-entrant schedule scanning.
-	 */
-	if (fotg210->scanning) {
-		fotg210->need_rescan = true;
-		return;
-	}
-	fotg210->scanning = true;
-
-rescan:
-	fotg210->need_rescan = false;
-	if (fotg210->async_count)
-		scan_async(fotg210);
-	if (fotg210->intr_count > 0)
-		scan_intr(fotg210);
-	if (fotg210->isoc_count > 0)
-		scan_isoc(fotg210);
-	if (fotg210->need_rescan)
-		goto rescan;
-	fotg210->scanning = false;
-
-	/* the IO watchdog guards against hardware or driver bugs that
-	 * misplace IRQs, and should let us run completely without IRQs.
-	 * such lossage has been observed on both VT6202 and VT8235.
-	 */
-	turn_on_io_watchdog(fotg210);
-}
-
-/* Called when the fotg210_hcd module is removed.
- */
-static void fotg210_stop(struct usb_hcd *hcd)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-
-	fotg210_dbg(fotg210, "stop\n");
-
-	/* no more interrupts ... */
-
-	spin_lock_irq(&fotg210->lock);
-	fotg210->enabled_hrtimer_events = 0;
-	spin_unlock_irq(&fotg210->lock);
-
-	fotg210_quiesce(fotg210);
-	fotg210_silence_controller(fotg210);
-	fotg210_reset(fotg210);
-
-	hrtimer_cancel(&fotg210->hrtimer);
-	remove_sysfs_files(fotg210);
-	remove_debug_files(fotg210);
-
-	/* root hub is shut down separately (first, when possible) */
-	spin_lock_irq(&fotg210->lock);
-	end_free_itds(fotg210);
-	spin_unlock_irq(&fotg210->lock);
-	fotg210_mem_cleanup(fotg210);
-
-#ifdef FOTG210_STATS
-	fotg210_dbg(fotg210, "irq normal %ld err %ld iaa %ld (lost %ld)\n",
-			fotg210->stats.normal, fotg210->stats.error,
-			fotg210->stats.iaa, fotg210->stats.lost_iaa);
-	fotg210_dbg(fotg210, "complete %ld unlink %ld\n",
-			fotg210->stats.complete, fotg210->stats.unlink);
-#endif
-
-	dbg_status(fotg210, "fotg210_stop completed",
-			fotg210_readl(fotg210, &fotg210->regs->status));
-}
-
-/* one-time init, only for memory state */
-static int hcd_fotg210_init(struct usb_hcd *hcd)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	u32 temp;
-	int retval;
-	u32 hcc_params;
-	struct fotg210_qh_hw *hw;
-
-	spin_lock_init(&fotg210->lock);
-
-	/*
-	 * keep io watchdog by default, those good HCDs could turn off it later
-	 */
-	fotg210->need_io_watchdog = 1;
-
-	hrtimer_setup(&fotg210->hrtimer, fotg210_hrtimer_func, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
-	fotg210->next_hrtimer_event = FOTG210_HRTIMER_NO_EVENT;
-
-	hcc_params = fotg210_readl(fotg210, &fotg210->caps->hcc_params);
-
-	/*
-	 * by default set standard 80% (== 100 usec/uframe) max periodic
-	 * bandwidth as required by USB 2.0
-	 */
-	fotg210->uframe_periodic_max = 100;
-
-	/*
-	 * hw default: 1K periodic list heads, one per frame.
-	 * periodic_size can shrink by USBCMD update if hcc_params allows.
-	 */
-	fotg210->periodic_size = DEFAULT_I_TDPS;
-	INIT_LIST_HEAD(&fotg210->intr_qh_list);
-	INIT_LIST_HEAD(&fotg210->cached_itd_list);
-
-	if (HCC_PGM_FRAMELISTLEN(hcc_params)) {
-		/* periodic schedule size can be smaller than default */
-		switch (FOTG210_TUNE_FLS) {
-		case 0:
-			fotg210->periodic_size = 1024;
-			break;
-		case 1:
-			fotg210->periodic_size = 512;
-			break;
-		case 2:
-			fotg210->periodic_size = 256;
-			break;
-		default:
-			BUG();
-		}
-	}
-	retval = fotg210_mem_init(fotg210, GFP_KERNEL);
-	if (retval < 0)
-		return retval;
-
-	/* controllers may cache some of the periodic schedule ... */
-	fotg210->i_thresh = 2;
-
-	/*
-	 * dedicate a qh for the async ring head, since we couldn't unlink
-	 * a 'real' qh without stopping the async schedule [4.8].  use it
-	 * as the 'reclamation list head' too.
-	 * its dummy is used in hw_alt_next of many tds, to prevent the qh
-	 * from automatically advancing to the next td after short reads.
-	 */
-	fotg210->async->qh_next.qh = NULL;
-	hw = fotg210->async->hw;
-	hw->hw_next = QH_NEXT(fotg210, fotg210->async->qh_dma);
-	hw->hw_info1 = cpu_to_hc32(fotg210, QH_HEAD);
-	hw->hw_token = cpu_to_hc32(fotg210, QTD_STS_HALT);
-	hw->hw_qtd_next = FOTG210_LIST_END(fotg210);
-	fotg210->async->qh_state = QH_STATE_LINKED;
-	hw->hw_alt_next = QTD_NEXT(fotg210, fotg210->async->dummy->qtd_dma);
-
-	/* clear interrupt enables, set irq latency */
-	if (log2_irq_thresh < 0 || log2_irq_thresh > 6)
-		log2_irq_thresh = 0;
-	temp = 1 << (16 + log2_irq_thresh);
-	if (HCC_CANPARK(hcc_params)) {
-		/* HW default park == 3, on hardware that supports it (like
-		 * NVidia and ALI silicon), maximizes throughput on the async
-		 * schedule by avoiding QH fetches between transfers.
-		 *
-		 * With fast usb storage devices and NForce2, "park" seems to
-		 * make problems:  throughput reduction (!), data errors...
-		 */
-		if (park) {
-			park = min_t(unsigned, park, 3);
-			temp |= CMD_PARK;
-			temp |= park << 8;
-		}
-		fotg210_dbg(fotg210, "park %d\n", park);
+	if (fotg->port != GEMINI_PORT_NONE) {
+		/* Work around the Gemini full-speed EOF1 timing erratum. */
+		val = readl(fotg->base + FOTG210_MISC);
+		val &= ~(FOTG210_MISC_EOF1_MASK | FOTG210_MISC_AS_SLP_MASK);
+		val |= FOTG210_MISC_EOF1_720_21000_4000 |
+		       FOTG210_MISC_AS_SLP_10US;
+		writel(val, fotg->base + FOTG210_MISC);
 	}
-	if (HCC_PGM_FRAMELISTLEN(hcc_params)) {
-		/* periodic schedule size can be smaller than default */
-		temp &= ~(3 << 2);
-		temp |= (FOTG210_TUNE_FLS << 2);
-	}
-	fotg210->command = temp;
-
-	/* Accept arbitrarily long scatter-gather lists */
-	if (!hcd->localmem_pool)
-		hcd->self.sg_tablesize = ~0;
-	return 0;
-}
-
-/* start HC running; it's halted, hcd_fotg210_init() has been run (once) */
-static int fotg210_run(struct usb_hcd *hcd)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	u32 temp;
-
-	hcd->uses_new_polling = 1;
-
-	/* EHCI spec section 4.1 */
-
-	fotg210_writel(fotg210, fotg210->periodic_dma,
-			&fotg210->regs->frame_list);
-	fotg210_writel(fotg210, (u32)fotg210->async->qh_dma,
-			&fotg210->regs->async_next);
-
-	/*
-	 * hcc_params controls whether fotg210->regs->segment must (!!!)
-	 * be used; it constrains QH/ITD/SITD and QTD locations.
-	 * dma_pool consistent memory always uses segment zero.
-	 * streaming mappings for I/O buffers, like dma_map_single(),
-	 * can return segments above 4GB, if the device allows.
-	 *
-	 * NOTE:  the dma mask is visible through dev->dma_mask, so
-	 * drivers can pass this info along ... like NETIF_F_HIGHDMA,
-	 * Scsi_Host.highmem_io, and so forth.  It's readonly to all
-	 * host side drivers though.
-	 */
-	fotg210_readl(fotg210, &fotg210->caps->hcc_params);
-
-	/*
-	 * Philips, Intel, and maybe others need CMD_RUN before the
-	 * root hub will detect new devices (why?); NEC doesn't
-	 */
-	fotg210->command &= ~(CMD_IAAD|CMD_PSE|CMD_ASE|CMD_RESET);
-	fotg210->command |= CMD_RUN;
-	fotg210_writel(fotg210, fotg210->command, &fotg210->regs->command);
-	dbg_cmd(fotg210, "init", fotg210->command);
-
-	/*
-	 * Start, enabling full USB 2.0 functionality ... usb 1.1 devices
-	 * are explicitly handed to companion controller(s), so no TT is
-	 * involved with the root hub.  (Except where one is integrated,
-	 * and there's no companion controller unless maybe for USB OTG.)
-	 *
-	 * Turning on the CF flag will transfer ownership of all ports
-	 * from the companions to the EHCI controller.  If any of the
-	 * companions are in the middle of a port reset at the time, it
-	 * could cause trouble.  Write-locking ehci_cf_port_reset_rwsem
-	 * guarantees that no resets are in progress.  After we set CF,
-	 * a short delay lets the hardware catch up; new resets shouldn't
-	 * be started before the port switching actions could complete.
-	 */
-	down_write(&ehci_cf_port_reset_rwsem);
-	fotg210->rh_state = FOTG210_RH_RUNNING;
-	/* unblock posted writes */
-	fotg210_readl(fotg210, &fotg210->regs->command);
-	usleep_range(5000, 10000);
-	up_write(&ehci_cf_port_reset_rwsem);
-	fotg210->last_periodic_enable = ktime_get_real();
-
-	temp = HC_VERSION(fotg210,
-			fotg210_readl(fotg210, &fotg210->caps->hc_capbase));
-	fotg210_info(fotg210,
-			"USB %x.%x started, EHCI %x.%02x\n",
-			((fotg210->sbrn & 0xf0) >> 4), (fotg210->sbrn & 0x0f),
-			temp >> 8, temp & 0xff);
-
-	fotg210_writel(fotg210, INTR_MASK,
-			&fotg210->regs->intr_enable); /* Turn On Interrupts */
-
-	/* GRR this is run-once init(), being done every time the HC starts.
-	 * So long as they're part of class devices, we can't do it init()
-	 * since the class device isn't created that early.
-	 */
-	create_debug_files(fotg210);
-	create_sysfs_files(fotg210);
-
-	return 0;
-}
-
-static int fotg210_setup(struct usb_hcd *hcd)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	int retval;
-
-	fotg210->regs = (void __iomem *)fotg210->caps +
-			HC_LENGTH(fotg210,
-			fotg210_readl(fotg210, &fotg210->caps->hc_capbase));
-	dbg_hcs_params(fotg210, "reset");
-	dbg_hcc_params(fotg210, "reset");
-
-	/* cache this readonly data; minimize chip reads */
-	fotg210->hcs_params = fotg210_readl(fotg210,
-			&fotg210->caps->hcs_params);
-
-	fotg210->sbrn = HCD_USB2;
-
-	/* data structure init */
-	retval = hcd_fotg210_init(hcd);
-	if (retval)
-		return retval;
-
-	retval = fotg210_halt(fotg210);
-	if (retval)
-		return retval;
-
-	fotg210_reset(fotg210);
 
 	return 0;
 }
 
-static irqreturn_t fotg210_irq(struct usb_hcd *hcd)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	u32 status, masked_status, pcd_status = 0, cmd;
-	int bh;
-
-	spin_lock(&fotg210->lock);
-
-	status = fotg210_readl(fotg210, &fotg210->regs->status);
-
-	/* e.g. cardbus physical eject */
-	if (status == ~(u32) 0) {
-		fotg210_dbg(fotg210, "device removed\n");
-		goto dead;
-	}
-
-	/*
-	 * We don't use STS_FLR, but some controllers don't like it to
-	 * remain on, so mask it out along with the other status bits.
-	 */
-	masked_status = status & (INTR_MASK | STS_FLR);
-
-	/* Shared IRQ? */
-	if (!masked_status ||
-			unlikely(fotg210->rh_state == FOTG210_RH_HALTED)) {
-		spin_unlock(&fotg210->lock);
-		return IRQ_NONE;
-	}
-
-	/* clear (just) interrupts */
-	fotg210_writel(fotg210, masked_status, &fotg210->regs->status);
-	cmd = fotg210_readl(fotg210, &fotg210->regs->command);
-	bh = 0;
-
-	/* unrequested/ignored: Frame List Rollover */
-	dbg_status(fotg210, "irq", status);
-
-	/* INT, ERR, and IAA interrupt rates can be throttled */
-
-	/* normal [4.15.1.2] or error [4.15.1.1] completion */
-	if (likely((status & (STS_INT|STS_ERR)) != 0)) {
-		if (likely((status & STS_ERR) == 0))
-			INCR(fotg210->stats.normal);
-		else
-			INCR(fotg210->stats.error);
-		bh = 1;
-	}
-
-	/* complete the unlinking of some qh [4.15.2.3] */
-	if (status & STS_IAA) {
-
-		/* Turn off the IAA watchdog */
-		fotg210->enabled_hrtimer_events &=
-			~BIT(FOTG210_HRTIMER_IAA_WATCHDOG);
-
-		/*
-		 * Mild optimization: Allow another IAAD to reset the
-		 * hrtimer, if one occurs before the next expiration.
-		 * In theory we could always cancel the hrtimer, but
-		 * tests show that about half the time it will be reset
-		 * for some other event anyway.
-		 */
-		if (fotg210->next_hrtimer_event == FOTG210_HRTIMER_IAA_WATCHDOG)
-			++fotg210->next_hrtimer_event;
-
-		/* guard against (alleged) silicon errata */
-		if (cmd & CMD_IAAD)
-			fotg210_dbg(fotg210, "IAA with IAAD still set?\n");
-		if (fotg210->async_iaa) {
-			INCR(fotg210->stats.iaa);
-			end_unlink_async(fotg210);
-		} else
-			fotg210_dbg(fotg210, "IAA with nothing unlinked?\n");
-	}
-
-	/* remote wakeup [4.3.1] */
-	if (status & STS_PCD) {
-		int pstatus;
-		u32 __iomem *status_reg = &fotg210->regs->port_status;
-
-		/* kick root hub later */
-		pcd_status = status;
-
-		/* resume root hub? */
-		if (fotg210->rh_state == FOTG210_RH_SUSPENDED)
-			usb_hcd_resume_root_hub(hcd);
-
-		pstatus = fotg210_readl(fotg210, status_reg);
-
-		if (test_bit(0, &fotg210->suspended_ports) &&
-				((pstatus & PORT_RESUME) ||
-				!(pstatus & PORT_SUSPEND)) &&
-				(pstatus & PORT_PE) &&
-				fotg210->reset_done[0] == 0) {
-
-			/* start 20 msec resume signaling from this port,
-			 * and make hub_wq collect PORT_STAT_C_SUSPEND to
-			 * stop that signaling.  Use 5 ms extra for safety,
-			 * like usb_port_resume() does.
-			 */
-			fotg210->reset_done[0] = jiffies + msecs_to_jiffies(25);
-			set_bit(0, &fotg210->resuming_ports);
-			fotg210_dbg(fotg210, "port 1 remote wakeup\n");
-			mod_timer(&hcd->rh_timer, fotg210->reset_done[0]);
-		}
-	}
-
-	/* PCI errors [4.15.2.4] */
-	if (unlikely((status & STS_FATAL) != 0)) {
-		fotg210_err(fotg210, "fatal error\n");
-		dbg_cmd(fotg210, "fatal", cmd);
-		dbg_status(fotg210, "fatal", status);
-dead:
-		usb_hc_died(hcd);
-
-		/* Don't let the controller do anything more */
-		fotg210->shutdown = true;
-		fotg210->rh_state = FOTG210_RH_STOPPING;
-		fotg210->command &= ~(CMD_RUN | CMD_ASE | CMD_PSE);
-		fotg210_writel(fotg210, fotg210->command,
-				&fotg210->regs->command);
-		fotg210_writel(fotg210, 0, &fotg210->regs->intr_enable);
-		fotg210_handle_controller_death(fotg210);
-
-		/* Handle completions when the controller stops */
-		bh = 0;
-	}
-
-	if (bh)
-		fotg210_work(fotg210);
-	spin_unlock(&fotg210->lock);
-	if (pcd_status)
-		usb_hcd_poll_rh_status(hcd);
-	return IRQ_HANDLED;
-}
-
-/* non-error returns are a promise to giveback() the urb later
- * we drop ownership so next owner (or urb unlink) can get it
- *
- * urb + dev is in hcd.self.controller.urb_list
- * we're queueing TDs onto software and hardware lists
- *
- * hcd-specific init for hcpriv hasn't been done yet
- *
- * NOTE:  control, bulk, and interrupt share the same code to append TDs
- * to a (possibly active) QH, and the same QH scanning code.
- */
-static int fotg210_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
-		gfp_t mem_flags)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	struct list_head qtd_list;
-
-	INIT_LIST_HEAD(&qtd_list);
-
-	switch (usb_pipetype(urb->pipe)) {
-	case PIPE_CONTROL:
-		/* qh_completions() code doesn't handle all the fault cases
-		 * in multi-TD control transfers.  Even 1KB is rare anyway.
-		 */
-		if (urb->transfer_buffer_length > (16 * 1024))
-			return -EMSGSIZE;
-		fallthrough;
-	/* case PIPE_BULK: */
-	default:
-		if (!qh_urb_transaction(fotg210, urb, &qtd_list, mem_flags))
-			return -ENOMEM;
-		return submit_async(fotg210, urb, &qtd_list, mem_flags);
-
-	case PIPE_INTERRUPT:
-		if (!qh_urb_transaction(fotg210, urb, &qtd_list, mem_flags))
-			return -ENOMEM;
-		return intr_submit(fotg210, urb, &qtd_list, mem_flags);
-
-	case PIPE_ISOCHRONOUS:
-		return itd_submit(fotg210, urb, mem_flags);
-	}
-}
-
-/* remove from hardware lists
- * completions normally happen asynchronously
- */
-
-static int fotg210_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	struct fotg210_qh *qh;
-	unsigned long flags;
-	int rc;
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-	rc = usb_hcd_check_unlink_urb(hcd, urb, status);
-	if (rc)
-		goto done;
-
-	switch (usb_pipetype(urb->pipe)) {
-	/* case PIPE_CONTROL: */
-	/* case PIPE_BULK:*/
-	default:
-		qh = (struct fotg210_qh *) urb->hcpriv;
-		if (!qh)
-			break;
-		switch (qh->qh_state) {
-		case QH_STATE_LINKED:
-		case QH_STATE_COMPLETING:
-			start_unlink_async(fotg210, qh);
-			break;
-		case QH_STATE_UNLINK:
-		case QH_STATE_UNLINK_WAIT:
-			/* already started */
-			break;
-		case QH_STATE_IDLE:
-			/* QH might be waiting for a Clear-TT-Buffer */
-			qh_completions(fotg210, qh);
-			break;
-		}
-		break;
-
-	case PIPE_INTERRUPT:
-		qh = (struct fotg210_qh *) urb->hcpriv;
-		if (!qh)
-			break;
-		switch (qh->qh_state) {
-		case QH_STATE_LINKED:
-		case QH_STATE_COMPLETING:
-			start_unlink_intr(fotg210, qh);
-			break;
-		case QH_STATE_IDLE:
-			qh_completions(fotg210, qh);
-			break;
-		default:
-			fotg210_dbg(fotg210, "bogus qh %p state %d\n",
-					qh, qh->qh_state);
-			goto done;
-		}
-		break;
-
-	case PIPE_ISOCHRONOUS:
-		/* itd... */
-
-		/* wait till next completion, do it then. */
-		/* completion irqs can wait up to 1024 msec, */
-		break;
-	}
-done:
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-	return rc;
-}
-
-/* bulk qh holds the data toggle */
-
-static void fotg210_endpoint_disable(struct usb_hcd *hcd,
-		struct usb_host_endpoint *ep)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	unsigned long flags;
-	struct fotg210_qh *qh, *tmp;
-
-	/* ASSERT:  any requests/urbs are being unlinked */
-	/* ASSERT:  nobody can be submitting urbs for this any more */
-
-rescan:
-	spin_lock_irqsave(&fotg210->lock, flags);
-	qh = ep->hcpriv;
-	if (!qh)
-		goto done;
-
-	/* endpoints can be iso streams.  for now, we don't
-	 * accelerate iso completions ... so spin a while.
-	 */
-	if (qh->hw == NULL) {
-		struct fotg210_iso_stream *stream = ep->hcpriv;
-
-		if (!list_empty(&stream->td_list))
-			goto idle_timeout;
-
-		/* BUG_ON(!list_empty(&stream->free_list)); */
-		kfree(stream);
-		goto done;
-	}
-
-	if (fotg210->rh_state < FOTG210_RH_RUNNING)
-		qh->qh_state = QH_STATE_IDLE;
-	switch (qh->qh_state) {
-	case QH_STATE_LINKED:
-	case QH_STATE_COMPLETING:
-		for (tmp = fotg210->async->qh_next.qh;
-				tmp && tmp != qh;
-				tmp = tmp->qh_next.qh)
-			continue;
-		/* periodic qh self-unlinks on empty, and a COMPLETING qh
-		 * may already be unlinked.
-		 */
-		if (tmp)
-			start_unlink_async(fotg210, qh);
-		fallthrough;
-	case QH_STATE_UNLINK:		/* wait for hw to finish? */
-	case QH_STATE_UNLINK_WAIT:
-idle_timeout:
-		spin_unlock_irqrestore(&fotg210->lock, flags);
-		schedule_timeout_uninterruptible(1);
-		goto rescan;
-	case QH_STATE_IDLE:		/* fully unlinked */
-		if (qh->clearing_tt)
-			goto idle_timeout;
-		if (list_empty(&qh->qtd_list)) {
-			qh_destroy(fotg210, qh);
-			break;
-		}
-		fallthrough;
-	default:
-		/* caller was supposed to have unlinked any requests;
-		 * that's not our job.  just leak this memory.
-		 */
-		fotg210_err(fotg210, "qh %p (#%02x) state %d%s\n",
-				qh, ep->desc.bEndpointAddress, qh->qh_state,
-				list_empty(&qh->qtd_list) ? "" : "(has tds)");
-		break;
-	}
-done:
-	ep->hcpriv = NULL;
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-}
-
-static void fotg210_endpoint_reset(struct usb_hcd *hcd,
-		struct usb_host_endpoint *ep)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-	struct fotg210_qh *qh;
-	int eptype = usb_endpoint_type(&ep->desc);
-	int epnum = usb_endpoint_num(&ep->desc);
-	int is_out = usb_endpoint_dir_out(&ep->desc);
-	unsigned long flags;
-
-	if (eptype != USB_ENDPOINT_XFER_BULK && eptype != USB_ENDPOINT_XFER_INT)
-		return;
-
-	spin_lock_irqsave(&fotg210->lock, flags);
-	qh = ep->hcpriv;
-
-	/* For Bulk and Interrupt endpoints we maintain the toggle state
-	 * in the hardware; the toggle bits in udev aren't used at all.
-	 * When an endpoint is reset by usb_clear_halt() we must reset
-	 * the toggle bit in the QH.
-	 */
-	if (qh) {
-		usb_settoggle(qh->dev, epnum, is_out, 0);
-		if (!list_empty(&qh->qtd_list)) {
-			WARN_ONCE(1, "clear_halt for a busy endpoint\n");
-		} else if (qh->qh_state == QH_STATE_LINKED ||
-				qh->qh_state == QH_STATE_COMPLETING) {
-
-			/* The toggle value in the QH can't be updated
-			 * while the QH is active.  Unlink it now;
-			 * re-linking will call qh_refresh().
-			 */
-			if (eptype == USB_ENDPOINT_XFER_BULK)
-				start_unlink_async(fotg210, qh);
-			else
-				start_unlink_intr(fotg210, qh);
-		}
-	}
-	spin_unlock_irqrestore(&fotg210->lock, flags);
-}
-
-static int fotg210_get_frame(struct usb_hcd *hcd)
-{
-	struct fotg210_hcd *fotg210 = hcd_to_fotg210(hcd);
-
-	return (fotg210_read_frame_index(fotg210) >> 3) %
-		fotg210->periodic_size;
-}
-
-/* The EHCI in ChipIdea HDRC cannot be a separate module or device,
- * because its registers (and irq) are shared between host/gadget/otg
- * functions  and in order to facilitate role switching we cannot
- * give the fotg210 driver exclusive access to those.
- */
-
-static const struct hc_driver fotg210_fotg210_hc_driver = {
-	.description		= hcd_name,
-	.product_desc		= "Faraday USB2.0 Host Controller",
-	.hcd_priv_size		= sizeof(struct fotg210_hcd),
-
-	/*
-	 * generic hardware linkage
-	 */
-	.irq			= fotg210_irq,
-	.flags			= HCD_MEMORY | HCD_DMA | HCD_USB2,
-
-	/*
-	 * basic lifecycle operations
-	 */
-	.reset			= hcd_fotg210_init,
-	.start			= fotg210_run,
-	.stop			= fotg210_stop,
-	.shutdown		= fotg210_shutdown,
-
-	/*
-	 * managing i/o requests and associated device resources
-	 */
-	.urb_enqueue		= fotg210_urb_enqueue,
-	.urb_dequeue		= fotg210_urb_dequeue,
-	.endpoint_disable	= fotg210_endpoint_disable,
-	.endpoint_reset		= fotg210_endpoint_reset,
-
-	/*
-	 * scheduling support
-	 */
-	.get_frame_number	= fotg210_get_frame,
-
-	/*
-	 * root hub support
-	 */
-	.hub_status_data	= fotg210_hub_status_data,
-	.hub_control		= fotg210_hub_control,
-	.bus_suspend		= fotg210_bus_suspend,
-	.bus_resume		= fotg210_bus_resume,
-
-	.relinquish_port	= fotg210_relinquish_port,
-	.port_handed_over	= fotg210_port_handed_over,
-
-	.clear_tt_buffer_complete = fotg210_clear_tt_buffer_complete,
+static const struct ehci_driver_overrides fotg210_hc_overrides __initconst = {
+	.reset = fotg210_hcd_reset,
+	.extra_priv_size = sizeof(struct fotg210_hcd),
 };
 
-static void fotg210_init(struct fotg210_hcd *fotg210)
-{
-	u32 value;
-
-	iowrite32(GMIR_MDEV_INT | GMIR_MOTG_INT | GMIR_INT_POLARITY,
-			&fotg210->regs->gmir);
-
-	value = ioread32(&fotg210->regs->otgcsr);
-	value &= ~OTGCSR_A_BUS_DROP;
-	value |= OTGCSR_A_BUS_REQ;
-	iowrite32(value, &fotg210->regs->otgcsr);
-}
-
 /*
  * fotg210_hcd_probe - initialize faraday FOTG210 HCDs
  *
@@ -5554,57 +134,39 @@ int fotg210_hcd_probe(struct platform_device *pdev, struct fotg210 *fotg)
 	struct device *dev = &pdev->dev;
 	struct usb_hcd *hcd;
 	int irq;
-	int retval;
-	struct fotg210_hcd *fotg210;
+	int ret;
 
 	if (usb_disabled())
 		return -ENODEV;
 
-	pdev->dev.power.power_state = PMSG_ON;
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	hcd = usb_create_hcd(&fotg210_fotg210_hc_driver, dev,
-			dev_name(dev));
-	if (!hcd) {
-		retval = dev_err_probe(dev, -ENOMEM, "failed to create hcd\n");
-		goto fail_create_hcd;
-	}
-
-	hcd->has_tt = 1;
-
-	hcd->regs = fotg->base;
+	hcd = usb_create_hcd(&fotg210_hc_driver, dev, dev_name(dev));
+	if (!hcd)
+		return dev_err_probe(dev, -ENOMEM, "failed to create hcd\n");
 
 	hcd->rsrc_start = fotg->res->start;
 	hcd->rsrc_len = resource_size(fotg->res);
+	hcd->regs = fotg->base;
+	hcd_to_fotg210_priv(hcd)->fotg = fotg;
+	hcd_to_ehci(hcd)->caps = fotg->base;
 
-	fotg210 = hcd_to_fotg210(hcd);
-
-	fotg210->fotg = fotg;
-	fotg210->caps = hcd->regs;
-
-	retval = fotg210_setup(hcd);
-	if (retval)
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
 		goto failed_put_hcd;
+	}
 
-	fotg210_init(fotg210);
-
-	retval = usb_add_hcd(hcd, irq, IRQF_SHARED);
-	if (retval) {
-		dev_err_probe(dev, retval, "failed to add hcd\n");
+	ret = usb_add_hcd(hcd, irq, 0);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to add HCD\n");
 		goto failed_put_hcd;
 	}
 	device_wakeup_enable(hcd->self.controller);
-	platform_set_drvdata(pdev, hcd);
 
-	return retval;
+	return ret;
 
 failed_put_hcd:
 	usb_put_hcd(hcd);
-fail_create_hcd:
-	return dev_err_probe(dev, retval, "init %s fail\n", dev_name(dev));
+	return ret;
 }
 
 /*
@@ -5615,29 +177,43 @@ int fotg210_hcd_probe(struct platform_device *pdev, struct fotg210 *fotg)
 int fotg210_hcd_remove(struct platform_device *pdev)
 {
 	struct usb_hcd *hcd = platform_get_drvdata(pdev);
+	struct fotg210 *fotg = hcd_to_fotg210_priv(hcd)->fotg;
 
 	usb_remove_hcd(hcd);
 	usb_put_hcd(hcd);
+	fotg210_vbus(fotg, false);
 
 	return 0;
 }
 
-int __init fotg210_hcd_init(void)
+int fotg210_hcd_suspend(struct device *dev)
 {
-	if (usb_disabled())
-		return -ENODEV;
+#ifdef CONFIG_PM
+	struct usb_hcd *hcd = dev_get_drvdata(dev);
+
+	return ehci_suspend(hcd, device_may_wakeup(dev));
+#else
+	return 0;
+#endif
+}
 
-	pr_debug("%s: block sizes: qh %zd qtd %zd itd %zd\n",
-			hcd_name, sizeof(struct fotg210_qh),
-			sizeof(struct fotg210_qtd),
-			sizeof(struct fotg210_itd));
+int fotg210_hcd_resume(struct device *dev)
+{
+#ifdef CONFIG_PM
+	struct usb_hcd *hcd = dev_get_drvdata(dev);
 
-	fotg210_debug_root = debugfs_create_dir("fotg210", usb_debug_root);
+	return ehci_resume(hcd, false);
+#else
+	return 0;
+#endif
+}
 
+int __init fotg210_hcd_init(void)
+{
+	ehci_init_driver(&fotg210_hc_driver, &fotg210_hc_overrides);
 	return 0;
 }
 
-void __exit fotg210_hcd_cleanup(void)
+void fotg210_hcd_cleanup(void)
 {
-	debugfs_remove(fotg210_debug_root);
 }
diff --git a/drivers/usb/fotg210/fotg210-hcd.h b/drivers/usb/fotg210/fotg210-hcd.h
deleted file mode 100644
index 13c9342982ee97..00000000000000
--- a/drivers/usb/fotg210/fotg210-hcd.h
+++ /dev/null
@@ -1,689 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __LINUX_FOTG210_H
-#define __LINUX_FOTG210_H
-
-#include <linux/usb/ehci-dbgp.h>
-
-/* definitions used for the EHCI driver */
-
-/*
- * __hc32 and __hc16 are "Host Controller" types, they may be equivalent to
- * __leXX (normally) or __beXX (given FOTG210_BIG_ENDIAN_DESC), depending on
- * the host controller implementation.
- *
- * To facilitate the strongest possible byte-order checking from "sparse"
- * and so on, we use __leXX unless that's not practical.
- */
-#define __hc32	__le32
-#define __hc16	__le16
-
-/* statistics can be kept for tuning/monitoring */
-struct fotg210_stats {
-	/* irq usage */
-	unsigned long		normal;
-	unsigned long		error;
-	unsigned long		iaa;
-	unsigned long		lost_iaa;
-
-	/* termination of urbs from core */
-	unsigned long		complete;
-	unsigned long		unlink;
-};
-
-/* fotg210_hcd->lock guards shared data against other CPUs:
- *   fotg210_hcd:	async, unlink, periodic (and shadow), ...
- *   usb_host_endpoint: hcpriv
- *   fotg210_qh:	qh_next, qtd_list
- *   fotg210_qtd:	qtd_list
- *
- * Also, hold this lock when talking to HC registers or
- * when updating hw_* fields in shared qh/qtd/... structures.
- */
-
-#define	FOTG210_MAX_ROOT_PORTS	1		/* see HCS_N_PORTS */
-
-/*
- * fotg210_rh_state values of FOTG210_RH_RUNNING or above mean that the
- * controller may be doing DMA.  Lower values mean there's no DMA.
- */
-enum fotg210_rh_state {
-	FOTG210_RH_HALTED,
-	FOTG210_RH_SUSPENDED,
-	FOTG210_RH_RUNNING,
-	FOTG210_RH_STOPPING
-};
-
-/*
- * Timer events, ordered by increasing delay length.
- * Always update event_delays_ns[] and event_handlers[] (defined in
- * ehci-timer.c) in parallel with this list.
- */
-enum fotg210_hrtimer_event {
-	FOTG210_HRTIMER_POLL_ASS,	/* Poll for async schedule off */
-	FOTG210_HRTIMER_POLL_PSS,	/* Poll for periodic schedule off */
-	FOTG210_HRTIMER_POLL_DEAD,	/* Wait for dead controller to stop */
-	FOTG210_HRTIMER_UNLINK_INTR,	/* Wait for interrupt QH unlink */
-	FOTG210_HRTIMER_FREE_ITDS,	/* Wait for unused iTDs and siTDs */
-	FOTG210_HRTIMER_ASYNC_UNLINKS,	/* Unlink empty async QHs */
-	FOTG210_HRTIMER_IAA_WATCHDOG,	/* Handle lost IAA interrupts */
-	FOTG210_HRTIMER_DISABLE_PERIODIC, /* Wait to disable periodic sched */
-	FOTG210_HRTIMER_DISABLE_ASYNC,	/* Wait to disable async sched */
-	FOTG210_HRTIMER_IO_WATCHDOG,	/* Check for missing IRQs */
-	FOTG210_HRTIMER_NUM_EVENTS	/* Must come last */
-};
-#define FOTG210_HRTIMER_NO_EVENT	99
-
-struct fotg210_hcd {			/* one per controller */
-	/* timing support */
-	enum fotg210_hrtimer_event	next_hrtimer_event;
-	unsigned		enabled_hrtimer_events;
-	ktime_t			hr_timeouts[FOTG210_HRTIMER_NUM_EVENTS];
-	struct hrtimer		hrtimer;
-
-	int			PSS_poll_count;
-	int			ASS_poll_count;
-	int			died_poll_count;
-
-	/* glue to PCI and HCD framework */
-	struct fotg210_caps __iomem *caps;
-	struct fotg210_regs __iomem *regs;
-	struct ehci_dbg_port __iomem *debug;
-
-	__u32			hcs_params;	/* cached register copy */
-	spinlock_t		lock;
-	enum fotg210_rh_state	rh_state;
-
-	/* general schedule support */
-	bool			scanning:1;
-	bool			need_rescan:1;
-	bool			intr_unlinking:1;
-	bool			async_unlinking:1;
-	bool			shutdown:1;
-	struct fotg210_qh		*qh_scan_next;
-
-	/* async schedule support */
-	struct fotg210_qh		*async;
-	struct fotg210_qh		*dummy;		/* For AMD quirk use */
-	struct fotg210_qh		*async_unlink;
-	struct fotg210_qh		*async_unlink_last;
-	struct fotg210_qh		*async_iaa;
-	unsigned		async_unlink_cycle;
-	unsigned		async_count;	/* async activity count */
-
-	/* periodic schedule support */
-#define	DEFAULT_I_TDPS		1024		/* some HCs can do less */
-	unsigned		periodic_size;
-	__hc32			*periodic;	/* hw periodic table */
-	dma_addr_t		periodic_dma;
-	struct list_head	intr_qh_list;
-	unsigned		i_thresh;	/* uframes HC might cache */
-
-	union fotg210_shadow	*pshadow;	/* mirror hw periodic table */
-	struct fotg210_qh		*intr_unlink;
-	struct fotg210_qh		*intr_unlink_last;
-	unsigned		intr_unlink_cycle;
-	unsigned		now_frame;	/* frame from HC hardware */
-	unsigned		next_frame;	/* scan periodic, start here */
-	unsigned		intr_count;	/* intr activity count */
-	unsigned		isoc_count;	/* isoc activity count */
-	unsigned		periodic_count;	/* periodic activity count */
-	/* max periodic time per uframe */
-	unsigned		uframe_periodic_max;
-
-
-	/* list of itds completed while now_frame was still active */
-	struct list_head	cached_itd_list;
-	struct fotg210_itd	*last_itd_to_free;
-
-	/* per root hub port */
-	unsigned long		reset_done[FOTG210_MAX_ROOT_PORTS];
-
-	/* bit vectors (one bit per port)
-	 * which ports were already suspended at the start of a bus suspend
-	 */
-	unsigned long		bus_suspended;
-
-	/* which ports are edicated to the companion controller */
-	unsigned long		companion_ports;
-
-	/* which ports are owned by the companion during a bus suspend */
-	unsigned long		owned_ports;
-
-	/* which ports have the change-suspend feature turned on */
-	unsigned long		port_c_suspend;
-
-	/* which ports are suspended */
-	unsigned long		suspended_ports;
-
-	/* which ports have started to resume */
-	unsigned long		resuming_ports;
-
-	/* per-HC memory pools (could be per-bus, but ...) */
-	struct dma_pool		*qh_pool;	/* qh per active urb */
-	struct dma_pool		*qtd_pool;	/* one or more per qh */
-	struct dma_pool		*itd_pool;	/* itd per iso urb */
-
-	unsigned		random_frame;
-	unsigned long		next_statechange;
-	ktime_t			last_periodic_enable;
-	u32			command;
-
-	/* SILICON QUIRKS */
-	unsigned		need_io_watchdog:1;
-	unsigned		fs_i_thresh:1;	/* Intel iso scheduling */
-
-	u8			sbrn;		/* packed release number */
-
-	/* irq statistics */
-#ifdef FOTG210_STATS
-	struct fotg210_stats	stats;
-#	define INCR(x) ((x)++)
-#else
-#	define INCR(x) do {} while (0)
-#endif
-
-	struct fotg210		*fotg;		/* Overarching FOTG210 device */
-	/* silicon clock */
-	struct clk		*pclk;
-};
-
-/* convert between an HCD pointer and the corresponding FOTG210_HCD */
-static inline struct fotg210_hcd *hcd_to_fotg210(struct usb_hcd *hcd)
-{
-	return (struct fotg210_hcd *)(hcd->hcd_priv);
-}
-static inline struct usb_hcd *fotg210_to_hcd(struct fotg210_hcd *fotg210)
-{
-	return container_of((void *) fotg210, struct usb_hcd, hcd_priv);
-}
-
-/*-------------------------------------------------------------------------*/
-
-/* EHCI register interface, corresponds to EHCI Revision 0.95 specification */
-
-/* Section 2.2 Host Controller Capability Registers */
-struct fotg210_caps {
-	/* these fields are specified as 8 and 16 bit registers,
-	 * but some hosts can't perform 8 or 16 bit PCI accesses.
-	 * some hosts treat caplength and hciversion as parts of a 32-bit
-	 * register, others treat them as two separate registers, this
-	 * affects the memory map for big endian controllers.
-	 */
-	u32		hc_capbase;
-#define HC_LENGTH(fotg210, p)	(0x00ff&((p) >> /* bits 7:0 / offset 00h */ \
-				(fotg210_big_endian_capbase(fotg210) ? 24 : 0)))
-#define HC_VERSION(fotg210, p)	(0xffff&((p) >> /* bits 31:16 / offset 02h */ \
-				(fotg210_big_endian_capbase(fotg210) ? 0 : 16)))
-	u32		hcs_params;     /* HCSPARAMS - offset 0x4 */
-#define HCS_N_PORTS(p)		(((p)>>0)&0xf)	/* bits 3:0, ports on HC */
-
-	u32		hcc_params;	/* HCCPARAMS - offset 0x8 */
-#define HCC_CANPARK(p)		((p)&(1 << 2))  /* true: can park on async qh */
-#define HCC_PGM_FRAMELISTLEN(p) ((p)&(1 << 1))  /* true: periodic_size changes*/
-	u8		portroute[8];	 /* nibbles for routing - offset 0xC */
-};
-
-
-/* Section 2.3 Host Controller Operational Registers */
-struct fotg210_regs {
-
-	/* USBCMD: offset 0x00 */
-	u32		command;
-
-/* EHCI 1.1 addendum */
-/* 23:16 is r/w intr rate, in microframes; default "8" == 1/msec */
-#define CMD_PARK	(1<<11)		/* enable "park" on async qh */
-#define CMD_PARK_CNT(c)	(((c)>>8)&3)	/* how many transfers to park for */
-#define CMD_IAAD	(1<<6)		/* "doorbell" interrupt async advance */
-#define CMD_ASE		(1<<5)		/* async schedule enable */
-#define CMD_PSE		(1<<4)		/* periodic schedule enable */
-/* 3:2 is periodic frame list size */
-#define CMD_RESET	(1<<1)		/* reset HC not bus */
-#define CMD_RUN		(1<<0)		/* start/stop HC */
-
-	/* USBSTS: offset 0x04 */
-	u32		status;
-#define STS_ASS		(1<<15)		/* Async Schedule Status */
-#define STS_PSS		(1<<14)		/* Periodic Schedule Status */
-#define STS_RECL	(1<<13)		/* Reclamation */
-#define STS_HALT	(1<<12)		/* Not running (any reason) */
-/* some bits reserved */
-	/* these STS_* flags are also intr_enable bits (USBINTR) */
-#define STS_IAA		(1<<5)		/* Interrupted on async advance */
-#define STS_FATAL	(1<<4)		/* such as some PCI access errors */
-#define STS_FLR		(1<<3)		/* frame list rolled over */
-#define STS_PCD		(1<<2)		/* port change detect */
-#define STS_ERR		(1<<1)		/* "error" completion (overflow, ...) */
-#define STS_INT		(1<<0)		/* "normal" completion (short, ...) */
-
-	/* USBINTR: offset 0x08 */
-	u32		intr_enable;
-
-	/* FRINDEX: offset 0x0C */
-	u32		frame_index;	/* current microframe number */
-	/* CTRLDSSEGMENT: offset 0x10 */
-	u32		segment;	/* address bits 63:32 if needed */
-	/* PERIODICLISTBASE: offset 0x14 */
-	u32		frame_list;	/* points to periodic list */
-	/* ASYNCLISTADDR: offset 0x18 */
-	u32		async_next;	/* address of next async queue head */
-
-	u32	reserved1;
-	/* PORTSC: offset 0x20 */
-	u32	port_status;
-/* 31:23 reserved */
-#define PORT_USB11(x) (((x)&(3<<10)) == (1<<10))	/* USB 1.1 device */
-#define PORT_RESET	(1<<8)		/* reset port */
-#define PORT_SUSPEND	(1<<7)		/* suspend port */
-#define PORT_RESUME	(1<<6)		/* resume it */
-#define PORT_PEC	(1<<3)		/* port enable change */
-#define PORT_PE		(1<<2)		/* port enable */
-#define PORT_CSC	(1<<1)		/* connect status change */
-#define PORT_CONNECT	(1<<0)		/* device connected */
-#define PORT_RWC_BITS   (PORT_CSC | PORT_PEC)
-	u32     reserved2[19];
-
-	/* OTGCSR: offet 0x70 */
-	u32     otgcsr;
-#define OTGCSR_HOST_SPD_TYP     (3 << 22)
-#define OTGCSR_A_BUS_DROP	(1 << 5)
-#define OTGCSR_A_BUS_REQ	(1 << 4)
-
-	/* OTGISR: offset 0x74 */
-	u32     otgisr;
-#define OTGISR_OVC	(1 << 10)
-
-	u32     reserved3[15];
-
-	/* GMIR: offset 0xB4 */
-	u32     gmir;
-#define GMIR_INT_POLARITY	(1 << 3) /*Active High*/
-#define GMIR_MHC_INT		(1 << 2)
-#define GMIR_MOTG_INT		(1 << 1)
-#define GMIR_MDEV_INT	(1 << 0)
-};
-
-/*-------------------------------------------------------------------------*/
-
-#define	QTD_NEXT(fotg210, dma)	cpu_to_hc32(fotg210, (u32)dma)
-
-/*
- * EHCI Specification 0.95 Section 3.5
- * QTD: describe data transfer components (buffer, direction, ...)
- * See Fig 3-6 "Queue Element Transfer Descriptor Block Diagram".
- *
- * These are associated only with "QH" (Queue Head) structures,
- * used with control, bulk, and interrupt transfers.
- */
-struct fotg210_qtd {
-	/* first part defined by EHCI spec */
-	__hc32			hw_next;	/* see EHCI 3.5.1 */
-	__hc32			hw_alt_next;    /* see EHCI 3.5.2 */
-	__hc32			hw_token;	/* see EHCI 3.5.3 */
-#define	QTD_TOGGLE	(1 << 31)	/* data toggle */
-#define	QTD_LENGTH(tok)	(((tok)>>16) & 0x7fff)
-#define	QTD_IOC		(1 << 15)	/* interrupt on complete */
-#define	QTD_CERR(tok)	(((tok)>>10) & 0x3)
-#define	QTD_PID(tok)	(((tok)>>8) & 0x3)
-#define	QTD_STS_ACTIVE	(1 << 7)	/* HC may execute this */
-#define	QTD_STS_HALT	(1 << 6)	/* halted on error */
-#define	QTD_STS_DBE	(1 << 5)	/* data buffer error (in HC) */
-#define	QTD_STS_BABBLE	(1 << 4)	/* device was babbling (qtd halted) */
-#define	QTD_STS_XACT	(1 << 3)	/* device gave illegal response */
-#define	QTD_STS_MMF	(1 << 2)	/* incomplete split transaction */
-#define	QTD_STS_STS	(1 << 1)	/* split transaction state */
-#define	QTD_STS_PING	(1 << 0)	/* issue PING? */
-
-#define ACTIVE_BIT(fotg210)	cpu_to_hc32(fotg210, QTD_STS_ACTIVE)
-#define HALT_BIT(fotg210)		cpu_to_hc32(fotg210, QTD_STS_HALT)
-#define STATUS_BIT(fotg210)	cpu_to_hc32(fotg210, QTD_STS_STS)
-
-	__hc32			hw_buf[5];	/* see EHCI 3.5.4 */
-	__hc32			hw_buf_hi[5];	/* Appendix B */
-
-	/* the rest is HCD-private */
-	dma_addr_t		qtd_dma;		/* qtd address */
-	struct list_head	qtd_list;		/* sw qtd list */
-	struct urb		*urb;			/* qtd's urb */
-	size_t			length;			/* length of buffer */
-} __aligned(32);
-
-/* mask NakCnt+T in qh->hw_alt_next */
-#define QTD_MASK(fotg210)	cpu_to_hc32(fotg210, ~0x1f)
-
-#define IS_SHORT_READ(token) (QTD_LENGTH(token) != 0 && QTD_PID(token) == 1)
-
-/*-------------------------------------------------------------------------*/
-
-/* type tag from {qh,itd,fstn}->hw_next */
-#define Q_NEXT_TYPE(fotg210, dma)	((dma) & cpu_to_hc32(fotg210, 3 << 1))
-
-/*
- * Now the following defines are not converted using the
- * cpu_to_le32() macro anymore, since we have to support
- * "dynamic" switching between be and le support, so that the driver
- * can be used on one system with SoC EHCI controller using big-endian
- * descriptors as well as a normal little-endian PCI EHCI controller.
- */
-/* values for that type tag */
-#define Q_TYPE_ITD	(0 << 1)
-#define Q_TYPE_QH	(1 << 1)
-#define Q_TYPE_SITD	(2 << 1)
-#define Q_TYPE_FSTN	(3 << 1)
-
-/* next async queue entry, or pointer to interrupt/periodic QH */
-#define QH_NEXT(fotg210, dma) \
-	(cpu_to_hc32(fotg210, (((u32)dma)&~0x01f)|Q_TYPE_QH))
-
-/* for periodic/async schedules and qtd lists, mark end of list */
-#define FOTG210_LIST_END(fotg210) \
-	cpu_to_hc32(fotg210, 1) /* "null pointer" to hw */
-
-/*
- * Entries in periodic shadow table are pointers to one of four kinds
- * of data structure.  That's dictated by the hardware; a type tag is
- * encoded in the low bits of the hardware's periodic schedule.  Use
- * Q_NEXT_TYPE to get the tag.
- *
- * For entries in the async schedule, the type tag always says "qh".
- */
-union fotg210_shadow {
-	struct fotg210_qh	*qh;		/* Q_TYPE_QH */
-	struct fotg210_itd	*itd;		/* Q_TYPE_ITD */
-	struct fotg210_fstn	*fstn;		/* Q_TYPE_FSTN */
-	__hc32			*hw_next;	/* (all types) */
-	void			*ptr;
-};
-
-/*-------------------------------------------------------------------------*/
-
-/*
- * EHCI Specification 0.95 Section 3.6
- * QH: describes control/bulk/interrupt endpoints
- * See Fig 3-7 "Queue Head Structure Layout".
- *
- * These appear in both the async and (for interrupt) periodic schedules.
- */
-
-/* first part defined by EHCI spec */
-struct fotg210_qh_hw {
-	__hc32			hw_next;	/* see EHCI 3.6.1 */
-	__hc32			hw_info1;	/* see EHCI 3.6.2 */
-#define	QH_CONTROL_EP	(1 << 27)	/* FS/LS control endpoint */
-#define	QH_HEAD		(1 << 15)	/* Head of async reclamation list */
-#define	QH_TOGGLE_CTL	(1 << 14)	/* Data toggle control */
-#define	QH_HIGH_SPEED	(2 << 12)	/* Endpoint speed */
-#define	QH_LOW_SPEED	(1 << 12)
-#define	QH_FULL_SPEED	(0 << 12)
-#define	QH_INACTIVATE	(1 << 7)	/* Inactivate on next transaction */
-	__hc32			hw_info2;	/* see EHCI 3.6.2 */
-#define	QH_SMASK	0x000000ff
-#define	QH_CMASK	0x0000ff00
-#define	QH_HUBADDR	0x007f0000
-#define	QH_HUBPORT	0x3f800000
-#define	QH_MULT		0xc0000000
-	__hc32			hw_current;	/* qtd list - see EHCI 3.6.4 */
-
-	/* qtd overlay (hardware parts of a struct fotg210_qtd) */
-	__hc32			hw_qtd_next;
-	__hc32			hw_alt_next;
-	__hc32			hw_token;
-	__hc32			hw_buf[5];
-	__hc32			hw_buf_hi[5];
-} __aligned(32);
-
-struct fotg210_qh {
-	struct fotg210_qh_hw	*hw;		/* Must come first */
-	/* the rest is HCD-private */
-	dma_addr_t		qh_dma;		/* address of qh */
-	union fotg210_shadow	qh_next;	/* ptr to qh; or periodic */
-	struct list_head	qtd_list;	/* sw qtd list */
-	struct list_head	intr_node;	/* list of intr QHs */
-	struct fotg210_qtd	*dummy;
-	struct fotg210_qh	*unlink_next;	/* next on unlink list */
-
-	unsigned		unlink_cycle;
-
-	u8			needs_rescan;	/* Dequeue during giveback */
-	u8			qh_state;
-#define	QH_STATE_LINKED		1		/* HC sees this */
-#define	QH_STATE_UNLINK		2		/* HC may still see this */
-#define	QH_STATE_IDLE		3		/* HC doesn't see this */
-#define	QH_STATE_UNLINK_WAIT	4		/* LINKED and on unlink q */
-#define	QH_STATE_COMPLETING	5		/* don't touch token.HALT */
-
-	u8			xacterrs;	/* XactErr retry counter */
-#define	QH_XACTERR_MAX		32		/* XactErr retry limit */
-
-	/* periodic schedule info */
-	u8			usecs;		/* intr bandwidth */
-	u8			gap_uf;		/* uframes split/csplit gap */
-	u8			c_usecs;	/* ... split completion bw */
-	u16			tt_usecs;	/* tt downstream bandwidth */
-	unsigned short		period;		/* polling interval */
-	unsigned short		start;		/* where polling starts */
-#define NO_FRAME ((unsigned short)~0)			/* pick new start */
-
-	struct usb_device	*dev;		/* access to TT */
-	unsigned		is_out:1;	/* bulk or intr OUT */
-	unsigned		clearing_tt:1;	/* Clear-TT-Buf in progress */
-};
-
-/*-------------------------------------------------------------------------*/
-
-/* description of one iso transaction (up to 3 KB data if highspeed) */
-struct fotg210_iso_packet {
-	/* These will be copied to iTD when scheduling */
-	u64			bufp;		/* itd->hw_bufp{,_hi}[pg] |= */
-	__hc32			transaction;	/* itd->hw_transaction[i] |= */
-	u8			cross;		/* buf crosses pages */
-	/* for full speed OUT splits */
-	u32			buf1;
-};
-
-/* temporary schedule data for packets from iso urbs (both speeds)
- * each packet is one logical usb transaction to the device (not TT),
- * beginning at stream->next_uframe
- */
-struct fotg210_iso_sched {
-	struct list_head	td_list;
-	unsigned		span;
-	struct fotg210_iso_packet	packet[];
-};
-
-/*
- * fotg210_iso_stream - groups all (s)itds for this endpoint.
- * acts like a qh would, if EHCI had them for ISO.
- */
-struct fotg210_iso_stream {
-	/* first field matches fotg210_hq, but is NULL */
-	struct fotg210_qh_hw	*hw;
-
-	u8			bEndpointAddress;
-	u8			highspeed;
-	struct list_head	td_list;	/* queued itds */
-	struct list_head	free_list;	/* list of unused itds */
-	struct usb_device	*udev;
-	struct usb_host_endpoint *ep;
-
-	/* output of (re)scheduling */
-	int			next_uframe;
-	__hc32			splits;
-
-	/* the rest is derived from the endpoint descriptor,
-	 * trusting urb->interval == f(epdesc->bInterval) and
-	 * including the extra info for hw_bufp[0..2]
-	 */
-	u8			usecs, c_usecs;
-	u16			interval;
-	u16			tt_usecs;
-	u16			maxp;
-	u16			raw_mask;
-	unsigned		bandwidth;
-
-	/* This is used to initialize iTD's hw_bufp fields */
-	__hc32			buf0;
-	__hc32			buf1;
-	__hc32			buf2;
-
-	/* this is used to initialize sITD's tt info */
-	__hc32			address;
-};
-
-/*-------------------------------------------------------------------------*/
-
-/*
- * EHCI Specification 0.95 Section 3.3
- * Fig 3-4 "Isochronous Transaction Descriptor (iTD)"
- *
- * Schedule records for high speed iso xfers
- */
-struct fotg210_itd {
-	/* first part defined by EHCI spec */
-	__hc32			hw_next;	/* see EHCI 3.3.1 */
-	__hc32			hw_transaction[8]; /* see EHCI 3.3.2 */
-#define FOTG210_ISOC_ACTIVE	(1<<31)	/* activate transfer this slot */
-#define FOTG210_ISOC_BUF_ERR	(1<<30)	/* Data buffer error */
-#define FOTG210_ISOC_BABBLE	(1<<29)	/* babble detected */
-#define FOTG210_ISOC_XACTERR	(1<<28)	/* XactErr - transaction error */
-#define	FOTG210_ITD_LENGTH(tok)	(((tok)>>16) & 0x0fff)
-#define	FOTG210_ITD_IOC		(1 << 15)	/* interrupt on complete */
-
-#define ITD_ACTIVE(fotg210)	cpu_to_hc32(fotg210, FOTG210_ISOC_ACTIVE)
-
-	__hc32			hw_bufp[7];	/* see EHCI 3.3.3 */
-	__hc32			hw_bufp_hi[7];	/* Appendix B */
-
-	/* the rest is HCD-private */
-	dma_addr_t		itd_dma;	/* for this itd */
-	union fotg210_shadow	itd_next;	/* ptr to periodic q entry */
-
-	struct urb		*urb;
-	struct fotg210_iso_stream	*stream;	/* endpoint's queue */
-	struct list_head	itd_list;	/* list of stream's itds */
-
-	/* any/all hw_transactions here may be used by that urb */
-	unsigned		frame;		/* where scheduled */
-	unsigned		pg;
-	unsigned		index[8];	/* in urb->iso_frame_desc */
-} __aligned(32);
-
-/*-------------------------------------------------------------------------*/
-
-/*
- * EHCI Specification 0.96 Section 3.7
- * Periodic Frame Span Traversal Node (FSTN)
- *
- * Manages split interrupt transactions (using TT) that span frame boundaries
- * into uframes 0/1; see 4.12.2.2.  In those uframes, a "save place" FSTN
- * makes the HC jump (back) to a QH to scan for fs/ls QH completions until
- * it hits a "restore" FSTN; then it returns to finish other uframe 0/1 work.
- */
-struct fotg210_fstn {
-	__hc32			hw_next;	/* any periodic q entry */
-	__hc32			hw_prev;	/* qh or FOTG210_LIST_END */
-
-	/* the rest is HCD-private */
-	dma_addr_t		fstn_dma;
-	union fotg210_shadow	fstn_next;	/* ptr to periodic q entry */
-} __aligned(32);
-
-/*-------------------------------------------------------------------------*/
-
-/* Prepare the PORTSC wakeup flags during controller suspend/resume */
-
-#define fotg210_prepare_ports_for_controller_suspend(fotg210, do_wakeup) \
-		fotg210_adjust_port_wakeup_flags(fotg210, true, do_wakeup)
-
-#define fotg210_prepare_ports_for_controller_resume(fotg210)		\
-		fotg210_adjust_port_wakeup_flags(fotg210, false, false)
-
-/*-------------------------------------------------------------------------*/
-
-/*
- * Some EHCI controllers have a Transaction Translator built into the
- * root hub. This is a non-standard feature.  Each controller will need
- * to add code to the following inline functions, and call them as
- * needed (mostly in root hub code).
- */
-
-static inline unsigned int
-fotg210_get_speed(struct fotg210_hcd *fotg210, unsigned int portsc)
-{
-	return (readl(&fotg210->regs->otgcsr)
-		& OTGCSR_HOST_SPD_TYP) >> 22;
-}
-
-/* Returns the speed of a device attached to a port on the root hub. */
-static inline unsigned int
-fotg210_port_speed(struct fotg210_hcd *fotg210, unsigned int portsc)
-{
-	switch (fotg210_get_speed(fotg210, portsc)) {
-	case 0:
-		return 0;
-	case 1:
-		return USB_PORT_STAT_LOW_SPEED;
-	case 2:
-	default:
-		return USB_PORT_STAT_HIGH_SPEED;
-	}
-}
-
-/*-------------------------------------------------------------------------*/
-
-#define	fotg210_has_fsl_portno_bug(e)		(0)
-
-/*
- * While most USB host controllers implement their registers in
- * little-endian format, a minority (celleb companion chip) implement
- * them in big endian format.
- *
- * This attempts to support either format at compile time without a
- * runtime penalty, or both formats with the additional overhead
- * of checking a flag bit.
- *
- */
-
-#define fotg210_big_endian_mmio(e)	0
-#define fotg210_big_endian_capbase(e)	0
-
-static inline unsigned int fotg210_readl(const struct fotg210_hcd *fotg210,
-		__u32 __iomem *regs)
-{
-	return readl(regs);
-}
-
-static inline void fotg210_writel(const struct fotg210_hcd *fotg210,
-		const unsigned int val, __u32 __iomem *regs)
-{
-	writel(val, regs);
-}
-
-/* cpu to fotg210 */
-static inline __hc32 cpu_to_hc32(const struct fotg210_hcd *fotg210, const u32 x)
-{
-	return cpu_to_le32(x);
-}
-
-/* fotg210 to cpu */
-static inline u32 hc32_to_cpu(const struct fotg210_hcd *fotg210, const __hc32 x)
-{
-	return le32_to_cpu(x);
-}
-
-static inline u32 hc32_to_cpup(const struct fotg210_hcd *fotg210,
-			       const __hc32 *x)
-{
-	return le32_to_cpup(x);
-}
-
-/*-------------------------------------------------------------------------*/
-
-static inline unsigned fotg210_read_frame_index(struct fotg210_hcd *fotg210)
-{
-	return fotg210_readl(fotg210, &fotg210->regs->frame_index);
-}
-
-/*-------------------------------------------------------------------------*/
-
-#endif /* __LINUX_FOTG210_H */
diff --git a/drivers/usb/fotg210/fotg210-udc.c b/drivers/usb/fotg210/fotg210-udc.c
index d9e024873a4291..e172f6a7856d8f 100644
--- a/drivers/usb/fotg210/fotg210-udc.c
+++ b/drivers/usb/fotg210/fotg210-udc.c
@@ -124,6 +124,7 @@ static void fotg210_set_fifo_dir(struct fotg210_ep *ep, u32 epnum, u32 dir_in)
 	u32 val;
 
 	val = ioread32(fotg210->reg + FOTG210_FIFOMAP);
+	val &= ~FIFOMAP_NA(epnum - 1);
 	val |= (dir_in ? FIFOMAP_DIRIN(epnum - 1) : FIFOMAP_DIROUT(epnum - 1));
 	iowrite32(val, fotg210->reg + FOTG210_FIFOMAP);
 }
@@ -134,6 +135,7 @@ static void fotg210_set_tfrtype(struct fotg210_ep *ep, u32 epnum, u32 type)
 	u32 val;
 
 	val = ioread32(fotg210->reg + FOTG210_FIFOCF);
+	val &= ~FIFOCF_TYPE(3, epnum - 1);
 	val |= FIFOCF_TYPE(type, epnum - 1);
 	iowrite32(val, fotg210->reg + FOTG210_FIFOCF);
 }
@@ -147,6 +149,7 @@ static void fotg210_set_mps(struct fotg210_ep *ep, u32 epnum, u32 mps,
 				FOTG210_OUTEPMPSR(epnum);
 
 	val = ioread32(fotg210->reg + offset);
+	val &= ~INOUTEPMPSR_MPS(~0);
 	val |= INOUTEPMPSR_MPS(mps);
 	iowrite32(val, fotg210->reg + offset);
 }
@@ -209,12 +212,13 @@ static int fotg210_ep_release(struct fotg210_ep *ep)
 {
 	if (!ep->epnum)
 		return 0;
+
+	fotg210_reset_tseq(ep->fotg210, ep->epnum);
+
 	ep->epnum = 0;
 	ep->stall = 0;
 	ep->wedged = 0;
 
-	fotg210_reset_tseq(ep->fotg210, ep->epnum);
-
 	return 0;
 }
 
@@ -338,6 +342,7 @@ static void fotg210_start_dma(struct fotg210_ep *ep,
 			struct fotg210_request *req)
 {
 	struct device *dev = &ep->fotg210->gadget.dev;
+	enum dma_data_direction direction;
 	dma_addr_t d;
 	u8 *buffer;
 	u32 length;
@@ -361,8 +366,8 @@ static void fotg210_start_dma(struct fotg210_ep *ep,
 			length = req->req.length - req->req.actual;
 	}
 
-	d = dma_map_single(dev, buffer, length,
-			ep->dir_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
+	direction = ep->dir_in ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	d = dma_map_single(dev, buffer, length, direction);
 
 	if (dma_mapping_error(dev, d)) {
 		pr_err("dma_mapping_error\n");
@@ -379,7 +384,7 @@ static void fotg210_start_dma(struct fotg210_ep *ep,
 	/* update actual transfer length */
 	req->req.actual += length;
 
-	dma_unmap_single(dev, d, length, DMA_TO_DEVICE);
+	dma_unmap_single(dev, d, length, direction);
 }
 
 static void fotg210_ep0_queue(struct fotg210_ep *ep,
@@ -445,7 +450,7 @@ static int fotg210_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
 	req = container_of(_req, struct fotg210_request, req);
 
 	spin_lock_irqsave(&ep->fotg210->lock, flags);
-	if (!list_empty(&ep->queue))
+	if (!list_empty(&req->queue))
 		fotg210_done(ep, req, -ECONNRESET);
 	spin_unlock_irqrestore(&ep->fotg210->lock, flags);
 
@@ -661,7 +666,7 @@ static void fotg210_set_feature(struct fotg210_udc *fotg210,
 	case USB_RECIP_ENDPOINT: {
 		u8 epnum;
 		epnum = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK;
-		if (epnum)
+		if (epnum && epnum < FOTG210_MAX_NUM_EP)
 			fotg210_set_epnstall(fotg210->ep[epnum]);
 		else
 			fotg210_set_cxstall(fotg210);
@@ -677,8 +682,8 @@ static void fotg210_set_feature(struct fotg210_udc *fotg210,
 static void fotg210_clear_feature(struct fotg210_udc *fotg210,
 				struct usb_ctrlrequest *ctrl)
 {
-	struct fotg210_ep *ep =
-		fotg210->ep[ctrl->wIndex & USB_ENDPOINT_NUMBER_MASK];
+	u8 epnum = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK;
+	struct fotg210_ep *ep;
 
 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
 	case USB_RECIP_DEVICE:
@@ -688,7 +693,12 @@ static void fotg210_clear_feature(struct fotg210_udc *fotg210,
 		fotg210_set_cxdone(fotg210);
 		break;
 	case USB_RECIP_ENDPOINT:
-		if (ctrl->wIndex & USB_ENDPOINT_NUMBER_MASK) {
+		if (epnum >= FOTG210_MAX_NUM_EP) {
+			fotg210_request_error(fotg210);
+			break;
+		}
+		if (epnum) {
+			ep = fotg210->ep[epnum];
 			if (ep->wedged) {
 				fotg210_set_cxdone(fotg210);
 				break;
@@ -744,8 +754,8 @@ static void fotg210_get_status(struct fotg210_udc *fotg210,
 		fotg210->ep0_data = cpu_to_le16(0);
 		break;
 	case USB_RECIP_ENDPOINT:
-		epnum = ctrl->wIndex & USB_ENDPOINT_NUMBER_MASK;
-		if (epnum)
+		epnum = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK;
+		if (epnum && epnum < FOTG210_MAX_NUM_EP)
 			fotg210->ep0_data =
 				cpu_to_le16(fotg210_is_epnstall(fotg210->ep[epnum])
 					    << USB_ENDPOINT_HALT);
@@ -886,6 +896,8 @@ static irqreturn_t fotg210_irq(int irq, void *_fotg210)
 	u32 int_msk = ioread32(fotg210->reg + FOTG210_DMIGR);
 
 	int_grp &= ~int_msk;
+	if (!int_grp)
+		return IRQ_NONE;
 
 	spin_lock(&fotg210->lock);
 
@@ -1002,6 +1014,14 @@ static void fotg210_disable_unplug(struct fotg210_udc *fotg210)
 	iowrite32(reg, fotg210->reg + FOTG210_PHYTMSR);
 }
 
+static void fotg210_enable_unplug(struct fotg210_udc *fotg210)
+{
+	u32 reg = ioread32(fotg210->reg + FOTG210_PHYTMSR);
+
+	reg |= PHYTMSR_UNPLUG;
+	iowrite32(reg, fotg210->reg + FOTG210_PHYTMSR);
+}
+
 static int fotg210_udc_start(struct usb_gadget *g,
 		struct usb_gadget_driver *driver)
 {
@@ -1009,20 +1029,23 @@ static int fotg210_udc_start(struct usb_gadget *g,
 	u32 value;
 	int ret;
 
-	/* hook up the driver */
+	/* Hook up the driver before enabling device interrupts. */
 	fotg210->driver = driver;
 	fotg210->gadget.dev.of_node = fotg210->dev->of_node;
 	fotg210->gadget.speed = USB_SPEED_UNKNOWN;
 
-	dev_info(fotg210->dev, "bound driver %s\n", driver->driver.name);
-
 	if (!IS_ERR_OR_NULL(fotg210->phy)) {
 		ret = otg_set_peripheral(fotg210->phy->otg,
 					 &fotg210->gadget);
-		if (ret)
-			dev_err(fotg210->dev, "can't bind to phy\n");
+		if (ret) {
+			fotg210->driver = NULL;
+			return dev_err_probe(fotg210->dev, ret,
+					     "can't bind to PHY\n");
+		}
 	}
 
+	dev_info(fotg210->dev, "bound driver %s\n", driver->driver.name);
+
 	/* chip enable */
 	value = ioread32(fotg210->reg + FOTG210_DMCR);
 	value |= DMCR_CHIP_EN;
@@ -1076,20 +1099,23 @@ static void fotg210_init(struct fotg210_udc *fotg210)
 static int fotg210_udc_stop(struct usb_gadget *g)
 {
 	struct fotg210_udc *fotg210 = gadget_to_fotg210(g);
-	unsigned long	flags;
+	unsigned long flags;
+	int ret = 0;
 
 	if (!IS_ERR_OR_NULL(fotg210->phy))
-		return otg_set_peripheral(fotg210->phy->otg, NULL);
+		ret = otg_set_peripheral(fotg210->phy->otg, NULL);
+
+	/* fotg210_init() sleeps, so it must run outside the spinlock. */
+	fotg210_init(fotg210);
 
 	spin_lock_irqsave(&fotg210->lock, flags);
 
-	fotg210_init(fotg210);
 	fotg210->driver = NULL;
 	fotg210->gadget.speed = USB_SPEED_UNKNOWN;
 
 	spin_unlock_irqrestore(&fotg210->lock, flags);
 
-	return 0;
+	return ret;
 }
 
 /**
@@ -1103,8 +1129,12 @@ static int fotg210_vbus_session(struct usb_gadget *g, int is_active)
 {
 	struct fotg210_udc *fotg210 = gadget_to_fotg210(g);
 
-	/* Call down to core integration layer to drive or disable VBUS */
-	fotg210_vbus(fotg210->fotg, is_active);
+	/* A peripheral must never source VBUS; only control its pull-up. */
+	if (is_active)
+		fotg210_disable_unplug(fotg210);
+	else
+		fotg210_enable_unplug(fotg210);
+
 	return 0;
 }
 
@@ -1144,28 +1174,24 @@ static int fotg210_phy_event(struct notifier_block *nb, unsigned long action,
 	}
 }
 
-static struct notifier_block fotg210_phy_notifier = {
-	.notifier_call = fotg210_phy_event,
-};
-
 int fotg210_udc_remove(struct platform_device *pdev)
 {
 	struct fotg210_udc *fotg210 = platform_get_drvdata(pdev);
 	int i;
 
 	usb_del_gadget_udc(&fotg210->gadget);
-	if (!IS_ERR_OR_NULL(fotg210->phy)) {
-		usb_unregister_notifier(fotg210->phy, &fotg210_phy_notifier);
-		usb_put_phy(fotg210->phy);
-	}
-	iounmap(fotg210->reg);
+	if (!IS_ERR_OR_NULL(fotg210->phy))
+		usb_unregister_notifier(fotg210->phy,
+					&fotg210->phy_notifier);
 	free_irq(platform_get_irq(pdev, 0), fotg210);
+	usb_phy_shutdown(fotg210->phy);
 
 	fotg210_ep_free_request(&fotg210->ep[0]->ep, fotg210->ep0_req);
 	for (i = 0; i < FOTG210_MAX_NUM_EP; i++)
 		kfree(fotg210->ep[i]);
 
 	kfree(fotg210);
+	platform_set_drvdata(pdev, NULL);
 
 	return 0;
 }
@@ -1188,7 +1214,6 @@ int fotg210_udc_probe(struct platform_device *pdev, struct fotg210 *fotg)
 		return -ENOMEM;
 
 	fotg210->dev = dev;
-	fotg210->fotg = fotg;
 
 	fotg210->phy = devm_usb_get_phy_by_phandle(dev, "usb-phy", 0);
 	if (IS_ERR(fotg210->phy)) {
@@ -1216,8 +1241,6 @@ int fotg210_udc_probe(struct platform_device *pdev, struct fotg210 *fotg)
 
 	spin_lock_init(&fotg210->lock);
 
-	platform_set_drvdata(pdev, fotg210);
-
 	fotg210->gadget.ops = &fotg210_gadget_ops;
 
 	fotg210->gadget.max_speed = USB_SPEED_HIGH;
@@ -1267,38 +1290,45 @@ int fotg210_udc_probe(struct platform_device *pdev, struct fotg210 *fotg)
 
 	fotg210_disable_unplug(fotg210);
 
-	ret = request_irq(irq, fotg210_irq, IRQF_SHARED,
-			  udc_name, fotg210);
+	ret = request_irq(irq, fotg210_irq, 0, udc_name, fotg210);
 	if (ret < 0) {
 		dev_err_probe(dev, ret, "request_irq error\n");
 		goto err_req;
 	}
 
-	if (!IS_ERR_OR_NULL(fotg210->phy))
-		usb_register_notifier(fotg210->phy, &fotg210_phy_notifier);
+	if (!IS_ERR_OR_NULL(fotg210->phy)) {
+		fotg210->phy_notifier.notifier_call = fotg210_phy_event;
+		ret = usb_register_notifier(fotg210->phy,
+					    &fotg210->phy_notifier);
+		if (ret)
+			goto err_notifier;
+	}
 
 	ret = usb_add_gadget_udc(dev, &fotg210->gadget);
 	if (ret)
 		goto err_add_udc;
 
+	platform_set_drvdata(pdev, fotg210);
+
 	dev_info(dev, "version %s\n", DRIVER_VERSION);
 
 	return 0;
 
 err_add_udc:
 	if (!IS_ERR_OR_NULL(fotg210->phy))
-		usb_unregister_notifier(fotg210->phy, &fotg210_phy_notifier);
+		usb_unregister_notifier(fotg210->phy,
+					&fotg210->phy_notifier);
+err_notifier:
 	free_irq(irq, fotg210);
 
 err_req:
 	fotg210_ep_free_request(&fotg210->ep[0]->ep, fotg210->ep0_req);
 
 err_map:
-	iounmap(fotg210->reg);
-
 err_alloc:
 	for (i = 0; i < FOTG210_MAX_NUM_EP; i++)
 		kfree(fotg210->ep[i]);
+	usb_phy_shutdown(fotg210->phy);
 
 err_free:
 	kfree(fotg210);
diff --git a/drivers/usb/fotg210/fotg210-udc.h b/drivers/usb/fotg210/fotg210-udc.h
index 252cb2b8e2fefa..11fe6780253772 100644
--- a/drivers/usb/fotg210/fotg210-udc.h
+++ b/drivers/usb/fotg210/fotg210-udc.h
@@ -237,8 +237,8 @@ struct fotg210_udc {
 	unsigned long		irq_trigger;
 
 	struct device			*dev;
-	struct fotg210			*fotg;
 	struct usb_phy			*phy;
+	struct notifier_block		phy_notifier;
 	struct usb_gadget		gadget;
 	struct usb_gadget_driver	*driver;
 
diff --git a/drivers/usb/fotg210/fotg210.h b/drivers/usb/fotg210/fotg210.h
index c44c0afe295692..aab0f0300dc8a6 100644
--- a/drivers/usb/fotg210/fotg210.h
+++ b/drivers/usb/fotg210/fotg210.h
@@ -2,6 +2,8 @@
 #ifndef __FOTG210_H
 #define __FOTG210_H
 
+#include <linux/usb/otg.h>
+
 enum gemini_port {
 	GEMINI_PORT_NONE = 0,
 	GEMINI_PORT_0,
@@ -15,6 +17,8 @@ struct fotg210 {
 	struct clk *pclk;
 	struct regmap *map;
 	enum gemini_port port;
+	enum usb_dr_mode mode;
+	bool is_fotg210;
 };
 
 void fotg210_vbus(struct fotg210 *fotg, bool enable);
@@ -22,22 +26,37 @@ void fotg210_vbus(struct fotg210 *fotg, bool enable);
 #ifdef CONFIG_USB_FOTG210_HCD
 int fotg210_hcd_probe(struct platform_device *pdev, struct fotg210 *fotg);
 int fotg210_hcd_remove(struct platform_device *pdev);
+int fotg210_hcd_suspend(struct device *dev);
+int fotg210_hcd_resume(struct device *dev);
 int fotg210_hcd_init(void);
 void fotg210_hcd_cleanup(void);
 #else
 static inline int fotg210_hcd_probe(struct platform_device *pdev,
 				    struct fotg210 *fotg)
 {
-	return 0;
+	return -ENODEV;
 }
+
 static inline int fotg210_hcd_remove(struct platform_device *pdev)
 {
-	return 0;
+	return -ENODEV;
+}
+
+static inline int fotg210_hcd_suspend(struct device *dev)
+{
+	return -ENODEV;
 }
+
+static inline int fotg210_hcd_resume(struct device *dev)
+{
+	return -ENODEV;
+}
+
 static inline int fotg210_hcd_init(void)
 {
 	return 0;
 }
+
 static inline void fotg210_hcd_cleanup(void)
 {
 }
@@ -50,8 +69,9 @@ int fotg210_udc_remove(struct platform_device *pdev);
 static inline int fotg210_udc_probe(struct platform_device *pdev,
 				    struct fotg210 *fotg)
 {
-	return 0;
+	return -ENODEV;
 }
+
 static inline int fotg210_udc_remove(struct platform_device *pdev)
 {
 	return 0;
diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
index df39e3487c1f3b..1b2238147cfe38 100644
--- a/drivers/usb/gadget/composite.c
+++ b/drivers/usb/gadget/composite.c
@@ -2740,7 +2740,7 @@ void usb_composite_setup_continue(struct usb_composite_dev *cdev)
 	spin_lock_irqsave(&cdev->lock, flags);
 
 	if (cdev->delayed_status == 0) {
-		WARN(cdev, "%s: Unexpected call\n", __func__);
+		DBG(cdev, "%s: Ignoring obsolete call\n", __func__);
 
 	} else if (--cdev->delayed_status == 0) {
 		DBG(cdev, "%s: Completing delayed status\n", __func__);
diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c
index fc4818fb2a8b82..e3c9b7ab9f39ba 100644
--- a/drivers/usb/gadget/function/f_mass_storage.c
+++ b/drivers/usb/gadget/function/f_mass_storage.c
@@ -463,7 +463,12 @@ static int fsg_setup(struct usb_function *f,
 	u16			w_value = le16_to_cpu(ctrl->wValue);
 	u16			w_length = le16_to_cpu(ctrl->wLength);
 
-	if (!fsg_is_set(fsg->common))
+	/*
+	 * fsg_is_set() warns when common->fsg is NULL, but that can happen
+	 * normally while a configuration change installs or removes the active
+	 * function.
+	 */
+	if (!fsg->common->fsg)
 		return -EOPNOTSUPP;
 
 	++fsg->common->ep0_req_tag;	/* Record arrival of a new request */
@@ -1103,9 +1108,9 @@ static int do_request_sense(struct fsg_common *common, struct fsg_buffhd *bh)
 	 * FSG normally uses option a); enable this code to use option b).
 	 */
 #if 0
-	if (curlun && curlun->unit_attention_data != SS_NO_SENSE) {
+	if (curlun && curlun->unit_attention_data != NO_SENSE) {
 		curlun->sense_data = curlun->unit_attention_data;
-		curlun->unit_attention_data = SS_NO_SENSE;
+		curlun->unit_attention_data = NO_SENSE;
 	}
 #endif
 
@@ -1118,18 +1123,18 @@ static int do_request_sense(struct fsg_common *common, struct fsg_buffhd *bh)
 		sd = curlun->sense_data;
 		sdinfo = curlun->sense_data_info;
 		valid = curlun->info_valid << 7;
-		curlun->sense_data = SS_NO_SENSE;
+		curlun->sense_data = NO_SENSE;
 		curlun->sense_data_info = 0;
 		curlun->info_valid = 0;
 	}
 
 	memset(buf, 0, 18);
 	buf[0] = valid | 0x70;			/* Valid, current error */
-	buf[2] = SK(sd);
+	buf[2] = usb_sense_key(sd);
 	put_unaligned_be32(sdinfo, &buf[3]);	/* Sense information */
 	buf[7] = 18 - 8;			/* Additional sense length */
-	buf[12] = ASC(sd);
-	buf[13] = ASCQ(sd);
+	buf[12] = usb_sense_asc(sd);
+	buf[13] = usb_sense_ascq(sd);
 	return 18;
 }
 
@@ -1678,21 +1683,23 @@ static void send_status(struct fsg_common *common)
 	if (curlun) {
 		sd = curlun->sense_data;
 		sdinfo = curlun->sense_data_info;
-	} else if (common->bad_lun_okay)
-		sd = SS_NO_SENSE;
-	else
+	} else if (common->bad_lun_okay) {
+		sd = NO_SENSE;
+	} else {
 		sd = SS_LOGICAL_UNIT_NOT_SUPPORTED;
+	}
 
 	if (common->phase_error) {
 		DBG(common, "sending phase-error status\n");
 		status = US_BULK_STAT_PHASE;
 		sd = SS_INVALID_COMMAND;
-	} else if (sd != SS_NO_SENSE) {
+	} else if (sd != NO_SENSE) {
 		DBG(common, "sending command-failure status\n");
 		status = US_BULK_STAT_FAIL;
 		VDBG(common, "  sense data: SK x%02x, ASC x%02x, ASCQ x%02x;"
 				"  info x%x\n",
-				SK(sd), ASC(sd), ASCQ(sd), sdinfo);
+		     usb_sense_key(sd), usb_sense_asc(sd),
+		     usb_sense_ascq(sd), sdinfo);
 	}
 
 	/* Store and send the Bulk-only CSW */
@@ -1798,7 +1805,7 @@ static int check_command(struct fsg_common *common, int cmnd_size,
 	curlun = common->curlun;
 	if (curlun) {
 		if (common->cmnd[0] != REQUEST_SENSE) {
-			curlun->sense_data = SS_NO_SENSE;
+			curlun->sense_data = NO_SENSE;
 			curlun->sense_data_info = 0;
 			curlun->info_valid = 0;
 		}
@@ -1820,11 +1827,11 @@ static int check_command(struct fsg_common *common, int cmnd_size,
 	 * If a unit attention condition exists, only INQUIRY and
 	 * REQUEST SENSE commands are allowed; anything else must fail.
 	 */
-	if (curlun && curlun->unit_attention_data != SS_NO_SENSE &&
+	if (curlun && curlun->unit_attention_data != NO_SENSE &&
 	    common->cmnd[0] != INQUIRY &&
 	    common->cmnd[0] != REQUEST_SENSE) {
 		curlun->sense_data = curlun->unit_attention_data;
-		curlun->unit_attention_data = SS_NO_SENSE;
+		curlun->unit_attention_data = NO_SENSE;
 		return -EINVAL;
 	}
 
@@ -2501,8 +2508,8 @@ static void handle_exception(struct fsg_common *common)
 			if (!curlun)
 				continue;
 			curlun->prevent_medium_removal = 0;
-			curlun->sense_data = SS_NO_SENSE;
-			curlun->unit_attention_data = SS_NO_SENSE;
+			curlun->sense_data = NO_SENSE;
+			curlun->unit_attention_data = NO_SENSE;
 			curlun->sense_data_info = 0;
 			curlun->info_valid = 0;
 		}
diff --git a/drivers/usb/gadget/function/f_midi2.c b/drivers/usb/gadget/function/f_midi2.c
index 5b8b18281989c0..d693de427641dd 100644
--- a/drivers/usb/gadget/function/f_midi2.c
+++ b/drivers/usb/gadget/function/f_midi2.c
@@ -1633,8 +1633,8 @@ struct f_midi2_usb_config {
 
 	/* MIDI 1.0 jacks */
 	unsigned char jack_in, jack_out, jack_id;
-	struct usb_midi_in_jack_descriptor jack_ins[MAX_CABLES];
-	struct usb_midi_out_jack_descriptor_1 jack_outs[MAX_CABLES];
+	struct usb_midi_in_jack_descriptor jack_ins[MAX_CABLES * 2];
+	struct usb_midi_out_jack_descriptor_1 jack_outs[MAX_CABLES * 2];
 };
 
 static int append_config(struct f_midi2_usb_config *config, void *d)
diff --git a/drivers/usb/gadget/function/storage_common.h b/drivers/usb/gadget/function/storage_common.h
index 11ac785d5eee26..e73a30cc553fc4 100644
--- a/drivers/usb/gadget/function/storage_common.h
+++ b/drivers/usb/gadget/function/storage_common.h
@@ -70,24 +70,59 @@ do {									\
 #define MAX_COMMAND_SIZE	16
 
 /* SCSI Sense Key/Additional Sense Code/ASC Qualifier values */
-#define SS_NO_SENSE				0
-#define SS_COMMUNICATION_FAILURE		0x040800
-#define SS_INVALID_COMMAND			0x052000
-#define SS_INVALID_FIELD_IN_CDB			0x052400
-#define SS_LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE	0x052100
-#define SS_LOGICAL_UNIT_NOT_SUPPORTED		0x052500
-#define SS_MEDIUM_NOT_PRESENT			0x023a00
-#define SS_MEDIUM_REMOVAL_PREVENTED		0x055302
-#define SS_NOT_READY_TO_READY_TRANSITION	0x062800
-#define SS_RESET_OCCURRED			0x062900
-#define SS_SAVING_PARAMETERS_NOT_SUPPORTED	0x053900
-#define SS_UNRECOVERED_READ_ERROR		0x031100
-#define SS_WRITE_ERROR				0x030c02
-#define SS_WRITE_PROTECTED			0x072700
-
-#define SK(x)		((u8) ((x) >> 16))	/* Sense Key byte, etc. */
-#define ASC(x)		((u8) ((x) >> 8))
-#define ASCQ(x)		((u8) (x))
+static inline u32 usb_sense_kc(u8 sense_key, u16 sense_code)
+{
+	return ((u32)sense_key << 16) | (u32)sense_code;
+}
+
+static inline u8 usb_sense_key(u32 kc)
+{
+	return kc >> 16;
+}
+
+static inline u16 usb_sense_code(u32 kc)
+{
+	return kc & 0xFFFF;
+}
+
+static inline u8 usb_sense_asc(u32 kc)
+{
+	return scsi_sense_code_asc(usb_sense_code(kc));
+}
+
+static inline u8 usb_sense_ascq(u32 kc)
+{
+	return scsi_sense_code_ascq(usb_sense_code(kc));
+}
+
+#define SS_COMMUNICATION_FAILURE		\
+	usb_sense_kc(HARDWARE_ERROR, LU_COMMUNICATION_FAILURE)
+#define SS_INVALID_COMMAND			\
+	usb_sense_kc(ILLEGAL_REQUEST, INVALID_COMMAND_OP_CODE)
+#define SS_INVALID_FIELD_IN_CDB			\
+	usb_sense_kc(ILLEGAL_REQUEST, INVALID_FIELD_IN_CDB)
+#define SS_LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE	\
+	usb_sense_kc(ILLEGAL_REQUEST, LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE)
+#define SS_LOGICAL_UNIT_NOT_SUPPORTED		\
+	usb_sense_kc(ILLEGAL_REQUEST, LU_NOT_SUPPORTED)
+#define SS_MEDIUM_NOT_PRESENT			\
+	usb_sense_kc(NOT_READY, MEDIUM_NOT_PRESENT)
+#define SS_MEDIUM_REMOVAL_PREVENTED		\
+	usb_sense_kc(ILLEGAL_REQUEST, MEDIUM_REMOVAL_PREVENTED)
+#define SS_NOT_READY_TO_READY_TRANSITION	\
+	usb_sense_kc(UNIT_ATTENTION,		\
+		     NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED)
+#define SS_RESET_OCCURRED			\
+	usb_sense_kc(UNIT_ATTENTION,		\
+		     POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED)
+#define SS_SAVING_PARAMETERS_NOT_SUPPORTED	\
+	usb_sense_kc(ILLEGAL_REQUEST, SAVING_PARAMETERS_NOT_SUPPORTED)
+#define SS_UNRECOVERED_READ_ERROR		\
+	usb_sense_kc(MEDIUM_ERROR, UNRECOVERED_READ_ERROR)
+#define SS_WRITE_ERROR				\
+	usb_sense_kc(MEDIUM_ERROR, WRITE_ERROR_AUTO_REALLOCATION_FAILED)
+#define SS_WRITE_PROTECTED			\
+	usb_sense_kc(DATA_PROTECT, WRITE_PROTECTED)
 
 /*
  * Vendor (8 chars), product (16 chars), release (4 hexadecimal digits) and NUL
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index 67c6ffaf4f72d6..7e383e3e0f664d 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -613,7 +613,7 @@ ep_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		return -EBADMSG;
 	}
 
-	buf = kmalloc(len, GFP_KERNEL);
+	buf = kmalloc(len, GFP_KERNEL | __GFP_NOWARN);
 	if (unlikely(!buf)) {
 		mutex_unlock(&epdata->lock);
 		return -ENOMEM;
@@ -675,7 +675,7 @@ ep_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		return -EBADMSG;
 	}
 
-	buf = kmalloc(len, GFP_KERNEL);
+	buf = kmalloc(len, GFP_KERNEL | __GFP_NOWARN);
 	if (unlikely(!buf)) {
 		mutex_unlock(&epdata->lock);
 		return -ENOMEM;
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/core.c b/drivers/usb/gadget/udc/aspeed-vhub/core.c
index 4a8b9ff8368fcf..4ac990553a86e0 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/core.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/core.c
@@ -379,10 +379,8 @@ static int ast_vhub_probe(struct platform_device *pdev)
 	}
 	rc = devm_request_irq(&pdev->dev, vhub->irq, ast_vhub_irq, 0,
 			      KBUILD_MODNAME, vhub);
-	if (rc) {
-		dev_err(&pdev->dev, "Failed to request interrupt\n");
+	if (rc)
 		goto err;
-	}
 
 	dma_mask_ptr = (u64 *)of_device_get_match_data(&pdev->dev);
 	if (dma_mask_ptr) {
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/dev.c b/drivers/usb/gadget/udc/aspeed-vhub/dev.c
index 8b9449d16324e3..0dccdbfd6aefd7 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/dev.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/dev.c
@@ -592,8 +592,6 @@ int ast_vhub_init_dev(struct ast_vhub *vhub, unsigned int idx)
 	else
 		d->gadget.max_speed = USB_SPEED_HIGH;
 	d->gadget.speed = USB_SPEED_UNKNOWN;
-	d->gadget.dev.of_node = vhub->pdev->dev.of_node;
-	dev_set_of_node_reused(&d->gadget.dev);
 
 	rc = usb_add_gadget_udc(d->port_dev, &d->gadget);
 	if (rc != 0)
diff --git a/drivers/usb/gadget/udc/aspeed_udc.c b/drivers/usb/gadget/udc/aspeed_udc.c
index 54f81e66800945..81fa495bbacd15 100644
--- a/drivers/usb/gadget/udc/aspeed_udc.c
+++ b/drivers/usb/gadget/udc/aspeed_udc.c
@@ -1299,7 +1299,6 @@ static int ast_udc_start(struct usb_gadget *gadget,
 
 	UDC_DBG(udc, "\n");
 	udc->driver = driver;
-	udc->gadget.dev.of_node = udc->pdev->dev.of_node;
 
 	for (i = 0; i < AST_UDC_NUM_ENDPOINTS; i++) {
 		ep = &udc->ep[i];
@@ -1568,10 +1567,8 @@ static int ast_udc_probe(struct platform_device *pdev)
 
 	rc = devm_request_irq(&pdev->dev, udc->irq, ast_udc_isr, 0,
 			      KBUILD_MODNAME, udc);
-	if (rc) {
-		dev_err(&pdev->dev, "Failed to request interrupt\n");
+	if (rc)
 		goto err_cleanup;
-	}
 
 	rc = usb_add_gadget_udc(&pdev->dev, &udc->gadget);
 	if (rc) {
diff --git a/drivers/usb/gadget/udc/at91_udc.c b/drivers/usb/gadget/udc/at91_udc.c
index 8e5d7fb7154920..0fccca1878033a 100644
--- a/drivers/usb/gadget/udc/at91_udc.c
+++ b/drivers/usb/gadget/udc/at91_udc.c
@@ -1558,7 +1558,6 @@ static int at91_start(struct usb_gadget *gadget,
 
 	udc = container_of(gadget, struct at91_udc, gadget);
 	udc->driver = driver;
-	udc->gadget.dev.of_node = udc->pdev->dev.of_node;
 	udc->enabled = 1;
 	udc->gadget.is_selfpowered = 1;
 
diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.c b/drivers/usb/gadget/udc/atmel_usba_udc.c
index 72a2f95ff0188f..18b4b2358d1ea0 100644
--- a/drivers/usb/gadget/udc/atmel_usba_udc.c
+++ b/drivers/usb/gadget/udc/atmel_usba_udc.c
@@ -2332,11 +2332,8 @@ static int usba_udc_probe(struct platform_device *pdev)
 
 	ret = devm_request_irq(&pdev->dev, irq, usba_udc_irq, 0,
 				"atmel_usba_udc", udc);
-	if (ret) {
-		dev_err(&pdev->dev, "Cannot request irq %d (error %d)\n",
-			irq, ret);
+	if (ret)
 		return ret;
-	}
 	udc->irq = irq;
 
 	if (udc->vbus_pin) {
diff --git a/drivers/usb/gadget/udc/bcm63xx_udc.c b/drivers/usb/gadget/udc/bcm63xx_udc.c
index c4f9ea45bdbb6e..ba91b1ebbf57eb 100644
--- a/drivers/usb/gadget/udc/bcm63xx_udc.c
+++ b/drivers/usb/gadget/udc/bcm63xx_udc.c
@@ -1830,7 +1830,6 @@ static int bcm63xx_udc_start(struct usb_gadget *gadget,
 	bcm63xx_select_phy_mode(udc, true);
 
 	udc->driver = driver;
-	udc->gadget.dev.of_node = udc->dev->of_node;
 
 	spin_unlock_irqrestore(&udc->lock, flags);
 
diff --git a/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
index 308d3c468ab197..8719f1f86e6106 100644
--- a/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
+++ b/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c
@@ -2341,7 +2341,7 @@ static int cdns2_gadget_start(struct cdns2_device *pdev)
 
 	/* Allocate memory for setup packet buffer. */
 	buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma,
-				 GFP_DMA);
+				 GFP_KERNEL);
 	pdev->ep0_preq.request.buf = buf;
 
 	if (!pdev->ep0_preq.request.buf) {
diff --git a/drivers/usb/gadget/udc/fsl_udc_core.c b/drivers/usb/gadget/udc/fsl_udc_core.c
index 20392409afb152..878d70b55530cb 100644
--- a/drivers/usb/gadget/udc/fsl_udc_core.c
+++ b/drivers/usb/gadget/udc/fsl_udc_core.c
@@ -2465,9 +2465,6 @@ static int fsl_udc_probe(struct platform_device *pdev)
 	udc_controller->gadget.speed = USB_SPEED_UNKNOWN;
 	udc_controller->gadget.name = driver_name;
 
-	/* Setup gadget.dev and register with kernel */
-	udc_controller->gadget.dev.of_node = pdev->dev.of_node;
-
 	if (!IS_ERR_OR_NULL(udc_controller->transceiver))
 		udc_controller->gadget.is_otg = 1;
 
diff --git a/drivers/usb/gadget/udc/lpc32xx_udc.c b/drivers/usb/gadget/udc/lpc32xx_udc.c
index 044c31869cfb26..94355d3d930fd6 100644
--- a/drivers/usb/gadget/udc/lpc32xx_udc.c
+++ b/drivers/usb/gadget/udc/lpc32xx_udc.c
@@ -2889,7 +2889,6 @@ static int lpc32xx_start(struct usb_gadget *gadget,
 	}
 
 	udc->driver = driver;
-	udc->gadget.dev.of_node = udc->dev->of_node;
 	udc->enabled = 1;
 	udc->gadget.is_selfpowered = 1;
 	udc->vbus = 0;
@@ -3080,7 +3079,7 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	/* Allocate memory for the UDCA */
 	udc->udca_v_base = dma_alloc_coherent(&pdev->dev, UDCA_BUFF_SIZE,
 					      &dma_handle,
-					      (GFP_KERNEL | GFP_DMA));
+					      GFP_KERNEL);
 	if (!udc->udca_v_base) {
 		dev_err(udc->dev, "error getting UDCA region\n");
 		retval = -ENOMEM;
diff --git a/drivers/usb/gadget/udc/r8a66597-udc.c b/drivers/usb/gadget/udc/r8a66597-udc.c
index d190e16d43fc03..915786c9628c14 100644
--- a/drivers/usb/gadget/udc/r8a66597-udc.c
+++ b/drivers/usb/gadget/udc/r8a66597-udc.c
@@ -1894,10 +1894,8 @@ static int r8a66597_probe(struct platform_device *pdev)
 
 	ret = devm_request_irq(dev, irq, r8a66597_irq, IRQF_SHARED,
 			       udc_name, r8a66597);
-	if (ret < 0) {
-		dev_err(dev, "request_irq error (%d)\n", ret);
+	if (ret < 0)
 		goto clean_up2;
-	}
 
 	INIT_LIST_HEAD(&r8a66597->gadget.ep_list);
 	r8a66597->gadget.ep0 = &r8a66597->ep[0].ep;
diff --git a/drivers/usb/gadget/udc/renesas_usbf.c b/drivers/usb/gadget/udc/renesas_usbf.c
index d67002ea049a08..367c9e73f366aa 100644
--- a/drivers/usb/gadget/udc/renesas_usbf.c
+++ b/drivers/usb/gadget/udc/renesas_usbf.c
@@ -3335,19 +3335,15 @@ static int usbf_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 	ret = devm_request_irq(dev, irq, usbf_epc_irq, 0, "usbf-epc", udc);
-	if (ret) {
-		dev_err(dev, "cannot request irq %d err %d\n", irq, ret);
+	if (ret)
 		return ret;
-	}
 
 	irq = platform_get_irq(pdev, 1);
 	if (irq < 0)
 		return irq;
 	ret = devm_request_irq(dev, irq, usbf_ahb_epc_irq, 0, "usbf-ahb-epc", udc);
-	if (ret) {
-		dev_err(dev, "cannot request irq %d err %d\n", irq, ret);
+	if (ret)
 		return ret;
-	}
 
 	usbf_reg_bitset(udc, USBF_REG_AHBMCTR, USBF_SYS_WBURST_TYPE);
 
diff --git a/drivers/usb/gadget/udc/snps_udc_plat.c b/drivers/usb/gadget/udc/snps_udc_plat.c
index 0ee3ed185d9c3b..8ce43c4f8fba62 100644
--- a/drivers/usb/gadget/udc/snps_udc_plat.c
+++ b/drivers/usb/gadget/udc/snps_udc_plat.c
@@ -192,10 +192,8 @@ static int udc_plat_probe(struct platform_device *pdev)
 
 	ret = devm_request_irq(dev, udc->irq, udc_irq, IRQF_SHARED,
 			       "snps-udc", udc);
-	if (ret < 0) {
-		dev_err(dev, "Request irq %d failed for UDC\n", udc->irq);
+	if (ret < 0)
 		goto exit_dma;
-	}
 
 	platform_set_drvdata(pdev, udc);
 	udc->chiprev = UDC_BCM_REV;
diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c
index e9d33be0286646..5042c13f92be67 100644
--- a/drivers/usb/gadget/udc/tegra-xudc.c
+++ b/drivers/usb/gadget/udc/tegra-xudc.c
@@ -3803,11 +3803,8 @@ static int tegra_xudc_probe(struct platform_device *pdev)
 
 	err = devm_request_irq(&pdev->dev, xudc->irq, tegra_xudc_irq, 0,
 			       dev_name(&pdev->dev), xudc);
-	if (err < 0) {
-		dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
-			err);
+	if (err < 0)
 		return err;
-	}
 
 	xudc->clks = devm_kcalloc(&pdev->dev, xudc->soc->num_clks, sizeof(*xudc->clks),
 				  GFP_KERNEL);
diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig
index 10cff12a3f31ea..05a88969a6a031 100644
--- a/drivers/usb/host/Kconfig
+++ b/drivers/usb/host/Kconfig
@@ -211,6 +211,19 @@ config USB_EHCI_PCI
 	depends on USB_PCI
 	default y
 
+config USB_EHCI_DEVIANT_PORT_STATUS_REG
+	bool
+	# Used for hosts with a deviant port status register location
+
+config USB_EHCI_PORT_RESET_HOOKS
+	bool
+	# Used for hosts with controller-specific port reset sequencing
+
+config USB_EHCI_PORT_SPEED_HOOK
+	bool
+	depends on USB_EHCI_ROOT_HUB_TT
+	# Used for hosts with controller-specific port speed reporting
+
 config XPS_USB_HCD_XILINX
 	bool "Use Xilinx usb host EHCI controller core"
 	depends on (PPC32 || MICROBLAZE)
diff --git a/drivers/usb/host/ehci-dbg.c b/drivers/usb/host/ehci-dbg.c
index 3cf743c81eb436..51ec6ea4315841 100644
--- a/drivers/usb/host/ehci-dbg.c
+++ b/drivers/usb/host/ehci-dbg.c
@@ -869,8 +869,7 @@ static ssize_t fill_registers_buffer(struct debug_buffer *buf)
 
 	for (i = 1; i <= HCS_N_PORTS(ehci->hcs_params); i++) {
 		temp = dbg_port_buf(scratch, sizeof(scratch), label, i,
-				ehci_readl(ehci,
-					&ehci->regs->port_status[i - 1]));
+				ehci_readl(ehci, ehci_portsc(ehci, i - 1)));
 		temp = scnprintf(next, size, fmt, temp, scratch);
 		size -= temp;
 		next += temp;
diff --git a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c
index 3c46bb18c7f3aa..75f780ec3ba0f4 100644
--- a/drivers/usb/host/ehci-hcd.c
+++ b/drivers/usb/host/ehci-hcd.c
@@ -187,7 +187,8 @@ static int ehci_halt (struct ehci_hcd *ehci)
 	/* disable any irqs left enabled by previous code */
 	ehci_writel(ehci, 0, &ehci->regs->intr_enable);
 
-	if (ehci_is_TDI(ehci) && !tdi_in_host_mode(ehci)) {
+	if (ehci_is_TDI(ehci) && !ehci->no_tdi_mode &&
+	    !tdi_in_host_mode(ehci)) {
 		spin_unlock_irq(&ehci->lock);
 		return 0;
 	}
@@ -254,7 +255,7 @@ int ehci_reset(struct ehci_hcd *ehci)
 	if (retval)
 		return retval;
 
-	if (ehci_is_TDI(ehci))
+	if (ehci_is_TDI(ehci) && !ehci->no_tdi_mode)
 		tdi_reset (ehci);
 
 	if (ehci->debug)
@@ -325,7 +326,7 @@ static void ehci_turn_off_all_ports(struct ehci_hcd *ehci)
 		ehci_port_power(ehci, port, false);
 		spin_lock_irq(&ehci->lock);
 		ehci_writel(ehci, PORT_RWC_BITS,
-				&ehci->regs->port_status[port]);
+				ehci_portsc(ehci, port));
 	}
 }
 
@@ -342,10 +343,12 @@ static void ehci_silence_controller(struct ehci_hcd *ehci)
 	ehci_turn_off_all_ports(ehci);
 
 	/* make BIOS/etc use companion controller during reboot */
-	ehci_writel(ehci, 0, &ehci->regs->configured_flag);
+	if (!ehci->no_configured_flag) {
+		ehci_writel(ehci, 0, &ehci->regs->configured_flag);
 
-	/* unblock posted writes */
-	ehci_readl(ehci, &ehci->regs->configured_flag);
+		/* unblock posted writes */
+		ehci_readl(ehci, &ehci->regs->configured_flag);
+	}
 	spin_unlock_irq(&ehci->lock);
 }
 
@@ -629,7 +632,8 @@ static int ehci_run (struct usb_hcd *hcd)
 	 */
 	down_write(&ehci_cf_port_reset_rwsem);
 	ehci->rh_state = EHCI_RH_RUNNING;
-	ehci_writel(ehci, FLAG_CF, &ehci->regs->configured_flag);
+	if (!ehci->no_configured_flag)
+		ehci_writel(ehci, FLAG_CF, &ehci->regs->configured_flag);
 
 	/* Wait until HC become operational */
 	ehci_readl(ehci, &ehci->regs->command);	/* unblock posted writes */
@@ -811,8 +815,7 @@ static irqreturn_t ehci_irq (struct usb_hcd *hcd)
 			/* leverage per-port change bits feature */
 			if (!(ppcd & (1 << i)))
 				continue;
-			pstatus = ehci_readl(ehci,
-					 &ehci->regs->port_status[i]);
+			pstatus = ehci_readl(ehci, ehci_portsc(ehci, i));
 
 			if (pstatus & PORT_OWNER)
 				continue;
@@ -910,6 +913,8 @@ static int ehci_urb_enqueue (
 	case PIPE_ISOCHRONOUS:
 		if (urb->dev->speed == USB_SPEED_HIGH)
 			return itd_submit (ehci, urb, mem_flags);
+		else if (ehci->no_fsls_isoc)
+			return -EOPNOTSUPP;
 		else
 			return sitd_submit (ehci, urb, mem_flags);
 	}
@@ -1109,7 +1114,7 @@ static void ehci_remove_device(struct usb_hcd *hcd, struct usb_device *udev)
 /* Clear wakeup signal locked in zhaoxin platform when device plug in. */
 static void ehci_zx_wakeup_clear(struct ehci_hcd *ehci)
 {
-	u32 __iomem	*reg = &ehci->regs->port_status[4];
+	u32 __iomem	*reg = ehci_portsc(ehci, 4);
 	u32 		t1 = ehci_readl(ehci, reg);
 
 	t1 &= (u32)~0xf0000;
@@ -1185,7 +1190,8 @@ int ehci_resume(struct usb_hcd *hcd, bool force_reset)
 	 * then we maintained suspend power.
 	 * Just undo the effect of ehci_suspend().
 	 */
-	if (ehci_readl(ehci, &ehci->regs->configured_flag) == FLAG_CF &&
+	if ((ehci->no_configured_flag ||
+	     ehci_readl(ehci, &ehci->regs->configured_flag) == FLAG_CF) &&
 			!force_reset) {
 		int	mask = INTR_MASK;
 
@@ -1217,7 +1223,8 @@ int ehci_resume(struct usb_hcd *hcd, bool force_reset)
 		goto skip;
 
 	ehci_writel(ehci, ehci->command, &ehci->regs->command);
-	ehci_writel(ehci, FLAG_CF, &ehci->regs->configured_flag);
+	if (!ehci->no_configured_flag)
+		ehci_writel(ehci, FLAG_CF, &ehci->regs->configured_flag);
 	ehci_readl(ehci, &ehci->regs->command);	/* unblock posted writes */
 
 	ehci->rh_state = EHCI_RH_SUSPENDED;
diff --git a/drivers/usb/host/ehci-hub.c b/drivers/usb/host/ehci-hub.c
index 1aee392e84927c..e676c98b6baafe 100644
--- a/drivers/usb/host/ehci-hub.c
+++ b/drivers/usb/host/ehci-hub.c
@@ -15,6 +15,44 @@
 
 /*-------------------------------------------------------------------------*/
 
+/*
+ * Some EHCI controllers have a Transaction Translator built into the
+ * root hub. This is a non-standard feature.  Each controller will need
+ * to add code to the following function, and call it as needed.
+ */
+
+/*
+ * In a bunch of EHCI implementations with transaction translators,
+ * the port speed can be found in the reserved bits in position 26 and
+ * 27. Implementations with the HOSTPC register will have this in
+ * bits 25 and 26 of the HOSTPC registers.
+ */
+#define PORTSC_SPEED_BITS(a)	(((a) >> 26) & 3)
+#define HOSTPC_SPEED_BITS(a)	(((a) >> 25) & 3)
+
+/* Returns the speed of a device attached to a port on the root hub. */
+static unsigned int ehci_port_speed(struct ehci_hcd *ehci,
+				    unsigned int port, unsigned int speed)
+{
+#ifdef CONFIG_USB_EHCI_PORT_SPEED_HOOK
+	if (ehci->get_port_speed)
+		return ehci->get_port_speed(ehci, port);
+#endif
+
+	if (!IS_ENABLED(CONFIG_USB_EHCI_ROOT_HUB_TT) || !ehci_is_TDI(ehci))
+		return USB_PORT_STAT_HIGH_SPEED;
+
+	switch (speed) {
+	case 0:
+		return 0;
+	case 1:
+		return USB_PORT_STAT_LOW_SPEED;
+	case 2:
+	default:
+		return USB_PORT_STAT_HIGH_SPEED;
+	}
+}
+
 #define	PORT_WAKE_BITS	(PORT_WKOC_E|PORT_WKDISC_E|PORT_WKCONN_E)
 
 #ifdef	CONFIG_PM
@@ -55,7 +93,7 @@ static void ehci_handover_companion_ports(struct ehci_hcd *ehci)
 	port = HCS_N_PORTS(ehci->hcs_params);
 	while (port--) {
 		if (test_bit(port, &ehci->owned_ports)) {
-			reg = &ehci->regs->port_status[port];
+			reg = ehci_portsc(ehci, port);
 			status = ehci_readl(ehci, reg) & ~PORT_RWC_BITS;
 			if (!(status & PORT_POWER))
 				ehci_port_power(ehci, port, true);
@@ -69,7 +107,7 @@ static void ehci_handover_companion_ports(struct ehci_hcd *ehci)
 	port = HCS_N_PORTS(ehci->hcs_params);
 	while (port--) {
 		if (test_bit(port, &ehci->owned_ports)) {
-			reg = &ehci->regs->port_status[port];
+			reg = ehci_portsc(ehci, port);
 			status = ehci_readl(ehci, reg) & ~PORT_RWC_BITS;
 
 			/* Port already owned by companion? */
@@ -106,7 +144,7 @@ static void ehci_handover_companion_ports(struct ehci_hcd *ehci)
 			 * but if something went wrong the port must not
 			 * remain enabled.
 			 */
-			reg = &ehci->regs->port_status[port];
+			reg = ehci_portsc(ehci, port);
 			status = ehci_readl(ehci, reg) & ~PORT_RWC_BITS;
 			if (status & PORT_OWNER)
 				ehci_writel(ehci, status | PORT_CSC, reg);
@@ -137,7 +175,7 @@ static int ehci_port_change(struct ehci_hcd *ehci)
 	 */
 
 	while (i--)
-		if (ehci_readl(ehci, &ehci->regs->port_status[i]) & PORT_CSC)
+		if (ehci_readl(ehci, ehci_portsc(ehci, i)) & PORT_CSC)
 			return 1;
 
 	return 0;
@@ -175,7 +213,7 @@ void ehci_adjust_port_wakeup_flags(struct ehci_hcd *ehci,
 
 	port = HCS_N_PORTS(ehci->hcs_params);
 	while (port--) {
-		u32 __iomem	*reg = &ehci->regs->port_status[port];
+		u32 __iomem	*reg = ehci_portsc(ehci, port);
 		u32		t1 = ehci_readl(ehci, reg) & ~PORT_RWC_BITS;
 		u32		t2 = t1 & ~PORT_WAKE_BITS;
 
@@ -254,7 +292,7 @@ static int ehci_bus_suspend (struct usb_hcd *hcd)
 	fs_idle_delay = false;
 	port = HCS_N_PORTS(ehci->hcs_params);
 	while (port--) {
-		u32 __iomem	*reg = &ehci->regs->port_status [port];
+		u32 __iomem	*reg = ehci_portsc(ehci, port);
 		u32		t1 = ehci_readl(ehci, reg) & ~PORT_RWC_BITS;
 		u32		t2 = t1 & ~PORT_WAKE_BITS;
 
@@ -287,8 +325,9 @@ static int ehci_bus_suspend (struct usb_hcd *hcd)
 			 * sake, add a delay if we need one.
 			 */
 			if ((t2 & PORT_WKDISC_E) &&
-					ehci_port_speed(ehci, t2) ==
-						USB_PORT_STAT_HIGH_SPEED)
+			    ehci_port_speed(ehci, port,
+					    PORTSC_SPEED_BITS(t2)) ==
+					    USB_PORT_STAT_HIGH_SPEED)
 				fs_idle_delay = true;
 			ehci_writel(ehci, t2, reg);
 			changed = 1;
@@ -426,7 +465,7 @@ static int ehci_bus_resume (struct usb_hcd *hcd)
 	 */
 	i = HCS_N_PORTS(ehci->hcs_params);
 	while (i--) {
-		temp = ehci_readl(ehci, &ehci->regs->port_status[i]);
+		temp = ehci_readl(ehci, ehci_portsc(ehci, i));
 		if ((temp & PORT_PE) &&
 				!(temp & (PORT_SUSPEND | PORT_RESUME))) {
 			ehci_dbg(ehci, "Port status(0x%x) is wrong\n", temp);
@@ -463,14 +502,14 @@ static int ehci_bus_resume (struct usb_hcd *hcd)
 	/* manually resume the ports we suspended during bus_suspend() */
 	i = HCS_N_PORTS (ehci->hcs_params);
 	while (i--) {
-		temp = ehci_readl(ehci, &ehci->regs->port_status [i]);
+		temp = ehci_readl(ehci, ehci_portsc(ehci, i));
 		temp &= ~(PORT_RWC_BITS | PORT_WAKE_BITS);
 		if (test_bit(i, &ehci->bus_suspended) &&
 				(temp & PORT_SUSPEND)) {
 			temp |= PORT_RESUME;
 			set_bit(i, &resume_needed);
 		}
-		ehci_writel(ehci, temp, &ehci->regs->port_status [i]);
+		ehci_writel(ehci, temp, ehci_portsc(ehci, i));
 	}
 
 	/*
@@ -487,10 +526,10 @@ static int ehci_bus_resume (struct usb_hcd *hcd)
 
 	i = HCS_N_PORTS (ehci->hcs_params);
 	while (i--) {
-		temp = ehci_readl(ehci, &ehci->regs->port_status [i]);
+		temp = ehci_readl(ehci, ehci_portsc(ehci, i));
 		if (test_bit(i, &resume_needed)) {
 			temp &= ~(PORT_RWC_BITS | PORT_SUSPEND | PORT_RESUME);
-			ehci_writel(ehci, temp, &ehci->regs->port_status [i]);
+			ehci_writel(ehci, temp, ehci_portsc(ehci, i));
 		}
 	}
 
@@ -540,7 +579,7 @@ static void set_owner(struct ehci_hcd *ehci, int portnum, int new_owner)
 	u32			port_status;
 	int 			try;
 
-	status_reg = &ehci->regs->port_status[portnum];
+	status_reg = ehci_portsc(ehci, portnum);
 
 	/*
 	 * The controller won't set the OWNER bit if the port is
@@ -661,7 +700,7 @@ ehci_hub_status_data (struct usb_hcd *hcd, char *buf)
 	for (i = 0; i < ports; i++) {
 		/* leverage per-port change bits feature */
 		if (ppcd & (1 << i))
-			temp = ehci_readl(ehci, &ehci->regs->port_status[i]);
+			temp = ehci_readl(ehci, ehci_portsc(ehci, i));
 		else
 			temp = 0;
 
@@ -753,7 +792,7 @@ int ehci_hub_control(
 	temp = (wIndex - 1) & 0xff;
 	if (temp >= HCS_N_PORTS_MAX)
 		temp = 0;
-	status_reg = &ehci->regs->port_status[temp];
+	status_reg = ehci_portsc(ehci, temp);
 	hostpc_reg = &ehci->regs->hostpc[temp];
 
 	/*
@@ -963,6 +1002,13 @@ int ehci_hub_control(
 			/* see what we found out */
 			temp = check_reset_complete (ehci, wIndex, status_reg,
 					ehci_readl(ehci, status_reg));
+#ifdef CONFIG_USB_EHCI_PORT_RESET_HOOKS
+			if (ehci->post_port_reset) {
+				retval = ehci->post_port_reset(ehci, wIndex);
+				if (retval)
+					goto error_exit;
+			}
+#endif
 		}
 
 		/* transfer dedicated ports to the companion hc */
@@ -983,12 +1029,14 @@ int ehci_hub_control(
 
 		if (temp & PORT_CONNECT) {
 			status |= USB_PORT_STAT_CONNECTION;
-			// status may be from integrated TT
 			if (ehci->has_hostpc) {
 				temp1 = ehci_readl(ehci, hostpc_reg);
-				status |= ehci_port_speed(ehci, temp1);
-			} else
-				status |= ehci_port_speed(ehci, temp);
+				status |= ehci_port_speed(ehci, wIndex,
+							  HOSTPC_SPEED_BITS(temp1));
+			} else {
+				status |= ehci_port_speed(ehci, wIndex,
+							  PORTSC_SPEED_BITS(temp));
+			}
 		}
 		if (temp & PORT_PE)
 			status |= USB_PORT_STAT_ENABLE;
@@ -1121,7 +1169,16 @@ int ehci_hub_control(
 				 */
 				if (ehci_has_fsl_hs_errata(ehci))
 					temp |= (1 << PORTSC_FSL_PFSC);
+
+#ifdef CONFIG_USB_EHCI_PORT_RESET_HOOKS
+				if (ehci->pre_port_reset) {
+					retval = ehci->pre_port_reset(ehci, wIndex);
+					if (retval)
+						goto error_exit;
+				}
+#endif
 			}
+
 			ehci_writel(ehci, temp, status_reg);
 			break;
 
@@ -1149,8 +1206,7 @@ int ehci_hub_control(
 
 			/* Put all enabled ports into suspend */
 			while (ports--) {
-				u32 __iomem *sreg =
-						&ehci->regs->port_status[ports];
+				u32 __iomem *sreg = ehci_portsc(ehci, ports);
 
 				temp = ehci_readl(ehci, sreg) & ~PORT_RWC_BITS;
 				if (temp & PORT_PE)
@@ -1200,14 +1256,14 @@ static int ehci_port_handed_over(struct usb_hcd *hcd, int portnum)
 
 	if (ehci_is_TDI(ehci))
 		return 0;
-	reg = &ehci->regs->port_status[portnum - 1];
+	reg = ehci_portsc(ehci, portnum - 1);
 	return ehci_readl(ehci, reg) & PORT_OWNER;
 }
 
 static int ehci_port_power(struct ehci_hcd *ehci, int portnum, bool enable)
 {
 	struct usb_hcd *hcd = ehci_to_hcd(ehci);
-	u32 __iomem *status_reg = &ehci->regs->port_status[portnum];
+	u32 __iomem *status_reg = ehci_portsc(ehci, portnum);
 	u32 temp = ehci_readl(ehci, status_reg) & ~PORT_RWC_BITS;
 
 	if (enable)
diff --git a/drivers/usb/host/ehci-ppc-of.c b/drivers/usb/host/ehci-ppc-of.c
index 8063b9d3aebd08..457d15da40a3ce 100644
--- a/drivers/usb/host/ehci-ppc-of.c
+++ b/drivers/usb/host/ehci-ppc-of.c
@@ -18,8 +18,6 @@
 
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/of_platform.h>
 
 
 static const struct hc_driver ehci_ppc_of_hc_driver = {
@@ -96,7 +94,9 @@ static int ehci_hcd_ppc_of_probe(struct platform_device *op)
 	struct device_node *dn = op->dev.of_node;
 	struct usb_hcd *hcd;
 	struct ehci_hcd	*ehci = NULL;
-	struct resource res;
+	struct resource *res;
+	struct resource ohci_res;
+	void __iomem *regs;
 	int irq;
 	int rv;
 
@@ -107,39 +107,30 @@ static int ehci_hcd_ppc_of_probe(struct platform_device *op)
 
 	dev_dbg(&op->dev, "initializing PPC-OF USB Controller\n");
 
-	rv = of_address_to_resource(dn, 0, &res);
-	if (rv)
-		return rv;
+	regs = devm_platform_get_and_ioremap_resource(op, 0, &res);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	irq = platform_get_irq(op, 0);
+	if (irq	< 0)
+		return irq;
 
 	hcd = usb_create_hcd(&ehci_ppc_of_hc_driver, &op->dev, "PPC-OF USB");
 	if (!hcd)
 		return -ENOMEM;
 
-	hcd->rsrc_start = res.start;
-	hcd->rsrc_len = resource_size(&res);
-
-	irq = irq_of_parse_and_map(dn, 0);
-	if (!irq) {
-		dev_err(&op->dev, "%s: irq_of_parse_and_map failed\n",
-			__FILE__);
-		rv = -EBUSY;
-		goto err_irq;
-	}
-
-	hcd->regs = devm_ioremap_resource(&op->dev, &res);
-	if (IS_ERR(hcd->regs)) {
-		rv = PTR_ERR(hcd->regs);
-		goto err_ioremap;
-	}
+	hcd->regs = regs;
+	hcd->rsrc_start = res->start;
+	hcd->rsrc_len = resource_size(res);
 
 	ehci = hcd_to_ehci(hcd);
 	np = of_find_compatible_node(NULL, NULL, "ibm,usb-ohci-440epx");
 	if (np != NULL) {
 		/* claim we really affected by usb23 erratum */
-		if (!of_address_to_resource(np, 0, &res))
+		if (!of_address_to_resource(np, 0, &ohci_res))
 			ehci->ohci_hcctrl_reg =
 				devm_ioremap(&op->dev,
-					     res.start + OHCI_HCCTRL_OFFSET,
+					     ohci_res.start + OHCI_HCCTRL_OFFSET,
 					     OHCI_HCCTRL_LEN);
 		else
 			pr_debug("%s: no ohci offset in fdt\n", __FILE__);
@@ -170,14 +161,12 @@ static int ehci_hcd_ppc_of_probe(struct platform_device *op)
 
 	rv = usb_add_hcd(hcd, irq, 0);
 	if (rv)
-		goto err_ioremap;
+		goto err;
 
 	device_wakeup_enable(hcd->self.controller);
 	return 0;
 
-err_ioremap:
-	irq_dispose_mapping(irq);
-err_irq:
+err:
 	usb_put_hcd(hcd);
 
 	return rv;
@@ -196,8 +185,6 @@ static void ehci_hcd_ppc_of_remove(struct platform_device *op)
 
 	usb_remove_hcd(hcd);
 
-	irq_dispose_mapping(hcd->irq);
-
 	/* use request_mem_region to test if the ohci driver is loaded.  if so
 	 * ensure the ohci core is operational.
 	 */
diff --git a/drivers/usb/host/ehci-timer.c b/drivers/usb/host/ehci-timer.c
index a79c8ac0a55fad..404069a3dd0bc5 100644
--- a/drivers/usb/host/ehci-timer.c
+++ b/drivers/usb/host/ehci-timer.c
@@ -199,7 +199,8 @@ static void ehci_handle_controller_death(struct ehci_hcd *ehci)
 
 	/* Clean up the mess */
 	ehci->rh_state = EHCI_RH_HALTED;
-	ehci_writel(ehci, 0, &ehci->regs->configured_flag);
+	if (!ehci->no_configured_flag)
+		ehci_writel(ehci, 0, &ehci->regs->configured_flag);
 	ehci_writel(ehci, 0, &ehci->regs->intr_enable);
 	ehci_work(ehci);
 	end_unlink_async(ehci);
diff --git a/drivers/usb/host/ehci.h b/drivers/usb/host/ehci.h
index d7a3c8d13f6b8a..855a9d08b0192b 100644
--- a/drivers/usb/host/ehci.h
+++ b/drivers/usb/host/ehci.h
@@ -123,6 +123,19 @@ struct ehci_hcd {			/* one per controller */
 	struct ehci_caps __iomem *caps;
 	struct ehci_regs __iomem *regs;
 	struct ehci_dbg_port __iomem *debug;
+#ifdef CONFIG_USB_EHCI_DEVIANT_PORT_STATUS_REG
+	u32 __iomem		*port_status;
+#endif
+#ifdef CONFIG_USB_EHCI_PORT_SPEED_HOOK
+	unsigned int		(*get_port_speed)(struct ehci_hcd *ehci,
+						  unsigned int port);
+#endif
+#ifdef CONFIG_USB_EHCI_PORT_RESET_HOOKS
+	int			(*pre_port_reset)(struct ehci_hcd *ehci,
+						  unsigned int port);
+	int			(*post_port_reset)(struct ehci_hcd *ehci,
+						   unsigned int port);
+#endif
 
 	__u32			hcs_params;	/* cached register copy */
 	spinlock_t		lock;
@@ -222,6 +235,9 @@ struct ehci_hcd {			/* one per controller */
 	unsigned		spurious_oc:1;
 	unsigned		is_aspeed:1;
 	unsigned		zx_wakeup_clear_needed:1;
+	unsigned		no_configured_flag:1;
+	unsigned		no_tdi_mode:1;
+	unsigned		no_fsls_isoc:1;
 
 	/* required for usb32 quirk */
 	#define OHCI_CTRL_HCFS          (3 << 6)
@@ -278,6 +294,19 @@ static inline struct usb_hcd *ehci_to_hcd(struct ehci_hcd *ehci)
 
 /*-------------------------------------------------------------------------*/
 
+static inline u32 __iomem *ehci_portsc(struct ehci_hcd *ehci,
+				       unsigned int port)
+{
+#ifdef CONFIG_USB_EHCI_DEVIANT_PORT_STATUS_REG
+	if (ehci->port_status)
+		return ehci->port_status + port;
+#endif
+
+	return &ehci->regs->port_status[port];
+}
+
+/*-------------------------------------------------------------------------*/
+
 #define	QTD_NEXT(ehci, dma)	cpu_to_hc32(ehci, (u32)dma)
 
 /*
@@ -651,38 +680,12 @@ struct ehci_tt {
 
 #ifdef CONFIG_USB_EHCI_ROOT_HUB_TT
 
-/*
- * Some EHCI controllers have a Transaction Translator built into the
- * root hub. This is a non-standard feature.  Each controller will need
- * to add code to the following inline functions, and call them as
- * needed (mostly in root hub code).
- */
-
 #define	ehci_is_TDI(e)			(ehci_to_hcd(e)->has_tt)
 
-/* Returns the speed of a device attached to a port on the root hub. */
-static inline unsigned int
-ehci_port_speed(struct ehci_hcd *ehci, unsigned int portsc)
-{
-	if (ehci_is_TDI(ehci)) {
-		switch ((portsc >> (ehci->has_hostpc ? 25 : 26)) & 3) {
-		case 0:
-			return 0;
-		case 1:
-			return USB_PORT_STAT_LOW_SPEED;
-		case 2:
-		default:
-			return USB_PORT_STAT_HIGH_SPEED;
-		}
-	}
-	return USB_PORT_STAT_HIGH_SPEED;
-}
-
 #else
 
 #define	ehci_is_TDI(e)			(0)
 
-#define	ehci_port_speed(ehci, portsc)	USB_PORT_STAT_HIGH_SPEED
 #endif
 
 /*-------------------------------------------------------------------------*/
diff --git a/drivers/usb/host/fhci-hcd.c b/drivers/usb/host/fhci-hcd.c
index 71e785f445a30e..d572f0c1dffaf3 100644
--- a/drivers/usb/host/fhci-hcd.c
+++ b/drivers/usb/host/fhci-hcd.c
@@ -643,7 +643,7 @@ static int of_fhci_probe(struct platform_device *ofdev)
 	}
 
 	for (j = 0; j < NUM_PINS; j++) {
-		fhci->pins[j] = qe_pin_request(dev, j);
+		fhci->pins[j] = qe_pin_request(dev, fhci->gpiods[j]);
 		if (IS_ERR(fhci->pins[j])) {
 			ret = PTR_ERR(fhci->pins[j]);
 			dev_err(dev, "can't get pin %d: %d\n", j, ret);
diff --git a/drivers/usb/host/octeon-hcd.c b/drivers/usb/host/octeon-hcd.c
index 34ce771fa0f52b..256e459ff6d8dc 100644
--- a/drivers/usb/host/octeon-hcd.c
+++ b/drivers/usb/host/octeon-hcd.c
@@ -378,8 +378,22 @@ struct octeon_hcd {
 	struct cvmx_usb_transaction *active_split;
 	struct cvmx_usb_tx_fifo periodic;
 	struct cvmx_usb_tx_fifo nonperiodic;
+	struct hrtimer sof_timer;
 };
 
+static int cvmx_usb_poll(struct octeon_hcd *usb);
+
+static enum hrtimer_restart octeon_usb_sof_timer(struct hrtimer *t)
+{
+	struct octeon_hcd *usb = container_of(t, struct octeon_hcd, sof_timer);
+	unsigned long flags;
+
+	spin_lock_irqsave(&usb->lock, flags);
+	cvmx_usb_poll(usb);
+	spin_unlock_irqrestore(&usb->lock, flags);
+	return HRTIMER_NORESTART;
+}
+
 /*
  * This macro logically sets a single field in a CSR. It does the sequence
  * read, modify, and write
@@ -578,7 +592,7 @@ static int cvmx_wait_tx_rx(struct octeon_hcd *usb, int fflsh_type)
 	int result;
 	u64 address = CVMX_USBCX_GRSTCTL(usb->index);
 	u64 done = cvmx_get_cycle() + 100 *
-		   (u64)octeon_get_clock_rate / 1000000;
+		   (u64)octeon_get_clock_rate() / 1000000;
 	union cvmx_usbcx_grstctl c;
 
 	while (1) {
@@ -853,6 +867,14 @@ static int cvmx_usb_initialize(struct device *dev,
 	 *    USBC_GAHBCFG[PTXFEMPLVL]
 	 *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 	 */
+	usbcx_gahbcfg.u32 = cvmx_usb_read_csr32(usb,
+						CVMX_USBCX_GHWCFG3(usb->index));
+	if (usbcx_gahbcfg.u32 == 0xffffffff || usbcx_gahbcfg.u32 == 0) {
+		dev_err(dev, "USB core is not responding (GHWCFG3=0x%08x)\n",
+			usbcx_gahbcfg.u32);
+		return -ENODEV;
+	}
+
 	usbcx_gahbcfg.u32 = 0;
 	usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 				  CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
@@ -1900,6 +1922,7 @@ static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof)
 	int channel;
 	struct cvmx_usb_pipe *pipe;
 	int need_sof;
+	u64 min_due;
 	enum cvmx_usb_transfer ttype;
 
 	if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
@@ -1940,15 +1963,33 @@ static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof)
 	 * future that might need to be scheduled
 	 */
 	need_sof = 0;
+	min_due = ~0ull;
 	for (ttype = CVMX_USB_TRANSFER_CONTROL;
 	     ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
 		list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
-			if (pipe->next_tx_frame > usb->frame_number) {
-				need_sof = 1;
-				break;
-			}
+			if (pipe->next_tx_frame > usb->frame_number &&
+			    pipe->next_tx_frame < min_due)
+				min_due = pipe->next_tx_frame;
 		}
 	}
+	if (min_due != ~0ull) {
+		u64 delta = min_due - usb->frame_number;
+
+		/*
+		 * frame_number is resynced from HFNUM on every poll, so a
+		 * deadline that is many frames away does not need an
+		 * interrupt on every SOF to count them down - sleep on the
+		 * timer instead and keep SOF interrupts for deadlines within
+		 * a few frames. Stay well below the 16383-frame wrap of
+		 * HFNUM. One (micro)frame is 125us in high-speed mode.
+		 */
+		if (delta <= 4 || delta > 8000)
+			need_sof = 1;
+		else
+			hrtimer_start(&usb->sof_timer,
+				      ns_to_ktime((delta - 2) * 125000),
+				      HRTIMER_MODE_REL);
+	}
 	USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 			cvmx_usbcx_gintmsk, sofmsk, need_sof);
 }
@@ -3638,6 +3679,8 @@ static int octeon_usb_probe(struct platform_device *pdev)
 	usb = (struct octeon_hcd *)hcd->hcd_priv;
 
 	spin_lock_init(&usb->lock);
+	hrtimer_setup(&usb->sof_timer, octeon_usb_sof_timer, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL);
 
 	usb->init_flags = initialize_flags;
 
@@ -3688,6 +3731,7 @@ static void octeon_usb_remove(struct platform_device *pdev)
 	unsigned long flags;
 
 	usb_remove_hcd(hcd);
+	hrtimer_cancel(&usb->sof_timer);
 	spin_lock_irqsave(&usb->lock, flags);
 	status = cvmx_usb_shutdown(usb);
 	spin_unlock_irqrestore(&usb->lock, flags);
diff --git a/drivers/usb/host/ohci-da8xx.c b/drivers/usb/host/ohci-da8xx.c
index 0938c0e7a8b6d5..a7cd5fb4f11946 100644
--- a/drivers/usb/host/ohci-da8xx.c
+++ b/drivers/usb/host/ohci-da8xx.c
@@ -447,6 +447,7 @@ static int ohci_da8xx_probe(struct platform_device *pdev)
 
 err_remove_hcd:
 	usb_remove_hcd(hcd);
+	device_wakeup_disable(hcd->self.controller);
 err:
 	usb_put_hcd(hcd);
 	return error;
@@ -457,6 +458,7 @@ static void ohci_da8xx_remove(struct platform_device *pdev)
 	struct usb_hcd	*hcd = platform_get_drvdata(pdev);
 
 	usb_remove_hcd(hcd);
+	device_wakeup_disable(hcd->self.controller);
 	usb_put_hcd(hcd);
 }
 
diff --git a/drivers/usb/host/ohci-dbg.c b/drivers/usb/host/ohci-dbg.c
index 23dc9eddc06c44..d945737f019685 100644
--- a/drivers/usb/host/ohci-dbg.c
+++ b/drivers/usb/host/ohci-dbg.c
@@ -184,7 +184,7 @@ ohci_dump_status (struct ohci_hcd *controller, char **next, unsigned *size)
 		\
 		(temp & RH_PS_PES) ? " PES" : "", \
 		(temp & RH_PS_CCS) ? " CCS" : "" \
-		);
+		)
 
 
 static void
diff --git a/drivers/usb/host/ohci-hub.c b/drivers/usb/host/ohci-hub.c
index b3d734ab6201b6..84d10a932942b1 100644
--- a/drivers/usb/host/ohci-hub.c
+++ b/drivers/usb/host/ohci-hub.c
@@ -33,7 +33,7 @@
 		\
 		(value & RH_PS_PES) ? " PES" : "", \
 		(value & RH_PS_CCS) ? " CCS" : "" \
-		);
+		)
 
 /*-------------------------------------------------------------------------*/
 
diff --git a/drivers/usb/host/ohci-ppc-of.c b/drivers/usb/host/ohci-ppc-of.c
index acd0a0e398a430..3a965b9c04fa87 100644
--- a/drivers/usb/host/ohci-ppc-of.c
+++ b/drivers/usb/host/ohci-ppc-of.c
@@ -17,7 +17,6 @@
 #include <linux/signal.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/of_irq.h>
 #include <linux/platform_device.h>
 
 static int
@@ -87,7 +86,9 @@ static int ohci_hcd_ppc_of_probe(struct platform_device *op)
 	struct device_node *dn = op->dev.of_node;
 	struct usb_hcd *hcd;
 	struct ohci_hcd	*ohci;
-	struct resource res;
+	struct resource *res;
+	struct resource ehci_res;
+	void __iomem *regs;
 	int irq;
 
 	int rv;
@@ -103,30 +104,21 @@ static int ohci_hcd_ppc_of_probe(struct platform_device *op)
 
 	dev_dbg(&op->dev, "initializing PPC-OF USB Controller\n");
 
-	rv = of_address_to_resource(dn, 0, &res);
-	if (rv)
-		return rv;
+	regs = devm_platform_get_and_ioremap_resource(op, 0, &res);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	irq = platform_get_irq(op, 0);
+	if (irq < 0)
+		return irq;
 
 	hcd = usb_create_hcd(&ohci_ppc_of_hc_driver, &op->dev, "PPC-OF USB");
 	if (!hcd)
 		return -ENOMEM;
 
-	hcd->rsrc_start = res.start;
-	hcd->rsrc_len = resource_size(&res);
-
-	hcd->regs = devm_ioremap_resource(&op->dev, &res);
-	if (IS_ERR(hcd->regs)) {
-		rv = PTR_ERR(hcd->regs);
-		goto err_rmr;
-	}
-
-	irq = irq_of_parse_and_map(dn, 0);
-	if (!irq) {
-		dev_err(&op->dev, "%s: irq_of_parse_and_map failed\n",
-			__FILE__);
-		rv = -EBUSY;
-		goto err_rmr;
-	}
+	hcd->rsrc_start = res->start;
+	hcd->rsrc_len = resource_size(res);
+	hcd->regs = regs;
 
 	ohci = hcd_to_ohci(hcd);
 	if (is_bigendian) {
@@ -158,20 +150,18 @@ static int ohci_hcd_ppc_of_probe(struct platform_device *op)
 	* the ehci driver is loaded.
 	*/
 	if (np !=  NULL) {
-		if (!of_address_to_resource(np, 0, &res)) {
-			if (!request_mem_region(res.start, 0x4, hcd_name)) {
+		if (!of_address_to_resource(np, 0, &ehci_res)) {
+			if (!request_mem_region(ehci_res.start, 0x4, hcd_name)) {
 				writel_be((readl_be(&ohci->regs->control) |
 					OHCI_USB_SUSPEND), &ohci->regs->control);
 					(void) readl_be(&ohci->regs->control);
 			} else
-				release_mem_region(res.start, 0x4);
+				release_mem_region(ehci_res.start, 0x4);
 		} else
 			pr_debug("%s: cannot get ehci offset from fdt\n", __FILE__);
 		of_node_put(np);
 	}
 
-	irq_dispose_mapping(irq);
-err_rmr:
  	usb_put_hcd(hcd);
 
 	return rv;
@@ -185,8 +175,6 @@ static void ohci_hcd_ppc_of_remove(struct platform_device *op)
 
 	usb_remove_hcd(hcd);
 
-	irq_dispose_mapping(hcd->irq);
-
 	usb_put_hcd(hcd);
 }
 
diff --git a/drivers/usb/host/ohci-s3c2410.c b/drivers/usb/host/ohci-s3c2410.c
index e623e24d3f8edf..7c0e35be58f1cc 100644
--- a/drivers/usb/host/ohci-s3c2410.c
+++ b/drivers/usb/host/ohci-s3c2410.c
@@ -335,6 +335,7 @@ ohci_hcd_s3c2410_remove(struct platform_device *dev)
 	struct usb_hcd *hcd = platform_get_drvdata(dev);
 
 	usb_remove_hcd(hcd);
+	device_wakeup_disable(hcd->self.controller);
 	s3c2410_stop_hc(dev);
 	usb_put_hcd(hcd);
 }
diff --git a/drivers/usb/host/ohci-spear.c b/drivers/usb/host/ohci-spear.c
index 6843d7cb3f9f37..5aeabb0512659f 100644
--- a/drivers/usb/host/ohci-spear.c
+++ b/drivers/usb/host/ohci-spear.c
@@ -103,6 +103,7 @@ static void spear_ohci_hcd_drv_remove(struct platform_device *pdev)
 	struct spear_ohci *sohci_p = to_spear_ohci(hcd);
 
 	usb_remove_hcd(hcd);
+	device_wakeup_disable(hcd->self.controller);
 	clk_disable_unprepare(sohci_p->clk);
 
 	usb_put_hcd(hcd);
diff --git a/drivers/usb/host/ohci-st.c b/drivers/usb/host/ohci-st.c
index d1656fce540047..53010a277b67fc 100644
--- a/drivers/usb/host/ohci-st.c
+++ b/drivers/usb/host/ohci-st.c
@@ -235,6 +235,7 @@ static void st_ohci_platform_remove(struct platform_device *dev)
 	int clk;
 
 	usb_remove_hcd(hcd);
+	device_wakeup_disable(hcd->self.controller);
 
 	if (pdata->power_off)
 		pdata->power_off(dev);
diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c
index 0404489c2f6a9c..ba158d63bb8bdf 100644
--- a/drivers/usb/host/pci-quirks.c
+++ b/drivers/usb/host/pci-quirks.c
@@ -1073,6 +1073,7 @@ void usb_enable_intel_xhci_ports(struct pci_dev *xhci_pdev)
 		if (companion->class == PCI_CLASS_SERIAL_USB_EHCI &&
 		    companion->vendor == PCI_VENDOR_ID_INTEL) {
 			ehci_found = true;
+			pci_dev_put(companion);
 			break;
 		}
 	}
diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
index 6f235d1e117e86..15dfa7e1a7dd64 100644
--- a/drivers/usb/host/xhci-tegra.c
+++ b/drivers/usb/host/xhci-tegra.c
@@ -1909,20 +1909,16 @@ static int tegra_xusb_probe(struct platform_device *pdev)
 					tegra_xusb_mbox_irq,
 					tegra_xusb_mbox_thread, 0,
 					dev_name(&pdev->dev), tegra);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to request IRQ: %d\n", err);
+	if (err < 0)
 		goto remove_usb3;
-	}
 
 	if (tegra->padctl_irq) {
 		err = devm_request_threaded_irq(&pdev->dev, tegra->padctl_irq,
 						NULL, tegra_xusb_padctl_irq,
 						IRQF_ONESHOT, dev_name(&pdev->dev),
 						tegra);
-		if (err < 0) {
-			dev_err(&pdev->dev, "failed to request padctl IRQ: %d\n", err);
+		if (err < 0)
 			goto remove_usb3;
-		}
 	}
 
 	err = tegra_xusb_enable_firmware_messages(tegra);
diff --git a/drivers/usb/image/microtek.c b/drivers/usb/image/microtek.c
index 1f0b3c44d3881e..787cf5aba55de6 100644
--- a/drivers/usb/image/microtek.c
+++ b/drivers/usb/image/microtek.c
@@ -475,6 +475,7 @@ static void mts_do_sg (struct urb* transfer)
 	if (unlikely(status)) {
                 set_host_byte(context->srb, (status == -ENOENT ? DID_ABORT : DID_ERROR));
 		mts_transfer_cleanup(transfer);
+		return;
         }
 
 	context->curr_sg = sg_next(context->curr_sg);
diff --git a/drivers/usb/misc/apple-mfi-fastcharge.c b/drivers/usb/misc/apple-mfi-fastcharge.c
index af266e19f2fdcb..02adec28d9e691 100644
--- a/drivers/usb/misc/apple-mfi-fastcharge.c
+++ b/drivers/usb/misc/apple-mfi-fastcharge.c
@@ -245,7 +245,7 @@ static struct usb_device_driver mfi_fc_driver = {
 
 static int __init mfi_fc_driver_init(void)
 {
-	return usb_register_device_driver(&mfi_fc_driver, THIS_MODULE);
+	return usb_register_device_driver(&mfi_fc_driver);
 }
 
 static void __exit mfi_fc_driver_exit(void)
diff --git a/drivers/usb/misc/brcmstb-usb-pinmap.c b/drivers/usb/misc/brcmstb-usb-pinmap.c
index 1ce885e4184c2e..d8e9c80a9c2e02 100644
--- a/drivers/usb/misc/brcmstb-usb-pinmap.c
+++ b/drivers/usb/misc/brcmstb-usb-pinmap.c
@@ -299,10 +299,8 @@ static int __init brcmstb_usb_pinmap_probe(struct platform_device *pdev)
 				       brcmstb_usb_pinmap_ovr_isr,
 				       IRQF_TRIGGER_RISING,
 				       pdev->name, pdata);
-		if (err < 0) {
-			dev_err(&pdev->dev, "Error requesting IRQ\n");
+		if (err < 0)
 			return err;
-		}
 	}
 
 	for (x = 0, pin = pdata->in_pins; x < pdata->in_count; x++, pin++) {
@@ -317,11 +315,8 @@ static int __init brcmstb_usb_pinmap_probe(struct platform_device *pdev)
 				       IRQF_SHARED | IRQF_TRIGGER_RISING |
 				       IRQF_TRIGGER_FALLING,
 				       pdev->name, pin);
-		if (err < 0) {
-			dev_err(&pdev->dev, "Error requesting IRQ for %s pin\n",
-				pin->name);
+		if (err < 0)
 			return err;
-		}
 	}
 
 	dev_dbg(&pdev->dev, "Driver probe succeeded\n");
diff --git a/drivers/usb/misc/onboard_usb_dev.c b/drivers/usb/misc/onboard_usb_dev.c
index 11508ed4df2516..c604062404162e 100644
--- a/drivers/usb/misc/onboard_usb_dev.c
+++ b/drivers/usb/misc/onboard_usb_dev.c
@@ -706,7 +706,7 @@ static int __init onboard_dev_init(void)
 {
 	int ret;
 
-	ret = usb_register_device_driver(&onboard_dev_usbdev_driver, THIS_MODULE);
+	ret = usb_register_device_driver(&onboard_dev_usbdev_driver);
 	if (ret)
 		return ret;
 
diff --git a/drivers/usb/misc/qcom_eud.c b/drivers/usb/misc/qcom_eud.c
index 926419ca560fc8..a7b9ed4ac07156 100644
--- a/drivers/usb/misc/qcom_eud.c
+++ b/drivers/usb/misc/qcom_eud.c
@@ -225,7 +225,7 @@ static int eud_probe(struct platform_device *pdev)
 	ret = devm_request_threaded_irq(&pdev->dev, chip->irq, handle_eud_irq,
 			handle_eud_irq_thread, IRQF_ONESHOT, NULL, chip);
 	if (ret)
-		return dev_err_probe(chip->dev, ret, "failed to allocate irq\n");
+		return ret;
 
 	enable_irq_wake(chip->irq);
 
diff --git a/drivers/usb/mon/mon_bin.c b/drivers/usb/mon/mon_bin.c
index 687f6a8981f34f..9d00b21a8153be 100644
--- a/drivers/usb/mon/mon_bin.c
+++ b/drivers/usb/mon/mon_bin.c
@@ -1219,6 +1219,15 @@ mon_bin_poll(struct file *file, struct poll_table_struct *wait)
 	return mask;
 }
 
+static void __mon_bin_vma_open(struct mon_reader_bin *rp)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&rp->b_lock, flags);
+	rp->mmap_active++;
+	spin_unlock_irqrestore(&rp->b_lock, flags);
+}
+
 /*
  * open and close: just keep track of how many times the device is
  * mapped, to use the proper memory allocation function.
@@ -1226,64 +1235,79 @@ mon_bin_poll(struct file *file, struct poll_table_struct *wait)
 static void mon_bin_vma_open(struct vm_area_struct *vma)
 {
 	struct mon_reader_bin *rp = vma->vm_private_data;
-	unsigned long flags;
 
-	spin_lock_irqsave(&rp->b_lock, flags);
-	rp->mmap_active++;
-	spin_unlock_irqrestore(&rp->b_lock, flags);
+	__mon_bin_vma_open(rp);
 }
 
-static void mon_bin_vma_close(struct vm_area_struct *vma)
+static void __mon_bin_vma_close(struct mon_reader_bin *rp)
 {
 	unsigned long flags;
 
-	struct mon_reader_bin *rp = vma->vm_private_data;
 	spin_lock_irqsave(&rp->b_lock, flags);
 	rp->mmap_active--;
 	spin_unlock_irqrestore(&rp->b_lock, flags);
 }
 
-/*
- * Map ring pages to user space.
- */
-static vm_fault_t mon_bin_vma_fault(struct vm_fault *vmf)
+static void mon_bin_vma_close(struct vm_area_struct *vma)
 {
-	struct mon_reader_bin *rp = vmf->vma->vm_private_data;
+	struct mon_reader_bin *rp = vma->vm_private_data;
+
+	__mon_bin_vma_close(rp);
+}
+
+static const struct vm_operations_struct mon_bin_vm_ops = {
+	.open =     mon_bin_vma_open,
+	.close =    mon_bin_vma_close,
+};
+
+static int mon_bin_discontig_init(void *vm_private_data, void **private)
+{
+	struct mon_reader_bin *rp = vm_private_data;
+
+	/* Dropped by mon_bin_vma_close() on unmap, including on error. */
+	__mon_bin_vma_open(rp);
+	return 0;
+}
+
+static int mon_bin_discontig_get(struct discontig_kernel_page_state *state)
+{
+	struct mon_reader_bin *rp = state->vm_private_data;
 	unsigned long offset, chunk_idx;
-	struct page *pageptr;
 	unsigned long flags;
 
 	spin_lock_irqsave(&rp->b_lock, flags);
-	offset = vmf->pgoff << PAGE_SHIFT;
+
+	offset = state->pgoff << PAGE_SHIFT;
 	if (offset >= rp->b_size) {
 		spin_unlock_irqrestore(&rp->b_lock, flags);
-		return VM_FAULT_SIGBUS;
+		discontig_kernel_map_abort(state);
+		return 0;
 	}
 	chunk_idx = offset / CHUNK_SIZE;
-	pageptr = rp->b_vec[chunk_idx].pg;
-	get_page(pageptr);
-	vmf->page = pageptr;
+	discontig_kernel_map_page(state, rp->b_vec[chunk_idx].pg);
+
 	spin_unlock_irqrestore(&rp->b_lock, flags);
 	return 0;
 }
 
-static const struct vm_operations_struct mon_bin_vm_ops = {
-	.open =     mon_bin_vma_open,
-	.close =    mon_bin_vma_close,
-	.fault =    mon_bin_vma_fault,
+static const struct discontig_kernel_page_ops mon_discontig_ops = {
+	.init = mon_bin_discontig_init,
+	.get = mon_bin_discontig_get,
 };
 
-static int mon_bin_mmap(struct file *filp, struct vm_area_struct *vma)
+static int mon_bin_mmap_prepare(struct vm_area_desc *desc)
 {
-	/* don't do anything here: "fault" will set up page table entries */
-	vma->vm_ops = &mon_bin_vm_ops;
+	const struct file *filp = desc->file;
 
-	if (vma->vm_flags & VM_WRITE)
+	if (vma_desc_test(desc, VMA_WRITE_BIT))
 		return -EPERM;
 
-	vm_flags_mod(vma, VM_DONTEXPAND | VM_DONTDUMP, VM_MAYWRITE);
-	vma->vm_private_data = filp->private_data;
-	mon_bin_vma_open(vma);
+	desc->vm_ops = &mon_bin_vm_ops;
+	vma_desc_clear_flags(desc, VMA_MAYWRITE_BIT);
+	vma_desc_set_flags(desc, VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT);
+	desc->private_data = filp->private_data;
+
+	mmap_action_map_discontig_kernel_pages(desc, NULL, &mon_discontig_ops);
 	return 0;
 }
 
@@ -1298,7 +1322,7 @@ static const struct file_operations mon_fops_binary = {
 	.compat_ioctl =	mon_bin_compat_ioctl,
 #endif
 	.release =	mon_bin_release,
-	.mmap =		mon_bin_mmap,
+	.mmap_prepare = mon_bin_mmap_prepare,
 };
 
 static int mon_bin_wait_event(struct file *file, struct mon_reader_bin *rp)
diff --git a/drivers/usb/mon/mon_main.c b/drivers/usb/mon/mon_main.c
index e4fff194fea054..600c9de84c7e16 100644
--- a/drivers/usb/mon/mon_main.c
+++ b/drivers/usb/mon/mon_main.c
@@ -199,6 +199,9 @@ static void mon_bus_remove(struct usb_bus *ubus)
 {
 	struct mon_bus *mbus = ubus->mon_bus;
 
+	if (mbus == NULL)
+		return;
+
 	mutex_lock(&mon_lock);
 	list_del(&mbus->bus_link);
 	if (mbus->text_inited)
diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c
index a40bf5bad2d5d5..990e2e8e6bc2ab 100644
--- a/drivers/usb/mtu3/mtu3_core.c
+++ b/drivers/usb/mtu3/mtu3_core.c
@@ -973,10 +973,8 @@ int ssusb_gadget_init(struct ssusb_mtk *ssusb)
 
 	ret = devm_request_threaded_irq(dev, mtu->irq, NULL, mtu3_irq,
 					IRQF_ONESHOT, dev_name(dev), mtu);
-	if (ret) {
-		dev_err(dev, "request irq %d failed!\n", mtu->irq);
+	if (ret)
 		goto irq_err;
-	}
 
 	/* power down device IP for power saving by default */
 	mtu3_stop(mtu);
diff --git a/drivers/usb/mtu3/mtu3_trace.h b/drivers/usb/mtu3/mtu3_trace.h
index 89870175d63561..9aaa167d69c130 100644
--- a/drivers/usb/mtu3/mtu3_trace.h
+++ b/drivers/usb/mtu3/mtu3_trace.h
@@ -224,6 +224,7 @@ DECLARE_EVENT_CLASS(mtu3_log_ep,
 		__field(unsigned int, flags)
 		__field(unsigned int, direction)
 		__field(struct mtu3_gpd_ring *, gpd_ring)
+		__field(dma_addr_t, gpd_ring_dma)
 	),
 	TP_fast_assign(
 		__assign_str(name);
@@ -235,12 +236,13 @@ DECLARE_EVENT_CLASS(mtu3_log_ep,
 		__entry->flags = mep->flags;
 		__entry->direction = mep->is_in;
 		__entry->gpd_ring = &mep->gpd_ring;
+		__entry->gpd_ring_dma = mep->gpd_ring.dma
 	),
 	TP_printk("%s: type %s maxp %d slot %d mult %d burst %d ring %p/%pad flags %c:%c%c%c:%c",
 		__get_str(name), usb_ep_type_string(__entry->type),
 		__entry->maxp, __entry->slot,
 		__entry->mult, __entry->maxburst,
-		__entry->gpd_ring, &__entry->gpd_ring->dma,
+		__entry->gpd_ring, &__entry->gpd_ring_dma,
 		__entry->flags & MTU3_EP_ENABLED ? 'E' : 'e',
 		__entry->flags & MTU3_EP_STALL ? 'S' : 's',
 		__entry->flags & MTU3_EP_WEDGE ? 'W' : 'w',
diff --git a/drivers/usb/musb/da8xx.c b/drivers/usb/musb/da8xx.c
index 4209f438ba1840..6c859a1a11d042 100644
--- a/drivers/usb/musb/da8xx.c
+++ b/drivers/usb/musb/da8xx.c
@@ -562,7 +562,7 @@ static int da8xx_probe(struct platform_device *pdev)
 	ret = of_platform_populate(pdev->dev.of_node, NULL,
 				   da8xx_auxdata_lookup, &pdev->dev);
 	if (ret)
-		goto err_unregister_phy;
+		goto err_depopulate;
 
 	pinfo = da8xx_dev_info;
 	pinfo.parent = &pdev->dev;
@@ -577,12 +577,13 @@ static int da8xx_probe(struct platform_device *pdev)
 	ret = PTR_ERR_OR_ZERO(glue->musb);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register musb device: %d\n", ret);
-		goto err_unregister_phy;
+		goto err_depopulate;
 	}
 
 	return 0;
 
-err_unregister_phy:
+err_depopulate:
+	of_platform_depopulate(&pdev->dev);
 	usb_phy_generic_unregister(glue->usb_phy);
 	return ret;
 }
@@ -592,6 +593,7 @@ static void da8xx_remove(struct platform_device *pdev)
 	struct da8xx_glue		*glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->musb);
+	of_platform_depopulate(&pdev->dev);
 	usb_phy_generic_unregister(glue->usb_phy);
 }
 
diff --git a/drivers/usb/musb/musb_dsps.c b/drivers/usb/musb/musb_dsps.c
index e3935f18dd56c2..efd1f0dbb92640 100644
--- a/drivers/usb/musb/musb_dsps.c
+++ b/drivers/usb/musb/musb_dsps.c
@@ -618,6 +618,29 @@ static int dsps_musb_recover(struct musb *musb)
 	return session_restart ? 0 : -EPIPE;
 }
 
+static int dsps_musb_vbus_status(struct musb *musb)
+{
+	u8 devctl = musb_readb(musb->mregs, MUSB_DEVCTL);
+
+	return (devctl & MUSB_DEVCTL_VBUS) == MUSB_DEVCTL_VBUS;
+}
+
+static void dsps_musb_set_vbus(struct musb *musb, int is_on)
+{
+	u8 devctl;
+
+	devctl = musb_readb(musb->mregs, MUSB_DEVCTL);
+
+	if (is_on) {
+		devctl |= MUSB_DEVCTL_SESSION;
+	} else {
+		musb_root_disconnect(musb);
+		devctl &= ~MUSB_DEVCTL_SESSION;
+	}
+
+	musb_writeb(musb->mregs, MUSB_DEVCTL, devctl);
+}
+
 /* Similar to am35x, dm81xx support only 32-bit read operation */
 static void dsps_read_fifo32(struct musb_hw_ep *hw_ep, u16 len, u8 *dst)
 {
@@ -702,6 +725,8 @@ static struct musb_platform_ops dsps_ops = {
 
 	.set_mode	= dsps_musb_set_mode,
 	.recover	= dsps_musb_recover,
+	.vbus_status	= dsps_musb_vbus_status,
+	.set_vbus	= dsps_musb_set_vbus,
 	.clear_ep_rxintr = dsps_musb_clear_ep_rxintr,
 };
 
diff --git a/drivers/usb/phy/phy-ab8500-usb.c b/drivers/usb/phy/phy-ab8500-usb.c
index 6a98aeeeae319c..e2e06e57dced55 100644
--- a/drivers/usb/phy/phy-ab8500-usb.c
+++ b/drivers/usb/phy/phy-ab8500-usb.c
@@ -748,10 +748,8 @@ static int ab8500_usb_irq_setup(struct platform_device *pdev,
 				ab8500_usb_link_status_irq,
 				IRQF_NO_SUSPEND | IRQF_SHARED | IRQF_ONESHOT,
 				"usb-link-status", ab);
-		if (err < 0) {
-			dev_err(ab->dev, "request_irq failed for link status irq\n");
+		if (err < 0)
 			return err;
-		}
 	}
 
 	if (ab->flags & AB8500_USB_FLAG_USE_ID_WAKEUP_IRQ) {
@@ -762,10 +760,8 @@ static int ab8500_usb_irq_setup(struct platform_device *pdev,
 				ab8500_usb_disconnect_irq,
 				IRQF_NO_SUSPEND | IRQF_SHARED | IRQF_ONESHOT,
 				"usb-id-fall", ab);
-		if (err < 0) {
-			dev_err(ab->dev, "request_irq failed for ID fall irq\n");
+		if (err < 0)
 			return err;
-		}
 	}
 
 	if (ab->flags & AB8500_USB_FLAG_USE_VBUS_DET_IRQ) {
@@ -776,10 +772,8 @@ static int ab8500_usb_irq_setup(struct platform_device *pdev,
 				ab8500_usb_disconnect_irq,
 				IRQF_NO_SUSPEND | IRQF_SHARED | IRQF_ONESHOT,
 				"usb-vbus-fall", ab);
-		if (err < 0) {
-			dev_err(ab->dev, "request_irq failed for Vbus fall irq\n");
+		if (err < 0)
 			return err;
-		}
 	}
 
 	return 0;
diff --git a/drivers/usb/phy/phy-generic.c b/drivers/usb/phy/phy-generic.c
index de26b302334de9..a41671eda2a144 100644
--- a/drivers/usb/phy/phy-generic.c
+++ b/drivers/usb/phy/phy-generic.c
@@ -282,8 +282,7 @@ static int usb_phy_generic_probe(struct platform_device *pdev)
 						VBUS_IRQ_FLAGS, "vbus_detect",
 						nop);
 		if (err)
-			return dev_err_probe(dev, err, "can't request irq %i\n",
-					     gpiod_to_irq(nop->gpiod_vbus));
+			return err;
 
 		nop->phy.otg->state = gpiod_get_value(nop->gpiod_vbus) ?
 			OTG_STATE_B_PERIPHERAL : OTG_STATE_B_IDLE;
diff --git a/drivers/usb/phy/phy-gpio-vbus-usb.c b/drivers/usb/phy/phy-gpio-vbus-usb.c
index ce09e789afd882..c0135a30cabd11 100644
--- a/drivers/usb/phy/phy-gpio-vbus-usb.c
+++ b/drivers/usb/phy/phy-gpio-vbus-usb.c
@@ -297,11 +297,8 @@ static int gpio_vbus_probe(struct platform_device *pdev)
 
 	err = devm_request_irq(&pdev->dev, irq, gpio_vbus_irq, irqflags,
 			       "vbus_detect", pdev);
-	if (err) {
-		dev_err(&pdev->dev, "can't request irq %i, err: %d\n",
-			irq, err);
+	if (err)
 		return err;
-	}
 
 	INIT_DELAYED_WORK(&gpio_vbus->work, gpio_vbus_work);
 
diff --git a/drivers/usb/renesas_usbhs/mod.c b/drivers/usb/renesas_usbhs/mod.c
index f2ea3e1412d2e8..dcaebcc0614d04 100644
--- a/drivers/usb/renesas_usbhs/mod.c
+++ b/drivers/usb/renesas_usbhs/mod.c
@@ -143,10 +143,8 @@ int usbhs_mod_probe(struct usbhs_priv *priv)
 	/* irq settings */
 	ret = devm_request_irq(dev, priv->irq, usbhs_interrupt,
 			       0, dev_name(dev), priv);
-	if (ret) {
-		dev_err(dev, "irq request err\n");
+	if (ret)
 		goto mod_init_gadget_err;
-	}
 
 	return ret;
 
diff --git a/drivers/usb/serial/bus.c b/drivers/usb/serial/bus.c
index 9e2a18c0b21839..ea1fe5d1e44908 100644
--- a/drivers/usb/serial/bus.c
+++ b/drivers/usb/serial/bus.c
@@ -165,7 +165,11 @@ int usb_serial_bus_register(struct usb_serial_driver *driver)
 
 void usb_serial_bus_deregister(struct usb_serial_driver *driver)
 {
-	free_dynids(driver);
 	driver_unregister(&driver->driver);
+	free_dynids(driver);
 }
 
+void usb_serial_bus_remove_new_id(struct usb_serial_driver *driver)
+{
+	driver_remove_file(&driver->driver, &driver_attr_new_id);
+}
diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c
index 333b8b46370483..c3f9add9833341 100644
--- a/drivers/usb/serial/cp210x.c
+++ b/drivers/usb/serial/cp210x.c
@@ -223,6 +223,7 @@ static const struct usb_device_id id_table[] = {
 	{ USB_DEVICE(0x19CF, 0x3000) }, /* Parrot NMEA GPS Flight Recorder */
 	{ USB_DEVICE(0x1ADB, 0x0001) }, /* Schweitzer Engineering C662 Cable */
 	{ USB_DEVICE(0x1B1C, 0x1C00) }, /* Corsair USB Dongle */
+	{ USB_DEVICE(0x1B1C, 0x1C02) }, /* Corsair AX1500i Power Supply */
 	{ USB_DEVICE(0x1B93, 0x1013) }, /* Phoenix Contact UPS Device */
 	{ USB_DEVICE(0x1BA4, 0x0002) },	/* Silicon Labs 358x factory default */
 	{ USB_DEVICE(0x1BE3, 0x07A6) }, /* WAGO 750-923 USB Service Cable */
diff --git a/drivers/usb/serial/generic.c b/drivers/usb/serial/generic.c
index 6eaf74930aa3f7..17272701fab04c 100644
--- a/drivers/usb/serial/generic.c
+++ b/drivers/usb/serial/generic.c
@@ -266,6 +266,7 @@ EXPORT_SYMBOL_GPL(usb_serial_generic_chars_in_buffer);
 void usb_serial_generic_wait_until_sent(struct tty_struct *tty, long timeout)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct tty_port *tport = &port->port;
 	unsigned int bps;
 	unsigned long period;
 	unsigned long expire;
@@ -285,7 +286,14 @@ void usb_serial_generic_wait_until_sent(struct tty_struct *tty, long timeout)
 					__func__, jiffies_to_msecs(timeout),
 					jiffies_to_msecs(period));
 	expire = jiffies + timeout;
-	while (!port->serial->type->tx_empty(port)) {
+	for (;;) {
+		mutex_lock(&tport->mutex);
+		if (tty_io_error(tty) || port->serial->type->tx_empty(port)) {
+			mutex_unlock(&tport->mutex);
+			break;
+		}
+		mutex_unlock(&tport->mutex);
+
 		schedule_timeout_interruptible(period);
 		if (signal_pending(current))
 			break;
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index e4ad14375d6a73..828b1be3fee2d1 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -260,6 +260,7 @@ static void option_instat_callback(struct urb *urb);
 #define QUECTEL_PRODUCT_EM060K_12a		0x012a
 #define QUECTEL_PRODUCT_EM060K_12b		0x012b
 #define QUECTEL_PRODUCT_EM060K_12c		0x012c
+#define QUECTEL_PRODUCT_RG660QB			0x013d
 #define QUECTEL_PRODUCT_EG91			0x0191
 #define QUECTEL_PRODUCT_EG95			0x0195
 #define QUECTEL_PRODUCT_BG96			0x0296
@@ -280,6 +281,7 @@ static void option_instat_callback(struct urb *urb);
 #define QUECTEL_PRODUCT_EC200U			0x0901
 #define QUECTEL_PRODUCT_EG912Y			0x6001
 #define QUECTEL_PRODUCT_EC200S_CN		0x6002
+#define QUECTEL_PRODUCT_EG060W			0x6004
 #define QUECTEL_PRODUCT_EC200A			0x6005
 #define QUECTEL_PRODUCT_EG916Q			0x6007
 #define QUECTEL_PRODUCT_EM061K_LWW		0x6008
@@ -1268,12 +1270,15 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200A, 0xff, 0, 0) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200U, 0xff, 0, 0) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200S_CN, 0xff, 0, 0) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG060W, 0xff, 0, 0) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200T, 0xff, 0, 0) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG912Y, 0xff, 0, 0) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG916Q, 0xff, 0x00, 0x00) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500K, 0xff, 0x00, 0x00) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG650V, 0xff, 0xff, 0x30) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG650V, 0xff, 0, 0) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG660QB, 0xff, 0xff, 0x30) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG660QB, 0xff, 0, 0) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0xff, 0x30) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0, 0) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0xff, 0x40) },
@@ -2168,6 +2173,8 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9003, 0xff) },	/* Simcom SIM7500/SIM7600 MBIM mode */
 	{ USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9011, 0xff),	/* Simcom SIM7500/SIM7600 RNDIS mode */
 	  .driver_info = RSVD(7) },
+	{ USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x902b, 0xff),
+	  .driver_info = RSVD(4) },
 	{ USB_DEVICE(0x1e0e, 0x9071),				/* Simcom SIM8230 RMNET mode */
 	  .driver_info = RSVD(3) | RSVD(4) },
 	{ USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9078, 0xff),	/* Simcom SIM8230 ECM mode */
@@ -2429,6 +2436,13 @@ static const struct usb_device_id option_ids[] = {
 	  .driver_info = RSVD(5) },
 	{ USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe167, 0xff),                     /* Foxconn T99W640 MBIM */
 	  .driver_info = RSVD(3) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x01) },	/* Compal EXC-T1 */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x02) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x03) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x04) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x8217, 0xff, 0xff, 0x30) },	/* Compal EXM-G1x */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x8217, 0xff, 0xff, 0x40) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x8217, 0xff, 0xff, 0x60) },
 	{ USB_DEVICE(0x1508, 0x1001),						/* Fibocom NL668 (IOT version) */
 	  .driver_info = RSVD(4) | RSVD(5) | RSVD(6) },
 	{ USB_DEVICE(0x1782, 0x4d10) },						/* Fibocom L610 (AT mode) */
diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c
index dc2b39810a26c2..3b4df3ac7c7399 100644
--- a/drivers/usb/serial/quatech2.c
+++ b/drivers/usb/serial/quatech2.c
@@ -52,6 +52,7 @@
 #define QT2_XMIT_FLUSH      0x05  /* no following info */
 #define QT2_CONTROL_ESCAPE  0xff  /* pass through previous 2 control bytes */
 
+#define  MIN_BAUD_RATE              50
 #define  MAX_BAUD_RATE              921600
 #define  DEFAULT_BAUD_RATE          9600
 
@@ -156,7 +157,7 @@ static inline int calc_baud_divisor(int baudrate)
 
 static inline int qt2_set_port_config(struct usb_device *dev,
 				      unsigned char port_number,
-				      u16 baudrate, u16 lcr)
+				      speed_t baudrate, u16 lcr)
 {
 	int divisor = calc_baud_divisor(baudrate);
 	u16 index = ((u16) (lcr << 8) | (u16) (port_number));
@@ -257,9 +258,9 @@ static void qt2_set_termios(struct tty_struct *tty,
 	struct usb_device *dev = port->serial->dev;
 	struct qt2_port_private *port_priv;
 	struct ktermios *termios = &tty->termios;
-	u16 baud;
 	unsigned int cflag = termios->c_cflag;
 	u16 new_lcr = 0;
+	speed_t baud;
 	int status;
 
 	port_priv = usb_get_serial_port_data(port);
@@ -277,6 +278,18 @@ static void qt2_set_termios(struct tty_struct *tty,
 	if (!baud)
 		baud = 9600;
 
+	if (baud < MIN_BAUD_RATE || baud > MAX_BAUD_RATE) {
+		if (old_termios)
+			baud = tty_termios_baud_rate(old_termios);
+		else
+			baud = clamp(baud, MIN_BAUD_RATE, MAX_BAUD_RATE);
+
+		tty_encode_baud_rate(tty, baud, baud);
+
+		if (!baud)
+			baud = 9600;
+	}
+
 	status = qt2_set_port_config(dev, port_priv->device_port, baud,
 				     new_lcr);
 	if (status < 0)
@@ -373,7 +386,7 @@ static int qt2_open(struct tty_struct *tty, struct usb_serial_port *port)
 	port_priv->device_port = (u8) device_port;
 
 	if (tty)
-		qt2_set_termios(tty, port, &tty->termios);
+		qt2_set_termios(tty, port, NULL);
 
 	return 0;
 
diff --git a/drivers/usb/serial/ssu100.c b/drivers/usb/serial/ssu100.c
index b0d51558b73cea..8c5b9ef708b359 100644
--- a/drivers/usb/serial/ssu100.c
+++ b/drivers/usb/serial/ssu100.c
@@ -32,6 +32,7 @@
 
 #define  SERIAL_EVEN_PARITY         (UART_LCR_PARITY | UART_LCR_EPAR)
 
+#define  MIN_BAUD_RATE              50
 #define  MAX_BAUD_RATE              460800
 
 #define ATC_DISABLED                0x00
@@ -217,9 +218,10 @@ static void ssu100_set_termios(struct tty_struct *tty,
 {
 	struct usb_device *dev = port->serial->dev;
 	struct ktermios *termios = &tty->termios;
-	u16 baud, divisor, remainder;
+	speed_t baud, remainder;
 	unsigned int cflag = termios->c_cflag;
 	u16 urb_value = 0; /* will hold the new flags */
+	u16 divisor;
 	int result;
 
 	if (cflag & PARENB) {
@@ -235,6 +237,18 @@ static void ssu100_set_termios(struct tty_struct *tty,
 	if (!baud)
 		baud = 9600;
 
+	if (baud < MIN_BAUD_RATE || baud > MAX_BAUD_RATE) {
+		if (old_termios)
+			baud = tty_termios_baud_rate(old_termios);
+		else
+			baud = clamp(baud, MIN_BAUD_RATE, MAX_BAUD_RATE);
+
+		tty_encode_baud_rate(tty, baud, baud);
+
+		if (!baud)
+			baud = 9600;
+	}
+
 	dev_dbg(&port->dev, "%s - got baud = %d\n", __func__, baud);
 
 
@@ -310,7 +324,7 @@ static int ssu100_open(struct tty_struct *tty, struct usb_serial_port *port)
 		dev_dbg(&port->dev, "%s - set uart failed\n", __func__);
 
 	if (tty)
-		ssu100_set_termios(tty, port, &tty->termios);
+		ssu100_set_termios(tty, port, NULL);
 
 	return usb_serial_generic_open(tty, port);
 }
diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c
index 17edc057a31101..f3c594f1a80634 100644
--- a/drivers/usb/serial/usb-serial.c
+++ b/drivers/usb/serial/usb-serial.c
@@ -400,17 +400,13 @@ static unsigned int serial_chars_in_buffer(struct tty_struct *tty)
 static void serial_wait_until_sent(struct tty_struct *tty, int timeout)
 {
 	struct usb_serial_port *port = tty->driver_data;
-	struct usb_serial *serial = port->serial;
 
 	dev_dbg(&port->dev, "%s\n", __func__);
 
 	if (!port->serial->type->wait_until_sent)
 		return;
 
-	mutex_lock(&serial->disc_mutex);
-	if (!serial->disconnected)
-		port->serial->type->wait_until_sent(tty, timeout);
-	mutex_unlock(&serial->disc_mutex);
+	port->serial->type->wait_until_sent(tty, timeout);
 }
 
 static void serial_throttle(struct tty_struct *tty)
@@ -438,8 +434,13 @@ static int serial_get_serial(struct tty_struct *tty, struct serial_struct *ss)
 	struct usb_serial_port *port = tty->driver_data;
 	struct tty_port *tport = &port->port;
 	unsigned int close_delay, closing_wait;
+	int ret = 0;
 
 	mutex_lock(&tport->mutex);
+	if (tty_io_error(tty)) {
+		ret = -EIO;
+		goto out_unlock;
+	}
 
 	close_delay = jiffies_to_msecs(tport->close_delay) / 10;
 	closing_wait = tport->closing_wait;
@@ -452,10 +453,10 @@ static int serial_get_serial(struct tty_struct *tty, struct serial_struct *ss)
 
 	if (port->serial->type->get_serial)
 		port->serial->type->get_serial(tty, ss);
-
+out_unlock:
 	mutex_unlock(&tport->mutex);
 
-	return 0;
+	return ret;
 }
 
 static int serial_set_serial(struct tty_struct *tty, struct serial_struct *ss)
@@ -471,6 +472,10 @@ static int serial_set_serial(struct tty_struct *tty, struct serial_struct *ss)
 		closing_wait = msecs_to_jiffies(closing_wait * 10);
 
 	mutex_lock(&tport->mutex);
+	if (tty_io_error(tty)) {
+		ret = -EIO;
+		goto out_unlock;
+	}
 
 	if (!capable(CAP_SYS_ADMIN)) {
 		if (close_delay != tport->close_delay ||
@@ -498,6 +503,7 @@ static int serial_ioctl(struct tty_struct *tty,
 					unsigned int cmd, unsigned long arg)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct tty_port *tport = &port->port;
 	int retval = -ENOIOCTLCMD;
 
 	dev_dbg(&port->dev, "%s - cmd 0x%04x\n", __func__, cmd);
@@ -508,8 +514,21 @@ static int serial_ioctl(struct tty_struct *tty,
 			retval = port->serial->type->tiocmiwait(tty, arg);
 		break;
 	default:
-		if (port->serial->type->ioctl)
+		if (!port->serial->type->ioctl)
+			break;
+
+		if (cmd == TIOCSRS485)
+			down_write(&tty->termios_rwsem);
+
+		mutex_lock(&tport->mutex);
+		if (tty_io_error(tty))
+			retval = -EIO;
+		else
 			retval = port->serial->type->ioctl(tty, cmd, arg);
+		mutex_unlock(&tport->mutex);
+
+		if (cmd == TIOCSRS485)
+			up_write(&tty->termios_rwsem);
 	}
 
 	return retval;
@@ -519,25 +538,40 @@ static void serial_set_termios(struct tty_struct *tty,
 		               const struct ktermios *old)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct tty_port *tport = &port->port;
 
 	dev_dbg(&port->dev, "%s\n", __func__);
 
-	if (port->serial->type->set_termios)
-		port->serial->type->set_termios(tty, port, old);
-	else
+	if (!port->serial->type->set_termios) {
 		tty_termios_copy_hw(&tty->termios, old);
+		return;
+	}
+
+	mutex_lock(&tport->mutex);
+	if (!tty_io_error(tty))
+		port->serial->type->set_termios(tty, port, old);
+	mutex_unlock(&tport->mutex);
 }
 
 static int serial_break(struct tty_struct *tty, int break_state)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct tty_port *tport = &port->port;
+	int ret;
 
 	dev_dbg(&port->dev, "%s\n", __func__);
 
-	if (port->serial->type->break_ctl)
-		return port->serial->type->break_ctl(tty, break_state);
+	if (!port->serial->type->break_ctl)
+		return -ENOTTY;
 
-	return -ENOTTY;
+	mutex_lock(&tport->mutex);
+	if (tty_io_error(tty))
+		ret = -EIO;
+	else
+		ret = port->serial->type->break_ctl(tty, break_state);
+	mutex_unlock(&tport->mutex);
+
+	return ret;
 }
 
 static int serial_proc_show(struct seq_file *m, void *v)
@@ -578,24 +612,44 @@ static int serial_proc_show(struct seq_file *m, void *v)
 static int serial_tiocmget(struct tty_struct *tty)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct tty_port *tport = &port->port;
+	int ret;
 
 	dev_dbg(&port->dev, "%s\n", __func__);
 
-	if (port->serial->type->tiocmget)
-		return port->serial->type->tiocmget(tty);
-	return -ENOTTY;
+	if (!port->serial->type->tiocmget)
+		return -ENOTTY;
+
+	mutex_lock(&tport->mutex);
+	if (tty_io_error(tty))
+		ret = -EIO;
+	else
+		ret = port->serial->type->tiocmget(tty);
+	mutex_unlock(&tport->mutex);
+
+	return ret;
 }
 
 static int serial_tiocmset(struct tty_struct *tty,
 			    unsigned int set, unsigned int clear)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct tty_port *tport = &port->port;
+	int ret;
 
 	dev_dbg(&port->dev, "%s\n", __func__);
 
-	if (port->serial->type->tiocmset)
-		return port->serial->type->tiocmset(tty, set, clear);
-	return -ENOTTY;
+	if (!port->serial->type->tiocmset)
+		return -ENOTTY;
+
+	mutex_lock(&tport->mutex);
+	if (tty_io_error(tty))
+		ret = -EIO;
+	else
+		ret = port->serial->type->tiocmset(tty, set, clear);
+	mutex_unlock(&tport->mutex);
+
+	return ret;
 }
 
 static int serial_get_icount(struct tty_struct *tty,
@@ -1191,12 +1245,17 @@ static void usb_serial_disconnect(struct usb_interface *interface)
 		usb_serial_port_poison_urbs(port);
 		wake_up_interruptible(&port->port.delta_msr_wait);
 		cancel_work_sync(&port->work);
-		if (device_is_registered(&port->dev))
-			device_del(&port->dev);
 	}
+
 	if (serial->type->disconnect)
 		serial->type->disconnect(serial);
 
+	for (i = 0; i < serial->num_ports; ++i) {
+		port = serial->port[i];
+		if (device_is_registered(&port->dev))
+			device_del(&port->dev);
+	}
+
 	release_sibling(serial, interface);
 
 	/* let the last holder of this object cause it to be cleaned up */
@@ -1464,7 +1523,7 @@ int __usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers
 {
 	int rc;
 	struct usb_driver *udriver;
-	struct usb_serial_driver * const *sd;
+	struct usb_serial_driver * const *sd, * const *s;
 
 	/*
 	 * udriver must be registered before any of the serial drivers,
@@ -1517,9 +1576,11 @@ int __usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers
 	return 0;
 
 err_deregister_drivers:
+	for (s = serial_drivers; s < sd; ++s)
+		usb_serial_bus_remove_new_id(*s);
+	usb_deregister(udriver);
 	while (sd-- > serial_drivers)
 		usb_serial_deregister(*sd);
-	usb_deregister(udriver);
 err_free_driver:
 	kfree(udriver);
 	return rc;
@@ -1537,10 +1598,23 @@ EXPORT_SYMBOL_GPL(__usb_serial_register_drivers);
 void usb_serial_deregister_drivers(struct usb_serial_driver *const serial_drivers[])
 {
 	struct usb_driver *udriver = (*serial_drivers)->usb_driver;
+	struct usb_serial_driver * const *sd;
+
+	/*
+	 * udriver must be deregistered before the serial drivers so that
+	 * I/O is stopped before unbinding the ports.
+	 *
+	 * Remove the new_id attributes to prevent ids from being added and
+	 * triggering a probe of udriver after it has been deregistered.
+	 */
+	for (sd = serial_drivers; *sd; ++sd)
+		usb_serial_bus_remove_new_id(*sd);
 
-	for (; *serial_drivers; ++serial_drivers)
-		usb_serial_deregister(*serial_drivers);
 	usb_deregister(udriver);
+
+	for (sd = serial_drivers; *sd; ++sd)
+		usb_serial_deregister(*sd);
+
 	kfree(udriver);
 }
 EXPORT_SYMBOL_GPL(usb_serial_deregister_drivers);
diff --git a/drivers/usb/serial/usb_wwan.c b/drivers/usb/serial/usb_wwan.c
index a183cc4515edb7..a3929d3a3ec17d 100644
--- a/drivers/usb/serial/usb_wwan.c
+++ b/drivers/usb/serial/usb_wwan.c
@@ -455,7 +455,7 @@ int usb_wwan_port_probe(struct usb_serial_port *port)
 	init_usb_anchor(&portdata->delayed);
 
 	for (i = 0; i < N_IN_URB; i++) {
-		buffer = (u8 *)__get_free_page(GFP_KERNEL);
+		buffer = kmalloc(IN_BUFLEN, GFP_KERNEL);
 		if (!buffer)
 			goto bail_out_error;
 		portdata->in_buffer[i] = buffer;
@@ -492,7 +492,7 @@ int usb_wwan_port_probe(struct usb_serial_port *port)
 bail_out_error:
 	for (i = 0; i < N_IN_URB; i++) {
 		usb_free_urb(portdata->in_urbs[i]);
-		free_page((unsigned long)portdata->in_buffer[i]);
+		kfree(portdata->in_buffer[i]);
 	}
 	kfree(portdata);
 
@@ -510,7 +510,7 @@ void usb_wwan_port_remove(struct usb_serial_port *port)
 
 	for (i = 0; i < N_IN_URB; i++) {
 		usb_free_urb(portdata->in_urbs[i]);
-		free_page((unsigned long)portdata->in_buffer[i]);
+		kfree(portdata->in_buffer[i]);
 	}
 	for (i = 0; i < N_OUT_URB; i++) {
 		usb_free_urb(portdata->out_urbs[i]);
diff --git a/drivers/usb/serial/xr_serial.c b/drivers/usb/serial/xr_serial.c
index 352c765d88034e..c08f4aa14a3d90 100644
--- a/drivers/usb/serial/xr_serial.c
+++ b/drivers/usb/serial/xr_serial.c
@@ -850,12 +850,9 @@ static int xr_get_rs485_config(struct tty_struct *tty,
 	struct usb_serial_port *port = tty->driver_data;
 	struct xr_data *data = usb_get_serial_port_data(port);
 
-	down_read(&tty->termios_rwsem);
-	if (copy_to_user(argp, &data->rs485, sizeof(data->rs485))) {
-		up_read(&tty->termios_rwsem);
+	/* core holds port mutex */
+	if (copy_to_user(argp, &data->rs485, sizeof(data->rs485)))
 		return -EFAULT;
-	}
-	up_read(&tty->termios_rwsem);
 
 	return 0;
 }
@@ -871,10 +868,9 @@ static int xr_set_rs485_config(struct tty_struct *tty,
 		return -EFAULT;
 	xr_sanitize_serial_rs485(&rs485);
 
-	down_write(&tty->termios_rwsem);
+	/* core holds termios rwsem and port mutex */
 	data->rs485 = rs485;
 	xr_set_flow_mode(tty, port, NULL);
-	up_write(&tty->termios_rwsem);
 
 	if (copy_to_user(argp, &rs485, sizeof(rs485)))
 		return -EFAULT;
diff --git a/drivers/usb/storage/alauda.c b/drivers/usb/storage/alauda.c
index 691fe47009cf6c..5e7f99ba8ac014 100644
--- a/drivers/usb/storage/alauda.c
+++ b/drivers/usb/storage/alauda.c
@@ -500,11 +500,14 @@ static int alauda_check_media(struct us_data *us)
 static int alauda_check_status2(struct us_data *us)
 {
 	int rc;
-	unsigned char command[] = {
-		ALAUDA_BULK_CMD, ALAUDA_BULK_GET_STATUS2,
-		0, 0, 0, 0, 3, 0, MEDIA_PORT(us)
-	};
-	unsigned char data[3];
+	unsigned char *command = us->iobuf;
+	unsigned char *data = us->iobuf;
+
+	memset(command, 0, 9);
+	command[0] = ALAUDA_BULK_CMD;
+	command[1] = ALAUDA_BULK_GET_STATUS2;
+	command[6] = 3;
+	command[8] = MEDIA_PORT(us);
 
 	rc = usb_stor_bulk_transfer_buf(us, us->send_bulk_pipe,
 		command, 9, NULL);
@@ -530,10 +533,15 @@ static int alauda_check_status2(struct us_data *us)
 static int alauda_get_redu_data(struct us_data *us, u16 pba, unsigned char *data)
 {
 	int rc;
-	unsigned char command[] = {
-		ALAUDA_BULK_CMD, ALAUDA_BULK_GET_REDU_DATA,
-		PBA_HI(pba), PBA_ZONE(pba), 0, PBA_LO(pba), 0, 0, MEDIA_PORT(us)
-	};
+	unsigned char *command = us->iobuf;
+
+	memset(command, 0, 9);
+	command[0] = ALAUDA_BULK_CMD;
+	command[1] = ALAUDA_BULK_GET_REDU_DATA;
+	command[2] = PBA_HI(pba);
+	command[3] = PBA_ZONE(pba);
+	command[5] = PBA_LO(pba);
+	command[8] = MEDIA_PORT(us);
 
 	rc = usb_stor_bulk_transfer_buf(us, us->send_bulk_pipe,
 		command, 9, NULL);
@@ -689,11 +697,13 @@ static int alauda_read_map(struct us_data *us, unsigned int zone)
  * Checks to see whether we have already mapped a certain zone
  * If we haven't, the map is generated
  */
-static void alauda_ensure_map_for_zone(struct us_data *us, unsigned int zone)
+static int alauda_ensure_map_for_zone(struct us_data *us, unsigned int zone)
 {
 	if (MEDIA_INFO(us).lba_to_pba[zone] == NULL
 		|| MEDIA_INFO(us).pba_to_lba[zone] == NULL)
-		alauda_read_map(us, zone);
+		return alauda_read_map(us, zone);
+
+	return 0;
 }
 
 /*
@@ -702,14 +712,20 @@ static void alauda_ensure_map_for_zone(struct us_data *us, unsigned int zone)
 static int alauda_erase_block(struct us_data *us, u16 pba)
 {
 	int rc;
-	unsigned char command[] = {
-		ALAUDA_BULK_CMD, ALAUDA_BULK_ERASE_BLOCK, PBA_HI(pba),
-		PBA_ZONE(pba), 0, PBA_LO(pba), 0x02, 0, MEDIA_PORT(us)
-	};
-	unsigned char buf[2];
+	unsigned char *command = us->iobuf;
+	unsigned char *buf = us->iobuf;
 
 	usb_stor_dbg(us, "Erasing PBA %d\n", pba);
 
+	memset(command, 0, 9);
+	command[0] = ALAUDA_BULK_CMD;
+	command[1] = ALAUDA_BULK_ERASE_BLOCK;
+	command[2] = PBA_HI(pba);
+	command[3] = PBA_ZONE(pba);
+	command[5] = PBA_LO(pba);
+	command[6] = 0x02;
+	command[8] = MEDIA_PORT(us);
+
 	rc = usb_stor_bulk_transfer_buf(us, us->send_bulk_pipe,
 		command, 9, NULL);
 	if (rc != USB_STOR_XFER_GOOD)
@@ -732,13 +748,19 @@ static int alauda_read_block_raw(struct us_data *us, u16 pba,
 		unsigned int page, unsigned int pages, unsigned char *data)
 {
 	int rc;
-	unsigned char command[] = {
-		ALAUDA_BULK_CMD, ALAUDA_BULK_READ_BLOCK, PBA_HI(pba),
-		PBA_ZONE(pba), 0, PBA_LO(pba) + page, pages, 0, MEDIA_PORT(us)
-	};
+	unsigned char *command = us->iobuf;
 
 	usb_stor_dbg(us, "pba %d page %d count %d\n", pba, page, pages);
 
+	memset(command, 0, 9);
+	command[0] = ALAUDA_BULK_CMD;
+	command[1] = ALAUDA_BULK_READ_BLOCK;
+	command[2] = PBA_HI(pba);
+	command[3] = PBA_ZONE(pba);
+	command[5] = PBA_LO(pba) + page;
+	command[6] = pages;
+	command[8] = MEDIA_PORT(us);
+
 	rc = usb_stor_bulk_transfer_buf(us, us->send_bulk_pipe,
 		command, 9, NULL);
 	if (rc != USB_STOR_XFER_GOOD)
@@ -783,13 +805,19 @@ static int alauda_write_block(struct us_data *us, u16 pba, unsigned char *data)
 {
 	int rc;
 	struct alauda_info *info = (struct alauda_info *) us->extra;
-	unsigned char command[] = {
-		ALAUDA_BULK_CMD, ALAUDA_BULK_WRITE_BLOCK, PBA_HI(pba),
-		PBA_ZONE(pba), 0, PBA_LO(pba), 32, 0, MEDIA_PORT(us)
-	};
+	unsigned char *command = us->iobuf;
 
 	usb_stor_dbg(us, "pba %d\n", pba);
 
+	memset(command, 0, 9);
+	command[0] = ALAUDA_BULK_CMD;
+	command[1] = ALAUDA_BULK_WRITE_BLOCK;
+	command[2] = PBA_HI(pba);
+	command[3] = PBA_ZONE(pba);
+	command[5] = PBA_LO(pba);
+	command[6] = 32;
+	command[8] = MEDIA_PORT(us);
+
 	rc = usb_stor_bulk_transfer_buf(us, us->send_bulk_pipe,
 		command, 9, NULL);
 	if (rc != USB_STOR_XFER_GOOD)
@@ -823,7 +851,9 @@ static int alauda_write_lba(struct us_data *us, u16 lba,
 	unsigned int new_pba_offset;
 	unsigned int zone = lba / uzonesize;
 
-	alauda_ensure_map_for_zone(us, zone);
+	result = alauda_ensure_map_for_zone(us, zone);
+	if (result != USB_STOR_TRANSPORT_GOOD)
+		return result;
 
 	pba = MEDIA_INFO(us).lba_to_pba[zone][lba_offset];
 	if (pba == 1) {
@@ -919,6 +949,7 @@ static int alauda_read_data(struct us_data *us, unsigned long address,
 	unsigned char *buffer;
 	u16 lba, max_lba;
 	unsigned int page, len, offset;
+	unsigned int num_zones;
 	unsigned int blockshift = MEDIA_INFO(us).blockshift;
 	unsigned int pageshift = MEDIA_INFO(us).pageshift;
 	unsigned int blocksize = MEDIA_INFO(us).blocksize;
@@ -943,7 +974,9 @@ static int alauda_read_data(struct us_data *us, unsigned long address,
 	/* Figure out the initial LBA and page */
 	lba = address >> blockshift;
 	page = (address & MEDIA_INFO(us).blockmask);
-	max_lba = MEDIA_INFO(us).capacity >> (blockshift + pageshift);
+	num_zones = MEDIA_INFO(us).capacity >> (MEDIA_INFO(us).zoneshift
+		+ blockshift + pageshift);
+	max_lba = num_zones * uzonesize;
 
 	result = USB_STOR_TRANSPORT_GOOD;
 	offset = 0;
@@ -954,7 +987,6 @@ static int alauda_read_data(struct us_data *us, unsigned long address,
 		unsigned int lba_offset = lba - (zone * uzonesize);
 		unsigned int pages;
 		u16 pba;
-		alauda_ensure_map_for_zone(us, zone);
 
 		/* Not overflowing capacity? */
 		if (lba >= max_lba) {
@@ -964,6 +996,10 @@ static int alauda_read_data(struct us_data *us, unsigned long address,
 			break;
 		}
 
+		result = alauda_ensure_map_for_zone(us, zone);
+		if (result != USB_STOR_TRANSPORT_GOOD)
+			break;
+
 		/* Find number of pages we can read in this block */
 		pages = min(sectors, blocksize - page);
 		len = pages << pageshift;
@@ -1019,6 +1055,7 @@ static int alauda_write_data(struct us_data *us, unsigned long address,
 	unsigned int pagesize = MEDIA_INFO(us).pagesize;
 	struct scatterlist *sg;
 	u16 lba, max_lba;
+	unsigned int num_zones;
 	int result;
 
 	/*
@@ -1045,7 +1082,9 @@ static int alauda_write_data(struct us_data *us, unsigned long address,
 	/* Figure out the initial LBA and page */
 	lba = address >> blockshift;
 	page = (address & MEDIA_INFO(us).blockmask);
-	max_lba = MEDIA_INFO(us).capacity >> (pageshift + blockshift);
+	num_zones = MEDIA_INFO(us).capacity >> (MEDIA_INFO(us).zoneshift
+		+ blockshift + pageshift);
+	max_lba = num_zones * MEDIA_INFO(us).uzonesize;
 
 	result = USB_STOR_TRANSPORT_GOOD;
 	offset = 0;
diff --git a/drivers/usb/storage/debug.c b/drivers/usb/storage/debug.c
index dda610f689b73a..71923cf2853667 100644
--- a/drivers/usb/storage/debug.c
+++ b/drivers/usb/storage/debug.c
@@ -141,14 +141,12 @@ void usb_stor_show_command(const struct us_data *us, struct scsi_cmnd *srb)
 }
 
 void usb_stor_show_sense(const struct us_data *us,
-			 unsigned char key,
-			 unsigned char asc,
-			 unsigned char ascq)
+			 const struct scsi_sense_hdr *sshdr)
 {
 	const char *what, *keystr, *fmt;
 
-	keystr = scsi_sense_key_string(key);
-	what = scsi_extd_sense_format(asc, ascq, &fmt);
+	keystr = scsi_sense_key_string(sshdr->sense_key);
+	what = scsi_extd_sense_format(sshdr, &fmt);
 
 	if (keystr == NULL)
 		keystr = "(Unknown Key)";
@@ -156,7 +154,8 @@ void usb_stor_show_sense(const struct us_data *us,
 		what = "(unknown ASC/ASCQ)";
 
 	if (fmt)
-		usb_stor_dbg(us, "%s: %s (%s%x)\n", keystr, what, fmt, ascq);
+		usb_stor_dbg(us, "%s: %s (%s%x)\n", keystr, what, fmt,
+			     scsi_sense_ascq(sshdr));
 	else
 		usb_stor_dbg(us, "%s: %s\n", keystr, what);
 }
diff --git a/drivers/usb/storage/debug.h b/drivers/usb/storage/debug.h
index a6505ceb66934c..34e20fff13b5e6 100644
--- a/drivers/usb/storage/debug.h
+++ b/drivers/usb/storage/debug.h
@@ -31,8 +31,8 @@
 
 #ifdef CONFIG_USB_STORAGE_DEBUG
 void usb_stor_show_command(const struct us_data *us, struct scsi_cmnd *srb);
-void usb_stor_show_sense(const struct us_data *us, unsigned char key,
-			 unsigned char asc, unsigned char ascq);
+void usb_stor_show_sense(const struct us_data *us,
+			 const struct scsi_sense_hdr *sshdr);
 __printf(2, 3) void usb_stor_dbg(const struct us_data *us,
 				 const char *fmt, ...);
 
diff --git a/drivers/usb/storage/ene_ub6250.c b/drivers/usb/storage/ene_ub6250.c
index 895f90c7a3faeb..a9358e04ab4b63 100644
--- a/drivers/usb/storage/ene_ub6250.c
+++ b/drivers/usb/storage/ene_ub6250.c
@@ -2350,6 +2350,8 @@ static int ene_ub6250_probe(struct usb_interface *intf,
 
 	us->transport_name = "ene_ub6250";
 	us->transport = ene_transport;
+	/* get_transport() only sets this for the standard USB_PR_* protocols */
+	us->transport_reset = usb_stor_Bulk_reset;
 	us->max_lun = 0;
 
 	result = usb_stor_probe2(us);
diff --git a/drivers/usb/storage/sddr09.c b/drivers/usb/storage/sddr09.c
index 3d45e1b54c6682..313313508bde70 100644
--- a/drivers/usb/storage/sddr09.c
+++ b/drivers/usb/storage/sddr09.c
@@ -1339,6 +1339,15 @@ sddr09_read_map(struct us_data *us) {
 
 		lba += 1000*(i/0x400);
 
+		/* The LBA comes from the card and may exceed the table size */
+		if (lba >= numblocks) {
+			dev_warn_ratelimited(&us->pusb_dev->dev,
+					"sddr09: Bad LBA %d for block %d exceeds the translation table size %d\n",
+					lba, i, numblocks);
+			info->pba_to_lba[i] = UNUSABLE;
+			continue;
+		}
+
 		if (info->lba_to_pba[lba] != UNDEF) {
 			printk(KERN_WARNING
 			       "sddr09: LBA %d seen for PBA %d and %d\n",
diff --git a/drivers/usb/storage/sierra_ms.c b/drivers/usb/storage/sierra_ms.c
index c5e3eeeb2144d0..992685ca6f1afb 100644
--- a/drivers/usb/storage/sierra_ms.c
+++ b/drivers/usb/storage/sierra_ms.c
@@ -77,6 +77,13 @@ static int sierra_get_swoc_info(struct usb_device *udev,
 			sizeof(struct swoc_info),	/* __u16 size 	     */
 			USB_CTRL_SET_TIMEOUT);		/* int timeout 	     */
 
+	/*
+	 * A short IN transfer leaves the tail of swocInfo uninitialized;
+	 * only a full transfer is valid.
+	 */
+	if (result != sizeof(struct swoc_info))
+		return -EIO;
+
 	swocInfo->LinuxSKU = le16_to_cpu(swocInfo->LinuxSKU);
 	swocInfo->LinuxVer = le16_to_cpu(swocInfo->LinuxVer);
 	return result;
diff --git a/drivers/usb/storage/transport.c b/drivers/usb/storage/transport.c
index 9a4bf86e7b6a31..8de3015bce0502 100644
--- a/drivers/usb/storage/transport.c
+++ b/drivers/usb/storage/transport.c
@@ -803,9 +803,9 @@ void usb_stor_invoke_transport(struct scsi_cmnd *srb, struct us_data *us)
 			     temp_result);
 		usb_stor_dbg(us, "-- code: 0x%x, key: 0x%x, ASC: 0x%x, ASCQ: 0x%x\n",
 			     sshdr.response_code, sshdr.sense_key,
-			     sshdr.asc, sshdr.ascq);
+			     scsi_sense_asc(&sshdr), scsi_sense_ascq(&sshdr));
 #ifdef CONFIG_USB_STORAGE_DEBUG
-		usb_stor_show_sense(us, sshdr.sense_key, sshdr.asc, sshdr.ascq);
+		usb_stor_show_sense(us, &sshdr);
 #endif
 
 		/* set the result so the higher layers expect this data */
@@ -820,7 +820,8 @@ void usb_stor_invoke_transport(struct scsi_cmnd *srb, struct us_data *us)
 		 * everything worked or that there was an unspecified
 		 * problem.  We have to decide which.
 		 */
-		if (sshdr.sense_key == 0 && sshdr.asc == 0 && sshdr.ascq == 0 &&
+		if (sshdr.sense_key == NO_SENSE &&
+		    sshdr.sense_code == NO_ADDITIONAL_SENSE_INFORMATION &&
 		    fm_ili == 0) {
 			/*
 			 * If things are really okay, then let's show that.
diff --git a/drivers/usb/storage/uas.c b/drivers/usb/storage/uas.c
index 26516298126948..8655edbd66b168 100644
--- a/drivers/usb/storage/uas.c
+++ b/drivers/usb/storage/uas.c
@@ -1130,9 +1130,9 @@ static int uas_pre_reset(struct usb_interface *intf)
 		return 0;
 
 	/* Block new requests */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	scsi_block_requests(shost);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	if (uas_wait_for_pending_cmnds(devinfo) != 0) {
 		shost_printk(KERN_ERR, shost, "%s: timed out\n", __func__);
@@ -1162,9 +1162,9 @@ static int uas_post_reset(struct usb_interface *intf)
 			     __func__, err);
 
 	/* we must unblock the host in every case lest we deadlock */
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	scsi_report_bus_reset(shost, 0);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	scsi_unblock_requests(shost);
 
@@ -1204,9 +1204,9 @@ static int uas_reset_resume(struct usb_interface *intf)
 		return -EIO;
 	}
 
-	spin_lock_irqsave(shost->host_lock, flags);
+	spin_lock_irqsave(&shost->host_lock, flags);
 	scsi_report_bus_reset(shost, 0);
-	spin_unlock_irqrestore(shost->host_lock, flags);
+	spin_unlock_irqrestore(&shost->host_lock, flags);
 
 	return 0;
 }
diff --git a/drivers/usb/storage/usb.c b/drivers/usb/storage/usb.c
index 064c7fc8e368a8..823a4b2e9033b5 100644
--- a/drivers/usb/storage/usb.c
+++ b/drivers/usb/storage/usb.c
@@ -1100,8 +1100,10 @@ int usb_stor_probe2(struct us_data *us)
 	int result;
 	struct device *dev = &us->pusb_intf->dev;
 
-	/* Make sure the transport and protocol have both been set */
-	if (!us->transport || !us->proto_handler) {
+	/* Make sure the transport, its reset method, and the protocol
+	 * have all been set
+	 */
+	if (!us->transport || !us->transport_reset || !us->proto_handler) {
 		result = -ENXIO;
 		goto BadDevice;
 	}
diff --git a/drivers/usb/storage/usb.h b/drivers/usb/storage/usb.h
index 97c6196d639b4a..c26d008e5184eb 100644
--- a/drivers/usb/storage/usb.h
+++ b/drivers/usb/storage/usb.h
@@ -166,8 +166,8 @@ extern void fill_inquiry_response(struct us_data *us,
  * The scsi_lock() and scsi_unlock() macros protect the sm_state and the
  * single queue element srb for write access
  */
-#define scsi_unlock(host)	spin_unlock_irq(host->host_lock)
-#define scsi_lock(host)		spin_lock_irq(host->host_lock)
+#define scsi_unlock(host)	spin_unlock_irq(&host->host_lock)
+#define scsi_lock(host)		spin_lock_irq(&host->host_lock)
 
 /* General routines provided by the usb-storage standard core */
 #ifdef CONFIG_PM
diff --git a/drivers/usb/typec/anx7411.c b/drivers/usb/typec/anx7411.c
index 41df115912b93f..06c25af094b3e0 100644
--- a/drivers/usb/typec/anx7411.c
+++ b/drivers/usb/typec/anx7411.c
@@ -1529,10 +1529,8 @@ static int anx7411_i2c_probe(struct i2c_client *client)
 					IRQF_TRIGGER_FALLING |
 					IRQF_ONESHOT,
 					"anx7411-intp", plat);
-	if (ret) {
-		dev_err(dev, "fail to request irq\n");
+	if (ret)
 		goto free_wq;
-	}
 
 	if (anx7411_typec_check_connection(plat))
 		dev_err(dev, "check status\n");
@@ -1566,8 +1564,11 @@ static void anx7411_i2c_remove(struct i2c_client *client)
 	anx7411_partner_unregister_altmode(plat);
 	anx7411_unregister_partner(plat);
 
-	if (plat->workqueue)
+	if (plat->workqueue) {
+		disable_irq(plat->intp_irq);
+		cancel_work_sync(&plat->work);
 		destroy_workqueue(plat->workqueue);
+	}
 
 	i2c_unregister_device(plat->spi_client);
 
diff --git a/drivers/usb/typec/bus.c b/drivers/usb/typec/bus.c
index e84b134a338133..587d3b06ee8d2a 100644
--- a/drivers/usb/typec/bus.c
+++ b/drivers/usb/typec/bus.c
@@ -383,12 +383,14 @@ void typec_altmode_put_plug(struct typec_altmode *plug)
 EXPORT_SYMBOL_GPL(typec_altmode_put_plug);
 
 int __typec_altmode_register_driver(struct typec_altmode_driver *drv,
-				    struct module *module)
+				    struct module *module,
+				    const char *mod_name)
 {
 	if (!drv->probe)
 		return -EINVAL;
 
 	drv->driver.owner = module;
+	drv->driver.mod_name = mod_name;
 	drv->driver.bus = &typec_bus;
 
 	return driver_register(&drv->driver);
diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c
index 4eec90c82bae4b..a717d0f8a4e563 100644
--- a/drivers/usb/typec/hd3ss3220.c
+++ b/drivers/usb/typec/hd3ss3220.c
@@ -49,6 +49,9 @@
 #define HD3SS3220_REG_GEN_CTRL_MODE_SELECT_UFP		BIT(4)
 #define HD3SS3220_REG_GEN_CTRL_MODE_SELECT_DRP		(BIT(5) | BIT(4))
 
+/* Minimum time VDD5 has to be stable before VCC33 starts ramping up */
+#define HD3SS3220_TVDD5V_PG_US				2000
+
 struct hd3ss3220 {
 	struct device *dev;
 	struct regmap *regmap;
@@ -363,6 +366,31 @@ static irqreturn_t hd3ss3220_id_isr(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+/*
+ * Bring both supplies up in the order the datasheet asks for. Powering VCC33
+ * first can back-power the device in a non-functioning state, which grounds
+ * the I2C bus and takes both this device and any others on the same bus down
+ */
+static int hd3ss3220_power_up(struct device *dev)
+{
+	int ret;
+
+	ret = devm_regulator_get_enable(dev, "vdd5");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable VDD5\n");
+
+	/* Nothing to stagger against unless the board describes both rails */
+	if (device_property_present(dev, "vdd5-supply") &&
+	    device_property_present(dev, "vcc33-supply"))
+		fsleep(HD3SS3220_TVDD5V_PG_US);
+
+	ret = devm_regulator_get_enable(dev, "vcc33");
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to enable VCC33\n");
+
+	return 0;
+}
+
 static int hd3ss3220_probe(struct i2c_client *client)
 {
 	struct typec_capability typec_cap = { };
@@ -384,6 +412,10 @@ static int hd3ss3220_probe(struct i2c_client *client)
 	if (IS_ERR(hd3ss3220->regmap))
 		return PTR_ERR(hd3ss3220->regmap);
 
+	ret = hd3ss3220_power_up(hd3ss3220->dev);
+	if (ret)
+		return ret;
+
 	/* For backward compatibility check the connector child node first */
 	connector = device_get_named_child_node(hd3ss3220->dev, "connector");
 	if (connector) {
@@ -441,10 +473,8 @@ static int hd3ss3220_probe(struct i2c_client *client)
 						IRQF_TRIGGER_RISING |
 						IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
 						dev_name(hd3ss3220->dev), hd3ss3220);
-		if (ret < 0) {
-			dev_err(hd3ss3220->dev, "failed to get ID irq: %d\n", ret);
+		if (ret < 0)
 			goto err_put_fwnode;
-		}
 	}
 
 	typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE;
diff --git a/drivers/usb/typec/mux/gpio-sbu-mux.c b/drivers/usb/typec/mux/gpio-sbu-mux.c
index 1834f1a2dd9dd0..b1aa976aca240f 100644
--- a/drivers/usb/typec/mux/gpio-sbu-mux.c
+++ b/drivers/usb/typec/mux/gpio-sbu-mux.c
@@ -17,6 +17,7 @@ struct gpio_sbu_mux {
 
 	struct typec_switch_dev *sw;
 	struct typec_mux_dev *mux;
+	bool mode_switch;
 
 	struct mutex lock; /* protect enabled and swapped */
 	bool enabled;
@@ -40,19 +41,24 @@ static int gpio_sbu_switch_set(struct typec_switch_dev *sw,
 		enabled = false;
 		break;
 	case TYPEC_ORIENTATION_NORMAL:
+		if (!sbu_mux->mode_switch)
+			enabled = true;
 		swapped = false;
 		break;
 	case TYPEC_ORIENTATION_REVERSE:
+		if (!sbu_mux->mode_switch)
+			enabled = true;
 		swapped = true;
 		break;
 	}
 
-	if (enabled != sbu_mux->enabled)
-		gpiod_set_value_cansleep(sbu_mux->enable_gpio, enabled);
-
 	if (swapped != sbu_mux->swapped)
 		gpiod_set_value_cansleep(sbu_mux->select_gpio, swapped);
 
+	/* If both enabled and swapped transition, set enable GPIO after to avoid glitches */
+	if (enabled != sbu_mux->enabled)
+		gpiod_set_value_cansleep(sbu_mux->enable_gpio, enabled);
+
 	sbu_mux->enabled = enabled;
 	sbu_mux->swapped = swapped;
 
@@ -120,6 +126,7 @@ static int gpio_sbu_mux_probe(struct platform_device *pdev)
 	sw_desc.fwnode = dev_fwnode(dev);
 	sw_desc.set = gpio_sbu_switch_set;
 
+	sbu_mux->mode_switch = device_property_read_bool(dev, "mode-switch");
 	sbu_mux->sw = typec_switch_register(dev, &sw_desc);
 	if (IS_ERR(sbu_mux->sw))
 		return dev_err_probe(dev, PTR_ERR(sbu_mux->sw),
diff --git a/drivers/usb/typec/mux/it5205.c b/drivers/usb/typec/mux/it5205.c
index 5e1a120b2e3b4d..c6bdf74cb65746 100644
--- a/drivers/usb/typec/mux/it5205.c
+++ b/drivers/usb/typec/mux/it5205.c
@@ -10,6 +10,7 @@
 
 #include <linux/delay.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of_platform.h>
@@ -250,7 +251,7 @@ static int it5205_probe(struct i2c_client *client)
 		if (ret) {
 			typec_mux_unregister(it->mux);
 			typec_switch_unregister(it->sw);
-			return dev_err_probe(dev, ret, "Failed to request irq\n");
+			return ret;
 		}
 	}
 
diff --git a/drivers/usb/typec/tcpm/tcpci_maxim_core.c b/drivers/usb/typec/tcpm/tcpci_maxim_core.c
index 998693a618393e..52dbbbe53691ec 100644
--- a/drivers/usb/typec/tcpm/tcpci_maxim_core.c
+++ b/drivers/usb/typec/tcpm/tcpci_maxim_core.c
@@ -542,8 +542,7 @@ static int max_tcpci_probe(struct i2c_client *client)
 					(IRQF_TRIGGER_LOW | IRQF_ONESHOT), dev_name(chip->dev),
 					chip);
 	if (ret < 0)
-		return dev_err_probe(&client->dev, ret,
-				     "IRQ initialization failed\n");
+		return ret;
 
 	ret = devm_device_init_wakeup(chip->dev);
 	if (ret)
diff --git a/drivers/usb/typec/tcpm/tcpci_mt6360.c b/drivers/usb/typec/tcpm/tcpci_mt6360.c
index 881ffacbfd7737..d4010ea5183b04 100644
--- a/drivers/usb/typec/tcpm/tcpci_mt6360.c
+++ b/drivers/usb/typec/tcpm/tcpci_mt6360.c
@@ -167,7 +167,6 @@ static int mt6360_tcpc_probe(struct platform_device *pdev)
 	ret = devm_request_threaded_irq(mti->dev, mti->irq, NULL, mt6360_irq, IRQF_ONESHOT,
 					dev_name(&pdev->dev), mti);
 	if (ret) {
-		dev_err(mti->dev, "Failed to register irq\n");
 		tcpci_unregister_port(mti->tcpci);
 		return ret;
 	}
diff --git a/drivers/usb/typec/tcpm/tcpci_mt6370.c b/drivers/usb/typec/tcpm/tcpci_mt6370.c
index 7d6c75c7098532..2e85cb05ed3776 100644
--- a/drivers/usb/typec/tcpm/tcpci_mt6370.c
+++ b/drivers/usb/typec/tcpm/tcpci_mt6370.c
@@ -168,7 +168,7 @@ static int mt6370_tcpc_probe(struct platform_device *pdev)
 	ret = devm_request_threaded_irq(dev, irq, NULL, mt6370_irq_handler,
 					IRQF_ONESHOT, dev_name(dev), priv);
 	if (ret)
-		return dev_err_probe(dev, ret, "Failed to allocate irq\n");
+		return ret;
 
 	device_init_wakeup(dev, true);
 	dev_pm_set_wake_irq(dev, irq);
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index 2d6b14aa208569..d721baef52104f 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -410,6 +410,9 @@ struct pd_timings {
 /* Convert microwatt to watt */
 #define UW_TO_W(pow)				(div_u64((pow), 1000000))
 
+/* Convert micro-Watt-hours to USB PD energy (increments of 0.1Wh) */
+#define UWH_TO_PD_ENERGY(energy)		(UW_TO_W((u64)(energy) * 10))
+
 /*
  * As per USB PD Spec Rev 3.18 (Sec. 6.5.13.11), the number of fixed batteries
  * that a port can be queried is restricted to 4.
@@ -1580,8 +1583,7 @@ static u16 tcpm_charge_to_energy(int charge, int voltage)
 {
 	u64 energy = div_u64((u64)charge * voltage, 1000000);
 
-	/* Battery telemetry is reported in increments of 0.1Wh */
-	return (u16)UW_TO_W(energy * 10);
+	return UWH_TO_PD_ENERGY(energy);
 }
 
 static int tcpm_pd_send_batt_status(struct tcpm_port *port)
@@ -1612,16 +1614,17 @@ static int tcpm_pd_send_batt_status(struct tcpm_port *port)
 	else
 		batt_present = val.intval > 0;
 
-	ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_CHARGE_NOW,
-					&val);
+	ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_ENERGY_NOW, &val);
 	if (!ret) {
-		charge_now = val.intval;
-		ret = power_supply_get_property(batt,
-						POWER_SUPPLY_PROP_VOLTAGE_AVG,
-						&val);
-		if (!ret)
-			present_charge = tcpm_charge_to_energy(charge_now,
-							       val.intval);
+		present_charge = UWH_TO_PD_ENERGY(val.intval);
+	} else {
+		ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_CHARGE_NOW, &val);
+		if (!ret) {
+			charge_now = val.intval;
+			ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_VOLTAGE_AVG, &val);
+			if (!ret)
+				present_charge = tcpm_charge_to_energy(charge_now, val.intval);
+		}
 	}
 
 	ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_STATUS, &val);
@@ -1678,19 +1681,31 @@ static int tcpm_pd_send_batt_cap(struct tcpm_port *port)
 
 	invalid_ref = false;
 	batt = port->fixed_batt[batt_id];
-	ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_VOLTAGE_AVG,
-					&val);
-	if (!ret) {
-		vol = val.intval;
-		ret = power_supply_get_property(batt,
-						POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN,
-						&val);
+
+	ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN, &val);
+	if (!ret)
+		design_cap = UWH_TO_PD_ENERGY(val.intval);
+
+	ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_ENERGY_FULL, &val);
+	if (!ret)
+		charge_cap = UWH_TO_PD_ENERGY(val.intval);
+
+	if (design_cap != BATTERY_PROPERTY_UNKNOWN && charge_cap != BATTERY_PROPERTY_UNKNOWN)
+		goto send_cap;
+
+	ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_VOLTAGE_AVG, &val);
+	if (ret)
+		goto send_cap;
+
+	vol = val.intval;
+	if (design_cap == BATTERY_PROPERTY_UNKNOWN) {
+		ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, &val);
 		if (!ret)
 			design_cap = tcpm_charge_to_energy(val.intval, vol);
+	}
 
-		ret = power_supply_get_property(batt,
-						POWER_SUPPLY_PROP_CHARGE_FULL,
-						&val);
+	if (charge_cap == BATTERY_PROPERTY_UNKNOWN) {
+		ret = power_supply_get_property(batt, POWER_SUPPLY_PROP_CHARGE_FULL, &val);
 		if (!ret)
 			charge_cap = tcpm_charge_to_energy(val.intval, vol);
 	}
@@ -1756,7 +1771,8 @@ static void mod_enable_frs_delayed_work(struct tcpm_port *port, unsigned int del
 static void mod_vdm_discovery_cancel_delayed_work(struct tcpm_port *port)
 {
 	hrtimer_cancel(&port->vdm_discovery_timer);
-	kthread_cancel_work_sync(&port->vdm_discovery_work);
+	if (port->wq)
+		kthread_cancel_work_sync(&port->vdm_discovery_work);
 }
 
 static void mod_vdm_discovery_delayed_work(struct tcpm_port *port, unsigned int delay_ms)
@@ -7113,7 +7129,7 @@ static void tcpm_pd_event_handler(struct kthread_work *work)
 				port->upcoming_state = FR_SWAP_SEND;
 				ret = tcpm_ams_start(port, FAST_ROLE_SWAP);
 				if (ret == -EAGAIN)
-					port->upcoming_state = INVALID_STATE;
+					tcpm_set_state(port, ERROR_RECOVERY, 0);
 			} else {
 				tcpm_log(port, "Discarding FRS_SIGNAL! Not in sink ready");
 			}
@@ -8977,6 +8993,7 @@ void tcpm_unregister_port(struct tcpm_port *port)
 
 	port->registered = false;
 	kthread_destroy_worker(port->wq);
+	port->wq = NULL;
 
 	hrtimer_cancel(&port->vdm_discovery_timer);
 	hrtimer_cancel(&port->enable_frs_timer);
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index f76f563dc42b8f..02539378ffedae 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -150,6 +150,7 @@ struct tipd_data {
 	irq_handler_t irq_handler;
 	u64 irq_mask1;
 	size_t tps_struct_size;
+	bool no_mode_control;
 	void (*remove)(struct tps6598x *tps);
 	int (*register_port)(struct tps6598x *tps, struct fwnode_handle *node);
 	void (*unregister_port)(struct tps6598x *tps);
@@ -343,7 +344,7 @@ static void tps6598x_set_data_role(struct tps6598x *tps,
 
 static int tps6598x_connect(struct tps6598x *tps, u32 status)
 {
-	struct typec_partner_desc desc;
+	struct typec_partner_desc desc = { };
 	enum typec_pwr_opmode mode;
 	int ret;
 
@@ -354,7 +355,6 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status)
 
 	desc.usb_pd = mode == TYPEC_PWR_MODE_PD;
 	desc.accessory = TYPEC_ACCESSORY_NONE; /* XXX: handle accessories */
-	desc.identity = NULL;
 
 	if (desc.usb_pd) {
 		ret = tps6598x_read_partner_identity(tps);
@@ -850,11 +850,10 @@ static void cd321x_update_work(struct work_struct *work)
 
 	/* Set up partner if we were previously disconnected (or changed). */
 	if (!tps->partner) {
-		struct typec_partner_desc desc;
+		struct typec_partner_desc desc = { };
 
 		desc.usb_pd = is_pd;
 		desc.accessory = TYPEC_ACCESSORY_NONE; /* XXX: handle accessories */
-		desc.identity = NULL;
 
 		if (desc.usb_pd)
 			desc.identity = &st.partner_identity;
@@ -1251,6 +1250,7 @@ tps6598x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode)
 	typec_cap.driver_data = tps;
 	typec_cap.ops = &tps6598x_ops;
 	typec_cap.fwnode = fwnode;
+	typec_cap.no_mode_control = tps->data->no_mode_control;
 
 	switch (TPS_SYSCONF_PORTINFO(conf)) {
 	case TPS_PORTINFO_SINK_ACCESSORY:
@@ -1792,6 +1792,7 @@ static int tps6598x_probe(struct i2c_client *client)
 	const struct tipd_data *data;
 	struct tps6598x *tps;
 	struct fwnode_handle *fwnode;
+	bool patch_loaded = false;
 	u32 status;
 	u32 vid = 0;
 	int ret;
@@ -1847,6 +1848,7 @@ static int tps6598x_probe(struct i2c_client *client)
 		return ret;
 
 	if (ret == TPS_MODE_PTCH) {
+		patch_loaded = true;
 		ret = tps->data->init(tps);
 		if (ret)
 			return ret;
@@ -1937,7 +1939,8 @@ static int tps6598x_probe(struct i2c_client *client)
 	tps6598x_write64(tps, TPS_REG_INT_MASK1, 0);
 err_reset_controller:
 	/* Reset PD controller to remove any applied patch */
-	tps->data->reset(tps);
+	if (patch_loaded)
+		tps->data->reset(tps);
 
 	return ret;
 }
@@ -2024,6 +2027,7 @@ static const struct tipd_data cd321x_data = {
 		     APPLE_CD_REG_INT_DATA_STATUS_UPDATE |
 		     APPLE_CD_REG_INT_PLUG_EVENT,
 	.tps_struct_size = sizeof(struct cd321x),
+	.no_mode_control = true,
 	.remove = cd321x_remove,
 	.register_port = cd321x_register_port,
 	.unregister_port = cd321x_unregister_port,
diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c
index 8d2032d0762ca8..572da7bbd29c73 100644
--- a/drivers/usb/typec/ucsi/displayport.c
+++ b/drivers/usb/typec/ucsi/displayport.c
@@ -71,11 +71,14 @@ static int ucsi_displayport_enter(struct typec_altmode *alt, u32 *vdo)
 	if (ret < 0) {
 		if (ucsi->version > 0x0100)
 			goto err_unlock;
-		cur = 0xff;
 	}
 
-	if (cur < UCSI_MAX_ALTMODES) {
-		ret = dp->con->port_altmode[cur] == alt ? 0 : -EBUSY;
+	if (cur != 0xff) {
+		if (cur < UCSI_MAX_ALTMODES)
+			ret = dp->con->port_altmode[cur] == alt ? 0 : -EBUSY;
+		else
+			ret = -EINVAL;
+
 		goto err_unlock;
 	}
 
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index bef3f9b71d718d..577e764a18e76f 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -2158,7 +2158,7 @@ static int ucsi_init(struct ucsi *ucsi)
 
 static void ucsi_resume_work(struct work_struct *work)
 {
-	struct ucsi *ucsi = container_of(work, struct ucsi, resume_work);
+	struct ucsi *ucsi = container_of(to_delayed_work(work), struct ucsi, resume_work);
 	struct ucsi_connector *con;
 	u64 command;
 	int ret;
@@ -2167,6 +2167,11 @@ static void ucsi_resume_work(struct work_struct *work)
 	command = UCSI_SET_NOTIFICATION_ENABLE | ucsi->ntfy;
 	ret = ucsi_send_command(ucsi, command, NULL, 0);
 	if (ret < 0) {
+		if (++ucsi->resume_retries < 5) {
+			queue_delayed_work(system_dfl_long_wq, &ucsi->resume_work,
+					   msecs_to_jiffies(500));
+			return;
+		}
 		dev_err(ucsi->dev, "failed to re-enable notifications (%d)\n", ret);
 		return;
 	}
@@ -2187,6 +2192,7 @@ int ucsi_suspend(struct ucsi *ucsi)
 	 * EC is stopped for suspend; state is re-read on resume.
 	 */
 	cancel_delayed_work_sync(&ucsi->work);
+	cancel_delayed_work_sync(&ucsi->resume_work);
 
 	if (!ucsi->connector)
 		return 0;
@@ -2200,8 +2206,10 @@ EXPORT_SYMBOL_GPL(ucsi_suspend);
 
 int ucsi_resume(struct ucsi *ucsi)
 {
-	if (ucsi->connector)
-		queue_work(system_long_wq, &ucsi->resume_work);
+	if (ucsi->connector) {
+		ucsi->resume_retries = 0;
+		queue_delayed_work(system_dfl_long_wq, &ucsi->resume_work, 0);
+	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(ucsi_resume);
@@ -2212,18 +2220,36 @@ static void ucsi_init_work(struct work_struct *work)
 	int ret;
 
 	ret = ucsi_init(ucsi);
-	if (ret)
-		dev_err_probe(ucsi->dev, ret, "PPM init failed\n");
+	if (!ret) {
+		if (ucsi->work_count)
+			dev_info(ucsi->dev,
+				 "PPM init succeeded after %u attempts\n",
+				 ucsi->work_count + 1);
+		return;
+	}
 
-	if (ret == -EPROBE_DEFER) {
-		if (ucsi->work_count++ > UCSI_ROLE_SWITCH_WAIT_COUNT) {
-			dev_err(ucsi->dev, "PPM init failed, stop trying\n");
+	/*
+	 * On some platforms the PPM is not ready to answer commands
+	 * correctly for a short window during boot: standard commands are
+	 * rejected or GET_CAPABILITY reports zero connectors, seen as
+	 * -EINVAL or -ENODEV from ucsi_init(), and a retry moments later
+	 * succeeds (observed on Lenovo ThinkPad X1 Carbon Gen 14, where
+	 * this affects roughly half of all boots). Retry those like the
+	 * USB role switch wait instead of giving up on the first attempt.
+	 */
+	if (ret == -EPROBE_DEFER || ret == -ENODEV || ret == -EINVAL) {
+		if (ucsi->work_count++ < UCSI_ROLE_SWITCH_WAIT_COUNT) {
+			dev_dbg(ucsi->dev, "PPM init failed (%pe), retrying\n",
+				ERR_PTR(ret));
+			queue_delayed_work(system_dfl_long_wq, &ucsi->work,
+					   UCSI_ROLE_SWITCH_INTERVAL);
 			return;
 		}
-
-		queue_delayed_work(system_dfl_long_wq, &ucsi->work,
-				   UCSI_ROLE_SWITCH_INTERVAL);
+		dev_err(ucsi->dev, "PPM init failed, stop trying\n");
+		return;
 	}
+
+	dev_err_probe(ucsi->dev, ret, "PPM init failed\n");
 }
 
 /**
@@ -2299,7 +2325,7 @@ struct ucsi *ucsi_create(struct device *dev, const struct ucsi_operations *ops)
 	if (!ucsi)
 		return ERR_PTR(-ENOMEM);
 
-	INIT_WORK(&ucsi->resume_work, ucsi_resume_work);
+	INIT_DELAYED_WORK(&ucsi->resume_work, ucsi_resume_work);
 	INIT_DELAYED_WORK(&ucsi->work, ucsi_init_work);
 	mutex_init(&ucsi->ppm_lock);
 	init_completion(&ucsi->complete);
@@ -2365,7 +2391,7 @@ void ucsi_unregister(struct ucsi *ucsi)
 
 	/* Make sure that we are not in the middle of driver initialization */
 	cancel_delayed_work_sync(&ucsi->work);
-	cancel_work_sync(&ucsi->resume_work);
+	cancel_delayed_work_sync(&ucsi->resume_work);
 
 	ucsi_debugfs_unregister(ucsi);
 
diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h
index dc594388dcd081..1996f04ed5e68d 100644
--- a/drivers/usb/typec/ucsi/ucsi.h
+++ b/drivers/usb/typec/ucsi/ucsi.h
@@ -484,7 +484,8 @@ struct ucsi {
 	struct ucsi_connector *connector;
 	struct ucsi_debugfs_entry *debugfs;
 
-	struct work_struct resume_work;
+	struct delayed_work resume_work;
+	int resume_retries;
 	struct delayed_work work;
 	int work_count;
 #define UCSI_ROLE_SWITCH_RETRY_PER_HZ	10
diff --git a/drivers/usb/typec/ucsi/ucsi_acpi.c b/drivers/usb/typec/ucsi/ucsi_acpi.c
index 18286d3e9cc59a..5697ccc24ec3fc 100644
--- a/drivers/usb/typec/ucsi/ucsi_acpi.c
+++ b/drivers/usb/typec/ucsi/ucsi_acpi.c
@@ -121,6 +121,33 @@ static const struct ucsi_operations ucsi_acpi_ops = {
 	.async_control = ucsi_acpi_async_control
 };
 
+static int ucsi_acer_read_version(struct ucsi *ucsi, u16 *version)
+{
+	struct ucsi_acpi *ua = ucsi_get_drvdata(ucsi);
+	int ret;
+
+	ret = ucsi_acpi_read_version(ucsi, version);
+	if (ret)
+		return ret;
+
+	if (!*version) {
+		dev_warn(ua->dev, "UCSI version is zero, assuming 1.2\n");
+		*version = UCSI_VERSION_1_2;
+	}
+
+	return 0;
+}
+
+static const struct ucsi_operations ucsi_acer_ops = {
+	.read_version = ucsi_acer_read_version,
+	.read_cci = ucsi_acpi_read_cci,
+	.poll_cci = ucsi_acpi_poll_cci,
+	.read_message_in = ucsi_acpi_read_message_in,
+	.write_message_out = ucsi_acpi_write_message_out,
+	.sync_control = ucsi_sync_control_common,
+	.async_control = ucsi_acpi_async_control
+};
+
 static int ucsi_gram_sync_control(struct ucsi *ucsi, u64 command, u32 *cci,
 				  void *val, size_t len, void *msg_out,
 				  size_t msg_out_size)
@@ -162,6 +189,13 @@ static const struct ucsi_operations ucsi_gram_ops = {
 };
 
 static const struct dmi_system_id ucsi_acpi_quirks[] = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Nitro ANV15-41"),
+		},
+		.driver_data = (void *)&ucsi_acer_ops,
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LG Electronics"),
diff --git a/drivers/usb/typec/ucsi/ucsi_glink.c b/drivers/usb/typec/ucsi/ucsi_glink.c
index 12e07b9fe62280..1db0c88d817e6d 100644
--- a/drivers/usb/typec/ucsi/ucsi_glink.c
+++ b/drivers/usb/typec/ucsi/ucsi_glink.c
@@ -362,6 +362,10 @@ static void pmic_glink_ucsi_destroy(void *data)
 {
 	struct pmic_glink_ucsi *ucsi = data;
 
+	/* Drain the work items before ucsi_destroy() frees the ucsi instance */
+	cancel_work_sync(&ucsi->notify_work);
+	cancel_work_sync(&ucsi->register_work);
+
 	/* Protect to make sure we're not in a middle of a transaction from a glink callback */
 	mutex_lock(&ucsi->lock);
 	ucsi_destroy(ucsi->ucsi);
@@ -467,8 +471,15 @@ static void pmic_glink_ucsi_remove(struct auxiliary_device *adev)
 {
 	struct pmic_glink_ucsi *ucsi = dev_get_drvdata(&adev->dev);
 
-	/* Unregister first to stop having read & writes */
-	ucsi_unregister(ucsi->ucsi);
+	/* Callbacks can queue work until devres releases the client */
+	disable_work_sync(&ucsi->notify_work);
+	disable_work_sync(&ucsi->register_work);
+
+	/* register_work may have unregistered the instance already */
+	if (ucsi->ucsi_registered) {
+		ucsi->ucsi_registered = false;
+		ucsi_unregister(ucsi->ucsi);
+	}
 }
 
 static const struct auxiliary_device_id pmic_glink_ucsi_id_table[] = {
diff --git a/drivers/usb/typec/wusb3801.c b/drivers/usb/typec/wusb3801.c
index 6062875fb04a9d..3ca66138f7be36 100644
--- a/drivers/usb/typec/wusb3801.c
+++ b/drivers/usb/typec/wusb3801.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
diff --git a/drivers/usb/usbip/stub_main.c b/drivers/usb/usbip/stub_main.c
index 79110a69d697c1..01452f7bdabbdc 100644
--- a/drivers/usb/usbip/stub_main.c
+++ b/drivers/usb/usbip/stub_main.c
@@ -371,7 +371,7 @@ static int __init usbip_host_init(void)
 		return -ENOMEM;
 	}
 
-	ret = usb_register_device_driver(&stub_driver, THIS_MODULE);
+	ret = usb_register_device_driver(&stub_driver);
 	if (ret) {
 		pr_err("usb_register failed %d\n", ret);
 		goto err_usb_register;
diff --git a/drivers/vfio/cdx/main.c b/drivers/vfio/cdx/main.c
index b31ed4be7bdc19..28dc8c82560063 100644
--- a/drivers/vfio/cdx/main.c
+++ b/drivers/vfio/cdx/main.c
@@ -254,10 +254,10 @@ static int vfio_cdx_mmap_mmio(struct vfio_cdx_region region,
 	if (base + size > region.size)
 		return -EINVAL;
 
-	vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
 	vma->vm_page_prot = pgprot_device(vma->vm_page_prot);
 
-	return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+	return io_remap_pfn_range(vma, vma->vm_start,
+				  (region.addr >> PAGE_SHIFT) + pgoff,
 				  size, vma->vm_page_prot);
 }
 
@@ -271,6 +271,9 @@ static int vfio_cdx_mmap(struct vfio_device *core_vdev,
 
 	index = vma->vm_pgoff >> (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT);
 
+	if (!(vma->vm_flags & VM_SHARED))
+		return -EINVAL;
+
 	if (index >= cdx_dev->res_count)
 		return -EINVAL;
 
@@ -281,9 +284,13 @@ static int vfio_cdx_mmap(struct vfio_device *core_vdev,
 	    (vma->vm_flags & VM_READ))
 		return -EPERM;
 
-	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE) &&
-	    (vma->vm_flags & VM_WRITE))
-		return -EPERM;
+	/* Prevent read-only region mappings from being upgraded with mprotect() */
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)) {
+		if (vma->vm_flags & VM_WRITE)
+			return -EPERM;
+
+		vm_flags_clear(vma, VM_MAYWRITE);
+	}
 
 	return vfio_cdx_mmap_mmio(vdev->regions[index], vma);
 }
diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c
index 1d9515c967b017..30362936c7b597 100644
--- a/drivers/vfio/device_cdev.c
+++ b/drivers/vfio/device_cdev.c
@@ -130,6 +130,18 @@ long vfio_df_ioctl_bind_iommufd(struct vfio_device_file *df,
 		goto out_unlock;
 	}
 
+	/*
+	 * The cdev path allows only a single open.  Reject a second open here,
+	 * before the VF token and device->kvm updates below would clobber the
+	 * current opener's state on a bind that cannot complete.  Return -EBUSY
+	 * rather than -EINVAL since a delayed release of the prior opener can
+	 * make this transient.
+	 */
+	if (device->open_count) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
 	ret = vfio_df_check_token(device, &bind);
 	if (ret)
 		goto out_unlock;
diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c
index b4c3958201b258..15b05b8c0c9086 100644
--- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c
+++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c
@@ -369,9 +369,8 @@ static int vfio_fsl_mc_mmap_mmio(struct vfio_fsl_mc_region region,
 	if (!region_cacheable)
 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
-	vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
-
-	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+	return remap_pfn_range(vma, vma->vm_start,
+			       (region.addr >> PAGE_SHIFT) + pgoff,
 			       size, vma->vm_page_prot);
 }
 
@@ -403,9 +402,13 @@ static int vfio_fsl_mc_mmap(struct vfio_device *core_vdev,
 			&& (vma->vm_flags & VM_READ))
 		return -EINVAL;
 
-	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)
-			&& (vma->vm_flags & VM_WRITE))
-		return -EINVAL;
+	/* Prevent read-only region mappings from being upgraded with mprotect() */
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)) {
+		if (vma->vm_flags & VM_WRITE)
+			return -EINVAL;
+
+		vm_flags_clear(vma, VM_MAYWRITE);
+	}
 
 	vma->vm_private_data = mc_dev;
 
diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c
index 3b4aa0632fccd1..34337292a9b19d 100644
--- a/drivers/vfio/mdev/mdev_core.c
+++ b/drivers/vfio/mdev/mdev_core.c
@@ -50,7 +50,7 @@ static int mdev_device_remove_cb(struct device *dev, void *data)
  * @types: Array of supported mdev types
  * @nr_types: Number of entries in @types
  *
- * Registers the @parent stucture as a parent for mdev types and thus mdev
+ * Registers the @parent structure as a parent for mdev types and thus mdev
  * devices.  The caller needs to hold a reference on @dev that must not be
  * released until after the call to mdev_unregister_parent().
  *
diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c
index 7f14dd46de17a0..df884808768157 100644
--- a/drivers/vfio/pci/vfio_pci_rdwr.c
+++ b/drivers/vfio/pci/vfio_pci_rdwr.c
@@ -423,9 +423,6 @@ int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
 	      pos >= vdev->msix_offset + vdev->msix_size))
 		return -EINVAL;
 
-	if (count == 8)
-		return -EINVAL;
-
 	io = vfio_pci_core_get_iomap(vdev, bar);
 	if (IS_ERR(io))
 		return PTR_ERR(io);
diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index c72db5a99ebda2..e8cfa76ad7f87d 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -559,9 +559,8 @@ static int vfio_platform_mmap_mmio(struct vfio_platform_region region,
 		return -EINVAL;
 
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-	vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
-
-	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+	return remap_pfn_range(vma, vma->vm_start,
+			       (region.addr >> PAGE_SHIFT) + pgoff,
 			       req_len, vma->vm_page_prot);
 }
 
@@ -591,9 +590,13 @@ int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
 			&& (vma->vm_flags & VM_READ))
 		return -EINVAL;
 
-	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)
-			&& (vma->vm_flags & VM_WRITE))
-		return -EINVAL;
+	/* Prevent read-only region mappings from being upgraded with mprotect() */
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)) {
+		if (vma->vm_flags & VM_WRITE)
+			return -EINVAL;
+
+		vm_flags_clear(vma, VM_MAYWRITE);
+	}
 
 	vma->vm_private_data = vdev;
 
diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index c8151ba54de33a..f7addfbe1aab4f 100644
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -94,6 +94,7 @@ struct vfio_dma {
 	bool			lock_cap;	/* capable(CAP_IPC_LOCK) */
 	bool			vaddr_invalid;
 	bool			has_rsvd;	/* has 1 or more rsvd pfns */
+	bool			has_non_rsvd;	/* has 1 or more !rsvd pfns */
 	struct task_struct	*task;
 	struct rb_root		pfn_list;	/* Ex-user pinned pfn list */
 	unsigned long		*bitmap;
@@ -791,6 +792,7 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr,
 
 out:
 	dma->has_rsvd |= rsvd;
+	dma->has_non_rsvd |= !rsvd;
 	ret = vfio_lock_acct(dma, lock_acct, false);
 
 unpin_out:
@@ -821,11 +823,13 @@ static long vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova,
 	long unlocked = 0, locked = vpfn_pages(dma, iova, npage);
 
 	if (dma->has_rsvd) {
-		unsigned long i;
+		if (dma->has_non_rsvd) {
+			unsigned long i;
 
-		for (i = 0; i < npage; i++)
-			if (put_pfn(pfn++, dma->prot))
-				unlocked++;
+			for (i = 0; i < npage; i++)
+				if (put_pfn(pfn++, dma->prot))
+					unlocked++;
+		}
 	} else {
 		put_valid_unreserved_pfns(pfn, npage, dma->prot);
 		unlocked = npage;
diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c
index 423ead48aafe33..cb3deb5a485745 100644
--- a/drivers/vfio/vfio_main.c
+++ b/drivers/vfio/vfio_main.c
@@ -603,13 +603,6 @@ int vfio_df_open(struct vfio_device_file *df)
 
 	lockdep_assert_held(&device->dev_set->lock);
 
-	/*
-	 * Only the group path allows the device to be opened multiple
-	 * times.  The device cdev path doesn't have a secure way for it.
-	 */
-	if (device->open_count != 0 && !df->group)
-		return -EINVAL;
-
 	device->open_count++;
 	if (device->open_count == 1) {
 		ret = vfio_df_device_first_open(df);
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 2cc730729e08a5..4570a4ffbab2c5 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -1452,7 +1452,7 @@ static int vhost_net_release(struct inode *inode, struct file *f)
 	vhost_net_stop(n, &tx_sock, &rx_sock);
 	vhost_net_flush(n);
 	vhost_dev_stop(&n->dev);
-	vhost_dev_cleanup(&n->dev);
+	vhost_dev_cleanup(&n->dev, false);
 	vhost_net_vq_reset(n);
 	if (tx_sock)
 		sockfd_put(tx_sock);
@@ -1661,7 +1661,7 @@ static long vhost_net_reset_owner(struct vhost_net *n)
 	vhost_net_stop(n, &tx_sock, &rx_sock);
 	vhost_net_flush(n);
 	vhost_dev_stop(&n->dev);
-	vhost_dev_reset_owner(&n->dev, umem);
+	vhost_dev_reset_owner(&n->dev, umem, false);
 	vhost_net_vq_reset(n);
 done:
 	mutex_unlock(&n->dev.mutex);
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 4f8c0260bc9e78..885fac5ed042ea 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -2390,7 +2390,7 @@ static int vhost_scsi_release(struct inode *inode, struct file *f)
 	mutex_unlock(&vs->dev.mutex);
 	vhost_scsi_clear_endpoint(vs, &t);
 	vhost_dev_stop(&vs->dev);
-	vhost_dev_cleanup(&vs->dev);
+	vhost_dev_cleanup(&vs->dev, false);
 	kfree(vs->dev.vqs);
 	kvfree(vs->vqs);
 	kfree(vs->old_inflight);
diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c
index 24514c8fdee494..84ed9ea81219d3 100644
--- a/drivers/vhost/test.c
+++ b/drivers/vhost/test.c
@@ -163,7 +163,7 @@ static int vhost_test_release(struct inode *inode, struct file *f)
 	vhost_test_stop(n, &private);
 	vhost_test_flush(n);
 	vhost_dev_stop(&n->dev);
-	vhost_dev_cleanup(&n->dev);
+	vhost_dev_cleanup(&n->dev, false);
 	kfree(n->dev.vqs);
 	kfree(n);
 	return 0;
@@ -238,7 +238,7 @@ static long vhost_test_reset_owner(struct vhost_test *n)
 	vhost_test_stop(n, &priv);
 	vhost_test_flush(n);
 	vhost_dev_stop(&n->dev);
-	vhost_dev_reset_owner(&n->dev, umem);
+	vhost_dev_reset_owner(&n->dev, umem, false);
 done:
 	mutex_unlock(&n->dev.mutex);
 	return err;
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index a31786796d4cef..ff7d9071917120 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -948,7 +948,7 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep,
 	case VHOST_SET_OWNER:
 		r = vhost_vdpa_bind_mm(v);
 		if (r)
-			vhost_dev_reset_owner(d, NULL);
+			vhost_dev_reset_owner(d, NULL, false);
 		break;
 	}
 out:
@@ -1453,7 +1453,7 @@ static void vhost_vdpa_cleanup(struct vhost_vdpa *v)
 	}
 
 	vhost_vdpa_free_domain(v);
-	vhost_dev_cleanup(&v->vdev);
+	vhost_dev_cleanup(&v->vdev, false);
 	kfree(v->vdev.vqs);
 	v->vdev.vqs = NULL;
 }
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 44cac11b68d2a7..4d5d32e68d35f0 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -731,7 +731,7 @@ static void vhost_worker_destroy(struct vhost_dev *dev,
 	kfree(worker);
 }
 
-static void vhost_workers_free(struct vhost_dev *dev)
+static void vhost_workers_free(struct vhost_dev *dev, bool sync)
 {
 	struct vhost_worker *worker;
 	unsigned long i;
@@ -741,6 +741,21 @@ static void vhost_workers_free(struct vhost_dev *dev)
 
 	for (i = 0; i < dev->nvqs; i++)
 		rcu_assign_pointer(dev->vqs[i]->worker, NULL);
+
+	/*
+	 * This path can be reached by lockless work queuers.  In this case
+	 * vhost_vq_work_queue() reads vq->worker under rcu_read_lock(), so a
+	 * RCU reader that fetched a worker before we cleared the pointers above
+	 * may still be queueing work on it.  Wait for those readers to finish,
+	 * then flush so any work they queued runs (clearing VHOST_WORK_QUEUED)
+	 * before the workers are freed.  Notably, that is the case for the
+	 * vsock RESET_OWNER path.
+	 */
+	if (sync) {
+		synchronize_rcu();
+		vhost_dev_flush(dev);
+	}
+
 	/*
 	 * Free the default worker we created and cleanup workers userspace
 	 * created but couldn't clean up (it forgot or crashed).
@@ -1163,11 +1178,12 @@ struct vhost_iotlb *vhost_dev_reset_owner_prepare(void)
 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
 
 /* Caller should have device mutex */
-void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem)
+void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *umem,
+			   bool sync)
 {
 	int i;
 
-	vhost_dev_cleanup(dev);
+	vhost_dev_cleanup(dev, sync);
 
 	dev->fork_owner = fork_from_owner_default;
 	dev->umem = umem;
@@ -1227,7 +1243,7 @@ void vhost_clear_msg(struct vhost_dev *dev)
 }
 EXPORT_SYMBOL_GPL(vhost_clear_msg);
 
-void vhost_dev_cleanup(struct vhost_dev *dev)
+void vhost_dev_cleanup(struct vhost_dev *dev, bool sync)
 {
 	int i;
 
@@ -1251,7 +1267,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev)
 	dev->iotlb = NULL;
 	vhost_clear_msg(dev);
 	wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
-	vhost_workers_free(dev);
+	vhost_workers_free(dev, sync);
 	vhost_detach_mm(dev);
 }
 EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 39e6121f752599..5a684ba5e38913 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -219,8 +219,9 @@ long vhost_dev_set_owner(struct vhost_dev *dev);
 bool vhost_dev_has_owner(struct vhost_dev *dev);
 long vhost_dev_check_owner(struct vhost_dev *);
 struct vhost_iotlb *vhost_dev_reset_owner_prepare(void);
-void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb);
-void vhost_dev_cleanup(struct vhost_dev *);
+void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb,
+			   bool sync);
+void vhost_dev_cleanup(struct vhost_dev *dev, bool sync);
 void vhost_dev_stop(struct vhost_dev *);
 long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp);
 long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp);
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index abed1fbcf66cc5..0cb87971d74919 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -61,6 +61,7 @@ struct vhost_vsock {
 
 	u32 guest_cid;
 	bool seqpacket_allow;
+	bool ever_started; /* set on first SET_RUNNING(1); never cleared */
 };
 
 static u32 vhost_transport_get_local_cid(void)
@@ -302,17 +303,12 @@ vhost_transport_send_pkt(struct sk_buff *skb, struct net *net)
 		return -ENODEV;
 	}
 
-	/* Fast-fail if the guest hasn't enabled the RX vq yet. Queuing the packet
-	 * and making the caller wait is pointless: even if the guest manages to init
-	 * within the timeout, it'll immediately reply with RST, because there's no
-	 * listener on the port yet.
-	 *
-	 * vhost_vq_get_backend() without vq->mutex is acceptable here: locking
-	 * the mutex would be too expensive in this hot path, and we already have
-	 * all the outcomes covered: if the backend becomes NULL right after the check,
-	 * vhost_transport_do_send_pkt() will check it under the mutex anyway.
+	/* Fast-fail until the guest first enables the device (SET_RUNNING(1)).
+	 * Before that there is no listener, so queuing is pointless.
+	 * 'ever_started' is never cleared, so once we're up we keep queuing
+	 * across later stop / CPR-pause windows.
 	 */
-	if (unlikely(!data_race(vhost_vq_get_backend(&vsock->vqs[VSOCK_VQ_RX])))) {
+	if (unlikely(!READ_ONCE(vsock->ever_started))) {
 		rcu_read_unlock();
 		kfree_skb(skb);
 		return -EHOSTUNREACH;
@@ -640,11 +636,23 @@ static int vhost_vsock_start(struct vhost_vsock *vsock)
 		mutex_unlock(&vq->mutex);
 	}
 
+	/* Set 'ever_started' flag on the first start; never cleared, so send_pkt
+	 * keeps queuing (instead of fast-failing) on later stop / CPR pauses.
+	 */
+	WRITE_ONCE(vsock->ever_started, true);
+
 	/* Some packets may have been queued before the device was started,
 	 * let's kick the send worker to send them.
 	 */
 	vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work);
 
+	/* The guest may have added TX buffers while the device was stopped
+	 * (e.g. across VHOST_RESET_OWNER) and their kicks got consumed by
+	 * the NULL-backend window.  Re-scan the TX VQ, mirroring the RX
+	 * send-worker kick above.
+	 */
+	vhost_poll_queue(&vsock->vqs[VSOCK_VQ_TX].poll);
+
 	mutex_unlock(&vsock->dev.mutex);
 	return 0;
 
@@ -664,9 +672,24 @@ static int vhost_vsock_start(struct vhost_vsock *vsock)
 	return ret;
 }
 
-static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
+static void vhost_vsock_drop_backends(struct vhost_vsock *vsock)
 {
+	struct vhost_virtqueue *vq;
 	size_t i;
+
+	lockdep_assert_held(&vsock->dev.mutex);
+
+	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
+		vq = &vsock->vqs[i];
+
+		mutex_lock(&vq->mutex);
+		vhost_vq_set_backend(vq, NULL);
+		mutex_unlock(&vq->mutex);
+	}
+}
+
+static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
+{
 	int ret = 0;
 
 	mutex_lock(&vsock->dev.mutex);
@@ -677,14 +700,7 @@ static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)
 			goto err;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
-		struct vhost_virtqueue *vq = &vsock->vqs[i];
-
-		mutex_lock(&vq->mutex);
-		vhost_vq_set_backend(vq, NULL);
-		mutex_unlock(&vq->mutex);
-	}
-
+	vhost_vsock_drop_backends(vsock);
 err:
 	mutex_unlock(&vsock->dev.mutex);
 	return ret;
@@ -720,6 +736,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
 
 	vsock->guest_cid = 0; /* no CID assigned yet */
 	vsock->seqpacket_allow = false;
+	vsock->ever_started = false;
 
 	atomic_set(&vsock->queued_replies, 0);
 
@@ -807,7 +824,7 @@ static int vhost_vsock_dev_release(struct inode *inode, struct file *file)
 
 	virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue);
 
-	vhost_dev_cleanup(&vsock->dev);
+	vhost_dev_cleanup(&vsock->dev, false);
 	put_net_track(vsock->net, &vsock->ns_tracker);
 	kfree(vsock->dev.vqs);
 	vhost_vsock_free(vsock);
@@ -888,6 +905,36 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 	return -EFAULT;
 }
 
+static long vhost_vsock_reset_owner(struct vhost_vsock *vsock)
+{
+	struct vhost_iotlb *umem;
+	long err;
+
+	mutex_lock(&vsock->dev.mutex);
+	err = vhost_dev_check_owner(&vsock->dev);
+	if (err)
+		goto done;
+	umem = vhost_dev_reset_owner_prepare();
+	if (!umem) {
+		err = -ENOMEM;
+		goto done;
+	}
+	vhost_vsock_drop_backends(vsock);
+	vhost_vsock_flush(vsock);
+	vhost_dev_stop(&vsock->dev);
+
+	/*
+	 * vsock keeps the CID hashed across RESET_OWNER, so a lockless sender
+	 * can still reach the device while its workers are freed.  Pass
+	 * sync=true so vhost_workers_free() fences RCU readers and flushes
+	 * before freeing them.
+	 */
+	vhost_dev_reset_owner(&vsock->dev, umem, true);
+done:
+	mutex_unlock(&vsock->dev.mutex);
+	return err;
+}
+
 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
 				  unsigned long arg)
 {
@@ -931,6 +978,8 @@ static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
 			return -EOPNOTSUPP;
 		vhost_set_backend_features(&vsock->dev, features);
 		return 0;
+	case VHOST_RESET_OWNER:
+		return vhost_vsock_reset_owner(vsock);
 	default:
 		mutex_lock(&vsock->dev.mutex);
 		r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);
diff --git a/drivers/video/backlight/ili9320.c b/drivers/video/backlight/ili9320.c
index 2df96a882119fd..d2f094dd7fc35f 100644
--- a/drivers/video/backlight/ili9320.c
+++ b/drivers/video/backlight/ili9320.c
@@ -30,7 +30,7 @@ static inline int ili9320_write_spi(struct ili9320 *ili,
 	unsigned char *addr = spi->buffer_addr;
 	unsigned char *data = spi->buffer_data;
 
-	/* spi message consits of:
+	/* spi message consists of:
 	 * first byte: ID and operation
 	 */
 
diff --git a/drivers/video/backlight/ktd253-backlight.c b/drivers/video/backlight/ktd253-backlight.c
index 327b4ee75254e9..f2c5527d02e0ff 100644
--- a/drivers/video/backlight/ktd253-backlight.c
+++ b/drivers/video/backlight/ktd253-backlight.c
@@ -53,7 +53,7 @@ static int ktd253_backlight_stepdown(struct ktd253_backlight *ktd253)
 	 *
 	 * Architectures do not always support ndelay() and we will get a few us
 	 * instead. If we get to a critical time limit an interrupt has likely
-	 * occured in the low part of the loop and we need to restart from the
+	 * occurred in the low part of the loop and we need to restart from the
 	 * top so we have the backlight in a known state.
 	 */
 	u64 ns;
diff --git a/drivers/video/backlight/lm3533_bl.c b/drivers/video/backlight/lm3533_bl.c
index babfd3ceec8663..adcd6ac7cf2293 100644
--- a/drivers/video/backlight/lm3533_bl.c
+++ b/drivers/video/backlight/lm3533_bl.c
@@ -10,7 +10,9 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/backlight.h>
+#include <linux/regmap.h>
 #include <linux/slab.h>
 
 #include <linux/mfd/lm3533.h>
@@ -20,13 +22,21 @@
 #define LM3533_BL_MAX_BRIGHTNESS	255
 
 #define LM3533_REG_CTRLBANK_AB_BCONF	0x1a
+#define   CTRLBANK_AB_BCONF_ALS(n)	BIT(2 * (n))
+#define   CTRLBANK_AB_BCONF_MODE(n)	BIT(2 * (n) + 1)
 
 
 struct lm3533_bl {
-	struct lm3533 *lm3533;
+	struct regmap *regmap;
 	struct lm3533_ctrlbank cb;
 	struct backlight_device *bd;
 	int id;
+
+	u32 max_current;
+	u32 pwm;
+
+	bool have_als;
+	bool linear;
 };
 
 
@@ -45,7 +55,7 @@ static int lm3533_bl_update_status(struct backlight_device *bd)
 static int lm3533_bl_get_brightness(struct backlight_device *bd)
 {
 	struct lm3533_bl *bl = bl_get_data(bd);
-	u8 val;
+	u32 val;
 	int ret;
 
 	ret = lm3533_ctrlbank_get_brightness(&bl->cb, &val);
@@ -77,103 +87,75 @@ static ssize_t show_als_channel(struct device *dev,
 	return scnprintf(buf, PAGE_SIZE, "%u\n", channel);
 }
 
-static ssize_t show_als_en(struct device *dev,
-				struct device_attribute *attr, char *buf)
+static ssize_t show_als_en(struct device *dev, struct device_attribute *attr,
+			   char *buf)
 {
 	struct lm3533_bl *bl = dev_get_drvdata(dev);
 	int ctrlbank = lm3533_bl_get_ctrlbank_id(bl);
-	u8 val;
-	u8 mask;
-	bool enable;
 	int ret;
 
-	ret = lm3533_read(bl->lm3533, LM3533_REG_CTRLBANK_AB_BCONF, &val);
-	if (ret)
+	ret = regmap_test_bits(bl->regmap, LM3533_REG_CTRLBANK_AB_BCONF,
+			       CTRLBANK_AB_BCONF_ALS(ctrlbank));
+	if (ret < 0)
 		return ret;
 
-	mask = 1 << (2 * ctrlbank);
-	enable = val & mask;
-
-	return scnprintf(buf, PAGE_SIZE, "%d\n", enable);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", ret);
 }
 
-static ssize_t store_als_en(struct device *dev,
-					struct device_attribute *attr,
-					const char *buf, size_t len)
+static ssize_t store_als_en(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t len)
 {
 	struct lm3533_bl *bl = dev_get_drvdata(dev);
 	int ctrlbank = lm3533_bl_get_ctrlbank_id(bl);
 	int enable;
-	u8 val;
-	u8 mask;
 	int ret;
 
 	if (kstrtoint(buf, 0, &enable))
 		return -EINVAL;
 
-	mask = 1 << (2 * ctrlbank);
-
-	if (enable)
-		val = mask;
-	else
-		val = 0;
-
-	ret = lm3533_update(bl->lm3533, LM3533_REG_CTRLBANK_AB_BCONF, val,
-									mask);
+	ret = regmap_assign_bits(bl->regmap, LM3533_REG_CTRLBANK_AB_BCONF,
+				 CTRLBANK_AB_BCONF_ALS(ctrlbank), enable);
 	if (ret)
 		return ret;
 
 	return len;
 }
 
-static ssize_t show_linear(struct device *dev,
-				struct device_attribute *attr, char *buf)
+static ssize_t show_linear(struct device *dev, struct device_attribute *attr,
+			   char *buf)
 {
 	struct lm3533_bl *bl = dev_get_drvdata(dev);
-	u8 val;
-	u8 mask;
-	int linear;
+	int ctrlbank = lm3533_bl_get_ctrlbank_id(bl);
 	int ret;
 
-	ret = lm3533_read(bl->lm3533, LM3533_REG_CTRLBANK_AB_BCONF, &val);
-	if (ret)
+	ret = regmap_test_bits(bl->regmap, LM3533_REG_CTRLBANK_AB_BCONF,
+			       CTRLBANK_AB_BCONF_MODE(ctrlbank));
+	if (ret < 0)
 		return ret;
 
-	mask = 1 << (2 * lm3533_bl_get_ctrlbank_id(bl) + 1);
-
-	if (val & mask)
-		linear = 1;
-	else
-		linear = 0;
-
-	return scnprintf(buf, PAGE_SIZE, "%x\n", linear);
+	return scnprintf(buf, PAGE_SIZE, "%x\n", ret);
 }
 
-static ssize_t store_linear(struct device *dev,
-					struct device_attribute *attr,
-					const char *buf, size_t len)
+static ssize_t store_linear(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t len)
 {
 	struct lm3533_bl *bl = dev_get_drvdata(dev);
+	int ctrlbank = lm3533_bl_get_ctrlbank_id(bl);
+	struct backlight_device *bd = bl->bd;
 	unsigned long linear;
-	u8 mask;
-	u8 val;
 	int ret;
 
 	if (kstrtoul(buf, 0, &linear))
 		return -EINVAL;
 
-	mask = 1 << (2 * lm3533_bl_get_ctrlbank_id(bl) + 1);
-
-	if (linear)
-		val = mask;
-	else
-		val = 0;
-
-	ret = lm3533_update(bl->lm3533, LM3533_REG_CTRLBANK_AB_BCONF, val,
-									mask);
+	ret = regmap_assign_bits(bl->regmap, LM3533_REG_CTRLBANK_AB_BCONF,
+				 CTRLBANK_AB_BCONF_MODE(ctrlbank), linear);
 	if (ret)
 		return ret;
 
+	bd->props.scale = linear ? BACKLIGHT_SCALE_LINEAR :
+				   BACKLIGHT_SCALE_NON_LINEAR;
+
 	return len;
 }
 
@@ -182,7 +164,7 @@ static ssize_t show_pwm(struct device *dev,
 					char *buf)
 {
 	struct lm3533_bl *bl = dev_get_drvdata(dev);
-	u8 val;
+	u32 val;
 	int ret;
 
 	ret = lm3533_ctrlbank_get_pwm(&bl->cb, &val);
@@ -197,10 +179,10 @@ static ssize_t store_pwm(struct device *dev,
 					const char *buf, size_t len)
 {
 	struct lm3533_bl *bl = dev_get_drvdata(dev);
-	u8 val;
+	u32 val;
 	int ret;
 
-	if (kstrtou8(buf, 0, &val))
+	if (kstrtou32(buf, 0, &val))
 		return -EINVAL;
 
 	ret = lm3533_ctrlbank_set_pwm(&bl->cb, val);
@@ -234,7 +216,7 @@ static umode_t lm3533_bl_attr_is_visible(struct kobject *kobj,
 
 	if (attr == &dev_attr_als_channel.attr ||
 					attr == &dev_attr_als_en.attr) {
-		if (!bl->lm3533->have_als)
+		if (!bl->have_als)
 			mode = 0;
 	}
 
@@ -246,25 +228,36 @@ static struct attribute_group lm3533_bl_attribute_group = {
 	.attrs		= lm3533_bl_attributes
 };
 
-static int lm3533_bl_setup(struct lm3533_bl *bl,
-					struct lm3533_bl_platform_data *pdata)
+static const struct attribute_group *lm3533_bl_attribute_groups[] = {
+	&lm3533_bl_attribute_group,
+	NULL,
+};
+
+static int lm3533_bl_setup(struct lm3533_bl *bl)
 {
+	int ctrlbank = lm3533_bl_get_ctrlbank_id(bl);
 	int ret;
 
-	ret = lm3533_ctrlbank_set_max_current(&bl->cb, pdata->max_current);
+	ret = regmap_assign_bits(bl->regmap, LM3533_REG_CTRLBANK_AB_BCONF,
+				 CTRLBANK_AB_BCONF_MODE(ctrlbank), bl->linear);
+	if (ret)
+		return ret;
+
+	ret = lm3533_ctrlbank_set_max_current(&bl->cb, bl->max_current);
 	if (ret)
 		return ret;
 
-	return lm3533_ctrlbank_set_pwm(&bl->cb, pdata->pwm);
+	return lm3533_ctrlbank_set_pwm(&bl->cb, bl->pwm);
 }
 
 static int lm3533_bl_probe(struct platform_device *pdev)
 {
 	struct lm3533 *lm3533;
-	struct lm3533_bl_platform_data *pdata;
 	struct lm3533_bl *bl;
 	struct backlight_device *bd;
 	struct backlight_properties props;
+	char *name = NULL;
+	u32 default_brightness = LM3533_BL_MAX_BRIGHTNESS;
 	int ret;
 
 	dev_dbg(&pdev->dev, "%s\n", __func__);
@@ -273,12 +266,6 @@ static int lm3533_bl_probe(struct platform_device *pdev)
 	if (!lm3533)
 		return -EINVAL;
 
-	pdata = dev_get_platdata(&pdev->dev);
-	if (!pdata) {
-		dev_err(&pdev->dev, "no platform data\n");
-		return -EINVAL;
-	}
-
 	if (pdev->id < 0 || pdev->id >= LM3533_HVCTRLBANK_COUNT) {
 		dev_err(&pdev->dev, "illegal backlight id %d\n", pdev->id);
 		return -EINVAL;
@@ -288,20 +275,34 @@ static int lm3533_bl_probe(struct platform_device *pdev)
 	if (!bl)
 		return -ENOMEM;
 
-	bl->lm3533 = lm3533;
+	bl->regmap = lm3533->regmap;
+	bl->have_als = lm3533->have_als;
 	bl->id = pdev->id;
 
-	bl->cb.lm3533 = lm3533;
+	bl->cb.regmap = lm3533->regmap;
 	bl->cb.id = lm3533_bl_get_ctrlbank_id(bl);
 	bl->cb.dev = NULL;			/* until registered */
 
+	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-%d",
+			      pdev->name, pdev->id);
+	if (!name)
+		return -ENOMEM;
+
+	device_property_read_u32(&pdev->dev, "default-brightness",
+				 &default_brightness);
+
 	memset(&props, 0, sizeof(props));
 	props.type = BACKLIGHT_RAW;
 	props.max_brightness = LM3533_BL_MAX_BRIGHTNESS;
-	props.brightness = pdata->default_brightness;
-	bd = devm_backlight_device_register(&pdev->dev, pdata->name,
-					pdev->dev.parent, bl, &lm3533_bl_ops,
-					&props);
+	props.brightness = default_brightness;
+
+	bl->linear = device_property_read_bool(&pdev->dev,
+					       "ti,linear-mapping-mode");
+	props.scale = bl->linear ? BACKLIGHT_SCALE_LINEAR :
+				   BACKLIGHT_SCALE_NON_LINEAR;
+
+	bd = devm_backlight_device_register(&pdev->dev, name, &pdev->dev,
+					    bl, &lm3533_bl_ops, &props);
 	if (IS_ERR(bd)) {
 		dev_err(&pdev->dev, "failed to register backlight device\n");
 		return PTR_ERR(bd);
@@ -312,28 +313,24 @@ static int lm3533_bl_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, bl);
 
-	ret = sysfs_create_group(&bd->dev.kobj, &lm3533_bl_attribute_group);
-	if (ret < 0) {
-		dev_err(&pdev->dev, "failed to create sysfs attributes\n");
-		return ret;
-	}
+	device_property_read_u32(&pdev->dev, "led-max-microamp",
+				 &bl->max_current);
+	bl->max_current = clamp(bl->max_current, LM3533_MAX_CURRENT_MIN,
+				LM3533_MAX_CURRENT_MAX);
 
-	backlight_update_status(bd);
+	device_property_read_u32(&pdev->dev, "ti,pwm-config-mask", &bl->pwm);
 
-	ret = lm3533_bl_setup(bl, pdata);
+	ret = lm3533_bl_setup(bl);
 	if (ret)
-		goto err_sysfs_remove;
+		return ret;
+
+	backlight_update_status(bd);
 
 	ret = lm3533_ctrlbank_enable(&bl->cb);
 	if (ret)
-		goto err_sysfs_remove;
+		return ret;
 
 	return 0;
-
-err_sysfs_remove:
-	sysfs_remove_group(&bd->dev.kobj, &lm3533_bl_attribute_group);
-
-	return ret;
 }
 
 static void lm3533_bl_remove(struct platform_device *pdev)
@@ -347,7 +344,6 @@ static void lm3533_bl_remove(struct platform_device *pdev)
 	bd->props.brightness = 0;
 
 	lm3533_ctrlbank_disable(&bl->cb);
-	sysfs_remove_group(&bd->dev.kobj, &lm3533_bl_attribute_group);
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -381,10 +377,18 @@ static void lm3533_bl_shutdown(struct platform_device *pdev)
 	lm3533_ctrlbank_disable(&bl->cb);
 }
 
+static const struct of_device_id lm3533_bl_match_table[] = {
+	{ .compatible = "ti,lm3533-backlight" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lm3533_bl_match_table);
+
 static struct platform_driver lm3533_bl_driver = {
 	.driver = {
 		.name	= "lm3533-backlight",
 		.pm	= &lm3533_bl_pm_ops,
+		.dev_groups = lm3533_bl_attribute_groups,
+		.of_match_table = lm3533_bl_match_table,
 	},
 	.probe		= lm3533_bl_probe,
 	.remove		= lm3533_bl_remove,
diff --git a/drivers/video/fbdev/acornfb.c b/drivers/video/fbdev/acornfb.c
index f0600f6ca2548e..30781679c90af2 100644
--- a/drivers/video/fbdev/acornfb.c
+++ b/drivers/video/fbdev/acornfb.c
@@ -1,7 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- *  linux/drivers/video/acornfb.c
- *
  *  Copyright (C) 1998-2001 Russell King
  *
  * Frame buffer code for Acorn platforms
@@ -94,7 +92,6 @@ static struct vidc_timing current_vidc;
 
 extern unsigned int vram_size;	/* set by setup.c */
 
-#ifdef HAS_VIDC20
 #include <mach/acornfb.h>
 
 #define MAX_SIZE	(2*1024*1024)
@@ -306,7 +303,6 @@ acornfb_setcolreg(u_int regno, u_int red, u_int green, u_int blue,
 
 	return 0;
 }
-#endif
 
 /*
  * Before selecting the timing parameters, adjust
@@ -390,7 +386,6 @@ acornfb_adjust_timing(struct fb_info *info, struct fb_var_screeninfo *var, u_int
 	/* hsync_len must be even */
 	var->hsync_len = (var->hsync_len + 1) & ~1;
 
-#if defined(HAS_VIDC20)
 	/* left_margin must be even */
 	if (var->left_margin & 1) {
 		var->left_margin += 1;
@@ -400,7 +395,6 @@ acornfb_adjust_timing(struct fb_info *info, struct fb_var_screeninfo *var, u_int
 	/* right_margin must be even */
 	if (var->right_margin & 1)
 		var->right_margin += 1;
-#endif
 
 	if (var->vsync_len < 1)
 		var->vsync_len = 1;
@@ -436,11 +430,7 @@ acornfb_update_dma(struct fb_info *info, struct fb_var_screeninfo *var)
 {
 	u_int off = var->yoffset * info->fix.line_length;
 
-#if defined(HAS_MEMC)
-	memc_write(VDMA_INIT, off >> 2);
-#elif defined(HAS_IOMD)
 	iomd_writel(info->fix.smem_start + off, IOMD_VIDINIT);
-#endif
 }
 
 static int
@@ -469,7 +459,6 @@ acornfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 		var->transp.length = 0;
 		break;
 
-#ifdef HAS_VIDC20
 	case 16:
 		var->red.offset    = 0;
 		var->red.length    = 5;
@@ -491,7 +480,6 @@ acornfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 		var->transp.offset = 24;
 		var->transp.length = 4;
 		break;
-#endif
 	default:
 		return -EINVAL;
 	}
@@ -519,6 +507,9 @@ acornfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 
 static int acornfb_set_par(struct fb_info *info)
 {
+	unsigned long start, size;
+	u_int control;
+
 	switch (info->var.bits_per_pixel) {
 	case 1:
 		current_par.palette_size = 2;
@@ -536,7 +527,6 @@ static int acornfb_set_par(struct fb_info *info)
 		current_par.palette_size = VIDC_PALETTE_SIZE;
 		info->fix.visual = FB_VISUAL_PSEUDOCOLOR;
 		break;
-#ifdef HAS_VIDC20
 	case 16:
 		current_par.palette_size = 32;
 		info->fix.visual = FB_VISUAL_DIRECTCOLOR;
@@ -545,41 +535,26 @@ static int acornfb_set_par(struct fb_info *info)
 		current_par.palette_size = VIDC_PALETTE_SIZE;
 		info->fix.visual = FB_VISUAL_DIRECTCOLOR;
 		break;
-#endif
 	default:
 		BUG();
 	}
 
 	info->fix.line_length	= (info->var.xres * info->var.bits_per_pixel) / 8;
 
-#if defined(HAS_MEMC)
-	{
-		unsigned long size = info->fix.smem_len - VDMA_XFERSIZE;
+	start = info->fix.smem_start;
+	size  = current_par.screen_end;
 
-		memc_write(VDMA_START, 0);
-		memc_write(VDMA_END, size >> 2);
+	if (current_par.using_vram) {
+		size -= current_par.vram_half_sam;
+		control = DMA_CR_E | (current_par.vram_half_sam / 256);
+	} else {
+		size -= 16;
+		control = DMA_CR_E | DMA_CR_D | 16;
 	}
-#elif defined(HAS_IOMD)
-	{
-		unsigned long start, size;
-		u_int control;
-
-		start = info->fix.smem_start;
-		size  = current_par.screen_end;
-
-		if (current_par.using_vram) {
-			size -= current_par.vram_half_sam;
-			control = DMA_CR_E | (current_par.vram_half_sam / 256);
-		} else {
-			size -= 16;
-			control = DMA_CR_E | DMA_CR_D | 16;
-		}
 
-		iomd_writel(start,   IOMD_VIDSTART);
-		iomd_writel(size,    IOMD_VIDEND);
-		iomd_writel(control, IOMD_VIDCR);
-	}
-#endif
+	iomd_writel(start,   IOMD_VIDSTART);
+	iomd_writel(size,    IOMD_VIDEND);
+	iomd_writel(control, IOMD_VIDCR);
 
 	acornfb_update_dma(info, &info->var);
 	acornfb_set_timing(info);
@@ -709,10 +684,8 @@ static void acornfb_init_fbinfo(void)
 	 */
 	memset(&fb_info.var, 0, sizeof(fb_info.var));
 
-#if defined(HAS_VIDC20)
 	fb_info.var.red.length	   = 8;
 	fb_info.var.transp.length  = 4;
-#endif
 	fb_info.var.green	   = fb_info.var.red;
 	fb_info.var.blue	   = fb_info.var.red;
 	fb_info.var.nonstd	   = 0;
@@ -991,7 +964,6 @@ static int acornfb_probe(struct platform_device *dev)
 
 	size = PAGE_ALIGN(size);
 
-#if defined(HAS_VIDC20)
 	if (!current_par.using_vram) {
 		dma_addr_t handle;
 		void *base;
@@ -1011,7 +983,6 @@ static int acornfb_probe(struct platform_device *dev)
 		fb_info.screen_base = base;
 		fb_info.fix.smem_start = handle;
 	}
-#endif
 	fb_info.fix.smem_len = size;
 	current_par.palette_size   = VIDC_PALETTE_SIZE;
 
diff --git a/drivers/video/fbdev/acornfb.h b/drivers/video/fbdev/acornfb.h
index f8df4ecb4fd7b2..c5d0fa04adc884 100644
--- a/drivers/video/fbdev/acornfb.h
+++ b/drivers/video/fbdev/acornfb.h
@@ -1,16 +1,14 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- *  linux/drivers/video/acornfb.h
- *
  *  Copyright (C) 1998,1999 Russell King
  *
  *  Frame buffer code for Acorn platforms
  */
-#if defined(HAS_VIDC20)
+
 #include <asm/hardware/iomd.h>
+
 #define VIDC_PALETTE_SIZE	256
 #define VIDC_NAME		"VIDC20"
-#endif
 
 #define EXTEND8(x) ((x)|(x)<<8)
 #define EXTEND4(x) ((x)|(x)<<4|(x)<<8|(x)<<12)
@@ -94,7 +92,6 @@ struct modex_params {
 	const struct modey_params *modey;
 };
 
-#ifdef HAS_VIDC20
 /*
  * VIDC20 registers
  */
@@ -162,5 +159,3 @@ struct modex_params {
 #define VIDC20_DCTL_VRAM_PXCLK	(1 << 18)
 #define VIDC20_DCTL_VRAM_PXCLK2	(2 << 18)
 #define VIDC20_DCTL_VRAM_PXCLK4	(3 << 18)
-
-#endif
diff --git a/drivers/video/fbdev/atafb.c b/drivers/video/fbdev/atafb.c
index c3011b61a94b90..c32576cb172029 100644
--- a/drivers/video/fbdev/atafb.c
+++ b/drivers/video/fbdev/atafb.c
@@ -1089,7 +1089,7 @@ static int falcon_decode_var(struct fb_info *info, struct fb_var_screeninfo *var
 			/* Choose largest pixelclock <= wanted clock */
 			int i;
 			unsigned long pcl = ULONG_MAX;
-			pclock = 0;
+			pclock = NULL;
 			for (i = 1; i <= 4; i *= 2) {
 				if (f25.t * i >= var->pixclock &&
 				    f25.t * i < pcl) {
@@ -2454,7 +2454,7 @@ static void svblit_fillrect(struct fb_info *info,
 		((u32 *)info->pseudo_palette)[rect->color] : rect->color;
 
 	/* draw the first line with the CPU ... */
-	line = (u8 *)info->screen_base + rect->dy * pitch +
+	line = info->screen_buffer + rect->dy * pitch +
 	       rect->dx * bytespp;
 	switch (bytespp) {
 	case 1:
@@ -2818,29 +2818,6 @@ static void atafb_imageblit(struct fb_info *info, const struct fb_image *image)
 	}
 }
 
-static int
-atafb_ioctl(struct fb_info *info, unsigned int cmd, unsigned long arg)
-{
-	switch (cmd) {
-#ifdef FBCMD_GET_CURRENTPAR
-	case FBCMD_GET_CURRENTPAR:
-		if (copy_to_user((void *)arg, &current_par,
-				 sizeof(struct atafb_par)))
-			return -EFAULT;
-		return 0;
-#endif
-#ifdef FBCMD_SET_CURRENTPAR
-	case FBCMD_SET_CURRENTPAR:
-		if (copy_from_user(&current_par, (void *)arg,
-				   sizeof(struct atafb_par)))
-			return -EFAULT;
-		ata_set_par(&current_par);
-		return 0;
-#endif
-	}
-	return -EINVAL;
-}
-
 /* (un)blank/poweroff
  * 0 = unblank
  * 1 = blank
@@ -2922,7 +2899,6 @@ static struct fb_ops atafb_ops = {
 	.fb_fillrect	= atafb_fillrect,
 	.fb_copyarea	= atafb_copyarea,
 	.fb_imageblit	= atafb_imageblit,
-	.fb_ioctl =	atafb_ioctl,
 	__FB_DEFAULT_IOMEM_OPS_MMAP,
 };
 
diff --git a/drivers/video/fbdev/aty/aty128fb.c b/drivers/video/fbdev/aty/aty128fb.c
index c349a1711ff56a..6f71c3c6307e8e 100644
--- a/drivers/video/fbdev/aty/aty128fb.c
+++ b/drivers/video/fbdev/aty/aty128fb.c
@@ -753,7 +753,7 @@ static u32 depth_to_dst(u32 depth)
 }
 
 /*
- * PLL informations retreival
+ * PLL information retreival
  */
 
 
diff --git a/drivers/video/fbdev/aty/atyfb_base.c b/drivers/video/fbdev/aty/atyfb_base.c
index 97cc8b6a936149..b1f33642e7226c 100644
--- a/drivers/video/fbdev/aty/atyfb_base.c
+++ b/drivers/video/fbdev/aty/atyfb_base.c
@@ -2966,7 +2966,7 @@ static int atyfb_setup_sparc(struct pci_dev *pdev, struct fb_info *info,
 
 	/*
 	 * Figure mmap addresses from PCI config space.
-	 * Split Framebuffer in big- and little-endian halfs.
+	 * Split Framebuffer in big- and little-endian halves.
 	 */
 	for (i = 0; i < 6 && pdev->resource[i].start; i++)
 		/* nothing */ ;
@@ -3387,7 +3387,7 @@ static int init_from_bios(struct atyfb_par *par)
 
 		PRINTKI("Mach64 BIOS is located at %x, mapped at %x.\n", rom_addr, bios_base);
 
-		/* check for frequncy table */
+		/* check for frequency table */
 		bios_ptr = (u8*)bios_base;
 		rom_table_offset = (u16)(bios_ptr[0x48] | (bios_ptr[0x49] << 8));
 		freq_table_offset = bios_ptr[rom_table_offset + 16] | (bios_ptr[rom_table_offset + 17] << 8);
diff --git a/drivers/video/fbdev/aty/mach64_cursor.c b/drivers/video/fbdev/aty/mach64_cursor.c
index e826cb7dd55daa..60abb96663958a 100644
--- a/drivers/video/fbdev/aty/mach64_cursor.c
+++ b/drivers/video/fbdev/aty/mach64_cursor.c
@@ -43,7 +43,7 @@
  *
  *
  * The Screen position of the top left corner of the displayed
- * cursor is specificed by CURS_HORZ_VERT_POSN. Care must be taken
+ * cursor is specified by CURS_HORZ_VERT_POSN. Care must be taken
  * when the cursor hot spot is not the top left corner and the
  * physical cursor position becomes negative. It will be displayed
  * if either the horizontal or vertical cursor position is negative
diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c
index b6b058cee75169..387bc5f9676517 100644
--- a/drivers/video/fbdev/aty/radeon_base.c
+++ b/drivers/video/fbdev/aty/radeon_base.c
@@ -779,7 +779,7 @@ static void radeon_get_pllinfo(struct radeonfb_info *rinfo)
 #endif /* CONFIG_PPC || CONFIG_SPARC */
 
 	/*
-	 * Check out if we have an X86 which gave us some PLL informations
+	 * Check out if we have an X86 which gave us some PLL information
 	 * and if yes, retrieve them
 	 */
 	if (!force_measure_pll && rinfo->bios_seg) {
@@ -2407,7 +2407,7 @@ static int radeonfb_pci_register(struct pci_dev *pdev,
 	if (rinfo->bios_seg == NULL && rinfo->is_mobility)
 		radeon_map_ROM(rinfo, pdev);
 
-	/* Get informations about the board's PLL */
+	/* Get information about the board's PLL */
 	radeon_get_pllinfo(rinfo);
 
 #ifdef CONFIG_FB_RADEON_I2C
diff --git a/drivers/video/fbdev/aty/radeon_monitor.c b/drivers/video/fbdev/aty/radeon_monitor.c
index 621d13a1a1d98c..775888c43ef047 100644
--- a/drivers/video/fbdev/aty/radeon_monitor.c
+++ b/drivers/video/fbdev/aty/radeon_monitor.c
@@ -58,7 +58,7 @@ static char *radeon_get_mon_name(int type)
 
 #if defined(CONFIG_PPC) || defined(CONFIG_SPARC)
 /*
- * Try to find monitor informations & EDID data out of the Open Firmware
+ * Try to find monitor information & EDID data out of the Open Firmware
  * device-tree. This also contains some "hacks" to work around a few machine
  * models with broken OF probing by hard-coding known EDIDs for some Mac
  * laptops internal LVDS panel. (XXX: not done yet)
@@ -248,7 +248,7 @@ static int radeon_get_panel_info_BIOS(struct radeonfb_info *rinfo)
 	return 0;
 }
 
-/* Try to extract the connector informations from the BIOS. This
+/* Try to extract the connector information from the BIOS. This
  * doesn't quite work yet, but it's output is still useful for
  * debugging
  */
diff --git a/drivers/video/fbdev/aty/radeonfb.h b/drivers/video/fbdev/aty/radeonfb.h
index 91d81b5762313e..0781caea2c7155 100644
--- a/drivers/video/fbdev/aty/radeonfb.h
+++ b/drivers/video/fbdev/aty/radeonfb.h
@@ -148,7 +148,7 @@ struct pll_info {
  * This structure contains the various registers manipulated by this
  * driver for setting or restoring a mode. It's mostly copied from
  * XFree's RADEONSaveRec structure. A few chip settings might still be
- * tweaked without beeing reflected or saved in these registers though
+ * tweaked without being reflected or saved in these registers though
  */
 struct radeon_regs {
 	/* Common registers */
diff --git a/drivers/video/fbdev/au1100fb.c b/drivers/video/fbdev/au1100fb.c
index 39f86ffb1ae3d3..01c836cb5205f4 100644
--- a/drivers/video/fbdev/au1100fb.c
+++ b/drivers/video/fbdev/au1100fb.c
@@ -528,7 +528,7 @@ static int au1100fb_setmode(struct au1100fb_device *fbdev)
 	if (!info->var.rotate || (info->var.rotate == 180)) {
 		words *= info->var.yres_virtual;
 		if (info->var.rotate /* 180 */) {
-			words -= (words % 8); /* should be divisable by 8 */
+			words -= (words % 8); /* should be divisible by 8 */
 		}
 	}
 	fbdev->regs->lcd_words = LCD_WRD_WRDS_N(words);
diff --git a/drivers/video/fbdev/core/Kconfig b/drivers/video/fbdev/core/Kconfig
index 8d1993e0b59175..14860a1ae6bc62 100644
--- a/drivers/video/fbdev/core/Kconfig
+++ b/drivers/video/fbdev/core/Kconfig
@@ -155,12 +155,6 @@ config FB_IOMEM_HELPERS
 	select FB_CFB_IMAGEBLIT
 	select FB_IOMEM_FOPS
 
-config FB_IOMEM_HELPERS_DEFERRED
-	bool
-	depends on FB_CORE
-	select FB_DEFERRED_IO
-	select FB_IOMEM_HELPERS
-
 config FB_SYSMEM_HELPERS
 	bool
 	depends on FB_CORE
diff --git a/drivers/video/fbdev/core/fb_defio.c b/drivers/video/fbdev/core/fb_defio.c
index fd00b86e1ae602..fb359ecc396619 100644
--- a/drivers/video/fbdev/core/fb_defio.c
+++ b/drivers/video/fbdev/core/fb_defio.c
@@ -366,13 +366,13 @@ int fb_deferred_io_mmap(struct fb_info *info, struct vm_area_struct *vma)
 {
 	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 
+	if (WARN_ON_ONCE(!(info->flags & FBINFO_VIRTFB)))
+		return -EINVAL;
 	if (!try_module_get(THIS_MODULE))
 		return -EINVAL;
 
 	vma->vm_ops = &fb_deferred_io_vm_ops;
-	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
-	if (!(info->flags & FBINFO_VIRTFB))
-		vm_flags_set(vma, VM_IO);
+	vm_flags_set(vma, VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP);
 	vma->vm_private_data = info->fbdefio_state;
 
 	fb_deferred_io_state_get(info->fbdefio_state); /* released in vma->vm_ops->close() */
diff --git a/drivers/video/fbdev/core/fb_procfs.c b/drivers/video/fbdev/core/fb_procfs.c
index 59641142f8aadc..3003fa62d342a3 100644
--- a/drivers/video/fbdev/core/fb_procfs.c
+++ b/drivers/video/fbdev/core/fb_procfs.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 #include <linux/proc_fs.h>
+#include <linux/seq_file.h>
 
 #include "fb_internal.h"
 
diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c
index 01715873ea49b6..615fa596b7cced 100644
--- a/drivers/video/fbdev/core/fbcon.c
+++ b/drivers/video/fbdev/core/fbcon.c
@@ -78,7 +78,6 @@
 #include <linux/interrupt.h>
 #include <linux/crc32.h> /* For counting font checksums */
 #include <linux/uaccess.h>
-#include <linux/vga_switcheroo.h>
 #include <asm/irq.h>
 
 #include "fbcon.h"
@@ -2883,9 +2882,6 @@ void fbcon_fb_unregistered(struct fb_info *info)
 
 	console_lock();
 
-	if (info->device && dev_is_pci(info->device))
-		vga_switcheroo_client_fb_set(to_pci_dev(info->device), NULL);
-
 	fbcon_registered_fb[info->node] = NULL;
 	fbcon_num_registered_fb--;
 
@@ -3019,10 +3015,6 @@ static int do_fb_registered(struct fb_info *info)
 		}
 	}
 
-	/* Set the fb info for vga_switcheroo clients. Does nothing otherwise. */
-	if (info->device && dev_is_pci(info->device))
-		vga_switcheroo_client_fb_set(to_pci_dev(info->device), info);
-
 	return ret;
 }
 
diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c
index ad8dec7807c3a5..e1c7648540041e 100644
--- a/drivers/video/fbdev/efifb.c
+++ b/drivers/video/fbdev/efifb.c
@@ -97,7 +97,7 @@ static int efifb_setcolreg(unsigned regno, unsigned red, unsigned green,
 }
 
 /*
- * If fbcon deffered console takeover is configured, the intent is for the
+ * If fbcon deferred console takeover is configured, the intent is for the
  * framebuffer to show the boot graphics (e.g. vendor logo) until there is some
  * (error) message to display. But the boot graphics may have been destroyed by
  * e.g. option ROM output, detect this and restore the boot graphics.
@@ -421,7 +421,7 @@ static int efifb_probe(struct platform_device *dev)
 	size_vmode = efifb_defined.yres * efifb_fix.line_length;
 
 	/*   size_total -- all video memory we have. Used for
-	 *                 entries, ressource allocation and bounds
+	 *                 entries, resource allocation and bounds
 	 *                 checking. */
 	size_total = si->lfb_size;
 	if (size_total < size_vmode)
diff --git a/drivers/video/fbdev/fsl-diu-fb.c b/drivers/video/fbdev/fsl-diu-fb.c
index b71d15794ce8b8..d7ed007915c189 100644
--- a/drivers/video/fbdev/fsl-diu-fb.c
+++ b/drivers/video/fbdev/fsl-diu-fb.c
@@ -1690,7 +1690,7 @@ static int fsl_diu_probe(struct platform_device *pdev)
 	int ret;
 
 	data = dmam_alloc_coherent(&pdev->dev, sizeof(struct fsl_diu_data),
-				   &dma_addr, GFP_DMA | __GFP_ZERO);
+				   &dma_addr, __GFP_ZERO);
 	if (!data)
 		return -ENOMEM;
 	data->dma_addr = dma_addr;
diff --git a/drivers/video/fbdev/grvga.c b/drivers/video/fbdev/grvga.c
index a6594bcd74e827..771d9491b49ed4 100644
--- a/drivers/video/fbdev/grvga.c
+++ b/drivers/video/fbdev/grvga.c
@@ -432,7 +432,7 @@ static int grvga_probe(struct platform_device *dev)
 			retval = -ENOMEM;
 			goto dealloc_cmap;
 		}
-	} else {	/* Allocate frambuffer memory */
+	} else {	/* Allocate framebuffer memory */
 
 		unsigned long page;
 
diff --git a/drivers/video/fbdev/gxt4500.c b/drivers/video/fbdev/gxt4500.c
index 1ee0a1efa18e0c..906b8a0dea774d 100644
--- a/drivers/video/fbdev/gxt4500.c
+++ b/drivers/video/fbdev/gxt4500.c
@@ -719,6 +719,7 @@ static int gxt4500_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  err_free_cmap:
 	fb_dealloc_cmap(&info->cmap);
  err_unmap_all:
+	arch_phys_wc_del(par->wc_cookie);
 	iounmap(info->screen_base);
  err_unmap_regs:
 	iounmap(par->regs);
diff --git a/drivers/video/fbdev/kyro/STG4000OverlayDevice.c b/drivers/video/fbdev/kyro/STG4000OverlayDevice.c
index 9fde0e3b69ec4c..cb6fa0991fca24 100644
--- a/drivers/video/fbdev/kyro/STG4000OverlayDevice.c
+++ b/drivers/video/fbdev/kyro/STG4000OverlayDevice.c
@@ -60,7 +60,7 @@ typedef struct _OVRL_SRC_DEST {
 	u32 ulDstX2;
 	u32 ulDstY2;
 
-	/*clipped pixel pos of source data within buffer thses need to be 128 bit word aligned */
+	/* clipped pixel pos of source data within buffer these need to be 128 bit word aligned */
 	u32 ulSrcX1;
 	u32 ulSrcY1;
 	u32 ulSrcX2;
diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c
index c2373898882290..94d6af54e6c461 100644
--- a/drivers/video/fbdev/kyro/fbdev.c
+++ b/drivers/video/fbdev/kyro/fbdev.c
@@ -765,7 +765,7 @@ static int kyrofb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	fb_memset_io(info->screen_base, 0, size);
 
 	if (register_framebuffer(info) < 0)
-		goto out_free_fb;
+		goto out_free_wc;
 
 	fb_info(info, "%s frame buffer device, at %dx%d@%d using %ldk/%ldk of VRAM\n",
 		info->fix.id,
@@ -776,6 +776,8 @@ static int kyrofb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	return 0;
 
+out_free_wc:
+	arch_phys_wc_del(currentpar->wc_cookie);
 out_free_fb:
 	framebuffer_release(info);
 
diff --git a/drivers/video/fbdev/maxinefb.c b/drivers/video/fbdev/maxinefb.c
index 67d4bd88104149..88aec13a98fbc3 100644
--- a/drivers/video/fbdev/maxinefb.c
+++ b/drivers/video/fbdev/maxinefb.c
@@ -98,7 +98,7 @@ static int maxinefb_setcolreg(unsigned regno, unsigned red, unsigned green,
 		return 1;
 
 	red   >>= 8;    /* The cmap fields are 16 bits    */
-	green >>= 8;    /* wide, but the harware colormap */
+	green >>= 8;    /* wide, but the hardware colormap */
 	blue  >>= 8;    /* registers are only 8 bits wide */
 
 	hw_colorvalue = (blue << 16) + (green << 8) + (red);
diff --git a/drivers/video/fbdev/mmp/core.c b/drivers/video/fbdev/mmp/core.c
index 3c35b0a3ea0983..9d4fda2fc83fa4 100644
--- a/drivers/video/fbdev/mmp/core.c
+++ b/drivers/video/fbdev/mmp/core.c
@@ -65,7 +65,7 @@ static DEFINE_MUTEX(disp_lock);
  * @p: panel to be registered
  *
  * this function provides interface for panel drivers to register panel
- * to panel_list and connect to path which matchs panel->plat_path_name.
+ * to panel_list and connect to path which matches panel->plat_path_name.
  * no error returns when no matching path is found as path register after
  * panel register is permitted.
  */
diff --git a/drivers/video/fbdev/mmp/hw/mmp_ctrl.c b/drivers/video/fbdev/mmp/hw/mmp_ctrl.c
index 2c457da67a9f16..3db608e5aef19a 100644
--- a/drivers/video/fbdev/mmp/hw/mmp_ctrl.c
+++ b/drivers/video/fbdev/mmp/hw/mmp_ctrl.c
@@ -325,7 +325,7 @@ static void ctrl_set_default(struct mmphw_ctrl *ctrl)
 	u32 tmp, irq_mask;
 
 	/*
-	 * LCD Global control(LCD_TOP_CTRL) should be configed before
+	 * LCD Global control(LCD_TOP_CTRL) should be configured before
 	 * any other LCD registers read/write, or there maybe issues.
 	 */
 	tmp = readl_relaxed(ctrl->reg_base + LCD_TOP_CTRL);
@@ -521,7 +521,7 @@ static int mmphw_probe(struct platform_device *pdev)
 	/* init global regs */
 	ctrl_set_default(ctrl);
 
-	/* init pathes from machine info and register them */
+	/* init paths from machine info and register them */
 	for (i = 0; i < ctrl->path_num; i++) {
 		/* get from config and machine info */
 		path_plat = &ctrl->path_plats[i];
diff --git a/drivers/video/fbdev/mmp/hw/mmp_ctrl.h b/drivers/video/fbdev/mmp/hw/mmp_ctrl.h
index 719b99a9bc7762..b49d870df44fd1 100644
--- a/drivers/video/fbdev/mmp/hw/mmp_ctrl.h
+++ b/drivers/video/fbdev/mmp/hw/mmp_ctrl.h
@@ -1404,7 +1404,7 @@ struct mmphw_ctrl {
 	int status;
 	struct mutex access_ok;
 
-	/*pathes*/
+	/* paths */
 	int path_num;
 	struct mmphw_path_plat path_plats[] __counted_by(path_num);
 };
@@ -1434,7 +1434,7 @@ static inline void __iomem *ctrl_regs(struct mmp_path *path)
 	return path_to_ctrl(path)->reg_base;
 }
 
-/* path regs, for regs symmetrical for both pathes */
+/* path regs, for regs symmetrical for both paths */
 static inline struct lcd_regs *path_regs(struct mmp_path *path)
 {
 	if (path->id == PATH_PN)
diff --git a/drivers/video/fbdev/mmp/hw/mmp_spi.c b/drivers/video/fbdev/mmp/hw/mmp_spi.c
index 3f253f4271ac9c..8e6e8be441d239 100644
--- a/drivers/video/fbdev/mmp/hw/mmp_spi.c
+++ b/drivers/video/fbdev/mmp/hw/mmp_spi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * linux/drivers/video/mmp/hw/mmp_spi.c
- * using the spi in LCD controler for commands send
+ * using the spi in LCD controller for commands send
  *
  * Copyright (C) 2012 Marvell Technology Group Ltd.
  * Authors:  Guoqing Li <ligq@marvell.com>
diff --git a/drivers/video/fbdev/nvidia/nvidia.c b/drivers/video/fbdev/nvidia/nvidia.c
index 4ef7a5ba43cc44..395f5d0455fa4f 100644
--- a/drivers/video/fbdev/nvidia/nvidia.c
+++ b/drivers/video/fbdev/nvidia/nvidia.c
@@ -1423,6 +1423,7 @@ static int nvidiafb_probe(struct pci_dev *pd, const struct pci_device_id *ent)
 	return 0;
 
 err_out_iounmap_fb:
+	arch_phys_wc_del(par->wc_cookie);
 	fb_destroy_modelist(&info->modelist);
 err_out_free_base1:
 	fb_destroy_modedb(info->monspecs.modedb);
diff --git a/drivers/video/fbdev/ocfb.c b/drivers/video/fbdev/ocfb.c
index 893888260c21c4..c16e907e57101a 100644
--- a/drivers/video/fbdev/ocfb.c
+++ b/drivers/video/fbdev/ocfb.c
@@ -115,7 +115,7 @@ static int ocfb_setupfb(struct ocfb_dev *fbdev)
 	fbdev->little_endian = 0;
 	ocfb_writereg(fbdev, OCFB_VBARA, fbdev->fb_phys);
 
-	/* Detect endianess */
+	/* Detect endianness */
 	if (ocfb_readreg(fbdev, OCFB_VBARA) != fbdev->fb_phys) {
 		fbdev->little_endian = 1;
 		ocfb_writereg(fbdev, OCFB_VBARA, fbdev->fb_phys);
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
index 1dc70c96d81326..cbe3cc5451317f 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
@@ -2119,7 +2119,7 @@ static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk,
 
 	/*
 	 * Pixel data should be prepared before visible display point starts.
-	 * So, atleast DS-2 lines must have already been fetched by DISPC
+	 * So, at least DS-2 lines must have already been fetched by DISPC
 	 * during nonactive - pos_x period.
 	 */
 	val = div_u64((u64)(nonactive - pos_x) * lclk, pclk);
@@ -2130,7 +2130,7 @@ static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk,
 
 	/*
 	 * All lines need to be refilled during the nonactive period of which
-	 * only one line can be loaded during the active period. So, atleast
+	 * only one line can be loaded during the active period. So, at least
 	 * DS - 1 lines should be loaded during nonactive period.
 	 */
 	val =  div_u64((u64)nonactive * lclk, pclk);
diff --git a/drivers/video/fbdev/pm2fb.c b/drivers/video/fbdev/pm2fb.c
index 058075a34c5dc2..f287c9059489c1 100644
--- a/drivers/video/fbdev/pm2fb.c
+++ b/drivers/video/fbdev/pm2fb.c
@@ -428,7 +428,7 @@ static void set_aperture(struct pm2fb_par *p, u32 depth)
 	}
 #endif
 
-	/* We don't use aperture two, so this may be superflous */
+	/* We don't use aperture two, so this may be superfluous */
 	pm2_WR(p, PM2R_APERTURE_TWO, PM2F_APERTURE_STANDARD);
 }
 
diff --git a/drivers/video/fbdev/s1d13xxxfb.c b/drivers/video/fbdev/s1d13xxxfb.c
index e80c806ef520dc..ab09790d6268d3 100644
--- a/drivers/video/fbdev/s1d13xxxfb.c
+++ b/drivers/video/fbdev/s1d13xxxfb.c
@@ -400,7 +400,7 @@ s1d13xxxfb_pan_display(struct fb_var_screeninfo *var, struct fb_info *info)
 
 /**
  *	bltbit_wait_bitclear - waits for change in register value
- *	@info : frambuffer structure
+ *	@info : framebuffer structure
  *	@bit  : value currently in register
  *	@timeout : ...
  *
@@ -426,7 +426,7 @@ bltbit_wait_bitclear(struct fb_info *info, u8 bit, int timeout)
  *	@info : framebuffer structure
  *	@area : fb_copyarea structure
  *
- *	supports (atleast) S1D13506
+ *	supports (at least) S1D13506
  *
  */
 static void
@@ -516,7 +516,7 @@ s1d13xxxfb_bitblt_copyarea(struct fb_info *info, const struct fb_copyarea *area)
  *	@info : framebuffer structure
  *	@rect : fb_fillrect structure
  *
- *	supports (atleast 13506)
+ *	supports (at least 13506)
  *
  **/
 static void
diff --git a/drivers/video/fbdev/sh_mobile_lcdcfb.c b/drivers/video/fbdev/sh_mobile_lcdcfb.c
index e8324b01700f6d..5743c96d548148 100644
--- a/drivers/video/fbdev/sh_mobile_lcdcfb.c
+++ b/drivers/video/fbdev/sh_mobile_lcdcfb.c
@@ -1337,22 +1337,24 @@ overlay_rop3_store(struct device *dev, struct device_attribute *attr,
 	return count;
 }
 
-static DEVICE_ATTR_RW(overlay_alpha);
-static DEVICE_ATTR_RW(overlay_mode);
-static DEVICE_ATTR_RW(overlay_position);
-static DEVICE_ATTR_RW(overlay_rop3);
-
-static struct attribute *overlay_sysfs_attrs[] __maybe_unused = {
-	&dev_attr_overlay_alpha.attr,
-	&dev_attr_overlay_mode.attr,
-	&dev_attr_overlay_position.attr,
-	&dev_attr_overlay_rop3.attr,
+static struct device_attribute dev_attr_ovl_alpha =
+	__ATTR(ovl_alpha, 0644, overlay_alpha_show, overlay_alpha_store);
+static struct device_attribute dev_attr_ovl_mode =
+	__ATTR(ovl_mode, 0644, overlay_mode_show, overlay_mode_store);
+static struct device_attribute dev_attr_ovl_position =
+	__ATTR(ovl_position, 0644, overlay_position_show, overlay_position_store);
+static struct device_attribute dev_attr_ovl_rop3 =
+	__ATTR(ovl_rop3, 0644, overlay_rop3_show, overlay_rop3_store);
+
+static struct attribute *overlay_sysfs_attrs[] = {
+	&dev_attr_ovl_alpha.attr,
+	&dev_attr_ovl_mode.attr,
+	&dev_attr_ovl_position.attr,
+	&dev_attr_ovl_rop3.attr,
 	NULL,
 };
 
-#ifdef CONFIG_FB_DEVICE
 ATTRIBUTE_GROUPS(overlay_sysfs);
-#endif
 
 static const struct fb_fix_screeninfo sh_mobile_lcdc_overlay_fix  = {
 	.id =		"SH Mobile LCDC",
@@ -1510,9 +1512,10 @@ sh_mobile_lcdc_overlay_fb_unregister(struct sh_mobile_lcdc_overlay *ovl)
 {
 	struct fb_info *info = ovl->info;
 
-	if (info == NULL || info->dev == NULL)
+	if (!info || !dev_of_fbinfo(info))
 		return;
 
+	device_remove_groups(dev_of_fbinfo(info), overlay_sysfs_groups);
 	unregister_framebuffer(ovl->info);
 }
 
@@ -1530,6 +1533,15 @@ sh_mobile_lcdc_overlay_fb_register(struct sh_mobile_lcdc_overlay *ovl)
 	if (ret < 0)
 		return ret;
 
+	/* The framebuffer device is optional, see fb_device_create(). */
+	if (dev_of_fbinfo(info)) {
+		ret = device_add_groups(dev_of_fbinfo(info), overlay_sysfs_groups);
+		if (ret < 0) {
+			unregister_framebuffer(info);
+			return ret;
+		}
+	}
+
 	dev_info(lcdc->dev, "registered %s/overlay %u as %dx%d %dbpp.\n",
 		 dev_name(lcdc->dev), ovl->index, info->var.xres,
 		 info->var.yres, info->var.bits_per_pixel);
@@ -2637,7 +2649,6 @@ static int sh_mobile_lcdc_probe(struct platform_device *pdev)
 static struct platform_driver sh_mobile_lcdc_driver = {
 	.driver		= {
 		.name		= "sh_mobile_lcdc_fb",
-		.dev_groups	= overlay_sysfs_groups,
 		.pm		= &sh_mobile_lcdc_dev_pm_ops,
 	},
 	.probe		= sh_mobile_lcdc_probe,
diff --git a/drivers/video/fbdev/skeletonfb.c b/drivers/video/fbdev/skeletonfb.c
index 7e98850d9bde9b..bbfbf55cdab807 100644
--- a/drivers/video/fbdev/skeletonfb.c
+++ b/drivers/video/fbdev/skeletonfb.c
@@ -478,7 +478,7 @@ static int xxxfb_blank(int blank_mode, struct fb_info *info)
 
 /**
  *      xxxfb_fillrect - REQUIRED function. Can use generic routines if
- *		 	 non acclerated hardware and packed pixel based.
+ *		 	 non accelerated hardware and packed pixel based.
  *			 Draws a rectangle on the screen.
  *
  *      @info: frame buffer structure that represents a single frame buffer
@@ -505,7 +505,7 @@ void xxxfb_fillrect(struct fb_info *p, const struct fb_fillrect *region)
 
 /**
  *      xxxfb_copyarea - REQUIRED function. Can use generic routines if
- *                       non acclerated hardware and packed pixel based.
+ *                       non accelerated hardware and packed pixel based.
  *                       Copies one area of the screen to another area.
  *
  *      @info: frame buffer structure that represents a single frame buffer
@@ -530,7 +530,7 @@ void xxxfb_copyarea(struct fb_info *p, const struct fb_copyarea *area)
 
 /**
  *      xxxfb_imageblit - REQUIRED function. Can use generic routines if
- *                        non acclerated hardware and packed pixel based.
+ *                        non accelerated hardware and packed pixel based.
  *                        Copies a image from system memory to the screen.
  *
  *      @info: frame buffer structure that represents a single frame buffer
diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c
index 4d185c75428438..db61d710fccb01 100644
--- a/drivers/video/fbdev/ssd1307fb.c
+++ b/drivers/video/fbdev/ssd1307fb.c
@@ -767,6 +767,8 @@ static int ssd1307fb_probe(struct i2c_client *client)
 	info->fix.smem_start = __pa(vmem);
 	info->fix.smem_len = vmem_size;
 
+	info->flags = FBINFO_VIRTFB;
+
 	fb_deferred_io_init(info);
 
 	i2c_set_clientdata(client, info);
diff --git a/drivers/video/fbdev/sstfb.c b/drivers/video/fbdev/sstfb.c
index 2745557822f76e..8e2c2421028df7 100644
--- a/drivers/video/fbdev/sstfb.c
+++ b/drivers/video/fbdev/sstfb.c
@@ -15,7 +15,7 @@
  * 09/2002 Helge Deller    <deller@gmx.de>
  *	(enable driver on big-endian machines (hppa), ioctl fixes)
  * 12/2002 Helge Deller    <deller@gmx.de>
- *	(port driver to new frambuffer infrastructure)
+ *	(port driver to new framebuffer infrastructure)
  * 01/2003 Helge Deller    <deller@gmx.de>
  *	(initial work on fb hardware acceleration for voodoo2)
  * 08/2006 Alan Cox 	   <alan@redhat.com>
@@ -1332,7 +1332,8 @@ static int sstfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		return err;
 
 	/* Enable device in PCI config. */
-	if ((err=pci_enable_device(pdev))) {
+	err = pcim_enable_device(pdev);
+	if (err) {
 		printk(KERN_ERR "cannot enable device\n");
 		return err;
 	}
diff --git a/drivers/video/fbdev/tgafb.c b/drivers/video/fbdev/tgafb.c
index dccfc38cfbd523..32296f926196ef 100644
--- a/drivers/video/fbdev/tgafb.c
+++ b/drivers/video/fbdev/tgafb.c
@@ -884,7 +884,7 @@ tgafb_clut_imageblit(struct fb_info *info, const struct fb_image *image)
 
 /**
  *      tgafb_imageblit - REQUIRED function. Can use generic routines if
- *                        non acclerated hardware and packed pixel based.
+ *                        non accelerated hardware and packed pixel based.
  *                        Copies a image from system memory to the screen.
  *
  *      @info: frame buffer structure that represents a single frame buffer
@@ -921,7 +921,7 @@ tgafb_imageblit(struct fb_info *info, const struct fb_image *image)
 
 /**
  *      tgafb_fillrect - REQUIRED function. Can use generic routines if
- *                       non acclerated hardware and packed pixel based.
+ *                       non accelerated hardware and packed pixel based.
  *                       Draws a rectangle on the screen.
  *
  *      @info: frame buffer structure that represents a single frame buffer
@@ -1047,7 +1047,7 @@ tgafb_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 
 /*
  *      tgafb_copyarea - REQUIRED function. Can use generic routines if
- *                       non acclerated hardware and packed pixel based.
+ *                       non accelerated hardware and packed pixel based.
  *                       Copies on area of the screen to another area.
  *
  *      @info: frame buffer structure that represents a single frame buffer
diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c
index 9f055ba776c81a..6501f6e2615a4a 100644
--- a/drivers/video/fbdev/tridentfb.c
+++ b/drivers/video/fbdev/tridentfb.c
@@ -58,7 +58,7 @@ static struct fb_fix_screeninfo tridentfb_fix = {
 	.accel = FB_ACCEL_NONE,
 };
 
-/* defaults which are normally overriden by user values */
+/* defaults which are normally overridden by user values */
 
 /* video mode */
 static char *mode_option;
diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c
index e78d6f95c9c5ce..92b352bc7e969c 100644
--- a/drivers/video/fbdev/udlfb.c
+++ b/drivers/video/fbdev/udlfb.c
@@ -1293,6 +1293,11 @@ static int dlfb_setup_modes(struct dlfb_data *dlfb,
 		goto error;
 	}
 
+	if (dlfb->fb_count) {
+		result = -EINVAL;
+		goto error;
+	}
+
 	fb_destroy_modelist(&info->modelist);
 	memset(&info->monspecs, 0, sizeof(info->monspecs));
 
diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c
index ccc9dbc25813cf..a13dff332aefec 100644
--- a/drivers/video/fbdev/uvesafb.c
+++ b/drivers/video/fbdev/uvesafb.c
@@ -23,6 +23,7 @@
 #include <linux/io.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
+#include <linux/umh.h>
 #include <video/edid.h>
 #include <video/uvesafb.h>
 #ifdef CONFIG_X86
diff --git a/drivers/video/fbdev/via/dvi.c b/drivers/video/fbdev/via/dvi.c
index 27990a73bfa391..845f513226972c 100644
--- a/drivers/video/fbdev/via/dvi.c
+++ b/drivers/video/fbdev/via/dvi.c
@@ -399,7 +399,7 @@ void viafb_dvi_enable(void)
 			tmds_register_write(0x88, 0x3b);
 		else
 			/*clear CR91[5] to direct on display period
-			   in the secondary diplay path */
+			   in the secondary display path */
 			via_write_reg_mask(VIACR, 0x91, 0x00, 0x20);
 		break;
 
@@ -407,12 +407,12 @@ void viafb_dvi_enable(void)
 		if (viaparinfo->chip_info->gfx_chip_name == UNICHROME_CLE266)
 			viafb_write_reg_mask(CR93, VIACR, 0x21, BIT0 + BIT5);
 
-		/*fix dvi cann't be enabled with MB VT5718C4 - Al Zhang */
+		/* fix dvi can't be enabled with MB VT5718C4 - Al Zhang */
 		if (viaparinfo->chip_info->gfx_chip_name == UNICHROME_CLE266)
 			tmds_register_write(0x88, 0x3b);
 		else
 			/*clear CR91[5] to direct on display period
-			  in the secondary diplay path */
+			  in the secondary display path */
 			via_write_reg_mask(VIACR, 0x91, 0x00, 0x20);
 
 		/*fix DVI cannot enable on EPIA-M board */
diff --git a/drivers/video/fbdev/via/hw.c b/drivers/video/fbdev/via/hw.c
index 66afd7be00cc74..e9d0fac8fb17f5 100644
--- a/drivers/video/fbdev/via/hw.c
+++ b/drivers/video/fbdev/via/hw.c
@@ -1542,7 +1542,7 @@ static void init_gfx_chip_info(int chip_type)
 		else
 			viaparinfo->chip_info->gfx_chip_revision =
 			CLE266_REVISION_CX;
-		/* restore orignal CR4F value */
+		/* restore original CR4F value */
 		viafb_write_reg(CR4F, VIACR, tmp);
 	}
 
diff --git a/drivers/video/fbdev/via/viafbdev.c b/drivers/video/fbdev/via/viafbdev.c
index dd9374e8ad098b..2c67a18d16ad38 100644
--- a/drivers/video/fbdev/via/viafbdev.c
+++ b/drivers/video/fbdev/via/viafbdev.c
@@ -190,7 +190,7 @@ static int viafb_check_var(struct fb_var_screeninfo *var,
 
 	DEBUG_MSG(KERN_INFO "viafb_check_var!\n");
 	/* Sanity check */
-	/* HW neither support interlacte nor double-scaned mode */
+	/* HW neither support interlacte nor double-scanned mode */
 	if (var->vmode & FB_VMODE_INTERLACED || var->vmode & FB_VMODE_DOUBLE)
 		return -EINVAL;
 
@@ -765,7 +765,7 @@ static int viafb_cursor(struct fb_info *info, struct fb_cursor *cursor)
 	if (info->flags & FBINFO_HWACCEL_DISABLED || info != viafbinfo)
 		return -ENODEV;
 
-	/* LCD ouput does not support hw cursors (at least on VN896) */
+	/* LCD output does not support hw cursors (at least on VN896) */
 	if ((chip_name == UNICHROME_CLE266 && viapar->iga_path == IGA2) ||
 		viafb_LCD_ON)
 		return -ENODEV;
diff --git a/drivers/video/sticore.c b/drivers/video/sticore.c
index 1d4477f2045049..6876a03c464a96 100644
--- a/drivers/video/sticore.c
+++ b/drivers/video/sticore.c
@@ -389,7 +389,7 @@ static int sti_font_setup(char *str)
  *	- sti_font=<number>	(<number> = 1,2,3,...)
  *		most STI ROMs have built-in HP specific fonts, which can be selected
  *		by giving the desired number to the sticon driver.
- *		NOTE: This number is machine and STI ROM dependend.
+ *		NOTE: This number is machine and STI ROM dependent.
  *	- sti_font=<height>x<width>  (e.g. sti_font=16x8)
  *		<height> and <width> gives hints to the height and width of the
  *		font which the user wants. The sticon driver will try to use
diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig
index 52eb7e4ba71f65..eeb84e578ddf96 100644
--- a/drivers/virt/Kconfig
+++ b/drivers/virt/Kconfig
@@ -41,6 +41,23 @@ config FSL_HV_MANAGER
           4) A kernel interface for receiving callbacks when a managed
 	     partition shuts down.
 
+config STEAL_GOVERNOR
+	tristate "Dynamic vCPU management based on steal time"
+	depends on PARAVIRT && SMP
+	select PREFERRED_CPU
+	default m
+	help
+	  This driver helps to reduce the steal time in paravirtualized
+	  environments, thereby reducing vCPU preemption costs.
+
+	  By default preferred CPUs will be same as active CPUs. Depending
+	  on the steal time when steal_governor driver is enabled,
+	  preferred CPUs could become subset of active CPUs.
+	  More details are at: Documentation/driver-api/steal-governor.rst
+
+	  It is recommended to build it as module and load the module
+	  to enable it.
+
 source "drivers/virt/vboxguest/Kconfig"
 
 source "drivers/virt/nitro_enclaves/Kconfig"
diff --git a/drivers/virt/Makefile b/drivers/virt/Makefile
index f29901bd782058..05fb075ef5b856 100644
--- a/drivers/virt/Makefile
+++ b/drivers/virt/Makefile
@@ -5,6 +5,7 @@
 
 obj-$(CONFIG_FSL_HV_MANAGER)	+= fsl_hypervisor.o
 obj-$(CONFIG_VMGENID)		+= vmgenid.o
+obj-$(CONFIG_STEAL_GOVERNOR)	+= steal_governor.o
 obj-y				+= vboxguest/
 
 obj-$(CONFIG_NITRO_ENCLAVES)	+= nitro_enclaves/
diff --git a/drivers/virt/coco/arm-cca-guest/Kconfig b/drivers/virt/coco/arm-cca-guest/Kconfig
index 3f0f013f03f159..0d4ce72e2d8622 100644
--- a/drivers/virt/coco/arm-cca-guest/Kconfig
+++ b/drivers/virt/coco/arm-cca-guest/Kconfig
@@ -1,6 +1,7 @@
 config ARM_CCA_GUEST
 	tristate "Arm CCA Guest driver"
-	depends on ARM64
+	depends on ARM_RMM_RSI
+	depends on HAVE_ARM_SMCCC_DISCOVERY
 	select TSM_REPORTS
 	help
 	  The driver provides userspace interface to request and
diff --git a/drivers/virt/coco/arm-cca-guest/Makefile b/drivers/virt/coco/arm-cca-guest/Makefile
index 69eeba08e98a85..7564f4fcc4a41a 100644
--- a/drivers/virt/coco/arm-cca-guest/Makefile
+++ b/drivers/virt/coco/arm-cca-guest/Makefile
@@ -1,2 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest.o
+
+arm-cca-guest-y += main.o
diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c
deleted file mode 100644
index 0eeddd1ff05b51..00000000000000
--- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c
+++ /dev/null
@@ -1,208 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2023 ARM Ltd.
- */
-
-#include <linux/arm-smccc.h>
-#include <linux/cc_platform.h>
-#include <linux/kernel.h>
-#include <linux/device-id/platform.h>
-#include <linux/module.h>
-#include <linux/smp.h>
-#include <linux/tsm.h>
-#include <linux/types.h>
-
-#include <asm/rsi.h>
-
-/**
- * struct arm_cca_token_info - a descriptor for the token buffer.
- * @granule:		PA of the granule to which the token will be written
- * @offset:		Offset within granule to start of buffer in bytes
- */
-struct arm_cca_token_info {
-	phys_addr_t     granule;
-	unsigned long   offset;
-};
-
-/**
- * arm_cca_attestation_continue - Retrieve the attestation token data.
- *
- * @info: pointer to the arm_cca_token_info
- *
- * Attestation token generation is a long running operation and therefore
- * the token data may not be retrieved in a single call. Moreover, the
- * token retrieval operation must be requested on the same CPU on which the
- * attestation token generation was initialised.
- * This helper function must therefore be executed on the same CPU multiple
- * times until the entire token data is retrieved.
- */
-static unsigned long
-arm_cca_attestation_continue(struct arm_cca_token_info *info)
-{
-	unsigned long ret;
-	unsigned long len;
-	unsigned long size;
-
-	size = RSI_GRANULE_SIZE - info->offset;
-	ret = rsi_attestation_token_continue(info->granule, info->offset, size,
-					     &len);
-	info->offset += len;
-	return ret;
-}
-
-/**
- * arm_cca_report_new - Generate a new attestation token.
- *
- * @report: pointer to the TSM report context information.
- * @data:  pointer to the context specific data for this module.
- *
- * Initialise the attestation token generation using the challenge data
- * passed in the TSM descriptor. Allocate memory for the attestation token
- * and retrieve the attestation token on the same CPU on which the
- * attestation token generation was initialised.
- *
- * The challenge data must be at least 32 bytes and no more than 64 bytes. If
- * less than 64 bytes are provided it will be zero padded to 64 bytes.
- *
- * Return:
- * * %0        - Attestation token generated successfully.
- * * %-EINVAL  - A parameter was not valid.
- * * %-ENOMEM  - Out of memory.
- * * %-EFAULT  - Failed to get IPA for memory page(s).
- */
-static int arm_cca_report_new(struct tsm_report *report, void *data)
-{
-	int ret = 0;
-	unsigned long rsi_result;
-	long max_size;
-	unsigned long token_size = 0;
-	struct arm_cca_token_info info;
-	void *buf;
-	u8 *token __free(kvfree) = NULL;
-	struct tsm_report_desc *desc = &report->desc;
-
-	if (desc->inblob_len < 32 || desc->inblob_len > 64)
-		return -EINVAL;
-
-	/*
-	 * The attestation token 'init' and 'continue' calls must be
-	 * performed on the same CPU, so disable CPU migration around
-	 * those operations.
-	 */
-	migrate_disable();
-
-	max_size = rsi_attestation_token_init(desc->inblob, desc->inblob_len);
-	if (max_size <= 0) {
-		ret = -EINVAL;
-		goto exit_migrate_enable;
-	}
-
-	/* Allocate outblob */
-	token = kvzalloc(max_size, GFP_KERNEL);
-	if (!token) {
-		ret = -ENOMEM;
-		goto exit_migrate_enable;
-	}
-
-	/*
-	 * Since the outblob may not be physically contiguous, use a page
-	 * to bounce the buffer from RMM.
-	 */
-	buf = alloc_pages_exact(RSI_GRANULE_SIZE, GFP_KERNEL);
-	if (!buf) {
-		ret = -ENOMEM;
-		goto exit_migrate_enable;
-	}
-
-	/* Get the PA of the memory page(s) that were allocated */
-	info.granule = (unsigned long)virt_to_phys(buf);
-
-	/* Loop until the token is ready or there is an error */
-	do {
-		/* Retrieve one RSI_GRANULE_SIZE data per loop iteration */
-		info.offset = 0;
-		do {
-			/*
-			 * Retrieve a sub-granule chunk of data per loop
-			 * iteration.
-			 */
-			rsi_result = arm_cca_attestation_continue(&info);
-		} while (rsi_result == RSI_INCOMPLETE &&
-			 info.offset < RSI_GRANULE_SIZE);
-
-		/* Break out in case of failure */
-		if (rsi_result != RSI_SUCCESS && rsi_result != RSI_INCOMPLETE) {
-			ret = -ENXIO;
-			token_size = 0;
-			goto exit_free_granule_page;
-		}
-
-		/*
-		 * Copy the retrieved token data from the granule
-		 * to the token buffer, ensuring that the RMM doesn't
-		 * overflow the buffer.
-		 */
-		if (WARN_ON(token_size + info.offset > max_size))
-			break;
-		memcpy(&token[token_size], buf, info.offset);
-		token_size += info.offset;
-	} while (rsi_result == RSI_INCOMPLETE);
-
-	report->outblob = no_free_ptr(token);
-exit_free_granule_page:
-	report->outblob_len = token_size;
-	free_pages_exact(buf, RSI_GRANULE_SIZE);
-exit_migrate_enable:
-	migrate_enable();
-	return ret;
-}
-
-static const struct tsm_report_ops arm_cca_tsm_ops = {
-	.name = KBUILD_MODNAME,
-	.report_new = arm_cca_report_new,
-};
-
-/**
- * arm_cca_guest_init - Register with the Trusted Security Module (TSM)
- * interface.
- *
- * Return:
- * * %0        - Registered successfully with the TSM interface.
- * * %-ENODEV  - The execution context is not an Arm Realm.
- * * %-EBUSY   - Already registered.
- */
-static int __init arm_cca_guest_init(void)
-{
-	int ret;
-
-	if (!is_realm_world())
-		return -ENODEV;
-
-	ret = tsm_report_register(&arm_cca_tsm_ops, NULL);
-	if (ret < 0)
-		pr_err("Error %d registering with TSM\n", ret);
-
-	return ret;
-}
-module_init(arm_cca_guest_init);
-
-/**
- * arm_cca_guest_exit - unregister with the Trusted Security Module (TSM)
- * interface.
- */
-static void __exit arm_cca_guest_exit(void)
-{
-	tsm_report_unregister(&arm_cca_tsm_ops);
-}
-module_exit(arm_cca_guest_exit);
-
-/* modalias, so userspace can autoload this module when RSI is available */
-static const struct platform_device_id arm_cca_match[] __maybe_unused = {
-	{ .name = RSI_PDEV_NAME },
-	{ }
-};
-
-MODULE_DEVICE_TABLE(platform, arm_cca_match);
-MODULE_AUTHOR("Sami Mujawar <sami.mujawar@arm.com>");
-MODULE_DESCRIPTION("Arm CCA Guest TSM Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/virt/coco/arm-cca-guest/main.c b/drivers/virt/coco/arm-cca-guest/main.c
new file mode 100644
index 00000000000000..f97f593da6e3d3
--- /dev/null
+++ b/drivers/virt/coco/arm-cca-guest/main.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023 ARM Ltd.
+ */
+
+#include <linux/arm-rsi-cmds.h>
+#include <linux/arm-smccc-bus.h>
+#include <linux/arm-smccc-rsi.h>
+#include <linux/cc_platform.h>
+#include <linux/kernel.h>
+#include <linux/device-id/platform.h>
+#include <linux/module.h>
+#include <linux/smp.h>
+#include <linux/tsm.h>
+#include <linux/types.h>
+
+/**
+ * struct arm_cca_token_info - a descriptor for the token buffer.
+ * @granule:		PA of the granule to which the token will be written
+ * @offset:		Offset within granule to start of buffer in bytes
+ */
+struct arm_cca_token_info {
+	phys_addr_t     granule;
+	unsigned long   offset;
+};
+
+/**
+ * arm_cca_attestation_continue - Retrieve the attestation token data.
+ *
+ * @info: pointer to the arm_cca_token_info
+ *
+ * Attestation token generation is a long running operation and therefore
+ * the token data may not be retrieved in a single call. Moreover, the
+ * token retrieval operation must be requested on the same CPU on which the
+ * attestation token generation was initialised.
+ * This helper function must therefore be executed on the same CPU multiple
+ * times until the entire token data is retrieved.
+ */
+static unsigned long
+arm_cca_attestation_continue(struct arm_cca_token_info *info)
+{
+	unsigned long ret;
+	unsigned long len;
+	unsigned long size;
+
+	size = RSI_GRANULE_SIZE - info->offset;
+	ret = rsi_attestation_token_continue(info->granule, info->offset, size,
+					     &len);
+	info->offset += len;
+	return ret;
+}
+
+/**
+ * arm_cca_report_new - Generate a new attestation token.
+ *
+ * @report: pointer to the TSM report context information.
+ * @data:  pointer to the context specific data for this module.
+ *
+ * Initialise the attestation token generation using the challenge data
+ * passed in the TSM descriptor. Allocate memory for the attestation token
+ * and retrieve the attestation token on the same CPU on which the
+ * attestation token generation was initialised.
+ *
+ * The challenge data must be at least 32 bytes and no more than 64 bytes. If
+ * less than 64 bytes are provided it will be zero padded to 64 bytes.
+ *
+ * Return:
+ * * %0        - Attestation token generated successfully.
+ * * %-EINVAL  - A parameter was not valid.
+ * * %-ENOMEM  - Out of memory.
+ * * %-EFAULT  - Failed to get IPA for memory page(s).
+ */
+static int arm_cca_report_new(struct tsm_report *report, void *data)
+{
+	int ret = 0;
+	unsigned long rsi_result;
+	long max_size;
+	unsigned long token_size = 0;
+	struct arm_cca_token_info info;
+	void *buf;
+	u8 *token __free(kvfree) = NULL;
+	struct tsm_report_desc *desc = &report->desc;
+
+	if (desc->inblob_len < 32 || desc->inblob_len > 64)
+		return -EINVAL;
+
+	/*
+	 * The attestation token 'init' and 'continue' calls must be
+	 * performed on the same CPU, so disable CPU migration around
+	 * those operations.
+	 */
+	migrate_disable();
+
+	max_size = rsi_attestation_token_init(desc->inblob, desc->inblob_len);
+	if (max_size <= 0) {
+		ret = -EINVAL;
+		goto exit_migrate_enable;
+	}
+
+	/* Allocate outblob */
+	token = kvzalloc(max_size, GFP_KERNEL);
+	if (!token) {
+		ret = -ENOMEM;
+		goto exit_migrate_enable;
+	}
+
+	/*
+	 * Since the outblob may not be physically contiguous, use a page
+	 * to bounce the buffer from RMM.
+	 */
+	buf = alloc_pages_exact(RSI_GRANULE_SIZE, GFP_KERNEL);
+	if (!buf) {
+		ret = -ENOMEM;
+		goto exit_migrate_enable;
+	}
+
+	/* Get the PA of the memory page(s) that were allocated */
+	info.granule = (unsigned long)virt_to_phys(buf);
+
+	/* Loop until the token is ready or there is an error */
+	do {
+		/* Retrieve one RSI_GRANULE_SIZE data per loop iteration */
+		info.offset = 0;
+		do {
+			/*
+			 * Retrieve a sub-granule chunk of data per loop
+			 * iteration.
+			 */
+			rsi_result = arm_cca_attestation_continue(&info);
+		} while (rsi_result == RSI_INCOMPLETE &&
+			 info.offset < RSI_GRANULE_SIZE);
+
+		/* Break out in case of failure */
+		if (rsi_result != RSI_SUCCESS && rsi_result != RSI_INCOMPLETE) {
+			ret = -ENXIO;
+			token_size = 0;
+			goto exit_free_granule_page;
+		}
+
+		/*
+		 * Copy the retrieved token data from the granule
+		 * to the token buffer, ensuring that the RMM doesn't
+		 * overflow the buffer.
+		 */
+		if (WARN_ON(token_size + info.offset > max_size))
+			break;
+		memcpy(&token[token_size], buf, info.offset);
+		token_size += info.offset;
+	} while (rsi_result == RSI_INCOMPLETE);
+
+	report->outblob = no_free_ptr(token);
+exit_free_granule_page:
+	report->outblob_len = token_size;
+	free_pages_exact(buf, RSI_GRANULE_SIZE);
+exit_migrate_enable:
+	migrate_enable();
+	return ret;
+}
+
+static const struct tsm_report_ops arm_cca_tsm_report_ops = {
+	.name = KBUILD_MODNAME,
+	.report_new = arm_cca_report_new,
+};
+
+static int cca_tsm_probe(struct arm_smccc_device *sdev)
+{
+	int ret;
+
+	if (!is_realm_world())
+		return -ENODEV;
+
+	ret = tsm_report_register(&arm_cca_tsm_report_ops, NULL);
+	if (ret < 0)
+		return dev_err_probe(&sdev->dev,
+				     ret, "Error registering with TSM\n");
+
+	return 0;
+}
+
+static void cca_tsm_remove(struct arm_smccc_device *sdev)
+{
+	tsm_report_unregister(&arm_cca_tsm_report_ops);
+}
+
+static const struct arm_smccc_device_id cca_tsm_id_table[] = {
+	{ .func_id = SMC_RSI_ABI_VERSION },
+	{}
+};
+MODULE_DEVICE_TABLE(arm_smccc, cca_tsm_id_table);
+
+static struct arm_smccc_driver cca_tsm_driver = {
+	.name = KBUILD_MODNAME,
+	.probe = cca_tsm_probe,
+	.remove = cca_tsm_remove,
+	.id_table = cca_tsm_id_table,
+};
+module_arm_smccc_driver(cca_tsm_driver);
+MODULE_AUTHOR("Sami Mujawar <sami.mujawar@arm.com>");
+MODULE_DESCRIPTION("Arm CCA Guest TSM Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/virt/coco/guest/report.c b/drivers/virt/coco/guest/report.c
index b254a14162865e..ad400fbe53f01e 100644
--- a/drivers/virt/coco/guest/report.c
+++ b/drivers/virt/coco/guest/report.c
@@ -356,7 +356,7 @@ static struct configfs_attribute *tsm_report_attrs[] = {
 	NULL,
 };
 
-static struct configfs_bin_attribute *tsm_report_bin_attrs[] = {
+static const struct configfs_bin_attribute *const tsm_report_bin_attrs[] = {
 	[TSM_REPORT_INBLOB] = &tsm_report_attr_inblob,
 	[TSM_REPORT_OUTBLOB] = &tsm_report_attr_outblob,
 	[TSM_REPORT_AUXBLOB] = &tsm_report_attr_auxblob,
@@ -381,7 +381,7 @@ static struct configfs_item_operations tsm_report_item_ops = {
 };
 
 static bool tsm_report_is_visible(struct config_item *item,
-				  struct configfs_attribute *attr, int n)
+				  const struct configfs_attribute *attr, int n)
 {
 	guard(rwsem_read)(&tsm_rwsem);
 	if (!provider.ops)
@@ -394,7 +394,7 @@ static bool tsm_report_is_visible(struct config_item *item,
 }
 
 static bool tsm_report_is_bin_visible(struct config_item *item,
-				      struct configfs_bin_attribute *attr, int n)
+				      const struct configfs_bin_attribute *attr, int n)
 {
 	guard(rwsem_read)(&tsm_rwsem);
 	if (!provider.ops)
diff --git a/drivers/virt/coco/tdx-guest/tdx-guest.c b/drivers/virt/coco/tdx-guest/tdx-guest.c
index d0303e31e81610..a21bd0376b74ad 100644
--- a/drivers/virt/coco/tdx-guest/tdx-guest.c
+++ b/drivers/virt/coco/tdx-guest/tdx-guest.c
@@ -265,7 +265,7 @@ static int wait_for_quote_completion(struct tdx_quote_buf *quote_buf, u32 timeou
 	return (i == timeout) ? -ETIMEDOUT : 0;
 }
 
-static int tdx_report_new_locked(struct tsm_report *report, void *data)
+static int tdx_report_new_locked(struct tsm_report *report)
 {
 	u8 *buf;
 	struct tdx_quote_buf *quote_buf = quote_data;
@@ -333,10 +333,10 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data)
 	return ret;
 }
 
-static int tdx_report_new(struct tsm_report *report, void *data)
+static int tdx_report_new(struct tsm_report *report, void *unused)
 {
 	scoped_cond_guard(mutex_intr, return -EINTR, &quote_lock)
-		return tdx_report_new_locked(report, data);
+		return tdx_report_new_locked(report);
 }
 
 static bool tdx_report_attr_visible(int n)
diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c
index c91300a73f5023..0d7bdad924b73c 100644
--- a/drivers/virt/nitro_enclaves/ne_misc_dev.c
+++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c
@@ -1706,6 +1706,7 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_ui
 	return enclave_fd;
 
 put_file:
+	enclave_file->private_data = NULL;
 	fput(enclave_file);
 put_fd:
 	put_unused_fd(enclave_fd);
diff --git a/drivers/virt/steal_governor.c b/drivers/virt/steal_governor.c
new file mode 100644
index 00000000000000..6e31f9923dea4d
--- /dev/null
+++ b/drivers/virt/steal_governor.c
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Steal time governor driver periodically computes steal time.
+ * Based on the thresholds it either reduce/increase the preferred
+ * CPUs which can be used by the workload to avoid vCPU preemption
+ * to an extent possible in paravirtualized environment.
+ *
+ * Available with CONFIG_STEAL_GOVERNOR
+ *
+ * Copyright (C) 2026 IBM
+ * Author: Shrikanth Hegde <sshegde@linux.ibm.com>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/cleanup.h>
+#include <linux/cpuhplock.h>
+#include <linux/cpumask.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/kernel_stat.h>
+#include <linux/kconfig.h>
+#include <linux/ktime.h>
+#include <linux/math64.h>
+#include <linux/module.h>
+#include <linux/sched/isolation.h>
+#include <linux/topology.h>
+#include <linux/types.h>
+#include <linux/workqueue.h>
+#ifdef CONFIG_XEN
+#include <xen/xen.h>
+#endif
+
+#if !IS_ENABLED(CONFIG_PREFERRED_CPU)
+#error "Steal Governor requires CONFIG_PREFERRED_CPU"
+#endif
+
+struct steal_governor {
+	ktime_t			time;
+	u64			steal;
+	unsigned long		delay;
+	unsigned int		interval_ms;
+	unsigned int		high_threshold;
+	unsigned int		low_threshold;
+	struct delayed_work	work;
+};
+
+static struct steal_governor sg_ctx = {
+	.interval_ms	=	1000,	/* 1 second */
+	.high_threshold =	500,	/* 5% */
+	.low_threshold	=	200,	/* 2% */
+};
+
+static void restore_preferred_to_active(void)
+{
+	int cpu;
+
+	guard(cpus_read_lock)();
+	for_each_cpu(cpu, cpu_active_mask)
+		set_cpu_preferred(cpu, true);
+}
+
+static int param_set_interval_ms(const char *val, const struct kernel_param *kp)
+{
+	unsigned int interval;
+	int ret;
+
+	ret = kstrtouint(val, 0, &interval);
+	if (ret)
+		return ret;
+
+	if (interval < 100 || interval > 100000) {
+		pr_err("interval_ms must be between 100 and 100000\n");
+		return -EINVAL;
+	}
+
+	return param_set_uint(val, kp);
+}
+
+static const struct kernel_param_ops interval_ms_ops = {
+	.set = param_set_interval_ms,
+	.get = param_get_uint,
+};
+
+module_param_cb(interval_ms, &interval_ms_ops, &sg_ctx.interval_ms, 0444);
+MODULE_PARM_DESC(interval_ms,
+		 "Sampling frequency in milliseconds. default: 1000");
+
+static int param_set_high_threshold(const char *val, const struct kernel_param *kp)
+{
+	unsigned int threshold;
+	int ret;
+
+	ret = kstrtouint(val, 0, &threshold);
+	if (ret)
+		return ret;
+
+	if (threshold >= 100 * 100) {
+		pr_err("high_threshold (%u) can't be more than 99.99%%\n", threshold);
+		return -EINVAL;
+	}
+
+	return param_set_uint(val, kp);
+}
+
+static const struct kernel_param_ops high_threshold_ops = {
+	.set = param_set_high_threshold,
+	.get = param_get_uint,
+};
+
+module_param_cb(high_threshold, &high_threshold_ops, &sg_ctx.high_threshold, 0444);
+MODULE_PARM_DESC(high_threshold,
+		 "High steal threshold. default: 500 i.e 5%. Must be > low_threshold");
+
+module_param_named(low_threshold, sg_ctx.low_threshold, uint, 0444);
+MODULE_PARM_DESC(low_threshold,
+		 "Low steal threshold. default: 200 i.e 2%. Must be < high_threshold");
+
+/* Return collective steal time across system. */
+static u64 get_system_steal_time(void)
+{
+	return kcpustat_field_total(CPUTIME_STEAL, cpu_possible_mask);
+}
+
+/* Return number of CPUs to consider for steal ratio. */
+static unsigned int get_system_cpus(void)
+{
+	return num_active_cpus();
+}
+
+/*
+ * Called when the steal governor detects high physical CPU contention.
+ * It finds the last active core in the preferred mask and mark those
+ * CPUs as non-preferred.
+ *
+ * Must ensure:
+ * - at least one core is always kept as preferred
+ * - preferred is always subset of active.
+ */
+static void decrease_preferred_cpus(void)
+{
+	const struct cpumask *first_hk_core;
+	int target_cpu = nr_cpu_ids;
+	int cpu;
+
+	guard(cpus_read_lock)();
+	cpu = cpumask_first_and(housekeeping_cpumask(HK_TYPE_KERNEL_NOISE),
+				cpu_preferred_mask);
+	if (cpu >= nr_cpu_ids)
+		return;
+
+	/* Always leave first housekeeping core as preferred. */
+	first_hk_core = topology_sibling_cpumask(cpu);
+	cpu = cpumask_last(cpu_preferred_mask);
+	if (cpu >= nr_cpu_ids)
+		return;
+
+	/* Find the last CPU which doesn't belong to that first hk_core. */
+	if (!cpumask_test_cpu(cpu, first_hk_core)) {
+		target_cpu = cpu;
+	} else {
+		for_each_cpu_andnot(cpu, cpu_preferred_mask, first_hk_core)
+			target_cpu = cpu;
+	}
+
+	/* Only the first housekeeping core remains */
+	if (target_cpu >= nr_cpu_ids)
+		return;
+
+	for_each_cpu_and(cpu, topology_sibling_cpumask(target_cpu),
+			 cpu_preferred_mask)
+		set_cpu_preferred(cpu, false);
+}
+
+/*
+ * Called when the steal governor detects no/low physical CPU contention.
+ * It finds the first active core outside of preferred mask and mark
+ * those CPUs as preferred.
+ *
+ * Must ensure preferred is subset of active.
+ */
+static void increase_preferred_cpus(void)
+{
+	int first_cpu, cpu;
+
+	guard(cpus_read_lock)();
+	first_cpu = cpumask_first_andnot(cpu_active_mask, cpu_preferred_mask);
+
+	/* All CPUs are preferred. Nothing to increase further */
+	if (first_cpu >= nr_cpu_ids)
+		return;
+
+	for_each_cpu_and(cpu, topology_sibling_cpumask(first_cpu),
+			 cpu_active_mask)
+		set_cpu_preferred(cpu, true);
+}
+
+static bool preferred_cpus_valid(void)
+{
+	if (cpumask_empty(cpu_preferred_mask)) {
+		pr_err("empty preferred mask. stopping\n");
+		return false;
+	}
+
+	if (!cpumask_subset(cpu_preferred_mask, cpu_active_mask)) {
+		pr_err("preferred: %*pbl is not subset of active: %*pbl, stopping\n",
+		       cpumask_pr_args(cpu_preferred_mask),
+		       cpumask_pr_args(cpu_active_mask));
+		return false;
+	}
+
+	return true;
+}
+
+static void steal_governor_loop(struct work_struct *work)
+{
+	u64 curr_steal, delta_steal, delta_ns, steal_ratio;
+	ktime_t now;
+
+	now = ktime_get();
+	delta_ns = ktime_to_ns(ktime_sub(now, sg_ctx.time));
+
+	if (unlikely(delta_ns < NSEC_PER_MSEC)) {
+		pr_err_ratelimited("work scheduled too soon delta_ns: %llu\n", delta_ns);
+		goto requeue_work;
+	}
+
+	curr_steal = get_system_steal_time();
+	delta_steal = curr_steal > sg_ctx.steal ? curr_steal - sg_ctx.steal : 0;
+	sg_ctx.steal = curr_steal;
+	sg_ctx.time = now;
+
+	/*
+	 * steal_ratio = (delta_steal * 100*100)/(delta_ns * num_cpus())
+	 * To avoid possible overflow, divide the denominator early.
+	 * Note minimum interval is 100ms.
+	 */
+	delta_ns = max_t(u64, div_u64(delta_ns * get_system_cpus(), 10000), 1);
+	steal_ratio = div64_u64(delta_steal, delta_ns);
+
+	if (steal_ratio > sg_ctx.high_threshold)
+		decrease_preferred_cpus();
+	else if (steal_ratio <= sg_ctx.low_threshold)
+		increase_preferred_cpus();
+	/*
+	 * else: steal ratio is within bounds. Still do design checks so that
+	 * module restores to active if CPU hotplug breaks those assumptions.
+	 */
+	if (!preferred_cpus_valid()) {
+		restore_preferred_to_active();
+		return;
+	}
+
+requeue_work:
+	schedule_delayed_work(&sg_ctx.work, sg_ctx.delay);
+}
+
+static int __init steal_governor_init(void)
+{
+#ifdef CONFIG_XEN
+	if (xen_initial_domain()) {
+		pr_err("Cannot load in Xen Dom0 (Host OS). Driver is for guests only.\n");
+		return -ENODEV;
+	}
+#endif
+
+	if (sg_ctx.low_threshold >= sg_ctx.high_threshold) {
+		pr_err("low_threshold (%u) must be less than high_threshold (%u)\n",
+		       sg_ctx.low_threshold, sg_ctx.high_threshold);
+		return -EINVAL;
+	}
+
+	sg_ctx.delay = msecs_to_jiffies(sg_ctx.interval_ms);
+	INIT_DELAYED_WORK(&sg_ctx.work, steal_governor_loop);
+	sg_ctx.steal = get_system_steal_time();
+	sg_ctx.time = ktime_get();
+	schedule_delayed_work(&sg_ctx.work, sg_ctx.delay);
+	pr_info("enabled. interval: %ums, high_threshold: %u, low_threshold: %u\n",
+		sg_ctx.interval_ms, sg_ctx.high_threshold, sg_ctx.low_threshold);
+
+	return 0;
+}
+
+static void __exit steal_governor_exit(void)
+{
+	disable_delayed_work_sync(&sg_ctx.work);
+	restore_preferred_to_active();
+	pr_info("disabled\n");
+}
+
+module_init(steal_governor_init);
+module_exit(steal_governor_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("Virtualization Steal Time Governor");
diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c
index 66135eac3abffa..ce957f5b1c317d 100644
--- a/drivers/virt/vmgenid.c
+++ b/drivers/virt/vmgenid.c
@@ -25,7 +25,7 @@ struct vmgenid_state {
 
 static void vmgenid_notify(struct device *device)
 {
-	struct vmgenid_state *state = device->driver_data;
+	struct vmgenid_state *state = dev_get_drvdata(device);
 	u8 old_id[VMGENID_SIZE];
 
 	memcpy(old_id, state->this_id, sizeof(old_id));
@@ -75,13 +75,16 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state)
 	phys_addr = (obj->package.elements[0].integer.value << 0) |
 		    (obj->package.elements[1].integer.value << 32);
 
-	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB);
+	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE,
+				  MEMREMAP_WB | MEMREMAP_DEC);
 	if (IS_ERR(virt_addr)) {
 		ret = PTR_ERR(virt_addr);
 		goto out;
 	}
 	setup_vmgenid_state(state, virt_addr);
 
+	dev_set_drvdata(dev, state);
+
 	status = acpi_install_notify_handler(device->handle, ACPI_DEVICE_NOTIFY,
 					     vmgenid_acpi_handler, dev);
 	if (ACPI_FAILURE(status)) {
@@ -89,7 +92,6 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state)
 		goto out;
 	}
 
-	dev->driver_data = state;
 out:
 	ACPI_FREE(parsed.pointer);
 	return ret;
@@ -123,12 +125,13 @@ static int vmgenid_add_of(struct platform_device *pdev,
 	if (ret < 0)
 		return ret;
 
+	dev_set_drvdata(&pdev->dev, state);
+
 	ret = devm_request_irq(&pdev->dev, ret, vmgenid_of_irq_handler,
 			       IRQF_SHARED, "vmgenid", &pdev->dev);
 	if (ret < 0)
 		return ret;
 
-	pdev->dev.driver_data = state;
 	return 0;
 }
 
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
index b6c9e927bef570..a8588d7ad109f1 100644
--- a/drivers/virtio/virtio.c
+++ b/drivers/virtio/virtio.c
@@ -264,6 +264,8 @@ void virtio_reset_device(struct virtio_device *dev)
 #endif
 
 	dev->config->reset(dev);
+	/* Flush pending VQ/configuration callbacks. */
+	virtio_synchronize_cbs(dev);
 }
 EXPORT_SYMBOL_GPL(virtio_reset_device);
 
diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c
index d9cbb02b35a112..8115aa39e01e9b 100644
--- a/drivers/virtio/virtio_pci_legacy.c
+++ b/drivers/virtio/virtio_pci_legacy.c
@@ -98,8 +98,6 @@ static void vp_reset(struct virtio_device *vdev)
 	/* Flush out the status write, and flush in device writes,
 	 * including MSi-X interrupts, if any. */
 	vp_legacy_get_status(&vp_dev->ldev);
-	/* Flush pending VQ/configuration callbacks. */
-	vp_synchronize_vectors(vdev);
 }
 
 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c
index 6d8ae2a6a8caf2..922e2df027de94 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -364,6 +364,12 @@ static void vp_modern_avq_cleanup(struct virtio_device *vdev)
 	}
 }
 
+static void vp_modern_del_vqs(struct virtio_device *vdev)
+{
+	vp_modern_avq_cleanup(vdev);
+	vp_del_vqs(vdev);
+}
+
 static void vp_transport_features(struct virtio_device *vdev, u64 features)
 {
 	struct virtio_pci_device *vp_dev = to_vp_device(vdev);
@@ -557,11 +563,6 @@ static void vp_reset(struct virtio_device *vdev)
 	 */
 	while (vp_modern_get_status(mdev))
 		msleep(1);
-
-	vp_modern_avq_cleanup(vdev);
-
-	/* Flush pending VQ/configuration callbacks. */
-	vp_synchronize_vectors(vdev);
 }
 
 static int vp_active_vq(struct virtqueue *vq, u16 msix_vec)
@@ -1232,7 +1233,7 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
 	.set_status	= vp_set_status,
 	.reset		= vp_reset,
 	.find_vqs	= vp_modern_find_vqs,
-	.del_vqs	= vp_del_vqs,
+	.del_vqs	= vp_modern_del_vqs,
 	.synchronize_cbs = vp_synchronize_vectors,
 	.get_extended_features = vp_get_features,
 	.finalize_features = vp_finalize_features,
@@ -1252,7 +1253,7 @@ static const struct virtio_config_ops virtio_pci_config_ops = {
 	.set_status	= vp_set_status,
 	.reset		= vp_reset,
 	.find_vqs	= vp_modern_find_vqs,
-	.del_vqs	= vp_del_vqs,
+	.del_vqs	= vp_modern_del_vqs,
 	.synchronize_cbs = vp_synchronize_vectors,
 	.get_extended_features = vp_get_features,
 	.finalize_features = vp_finalize_features,
diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
index 6bcf4567a92928..f9559b6b2e0b68 100644
--- a/drivers/virtio/virtio_vdpa.c
+++ b/drivers/virtio/virtio_vdpa.c
@@ -12,6 +12,7 @@
 #include <linux/device.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
+#include <linux/srcu.h>
 #include <linux/uuid.h>
 #include <linux/group_cpus.h>
 #include <linux/virtio.h>
@@ -27,6 +28,7 @@
 struct virtio_vdpa_device {
 	struct virtio_device vdev;
 	struct vdpa_device *vdpa;
+	struct srcu_struct cb_srcu;
 	u64 features;
 };
 
@@ -114,8 +116,11 @@ static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
 static irqreturn_t virtio_vdpa_config_cb(void *private)
 {
 	struct virtio_vdpa_device *vd_dev = private;
+	int idx;
 
+	idx = srcu_read_lock(&vd_dev->cb_srcu);
 	virtio_config_changed(&vd_dev->vdev);
+	srcu_read_unlock(&vd_dev->cb_srcu, idx);
 
 	return IRQ_HANDLED;
 }
@@ -123,8 +128,22 @@ static irqreturn_t virtio_vdpa_config_cb(void *private)
 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
 {
 	struct virtqueue *vq = private;
+	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
+	irqreturn_t ret;
+	int idx;
+
+	idx = srcu_read_lock(&vd_dev->cb_srcu);
+	ret = vring_interrupt(0, vq);
+	srcu_read_unlock(&vd_dev->cb_srcu, idx);
+
+	return ret;
+}
+
+static void virtio_vdpa_synchronize_cbs(struct virtio_device *vdev)
+{
+	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
 
-	return vring_interrupt(0, vq);
+	synchronize_srcu(&vd_dev->cb_srcu);
 }
 
 static struct virtqueue *
@@ -440,6 +459,7 @@ static const struct virtio_config_ops virtio_vdpa_config_ops = {
 	.reset		= virtio_vdpa_reset,
 	.find_vqs	= virtio_vdpa_find_vqs,
 	.del_vqs	= virtio_vdpa_del_vqs,
+	.synchronize_cbs = virtio_vdpa_synchronize_cbs,
 	.get_features	= virtio_vdpa_get_features,
 	.finalize_features = virtio_vdpa_finalize_features,
 	.bus_name	= virtio_vdpa_bus_name,
@@ -454,6 +474,7 @@ static void virtio_vdpa_release_dev(struct device *_d)
 	struct virtio_vdpa_device *vd_dev =
 	       container_of(vdev, struct virtio_vdpa_device, vdev);
 
+	cleanup_srcu_struct(&vd_dev->cb_srcu);
 	kfree(vd_dev);
 }
 
@@ -467,6 +488,11 @@ static int virtio_vdpa_probe(struct vdpa_device *vdpa)
 	if (!vd_dev)
 		return -ENOMEM;
 
+	if (init_srcu_struct(&vd_dev->cb_srcu)) {
+		kfree(vd_dev);
+		return -ENOMEM;
+	}
+
 	vd_dev->vdev.dev.parent = vdpa->map ? &vdpa->dev :
 				  vdpa_get_map(vdpa).dma_dev;
 	vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
@@ -489,10 +515,12 @@ static int virtio_vdpa_probe(struct vdpa_device *vdpa)
 	return 0;
 
 err:
-	if (reg_dev)
+	if (reg_dev) {
 		put_device(&vd_dev->vdev.dev);
-	else
+	} else {
+		cleanup_srcu_struct(&vd_dev->cb_srcu);
 		kfree(vd_dev);
+	}
 	return ret;
 }
 
diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c
index cbd11bc6127493..70854336a17311 100644
--- a/drivers/w1/masters/ds2482.c
+++ b/drivers/w1/masters/ds2482.c
@@ -425,7 +425,7 @@ static u8 ds2482_w1_set_pullup(void *data, int delay)
 
 	/* if delay is non-zero activate the pullup,
 	 * the strong pullup will be automatically deactivated
-	 * by the master, so do not explicitly deactive it
+	 * by the master, so do not explicitly deactivate it
 	 */
 	if (delay) {
 		/* both waits are crucial, otherwise devices might not be
diff --git a/drivers/w1/masters/ds2490.c b/drivers/w1/masters/ds2490.c
index 3cbff88c339e96..075ffd6c3f55d2 100644
--- a/drivers/w1/masters/ds2490.c
+++ b/drivers/w1/masters/ds2490.c
@@ -561,7 +561,7 @@ static int ds_write_bit(struct ds_device *dev, u8 bit)
 
 	/* Set COMM_ICP to write without a readback.  Note, this will
 	 * produce one time slot, a down followed by an up with COMM_D
-	 * only determing the timing.
+	 * only determining the timing.
 	 */
 	err = ds_send_control(dev, COMM_BIT_IO | COMM_IM | COMM_ICP |
 		(bit ? COMM_D : 0), 0);
@@ -690,7 +690,7 @@ static void ds9490r_search(void *data, struct w1_master *master,
 	 * If the number of devices found is less than or equal to the
 	 * search_limit, that number of IDs will be returned.  If there are
 	 * more, search_limit IDs will be returned followed by a non-zero
-	 * discrepency value.
+	 * discrepancy value.
 	 */
 	struct ds_device *dev = data;
 	int err;
diff --git a/drivers/w1/masters/omap_hdq.c b/drivers/w1/masters/omap_hdq.c
index d13db339657094..0982efa2b25b4e 100644
--- a/drivers/w1/masters/omap_hdq.c
+++ b/drivers/w1/masters/omap_hdq.c
@@ -282,7 +282,7 @@ static int omap_hdq_break(struct hdq_data *hdq_data)
 	}
 
 	/*
-	 * wait for both INIT and GO bits rerurn to zero.
+	 * wait for both INIT and GO bits return to zero.
 	 * zero wait time expected for interrupt mode.
 	 */
 	ret = hdq_wait_for_flag(hdq_data, OMAP_HDQ_CTRL_STATUS,
diff --git a/drivers/w1/slaves/w1_ds28e17.c b/drivers/w1/slaves/w1_ds28e17.c
index b638963d4b5951..f3d51cf3998b52 100644
--- a/drivers/w1/slaves/w1_ds28e17.c
+++ b/drivers/w1/slaves/w1_ds28e17.c
@@ -101,7 +101,7 @@ static int w1_f19_i2c_busy_wait(struct w1_slave *sl, size_t count)
 		timebases[data->speed] * (data->stretch) * count
 		+ W1_F19_BUSY_GRATUITY);
 
-	/* Now continusly check the busy flag sent by the DS28E17. */
+	/* Now continuously check the busy flag sent by the DS28E17. */
 	checks = W1_F19_BUSY_CHECKS;
 	while ((checks--) > 0) {
 		/* Return success if the busy flag is cleared. */
diff --git a/drivers/w1/slaves/w1_therm.c b/drivers/w1/slaves/w1_therm.c
index d96b224e221503..02acf84ffec539 100644
--- a/drivers/w1/slaves/w1_therm.c
+++ b/drivers/w1/slaves/w1_therm.c
@@ -355,7 +355,7 @@ static DEVICE_ATTR_RW(alarms);
 static DEVICE_ATTR_RW(conv_time);
 static DEVICE_ATTR_RW(features);
 
-static DEVICE_ATTR_RW(therm_bulk_read); /* attribut at master level */
+static DEVICE_ATTR_RW(therm_bulk_read); /* attribute at master level */
 
 /* Interface Functions declaration */
 
diff --git a/drivers/w1/w1.c b/drivers/w1/w1.c
index c1694664278929..c40dc5153b8e76 100644
--- a/drivers/w1/w1.c
+++ b/drivers/w1/w1.c
@@ -1182,7 +1182,7 @@ int w1_process(void *data)
 
 		__set_current_state(TASK_INTERRUPTIBLE);
 
-		/* hold list_mutex until after interruptible to prevent loosing
+		/* hold list_mutex until after interruptible to prevent losing
 		 * the wakeup signal when async_cmd is added.
 		 */
 		mutex_unlock(&dev->list_mutex);
diff --git a/drivers/w1/w1_family.c b/drivers/w1/w1_family.c
index 97da4f156e9a2e..8152a43c8c76b0 100644
--- a/drivers/w1/w1_family.c
+++ b/drivers/w1/w1_family.c
@@ -67,7 +67,7 @@ void w1_unregister_family(struct w1_family *fent)
 	}
 	spin_unlock(&w1_flock);
 
-	/* deatch devices using this family code */
+	/* detach devices using this family code */
 	w1_reconnect_slaves(fent, 0);
 
 	while (atomic_read(&fent->refcnt)) {
diff --git a/drivers/w1/w1_netlink.h b/drivers/w1/w1_netlink.h
index 449680a615699a..1818a06740eb4e 100644
--- a/drivers/w1/w1_netlink.h
+++ b/drivers/w1/w1_netlink.h
@@ -14,7 +14,7 @@
 /**
  * enum w1_cn_msg_flags - bitfield flags for struct cn_msg.flags
  *
- * @W1_CN_BUNDLE: Request bundling replies into fewer messagse.  Be prepared
+ * @W1_CN_BUNDLE: Request bundling replies into fewer messages.  Be prepared
  * to handle multiple struct cn_msg, struct w1_netlink_msg, and
  * struct w1_netlink_cmd in one packet.
  */
diff --git a/drivers/watchdog/acquirewdt.c b/drivers/watchdog/acquirewdt.c
index 052f65c48a703e..cb62800e89ed4f 100644
--- a/drivers/watchdog/acquirewdt.c
+++ b/drivers/watchdog/acquirewdt.c
@@ -64,6 +64,7 @@
 #include <linux/init.h>			/* For __init/__exit/... */
 #include <linux/uaccess.h>		/* For copy_to_user/put_user/... */
 #include <linux/io.h>			/* For inb/outb/... */
+#include <linux/stringify.h>		/* For __stringify */
 
 /* Module information */
 #define DRV_NAME "acquirewdt"
@@ -92,7 +93,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	Watchdog Operations
diff --git a/drivers/watchdog/advantech_ec_wdt.c b/drivers/watchdog/advantech_ec_wdt.c
index 7c380f90ca5831..2470c43032672e 100644
--- a/drivers/watchdog/advantech_ec_wdt.c
+++ b/drivers/watchdog/advantech_ec_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DRIVER_NAME		"advantech_ec_wdt"
@@ -49,7 +50,7 @@ static ktime_t ec_timestamp;
 
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
-		 "Default Watchdog timer setting (" __MODULE_STRING(DEFAULT_TIME) "s). The range is from " __MODULE_STRING(MIN_TIME) " to " __MODULE_STRING(MAX_TIME) ".");
+		 "Default Watchdog timer setting (" __stringify(DEFAULT_TIME) "s). The range is from " __stringify(MIN_TIME) " to " __stringify(MAX_TIME) ".");
 
 static void adv_ec_wdt_timing_gate(void)
 {
diff --git a/drivers/watchdog/advantechwdt.c b/drivers/watchdog/advantechwdt.c
index 42d3f377178109..6d24a6bc8f9089 100644
--- a/drivers/watchdog/advantechwdt.c
+++ b/drivers/watchdog/advantechwdt.c
@@ -37,6 +37,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 #define DRV_NAME "advantechwdt"
@@ -71,13 +72,13 @@ static int timeout = WATCHDOG_TIMEOUT;	/* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. 1<= timeout <=63, default="
-		__MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+		__stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-		__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	Watchdog Operations
diff --git a/drivers/watchdog/airoha_wdt.c b/drivers/watchdog/airoha_wdt.c
index 4bd333189b87ec..c535c48ab6a86d 100644
--- a/drivers/watchdog/airoha_wdt.c
+++ b/drivers/watchdog/airoha_wdt.c
@@ -20,6 +20,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/watchdog.h>
+#include <linux/stringify.h>
 
 /* Base address of timer and watchdog registers */
 #define TIMER_CTRL			0x0
@@ -44,12 +45,12 @@ struct airoha_wdt_desc {
 static int heartbeat = WDT_HEARTBEAT;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeats in seconds. (default="
-		 __MODULE_STRING(WDT_HEARTBEAT) ")");
+		 __stringify(WDT_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int airoha_wdt_start(struct watchdog_device *wdog_dev)
 {
diff --git a/drivers/watchdog/alim1535_wdt.c b/drivers/watchdog/alim1535_wdt.c
index 1ecbd1ac5c3a08..d2f8d3331fe409 100644
--- a/drivers/watchdog/alim1535_wdt.c
+++ b/drivers/watchdog/alim1535_wdt.c
@@ -18,6 +18,7 @@
 #include <linux/pci.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/stringify.h>
 
 #define WATCHDOG_NAME "ALi_M1535"
 #define WATCHDOG_TIMEOUT 60	/* 60 sec default timeout */
@@ -34,13 +35,13 @@ static int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		"Watchdog timeout in seconds. (0 < timeout < 18000, default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+				__stringify(WATCHDOG_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	ali_start	-	start watchdog countdown
diff --git a/drivers/watchdog/alim7101_wdt.c b/drivers/watchdog/alim7101_wdt.c
index 03a559b41f5b88..e9611c76a26963 100644
--- a/drivers/watchdog/alim7101_wdt.c
+++ b/drivers/watchdog/alim7101_wdt.c
@@ -36,6 +36,7 @@
 #include <linux/pci.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 #define WDT_ENABLE 0x9C
@@ -64,7 +65,7 @@ static int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		"Watchdog timeout in seconds. (1<=timeout<=3600, default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+				__stringify(WATCHDOG_TIMEOUT) ")");
 
 static int use_gpio; /* Use the pic (for a1d revision alim7101) */
 module_param(use_gpio, int, 0);
@@ -82,7 +83,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	Whack the dog
diff --git a/drivers/watchdog/arm_smc_wdt.c b/drivers/watchdog/arm_smc_wdt.c
index bbba23ace7b859..c73ed1dcdfcb02 100644
--- a/drivers/watchdog/arm_smc_wdt.c
+++ b/drivers/watchdog/arm_smc_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/moduleparam.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 #include <uapi/linux/psci.h>
@@ -191,7 +192,7 @@ MODULE_PARM_DESC(timeout, "Watchdog heartbeat in seconds");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			__stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Julius Werner <jwerner@chromium.org>");
diff --git a/drivers/watchdog/armada_37xx_wdt.c b/drivers/watchdog/armada_37xx_wdt.c
index a17a7911771ace..d62b02c40b0eba 100644
--- a/drivers/watchdog/armada_37xx_wdt.c
+++ b/drivers/watchdog/armada_37xx_wdt.c
@@ -16,6 +16,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -70,7 +71,7 @@ MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds");
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			   __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			   __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct armada_37xx_watchdog {
 	struct watchdog_device wdt;
diff --git a/drivers/watchdog/aspeed_wdt.c b/drivers/watchdog/aspeed_wdt.c
index 48454ac8607461..d4bd41cb801e60 100644
--- a/drivers/watchdog/aspeed_wdt.c
+++ b/drivers/watchdog/aspeed_wdt.c
@@ -17,12 +17,13 @@
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 struct aspeed_wdt_scu {
 	const char *compatible;
 	u32 reset_status_reg;
diff --git a/drivers/watchdog/at91rm9200_wdt.c b/drivers/watchdog/at91rm9200_wdt.c
index d5afcc63422155..8c292d24698134 100644
--- a/drivers/watchdog/at91rm9200_wdt.c
+++ b/drivers/watchdog/at91rm9200_wdt.c
@@ -23,6 +23,7 @@
 #include <linux/platform_device.h>
 #include <linux/reboot.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 #include <linux/uaccess.h>
@@ -36,13 +37,13 @@ static struct regmap *regmap_st;
 
 module_param(wdt_time, int, 0);
 MODULE_PARM_DESC(wdt_time, "Watchdog time in seconds. (default="
-				__MODULE_STRING(WDT_DEFAULT_TIME) ")");
+				__stringify(WDT_DEFAULT_TIME) ")");
 
 #ifdef CONFIG_WATCHDOG_NOWAYOUT
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 #endif
 
 
diff --git a/drivers/watchdog/at91sam9_wdt.c b/drivers/watchdog/at91sam9_wdt.c
index 80ba04df54adfd..f966da6020111c 100644
--- a/drivers/watchdog/at91sam9_wdt.c
+++ b/drivers/watchdog/at91sam9_wdt.c
@@ -32,6 +32,7 @@
 #include <linux/uaccess.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
+#include <linux/stringify.h>
 
 #include "at91sam9_wdt.h"
 
@@ -70,12 +71,12 @@
 static int heartbeat;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeats in seconds. "
-	"(default = " __MODULE_STRING(WDT_HEARTBEAT) ")");
+	"(default = " __stringify(WDT_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 #define to_wdt(wdd) container_of(wdd, struct at91wdt, wdd)
 struct at91wdt {
diff --git a/drivers/watchdog/atcwdt200_wdt.c b/drivers/watchdog/atcwdt200_wdt.c
index 7e757c217b0b53..ceedc3f6763207 100644
--- a/drivers/watchdog/atcwdt200_wdt.c
+++ b/drivers/watchdog/atcwdt200_wdt.c
@@ -19,6 +19,7 @@
 #include <linux/pm.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* Register definitions */
@@ -570,11 +571,11 @@ module_platform_driver(atcwdt_driver);
 
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds (default="
-		 __MODULE_STRING(ATCWDT_TIMEOUT) ")");
+		 __stringify(ATCWDT_TIMEOUT) ")");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("CL Wang <cl634@andestech.com>");
diff --git a/drivers/watchdog/ath79_wdt.c b/drivers/watchdog/ath79_wdt.c
index 7df703e9852a6e..02783c1ec52646 100644
--- a/drivers/watchdog/ath79_wdt.c
+++ b/drivers/watchdog/ath79_wdt.c
@@ -32,6 +32,7 @@
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #define DRIVER_NAME	"ath79-wdt"
 
@@ -50,12 +51,12 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-			   "(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			   "(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int timeout = WDT_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds "
-			  "(default=" __MODULE_STRING(WDT_TIMEOUT) "s)");
+			  "(default=" __stringify(WDT_TIMEOUT) "s)");
 
 static unsigned long wdt_flags;
 
diff --git a/drivers/watchdog/bcm2835_wdt.c b/drivers/watchdog/bcm2835_wdt.c
index 6fd8b1b8e386ac..1a0fc66cc0930a 100644
--- a/drivers/watchdog/bcm2835_wdt.c
+++ b/drivers/watchdog/bcm2835_wdt.c
@@ -20,6 +20,7 @@
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/reboot.h>
+#include <linux/stringify.h>
 
 #define PM_RSTC				0x1c
 #define PM_RSTS				0x20
@@ -229,7 +230,7 @@ MODULE_PARM_DESC(heartbeat, "Initial watchdog heartbeat in seconds");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_ALIAS("platform:bcm2835-wdt");
 MODULE_AUTHOR("Lubomir Rintel <lkundrak@v3.sk>");
diff --git a/drivers/watchdog/bcm47xx_wdt.c b/drivers/watchdog/bcm47xx_wdt.c
index e13ec0975bef35..0d7e8a87074a53 100644
--- a/drivers/watchdog/bcm47xx_wdt.c
+++ b/drivers/watchdog/bcm47xx_wdt.c
@@ -21,6 +21,7 @@
 #include <linux/watchdog.h>
 #include <linux/timer.h>
 #include <linux/jiffies.h>
+#include <linux/stringify.h>
 
 #define DRV_NAME		"bcm47xx_wdt"
 
@@ -33,12 +34,12 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog time in seconds. (default="
-				__MODULE_STRING(WDT_DEFAULT_TIME) ")");
+				__stringify(WDT_DEFAULT_TIME) ")");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static inline struct bcm47xx_wdt *bcm47xx_wdt_get(struct watchdog_device *wdd)
 {
diff --git a/drivers/watchdog/bcm7038_wdt.c b/drivers/watchdog/bcm7038_wdt.c
index e038dd66b81982..1117026cf95945 100644
--- a/drivers/watchdog/bcm7038_wdt.c
+++ b/drivers/watchdog/bcm7038_wdt.c
@@ -12,6 +12,7 @@
 #include <linux/platform_device.h>
 #include <linux/platform_data/bcm7038_wdt.h>
 #include <linux/pm.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define WDT_START_1		0xff00
@@ -228,7 +229,7 @@ module_platform_driver(bcm7038_wdt_driver);
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Driver for Broadcom 7038 SoCs Watchdog");
 MODULE_AUTHOR("Justin Chen");
diff --git a/drivers/watchdog/booke_wdt.c b/drivers/watchdog/booke_wdt.c
index 1b1fb389b89ba3..1224d1737e87b5 100644
--- a/drivers/watchdog/booke_wdt.c
+++ b/drivers/watchdog/booke_wdt.c
@@ -12,6 +12,7 @@
 
 #include <linux/module.h>
 #include <linux/smp.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #include <asm/reg_booke.h>
@@ -43,7 +44,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 #ifdef CONFIG_PPC_E500
 
diff --git a/drivers/watchdog/cadence_wdt.c b/drivers/watchdog/cadence_wdt.c
index ad46b33ce5f2ed..55a7c08dc1c159 100644
--- a/drivers/watchdog/cadence_wdt.c
+++ b/drivers/watchdog/cadence_wdt.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define CDNS_WDT_DEFAULT_TIMEOUT	10
@@ -52,12 +53,12 @@ static int nowayout = WATCHDOG_NOWAYOUT;
 module_param(wdt_timeout, int, 0644);
 MODULE_PARM_DESC(wdt_timeout,
 		 "Watchdog time in seconds. (default="
-		 __MODULE_STRING(CDNS_WDT_DEFAULT_TIMEOUT) ")");
+		 __stringify(CDNS_WDT_DEFAULT_TIMEOUT) ")");
 
 module_param(nowayout, int, 0644);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 /**
  * struct cdns_wdt - Watchdog device structure
@@ -359,7 +360,7 @@ static int cdns_wdt_probe(struct platform_device *pdev)
  * @dev: handle to the device structure.
  * Return: 0 always.
  */
-static int __maybe_unused cdns_wdt_suspend(struct device *dev)
+static int cdns_wdt_suspend(struct device *dev)
 {
 	struct cdns_wdt *wdt = dev_get_drvdata(dev);
 
@@ -377,7 +378,7 @@ static int __maybe_unused cdns_wdt_suspend(struct device *dev)
  * @dev: handle to the device structure.
  * Return: 0 on success, errno otherwise.
  */
-static int __maybe_unused cdns_wdt_resume(struct device *dev)
+static int cdns_wdt_resume(struct device *dev)
 {
 	int ret;
 	struct cdns_wdt *wdt = dev_get_drvdata(dev);
@@ -394,7 +395,7 @@ static int __maybe_unused cdns_wdt_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(cdns_wdt_pm_ops, cdns_wdt_suspend, cdns_wdt_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(cdns_wdt_pm_ops, cdns_wdt_suspend, cdns_wdt_resume);
 
 static const struct of_device_id cdns_wdt_of_match[] = {
 	{ .compatible = "cdns,wdt-r1p2", },
@@ -408,7 +409,7 @@ static struct platform_driver cdns_wdt_driver = {
 	.driver		= {
 		.name	= "cdns-wdt",
 		.of_match_table = cdns_wdt_of_match,
-		.pm	= &cdns_wdt_pm_ops,
+		.pm	= pm_sleep_ptr(&cdns_wdt_pm_ops),
 	},
 };
 
diff --git a/drivers/watchdog/cgbc_wdt.c b/drivers/watchdog/cgbc_wdt.c
index 702b055ba6f4bf..a90e6af13bed30 100644
--- a/drivers/watchdog/cgbc_wdt.c
+++ b/drivers/watchdog/cgbc_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/limits.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #include <linux/mfd/cgbc.h>
@@ -38,19 +39,19 @@ static unsigned int timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. (>=0, default="
-		 __MODULE_STRING(CGBC_WDT_DEFAULT_TIMEOUT) ")");
+		 __stringify(CGBC_WDT_DEFAULT_TIMEOUT) ")");
 
 static unsigned int pretimeout = CGBC_WDT_DEFAULT_PRETIMEOUT;
 module_param(pretimeout, uint, 0);
 MODULE_PARM_DESC(pretimeout,
 		 "Watchdog pretimeout in seconds. (>=0, default="
-		 __MODULE_STRING(CGBC_WDT_DEFAULT_PRETIMEOUT) ")");
+		 __stringify(CGBC_WDT_DEFAULT_PRETIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct cgbc_wdt_data {
 	struct cgbc_device_data	*cgbc;
diff --git a/drivers/watchdog/da9052_wdt.c b/drivers/watchdog/da9052_wdt.c
index afb7887c3a1e86..959cba0a881951 100644
--- a/drivers/watchdog/da9052_wdt.c
+++ b/drivers/watchdog/da9052_wdt.c
@@ -17,6 +17,7 @@
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/jiffies.h>
+#include <linux/stringify.h>
 
 #include <linux/mfd/da9052/reg.h>
 #include <linux/mfd/da9052/da9052.h>
@@ -34,13 +35,13 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int timeout;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. (default = "
-	__MODULE_STRING(WDT_DEFAULT_TIMEOUT) ")");
+	__stringify(WDT_DEFAULT_TIMEOUT) ")");
 
 static const struct {
 	u8 reg_val;
diff --git a/drivers/watchdog/da9055_wdt.c b/drivers/watchdog/da9055_wdt.c
index 9d5a2009466f74..cbb87855bd6a45 100644
--- a/drivers/watchdog/da9055_wdt.c
+++ b/drivers/watchdog/da9055_wdt.c
@@ -15,6 +15,7 @@
 #include <linux/platform_device.h>
 #include <linux/watchdog.h>
 #include <linux/delay.h>
+#include <linux/stringify.h>
 
 #include <linux/mfd/da9055/core.h>
 #include <linux/mfd/da9055/reg.h>
@@ -23,7 +24,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 #define DA9055_DEF_TIMEOUT	4
 #define DA9055_TWDMIN		256
diff --git a/drivers/watchdog/da9062_wdt.c b/drivers/watchdog/da9062_wdt.c
index 4d558652e9e710..3d7bf2f351db1a 100644
--- a/drivers/watchdog/da9062_wdt.c
+++ b/drivers/watchdog/da9062_wdt.c
@@ -248,7 +248,7 @@ static int da9062_wdt_probe(struct platform_device *pdev)
 	return devm_watchdog_register_device(dev, &wdt->wdtdev);
 }
 
-static int __maybe_unused da9062_wdt_suspend(struct device *dev)
+static int da9062_wdt_suspend(struct device *dev)
 {
 	struct watchdog_device *wdd = dev_get_drvdata(dev);
 	struct da9062_watchdog *wdt = watchdog_get_drvdata(wdd);
@@ -262,7 +262,7 @@ static int __maybe_unused da9062_wdt_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused da9062_wdt_resume(struct device *dev)
+static int da9062_wdt_resume(struct device *dev)
 {
 	struct watchdog_device *wdd = dev_get_drvdata(dev);
 	struct da9062_watchdog *wdt = watchdog_get_drvdata(wdd);
@@ -276,14 +276,14 @@ static int __maybe_unused da9062_wdt_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(da9062_wdt_pm_ops,
+static DEFINE_SIMPLE_DEV_PM_OPS(da9062_wdt_pm_ops,
 			 da9062_wdt_suspend, da9062_wdt_resume);
 
 static struct platform_driver da9062_wdt_driver = {
 	.probe = da9062_wdt_probe,
 	.driver = {
 		.name = "da9062-watchdog",
-		.pm = &da9062_wdt_pm_ops,
+		.pm = pm_sleep_ptr(&da9062_wdt_pm_ops),
 		.of_match_table = da9062_compatible_id_table,
 	},
 };
diff --git a/drivers/watchdog/davinci_wdt.c b/drivers/watchdog/davinci_wdt.c
index 6ea0434f45c225..074e95cf2fe7b5 100644
--- a/drivers/watchdog/davinci_wdt.c
+++ b/drivers/watchdog/davinci_wdt.c
@@ -19,6 +19,7 @@
 #include <linux/device.h>
 #include <linux/clk.h>
 #include <linux/err.h>
+#include <linux/stringify.h>
 
 #define MODULE_NAME "DAVINCI-WDT: "
 
@@ -120,7 +121,7 @@ static unsigned int davinci_wdt_get_timeleft(struct watchdog_device *wdd)
 	u32 val;
 	struct davinci_wdt_device *davinci_wdt = watchdog_get_drvdata(wdd);
 
-	/* if timeout has occured then return 0 */
+	/* if timeout has occurred then return 0 */
 	val = ioread32(davinci_wdt->base + WDTCR);
 	if (val & WDFLAG)
 		return 0;
@@ -250,8 +251,8 @@ MODULE_DESCRIPTION("DaVinci Watchdog Driver");
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 		 "Watchdog heartbeat period in seconds from 1 to "
-		 __MODULE_STRING(MAX_HEARTBEAT) ", default "
-		 __MODULE_STRING(DEFAULT_HEARTBEAT));
+		 __stringify(MAX_HEARTBEAT) ", default "
+		 __stringify(DEFAULT_HEARTBEAT));
 
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:davinci-wdt");
diff --git a/drivers/watchdog/db8500_wdt.c b/drivers/watchdog/db8500_wdt.c
index 70ccea13288d87..7e11c28997ec6f 100644
--- a/drivers/watchdog/db8500_wdt.c
+++ b/drivers/watchdog/db8500_wdt.c
@@ -15,6 +15,7 @@
 #include <linux/uaccess.h>
 #include <linux/watchdog.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 
 #include <linux/mfd/db8500-prcmu.h>
 
@@ -27,13 +28,13 @@ static unsigned int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+				__stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int db8500_wdt_start(struct watchdog_device *wdd)
 {
diff --git a/drivers/watchdog/dw_wdt.c b/drivers/watchdog/dw_wdt.c
index 39031445d754b3..c1d40d7b2165f5 100644
--- a/drivers/watchdog/dw_wdt.c
+++ b/drivers/watchdog/dw_wdt.c
@@ -27,6 +27,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm.h>
 #include <linux/reset.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define WDOG_CONTROL_REG_OFFSET		    0x00
@@ -66,7 +67,7 @@ static const u32 dw_wdt_fix_tops[DW_WDT_NUM_TOPS] = {
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-		 "(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 "(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 enum dw_wdt_rmod {
 	DW_WDT_RMOD_RESET = 1,
diff --git a/drivers/watchdog/ebc-c384_wdt.c b/drivers/watchdog/ebc-c384_wdt.c
index 8ef4b0df3855f4..39cc1f7e1cdb0e 100644
--- a/drivers/watchdog/ebc-c384_wdt.c
+++ b/drivers/watchdog/ebc-c384_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/types.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define MODULE_NAME		"ebc-c384_wdt"
@@ -30,12 +31,12 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds (default="
-	__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+	__stringify(WATCHDOG_TIMEOUT) ")");
 
 static int ebc_c384_wdt_start(struct watchdog_device *wdev)
 {
diff --git a/drivers/watchdog/eurotechwdt.c b/drivers/watchdog/eurotechwdt.c
index 10c647b1226af5..4d671c4e148afd 100644
--- a/drivers/watchdog/eurotechwdt.c
+++ b/drivers/watchdog/eurotechwdt.c
@@ -56,6 +56,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 static unsigned long eurwdt_is_open;
@@ -77,7 +78,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  * Some symbolic names
diff --git a/drivers/watchdog/exar_wdt.c b/drivers/watchdog/exar_wdt.c
index db62c9797406f5..553be158a7252c 100644
--- a/drivers/watchdog/exar_wdt.c
+++ b/drivers/watchdog/exar_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DRV_NAME	"exar_wdt"
@@ -70,13 +71,13 @@ static int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. 1<=timeout<=15300, default="
-		 __MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+		 __stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int exar_sio_enter(const unsigned short config_port,
 			  const unsigned char key)
diff --git a/drivers/watchdog/f71808e_wdt.c b/drivers/watchdog/f71808e_wdt.c
index 6a16d3d0bb1e62..2790061a68f343 100644
--- a/drivers/watchdog/f71808e_wdt.c
+++ b/drivers/watchdog/f71808e_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/ioport.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DRVNAME "f71808e_wdt"
@@ -81,20 +82,20 @@ static int timeout = WATCHDOG_TIMEOUT;	/* default timeout in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. 1<= timeout <="
-			__MODULE_STRING(WATCHDOG_MAX_TIMEOUT) " (default="
-			__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+			__stringify(WATCHDOG_MAX_TIMEOUT) " (default="
+			__stringify(WATCHDOG_TIMEOUT) ")");
 
 static unsigned int pulse_width = WATCHDOG_PULSE_WIDTH;
 module_param(pulse_width, uint, 0);
 MODULE_PARM_DESC(pulse_width,
 	"Watchdog signal pulse width. 0(=level), 1, 25, 30, 125, 150, 5000 or 6000 ms"
-			" (default=" __MODULE_STRING(WATCHDOG_PULSE_WIDTH) ")");
+			" (default=" __stringify(WATCHDOG_PULSE_WIDTH) ")");
 
 static unsigned int f71862fg_pin = WATCHDOG_F71862FG_PIN;
 module_param(f71862fg_pin, uint, 0);
 MODULE_PARM_DESC(f71862fg_pin,
 	"Watchdog f71862fg reset output pin configuration. Choose pin 56 or 63"
-			" (default=" __MODULE_STRING(WATCHDOG_F71862FG_PIN)")");
+			" (default=" __stringify(WATCHDOG_F71862FG_PIN)")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0444);
diff --git a/drivers/watchdog/gef_wdt.c b/drivers/watchdog/gef_wdt.c
index bf6f733dfb5f7b..4d39588a49c325 100644
--- a/drivers/watchdog/gef_wdt.c
+++ b/drivers/watchdog/gef_wdt.c
@@ -34,6 +34,7 @@
 #include <linux/platform_device.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #include <sysdev/fsl_soc.h>
 
@@ -70,7 +71,7 @@ static DEFINE_SPINLOCK(gef_wdt_spinlock);
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 
 static int gef_wdt_toggle_wdc(int enabled_predicate, int field_shift)
diff --git a/drivers/watchdog/geodewdt.c b/drivers/watchdog/geodewdt.c
index 5b80ade1c6814a..437cc0a536545b 100644
--- a/drivers/watchdog/geodewdt.c
+++ b/drivers/watchdog/geodewdt.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/reboot.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #include <linux/cs5535.h>
 
@@ -34,13 +35,13 @@ static int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. 1<= timeout <=131, default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+				__stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static struct platform_device *geodewdt_platform_device;
 static unsigned long wdt_flags;
diff --git a/drivers/watchdog/gpio_wdt.c b/drivers/watchdog/gpio_wdt.c
index 1abc7d0b78a627..2cd79c8f60dcd8 100644
--- a/drivers/watchdog/gpio_wdt.c
+++ b/drivers/watchdog/gpio_wdt.c
@@ -11,13 +11,14 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/property.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 #define SOFT_TIMEOUT_MIN	1
 #define SOFT_TIMEOUT_DEF	60
diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c
index 8af1fad2de0bd3..18f486e553231e 100644
--- a/drivers/watchdog/hpwdt.c
+++ b/drivers/watchdog/hpwdt.c
@@ -20,6 +20,7 @@
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
 #include <linux/types.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 #ifdef CONFIG_HPWDT_NMI_DECODING
 #include <asm/nmi.h>
@@ -199,7 +200,7 @@ static int hpwdt_pretimeout(unsigned int ulReason, struct pt_regs *regs)
 	}
 
 	hex_byte_pack(panic_msg, nmistat);
-	nmi_panic(regs, panic_msg);
+	nmi_panic(regs, "%s", panic_msg);
 
 	return NMI_HANDLED;
 }
@@ -432,7 +433,7 @@ MODULE_PARM_DESC(timeout, "Alias of soft_margin");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		__stringify(WATCHDOG_NOWAYOUT) ")");
 
 module_param(kdumptimeout, int, 0444);
 MODULE_PARM_DESC(kdumptimeout, "Timeout applied for crash kernel transition in seconds");
diff --git a/drivers/watchdog/i6300esb.c b/drivers/watchdog/i6300esb.c
index a30835f547b32e..7f08d6d2ca7789 100644
--- a/drivers/watchdog/i6300esb.c
+++ b/drivers/watchdog/i6300esb.c
@@ -37,6 +37,7 @@
 #include <linux/ioport.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/stringify.h>
 
 /* Module and version information */
 #define ESB_MODULE_NAME "i6300ESB timer"
@@ -74,19 +75,19 @@
 #define ESB_HEARTBEAT_MIN	1
 #define ESB_HEARTBEAT_MAX	2046
 #define ESB_HEARTBEAT_DEFAULT	30
-#define ESB_HEARTBEAT_RANGE __MODULE_STRING(ESB_HEARTBEAT_MIN) \
-	"<heartbeat<" __MODULE_STRING(ESB_HEARTBEAT_MAX)
+#define ESB_HEARTBEAT_RANGE __stringify(ESB_HEARTBEAT_MIN) \
+	"<heartbeat<" __stringify(ESB_HEARTBEAT_MAX)
 static int heartbeat; /* in seconds */
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 	"Watchdog heartbeat in seconds. (" ESB_HEARTBEAT_RANGE
-	", default=" __MODULE_STRING(ESB_HEARTBEAT_DEFAULT) ")");
+	", default=" __stringify(ESB_HEARTBEAT_DEFAULT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /* internal variables */
 struct esb_dev {
@@ -264,13 +265,21 @@ static void esb_initdevice(struct esb_dev *edev)
 	 */
 	pci_write_config_word(edev->pdev, ESB_CONFIG_REG, 0x0003);
 
-	/* Check that the WDT isn't already locked */
+	/* Check the current state of the WDT */
 	pci_read_config_byte(edev->pdev, ESB_LOCK_REG, &val1);
 	if (val1 & ESB_WDT_LOCK)
 		dev_warn(&edev->pdev->dev, "nowayout already set\n");
 
-	/* Set the timer to watchdog mode and disable it for now */
-	pci_write_config_byte(edev->pdev, ESB_LOCK_REG, 0x00);
+	if (val1 & ESB_WDT_ENABLE) {
+		/*
+		 * The watchdog is already running, e.g. enabled by
+		 * firmware. Do not stop it, just mark it as running.
+		 */
+		set_bit(WDOG_HW_RUNNING, &edev->wdd.status);
+	} else {
+		/* Set the timer to watchdog mode and disable it for now */
+		pci_write_config_byte(edev->pdev, ESB_LOCK_REG, 0x00);
+	}
 
 	/* Check if the watchdog was previously triggered */
 	esb_unlock_registers(edev);
@@ -301,7 +310,10 @@ static int esb_probe(struct pci_dev *pdev,
 	if (!esb_getdevice(edev))
 		return -ENODEV;
 
-	/* Initialize the watchdog and make sure it does not run */
+	/*
+	 * Initialize the watchdog, keeping it running if it was already
+	 * started, e.g. by firmware.
+	 */
 	edev->wdd.info = &esb_info;
 	edev->wdd.ops = &esb_ops;
 	edev->wdd.min_timeout = ESB_HEARTBEAT_MIN;
diff --git a/drivers/watchdog/iTCO_wdt.c b/drivers/watchdog/iTCO_wdt.c
index 1dec9cd155b11e..0a8bf28f57688b 100644
--- a/drivers/watchdog/iTCO_wdt.c
+++ b/drivers/watchdog/iTCO_wdt.c
@@ -60,6 +60,7 @@
 #include <linux/ioport.h>		/* For io-port access */
 #include <linux/uaccess.h>		/* For copy_to_user/put_user/... */
 #include <linux/io.h>			/* For inb/outb/... */
+#include <linux/stringify.h>		/* For __stringify */
 #include <linux/platform_data/itco_wdt.h>
 #include <linux/mfd/intel_pmc_bxt.h>
 
@@ -115,13 +116,13 @@ static int heartbeat = WATCHDOG_TIMEOUT;  /* in seconds */
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog timeout in seconds. "
 	"5..76 (TCO v1) or 3..614 (TCO v2), default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+				__stringify(WATCHDOG_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int turn_SMI_watchdog_clear_off = 1;
 module_param(turn_SMI_watchdog_clear_off, int, 0);
diff --git a/drivers/watchdog/ib700wdt.c b/drivers/watchdog/ib700wdt.c
index 5ce6101d236d7a..3806d86eb36adf 100644
--- a/drivers/watchdog/ib700wdt.c
+++ b/drivers/watchdog/ib700wdt.c
@@ -41,6 +41,7 @@
 #include <linux/platform_device.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 static struct platform_device *ibwdt_platform_device;
@@ -96,13 +97,13 @@ static int timeout = WATCHDOG_TIMEOUT;	/* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. 0<= timeout <=30, default="
-		__MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+		__stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 
 /*
diff --git a/drivers/watchdog/ibmasr.c b/drivers/watchdog/ibmasr.c
index cf845f865945d6..53bd84e6d57f1b 100644
--- a/drivers/watchdog/ibmasr.c
+++ b/drivers/watchdog/ibmasr.c
@@ -21,6 +21,7 @@
 #include <linux/dmi.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 enum {
@@ -413,7 +414,7 @@ module_exit(ibmasr_exit);
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_DESCRIPTION("IBM Automatic Server Restart driver");
 MODULE_AUTHOR("Andrey Panin");
diff --git a/drivers/watchdog/ie6xx_wdt.c b/drivers/watchdog/ie6xx_wdt.c
index 5a7bb7e84653f5..eda515b917cc49 100644
--- a/drivers/watchdog/ie6xx_wdt.c
+++ b/drivers/watchdog/ie6xx_wdt.c
@@ -17,6 +17,7 @@
 #include <linux/debugfs.h>
 #include <linux/uaccess.h>
 #include <linux/spinlock.h>
+#include <linux/stringify.h>
 
 #define DRIVER_NAME "ie6xx_wdt"
 
@@ -49,14 +50,14 @@ static unsigned int timeout = DEFAULT_TIME;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 		"Default Watchdog timer setting ("
-		__MODULE_STRING(DEFAULT_TIME) "s)."
+		__stringify(DEFAULT_TIME) "s)."
 		"The range is from 1 to 600");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-		__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static u8 resetmode = 0x10;
 module_param(resetmode, byte, 0);
diff --git a/drivers/watchdog/imgpdc_wdt.c b/drivers/watchdog/imgpdc_wdt.c
index 65cc8396aa5958..bb6227c40de433 100644
--- a/drivers/watchdog/imgpdc_wdt.c
+++ b/drivers/watchdog/imgpdc_wdt.c
@@ -43,6 +43,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* registers */
@@ -71,12 +72,12 @@
 static int heartbeat;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeats in seconds "
-	"(default=" __MODULE_STRING(PDC_WDT_DEF_TIMEOUT) ")");
+	"(default=" __stringify(PDC_WDT_DEF_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct pdc_wdt_dev {
 	struct watchdog_device wdt_dev;
diff --git a/drivers/watchdog/imx2_wdt.c b/drivers/watchdog/imx2_wdt.c
index 4b3a192ee3e825..58aa5f999f4feb 100644
--- a/drivers/watchdog/imx2_wdt.c
+++ b/drivers/watchdog/imx2_wdt.c
@@ -29,6 +29,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DRIVER_NAME "imx2-wdt"
@@ -79,12 +80,12 @@ struct imx2_wdt_device {
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds (default="
-				__MODULE_STRING(IMX2_WDT_DEFAULT_TIME) ")");
+				__stringify(IMX2_WDT_DEFAULT_TIME) ")");
 
 static const struct watchdog_info imx2_wdt_info = {
 	.identity = "imx2+ watchdog",
diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c
index 855dc9d5083a53..f61936194aea82 100644
--- a/drivers/watchdog/imx7ulp_wdt.c
+++ b/drivers/watchdog/imx7ulp_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/reboot.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define WDOG_CS			0x0
@@ -51,7 +52,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0000);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct imx_wdt_hw_feature {
 	bool prescaler_enable;
diff --git a/drivers/watchdog/imx_sc_wdt.c b/drivers/watchdog/imx_sc_wdt.c
index 1280b9b1ec2ac4..fc39f9db4c06f5 100644
--- a/drivers/watchdog/imx_sc_wdt.c
+++ b/drivers/watchdog/imx_sc_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/moduleparam.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DEFAULT_TIMEOUT 60
@@ -39,7 +40,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0000);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct imx_sc_wdt_device {
 	struct watchdog_device wdd;
diff --git a/drivers/watchdog/indydog.c b/drivers/watchdog/indydog.c
index d3092d2613452c..5268b1feb7ce77 100644
--- a/drivers/watchdog/indydog.c
+++ b/drivers/watchdog/indydog.c
@@ -22,6 +22,7 @@
 #include <linux/reboot.h>
 #include <linux/init.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 #include <asm/sgi/mc.h>
 
 static unsigned long indydog_alive;
@@ -33,7 +34,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static void indydog_start(void)
 {
diff --git a/drivers/watchdog/intel_oc_wdt.c b/drivers/watchdog/intel_oc_wdt.c
index a39892c10770eb..d1e9a3f62369f5 100644
--- a/drivers/watchdog/intel_oc_wdt.c
+++ b/drivers/watchdog/intel_oc_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define INTEL_OC_WDT_TOV		GENMASK(9, 0)
@@ -48,12 +49,12 @@ struct intel_oc_wdt {
 static int heartbeat;
 module_param(heartbeat, uint, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeats in seconds. (default="
-		 __MODULE_STRING(WDT_HEARTBEAT) ")");
+		 __stringify(WDT_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int intel_oc_wdt_start(struct watchdog_device *wdd)
 {
diff --git a/drivers/watchdog/it87_wdt.c b/drivers/watchdog/it87_wdt.c
index 1d9f8591f38d87..087250ab036a93 100644
--- a/drivers/watchdog/it87_wdt.c
+++ b/drivers/watchdog/it87_wdt.c
@@ -29,6 +29,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/printk.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -105,13 +106,13 @@ static bool nowayout = DEFAULT_NOWAYOUT;
 
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds, default="
-		__MODULE_STRING(DEFAULT_TIMEOUT));
+		__stringify(DEFAULT_TIMEOUT));
 module_param(testmode, int, 0);
 MODULE_PARM_DESC(testmode, "Watchdog test mode (1 = no reboot), default="
-		__MODULE_STRING(DEFAULT_TESTMODE));
+		__stringify(DEFAULT_TESTMODE));
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started, default="
-		__MODULE_STRING(WATCHDOG_NOWAYOUT));
+		__stringify(WATCHDOG_NOWAYOUT));
 
 /* Superio Chip */
 
diff --git a/drivers/watchdog/jz4740_wdt.c b/drivers/watchdog/jz4740_wdt.c
index 395bde79e29201..9dec1b82241014 100644
--- a/drivers/watchdog/jz4740_wdt.c
+++ b/drivers/watchdog/jz4740_wdt.c
@@ -19,6 +19,7 @@
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 
 #define DEFAULT_HEARTBEAT 5
 #define MAX_HEARTBEAT     2048
@@ -27,14 +28,14 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned int heartbeat = DEFAULT_HEARTBEAT;
 module_param(heartbeat, uint, 0);
 MODULE_PARM_DESC(heartbeat,
 		"Watchdog heartbeat period in seconds from 1 to "
-		__MODULE_STRING(MAX_HEARTBEAT) ", default "
-		__MODULE_STRING(DEFAULT_HEARTBEAT));
+		__stringify(MAX_HEARTBEAT) ", default "
+		__stringify(DEFAULT_HEARTBEAT));
 
 struct jz4740_wdt_drvdata {
 	struct watchdog_device wdt;
diff --git a/drivers/watchdog/keembay_wdt.c b/drivers/watchdog/keembay_wdt.c
index 7c8e53921e4079..76246a7714ca43 100644
--- a/drivers/watchdog/keembay_wdt.c
+++ b/drivers/watchdog/keembay_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/reboot.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* Non-secure watchdog register offsets */
@@ -39,12 +40,12 @@
 static unsigned int timeout = WDT_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout period in seconds (default = "
-		 __MODULE_STRING(WDT_TIMEOUT) ")");
+		 __stringify(WDT_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default = "
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct keembay_wdt {
 	struct watchdog_device	wdd;
@@ -250,7 +251,7 @@ static int keembay_wdt_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int __maybe_unused keembay_wdt_suspend(struct device *dev)
+static int keembay_wdt_suspend(struct device *dev)
 {
 	struct keembay_wdt *wdt = dev_get_drvdata(dev);
 
@@ -260,7 +261,7 @@ static int __maybe_unused keembay_wdt_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused keembay_wdt_resume(struct device *dev)
+static int keembay_wdt_resume(struct device *dev)
 {
 	struct keembay_wdt *wdt = dev_get_drvdata(dev);
 
@@ -270,7 +271,7 @@ static int __maybe_unused keembay_wdt_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(keembay_wdt_pm_ops, keembay_wdt_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(keembay_wdt_pm_ops, keembay_wdt_suspend,
 			 keembay_wdt_resume);
 
 static const struct of_device_id keembay_wdt_match[] = {
@@ -284,7 +285,7 @@ static struct platform_driver keembay_wdt_driver = {
 	.driver	= {
 		.name		= "keembay_wdt",
 		.of_match_table	= keembay_wdt_match,
-		.pm		= &keembay_wdt_pm_ops,
+		.pm		= pm_sleep_ptr(&keembay_wdt_pm_ops),
 	},
 };
 
diff --git a/drivers/watchdog/kempld_wdt.c b/drivers/watchdog/kempld_wdt.c
index e6c7a290668043..b4cde48d2b10a7 100644
--- a/drivers/watchdog/kempld_wdt.c
+++ b/drivers/watchdog/kempld_wdt.c
@@ -21,6 +21,7 @@
 #include <linux/uaccess.h>
 #include <linux/watchdog.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/mfd/kempld.h>
 
 #define KEMPLD_WDT_STAGE_TIMEOUT(x)	(0x1b + (x) * 4)
@@ -85,19 +86,19 @@ static unsigned int timeout = DEFAULT_TIMEOUT;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. (>=0, default="
-	__MODULE_STRING(DEFAULT_TIMEOUT) ")");
+	__stringify(DEFAULT_TIMEOUT) ")");
 
 static unsigned int pretimeout = DEFAULT_PRETIMEOUT;
 module_param(pretimeout, uint, 0);
 MODULE_PARM_DESC(pretimeout,
 	"Watchdog pretimeout in seconds. (>=0, default="
-	__MODULE_STRING(DEFAULT_PRETIMEOUT) ")");
+	__stringify(DEFAULT_PRETIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int kempld_wdt_set_stage_action(struct kempld_wdt_data *wdt_data,
 					struct kempld_wdt_stage *stage,
diff --git a/drivers/watchdog/lenovo_se10_wdt.c b/drivers/watchdog/lenovo_se10_wdt.c
index 503e220263f95a..1eaf86c85cc0e7 100644
--- a/drivers/watchdog/lenovo_se10_wdt.c
+++ b/drivers/watchdog/lenovo_se10_wdt.c
@@ -10,6 +10,7 @@
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
 #include <linux/string.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -48,13 +49,13 @@ static int timeout; /* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. 1 <= timeout <= 255, default="
-		 __MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+		 __stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct se10_wdt {
 	struct watchdog_device wdd;
diff --git a/drivers/watchdog/lenovo_se30_wdt.c b/drivers/watchdog/lenovo_se30_wdt.c
index 1c73bb7eeeeed1..127ea9a1933ffb 100644
--- a/drivers/watchdog/lenovo_se30_wdt.c
+++ b/drivers/watchdog/lenovo_se30_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define IOREGION_OFFSET	4 /* Use EC port 1 */
@@ -28,13 +29,13 @@ static int timeout; /* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. 1 <= timeout <= 255, default="
-		 __MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+		 __stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 #define LNV_SE30_NAME	"lenovo-se30-wdt"
 #define LNV_SE30_ID	0x0110
diff --git a/drivers/watchdog/lenovo_se30g2_se60_wdt.c b/drivers/watchdog/lenovo_se30g2_se60_wdt.c
index f31ecdb82ad052..c87abdb1a1b2d1 100644
--- a/drivers/watchdog/lenovo_se30g2_se60_wdt.c
+++ b/drivers/watchdog/lenovo_se30g2_se60_wdt.c
@@ -10,6 +10,7 @@
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
 #include <linux/string.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -37,13 +38,13 @@ static int timeout; /* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. 1 <= timeout <= 255, default="
-		 __MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+		 __stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct se_30g2_60_wdt {
 	struct watchdog_device wdd;
diff --git a/drivers/watchdog/loongson1_wdt.c b/drivers/watchdog/loongson1_wdt.c
index 2417519717cc48..7f8de11f1b4ab1 100644
--- a/drivers/watchdog/loongson1_wdt.c
+++ b/drivers/watchdog/loongson1_wdt.c
@@ -9,6 +9,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* Loongson Watchdog Register Definitions */
@@ -19,12 +20,12 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned int heartbeat;
 module_param(heartbeat, uint, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. (default="
-		 __MODULE_STRING(DEFAULT_HEARTBEAT) ")");
+		 __stringify(DEFAULT_HEARTBEAT) ")");
 
 struct ls1x_wdt_pdata {
 	u32 timer_offset;
diff --git a/drivers/watchdog/lpc18xx_wdt.c b/drivers/watchdog/lpc18xx_wdt.c
index 3e8e80bbcb93b9..633119e4b126d8 100644
--- a/drivers/watchdog/lpc18xx_wdt.c
+++ b/drivers/watchdog/lpc18xx_wdt.c
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* Registers */
@@ -41,12 +42,12 @@
 static int heartbeat;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeats in seconds (default="
-		 __MODULE_STRING(LPC18XX_WDT_DEF_TIMEOUT) ")");
+		 __stringify(LPC18XX_WDT_DEF_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct lpc18xx_wdt_dev {
 	struct watchdog_device	wdt_dev;
diff --git a/drivers/watchdog/ma35d1_wdt.c b/drivers/watchdog/ma35d1_wdt.c
index 119d58f338bf0a..ede4bee594f1f3 100644
--- a/drivers/watchdog/ma35d1_wdt.c
+++ b/drivers/watchdog/ma35d1_wdt.c
@@ -20,6 +20,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_wakeirq.h>
 #include <linux/property.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -355,7 +356,7 @@ MODULE_PARM_DESC(timeout, "Watchdog heartbeat in seconds");
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(
 	nowayout,
-	"Watchdog cannot be stopped once started (default=" __MODULE_STRING(
+	"Watchdog cannot be stopped once started (default=" __stringify(
 		WATCHDOG_NOWAYOUT) ")");
 
 MODULE_AUTHOR("Zi-Yu Chen <zychennvt@gmail.com>");
diff --git a/drivers/watchdog/max63xx_wdt.c b/drivers/watchdog/max63xx_wdt.c
index e9073df1a7de32..17b19dfdad16f0 100644
--- a/drivers/watchdog/max63xx_wdt.c
+++ b/drivers/watchdog/max63xx_wdt.c
@@ -23,6 +23,7 @@
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/property.h>
+#include <linux/stringify.h>
 
 #define DEFAULT_HEARTBEAT 60
 #define MAX_HEARTBEAT     60
@@ -64,7 +65,7 @@ struct max63xx_wdt {
  * delay (selected with the "nodelay" parameter).
  *
  * I also decided to remove from the tables any timeout smaller than a
- * second, as it looked completly overkill...
+ * second, as it looked completely overkill...
  */
 
 /* Timeouts in second */
@@ -283,12 +284,12 @@ MODULE_DESCRIPTION("max63xx Watchdog Driver");
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 		 "Watchdog heartbeat period in seconds from 1 to "
-		 __MODULE_STRING(MAX_HEARTBEAT) ", default "
-		 __MODULE_STRING(DEFAULT_HEARTBEAT));
+		 __stringify(MAX_HEARTBEAT) ", default "
+		 __stringify(DEFAULT_HEARTBEAT));
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 module_param(nodelay, int, 0);
 MODULE_PARM_DESC(nodelay,
diff --git a/drivers/watchdog/max77620_wdt.c b/drivers/watchdog/max77620_wdt.c
index ed5a98cea525db..79148cbad4f26b 100644
--- a/drivers/watchdog/max77620_wdt.c
+++ b/drivers/watchdog/max77620_wdt.c
@@ -18,6 +18,7 @@
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
@@ -255,7 +256,7 @@ MODULE_DESCRIPTION("Max77620 watchdog timer driver");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
 MODULE_AUTHOR("Luca Ceresoli <luca.ceresoli@bootlin.com>");
diff --git a/drivers/watchdog/mena21_wdt.c b/drivers/watchdog/mena21_wdt.c
index 99d2359d5a8a76..a762462c25c543 100644
--- a/drivers/watchdog/mena21_wdt.c
+++ b/drivers/watchdog/mena21_wdt.c
@@ -17,6 +17,7 @@
 #include <linux/delay.h>
 #include <linux/bitops.h>
 #include <linux/of.h>
+#include <linux/stringify.h>
 
 #define NUM_GPIOS 6
 
@@ -37,7 +38,7 @@ struct a21_wdt_drv {
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			    __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			    __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned int a21_wdt_get_bootstatus(struct a21_wdt_drv *drv)
 {
diff --git a/drivers/watchdog/menf21bmc_wdt.c b/drivers/watchdog/menf21bmc_wdt.c
index 81ebdfc371f4db..bb8eda18fbad6b 100644
--- a/drivers/watchdog/menf21bmc_wdt.c
+++ b/drivers/watchdog/menf21bmc_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/watchdog.h>
 #include <linux/platform_device.h>
 #include <linux/i2c.h>
+#include <linux/stringify.h>
 
 #define DEVNAME "menf21bmc_wdt"
 
@@ -28,7 +29,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct menf21bmc_wdt {
 	struct watchdog_device wdt;
diff --git a/drivers/watchdog/menz69_wdt.c b/drivers/watchdog/menz69_wdt.c
index 3fe23451135d70..962c6867bc0734 100644
--- a/drivers/watchdog/menz69_wdt.c
+++ b/drivers/watchdog/menz69_wdt.c
@@ -8,6 +8,7 @@
 #include <linux/kernel.h>
 #include <linux/mcb.h>
 #include <linux/module.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 struct men_z069_drv {
@@ -29,7 +30,7 @@ struct men_z069_drv {
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			    __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			    __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int men_z069_wdt_start(struct watchdog_device *wdt)
 {
diff --git a/drivers/watchdog/meson_gxbb_wdt.c b/drivers/watchdog/meson_gxbb_wdt.c
index a48622d11ad728..fb276060e8314e 100644
--- a/drivers/watchdog/meson_gxbb_wdt.c
+++ b/drivers/watchdog/meson_gxbb_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -31,12 +32,12 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned int timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout, "Watchdog heartbeat in seconds="
-		 __MODULE_STRING(DEFAULT_TIMEOUT) ")");
+		 __stringify(DEFAULT_TIMEOUT) ")");
 
 struct meson_gxbb_wdt {
 	void __iomem *reg_base;
diff --git a/drivers/watchdog/meson_wdt.c b/drivers/watchdog/meson_wdt.c
index 44db4ebc8599f6..80a4f877133596 100644
--- a/drivers/watchdog/meson_wdt.c
+++ b/drivers/watchdog/meson_wdt.c
@@ -15,6 +15,7 @@
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
 #include <linux/property.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -218,7 +219,7 @@ MODULE_PARM_DESC(timeout, "Watchdog heartbeat in seconds");
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Carlo Caione <carlo@caione.org>");
diff --git a/drivers/watchdog/mixcomwd.c b/drivers/watchdog/mixcomwd.c
index 1ecd5c48a005a1..db358df2f22287 100644
--- a/drivers/watchdog/mixcomwd.c
+++ b/drivers/watchdog/mixcomwd.c
@@ -52,6 +52,7 @@
 #include <linux/timer.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/stringify.h>
 
 /*
  * We have two types of cards that can be probed:
@@ -107,7 +108,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static void mixcomwd_ping(void)
 {
diff --git a/drivers/watchdog/mpc8xxx_wdt.c b/drivers/watchdog/mpc8xxx_wdt.c
index a4b497ecfa2051..0d5a360b56292b 100644
--- a/drivers/watchdog/mpc8xxx_wdt.c
+++ b/drivers/watchdog/mpc8xxx_wdt.c
@@ -22,6 +22,7 @@
 #include <linux/watchdog.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 #include <sysdev/fsl_soc.h>
 
 #define WATCHDOG_TIMEOUT 10
@@ -57,7 +58,7 @@ static u16 timeout;
 module_param(timeout, ushort, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. (1<timeout<65535, default="
-	__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+	__stringify(WATCHDOG_TIMEOUT) ")");
 
 static bool reset = 1;
 module_param(reset, bool, 0);
@@ -67,7 +68,7 @@ MODULE_PARM_DESC(reset,
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-		 "(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 "(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static void mpc8xxx_wdt_keepalive(struct mpc8xxx_wdt_ddata *ddata)
 {
diff --git a/drivers/watchdog/msc313e_wdt.c b/drivers/watchdog/msc313e_wdt.c
index d62586f6e09d0a..0a68be7a217b6d 100644
--- a/drivers/watchdog/msc313e_wdt.c
+++ b/drivers/watchdog/msc313e_wdt.c
@@ -140,8 +140,8 @@ static int msc313e_wdt_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->clk);
 	}
 
-	priv->wdev.info = &msc313e_wdt_ident,
-	priv->wdev.ops = &msc313e_wdt_ops,
+	priv->wdev.info = &msc313e_wdt_ident;
+	priv->wdev.ops = &msc313e_wdt_ops;
 	priv->wdev.parent = dev;
 	priv->wdev.min_timeout = MSC313E_WDT_MIN_TIMEOUT;
 	rate = clk_get_rate(priv->clk);
@@ -184,7 +184,7 @@ static int msc313e_wdt_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int __maybe_unused msc313e_wdt_suspend(struct device *dev)
+static int msc313e_wdt_suspend(struct device *dev)
 {
 	struct msc313e_wdt_priv *priv = dev_get_drvdata(dev);
 
@@ -194,7 +194,7 @@ static int __maybe_unused msc313e_wdt_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused msc313e_wdt_resume(struct device *dev)
+static int msc313e_wdt_resume(struct device *dev)
 {
 	struct msc313e_wdt_priv *priv = dev_get_drvdata(dev);
 	int ret = 0;
@@ -208,13 +208,13 @@ static int __maybe_unused msc313e_wdt_resume(struct device *dev)
 	return ret;
 }
 
-static SIMPLE_DEV_PM_OPS(msc313e_wdt_pm_ops, msc313e_wdt_suspend, msc313e_wdt_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(msc313e_wdt_pm_ops, msc313e_wdt_suspend, msc313e_wdt_resume);
 
 static struct platform_driver msc313e_wdt_driver = {
 	.driver = {
 		.name = "msc313e-wdt",
 		.of_match_table = msc313e_wdt_of_match,
-		.pm = &msc313e_wdt_pm_ops,
+		.pm = pm_sleep_ptr(&msc313e_wdt_pm_ops),
 	},
 	.probe = msc313e_wdt_probe,
 };
diff --git a/drivers/watchdog/mt7621_wdt.c b/drivers/watchdog/mt7621_wdt.c
index db47b131f78062..9e0ebec43c8887 100644
--- a/drivers/watchdog/mt7621_wdt.c
+++ b/drivers/watchdog/mt7621_wdt.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 
 #define SYSC_RSTSTAT			0x38
 #define WDT_RST_CAUSE			BIT(1)
@@ -41,7 +42,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static inline void rt_wdt_w32(void __iomem *base, unsigned int reg, u32 val)
 {
diff --git a/drivers/watchdog/mtk_wdt.c b/drivers/watchdog/mtk_wdt.c
index d9c30e4c80e3ff..1630ab65d59329 100644
--- a/drivers/watchdog/mtk_wdt.c
+++ b/drivers/watchdog/mtk_wdt.c
@@ -12,6 +12,7 @@
 #include <dt-bindings/reset/mt2712-resets.h>
 #include <dt-bindings/reset/mediatek,mt6735-wdt.h>
 #include <dt-bindings/reset/mediatek,mt6795-resets.h>
+#include <dt-bindings/reset/mediatek,mt8167-wdt.h>
 #include <dt-bindings/reset/mt7986-resets.h>
 #include <dt-bindings/reset/mt8183-resets.h>
 #include <dt-bindings/reset/mt8186-resets.h>
@@ -31,10 +32,12 @@
 #include <linux/types.h>
 #include <linux/watchdog.h>
 #include <linux/interrupt.h>
+#include <linux/stringify.h>
 
 #define WDT_MAX_TIMEOUT		31
 #define WDT_MIN_TIMEOUT		2
 #define WDT_LENGTH_TIMEOUT(n)	((n) << 5)
+#define WDT_IRQ_LEVEL_SYNC_US	70
 
 #define WDT_LENGTH		0x04
 #define WDT_LENGTH_KEY		0x8
@@ -49,6 +52,7 @@
 #define WDT_MODE_EXRST_EN	(1 << 2)
 #define WDT_MODE_IRQ_EN		(1 << 3)
 #define WDT_MODE_AUTO_START	(1 << 4)
+#define WDT_MODE_IRQ_LEVEL_EN	(1 << 5)
 #define WDT_MODE_DUAL_EN	(1 << 6)
 #define WDT_MODE_CNT_SEL	(1 << 8)
 #define WDT_MODE_KEY		0x22000000
@@ -64,6 +68,7 @@
 #define DRV_NAME		"mtk-wdt"
 #define DRV_VERSION		"1.0"
 
+#define MT6589_TOPRGU_SW_RST_NUM	12
 #define MT7988_TOPRGU_SW_RST_NUM	24
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
@@ -72,22 +77,29 @@ static unsigned int timeout;
 struct mtk_wdt_dev {
 	struct watchdog_device wdt_dev;
 	void __iomem *wdt_base;
-	spinlock_t lock; /* protects WDT_SWSYSRST reg */
+	spinlock_t lock; /* protects WDT_MODE and WDT_SWSYSRST reg */
 	struct reset_controller_dev rcdev;
 	bool disable_wdt_extrst;
 	bool reset_by_toprgu;
 	bool has_swsysrst_en;
+	const u8 *toprgu_sw_rst_tb;
+	int toprgu_sw_rst_num;
 };
 
 struct mtk_wdt_data {
-	int toprgu_sw_rst_num;
-	bool has_swsysrst_en;
+	const u8 *toprgu_sw_rst_tb;
+	const int toprgu_sw_rst_num;
+	const bool has_swsysrst_en;
 };
 
 static const struct mtk_wdt_data mt2712_data = {
 	.toprgu_sw_rst_num = MT2712_TOPRGU_SW_RST_NUM,
 };
 
+static const struct mtk_wdt_data mt6589_data = {
+	.toprgu_sw_rst_num = MT6589_TOPRGU_SW_RST_NUM,
+};
+
 static const struct mtk_wdt_data mt6735_data = {
 	.toprgu_sw_rst_num = MT6735_TOPRGU_RST_NUM,
 };
@@ -125,6 +137,29 @@ static const struct mtk_wdt_data mt8195_data = {
 	.toprgu_sw_rst_num = MT8195_TOPRGU_SW_RST_NUM,
 };
 
+static const u8 mt8167_toprgu_sw_rst_tb[] = {
+	[MT8167_TOPRGU_DDRPHY_FLASH_RST]	= 0,
+	[MT8167_TOPRGU_AUD_PAD_RST]		= 1,
+	[MT8167_TOPRGU_MM_RST]			= 2,
+	[MT8167_TOPRGU_MFG_RST]			= 3,
+	[MT8167_TOPRGU_MDSYS_RST]		= 4,
+	[MT8167_TOPRGU_CONN_RST]		= 5,
+	[MT8167_TOPRGU_PAD2CAM_DIG_MIPI_RX_RST]	= 6,
+	[MT8167_TOPRGU_DIG_MIPI_TX_RST]		= 7,
+	[MT8167_TOPRGU_SPI_PAD_MACRO_RST]	= 8,
+	/* The data sheet describes bit 9 as "reserved, unused" */
+	[MT8167_TOPRGU_APMIXED_RST]		= 10,
+	[MT8167_TOPRGU_VDEC_RST]		= 11,
+	[MT8167_TOPRGU_CONN_MCU_RST]		= 12,
+	[MT8167_TOPRGU_EFUSE_RST]		= 13,
+	[MT8167_TOPRGU_PWRAP_SPICTL_RST]	= 14
+};
+
+static const struct mtk_wdt_data mt8167_data = {
+	.toprgu_sw_rst_tb = mt8167_toprgu_sw_rst_tb,
+	.toprgu_sw_rst_num = ARRAY_SIZE(mt8167_toprgu_sw_rst_tb),
+};
+
 /**
  * toprgu_reset_sw_en_unlocked() - enable/disable software control for reset bit
  * @data: Pointer to instance of driver data.
@@ -155,6 +190,15 @@ static int toprgu_reset_update(struct reset_controller_dev *rcdev,
 	struct mtk_wdt_dev *data =
 		 container_of(rcdev, struct mtk_wdt_dev, rcdev);
 
+	if (data->toprgu_sw_rst_tb) {
+		if (id >= data->toprgu_sw_rst_num) {
+			dev_err(rcdev->dev, "Invalid reset ID: %lu (>=%u)\n",
+				id, data->toprgu_sw_rst_num);
+			return -EINVAL;
+		}
+		id = data->toprgu_sw_rst_tb[id];
+	}
+
 	spin_lock_irqsave(&data->lock, flags);
 
 	if (assert && data->has_swsysrst_en)
@@ -212,8 +256,6 @@ static int toprgu_register_reset_controller(struct platform_device *pdev,
 	int ret;
 	struct mtk_wdt_dev *mtk_wdt = platform_get_drvdata(pdev);
 
-	spin_lock_init(&mtk_wdt->lock);
-
 	mtk_wdt->rcdev.owner = THIS_MODULE;
 	mtk_wdt->rcdev.nr_resets = rst_num;
 	mtk_wdt->rcdev.ops = &toprgu_reset_ops;
@@ -285,29 +327,19 @@ static int mtk_wdt_set_timeout(struct watchdog_device *wdt_dev,
 	return 0;
 }
 
-static void mtk_wdt_init(struct watchdog_device *wdt_dev)
-{
-	struct mtk_wdt_dev *mtk_wdt = watchdog_get_drvdata(wdt_dev);
-	void __iomem *wdt_base;
-
-	wdt_base = mtk_wdt->wdt_base;
-
-	if (readl(wdt_base + WDT_MODE) & WDT_MODE_EN) {
-		set_bit(WDOG_HW_RUNNING, &wdt_dev->status);
-		mtk_wdt_set_timeout(wdt_dev, wdt_dev->timeout);
-	}
-}
-
 static int mtk_wdt_stop(struct watchdog_device *wdt_dev)
 {
 	struct mtk_wdt_dev *mtk_wdt = watchdog_get_drvdata(wdt_dev);
 	void __iomem *wdt_base = mtk_wdt->wdt_base;
+	unsigned long flags;
 	u32 reg;
 
+	spin_lock_irqsave(&mtk_wdt->lock, flags);
 	reg = readl(wdt_base + WDT_MODE);
 	reg &= ~WDT_MODE_EN;
 	reg |= WDT_MODE_KEY;
 	iowrite32(reg, wdt_base + WDT_MODE);
+	spin_unlock_irqrestore(&mtk_wdt->lock, flags);
 
 	return 0;
 }
@@ -317,12 +349,14 @@ static int mtk_wdt_start(struct watchdog_device *wdt_dev)
 	u32 reg;
 	struct mtk_wdt_dev *mtk_wdt = watchdog_get_drvdata(wdt_dev);
 	void __iomem *wdt_base = mtk_wdt->wdt_base;
+	unsigned long flags;
 	int ret;
 
 	ret = mtk_wdt_set_timeout(wdt_dev, wdt_dev->timeout);
 	if (ret < 0)
 		return ret;
 
+	spin_lock_irqsave(&mtk_wdt->lock, flags);
 	reg = ioread32(wdt_base + WDT_MODE);
 	if (wdt_dev->pretimeout)
 		reg |= (WDT_MODE_IRQ_EN | WDT_MODE_DUAL_EN);
@@ -334,16 +368,39 @@ static int mtk_wdt_start(struct watchdog_device *wdt_dev)
 		reg |= WDT_MODE_CNT_SEL;
 	reg |= (WDT_MODE_EN | WDT_MODE_KEY);
 	iowrite32(reg, wdt_base + WDT_MODE);
+	spin_unlock_irqrestore(&mtk_wdt->lock, flags);
 
 	return 0;
 }
 
+static void mtk_wdt_init(struct watchdog_device *wdt_dev)
+{
+	struct mtk_wdt_dev *mtk_wdt = watchdog_get_drvdata(wdt_dev);
+	void __iomem *wdt_base;
+
+	wdt_base = mtk_wdt->wdt_base;
+
+	/*
+	 * The bootloader may have left it in dual mode, where a timeout only
+	 * raises the bark IRQ and the reset comes one timeout later. Apply
+	 * this driver's mode instead, as mtk_wdt_start() does.
+	 */
+	if (readl(wdt_base + WDT_MODE) & WDT_MODE_EN) {
+		set_bit(WDOG_HW_RUNNING, &wdt_dev->status);
+		mtk_wdt_start(wdt_dev);
+	}
+}
+
 static int mtk_wdt_set_pretimeout(struct watchdog_device *wdd,
 				  unsigned int timeout)
 {
 	struct mtk_wdt_dev *mtk_wdt = watchdog_get_drvdata(wdd);
 	void __iomem *wdt_base = mtk_wdt->wdt_base;
-	u32 reg = ioread32(wdt_base + WDT_MODE);
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&mtk_wdt->lock, flags);
+	reg = ioread32(wdt_base + WDT_MODE);
 
 	if (timeout && !wdd->pretimeout) {
 		wdd->pretimeout = wdd->timeout / 2;
@@ -352,19 +409,54 @@ static int mtk_wdt_set_pretimeout(struct watchdog_device *wdd,
 		wdd->pretimeout = 0;
 		reg &= ~(WDT_MODE_IRQ_EN | WDT_MODE_DUAL_EN);
 	} else {
+		spin_unlock_irqrestore(&mtk_wdt->lock, flags);
 		return 0;
 	}
 
 	reg |= WDT_MODE_KEY;
 	iowrite32(reg, wdt_base + WDT_MODE);
+	spin_unlock_irqrestore(&mtk_wdt->lock, flags);
 
 	return mtk_wdt_set_timeout(wdd, wdd->timeout);
 }
 
+static void mtk_wdt_deassert_irq(struct watchdog_device *wdd)
+{
+	struct mtk_wdt_dev *mtk_wdt = watchdog_get_drvdata(wdd);
+	void __iomem *wdt_base = mtk_wdt->wdt_base;
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&mtk_wdt->lock, flags);
+
+	reg = ioread32(wdt_base + WDT_MODE);
+	reg ^= WDT_MODE_IRQ_LEVEL_EN;
+	iowrite32(reg | WDT_MODE_KEY, wdt_base + WDT_MODE);
+
+	/*
+	 * Wait for two 32 kHz watchdog clock cycles so the IRQ level
+	 * change can propagate across the clock domain.
+	 */
+	udelay(WDT_IRQ_LEVEL_SYNC_US);
+
+	reg = ioread32(wdt_base + WDT_MODE);
+	reg ^= WDT_MODE_IRQ_LEVEL_EN;
+	iowrite32(reg | WDT_MODE_KEY, wdt_base + WDT_MODE);
+
+	/*
+	 * Flush the posted write before releasing the lock and notifying
+	 * the watchdog core.
+	 */
+	ioread32(wdt_base + WDT_MODE);
+
+	spin_unlock_irqrestore(&mtk_wdt->lock, flags);
+}
+
 static irqreturn_t mtk_wdt_isr(int irq, void *arg)
 {
 	struct watchdog_device *wdd = arg;
 
+	mtk_wdt_deassert_irq(wdd);
 	watchdog_notify_pretimeout(wdd);
 
 	return IRQ_HANDLED;
@@ -406,6 +498,8 @@ static int mtk_wdt_probe(struct platform_device *pdev)
 	if (!mtk_wdt)
 		return -ENOMEM;
 
+	spin_lock_init(&mtk_wdt->lock);
+
 	platform_set_drvdata(pdev, mtk_wdt);
 
 	mtk_wdt->wdt_base = devm_platform_ioremap_resource(pdev, 0);
@@ -414,8 +508,8 @@ static int mtk_wdt_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq_optional(pdev, 0);
 	if (irq > 0) {
-		err = devm_request_irq(&pdev->dev, irq, mtk_wdt_isr, 0, "wdt_bark",
-				       &mtk_wdt->wdt_dev);
+		err = devm_request_irq(&pdev->dev, irq, mtk_wdt_isr, IRQF_NO_AUTOEN,
+				       "wdt_bark", &mtk_wdt->wdt_dev);
 		if (err)
 			return err;
 
@@ -440,6 +534,12 @@ static int mtk_wdt_probe(struct platform_device *pdev)
 
 	watchdog_set_drvdata(&mtk_wdt->wdt_dev, mtk_wdt);
 
+	mtk_wdt->disable_wdt_extrst =
+		of_property_read_bool(dev->of_node, "mediatek,disable-extrst");
+
+	mtk_wdt->reset_by_toprgu =
+		of_property_read_bool(dev->of_node, "mediatek,reset-by-toprgu");
+
 	mtk_wdt_init(&mtk_wdt->wdt_dev);
 
 	watchdog_stop_on_reboot(&mtk_wdt->wdt_dev);
@@ -447,25 +547,24 @@ static int mtk_wdt_probe(struct platform_device *pdev)
 	if (unlikely(err))
 		return err;
 
+	if (irq > 0)
+		enable_irq(irq);
+
 	dev_info(dev, "Watchdog enabled (timeout=%d sec, nowayout=%d)\n",
 		 mtk_wdt->wdt_dev.timeout, nowayout);
 
 	wdt_data = of_device_get_match_data(dev);
 	if (wdt_data) {
+		mtk_wdt->toprgu_sw_rst_num = wdt_data->toprgu_sw_rst_num;
+		mtk_wdt->toprgu_sw_rst_tb = wdt_data->toprgu_sw_rst_tb;
+		mtk_wdt->has_swsysrst_en = wdt_data->has_swsysrst_en;
+
 		err = toprgu_register_reset_controller(pdev,
 						       wdt_data->toprgu_sw_rst_num);
 		if (err)
 			return err;
-
-		mtk_wdt->has_swsysrst_en = wdt_data->has_swsysrst_en;
 	}
 
-	mtk_wdt->disable_wdt_extrst =
-		of_property_read_bool(dev->of_node, "mediatek,disable-extrst");
-
-	mtk_wdt->reset_by_toprgu =
-		of_property_read_bool(dev->of_node, "mediatek,reset-by-toprgu");
-
 	return 0;
 }
 
@@ -493,11 +592,12 @@ static int mtk_wdt_resume(struct device *dev)
 
 static const struct of_device_id mtk_wdt_dt_ids[] = {
 	{ .compatible = "mediatek,mt2712-wdt", .data = &mt2712_data },
-	{ .compatible = "mediatek,mt6589-wdt" },
+	{ .compatible = "mediatek,mt6589-wdt", .data = &mt6589_data },
 	{ .compatible = "mediatek,mt6735-wdt", .data = &mt6735_data },
 	{ .compatible = "mediatek,mt6795-wdt", .data = &mt6795_data },
 	{ .compatible = "mediatek,mt7986-wdt", .data = &mt7986_data },
 	{ .compatible = "mediatek,mt7988-wdt", .data = &mt7988_data },
+	{ .compatible = "mediatek,mt8167-wdt", .data = &mt8167_data },
 	{ .compatible = "mediatek,mt8183-wdt", .data = &mt8183_data },
 	{ .compatible = "mediatek,mt8186-wdt", .data = &mt8186_data },
 	{ .compatible = "mediatek,mt8188-wdt", .data = &mt8188_data },
@@ -526,7 +626,7 @@ MODULE_PARM_DESC(timeout, "Watchdog heartbeat in seconds");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			__stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Matthias Brugger <matthias.bgg@gmail.com>");
diff --git a/drivers/watchdog/nct6694_wdt.c b/drivers/watchdog/nct6694_wdt.c
index bc3689bd4b6bb0..169a6736e9ea0e 100644
--- a/drivers/watchdog/nct6694_wdt.c
+++ b/drivers/watchdog/nct6694_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DEVICE_NAME "nct6694-wdt"
@@ -50,7 +51,7 @@ MODULE_PARM_DESC(pretimeout, "Watchdog pre-timeout in seconds");
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			   __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			   __stringify(WATCHDOG_NOWAYOUT) ")");
 
 enum {
 	NCT6694_ACTION_NONE = 0,
diff --git a/drivers/watchdog/ni903x_wdt.c b/drivers/watchdog/ni903x_wdt.c
index c72a9ee9cb8e19..68b7c485726b23 100644
--- a/drivers/watchdog/ni903x_wdt.c
+++ b/drivers/watchdog/ni903x_wdt.c
@@ -9,6 +9,7 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define NIWD_CONTROL	0x01
@@ -46,13 +47,13 @@ static unsigned int timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. (default="
-		 __MODULE_STRING(NIWD_DEFAULT_TIMEOUT) ")");
+		 __stringify(NIWD_DEFAULT_TIMEOUT) ")");
 
 static int nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, int, S_IRUGO);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static void ni903x_start(struct ni903x_wdt *wdt)
 {
diff --git a/drivers/watchdog/nic7018_wdt.c b/drivers/watchdog/nic7018_wdt.c
index 8169423801f061..2b122ba8733a5a 100644
--- a/drivers/watchdog/nic7018_wdt.c
+++ b/drivers/watchdog/nic7018_wdt.c
@@ -8,6 +8,7 @@
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -34,13 +35,13 @@ static unsigned int timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. (default="
-		 __MODULE_STRING(WDT_DEFAULT_TIMEOUT) ")");
+		 __stringify(WDT_DEFAULT_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started. (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct nic7018_wdt {
 	u16 io_base;
diff --git a/drivers/watchdog/nv_tco.c b/drivers/watchdog/nv_tco.c
index f16cee5173d504..0d671b8bb39146 100644
--- a/drivers/watchdog/nv_tco.c
+++ b/drivers/watchdog/nv_tco.c
@@ -32,6 +32,7 @@
 #include <linux/platform_device.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/stringify.h>
 
 #include "nv_tco.h"
 
@@ -55,12 +56,12 @@ static struct platform_device *nv_tco_platform_device;
 static int heartbeat = WATCHDOG_HEARTBEAT;  /* in seconds */
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. (2<heartbeat<39, "
-			    "default=" __MODULE_STRING(WATCHDOG_HEARTBEAT) ")");
+			    "default=" __stringify(WATCHDOG_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started"
-		" (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		" (default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  * Some TCO specific functions
diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c
index 0615bb81608212..171b46d780e6f2 100644
--- a/drivers/watchdog/octeon-wdt-main.c
+++ b/drivers/watchdog/octeon-wdt-main.c
@@ -55,6 +55,7 @@
 #include <linux/cpu.h>
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
+#include <linux/stringify.h>
 
 #include <asm/mipsregs.h>
 #include <asm/uasm.h>
@@ -93,13 +94,13 @@ static int heartbeat = WD_TIMO;
 module_param(heartbeat, int, 0444);
 MODULE_PARM_DESC(heartbeat,
 	"Watchdog heartbeat in seconds. (0 < heartbeat, default="
-				__MODULE_STRING(WD_TIMO) ")");
+				__stringify(WD_TIMO) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0444);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int disable;
 module_param(disable, int, 0444);
diff --git a/drivers/watchdog/of_xilinx_wdt.c b/drivers/watchdog/of_xilinx_wdt.c
index 352853e6fe7120..d6b1adfe31d11a 100644
--- a/drivers/watchdog/of_xilinx_wdt.c
+++ b/drivers/watchdog/of_xilinx_wdt.c
@@ -251,7 +251,7 @@ static int xwdt_probe(struct platform_device *pdev)
  * @dev: handle to the device structure.
  * Return: 0 always.
  */
-static int __maybe_unused xwdt_suspend(struct device *dev)
+static int xwdt_suspend(struct device *dev)
 {
 	struct xwdt_device *xdev = dev_get_drvdata(dev);
 
@@ -267,7 +267,7 @@ static int __maybe_unused xwdt_suspend(struct device *dev)
  * @dev: handle to the device structure.
  * Return: 0 on success, errno otherwise.
  */
-static int __maybe_unused xwdt_resume(struct device *dev)
+static int xwdt_resume(struct device *dev)
 {
 	struct xwdt_device *xdev = dev_get_drvdata(dev);
 	int ret = 0;
@@ -278,7 +278,7 @@ static int __maybe_unused xwdt_resume(struct device *dev)
 	return ret;
 }
 
-static SIMPLE_DEV_PM_OPS(xwdt_pm_ops, xwdt_suspend, xwdt_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(xwdt_pm_ops, xwdt_suspend, xwdt_resume);
 
 /* Match table for of_platform binding */
 static const struct of_device_id xwdt_of_match[] = {
@@ -293,7 +293,7 @@ static struct platform_driver xwdt_driver = {
 	.driver = {
 		.name  = WATCHDOG_NAME,
 		.of_match_table = xwdt_of_match,
-		.pm = &xwdt_pm_ops,
+		.pm = pm_sleep_ptr(&xwdt_pm_ops),
 	},
 };
 
diff --git a/drivers/watchdog/omap_wdt.c b/drivers/watchdog/omap_wdt.c
index 95c7e44b7baac0..2a6de73e6f0b36 100644
--- a/drivers/watchdog/omap_wdt.c
+++ b/drivers/watchdog/omap_wdt.c
@@ -38,6 +38,7 @@
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/pm_runtime.h>
+#include <linux/stringify.h>
 #include <linux/platform_data/omap-wd-timer.h>
 
 #include "omap_wdt.h"
@@ -45,7 +46,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned timer_margin;
 module_param(timer_margin, uint, 0);
diff --git a/drivers/watchdog/orion_wdt.c b/drivers/watchdog/orion_wdt.c
index 0ea946823b0a73..5d56e93f040646 100644
--- a/drivers/watchdog/orion_wdt.c
+++ b/drivers/watchdog/orion_wdt.c
@@ -22,6 +22,7 @@
 #include <linux/err.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/stringify.h>
 
 /* RSTOUT mask register physical address for Orion5x, Kirkwood and Dove */
 #define ORION_RSTOUT_MASK_OFFSET	0x20108
@@ -687,7 +688,7 @@ MODULE_PARM_DESC(heartbeat, "Initial watchdog heartbeat in seconds");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:orion_wdt");
diff --git a/drivers/watchdog/pc87413_wdt.c b/drivers/watchdog/pc87413_wdt.c
index fbf835d112b844..bbd8638cd5f2f8 100644
--- a/drivers/watchdog/pc87413_wdt.c
+++ b/drivers/watchdog/pc87413_wdt.c
@@ -30,6 +30,7 @@
 #include <linux/moduleparam.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 /* #define DEBUG 1 */
@@ -333,7 +334,7 @@ static int pc87413_status(void)
  *	@ppos: pointer to the position to write. No seeks allowed
  *
  *	A write to a watchdog device is defined as a keepalive signal. Any
- *	write of data will do, as we we don't define content meaning.
+ *	write of data will do, as we don't define content meaning.
  */
 
 static ssize_t pc87413_write(struct file *file, const char __user *data,
@@ -577,15 +578,15 @@ MODULE_LICENSE("GPL");
 
 module_param_hw(io, int, ioport, 0);
 MODULE_PARM_DESC(io, MODNAME " I/O port (default: "
-					__MODULE_STRING(IO_DEFAULT) ").");
+					__stringify(IO_DEFAULT) ").");
 
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		"Watchdog timeout in minutes (default="
-				__MODULE_STRING(DEFAULT_TIMEOUT) ").");
+				__stringify(DEFAULT_TIMEOUT) ").");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
diff --git a/drivers/watchdog/pcwd.c b/drivers/watchdog/pcwd.c
index d4ea7d6ccd6a71..3370a915527666 100644
--- a/drivers/watchdog/pcwd.c
+++ b/drivers/watchdog/pcwd.c
@@ -72,6 +72,7 @@
 #include <linux/spinlock.h>	/* For spin_lock/spin_unlock/... */
 #include <linux/uaccess.h>	/* For copy_to_user/put_user/... */
 #include <linux/io.h>		/* For inb/outb/... */
+#include <linux/stringify.h>	/* For __stringify */
 
 /* Module and version information */
 #define WATCHDOG_VERSION "1.20"
@@ -203,13 +204,13 @@ static int heartbeat = WATCHDOG_HEARTBEAT;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. "
 	"(2 <= heartbeat <= 7200 or 0=delay-time from dip-switches, default="
-				__MODULE_STRING(WATCHDOG_HEARTBEAT) ")");
+				__stringify(WATCHDOG_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	Internal functions
diff --git a/drivers/watchdog/pcwd_pci.c b/drivers/watchdog/pcwd_pci.c
index a489b426f2ba67..8c232c760ee54f 100644
--- a/drivers/watchdog/pcwd_pci.c
+++ b/drivers/watchdog/pcwd_pci.c
@@ -47,6 +47,7 @@
 #include <linux/spinlock.h>	/* For spin_lock/spin_unlock/... */
 #include <linux/uaccess.h>	/* For copy_to_user/put_user/... */
 #include <linux/io.h>		/* For inb/outb/... */
+#include <linux/stringify.h>	/* For __stringify */
 
 /* Module and version information */
 #define WATCHDOG_VERSION "1.03"
@@ -114,7 +115,7 @@ static unsigned long is_active;
 static char expect_release;
 /* this is private data for each PCI-PC watchdog card */
 static struct {
-	/* Wether or not the card has a temperature device */
+	/* Whether or not the card has a temperature device */
 	int supports_temp;
 	/* The card's boot status */
 	int boot_status;
@@ -140,12 +141,12 @@ static int heartbeat = WATCHDOG_HEARTBEAT;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. "
 	"(0<heartbeat<65536 or 0=delay-time from dip-switches, default="
-				__MODULE_STRING(WATCHDOG_HEARTBEAT) ")");
+				__stringify(WATCHDOG_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-					__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+					__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	Internal functions
diff --git a/drivers/watchdog/pcwd_usb.c b/drivers/watchdog/pcwd_usb.c
index d7c18c990649ae..fc175aa2f9924b 100644
--- a/drivers/watchdog/pcwd_usb.c
+++ b/drivers/watchdog/pcwd_usb.c
@@ -39,6 +39,7 @@
 #include <linux/mutex.h>	/* For mutex locking */
 #include <linux/hid.h>		/* For HID_REQ_SET_REPORT & HID_DT_REPORT */
 #include <linux/uaccess.h>	/* For copy_to_user/put_user/... */
+#include <linux/stringify.h>	/* For __stringify */
 
 
 /* Module and Version Information */
@@ -57,12 +58,12 @@ static int heartbeat = WATCHDOG_HEARTBEAT;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. "
 	"(0<heartbeat<65536 or 0=delay-time from dip-switches, default="
-				__MODULE_STRING(WATCHDOG_HEARTBEAT) ")");
+				__stringify(WATCHDOG_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /* The vendor and product id's for the USB-PC Watchdog card */
 #define USB_PCWD_VENDOR_ID	0x0c98
@@ -138,7 +139,7 @@ struct usb_pcwd_private {
 	/* true if we received a report after a command */
 	atomic_t		cmd_received;
 
-	/* Wether or not the device exists */
+	/* Whether or not the device exists */
 	int			exists;
 	/* locks this structure */
 	struct mutex		mtx;
diff --git a/drivers/watchdog/pika_wdt.c b/drivers/watchdog/pika_wdt.c
index 87b8988d252067..09fc76aaa0513b 100644
--- a/drivers/watchdog/pika_wdt.c
+++ b/drivers/watchdog/pika_wdt.c
@@ -25,6 +25,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
+#include <linux/stringify.h>
 
 #define DRV_NAME "PIKA-WDT"
 
@@ -39,12 +40,12 @@
 static int heartbeat = WDT_HEARTBEAT;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeats in seconds. "
-	"(default = " __MODULE_STRING(WDT_HEARTBEAT) ")");
+	"(default = " __stringify(WDT_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static struct {
 	void __iomem *fpga;
diff --git a/drivers/watchdog/pm8916_wdt.c b/drivers/watchdog/pm8916_wdt.c
index 007ed139ab9653..a7ae57fb24532a 100644
--- a/drivers/watchdog/pm8916_wdt.c
+++ b/drivers/watchdog/pm8916_wdt.c
@@ -233,7 +233,7 @@ static int pm8916_wdt_probe(struct platform_device *pdev)
 	return devm_watchdog_register_device(dev, &wdt->wdev);
 }
 
-static int __maybe_unused pm8916_wdt_suspend(struct device *dev)
+static int pm8916_wdt_suspend(struct device *dev)
 {
 	struct pm8916_wdt *wdt = dev_get_drvdata(dev);
 
@@ -243,7 +243,7 @@ static int __maybe_unused pm8916_wdt_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused pm8916_wdt_resume(struct device *dev)
+static int pm8916_wdt_resume(struct device *dev)
 {
 	struct pm8916_wdt *wdt = dev_get_drvdata(dev);
 
@@ -253,7 +253,7 @@ static int __maybe_unused pm8916_wdt_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(pm8916_wdt_pm_ops, pm8916_wdt_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(pm8916_wdt_pm_ops, pm8916_wdt_suspend,
 			 pm8916_wdt_resume);
 
 static const struct of_device_id pm8916_wdt_id_table[] = {
@@ -267,7 +267,7 @@ static struct platform_driver pm8916_wdt_driver = {
 	.driver = {
 		.name = "pm8916-wdt",
 		.of_match_table = pm8916_wdt_id_table,
-		.pm = &pm8916_wdt_pm_ops,
+		.pm = pm_sleep_ptr(&pm8916_wdt_pm_ops),
 	},
 };
 module_platform_driver(pm8916_wdt_driver);
diff --git a/drivers/watchdog/pnx4008_wdt.c b/drivers/watchdog/pnx4008_wdt.c
index 87a44a5675a143..a7c39849cbfdc7 100644
--- a/drivers/watchdog/pnx4008_wdt.c
+++ b/drivers/watchdog/pnx4008_wdt.c
@@ -30,6 +30,7 @@
 #include <linux/of.h>
 #include <linux/delay.h>
 #include <linux/reboot.h>
+#include <linux/stringify.h>
 
 /* WatchDog Timer - Chapter 23 Page 207 */
 
@@ -237,8 +238,8 @@ MODULE_DESCRIPTION("PNX4008 Watchdog Driver");
 module_param(heartbeat, uint, 0);
 MODULE_PARM_DESC(heartbeat,
 		 "Watchdog heartbeat period in seconds from 1 to "
-		 __MODULE_STRING(MAX_HEARTBEAT) ", default "
-		 __MODULE_STRING(DEFAULT_HEARTBEAT));
+		 __stringify(MAX_HEARTBEAT) ", default "
+		 __stringify(DEFAULT_HEARTBEAT));
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
diff --git a/drivers/watchdog/pseries-wdt.c b/drivers/watchdog/pseries-wdt.c
index e97b943e1d3c50..5f531a03a0fc6f 100644
--- a/drivers/watchdog/pseries-wdt.c
+++ b/drivers/watchdog/pseries-wdt.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/time64.h>
 #include <linux/watchdog.h>
 #include <asm/papr-watchdog.h>
@@ -26,18 +27,18 @@ static const unsigned long pseries_wdt_action[] = {
 static unsigned int action = WATCHDOG_ACTION;
 module_param(action, uint, 0444);
 MODULE_PARM_DESC(action, "Action taken when watchdog expires (default="
-		 __MODULE_STRING(WATCHDOG_ACTION) ")");
+		 __stringify(WATCHDOG_ACTION) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0444);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 #define WATCHDOG_TIMEOUT 60
 static unsigned int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, uint, 0444);
 MODULE_PARM_DESC(timeout, "Initial watchdog timeout in seconds (default="
-		 __MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+		 __stringify(WATCHDOG_TIMEOUT) ")");
 
 struct pseries_wdt {
 	struct watchdog_device wd;
diff --git a/drivers/watchdog/rc32434_wdt.c b/drivers/watchdog/rc32434_wdt.c
index 0e5c5c96af58d2..5301a93abf55f5 100644
--- a/drivers/watchdog/rc32434_wdt.c
+++ b/drivers/watchdog/rc32434_wdt.c
@@ -27,6 +27,7 @@
 #include <linux/spinlock.h>		/* For spin_lock/spin_unlock/... */
 #include <linux/uaccess.h>		/* For copy_to_user/put_user/... */
 #include <linux/io.h>			/* For devm_ioremap */
+#include <linux/stringify.h>		/* For __stringify */
 
 #include <asm/mach-rc32434/integ.h>	/* For the Watchdog registers */
 
@@ -57,12 +58,12 @@ extern unsigned int idt_cpu_freq;
 static int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout value, in seconds (default="
-		__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+		__stringify(WATCHDOG_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /* apply or and nand masks to data read from addr and write back */
 #define SET_BITS(addr, or, nand) \
diff --git a/drivers/watchdog/renesas_wdt.c b/drivers/watchdog/renesas_wdt.c
index 97bcd32bade521..8c0041ce555214 100644
--- a/drivers/watchdog/renesas_wdt.c
+++ b/drivers/watchdog/renesas_wdt.c
@@ -16,6 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/smp.h>
+#include <linux/stringify.h>
 #include <linux/sys_soc.h>
 #include <linux/watchdog.h>
 
@@ -44,7 +45,7 @@ static const unsigned int clk_divs[] = { 1, 4, 16, 32, 64, 128, 1024, 4096 };
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct rwdt_priv {
 	void __iomem *base;
diff --git a/drivers/watchdog/rn5t618_wdt.c b/drivers/watchdog/rn5t618_wdt.c
index 87d06d210ac9ed..14056af36445cc 100644
--- a/drivers/watchdog/rn5t618_wdt.c
+++ b/drivers/watchdog/rn5t618_wdt.c
@@ -9,6 +9,7 @@
 #include <linux/mfd/rn5t618.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DRIVER_NAME "rn5t618-wdt"
@@ -21,7 +22,7 @@ MODULE_PARM_DESC(timeout, "Initial watchdog timeout in seconds");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct rn5t618_wdt {
 	struct watchdog_device wdt_dev;
diff --git a/drivers/watchdog/rt2880_wdt.c b/drivers/watchdog/rt2880_wdt.c
index c8edd83bb502e2..38f31d302a4b36 100644
--- a/drivers/watchdog/rt2880_wdt.c
+++ b/drivers/watchdog/rt2880_wdt.c
@@ -15,6 +15,7 @@
 #include <linux/watchdog.h>
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 
 #include <asm/mach-ralink/ralink_regs.h>
 
@@ -51,7 +52,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-		__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static inline void rt_wdt_w32(void __iomem *base, unsigned int reg, u32 val)
 {
diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c
index 7c1bd83b056a80..6a64adbe1a8eac 100644
--- a/drivers/watchdog/rti_wdt.c
+++ b/drivers/watchdog/rti_wdt.c
@@ -17,6 +17,7 @@
 #include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -414,8 +415,8 @@ MODULE_DESCRIPTION("K3 RTI Watchdog Driver");
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 		 "Watchdog heartbeat period in seconds from 1 to "
-		 __MODULE_STRING(MAX_HEARTBEAT) ", default "
-		 __MODULE_STRING(DEFAULT_HEARTBEAT));
+		 __stringify(MAX_HEARTBEAT) ", default "
+		 __stringify(DEFAULT_HEARTBEAT));
 
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:rti-wdt");
diff --git a/drivers/watchdog/rzg2l_wdt.c b/drivers/watchdog/rzg2l_wdt.c
index 509f9dffdacde8..592ea7ce6d69a7 100644
--- a/drivers/watchdog/rzg2l_wdt.c
+++ b/drivers/watchdog/rzg2l_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
+#include <linux/stringify.h>
 #include <linux/reset.h>
 #include <linux/units.h>
 #include <linux/watchdog.h>
@@ -41,7 +42,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 enum rz_wdt_type {
 	WDT_RZG2L,
diff --git a/drivers/watchdog/rzv2h_wdt.c b/drivers/watchdog/rzv2h_wdt.c
index 83540dd9a37b6d..a8db85e85c73a6 100644
--- a/drivers/watchdog/rzv2h_wdt.c
+++ b/drivers/watchdog/rzv2h_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
+#include <linux/stringify.h>
 #include <linux/units.h>
 #include <linux/watchdog.h>
 
@@ -48,7 +49,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 enum rzv2h_wdt_count_source {
 	COUNT_SOURCE_LOCO,
diff --git a/drivers/watchdog/s32g_wdt.c b/drivers/watchdog/s32g_wdt.c
index 6422a694fc65ff..bd8740153c17f4 100644
--- a/drivers/watchdog/s32g_wdt.c
+++ b/drivers/watchdog/s32g_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/moduleparam.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define DRIVER_NAME "s32g-swt"
@@ -42,12 +43,12 @@ struct s32g_wdt_device {
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static unsigned int timeout_param = S32G_WDT_DEFAULT_TIMEOUT;
 module_param(timeout_param, uint, 0);
 MODULE_PARM_DESC(timeout_param, "Watchdog timeout in seconds (default="
-		 __MODULE_STRING(S32G_WDT_DEFAULT_TIMEOUT) ")");
+		 __stringify(S32G_WDT_DEFAULT_TIMEOUT) ")");
 
 static bool early_enable;
 module_param(early_enable, bool, 0);
diff --git a/drivers/watchdog/s3c2410_wdt.c b/drivers/watchdog/s3c2410_wdt.c
index e31f93db050966..7053963fb0ac63 100644
--- a/drivers/watchdog/s3c2410_wdt.c
+++ b/drivers/watchdog/s3c2410_wdt.c
@@ -28,6 +28,7 @@
 #include <linux/regmap.h>
 #include <linux/delay.h>
 #include <linux/math64.h>
+#include <linux/stringify.h>
 
 #define S3C2410_WTCON		0x00
 #define S3C2410_WTDAT		0x04
@@ -154,12 +155,12 @@ module_param(nowayout,   bool, 0);
 module_param(soft_noboot, int, 0);
 
 MODULE_PARM_DESC(tmr_margin, "Watchdog tmr_margin in seconds. (default="
-		__MODULE_STRING(S3C2410_WATCHDOG_DEFAULT_TIME) ")");
+		__stringify(S3C2410_WATCHDOG_DEFAULT_TIME) ")");
 MODULE_PARM_DESC(tmr_atboot,
 		"Watchdog is started at boot time if set to 1, default="
-			__MODULE_STRING(S3C2410_WATCHDOG_ATBOOT));
+			__stringify(S3C2410_WATCHDOG_ATBOOT));
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			__stringify(WATCHDOG_NOWAYOUT) ")");
 MODULE_PARM_DESC(soft_noboot, "Watchdog action, set to 1 to ignore reboots, 0 to reboot (default 0)");
 
 /**
@@ -232,6 +233,20 @@ static const struct s3c2410_wdt_variant drv_data_exynos5420 = {
 		  QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_PMU_AUTO_DISABLE,
 };
 
+/*
+ * Unlike similar SoCs like GS101, Exynos5515's PMU does not require
+ * explicit counter enablement. Hence, QUIRK_HAS_PMU_CNT_EN is not set.
+ */
+static const struct s3c2410_wdt_variant drv_data_exynos5515 = {
+	.mask_reset_reg = EXYNOSAUTOV920_CLUSTER0_NONCPU_INT_EN,
+	.mask_bit = 2,
+	.mask_reset_inv = true,
+	.rst_stat_reg = EXYNOS5_RST_STAT_REG_OFFSET,
+	.rst_stat_bit = 24,
+	.quirks = QUIRK_HAS_WTCLRINT_REG | QUIRK_HAS_PMU_MASK_RESET | \
+		  QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_DBGACK_BIT,
+};
+
 static const struct s3c2410_wdt_variant drv_data_exynos7 = {
 	.disable_reg = EXYNOS5_WDT_DISABLE_REG_OFFSET,
 	.mask_reset_reg = EXYNOS5_WDT_MASK_RESET_REG_OFFSET,
@@ -379,6 +394,8 @@ static const struct of_device_id s3c2410_wdt_match[] = {
 	  .data = &drv_data_exynos5250 },
 	{ .compatible = "samsung,exynos5420-wdt",
 	  .data = &drv_data_exynos5420 },
+	{ .compatible = "samsung,exynos5515-wdt",
+	  .data = &drv_data_exynos5515 },
 	{ .compatible = "samsung,exynos7-wdt",
 	  .data = &drv_data_exynos7 },
 	{ .compatible = "samsung,exynos850-wdt",
diff --git a/drivers/watchdog/sama5d4_wdt.c b/drivers/watchdog/sama5d4_wdt.c
index ecdd9aeab79cc5..85efe4db40f0b5 100644
--- a/drivers/watchdog/sama5d4_wdt.c
+++ b/drivers/watchdog/sama5d4_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/reboot.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #include "at91sam9_wdt.h"
@@ -42,12 +43,12 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(wdt_timeout, int, 0);
 MODULE_PARM_DESC(wdt_timeout,
 	"Watchdog timeout in seconds. (default = "
-	__MODULE_STRING(WDT_DEFAULT_TIMEOUT) ")");
+	__stringify(WDT_DEFAULT_TIMEOUT) ")");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static inline bool wdt_enabled(struct sama5d4_wdt *wdt)
 {
diff --git a/drivers/watchdog/sbc60xxwdt.c b/drivers/watchdog/sbc60xxwdt.c
index 03eaf48c8f0f1e..fffe58dcbf9340 100644
--- a/drivers/watchdog/sbc60xxwdt.c
+++ b/drivers/watchdog/sbc60xxwdt.c
@@ -58,6 +58,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 #define OUR_NAME "sbc60xxwdt"
@@ -98,13 +99,13 @@ static int timeout = WATCHDOG_TIMEOUT;	/* in seconds, multiplied by HZ to
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. (1<=timeout<=3600, default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+				__stringify(WATCHDOG_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static void wdt_timer_ping(struct timer_list *);
 static DEFINE_TIMER(timer, wdt_timer_ping);
diff --git a/drivers/watchdog/sbc7240_wdt.c b/drivers/watchdog/sbc7240_wdt.c
index 21a1f0b32070b8..077abbdbf5828c 100644
--- a/drivers/watchdog/sbc7240_wdt.c
+++ b/drivers/watchdog/sbc7240_wdt.c
@@ -23,6 +23,7 @@
 #include <linux/io.h>
 #include <linux/uaccess.h>
 #include <linux/atomic.h>
+#include <linux/stringify.h>
 
 #define SBC7240_ENABLE_PORT		0x443
 #define SBC7240_DISABLE_PORT		0x043
@@ -34,8 +35,8 @@
 static int timeout = SBC7240_TIMEOUT;	/* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds. (1<=timeout<="
-		 __MODULE_STRING(SBC7240_MAX_TIMEOUT) ", default="
-		 __MODULE_STRING(SBC7240_TIMEOUT) ")");
+		 __stringify(SBC7240_MAX_TIMEOUT) ", default="
+		 __stringify(SBC7240_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
diff --git a/drivers/watchdog/sbc8360.c b/drivers/watchdog/sbc8360.c
index a9fd1615b4c389..dafef1df971913 100644
--- a/drivers/watchdog/sbc8360.c
+++ b/drivers/watchdog/sbc8360.c
@@ -48,6 +48,7 @@
 #include <linux/moduleparam.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 static unsigned long sbc8360_is_open;
@@ -199,7 +200,7 @@ MODULE_PARM_DESC(timeout, "Index into timeout table (0-63) (default=27 (60s))");
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	Kernel methods.
diff --git a/drivers/watchdog/sbc_epx_c3.c b/drivers/watchdog/sbc_epx_c3.c
index 1d291dc0a4a693..b17bd1db7b103e 100644
--- a/drivers/watchdog/sbc_epx_c3.c
+++ b/drivers/watchdog/sbc_epx_c3.c
@@ -25,6 +25,7 @@
 #include <linux/ioport.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/stringify.h>
 
 static int epx_c3_alive;
 
@@ -33,7 +34,7 @@ static int epx_c3_alive;
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-					__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+					__stringify(WATCHDOG_NOWAYOUT) ")");
 
 #define EPXC3_WATCHDOG_CTL_REG 0x1ee /* write 1 to enable, 0 to disable */
 #define EPXC3_WATCHDOG_PET_REG 0x1ef /* write anything to pet once enabled */
diff --git a/drivers/watchdog/sbsa_gwdt.c b/drivers/watchdog/sbsa_gwdt.c
index e04d42cc7774da..3a5ff48a58f20e 100644
--- a/drivers/watchdog/sbsa_gwdt.c
+++ b/drivers/watchdog/sbsa_gwdt.c
@@ -46,6 +46,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/uaccess.h>
 #include <linux/watchdog.h>
 #include <asm/arch_timer.h>
@@ -103,7 +104,7 @@ static unsigned int timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 		 "Watchdog timeout in seconds. (>=0, default="
-		 __MODULE_STRING(DEFAULT_TIMEOUT) ")");
+		 __stringify(DEFAULT_TIMEOUT) ")");
 
 /*
  * action refers to action taken when watchdog gets WS0
@@ -120,7 +121,12 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, S_IRUGO);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
+
+static bool early_enable;
+module_param(early_enable, bool, 0);
+MODULE_PARM_DESC(early_enable,
+		 "Watchdog is started on module insertion (default=0)");
 
 /*
  * Arm Base System Architecture 1.0 introduces watchdog v1 which
@@ -296,6 +302,7 @@ static int sbsa_gwdt_probe(struct platform_device *pdev)
 	struct sbsa_gwdt *gwdt;
 	int ret, irq;
 	u32 status;
+	bool early_action;
 
 	gwdt = devm_kzalloc(dev, sizeof(*gwdt), GFP_KERNEL);
 	if (!gwdt)
@@ -386,14 +393,23 @@ static int sbsa_gwdt_probe(struct platform_device *pdev)
 	 */
 	sbsa_gwdt_set_timeout(wdd, wdd->timeout);
 
+	early_action = early_enable && !(status & SBSA_GWDT_WCS_EN);
+	if (early_action) {
+		sbsa_gwdt_start(wdd);
+		set_bit(WDOG_HW_RUNNING, &wdd->status);
+	}
+
 	watchdog_stop_on_reboot(wdd);
 	ret = devm_watchdog_register_device(dev, wdd);
-	if (ret)
+	if (ret) {
+		if (early_action)
+			sbsa_gwdt_stop(wdd);
 		return ret;
+	}
 
 	dev_info(dev, "Initialized with %ds timeout @ %u Hz, action=%d.%s\n",
 		 wdd->timeout, gwdt->clk, action,
-		 status & SBSA_GWDT_WCS_EN ? " [enabled]" : "");
+		 watchdog_hw_running(wdd) ? " [enabled]" : "");
 
 	return 0;
 }
diff --git a/drivers/watchdog/sc1200wdt.c b/drivers/watchdog/sc1200wdt.c
index 1d3300c98d43ec..34cd57420c86d8 100644
--- a/drivers/watchdog/sc1200wdt.c
+++ b/drivers/watchdog/sc1200wdt.c
@@ -42,6 +42,7 @@
 #include <linux/semaphore.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #define SC1200_MODULE_VER	"build 20020303"
 #define SC1200_MODULE_NAME	"sc1200wdt"
@@ -92,7 +93,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 
 
diff --git a/drivers/watchdog/sch311x_wdt.c b/drivers/watchdog/sch311x_wdt.c
index 9670a1ea57cbd9..a2a0b5c2b55b7d 100644
--- a/drivers/watchdog/sch311x_wdt.c
+++ b/drivers/watchdog/sch311x_wdt.c
@@ -31,6 +31,7 @@
 #include <linux/spinlock.h>		/* For spin_lock/spin_unlock/... */
 #include <linux/uaccess.h>		/* For copy_to_user/put_user/... */
 #include <linux/io.h>			/* For inb/outb/... */
+#include <linux/stringify.h>		/* For __stringify */
 
 /* Module and version information */
 #define DRV_NAME	"sch311x_wdt"
@@ -68,13 +69,13 @@ static int timeout = WATCHDOG_TIMEOUT;	/* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. 1<= timeout <=15300, default="
-		__MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+		__stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-		__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  *	Super-IO functions
diff --git a/drivers/watchdog/shwdt.c b/drivers/watchdog/shwdt.c
index 719f100aae60dd..8adecf54b9bc84 100644
--- a/drivers/watchdog/shwdt.c
+++ b/drivers/watchdog/shwdt.c
@@ -30,6 +30,7 @@
 #include <linux/io.h>
 #include <linux/clk.h>
 #include <linux/err.h>
+#include <linux/stringify.h>
 #include <asm/watchdog.h>
 
 #define DRV_NAME "sh-wdt"
@@ -329,14 +330,14 @@ MODULE_ALIAS("platform:" DRV_NAME);
 module_param(clock_division_ratio, int, 0);
 MODULE_PARM_DESC(clock_division_ratio,
 	"Clock division ratio. Valid ranges are from 0x5 (1.31ms) "
-	"to 0x7 (5.25ms). (default=" __MODULE_STRING(WTCSR_CKS_4096) ")");
+	"to 0x7 (5.25ms). (default=" __stringify(WTCSR_CKS_4096) ")");
 
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 	"Watchdog heartbeat in seconds. (1 <= heartbeat <= 3600, default="
-				__MODULE_STRING(WATCHDOG_HEARTBEAT) ")");
+				__stringify(WATCHDOG_HEARTBEAT) ")");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
diff --git a/drivers/watchdog/simatic-ipc-wdt.c b/drivers/watchdog/simatic-ipc-wdt.c
index 1e91f0a560ffce..addeecccf2e9be 100644
--- a/drivers/watchdog/simatic-ipc-wdt.c
+++ b/drivers/watchdog/simatic-ipc-wdt.c
@@ -22,6 +22,7 @@
 #include <linux/sizes.h>
 #include <linux/util_macros.h>
 #include <linux/watchdog.h>
+#include <linux/stringify.h>
 
 #define WD_ENABLE_IOADR			0x62
 #define WD_TRIGGER_IOADR		0x66
@@ -42,7 +43,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0000);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static struct resource gp_status_reg_227e_res =
 	DEFINE_RES_IO_NAMED(GP_STATUS_REG_227E, SZ_1, KBUILD_MODNAME);
diff --git a/drivers/watchdog/sl28cpld_wdt.c b/drivers/watchdog/sl28cpld_wdt.c
index c0b5e5fff5d7af..e32b757372eb64 100644
--- a/drivers/watchdog/sl28cpld_wdt.c
+++ b/drivers/watchdog/sl28cpld_wdt.c
@@ -10,6 +10,7 @@
 #include <linux/platform_device.h>
 #include <linux/property.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /*
@@ -30,7 +31,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int timeout;
 module_param(timeout, int, 0);
diff --git a/drivers/watchdog/smsc37b787_wdt.c b/drivers/watchdog/smsc37b787_wdt.c
index 9f2c2ab4e2f157..0ee36ae3113e0c 100644
--- a/drivers/watchdog/smsc37b787_wdt.c
+++ b/drivers/watchdog/smsc37b787_wdt.c
@@ -55,6 +55,7 @@
 #include <linux/spinlock.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 /* enable support for minutes as units? */
@@ -611,4 +612,4 @@ MODULE_PARM_DESC(timeout, "range is 1-255 units, default is 60");
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
diff --git a/drivers/watchdog/softdog.c b/drivers/watchdog/softdog.c
index 0820e35ad2e3cd..0ff125958b8a25 100644
--- a/drivers/watchdog/softdog.c
+++ b/drivers/watchdog/softdog.c
@@ -24,6 +24,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/reboot.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 #include <linux/workqueue.h>
@@ -33,13 +34,13 @@ static unsigned int soft_margin = TIMER_MARGIN;	/* in seconds */
 module_param(soft_margin, uint, 0);
 MODULE_PARM_DESC(soft_margin,
 	"Watchdog soft_margin in seconds. (0 < soft_margin < 65536, default="
-					__MODULE_STRING(TIMER_MARGIN) ")");
+					__stringify(TIMER_MARGIN) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int soft_noboot;
 module_param(soft_noboot, int, 0);
diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c
index 72ad59649ccc41..14d4bffface41e 100644
--- a/drivers/watchdog/sp5100_tco.c
+++ b/drivers/watchdog/sp5100_tco.c
@@ -33,12 +33,14 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/init.h>
+#include <linux/dmi.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/types.h>
 #include <linux/watchdog.h>
 
@@ -69,18 +71,18 @@ static struct pci_dev *sp5100_tco_pci;
 static bool action = WATCHDOG_ACTION;
 module_param(action, bool, 0);
 MODULE_PARM_DESC(action, "Action taken when watchdog expires, 0 to reset, 1 to poweroff (default="
-		 __MODULE_STRING(WATCHDOG_ACTION) ")");
+		 __stringify(WATCHDOG_ACTION) ")");
 
 #define WATCHDOG_HEARTBEAT 60	/* 60 sec default heartbeat. */
 static int heartbeat = WATCHDOG_HEARTBEAT;  /* in seconds */
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. (default="
-		 __MODULE_STRING(WATCHDOG_HEARTBEAT) ")");
+		 __stringify(WATCHDOG_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started."
-		" (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		" (default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  * Some TCO specific functions
@@ -240,6 +242,35 @@ static u32 sp5100_tco_read_pm_reg32(u8 index)
 	return val;
 }
 
+/*
+ * The Gigabyte GA-78LMT-USB3 firmware programs the legacy SP5100 watchdog
+ * MMIO window at 0xfec000f0. This address lies inside the IOAPIC resource,
+ * so the generic resource reservation fails even though firmware explicitly
+ * assigns the watchdog to this address.
+ *
+ * Keep this exception narrowly scoped to the affected system and firmware
+ * address. Do not relocate or otherwise reprogram the watchdog.
+ */
+#define SP5100_WDT_GA78LMT_MMIO	0xfec000f0
+
+static const struct dmi_system_id sp5100_tco_unreserved_mmio_dmi[] = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Gigabyte Technology Co., Ltd."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "GA-78LMT-USB3"),
+		},
+	},
+	{}
+};
+
+static bool sp5100_tco_allow_unreserved_mmio(struct sp5100_tco *tco,
+					     u32 mmio_addr)
+{
+	return tco->tco_reg_layout == sp5100 &&
+	       mmio_addr == SP5100_WDT_GA78LMT_MMIO &&
+	       dmi_check_system(sp5100_tco_unreserved_mmio_dmi);
+}
+
 static u32 sp5100_tco_request_region(struct device *dev,
 				     u32 mmio_addr,
 				     const char *dev_name)
@@ -259,6 +290,7 @@ static u32 sp5100_tco_prepare_base(struct sp5100_tco *tco,
 				   const char *dev_name)
 {
 	struct device *dev = tco->wdd.parent;
+	bool reserved = false;
 
 	dev_dbg(dev, "Got 0x%08x from SBResource_MMIO register\n", mmio_addr);
 
@@ -266,11 +298,29 @@ static u32 sp5100_tco_prepare_base(struct sp5100_tco *tco,
 		return -ENODEV;
 
 	/* Check for MMIO address and alternate MMIO address conflicts */
-	if (mmio_addr)
-		mmio_addr = sp5100_tco_request_region(dev, mmio_addr, dev_name);
+	if (mmio_addr) {
+		u32 requested_addr;
+
+		requested_addr = sp5100_tco_request_region(dev, mmio_addr,
+							   dev_name);
+		if (requested_addr) {
+			mmio_addr = requested_addr;
+			reserved = true;
+		} else if (sp5100_tco_allow_unreserved_mmio(tco, mmio_addr)) {
+			dev_info(dev,
+				 "Using firmware watchdog MMIO 0x%08x without reserving it\n",
+				 mmio_addr);
+		} else {
+			mmio_addr = 0;
+		}
+	}
 
-	if (!mmio_addr && alt_mmio_addr)
-		mmio_addr = sp5100_tco_request_region(dev, alt_mmio_addr, dev_name);
+	if (!mmio_addr && alt_mmio_addr) {
+		mmio_addr = sp5100_tco_request_region(dev, alt_mmio_addr,
+						      dev_name);
+		if (mmio_addr)
+			reserved = true;
+	}
 
 	if (!mmio_addr) {
 		dev_err(dev, "Failed to reserve MMIO or alternate MMIO region\n");
@@ -280,7 +330,9 @@ static u32 sp5100_tco_prepare_base(struct sp5100_tco *tco,
 	tco->tcobase = devm_ioremap(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE);
 	if (!tco->tcobase) {
 		dev_err(dev, "MMIO address 0x%08x failed mapping\n", mmio_addr);
-		devm_release_mem_region(dev, mmio_addr, SP5100_WDT_MEM_MAP_SIZE);
+		if (reserved)
+			devm_release_mem_region(dev, mmio_addr,
+						SP5100_WDT_MEM_MAP_SIZE);
 		return -ENOMEM;
 	}
 
diff --git a/drivers/watchdog/sp805_wdt.c b/drivers/watchdog/sp805_wdt.c
index c2125f204a137f..925f1215718137 100644
--- a/drivers/watchdog/sp805_wdt.c
+++ b/drivers/watchdog/sp805_wdt.c
@@ -321,27 +321,29 @@ static void sp805_wdt_remove(struct amba_device *adev)
 	watchdog_set_drvdata(&wdt->wdd, NULL);
 }
 
-static int __maybe_unused sp805_wdt_suspend(struct device *dev)
+static int sp805_wdt_suspend(struct device *dev)
 {
 	struct sp805_wdt *wdt = dev_get_drvdata(dev);
 
-	if (watchdog_active(&wdt->wdd))
+	if (watchdog_active(&wdt->wdd) ||
+	    watchdog_hw_running(&wdt->wdd))
 		return wdt_disable(&wdt->wdd);
 
 	return 0;
 }
 
-static int __maybe_unused sp805_wdt_resume(struct device *dev)
+static int sp805_wdt_resume(struct device *dev)
 {
 	struct sp805_wdt *wdt = dev_get_drvdata(dev);
 
-	if (watchdog_active(&wdt->wdd))
+	if (watchdog_active(&wdt->wdd) ||
+	    watchdog_hw_running(&wdt->wdd))
 		return wdt_enable(&wdt->wdd);
 
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(sp805_wdt_dev_pm_ops, sp805_wdt_suspend,
+static DEFINE_SIMPLE_DEV_PM_OPS(sp805_wdt_dev_pm_ops, sp805_wdt_suspend,
 		sp805_wdt_resume);
 
 static const struct amba_id sp805_wdt_ids[] = {
@@ -361,7 +363,7 @@ MODULE_DEVICE_TABLE(amba, sp805_wdt_ids);
 static struct amba_driver sp805_wdt_driver = {
 	.drv = {
 		.name	= MODULE_NAME,
-		.pm	= &sp805_wdt_dev_pm_ops,
+		.pm	= pm_sleep_ptr(&sp805_wdt_dev_pm_ops),
 	},
 	.id_table	= sp805_wdt_ids,
 	.probe		= sp805_wdt_probe,
diff --git a/drivers/watchdog/starfive-wdt.c b/drivers/watchdog/starfive-wdt.c
index 5a3254c83d3bb8..044b4fdb73505b 100644
--- a/drivers/watchdog/starfive-wdt.c
+++ b/drivers/watchdog/starfive-wdt.c
@@ -12,6 +12,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* JH7100 Watchdog register define */
@@ -72,12 +73,12 @@ module_param(early_enable, bool, 0);
 module_param(nowayout, bool, 0);
 
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. (default="
-		 __MODULE_STRING(STARFIVE_WDT_DEFAULT_TIME) ")");
+		 __stringify(STARFIVE_WDT_DEFAULT_TIME) ")");
 MODULE_PARM_DESC(early_enable,
 		 "Watchdog is started at boot time if set to 1, default="
-		 __MODULE_STRING(STARFIVE_WDT_EARLY_ENA));
+		 __stringify(STARFIVE_WDT_EARLY_ENA));
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct starfive_wdt_variant {
 	unsigned int control;		/* Watchdog Control Register for reset enable */
diff --git a/drivers/watchdog/stmp3xxx_rtc_wdt.c b/drivers/watchdog/stmp3xxx_rtc_wdt.c
index 060447101f4866..7511599f3c5314 100644
--- a/drivers/watchdog/stmp3xxx_rtc_wdt.c
+++ b/drivers/watchdog/stmp3xxx_rtc_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/stmp3xxx_rtc_wdt.h>
 #include <linux/notifier.h>
 #include <linux/reboot.h>
+#include <linux/stringify.h>
 
 #define WDOG_TICK_RATE 1000 /* 1 kHz clock */
 #define STMP3XXX_DEFAULT_TIMEOUT 19
@@ -21,8 +22,8 @@
 static int heartbeat = STMP3XXX_DEFAULT_TIMEOUT;
 module_param(heartbeat, uint, 0);
 MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat period in seconds from 1 to "
-		 __MODULE_STRING(STMP3XXX_MAX_TIMEOUT) ", default "
-		 __MODULE_STRING(STMP3XXX_DEFAULT_TIMEOUT));
+		 __stringify(STMP3XXX_MAX_TIMEOUT) ", default "
+		 __stringify(STMP3XXX_DEFAULT_TIMEOUT));
 
 static int wdt_start(struct watchdog_device *wdd)
 {
@@ -114,7 +115,7 @@ static void stmp3xxx_wdt_remove(struct platform_device *pdev)
 	unregister_reboot_notifier(&wdt_notifier);
 }
 
-static int __maybe_unused stmp3xxx_wdt_suspend(struct device *dev)
+static int stmp3xxx_wdt_suspend(struct device *dev)
 {
 	struct watchdog_device *wdd = &stmp3xxx_wdd;
 
@@ -124,7 +125,7 @@ static int __maybe_unused stmp3xxx_wdt_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused stmp3xxx_wdt_resume(struct device *dev)
+static int stmp3xxx_wdt_resume(struct device *dev)
 {
 	struct watchdog_device *wdd = &stmp3xxx_wdd;
 
@@ -134,13 +135,13 @@ static int __maybe_unused stmp3xxx_wdt_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(stmp3xxx_wdt_pm_ops,
+static DEFINE_SIMPLE_DEV_PM_OPS(stmp3xxx_wdt_pm_ops,
 			 stmp3xxx_wdt_suspend, stmp3xxx_wdt_resume);
 
 static struct platform_driver stmp3xxx_wdt_driver = {
 	.driver = {
 		.name = "stmp3xxx_rtc_wdt",
-		.pm = &stmp3xxx_wdt_pm_ops,
+		.pm = pm_sleep_ptr(&stmp3xxx_wdt_pm_ops),
 	},
 	.probe = stmp3xxx_wdt_probe,
 	.remove = stmp3xxx_wdt_remove,
diff --git a/drivers/watchdog/stpmic1_wdt.c b/drivers/watchdog/stpmic1_wdt.c
index 45d0c543466fd2..17f0cdecd22c3c 100644
--- a/drivers/watchdog/stpmic1_wdt.c
+++ b/drivers/watchdog/stpmic1_wdt.c
@@ -9,6 +9,7 @@
 #include <linux/of.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* WATCHDOG CONTROL REGISTER bit */
@@ -25,7 +26,7 @@
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct stpmic1_wdt {
 	struct stpmic1 *pmic;
diff --git a/drivers/watchdog/sun4v_wdt.c b/drivers/watchdog/sun4v_wdt.c
index 8db86ad5ee3dfd..ad56d778776e7a 100644
--- a/drivers/watchdog/sun4v_wdt.c
+++ b/drivers/watchdog/sun4v_wdt.c
@@ -15,6 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 #include <asm/hypervisor.h>
 #include <asm/mdesc.h>
@@ -27,12 +28,12 @@
 static unsigned int timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds (default="
-	__MODULE_STRING(WDT_TIMEOUT) ")");
+	__stringify(WDT_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, S_IRUGO);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int sun4v_wdt_stop(struct watchdog_device *wdd)
 {
diff --git a/drivers/watchdog/sunplus_wdt.c b/drivers/watchdog/sunplus_wdt.c
index ae0c11a15a0512..f2e45907d882c4 100644
--- a/drivers/watchdog/sunplus_wdt.c
+++ b/drivers/watchdog/sunplus_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/reset.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define WDT_CTRL		0x00
@@ -38,7 +39,7 @@ MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds");
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-			__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+			__stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct sp_wdt_priv {
 	struct watchdog_device wdev;
diff --git a/drivers/watchdog/sunxi_wdt.c b/drivers/watchdog/sunxi_wdt.c
index 3db34524ed13bf..1e55c5fd43b8a9 100644
--- a/drivers/watchdog/sunxi_wdt.c
+++ b/drivers/watchdog/sunxi_wdt.c
@@ -20,6 +20,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/types.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define WDT_MAX_TIMEOUT         16
@@ -357,7 +358,7 @@ MODULE_PARM_DESC(timeout, "Watchdog heartbeat in seconds");
 
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-		"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Carlo Caione <carlo.caione@gmail.com>");
diff --git a/drivers/watchdog/tegra_wdt.c b/drivers/watchdog/tegra_wdt.c
index d5de6c0657a553..6b3f65c4d593b8 100644
--- a/drivers/watchdog/tegra_wdt.c
+++ b/drivers/watchdog/tegra_wdt.c
@@ -9,6 +9,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* minimum and maximum watchdog trigger timeout, in seconds */
@@ -67,13 +68,13 @@ static int heartbeat = WDT_HEARTBEAT;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 	"Watchdog heartbeats in seconds. (default = "
-	__MODULE_STRING(WDT_HEARTBEAT) ")");
+	__stringify(WDT_HEARTBEAT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int tegra_wdt_start(struct watchdog_device *wdd)
 {
diff --git a/drivers/watchdog/tqmx86_wdt.c b/drivers/watchdog/tqmx86_wdt.c
index 83860e94ce9d72..1da45a3f8cf3f0 100644
--- a/drivers/watchdog/tqmx86_wdt.c
+++ b/drivers/watchdog/tqmx86_wdt.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/timer.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* default timeout (secs) */
@@ -23,7 +24,7 @@ static unsigned int timeout;
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. (1<=timeout<=4096, default="
-				__MODULE_STRING(WDT_TIMEOUT) ")");
+				__stringify(WDT_TIMEOUT) ")");
 struct tqmx86_wdt {
 	struct watchdog_device wdd;
 	void __iomem *io_base;
diff --git a/drivers/watchdog/ts4800_wdt.c b/drivers/watchdog/ts4800_wdt.c
index 24b1ad52102e81..1bfc8db323c6ae 100644
--- a/drivers/watchdog/ts4800_wdt.c
+++ b/drivers/watchdog/ts4800_wdt.c
@@ -12,13 +12,14 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /* possible feed values */
 #define TS4800_WDT_FEED_2S       0x1
diff --git a/drivers/watchdog/twl4030_wdt.c b/drivers/watchdog/twl4030_wdt.c
index 69a622646d75c8..d5e06dcf74d9a0 100644
--- a/drivers/watchdog/twl4030_wdt.c
+++ b/drivers/watchdog/twl4030_wdt.c
@@ -12,13 +12,14 @@
 #include <linux/watchdog.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/twl.h>
+#include <linux/stringify.h>
 
 #define TWL4030_WATCHDOG_CFG_REG_OFFS	0x3
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int twl4030_wdt_write(unsigned char val)
 {
diff --git a/drivers/watchdog/txx9wdt.c b/drivers/watchdog/txx9wdt.c
index 305349844b4f1d..161a779dc74d8e 100644
--- a/drivers/watchdog/txx9wdt.c
+++ b/drivers/watchdog/txx9wdt.c
@@ -16,6 +16,7 @@
 #include <linux/clk.h>
 #include <linux/err.h>
 #include <linux/io.h>
+#include <linux/stringify.h>
 #include <asm/txx9tmr.h>
 
 #define WD_TIMER_CCD	7		/* 1/256 */
@@ -27,14 +28,14 @@ static unsigned int timeout = TIMER_MARGIN;	/* in seconds */
 module_param(timeout, uint, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. "
-	"(0<timeout<((2^" __MODULE_STRING(TXX9_TIMER_BITS) ")/(IMCLK/256)), "
-	"default=" __MODULE_STRING(TIMER_MARGIN) ")");
+	"(0<timeout<((2^" __stringify(TXX9_TIMER_BITS) ")/(IMCLK/256)), "
+	"default=" __stringify(TIMER_MARGIN) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static struct txx9_tmr_reg __iomem *txx9wdt_reg;
 static struct clk *txx9_imclk;
diff --git a/drivers/watchdog/uniphier_wdt.c b/drivers/watchdog/uniphier_wdt.c
index 8e9242c2302222..d09fb2acfdde3a 100644
--- a/drivers/watchdog/uniphier_wdt.c
+++ b/drivers/watchdog/uniphier_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* WDT timer setting register */
@@ -245,12 +246,12 @@ module_platform_driver(uniphier_wdt_driver);
 module_param(timeout, uint, 0000);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout seconds in power of 2. (0 < timeout < 128, default="
-				__MODULE_STRING(WDT_DEFAULT_TIMEOUT) ")");
+				__stringify(WDT_DEFAULT_TIMEOUT) ")");
 
 module_param(nowayout, bool, 0000);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 MODULE_AUTHOR("Keiji Hayashibara <hayashibara.keiji@socionext.com>");
 MODULE_DESCRIPTION("UniPhier Watchdog Device Driver");
diff --git a/drivers/watchdog/via_wdt.c b/drivers/watchdog/via_wdt.c
index be4ca619da65a8..effa9bc8d354d1 100644
--- a/drivers/watchdog/via_wdt.c
+++ b/drivers/watchdog/via_wdt.c
@@ -18,6 +18,7 @@
 #include <linux/jiffies.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/stringify.h>
 #include <linux/timer.h>
 #include <linux/watchdog.h>
 
@@ -56,12 +57,12 @@
 static int timeout = WDT_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds, between 1 and 1023 "
-	"(default = " __MODULE_STRING(WDT_TIMEOUT) ")");
+	"(default = " __stringify(WDT_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default = " __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default = " __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static struct watchdog_device wdt_dev;
 static struct resource wdt_res;
diff --git a/drivers/watchdog/visconti_wdt.c b/drivers/watchdog/visconti_wdt.c
index 7795e7fbf67e66..fc4f0d4d3d5542 100644
--- a/drivers/watchdog/visconti_wdt.c
+++ b/drivers/watchdog/visconti_wdt.c
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define WDT_CNT			0x00
@@ -29,7 +30,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(
 	nowayout,
-	"Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT)")");
+	"Watchdog cannot be stopped once started (default=" __stringify(WATCHDOG_NOWAYOUT)")");
 
 struct visconti_wdt_priv {
 	struct watchdog_device wdev;
diff --git a/drivers/watchdog/w83627hf_wdt.c b/drivers/watchdog/w83627hf_wdt.c
index db77599e43a06b..5aa4d9c0fc001b 100644
--- a/drivers/watchdog/w83627hf_wdt.c
+++ b/drivers/watchdog/w83627hf_wdt.c
@@ -35,6 +35,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/dmi.h>
+#include <linux/stringify.h>
 
 #define WATCHDOG_NAME "w83627hf/thf/hg/dhg WDT"
 #define WATCHDOG_TIMEOUT 60		/* 60 sec default timeout */
@@ -55,13 +56,13 @@ static int timeout;			/* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		"Watchdog timeout in seconds. 1 <= timeout <= 255, default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ".");
+				__stringify(WATCHDOG_TIMEOUT) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int early_disable;
 module_param(early_disable, int, 0);
diff --git a/drivers/watchdog/w83877f_wdt.c b/drivers/watchdog/w83877f_wdt.c
index 53db59ef774b80..3187ea9d1b5746 100644
--- a/drivers/watchdog/w83877f_wdt.c
+++ b/drivers/watchdog/w83877f_wdt.c
@@ -54,6 +54,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #define OUR_NAME "w83877f_wdt"
 
@@ -84,14 +85,14 @@ static int timeout = WATCHDOG_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 	"Watchdog timeout in seconds. (1<=timeout<=3600, default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+				__stringify(WATCHDOG_TIMEOUT) ")");
 
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static void wdt_timer_ping(struct timer_list *);
 static DEFINE_TIMER(timer, wdt_timer_ping);
diff --git a/drivers/watchdog/w83977f_wdt.c b/drivers/watchdog/w83977f_wdt.c
index 3776030fa7c62b..019fbf65eeb45e 100644
--- a/drivers/watchdog/w83977f_wdt.c
+++ b/drivers/watchdog/w83977f_wdt.c
@@ -25,6 +25,7 @@
 #include <linux/reboot.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/stringify.h>
 
 
 #define WATCHDOG_VERSION  "1.00"
@@ -49,7 +50,7 @@ static	DEFINE_SPINLOCK(spinlock);
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		"Watchdog timeout in seconds (15..7635), default="
-				__MODULE_STRING(DEFAULT_TIMEOUT) ")");
+				__stringify(DEFAULT_TIMEOUT) ")");
 module_param(testmode, int, 0);
 MODULE_PARM_DESC(testmode, "Watchdog testmode (1 = no reboot), default=0");
 
@@ -57,7 +58,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  * Start the watchdog
diff --git a/drivers/watchdog/wafer5823wdt.c b/drivers/watchdog/wafer5823wdt.c
index 291109349e73aa..c564d51fe5a5bd 100644
--- a/drivers/watchdog/wafer5823wdt.c
+++ b/drivers/watchdog/wafer5823wdt.c
@@ -36,6 +36,7 @@
 #include <linux/spinlock.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #define WATCHDOG_NAME "Wafer 5823 WDT"
 #define PFX WATCHDOG_NAME ": "
@@ -61,13 +62,13 @@ static int timeout = WD_TIMO;  /* in seconds */
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout,
 		"Watchdog timeout in seconds. 1 <= timeout <= 255, default="
-				__MODULE_STRING(WD_TIMO) ".");
+				__stringify(WD_TIMO) ".");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 static void wafwdt_ping(void)
 {
diff --git a/drivers/watchdog/watchdog_core.c b/drivers/watchdog/watchdog_core.c
index 38c1e63a8e2c16..2948fe52e09362 100644
--- a/drivers/watchdog/watchdog_core.c
+++ b/drivers/watchdog/watchdog_core.c
@@ -314,10 +314,7 @@ static int ___watchdog_register_device(struct watchdog_device *wdd)
 
 	/* Module parameter to force watchdog policy on reboot. */
 	if (stop_on_reboot != -1) {
-		if (stop_on_reboot)
-			set_bit(WDOG_STOP_ON_REBOOT, &wdd->status);
-		else
-			clear_bit(WDOG_STOP_ON_REBOOT, &wdd->status);
+		assign_bit(WDOG_STOP_ON_REBOOT, &wdd->status, stop_on_reboot);
 	}
 
 	if (test_bit(WDOG_STOP_ON_REBOOT, &wdd->status)) {
diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c
index d7895009a2de39..e8c1fcb53c56c0 100644
--- a/drivers/watchdog/watchdog_dev.c
+++ b/drivers/watchdog/watchdog_dev.c
@@ -44,6 +44,7 @@
 #include <linux/types.h>	/* For standard types (like size_t) */
 #include <linux/watchdog.h>	/* For watchdog specific items */
 #include <linux/uaccess.h>	/* For copy_to_user/put_user/... */
+#include <linux/stringify.h>	/* For __stringify */
 
 #include "watchdog_core.h"
 #include "watchdog_pretimeout.h"
@@ -305,6 +306,10 @@ static int watchdog_stop(struct watchdog_device *wdd)
 	if (wdd->ops->stop) {
 		clear_bit(WDOG_HW_RUNNING, &wdd->status);
 		err = wdd->ops->stop(wdd);
+		if (err < 0) {
+			pr_err("watchdog%d: Failed to stop watchdog: %pe\n",
+			       wdd->id, ERR_PTR(err));
+		}
 		trace_watchdog_stop(wdd, err);
 	} else {
 		set_bit(WDOG_HW_RUNNING, &wdd->status);
@@ -1314,9 +1319,9 @@ int watchdog_dev_resume(struct watchdog_device *wdd)
 module_param(handle_boot_enabled, bool, 0444);
 MODULE_PARM_DESC(handle_boot_enabled,
 	"Watchdog core auto-updates boot enabled watchdogs before userspace takes over (default="
-	__MODULE_STRING(IS_ENABLED(CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED)) ")");
+	__stringify(IS_ENABLED(CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED)) ")");
 
 module_param(open_timeout, uint, 0644);
 MODULE_PARM_DESC(open_timeout,
 	"Maximum time (in seconds, 0 means infinity) for userspace to take over a running watchdog (default="
-	__MODULE_STRING(CONFIG_WATCHDOG_OPEN_TIMEOUT) ")");
+	__stringify(CONFIG_WATCHDOG_OPEN_TIMEOUT) ")");
diff --git a/drivers/watchdog/wdat_wdt.c b/drivers/watchdog/wdat_wdt.c
index 55ea16e73652fc..a418da46bda2c0 100644
--- a/drivers/watchdog/wdat_wdt.c
+++ b/drivers/watchdog/wdat_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #define MAX_WDAT_ACTIONS ACPI_WDAT_ACTION_RESERVED
@@ -54,14 +55,14 @@ struct wdat_wdt {
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 #define WDAT_DEFAULT_TIMEOUT	30
 
 static int timeout = WDAT_DEFAULT_TIMEOUT;
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds (default="
-		 __MODULE_STRING(WDAT_DEFAULT_TIMEOUT) ")");
+		 __stringify(WDAT_DEFAULT_TIMEOUT) ")");
 
 static int wdat_wdt_read(struct wdat_wdt *wdat,
 	 const struct wdat_instruction *instr, u32 *value)
diff --git a/drivers/watchdog/wdt.c b/drivers/watchdog/wdt.c
index 3980d60bacd827..71517a9f8a7b96 100644
--- a/drivers/watchdog/wdt.c
+++ b/drivers/watchdog/wdt.c
@@ -43,6 +43,7 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #include "wd501p.h"
 
@@ -60,13 +61,13 @@ static int wd_heartbeat;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 	"Watchdog heartbeat in seconds. (0 < heartbeat < 65536, default="
-				__MODULE_STRING(WD_TIMO) ")");
+				__stringify(WD_TIMO) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /* You must set these - there is no sane way to probe for this board. */
 static int io = 0x240;
@@ -311,7 +312,7 @@ static irqreturn_t wdt_interrupt(int irq, void *dev_id)
  *	@ppos: pointer to the position to write. No seeks allowed
  *
  *	A write to a watchdog device is defined as a keepalive signal. Any
- *	write of data will do, as we we don't define content meaning.
+ *	write of data will do, as we don't define content meaning.
  */
 
 static ssize_t wdt_write(struct file *file, const char __user *buf,
@@ -406,7 +407,7 @@ static long wdt_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  *	open and on opening we load the counters. Counter zero is a 100Hz
  *	cascade, into counter 1 which downcounts to reboot. When the counter
  *	triggers counter 2 downcounts the length of the reset pulse which
- *	set set to be as long as possible.
+ *	set to be as long as possible.
  */
 
 static int wdt_open(struct inode *inode, struct file *file)
diff --git a/drivers/watchdog/wdt977.c b/drivers/watchdog/wdt977.c
index 4f449ac4dda412..0d2e1e0c16cf99 100644
--- a/drivers/watchdog/wdt977.c
+++ b/drivers/watchdog/wdt977.c
@@ -34,6 +34,7 @@
 #include <linux/reboot.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #include <asm/mach-types.h>
 
@@ -59,7 +60,7 @@ static	DEFINE_SPINLOCK(spinlock);
 
 module_param(timeout, int, 0);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds (60..15300, default="
-				__MODULE_STRING(DEFAULT_TIMEOUT) ")");
+				__stringify(DEFAULT_TIMEOUT) ")");
 module_param(testmode, int, 0);
 MODULE_PARM_DESC(testmode, "Watchdog testmode (1 = no reboot), default=0");
 
@@ -67,7 +68,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /*
  * Start the watchdog
@@ -298,7 +299,7 @@ static int wdt977_release(struct inode *inode, struct file *file)
  *      @ppos: pointer to the position to write. No seeks allowed
  *
  *      A write to a watchdog device is defined as a keepalive signal. Any
- *      write of data will do, as we we don't define content meaning.
+ *      write of data will do, as we don't define content meaning.
  */
 
 static ssize_t wdt977_write(struct file *file, const char __user *buf,
diff --git a/drivers/watchdog/wdt_pci.c b/drivers/watchdog/wdt_pci.c
index 3918a600f2a00b..dd5944c87a96f1 100644
--- a/drivers/watchdog/wdt_pci.c
+++ b/drivers/watchdog/wdt_pci.c
@@ -49,6 +49,7 @@
 #include <linux/pci.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 
 #define WDT_IS_PCI
@@ -72,13 +73,13 @@ static int wd_heartbeat;
 module_param(heartbeat, int, 0);
 MODULE_PARM_DESC(heartbeat,
 		"Watchdog heartbeat in seconds. (0<heartbeat<65536, default="
-				__MODULE_STRING(WD_TIMO) ")");
+				__stringify(WD_TIMO) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+				__stringify(WATCHDOG_NOWAYOUT) ")");
 
 /* Support for the Fan Tachometer on the PCI-WDT501 */
 static int tachometer;
@@ -347,7 +348,7 @@ static irqreturn_t wdtpci_interrupt(int irq, void *dev_id)
  *	@ppos: pointer to the position to write. No seeks allowed
  *
  *	A write to a watchdog device is defined as a keepalive signal. Any
- *	write of data will do, as we we don't define content meaning.
+ *	write of data will do, as we don't define content meaning.
  */
 
 static ssize_t wdtpci_write(struct file *file, const char __user *buf,
@@ -443,7 +444,7 @@ static long wdtpci_ioctl(struct file *file, unsigned int cmd,
  *	open and on opening we load the counters. Counter zero is a 100Hz
  *	cascade, into counter 1 which downcounts to reboot. When the counter
  *	triggers counter 2 downcounts the length of the reset pulse which
- *	set set to be as long as possible.
+ *	set to be as long as possible.
  */
 
 static int wdtpci_open(struct inode *inode, struct file *file)
diff --git a/drivers/watchdog/wm831x_wdt.c b/drivers/watchdog/wm831x_wdt.c
index d96ad8f38bd2e6..1aeb532dfebae7 100644
--- a/drivers/watchdog/wm831x_wdt.c
+++ b/drivers/watchdog/wm831x_wdt.c
@@ -13,6 +13,7 @@
 #include <linux/platform_device.h>
 #include <linux/watchdog.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 
 #include <linux/mfd/wm831x/core.h>
 #include <linux/mfd/wm831x/pdata.h>
@@ -22,7 +23,7 @@ static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 struct wm831x_wdt_drvdata {
 	struct watchdog_device wdt;
diff --git a/drivers/watchdog/wm8350_wdt.c b/drivers/watchdog/wm8350_wdt.c
index c82c1b77d91b51..5eca195eac1886 100644
--- a/drivers/watchdog/wm8350_wdt.c
+++ b/drivers/watchdog/wm8350_wdt.c
@@ -14,13 +14,14 @@
 #include <linux/platform_device.h>
 #include <linux/watchdog.h>
 #include <linux/uaccess.h>
+#include <linux/stringify.h>
 #include <linux/mfd/wm8350/core.h>
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout,
 		 "Watchdog cannot be stopped once started (default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static DEFINE_MUTEX(wdt_mutex);
 
diff --git a/drivers/watchdog/xen_wdt.c b/drivers/watchdog/xen_wdt.c
index b343f421dc72c5..3bc323d886d0be 100644
--- a/drivers/watchdog/xen_wdt.c
+++ b/drivers/watchdog/xen_wdt.c
@@ -17,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 #include <xen/xen.h>
 #include <asm/xen/hypercall.h>
@@ -30,12 +31,12 @@ static time64_t wdt_expires;
 static unsigned int timeout;
 module_param(timeout, uint, S_IRUGO);
 MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds "
-	"(default=" __MODULE_STRING(WATCHDOG_TIMEOUT) ")");
+	"(default=" __stringify(WATCHDOG_TIMEOUT) ")");
 
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, S_IRUGO);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started "
-	"(default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+	"(default=" __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static inline time64_t set_timeout(struct watchdog_device *wdd)
 {
diff --git a/drivers/watchdog/xilinx_wwdt.c b/drivers/watchdog/xilinx_wwdt.c
index 799ce8d22b2f3a..b7cdf8140666f9 100644
--- a/drivers/watchdog/xilinx_wwdt.c
+++ b/drivers/watchdog/xilinx_wwdt.c
@@ -12,6 +12,7 @@
 #include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 /* Max timeout is calculated at 100MHz source clock */
@@ -47,11 +48,11 @@ static int closed_window_percent;
 module_param(wwdt_timeout, int, 0);
 MODULE_PARM_DESC(wwdt_timeout,
 		 "Watchdog time in seconds. (default="
-		 __MODULE_STRING(XWWDT_DEFAULT_TIMEOUT) ")");
+		 __stringify(XWWDT_DEFAULT_TIMEOUT) ")");
 module_param(closed_window_percent, int, 0);
 MODULE_PARM_DESC(closed_window_percent,
 		 "Watchdog closed window percentage. (default="
-		 __MODULE_STRING(XWWDT_CLOSE_WINDOW_PERCENT) ")");
+		 __stringify(XWWDT_CLOSE_WINDOW_PERCENT) ")");
 /**
  * struct xwwdt_device - Watchdog device structure
  * @base: base io address of WDT device
diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c
index f3bb935c08c2ed..10d4eaa8592a04 100644
--- a/drivers/watchdog/ziirave_wdt.c
+++ b/drivers/watchdog/ziirave_wdt.c
@@ -18,6 +18,7 @@
 #include <linux/slab.h>
 #include <linux/sysfs.h>
 #include <linux/types.h>
+#include <linux/stringify.h>
 #include <linux/watchdog.h>
 
 #include <linux/unaligned.h>
@@ -94,7 +95,7 @@ MODULE_PARM_DESC(reset_duration,
 static bool nowayout = WATCHDOG_NOWAYOUT;
 module_param(nowayout, bool, 0);
 MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started default="
-		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+		 __stringify(WATCHDOG_NOWAYOUT) ")");
 
 static int ziirave_wdt_revision(struct i2c_client *client,
 				struct ziirave_wdt_rev *rev, u8 command)
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index e7f74ea7cd5ebc..373042fe501bdd 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -48,6 +48,7 @@
 #include <linux/mm.h>
 #include <linux/memblock.h>
 #include <linux/pagemap.h>
+#include <linux/swap.h>
 #include <linux/highmem.h>
 #include <linux/mutex.h>
 #include <linux/list.h>
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 076c1b0ab87fd1..f75b5cc71cfc5a 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -863,10 +863,10 @@ EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
 
 int gnttab_pages_set_private(int nr_pages, struct page **pages)
 {
+#if BITS_PER_LONG < 64
 	int i;
 
 	for (i = 0; i < nr_pages; i++) {
-#if BITS_PER_LONG < 64
 		struct xen_page_foreign *foreign;
 
 		foreign = kzalloc_obj(*foreign);
@@ -874,9 +874,9 @@ int gnttab_pages_set_private(int nr_pages, struct page **pages)
 			return -ENOMEM;
 
 		set_page_private(pages[i], (unsigned long)foreign);
-#endif
-		SetPagePrivate(pages[i]);
 	}
+#endif
+	/* Data is stored in page->private on 64-bit */
 
 	return 0;
 }
@@ -1031,12 +1031,11 @@ void gnttab_pages_clear_private(int nr_pages, struct page **pages)
 	int i;
 
 	for (i = 0; i < nr_pages; i++) {
-		if (PagePrivate(pages[i])) {
 #if BITS_PER_LONG < 64
+		if (page_private(pages[i]))
 			kfree((void *)page_private(pages[i]));
 #endif
-			ClearPagePrivate(pages[i]);
-		}
+		set_page_private(pages[i], 0);
 	}
 }
 EXPORT_SYMBOL_GPL(gnttab_pages_clear_private);
diff --git a/drivers/xen/time.c b/drivers/xen/time.c
index a2be0a4d45b0ba..a02d48a2aa684e 100644
--- a/drivers/xen/time.c
+++ b/drivers/xen/time.c
@@ -169,7 +169,7 @@ void __init xen_time_setup_guest(void)
 
 	static_call_update(pv_steal_clock, xen_steal_clock);
 
-	static_key_slow_inc(&paravirt_steal_enabled);
+	static_branch_inc(&paravirt_steal_enabled);
 	if (xen_runstate_remote)
-		static_key_slow_inc(&paravirt_steal_rq_enabled);
+		static_branch_inc(&paravirt_steal_rq_enabled);
 }
diff --git a/drivers/zorro/zorro.c b/drivers/zorro/zorro.c
index 45495c423c79d7..cc3177cc681c45 100644
--- a/drivers/zorro/zorro.c
+++ b/drivers/zorro/zorro.c
@@ -103,10 +103,7 @@ static void __init mark_region(unsigned long start, unsigned long end,
 	while (start < end) {
 		u32 chunk = start>>Z2RAM_CHUNKSHIFT;
 
-		if (flag)
-			set_bit(chunk, zorro_unused_z2ram);
-		else
-			clear_bit(chunk, zorro_unused_z2ram);
+		assign_bit(chunk, zorro_unused_z2ram, flag);
 		start += Z2RAM_CHUNKSIZE;
 	}
 }
diff --git a/fs/9p/acl.c b/fs/9p/acl.c
index ae7e7cf7523a16..c6c7c47d32b9e8 100644
--- a/fs/9p/acl.c
+++ b/fs/9p/acl.c
@@ -140,7 +140,7 @@ struct posix_acl *v9fs_iop_get_inode_acl(struct inode *inode, int type, bool rcu
 
 }
 
-struct posix_acl *v9fs_iop_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *v9fs_iop_get_acl(const struct mnt_idmap *idmap,
 				   struct dentry *dentry, int type)
 {
 	struct v9fs_session_info *v9ses;
@@ -152,7 +152,7 @@ struct posix_acl *v9fs_iop_get_acl(struct mnt_idmap *idmap,
 	return v9fs_get_cached_acl(d_inode(dentry), type);
 }
 
-int v9fs_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int v9fs_iop_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		     struct posix_acl *acl, int type)
 {
 	int retval;
diff --git a/fs/9p/acl.h b/fs/9p/acl.h
index 333cfcc281da44..2d1b24abcd3f50 100644
--- a/fs/9p/acl.h
+++ b/fs/9p/acl.h
@@ -10,9 +10,9 @@
 int v9fs_get_acl(struct inode *inode, struct p9_fid *fid);
 struct posix_acl *v9fs_iop_get_inode_acl(struct inode *inode, int type,
 				   bool rcu);
-struct posix_acl *v9fs_iop_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *v9fs_iop_get_acl(const struct mnt_idmap *idmap,
 					  struct dentry *dentry, int type);
-int v9fs_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int v9fs_iop_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		     struct posix_acl *acl, int type);
 int v9fs_acl_chmod(struct inode *inode, struct p9_fid *fid);
 int v9fs_set_create_acl(struct inode *inode, struct p9_fid *fid,
diff --git a/fs/9p/v9fs.h b/fs/9p/v9fs.h
index a462bcbfc7da48..54a4a4ec5c15bc 100644
--- a/fs/9p/v9fs.h
+++ b/fs/9p/v9fs.h
@@ -188,7 +188,7 @@ extern struct dentry *v9fs_vfs_lookup(struct inode *dir, struct dentry *dentry,
 				      unsigned int flags);
 extern int v9fs_vfs_unlink(struct inode *i, struct dentry *d);
 extern int v9fs_vfs_rmdir(struct inode *i, struct dentry *d);
-extern int v9fs_vfs_rename(struct mnt_idmap *idmap,
+extern int v9fs_vfs_rename(const struct mnt_idmap *idmap,
 			   struct inode *old_dir, struct dentry *old_dentry,
 			   struct inode *new_dir, struct dentry *new_dentry,
 			   unsigned int flags);
diff --git a/fs/9p/v9fs_vfs.h b/fs/9p/v9fs_vfs.h
index 1856d91f87030a..5e7b600424988d 100644
--- a/fs/9p/v9fs_vfs.h
+++ b/fs/9p/v9fs_vfs.h
@@ -74,7 +74,7 @@ int v9fs_file_open(struct inode *inode, struct file *file);
 int v9fs_uflags2omode(int uflags, int extended);
 
 void v9fs_blank_wstat(struct p9_wstat *wstat);
-int v9fs_vfs_setattr_dotl(struct mnt_idmap *idmap,
+int v9fs_vfs_setattr_dotl(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, struct iattr *iattr);
 int v9fs_file_fsync_dotl(struct file *filp, loff_t start, loff_t end,
 			 int datasync);
diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c
index 13cf87a5f90cb7..170a2b91c5f07a 100644
--- a/fs/9p/vfs_addr.c
+++ b/fs/9p/vfs_addr.c
@@ -150,7 +150,6 @@ static int v9fs_init_request(struct netfs_io_request *rreq, struct file *file)
 	struct p9_fid *fid;
 	struct dentry *dentry;
 	bool writing = (rreq->origin == NETFS_READ_FOR_WRITE ||
-			rreq->origin == NETFS_WRITETHROUGH ||
 			rreq->origin == NETFS_UNBUFFERED_WRITE ||
 			rreq->origin == NETFS_DIO_WRITE);
 
diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c
index 3829554ca36929..c95e653344f63e 100644
--- a/fs/9p/vfs_inode.c
+++ b/fs/9p/vfs_inode.c
@@ -652,7 +652,7 @@ v9fs_create(struct v9fs_session_info *v9ses, struct inode *dir,
  */
 
 static int
-v9fs_vfs_create(struct mnt_idmap *idmap, struct inode *dir,
+v9fs_vfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *dentry, umode_t mode)
 {
 	struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir);
@@ -679,7 +679,7 @@ v9fs_vfs_create(struct mnt_idmap *idmap, struct inode *dir,
  *
  */
 
-static struct dentry *v9fs_vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *v9fs_vfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				     struct dentry *dentry, umode_t mode)
 {
 	u32 perm;
@@ -858,7 +858,7 @@ int v9fs_vfs_rmdir(struct inode *i, struct dentry *d)
  */
 
 int
-v9fs_vfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+v9fs_vfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		struct dentry *old_dentry, struct inode *new_dir,
 		struct dentry *new_dentry, unsigned int flags)
 {
@@ -966,7 +966,7 @@ v9fs_vfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
  */
 
 static int
-v9fs_vfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+v9fs_vfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	struct dentry *dentry = path->dentry;
@@ -1014,7 +1014,7 @@ v9fs_vfs_getattr(struct mnt_idmap *idmap, const struct path *path,
  *
  */
 
-static int v9fs_vfs_setattr(struct mnt_idmap *idmap,
+static int v9fs_vfs_setattr(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct iattr *iattr)
 {
 	int retval, use_dentry = 0;
@@ -1249,7 +1249,7 @@ static int v9fs_vfs_mkspecial(struct inode *dir, struct dentry *dentry,
  */
 
 static int
-v9fs_vfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+v9fs_vfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		 struct dentry *dentry, const char *symname)
 {
 	p9_debug(P9_DEBUG_VFS, " %llu,%pd,%s\n",
@@ -1304,7 +1304,7 @@ v9fs_vfs_link(struct dentry *old_dentry, struct inode *dir,
  */
 
 static int
-v9fs_vfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+v9fs_vfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 	       struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir);
diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
index 116b29e95f21ee..2cd2898580a94b 100644
--- a/fs/9p/vfs_inode_dotl.c
+++ b/fs/9p/vfs_inode_dotl.c
@@ -29,7 +29,7 @@
 #include "acl.h"
 
 static int
-v9fs_vfs_mknod_dotl(struct mnt_idmap *idmap, struct inode *dir,
+v9fs_vfs_mknod_dotl(const struct mnt_idmap *idmap, struct inode *dir,
 		    struct dentry *dentry, umode_t omode, dev_t rdev);
 
 /**
@@ -216,7 +216,7 @@ int v9fs_open_to_dotl_flags(int flags)
  *
  */
 static int
-v9fs_vfs_create_dotl(struct mnt_idmap *idmap, struct inode *dir,
+v9fs_vfs_create_dotl(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct dentry *dentry, umode_t omode)
 {
 	return v9fs_vfs_mknod_dotl(idmap, dir, dentry, omode, 0);
@@ -344,7 +344,7 @@ v9fs_vfs_atomic_open_dotl(struct inode *dir, struct dentry *dentry,
  *
  */
 
-static struct dentry *v9fs_vfs_mkdir_dotl(struct mnt_idmap *idmap,
+static struct dentry *v9fs_vfs_mkdir_dotl(const struct mnt_idmap *idmap,
 					  struct inode *dir, struct dentry *dentry,
 					  umode_t omode)
 {
@@ -414,7 +414,7 @@ static struct dentry *v9fs_vfs_mkdir_dotl(struct mnt_idmap *idmap,
 }
 
 static int
-v9fs_vfs_getattr_dotl(struct mnt_idmap *idmap,
+v9fs_vfs_getattr_dotl(const struct mnt_idmap *idmap,
 		      const struct path *path, struct kstat *stat,
 		      u32 request_mask, unsigned int flags)
 {
@@ -508,7 +508,7 @@ static int v9fs_mapped_iattr_valid(int iattr_valid)
  *
  */
 
-int v9fs_vfs_setattr_dotl(struct mnt_idmap *idmap,
+int v9fs_vfs_setattr_dotl(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, struct iattr *iattr)
 {
 	int retval, use_dentry = 0;
@@ -682,7 +682,7 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode,
 }
 
 static int
-v9fs_vfs_symlink_dotl(struct mnt_idmap *idmap, struct inode *dir,
+v9fs_vfs_symlink_dotl(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, const char *symname)
 {
 	int err;
@@ -809,7 +809,7 @@ v9fs_vfs_link_dotl(struct dentry *old_dentry, struct inode *dir,
  *
  */
 static int
-v9fs_vfs_mknod_dotl(struct mnt_idmap *idmap, struct inode *dir,
+v9fs_vfs_mknod_dotl(const struct mnt_idmap *idmap, struct inode *dir,
 		    struct dentry *dentry, umode_t omode, dev_t rdev)
 {
 	int err;
diff --git a/fs/9p/xattr.c b/fs/9p/xattr.c
index 8604e3377ee7ab..dac06587f67ace 100644
--- a/fs/9p/xattr.c
+++ b/fs/9p/xattr.c
@@ -153,7 +153,7 @@ static int v9fs_xattr_handler_get(const struct xattr_handler *handler,
 }
 
 static int v9fs_xattr_handler_set(const struct xattr_handler *handler,
-				  struct mnt_idmap *idmap,
+				  const struct mnt_idmap *idmap,
 				  struct dentry *dentry, struct inode *inode,
 				  const char *name, const void *value,
 				  size_t size, int flags)
diff --git a/fs/Kconfig b/fs/Kconfig
index e05917adcd608e..70a1d0760d2897 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -278,7 +278,7 @@ config HUGETLB_PAGE_OPTIMIZE_VMEMMAP
 	def_bool HUGETLB_PAGE
 	depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
 	depends on SPARSEMEM_VMEMMAP
-	select SPARSEMEM_VMEMMAP_PREINIT if ARCH_WANT_HUGETLB_VMEMMAP_PREINIT
+	select VMEMMAP_OPTIMIZATION
 
 config HUGETLB_PMD_PAGE_TABLE_SHARING
 	def_bool HUGETLB_PAGE
@@ -314,7 +314,6 @@ source "fs/ecryptfs/Kconfig"
 source "fs/hfs/Kconfig"
 source "fs/hfsplus/Kconfig"
 source "fs/befs/Kconfig"
-source "fs/bfs/Kconfig"
 source "fs/jffs2/Kconfig"
 # UBIFS File system configuration
 source "fs/ubifs/Kconfig"
@@ -421,4 +420,12 @@ source "fs/unicode/Kconfig"
 config IO_WQ
 	bool
 
+config FDTABLE_KUNIT_TEST
+	bool "KUnit test for fdtable" if !KUNIT_ALL_TESTS
+	depends on KUNIT=y
+	default KUNIT_ALL_TESTS
+	help
+	  This builds the fdtable KUnit tests, which tests various aspects
+	  of the fdtable structure and allocation.
+
 endmenu
diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt
index daeac4889d03ef..e36cd94513b9bd 100644
--- a/fs/Kconfig.binfmt
+++ b/fs/Kconfig.binfmt
@@ -58,7 +58,7 @@ config ARCH_USE_GNU_PROPERTY
 config BINFMT_ELF_FDPIC
 	bool "Kernel support for FDPIC ELF binaries"
 	default y if !BINFMT_ELF
-	depends on ARM || ((M68K || RISCV || SUPERH || XTENSA) && !MMU)
+	depends on ARM || ((M68K || RISCV || SUPERH || UML || XTENSA) && !MMU)
 	select ELFCORE
 	help
 	  ELF FDPIC binaries are based on ELF, but allow the individual load
diff --git a/fs/Makefile b/fs/Makefile
index 055dfc23d82b6f..16f1108b64e1ab 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -76,7 +76,6 @@ obj-$(CONFIG_CODA_FS)		+= coda/
 obj-$(CONFIG_MINIX_FS)		+= minix/
 obj-$(CONFIG_FAT_FS)		+= fat/
 obj-$(CONFIG_EXFAT_FS)		+= exfat/
-obj-$(CONFIG_BFS_FS)		+= bfs/
 obj-$(CONFIG_ISO9660_FS)	+= isofs/
 obj-$(CONFIG_HFSPLUS_FS)	+= hfsplus/ # Before hfs to find wrapped HFS+
 obj-$(CONFIG_HFS_FS)		+= hfs/
diff --git a/fs/adfs/adfs.h b/fs/adfs/adfs.h
index 0d32b7cd99b436..6003832277f87e 100644
--- a/fs/adfs/adfs.h
+++ b/fs/adfs/adfs.h
@@ -144,7 +144,7 @@ struct adfs_discmap {
 /* Inode stuff */
 struct inode *adfs_iget(struct super_block *sb, struct object_info *obj);
 int adfs_write_inode(struct inode *inode, struct writeback_control *wbc);
-int adfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int adfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *attr);
 
 /* map.c */
diff --git a/fs/adfs/dir.c b/fs/adfs/dir.c
index 11afa9e157aa70..b8cc6a697a05dd 100644
--- a/fs/adfs/dir.c
+++ b/fs/adfs/dir.c
@@ -191,7 +191,7 @@ static int adfs_dir_sync(struct adfs_dir *dir)
 	for (i = dir->nr_buffers - 1; i >= 0; i--) {
 		struct buffer_head *bh = dir->bhs[i];
 		sync_dirty_buffer(bh);
-		if (buffer_req(bh) && !buffer_uptodate(bh))
+		if (buffer_write_io_error(bh))
 			err = -EIO;
 	}
 
diff --git a/fs/adfs/inode.c b/fs/adfs/inode.c
index 4ac442d0a8c044..34598b499372b6 100644
--- a/fs/adfs/inode.c
+++ b/fs/adfs/inode.c
@@ -299,7 +299,7 @@ adfs_iget(struct super_block *sb, struct object_info *obj)
  * later.
  */
 int
-adfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr)
+adfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
 	struct super_block *sb = inode->i_sb;
diff --git a/fs/affs/affs.h b/fs/affs/affs.h
index d1c506c1f3105f..2518de96a0c8ea 100644
--- a/fs/affs/affs.h
+++ b/fs/affs/affs.h
@@ -166,17 +166,17 @@ extern const struct export_operations affs_export_ops;
 extern int	affs_hash_name(struct super_block *sb, const u8 *name, unsigned int len);
 extern struct dentry *affs_lookup(struct inode *dir, struct dentry *dentry, unsigned int);
 extern int	affs_unlink(struct inode *dir, struct dentry *dentry);
-extern int	affs_create(struct mnt_idmap *idmap, struct inode *dir,
+extern int	affs_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode);
-extern struct dentry *affs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+extern struct dentry *affs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode);
 extern int	affs_rmdir(struct inode *dir, struct dentry *dentry);
 extern int	affs_link(struct dentry *olddentry, struct inode *dir,
 			  struct dentry *dentry);
-extern int	affs_symlink(struct mnt_idmap *idmap,
+extern int	affs_symlink(const struct mnt_idmap *idmap,
 			struct inode *dir, struct dentry *dentry,
 			const char *symname);
-extern int	affs_rename2(struct mnt_idmap *idmap,
+extern int	affs_rename2(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags);
@@ -184,7 +184,7 @@ extern int	affs_rename2(struct mnt_idmap *idmap,
 /* inode.c */
 
 extern struct inode		*affs_new_inode(struct inode *dir);
-extern int			 affs_setattr(struct mnt_idmap *idmap,
+extern int			 affs_setattr(const struct mnt_idmap *idmap,
 					struct dentry *dentry, struct iattr *attr);
 extern void			 affs_evict_inode(struct inode *inode);
 extern struct inode		*affs_iget(struct super_block *sb,
diff --git a/fs/affs/inode.c b/fs/affs/inode.c
index d4a3f381c4bcff..2a48d742209171 100644
--- a/fs/affs/inode.c
+++ b/fs/affs/inode.c
@@ -213,7 +213,7 @@ affs_write_inode(struct inode *inode, struct writeback_control *wbc)
 }
 
 int
-affs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr)
+affs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
 	int error;
diff --git a/fs/affs/namei.c b/fs/affs/namei.c
index 6cb52efafe5fed..2e32899a32d5cc 100644
--- a/fs/affs/namei.c
+++ b/fs/affs/namei.c
@@ -242,7 +242,7 @@ affs_unlink(struct inode *dir, struct dentry *dentry)
 }
 
 int
-affs_create(struct mnt_idmap *idmap, struct inode *dir,
+affs_create(const struct mnt_idmap *idmap, struct inode *dir,
 	    struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -274,7 +274,7 @@ affs_create(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 struct dentry *
-affs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+affs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 	   struct dentry *dentry, umode_t mode)
 {
 	struct inode		*inode;
@@ -313,7 +313,7 @@ affs_rmdir(struct inode *dir, struct dentry *dentry)
 }
 
 int
-affs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+affs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 	     struct dentry *dentry, const char *symname)
 {
 	struct super_block	*sb = dir->i_sb;
@@ -503,7 +503,7 @@ affs_xrename(struct inode *old_dir, struct dentry *old_dentry,
 	return retval;
 }
 
-int affs_rename2(struct mnt_idmap *idmap, struct inode *old_dir,
+int affs_rename2(const struct mnt_idmap *idmap, struct inode *old_dir,
 		 struct dentry *old_dentry, struct inode *new_dir,
 		 struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index 2db534a2c7cc49..75f8f70cfeff02 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -33,17 +33,17 @@ static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, in
 static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen,
 			       u64 ino, u32 uniquifier);
 #define AFS_LOOKUP ((filldir_t)0x137UL)
-static int afs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int afs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode);
-static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *afs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode);
 static int afs_rmdir(struct inode *dir, struct dentry *dentry);
 static int afs_unlink(struct inode *dir, struct dentry *dentry);
 static int afs_link(struct dentry *from, struct inode *dir,
 		    struct dentry *dentry);
-static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int afs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, const char *content);
-static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int afs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags);
 static int afs_dir_writepages(struct address_space *mapping,
@@ -1310,7 +1310,7 @@ static const struct afs_operation_ops afs_mkdir_operation = {
 /*
  * create a directory on an AFS filesystem
  */
-static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *afs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode)
 {
 	struct afs_operation *op;
@@ -1632,7 +1632,7 @@ static const struct afs_operation_ops afs_create_operation = {
 /*
  * create a regular file on an AFS filesystem
  */
-static int afs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int afs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode)
 {
 	struct afs_operation *op;
@@ -1779,7 +1779,7 @@ static const struct afs_operation_ops afs_symlink_operation = {
 /*
  * create a symlink in an AFS filesystem
  */
-static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int afs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, const char *content)
 {
 	struct afs_operation *op;
@@ -2067,7 +2067,7 @@ static const struct afs_operation_ops afs_rename_exchange_operation = {
 /*
  * rename a file in an AFS filesystem and/or move it between directories
  */
-static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int afs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/afs/file.c b/fs/afs/file.c
index 0467742bfeee34..4c78d3441785dd 100644
--- a/fs/afs/file.c
+++ b/fs/afs/file.c
@@ -400,7 +400,6 @@ static int afs_init_request(struct netfs_io_request *rreq, struct file *file)
 		}
 		break;
 	case NETFS_WRITEBACK:
-	case NETFS_WRITETHROUGH:
 	case NETFS_UNBUFFERED_WRITE:
 	case NETFS_DIO_WRITE:
 		if (S_ISREG(rreq->inode->i_mode))
@@ -413,7 +412,7 @@ static int afs_init_request(struct netfs_io_request *rreq, struct file *file)
 	return 0;
 }
 
-static int afs_check_write_begin(struct file *file, loff_t pos, unsigned len,
+static int afs_check_write_begin(struct file *file, uoff_t pos, unsigned len,
 				 struct folio **foliop, void **_fsdata)
 {
 	struct afs_vnode *vnode = AFS_FS_I(file_inode(file));
@@ -434,7 +433,7 @@ static void afs_free_request(struct netfs_io_request *rreq)
  * Also, estimate the number of 512 bytes blocks used, rounded up to nearest 1K
  * for consistency with other AFS clients.
  */
-void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size)
+void afs_set_i_size(struct afs_vnode *vnode, uoff_t new_i_size)
 {
 	struct inode *inode = &vnode->netfs.inode;
 	loff_t i_size;
@@ -448,10 +447,9 @@ void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size)
 	}
 	spin_unlock(&inode->i_lock);
 	write_sequnlock(&vnode->cb_lock);
-	fscache_update_cookie(afs_vnode_cache(vnode), NULL, &new_i_size);
 }
 
-static void afs_update_i_size(struct inode *inode, loff_t new_i_size)
+static void afs_update_i_size(struct inode *inode, uoff_t new_i_size)
 {
 	afs_set_i_size(AFS_FS_I(inode), new_i_size);
 }
diff --git a/fs/afs/inode.c b/fs/afs/inode.c
index 14f39a9bea6cfe..8e6ca6b45c6b1e 100644
--- a/fs/afs/inode.c
+++ b/fs/afs/inode.c
@@ -596,7 +596,7 @@ struct inode *afs_root_iget(struct super_block *sb, struct key *key)
 /*
  * read the attributes of an inode
  */
-int afs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int afs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -759,7 +759,7 @@ static const struct afs_operation_ops afs_setattr_operation = {
 /*
  * set the attributes of an inode
  */
-int afs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int afs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *attr)
 {
 	const unsigned int supported =
diff --git a/fs/afs/internal.h b/fs/afs/internal.h
index 330654ed16ecea..5744a347ce2e79 100644
--- a/fs/afs/internal.h
+++ b/fs/afs/internal.h
@@ -1171,7 +1171,7 @@ extern int afs_open(struct inode *, struct file *);
 extern int afs_release(struct inode *, struct file *);
 void afs_fetch_data_async_rx(struct work_struct *work);
 void afs_fetch_data_immediate_cancel(struct afs_call *call);
-void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size);
+void afs_set_i_size(struct afs_vnode *vnode, uoff_t new_i_size);
 
 /*
  * flock.c
@@ -1266,9 +1266,9 @@ extern int afs_fetch_status(struct afs_vnode *, struct key *, bool, afs_access_t
 extern int afs_ilookup5_test_by_fid(struct inode *, void *);
 extern struct inode *afs_iget(struct afs_operation *, struct afs_vnode_param *);
 extern struct inode *afs_root_iget(struct super_block *, struct key *);
-extern int afs_getattr(struct mnt_idmap *idmap, const struct path *,
+extern int afs_getattr(const struct mnt_idmap *idmap, const struct path *,
 		       struct kstat *, u32, unsigned int);
-extern int afs_setattr(struct mnt_idmap *idmap, struct dentry *, struct iattr *);
+extern int afs_setattr(const struct mnt_idmap *idmap, struct dentry *, struct iattr *);
 extern void afs_evict_inode(struct inode *);
 extern int afs_drop_inode(struct inode *);
 
@@ -1538,7 +1538,7 @@ extern void afs_cache_permit(struct afs_vnode *, struct key *, unsigned int,
 extern struct key *afs_request_key(struct afs_cell *);
 extern struct key *afs_request_key_rcu(struct afs_cell *);
 extern int afs_check_permit(struct afs_vnode *, struct key *, afs_access_t *);
-extern int afs_permission(struct mnt_idmap *, struct inode *, int);
+extern int afs_permission(const struct mnt_idmap *, struct inode *, int);
 extern void __exit afs_clean_up_permit_cache(void);
 
 /*
diff --git a/fs/afs/security.c b/fs/afs/security.c
index 6d00d62a65edf6..fd040f7c6766c7 100644
--- a/fs/afs/security.c
+++ b/fs/afs/security.c
@@ -428,7 +428,7 @@ int afs_check_permit(struct afs_vnode *vnode, struct key *key,
  * - AFS ACLs are attached to directories only, and a file is controlled by its
  *   parent directory's ACL
  */
-int afs_permission(struct mnt_idmap *idmap, struct inode *inode,
+int afs_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		   int mask)
 {
 	struct afs_vnode *vnode = AFS_FS_I(inode);
diff --git a/fs/afs/xattr.c b/fs/afs/xattr.c
index 3770ed236f671c..bcffd7236cc9c9 100644
--- a/fs/afs/xattr.c
+++ b/fs/afs/xattr.c
@@ -97,7 +97,7 @@ static const struct afs_operation_ops afs_store_acl_operation = {
  * Set a file's AFS3 ACL.
  */
 static int afs_xattr_set_acl(const struct xattr_handler *handler,
-			     struct mnt_idmap *idmap,
+			     const struct mnt_idmap *idmap,
                              struct dentry *dentry,
                              struct inode *inode, const char *name,
                              const void *buffer, size_t size, int flags)
@@ -228,7 +228,7 @@ static const struct afs_operation_ops yfs_store_opaque_acl2_operation = {
  * Set a file's YFS ACL.
  */
 static int afs_xattr_set_yfs(const struct xattr_handler *handler,
-			     struct mnt_idmap *idmap,
+			     const struct mnt_idmap *idmap,
                              struct dentry *dentry,
                              struct inode *inode, const char *name,
                              const void *buffer, size_t size, int flags)
diff --git a/fs/aio.c b/fs/aio.c
index d78acc69f48701..8a62b770a70e01 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -27,6 +27,7 @@
 #include <linux/file.h>
 #include <linux/mm.h>
 #include <linux/mman.h>
+#include <linux/pagemap.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
 #include <linux/timer.h>
@@ -936,7 +937,7 @@ static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx,
 
 /*
  * exit_aio: called when the last user of mm goes away.  At this point, there is
- * no way for any new requests to be submited or any of the io_* syscalls to be
+ * no way for any new requests to be submitted or any of the io_* syscalls to be
  * called on the context.
  *
  * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on
@@ -1280,7 +1281,7 @@ static long aio_read_events_ring(struct kioctx *ctx,
 	 * The mutex can block and wake us up and that will cause
 	 * wait_event_interruptible_hrtimeout() to schedule without sleeping
 	 * and repeat. This should be rare enough that it doesn't cause
-	 * peformance issues. See the comment in read_events() for more detail.
+	 * performance issues. See the comment in read_events() for more detail.
 	 */
 	sched_annotate_sleep();
 	mutex_lock(&ctx->ring_lock);
@@ -1402,7 +1403,7 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr,
 		w.min_nr = min_nr - ret;
 
 		ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE);
-		if (!ret2 && !t.task)
+		if (!ret2 && !hrtimer_sleeper_task_get(&t))
 			ret2 = -ETIME;
 
 		if (aio_read_events(ctx, min_nr, nr, event, &ret) || ret2)
@@ -1869,7 +1870,12 @@ static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
 		list_del_init(&req->wait.entry);
 		list_del(&iocb->ki_list);
 		iocb->ki_res.res = mangle_poll(mask);
-		if (iocb->ki_eventfd && !eventfd_signal_allowed()) {
+		/*
+		 * We hold an arbitrary provider waitqueue lock here.  Signaling a
+		 * result eventfd can feed back through epoll and try to take the same
+		 * lock again.  Defer all eventfd-backed poll completions.
+		 */
+		if (iocb->ki_eventfd) {
 			iocb = NULL;
 			INIT_WORK(&req->work, aio_poll_put_work);
 			schedule_work(&req->work);
diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c
index a7b9b948e33d11..8f07c8d9bda0c4 100644
--- a/fs/anon_inodes.c
+++ b/fs/anon_inodes.c
@@ -46,7 +46,7 @@ static struct inode *anon_inode_inode __ro_after_init;
  * Rather than mess with our internal sane inode data, just fix it
  * up here in getattr() by masking off the format bits.
  */
-int anon_inode_getattr(struct mnt_idmap *idmap, const struct path *path,
+int anon_inode_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		       struct kstat *stat, u32 request_mask,
 		       unsigned int query_flags)
 {
@@ -57,7 +57,7 @@ int anon_inode_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return 0;
 }
 
-int anon_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int anon_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct iattr *attr)
 {
 	return -EOPNOTSUPP;
diff --git a/fs/attr.c b/fs/attr.c
index 71888ac903c27d..9706d370f34afb 100644
--- a/fs/attr.c
+++ b/fs/attr.c
@@ -30,7 +30,7 @@
  *
  * Return: ATTR_KILL_SGID if setgid bit needs to be removed, 0 otherwise.
  */
-int setattr_should_drop_sgid(struct mnt_idmap *idmap,
+int setattr_should_drop_sgid(const struct mnt_idmap *idmap,
 			     const struct inode *inode)
 {
 	umode_t mode = inode->i_mode;
@@ -60,7 +60,7 @@ EXPORT_SYMBOL(setattr_should_drop_sgid);
  * Return: A mask of ATTR_KILL_S{G,U}ID indicating which - if any - setid bits
  * to remove, 0 otherwise.
  */
-int setattr_should_drop_suidgid(struct mnt_idmap *idmap,
+int setattr_should_drop_suidgid(const struct mnt_idmap *idmap,
 				struct inode *inode)
 {
 	umode_t mode = inode->i_mode;
@@ -91,7 +91,7 @@ EXPORT_SYMBOL(setattr_should_drop_suidgid);
  * permissions. On non-idmapped mounts or if permission checking is to be
  * performed on the raw inode simply pass @nop_mnt_idmap.
  */
-static bool chown_ok(struct mnt_idmap *idmap,
+static bool chown_ok(const struct mnt_idmap *idmap,
 		     const struct inode *inode, vfsuid_t ia_vfsuid)
 {
 	vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
@@ -118,7 +118,7 @@ static bool chown_ok(struct mnt_idmap *idmap,
  * permissions. On non-idmapped mounts or if permission checking is to be
  * performed on the raw inode simply pass @nop_mnt_idmap.
  */
-static bool chgrp_ok(struct mnt_idmap *idmap,
+static bool chgrp_ok(const struct mnt_idmap *idmap,
 		     const struct inode *inode, vfsgid_t ia_vfsgid)
 {
 	vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode);
@@ -158,7 +158,7 @@ static bool chgrp_ok(struct mnt_idmap *idmap,
  * Should be called as the first thing in ->setattr implementations,
  * possibly after taking additional locks.
  */
-int setattr_prepare(struct mnt_idmap *idmap, struct dentry *dentry,
+int setattr_prepare(const struct mnt_idmap *idmap, struct dentry *dentry,
 		    struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -339,7 +339,7 @@ static void setattr_copy_mgtime(struct inode *inode, const struct iattr *attr)
  * that for "simple" filesystems, the struct inode is the inode storage.
  * The caller is free to mark the inode dirty afterwards if needed.
  */
-void setattr_copy(struct mnt_idmap *idmap, struct inode *inode,
+void setattr_copy(const struct mnt_idmap *idmap, struct inode *inode,
 		  const struct iattr *attr)
 {
 	unsigned int ia_valid = attr->ia_valid;
@@ -369,7 +369,7 @@ void setattr_copy(struct mnt_idmap *idmap, struct inode *inode,
 }
 EXPORT_SYMBOL(setattr_copy);
 
-int may_setattr(struct mnt_idmap *idmap, struct inode *inode,
+int may_setattr(const struct mnt_idmap *idmap, struct inode *inode,
 		unsigned int ia_valid)
 {
 	int error;
@@ -424,7 +424,7 @@ EXPORT_SYMBOL(may_setattr);
  * permissions. On non-idmapped mounts or if permission checking is to be
  * performed on the raw inode simply pass @nop_mnt_idmap.
  */
-int notify_change(struct mnt_idmap *idmap, struct dentry *dentry,
+int notify_change(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr, struct delegated_inode *delegated_inode)
 {
 	struct inode *inode = dentry->d_inode;
diff --git a/fs/autofs/inode.c b/fs/autofs/inode.c
index 6b15a3717ba7e4..066c16f2ea5695 100644
--- a/fs/autofs/inode.c
+++ b/fs/autofs/inode.c
@@ -51,6 +51,10 @@ void autofs_kill_sb(struct super_block *sb)
 	if (sbi) {
 		/* Free wait queues, close pipe */
 		autofs_catatonic_mode(sbi);
+		if (sbi->pipe) {
+			fput(sbi->pipe);
+			sbi->pipe = NULL;
+		}
 		put_pid(sbi->oz_pgrp);
 	}
 
diff --git a/fs/autofs/root.c b/fs/autofs/root.c
index b36439f4521ebe..28f38f5d023650 100644
--- a/fs/autofs/root.c
+++ b/fs/autofs/root.c
@@ -11,12 +11,12 @@
 
 #include "autofs_i.h"
 
-static int autofs_dir_permission(struct mnt_idmap *, struct inode *, int);
-static int autofs_dir_symlink(struct mnt_idmap *, struct inode *,
+static int autofs_dir_permission(const struct mnt_idmap *, struct inode *, int);
+static int autofs_dir_symlink(const struct mnt_idmap *, struct inode *,
 			      struct dentry *, const char *);
 static int autofs_dir_unlink(struct inode *, struct dentry *);
 static int autofs_dir_rmdir(struct inode *, struct dentry *);
-static struct dentry *autofs_dir_mkdir(struct mnt_idmap *, struct inode *,
+static struct dentry *autofs_dir_mkdir(const struct mnt_idmap *, struct inode *,
 				       struct dentry *, umode_t);
 static long autofs_root_ioctl(struct file *, unsigned int, unsigned long);
 #ifdef CONFIG_COMPAT
@@ -552,7 +552,7 @@ static struct dentry *autofs_lookup(struct inode *dir,
 	return NULL;
 }
 
-static int autofs_dir_permission(struct mnt_idmap *idmap,
+static int autofs_dir_permission(const struct mnt_idmap *idmap,
 				 struct inode *inode, int mask)
 {
 	if (mask & MAY_WRITE) {
@@ -572,7 +572,7 @@ static int autofs_dir_permission(struct mnt_idmap *idmap,
 	return generic_permission(idmap, inode, mask);
 }
 
-static int autofs_dir_symlink(struct mnt_idmap *idmap,
+static int autofs_dir_symlink(const struct mnt_idmap *idmap,
 			      struct inode *dir, struct dentry *dentry,
 			      const char *symname)
 {
@@ -724,7 +724,7 @@ static int autofs_dir_rmdir(struct inode *dir, struct dentry *dentry)
 	return 0;
 }
 
-static struct dentry *autofs_dir_mkdir(struct mnt_idmap *idmap,
+static struct dentry *autofs_dir_mkdir(const struct mnt_idmap *idmap,
 				       struct inode *dir, struct dentry *dentry,
 				       umode_t mode)
 {
diff --git a/fs/backing-file.c b/fs/backing-file.c
index cc101143f92129..5614cb7801e1b5 100644
--- a/fs/backing-file.c
+++ b/fs/backing-file.c
@@ -59,7 +59,7 @@ struct file *backing_tmpfile_open(const struct file *user_file, int flags,
 				  const struct path *real_parentpath,
 				  umode_t mode, const struct cred *cred)
 {
-	struct mnt_idmap *real_idmap = mnt_idmap(real_parentpath->mnt);
+	const struct mnt_idmap *real_idmap = mnt_idmap(real_parentpath->mnt);
 	const struct path *user_path = &user_file->f_path;
 	struct file *f;
 	int error;
diff --git a/fs/bad_inode.c b/fs/bad_inode.c
index 486c40f73e51e0..bea9f4876ee086 100644
--- a/fs/bad_inode.c
+++ b/fs/bad_inode.c
@@ -27,7 +27,7 @@ static const struct file_operations bad_file_ops =
 	.open		= bad_file_open,
 };
 
-static int bad_inode_create(struct mnt_idmap *idmap,
+static int bad_inode_create(const struct mnt_idmap *idmap,
 			    struct inode *dir, struct dentry *dentry,
 			    umode_t mode)
 {
@@ -51,14 +51,14 @@ static int bad_inode_unlink(struct inode *dir, struct dentry *dentry)
 	return -EIO;
 }
 
-static int bad_inode_symlink(struct mnt_idmap *idmap,
+static int bad_inode_symlink(const struct mnt_idmap *idmap,
 			     struct inode *dir, struct dentry *dentry,
 			     const char *symname)
 {
 	return -EIO;
 }
 
-static struct dentry *bad_inode_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *bad_inode_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				      struct dentry *dentry, umode_t mode)
 {
 	return ERR_PTR(-EIO);
@@ -69,13 +69,13 @@ static int bad_inode_rmdir (struct inode *dir, struct dentry *dentry)
 	return -EIO;
 }
 
-static int bad_inode_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int bad_inode_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 			   struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	return -EIO;
 }
 
-static int bad_inode_rename2(struct mnt_idmap *idmap,
+static int bad_inode_rename2(const struct mnt_idmap *idmap,
 			     struct inode *old_dir, struct dentry *old_dentry,
 			     struct inode *new_dir, struct dentry *new_dentry,
 			     unsigned int flags)
@@ -89,20 +89,20 @@ static int bad_inode_readlink(struct dentry *dentry, char __user *buffer,
 	return -EIO;
 }
 
-static int bad_inode_permission(struct mnt_idmap *idmap,
+static int bad_inode_permission(const struct mnt_idmap *idmap,
 				struct inode *inode, int mask)
 {
 	return -EIO;
 }
 
-static int bad_inode_getattr(struct mnt_idmap *idmap,
+static int bad_inode_getattr(const struct mnt_idmap *idmap,
 			     const struct path *path, struct kstat *stat,
 			     u32 request_mask, unsigned int query_flags)
 {
 	return -EIO;
 }
 
-static int bad_inode_setattr(struct mnt_idmap *idmap,
+static int bad_inode_setattr(const struct mnt_idmap *idmap,
 			     struct dentry *direntry, struct iattr *attrs)
 {
 	return -EIO;
@@ -146,14 +146,14 @@ static int bad_inode_atomic_open(struct inode *inode, struct dentry *dentry,
 	return -EIO;
 }
 
-static int bad_inode_tmpfile(struct mnt_idmap *idmap,
+static int bad_inode_tmpfile(const struct mnt_idmap *idmap,
 			     struct inode *inode, struct file *file,
 			     umode_t mode)
 {
 	return -EIO;
 }
 
-static int bad_inode_set_acl(struct mnt_idmap *idmap,
+static int bad_inode_set_acl(const struct mnt_idmap *idmap,
 			     struct dentry *dentry, struct posix_acl *acl,
 			     int type)
 {
diff --git a/fs/bfs/Kconfig b/fs/bfs/Kconfig
deleted file mode 100644
index 8e7ef866b62a62..00000000000000
--- a/fs/bfs/Kconfig
+++ /dev/null
@@ -1,21 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config BFS_FS
-	tristate "BFS file system support"
-	depends on BLOCK
-	select BUFFER_HEAD
-	help
-	  Boot File System (BFS) is a file system used under SCO UnixWare to
-	  allow the bootloader access to the kernel image and other important
-	  files during the boot process.  It is usually mounted under /stand
-	  and corresponds to the slice marked as "STAND" in the UnixWare
-	  partition.  You should say Y if you want to read or write the files
-	  on your /stand slice from within Linux.  You then also need to say Y
-	  to "UnixWare slices support", below.  More information about the BFS
-	  file system is contained in the file
-	  <file:Documentation/filesystems/bfs.rst>.
-
-	  If you don't know what this is about, say N.
-
-	  To compile this as a module, choose M here: the module will be called
-	  bfs.  Note that the file system of your root partition (the one
-	  containing the directory /) cannot be compiled as a module.
diff --git a/fs/bfs/Makefile b/fs/bfs/Makefile
deleted file mode 100644
index 2b6bc5eb4de9c5..00000000000000
--- a/fs/bfs/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Makefile for BFS filesystem.
-#
-
-obj-$(CONFIG_BFS_FS) += bfs.o
-
-bfs-objs := inode.o file.o dir.o
diff --git a/fs/bfs/bfs.h b/fs/bfs/bfs.h
deleted file mode 100644
index b08afe733e6326..00000000000000
--- a/fs/bfs/bfs.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- *	fs/bfs/bfs.h
- *	Copyright (C) 1999-2018 Tigran Aivazian <aivazian.tigran@gmail.com>
- */
-#ifndef _FS_BFS_BFS_H
-#define _FS_BFS_BFS_H
-
-#include <linux/bfs_fs.h>
-
-/* In theory BFS supports up to 512 inodes, numbered from 2 (for /) up to 513 inclusive.
-   In actual fact, attempting to create the 512th inode (i.e. inode No. 513 or file No. 511)
-   will fail with ENOSPC in bfs_add_entry(): the root directory cannot contain so many entries, counting '..'.
-   So, mkfs.bfs(8) should really limit its -N option to 511 and not 512. For now, we just print a warning
-   if a filesystem is mounted with such "impossible to fill up" number of inodes */
-#define BFS_MAX_LASTI	513
-
-/*
- * BFS file system in-core superblock info
- */
-struct bfs_sb_info {
-	unsigned long si_blocks;
-	unsigned long si_freeb;
-	unsigned long si_freei;
-	unsigned long si_lf_eblk;
-	unsigned long si_lasti;
-	DECLARE_BITMAP(si_imap, BFS_MAX_LASTI+1);
-	struct mutex bfs_lock;
-};
-
-/*
- * BFS file system in-core inode info
- */
-struct bfs_inode_info {
-	unsigned long i_dsk_ino; /* inode number from the disk, can be 0 */
-	unsigned long i_sblock;
-	unsigned long i_eblock;
-	struct mapping_metadata_bhs i_metadata_bhs;
-	struct inode vfs_inode;
-};
-
-static inline struct bfs_sb_info *BFS_SB(struct super_block *sb)
-{
-	return sb->s_fs_info;
-}
-
-static inline struct bfs_inode_info *BFS_I(struct inode *inode)
-{
-	return container_of(inode, struct bfs_inode_info, vfs_inode);
-}
-
-
-#define printf(format, args...) \
-	printk(KERN_ERR "BFS-fs: %s(): " format, __func__, ## args)
-
-/* inode.c */
-extern struct inode *bfs_iget(struct super_block *sb, unsigned long ino);
-extern void bfs_dump_imap(const char *, struct super_block *);
-
-/* file.c */
-extern const struct inode_operations bfs_file_inops;
-extern const struct file_operations bfs_file_operations;
-extern const struct address_space_operations bfs_aops;
-
-/* dir.c */
-extern const struct inode_operations bfs_dir_inops;
-extern const struct file_operations bfs_dir_operations;
-
-#endif /* _FS_BFS_BFS_H */
diff --git a/fs/bfs/dir.c b/fs/bfs/dir.c
index 91a4871fa05191..b944bd62f5d025 100644
--- a/fs/bfs/dir.c
+++ b/fs/bfs/dir.c
@@ -75,7 +75,7 @@ const struct file_operations bfs_dir_operations = {
 	.llseek		= generic_file_llseek,
 };
 
-static int bfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int bfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode)
 {
 	int err;
@@ -199,7 +199,7 @@ static int bfs_unlink(struct inode *dir, struct dentry *dentry)
 	return error;
 }
 
-static int bfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int bfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/bfs/file.c b/fs/bfs/file.c
deleted file mode 100644
index d33d6bde992b8b..00000000000000
--- a/fs/bfs/file.c
+++ /dev/null
@@ -1,203 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *	fs/bfs/file.c
- *	BFS file operations.
- *	Copyright (C) 1999-2018 Tigran Aivazian <aivazian.tigran@gmail.com>
- *
- *	Make the file block allocation algorithm understand the size
- *	of the underlying block device.
- *	Copyright (C) 2007 Dmitri Vorobiev <dmitri.vorobiev@gmail.com>
- *
- */
-
-#include <linux/fs.h>
-#include <linux/mpage.h>
-#include <linux/buffer_head.h>
-#include "bfs.h"
-
-#undef DEBUG
-
-#ifdef DEBUG
-#define dprintf(x...)	printf(x)
-#else
-#define dprintf(x...)
-#endif
-
-const struct file_operations bfs_file_operations = {
-	.llseek 	= generic_file_llseek,
-	.read_iter	= generic_file_read_iter,
-	.write_iter	= generic_file_write_iter,
-	.mmap_prepare	= generic_file_mmap_prepare,
-	.splice_read	= filemap_splice_read,
-};
-
-static int bfs_move_block(unsigned long from, unsigned long to,
-					struct super_block *sb)
-{
-	struct buffer_head *bh, *new;
-
-	bh = sb_bread(sb, from);
-	if (!bh)
-		return -EIO;
-	new = sb_getblk(sb, to);
-	memcpy(new->b_data, bh->b_data, bh->b_size);
-	mark_buffer_dirty(new);
-	bforget(bh);
-	brelse(new);
-	return 0;
-}
-
-static int bfs_move_blocks(struct super_block *sb, unsigned long start,
-				unsigned long end, unsigned long where)
-{
-	unsigned long i;
-
-	dprintf("%08lx-%08lx->%08lx\n", start, end, where);
-	for (i = start; i <= end; i++)
-		if(bfs_move_block(i, where + i, sb)) {
-			dprintf("failed to move block %08lx -> %08lx\n", i,
-								where + i);
-			return -EIO;
-		}
-	return 0;
-}
-
-static int bfs_get_block(struct inode *inode, sector_t block,
-			struct buffer_head *bh_result, int create)
-{
-	unsigned long phys;
-	int err;
-	struct super_block *sb = inode->i_sb;
-	struct bfs_sb_info *info = BFS_SB(sb);
-	struct bfs_inode_info *bi = BFS_I(inode);
-
-	phys = bi->i_sblock + block;
-	if (!create) {
-		if (phys <= bi->i_eblock) {
-			dprintf("c=%d, b=%08lx, phys=%09lx (granted)\n",
-                                create, (unsigned long)block, phys);
-			map_bh(bh_result, sb, phys);
-		}
-		return 0;
-	}
-
-	/*
-	 * If the file is not empty and the requested block is within the
-	 * range of blocks allocated for this file, we can grant it.
-	 */
-	if (bi->i_sblock && (phys <= bi->i_eblock)) {
-		dprintf("c=%d, b=%08lx, phys=%08lx (interim block granted)\n", 
-				create, (unsigned long)block, phys);
-		map_bh(bh_result, sb, phys);
-		return 0;
-	}
-
-	/* The file will be extended, so let's see if there is enough space. */
-	if (phys >= info->si_blocks)
-		return -ENOSPC;
-
-	/* The rest has to be protected against itself. */
-	mutex_lock(&info->bfs_lock);
-
-	/*
-	 * If the last data block for this file is the last allocated
-	 * block, we can extend the file trivially, without moving it
-	 * anywhere.
-	 */
-	if (bi->i_eblock == info->si_lf_eblk) {
-		dprintf("c=%d, b=%08lx, phys=%08lx (simple extension)\n", 
-				create, (unsigned long)block, phys);
-		map_bh(bh_result, sb, phys);
-		info->si_freeb -= phys - bi->i_eblock;
-		info->si_lf_eblk = bi->i_eblock = phys;
-		mark_inode_dirty(inode);
-		err = 0;
-		goto out;
-	}
-
-	/* Ok, we have to move this entire file to the next free block. */
-	phys = info->si_lf_eblk + 1;
-	if (phys + block >= info->si_blocks) {
-		err = -ENOSPC;
-		goto out;
-	}
-
-	if (bi->i_sblock) {
-		err = bfs_move_blocks(inode->i_sb, bi->i_sblock, 
-						bi->i_eblock, phys);
-		if (err) {
-			dprintf("failed to move ino=%08lx -> fs corruption\n",
-								inode->i_ino);
-			goto out;
-		}
-	} else
-		err = 0;
-
-	dprintf("c=%d, b=%08lx, phys=%08lx (moved)\n",
-                create, (unsigned long)block, phys);
-	bi->i_sblock = phys;
-	phys += block;
-	info->si_lf_eblk = bi->i_eblock = phys;
-
-	/*
-	 * This assumes nothing can write the inode back while we are here
-	 * and thus update inode->i_blocks! (XXX)
-	 */
-	info->si_freeb -= bi->i_eblock - bi->i_sblock + 1 - inode->i_blocks;
-	mark_inode_dirty(inode);
-	map_bh(bh_result, sb, phys);
-out:
-	mutex_unlock(&info->bfs_lock);
-	return err;
-}
-
-static int bfs_writepages(struct address_space *mapping,
-		struct writeback_control *wbc)
-{
-	return mpage_writepages(mapping, wbc, bfs_get_block);
-}
-
-static int bfs_read_folio(struct file *file, struct folio *folio)
-{
-	return block_read_full_folio(folio, bfs_get_block);
-}
-
-static void bfs_write_failed(struct address_space *mapping, loff_t to)
-{
-	struct inode *inode = mapping->host;
-
-	if (to > inode->i_size)
-		truncate_pagecache(inode, inode->i_size);
-}
-
-static int bfs_write_begin(const struct kiocb *iocb,
-			   struct address_space *mapping,
-			   loff_t pos, unsigned len,
-			   struct folio **foliop, void **fsdata)
-{
-	int ret;
-
-	ret = block_write_begin(mapping, pos, len, foliop, bfs_get_block);
-	if (unlikely(ret))
-		bfs_write_failed(mapping, pos + len);
-
-	return ret;
-}
-
-static sector_t bfs_bmap(struct address_space *mapping, sector_t block)
-{
-	return generic_block_bmap(mapping, block, bfs_get_block);
-}
-
-const struct address_space_operations bfs_aops = {
-	.dirty_folio	= block_dirty_folio,
-	.invalidate_folio = block_invalidate_folio,
-	.read_folio	= bfs_read_folio,
-	.writepages	= bfs_writepages,
-	.write_begin	= bfs_write_begin,
-	.write_end	= generic_write_end,
-	.migrate_folio	= buffer_migrate_folio,
-	.bmap		= bfs_bmap,
-};
-
-const struct inode_operations bfs_file_inops;
diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c
deleted file mode 100644
index 06e3a848b4efd8..00000000000000
--- a/fs/bfs/inode.c
+++ /dev/null
@@ -1,538 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *	fs/bfs/inode.c
- *	BFS superblock and inode operations.
- *	Copyright (C) 1999-2018 Tigran Aivazian <aivazian.tigran@gmail.com>
- *	From fs/minix, Copyright (C) 1991, 1992 Linus Torvalds.
- *	Made endianness-clean by Andrew Stribblehill <ads@wompom.org>, 2005.
- */
-
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/fs.h>
-#include <linux/buffer_head.h>
-#include <linux/vfs.h>
-#include <linux/writeback.h>
-#include <linux/uio.h>
-#include <linux/uaccess.h>
-#include <linux/fs_context.h>
-#include "bfs.h"
-
-MODULE_AUTHOR("Tigran Aivazian <aivazian.tigran@gmail.com>");
-MODULE_DESCRIPTION("SCO UnixWare BFS filesystem for Linux");
-MODULE_LICENSE("GPL");
-
-#undef DEBUG
-
-#ifdef DEBUG
-#define dprintf(x...)	printf(x)
-#else
-#define dprintf(x...)
-#endif
-
-struct inode *bfs_iget(struct super_block *sb, unsigned long ino)
-{
-	struct bfs_inode *di;
-	struct inode *inode;
-	struct buffer_head *bh;
-	int block, off;
-
-	inode = iget_locked(sb, ino);
-	if (!inode)
-		return ERR_PTR(-ENOMEM);
-	if (!(inode_state_read_once(inode) & I_NEW))
-		return inode;
-
-	if ((ino < BFS_ROOT_INO) || (ino > BFS_SB(inode->i_sb)->si_lasti)) {
-		printf("Bad inode number %s:%08lx\n", inode->i_sb->s_id, ino);
-		goto error;
-	}
-
-	block = (ino - BFS_ROOT_INO) / BFS_INODES_PER_BLOCK + 1;
-	bh = sb_bread(inode->i_sb, block);
-	if (!bh) {
-		printf("Unable to read inode %s:%08lx\n", inode->i_sb->s_id,
-									ino);
-		goto error;
-	}
-
-	off = (ino - BFS_ROOT_INO) % BFS_INODES_PER_BLOCK;
-	di = (struct bfs_inode *)bh->b_data + off;
-
-	/*
-	 * https://martin.hinner.info/fs/bfs/bfs-structure.html explains that
-	 * BFS in SCO UnixWare environment used only lower 9 bits of di->i_mode
-	 * value. This means that, although bfs_write_inode() saves whole
-	 * inode->i_mode bits (which include S_IFMT bits and S_IS{UID,GID,VTX}
-	 * bits), middle 7 bits of di->i_mode value can be garbage when these
-	 * bits were not saved by bfs_write_inode().
-	 * Since we can't tell whether middle 7 bits are garbage, use only
-	 * lower 12 bits (i.e. tolerate S_IS{UID,GID,VTX} bits possibly being
-	 * garbage) and reconstruct S_IFMT bits for Linux environment from
-	 * di->i_vtype value.
-	 */
-	inode->i_mode = 0x00000FFF & le32_to_cpu(di->i_mode);
-	if (le32_to_cpu(di->i_vtype) == BFS_VDIR) {
-		inode->i_mode |= S_IFDIR;
-		inode->i_op = &bfs_dir_inops;
-		inode->i_fop = &bfs_dir_operations;
-	} else if (le32_to_cpu(di->i_vtype) == BFS_VREG) {
-		inode->i_mode |= S_IFREG;
-		inode->i_op = &bfs_file_inops;
-		inode->i_fop = &bfs_file_operations;
-		inode->i_mapping->a_ops = &bfs_aops;
-	} else {
-		brelse(bh);
-		printf("Unknown vtype=%u %s:%08lx\n",
-		       le32_to_cpu(di->i_vtype), inode->i_sb->s_id, ino);
-		goto error;
-	}
-
-	BFS_I(inode)->i_sblock =  le32_to_cpu(di->i_sblock);
-	BFS_I(inode)->i_eblock =  le32_to_cpu(di->i_eblock);
-	BFS_I(inode)->i_dsk_ino = le16_to_cpu(di->i_ino);
-	i_uid_write(inode, le32_to_cpu(di->i_uid));
-	i_gid_write(inode,  le32_to_cpu(di->i_gid));
-	set_nlink(inode, le32_to_cpu(di->i_nlink));
-	inode->i_size = BFS_FILESIZE(di);
-	inode->i_blocks = BFS_FILEBLOCKS(di);
-	inode_set_atime(inode, le32_to_cpu(di->i_atime), 0);
-	inode_set_mtime(inode, le32_to_cpu(di->i_mtime), 0);
-	inode_set_ctime(inode, le32_to_cpu(di->i_ctime), 0);
-
-	brelse(bh);
-	unlock_new_inode(inode);
-	return inode;
-
-error:
-	iget_failed(inode);
-	return ERR_PTR(-EIO);
-}
-
-static struct bfs_inode *find_inode(struct super_block *sb, u16 ino, struct buffer_head **p)
-{
-	if ((ino < BFS_ROOT_INO) || (ino > BFS_SB(sb)->si_lasti)) {
-		printf("Bad inode number %s:%08x\n", sb->s_id, ino);
-		return ERR_PTR(-EIO);
-	}
-
-	ino -= BFS_ROOT_INO;
-
-	*p = sb_bread(sb, 1 + ino / BFS_INODES_PER_BLOCK);
-	if (!*p) {
-		printf("Unable to read inode %s:%08x\n", sb->s_id, ino);
-		return ERR_PTR(-EIO);
-	}
-
-	return (struct bfs_inode *)(*p)->b_data +  ino % BFS_INODES_PER_BLOCK;
-}
-
-static int bfs_write_inode(struct inode *inode, struct writeback_control *wbc)
-{
-	struct bfs_sb_info *info = BFS_SB(inode->i_sb);
-	unsigned int ino = (u16)inode->i_ino;
-	unsigned long i_sblock;
-	struct bfs_inode *di;
-	struct buffer_head *bh;
-
-	dprintf("ino=%08x\n", ino);
-
-	di = find_inode(inode->i_sb, ino, &bh);
-	if (IS_ERR(di))
-		return PTR_ERR(di);
-
-	mutex_lock(&info->bfs_lock);
-
-	if (ino == BFS_ROOT_INO)
-		di->i_vtype = cpu_to_le32(BFS_VDIR);
-	else
-		di->i_vtype = cpu_to_le32(BFS_VREG);
-
-	di->i_ino = cpu_to_le16(ino);
-	di->i_mode = cpu_to_le32(inode->i_mode);
-	di->i_uid = cpu_to_le32(i_uid_read(inode));
-	di->i_gid = cpu_to_le32(i_gid_read(inode));
-	di->i_nlink = cpu_to_le32(inode->i_nlink);
-	di->i_atime = cpu_to_le32(inode_get_atime_sec(inode));
-	di->i_mtime = cpu_to_le32(inode_get_mtime_sec(inode));
-	di->i_ctime = cpu_to_le32(inode_get_ctime_sec(inode));
-	i_sblock = BFS_I(inode)->i_sblock;
-	di->i_sblock = cpu_to_le32(i_sblock);
-	di->i_eblock = cpu_to_le32(BFS_I(inode)->i_eblock);
-	di->i_eoffset = cpu_to_le32(i_sblock * BFS_BSIZE + inode->i_size - 1);
-
-	mark_buffer_dirty(bh);
-	brelse(bh);
-	mutex_unlock(&info->bfs_lock);
-	set_inode_metadata_writeback(inode);
-	return 0;
-}
-
-static int bfs_sync_inode_metadata(struct inode *inode,
-				   struct writeback_control *wbc)
-{
-	int err = 0;
-	struct bfs_inode *di;
-	struct buffer_head *bh;
-
-	di = find_inode(inode->i_sb, (u16)inode->i_ino, &bh);
-	if (IS_ERR(di))
-		return PTR_ERR(di);
-
-	sync_dirty_buffer(bh);
-	if (buffer_write_io_error(bh)) {
-		err = -EIO;
-		goto out;
-	}
-	err = mmb_sync(&BFS_I(inode)->i_metadata_bhs);
-out:
-	brelse(bh);
-	return err;
-}
-
-static void bfs_evict_inode(struct inode *inode)
-{
-	unsigned long ino = inode->i_ino;
-	struct bfs_inode *di;
-	struct buffer_head *bh;
-	struct super_block *s = inode->i_sb;
-	struct bfs_sb_info *info = BFS_SB(s);
-	struct bfs_inode_info *bi = BFS_I(inode);
-
-	dprintf("ino=%08lx\n", ino);
-
-	truncate_inode_pages_final(&inode->i_data);
-	if (inode->i_nlink)
-		mmb_sync(&BFS_I(inode)->i_metadata_bhs);
-	mmb_invalidate(&BFS_I(inode)->i_metadata_bhs);
-	clear_inode(inode);
-
-	if (inode->i_nlink)
-		return;
-
-	di = find_inode(s, inode->i_ino, &bh);
-	if (IS_ERR(di))
-		return;
-
-	mutex_lock(&info->bfs_lock);
-	/* clear on-disk inode */
-	memset(di, 0, sizeof(struct bfs_inode));
-	mark_buffer_dirty(bh);
-	brelse(bh);
-
-	if (bi->i_dsk_ino) {
-		if (bi->i_sblock)
-			info->si_freeb += bi->i_eblock + 1 - bi->i_sblock;
-		info->si_freei++;
-		clear_bit(ino, info->si_imap);
-		bfs_dump_imap("evict_inode", s);
-	}
-
-	/*
-	 * If this was the last file, make the previous block
-	 * "last block of the last file" even if there is no
-	 * real file there, saves us 1 gap.
-	 */
-	if (info->si_lf_eblk == bi->i_eblock)
-		info->si_lf_eblk = bi->i_sblock - 1;
-	mutex_unlock(&info->bfs_lock);
-}
-
-static void bfs_put_super(struct super_block *s)
-{
-	struct bfs_sb_info *info = BFS_SB(s);
-
-	if (!info)
-		return;
-
-	mutex_destroy(&info->bfs_lock);
-	kfree(info);
-	s->s_fs_info = NULL;
-}
-
-static int bfs_statfs(struct dentry *dentry, struct kstatfs *buf)
-{
-	struct super_block *s = dentry->d_sb;
-	struct bfs_sb_info *info = BFS_SB(s);
-	u64 id = huge_encode_dev(s->s_bdev->bd_dev);
-	buf->f_type = BFS_MAGIC;
-	buf->f_bsize = s->s_blocksize;
-	buf->f_blocks = info->si_blocks;
-	buf->f_bfree = buf->f_bavail = info->si_freeb;
-	buf->f_files = info->si_lasti + 1 - BFS_ROOT_INO;
-	buf->f_ffree = info->si_freei;
-	buf->f_fsid = u64_to_fsid(id);
-	buf->f_namelen = BFS_NAMELEN;
-	return 0;
-}
-
-static struct kmem_cache *bfs_inode_cachep;
-
-static struct inode *bfs_alloc_inode(struct super_block *sb)
-{
-	struct bfs_inode_info *bi;
-	bi = alloc_inode_sb(sb, bfs_inode_cachep, GFP_KERNEL);
-	if (!bi)
-		return NULL;
-	mmb_init(&bi->i_metadata_bhs, &bi->vfs_inode.i_data);
-
-	return &bi->vfs_inode;
-}
-
-static void bfs_free_inode(struct inode *inode)
-{
-	kmem_cache_free(bfs_inode_cachep, BFS_I(inode));
-}
-
-static void init_once(void *foo)
-{
-	struct bfs_inode_info *bi = foo;
-
-	inode_init_once(&bi->vfs_inode);
-}
-
-static int __init init_inodecache(void)
-{
-	bfs_inode_cachep = kmem_cache_create("bfs_inode_cache",
-					     sizeof(struct bfs_inode_info),
-					     0, (SLAB_RECLAIM_ACCOUNT|
-						SLAB_ACCOUNT),
-					     init_once);
-	if (bfs_inode_cachep == NULL)
-		return -ENOMEM;
-	return 0;
-}
-
-static void destroy_inodecache(void)
-{
-	/*
-	 * Make sure all delayed rcu free inodes are flushed before we
-	 * destroy cache.
-	 */
-	rcu_barrier();
-	kmem_cache_destroy(bfs_inode_cachep);
-}
-
-static const struct super_operations bfs_sops = {
-	.alloc_inode	= bfs_alloc_inode,
-	.free_inode	= bfs_free_inode,
-	.write_inode	= bfs_write_inode,
-	.sync_inode_metadata = bfs_sync_inode_metadata,
-	.evict_inode	= bfs_evict_inode,
-	.put_super	= bfs_put_super,
-	.statfs		= bfs_statfs,
-};
-
-void bfs_dump_imap(const char *prefix, struct super_block *s)
-{
-#ifdef DEBUG
-	int i;
-	char *tmpbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
-
-	if (!tmpbuf)
-		return;
-	for (i = BFS_SB(s)->si_lasti; i >= 0; i--) {
-		if (i > PAGE_SIZE - 100) break;
-		if (test_bit(i, BFS_SB(s)->si_imap))
-			strcat(tmpbuf, "1");
-		else
-			strcat(tmpbuf, "0");
-	}
-	printf("%s: lasti=%08lx <%s>\n", prefix, BFS_SB(s)->si_lasti, tmpbuf);
-	kfree(tmpbuf);
-#endif
-}
-
-static int bfs_fill_super(struct super_block *s, struct fs_context *fc)
-{
-	struct buffer_head *bh, *sbh;
-	struct bfs_super_block *bfs_sb;
-	struct inode *inode;
-	unsigned i;
-	struct bfs_sb_info *info;
-	int ret = -EINVAL;
-	unsigned long i_sblock, i_eblock, i_eoff, s_size;
-	int silent = fc->sb_flags & SB_SILENT;
-
-	info = kzalloc_obj(*info);
-	if (!info)
-		return -ENOMEM;
-	mutex_init(&info->bfs_lock);
-	s->s_fs_info = info;
-	s->s_time_min = 0;
-	s->s_time_max = U32_MAX;
-
-	if (!sb_set_blocksize(s, BFS_BSIZE))
-		goto out;
-
-	sbh = sb_bread(s, 0);
-	if (!sbh)
-		goto out;
-	bfs_sb = (struct bfs_super_block *)sbh->b_data;
-	if (le32_to_cpu(bfs_sb->s_magic) != BFS_MAGIC) {
-		if (!silent)
-			printf("No BFS filesystem on %s (magic=%08x)\n", s->s_id,  le32_to_cpu(bfs_sb->s_magic));
-		goto out1;
-	}
-	if (BFS_UNCLEAN(bfs_sb, s) && !silent)
-		printf("%s is unclean, continuing\n", s->s_id);
-
-	s->s_magic = BFS_MAGIC;
-
-	if (le32_to_cpu(bfs_sb->s_start) > le32_to_cpu(bfs_sb->s_end) ||
-	    le32_to_cpu(bfs_sb->s_start) < sizeof(struct bfs_super_block) + sizeof(struct bfs_dirent)) {
-		printf("Superblock is corrupted on %s\n", s->s_id);
-		goto out1;
-	}
-
-	info->si_lasti = (le32_to_cpu(bfs_sb->s_start) - BFS_BSIZE) / sizeof(struct bfs_inode) + BFS_ROOT_INO - 1;
-	if (info->si_lasti == BFS_MAX_LASTI)
-		printf("NOTE: filesystem %s was created with 512 inodes, the real maximum is 511, mounting anyway\n", s->s_id);
-	else if (info->si_lasti > BFS_MAX_LASTI) {
-		printf("Impossible last inode number %lu > %d on %s\n", info->si_lasti, BFS_MAX_LASTI, s->s_id);
-		goto out1;
-	}
-	for (i = 0; i < BFS_ROOT_INO; i++)
-		set_bit(i, info->si_imap);
-
-	s->s_op = &bfs_sops;
-	inode = bfs_iget(s, BFS_ROOT_INO);
-	if (IS_ERR(inode)) {
-		ret = PTR_ERR(inode);
-		goto out1;
-	}
-	s->s_root = d_make_root(inode);
-	if (!s->s_root) {
-		ret = -ENOMEM;
-		goto out1;
-	}
-
-	info->si_blocks = (le32_to_cpu(bfs_sb->s_end) + 1) >> BFS_BSIZE_BITS;
-	info->si_freeb = (le32_to_cpu(bfs_sb->s_end) + 1 - le32_to_cpu(bfs_sb->s_start)) >> BFS_BSIZE_BITS;
-	info->si_freei = 0;
-	info->si_lf_eblk = 0;
-
-	/* can we read the last block? */
-	bh = sb_bread(s, info->si_blocks - 1);
-	if (!bh) {
-		printf("Last block not available on %s: %lu\n", s->s_id, info->si_blocks - 1);
-		ret = -EIO;
-		goto out2;
-	}
-	brelse(bh);
-
-	bh = NULL;
-	for (i = BFS_ROOT_INO; i <= info->si_lasti; i++) {
-		struct bfs_inode *di;
-		int block = (i - BFS_ROOT_INO) / BFS_INODES_PER_BLOCK + 1;
-		int off = (i - BFS_ROOT_INO) % BFS_INODES_PER_BLOCK;
-		unsigned long eblock;
-
-		if (!off) {
-			brelse(bh);
-			bh = sb_bread(s, block);
-		}
-
-		if (!bh)
-			continue;
-
-		di = (struct bfs_inode *)bh->b_data + off;
-
-		/* test if filesystem is not corrupted */
-
-		i_eoff = le32_to_cpu(di->i_eoffset);
-		i_sblock = le32_to_cpu(di->i_sblock);
-		i_eblock = le32_to_cpu(di->i_eblock);
-		s_size = le32_to_cpu(bfs_sb->s_end);
-
-		if (i_sblock > info->si_blocks ||
-			i_eblock > info->si_blocks ||
-			i_sblock > i_eblock ||
-			(i_eoff != le32_to_cpu(-1) && i_eoff > s_size) ||
-			i_sblock * BFS_BSIZE > i_eoff) {
-
-			printf("Inode 0x%08x corrupted on %s\n", i, s->s_id);
-
-			brelse(bh);
-			ret = -EIO;
-			goto out2;
-		}
-
-		if (!di->i_ino) {
-			info->si_freei++;
-			continue;
-		}
-		set_bit(i, info->si_imap);
-		info->si_freeb -= BFS_FILEBLOCKS(di);
-
-		eblock =  le32_to_cpu(di->i_eblock);
-		if (eblock > info->si_lf_eblk)
-			info->si_lf_eblk = eblock;
-	}
-	brelse(bh);
-	brelse(sbh);
-	bfs_dump_imap("fill_super", s);
-	return 0;
-
-out2:
-	dput(s->s_root);
-	s->s_root = NULL;
-out1:
-	brelse(sbh);
-out:
-	mutex_destroy(&info->bfs_lock);
-	kfree(info);
-	s->s_fs_info = NULL;
-	return ret;
-}
-
-static int bfs_get_tree(struct fs_context *fc)
-{
-	return get_tree_bdev(fc, bfs_fill_super);
-}
-
-static const struct fs_context_operations bfs_context_ops = {
-	.get_tree = bfs_get_tree,
-};
-
-static int bfs_init_fs_context(struct fs_context *fc)
-{
-	fc->ops = &bfs_context_ops;
-
-	return 0;
-}
-
-static struct file_system_type bfs_fs_type = {
-	.owner			= THIS_MODULE,
-	.name			= "bfs",
-	.init_fs_context	= bfs_init_fs_context,
-	.kill_sb		= kill_block_super,
-	.fs_flags		= FS_REQUIRES_DEV,
-};
-MODULE_ALIAS_FS("bfs");
-
-static int __init init_bfs_fs(void)
-{
-	int err = init_inodecache();
-	if (err)
-		goto out1;
-	err = register_filesystem(&bfs_fs_type);
-	if (err)
-		goto out;
-	return 0;
-out:
-	destroy_inodecache();
-out1:
-	return err;
-}
-
-static void __exit exit_bfs_fs(void)
-{
-	unregister_filesystem(&bfs_fs_type);
-	destroy_inodecache();
-}
-
-module_init(init_bfs_fs)
-module_exit(exit_bfs_fs)
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index 06d0df1053823e..bf7f8f47548d30 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -74,7 +74,7 @@ static int load_elf_binary(struct linux_binprm *bprm);
  * don't even try.
  */
 #ifdef CONFIG_ELF_CORE
-static int elf_core_dump(struct coredump_params *cprm);
+static bool elf_core_dump(struct coredump_params *cprm);
 #else
 #define elf_core_dump	NULL
 #endif
@@ -1875,7 +1875,7 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
 		return 0;
 
 	info->thread->task = dump_task;
-	for (ct = dump_task->signal->core_state->dumper.next; ct; ct = ct->next) {
+	for (ct = dump_task->signal->core_state->tasks; ct; ct = ct->next) {
 		t = kzalloc_flex(*t, notes, info->thread_notes);
 		if (unlikely(!t))
 			return 0;
@@ -1987,9 +1987,9 @@ static void fill_extnum_info(struct elfhdr *elf, struct elf_shdr *shdr4extnum,
  * and then they are actually written out.  If we run out of core limit
  * we just truncate.
  */
-static int elf_core_dump(struct coredump_params *cprm)
+static bool elf_core_dump(struct coredump_params *cprm)
 {
-	int has_dumped = 0;
+	bool ret = false;
 	int segs, i;
 	struct elfhdr elf;
 	loff_t offset = 0, dataoff;
@@ -2020,7 +2020,7 @@ static int elf_core_dump(struct coredump_params *cprm)
 	if (!fill_note_info(&elf, e_phnum, &info, cprm))
 		goto end_coredump;
 
-	has_dumped = 1;
+	cprm->state |= COREDUMP_STATE_STARTED;
 
 	offset += sizeof(elf);				/* ELF header */
 	offset += segs * sizeof(struct elf_phdr);	/* Program headers */
@@ -2029,7 +2029,7 @@ static int elf_core_dump(struct coredump_params *cprm)
 	{
 		size_t sz = info.size;
 
-		/* For cell spufs and x86 xstate */
+		/* For x86 xstate */
 		sz += elf_coredump_extra_notes_size();
 
 		phdr4note = kmalloc_obj(*phdr4note);
@@ -2093,7 +2093,7 @@ static int elf_core_dump(struct coredump_params *cprm)
 	if (!write_note_info(&info, cprm))
 		goto end_coredump;
 
-	/* For cell spufs and x86 xstate */
+	/* For x86 xstate */
 	if (elf_coredump_extra_notes_write(cprm))
 		goto end_coredump;
 
@@ -2115,11 +2115,13 @@ static int elf_core_dump(struct coredump_params *cprm)
 			goto end_coredump;
 	}
 
+	ret = true;
+
 end_coredump:
 	free_note_info(&info);
 	kfree(shdr4extnum);
 	kfree(phdr4note);
-	return has_dumped;
+	return ret;
 }
 
 #endif		/* CONFIG_ELF_CORE */
diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
index 068c46875c7442..d3872169f55e5b 100644
--- a/fs/binfmt_elf_fdpic.c
+++ b/fs/binfmt_elf_fdpic.c
@@ -75,7 +75,7 @@ static int elf_fdpic_map_file_by_direct_mmap(struct elf_fdpic_params *,
 					     struct file *, struct mm_struct *);
 
 #ifdef CONFIG_ELF_CORE
-static int elf_fdpic_core_dump(struct coredump_params *cprm);
+static bool elf_fdpic_core_dump(struct coredump_params *cprm);
 #endif
 
 static struct linux_binfmt elf_fdpic_format = {
@@ -1477,9 +1477,9 @@ static bool elf_fdpic_dump_segments(struct coredump_params *cprm,
  * and then they are actually written out.  If we run out of core limit
  * we just truncate.
  */
-static int elf_fdpic_core_dump(struct coredump_params *cprm)
+static bool elf_fdpic_core_dump(struct coredump_params *cprm)
 {
-	int has_dumped = 0;
+	bool ret = false;
 	int segs;
 	int i;
 	struct elfhdr *elf = NULL;
@@ -1504,7 +1504,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
 	if (!psinfo)
 		goto end_coredump;
 
-	for (ct = current->signal->core_state->dumper.next;
+	for (ct = current->signal->core_state->tasks;
 					ct; ct = ct->next) {
 		tmp = elf_dump_thread_status(cprm->siginfo->si_signo,
 					     ct->task, &thread_status_size);
@@ -1536,7 +1536,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
 	/* Set up header */
 	fill_elf_fdpic_header(elf, e_phnum);
 
-	has_dumped = 1;
+	cprm->state |= COREDUMP_STATE_STARTED;
 	/*
 	 * Set up the notes in similar form to SVR4 core dumps made
 	 * with info from their /proc.
@@ -1656,6 +1656,8 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
 		       cprm->file->f_pos, offset);
 	}
 
+	ret = true;
+
 end_coredump:
 	while (thread_list) {
 		tmp = thread_list;
@@ -1666,7 +1668,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
 	kfree(elf);
 	kfree(psinfo);
 	kfree(shdr4extnum);
-	return has_dumped;
+	return ret;
 }
 
 #endif		/* CONFIG_ELF_CORE */
diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c
index 620da85948b4d5..d945b4f6e15813 100644
--- a/fs/binfmt_misc.c
+++ b/fs/binfmt_misc.c
@@ -100,6 +100,13 @@ static const struct binfmt_misc_flag *misc_flag_by_char(const char c)
 	return NULL;
 }
 
+static bool misc_valid_delim(const char c)
+{
+	if (!isascii(c) || !ispunct(c))
+		return false;
+	return c != '\\';
+}
+
 struct binfmt_misc_entry {
 	struct hlist_node node;
 	unsigned long flags;		/* type, status, etc. */
@@ -871,10 +878,9 @@ static struct binfmt_misc_entry *create_entry(const char __user *buffer,
 
 	del = *p++;	/* delimiter */
 
-	pr_debug("register: delim: %#x {%c}\n", del, del);
+	pr_debug("register: delim: %#x\n", del);
 
-	/* A flag-char delimiter runs the flag scan off the buffer. */
-	if (misc_flag_by_char(del))
+	if (!misc_valid_delim(del))
 		return ERR_PTR(-EINVAL);
 
 	/* Pad the buffer with the delim to simplify parsing below. */
diff --git a/fs/binfmt_misc_bpf.c b/fs/binfmt_misc_bpf.c
index 91576ff05911d0..a3e26e8a402702 100644
--- a/fs/binfmt_misc_bpf.c
+++ b/fs/binfmt_misc_bpf.c
@@ -141,8 +141,6 @@ __bpf_kfunc int bpf_binprm_set_interp(struct linux_binprm *bprm,
 	len = strnlen(path, path__sz);
 	if (len == path__sz)
 		return -EINVAL;
-	if (path[0] != '/')
-		return -EINVAL;
 	if (len >= PATH_MAX)
 		return -ENAMETOOLONG;
 
@@ -150,6 +148,15 @@ __bpf_kfunc int bpf_binprm_set_interp(struct linux_binprm *bprm,
 	if (!interp)
 		return -ENOMEM;
 
+	/*
+	 * The program may pass memory that is written to while this runs,
+	 * so check the private copy and not the buffer it was made from.
+	 */
+	if (interp[0] != '/') {
+		kfree(interp);
+		return -EINVAL;
+	}
+
 	bm_bpf_stage_selection(bprm, interp, NULL);
 	return 0;
 }
@@ -176,6 +183,7 @@ __bpf_kfunc int bpf_binprm_select_interp(struct linux_binprm *bprm,
 					 const char *name, size_t name__sz)
 {
 	const struct binfmt_misc_interp *interp;
+	char buf[BINFMT_MISC_INTERP_NAME_MAX + 1];
 	size_t len;
 	char *path;
 
@@ -184,8 +192,20 @@ __bpf_kfunc int bpf_binprm_select_interp(struct linux_binprm *bprm,
 	len = strnlen(name, name__sz);
 	if (len == name__sz || !len)
 		return -EINVAL;
+	/* No entry binds a longer name, so it cannot be found. */
+	if (len > BINFMT_MISC_INTERP_NAME_MAX)
+		return -ENOENT;
+
+	/*
+	 * The program may pass memory that is written to while this runs,
+	 * so look the name up in a private copy and check that instead.
+	 */
+	memcpy(buf, name, len);
+	buf[len] = '\0';
+	if (!buf[0])
+		return -EINVAL;
 
-	interp = binfmt_misc_find_interp(bprm->bpf_interps, name);
+	interp = binfmt_misc_find_interp(bprm->bpf_interps, buf);
 	if (!interp)
 		return -ENOENT;
 
@@ -228,6 +248,15 @@ __bpf_kfunc int bpf_binprm_set_interp_arg(struct linux_binprm *bprm,
 	if (!val)
 		return -ENOMEM;
 
+	/*
+	 * The program may pass memory that is written to while this runs,
+	 * so check the private copy and not the buffer it was made from.
+	 */
+	if (!val[0]) {
+		kfree(val);
+		return -EINVAL;
+	}
+
 	kfree(bprm->bpf_interp_arg);
 	bprm->bpf_interp_arg = val;
 	return 0;
diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c
index 6cb8772679782d..35ccd98a4de642 100644
--- a/fs/bpf_fs_kfuncs.c
+++ b/fs/bpf_fs_kfuncs.c
@@ -11,6 +11,7 @@
 #include <linux/fsnotify.h>
 #include <linux/file.h>
 #include <linux/kernfs.h>
+#include <linux/lsm_hooks.h>
 #include <linux/mm.h>
 #include <linux/net.h>
 #include <linux/xattr.h>
@@ -237,7 +238,7 @@ int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
  * @dentry: dentry to get xattr from
  * @name__str: name of the xattr
  *
- * Rmove xattr *name__str* of *dentry*.
+ * Remove xattr *name__str* of *dentry*.
  *
  * For security reasons, only *name__str* with prefix "security.bpf."
  * is allowed.
@@ -305,7 +306,7 @@ __bpf_kfunc int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__st
  * @dentry: dentry to get xattr from
  * @name__str: name of the xattr
  *
- * Rmove xattr *name__str* of *dentry*.
+ * Remove xattr *name__str* of *dentry*.
  *
  * For security reasons, only *name__str* with prefix "security.bpf."
  * is allowed.
@@ -328,6 +329,89 @@ __bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name_
 	return ret;
 }
 
+static int bpf_inode_init_xattrs_claimed(const struct xattr *xattrs, int xattr_count)
+{
+	const size_t suffix_len = sizeof(XATTR_BPF_LSM_SUFFIX) - 1;
+	int i, claimed = 0;
+
+	for (i = 0; i < xattr_count; i++) {
+		const char *name = xattrs[i].name;
+
+		if (name && !strncmp(name, XATTR_BPF_LSM_SUFFIX, suffix_len))
+			claimed++;
+	}
+	return claimed;
+}
+
+/**
+ * bpf_inode_init_xattr - attach a xattr to an inode that is being created
+ * @xattrs: xattr array the inode_init_security hook was handed
+ * @xattr_count__ctx_out: xattr count the inode_init_security hook was handed
+ * @name__str: name of the xattr
+ * @value_p: xattr value
+ *
+ * Claim one of the slots the BPF LSM reserved in the xattr array, so that
+ * the xattr is written out as part of the transaction creating the inode.
+ *
+ * For security reasons, only *name__str* with prefix "security.bpf." is
+ * allowed. The slot stores the name without that "security." prefix, which
+ * the filesystem's initxattrs() callback puts back.
+ *
+ * At most BPF_LSM_INODE_INIT_XATTRS xattrs can be attached to one inode,
+ * matching the number of slots the BPF LSM reserves.
+ *
+ * Return: 0 on success, a negative value on error.
+ */
+__bpf_kfunc int bpf_inode_init_xattr(struct xattr *xattrs,
+				     int *xattr_count__ctx_out,
+				     const char *name__str,
+				     const struct bpf_dynptr *value_p)
+{
+	const struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p;
+	int *xattr_count = xattr_count__ctx_out;
+	const char *suffix;
+	struct xattr *slot;
+	const void *value;
+	size_t name_len;
+	u32 value_len;
+	char *buf;
+
+	if (!match_security_bpf_prefix(name__str))
+		return -EPERM;
+	if (bpf_inode_init_xattrs_claimed(xattrs, *xattr_count) >=
+	    BPF_LSM_INODE_INIT_XATTRS)
+		return -ENOSPC;
+
+	suffix = name__str + XATTR_SECURITY_PREFIX_LEN;
+	name_len = strlen(suffix);
+	if (name_len <= sizeof(XATTR_BPF_LSM_SUFFIX) - 1 ||
+	    name_len > XATTR_NAME_MAX - XATTR_SECURITY_PREFIX_LEN)
+		return -EINVAL;
+
+	value_len = __bpf_dynptr_size(value_ptr);
+	value = __bpf_dynptr_data(value_ptr, value_len);
+	if (!value)
+		return -EINVAL;
+	if (value_len > XATTR_SIZE_MAX)
+		return -E2BIG;
+
+	buf = kmalloc(value_len + name_len + 1, GFP_NOWAIT | __GFP_NOWARN);
+	if (!buf)
+		return -ENOMEM;
+	memcpy(buf, value, value_len);
+	memcpy(buf + value_len, suffix, name_len + 1);
+
+	slot = lsm_get_xattr_slot(xattrs, xattr_count);
+	if (!slot) {
+		kfree(buf);
+		return -ENOSPC;
+	}
+	slot->value = buf;
+	slot->value_len = value_len;
+	slot->name = buf + value_len;
+	return 0;
+}
+
 #ifdef CONFIG_CGROUPS
 /**
  * bpf_cgroup_read_xattr - read xattr of a cgroup's node in cgroupfs
@@ -425,6 +509,7 @@ BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_real_data_inode, KF_SLEEPABLE | KF_RET_NULL)
+BTF_ID_FLAGS(func, bpf_inode_init_xattr)
 #ifdef CONFIG_NET
 BTF_ID_FLAGS(func, bpf_sock_read_xattr, KF_RCU)
 #endif
@@ -436,10 +521,24 @@ BTF_ID(func, bpf_set_dentry_xattr)
 BTF_ID(func, bpf_remove_dentry_xattr)
 BTF_SET_END(bpf_fs_kfunc_lsm_only_ids)
 
+BTF_ID_LIST_SINGLE(bpf_inode_init_xattr_ids, func, bpf_inode_init_xattr)
+
+BTF_SET_START(bpf_inode_init_xattr_hooks)
+BTF_ID(func, bpf_lsm_inode_init_security)
+BTF_SET_END(bpf_inode_init_xattr_hooks)
+
 static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
 {
 	if (!btf_id_set8_contains(&bpf_fs_kfunc_set_ids, kfunc_id))
 		return 0;
+	if (kfunc_id == bpf_inode_init_xattr_ids[0]) {
+		if (prog->type != BPF_PROG_TYPE_LSM ||
+		    prog->expected_attach_type != BPF_LSM_MAC ||
+		    !btf_id_set_contains(&bpf_inode_init_xattr_hooks,
+					 prog->aux->attach_btf_id))
+			return -EACCES;
+		return 0;
+	}
 	if (prog->type == BPF_PROG_TYPE_LSM)
 		return 0;
 	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
@@ -472,10 +571,6 @@ BTF_ID(func, bpf_lsm_inode_rmdir)
 BTF_ID(func, bpf_lsm_inode_setattr)
 BTF_ID(func, bpf_lsm_inode_setxattr)
 BTF_ID(func, bpf_lsm_inode_unlink)
-#ifdef CONFIG_SECURITY_PATH
-BTF_ID(func, bpf_lsm_path_unlink)
-BTF_ID(func, bpf_lsm_path_rmdir)
-#endif /* CONFIG_SECURITY_PATH */
 BTF_SET_END(d_inode_locked_hooks)
 
 bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
index 4b10d78ed99b16..0b8d8905e38e25 100644
--- a/fs/btrfs/Kconfig
+++ b/fs/btrfs/Kconfig
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
+# misc-next marker
 
 config BTRFS_FS
 	tristate "Btrfs filesystem support"
diff --git a/fs/btrfs/acl.c b/fs/btrfs/acl.c
index 662cdd1cbdef6e..10a0d733bfd174 100644
--- a/fs/btrfs/acl.c
+++ b/fs/btrfs/acl.c
@@ -101,7 +101,7 @@ int __btrfs_set_acl(struct btrfs_trans_handle *trans, struct inode *inode,
 	return 0;
 }
 
-int btrfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int btrfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct posix_acl *acl, int type)
 {
 	int ret;
diff --git a/fs/btrfs/acl.h b/fs/btrfs/acl.h
index 0458cd51ed4892..6eae2db3654d1a 100644
--- a/fs/btrfs/acl.h
+++ b/fs/btrfs/acl.h
@@ -15,7 +15,7 @@ struct mnt_idmap;
 struct dentry;
 
 struct posix_acl *btrfs_get_acl(struct inode *inode, int type, bool rcu);
-int btrfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int btrfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct posix_acl *acl, int type);
 int __btrfs_set_acl(struct btrfs_trans_handle *trans, struct inode *inode,
 		    struct posix_acl *acl, int type);
diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c
index cc0bd03048bae6..2c6234f6182f89 100644
--- a/fs/btrfs/bio.c
+++ b/fs/btrfs/bio.c
@@ -27,15 +27,9 @@ struct btrfs_failed_bio {
 	atomic_t repair_count;
 };
 
-/* Is this a data path I/O that needs storage layer checksum and repair? */
-static inline bool is_data_bbio(const struct btrfs_bio *bbio)
-{
-	return bbio->inode && is_data_inode(bbio->inode);
-}
-
 static bool bbio_has_ordered_extent(const struct btrfs_bio *bbio)
 {
-	return is_data_bbio(bbio) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE;
+	return is_data_inode(bbio->inode) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE;
 }
 
 /*
@@ -103,7 +97,6 @@ static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info,
 	bbio->can_use_append = orig_bbio->can_use_append;
 	bbio->is_scrub = orig_bbio->is_scrub;
 	bbio->is_remap = orig_bbio->is_remap;
-	bbio->async_csum = orig_bbio->async_csum;
 
 	atomic_inc(&orig_bbio->pending_ios);
 	return bbio;
@@ -114,9 +107,6 @@ void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status)
 	/* Make sure we're already in task context. */
 	ASSERT(in_task());
 
-	if (bbio->async_csum)
-		wait_for_completion(&bbio->csum_done);
-
 	bbio->bio.bi_status = status;
 	if (bbio->bio.bi_pool == &btrfs_clone_bioset) {
 		struct btrfs_bio *orig_bbio = bbio->private;
@@ -180,30 +170,13 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio,
 	struct btrfs_failed_bio *fbio = repair_bbio->private;
 	struct btrfs_inode *inode = repair_bbio->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	/*
-	 * We can not move forward the saved_iter, as it will be later
-	 * utilized by repair_bbio again.
-	 */
-	struct bvec_iter saved_iter = repair_bbio->saved_iter;
-	const u32 step = min(fs_info->sectorsize, PAGE_SIZE);
-	const u64 logical = repair_bbio->saved_iter.bi_sector << SECTOR_SHIFT;
-	const u32 nr_steps = repair_bbio->saved_iter.bi_size / step;
 	int mirror = repair_bbio->mirror_num;
-	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
-	phys_addr_t paddr;
-	unsigned int slot = 0;
 
-	/* Repair bbio should be eaxctly one block sized. */
+	/* Repair bbio should be exactly one block sized. */
 	ASSERT(repair_bbio->saved_iter.bi_size == fs_info->sectorsize);
 
-	btrfs_bio_for_each_block(paddr, &repair_bbio->bio, &saved_iter, step) {
-		ASSERT(slot < nr_steps);
-		paddrs[slot] = paddr;
-		slot++;
-	}
-
 	if (repair_bbio->bio.bi_status ||
-	    !btrfs_data_csum_ok(repair_bbio, dev, 0, paddrs)) {
+	    !btrfs_bio_data_csum_ok(repair_bbio, &repair_bbio->saved_iter, dev)) {
 		bio_reset(&repair_bbio->bio, NULL, REQ_OP_READ);
 		repair_bbio->bio.bi_iter = repair_bbio->saved_iter;
 
@@ -220,9 +193,8 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio,
 
 	do {
 		mirror = prev_repair_mirror(fbio, mirror);
-		btrfs_repair_io_failure(fs_info, btrfs_ino(inode),
-				  repair_bbio->file_offset, fs_info->sectorsize,
-				  logical, paddrs, step, mirror);
+		btrfs_repair_bbio_failure(repair_bbio, &repair_bbio->saved_iter,
+					  fs_info->sectorsize, mirror);
 	} while (mirror != fbio->bbio->mirror_num);
 
 done:
@@ -238,25 +210,21 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio,
  * read succeeded to restore the redundancy.
  */
 static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio,
-						  u32 bio_offset,
-						  phys_addr_t paddrs[],
+						  const struct bvec_iter *orig_iter,
 						  struct btrfs_failed_bio *fbio)
 {
 	struct btrfs_inode *inode = failed_bbio->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	const u32 sectorsize = fs_info->sectorsize;
-	const u32 step = min(fs_info->sectorsize, PAGE_SIZE);
-	const u32 nr_steps = sectorsize / step;
-	/*
-	 * For bs > ps cases, the saved_iter can be partially moved forward.
-	 * In that case we should round it down to the block boundary.
-	 */
-	const u64 logical = round_down(failed_bbio->saved_iter.bi_sector << SECTOR_SHIFT,
-				       sectorsize);
 	struct btrfs_bio *repair_bbio;
 	struct bio *repair_bio;
+	struct bvec_iter iter = *orig_iter;
+	const u32 sectorsize = fs_info->sectorsize;
+	const u32 bio_offset = ((iter.bi_sector - failed_bbio->saved_iter.bi_sector) <<
+				SECTOR_SHIFT);
+	const u64 logical = (iter.bi_sector << SECTOR_SHIFT);
 	int num_copies;
 	int mirror;
+	u32 cur = 0;
 
 	btrfs_debug(fs_info, "repair read error: read error at %llu",
 		    failed_bbio->file_offset + bio_offset);
@@ -277,17 +245,21 @@ static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio,
 
 	atomic_inc(&fbio->repair_count);
 
-	repair_bio = bio_alloc_bioset(NULL, nr_steps, REQ_OP_READ, GFP_NOFS,
-				      &btrfs_repair_bioset);
+	repair_bio = bio_alloc_bioset(NULL, max(1, sectorsize >> PAGE_SHIFT),
+				      REQ_OP_READ, GFP_NOFS, &btrfs_repair_bioset);
 	repair_bio->bi_iter.bi_sector = logical >> SECTOR_SHIFT;
-	for (int i = 0; i < nr_steps; i++) {
+	while (cur < sectorsize) {
+		struct page *page = bio_iter_page(&failed_bbio->bio, iter);
+		const u32 pg_off = bio_iter_offset(&failed_bbio->bio, iter);
+		const u32 cur_len = min(bio_iter_len(&failed_bbio->bio, iter),
+					sectorsize - cur);
 		int ret;
 
-		ASSERT(offset_in_page(paddrs[i]) + step <= PAGE_SIZE);
+		ret = bio_add_page(repair_bio, page, cur_len, pg_off);
+		ASSERT(ret == cur_len);
 
-		ret = bio_add_page(repair_bio, phys_to_page(paddrs[i]), step,
-				   offset_in_page(paddrs[i]));
-		ASSERT(ret == step);
+		bio_advance_iter_single(&failed_bbio->bio, &iter, cur_len);
+		cur += cur_len;
 	}
 
 	repair_bbio = btrfs_bio(repair_bio);
@@ -305,18 +277,16 @@ static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *de
 	struct btrfs_inode *inode = bbio->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 	const u32 sectorsize = fs_info->sectorsize;
-	const u32 step = min(sectorsize, PAGE_SIZE);
-	const u32 nr_steps = sectorsize / step;
-	struct bvec_iter *iter = &bbio->saved_iter;
+	struct bvec_iter iter;
 	blk_status_t status = bbio->bio.bi_status;
 	struct btrfs_failed_bio *fbio = NULL;
-	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
-	phys_addr_t paddr;
-	u32 offset = 0;
 
 	/* Read-repair requires the inode field to be set by the submitter. */
 	ASSERT(inode);
 
+	/* The original bbio should be sectorsize aligned. */
+	ASSERT(IS_ALIGNED(bbio->saved_iter.bi_size, sectorsize));
+
 	/*
 	 * Hand off repair bios to the repair code as there is no upper level
 	 * submitter for them.
@@ -329,16 +299,10 @@ static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *de
 	/* Clear the I/O error. A failed repair will reset it. */
 	bbio->bio.bi_status = BLK_STS_OK;
 
-	btrfs_bio_for_each_block(paddr, &bbio->bio, iter, step) {
-		paddrs[(offset / step) % nr_steps] = paddr;
-		offset += step;
-
-		if (IS_ALIGNED(offset, sectorsize)) {
-			if (status ||
-			    !btrfs_data_csum_ok(bbio, dev, offset - sectorsize, paddrs))
-				fbio = repair_one_sector(bbio, offset - sectorsize,
-							 paddrs, fbio);
-		}
+	for (iter = bbio->saved_iter; iter.bi_size;
+	     bio_advance_iter(&bbio->bio, &iter, sectorsize)) {
+		if (status || !btrfs_bio_data_csum_ok(bbio, &iter, dev))
+			fbio = repair_one_sector(bbio, &iter, fbio);
 	}
 	if (bbio->csum != bbio->csum_inline)
 		kvfree(bbio->csum);
@@ -386,7 +350,7 @@ static void simple_end_io_work(struct work_struct *work)
 
 	if (bio_op(bio) == REQ_OP_READ) {
 		/* Metadata reads are checked and repaired by the submitter. */
-		if (is_data_bbio(bbio))
+		if (is_data_inode(bbio->inode))
 			return btrfs_check_read_bio(bbio, bbio->bio.bi_private);
 		return btrfs_bio_end_io(bbio, bbio->bio.bi_status);
 	}
@@ -420,7 +384,7 @@ static void btrfs_raid56_end_io(struct bio *bio)
 
 	btrfs_bio_counter_dec(bioc->fs_info);
 	bbio->mirror_num = bioc->mirror_num;
-	if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio))
+	if (bio_op(bio) == REQ_OP_READ && is_data_inode(bbio->inode))
 		btrfs_check_read_bio(bbio, NULL);
 	else
 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);
@@ -783,7 +747,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
 	 * our bio to the physical disk location, so we need to save the
 	 * original bytenr so we know what we're checksumming.
 	 */
-	if (bio_op(bio) == REQ_OP_WRITE && is_data_bbio(bbio))
+	if (bio_op(bio) == REQ_OP_WRITE && is_data_inode(bbio->inode))
 		bbio->orig_logical = logical;
 
 	bbio->can_use_append = btrfs_use_zone_append(bbio);
@@ -809,7 +773,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
 	 * Save the iter for the end_io handler and preload the checksums for
 	 * data reads.
 	 */
-	if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio)) {
+	if (bio_op(bio) == REQ_OP_READ && is_data_inode(bbio->inode)) {
 		bbio->saved_iter = bio->bi_iter;
 		ret = btrfs_lookup_bio_sums(bbio);
 		status = errno_to_blk_status(ret);
@@ -818,7 +782,7 @@ static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num)
 	}
 
 	if (btrfs_op(bio) == BTRFS_MAP_WRITE) {
-		if (is_data_bbio(bbio) && bioc && bioc->use_rst) {
+		if (is_data_inode(bbio->inode) && bioc && bioc->use_rst) {
 			/*
 			 * No locking for the list update, as we only add to
 			 * the list in the I/O submission path, and list
@@ -925,21 +889,23 @@ void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num)
  * The I/O is issued synchronously to block the repair read completion from
  * freeing the bio.
  *
- * @ino:	Offending inode number
- * @fileoff:	File offset inside the inode
+ * @bbio:	Original bbio where the repair is needed
+ * @orig_iter:	Points to where the repair starts
  * @length:	Length of the repair write
- * @logical:	Logical address of the range
- * @paddrs:	Physical address array of the content
- * @step:	Length of for each paddrs
  * @mirror_num: Mirror number to write to. Must not be zero
  */
-int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff,
-			    u32 length, u64 logical, const phys_addr_t paddrs[],
-			    unsigned int step, int mirror_num)
+int btrfs_repair_bbio_failure(struct btrfs_bio *bbio, const struct bvec_iter *orig_iter,
+			      u32 length, int mirror_num)
 {
-	const u32 nr_steps = DIV_ROUND_UP_POW2(length, step);
+	struct btrfs_inode *inode = bbio->inode;
+	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 	struct btrfs_io_stripe smap = { 0 };
-	struct bio *bio = NULL;
+	struct bvec_iter iter = *orig_iter;
+	struct bio *repair_bio = NULL;
+	const u64 logical = iter.bi_sector << SECTOR_SHIFT;
+	const u64 fileoff = bbio->file_offset +
+			    ((iter.bi_sector - bbio->saved_iter.bi_sector) << SECTOR_SHIFT);
+	u32 cur = 0;
 	int ret = 0;
 
 	BUG_ON(!mirror_num);
@@ -950,8 +916,9 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff,
 	ASSERT(IS_ALIGNED(fileoff, fs_info->sectorsize));
 	/* Either it's a single data or metadata block. */
 	ASSERT(length <= BTRFS_MAX_BLOCKSIZE);
-	ASSERT(step <= length);
-	ASSERT(is_power_of_2(step));
+
+	/* Our current iter should not be before the original bbio saved_iter. */
+	ASSERT(iter.bi_sector >= bbio->saved_iter.bi_sector);
 
 	/*
 	 * The fs either mounted RO or hit critical errors, no need
@@ -979,15 +946,22 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff,
 		goto out_counter_dec;
 	}
 
-	bio = bio_alloc(smap.dev->bdev, nr_steps, REQ_OP_WRITE | REQ_SYNC, GFP_NOFS);
-	bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT;
-	for (int i = 0; i < nr_steps; i++) {
-		ret = bio_add_page(bio, phys_to_page(paddrs[i]), step, offset_in_page(paddrs[i]));
-		/* We should have allocated enough slots to contain all the different pages. */
-		ASSERT(ret == step);
+	repair_bio = bio_alloc(smap.dev->bdev, max(1, length >> PAGE_SHIFT),
+			       REQ_OP_WRITE | REQ_SYNC, GFP_NOFS);
+	repair_bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT;
+	while (cur < length) {
+		struct page *page = bio_iter_page(&bbio->bio, iter);
+		const u32 pg_off = bio_iter_offset(&bbio->bio, iter);
+		const u32 cur_len = min(bio_iter_len(&bbio->bio, iter), length - cur);
+
+		ret = bio_add_page(repair_bio, page, cur_len, pg_off);
+		ASSERT(ret == cur_len);
+		bio_advance_iter_single(&bbio->bio, &iter, cur_len);
+		cur += cur_len;
 	}
-	ret = submit_bio_wait(bio);
-	bio_put(bio);
+
+	ret = submit_bio_wait(repair_bio);
+	bio_put(repair_bio);
 	if (ret) {
 		/* try to remap that extent elsewhere? */
 		btrfs_dev_stat_inc_and_print(smap.dev, BTRFS_DEV_STAT_WRITE_ERRS);
@@ -995,8 +969,9 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff,
 	}
 
 	btrfs_info_rl(fs_info,
-		"read error corrected: ino %llu off %llu (dev %s sector %llu)",
-			     ino, fileoff, btrfs_dev_name(smap.dev),
+		"read error corrected: root %llu ino %llu off %llu (dev %s sector %llu)",
+			     btrfs_root_id(inode->root), btrfs_ino(inode), fileoff,
+			     btrfs_dev_name(smap.dev),
 			     smap.physical >> SECTOR_SHIFT);
 	ret = 0;
 
diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h
index 303ed6c7103d92..bbf362b8668bbc 100644
--- a/fs/btrfs/bio.h
+++ b/fs/btrfs/bio.h
@@ -58,7 +58,6 @@ struct btrfs_bio {
 			struct btrfs_ordered_extent *ordered;
 			struct btrfs_ordered_sum *sums;
 			struct work_struct csum_work;
-			struct completion csum_done;
 			struct bvec_iter csum_saved_iter;
 			u64 orig_physical;
 			u64 orig_logical;
@@ -93,9 +92,6 @@ struct btrfs_bio {
 	/* Whether the bio is coming from copy_remapped_data_io(). */
 	bool is_remap:1;
 
-	/* Whether the csum generation for data write is async. */
-	bool async_csum:1;
-
 	/* Whether the bio is written using zone append. */
 	bool can_use_append:1;
 
@@ -126,8 +122,7 @@ void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status);
 
 void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num);
 void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace);
-int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff,
-			    u32 length, u64 logical, const phys_addr_t paddrs[],
-			    unsigned int step, int mirror_num);
+int btrfs_repair_bbio_failure(struct btrfs_bio *bbio, const struct bvec_iter *orig_iter,
+			      u32 length, int mirror_num);
 
 #endif
diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c
index ee182369254c08..e6080cb47c895c 100644
--- a/fs/btrfs/block-group.c
+++ b/fs/btrfs/block-group.c
@@ -904,22 +904,6 @@ static noinline void caching_thread(struct btrfs_work *work)
 	down_read(&fs_info->commit_root_sem);
 
 	load_block_group_size_class(caching_ctl);
-	if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
-		ret = load_free_space_cache(block_group);
-		if (ret == 1) {
-			ret = 0;
-			goto done;
-		}
-
-		/*
-		 * We failed to load the space cache, set ourselves to
-		 * CACHE_STARTED and carry on.
-		 */
-		spin_lock(&block_group->lock);
-		block_group->cached = BTRFS_CACHE_STARTED;
-		spin_unlock(&block_group->lock);
-		wake_up(&caching_ctl->wait);
-	}
 
 	/*
 	 * If we are in the transaction that populated the free space tree we
@@ -933,7 +917,7 @@ static noinline void caching_thread(struct btrfs_work *work)
 		ret = btrfs_load_free_space_tree(caching_ctl);
 	else
 		ret = load_extent_tree_free(caching_ctl);
-done:
+
 	spin_lock(&block_group->lock);
 	block_group->caching_ctl = NULL;
 	block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
@@ -1194,36 +1178,21 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
 		goto out;
 	}
 
-	/*
-	 * get the inode first so any iput calls done for the io_list
-	 * aren't the final iput (no unlinks allowed now)
-	 */
 	inode = lookup_free_space_inode(block_group, path);
 
-	mutex_lock(&trans->transaction->cache_write_mutex);
 	/*
-	 * Make sure our free space cache IO is done before removing the
-	 * free space inode
+	 * Do not delete the block group item while
+	 * btrfs_start_dirty_block_groups() is updating it.
 	 */
+	mutex_lock(&trans->transaction->dirty_bgs_update_mutex);
 	spin_lock(&trans->transaction->dirty_bgs_lock);
-	if (!list_empty(&block_group->io_list)) {
-		list_del_init(&block_group->io_list);
-
-		WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode);
-
-		spin_unlock(&trans->transaction->dirty_bgs_lock);
-		btrfs_wait_cache_io(trans, block_group, path);
-		btrfs_put_block_group(block_group);
-		spin_lock(&trans->transaction->dirty_bgs_lock);
-	}
-
 	if (!list_empty(&block_group->dirty_list)) {
 		list_del_init(&block_group->dirty_list);
 		remove_rsv = true;
 		btrfs_put_block_group(block_group);
 	}
 	spin_unlock(&trans->transaction->dirty_bgs_lock);
-	mutex_unlock(&trans->transaction->cache_write_mutex);
+	mutex_unlock(&trans->transaction->dirty_bgs_update_mutex);
 
 	ret = btrfs_remove_free_space_inode(trans, inode, block_group);
 	if (unlikely(ret)) {
@@ -1287,7 +1256,6 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
 
 	spin_lock(&trans->transaction->dirty_bgs_lock);
 	WARN_ON(!list_empty(&block_group->dirty_list));
-	WARN_ON(!list_empty(&block_group->io_list));
 	spin_unlock(&trans->transaction->dirty_bgs_lock);
 
 	btrfs_remove_free_space_cache(block_group);
@@ -1612,7 +1580,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
 
 		space_info = block_group->space_info;
 
-		if (ret || btrfs_mixed_space_info(space_info)) {
+		if (btrfs_mixed_space_info(space_info)) {
 			btrfs_put_block_group(block_group);
 			continue;
 		}
@@ -1727,6 +1695,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
 		ret = inc_block_group_ro(block_group, false);
 		up_write(&space_info->groups_sem);
 		if (ret < 0) {
+			btrfs_link_bg_list(block_group, &retry_list);
 			ret = 0;
 			goto next;
 		}
@@ -1749,6 +1718,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
 						     block_group->start);
 		if (IS_ERR(trans)) {
 			btrfs_dec_block_group_ro(block_group);
+			btrfs_link_bg_list(block_group, &retry_list);
 			ret = PTR_ERR(trans);
 			goto next;
 		}
@@ -1759,6 +1729,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
 		 */
 		if (!clean_pinned_extents(trans, block_group)) {
 			btrfs_dec_block_group_ro(block_group);
+			btrfs_link_bg_list(block_group, &retry_list);
 			goto end_trans;
 		}
 
@@ -1845,6 +1816,8 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
 next:
 		btrfs_put_block_group(block_group);
 		spin_lock(&fs_info->unused_bgs_lock);
+		if (ret)
+			break;
 	}
 	list_splice_tail(&retry_list, &fs_info->unused_bgs);
 	spin_unlock(&fs_info->unused_bgs_lock);
@@ -2431,7 +2404,6 @@ static struct btrfs_block_group *btrfs_create_block_group(
 	INIT_LIST_HEAD(&cache->ro_list);
 	INIT_LIST_HEAD(&cache->discard_list);
 	INIT_LIST_HEAD(&cache->dirty_list);
-	INIT_LIST_HEAD(&cache->io_list);
 	INIT_LIST_HEAD(&cache->active_bg_list);
 	btrfs_init_free_space_ctl(cache, cache->free_space_ctl);
 	atomic_set(&cache->frozen, 0);
@@ -2487,8 +2459,7 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info)
 
 static int read_one_block_group(struct btrfs_fs_info *info,
 				struct btrfs_block_group_item_v2 *bgi,
-				const struct btrfs_key *key,
-				bool need_clear)
+				const struct btrfs_key *key)
 {
 	struct btrfs_block_group *cache;
 	const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS);
@@ -2514,20 +2485,6 @@ static int read_one_block_group(struct btrfs_fs_info *info,
 
 	btrfs_set_free_space_tree_thresholds(cache);
 
-	if (need_clear) {
-		/*
-		 * When we mount with old space cache, we need to
-		 * set BTRFS_DC_CLEAR and set dirty flag.
-		 *
-		 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
-		 *    truncate the old free space cache inode and
-		 *    setup a new one.
-		 * b) Setting 'dirty flag' makes sure that we flush
-		 *    the new space cache info onto disk.
-		 */
-		if (btrfs_test_opt(info, SPACE_CACHE))
-			cache->disk_cache_state = BTRFS_DC_CLEAR;
-	}
 	if (!mixed && ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) &&
 	    (cache->flags & BTRFS_BLOCK_GROUP_DATA))) {
 			btrfs_err(info,
@@ -2668,8 +2625,6 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info)
 	struct btrfs_block_group *cache;
 	struct btrfs_space_info *space_info;
 	struct btrfs_key key;
-	bool need_clear = false;
-	u64 cache_gen;
 
 	/*
 	 * Either no extent root (with ibadroots rescue option) or we have
@@ -2690,13 +2645,6 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info)
 	if (!path)
 		return -ENOMEM;
 
-	cache_gen = btrfs_super_cache_generation(info->super_copy);
-	if (btrfs_test_opt(info, SPACE_CACHE) &&
-	    btrfs_super_generation(info->super_copy) != cache_gen)
-		need_clear = true;
-	if (btrfs_test_opt(info, CLEAR_CACHE))
-		need_clear = true;
-
 	while (1) {
 		struct btrfs_block_group_item_v2 bgi;
 		struct extent_buffer *leaf;
@@ -2725,7 +2673,7 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info)
 
 		btrfs_item_key_to_cpu(leaf, &key, slot);
 		btrfs_release_path(path);
-		ret = read_one_block_group(info, &bgi, &key, need_clear);
+		ret = read_one_block_group(info, &bgi, &key);
 		if (ret < 0)
 			goto error;
 		key.objectid += key.offset;
@@ -3373,207 +3321,16 @@ static int update_block_group_item(struct btrfs_trans_handle *trans,
 
 }
 
-static void cache_save_setup(struct btrfs_block_group *block_group,
-			     struct btrfs_trans_handle *trans,
-			     struct btrfs_path *path)
-{
-	struct btrfs_fs_info *fs_info = block_group->fs_info;
-	struct inode *inode = NULL;
-	struct extent_changeset *data_reserved = NULL;
-	u64 alloc_hint = 0;
-	int dcs = BTRFS_DC_ERROR;
-	u64 cache_size = 0;
-	int retries = 0;
-	int ret = 0;
-
-	if (!btrfs_test_opt(fs_info, SPACE_CACHE))
-		return;
-
-	/*
-	 * If this block group is smaller than 100 megs don't bother caching the
-	 * block group.
-	 */
-	if (block_group->length < (100 * SZ_1M)) {
-		spin_lock(&block_group->lock);
-		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
-		spin_unlock(&block_group->lock);
-		return;
-	}
-
-	if (TRANS_ABORTED(trans))
-		return;
-again:
-	inode = lookup_free_space_inode(block_group, path);
-	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
-		ret = PTR_ERR(inode);
-		btrfs_release_path(path);
-		goto out;
-	}
-
-	if (IS_ERR(inode)) {
-		if (retries) {
-			ret = PTR_ERR(inode);
-			btrfs_err(fs_info,
-				  "failed to lookup free space inode after creation for block group %llu: %d",
-				  block_group->start, ret);
-			goto out_free;
-		}
-		retries++;
-
-		if (block_group->ro)
-			goto out_free;
-
-		ret = create_free_space_inode(trans, block_group, path);
-		if (ret)
-			goto out_free;
-		goto again;
-	}
-
-	/*
-	 * We want to set the generation to 0, that way if anything goes wrong
-	 * from here on out we know not to trust this cache when we load up next
-	 * time.
-	 */
-	BTRFS_I(inode)->generation = 0;
-	ret = btrfs_update_inode(trans, BTRFS_I(inode));
-	if (unlikely(ret)) {
-		/*
-		 * So theoretically we could recover from this, simply set the
-		 * super cache generation to 0 so we know to invalidate the
-		 * cache, but then we'd have to keep track of the block groups
-		 * that fail this way so we know we _have_ to reset this cache
-		 * before the next commit or risk reading stale cache.  So to
-		 * limit our exposure to horrible edge cases lets just abort the
-		 * transaction, this only happens in really bad situations
-		 * anyway.
-		 */
-		btrfs_abort_transaction(trans, ret);
-		goto out_put;
-	}
-
-	/* We've already setup this transaction, go ahead and exit */
-	if (block_group->cache_generation == trans->transid &&
-	    i_size_read(inode)) {
-		dcs = BTRFS_DC_SETUP;
-		goto out_put;
-	}
-
-	if (i_size_read(inode) > 0) {
-		ret = btrfs_check_trunc_cache_free_space(fs_info,
-					&fs_info->global_block_rsv);
-		if (ret)
-			goto out_put;
-
-		ret = btrfs_truncate_free_space_cache(trans, NULL, inode);
-		if (ret)
-			goto out_put;
-	}
-
-	spin_lock(&block_group->lock);
-	if (block_group->cached != BTRFS_CACHE_FINISHED ||
-	    !btrfs_test_opt(fs_info, SPACE_CACHE)) {
-		/*
-		 * don't bother trying to write stuff out _if_
-		 * a) we're not cached,
-		 * b) we're with nospace_cache mount option,
-		 * c) we're with v2 space_cache (FREE_SPACE_TREE).
-		 */
-		dcs = BTRFS_DC_WRITTEN;
-		spin_unlock(&block_group->lock);
-		goto out_put;
-	}
-	spin_unlock(&block_group->lock);
-
-	/*
-	 * We hit an ENOSPC when setting up the cache in this transaction, just
-	 * skip doing the setup, we've already cleared the cache so we're safe.
-	 */
-	if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags))
-		goto out_put;
-
-	/*
-	 * Try to preallocate enough space based on how big the block group is.
-	 * Keep in mind this has to include any pinned space which could end up
-	 * taking up quite a bit since it's not folded into the other space
-	 * cache.
-	 */
-	cache_size = div_u64(block_group->length, SZ_256M);
-	if (!cache_size)
-		cache_size = 1;
-
-	cache_size *= 16;
-	cache_size *= fs_info->sectorsize;
-
-	ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 0,
-					  cache_size, false);
-	if (ret)
-		goto out_put;
-
-	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, cache_size,
-					      cache_size, cache_size,
-					      &alloc_hint);
-	/*
-	 * Our cache requires contiguous chunks so that we don't modify a bunch
-	 * of metadata or split extents when writing the cache out, which means
-	 * we can enospc if we are heavily fragmented in addition to just normal
-	 * out of space conditions.  So if we hit this just skip setting up any
-	 * other block groups for this transaction, maybe we'll unpin enough
-	 * space the next time around.
-	 */
-	if (!ret)
-		dcs = BTRFS_DC_SETUP;
-	else if (ret == -ENOSPC)
-		set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags);
-
-out_put:
-	iput(inode);
-out_free:
-	btrfs_release_path(path);
-out:
-	spin_lock(&block_group->lock);
-	if (!ret && dcs == BTRFS_DC_SETUP)
-		block_group->cache_generation = trans->transid;
-	block_group->disk_cache_state = dcs;
-	spin_unlock(&block_group->lock);
-
-	extent_changeset_free(data_reserved);
-}
-
-int btrfs_setup_space_cache(struct btrfs_trans_handle *trans)
-{
-	struct btrfs_fs_info *fs_info = trans->fs_info;
-	struct btrfs_block_group *cache, *tmp;
-	struct btrfs_transaction *cur_trans = trans->transaction;
-	BTRFS_PATH_AUTO_FREE(path);
-
-	if (list_empty(&cur_trans->dirty_bgs) ||
-	    !btrfs_test_opt(fs_info, SPACE_CACHE))
-		return 0;
-
-	path = btrfs_alloc_path();
-	if (!path)
-		return -ENOMEM;
-
-	/* Could add new block groups, use _safe just in case */
-	list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs,
-				 dirty_list) {
-		if (cache->disk_cache_state == BTRFS_DC_CLEAR)
-			cache_save_setup(cache, trans, path);
-	}
-
-	return 0;
-}
-
 /*
- * Transaction commit does final block group cache writeback during a critical
+ * Transaction commit does the final block group item updates during a critical
  * section where nothing is allowed to change the FS.  This is required in
- * order for the cache to actually match the block group, but can introduce a
+ * order for the items to actually match the block groups, but can introduce a
  * lot of latency into the commit.
  *
- * So, btrfs_start_dirty_block_groups is here to kick off block group cache IO.
- * There's a chance we'll have to redo some of it if the block group changes
- * again during the commit, but it greatly reduces the commit latency by
- * getting rid of the easy block groups while we're still allowing others to
+ * So, btrfs_start_dirty_block_groups is here to update the block group items
+ * early.  There's a chance we'll have to redo some of it if the block group
+ * changes again during the commit, but it greatly reduces the commit latency
+ * by getting rid of the easy block groups while we're still allowing others to
  * join the commit.
  */
 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
@@ -3582,10 +3339,8 @@ int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
 	struct btrfs_block_group *cache;
 	struct btrfs_transaction *cur_trans = trans->transaction;
 	int ret = 0;
-	int should_put;
 	BTRFS_PATH_AUTO_FREE(path);
 	LIST_HEAD(dirty);
-	struct list_head *io = &cur_trans->io_bgs;
 	int loops = 0;
 
 	spin_lock(&cur_trans->dirty_bgs_lock);
@@ -3609,33 +3364,18 @@ int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
 	}
 
 	/*
-	 * cache_write_mutex is here only to save us from balance or automatic
-	 * removal of empty block groups deleting this block group while we are
-	 * writing out the cache
+	 * dirty_bgs_update_mutex is here only to save us from balance or
+	 * automatic removal of empty block groups deleting this block group
+	 * while we are updating its item
 	 */
-	mutex_lock(&trans->transaction->cache_write_mutex);
+	mutex_lock(&trans->transaction->dirty_bgs_update_mutex);
 	while (!list_empty(&dirty)) {
 		bool drop_reserve = true;
 
 		cache = list_first_entry(&dirty, struct btrfs_block_group,
 					 dirty_list);
-		/*
-		 * This can happen if something re-dirties a block group that
-		 * is already under IO.  Just wait for it to finish and then do
-		 * it all again
-		 */
-		if (!list_empty(&cache->io_list)) {
-			list_del_init(&cache->io_list);
-			btrfs_wait_cache_io(trans, cache, path);
-			btrfs_put_block_group(cache);
-		}
-
 
 		/*
-		 * btrfs_wait_cache_io uses the cache->dirty_list to decide if
-		 * it should update the cache_state.  Don't delete until after
-		 * we wait.
-		 *
 		 * Since we're not running in the commit critical section
 		 * we need the dirty_bgs_lock to protect from update_block_group
 		 */
@@ -3643,72 +3383,39 @@ int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
 		list_del_init(&cache->dirty_list);
 		spin_unlock(&cur_trans->dirty_bgs_lock);
 
-		should_put = 1;
-
-		cache_save_setup(cache, trans, path);
-
-		if (cache->disk_cache_state == BTRFS_DC_SETUP) {
-			cache->io_ctl.inode = NULL;
-			ret = btrfs_write_out_cache(trans, cache, path);
-			if (ret == 0 && cache->io_ctl.inode) {
-				should_put = 0;
-
-				/*
-				 * The cache_write_mutex is protecting the
-				 * io_list, also refer to the definition of
-				 * btrfs_transaction::io_bgs for more details
-				 */
-				list_add_tail(&cache->io_list, io);
-			} else {
-				/*
-				 * If we failed to write the cache, the
-				 * generation will be bad and life goes on
-				 */
-				ret = 0;
-			}
-		}
-		if (!ret) {
-			ret = update_block_group_item(trans, path, cache);
-			/*
-			 * Our block group might still be attached to the list
-			 * of new block groups in the transaction handle of some
-			 * other task (struct btrfs_trans_handle->new_bgs). This
-			 * means its block group item isn't yet in the extent
-			 * tree. If this happens ignore the error, as we will
-			 * try again later in the critical section of the
-			 * transaction commit.
-			 */
-			if (ret == -ENOENT) {
-				ret = 0;
-				spin_lock(&cur_trans->dirty_bgs_lock);
-				if (list_empty(&cache->dirty_list)) {
-					list_add_tail(&cache->dirty_list,
-						      &cur_trans->dirty_bgs);
-					btrfs_get_block_group(cache);
-					drop_reserve = false;
-				}
-				spin_unlock(&cur_trans->dirty_bgs_lock);
-			} else if (ret) {
-				btrfs_abort_transaction(trans, ret);
+		ret = update_block_group_item(trans, path, cache);
+		/*
+		 * Our block group might still be attached to the list of new
+		 * block groups in the transaction handle of some other task
+		 * (struct btrfs_trans_handle->new_bgs). This means its block
+		 * group item isn't yet in the extent tree. If this happens
+		 * ignore the error, as we will try again later in the critical
+		 * section of the transaction commit.
+		 */
+		if (ret == -ENOENT) {
+			ret = 0;
+			spin_lock(&cur_trans->dirty_bgs_lock);
+			if (list_empty(&cache->dirty_list)) {
+				list_add_tail(&cache->dirty_list,
+					      &cur_trans->dirty_bgs);
+				btrfs_get_block_group(cache);
+				drop_reserve = false;
 			}
+			spin_unlock(&cur_trans->dirty_bgs_lock);
+		} else if (ret) {
+			btrfs_abort_transaction(trans, ret);
 		}
 
-		/* If it's not on the io list, we need to put the block group */
-		if (should_put)
-			btrfs_put_block_group(cache);
+		btrfs_put_block_group(cache);
 		if (drop_reserve)
 			btrfs_dec_delayed_refs_rsv_bg_updates(fs_info);
-		/*
-		 * Avoid blocking other tasks for too long. It might even save
-		 * us from writing caches for block groups that are going to be
-		 * removed.
-		 */
-		mutex_unlock(&trans->transaction->cache_write_mutex);
+		/* Avoid blocking other tasks for too long. */
+		mutex_unlock(&trans->transaction->dirty_bgs_update_mutex);
 		if (ret)
 			goto out;
-		mutex_lock(&trans->transaction->cache_write_mutex);
+		mutex_lock(&trans->transaction->dirty_bgs_update_mutex);
 	}
-	mutex_unlock(&trans->transaction->cache_write_mutex);
+	mutex_unlock(&trans->transaction->dirty_bgs_update_mutex);
 
 	/*
 	 * Go through delayed refs for all the stuff we've just kicked off
@@ -3722,7 +3429,7 @@ int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
 		list_splice_init(&cur_trans->dirty_bgs, &dirty);
 		/*
 		 * dirty_bgs_lock protects us from concurrent block group
-		 * deletes too (not just cache_write_mutex).
+		 * deletes too (not just dirty_bgs_update_mutex).
 		 */
 		if (!list_empty(&dirty)) {
 			spin_unlock(&cur_trans->dirty_bgs_lock);
@@ -3747,121 +3454,34 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans)
 	struct btrfs_block_group *cache;
 	struct btrfs_transaction *cur_trans = trans->transaction;
 	int ret = 0;
-	int should_put;
 	BTRFS_PATH_AUTO_FREE(path);
-	struct list_head *io = &cur_trans->io_bgs;
 
 	path = btrfs_alloc_path();
 	if (!path)
 		return -ENOMEM;
 
-	/*
-	 * Even though we are in the critical section of the transaction commit,
-	 * we can still have concurrent tasks adding elements to this
-	 * transaction's list of dirty block groups. These tasks correspond to
-	 * endio free space workers started when writeback finishes for a
-	 * space cache, which run inode.c:btrfs_finish_ordered_io(), and can
-	 * allocate new block groups as a result of COWing nodes of the root
-	 * tree when updating the free space inode. The writeback for the space
-	 * caches is triggered by an earlier call to
-	 * btrfs_start_dirty_block_groups() and iterations of the following
-	 * loop.
-	 * Also we want to do the cache_save_setup first and then run the
-	 * delayed refs to make sure we have the best chance at doing this all
-	 * in one shot.
-	 */
 	spin_lock(&cur_trans->dirty_bgs_lock);
 	while (!list_empty(&cur_trans->dirty_bgs)) {
 		cache = list_first_entry(&cur_trans->dirty_bgs,
 					 struct btrfs_block_group,
 					 dirty_list);
-
-		/*
-		 * This can happen if cache_save_setup re-dirties a block group
-		 * that is already under IO.  Just wait for it to finish and
-		 * then do it all again
-		 */
-		if (!list_empty(&cache->io_list)) {
-			spin_unlock(&cur_trans->dirty_bgs_lock);
-			list_del_init(&cache->io_list);
-			btrfs_wait_cache_io(trans, cache, path);
-			btrfs_put_block_group(cache);
-			spin_lock(&cur_trans->dirty_bgs_lock);
-		}
-
-		/*
-		 * Don't remove from the dirty list until after we've waited on
-		 * any pending IO
-		 */
 		list_del_init(&cache->dirty_list);
 		spin_unlock(&cur_trans->dirty_bgs_lock);
-		should_put = 1;
-
-		cache_save_setup(cache, trans, path);
 
 		if (!ret)
 			ret = btrfs_run_delayed_refs(trans, U64_MAX);
-
-		if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) {
-			cache->io_ctl.inode = NULL;
-			ret = btrfs_write_out_cache(trans, cache, path);
-			if (ret == 0 && cache->io_ctl.inode) {
-				should_put = 0;
-				list_add_tail(&cache->io_list, io);
-			} else {
-				/*
-				 * If we failed to write the cache, the
-				 * generation will be bad and life goes on
-				 */
-				ret = 0;
-			}
-		}
 		if (!ret) {
 			ret = update_block_group_item(trans, path, cache);
-			/*
-			 * One of the free space endio workers might have
-			 * created a new block group while updating a free space
-			 * cache's inode (at inode.c:btrfs_finish_ordered_io())
-			 * and hasn't released its transaction handle yet, in
-			 * which case the new block group is still attached to
-			 * its transaction handle and its creation has not
-			 * finished yet (no block group item in the extent tree
-			 * yet, etc). If this is the case, wait for all free
-			 * space endio workers to finish and retry. This is a
-			 * very rare case so no need for a more efficient and
-			 * complex approach.
-			 */
-			if (ret == -ENOENT) {
-				wait_event(cur_trans->writer_wait,
-				   atomic_read(&cur_trans->num_writers) == 1);
-				ret = update_block_group_item(trans, path, cache);
-				if (ret)
-					btrfs_abort_transaction(trans, ret);
-			} else if (ret) {
+			if (ret)
 				btrfs_abort_transaction(trans, ret);
-			}
 		}
 
-		/* If its not on the io list, we need to put the block group */
-		if (should_put)
-			btrfs_put_block_group(cache);
+		btrfs_put_block_group(cache);
 		btrfs_dec_delayed_refs_rsv_bg_updates(fs_info);
 		spin_lock(&cur_trans->dirty_bgs_lock);
 	}
 	spin_unlock(&cur_trans->dirty_bgs_lock);
 
-	/*
-	 * Refer to the definition of io_bgs member for details why it's safe
-	 * to use it without any locking
-	 */
-	while (!list_empty(io)) {
-		cache = list_first_entry(io, struct btrfs_block_group,
-					 io_list);
-		list_del_init(&cache->io_list);
-		btrfs_wait_cache_io(trans, cache, path);
-		btrfs_put_block_group(cache);
-	}
-
 	return ret;
 }
 
@@ -3905,10 +3525,10 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans,
 	factor = btrfs_bg_type_to_factor(cache->flags);
 
 	/*
-	 * If this block group has free space cache written out, we need to make
-	 * sure to load it if we are removing space.  This is because we need
-	 * the unpinning stage to actually add the space back to the block group,
-	 * otherwise we will leak space.
+	 * Make sure the free space of this block group is loaded if we are
+	 * removing space.  This is because we need the unpinning stage to
+	 * actually add the space back to the block group, otherwise we will
+	 * leak space.
 	 */
 	if (!alloc && !btrfs_block_group_done(cache))
 		btrfs_cache_block_group(cache, true);
@@ -3916,10 +3536,6 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans,
 	spin_lock(&space_info->lock);
 	spin_lock(&cache->lock);
 
-	if (btrfs_test_opt(info, SPACE_CACHE) &&
-	    cache->disk_cache_state < BTRFS_DC_CLEAR)
-		cache->disk_cache_state = BTRFS_DC_CLEAR;
-
 	old_val = cache->used;
 	if (alloc) {
 		old_val += num_bytes;
@@ -3964,7 +3580,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans,
 	/*
 	 * No longer have used bytes in this block group, queue it for deletion.
 	 * We do this after adding the block group to the dirty list to avoid
-	 * races between cleaner kthread and space cache writeout.
+	 * races between the cleaner kthread and the dirty block group writeout.
 	 */
 	if (!alloc && old_val == 0) {
 		if (!btrfs_test_opt(info, DISCARD_ASYNC))
@@ -4640,7 +4256,6 @@ void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
 			block_group->inode = NULL;
 			spin_unlock(&block_group->lock);
 
-			ASSERT(block_group->io_ctl.inode == NULL);
 			iput(&inode->vfs_inode);
 		} else {
 			spin_unlock(&block_group->lock);
@@ -4777,7 +4392,6 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info)
 		btrfs_remove_free_space_cache(block_group);
 		ASSERT(block_group->cached != BTRFS_CACHE_STARTED);
 		ASSERT(list_empty(&block_group->dirty_list));
-		ASSERT(list_empty(&block_group->io_list));
 		ASSERT(list_empty(&block_group->bg_list));
 		ASSERT(refcount_read(&block_group->refs) == 1);
 		ASSERT(block_group->swap_extents == 0);
diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h
index 69d56864d4baa0..d567ed822e55da 100644
--- a/fs/btrfs/block-group.h
+++ b/fs/btrfs/block-group.h
@@ -20,13 +20,6 @@ struct btrfs_fs_info;
 struct btrfs_inode;
 struct btrfs_trans_handle;
 
-enum btrfs_disk_cache_state {
-	BTRFS_DC_WRITTEN,
-	BTRFS_DC_ERROR,
-	BTRFS_DC_CLEAR,
-	BTRFS_DC_SETUP,
-};
-
 enum btrfs_block_group_size_class {
 	/* Unset */
 	BTRFS_BG_SZ_NONE,
@@ -131,11 +124,10 @@ struct btrfs_block_group {
 	u64 delalloc_bytes;
 	u64 bytes_super;
 	u64 flags;
-	u64 cache_generation;
 	u64 global_root_id;
 	u64 remap_bytes;
 	u32 identity_remap_count;
-	/* The last commited identity_remap_count value of this block group. */
+	/* The last committed identity_remap_count value of this block group. */
 	u32 last_identity_remap_count;
 	/*
 	 * The last committed used bytes of this block group, if the above @used
@@ -171,8 +163,6 @@ struct btrfs_block_group {
 	unsigned long full_stripe_len;
 	unsigned long runtime_flags;
 
-	enum btrfs_disk_cache_state disk_cache_state;
-
 	/* Cache tracking stuff */
 	enum btrfs_caching_type cached;
 	struct btrfs_caching_control *caching_ctl;
@@ -228,9 +218,6 @@ struct btrfs_block_group {
 
 	/* For dirty block groups */
 	struct list_head dirty_list;
-	struct list_head io_list;
-
-	struct btrfs_io_ctl io_ctl;
 
 	/*
 	 * Incremented when doing extent allocations and holding a read lock
@@ -368,7 +355,6 @@ int btrfs_inc_block_group_ro(struct btrfs_block_group *cache,
 void btrfs_dec_block_group_ro(struct btrfs_block_group *cache);
 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans);
 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans);
-int btrfs_setup_space_cache(struct btrfs_trans_handle *trans);
 int btrfs_update_block_group(struct btrfs_trans_handle *trans,
 			     u64 bytenr, u64 num_bytes, bool alloc);
 int btrfs_add_reserved_bytes(struct btrfs_block_group *cache,
diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h
index 1082fa92c1457a..b673851d8d2ed9 100644
--- a/fs/btrfs/btrfs_inode.h
+++ b/fs/btrfs/btrfs_inode.h
@@ -32,6 +32,7 @@ struct btrfs_trans_handle;
 struct btrfs_bio;
 struct btrfs_file_extent;
 struct btrfs_delayed_node;
+struct btrfs_dir_index_prealloc;
 
 /*
  * Since we search a directory based on f_pos (struct dir_context::pos) we have
@@ -402,8 +403,6 @@ static inline void btrfs_mod_outstanding_extents(struct btrfs_inode *inode,
 {
 	lockdep_assert_held(&inode->lock);
 	inode->outstanding_extents += mod;
-	if (btrfs_is_free_space_inode(inode))
-		return;
 	trace_btrfs_inode_mod_outstanding_extents(inode->root, btrfs_ino(inode),
 						  mod, inode->outstanding_extents);
 }
@@ -507,14 +506,10 @@ static inline void btrfs_set_inode_mapping_order(struct btrfs_inode *inode)
 				      inode->root->fs_info->block_max_order);
 }
 
-void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info,
-				      const phys_addr_t paddr, u8 *dest);
-void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info,
-				      const phys_addr_t paddrs[], u8 *dest);
-int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum,
-			   const u8 * const csum_expected);
-bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
-			u32 bio_offset, const phys_addr_t paddrs[]);
+bool btrfs_bio_data_csum_ok(struct btrfs_bio *bbio, const struct bvec_iter *orig_iter,
+			    struct btrfs_device *dev);
+void btrfs_csum_one_bio_block(struct btrfs_fs_info *fs_info, struct bio *bio,
+			      const struct bvec_iter *orig_iter, u8 *csum);
 noinline int can_nocow_extent(struct btrfs_inode *inode, u64 offset, u64 *len,
 			      struct btrfs_file_extent *file_extent,
 			      bool nowait);
@@ -527,7 +522,8 @@ int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
 		       const struct fscrypt_str *name);
 int btrfs_add_link(struct btrfs_trans_handle *trans,
 		   struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
-		   const struct fscrypt_str *name, bool add_backref, u64 index);
+		   const struct fscrypt_str *name, bool add_backref, u64 index,
+		   struct btrfs_dir_index_prealloc *prealloc);
 int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry);
 int btrfs_truncate_block(struct btrfs_inode *inode, u64 offset, u64 start, u64 end);
 
@@ -559,7 +555,7 @@ int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args,
 int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
 			   struct btrfs_new_inode_args *args);
 void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args);
-struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap,
+struct inode *btrfs_new_subvol_inode(const struct mnt_idmap *idmap,
 				     struct inode *dir);
  void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state,
 			        u32 bits);
@@ -594,10 +590,6 @@ int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info);
 int btrfs_prealloc_file_range(struct inode *inode, int mode,
 			      u64 start, u64 num_bytes, u64 min_size,
 			      loff_t actual_len, u64 *alloc_hint);
-int btrfs_prealloc_file_range_trans(struct inode *inode,
-				    struct btrfs_trans_handle *trans, int mode,
-				    u64 start, u64 num_bytes, u64 min_size,
-				    loff_t actual_len, u64 *alloc_hint);
 int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio,
 			     u64 start, u64 end, struct writeback_control *wbc);
 void btrfs_queue_writepage_fixup(struct btrfs_inode *inode, struct folio *folio);
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index c62b5148d5ac19..ad90e14032dbb3 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -168,10 +168,10 @@ static unsigned long btrfs_compr_pool_scan(struct shrinker *sh, struct shrink_co
 	spin_unlock(&compr_pool.lock);
 
 	list_for_each_safe(tmp, next, &remove) {
-		struct page *page = list_entry(tmp, struct page, lru);
+		struct folio *folio = list_entry(tmp, struct folio, lru);
 
-		ASSERT(page_ref_count(page) == 1);
-		put_page(page);
+		ASSERT(folio_ref_count(folio) == 1);
+		folio_put(folio);
 	}
 
 	return freed;
@@ -431,7 +431,7 @@ static noinline int add_ra_bio_folios(struct inode *inode, u64 compressed_end,
 		}
 
 		/*
-		 * Since add_ra_bio_pages() is always speculative, suppress
+		 * Since add_ra_bio_folios() is always speculative, suppress
 		 * allocation warnings.
 		 */
 		masked_constraint_gfp = mapping_gfp_constraint(mapping, constraint_gfp);
@@ -960,7 +960,7 @@ bool btrfs_compress_level_valid(unsigned int type, int level)
 	return levels->min_level <= level && level <= levels->max_level;
 }
 
-/* Wrapper around find_get_page(), with extra error message. */
+/* Wrapper around filemap_get_folio(), with extra error message. */
 int btrfs_compress_filemap_get_folio(struct address_space *mapping, u64 start,
 				     struct folio **in_folio_ret)
 {
@@ -1488,10 +1488,11 @@ static bool sample_repeated_patterns(struct heuristic_ws *ws)
 static void heuristic_collect_sample(struct inode *inode, u64 start, u64 end,
 				     struct heuristic_ws *ws)
 {
-	struct page *page;
-	pgoff_t index, index_end;
-	u32 i, curr_sample_pos;
-	u8 *in_data;
+	const u32 blocksize = BTRFS_I(inode)->root->fs_info->sectorsize;
+	u64 cur = start;
+	u32 curr_sample_pos = 0;
+
+	ASSERT(IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize));
 
 	/*
 	 * Compression handles the input data by chunks of 128KiB
@@ -1502,38 +1503,30 @@ static void heuristic_collect_sample(struct inode *inode, u64 start, u64 end,
 	 * MAX_SAMPLE_SIZE - calculated under assumption that heuristic will
 	 * process no more than BTRFS_MAX_UNCOMPRESSED at a time.
 	 */
-	if (end - start > BTRFS_MAX_UNCOMPRESSED)
-		end = start + BTRFS_MAX_UNCOMPRESSED;
-
-	index = start >> PAGE_SHIFT;
-	index_end = end >> PAGE_SHIFT;
-
-	/* Don't miss unaligned end */
-	if (!PAGE_ALIGNED(end))
-		index_end++;
-
-	curr_sample_pos = 0;
-	while (index < index_end) {
-		page = find_get_page(inode->i_mapping, index);
-		in_data = kmap_local_page(page);
-		/* Handle case where the start is not aligned to PAGE_SIZE */
-		i = start % PAGE_SIZE;
-		while (i < PAGE_SIZE - SAMPLING_READ_SIZE) {
-			/* Don't sample any garbage from the last page */
-			if (start > end - SAMPLING_READ_SIZE)
-				break;
-			memcpy(&ws->sample[curr_sample_pos], &in_data[i],
-					SAMPLING_READ_SIZE);
-			i += SAMPLING_INTERVAL;
-			start += SAMPLING_INTERVAL;
+	if (end + 1 - start > BTRFS_MAX_UNCOMPRESSED)
+		end = start + BTRFS_MAX_UNCOMPRESSED - 1;
+
+	while (cur < end) {
+		struct folio *folio;
+		void *in_data;
+		u64 next_pos;
+
+		folio = filemap_get_folio(inode->i_mapping, cur >> PAGE_SHIFT);
+		/* All folios inside the range should exist and be locked. */
+		ASSERT(!IS_ERR(folio));
+		next_pos = min_t(u64, end + 1, folio_next_pos(folio));
+		in_data = kmap_local_folio(folio, 0);
+
+		for (; cur < next_pos; cur += SAMPLING_INTERVAL) {
+			memcpy(&ws->sample[curr_sample_pos],
+			       in_data + offset_in_folio(folio, cur),
+			       SAMPLING_READ_SIZE);
 			curr_sample_pos += SAMPLING_READ_SIZE;
 		}
 		kunmap_local(in_data);
-		put_page(page);
-
-		index++;
+		folio_put(folio);
+		cur = next_pos;
 	}
-
 	ws->sample_size = curr_sample_pos;
 }
 
diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index 8fe330d81b8ff3..fef0e49dd91882 100644
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -3943,7 +3943,7 @@ static noinline int split_item(struct btrfs_trans_handle *trans,
 	orig_offset = btrfs_item_offset(leaf, path->slots[0]);
 	item_size = btrfs_item_size(leaf, path->slots[0]);
 
-	buf = kmalloc(item_size, GFP_NOFS);
+	buf = kvmalloc(item_size, GFP_NOFS);
 	if (!buf)
 		return -ENOMEM;
 
@@ -3981,7 +3981,7 @@ static noinline int split_item(struct btrfs_trans_handle *trans,
 	btrfs_mark_buffer_dirty(trans, leaf);
 
 	BUG_ON(btrfs_leaf_free_space(leaf) < 0);
-	kfree(buf);
+	kvfree(buf);
 	return 0;
 }
 
diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c
index d357ed7efd99bd..77781852e41782 100644
--- a/fs/btrfs/delalloc-space.c
+++ b/fs/btrfs/delalloc-space.c
@@ -132,9 +132,7 @@ int btrfs_alloc_data_chunk_ondemand(const struct btrfs_inode *inode, u64 bytes)
 	/* Make sure bytes are sectorsize aligned */
 	bytes = ALIGN(bytes, fs_info->sectorsize);
 
-	if (btrfs_is_free_space_inode(inode))
-		flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE;
-	else if (btrfs_is_zoned(fs_info) && btrfs_is_data_reloc_root(root))
+	if (btrfs_is_zoned(fs_info) && btrfs_is_data_reloc_root(root))
 		flush = BTRFS_RESERVE_FLUSH_ZONED_RELOCATION;
 
 	return btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), bytes, flush);
@@ -155,8 +153,6 @@ int btrfs_check_data_free_space(struct btrfs_inode *inode,
 
 	if (noflush)
 		flush = BTRFS_RESERVE_NO_FLUSH;
-	else if (btrfs_is_free_space_inode(inode))
-		flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE;
 
 	ret = btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), len, flush);
 	if (ret < 0)
@@ -326,15 +322,10 @@ int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes,
 	int ret = 0;
 
 	/*
-	 * If we are a free space inode we need to not flush since we will be in
-	 * the middle of a transaction commit.  We also don't need the delalloc
-	 * mutex since we won't race with anybody.  We need this mostly to make
-	 * lockdep shut its filthy mouth.
-	 *
 	 * If we have a transaction open (can happen if we call truncate_block
 	 * from truncate), then we need FLUSH_LIMIT so we don't deadlock.
 	 */
-	if (noflush || btrfs_is_free_space_inode(inode)) {
+	if (noflush) {
 		flush = BTRFS_RESERVE_NO_FLUSH;
 	} else {
 		if (current->journal_info)
diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c
index db2ffab0941a6a..1f20148c1fd44e 100644
--- a/fs/btrfs/delayed-inode.c
+++ b/fs/btrfs/delayed-inode.c
@@ -6,6 +6,7 @@
 
 #include <linux/slab.h>
 #include <linux/iversion.h>
+#include <linux/error-injection.h>
 #include "ctree.h"
 #include "fs.h"
 #include "messages.h"
@@ -686,7 +687,7 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans,
 	/*
 	 * For delayed items to insert, we track reserved metadata bytes based
 	 * on the number of leaves that we will use.
-	 * See btrfs_insert_delayed_dir_index() and
+	 * See btrfs_insert_delayed_dir_index_prealloc() and
 	 * btrfs_delayed_item_reserve_metadata()).
 	 */
 	ASSERT(first_item->bytes_reserved == 0);
@@ -1469,35 +1470,93 @@ static void btrfs_release_dir_index_item_space(struct btrfs_trans_handle *trans)
 	trans->bytes_reserved -= bytes;
 }
 
-/* Will return 0, -ENOMEM or -EEXIST (index number collision, unexpected). */
-int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans,
-				   const char *name, int name_len,
-				   struct btrfs_inode *dir,
-				   const struct btrfs_disk_key *disk_key, u8 flags,
-				   u64 index)
+/*
+ * Pre-allocate a delayed node and delayed item for a dir index insertion and
+ * copy the name into the item. Call this before modifying the btree so that
+ * ENOMEM can be returned before any on-disk state has changed.
+ *
+ * The returned prealloc is consumed by either
+ * btrfs_insert_delayed_dir_index_prealloc() or
+ * btrfs_free_delayed_dir_index_prealloc(); it must not be used afterwards.
+ *
+ * Returns a prealloc on success, ERR_PTR on allocation failure.
+ */
+struct btrfs_dir_index_prealloc *btrfs_prealloc_delayed_dir_index(struct btrfs_inode *dir,
+								  const char *name,
+								  int name_len)
+{
+	struct btrfs_dir_index_prealloc *prealloc;
+	struct btrfs_delayed_node *node;
+	struct btrfs_delayed_item *item;
+
+	prealloc = kzalloc_obj(*prealloc, GFP_NOFS);
+	if (!prealloc)
+		return ERR_PTR(-ENOMEM);
+
+	node = btrfs_get_or_create_delayed_node(dir, &prealloc->tracker);
+	if (IS_ERR(node)) {
+		kfree(prealloc);
+		return ERR_CAST(node);
+	}
+
+	item = btrfs_alloc_delayed_item(sizeof(struct btrfs_dir_item) + name_len,
+					node, BTRFS_DELAYED_INSERTION_ITEM);
+	if (!item) {
+		btrfs_release_delayed_node(node, &prealloc->tracker);
+		kfree(prealloc);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	memcpy(item->data + sizeof(struct btrfs_dir_item), name, name_len);
+
+	prealloc->node = node;
+	prealloc->item = item;
+	return prealloc;
+}
+ALLOW_ERROR_INJECTION(btrfs_prealloc_delayed_dir_index, ERRNO);
+
+/*
+ * Free resources from btrfs_prealloc_delayed_dir_index() when the btree
+ * insertion failed and we will not commit the delayed dir index. Does nothing
+ * if @prealloc is NULL.
+ */
+void btrfs_free_delayed_dir_index_prealloc(struct btrfs_trans_handle *trans,
+					   struct btrfs_dir_index_prealloc *prealloc)
 {
+	if (!prealloc)
+		return;
+
+	btrfs_release_delayed_item(prealloc->item);
+	btrfs_release_dir_index_item_space(trans);
+	btrfs_release_delayed_node(prealloc->node, &prealloc->tracker);
+	kfree(prealloc);
+}
+
+/*
+ * Commit a pre-allocated delayed dir index item. @prealloc must have been
+ * returned by btrfs_prealloc_delayed_dir_index(). This populates the item,
+ * adds it to the delayed node's rb-tree, and reserves metadata space. It
+ * cannot fail with ENOMEM. @prealloc is freed here in all cases.
+ *
+ * Return 0 or -EEXIST (index number collision, unexpected).
+ */
+int btrfs_insert_delayed_dir_index_prealloc(struct btrfs_trans_handle *trans,
+					    struct btrfs_inode *dir,
+					    struct btrfs_dir_index_prealloc *prealloc,
+					    const struct btrfs_disk_key *disk_key,
+					    u8 flags, u64 index)
+{
+	struct btrfs_delayed_node *delayed_node = prealloc->node;
+	struct btrfs_ref_tracker *tracker = &prealloc->tracker;
+	struct btrfs_delayed_item *delayed_item = prealloc->item;
 	struct btrfs_fs_info *fs_info = trans->fs_info;
 	const unsigned int leaf_data_size = BTRFS_LEAF_DATA_SIZE(fs_info);
-	struct btrfs_delayed_node *delayed_node;
-	struct btrfs_ref_tracker delayed_node_tracker;
-	struct btrfs_delayed_item *delayed_item;
+	const int name_len = delayed_item->data_len - sizeof(struct btrfs_dir_item);
 	struct btrfs_dir_item *dir_item;
 	bool reserve_leaf_space;
 	u32 data_len;
 	int ret;
 
-	delayed_node = btrfs_get_or_create_delayed_node(dir, &delayed_node_tracker);
-	if (IS_ERR(delayed_node))
-		return PTR_ERR(delayed_node);
-
-	delayed_item = btrfs_alloc_delayed_item(sizeof(*dir_item) + name_len,
-						delayed_node,
-						BTRFS_DELAYED_INSERTION_ITEM);
-	if (!delayed_item) {
-		ret = -ENOMEM;
-		goto release_node;
-	}
-
 	delayed_item->index = index;
 
 	dir_item = (struct btrfs_dir_item *)delayed_item->data;
@@ -1506,7 +1565,7 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans,
 	btrfs_set_stack_dir_data_len(dir_item, 0);
 	btrfs_set_stack_dir_name_len(dir_item, name_len);
 	btrfs_set_stack_dir_flags(dir_item, flags);
-	memcpy((char *)(dir_item + 1), name, name_len);
+	/* Name was already copied by btrfs_prealloc_delayed_dir_index(). */
 
 	data_len = delayed_item->data_len + sizeof(struct btrfs_item);
 
@@ -1524,7 +1583,9 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans,
 	if (unlikely(ret)) {
 		btrfs_err(trans->fs_info,
 "error adding delayed dir index item, name: %.*s, index: %llu, root: %llu, dir: %llu, dir->index_cnt: %llu, delayed_node->index_cnt: %llu, error: %pe",
-			  name_len, name, index, btrfs_root_id(delayed_node->root),
+			  name_len,
+			  (const char *)(dir_item + 1),
+			  index, btrfs_root_id(delayed_node->root),
 			  delayed_node->inode_id, dir->index_cnt,
 			  delayed_node->index_cnt, ERR_PTR(ret));
 		btrfs_release_delayed_item(delayed_item);
@@ -1562,7 +1623,9 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans,
 	mutex_unlock(&delayed_node->mutex);
 
 release_node:
-	btrfs_release_delayed_node(delayed_node, &delayed_node_tracker);
+	/* Must release the node before freeing @tracker's containing struct. */
+	btrfs_release_delayed_node(delayed_node, tracker);
+	kfree(prealloc);
 	return ret;
 }
 
@@ -1581,7 +1644,7 @@ static bool btrfs_delete_delayed_insertion_item(struct btrfs_delayed_node *node,
 	/*
 	 * For delayed items to insert, we track reserved metadata bytes based
 	 * on the number of leaves that we will use.
-	 * See btrfs_insert_delayed_dir_index() and
+	 * See btrfs_insert_delayed_dir_index_prealloc() and
 	 * btrfs_delayed_item_reserve_metadata()).
 	 */
 	ASSERT(item->bytes_reserved == 0);
diff --git a/fs/btrfs/delayed-inode.h b/fs/btrfs/delayed-inode.h
index fc752863f89bcd..57ba96cfaf9cb3 100644
--- a/fs/btrfs/delayed-inode.h
+++ b/fs/btrfs/delayed-inode.h
@@ -115,11 +115,23 @@ struct btrfs_delayed_item {
 };
 
 void btrfs_init_delayed_root(struct btrfs_delayed_root *delayed_root);
-int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans,
-				   const char *name, int name_len,
-				   struct btrfs_inode *dir,
-				   const struct btrfs_disk_key *disk_key, u8 flags,
-				   u64 index);
+
+struct btrfs_dir_index_prealloc {
+	struct btrfs_delayed_node *node;
+	struct btrfs_ref_tracker tracker;
+	struct btrfs_delayed_item *item;
+};
+
+struct btrfs_dir_index_prealloc *btrfs_prealloc_delayed_dir_index(struct btrfs_inode *dir,
+								  const char *name,
+								  int name_len);
+void btrfs_free_delayed_dir_index_prealloc(struct btrfs_trans_handle *trans,
+					   struct btrfs_dir_index_prealloc *prealloc);
+int btrfs_insert_delayed_dir_index_prealloc(struct btrfs_trans_handle *trans,
+					    struct btrfs_inode *dir,
+					    struct btrfs_dir_index_prealloc *prealloc,
+					    const struct btrfs_disk_key *disk_key,
+					    u8 flags, u64 index);
 
 int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans,
 				   struct btrfs_inode *dir, u64 index);
diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c
index af1b898029e82a..cdfe093e5c5c4f 100644
--- a/fs/btrfs/dev-replace.c
+++ b/fs/btrfs/dev-replace.c
@@ -235,7 +235,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
 				  struct btrfs_device **device_out)
 {
 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
-	struct btrfs_device *device;
+	struct btrfs_device *device = NULL;
+	struct btrfs_device *tmp_device;
 	struct file *bdev_file;
 	struct block_device *bdev;
 	u64 devid = BTRFS_DEV_REPLACE_DEVID;
@@ -264,8 +265,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
 
 	sync_blockdev(bdev);
 
-	list_for_each_entry(device, &fs_devices->devices, dev_list) {
-		if (device->bdev == bdev) {
+	list_for_each_entry(tmp_device, &fs_devices->devices, dev_list) {
+		if (tmp_device->bdev == bdev) {
 			btrfs_err(fs_info,
 				  "target device is in the filesystem!");
 			ret = -EEXIST;
@@ -285,6 +286,7 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
 	device = btrfs_alloc_device(NULL, &devid, NULL, device_path);
 	if (IS_ERR(device)) {
 		ret = PTR_ERR(device);
+		device = NULL;
 		goto error;
 	}
 
@@ -328,6 +330,8 @@ static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info,
 
 error:
 	/* Undo the open-time freeze deny. */
+	if (device)
+		btrfs_free_device(device);
 	btrfs_release_device_allow_freeze(bdev_file);
 	return ret;
 }
@@ -494,6 +498,7 @@ static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info,
 	path->reada = READA_FORWARD;
 	path->search_commit_root = true;
 	path->skip_locking = true;
+	path->need_commit_sem = true;
 
 	key.objectid = src_dev->devid;
 	key.type = BTRFS_DEV_EXTENT_KEY;
diff --git a/fs/btrfs/dir-item.c b/fs/btrfs/dir-item.c
index 84f1c64423d328..3a90736915af6f 100644
--- a/fs/btrfs/dir-item.c
+++ b/fs/btrfs/dir-item.c
@@ -106,8 +106,11 @@ int btrfs_insert_xattr_item(struct btrfs_trans_handle *trans,
  * Will return 0 or -ENOMEM
  */
 int btrfs_insert_dir_item(struct btrfs_trans_handle *trans,
-			  const struct fscrypt_str *name, struct btrfs_inode *dir,
-			  const struct btrfs_key *location, u8 type, u64 index)
+			  const struct fscrypt_str *name,
+			  struct btrfs_inode *dir,
+			  const struct btrfs_key *location, u8 type,
+			  u64 index,
+			  struct btrfs_dir_index_prealloc *prealloc)
 {
 	int ret = 0;
 	int ret2 = 0;
@@ -119,17 +122,27 @@ int btrfs_insert_dir_item(struct btrfs_trans_handle *trans,
 	struct btrfs_key key;
 	struct btrfs_disk_key disk_key;
 	u32 data_size;
+	const bool need_delayed_index = (root != root->fs_info->tree_root);
 
 	key.objectid = btrfs_ino(dir);
 	key.type = BTRFS_DIR_ITEM_KEY;
 	key.offset = btrfs_name_hash(name->name, name->len);
 
 	path = btrfs_alloc_path();
-	if (!path)
-		return -ENOMEM;
+	if (!path) {
+		ret = -ENOMEM;
+		goto out_free_prealloc;
+	}
 
 	btrfs_cpu_key_to_disk(&disk_key, location);
 
+	/* Pre-allocate the delayed dir index before modifying the btree. */
+	if (need_delayed_index && !prealloc) {
+		prealloc = btrfs_prealloc_delayed_dir_index(dir, name->name, name->len);
+		if (IS_ERR(prealloc))
+			return PTR_ERR(prealloc);
+	}
+
 	data_size = sizeof(*dir_item) + name->len;
 	dir_item = insert_with_overflow(trans, root, path, &key, data_size,
 					name->name, name->len);
@@ -137,7 +150,7 @@ int btrfs_insert_dir_item(struct btrfs_trans_handle *trans,
 		ret = PTR_ERR(dir_item);
 		if (ret == -EEXIST)
 			goto second_insert;
-		goto out_free;
+		goto out_free_prealloc;
 	}
 
 	if (IS_ENCRYPTED(&dir->vfs_inode))
@@ -154,21 +167,21 @@ int btrfs_insert_dir_item(struct btrfs_trans_handle *trans,
 	write_extent_buffer(leaf, name->name, name_ptr, name->len);
 
 second_insert:
-	/* FIXME, use some real flag for selecting the extra index */
-	if (root == root->fs_info->tree_root) {
+	if (!need_delayed_index) {
 		ret = 0;
-		goto out_free;
+		goto out_free_prealloc;
 	}
 	btrfs_release_path(path);
 
-	ret2 = btrfs_insert_delayed_dir_index(trans, name->name, name->len, dir,
-					      &disk_key, type, index);
-out_free:
+	ret2 = btrfs_insert_delayed_dir_index_prealloc(trans, dir, prealloc,
+						       &disk_key, type, index);
 	if (ret)
 		return ret;
-	if (ret2)
-		return ret2;
-	return 0;
+	return ret2;
+
+out_free_prealloc:
+	btrfs_free_delayed_dir_index_prealloc(trans, prealloc);
+	return ret;
 }
 
 static struct btrfs_dir_item *btrfs_lookup_match_dir(
diff --git a/fs/btrfs/dir-item.h b/fs/btrfs/dir-item.h
index e52174a8baf92c..8d22976a0a7712 100644
--- a/fs/btrfs/dir-item.h
+++ b/fs/btrfs/dir-item.h
@@ -13,12 +13,14 @@ struct btrfs_path;
 struct btrfs_inode;
 struct btrfs_root;
 struct btrfs_trans_handle;
+struct btrfs_dir_index_prealloc;
 
 int btrfs_check_dir_item_collision(struct btrfs_root *root, u64 dir_ino,
 			  const struct fscrypt_str *name);
 int btrfs_insert_dir_item(struct btrfs_trans_handle *trans,
 			  const struct fscrypt_str *name, struct btrfs_inode *dir,
-			  const struct btrfs_key *location, u8 type, u64 index);
+			  const struct btrfs_key *location, u8 type, u64 index,
+			  struct btrfs_dir_index_prealloc *prealloc);
 struct btrfs_dir_item *btrfs_lookup_dir_item(struct btrfs_trans_handle *trans,
 					     struct btrfs_root *root,
 					     struct btrfs_path *path, u64 dir,
@@ -53,5 +55,4 @@ static inline u64 btrfs_name_hash(const char *name, int len)
 {
        return crc32c((u32)~1, name, len);
 }
-
 #endif
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 819727460bcf4d..7fd8babd74a61e 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -125,15 +125,13 @@ int btrfs_buffer_uptodate(struct extent_buffer *eb, u64 parent_transid,
 		return 1;
 	}
 
-	if (btrfs_header_generation(eb) != parent_transid) {
-		btrfs_err_rl(eb->fs_info,
+	btrfs_err_rl(eb->fs_info,
 "parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
-			eb->start, eb->read_mirror,
-			parent_transid, btrfs_header_generation(eb));
-		clear_extent_buffer_uptodate(eb);
-		return 0;
-	}
-	return 1;
+		     eb->start, eb->read_mirror,
+		     parent_transid, btrfs_header_generation(eb));
+	clear_extent_buffer_uptodate(eb);
+
+	return 0;
 }
 
 static bool btrfs_supported_super_csum(u16 csum_type)
@@ -176,19 +174,24 @@ static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb,
 				      int mirror_num)
 {
 	struct btrfs_fs_info *fs_info = eb->fs_info;
-	const u32 step = min(fs_info->nodesize, PAGE_SIZE);
-	const u32 nr_steps = eb->len / step;
-	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
+	struct btrfs_bio *bbio;
+	int ret;
 
 	if (sb_rdonly(fs_info->sb))
 		return -EROFS;
 
+	/*
+	 * This bbio is only to queue all pages for btrfs_repair_bbio_failure().
+	 * Thus it will never get its endio called.
+	 */
+	bbio = btrfs_bio_alloc(max(1, fs_info->nodesize >> PAGE_SHIFT), REQ_OP_READ,
+			       BTRFS_I(fs_info->btree_inode), eb->start, NULL, NULL);
+	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
 	for (int i = 0; i < num_extent_pages(eb); i++) {
 		struct folio *folio = eb->folios[i];
 
 		/* No large folio support yet. */
 		ASSERT(folio_order(folio) == 0);
-		ASSERT(i < nr_steps);
 
 		/*
 		 * For nodesize < page size, there is just one paddr, with some
@@ -197,11 +200,17 @@ static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb,
 		 * For nodesize >= page size, it's one or more paddrs, and eb->start
 		 * must be aligned to page boundary.
 		 */
-		paddrs[i] = page_to_phys(&folio->page) + offset_in_page(eb->start);
+		ret = bio_add_page(&bbio->bio, &folio->page, min(PAGE_SIZE, fs_info->nodesize),
+				   offset_in_page(eb->start));
+		ASSERT(ret == min(PAGE_SIZE, fs_info->nodesize));
 	}
+	/* Since the bbio is never submitted, we have to save the iter manually. */
+	bbio->saved_iter = bbio->bio.bi_iter;
 
-	return btrfs_repair_io_failure(fs_info, 0, eb->start, eb->len,
-				       eb->start, paddrs, step, mirror_num);
+	ret = btrfs_repair_bbio_failure(bbio, &bbio->saved_iter, fs_info->nodesize,
+					mirror_num);
+	bio_put(&bbio->bio);
+	return ret;
 }
 
 /*
@@ -1485,7 +1494,9 @@ static int cleaner_kthread(void *arg)
 
 		btrfs_run_delayed_iputs(fs_info);
 
+		set_bit(BTRFS_QGROUP_RUNTIME_BIT_REJECT_RESCAN, &fs_info->qgroup_flags);
 		again = btrfs_clean_one_deleted_snapshot(fs_info);
+		clear_bit(BTRFS_QGROUP_RUNTIME_BIT_REJECT_RESCAN, &fs_info->qgroup_flags);
 		mutex_unlock(&fs_info->cleaner_mutex);
 
 		/*
@@ -1769,7 +1780,6 @@ static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
 	if (fs_info->rmw_workers)
 		destroy_workqueue(fs_info->rmw_workers);
 	btrfs_destroy_workqueue(fs_info->endio_write_workers);
-	btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
 	btrfs_destroy_workqueue(fs_info->delayed_workers);
 	btrfs_destroy_workqueue(fs_info->caching_workers);
 	btrfs_destroy_workqueue(fs_info->flush_workers);
@@ -1980,9 +1990,6 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info)
 	fs_info->endio_write_workers =
 		btrfs_alloc_workqueue(fs_info, "endio-write", flags,
 				      max_active, 2);
-	fs_info->endio_freespace_worker =
-		btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
-				      max_active, 0);
 	fs_info->delayed_workers =
 		btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
 				      max_active, 0);
@@ -1995,8 +2002,7 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info)
 	if (!(fs_info->workers &&
 	      fs_info->delalloc_workers && fs_info->flush_workers &&
 	      fs_info->endio_workers && fs_info->endio_meta_workers &&
-	      fs_info->endio_write_workers &&
-	      fs_info->endio_freespace_worker && fs_info->rmw_workers &&
+	      fs_info->endio_write_workers && fs_info->rmw_workers &&
 	      fs_info->caching_workers && fs_info->fixup_workers &&
 	      fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&
 	      fs_info->discard_ctl.discard_workers)) {
@@ -2357,6 +2363,10 @@ static int validate_sys_chunk_array(const struct btrfs_fs_info *fs_info,
 				  key.type, cur);
 			return -EUCLEAN;
 		}
+
+		if (unlikely(cur + sizeof(*chunk) > sys_array_size))
+			goto short_read;
+
 		chunk = (struct btrfs_chunk *)(sb->sys_chunk_array + cur);
 		num_stripes = btrfs_stack_chunk_num_stripes(chunk);
 		if (unlikely(cur + btrfs_chunk_item_size(num_stripes) > sys_array_size))
@@ -2370,7 +2380,7 @@ static int validate_sys_chunk_array(const struct btrfs_fs_info *fs_info,
 		}
 		ret = btrfs_check_chunk_valid(fs_info, NULL, chunk, key.offset,
 					      sectorsize);
-		if (ret < 0)
+		if (unlikely(ret < 0))
 			return ret;
 		cur += btrfs_chunk_item_size(num_stripes);
 	}
@@ -3059,7 +3069,6 @@ static int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
 int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info)
 {
 	int ret;
-	const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE);
 	bool rebuild_free_space_tree = false;
 
 	if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
@@ -3154,8 +3163,8 @@ int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info)
 		}
 	}
 
-	if (cache_opt != btrfs_free_space_cache_v1_active(fs_info)) {
-		ret = btrfs_set_free_space_cache_v1_active(fs_info, cache_opt);
+	if (btrfs_free_space_cache_v1_active(fs_info)) {
+		ret = btrfs_cleanup_free_space_cache_v1(fs_info);
 		if (ret)
 			return ret;
 	}
@@ -3271,20 +3280,6 @@ int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount)
 		return -EINVAL;
 	}
 
-	/*
-	 * Subpage/bs > ps runtime limitation on v1 cache.
-	 *
-	 * V1 space cache still has some hard coded PAGE_SIZE usage, while
-	 * we're already defaulting to v2 cache, no need to bother v1 as it's
-	 * going to be deprecated anyway.
-	 */
-	if (fs_info->sectorsize != PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) {
-		btrfs_warn(fs_info,
-	"v1 space cache is not supported for page size %lu with sectorsize %u",
-			   PAGE_SIZE, fs_info->sectorsize);
-		return -EINVAL;
-	}
-
 	/* This can be called by remount, we need to protect the super block. */
 	spin_lock(&fs_info->super_lock);
 	btrfs_set_super_incompat_flags(disk_super, incompat);
@@ -4213,8 +4208,6 @@ int write_all_supers(struct btrfs_trans_handle *trans)
 			total_errors++;
 	}
 	if (unlikely(total_errors > max_errors)) {
-		btrfs_err(fs_info, "%d errors while writing supers",
-			  total_errors);
 		mutex_unlock(&fs_info->fs_devices->device_list_mutex);
 
 		/* FUA is masked off if unsupported and can't be the reason */
@@ -4442,9 +4435,8 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info)
 	 * to finish an ordered extent - end_bbio_compressed_write()
 	 * calls btrfs_finish_ordered_extent() which in turns does a call to
 	 * btrfs_queue_ordered_fn(), and that queues the ordered extent
-	 * completion either in the endio_write_workers work queue or in the
-	 * fs_info->endio_freespace_worker work queue. We flush those queues
-	 * below, so before we flush them we must flush this queue for the
+	 * completion in the endio_write_workers work queue. We flush that
+	 * queue below, so before we flush it we must flush this queue for the
 	 * workers of compressed writes.
 	 */
 	flush_workqueue(fs_info->endio_workers);
@@ -4470,8 +4462,6 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info)
 	 * btrfs_finish_ordered_io() when we are unmounting).
 	 */
 	btrfs_flush_workqueue(fs_info->endio_write_workers);
-	/* Ordered extents for free space inodes. */
-	btrfs_flush_workqueue(fs_info->endio_freespace_worker);
 	/*
 	 * Run delayed iputs in case an async reclaim worker is waiting for them
 	 * to be run as mentioned above.
@@ -4860,26 +4850,6 @@ static void btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
 	}
 }
 
-static void btrfs_cleanup_bg_io(struct btrfs_block_group *cache)
-{
-	struct inode *inode;
-
-	inode = cache->io_ctl.inode;
-	if (inode) {
-		unsigned int nofs_flag;
-
-		nofs_flag = memalloc_nofs_save();
-		invalidate_inode_pages2(inode->i_mapping);
-		memalloc_nofs_restore(nofs_flag);
-
-		BTRFS_I(inode)->generation = 0;
-		cache->io_ctl.inode = NULL;
-		iput(inode);
-	}
-	ASSERT(cache->io_ctl.pages == NULL);
-	btrfs_put_block_group(cache);
-}
-
 void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
 			     struct btrfs_fs_info *fs_info)
 {
@@ -4891,40 +4861,13 @@ void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
 					 struct btrfs_block_group,
 					 dirty_list);
 
-		if (!list_empty(&cache->io_list)) {
-			spin_unlock(&cur_trans->dirty_bgs_lock);
-			list_del_init(&cache->io_list);
-			btrfs_cleanup_bg_io(cache);
-			spin_lock(&cur_trans->dirty_bgs_lock);
-		}
-
 		list_del_init(&cache->dirty_list);
-		spin_lock(&cache->lock);
-		cache->disk_cache_state = BTRFS_DC_ERROR;
-		spin_unlock(&cache->lock);
-
 		spin_unlock(&cur_trans->dirty_bgs_lock);
 		btrfs_put_block_group(cache);
 		btrfs_dec_delayed_refs_rsv_bg_updates(fs_info);
 		spin_lock(&cur_trans->dirty_bgs_lock);
 	}
 	spin_unlock(&cur_trans->dirty_bgs_lock);
-
-	/*
-	 * Refer to the definition of io_bgs member for details why it's safe
-	 * to use it without any locking
-	 */
-	while (!list_empty(&cur_trans->io_bgs)) {
-		cache = list_first_entry(&cur_trans->io_bgs,
-					 struct btrfs_block_group,
-					 io_list);
-
-		list_del_init(&cache->io_list);
-		spin_lock(&cache->lock);
-		cache->disk_cache_state = BTRFS_DC_ERROR;
-		spin_unlock(&cache->lock);
-		btrfs_cleanup_bg_io(cache);
-	}
 }
 
 static void btrfs_free_all_qgroup_pertrans(struct btrfs_fs_info *fs_info)
@@ -4960,7 +4903,6 @@ void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans)
 
 	btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
 	ASSERT(list_empty(&cur_trans->dirty_bgs));
-	ASSERT(list_empty(&cur_trans->io_bgs));
 
 	list_for_each_entry_safe(dev, tmp, &cur_trans->dev_update_list,
 				 post_commit_list) {
diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c
index d6df11f6088c44..992b8b42bdb4e3 100644
--- a/fs/btrfs/extent-io-tree.c
+++ b/fs/btrfs/extent-io-tree.c
@@ -751,7 +751,7 @@ int btrfs_clear_extent_bit_changeset(struct extent_io_tree *tree, u64 start, u64
 				btrfs_split_delalloc_extent(tree->inode, state, start);
 
 			/*
-			 * Temporarilly ajdust this state's range to match the
+			 * Temporarily ajdust this state's range to match the
 			 * range for which we are clearing bits.
 			 */
 			state->start = start;
diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
index d6a4390ee34ac9..a0d5ab03aae264 100644
--- a/fs/btrfs/extent-tree.c
+++ b/fs/btrfs/extent-tree.c
@@ -6315,6 +6315,16 @@ int btrfs_drop_snapshot(struct btrfs_root *root, bool update_ref, bool for_reloc
 	set_bit(BTRFS_ROOT_DELETING, &root->state);
 	unfinished_drop = test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state);
 
+	/*
+	 * For subvolume dropping, check if the subvolume is large enough so
+	 * that we need to mark qgroup inconsistent to avoid long qgroup stall.
+	 *
+	 * Even for a subvolume without any snapshot, there can still be
+	 * a lot of qgroup records queued into one transaction.
+	 */
+	if (!for_reloc)
+		btrfs_qgroup_check_tree_drop(fs_info, rootid,
+					     btrfs_header_level(root->node));
 	if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
 		level = btrfs_header_level(root->node);
 		path->nodes[level] = btrfs_lock_root_node(root);
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index d7600e5fa3d95d..55e9144d47595f 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -110,14 +110,14 @@ struct btrfs_bio_ctrl {
 	 * make the decision when submitting the bio.
 	 *
 	 * The pattern between do_readpage(), submit_one_bio() and
-	 * submit_extent_folio() is quite subtle, so tracking this is tricky.
+	 * submit_one_block() is quite subtle, so tracking this is tricky.
 	 *
 	 * As we process extent E, we might submit a bio with existing built up
 	 * extents before adding E to a new bio, or we might just add E to the
 	 * bio. As a result, E's generation could apply to the current bio or
 	 * to the next one, so we need to be careful to update the bio_ctrl's
 	 * generation with E's only when we are sure E is added to bio_ctrl->bbio
-	 * in submit_extent_folio().
+	 * in submit_one_block().
 	 *
 	 * See the comment in btrfs_lookup_bio_sums() for more detail on the
 	 * need for this optimization.
@@ -797,118 +797,95 @@ static int alloc_new_bio(struct btrfs_inode *inode,
 }
 
 /*
- * @disk_bytenr: logical bytenr where the write will be
- * @page:	page to add to the bio
- * @size:	portion of page that we want to write to
- * @pg_offset:	offset of the new bio or to check whether we are adding
- *              a contiguous page to the previous one
+ * @disk_bytenr: logical bytenr where the read/write will be
+ * @folio:	 the folio the block belongs to
+ * @pg_offset:	 the offset inside the folio
  * @read_em_generation: generation of the extent_map we are submitting
  *			(only used for read)
  *
- * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
+ * This will either add the block into the existing @bio_ctrl->bbio, or allocate a
  * new one in @bio_ctrl->bbio.
  * The mirror number for this IO should already be initialized in
  * @bio_ctrl->mirror_num.
  *
- * Return the number of bytes that are queued into a bio.
- * If the returned bytes is smaller than @size, it means we hit a critical error
- * for data write, where there is no ordered extent for the range.
+ * Return 0 if the block is queued or submitted.
+ * Return <0 for error.
  */
-static unsigned int submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl,
-					u64 disk_bytenr, struct folio *folio,
-					size_t size, unsigned long pg_offset,
-					u64 read_em_generation)
+static int submit_one_block(struct btrfs_bio_ctrl *bio_ctrl,
+			    u64 disk_bytenr, struct folio *folio,
+			    unsigned long pg_offset, u64 read_em_generation)
 {
 	struct btrfs_inode *inode = folio_to_inode(folio);
+	const struct btrfs_fs_info *fs_info = inode->root->fs_info;
+	const u32 blocksize = fs_info->sectorsize;
 	loff_t file_offset = folio_pos(folio) + pg_offset;
-	unsigned int queued = 0;
 
-	ASSERT(pg_offset + size <= folio_size(folio));
+	ASSERT(pg_offset + blocksize <= folio_size(folio));
 	ASSERT(bio_ctrl->end_io_func);
 
 	if (bio_ctrl->bbio &&
 	    !btrfs_bio_is_contig(bio_ctrl, disk_bytenr, file_offset))
 		submit_one_bio(bio_ctrl);
 
-	do {
-		u32 len = size;
-
-		/* Allocate new bio if needed */
-		if (!bio_ctrl->bbio) {
-			int ret;
-
-			ret = alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset);
-			if (ret < 0)
-				break;
-		}
-
-		/* Cap to the current ordered extent boundary if there is one. */
-		if (len > bio_ctrl->len_to_oe_boundary) {
-			ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
-			ASSERT(is_data_inode(inode));
-			len = bio_ctrl->len_to_oe_boundary;
-		}
+again:
+	/* Allocate new bio if needed */
+	if (!bio_ctrl->bbio) {
+		int ret;
 
-		if (!bio_add_folio(&bio_ctrl->bbio->bio, folio, len, pg_offset)) {
-			/* bio full: move on to a new one */
-			submit_one_bio(bio_ctrl);
-			continue;
-		}
-		/*
-		 * Now that the folio is definitely added to the bio, include its
-		 * generation in the max generation calculation.
-		 */
-		bio_ctrl->generation = max(bio_ctrl->generation, read_em_generation);
-		bio_ctrl->next_file_offset += len;
+		ret = alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset);
+		if (ret < 0)
+			return ret;
+	}
 
-		if (bio_ctrl->wbc)
-			wbc_account_cgroup_owner(bio_ctrl->wbc, folio, len);
+	if (!bio_add_folio(&bio_ctrl->bbio->bio, folio, blocksize, pg_offset)) {
+		/* bio full: move on to a new one */
+		submit_one_bio(bio_ctrl);
+		goto again;
+	}
 
-		size -= len;
-		pg_offset += len;
-		disk_bytenr += len;
-		file_offset += len;
-		queued += len;
+	/*
+	 * Now that the folio is definitely added to the bio, include its
+	 * generation in the max generation calculation.
+	 */
+	bio_ctrl->generation = max(bio_ctrl->generation, read_em_generation);
+	bio_ctrl->next_file_offset += blocksize;
 
-		/*
-		 * len_to_oe_boundary defaults to U32_MAX, which isn't folio or
-		 * sector aligned.  alloc_new_bio() then sets it to the end of
-		 * our ordered extent for writes into zoned devices.
-		 *
-		 * When len_to_oe_boundary is tracking an ordered extent, we
-		 * trust the ordered extent code to align things properly, and
-		 * the check above to cap our write to the ordered extent
-		 * boundary is correct.
-		 *
-		 * When len_to_oe_boundary is U32_MAX, the cap above would
-		 * result in a 4095 byte IO for the last folio right before
-		 * we hit the bio limit of UINT_MAX.  bio_add_folio() has all
-		 * the checks required to make sure we don't overflow the bio,
-		 * and we should just ignore len_to_oe_boundary completely
-		 * unless we're using it to track an ordered extent.
-		 *
-		 * It's pretty hard to make a bio sized U32_MAX, but it can
-		 * happen when the page cache is able to feed us contiguous
-		 * folios for large extents.
-		 */
-		if (bio_ctrl->len_to_oe_boundary != U32_MAX)
-			bio_ctrl->len_to_oe_boundary -= len;
+	if (bio_ctrl->wbc)
+		wbc_account_cgroup_owner(bio_ctrl->wbc, folio, blocksize);
 
-		/* Ordered extent boundary: move on to a new bio. */
-		if (bio_ctrl->len_to_oe_boundary == 0)
-			submit_one_bio(bio_ctrl);
-		/*
-		 * If we have accumulated decent amount of IO, send it to the
-		 * block layer so that IO can run while we are accumulating
-		 * more folios to write.
-		 */
-		else if (bio_ctrl->wbc &&
-			 bio_ctrl->bbio->bio.bi_iter.bi_size >=
-			    inode->root->fs_info->writeback_bio_size)
-			submit_one_bio(bio_ctrl);
+	/*
+	 * len_to_oe_boundary defaults to U32_MAX, which isn't folio or sector
+	 * aligned.  alloc_new_bio() then sets it to the end of our ordered
+	 * extent for writes into zoned devices.
+	 *
+	 * When len_to_oe_boundary is tracking an ordered extent, the
+	 * len_to_oe_boundary should follow that OE and never go beyond the max
+	 * extent size (128MiB).
+	 *
+	 * When len_to_oe_boundary is U32_MAX, decreasing the length by
+	 * blocksize will never make it reach 0, thus skipping the later
+	 * submit_one_bio() call.  So if len_to_oe_boundary() is not tracking
+	 * an OE, do not decrease it.
+	 *
+	 * It's pretty hard to make a bio sized U32_MAX, but it can happen when
+	 * the page cache is able to feed us contiguous folios for large
+	 * extents.
+	 */
+	if (bio_ctrl->len_to_oe_boundary != U32_MAX)
+		bio_ctrl->len_to_oe_boundary -= blocksize;
 
-	} while (size);
-	return queued;
+	/* Ordered extent boundary: move on to a new bio. */
+	if (bio_ctrl->len_to_oe_boundary == 0)
+		submit_one_bio(bio_ctrl);
+	/*
+	 * If we have accumulated decent amount of IO, send it to the block
+	 * layer so that IO can run while we are accumulating more folios to
+	 * write.
+	 */
+	else if (bio_ctrl->wbc &&
+		 bio_ctrl->bbio->bio.bi_iter.bi_size >= fs_info->writeback_bio_size)
+		submit_one_bio(bio_ctrl);
+	return 0;
 }
 
 static int attach_extent_buffer_folio(struct extent_buffer *eb,
@@ -1092,7 +1069,6 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
 		u64 disk_bytenr;
 		u64 block_start;
 		u64 em_gen;
-		unsigned int queued;
 
 		ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
 		if (cur >= last_byte) {
@@ -1206,10 +1182,9 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
 
 		if (force_bio_submit)
 			submit_one_bio(bio_ctrl);
-		queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize,
-					     pg_offset, em_gen);
+		ret = submit_one_block(bio_ctrl, disk_bytenr, folio, pg_offset, em_gen);
 		/* Read submission should not fail. */
-		ASSERT(queued == blocksize);
+		ASSERT(ret == 0);
 	}
 	return 0;
 }
@@ -1808,33 +1783,51 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
 	return 0;
 }
 
+static struct btrfs_ordered_extent *get_oe_from_bbio(const struct btrfs_bio *bbio,
+						     u64 filepos)
+{
+	struct btrfs_ordered_extent *oe;
+
+	if (!bbio || !bbio->ordered)
+		return NULL;
+
+	oe = bbio->ordered;
+	if (!in_range(filepos, oe->file_offset, oe->num_bytes))
+		return NULL;
+
+	refcount_inc(&oe->refs);
+	return oe;
+}
+
 /*
  * Return 0 if we have submitted or queued the sector for submission.
  * Return <0 for critical errors, and the involved sector will be cleaned up.
  *
  * Caller should make sure filepos < i_size and handle filepos >= i_size case.
  */
-static int submit_one_sector(struct btrfs_inode *inode,
-			     struct folio *folio,
-			     u64 filepos, struct btrfs_bio_ctrl *bio_ctrl,
-			     loff_t i_size)
+static int submit_write_sector(struct btrfs_inode *inode,
+			       struct folio *folio,
+			       u64 filepos, struct btrfs_bio_ctrl *bio_ctrl,
+			       loff_t i_size)
 {
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	struct extent_map *em;
+	struct btrfs_ordered_extent *oe;
 	u64 block_start;
 	u64 disk_bytenr;
 	u64 extent_offset;
-	u64 em_end;
 	const u32 sectorsize = fs_info->sectorsize;
-	unsigned int queued;
+	int ret;
 
 	ASSERT(IS_ALIGNED(filepos, sectorsize));
 
 	/* @filepos >= i_size case should be handled by the caller. */
 	ASSERT(filepos < i_size);
 
-	em = btrfs_get_extent(inode, NULL, filepos, sectorsize);
-	if (IS_ERR(em)) {
+	/* Try to reuse the existing OE from bbio first. */
+	oe = get_oe_from_bbio(bio_ctrl->bbio, filepos);
+	if (!oe)
+		oe = btrfs_lookup_ordered_extent(inode, filepos);
+	if (unlikely(!oe)) {
 		/*
 		 * bio_ctrl may contain a bio crossing several folios.
 		 * Submit it immediately so that the bio has a chance
@@ -1857,31 +1850,25 @@ static int submit_one_sector(struct btrfs_inode *inode,
 		 */
 		btrfs_mark_ordered_io_finished(inode, filepos, fs_info->sectorsize,
 					       false);
-		return PTR_ERR(em);
+		btrfs_err_rl(fs_info,
+		"no ordered extent for root %lld ino %llu filepos %llu",
+			     btrfs_root_id(inode->root), btrfs_ino(inode),
+			     filepos);
+		return -EUCLEAN;
 	}
 
-	extent_offset = filepos - em->start;
-	em_end = btrfs_extent_map_end(em);
-	ASSERT(filepos <= em_end);
-	ASSERT(IS_ALIGNED(em->start, sectorsize));
-	ASSERT(IS_ALIGNED(em->len, sectorsize));
+	extent_offset = filepos - oe->file_offset;
+	ASSERT(filepos < oe->file_offset + oe->num_bytes);
+	ASSERT(IS_ALIGNED(oe->file_offset, sectorsize));
+	ASSERT(IS_ALIGNED(oe->num_bytes, sectorsize));
+	ASSERT(oe->compress_type == BTRFS_COMPRESS_NONE);
+	ASSERT(!test_bit(BTRFS_ORDERED_COMPRESSED, &oe->flags));
 
-	block_start = btrfs_extent_map_block_start(em);
-	disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset;
+	block_start = oe->disk_bytenr + oe->offset;
+	disk_bytenr = block_start + extent_offset;
 
-	ASSERT(!btrfs_extent_map_is_compressed(em));
-	ASSERT(block_start != EXTENT_MAP_HOLE);
-	ASSERT(block_start != EXTENT_MAP_INLINE);
+	btrfs_put_ordered_extent(oe);
 
-	btrfs_free_extent_map(em);
-	em = NULL;
-
-	/*
-	 * Although the PageDirty bit is cleared before entering this
-	 * function, subpage dirty bit is not cleared.
-	 * So clear subpage dirty bit here so next time we won't submit
-	 * a folio for a range already written to disk.
-	 */
 	btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize);
 	btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize);
 	/*
@@ -1892,13 +1879,17 @@ static int submit_one_sector(struct btrfs_inode *inode,
 	 */
 	ASSERT(folio_test_writeback(folio));
 
-	queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio,
-				     sectorsize, filepos - folio_pos(folio), 0);
-	if (unlikely(queued < sectorsize)) {
+	ret = submit_one_block(bio_ctrl, disk_bytenr, folio,
+			       offset_in_folio(folio, filepos), 0);
+	if (unlikely(ret < 0)) {
 		btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize);
 		btrfs_mark_ordered_io_finished(inode, filepos, fs_info->sectorsize,
 					       false);
-		return -EUCLEAN;
+		btrfs_err_rl(fs_info,
+		"failed to queue sector for root %lld ino %llu filepos %llu: %pe",
+			     btrfs_root_id(inode->root),
+			     btrfs_ino(inode), filepos, ERR_PTR(ret));
+		return ret;
 	}
 	return 0;
 }
@@ -1983,7 +1974,7 @@ static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
 			btrfs_folio_clear_dirty(fs_info, folio, cur, fs_info->sectorsize);
 			continue;
 		}
-		ret = submit_one_sector(inode, folio, cur, bio_ctrl, i_size);
+		ret = submit_write_sector(inode, folio, cur, bio_ctrl, i_size);
 		if (unlikely(ret < 0)) {
 			if (!found_error)
 				found_error = ret;
diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c
index 6ad7b39ae358b2..86d9c6f5ff4bd2 100644
--- a/fs/btrfs/extent_map.c
+++ b/fs/btrfs/extent_map.c
@@ -1219,6 +1219,14 @@ static struct btrfs_inode *find_first_inode_to_shrink(struct btrfs_root *root,
 
 		tree = &inode->extent_tree;
 
+		/*
+		 * Most inodes have no extent maps, so check without the lock.
+		 * The race is harmless: a false empty just defers the inode to
+		 * a later scan, and a false non-empty is caught under the lock.
+		 */
+		if (data_race(RB_EMPTY_ROOT(&tree->root)))
+			goto next;
+
 		/*
 		 * We want to be fast so if the lock is busy we don't want to
 		 * spend time waiting for it (some task is about to do IO for
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index cf50fd623f41a8..ff8f8cad00fc0d 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -396,17 +396,6 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio)
 	if (nblocks > fs_info->csums_per_leaf)
 		path->reada = READA_FORWARD;
 
-	/*
-	 * the free space stuff is only read when it hasn't been
-	 * updated in the current transaction.  So, we can safely
-	 * read from the commit root and sidestep a nasty deadlock
-	 * between reading the free space cache and updating the csum tree.
-	 */
-	if (btrfs_is_free_space_inode(inode)) {
-		path->search_commit_root = true;
-		path->skip_locking = true;
-	}
-
 	/*
 	 * If we are searching for a csum of an extent from a past
 	 * transaction, we can search in the commit root and reduce
@@ -797,29 +786,20 @@ int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path,
 	return ret;
 }
 
-static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src)
+static void csum_one_bio(struct btrfs_bio *bbio)
 {
 	struct btrfs_inode *inode = bbio->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	struct bio *bio = &bbio->bio;
 	struct btrfs_ordered_sum *sums = bbio->sums;
-	struct bvec_iter iter = *src;
-	phys_addr_t paddr;
 	const u32 blocksize = fs_info->sectorsize;
-	const u32 step = min(blocksize, PAGE_SIZE);
-	const u32 nr_steps = blocksize / step;
-	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
-	u32 offset = 0;
 	int index = 0;
 
-	btrfs_bio_for_each_block(paddr, bio, &iter, step) {
-		paddrs[(offset / step) % nr_steps] = paddr;
-		offset += step;
+	for (struct bvec_iter *iter = &bbio->csum_saved_iter;
+	     iter->bi_size;
+	     bio_advance_iter(&bbio->bio, iter, blocksize)) {
+		btrfs_csum_one_bio_block(fs_info, &bbio->bio, iter, sums->sums + index);
 
-		if (IS_ALIGNED(offset, blocksize)) {
-			btrfs_calculate_block_csum_pages(fs_info, paddrs, sums->sums + index);
-			index += fs_info->csum_size;
-		}
+		index += fs_info->csum_size;
 	}
 }
 
@@ -828,9 +808,8 @@ static void csum_one_bio_work(struct work_struct *work)
 	struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, csum_work);
 
 	ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE);
-	ASSERT(bbio->async_csum == true);
-	csum_one_bio(bbio, &bbio->csum_saved_iter);
-	complete(&bbio->csum_done);
+	csum_one_bio(bbio);
+	bio_endio(&bbio->bio);
 }
 
 /*
@@ -859,15 +838,14 @@ int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async)
 	bbio->sums = sums;
 	btrfs_add_ordered_sum(ordered, sums);
 
+	bbio->csum_saved_iter = bio->bi_iter;
 	if (!async) {
-		csum_one_bio(bbio, &bbio->bio.bi_iter);
+		csum_one_bio(bbio);
 		return 0;
 	}
-	init_completion(&bbio->csum_done);
-	bbio->async_csum = true;
-	bbio->csum_saved_iter = bbio->bio.bi_iter;
+	bio_inc_remaining(bio);
 	INIT_WORK(&bbio->csum_work, csum_one_bio_work);
-	schedule_work(&bbio->csum_work);
+	queue_work(fs_info->endio_workers, &bbio->csum_work);
 	return 0;
 }
 
diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c
index 20e15dc30bfb55..f978c6524aa03f 100644
--- a/fs/btrfs/file.c
+++ b/fs/btrfs/file.c
@@ -2509,8 +2509,10 @@ int btrfs_replace_file_extents(struct btrfs_inode *inode,
 					      inode_set_ctime_current(&inode->vfs_inode));
 
 		ret = btrfs_update_inode(trans, inode);
-		if (ret)
+		if (unlikely(ret)) {
+			btrfs_abort_transaction(trans, ret);
 			break;
+		}
 
 		btrfs_end_transaction(trans);
 		btrfs_btree_balance_dirty(fs_info);
diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c
index e2af75a205ea53..2a40167c3fb657 100644
--- a/fs/btrfs/free-space-cache.c
+++ b/fs/btrfs/free-space-cache.c
@@ -9,7 +9,6 @@
 #include <linux/slab.h>
 #include <linux/math64.h>
 #include <linux/ratelimit.h>
-#include <linux/error-injection.h>
 #include <linux/sched/mm.h>
 #include <linux/string_choices.h>
 #include "extent-tree.h"
@@ -23,7 +22,6 @@
 #include "space-info.h"
 #include "block-group.h"
 #include "discard.h"
-#include "subpage.h"
 #include "inode-item.h"
 #include "accessors.h"
 #include "file-item.h"
@@ -38,12 +36,6 @@
 static struct kmem_cache *btrfs_free_space_cachep;
 static struct kmem_cache *btrfs_free_space_bitmap_cachep;
 
-struct btrfs_trim_range {
-	u64 start;
-	u64 bytes;
-	struct list_head list;
-};
-
 static int link_free_space(struct btrfs_free_space_ctl *ctl,
 			   struct btrfs_free_space *info);
 static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
@@ -57,11 +49,6 @@ static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
 			      struct btrfs_free_space *info, u64 offset,
 			      u64 bytes, bool update_stats);
 
-static void btrfs_crc32c_final(u32 crc, u8 *result)
-{
-	put_unaligned_le32(~crc, result);
-}
-
 static void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl)
 {
 	struct btrfs_free_space *info;
@@ -102,1451 +89,197 @@ static struct inode *__lookup_free_space_inode(struct btrfs_root *root,
 		return ERR_PTR(ret);
 	if (ret > 0) {
 		btrfs_release_path(path);
-		return ERR_PTR(-ENOENT);
-	}
-
-	leaf = path->nodes[0];
-	header = btrfs_item_ptr(leaf, path->slots[0],
-				struct btrfs_free_space_header);
-	btrfs_free_space_key(leaf, header, &disk_key);
-	btrfs_disk_key_to_cpu(&location, &disk_key);
-	btrfs_release_path(path);
-
-	/*
-	 * We are often under a trans handle at this point, so we need to make
-	 * sure NOFS is set to keep us from deadlocking.
-	 */
-	nofs_flag = memalloc_nofs_save();
-	inode = btrfs_iget_path(location.objectid, root, path);
-	btrfs_release_path(path);
-	memalloc_nofs_restore(nofs_flag);
-	if (IS_ERR(inode))
-		return ERR_CAST(inode);
-
-	mapping_set_gfp_mask(inode->vfs_inode.i_mapping,
-			mapping_gfp_constraint(inode->vfs_inode.i_mapping,
-			~(__GFP_FS | __GFP_HIGHMEM)));
-
-	return &inode->vfs_inode;
-}
-
-struct inode *lookup_free_space_inode(struct btrfs_block_group *block_group,
-		struct btrfs_path *path)
-{
-	struct btrfs_fs_info *fs_info = block_group->fs_info;
-	struct inode *inode = NULL;
-	u32 flags = BTRFS_INODE_NODATASUM | BTRFS_INODE_NODATACOW;
-
-	spin_lock(&block_group->lock);
-	if (block_group->inode)
-		inode = igrab(&block_group->inode->vfs_inode);
-	spin_unlock(&block_group->lock);
-	if (inode)
-		return inode;
-
-	inode = __lookup_free_space_inode(fs_info->tree_root, path,
-					  block_group->start);
-	if (IS_ERR(inode))
-		return inode;
-
-	spin_lock(&block_group->lock);
-	if (!((BTRFS_I(inode)->flags & flags) == flags)) {
-		btrfs_info(fs_info, "Old style space inode found, converting.");
-		BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM |
-			BTRFS_INODE_NODATACOW;
-		block_group->disk_cache_state = BTRFS_DC_CLEAR;
-	}
-
-	if (!test_and_set_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags))
-		block_group->inode = BTRFS_I(igrab(inode));
-	spin_unlock(&block_group->lock);
-
-	return inode;
-}
-
-static int __create_free_space_inode(struct btrfs_root *root,
-				     struct btrfs_trans_handle *trans,
-				     struct btrfs_path *path,
-				     u64 ino, u64 offset)
-{
-	struct btrfs_key key;
-	struct btrfs_disk_key disk_key;
-	struct btrfs_free_space_header *header;
-	struct btrfs_inode_item *inode_item;
-	struct extent_buffer *leaf;
-	/* We inline CRCs for the free disk space cache */
-	const u64 flags = BTRFS_INODE_NOCOMPRESS | BTRFS_INODE_PREALLOC |
-			  BTRFS_INODE_NODATASUM | BTRFS_INODE_NODATACOW;
-	int ret;
-
-	ret = btrfs_insert_empty_inode(trans, root, path, ino);
-	if (ret)
-		return ret;
-
-	leaf = path->nodes[0];
-	inode_item = btrfs_item_ptr(leaf, path->slots[0],
-				    struct btrfs_inode_item);
-	btrfs_item_key(leaf, &disk_key, path->slots[0]);
-	memzero_extent_buffer(leaf, (unsigned long)inode_item,
-			     sizeof(*inode_item));
-	btrfs_set_inode_generation(leaf, inode_item, trans->transid);
-	btrfs_set_inode_size(leaf, inode_item, 0);
-	btrfs_set_inode_nbytes(leaf, inode_item, 0);
-	btrfs_set_inode_uid(leaf, inode_item, 0);
-	btrfs_set_inode_gid(leaf, inode_item, 0);
-	btrfs_set_inode_mode(leaf, inode_item, S_IFREG | 0600);
-	btrfs_set_inode_flags(leaf, inode_item, flags);
-	btrfs_set_inode_nlink(leaf, inode_item, 1);
-	btrfs_set_inode_transid(leaf, inode_item, trans->transid);
-	btrfs_set_inode_block_group(leaf, inode_item, offset);
-	btrfs_release_path(path);
-
-	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
-	key.type = 0;
-	key.offset = offset;
-	ret = btrfs_insert_empty_item(trans, root, path, &key,
-				      sizeof(struct btrfs_free_space_header));
-	if (ret < 0) {
-		btrfs_release_path(path);
-		return ret;
-	}
-
-	leaf = path->nodes[0];
-	header = btrfs_item_ptr(leaf, path->slots[0],
-				struct btrfs_free_space_header);
-	memzero_extent_buffer(leaf, (unsigned long)header, sizeof(*header));
-	btrfs_set_free_space_key(leaf, header, &disk_key);
-	btrfs_release_path(path);
-
-	return 0;
-}
-
-int create_free_space_inode(struct btrfs_trans_handle *trans,
-			    struct btrfs_block_group *block_group,
-			    struct btrfs_path *path)
-{
-	int ret;
-	u64 ino;
-
-	ret = btrfs_get_free_objectid(trans->fs_info->tree_root, &ino);
-	if (ret < 0)
-		return ret;
-
-	return __create_free_space_inode(trans->fs_info->tree_root, trans, path,
-					 ino, block_group->start);
-}
-
-/*
- * inode is an optional sink: if it is NULL, btrfs_remove_free_space_inode
- * handles lookup, otherwise it takes ownership and iputs the inode.
- * Don't reuse an inode pointer after passing it into this function.
- */
-int btrfs_remove_free_space_inode(struct btrfs_trans_handle *trans,
-				  struct inode *inode,
-				  struct btrfs_block_group *block_group)
-{
-	BTRFS_PATH_AUTO_FREE(path);
-	struct btrfs_key key;
-	int ret = 0;
-
-	path = btrfs_alloc_path();
-	if (!path)
-		return -ENOMEM;
-
-	if (!inode)
-		inode = lookup_free_space_inode(block_group, path);
-	if (IS_ERR(inode)) {
-		if (PTR_ERR(inode) != -ENOENT)
-			ret = PTR_ERR(inode);
-		return ret;
-	}
-	ret = btrfs_orphan_add(trans, BTRFS_I(inode));
-	if (ret) {
-		btrfs_add_delayed_iput(BTRFS_I(inode));
-		return ret;
-	}
-	clear_nlink(inode);
-	/* One for the block groups ref */
-	spin_lock(&block_group->lock);
-	if (test_and_clear_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags)) {
-		block_group->inode = NULL;
-		spin_unlock(&block_group->lock);
-		iput(inode);
-	} else {
-		spin_unlock(&block_group->lock);
-	}
-	/* One for the lookup ref */
-	btrfs_add_delayed_iput(BTRFS_I(inode));
-
-	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
-	key.type = 0;
-	key.offset = block_group->start;
-	ret = btrfs_search_slot(trans, trans->fs_info->tree_root, &key, path,
-				-1, 1);
-	if (ret) {
-		if (ret > 0)
-			ret = 0;
-		return ret;
-	}
-	return btrfs_del_item(trans, trans->fs_info->tree_root, path);
-}
-
-int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans,
-				    struct btrfs_block_group *block_group,
-				    struct inode *vfs_inode)
-{
-	struct btrfs_truncate_control control = {
-		.inode = BTRFS_I(vfs_inode),
-		.new_size = 0,
-		.ino = btrfs_ino(BTRFS_I(vfs_inode)),
-		.min_type = BTRFS_EXTENT_DATA_KEY,
-		.clear_extent_range = true,
-	};
-	struct btrfs_inode *inode = BTRFS_I(vfs_inode);
-	struct btrfs_root *root = inode->root;
-	struct extent_state *cached_state = NULL;
-	int ret = 0;
-	bool locked = false;
-
-	if (block_group) {
-		BTRFS_PATH_AUTO_FREE(path);
-
-		path = btrfs_alloc_path();
-		if (!path) {
-			ret = -ENOMEM;
-			goto fail;
-		}
-		locked = true;
-		mutex_lock(&trans->transaction->cache_write_mutex);
-		if (!list_empty(&block_group->io_list)) {
-			list_del_init(&block_group->io_list);
-
-			btrfs_wait_cache_io(trans, block_group, path);
-			btrfs_put_block_group(block_group);
-		}
-
-		/*
-		 * now that we've truncated the cache away, its no longer
-		 * setup or written
-		 */
-		spin_lock(&block_group->lock);
-		block_group->disk_cache_state = BTRFS_DC_CLEAR;
-		spin_unlock(&block_group->lock);
-	}
-
-	btrfs_i_size_write(inode, 0);
-	truncate_pagecache(vfs_inode, 0);
-
-	btrfs_lock_extent(&inode->io_tree, 0, (u64)-1, &cached_state);
-	btrfs_drop_extent_map_range(inode, 0, (u64)-1, false);
-
-	/*
-	 * We skip the throttling logic for free space cache inodes, so we don't
-	 * need to check for -EAGAIN.
-	 */
-	ret = btrfs_truncate_inode_items(trans, root, &control);
-
-	inode_sub_bytes(&inode->vfs_inode, control.sub_bytes);
-	btrfs_inode_safe_disk_i_size_write(inode, control.last_size);
-
-	btrfs_unlock_extent(&inode->io_tree, 0, (u64)-1, &cached_state);
-	if (ret)
-		goto fail;
-
-	ret = btrfs_update_inode(trans, inode);
-
-fail:
-	if (locked)
-		mutex_unlock(&trans->transaction->cache_write_mutex);
-	if (ret)
-		btrfs_abort_transaction(trans, ret);
-
-	return ret;
-}
-
-static void readahead_cache(struct inode *inode)
-{
-	struct file_ra_state ra;
-	pgoff_t last_index;
-
-	file_ra_state_init(&ra, inode->i_mapping);
-	last_index = (i_size_read(inode) - 1) >> PAGE_SHIFT;
-
-	page_cache_sync_readahead(inode->i_mapping, &ra, NULL, 0, last_index);
-}
-
-static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,
-		       int write)
-{
-	int num_pages;
-
-	num_pages = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
-
-	/* Make sure we can fit our crcs and generation into the first page */
-	if (write && (num_pages * sizeof(u32) + sizeof(u64)) > PAGE_SIZE)
-		return -ENOSPC;
-
-	memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));
-
-	io_ctl->pages = kzalloc_objs(struct page *, num_pages, GFP_NOFS);
-	if (!io_ctl->pages)
-		return -ENOMEM;
-
-	io_ctl->num_pages = num_pages;
-	io_ctl->fs_info = inode_to_fs_info(inode);
-	io_ctl->inode = inode;
-
-	return 0;
-}
-ALLOW_ERROR_INJECTION(io_ctl_init, ERRNO);
-
-static void io_ctl_free(struct btrfs_io_ctl *io_ctl)
-{
-	kfree(io_ctl->pages);
-	io_ctl->pages = NULL;
-}
-
-static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)
-{
-	if (io_ctl->cur) {
-		io_ctl->cur = NULL;
-		io_ctl->orig = NULL;
-	}
-}
-
-static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)
-{
-	ASSERT(io_ctl->index < io_ctl->num_pages);
-	io_ctl->page = io_ctl->pages[io_ctl->index++];
-	io_ctl->cur = page_address(io_ctl->page);
-	io_ctl->orig = io_ctl->cur;
-	io_ctl->size = PAGE_SIZE;
-	if (clear)
-		clear_page(io_ctl->cur);
-}
-
-static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)
-{
-	int i;
-
-	io_ctl_unmap_page(io_ctl);
-
-	for (i = 0; i < io_ctl->num_pages; i++) {
-		if (io_ctl->pages[i]) {
-			unlock_page(io_ctl->pages[i]);
-			put_page(io_ctl->pages[i]);
-		}
-	}
-}
-
-static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate)
-{
-	struct folio *folio;
-	struct inode *inode = io_ctl->inode;
-	gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
-	int i;
-
-	for (i = 0; i < io_ctl->num_pages; i++) {
-		int ret;
-
-		folio = __filemap_get_folio(inode->i_mapping, i,
-					    FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
-					    mask);
-		if (IS_ERR(folio)) {
-			io_ctl_drop_pages(io_ctl);
-			return PTR_ERR(folio);
-		}
-
-		ret = set_folio_extent_mapped(folio);
-		if (ret < 0) {
-			folio_unlock(folio);
-			folio_put(folio);
-			io_ctl_drop_pages(io_ctl);
-			return ret;
-		}
-
-		io_ctl->pages[i] = &folio->page;
-		if (uptodate && !folio_test_uptodate(folio)) {
-			btrfs_read_folio(NULL, folio);
-			folio_lock(folio);
-			if (folio->mapping != inode->i_mapping) {
-				btrfs_err(BTRFS_I(inode)->root->fs_info,
-					  "free space cache page truncated");
-				io_ctl_drop_pages(io_ctl);
-				return -EIO;
-			}
-			if (!folio_test_uptodate(folio)) {
-				btrfs_err(BTRFS_I(inode)->root->fs_info,
-					   "error reading free space cache");
-				io_ctl_drop_pages(io_ctl);
-				return -EIO;
-			}
-		}
-	}
-
-	for (i = 0; i < io_ctl->num_pages; i++)
-		clear_page_dirty_for_io(io_ctl->pages[i]);
-
-	return 0;
-}
-
-static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
-{
-	io_ctl_map_page(io_ctl, 1);
-
-	/*
-	 * Skip the csum areas.  If we don't check crcs then we just have a
-	 * 64bit chunk at the front of the first page.
-	 */
-	io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);
-	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);
-
-	put_unaligned_le64(generation, io_ctl->cur);
-	io_ctl->cur += sizeof(u64);
-}
-
-static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
-{
-	u64 cache_gen;
-
-	/*
-	 * Skip the crc area.  If we don't check crcs then we just have a 64bit
-	 * chunk at the front of the first page.
-	 */
-	io_ctl->cur += sizeof(u32) * io_ctl->num_pages;
-	io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);
-
-	cache_gen = get_unaligned_le64(io_ctl->cur);
-	if (cache_gen != generation) {
-		btrfs_err_rl(io_ctl->fs_info,
-			"space cache generation (%llu) does not match inode (%llu)",
-				cache_gen, generation);
-		io_ctl_unmap_page(io_ctl);
-		return -EIO;
-	}
-	io_ctl->cur += sizeof(u64);
-	return 0;
-}
-
-static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)
-{
-	u32 *tmp;
-	u32 crc = ~(u32)0;
-	unsigned offset = 0;
-
-	if (index == 0)
-		offset = sizeof(u32) * io_ctl->num_pages;
-
-	crc = crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);
-	btrfs_crc32c_final(crc, (u8 *)&crc);
-	io_ctl_unmap_page(io_ctl);
-	tmp = page_address(io_ctl->pages[0]);
-	tmp += index;
-	*tmp = crc;
-}
-
-static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)
-{
-	u32 *tmp, val;
-	u32 crc = ~(u32)0;
-	unsigned offset = 0;
-
-	if (index >= io_ctl->num_pages)
-		return -EIO;
-
-	if (index == 0)
-		offset = sizeof(u32) * io_ctl->num_pages;
-
-	tmp = page_address(io_ctl->pages[0]);
-	tmp += index;
-	val = *tmp;
-
-	io_ctl_map_page(io_ctl, 0);
-	crc = crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);
-	btrfs_crc32c_final(crc, (u8 *)&crc);
-	if (val != crc) {
-		btrfs_err_rl(io_ctl->fs_info,
-			"csum mismatch on free space cache");
-		io_ctl_unmap_page(io_ctl);
-		return -EIO;
-	}
-
-	return 0;
-}
-
-static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,
-			    void *bitmap)
-{
-	struct btrfs_free_space_entry *entry;
-
-	if (!io_ctl->cur)
-		return -ENOSPC;
-
-	entry = io_ctl->cur;
-	put_unaligned_le64(offset, &entry->offset);
-	put_unaligned_le64(bytes, &entry->bytes);
-	entry->type = (bitmap) ? BTRFS_FREE_SPACE_BITMAP :
-		BTRFS_FREE_SPACE_EXTENT;
-	io_ctl->cur += sizeof(struct btrfs_free_space_entry);
-	io_ctl->size -= sizeof(struct btrfs_free_space_entry);
-
-	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))
-		return 0;
-
-	io_ctl_set_crc(io_ctl, io_ctl->index - 1);
-
-	/* No more pages to map */
-	if (io_ctl->index >= io_ctl->num_pages)
-		return 0;
-
-	/* map the next page */
-	io_ctl_map_page(io_ctl, 1);
-	return 0;
-}
-
-static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)
-{
-	if (!io_ctl->cur)
-		return -ENOSPC;
-
-	/*
-	 * If we aren't at the start of the current page, unmap this one and
-	 * map the next one if there is any left.
-	 */
-	if (io_ctl->cur != io_ctl->orig) {
-		io_ctl_set_crc(io_ctl, io_ctl->index - 1);
-		if (io_ctl->index >= io_ctl->num_pages)
-			return -ENOSPC;
-		io_ctl_map_page(io_ctl, 0);
-	}
-
-	copy_page(io_ctl->cur, bitmap);
-	io_ctl_set_crc(io_ctl, io_ctl->index - 1);
-	if (io_ctl->index < io_ctl->num_pages)
-		io_ctl_map_page(io_ctl, 0);
-	return 0;
-}
-
-static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)
-{
-	/*
-	 * If we're not on the boundary we know we've modified the page and we
-	 * need to crc the page.
-	 */
-	if (io_ctl->cur != io_ctl->orig)
-		io_ctl_set_crc(io_ctl, io_ctl->index - 1);
-	else
-		io_ctl_unmap_page(io_ctl);
-
-	while (io_ctl->index < io_ctl->num_pages) {
-		io_ctl_map_page(io_ctl, 1);
-		io_ctl_set_crc(io_ctl, io_ctl->index - 1);
-	}
-}
-
-static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,
-			    struct btrfs_free_space *entry, u8 *type)
-{
-	struct btrfs_free_space_entry *e;
-	int ret;
-
-	if (!io_ctl->cur) {
-		ret = io_ctl_check_crc(io_ctl, io_ctl->index);
-		if (ret)
-			return ret;
-	}
-
-	e = io_ctl->cur;
-	entry->offset = get_unaligned_le64(&e->offset);
-	entry->bytes = get_unaligned_le64(&e->bytes);
-	*type = e->type;
-	io_ctl->cur += sizeof(struct btrfs_free_space_entry);
-	io_ctl->size -= sizeof(struct btrfs_free_space_entry);
-
-	if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))
-		return 0;
-
-	io_ctl_unmap_page(io_ctl);
-
-	return 0;
-}
-
-static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,
-			      struct btrfs_free_space *entry)
-{
-	int ret;
-
-	ret = io_ctl_check_crc(io_ctl, io_ctl->index);
-	if (ret)
-		return ret;
-
-	copy_page(entry->bitmap, io_ctl->cur);
-	io_ctl_unmap_page(io_ctl);
-
-	return 0;
-}
-
-static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl)
-{
-	struct btrfs_block_group *block_group = ctl->block_group;
-	const int unit = block_group->fs_info->sectorsize;
-	u64 max_bytes;
-	u64 bitmap_bytes;
-	u64 extent_bytes;
-	u64 size = block_group->length;
-	u64 bytes_per_bg = BITS_PER_BITMAP * unit;
-	u64 max_bitmaps = div64_u64(size + bytes_per_bg - 1, bytes_per_bg);
-
-	max_bitmaps = max_t(u64, max_bitmaps, 1);
-
-	if (ctl->total_bitmaps > max_bitmaps)
-		btrfs_err(block_group->fs_info,
-"invalid free space control: bg start=%llu len=%llu total_bitmaps=%u unit=%u max_bitmaps=%llu bytes_per_bg=%llu",
-			  block_group->start, block_group->length,
-			  ctl->total_bitmaps, unit, max_bitmaps,
-			  bytes_per_bg);
-	ASSERT(ctl->total_bitmaps <= max_bitmaps);
-
-	/*
-	 * We are trying to keep the total amount of memory used per 1GiB of
-	 * space to be MAX_CACHE_BYTES_PER_GIG.  However, with a reclamation
-	 * mechanism of pulling extents >= FORCE_EXTENT_THRESHOLD out of
-	 * bitmaps, we may end up using more memory than this.
-	 */
-	if (size < SZ_1G)
-		max_bytes = MAX_CACHE_BYTES_PER_GIG;
-	else
-		max_bytes = MAX_CACHE_BYTES_PER_GIG * div_u64(size, SZ_1G);
-
-	bitmap_bytes = ctl->total_bitmaps * unit;
-
-	/*
-	 * we want the extent entry threshold to always be at most 1/2 the max
-	 * bytes we can have, or whatever is less than that.
-	 */
-	extent_bytes = max_bytes - bitmap_bytes;
-	extent_bytes = min_t(u64, extent_bytes, max_bytes >> 1);
-
-	ctl->extents_thresh =
-		div_u64(extent_bytes, sizeof(struct btrfs_free_space));
-}
-
-static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode,
-				   struct btrfs_free_space_ctl *ctl,
-				   struct btrfs_path *path, u64 offset)
-{
-	struct btrfs_fs_info *fs_info = root->fs_info;
-	struct btrfs_free_space_header *header;
-	struct extent_buffer *leaf;
-	struct btrfs_io_ctl io_ctl;
-	struct btrfs_key key;
-	struct btrfs_free_space *e, *n;
-	LIST_HEAD(bitmaps);
-	u64 num_entries;
-	u64 num_bitmaps;
-	u64 generation;
-	u8 type;
-	int ret = 0;
-
-	/* Nothing in the space cache, goodbye */
-	if (!i_size_read(inode))
-		return 0;
-
-	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
-	key.type = 0;
-	key.offset = offset;
-
-	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
-	if (ret < 0)
-		return 0;
-	else if (ret > 0) {
-		btrfs_release_path(path);
-		return 0;
-	}
-
-	ret = -1;
-
-	leaf = path->nodes[0];
-	header = btrfs_item_ptr(leaf, path->slots[0],
-				struct btrfs_free_space_header);
-	num_entries = btrfs_free_space_entries(leaf, header);
-	num_bitmaps = btrfs_free_space_bitmaps(leaf, header);
-	generation = btrfs_free_space_generation(leaf, header);
-	btrfs_release_path(path);
-
-	if (!BTRFS_I(inode)->generation) {
-		btrfs_info(fs_info,
-			   "the free space cache file (%llu) is invalid, skip it",
-			   offset);
-		return 0;
-	}
-
-	if (BTRFS_I(inode)->generation != generation) {
-		btrfs_err(fs_info,
-			  "free space inode generation (%llu) did not match free space cache generation (%llu)",
-			  BTRFS_I(inode)->generation, generation);
-		return 0;
-	}
-
-	if (!num_entries)
-		return 0;
-
-	ret = io_ctl_init(&io_ctl, inode, 0);
-	if (ret)
-		return ret;
-
-	readahead_cache(inode);
-
-	ret = io_ctl_prepare_pages(&io_ctl, true);
-	if (ret)
-		goto out;
-
-	ret = io_ctl_check_crc(&io_ctl, 0);
-	if (ret)
-		goto free_cache;
-
-	ret = io_ctl_check_generation(&io_ctl, generation);
-	if (ret)
-		goto free_cache;
-
-	while (num_entries) {
-		e = kmem_cache_zalloc(btrfs_free_space_cachep,
-				      GFP_NOFS);
-		if (!e) {
-			ret = -ENOMEM;
-			goto free_cache;
-		}
-
-		ret = io_ctl_read_entry(&io_ctl, e, &type);
-		if (ret) {
-			kmem_cache_free(btrfs_free_space_cachep, e);
-			goto free_cache;
-		}
-
-		if (!e->bytes) {
-			ret = -1;
-			kmem_cache_free(btrfs_free_space_cachep, e);
-			goto free_cache;
-		}
-
-		if (type == BTRFS_FREE_SPACE_EXTENT) {
-			spin_lock(&ctl->tree_lock);
-			ret = link_free_space(ctl, e);
-			spin_unlock(&ctl->tree_lock);
-			if (ret) {
-				btrfs_err(fs_info,
-					"Duplicate entries in free space cache, dumping");
-				kmem_cache_free(btrfs_free_space_cachep, e);
-				goto free_cache;
-			}
-		} else {
-			ASSERT(num_bitmaps);
-			num_bitmaps--;
-			e->bitmap = kmem_cache_zalloc(
-					btrfs_free_space_bitmap_cachep, GFP_NOFS);
-			if (!e->bitmap) {
-				ret = -ENOMEM;
-				kmem_cache_free(
-					btrfs_free_space_cachep, e);
-				goto free_cache;
-			}
-			spin_lock(&ctl->tree_lock);
-			ret = link_free_space(ctl, e);
-			if (ret) {
-				spin_unlock(&ctl->tree_lock);
-				btrfs_err(fs_info,
-					"Duplicate entries in free space cache, dumping");
-				kmem_cache_free(btrfs_free_space_bitmap_cachep, e->bitmap);
-				kmem_cache_free(btrfs_free_space_cachep, e);
-				goto free_cache;
-			}
-			ctl->total_bitmaps++;
-			recalculate_thresholds(ctl);
-			spin_unlock(&ctl->tree_lock);
-			list_add_tail(&e->list, &bitmaps);
-		}
-
-		num_entries--;
-	}
-
-	io_ctl_unmap_page(&io_ctl);
-
-	/*
-	 * We add the bitmaps at the end of the entries in order that
-	 * the bitmap entries are added to the cache.
-	 */
-	list_for_each_entry_safe(e, n, &bitmaps, list) {
-		list_del_init(&e->list);
-		ret = io_ctl_read_bitmap(&io_ctl, e);
-		if (ret)
-			goto free_cache;
-	}
-
-	io_ctl_drop_pages(&io_ctl);
-	ret = 1;
-out:
-	io_ctl_free(&io_ctl);
-	return ret;
-free_cache:
-	io_ctl_drop_pages(&io_ctl);
-
-	spin_lock(&ctl->tree_lock);
-	__btrfs_remove_free_space_cache(ctl);
-	spin_unlock(&ctl->tree_lock);
-	goto out;
-}
-
-static int copy_free_space_cache(struct btrfs_free_space_ctl *ctl)
-{
-	struct btrfs_free_space *info;
-	struct rb_node *n;
-	int ret = 0;
-
-	while (!ret && (n = rb_first(&ctl->free_space_offset)) != NULL) {
-		info = rb_entry(n, struct btrfs_free_space, offset_index);
-		if (!info->bitmap) {
-			const u64 offset = info->offset;
-			const u64 bytes = info->bytes;
-
-			unlink_free_space(ctl, info, true);
-			spin_unlock(&ctl->tree_lock);
-			kmem_cache_free(btrfs_free_space_cachep, info);
-			ret = btrfs_add_free_space(ctl->block_group, offset, bytes);
-			spin_lock(&ctl->tree_lock);
-		} else {
-			u64 offset = info->offset;
-			u64 bytes = ctl->block_group->fs_info->sectorsize;
-
-			ret = search_bitmap(ctl, info, &offset, &bytes, false);
-			if (ret == 0) {
-				bitmap_clear_bits(ctl, info, offset, bytes, true);
-				spin_unlock(&ctl->tree_lock);
-				ret = btrfs_add_free_space(ctl->block_group, offset,
-							   bytes);
-				spin_lock(&ctl->tree_lock);
-			} else {
-				free_bitmap(ctl, info);
-				ret = 0;
-			}
-		}
-		cond_resched_lock(&ctl->tree_lock);
-	}
-	return ret;
-}
-
-static struct lock_class_key btrfs_free_space_inode_key;
-
-int load_free_space_cache(struct btrfs_block_group *block_group)
-{
-	struct btrfs_fs_info *fs_info = block_group->fs_info;
-	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
-	struct btrfs_free_space_ctl tmp_ctl = {};
-	struct inode *inode;
-	struct btrfs_path *path;
-	int ret = 0;
-	bool matched;
-	u64 used = block_group->used;
-
-	/*
-	 * Because we could potentially discard our loaded free space, we want
-	 * to load everything into a temporary structure first, and then if it's
-	 * valid copy it all into the actual free space ctl.
-	 */
-	btrfs_init_free_space_ctl(block_group, &tmp_ctl);
-
-	/*
-	 * If this block group has been marked to be cleared for one reason or
-	 * another then we can't trust the on disk cache, so just return.
-	 */
-	spin_lock(&block_group->lock);
-	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
-		spin_unlock(&block_group->lock);
-		return 0;
-	}
-	spin_unlock(&block_group->lock);
-
-	path = btrfs_alloc_path();
-	if (!path)
-		return 0;
-	path->search_commit_root = true;
-	path->skip_locking = true;
-
-	/*
-	 * We must pass a path with search_commit_root set to btrfs_iget in
-	 * order to avoid a deadlock when allocating extents for the tree root.
-	 *
-	 * When we are COWing an extent buffer from the tree root, when looking
-	 * for a free extent, at extent-tree.c:find_free_extent(), we can find
-	 * block group without its free space cache loaded. When we find one
-	 * we must load its space cache which requires reading its free space
-	 * cache's inode item from the root tree. If this inode item is located
-	 * in the same leaf that we started COWing before, then we end up in
-	 * deadlock on the extent buffer (trying to read lock it when we
-	 * previously write locked it).
-	 *
-	 * It's safe to read the inode item using the commit root because
-	 * block groups, once loaded, stay in memory forever (until they are
-	 * removed) as well as their space caches once loaded. New block groups
-	 * once created get their ->cached field set to BTRFS_CACHE_FINISHED so
-	 * we will never try to read their inode item while the fs is mounted.
-	 */
-	inode = lookup_free_space_inode(block_group, path);
-	if (IS_ERR(inode)) {
-		btrfs_free_path(path);
-		return 0;
-	}
-
-	/* We may have converted the inode and made the cache invalid. */
-	spin_lock(&block_group->lock);
-	if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
-		spin_unlock(&block_group->lock);
-		btrfs_free_path(path);
-		goto out;
-	}
-	spin_unlock(&block_group->lock);
-
-	/*
-	 * Reinitialize the class of struct inode's mapping->invalidate_lock for
-	 * free space inodes to prevent false positives related to locks for normal
-	 * inodes.
-	 */
-	lockdep_set_class(&(&inode->i_data)->invalidate_lock,
-			  &btrfs_free_space_inode_key);
-
-	ret = __load_free_space_cache(fs_info->tree_root, inode, &tmp_ctl,
-				      path, block_group->start);
-	btrfs_free_path(path);
-	if (ret <= 0)
-		goto out;
-
-	matched = (tmp_ctl.free_space == (block_group->length - used -
-					  block_group->bytes_super));
-
-	if (matched) {
-		spin_lock(&tmp_ctl.tree_lock);
-		ret = copy_free_space_cache(&tmp_ctl);
-		spin_unlock(&tmp_ctl.tree_lock);
-		/*
-		 * ret == 1 means we successfully loaded the free space cache,
-		 * so we need to re-set it here.
-		 */
-		if (ret == 0)
-			ret = 1;
-	} else {
-		/*
-		 * We need to call the _locked variant so we don't try to update
-		 * the discard counters.
-		 */
-		spin_lock(&tmp_ctl.tree_lock);
-		__btrfs_remove_free_space_cache(&tmp_ctl);
-		spin_unlock(&tmp_ctl.tree_lock);
-		btrfs_warn(fs_info,
-			   "block group %llu has wrong amount of free space",
-			   block_group->start);
-		ret = -1;
-	}
-out:
-	if (ret < 0) {
-		/* This cache is bogus, make sure it gets cleared */
-		spin_lock(&block_group->lock);
-		block_group->disk_cache_state = BTRFS_DC_CLEAR;
-		spin_unlock(&block_group->lock);
-		ret = 0;
-
-		btrfs_warn(fs_info,
-			   "failed to load free space cache for block group %llu, rebuilding it now",
-			   block_group->start);
-	}
-
-	spin_lock(&ctl->tree_lock);
-	btrfs_discard_update_discardable(block_group);
-	spin_unlock(&ctl->tree_lock);
-	iput(inode);
-	return ret;
-}
-
-static noinline_for_stack
-int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,
-			      struct btrfs_block_group *block_group,
-			      int *entries, int *bitmaps,
-			      struct list_head *bitmap_list)
-{
-	int ret;
-	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
-	struct btrfs_free_cluster *cluster = NULL;
-	struct btrfs_free_cluster *cluster_locked = NULL;
-	struct rb_node *node = rb_first(&ctl->free_space_offset);
-	struct btrfs_trim_range *trim_entry;
-
-	/* Get the cluster for this block_group if it exists */
-	if (!list_empty(&block_group->cluster_list)) {
-		cluster = list_first_entry(&block_group->cluster_list,
-					   struct btrfs_free_cluster, block_group_list);
-	}
-
-	if (!node && cluster) {
-		cluster_locked = cluster;
-		spin_lock(&cluster_locked->lock);
-		node = rb_first(&cluster->root);
-		cluster = NULL;
-	}
-
-	/* Write out the extent entries */
-	while (node) {
-		struct btrfs_free_space *e;
-
-		e = rb_entry(node, struct btrfs_free_space, offset_index);
-		*entries += 1;
-
-		ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,
-				       e->bitmap);
-		if (ret)
-			goto fail;
-
-		if (e->bitmap) {
-			list_add_tail(&e->list, bitmap_list);
-			*bitmaps += 1;
-		}
-		node = rb_next(node);
-		if (!node && cluster) {
-			node = rb_first(&cluster->root);
-			cluster_locked = cluster;
-			spin_lock(&cluster_locked->lock);
-			cluster = NULL;
-		}
-	}
-	if (cluster_locked) {
-		spin_unlock(&cluster_locked->lock);
-		cluster_locked = NULL;
-	}
-
-	/*
-	 * Make sure we don't miss any range that was removed from our rbtree
-	 * because trimming is running. Otherwise after a umount+mount (or crash
-	 * after committing the transaction) we would leak free space and get
-	 * an inconsistent free space cache report from fsck.
-	 */
-	list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) {
-		ret = io_ctl_add_entry(io_ctl, trim_entry->start,
-				       trim_entry->bytes, NULL);
-		if (ret)
-			goto fail;
-		*entries += 1;
-	}
-
-	return 0;
-fail:
-	if (cluster_locked)
-		spin_unlock(&cluster_locked->lock);
-	return -ENOSPC;
-}
-
-static noinline_for_stack int
-update_cache_item(struct btrfs_trans_handle *trans,
-		  struct btrfs_root *root,
-		  struct inode *inode,
-		  struct btrfs_path *path, u64 offset,
-		  int entries, int bitmaps)
-{
-	struct btrfs_key key;
-	struct btrfs_free_space_header *header;
-	struct extent_buffer *leaf;
-	int ret;
-
-	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
-	key.type = 0;
-	key.offset = offset;
-
-	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
-	if (ret < 0) {
-		btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1,
-				       EXTENT_DELALLOC, NULL);
-		return ret;
-	}
-	leaf = path->nodes[0];
-	if (ret > 0) {
-		struct btrfs_key found_key;
-		ASSERT(path->slots[0]);
-		path->slots[0]--;
-		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
-		if (found_key.objectid != BTRFS_FREE_SPACE_OBJECTID ||
-		    found_key.offset != offset) {
-			btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0,
-					       inode->i_size - 1, EXTENT_DELALLOC,
-					       NULL);
-			btrfs_release_path(path);
-			return -ENOENT;
-		}
-	}
-
-	BTRFS_I(inode)->generation = trans->transid;
-	header = btrfs_item_ptr(leaf, path->slots[0],
-				struct btrfs_free_space_header);
-	btrfs_set_free_space_entries(leaf, header, entries);
-	btrfs_set_free_space_bitmaps(leaf, header, bitmaps);
-	btrfs_set_free_space_generation(leaf, header, trans->transid);
-	btrfs_release_path(path);
-
-	return 0;
-}
-
-static noinline_for_stack int write_pinned_extent_entries(
-			    struct btrfs_trans_handle *trans,
-			    struct btrfs_block_group *block_group,
-			    struct btrfs_io_ctl *io_ctl,
-			    int *entries)
-{
-	u64 start, extent_start, extent_end, len;
-	const u64 block_group_end = btrfs_block_group_end(block_group);
-	struct extent_io_tree *unpin = NULL;
-	int ret;
-
-	/*
-	 * We want to add any pinned extents to our free space cache
-	 * so we don't leak the space
-	 *
-	 * We shouldn't have switched the pinned extents yet so this is the
-	 * right one
-	 */
-	unpin = &trans->transaction->pinned_extents;
-
-	start = block_group->start;
-
-	while (start < block_group_end) {
-		if (!btrfs_find_first_extent_bit(unpin, start,
-						 &extent_start, &extent_end,
-						 EXTENT_DIRTY, NULL))
-			return 0;
-
-		/* This pinned extent is out of our range */
-		if (extent_start >= block_group_end)
-			return 0;
-
-		extent_start = max(extent_start, start);
-		extent_end = min(block_group_end, extent_end + 1);
-		len = extent_end - extent_start;
-
-		*entries += 1;
-		ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);
-		if (ret)
-			return -ENOSPC;
-
-		start = extent_end;
-	}
-
-	return 0;
-}
-
-static noinline_for_stack int
-write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)
-{
-	struct btrfs_free_space *entry, *next;
-	int ret;
-
-	/* Write out the bitmaps */
-	list_for_each_entry_safe(entry, next, bitmap_list, list) {
-		ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);
-		if (ret)
-			return -ENOSPC;
-		list_del_init(&entry->list);
+		return ERR_PTR(-ENOENT);
 	}
 
-	return 0;
+	leaf = path->nodes[0];
+	header = btrfs_item_ptr(leaf, path->slots[0],
+				struct btrfs_free_space_header);
+	btrfs_free_space_key(leaf, header, &disk_key);
+	btrfs_disk_key_to_cpu(&location, &disk_key);
+	btrfs_release_path(path);
+
+	/*
+	 * We are often under a trans handle at this point, so we need to make
+	 * sure NOFS is set to keep us from deadlocking.
+	 */
+	nofs_flag = memalloc_nofs_save();
+	inode = btrfs_iget_path(location.objectid, root, path);
+	btrfs_release_path(path);
+	memalloc_nofs_restore(nofs_flag);
+	if (IS_ERR(inode))
+		return ERR_CAST(inode);
+
+	return &inode->vfs_inode;
 }
 
-static int flush_dirty_cache(struct inode *inode)
+struct inode *lookup_free_space_inode(struct btrfs_block_group *block_group,
+		struct btrfs_path *path)
 {
-	int ret;
+	struct btrfs_fs_info *fs_info = block_group->fs_info;
+	struct inode *inode = NULL;
 
-	ret = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1);
-	if (ret)
-		btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1,
-				       EXTENT_DELALLOC, NULL);
+	spin_lock(&block_group->lock);
+	if (block_group->inode)
+		inode = igrab(&block_group->inode->vfs_inode);
+	spin_unlock(&block_group->lock);
+	if (inode)
+		return inode;
 
-	return ret;
-}
+	inode = __lookup_free_space_inode(fs_info->tree_root, path,
+					  block_group->start);
+	if (IS_ERR(inode))
+		return inode;
 
-static void noinline_for_stack
-cleanup_bitmap_list(struct list_head *bitmap_list)
-{
-	struct btrfs_free_space *entry, *next;
+	spin_lock(&block_group->lock);
+	if (!test_and_set_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags))
+		block_group->inode = BTRFS_I(igrab(inode));
+	spin_unlock(&block_group->lock);
 
-	list_for_each_entry_safe(entry, next, bitmap_list, list)
-		list_del_init(&entry->list);
+	return inode;
 }
 
-static void noinline_for_stack
-cleanup_write_cache_enospc(struct inode *inode,
-			   struct btrfs_io_ctl *io_ctl,
-			   struct extent_state **cached_state)
+/*
+ * inode is an optional sink: if it is NULL, btrfs_remove_free_space_inode
+ * handles lookup, otherwise it takes ownership and iputs the inode.
+ * Don't reuse an inode pointer after passing it into this function.
+ */
+int btrfs_remove_free_space_inode(struct btrfs_trans_handle *trans,
+				  struct inode *inode,
+				  struct btrfs_block_group *block_group)
 {
-	io_ctl_drop_pages(io_ctl);
-	btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1,
-			    cached_state);
-}
+	BTRFS_PATH_AUTO_FREE(path);
+	struct btrfs_key key;
+	int ret = 0;
 
-static int __btrfs_wait_cache_io(struct btrfs_root *root,
-				 struct btrfs_trans_handle *trans,
-				 struct btrfs_block_group *block_group,
-				 struct btrfs_io_ctl *io_ctl,
-				 struct btrfs_path *path, u64 offset)
-{
-	int ret;
-	struct inode *inode = io_ctl->inode;
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
 
 	if (!inode)
-		return 0;
-
-	/* Flush the dirty pages in the cache file. */
-	ret = flush_dirty_cache(inode);
-	if (ret)
-		goto out;
-
-	/* Update the cache item to tell everyone this cache file is valid. */
-	ret = update_cache_item(trans, root, inode, path, offset,
-				io_ctl->entries, io_ctl->bitmaps);
-out:
+		inode = lookup_free_space_inode(block_group, path);
+	if (IS_ERR(inode)) {
+		if (PTR_ERR(inode) != -ENOENT)
+			ret = PTR_ERR(inode);
+		return ret;
+	}
+	ret = btrfs_orphan_add(trans, BTRFS_I(inode));
 	if (ret) {
-		invalidate_inode_pages2(inode->i_mapping);
-		BTRFS_I(inode)->generation = 0;
-		if (block_group)
-			btrfs_debug(root->fs_info,
-	  "failed to write free space cache for block group %llu error %d",
-				  block_group->start, ret);
+		btrfs_add_delayed_iput(BTRFS_I(inode));
+		return ret;
 	}
-	btrfs_update_inode(trans, BTRFS_I(inode));
-
-	if (block_group) {
-		/* the dirty list is protected by the dirty_bgs_lock */
-		spin_lock(&trans->transaction->dirty_bgs_lock);
-
-		/* the disk_cache_state is protected by the block group lock */
-		spin_lock(&block_group->lock);
-
-		/*
-		 * only mark this as written if we didn't get put back on
-		 * the dirty list while waiting for IO.   Otherwise our
-		 * cache state won't be right, and we won't get written again
-		 */
-		if (!ret && list_empty(&block_group->dirty_list))
-			block_group->disk_cache_state = BTRFS_DC_WRITTEN;
-		else if (ret)
-			block_group->disk_cache_state = BTRFS_DC_ERROR;
-
+	clear_nlink(inode);
+	/* One for the block groups ref */
+	spin_lock(&block_group->lock);
+	if (test_and_clear_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags)) {
+		block_group->inode = NULL;
 		spin_unlock(&block_group->lock);
-		spin_unlock(&trans->transaction->dirty_bgs_lock);
-		io_ctl->inode = NULL;
 		iput(inode);
+	} else {
+		spin_unlock(&block_group->lock);
 	}
+	/* One for the lookup ref */
+	btrfs_add_delayed_iput(BTRFS_I(inode));
 
-	return ret;
-
-}
-
-int btrfs_wait_cache_io(struct btrfs_trans_handle *trans,
-			struct btrfs_block_group *block_group,
-			struct btrfs_path *path)
-{
-	return __btrfs_wait_cache_io(block_group->fs_info->tree_root, trans,
-				     block_group, &block_group->io_ctl,
-				     path, block_group->start);
+	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
+	key.type = 0;
+	key.offset = block_group->start;
+	ret = btrfs_search_slot(trans, trans->fs_info->tree_root, &key, path,
+				-1, 1);
+	if (ret) {
+		if (ret > 0)
+			ret = 0;
+		return ret;
+	}
+	return btrfs_del_item(trans, trans->fs_info->tree_root, path);
 }
 
-/*
- * Write out cached info to an inode.
- *
- * @inode:       freespace inode we are writing out
- * @ctl:         free space cache we are going to write out
- * @block_group: block_group for this cache if it belongs to a block_group
- * @io_ctl:      holds context for the io
- * @trans:       the trans handle
- *
- * This function writes out a free space cache struct to disk for quick recovery
- * on mount.  This will return 0 if it was successful in writing the cache out,
- * or an errno if it was not.
- */
-static int __btrfs_write_out_cache(struct inode *inode,
-				   struct btrfs_block_group *block_group,
-				   struct btrfs_trans_handle *trans)
+int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans,
+				    struct inode *vfs_inode)
 {
-	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
-	struct btrfs_io_ctl *io_ctl = &block_group->io_ctl;
+	struct btrfs_truncate_control control = {
+		.inode = BTRFS_I(vfs_inode),
+		.new_size = 0,
+		.ino = btrfs_ino(BTRFS_I(vfs_inode)),
+		.min_type = BTRFS_EXTENT_DATA_KEY,
+		.clear_extent_range = true,
+	};
+	struct btrfs_inode *inode = BTRFS_I(vfs_inode);
+	struct btrfs_root *root = inode->root;
 	struct extent_state *cached_state = NULL;
-	LIST_HEAD(bitmap_list);
-	int entries = 0;
-	int bitmaps = 0;
-	int ret;
-	bool must_iput = false;
-	int i_size;
-
-	if (!i_size_read(inode))
-		return -EIO;
-
-	WARN_ON(io_ctl->pages);
-	ret = io_ctl_init(io_ctl, inode, 1);
-	if (ret)
-		return ret;
-
-	if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) {
-		down_write(&block_group->data_rwsem);
-		spin_lock(&block_group->lock);
-		if (block_group->delalloc_bytes) {
-			block_group->disk_cache_state = BTRFS_DC_WRITTEN;
-			spin_unlock(&block_group->lock);
-			up_write(&block_group->data_rwsem);
-			BTRFS_I(inode)->generation = 0;
-			ret = 0;
-			must_iput = true;
-			goto out;
-		}
-		spin_unlock(&block_group->lock);
-	}
-
-	/* Lock all pages first so we can lock the extent safely. */
-	ret = io_ctl_prepare_pages(io_ctl, false);
-	if (ret)
-		goto out_unlock;
-
-	btrfs_lock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1,
-			  &cached_state);
-
-	io_ctl_set_generation(io_ctl, trans->transid);
-
-	mutex_lock(&ctl->cache_writeout_mutex);
-	/* Write out the extent entries in the free space cache */
-	spin_lock(&ctl->tree_lock);
-	ret = write_cache_extent_entries(io_ctl, block_group, &entries, &bitmaps,
-					 &bitmap_list);
-	if (ret)
-		goto out_nospc_locked;
-
-	/*
-	 * Some spaces that are freed in the current transaction are pinned,
-	 * they will be added into free space cache after the transaction is
-	 * committed, we shouldn't lose them.
-	 *
-	 * If this changes while we are working we'll get added back to
-	 * the dirty list and redo it.  No locking needed
-	 */
-	ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries);
-	if (ret)
-		goto out_nospc_locked;
-
-	/*
-	 * At last, we write out all the bitmaps and keep cache_writeout_mutex
-	 * locked while doing it because a concurrent trim can be manipulating
-	 * or freeing the bitmap.
-	 */
-	ret = write_bitmap_entries(io_ctl, &bitmap_list);
-	spin_unlock(&ctl->tree_lock);
-	mutex_unlock(&ctl->cache_writeout_mutex);
-	if (ret)
-		goto out_nospc;
-
-	/* Zero out the rest of the pages just to make sure */
-	io_ctl_zero_remaining_pages(io_ctl);
+	int ret = 0;
 
-	/* Everything is written out, now we dirty the pages in the file. */
-	i_size = i_size_read(inode);
-	for (int i = 0; i < round_up(i_size, PAGE_SIZE) / PAGE_SIZE; i++) {
-		u64 dirty_start = i * PAGE_SIZE;
-		u64 dirty_len = min_t(u64, dirty_start + PAGE_SIZE, i_size) - dirty_start;
+	btrfs_i_size_write(inode, 0);
+	truncate_pagecache(vfs_inode, 0);
 
-		ret = btrfs_dirty_folio(BTRFS_I(inode), page_folio(io_ctl->pages[i]),
-					dirty_start, dirty_len, &cached_state, false);
-		if (ret < 0)
-			goto out_nospc;
-	}
+	btrfs_lock_extent(&inode->io_tree, 0, (u64)-1, &cached_state);
+	btrfs_drop_extent_map_range(inode, 0, (u64)-1, false);
 
-	if (block_group->flags & BTRFS_BLOCK_GROUP_DATA)
-		up_write(&block_group->data_rwsem);
 	/*
-	 * Release the pages and unlock the extent, we will flush
-	 * them out later
+	 * We skip the throttling logic for free space cache inodes, so we don't
+	 * need to check for -EAGAIN.
 	 */
-	io_ctl_drop_pages(io_ctl);
-	io_ctl_free(io_ctl);
-
-	btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1,
-			    &cached_state);
+	ret = btrfs_truncate_inode_items(trans, root, &control);
 
-	/*
-	 * at this point the pages are under IO and we're happy,
-	 * The caller is responsible for waiting on them and updating
-	 * the cache and the inode
-	 */
-	io_ctl->entries = entries;
-	io_ctl->bitmaps = bitmaps;
+	inode_sub_bytes(&inode->vfs_inode, control.sub_bytes);
+	btrfs_inode_safe_disk_i_size_write(inode, control.last_size);
 
-	ret = btrfs_fdatawrite_range(BTRFS_I(inode), 0, (u64)-1);
+	btrfs_unlock_extent(&inode->io_tree, 0, (u64)-1, &cached_state);
 	if (ret)
-		goto out;
-
-	return 0;
-
-out_nospc_locked:
-	cleanup_bitmap_list(&bitmap_list);
-	spin_unlock(&ctl->tree_lock);
-	mutex_unlock(&ctl->cache_writeout_mutex);
+		goto fail;
 
-out_nospc:
-	cleanup_write_cache_enospc(inode, io_ctl, &cached_state);
+	ret = btrfs_update_inode(trans, inode);
 
-out_unlock:
-	if (block_group->flags & BTRFS_BLOCK_GROUP_DATA)
-		up_write(&block_group->data_rwsem);
+fail:
+	if (ret)
+		btrfs_abort_transaction(trans, ret);
 
-out:
-	io_ctl->inode = NULL;
-	io_ctl_free(io_ctl);
-	if (ret) {
-		invalidate_inode_pages2(inode->i_mapping);
-		BTRFS_I(inode)->generation = 0;
-	}
-	btrfs_update_inode(trans, BTRFS_I(inode));
-	if (must_iput)
-		iput(inode);
 	return ret;
 }
 
-int btrfs_write_out_cache(struct btrfs_trans_handle *trans,
-			  struct btrfs_block_group *block_group,
-			  struct btrfs_path *path)
+static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl)
 {
-	struct btrfs_fs_info *fs_info = trans->fs_info;
-	struct inode *inode;
-	int ret = 0;
+	struct btrfs_block_group *block_group = ctl->block_group;
+	const int unit = block_group->fs_info->sectorsize;
+	u64 max_bytes;
+	u64 bitmap_bytes;
+	u64 extent_bytes;
+	u64 size = block_group->length;
+	u64 bytes_per_bg = BITS_PER_BITMAP * unit;
+	u64 max_bitmaps = div64_u64(size + bytes_per_bg - 1, bytes_per_bg);
 
-	spin_lock(&block_group->lock);
-	if (block_group->disk_cache_state < BTRFS_DC_SETUP) {
-		spin_unlock(&block_group->lock);
-		return 0;
-	}
-	spin_unlock(&block_group->lock);
+	max_bitmaps = max_t(u64, max_bitmaps, 1);
 
-	inode = lookup_free_space_inode(block_group, path);
-	if (IS_ERR(inode))
-		return 0;
+	if (ctl->total_bitmaps > max_bitmaps)
+		btrfs_err(block_group->fs_info,
+"invalid free space control: bg start=%llu len=%llu total_bitmaps=%u unit=%u max_bitmaps=%llu bytes_per_bg=%llu",
+			  block_group->start, block_group->length,
+			  ctl->total_bitmaps, unit, max_bitmaps,
+			  bytes_per_bg);
+	ASSERT(ctl->total_bitmaps <= max_bitmaps);
 
-	ret = __btrfs_write_out_cache(inode, block_group, trans);
-	if (ret) {
-		btrfs_debug(fs_info,
-	  "failed to write free space cache for block group %llu error %d",
-			  block_group->start, ret);
-		spin_lock(&block_group->lock);
-		block_group->disk_cache_state = BTRFS_DC_ERROR;
-		spin_unlock(&block_group->lock);
+	/*
+	 * We are trying to keep the total amount of memory used per 1GiB of
+	 * space to be MAX_CACHE_BYTES_PER_GIG.  However, with a reclamation
+	 * mechanism of pulling extents >= FORCE_EXTENT_THRESHOLD out of
+	 * bitmaps, we may end up using more memory than this.
+	 */
+	if (size < SZ_1G)
+		max_bytes = MAX_CACHE_BYTES_PER_GIG;
+	else
+		max_bytes = MAX_CACHE_BYTES_PER_GIG * div_u64(size, SZ_1G);
 
-		block_group->io_ctl.inode = NULL;
-		iput(inode);
-	}
+	bitmap_bytes = ctl->total_bitmaps * unit;
 
 	/*
-	 * if ret == 0 the caller is expected to call btrfs_wait_cache_io
-	 * to wait for IO and put the inode
+	 * we want the extent entry threshold to always be at most 1/2 the max
+	 * bytes we can have, or whatever is less than that.
 	 */
+	extent_bytes = max_bytes - bitmap_bytes;
+	extent_bytes = min_t(u64, extent_bytes, max_bytes >> 1);
 
-	return ret;
+	ctl->extents_thresh =
+		div_u64(extent_bytes, sizeof(struct btrfs_free_space));
 }
 
 static inline unsigned long offset_to_bit(u64 bitmap_start, u32 unit,
@@ -2953,8 +1686,6 @@ void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group,
 	spin_lock_init(&ctl->tree_lock);
 	ctl->block_group = block_group;
 	ctl->free_space_bytes = RB_ROOT_CACHED;
-	INIT_LIST_HEAD(&ctl->trimming_ranges);
-	mutex_init(&ctl->cache_writeout_mutex);
 
 	/*
 	 * we only want to have 32k of ram per block group for keeping
@@ -3650,12 +2381,10 @@ void btrfs_init_free_cluster(struct btrfs_free_cluster *cluster)
 static int do_trimming(struct btrfs_block_group *block_group,
 		       u64 *total_trimmed, u64 start, u64 bytes,
 		       u64 reserved_start, u64 reserved_bytes,
-		       enum btrfs_trim_state reserved_trim_state,
-		       struct btrfs_trim_range *trim_entry)
+		       enum btrfs_trim_state reserved_trim_state)
 {
 	struct btrfs_space_info *space_info = block_group->space_info;
 	struct btrfs_fs_info *fs_info = block_group->fs_info;
-	struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
 	int ret;
 	bool bg_ro;
 	const u64 end = start + bytes;
@@ -3681,7 +2410,6 @@ static int do_trimming(struct btrfs_block_group *block_group,
 		trim_state = BTRFS_TRIM_STATE_TRIMMED;
 	}
 
-	mutex_lock(&ctl->cache_writeout_mutex);
 	if (reserved_start < start)
 		__btrfs_add_free_space(block_group, reserved_start,
 				       start - reserved_start,
@@ -3690,8 +2418,6 @@ static int do_trimming(struct btrfs_block_group *block_group,
 		__btrfs_add_free_space(block_group, end, reserved_end - end,
 				       reserved_trim_state);
 	__btrfs_add_free_space(block_group, start, bytes, trim_state);
-	list_del(&trim_entry->list);
-	mutex_unlock(&ctl->cache_writeout_mutex);
 
 	if (!bg_ro) {
 		spin_lock(&space_info->lock);
@@ -3729,9 +2455,6 @@ static int trim_no_bitmap(struct btrfs_block_group *block_group,
 	const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size);
 
 	while (start < end) {
-		struct btrfs_trim_range trim_entry;
-
-		mutex_lock(&ctl->cache_writeout_mutex);
 		spin_lock(&ctl->tree_lock);
 
 		if (ctl->free_space < minlen)
@@ -3762,7 +2485,6 @@ static int trim_no_bitmap(struct btrfs_block_group *block_group,
 			bytes = entry->bytes;
 			if (bytes < minlen) {
 				spin_unlock(&ctl->tree_lock);
-				mutex_unlock(&ctl->cache_writeout_mutex);
 				goto next;
 			}
 			unlink_free_space(ctl, entry, true);
@@ -3787,7 +2509,6 @@ static int trim_no_bitmap(struct btrfs_block_group *block_group,
 			bytes = min(extent_start + extent_bytes, end) - start;
 			if (bytes < minlen) {
 				spin_unlock(&ctl->tree_lock);
-				mutex_unlock(&ctl->cache_writeout_mutex);
 				goto next;
 			}
 
@@ -3796,14 +2517,9 @@ static int trim_no_bitmap(struct btrfs_block_group *block_group,
 		}
 
 		spin_unlock(&ctl->tree_lock);
-		trim_entry.start = extent_start;
-		trim_entry.bytes = extent_bytes;
-		list_add_tail(&trim_entry.list, &ctl->trimming_ranges);
-		mutex_unlock(&ctl->cache_writeout_mutex);
 
 		ret = do_trimming(block_group, total_trimmed, start, bytes,
-				  extent_start, extent_bytes, extent_trim_state,
-				  &trim_entry);
+				  extent_start, extent_bytes, extent_trim_state);
 		if (ret) {
 			block_group->discard_cursor = start + bytes;
 			break;
@@ -3827,7 +2543,6 @@ static int trim_no_bitmap(struct btrfs_block_group *block_group,
 out_unlock:
 	block_group->discard_cursor = btrfs_block_group_end(block_group);
 	spin_unlock(&ctl->tree_lock);
-	mutex_unlock(&ctl->cache_writeout_mutex);
 
 	return ret;
 }
@@ -3938,16 +2653,13 @@ static int trim_bitmaps(struct btrfs_block_group *block_group,
 
 	while (offset < end) {
 		bool next_bitmap = false;
-		struct btrfs_trim_range trim_entry;
 
-		mutex_lock(&ctl->cache_writeout_mutex);
 		spin_lock(&ctl->tree_lock);
 
 		if (ctl->free_space < minlen) {
 			block_group->discard_cursor =
 				btrfs_block_group_end(block_group);
 			spin_unlock(&ctl->tree_lock);
-			mutex_unlock(&ctl->cache_writeout_mutex);
 			break;
 		}
 
@@ -3963,7 +2675,6 @@ static int trim_bitmaps(struct btrfs_block_group *block_group,
 		if (!entry || (async && minlen && start == offset &&
 			       btrfs_free_space_trimmed(entry))) {
 			spin_unlock(&ctl->tree_lock);
-			mutex_unlock(&ctl->cache_writeout_mutex);
 			next_bitmap = true;
 			goto next;
 		}
@@ -3989,7 +2700,6 @@ static int trim_bitmaps(struct btrfs_block_group *block_group,
 			else
 				entry->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
 			spin_unlock(&ctl->tree_lock);
-			mutex_unlock(&ctl->cache_writeout_mutex);
 			next_bitmap = true;
 			goto next;
 		}
@@ -4000,14 +2710,12 @@ static int trim_bitmaps(struct btrfs_block_group *block_group,
 		 */
 		if (async && *total_trimmed) {
 			spin_unlock(&ctl->tree_lock);
-			mutex_unlock(&ctl->cache_writeout_mutex);
 			return ret;
 		}
 
 		bytes = min(bytes, end - start);
 		if (bytes < minlen || (async && maxlen && bytes > maxlen)) {
 			spin_unlock(&ctl->tree_lock);
-			mutex_unlock(&ctl->cache_writeout_mutex);
 			goto next;
 		}
 
@@ -4027,13 +2735,9 @@ static int trim_bitmaps(struct btrfs_block_group *block_group,
 			free_bitmap(ctl, entry);
 
 		spin_unlock(&ctl->tree_lock);
-		trim_entry.start = start;
-		trim_entry.bytes = bytes;
-		list_add_tail(&trim_entry.list, &ctl->trimming_ranges);
-		mutex_unlock(&ctl->cache_writeout_mutex);
 
 		ret = do_trimming(block_group, total_trimmed, start, bytes,
-				  start, bytes, 0, &trim_entry);
+				  start, bytes, 0);
 		if (ret) {
 			reset_trimming_bitmap(ctl, offset);
 			block_group->discard_cursor =
@@ -4152,47 +2856,29 @@ bool btrfs_free_space_cache_v1_active(struct btrfs_fs_info *fs_info)
 	return btrfs_super_cache_generation(fs_info->super_copy);
 }
 
-static int cleanup_free_space_cache_v1(struct btrfs_fs_info *fs_info,
-				       struct btrfs_trans_handle *trans)
+int btrfs_cleanup_free_space_cache_v1(struct btrfs_fs_info *fs_info)
 {
-	struct btrfs_block_group *block_group;
+	struct btrfs_trans_handle *trans;
 	struct rb_node *node;
+	int ret;
 
 	btrfs_info(fs_info, "cleaning free space cache v1");
 
-	node = rb_first_cached(&fs_info->block_group_cache_tree);
-	while (node) {
-		int ret;
-
-		block_group = rb_entry(node, struct btrfs_block_group, cache_node);
-		ret = btrfs_remove_free_space_inode(trans, NULL, block_group);
-		if (ret)
-			return ret;
-		node = rb_next(node);
-	}
-	return 0;
-}
-
-int btrfs_set_free_space_cache_v1_active(struct btrfs_fs_info *fs_info, bool active)
-{
-	struct btrfs_trans_handle *trans;
-	int ret;
-
 	/*
-	 * update_super_roots will appropriately set or unset
-	 * super_copy->cache_generation based on SPACE_CACHE and
-	 * BTRFS_FS_CLEANUP_SPACE_CACHE_V1. For this reason, we need a
-	 * transaction commit whether we are enabling space cache v1 and don't
-	 * have any other work to do, or are disabling it and removing free
-	 * space inodes.
+	 * update_super_roots() zeroes super_copy->cache_generation while
+	 * BTRFS_FS_CLEANUP_SPACE_CACHE_V1 is set, so this needs a commit.
 	 */
 	trans = btrfs_start_transaction(fs_info->tree_root, 0);
 	if (IS_ERR(trans))
 		return PTR_ERR(trans);
 
-	if (!active) {
-		set_bit(BTRFS_FS_CLEANUP_SPACE_CACHE_V1, &fs_info->flags);
-		ret = cleanup_free_space_cache_v1(fs_info, trans);
+	set_bit(BTRFS_FS_CLEANUP_SPACE_CACHE_V1, &fs_info->flags);
+	for (node = rb_first_cached(&fs_info->block_group_cache_tree); node;
+	     node = rb_next(node)) {
+		struct btrfs_block_group *block_group;
+
+		block_group = rb_entry(node, struct btrfs_block_group, cache_node);
+		ret = btrfs_remove_free_space_inode(trans, NULL, block_group);
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
 			btrfs_end_transaction(trans);
diff --git a/fs/btrfs/free-space-cache.h b/fs/btrfs/free-space-cache.h
index 53fe8e293af1d5..e22443598b8efe 100644
--- a/fs/btrfs/free-space-cache.h
+++ b/fs/btrfs/free-space-cache.h
@@ -14,7 +14,6 @@
 #include "fs.h"
 
 struct inode;
-struct page;
 struct btrfs_fs_info;
 struct btrfs_path;
 struct btrfs_trans_handle;
@@ -84,44 +83,18 @@ struct btrfs_free_space_ctl {
 	s32 discardable_extents[BTRFS_STAT_NR_ENTRIES];
 	s64 discardable_bytes[BTRFS_STAT_NR_ENTRIES];
 	struct btrfs_block_group *block_group;
-	struct mutex cache_writeout_mutex;
-	struct list_head trimming_ranges;
-};
-
-struct btrfs_io_ctl {
-	void *cur, *orig;
-	struct page *page;
-	struct page **pages;
-	struct btrfs_fs_info *fs_info;
-	struct inode *inode;
-	unsigned long size;
-	int index;
-	int num_pages;
-	int entries;
-	int bitmaps;
 };
 
 int __init btrfs_free_space_init(void);
 void __cold btrfs_free_space_exit(void);
 struct inode *lookup_free_space_inode(struct btrfs_block_group *block_group,
 		struct btrfs_path *path);
-int create_free_space_inode(struct btrfs_trans_handle *trans,
-			    struct btrfs_block_group *block_group,
-			    struct btrfs_path *path);
 int btrfs_remove_free_space_inode(struct btrfs_trans_handle *trans,
 				  struct inode *inode,
 				  struct btrfs_block_group *block_group);
 
 int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans,
-				    struct btrfs_block_group *block_group,
 				    struct inode *inode);
-int load_free_space_cache(struct btrfs_block_group *block_group);
-int btrfs_wait_cache_io(struct btrfs_trans_handle *trans,
-			struct btrfs_block_group *block_group,
-			struct btrfs_path *path);
-int btrfs_write_out_cache(struct btrfs_trans_handle *trans,
-			  struct btrfs_block_group *block_group,
-			  struct btrfs_path *path);
 
 void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group,
 			       struct btrfs_free_space_ctl *ctl);
@@ -161,7 +134,7 @@ int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group,
 void btrfs_trim_fully_remapped_block_group(struct btrfs_block_group *bg);
 
 bool btrfs_free_space_cache_v1_active(struct btrfs_fs_info *fs_info);
-int btrfs_set_free_space_cache_v1_active(struct btrfs_fs_info *fs_info, bool active);
+int btrfs_cleanup_free_space_cache_v1(struct btrfs_fs_info *fs_info);
 /* Support functions for running our sanity tests */
 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
 bool btrfs_use_bitmap(struct btrfs_free_space_ctl *ctl,
diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c
index 1b3d82ae3de80e..b7a4a6ade30f0e 100644
--- a/fs/btrfs/free-space-tree.c
+++ b/fs/btrfs/free-space-tree.c
@@ -1353,7 +1353,7 @@ int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info)
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		btrfs_end_transaction(trans);
-		return ret;
+		goto out_clear;
 	}
 
 	node = rb_first_cached(&fs_info->block_group_cache_tree);
@@ -1371,14 +1371,16 @@ int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info)
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
 			btrfs_end_transaction(trans);
-			return ret;
+			goto out_clear;
 		}
 next:
 		if (btrfs_should_end_transaction(trans)) {
 			btrfs_end_transaction(trans);
 			trans = btrfs_start_transaction(free_space_root, 1);
-			if (IS_ERR(trans))
-				return PTR_ERR(trans);
+			if (IS_ERR(trans)) {
+				ret = PTR_ERR(trans);
+				goto out_clear;
+			}
 		}
 		node = rb_next(node);
 	}
@@ -1390,6 +1392,10 @@ int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info)
 	ret = btrfs_commit_transaction(trans);
 	clear_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags);
 	return ret;
+
+out_clear:
+	clear_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags);
+	return ret;
 }
 
 static int __add_block_group_free_space(struct btrfs_trans_handle *trans,
diff --git a/fs/btrfs/fs.c b/fs/btrfs/fs.c
index de160d29dde82f..75a1217727a7bb 100644
--- a/fs/btrfs/fs.c
+++ b/fs/btrfs/fs.c
@@ -79,7 +79,7 @@ void btrfs_csum_init(struct btrfs_csum_ctx *ctx, u16 csum_type)
 		blake2b_init(&ctx->blake2b, 32);
 		break;
 	default:
-		/* Checksume type is validated at mount time. */
+		/* Checksum type is validated at mount time. */
 		BUG();
 	}
 }
diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h
index 10e15a319b93de..79d0828c51c7d7 100644
--- a/fs/btrfs/fs.h
+++ b/fs/btrfs/fs.h
@@ -259,7 +259,6 @@ enum {
 	BTRFS_MOUNT_NOSSD			= (1ULL << 9),
 	BTRFS_MOUNT_DISCARD_SYNC		= (1ULL << 10),
 	BTRFS_MOUNT_FORCE_COMPRESS		= (1ULL << 11),
-	BTRFS_MOUNT_SPACE_CACHE			= (1ULL << 12),
 	BTRFS_MOUNT_CLEAR_CACHE			= (1ULL << 13),
 	BTRFS_MOUNT_USER_SUBVOL_RM_ALLOWED	= (1ULL << 14),
 	BTRFS_MOUNT_ENOSPC_DEBUG		= (1ULL << 15),
@@ -712,7 +711,6 @@ struct btrfs_fs_info {
 	struct workqueue_struct *endio_meta_workers;
 	struct workqueue_struct *rmw_workers;
 	struct btrfs_workqueue *endio_write_workers;
-	struct btrfs_workqueue *endio_freespace_worker;
 	struct btrfs_workqueue *caching_workers;
 
 	struct workqueue_struct *fixup_workers;
@@ -811,7 +809,7 @@ struct btrfs_fs_info {
 	struct btrfs_discard_ctl discard_ctl;
 
 	/* Is qgroup tracking in a consistent state? */
-	u64 qgroup_flags;
+	unsigned long qgroup_flags;
 
 	/* Holds configuration and tracking. Protected by qgroup_lock. */
 	struct rb_root qgroup_tree;
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 93ef3cec191e36..1d79f5263de3d6 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -730,6 +730,9 @@ static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,
 				      u64 end, bool check_inline)
 {
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
+	const u32 blocksize = fs_info->sectorsize;
+
+	ASSERT(IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize));
 
 	if (unlikely(!btrfs_inode_can_compress(inode))) {
 		DEBUG_WARN("BTRFS: unexpected compression for ino %llu", btrfs_ino(inode));
@@ -1366,11 +1369,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode,
 		goto out_unlock;
 	}
 
-	if (btrfs_is_free_space_inode(inode)) {
-		ret = -EINVAL;
-		goto out_unlock;
-	}
-
 	num_bytes = ALIGN(end - start + 1, blocksize);
 	num_bytes = max(blocksize,  num_bytes);
 	ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
@@ -1680,7 +1678,6 @@ static int fallback_to_cow(struct btrfs_inode *inode,
 			   struct folio *locked_folio, const u64 start,
 			   const u64 end)
 {
-	const bool is_space_ino = btrfs_is_free_space_inode(inode);
 	const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root);
 	const u64 range_bytes = end + 1 - start;
 	struct extent_io_tree *io_tree = &inode->io_tree;
@@ -1713,23 +1710,22 @@ static int fallback_to_cow(struct btrfs_inode *inode,
 	 *    extent_clear_unlock_delalloc()) the bytes_may_use counter of the
 	 *    data space info, which we incremented in the step above.
 	 *
-	 * If we need to fallback to cow and the inode corresponds to a free
-	 * space cache inode or an inode of the data relocation tree, we must
-	 * also increment bytes_may_use of the data space_info for the same
-	 * reason. Space caches and relocated data extents always get a prealloc
-	 * extent for them, however scrub or balance may have set the block
-	 * group that contains that extent to RO mode and therefore force COW
-	 * when starting writeback.
+	 * If we need to fallback to cow and the inode is in the data relocation
+	 * tree, we must also increment bytes_may_use of the data space_info for
+	 * the same reason. Relocated data extents always get a prealloc extent,
+	 * however scrub or balance may have set the block group that contains
+	 * that extent to RO mode and therefore force COW when starting
+	 * writeback.
 	 */
 	btrfs_lock_extent(io_tree, start, end, &cached_state);
 	count = btrfs_count_range_bits(io_tree, &range_start, end, range_bytes,
 				       EXTENT_NORESERVE, false, NULL);
-	if (count > 0 || is_space_ino || is_reloc_ino) {
+	if (count > 0 || is_reloc_ino) {
 		u64 bytes = count;
 		struct btrfs_fs_info *fs_info = inode->root->fs_info;
 		struct btrfs_space_info *sinfo = fs_info->data_sinfo;
 
-		if (is_space_ino || is_reloc_ino)
+		if (is_reloc_ino)
 			bytes = range_bytes;
 
 		spin_lock(&sinfo->lock);
@@ -1794,7 +1790,6 @@ static int can_nocow_file_extent(struct btrfs_path *path,
 				 struct btrfs_inode *inode,
 				 struct can_nocow_file_extent_args *args)
 {
-	const bool is_freespace_inode = btrfs_is_free_space_inode(inode);
 	struct extent_buffer *leaf = path->nodes[0];
 	struct btrfs_root *root = inode->root;
 	struct btrfs_file_extent_item *fi;
@@ -1807,8 +1802,7 @@ static int can_nocow_file_extent(struct btrfs_path *path,
 	bool nowait = path->nowait;
 
 	/* If there are pending snapshots for this root, we must do COW. */
-	if (args->writeback_path && !is_freespace_inode &&
-	    atomic_read(&root->snapshot_force_cow))
+	if (args->writeback_path && atomic_read(&root->snapshot_force_cow))
 		goto out;
 
 	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
@@ -1857,7 +1851,6 @@ static int can_nocow_file_extent(struct btrfs_path *path,
 
 	ret = btrfs_cross_ref_exist(inode, key->offset - args->file_extent.offset,
 				    args->file_extent.disk_bytenr, path);
-	WARN_ON_ONCE(ret > 0 && is_freespace_inode);
 	if (ret != 0)
 		goto out;
 
@@ -1892,7 +1885,6 @@ static int can_nocow_file_extent(struct btrfs_path *path,
 	ret = btrfs_lookup_csums_list(csum_root, io_start,
 				      io_start + args->file_extent.num_bytes - 1,
 				      NULL, nowait);
-	WARN_ON_ONCE(ret > 0 && is_freespace_inode);
 	if (ret != 0)
 		goto out;
 
@@ -2331,7 +2323,7 @@ static int run_delalloc_inline(struct btrfs_inode *inode, struct folio *locked_f
 	btrfs_check_folio_write_protected(locked_folio);
 
 	if (btrfs_inode_can_compress(inode) &&
-	    inode_need_compress(inode, 0, blocksize, true)) {
+	    inode_need_compress(inode, 0, blocksize - 1, true)) {
 		if (inode->defrag_compress > 0 &&
 		    inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) {
 			compress_type = inode->defrag_compress;
@@ -2339,12 +2331,27 @@ static int run_delalloc_inline(struct btrfs_inode *inode, struct folio *locked_f
 		} else if (inode->prop_compress) {
 			compress_type = inode->prop_compress;
 		}
+		/*
+		 * We need to pass blocksize and not i_size, otherwise we can't
+		 * create compressed inline extents for data smaller than sector
+		 * size with lzo.
+		 */
 		cb = btrfs_compress_bio(inode, 0, blocksize, compress_type, compress_level, 0);
 		if (IS_ERR(cb)) {
 			cb = NULL;
 			/* Just fall back to non-compressed case. */
 		} else {
 			compressed_size = cb->bbio.bio.bi_iter.bi_size;
+			/*
+			 * If we did not save space, it's pointless and wasteful
+			 * to have an inline compressed extent, so fallback to
+			 * an uncompressed inline extent.
+			 */
+			if (compressed_size >= i_size) {
+				cleanup_compressed_bio(cb);
+				cb = NULL;
+				compressed_size = 0;
+			}
 		}
 	}
 	if (!can_cow_file_range_inline(inode, 0, i_size, compressed_size)) {
@@ -2625,7 +2632,7 @@ void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *s
 		 * and are therefore protected against concurrent calls of this
 		 * function and btrfs_clear_delalloc_extent().
 		 */
-		if (!btrfs_is_free_space_inode(inode) && prev_delalloc_bytes == 0)
+		if (prev_delalloc_bytes == 0)
 			btrfs_add_delalloc_inode(inode);
 	}
 
@@ -2683,7 +2690,6 @@ void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,
 			return;
 
 		if (!btrfs_is_data_reloc_root(root) &&
-		    !btrfs_is_free_space_inode(inode) &&
 		    !(state->state & EXTENT_NORESERVE) &&
 		    (bits & EXTENT_CLEAR_DATA_RESV))
 			btrfs_free_reserved_data_space_noquota(inode, len);
@@ -2701,7 +2707,7 @@ void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,
 		 * and are therefore protected against concurrent calls of this
 		 * function and btrfs_set_delalloc_extent().
 		 */
-		if (!btrfs_is_free_space_inode(inode) && new_delalloc_bytes == 0) {
+		if (new_delalloc_bytes == 0) {
 			spin_lock(&root->delalloc_lock);
 			btrfs_del_delalloc_inode(inode);
 			spin_unlock(&root->delalloc_lock);
@@ -3203,7 +3209,7 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 	int compress_type = 0;
 	int ret = 0;
 	u64 logical_len = ordered_extent->num_bytes;
-	bool freespace_inode;
+	u64 unwritten_start;
 	bool truncated = false;
 	bool clear_reserved_extent = true;
 	unsigned int clear_bits = 0;
@@ -3220,9 +3226,7 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags))
 		clear_bits |= EXTENT_DEFRAG;
 
-	freespace_inode = btrfs_is_free_space_inode(inode);
-	if (!freespace_inode)
-		btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent);
+	btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent);
 
 	if (unlikely(test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags))) {
 		ret = -EIO;
@@ -3257,10 +3261,7 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 				       &cached_state);
 	}
 
-	if (freespace_inode)
-		trans = btrfs_join_transaction_spacecache(root);
-	else
-		trans = btrfs_join_transaction(root);
+	trans = btrfs_join_transaction(root);
 	if (IS_ERR(trans)) {
 		ret = PTR_ERR(trans);
 		trans = NULL;
@@ -3370,29 +3371,11 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
 		if (ret)
 			btrfs_mark_ordered_extent_error(ordered_extent);
 
-		/*
-		 * Drop extent maps for the part of the extent we didn't write.
-		 *
-		 * We have an exception here for the free_space_inode, this is
-		 * because when we do btrfs_get_extent() on the free space inode
-		 * we will search the commit root.  If this is a new block group
-		 * we won't find anything, and we will trip over the assert in
-		 * writepage where we do ASSERT(em->block_start !=
-		 * EXTENT_MAP_HOLE).
-		 *
-		 * Theoretically we could also skip this for any NOCOW extent as
-		 * we don't mess with the extent map tree in the NOCOW case, but
-		 * for now simply skip this if we are the free space inode.
-		 */
-		if (!btrfs_is_free_space_inode(inode)) {
-			u64 unwritten_start = start;
-
-			if (truncated)
-				unwritten_start += logical_len;
-
-			btrfs_drop_extent_map_range(inode, unwritten_start,
-						    end, false);
-		}
+		/* Drop extent maps for the part of the extent we didn't write. */
+		unwritten_start = start;
+		if (truncated)
+			unwritten_start += logical_len;
+		btrfs_drop_extent_map_range(inode, unwritten_start, end, false);
 
 		/*
 		 * If the ordered extent had an IOERR or something else went
@@ -3456,79 +3439,30 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)
 	return btrfs_finish_one_ordered(ordered);
 }
 
-/*
- * Calculate the checksum of an fs block at physical memory address @paddr,
- * and save the result to @dest.
- *
- * The folio containing @paddr must be large enough to contain a full fs block.
- */
-void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info,
-				      const phys_addr_t paddr, u8 *dest)
-{
-	struct folio *folio = page_folio(phys_to_page(paddr));
-	const u32 blocksize = fs_info->sectorsize;
-	const u32 step = min(blocksize, PAGE_SIZE);
-	const u32 nr_steps = blocksize / step;
-	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
-
-	/* The full block must be inside the folio. */
-	ASSERT(offset_in_folio(folio, paddr) + blocksize <= folio_size(folio));
-
-	for (int i = 0; i < nr_steps; i++) {
-		u32 pindex = offset_in_folio(folio, paddr + i * step) >> PAGE_SHIFT;
-
-		/*
-		 * For bs <= ps cases, we will only run the loop once, so the offset
-		 * inside the page will only added to paddrs[0].
-		 *
-		 * For bs > ps cases, the block must be page aligned, thus offset
-		 * inside the page will always be 0.
-		 */
-		paddrs[i] = page_to_phys(folio_page(folio, pindex)) + offset_in_page(paddr);
-	}
-	return btrfs_calculate_block_csum_pages(fs_info, paddrs, dest);
-}
-
-/*
- * Calculate the checksum of a fs block backed by multiple noncontiguous pages
- * at @paddrs[] and save the result to @dest.
- *
- * The folio containing @paddr must be large enough to contain a full fs block.
- */
-void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info,
-				      const phys_addr_t paddrs[], u8 *dest)
+/* Generate data checksum for a single fs block, pointed to by @orig_iter. */
+void btrfs_csum_one_bio_block(struct btrfs_fs_info *fs_info, struct bio *bio,
+			      const struct bvec_iter *orig_iter, u8 *csum)
 {
+	struct btrfs_csum_ctx cctx;
+	struct bvec_iter iter = *orig_iter;
 	const u32 blocksize = fs_info->sectorsize;
-	const u32 step = min(blocksize, PAGE_SIZE);
-	const u32 nr_steps = blocksize / step;
-	struct btrfs_csum_ctx csum;
+	u32 cur = 0;
 
-	btrfs_csum_init(&csum, fs_info->csum_type);
-	for (int i = 0; i < nr_steps; i++) {
-		const phys_addr_t paddr = paddrs[i];
+	btrfs_csum_init(&cctx, fs_info->csum_type);
+	while (cur < blocksize) {
+		struct page *page = bio_iter_page(bio, iter);
+		const u32 pg_off = bio_iter_offset(bio, iter);
+		const u32 cur_len = min(bio_iter_len(bio, iter), blocksize - cur);
 		void *kaddr;
 
-		ASSERT(offset_in_page(paddr) + step <= PAGE_SIZE);
-		kaddr = kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr);
-		btrfs_csum_update(&csum, kaddr, step);
+		kaddr = kmap_local_page(page) + pg_off;
+		btrfs_csum_update(&cctx, kaddr, cur_len);
 		kunmap_local(kaddr);
-	}
-	btrfs_csum_final(&csum, dest);
-}
 
-/*
- * Verify the checksum for a single sector without any extra action that depend
- * on the type of I/O.
- *
- * @kaddr must be a properly kmapped address.
- */
-int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum,
-			   const u8 * const csum_expected)
-{
-	btrfs_calculate_block_csum_folio(fs_info, paddr, csum);
-	if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0))
-		return -EIO;
-	return 0;
+		bio_advance_iter_single(bio, &iter, cur_len);
+		cur += cur_len;
+	}
+	btrfs_csum_final(&cctx, csum);
 }
 
 /*
@@ -3536,27 +3470,30 @@ int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8
  * different noncontiguous pages.
  *
  * @bbio:	btrfs_io_bio which contains the csum
- * @dev:	device the sector is on
- * @bio_offset:	offset to the beginning of the bio (in bytes)
- * @paddrs:	physical addresses which back the fs block
+ * @orig_iter:	bvec iter pointing to the start of the block
+ * @dev:	device the sector is on (optional)
  *
  * Check if the checksum on a data block is valid.  When a checksum mismatch is
  * detected, report the error and fill the corrupted range with zero.
  *
  * Return %true if the sector is ok or had no checksum to start with, else %false.
  */
-bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
-			u32 bio_offset, const phys_addr_t paddrs[])
+bool btrfs_bio_data_csum_ok(struct btrfs_bio *bbio,
+			    const struct bvec_iter *orig_iter,
+			    struct btrfs_device *dev)
 {
 	struct btrfs_inode *inode = bbio->inode;
 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
+	struct bvec_iter iter = *orig_iter;
 	const u32 blocksize = fs_info->sectorsize;
-	const u32 step = min(blocksize, PAGE_SIZE);
-	const u32 nr_steps = blocksize / step;
+	const u32 bio_offset = (iter.bi_sector - bbio->saved_iter.bi_sector) << SECTOR_SHIFT;
 	u64 file_offset = bbio->file_offset + bio_offset;
 	u64 end = file_offset + blocksize - 1;
 	u8 *csum_expected;
 	u8 csum[BTRFS_CSUM_SIZE];
+	u32 cur = 0;
+
+	ASSERT(iter.bi_sector >= bbio->saved_iter.bi_sector);
 
 	if (!bbio->csum)
 		return true;
@@ -3572,7 +3509,7 @@ bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
 
 	csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) *
 				fs_info->csum_size;
-	btrfs_calculate_block_csum_pages(fs_info, paddrs, csum);
+	btrfs_csum_one_bio_block(fs_info, &bbio->bio, orig_iter, csum);
 	if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0))
 		goto zeroit;
 	return true;
@@ -3582,8 +3519,16 @@ bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
 				    bbio->mirror_num);
 	if (dev)
 		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS);
-	for (int i = 0; i < nr_steps; i++)
-		memzero_page(phys_to_page(paddrs[i]), offset_in_page(paddrs[i]), step);
+	while (cur < blocksize) {
+		struct page *page = bio_iter_page(&bbio->bio, iter);
+		const u32 pg_off = bio_iter_offset(&bbio->bio, iter);
+		const u32 cur_len = min(bio_iter_len(&bbio->bio, iter), blocksize - cur);
+
+		memzero_page(page, pg_off, cur_len);
+
+		bio_advance_iter_single(&bbio->bio, &iter, cur_len);
+		cur += cur_len;
+	}
 	return false;
 }
 
@@ -3877,7 +3822,8 @@ int btrfs_orphan_cleanup(struct btrfs_root *root)
 				if (ret)
 					goto out;
 			}
-			trans = btrfs_start_transaction(root, 1);
+			/* Only deletes the orphan. */
+			trans = btrfs_start_transaction_fallback_global_rsv(root, 1);
 			if (IS_ERR(trans)) {
 				ret = PTR_ERR(trans);
 				goto out;
@@ -5482,7 +5428,7 @@ static int btrfs_setsize(struct inode *inode, struct iattr *attr)
 	return ret;
 }
 
-static int btrfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int btrfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			 struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -6865,8 +6811,28 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
 		}
 	} else {
 		ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,
-				     false, BTRFS_I(inode)->dir_index);
-		if (unlikely(ret)) {
+				     false, BTRFS_I(inode)->dir_index, NULL);
+		if (ret == -ENOMEM) {
+			/*
+			 * Orphan the new inode instead of aborting. The inode
+			 * item was already written with nlink 1, and discard's
+			 * eviction won't delete a bad inode, so nlink 0 must be
+			 * persisted here or orphan cleanup would see nlink > 0,
+			 * drop the orphan item, and leak the inode.
+			 */
+			clear_nlink(inode);
+			/* btrfs_orphan_add() aborts the transaction on failure. */
+			ret = btrfs_orphan_add(trans, BTRFS_I(inode));
+			if (ret)
+				goto discard;
+			ret = btrfs_update_inode(trans, BTRFS_I(inode));
+			if (ret) {
+				btrfs_abort_transaction(trans, ret);
+				goto discard;
+			}
+			ret = -ENOMEM;
+			goto discard;
+		} else if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
 			goto discard;
 		}
@@ -6897,7 +6863,8 @@ int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
  */
 int btrfs_add_link(struct btrfs_trans_handle *trans,
 		   struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
-		   const struct fscrypt_str *name, bool add_backref, u64 index)
+		   const struct fscrypt_str *name, bool add_backref, u64 index,
+		   struct btrfs_dir_index_prealloc *prealloc)
 {
 	int ret = 0;
 	struct btrfs_key key;
@@ -6923,12 +6890,14 @@ int btrfs_add_link(struct btrfs_trans_handle *trans,
 	}
 
 	/* Nothing to clean up yet */
-	if (ret)
+	if (ret) {
+		btrfs_free_delayed_dir_index_prealloc(trans, prealloc);
 		return ret;
+	}
 
 	ret = btrfs_insert_dir_item(trans, name, parent_inode, &key,
-				    btrfs_inode_type(inode), index);
-	if (ret == -EEXIST || ret == -EOVERFLOW)
+				    btrfs_inode_type(inode), index, prealloc);
+	if (ret == -EEXIST || ret == -EOVERFLOW || ret == -ENOMEM)
 		goto fail_dir_item;
 	else if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
@@ -7007,7 +6976,7 @@ static int btrfs_create_common(struct inode *dir, struct dentry *dentry,
 	return ret;
 }
 
-static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int btrfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct inode *inode;
@@ -7021,7 +6990,7 @@ static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return btrfs_create_common(dir, dentry, inode);
 }
 
-static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int btrfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -7081,7 +7050,7 @@ static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
 	inode_set_ctime_current(inode);
 
 	ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
-			     &fname.disk_name, true, index);
+			     &fname.disk_name, true, index, NULL);
 	if (ret)
 		goto fail;
 
@@ -7118,7 +7087,7 @@ static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
 	return ret;
 }
 
-static struct dentry *btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *btrfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -7148,7 +7117,7 @@ static noinline int uncompress_inline(struct btrfs_path *path,
 	compress_type = btrfs_file_extent_compression(leaf, item);
 	max_size = btrfs_file_extent_ram_bytes(leaf, item);
 	inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]);
-	tmp = kmalloc(inline_size, GFP_NOFS);
+	tmp = kvmalloc(inline_size, GFP_NOFS);
 	if (!tmp)
 		return -ENOMEM;
 	ptr = btrfs_file_extent_inline_start(item);
@@ -7169,7 +7138,7 @@ static noinline int uncompress_inline(struct btrfs_path *path,
 
 	if (max_size < blocksize)
 		folio_zero_range(folio, max_size, blocksize - max_size);
-	kfree(tmp);
+	kvfree(tmp);
 	return ret;
 }
 
@@ -7265,16 +7234,6 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
 	/* Chances are we'll be called again, so go ahead and do readahead */
 	path->reada = READA_FORWARD;
 
-	/*
-	 * The same explanation in load_free_space_cache applies here as well,
-	 * we only read when we're loading the free space cache, and at that
-	 * point the commit_root has everything we need.
-	 */
-	if (btrfs_is_free_space_inode(inode)) {
-		path->search_commit_root = true;
-		path->skip_locking = true;
-	}
-
 	ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
 	if (ret < 0) {
 		goto out;
@@ -8025,7 +7984,7 @@ static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback)
 	return ret;
 }
 
-struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap,
+struct inode *btrfs_new_subvol_inode(const struct mnt_idmap *idmap,
 				     struct inode *dir)
 {
 	struct inode *inode;
@@ -8131,7 +8090,6 @@ void btrfs_destroy_inode(struct inode *vfs_inode)
 	struct btrfs_ordered_extent *ordered;
 	struct btrfs_inode *inode = BTRFS_I(vfs_inode);
 	struct btrfs_root *root = inode->root;
-	bool freespace_inode;
 
 	WARN_ON(!hlist_empty(&vfs_inode->i_dentry));
 	WARN_ON(vfs_inode->i_data.nrpages);
@@ -8154,12 +8112,6 @@ void btrfs_destroy_inode(struct inode *vfs_inode)
 	if (!root)
 		return;
 
-	/*
-	 * If this is a free space inode do not take the ordered extents lockdep
-	 * map.
-	 */
-	freespace_inode = btrfs_is_free_space_inode(inode);
-
 	while (1) {
 		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
 		if (!ordered)
@@ -8169,8 +8121,7 @@ void btrfs_destroy_inode(struct inode *vfs_inode)
 				  "found ordered extent %llu %llu on inode cleanup",
 				  ordered->file_offset, ordered->num_bytes);
 
-			if (!freespace_inode)
-				btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent);
+			btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent);
 
 			btrfs_remove_ordered_extent(ordered);
 			btrfs_put_ordered_extent(ordered);
@@ -8227,7 +8178,7 @@ int __init btrfs_init_cachep(void)
 	return 0;
 }
 
-static int btrfs_getattr(struct mnt_idmap *idmap,
+static int btrfs_getattr(const struct mnt_idmap *idmap,
 			 const struct path *path, struct kstat *stat,
 			 u32 request_mask, unsigned int flags)
 {
@@ -8495,14 +8446,14 @@ static int btrfs_rename_exchange(struct inode *old_dir,
 	}
 
 	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
-			     new_name, false, old_idx);
+			     new_name, false, old_idx, NULL);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto out_fail;
 	}
 
 	ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
-			     old_name, false, new_idx);
+			     old_name, false, new_idx, NULL);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto out_fail;
@@ -8543,7 +8494,7 @@ static int btrfs_rename_exchange(struct inode *old_dir,
 	return ret;
 }
 
-static struct inode *new_whiteout_inode(struct mnt_idmap *idmap,
+static struct inode *new_whiteout_inode(const struct mnt_idmap *idmap,
 					struct inode *dir)
 {
 	struct inode *inode;
@@ -8558,7 +8509,7 @@ static struct inode *new_whiteout_inode(struct mnt_idmap *idmap,
 	return inode;
 }
 
-static int btrfs_rename(struct mnt_idmap *idmap,
+static int btrfs_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
@@ -8575,6 +8526,7 @@ static int btrfs_rename(struct mnt_idmap *idmap,
 	struct inode *new_inode = d_inode(new_dentry);
 	struct inode *old_inode = d_inode(old_dentry);
 	struct btrfs_rename_ctx rename_ctx;
+	struct btrfs_dir_index_prealloc *prealloc = NULL;
 	u64 index = 0;
 	int ret;
 	int ret2;
@@ -8698,6 +8650,23 @@ static int btrfs_rename(struct mnt_idmap *idmap,
 	if (ret)
 		goto out_fail;
 
+	/*
+	 * When not overwriting an existing entry, pre-allocate the delayed dir
+	 * index now so that ENOMEM is returned before any btree modifications.
+	 * For the overwrite case, too many btree changes have already happened
+	 * by the time btrfs_add_link() is called.
+	 */
+	if (!new_inode) {
+		prealloc = btrfs_prealloc_delayed_dir_index(BTRFS_I(new_dir),
+							    new_fname.disk_name.name,
+							    new_fname.disk_name.len);
+		if (IS_ERR(prealloc)) {
+			ret = PTR_ERR(prealloc);
+			prealloc = NULL;
+			goto out_fail;
+		}
+	}
+
 	BTRFS_I(old_inode)->dir_index = 0ULL;
 	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
 		/* force full log commit if subvolume involved. */
@@ -8793,7 +8762,8 @@ static int btrfs_rename(struct mnt_idmap *idmap,
 	}
 
 	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
-			     &new_fname.disk_name, false, index);
+			     &new_fname.disk_name, false, index, prealloc);
+	prealloc = NULL;
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto out_fail;
@@ -8818,6 +8788,7 @@ static int btrfs_rename(struct mnt_idmap *idmap,
 		}
 	}
 out_fail:
+	btrfs_free_delayed_dir_index_prealloc(trans, prealloc);
 	if (logs_pinned) {
 		btrfs_end_log_trans(root);
 		btrfs_end_log_trans(dest);
@@ -8838,7 +8809,7 @@ static int btrfs_rename(struct mnt_idmap *idmap,
 	return ret;
 }
 
-static int btrfs_rename2(struct mnt_idmap *idmap, struct inode *old_dir,
+static int btrfs_rename2(const struct mnt_idmap *idmap, struct inode *old_dir,
 			 struct dentry *old_dentry, struct inode *new_dir,
 			 struct dentry *new_dentry, unsigned int flags)
 {
@@ -9026,7 +8997,7 @@ int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr,
 	return ret;
 }
 
-static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int btrfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, const char *symname)
 {
 	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
@@ -9132,14 +9103,13 @@ static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static struct btrfs_trans_handle *insert_prealloc_file_extent(
-				       struct btrfs_trans_handle *trans_in,
 				       struct btrfs_inode *inode,
 				       struct btrfs_key *ins,
 				       u64 file_offset)
 {
 	struct btrfs_file_extent_item stack_fi;
 	struct btrfs_replace_extent_info extent_info;
-	struct btrfs_trans_handle *trans = trans_in;
+	struct btrfs_trans_handle *trans;
 	struct btrfs_path *path;
 	u64 start = ins->objectid;
 	u64 len = ins->offset;
@@ -9160,15 +9130,6 @@ static struct btrfs_trans_handle *insert_prealloc_file_extent(
 	if (ret < 0)
 		return ERR_PTR(ret);
 
-	if (trans) {
-		ret = insert_reserved_file_extent(trans, inode,
-						  file_offset, &stack_fi,
-						  true, qgroup_released);
-		if (ret)
-			goto free_qgroup;
-		return trans;
-	}
-
 	extent_info.disk_offset = start;
 	extent_info.disk_len = len;
 	extent_info.data_offset = 0;
@@ -9208,12 +9169,12 @@ static struct btrfs_trans_handle *insert_prealloc_file_extent(
 	return ERR_PTR(ret);
 }
 
-static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
-				       u64 start, u64 num_bytes, u64 min_size,
-				       loff_t actual_len, u64 *alloc_hint,
-				       struct btrfs_trans_handle *trans)
+int btrfs_prealloc_file_range(struct inode *inode, int mode,
+			      u64 start, u64 num_bytes, u64 min_size,
+			      loff_t actual_len, u64 *alloc_hint)
 {
 	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
+	struct btrfs_trans_handle *trans;
 	struct extent_map *em;
 	struct btrfs_root *root = BTRFS_I(inode)->root;
 	struct btrfs_key ins;
@@ -9223,11 +9184,8 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
 	u64 cur_bytes;
 	u64 last_alloc = (u64)-1;
 	int ret = 0;
-	bool own_trans = true;
 	u64 end = start + num_bytes - 1;
 
-	if (trans)
-		own_trans = false;
 	while (num_bytes > 0) {
 		cur_bytes = min_t(u64, num_bytes, SZ_256M);
 		cur_bytes = max(cur_bytes, min_size);
@@ -9253,8 +9211,8 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
 		clear_offset += ins.offset;
 
 		last_alloc = ins.offset;
-		trans = insert_prealloc_file_extent(trans, BTRFS_I(inode),
-						    &ins, cur_offset);
+		trans = insert_prealloc_file_extent(BTRFS_I(inode), &ins,
+						    cur_offset);
 		/*
 		 * Now that we inserted the prealloc extent we can finally
 		 * decrement the number of reservations in the block group.
@@ -9326,8 +9284,7 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
 					range_start, range_end - range_start);
 			if (ret) {
 				btrfs_abort_transaction(trans, ret);
-				if (own_trans)
-					btrfs_end_transaction(trans);
+				btrfs_end_transaction(trans);
 				break;
 			}
 
@@ -9339,15 +9296,11 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
 
 		if (unlikely(ret)) {
 			btrfs_abort_transaction(trans, ret);
-			if (own_trans)
-				btrfs_end_transaction(trans);
+			btrfs_end_transaction(trans);
 			break;
 		}
 
-		if (own_trans) {
-			btrfs_end_transaction(trans);
-			trans = NULL;
-		}
+		btrfs_end_transaction(trans);
 	}
 	if (clear_offset < end)
 		btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset,
@@ -9355,30 +9308,12 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
 	return ret;
 }
 
-int btrfs_prealloc_file_range(struct inode *inode, int mode,
-			      u64 start, u64 num_bytes, u64 min_size,
-			      loff_t actual_len, u64 *alloc_hint)
-{
-	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
-					   min_size, actual_len, alloc_hint,
-					   NULL);
-}
-
-int btrfs_prealloc_file_range_trans(struct inode *inode,
-				    struct btrfs_trans_handle *trans, int mode,
-				    u64 start, u64 num_bytes, u64 min_size,
-				    loff_t actual_len, u64 *alloc_hint)
-{
-	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
-					   min_size, actual_len, alloc_hint, trans);
-}
-
 /*
  * NOTE: in case you are adding MAY_EXEC check for directories:
  * we are marking them with IOP_FASTPERM_MAY_EXEC, allowing path lookup to
  * elide calls here.
  */
-static int btrfs_permission(struct mnt_idmap *idmap,
+static int btrfs_permission(const struct mnt_idmap *idmap,
 			    struct inode *inode, int mask)
 {
 	struct btrfs_root *root = BTRFS_I(inode)->root;
@@ -9394,7 +9329,7 @@ static int btrfs_permission(struct mnt_idmap *idmap,
 	return generic_permission(idmap, inode, mask);
 }
 
-static int btrfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int btrfs_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct file *file, umode_t mode)
 {
 	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
@@ -9607,7 +9542,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
 	struct completion sync_reads;
 	unsigned long i = 0;
 	struct btrfs_bio *bbio;
-	int ret;
 
 	/*
 	 * Fast path for synchronous reads which completes in this call, io_uring
@@ -9654,10 +9588,10 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
 
 	if (uring_ctx) {
 		if (refcount_dec_and_test(&priv->pending_refs)) {
-			ret = blk_status_to_errno(READ_ONCE(priv->status));
-			btrfs_uring_read_extent_endio(uring_ctx, ret);
+			int err = blk_status_to_errno(READ_ONCE(priv->status));
+
+			btrfs_uring_read_extent_endio(uring_ctx, err);
 			kfree(priv);
-			return ret;
 		}
 
 		return -EIOCBQUEUED;
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index e4b2da31a0d5de..f3e2afe221be35 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -278,7 +278,7 @@ int btrfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int btrfs_fileattr_set(struct mnt_idmap *idmap,
+int btrfs_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa)
 {
 	struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
@@ -549,7 +549,7 @@ static unsigned int create_subvol_num_items(const struct btrfs_qgroup_inherit *i
 	return num_items;
 }
 
-static noinline int create_subvol(struct mnt_idmap *idmap,
+static noinline int create_subvol(const struct mnt_idmap *idmap,
 				  struct inode *dir, struct dentry *dentry,
 				  struct btrfs_qgroup_inherit *inherit)
 {
@@ -879,7 +879,7 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir,
  * inside this filesystem so it's quite a bit simpler.
  */
 static noinline int btrfs_mksubvol(struct dentry *parent,
-				   struct mnt_idmap *idmap,
+				   const struct mnt_idmap *idmap,
 				   struct qstr *qname, struct btrfs_root *snap_src,
 				   bool readonly,
 				   struct btrfs_qgroup_inherit *inherit)
@@ -926,7 +926,7 @@ static noinline int btrfs_mksubvol(struct dentry *parent,
 }
 
 static noinline int btrfs_mksnapshot(struct dentry *parent,
-				   struct mnt_idmap *idmap,
+				   const struct mnt_idmap *idmap,
 				   struct qstr *qname,
 				   struct btrfs_root *root,
 				   bool readonly,
@@ -1164,7 +1164,7 @@ static noinline int __btrfs_ioctl_snap_create(struct file *file,
 {
 	int ret;
 	struct qstr qname = QSTR(name);
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 
 	if (!S_ISDIR(file_inode(file)->i_mode))
 		return -ENOTDIR;
@@ -1741,7 +1741,7 @@ static noinline int btrfs_search_path_in_tree(struct btrfs_root *root, u64 dirid
 	return 0;
 }
 
-static int btrfs_search_path_in_tree_user(struct mnt_idmap *idmap,
+static int btrfs_search_path_in_tree_user(const struct mnt_idmap *idmap,
 				struct inode *inode,
 				struct btrfs_ioctl_ino_lookup_user_args *args)
 {
@@ -2241,7 +2241,7 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file,
 	struct btrfs_root *dest = NULL;
 	struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args);
 	struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args2);
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	char *subvol_name, *subvol_name_ptr = NULL;
 	int ret = 0;
 	bool destroy_parent = false;
@@ -3881,7 +3881,7 @@ static long btrfs_ioctl_quota_rescan_status(struct btrfs_fs_info *fs_info,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 
-	if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) {
+	if (test_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags)) {
 		qsa.flags = 1;
 		qsa.progress = fs_info->qgroup_rescan_progress.objectid;
 	}
@@ -3901,7 +3901,7 @@ static long btrfs_ioctl_quota_rescan_wait(struct btrfs_fs_info *fs_info)
 }
 
 static long _btrfs_ioctl_set_received_subvol(struct file *file,
-					    struct mnt_idmap *idmap,
+					    const struct mnt_idmap *idmap,
 					    struct btrfs_ioctl_received_subvol_args *sa)
 {
 	struct inode *inode = file_inode(file);
@@ -4601,7 +4601,7 @@ static void btrfs_uring_read_finished(struct io_tw_req tw_req, io_tw_token_t tw)
 	size_t page_offset;
 	ssize_t ret;
 
-	/* The inode lock has already been acquired in btrfs_uring_read_extent.  */
+	/* The inode lock has already been acquired in btrfs_encoded_read(). */
 	btrfs_lockdep_inode_acquire(inode, i_rwsem);
 
 	if (priv->err) {
@@ -4667,7 +4667,6 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter,
 				   struct iovec *iov, struct io_uring_cmd *cmd)
 {
 	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
-	struct extent_io_tree *io_tree = &inode->io_tree;
 	struct page **pages = NULL;
 	struct btrfs_uring_priv *priv = NULL;
 	unsigned long nr_pages;
@@ -4723,8 +4722,6 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter,
 	return -EIOCBQUEUED;
 
 out_fail:
-	btrfs_unlock_extent(io_tree, start, lockend, &cached_state);
-	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
 	kfree(priv);
 	for (int i = 0; i < nr_pages; i++) {
 		if (pages[i])
@@ -4752,9 +4749,6 @@ static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue
 	struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd);
 	struct btrfs_uring_encoded_data *data = NULL;
 
-	if (cmd->flags & IORING_URING_CMD_REISSUE)
-		data = bc->data;
-
 	if (!capable(CAP_SYS_ADMIN)) {
 		ret = -EPERM;
 		goto out_acct;
@@ -4836,8 +4830,6 @@ static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue
 
 	ret = btrfs_encoded_read(&kiocb, &data->iter, &data->args, &cached_state,
 				 &disk_bytenr, &disk_io_size);
-	if (ret == -EAGAIN)
-		goto out_acct;
 	if (ret < 0 && ret != -EIOCBQUEUED)
 		goto out_free;
 
@@ -4865,8 +4857,10 @@ static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue
 					      cached_state, disk_bytenr, disk_io_size,
 					      count, data->args.compression,
 					      data->iov, cmd);
-
-		goto out_acct;
+		if (ret == -EIOCBQUEUED)
+			goto out_acct;
+		btrfs_unlock_extent(io_tree, start, lockend, &cached_state);
+		btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
 	}
 
 out_free:
@@ -4877,8 +4871,10 @@ static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue
 		add_rchar(current, ret);
 	inc_syscr(current);
 
-	if (ret != -EIOCBQUEUED && ret != -EAGAIN)
+	if (ret != -EIOCBQUEUED) {
 		kfree(data);
+		bc->data = NULL;
+	}
 
 	return ret;
 }
@@ -4890,12 +4886,8 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu
 	struct kiocb kiocb;
 	ssize_t ret;
 	void __user *sqe_addr;
-	struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd);
 	struct btrfs_uring_encoded_data *data = NULL;
 
-	if (cmd->flags & IORING_URING_CMD_REISSUE)
-		data = bc->data;
-
 	if (!capable(CAP_SYS_ADMIN)) {
 		ret = -EPERM;
 		goto out_acct;
@@ -4907,6 +4899,11 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu
 		goto out_acct;
 	}
 
+	if (issue_flags & IO_URING_F_NONBLOCK) {
+		ret = -EAGAIN;
+		goto out_acct;
+	}
+
 	if (!data) {
 		data = kzalloc_obj(*data, GFP_NOFS);
 		if (!data) {
@@ -4914,8 +4911,6 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu
 			goto out_acct;
 		}
 
-		bc->data = data;
-
 		if (issue_flags & IO_URING_F_COMPAT) {
 #if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT)
 			struct btrfs_ioctl_encoded_io_args_32 args32;
@@ -4975,11 +4970,6 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu
 		}
 	}
 
-	if (issue_flags & IO_URING_F_NONBLOCK) {
-		ret = -EAGAIN;
-		goto out_acct;
-	}
-
 	pos = data->args.offset;
 	ret = rw_verify_area(WRITE, file, &pos, data->args.len);
 	if (ret < 0)
@@ -5005,8 +4995,7 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu
 		add_wchar(current, ret);
 	inc_syscw(current);
 
-	if (ret != -EAGAIN)
-		kfree(data);
+	kfree(data);
 	return ret;
 }
 
diff --git a/fs/btrfs/ioctl.h b/fs/btrfs/ioctl.h
index ccf6bed9cc24ee..55f86aeb3500ea 100644
--- a/fs/btrfs/ioctl.h
+++ b/fs/btrfs/ioctl.h
@@ -17,7 +17,7 @@ struct btrfs_ioctl_balance_args;
 long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 long btrfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 int btrfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int btrfs_fileattr_set(struct mnt_idmap *idmap,
+int btrfs_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa);
 int btrfs_ioctl_get_supported_features(void __user *arg);
 void btrfs_sync_inode_flags_to_i_flags(struct btrfs_inode *inode);
diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c
index b32d4eabe0abe4..df74c75d6c2991 100644
--- a/fs/btrfs/ordered-data.c
+++ b/fs/btrfs/ordered-data.c
@@ -417,13 +417,10 @@ static bool can_finish_ordered_extent(struct btrfs_ordered_extent *ordered,
 
 static void btrfs_queue_ordered_fn(struct btrfs_ordered_extent *ordered)
 {
-	struct btrfs_inode *inode = ordered->inode;
-	struct btrfs_fs_info *fs_info = inode->root->fs_info;
-	struct btrfs_workqueue *wq = btrfs_is_free_space_inode(inode) ?
-		fs_info->endio_freespace_worker : fs_info->endio_write_workers;
+	struct btrfs_fs_info *fs_info = ordered->inode->root->fs_info;
 
 	btrfs_init_work(&ordered->work, finish_ordered_fn, NULL);
-	btrfs_queue_work(wq, &ordered->work);
+	btrfs_queue_work(fs_info->endio_write_workers, &ordered->work);
 }
 
 void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered,
@@ -657,13 +654,6 @@ void btrfs_remove_ordered_extent(struct btrfs_ordered_extent *entry)
 	struct btrfs_fs_info *fs_info = root->fs_info;
 	struct rb_node *node;
 	bool pending;
-	bool freespace_inode;
-
-	/*
-	 * If this is a free space inode the thread has not acquired the ordered
-	 * extents lockdep map.
-	 */
-	freespace_inode = btrfs_is_free_space_inode(btrfs_inode);
 
 	btrfs_lockdep_acquire(fs_info, btrfs_trans_pending_ordered);
 	/* This is paired with alloc_ordered_extent(). */
@@ -738,8 +728,7 @@ void btrfs_remove_ordered_extent(struct btrfs_ordered_extent *entry)
 	}
 	spin_unlock(&root->ordered_extent_lock);
 	wake_up(&entry->wait);
-	if (!freespace_inode)
-		btrfs_lockdep_release(fs_info, btrfs_ordered_extent);
+	btrfs_lockdep_release(fs_info, btrfs_ordered_extent);
 }
 
 static void btrfs_run_ordered_extent_work(struct btrfs_work *work)
@@ -870,16 +859,9 @@ void btrfs_start_ordered_extent_nowriteback(struct btrfs_ordered_extent *entry,
 	u64 start = entry->file_offset;
 	u64 end = start + entry->num_bytes - 1;
 	struct btrfs_inode *inode = entry->inode;
-	bool freespace_inode;
 
 	trace_btrfs_ordered_extent_start(inode, entry);
 
-	/*
-	 * If this is a free space inode do not take the ordered extents lockdep
-	 * map.
-	 */
-	freespace_inode = btrfs_is_free_space_inode(inode);
-
 	/*
 	 * pages in the range can be dirty, clean or writeback.  We
 	 * start IO on any dirty ones so the wait doesn't stall waiting
@@ -899,8 +881,7 @@ void btrfs_start_ordered_extent_nowriteback(struct btrfs_ordered_extent *entry,
 		}
 	}
 
-	if (!freespace_inode)
-		btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent);
+	btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent);
 	wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags));
 }
 
diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index f68b696b4bf720..05e35eb126dc5b 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -34,7 +34,7 @@ enum btrfs_qgroup_mode btrfs_qgroup_mode(const struct btrfs_fs_info *fs_info)
 {
 	if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags))
 		return BTRFS_QGROUP_MODE_DISABLED;
-	if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE)
+	if (test_bit(BTRFS_QGROUP_STATUS_BIT_SIMPLE_MODE, &fs_info->qgroup_flags))
 		return BTRFS_QGROUP_MODE_SIMPLE;
 	return BTRFS_QGROUP_MODE_FULL;
 }
@@ -384,14 +384,14 @@ static bool squota_check_parent_usage(struct btrfs_fs_info *fs_info, struct btrf
 __printf(2, 3)
 static void qgroup_mark_inconsistent(struct btrfs_fs_info *fs_info, const char *fmt, ...)
 {
-	const u64 old_flags = fs_info->qgroup_flags;
+	const unsigned long old_flags = fs_info->qgroup_flags;
 
 	if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
 		return;
-	fs_info->qgroup_flags |= (BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT |
-				  BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN |
-				  BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING);
-	if (!(old_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT)) {
+	set_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags);
+	set_bit(BTRFS_QGROUP_RUNTIME_BIT_CANCEL_RESCAN, &fs_info->qgroup_flags);
+	set_bit(BTRFS_QGROUP_RUNTIME_BIT_NO_ACCOUNTING, &fs_info->qgroup_flags);
+	if (!test_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &old_flags)) {
 		struct va_format vaf;
 		va_list args;
 
@@ -426,7 +426,6 @@ int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info)
 	struct extent_buffer *l;
 	int slot;
 	int ret = 0;
-	u64 flags = 0;
 	u64 rescan_progress = 0;
 
 	if (!fs_info->quota_root)
@@ -473,8 +472,12 @@ int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info)
 				 "old qgroup version, quota disabled");
 				goto out;
 			}
-			fs_info->qgroup_flags = btrfs_qgroup_status_flags(l, ptr);
-			if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE)
+			if (btrfs_qgroup_status_flags(l, ptr) > ULONG_MAX) {
+				btrfs_err(fs_info, "invalid qgroup status flags, quota disabled");
+				goto out;
+			}
+			fs_info->qgroup_flags = (unsigned long)btrfs_qgroup_status_flags(l, ptr);
+			if (test_bit(BTRFS_QGROUP_STATUS_BIT_SIMPLE_MODE, &fs_info->qgroup_flags))
 				qgroup_read_enable_gen(fs_info, l, slot, ptr);
 			else if (btrfs_qgroup_status_generation(l, ptr) != fs_info->generation)
 				qgroup_mark_inconsistent(fs_info, "qgroup generation mismatch");
@@ -609,14 +612,13 @@ int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info)
 	}
 out:
 	btrfs_free_path(path);
-	fs_info->qgroup_flags |= flags;
 	if (ret >= 0) {
-		if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_ON)
+		if (test_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags))
 			set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
-		if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN)
+		if (test_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags))
 			ret = qgroup_rescan_init(fs_info, rescan_progress, 0);
 	} else {
-		fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
+		clear_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags);
 		btrfs_sysfs_del_qgroups(fs_info);
 	}
 
@@ -1101,9 +1103,9 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
 				 struct btrfs_qgroup_status_item);
 	btrfs_set_qgroup_status_generation(leaf, ptr, trans->transid);
 	btrfs_set_qgroup_status_version(leaf, ptr, BTRFS_QGROUP_STATUS_VERSION);
-	fs_info->qgroup_flags = BTRFS_QGROUP_STATUS_FLAG_ON;
+	set_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags);
 	if (simple) {
-		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE;
+		set_bit(BTRFS_QGROUP_STATUS_BIT_SIMPLE_MODE, &fs_info->qgroup_flags);
 		btrfs_set_fs_incompat(fs_info, SIMPLE_QUOTA);
 		/*
 		 * Set the enable generation to the next transaction, as we cannot
@@ -1113,7 +1115,7 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
 		 */
 		btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid + 1);
 	} else {
-		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
+		set_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags);
 	}
 	btrfs_set_qgroup_status_flags(leaf, ptr, fs_info->qgroup_flags &
 				      BTRFS_QGROUP_STATUS_FLAGS_MASK);
@@ -1403,8 +1405,14 @@ int btrfs_quota_disable(struct btrfs_fs_info *fs_info)
 	spin_lock(&fs_info->qgroup_lock);
 	quota_root = fs_info->quota_root;
 	fs_info->quota_root = NULL;
-	fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_ON;
-	fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE;
+	/*
+	 * Clear all on-disk and runtime bits, except RESCAN related ones, that
+	 * are either handled by rescan thread, or the caller who rejects rescan.
+	 */
+	clear_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags);
+	clear_bit(BTRFS_QGROUP_STATUS_BIT_SIMPLE_MODE, &fs_info->qgroup_flags);
+	clear_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags);
+	clear_bit(BTRFS_QGROUP_RUNTIME_BIT_NO_ACCOUNTING, &fs_info->qgroup_flags);
 	fs_info->qgroup_drop_subtree_thres = BTRFS_QGROUP_DROP_SUBTREE_THRES_DEFAULT;
 	spin_unlock(&fs_info->qgroup_lock);
 
@@ -1554,7 +1562,7 @@ static int quick_update_accounting(struct btrfs_fs_info *fs_info,
 	}
 out:
 	if (ret)
-		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
+		set_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags);
 	return ret;
 }
 
@@ -1875,7 +1883,7 @@ int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
 	 * very frequently.
 	 */
 	if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL &&
-	    !(fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT)) {
+	    !test_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags)) {
 		if (unlikely(qgroup->rfer || qgroup->excl ||
 			     qgroup->rfer_cmpr || qgroup->excl_cmpr)) {
 			DEBUG_WARN();
@@ -2120,7 +2128,7 @@ int btrfs_qgroup_trace_extent_post(struct btrfs_trans_handle *trans,
 	 */
 	ASSERT(trans != NULL);
 
-	if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)
+	if (test_bit(BTRFS_QGROUP_RUNTIME_BIT_NO_ACCOUNTING, &fs_info->qgroup_flags))
 		return 0;
 
 	ret = btrfs_find_all_roots(&ctx, true);
@@ -2740,6 +2748,24 @@ int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans,
 	return 0;
 }
 
+void btrfs_qgroup_check_tree_drop(struct btrfs_fs_info *fs_info, u64 rootid, u8 level)
+{
+	u8 drop_subtree_thres;
+
+	if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_FULL)
+		return;
+
+	if (!btrfs_is_fstree(rootid))
+		return;
+
+	spin_lock(&fs_info->qgroup_lock);
+	drop_subtree_thres = fs_info->qgroup_drop_subtree_thres;
+	spin_unlock(&fs_info->qgroup_lock);
+
+	if (level >= drop_subtree_thres)
+		qgroup_mark_inconsistent(fs_info, "subtree level reached threshold");
+}
+
 static void qgroup_iterator_nested_add(struct list_head *head, struct btrfs_qgroup *qgroup)
 {
 	if (!list_empty(&qgroup->nested_iterator))
@@ -2961,7 +2987,7 @@ int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr,
 	 * we can't just exit here.
 	 */
 	if (!btrfs_qgroup_full_accounting(fs_info) ||
-	    fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)
+	    test_bit(BTRFS_QGROUP_RUNTIME_BIT_NO_ACCOUNTING, &fs_info->qgroup_flags))
 		goto out_free;
 
 	if (new_roots) {
@@ -2983,7 +3009,7 @@ int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr,
 					num_bytes, nr_old_roots, nr_new_roots);
 
 	mutex_lock(&fs_info->qgroup_rescan_lock);
-	if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) {
+	if (test_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags)) {
 		if (fs_info->qgroup_rescan_progress.objectid <= bytenr) {
 			mutex_unlock(&fs_info->qgroup_rescan_lock);
 			ret = 0;
@@ -3044,8 +3070,8 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans)
 		num_dirty_extents++;
 		trace_btrfs_qgroup_account_extents(fs_info, record, bytenr);
 
-		if (!ret && !(fs_info->qgroup_flags &
-			      BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)) {
+		if (!ret && !test_bit(BTRFS_QGROUP_RUNTIME_BIT_NO_ACCOUNTING,
+				      &fs_info->qgroup_flags)) {
 			struct btrfs_backref_walk_ctx ctx = { 0 };
 
 			ctx.bytenr = bytenr;
@@ -3152,9 +3178,9 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans)
 		spin_lock(&fs_info->qgroup_lock);
 	}
 	if (btrfs_qgroup_enabled(fs_info))
-		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_ON;
+		set_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags);
 	else
-		fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_ON;
+		clear_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags);
 	spin_unlock(&fs_info->qgroup_lock);
 
 	ret = update_qgroup_status_item(trans);
@@ -3844,7 +3870,7 @@ static bool rescan_should_stop(struct btrfs_fs_info *fs_info)
 		return true;
 	if (!btrfs_qgroup_enabled(fs_info))
 		return true;
-	if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN)
+	if (test_bit(BTRFS_QGROUP_RUNTIME_BIT_CANCEL_RESCAN, &fs_info->qgroup_flags))
 		return true;
 	return false;
 }
@@ -3894,12 +3920,10 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
 	btrfs_free_path(path);
 
 	mutex_lock(&fs_info->qgroup_rescan_lock);
-	if (ret > 0 &&
-	    fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT) {
-		fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
-	} else if (ret < 0 || stopped) {
-		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
-	}
+	if (ret > 0)
+		clear_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags);
+	else if (ret < 0 || stopped)
+		set_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags);
 	mutex_unlock(&fs_info->qgroup_rescan_lock);
 
 	/*
@@ -3923,9 +3947,9 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
 	}
 
 	mutex_lock(&fs_info->qgroup_rescan_lock);
-	if (!stopped ||
-	    fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN)
-		fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
+	if (!stopped || test_bit(BTRFS_QGROUP_RUNTIME_BIT_CANCEL_RESCAN,
+				 &fs_info->qgroup_flags))
+		clear_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags);
 	if (trans) {
 		int ret2 = update_qgroup_status_item(trans);
 
@@ -3935,7 +3959,7 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
 		}
 	}
 	fs_info->qgroup_rescan_running = false;
-	fs_info->qgroup_flags &= ~BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN;
+	clear_bit(BTRFS_QGROUP_RUNTIME_BIT_CANCEL_RESCAN, &fs_info->qgroup_flags);
 	complete_all(&fs_info->qgroup_rescan_completion);
 	mutex_unlock(&fs_info->qgroup_rescan_lock);
 
@@ -3946,7 +3970,7 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
 
 	if (stopped) {
 		btrfs_info(fs_info, "qgroup scan paused");
-	} else if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN) {
+	} else if (test_bit(BTRFS_QGROUP_RUNTIME_BIT_CANCEL_RESCAN, &fs_info->qgroup_flags)) {
 		btrfs_info(fs_info, "qgroup scan cancelled");
 	} else if (ret >= 0) {
 		btrfs_info(fs_info, "qgroup scan completed%s",
@@ -3973,13 +3997,11 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid,
 
 	if (!init_flags) {
 		/* we're resuming qgroup rescan at mount time */
-		if (!(fs_info->qgroup_flags &
-		      BTRFS_QGROUP_STATUS_FLAG_RESCAN)) {
+		if (!(test_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags))) {
 			btrfs_debug(fs_info,
 			"qgroup rescan init failed, qgroup rescan is not queued");
 			ret = -EINVAL;
-		} else if (!(fs_info->qgroup_flags &
-			     BTRFS_QGROUP_STATUS_FLAG_ON)) {
+		} else if (!(test_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags))) {
 			btrfs_debug(fs_info,
 			"qgroup rescan init failed, qgroup is not enabled");
 			ret = -ENOTCONN;
@@ -3992,10 +4014,12 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid,
 	mutex_lock(&fs_info->qgroup_rescan_lock);
 
 	if (init_flags) {
-		if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) {
+		if (test_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN,
+			     &fs_info->qgroup_flags) ||
+		    test_bit(BTRFS_QGROUP_RUNTIME_BIT_REJECT_RESCAN,
+			     &fs_info->qgroup_flags)) {
 			ret = -EINPROGRESS;
-		} else if (!(fs_info->qgroup_flags &
-			     BTRFS_QGROUP_STATUS_FLAG_ON)) {
+		} else if (!test_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags)) {
 			btrfs_debug(fs_info,
 			"qgroup rescan init failed, qgroup is not enabled");
 			ret = -ENOTCONN;
@@ -4008,13 +4032,13 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid,
 			mutex_unlock(&fs_info->qgroup_rescan_lock);
 			return ret;
 		}
-		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_RESCAN;
+		set_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags);
 	}
 
 	memset(&fs_info->qgroup_rescan_progress, 0,
 		sizeof(fs_info->qgroup_rescan_progress));
-	fs_info->qgroup_flags &= ~(BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN |
-				   BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING);
+	clear_bit(BTRFS_QGROUP_RUNTIME_BIT_CANCEL_RESCAN, &fs_info->qgroup_flags);
+	clear_bit(BTRFS_QGROUP_RUNTIME_BIT_NO_ACCOUNTING, &fs_info->qgroup_flags);
 	fs_info->qgroup_rescan_progress.objectid = progress_objectid;
 	init_completion(&fs_info->qgroup_rescan_completion);
 	mutex_unlock(&fs_info->qgroup_rescan_lock);
@@ -4065,7 +4089,7 @@ btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info)
 
 	ret = btrfs_commit_current_transaction(fs_info->fs_root);
 	if (ret) {
-		fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
+		clear_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags);
 		return ret;
 	}
 
@@ -4118,7 +4142,7 @@ int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info,
 void
 btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info)
 {
-	if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) {
+	if (test_bit(BTRFS_QGROUP_STATUS_BIT_RESCAN, &fs_info->qgroup_flags)) {
 		mutex_lock(&fs_info->qgroup_rescan_lock);
 		fs_info->qgroup_rescan_running = true;
 		btrfs_queue_work(fs_info->qgroup_rescan_workers,
diff --git a/fs/btrfs/qgroup.h b/fs/btrfs/qgroup.h
index 80dd2dacd56db4..c64b26b09c22f2 100644
--- a/fs/btrfs/qgroup.h
+++ b/fs/btrfs/qgroup.h
@@ -121,8 +121,19 @@ struct btrfs_qgroup_swapped_blocks;
  * To minimize the chance of collision with new persisted status flags, these
  * count backwards from the MSB.
  */
-#define BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN		(1ULL << 63)
-#define BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING		(1ULL << 62)
+#define BTRFS_QGROUP_RUNTIME_BIT_CANCEL_RESCAN		(BITS_PER_LONG - 1)
+#define BTRFS_QGROUP_RUNTIME_BIT_NO_ACCOUNTING		(BITS_PER_LONG - 2)
+
+/*
+ * No new rescan allowed when set.
+ *
+ * During huge subtree dropping, qgroup will be marked inconsistent, and skip
+ * all future accounting to avoid long stall.  But, an immediate rescan will
+ * re-enable qgroup and still stall the system.
+ *
+ * This bit is to avoid such rescan during the duration of a subvolume dropping.
+ */
+#define BTRFS_QGROUP_RUNTIME_BIT_REJECT_RESCAN		(BITS_PER_LONG - 3)
 
 #define BTRFS_QGROUP_DROP_SUBTREE_THRES_DEFAULT		(3)
 
@@ -365,6 +376,7 @@ int btrfs_qgroup_trace_leaf_items(struct btrfs_trans_handle *trans,
 int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans,
 			       struct extent_buffer *root_eb,
 			       u64 root_gen, int root_level);
+void btrfs_qgroup_check_tree_drop(struct btrfs_fs_info *fs_info, u64 rootid, u8 level);
 int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr,
 				u64 num_bytes, struct ulist *old_roots,
 				struct ulist *new_roots);
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index 1ee52a9dcee36f..8ec24dbb180f92 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -953,7 +953,7 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t status)
 
 	/*
 	 * Clear the data bitmap, as the rbio may be cached for later usage.
-	 * do this before before unlock_stripe() so there will be no new bio
+	 * do this before unlock_stripe() so there will be no new bio
 	 * for this bio.
 	 */
 	bitmap_clear(&rbio->dbitmap, 0, rbio->stripe_nsectors);
@@ -988,7 +988,7 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t status)
  * as possible, and only use stripe_sectors as fallback.
  *
  * Return NULL if bio_list_only is set but the specified sector has no
- * coresponding bio.
+ * corresponding bio.
  */
 static phys_addr_t *sector_paddrs_in_rbio(struct btrfs_raid_bio *rbio,
 					  int stripe_nr, int sector_nr,
@@ -1450,10 +1450,7 @@ static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio,
 	/* We should have at least one data sector. */
 	ASSERT(bitmap_weight(&rbio->dbitmap, rbio->stripe_nsectors));
 
-	/*
-	 * Reset errors, as we may have errors inherited from from degraded
-	 * write.
-	 */
+	/* Reset errors, as we may have errors inherited from degraded write. */
 	bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors);
 
 	/*
@@ -1652,12 +1649,7 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,
 				    struct bio *bio)
 {
 	struct btrfs_fs_info *fs_info = rbio->bioc->fs_info;
-	const u32 step = min(fs_info->sectorsize, PAGE_SIZE);
-	const u32 nr_steps = rbio->sector_nsteps;
 	int total_sector_nr = get_bio_sector_nr(rbio, bio);
-	u32 offset = 0;
-	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
-	phys_addr_t paddr;
 
 	/* No data csum for the whole stripe, no need to verify. */
 	if (!rbio->csum_bitmap || !rbio->csum_buf)
@@ -1667,28 +1659,20 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio,
 	if (total_sector_nr >= rbio->nr_data * rbio->stripe_nsectors)
 		return;
 
-	btrfs_bio_for_each_block_all(paddr, bio, step) {
+	for (struct bvec_iter iter = init_bvec_iter_for_bio(bio);
+	     iter.bi_size;
+	     bio_advance_iter(bio, &iter, fs_info->sectorsize), total_sector_nr++) {
 		u8 csum_buf[BTRFS_CSUM_SIZE];
 		u8 *expected_csum;
 
-		paddrs[(offset / step) % nr_steps] = paddr;
-		offset += step;
-
-		/* Not yet covering the full fs block, continue to the next step. */
-		if (!IS_ALIGNED(offset, fs_info->sectorsize))
-			continue;
-
 		/* No csum for this sector, skip to the next sector. */
-		if (!test_bit(total_sector_nr, rbio->csum_bitmap)) {
-			total_sector_nr++;
+		if (!test_bit(total_sector_nr, rbio->csum_bitmap))
 			continue;
-		}
 
 		expected_csum = rbio->csum_buf + total_sector_nr * fs_info->csum_size;
-		btrfs_calculate_block_csum_pages(fs_info, paddrs, csum_buf);
+		btrfs_csum_one_bio_block(fs_info, bio, &iter, csum_buf);
 		if (unlikely(memcmp(csum_buf, expected_csum, fs_info->csum_size) != 0))
 			set_bit(total_sector_nr, rbio->error_bitmap);
-		total_sector_nr++;
 	}
 }
 
@@ -1879,6 +1863,27 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc)
 	start_async_work(rbio, rmw_rbio_work);
 }
 
+static void calculate_block_csum_paddrs(struct btrfs_fs_info *fs_info,
+					const phys_addr_t paddrs[], u8 *dest)
+{
+	const u32 blocksize = fs_info->sectorsize;
+	const u32 step = min(blocksize, PAGE_SIZE);
+	const u32 nr_steps = blocksize / step;
+	struct btrfs_csum_ctx csum;
+
+	btrfs_csum_init(&csum, fs_info->csum_type);
+	for (int i = 0; i < nr_steps; i++) {
+		const phys_addr_t paddr = paddrs[i];
+		void *kaddr;
+
+		ASSERT(offset_in_page(paddr) + step <= PAGE_SIZE);
+		kaddr = kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr);
+		btrfs_csum_update(&csum, kaddr, step);
+		kunmap_local(kaddr);
+	}
+	btrfs_csum_final(&csum, dest);
+}
+
 static int verify_one_sector(struct btrfs_raid_bio *rbio,
 			     int stripe_nr, int sector_nr)
 {
@@ -1906,7 +1911,7 @@ static int verify_one_sector(struct btrfs_raid_bio *rbio,
 	csum_expected = rbio->csum_buf +
 			(stripe_nr * rbio->stripe_nsectors + sector_nr) *
 			fs_info->csum_size;
-	btrfs_calculate_block_csum_pages(fs_info, paddrs, csum_buf);
+	calculate_block_csum_paddrs(fs_info, paddrs, csum_buf);
 	if (unlikely(memcmp(csum_buf, csum_expected, fs_info->csum_size) != 0))
 		return -EIO;
 	return 0;
@@ -2624,7 +2629,7 @@ static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio)
 	return 0;
 }
 
-/* Return true if the content of the step matches the caclulated one. */
+/* Return true if the content of the step matches the calculated one. */
 static bool verify_one_parity_step(struct btrfs_raid_bio *rbio,
 				   void *pointers[], unsigned int sector_nr,
 				   unsigned int step_nr)
diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c
index da54db75e7a9b9..630a7ad8f8e1ac 100644
--- a/fs/btrfs/relocation.c
+++ b/fs/btrfs/relocation.c
@@ -3357,7 +3357,7 @@ static int delete_block_group_cache(struct btrfs_block_group *block_group,
 		goto out;
 	}
 
-	ret = btrfs_truncate_free_space_cache(trans, block_group, inode);
+	ret = btrfs_truncate_free_space_cache(trans, inode);
 
 	btrfs_end_transaction(trans);
 	btrfs_btree_balance_dirty(fs_info);
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index 5c59b9abedcd71..c523bf950c898f 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -7023,7 +7023,7 @@ static int changed_extent(struct send_ctx *sctx,
 	 * get modified or replaced with a new one). Note that deduplication
 	 * updates the inode item, but it only changes the iversion (sequence
 	 * field in the inode item) of the inode, so if a file is deduplicated
-	 * the same amount of times in both the parent and send snapshots, its
+	 * the same number of times in both the parent and send snapshots, its
 	 * iversion becomes the same in both snapshots, whence the inode item is
 	 * the same on both snapshots.
 	 */
diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c
index 39a28e1bec8ad8..01018152c054b0 100644
--- a/fs/btrfs/space-info.c
+++ b/fs/btrfs/space-info.c
@@ -1704,7 +1704,6 @@ static int handle_reserve_ticket(struct btrfs_space_info *space_info,
 						evict_flush_states,
 						ARRAY_SIZE(evict_flush_states));
 		break;
-	case BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE:
 	case BTRFS_RESERVE_FLUSH_ZONED_RELOCATION:
 		priority_reclaim_data_space(space_info, ticket);
 		break;
@@ -1968,7 +1967,6 @@ int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes,
 	int ret;
 
 	ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA ||
-	       flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE ||
 	       flush == BTRFS_RESERVE_FLUSH_ZONED_RELOCATION ||
 	       flush == BTRFS_RESERVE_NO_FLUSH, "flush=%d", flush);
 	ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA,
diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h
index aa836e8a9d4a6f..d0130c8ba3ddaf 100644
--- a/fs/btrfs/space-info.h
+++ b/fs/btrfs/space-info.h
@@ -66,7 +66,6 @@ enum btrfs_reserve_flush_enum {
 	 * Can be interrupted by a fatal signal.
 	 */
 	BTRFS_RESERVE_FLUSH_DATA,
-	BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE,
 	BTRFS_RESERVE_FLUSH_ALL,
 
 	/*
@@ -82,9 +81,6 @@ enum btrfs_reserve_flush_enum {
 	 * priority flushing for this, because otherwise we can deadlock on
 	 * waiting for a ticket, that cannot be granted, because we cannot do
 	 * any allocations.
-	 *
-	 * Apart from being specific to zoned relocation, it is equal to
-	 * BTRFS_FLUSH_FREE_SPACE_INODE.
 	 */
 	BTRFS_RESERVE_FLUSH_ZONED_RELOCATION,
 
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 464129b1b0d4cb..14ed0ed823a3c6 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -515,7 +515,6 @@ static int btrfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
 			btrfs_warn(NULL,
 			"v1 space cache is deprecated, falling back to no space cache");
 		btrfs_set_opt(ctx->mount_opt, NOSPACECACHE);
-		btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE);
 		btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE);
 		break;
 	case Opt_space_cache_version:
@@ -524,11 +523,9 @@ static int btrfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
 			btrfs_warn(NULL,
 			"v1 space cache is deprecated, falling back to no space cache");
 			btrfs_set_opt(ctx->mount_opt, NOSPACECACHE);
-			btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE);
 			btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE);
 			break;
 		case Opt_space_cache_v2:
-			btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE);
 			btrfs_set_opt(ctx->mount_opt, FREE_SPACE_TREE);
 			break;
 		default:
@@ -705,13 +702,6 @@ bool btrfs_check_options(const struct btrfs_fs_info *info,
 	if (btrfs_check_mountopts_zoned(info, mount_opt))
 		ret = false;
 
-	if (!test_bit(BTRFS_FS_STATE_REMOUNTING, &info->fs_state)) {
-		if (btrfs_raw_test_opt(*mount_opt, SPACE_CACHE)) {
-			btrfs_warn(info,
-"space cache v1 is being deprecated and will be removed in a future release, please use -o space_cache=v2");
-		}
-	}
-
 	return ret;
 }
 
@@ -729,14 +719,6 @@ bool btrfs_check_options(const struct btrfs_fs_info *info,
  */
 void btrfs_set_free_space_cache_settings(struct btrfs_fs_info *fs_info)
 {
-	if (fs_info->sectorsize != PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) {
-		btrfs_info(fs_info,
-			   "forcing free space tree for sector size %u with page size %lu",
-			   fs_info->sectorsize, PAGE_SIZE);
-		btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
-		btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE);
-	}
-
 	/*
 	 * At this point our mount options are populated, so we only mess with
 	 * these settings if we don't have any settings already.
@@ -751,20 +733,17 @@ void btrfs_set_free_space_cache_settings(struct btrfs_fs_info *fs_info)
 		return;
 	}
 
-	if (btrfs_test_opt(fs_info, SPACE_CACHE))
-		return;
-
 	if (btrfs_test_opt(fs_info, NOSPACECACHE))
 		return;
 
 	/*
 	 * At this point we don't have explicit options set by the user, set
-	 * them ourselves based on the state of the file system.
+	 * them ourselves based on the state of the file system. An existing
+	 * v1 space cache is no longer used and gets cleaned up once the
+	 * filesystem is mounted read-write.
 	 */
 	if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
 		btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE);
-	else if (btrfs_free_space_cache_v1_active(fs_info))
-		btrfs_set_opt(fs_info->mount_opt, SPACE_CACHE);
 }
 
 static void set_device_specific_options(struct btrfs_fs_info *fs_info)
@@ -1107,9 +1086,7 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
 		seq_puts(seq, ",discard=async");
 	if (!(info->sb->s_flags & SB_POSIXACL))
 		seq_puts(seq, ",noacl");
-	if (btrfs_free_space_cache_v1_active(info))
-		seq_puts(seq, ",space_cache");
-	else if (btrfs_fs_compat_ro(info, FREE_SPACE_TREE))
+	if (btrfs_fs_compat_ro(info, FREE_SPACE_TREE))
 		seq_puts(seq, ",space_cache=v2");
 	else
 		seq_puts(seq, ",nospace_cache");
@@ -1243,7 +1220,6 @@ static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info,
 	workqueue_set_max_active(fs_info->endio_workers, new_pool_size);
 	workqueue_set_max_active(fs_info->endio_meta_workers, new_pool_size);
 	btrfs_workqueue_set_max(fs_info->endio_write_workers, new_pool_size);
-	btrfs_workqueue_set_max(fs_info->endio_freespace_worker, new_pool_size);
 	btrfs_workqueue_set_max(fs_info->delayed_workers, new_pool_size);
 }
 
@@ -1264,8 +1240,6 @@ static inline void btrfs_remount_begin(struct btrfs_fs_info *fs_info,
 static inline void btrfs_remount_cleanup(struct btrfs_fs_info *fs_info,
 					 unsigned long long old_opts)
 {
-	const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE);
-
 	/*
 	 * We need to cleanup all defraggable inodes if the autodefragment is
 	 * close or the filesystem is read only.
@@ -1282,10 +1256,6 @@ static inline void btrfs_remount_cleanup(struct btrfs_fs_info *fs_info,
 	else if (btrfs_raw_test_opt(old_opts, DISCARD_ASYNC) &&
 		 !btrfs_test_opt(fs_info, DISCARD_ASYNC))
 		btrfs_discard_cleanup(fs_info);
-
-	/* If we toggled space cache */
-	if (cache_opt != btrfs_free_space_cache_v1_active(fs_info))
-		btrfs_set_free_space_cache_v1_active(fs_info, cache_opt);
 }
 
 static int btrfs_remount_rw(struct btrfs_fs_info *fs_info)
@@ -1448,7 +1418,6 @@ static void btrfs_emit_options(struct btrfs_fs_info *info,
 	btrfs_info_if_set(info, old, DISCARD_SYNC, "turning on sync discard");
 	btrfs_info_if_set(info, old, DISCARD_ASYNC, "turning on async discard");
 	btrfs_info_if_set(info, old, FREE_SPACE_TREE, "enabling free space tree");
-	btrfs_info_if_set(info, old, SPACE_CACHE, "enabling disk space caching");
 	btrfs_info_if_set(info, old, CLEAR_CACHE, "force clearing of disk cache");
 	btrfs_info_if_set(info, old, AUTO_DEFRAG, "enabling auto defrag");
 	btrfs_info_if_set(info, old, FRAGMENT_DATA, "fragmenting data");
@@ -1466,7 +1435,6 @@ static void btrfs_emit_options(struct btrfs_fs_info *info,
 	btrfs_info_if_unset(info, old, SSD_SPREAD, "not using spread ssd allocation scheme");
 	btrfs_info_if_unset(info, old, NOBARRIER, "turning on barriers");
 	btrfs_info_if_unset(info, old, NOTREELOG, "enabling tree log");
-	btrfs_info_if_unset(info, old, SPACE_CACHE, "disabling disk space caching");
 	btrfs_info_if_unset(info, old, FREE_SPACE_TREE, "disabling free space tree");
 	btrfs_info_if_unset(info, old, AUTO_DEFRAG, "disabling auto defrag");
 	btrfs_info_if_unset(info, old, COMPRESS, "use no compression");
@@ -1531,14 +1499,8 @@ static int btrfs_reconfigure(struct fs_context *fc)
 		btrfs_warn(fs_info,
 		"remount supports changing free space tree only from RO to RW");
 		/* Make sure free space cache options match the state on disk. */
-		if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
+		if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
 			btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE);
-			btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
-		}
-		if (btrfs_free_space_cache_v1_active(fs_info)) {
-			btrfs_clear_opt(fs_info->mount_opt, FREE_SPACE_TREE);
-			btrfs_set_opt(fs_info->mount_opt, SPACE_CACHE);
-		}
 	}
 
 	ret = 0;
@@ -1836,8 +1798,12 @@ static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf)
 	f_fsid.val[0] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root) >> 32;
 	f_fsid.val[1] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root);
 
-	/* Hash dev_t to avoid f_fsid collision with cloned filesystems. */
-	if (fs_info->fs_devices->total_devices == 1) {
+	/*
+	 * Hash dev_t to avoid f_fsid collisions with cloned filesystems.
+	 * Only do this when a clone is present so the original filesystem
+	 * (mounted first) maintains backward-compatible f_fsid behavior.
+	 */
+	if (fs_info->fs_devices->temp_fsid) {
 		__kernel_fsid_t dev_fsid =
 			u64_to_fsid(huge_encode_dev(fs_info->fs_devices->latest_dev->bdev->bd_dev));
 
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 39cb01ee441ab8..c5bb1c7eac6afe 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -83,8 +83,7 @@ struct raid_kobject {
 #define BTRFS_FEAT_ATTR(_name, _feature_set, _feature_prefix, _feature_bit)  \
 static struct btrfs_feature_attr btrfs_attr_features_##_name = {	     \
 	.kobj_attr = __INIT_KOBJ_ATTR(_name, S_IRUGO,			     \
-				      btrfs_feature_attr_show,		     \
-				      btrfs_feature_attr_store),	     \
+				      btrfs_feature_attr_show, NULL),	     \
 	.feature_set	= _feature_set,					     \
 	.feature_bit	= _feature_prefix ##_## _feature_bit,		     \
 }
@@ -130,130 +129,20 @@ static u64 get_features(struct btrfs_fs_info *fs_info,
 		return btrfs_super_incompat_flags(disk_super);
 }
 
-static void set_features(struct btrfs_fs_info *fs_info,
-			 enum btrfs_feature_set set, u64 features)
-{
-	struct btrfs_super_block *disk_super = fs_info->super_copy;
-	if (set == FEAT_COMPAT)
-		btrfs_set_super_compat_flags(disk_super, features);
-	else if (set == FEAT_COMPAT_RO)
-		btrfs_set_super_compat_ro_flags(disk_super, features);
-	else
-		btrfs_set_super_incompat_flags(disk_super, features);
-}
-
-static int can_modify_feature(struct btrfs_feature_attr *fa)
-{
-	int val = 0;
-	u64 set, clear;
-	switch (fa->feature_set) {
-	case FEAT_COMPAT:
-		set = BTRFS_FEATURE_COMPAT_SAFE_SET;
-		clear = BTRFS_FEATURE_COMPAT_SAFE_CLEAR;
-		break;
-	case FEAT_COMPAT_RO:
-		set = BTRFS_FEATURE_COMPAT_RO_SAFE_SET;
-		clear = BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR;
-		break;
-	case FEAT_INCOMPAT:
-		set = BTRFS_FEATURE_INCOMPAT_SAFE_SET;
-		clear = BTRFS_FEATURE_INCOMPAT_SAFE_CLEAR;
-		break;
-	default:
-		btrfs_warn(NULL, "sysfs: unknown feature set %d", fa->feature_set);
-		return 0;
-	}
-
-	if (set & fa->feature_bit)
-		val |= 1;
-	if (clear & fa->feature_bit)
-		val |= 2;
-
-	return val;
-}
-
 static ssize_t btrfs_feature_attr_show(struct kobject *kobj,
 				       struct kobj_attribute *a, char *buf)
 {
 	int val = 0;
 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
 	struct btrfs_feature_attr *fa = to_btrfs_feature_attr(a);
+
 	if (fs_info) {
 		u64 features = get_features(fs_info, fa->feature_set);
 		if (features & fa->feature_bit)
 			val = 1;
-	} else
-		val = can_modify_feature(fa);
-
-	return sysfs_emit(buf, "%d\n", val);
-}
-
-static ssize_t btrfs_feature_attr_store(struct kobject *kobj,
-					struct kobj_attribute *a,
-					const char *buf, size_t count)
-{
-	struct btrfs_fs_info *fs_info;
-	struct btrfs_feature_attr *fa = to_btrfs_feature_attr(a);
-	u64 features, set, clear;
-	unsigned long val;
-	int ret;
-
-	fs_info = to_fs_info(kobj);
-	if (!fs_info)
-		return -EPERM;
-
-	if (sb_rdonly(fs_info->sb))
-		return -EROFS;
-
-	ret = kstrtoul(skip_spaces(buf), 0, &val);
-	if (ret)
-		return ret;
-
-	if (fa->feature_set == FEAT_COMPAT) {
-		set = BTRFS_FEATURE_COMPAT_SAFE_SET;
-		clear = BTRFS_FEATURE_COMPAT_SAFE_CLEAR;
-	} else if (fa->feature_set == FEAT_COMPAT_RO) {
-		set = BTRFS_FEATURE_COMPAT_RO_SAFE_SET;
-		clear = BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR;
-	} else {
-		set = BTRFS_FEATURE_INCOMPAT_SAFE_SET;
-		clear = BTRFS_FEATURE_INCOMPAT_SAFE_CLEAR;
-	}
-
-	features = get_features(fs_info, fa->feature_set);
-
-	/* Nothing to do */
-	if ((val && (features & fa->feature_bit)) ||
-	    (!val && !(features & fa->feature_bit)))
-		return count;
-
-	if ((val && !(set & fa->feature_bit)) ||
-	    (!val && !(clear & fa->feature_bit))) {
-		btrfs_info(fs_info,
-			"%sabling feature %s on mounted fs is not supported.",
-			val ? "En" : "Dis", fa->kobj_attr.attr.name);
-		return -EPERM;
 	}
 
-	btrfs_info(fs_info, "%s %s feature flag",
-		   val ? "Setting" : "Clearing", fa->kobj_attr.attr.name);
-
-	spin_lock(&fs_info->super_lock);
-	features = get_features(fs_info, fa->feature_set);
-	if (val)
-		features |= fa->feature_bit;
-	else
-		features &= ~fa->feature_bit;
-	set_features(fs_info, fa->feature_set, features);
-	spin_unlock(&fs_info->super_lock);
-
-	/*
-	 * We don't want to do full transaction commit from inside sysfs
-	 */
-	set_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags);
-	wake_up_process(fs_info->transaction_kthread);
-
-	return count;
+	return sysfs_emit(buf, "%d\n", val);
 }
 
 static umode_t btrfs_feature_visible(struct kobject *kobj,
@@ -269,9 +158,7 @@ static umode_t btrfs_feature_visible(struct kobject *kobj,
 		fa = attr_to_btrfs_feature_attr(attr);
 		features = get_features(fs_info, fa->feature_set);
 
-		if (can_modify_feature(fa))
-			mode |= S_IWUSR;
-		else if (!(features & fa->feature_bit))
+		if (!(features & fa->feature_bit))
 			mode = 0;
 	}
 
@@ -2359,9 +2246,7 @@ static ssize_t qgroup_enabled_show(struct kobject *qgroups_kobj,
 	struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent);
 	bool enabled;
 
-	spin_lock(&fs_info->qgroup_lock);
-	enabled = fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_ON;
-	spin_unlock(&fs_info->qgroup_lock);
+	enabled = test_bit(BTRFS_QGROUP_STATUS_BIT_ON, &fs_info->qgroup_flags);
 
 	return sysfs_emit(buf, "%d\n", enabled);
 }
@@ -2401,9 +2286,7 @@ static ssize_t qgroup_inconsistent_show(struct kobject *qgroups_kobj,
 	struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent);
 	bool inconsistent;
 
-	spin_lock(&fs_info->qgroup_lock);
-	inconsistent = (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT);
-	spin_unlock(&fs_info->qgroup_lock);
+	inconsistent = test_bit(BTRFS_QGROUP_STATUS_BIT_INCONSISTENT, &fs_info->qgroup_flags);
 
 	return sysfs_emit(buf, "%d\n", inconsistent);
 }
diff --git a/fs/btrfs/tests/extent-io-tests.c b/fs/btrfs/tests/extent-io-tests.c
index 23459cd4e50388..cd045778400dc0 100644
--- a/fs/btrfs/tests/extent-io-tests.c
+++ b/fs/btrfs/tests/extent-io-tests.c
@@ -18,8 +18,8 @@
 #define PROCESS_RELEASE		(1U << 1)
 #define PROCESS_TEST_LOCKED	(1U << 2)
 
-static noinline int process_page_range(struct inode *inode, u64 start, u64 end,
-				       unsigned long flags)
+static noinline int process_folio_range(struct inode *inode, u64 start, u64 end,
+					unsigned long flags)
 {
 	int ret;
 	struct folio_batch fbatch;
@@ -112,8 +112,8 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 	struct btrfs_root *root = NULL;
 	struct inode *inode = NULL;
 	struct extent_io_tree *tmp;
-	struct page *page;
-	struct page *locked_page = NULL;
+	struct folio *folio;
+	struct folio *locked_folio = NULL;
 	/* In this test we need at least 2 file extents at its maximum size */
 	u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
 	u64 total_dirty = 2 * max_bytes;
@@ -152,23 +152,27 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 	btrfs_extent_io_tree_init(NULL, tmp, IO_TREE_SELFTEST);
 
 	/*
-	 * First go through and create and mark all of our pages dirty, we pin
-	 * everything to make sure our pages don't get evicted and screw up our
+	 * First go through and create and mark all of our folios dirty, we pin
+	 * everything to make sure our folios don't get evicted and screw up our
 	 * test.
 	 */
 	for (pgoff_t index = 0; index < (total_dirty >> PAGE_SHIFT); index++) {
-		page = find_or_create_page(inode->i_mapping, index, GFP_KERNEL);
-		if (!page) {
-			test_err("failed to allocate test page");
-			ret = -ENOMEM;
+		folio = __filemap_get_folio(inode->i_mapping, index,
+					    FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
+					    GFP_KERNEL);
+		if (IS_ERR(folio)) {
+			test_err("failed to allocate test folio");
+			ret = PTR_ERR(folio);
 			goto out;
 		}
-		SetPageDirty(page);
+		/* The ranges below assume page sized folios. */
+		ASSERT(folio_order(folio) == 0);
+		folio_set_dirty(folio);
 		if (index) {
-			unlock_page(page);
+			folio_unlock(folio);
 		} else {
-			get_page(page);
-			locked_page = page;
+			folio_get(folio);
+			locked_folio = folio;
 		}
 	}
 
@@ -179,8 +183,7 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 	btrfs_set_extent_bit(tmp, 0, sectorsize - 1, EXTENT_DELALLOC, NULL);
 	start = 0;
 	end = start + PAGE_SIZE - 1;
-	found = find_lock_delalloc_range(inode, page_folio(locked_page), &start,
-					 &end);
+	found = find_lock_delalloc_range(inode, locked_folio, &start, &end);
 	if (!found) {
 		test_err("should have found at least one delalloc");
 		goto out_bits;
@@ -191,8 +194,8 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 		goto out_bits;
 	}
 	btrfs_unlock_extent(tmp, start, end, NULL);
-	unlock_page(locked_page);
-	put_page(locked_page);
+	folio_unlock(locked_folio);
+	folio_put(locked_folio);
 
 	/*
 	 * Test this scenario
@@ -201,17 +204,17 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 	 *           |--- search ---|
 	 */
 	test_start = SZ_64M;
-	locked_page = find_lock_page(inode->i_mapping,
-				     test_start >> PAGE_SHIFT);
-	if (!locked_page) {
-		test_err("couldn't find the locked page");
+	locked_folio = filemap_lock_folio(inode->i_mapping, test_start >> PAGE_SHIFT);
+	if (IS_ERR(locked_folio)) {
+		test_err("couldn't find the locked folio");
+		locked_folio = NULL;
 		goto out_bits;
 	}
+	ASSERT(folio_order(locked_folio) == 0);
 	btrfs_set_extent_bit(tmp, sectorsize, max_bytes - 1, EXTENT_DELALLOC, NULL);
 	start = test_start;
 	end = start + PAGE_SIZE - 1;
-	found = find_lock_delalloc_range(inode, page_folio(locked_page), &start,
-					 &end);
+	found = find_lock_delalloc_range(inode, locked_folio, &start, &end);
 	if (!found) {
 		test_err("couldn't find delalloc in our range");
 		goto out_bits;
@@ -221,14 +224,14 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 				test_start, max_bytes - 1, start, end);
 		goto out_bits;
 	}
-	if (process_page_range(inode, start, end,
-			       PROCESS_TEST_LOCKED | PROCESS_UNLOCK)) {
-		test_err("there were unlocked pages in the range");
+	if (process_folio_range(inode, start, end,
+				PROCESS_TEST_LOCKED | PROCESS_UNLOCK)) {
+		test_err("there were unlocked folios in the range");
 		goto out_bits;
 	}
 	btrfs_unlock_extent(tmp, start, end, NULL);
-	/* locked_page was unlocked above */
-	put_page(locked_page);
+	/* locked_folio was unlocked above */
+	folio_put(locked_folio);
 
 	/*
 	 * Test this scenario
@@ -236,16 +239,16 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 	 *                    |--- search ---|
 	 */
 	test_start = max_bytes + sectorsize;
-	locked_page = find_lock_page(inode->i_mapping, test_start >>
-				     PAGE_SHIFT);
-	if (!locked_page) {
-		test_err("couldn't find the locked page");
+	locked_folio = filemap_lock_folio(inode->i_mapping, test_start >> PAGE_SHIFT);
+	if (IS_ERR(locked_folio)) {
+		test_err("couldn't find the locked folio");
+		locked_folio = NULL;
 		goto out_bits;
 	}
+	ASSERT(folio_order(locked_folio) == 0);
 	start = test_start;
 	end = start + PAGE_SIZE - 1;
-	found = find_lock_delalloc_range(inode, page_folio(locked_page), &start,
-					 &end);
+	found = find_lock_delalloc_range(inode, locked_folio, &start, &end);
 	if (found) {
 		test_err("found range when we shouldn't have");
 		goto out_bits;
@@ -265,8 +268,7 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 	btrfs_set_extent_bit(tmp, max_bytes, total_dirty - 1, EXTENT_DELALLOC, NULL);
 	start = test_start;
 	end = start + PAGE_SIZE - 1;
-	found = find_lock_delalloc_range(inode, page_folio(locked_page), &start,
-					 &end);
+	found = find_lock_delalloc_range(inode, locked_folio, &start, &end);
 	if (!found) {
 		test_err("didn't find our range");
 		goto out_bits;
@@ -276,38 +278,37 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 			 test_start, total_dirty - 1, start, end);
 		goto out_bits;
 	}
-	if (process_page_range(inode, start, end,
-			       PROCESS_TEST_LOCKED | PROCESS_UNLOCK)) {
-		test_err("pages in range were not all locked");
+	if (process_folio_range(inode, start, end,
+				PROCESS_TEST_LOCKED | PROCESS_UNLOCK)) {
+		test_err("folios in range were not all locked");
 		goto out_bits;
 	}
 	btrfs_unlock_extent(tmp, start, end, NULL);
 
 	/*
-	 * Now to test where we run into a page that is no longer dirty in the
+	 * Now to test where we run into a folio that is no longer dirty in the
 	 * range we want to find.
 	 */
-	page = find_get_page(inode->i_mapping,
-			     (max_bytes + SZ_1M) >> PAGE_SHIFT);
-	if (!page) {
-		test_err("couldn't find our page");
+	folio = filemap_get_folio(inode->i_mapping, (max_bytes + SZ_1M) >> PAGE_SHIFT);
+	if (IS_ERR(folio)) {
+		test_err("couldn't find our folio");
 		goto out_bits;
 	}
-	ClearPageDirty(page);
-	put_page(page);
+	ASSERT(folio_order(folio) == 0);
+	folio_clear_dirty(folio);
+	folio_put(folio);
 
 	/* We unlocked it in the previous test */
-	lock_page(locked_page);
+	folio_lock(locked_folio);
 	start = test_start;
 	end = start + PAGE_SIZE - 1;
 	/*
-	 * Currently if we fail to find dirty pages in the delalloc range we
+	 * Currently if we fail to find dirty folios in the delalloc range we
 	 * will adjust max_bytes down to PAGE_SIZE and then re-search.  If
 	 * this changes at any point in the future we will need to fix this
 	 * tests expected behavior.
 	 */
-	found = find_lock_delalloc_range(inode, page_folio(locked_page), &start,
-					 &end);
+	found = find_lock_delalloc_range(inode, locked_folio, &start, &end);
 	if (!found) {
 		test_err("didn't find our range");
 		goto out_bits;
@@ -317,9 +318,9 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 			 test_start, test_start + PAGE_SIZE - 1, start, end);
 		goto out_bits;
 	}
-	if (process_page_range(inode, start, end, PROCESS_TEST_LOCKED |
-			       PROCESS_UNLOCK)) {
-		test_err("pages in range were not all locked");
+	if (process_folio_range(inode, start, end, PROCESS_TEST_LOCKED |
+				PROCESS_UNLOCK)) {
+		test_err("folios in range were not all locked");
 		goto out_bits;
 	}
 	ret = 0;
@@ -328,10 +329,10 @@ static int test_find_delalloc(u32 sectorsize, u32 nodesize)
 		dump_extent_io_tree(tmp);
 	btrfs_clear_extent_bit(tmp, 0, total_dirty - 1, (unsigned)-1, NULL);
 out:
-	if (locked_page)
-		put_page(locked_page);
-	process_page_range(inode, 0, total_dirty - 1,
-			   PROCESS_UNLOCK | PROCESS_RELEASE);
+	if (locked_folio)
+		folio_put(locked_folio);
+	process_folio_range(inode, 0, total_dirty - 1,
+			    PROCESS_UNLOCK | PROCESS_RELEASE);
 	iput(inode);
 out_root_info:
 	btrfs_free_dummy_root(root);
@@ -671,8 +672,9 @@ static void dump_eb_and_memory_contents(struct extent_buffer *eb, void *memory,
 					const char *test_name)
 {
 	for (int i = 0; i < eb->len; i++) {
-		struct page *page = folio_page(eb->folios[i >> PAGE_SHIFT], 0);
-		void *addr = page_address(page) + offset_in_page(i);
+		const unsigned long idx = get_eb_folio_index(eb, i);
+		void *addr = folio_address(eb->folios[idx]) +
+			     get_eb_offset_in_folio(eb, i);
 
 		if (memcmp(addr, memory + i, 1) != 0) {
 			test_err("%s failed", test_name);
@@ -687,9 +689,12 @@ static int verify_eb_and_memory(struct extent_buffer *eb, void *memory,
 				const char *test_name)
 {
 	for (int i = 0; i < (eb->len >> PAGE_SHIFT); i++) {
-		void *eb_addr = folio_address(eb->folios[i]);
+		const unsigned long offset = i << PAGE_SHIFT;
+		const unsigned long idx = get_eb_folio_index(eb, offset);
+		void *eb_addr = folio_address(eb->folios[idx]) +
+				get_eb_offset_in_folio(eb, offset);
 
-		if (memcmp(memory + (i << PAGE_SHIFT), eb_addr, PAGE_SIZE) != 0) {
+		if (memcmp(memory + offset, eb_addr, PAGE_SIZE) != 0) {
 			dump_eb_and_memory_contents(eb, memory, test_name);
 			return -EUCLEAN;
 		}
diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c
index 6802b94ed76ff4..84f012bfcffca6 100644
--- a/fs/btrfs/transaction.c
+++ b/fs/btrfs/transaction.c
@@ -125,17 +125,14 @@ static const unsigned int btrfs_blocked_trans_types[TRANS_STATE_MAX] = {
 	[TRANS_STATE_UNBLOCKED]		= (__TRANS_START |
 					   __TRANS_ATTACH |
 					   __TRANS_JOIN |
-					   __TRANS_JOIN_NOLOCK |
 					   __TRANS_JOIN_NOSTART),
 	[TRANS_STATE_SUPER_COMMITTED]	= (__TRANS_START |
 					   __TRANS_ATTACH |
 					   __TRANS_JOIN |
-					   __TRANS_JOIN_NOLOCK |
 					   __TRANS_JOIN_NOSTART),
 	[TRANS_STATE_COMPLETED]		= (__TRANS_START |
 					   __TRANS_ATTACH |
 					   __TRANS_JOIN |
-					   __TRANS_JOIN_NOLOCK |
 					   __TRANS_JOIN_NOSTART),
 };
 
@@ -310,12 +307,6 @@ static noinline int join_transaction(struct btrfs_fs_info *fs_info,
 	if (type == TRANS_ATTACH || type == TRANS_JOIN_NOSTART)
 		return -ENOENT;
 
-	/*
-	 * JOIN_NOLOCK only happens during the transaction commit, so
-	 * it is impossible that ->running_transaction is NULL
-	 */
-	BUG_ON(type == TRANS_JOIN_NOLOCK);
-
 	cur_trans = kmalloc_obj(*cur_trans, GFP_NOFS);
 	if (!cur_trans)
 		return -ENOMEM;
@@ -379,9 +370,8 @@ static noinline int join_transaction(struct btrfs_fs_info *fs_info,
 	INIT_LIST_HEAD(&cur_trans->dev_update_list);
 	INIT_LIST_HEAD(&cur_trans->switch_commits);
 	INIT_LIST_HEAD(&cur_trans->dirty_bgs);
-	INIT_LIST_HEAD(&cur_trans->io_bgs);
 	INIT_LIST_HEAD(&cur_trans->dropped_roots);
-	mutex_init(&cur_trans->cache_write_mutex);
+	mutex_init(&cur_trans->dirty_bgs_update_mutex);
 	spin_lock_init(&cur_trans->dirty_bgs_lock);
 	INIT_LIST_HEAD(&cur_trans->deleted_bgs);
 	spin_lock_init(&cur_trans->dropped_roots_lock);
@@ -710,14 +700,8 @@ start_transaction(struct btrfs_root *root, unsigned int num_items,
 	}
 
 	/*
-	 * If we are JOIN_NOLOCK we're already committing a transaction and
-	 * waiting on this guy, so we don't need to do the sb_start_intwrite
-	 * because we're already holding a ref.  We need this because we could
-	 * have raced in and did an fsync() on a file which can kick a commit
-	 * and then we deadlock with somebody doing a freeze.
-	 *
 	 * If we are ATTACH, it means we just want to catch the current
-	 * transaction and commit it, so we needn't do sb_start_intwrite(). 
+	 * transaction and commit it, so we needn't do sb_start_intwrite().
 	 */
 	if (type & __TRANS_FREEZABLE)
 		sb_start_intwrite(fs_info->sb);
@@ -855,12 +839,6 @@ struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root)
 				 true);
 }
 
-struct btrfs_trans_handle *btrfs_join_transaction_spacecache(struct btrfs_root *root)
-{
-	return start_transaction(root, 0, TRANS_JOIN_NOLOCK,
-				 BTRFS_RESERVE_NO_FLUSH, true);
-}
-
 /*
  * Similar to regular join but it never starts a transaction when none is
  * running or when there's a running one at a state >= TRANS_STATE_UNBLOCKED.
@@ -1363,7 +1341,6 @@ static noinline int commit_cowonly_roots(struct btrfs_trans_handle *trans)
 {
 	struct btrfs_fs_info *fs_info = trans->fs_info;
 	struct list_head *dirty_bgs = &trans->transaction->dirty_bgs;
-	struct list_head *io_bgs = &trans->transaction->io_bgs;
 	struct extent_buffer *eb;
 	int ret;
 
@@ -1393,10 +1370,6 @@ static noinline int commit_cowonly_roots(struct btrfs_trans_handle *trans)
 	if (ret)
 		return ret;
 
-	ret = btrfs_setup_space_cache(trans);
-	if (ret)
-		return ret;
-
 again:
 	while (!list_empty(&fs_info->dirty_cowonly_roots)) {
 		struct btrfs_root *root;
@@ -1417,7 +1390,7 @@ static noinline int commit_cowonly_roots(struct btrfs_trans_handle *trans)
 	if (ret)
 		return ret;
 
-	while (!list_empty(dirty_bgs) || !list_empty(io_bgs)) {
+	while (!list_empty(dirty_bgs)) {
 		ret = btrfs_write_dirty_block_groups(trans);
 		if (ret)
 			return ret;
@@ -1890,8 +1863,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 		goto fail;
 
 	ret = btrfs_insert_dir_item(trans, &fname.disk_name,
-				    parent_inode, &key, BTRFS_FT_DIR,
-				    index);
+				    parent_inode, &key, BTRFS_FT_DIR, index, NULL);
 	if (unlikely(ret)) {
 		btrfs_abort_transaction(trans, ret);
 		goto fail;
@@ -1990,9 +1962,7 @@ static void update_super_roots(struct btrfs_fs_info *fs_info)
 	super->root = root_item->bytenr;
 	super->generation = root_item->generation;
 	super->root_level = root_item->level;
-	if (btrfs_test_opt(fs_info, SPACE_CACHE))
-		super->cache_generation = root_item->generation;
-	else if (test_bit(BTRFS_FS_CLEANUP_SPACE_CACHE_V1, &fs_info->flags))
+	if (test_bit(BTRFS_FS_CLEANUP_SPACE_CACHE_V1, &fs_info->flags))
 		super->cache_generation = 0;
 	if (test_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags))
 		super->uuid_tree_generation = root_item->generation;
@@ -2274,18 +2244,16 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
 	if (!test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &cur_trans->flags)) {
 		bool run_it = false;
 
-		/* this mutex is also taken before trying to set
-		 * block groups readonly.  We need to make sure
-		 * that nobody has set a block group readonly
-		 * after a extents from that block group have been
-		 * allocated for cache files.  btrfs_set_block_group_ro
-		 * will wait for the transaction to commit if it
-		 * finds BTRFS_TRANS_DIRTY_BG_RUN set.
+		/*
+		 * This mutex is also taken before trying to set block groups
+		 * readonly.  btrfs_inc_block_group_ro() will wait for the
+		 * transaction to commit if it finds BTRFS_TRANS_DIRTY_BG_RUN
+		 * set.
 		 *
 		 * The BTRFS_TRANS_DIRTY_BG_RUN flag is also used to make sure
-		 * only one process starts all the block group IO.  It wouldn't
-		 * hurt to have more than one go through, but there's no
-		 * real advantage to it either.
+		 * only one process starts all the block group item updates.  It
+		 * wouldn't hurt to have more than one go through, but there's
+		 * no real advantage to it either.
 		 */
 		mutex_lock(&fs_info->ro_block_group_mutex);
 		if (!test_and_set_bit(BTRFS_TRANS_DIRTY_BG_RUN,
@@ -2519,10 +2487,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
 	if (unlikely(ret))
 		goto unlock_reloc;
 
-	/*
-	 * The tasks which save the space cache and inode cache may also
-	 * update ->aborted, check it.
-	 */
+	/* Other tasks may also have updated ->aborted, check it. */
 	if (TRANS_ABORTED(cur_trans)) {
 		ret = cur_trans->aborted;
 		goto unlock_reloc;
@@ -2543,7 +2508,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans)
 	switch_commit_roots(trans);
 
 	ASSERT(list_empty(&cur_trans->dirty_bgs));
-	ASSERT(list_empty(&cur_trans->io_bgs));
 	update_super_roots(fs_info);
 
 	btrfs_set_super_log_root(fs_info->super_copy, 0);
diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h
index 3a57f227b5ed28..68c724e708095a 100644
--- a/fs/btrfs/transaction.h
+++ b/fs/btrfs/transaction.h
@@ -47,7 +47,6 @@ enum btrfs_trans_state {
 
 #define BTRFS_TRANS_HAVE_FREE_BGS	0
 #define BTRFS_TRANS_DIRTY_BG_RUN	1
-#define BTRFS_TRANS_CACHE_ENOSPC	2
 
 struct btrfs_transaction {
 	u64 transid;
@@ -78,32 +77,15 @@ struct btrfs_transaction {
 	struct list_head dev_update_list;
 	struct list_head switch_commits;
 	struct list_head dirty_bgs;
-
-	/*
-	 * There is no explicit lock which protects io_bgs, rather its
-	 * consistency is implied by the fact that all the sites which modify
-	 * it do so under some form of transaction critical section, namely:
-	 *
-	 * - btrfs_start_dirty_block_groups - This function can only ever be
-	 *   run by one of the transaction committers. Refer to
-	 *   BTRFS_TRANS_DIRTY_BG_RUN usage in btrfs_commit_transaction
-	 *
-	 * - btrfs_write_dirty_blockgroups - this is called by
-	 *   commit_cowonly_roots from transaction critical section
-	 *   (TRANS_STATE_COMMIT_DOING)
-	 *
-	 * - btrfs_cleanup_dirty_bgs - called on transaction abort
-	 */
-	struct list_head io_bgs;
 	struct list_head dropped_roots;
 	struct extent_io_tree pinned_extents;
 
 	/*
-	 * we need to make sure block group deletion doesn't race with
-	 * free space cache writeout.  This mutex keeps them from stomping
-	 * on each other
+	 * We need to make sure block group deletion doesn't race with the
+	 * dirty block group item updates done outside the commit critical
+	 * section. This mutex keeps them from stomping on each other.
 	 */
-	struct mutex cache_write_mutex;
+	struct mutex dirty_bgs_update_mutex;
 	spinlock_t dirty_bgs_lock;
 	/* Protected by spin lock fs_info->unused_bgs_lock. */
 	struct list_head deleted_bgs;
@@ -124,7 +106,6 @@ enum {
 	ENUM_BIT(__TRANS_START),
 	ENUM_BIT(__TRANS_ATTACH),
 	ENUM_BIT(__TRANS_JOIN),
-	ENUM_BIT(__TRANS_JOIN_NOLOCK),
 	ENUM_BIT(__TRANS_DUMMY),
 	ENUM_BIT(__TRANS_JOIN_NOSTART),
 };
@@ -132,7 +113,6 @@ enum {
 #define TRANS_START		(__TRANS_START | __TRANS_FREEZABLE)
 #define TRANS_ATTACH		(__TRANS_ATTACH)
 #define TRANS_JOIN		(__TRANS_JOIN | __TRANS_FREEZABLE)
-#define TRANS_JOIN_NOLOCK	(__TRANS_JOIN_NOLOCK)
 #define TRANS_JOIN_NOSTART	(__TRANS_JOIN_NOSTART)
 
 #define TRANS_EXTWRITERS	(__TRANS_START | __TRANS_ATTACH)
@@ -288,7 +268,7 @@ do {										\
  * Call btrfs_abort_transaction() as early as possible when an error condition
  * is detected, that way the exact stack trace is reported for some errors.
  *
- * Error number must be negative as it encodes wheather it's the first abort.
+ * Error number must be negative as it encodes whether it's the first abort.
  */
 #define btrfs_abort_transaction(trans, error)		\
 do {								\
@@ -312,7 +292,6 @@ struct btrfs_trans_handle *btrfs_start_transaction_fallback_global_rsv(
 					struct btrfs_root *root,
 					unsigned int num_items);
 struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root);
-struct btrfs_trans_handle *btrfs_join_transaction_spacecache(struct btrfs_root *root);
 struct btrfs_trans_handle *btrfs_join_transaction_nostart(struct btrfs_root *root);
 struct btrfs_trans_handle *btrfs_attach_transaction(struct btrfs_root *root);
 struct btrfs_trans_handle *btrfs_attach_transaction_barrier(
diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
index ab5abbb475e2ca..9cd79d97b9b5e7 100644
--- a/fs/btrfs/tree-checker.c
+++ b/fs/btrfs/tree-checker.c
@@ -108,13 +108,14 @@ static void file_extent_err(const struct extent_buffer *eb, int slot,
  */
 #define CHECK_FE_ALIGNED(leaf, slot, fi, name, alignment)		      \
 ({									      \
-	if (unlikely(!IS_ALIGNED(btrfs_file_extent_##name((leaf), (fi)),      \
-				 (alignment))))				      \
+	const u64 val = btrfs_file_extent_##name((leaf), (fi));               \
+	const bool not_aligned = !IS_ALIGNED(val, (alignment));		      \
+									      \
+	if (unlikely(not_aligned))			      		      \
 		file_extent_err((leaf), (slot),				      \
 	"invalid %s for file extent, have %llu, should be aligned to %u",     \
-			(#name), btrfs_file_extent_##name((leaf), (fi)),      \
-			(alignment));					      \
-	(!IS_ALIGNED(btrfs_file_extent_##name((leaf), (fi)), (alignment)));   \
+			(#name), val, (alignment));			      \
+	not_aligned;			                                      \
 })
 
 static u64 file_extent_end(struct extent_buffer *leaf,
@@ -163,6 +164,12 @@ static void dir_item_err(const struct extent_buffer *eb, int slot,
 	va_end(args);
 }
 
+/* Record info for the last hit inode. */
+struct saved_inode_info {
+	u64 ino;
+	u32 mode;
+};
+
 /*
  * This functions checks prev_key->objectid, to ensure current key and prev_key
  * share the same objectid as inode number.
@@ -204,15 +211,41 @@ static bool check_prev_ino(struct extent_buffer *leaf,
 		prev_key->objectid, key->objectid);
 	return false;
 }
+
+static bool can_have_extent_data(struct extent_buffer *leaf,
+				 struct btrfs_key *key, int slot, u8 fi_type,
+				 const struct saved_inode_info *inode_info)
+{
+	/* No inode item in this leaf. */
+	if (inode_info->ino != key->objectid)
+		return true;
+	if (S_ISREG(inode_info->mode))
+		return true;
+	if (S_ISLNK(inode_info->mode)) {
+		/* For a symlink, the file extent item should always be inlined. */
+		if (unlikely(fi_type != BTRFS_FILE_EXTENT_INLINE))
+			return false;
+		return true;
+	}
+
+	/*
+	 * The rest are special files, e.g. block/FIFO files, which cannot
+	 * have any file extent.
+	 */
+	return false;
+}
+
 static int check_extent_data_item(struct extent_buffer *leaf,
 				  struct btrfs_key *key, int slot,
-				  struct btrfs_key *prev_key)
+				  struct btrfs_key *prev_key,
+				  const struct saved_inode_info *inode_info)
 {
 	struct btrfs_fs_info *fs_info = leaf->fs_info;
 	struct btrfs_file_extent_item *fi;
 	u32 sectorsize = fs_info->sectorsize;
 	u32 item_size = btrfs_item_size(leaf, slot);
 	u64 extent_end;
+	u8 fi_type;
 
 	if (unlikely(!IS_ALIGNED(key->offset, sectorsize))) {
 		file_extent_err(leaf, slot,
@@ -243,12 +276,18 @@ static int check_extent_data_item(struct extent_buffer *leaf,
 				SZ_4K);
 		return -EUCLEAN;
 	}
-	if (unlikely(btrfs_file_extent_type(leaf, fi) >=
-		     BTRFS_NR_FILE_EXTENT_TYPES)) {
+	fi_type = btrfs_file_extent_type(leaf, fi);
+	if (unlikely(fi_type >= BTRFS_NR_FILE_EXTENT_TYPES)) {
 		file_extent_err(leaf, slot,
 		"invalid type for file extent, have %u expect range [0, %u]",
-			btrfs_file_extent_type(leaf, fi),
-			BTRFS_NR_FILE_EXTENT_TYPES - 1);
+			fi_type, BTRFS_NR_FILE_EXTENT_TYPES - 1);
+		return -EUCLEAN;
+	}
+
+	if (unlikely(!can_have_extent_data(leaf, key, slot, fi_type, inode_info))) {
+		file_extent_err(leaf, slot,
+				"unexpected file extent item type %u for inode mode 0%o",
+				fi_type, inode_info->mode);
 		return -EUCLEAN;
 	}
 
@@ -270,7 +309,8 @@ static int check_extent_data_item(struct extent_buffer *leaf,
 			btrfs_file_extent_encryption(leaf, fi));
 		return -EUCLEAN;
 	}
-	if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) {
+
+	if (fi_type == BTRFS_FILE_EXTENT_INLINE) {
 		/* Inline extent must have 0 as key offset */
 		if (unlikely(key->offset)) {
 			file_extent_err(leaf, slot,
@@ -1081,9 +1121,9 @@ int btrfs_check_chunk_valid(const struct btrfs_fs_info *fs_info,
 		return -EUCLEAN;
 	}
 
-	if (!remapped &&
-	    !valid_stripe_count(type & BTRFS_BLOCK_GROUP_PROFILE_MASK,
-				num_stripes, sub_stripes)) {
+	if (unlikely(!remapped &&
+		     !valid_stripe_count(type & BTRFS_BLOCK_GROUP_PROFILE_MASK,
+					 num_stripes, sub_stripes))) {
 		chunk_err(fs_info, leaf, chunk, logical,
 			"invalid num_stripes:sub_stripes %u:%u for profile %llu",
 			num_stripes, sub_stripes,
@@ -1206,7 +1246,8 @@ static int check_dev_item(struct extent_buffer *leaf,
 }
 
 static int check_inode_item(struct extent_buffer *leaf,
-			    struct btrfs_key *key, int slot)
+			    struct btrfs_key *key, int slot,
+			    struct saved_inode_info *inode_info)
 {
 	struct btrfs_fs_info *fs_info = leaf->fs_info;
 	struct btrfs_inode_item *iitem;
@@ -1291,6 +1332,8 @@ static int check_inode_item(struct extent_buffer *leaf,
 			ro_flags);
 		return -EUCLEAN;
 	}
+	inode_info->ino = key->objectid;
+	inode_info->mode = mode;
 	return 0;
 }
 
@@ -2129,7 +2172,7 @@ static int check_dev_extent_item(const struct extent_buffer *leaf,
 				 sectorsize))) {
 		generic_err(leaf, slot,
 			    "invalid dev extent chunk offset, has %llu not aligned to %u",
-			    btrfs_dev_extent_chunk_objectid(leaf, de),
+			    btrfs_dev_extent_chunk_offset(leaf, de),
 			    sectorsize);
 		return -EUCLEAN;
 	}
@@ -2306,7 +2349,7 @@ static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key
 
 	if (unlikely(btrfs_item_size(leaf, slot) != 0)) {
 		generic_err(leaf, slot,
-			    "invalid item size for free space info, has %u expect 0",
+			    "invalid item size for free space extent, has %u expect 0",
 			    btrfs_item_size(leaf, slot));
 		return -EUCLEAN;
 	}
@@ -2348,14 +2391,15 @@ static int check_free_space_bitmap(struct extent_buffer *leaf,
 static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf,
 						    struct btrfs_key *key,
 						    int slot,
-						    struct btrfs_key *prev_key)
+						    struct btrfs_key *prev_key,
+						    struct saved_inode_info *inode_info)
 {
 	int ret = 0;
 	struct btrfs_chunk *chunk;
 
 	switch (key->type) {
 	case BTRFS_EXTENT_DATA_KEY:
-		ret = check_extent_data_item(leaf, key, slot, prev_key);
+		ret = check_extent_data_item(leaf, key, slot, prev_key, inode_info);
 		break;
 	case BTRFS_EXTENT_CSUM_KEY:
 		ret = check_csum_item(leaf, key, slot, prev_key);
@@ -2385,7 +2429,7 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf,
 		ret = check_dev_extent_item(leaf, key, slot, prev_key);
 		break;
 	case BTRFS_INODE_ITEM_KEY:
-		ret = check_inode_item(leaf, key, slot);
+		ret = check_inode_item(leaf, key, slot, inode_info);
 		break;
 	case BTRFS_ROOT_ITEM_KEY:
 		ret = check_root_item(leaf, key, slot);
@@ -2433,6 +2477,7 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf,
 enum btrfs_tree_block_status __btrfs_check_leaf(struct extent_buffer *leaf)
 {
 	struct btrfs_fs_info *fs_info = leaf->fs_info;
+	struct saved_inode_info inode_info = { 0 };
 	/* No valid key type is 0, so all key should be larger than this key */
 	struct btrfs_key prev_key = {0, 0, 0};
 	struct btrfs_key key;
@@ -2568,7 +2613,7 @@ enum btrfs_tree_block_status __btrfs_check_leaf(struct extent_buffer *leaf)
 		}
 
 		/* Check if the item size and content meet other criteria. */
-		ret = check_leaf_item(leaf, &key, slot, &prev_key);
+		ret = check_leaf_item(leaf, &key, slot, &prev_key, &inode_info);
 		if (unlikely(ret != BTRFS_TREE_BLOCK_CLEAN))
 			return ret;
 
diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c
index a00094604e544e..1d9dfb63fc5a4f 100644
--- a/fs/btrfs/tree-log.c
+++ b/fs/btrfs/tree-log.c
@@ -503,7 +503,7 @@ static int overwrite_item(struct walk_control *wc)
 			btrfs_release_path(wc->subvol_path);
 			return 0;
 		}
-		src_copy = kmalloc(item_size, GFP_NOFS);
+		src_copy = kvmalloc(item_size, GFP_NOFS);
 		if (!src_copy) {
 			btrfs_abort_log_replay(wc, -ENOMEM,
 			       "failed to allocate memory for log leaf item");
@@ -514,7 +514,7 @@ static int overwrite_item(struct walk_control *wc)
 		dst_ptr = btrfs_item_ptr_offset(dst_eb, dst_slot);
 		ret = memcmp_extent_buffer(dst_eb, src_copy, dst_ptr, item_size);
 
-		kfree(src_copy);
+		kvfree(src_copy);
 		/*
 		 * they have the same contents, just return, this saves
 		 * us from cowing blocks in the destination tree and doing
@@ -1683,7 +1683,7 @@ static noinline int add_inode_ref(struct walk_control *wc)
 			}
 
 			/* insert our name */
-			ret = btrfs_add_link(trans, dir, inode, &name, false, ref_index);
+			ret = btrfs_add_link(trans, dir, inode, &name, false, ref_index, NULL);
 			if (ret) {
 				btrfs_abort_log_replay(wc, ret,
 "failed to add link for inode %llu in dir %llu ref_index %llu name %.*s root %llu",
@@ -2031,7 +2031,7 @@ static noinline int insert_one_name(struct btrfs_trans_handle *trans,
 		return PTR_ERR(dir);
 	}
 
-	ret = btrfs_add_link(trans, dir, inode, name, true, index);
+	ret = btrfs_add_link(trans, dir, inode, name, true, index, NULL);
 
 	/* FIXME, put inode into FIXUP list */
 
@@ -5350,7 +5350,7 @@ static int btrfs_log_changed_extents(struct btrfs_trans_handle *trans,
 		 * have a bunch of extents we just want to commit since it will
 		 * be faster.
 		 */
-		if (++num > 32768) {
+		if (++num > SZ_16K) {
 			list_del_init(&tree->modified_extents);
 			ret = -EFBIG;
 			goto process;
@@ -5368,7 +5368,6 @@ static int btrfs_log_changed_extents(struct btrfs_trans_handle *trans,
 		refcount_inc(&em->refs);
 		em->flags |= EXTENT_FLAG_LOGGING;
 		list_add_tail(&em->list, &extents);
-		num++;
 	}
 
 	list_sort(NULL, &extents, extent_cmp);
diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c
index 4e0ab584227419..83dc7dee14cfb3 100644
--- a/fs/btrfs/verity.c
+++ b/fs/btrfs/verity.c
@@ -93,6 +93,20 @@ static loff_t merkle_file_pos(const struct inode *inode)
 	return rounded;
 }
 
+/*
+ * Start a transaction for removing verity items or the verity orphan.
+ *
+ * Like unlink, this only deletes items and frees space in the end, so the
+ * reservation may come from the global reserve when the filesystem is full
+ * (-ENOSPC) and is not subject to the qgroup limit (-EDQUOT). Otherwise a
+ * failed enable could never be cleaned up in either situation.
+ */
+static struct btrfs_trans_handle *start_verity_cleanup_trans(struct btrfs_root *root,
+							     unsigned int num_items)
+{
+	return btrfs_start_transaction_fallback_global_rsv(root, num_items);
+}
+
 /*
  * Drop all the items for this inode with this key_type.
  *
@@ -120,7 +134,7 @@ static int drop_verity_items(struct btrfs_inode *inode, u8 key_type)
 
 	while (1) {
 		/* 1 for the item being dropped */
-		trans = btrfs_start_transaction(root, 1);
+		trans = start_verity_cleanup_trans(root, 1);
 		if (IS_ERR(trans))
 			return PTR_ERR(trans);
 
@@ -272,21 +286,17 @@ static int write_key_bytes(struct btrfs_inode *inode, u8 key_type, u64 offset,
  * @dest:       Buffer to read into. This parameter has slightly tricky
  *              semantics.  If it is NULL, the function will not do any copying
  *              and will just return the size of all the items up to len bytes.
- *              If dest_page is passed, then the function will kmap_local the
- *              page and ignore dest, but it must still be non-NULL to avoid the
- *              counting-only behavior.
  * @len:        length in bytes to read
- * @dest_folio: copy into this folio instead of the dest buffer
  *
  * Helper function to read items from the btree.  This returns the number of
  * bytes read or < 0 for errors.  We can return short reads if the items don't
  * exist on disk or aren't big enough to fill the desired length.  Supports
- * reading into a provided buffer (dest) or into the page cache
+ * reading into a provided buffer (dest).
  *
  * Returns number of bytes read or a negative error code on failure.
  */
 static int read_key_bytes(struct btrfs_inode *inode, u8 key_type, u64 offset,
-			  char *dest, u64 len, struct folio *dest_folio)
+			  char *dest, u64 len)
 {
 	BTRFS_PATH_AUTO_FREE(path);
 	struct btrfs_root *root = inode->root;
@@ -306,7 +316,11 @@ static int read_key_bytes(struct btrfs_inode *inode, u8 key_type, u64 offset,
 	if (!path)
 		return -ENOMEM;
 
-	if (dest_folio)
+	/*
+	 * Merkle items can be large and split across multiple items, so enable
+	 * readahead for such cases.
+	 */
+	if (key_type == BTRFS_VERITY_MERKLE_ITEM_KEY)
 		path->reada = READA_FORWARD;
 
 	key.objectid = btrfs_ino(inode);
@@ -350,7 +364,7 @@ static int read_key_bytes(struct btrfs_inode *inode, u8 key_type, u64 offset,
 				break;
 		}
 
-		/* desc = NULL to just sum all the item lengths */
+		/* dest == NULL to just sum all the item lengths */
 		if (!dest)
 			copy_end = item_end;
 		else
@@ -363,16 +377,10 @@ static int read_key_bytes(struct btrfs_inode *inode, u8 key_type, u64 offset,
 		copy_offset = offset - key.offset;
 
 		if (dest) {
-			if (dest_folio)
-				kaddr = kmap_local_folio(dest_folio, 0);
-
 			data = btrfs_item_ptr(leaf, path->slots[0], void);
 			read_extent_buffer(leaf, kaddr + dest_offset,
 					   (unsigned long)data + copy_offset,
 					   copy_bytes);
-
-			if (dest_folio)
-				kunmap_local(kaddr);
 		}
 
 		offset += copy_bytes;
@@ -466,7 +474,7 @@ static int rollback_verity(struct btrfs_inode *inode)
 	 * 1 for updating the inode flag
 	 * 1 for deleting the orphan
 	 */
-	trans = btrfs_start_transaction(root, 2);
+	trans = start_verity_cleanup_trans(root, 2);
 	if (IS_ERR(trans)) {
 		ret = PTR_ERR(trans);
 		trans = NULL;
@@ -663,7 +671,7 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size)
 
 	memset(&item, 0, sizeof(item));
 	ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_DESC_ITEM_KEY, 0,
-			     (char *)&item, sizeof(item), NULL);
+			     (char *)&item, sizeof(item));
 	if (ret < 0)
 		return ret;
 
@@ -680,7 +688,7 @@ int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size)
 		return -ERANGE;
 
 	ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_DESC_ITEM_KEY, 1,
-			     buf, buf_size, NULL);
+			     buf, buf_size);
 	if (ret < 0)
 		return ret;
 	if (ret != true_size)
@@ -706,6 +714,7 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode,
 	struct folio *folio;
 	u64 off = (u64)index << PAGE_SHIFT;
 	loff_t merkle_pos = merkle_file_pos(inode);
+	void *kaddr;
 	int ret;
 
 	if (merkle_pos < 0)
@@ -749,6 +758,7 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode,
 	}
 
 read_folio:
+	kaddr = kmap_local_folio(folio, 0);
 	/*
 	 * Merkle item keys are indexed from byte 0 in the merkle tree.
 	 * They have the form:
@@ -756,7 +766,8 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode,
 	 * [ inode objectid, BTRFS_MERKLE_ITEM_KEY, offset in bytes ]
 	 */
 	ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_MERKLE_ITEM_KEY, off,
-			     folio_address(folio), PAGE_SIZE, folio);
+			     kaddr, PAGE_SIZE);
+	kunmap_local(kaddr);
 	if (ret < 0) {
 		folio_unlock(folio);
 		folio_put(folio);
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 74584669507fd7..7c040f22dbc3e5 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -403,7 +403,7 @@ static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid)
 	return fs_devs;
 }
 
-static void btrfs_free_device(struct btrfs_device *device)
+void btrfs_free_device(struct btrfs_device *device)
 {
 	WARN_ON(!list_empty(&device->post_commit_list));
 	/*
@@ -749,6 +749,36 @@ const u8 *btrfs_sb_fsid_ptr(const struct btrfs_super_block *sb)
 	return has_metadata_uuid ? sb->metadata_uuid : sb->fsid;
 }
 
+static bool should_rename_device(const struct btrfs_device *dev)
+{
+	bool ret;
+	const char *old_name;
+
+	rcu_read_lock();
+	old_name = rcu_dereference(dev->name);
+	/*
+	 * For systems booted without an initramfs, the rootfs has the device
+	 * name "/dev/root".
+	 *
+	 * Although using btrfs without an initramfs is not recommended (if a
+	 * new device is added to the rootfs, the system can no longer boot, as
+	 * there is no way to register all devices), there is still a minority
+	 * of users doing this.
+	 *
+	 * And after the system is up, a later device scan on the real block
+	 * device file will never get this device's name updated, as the
+	 * device->devt is still the same.
+	 *
+	 * Here we add one and only one exception for "/dev/root", to allow the
+	 * device name to be updated even if the new path points to the same
+	 * block device.
+	 */
+	ret = (strcmp(old_name, "/dev/root") == 0);
+	rcu_read_unlock();
+
+	return ret;
+}
+
 /*
  * Add new device to list of registered devices
  *
@@ -869,7 +899,8 @@ static noinline struct btrfs_device *device_list_add(const char *path,
 				MAJOR(path_devt), MINOR(path_devt),
 				current->comm, task_pid_nr(current));
 
-	} else if (!device->name || device->devt != path_devt) {
+	} else if (!device->name || device->devt != path_devt ||
+		   should_rename_device(device)) {
 		const char *old_name;
 
 		/*
@@ -1327,16 +1358,14 @@ int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
 
 void btrfs_release_disk_super(struct btrfs_super_block *super)
 {
-	struct page *page = virt_to_page(super);
-
-	put_page(page);
+	folio_put(virt_to_folio(super));
 }
 
 struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
 						int copy_num, bool drop_cache)
 {
 	struct btrfs_super_block *super;
-	struct page *page;
+	struct folio *folio;
 	u64 bytenr, bytenr_orig;
 	struct address_space *mapping = bdev->bd_mapping;
 	int ret;
@@ -1357,7 +1386,7 @@ struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
 		ASSERT(copy_num == 0);
 
 		/*
-		 * Drop the page of the primary superblock, so later read will
+		 * Drop the folio of the primary superblock, so later read will
 		 * always read from the device.
 		 */
 		invalidate_inode_pages2_range(mapping, bytenr >> PAGE_SHIFT,
@@ -1365,12 +1394,12 @@ struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
 	}
 
 	filemap_invalidate_lock_shared(mapping);
-	page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS);
+	folio = mapping_read_folio_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS);
 	filemap_invalidate_unlock_shared(mapping);
-	if (IS_ERR(page))
-		return ERR_CAST(page);
+	if (IS_ERR(folio))
+		return ERR_CAST(folio);
 
-	super = page_address(page);
+	super = folio_address(folio) + offset_in_folio(folio, bytenr);
 	if (btrfs_super_magic(super) != BTRFS_MAGIC ||
 	    btrfs_super_bytenr(super) != bytenr_orig) {
 		btrfs_release_disk_super(super);
@@ -2783,6 +2812,41 @@ static void btrfs_setup_sprout(struct btrfs_fs_info *fs_info,
 	btrfs_set_super_flags(disk_super, super_flags);
 }
 
+static void btrfs_rollback_sprout(struct btrfs_fs_info *fs_info,
+				  struct btrfs_fs_devices *seed_devices)
+{
+	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+	struct btrfs_super_block *disk_super = fs_info->super_copy;
+	struct btrfs_device *device;
+	u64 super_flags;
+
+	lockdep_assert_held(&uuid_mutex);
+	lockdep_assert_held(&fs_devices->device_list_mutex);
+
+	list_del_init(&seed_devices->seed_list);
+	list_splice_init_rcu(&seed_devices->devices, &fs_devices->devices, synchronize_rcu);
+	list_for_each_entry(device, &fs_devices->devices, dev_list) {
+		device->fs_devices = fs_devices;
+	}
+
+	fs_devices->seeding = true;
+	fs_devices->num_devices = seed_devices->num_devices;
+	fs_devices->open_devices = seed_devices->open_devices;
+	fs_devices->missing_devices = seed_devices->missing_devices;
+	fs_devices->rotating = seed_devices->rotating;
+	fs_devices->latest_dev = seed_devices->latest_dev;
+
+	memcpy(fs_devices->fsid, seed_devices->fsid, BTRFS_FSID_SIZE);
+	memcpy(fs_devices->metadata_uuid, seed_devices->metadata_uuid, BTRFS_FSID_SIZE);
+	memcpy(disk_super->fsid, seed_devices->fsid, BTRFS_FSID_SIZE);
+
+	super_flags = (btrfs_super_flags(disk_super) | BTRFS_SUPER_FLAG_SEEDING);
+	btrfs_set_super_flags(disk_super, super_flags);
+
+	seed_devices->opened = 0;
+	free_fs_devices(seed_devices);
+}
+
 /*
  * Store the expected generation for seed devices in device items.
  */
@@ -3134,6 +3198,8 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path
 				    orig_super_total_bytes);
 	btrfs_set_super_num_devices(fs_info->super_copy,
 				    orig_super_num_devices);
+	if (seeding_dev)
+		btrfs_rollback_sprout(fs_info, seed_devices);
 	btrfs_update_per_profile_avail(fs_info);
 	mutex_unlock(&fs_info->chunk_mutex);
 	mutex_unlock(&fs_info->fs_devices->device_list_mutex);
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index 0415d74cad9ba9..337d7007d9e225 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -799,6 +799,7 @@ void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev);
 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev);
 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev,
 				      bool allow_freeze);
+void btrfs_free_device(struct btrfs_device *device);
 unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
 				    u64 logical);
 u64 btrfs_calc_stripe_length(const struct btrfs_chunk_map *map);
diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c
index ab55d10bd71fdb..a06420b9c66288 100644
--- a/fs/btrfs/xattr.c
+++ b/fs/btrfs/xattr.c
@@ -353,7 +353,7 @@ static int btrfs_xattr_handler_get(const struct xattr_handler *handler,
 }
 
 static int btrfs_xattr_handler_set(const struct xattr_handler *handler,
-				   struct mnt_idmap *idmap,
+				   const struct mnt_idmap *idmap,
 				   struct dentry *unused, struct inode *inode,
 				   const char *name, const void *buffer,
 				   size_t size, int flags)
@@ -395,7 +395,7 @@ static int btrfs_xattr_handler_get_security(const struct xattr_handler *handler,
 }
 
 static int btrfs_xattr_handler_set_security(const struct xattr_handler *handler,
-					    struct mnt_idmap *idmap,
+					    const struct mnt_idmap *idmap,
 					    struct dentry *unused,
 					    struct inode *inode,
 					    const char *name,
@@ -413,7 +413,7 @@ static int btrfs_xattr_handler_set_security(const struct xattr_handler *handler,
 }
 
 static int btrfs_xattr_handler_set_prop(const struct xattr_handler *handler,
-					struct mnt_idmap *idmap,
+					const struct mnt_idmap *idmap,
 					struct dentry *unused, struct inode *inode,
 					const char *name, const void *value,
 					size_t size, int flags)
diff --git a/fs/btrfs/zlib.c b/fs/btrfs/zlib.c
index 486b52db583ecb..e995d0b1452b92 100644
--- a/fs/btrfs/zlib.c
+++ b/fs/btrfs/zlib.c
@@ -49,7 +49,7 @@ void zlib_free_workspace(struct list_head *ws)
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
 
 	kvfree(workspace->strm.workspace);
-	kfree(workspace->buf);
+	kvfree(workspace->buf);
 	kfree(workspace);
 }
 
@@ -84,13 +84,13 @@ struct list_head *zlib_alloc_workspace(struct btrfs_fs_info *fs_info, unsigned i
 	workspace->level = level;
 	workspace->buf = NULL;
 	if (need_special_buffer(fs_info)) {
-		workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE,
-					 __GFP_NOMEMALLOC | __GFP_NORETRY |
-					 __GFP_NOWARN | GFP_NOIO);
+		workspace->buf = kvmalloc(ZLIB_DFLTCC_BUF_SIZE,
+					  __GFP_NOMEMALLOC | __GFP_NORETRY |
+					  __GFP_NOWARN | GFP_NOIO);
 		workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE;
 	}
 	if (!workspace->buf) {
-		workspace->buf = kmalloc(fs_info->sectorsize, GFP_KERNEL);
+		workspace->buf = kvmalloc(fs_info->sectorsize, GFP_KERNEL);
 		workspace->buf_size = fs_info->sectorsize;
 	}
 	if (!workspace->strm.workspace || !workspace->buf)
diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c
index 9cc2c9c1a606b6..c04a9955f2d39b 100644
--- a/fs/btrfs/zoned.c
+++ b/fs/btrfs/zoned.c
@@ -123,24 +123,24 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones,
 	} else if (full[0] && full[1]) {
 		/* Compare two super blocks */
 		struct address_space *mapping = bdev->bd_mapping;
-		struct page *page[BTRFS_NR_SB_LOG_ZONES];
 		struct btrfs_super_block *super[BTRFS_NR_SB_LOG_ZONES];
 
 		for (int i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) {
 			u64 zone_end = (zones[i].start + zones[i].capacity) << SECTOR_SHIFT;
 			u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) -
 						BTRFS_SUPER_INFO_SIZE;
+			struct folio *folio;
 
 			filemap_invalidate_lock_shared(mapping);
-			page[i] = read_cache_page_gfp(mapping,
-					bytenr >> PAGE_SHIFT, GFP_NOFS);
+			folio = mapping_read_folio_gfp(mapping, bytenr >> PAGE_SHIFT,
+						       GFP_NOFS);
 			filemap_invalidate_unlock_shared(mapping);
-			if (IS_ERR(page[i])) {
+			if (IS_ERR(folio)) {
 				if (i == 1)
 					btrfs_release_disk_super(super[0]);
-				return PTR_ERR(page[i]);
+				return PTR_ERR(folio);
 			}
-			super[i] = page_address(page[i]);
+			super[i] = folio_address(folio) + offset_in_folio(folio, bytenr);
 		}
 
 		if (btrfs_super_generation(super[0]) >
@@ -804,15 +804,6 @@ int btrfs_check_mountopts_zoned(const struct btrfs_fs_info *info,
 	if (!btrfs_is_zoned(info))
 		return 0;
 
-	/*
-	 * Space cache writing is not COWed. Disable that to avoid write errors
-	 * in sequential zones.
-	 */
-	if (btrfs_raw_test_opt(*mount_opt, SPACE_CACHE)) {
-		btrfs_err(info, "zoned: space cache v1 is not supported");
-		return -EINVAL;
-	}
-
 	if (btrfs_raw_test_opt(*mount_opt, NODATACOW)) {
 		btrfs_err(info, "zoned: NODATACOW not supported");
 		return -EINVAL;
@@ -2688,6 +2679,11 @@ bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags)
 
 		switch (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
 		case 0: /* single */
+		case BTRFS_BLOCK_GROUP_RAID0:
+		case BTRFS_BLOCK_GROUP_RAID1:
+		case BTRFS_BLOCK_GROUP_RAID1C3:
+		case BTRFS_BLOCK_GROUP_RAID1C4:
+		case BTRFS_BLOCK_GROUP_RAID10:
 			ret = (atomic_read(&zinfo->active_zones_left) >= (1 + reserved));
 			break;
 		case BTRFS_BLOCK_GROUP_DUP:
diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c
index 58d9ff76fe07bb..cc92d0b1b948cd 100644
--- a/fs/btrfs/zstd.c
+++ b/fs/btrfs/zstd.c
@@ -373,7 +373,7 @@ void zstd_free_workspace(struct list_head *ws)
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
 
 	kvfree(workspace->mem);
-	kfree(workspace->buf);
+	kvfree(workspace->buf);
 	kfree(workspace);
 }
 
@@ -391,7 +391,7 @@ struct list_head *zstd_alloc_workspace(struct btrfs_fs_info *fs_info, int level)
 	workspace->req_level = level;
 	workspace->last_used = jiffies;
 	workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN);
-	workspace->buf = kmalloc(fs_info->sectorsize, GFP_KERNEL);
+	workspace->buf = kvmalloc(fs_info->sectorsize, GFP_KERNEL);
 	if (!workspace->mem || !workspace->buf)
 		goto fail;
 
@@ -589,10 +589,48 @@ int zstd_compress_bio(struct list_head *ws, struct compressed_bio *cb)
 	return ret;
 }
 
+/*
+ * Map the destination for the next chunk of output.
+ *
+ * @decompressed is the offset of the next output byte inside the fully
+ * decompressed extent.  If that offset has reached the current destination
+ * segment, its page-bounded bio_vec is kmapped so that zstd can write into the
+ * page cache directly, and the number of bytes writable there is returned.
+ * Otherwise @kaddr_ret is set to NULL and the number of bytes to skip before
+ * that segment is returned.  This covers both the initial prefix and gaps in
+ * the destination bio.
+ */
+static u32 zstd_map_dest(struct compressed_bio *cb, u32 decompressed,
+			 void **kaddr_ret)
+{
+	struct bio *orig_bio = &cb->orig_bbio->bio;
+	struct bio_vec bvec;
+	u32 bvec_offset;
+	u32 off;
+
+	bvec = bio_iter_iovec(orig_bio, orig_bio->bi_iter);
+	/*
+	 * cb->start may underflow, but subtracting that value can still give us
+	 * the correct offset inside the full decompressed extent.
+	 */
+	bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start;
+
+	if (decompressed < bvec_offset) {
+		*kaddr_ret = NULL;
+		return bvec_offset - decompressed;
+	}
+
+	off = decompressed - bvec_offset;
+	ASSERT(off < bvec.bv_len);
+	*kaddr_ret = bvec_kmap_local(&bvec) + off;
+	return bvec.bv_len - off;
+}
+
 int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 {
 	struct btrfs_fs_info *fs_info = cb_to_fs_info(cb);
 	struct workspace *workspace = list_entry(ws, struct workspace, list);
+	struct bio *orig_bio = &cb->orig_bbio->bio;
 	struct folio_iter fi;
 	size_t srclen = bio_get_size(&cb->bbio.bio);
 	zstd_dstream *stream;
@@ -600,7 +638,6 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 	const unsigned int min_folio_size = btrfs_min_folio_size(fs_info);
 	unsigned long folio_in_index = 0;
 	unsigned long total_folios_in = DIV_ROUND_UP(srclen, min_folio_size);
-	unsigned long buf_start;
 	unsigned long total_out = 0;
 
 	bio_first_folio(&fi, &cb->bbio.bio, 0);
@@ -624,15 +661,26 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 	workspace->in_buf.pos = 0;
 	workspace->in_buf.size = min_t(size_t, srclen, min_folio_size);
 
-	workspace->out_buf.dst = workspace->buf;
-	workspace->out_buf.pos = 0;
-	workspace->out_buf.size = fs_info->sectorsize;
-
-	while (1) {
+	while (orig_bio->bi_iter.bi_size) {
 		size_t ret2;
+		void *kaddr;
+		u32 dstlen;
+
+		dstlen = zstd_map_dest(cb, total_out, &kaddr);
+		if (kaddr) {
+			workspace->out_buf.dst = kaddr;
+			workspace->out_buf.size = dstlen;
+		} else {
+			workspace->out_buf.dst = workspace->buf;
+			workspace->out_buf.size = min_t(u32, dstlen,
+							fs_info->sectorsize);
+		}
+		workspace->out_buf.pos = 0;
 
 		ret2 = zstd_decompress_stream(stream, &workspace->out_buf,
 				&workspace->in_buf);
+		if (kaddr)
+			kunmap_local(kaddr);
 		if (unlikely(zstd_is_error(ret2))) {
 			struct btrfs_inode *inode = cb->bbio.inode;
 
@@ -643,14 +691,9 @@ int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 			ret = -EIO;
 			goto done;
 		}
-		buf_start = total_out;
 		total_out += workspace->out_buf.pos;
-		workspace->out_buf.pos = 0;
-
-		ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
-				total_out - buf_start, cb, buf_start);
-		if (ret == 0)
-			break;
+		if (kaddr)
+			bio_advance(orig_bio, workspace->out_buf.pos);
 
 		if (workspace->in_buf.pos >= srclen)
 			break;
diff --git a/fs/buffer.c b/fs/buffer.c
index ed966fa73b1ba2..60f7a588fff0cf 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -25,6 +25,7 @@
 #include <linux/fs.h>
 #include <linux/iomap.h>
 #include <linux/mm.h>
+#include <linux/swap.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
 #include <linux/capability.h>
@@ -203,11 +204,10 @@ void bh_end_write(struct bio *bio)
 	bool success = bio_endio_bh(bio, &bh);
 
 	if (success) {
-		set_buffer_uptodate(bh);
+		clear_buffer_write_io_error(bh);
 	} else {
 		buffer_io_error(bh, ", lost sync page write");
 		mark_buffer_write_io_error(bh);
-		clear_buffer_uptodate(bh);
 	}
 	unlock_buffer(bh);
 }
@@ -265,7 +265,7 @@ __find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic)
 		bh = bh->b_this_page;
 	} while (bh != head);
 
-	/* we might be here because some of the buffers on this page are
+	/* we might be here because some of the buffers on this folio are
 	 * not mapped.  This is due to various races between
 	 * file io on the block device and getblk.  It gets dealt with
 	 * elsewhere, don't buffer_error if we had some unmapped buffers
@@ -311,7 +311,7 @@ static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
 	/*
 	 * Be _very_ careful from here on. Bad things can happen if
 	 * two buffer heads end IO at almost the same time and both
-	 * decide that the page is now completely done.
+	 * decide that the folio is now completely done.
 	 */
 	first = folio_buffers(folio);
 	spin_lock_irqsave(&first->b_uptodate_lock, flags);
@@ -408,11 +408,10 @@ void bh_end_async_write(struct bio *bio)
 
 	folio = bh->b_folio;
 	if (success) {
-		set_buffer_uptodate(bh);
+		clear_buffer_write_io_error(bh);
 	} else {
 		buffer_io_error(bh, ", lost async page write");
 		mark_buffer_write_io_error(bh);
-		clear_buffer_uptodate(bh);
 	}
 
 	first = folio_buffers(folio);
@@ -520,8 +519,8 @@ EXPORT_SYMBOL_GPL(mmb_has_buffers);
  *
  * Do this in two main stages: first we copy dirty buffers to a
  * temporary inode list, queueing the writes as we go. Then we clean
- * up, waiting for those writes to complete. mark_buffer_dirty_inode()
- * doesn't touch b_assoc_buffers list if b_mmb is not NULL so we are sure the
+ * up, waiting for those writes to complete. mmb_mark_buffer_dirty()
+ * doesn't touch b_assoc_buffers list if b_mmb is set so we are sure the
  * buffer stays on our list until IO completes (at which point it can be
  * reaped).
  */
@@ -542,7 +541,7 @@ int mmb_sync(struct mapping_metadata_bhs *mmb)
 		bh = BH_ENTRY(mmb->list.next);
 		WARN_ON_ONCE(bh->b_mmb != mmb);
 		__remove_assoc_queue(mmb, bh);
-		/* Avoid race with mark_buffer_dirty_inode() which does
+		/* Avoid race with mmb_mark_buffer_dirty() which does
 		 * a lockless check and we rely on seeing the dirty bit */
 		smp_mb();
 		if (buffer_dirty(bh) || buffer_locked(bh)) {
@@ -580,7 +579,7 @@ int mmb_sync(struct mapping_metadata_bhs *mmb)
 		bh = BH_ENTRY(tmp.prev);
 		get_bh(bh);
 		__remove_assoc_queue(mmb, bh);
-		/* Avoid race with mark_buffer_dirty_inode() which does
+		/* Avoid race with mmb_mark_buffer_dirty() which does
 		 * a lockless check and we rely on seeing the dirty bit */
 		smp_mb();
 		if (buffer_dirty(bh)) {
@@ -589,7 +588,7 @@ int mmb_sync(struct mapping_metadata_bhs *mmb)
 		}
 		spin_unlock(&mmb->lock);
 		wait_on_buffer(bh);
-		if (!buffer_uptodate(bh))
+		if (buffer_write_io_error(bh))
 			err = -EIO;
 		brelse(bh);
 		spin_lock(&mmb->lock);
@@ -618,6 +617,14 @@ void write_boundary_block(struct block_device *bdev,
 	}
 }
 
+/**
+ * mmb_mark_buffer_dirty - Mark a metadata buffer dirty.
+ * @bh: The buffer to mark dirty.
+ * @mmb: The list of buffers to add the buffer to.
+ *
+ * Mark the buffer dirty and add it to the list if it is not already on
+ * a list.
+ */
 void mmb_mark_buffer_dirty(struct buffer_head *bh,
 			   struct mapping_metadata_bhs *mmb)
 {
@@ -686,7 +693,7 @@ bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
 		} while (bh != head);
 	}
 	/*
-	 * Lock out page's memcg migration to keep PageDirty
+	 * Lock out folio's memcg migration to keep folio dirty flag
 	 * synchronized with per-memcg dirty page counters.
 	 */
 	newly_dirty = !folio_test_set_dirty(folio);
@@ -773,14 +780,6 @@ struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
 }
 EXPORT_SYMBOL_GPL(folio_alloc_buffers);
 
-struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size)
-{
-	gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
-
-	return folio_alloc_buffers(page_folio(page), size, gfp);
-}
-EXPORT_SYMBOL_GPL(alloc_page_buffers);
-
 static inline void link_dev_buffers(struct folio *folio,
 		struct buffer_head *head)
 {
@@ -952,23 +951,23 @@ __getblk_slow(struct block_device *bdev, sector_t block,
 }
 
 /*
- * The relationship between dirty buffers and dirty pages:
+ * The relationship between dirty buffers and dirty folios:
  *
- * Whenever a page has any dirty buffers, the page's dirty bit is set, and
- * the page is tagged dirty in the page cache.
+ * Whenever a folio has any dirty buffers, the folio's dirty flag is set, and
+ * the folio is tagged dirty in the page cache.
  *
  * At all times, the dirtiness of the buffers represents the dirtiness of
- * subsections of the page.  If the page has buffers, the page dirty bit is
+ * subsections of the folio.  If the folio has buffers, the folio dirty flag is
  * merely a hint about the true dirty state.
  *
- * When a page is set dirty in its entirety, all its buffers are marked dirty
- * (if the page has buffers).
+ * When a folio is set dirty in its entirety, all its buffers are marked dirty
+ * (if the folio has buffers).
  *
- * When a buffer is marked dirty, its page is dirtied, but the page's other
+ * When a buffer is marked dirty, its folio is dirtied, but the folio's other
  * buffers are not.
  *
  * Also.  When blockdev buffers are explicitly read with bread(), they
- * individually become uptodate.  But their backing page remains not
+ * individually become uptodate.  But their backing folio remains not
  * uptodate - even if all of its buffers are uptodate.  A subsequent
  * block_read_full_folio() against that folio will discover all the uptodate
  * buffers, will set the folio uptodate and will perform no I/O.
@@ -979,7 +978,7 @@ __getblk_slow(struct block_device *bdev, sector_t block,
  * @bh: the buffer_head to mark dirty
  *
  * mark_buffer_dirty() will set the dirty bit against the buffer, then set
- * its backing page dirty, then tag the page as dirty in the page cache
+ * its backing folio dirty, then tag the folio as dirty in the page cache
  * and then attach the address_space's inode to its superblock's dirty
  * inode list.
  *
@@ -1062,6 +1061,7 @@ EXPORT_SYMBOL(__brelse);
 void __bforget(struct buffer_head *bh)
 {
 	clear_buffer_dirty(bh);
+	clear_buffer_write_io_error(bh);
 	remove_assoc_queue(bh);
 	__brelse(bh);
 }
@@ -1070,12 +1070,16 @@ EXPORT_SYMBOL(__bforget);
 static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh,
 				  gfp_t gfp_mask)
 {
-	const struct address_space *mapping = folio_mapping(bh->b_folio);
+	const struct address_space *mapping;
 
 	/*
 	 * The ext4 journal (jbd2) can submit a buffer_head it directly created
-	 * for a non-pagecache page.  fscrypt doesn't care about these.
+	 * for memory that is not in the page cache at all.  fscrypt doesn't
+	 * care about these.
 	 */
+	if (!bh->b_folio)
+		return;
+	mapping = bh->b_folio->mapping;
 	if (!mapping)
 		return;
 	fscrypt_set_bio_crypt_ctx(bio, mapping->host,
@@ -1086,7 +1090,6 @@ static void __bh_submit(struct buffer_head *bh, blk_opf_t opf,
 		enum rw_hint write_hint, struct writeback_control *wbc,
 		bio_end_io_t end_bio)
 {
-	const enum req_op op = opf & REQ_OP_MASK;
 	struct bio *bio;
 
 	BUG_ON(!buffer_locked(bh));
@@ -1094,11 +1097,7 @@ static void __bh_submit(struct buffer_head *bh, blk_opf_t opf,
 	BUG_ON(buffer_delay(bh));
 	BUG_ON(buffer_unwritten(bh));
 
-	/*
-	 * Only clear out a write error when rewriting
-	 */
-	if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
-		clear_buffer_write_io_error(bh);
+	set_buffer_req(bh);
 
 	if (buffer_meta(bh))
 		opf |= REQ_META;
@@ -1107,7 +1106,8 @@ static void __bh_submit(struct buffer_head *bh, blk_opf_t opf,
 
 	bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
 
-	if (folio_test_dropbehind(bh->b_folio) && op_is_write(opf))
+	if (bh->b_folio && folio_test_dropbehind(bh->b_folio) &&
+	    op_is_write(opf))
 		bio_set_flag(bio, BIO_COMPLETE_IN_TASK);
 
 	if (IS_ENABLED(CONFIG_FS_ENCRYPTION))
@@ -1116,7 +1116,11 @@ static void __bh_submit(struct buffer_head *bh, blk_opf_t opf,
 	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
 	bio->bi_write_hint = write_hint;
 
-	bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh));
+	if (bh->b_folio)
+		bio_add_folio_nofail(bio, bh->b_folio, bh->b_size,
+				     bh_offset(bh));
+	else
+		bio_add_virt_nofail(bio, bh->b_data, bh->b_size);
 
 	bio->bi_end_io = end_bio;
 	bio->bi_private = bh;
@@ -1126,7 +1130,8 @@ static void __bh_submit(struct buffer_head *bh, blk_opf_t opf,
 
 	if (wbc) {
 		wbc_init_bio(wbc, bio);
-		wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size);
+		if (bh->b_folio)
+			wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size);
 	}
 
 	blk_crypto_submit_bio(bio);
@@ -1216,7 +1221,7 @@ static void bh_lru_install(struct buffer_head *bh)
 
 	/*
 	 * the refcount of buffer_head in bh_lru prevents dropping the
-	 * attached page(i.e., try_to_free_buffers) so it could cause
+	 * attached folio (i.e., try_to_free_buffers) so it could cause
 	 * failing page migration.
 	 * Skip putting upcoming bh into bh_lru until migration is done.
 	 */
@@ -1280,7 +1285,7 @@ lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
  * it in the LRU and mark it as accessed.  If it is not present then return
  * NULL. Atomic context callers may also return NULL if the buffer is being
- * migrated; similarly the page is not marked accessed either.
+ * migrated; similarly the folio is not marked accessed either.
  */
 static struct buffer_head *
 find_get_block_common(struct block_device *bdev, sector_t block,
@@ -1289,7 +1294,7 @@ find_get_block_common(struct block_device *bdev, sector_t block,
 	struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
 
 	if (bh == NULL) {
-		/* __find_get_block_slow will mark the page accessed */
+		/* __find_get_block_slow will mark the folio accessed */
 		bh = __find_get_block_slow(bdev, block, atomic);
 		if (bh)
 			bh_lru_install(bh);
@@ -1475,15 +1480,14 @@ void folio_set_bh(struct buffer_head *bh, struct folio *folio,
 }
 EXPORT_SYMBOL(folio_set_bh);
 
-/*
- * Called when truncating a buffer on a page completely.
- */
-
 /* Bits that are cleared during an invalidate */
 #define BUFFER_FLAGS_DISCARD \
 	(1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
-	 1 << BH_Delay | 1 << BH_Unwritten)
+	 1 << BH_Delay | 1 << BH_Unwritten | 1 << BH_Write_EIO)
 
+/*
+ * Called when truncating a buffer on a folio completely.
+ */
 static void discard_buffer(struct buffer_head * bh)
 {
 	unsigned long b_state;
@@ -1611,9 +1615,7 @@ EXPORT_SYMBOL(create_empty_buffers);
  * moment when something will explicitly mark the buffer dirty (hopefully that
  * will not happen until we will free that block ;-) We don't even need to mark
  * it not-uptodate - nobody can expect anything from a newly allocated buffer
- * anyway. We used to use unmap_buffer() for such invalidation, but that was
- * wrong. We definitely don't want to mark the alias unmapped, for example - it
- * would confuse anyone who might pick it with bread() afterwards...
+ * anyway.
  *
  * Also..  Note that bforget() doesn't lock the buffer.  So there can be
  * writeout I/O going on against recently-freed buffers.  We don't wait on that
@@ -1649,7 +1651,7 @@ void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
 			/* Recheck when the folio is locked which pins bhs */
 			head = folio_buffers(folio);
 			if (!head)
-				goto unlock_page;
+				goto unlock_folio;
 			bh = head;
 			do {
 				if (!buffer_mapped(bh) || (bh->b_blocknr < block))
@@ -1662,7 +1664,7 @@ void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
 next:
 				bh = bh->b_this_page;
 			} while (bh != head);
-unlock_page:
+unlock_folio:
 			folio_unlock(folio);
 		}
 		folio_batch_release(&fbatch);
@@ -1710,7 +1712,7 @@ static struct buffer_head *folio_create_buffers(struct folio *folio,
  *
  * If block_write_full_folio() is called for regular writeback
  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a folio which
- * has a locked buffer.   This only can happen if someone has written
+ * has a locked buffer.   This can only happen if someone has written
  * the buffer directly, with bh_submit().  At the address_space level
  * the folio writeback flag prevents this contention from occurring.
  *
@@ -2213,9 +2215,9 @@ int generic_write_end(const struct kiocb *iocb, struct address_space *mapping,
 	if (old_size < pos)
 		pagecache_isize_extended(inode, old_size, pos);
 	/*
-	 * Don't mark the inode dirty under page lock. First, it unnecessarily
-	 * makes the holding time of page lock longer. Second, it forces lock
-	 * ordering of page lock and transaction start for journaling
+	 * Don't mark the inode dirty under folio lock. First, it unnecessarily
+	 * makes the holding time of folio lock longer. Second, it forces lock
+	 * ordering of folio lock and transaction start for journaling
 	 * filesystems.
 	 */
 	if (i_size_changed)
@@ -2341,7 +2343,7 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block)
 		 * BH_Async_Read tells end_buffer_async_read() that this
 		 * buffer is not under async I/O.
 		 *
-		 * The folio comes unlocked when it has no locked
+		 * The folio is unlocked when it has no locked
 		 * buffer_async buffers left.
 		 *
 		 * The folio lock prevents anyone starting new async
@@ -2451,7 +2453,7 @@ static int cont_expand_zero(const struct kiocb *iocb,
 		}
 	}
 
-	/* page covers the boundary, find the boundary offset */
+	/* folio crosses the boundary, find the boundary offset */
 	if (index == curidx) {
 		zerofrom = curpos & ~PAGE_MASK;
 		/* if we will expand the thing last block will be filled */
@@ -2509,18 +2511,18 @@ EXPORT_SYMBOL(cont_write_begin);
 
 /*
  * block_page_mkwrite() is not allowed to change the file size as it gets
- * called from a page fault handler when a page is first dirtied. Hence we must
- * be careful to check for EOF conditions here. We set the page up correctly
- * for a written page which means we get ENOSPC checking when writing into
+ * called from a page fault handler when a folio is first dirtied. Hence we must
+ * be careful to check for EOF conditions here. We set the folio up correctly
+ * for a written folio which means we get ENOSPC checking when writing into
  * holes and correct delalloc and unwritten extent mapping on filesystems that
  * support these features.
  *
  * We are not allowed to take the i_rwsem here so we have to play games to
- * protect against truncate races as the page could now be beyond EOF.  Because
- * truncate writes the inode size before removing pages, once we have the
- * page lock we can determine safely if the page is beyond EOF. If it is not
- * beyond EOF, then the page is guaranteed safe against truncation until we
- * unlock the page.
+ * protect against truncate races as the folio could now be beyond EOF.  Because
+ * truncate writes the inode size before removing folios, once we have the
+ * folio lock we can determine safely if the folio is beyond EOF. If it is not
+ * beyond EOF, then the folio is guaranteed safe against truncation until we
+ * unlock the folio.
  *
  * Direct callers of this function should protect against filesystem freezing
  * using sb_start_pagefault() - sb_end_pagefault() functions.
@@ -2538,7 +2540,7 @@ int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
 	size = i_size_read(inode);
 	if ((folio->mapping != inode->i_mapping) ||
 	    (folio_pos(folio) >= size)) {
-		/* We overload EFAULT to mean page got truncated */
+		/* We overload EFAULT to mean folio got truncated */
 		ret = -EFAULT;
 		goto out_unlock;
 	}
@@ -2710,7 +2712,7 @@ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
 
 		bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write);
 		wait_on_buffer(bh);
-		if (!buffer_uptodate(bh))
+		if (buffer_write_io_error(bh))
 			return -EIO;
 	} else {
 		unlock_buffer(bh);
diff --git a/fs/cachefiles/Kconfig b/fs/cachefiles/Kconfig
index afb25b6af5aa17..c9c168c7e07268 100644
--- a/fs/cachefiles/Kconfig
+++ b/fs/cachefiles/Kconfig
@@ -17,7 +17,7 @@ config CACHEFILES_DEBUG
 	help
 	  This permits debugging to be dynamically enabled in the filesystem
 	  caching on files module.  If this is set, the debugging output may be
-	  enabled by setting bits in /sys/modules/cachefiles/parameter/debug or
+	  enabled by setting bits in /sys/module/cachefiles/parameters/debug or
 	  by including a debugging specifier in /etc/cachefilesd.conf.
 
 config CACHEFILES_ERROR_INJECTION
diff --git a/fs/cachefiles/interface.c b/fs/cachefiles/interface.c
index 50a000310a8c54..789ff6abe92624 100644
--- a/fs/cachefiles/interface.c
+++ b/fs/cachefiles/interface.c
@@ -99,73 +99,6 @@ void cachefiles_put_object(struct cachefiles_object *object,
 	_leave("");
 }
 
-/*
- * Adjust the size of a cache file if necessary to match the DIO size.  We keep
- * the EOF marker a multiple of DIO blocks so that we don't fall back to doing
- * non-DIO for a partial block straddling the EOF, but we also have to be
- * careful of someone expanding the file and accidentally accreting the
- * padding.
- */
-static int cachefiles_adjust_size(struct cachefiles_object *object)
-{
-	struct iattr newattrs;
-	struct file *file = object->file;
-	uint64_t ni_size;
-	loff_t oi_size;
-	int ret;
-
-	ni_size = object->cookie->object_size;
-	ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE);
-
-	_enter("{OBJ%x},[%llu]",
-	       object->debug_id, (unsigned long long) ni_size);
-
-	if (!file)
-		return -ENOBUFS;
-
-	oi_size = i_size_read(file_inode(file));
-	if (oi_size == ni_size)
-		return 0;
-
-	inode_lock(file_inode(file));
-
-	/* if there's an extension to a partial page at the end of the backing
-	 * file, we need to discard the partial page so that we pick up new
-	 * data after it */
-	if (oi_size & ~PAGE_MASK && ni_size > oi_size) {
-		_debug("discard tail %llx", oi_size);
-		newattrs.ia_valid = ATTR_SIZE;
-		newattrs.ia_size = oi_size & PAGE_MASK;
-		ret = cachefiles_inject_remove_error();
-		if (ret == 0)
-			ret = notify_change(&nop_mnt_idmap, file->f_path.dentry,
-					    &newattrs, NULL);
-		if (ret < 0)
-			goto truncate_failed;
-	}
-
-	newattrs.ia_valid = ATTR_SIZE;
-	newattrs.ia_size = ni_size;
-	ret = cachefiles_inject_write_error();
-	if (ret == 0)
-		ret = notify_change(&nop_mnt_idmap, file->f_path.dentry,
-				    &newattrs, NULL);
-
-truncate_failed:
-	inode_unlock(file_inode(file));
-
-	if (ret < 0)
-		trace_cachefiles_io_error(NULL, file_inode(file), ret,
-					  cachefiles_trace_notify_change_error);
-	if (ret == -EIO) {
-		cachefiles_io_error_obj(object, "Size set failed");
-		ret = -ENOBUFS;
-	}
-
-	_leave(" = %d", ret);
-	return ret;
-}
-
 /*
  * Attempt to look up the nominated node in this cache
  */
@@ -198,7 +131,6 @@ static bool cachefiles_lookup_cookie(struct fscache_cookie *cookie)
 	spin_lock(&cache->object_list_lock);
 	list_add(&object->cache_link, &cache->object_list);
 	spin_unlock(&cache->object_list_lock);
-	cachefiles_adjust_size(object);
 
 	cachefiles_end_secure(cache, saved_cred);
 	_leave(" = t");
@@ -225,14 +157,14 @@ static bool cachefiles_lookup_cookie(struct fscache_cookie *cookie)
  * any unused granules.
  */
 static bool cachefiles_shorten_object(struct cachefiles_object *object,
-				      struct file *file, loff_t new_size)
+				      struct file *file, uoff_t new_size)
 {
 	struct cachefiles_cache *cache = object->volume->cache;
 	struct inode *inode = file_inode(file);
-	loff_t i_size, dio_size;
+	uoff_t i_size, dio_size;
 	int ret;
 
-	dio_size = round_up(new_size, CACHEFILES_DIO_BLOCK_SIZE);
+	dio_size = round_up(new_size, cache->bsize);
 	i_size = i_size_read(inode);
 
 	trace_cachefiles_trunc(object, inode, i_size, dio_size,
@@ -264,6 +196,7 @@ static bool cachefiles_shorten_object(struct cachefiles_object *object,
 		}
 	}
 
+	object->object_size = new_size;
 	return true;
 }
 
@@ -271,29 +204,38 @@ static bool cachefiles_shorten_object(struct cachefiles_object *object,
  * Resize the backing object.
  */
 static void cachefiles_resize_cookie(struct netfs_cache_resources *cres,
-				     loff_t new_size)
+				     uoff_t new_size)
 {
 	struct cachefiles_object *object = cachefiles_cres_object(cres);
 	struct cachefiles_cache *cache = object->volume->cache;
 	struct fscache_cookie *cookie = object->cookie;
 	const struct cred *saved_cred;
 	struct file *file = cachefiles_cres_file(cres);
-	loff_t old_size = cookie->object_size;
+	uoff_t i_size = i_size_read(file_inode(file));
 
-	_enter("%llu->%llu", old_size, new_size);
+	_enter("%llu->%llu", object->object_size, new_size);
 
-	if (new_size < old_size) {
+	/* If the file is being shrunk, we need to downsize the backing file
+	 * and clear the end of the final block.
+	 */
+	if (new_size < object->object_size) {
+		if (new_size >= i_size)
+			goto out;
 		cachefiles_begin_secure(cache, &saved_cred);
 		cachefiles_shorten_object(object, file, new_size);
 		cachefiles_end_secure(cache, saved_cred);
 		object->cookie->object_size = new_size;
+		if (new_size == 0)
+			object->content_info = CACHEFILES_CONTENT_NO_DATA;
 		return;
 	}
 
 	/* The file is being expanded.  We don't need to do anything
-	 * particularly.  cookie->initial_size doesn't change and so the point
-	 * at which we have to download before doesn't change.
+	 * particularly.  The tail of the last block should have been cleared
+	 * both when it is written and when it is shrunk.
 	 */
+out:
+	object->object_size = new_size;
 	cookie->object_size = new_size;
 }
 
diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h
index c93324e0f98c76..664be64ab5384b 100644
--- a/fs/cachefiles/internal.h
+++ b/fs/cachefiles/internal.h
@@ -16,8 +16,6 @@
 #include <linux/cred.h>
 #include <linux/security.h>
 
-#define CACHEFILES_DIO_BLOCK_SIZE 4096
-
 struct cachefiles_cache;
 struct cachefiles_object;
 
@@ -51,12 +49,17 @@ struct cachefiles_object {
 	struct list_head		cache_link;	/* Link in cache->*_list */
 	struct file			*file;		/* The file representing this object */
 	char				*d_name;	/* Backing file name */
+	unsigned long			flags;
+#define CACHEFILES_OBJECT_USING_TMPFILE	0		/* Have an unlinked tmpfile */
+	uoff_t				object_size;	/* Size of the object stored
+							 * (independent of cookie->object_size for
+							 * coherency reasons)
+							 */
+	atomic64_t			read_limit;	/* Point beyond which uncommitted writes */
 	int				debug_id;
 	spinlock_t			lock;
 	refcount_t			ref;
-	enum cachefiles_content		content_info:8;	/* Info about content presence */
-	unsigned long			flags;
-#define CACHEFILES_OBJECT_USING_TMPFILE	0		/* Have an unlinked tmpfile */
+	enum cachefiles_content		content_info;	/* Info about content presence */
 };
 
 /*
@@ -203,11 +206,11 @@ extern bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
 				       enum fscache_want_state want_state);
 extern int __cachefiles_prepare_write(struct cachefiles_object *object,
 				      struct file *file,
-				      loff_t *_start, size_t *_len, size_t upper_len,
+				      uoff_t *_start, size_t *_len, size_t upper_len,
 				      bool no_space_allocated_yet);
 extern int __cachefiles_write(struct cachefiles_object *object,
 			      struct file *file,
-			      loff_t start_pos,
+			      uoff_t start_pos,
 			      struct iov_iter *iter,
 			      netfs_io_terminated_t term_func,
 			      void *term_func_priv);
@@ -280,6 +283,7 @@ void cachefiles_withdraw_volume(struct cachefiles_volume *volume);
 /*
  * xattr.c
  */
+int cachefiles_preset_object_xattr(struct cachefiles_object *object, struct file *file);
 extern int cachefiles_set_object_xattr(struct cachefiles_object *object);
 extern int cachefiles_check_auxdata(struct cachefiles_object *object,
 				    struct file *file);
diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c
index 9540ec25b3cb6c..4f547d97356ec3 100644
--- a/fs/cachefiles/io.c
+++ b/fs/cachefiles/io.c
@@ -19,7 +19,7 @@
 struct cachefiles_kiocb {
 	struct kiocb		iocb;
 	refcount_t		ki_refcnt;
-	loff_t			start;
+	uoff_t			start;
 	union {
 		size_t		skipped;
 		size_t		len;
@@ -32,6 +32,8 @@ struct cachefiles_kiocb {
 	u64			b_writing;
 };
 
+#define IS_ERR_VALUE_LL(x) unlikely((x) >= (unsigned long long)-MAX_ERRNO)
+
 static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
 {
 	if (refcount_dec_and_test(&ki->ki_refcnt)) {
@@ -73,7 +75,7 @@ static void cachefiles_read_complete(struct kiocb *iocb, long ret)
  * Initiate a read from the cache.
  */
 static int cachefiles_read(struct netfs_cache_resources *cres,
-			   loff_t start_pos,
+			   uoff_t start_pos,
 			   struct iov_iter *iter,
 			   enum netfs_read_from_hole read_hole,
 			   netfs_io_terminated_t term_func,
@@ -193,60 +195,81 @@ static int cachefiles_read(struct netfs_cache_resources *cres,
 }
 
 /*
- * Query the occupancy of the cache in a region, returning where the next chunk
- * of data starts and how long it is.
+ * Query the occupancy of the cache in a region, returning the extent of the
+ * next two chunks of cached data and the next hole.
  */
 static int cachefiles_query_occupancy(struct netfs_cache_resources *cres,
-				      loff_t start, size_t len, size_t granularity,
-				      loff_t *_data_start, size_t *_data_len)
+				      struct fscache_occupancy *occ)
 {
 	struct cachefiles_object *object;
+	struct inode *inode;
 	struct file *file;
-	loff_t off, off2;
-
-	*_data_start = -1;
-	*_data_len = 0;
+	uoff_t read_limit;
+	loff_t ret;
+	int i;
 
 	if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
 		return -ENOBUFS;
 
 	object = cachefiles_cres_object(cres);
 	file = cachefiles_cres_file(cres);
-	granularity = max_t(size_t, object->volume->cache->bsize, granularity);
+	inode = file_inode(file);
+	occ->granularity = object->volume->cache->bsize;
+	/* Read read_limit before content_info. */
+	read_limit = atomic64_read_acquire(&object->read_limit);
+
+	_enter("%pD,%llu,%llx-%llx/%llx",
+	       file, inode->i_ino, occ->query_from, occ->query_to, read_limit);
+
+	if (read_limit == 0)
+		goto done;
+
+	switch (READ_ONCE(object->content_info)) {
+	case CACHEFILES_CONTENT_ALL:
+	case CACHEFILES_CONTENT_SINGLE:
+		if (read_limit > occ->query_from) {
+			occ->cached_from[0] = 0;
+			occ->cached_to[0] = read_limit;
+			occ->cached_type[0] = FSCACHE_EXTENT_DATA;
+			occ->query_from = ULLONG_MAX;
+		}
+		goto done;
+	default:
+		break;
+	}
 
-	_enter("%pD,%llu,%llx,%zx/%llx",
-	       file, file_inode(file)->i_ino, start, len,
-	       i_size_read(file_inode(file)));
+	for (i = 0; i < ARRAY_SIZE(occ->cached_from); i++) {
+		ret = cachefiles_inject_read_error();
+		if (ret == 0)
+			ret = vfs_llseek(file, occ->query_from, SEEK_DATA);
+		if (IS_ERR_VALUE_LL(ret)) {
+			if (ret != -ENXIO)
+				return ret;
+			occ->query_from = ULLONG_MAX;
+			goto done;
+		}
+		occ->cached_type[i] = FSCACHE_EXTENT_DATA;
+		occ->cached_from[i] = ret;
+		occ->query_from = ret;
+
+		ret = cachefiles_inject_read_error();
+		if (ret == 0)
+			ret = vfs_llseek(file, occ->query_from, SEEK_HOLE);
+		if (IS_ERR_VALUE_LL(ret)) {
+			if (ret != -ENXIO)
+				return ret;
+			occ->query_from = ULLONG_MAX;
+			goto done;
+		}
+		occ->cached_to[i] = ret;
+		occ->query_from = ret;
+		if (occ->query_from >= occ->query_to)
+			break;
+	}
 
-	off = cachefiles_inject_read_error();
-	if (off == 0)
-		off = vfs_llseek(file, start, SEEK_DATA);
-	if (off == -ENXIO)
-		return -ENODATA; /* Beyond EOF */
-	if (off < 0 && off >= (loff_t)-MAX_ERRNO)
-		return -ENOBUFS; /* Error. */
-	if (round_up(off, granularity) >= start + len)
-		return -ENODATA; /* No data in range */
-
-	off2 = cachefiles_inject_read_error();
-	if (off2 == 0)
-		off2 = vfs_llseek(file, off, SEEK_HOLE);
-	if (off2 == -ENXIO)
-		return -ENODATA; /* Beyond EOF */
-	if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO)
-		return -ENOBUFS; /* Error. */
-
-	/* Round away partial blocks */
-	off = round_up(off, granularity);
-	off2 = round_down(off2, granularity);
-	if (off2 <= off)
-		return -ENODATA;
-
-	*_data_start = off;
-	if (off2 > start + len)
-		*_data_len = len;
-	else
-		*_data_len = off2 - off;
+done:
+	_debug("query[0] %llx-%llx", occ->cached_from[0], occ->cached_to[0]);
+	_debug("query[1] %llx-%llx", occ->cached_from[1], occ->cached_to[1]);
 	return 0;
 }
 
@@ -280,7 +303,7 @@ static void cachefiles_write_complete(struct kiocb *iocb, long ret)
  */
 int __cachefiles_write(struct cachefiles_object *object,
 		       struct file *file,
-		       loff_t start_pos,
+		       uoff_t start_pos,
 		       struct iov_iter *iter,
 		       netfs_io_terminated_t term_func,
 		       void *term_func_priv)
@@ -357,7 +380,7 @@ int __cachefiles_write(struct cachefiles_object *object,
 }
 
 static int cachefiles_write(struct netfs_cache_resources *cres,
-			    loff_t start_pos,
+			    uoff_t start_pos,
 			    struct iov_iter *iter,
 			    netfs_io_terminated_t term_func,
 			    void *term_func_priv)
@@ -375,127 +398,12 @@ static int cachefiles_write(struct netfs_cache_resources *cres,
 				  term_func, term_func_priv);
 }
 
-static inline enum netfs_io_source
-cachefiles_do_prepare_read(struct netfs_cache_resources *cres,
-			   loff_t start, size_t *_len, loff_t i_size,
-			   unsigned long *_flags, ino_t netfs_ino)
-{
-	enum cachefiles_prepare_read_trace why;
-	struct cachefiles_object *object = NULL;
-	struct cachefiles_cache *cache;
-	struct fscache_cookie *cookie = fscache_cres_cookie(cres);
-	const struct cred *saved_cred;
-	struct file *file = cachefiles_cres_file(cres);
-	enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER;
-	size_t len = *_len;
-	loff_t off, to;
-	ino_t ino = file ? file_inode(file)->i_ino : 0;
-
-	_enter("%zx @%llx/%llx", len, start, i_size);
-
-	if (start >= i_size) {
-		ret = NETFS_FILL_WITH_ZEROES;
-		why = cachefiles_trace_read_after_eof;
-		goto out_no_object;
-	}
-
-	if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) {
-		__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
-		why = cachefiles_trace_read_no_data;
-		goto out_no_object;
-	}
-
-	/* The object and the file may be being created in the background. */
-	if (!file) {
-		why = cachefiles_trace_read_no_file;
-		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ))
-			goto out_no_object;
-		file = cachefiles_cres_file(cres);
-		if (!file)
-			goto out_no_object;
-		ino = file_inode(file)->i_ino;
-	}
-
-	object = cachefiles_cres_object(cres);
-	cache = object->volume->cache;
-	cachefiles_begin_secure(cache, &saved_cred);
-	off = cachefiles_inject_read_error();
-	if (off == 0)
-		off = vfs_llseek(file, start, SEEK_DATA);
-	if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
-		if (off == (loff_t)-ENXIO) {
-			why = cachefiles_trace_read_seek_nxio;
-			goto download_and_store;
-		}
-		trace_cachefiles_io_error(object, file_inode(file), off,
-					  cachefiles_trace_seek_error);
-		why = cachefiles_trace_read_seek_error;
-		goto out;
-	}
-
-	if (off >= start + len) {
-		why = cachefiles_trace_read_found_hole;
-		goto download_and_store;
-	}
-
-	if (off > start) {
-		off = round_up(off, cache->bsize);
-		len = off - start;
-		*_len = len;
-		why = cachefiles_trace_read_found_part;
-		goto download_and_store;
-	}
-
-	to = cachefiles_inject_read_error();
-	if (to == 0)
-		to = vfs_llseek(file, start, SEEK_HOLE);
-	if (to < 0 && to >= (loff_t)-MAX_ERRNO) {
-		trace_cachefiles_io_error(object, file_inode(file), to,
-					  cachefiles_trace_seek_error);
-		why = cachefiles_trace_read_seek_error;
-		goto out;
-	}
-
-	if (to < start + len) {
-		if (start + len >= i_size)
-			to = round_up(to, cache->bsize);
-		else
-			to = round_down(to, cache->bsize);
-		len = to - start;
-		*_len = len;
-	}
-
-	why = cachefiles_trace_read_have_data;
-	ret = NETFS_READ_FROM_CACHE;
-	goto out;
-
-download_and_store:
-	__set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags);
-out:
-	cachefiles_end_secure(cache, saved_cred);
-out_no_object:
-	trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino);
-	return ret;
-}
-
-/*
- * Prepare a read operation, shortening it to a cached/uncached
- * boundary as appropriate.
- */
-static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq,
-						    unsigned long long i_size)
-{
-	return cachefiles_do_prepare_read(&subreq->rreq->cache_resources,
-					  subreq->start, &subreq->len, i_size,
-					  &subreq->flags, subreq->rreq->inode->i_ino);
-}
-
 /*
  * Prepare for a write to occur.
  */
 int __cachefiles_prepare_write(struct cachefiles_object *object,
 			       struct file *file,
-			       loff_t *_start, size_t *_len, size_t upper_len,
+			       uoff_t *_start, size_t *_len, size_t upper_len,
 			       bool no_space_allocated_yet)
 {
 	struct cachefiles_cache *cache = object->volume->cache;
@@ -504,7 +412,7 @@ int __cachefiles_prepare_write(struct cachefiles_object *object,
 	int ret;
 
 	/* Round to DIO size */
-	start = round_down(*_start, PAGE_SIZE);
+	start = round_down(*_start, cache->bsize);
 	if (start != *_start || *_len > upper_len) {
 		/* Probably asked to cache a streaming write written into the
 		 * pagecache when the cookie was temporarily out of service to
@@ -514,7 +422,7 @@ int __cachefiles_prepare_write(struct cachefiles_object *object,
 		return -ENOBUFS;
 	}
 
-	*_len = round_up(len, PAGE_SIZE);
+	*_len = round_up(len, cache->bsize);
 
 	/* We need to work out whether there's sufficient disk space to perform
 	 * the write - but we can skip that check if we have space already
@@ -540,10 +448,14 @@ int __cachefiles_prepare_write(struct cachefiles_object *object,
 	 * space, we need to see if it's fully allocated.  If it's not, we may
 	 * want to cull it.
 	 */
-	if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
-				 cachefiles_has_space_check) == 0)
+	ret = cachefiles_has_space(cache, 0, *_len / cache->bsize,
+				   cachefiles_has_space_check);
+	if (ret == 0)
 		return 0; /* Enough space to simply overwrite the whole block */
 
+	if (ret == -ENOBUFS)
+		trace_cachefiles_no_space(object, cachefiles_trace_write_nospace_2);
+
 	pos = cachefiles_inject_read_error();
 	if (pos == 0)
 		pos = vfs_llseek(file, start, SEEK_HOLE);
@@ -572,13 +484,16 @@ int __cachefiles_prepare_write(struct cachefiles_object *object,
 	return ret;
 
 check_space:
-	return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE,
-				    cachefiles_has_space_for_write);
+	ret = cachefiles_has_space(cache, 0, *_len / cache->bsize,
+				   cachefiles_has_space_for_write);
+	if (ret == -ENOBUFS)
+		trace_cachefiles_no_space(object, cachefiles_trace_write_nospace);
+	return ret;
 }
 
 static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
-				    loff_t *_start, size_t *_len, size_t upper_len,
-				    loff_t i_size, bool no_space_allocated_yet)
+				    uoff_t *_start, size_t *_len, size_t upper_len,
+				    uoff_t i_size, bool no_space_allocated_yet)
 {
 	struct cachefiles_object *object = cachefiles_cres_object(cres);
 	struct cachefiles_cache *cache = object->volume->cache;
@@ -612,10 +527,14 @@ static void cachefiles_prepare_write_subreq(struct netfs_io_subrequest *subreq)
 	stream->sreq_max_segs = BIO_MAX_VECS;
 
 	if (!cachefiles_cres_file(cres)) {
-		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE))
+		if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) {
+			trace_netfs_sreq(subreq, netfs_sreq_trace_cache_waitfail);
 			return netfs_prepare_write_failed(subreq);
-		if (!cachefiles_cres_file(cres))
+		}
+		if (!cachefiles_cres_file(cres)) {
+			trace_netfs_sreq(subreq, netfs_sreq_trace_cache_nofile);
 			return netfs_prepare_write_failed(subreq);
+		}
 	}
 }
 
@@ -628,16 +547,16 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
 	struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr];
 	const struct cred *saved_cred;
 	size_t off, pre, post, len = subreq->len;
-	loff_t start = subreq->start;
+	uoff_t start = subreq->start;
 	int ret;
 
 	_enter("W=%x[%x] %llx-%llx",
 	       wreq->debug_id, subreq->debug_index, start, start + len - 1);
 
 	/* We need to start on the cache granularity boundary */
-	off = start & (CACHEFILES_DIO_BLOCK_SIZE - 1);
+	off = start & (cache->bsize - 1);
 	if (off) {
-		pre = CACHEFILES_DIO_BLOCK_SIZE - off;
+		pre = cache->bsize - off;
 		if (pre >= len) {
 			fscache_count_dio_misfit();
 			netfs_write_subrequest_terminated(subreq, len);
@@ -651,8 +570,8 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
 
 	/* We also need to end on the cache granularity boundary */
 	if (start + len == wreq->i_size) {
-		size_t part = len % CACHEFILES_DIO_BLOCK_SIZE;
-		size_t need = CACHEFILES_DIO_BLOCK_SIZE - part;
+		size_t part = len & (cache->bsize - 1);
+		size_t need = cache->bsize - part;
 
 		if (part && stream->submit_extendable_to >= need) {
 			len += need;
@@ -661,7 +580,7 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
 		}
 	}
 
-	post = len & (CACHEFILES_DIO_BLOCK_SIZE - 1);
+	post = len & (cache->bsize - 1);
 	if (post) {
 		len -= post;
 		if (len == 0) {
@@ -688,6 +607,198 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq)
 			 netfs_write_subrequest_terminated, subreq);
 }
 
+/*
+ * Collect the result of buffered writeback to the cache.  This includes
+ * copying a read to the cache.  Netfslib collates the results, which might
+ * occur out of order, and delivers them to the cache so that it can update its
+ * content record.
+ *
+ * block_type is one of:
+ * - NETFS_CACHE_COLLECT_WRITE_DATA for a contiguous block of data
+ * - NETFS_CACHE_COLLECT_WRITE_GAP if a discontiguity was skipped
+ * - NETFS_CACHE_COLLECT_WRITE_CANCEL for a hole due to a failed/cancelled write
+ *
+ * The writes we made are all rounded out at both sides to the nearest DIO
+ * block boundary, so if the final block contains the EOF in the middle of it
+ * (rather than at the end), padding will have been written to the file.  The
+ * backing file's filesize will have been updated if the write extended the
+ * file; the filesize may still change due to outstanding subreqs.
+ *
+ * The metadata in the cache file xattr records the size of the object we have
+ * stored, but the cache file EOF only goes up to where we've cached data to
+ * and, furthermore, is rounded up to the nearest DIO block boundary.
+ *
+ * Concurrent updates should be protected against by the caller.  Netfslib
+ * holds NETFS_ICTX_WB_LOCK as a lock on writeback requests.  DIO writes
+ * invalidate the cookie and caching is kept disabled until all users have
+ * unused the cookie.
+ */
+static void cachefiles_collect_write(struct netfs_io_request *wreq,
+				     uoff_t start, size_t len,
+				     enum netfs_cache_collect block_type)
+{
+	struct netfs_cache_resources *cres = &wreq->cache_resources;
+	struct cachefiles_object *object = cachefiles_cres_object(cres);
+	struct cachefiles_cache *cache = object->volume->cache;
+	struct inode *inode;
+	struct file *file = cachefiles_cres_file(cres);
+	uoff_t read_limit;
+	uoff_t old_size = cres->cache_i_size;
+	uoff_t new_size;
+	uoff_t data_to = object->object_size;
+	uoff_t end = start + len;
+	int ret;
+
+	if (!file)
+		return;
+
+	inode = file_inode(file);
+	new_size = i_size_read(inode);
+
+	_enter("%llx,%zx,%x", start, len, cache->bsize);
+
+	if (WARN_ON(old_size	& (cache->bsize - 1)) ||
+	    WARN_ON(new_size	& (cache->bsize - 1)) ||
+	    WARN_ON(start	& (cache->bsize - 1)) ||
+	    WARN_ON(len		& (cache->bsize - 1))) {
+		trace_cachefiles_io_error(object, inode, -EIO,
+					  cachefiles_trace_alignment_error);
+		cachefiles_remove_object_xattr(cache, object, file->f_path.dentry);
+		return;
+	}
+
+	/* If this is recording a gap, due to discontiguous writes or lack of
+	 * cache space, then a hole may have been introduced into the backing
+	 * file.  Treat it as a zero-length data block.
+	 */
+	if (block_type == NETFS_CACHE_COLLECT_WRITE_GAP ||
+	    block_type == NETFS_CACHE_COLLECT_WRITE_CANCEL) {
+		start = end;
+		len = 0;
+	}
+
+	/* Zeroth case: Single monolithic files are handled specially.
+	 */
+	if (wreq->origin == NETFS_WRITEBACK_SINGLE) {
+		if (block_type == NETFS_CACHE_COLLECT_WRITE_GAP ||
+		    block_type == NETFS_CACHE_COLLECT_WRITE_CANCEL) {
+			trace_cachefiles_trunc(object, inode, data_to, 0,
+					       cachefiles_trunc_zap);
+			ret = cachefiles_inject_remove_error();
+			if (ret == 0)
+				ret = vfs_truncate(&file->f_path, 0);
+			if (ret < 0) {
+				trace_cachefiles_io_error(object, inode, ret,
+							  cachefiles_trace_trunc_error);
+				cachefiles_io_error_obj(object, "truncate failed %d", ret);
+				cachefiles_remove_object_xattr(cache, object, file->f_path.dentry);
+				return;
+			}
+
+			object->content_info = CACHEFILES_CONTENT_NO_DATA;
+			read_limit = 0;
+		} else {
+			object->content_info = CACHEFILES_CONTENT_SINGLE;
+			read_limit = len;
+		}
+		goto update_sizes_2;
+	}
+
+	/* First case: The backing file was empty. */
+	if (old_size == 0) {
+		if (start == 0)
+			object->content_info = CACHEFILES_CONTENT_ALL;
+		else
+			object->content_info = CACHEFILES_CONTENT_BACKFS_MAP;
+		goto update_sizes;
+	}
+
+	/* Second case: The backing file is entirely within the old object size
+	 * and thus there can be no partial tail block to deal with in the
+	 * cache file.
+	 */
+	if (old_size <= data_to) {
+		if (start > old_size)
+			goto discontiguous;
+		goto update_sizes;
+	}
+
+	/* Third case: The write happened entirely within the bounds of the
+	 * current cache file's size.
+	 */
+	if (end <= old_size)
+		goto update_sizes;
+
+	/* Fourth case: The write overwrote the partial tail block and extended
+	 * the file.  We only need to update the object size because netfslib
+	 * rounds out/pads cache writes to whole disk blocks.
+	 */
+	if (start < old_size)
+		goto update_sizes;
+
+	/* Fifth case: The write started from the end of the whole tail block
+	 * and extended the file.  Just extend our notion of the filesize.
+	 */
+	if (start == old_size && old_size == data_to)
+		goto update_sizes;
+
+	/* Sixth case: The write continued on from the partial tail block and
+	 * extended the file.  Need to clear the gap.
+	 */
+	if (start == old_size && old_size > data_to)
+		goto clear_gap;
+
+discontiguous:
+	/* Seventh case: The write was beyond the EOF on the cache file, so now
+	 * there's a hole in the file and we can no longer say in the metadata
+	 * that we can assume we have it all.  We may also need to clear the
+	 * end of the partial tail block.
+	 */
+	/* TODO: For the moment, we will have to use SEEK_HOLE/SEEK_DATA. */
+	if (object->content_info != CACHEFILES_CONTENT_BACKFS_MAP) {
+		object->content_info = CACHEFILES_CONTENT_BACKFS_MAP;
+		trace_cachefiles_coherency(object, inode->i_ino, data_to, NULL,
+					   CACHEFILES_CONTENT_BACKFS_MAP,
+					   cachefiles_coherency_discontiguous);
+	}
+
+clear_gap:
+	/* We need to clear any partial padding that got jumped over.  It
+	 * *should* be all zeros, but shared-writable mmap exists...
+	 */
+	if (old_size > data_to) {
+		trace_cachefiles_trunc(object, inode, data_to, old_size,
+				       cachefiles_trunc_clear_padding);
+		ret = cachefiles_inject_write_error();
+		if (ret == 0)
+			ret = vfs_fallocate(file, FALLOC_FL_ZERO_RANGE,
+					    data_to, old_size - data_to);
+		if (ret < 0) {
+			trace_cachefiles_io_error(object, inode, ret,
+						  cachefiles_trace_fallocate_error);
+			cachefiles_io_error_obj(object, "fallocate zero pad failed %d", ret);
+			cachefiles_remove_object_xattr(cache, object, file->f_path.dentry);
+			return;
+		}
+	}
+
+update_sizes:
+	read_limit = umax(old_size, end);
+update_sizes_2:
+	cres->cache_i_size = read_limit;
+
+	/* We need to be careful setting the object_size: we may have written
+	 * more to the cache than to the server (due to cache DIO rounding) and
+	 * the i_size set on the netfs inode may include unwritten data that
+	 * the server doesn't know about yet.
+	 */
+	object->object_size = umin(read_limit, wreq->i_size);
+
+	/* Raise the limit at which reads can access the file. */
+	/* Update read_limit after content_info */
+	atomic64_set_release(&object->read_limit, read_limit);
+}
+
 /*
  * Clean up an operation.
  */
@@ -705,10 +816,10 @@ static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
 	.read			= cachefiles_read,
 	.write			= cachefiles_write,
 	.issue_write		= cachefiles_issue_write,
-	.prepare_read		= cachefiles_prepare_read,
 	.prepare_write		= cachefiles_prepare_write,
 	.prepare_write_subreq	= cachefiles_prepare_write_subreq,
 	.query_occupancy	= cachefiles_query_occupancy,
+	.collect_write		= cachefiles_collect_write,
 };
 
 /*
@@ -718,13 +829,20 @@ bool cachefiles_begin_operation(struct netfs_cache_resources *cres,
 				enum fscache_want_state want_state)
 {
 	struct cachefiles_object *object = cachefiles_cres_object(cres);
+	struct file *file;
+
+	cres->dio_size = object->volume->cache->bsize;
 
 	if (!cachefiles_cres_file(cres)) {
 		cres->ops = &cachefiles_netfs_cache_ops;
+		cres->object_id = object->debug_id;
 		if (object->file) {
 			spin_lock(&object->lock);
-			if (!cres->cache_priv2 && object->file)
-				cres->cache_priv2 = get_file(object->file);
+			file = object->file;
+			if (!cres->cache_priv2 && file) {
+				cres->cache_priv2 = get_file(file);
+				cres->cache_i_size = i_size_read(file_inode(file));
+			}
 			spin_unlock(&object->lock);
 		}
 	}
diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c
index 88955249a1a64b..ef656a319ede2e 100644
--- a/fs/cachefiles/namei.c
+++ b/fs/cachefiles/namei.c
@@ -117,8 +117,11 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache,
 	if (d_is_negative(subdir)) {
 		ret = cachefiles_has_space(cache, 1, 0,
 					   cachefiles_has_space_for_create);
-		if (ret < 0)
+		if (ret < 0) {
+			if (ret == -ENOBUFS)
+				trace_cachefiles_no_space(NULL, cachefiles_trace_mkdir_nospace);
 			goto mkdir_error;
+		}
 
 		_debug("attempt mkdir");
 
@@ -414,7 +417,6 @@ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object)
 	struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash];
 	struct file *file;
 	const struct path parentpath = { .mnt = cache->mnt, .dentry = fan };
-	uint64_t ni_size;
 	long ret;
 
 
@@ -442,31 +444,20 @@ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object)
 	if (!cachefiles_mark_inode_in_use(object, file_inode(file)))
 		WARN_ON(1);
 
-	ni_size = object->cookie->object_size;
-	ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE);
-
-	if (ni_size > 0) {
-		trace_cachefiles_trunc(object, file_inode(file), 0, ni_size,
-				       cachefiles_trunc_expand_tmpfile);
-		ret = cachefiles_inject_write_error();
-		if (ret == 0)
-			ret = vfs_truncate(&file->f_path, ni_size);
-		if (ret < 0) {
-			trace_cachefiles_vfs_error(
-				object, file_inode(file), ret,
-				cachefiles_trace_trunc_error);
-			goto err_unuse;
-		}
-	}
-
 	ret = -EINVAL;
 	if (unlikely(!file->f_op->read_iter) ||
 	    unlikely(!file->f_op->write_iter)) {
 		pr_notice("Cache does not support read_iter and write_iter\n");
 		goto err_unuse;
 	}
+
+	/* Preallocate space for the xattr. */
+	ret = cachefiles_preset_object_xattr(object, file);
+	if (ret < 0)
+		goto err_unuse;
 out:
 	cachefiles_end_secure(cache, saved_cred);
+	object->content_info = CACHEFILES_CONTENT_ALL;
 	return file;
 
 err_unuse:
@@ -487,8 +478,11 @@ static bool cachefiles_create_file(struct cachefiles_object *object)
 
 	ret = cachefiles_has_space(object->volume->cache, 1, 0,
 				   cachefiles_has_space_for_create);
-	if (ret < 0)
+	if (ret < 0) {
+		if (ret == -ENOBUFS)
+			trace_cachefiles_no_space(object, cachefiles_trace_create_nospace);
 		return false;
+	}
 
 	file = cachefiles_create_tmpfile(object);
 	if (IS_ERR(file))
diff --git a/fs/cachefiles/security.c b/fs/cachefiles/security.c
index fc6611886b3b5e..eefe5453b904e6 100644
--- a/fs/cachefiles/security.c
+++ b/fs/cachefiles/security.c
@@ -51,14 +51,14 @@ static int cachefiles_check_cache_dir(struct cachefiles_cache *cache,
 {
 	int ret;
 
-	ret = security_inode_mkdir(d_backing_inode(root), root, 0);
+	ret = security_inode_mkdir(&nop_mnt_idmap, d_backing_inode(root), root, 0);
 	if (ret < 0) {
 		pr_err("Security denies permission to make dirs: error %d",
 		       ret);
 		return ret;
 	}
 
-	ret = security_inode_create(d_backing_inode(root), root, 0);
+	ret = security_inode_create(&nop_mnt_idmap, d_backing_inode(root), root, 0);
 	if (ret < 0)
 		pr_err("Security denies permission to create files: error %d",
 		       ret);
diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c
index c70bf67e52b01a..8ebb713482e3e7 100644
--- a/fs/cachefiles/xattr.c
+++ b/fs/cachefiles/xattr.c
@@ -34,6 +34,57 @@ struct cachefiles_vol_xattr {
 	__u8	data[];		/* netfs volume coherency data */
 } __packed;
 
+/*
+ * Preset the state xattr on a cache file to allocate space for it.
+ */
+int cachefiles_preset_object_xattr(struct cachefiles_object *object, struct file *file)
+{
+	struct cachefiles_xattr *buf;
+	struct dentry *dentry = file->f_path.dentry;
+	unsigned int len = object->cookie->aux_len;
+	int ret;
+
+	buf = kzalloc(sizeof(struct cachefiles_xattr) + min(len, sizeof(__be64)), GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	buf->type		= CACHEFILES_COOKIE_TYPE_DATA;
+	buf->content		= CACHEFILES_CONTENT_DIRTY;
+
+	ret = cachefiles_inject_write_error();
+	if (ret == 0) {
+		ret = mnt_want_write_file(file);
+		if (ret == 0) {
+			ret = vfs_setxattr(&nop_mnt_idmap, dentry,
+					   cachefiles_xattr_cache, buf,
+					   sizeof(struct cachefiles_xattr) + len, 0);
+			mnt_drop_write_file(file);
+		}
+	}
+	if (ret < 0) {
+		trace_cachefiles_vfs_error(object, file_inode(file), ret,
+					   cachefiles_trace_setxattr_error);
+		trace_cachefiles_coherency(object, file_inode(file)->i_ino,
+					   object->object_size,
+					   buf->data, buf->content,
+					   cachefiles_coherency_set_fail);
+		switch (ret) {
+		case -ENOMEM:
+		case -ENOSPC:
+			break;
+		default:
+			cachefiles_io_error_obj(
+				object,
+				"Failed to set xattr with error %d", ret);
+			break;
+		}
+	}
+
+	kfree(buf);
+	_leave(" = %d", ret);
+	return ret;
+}
+
 /*
  * set the state xattr on a cache file
  */
@@ -43,6 +94,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object)
 	struct dentry *dentry;
 	struct file *file = object->file;
 	unsigned int len = object->cookie->aux_len;
+	uoff_t object_size = object->cookie->object_size;
 	int ret;
 
 	if (!file)
@@ -55,7 +107,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object)
 	if (!buf)
 		return -ENOMEM;
 
-	buf->object_size	= cpu_to_be64(object->cookie->object_size);
+	buf->object_size	= cpu_to_be64(object_size);
 	buf->zero_point		= 0;
 	buf->type		= CACHEFILES_COOKIE_TYPE_DATA;
 	buf->content		= object->content_info;
@@ -79,15 +131,21 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object)
 		trace_cachefiles_vfs_error(object, file_inode(file), ret,
 					   cachefiles_trace_setxattr_error);
 		trace_cachefiles_coherency(object, file_inode(file)->i_ino,
-					   buf->data, buf->content,
+					   object_size, buf->data, buf->content,
 					   cachefiles_coherency_set_fail);
-		if (ret != -ENOMEM)
+		switch (ret) {
+		case -ENOMEM:
+			break;
+		case -ENOSPC:
+		default:
 			cachefiles_io_error_obj(
 				object,
 				"Failed to set xattr with error %d", ret);
+			break;
+		}
 	} else {
 		trace_cachefiles_coherency(object, file_inode(file)->i_ino,
-					   buf->data, buf->content,
+					   object_size, buf->data, buf->content,
 					   cachefiles_coherency_set_ok);
 	}
 
@@ -103,10 +161,12 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file
 {
 	struct cachefiles_xattr *buf;
 	struct dentry *dentry = file->f_path.dentry;
+	struct inode *inode = file_inode(file);
 	unsigned int len = object->cookie->aux_len, tlen;
 	const void *p = fscache_get_aux(object->cookie);
 	enum cachefiles_coherency_trace why;
 	ssize_t xlen;
+	uoff_t obj_size;
 	int ret = -ESTALE;
 
 	tlen = sizeof(struct cachefiles_xattr) + len;
@@ -121,34 +181,39 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file
 	if (xlen != tlen) {
 		if (xlen < 0) {
 			ret = xlen;
-			trace_cachefiles_vfs_error(object, file_inode(file), xlen,
+			trace_cachefiles_vfs_error(object, inode, xlen,
 						   cachefiles_trace_getxattr_error);
 		}
 		if (xlen == -EIO)
 			cachefiles_io_error_obj(
 				object,
 				"Failed to read aux with error %zd", xlen);
+		obj_size = 0;
 		why = cachefiles_coherency_check_xattr;
 		goto out;
 	}
 
+	obj_size = be64_to_cpu(buf->object_size);
 	if (buf->type != CACHEFILES_COOKIE_TYPE_DATA) {
 		why = cachefiles_coherency_check_type;
 	} else if (memcmp(buf->data, p, len) != 0) {
 		why = cachefiles_coherency_check_aux;
-	} else if (be64_to_cpu(buf->object_size) != object->cookie->object_size) {
+	} else if (obj_size != object->cookie->object_size) {
 		why = cachefiles_coherency_check_objsize;
 	} else if (buf->content == CACHEFILES_CONTENT_DIRTY) {
 		// TODO: Begin conflict resolution
 		pr_warn("Dirty object in cache\n");
 		why = cachefiles_coherency_check_dirty;
 	} else {
+		object->content_info = buf->content;
+		object->object_size = obj_size;
+		atomic64_set(&object->read_limit, i_size_read(inode));
 		why = cachefiles_coherency_check_ok;
 		ret = 0;
 	}
 
 out:
-	trace_cachefiles_coherency(object, file_inode(file)->i_ino,
+	trace_cachefiles_coherency(object, inode->i_ino, obj_size,
 				   buf->data, buf->content, why);
 	kfree(buf);
 	return ret;
@@ -163,6 +228,9 @@ int cachefiles_remove_object_xattr(struct cachefiles_cache *cache,
 {
 	int ret;
 
+	trace_cachefiles_coherency(object, d_inode(dentry)->i_ino, 0, NULL, 0,
+				   cachefiles_coherency_remove);
+
 	ret = cachefiles_inject_remove_error();
 	if (ret == 0) {
 		ret = mnt_want_write(cache->mnt);
diff --git a/fs/ceph/Kconfig b/fs/ceph/Kconfig
index 3d64a316ca31d3..aa6ccd7794d233 100644
--- a/fs/ceph/Kconfig
+++ b/fs/ceph/Kconfig
@@ -4,6 +4,7 @@ config CEPH_FS
 	depends on INET
 	select CEPH_LIB
 	select NETFS_SUPPORT
+	select NETFS_PGPRIV2
 	select FS_ENCRYPTION_ALGS if FS_ENCRYPTION
 	default n
 	help
diff --git a/fs/ceph/acl.c b/fs/ceph/acl.c
index 85d3dd48b167a3..124f07ae5b2db6 100644
--- a/fs/ceph/acl.c
+++ b/fs/ceph/acl.c
@@ -87,7 +87,7 @@ struct posix_acl *ceph_get_acl(struct inode *inode, int type, bool rcu)
 	return acl;
 }
 
-int ceph_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ceph_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 	int ret = 0;
diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index e598b2d424ec16..21d52d84984244 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -65,14 +65,12 @@
 	(CONGESTION_ON_THRESH(congestion_kb) -				\
 	 (CONGESTION_ON_THRESH(congestion_kb) >> 2))
 
-static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len,
+static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len,
 					struct folio **foliop, void **_fsdata);
 
 static inline struct ceph_snap_context *page_snap_context(struct page *page)
 {
-	if (PagePrivate(page))
-		return (void *)page->private;
-	return NULL;
+	return (void *)page_private(page);
 }
 
 /*
@@ -124,8 +122,8 @@ static bool ceph_dirty_folio(struct address_space *mapping, struct folio *folio)
 	spin_unlock(&ci->i_ceph_lock);
 
 	/*
-	 * Reference snap context in folio->private.  Also set
-	 * PagePrivate so that we get invalidate_folio callback.
+	 * Reference snap context in folio->private. Setting folio->private is
+	 * what gets us the invalidate_folio callback.
 	 */
 	VM_WARN_ON_FOLIO(folio->private, folio);
 	folio_attach_private(folio, snapc);
@@ -1868,7 +1866,7 @@ ceph_find_incompatible(struct folio *folio)
 	return NULL;
 }
 
-static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len,
+static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len,
 					struct folio **foliop, void **_fsdata)
 {
 	struct inode *inode = file_inode(file);
diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c
index 2e5c0ccb1b3464..d9615d67bf1c01 100644
--- a/fs/ceph/dir.c
+++ b/fs/ceph/dir.c
@@ -921,7 +921,7 @@ int ceph_handle_notrace_create(struct inode *dir, struct dentry *dentry)
 	return PTR_ERR(result);
 }
 
-static int ceph_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int ceph_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb);
@@ -988,7 +988,7 @@ static int ceph_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int ceph_create(struct mnt_idmap *idmap, struct inode *dir,
+static int ceph_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	return ceph_mknod(idmap, dir, dentry, mode, 0);
@@ -1032,7 +1032,7 @@ static int prep_encrypted_symlink_target(struct ceph_mds_request *req,
 }
 #endif
 
-static int ceph_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int ceph_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, const char *dest)
 {
 	struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb);
@@ -1106,7 +1106,7 @@ static int ceph_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static struct dentry *ceph_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ceph_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb);
@@ -1478,7 +1478,7 @@ static int ceph_unlink(struct inode *dir, struct dentry *dentry)
 	return err;
 }
 
-static int ceph_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int ceph_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		       struct dentry *old_dentry, struct inode *new_dir,
 		       struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index bd3e3f5c269e89..2c994c08ed4b3e 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -795,7 +795,7 @@ static int ceph_finish_async_create(struct inode *dir, struct inode *inode,
 int ceph_atomic_open(struct inode *dir, struct dentry *dentry,
 		     struct file *file, unsigned flags, umode_t mode)
 {
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dir->i_sb);
 	struct ceph_client *cl = fsc->client;
 	struct ceph_mds_client *mdsc = fsc->mdsc;
diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c
index d52e2b389e0b62..a695dba82554ee 100644
--- a/fs/ceph/inode.c
+++ b/fs/ceph/inode.c
@@ -2398,7 +2398,7 @@ static const char *ceph_encrypted_get_link(struct dentry *dentry,
 				   done);
 }
 
-static int ceph_encrypted_symlink_getattr(struct mnt_idmap *idmap,
+static int ceph_encrypted_symlink_getattr(const struct mnt_idmap *idmap,
 					  const struct path *path,
 					  struct kstat *stat, u32 request_mask,
 					  unsigned int query_flags)
@@ -2568,7 +2568,7 @@ static int fill_fscrypt_truncate(struct inode *inode,
 	return ret;
 }
 
-int __ceph_setattr(struct mnt_idmap *idmap, struct inode *inode,
+int __ceph_setattr(const struct mnt_idmap *idmap, struct inode *inode,
 		   struct iattr *attr, struct ceph_iattr *cia)
 {
 	struct ceph_inode_info *ci = ceph_inode(inode);
@@ -2921,7 +2921,7 @@ int __ceph_setattr(struct mnt_idmap *idmap, struct inode *inode,
 /*
  * setattr
  */
-int ceph_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ceph_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -3098,7 +3098,7 @@ int ceph_do_getvxattr(struct inode *inode, const char *name, void *value,
  * Check inode permissions.  We verify we have a valid value for
  * the AUTH cap, then call the generic handler.
  */
-int ceph_permission(struct mnt_idmap *idmap, struct inode *inode,
+int ceph_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		    int mask)
 {
 	int err;
@@ -3145,7 +3145,7 @@ static int statx_to_caps(u32 want, umode_t mode)
  * Get all the attributes. If we have sufficient caps for the requested attrs,
  * then we can avoid talking to the MDS at all.
  */
-int ceph_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ceph_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	struct inode *inode = d_inode(path->dentry);
diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h
index e7a262c9c2ab4b..ea48ec5383ef8a 100644
--- a/fs/ceph/mds_client.h
+++ b/fs/ceph/mds_client.h
@@ -375,7 +375,7 @@ struct ceph_mds_request {
 	int r_fmode;        /* file mode, if expecting cap */
 	int r_request_release_offset;
 	const struct cred *r_cred;
-	struct mnt_idmap *r_mnt_idmap;
+	const struct mnt_idmap *r_mnt_idmap;
 	struct timespec64 r_stamp;
 
 	/* for choosing which mds to send this request to */
diff --git a/fs/ceph/super.h b/fs/ceph/super.h
index 72d4e30304dcab..a033331bb151a5 100644
--- a/fs/ceph/super.h
+++ b/fs/ceph/super.h
@@ -1166,18 +1166,18 @@ static inline int ceph_do_getattr(struct inode *inode, int mask, bool force)
 {
 	return __ceph_do_getattr(inode, NULL, mask, force);
 }
-extern int ceph_permission(struct mnt_idmap *idmap,
+extern int ceph_permission(const struct mnt_idmap *idmap,
 			   struct inode *inode, int mask);
 
 struct ceph_iattr {
 	struct ceph_fscrypt_auth	*fscrypt_auth;
 };
 
-extern int __ceph_setattr(struct mnt_idmap *idmap, struct inode *inode,
+extern int __ceph_setattr(const struct mnt_idmap *idmap, struct inode *inode,
 			  struct iattr *attr, struct ceph_iattr *cia);
-extern int ceph_setattr(struct mnt_idmap *idmap,
+extern int ceph_setattr(const struct mnt_idmap *idmap,
 			struct dentry *dentry, struct iattr *attr);
-extern int ceph_getattr(struct mnt_idmap *idmap,
+extern int ceph_getattr(const struct mnt_idmap *idmap,
 			const struct path *path, struct kstat *stat,
 			u32 request_mask, unsigned int flags);
 void ceph_inode_shutdown(struct inode *inode);
@@ -1252,7 +1252,7 @@ void ceph_release_acl_sec_ctx(struct ceph_acl_sec_ctx *as_ctx);
 #ifdef CONFIG_CEPH_FS_POSIX_ACL
 
 struct posix_acl *ceph_get_acl(struct inode *, int, bool);
-int ceph_set_acl(struct mnt_idmap *idmap,
+int ceph_set_acl(const struct mnt_idmap *idmap,
 		 struct dentry *dentry, struct posix_acl *acl, int type);
 int ceph_pre_init_acls(struct inode *dir, umode_t *mode,
 		       struct ceph_acl_sec_ctx *as_ctx);
diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c
index cc4ffbbcb719a7..7d77214c76c69e 100644
--- a/fs/ceph/xattr.c
+++ b/fs/ceph/xattr.c
@@ -1352,7 +1352,7 @@ static int ceph_get_xattr_handler(const struct xattr_handler *handler,
 }
 
 static int ceph_set_xattr_handler(const struct xattr_handler *handler,
-				  struct mnt_idmap *idmap,
+				  const struct mnt_idmap *idmap,
 				  struct dentry *unused, struct inode *inode,
 				  const char *name, const void *value,
 				  size_t size, int flags)
diff --git a/fs/char_dev.c b/fs/char_dev.c
index 00229e25c10f61..5ce5423f6c991c 100644
--- a/fs/char_dev.c
+++ b/fs/char_dev.c
@@ -280,7 +280,9 @@ int __register_chrdev(unsigned int major, unsigned int baseminor,
 
 	cdev->owner = fops->owner;
 	cdev->ops = fops;
-	kobject_set_name(&cdev->kobj, "%s", name);
+	err = kobject_set_name(&cdev->kobj, "%s", name);
+	if (err)
+		goto out;
 
 	err = cdev_add(cdev, MKDEV(cd->major, baseminor), count);
 	if (err)
diff --git a/fs/coda/coda_linux.h b/fs/coda/coda_linux.h
index dd6277d87afb7d..0c0d5f81653c60 100644
--- a/fs/coda/coda_linux.h
+++ b/fs/coda/coda_linux.h
@@ -46,12 +46,12 @@ extern const struct file_operations coda_ioctl_operations;
 /* operations shared over more than one file */
 int coda_open(struct inode *i, struct file *f);
 int coda_release(struct inode *i, struct file *f);
-int coda_permission(struct mnt_idmap *idmap, struct inode *inode,
+int coda_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		    int mask);
 int coda_revalidate_inode(struct inode *);
-int coda_getattr(struct mnt_idmap *, const struct path *, struct kstat *,
+int coda_getattr(const struct mnt_idmap *, const struct path *, struct kstat *,
 		 u32, unsigned int);
-int coda_setattr(struct mnt_idmap *, struct dentry *, struct iattr *);
+int coda_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *);
 
 /* this file:  helpers */
 char *coda_f2s(struct CodaFid *f);
diff --git a/fs/coda/dir.c b/fs/coda/dir.c
index 67148edfadeef4..a85be5962e6264 100644
--- a/fs/coda/dir.c
+++ b/fs/coda/dir.c
@@ -73,7 +73,7 @@ static struct dentry *coda_lookup(struct inode *dir, struct dentry *entry, unsig
 }
 
 
-int coda_permission(struct mnt_idmap *idmap, struct inode *inode,
+int coda_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		    int mask)
 {
 	int error;
@@ -133,7 +133,7 @@ static inline void coda_dir_drop_nlink(struct inode *dir)
 }
 
 /* creation routines: create, mknod, mkdir, link, symlink */
-static int coda_create(struct mnt_idmap *idmap, struct inode *dir,
+static int coda_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *de, umode_t mode)
 {
 	int error;
@@ -166,7 +166,7 @@ static int coda_create(struct mnt_idmap *idmap, struct inode *dir,
 	return error;
 }
 
-static struct dentry *coda_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *coda_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *de, umode_t mode)
 {
 	struct inode *inode;
@@ -233,7 +233,7 @@ static int coda_link(struct dentry *source_de, struct inode *dir_inode,
 }
 
 
-static int coda_symlink(struct mnt_idmap *idmap,
+static int coda_symlink(const struct mnt_idmap *idmap,
 			struct inode *dir_inode, struct dentry *de,
 			const char *symname)
 {
@@ -300,7 +300,7 @@ static int coda_rmdir(struct inode *dir, struct dentry *de)
 }
 
 /* rename */
-static int coda_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int coda_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		       struct dentry *old_dentry, struct inode *new_dir,
 		       struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/coda/inode.c b/fs/coda/inode.c
index 40b43866e6a549..c449954e23c216 100644
--- a/fs/coda/inode.c
+++ b/fs/coda/inode.c
@@ -294,7 +294,7 @@ static void coda_evict_inode(struct inode *inode)
 	coda_cache_clear_inode(inode);
 }
 
-int coda_getattr(struct mnt_idmap *idmap, const struct path *path,
+int coda_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	int err = coda_revalidate_inode(d_inode(path->dentry));
@@ -304,7 +304,7 @@ int coda_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return err;
 }
 
-int coda_setattr(struct mnt_idmap *idmap, struct dentry *de,
+int coda_setattr(const struct mnt_idmap *idmap, struct dentry *de,
 		 struct iattr *iattr)
 {
 	struct inode *inode = d_inode(de);
diff --git a/fs/coda/pioctl.c b/fs/coda/pioctl.c
index 36e35c15561a98..c457e9bab94bdc 100644
--- a/fs/coda/pioctl.c
+++ b/fs/coda/pioctl.c
@@ -24,7 +24,7 @@
 #include "coda_linux.h"
 
 /* pioctl ops */
-static int coda_ioctl_permission(struct mnt_idmap *idmap,
+static int coda_ioctl_permission(const struct mnt_idmap *idmap,
 				 struct inode *inode, int mask);
 static long coda_pioctl(struct file *filp, unsigned int cmd,
 			unsigned long user_data);
@@ -41,7 +41,7 @@ const struct file_operations coda_ioctl_operations = {
 };
 
 /* the coda pioctl inode ops */
-static int coda_ioctl_permission(struct mnt_idmap *idmap,
+static int coda_ioctl_permission(const struct mnt_idmap *idmap,
 				 struct inode *inode, int mask)
 {
 	return (mask & MAY_EXEC) ? -EACCES : 0;
diff --git a/fs/configfs/configfs_internal.h b/fs/configfs/configfs_internal.h
index acdeea8e2d69ac..5f627e58f13504 100644
--- a/fs/configfs/configfs_internal.h
+++ b/fs/configfs/configfs_internal.h
@@ -76,7 +76,7 @@ extern int configfs_make_dirent(struct configfs_dirent *, struct dentry *,
 extern int configfs_dirent_is_ready(struct configfs_dirent *);
 
 extern const unsigned char * configfs_get_name(struct configfs_dirent *sd);
-extern int configfs_setattr(struct mnt_idmap *idmap,
+extern int configfs_setattr(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct iattr *iattr);
 
 extern struct dentry *configfs_pin_fs(void);
@@ -90,7 +90,7 @@ extern const struct inode_operations configfs_root_inode_operations;
 extern const struct inode_operations configfs_symlink_inode_operations;
 extern const struct dentry_operations configfs_dentry_ops;
 
-extern int configfs_symlink(struct mnt_idmap *idmap,
+extern int configfs_symlink(const struct mnt_idmap *idmap,
 			    struct inode *dir, struct dentry *dentry,
 			    const char *symname);
 extern int configfs_unlink(struct inode *dir, struct dentry *dentry);
@@ -104,17 +104,17 @@ static inline struct config_item * to_item(struct dentry * dentry)
 	return ((struct config_item *) sd->s_element);
 }
 
-static inline struct configfs_attribute * to_attr(struct dentry * dentry)
+static inline const struct configfs_attribute * to_attr(struct dentry * dentry)
 {
 	struct configfs_dirent * sd = dentry->d_fsdata;
-	return ((struct configfs_attribute *) sd->s_element);
+	return ((const struct configfs_attribute *) sd->s_element);
 }
 
-static inline struct configfs_bin_attribute *to_bin_attr(struct dentry *dentry)
+static inline const struct configfs_bin_attribute *to_bin_attr(struct dentry *dentry)
 {
-	struct configfs_attribute *attr = to_attr(dentry);
+	const struct configfs_attribute *attr = to_attr(dentry);
 
-	return container_of(attr, struct configfs_bin_attribute, cb_attr);
+	return container_of_const(attr, struct configfs_bin_attribute, cb_attr);
 }
 
 static inline struct config_item *configfs_get_config_item(struct dentry *dentry)
diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c
index eda80c2a2d3888..0c80feec5926b1 100644
--- a/fs/configfs/dir.c
+++ b/fs/configfs/dir.c
@@ -470,7 +470,7 @@ static struct dentry * configfs_lookup(struct inode *dir,
 		 */
 		if ((sd->s_type & CONFIGFS_NOT_PINNED) &&
 		    !strcmp(configfs_get_name(sd), dentry->d_name.name)) {
-			struct configfs_attribute *attr = sd->s_element;
+			const struct configfs_attribute *attr = sd->s_element;
 			umode_t mode = (attr->ca_mode & S_IALLUGO) | S_IFREG;
 
 			dentry->d_fsdata = configfs_get(sd);
@@ -631,8 +631,8 @@ static int populate_attrs(struct config_item *item)
 {
 	const struct config_item_type *t = item->ci_type;
 	const struct configfs_group_operations *ops;
-	struct configfs_attribute *attr;
-	struct configfs_bin_attribute *bin_attr;
+	const struct configfs_attribute *attr;
+	const struct configfs_bin_attribute *bin_attr;
 	int error = 0;
 	int i;
 
@@ -641,8 +641,8 @@ static int populate_attrs(struct config_item *item)
 
 	ops = t->ct_group_ops;
 
-	if (t->ct_attrs) {
-		for (i = 0; (attr = t->ct_attrs[i]) != NULL; i++) {
+	if (t->ct_attrs_const) {
+		for (i = 0; (attr = t->ct_attrs_const[i]) != NULL; i++) {
 			if (ops && ops->is_visible && !ops->is_visible(item, attr, i))
 				continue;
 
@@ -1295,7 +1295,7 @@ int configfs_depend_item_unlocked(struct configfs_subsystem *caller_subsys,
 }
 EXPORT_SYMBOL(configfs_depend_item_unlocked);
 
-static struct dentry *configfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *configfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				     struct dentry *dentry, umode_t mode)
 {
 	int ret = 0;
diff --git a/fs/configfs/file.c b/fs/configfs/file.c
index a48cece775a39f..6460b000c5938d 100644
--- a/fs/configfs/file.c
+++ b/fs/configfs/file.c
@@ -41,8 +41,8 @@ struct configfs_buffer {
 	struct config_item	*item;
 	struct module		*owner;
 	union {
-		struct configfs_attribute	*attr;
-		struct configfs_bin_attribute	*bin_attr;
+		const struct configfs_attribute		*attr;
+		const struct configfs_bin_attribute	*bin_attr;
 	};
 };
 
@@ -291,7 +291,7 @@ static int __configfs_open_file(struct inode *inode, struct file *file, int type
 {
 	struct dentry *dentry = file->f_path.dentry;
 	struct configfs_fragment *frag = to_frag(file);
-	struct configfs_attribute *attr;
+	const struct configfs_attribute *attr;
 	struct configfs_buffer *buffer;
 	int error;
 
diff --git a/fs/configfs/inode.c b/fs/configfs/inode.c
index 68290fe0e374fc..c92a05251a4794 100644
--- a/fs/configfs/inode.c
+++ b/fs/configfs/inode.c
@@ -32,7 +32,7 @@ static const struct inode_operations configfs_inode_operations ={
 	.setattr	= configfs_setattr,
 };
 
-int configfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int configfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		     struct iattr *iattr)
 {
 	struct inode * inode = d_inode(dentry);
@@ -178,7 +178,7 @@ struct inode *configfs_create(struct dentry *dentry, umode_t mode)
  */
 const unsigned char * configfs_get_name(struct configfs_dirent *sd)
 {
-	struct configfs_attribute *attr;
+	const struct configfs_attribute *attr;
 
 	BUG_ON(!sd || !sd->s_element);
 
diff --git a/fs/configfs/symlink.c b/fs/configfs/symlink.c
index 3b31c714400f99..89178f5d371a6e 100644
--- a/fs/configfs/symlink.c
+++ b/fs/configfs/symlink.c
@@ -146,7 +146,7 @@ static int get_target(const char *symname, struct config_item **target,
 }
 
 
-int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+int configfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct dentry *dentry, const char *symname)
 {
 	int ret;
diff --git a/fs/coredump.c b/fs/coredump.c
index 6114839f5178b0..af372725d4590a 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -32,13 +32,14 @@
 #include <linux/tsacct_kern.h>
 #include <linux/cn_proc.h>
 #include <linux/audit.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/fsnotify.h>
 #include <linux/fs_struct.h>
 #include <linux/pipe_fs_i.h>
 #include <linux/oom.h>
 #include <linux/compat.h>
 #include <linux/fs.h>
+#include <linux/wait_bit.h>
 #include <linux/path.h>
 #include <linux/timekeeping.h>
 #include <linux/sysctl.h>
@@ -51,7 +52,6 @@
 #include <net/sock.h>
 #include <uapi/linux/pidfd.h>
 #include <uapi/linux/un.h>
-#include <uapi/linux/coredump.h>
 
 #include <linux/uaccess.h>
 #include <asm/mmu_context.h>
@@ -68,6 +68,8 @@
 
 static bool dump_vma_snapshot(struct coredump_params *cprm);
 static void free_vma_snapshot(struct coredump_params *cprm);
+static void dump_end_record(struct coredump_params *cprm);
+static bool dump_flush_skip(struct coredump_params *cprm);
 
 #define CORE_FILE_NOTE_SIZE_DEFAULT (4*1024*1024)
 /* Define a reasonable max cap */
@@ -83,6 +85,8 @@ static int core_uses_pid;
 static unsigned int core_pipe_limit;
 static unsigned int core_sort_vma;
 static char core_pattern[CORENAME_MAX_SIZE] = "core";
+/* Taken around every copy in and out of core_pattern. */
+static DEFINE_SPINLOCK(core_pattern_lock);
 static int core_name_size = CORENAME_MAX_SIZE;
 unsigned int core_file_note_size_limit = CORE_FILE_NOTE_SIZE_DEFAULT;
 static atomic_t core_pipe_count = ATOMIC_INIT(0);
@@ -98,9 +102,7 @@ struct core_name {
 	char *corename __counted_by_ptr(size);
 	int used, size;
 	unsigned int core_pipe_limit;
-	bool core_dumped;
 	enum coredump_type_t core_type;
-	u64 mask;
 };
 
 static int expand_corename(struct core_name *cn, int size)
@@ -240,18 +242,22 @@ static bool coredump_parse(struct core_name *cn, struct coredump_params *cprm,
 			   size_t **argv, int *argc)
 {
 	const struct cred *cred = current_cred();
-	const char *pat_ptr = core_pattern;
+	char pattern[CORENAME_MAX_SIZE];
+	const char *pat_ptr = pattern;
 	bool was_space = false;
 	int pid_in_pattern = 0;
 	int err = 0;
 
-	cn->mask = COREDUMP_KERNEL;
+	/* The sysctl handler may be publishing a new pattern. */
+	scoped_guard(spinlock, &core_pattern_lock)
+		strscpy(pattern, core_pattern);
+
+	cprm->mask = COREDUMP_KERNEL;
 	if (core_pipe_limit)
-		cn->mask |= COREDUMP_WAIT;
+		cprm->mask |= COREDUMP_WAIT;
 	cn->used = 0;
 	cn->corename = NULL;
 	cn->core_pipe_limit = 0;
-	cn->core_dumped = false;
 	if (*pat_ptr == '|')
 		cn->core_type = COREDUMP_PIPE;
 	else if (*pat_ptr == '@')
@@ -508,60 +514,64 @@ static int zap_threads(struct task_struct *tsk,
 	int nr = -EAGAIN;
 
 	spin_lock_irq(&tsk->sighand->siglock);
-	if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task) {
+	/* A freeze requested before the dump would be lost with TIF_SIGPENDING. */
+	if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task &&
+	    !freezing(tsk) && !(tsk->jobctl & JOBCTL_TRAP_FREEZE)) {
 		/* Allow SIGKILL, see prepare_signal() */
 		signal->core_state = core_state;
 		nr = zap_process(signal, exit_code);
 		clear_tsk_thread_flag(tsk, TIF_SIGPENDING);
 		tsk->flags |= PF_DUMPCORE;
-		atomic_set(&core_state->nr_threads, nr);
+		atomic_set(&core_state->threads_remaining, nr);
 	}
 	spin_unlock_irq(&tsk->sighand->siglock);
 	return nr;
 }
 
+static void coredump_wait_inactive(struct core_state *core_state)
+{
+	struct core_thread *ptr;
+
+	wait_var_event_state(&core_state->threads_remaining,
+			     !atomic_read_acquire(&core_state->threads_remaining),
+			     TASK_UNINTERRUPTIBLE | TASK_FREEZABLE);
+	/*
+	 * Wait for all the threads to become inactive, so that
+	 * all the thread context (extended register state, like
+	 * fpu etc) gets copied to the memory.
+	 */
+	for (ptr = core_state->tasks; ptr; ptr = ptr->next)
+		wait_task_inactive(ptr->task, TASK_ANY);
+}
+
 static int coredump_wait(int exit_code, struct core_state *core_state)
 {
 	struct task_struct *tsk = current;
 	int core_waiters = -EBUSY;
 
-	init_completion(&core_state->startup);
-	core_state->dumper.task = tsk;
-	core_state->dumper.next = NULL;
+	core_state->tasks = NULL;
 
 	core_waiters = zap_threads(tsk, core_state, exit_code);
-	if (core_waiters > 0) {
-		struct core_thread *ptr;
-
-		wait_for_completion_state(&core_state->startup,
-					  TASK_UNINTERRUPTIBLE|TASK_FREEZABLE);
-		/*
-		 * Wait for all the threads to become inactive, so that
-		 * all the thread context (extended register state, like
-		 * fpu etc) gets copied to the memory.
-		 */
-		ptr = core_state->dumper.next;
-		while (ptr != NULL) {
-			wait_task_inactive(ptr->task, TASK_ANY);
-			ptr = ptr->next;
-		}
-	}
+	if (core_waiters > 0)
+		coredump_wait_inactive(core_state);
 
 	return core_waiters;
 }
 
-static void coredump_finish(bool core_dumped)
+static void coredump_finish(enum coredump_state state)
 {
 	struct core_thread *curr, *next;
 	struct task_struct *task;
 
 	spin_lock_irq(&current->sighand->siglock);
-	if (core_dumped && !__fatal_signal_pending(current))
+	if ((state & COREDUMP_STATE_STARTED) && !__fatal_signal_pending(current))
 		current->signal->group_exit_code |= 0x80;
-	next = current->signal->core_state->dumper.next;
+	next = current->signal->core_state->tasks;
 	current->signal->core_state = NULL;
 	spin_unlock_irq(&current->sighand->siglock);
 
+	/* A released thread may exit and be freed before it is woken. */
+	guard(rcu)();
 	while ((curr = next) != NULL) {
 		next = curr->next;
 		task = curr->task;
@@ -570,6 +580,7 @@ static void coredump_finish(bool core_dumped)
 		 * ->task == NULL before we read ->next.
 		 */
 		smp_mb();
+		/* Any wakeup now lets the thread exit, rcu keeps it alive. */
 		curr->task = NULL;
 		wake_up_process(task);
 	}
@@ -577,13 +588,8 @@ static void coredump_finish(bool core_dumped)
 
 static bool dump_interrupted(void)
 {
-	/*
-	 * SIGKILL or freezing() interrupt the coredumping. Perhaps we
-	 * can do try_to_freeze() and check __fatal_signal_pending(),
-	 * but then we need to teach dump_write() to restart and clear
-	 * TIF_SIGPENDING.
-	 */
-	return fatal_signal_pending(current) || freezing(current);
+	/* Only SIGKILL and the freezers set it after zap_threads(). */
+	return task_sigpending(current);
 }
 
 static void wait_for_dump_helpers(struct file *file)
@@ -664,7 +670,12 @@ static int umh_coredump_setup(struct subprocess_info *info, struct cred *new)
 	return 0;
 }
 
+static_assert(sizeof(struct coredump_record_header) == COREDUMP_RECORD_HEADER_SIZE_VER0);
+
 #ifdef CONFIG_UNIX
+/* af_unix halves the send buffer to size a single skb. */
+#define COREDUMP_SOCK_SNDBUF_MIN (3 * PAGE_SIZE)
+
 static bool coredump_sock_connect(struct core_name *cn, struct coredump_params *cprm)
 {
 	struct file *file __free(fput) = NULL;
@@ -690,6 +701,10 @@ static bool coredump_sock_connect(struct core_name *cn, struct coredump_params *
 	if (retval < 0)
 		return false;
 
+	/* Don't let a page-sized write split into several skbs. */
+	socket->sk->sk_sndbuf = max_t(int, socket->sk->sk_sndbuf,
+				      COREDUMP_SOCK_SNDBUF_MIN);
+
 	file = sock_alloc_file(socket, 0, NULL);
 	if (IS_ERR(file))
 		return false;
@@ -752,8 +767,39 @@ static inline bool coredump_sock_send(struct file *file, struct coredump_req *re
 	return ret == sizeof(*req);
 }
 
+static_assert(sizeof(struct coredump_req) == COREDUMP_REQ_SIZE_VER1);
+static_assert(sizeof(struct coredump_ack) == COREDUMP_ACK_SIZE_VER1);
 static_assert(sizeof(enum coredump_mark) == sizeof(__u32));
 
+/* Every memory type this kernel knows. */
+#define COREDUMP_MEMORY_ALL						\
+	(COREDUMP_MEMORY_ANON_PRIVATE | COREDUMP_MEMORY_ANON_SHARED |	\
+	 COREDUMP_MEMORY_FILE_PRIVATE | COREDUMP_MEMORY_FILE_SHARED |	\
+	 COREDUMP_MEMORY_ELF_HEADERS |					\
+	 COREDUMP_MEMORY_HUGETLB_PRIVATE | COREDUMP_MEMORY_HUGETLB_SHARED | \
+	 COREDUMP_MEMORY_DAX_PRIVATE | COREDUMP_MEMORY_DAX_SHARED)
+
+#define COREDUMP_MEMORY_TYPE_BIT(mmf) BIT((mmf) - MMF_DUMP_FILTER_SHIFT)
+static_assert(COREDUMP_MEMORY_ALL == (MMF_DUMP_FILTER_MASK >> MMF_DUMP_FILTER_SHIFT));
+static_assert(COREDUMP_MEMORY_ANON_PRIVATE ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_ANON_PRIVATE));
+static_assert(COREDUMP_MEMORY_ANON_SHARED ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_ANON_SHARED));
+static_assert(COREDUMP_MEMORY_FILE_PRIVATE ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_MAPPED_PRIVATE));
+static_assert(COREDUMP_MEMORY_FILE_SHARED ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_MAPPED_SHARED));
+static_assert(COREDUMP_MEMORY_ELF_HEADERS ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_ELF_HEADERS));
+static_assert(COREDUMP_MEMORY_HUGETLB_PRIVATE ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_HUGETLB_PRIVATE));
+static_assert(COREDUMP_MEMORY_HUGETLB_SHARED ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_HUGETLB_SHARED));
+static_assert(COREDUMP_MEMORY_DAX_PRIVATE ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_DAX_PRIVATE));
+static_assert(COREDUMP_MEMORY_DAX_SHARED ==
+	      COREDUMP_MEMORY_TYPE_BIT(MMF_DUMP_DAX_SHARED));
+
 static inline bool coredump_sock_mark(struct file *file, enum coredump_mark mark)
 {
 	struct msghdr msg = { .msg_flags = MSG_NOSIGNAL };
@@ -795,10 +841,14 @@ static inline void coredump_sock_shutdown(struct file *file)
 static bool coredump_sock_request(struct core_name *cn, struct coredump_params *cprm)
 {
 	struct coredump_req req = {
-		.size		= sizeof(struct coredump_req),
-		.mask		= COREDUMP_KERNEL | COREDUMP_USERSPACE |
-				  COREDUMP_REJECT | COREDUMP_WAIT,
-		.size_ack	= sizeof(struct coredump_ack),
+		.size			= sizeof(struct coredump_req),
+		.mask			= COREDUMP_KERNEL | COREDUMP_USERSPACE |
+					  COREDUMP_REJECT | COREDUMP_WAIT |
+					  COREDUMP_RECORDS | COREDUMP_SPARSE |
+					  COREDUMP_MEMORY_TYPES,
+		.size_ack		= sizeof(struct coredump_ack),
+		.memory_types		= cprm->memory_types,
+		.memory_types_mask	= COREDUMP_MEMORY_ALL,
 	};
 	struct coredump_ack ack = {};
 	ssize_t usize;
@@ -851,7 +901,54 @@ static bool coredump_sock_request(struct core_name *cn, struct coredump_params *
 		return false;
 	}
 
-	cn->mask = ack.mask;
+	/* Records only describe a coredump the kernel writes. */
+	if ((ack.mask & COREDUMP_RECORDS) && !(ack.mask & COREDUMP_KERNEL)) {
+		coredump_sock_mark(cprm->file, COREDUMP_MARK_CONFLICTING);
+		return false;
+	}
+
+	/* Zero records only exist inside a record stream. */
+	if ((ack.mask & COREDUMP_SPARSE) && !(ack.mask & COREDUMP_RECORDS)) {
+		coredump_sock_mark(cprm->file, COREDUMP_MARK_CONFLICTING);
+		return false;
+	}
+
+	if (ack.mask & COREDUMP_MEMORY_TYPES) {
+		/* The memory types need the whole field. */
+		if (usize < COREDUMP_ACK_SIZE_VER1) {
+			coredump_sock_mark(cprm->file, COREDUMP_MARK_MINSIZE);
+			return false;
+		}
+
+		/* The memory types only select what the kernel writes. */
+		if (!(ack.mask & COREDUMP_KERNEL)) {
+			coredump_sock_mark(cprm->file, COREDUMP_MARK_CONFLICTING);
+			return false;
+		}
+
+		/* Refuse unknown memory types. */
+		if (ack.memory_types & ~req.memory_types_mask) {
+			coredump_sock_mark(cprm->file, COREDUMP_MARK_UNSUPPORTED);
+			return false;
+		}
+	} else if (ack.memory_types) {
+		/* Like @spare the field must be zero when it isn't used. */
+		coredump_sock_mark(cprm->file, COREDUMP_MARK_UNSUPPORTED);
+		return false;
+	}
+
+	/* Record header scratch; a bvec can't point at the stack. */
+	if (ack.mask & COREDUMP_RECORDS) {
+		cprm->record_hdr = kmalloc_obj(*cprm->record_hdr);
+		if (!cprm->record_hdr)
+			return false;
+	}
+
+	/* The server's selection replaces the task's entirely. */
+	if (ack.mask & COREDUMP_MEMORY_TYPES)
+		cprm->memory_types = ack.memory_types;
+
+	cprm->mask = ack.mask;
 	return coredump_sock_mark(cprm->file, COREDUMP_MARK_REQACK);
 }
 
@@ -878,7 +975,7 @@ static inline bool coredump_force_suid_safe(const struct coredump_params *cprm)
 static bool coredump_file(struct core_name *cn, struct coredump_params *cprm,
 			  const struct linux_binfmt *binfmt)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct inode *inode;
 	struct file *file __free(fput) = NULL;
 	int open_flags = O_CREAT | O_WRONLY | O_NOFOLLOW | O_LARGEFILE | O_EXCL;
@@ -1032,29 +1129,41 @@ static bool coredump_pipe(struct core_name *cn, struct coredump_params *cprm,
 	return true;
 }
 
-static bool coredump_write(struct core_name *cn,
-			  struct coredump_params *cprm,
-			  const struct linux_binfmt *binfmt)
+static bool coredump_write(struct coredump_params *cprm,
+			   const struct linux_binfmt *binfmt)
 {
-
-	if (dump_interrupted())
+	if (dump_interrupted()) {
+		cprm->state |= COREDUMP_STATE_TRUNCATED;
 		return true;
+	}
 
-	if (!dump_vma_snapshot(cprm))
+	if (!dump_vma_snapshot(cprm)) {
+		cprm->state |= COREDUMP_STATE_TRUNCATED;
 		return false;
+	}
 
 	file_start_write(cprm->file);
-	cn->core_dumped = binfmt->core_dump(cprm);
+	if (!binfmt->core_dump(cprm))
+		cprm->state |= COREDUMP_STATE_TRUNCATED;
 	/*
-	 * Ensures that file size is big enough to contain the current
-	 * file postion. This prevents gdb from complaining about
-	 * a truncated file if the last "write" to the file was
-	 * dump_skip.
+	 * A trailing hole still has to land in the coredump. Seeking over
+	 * it doesn't grow the file, so the last byte of it is written
+	 * instead and gdb doesn't see a truncated file. Everything else
+	 * puts the hole on the wire as it flushes it.
 	 */
 	if (cprm->to_skip) {
-		cprm->to_skip--;
-		dump_emit(cprm, "", 1);
+		bool flushed;
+
+		if (cprm->file->f_mode & FMODE_LSEEK) {
+			cprm->to_skip--;
+			flushed = dump_emit(cprm, "", 1);
+		} else {
+			flushed = dump_flush_skip(cprm);
+		}
+		if (!flushed)
+			cprm->state |= COREDUMP_STATE_TRUNCATED;
 	}
+	dump_end_record(cprm);
 	file_end_write(cprm->file);
 	free_vma_snapshot(cprm);
 	return true;
@@ -1069,7 +1178,8 @@ static void coredump_cleanup(struct core_name *cn, struct coredump_params *cprm)
 		atomic_dec(&core_pipe_count);
 	}
 	kfree(cn->corename);
-	coredump_finish(cn->core_dumped);
+	kfree(cprm->record_hdr);
+	coredump_finish(cprm->state);
 }
 
 static inline bool coredump_skip(const struct coredump_params *cprm,
@@ -1115,29 +1225,24 @@ static void do_coredump(struct core_name *cn, struct coredump_params *cprm,
 	}
 
 	/* Don't even generate the coredump. */
-	if (cn->mask & COREDUMP_REJECT)
-		return;
-
-	/* get us an unshared descriptor table; almost always a no-op */
-	/* The cell spufs coredump code reads the file descriptor tables */
-	if (unshare_files())
+	if (cprm->mask & COREDUMP_REJECT)
 		return;
 
-	if ((cn->mask & COREDUMP_KERNEL) && !coredump_write(cn, cprm, binfmt))
+	if ((cprm->mask & COREDUMP_KERNEL) && !coredump_write(cprm, binfmt))
 		return;
 
 	coredump_sock_shutdown(cprm->file);
 
 	/* Let the parent know that a coredump was generated. */
-	if (cn->mask & COREDUMP_USERSPACE)
-		cn->core_dumped = true;
+	if (cprm->mask & COREDUMP_USERSPACE)
+		cprm->state |= COREDUMP_STATE_STARTED;
 
 	/*
 	 * When core_pipe_limit is set we wait for the coredump server
 	 * or usermodehelper to finish before exiting so it can e.g.,
 	 * inspect /proc/<pid>.
 	 */
-	if (cn->mask & COREDUMP_WAIT) {
+	if (cprm->mask & COREDUMP_WAIT) {
 		switch (cn->core_type) {
 		case COREDUMP_PIPE:
 			wait_for_dump_helpers(cprm->file);
@@ -1153,6 +1258,10 @@ static void do_coredump(struct core_name *cn, struct coredump_params *cprm,
 	}
 }
 
+#define COREDUMP_TASK_MEMORY_TYPES(mm)                         \
+	((__mm_flags_get_word((mm)) & MMF_DUMP_FILTER_MASK) >> \
+	 MMF_DUMP_FILTER_SHIFT)
+
 void vfs_coredump(const kernel_siginfo_t *siginfo)
 {
 	size_t *argv __free(kfree) = NULL;
@@ -1164,8 +1273,8 @@ void vfs_coredump(const kernel_siginfo_t *siginfo)
 	struct coredump_params cprm = {
 		.siginfo = siginfo,
 		.limit = rlimit(RLIMIT_CORE),
-		/* Snapshot MMF_DUMP_FILTER_* (unlocked) and dumpable for the dump. */
-		.mm_flags = __mm_flags_get_word(mm),
+		/* Snapshot the memory types (unlocked) and dumpable for the dump. */
+		.memory_types = COREDUMP_TASK_MEMORY_TYPES(mm),
 		.dumpable = task_exec_state_get_dumpable(current),
 		.vma_meta = NULL,
 		.cpu = raw_smp_processor_id(),
@@ -1191,6 +1300,8 @@ void vfs_coredump(const kernel_siginfo_t *siginfo)
 	if (coredump_wait(siginfo->si_signo, &core_state) < 0)
 		return;
 
+	/* Task work must not cut the dump short, see signal_pending(). */
+	guard(no_notify_signal)();
 	scoped_with_creds(cred)
 		do_coredump(&cn, &cprm, &argv, &argc, binfmt);
 	coredump_cleanup(&cn, &cprm);
@@ -1202,60 +1313,181 @@ void vfs_coredump(const kernel_siginfo_t *siginfo)
  * do on a core-file: use only these functions to write out all the
  * necessary info.
  */
-static int __dump_emit(struct coredump_params *cprm, const void *addr, int nr)
+static bool dump_records(const struct coredump_params *cprm)
+{
+	return cprm->mask & COREDUMP_RECORDS;
+}
+
+static bool dump_sparse(const struct coredump_params *cprm)
+{
+	return cprm->mask & COREDUMP_SPARSE;
+}
+
+/* Describe the next @len bytes of the coredump. Returns the header size. */
+static size_t dump_record_init(struct coredump_params *cprm,
+			       enum coredump_record_type type, u64 flags,
+			       u64 len)
+{
+	if (!dump_records(cprm))
+		return 0;
+
+	*cprm->record_hdr = (struct coredump_record_header) {
+		.size	= sizeof(*cprm->record_hdr),
+		.type	= type,
+		.flags	= flags,
+		.offset	= cprm->pos,
+		.len	= len,
+	};
+
+	return sizeof(*cprm->record_hdr);
+}
+
+/* Write @iter whole or fail. @len is what it advances the coredump by. */
+static bool dump_write_iter(struct coredump_params *cprm, struct iov_iter *iter,
+			    size_t len)
 {
 	struct file *file = cprm->file;
+	size_t count = iov_iter_count(iter);
 	loff_t pos = file->f_pos;
 	ssize_t n;
 
-	if (cprm->written + nr > cprm->limit)
-		return 0;
-	if (dump_interrupted())
-		return 0;
-	n = __kernel_write(file, addr, nr, &pos);
-	if (n != nr)
-		return 0;
+	n = __kernel_write_iter(file, iter, &pos);
+	if (n != (ssize_t)count)
+		return false;
 	file->f_pos = pos;
-	cprm->written += n;
-	cprm->pos += n;
+	cprm->written += count;
+	cprm->pos += len;
+
+	return true;
+}
+
+/* One record, never more than a page. See __dump_emit(). */
+static bool dump_emit_chunk(struct coredump_params *cprm, const void *addr,
+			    int nr)
+{
+	struct kvec kvec[2];
+	struct iov_iter iter;
+	unsigned int nseg = 0;
+	size_t hdrlen;
 
-	return 1;
+	if (dump_interrupted())
+		return false;
+
+	hdrlen = dump_record_init(cprm, COREDUMP_RECORD_DATA, 0, nr);
+	if (hdrlen) {
+		kvec[nseg].iov_base = cprm->record_hdr;
+		kvec[nseg].iov_len = hdrlen;
+		nseg++;
+	}
+	kvec[nseg].iov_base = (void *)addr;
+	kvec[nseg].iov_len = nr;
+	nseg++;
+
+	iov_iter_kvec(&iter, ITER_SOURCE, kvec, nseg, hdrlen + nr);
+
+	return dump_write_iter(cprm, &iter, nr);
+}
+
+static bool __dump_emit(struct coredump_params *cprm, const void *addr, int nr)
+{
+	if (cprm->written + nr > cprm->limit)
+		return false;
+
+	while (nr) {
+		int chunk = min_t(int, nr, PAGE_SIZE);
+
+		if (!dump_emit_chunk(cprm, addr, chunk))
+			return false;
+
+		addr += chunk;
+		nr -= chunk;
+	}
+
+	return true;
 }
 
-static int __dump_skip(struct coredump_params *cprm, size_t nr)
+/* Send a record that stands on its own: a header and nothing else. */
+static bool dump_emit_record(struct coredump_params *cprm,
+			     enum coredump_record_type type, u64 flags, u64 len)
+{
+	struct kvec kvec;
+	struct iov_iter iter;
+	size_t hdrlen;
+
+	hdrlen = dump_record_init(cprm, type, flags, len);
+	if (!hdrlen)
+		return false;
+
+	kvec.iov_base = cprm->record_hdr;
+	kvec.iov_len = hdrlen;
+	iov_iter_kvec(&iter, ITER_SOURCE, &kvec, 1, hdrlen);
+
+	return dump_write_iter(cprm, &iter, len);
+}
+
+/* Close the record stream. Only a whole coredump gets an end record. */
+static void dump_end_record(struct coredump_params *cprm)
+{
+	if (cprm->state & COREDUMP_STATE_TRUNCATED)
+		return;
+
+	dump_emit_record(cprm, COREDUMP_RECORD_END, 0, 0);
+}
+
+static bool __dump_skip(struct coredump_params *cprm, size_t nr)
 {
 	static char zeroes[PAGE_SIZE];
 	struct file *file = cprm->file;
 
+	if (dump_sparse(cprm)) {
+		/* Hand the server the length of the hole instead of the hole itself. */
+		if (dump_interrupted())
+			return false;
+		return dump_emit_record(cprm, COREDUMP_RECORD_ZERO, 0, nr);
+	}
+
 	if (file->f_mode & FMODE_LSEEK) {
 		if (dump_interrupted() || vfs_llseek(file, nr, SEEK_CUR) < 0)
-			return 0;
+			return false;
 		cprm->pos += nr;
-		return 1;
+		return true;
 	}
 
-	while (nr > PAGE_SIZE) {
-		if (!__dump_emit(cprm, zeroes, PAGE_SIZE))
-			return 0;
-		nr -= PAGE_SIZE;
+	while (nr) {
+		size_t chunk = min_t(size_t, nr, PAGE_SIZE);
+
+		if (!__dump_emit(cprm, zeroes, chunk))
+			return false;
+
+		nr -= chunk;
 	}
 
-	return __dump_emit(cprm, zeroes, nr);
+	return true;
 }
 
-int dump_emit(struct coredump_params *cprm, const void *addr, int nr)
+/* Flush the accumulated hole before writing data. */
+static bool dump_flush_skip(struct coredump_params *cprm)
 {
 	if (cprm->to_skip) {
 		if (!__dump_skip(cprm, cprm->to_skip))
-			return 0;
+			return false;
 		cprm->to_skip = 0;
 	}
+	return true;
+}
+
+bool dump_emit(struct coredump_params *cprm, const void *addr, int nr)
+{
+	if (!dump_flush_skip(cprm))
+		return false;
 	return __dump_emit(cprm, addr, nr);
 }
 EXPORT_SYMBOL(dump_emit);
 
 void dump_skip_to(struct coredump_params *cprm, unsigned long pos)
 {
+	if (WARN_ON_ONCE(pos < cprm->pos))
+		return;
 	cprm->to_skip = pos - cprm->pos;
 }
 EXPORT_SYMBOL(dump_skip_to);
@@ -1267,37 +1499,32 @@ void dump_skip(struct coredump_params *cprm, size_t nr)
 EXPORT_SYMBOL(dump_skip);
 
 #ifdef CONFIG_ELF_CORE
-static int dump_emit_page(struct coredump_params *cprm, struct page *page)
+static bool dump_emit_page(struct coredump_params *cprm, struct page *page)
 {
-	struct bio_vec bvec;
+	struct bio_vec bvec[2];
 	struct iov_iter iter;
-	struct file *file = cprm->file;
-	loff_t pos;
-	ssize_t n;
+	unsigned int nseg = 0;
+	size_t hdrlen;
 
 	if (!page)
-		return 0;
+		return false;
 
-	if (cprm->to_skip) {
-		if (!__dump_skip(cprm, cprm->to_skip))
-			return 0;
-		cprm->to_skip = 0;
-	}
+	if (!dump_flush_skip(cprm))
+		return false;
 	if (cprm->written + PAGE_SIZE > cprm->limit)
-		return 0;
+		return false;
 	if (dump_interrupted())
-		return 0;
-	pos = file->f_pos;
-	bvec_set_page(&bvec, page, PAGE_SIZE, 0);
-	iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE);
-	n = __kernel_write_iter(cprm->file, &iter, &pos);
-	if (n != PAGE_SIZE)
-		return 0;
-	file->f_pos = pos;
-	cprm->written += PAGE_SIZE;
-	cprm->pos += PAGE_SIZE;
+		return false;
 
-	return 1;
+	/* Hand the record header to the same write as the page it describes. */
+	hdrlen = dump_record_init(cprm, COREDUMP_RECORD_DATA, 0, PAGE_SIZE);
+	if (hdrlen)
+		bvec_set_virt(&bvec[nseg++], cprm->record_hdr, hdrlen);
+	bvec_set_page(&bvec[nseg++], page, PAGE_SIZE, 0);
+
+	iov_iter_bvec(&iter, ITER_SOURCE, bvec, nseg, hdrlen + PAGE_SIZE);
+
+	return dump_write_iter(cprm, &iter, PAGE_SIZE);
 }
 
 /*
@@ -1329,18 +1556,19 @@ static inline struct page *dump_page_copy(struct page *src, struct page *dst)
 }
 #endif
 
-int dump_user_range(struct coredump_params *cprm, unsigned long start,
-		    unsigned long len)
+bool dump_user_range(struct coredump_params *cprm, unsigned long start,
+		     unsigned long len)
 {
 	unsigned long addr;
 	struct page *dump_page;
-	int locked, ret;
+	int locked;
+	bool ret;
 
 	dump_page = dump_page_alloc();
 	if (!dump_page)
-		return 0;
+		return false;
 
-	ret = 0;
+	ret = false;
 	locked = 0;
 	for (addr = start; addr < start + len; addr += PAGE_SIZE) {
 		struct page *page;
@@ -1364,7 +1592,7 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start,
 				mmap_read_unlock(current->mm);
 				locked = 0;
 			}
-			int stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page));
+			bool stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page));
 			put_page(page);
 			if (stop)
 				goto out;
@@ -1383,7 +1611,7 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start,
 		}
 		cond_resched();
 	}
-	ret = 1;
+	ret = true;
 out:
 	if (locked)
 		mmap_read_unlock(current->mm);
@@ -1393,14 +1621,14 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start,
 }
 #endif
 
-int dump_align(struct coredump_params *cprm, int align)
+bool dump_align(struct coredump_params *cprm, int align)
 {
 	unsigned mod = (cprm->pos + cprm->to_skip) & (align - 1);
 	if (align & (align - 1))
-		return 0;
+		return false;
 	if (mod)
 		cprm->to_skip += align - mod;
-	return 1;
+	return true;
 }
 EXPORT_SYMBOL(dump_align);
 
@@ -1417,11 +1645,11 @@ void validate_coredump_safety(void)
 	}
 }
 
-static inline bool check_coredump_socket(void)
+static inline bool check_coredump_socket(const char *pattern)
 {
 	const char *p;
 
-	if (core_pattern[0] != '@')
+	if (pattern[0] != '@')
 		return true;
 
 	/*
@@ -1433,16 +1661,16 @@ static inline bool check_coredump_socket(void)
 		return false;
 
 	/* Must be an absolute path... */
-	if (core_pattern[1] != '/') {
+	if (pattern[1] != '/') {
 		/* ... or the socket request protocol... */
-		if (core_pattern[1] != '@')
+		if (pattern[1] != '@')
 			return false;
 		/* ... and if so must be an absolute path. */
-		if (core_pattern[2] != '/')
+		if (pattern[2] != '/')
 			return false;
-		p = &core_pattern[2];
+		p = &pattern[2];
 	} else {
-		p = &core_pattern[1];
+		p = &pattern[1];
 	}
 
 	/* The path obviously cannot exceed UNIX_PATH_MAX. */
@@ -1450,7 +1678,7 @@ static inline bool check_coredump_socket(void)
 		return false;
 
 	/* Must not contain ".." in the path. */
-	if (name_contains_dotdot(core_pattern))
+	if (name_contains_dotdot(pattern))
 		return false;
 
 	return true;
@@ -1459,27 +1687,35 @@ static inline bool check_coredump_socket(void)
 static int proc_dostring_coredump(const struct ctl_table *table, int write,
 		  void *buffer, size_t *lenp, loff_t *ppos)
 {
+	char pattern[CORENAME_MAX_SIZE];
+	const struct ctl_table tmp = {
+		.procname	= table->procname,
+		.data		= pattern,
+		.maxlen		= sizeof(pattern),
+	};
+	bool changed = false;
 	int error;
-	ssize_t retval;
-	char old_core_pattern[CORENAME_MAX_SIZE];
 
-	if (!write)
-		return proc_dostring(table, write, buffer, lenp, ppos);
+	/* Work on a copy, proc_dostring() appends at *ppos. */
+	scoped_guard(spinlock, &core_pattern_lock)
+		strscpy(pattern, core_pattern);
 
-	retval = strscpy(old_core_pattern, core_pattern, CORENAME_MAX_SIZE);
-
-	error = proc_dostring(table, write, buffer, lenp, ppos);
-	if (error)
+	error = proc_dostring(&tmp, write, buffer, lenp, ppos);
+	if (error || !write)
 		return error;
 
-	if (!check_coredump_socket()) {
-		strscpy(core_pattern, old_core_pattern, retval + 1);
+	if (!check_coredump_socket(pattern))
 		return -EINVAL;
-	}
 
-	if (strncmp(old_core_pattern, core_pattern, CORENAME_MAX_SIZE))
+	/* Publish the validated pattern whole. */
+	scoped_guard(spinlock, &core_pattern_lock) {
+		changed = strncmp(pattern, core_pattern, CORENAME_MAX_SIZE);
+		if (changed)
+			strscpy(core_pattern, pattern);
+	}
+	if (changed)
 		validate_coredump_safety();
-	return error;
+	return 0;
 }
 
 static const unsigned int core_file_note_size_min = CORE_FILE_NOTE_SIZE_DEFAULT;
@@ -1582,15 +1818,15 @@ static bool always_dump_vma(struct vm_area_struct *vma)
 }
 
 #define DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER 1
+#define COREDUMP_MEMORY_TYPE_INCLUDE(types, type) \
+	((types) & COREDUMP_MEMORY_##type)
 
 /*
  * Decide how much of @vma's contents should be included in a core dump.
  */
 static unsigned long vma_dump_size(struct vm_area_struct *vma,
-				   unsigned long mm_flags)
+				   u64 memory_types)
 {
-#define FILTER(type)	(mm_flags & (1UL << MMF_DUMP_##type))
-
 	/* always dump the vdso and vsyscall sections */
 	if (always_dump_vma(vma))
 		goto whole;
@@ -1600,48 +1836,54 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma,
 
 	/* support for DAX */
 	if (vma_is_dax(vma)) {
-		if ((vma->vm_flags & VM_SHARED) && FILTER(DAX_SHARED))
+		if ((vma->vm_flags & VM_SHARED) &&
+		    COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, DAX_SHARED))
 			goto whole;
-		if (!(vma->vm_flags & VM_SHARED) && FILTER(DAX_PRIVATE))
+		if (!(vma->vm_flags & VM_SHARED) &&
+		    COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, DAX_PRIVATE))
 			goto whole;
 		return 0;
 	}
 
 	/* Hugetlb memory check */
-	if (is_vm_hugetlb_page(vma)) {
-		if ((vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_SHARED))
+	if (vma_is_hugetlb(vma)) {
+		if ((vma->vm_flags & VM_SHARED) &&
+		    COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, HUGETLB_SHARED))
 			goto whole;
-		if (!(vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_PRIVATE))
+		if (!(vma->vm_flags & VM_SHARED) &&
+		    COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, HUGETLB_PRIVATE))
 			goto whole;
 		return 0;
 	}
 
-	/* Do not dump I/O mapped devices or special mappings */
-	if (vma->vm_flags & VM_IO)
+	/* Do not dump memory-mapped I/O, which may have side effects on read. */
+	if (vma_test(vma, VMA_IO_BIT))
 		return 0;
 
 	/* By default, dump shared memory if mapped from an anonymous file. */
 	if (vma->vm_flags & VM_SHARED) {
 		if (file_inode(vma->vm_file)->i_nlink == 0 ?
-		    FILTER(ANON_SHARED) : FILTER(MAPPED_SHARED))
+			    COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, ANON_SHARED) :
+			    COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, FILE_SHARED))
 			goto whole;
 		return 0;
 	}
 
 	/* Dump segments that have been written to.  */
-	if ((!IS_ENABLED(CONFIG_MMU) || vma->anon_vma) && FILTER(ANON_PRIVATE))
+	if ((!IS_ENABLED(CONFIG_MMU) || vma->anon_vma) &&
+	    COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, ANON_PRIVATE))
 		goto whole;
 	if (vma->vm_file == NULL)
 		return 0;
 
-	if (FILTER(MAPPED_PRIVATE))
+	if (COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, FILE_PRIVATE))
 		goto whole;
 
 	/*
 	 * If this is the beginning of an executable file mapping,
 	 * dump the first page to aid in determining what was mapped here.
 	 */
-	if (FILTER(ELF_HEADERS) &&
+	if (COREDUMP_MEMORY_TYPE_INCLUDE(memory_types, ELF_HEADERS) &&
 	    vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) {
 		if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0)
 			return PAGE_SIZE;
@@ -1657,8 +1899,6 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma,
 		return DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER;
 	}
 
-#undef	FILTER
-
 	return 0;
 
 whole:
@@ -1743,7 +1983,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm)
 		m->start = vma->vm_start;
 		m->end = vma->vm_end;
 		m->flags = vma->vm_flags;
-		m->dump_size = vma_dump_size(vma, cprm->mm_flags);
+		m->dump_size = vma_dump_size(vma, cprm->memory_types);
 		m->pgoff = vma->vm_pgoff;
 		m->file = vma->vm_file;
 		if (m->file)
diff --git a/fs/crypto/Kconfig b/fs/crypto/Kconfig
index cd934e31dec40b..4cfbf4b51be7a6 100644
--- a/fs/crypto/Kconfig
+++ b/fs/crypto/Kconfig
@@ -5,7 +5,7 @@ config FS_ENCRYPTION
 	select BLK_INLINE_ENCRYPTION_FALLBACK if BLOCK
 	select CRYPTO
 	select CRYPTO_SKCIPHER
-	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_AES_ECB # for deprecated v1 key derivation function
 	select CRYPTO_LIB_SHA256
 	select CRYPTO_LIB_SHA512
 	select KEYS
diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c
index 5286a124b0d982..b6a253217a5ac2 100644
--- a/fs/crypto/crypto.c
+++ b/fs/crypto/crypto.c
@@ -65,7 +65,6 @@ void fscrypt_free_bounce_page(struct page *bounce_page)
 	if (!bounce_page)
 		return;
 	set_page_private(bounce_page, (unsigned long)NULL);
-	ClearPagePrivate(bounce_page);
 	mempool_free(bounce_page, fscrypt_bounce_page_pool);
 }
 EXPORT_SYMBOL(fscrypt_free_bounce_page);
@@ -98,6 +97,11 @@ void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index,
 	iv->index = cpu_to_le64(index);
 }
 
+typedef enum {
+	FS_DECRYPT = 0,
+	FS_ENCRYPT,
+} fscrypt_direction_t;
+
 /* Encrypt or decrypt a single "data unit" of file contents. */
 static int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci,
 				   fscrypt_direction_t rw, u64 index,
@@ -210,7 +214,6 @@ struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio,
 			return ERR_PTR(err);
 		}
 	}
-	SetPagePrivate(ciphertext_page);
 	set_page_private(ciphertext_page, (unsigned long)folio);
 	return ciphertext_page;
 }
diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
index 74329e0953d147..e602c31b3844e7 100644
--- a/fs/crypto/fscrypt_private.h
+++ b/fs/crypto/fscrypt_private.h
@@ -219,15 +219,6 @@ fscrypt_policy_du_bits(const union fscrypt_policy *policy,
 	BUG();
 }
 
-/*
- * For encrypted symlinks, the ciphertext length is stored at the beginning
- * of the string in little-endian format.
- */
-struct fscrypt_symlink_data {
-	__le16 len;
-	char encrypted_path[];
-} __packed;
-
 /**
  * struct fscrypt_prepared_key - a key prepared for actual encryption/decryption
  * @tfm: crypto API transform object
@@ -321,11 +312,6 @@ struct fscrypt_inode_info {
 	u8 ci_nonce[FSCRYPT_FILE_NONCE_SIZE];
 };
 
-typedef enum {
-	FS_DECRYPT = 0,
-	FS_ENCRYPT,
-} fscrypt_direction_t;
-
 /* crypto.c */
 extern struct kmem_cache *fscrypt_inode_info_cachep;
 int fscrypt_initialize(struct super_block *sb);
@@ -473,19 +459,6 @@ fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key,
 
 /* keyring.c */
 
-/*
- * fscrypt_master_key_user - a user's claim to a master key
- */
-struct fscrypt_master_key_user {
-	struct list_head link;
-	kuid_t uid;
-	/*
-	 * This 'struct key' contains no secret.  It exists solely to charge the
-	 * appropriate user's key quota.
-	 */
-	struct key *quota_key;
-};
-
 /*
  * fscrypt_master_key_secret - secret key material of an in-use master key
  */
diff --git a/fs/crypto/hooks.c b/fs/crypto/hooks.c
index a7a8a3f581a046..149498bdab94e7 100644
--- a/fs/crypto/hooks.c
+++ b/fs/crypto/hooks.c
@@ -9,6 +9,15 @@
 
 #include "fscrypt_private.h"
 
+/*
+ * For encrypted symlinks, the ciphertext length is stored at the beginning
+ * of the string in little-endian format.
+ */
+struct fscrypt_symlink_data {
+	__le16 len;
+	char encrypted_path[];
+} __packed;
+
 /**
  * fscrypt_file_open() - prepare to open a possibly-encrypted regular file
  * @inode: the inode being opened
diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c
index 76e28d1e00640d..874cc183ee6244 100644
--- a/fs/crypto/keyring.c
+++ b/fs/crypto/keyring.c
@@ -145,6 +145,19 @@ static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec)
 	return master_key_spec_len(spec) != 0;
 }
 
+/*
+ * fscrypt_master_key_user - a user's claim to a master key
+ */
+struct fscrypt_master_key_user {
+	struct list_head link;
+	kuid_t uid;
+	/*
+	 * This 'struct key' contains no secret.  It exists solely to charge the
+	 * appropriate user's key quota.
+	 */
+	struct key *quota_key;
+};
+
 static int fscrypt_user_key_instantiate(struct key *key,
 					struct key_preparsed_payload *prep)
 {
diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c
index 87fe13ccb25359..d9ac7c4328f822 100644
--- a/fs/crypto/keysetup_v1.c
+++ b/fs/crypto/keysetup_v1.c
@@ -20,7 +20,7 @@
  *    managed alongside the master keys in the filesystem-level keyring)
  */
 
-#include <crypto/aes.h>
+#include <crypto/aes-ecb.h>
 #include <crypto/utils.h>
 #include <keys/user-type.h>
 #include <linux/hashtable.h>
@@ -246,8 +246,7 @@ static int setup_v1_file_key_derived(struct fscrypt_inode_info *ci,
 
 	static_assert(FSCRYPT_FILE_NONCE_SIZE == AES_KEYSIZE_128);
 	aes_prepareenckey(&aes, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE);
-	for (unsigned int i = 0; i < derived_keysize; i += AES_BLOCK_SIZE)
-		aes_encrypt(&aes, &derived_key[i], &raw_master_key[i]);
+	aes_ecb_encrypt(derived_key, raw_master_key, derived_keysize, &aes);
 
 	err = fscrypt_set_per_file_enc_key(ci, derived_key);
 
diff --git a/fs/dax.c b/fs/dax.c
index 1fbba0d21c13d8..08adeaed2baf15 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -469,8 +469,6 @@ static void dax_folio_init(void *entry)
 
 	if (order > 0) {
 		prep_compound_page(&folio->page, order);
-		if (order > 1)
-			INIT_LIST_HEAD(&folio->_deferred_list);
 		WARN_ON_ONCE(folio_ref_count(folio));
 	}
 }
@@ -775,24 +773,23 @@ static void *grab_mapping_entry(struct xa_state *xas,
 
 /**
  * dax_layout_busy_page_range - find first pinned page in @mapping
- * @mapping: address space to scan for a page with ref count > 1
+ * @mapping: address space to scan for a pinned page
  * @start: Starting offset. Page containing 'start' is included.
  * @end: End offset. Page containing 'end' is included. If 'end' is LLONG_MAX,
  *       pages from 'start' till the end of file are included.
  *
- * DAX requires ZONE_DEVICE mapped pages. These pages are never
- * 'onlined' to the page allocator so they are considered idle when
- * page->count == 1. A filesystem uses this interface to determine if
- * any page in the mapping is busy, i.e. for DMA, or other
- * get_user_pages() usages.
+ * DAX requires ZONE_DEVICE mapped pages. A page is considered busy when
+ * folio_ref_count(folio) exceeds folio_mapcount(folio). This helper is
+ * used to determine if any page in the mapping is busy, i.e. for DMA,
+ * or other get_user_pages() usages.
  *
  * It is expected that the filesystem is holding locks to block the
  * establishment of new mappings in this address_space. I.e. it expects
- * to be able to run unmap_mapping_range() and subsequently not race
+ * to be able to run unmap_mapping_pages() and subsequently not race
  * mapping_mapped() becoming true.
  */
-struct page *dax_layout_busy_page_range(struct address_space *mapping,
-					loff_t start, loff_t end)
+static struct page *dax_layout_busy_page_range(struct address_space *mapping,
+					       loff_t start, loff_t end)
 {
 	void *entry;
 	unsigned int scanned = 0;
@@ -844,13 +841,6 @@ struct page *dax_layout_busy_page_range(struct address_space *mapping,
 	xas_unlock_irq(&xas);
 	return page;
 }
-EXPORT_SYMBOL_GPL(dax_layout_busy_page_range);
-
-struct page *dax_layout_busy_page(struct address_space *mapping)
-{
-	return dax_layout_busy_page_range(mapping, 0, LLONG_MAX);
-}
-EXPORT_SYMBOL_GPL(dax_layout_busy_page);
 
 static int __dax_invalidate_entry(struct address_space *mapping,
 				  pgoff_t index, bool trunc)
diff --git a/fs/dcache.c b/fs/dcache.c
index 1b1a81f10da67c..71918fe19d4993 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -32,6 +32,7 @@
 #include <linux/bit_spinlock.h>
 #include <linux/rculist_bl.h>
 #include <linux/list_lru.h>
+#include <linux/namei.h>
 #include "internal.h"
 #include "mount.h"
 
@@ -144,7 +145,7 @@ static DEFINE_PER_CPU(long, nr_dentry_unused);
 static DEFINE_PER_CPU(long, nr_dentry_negative);
 static int dentry_negative_policy;
 
-#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
+#ifdef CONFIG_SYSCTL
 /* Statistics gathering. */
 static struct dentry_stat_t dentry_stat = {
 	.age_limit = 45,
@@ -450,6 +451,17 @@ static void dentry_free(struct dentry *dentry)
 		call_rcu(&dentry->d_rcu, __d_free);
 }
 
+/*
+ * If inode is unlinked and doesn't have any aliases (i.e., all fds pointing to
+ * it are closed), it is pretty much dead. Except that file handle lookup could
+ * still revive it which causes issues to fsnotify. So once inode reaches this
+ * state we make sure to block creating any new aliases.
+ */
+static bool inode_notify_dead(struct inode *inode)
+{
+	return !inode->i_nlink && hlist_empty(&inode->i_dentry);
+}
+
 /*
  * Release the dentry's inode, using the filesystem
  * d_iput() operation if defined.
@@ -459,6 +471,7 @@ static void dentry_unlink_inode(struct dentry * dentry)
 	__releases(dentry->d_inode->i_lock)
 {
 	struct inode *inode = dentry->d_inode;
+	bool notify_dead;
 
 	raw_write_seqcount_begin(&dentry->d_seq);
 	__d_clear_type_and_inode(dentry);
@@ -469,9 +482,10 @@ static void dentry_unlink_inode(struct dentry * dentry)
 	 */
 	dentry->waiters = NULL;
 	raw_write_seqcount_end(&dentry->d_seq);
+	notify_dead = inode_notify_dead(inode);
 	spin_unlock(&dentry->d_lock);
 	spin_unlock(&inode->i_lock);
-	if (!inode->i_nlink)
+	if (notify_dead)
 		fsnotify_inoderemove(inode);
 	if (dentry->d_op && dentry->d_op->d_iput)
 		dentry->d_op->d_iput(dentry, inode);
@@ -830,7 +844,7 @@ static struct dentry *dentry_kill(struct dentry *dentry)
 	if (dentry->d_op && dentry->d_op->d_release)
 		dentry->d_op->d_release(dentry);
 
-	cond_resched();
+	cond_resched_tasks_rcu_qs();
 	/* now that it's negative, ->d_parent is stable */
 	if (!IS_ROOT(dentry)) {
 		parent = dentry->d_parent;
@@ -1900,6 +1914,7 @@ EXPORT_SYMBOL(d_invalidate);
  
 static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
 {
+	static struct lock_class_key __lookup_key;
 	struct dentry *dentry;
 	char *dname;
 	int err;
@@ -1916,6 +1931,10 @@ static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
 	 * be overwriting an internal NUL character
 	 */
 	dentry->d_shortname.string[DNAME_INLINE_LEN-1] = 0;
+
+	/* Racy __d_lookup_rcu() walk may read past the NUL; harmless */
+	kmsan_unpoison_memory(dentry->d_shortname.string, DNAME_INLINE_LEN);
+
 	if (unlikely(!name)) {
 		name = &slash_name;
 		dname = dentry->d_shortname.string;
@@ -1957,6 +1976,8 @@ static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
 	dentry->waiters = NULL;
 	INIT_HLIST_NODE(&dentry->d_sib);
 
+	lockdep_init_map(&dentry->lookup_map, "DCACHE_PAR_LOOKUP", &__lookup_key, 0);
+
 	if (dentry->d_op && dentry->d_op->d_init) {
 		err = dentry->d_op->d_init(dentry);
 		if (err) {
@@ -1999,6 +2020,58 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
 }
 EXPORT_SYMBOL(d_alloc);
 
+/**
+ * d_duplicate - duplicate a dentry for combined atomic operation
+ * @dentry: the dentry to duplicate
+ *
+ * Some rename operations need to be combined with another operation
+ * inside the filesystem.
+ * 1/ A cluster filesystem when renaming to an in-use file might need to
+ *   first "silly-rename" that target out of the way before the main rename
+ * 2/ A filesystem that supports white-out might want to create a whiteout
+ *   in place of the file being moved.
+ *
+ * For this they need two dentries which temporarily have the same name,
+ * before one is renamed.  d_duplicate() provides for this.  Given a
+ * positive hashed dentry, it creates a second in-lookup dentry.
+ * Because the original dentry exists, no other thread will try to
+ * create an in-lookup dentry, so there can be no race in this create.
+ *
+ * The caller should d_move() the original to a new name, often via a
+ * rename request, and should call d_lookup_done() on the newly created
+ * dentry.  If the new is instantiated then the old MUST either be moved
+ * or dropped.
+ *
+ * Parent must be locked.
+ *
+ * Returns: an in-lookup dentry, or -ENOMEM.
+ */
+struct dentry *d_duplicate(struct dentry *dentry)
+{
+	unsigned int hash = dentry->d_name.hash;
+	struct dentry *parent = dentry->d_parent;
+	struct hlist_bl_head *b = in_lookup_hash(parent, hash);
+	struct dentry *new = __d_alloc(parent->d_sb, &dentry->d_name);
+
+	if (unlikely(!new))
+		return ERR_PTR(-ENOMEM);
+
+	new->d_flags |= DCACHE_PAR_LOOKUP;
+	lock_map_acquire_try(&new->lookup_map);
+	spin_lock(&parent->d_lock);
+	new->d_parent = dget_dlock(parent);
+	hlist_add_head(&new->d_sib, &parent->d_children);
+	if (parent->d_flags & DCACHE_DISCONNECTED)
+		new->d_flags |= DCACHE_DISCONNECTED;
+	spin_unlock(&parent->d_lock);
+
+	hlist_bl_lock(b);
+	hlist_bl_add_head(&new->d_in_lookup_hash, b);
+	hlist_bl_unlock(b);
+	return new;
+}
+EXPORT_SYMBOL(d_duplicate);
+
 struct dentry *d_alloc_anon(struct super_block *sb)
 {
 	return __d_alloc(sb, NULL);
@@ -2168,7 +2241,6 @@ static void __d_instantiate(struct dentry *dentry, struct inode *inode)
  * (or otherwise set) by the caller to indicate that it is now
  * in use by the dcache.
  */
- 
 void d_instantiate(struct dentry *entry, struct inode * inode)
 {
 	BUG_ON(d_really_is_positive(entry));
@@ -2237,7 +2309,12 @@ static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
 
 	sb = inode->i_sb;
 
-	res = d_find_any_alias(inode); /* existing alias? */
+	spin_lock(&inode->i_lock);
+	if (!inode_notify_dead(inode))
+		res = __d_find_any_alias(inode);	/* existing alias? */
+	else
+		res = ERR_PTR(-ESTALE);
+	spin_unlock(&inode->i_lock);
 	if (res)
 		goto out;
 
@@ -2249,7 +2326,10 @@ static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
 
 	security_d_instantiate(new, inode);
 	spin_lock(&inode->i_lock);
-	res = __d_find_any_alias(inode); /* recheck under lock */
+	if (!inode_notify_dead(inode))
+		res = __d_find_any_alias(inode);	/* recheck under lock */
+	else
+		res = ERR_PTR(-ESTALE);
 	if (likely(!res)) { /* still no alias, attach a disconnected dentry */
 		unsigned add_flags = d_flags_for_inode(inode);
 
@@ -2750,6 +2830,15 @@ static inline void end_dir_add(struct inode *dir, unsigned int n)
 static void d_wait_lookup(struct dentry *dentry)
 {
 	if (likely(d_in_lookup(dentry))) {
+		/*
+		 * Tell lockdep we will wait for the lookup lock, after
+		 * dropping ->d_lock, but won't actually take it.
+		 */
+		spin_release(&dentry->d_lock.dep_map, _THIS_IP_);
+		lock_map_acquire(&dentry->lookup_map);
+		lock_map_release(&dentry->lookup_map);
+		spin_acquire(&dentry->d_lock.dep_map, 0, 1, _THIS_IP_);
+
 		dentry->d_flags |= DCACHE_LOOKUP_WAITERS;
 		wait_var_event_spinlock(&dentry->d_flags,
 					!d_in_lookup(dentry),
@@ -2757,8 +2846,16 @@ static void d_wait_lookup(struct dentry *dentry)
 	}
 }
 
-struct dentry *d_alloc_parallel(struct dentry *parent,
-				const struct qstr *name)
+/* What to do when __d_alloc_parallel finds a d_in_lookup dentry */
+enum alloc_para {
+	ALLOC_PARA_WAIT,
+	ALLOC_PARA_FAIL,
+};
+
+static inline
+struct dentry *__d_alloc_parallel(struct dentry *parent,
+				  const struct qstr *name,
+				  enum alloc_para how)
 {
 	unsigned int hash = name->hash;
 	struct hlist_bl_head *b = in_lookup_hash(parent, hash);
@@ -2831,6 +2928,12 @@ struct dentry *d_alloc_parallel(struct dentry *parent,
 			spin_unlock(&dentry->d_lock);
 			goto retry;
 		}
+		if (unlikely(how == ALLOC_PARA_FAIL)) {
+			/* mustn't wait for concurrent lookup to complete */
+			spin_unlock(&dentry->d_lock);
+			dput(new);
+			return ERR_PTR(-EWOULDBLOCK);
+		}
 		/*
 		 * somebody is likely to be still doing lookup for it;
 		 * pin it and wait for them to finish
@@ -2858,14 +2961,77 @@ struct dentry *d_alloc_parallel(struct dentry *parent,
 	}
 	hlist_bl_add_head(&new->d_in_lookup_hash, b);
 	hlist_bl_unlock(b);
+	lock_map_acquire_try(&new->lookup_map);
 	return new;
 mismatch:
 	spin_unlock(&dentry->d_lock);
 	dput(dentry);
 	goto retry;
 }
+
+/**
+ * d_alloc_parallel() - allocate a new dentry and ensure uniqueness
+ * @parent: dentry of the parent
+ * @name:   name of the dentry within that parent.
+ *
+ * A new dentry is allocated and, providing it is unique, added to the
+ * relevant index.
+ * If an existing dentry is found with the same parent/name that is
+ * not d_in_lookup(), then that is returned instead.
+ * If the existing dentry is d_in_lookup(), d_alloc_parallel() waits for
+ * that lookup to complete before returning the dentry and then ensures the
+ * match is still valid.
+ * Thus if the returned dentry is d_in_lookup() then the caller has
+ * exclusive access until it completes the lookup.
+ * If the returned dentry is not d_in_lookup() then a lookup has
+ * already completed.
+ *
+ * The @name must already have ->hash set, as can be achieved
+ * by e.g. try_lookup_noperm().
+ *
+ * Returns: the dentry, whether found or allocated, or an error %-ENOMEM.
+ */
+struct dentry *d_alloc_parallel(struct dentry *parent,
+				const struct qstr *name)
+{
+	return __d_alloc_parallel(parent, name, ALLOC_PARA_WAIT);
+}
 EXPORT_SYMBOL(d_alloc_parallel);
 
+/**
+ * d_alloc_trylock() - find or allocate a new dentry
+ * @parent: dentry of the parent
+ * @name:   name of the dentry within that parent.
+ *
+ * A new dentry is allocated and, providing it is unique, added to the
+ * relevant index.
+ * If an existing dentry is found with the same parent/name that is
+ * not d_in_lookup() then that is returned instead.
+ * If the existing dentry is d_in_lookup(), d_alloc_trylock()
+ * returns with error %-EWOULDBLOCK.
+ * Thus if the returned dentry is d_in_lookup() then the caller has
+ * exclusive access until it completes the lookup.
+ * If the returned dentry is not d_in_lookup() then a lookup has
+ * already completed.
+ *
+ * The @name need not already have ->hash set.
+ *
+ * Returns: the dentry, whether found or allocated, or an error
+ *    %-ENOMEM, %-EWOULDBLOCK, %-EACCES (for a bad name) or
+ *    anything returned by ->d_hash().
+ */
+struct dentry *d_alloc_trylock(struct dentry *parent,
+			       struct qstr *name)
+{
+	struct dentry *de;
+
+	de = try_lookup_noperm(name, parent);
+	if (!de)
+		de = __d_alloc_parallel(parent, name, ALLOC_PARA_FAIL);
+	return de;
+}
+EXPORT_SYMBOL(d_alloc_trylock);
+
 /*
  * Move dentry from in-lookup state to busy-negative one.
  *
@@ -2894,6 +3060,7 @@ static void __d_lookup_unhash(struct dentry *dentry)
 	b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash);
 	hlist_bl_lock(b);
 	dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
+	lock_map_release(&dentry->lookup_map);
 	__hlist_bl_del(&dentry->d_in_lookup_hash);
 	hlist_bl_unlock(b);
 	dentry->waiters = NULL;
@@ -2931,15 +3098,10 @@ static inline void __d_add(struct dentry *dentry, struct inode *inode,
 	}
 	if (unlikely(ops))
 		d_set_d_op(dentry, ops);
-	if (inode) {
-		unsigned add_flags = d_flags_for_inode(inode);
-		hlist_add_head(&dentry->d_alias, &inode->i_dentry);
-		raw_write_seqcount_begin(&dentry->d_seq);
-		__d_set_inode_and_type(dentry, inode, add_flags);
-		raw_write_seqcount_end(&dentry->d_seq);
-		fsnotify_update_flags(dentry);
-	}
-	__d_rehash(dentry);
+	if (inode)
+		__d_instantiate(dentry, inode);
+	if (d_unhashed(dentry))
+		__d_rehash(dentry);
 	if (dir) {
 		end_dir_add(dir, n);
 		__d_wake_in_lookup_waiters(dentry);
@@ -3241,7 +3403,7 @@ struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry,
 	if (IS_ERR(inode))
 		return ERR_CAST(inode);
 
-	BUG_ON(!d_unhashed(dentry));
+	BUG_ON(d_really_is_positive(dentry));
 
 	if (!inode)
 		goto out;
@@ -3297,6 +3459,8 @@ struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry,
  * @inode:  the inode which may have a disconnected dentry
  * @dentry: a negative dentry which we want to point to the inode.
  *
+ * @dentry must be negative and may be in-lookup or unhashed or hashed.
+ *
  * If inode is a directory and has an IS_ROOT alias, then d_move that in
  * place of the given dentry and return it, else simply d_add the inode
  * to the dentry and return NULL.
@@ -3304,16 +3468,14 @@ struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry,
  * If a non-IS_ROOT directory is found, the filesystem is corrupt, and
  * we should error out: directories can't have multiple aliases.
  *
- * This is needed in the lookup routine of any filesystem that is exportable
- * (via knfsd) so that we can build dcache paths to directories effectively.
+ * This should be used to return the result of ->lookup() and to
+ * instantiate the result of ->mkdir(), is often useful for
+ * ->atomic_open, and may be used to instantiate other objects.
  *
  * If a dentry was found and moved, then it is returned.  Otherwise NULL
- * is returned.  This matches the expected return value of ->lookup.
+ * is returned.  This matches the expected return value of ->lookup and
+ * ->mkdir.
  *
- * Cluster filesystems may call this function with a negative, hashed dentry.
- * In that case, we know that the inode will be a regular file, and also this
- * will only occur during atomic_open. So we need to check for the dentry
- * being already hashed only in the final case.
  */
 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
 {
diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c
index e054e62919ec6d..b4915551ad315a 100644
--- a/fs/debugfs/inode.c
+++ b/fs/debugfs/inode.c
@@ -42,7 +42,7 @@ static bool debugfs_enabled __ro_after_init = IS_ENABLED(CONFIG_DEBUG_FS_ALLOW_A
  * so that we can use the file mode as part of a heuristic to determine whether
  * to lock down individual files.
  */
-static int debugfs_setattr(struct mnt_idmap *idmap,
+static int debugfs_setattr(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, struct iattr *ia)
 {
 	int ret;
@@ -368,6 +368,9 @@ static struct dentry *debugfs_start_creating(const char *name,
 	if (!debugfs_enabled)
 		return ERR_PTR(-EPERM);
 
+	if (IS_ERR(parent))
+		return parent;
+
 	if (!debugfs_initialized()) {
 		pr_err("Unable to create file '%s', debugfs is not initialized yet\n",
 		       name);
@@ -376,9 +379,6 @@ static struct dentry *debugfs_start_creating(const char *name,
 
 	pr_debug("creating file '%s'\n", name);
 
-	if (IS_ERR(parent))
-		return parent;
-
 	error = simple_pin_fs(&debug_fs_type, &debugfs_mount,
 			      &debugfs_mount_count);
 	if (error) {
diff --git a/fs/devpts/inode.c b/fs/devpts/inode.c
index 9844dcf354ee90..bd1e8eb26edb70 100644
--- a/fs/devpts/inode.c
+++ b/fs/devpts/inode.c
@@ -249,6 +249,8 @@ static int devpts_parse_param(struct fs_context *fc, struct fs_parameter *param)
 	case Opt_max:
 		if (result.uint_32 > NR_UNIX98_PTY_MAX)
 			return invalf(fc, "max out of range");
+		if (result.uint_32 == 0)
+			return invalf(fc, "max must be greater than 0");
 		opts->max = result.uint_32;
 		break;
 	}
diff --git a/fs/dlm/config.c b/fs/dlm/config.c
index 53cd3329304231..6c5c3f049b33a0 100644
--- a/fs/dlm/config.c
+++ b/fs/dlm/config.c
@@ -64,11 +64,37 @@ static void release_node(struct config_item *);
 static struct configfs_attribute *comm_attrs[];
 static struct configfs_attribute *node_attrs[];
 
+static u32 rsb_hashfn(const void *data, u32 len, u32 seed)
+{
+	const struct dlm_rsb_key *key = data;
+
+	return jhash(key->name, key->len, 0);
+}
+
+static u32 rsb_obj_hashfn(const void *data, u32 len, u32 seed)
+{
+	const struct dlm_rsb *r = data;
+
+	return r->res_hash;
+}
+
+static int rsb_obj_cmpfn(struct rhashtable_compare_arg *arg, const void *obj)
+{
+	const struct dlm_rsb_key *key = arg->key;
+	const struct dlm_rsb *r = obj;
+
+	if (key->len != r->res_length)
+		return -1;
+
+	return memcmp(&r->res_name, key->name, key->len);
+}
+
 const struct rhashtable_params dlm_rhash_rsb_params = {
 	.nelem_hint = 3, /* start small */
-	.key_len = DLM_RESNAME_MAXLEN,
-	.key_offset = offsetof(struct dlm_rsb, res_name),
 	.head_offset = offsetof(struct dlm_rsb, res_node),
+	.hashfn = rsb_hashfn,
+	.obj_hashfn = rsb_obj_hashfn,
+	.obj_cmpfn = rsb_obj_cmpfn,
 	.automatic_shrinking = true,
 };
 
diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h
index 9df842421ae06d..74c2e77d1b553e 100644
--- a/fs/dlm/dlm_internal.h
+++ b/fs/dlm/dlm_internal.h
@@ -340,6 +340,11 @@ struct dlm_rsb {
 	char			res_name[DLM_RESNAME_MAXLEN+1];
 };
 
+struct dlm_rsb_key {
+	char name[DLM_RESNAME_MAXLEN];
+	size_t len;
+};
+
 /* dlm_master_lookup() flags */
 
 #define DLM_LU_RECOVER_DIR	1
diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c
index c381e102844656..2609e4fdeba847 100644
--- a/fs/dlm/lock.c
+++ b/fs/dlm/lock.c
@@ -622,13 +622,17 @@ static int get_rsb_struct(struct dlm_ls *ls, const void *name, int len,
 	return 0;
 }
 
-int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len,
-			struct dlm_rsb **r_ret)
+int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name,
+			unsigned int len, struct dlm_rsb **r_ret)
 {
-	char key[DLM_RESNAME_MAXLEN] = {};
+	struct dlm_rsb_key key = {
+		.len = len,
+	};
+
 	if (len > DLM_RESNAME_MAXLEN)
 		return -EINVAL;
-	memcpy(key, name, len);
+
+	memcpy(key.name, name, len);
 	*r_ret = rhashtable_lookup_fast(rhash, &key, dlm_rhash_rsb_params);
 	if (*r_ret)
 		return 0;
@@ -1421,7 +1425,7 @@ void dlm_dump_rsb_name(struct dlm_ls *ls, const char *name, int len)
 
 	rcu_read_lock();
 	error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r);
-	if (!error)
+	if (error)
 		goto out;
 
 	dlm_dump_rsb(r);
@@ -5529,6 +5533,10 @@ static int receive_rcom_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
 {
 	struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf;
 
+	if (rl->rl_rqmode < DLM_LOCK_IV || rl->rl_rqmode > DLM_LOCK_EX ||
+	    rl->rl_grmode < DLM_LOCK_IV || rl->rl_grmode > DLM_LOCK_EX)
+		return -EINVAL;
+
 	lkb->lkb_nodeid = le32_to_cpu(rc->rc_header.h_nodeid);
 	lkb->lkb_ownpid = le32_to_cpu(rl->rl_ownpid);
 	lkb->lkb_remid = le32_to_cpu(rl->rl_lkid);
diff --git a/fs/dlm/lock.h b/fs/dlm/lock.h
index b23d7b854ed468..c75975937331cb 100644
--- a/fs/dlm/lock.h
+++ b/fs/dlm/lock.h
@@ -31,8 +31,8 @@ void resume_scan_timer(struct dlm_ls *ls);
 int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name,
 		      int len, unsigned int flags, int *r_nodeid, int *result);
 
-int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len,
-			struct dlm_rsb **r_ret);
+int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name,
+			unsigned int len, struct dlm_rsb **r_ret);
 
 void dlm_recover_purge(struct dlm_ls *ls, const struct list_head *root_list);
 void dlm_purge_mstcpy_locks(struct dlm_rsb *r);
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index 2aff1c7c17de49..ea8353c4638d05 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -1984,4 +1984,5 @@ void dlm_lowcomms_exit(void)
 		}
 	}
 	srcu_read_unlock(&connections_srcu, idx);
+	srcu_barrier(&connections_srcu);
 }
diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c
index 8964164600d2d7..0454315244945d 100644
--- a/fs/dlm/midcomms.c
+++ b/fs/dlm/midcomms.c
@@ -1178,6 +1178,7 @@ void dlm_midcomms_exit(void)
 		}
 	}
 	srcu_read_unlock(&nodes_srcu, idx);
+	srcu_barrier(&nodes_srcu);
 
 	dlm_lowcomms_exit();
 }
diff --git a/fs/dlm/plock.c b/fs/dlm/plock.c
index e9598b3fe5d09e..711e8bc3a46a65 100644
--- a/fs/dlm/plock.c
+++ b/fs/dlm/plock.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/fs.h>
+#include <linux/capability.h>
 #include <linux/filelock.h>
 #include <linux/miscdevice.h>
 #include <linux/poll.h>
@@ -477,6 +478,15 @@ int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file,
 }
 EXPORT_SYMBOL_GPL(dlm_posix_get);
 
+static int dev_open(struct inode *inode, struct file *file)
+{
+	/* Userspace plock daemon is a privileged cluster component. */
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	return 0;
+}
+
 /* a read copies out one plock request from the send list */
 static ssize_t dev_read(struct file *file, char __user *u, size_t count,
 			loff_t *ppos)
@@ -598,6 +608,7 @@ static __poll_t dev_poll(struct file *file, poll_table *wait)
 }
 
 static const struct file_operations dev_fops = {
+	.open    = dev_open,
 	.read    = dev_read,
 	.write   = dev_write,
 	.poll    = dev_poll,
@@ -608,7 +619,8 @@ static const struct file_operations dev_fops = {
 static struct miscdevice plock_dev_misc = {
 	.minor = MISC_DYNAMIC_MINOR,
 	.name = DLM_PLOCK_MISC_NAME,
-	.fops = &dev_fops
+	.fops = &dev_fops,
+	.mode = 0600,
 };
 
 int dlm_plock_init(void)
diff --git a/fs/dlm/user.c b/fs/dlm/user.c
index a8ed4c8fdc5bdf..0b0a7e1bd10958 100644
--- a/fs/dlm/user.c
+++ b/fs/dlm/user.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/miscdevice.h>
+#include <linux/capability.h>
 #include <linux/init.h>
 #include <linux/wait.h>
 #include <linux/file.h>
@@ -511,6 +512,7 @@ static ssize_t device_write(struct file *file, const char __user *buf,
 			    size_t count, loff_t *ppos)
 {
 	struct dlm_user_proc *proc = file->private_data;
+	size_t name_payload = 0;
 	struct dlm_write_request *kbuf;
 	int error;
 
@@ -544,6 +546,7 @@ static ssize_t device_write(struct file *file, const char __user *buf,
 
 		if (count > sizeof(struct dlm_write_request32))
 			namelen = count - sizeof(struct dlm_write_request32);
+		name_payload = namelen;
 
 		k32buf = (struct dlm_write_request32 *)kbuf;
 
@@ -560,7 +563,13 @@ static ssize_t device_write(struct file *file, const char __user *buf,
 
 		compat_input(kbuf, k32buf, namelen);
 		kfree(k32buf);
+	} else {
+		if (count > sizeof(*kbuf))
+			name_payload = count - sizeof(*kbuf);
 	}
+#else
+	if (count > sizeof(*kbuf))
+		name_payload = count - sizeof(*kbuf);
 #endif
 
 	/* do we really need this? can a write happen after a close? */
@@ -570,6 +579,15 @@ static ssize_t device_write(struct file *file, const char __user *buf,
 		goto out_free;
 	}
 
+	if (kbuf->cmd == DLM_USER_LOCK &&
+	    !(kbuf->i.lock.flags & DLM_LKF_CONVERT)) {
+		if (kbuf->i.lock.namelen > name_payload ||
+		    kbuf->i.lock.namelen > DLM_RESNAME_MAXLEN) {
+			error = -EINVAL;
+			goto out_free;
+		}
+	}
+
 	error = -EINVAL;
 
 	switch (kbuf->cmd)
@@ -910,6 +928,10 @@ static int ctl_device_close(struct inode *inode, struct file *file)
 
 static int monitor_device_open(struct inode *inode, struct file *file)
 {
+	/* dlm_controld is the only expected opener; last close stops LS. */
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
 	atomic_inc(&dlm_monitor_opened);
 	dlm_monitor_unused = 0;
 	return 0;
@@ -958,6 +980,7 @@ static struct miscdevice monitor_device = {
 	.name  = "dlm-monitor",
 	.fops  = &monitor_device_fops,
 	.minor = MISC_DYNAMIC_MINOR,
+	.mode  = 0600,
 };
 
 int __init dlm_user_init(void)
diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c
index 525297c7ebd8c4..48e520960d6657 100644
--- a/fs/ecryptfs/inode.c
+++ b/fs/ecryptfs/inode.c
@@ -266,7 +266,7 @@ int ecryptfs_initialize_file(struct dentry *ecryptfs_dentry,
  * Returns zero on success; non-zero on error condition
  */
 static int
-ecryptfs_create(struct mnt_idmap *idmap,
+ecryptfs_create(const struct mnt_idmap *idmap,
 		struct inode *directory_inode, struct dentry *ecryptfs_dentry,
 		umode_t mode)
 {
@@ -462,7 +462,7 @@ static int ecryptfs_unlink(struct inode *dir, struct dentry *dentry)
 	return ecryptfs_do_unlink(dir, dentry, d_inode(dentry));
 }
 
-static int ecryptfs_symlink(struct mnt_idmap *idmap,
+static int ecryptfs_symlink(const struct mnt_idmap *idmap,
 			    struct inode *dir, struct dentry *dentry,
 			    const char *symname)
 {
@@ -503,7 +503,7 @@ static int ecryptfs_symlink(struct mnt_idmap *idmap,
 	return rc;
 }
 
-static struct dentry *ecryptfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ecryptfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				     struct dentry *dentry, umode_t mode)
 {
 	int rc;
@@ -562,7 +562,7 @@ static int ecryptfs_rmdir(struct inode *dir, struct dentry *dentry)
 }
 
 static int
-ecryptfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+ecryptfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 	       struct dentry *dentry, umode_t mode, dev_t dev)
 {
 	int rc;
@@ -590,7 +590,7 @@ ecryptfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int
-ecryptfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+ecryptfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		struct dentry *old_dentry, struct inode *new_dir,
 		struct dentry *new_dentry, unsigned int flags)
 {
@@ -849,7 +849,7 @@ int ecryptfs_truncate(struct dentry *dentry, loff_t new_length)
 }
 
 static int
-ecryptfs_permission(struct mnt_idmap *idmap, struct inode *inode,
+ecryptfs_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		    int mask)
 {
 	return inode_permission(&nop_mnt_idmap,
@@ -869,7 +869,7 @@ ecryptfs_permission(struct mnt_idmap *idmap, struct inode *inode,
  * All other metadata changes will be passed right to the lower filesystem,
  * and we will just update our inode to look like the lower.
  */
-static int ecryptfs_setattr(struct mnt_idmap *idmap,
+static int ecryptfs_setattr(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct iattr *ia)
 {
 	struct inode *inode = d_inode(dentry);
@@ -939,7 +939,7 @@ static int ecryptfs_setattr(struct mnt_idmap *idmap,
 	return rc;
 }
 
-static int ecryptfs_getattr_link(struct mnt_idmap *idmap,
+static int ecryptfs_getattr_link(const struct mnt_idmap *idmap,
 				 const struct path *path, struct kstat *stat,
 				 u32 request_mask, unsigned int flags)
 {
@@ -965,7 +965,7 @@ static int ecryptfs_getattr_link(struct mnt_idmap *idmap,
 	return rc;
 }
 
-static int ecryptfs_getattr(struct mnt_idmap *idmap,
+static int ecryptfs_getattr(const struct mnt_idmap *idmap,
 			    const struct path *path, struct kstat *stat,
 			    u32 request_mask, unsigned int flags)
 {
@@ -1078,7 +1078,7 @@ static int ecryptfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return vfs_fileattr_get(ecryptfs_dentry_to_lower(dentry), fa);
 }
 
-static int ecryptfs_fileattr_set(struct mnt_idmap *idmap,
+static int ecryptfs_fileattr_set(const struct mnt_idmap *idmap,
 				 struct dentry *dentry, struct file_kattr *fa)
 {
 	struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry);
@@ -1090,14 +1090,14 @@ static int ecryptfs_fileattr_set(struct mnt_idmap *idmap,
 	return rc;
 }
 
-static struct posix_acl *ecryptfs_get_acl(struct mnt_idmap *idmap,
+static struct posix_acl *ecryptfs_get_acl(const struct mnt_idmap *idmap,
 					  struct dentry *dentry, int type)
 {
 	return vfs_get_acl(idmap, ecryptfs_dentry_to_lower(dentry),
 			   posix_acl_xattr_name(type));
 }
 
-static int ecryptfs_set_acl(struct mnt_idmap *idmap,
+static int ecryptfs_set_acl(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct posix_acl *acl,
 			    int type)
 {
@@ -1158,7 +1158,7 @@ static int ecryptfs_xattr_get(const struct xattr_handler *handler,
 }
 
 static int ecryptfs_xattr_set(const struct xattr_handler *handler,
-			      struct mnt_idmap *idmap,
+			      const struct mnt_idmap *idmap,
 			      struct dentry *dentry, struct inode *inode,
 			      const char *name, const void *value, size_t size,
 			      int flags)
diff --git a/fs/efivarfs/inode.c b/fs/efivarfs/inode.c
index f0d009555fc660..07602cd5d33c3c 100644
--- a/fs/efivarfs/inode.c
+++ b/fs/efivarfs/inode.c
@@ -74,7 +74,7 @@ static bool efivarfs_valid_name(const char *str, int len)
 	return uuid_is_valid(s);
 }
 
-static int efivarfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int efivarfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 			   struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode = NULL;
@@ -150,7 +150,7 @@ efivarfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 }
 
 static int
-efivarfs_fileattr_set(struct mnt_idmap *idmap,
+efivarfs_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa)
 {
 	unsigned int i_flags = 0;
@@ -170,7 +170,7 @@ efivarfs_fileattr_set(struct mnt_idmap *idmap,
 }
 
 /* copy of simple_setattr except that it doesn't do i_size updates */
-static int efivarfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int efivarfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig
index 37495e95a78d9b..2dfc313588d283 100644
--- a/fs/erofs/Kconfig
+++ b/fs/erofs/Kconfig
@@ -132,13 +132,11 @@ config EROFS_FS_ZIP_LZMA
 	  Say N if you want to disable LZMA compression support.
 
 config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS
-	int "EROFS LZMA default maximum decompression streams" if SMP
+	int "EROFS LZMA default maximum decompression streams"
 	depends on EROFS_FS_ZIP_LZMA
 	range 1 NR_CPUS if SMP
 	range 1 1 if !SMP
-	default 16 if SMP && NR_CPUS >= 16
-	default NR_CPUS if SMP
-	default 1
+	default 16
 	help
 	  By default EROFS allocates one LZMA decompression stream per CPU.
 	  Each stream can hold a dictionary of up to 8 MiB taken from the
@@ -146,7 +144,7 @@ config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS
 	  of memory.  This caps the default; the lzma_streams module parameter
 	  still overrides it.
 
-	  If unsure, keep the suggested default (16 or lower).
+	  If unsure, keep the default of 16.
 
 config EROFS_FS_ZIP_DEFLATE
 	bool "EROFS DEFLATE compressed data support"
diff --git a/fs/erofs/data.c b/fs/erofs/data.c
index be63b89f086229..8b150aebf0af8c 100644
--- a/fs/erofs/data.c
+++ b/fs/erofs/data.c
@@ -239,19 +239,23 @@ int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map)
 /*
  * bit 30: I/O error occurred on this folio
  * bit 29: CPU has dirty data in D-cache (needs aliasing handling);
- * bit 0 - 29: remaining parts to complete this folio
+ * bit 0 - 28: remaining parts to complete this folio, biased by 1 so that
+ *	       ->private stays non-NULL while the folio is attached
  */
 #define EROFS_ONLINEFOLIO_EIO		30
 #define EROFS_ONLINEFOLIO_DIRTY		29
+#define EROFS_ONLINEFOLIO_COUNT_MASK	(BIT(EROFS_ONLINEFOLIO_DIRTY) - 1)
+#define EROFS_ONLINEFOLIO_BIAS		1
 
 void erofs_onlinefolio_init(struct folio *folio)
 {
 	union {
 		atomic_t o;
 		void *v;
-	} u = { .o = ATOMIC_INIT(1) };
+	} u = { .o = ATOMIC_INIT(1 + EROFS_ONLINEFOLIO_BIAS) };
 
-	folio->private = u.v;	/* valid only if file-backed folio is locked */
+	/* valid only if file-backed folio is locked */
+	folio_attach_private(folio, u.v);
 }
 
 void erofs_onlinefolio_split(struct folio *folio)
@@ -265,14 +269,14 @@ void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty)
 
 	do {
 		orig = atomic_read((atomic_t *)&folio->private);
-		DBG_BUGON(orig <= 0);
+		DBG_BUGON((orig & EROFS_ONLINEFOLIO_COUNT_MASK) <= EROFS_ONLINEFOLIO_BIAS);
 		v = dirty << EROFS_ONLINEFOLIO_DIRTY;
 		v |= (orig - 1) | (!!err << EROFS_ONLINEFOLIO_EIO);
 	} while (atomic_cmpxchg((atomic_t *)&folio->private, orig, v) != orig);
 
-	if (v & (BIT(EROFS_ONLINEFOLIO_DIRTY) - 1))
+	if ((v & EROFS_ONLINEFOLIO_COUNT_MASK) != EROFS_ONLINEFOLIO_BIAS)
 		return;
-	folio->private = 0;
+	folio_detach_private(folio);
 	if (v & BIT(EROFS_ONLINEFOLIO_DIRTY))
 		flush_dcache_folio(folio);
 	folio_end_read(folio, !(v & BIT(EROFS_ONLINEFOLIO_EIO)));
diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c
index 45afe5c50de830..26ea3790ff2134 100644
--- a/fs/erofs/inode.c
+++ b/fs/erofs/inode.c
@@ -311,7 +311,7 @@ struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid)
 	return inode;
 }
 
-int erofs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int erofs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, u32 request_mask,
 		  unsigned int query_flags)
 {
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 12e3a5b80a5a63..ab817091bd2947 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -417,7 +417,7 @@ void erofs_onlinefolio_init(struct folio *folio);
 void erofs_onlinefolio_split(struct folio *folio);
 void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty);
 struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid);
-int erofs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int erofs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, u32 request_mask,
 		  unsigned int query_flags);
 int erofs_namei(struct inode *dir, const struct qstr *name,
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 6b07e73ee2aaa0..e981e371d6c294 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -1886,21 +1886,14 @@ static void z_erofs_readahead(struct readahead_control *rac)
 	struct inode *realinode = erofs_real_inode(sharedinode, &need_iput);
 	Z_EROFS_DEFINE_FRONTEND(f, realinode, sharedinode, readahead_pos(rac));
 	unsigned int nrpages = readahead_count(rac);
-	struct folio *head = NULL, *folio;
+	struct folio *folio;
 	int err;
 
 	trace_erofs_readahead(realinode, readahead_index(rac), nrpages, false);
 	z_erofs_pcluster_readmore(&f, rac, true);
-	while ((folio = readahead_folio(rac))) {
-		folio->private = head;
-		head = folio;
-	}
-
-	/* traverse in reverse order for best metadata I/O performance */
-	while (head) {
-		folio = head;
-		head = folio_get_private(folio);
 
+	/* traverse from last to first for best metadata I/O performance */
+	while ((folio = readahead_folio_last(rac))) {
 		err = z_erofs_scan_folio(&f, folio, true);
 		if (err && err != -EINTR)
 			erofs_err(realinode->i_sb, "readahead error at folio %lu @ nid %llu",
diff --git a/fs/eventfd.c b/fs/eventfd.c
index 9d33a02757d52b..52426795752ece 100644
--- a/fs/eventfd.c
+++ b/fs/eventfd.c
@@ -403,8 +403,8 @@ static int do_eventfd(unsigned int count, int flags)
 	FD_PREPARE(fdf, flags,
 		   anon_inode_getfile_fmode("[eventfd]", &eventfd_fops, ctx,
 					    flags, FMODE_NOWAIT));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
 	ctx->id = ida_alloc(&eventfd_ida, GFP_KERNEL);
 	retain_and_null_ptr(ctx);
diff --git a/fs/eventpoll.c b/fs/eventpoll.c
index e0c4bf88a83836..f48b829a710f75 100644
--- a/fs/eventpoll.c
+++ b/fs/eventpoll.c
@@ -2514,11 +2514,11 @@ static int do_epoll_create(int flags)
 	FD_PREPARE(fdf, O_RDWR | (flags & O_CLOEXEC),
 		   anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,
 				      O_RDWR | (flags & O_CLOEXEC)));
-	if (fdf.err) {
+	if (fdf->fd < 0) {
 		ep_clear_and_put(ep);
-		return fdf.err;
+		return fdf->fd;
 	}
-	ep->file = fd_prepare_file(fdf);
+	ep->file = fdf->file;
 	return fd_publish(fdf);
 }
 
diff --git a/fs/exec.c b/fs/exec.c
index d3081c8f7c10c0..8d2b14e1d455cf 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -882,6 +882,7 @@ static int exec_mmap(struct linux_binprm *bprm)
 	active_mm = tsk->active_mm;
 	tsk->active_mm = mm;
 	tsk->mm = mm;
+	sched_cache_exec_mmap(tsk, mm);
 	mm_init_cid(mm, tsk);
 	exec_state = task_exec_state_replace(tsk, exec_state);
 	/*
@@ -1124,6 +1125,7 @@ static struct file *bprm_identity_file(const struct linux_binprm *bprm)
 int begin_new_exec(struct linux_binprm * bprm)
 {
 	struct task_struct *me = current;
+	struct files_struct *files = NULL;
 	int retval;
 
 	/* A pending PT_INTERP substitution this format cannot consume. */
@@ -1148,21 +1150,36 @@ int begin_new_exec(struct linux_binprm * bprm)
 	 */
 	bprm->point_of_no_return = true;
 
+	/*
+	 * Cancel any io_uring activity across execve. This runs task work
+	 * that may still create an io-wq worker, so do it while de_thread()
+	 * can still zap it.
+	 */
+	io_uring_task_cancel();
+
 	/* Make this the only thread in the thread group */
 	retval = de_thread(me);
 	if (retval)
 		goto out;
-	/* see the comment in check_unsafe_exec() */
-	current->fs->in_exec = 0;
+
 	/*
-	 * Cancel any io_uring activity across execve
+	 * This must be done here to ensure that POSIX CPU timers which were
+	 * armed on the current task are dequeued from me::posix_cputimers.
+	 * Otherwise in case of a TID switch the deletion of the related POSIX
+	 * timer would not remove an enqueued timer because the TID lookup
+	 * of the old TID fails.
 	 */
-	io_uring_task_cancel();
+	posixtimer_exec();
+
+	/* see the comment in check_unsafe_exec() */
+	current->fs->in_exec = 0;
 
 	/* Ensure the files table is not shared. */
-	retval = unshare_files();
+	retval = unshare_fd(CLONE_FILES, &files);
 	if (retval)
 		goto out;
+	if (files)
+		switch_files_struct(me, files);
 
 	/*
 	 * We have to apply CLOEXEC before we change whether the process is
@@ -1170,13 +1187,13 @@ int begin_new_exec(struct linux_binprm * bprm)
 	 * trying to access the should-be-closed file descriptors of a process
 	 * undergoing exec(2).
 	 *
-	 * This can block on filesystem ->flush() handlers, including waiting
-	 * for FUSE daemons, so do it before exec_mmap takes the
-	 * exec_update_lock.
+	 * This can block on filesystem ->flush() and ->release() handlers,
+	 * including waiting for FUSE daemons, so do it before exec_mmap
+	 * takes the exec_update_lock.
 	 * This must happen after the point of no return, and after unsharing
 	 * the FD table.
 	 */
-	do_close_on_exec(me->files);
+	close_cloexec_files(me->files);
 
 	/*
 	 * Must be called _before_ exec_mmap() as bprm->mm is
@@ -1206,14 +1223,6 @@ int begin_new_exec(struct linux_binprm * bprm)
 	if (retval)
 		goto out_unlock;
 
-#ifdef CONFIG_POSIX_TIMERS
-	spin_lock_irq(&me->sighand->siglock);
-	posix_cpu_timers_exit(me);
-	spin_unlock_irq(&me->sighand->siglock);
-	exit_itimers(me);
-	flush_itimer_signals();
-#endif
-
 	/*
 	 * Make the signal table private.
 	 */
@@ -1345,7 +1354,7 @@ EXPORT_SYMBOL(begin_new_exec);
 void would_dump(struct linux_binprm *bprm, struct file *file)
 {
 	struct inode *inode = file_inode(file);
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	if (inode_permission(idmap, inode, MAY_READ) < 0) {
 		struct user_namespace *old, *user_ns;
 		bprm->interp_flags |= BINPRM_FLAGS_ENFORCE_NONDUMP;
@@ -1462,9 +1471,12 @@ void bprm_drop_loader(struct linux_binprm *bprm)
 
 static void free_bprm(struct linux_binprm *bprm)
 {
-	if (bprm->mm) {
+	struct mm_struct *mm = bprm->mm;
+
+	if (mm) {
 		acct_arg_size(bprm, 0);
-		mmput(bprm->mm);
+		bprm->mm = NULL;
+		mmput(mm);
 	}
 	if (bprm->user_ns)
 		put_user_ns(bprm->user_ns);
@@ -1629,7 +1641,7 @@ static void check_unsafe_exec(struct linux_binprm *bprm)
 static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file)
 {
 	/* Handle suid and sgid on files */
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct inode *inode = file_inode(file);
 	unsigned int mode;
 	vfsuid_t vfsuid;
diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c
index fe73b1380c5d87..46514b13bebda4 100644
--- a/fs/exfat/dir.c
+++ b/fs/exfat/dir.c
@@ -678,11 +678,54 @@ enum exfat_validate_dentry_mode {
 	ES_MODE_GET_BENIGN_SEC_ENTRY,
 };
 
-static bool exfat_validate_entry(unsigned int type,
-		enum exfat_validate_dentry_mode *mode)
+static bool exfat_validate_vendor_alloc(struct super_block *sb,
+		struct exfat_dentry *ep)
+{
+	struct exfat_sb_info *sbi = EXFAT_SB(sb);
+	u8 flags = ep->dentry.vendor_alloc.flags;
+	u32 start_clu = le32_to_cpu(ep->dentry.vendor_alloc.start_clu);
+	u64 size = le64_to_cpu(ep->dentry.vendor_alloc.size);
+	u64 max_size = exfat_cluster_to_bytes(sbi,
+			(u64)EXFAT_DATA_CLUSTER_COUNT(sbi));
+	u64 num_clusters;
+
+	/* AllocationPossible is required for Vendor Allocation entries. */
+	if (!(flags & ALLOC_POSSIBLE))
+		return false;
+
+	/* The null GUID does not identify a valid vendor allocation. */
+	if (!memchr_inv(ep->dentry.vendor_alloc.vendor_guid, 0,
+			sizeof(ep->dentry.vendor_alloc.vendor_guid)))
+		return false;
+
+	if (!start_clu)
+		return !size && !(flags & (ALLOC_NO_FAT_CHAIN ^ ALLOC_FAT_CHAIN));
+
+	if (!is_valid_cluster(sbi, start_clu) || size > max_size)
+		return false;
+
+	if ((flags & ALLOC_NO_FAT_CHAIN) == ALLOC_NO_FAT_CHAIN) {
+		if (!size)
+			return false;
+
+		num_clusters = DIV_ROUND_UP_ULL(size, sbi->cluster_size);
+		if (num_clusters > sbi->num_clusters - start_clu)
+			return false;
+	}
+
+	return true;
+}
+
+static bool exfat_validate_entry(struct super_block *sb,
+		struct exfat_dentry *ep, enum exfat_validate_dentry_mode *mode)
 {
+	unsigned int type = exfat_get_entry_type(ep);
+
 	if (type == TYPE_UNUSED || type == TYPE_DELETED)
 		return false;
+	if (type == TYPE_VENDOR_ALLOC &&
+	    !exfat_validate_vendor_alloc(sb, ep))
+		return false;
 
 	switch (*mode) {
 	case ES_MODE_GET_FILE_ENTRY:
@@ -836,7 +879,7 @@ int exfat_get_dentry_set(struct exfat_entry_set_cache *es,
 	/* validate cached dentries */
 	for (i = ES_IDX_STREAM; i < es->num_entries; i++) {
 		ep = exfat_get_dentry_cached(es, i);
-		if (!exfat_validate_entry(exfat_get_entry_type(ep), &mode))
+		if (!exfat_validate_entry(sb, ep, &mode))
 			goto put_es;
 	}
 	return 0;
@@ -1266,7 +1309,8 @@ static int exfat_get_volume_label_dentry(struct super_block *sb,
 			es->bh = es->__bh;
 			es->bh[0] = bh;
 			es->num_bh = 1;
-			es->start_off = exfat_dentries_to_bytes(i) % sb->s_blocksize;
+			es->start_off = exfat_dentries_to_bytes(i) &
+					((u32)sb->s_blocksize - 1);
 
 			return 0;
 		}
diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h
index a9131fe03302e5..5f258e96fce90d 100644
--- a/fs/exfat/exfat_fs.h
+++ b/fs/exfat/exfat_fs.h
@@ -232,6 +232,7 @@ struct exfat_sb_info {
 	unsigned int num_FAT_sectors; /* num of FAT sectors */
 	unsigned int root_dir; /* root dir cluster */
 	unsigned int dentries_per_clu; /* num of dentries per cluster */
+	unsigned int dentries_per_clu_bits;
 	unsigned int vol_flags; /* volume flags */
 	unsigned int vol_flags_persistent; /* volume flags to retain */
 	struct buffer_head *boot_bh; /* buffer_head of BOOT sector */
@@ -555,9 +556,9 @@ int exfat_trim_fs(struct inode *inode, struct fstrim_range *range);
 /* file.c */
 extern const struct file_operations exfat_file_operations;
 int __exfat_truncate(struct inode *inode);
-int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int exfat_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr);
-int exfat_getattr(struct mnt_idmap *idmap, const struct path *path,
+int exfat_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, unsigned int request_mask,
 		  unsigned int query_flags);
 struct file_kattr;
diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c
index a8b11e2ce43f1c..3c8bdc131f6f3e 100644
--- a/fs/exfat/fatent.c
+++ b/fs/exfat/fatent.c
@@ -427,19 +427,22 @@ int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc,
 	struct super_block *sb = inode->i_sb;
 	struct exfat_sb_info *sbi = EXFAT_SB(sb);
 
+	mutex_lock(&sbi->bitmap_lock);
+
 	total_cnt = EXFAT_DATA_CLUSTER_COUNT(sbi);
 
 	if (unlikely(total_cnt < sbi->used_clusters)) {
 		exfat_fs_error_ratelimit(sb,
 			"%s: invalid used clusters(t:%u,u:%u)\n",
 			__func__, total_cnt, sbi->used_clusters);
-		return -EIO;
+		ret = -EIO;
+		goto unlock;
 	}
 
-	if (num_alloc > total_cnt - sbi->used_clusters)
-		return -ENOSPC;
-
-	mutex_lock(&sbi->bitmap_lock);
+	if (num_alloc > total_cnt - sbi->used_clusters) {
+		ret = -ENOSPC;
+		goto unlock;
+	}
 
 	hint_clu = p_chain->dir;
 	/* find new cluster */
@@ -509,15 +512,15 @@ int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc,
 			}
 		}
 		p_chain->size++;
+		sbi->used_clusters++;
 
 		last_clu = new_clu;
 
 		if (p_chain->size == num_alloc) {
 done:
 			sbi->clu_srch_ptr = hint_clu;
-			sbi->used_clusters += p_chain->size;
-			mutex_unlock(&sbi->bitmap_lock);
-			return 0;
+			ret = 0;
+			goto unlock;
 		}
 
 		hint_clu = new_clu + 1;
diff --git a/fs/exfat/file.c b/fs/exfat/file.c
index a2a9ee1a20048e..3867e78c23125b 100644
--- a/fs/exfat/file.c
+++ b/fs/exfat/file.c
@@ -143,7 +143,7 @@ static long exfat_fallocate(struct file *file, int mode,
 	return err;
 }
 
-static bool exfat_allow_set_time(struct mnt_idmap *idmap,
+static bool exfat_allow_set_time(const struct mnt_idmap *idmap,
 				 struct exfat_sb_info *sbi, struct inode *inode)
 {
 	mode_t allow_utime = sbi->options.allow_utime;
@@ -319,7 +319,7 @@ static void exfat_truncate(struct inode *inode)
 	mutex_unlock(&sbi->s_lock);
 }
 
-int exfat_getattr(struct mnt_idmap *idmap, const struct path *path,
+int exfat_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, unsigned int request_mask,
 		  unsigned int query_flags)
 {
@@ -347,7 +347,7 @@ int exfat_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int exfat_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr)
 {
 	struct exfat_sb_info *sbi = EXFAT_SB(dentry->d_sb);
diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c
index 8911aa84a730ea..533cdb4f0929b8 100644
--- a/fs/exfat/iomap.c
+++ b/fs/exfat/iomap.c
@@ -157,6 +157,27 @@ const struct iomap_ops exfat_iomap_ops = {
 	.iomap_next = exfat_iomap_next,
 };
 
+#ifdef CONFIG_SWAP
+static int exfat_swap_iomap_begin(struct inode *inode, loff_t offset,
+		loff_t length, unsigned int flags, struct iomap *iomap,
+		struct iomap *srcmap)
+{
+	/*
+	 * Swap activation needs the physical mappings of preallocated
+	 * ranges. Do not report the VDL tail as a hole.
+	 */
+	return __exfat_iomap_begin(inode, offset, length,
+			flags & ~IOMAP_REPORT, iomap, false);
+}
+
+static DEFINE_IOMAP_ITER_NEXT(exfat_swap_iomap_next,
+		exfat_swap_iomap_begin);
+
+static const struct iomap_ops exfat_swap_iomap_ops = {
+	.iomap_next = exfat_swap_iomap_next,
+};
+#endif
+
 /*
  * exfat_write_iomap_end - Update the state after write
  *
@@ -275,5 +296,10 @@ const struct iomap_read_ops exfat_iomap_bio_read_ops = {
 int exfat_iomap_swap_activate(struct swap_info_struct *sis,
 			       struct file *file, sector_t *span)
 {
-	return iomap_swapfile_activate(sis, file, span, &exfat_iomap_ops);
+#ifdef CONFIG_SWAP
+	return iomap_swapfile_activate(sis, file, span,
+				       &exfat_swap_iomap_ops);
+#else
+	return -EIO;
+#endif
 }
diff --git a/fs/exfat/misc.c b/fs/exfat/misc.c
index 6f11a96a4ffa84..dfd0bbf31c940a 100644
--- a/fs/exfat/misc.c
+++ b/fs/exfat/misc.c
@@ -187,7 +187,7 @@ int exfat_update_bhs(struct buffer_head **bhs, int nr_bhs, int sync)
 
 	for (i = 0; i < nr_bhs && sync; i++) {
 		wait_on_buffer(bhs[i]);
-		if (!err && !buffer_uptodate(bhs[i]))
+		if (!err && buffer_write_io_error(bhs[i]))
 			err = -EIO;
 	}
 	return err;
diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c
index a4dc83b5949c46..d116c89d724eb2 100644
--- a/fs/exfat/namei.c
+++ b/fs/exfat/namei.c
@@ -386,7 +386,7 @@ int exfat_find_empty_entry(struct inode *inode,
 	}
 
 	p_dir->dir = exfat_sector_to_cluster(sbi, es->bh[0]->b_blocknr);
-	p_dir->size -= dentry / sbi->dentries_per_clu;
+	p_dir->size -= dentry >> sbi->dentries_per_clu_bits;
 
 	return dentry & (sbi->dentries_per_clu - 1);
 }
@@ -552,7 +552,7 @@ static int exfat_add_entry(struct inode *inode, const char *path,
 	return ret;
 }
 
-static int exfat_create(struct mnt_idmap *idmap, struct inode *dir,
+static int exfat_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -638,14 +638,15 @@ static int exfat_find(struct inode *dir, const struct qstr *qname,
 	/* adjust cdir to the optimized value */
 	cdir.dir = hint_opt.clu;
 	if (cdir.flags & ALLOC_NO_FAT_CHAIN)
-		cdir.size -= dentry / sbi->dentries_per_clu;
+		cdir.size -= dentry >> sbi->dentries_per_clu_bits;
 	dentry = hint_opt.eidx;
 
 	info->dir = cdir;
 	info->entry = dentry;
 	info->num_subdirs = 0;
 
-	if (exfat_get_dentry_set(&es, sb, &cdir, dentry, ES_2_ENTRIES))
+	/* Validate the complete set, including recognized benign entries. */
+	if (exfat_get_dentry_set(&es, sb, &cdir, dentry, ES_ALL_ENTRIES))
 		return -EIO;
 	ep = exfat_get_dentry_cached(&es, ES_IDX_FILE);
 	ep2 = exfat_get_dentry_cached(&es, ES_IDX_STREAM);
@@ -825,7 +826,7 @@ static int exfat_unlink(struct inode *dir, struct dentry *dentry)
 	return err;
 }
 
-static struct dentry *exfat_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *exfat_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -1263,7 +1264,7 @@ static int __exfat_rename(struct inode *old_parent_inode,
 	return ret;
 }
 
-static int exfat_rename(struct mnt_idmap *idmap,
+static int exfat_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
diff --git a/fs/exfat/super.c b/fs/exfat/super.c
index a9ea36ba26938a..4943cef9774181 100644
--- a/fs/exfat/super.c
+++ b/fs/exfat/super.c
@@ -18,6 +18,7 @@
 #include <linux/nls.h>
 #include <linux/buffer_head.h>
 #include <linux/magic.h>
+#include <linux/math64.h>
 
 #include "exfat_raw.h"
 #include "exfat_fs.h"
@@ -69,16 +70,28 @@ static int exfat_statfs(struct dentry *dentry, struct kstatfs *buf)
 	return 0;
 }
 
+static inline __u8 exfat_calc_perc_in_use(const struct exfat_sb_info *sbi)
+{
+	return (__u8)mul_u64_u32_div(sbi->used_clusters, 100,
+			EXFAT_DATA_CLUSTER_COUNT(sbi));
+}
+
 static int exfat_set_vol_flags(struct super_block *sb, unsigned short new_flags)
 {
 	struct exfat_sb_info *sbi = EXFAT_SB(sb);
 	struct boot_sector *p_boot = (struct boot_sector *)sbi->boot_bh->b_data;
+	__u8 new_piu;
 
 	/* retain persistent-flags */
 	new_flags |= sbi->vol_flags_persistent;
 
+	if (new_flags & VOLUME_DIRTY)
+		new_piu = 0xFF;
+	else
+		new_piu = exfat_calc_perc_in_use(sbi);
+
 	/* flags are not changed */
-	if (sbi->vol_flags == new_flags)
+	if (sbi->vol_flags == new_flags && new_piu == p_boot->percent_in_use)
 		return 0;
 
 	sbi->vol_flags = new_flags;
@@ -90,6 +103,7 @@ static int exfat_set_vol_flags(struct super_block *sb, unsigned short new_flags)
 		return 0;
 
 	p_boot->vol_flags = cpu_to_le16(new_flags);
+	p_boot->percent_in_use = new_piu;
 
 	set_buffer_uptodate(sbi->boot_bh);
 	mark_buffer_dirty(sbi->boot_bh);
@@ -505,8 +519,8 @@ static int exfat_read_boot_sector(struct super_block *sb)
 		EXFAT_RESERVED_CLUSTERS;
 
 	sbi->root_dir = le32_to_cpu(p_boot->root_cluster);
-	sbi->dentries_per_clu = 1 <<
-		(sbi->cluster_size_bits - DENTRY_SIZE_BITS);
+	sbi->dentries_per_clu_bits = sbi->cluster_size_bits - DENTRY_SIZE_BITS;
+	sbi->dentries_per_clu = 1 << sbi->dentries_per_clu_bits;
 
 	sbi->vol_flags = le16_to_cpu(p_boot->vol_flags);
 	sbi->vol_flags_persistent = sbi->vol_flags & (VOLUME_DIRTY | MEDIA_FAILURE);
@@ -775,9 +789,13 @@ static int exfat_reconfigure(struct fs_context *fc)
 	fc->sb_flags |= SB_NODIRATIME;
 
 	sync_filesystem(sb);
-	mutex_lock(&sbi->s_lock);
-	exfat_clear_volume_dirty(sb);
-	mutex_unlock(&sbi->s_lock);
+
+	if ((fc->sb_flags & (SB_FORCE | SB_RDONLY)) == SB_RDONLY &&
+	    !sb_rdonly(sb)) {
+		mutex_lock(&sbi->s_lock);
+		exfat_clear_volume_dirty(sb);
+		mutex_unlock(&sbi->s_lock);
+	}
 
 	if (new_opts->allow_utime == (unsigned short)-1)
 		new_opts->allow_utime = ~new_opts->fs_dmask & 0022;
diff --git a/fs/ext2/Makefile b/fs/ext2/Makefile
index 8860948ef9ca4e..33db2e9dc90827 100644
--- a/fs/ext2/Makefile
+++ b/fs/ext2/Makefile
@@ -3,6 +3,8 @@
 # Makefile for the linux ext2-filesystem routines.
 #
 
+CONTEXT_ANALYSIS := y
+
 obj-$(CONFIG_EXT2_FS) += ext2.o
 
 ext2-y := balloc.o dir.o file.o ialloc.o inode.o \
diff --git a/fs/ext2/acl.c b/fs/ext2/acl.c
index 7e54c31589c7ca..b2746657fc533a 100644
--- a/fs/ext2/acl.c
+++ b/fs/ext2/acl.c
@@ -219,7 +219,7 @@ __ext2_set_acl(struct inode *inode, struct posix_acl *acl, int type)
  * inode->i_mutex: down
  */
 int
-ext2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+ext2_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 	     struct posix_acl *acl, int type)
 {
 	int error;
diff --git a/fs/ext2/acl.h b/fs/ext2/acl.h
index 4a8443a2b8ec79..e68bc354560811 100644
--- a/fs/ext2/acl.h
+++ b/fs/ext2/acl.h
@@ -56,7 +56,7 @@ static inline int ext2_acl_count(size_t size)
 
 /* acl.c */
 extern struct posix_acl *ext2_get_acl(struct inode *inode, int type, bool rcu);
-extern int ext2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+extern int ext2_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			struct posix_acl *acl, int type);
 extern int ext2_init_acl (struct inode *, struct inode *);
 
diff --git a/fs/ext2/balloc.c b/fs/ext2/balloc.c
index adf0f31fbddd16..80acc1e1938710 100644
--- a/fs/ext2/balloc.c
+++ b/fs/ext2/balloc.c
@@ -334,6 +334,7 @@ search_reserve_window(struct rb_root *root, ext2_fsblk_t goal)
  */
 void ext2_rsv_window_add(struct super_block *sb,
 		    struct ext2_reserve_window_node *rsv)
+	__must_hold(&EXT2_SB(sb)->s_rsv_window_lock)
 {
 	struct rb_root *root = &EXT2_SB(sb)->s_rsv_window_root;
 	struct rb_node *node = &rsv->rsv_node;
@@ -373,6 +374,7 @@ void ext2_rsv_window_add(struct super_block *sb,
  */
 static void rsv_window_remove(struct super_block *sb,
 			      struct ext2_reserve_window_node *rsv)
+	__must_hold(&EXT2_SB(sb)->s_rsv_window_lock)
 {
 	rsv->rsv_start = EXT2_RESERVE_WINDOW_NOT_ALLOCATED;
 	rsv->rsv_end = EXT2_RESERVE_WINDOW_NOT_ALLOCATED;
@@ -414,6 +416,7 @@ static inline int rsv_is_empty(struct ext2_reserve_window *rsv)
  * Needs truncate_mutex protection prior to calling this function.
  */
 void ext2_init_block_alloc_info(struct inode *inode)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	struct ext2_inode_info *ei = EXT2_I(inode);
 	struct ext2_block_alloc_info *block_i;
@@ -758,6 +761,7 @@ static int find_next_reservable_window(
 				struct super_block * sb,
 				ext2_fsblk_t start_block,
 				ext2_fsblk_t last_block)
+	__must_hold(&EXT2_SB(sb)->s_rsv_window_lock)
 {
 	struct rb_node *next;
 	struct ext2_reserve_window_node *rsv, *prev;
diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h
index 5642451bf19173..7bdada93dd060e 100644
--- a/fs/ext2/ext2.h
+++ b/fs/ext2/ext2.h
@@ -77,8 +77,8 @@ struct ext2_sb_info {
 	unsigned long s_gdb_count;	/* Number of group descriptor blocks */
 	unsigned long s_desc_per_block;	/* Number of group descriptors per block */
 	unsigned long s_groups_count;	/* Number of groups in the fs */
-	unsigned long s_overhead_last;  /* Last calculated overhead */
-	unsigned long s_blocks_last;    /* Last seen block count */
+	unsigned long s_overhead_last __guarded_by(&s_lock);  /* Last calculated overhead */
+	unsigned long s_blocks_last __guarded_by(&s_lock);    /* Last seen block count */
 	struct buffer_head * s_sbh;	/* Buffer containing the super block */
 	struct ext2_super_block * s_es;	/* Pointer to the super block in the buffer */
 	struct buffer_head ** s_group_desc;
@@ -86,14 +86,14 @@ struct ext2_sb_info {
 	unsigned long s_sb_block;
 	kuid_t s_resuid;
 	kgid_t s_resgid;
-	unsigned short s_mount_state;
+	unsigned short s_mount_state __guarded_by(&s_lock);
 	unsigned short s_pad;
 	int s_addr_per_block_bits;
 	int s_desc_per_block_bits;
 	int s_inode_size;
 	int s_first_ino;
 	spinlock_t s_next_gen_lock;
-	u32 s_next_generation;
+	u32 s_next_generation __guarded_by(&s_next_gen_lock);
 	unsigned long s_dir_count;
 	u8 *s_debts;
 	struct percpu_counter s_freeblocks_counter;
@@ -102,7 +102,7 @@ struct ext2_sb_info {
 	struct blockgroup_lock *s_blockgroup_lock;
 	/* root of the per fs reservation window tree */
 	spinlock_t s_rsv_window_lock;
-	struct rb_root s_rsv_window_root;
+	struct rb_root s_rsv_window_root __guarded_by(&s_rsv_window_lock);
 	struct ext2_reserve_window_node s_rsv_window_head;
 	/*
 	 * s_lock protects against concurrent modifications of s_mount_state,
@@ -710,8 +710,10 @@ extern struct ext2_group_desc * ext2_get_group_desc(struct super_block * sb,
 						    struct buffer_head ** bh);
 extern void ext2_discard_reservation (struct inode *);
 extern int ext2_should_retry_alloc(struct super_block *sb, int *retries);
-extern void ext2_init_block_alloc_info(struct inode *);
-extern void ext2_rsv_window_add(struct super_block *sb, struct ext2_reserve_window_node *rsv);
+extern void ext2_init_block_alloc_info(struct inode *inode)
+	__must_hold(&EXT2_I(inode)->truncate_mutex);
+extern void ext2_rsv_window_add(struct super_block *sb, struct ext2_reserve_window_node *rsv)
+	__must_hold(&EXT2_SB(sb)->s_rsv_window_lock);
 
 /* dir.c */
 int ext2_add_link(struct dentry *, struct inode *);
@@ -739,8 +741,8 @@ extern int ext2_sync_inode_metadata(struct inode *, struct writeback_control *);
 extern void ext2_evict_inode(struct inode *);
 void ext2_write_failed(struct address_space *mapping, loff_t to);
 extern int ext2_get_block(struct inode *, sector_t, struct buffer_head *, int);
-extern int ext2_setattr (struct mnt_idmap *, struct dentry *, struct iattr *);
-extern int ext2_getattr (struct mnt_idmap *, const struct path *,
+extern int ext2_setattr (const struct mnt_idmap *, struct dentry *, struct iattr *);
+extern int ext2_getattr (const struct mnt_idmap *, const struct path *,
 			 struct kstat *, u32, unsigned int);
 extern void ext2_set_inode_flags(struct inode *inode);
 extern int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
@@ -748,7 +750,7 @@ extern int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 
 /* ioctl.c */
 extern int ext2_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-extern int ext2_fileattr_set(struct mnt_idmap *idmap,
+extern int ext2_fileattr_set(const struct mnt_idmap *idmap,
 			     struct dentry *dentry, struct file_kattr *fa);
 extern long ext2_ioctl(struct file *, unsigned int, unsigned long);
 extern long ext2_compat_ioctl(struct file *, unsigned int, unsigned long);
@@ -761,7 +763,8 @@ extern __printf(3, 4)
 void ext2_error(struct super_block *, const char *, const char *, ...);
 extern __printf(3, 4)
 void ext2_msg(struct super_block *, const char *, const char *, ...);
-extern void ext2_update_dynamic_rev (struct super_block *sb);
+extern void ext2_update_dynamic_rev(struct super_block *sb)
+	__must_hold(&EXT2_SB(sb)->s_lock);
 extern void ext2_sync_super(struct super_block *sb, struct ext2_super_block *es,
 			    int wait);
 
diff --git a/fs/ext2/file.c b/fs/ext2/file.c
index b9020df7d89e03..67fe423c3828d8 100644
--- a/fs/ext2/file.c
+++ b/fs/ext2/file.c
@@ -135,6 +135,13 @@ static ssize_t ext2_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		int ret2;
 
 		iocb->ki_flags &= ~IOCB_DIRECT;
+
+		/*
+		 * Prevent concurrent direct I/O and buffered I/O to the same file
+		 * range. Wait for in-flight DIO to finish before dirtying pages.
+		 */
+		inode_dio_wait(inode);
+
 		pos = iocb->ki_pos;
 		status = generic_perform_write(iocb, from);
 		if (unlikely(status < 0)) {
diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c
index a9245f0cda4d46..12ac4cfe150038 100644
--- a/fs/ext2/inode.c
+++ b/fs/ext2/inode.c
@@ -327,6 +327,7 @@ static ext2_fsblk_t ext2_find_near(struct inode *inode, Indirect *ind)
 
 static inline ext2_fsblk_t ext2_find_goal(struct inode *inode, long block,
 					  Indirect *partial)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	struct ext2_block_alloc_info *block_i;
 
@@ -397,6 +398,7 @@ ext2_blks_to_allocate(Indirect * branch, int k, unsigned long blks,
 static int ext2_alloc_blocks(struct inode *inode,
 			ext2_fsblk_t goal, int indirect_blks, int blks,
 			ext2_fsblk_t new_blocks[4], int *err)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	int target, i;
 	unsigned long count = 0;
@@ -477,6 +479,7 @@ static int ext2_alloc_blocks(struct inode *inode,
 static int ext2_alloc_branch(struct inode *inode,
 			int indirect_blks, int *blks, ext2_fsblk_t goal,
 			int *offsets, Indirect *branch)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	int blocksize = inode->i_sb->s_blocksize;
 	int i, n = 0;
@@ -558,6 +561,7 @@ static int ext2_alloc_branch(struct inode *inode,
  */
 static void ext2_splice_branch(struct inode *inode,
 			long block, Indirect *where, int num, int blks)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	int i;
 	struct ext2_block_alloc_info *block_i;
@@ -1002,6 +1006,7 @@ static Indirect *ext2_find_shared(struct inode *inode,
 				int offsets[4],
 				Indirect chain[4],
 				__le32 *top)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	Indirect *partial, *p;
 	int k, err;
@@ -1057,6 +1062,7 @@ static Indirect *ext2_find_shared(struct inode *inode,
  *	appropriately.
  */
 static inline void ext2_free_data(struct inode *inode, __le32 *p, __le32 *q)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	ext2_fsblk_t block_to_free = 0, count = 0;
 	ext2_fsblk_t nr;
@@ -1097,6 +1103,7 @@ static inline void ext2_free_data(struct inode *inode, __le32 *p, __le32 *q)
  *	appropriately.
  */
 static void ext2_free_branches(struct inode *inode, __le32 *p, __le32 *q, int depth)
+	__must_hold(&EXT2_I(inode)->truncate_mutex)
 {
 	struct buffer_head * bh;
 	ext2_fsblk_t nr;
@@ -1591,7 +1598,7 @@ int ext2_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc)
 	return err;
 }
 
-int ext2_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ext2_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -1617,7 +1624,7 @@ int ext2_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return 0;
 }
 
-int ext2_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ext2_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/ext2/ioctl.c b/fs/ext2/ioctl.c
index c3fea55b8efa4c..f2218455fa4771 100644
--- a/fs/ext2/ioctl.c
+++ b/fs/ext2/ioctl.c
@@ -27,7 +27,7 @@ int ext2_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int ext2_fileattr_set(struct mnt_idmap *idmap,
+int ext2_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c
index 8666233ec63bc7..bfb6a463a95e06 100644
--- a/fs/ext2/namei.c
+++ b/fs/ext2/namei.c
@@ -97,7 +97,7 @@ struct dentry *ext2_get_parent(struct dentry *child)
  * If the create succeeds, we fill in the inode information
  * with d_instantiate(). 
  */
-static int ext2_create (struct mnt_idmap * idmap,
+static int ext2_create (const struct mnt_idmap * idmap,
 			struct inode * dir, struct dentry * dentry,
 			umode_t mode)
 {
@@ -117,7 +117,7 @@ static int ext2_create (struct mnt_idmap * idmap,
 	return ext2_add_nondir(dentry, inode);
 }
 
-static int ext2_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int ext2_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			struct file *file, umode_t mode)
 {
 	struct inode *inode = ext2_new_inode(dir, mode, NULL);
@@ -131,7 +131,7 @@ static int ext2_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	return finish_open_simple(file, 0);
 }
 
-static int ext2_mknod (struct mnt_idmap * idmap, struct inode * dir,
+static int ext2_mknod (const struct mnt_idmap * idmap, struct inode * dir,
 	struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct inode * inode;
@@ -152,7 +152,7 @@ static int ext2_mknod (struct mnt_idmap * idmap, struct inode * dir,
 	return err;
 }
 
-static int ext2_symlink (struct mnt_idmap * idmap, struct inode * dir,
+static int ext2_symlink (const struct mnt_idmap * idmap, struct inode * dir,
 	struct dentry * dentry, const char * symname)
 {
 	struct super_block * sb = dir->i_sb;
@@ -223,7 +223,7 @@ static int ext2_link (struct dentry * old_dentry, struct inode * dir,
 	return err;
 }
 
-static struct dentry *ext2_mkdir(struct mnt_idmap * idmap,
+static struct dentry *ext2_mkdir(const struct mnt_idmap * idmap,
 				 struct inode * dir, struct dentry * dentry,
 				 umode_t mode)
 {
@@ -316,7 +316,7 @@ static int ext2_rmdir (struct inode * dir, struct dentry *dentry)
 	return err;
 }
 
-static int ext2_rename (struct mnt_idmap * idmap,
+static int ext2_rename (const struct mnt_idmap * idmap,
 			struct inode * old_dir, struct dentry * old_dentry,
 			struct inode * new_dir, struct dentry * new_dentry,
 			unsigned int flags)
diff --git a/fs/ext2/super.c b/fs/ext2/super.c
index a40f530872a407..fb217ef2d69f07 100644
--- a/fs/ext2/super.c
+++ b/fs/ext2/super.c
@@ -127,6 +127,7 @@ void ext2_msg(struct super_block *sb, const char *prefix,
  * This must be called with sbi->s_lock held.
  */
 void ext2_update_dynamic_rev(struct super_block *sb)
+	__must_hold(&EXT2_SB(sb)->s_lock)
 {
 	struct ext2_super_block *es = EXT2_SB(sb)->s_es;
 
@@ -633,6 +634,7 @@ static int ext2_parse_param(struct fs_context *fc, struct fs_parameter *param)
 static int ext2_setup_super (struct super_block * sb,
 			      struct ext2_super_block * es,
 			      int read_only)
+	__must_hold(&EXT2_SB(sb)->s_lock)
 {
 	int res = 0;
 	struct ext2_sb_info *sbi = EXT2_SB(sb);
@@ -894,7 +896,7 @@ static int ext2_fill_super(struct super_block *sb, struct fs_context *fc)
 	sb->s_fs_info = sbi;
 	sbi->s_sb_block = sb_block;
 
-	spin_lock_init(&sbi->s_lock);
+	guard(spinlock_init)(&EXT2_SB(sb)->s_lock);
 	ret = -EINVAL;
 
 	/*
@@ -1126,23 +1128,26 @@ static int ext2_fill_super(struct super_block *sb, struct fs_context *fc)
 		goto failed_mount2;
 	}
 	sbi->s_gdb_count = db_count;
-	sbi->s_next_generation = get_random_u32();
 	spin_lock_init(&sbi->s_next_gen_lock);
+	scoped_guard(spinlock, &sbi->s_next_gen_lock)
+		sbi->s_next_generation = get_random_u32();
 
 	/* per filesystem reservation list head & lock */
 	spin_lock_init(&sbi->s_rsv_window_lock);
-	sbi->s_rsv_window_root = RB_ROOT;
-	/*
-	 * Add a single, static dummy reservation to the start of the
-	 * reservation window list --- it gives us a placeholder for
-	 * append-at-start-of-list which makes the allocation logic
-	 * _much_ simpler.
-	 */
-	sbi->s_rsv_window_head.rsv_start = EXT2_RESERVE_WINDOW_NOT_ALLOCATED;
-	sbi->s_rsv_window_head.rsv_end = EXT2_RESERVE_WINDOW_NOT_ALLOCATED;
-	sbi->s_rsv_window_head.rsv_alloc_hit = 0;
-	sbi->s_rsv_window_head.rsv_goal_size = 0;
-	ext2_rsv_window_add(sb, &sbi->s_rsv_window_head);
+	scoped_guard(spinlock, &EXT2_SB(sb)->s_rsv_window_lock) {
+		sbi->s_rsv_window_root = RB_ROOT;
+		/*
+		 * Add a single, static dummy reservation to the start of the
+		 * reservation window list --- it gives us a placeholder for
+		 * append-at-start-of-list which makes the allocation logic
+		 * _much_ simpler.
+		 */
+		sbi->s_rsv_window_head.rsv_start = EXT2_RESERVE_WINDOW_NOT_ALLOCATED;
+		sbi->s_rsv_window_head.rsv_end = EXT2_RESERVE_WINDOW_NOT_ALLOCATED;
+		sbi->s_rsv_window_head.rsv_alloc_hit = 0;
+		sbi->s_rsv_window_head.rsv_goal_size = 0;
+		ext2_rsv_window_add(sb, &sbi->s_rsv_window_head);
+	}
 
 	err = percpu_counter_init(&sbi->s_freeblocks_counter,
 				ext2_count_free_blocks(sb), GFP_KERNEL);
diff --git a/fs/ext2/xattr.c b/fs/ext2/xattr.c
index 9b68c490ab26de..8f608930a48c64 100644
--- a/fs/ext2/xattr.c
+++ b/fs/ext2/xattr.c
@@ -769,7 +769,7 @@ ext2_xattr_set2(struct inode *inode, struct buffer_head *old_bh,
 		if (IS_SYNC(inode)) {
 			sync_dirty_buffer(new_bh);
 			error = -EIO;
-			if (buffer_req(new_bh) && !buffer_uptodate(new_bh))
+			if (buffer_write_io_error(new_bh))
 				goto cleanup;
 		}
 	}
diff --git a/fs/ext2/xattr_security.c b/fs/ext2/xattr_security.c
index db47b8ab153e7d..ade074354258ea 100644
--- a/fs/ext2/xattr_security.c
+++ b/fs/ext2/xattr_security.c
@@ -19,7 +19,7 @@ ext2_xattr_security_get(const struct xattr_handler *handler,
 
 static int
 ext2_xattr_security_set(const struct xattr_handler *handler,
-			struct mnt_idmap *idmap,
+			const struct mnt_idmap *idmap,
 			struct dentry *unused, struct inode *inode,
 			const char *name, const void *value,
 			size_t size, int flags)
diff --git a/fs/ext2/xattr_trusted.c b/fs/ext2/xattr_trusted.c
index 995f931228ce8b..0f12d634d6d08a 100644
--- a/fs/ext2/xattr_trusted.c
+++ b/fs/ext2/xattr_trusted.c
@@ -26,7 +26,7 @@ ext2_xattr_trusted_get(const struct xattr_handler *handler,
 
 static int
 ext2_xattr_trusted_set(const struct xattr_handler *handler,
-		       struct mnt_idmap *idmap,
+		       const struct mnt_idmap *idmap,
 		       struct dentry *unused, struct inode *inode,
 		       const char *name, const void *value,
 		       size_t size, int flags)
diff --git a/fs/ext2/xattr_user.c b/fs/ext2/xattr_user.c
index dd15072310818b..48002c033e9c92 100644
--- a/fs/ext2/xattr_user.c
+++ b/fs/ext2/xattr_user.c
@@ -30,7 +30,7 @@ ext2_xattr_user_get(const struct xattr_handler *handler,
 
 static int
 ext2_xattr_user_set(const struct xattr_handler *handler,
-		    struct mnt_idmap *idmap,
+		    const struct mnt_idmap *idmap,
 		    struct dentry *unused, struct inode *inode,
 		    const char *name, const void *value,
 		    size_t size, int flags)
diff --git a/fs/ext4/acl.c b/fs/ext4/acl.c
index 3bffe862f95499..59fac55a242610 100644
--- a/fs/ext4/acl.c
+++ b/fs/ext4/acl.c
@@ -225,7 +225,7 @@ __ext4_set_acl(handle_t *handle, struct inode *inode, int type,
 }
 
 int
-ext4_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+ext4_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 	     struct posix_acl *acl, int type)
 {
 	handle_t *handle;
diff --git a/fs/ext4/acl.h b/fs/ext4/acl.h
index 0c5a79c3b5d480..a14838c5bc42d8 100644
--- a/fs/ext4/acl.h
+++ b/fs/ext4/acl.h
@@ -56,7 +56,7 @@ static inline int ext4_acl_count(size_t size)
 
 /* acl.c */
 struct posix_acl *ext4_get_acl(struct inode *inode, int type, bool rcu);
-int ext4_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ext4_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
 extern int ext4_init_acl(handle_t *, struct inode *, struct inode *);
 
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index 724a27e8be613e..cbc59d03ca8159 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -3044,7 +3044,7 @@ extern int ext4fs_dirhash(const struct inode *dir, const char *name, int len,
 
 /* ialloc.c */
 extern int ext4_mark_inode_used(struct super_block *sb, int ino);
-extern struct inode *__ext4_new_inode(struct mnt_idmap *, handle_t *,
+extern struct inode *__ext4_new_inode(const struct mnt_idmap *, handle_t *,
 				      struct inode *, umode_t,
 				      const struct qstr *qstr, __u32 goal,
 				      uid_t *owner, __u32 i_flags,
@@ -3179,14 +3179,14 @@ extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
 
 extern int  ext4_write_inode(struct inode *, struct writeback_control *);
 extern int  ext4_sync_inode_metadata(struct inode *, struct writeback_control *);
-extern int  ext4_setattr(struct mnt_idmap *, struct dentry *,
+extern int  ext4_setattr(const struct mnt_idmap *, struct dentry *,
 			 struct iattr *);
 extern u32  ext4_dio_alignment(struct inode *inode);
-extern int  ext4_getattr(struct mnt_idmap *, const struct path *,
+extern int  ext4_getattr(const struct mnt_idmap *, const struct path *,
 			 struct kstat *, u32, unsigned int);
 extern void ext4_evict_inode(struct inode *);
 extern void ext4_clear_inode(struct inode *);
-extern int  ext4_file_getattr(struct mnt_idmap *, const struct path *,
+extern int  ext4_file_getattr(const struct mnt_idmap *, const struct path *,
 			      struct kstat *, u32, unsigned int);
 extern void ext4_dirty_inode(struct inode *, int);
 extern int ext4_change_inode_journal_flag(struct inode *, int);
@@ -3246,7 +3246,7 @@ extern int ext4_ind_remove_space(handle_t *handle, struct inode *inode,
 /* ioctl.c */
 extern long ext4_ioctl(struct file *, unsigned int, unsigned long);
 extern long ext4_compat_ioctl(struct file *, unsigned int, unsigned long);
-int ext4_fileattr_set(struct mnt_idmap *idmap,
+int ext4_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa);
 int ext4_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
 extern void ext4_reset_inode_seed(struct inode *inode);
diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c
index 53ddedb52a6f73..c241f50b97bc0a 100644
--- a/fs/ext4/ext4_jbd2.c
+++ b/fs/ext4/ext4_jbd2.c
@@ -421,7 +421,7 @@ int __ext4_handle_dirty_metadata(const char *where, unsigned int line,
 		}
 		if (inode && inode_needs_sync(inode)) {
 			sync_dirty_buffer(bh);
-			if (buffer_req(bh) && !buffer_uptodate(bh)) {
+			if (buffer_write_io_error(bh)) {
 				ext4_error_inode_err(inode, where, line,
 						     bh->b_blocknr, EIO,
 					"IO error syncing itable block");
diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c
index a5831fc536db90..529623103ae7ac 100644
--- a/fs/ext4/ialloc.c
+++ b/fs/ext4/ialloc.c
@@ -930,7 +930,7 @@ static int ext4_xattr_credits_for_new_inode(struct inode *dir, mode_t mode,
  * For other inodes, search forward from the parent directory's block
  * group to find a free inode.
  */
-struct inode *__ext4_new_inode(struct mnt_idmap *idmap,
+struct inode *__ext4_new_inode(const struct mnt_idmap *idmap,
 			       handle_t *handle, struct inode *dir,
 			       umode_t mode, const struct qstr *qstr,
 			       __u32 goal, uid_t *owner, __u32 i_flags,
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 26f0f9714f03cf..cb68bf50a3d6a0 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -6006,7 +6006,7 @@ static void ext4_wait_for_tail_page_commit(struct inode *inode)
  *
  * Called with inode->i_rwsem down.
  */
-int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ext4_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -6263,7 +6263,7 @@ u32 ext4_dio_alignment(struct inode *inode)
 	return 1; /* use the iomap defaults */
 }
 
-int ext4_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ext4_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -6332,7 +6332,7 @@ int ext4_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return 0;
 }
 
-int ext4_file_getattr(struct mnt_idmap *idmap,
+int ext4_file_getattr(const struct mnt_idmap *idmap,
 		      const struct path *path, struct kstat *stat,
 		      u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c
index c8387e6a2c6e90..0a54b00e5be5a5 100644
--- a/fs/ext4/ioctl.c
+++ b/fs/ext4/ioctl.c
@@ -373,7 +373,7 @@ void ext4_reset_inode_seed(struct inode *inode)
  *
  */
 static long swap_inode_boot_loader(struct super_block *sb,
-				struct mnt_idmap *idmap,
+				const struct mnt_idmap *idmap,
 				struct inode *inode)
 {
 	handle_t *handle;
@@ -1008,7 +1008,7 @@ int ext4_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int ext4_fileattr_set(struct mnt_idmap *idmap,
+int ext4_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
@@ -1539,7 +1539,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 {
 	struct inode *inode = file_inode(filp);
 	struct super_block *sb = inode->i_sb;
-	struct mnt_idmap *idmap = file_mnt_idmap(filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(filp);
 
 	ext4_debug("cmd = %u, arg = %lu\n", cmd, arg);
 
diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c
index 7ce361484b3821..4b18ddef468d71 100644
--- a/fs/ext4/mmp.c
+++ b/fs/ext4/mmp.c
@@ -49,7 +49,7 @@ static int write_mmp_block_thawed(struct super_block *sb,
 	bh_submit(bh, REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO,
 			bh_end_write);
 	wait_on_buffer(bh);
-	if (unlikely(!buffer_uptodate(bh)))
+	if (unlikely(buffer_write_io_error(bh)))
 		return -EIO;
 	return 0;
 }
diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
index a6386c1d237f1a..6e0630a49e4863 100644
--- a/fs/ext4/namei.c
+++ b/fs/ext4/namei.c
@@ -2812,7 +2812,7 @@ static int ext4_add_nondir(handle_t *handle,
  * If the create succeeds, we fill in the inode information
  * with d_instantiate().
  */
-static int ext4_create(struct mnt_idmap *idmap, struct inode *dir,
+static int ext4_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	handle_t *handle;
@@ -2847,7 +2847,7 @@ static int ext4_create(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int ext4_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int ext4_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	handle_t *handle;
@@ -2881,7 +2881,7 @@ static int ext4_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int ext4_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int ext4_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			struct file *file, umode_t mode)
 {
 	handle_t *handle;
@@ -2994,7 +2994,7 @@ int ext4_init_new_dir(handle_t *handle, struct inode *dir,
 	return err;
 }
 
-static struct dentry *ext4_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ext4_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	handle_t *handle;
@@ -3360,7 +3360,7 @@ static int ext4_init_symlink_block(handle_t *handle, struct inode *inode,
 	return err;
 }
 
-static int ext4_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int ext4_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, const char *symname)
 {
 	handle_t *handle;
@@ -3753,7 +3753,7 @@ static void ext4_update_dir_count(handle_t *handle, struct ext4_renament *ent)
 	}
 }
 
-static struct inode *ext4_whiteout_for_rename(struct mnt_idmap *idmap,
+static struct inode *ext4_whiteout_for_rename(const struct mnt_idmap *idmap,
 					      struct ext4_renament *ent,
 					      int credits, handle_t **h)
 {
@@ -3796,7 +3796,7 @@ static struct inode *ext4_whiteout_for_rename(struct mnt_idmap *idmap,
  * while new_{dentry,inode) refers to the destination dentry/inode
  * This comes from rename(const char *oldpath, const char *newpath)
  */
-static int ext4_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int ext4_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		       struct dentry *old_dentry, struct inode *new_dir,
 		       struct dentry *new_dentry, unsigned int flags)
 {
@@ -4191,7 +4191,7 @@ static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 	return retval;
 }
 
-static int ext4_rename2(struct mnt_idmap *idmap,
+static int ext4_rename2(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
diff --git a/fs/ext4/symlink.c b/fs/ext4/symlink.c
index b612262719ede3..e680d1e45b4775 100644
--- a/fs/ext4/symlink.c
+++ b/fs/ext4/symlink.c
@@ -55,7 +55,7 @@ static const char *ext4_encrypted_get_link(struct dentry *dentry,
 	return paddr;
 }
 
-static int ext4_encrypted_symlink_getattr(struct mnt_idmap *idmap,
+static int ext4_encrypted_symlink_getattr(const struct mnt_idmap *idmap,
 					  const struct path *path,
 					  struct kstat *stat, u32 request_mask,
 					  unsigned int query_flags)
diff --git a/fs/ext4/xattr_hurd.c b/fs/ext4/xattr_hurd.c
index 8a5842e4cd95c6..a3ecbff72b1015 100644
--- a/fs/ext4/xattr_hurd.c
+++ b/fs/ext4/xattr_hurd.c
@@ -32,7 +32,7 @@ ext4_xattr_hurd_get(const struct xattr_handler *handler,
 
 static int
 ext4_xattr_hurd_set(const struct xattr_handler *handler,
-		    struct mnt_idmap *idmap,
+		    const struct mnt_idmap *idmap,
 		    struct dentry *unused, struct inode *inode,
 		    const char *name, const void *value,
 		    size_t size, int flags)
diff --git a/fs/ext4/xattr_security.c b/fs/ext4/xattr_security.c
index 776cf11d24ca10..af5b8a93fed13a 100644
--- a/fs/ext4/xattr_security.c
+++ b/fs/ext4/xattr_security.c
@@ -23,7 +23,7 @@ ext4_xattr_security_get(const struct xattr_handler *handler,
 
 static int
 ext4_xattr_security_set(const struct xattr_handler *handler,
-			struct mnt_idmap *idmap,
+			const struct mnt_idmap *idmap,
 			struct dentry *unused, struct inode *inode,
 			const char *name, const void *value,
 			size_t size, int flags)
diff --git a/fs/ext4/xattr_trusted.c b/fs/ext4/xattr_trusted.c
index 9811eb0ab27623..458e1982ef83a3 100644
--- a/fs/ext4/xattr_trusted.c
+++ b/fs/ext4/xattr_trusted.c
@@ -30,7 +30,7 @@ ext4_xattr_trusted_get(const struct xattr_handler *handler,
 
 static int
 ext4_xattr_trusted_set(const struct xattr_handler *handler,
-		       struct mnt_idmap *idmap,
+		       const struct mnt_idmap *idmap,
 		       struct dentry *unused, struct inode *inode,
 		       const char *name, const void *value,
 		       size_t size, int flags)
diff --git a/fs/ext4/xattr_user.c b/fs/ext4/xattr_user.c
index 4b70bf4e762633..ad35215f66104c 100644
--- a/fs/ext4/xattr_user.c
+++ b/fs/ext4/xattr_user.c
@@ -31,7 +31,7 @@ ext4_xattr_user_get(const struct xattr_handler *handler,
 
 static int
 ext4_xattr_user_set(const struct xattr_handler *handler,
-		    struct mnt_idmap *idmap,
+		    const struct mnt_idmap *idmap,
 		    struct dentry *unused, struct inode *inode,
 		    const char *name, const void *value,
 		    size_t size, int flags)
diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile
index 8a7322d229e4d2..fbf49c30b06646 100644
--- a/fs/f2fs/Makefile
+++ b/fs/f2fs/Makefile
@@ -3,7 +3,7 @@ obj-$(CONFIG_F2FS_FS) += f2fs.o
 
 f2fs-y		:= dir.o file.o inode.o namei.o hash.o super.o inline.o
 f2fs-y		+= checkpoint.o gc.o data.o node.o segment.o recovery.o
-f2fs-y		+= shrinker.o extent_cache.o sysfs.o
+f2fs-y		+= shrinker.o extent_cache.o sysfs.o cache.o
 f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o
 f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o
 f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o
diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c
index d3253549173e68..a7485bc382526f 100644
--- a/fs/f2fs/acl.c
+++ b/fs/f2fs/acl.c
@@ -181,7 +181,7 @@ static void *f2fs_acl_to_disk(struct f2fs_sb_info *sbi,
 }
 
 static struct posix_acl *__f2fs_get_acl(struct inode *inode, int type,
-						struct folio *dfolio)
+						struct f2fs_cached_block *entry)
 {
 	int name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT;
 	void *value = NULL;
@@ -191,13 +191,13 @@ static struct posix_acl *__f2fs_get_acl(struct inode *inode, int type,
 	if (type == ACL_TYPE_ACCESS)
 		name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS;
 
-	retval = f2fs_getxattr(inode, name_index, "", NULL, 0, dfolio);
+	retval = f2fs_getxattr(inode, name_index, "", NULL, 0, entry);
 	if (retval > 0) {
 		value = f2fs_kmalloc(F2FS_I_SB(inode), retval, GFP_F2FS_ZERO);
 		if (!value)
 			return ERR_PTR(-ENOMEM);
 		retval = f2fs_getxattr(inode, name_index, "", value,
-							retval, dfolio);
+							retval, entry);
 	}
 
 	if (retval > 0)
@@ -219,7 +219,7 @@ struct posix_acl *f2fs_get_acl(struct inode *inode, int type, bool rcu)
 	return __f2fs_get_acl(inode, type, NULL);
 }
 
-static int f2fs_acl_update_mode(struct mnt_idmap *idmap,
+static int f2fs_acl_update_mode(const struct mnt_idmap *idmap,
 				struct inode *inode, umode_t *mode_p,
 				struct posix_acl **acl)
 {
@@ -240,9 +240,9 @@ static int f2fs_acl_update_mode(struct mnt_idmap *idmap,
 	return 0;
 }
 
-static int __f2fs_set_acl(struct mnt_idmap *idmap,
+static int __f2fs_set_acl(const struct mnt_idmap *idmap,
 			struct inode *inode, int type,
-			struct posix_acl *acl, struct folio *ifolio)
+			struct posix_acl *acl, struct f2fs_cached_block *ientry)
 {
 	int name_index;
 	void *value = NULL;
@@ -253,7 +253,7 @@ static int __f2fs_set_acl(struct mnt_idmap *idmap,
 	switch (type) {
 	case ACL_TYPE_ACCESS:
 		name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS;
-		if (acl && !ifolio) {
+		if (acl && !ientry) {
 			error = f2fs_acl_update_mode(idmap, inode, &mode, &acl);
 			if (error)
 				return error;
@@ -279,7 +279,7 @@ static int __f2fs_set_acl(struct mnt_idmap *idmap,
 		}
 	}
 
-	error = f2fs_setxattr(inode, name_index, "", value, size, ifolio, 0);
+	error = f2fs_setxattr(inode, name_index, "", value, size, ientry, 0);
 
 	kfree(value);
 	if (!error)
@@ -289,7 +289,7 @@ static int __f2fs_set_acl(struct mnt_idmap *idmap,
 	return error;
 }
 
-int f2fs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int f2fs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 	struct inode *inode = d_inode(dentry);
@@ -374,7 +374,7 @@ static int f2fs_acl_create_masq(struct posix_acl *acl, umode_t *mode_p)
 
 static int f2fs_acl_create(struct inode *dir, umode_t *mode,
 		struct posix_acl **default_acl, struct posix_acl **acl,
-		struct folio *dfolio)
+		struct f2fs_cached_block *entry)
 {
 	struct posix_acl *p;
 	struct posix_acl *clone;
@@ -386,7 +386,7 @@ static int f2fs_acl_create(struct inode *dir, umode_t *mode,
 	if (S_ISLNK(*mode) || !IS_POSIXACL(dir))
 		return 0;
 
-	p = __f2fs_get_acl(dir, ACL_TYPE_DEFAULT, dfolio);
+	p = __f2fs_get_acl(dir, ACL_TYPE_DEFAULT, entry);
 	if (!p || p == ERR_PTR(-EOPNOTSUPP)) {
 		*mode &= ~current_umask();
 		return 0;
@@ -423,13 +423,13 @@ static int f2fs_acl_create(struct inode *dir, umode_t *mode,
 	return ret;
 }
 
-int f2fs_init_acl(struct inode *inode, struct inode *dir, struct folio *ifolio,
-							struct folio *dfolio)
+int f2fs_init_acl(struct inode *inode, struct inode *dir, struct f2fs_cached_block *ientry,
+							struct f2fs_cached_block *dentry)
 {
 	struct posix_acl *default_acl = NULL, *acl = NULL;
 	int error;
 
-	error = f2fs_acl_create(dir, &inode->i_mode, &default_acl, &acl, dfolio);
+	error = f2fs_acl_create(dir, &inode->i_mode, &default_acl, &acl, dentry);
 	if (error)
 		return error;
 
@@ -437,7 +437,7 @@ int f2fs_init_acl(struct inode *inode, struct inode *dir, struct folio *ifolio,
 
 	if (default_acl) {
 		error = __f2fs_set_acl(NULL, inode, ACL_TYPE_DEFAULT,
-				default_acl, ifolio);
+				default_acl, ientry);
 		posix_acl_release(default_acl);
 	} else {
 		inode->i_default_acl = NULL;
@@ -445,7 +445,7 @@ int f2fs_init_acl(struct inode *inode, struct inode *dir, struct folio *ifolio,
 	if (acl) {
 		if (!error)
 			error = __f2fs_set_acl(NULL, inode, ACL_TYPE_ACCESS,
-					acl, ifolio);
+					acl, ientry);
 		posix_acl_release(acl);
 	} else {
 		inode->i_acl = NULL;
diff --git a/fs/f2fs/acl.h b/fs/f2fs/acl.h
index 20e87e63c08958..b1085efcc05e9c 100644
--- a/fs/f2fs/acl.h
+++ b/fs/f2fs/acl.h
@@ -34,16 +34,18 @@ struct f2fs_acl_header {
 #ifdef CONFIG_F2FS_FS_POSIX_ACL
 
 struct posix_acl *f2fs_get_acl(struct inode *, int, bool);
-int f2fs_set_acl(struct mnt_idmap *, struct dentry *,
+int f2fs_set_acl(const struct mnt_idmap *, struct dentry *,
 			struct posix_acl *, int);
-int f2fs_init_acl(struct inode *, struct inode *, struct folio *ifolio,
-		struct folio *dfolio);
+int f2fs_init_acl(struct inode *inode, struct inode *dir,
+			struct f2fs_cached_block *ientry,
+			struct f2fs_cached_block *dentry);
 #else
 #define f2fs_get_acl	NULL
 #define f2fs_set_acl	NULL
 
 static inline int f2fs_init_acl(struct inode *inode, struct inode *dir,
-				struct folio *ifolio, struct folio *dfolio)
+				struct f2fs_cached_block *ientry,
+				struct f2fs_cached_block *dentry)
 {
 	return 0;
 }
diff --git a/fs/f2fs/cache.c b/fs/f2fs/cache.c
new file mode 100644
index 00000000000000..38fc5eb17f9282
--- /dev/null
+++ b/fs/f2fs/cache.c
@@ -0,0 +1,720 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Google LLC
+ * Author: Chao Yu <chaseyu@google.com>
+ */
+#include <linux/fs.h>
+#include <linux/f2fs_fs.h>
+#include <linux/radix-tree.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/pagemap.h>
+#include <linux/kthread.h>
+#include <linux/freezer.h>
+#include <linux/delay.h>
+#include "f2fs.h"
+#include "cache.h"
+#include "node.h"
+#include <trace/events/f2fs.h>
+#include "segment.h"
+
+void f2fs_cache_wait_writeback_cond(struct f2fs_cached_block *entry,
+					enum page_type type)
+{
+	/* in case the entry was truncated or on-going shrink */
+	if (!entry->cache)
+		return;
+
+	if (!f2fs_cache_test_writeback(entry))
+		return;
+
+	/* submit cached bio */
+	f2fs_submit_merged_write_cache(entry->cache->sbi, entry, 0, type);
+
+	wait_on_bit_io(&entry->state, F2FS_BLOCK_WRITEBACK,
+					TASK_UNINTERRUPTIBLE);
+}
+
+void f2fs_cache_wait_writeback(struct f2fs_cached_block *entry)
+{
+	/* in case the entry was truncated or on-going shrink */
+	if (!entry->cache)
+		return;
+
+	f2fs_cache_wait_writeback_cond(entry,
+			IS_META_CACHE(entry->cache) ? META : NODE);
+}
+
+void f2fs_cache_update_tag(struct f2fs_cached_block *entry,
+		unsigned int clear_from, unsigned int set_to)
+{
+	struct f2fs_cached_block_list *cache = entry->cache;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cache->tree_lock, flags);
+	if (clear_from != F2FS_CACHE_TAG_NONE)
+		radix_tree_tag_clear(&cache->root, entry->index, clear_from);
+	if (set_to != F2FS_CACHE_TAG_NONE)
+		radix_tree_tag_set(&cache->root, entry->index, set_to);
+	spin_unlock_irqrestore(&cache->tree_lock, flags);
+}
+
+bool f2fs_mark_cache_dirty(struct f2fs_cached_block *entry)
+{
+	struct f2fs_cached_block_list *cache = entry->cache;
+
+	f2fs_cache_set_uptodate(entry);
+
+#ifdef CONFIG_F2FS_CHECK_FS
+	if (f2fs_is_node_cache(entry) && IS_INODE(cache->sbi, entry))
+		f2fs_inode_chksum_set(cache->sbi, entry);
+#endif
+
+	if (!f2fs_cache_test_and_set_dirty(entry)) {
+		enum count_type type = IS_META_CACHE(cache) ?
+				F2FS_DIRTY_META : F2FS_DIRTY_NODES;
+
+		trace_f2fs_cache_set_dirty(entry,
+				IS_META_CACHE(cache) ? META : NODE);
+		f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_NONE,
+						F2FS_CACHE_TAG_DIRTY);
+		inc_cache_count(cache->sbi, type);
+		return true;
+	}
+
+	return false;
+}
+
+void f2fs_drop_cache_dirty(struct f2fs_cached_block *entry)
+{
+
+	struct f2fs_cached_block_list *cache = entry->cache;
+	enum count_type type = IS_META_CACHE(cache) ?
+				F2FS_DIRTY_META : F2FS_DIRTY_NODES;
+
+	f2fs_cache_clear_uptodate(entry);
+
+	if (!f2fs_cache_test_and_clear_dirty(entry))
+		return;
+
+	f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY,
+					F2FS_CACHE_TAG_NONE);
+	dec_cache_count(cache->sbi, type);
+}
+
+void f2fs_start_cache_writeback(struct f2fs_cached_block *entry)
+{
+	f2fs_cache_set_writeback(entry);
+	f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY,
+				F2FS_CACHE_TAG_WRITEBACK);
+}
+
+void f2fs_end_cache_writeback(struct f2fs_cached_block *entry)
+{
+	/*
+	 * should call f2fs_cache_update_tag() before clearing writeback bit,
+	 * in case f2fs_truncate_cache() set entry->cache to NULL.
+	 */
+	f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_WRITEBACK,
+						F2FS_CACHE_TAG_NONE);
+	clear_and_wake_up_bit(F2FS_BLOCK_WRITEBACK, &entry->state);
+}
+
+static int f2fs_cache_refcount(struct f2fs_cached_block *entry)
+{
+	return atomic_read(&entry->refcount);
+}
+
+static void f2fs_do_free_cache(struct f2fs_cached_block *entry)
+{
+	kfree(entry->data);
+	kfree(entry);
+}
+
+static void f2fs_free_cache(struct f2fs_cached_block *entry)
+{
+	WARN_ON_ONCE(!list_empty(&entry->list));
+	WARN_ON_ONCE(f2fs_cache_refcount(entry));
+	f2fs_do_free_cache(entry);
+}
+
+void f2fs_cache_get(struct f2fs_cached_block *entry)
+{
+	atomic_inc(&entry->refcount);
+}
+
+static bool f2fs_cache_put(struct f2fs_cached_block *entry)
+{
+	WARN_ON_ONCE(!f2fs_cache_refcount(entry));
+	if (atomic_dec_and_test(&entry->refcount)) {
+		f2fs_free_cache(entry);
+		return true;
+	}
+	return false;
+}
+
+static struct f2fs_cached_block *f2fs_create_cache(
+		struct f2fs_cached_block_list *cache,
+		unsigned long index, bool nofail)
+{
+	struct f2fs_sb_info *sbi = cache->sbi;
+	struct f2fs_cached_block *entry;
+	unsigned int flags = GFP_NOFS;
+
+	if (index == ULONG_MAX)
+		return ERR_PTR(-ERANGE);
+
+	if (nofail) {
+		flags |= __GFP_NOFAIL;
+		entry = kzalloc_obj(*entry, flags);
+		entry->data = kmalloc(sbi->blocksize, flags);
+	} else {
+		entry = f2fs_kzalloc(sbi, sizeof(*entry), flags);
+		if (!entry)
+			return ERR_PTR(-ENOMEM);
+
+		entry->data = f2fs_kmalloc(sbi, sbi->blocksize, flags);
+		if (!entry->data) {
+			kfree(entry);
+			return ERR_PTR(-ENOMEM);
+		}
+	}
+	entry->index = index;
+
+	atomic_set(&entry->refcount, 0);
+	if (!IS_COMPRESS_CACHE(cache))
+		entry->next_entry = NULL;
+	else
+		entry->ino = 0;
+	INIT_LIST_HEAD(&entry->list);
+
+	entry->cache = cache;
+
+	return entry;
+}
+
+static struct f2fs_cached_block *f2fs_insert_cache(
+			struct f2fs_cached_block_list *cache,
+			unsigned long index,
+			struct f2fs_cached_block *new)
+{
+	struct f2fs_cached_block *e;
+	int ret;
+	unsigned long flags;
+
+	ret = radix_tree_preload(GFP_NOFS | __GFP_NOFAIL);
+	f2fs_bug_on(cache->sbi, ret);
+
+	spin_lock(&cache->list_lock);
+	spin_lock_irqsave(&cache->tree_lock, flags);
+	e = radix_tree_lookup(&cache->root, index);
+	if (!e) {
+		e = new;
+		f2fs_bug_on(cache->sbi, f2fs_cache_refcount(e));
+
+		ret = radix_tree_insert(&cache->root, index, e);
+		f2fs_bug_on(cache->sbi, ret);
+
+		/* radix tree referenced cache entry */
+		f2fs_cache_get(e);
+		f2fs_bug_on(cache->sbi, !list_empty(&e->list));
+		list_add_tail(&e->list, &cache->lru_list);
+		cache->num_entries++;
+	}
+	f2fs_cache_get(e);
+	spin_unlock_irqrestore(&cache->tree_lock, flags);
+	spin_unlock(&cache->list_lock);
+	radix_tree_preload_end();
+
+	if (new != e) {
+		f2fs_bug_on(cache->sbi, f2fs_cache_refcount(new));
+		f2fs_do_free_cache(new);
+	}
+
+	return e;
+}
+
+struct f2fs_cached_block *f2fs_find_cache(
+			struct f2fs_cached_block_list *cache,
+			unsigned long index,
+			enum f2fs_cache_request_flag rflag)
+{
+	struct f2fs_cached_block *entry;
+	unsigned long flags;
+	bool access = rflag & F2FS_CACHE_ACCESS;
+
+	spin_lock_irqsave(&cache->tree_lock, flags);
+	entry = radix_tree_lookup(&cache->root, index);
+	if (entry) {
+		f2fs_bug_on(cache->sbi, !f2fs_cache_refcount(entry));
+		f2fs_cache_get(entry);
+		if (access && !f2fs_cache_test_referenced(entry))
+			f2fs_cache_set_referenced(entry);
+	} else {
+		entry = ERR_PTR(-ENOENT);
+	}
+	spin_unlock_irqrestore(&cache->tree_lock, flags);
+
+	return entry;
+}
+
+struct f2fs_cached_block *f2fs_grab_cache(
+			struct f2fs_cached_block_list *cache,
+			unsigned long index, int flags)
+
+{
+	struct f2fs_cached_block *entry, *new;
+	bool create = flags & F2FS_CACHE_CREATE;
+	bool nofail = flags & F2FS_CACHE_NOFAIL;
+	bool lock = flags & F2FS_CACHE_LOCK;
+
+repeat:
+	entry = f2fs_find_cache(cache, index, F2FS_CACHE_ACCESS);
+	if (!IS_ERR(entry))
+		goto found;
+
+	if (!create)
+		return ERR_PTR(-ENOENT);
+
+	new = f2fs_create_cache(cache, index, nofail);
+	if (IS_ERR(new))
+		return new;
+
+	entry = f2fs_insert_cache(cache, index, new);
+found:
+	if (lock) {
+		f2fs_lock_cache(entry);
+		/* has been truncated */
+		if (entry->cache != cache) {
+			f2fs_put_cache(entry, true);
+			goto repeat;
+		}
+	}
+	return entry;
+}
+
+bool f2fs_trylock_cache(struct f2fs_cached_block *entry)
+{
+	return !test_and_set_bit(F2FS_BLOCK_LOCKED, &entry->state);
+}
+
+void f2fs_lock_cache(struct f2fs_cached_block *entry)
+{
+	wait_on_bit_lock(&entry->state, F2FS_BLOCK_LOCKED,
+					TASK_UNINTERRUPTIBLE);
+}
+
+void f2fs_unlock_cache(struct f2fs_cached_block *entry)
+{
+	clear_and_wake_up_bit(F2FS_BLOCK_LOCKED, &entry->state);
+}
+
+bool f2fs_put_cache(struct f2fs_cached_block *entry, bool unlock)
+{
+	if (IS_ERR_OR_NULL(entry))
+		return false;
+	if (unlock)
+		f2fs_unlock_cache(entry);
+	return f2fs_cache_put(entry);
+}
+
+unsigned int f2fs_cache_gang_lookup(struct f2fs_cached_block_list *cache,
+				struct f2fs_cached_block **entries,
+				pgoff_t *index, unsigned long end)
+{
+	unsigned long flags;
+	unsigned int max_nr = min((unsigned long)F2FS_ONSTACK_CACHES, end - *index);
+	int nr, i;
+
+	if (*index >= end || *index == ULONG_MAX)
+		return 0;
+
+	spin_lock_irqsave(&cache->tree_lock, flags);
+	nr = radix_tree_gang_lookup(&cache->root, (void **)entries,
+				*index, max_nr);
+	if (!nr)
+		goto out_unlock;
+
+	for (i = 0; i < nr; i++) {
+		struct f2fs_cached_block *entry = entries[i];
+
+		if (entry->index >= end) {
+			nr = i;
+			break;
+		}
+		f2fs_cache_get(entry);
+	}
+	if (nr)
+		*index = entries[nr - 1]->index + 1;
+out_unlock:
+	spin_unlock_irqrestore(&cache->tree_lock, flags);
+	return nr;
+}
+
+unsigned int f2fs_cache_gang_lookup_tag(struct f2fs_cached_block_list *cache,
+				struct f2fs_cached_block **entries,
+				pgoff_t *index, unsigned int max_nr,
+				int tag)
+{
+	unsigned long flags;
+	int nr, i;
+
+	if (*index == ULONG_MAX)
+		return 0;
+
+	spin_lock_irqsave(&cache->tree_lock, flags);
+	nr = radix_tree_gang_lookup_tag(&cache->root, (void **)entries,
+				*index, max_nr, tag);
+	if (!nr)
+		goto out;
+
+	for (i = 0; i < nr; i++)
+		f2fs_cache_get(entries[i]);
+	*index = entries[nr - 1]->index + 1;
+out:
+	spin_unlock_irqrestore(&cache->tree_lock, flags);
+	return nr;
+}
+
+void f2fs_cache_gang_release(struct f2fs_cached_block **entries,
+				unsigned int nr_entries)
+{
+	int i;
+
+	for (i = 0; i < nr_entries; i++)
+		f2fs_put_cache(entries[i], false);
+}
+
+void f2fs_cache_wait_on_all_writeback(struct f2fs_cached_block_list *cache)
+{
+	unsigned long index = 0;
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
+	int nr, i;
+
+next:
+	nr = f2fs_cache_gang_lookup_tag(cache, entries, &index,
+			F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_WRITEBACK);
+	if (!nr)
+		return;
+
+	for (i = 0; i < nr; i++)
+		f2fs_cache_wait_writeback(entries[i]);
+	f2fs_cache_gang_release(entries, nr);
+
+	cond_resched();
+	goto next;
+}
+
+static void f2fs_do_truncate_cache(struct f2fs_cached_block *entry,
+						bool drop_dirty)
+{
+	struct f2fs_cached_block_list *cache = entry->cache;
+	unsigned long flags;
+
+	if (drop_dirty)
+		goto drop_it;
+
+	if (f2fs_cache_test_dirty(entry) ||
+	    f2fs_cache_test_writeback(entry))
+		return;
+
+drop_it:
+	f2fs_cache_wait_writeback(entry);
+	f2fs_drop_cache_dirty(entry);
+
+	spin_lock(&cache->list_lock);
+	spin_lock_irqsave(&cache->tree_lock, flags);
+
+	f2fs_bug_on(cache->sbi, !entry->cache);
+	if (!radix_tree_delete(&cache->root, entry->index))
+		f2fs_bug_on(cache->sbi, !entry->cache);
+
+	entry->cache = NULL;
+	cache->num_entries--;
+
+	atomic_dec(&entry->refcount);
+	f2fs_bug_on(cache->sbi, !f2fs_cache_refcount(entry));
+
+	f2fs_bug_on(cache->sbi, list_empty(&entry->list));
+	list_del_init(&entry->list);
+
+	spin_unlock_irqrestore(&cache->tree_lock, flags);
+	spin_unlock(&cache->list_lock);
+}
+
+void f2fs_truncate_locked_cache(struct f2fs_cached_block *entry,
+				bool drop_dirty)
+{
+	if (!entry->cache)
+		return;
+	f2fs_do_truncate_cache(entry, drop_dirty);
+}
+
+void f2fs_truncate_cache(struct f2fs_cached_block *entry, bool drop_dirty)
+{
+	f2fs_lock_cache(entry);
+	f2fs_truncate_locked_cache(entry, drop_dirty);
+	f2fs_unlock_cache(entry);
+}
+
+static void f2fs_drop_cache(struct f2fs_cached_block_list *cache,
+			block_t blkaddr, bool drop_dirty)
+{
+	struct f2fs_cached_block *entry;
+
+	entry = f2fs_find_cache(cache, blkaddr, 0);
+	if (IS_ERR(entry))
+		return;
+
+	f2fs_truncate_cache(entry, drop_dirty);
+	f2fs_put_cache(entry, false);
+}
+
+void f2fs_drop_cache_range(struct f2fs_cached_block_list *cache,
+		unsigned long start, unsigned long len, bool drop_dirty)
+{
+	unsigned long index = start;
+	unsigned long end = (ULONG_MAX - start < len) ?
+				ULONG_MAX : (start + len);
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
+	int nr, i;
+
+	if (len == 1)
+		return f2fs_drop_cache(cache, index, drop_dirty);
+
+next:
+	nr = f2fs_cache_gang_lookup(cache, entries, &index, end);
+	if (!nr)
+		return;
+
+	for (i = 0; i < nr; i++)
+		f2fs_truncate_cache(entries[i], drop_dirty);
+	f2fs_cache_gang_release(entries, nr);
+
+	if (index < end) {
+		cond_resched();
+		goto next;
+	}
+}
+
+int f2fs_init_cache(struct f2fs_sb_info *sbi,
+		struct f2fs_cached_block_list *cache,
+		enum f2fs_cache_type type)
+{
+	cache->sbi = sbi;
+	cache->type = type;
+	INIT_RADIX_TREE(&cache->root, GFP_ATOMIC);
+	spin_lock_init(&cache->tree_lock);
+	spin_lock_init(&cache->list_lock);
+	INIT_LIST_HEAD(&cache->lru_list);
+	cache->num_entries = 0;
+
+	return 0;
+}
+
+void f2fs_destroy_cache(struct f2fs_cached_block_list *cache)
+{
+	struct list_head *head = &cache->lru_list;
+	struct f2fs_cached_block *entry;
+	unsigned long flags;
+
+	f2fs_cache_wait_on_all_writeback(cache);
+next:
+	spin_lock(&cache->list_lock);
+	if (list_empty(head)) {
+		spin_unlock(&cache->list_lock);
+		return;
+	}
+	entry = list_first_entry(head, struct f2fs_cached_block, list);
+
+	spin_lock_irqsave(&cache->tree_lock, flags);
+	radix_tree_delete(&cache->root, entry->index);
+	cache->num_entries--;
+	list_del_init(&entry->list);
+	spin_unlock_irqrestore(&cache->tree_lock, flags);
+
+	spin_unlock(&cache->list_lock);
+
+	/* wait on read cache IO */
+	f2fs_lock_cache(entry);
+	/* wait on write cache IO */
+	f2fs_cache_wait_writeback(entry);
+	f2fs_bug_on(cache->sbi, f2fs_cache_test_dirty(entry));
+	f2fs_bug_on(cache->sbi, f2fs_cache_test_writeback(entry));
+	f2fs_bug_on(cache->sbi, !list_empty(&entry->list));
+	f2fs_bug_on(cache->sbi, f2fs_cache_refcount(entry) != 1);
+	f2fs_put_cache(entry, true);
+	goto next;
+}
+
+static unsigned long f2fs_do_shrink_cache(struct f2fs_cached_block_list *cache,
+						unsigned long nr_to_scan)
+{
+	struct f2fs_cached_block *entry, *next;
+	LIST_HEAD(dispose_list);
+	LIST_HEAD(keep_list);
+	unsigned long freed = 0;
+	unsigned long scanned = 0;
+
+	/* Phase 1: Isolate candidate entries from LRU list into dispose_list */
+	spin_lock(&cache->list_lock);
+	list_for_each_entry_safe(entry, next, &cache->lru_list, list) {
+		if (scanned >= cache->num_entries)
+			break;
+		if (scanned++ >= nr_to_scan)
+			break;
+
+		/* If accessed, give it a second chance to rotate to tail */
+		if (f2fs_cache_test_and_clear_referenced(entry)) {
+			list_move_tail(&entry->list, &cache->lru_list);
+			continue;
+		}
+
+		if (f2fs_cache_test_dirty(entry) ||
+		    f2fs_cache_test_writeback(entry) ||
+		    f2fs_cache_test_locked(entry))
+			continue;
+
+		if (f2fs_cache_refcount(entry) != 1)
+			continue;
+
+		list_move_tail(&entry->list, &dispose_list);
+	}
+	spin_unlock(&cache->list_lock);
+
+	/* Phase 2: Process isolated candidates one by one */
+	while (1) {
+		spin_lock(&cache->list_lock);
+		entry = list_first_entry_or_null(&dispose_list,
+						struct f2fs_cached_block, list);
+		if (!entry) {
+			spin_unlock(&cache->list_lock);
+			break;
+		}
+		f2fs_cache_get(entry);
+		list_move_tail(&entry->list, &keep_list);
+		spin_unlock(&cache->list_lock);
+
+		if (!f2fs_trylock_cache(entry)) {
+			f2fs_put_cache(entry, false);
+			continue;
+		}
+
+		/* the entry has been truncated */
+		if (!entry->cache) {
+			f2fs_put_cache(entry, true);
+			continue;
+		}
+		/*
+		 * at least there are shrinker, radix tree and another user
+		 * has referenced the entry.
+		 */
+		if (f2fs_cache_refcount(entry) >= 3) {
+			f2fs_put_cache(entry, true);
+			continue;
+		}
+
+		f2fs_do_truncate_cache(entry, false);
+
+		if (f2fs_put_cache(entry, true))
+			freed++;
+	}
+
+	/* Phase 3: Splice un-reclaimed entries back onto cache->lru_list */
+	if (!list_empty(&keep_list)) {
+		spin_lock(&cache->list_lock);
+		list_splice_tail(&keep_list, &cache->lru_list);
+		spin_unlock(&cache->list_lock);
+	}
+
+	return freed;
+}
+
+unsigned long f2fs_shrink_cache(struct f2fs_sb_info *sbi,
+					unsigned long nr_to_scan)
+{
+	unsigned long freed;
+
+	freed = f2fs_do_shrink_cache(META_CACHE(sbi), nr_to_scan);
+	if (freed >= nr_to_scan)
+		return freed;
+
+	freed += f2fs_do_shrink_cache(NODE_CACHE(sbi), nr_to_scan - freed);
+	if (freed >= nr_to_scan)
+		return freed;
+
+	freed += f2fs_do_shrink_cache(COMPRESS_CACHE(sbi), nr_to_scan - freed);
+	return freed;
+}
+
+static int f2fs_cache_writeback_kthread(void *data)
+{
+	struct f2fs_sb_info *sbi = data;
+	struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread;
+	wait_queue_head_t *wq = &cache_thread->cache_wb_wq;
+
+	set_freezable();
+
+	while (!kthread_should_stop()) {
+		unsigned int interval = cache_thread->cache_wb_interval;
+
+		wait_event_freezable_timeout(*wq,
+				kthread_should_stop(),
+				msecs_to_jiffies(interval));
+
+		if (kthread_should_stop())
+			break;
+
+		if (f2fs_readonly(sbi->sb))
+			continue;
+
+		if (f2fs_cp_error(sbi))
+			continue;
+
+		if (unlikely(freezing(current)))
+			continue;
+
+		if (!sb_start_write_trylock(sbi->sb))
+			continue;
+
+		f2fs_write_meta_caches(sbi);
+		f2fs_write_node_caches(sbi);
+
+		sb_end_write(sbi->sb);
+	}
+	return 0;
+}
+
+int f2fs_start_cache_wb_thread(struct f2fs_sb_info *sbi)
+{
+	struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread;
+	struct task_struct *task;
+	dev_t dev = sbi->sb->s_dev;
+	char name[36];
+
+	if (cache_thread->cache_wb_task)
+		return 0;
+
+	init_waitqueue_head(&cache_thread->cache_wb_wq);
+	cache_thread->cache_wb_interval = DEF_DIRTY_CACHE_TIMEOUT;
+	snprintf(name, sizeof(name), "f2fs_writeback-%u:%u",
+			MAJOR(dev), MINOR(dev));
+
+	task = kthread_run(f2fs_cache_writeback_kthread, sbi, "%s", name);
+	if (IS_ERR(task))
+		return PTR_ERR(task);
+
+	cache_thread->cache_wb_task = task;
+	return 0;
+}
+
+void f2fs_stop_cache_wb_thread(struct f2fs_sb_info *sbi)
+{
+	struct f2fs_cache_kthread *cache_thread = &sbi->cache_thread;
+
+	if (!cache_thread->cache_wb_task)
+		return;
+
+	kthread_stop(cache_thread->cache_wb_task);
+	cache_thread->cache_wb_task = NULL;
+}
diff --git a/fs/f2fs/cache.h b/fs/f2fs/cache.h
new file mode 100644
index 00000000000000..6c4db910d767a5
--- /dev/null
+++ b/fs/f2fs/cache.h
@@ -0,0 +1,240 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Google LLC
+ * Author: Chao Yu <chaseyu@google.com>
+ */
+#ifndef _LINUX_F2FS_CACHE_H
+#define _LINUX_F2FS_CACHE_H
+
+#include <linux/pagemap.h>
+#include <linux/mm.h>
+#include <linux/list.h>
+#include <linux/radix-tree.h>
+#include <linux/spinlock.h>
+#include <linux/wait.h>
+#include <linux/types.h>
+
+struct f2fs_rwsem;
+struct f2fs_io_info;
+enum page_type;
+
+/* Represents a single cached block (meta, node or compress) */
+struct f2fs_cached_block {
+	struct list_head list;		/* LRU list head */
+	struct f2fs_cached_block_list *cache;	/* parent cache list */
+	union {
+		struct f2fs_cached_block *next_entry;/* chain for merged BIO */
+		nid_t ino;		/* inode number for compress cache */
+	};
+	unsigned long index;		/* key in radix tree, (meta/compress: pba, node: nid) */
+	unsigned long state;		/* cache entry state (e.g., Dirty, UpToDate) */
+	void *data;			/* blocksize-aligned memory (4KB or 16KB) */
+	atomic_t refcount;		/* reference count */
+};
+
+struct f2fs_sb_info;
+
+enum f2fs_cache_type {
+	F2FS_META_CACHE,
+	F2FS_NODE_CACHE,
+	F2FS_COMPRESS_CACHE,
+	NR_CACHE_TYPES,
+};
+
+/* Main cache control structure (per sb_info) */
+struct f2fs_cached_block_list {
+	struct f2fs_sb_info *sbi;	/* Pointer to f2fs_sb_info */
+	struct radix_tree_root root;	/* Radix tree for cache lookup */
+	spinlock_t tree_lock;		/* Lock for radix tree */
+	struct list_head lru_list;	/* Single global LRU list */
+	spinlock_t list_lock;		/* Lock for LRU list */
+	enum f2fs_cache_type type;	/* Cache type (Node, Meta, Compress) */
+	unsigned long num_entries;	/* Current number of entries */
+};
+
+#define IS_META_CACHE(cache) ((cache)->type == F2FS_META_CACHE)
+#define IS_NODE_CACHE(cache) ((cache)->type == F2FS_NODE_CACHE)
+#define IS_COMPRESS_CACHE(cache) ((cache)->type == F2FS_COMPRESS_CACHE)
+
+/* Flags for f2fs_cached_block state */
+enum f2fs_cached_state {
+	F2FS_BLOCK_LOCKED,		/* cache entry is locked */
+	F2FS_BLOCK_UPTODATE,		/* cache data is valid */
+	F2FS_BLOCK_DIRTY,		/* cache data is dirty, need to writeback the data */
+	F2FS_BLOCK_WRITEBACK,		/* cache data is writeback state */
+	F2FS_BLOCK_REFERENCED,		/* cache was accessed recently, shrinker will skip it for once */
+	F2FS_BLOCK_INLINE_DATA,		/* indicate inline data */
+};
+
+enum {
+	__F2FS_CACHE_CREATE,		/* create the cache if there is no cache entry */
+	__F2FS_CACHE_LOCK,		/* get and lock the cache entry */
+	__F2FS_CACHE_NOFAIL,		/* do not allow failure */
+	__F2FS_CACHE_ACCESS,		/* give a chance to add referenced tag */
+};
+
+enum f2fs_cache_request_flag {
+	F2FS_CACHE_CREATE	= 1 << __F2FS_CACHE_CREATE,
+	F2FS_CACHE_LOCK		= 1 << __F2FS_CACHE_LOCK,
+	F2FS_CACHE_NOFAIL	= 1 << __F2FS_CACHE_NOFAIL,
+	F2FS_CACHE_ACCESS	= 1 << __F2FS_CACHE_ACCESS,
+};
+
+#define F2FS_CACHE_LOCK_CREATE	(F2FS_CACHE_LOCK | F2FS_CACHE_CREATE)
+
+#define F2FS_ONSTACK_CACHES		(32)
+
+#define F2FS_CACHE_FLAG_TEST_FUNC(name, flagname)			\
+static inline bool f2fs_cache_test_##name(				\
+			const struct f2fs_cached_block *entry)		\
+{									\
+	return test_bit(F2FS_BLOCK_##flagname, &entry->state);		\
+}									\
+
+#define F2FS_CACHE_FLAG_SET_FUNC(name, flagname)			\
+static inline void f2fs_cache_set_##name(				\
+			struct f2fs_cached_block *entry)		\
+{									\
+	set_bit(F2FS_BLOCK_##flagname, &entry->state);			\
+}									\
+
+#define F2FS_CACHE_FLAG_CLEAR_FUNC(name, flagname)			\
+static inline void f2fs_cache_clear_##name(				\
+			struct f2fs_cached_block *entry)		\
+{									\
+	clear_bit(F2FS_BLOCK_##flagname, &entry->state);		\
+}									\
+
+#define F2FS_CACHE_FLAG_TEST_AND_SET_FUNC(name, flagname)		\
+static inline bool f2fs_cache_test_and_set_##name(			\
+			struct f2fs_cached_block *entry)		\
+{									\
+	return test_and_set_bit(F2FS_BLOCK_##flagname, &entry->state);	\
+}									\
+
+#define F2FS_CACHE_FLAG_TEST_AND_CLEAR_FUNC(name, flagname)		\
+static inline bool f2fs_cache_test_and_clear_##name(			\
+			struct f2fs_cached_block *entry)		\
+{									\
+	return test_and_clear_bit(F2FS_BLOCK_##flagname, &entry->state);\
+}									\
+
+F2FS_CACHE_FLAG_TEST_FUNC(locked, LOCKED);
+F2FS_CACHE_FLAG_SET_FUNC(locked, LOCKED);
+F2FS_CACHE_FLAG_CLEAR_FUNC(locked, LOCKED);
+
+F2FS_CACHE_FLAG_TEST_FUNC(uptodate, UPTODATE);
+F2FS_CACHE_FLAG_SET_FUNC(uptodate, UPTODATE);
+F2FS_CACHE_FLAG_CLEAR_FUNC(uptodate, UPTODATE);
+
+F2FS_CACHE_FLAG_TEST_FUNC(dirty, DIRTY);
+F2FS_CACHE_FLAG_SET_FUNC(dirty, DIRTY);
+F2FS_CACHE_FLAG_CLEAR_FUNC(dirty, DIRTY);
+F2FS_CACHE_FLAG_TEST_AND_SET_FUNC(dirty, DIRTY);
+F2FS_CACHE_FLAG_TEST_AND_CLEAR_FUNC(dirty, DIRTY);
+
+F2FS_CACHE_FLAG_TEST_FUNC(writeback, WRITEBACK);
+F2FS_CACHE_FLAG_SET_FUNC(writeback, WRITEBACK);
+F2FS_CACHE_FLAG_CLEAR_FUNC(writeback, WRITEBACK);
+
+F2FS_CACHE_FLAG_TEST_FUNC(inline, INLINE_DATA);
+F2FS_CACHE_FLAG_SET_FUNC(inline, INLINE_DATA);
+F2FS_CACHE_FLAG_CLEAR_FUNC(inline, INLINE_DATA);
+
+F2FS_CACHE_FLAG_TEST_FUNC(referenced, REFERENCED);
+F2FS_CACHE_FLAG_SET_FUNC(referenced, REFERENCED);
+F2FS_CACHE_FLAG_TEST_AND_CLEAR_FUNC(referenced, REFERENCED);
+
+static inline void *cache_address(const struct f2fs_cached_block *entry)
+{
+	return entry->data;
+}
+
+#define CACHED_NODE(entry)	((struct f2fs_node *)(cache_address(entry)))
+
+static inline struct folio *cache_folio(const struct f2fs_cached_block *entry)
+{
+	return virt_to_folio(entry->data);
+}
+
+int f2fs_init_cache(struct f2fs_sb_info *sbi,
+				struct f2fs_cached_block_list *cache,
+				enum f2fs_cache_type type);
+void f2fs_destroy_cache(struct f2fs_cached_block_list *cache);
+void f2fs_cache_get(struct f2fs_cached_block *entry);
+struct f2fs_cached_block *f2fs_find_cache(
+			struct f2fs_cached_block_list *cache,
+			unsigned long index,
+			enum f2fs_cache_request_flag rflag);
+#define F2FS_CACHE_TAG_NONE		0
+#define F2FS_CACHE_TAG_DIRTY		1
+#define F2FS_CACHE_TAG_WRITEBACK	2
+
+bool f2fs_trylock_cache(struct f2fs_cached_block *entry);
+void f2fs_lock_cache(struct f2fs_cached_block *entry);
+void f2fs_unlock_cache(struct f2fs_cached_block *entry);
+bool f2fs_put_cache(struct f2fs_cached_block *entry, bool unlock);
+bool f2fs_mark_cache_dirty(struct f2fs_cached_block *entry);
+void f2fs_drop_cache_dirty(struct f2fs_cached_block *entry);
+void f2fs_start_cache_writeback(struct f2fs_cached_block *entry);
+void f2fs_end_cache_writeback(struct f2fs_cached_block *entry);
+unsigned int f2fs_cache_gang_lookup(struct f2fs_cached_block_list *cache,
+				struct f2fs_cached_block **entries,
+				pgoff_t *index, unsigned long end);
+unsigned int f2fs_cache_gang_lookup_tag(struct f2fs_cached_block_list *cache,
+		struct f2fs_cached_block **results, pgoff_t *first_index,
+		unsigned int max_items, int tag);
+void f2fs_cache_gang_release(struct f2fs_cached_block **entries,
+				unsigned int nr_entries);
+void f2fs_cache_wait_on_all_writeback(struct f2fs_cached_block_list *cache);
+void f2fs_cache_wait_writeback_cond(struct f2fs_cached_block *entry,
+					enum page_type type);
+void f2fs_cache_wait_writeback(struct f2fs_cached_block *entry);
+void f2fs_cache_update_tag(struct f2fs_cached_block *entry,
+		unsigned int clear_from, unsigned int set_to);
+struct f2fs_cached_block *f2fs_grab_cache(struct f2fs_cached_block_list *cache,
+				unsigned long index, int flags);
+void f2fs_truncate_locked_cache(struct f2fs_cached_block *entry,
+				bool drop_dirty);
+void f2fs_truncate_cache(struct f2fs_cached_block *entry, bool drop_dirty);
+void f2fs_drop_cache_range(struct f2fs_cached_block_list *cache,
+		unsigned long start, unsigned long len, bool drop_dirty);
+
+#define META_CACHE(sbi)		(&(sbi)->meta_blocks)
+#define NODE_CACHE(sbi)		(&(sbi)->node_blocks)
+#define COMPRESS_CACHE(sbi)	(&(sbi)->compress_blocks)
+
+#define f2fs_find_meta_cache(sbi, blkaddr)		\
+	f2fs_find_cache(META_CACHE(sbi), blkaddr, 0)
+#define f2fs_invalidate_meta_caches(sbi, start, len)	\
+	f2fs_drop_cache_range(META_CACHE(sbi), start, len, false)
+#define f2fs_truncate_meta_caches(sbi, start, len)	\
+	f2fs_drop_cache_range(META_CACHE(sbi), start, len, true)
+
+#define f2fs_grab_node_cache(sbi, blkaddr)		\
+	f2fs_grab_cache(NODE_CACHE(sbi), blkaddr,	\
+	F2FS_CACHE_LOCK_CREATE)
+#define f2fs_find_node_cache(sbi, blkaddr)		\
+	f2fs_find_cache(NODE_CACHE(sbi), blkaddr, F2FS_CACHE_ACCESS)
+#define f2fs_invalidate_node_cache(sbi, blkaddr)	\
+	f2fs_drop_cache_range(NODE_CACHE(sbi), blkaddr, 1, false)
+#define f2fs_truncate_node_caches(sbi, start, len)	\
+	f2fs_drop_cache_range(NODE_CACHE(sbi), start, len, true)
+
+unsigned long f2fs_shrink_cache(struct f2fs_sb_info *sbi,
+				unsigned long nr_to_scan);
+
+#define DEF_DIRTY_CACHE_TIMEOUT 5000
+#define MIN_DIRTY_CACHE_TIMEOUT 100
+#define MAX_DIRTY_CACHE_TIMEOUT 30000
+
+struct f2fs_cache_kthread {
+	struct task_struct *cache_wb_task;
+	wait_queue_head_t cache_wb_wq;
+	unsigned int cache_wb_interval;
+};
+
+int f2fs_start_cache_wb_thread(struct f2fs_sb_info *sbi);
+void f2fs_stop_cache_wb_thread(struct f2fs_sb_info *sbi);
+
+#endif /* _LINUX_F2FS_CACHE_H */
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index 4b59f30ef45d58..7b89bb838946de 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -7,7 +7,6 @@
  */
 #include <linux/fs.h>
 #include <linux/bio.h>
-#include <linux/mpage.h>
 #include <linux/writeback.h>
 #include <linux/blkdev.h>
 #include <linux/f2fs_fs.h>
@@ -17,6 +16,7 @@
 #include <linux/delayacct.h>
 #include <linux/ioprio.h>
 #include <linux/math64.h>
+#include <linux/freezer.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -234,29 +234,29 @@ static struct kmem_cache *ino_entry_slab;
 struct kmem_cache *f2fs_inode_entry_slab;
 
 /*
- * We guarantee no failure on the returned page.
+ * We guarantee no failure on the returned cache.
  */
-struct folio *f2fs_grab_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index)
+struct f2fs_cached_block *f2fs_grab_meta_cache(struct f2fs_sb_info *sbi,
+							pgoff_t index)
 {
-	struct address_space *mapping = META_MAPPING(sbi);
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 repeat:
-	folio = f2fs_grab_cache_folio(mapping, index, false);
-	if (IS_ERR(folio)) {
+	entry = f2fs_grab_cache(META_CACHE(sbi), index,
+					F2FS_CACHE_LOCK_CREATE);
+	if (IS_ERR(entry)) {
 		cond_resched();
 		goto repeat;
 	}
-	f2fs_folio_wait_writeback(folio, META, true, true);
-	if (!folio_test_uptodate(folio))
-		folio_mark_uptodate(folio);
-	return folio;
+	f2fs_cache_wait_writeback(entry);
+	if (!f2fs_cache_test_uptodate(entry))
+		f2fs_cache_set_uptodate(entry);
+	return entry;
 }
 
-static struct folio *__get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index,
-							bool is_meta)
+static struct f2fs_cached_block *__get_meta_cache(struct f2fs_sb_info *sbi,
+					pgoff_t index, bool is_meta)
 {
-	struct address_space *mapping = META_MAPPING(sbi);
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	struct f2fs_io_info fio = {
 		.sbi = sbi,
 		.type = META,
@@ -266,70 +266,75 @@ static struct folio *__get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index,
 		.new_blkaddr = index,
 		.encrypted_page = NULL,
 		.is_por = !is_meta ? 1 : 0,
+		.is_cache = 1,
 	};
 	int err;
 
 	if (unlikely(!is_meta))
 		fio.op_flags &= ~REQ_META;
 repeat:
-	folio = f2fs_grab_cache_folio(mapping, index, false);
-	if (IS_ERR(folio)) {
+	entry = f2fs_grab_cache(META_CACHE(sbi), index,
+					F2FS_CACHE_LOCK_CREATE);
+	if (IS_ERR(entry)) {
 		cond_resched();
 		goto repeat;
 	}
-	if (folio_test_uptodate(folio))
+	if (f2fs_cache_test_uptodate(entry))
 		goto out;
 
-	fio.folio = folio;
+	fio.cache_entry = entry;
 
-	err = f2fs_submit_page_bio(&fio);
+	err = f2fs_submit_cache_read(&fio);
 	if (err) {
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 		return ERR_PTR(err);
 	}
 
-	f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE);
+	f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE(sbi));
 
-	folio_lock(folio);
-	if (unlikely(!is_meta_folio(folio))) {
-		f2fs_folio_put(folio, true);
+	f2fs_lock_cache(entry);
+	if (unlikely(!f2fs_is_meta_cache(entry))) {
+		f2fs_put_cache(entry, true);
 		goto repeat;
 	}
 
-	if (unlikely(!folio_test_uptodate(folio))) {
-		f2fs_handle_page_eio(sbi, folio, META);
-		f2fs_folio_put(folio, true);
+	if (unlikely(!f2fs_cache_test_uptodate(entry))) {
+		f2fs_handle_page_eio(sbi, entry->index, META);
+		f2fs_put_cache(entry, true);
 		return ERR_PTR(-EIO);
 	}
 out:
-	return folio;
+	return entry;
 }
 
-struct folio *f2fs_get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index)
+struct f2fs_cached_block *f2fs_get_meta_cache(struct f2fs_sb_info *sbi,
+							pgoff_t index)
 {
-	return __get_meta_folio(sbi, index, true);
+	return __get_meta_cache(sbi, index, true);
 }
 
-struct folio *f2fs_get_meta_folio_retry(struct f2fs_sb_info *sbi, pgoff_t index)
+struct f2fs_cached_block *f2fs_get_meta_cache_retry(struct f2fs_sb_info *sbi,
+							pgoff_t index)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	int count = 0;
 
 retry:
-	folio = __get_meta_folio(sbi, index, true);
-	if (IS_ERR(folio)) {
-		if (PTR_ERR(folio) == -EIO &&
+	entry = __get_meta_cache(sbi, index, true);
+	if (IS_ERR(entry)) {
+		if (PTR_ERR(entry) == -EIO &&
 				++count <= DEFAULT_RETRY_IO_COUNT)
 			goto retry;
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE);
 	}
-	return folio;
+	return entry;
 }
 
 /* for POR only */
-struct folio *f2fs_get_tmp_folio(struct f2fs_sb_info *sbi, pgoff_t index)
+struct f2fs_cached_block *f2fs_get_tmp_cache(struct f2fs_sb_info *sbi,
+							pgoff_t index)
 {
-	return __get_meta_folio(sbi, index, false);
+	return __get_meta_cache(sbi, index, false);
 }
 
 static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr,
@@ -445,11 +450,12 @@ bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi,
 }
 
 /*
- * Readahead CP/NAT/SIT/SSA/POR pages
+ * Readahead CP/NAT/SIT/SSA/POR blocks
  */
-int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
-							int type, bool sync)
+int f2fs_ra_meta_caches(struct f2fs_sb_info *sbi, block_t start,
+				int nrblocks, int type, bool sync)
 {
+	struct f2fs_cached_block_list *cache = META_CACHE(sbi);
 	block_t blkno = start;
 	struct f2fs_io_info fio = {
 		.sbi = sbi,
@@ -459,6 +465,7 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 		.encrypted_page = NULL,
 		.in_list = 0,
 		.is_por = (type == META_POR) ? 1 : 0,
+		.is_cache = 1,
 	};
 	struct blk_plug plug;
 	int err;
@@ -467,8 +474,8 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 		fio.op_flags &= ~REQ_META;
 
 	blk_start_plug(&plug);
-	for (; nrpages-- > 0; blkno++) {
-		struct folio *folio;
+	for (; nrblocks-- > 0; blkno++) {
+		struct f2fs_cached_block *entry;
 
 		if (!f2fs_is_valid_blkaddr(sbi, blkno, type))
 			goto out;
@@ -476,18 +483,18 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 		switch (type) {
 		case META_NAT:
 			if (unlikely(blkno >=
-					NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid)))
+					NAT_BLOCK_OFFSET(sbi, NM_I(sbi)->max_nid)))
 				blkno = 0;
 			/* get nat block addr */
 			fio.new_blkaddr = current_nat_addr(sbi,
-					blkno * NAT_ENTRY_PER_BLOCK);
+					blkno * NAT_ENTRY_PER_BLOCK(sbi));
 			break;
 		case META_SIT:
 			if (unlikely(blkno >= TOTAL_SEGS(sbi)))
 				goto out;
 			/* get sit block addr */
 			fio.new_blkaddr = current_sit_addr(sbi,
-					blkno * SIT_ENTRY_PER_BLOCK);
+					blkno * SIT_ENTRY_PER_BLOCK(sbi));
 			break;
 		case META_SSA:
 		case META_CP:
@@ -495,62 +502,63 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 			fio.new_blkaddr = blkno;
 			break;
 		default:
-			BUG();
+			f2fs_bug_on(sbi, 1);
 		}
 
-		folio = f2fs_grab_cache_folio(META_MAPPING(sbi),
-						fio.new_blkaddr, false);
-		if (IS_ERR(folio))
+		entry = f2fs_grab_cache(cache, fio.new_blkaddr,
+						F2FS_CACHE_LOCK_CREATE);
+		if (IS_ERR(entry))
 			continue;
-		if (folio_test_uptodate(folio)) {
-			f2fs_folio_put(folio, true);
+		if (f2fs_cache_test_uptodate(entry)) {
+			f2fs_put_cache(entry, true);
 			continue;
 		}
 
-		fio.folio = folio;
-		err = f2fs_submit_page_bio(&fio);
-		f2fs_folio_put(folio, err ? true : false);
+		fio.cache_entry = entry;
+		err = f2fs_submit_cache_read(&fio);
+		f2fs_put_cache(entry, err ? true : false);
 
 		if (!err)
 			f2fs_update_iostat(sbi, NULL, FS_META_READ_IO,
-							F2FS_BLKSIZE);
+							F2FS_BLKSIZE(sbi));
 	}
 out:
 	blk_finish_plug(&plug);
 	return blkno - start;
 }
 
-void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index,
-							unsigned int ra_blocks)
+void f2fs_ra_meta_caches_cond(struct f2fs_sb_info *sbi, pgoff_t index,
+						unsigned int ra_blocks)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	bool readahead = false;
 
 	if (ra_blocks == RECOVERY_MIN_RA_BLOCKS)
 		return;
 
-	folio = filemap_get_folio(META_MAPPING(sbi), index);
-	if (IS_ERR(folio) || !folio_test_uptodate(folio))
+	entry = f2fs_find_cache(META_CACHE(sbi), index, 0);
+	if (IS_ERR(entry) || !f2fs_cache_test_uptodate(entry))
 		readahead = true;
-	f2fs_folio_put(folio, false);
+	f2fs_put_cache(entry, false);
 
 	if (readahead)
-		f2fs_ra_meta_pages(sbi, index, ra_blocks, META_POR, true);
+		f2fs_ra_meta_caches(sbi, index, ra_blocks, META_POR, true);
 }
 
-static bool __f2fs_write_meta_folio(struct folio *folio,
-				struct writeback_control *wbc,
+static bool __f2fs_write_meta_cache(struct f2fs_cached_block *entry,
 				enum iostat_type io_type)
 {
-	struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
+	struct f2fs_sb_info *sbi = entry->cache->sbi;
 
-	trace_f2fs_writepage(folio, META);
+	trace_f2fs_write_cache(entry, META);
 
 	if (unlikely(f2fs_cp_error(sbi))) {
 		if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) {
-			folio_clear_uptodate(folio);
-			dec_page_count(sbi, F2FS_DIRTY_META);
-			folio_unlock(folio);
+			f2fs_cache_clear_uptodate(entry);
+			dec_cache_count(sbi, F2FS_DIRTY_META);
+			f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY,
+						F2FS_CACHE_TAG_NONE);
+			f2fs_unlock_cache(entry);
 			return true;
 		}
 		goto redirty_out;
@@ -558,10 +566,10 @@ static bool __f2fs_write_meta_folio(struct folio *folio,
 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 		goto redirty_out;
 
-	f2fs_do_write_meta_page(sbi, folio, io_type);
-	dec_page_count(sbi, F2FS_DIRTY_META);
+	f2fs_do_write_meta_cache(sbi, entry, io_type);
+	dec_cache_count(sbi, F2FS_DIRTY_META);
 
-	folio_unlock(folio);
+	f2fs_unlock_cache(entry);
 
 	if (unlikely(f2fs_cp_error(sbi)))
 		f2fs_submit_merged_write(sbi, META);
@@ -569,101 +577,91 @@ static bool __f2fs_write_meta_folio(struct folio *folio,
 	return true;
 
 redirty_out:
-	folio_redirty_for_writepage(wbc, folio);
+	f2fs_cache_set_dirty(entry);
 	return false;
 }
 
-static int f2fs_write_meta_pages(struct address_space *mapping,
-				struct writeback_control *wbc)
+void f2fs_write_meta_caches(struct f2fs_sb_info *sbi)
 {
-	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
 	struct f2fs_lock_context lc;
-	long diff, written;
+	long nr_to_write = LONG_MAX;
 
 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
-		goto skip_write;
+		return;
 
-	/* collect a number of dirty meta pages and write together */
-	if (wbc->sync_mode != WB_SYNC_ALL &&
-			get_pages(sbi, F2FS_DIRTY_META) <
-					nr_pages_to_skip(sbi, META))
-		goto skip_write;
+	/* collect a number of dirty meta caches and write together */
+	if (get_nr_caches(sbi, F2FS_DIRTY_META) <
+	    nr_caches_to_skip(sbi, META))
+		return;
 
-	/* if locked failed, cp will flush dirty pages instead */
+	/* if locked failed, cp will flush dirty caches instead */
 	if (!f2fs_down_write_trylock_trace(&sbi->cp_global_sem, &lc))
-		goto skip_write;
+		return;
 
-	trace_f2fs_writepages(mapping->host, wbc, META);
-	diff = nr_pages_to_write(sbi, META, wbc);
-	written = f2fs_sync_meta_pages(sbi, wbc->nr_to_write, FS_META_IO);
+	nr_to_write = adjust_flush_cache_number(sbi, META);
+	f2fs_sync_meta_caches(sbi, nr_to_write, false, FS_META_IO);
 	f2fs_up_write_trace(&sbi->cp_global_sem, &lc);
-	wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff);
-	return 0;
-
-skip_write:
-	wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META);
-	trace_f2fs_writepages(mapping->host, wbc, META);
-	return 0;
 }
 
-long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write,
-				enum iostat_type io_type)
+long f2fs_sync_meta_caches(struct f2fs_sb_info *sbi, long nr_to_write,
+				bool sync, enum iostat_type io_type)
 {
-	struct address_space *mapping = META_MAPPING(sbi);
 	pgoff_t index = 0, prev = ULONG_MAX;
-	struct folio_batch fbatch;
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
 	long nwritten = 0;
-	int nr_folios;
-	struct writeback_control wbc = {};
+	int nr;
 	struct blk_plug plug;
+	bool background = nr_to_write != LONG_MAX;
 
-	folio_batch_init(&fbatch);
+	trace_f2fs_write_caches(sbi, nr_to_write, 0, META);
 
 	blk_start_plug(&plug);
 
-	while ((nr_folios = filemap_get_folios_tag(mapping, &index,
-					(pgoff_t)-1,
-					PAGECACHE_TAG_DIRTY, &fbatch))) {
+	while ((nr = f2fs_cache_gang_lookup_tag(META_CACHE(sbi), entries,
+			&index, F2FS_ONSTACK_CACHES, F2FS_CACHE_TAG_DIRTY))) {
 		int i;
 
-		for (i = 0; i < nr_folios; i++) {
-			struct folio *folio = fbatch.folios[i];
+		for (i = 0; i < nr; i++) {
+			struct f2fs_cached_block *entry = entries[i];
 
-			if (nr_to_write != LONG_MAX && i != 0 &&
-					folio->index != prev +
-					folio_nr_pages(fbatch.folios[i-1])) {
-				folio_batch_release(&fbatch);
+			if (background && unlikely(freezing(current))) {
+				f2fs_cache_gang_release(entries, nr);
 				goto stop;
 			}
 
-			folio_lock(folio);
+			if (background && i != 0 &&
+					entry->index != prev + 1) {
+				f2fs_cache_gang_release(entries, nr);
+				goto stop;
+			}
 
-			if (unlikely(!is_meta_folio(folio))) {
+			f2fs_lock_cache(entry);
+
+			if (unlikely(!f2fs_is_meta_cache(entry))) {
 continue_unlock:
-				folio_unlock(folio);
+				f2fs_unlock_cache(entry);
 				continue;
 			}
-			if (!folio_test_dirty(folio)) {
+			if (!f2fs_cache_test_dirty(entry)) {
 				/* someone wrote it for us */
 				goto continue_unlock;
 			}
 
-			f2fs_folio_wait_writeback(folio, META, true, true);
+			f2fs_cache_wait_writeback(entry);
 
-			if (!folio_clear_dirty_for_io(folio))
+			if (!f2fs_cache_test_and_clear_dirty(entry))
 				goto continue_unlock;
 
-			if (!__f2fs_write_meta_folio(folio, &wbc,
-						io_type)) {
-				folio_unlock(folio);
+			if (!__f2fs_write_meta_cache(entry, io_type)) {
+				f2fs_unlock_cache(entry);
 				break;
 			}
-			nwritten += folio_nr_pages(folio);
-			prev = folio->index;
+			nwritten++;
+			prev = entry->index;
 			if (unlikely(nwritten >= nr_to_write))
 				break;
 		}
-		folio_batch_release(&fbatch);
+		f2fs_cache_gang_release(entries, nr);
 		cond_resched();
 	}
 stop:
@@ -672,32 +670,11 @@ long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write,
 
 	blk_finish_plug(&plug);
 
-	return nwritten;
-}
-
-static bool f2fs_dirty_meta_folio(struct address_space *mapping,
-		struct folio *folio)
-{
-	trace_f2fs_set_page_dirty(folio, META);
+	trace_f2fs_write_caches(sbi, nr_to_write, nwritten, META);
 
-	if (!folio_test_uptodate(folio))
-		folio_mark_uptodate(folio);
-	if (filemap_dirty_folio(mapping, folio)) {
-		inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META);
-		folio_set_f2fs_reference(folio);
-		return true;
-	}
-	return false;
+	return nwritten;
 }
 
-const struct address_space_operations f2fs_meta_aops = {
-	.writepages	= f2fs_write_meta_pages,
-	.dirty_folio	= f2fs_dirty_meta_folio,
-	.invalidate_folio = f2fs_invalidate_folio,
-	.release_folio	= f2fs_release_folio,
-	.migrate_folio	= filemap_migrate_folio,
-};
-
 static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino,
 						unsigned int devidx, int type)
 {
@@ -825,15 +802,6 @@ static void __clear_ino_bitmap(struct f2fs_sb_info *sbi, nid_t ino, int type)
 	spin_unlock(&im->ino_lock);
 }
 
-static void f2fs_wait_for_inode_record(struct f2fs_sb_info *sbi, int mode)
-{
-	if (mode != APPEND_INO && mode != UPDATE_INO)
-		return;
-
-	/* Let's wait for some pending updates for APPEND_INO and UPDATE_INO. */
-	flush_workqueue(sbi->evict_wq);
-}
-
 static void __f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino,
 					unsigned int devidx, int type)
 {
@@ -887,8 +855,6 @@ void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all)
 	for (i = all ? ORPHAN_INO : FLUSH_INO; i <= FLUSH_INO; i++) {
 		struct inode_management *im = &sbi->im[i];
 
-		f2fs_wait_for_inode_record(sbi, i);
-
 		spin_lock(&im->ino_lock);
 		list_for_each_entry_safe(e, tmp, &im->ino_list, list) {
 			list_del(&e->list);
@@ -899,6 +865,9 @@ void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all)
 		spin_unlock(&im->ino_lock);
 	}
 
+	/* Wait for pending APPEND/UPDATE inode state updates. */
+	flush_workqueue(sbi->evict_wq);
+
 	for (i = APPEND_INO; i < MAX_INO_ENTRY; i++) {
 		struct inode_management *im = &sbi->im[i];
 
@@ -964,7 +933,7 @@ void f2fs_add_orphan_inode(struct inode *inode)
 {
 	/* add new orphan ino entry into list */
 	f2fs_add_ino_entry(F2FS_I_SB(inode), inode->i_ino, ORPHAN_INO);
-	f2fs_update_inode_page(inode);
+	f2fs_update_inode_cache(inode);
 }
 
 void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
@@ -1037,28 +1006,30 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi)
 	start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi);
 	orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi);
 
-	f2fs_ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true);
+	f2fs_ra_meta_caches(sbi, start_blk, orphan_blocks, META_CP, true);
 
 	for (i = 0; i < orphan_blocks; i++) {
-		struct folio *folio;
+		struct f2fs_cached_block *entry;
 		struct f2fs_orphan_block *orphan_blk;
+		struct f2fs_orphan_footer *footer;
 		unsigned int entry_count;
 
-		folio = f2fs_get_meta_folio(sbi, start_blk + i);
-		if (IS_ERR(folio)) {
-			err = PTR_ERR(folio);
+		entry = f2fs_get_meta_cache(sbi, start_blk + i);
+		if (IS_ERR(entry)) {
+			err = PTR_ERR(entry);
 			goto out;
 		}
 
-		orphan_blk = folio_address(folio);
-		entry_count = le32_to_cpu(orphan_blk->entry_count);
-		if (entry_count > F2FS_ORPHANS_PER_BLOCK) {
+		orphan_blk = cache_address(entry);
+		footer = f2fs_orphan_footer(orphan_blk, sbi);
+		entry_count = le32_to_cpu(footer->entry_count);
+		if (entry_count > F2FS_ORPHANS_PER_BLOCK(sbi)) {
 			f2fs_err(sbi, "invalid orphan inode entry count %u",
 				 entry_count);
 			set_sbi_flag(sbi, SBI_NEED_FSCK);
 			f2fs_handle_error(sbi, ERROR_INCONSISTENT_ORPHAN);
 			err = -EFSCORRUPTED;
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 			goto out;
 		}
 
@@ -1067,11 +1038,11 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi)
 
 			err = recover_orphan_inode(sbi, ino);
 			if (err) {
-				f2fs_folio_put(folio, true);
+				f2fs_put_cache(entry, true);
 				goto out;
 			}
 		}
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 	}
 	/* clear Orphan Flag */
 	clear_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG);
@@ -1085,96 +1056,99 @@ static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
 {
 	struct list_head *head;
 	struct f2fs_orphan_block *orphan_blk = NULL;
+	struct f2fs_orphan_footer *footer = NULL;
 	unsigned int nentries = 0;
 	unsigned short index = 1;
 	unsigned short orphan_blocks;
-	struct folio *folio = NULL;
 	struct ino_entry *orphan = NULL;
 	struct inode_management *im = &sbi->im[ORPHAN_INO];
+	struct f2fs_cached_block *entry = NULL;
 
-	orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num);
+	orphan_blocks = GET_ORPHAN_BLOCKS(sbi, im->ino_num);
 
 	/*
 	 * we don't need to do spin_lock(&im->ino_lock) here, since all the
 	 * orphan inode operations are covered under f2fs_lock_op().
-	 * And, spin_lock should be avoided due to page operations below.
+	 * And, spin_lock should be avoided due to cache operations below.
 	 */
 	head = &im->ino_list;
 
 	/* loop for each orphan inode entry and write them in journal block */
 	list_for_each_entry(orphan, head, list) {
-		if (!folio) {
-			folio = f2fs_grab_meta_folio(sbi, start_blk++);
-			orphan_blk = folio_address(folio);
-			memset(orphan_blk, 0, sizeof(*orphan_blk));
+		if (!entry) {
+			entry = f2fs_grab_meta_cache(sbi, start_blk++);
+			orphan_blk = cache_address(entry);
+			footer = f2fs_orphan_footer(orphan_blk, sbi);
+			memset(orphan_blk, 0, sbi->blocksize);
 		}
 
 		orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino);
 
-		if (nentries == F2FS_ORPHANS_PER_BLOCK) {
+		if (nentries == F2FS_ORPHANS_PER_BLOCK(sbi)) {
 			/*
-			 * an orphan block is full of 1020 entries,
+			 * an orphan block is full,
 			 * then we need to flush current orphan blocks
 			 * and bring another one in memory
 			 */
-			orphan_blk->blk_addr = cpu_to_le16(index);
-			orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
-			orphan_blk->entry_count = cpu_to_le32(nentries);
-			folio_mark_dirty(folio);
-			f2fs_folio_put(folio, true);
+			footer->blk_addr = cpu_to_le16(index);
+			footer->blk_count = cpu_to_le16(orphan_blocks);
+			footer->entry_count = cpu_to_le32(nentries);
+			f2fs_mark_cache_dirty(entry);
+			f2fs_put_cache(entry, true);
 			index++;
 			nentries = 0;
-			folio = NULL;
+			entry = NULL;
 		}
 	}
 
-	if (folio) {
-		orphan_blk->blk_addr = cpu_to_le16(index);
-		orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
-		orphan_blk->entry_count = cpu_to_le32(nentries);
-		folio_mark_dirty(folio);
-		f2fs_folio_put(folio, true);
+	if (entry) {
+		footer->blk_addr = cpu_to_le16(index);
+		footer->blk_count = cpu_to_le16(orphan_blocks);
+		footer->entry_count = cpu_to_le32(nentries);
+		f2fs_mark_cache_dirty(entry);
+		f2fs_put_cache(entry, true);
 	}
 }
 
-static __u32 f2fs_checkpoint_chksum(struct f2fs_checkpoint *ckpt)
+static __u32 f2fs_checkpoint_chksum(struct f2fs_sb_info *sbi,
+				     struct f2fs_checkpoint *ckpt)
 {
 	unsigned int chksum_ofs = le32_to_cpu(ckpt->checksum_offset);
 	__u32 chksum;
 
 	chksum = f2fs_crc32(ckpt, chksum_ofs);
-	if (chksum_ofs < CP_CHKSUM_OFFSET) {
+	if (chksum_ofs < CP_CHKSUM_OFFSET(sbi)) {
 		chksum_ofs += sizeof(chksum);
 		chksum = f2fs_chksum(chksum, (__u8 *)ckpt + chksum_ofs,
-				     F2FS_BLKSIZE - chksum_ofs);
+					F2FS_BLKSIZE(sbi) - chksum_ofs);
 	}
 	return chksum;
 }
 
 static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr,
-		struct f2fs_checkpoint **cp_block, struct folio **cp_folio,
+		struct f2fs_checkpoint **cp_block, struct f2fs_cached_block **cp_entry,
 		unsigned long long *version)
 {
 	size_t crc_offset = 0;
 	__u32 crc;
 
-	*cp_folio = f2fs_get_meta_folio(sbi, cp_addr);
-	if (IS_ERR(*cp_folio))
-		return PTR_ERR(*cp_folio);
+	*cp_entry = f2fs_get_meta_cache(sbi, cp_addr);
+	if (IS_ERR(*cp_entry))
+		return PTR_ERR(*cp_entry);
 
-	*cp_block = folio_address(*cp_folio);
+	*cp_block = cache_address(*cp_entry);
 
 	crc_offset = le32_to_cpu((*cp_block)->checksum_offset);
 	if (crc_offset < CP_MIN_CHKSUM_OFFSET ||
-			crc_offset > CP_CHKSUM_OFFSET) {
-		f2fs_folio_put(*cp_folio, true);
+			crc_offset > CP_CHKSUM_OFFSET(sbi)) {
+		f2fs_put_cache(*cp_entry, true);
 		f2fs_warn(sbi, "invalid crc_offset: %zu", crc_offset);
 		return -EINVAL;
 	}
 
-	crc = f2fs_checkpoint_chksum(*cp_block);
+	crc = f2fs_checkpoint_chksum(sbi, *cp_block);
 	if (crc != cur_cp_crc(*cp_block)) {
-		f2fs_folio_put(*cp_folio, true);
+		f2fs_put_cache(*cp_entry, true);
 		f2fs_warn(sbi, "invalid crc value");
 		return -EINVAL;
 	}
@@ -1183,17 +1157,17 @@ static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr,
 	return 0;
 }
 
-static struct folio *validate_checkpoint(struct f2fs_sb_info *sbi,
+static struct f2fs_cached_block *validate_checkpoint(struct f2fs_sb_info *sbi,
 				block_t cp_addr, unsigned long long *version)
 {
-	struct folio *cp_folio_1 = NULL, *cp_folio_2 = NULL;
+	struct f2fs_cached_block *cp_entry_1 = NULL, *cp_entry_2 = NULL;
 	struct f2fs_checkpoint *cp_block = NULL;
 	unsigned long long cur_version = 0, pre_version = 0;
 	unsigned int cp_blocks;
 	int err;
 
 	err = get_checkpoint_version(sbi, cp_addr, &cp_block,
-					&cp_folio_1, version);
+					&cp_entry_1, version);
 	if (err)
 		return NULL;
 
@@ -1208,19 +1182,19 @@ static struct folio *validate_checkpoint(struct f2fs_sb_info *sbi,
 
 	cp_addr += cp_blocks - 1;
 	err = get_checkpoint_version(sbi, cp_addr, &cp_block,
-					&cp_folio_2, version);
+					&cp_entry_2, version);
 	if (err)
 		goto invalid_cp;
 	cur_version = *version;
 
 	if (cur_version == pre_version) {
 		*version = cur_version;
-		f2fs_folio_put(cp_folio_2, true);
-		return cp_folio_1;
+		f2fs_put_cache(cp_entry_2, true);
+		return cp_entry_1;
 	}
-	f2fs_folio_put(cp_folio_2, true);
+	f2fs_put_cache(cp_entry_2, true);
 invalid_cp:
-	f2fs_folio_put(cp_folio_1, true);
+	f2fs_put_cache(cp_entry_1, true);
 	return NULL;
 }
 
@@ -1228,7 +1202,7 @@ int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_checkpoint *cp_block;
 	struct f2fs_super_block *fsb = sbi->raw_super;
-	struct folio *cp1, *cp2, *cur_folio;
+	struct f2fs_cached_block *cp1 = NULL, *cp2 = NULL, *cur_entry;
 	unsigned long blk_size = sbi->blocksize;
 	unsigned long long cp1_version = 0, cp2_version = 0;
 	unsigned long long cp_start_blk_no;
@@ -1255,22 +1229,22 @@ int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi)
 
 	if (cp1 && cp2) {
 		if (ver_after(cp2_version, cp1_version))
-			cur_folio = cp2;
+			cur_entry = cp2;
 		else
-			cur_folio = cp1;
+			cur_entry = cp1;
 	} else if (cp1) {
-		cur_folio = cp1;
+		cur_entry = cp1;
 	} else if (cp2) {
-		cur_folio = cp2;
+		cur_entry = cp2;
 	} else {
 		err = -EFSCORRUPTED;
 		goto fail_no_cp;
 	}
 
-	cp_block = folio_address(cur_folio);
+	cp_block = cache_address(cur_entry);
 	memcpy(sbi->ckpt, cp_block, blk_size);
 
-	if (cur_folio == cp1)
+	if (cur_entry == cp1)
 		sbi->cur_cp_pack = 1;
 	else
 		sbi->cur_cp_pack = 2;
@@ -1285,30 +1259,31 @@ int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi)
 		goto done;
 
 	cp_blk_no = le32_to_cpu(fsb->cp_blkaddr);
-	if (cur_folio == cp2)
+	if (cur_entry == cp2)
 		cp_blk_no += BIT(le32_to_cpu(fsb->log_blocks_per_seg));
 
 	for (i = 1; i < cp_blks; i++) {
+		struct f2fs_cached_block *cur_entry_payload;
 		void *sit_bitmap_ptr;
 		unsigned char *ckpt = (unsigned char *)sbi->ckpt;
 
-		cur_folio = f2fs_get_meta_folio(sbi, cp_blk_no + i);
-		if (IS_ERR(cur_folio)) {
-			err = PTR_ERR(cur_folio);
+		cur_entry_payload = f2fs_get_meta_cache(sbi, cp_blk_no + i);
+		if (IS_ERR(cur_entry_payload)) {
+			err = PTR_ERR(cur_entry_payload);
 			goto free_fail_no_cp;
 		}
-		sit_bitmap_ptr = folio_address(cur_folio);
+		sit_bitmap_ptr = cache_address(cur_entry_payload);
 		memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size);
-		f2fs_folio_put(cur_folio, true);
+		f2fs_put_cache(cur_entry_payload, true);
 	}
 done:
-	f2fs_folio_put(cp1, true);
-	f2fs_folio_put(cp2, true);
+	f2fs_put_cache(cp1, true);
+	f2fs_put_cache(cp2, true);
 	return 0;
 
 free_fail_no_cp:
-	f2fs_folio_put(cp1, true);
-	f2fs_folio_put(cp2, true);
+	f2fs_put_cache(cp1, true);
+	f2fs_put_cache(cp2, true);
 fail_no_cp:
 	kvfree(sbi->ckpt);
 	return err;
@@ -1384,12 +1359,12 @@ int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
 	unsigned long ino = 0;
 
 	trace_f2fs_sync_dirty_inodes_enter(sbi->sb, is_dir,
-				get_pages(sbi, is_dir ?
+				get_nr_caches(sbi, is_dir ?
 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
 retry:
 	if (unlikely(f2fs_cp_error(sbi))) {
 		trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
-				get_pages(sbi, is_dir ?
+				get_nr_caches(sbi, is_dir ?
 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
 		return -EIO;
 	}
@@ -1400,11 +1375,12 @@ int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
 	if (list_empty(head)) {
 		spin_unlock(&sbi->inode_lock[type]);
 		trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
-				get_pages(sbi, is_dir ?
+				get_nr_caches(sbi, is_dir ?
 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
 		return 0;
 	}
 	fi = list_first_entry(head, struct f2fs_inode_info, dirty_list);
+	list_move_tail(&fi->dirty_list, head);
 	inode = igrab(&fi->vfs_inode);
 	spin_unlock(&sbi->inode_lock[type]);
 	if (inode) {
@@ -1427,11 +1403,6 @@ int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
 		else
 			ino = cur_ino;
 	} else {
-		/*
-		 * We should submit bio, since it exists several
-		 * writebacking dentry pages in the freeing inode.
-		 */
-		f2fs_submit_merged_write(sbi, DATA);
 		cond_resched();
 	}
 	goto retry;
@@ -1442,7 +1413,7 @@ static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
 	struct list_head *head = &sbi->inode_list[DIRTY_META];
 	struct inode *inode;
 	struct f2fs_inode_info *fi;
-	s64 total = get_pages(sbi, F2FS_DIRTY_IMETA);
+	s64 total = get_nr_caches(sbi, F2FS_DIRTY_IMETA);
 
 	while (total--) {
 		if (unlikely(f2fs_cp_error(sbi)))
@@ -1455,6 +1426,7 @@ static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
 		}
 		fi = list_first_entry(head, struct f2fs_inode_info,
 							gdirty_list);
+		list_move_tail(&fi->gdirty_list, head);
 		inode = igrab(&fi->vfs_inode);
 		spin_unlock(&sbi->inode_lock[DIRTY_META]);
 		if (inode) {
@@ -1462,8 +1434,10 @@ static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
 
 			/* it's on eviction */
 			if (is_inode_flag_set(inode, FI_DIRTY_INODE))
-				f2fs_update_inode_page(inode);
+				f2fs_update_inode_cache(inode);
 			iput(inode);
+		} else {
+			cond_resched();
 		}
 	}
 	return 0;
@@ -1503,7 +1477,7 @@ static bool __need_flush_quota(struct f2fs_sb_info *sbi)
 	} else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) {
 		clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
 		ret = true;
-	} else if (get_pages(sbi, F2FS_DIRTY_QDATA)) {
+	} else if (get_nr_caches(sbi, F2FS_DIRTY_QDATA)) {
 		ret = true;
 	}
 	f2fs_up_write(&sbi->quota_sem);
@@ -1515,14 +1489,10 @@ static bool __need_flush_quota(struct f2fs_sb_info *sbi)
  */
 static int block_operations(struct f2fs_sb_info *sbi)
 {
-	struct writeback_control wbc = {
-		.sync_mode = WB_SYNC_ALL,
-		.nr_to_write = LONG_MAX,
-	};
 	int err = 0, cnt = 0;
 
 	/*
-	 * Let's flush inline_data in dirty node pages.
+	 * Let's flush inline_data in dirty node caches.
 	 */
 	f2fs_flush_inline_data(sbi);
 
@@ -1551,7 +1521,7 @@ static int block_operations(struct f2fs_sb_info *sbi)
 
 retry_flush_dents:
 	/* write all the dirty dentry pages */
-	if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
+	if (get_nr_caches(sbi, F2FS_DIRTY_DENTS)) {
 		f2fs_unlock_all(sbi);
 		err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true);
 		if (err)
@@ -1561,12 +1531,12 @@ static int block_operations(struct f2fs_sb_info *sbi)
 	}
 
 	/*
-	 * POR: we should ensure that there are no dirty node pages
+	 * POR: we should ensure that there are no dirty node caches
 	 * until finishing nat/sit flush. inode->i_blocks can be updated.
 	 */
 	f2fs_down_write(&sbi->node_change);
 
-	if (get_pages(sbi, F2FS_DIRTY_IMETA)) {
+	if (get_nr_caches(sbi, F2FS_DIRTY_IMETA)) {
 		f2fs_up_write(&sbi->node_change);
 		f2fs_unlock_all(sbi);
 		err = f2fs_sync_inode_meta(sbi);
@@ -1579,10 +1549,11 @@ static int block_operations(struct f2fs_sb_info *sbi)
 retry_flush_nodes:
 	f2fs_down_write(&sbi->node_write);
 
-	if (get_pages(sbi, F2FS_DIRTY_NODES)) {
+	if (get_nr_caches(sbi, F2FS_DIRTY_NODES)) {
 		f2fs_up_write(&sbi->node_write);
 		atomic_inc(&sbi->wb_sync_req[NODE]);
-		err = f2fs_sync_node_pages(sbi, &wbc, false, FS_CP_NODE_IO);
+		err = f2fs_writeback_node_caches(sbi, LONG_MAX,
+			true, false, FS_CP_NODE_IO);
 		atomic_dec(&sbi->wb_sync_req[NODE]);
 		if (err) {
 			f2fs_up_write(&sbi->node_change);
@@ -1608,12 +1579,12 @@ static void unblock_operations(struct f2fs_sb_info *sbi)
 	f2fs_unlock_all(sbi);
 }
 
-void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type)
+void f2fs_sync_dirty_data(struct f2fs_sb_info *sbi, int type)
 {
 	DEFINE_WAIT(wait);
 
 	for (;;) {
-		if (!get_pages(sbi, type))
+		if (!get_nr_caches(sbi, type))
 			break;
 
 		if (unlikely(f2fs_cp_error(sbi) &&
@@ -1621,7 +1592,7 @@ void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type)
 			break;
 
 		if (type == F2FS_DIRTY_META)
-			f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO);
+			f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_META_IO);
 		else if (type == F2FS_WB_CP_DATA)
 			f2fs_submit_merged_write(sbi, DATA);
 
@@ -1700,31 +1671,24 @@ static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 static void commit_checkpoint(struct f2fs_sb_info *sbi,
 	void *src, block_t blk_addr)
 {
-	struct writeback_control wbc = {};
+	struct f2fs_cached_block *entry = f2fs_grab_meta_cache(sbi, blk_addr);
 
-	/*
-	 * filemap_get_folios_tag and folio_lock again will take
-	 * some extra time. Therefore, f2fs_update_meta_pages and
-	 * f2fs_sync_meta_pages are combined in this function.
-	 */
-	struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr);
+	memcpy(cache_address(entry), src, F2FS_BLKSIZE(sbi));
 
-	memcpy(folio_address(folio), src, PAGE_SIZE);
-
-	folio_mark_dirty(folio);
-	if (unlikely(!folio_clear_dirty_for_io(folio)))
+	f2fs_mark_cache_dirty(entry);
+	if (unlikely(!f2fs_cache_test_and_clear_dirty(entry)))
 		f2fs_bug_on(sbi, 1);
 
-	/* writeout cp pack 2 page */
-	if (unlikely(!__f2fs_write_meta_folio(folio, &wbc, FS_CP_META_IO))) {
+	/* writeout cp pack 2 cache */
+	if (unlikely(!__f2fs_write_meta_cache(entry, FS_CP_META_IO))) {
 		if (f2fs_cp_error(sbi)) {
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 			return;
 		}
 		f2fs_bug_on(sbi, true);
 	}
 
-	f2fs_folio_put(folio, false);
+	f2fs_put_cache(entry, false);
 
 	/* submit checkpoint (with barrier if NOBARRIER is not set) */
 	f2fs_submit_merged_write(sbi, META_FLUSH);
@@ -1792,8 +1756,8 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 	u64 kbytes_written;
 	int err;
 
-	/* Flush all the NAT/SIT pages */
-	f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO);
+	/* Flush all the NAT/SIT caches */
+	f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_CP_META_IO);
 
 	stat_cp_time(cpc, CP_TIME_SYNC_META);
 
@@ -1816,7 +1780,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 	}
 
 	/* 2 cp + n data seg summary + orphan inode blocks */
-	data_sum_blocks = f2fs_npages_for_summary_flush(sbi, false);
+	data_sum_blocks = f2fs_nblocks_for_summary_flush(sbi, false);
 	spin_lock_irqsave(&sbi->cp_lock, flags);
 	if (data_sum_blocks < NR_CURSEG_DATA_TYPE)
 		__set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
@@ -1824,7 +1788,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 		__clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
 	spin_unlock_irqrestore(&sbi->cp_lock, flags);
 
-	orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num);
+	orphan_blocks = GET_ORPHAN_BLOCKS(sbi, orphan_num);
 	ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks +
 			orphan_blocks);
 
@@ -1844,7 +1808,7 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 	get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP));
 	get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP));
 
-	crc32 = f2fs_checkpoint_chksum(ckpt);
+	crc32 = f2fs_checkpoint_chksum(sbi, ckpt);
 	*((__le32 *)((unsigned char *)ckpt +
 				le32_to_cpu(ckpt->checksum_offset)))
 				= cpu_to_le32(crc32);
@@ -1861,16 +1825,16 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 
 		blk = start_blk + BLKS_PER_SEG(sbi) - nm_i->nat_bits_blocks;
 		for (i = 0; i < nm_i->nat_bits_blocks; i++)
-			f2fs_update_meta_page(sbi, nm_i->nat_bits +
-					F2FS_BLK_TO_BYTES(i), blk + i);
+			f2fs_update_meta_block(sbi, nm_i->nat_bits +
+					F2FS_BLK_TO_BYTES(sbi, i), blk + i);
 	}
 
 	/* write out checkpoint buffer at block 0 */
-	f2fs_update_meta_page(sbi, ckpt, start_blk++);
+	f2fs_update_meta_block(sbi, ckpt, start_blk++);
 
 	for (i = 1; i < 1 + cp_payload_blks; i++)
-		f2fs_update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE,
-							start_blk++);
+		f2fs_update_meta_block(sbi, (char *)ckpt +
+				i * F2FS_BLKSIZE(sbi), start_blk++);
 
 	if (orphan_num) {
 		write_orphan_inodes(sbi, start_blk);
@@ -1891,16 +1855,16 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 		start_blk += NR_CURSEG_NODE_TYPE;
 	}
 
-	/* Here, we have one bio having CP pack except cp pack 2 page */
-	f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO);
+	/* Here, we have one bio having CP pack except cp pack 2 blocks */
+	f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_CP_META_IO);
 	stat_cp_time(cpc, CP_TIME_SYNC_CP_META);
 
-	/* Wait for all dirty meta pages to be submitted for IO */
-	f2fs_wait_on_all_pages(sbi, F2FS_DIRTY_META);
+	/* Wait for all dirty meta blocks to be submitted for IO */
+	f2fs_sync_dirty_data(sbi, F2FS_DIRTY_META);
 	stat_cp_time(cpc, CP_TIME_WAIT_DIRTY_META);
 
-	/* wait for previous submitted meta pages writeback */
-	f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
+	/* wait for previous submitted meta blocks writeback */
+	f2fs_sync_dirty_data(sbi, F2FS_WB_CP_DATA);
 	stat_cp_time(cpc, CP_TIME_WAIT_CP_DATA);
 
 	/* flush all device cache */
@@ -1909,20 +1873,20 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 		return err;
 	stat_cp_time(cpc, CP_TIME_FLUSH_DEVICE);
 
-	/* barrier and flush checkpoint cp pack 2 page if it can */
+	/* barrier and flush checkpoint cp pack 2 blocks if it can */
 	commit_checkpoint(sbi, ckpt, start_blk);
-	f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
+	f2fs_sync_dirty_data(sbi, F2FS_WB_CP_DATA);
 	stat_cp_time(cpc, CP_TIME_WAIT_LAST_CP);
 
 	/*
-	 * invalidate intermediate page cache borrowed from meta inode which are
-	 * used for migration of encrypted, verity or compressed inode's blocks.
+	 * invalidate intermediate meta cache used for migration of
+	 * encrypted, verity or compressed inode's blocks.
 	 */
 	if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) ||
-		f2fs_sb_has_compression(sbi))
-		f2fs_bug_on(sbi,
-			invalidate_inode_pages2_range(META_MAPPING(sbi),
-				MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1));
+		f2fs_sb_has_compression(sbi)) {
+		f2fs_truncate_meta_caches(sbi, MAIN_BLKADDR(sbi),
+				MAX_BLKADDR(sbi) - MAIN_BLKADDR(sbi));
+	}
 
 	f2fs_release_ino_entry(sbi, false);
 
@@ -1939,14 +1903,14 @@ static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 	__set_cp_next_pack(sbi);
 
 	/*
-	 * redirty superblock if metadata like node page or inode cache is
+	 * redirty superblock if metadata like node caches or inode cache is
 	 * updated during writing checkpoint.
 	 */
-	if (get_pages(sbi, F2FS_DIRTY_NODES) ||
-			get_pages(sbi, F2FS_DIRTY_IMETA))
+	if (get_nr_caches(sbi, F2FS_DIRTY_NODES) ||
+			get_nr_caches(sbi, F2FS_DIRTY_IMETA))
 		set_sbi_flag(sbi, SBI_IS_DIRTY);
 
-	f2fs_bug_on(sbi, get_pages(sbi, F2FS_DIRTY_DENTS));
+	f2fs_bug_on(sbi, get_nr_caches(sbi, F2FS_DIRTY_DENTS));
 
 	return unlikely(f2fs_cp_error(sbi)) ? -EIO : 0;
 }
@@ -2080,7 +2044,7 @@ void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi)
 
 	sbi->max_orphans = (BLKS_PER_SEG(sbi) - F2FS_CP_PACKS -
 			NR_CURSEG_PERSIST_TYPE - __cp_payload(sbi)) *
-			F2FS_ORPHANS_PER_BLOCK;
+			F2FS_ORPHANS_PER_BLOCK(sbi);
 }
 
 int __init f2fs_create_checkpoint_caches(void)
diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
index ce88092d9ce26d..f10c753800271a 100644
--- a/fs/f2fs/compress.c
+++ b/fs/f2fs/compress.c
@@ -807,7 +807,7 @@ void f2fs_end_read_compressed_page(struct folio *folio, bool failed,
 	struct decompress_io_ctx *dic = folio->private;
 	struct f2fs_sb_info *sbi = dic->sbi;
 
-	dec_page_count(sbi, F2FS_RD_DATA);
+	dec_cache_count(sbi, F2FS_RD_DATA);
 
 	if (failed)
 		WRITE_ONCE(dic->failed, true);
@@ -909,7 +909,7 @@ bool f2fs_sanity_check_cluster(struct dnode_of_data *dn)
 	}
 
 	for (i = 1, count = 1; i < cluster_size; i++, count++) {
-		block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+		block_t blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 							dn->ofs_in_node + i);
 
 		/* [COMPR_ADDR, ..., COMPR_ADDR] */
@@ -950,7 +950,7 @@ static int __f2fs_get_cluster_blocks(struct inode *inode,
 	int count, i;
 
 	for (i = 0, count = 0; i < cluster_size; i++) {
-		block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+		block_t blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 							dn->ofs_in_node + i);
 
 		if (__is_valid_data_blkaddr(blkaddr))
@@ -1064,13 +1064,15 @@ static void cancel_cluster_writeback(struct compress_ctx *cc,
 
 	/* Cancel writeback and stay locked. */
 	for (i = 0; i < cc->cluster_size; i++) {
+		struct folio *folio = page_folio(cc->rpages[i]);
+
 		if (i < submitted) {
 			inode_inc_dirty_pages(cc->inode);
-			lock_page(cc->rpages[i]);
+			folio_lock(folio);
 		}
-		clear_page_private_gcing(cc->rpages[i]);
-		if (folio_test_writeback(page_folio(cc->rpages[i])))
-			end_page_writeback(cc->rpages[i]);
+		folio_clear_f2fs_gcing(folio);
+		if (folio_test_writeback(folio))
+			folio_end_writeback(folio);
 	}
 }
 
@@ -1078,11 +1080,15 @@ static void set_cluster_dirty(struct compress_ctx *cc)
 {
 	int i;
 
-	for (i = 0; i < cc->cluster_size; i++)
-		if (cc->rpages[i]) {
-			set_page_dirty(cc->rpages[i]);
-			set_page_private_gcing(cc->rpages[i]);
-		}
+	for (i = 0; i < cc->cluster_size; i++) {
+		struct folio *folio;
+
+		if (!cc->rpages[i])
+			continue;
+		folio = page_folio(cc->rpages[i]);
+		folio_mark_dirty(folio);
+		folio_set_f2fs_gcing(folio);
+	}
 }
 
 static int prepare_compress_overwrite(struct compress_ctx *cc,
@@ -1146,11 +1152,11 @@ static int prepare_compress_overwrite(struct compress_ctx *cc,
 			goto release_and_retry;
 		}
 
-		f2fs_folio_wait_writeback(folio, DATA, true, true);
+		f2fs_folio_wait_writeback(folio, true, true);
 		f2fs_compress_ctx_add_page(cc, folio);
 
 		if (!folio_test_uptodate(folio)) {
-			f2fs_handle_page_eio(sbi, folio, DATA);
+			f2fs_handle_page_eio(sbi, folio->index, DATA);
 release_and_retry:
 			f2fs_put_rpages(cc);
 			f2fs_unlock_rpages(cc, i + 1);
@@ -1222,6 +1228,7 @@ int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock)
 	int i;
 	int err;
 
+repeat:
 	err = f2fs_is_compressed_cluster(inode, start_idx);
 	if (err < 0)
 		return err;
@@ -1233,12 +1240,11 @@ int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock)
 	/* truncate compressed cluster */
 	err = f2fs_prepare_compress_overwrite(inode, &pagep,
 						start_idx, &fsdata);
-
-	/* should not be a normal cluster */
-	f2fs_bug_on(F2FS_I_SB(inode), err == 0);
-
-	if (err <= 0)
+	if (err < 0)
 		return err;
+	else if (err == 0)
+		/* the cluster became non-compressed one due to race case */
+		goto repeat;
 
 	rpages = fsdata;
 
@@ -1281,7 +1287,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
 		.op = REQ_OP_WRITE,
 		.op_flags = wbc_to_write_flags(wbc),
 		.old_blkaddr = NEW_ADDR,
-		.page = NULL,
+		.folio = NULL,
 		.encrypted_page = NULL,
 		.compressed_page = NULL,
 		.io_type = io_type,
@@ -1322,7 +1328,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
 		goto out_unlock_op;
 
 	for (i = 0; i < cc->cluster_size; i++) {
-		if (data_blkaddr(dn.inode, dn.node_folio,
+		if (data_blkaddr(dn.inode, dn.node_entry,
 					dn.ofs_in_node + i) == NULL_ADDR)
 			goto out_put_dnode;
 	}
@@ -1354,10 +1360,10 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
 				page_folio(cc->rpages[i + 1])->index, cic);
 		fio.compressed_page = cc->cpages[i];
 
-		fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_folio,
+		fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_entry,
 						dn.ofs_in_node + i + 1);
 
-		/* wait for GCed page writeback via META_MAPPING */
+		/* wait for GCed page writeback via generic cache */
 		f2fs_wait_on_block_writeback(inode, fio.old_blkaddr);
 	}
 
@@ -1370,7 +1376,7 @@ static int f2fs_write_compressed_pages(struct compress_ctx *cc,
 		block_t blkaddr;
 
 		blkaddr = f2fs_data_blkaddr(&dn);
-		fio.page = cc->rpages[i];
+		fio.folio = page_folio(cc->rpages[i]);
 		fio.old_blkaddr = blkaddr;
 
 		/* cluster header */
@@ -1471,26 +1477,29 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio)
 	f2fs_compress_free_page(page);
 
 	if (atomic_dec_return(&cic->pending_pages)) {
-		dec_page_count(sbi, type);
+		dec_cache_count(sbi, type);
 		return;
 	}
 
 	for (i = 0; i < cic->nr_rpages; i++) {
+		struct folio *rfolio;
+
 		WARN_ON(!cic->rpages[i]);
-		clear_page_private_gcing(cic->rpages[i]);
-		end_page_writeback(cic->rpages[i]);
+		rfolio = page_folio(cic->rpages[i]);
+		folio_clear_f2fs_gcing(rfolio);
+		folio_end_writeback(rfolio);
 	}
 
 	page_array_free(sbi, cic->rpages, cic->nr_rpages);
 	kmem_cache_free(cic_entry_slab, cic);
 
 	/*
-	 * Make sure dec_page_count() is the last access to sbi.
+	 * Make sure dec_cache_count() is the last access to sbi.
 	 * Once it drops the F2FS_WB_CP_DATA counter to zero, the
 	 * unmount thread can proceed to destroy sbi and
 	 * sbi->page_array_slab.
 	 */
-	dec_page_count(sbi, type);
+	dec_cache_count(sbi, type);
 }
 
 static int f2fs_write_raw_pages(struct compress_ctx *cc,
@@ -1542,7 +1551,7 @@ static int f2fs_write_raw_pages(struct compress_ctx *cc,
 		if (folio_test_writeback(folio)) {
 			if (wbc->sync_mode == WB_SYNC_NONE)
 				goto continue_unlock;
-			f2fs_folio_wait_writeback(folio, DATA, true, true);
+			f2fs_folio_wait_writeback(folio, true, true);
 		}
 
 		if (!folio_clear_dirty_for_io(folio))
@@ -1874,14 +1883,14 @@ void f2fs_put_folio_dic(struct folio *folio, bool in_task)
 unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn,
 						unsigned int ofs_in_node)
 {
-	bool compressed = data_blkaddr(dn->inode, dn->node_folio,
+	bool compressed = data_blkaddr(dn->inode, dn->node_entry,
 					ofs_in_node) == COMPRESS_ADDR;
 	int i = compressed ? 1 : 0;
-	block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+	block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 							ofs_in_node + i);
 
 	for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) {
-		block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+		block_t blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 							ofs_in_node + i);
 
 		if (!__is_valid_data_blkaddr(blkaddr))
@@ -1893,30 +1902,19 @@ unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn,
 	return compressed ? i - 1 : i;
 }
 
-const struct address_space_operations f2fs_compress_aops = {
-	.release_folio = f2fs_release_folio,
-	.invalidate_folio = f2fs_invalidate_folio,
-	.migrate_folio	= filemap_migrate_folio,
-};
-
-struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi)
-{
-	return sbi->compress_inode->i_mapping;
-}
-
 void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi,
 				block_t blkaddr, unsigned int len)
 {
-	if (!sbi->compress_inode)
+	if (!test_opt(sbi, COMPRESS_CACHE))
 		return;
-	invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr + len - 1);
+
+	f2fs_drop_cache_range(COMPRESS_CACHE(sbi), blkaddr, len, false);
 }
 
 static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
 		struct folio *folio, nid_t ino, block_t blkaddr)
 {
-	struct folio *cfolio;
-	int ret;
+	struct f2fs_cached_block *entry;
 
 	if (!test_opt(sbi, COMPRESS_CACHE))
 		return;
@@ -1924,52 +1922,48 @@ static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi,
 	if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ))
 		return;
 
-	if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE))
+	if (!f2fs_available_free_memory(sbi, COMPRESS_BLOCK))
 		return;
 
-	cfolio = filemap_get_folio(COMPRESS_MAPPING(sbi), blkaddr);
-	if (!IS_ERR(cfolio)) {
-		f2fs_folio_put(cfolio, false);
+	entry = f2fs_find_cache(COMPRESS_CACHE(sbi), blkaddr,
+						F2FS_CACHE_ACCESS);
+	if (!IS_ERR(entry)) {
+		f2fs_put_cache(entry, false);
 		return;
 	}
 
-	cfolio = filemap_alloc_folio(__GFP_NOWARN | __GFP_IO, 0, NULL);
-	if (!cfolio)
+	entry = f2fs_grab_cache(COMPRESS_CACHE(sbi), blkaddr,
+					F2FS_CACHE_LOCK_CREATE);
+	if (IS_ERR(entry))
 		return;
 
-	ret = filemap_add_folio(COMPRESS_MAPPING(sbi), cfolio,
-						blkaddr, GFP_NOFS);
-	if (ret) {
-		f2fs_folio_put(cfolio, false);
-		return;
-	}
-
-	folio_set_f2fs_data(cfolio, ino);
-
-	memcpy(folio_address(cfolio), folio_address(folio), PAGE_SIZE);
-	folio_mark_uptodate(cfolio);
-	f2fs_folio_put(cfolio, true);
+	entry->ino = ino;
+	memcpy(cache_address(entry), folio_address(folio), sbi->blocksize);
+	f2fs_cache_set_uptodate(entry);
+	f2fs_put_cache(entry, true);
 }
 
 bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio,
 								block_t blkaddr)
 {
-	struct folio *cfolio;
+	struct f2fs_cached_block *entry;
 	bool hitted = false;
 
 	if (!test_opt(sbi, COMPRESS_CACHE))
 		return false;
 
-	cfolio = f2fs_filemap_get_folio(COMPRESS_MAPPING(sbi),
-				blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS);
-	if (!IS_ERR(cfolio)) {
-		if (folio_test_uptodate(cfolio)) {
+	entry = f2fs_find_cache(COMPRESS_CACHE(sbi), blkaddr,
+						F2FS_CACHE_ACCESS);
+	if (!IS_ERR(entry)) {
+		f2fs_lock_cache(entry);
+		if (f2fs_is_compress_cache(entry) &&
+				f2fs_cache_test_uptodate(entry)) {
 			atomic_inc(&sbi->compress_page_hit);
 			memcpy(folio_address(folio),
-				folio_address(cfolio), folio_size(folio));
+				cache_address(entry), folio_size(folio));
 			hitted = true;
 		}
-		f2fs_folio_put(cfolio, true);
+		f2fs_put_cache(entry, true);
 	}
 
 	return hitted;
@@ -1977,71 +1971,49 @@ bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio,
 
 void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino)
 {
-	struct address_space *mapping = COMPRESS_MAPPING(sbi);
-	struct folio_batch fbatch;
-	pgoff_t index = 0;
-	pgoff_t end = MAX_BLKADDR(sbi);
+	struct f2fs_cached_block_list *cache = COMPRESS_CACHE(sbi);
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
+	pgoff_t index = 0, end = ULONG_MAX;
+	int nr;
+	int i;
 
-	if (!mapping->nrpages)
+	if (!cache->num_entries)
 		return;
+next:
+	nr = f2fs_cache_gang_lookup(cache, entries, &index, end);
+	if (!nr)
+		return;
+	for (i = 0; i < nr; i++) {
+		struct f2fs_cached_block *entry = entries[i];
 
-	folio_batch_init(&fbatch);
-
-	do {
-		unsigned int nr, i;
-
-		nr = filemap_get_folios(mapping, &index, end - 1, &fbatch);
-		if (!nr)
-			break;
-
-		for (i = 0; i < nr; i++) {
-			struct folio *folio = fbatch.folios[i];
+		index = entry->index + 1;
 
-			folio_lock(folio);
-			if (folio->mapping != mapping) {
-				folio_unlock(folio);
-				continue;
-			}
+		f2fs_lock_cache(entry);
+		if (unlikely(!f2fs_is_compress_cache(entry)))
+			goto unlock;
+		if (entry->ino != ino)
+			goto unlock;
 
-			if (ino != folio_get_f2fs_data(folio)) {
-				folio_unlock(folio);
-				continue;
-			}
+		f2fs_truncate_locked_cache(entry, false);
+unlock:
+		f2fs_unlock_cache(entry);
+	}
+	f2fs_cache_gang_release(entries, nr);
 
-			generic_error_remove_folio(mapping, folio);
-			folio_unlock(folio);
-		}
-		folio_batch_release(&fbatch);
+	if (index < end) {
 		cond_resched();
-	} while (index < end);
+		goto next;
+	}
 }
 
-int f2fs_init_compress_inode(struct f2fs_sb_info *sbi)
+void f2fs_init_compress_cache_context(struct f2fs_sb_info *sbi)
 {
-	struct inode *inode;
-
 	if (!test_opt(sbi, COMPRESS_CACHE))
-		return 0;
-
-	inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi));
-	if (IS_ERR(inode))
-		return PTR_ERR(inode);
-	sbi->compress_inode = inode;
+		return;
 
 	sbi->compress_percent = COMPRESS_PERCENT;
 	sbi->compress_watermark = COMPRESS_WATERMARK;
-
 	atomic_set(&sbi->compress_page_hit, 0);
-
-	return 0;
-}
-
-void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi)
-{
-	if (!sbi->compress_inode)
-		return;
-	iput(sbi->compress_inode);
-	sbi->compress_inode = NULL;
 }
 
 int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi)
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 21f396ebe22ca9..29c4a81eb9475f 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -41,11 +41,6 @@ struct f2fs_folio_state {
 	unsigned int		read_pages_pending;
 };
 
-struct f2fs_bio {
-	struct work_struct work;
-	struct bio bio;
-};
-
 #define	F2FS_BIO_POOL_SIZE	NR_CURSEG_TYPE
 
 int __init f2fs_init_bioset(void)
@@ -63,14 +58,10 @@ bool f2fs_is_cp_guaranteed(const struct folio *folio)
 {
 	struct address_space *mapping = folio->mapping;
 	struct inode *inode;
-	struct f2fs_sb_info *sbi;
 
 	inode = mapping->host;
-	sbi = F2FS_I_SB(inode);
 
-	if (inode->i_ino == F2FS_META_INO(sbi) ||
-			inode->i_ino == F2FS_NODE_INO(sbi) ||
-			S_ISDIR(inode->i_mode))
+	if (S_ISDIR(inode->i_mode))
 		return true;
 
 	if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) ||
@@ -79,23 +70,6 @@ bool f2fs_is_cp_guaranteed(const struct folio *folio)
 	return false;
 }
 
-static enum count_type __read_io_type(struct folio *folio)
-{
-	struct address_space *mapping = folio->mapping;
-
-	if (mapping) {
-		struct inode *inode = mapping->host;
-		struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-
-		if (inode->i_ino == F2FS_META_INO(sbi))
-			return F2FS_RD_META;
-
-		if (inode->i_ino == F2FS_NODE_INO(sbi))
-			return F2FS_RD_NODE;
-	}
-	return F2FS_RD_DATA;
-}
-
 /* postprocessing steps for read bios */
 enum bio_post_read_step {
 #ifdef CONFIG_F2FS_FS_COMPRESSION
@@ -169,13 +143,7 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
 		}
 
 		while (nr_pages--)
-			dec_page_count(F2FS_F_SB(folio), __read_io_type(folio));
-
-		if (bio->bi_status == BLK_STS_OK &&
-			F2FS_F_SB(folio)->node_inode && is_node_folio(folio) &&
-			f2fs_sanity_check_node_footer(F2FS_F_SB(folio),
-				folio, folio->index, NODE_TYPE_REGULAR, true))
-			bio->bi_status = BLK_STS_IOERR;
+			dec_cache_count(F2FS_F_SB(folio), F2FS_RD_DATA);
 
 		if (finished)
 			folio_end_read(folio, bio->bi_status == BLK_STS_OK);
@@ -359,21 +327,13 @@ static void f2fs_write_end_bio(struct bio *bio)
 			}
 		}
 
-		if (is_node_folio(folio)) {
-			f2fs_sanity_check_node_footer(sbi, folio,
-				folio->index, NODE_TYPE_REGULAR, true);
-			f2fs_bug_on(sbi, folio->index != nid_of_node(folio));
-		}
-		if (f2fs_in_warm_node_list(folio))
-			f2fs_del_fsync_node_entry(sbi, folio);
-
-		dec_page_count(sbi, type);
+		dec_cache_count(sbi, type);
 
 		/*
 		 * we should access sbi before folio_end_writeback() to
 		 * avoid racing w/ kill_f2fs_super()
 		 */
-		if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) &&
+		if (type == F2FS_WB_CP_DATA && !get_nr_caches(sbi, type) &&
 				wq_has_sleeper(&sbi->cp_wait))
 			wake_up(&sbi->cp_wait);
 
@@ -410,6 +370,79 @@ static void f2fs_write_end_io(struct bio *bio)
 	}
 }
 
+static void f2fs_cache_read_end_io(struct bio *bio)
+{
+	struct f2fs_cached_block *entry = F2FS_BIO(bio)->entry;
+	struct f2fs_sb_info *sbi = entry->cache->sbi;
+	enum count_type io_type = IS_META_CACHE(entry->cache) ?
+					F2FS_RD_META : F2FS_RD_NODE;
+	struct f2fs_cached_block *next;
+
+	iostat_update_and_unbind_ctx(bio);
+
+	if (time_to_inject(sbi, FAULT_READ_IO))
+		bio->bi_status = BLK_STS_IOERR;
+
+	while (entry) {
+		next = entry->next_entry;
+		entry->next_entry = NULL;
+
+		if (bio->bi_status == BLK_STS_OK &&
+			f2fs_is_node_cache(entry) &&
+			f2fs_sanity_check_node_footer(sbi, entry,
+				entry->index, NODE_TYPE_REGULAR, true))
+			bio->bi_status = BLK_STS_IOERR;
+
+		if (bio->bi_status == BLK_STS_OK)
+			f2fs_cache_set_uptodate(entry);
+
+		dec_cache_count(sbi, io_type);
+
+		f2fs_unlock_cache(entry);
+		entry = next;
+	}
+	bio_put(bio);
+}
+
+static void f2fs_cache_write_end_io(struct bio *bio)
+{
+	struct f2fs_cached_block *entry = F2FS_BIO(bio)->entry;
+	struct f2fs_sb_info *sbi = entry->cache->sbi;
+	struct f2fs_cached_block *next;
+
+	iostat_update_and_unbind_ctx(bio);
+
+	if (time_to_inject(sbi, FAULT_WRITE_IO))
+		bio->bi_status = BLK_STS_IOERR;
+
+	if (bio->bi_status != BLK_STS_OK)
+		f2fs_stop_checkpoint(sbi, true,
+			STOP_CP_REASON_WRITE_FAIL);
+
+	while (entry) {
+		next = entry->next_entry;
+		entry->next_entry = NULL;
+
+		if (f2fs_is_node_cache(entry)) {
+			f2fs_sanity_check_node_footer(sbi, entry,
+				entry->index, NODE_TYPE_REGULAR, true);
+			f2fs_bug_on(sbi, entry->index != nid_of_node(sbi, entry));
+		}
+		if (f2fs_in_warm_node_list(sbi, entry))
+			f2fs_del_fsync_node_entry(sbi, entry);
+
+		dec_cache_count(sbi, F2FS_WB_CP_DATA);
+
+		if (!get_nr_caches(sbi, F2FS_WB_CP_DATA) &&
+				wq_has_sleeper(&sbi->cp_wait))
+			wake_up(&sbi->cp_wait);
+
+		f2fs_end_cache_writeback(entry);
+		entry = next;
+	}
+	bio_put(bio);
+}
+
 #ifdef CONFIG_BLK_DEV_ZONED
 static void f2fs_zone_write_end_io(struct bio *bio)
 {
@@ -417,7 +450,10 @@ static void f2fs_zone_write_end_io(struct bio *bio)
 
 	bio->bi_private = io->bi_private;
 	complete(&io->zone_wait);
-	f2fs_write_end_io(bio);
+	if (f2fs_is_cache_bio(bio))
+		f2fs_cache_write_end_io(bio);
+	else
+		f2fs_write_end_io(bio);
 }
 #endif
 
@@ -439,7 +475,7 @@ struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
 	}
 
 	if (sector)
-		*sector = SECTOR_FROM_BLOCK(blk_addr);
+		*sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
 	return bdev;
 }
 
@@ -504,12 +540,21 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
 				fio->op | fio->op_flags | f2fs_io_flags(fio),
 				GFP_NOIO, &f2fs_bioset);
 	bio->bi_iter.bi_sector = sector;
+	F2FS_BIO(bio)->entry = NULL;
+	bio->bi_private = NULL;
 	if (is_read_io(fio->op)) {
-		bio->bi_end_io = f2fs_read_end_io;
-		bio->bi_private = NULL;
+		if (fio->is_cache)
+			bio->bi_end_io = f2fs_cache_read_end_io;
+		else
+			bio->bi_end_io = f2fs_read_end_io;
 	} else {
-		bio->bi_end_io = f2fs_write_end_io;
-		bio->bi_private = sbi;
+		if (fio->is_cache) {
+			bio->bi_end_io = f2fs_cache_write_end_io;
+		} else {
+			bio->bi_end_io = f2fs_write_end_io;
+			bio->bi_private = sbi;
+		}
+
 		bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi,
 						fio->type, fio->temp);
 		bio->bi_write_stream = f2fs_io_type_to_write_stream(bdev, fio->type,
@@ -532,7 +577,7 @@ static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
 	 * The f2fs garbage collector sets ->encrypted_page when it wants to
 	 * read/write raw data without encryption.
 	 */
-	if (!fio || !fio->encrypted_page)
+	if (!fio || (!fio->encrypted_page && !fio->is_cache))
 		fscrypt_set_bio_crypt_ctx(bio, inode,
 				(loff_t)first_idx << inode->i_blkbits,
 				gfp_mask);
@@ -593,16 +638,19 @@ static void __submit_merged_bio(struct f2fs_bio_info *io)
 }
 
 static bool __has_merged_page(struct bio *bio, struct inode *inode,
-						struct folio *folio, nid_t ino)
+						struct folio *folio)
 {
 	struct folio_iter fi;
 
 	if (!bio)
 		return false;
 
-	if (!inode && !folio && !ino)
+	if (!inode && !folio)
 		return true;
 
+	if (f2fs_is_cache_bio(bio))
+		return false;
+
 	bio_for_each_folio_all(fi, bio) {
 		struct folio *target = fi.folio;
 
@@ -616,8 +664,6 @@ static bool __has_merged_page(struct bio *bio, struct inode *inode,
 			return true;
 		if (folio && folio == target)
 			return true;
-		if (ino && ino == ino_of_node(target))
-			return true;
 	}
 
 	return false;
@@ -684,26 +730,25 @@ static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
 	f2fs_up_write_trace(&io->io_rwsem, &lc);
 }
 
-static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
+static void __submit_merged_data_write_cond(struct f2fs_sb_info *sbi,
 				struct inode *inode, struct folio *folio,
-				nid_t ino, enum page_type type, bool writeback)
+				bool writeback)
 {
 	enum temp_type temp;
 	bool ret = true;
-	bool force = !inode && !folio && !ino;
+	bool force = !inode && !folio;
 
 	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
-		if (!force)	{
-			enum page_type btype = PAGE_TYPE_OF_BIO(type);
-			struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
+		if (!force) {
+			struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
 			struct f2fs_lock_context lc;
 
 			f2fs_down_read_trace(&io->io_rwsem, &lc);
-			ret = __has_merged_page(io->bio, inode, folio, ino);
+			ret = __has_merged_page(io->bio, inode, folio);
 			f2fs_up_read_trace(&io->io_rwsem, &lc);
 		}
 		if (ret) {
-			__f2fs_submit_merged_write(sbi, type, temp);
+			__f2fs_submit_merged_write(sbi, DATA, temp);
 			/*
 			 * For waitting writebck case, if the bio owned by the
 			 * folio is already submitted, we do not need to submit
@@ -712,29 +757,86 @@ static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
 			if (writeback)
 				break;
 		}
-
-		/* TODO: use HOT temp only for meta pages now. */
-		if (type >= META)
-			break;
 	}
 }
 
-void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
+void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
+				struct inode *inode, struct folio *folio)
 {
-	__submit_merged_write_cond(sbi, NULL, NULL, 0, type, false);
+	__submit_merged_data_write_cond(sbi, inode, folio, false);
 }
 
-void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
-				struct inode *inode, struct folio *folio,
+void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi,
+				struct folio *folio)
+{
+	__submit_merged_data_write_cond(sbi, NULL, folio, true);
+}
+
+static bool __has_merged_cache(struct f2fs_sb_info *sbi, struct bio *bio,
+		struct f2fs_cached_block *target, nid_t ino)
+{
+	struct f2fs_cached_block *entry;
+
+	if (!bio)
+		return false;
+
+	entry = F2FS_BIO(bio)->entry;
+
+	while (entry) {
+		if (target && entry == target)
+			return true;
+
+		if (ino) {
+			if (likely(f2fs_is_node_cache(entry))) {
+				if (ino_of_node(sbi, entry) == ino)
+					return true;
+			} else {
+				WARN_ON_ONCE(1);
+			}
+		}
+		entry = entry->next_entry;
+	}
+	return false;
+}
+
+bool f2fs_submit_merged_write_cache(struct f2fs_sb_info *sbi,
+				struct f2fs_cached_block *entry,
 				nid_t ino, enum page_type type)
 {
-	__submit_merged_write_cond(sbi, inode, folio, ino, type, false);
+	enum temp_type temp;
+	bool ret = false;
+	bool force = !entry && !ino;
+
+	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
+		enum page_type btype = PAGE_TYPE_OF_BIO(type);
+		struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
+		struct f2fs_lock_context lc;
+		bool merged = true;
+
+		if (!force) {
+			f2fs_down_read_trace(&io->io_rwsem, &lc);
+			merged = __has_merged_cache(sbi, io->bio, entry, ino);
+			f2fs_up_read_trace(&io->io_rwsem, &lc);
+		}
+
+		if (merged) {
+			__f2fs_submit_merged_write(sbi, type, temp);
+			ret = true;
+		}
+
+		/* TODO: use HOT temp only for meta pages now. */
+		if (type >= META)
+			break;
+	}
+	return ret;
 }
 
-void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi,
-				struct folio *folio, enum page_type type)
+void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
 {
-	__submit_merged_write_cond(sbi, NULL, folio, 0, type, true);
+	if (type == DATA)
+		__submit_merged_data_write_cond(sbi, NULL, NULL, false);
+	else
+		f2fs_submit_merged_write_cache(sbi, NULL, 0, type);
 }
 
 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
@@ -772,8 +874,8 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio)
 	if (fio->io_wbc && !is_read_io(fio->op))
 		wbc_account_cgroup_owner(fio->io_wbc, fio_folio, PAGE_SIZE);
 
-	inc_page_count(fio->sbi, is_read_io(fio->op) ?
-			__read_io_type(data_folio) : WB_DATA_TYPE(fio->folio, false));
+	inc_cache_count(fio->sbi, is_read_io(fio->op) ?
+			F2FS_RD_DATA : WB_DATA_TYPE(fio->folio, false));
 
 	if (is_read_io(bio_op(bio)))
 		f2fs_submit_read_bio(fio->sbi, bio, fio->type);
@@ -800,6 +902,8 @@ static bool io_type_is_mergeable(struct f2fs_bio_info *io,
 
 	if (io->fio.op != fio->op)
 		return false;
+	if (io->fio.is_cache != fio->is_cache)
+		return false;
 	return (io->fio.op_flags & mask) == (fio->op_flags & mask);
 }
 
@@ -907,8 +1011,8 @@ void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
 			if (target)
 				found = (target == be->bio);
 			else
-				found = __has_merged_page(be->bio, NULL,
-							folio, 0);
+				found = __has_merged_page(be->bio,
+							NULL, folio);
 			if (found)
 				break;
 		}
@@ -924,8 +1028,8 @@ void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
 			if (target)
 				found = (target == be->bio);
 			else
-				found = __has_merged_page(be->bio, NULL,
-							folio, 0);
+				found = __has_merged_page(be->bio,
+							NULL, folio);
 			if (found) {
 				target = be->bio;
 				del_bio_entry(be);
@@ -1003,7 +1107,7 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio)
 	if (fio->io_wbc)
 		wbc_account_cgroup_owner(fio->io_wbc, folio, folio_size(folio));
 
-	inc_page_count(fio->sbi, WB_DATA_TYPE(folio, false));
+	inc_cache_count(fio->sbi, WB_DATA_TYPE(folio, false));
 
 	*fio->last_block = fio->new_blkaddr;
 	*fio->bio = bio;
@@ -1031,6 +1135,33 @@ static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr)
 		f2fs_blkz_is_seq(sbi, devi, blkaddr) &&
 		(blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1);
 }
+
+static void f2fs_wait_zone_io_completion(struct f2fs_sb_info *sbi,
+			struct f2fs_bio_info *io, enum page_type btype)
+{
+	if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) {
+		wait_for_completion_io(&io->zone_wait);
+		bio_put(io->zone_pending_bio);
+		io->zone_pending_bio = NULL;
+		io->bi_private = NULL;
+	}
+}
+
+static void f2fs_submit_zone_io(struct f2fs_sb_info *sbi,
+			struct f2fs_io_info *fio, struct f2fs_bio_info *io,
+			enum page_type btype)
+{
+	if (f2fs_sb_has_blkzoned(sbi) && btype < META &&
+			is_end_zone_blkaddr(sbi, fio->new_blkaddr)) {
+		bio_get(io->bio);
+		reinit_completion(&io->zone_wait);
+		io->bi_private = io->bio->bi_private;
+		io->bio->bi_private = io;
+		io->bio->bi_end_io = f2fs_zone_write_end_io;
+		io->zone_pending_bio = io->bio;
+		__submit_merged_bio(io);
+	}
+}
 #endif
 
 void f2fs_submit_page_write(struct f2fs_io_info *fio)
@@ -1047,14 +1178,8 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio)
 	f2fs_down_write_trace(&io->io_rwsem, &lc);
 next:
 #ifdef CONFIG_BLK_DEV_ZONED
-	if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) {
-		wait_for_completion_io(&io->zone_wait);
-		bio_put(io->zone_pending_bio);
-		io->zone_pending_bio = NULL;
-		io->bi_private = NULL;
-	}
+	f2fs_wait_zone_io_completion(sbi, io, btype);
 #endif
-
 	if (fio->in_list) {
 		spin_lock(&io->io_lock);
 		if (list_empty(&io->io_list)) {
@@ -1080,7 +1205,7 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio)
 	fio->submitted = 1;
 
 	type = WB_DATA_TYPE(bio_folio, fio->compressed_page);
-	inc_page_count(sbi, type);
+	inc_cache_count(sbi, type);
 
 	if (io->bio &&
 	    (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,
@@ -1109,16 +1234,109 @@ void f2fs_submit_page_write(struct f2fs_io_info *fio)
 
 	trace_f2fs_submit_folio_write(fio->folio, fio);
 #ifdef CONFIG_BLK_DEV_ZONED
-	if (f2fs_sb_has_blkzoned(sbi) && btype < META &&
-			is_end_zone_blkaddr(sbi, fio->new_blkaddr)) {
-		bio_get(io->bio);
-		reinit_completion(&io->zone_wait);
-		io->bi_private = io->bio->bi_private;
-		io->bio->bi_private = io;
-		io->bio->bi_end_io = f2fs_zone_write_end_io;
-		io->zone_pending_bio = io->bio;
+	f2fs_submit_zone_io(sbi, fio, io, btype);
+#endif
+
+	if (fio->in_list)
+		goto next;
+out:
+	if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
+				!f2fs_is_checkpoint_ready(sbi))
 		__submit_merged_bio(io);
+	f2fs_up_write_trace(&io->io_rwsem, &lc);
+}
+
+static void f2fs_bio_add_cache(struct f2fs_io_info *fio, struct bio *bio)
+{
+	struct f2fs_bio *fbio = F2FS_BIO(bio);
+	struct f2fs_cached_block *head = fbio->entry;
+	struct f2fs_cached_block *new = fio->cache_entry;
+
+	new->next_entry = head;
+	fbio->entry = new;
+}
+
+int f2fs_submit_cache_read(struct f2fs_io_info *fio)
+{
+	struct f2fs_sb_info *sbi = fio->sbi;
+	struct f2fs_cached_block *entry = fio->cache_entry;
+	struct bio *bio;
+	enum count_type io_type = IS_META_CACHE(entry->cache) ?
+				F2FS_RD_META : F2FS_RD_NODE;
+
+	if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
+			fio->is_por ? META_POR : (__is_meta_io(fio) ?
+			META_GENERIC : DATA_GENERIC_ENHANCE)))
+		return -EFSCORRUPTED;
+
+	bio = __bio_alloc(fio, 1);
+
+	bio_add_virt_nofail(bio, cache_address(entry), sbi->blocksize);
+	f2fs_bio_add_cache(fio, bio);
+	inc_cache_count(sbi, io_type);
+
+	f2fs_submit_read_bio(sbi, bio, fio->type);
+	return 0;
+}
+
+void f2fs_submit_cache_write(struct f2fs_io_info *fio)
+{
+	struct f2fs_sb_info *sbi = fio->sbi;
+	enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
+	struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
+	struct f2fs_lock_context lc;
+	struct folio *folio;
+
+	f2fs_bug_on(sbi, is_read_io(fio->op));
+
+	f2fs_down_write_trace(&io->io_rwsem, &lc);
+next:
+#ifdef CONFIG_BLK_DEV_ZONED
+	f2fs_wait_zone_io_completion(sbi, io, btype);
+#endif
+	if (fio->in_list) {
+		spin_lock(&io->io_lock);
+		if (list_empty(&io->io_list)) {
+			spin_unlock(&io->io_lock);
+			goto out;
+		}
+		fio = list_first_entry(&io->io_list,
+						struct f2fs_io_info, list);
+		list_del(&fio->list);
+		spin_unlock(&io->io_lock);
+	}
+
+	verify_fio_blkaddr(fio);
+
+	fio->submitted = 1;
+	inc_cache_count(sbi, F2FS_WB_CP_DATA);
+
+	if (io->bio &&
+	    (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,
+			      fio->new_blkaddr)))
+		__submit_merged_bio(io);
+alloc_new:
+	if (io->bio == NULL) {
+		io->bio = __bio_alloc(fio, BIO_MAX_VECS);
+		io->fio = *fio;
+	}
+
+	folio = cache_folio(fio->cache_entry);
+
+	if (!bio_add_folio(io->bio, folio, sbi->blocksize,
+			offset_in_folio(folio, cache_address(fio->cache_entry)))) {
+		f2fs_bug_on(sbi, !F2FS_BIO(io->bio)->entry);
+
+		__submit_merged_bio(io);
+		goto alloc_new;
 	}
+
+	f2fs_bio_add_cache(fio, io->bio);
+
+	io->last_block_in_bio = fio->new_blkaddr;
+
+#ifdef CONFIG_BLK_DEV_ZONED
+	f2fs_submit_zone_io(sbi, fio, io, btype);
 #endif
 	if (fio->in_list)
 		goto next;
@@ -1185,20 +1403,20 @@ static void f2fs_submit_page_read(struct inode *inode, struct fsverity_info *vi,
 	bio = f2fs_grab_read_bio(inode, vi, blkaddr, 1, op_flags, folio->index,
 				 for_write);
 
-	/* wait for GCed page writeback via META_MAPPING */
+	/* wait for GCed page writeback via generic cache */
 	f2fs_wait_on_block_writeback(inode, blkaddr);
 
 	if (!bio_add_folio(bio, folio, PAGE_SIZE, 0))
 		f2fs_bug_on(sbi, 1);
 
-	inc_page_count(sbi, F2FS_RD_DATA);
-	f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE);
+	inc_cache_count(sbi, F2FS_RD_DATA);
+	f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE(sbi));
 	f2fs_submit_read_bio(sbi, bio, DATA);
 }
 
 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
 {
-	__le32 *addr = get_dnode_addr(dn->inode, dn->node_folio);
+	__le32 *addr = get_dnode_addr(dn->inode, dn->node_entry);
 
 	dn->data_blkaddr = blkaddr;
 	addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
@@ -1212,9 +1430,9 @@ static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
  */
 void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
 {
-	f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true);
+	f2fs_cache_wait_writeback(dn->node_entry);
 	__set_data_blkaddr(dn, blkaddr);
-	if (folio_mark_dirty(dn->node_folio))
+	if (f2fs_mark_cache_dirty(dn->node_entry))
 		dn->node_changed = true;
 }
 
@@ -1242,7 +1460,7 @@ int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
 	trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
 						dn->ofs_in_node, count);
 
-	f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true);
+	f2fs_cache_wait_writeback(dn->node_entry);
 
 	for (; count > 0; dn->ofs_in_node++) {
 		block_t blkaddr = f2fs_data_blkaddr(dn);
@@ -1253,7 +1471,7 @@ int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
 		}
 	}
 
-	if (folio_mark_dirty(dn->node_folio))
+	if (f2fs_mark_cache_dirty(dn->node_entry))
 		dn->node_changed = true;
 	return 0;
 }
@@ -1271,7 +1489,7 @@ int f2fs_reserve_new_block(struct dnode_of_data *dn)
 
 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
 {
-	bool need_put = dn->inode_folio ? false : true;
+	bool need_put = dn->inode_entry ? false : true;
 	int err;
 
 	err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
@@ -1437,11 +1655,11 @@ struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index,
  *
  * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
  * f2fs_unlock_op().
- * Note that, ifolio is set only by make_empty_dir, and if any error occur,
- * ifolio should be released by this function.
+ * Note that, ientry is set only by make_empty_dir, and if any error occur,
+ * ientry should be released by this function.
  */
 struct folio *f2fs_get_new_data_folio(struct inode *inode,
-		struct folio *ifolio, pgoff_t index, bool new_i_size)
+		struct f2fs_cached_block *ientry, pgoff_t index, bool new_i_size)
 {
 	struct address_space *mapping = inode->i_mapping;
 	struct folio *folio;
@@ -1451,20 +1669,20 @@ struct folio *f2fs_get_new_data_folio(struct inode *inode,
 	folio = f2fs_grab_cache_folio(mapping, index, true);
 	if (IS_ERR(folio)) {
 		/*
-		 * before exiting, we should make sure ifolio will be released
+		 * before exiting, we should make sure ientry will be released
 		 * if any error occur.
 		 */
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	set_new_dnode(&dn, inode, ifolio, NULL, 0);
+	set_new_dnode(&dn, inode, ientry, NULL, 0);
 	err = f2fs_reserve_block(&dn, index);
 	if (err) {
 		f2fs_folio_put(folio, true);
 		return ERR_PTR(err);
 	}
-	if (!ifolio)
+	if (!ientry)
 		f2fs_put_dnode(&dn);
 
 	if (folio_test_uptodate(folio))
@@ -1477,8 +1695,8 @@ struct folio *f2fs_get_new_data_folio(struct inode *inode,
 	} else {
 		f2fs_folio_put(folio, true);
 
-		/* if ifolio exists, blkaddr should be NEW_ADDR */
-		f2fs_bug_on(F2FS_I_SB(inode), ifolio);
+		/* if ientry exists, blkaddr should be NEW_ADDR */
+		f2fs_bug_on(F2FS_I_SB(inode), ientry);
 		folio = f2fs_get_lock_data_folio(inode, index, true);
 		if (IS_ERR(folio))
 			return folio;
@@ -1515,7 +1733,7 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
 
 	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 	old_blkaddr = dn->data_blkaddr;
-	err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr,
+	err = f2fs_allocate_data_block(sbi, old_blkaddr,
 				&dn->data_blkaddr, &sum, seg_type, NULL);
 	if (err) {
 		if (old_blkaddr == NULL_ADDR)
@@ -1727,7 +1945,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag)
 	start_pgofs = pgofs;
 	prealloc = 0;
 	last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
-	end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
+	end_offset = ADDRS_PER_PAGE(dn.node_entry, inode);
 
 next_block:
 	blkaddr = f2fs_data_blkaddr(&dn);
@@ -1942,12 +2160,12 @@ static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len,
 	if (pos + len > i_size_read(inode))
 		return false;
 
-	map.m_lblk = F2FS_BYTES_TO_BLK(pos);
+	map.m_lblk = F2FS_BYTES_TO_BLK(F2FS_I_SB(inode), pos);
 	map.m_next_pgofs = NULL;
 	map.m_next_extent = NULL;
 	map.m_seg_type = NO_CHECK_TYPE;
 	map.m_may_create = false;
-	last_lblk = F2FS_BLK_ALIGN(pos + len);
+	last_lblk = F2FS_BLK_ALIGN(F2FS_I_SB(inode), pos + len);
 
 	while (map.m_lblk < last_lblk) {
 		map.m_len = last_lblk - map.m_lblk;
@@ -1978,27 +2196,27 @@ static int f2fs_xattr_fiemap(struct inode *inode,
 
 	if (f2fs_has_inline_xattr(inode)) {
 		int offset;
-		struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi),
-				inode->i_ino, false);
+		struct f2fs_cached_block *entry =
+				f2fs_grab_node_cache(sbi, inode->i_ino);
 
-		if (IS_ERR(folio))
-			return PTR_ERR(folio);
+		if (IS_ERR(entry))
+			return PTR_ERR(entry);
 
 		err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false);
 		if (err) {
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 			return err;
 		}
 
-		phys = F2FS_BLK_TO_BYTES(ni.blk_addr);
+		phys = F2FS_BLK_TO_BYTES(sbi, ni.blk_addr);
 		offset = offsetof(struct f2fs_inode, i_addr) +
-					sizeof(__le32) * (DEF_ADDRS_PER_INODE -
+					sizeof(__le32) * (DEF_ADDRS_PER_INODE(sbi) -
 					get_inline_xattr_addrs(inode));
 
 		phys += offset;
 		len = inline_xattr_size(inode);
 
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 
 		flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
 
@@ -2012,22 +2230,22 @@ static int f2fs_xattr_fiemap(struct inode *inode,
 	}
 
 	if (xnid) {
-		struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi),
-				xnid, false);
+		struct f2fs_cached_block *entry =
+				f2fs_grab_node_cache(sbi, xnid);
 
-		if (IS_ERR(folio))
-			return PTR_ERR(folio);
+		if (IS_ERR(entry))
+			return PTR_ERR(entry);
 
 		err = f2fs_get_node_info(sbi, xnid, &ni, false);
 		if (err) {
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 			return err;
 		}
 
-		phys = F2FS_BLK_TO_BYTES(ni.blk_addr);
+		phys = F2FS_BLK_TO_BYTES(sbi, ni.blk_addr);
 		len = inode->i_sb->s_blocksize;
 
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 
 		flags = FIEMAP_EXTENT_LAST;
 	}
@@ -2043,6 +2261,7 @@ static int f2fs_xattr_fiemap(struct inode *inode,
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 		u64 start, u64 len)
 {
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_map_blocks map;
 	sector_t start_blk, last_blk, blk_len, max_len;
 	pgoff_t next_pgofs;
@@ -2066,7 +2285,7 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 
 	inode_lock_shared(inode);
 
-	maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode));
+	maxbytes = F2FS_BLK_TO_BYTES(sbi, max_file_blocks(sbi, inode));
 	if (start > maxbytes) {
 		ret = -EFBIG;
 		goto out;
@@ -2086,10 +2305,10 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 			goto out;
 	}
 
-	start_blk = F2FS_BYTES_TO_BLK(start);
-	last_blk = F2FS_BYTES_TO_BLK(start + len - 1);
+	start_blk = F2FS_BYTES_TO_BLK(sbi, start);
+	last_blk = F2FS_BYTES_TO_BLK(sbi, start + len - 1);
 	blk_len = last_blk - start_blk + 1;
-	max_len = F2FS_BYTES_TO_BLK(maxbytes) - start_blk;
+	max_len = F2FS_BYTES_TO_BLK(sbi, maxbytes) - start_blk;
 
 next:
 	memset(&map, 0, sizeof(map));
@@ -2111,7 +2330,7 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 	if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) {
 		start_blk = next_pgofs;
 
-		if (F2FS_BLK_TO_BYTES(start_blk) < maxbytes)
+		if (F2FS_BLK_TO_BYTES(sbi, start_blk) < maxbytes)
 			goto prep_next;
 
 		flags |= FIEMAP_EXTENT_LAST;
@@ -2159,14 +2378,14 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 	} else if (compr_appended) {
 		unsigned int appended_blks = cluster_size -
 						count_in_cluster + 1;
-		size += F2FS_BLK_TO_BYTES(appended_blks);
+		size += F2FS_BLK_TO_BYTES(sbi, appended_blks);
 		start_blk += appended_blks;
 		compr_cluster = false;
 	} else {
-		logical = F2FS_BLK_TO_BYTES(start_blk);
+		logical = F2FS_BLK_TO_BYTES(sbi, start_blk);
 		phys = __is_valid_data_blkaddr(map.m_pblk) ?
-			F2FS_BLK_TO_BYTES(map.m_pblk) : 0;
-		size = F2FS_BLK_TO_BYTES(map.m_len);
+			F2FS_BLK_TO_BYTES(sbi, map.m_pblk) : 0;
+		size = F2FS_BLK_TO_BYTES(sbi, map.m_len);
 		flags = 0;
 
 		if (compr_cluster) {
@@ -2174,13 +2393,13 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 			count_in_cluster += map.m_len;
 			if (count_in_cluster == cluster_size) {
 				compr_cluster = false;
-				size += F2FS_BLKSIZE;
+				size += F2FS_BLKSIZE(sbi);
 			}
 		} else if (map.m_flags & F2FS_MAP_DELALLOC) {
 			flags = FIEMAP_EXTENT_UNWRITTEN;
 		}
 
-		start_blk += F2FS_BYTES_TO_BLK(size);
+		start_blk += F2FS_BYTES_TO_BLK(sbi, size);
 	}
 
 prep_next:
@@ -2200,7 +2419,8 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 static inline loff_t f2fs_readpage_limit(struct inode *inode)
 {
 	if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode))
-		return F2FS_BLK_TO_BYTES(max_file_blocks(inode));
+		return F2FS_BLK_TO_BYTES(F2FS_I_SB(inode),
+				max_file_blocks(F2FS_I_SB(inode), inode));
 
 	return i_size_read(inode);
 }
@@ -2218,7 +2438,7 @@ static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi,
 				 struct readahead_control *rac)
 {
 	struct bio *bio = *bio_ret;
-	const unsigned int blocksize = F2FS_BLKSIZE;
+	const unsigned int blocksize = F2FS_BLKSIZE(F2FS_I_SB(inode));
 	sector_t block_in_file;
 	sector_t last_block;
 	sector_t last_block_in_file;
@@ -2228,7 +2448,8 @@ static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi,
 
 	block_in_file = (sector_t)index;
 	last_block = block_in_file + nr_pages;
-	last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) +
+	last_block_in_file = F2FS_BYTES_TO_BLK(F2FS_I_SB(inode),
+				f2fs_readpage_limit(inode) +
 							blocksize - 1);
 	if (last_block > last_block_in_file)
 		last_block = last_block_in_file;
@@ -2304,9 +2525,9 @@ static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi,
 	if (!bio_add_folio(bio, folio, blocksize, 0))
 		goto submit_and_realloc;
 
-	inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
+	inc_cache_count(F2FS_I_SB(inode), F2FS_RD_DATA);
 	f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO,
-							F2FS_BLKSIZE);
+					F2FS_BLKSIZE(F2FS_I_SB(inode)));
 	*last_block_in_bio = block_nr;
 out:
 	*bio_ret = bio;
@@ -2324,7 +2545,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
 	struct bio *bio = *bio_ret;
 	unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size;
 	sector_t last_block_in_file;
-	const unsigned int blocksize = F2FS_BLKSIZE;
+	const unsigned int blocksize = F2FS_BLKSIZE(sbi);
 	struct decompress_io_ctx *dic = NULL;
 	struct extent_info ei = {};
 	bool from_dnode = true;
@@ -2339,7 +2560,8 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
 
 	f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc));
 
-	last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) +
+	last_block_in_file = F2FS_BYTES_TO_BLK(sbi,
+				f2fs_readpage_limit(inode) +
 							blocksize - 1);
 
 	/* get rid of pages beyond EOF */
@@ -2386,7 +2608,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
 	for (i = 1; i < cc->cluster_size; i++) {
 		block_t blkaddr;
 
-		blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio,
+		blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_entry,
 					dn.ofs_in_node + i) :
 					ei.blk + i - 1;
 
@@ -2420,7 +2642,7 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
 		block_t blkaddr;
 		struct bio_post_read_ctx *ctx;
 
-		blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio,
+		blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_entry,
 					dn.ofs_in_node + i + 1) :
 					ei.blk + i;
 
@@ -2455,8 +2677,9 @@ int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
 		ctx->enabled_steps |= STEP_DECOMPRESS;
 		refcount_inc(&dic->refcnt);
 
-		inc_page_count(sbi, F2FS_RD_DATA);
-		f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE);
+		inc_cache_count(sbi, F2FS_RD_DATA);
+		f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO,
+				   F2FS_BLKSIZE(sbi));
 		*last_block_in_bio = blkaddr;
 	}
 
@@ -2630,14 +2853,14 @@ static int f2fs_read_data_large_folio(struct inode *inode,
 		 */
 		f2fs_wait_on_block_writeback(inode, block_nr);
 
-		if (!bio_add_folio(bio, folio, F2FS_BLKSIZE,
-					offset << PAGE_SHIFT))
+		if (!bio_add_folio(bio, folio, F2FS_BLKSIZE(F2FS_I_SB(inode)),
+				offset << PAGE_SHIFT))
 			goto submit_and_realloc;
 
 		folio_in_bio = true;
-		inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
+		inc_cache_count(F2FS_I_SB(inode), F2FS_RD_DATA);
 		f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO,
-				F2FS_BLKSIZE);
+				F2FS_BLKSIZE(F2FS_I_SB(inode)));
 		last_block_in_bio = block_nr;
 	}
 	trace_f2fs_read_folio(folio, DATA);
@@ -2909,10 +3132,12 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
 		return true;
 	if (f2fs_used_in_atomic_write(inode))
 		return true;
-	/* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */
-	if (f2fs_compressed_file(inode) &&
-		F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER &&
-		is_inode_flag_set(inode, FI_ENABLE_COMPRESS))
+	/*
+	 * rewrite low ratio compress data w/ OPU mode to avoid fragmentation.
+	 * If IO comes from compressed write path and fallback to raw write,
+	 * force out‑place to prevent metadata‑data inconsistency.
+	 */
+	if (f2fs_compressed_file(inode))
 		return true;
 
 	/* swap file is migrating in aligned write mode */
@@ -2923,7 +3148,7 @@ bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
 		return true;
 
 	if (fio) {
-		if (page_private_gcing(fio->page))
+		if (folio_test_f2fs_gcing(fio->folio))
 			return true;
 		if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
 			f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
@@ -3001,7 +3226,7 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio)
 		goto out_writepage;
 	}
 
-	/* wait for GCed page writeback via META_MAPPING */
+	/* wait for GCed page writeback via generic cache */
 	if (fio->meta_gc)
 		f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
 
@@ -3410,7 +3635,7 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
 			if (folio_test_writeback(folio)) {
 				if (wbc->sync_mode == WB_SYNC_NONE)
 					goto continue_unlock;
-				f2fs_folio_wait_writeback(folio, DATA, true, true);
+				f2fs_folio_wait_writeback(folio, true, true);
 			}
 
 			if (!folio_clear_dirty_for_io(folio))
@@ -3492,8 +3717,8 @@ static int f2fs_write_cache_pages(struct address_space *mapping,
 		mapping->writeback_index = done_index;
 
 	if (nwritten)
-		f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
-								NULL, 0, DATA);
+		f2fs_submit_merged_write_cond(F2FS_M_SB(mapping),
+					mapping->host, NULL);
 	/* submit cached bio of IPU write */
 	if (bio)
 		f2fs_submit_merged_ipu_write(sbi, &bio, NULL);
@@ -3549,7 +3774,7 @@ static inline void update_skipped_write(struct f2fs_sb_info *sbi,
 
 	if (is_sbi_flag_set(sbi, SBI_ENABLE_CHECKPOINT) && skipped &&
 		wbc->sync_mode == WB_SYNC_ALL)
-		atomic_add(skipped, &sbi->nr_pages[F2FS_SKIPPED_WRITE]);
+		atomic_add(skipped, &sbi->nr_caches[F2FS_SKIPPED_WRITE]);
 }
 
 static int __f2fs_write_data_pages(struct address_space *mapping,
@@ -3572,7 +3797,7 @@ static int __f2fs_write_data_pages(struct address_space *mapping,
 
 	if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) &&
 			wbc->sync_mode == WB_SYNC_NONE &&
-			get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
+			get_dirty_pages(inode) < nr_caches_to_skip(sbi, DATA) &&
 			f2fs_available_free_memory(sbi, DIRTY_DENTS))
 		goto skip_write;
 
@@ -3671,7 +3896,7 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi,
 	pgoff_t index = folio->index;
 	struct dnode_of_data dn;
 	struct f2fs_lock_context lc;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	bool locked = false;
 	int flag = F2FS_GET_BLOCK_PRE_AIO;
 	int err = 0;
@@ -3690,10 +3915,11 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi,
 
 	/* f2fs_lock_op avoids race between write CP and convert_inline_page */
 	if (f2fs_has_inline_data(inode)) {
-		if (pos + len > MAX_INLINE_DATA(inode))
+		if (pos + len > MAX_INLINE_DATA(inode)) {
 			flag = F2FS_GET_BLOCK_DEFAULT;
-		f2fs_map_lock(sbi, &lc, flag);
-		locked = true;
+			f2fs_map_lock(sbi, &lc, flag);
+			locked = true;
+		}
 	} else if ((pos & PAGE_MASK) >= i_size_read(inode)) {
 		f2fs_map_lock(sbi, &lc, flag);
 		locked = true;
@@ -3701,20 +3927,20 @@ static int prepare_write_begin(struct f2fs_sb_info *sbi,
 
 restart:
 	/* check inline_data */
-	ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(ifolio)) {
-		err = PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(ientry)) {
+		err = PTR_ERR(ientry);
 		goto unlock_out;
 	}
 
-	set_new_dnode(&dn, inode, ifolio, ifolio, 0);
+	set_new_dnode(&dn, inode, ientry, ientry, 0);
 
 	if (f2fs_has_inline_data(inode)) {
 		if (pos + len <= MAX_INLINE_DATA(inode)) {
-			f2fs_do_read_inline_data(folio, ifolio);
+			f2fs_do_read_inline_data(folio, ientry);
 			set_inode_flag(inode, FI_DATA_EXIST);
 			if (inode->i_nlink)
-				folio_set_f2fs_inline(ifolio);
+				f2fs_cache_set_inline(ientry);
 			goto out;
 		}
 		err = f2fs_convert_inline_folio(&dn, folio);
@@ -3761,14 +3987,14 @@ static int __find_data_block(struct inode *inode, pgoff_t index,
 				block_t *blk_addr)
 {
 	struct dnode_of_data dn;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	int err = 0;
 
-	ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
-	if (IS_ERR(ifolio))
-		return PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino);
+	if (IS_ERR(ientry))
+		return PTR_ERR(ientry);
 
-	set_new_dnode(&dn, inode, ifolio, ifolio, 0);
+	set_new_dnode(&dn, inode, ientry, ientry, 0);
 
 	if (!f2fs_lookup_read_extent_cache_block(inode, index,
 						 &dn.data_blkaddr)) {
@@ -3790,17 +4016,17 @@ static int __reserve_data_block(struct inode *inode, pgoff_t index,
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct dnode_of_data dn;
 	struct f2fs_lock_context lc;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	int err = 0;
 
 	f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO);
 
-	ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(ifolio)) {
-		err = PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(ientry)) {
+		err = PTR_ERR(ientry);
 		goto unlock_out;
 	}
-	set_new_dnode(&dn, inode, ifolio, ifolio, 0);
+	set_new_dnode(&dn, inode, ientry, ientry, 0);
 
 	if (!f2fs_lookup_read_extent_cache_block(dn.inode, index,
 						&dn.data_blkaddr))
@@ -3956,7 +4182,7 @@ static int f2fs_write_begin(const struct kiocb *iocb,
 		}
 	}
 
-	f2fs_folio_wait_writeback(folio, DATA, false, true);
+	f2fs_folio_wait_writeback(folio, false, true);
 
 	if (len == folio_size(folio) || folio_test_uptodate(folio))
 		return 0;
@@ -4073,14 +4299,8 @@ void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 		return;
 
 	if (folio_test_dirty(folio)) {
-		if (inode->i_ino == F2FS_META_INO(sbi)) {
-			dec_page_count(sbi, F2FS_DIRTY_META);
-		} else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
-			dec_page_count(sbi, F2FS_DIRTY_NODES);
-		} else {
-			inode_dec_dirty_pages(inode);
-			f2fs_remove_dirty_inode(inode);
-		}
+		inode_dec_dirty_pages(inode);
+		f2fs_remove_dirty_inode(inode);
 	}
 
 	if (offset || length != folio_size(folio))
@@ -4151,6 +4371,7 @@ static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block)
 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
 {
 	struct inode *inode = mapping->host;
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	sector_t blknr = 0;
 
 	if (f2fs_has_inline_data(inode))
@@ -4161,7 +4382,7 @@ static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
 		filemap_write_and_wait(mapping);
 
 	/* Block number less than F2FS MAX BLOCKS */
-	if (unlikely(block >= max_file_blocks(inode)))
+	if (unlikely(block >= max_file_blocks(sbi, inode)))
 		goto out;
 
 	if (f2fs_compressed_file(inode)) {
@@ -4277,7 +4498,7 @@ static int check_swap_activate(struct swap_info_struct *sis,
 	 * to be very smart.
 	 */
 	cur_lblock = 0;
-	last_lblock = F2FS_BYTES_TO_BLK(i_size_read(inode));
+	last_lblock = F2FS_BYTES_TO_BLK(sbi, i_size_read(inode));
 
 	while (cur_lblock < last_lblock && cur_lblock < sis->max) {
 		struct f2fs_map_blocks map;
@@ -4363,7 +4584,8 @@ static int check_swap_activate(struct swap_info_struct *sis,
 out:
 	if (not_aligned)
 		f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)",
-			  not_aligned, blks_per_sec * F2FS_BLKSIZE);
+			  not_aligned,
+			  (unsigned long)blks_per_sec * F2FS_BLKSIZE(sbi));
 	return ret;
 }
 
@@ -4532,12 +4754,14 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 			    unsigned int flags, struct iomap *iomap,
 			    struct iomap *srcmap)
 {
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_map_blocks map = { NULL, };
 	pgoff_t next_pgofs = 0;
 	int err;
 
-	map.m_lblk = F2FS_BYTES_TO_BLK(offset);
-	map.m_len = F2FS_BYTES_TO_BLK(offset + length - 1) - map.m_lblk + 1;
+	map.m_lblk = F2FS_BYTES_TO_BLK(sbi, offset);
+	map.m_len = F2FS_BYTES_TO_BLK(sbi, offset + length - 1) -
+							map.m_lblk + 1;
 	map.m_next_pgofs = &next_pgofs;
 	map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode),
 						inode->i_write_hint);
@@ -4558,7 +4782,7 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 	if (err)
 		return err;
 
-	iomap->offset = F2FS_BLK_TO_BYTES(map.m_lblk);
+	iomap->offset = F2FS_BLK_TO_BYTES(sbi, map.m_lblk);
 
 	/*
 	 * Sometimes I/O to an encrypted file has to be broken up to guarantee
@@ -4578,11 +4802,11 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 		if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR))
 			return -EINVAL;
 
-		iomap->length = F2FS_BLK_TO_BYTES(map.m_len);
+		iomap->length = F2FS_BLK_TO_BYTES(sbi, map.m_len);
 		iomap->type = IOMAP_MAPPED;
 		iomap->flags |= IOMAP_F_MERGED;
 		iomap->bdev = map.m_bdev;
-		iomap->addr = F2FS_BLK_TO_BYTES(map.m_pblk);
+		iomap->addr = F2FS_BLK_TO_BYTES(sbi, map.m_pblk);
 
 		if (flags & IOMAP_WRITE && map.m_last_pblk)
 			iomap->private = (void *)map.m_last_pblk;
@@ -4591,11 +4815,11 @@ static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 			return -ENOTBLK;
 
 		if (map.m_pblk == NULL_ADDR) {
-			iomap->length = F2FS_BLK_TO_BYTES(next_pgofs) -
+			iomap->length = F2FS_BLK_TO_BYTES(sbi, next_pgofs) -
 							iomap->offset;
 			iomap->type = IOMAP_HOLE;
 		} else if (map.m_pblk == NEW_ADDR) {
-			iomap->length = F2FS_BLK_TO_BYTES(map.m_len);
+			iomap->length = F2FS_BLK_TO_BYTES(sbi, map.m_len);
 			iomap->type = IOMAP_UNWRITTEN;
 		} else {
 			f2fs_bug_on(F2FS_I_SB(inode), 1);
diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
index ff379aff44726e..6fe606e2c70d24 100644
--- a/fs/f2fs/debug.c
+++ b/fs/f2fs/debug.c
@@ -156,12 +156,12 @@ static void update_general_status(struct f2fs_sb_info *sbi)
 	si->allocated_data_blocks = atomic64_read(&sbi->allocated_data_blocks);
 
 	/* validation check of the segment numbers */
-	si->ndirty_node = get_pages(sbi, F2FS_DIRTY_NODES);
-	si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS);
-	si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META);
-	si->ndirty_data = get_pages(sbi, F2FS_DIRTY_DATA);
-	si->ndirty_qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
-	si->ndirty_imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
+	si->ndirty_node = get_nr_caches(sbi, F2FS_DIRTY_NODES);
+	si->ndirty_dent = get_nr_caches(sbi, F2FS_DIRTY_DENTS);
+	si->ndirty_meta = get_nr_caches(sbi, F2FS_DIRTY_META);
+	si->ndirty_data = get_nr_caches(sbi, F2FS_DIRTY_DATA);
+	si->ndirty_qdata = get_nr_caches(sbi, F2FS_DIRTY_QDATA);
+	si->ndirty_imeta = get_nr_caches(sbi, F2FS_DIRTY_IMETA);
 	si->ndirty_dirs = sbi->ndirty_inode[DIR_INODE];
 	si->ndirty_files = sbi->ndirty_inode[FILE_INODE];
 	si->ndonate_files = sbi->donate_files;
@@ -169,13 +169,13 @@ static void update_general_status(struct f2fs_sb_info *sbi)
 	si->ndirty_all = sbi->ndirty_inode[DIRTY_META];
 	si->aw_cnt = atomic_read(&sbi->atomic_files);
 	si->max_aw_cnt = atomic_read(&sbi->max_aw_cnt);
-	si->nr_dio_read = get_pages(sbi, F2FS_DIO_READ);
-	si->nr_dio_write = get_pages(sbi, F2FS_DIO_WRITE);
-	si->nr_wb_cp_data = get_pages(sbi, F2FS_WB_CP_DATA);
-	si->nr_wb_data = get_pages(sbi, F2FS_WB_DATA);
-	si->nr_rd_data = get_pages(sbi, F2FS_RD_DATA);
-	si->nr_rd_node = get_pages(sbi, F2FS_RD_NODE);
-	si->nr_rd_meta = get_pages(sbi, F2FS_RD_META);
+	si->nr_dio_read = get_nr_caches(sbi, F2FS_DIO_READ);
+	si->nr_dio_write = get_nr_caches(sbi, F2FS_DIO_WRITE);
+	si->nr_wb_cp_data = get_nr_caches(sbi, F2FS_WB_CP_DATA);
+	si->nr_wb_data = get_nr_caches(sbi, F2FS_WB_DATA);
+	si->nr_rd_data = get_nr_caches(sbi, F2FS_RD_DATA);
+	si->nr_rd_node = get_nr_caches(sbi, F2FS_RD_NODE);
+	si->nr_rd_meta = get_nr_caches(sbi, F2FS_RD_META);
 	if (SM_I(sbi)->fcc_info) {
 		si->nr_flushed =
 			atomic_read(&SM_I(sbi)->fcc_info->issued_flush);
@@ -222,13 +222,11 @@ static void update_general_status(struct f2fs_sb_info *sbi)
 	si->free_secs = free_sections(sbi);
 	si->prefree_count = prefree_segments(sbi);
 	si->dirty_count = dirty_segments(sbi);
-	if (sbi->node_inode)
-		si->node_pages = NODE_MAPPING(sbi)->nrpages;
-	if (sbi->meta_inode)
-		si->meta_pages = META_MAPPING(sbi)->nrpages;
+	si->node_caches = NODE_CACHE(sbi)->num_entries;
+	si->meta_caches = META_CACHE(sbi)->num_entries;
 #ifdef CONFIG_F2FS_FS_COMPRESSION
-	if (sbi->compress_inode) {
-		si->compress_pages = COMPRESS_MAPPING(sbi)->nrpages;
+	if (test_opt(sbi, COMPRESS_CACHE)) {
+		si->compress_pages = COMPRESS_CACHE(sbi)->num_entries;
 		si->compress_page_hit = atomic_read(&sbi->compress_page_hit);
 	}
 #endif
@@ -343,9 +341,9 @@ static void update_mem_info(struct f2fs_sb_info *sbi)
 	/* build nm */
 	si->base_mem += sizeof(struct f2fs_nm_info);
 	si->base_mem += __bitmap_size(sbi, NAT_BITMAP);
-	si->base_mem += F2FS_BLK_TO_BYTES(NM_I(sbi)->nat_bits_blocks);
+	si->base_mem += F2FS_BLK_TO_BYTES(sbi, NM_I(sbi)->nat_bits_blocks);
 	si->base_mem += NM_I(sbi)->nat_blocks *
-				f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK);
+				f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK(sbi));
 	si->base_mem += NM_I(sbi)->nat_blocks / 8;
 	si->base_mem += NM_I(sbi)->nat_blocks * sizeof(unsigned short);
 
@@ -382,22 +380,36 @@ static void update_mem_info(struct f2fs_sb_info *sbi)
 		si->cache_mem += si->ext_mem[i];
 	}
 
-	si->page_mem = 0;
-	if (sbi->node_inode) {
-		unsigned long npages = NODE_MAPPING(sbi)->nrpages;
-
-		si->page_mem += (unsigned long long)npages << PAGE_SHIFT;
-	}
-	if (sbi->meta_inode) {
-		unsigned long npages = META_MAPPING(sbi)->nrpages;
-
-		si->page_mem += (unsigned long long)npages << PAGE_SHIFT;
-	}
+	si->cache_entry_mem[F2FS_META_CACHE] =
+		(unsigned long long)META_CACHE(sbi)->num_entries *
+		sizeof(struct f2fs_cached_block);
+	si->cache_data_mem[F2FS_META_CACHE] =
+		(unsigned long long)META_CACHE(sbi)->num_entries * sbi->blocksize;
+
+	si->cache_entry_mem[F2FS_NODE_CACHE] =
+		(unsigned long long)NODE_CACHE(sbi)->num_entries *
+		sizeof(struct f2fs_cached_block);
+	si->cache_data_mem[F2FS_NODE_CACHE] =
+		(unsigned long long)NODE_CACHE(sbi)->num_entries * sbi->blocksize;
+
+	si->cache_mem += si->cache_entry_mem[F2FS_META_CACHE] +
+			 si->cache_entry_mem[F2FS_NODE_CACHE];
+	si->page_mem = si->cache_data_mem[F2FS_META_CACHE] +
+			si->cache_data_mem[F2FS_NODE_CACHE];
 #ifdef CONFIG_F2FS_FS_COMPRESSION
-	if (sbi->compress_inode) {
-		unsigned long npages = COMPRESS_MAPPING(sbi)->nrpages;
-
-		si->page_mem += (unsigned long long)npages << PAGE_SHIFT;
+	if (test_opt(sbi, COMPRESS_CACHE)) {
+		si->cache_entry_mem[F2FS_COMPRESS_CACHE] =
+			(unsigned long long)COMPRESS_CACHE(sbi)->num_entries *
+			sizeof(struct f2fs_cached_block);
+		si->cache_data_mem[F2FS_COMPRESS_CACHE] =
+			(unsigned long long)COMPRESS_CACHE(sbi)->num_entries *
+			sbi->blocksize;
+
+		si->cache_mem += si->cache_entry_mem[F2FS_COMPRESS_CACHE];
+		si->page_mem += si->cache_data_mem[F2FS_COMPRESS_CACHE];
+	} else {
+		si->cache_entry_mem[F2FS_COMPRESS_CACHE] = 0;
+		si->cache_data_mem[F2FS_COMPRESS_CACHE] = 0;
 	}
 #endif
 }
@@ -700,7 +712,7 @@ static int stat_show(struct seq_file *s, void *v)
 			   si->aw_cnt, si->max_aw_cnt);
 		seq_printf(s, "  - compress: %4d, hit:%8d\n", si->compress_pages, si->compress_page_hit);
 		seq_printf(s, "  - nodes: %4d in %4d\n",
-			   si->ndirty_node, si->node_pages);
+			   si->ndirty_node, si->node_caches);
 		seq_printf(s, "  - dents: %4d in dirs:%4d (%4d)\n",
 			   si->ndirty_dent, si->ndirty_dirs, si->ndirty_all);
 		seq_printf(s, "  - data: %4d in files:%4d\n",
@@ -708,7 +720,7 @@ static int stat_show(struct seq_file *s, void *v)
 		seq_printf(s, "  - quota data: %4d in quota files:%4d\n",
 			   si->ndirty_qdata, si->nquota_files);
 		seq_printf(s, "  - meta: %4d in %4d\n",
-			   si->ndirty_meta, si->meta_pages);
+			   si->ndirty_meta, si->meta_caches);
 		seq_printf(s, "  - imeta: %4d\n",
 			   si->ndirty_imeta);
 		seq_printf(s, "  - fsync mark: %4lld\n",
@@ -756,6 +768,18 @@ static int stat_show(struct seq_file *s, void *v)
 				si->ext_mem[EX_READ] >> 10);
 		seq_printf(s, "  - block age extent cache: %llu KB\n",
 				si->ext_mem[EX_BLOCK_AGE] >> 10);
+		seq_printf(s, "  - meta entry: %llu KB, meta cache: %llu KB\n",
+				si->cache_entry_mem[F2FS_META_CACHE] >> 10,
+				si->cache_data_mem[F2FS_META_CACHE] >> 10);
+		seq_printf(s, "  - node entry: %llu KB, node cache: %llu KB\n",
+				si->cache_entry_mem[F2FS_NODE_CACHE] >> 10,
+				si->cache_data_mem[F2FS_NODE_CACHE] >> 10);
+#ifdef CONFIG_F2FS_FS_COMPRESSION
+		if (test_opt(sbi, COMPRESS_CACHE))
+			seq_printf(s, "  - compress entry: %llu KB, compress cache: %llu KB\n",
+					si->cache_entry_mem[F2FS_COMPRESS_CACHE] >> 10,
+					si->cache_data_mem[F2FS_COMPRESS_CACHE] >> 10);
+#endif
 		seq_printf(s, "  - paged : %llu KB\n",
 				si->page_mem >> 10);
 	}
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
index fd0e2cd31a814a..97f7fef6b068bb 100644
--- a/fs/f2fs/dir.c
+++ b/fs/f2fs/dir.c
@@ -195,7 +195,7 @@ static struct f2fs_dir_entry *find_in_block(struct inode *dir,
 				int *max_slots,
 				bool use_hash)
 {
-	struct f2fs_dentry_block *dentry_blk;
+	void *dentry_blk;
 	struct f2fs_dentry_ptr d;
 
 	dentry_blk = folio_address(dentry_folio);
@@ -282,7 +282,7 @@ struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d,
 static struct f2fs_dir_entry *find_in_level(struct inode *dir,
 					unsigned int level,
 					const struct f2fs_filename *fname,
-					struct folio **res_folio,
+					void **dentry_block,
 					bool use_hash)
 {
 	int s = GET_DENTRY_SLOTS(fname->disk_name.len);
@@ -313,7 +313,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
 				bidx = next_pgofs;
 				continue;
 			} else {
-				*res_folio = dentry_folio;
+				*dentry_block = dentry_folio;
 				break;
 			}
 		}
@@ -321,11 +321,11 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
 		de = find_in_block(dir, dentry_folio, fname, &max_slots, use_hash);
 		if (IS_ERR(de)) {
 			f2fs_folio_put(dentry_folio, false);
-			*res_folio = ERR_CAST(de);
+			*dentry_block = ERR_CAST(de);
 			de = NULL;
 			break;
 		} else if (de) {
-			*res_folio = dentry_folio;
+			*dentry_block = dentry_folio;
 			break;
 		}
 
@@ -352,7 +352,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
 
 struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
 					 const struct f2fs_filename *fname,
-					 struct folio **res_folio)
+					 void **dentry_block)
 {
 	unsigned long npages = dir_blocks(dir);
 	struct f2fs_dir_entry *de = NULL;
@@ -360,13 +360,13 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
 	unsigned int level;
 	bool use_hash = true;
 
-	*res_folio = NULL;
+	*dentry_block = NULL;
 
 #if IS_ENABLED(CONFIG_UNICODE)
 start_find_entry:
 #endif
 	if (f2fs_has_inline_dentry(dir)) {
-		de = f2fs_find_in_inline_dir(dir, fname, res_folio, use_hash);
+		de = f2fs_find_in_inline_dir(dir, fname, dentry_block, use_hash);
 		goto out;
 	}
 
@@ -382,8 +382,8 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
 	}
 
 	for (level = 0; level < max_depth; level++) {
-		de = find_in_level(dir, level, fname, res_folio, use_hash);
-		if (de || IS_ERR(*res_folio))
+		de = find_in_level(dir, level, fname, dentry_block, use_hash);
+		if (de || IS_ERR(*dentry_block))
 			break;
 	}
 
@@ -408,7 +408,7 @@ struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
  * Entry is guaranteed to be valid.
  */
 struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
-			const struct qstr *child, struct folio **res_folio)
+			const struct qstr *child, void **dentry_block)
 {
 	struct f2fs_dir_entry *de = NULL;
 	struct f2fs_filename fname;
@@ -417,67 +417,78 @@ struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
 	err = f2fs_setup_filename(dir, child, 1, &fname);
 	if (err) {
 		if (err == -ENOENT)
-			*res_folio = NULL;
+			*dentry_block = NULL;
 		else
-			*res_folio = ERR_PTR(err);
+			*dentry_block = ERR_PTR(err);
 		return NULL;
 	}
 
-	de = __f2fs_find_entry(dir, &fname, res_folio);
+	de = __f2fs_find_entry(dir, &fname, dentry_block);
 
 	f2fs_free_filename(&fname);
 	return de;
 }
 
-struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct folio **f)
+struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, void **dentry_block)
 {
-	return f2fs_find_entry(dir, &dotdot_name, f);
+	return f2fs_find_entry(dir, &dotdot_name, dentry_block);
 }
 
 ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr,
-							struct folio **folio)
+							void **dentry_block)
 {
 	ino_t res = 0;
 	struct f2fs_dir_entry *de;
 
-	de = f2fs_find_entry(dir, qstr, folio);
+	de = f2fs_find_entry(dir, qstr, dentry_block);
 	if (de) {
 		res = le32_to_cpu(de->ino);
-		f2fs_folio_put(*folio, false);
+		f2fs_put_dentry_block(*dentry_block, false);
 	}
 
 	return res;
 }
 
 void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
-		struct folio *folio, struct inode *inode)
+		void *dentry_block, struct inode *inode)
 {
-	enum page_type type = f2fs_has_inline_dentry(dir) ? NODE : DATA;
+	if (f2fs_dentry_is_cache(dentry_block)) {
+		struct f2fs_cached_block *entry =
+				f2fs_dentry_cache(dentry_block);
+
+		f2fs_lock_cache(entry);
+		f2fs_cache_wait_writeback(entry);
+		de->ino = cpu_to_le32(inode->i_ino);
+		de->file_type = fs_umode_to_ftype(inode->i_mode);
+		f2fs_mark_cache_dirty(entry);
+	} else {
+		struct folio *folio = f2fs_dentry_folio(dentry_block);
 
-	folio_lock(folio);
-	f2fs_folio_wait_writeback(folio, type, true, true);
-	de->ino = cpu_to_le32(inode->i_ino);
-	de->file_type = fs_umode_to_ftype(inode->i_mode);
-	folio_mark_dirty(folio);
+		folio_lock(folio);
+		f2fs_folio_wait_writeback(folio, true, true);
+		de->ino = cpu_to_le32(inode->i_ino);
+		de->file_type = fs_umode_to_ftype(inode->i_mode);
+		folio_mark_dirty(folio);
+	}
 
 	inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
 	f2fs_mark_inode_dirty_sync(dir, true);
-	f2fs_folio_put(folio, true);
+	f2fs_put_dentry_block(dentry_block, true);
 }
 
 static void init_dent_inode(struct inode *dir, struct inode *inode,
 			    const struct f2fs_filename *fname,
-			    struct folio *ifolio)
+			    struct f2fs_cached_block *ientry)
 {
 	struct f2fs_inode *ri;
 
 	if (!fname) /* tmpfile case? */
 		return;
 
-	f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+	f2fs_cache_wait_writeback(ientry);
 
 	/* copy name info. to this inode folio */
-	ri = F2FS_INODE(ifolio);
+	ri = F2FS_INODE(ientry);
 	ri->i_namelen = cpu_to_le32(fname->disk_name.len);
 	memcpy(ri->i_name, fname->disk_name.name, fname->disk_name.len);
 	if (IS_ENCRYPTED(dir)) {
@@ -498,7 +509,7 @@ static void init_dent_inode(struct inode *dir, struct inode *inode,
 				file_lost_pino(inode);
 		}
 	}
-	folio_mark_dirty(ifolio);
+	f2fs_mark_cache_dirty(ientry);
 }
 
 void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
@@ -515,22 +526,22 @@ void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
 }
 
 static int make_empty_dir(struct inode *inode,
-		struct inode *parent, struct folio *folio)
+		struct inode *parent, struct f2fs_cached_block *ientry)
 {
 	struct folio *dentry_folio;
-	struct f2fs_dentry_block *dentry_blk;
+	void *dentry_blk;
 	struct f2fs_dentry_ptr d;
 
 	if (f2fs_has_inline_dentry(inode))
-		return f2fs_make_empty_inline_dir(inode, parent, folio);
+		return f2fs_make_empty_inline_dir(inode, parent, ientry);
 
-	dentry_folio = f2fs_get_new_data_folio(inode, folio, 0, true);
+	dentry_folio = f2fs_get_new_data_folio(inode, ientry, 0, true);
 	if (IS_ERR(dentry_folio))
 		return PTR_ERR(dentry_folio);
 
 	dentry_blk = folio_address(dentry_folio);
 
-	make_dentry_ptr_block(NULL, &d, dentry_blk);
+	make_dentry_ptr_block(inode, &d, dentry_blk);
 	f2fs_do_make_empty_dir(inode, parent, &d);
 
 	folio_mark_dirty(dentry_folio);
@@ -538,50 +549,50 @@ static int make_empty_dir(struct inode *inode,
 	return 0;
 }
 
-struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
-		const struct f2fs_filename *fname, struct folio *dfolio)
+struct f2fs_cached_block *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
+		const struct f2fs_filename *fname, struct f2fs_cached_block *dentry)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *ientry;
 	int err;
 
 	if (is_inode_flag_set(inode, FI_NEW_INODE)) {
-		folio = f2fs_new_inode_folio(inode);
-		if (IS_ERR(folio))
-			return folio;
+		ientry = f2fs_new_inode_cache(inode);
+		if (IS_ERR(ientry))
+			return ientry;
 
 		if (S_ISDIR(inode->i_mode)) {
 			/* in order to handle error case */
-			folio_get(folio);
-			err = make_empty_dir(inode, dir, folio);
+			f2fs_cache_get(ientry);
+			err = make_empty_dir(inode, dir, ientry);
 			if (err) {
-				folio_lock(folio);
+				f2fs_lock_cache(ientry);
 				goto put_error;
 			}
-			folio_put(folio);
+			f2fs_put_cache(ientry, false);
 		}
 
-		err = f2fs_init_acl(inode, dir, folio, dfolio);
+		err = f2fs_init_acl(inode, dir, ientry, dentry);
 		if (err)
 			goto put_error;
 
 		err = f2fs_init_security(inode, dir,
 					 fname ? fname->usr_fname : NULL,
-					 folio);
+					 ientry);
 		if (err)
 			goto put_error;
 
 		if (IS_ENCRYPTED(inode)) {
-			err = fscrypt_set_context(inode, folio);
+			err = fscrypt_set_context(inode, ientry);
 			if (err)
 				goto put_error;
 		}
 	} else {
-		folio = f2fs_get_inode_folio(F2FS_I_SB(dir), inode->i_ino);
-		if (IS_ERR(folio))
-			return folio;
+		ientry = f2fs_get_inode_cache(F2FS_I_SB(dir), inode->i_ino);
+		if (IS_ERR(ientry))
+			return ientry;
 	}
 
-	init_dent_inode(dir, inode, fname, folio);
+	init_dent_inode(dir, inode, fname, ientry);
 
 	/*
 	 * This file should be checkpointed during fsync.
@@ -598,12 +609,12 @@ struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
 			f2fs_remove_orphan_inode(F2FS_I_SB(dir), inode->i_ino);
 		f2fs_i_links_write(inode, true);
 	}
-	return folio;
+	return ientry;
 
 put_error:
 	clear_nlink(inode);
-	f2fs_update_inode(inode, folio);
-	f2fs_folio_put(folio, true);
+	f2fs_update_inode(inode, ientry);
+	f2fs_put_cache(ientry, true);
 	return ERR_PTR(err);
 }
 
@@ -645,14 +656,14 @@ int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots)
 	goto next;
 }
 
-bool f2fs_has_enough_room(struct inode *dir, struct folio *ifolio,
+bool f2fs_has_enough_room(struct inode *dir, struct f2fs_cached_block *ientry,
 			  const struct f2fs_filename *fname)
 {
 	struct f2fs_dentry_ptr d;
 	unsigned int bit_pos;
 	int slots = GET_DENTRY_SLOTS(fname->disk_name.len);
 
-	make_dentry_ptr_inline(dir, &d, inline_data_addr(dir, ifolio));
+	make_dentry_ptr_inline(dir, &d, inline_data_addr(dir, ientry));
 
 	bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max);
 
@@ -690,9 +701,9 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
 	unsigned long bidx, block;
 	unsigned int nbucket, nblock;
 	struct folio *dentry_folio = NULL;
-	struct f2fs_dentry_block *dentry_blk = NULL;
+	void *dentry_blk = NULL;
 	struct f2fs_dentry_ptr d;
-	struct folio *folio = NULL;
+	struct f2fs_cached_block *entry = NULL;
 	int slots, err = 0;
 
 	level = 0;
@@ -727,9 +738,9 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
 			return PTR_ERR(dentry_folio);
 
 		dentry_blk = folio_address(dentry_folio);
-		bit_pos = f2fs_room_for_filename(&dentry_blk->dentry_bitmap,
-						slots, NR_DENTRY_IN_BLOCK);
-		if (bit_pos < NR_DENTRY_IN_BLOCK)
+		make_dentry_ptr_block(dir, &d, dentry_blk);
+		bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max);
+		if (bit_pos < d.max)
 			goto add_dentry;
 
 		f2fs_folio_put(dentry_folio, true);
@@ -739,18 +750,17 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
 	++level;
 	goto start;
 add_dentry:
-	f2fs_folio_wait_writeback(dentry_folio, DATA, true, true);
+	f2fs_folio_wait_writeback(dentry_folio, true, true);
 
 	if (inode) {
 		f2fs_down_write(&F2FS_I(inode)->i_sem);
-		folio = f2fs_init_inode_metadata(inode, dir, fname, NULL);
-		if (IS_ERR(folio)) {
-			err = PTR_ERR(folio);
+		entry = f2fs_init_inode_metadata(inode, dir, fname, NULL);
+		if (IS_ERR(entry)) {
+			err = PTR_ERR(entry);
 			goto fail;
 		}
 	}
 
-	make_dentry_ptr_block(NULL, &d, dentry_blk);
 	f2fs_update_dentry(ino, mode, &d, &fname->disk_name, fname->hash,
 			   bit_pos);
 
@@ -761,9 +771,9 @@ int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname,
 
 		/* synchronize inode page's data from inode cache */
 		if (is_inode_flag_set(inode, FI_NEW_INODE))
-			f2fs_update_inode(inode, folio);
+			f2fs_update_inode(inode, entry);
 
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 	}
 
 	f2fs_update_parent_metadata(dir, inode, current_depth);
@@ -805,7 +815,7 @@ int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
 				struct inode *inode, nid_t ino, umode_t mode)
 {
 	struct f2fs_filename fname;
-	struct folio *folio = NULL;
+	void *dentry_blk = NULL;
 	struct f2fs_dir_entry *de = NULL;
 	int err;
 
@@ -821,14 +831,14 @@ int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
 	 * consistency more.
 	 */
 	if (current != F2FS_I(dir)->task) {
-		de = __f2fs_find_entry(dir, &fname, &folio);
+		de = __f2fs_find_entry(dir, &fname, &dentry_blk);
 		F2FS_I(dir)->task = NULL;
 	}
 	if (de) {
-		f2fs_folio_put(folio, false);
+		f2fs_put_dentry_block(dentry_blk, false);
 		err = -EEXIST;
-	} else if (IS_ERR(folio)) {
-		err = PTR_ERR(folio);
+	} else if (IS_ERR(dentry_blk)) {
+		err = PTR_ERR(dentry_blk);
 	} else {
 		err = f2fs_add_dentry(dir, &fname, inode, ino, mode);
 	}
@@ -839,16 +849,16 @@ int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
 int f2fs_do_tmpfile(struct inode *inode, struct inode *dir,
 					struct f2fs_filename *fname)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *ientry;
 	int err = 0;
 
 	f2fs_down_write(&F2FS_I(inode)->i_sem);
-	folio = f2fs_init_inode_metadata(inode, dir, fname, NULL);
-	if (IS_ERR(folio)) {
-		err = PTR_ERR(folio);
+	ientry = f2fs_init_inode_metadata(inode, dir, fname, NULL);
+	if (IS_ERR(ientry)) {
+		err = PTR_ERR(ientry);
 		goto fail;
 	}
-	f2fs_folio_put(folio, true);
+	f2fs_put_cache(ientry, true);
 
 	clear_inode_flag(inode, FI_NEW_INODE);
 	f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
@@ -884,13 +894,15 @@ void f2fs_drop_nlink(struct inode *dir, struct inode *inode)
  * It only removes the dentry from the dentry page, corresponding name
  * entry in name page does not need to be touched during deletion.
  */
-void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio,
+void f2fs_delete_entry(struct f2fs_dir_entry *dentry, void *dentry_block,
 					struct inode *dir, struct inode *inode)
 {
-	struct f2fs_dentry_block *dentry_blk;
+	void *dentry_blk;
+	struct f2fs_dentry_ptr d;
+	struct folio *folio;
 	unsigned int bit_pos;
 	int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len));
-	pgoff_t index = folio->index;
+	pgoff_t index;
 	int i;
 
 	f2fs_update_time(F2FS_I_SB(dir), REQ_TIME);
@@ -899,24 +911,27 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio,
 		f2fs_add_ino_entry(F2FS_I_SB(dir), dir->i_ino, TRANS_DIR_INO);
 
 	if (f2fs_has_inline_dentry(dir))
-		return f2fs_delete_inline_entry(dentry, folio, dir, inode);
+		return f2fs_delete_inline_entry(dentry,
+				f2fs_dentry_cache(dentry_block), dir, inode);
+
+	folio = f2fs_dentry_folio(dentry_block);
+	index = folio->index;
 
 	folio_lock(folio);
-	f2fs_folio_wait_writeback(folio, DATA, true, true);
+	f2fs_folio_wait_writeback(folio, true, true);
 
 	dentry_blk = folio_address(folio);
-	bit_pos = dentry - dentry_blk->dentry;
+	make_dentry_ptr_block(dir, &d, dentry_blk);
+	bit_pos = dentry - d.dentry;
 	for (i = 0; i < slots; i++)
-		__clear_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap);
+		__clear_bit_le(bit_pos + i, d.bitmap);
 
 	/* Let's check and deallocate this dentry page */
-	bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
-			NR_DENTRY_IN_BLOCK,
-			0);
+	bit_pos = find_next_bit_le(d.bitmap, d.max, 0);
 	folio_mark_dirty(folio);
 
-	if (bit_pos == NR_DENTRY_IN_BLOCK &&
-		!f2fs_truncate_hole(dir, index, index + 1)) {
+	if (bit_pos == d.max &&
+	    !f2fs_truncate_hole(dir, index, index + 1)) {
 		f2fs_clear_page_cache_dirty_tag(folio);
 		folio_clear_dirty_for_io(folio);
 		folio_clear_uptodate(folio);
@@ -938,7 +953,8 @@ bool f2fs_empty_dir(struct inode *dir)
 {
 	unsigned long bidx = 0;
 	unsigned int bit_pos;
-	struct f2fs_dentry_block *dentry_blk;
+	void *dentry_blk;
+	struct f2fs_dentry_ptr d;
 	unsigned long nblock = dir_blocks(dir);
 
 	if (f2fs_has_inline_dentry(dir))
@@ -959,17 +975,16 @@ bool f2fs_empty_dir(struct inode *dir)
 		}
 
 		dentry_blk = folio_address(dentry_folio);
+		make_dentry_ptr_block(dir, &d, dentry_blk);
 		if (bidx == 0)
 			bit_pos = 2;
 		else
 			bit_pos = 0;
-		bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
-						NR_DENTRY_IN_BLOCK,
-						bit_pos);
+		bit_pos = find_next_bit_le(d.bitmap, d.max, bit_pos);
 
 		f2fs_folio_put(dentry_folio, false);
 
-		if (bit_pos < NR_DENTRY_IN_BLOCK)
+		if (bit_pos < d.max)
 			return false;
 
 		bidx++;
@@ -1051,7 +1066,7 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
 		}
 
 		if (readdir_ra)
-			f2fs_ra_node_page(sbi, le32_to_cpu(de->ino));
+			f2fs_ra_node_cache(sbi, le32_to_cpu(de->ino));
 
 		ctx->pos = start_pos + bit_pos;
 		found_valid_dirent = true;
@@ -1066,10 +1081,12 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx)
 {
 	struct inode *inode = file_inode(file);
 	unsigned long npages = dir_blocks(inode);
-	struct f2fs_dentry_block *dentry_blk = NULL;
+	void *dentry_blk = NULL;
 	struct file_ra_state *ra = &file->f_ra;
 	loff_t start_pos = ctx->pos;
-	unsigned int n = ((unsigned long)ctx->pos / NR_DENTRY_IN_BLOCK);
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+	unsigned int entries = sbi->dentries_per_block;
+	unsigned int n = (unsigned long)ctx->pos / entries;
 	struct f2fs_dentry_ptr d;
 	struct fscrypt_str fstr = FSTR_INIT(NULL, 0);
 	int err = 0;
@@ -1089,7 +1106,7 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx)
 		goto out_free;
 	}
 
-	for (; n < npages; ctx->pos = n * NR_DENTRY_IN_BLOCK) {
+	for (; n < npages; ctx->pos = n * entries) {
 		struct folio *dentry_folio;
 		pgoff_t next_pgofs;
 
@@ -1122,7 +1139,7 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx)
 		make_dentry_ptr_block(inode, &d, dentry_blk);
 
 		err = f2fs_fill_dentries(ctx, &d,
-				n * NR_DENTRY_IN_BLOCK, &fstr);
+				n * entries, &fstr);
 		f2fs_folio_put(dentry_folio, false);
 		if (err)
 			break;
diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c
index 37cf9fa8d53704..5a944428aca03f 100644
--- a/fs/f2fs/extent_cache.c
+++ b/fs/f2fs/extent_cache.c
@@ -20,10 +20,10 @@
 #include "segment.h"
 #include <trace/events/f2fs.h>
 
-bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio)
+bool sanity_check_extent_cache(struct inode *inode, struct f2fs_cached_block *ientry)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-	struct f2fs_extent *i_ext = &F2FS_INODE(ifolio)->i_ext;
+	struct f2fs_extent *i_ext = &F2FS_INODE(ientry)->i_ext;
 	struct extent_info ei;
 	int devi;
 
@@ -416,11 +416,11 @@ static void __drop_largest_extent(struct extent_tree *et,
 	}
 }
 
-void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio)
+void f2fs_init_read_extent_tree(struct inode *inode, struct f2fs_cached_block *ientry)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct extent_tree_info *eti = &sbi->extent_tree[EX_READ];
-	struct f2fs_extent *i_ext = &F2FS_INODE(ifolio)->i_ext;
+	struct f2fs_extent *i_ext = &F2FS_INODE(ientry)->i_ext;
 	struct extent_tree *et;
 	struct extent_node *en;
 	struct extent_info ei = {0};
@@ -428,9 +428,9 @@ void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio)
 	if (!__may_extent_tree(inode, EX_READ)) {
 		/* drop largest read extent */
 		if (i_ext->len) {
-			f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+			f2fs_cache_wait_writeback(ientry);
 			i_ext->len = 0;
-			folio_mark_dirty(ifolio);
+			f2fs_mark_cache_dirty(ientry);
 		}
 		set_inode_flag(inode, FI_NO_EXTENT);
 		return;
@@ -904,11 +904,12 @@ static int __get_new_block_age(struct inode *inode, struct extent_info *ei,
 	struct extent_info tei = *ei;	/* only fofs and len are valid */
 
 	/*
-	 * When I/O is not aligned to a PAGE_SIZE, update will happen to the last
-	 * file block even in seq write. So don't record age for newly last file
-	 * block here.
+	 * When I/O is not aligned to the filesystem block size, update will
+	 * happen to the last file block even in seq write. Do not record the age
+	 * of the new last file block here.
 	 */
-	if ((f_size >> PAGE_SHIFT) == ei->fofs && f_size & (PAGE_SIZE - 1) &&
+	if ((f_size >> sbi->log_blocksize) == ei->fofs &&
+	    f_size & F2FS_BLKSIZE_MASK(sbi) &&
 			blkaddr == NEW_ADDR)
 		return -EINVAL;
 
@@ -956,8 +957,8 @@ static void __update_extent_cache(struct dnode_of_data *dn, enum extent_type typ
 	if (!__may_extent_tree(dn->inode, type))
 		return;
 
-	ei.fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_folio), dn->inode) +
-								dn->ofs_in_node;
+	ei.fofs = f2fs_start_bidx_of_node(ofs_of_node(F2FS_I_SB(dn->inode),
+				dn->node_entry), dn->inode) + dn->ofs_in_node;
 	ei.len = 1;
 
 	if (type == EX_READ) {
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 9940a6cecf1a2c..dd9b4e4a88fcbb 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -222,6 +222,8 @@ struct f2fs_rwsem {
 #endif
 };
 
+#include "cache.h"
+
 struct f2fs_mount_info {
 	unsigned long long opt;
 	block_t root_reserved_blocks;	/* root reserved blocks */
@@ -416,7 +418,7 @@ struct inode_entry {
 
 struct fsync_node_entry {
 	struct list_head list;	/* list head */
-	struct folio *folio;	/* warm node folio pointer */
+	struct f2fs_cached_block *entry; /* warm node cache entry pointer */
 	unsigned int seq_id;	/* sequence id */
 };
 
@@ -466,6 +468,7 @@ struct discard_entry {
 #define MAX_PLIST_NUM		512
 #define plist_idx(blk_num)	((blk_num) >= MAX_PLIST_NUM ?		\
 					(MAX_PLIST_NUM - 1) : ((blk_num) - 1))
+#define MAX_DISCARD_DROP_COUNT	512
 
 enum {
 	D_PREP,			/* initial */
@@ -602,8 +605,9 @@ static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i)
 #define DEF_INLINE_RESERVED_SIZE	1
 static inline int get_extra_isize(struct inode *inode);
 static inline int get_inline_xattr_addrs(struct inode *inode);
+static inline unsigned int cur_addrs_per_inode(struct inode *inode);
 #define MAX_INLINE_DATA(inode)	(sizeof(__le32) *			\
-				(CUR_ADDRS_PER_INODE(inode) -		\
+				(cur_addrs_per_inode(inode) -		\
 				get_inline_xattr_addrs(inode) -	\
 				DEF_INLINE_RESERVED_SIZE))
 
@@ -669,17 +673,6 @@ struct f2fs_dentry_ptr {
 	int nr_bitmap;
 };
 
-static inline void make_dentry_ptr_block(struct inode *inode,
-		struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t)
-{
-	d->inode = inode;
-	d->max = NR_DENTRY_IN_BLOCK;
-	d->nr_bitmap = SIZE_OF_DENTRY_BITMAP;
-	d->bitmap = t->dentry_bitmap;
-	d->dentry = t->dentry;
-	d->filename = t->filename;
-}
-
 static inline void make_dentry_ptr_inline(struct inode *inode,
 					struct f2fs_dentry_ptr *d, void *t)
 {
@@ -1081,7 +1074,7 @@ struct f2fs_nm_info {
 	nid_t next_scan_nid;		/* the next nid to be scanned */
 	nid_t max_rf_node_blocks;	/* max # of nodes for recovery */
 	unsigned int ram_thresh;	/* control the memory footprint */
-	unsigned int ra_nid_pages;	/* # of nid pages to be readaheaded */
+	unsigned int ra_nid_blocks;	/* # of nid blocks to be readaheaded */
 	unsigned int dirty_nats_ratio;	/* control dirty nats ratio threshold */
 
 	/* NAT cache management */
@@ -1123,11 +1116,11 @@ struct f2fs_nm_info {
  */
 struct dnode_of_data {
 	struct inode *inode;		/* vfs inode pointer */
-	struct folio *inode_folio;	/* its inode folio, NULL is possible */
-	struct folio *node_folio;	/* cached direct node folio */
+	struct f2fs_cached_block *inode_entry; /* generic cache inode entry */
+	struct f2fs_cached_block *node_entry;  /* generic cache node entry */
 	nid_t nid;			/* node id of the direct node block */
 	unsigned int ofs_in_node;	/* data offset in the node page */
-	bool inode_folio_locked;	/* inode folio is locked or not */
+	bool inode_entry_locked;	/* inode entry is locked or not */
 	bool node_changed;		/* is node block changed */
 	char cur_level;			/* level of hole node page */
 	char max_level;			/* level of current page located */
@@ -1135,12 +1128,12 @@ struct dnode_of_data {
 };
 
 static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode,
-		struct folio *ifolio, struct folio *nfolio, nid_t nid)
+		struct f2fs_cached_block *ientry, struct f2fs_cached_block *nentry, nid_t nid)
 {
 	memset(dn, 0, sizeof(*dn));
 	dn->inode = inode;
-	dn->inode_folio = ifolio;
-	dn->node_folio = nfolio;
+	dn->inode_entry = ientry;
+	dn->node_entry = nentry;
 	dn->nid = nid;
 }
 
@@ -1357,10 +1350,7 @@ struct f2fs_io_info {
 	blk_opf_t op_flags;	/* req_flag_bits */
 	block_t new_blkaddr;	/* new block address to be written */
 	block_t old_blkaddr;	/* old block address before Cow */
-	union {
-		struct page *page;	/* page to be written */
-		struct folio *folio;
-	};
+	struct folio *folio;	/* folio to be written */
 	struct page *encrypted_page;	/* encrypted page */
 	struct page *compressed_page;	/* compressed page */
 	struct list_head list;		/* serialize IOs */
@@ -1371,8 +1361,10 @@ struct f2fs_io_info {
 	unsigned int in_list:1;		/* indicate fio is in io_list */
 	unsigned int is_por:1;		/* indicate IO is from recovery or not */
 	unsigned int meta_gc:1;		/* require meta inode GC */
+	unsigned int is_cache:1;	/* indicate IO is from internal cache */
 	enum iostat_type io_type;	/* io type */
 	struct writeback_control *io_wbc; /* writeback control */
+	struct f2fs_cached_block *cache_entry;
 	struct bio **bio;		/* bio for ipu */
 	sector_t *last_block;		/* last block number in bio */
 };
@@ -1613,27 +1605,25 @@ static inline void f2fs_set_bit(unsigned int nr, char *addr);
 static inline void f2fs_clear_bit(unsigned int nr, char *addr);
 
 /*
- * Layout of f2fs page.private:
+ * Layout of f2fs folio->private:
  *
  * Layout A: lowest bit should be 1
  * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... |
- * bit 0	PAGE_PRIVATE_NOT_POINTER
- * bit 1	PAGE_PRIVATE_ONGOING_MIGRATION
- * bit 2	PAGE_PRIVATE_INLINE_INODE
- * bit 3	PAGE_PRIVATE_REF_RESOURCE
- * bit 4	PAGE_PRIVATE_ATOMIC_WRITE
- * bit 5-	f2fs private data
+ * bit 0	F2FS_FOLIO_PRIVATE_NOT_POINTER
+ * bit 1	F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION
+ * bit 2	F2FS_FOLIO_PRIVATE_REF_RESOURCE
+ * bit 3	F2FS_FOLIO_PRIVATE_ATOMIC_WRITE
+ * bit 4-	f2fs private data
  *
  * Layout B: lowest bit should be 0
- * page.private is a wrapped pointer.
+ * folio->private is a wrapped pointer.
  */
 enum {
-	PAGE_PRIVATE_NOT_POINTER,		/* private contains non-pointer data */
-	PAGE_PRIVATE_ONGOING_MIGRATION,		/* data page which is on-going migrating */
-	PAGE_PRIVATE_INLINE_INODE,		/* inode page contains inline data */
-	PAGE_PRIVATE_REF_RESOURCE,		/* dirty page has referenced resources */
-	PAGE_PRIVATE_ATOMIC_WRITE,		/* data page from atomic write path */
-	PAGE_PRIVATE_MAX
+	F2FS_FOLIO_PRIVATE_NOT_POINTER,		/* private contains non-pointer data */
+	F2FS_FOLIO_PRIVATE_ONGOING_MIGRATION,		/* data page which is on-going migrating */
+	F2FS_FOLIO_PRIVATE_REF_RESOURCE,		/* dirty page has referenced resources */
+	F2FS_FOLIO_PRIVATE_ATOMIC_WRITE,		/* data page from atomic write path */
+	F2FS_FOLIO_PRIVATE_MAX
 };
 
 /* For compression */
@@ -1782,6 +1772,12 @@ struct f2fs_gc_kthread {
 	unsigned int boost_gc_greedy;
 };
 
+struct f2fs_bio {
+	struct work_struct work;
+	struct f2fs_cached_block *entry;
+	struct bio bio;
+};
+
 struct f2fs_sb_info {
 	struct super_block *sb;			/* pointer to VFS super block */
 	struct proc_dir_entry *s_proc;		/* proc entry */
@@ -1801,7 +1797,6 @@ struct f2fs_sb_info {
 
 	/* for node-related operations */
 	struct f2fs_nm_info *nm_info;		/* node manager */
-	struct inode *node_inode;		/* cache node blocks */
 
 	/* for segment-related operations */
 	struct f2fs_sm_info *sm_info;		/* segment manager */
@@ -1819,7 +1814,6 @@ struct f2fs_sb_info {
 	struct f2fs_checkpoint *ckpt;		/* raw checkpoint pointer */
 	int cur_cp_pack;			/* remain current cp pack */
 	spinlock_t cp_lock;			/* for flag in ckpt */
-	struct inode *meta_inode;		/* cache meta blocks */
 	struct f2fs_rwsem cp_global_sem;	/* checkpoint procedure lock */
 	struct f2fs_rwsem cp_rwsem;		/* blocking FS operations */
 	struct f2fs_rwsem node_write;		/* locking node writes */
@@ -1862,9 +1856,17 @@ struct f2fs_sb_info {
 	unsigned int log_sectors_per_block;	/* log2 sectors per block */
 	unsigned int log_blocksize;		/* log2 block size */
 	unsigned int blocksize;			/* block size */
+	unsigned int nat_entries_per_block;	/* NAT entries in a block */
+	unsigned int addrs_per_inode;		/* addresses in an inode block */
+	unsigned int addrs_per_block;		/* addresses in a direct node block */
+	unsigned int nids_per_block;		/* node IDs in an indirect node block */
+	unsigned int sit_entries_per_block;	/* SIT entries in a block */
+	unsigned int orphans_per_block;	/* orphan inodes in a block */
+	unsigned int dentries_per_block;	/* dentries in a block */
+	unsigned int dentry_bitmap_size;	/* dentry bitmap size in bytes */
+	unsigned int dentry_reserved_size;	/* dentry reserved bytes */
 	unsigned int root_ino_num;		/* root inode number*/
 	unsigned int node_ino_num;		/* node inode number*/
-	unsigned int meta_ino_num;		/* meta inode number*/
 	unsigned int log_blocks_per_seg;	/* log2 blocks per segment */
 	unsigned int blocks_per_seg;		/* blocks per segment */
 	unsigned int segs_per_sec;		/* segments per section */
@@ -1900,8 +1902,8 @@ struct f2fs_sb_info {
 	struct f2fs_rwsem quota_sem;		/* blocking cp for flags */
 	struct task_struct *umount_lock_holder;	/* s_umount lock holder */
 
-	/* # of pages, see count_type */
-	atomic_t nr_pages[NR_COUNT_TYPE];
+	/* # of cache entries, see count_type */
+	atomic_t nr_caches[NR_COUNT_TYPE];
 	/* # of allocated blocks */
 	struct percpu_counter alloc_valid_block_count;
 	/* # of node block writes as roll forward recovery */
@@ -2072,7 +2074,6 @@ struct f2fs_sb_info {
 	u32 compr_new_inode;
 
 	/* For compressed block cache */
-	struct inode *compress_inode;		/* cache compressed blocks */
 	unsigned int compress_percent;		/* cache page percentage */
 	unsigned int compress_watermark;	/* cache page watermark */
 	atomic_t compress_page_hit;		/* cache hit count */
@@ -2097,6 +2098,14 @@ struct f2fs_sb_info {
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	struct lock_class_key cp_global_sem_key;
 #endif
+
+	/* f2fs internal cache */
+	struct f2fs_cached_block_list meta_blocks;
+	struct f2fs_cached_block_list node_blocks;
+	struct f2fs_cached_block_list compress_blocks;
+
+	/* internal cache flush thread */
+	struct f2fs_cache_kthread cache_thread;
 };
 
 /* Definitions to access f2fs_sb_info */
@@ -2250,6 +2259,47 @@ static inline struct f2fs_sb_info *F2FS_F_SB(const struct folio *folio)
 	return F2FS_M_SB(folio->mapping);
 }
 
+#define SIT_ENTRY_PER_BLOCK(sbi)	((sbi)->sit_entries_per_block)
+#define NAT_ENTRY_PER_BLOCK(sbi)	((sbi)->nat_entries_per_block)
+#define DEF_ADDRS_PER_INODE(sbi)	((sbi)->addrs_per_inode)
+#define DEF_ADDRS_PER_BLOCK(sbi)	((sbi)->addrs_per_block)
+#define NIDS_PER_BLOCK(sbi)		((sbi)->nids_per_block)
+#define F2FS_ORPHANS_PER_BLOCK(sbi)	((sbi)->orphans_per_block)
+#define GET_ORPHAN_BLOCKS(sbi, n)	DIV_ROUND_UP((n), \
+					F2FS_ORPHANS_PER_BLOCK(sbi))
+#define CP_CHKSUM_OFFSET(sbi)		(F2FS_BLKSIZE(sbi) - sizeof(__le32))
+
+#define NODE_DIR1_BLOCK(sbi)		(DEF_ADDRS_PER_INODE(sbi) + 1)
+#define NODE_DIR2_BLOCK(sbi)		(DEF_ADDRS_PER_INODE(sbi) + 2)
+#define NODE_IND1_BLOCK(sbi)		(DEF_ADDRS_PER_INODE(sbi) + 3)
+#define NODE_IND2_BLOCK(sbi)		(DEF_ADDRS_PER_INODE(sbi) + 4)
+#define NODE_DIND_BLOCK(sbi)		(DEF_ADDRS_PER_INODE(sbi) + 5)
+
+static inline struct f2fs_orphan_footer *
+f2fs_orphan_footer(void *orphan_block, struct f2fs_sb_info *sbi)
+{
+	return (struct f2fs_orphan_footer *)
+		((char *)orphan_block + sbi->blocksize -
+		 sizeof(struct f2fs_orphan_footer));
+}
+
+static inline void make_dentry_ptr_block(struct inode *inode,
+				struct f2fs_dentry_ptr *d, void *t)
+{
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+	unsigned int entries = sbi->dentries_per_block;
+	unsigned int bitmap_size = sbi->dentry_bitmap_size;
+	unsigned int reserved_size = sbi->dentry_reserved_size;
+
+	d->inode = inode;
+	d->max = entries;
+	d->nr_bitmap = bitmap_size;
+	d->bitmap = t;
+	d->dentry = t + bitmap_size + reserved_size;
+	d->filename = t + bitmap_size + reserved_size +
+					SIZE_OF_DIR_ENTRY * entries;
+}
+
 static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi)
 {
 	return (struct f2fs_super_block *)(sbi->raw_super);
@@ -2270,14 +2320,30 @@ static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi)
 	return (struct f2fs_checkpoint *)(sbi->ckpt);
 }
 
-static inline struct f2fs_node *F2FS_NODE(const struct folio *folio)
+static inline struct node_footer *F2FS_NODE_FOOTER(struct f2fs_sb_info *sbi,
+						const struct f2fs_cached_block *entry)
+{
+	return (struct node_footer *)(cache_address(entry) +
+		F2FS_BLKSIZE(sbi) - sizeof(struct node_footer));
+}
+
+static inline struct f2fs_node *F2FS_NODE(const struct f2fs_cached_block *entry)
 {
-	return (struct f2fs_node *)folio_address(folio);
+	return (struct f2fs_node *)CACHED_NODE(entry);
 }
 
-static inline struct f2fs_inode *F2FS_INODE(const struct folio *folio)
+static inline struct f2fs_inode *F2FS_INODE(const struct f2fs_cached_block *entry)
 {
-	return &((struct f2fs_node *)folio_address(folio))->i;
+	return &CACHED_NODE(entry)->i;
+}
+
+static inline __le32 *F2FS_INODE_NIDS(struct f2fs_sb_info *sbi,
+						const struct f2fs_cached_block *entry)
+{
+	return (__le32 *)(cache_address(entry) +
+		F2FS_BLKSIZE(sbi) -
+		sizeof(struct node_footer) -
+		SIZE_OF_I_NID);
 }
 
 static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi)
@@ -2305,24 +2371,29 @@ static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi)
 	return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info);
 }
 
-static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi)
+static inline bool f2fs_is_meta_cache(struct f2fs_cached_block *entry)
+{
+	return entry->cache && entry->cache == META_CACHE(entry->cache->sbi);
+}
+
+static inline bool f2fs_is_node_cache(struct f2fs_cached_block *entry)
 {
-	return sbi->meta_inode->i_mapping;
+	return entry->cache && entry->cache == NODE_CACHE(entry->cache->sbi);
 }
 
-static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi)
+static inline bool f2fs_is_compress_cache(struct f2fs_cached_block *entry)
 {
-	return sbi->node_inode->i_mapping;
+	return entry->cache && entry->cache == COMPRESS_CACHE(entry->cache->sbi);
 }
 
-static inline bool is_meta_folio(struct folio *folio)
+static inline struct f2fs_bio *F2FS_BIO(struct bio *bio)
 {
-	return folio->mapping == META_MAPPING(F2FS_F_SB(folio));
+	return container_of(bio, struct f2fs_bio, bio);
 }
 
-static inline bool is_node_folio(struct folio *folio)
+static inline bool f2fs_is_cache_bio(struct bio *bio)
 {
-	return folio->mapping == NODE_MAPPING(F2FS_F_SB(folio));
+	return F2FS_BIO(bio)->entry != NULL;
 }
 
 static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type)
@@ -2566,7 +2637,8 @@ static inline int F2FS_HAS_BLOCKS(struct inode *inode)
 {
 	block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0;
 
-	return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block;
+	return (inode->i_blocks >>
+		F2FS_LOG_SECTORS_PER_BLOCK(F2FS_I_SB(inode))) > xattr_block;
 }
 
 static inline bool f2fs_has_xattr_block(unsigned int ofs)
@@ -2681,86 +2753,65 @@ static inline int inc_valid_block_count(struct f2fs_sb_info *sbi,
 	return -ENOSPC;
 }
 
-#define PAGE_PRIVATE_GET_FUNC(name, flagname) \
+#define F2FS_FOLIO_PRIVATE_GET_FUNC(name, flagname) \
 static inline bool folio_test_f2fs_##name(const struct folio *folio)	\
 {									\
 	unsigned long priv = (unsigned long)folio->private;		\
-	unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) |		\
-			     (1UL << PAGE_PRIVATE_##flagname);		\
+	unsigned long v = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) |	\
+			     (1UL << F2FS_FOLIO_PRIVATE_##flagname);	\
 	return (priv & v) == v;						\
-}									\
-static inline bool page_private_##name(struct page *page) \
-{ \
-	return PagePrivate(page) && \
-		test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \
-		test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
 }
 
-#define PAGE_PRIVATE_SET_FUNC(name, flagname) \
+#define F2FS_FOLIO_PRIVATE_SET_FUNC(name, flagname) \
 static inline void folio_set_f2fs_##name(struct folio *folio)		\
 {									\
-	unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) |		\
-			     (1UL << PAGE_PRIVATE_##flagname);		\
+	unsigned long v = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) |	\
+			     (1UL << F2FS_FOLIO_PRIVATE_##flagname);	\
 	if (!folio->private)						\
 		folio_attach_private(folio, (void *)v);			\
 	else {								\
 		v |= (unsigned long)folio->private;			\
 		folio->private = (void *)v;				\
 	}								\
-}									\
-static inline void set_page_private_##name(struct page *page) \
-{ \
-	if (!PagePrivate(page)) \
-		attach_page_private(page, (void *)0); \
-	set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \
-	set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
 }
 
-#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \
+#define F2FS_FOLIO_PRIVATE_CLEAR_FUNC(name, flagname) \
 static inline void folio_clear_f2fs_##name(struct folio *folio)		\
 {									\
 	unsigned long v = (unsigned long)folio->private;		\
 									\
-	v &= ~(1UL << PAGE_PRIVATE_##flagname);				\
-	if (v == (1UL << PAGE_PRIVATE_NOT_POINTER))			\
+	v &= ~(1UL << F2FS_FOLIO_PRIVATE_##flagname);			\
+	if (v == (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER))		\
 		folio_detach_private(folio);				\
 	else								\
 		folio->private = (void *)v;				\
-}									\
-static inline void clear_page_private_##name(struct page *page) \
-{ \
-	clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \
-	if (page_private(page) == BIT(PAGE_PRIVATE_NOT_POINTER)) \
-		detach_page_private(page); \
 }
 
-PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
-PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE);
-PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
-PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE);
+F2FS_FOLIO_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER);
+F2FS_FOLIO_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION);
+F2FS_FOLIO_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE);
 
-PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
-PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE);
-PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
-PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE);
+F2FS_FOLIO_PRIVATE_SET_FUNC(reference, REF_RESOURCE);
+F2FS_FOLIO_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION);
+F2FS_FOLIO_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE);
 
-PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
-PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE);
-PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
-PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
+F2FS_FOLIO_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE);
+F2FS_FOLIO_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION);
+F2FS_FOLIO_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE);
 
 static inline unsigned long folio_get_f2fs_data(struct folio *folio)
 {
 	unsigned long data = (unsigned long)folio->private;
 
-	if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data))
+	if (!test_bit(F2FS_FOLIO_PRIVATE_NOT_POINTER, &data))
 		return 0;
-	return data >> PAGE_PRIVATE_MAX;
+	return data >> F2FS_FOLIO_PRIVATE_MAX;
 }
 
 static inline void folio_set_f2fs_data(struct folio *folio, unsigned long data)
 {
-	data = (1UL << PAGE_PRIVATE_NOT_POINTER) | (data << PAGE_PRIVATE_MAX);
+	data = (1UL << F2FS_FOLIO_PRIVATE_NOT_POINTER) |
+	       (data << F2FS_FOLIO_PRIVATE_MAX);
 
 	if (!folio_test_private(folio))
 		folio_attach_private(folio, (void *)data);
@@ -2772,7 +2823,7 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
 						struct inode *inode,
 						block_t count)
 {
-	blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK;
+	blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK(sbi);
 
 	spin_lock(&sbi->stat_lock);
 	if (unlikely(sbi->total_valid_block_count < count)) {
@@ -2799,9 +2850,9 @@ static inline void dec_valid_block_count(struct f2fs_sb_info *sbi,
 	f2fs_i_blocks_write(inode, count, false, true);
 }
 
-static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
+static inline void inc_cache_count(struct f2fs_sb_info *sbi, int count_type)
 {
-	atomic_inc(&sbi->nr_pages[count_type]);
+	atomic_inc(&sbi->nr_caches[count_type]);
 
 	if (count_type == F2FS_DIRTY_DENTS ||
 			count_type == F2FS_DIRTY_NODES ||
@@ -2814,15 +2865,15 @@ static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type)
 static inline void inode_inc_dirty_pages(struct inode *inode)
 {
 	atomic_inc(&F2FS_I(inode)->dirty_pages);
-	inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
+	inc_cache_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
 	if (IS_NOQUOTA(inode))
-		inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
+		inc_cache_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
 }
 
-static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type)
+static inline void dec_cache_count(struct f2fs_sb_info *sbi, int count_type)
 {
-	atomic_dec(&sbi->nr_pages[count_type]);
+	atomic_dec(&sbi->nr_caches[count_type]);
 }
 
 static inline void inode_dec_dirty_pages(struct inode *inode)
@@ -2832,10 +2883,10 @@ static inline void inode_dec_dirty_pages(struct inode *inode)
 		return;
 
 	atomic_dec(&F2FS_I(inode)->dirty_pages);
-	dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
+	dec_cache_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ?
 				F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA);
 	if (IS_NOQUOTA(inode))
-		dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
+		dec_cache_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA);
 }
 
 static inline void inc_atomic_write_cnt(struct inode *inode)
@@ -2860,9 +2911,9 @@ static inline void release_atomic_write_cnt(struct inode *inode)
 	fi->atomic_write_cnt = 0;
 }
 
-static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type)
+static inline s64 get_nr_caches(struct f2fs_sb_info *sbi, int count_type)
 {
-	return atomic_read(&sbi->nr_pages[count_type]);
+	return atomic_read(&sbi->nr_caches[count_type]);
 }
 
 static inline int get_dirty_pages(struct inode *inode)
@@ -2872,7 +2923,7 @@ static inline int get_dirty_pages(struct inode *inode)
 
 static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type)
 {
-	return div_u64(get_pages(sbi, block_type) + BLKS_PER_SEC(sbi) - 1,
+	return div_u64(get_nr_caches(sbi, block_type) + BLKS_PER_SEC(sbi) - 1,
 							BLKS_PER_SEC(sbi));
 }
 
@@ -2924,7 +2975,7 @@ static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag)
 		if (flag == NAT_BITMAP)
 			return tmp_ptr;
 		else
-			return (unsigned char *)ckpt + F2FS_BLKSIZE;
+			return (unsigned char *)ckpt + F2FS_BLKSIZE(sbi);
 	} else {
 		offset = (flag == NAT_BITMAP) ?
 			le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0;
@@ -3154,14 +3205,51 @@ static inline void f2fs_put_page(struct page *page, bool unlock)
 	f2fs_folio_put(page_folio(page), unlock);
 }
 
+#define F2FS_DENTRY_TAG_CACHE	1UL
+#define F2FS_DENTRY_TAG_MASK	1UL
+
+static inline void *f2fs_cache_make_dentry_block(struct f2fs_cached_block *entry)
+{
+	if (IS_ERR_OR_NULL(entry))
+		return entry;
+	return (void *)((unsigned long)entry | F2FS_DENTRY_TAG_CACHE);
+}
+
+static inline bool f2fs_dentry_is_cache(void *dentry_block)
+{
+	return ((unsigned long)dentry_block & F2FS_DENTRY_TAG_MASK) ==
+						F2FS_DENTRY_TAG_CACHE;
+}
+
+static inline struct f2fs_cached_block *f2fs_dentry_cache(void *dentry_block)
+{
+	return (struct f2fs_cached_block *)
+		((unsigned long)dentry_block & ~F2FS_DENTRY_TAG_MASK);
+}
+
+static inline struct folio *f2fs_dentry_folio(void *dentry_block)
+{
+	return (struct folio *)dentry_block;
+}
+
+static inline void f2fs_put_dentry_block(void *dentry_block, bool unlock)
+{
+	if (IS_ERR_OR_NULL(dentry_block))
+		return;
+	if (f2fs_dentry_is_cache(dentry_block))
+		f2fs_put_cache(f2fs_dentry_cache(dentry_block), unlock);
+	else
+		f2fs_folio_put(f2fs_dentry_folio(dentry_block), unlock);
+}
+
 static inline void f2fs_put_dnode(struct dnode_of_data *dn)
 {
-	if (dn->node_folio)
-		f2fs_folio_put(dn->node_folio, true);
-	if (dn->inode_folio && dn->node_folio != dn->inode_folio)
-		f2fs_folio_put(dn->inode_folio, false);
-	dn->node_folio = NULL;
-	dn->inode_folio = NULL;
+	if (dn->node_entry)
+		f2fs_put_cache(dn->node_entry, true);
+	if (dn->inode_entry && dn->node_entry != dn->inode_entry)
+		f2fs_put_cache(dn->inode_entry, false);
+	dn->node_entry = NULL;
+	dn->inode_entry = NULL;
 }
 
 static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name,
@@ -3195,11 +3283,13 @@ static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep,
 
 static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
 {
-	if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) ||
-		get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) ||
-		get_pages(sbi, F2FS_WB_CP_DATA) ||
-		get_pages(sbi, F2FS_DIO_READ) ||
-		get_pages(sbi, F2FS_DIO_WRITE))
+	if (get_nr_caches(sbi, F2FS_RD_DATA) ||
+	    get_nr_caches(sbi, F2FS_RD_NODE) ||
+	    get_nr_caches(sbi, F2FS_RD_META) ||
+	    get_nr_caches(sbi, F2FS_WB_DATA) ||
+	    get_nr_caches(sbi, F2FS_WB_CP_DATA) ||
+	    get_nr_caches(sbi, F2FS_DIO_READ) ||
+	    get_nr_caches(sbi, F2FS_DIO_WRITE))
 		return true;
 
 	if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info &&
@@ -3214,7 +3304,8 @@ static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type)
 
 static inline bool is_inflight_read_io(struct f2fs_sb_info *sbi)
 {
-	return get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_DIO_READ);
+	return get_nr_caches(sbi, F2FS_RD_DATA) ||
+	       get_nr_caches(sbi, F2FS_DIO_READ);
 }
 
 static inline bool is_idle(struct f2fs_sb_info *sbi, int type)
@@ -3250,13 +3341,11 @@ static inline void f2fs_radix_tree_insert(struct radix_tree_root *root,
 		cond_resched();
 }
 
-#define RAW_IS_INODE(p)	((p)->footer.nid == (p)->footer.ino)
-
-static inline bool IS_INODE(const struct folio *folio)
+static inline bool IS_INODE(struct f2fs_sb_info *sbi, const struct f2fs_cached_block *entry)
 {
-	struct f2fs_node *p = F2FS_NODE(folio);
+	struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry);
 
-	return RAW_IS_INODE(p);
+	return footer->nid == footer->ino;
 }
 
 static inline int offset_in_addr(struct f2fs_inode *i)
@@ -3265,38 +3354,45 @@ static inline int offset_in_addr(struct f2fs_inode *i)
 			(le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0;
 }
 
-static inline __le32 *blkaddr_in_node(struct f2fs_node *node)
+static inline __le32 *blkaddr_in_node(struct f2fs_sb_info *sbi,
+						const struct f2fs_cached_block *entry)
 {
-	return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr;
+	struct f2fs_node *node = F2FS_NODE(entry);
+
+	return IS_INODE(sbi, entry) ? node->i.i_addr : node->dn.addr;
 }
 
 static inline int f2fs_has_extra_attr(struct inode *inode);
+
 static inline unsigned int get_dnode_base(struct inode *inode,
-					struct folio *node_folio)
+					const struct f2fs_cached_block *entry)
 {
-	if (!IS_INODE(node_folio))
+	if (!IS_INODE(inode ? F2FS_I_SB(inode) : entry->cache->sbi, entry))
 		return 0;
 
 	return inode ? get_extra_isize(inode) :
-			offset_in_addr(&F2FS_NODE(node_folio)->i);
+			offset_in_addr(&CACHED_NODE(entry)->i);
 }
 
 static inline __le32 *get_dnode_addr(struct inode *inode,
-					struct folio *node_folio)
+					const struct f2fs_cached_block *entry)
 {
-	return blkaddr_in_node(F2FS_NODE(node_folio)) +
-			get_dnode_base(inode, node_folio);
+	struct f2fs_sb_info *sbi = inode ? F2FS_I_SB(inode) : entry->cache->sbi;
+
+	return blkaddr_in_node(sbi, entry) +
+			get_dnode_base(inode, entry);
 }
 
 static inline block_t data_blkaddr(struct inode *inode,
-			struct folio *node_folio, unsigned int offset)
+			const struct f2fs_cached_block *entry,
+			unsigned int offset)
 {
-	return le32_to_cpu(*(get_dnode_addr(inode, node_folio) + offset));
+	return le32_to_cpu(*(get_dnode_addr(inode, entry) + offset));
 }
 
 static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn)
 {
-	return data_blkaddr(dn->inode, dn->node_folio, dn->ofs_in_node);
+	return data_blkaddr(dn->inode, dn->node_entry, dn->ofs_in_node);
 }
 
 static inline int f2fs_test_bit(unsigned int nr, char *addr)
@@ -3599,20 +3695,26 @@ static inline bool f2fs_need_compress_data(struct inode *inode)
 static inline unsigned int addrs_per_page(struct inode *inode,
 							bool is_inode)
 {
-	unsigned int addrs = is_inode ? (CUR_ADDRS_PER_INODE(inode) -
-			get_inline_xattr_addrs(inode)) : DEF_ADDRS_PER_BLOCK;
+	unsigned int addrs = is_inode ? (cur_addrs_per_inode(inode) -
+			get_inline_xattr_addrs(inode)) :
+			DEF_ADDRS_PER_BLOCK(F2FS_I_SB(inode));
 
 	if (f2fs_compressed_file(inode))
 		return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size);
 	return addrs;
 }
 
-static inline
-void *inline_xattr_addr(struct inode *inode, const struct folio *folio)
+static inline unsigned int cur_addrs_per_inode(struct inode *inode)
 {
-	struct f2fs_inode *ri = F2FS_INODE(folio);
+	return DEF_ADDRS_PER_INODE(F2FS_I_SB(inode)) - get_extra_isize(inode);
+}
 
-	return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE -
+static inline void *inline_xattr_addr(struct inode *inode,
+				const struct f2fs_cached_block *entry)
+{
+	struct f2fs_inode *ri = F2FS_INODE(entry);
+
+	return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE(F2FS_I_SB(inode)) -
 					get_inline_xattr_addrs(inode)]);
 }
 
@@ -3657,9 +3759,10 @@ static inline bool f2fs_is_cow_file(struct inode *inode)
 	return is_inode_flag_set(inode, FI_COW_FILE);
 }
 
-static inline void *inline_data_addr(struct inode *inode, struct folio *folio)
+static inline void *inline_data_addr(struct inode *inode,
+			const struct f2fs_cached_block *entry)
 {
-	__le32 *addr = get_dnode_addr(inode, folio);
+	__le32 *addr = get_dnode_addr(inode, entry);
 
 	return (void *)(addr + DEF_INLINE_RESERVED_SIZE);
 }
@@ -3848,9 +3951,9 @@ int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync);
 int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock);
 int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock);
 int f2fs_truncate(struct inode *inode);
-int f2fs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int f2fs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int flags);
-int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int f2fs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr);
 int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end);
 void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count);
@@ -3858,7 +3961,7 @@ int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag,
 						bool readonly, bool need_lock);
 int f2fs_precache_extents(struct inode *inode);
 int f2fs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int f2fs_fileattr_set(struct mnt_idmap *idmap,
+int f2fs_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa);
 long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
@@ -3869,13 +3972,13 @@ int f2fs_pin_file_control(struct inode *inode, bool inc);
  * inode.c
  */
 void f2fs_set_inode_flags(struct inode *inode);
-bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio);
-void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct folio *folio);
+bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry);
+void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry);
 struct inode *f2fs_iget(struct super_block *sb, unsigned long ino);
 struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino);
 int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink);
-void f2fs_update_inode(struct inode *inode, struct folio *node_folio);
-void f2fs_update_inode_page(struct inode *inode);
+void f2fs_update_inode(struct inode *inode, struct f2fs_cached_block *entry);
+void f2fs_update_inode_cache(struct inode *inode);
 int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc);
 void f2fs_remove_donate_inode(struct inode *inode);
 void f2fs_evict_inode(struct inode *inode);
@@ -3890,7 +3993,7 @@ void f2fs_destroy_evict_inode_work(void);
 int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name,
 							bool hot, bool set);
 struct dentry *f2fs_get_parent(struct dentry *child);
-int f2fs_get_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+int f2fs_get_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct inode **new_inode);
 
 /*
@@ -3924,22 +4027,22 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
 			unsigned int start_pos, struct fscrypt_str *fstr);
 void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent,
 			struct f2fs_dentry_ptr *d);
-struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
-		const struct f2fs_filename *fname, struct folio *dfolio);
+struct f2fs_cached_block *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir,
+		const struct f2fs_filename *fname, struct f2fs_cached_block *dentry);
 void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode,
 			unsigned int current_depth);
 int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots);
 void f2fs_drop_nlink(struct inode *dir, struct inode *inode);
 struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir,
-		const struct f2fs_filename *fname, struct folio **res_folio);
+		const struct f2fs_filename *fname, void **dentry_block);
 struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
-			const struct qstr *child, struct folio **res_folio);
-struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct folio **f);
+			const struct qstr *child, void **dentry_block);
+struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, void **dentry_block);
 ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr,
-			struct folio **folio);
+			void **dentry_block);
 void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
-			struct folio *folio, struct inode *inode);
-bool f2fs_has_enough_room(struct inode *dir, struct folio *ifolio,
+			void *dentry_blk, struct inode *inode);
+bool f2fs_has_enough_room(struct inode *dir, struct f2fs_cached_block *ientry,
 			  const struct f2fs_filename *fname);
 void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d,
 			const struct fscrypt_str *name, f2fs_hash_t name_hash,
@@ -3950,7 +4053,7 @@ int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname,
 			struct inode *inode, nid_t ino, umode_t mode);
 int f2fs_do_add_link(struct inode *dir, const struct qstr *name,
 			struct inode *inode, nid_t ino, umode_t mode);
-void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio,
+void f2fs_delete_entry(struct f2fs_dir_entry *dentry, void *dentry_blk,
 			struct inode *dir, struct inode *inode);
 int f2fs_do_tmpfile(struct inode *inode, struct inode *dir,
 					struct f2fs_filename *fname);
@@ -3972,7 +4075,7 @@ void f2fs_inode_synced(struct inode *inode);
 int f2fs_dquot_initialize(struct inode *inode);
 int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly);
 int f2fs_do_quota_sync(struct super_block *sb, int type);
-loff_t max_file_blocks(struct inode *inode);
+loff_t max_file_blocks(struct f2fs_sb_info *sbi, struct inode *inode);
 void f2fs_quota_off_umount(struct super_block *sb);
 void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag);
 void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error);
@@ -3993,9 +4096,9 @@ enum node_type;
 
 int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid);
 bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type);
-bool f2fs_in_warm_node_list(struct folio *folio);
+bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry);
 void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi);
-void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio);
+void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry);
 void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi);
 bool f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid);
 bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid);
@@ -4006,37 +4109,37 @@ pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs);
 int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode);
 int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from);
 int f2fs_truncate_xattr_node(struct inode *inode);
-int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
+int f2fs_wait_on_node_caches_writeback(struct f2fs_sb_info *sbi,
 					unsigned int seq_id);
-int f2fs_remove_inode_page(struct inode *inode);
-struct folio *f2fs_new_inode_folio(struct inode *inode);
-struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs);
-void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid);
-struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid,
+int f2fs_write_node_caches(struct f2fs_sb_info *sbi);
+int f2fs_remove_inode_cache(struct inode *inode);
+struct f2fs_cached_block *f2fs_new_inode_cache(struct inode *inode);
+struct f2fs_cached_block *f2fs_new_node_cache(struct dnode_of_data *dn, unsigned int ofs);
+void f2fs_ra_node_cache(struct f2fs_sb_info *sbi, nid_t nid);
+struct f2fs_cached_block *f2fs_get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid,
 						enum node_type node_type);
 int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi,
-					struct folio *folio, pgoff_t nid,
+					struct f2fs_cached_block *entry, pgoff_t nid,
 					enum node_type ntype, bool in_irq);
-struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino);
-struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid);
-int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode,
+struct f2fs_cached_block *f2fs_get_inode_cache(struct f2fs_sb_info *sbi, pgoff_t ino);
+struct f2fs_cached_block *f2fs_get_xnode_cache(struct f2fs_sb_info *sbi, pgoff_t xnid);
+int f2fs_write_node_cache(struct f2fs_cached_block *entry, int sync_mode,
 			bool mark_dirty, enum iostat_type io_type);
-int f2fs_move_node_folio(struct folio *node_folio, int gc_type);
+int f2fs_move_node_cache(struct f2fs_cached_block *entry, int gc_type);
 void f2fs_flush_inline_data(struct f2fs_sb_info *sbi);
-int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
-			struct writeback_control *wbc, bool atomic,
-			unsigned int *seq_id);
-int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
-			struct writeback_control *wbc,
-			bool do_balance, enum iostat_type io_type);
+int f2fs_fsync_node_caches(struct f2fs_sb_info *sbi, struct inode *inode,
+					bool atomic, unsigned int *seq_id);
+int f2fs_writeback_node_caches(struct f2fs_sb_info *sbi, long nr_to_write,
+			bool sync, bool do_balance, enum iostat_type io_type);
 int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount);
 bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid);
 void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid);
 void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid);
 int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink);
-int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio);
-int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio);
-int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio);
+int f2fs_recover_inline_xattr(struct inode *inode, struct f2fs_cached_block *entry);
+int f2fs_recover_xattr_data(struct inode *inode, struct f2fs_cached_block *entry);
+int f2fs_recover_inode_cache(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry);
 int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
 			unsigned int segno, struct f2fs_summary_block *sum);
 int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc);
@@ -4064,6 +4167,8 @@ void f2fs_reserve_device_alias(struct f2fs_sb_info *sbi, block_t addr,
 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr);
 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi);
 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi);
+void f2fs_drop_discard_cmd_range(struct f2fs_sb_info *sbi,
+					block_t start, block_t len);
 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi);
 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check);
 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
@@ -4072,7 +4177,7 @@ void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi);
 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi);
 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable);
 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi);
-int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
+int f2fs_nblocks_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra);
 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno);
 int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi);
 int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi);
@@ -4086,12 +4191,14 @@ int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi);
 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range);
 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
 					struct cp_control *cpc);
-struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno);
-void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src,
+struct f2fs_cached_block *f2fs_get_sum_cache(struct f2fs_sb_info *sbi,
+		unsigned int segno);
+void f2fs_update_meta_block(struct f2fs_sb_info *sbi, void *src,
 					block_t blk_addr);
-void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio,
-						enum iostat_type io_type);
-void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio);
+void f2fs_do_write_meta_cache(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry,
+			enum iostat_type io_type);
+void f2fs_do_write_node_cache(unsigned int nid, struct f2fs_io_info *fio);
 void f2fs_outplace_write_data(struct dnode_of_data *dn,
 			struct f2fs_io_info *fio);
 int f2fs_inplace_write_data(struct f2fs_io_info *fio);
@@ -4105,16 +4212,13 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
 			bool recover_newaddr);
 enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi,
 						enum log_type seg_type);
-int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio,
+int f2fs_allocate_data_block(struct f2fs_sb_info *sbi,
 			block_t old_blkaddr, block_t *new_blkaddr,
 			struct f2fs_summary *sum, int type,
 			struct f2fs_io_info *fio);
 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
 					block_t blkaddr, unsigned int blkcnt);
-void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type,
-		bool ordered, bool locked);
-#define f2fs_wait_on_page_writeback(page, type, ordered, locked)	\
-		f2fs_folio_wait_writeback(page_folio(page), type, ordered, locked)
+void f2fs_folio_wait_writeback(struct folio *folio, bool ordered, bool locked);
 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr);
 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
 								block_t len);
@@ -4180,20 +4284,21 @@ void f2fs_unlock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc);
 void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
 							unsigned char reason);
 void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi);
-struct folio *f2fs_grab_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index);
-struct folio *f2fs_get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index);
-struct folio *f2fs_get_meta_folio_retry(struct f2fs_sb_info *sbi, pgoff_t index);
-struct folio *f2fs_get_tmp_folio(struct f2fs_sb_info *sbi, pgoff_t index);
+struct f2fs_cached_block *f2fs_grab_meta_cache(struct f2fs_sb_info *sbi, pgoff_t index);
+struct f2fs_cached_block *f2fs_get_meta_cache(struct f2fs_sb_info *sbi, pgoff_t index);
+struct f2fs_cached_block *f2fs_get_meta_cache_retry(struct f2fs_sb_info *sbi, pgoff_t index);
+struct f2fs_cached_block *f2fs_get_tmp_cache(struct f2fs_sb_info *sbi, pgoff_t index);
 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
 					block_t blkaddr, int type);
 bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi,
 					block_t blkaddr, int type);
-int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
+int f2fs_ra_meta_caches(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 			int type, bool sync);
-void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index,
+void f2fs_ra_meta_caches_cond(struct f2fs_sb_info *sbi, pgoff_t index,
 							unsigned int ra_blocks);
-long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write,
-			enum iostat_type io_type);
+void f2fs_write_meta_caches(struct f2fs_sb_info *sbi);
+long f2fs_sync_meta_caches(struct f2fs_sb_info *sbi, long nr_to_write,
+				bool sync, enum iostat_type io_type);
 void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
 void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type);
 void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all);
@@ -4212,7 +4317,7 @@ void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio);
 void f2fs_remove_dirty_inode(struct inode *inode);
 int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
 								bool from_cp);
-void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type);
+void f2fs_sync_dirty_data(struct f2fs_sb_info *sbi, int type);
 u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi);
 int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc);
 void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi);
@@ -4234,12 +4339,14 @@ void f2fs_destroy_bio_entry_cache(void);
 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio,
 			  enum page_type type);
 int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi);
-void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
-				struct inode *inode, struct folio *folio,
-				nid_t ino, enum page_type type);
+				struct inode *inode, struct folio *folio);
 void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi,
-				struct folio *folio, enum page_type type);
+				struct folio *folio);
+bool f2fs_submit_merged_write_cache(struct f2fs_sb_info *sbi,
+				struct f2fs_cached_block *entry,
+				nid_t ino, enum page_type type);
+void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type);
 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
 					struct bio **bio, struct folio *folio);
 void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi);
@@ -4247,6 +4354,8 @@ void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
 int f2fs_submit_page_bio(struct f2fs_io_info *fio);
 int f2fs_merge_page_bio(struct f2fs_io_info *fio);
 void f2fs_submit_page_write(struct f2fs_io_info *fio);
+int f2fs_submit_cache_read(struct f2fs_io_info *fio);
+void f2fs_submit_cache_write(struct f2fs_io_info *fio);
 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
 		block_t blk_addr, sector_t *sector);
 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr);
@@ -4263,7 +4372,7 @@ struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index,
 struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index,
 			bool for_write);
 struct folio *f2fs_get_new_data_folio(struct inode *inode,
-			struct folio *ifolio, pgoff_t index, bool new_i_size);
+			struct f2fs_cached_block *ientry, pgoff_t index, bool new_i_size);
 int f2fs_do_write_data_page(struct f2fs_io_info *fio);
 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag);
 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
@@ -4375,7 +4484,7 @@ struct f2fs_stat_info {
 	unsigned int bimodal, avg_vblocks;
 	int util_free, util_valid, util_invalid;
 	int rsvd_segs, overp_segs;
-	int dirty_count, node_pages, meta_pages, compress_pages;
+	int dirty_count, node_caches, meta_caches, compress_pages;
 	int compress_page_hit;
 	int prefree_count, free_segs, free_secs;
 	int cp_call_count[MAX_CALL_TYPE], cp_count;
@@ -4398,6 +4507,8 @@ struct f2fs_stat_info {
 	unsigned int block_count[2];
 	unsigned int inplace_count;
 	unsigned long long base_mem, cache_mem, page_mem;
+	unsigned long long cache_entry_mem[NR_CACHE_TYPES];
+	unsigned long long cache_data_mem[NR_CACHE_TYPES];
 	struct f2fs_dev_stats *dev_stats;
 };
 
@@ -4576,8 +4687,6 @@ extern const struct file_operations f2fs_dir_operations;
 extern const struct file_operations f2fs_file_operations;
 extern const struct inode_operations f2fs_file_inode_operations;
 extern const struct address_space_operations f2fs_dblock_aops;
-extern const struct address_space_operations f2fs_node_aops;
-extern const struct address_space_operations f2fs_meta_aops;
 extern const struct inode_operations f2fs_dir_inode_operations;
 extern const struct inode_operations f2fs_symlink_inode_operations;
 extern const struct inode_operations f2fs_encrypted_symlink_inode_operations;
@@ -4588,26 +4697,27 @@ extern struct kmem_cache *f2fs_inode_entry_slab;
  * inline.c
  */
 bool f2fs_may_inline_data(struct inode *inode);
-bool f2fs_sanity_check_inline_data(struct inode *inode, struct folio *ifolio);
+bool f2fs_sanity_check_inline_data(struct inode *inode, struct f2fs_cached_block *ientry);
 bool f2fs_may_inline_dentry(struct inode *inode);
-void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio);
-void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio,
+void f2fs_do_read_inline_data(struct folio *folio, struct f2fs_cached_block *ientry);
+void f2fs_truncate_inline_inode(struct inode *inode, struct f2fs_cached_block *ientry,
 		u64 from);
 int f2fs_read_inline_data(struct inode *inode, struct folio *folio);
 int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio);
 int f2fs_convert_inline_inode(struct inode *inode);
 int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry);
 int f2fs_write_inline_data(struct inode *inode, struct folio *folio);
-int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio);
+int f2fs_recover_inline_data(struct inode *inode, struct f2fs_cached_block *entry);
 struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
-		const struct f2fs_filename *fname, struct folio **res_folio,
+		const struct f2fs_filename *fname, void **dentry_block,
 		bool use_hash);
 int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
-			struct folio *ifolio);
+			struct f2fs_cached_block *ientry);
 int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
 			struct inode *inode, nid_t ino, umode_t mode);
 void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
-		struct folio *folio, struct inode *dir, struct inode *inode);
+		struct f2fs_cached_block *ientry, struct inode *dir,
+		struct inode *inode);
 bool f2fs_empty_inline_dir(struct inode *dir);
 int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
 			struct fscrypt_str *fstr);
@@ -4630,7 +4740,7 @@ void f2fs_leave_shrinker(struct f2fs_sb_info *sbi);
 /*
  * extent_cache.c
  */
-bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio);
+bool sanity_check_extent_cache(struct inode *inode, struct f2fs_cached_block *ientry);
 void f2fs_init_extent_tree(struct inode *inode);
 void f2fs_drop_extent_tree(struct inode *inode);
 void f2fs_destroy_extent_node(struct inode *inode);
@@ -4640,7 +4750,7 @@ int __init f2fs_create_extent_cache(void);
 void f2fs_destroy_extent_cache(void);
 
 /* read extent cache ops */
-void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio);
+void f2fs_init_read_extent_tree(struct inode *inode, struct f2fs_cached_block *ientry);
 bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs,
 			struct extent_info *ei);
 bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index,
@@ -4762,13 +4872,11 @@ unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn,
 int f2fs_init_compress_ctx(struct compress_ctx *cc);
 void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse);
 void f2fs_init_compress_info(struct f2fs_sb_info *sbi);
-int f2fs_init_compress_inode(struct f2fs_sb_info *sbi);
-void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi);
+void f2fs_init_compress_cache_context(struct f2fs_sb_info *sbi);
 int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
 void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
 int __init f2fs_init_compress_cache(void);
 void f2fs_destroy_compress_cache(void);
-struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi);
 void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi,
 					block_t blkaddr, unsigned int len);
 bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio,
@@ -4817,8 +4925,7 @@ static inline void f2fs_put_folio_dic(struct folio *folio, bool in_task)
 static inline unsigned int f2fs_cluster_blocks_are_contiguous(
 			struct dnode_of_data *dn, unsigned int ofs_in_node) { return 0; }
 static inline bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) { return false; }
-static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; }
-static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { }
+static inline void f2fs_init_compress_cache_context(struct f2fs_sb_info *sbi) { }
 static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; }
 static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
 static inline int __init f2fs_init_compress_cache(void) { return 0; }
@@ -5158,10 +5265,8 @@ static inline void f2fs_schedule_timeout_killable(long timeout, bool io)
 }
 
 static inline void f2fs_handle_page_eio(struct f2fs_sb_info *sbi,
-				struct folio *folio, enum page_type type)
+				pgoff_t ofs, enum page_type type)
 {
-	pgoff_t ofs = folio->index;
-
 	if (unlikely(f2fs_cp_error(sbi)))
 		return;
 
@@ -5196,36 +5301,10 @@ static inline bool f2fs_is_readonly(struct f2fs_sb_info *sbi)
 	return f2fs_sb_has_readonly(sbi) || f2fs_readonly(sbi->sb);
 }
 
-static inline void f2fs_truncate_meta_inode_pages(struct f2fs_sb_info *sbi,
-					block_t blkaddr, unsigned int cnt)
-{
-	bool need_submit = false;
-	int i = 0;
-
-	do {
-		struct folio *folio;
-
-		folio = filemap_get_folio(META_MAPPING(sbi), blkaddr + i);
-		if (!IS_ERR(folio)) {
-			if (folio_test_writeback(folio))
-				need_submit = true;
-			f2fs_folio_put(folio, false);
-		}
-	} while (++i < cnt && !need_submit);
-
-	if (need_submit)
-		f2fs_submit_merged_write_cond(sbi, sbi->meta_inode,
-							NULL, 0, DATA);
-
-	truncate_inode_pages_range(META_MAPPING(sbi),
-			F2FS_BLK_TO_BYTES((loff_t)blkaddr),
-			F2FS_BLK_END_BYTES((loff_t)(blkaddr + cnt - 1)));
-}
-
 static inline void f2fs_invalidate_internal_cache(struct f2fs_sb_info *sbi,
 						block_t blkaddr, unsigned int len)
 {
-	f2fs_truncate_meta_inode_pages(sbi, blkaddr, len);
+	f2fs_truncate_meta_caches(sbi, blkaddr, len);
 	f2fs_invalidate_compress_pages_range(sbi, blkaddr, len);
 }
 
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index edc352569e87e3..626c6f97b6f8b1 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -110,7 +110,8 @@ static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf)
 	ret = filemap_fault(vmf);
 	if (ret & VM_FAULT_LOCKED)
 		f2fs_update_iostat(F2FS_I_SB(inode), inode,
-					APP_MAPPED_READ_IO, F2FS_BLKSIZE);
+					APP_MAPPED_READ_IO,
+					F2FS_BLKSIZE(F2FS_I_SB(inode)));
 
 	trace_f2fs_filemap_fault(inode, vmf->pgoff, flags, ret);
 
@@ -212,9 +213,9 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
 		goto out_sem;
 	}
 
-	f2fs_folio_wait_writeback(folio, DATA, false, true);
+	f2fs_folio_wait_writeback(folio, false, true);
 
-	/* wait for GCed page writeback via META_MAPPING */
+	/* wait for GCed page writeback via generic cache */
 	f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
 
 	/*
@@ -233,7 +234,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf)
 	}
 	folio_mark_dirty(folio);
 
-	f2fs_update_iostat(sbi, inode, APP_MAPPED_IO, F2FS_BLKSIZE);
+	f2fs_update_iostat(sbi, inode, APP_MAPPED_IO, F2FS_BLKSIZE(sbi));
 	f2fs_update_time(sbi, REQ_TIME);
 
 out_sem:
@@ -307,13 +308,13 @@ static inline enum cp_reason_type need_do_checkpoint(struct inode *inode)
 
 static bool need_inode_page_update(struct f2fs_sb_info *sbi, nid_t ino)
 {
-	struct folio *i = filemap_get_folio(NODE_MAPPING(sbi), ino);
+	struct f2fs_cached_block *entry = f2fs_find_node_cache(sbi, ino);
 	bool ret = false;
 	/* But we need to avoid that there are some inode updates */
-	if ((!IS_ERR(i) && folio_test_dirty(i)) ||
+	if ((!IS_ERR(entry) && f2fs_cache_test_dirty(entry)) ||
 	    f2fs_need_inode_block_update(sbi, ino))
 		ret = true;
-	f2fs_folio_put(i, false);
+	f2fs_put_cache(entry, false);
 	return ret;
 }
 
@@ -339,10 +340,6 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end,
 	nid_t ino = inode->i_ino;
 	int ret = 0;
 	enum cp_reason_type cp_reason = 0;
-	struct writeback_control wbc = {
-		.sync_mode = WB_SYNC_ALL,
-		.nr_to_write = LONG_MAX,
-	};
 	unsigned int seq_id = 0;
 
 	if (unlikely(f2fs_readonly(inode->i_sb)))
@@ -421,7 +418,7 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end,
 	}
 sync_nodes:
 	atomic_inc(&sbi->wb_sync_req[NODE]);
-	ret = f2fs_fsync_node_pages(sbi, inode, &wbc, atomic, &seq_id);
+	ret = f2fs_fsync_node_caches(sbi, inode, atomic, &seq_id);
 	atomic_dec(&sbi->wb_sync_req[NODE]);
 	if (ret)
 		goto out;
@@ -447,7 +444,7 @@ static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end,
 	 * given fsync mark.
 	 */
 	if (!atomic) {
-		ret = f2fs_wait_on_node_pages_writeback(sbi, seq_id);
+		ret = f2fs_wait_on_node_caches_writeback(sbi, seq_id);
 		if (ret)
 			goto out;
 	}
@@ -484,7 +481,7 @@ static bool __found_offset(struct address_space *mapping,
 	bool compressed_cluster = false;
 
 	if (f2fs_compressed_file(inode)) {
-		block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+		block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 		    ALIGN_DOWN(dn->ofs_in_node, F2FS_I(inode)->i_cluster_size));
 
 		compressed_cluster = first_blkaddr == COMPRESS_ADDR;
@@ -513,7 +510,8 @@ static bool __found_offset(struct address_space *mapping,
 static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence)
 {
 	struct inode *inode = file->f_mapping->host;
-	loff_t maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode));
+	loff_t maxbytes = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode),
+			max_file_blocks(F2FS_I_SB(inode), inode));
 	struct dnode_of_data dn;
 	pgoff_t pgofs, end_offset;
 	loff_t data_ofs = offset;
@@ -537,9 +535,10 @@ static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence)
 		}
 	}
 
-	pgofs = (pgoff_t)(offset >> PAGE_SHIFT);
+	pgofs = F2FS_BYTES_TO_BLK(F2FS_I_SB(inode), offset);
 
-	for (; data_ofs < isize; data_ofs = (loff_t)pgofs << PAGE_SHIFT) {
+	for (; data_ofs < isize;
+			data_ofs = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), pgofs)) {
 		set_new_dnode(&dn, inode, NULL, NULL, 0);
 		err = f2fs_get_dnode_of_data(&dn, pgofs, LOOKUP_NODE);
 		if (err && err != -ENOENT) {
@@ -554,12 +553,12 @@ static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence)
 			}
 		}
 
-		end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
+		end_offset = ADDRS_PER_PAGE(dn.node_entry, inode);
 
 		/* find data/hole in dnode block */
 		for (; dn.ofs_in_node < end_offset;
 				dn.ofs_in_node++, pgofs++,
-				data_ofs = (loff_t)pgofs << PAGE_SHIFT) {
+				data_ofs = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode), pgofs)) {
 			block_t blkaddr;
 
 			blkaddr = f2fs_data_blkaddr(&dn);
@@ -595,7 +594,8 @@ static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence)
 static loff_t f2fs_llseek(struct file *file, loff_t offset, int whence)
 {
 	struct inode *inode = file->f_mapping->host;
-	loff_t maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode));
+	loff_t maxbytes = F2FS_BLK_TO_BYTES(F2FS_I_SB(inode),
+			max_file_blocks(F2FS_I_SB(inode), inode));
 
 	switch (whence) {
 	case SEEK_SET:
@@ -716,7 +716,7 @@ void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count)
 	block_t blkstart;
 	int blklen = 0;
 
-	addr = get_dnode_addr(dn->inode, dn->node_folio) + ofs;
+	addr = get_dnode_addr(dn->inode, dn->node_entry) + ofs;
 	blkstart = le32_to_cpu(*addr);
 
 	/* Assumption: truncation starts with cluster */
@@ -780,7 +780,7 @@ void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count)
 		 * once we invalidate valid blkaddr in range [ofs, ofs + count],
 		 * we will invalidate all blkaddr in the whole range.
 		 */
-		fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_folio),
+		fofs = f2fs_start_bidx_of_node(ofs_of_node(sbi, dn->node_entry),
 							dn->inode) + ofs;
 		f2fs_update_read_extent_cache_range(dn, fofs, 0, len);
 		f2fs_update_age_extent_cache_range(dn, fofs, len);
@@ -818,7 +818,7 @@ static int truncate_partial_data_page(struct inode *inode, u64 from,
 	if (IS_ERR(folio))
 		return PTR_ERR(folio) == -ENOENT ? 0 : PTR_ERR(folio);
 truncate_out:
-	f2fs_folio_wait_writeback(folio, DATA, true, true);
+	f2fs_folio_wait_writeback(folio, true, true);
 	folio_zero_segment(folio, offset, folio_size(folio));
 
 	/* An encrypted inode should have a key and truncate the last page. */
@@ -836,7 +836,7 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock)
 	struct f2fs_lock_context lc;
 	pgoff_t free_from;
 	int count = 0, err = 0;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	bool truncate_page = false;
 
 	trace_f2fs_truncate_blocks_enter(inode, from);
@@ -846,17 +846,17 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock)
 		goto out_err;
 	}
 
-	free_from = (pgoff_t)F2FS_BLK_ALIGN(from);
+	free_from = (pgoff_t)F2FS_BLK_ALIGN(sbi, from);
 
-	if (free_from >= max_file_blocks(inode))
+	if (free_from >= max_file_blocks(sbi, inode))
 		goto free_partial;
 
 	if (lock)
 		f2fs_lock_op(sbi, &lc);
 
-	ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(ifolio)) {
-		err = PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(ientry)) {
+		err = PTR_ERR(ientry);
 		goto out;
 	}
 
@@ -875,18 +875,18 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock)
 
 		f2fs_drop_extent_tree(inode);
 
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 		goto out;
 	}
 
 	if (f2fs_has_inline_data(inode)) {
-		f2fs_truncate_inline_inode(inode, ifolio, from);
-		f2fs_folio_put(ifolio, true);
+		f2fs_truncate_inline_inode(inode, ientry, from);
+		f2fs_put_cache(ientry, true);
 		truncate_page = true;
 		goto out;
 	}
 
-	set_new_dnode(&dn, inode, ifolio, NULL, 0);
+	set_new_dnode(&dn, inode, ientry, NULL, 0);
 	err = f2fs_get_dnode_of_data(&dn, free_from, LOOKUP_NODE_RA);
 	if (err) {
 		if (err == -ENOENT)
@@ -894,12 +894,12 @@ int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock)
 		goto out;
 	}
 
-	count = ADDRS_PER_PAGE(dn.node_folio, inode);
+	count = ADDRS_PER_PAGE(dn.node_entry, inode);
 
 	count -= dn.ofs_in_node;
 	f2fs_bug_on(sbi, count < 0);
 
-	if (dn.ofs_in_node || IS_INODE(dn.node_folio)) {
+	if (dn.ofs_in_node || IS_INODE(sbi, dn.node_entry)) {
 		f2fs_truncate_data_blocks_range(&dn, count);
 		free_from += count;
 	}
@@ -959,9 +959,10 @@ int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock)
 
 int f2fs_truncate(struct inode *inode)
 {
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	int err;
 
-	if (unlikely(f2fs_cp_error(F2FS_I_SB(inode))))
+	if (unlikely(f2fs_cp_error(sbi)))
 		return -EIO;
 
 	if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
@@ -986,8 +987,8 @@ int f2fs_truncate(struct inode *inode)
 			 * leak in evict() path.
 			 */
 			truncate_inode_pages_range(inode->i_mapping,
-					F2FS_BLK_TO_BYTES(0),
-					F2FS_BLK_END_BYTES(0));
+					F2FS_BLK_TO_BYTES(sbi, 0),
+					F2FS_BLK_END_BYTES(sbi, 0));
 			return err;
 		}
 	}
@@ -1032,7 +1033,7 @@ static bool f2fs_force_buffered_io(struct inode *inode, int rw)
 	return false;
 }
 
-int f2fs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int f2fs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -1096,7 +1097,7 @@ int f2fs_getattr(struct mnt_idmap *idmap, const struct path *path,
 }
 
 #ifdef CONFIG_F2FS_FS_POSIX_ACL
-static void __setattr_copy(struct mnt_idmap *idmap,
+static void __setattr_copy(const struct mnt_idmap *idmap,
 			   struct inode *inode, const struct iattr *attr)
 {
 	unsigned int ia_valid = attr->ia_valid;
@@ -1121,7 +1122,7 @@ static void __setattr_copy(struct mnt_idmap *idmap,
 #define __setattr_copy setattr_copy
 #endif
 
-int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int f2fs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -1157,7 +1158,7 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 			return -EOPNOTSUPP;
 		if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED) &&
 			!IS_ALIGNED(attr->ia_size,
-			F2FS_BLK_TO_BYTES(fi->i_cluster_size)))
+			F2FS_BLK_TO_BYTES(sbi, fi->i_cluster_size)))
 			return -EINVAL;
 
 		if (f2fs_is_pinned_file(inode)) {
@@ -1173,7 +1174,7 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 			 * pinned file.
 			 */
 			else if (!IS_ALIGNED(attr->ia_size,
-				F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi))))
+				F2FS_BLK_TO_BYTES(sbi, CAP_BLKS_PER_SEC(sbi))))
 				return -EINVAL;
 		}
 	}
@@ -1304,7 +1305,7 @@ static int fill_zero(struct inode *inode, pgoff_t index,
 	if (IS_ERR(folio))
 		return PTR_ERR(folio);
 
-	f2fs_folio_wait_writeback(folio, DATA, true, true);
+	f2fs_folio_wait_writeback(folio, true, true);
 	folio_zero_range(folio, start, len);
 	folio_mark_dirty(folio);
 	f2fs_folio_put(folio, true);
@@ -1330,7 +1331,7 @@ int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end)
 			return err;
 		}
 
-		end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
+		end_offset = ADDRS_PER_PAGE(dn.node_entry, inode);
 		count = min(end_offset - dn.ofs_in_node, pg_end - pg_start);
 
 		f2fs_bug_on(F2FS_I_SB(inode), count == 0 || count > end_offset);
@@ -1430,7 +1431,7 @@ static int __read_out_blkaddrs(struct inode *inode, block_t *blkaddr,
 		goto next;
 	}
 
-	done = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, inode) -
+	done = min((pgoff_t)ADDRS_PER_PAGE(dn.node_entry, inode) -
 							dn.ofs_in_node, len);
 	for (i = 0; i < done; i++, blkaddr++, do_replace++, dn.ofs_in_node++) {
 		*blkaddr = f2fs_data_blkaddr(&dn);
@@ -1519,7 +1520,7 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode,
 			}
 
 			ilen = min((pgoff_t)
-				ADDRS_PER_PAGE(dn.node_folio, dst_inode) -
+				ADDRS_PER_PAGE(dn.node_entry, dst_inode) -
 						dn.ofs_in_node, len - i);
 			do {
 				dn.data_blkaddr = f2fs_data_blkaddr(&dn);
@@ -1557,7 +1558,7 @@ static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode,
 				return PTR_ERR(fdst);
 			}
 
-			f2fs_folio_wait_writeback(fdst, DATA, true, true);
+			f2fs_folio_wait_writeback(fdst, true, true);
 
 			memcpy_folio(fdst, 0, fsrc, 0, PAGE_SIZE);
 			folio_mark_dirty(fdst);
@@ -1667,7 +1668,8 @@ static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len)
 		return -EINVAL;
 
 	/* collapse range should be aligned to block size of f2fs. */
-	if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1))
+	if (offset & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode)) ||
+	    len & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode)))
 		return -EINVAL;
 
 	ret = f2fs_convert_inline_inode(inode);
@@ -1826,7 +1828,7 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len,
 				goto out;
 			}
 
-			end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
+			end_offset = ADDRS_PER_PAGE(dn.node_entry, inode);
 			end = min(pg_end, end_offset - dn.ofs_in_node + index);
 
 			ret = f2fs_do_zero_range(&dn, index, end);
@@ -1882,7 +1884,8 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len)
 		return -EINVAL;
 
 	/* insert range should be aligned to block size of f2fs. */
-	if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1))
+	if (offset & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode)) ||
+	    len & F2FS_BLKSIZE_MASK(F2FS_I_SB(inode)))
 		return -EINVAL;
 
 	ret = f2fs_convert_inline_inode(inode);
@@ -2361,7 +2364,7 @@ static int f2fs_ioc_getversion(struct file *filp, unsigned long arg)
 static int f2fs_ioc_start_atomic_write(struct file *filp, bool truncate)
 {
 	struct inode *inode = file_inode(filp);
-	struct mnt_idmap *idmap = file_mnt_idmap(filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(filp);
 	struct f2fs_inode_info *fi = F2FS_I(inode);
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	loff_t isize;
@@ -2473,7 +2476,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp, bool truncate)
 static int f2fs_ioc_commit_atomic_write(struct file *filp)
 {
 	struct inode *inode = file_inode(filp);
-	struct mnt_idmap *idmap = file_mnt_idmap(filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(filp);
 	int ret;
 
 	if (!(filp->f_mode & FMODE_WRITE))
@@ -2508,7 +2511,7 @@ static int f2fs_ioc_commit_atomic_write(struct file *filp)
 static int f2fs_ioc_abort_atomic_write(struct file *filp)
 {
 	struct inode *inode = file_inode(filp);
-	struct mnt_idmap *idmap = file_mnt_idmap(filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(filp);
 	int ret;
 
 	if (!(filp->f_mode & FMODE_WRITE))
@@ -2560,7 +2563,7 @@ int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag,
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN);
 		break;
 	case F2FS_GOING_DOWN_METAFLUSH:
-		f2fs_sync_meta_pages(sbi, LONG_MAX, FS_META_IO);
+		f2fs_sync_meta_caches(sbi, LONG_MAX, true, FS_META_IO);
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN);
 		break;
 	case F2FS_GOING_DOWN_NEED_FSCK:
@@ -2644,7 +2647,8 @@ static int f2fs_keep_noreuse_range(struct inode *inode,
 				loff_t offset, loff_t len)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-	u64 max_bytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode));
+	u64 max_bytes = F2FS_BLK_TO_BYTES(sbi,
+					max_file_blocks(sbi, inode));
 	u64 start, end;
 	int ret = 0;
 
@@ -3125,7 +3129,7 @@ static int f2fs_defragment_range(struct f2fs_sb_info *sbi,
 				goto clear_out;
 			}
 
-			f2fs_folio_wait_writeback(folio, DATA, true, true);
+			f2fs_folio_wait_writeback(folio, true, true);
 
 			folio_mark_dirty(folio);
 			folio_set_f2fs_gcing(folio);
@@ -3184,11 +3188,12 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg)
 		return -EFAULT;
 
 	/* verify alignment of offset & size */
-	if (range.start & (F2FS_BLKSIZE - 1) || range.len & (F2FS_BLKSIZE - 1))
+	if (range.start & F2FS_BLKSIZE_MASK(sbi) ||
+	    range.len & F2FS_BLKSIZE_MASK(sbi))
 		return -EINVAL;
 
-	if (unlikely((range.start + range.len) >> PAGE_SHIFT >
-					max_file_blocks(inode)))
+	if (unlikely(F2FS_BYTES_TO_BLK(sbi, range.start + range.len) >
+					max_file_blocks(sbi, inode)))
 		return -EINVAL;
 
 	err = mnt_want_write_file(filp);
@@ -3270,7 +3275,7 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in,
 	if (src == dst && pos_out > pos_in && pos_out < pos_in + len)
 		goto out_unlock;
 	if (pos_in + len == src->i_size)
-		len = ALIGN(src->i_size, F2FS_BLKSIZE) - pos_in;
+		len = ALIGN(src->i_size, F2FS_BLKSIZE(sbi)) - pos_in;
 	if (len == 0) {
 		ret = 0;
 		goto out_unlock;
@@ -3288,9 +3293,9 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in,
 	}
 
 	/* verify the end result is block aligned */
-	if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE) ||
-			!IS_ALIGNED(pos_in + len, F2FS_BLKSIZE) ||
-			!IS_ALIGNED(pos_out, F2FS_BLKSIZE))
+	if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE(sbi)) ||
+			!IS_ALIGNED(pos_in + len, F2FS_BLKSIZE(sbi)) ||
+			!IS_ALIGNED(pos_out, F2FS_BLKSIZE(sbi)))
 		goto out_unlock;
 
 	ret = f2fs_convert_inline_inode(src);
@@ -3322,9 +3327,9 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in,
 	}
 
 	f2fs_lock_op(sbi, &lc);
-	ret = __exchange_data_block(src, dst, F2FS_BYTES_TO_BLK(pos_in),
-				F2FS_BYTES_TO_BLK(pos_out),
-				F2FS_BYTES_TO_BLK(len), false);
+	ret = __exchange_data_block(src, dst, F2FS_BYTES_TO_BLK(sbi, pos_in),
+				F2FS_BYTES_TO_BLK(sbi, pos_out),
+				F2FS_BYTES_TO_BLK(sbi, len), false);
 
 	if (!ret) {
 		if (dst_max_i_size)
@@ -3580,7 +3585,7 @@ int f2fs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int f2fs_fileattr_set(struct mnt_idmap *idmap,
+int f2fs_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
@@ -3843,10 +3848,11 @@ static int f2fs_ioc_reserve_dev_alias(struct file *filp)
 	write_unlock(&et->lock);
 	clear_inode_flag(inode, FI_NO_EXTENT);
 
+	f2fs_drop_discard_cmd_range(sbi, ei.blk, ei.len);
 	f2fs_reserve_device_alias(sbi, ei.blk, ei.len);
 
 	i_size_write(inode, (loff_t)ei.len << sbi->log_blocksize);
-	f2fs_update_inode_page(inode);
+	f2fs_update_inode_cache(inode);
 
 	spin_lock(&FREE_I(sbi)->segmap_lock);
 	FDEV(f2fs_target_device_index(sbi, ei.blk)).is_reserving = false;
@@ -3940,7 +3946,7 @@ static int f2fs_ioc_release_dev_alias(struct file *filp)
 		goto out_inode_unlock;
 	}
 
-	f2fs_update_inode_page(inode);
+	f2fs_update_inode_cache(inode);
 
 	f2fs_unlock_op(sbi, &lc);
 	f2fs_up_write_trace(&sbi->gc_lock, &glc);
@@ -3990,7 +3996,7 @@ int f2fs_precache_extents(struct inode *inode)
 	map.m_next_extent = &m_next_extent;
 	map.m_seg_type = NO_CHECK_TYPE;
 	map.m_may_create = false;
-	end = F2FS_BLK_ALIGN(i_size_read(inode));
+	end = F2FS_BLK_ALIGN(F2FS_I_SB(inode), i_size_read(inode));
 
 	while (map.m_lblk < end) {
 		map.m_len = end - map.m_lblk;
@@ -4159,7 +4165,7 @@ static int release_compress_blocks(struct dnode_of_data *dn, pgoff_t count)
 	int i;
 
 	for (i = 0; i < count; i++) {
-		blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+		blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 						dn->ofs_in_node + i);
 
 		if (!__is_valid_data_blkaddr(blkaddr))
@@ -4278,7 +4284,7 @@ static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg)
 			break;
 		}
 
-		end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
+		end_offset = ADDRS_PER_PAGE(dn.node_entry, inode);
 		count = min(end_offset - dn.ofs_in_node, last_idx - page_idx);
 		count = round_up(count, fi->i_cluster_size);
 
@@ -4329,7 +4335,7 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count,
 	int i;
 
 	for (i = 0; i < count; i++) {
-		blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+		blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 						dn->ofs_in_node + i);
 
 		if (!__is_valid_data_blkaddr(blkaddr))
@@ -4346,7 +4352,7 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count,
 		int ret;
 
 		for (i = 0; i < cluster_size; i++) {
-			blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+			blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 						dn->ofs_in_node + i);
 
 			if (i == 0) {
@@ -4457,7 +4463,7 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg)
 			break;
 		}
 
-		end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
+		end_offset = ADDRS_PER_PAGE(dn.node_entry, inode);
 		count = min(end_offset - dn.ofs_in_node, last_idx - page_idx);
 		count = round_up(count, fi->i_cluster_size);
 
@@ -4506,8 +4512,8 @@ static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg)
 static int f2fs_secure_erase(struct block_device *bdev, struct inode *inode,
 		pgoff_t off, block_t block, block_t len, u32 flags)
 {
-	sector_t sector = SECTOR_FROM_BLOCK(block);
-	sector_t nr_sects = SECTOR_FROM_BLOCK(len);
+	sector_t sector = SECTOR_FROM_BLOCK(F2FS_I_SB(inode), block);
+	sector_t nr_sects = SECTOR_FROM_BLOCK(F2FS_I_SB(inode), len);
 	int ret = 0;
 
 	if (flags & F2FS_TRIM_FILE_DISCARD) {
@@ -4584,14 +4590,14 @@ static int f2fs_sec_trim_file(struct file *filp, unsigned long arg)
 		to_end = true;
 	}
 
-	if (!IS_ALIGNED(range.start, F2FS_BLKSIZE) ||
-			(!to_end && !IS_ALIGNED(end_addr, F2FS_BLKSIZE))) {
+	if (!IS_ALIGNED(range.start, F2FS_BLKSIZE(sbi)) ||
+	    (!to_end && !IS_ALIGNED(end_addr, F2FS_BLKSIZE(sbi)))) {
 		ret = -EINVAL;
 		goto err;
 	}
 
-	index = F2FS_BYTES_TO_BLK(range.start);
-	pg_end = DIV_ROUND_UP(end_addr, F2FS_BLKSIZE);
+	index = F2FS_BYTES_TO_BLK(sbi, range.start);
+	pg_end = F2FS_BLK_ALIGN(sbi, end_addr);
 
 	ret = f2fs_convert_inline_inode(inode);
 	if (ret)
@@ -4623,7 +4629,7 @@ static int f2fs_sec_trim_file(struct file *filp, unsigned long arg)
 			goto out;
 		}
 
-		end_offset = ADDRS_PER_PAGE(dn.node_folio, inode);
+		end_offset = ADDRS_PER_PAGE(dn.node_entry, inode);
 		count = min(end_offset - dn.ofs_in_node, pg_end - index);
 		for (i = 0; i < count; i++, index++, dn.ofs_in_node++) {
 			struct block_device *cur_bdev;
@@ -4819,7 +4825,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len)
 		/* It will never fail, when folio has pinned above */
 		f2fs_bug_on(F2FS_I_SB(inode), IS_ERR(folio));
 
-		f2fs_folio_wait_writeback(folio, DATA, true, true);
+		f2fs_folio_wait_writeback(folio, true, true);
 
 		folio_mark_dirty(folio);
 		folio_set_f2fs_gcing(folio);
@@ -5171,7 +5177,7 @@ static int f2fs_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error,
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(iocb->ki_filp));
 
-	dec_page_count(sbi, F2FS_DIO_READ);
+	dec_cache_count(sbi, F2FS_DIO_READ);
 	if (error)
 		return error;
 	f2fs_update_iostat(sbi, NULL, APP_DIRECT_READ_IO, size);
@@ -5229,13 +5235,13 @@ static ssize_t f2fs_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	 * the higher-level function iomap_dio_rw() in order to ensure that the
 	 * F2FS_DIO_READ counter will be decremented correctly in all cases.
 	 */
-	inc_page_count(sbi, F2FS_DIO_READ);
+	inc_cache_count(sbi, F2FS_DIO_READ);
 	dio = __iomap_dio_rw(iocb, to, &f2fs_iomap_ops,
 			     &f2fs_iomap_dio_read_ops, 0, NULL, 0);
 	if (IS_ERR_OR_NULL(dio)) {
 		ret = PTR_ERR_OR_ZERO(dio);
 		if (ret != -EIOCBQUEUED)
-			dec_page_count(sbi, F2FS_DIO_READ);
+			dec_cache_count(sbi, F2FS_DIO_READ);
 	} else {
 		ret = iomap_dio_complete(dio);
 	}
@@ -5288,7 +5294,7 @@ static ssize_t f2fs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
 	/* In LFS mode, if there is inflight dio, wait for its completion */
 	if (f2fs_lfs_mode(F2FS_I_SB(inode)) &&
-	    get_pages(F2FS_I_SB(inode), F2FS_DIO_WRITE) &&
+	    get_nr_caches(F2FS_I_SB(inode), F2FS_DIO_WRITE) &&
 		(!f2fs_is_pinned_file(inode) || !dio))
 		inode_dio_wait(inode);
 
@@ -5381,7 +5387,8 @@ static int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *iter,
 	 * buffered IO, if DIO meets any holes.
 	 */
 	if (dio && i_size_read(inode) &&
-		(F2FS_BYTES_TO_BLK(pos) < F2FS_BLK_ALIGN(i_size_read(inode))))
+		(F2FS_BYTES_TO_BLK(sbi, pos) <
+		 F2FS_BLK_ALIGN(sbi, i_size_read(inode))))
 		return 0;
 
 	/* No-wait I/O can't allocate blocks. */
@@ -5402,8 +5409,8 @@ static int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *iter,
 	}
 
 	/* Do not preallocate blocks that will be written partially in 4KB. */
-	map.m_lblk = F2FS_BLK_ALIGN(pos);
-	map.m_len = F2FS_BYTES_TO_BLK(pos + count);
+	map.m_lblk = F2FS_BLK_ALIGN(sbi, pos);
+	map.m_len = F2FS_BYTES_TO_BLK(sbi, pos + count);
 	if (map.m_len > map.m_lblk)
 		map.m_len -= map.m_lblk;
 	else
@@ -5453,7 +5460,7 @@ static int f2fs_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error,
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(iocb->ki_filp));
 
-	dec_page_count(sbi, F2FS_DIO_WRITE);
+	dec_cache_count(sbi, F2FS_DIO_WRITE);
 	if (error)
 		return error;
 	f2fs_update_time(sbi, REQ_TIME);
@@ -5564,7 +5571,7 @@ static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from,
 	 * the higher-level function iomap_dio_rw() in order to ensure that the
 	 * F2FS_DIO_WRITE counter will be decremented correctly in all cases.
 	 */
-	inc_page_count(sbi, F2FS_DIO_WRITE);
+	inc_cache_count(sbi, F2FS_DIO_WRITE);
 	dio_flags = 0;
 	if (pos + count > inode->i_size)
 		dio_flags |= IOMAP_DIO_FORCE_WAIT;
@@ -5575,7 +5582,7 @@ static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from,
 		if (ret == -ENOTBLK)
 			ret = 0;
 		if (ret != -EIOCBQUEUED)
-			dec_page_count(sbi, F2FS_DIO_WRITE);
+			dec_cache_count(sbi, F2FS_DIO_WRITE);
 	} else {
 		ret = iomap_dio_complete(dio);
 	}
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index 0a00180c21dc24..926e6976bc22a5 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -1054,7 +1054,7 @@ static int gc_node_segment(struct f2fs_sb_info *sbi,
 
 	for (off = 0; off < usable_blks_in_seg; off++, entry++) {
 		nid_t nid = le32_to_cpu(entry->nid);
-		struct folio *node_folio;
+		struct f2fs_cached_block *node_entry;
 		struct node_info ni;
 		int err;
 
@@ -1066,38 +1066,38 @@ static int gc_node_segment(struct f2fs_sb_info *sbi,
 			continue;
 
 		if (phase == 0) {
-			f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
+			f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nid), 1,
 							META_NAT, true);
 			continue;
 		}
 
 		if (phase == 1) {
-			f2fs_ra_node_page(sbi, nid);
+			f2fs_ra_node_cache(sbi, nid);
 			continue;
 		}
 
 		/* phase == 2 */
-		node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR);
-		if (IS_ERR(node_folio))
+		node_entry = f2fs_get_node_cache(sbi, nid, NODE_TYPE_REGULAR);
+		if (IS_ERR(node_entry))
 			continue;
 
 		/* block may become invalid during f2fs_get_node_folio */
 		if (check_valid_map(sbi, segno, off) == 0) {
-			f2fs_folio_put(node_folio, true);
+			f2fs_put_cache(node_entry, true);
 			continue;
 		}
 
 		if (f2fs_get_node_info(sbi, nid, &ni, false)) {
-			f2fs_folio_put(node_folio, true);
+			f2fs_put_cache(node_entry, true);
 			continue;
 		}
 
 		if (ni.blk_addr != start_addr + off) {
-			f2fs_folio_put(node_folio, true);
+			f2fs_put_cache(node_entry, true);
 			continue;
 		}
 
-		err = f2fs_move_node_folio(node_folio, gc_type);
+		err = f2fs_move_node_cache(node_entry, gc_type);
 		if (!err && gc_type == FG_GC)
 			submitted++;
 		stat_inc_node_blk_count(sbi, 1, gc_type);
@@ -1123,7 +1123,8 @@ static int gc_node_segment(struct f2fs_sb_info *sbi,
  */
 block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode)
 {
-	unsigned int indirect_blks = 2 * NIDS_PER_BLOCK + 4;
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+	unsigned int indirect_blks = 2 * NIDS_PER_BLOCK(sbi) + 4;
 	unsigned int bidx;
 
 	if (node_ofs == 0)
@@ -1132,11 +1133,12 @@ block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode)
 	if (node_ofs <= 2) {
 		bidx = node_ofs - 1;
 	} else if (node_ofs <= indirect_blks) {
-		int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1);
+		int dec = (node_ofs - 4) / (NIDS_PER_BLOCK(sbi) + 1);
 
 		bidx = node_ofs - 2 - dec;
 	} else {
-		int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1);
+		int dec = (node_ofs - indirect_blks - 3) /
+			(NIDS_PER_BLOCK(sbi) + 1);
 
 		bidx = node_ofs - 5 - dec;
 	}
@@ -1146,7 +1148,7 @@ block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode)
 static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
 		struct node_info *dni, block_t blkaddr, unsigned int *nofs)
 {
-	struct folio *node_folio;
+	struct f2fs_cached_block *node_entry;
 	nid_t nid;
 	unsigned int ofs_in_node, max_addrs, base;
 	block_t source_blkaddr;
@@ -1154,12 +1156,12 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
 	nid = le32_to_cpu(sum->nid);
 	ofs_in_node = le16_to_cpu(sum->ofs_in_node);
 
-	node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR);
-	if (IS_ERR(node_folio))
+	node_entry = f2fs_get_node_cache(sbi, nid, NODE_TYPE_REGULAR);
+	if (IS_ERR(node_entry))
 		return false;
 
 	if (f2fs_get_node_info(sbi, nid, dni, false)) {
-		f2fs_folio_put(node_folio, true);
+		f2fs_put_cache(node_entry, true);
 		return false;
 	}
 
@@ -1170,28 +1172,28 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
 	}
 
 	if (f2fs_check_nid_range(sbi, dni->ino)) {
-		f2fs_folio_put(node_folio, true);
+		f2fs_put_cache(node_entry, true);
 		return false;
 	}
 
-	if (IS_INODE(node_folio)) {
-		base = offset_in_addr(F2FS_INODE(node_folio));
-		max_addrs = DEF_ADDRS_PER_INODE;
+	if (IS_INODE(sbi, node_entry)) {
+		base = offset_in_addr(F2FS_INODE(node_entry));
+		max_addrs = DEF_ADDRS_PER_INODE(sbi);
 	} else {
 		base = 0;
-		max_addrs = DEF_ADDRS_PER_BLOCK;
+		max_addrs = DEF_ADDRS_PER_BLOCK(sbi);
 	}
 
 	if (base + ofs_in_node >= max_addrs) {
 		f2fs_err(sbi, "Inconsistent blkaddr offset: base:%u, ofs_in_node:%u, max:%u, ino:%u, nid:%u",
 			base, ofs_in_node, max_addrs, dni->ino, dni->nid);
-		f2fs_folio_put(node_folio, true);
+		f2fs_put_cache(node_entry, true);
 		return false;
 	}
 
-	*nofs = ofs_of_node(node_folio);
-	source_blkaddr = data_blkaddr(NULL, node_folio, ofs_in_node);
-	f2fs_folio_put(node_folio, true);
+	*nofs = ofs_of_node(sbi, node_entry);
+	source_blkaddr = data_blkaddr(NULL, node_entry, ofs_in_node);
+	f2fs_put_cache(node_entry, true);
 
 	if (source_blkaddr != blkaddr) {
 #ifdef CONFIG_F2FS_CHECK_FS
@@ -1217,7 +1219,8 @@ static int ra_data_block(struct inode *inode, pgoff_t index)
 	struct address_space *mapping = inode->i_mapping;
 	struct inode *atomic_inode = NULL;
 	struct dnode_of_data dn;
-	struct folio *folio, *efolio;
+	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	struct f2fs_io_info fio = {
 		.sbi = sbi,
 		.ino = inode->i_ino,
@@ -1227,6 +1230,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index)
 		.op_flags = 0,
 		.encrypted_page = NULL,
 		.in_list = 0,
+		.is_cache = 1,
 	};
 	int err = 0;
 
@@ -1281,36 +1285,35 @@ static int ra_data_block(struct inode *inode, pgoff_t index)
 	 * don't cache encrypted data into meta inode until previous dirty
 	 * data were writebacked to avoid racing between GC and flush.
 	 */
-	f2fs_folio_wait_writeback(folio, DATA, true, true);
+	f2fs_folio_wait_writeback(folio, true, true);
 
 	f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
 
-	efolio = f2fs_filemap_get_folio(META_MAPPING(sbi), dn.data_blkaddr,
-					FGP_LOCK | FGP_CREAT, GFP_NOFS);
-	if (IS_ERR(efolio)) {
-		err = PTR_ERR(efolio);
+	entry = f2fs_grab_cache(META_CACHE(sbi), dn.data_blkaddr,
+						F2FS_CACHE_LOCK_CREATE);
+	if (IS_ERR(entry)) {
+		err = PTR_ERR(entry);
 		goto put_folio;
 	}
 
-	fio.encrypted_page = &efolio->page;
+	if (f2fs_cache_test_uptodate(entry))
+		goto put_cache;
 
-	if (folio_test_uptodate(efolio))
-		goto put_encrypted_page;
-
-	err = f2fs_submit_page_bio(&fio);
+	fio.cache_entry = entry;
+	err = f2fs_submit_cache_read(&fio);
 	if (err)
-		goto put_encrypted_page;
-	f2fs_put_page(fio.encrypted_page, false);
+		goto put_cache;
+	f2fs_put_cache(entry, false);
 	f2fs_folio_put(folio, true);
 
-	f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE);
-	f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE);
+	f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE(sbi));
+	f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE(sbi));
 
 	if (atomic_inode)
 		iput(atomic_inode);
 	return 0;
-put_encrypted_page:
-	f2fs_put_page(fio.encrypted_page, true);
+put_cache:
+	f2fs_put_cache(entry, true);
 put_folio:
 	f2fs_folio_put(folio, true);
 out_iput:
@@ -1320,7 +1323,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index)
 }
 
 /*
- * Move data block via META_MAPPING while keeping locked data page.
+ * Move data block via meta cache while keeping locked data page.
  * This can be used to move blocks, aka LBAs, directly on disk.
  */
 static int move_data_block(struct inode *inode, block_t bidx,
@@ -1337,11 +1340,13 @@ static int move_data_block(struct inode *inode, block_t bidx,
 		.op_flags = 0,
 		.encrypted_page = NULL,
 		.in_list = 0,
+		.is_cache = 1,
 	};
 	struct dnode_of_data dn;
 	struct f2fs_summary sum;
 	struct node_info ni;
-	struct folio *folio, *mfolio, *efolio;
+	struct folio *folio;
+	struct f2fs_cached_block *sentry, *tentry;
 	block_t newaddr;
 	int err = 0;
 	bool lfs_mode = f2fs_lfs_mode(fio.sbi);
@@ -1391,7 +1396,7 @@ static int move_data_block(struct inode *inode, block_t bidx,
 	 * don't cache encrypted data into meta inode until previous dirty
 	 * data were writebacked to avoid racing between GC and flush.
 	 */
-	f2fs_folio_wait_writeback(folio, DATA, true, true);
+	f2fs_folio_wait_writeback(folio, true, true);
 
 	f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
 
@@ -1406,33 +1411,33 @@ static int move_data_block(struct inode *inode, block_t bidx,
 	if (lfs_mode)
 		f2fs_down_write(&fio.sbi->io_order_lock);
 
-	mfolio = f2fs_grab_cache_folio(META_MAPPING(fio.sbi),
-					fio.old_blkaddr, false);
-	if (IS_ERR(mfolio)) {
-		err = PTR_ERR(mfolio);
+	sentry = f2fs_grab_cache(META_CACHE(fio.sbi), fio.old_blkaddr,
+						F2FS_CACHE_LOCK_CREATE);
+	if (IS_ERR(sentry)) {
+		err = PTR_ERR(sentry);
 		goto up_out;
 	}
 
-	fio.encrypted_page = folio_file_page(mfolio, fio.old_blkaddr);
+	fio.cache_entry = sentry;
 
-	/* read source block in mfolio */
-	if (!folio_test_uptodate(mfolio)) {
-		err = f2fs_submit_page_bio(&fio);
+	/* read source block in cache */
+	if (!f2fs_cache_test_uptodate(sentry)) {
+		err = f2fs_submit_cache_read(&fio);
 		if (err) {
-			f2fs_folio_put(mfolio, true);
+			f2fs_put_cache(sentry, true);
 			goto up_out;
 		}
 
 		f2fs_update_iostat(fio.sbi, inode, FS_DATA_READ_IO,
-							F2FS_BLKSIZE);
+						F2FS_BLKSIZE(fio.sbi));
 		f2fs_update_iostat(fio.sbi, NULL, FS_GDATA_READ_IO,
-							F2FS_BLKSIZE);
+						F2FS_BLKSIZE(fio.sbi));
 
-		folio_lock(mfolio);
-		if (unlikely(!is_meta_folio(mfolio) ||
-			     !folio_test_uptodate(mfolio))) {
+		f2fs_lock_cache(sentry);
+		if (unlikely(!f2fs_is_meta_cache(sentry) ||
+				!f2fs_cache_test_uptodate(sentry))) {
 			err = -EIO;
-			f2fs_folio_put(mfolio, true);
+			f2fs_put_cache(sentry, true);
 			goto up_out;
 		}
 	}
@@ -1440,49 +1445,50 @@ static int move_data_block(struct inode *inode, block_t bidx,
 	set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version);
 
 	/* allocate block address */
-	err = f2fs_allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,
+	err = f2fs_allocate_data_block(fio.sbi, fio.old_blkaddr, &newaddr,
 				&sum, type, NULL);
 	if (err) {
-		f2fs_folio_put(mfolio, true);
+		f2fs_put_cache(sentry, true);
 		/* filesystem should shutdown, no need to recovery block */
 		goto up_out;
 	}
 
-	efolio = f2fs_filemap_get_folio(META_MAPPING(fio.sbi), newaddr,
-					FGP_LOCK | FGP_CREAT, GFP_NOFS);
-	if (IS_ERR(efolio)) {
-		err = PTR_ERR(efolio);
-		f2fs_folio_put(mfolio, true);
+	tentry = f2fs_grab_cache(META_CACHE(fio.sbi), newaddr,
+					F2FS_CACHE_LOCK_CREATE);
+	if (IS_ERR(tentry)) {
+		err = PTR_ERR(tentry);
+		f2fs_put_cache(sentry, true);
 		goto recover_block;
 	}
 
-	fio.encrypted_page = &efolio->page;
+	fio.cache_entry = tentry;
 
 	/* write target block */
-	f2fs_wait_on_page_writeback(fio.encrypted_page, DATA, true, true);
-	memcpy(page_address(fio.encrypted_page),
-				folio_address(mfolio), PAGE_SIZE);
-	f2fs_folio_put(mfolio, true);
+	f2fs_cache_wait_writeback_cond(tentry, DATA);
+	memcpy(cache_address(tentry), cache_address(sentry),
+							fio.sbi->blocksize);
+	f2fs_put_cache(sentry, true);
 
 	f2fs_invalidate_internal_cache(fio.sbi, fio.old_blkaddr, 1);
 
-	set_page_dirty(fio.encrypted_page);
-	if (clear_page_dirty_for_io(fio.encrypted_page))
-		dec_page_count(fio.sbi, F2FS_DIRTY_META);
+	f2fs_mark_cache_dirty(tentry);
+	if (f2fs_cache_test_and_clear_dirty(tentry))
+		dec_cache_count(fio.sbi, F2FS_DIRTY_META);
 
-	set_page_writeback(fio.encrypted_page);
+	f2fs_start_cache_writeback(tentry);
 
 	fio.op = REQ_OP_WRITE;
 	fio.op_flags = REQ_SYNC;
 	fio.new_blkaddr = newaddr;
-	f2fs_submit_page_write(&fio);
+	f2fs_submit_cache_write(&fio);
 
-	f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, F2FS_BLKSIZE);
+	f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO,
+					F2FS_BLKSIZE(fio.sbi));
 
 	f2fs_update_data_blkaddr(&dn, newaddr);
 	set_inode_flag(inode, FI_APPEND_WRITE);
 
-	f2fs_put_page(fio.encrypted_page, true);
+	f2fs_put_cache(tentry, true);
 recover_block:
 	if (err)
 		f2fs_do_replace_block(fio.sbi, &sum, newaddr, fio.old_blkaddr,
@@ -1543,7 +1549,7 @@ static int move_data_page(struct inode *inode, block_t bidx, int gc_type,
 		bool is_dirty = folio_test_dirty(folio);
 
 retry:
-		f2fs_folio_wait_writeback(folio, DATA, true, true);
+		f2fs_folio_wait_writeback(folio, true, true);
 
 		folio_mark_dirty(folio);
 		if (folio_clear_dirty_for_io(folio)) {
@@ -1614,13 +1620,13 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
 			continue;
 
 		if (phase == 0) {
-			f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
+			f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nid), 1,
 							META_NAT, true);
 			continue;
 		}
 
 		if (phase == 1) {
-			f2fs_ra_node_page(sbi, nid);
+			f2fs_ra_node_cache(sbi, nid);
 			continue;
 		}
 
@@ -1629,7 +1635,7 @@ static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
 			continue;
 
 		if (phase == 2) {
-			f2fs_ra_node_page(sbi, dni.ino);
+			f2fs_ra_node_cache(sbi, dni.ino);
 			continue;
 		}
 
@@ -1818,28 +1824,31 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 	sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, sbi->sums_per_block);
 	/* readahead multi ssa blocks those have contiguous address */
 	if (__is_large_section(sbi))
-		f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno),
+		f2fs_ra_meta_caches(sbi, GET_SUM_BLOCK(sbi, segno),
 					sum_blk_cnt, META_SSA, true);
 
 	/* reference all summary page */
 	while (segno < end_segno) {
-		struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno);
+		struct f2fs_cached_block *sum_entry =
+				f2fs_get_sum_cache(sbi, segno);
 
 		segno += sbi->sums_per_block;
-		if (IS_ERR(sum_folio)) {
-			int err = PTR_ERR(sum_folio);
+		if (IS_ERR(sum_entry)) {
+			int err = PTR_ERR(sum_entry);
 
 			end_segno = segno - sbi->sums_per_block;
 			segno = rounddown(start_segno, sbi->sums_per_block);
 			while (segno < end_segno) {
-				sum_folio = filemap_get_folio(META_MAPPING(sbi),
+				sum_entry = f2fs_find_meta_cache(sbi,
 						GET_SUM_BLOCK(sbi, segno));
-				folio_put_refs(sum_folio, 2);
+				f2fs_put_cache(sum_entry, false);
+				f2fs_put_cache(sum_entry, false);
 				segno += sbi->sums_per_block;
 			}
 			return err;
 		}
-		folio_unlock(sum_folio);
+		f2fs_unlock_cache(sum_entry);
+
 	}
 
 	blk_start_plug(&plug);
@@ -1847,11 +1856,16 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 	segno = start_segno;
 	while (segno < end_segno) {
 		unsigned int cur_segno;
+		unsigned int block_end_segno;
 
 		/* find segment summary of victim */
-		struct folio *sum_folio = filemap_get_folio(META_MAPPING(sbi),
+		struct f2fs_cached_block *sum_entry =
+				f2fs_find_meta_cache(sbi,
 					GET_SUM_BLOCK(sbi, segno));
-		unsigned int block_end_segno = rounddown(segno, sbi->sums_per_block)
+
+		f2fs_bug_on(sbi, IS_ERR(sum_entry));
+
+		block_end_segno = rounddown(segno, sbi->sums_per_block)
 					+ sbi->sums_per_block;
 
 		if (block_end_segno > end_segno)
@@ -1864,8 +1878,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 			goto next_block;
 		}
 
-		if (!folio_test_uptodate(sum_folio) ||
-		    unlikely(f2fs_cp_error(sbi)))
+		if (!f2fs_cache_test_uptodate(sum_entry) ||
+			unlikely(f2fs_cp_error(sbi)))
 			goto next_block;
 
 		for (cur_segno = segno; cur_segno < block_end_segno;
@@ -1884,7 +1898,7 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 				data_type = (type == SUM_TYPE_DATA) ? DATA : NODE;
 			}
 
-			sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno);
+			sum = SUM_BLK_ENTRY_ADDR(sbi, sum_entry, cur_segno);
 			if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) {
 				f2fs_err(sbi, "Inconsistent segment (%u) type "
 						"[%d, %d] in SIT and SSA",
@@ -1926,12 +1940,14 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 					cur_segno + 1 : NULL_SEGNO;
 
 			if (unlikely(freezing(current))) {
-				folio_put_refs(sum_folio, 2);
+				f2fs_put_cache(sum_entry, false);
+				f2fs_put_cache(sum_entry, false);
 				goto stop;
 			}
 		}
 next_block:
-		folio_put_refs(sum_folio, 2);
+		f2fs_put_cache(sum_entry, false);
+		f2fs_put_cache(sum_entry, false);
 		segno = block_end_segno;
 	}
 
@@ -1963,9 +1979,9 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control)
 
 	trace_f2fs_gc_begin(sbi->sb, gc_type, gc_control->no_bg_gc,
 				gc_control->nr_free_secs,
-				get_pages(sbi, F2FS_DIRTY_NODES),
-				get_pages(sbi, F2FS_DIRTY_DENTS),
-				get_pages(sbi, F2FS_DIRTY_IMETA),
+				get_nr_caches(sbi, F2FS_DIRTY_NODES),
+				get_nr_caches(sbi, F2FS_DIRTY_DENTS),
+				get_nr_caches(sbi, F2FS_DIRTY_IMETA),
 				free_sections(sbi),
 				free_segments(sbi),
 				reserved_segments(sbi),
@@ -2089,9 +2105,9 @@ int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control)
 		f2fs_unpin_all_sections(sbi, true);
 
 	trace_f2fs_gc_end(sbi->sb, ret, total_freed, total_sec_freed,
-				get_pages(sbi, F2FS_DIRTY_NODES),
-				get_pages(sbi, F2FS_DIRTY_DENTS),
-				get_pages(sbi, F2FS_DIRTY_IMETA),
+				get_nr_caches(sbi, F2FS_DIRTY_NODES),
+				get_nr_caches(sbi, F2FS_DIRTY_DENTS),
+				get_nr_caches(sbi, F2FS_DIRTY_IMETA),
 				free_sections(sbi),
 				free_segments(sbi),
 				reserved_segments(sbi),
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index aec06fb4fd76b2..273b6428c9f903 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -34,27 +34,26 @@ bool f2fs_may_inline_data(struct inode *inode)
 	return !f2fs_post_read_required(inode);
 }
 
-static bool inode_has_blocks(struct inode *inode, struct folio *ifolio)
+static bool inode_has_blocks(struct inode *inode, struct f2fs_cached_block *ientry)
 {
-	struct f2fs_inode *ri = F2FS_INODE(ifolio);
 	int i;
 
 	if (F2FS_HAS_BLOCKS(inode))
 		return true;
 
 	for (i = 0; i < DEF_NIDS_PER_INODE; i++) {
-		if (ri->i_nid[i])
+		if (F2FS_INODE_NIDS(F2FS_I_SB(inode), ientry)[i])
 			return true;
 	}
 	return false;
 }
 
-bool f2fs_sanity_check_inline_data(struct inode *inode, struct folio *ifolio)
+bool f2fs_sanity_check_inline_data(struct inode *inode, struct f2fs_cached_block *ientry)
 {
 	if (!f2fs_has_inline_data(inode))
 		return false;
 
-	if (inode_has_blocks(inode, ifolio))
+	if (inode_has_blocks(inode, ientry))
 		return false;
 
 	if (!support_inline_data(inode))
@@ -80,7 +79,7 @@ bool f2fs_may_inline_dentry(struct inode *inode)
 	return true;
 }
 
-void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio)
+void f2fs_do_read_inline_data(struct folio *folio, struct f2fs_cached_block *ientry)
 {
 	struct inode *inode = folio->mapping->host;
 
@@ -92,13 +91,13 @@ void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio)
 	folio_zero_segment(folio, MAX_INLINE_DATA(inode), folio_size(folio));
 
 	/* Copy the whole inline data block */
-	memcpy_to_folio(folio, 0, inline_data_addr(inode, ifolio),
+	memcpy_to_folio(folio, 0, inline_data_addr(inode, ientry),
 		       MAX_INLINE_DATA(inode));
 	if (!folio_test_uptodate(folio))
 		folio_mark_uptodate(folio);
 }
 
-void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio,
+void f2fs_truncate_inline_inode(struct inode *inode, struct f2fs_cached_block *ientry,
 		u64 from)
 {
 	void *addr;
@@ -106,11 +105,11 @@ void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio,
 	if (from >= MAX_INLINE_DATA(inode))
 		return;
 
-	addr = inline_data_addr(inode, ifolio);
+	addr = inline_data_addr(inode, ientry);
 
-	f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+	f2fs_cache_wait_writeback(ientry);
 	memset(addr + from, 0, MAX_INLINE_DATA(inode) - from);
-	folio_mark_dirty(ifolio);
+	f2fs_mark_cache_dirty(ientry);
 
 	if (from == 0)
 		clear_inode_flag(inode, FI_DATA_EXIST);
@@ -118,27 +117,27 @@ void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio,
 
 int f2fs_read_inline_data(struct inode *inode, struct folio *folio)
 {
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 
-	ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
-	if (IS_ERR(ifolio)) {
+	ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino);
+	if (IS_ERR(ientry)) {
 		folio_unlock(folio);
-		return PTR_ERR(ifolio);
+		return PTR_ERR(ientry);
 	}
 
 	if (!f2fs_has_inline_data(inode)) {
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 		return -EAGAIN;
 	}
 
 	if (folio->index)
 		folio_zero_segment(folio, 0, folio_size(folio));
 	else
-		f2fs_do_read_inline_data(folio, ifolio);
+		f2fs_do_read_inline_data(folio, ientry);
 
 	if (!folio_test_uptodate(folio))
 		folio_mark_uptodate(folio);
-	f2fs_folio_put(ifolio, true);
+	f2fs_put_cache(ientry, true);
 	folio_unlock(folio);
 	return 0;
 }
@@ -186,7 +185,7 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio)
 
 	f2fs_bug_on(F2FS_F_SB(folio), folio_test_writeback(folio));
 
-	f2fs_do_read_inline_data(folio, dn->inode_folio);
+	f2fs_do_read_inline_data(folio, dn->inode_entry);
 	folio_mark_dirty(folio);
 
 	/* clear dirty state */
@@ -197,7 +196,7 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio)
 	fio.old_blkaddr = dn->data_blkaddr;
 	set_inode_flag(dn->inode, FI_HOT_DATA);
 	f2fs_outplace_write_data(dn, &fio);
-	f2fs_folio_wait_writeback(folio, DATA, true, true);
+	f2fs_folio_wait_writeback(folio, true, true);
 	if (dirty) {
 		inode_dec_dirty_pages(dn->inode);
 		f2fs_remove_dirty_inode(dn->inode);
@@ -207,8 +206,8 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio)
 	set_inode_flag(dn->inode, FI_APPEND_WRITE);
 
 	/* clear inline data and flag after data writeback */
-	f2fs_truncate_inline_inode(dn->inode, dn->inode_folio, 0);
-	folio_clear_f2fs_inline(dn->inode_folio);
+	f2fs_truncate_inline_inode(dn->inode, dn->inode_entry, 0);
+	f2fs_cache_clear_inline(dn->inode_entry);
 clear_out:
 	stat_dec_inline_inode(dn->inode);
 	clear_inode_flag(dn->inode, FI_INLINE_DATA);
@@ -221,7 +220,8 @@ int f2fs_convert_inline_inode(struct inode *inode)
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct dnode_of_data dn;
 	struct f2fs_lock_context lc;
-	struct folio *ifolio, *folio;
+	struct f2fs_cached_block *ientry;
+	struct folio *folio;
 	int err = 0;
 
 	if (f2fs_hw_is_readonly(sbi) || f2fs_readonly(sbi->sb))
@@ -240,13 +240,13 @@ int f2fs_convert_inline_inode(struct inode *inode)
 
 	f2fs_lock_op(sbi, &lc);
 
-	ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(ifolio)) {
-		err = PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(ientry)) {
+		err = PTR_ERR(ientry);
 		goto out;
 	}
 
-	set_new_dnode(&dn, inode, ifolio, ifolio, 0);
+	set_new_dnode(&dn, inode, ientry, ientry, 0);
 
 	if (f2fs_has_inline_data(inode))
 		err = f2fs_convert_inline_folio(&dn, folio);
@@ -266,35 +266,35 @@ int f2fs_convert_inline_inode(struct inode *inode)
 int f2fs_write_inline_data(struct inode *inode, struct folio *folio)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 
-	ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(ifolio))
-		return PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(ientry))
+		return PTR_ERR(ientry);
 
 	if (!f2fs_has_inline_data(inode)) {
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 		return -EAGAIN;
 	}
 
 	f2fs_bug_on(F2FS_I_SB(inode), folio->index);
 
-	f2fs_folio_wait_writeback(ifolio, NODE, true, true);
-	memcpy_from_folio(inline_data_addr(inode, ifolio),
+	f2fs_cache_wait_writeback(ientry);
+	memcpy_from_folio(inline_data_addr(inode, ientry),
 			 folio, 0, MAX_INLINE_DATA(inode));
-	folio_mark_dirty(ifolio);
+	f2fs_mark_cache_dirty(ientry);
 
 	f2fs_clear_page_cache_dirty_tag(folio);
 
 	set_inode_flag(inode, FI_APPEND_WRITE);
 	set_inode_flag(inode, FI_DATA_EXIST);
 
-	folio_clear_f2fs_inline(ifolio);
-	f2fs_folio_put(ifolio, true);
+	f2fs_cache_clear_inline(ientry);
+	f2fs_put_cache(ientry, true);
 	return 0;
 }
 
-int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio)
+int f2fs_recover_inline_data(struct inode *inode, struct f2fs_cached_block *entry)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_inode *ri = NULL;
@@ -308,39 +308,41 @@ int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio)
 	 *    x       o  -> remove data blocks, and then recover inline_data
 	 *    x       x  -> recover data blocks
 	 */
-	if (IS_INODE(nfolio))
-		ri = F2FS_INODE(nfolio);
+	if (IS_INODE(F2FS_I_SB(inode), entry))
+		ri = &CACHED_NODE(entry)->i;
 
 	if (f2fs_has_inline_data(inode) &&
 			ri && (ri->i_inline & F2FS_INLINE_DATA)) {
-		struct folio *ifolio;
+		struct f2fs_cached_block *ientry;
+
 process_inline:
-		ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
-		if (IS_ERR(ifolio))
-			return PTR_ERR(ifolio);
+		ientry = f2fs_get_inode_cache(sbi, inode->i_ino);
+		if (IS_ERR(ientry))
+			return PTR_ERR(ientry);
 
-		f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+		f2fs_cache_wait_writeback(ientry);
 
-		src_addr = inline_data_addr(inode, nfolio);
-		dst_addr = inline_data_addr(inode, ifolio);
+		src_addr = inline_data_addr(inode, entry);
+		dst_addr = inline_data_addr(inode, ientry);
 		memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode));
 
 		set_inode_flag(inode, FI_INLINE_DATA);
 		set_inode_flag(inode, FI_DATA_EXIST);
 
-		folio_mark_dirty(ifolio);
-		f2fs_folio_put(ifolio, true);
+		f2fs_mark_cache_dirty(ientry);
+		f2fs_put_cache(ientry, true);
 		return 1;
 	}
 
 	if (f2fs_has_inline_data(inode)) {
-		struct folio *ifolio = f2fs_get_inode_folio(sbi, inode->i_ino);
-		if (IS_ERR(ifolio))
-			return PTR_ERR(ifolio);
-		f2fs_truncate_inline_inode(inode, ifolio, 0);
+		struct f2fs_cached_block *ientry = f2fs_get_inode_cache(sbi, inode->i_ino);
+
+		if (IS_ERR(ientry))
+			return PTR_ERR(ientry);
+		f2fs_truncate_inline_inode(inode, ientry, 0);
 		stat_dec_inline_inode(inode);
 		clear_inode_flag(inode, FI_INLINE_DATA);
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 	} else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) {
 		int ret;
 
@@ -355,50 +357,50 @@ int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio)
 
 struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
 					const struct f2fs_filename *fname,
-					struct folio **res_folio,
+					void **dentry_block,
 					bool use_hash)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb);
 	struct f2fs_dir_entry *de;
 	struct f2fs_dentry_ptr d;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	void *inline_dentry;
 
-	ifolio = f2fs_get_inode_folio(sbi, dir->i_ino);
-	if (IS_ERR(ifolio)) {
-		*res_folio = ifolio;
+	ientry = f2fs_get_inode_cache(sbi, dir->i_ino);
+	if (IS_ERR(ientry)) {
+		*dentry_block = ientry;
 		return NULL;
 	}
 
-	inline_dentry = inline_data_addr(dir, ifolio);
+	inline_dentry = inline_data_addr(dir, ientry);
 
 	make_dentry_ptr_inline(dir, &d, inline_dentry);
 	de = f2fs_find_target_dentry(&d, fname, NULL, use_hash);
-	folio_unlock(ifolio);
+	f2fs_unlock_cache(ientry);
 	if (IS_ERR(de)) {
-		*res_folio = ERR_CAST(de);
+		*dentry_block = ERR_CAST(de);
 		de = NULL;
 	}
 	if (de)
-		*res_folio = ifolio;
+		*dentry_block = f2fs_cache_make_dentry_block(ientry);
 	else
-		f2fs_folio_put(ifolio, false);
+		f2fs_put_cache(ientry, false);
 
 	return de;
 }
 
 int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
-							struct folio *ifolio)
+					struct f2fs_cached_block *ientry)
 {
 	struct f2fs_dentry_ptr d;
 	void *inline_dentry;
 
-	inline_dentry = inline_data_addr(inode, ifolio);
+	inline_dentry = inline_data_addr(inode, ientry);
 
 	make_dentry_ptr_inline(inode, &d, inline_dentry);
 	f2fs_do_make_empty_dir(inode, parent, &d);
 
-	folio_mark_dirty(ifolio);
+	f2fs_mark_cache_dirty(ientry);
 
 	/* update i_size to MAX_INLINE_DATA */
 	if (i_size_read(inode) < MAX_INLINE_DATA(inode))
@@ -410,22 +412,23 @@ int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
  * NOTE: ipage is grabbed by caller, but if any error occurs, we should
  * release ipage in this function.
  */
-static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio,
-							void *inline_dentry)
+static int f2fs_move_inline_dirents(struct inode *dir,
+			struct f2fs_cached_block *ientry,
+			void *inline_dentry)
 {
 	struct folio *folio;
 	struct dnode_of_data dn;
-	struct f2fs_dentry_block *dentry_blk;
+	void *dentry_blk;
 	struct f2fs_dentry_ptr src, dst;
 	int err;
 
 	folio = f2fs_grab_cache_folio(dir->i_mapping, 0, true);
 	if (IS_ERR(folio)) {
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 		return PTR_ERR(folio);
 	}
 
-	set_new_dnode(&dn, dir, ifolio, NULL, 0);
+	set_new_dnode(&dn, dir, ientry, NULL, 0);
 	err = f2fs_reserve_block(&dn, 0);
 	if (err)
 		goto out;
@@ -441,7 +444,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio,
 		goto out;
 	}
 
-	f2fs_folio_wait_writeback(folio, DATA, true, true);
+	f2fs_folio_wait_writeback(folio, true, true);
 
 	dentry_blk = folio_address(folio);
 
@@ -449,7 +452,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio,
 	 * Start by zeroing the full block, to ensure that all unused space is
 	 * zeroed and no uninitialized memory is leaked to disk.
 	 */
-	memset(dentry_blk, 0, F2FS_BLKSIZE);
+	memset(dentry_blk, 0, F2FS_BLKSIZE(F2FS_I_SB(dir)));
 
 	make_dentry_ptr_inline(dir, &src, inline_dentry);
 	make_dentry_ptr_block(dir, &dst, dentry_blk);
@@ -464,7 +467,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio,
 	folio_mark_dirty(folio);
 
 	/* clear inline dir and flag after data writeback */
-	f2fs_truncate_inline_inode(dir, ifolio, 0);
+	f2fs_truncate_inline_inode(dir, ientry, 0);
 
 	stat_dec_inline_dir(dir);
 	clear_inode_flag(dir, FI_INLINE_DENTRY);
@@ -478,8 +481,8 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio,
 		F2FS_I(dir)->i_inline_xattr_size = 0;
 
 	f2fs_i_depth_write(dir, 1);
-	if (i_size_read(dir) < PAGE_SIZE)
-		f2fs_i_size_write(dir, PAGE_SIZE);
+	if (i_size_read(dir) < F2FS_BLKSIZE(F2FS_I_SB(dir)))
+		f2fs_i_size_write(dir, F2FS_BLKSIZE(F2FS_I_SB(dir)));
 out:
 	f2fs_folio_put(folio, true);
 	return err;
@@ -544,8 +547,8 @@ static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry)
 	return err;
 }
 
-static int f2fs_move_rehashed_dirents(struct inode *dir, struct folio *ifolio,
-							void *inline_dentry)
+static int f2fs_move_rehashed_dirents(struct inode *dir,
+		struct f2fs_cached_block *ientry, void *inline_dentry)
 {
 	void *backup_dentry;
 	int err;
@@ -553,20 +556,20 @@ static int f2fs_move_rehashed_dirents(struct inode *dir, struct folio *ifolio,
 	backup_dentry = f2fs_kmalloc(F2FS_I_SB(dir),
 				MAX_INLINE_DATA(dir), GFP_F2FS_ZERO);
 	if (!backup_dentry) {
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 		return -ENOMEM;
 	}
 
 	memcpy(backup_dentry, inline_dentry, MAX_INLINE_DATA(dir));
-	f2fs_truncate_inline_inode(dir, ifolio, 0);
+	f2fs_truncate_inline_inode(dir, ientry, 0);
 
-	folio_unlock(ifolio);
+	f2fs_unlock_cache(ientry);
 
 	err = f2fs_add_inline_entries(dir, backup_dentry);
 	if (err)
 		goto recover;
 
-	folio_lock(ifolio);
+	f2fs_lock_cache(ientry);
 
 	stat_dec_inline_dir(dir);
 	clear_inode_flag(dir, FI_INLINE_DENTRY);
@@ -582,31 +585,31 @@ static int f2fs_move_rehashed_dirents(struct inode *dir, struct folio *ifolio,
 	kfree(backup_dentry);
 	return 0;
 recover:
-	folio_lock(ifolio);
-	f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+	f2fs_lock_cache(ientry);
+	f2fs_cache_wait_writeback(ientry);
 	memcpy(inline_dentry, backup_dentry, MAX_INLINE_DATA(dir));
 	f2fs_i_depth_write(dir, 0);
 	f2fs_i_size_write(dir, MAX_INLINE_DATA(dir));
-	folio_mark_dirty(ifolio);
-	f2fs_folio_put(ifolio, true);
+	f2fs_mark_cache_dirty(ientry);
+	f2fs_put_cache(ientry, true);
 
 	kfree(backup_dentry);
 	return err;
 }
 
-static int do_convert_inline_dir(struct inode *dir, struct folio *ifolio,
-							void *inline_dentry)
+static int do_convert_inline_dir(struct inode *dir,
+		struct f2fs_cached_block *ientry, void *inline_dentry)
 {
 	if (!F2FS_I(dir)->i_dir_level)
-		return f2fs_move_inline_dirents(dir, ifolio, inline_dentry);
+		return f2fs_move_inline_dirents(dir, ientry, inline_dentry);
 	else
-		return f2fs_move_rehashed_dirents(dir, ifolio, inline_dentry);
+		return f2fs_move_rehashed_dirents(dir, ientry, inline_dentry);
 }
 
 int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	struct f2fs_filename fname;
 	struct f2fs_lock_context lc;
 	void *inline_dentry = NULL;
@@ -621,22 +624,22 @@ int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry)
 	if (err)
 		goto out;
 
-	ifolio = f2fs_get_inode_folio(sbi, dir->i_ino);
-	if (IS_ERR(ifolio)) {
-		err = PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(sbi, dir->i_ino);
+	if (IS_ERR(ientry)) {
+		err = PTR_ERR(ientry);
 		goto out_fname;
 	}
 
-	if (f2fs_has_enough_room(dir, ifolio, &fname)) {
-		f2fs_folio_put(ifolio, true);
+	if (f2fs_has_enough_room(dir, ientry, &fname)) {
+		f2fs_put_cache(ientry, true);
 		goto out_fname;
 	}
 
-	inline_dentry = inline_data_addr(dir, ifolio);
+	inline_dentry = inline_data_addr(dir, ientry);
 
-	err = do_convert_inline_dir(dir, ifolio, inline_dentry);
+	err = do_convert_inline_dir(dir, ientry, inline_dentry);
 	if (!err)
-		f2fs_folio_put(ifolio, true);
+		f2fs_put_cache(ientry, true);
 out_fname:
 	f2fs_free_filename(&fname);
 out:
@@ -648,24 +651,24 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
 			  struct inode *inode, nid_t ino, umode_t mode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	unsigned int bit_pos;
 	void *inline_dentry = NULL;
 	struct f2fs_dentry_ptr d;
 	int slots = GET_DENTRY_SLOTS(fname->disk_name.len);
-	struct folio *folio = NULL;
+	struct f2fs_cached_block *nentry = NULL;
 	int err = 0;
 
-	ifolio = f2fs_get_inode_folio(sbi, dir->i_ino);
-	if (IS_ERR(ifolio))
-		return PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(sbi, dir->i_ino);
+	if (IS_ERR(ientry))
+		return PTR_ERR(ientry);
 
-	inline_dentry = inline_data_addr(dir, ifolio);
+	inline_dentry = inline_data_addr(dir, ientry);
 	make_dentry_ptr_inline(dir, &d, inline_dentry);
 
 	bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max);
 	if (bit_pos >= d.max) {
-		err = do_convert_inline_dir(dir, ifolio, inline_dentry);
+		err = do_convert_inline_dir(dir, ientry, inline_dentry);
 		if (err)
 			return err;
 		err = -EAGAIN;
@@ -675,19 +678,19 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
 	if (inode) {
 		f2fs_down_write_nested(&F2FS_I(inode)->i_sem,
 						SINGLE_DEPTH_NESTING);
-		folio = f2fs_init_inode_metadata(inode, dir, fname, ifolio);
-		if (IS_ERR(folio)) {
-			err = PTR_ERR(folio);
+		nentry = f2fs_init_inode_metadata(inode, dir, fname, ientry);
+		if (IS_ERR(nentry)) {
+			err = PTR_ERR(nentry);
 			goto fail;
 		}
 	}
 
-	f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+	f2fs_cache_wait_writeback(ientry);
 
 	f2fs_update_dentry(ino, mode, &d, &fname->disk_name, fname->hash,
 			   bit_pos);
 
-	folio_mark_dirty(ifolio);
+	f2fs_mark_cache_dirty(ientry);
 
 	/* we don't need to mark_inode_dirty now */
 	if (inode) {
@@ -695,9 +698,9 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
 
 		/* synchronize inode page's data from inode cache */
 		if (is_inode_flag_set(inode, FI_NEW_INODE))
-			f2fs_update_inode(inode, folio);
+			f2fs_update_inode(inode, nentry);
 
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(nentry, true);
 	}
 
 	f2fs_update_parent_metadata(dir, inode, 0);
@@ -705,12 +708,12 @@ int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname,
 	if (inode)
 		f2fs_up_write(&F2FS_I(inode)->i_sem);
 out:
-	f2fs_folio_put(ifolio, true);
+	f2fs_put_cache(ientry, true);
 	return err;
 }
 
 void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
-		struct folio *folio, struct inode *dir, struct inode *inode)
+		struct f2fs_cached_block *ientry, struct inode *dir, struct inode *inode)
 {
 	struct f2fs_dentry_ptr d;
 	void *inline_dentry;
@@ -718,18 +721,18 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
 	unsigned int bit_pos;
 	int i;
 
-	folio_lock(folio);
-	f2fs_folio_wait_writeback(folio, NODE, true, true);
+	f2fs_lock_cache(ientry);
+	f2fs_cache_wait_writeback(ientry);
 
-	inline_dentry = inline_data_addr(dir, folio);
+	inline_dentry = inline_data_addr(dir, ientry);
 	make_dentry_ptr_inline(dir, &d, inline_dentry);
 
 	bit_pos = dentry - d.dentry;
 	for (i = 0; i < slots; i++)
 		__clear_bit_le(bit_pos + i, d.bitmap);
 
-	folio_mark_dirty(folio);
-	f2fs_folio_put(folio, true);
+	f2fs_mark_cache_dirty(ientry);
+	f2fs_put_cache(ientry, true);
 
 	inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
 	f2fs_mark_inode_dirty_sync(dir, true);
@@ -741,21 +744,21 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry,
 bool f2fs_empty_inline_dir(struct inode *dir)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	unsigned int bit_pos = 2;
 	void *inline_dentry;
 	struct f2fs_dentry_ptr d;
 
-	ifolio = f2fs_get_inode_folio(sbi, dir->i_ino);
-	if (IS_ERR(ifolio))
+	ientry = f2fs_get_inode_cache(sbi, dir->i_ino);
+	if (IS_ERR(ientry))
 		return false;
 
-	inline_dentry = inline_data_addr(dir, ifolio);
+	inline_dentry = inline_data_addr(dir, ientry);
 	make_dentry_ptr_inline(dir, &d, inline_dentry);
 
 	bit_pos = find_next_bit_le(d.bitmap, d.max, bit_pos);
 
-	f2fs_folio_put(ifolio, true);
+	f2fs_put_cache(ientry, true);
 
 	if (bit_pos < d.max)
 		return false;
@@ -767,7 +770,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
 				struct fscrypt_str *fstr)
 {
 	struct inode *inode = file_inode(file);
-	struct folio *ifolio = NULL;
+	struct f2fs_cached_block *ientry = NULL;
 	struct f2fs_dentry_ptr d;
 	void *inline_dentry = NULL;
 	int err;
@@ -777,17 +780,17 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
 	if (ctx->pos == d.max)
 		return 0;
 
-	ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
-	if (IS_ERR(ifolio))
-		return PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino);
+	if (IS_ERR(ientry))
+		return PTR_ERR(ientry);
 
 	/*
 	 * f2fs_readdir was protected by inode.i_rwsem, it is safe to access
 	 * ipage without page's lock held.
 	 */
-	folio_unlock(ifolio);
+	f2fs_unlock_cache(ientry);
 
-	inline_dentry = inline_data_addr(inode, ifolio);
+	inline_dentry = inline_data_addr(inode, ientry);
 
 	make_dentry_ptr_inline(inode, &d, inline_dentry);
 
@@ -795,7 +798,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
 	if (!err)
 		ctx->pos = d.max;
 
-	f2fs_folio_put(ifolio, false);
+	f2fs_put_cache(ientry, false);
 	return err < 0 ? err : 0;
 }
 
@@ -806,12 +809,12 @@ int f2fs_inline_data_fiemap(struct inode *inode,
 	__u32 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED |
 		FIEMAP_EXTENT_LAST;
 	struct node_info ni;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	int err = 0;
 
-	ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
-	if (IS_ERR(ifolio))
-		return PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino);
+	if (IS_ERR(ientry))
+		return PTR_ERR(ientry);
 
 	if ((S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) &&
 				!f2fs_has_inline_data(inode)) {
@@ -825,13 +828,13 @@ int f2fs_inline_data_fiemap(struct inode *inode,
 	}
 
 	if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) {
-		err = f2fs_write_single_node_folio(ifolio, true, false, FS_NODE_IO);
+		err = f2fs_write_node_cache(ientry, true, false, FS_NODE_IO);
 		if (err)
 			return err;
-		ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
-		if (IS_ERR(ifolio))
-			return PTR_ERR(ifolio);
-		f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+		ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino);
+		if (IS_ERR(ientry))
+			return PTR_ERR(ientry);
+		f2fs_cache_wait_writeback(ientry);
 	}
 	ilen = min_t(size_t, MAX_INLINE_DATA(inode), i_size_read(inode));
 	if (start >= ilen)
@@ -846,8 +849,8 @@ int f2fs_inline_data_fiemap(struct inode *inode,
 
 	if (__is_valid_data_blkaddr(ni.blk_addr)) {
 		byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits;
-		byteaddr += (char *)inline_data_addr(inode, ifolio) -
-						(char *)F2FS_INODE(ifolio);
+		byteaddr += (char *)inline_data_addr(inode, ientry) -
+						(char *)F2FS_INODE(ientry);
 	} else {
 		f2fs_bug_on(F2FS_I_SB(inode), ni.blk_addr != NEW_ADDR);
 		flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN;
@@ -855,6 +858,6 @@ int f2fs_inline_data_fiemap(struct inode *inode,
 	err = fiemap_fill_next_extent(fieinfo, start, byteaddr, ilen, flags);
 	trace_f2fs_fiemap(inode, start, byteaddr, ilen, flags, err);
 out:
-	f2fs_folio_put(ifolio, true);
+	f2fs_put_cache(ientry, true);
 	return err;
 }
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index 96cc0e777567ef..9164a2b5d9f091 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -82,9 +82,9 @@ void f2fs_set_inode_flags(struct inode *inode)
 			S_ENCRYPTED|S_VERITY|S_CASEFOLD);
 }
 
-static void __get_inode_rdev(struct inode *inode, struct folio *node_folio)
+static void __get_inode_rdev(struct inode *inode, struct f2fs_cached_block *ientry)
 {
-	__le32 *addr = get_dnode_addr(inode, node_folio);
+	__le32 *addr = get_dnode_addr(inode, ientry);
 
 	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
 			S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
@@ -95,9 +95,9 @@ static void __get_inode_rdev(struct inode *inode, struct folio *node_folio)
 	}
 }
 
-static void __set_inode_rdev(struct inode *inode, struct folio *node_folio)
+static void __set_inode_rdev(struct inode *inode, struct f2fs_cached_block *ientry)
 {
-	__le32 *addr = get_dnode_addr(inode, node_folio);
+	__le32 *addr = get_dnode_addr(inode, ientry);
 
 	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
 		if (old_valid_dev(inode->i_rdev)) {
@@ -111,34 +111,33 @@ static void __set_inode_rdev(struct inode *inode, struct folio *node_folio)
 	}
 }
 
-static void __recover_inline_status(struct inode *inode, struct folio *ifolio)
+static void __recover_inline_status(struct inode *inode, struct f2fs_cached_block *ientry)
 {
-	void *inline_data = inline_data_addr(inode, ifolio);
+	void *inline_data = inline_data_addr(inode, ientry);
 	__le32 *start = inline_data;
 	__le32 *end = start + MAX_INLINE_DATA(inode) / sizeof(__le32);
 
 	while (start < end) {
 		if (*start++) {
-			f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+			f2fs_cache_wait_writeback(ientry);
 
 			set_inode_flag(inode, FI_DATA_EXIST);
-			set_raw_inline(inode, F2FS_INODE(ifolio));
-			folio_mark_dirty(ifolio);
+			set_raw_inline(inode, F2FS_INODE(ientry));
+			f2fs_mark_cache_dirty(ientry);
 			return;
 		}
 	}
 	return;
 }
 
-static
-bool f2fs_enable_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio)
+static bool f2fs_enable_inode_chksum(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry)
 {
-	struct f2fs_inode *ri = &F2FS_NODE(folio)->i;
+	struct f2fs_inode *ri = F2FS_INODE(entry);
 
 	if (!f2fs_sb_has_inode_chksum(sbi))
 		return false;
 
-	if (!IS_INODE(folio) || !(ri->i_inline & F2FS_EXTRA_ATTR))
+	if (!IS_INODE(sbi, entry) || !(ri->i_inline & F2FS_EXTRA_ATTR))
 		return false;
 
 	if (!F2FS_FITS_IN_INODE(ri, le16_to_cpu(ri->i_extra_isize),
@@ -148,11 +147,10 @@ bool f2fs_enable_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio)
 	return true;
 }
 
-static __u32 f2fs_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio)
+static __u32 f2fs_inode_chksum(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry)
 {
-	struct f2fs_node *node = F2FS_NODE(folio);
-	struct f2fs_inode *ri = &node->i;
-	__le32 ino = node->footer.ino;
+	struct f2fs_inode *ri = F2FS_INODE(entry);
+	__le32 ino = F2FS_NODE_FOOTER(sbi, entry)->ino;
 	__le32 gen = ri->i_generation;
 	__u32 chksum, chksum_seed;
 	__u32 dummy_cs = 0;
@@ -166,11 +164,11 @@ static __u32 f2fs_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio)
 	chksum = f2fs_chksum(chksum, (__u8 *)&dummy_cs, cs_size);
 	offset += cs_size;
 	chksum = f2fs_chksum(chksum, (__u8 *)ri + offset,
-			     F2FS_BLKSIZE - offset);
+			     F2FS_BLKSIZE(sbi) - offset);
 	return chksum;
 }
 
-bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio)
+bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry)
 {
 	struct f2fs_inode *ri;
 	__u32 provided, calculated;
@@ -178,35 +176,33 @@ bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio)
 	if (unlikely(is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN)))
 		return true;
 
-#ifdef CONFIG_F2FS_CHECK_FS
-	if (!f2fs_enable_inode_chksum(sbi, folio))
-#else
-	if (!f2fs_enable_inode_chksum(sbi, folio) ||
-			folio_test_dirty(folio) ||
-			folio_test_writeback(folio))
-#endif
+	if (!f2fs_enable_inode_chksum(sbi, entry))
+		return true;
+#ifndef CONFIG_F2FS_CHECK_FS
+	if (f2fs_cache_test_dirty(entry) || f2fs_cache_test_writeback(entry))
 		return true;
+#endif
 
-	ri = &F2FS_NODE(folio)->i;
+	ri = F2FS_INODE(entry);
 	provided = le32_to_cpu(ri->i_inode_checksum);
-	calculated = f2fs_inode_chksum(sbi, folio);
+	calculated = f2fs_inode_chksum(sbi, entry);
 
 	if (provided != calculated)
-		f2fs_warn(sbi, "checksum invalid, nid = %lu, ino_of_node = %x, %x vs. %x",
-			  folio->index, ino_of_node(folio),
+		f2fs_warn(sbi, "checksum invalid, nid = %lu, ino_of_node = %u, %x vs. %x",
+			  entry->index, ino_of_node(sbi, entry),
 			  provided, calculated);
 
 	return provided == calculated;
 }
 
-void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct folio *folio)
+void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry)
 {
-	struct f2fs_inode *ri = &F2FS_NODE(folio)->i;
+	struct f2fs_inode *ri = F2FS_INODE(entry);
 
-	if (!f2fs_enable_inode_chksum(sbi, folio))
+	if (!f2fs_enable_inode_chksum(sbi, entry))
 		return;
 
-	ri->i_inode_checksum = cpu_to_le32(f2fs_inode_chksum(sbi, folio));
+	ri->i_inode_checksum = cpu_to_le32(f2fs_inode_chksum(sbi, entry));
 }
 
 static bool sanity_check_compress_inode(struct inode *inode,
@@ -222,11 +218,11 @@ static bool sanity_check_compress_inode(struct inode *inode,
 		return false;
 	}
 	if (le64_to_cpu(ri->i_compr_blocks) >
-			SECTOR_TO_BLOCK(inode->i_blocks)) {
+			SECTOR_TO_BLOCK(sbi, inode->i_blocks)) {
 		f2fs_warn(sbi,
 			"%s: inode (ino=%llx) has inconsistent i_compr_blocks:%llu, i_blocks:%llu, run fsck to fix",
 			__func__, inode->i_ino, le64_to_cpu(ri->i_compr_blocks),
-			SECTOR_TO_BLOCK(inode->i_blocks));
+			SECTOR_TO_BLOCK(sbi, inode->i_blocks));
 		return false;
 	}
 	if (ri->i_log_cluster_size < MIN_COMPRESS_LOG_SIZE ||
@@ -281,28 +277,29 @@ static bool sanity_check_compress_inode(struct inode *inode,
 	return false;
 }
 
-static bool sanity_check_inode(struct inode *inode, struct folio *node_folio)
+static bool sanity_check_inode(struct inode *inode,
+			struct f2fs_cached_block *node_entry)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_inode_info *fi = F2FS_I(inode);
-	struct f2fs_inode *ri = F2FS_INODE(node_folio);
+	struct f2fs_inode *ri = F2FS_INODE(node_entry);
 	unsigned long long iblocks;
 
-	iblocks = le64_to_cpu(F2FS_INODE(node_folio)->i_blocks);
+	iblocks = le64_to_cpu(ri->i_blocks);
 	if (!iblocks) {
 		f2fs_warn(sbi, "%s: corrupted inode i_blocks i_ino=%llx iblocks=%llu, run fsck to fix.",
 			  __func__, inode->i_ino, iblocks);
 		return false;
 	}
 
-	if (ino_of_node(node_folio) != nid_of_node(node_folio)) {
+	if (ino_of_node(sbi, node_entry) != nid_of_node(sbi, node_entry)) {
 		f2fs_warn(sbi, "%s: corrupted inode footer i_ino=%llx, ino,nid: [%u, %u] run fsck to fix.",
 			  __func__, inode->i_ino,
-			  ino_of_node(node_folio), nid_of_node(node_folio));
+			  ino_of_node(sbi, node_entry), nid_of_node(sbi, node_entry));
 		return false;
 	}
 
-	if (ino_of_node(node_folio) == fi->i_xattr_nid) {
+	if (ino_of_node(sbi, node_entry) == fi->i_xattr_nid) {
 		f2fs_warn(sbi, "%s: corrupted inode i_ino=%llx, xnid=%x, run fsck to fix.",
 			  __func__, inode->i_ino, fi->i_xattr_nid);
 		return false;
@@ -338,12 +335,13 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio)
 	}
 
 	if (f2fs_sb_has_flexible_inline_xattr(sbi) &&
-		(fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE ||
+		(fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE(i_blocksize(inode)) ||
 		(f2fs_has_inline_xattr(inode) &&
 		fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE))) {
-		f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %lu",
+		f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %zu",
 			  __func__, inode->i_ino, fi->i_inline_xattr_size,
-			  MIN_INLINE_XATTR_SIZE, MAX_INLINE_XATTR_SIZE);
+			  MIN_INLINE_XATTR_SIZE,
+			  (size_t)MAX_INLINE_XATTR_SIZE(i_blocksize(inode)));
 		return false;
 	}
 
@@ -375,7 +373,7 @@ static bool sanity_check_inode(struct inode *inode, struct folio *node_folio)
 		}
 	}
 
-	if (f2fs_sanity_check_inline_data(inode, node_folio)) {
+	if (f2fs_sanity_check_inline_data(inode, node_entry)) {
 		f2fs_warn(sbi, "%s: inode (ino=%llx, mode=%u) should not have inline_data, run fsck to fix",
 			  __func__, inode->i_ino, inode->i_mode);
 		return false;
@@ -428,7 +426,7 @@ static int do_read_inode(struct inode *inode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_inode_info *fi = F2FS_I(inode);
-	struct folio *node_folio;
+	struct f2fs_cached_block *node_entry;
 	struct f2fs_inode *ri;
 	projid_t i_projid;
 
@@ -436,18 +434,19 @@ static int do_read_inode(struct inode *inode)
 	if (f2fs_check_nid_range(sbi, inode->i_ino))
 		return -EINVAL;
 
-	node_folio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(node_folio))
-		return PTR_ERR(node_folio);
+	node_entry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(node_entry))
+		return PTR_ERR(node_entry);
 
-	ri = F2FS_INODE(node_folio);
+	ri = F2FS_INODE(node_entry);
 
 	inode->i_mode = le16_to_cpu(ri->i_mode);
 	i_uid_write(inode, le32_to_cpu(ri->i_uid));
 	i_gid_write(inode, le32_to_cpu(ri->i_gid));
 	set_nlink(inode, le32_to_cpu(ri->i_links));
 	inode->i_size = le64_to_cpu(ri->i_size);
-	inode->i_blocks = SECTOR_FROM_BLOCK(le64_to_cpu(ri->i_blocks) - 1);
+	inode->i_blocks = SECTOR_FROM_BLOCK(sbi,
+			le64_to_cpu(ri->i_blocks) - 1);
 
 	inode_set_atime(inode, le64_to_cpu(ri->i_atime),
 			le32_to_cpu(ri->i_atime_nsec));
@@ -490,8 +489,8 @@ static int do_read_inode(struct inode *inode)
 		fi->i_inline_xattr_size = 0;
 	}
 
-	if (!sanity_check_inode(inode, node_folio)) {
-		f2fs_folio_put(node_folio, true);
+	if (!sanity_check_inode(inode, node_entry)) {
+		f2fs_put_cache(node_entry, true);
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
 		f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
 		fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS);
@@ -500,17 +499,17 @@ static int do_read_inode(struct inode *inode)
 
 	/* check data exist */
 	if (f2fs_has_inline_data(inode) && !f2fs_exist_data(inode))
-		__recover_inline_status(inode, node_folio);
+		__recover_inline_status(inode, node_entry);
 
 	/* try to recover cold bit for non-dir inode */
-	if (!S_ISDIR(inode->i_mode) && !is_cold_node(node_folio)) {
-		f2fs_folio_wait_writeback(node_folio, NODE, true, true);
-		set_cold_node(node_folio, false);
-		folio_mark_dirty(node_folio);
+	if (!S_ISDIR(inode->i_mode) && !is_cold_node(sbi, node_entry)) {
+		f2fs_cache_wait_writeback(node_entry);
+		set_cold_node(sbi, node_entry, false);
+		f2fs_mark_cache_dirty(node_entry);
 	}
 
 	/* get rdev by using inline_info */
-	__get_inode_rdev(inode, node_folio);
+	__get_inode_rdev(inode, node_entry);
 
 	if (!f2fs_need_inode_block_update(sbi, inode->i_ino))
 		fi->last_disk_size = inode->i_size;
@@ -553,18 +552,18 @@ static int do_read_inode(struct inode *inode)
 
 	init_idisk_time(inode);
 
-	if (!sanity_check_extent_cache(inode, node_folio)) {
-		f2fs_folio_put(node_folio, true);
+	if (!sanity_check_extent_cache(inode, node_entry)) {
+		f2fs_put_cache(node_entry, true);
 		f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
 		fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS);
 		return -EFSCORRUPTED;
 	}
 
 	/* Need all the flag bits */
-	f2fs_init_read_extent_tree(inode, node_folio);
+	f2fs_init_read_extent_tree(inode, node_entry);
 	f2fs_init_age_extent_tree(inode);
 
-	f2fs_folio_put(node_folio, true);
+	f2fs_put_cache(node_entry, true);
 
 	stat_inc_inline_xattr(inode);
 	stat_inc_inline_inode(inode);
@@ -575,17 +574,6 @@ static int do_read_inode(struct inode *inode)
 	return 0;
 }
 
-static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino)
-{
-	if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi))
-		return true;
-#ifdef CONFIG_F2FS_FS_COMPRESSION
-	if (test_opt(sbi, COMPRESS_CACHE) && ino == F2FS_COMPRESS_INO(sbi))
-		return true;
-#endif
-	return false;
-}
-
 struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(sb);
@@ -597,48 +585,17 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 		return ERR_PTR(-ENOMEM);
 
 	if (!(inode_state_read_once(inode) & I_NEW)) {
-		if (is_meta_ino(sbi, ino)) {
-			f2fs_err(sbi, "inaccessible inode: %lu, run fsck to repair", ino);
-			set_sbi_flag(sbi, SBI_NEED_FSCK);
-			ret = -EFSCORRUPTED;
-			trace_f2fs_iget_exit(inode, ret);
-			iput(inode);
-			f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
-			fserror_report_file_metadata(inode, ret, GFP_NOFS);
-			return ERR_PTR(ret);
-		}
-
 		trace_f2fs_iget(inode);
 		return inode;
 	}
 
-	if (is_meta_ino(sbi, ino))
-		goto make_now;
-
 	ret = do_read_inode(inode);
 	if (ret)
 		goto bad_inode;
-make_now:
+
 	f2fs_set_inode_flags(inode);
 
-	if (ino == F2FS_NODE_INO(sbi)) {
-		inode->i_mapping->a_ops = &f2fs_node_aops;
-		mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
-	} else if (ino == F2FS_META_INO(sbi)) {
-		inode->i_mapping->a_ops = &f2fs_meta_aops;
-		mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS);
-	} else if (ino == F2FS_COMPRESS_INO(sbi)) {
-#ifdef CONFIG_F2FS_FS_COMPRESSION
-		inode->i_mapping->a_ops = &f2fs_compress_aops;
-		/*
-		 * generic_error_remove_folio only truncates pages of regular
-		 * inode
-		 */
-		inode->i_mode |= S_IFREG;
-#endif
-		mapping_set_gfp_mask(inode->i_mapping,
-			GFP_NOFS | __GFP_HIGHMEM | __GFP_MOVABLE);
-	} else if (S_ISREG(inode->i_mode)) {
+	if (S_ISREG(inode->i_mode)) {
 		inode->i_op = &f2fs_file_inode_operations;
 		inode->i_fop = &f2fs_file_operations;
 		inode->i_mapping->a_ops = &f2fs_dblock_aops;
@@ -694,25 +651,28 @@ struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino)
 	return inode;
 }
 
-void f2fs_update_inode(struct inode *inode, struct folio *node_folio)
+void f2fs_update_inode(struct inode *inode,
+		struct f2fs_cached_block *node_entry)
 {
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_inode_info *fi = F2FS_I(inode);
 	struct f2fs_inode *ri;
 	struct extent_tree *et = fi->extent_tree[EX_READ];
 
-	f2fs_folio_wait_writeback(node_folio, NODE, true, true);
-	folio_mark_dirty(node_folio);
+	f2fs_cache_wait_writeback(node_entry);
+	f2fs_mark_cache_dirty(node_entry);
 
 	f2fs_inode_synced(inode);
 
-	ri = F2FS_INODE(node_folio);
+	ri = F2FS_INODE(node_entry);
 
 	ri->i_mode = cpu_to_le16(inode->i_mode);
 	ri->i_advise = fi->i_advise;
 	ri->i_uid = cpu_to_le32(i_uid_read(inode));
 	ri->i_gid = cpu_to_le32(i_gid_read(inode));
 	ri->i_links = cpu_to_le32(inode->i_nlink);
-	ri->i_blocks = cpu_to_le64(SECTOR_TO_BLOCK(READ_ONCE(inode->i_blocks)) + 1);
+	ri->i_blocks = cpu_to_le64(SECTOR_TO_BLOCK(sbi,
+					   READ_ONCE(inode->i_blocks)) + 1);
 
 	if (!f2fs_is_atomic_file(inode) ||
 			is_inode_flag_set(inode, FI_ATOMIC_COMMITTED))
@@ -780,27 +740,27 @@ void f2fs_update_inode(struct inode *inode, struct folio *node_folio)
 		}
 	}
 
-	__set_inode_rdev(inode, node_folio);
+	__set_inode_rdev(inode, node_entry);
 
 	/* deleted inode */
 	if (inode->i_nlink == 0)
-		folio_clear_f2fs_inline(node_folio);
+		f2fs_cache_clear_inline(node_entry);
 
 	init_idisk_time(inode);
 #ifdef CONFIG_F2FS_CHECK_FS
-	f2fs_inode_chksum_set(F2FS_I_SB(inode), node_folio);
+	f2fs_inode_chksum_set(F2FS_I_SB(inode), node_entry);
 #endif
 }
 
-void f2fs_update_inode_page(struct inode *inode)
+void f2fs_update_inode_cache(struct inode *inode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-	struct folio *node_folio;
+	struct f2fs_cached_block *node_entry;
 	int count = 0;
 retry:
-	node_folio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(node_folio)) {
-		int err = PTR_ERR(node_folio);
+	node_entry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(node_entry)) {
+		int err = PTR_ERR(node_entry);
 
 		/* The node block was truncated. */
 		if (err == -ENOENT)
@@ -816,18 +776,14 @@ void f2fs_update_inode_page(struct inode *inode)
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_UPDATE_INODE);
 		return;
 	}
-	f2fs_update_inode(inode, node_folio);
-	f2fs_folio_put(node_folio, true);
+	f2fs_update_inode(inode, node_entry);
+	f2fs_put_cache(node_entry, true);
 }
 
 int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 
-	if (inode->i_ino == F2FS_NODE_INO(sbi) ||
-			inode->i_ino == F2FS_META_INO(sbi))
-		return 0;
-
 	/*
 	 * atime could be updated without dirtying f2fs inode in lazytime mode
 	 */
@@ -851,7 +807,7 @@ int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc)
 	 * We need to balance fs here to prevent from producing dirty node pages
 	 * during the urgent cleaning time when running out of free sections.
 	 */
-	f2fs_update_inode_page(inode);
+	f2fs_update_inode_cache(inode);
 	if (wbc && wbc->nr_to_write)
 		f2fs_balance_fs(sbi, true);
 	return 0;
@@ -892,7 +848,7 @@ static void f2fs_evict_inode_work(struct work_struct *work)
 /*
  * Return true, if we shouldn't go through post_evict_inode.
  */
-static bool f2fs_pre_evict_inode(struct inode *inode)
+static void f2fs_pre_evict_inode(struct inode *inode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_inode_info *fi = F2FS_I(inode);
@@ -918,19 +874,12 @@ static bool f2fs_pre_evict_inode(struct inode *inode)
 	    test_opt(sbi, COMPRESS_CACHE) && f2fs_compressed_file(inode))
 		f2fs_invalidate_compress_pages(sbi, inode->i_ino);
 
-	if (inode->i_ino == F2FS_NODE_INO(sbi) ||
-	    inode->i_ino == F2FS_META_INO(sbi) ||
-	    inode->i_ino == F2FS_COMPRESS_INO(sbi))
-		return true;
-
 	f2fs_bug_on(sbi, get_dirty_pages(inode));
 	f2fs_remove_dirty_inode(inode);
 	f2fs_remove_donate_inode(inode);
 
 	if (!IS_DEVICE_ALIASING(inode))
 		f2fs_destroy_extent_tree(inode);
-
-	return false;
 }
 
 static void f2fs_delete_inode(struct inode *inode)
@@ -964,7 +913,7 @@ static void f2fs_delete_inode(struct inode *inode)
 		goto error_check;
 
 	f2fs_lock_op(sbi, &lc);
-	err = f2fs_remove_inode_page(inode);
+	err = f2fs_remove_inode_cache(inode);
 	f2fs_unlock_op(sbi, &lc);
 
 	if (err == -ENOENT) {
@@ -996,13 +945,13 @@ static void f2fs_delete_inode(struct inode *inode)
 	if (!err)
 		goto unfreeze_out;
 
-	f2fs_update_inode_page(inode);
+	f2fs_update_inode_cache(inode);
 
 	if (dquot_initialize_needed(inode))
 		set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
 
 	/*
-	 * If both f2fs_truncate() and f2fs_update_inode_page() failed
+	 * If both f2fs_truncate() and f2fs_update_inode_cache() failed
 	 * due to fuzzed corrupted inode, call f2fs_inode_synced() to
 	 * avoid triggering later f2fs_bug_on().
 	 */
@@ -1046,18 +995,17 @@ static void f2fs_post_evict_inode(struct inode *inode)
 
 	/* for the case f2fs_new_inode() was failed, .i_ino is zero, skip it */
 	if (inode->i_ino)
-		invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino,
-							inode->i_ino);
+		f2fs_invalidate_node_cache(sbi, inode->i_ino);
 	if (xnid)
-		invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid);
+		f2fs_invalidate_node_cache(sbi, xnid);
 
 	if (!inode->i_nlink)
 		goto skip_record;
 
 	if (is_inode_flag_set(inode, FI_APPEND_WRITE))
-		record_bits = BIT(APPEND_INO);
+		record_bits |= BIT(APPEND_INO);
 	if (is_inode_flag_set(inode, FI_UPDATE_WRITE))
-		record_bits = BIT(UPDATE_INO);
+		record_bits |= BIT(UPDATE_INO);
 
 	if (!record_bits)
 		goto skip_record;
@@ -1093,15 +1041,13 @@ static void f2fs_post_evict_inode(struct inode *inode)
  */
 void f2fs_evict_inode(struct inode *inode)
 {
-	if (f2fs_pre_evict_inode(inode))
-		goto clear_out;
+	f2fs_pre_evict_inode(inode);
 
 	if (!inode->i_nlink && !is_bad_inode(inode))
 		f2fs_delete_inode(inode);
 
 	f2fs_post_evict_inode(inode);
 
-clear_out:
 	fscrypt_put_encryption_info(inode);
 	clear_inode(inode);
 }
@@ -1124,7 +1070,7 @@ void f2fs_handle_failed_inode(struct inode *inode,
 	 * we must call this to avoid inode being remained as dirty, resulting
 	 * in a panic when flushing dirty inodes in gdirty_list.
 	 */
-	f2fs_update_inode_page(inode);
+	f2fs_update_inode_cache(inode);
 	f2fs_inode_synced(inode);
 
 	/* don't make bad inode, since it becomes a regular file. */
diff --git a/fs/f2fs/iostat.h b/fs/f2fs/iostat.h
index 2025225b5bedac..61c6bc8e311986 100644
--- a/fs/f2fs/iostat.h
+++ b/fs/f2fs/iostat.h
@@ -60,6 +60,13 @@ static inline struct bio_post_read_ctx *get_post_read_ctx(struct bio *bio)
 	return iostat_ctx->post_read_ctx;
 }
 
+static inline void iostat_set_post_read_ctx(struct bio *bio, void *ctx)
+{
+	struct bio_iostat_ctx *iostat_ctx = bio->bi_private;
+
+	iostat_ctx->post_read_ctx = ctx;
+}
+
 extern void iostat_update_and_unbind_ctx(struct bio *bio);
 extern void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
 		struct bio *bio, struct bio_post_read_ctx *ctx);
@@ -81,6 +88,10 @@ static inline struct bio_post_read_ctx *get_post_read_ctx(struct bio *bio)
 {
 	return bio->bi_private;
 }
+static inline void iostat_set_post_read_ctx(struct bio *bio, void *ctx)
+{
+	bio->bi_private = ctx;
+}
 static inline int f2fs_init_iostat_processing(void) { return 0; }
 static inline void f2fs_destroy_iostat_processing(void) {}
 static inline int f2fs_init_iostat(struct f2fs_sb_info *sbi) { return 0; }
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index ff86ee07290d18..ce5d536d892bdb 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -231,7 +231,7 @@ static void set_file_temperature(struct f2fs_sb_info *sbi, struct inode *inode,
 		file_set_hot(inode);
 }
 
-static struct inode *f2fs_new_inode(struct mnt_idmap *idmap,
+static struct inode *f2fs_new_inode(const struct mnt_idmap *idmap,
 						struct inode *dir, umode_t mode,
 						const char *name)
 {
@@ -365,7 +365,7 @@ static struct inode *f2fs_new_inode(struct mnt_idmap *idmap,
 	return ERR_PTR(err);
 }
 
-static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int f2fs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
@@ -473,12 +473,13 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir,
 
 struct dentry *f2fs_get_parent(struct dentry *child)
 {
-	struct folio *folio;
-	unsigned long ino = f2fs_inode_by_name(d_inode(child), &dotdot_name, &folio);
+	void *dentry_block = NULL;
+	unsigned long ino = f2fs_inode_by_name(d_inode(child),
+					&dotdot_name, &dentry_block);
 
 	if (!ino) {
-		if (IS_ERR(folio))
-			return ERR_CAST(folio);
+		if (IS_ERR(dentry_block))
+			return ERR_CAST(dentry_block);
 		return ERR_PTR(-ENOENT);
 	}
 	return d_obtain_alias(f2fs_iget(child->d_sb, ino));
@@ -489,7 +490,7 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
 {
 	struct inode *inode = NULL;
 	struct f2fs_dir_entry *de;
-	struct folio *folio;
+	void *dentry_block = NULL;
 	struct dentry *new;
 	nid_t ino = -1;
 	int err = 0;
@@ -507,12 +508,12 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
 		goto out_splice;
 	if (err)
 		goto out;
-	de = __f2fs_find_entry(dir, &fname, &folio);
+	de = __f2fs_find_entry(dir, &fname, &dentry_block);
 	f2fs_free_filename(&fname);
 
 	if (!de) {
-		if (IS_ERR(folio)) {
-			err = PTR_ERR(folio);
+		if (IS_ERR(dentry_block)) {
+			err = PTR_ERR(dentry_block);
 			goto out;
 		}
 		err = -ENOENT;
@@ -520,7 +521,7 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
 	}
 
 	ino = le32_to_cpu(de->ino);
-	f2fs_folio_put(folio, false);
+	f2fs_put_dentry_block(dentry_block, false);
 
 	inode = f2fs_iget(dir->i_sb, ino);
 	if (IS_ERR(inode)) {
@@ -572,7 +573,7 @@ static int __do_unlink(struct inode *dir, struct inode *inode,
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
 	struct f2fs_dir_entry *de;
 	struct f2fs_lock_context lc;
-	struct folio *folio;
+	void *dentry_block = NULL;
 	int err;
 
 	if (IS_DEVICE_ALIASING(inode))
@@ -588,9 +589,9 @@ static int __do_unlink(struct inode *dir, struct inode *inode,
 	if (err)
 		return err;
 
-	de = f2fs_find_entry(dir, name, &folio);
+	de = f2fs_find_entry(dir, name, &dentry_block);
 	if (!de)
-		return IS_ERR(folio) ? PTR_ERR(folio) : 0;
+		return IS_ERR(dentry_block) ? PTR_ERR(dentry_block) : 0;
 
 	if (unlikely(inode->i_nlink == 0)) {
 		f2fs_warn(sbi, "%s: inode (ino=%llx) has zero i_nlink",
@@ -610,7 +611,7 @@ static int __do_unlink(struct inode *dir, struct inode *inode,
 		f2fs_unlock_op(sbi, &lc);
 		goto err_out;
 	}
-	f2fs_delete_entry(de, folio, dir, inode);
+	f2fs_delete_entry(de, dentry_block, dir, inode);
 	f2fs_unlock_op(sbi, &lc);
 	return 0;
 
@@ -618,7 +619,7 @@ static int __do_unlink(struct inode *dir, struct inode *inode,
 	err = -EFSCORRUPTED;
 	set_sbi_flag(sbi, SBI_NEED_FSCK);
 err_out:
-	f2fs_folio_put(folio, false);
+	f2fs_put_dentry_block(dentry_block, false);
 	return err;
 }
 
@@ -662,7 +663,7 @@ static const char *f2fs_get_link(struct dentry *dentry,
 	return link;
 }
 
-static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int f2fs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, const char *symname)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
@@ -751,7 +752,7 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	goto out;
 }
 
-static struct dentry *f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *f2fs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
@@ -810,7 +811,7 @@ static int f2fs_rmdir(struct inode *dir, struct dentry *dentry)
 	return -ENOTEMPTY;
 }
 
-static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int f2fs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
@@ -857,7 +858,7 @@ static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int __f2fs_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			  struct file *file, umode_t mode, bool is_whiteout,
 			  struct inode **new_inode, struct f2fs_filename *fname)
 {
@@ -928,7 +929,7 @@ static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int f2fs_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			struct file *file, umode_t mode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
@@ -944,7 +945,7 @@ static int f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	return finish_open_simple(file, err);
 }
 
-static int f2fs_create_whiteout(struct mnt_idmap *idmap,
+static int f2fs_create_whiteout(const struct mnt_idmap *idmap,
 				struct inode *dir, struct inode **whiteout,
 				struct f2fs_filename *fname)
 {
@@ -952,14 +953,14 @@ static int f2fs_create_whiteout(struct mnt_idmap *idmap,
 						true, whiteout, fname);
 }
 
-int f2fs_get_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+int f2fs_get_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct inode **new_inode)
 {
 	return __f2fs_tmpfile(idmap, dir, NULL, S_IFREG,
 				false, new_inode, NULL);
 }
 
-static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int f2fs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 			struct dentry *old_dentry, struct inode *new_dir,
 			struct dentry *new_dentry, unsigned int flags)
 {
@@ -967,8 +968,9 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 	struct inode *old_inode = d_inode(old_dentry);
 	struct inode *new_inode = d_inode(new_dentry);
 	struct inode *whiteout = NULL;
-	struct folio *old_dir_folio = NULL;
-	struct folio *old_folio, *new_folio = NULL;
+	void *old_dir_block = NULL;
+	void *old_block = NULL;
+	void *new_block = NULL;
 	struct f2fs_dir_entry *old_dir_entry = NULL;
 	struct f2fs_dir_entry *old_entry;
 	struct f2fs_dir_entry *new_entry;
@@ -1032,18 +1034,18 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 	}
 
 	err = -ENOENT;
-	old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_folio);
+	old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_block);
 	if (!old_entry) {
-		if (IS_ERR(old_folio))
-			err = PTR_ERR(old_folio);
+		if (IS_ERR(old_block))
+			err = PTR_ERR(old_block);
 		goto out;
 	}
 
 	if (old_is_dir && old_dir != new_dir) {
-		old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_folio);
+		old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_block);
 		if (!old_dir_entry) {
-			if (IS_ERR(old_dir_folio))
-				err = PTR_ERR(old_dir_folio);
+			if (IS_ERR(old_dir_block))
+				err = PTR_ERR(old_dir_block);
 			goto out_old;
 		}
 	}
@@ -1060,10 +1062,10 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 
 		err = -ENOENT;
 		new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name,
-						&new_folio);
+						&new_block);
 		if (!new_entry) {
-			if (IS_ERR(new_folio))
-				err = PTR_ERR(new_folio);
+			if (IS_ERR(new_block))
+				err = PTR_ERR(new_block);
 			goto out_dir;
 		}
 
@@ -1075,8 +1077,8 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 		if (err)
 			goto put_out_dir;
 
-		f2fs_set_link(new_dir, new_entry, new_folio, old_inode);
-		new_folio = NULL;
+		f2fs_set_link(new_dir, new_entry, new_block, old_inode);
+		new_block = NULL;
 
 		inode_set_ctime_current(new_inode);
 		f2fs_down_write(&F2FS_I(new_inode)->i_sem);
@@ -1115,8 +1117,8 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 	inode_set_ctime_current(old_inode);
 	f2fs_mark_inode_dirty_sync(old_inode, true);
 
-	f2fs_delete_entry(old_entry, old_folio, old_dir, NULL);
-	old_folio = NULL;
+	f2fs_delete_entry(old_entry, old_block, old_dir, NULL);
+	old_block = NULL;
 
 	if (whiteout) {
 		set_inode_flag(whiteout, FI_INC_LINK);
@@ -1134,7 +1136,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 	}
 
 	if (old_dir_entry)
-		f2fs_set_link(old_inode, old_dir_entry, old_dir_folio, new_dir);
+		f2fs_set_link(old_inode, old_dir_entry, old_dir_block, new_dir);
 	if (old_is_dir)
 		f2fs_i_links_write(old_dir, false);
 
@@ -1158,12 +1160,12 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 
 put_out_dir:
 	f2fs_unlock_op(sbi, &lc);
-	f2fs_folio_put(new_folio, false);
+	f2fs_put_dentry_block(new_block, false);
 out_dir:
 	if (old_dir_entry)
-		f2fs_folio_put(old_dir_folio, false);
+		f2fs_put_dentry_block(old_dir_block, false);
 out_old:
-	f2fs_folio_put(old_folio, false);
+	f2fs_put_dentry_block(old_block, false);
 out:
 	iput(whiteout);
 	return err;
@@ -1175,8 +1177,10 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 	struct f2fs_sb_info *sbi = F2FS_I_SB(old_dir);
 	struct inode *old_inode = d_inode(old_dentry);
 	struct inode *new_inode = d_inode(new_dentry);
-	struct folio *old_dir_folio, *new_dir_folio;
-	struct folio *old_folio, *new_folio;
+	void *old_dir_block = NULL;
+	void *new_dir_block = NULL;
+	void *old_block = NULL;
+	void *new_block = NULL;
 	struct f2fs_dir_entry *old_dir_entry = NULL, *new_dir_entry = NULL;
 	struct f2fs_dir_entry *old_entry, *new_entry;
 	struct f2fs_lock_context lc;
@@ -1208,17 +1212,17 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 		goto out;
 
 	err = -ENOENT;
-	old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_folio);
+	old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_block);
 	if (!old_entry) {
-		if (IS_ERR(old_folio))
-			err = PTR_ERR(old_folio);
+		if (IS_ERR(old_block))
+			err = PTR_ERR(old_block);
 		goto out;
 	}
 
-	new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, &new_folio);
+	new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, &new_block);
 	if (!new_entry) {
-		if (IS_ERR(new_folio))
-			err = PTR_ERR(new_folio);
+		if (IS_ERR(new_block))
+			err = PTR_ERR(new_block);
 		goto out_old;
 	}
 
@@ -1226,20 +1230,20 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 	if (old_dir != new_dir) {
 		if (S_ISDIR(old_inode->i_mode)) {
 			old_dir_entry = f2fs_parent_dir(old_inode,
-							&old_dir_folio);
+							&old_dir_block);
 			if (!old_dir_entry) {
-				if (IS_ERR(old_dir_folio))
-					err = PTR_ERR(old_dir_folio);
+				if (IS_ERR(old_dir_block))
+					err = PTR_ERR(old_dir_block);
 				goto out_new;
 			}
 		}
 
 		if (S_ISDIR(new_inode->i_mode)) {
 			new_dir_entry = f2fs_parent_dir(new_inode,
-							&new_dir_folio);
+							&new_dir_block);
 			if (!new_dir_entry) {
-				if (IS_ERR(new_dir_folio))
-					err = PTR_ERR(new_dir_folio);
+				if (IS_ERR(new_dir_block))
+					err = PTR_ERR(new_dir_block);
 				goto out_old_dir;
 			}
 		}
@@ -1266,14 +1270,14 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 
 	/* update ".." directory entry info of old dentry */
 	if (old_dir_entry)
-		f2fs_set_link(old_inode, old_dir_entry, old_dir_folio, new_dir);
+		f2fs_set_link(old_inode, old_dir_entry, old_dir_block, new_dir);
 
 	/* update ".." directory entry info of new dentry */
 	if (new_dir_entry)
-		f2fs_set_link(new_inode, new_dir_entry, new_dir_folio, old_dir);
+		f2fs_set_link(new_inode, new_dir_entry, new_dir_block, old_dir);
 
 	/* update directory entry info of old dir inode */
-	f2fs_set_link(old_dir, old_entry, old_folio, new_inode);
+	f2fs_set_link(old_dir, old_entry, old_block, new_inode);
 
 	f2fs_down_write(&F2FS_I(old_inode)->i_sem);
 	if (!old_dir_entry)
@@ -1292,7 +1296,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 	f2fs_mark_inode_dirty_sync(old_dir, true);
 
 	/* update directory entry info of new dir inode */
-	f2fs_set_link(new_dir, new_entry, new_folio, old_inode);
+	f2fs_set_link(new_dir, new_entry, new_block, old_inode);
 
 	f2fs_down_write(&F2FS_I(new_inode)->i_sem);
 	if (!new_dir_entry)
@@ -1327,21 +1331,21 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
 	return 0;
 out_new_dir:
 	if (new_dir_entry) {
-		f2fs_folio_put(new_dir_folio, false);
+		f2fs_put_dentry_block(new_dir_block, false);
 	}
 out_old_dir:
 	if (old_dir_entry) {
-		f2fs_folio_put(old_dir_folio, false);
+		f2fs_put_dentry_block(old_dir_block, false);
 	}
 out_new:
-	f2fs_folio_put(new_folio, false);
+	f2fs_put_dentry_block(new_block, false);
 out_old:
-	f2fs_folio_put(old_folio, false);
+	f2fs_put_dentry_block(old_block, false);
 out:
 	return err;
 }
 
-static int f2fs_rename2(struct mnt_idmap *idmap,
+static int f2fs_rename2(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
@@ -1394,7 +1398,7 @@ static const char *f2fs_encrypted_get_link(struct dentry *dentry,
 	return target;
 }
 
-static int f2fs_encrypted_symlink_getattr(struct mnt_idmap *idmap,
+static int f2fs_encrypted_symlink_getattr(const struct mnt_idmap *idmap,
 					  const struct path *path,
 					  struct kstat *stat, u32 request_mask,
 					  unsigned int query_flags)
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index 86c2e67e43b6ae..be14dfa31e530d 100644
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -7,12 +7,12 @@
  */
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
-#include <linux/mpage.h>
 #include <linux/sched/mm.h>
 #include <linux/blkdev.h>
 #include <linux/folio_batch.h>
 #include <linux/swap.h>
 #include <linux/fserror.h>
+#include <linux/freezer.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -85,7 +85,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
 	} else if (type == DIRTY_DENTS) {
 		if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi))
 			return false;
-		mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);
+		mem_size = get_nr_caches(sbi, F2FS_DIRTY_DENTS);
 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
 	} else if (type == INO_ENTRIES) {
 		int i;
@@ -109,7 +109,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
 		mem_size = (atomic_read(&dcc->discard_cmd_cnt) *
 				sizeof(struct discard_cmd)) >> PAGE_SHIFT;
 		res = mem_size < (avail_ram * nm_i->ram_thresh / 100);
-	} else if (type == COMPRESS_PAGE) {
+	} else if (type == COMPRESS_BLOCK) {
 #ifdef CONFIG_F2FS_FS_COMPRESSION
 		unsigned long free_ram = val.freeram;
 
@@ -118,7 +118,7 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
 		 * exceed threshold, deny caching compress page.
 		 */
 		res = (free_ram > avail_ram * sbi->compress_watermark / 100) &&
-			(COMPRESS_MAPPING(sbi)->nrpages <
+			(COMPRESS_CACHE(sbi)->num_entries <
 			 free_ram * sbi->compress_percent / 100);
 #else
 		res = false;
@@ -130,48 +130,36 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
 	return res;
 }
 
-static void clear_node_folio_dirty(struct folio *folio)
+static struct f2fs_cached_block *get_current_nat_cache(struct f2fs_sb_info *sbi,
+		nid_t nid)
 {
-	if (folio_test_dirty(folio)) {
-		f2fs_clear_page_cache_dirty_tag(folio);
-		folio_clear_dirty_for_io(folio);
-		dec_page_count(F2FS_F_SB(folio), F2FS_DIRTY_NODES);
-	}
-	folio_clear_uptodate(folio);
-}
-
-static struct folio *get_current_nat_folio(struct f2fs_sb_info *sbi, nid_t nid)
-{
-	return f2fs_get_meta_folio_retry(sbi, current_nat_addr(sbi, nid));
+	return f2fs_get_meta_cache_retry(sbi, current_nat_addr(sbi, nid));
 }
 
-static struct folio *get_next_nat_folio(struct f2fs_sb_info *sbi, nid_t nid)
+static struct f2fs_cached_block *get_next_nat_cache(struct f2fs_sb_info *sbi,
+		nid_t nid)
 {
-	struct folio *src_folio;
-	struct folio *dst_folio;
+	struct f2fs_cached_block *src_entry;
+	struct f2fs_cached_block *dst_entry;
 	pgoff_t dst_off;
-	void *src_addr;
-	void *dst_addr;
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 
 	dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid));
 
-	/* get current nat block page with lock */
-	src_folio = get_current_nat_folio(sbi, nid);
-	if (IS_ERR(src_folio))
-		return src_folio;
-	dst_folio = f2fs_grab_meta_folio(sbi, dst_off);
-	f2fs_bug_on(sbi, folio_test_dirty(src_folio));
+	/* get current nat cached block with lock */
+	src_entry = get_current_nat_cache(sbi, nid);
+	if (IS_ERR(src_entry))
+		return src_entry;
+	dst_entry = f2fs_grab_meta_cache(sbi, dst_off);
+	f2fs_bug_on(sbi, f2fs_cache_test_dirty(src_entry));
 
-	src_addr = folio_address(src_folio);
-	dst_addr = folio_address(dst_folio);
-	memcpy(dst_addr, src_addr, PAGE_SIZE);
-	folio_mark_dirty(dst_folio);
-	f2fs_folio_put(src_folio, true);
+	memcpy(cache_address(dst_entry), cache_address(src_entry), F2FS_BLKSIZE(sbi));
+	f2fs_mark_cache_dirty(dst_entry);
+	f2fs_put_cache(src_entry, true);
 
-	set_to_next_nat(nm_i, nid);
+	set_to_next_nat(sbi, nm_i, nid);
 
-	return dst_folio;
+	return dst_entry;
 }
 
 static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi,
@@ -254,10 +242,11 @@ static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)
 	__free_nat_entry(e);
 }
 
-static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i,
+static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_sb_info *sbi,
+							struct f2fs_nm_info *nm_i,
 							struct nat_entry *ne)
 {
-	nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid);
+	nid_t set = NAT_BLOCK_OFFSET(sbi, ne->ni.nid);
 	struct nat_entry_set *head;
 
 	head = radix_tree_lookup(&nm_i->nat_set_root, set);
@@ -274,14 +263,15 @@ static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i,
 	return head;
 }
 
-static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,
+static void __set_nat_cache_dirty(struct f2fs_sb_info *sbi,
+		struct f2fs_nm_info *nm_i,
 		struct nat_entry *ne, bool init_dirty)
 {
 	struct nat_entry_set *head;
 	bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR;
 
 	if (!new_ne)
-		head = __grab_nat_entry_set(nm_i, ne);
+		head = __grab_nat_entry_set(sbi, nm_i, ne);
 
 	/*
 	 * update entry_cnt in below condition:
@@ -330,9 +320,11 @@ static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,
 							start, nr);
 }
 
-bool f2fs_in_warm_node_list(struct folio *folio)
+bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi,
+				struct f2fs_cached_block *entry)
 {
-	return is_node_folio(folio) && IS_DNODE(folio) && is_cold_node(folio);
+	return f2fs_is_node_cache(entry) &&
+		IS_DNODE(sbi, entry) && is_cold_node(sbi, entry);
 }
 
 void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi)
@@ -344,7 +336,7 @@ void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi)
 }
 
 static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi,
-		struct folio *folio)
+		struct f2fs_cached_block *entry)
 {
 	struct fsync_node_entry *fn;
 	unsigned long flags;
@@ -353,8 +345,8 @@ static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi,
 	fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab,
 					GFP_NOFS, true, NULL);
 
-	folio_get(folio);
-	fn->folio = folio;
+	f2fs_cache_get(entry);
+	fn->entry = entry;
 	INIT_LIST_HEAD(&fn->list);
 
 	spin_lock_irqsave(&sbi->fsync_node_lock, flags);
@@ -367,19 +359,19 @@ static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi,
 	return seq_id;
 }
 
-void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio)
+void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct f2fs_cached_block *entry)
 {
 	struct fsync_node_entry *fn;
 	unsigned long flags;
 
 	spin_lock_irqsave(&sbi->fsync_node_lock, flags);
 	list_for_each_entry(fn, &sbi->fsync_node_list, list) {
-		if (fn->folio == folio) {
+		if (fn->entry == entry) {
 			list_del(&fn->list);
 			sbi->fsync_node_num--;
 			spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
+			f2fs_put_cache(fn->entry, false);
 			kmem_cache_free(fsync_node_entry_slab, fn);
-			folio_put(folio);
 			return;
 		}
 	}
@@ -527,7 +519,7 @@ static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
 	nat_set_blkaddr(e, new_blkaddr);
 	if (!__is_valid_data_blkaddr(new_blkaddr))
 		set_nat_flag(e, IS_CHECKPOINTED, false);
-	__set_nat_cache_dirty(nm_i, e, init_dirty);
+	__set_nat_cache_dirty(sbi, nm_i, e, init_dirty);
 
 	/* update fsync_mark if its inode nat entry is still alive */
 	if (ni->nid != ni->ino)
@@ -578,9 +570,9 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
 	struct f2fs_journal *journal = curseg->journal;
-	nid_t start_nid = START_NID(nid);
+	nid_t start_nid = f2fs_start_nid(sbi, nid);
 	struct f2fs_nat_block *nat_blk;
-	struct folio *folio = NULL;
+	struct f2fs_cached_block *entry = NULL;
 	struct f2fs_nat_entry ne;
 	struct nat_entry *e;
 	pgoff_t index;
@@ -631,18 +623,18 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
 		goto sanity_check;
 	}
 
-	/* Fill node_info from nat page */
+	/* Fill node_info from nat block */
 	index = current_nat_addr(sbi, nid);
 	f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc);
 
-	folio = f2fs_get_meta_folio(sbi, index);
-	if (IS_ERR(folio))
-		return PTR_ERR(folio);
+	entry = f2fs_get_meta_cache(sbi, index);
+	if (IS_ERR(entry))
+		return PTR_ERR(entry);
 
-	nat_blk = folio_address(folio);
+	nat_blk = cache_address(entry);
 	ne = nat_blk->entries[nid - start_nid];
 	node_info_from_raw_nat(ni, &ne);
-	f2fs_folio_put(folio, true);
+	f2fs_put_cache(entry, true);
 sanity_check:
 	if (__is_valid_data_blkaddr(ni->blk_addr) &&
 		!f2fs_is_valid_blkaddr(sbi, ni->blk_addr,
@@ -676,11 +668,11 @@ int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
 }
 
 /*
- * readahead MAX_RA_NODE number of node pages.
+ * readahead MAX_RA_NODE number of node caches.
  */
-static void f2fs_ra_node_pages(struct folio *parent, int start, int n)
+static void f2fs_ra_node_caches(struct f2fs_cached_block *parent, int start, int n)
 {
-	struct f2fs_sb_info *sbi = F2FS_F_SB(parent);
+	struct f2fs_sb_info *sbi = parent->cache->sbi;
 	struct blk_plug plug;
 	int i, end;
 	nid_t nid;
@@ -689,10 +681,10 @@ static void f2fs_ra_node_pages(struct folio *parent, int start, int n)
 
 	/* Then, try readahead for siblings of the desired node */
 	end = start + n;
-	end = min(end, (int)NIDS_PER_BLOCK);
+	end = min_t(int, end, NIDS_PER_BLOCK(sbi));
 	for (i = start; i < end; i++) {
-		nid = get_nid(parent, i, false);
-		f2fs_ra_node_page(sbi, nid);
+		nid = get_nid(sbi, parent, i, false);
+		f2fs_ra_node_cache(sbi, nid);
 	}
 
 	blk_finish_plug(&plug);
@@ -700,9 +692,11 @@ static void f2fs_ra_node_pages(struct folio *parent, int start, int n)
 
 pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs)
 {
+	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 	const long direct_index = ADDRS_PER_INODE(dn->inode);
 	const long direct_blks = ADDRS_PER_BLOCK(dn->inode);
-	const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) * NIDS_PER_BLOCK;
+	const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) *
+		NIDS_PER_BLOCK(sbi);
 	unsigned int skipped_unit = ADDRS_PER_BLOCK(dn->inode);
 	int cur_level = dn->cur_level;
 	int max_level = dn->max_level;
@@ -712,7 +706,7 @@ pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs)
 		return pgofs + 1;
 
 	while (max_level-- > cur_level)
-		skipped_unit *= NIDS_PER_BLOCK;
+		skipped_unit *= NIDS_PER_BLOCK(sbi);
 
 	switch (dn->max_level) {
 	case 3:
@@ -738,11 +732,13 @@ pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs)
 static int get_node_path(struct inode *inode, long block,
 				int offset[4], unsigned int noffset[4])
 {
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	const long direct_index = ADDRS_PER_INODE(inode);
 	const long direct_blks = ADDRS_PER_BLOCK(inode);
-	const long dptrs_per_blk = NIDS_PER_BLOCK;
-	const long indirect_blks = ADDRS_PER_BLOCK(inode) * NIDS_PER_BLOCK;
-	const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;
+	const long dptrs_per_blk = NIDS_PER_BLOCK(sbi);
+	const long indirect_blks = ADDRS_PER_BLOCK(inode) *
+		NIDS_PER_BLOCK(sbi);
+	const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK(sbi);
 	int n = 0;
 	int level = 0;
 
@@ -754,7 +750,7 @@ static int get_node_path(struct inode *inode, long block,
 	}
 	block -= direct_index;
 	if (block < direct_blks) {
-		offset[n++] = NODE_DIR1_BLOCK;
+		offset[n++] = NODE_DIR1_BLOCK(sbi);
 		noffset[n] = 1;
 		offset[n] = block;
 		level = 1;
@@ -762,7 +758,7 @@ static int get_node_path(struct inode *inode, long block,
 	}
 	block -= direct_blks;
 	if (block < direct_blks) {
-		offset[n++] = NODE_DIR2_BLOCK;
+		offset[n++] = NODE_DIR2_BLOCK(sbi);
 		noffset[n] = 2;
 		offset[n] = block;
 		level = 1;
@@ -770,7 +766,7 @@ static int get_node_path(struct inode *inode, long block,
 	}
 	block -= direct_blks;
 	if (block < indirect_blks) {
-		offset[n++] = NODE_IND1_BLOCK;
+		offset[n++] = NODE_IND1_BLOCK(sbi);
 		noffset[n] = 3;
 		offset[n++] = block / direct_blks;
 		noffset[n] = 4 + offset[n - 1];
@@ -780,7 +776,7 @@ static int get_node_path(struct inode *inode, long block,
 	}
 	block -= indirect_blks;
 	if (block < indirect_blks) {
-		offset[n++] = NODE_IND2_BLOCK;
+		offset[n++] = NODE_IND2_BLOCK(sbi);
 		noffset[n] = 4 + dptrs_per_blk;
 		offset[n++] = block / direct_blks;
 		noffset[n] = 5 + dptrs_per_blk + offset[n - 1];
@@ -790,7 +786,7 @@ static int get_node_path(struct inode *inode, long block,
 	}
 	block -= indirect_blks;
 	if (block < dindirect_blks) {
-		offset[n++] = NODE_DIND_BLOCK;
+		offset[n++] = NODE_DIND_BLOCK(sbi);
 		noffset[n] = 5 + (dptrs_per_blk * 2);
 		offset[n++] = block / indirect_blks;
 		noffset[n] = 6 + (dptrs_per_blk * 2) +
@@ -809,7 +805,8 @@ static int get_node_path(struct inode *inode, long block,
 	return level;
 }
 
-static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start);
+static struct f2fs_cached_block *f2fs_get_node_cache_ra(
+		struct f2fs_cached_block *parent, int start);
 
 /*
  * Caller should call f2fs_put_dnode(dn).
@@ -819,8 +816,8 @@ static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start);
 int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
-	struct folio *nfolio[4];
-	struct folio *parent = NULL;
+	struct f2fs_cached_block *nentry[4];
+	struct f2fs_cached_block *parent = NULL;
 	int offset[4];
 	unsigned int noffset[4];
 	nid_t nids[4];
@@ -833,26 +830,26 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 
 	nids[0] = dn->inode->i_ino;
 
-	if (!dn->inode_folio) {
-		nfolio[0] = f2fs_get_inode_folio(sbi, nids[0]);
-		if (IS_ERR(nfolio[0]))
-			return PTR_ERR(nfolio[0]);
+	if (!dn->inode_entry) {
+		nentry[0] = f2fs_get_inode_cache(sbi, nids[0]);
+		if (IS_ERR(nentry[0]))
+			return PTR_ERR(nentry[0]);
 	} else {
-		nfolio[0] = dn->inode_folio;
+		nentry[0] = dn->inode_entry;
 	}
 
 	/* if inline_data is set, should not report any block indices */
 	if (f2fs_has_inline_data(dn->inode) && index) {
 		err = -ENOENT;
-		f2fs_folio_put(nfolio[0], true);
+		f2fs_put_cache(nentry[0], true);
 		goto release_out;
 	}
 
-	parent = nfolio[0];
+	parent = nentry[0];
 	if (level != 0)
-		nids[1] = get_nid(parent, offset[0], true);
-	dn->inode_folio = nfolio[0];
-	dn->inode_folio_locked = true;
+		nids[1] = get_nid(sbi, parent, offset[0], true);
+	dn->inode_entry = nentry[0];
+	dn->inode_entry_locked = true;
 
 	/* get indirect or direct nodes */
 	for (i = 1; i <= level; i++) {
@@ -865,59 +862,59 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 				"ino:%llu, nid:%u, level:%d, offset:%d",
 				dn->inode->i_ino, nids[i], level, offset[level]);
 			set_sbi_flag(sbi, SBI_NEED_FSCK);
-			goto release_pages;
+			goto release_caches;
 		}
 
 		if (!nids[i] && mode == ALLOC_NODE) {
 			/* alloc new node */
 			if (!f2fs_alloc_nid(sbi, &(nids[i]))) {
 				err = -ENOSPC;
-				goto release_pages;
+				goto release_caches;
 			}
 
 			dn->nid = nids[i];
-			nfolio[i] = f2fs_new_node_folio(dn, noffset[i]);
-			if (IS_ERR(nfolio[i])) {
+			nentry[i] = f2fs_new_node_cache(dn, noffset[i]);
+			if (IS_ERR(nentry[i])) {
 				f2fs_alloc_nid_failed(sbi, nids[i]);
-				err = PTR_ERR(nfolio[i]);
-				goto release_pages;
+				err = PTR_ERR(nentry[i]);
+				goto release_caches;
 			}
 
-			set_nid(parent, offset[i - 1], nids[i], i == 1);
+			set_nid(sbi, parent, offset[i - 1], nids[i], i == 1);
 			f2fs_alloc_nid_done(sbi, nids[i]);
 			done = true;
 		} else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
-			nfolio[i] = f2fs_get_node_folio_ra(parent, offset[i - 1]);
-			if (IS_ERR(nfolio[i])) {
-				err = PTR_ERR(nfolio[i]);
-				goto release_pages;
+			nentry[i] = f2fs_get_node_cache_ra(parent, offset[i - 1]);
+			if (IS_ERR(nentry[i])) {
+				err = PTR_ERR(nentry[i]);
+				goto release_caches;
 			}
 			done = true;
 		}
 		if (i == 1) {
-			dn->inode_folio_locked = false;
-			folio_unlock(parent);
+			dn->inode_entry_locked = false;
+			f2fs_unlock_cache(parent);
 		} else {
-			f2fs_folio_put(parent, true);
+			f2fs_put_cache(parent, true);
 		}
 
 		if (!done) {
-			nfolio[i] = f2fs_get_node_folio(sbi, nids[i],
+			nentry[i] = f2fs_get_node_cache(sbi, nids[i],
 						NODE_TYPE_NON_INODE);
-			if (IS_ERR(nfolio[i])) {
-				err = PTR_ERR(nfolio[i]);
-				f2fs_folio_put(nfolio[0], false);
+			if (IS_ERR(nentry[i])) {
+				err = PTR_ERR(nentry[i]);
+				f2fs_put_cache(nentry[0], false);
 				goto release_out;
 			}
 		}
 		if (i < level) {
-			parent = nfolio[i];
-			nids[i + 1] = get_nid(parent, offset[i], false);
+			parent = nentry[i];
+			nids[i + 1] = get_nid(sbi, parent, offset[i], false);
 		}
 	}
 	dn->nid = nids[level];
 	dn->ofs_in_node = offset[level];
-	dn->node_folio = nfolio[level];
+	dn->node_entry = nentry[level];
 	dn->data_blkaddr = f2fs_data_blkaddr(dn);
 
 	if (is_inode_flag_set(dn->inode, FI_COMPRESSED_FILE) &&
@@ -938,9 +935,9 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 		if (!c_len)
 			goto out;
 
-		blkaddr = data_blkaddr(dn->inode, dn->node_folio, ofs_in_node);
+		blkaddr = data_blkaddr(dn->inode, dn->node_entry, ofs_in_node);
 		if (blkaddr == COMPRESS_ADDR)
-			blkaddr = data_blkaddr(dn->inode, dn->node_folio,
+			blkaddr = data_blkaddr(dn->inode, dn->node_entry,
 						ofs_in_node + 1);
 
 		f2fs_update_read_extent_tree_range_compressed(dn->inode,
@@ -949,13 +946,13 @@ int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 out:
 	return 0;
 
-release_pages:
-	f2fs_folio_put(parent, true);
+release_caches:
+	f2fs_put_cache(parent, true);
 	if (i > 1)
-		f2fs_folio_put(nfolio[0], false);
+		f2fs_put_cache(nentry[0], false);
 release_out:
-	dn->inode_folio = NULL;
-	dn->node_folio = NULL;
+	dn->inode_entry = NULL;
+	dn->node_entry = NULL;
 	if (err == -ENOENT) {
 		dn->cur_level = i;
 		dn->max_level = level;
@@ -996,16 +993,15 @@ static int truncate_node(struct dnode_of_data *dn)
 		f2fs_inode_synced(dn->inode);
 	}
 
-	clear_node_folio_dirty(dn->node_folio);
+	f2fs_drop_cache_dirty(dn->node_entry);
 	set_sbi_flag(sbi, SBI_IS_DIRTY);
 
-	index = dn->node_folio->index;
-	f2fs_folio_put(dn->node_folio, true);
+	index = dn->node_entry->index;
+	f2fs_put_cache(dn->node_entry, true);
 
-	invalidate_mapping_pages(NODE_MAPPING(sbi),
-			index, index);
+	f2fs_invalidate_node_cache(sbi, index);
 
-	dn->node_folio = NULL;
+	dn->node_entry = NULL;
 	trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
 
 	return 0;
@@ -1014,35 +1010,35 @@ static int truncate_node(struct dnode_of_data *dn)
 static int truncate_dnode(struct dnode_of_data *dn)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	int err;
 
 	if (dn->nid == 0)
 		return 1;
 
 	/* get direct node */
-	folio = f2fs_get_node_folio(sbi, dn->nid, NODE_TYPE_NON_INODE);
-	if (PTR_ERR(folio) == -ENOENT)
+	entry = f2fs_get_node_cache(sbi, dn->nid, NODE_TYPE_NON_INODE);
+	if (PTR_ERR(entry) == -ENOENT)
 		return 1;
-	else if (IS_ERR(folio))
-		return PTR_ERR(folio);
+	else if (IS_ERR(entry))
+		return PTR_ERR(entry);
 
-	if (IS_INODE(folio) || ino_of_node(folio) != dn->inode->i_ino) {
+	if (IS_INODE(sbi, entry) || ino_of_node(sbi, entry) != dn->inode->i_ino) {
 		f2fs_err(sbi, "incorrect node reference, ino: %llu, nid: %u, ino_of_node: %u",
-				dn->inode->i_ino, dn->nid, ino_of_node(folio));
+				dn->inode->i_ino, dn->nid, ino_of_node(sbi, entry));
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
 		f2fs_handle_error(sbi, ERROR_INVALID_NODE_REFERENCE);
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 		return -EFSCORRUPTED;
 	}
 
 	/* Make dnode_of_data for parameter */
-	dn->node_folio = folio;
+	dn->node_entry = entry;
 	dn->ofs_in_node = 0;
 	f2fs_truncate_data_blocks_range(dn, ADDRS_PER_BLOCK(dn->inode));
 	err = truncate_node(dn);
 	if (err) {
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 		return err;
 	}
 
@@ -1052,8 +1048,9 @@ static int truncate_dnode(struct dnode_of_data *dn)
 static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
 						int ofs, int depth)
 {
+	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 	struct dnode_of_data rdn = *dn;
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	struct f2fs_node *rn;
 	nid_t child_nid;
 	unsigned int child_nofs;
@@ -1061,22 +1058,22 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
 	int i, ret;
 
 	if (dn->nid == 0)
-		return NIDS_PER_BLOCK + 1;
+		return NIDS_PER_BLOCK(sbi) + 1;
 
 	trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);
 
-	folio = f2fs_get_node_folio(F2FS_I_SB(dn->inode), dn->nid,
+	entry = f2fs_get_node_cache(F2FS_I_SB(dn->inode), dn->nid,
 						NODE_TYPE_NON_INODE);
-	if (IS_ERR(folio)) {
-		trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(folio));
-		return PTR_ERR(folio);
+	if (IS_ERR(entry)) {
+		trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(entry));
+		return PTR_ERR(entry);
 	}
 
-	f2fs_ra_node_pages(folio, ofs, NIDS_PER_BLOCK);
+	f2fs_ra_node_caches(entry, ofs, NIDS_PER_BLOCK(sbi));
 
-	rn = F2FS_NODE(folio);
+	rn = CACHED_NODE(entry);
 	if (depth < 3) {
-		for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) {
+		for (i = ofs; i < NIDS_PER_BLOCK(sbi); i++, freed++) {
 			child_nid = le32_to_cpu(rn->in.nid[i]);
 			if (child_nid == 0)
 				continue;
@@ -1084,21 +1081,21 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
 			ret = truncate_dnode(&rdn);
 			if (ret < 0)
 				goto out_err;
-			if (set_nid(folio, i, 0, false))
+			if (set_nid(sbi, entry, i, 0, false))
 				dn->node_changed = true;
 		}
 	} else {
-		child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1;
-		for (i = ofs; i < NIDS_PER_BLOCK; i++) {
+		child_nofs = nofs + ofs * (NIDS_PER_BLOCK(sbi) + 1) + 1;
+		for (i = ofs; i < NIDS_PER_BLOCK(sbi); i++) {
 			child_nid = le32_to_cpu(rn->in.nid[i]);
 			if (child_nid == 0) {
-				child_nofs += NIDS_PER_BLOCK + 1;
+				child_nofs += NIDS_PER_BLOCK(sbi) + 1;
 				continue;
 			}
 			rdn.nid = child_nid;
 			ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1);
-			if (ret == (NIDS_PER_BLOCK + 1)) {
-				if (set_nid(folio, i, 0, false))
+			if (ret == (NIDS_PER_BLOCK(sbi) + 1)) {
+				if (set_nid(sbi, entry, i, 0, false))
 					dn->node_changed = true;
 				child_nofs += ret;
 			} else if (ret < 0 && ret != -ENOENT) {
@@ -1110,19 +1107,19 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
 
 	if (!ofs) {
 		/* remove current indirect node */
-		dn->node_folio = folio;
+		dn->node_entry = entry;
 		ret = truncate_node(dn);
 		if (ret)
 			goto out_err;
 		freed++;
 	} else {
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 	}
 	trace_f2fs_truncate_nodes_exit(dn->inode, freed);
 	return freed;
 
 out_err:
-	f2fs_folio_put(folio, true);
+	f2fs_put_cache(entry, true);
 	trace_f2fs_truncate_nodes_exit(dn->inode, ret);
 	return ret;
 }
@@ -1130,60 +1127,62 @@ static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
 static int truncate_partial_nodes(struct dnode_of_data *dn,
 			int *offset, int depth)
 {
-	struct folio *folios[2];
+	struct f2fs_cached_block *entries[2];
 	nid_t nid[3];
 	nid_t child_nid;
 	int err = 0;
 	int i;
 	int idx = depth - 2;
 
-	nid[0] = get_nid(dn->inode_folio, offset[0], true);
+	nid[0] = get_nid(F2FS_I_SB(dn->inode), dn->inode_entry, offset[0], true);
 	if (!nid[0])
 		return 0;
 
 	/* get indirect nodes in the path */
 	for (i = 0; i < idx + 1; i++) {
 		/* reference count'll be increased */
-		folios[i] = f2fs_get_node_folio(F2FS_I_SB(dn->inode), nid[i],
+		entries[i] = f2fs_get_node_cache(F2FS_I_SB(dn->inode), nid[i],
 							NODE_TYPE_NON_INODE);
-		if (IS_ERR(folios[i])) {
-			err = PTR_ERR(folios[i]);
+		if (IS_ERR(entries[i])) {
+			err = PTR_ERR(entries[i]);
 			idx = i - 1;
 			goto fail;
 		}
-		nid[i + 1] = get_nid(folios[i], offset[i + 1], false);
+		nid[i + 1] = get_nid(F2FS_I_SB(dn->inode), entries[i], offset[i + 1], false);
 	}
 
-	f2fs_ra_node_pages(folios[idx], offset[idx + 1], NIDS_PER_BLOCK);
+	f2fs_ra_node_caches(entries[idx], offset[idx + 1],
+			   NIDS_PER_BLOCK(F2FS_I_SB(dn->inode)));
 
 	/* free direct nodes linked to a partial indirect node */
-	for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) {
-		child_nid = get_nid(folios[idx], i, false);
+	for (i = offset[idx + 1];
+			i < NIDS_PER_BLOCK(F2FS_I_SB(dn->inode)); i++) {
+		child_nid = get_nid(F2FS_I_SB(dn->inode), entries[idx], i, false);
 		if (!child_nid)
 			continue;
 		dn->nid = child_nid;
 		err = truncate_dnode(dn);
 		if (err < 0)
 			goto fail;
-		if (set_nid(folios[idx], i, 0, false))
+		if (set_nid(F2FS_I_SB(dn->inode), entries[idx], i, 0, false))
 			dn->node_changed = true;
 	}
 
 	if (offset[idx + 1] == 0) {
-		dn->node_folio = folios[idx];
+		dn->node_entry = entries[idx];
 		dn->nid = nid[idx];
 		err = truncate_node(dn);
 		if (err)
 			goto fail;
 	} else {
-		f2fs_folio_put(folios[idx], true);
+		f2fs_put_cache(entries[idx], true);
 	}
 	offset[idx]++;
 	offset[idx + 1] = 0;
 	idx--;
 fail:
 	for (i = idx; i >= 0; i--)
-		f2fs_folio_put(folios[i], true);
+		f2fs_put_cache(entries[i], true);
 
 	trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);
 
@@ -1191,7 +1190,9 @@ static int truncate_partial_nodes(struct dnode_of_data *dn,
 }
 
 /*
- * All the block addresses of data and nodes should be nullified.
+ * All the node blocks actually belong to the inode will be released.
+ * If the level is 0, we will simply truncate the dnode,
+ * or else we should do dynamic truncate for the node pointers with the depth.
  */
 int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 {
@@ -1200,7 +1201,7 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 	int level, offset[4], noffset[4];
 	unsigned int nofs = 0;
 	struct dnode_of_data dn;
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 
 	trace_f2fs_truncate_inode_blocks_enter(inode, from);
 
@@ -1217,14 +1218,14 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 		return level;
 	}
 
-	folio = f2fs_get_inode_folio(sbi, inode->i_ino);
-	if (IS_ERR(folio)) {
-		trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(folio));
-		return PTR_ERR(folio);
+	entry = f2fs_get_inode_cache(sbi, inode->i_ino);
+	if (IS_ERR(entry)) {
+		trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(entry));
+		return PTR_ERR(entry);
 	}
 
-	set_new_dnode(&dn, inode, folio, NULL, 0);
-	folio_unlock(folio);
+	set_new_dnode(&dn, inode, entry, NULL, 0);
+	f2fs_unlock_cache(entry);
 
 	switch (level) {
 	case 0:
@@ -1238,10 +1239,10 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 		err = truncate_partial_nodes(&dn, offset, level);
 		if (err < 0 && err != -ENOENT)
 			goto fail;
-		nofs += 1 + NIDS_PER_BLOCK;
+		nofs += 1 + NIDS_PER_BLOCK(sbi);
 		break;
 	case 3:
-		nofs = 5 + 2 * NIDS_PER_BLOCK;
+		nofs = 5 + 2 * NIDS_PER_BLOCK(sbi);
 		if (!offset[level - 1])
 			goto skip_partial;
 		err = truncate_partial_nodes(&dn, offset, level);
@@ -1254,28 +1255,21 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 
 skip_partial:
 	while (cont) {
-		dn.nid = get_nid(folio, offset[0], true);
-		switch (offset[0]) {
-		case NODE_DIR1_BLOCK:
-		case NODE_DIR2_BLOCK:
+		dn.nid = get_nid(sbi, entry, offset[0], true);
+		if (offset[0] == NODE_DIR1_BLOCK(sbi) ||
+		    offset[0] == NODE_DIR2_BLOCK(sbi)) {
 			err = truncate_dnode(&dn);
-			break;
-
-		case NODE_IND1_BLOCK:
-		case NODE_IND2_BLOCK:
+		} else if (offset[0] == NODE_IND1_BLOCK(sbi) ||
+				offset[0] == NODE_IND2_BLOCK(sbi)) {
 			err = truncate_nodes(&dn, nofs, offset[1], 2);
-			break;
-
-		case NODE_DIND_BLOCK:
+		} else if (offset[0] == NODE_DIND_BLOCK(sbi)) {
 			err = truncate_nodes(&dn, nofs, offset[1], 3);
 			cont = 0;
-			break;
-
-		default:
+		} else {
 			BUG();
 		}
 		if (err == -ENOENT) {
-			set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK);
+			set_sbi_flag(sbi, SBI_NEED_FSCK);
 			f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
 			fserror_report_file_metadata(dn.inode, -EFSCORRUPTED,
 								GFP_NOFS);
@@ -1288,42 +1282,42 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 		}
 		if (err < 0)
 			goto fail;
-		if (offset[1] == 0 && get_nid(folio, offset[0], true)) {
-			folio_lock(folio);
-			BUG_ON(!is_node_folio(folio));
-			set_nid(folio, offset[0], 0, true);
-			folio_unlock(folio);
+		if (offset[1] == 0 && get_nid(sbi, entry, offset[0], true)) {
+			f2fs_lock_cache(entry);
+			f2fs_bug_on(sbi, !f2fs_is_node_cache(entry));
+			set_nid(sbi, entry, offset[0], 0, true);
+			f2fs_unlock_cache(entry);
 		}
 		offset[1] = 0;
 		offset[0]++;
 		nofs += err;
 	}
 fail:
-	f2fs_folio_put(folio, false);
+	f2fs_put_cache(entry, false);
 	trace_f2fs_truncate_inode_blocks_exit(inode, err);
 	return err > 0 ? 0 : err;
 }
 
-/* caller must lock inode page */
+/* caller must lock inode cached block */
 int f2fs_truncate_xattr_node(struct inode *inode)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	nid_t nid = F2FS_I(inode)->i_xattr_nid;
 	struct dnode_of_data dn;
-	struct folio *nfolio;
+	struct f2fs_cached_block *nentry;
 	int err;
 
 	if (!nid)
 		return 0;
 
-	nfolio = f2fs_get_xnode_folio(sbi, nid);
-	if (IS_ERR(nfolio))
-		return PTR_ERR(nfolio);
+	nentry = f2fs_get_xnode_cache(sbi, nid);
+	if (IS_ERR(nentry))
+		return PTR_ERR(nentry);
 
-	set_new_dnode(&dn, inode, NULL, nfolio, nid);
+	set_new_dnode(&dn, inode, NULL, nentry, nid);
 	err = truncate_node(&dn);
 	if (err) {
-		f2fs_folio_put(nfolio, true);
+		f2fs_put_cache(nentry, true);
 		return err;
 	}
 
@@ -1336,7 +1330,7 @@ int f2fs_truncate_xattr_node(struct inode *inode)
  * Caller should grab and release a rwsem by calling f2fs_lock_op() and
  * f2fs_unlock_op().
  */
-int f2fs_remove_inode_page(struct inode *inode)
+int f2fs_remove_inode_cache(struct inode *inode)
 {
 	struct dnode_of_data dn;
 	int err;
@@ -1366,12 +1360,12 @@ int f2fs_remove_inode_page(struct inode *inode)
 
 	if (unlikely(inode->i_blocks != 0 && inode->i_blocks != 8)) {
 		f2fs_warn(F2FS_I_SB(inode),
-			"f2fs_remove_inode_page: inconsistent i_blocks, ino:%llu, iblocks:%llu",
+			"f2fs_remove_inode_cache: inconsistent i_blocks, ino:%llu, iblocks:%llu",
 			inode->i_ino, (unsigned long long)inode->i_blocks);
 		set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
 	}
 
-	/* will put inode & node pages */
+	/* will put inode & node cached blocks */
 	err = truncate_node(&dn);
 	if (err) {
 		f2fs_put_dnode(&dn);
@@ -1380,30 +1374,30 @@ int f2fs_remove_inode_page(struct inode *inode)
 	return 0;
 }
 
-struct folio *f2fs_new_inode_folio(struct inode *inode)
+struct f2fs_cached_block *f2fs_new_inode_cache(struct inode *inode)
 {
 	struct dnode_of_data dn;
 
-	/* allocate inode page for new inode */
+	/* allocate inode block for new inode */
 	set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
 
-	/* caller should f2fs_folio_put(folio, true); */
-	return f2fs_new_node_folio(&dn, 0);
+	/* caller should f2fs_put_cache(entry, true); */
+	return f2fs_new_node_cache(&dn, 0);
 }
 
-struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs)
+struct f2fs_cached_block *f2fs_new_node_cache(struct dnode_of_data *dn, unsigned int ofs)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 	struct node_info new_ni;
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	int err;
 
 	if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
 		return ERR_PTR(-EPERM);
 
-	folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), dn->nid, false);
-	if (IS_ERR(folio))
-		return folio;
+	entry = f2fs_grab_node_cache(sbi, dn->nid);
+	if (IS_ERR(entry))
+		return entry;
 
 	if (unlikely((err = inc_valid_node_count(sbi, dn->inode, !ofs))))
 		goto fail;
@@ -1419,7 +1413,7 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs)
 		dec_valid_node_count(sbi, dn->inode, !ofs);
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
 		f2fs_warn_ratelimited(sbi,
-			"f2fs_new_node_folio: inconsistent nat entry, "
+			"f2fs_new_node_cache: inconsistent nat entry, "
 			"ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u",
 			new_ni.ino, new_ni.nid, new_ni.blk_addr,
 			new_ni.version, new_ni.flag);
@@ -1434,12 +1428,11 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs)
 	new_ni.version = 0;
 	set_node_addr(sbi, &new_ni, NEW_ADDR, false);
 
-	f2fs_folio_wait_writeback(folio, NODE, true, true);
-	fill_node_footer(folio, dn->nid, dn->inode->i_ino, ofs, true);
-	set_cold_node(folio, S_ISDIR(dn->inode->i_mode));
-	if (!folio_test_uptodate(folio))
-		folio_mark_uptodate(folio);
-	if (folio_mark_dirty(folio))
+	f2fs_cache_wait_writeback(entry);
+	fill_node_footer(sbi, entry, dn->nid, dn->inode->i_ino, ofs, true);
+	set_cold_node(sbi, entry, S_ISDIR(dn->inode->i_mode));
+	f2fs_cache_set_uptodate(entry);
+	if (f2fs_mark_cache_dirty(entry))
 		dn->node_changed = true;
 
 	if (f2fs_has_xattr_block(ofs))
@@ -1447,66 +1440,67 @@ struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs)
 
 	if (ofs == 0)
 		inc_valid_inode_count(sbi);
-	return folio;
+	return entry;
 fail:
-	clear_node_folio_dirty(folio);
-	f2fs_folio_put(folio, true);
+	f2fs_drop_cache_dirty(entry);
+	f2fs_put_cache(entry, true);
 	return ERR_PTR(err);
 }
 
 /*
  * Caller should do after getting the following values.
- * 0: f2fs_folio_put(folio, false)
- * LOCKED_PAGE or error: f2fs_folio_put(folio, true)
+ * 0: f2fs_put_cache(cache, false)
+ * LOCKED_CACHE or error: f2fs_put_cache(entry, true)
  */
-static int read_node_folio(struct folio *folio, blk_opf_t op_flags)
+static int read_node_cache(struct f2fs_cached_block *entry, blk_opf_t op_flags)
 {
-	struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
+	struct f2fs_sb_info *sbi = entry->cache->sbi;
 	struct node_info ni;
 	struct f2fs_io_info fio = {
 		.sbi = sbi,
 		.type = NODE,
 		.op = REQ_OP_READ,
 		.op_flags = op_flags,
-		.folio = folio,
 		.encrypted_page = NULL,
+		.cache_entry = entry,
+		.is_cache = 1,
 	};
 	int err;
 
-	if (folio_test_uptodate(folio)) {
-		if (!f2fs_inode_chksum_verify(sbi, folio)) {
-			folio_clear_uptodate(folio);
+	if (f2fs_cache_test_uptodate(entry)) {
+		if (!f2fs_inode_chksum_verify(sbi, entry)) {
+			f2fs_cache_clear_uptodate(entry);
 			return -EFSBADCRC;
 		}
-		return LOCKED_PAGE;
+		return LOCKED_CACHE;
 	}
 
-	err = f2fs_get_node_info(sbi, folio->index, &ni, false);
+	err = f2fs_get_node_info(sbi, entry->index, &ni, false);
 	if (err)
 		return err;
 
-	/* NEW_ADDR can be seen, after cp_error drops some dirty node pages */
+	/* NEW_ADDR can be seen, after cp_error drops some dirty node caches */
 	if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR)) {
-		folio_clear_uptodate(folio);
+		f2fs_cache_clear_uptodate(entry);
 		return -ENOENT;
 	}
 
 	fio.new_blkaddr = fio.old_blkaddr = ni.blk_addr;
 
-	err = f2fs_submit_page_bio(&fio);
-
+	err = f2fs_submit_cache_read(&fio);
 	if (!err)
-		f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, F2FS_BLKSIZE);
+		f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO,
+				   F2FS_BLKSIZE(sbi));
 
 	return err;
 }
 
 /*
- * Readahead a node page
+ * Readahead a node cache
  */
-void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
+void f2fs_ra_node_cache(struct f2fs_sb_info *sbi, nid_t nid)
 {
-	struct folio *afolio;
+	struct f2fs_cached_block *entry;
 	int err;
 
 	if (!nid)
@@ -1514,29 +1508,29 @@ void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
 	if (f2fs_check_nid_range(sbi, nid))
 		return;
 
-	afolio = xa_load(&NODE_MAPPING(sbi)->i_pages, nid);
-	if (afolio)
+	entry = xa_load(&NODE_CACHE(sbi)->root, nid);
+	if (entry)
 		return;
 
-	afolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false);
-	if (IS_ERR(afolio))
+	entry = f2fs_grab_node_cache(sbi, nid);
+	if (IS_ERR(entry))
 		return;
 
-	err = read_node_folio(afolio, REQ_RAHEAD);
-	f2fs_folio_put(afolio, err ? true : false);
+	err = read_node_cache(entry, REQ_RAHEAD);
+	f2fs_put_cache(entry, err ? true : false);
 }
 
 int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi,
-					struct folio *folio, pgoff_t nid,
+					struct f2fs_cached_block *entry, pgoff_t nid,
 					enum node_type ntype, bool in_irq)
 {
 	bool is_inode, is_xnode;
 
-	if (unlikely(nid != nid_of_node(folio)))
+	if (unlikely(nid != nid_of_node(sbi, entry)))
 		goto out_err;
 
-	is_inode = IS_INODE(folio);
-	is_xnode = f2fs_has_xattr_block(ofs_of_node(folio));
+	is_inode = IS_INODE(sbi, entry);
+	is_xnode = f2fs_has_xattr_block(ofs_of_node(sbi, entry));
 
 	switch (ntype) {
 	case NODE_TYPE_REGULAR:
@@ -1569,20 +1563,20 @@ int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi,
 	set_sbi_flag(sbi, SBI_NEED_FSCK);
 	f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, "
 		"node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]",
-		ntype, nid, nid_of_node(folio), ino_of_node(folio),
-		ofs_of_node(folio), cpver_of_node(folio),
-		next_blkaddr_of_node(folio));
+		ntype, nid, nid_of_node(sbi, entry), ino_of_node(sbi, entry),
+		ofs_of_node(sbi, entry), cpver_of_node(sbi, entry),
+		next_blkaddr_of_node(sbi, entry));
 
 	f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER);
-	fserror_report_file_metadata(folio->mapping->host,
-			-EFSCORRUPTED, in_irq ? GFP_NOWAIT : GFP_NOFS);
+	fserror_report_metadata(sbi->sb, -EFSCORRUPTED,
+				in_irq ? GFP_NOWAIT : GFP_NOFS);
 	return -EFSCORRUPTED;
 }
 
-static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid,
-		struct folio *parent, int start, enum node_type ntype)
+static struct f2fs_cached_block *__get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid,
+		struct f2fs_cached_block *parent, int start, enum node_type ntype)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	int err;
 
 	if (!nid)
@@ -1590,71 +1584,71 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid,
 	if (f2fs_check_nid_range(sbi, nid))
 		return ERR_PTR(-EINVAL);
 repeat:
-	folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false);
-	if (IS_ERR(folio))
-		return folio;
+	entry = f2fs_grab_node_cache(sbi, nid);
+	if (IS_ERR(entry))
+		return entry;
 
-	err = read_node_folio(folio, 0);
+	err = read_node_cache(entry, 0);
 	if (err < 0)
 		goto out_put_err;
-	if (err == LOCKED_PAGE)
-		goto page_hit;
+	if (err == LOCKED_CACHE)
+		goto entry_hit;
 
 	if (parent)
-		f2fs_ra_node_pages(parent, start + 1, MAX_RA_NODE);
+		f2fs_ra_node_caches(parent, start + 1, MAX_RA_NODE);
 
-	folio_lock(folio);
+	f2fs_lock_cache(entry);
 
-	if (unlikely(!is_node_folio(folio))) {
-		f2fs_folio_put(folio, true);
+	if (unlikely(!f2fs_is_node_cache(entry))) {
+		f2fs_put_cache(entry, true);
 		goto repeat;
 	}
 
-	if (unlikely(!folio_test_uptodate(folio))) {
+	if (unlikely(!f2fs_cache_test_uptodate(entry))) {
 		err = -EIO;
 		goto out_put_err;
 	}
 
-	if (!f2fs_inode_chksum_verify(sbi, folio)) {
+	if (!f2fs_inode_chksum_verify(sbi, entry)) {
 		err = -EFSBADCRC;
 		goto out_err;
 	}
-page_hit:
-	err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false);
+entry_hit:
+	err = f2fs_sanity_check_node_footer(sbi, entry, nid, ntype, false);
 	if (!err)
-		return folio;
+		return entry;
 out_err:
-	clear_node_folio_dirty(folio);
+	f2fs_drop_cache_dirty(entry);
 out_put_err:
 	/* ENOENT comes from read_node_folio which is not an error. */
 	if (err != -ENOENT)
-		f2fs_handle_page_eio(sbi, folio, NODE);
-	f2fs_folio_put(folio, true);
+		f2fs_handle_page_eio(sbi, entry->index, NODE);
+	f2fs_put_cache(entry, true);
 	return ERR_PTR(err);
 }
 
-struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid,
+struct f2fs_cached_block *f2fs_get_node_cache(struct f2fs_sb_info *sbi, pgoff_t nid,
 						enum node_type node_type)
 {
-	return __get_node_folio(sbi, nid, NULL, 0, node_type);
+	return __get_node_cache(sbi, nid, NULL, 0, node_type);
 }
 
-struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino)
+struct f2fs_cached_block *f2fs_get_inode_cache(struct f2fs_sb_info *sbi, pgoff_t ino)
 {
-	return __get_node_folio(sbi, ino, NULL, 0, NODE_TYPE_INODE);
+	return __get_node_cache(sbi, ino, NULL, 0, NODE_TYPE_INODE);
 }
 
-struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid)
+struct f2fs_cached_block *f2fs_get_xnode_cache(struct f2fs_sb_info *sbi, pgoff_t xnid)
 {
-	return __get_node_folio(sbi, xnid, NULL, 0, NODE_TYPE_XATTR);
+	return __get_node_cache(sbi, xnid, NULL, 0, NODE_TYPE_XATTR);
 }
 
-static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start)
+static struct f2fs_cached_block *f2fs_get_node_cache_ra(struct f2fs_cached_block *parent, int start)
 {
-	struct f2fs_sb_info *sbi = F2FS_F_SB(parent);
-	nid_t nid = get_nid(parent, start, false);
+	struct f2fs_sb_info *sbi = parent->cache->sbi;
+	nid_t nid = get_nid(sbi, parent, start, false);
 
-	return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE);
+	return __get_node_cache(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE);
 }
 
 static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino)
@@ -1693,110 +1687,106 @@ static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino)
 	iput(inode);
 }
 
-static struct folio *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino)
+static struct f2fs_cached_block *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino)
 {
-	pgoff_t index;
-	struct folio_batch fbatch;
-	struct folio *last_folio = NULL;
-	int nr_folios;
-
-	folio_batch_init(&fbatch);
-	index = 0;
+	pgoff_t index = 0;
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
+	struct f2fs_cached_block *last_entry = NULL;
+	unsigned int nr;
 
-	while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index,
-					(pgoff_t)-1, PAGECACHE_TAG_DIRTY,
-					&fbatch))) {
+	while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi),
+				entries, &index, F2FS_ONSTACK_CACHES,
+				F2FS_CACHE_TAG_DIRTY))) {
 		int i;
 
-		for (i = 0; i < nr_folios; i++) {
-			struct folio *folio = fbatch.folios[i];
+		for (i = 0; i < nr; i++) {
+			struct f2fs_cached_block *entry = entries[i];
 
 			if (unlikely(f2fs_cp_error(sbi))) {
-				f2fs_folio_put(last_folio, false);
-				folio_batch_release(&fbatch);
+				f2fs_put_cache(last_entry, false);
+				f2fs_cache_gang_release(entries, nr);
 				return ERR_PTR(-EIO);
 			}
 
-			if (!IS_DNODE(folio) || !is_cold_node(folio))
+			if (!IS_DNODE(sbi, entry) || !is_cold_node(sbi, entry))
 				continue;
-			if (ino_of_node(folio) != ino)
+			if (ino_of_node(sbi, entry) != ino)
 				continue;
 
-			folio_lock(folio);
+			f2fs_lock_cache(entry);
 
-			if (unlikely(!is_node_folio(folio))) {
+			if (unlikely(!f2fs_is_node_cache(entry))) {
 continue_unlock:
-				folio_unlock(folio);
+				f2fs_unlock_cache(entry);
 				continue;
 			}
-			if (ino_of_node(folio) != ino)
+			if (ino_of_node(sbi, entry) != ino)
 				goto continue_unlock;
 
-			if (!folio_test_dirty(folio)) {
+			if (!f2fs_cache_test_dirty(entry)) {
 				/* someone wrote it for us */
 				goto continue_unlock;
 			}
 
-			if (last_folio)
-				f2fs_folio_put(last_folio, false);
+			if (last_entry)
+				f2fs_put_cache(last_entry, false);
 
-			folio_get(folio);
-			last_folio = folio;
-			folio_unlock(folio);
+			f2fs_cache_get(entry);
+			last_entry = entry;
+			f2fs_unlock_cache(entry);
 		}
-		folio_batch_release(&fbatch);
+		f2fs_cache_gang_release(entries, nr);
 		cond_resched();
 	}
-	return last_folio;
+	return last_entry;
 }
 
-static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync,
-				bool *submitted, struct writeback_control *wbc,
-				bool do_balance, enum iostat_type io_type,
-				unsigned int *seq_id)
+static bool __write_node_cache(struct f2fs_cached_block *entry,
+				bool atomic, bool do_fsync, bool *submitted,
+				bool sync, bool do_balance,
+				enum iostat_type io_type, unsigned int *seq_id)
 {
-	struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
+	struct f2fs_sb_info *sbi = entry->cache->sbi;
 	nid_t nid;
 	struct node_info ni;
 	struct f2fs_io_info fio = {
 		.sbi = sbi,
-		.ino = ino_of_node(folio),
+		.ino = ino_of_node(sbi, entry),
 		.type = NODE,
 		.op = REQ_OP_WRITE,
-		.op_flags = wbc_to_write_flags(wbc),
-		.folio = folio,
+		.op_flags = sync ? REQ_SYNC : REQ_BACKGROUND,
+		.cache_entry = entry,
 		.encrypted_page = NULL,
 		.submitted = 0,
 		.io_type = io_type,
-		.io_wbc = wbc,
+		.is_cache = true,
 	};
 	struct f2fs_lock_context lc;
 	unsigned int seq;
 
-	trace_f2fs_writepage(folio, NODE);
-
 	if (unlikely(f2fs_cp_error(sbi))) {
-		/* keep node pages in remount-ro mode */
+		/* keep node caches in remount-ro mode */
 		if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY)
 			goto redirty_out;
-		folio_clear_uptodate(folio);
-		dec_page_count(sbi, F2FS_DIRTY_NODES);
-		folio_unlock(folio);
+		f2fs_cache_clear_uptodate(entry);
+		f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY,
+					F2FS_CACHE_TAG_NONE);
+		dec_cache_count(entry->cache->sbi, F2FS_DIRTY_NODES);
+		f2fs_unlock_cache(entry);
 		return true;
 	}
 
 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 		goto redirty_out;
 
-	if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
-			wbc->sync_mode == WB_SYNC_NONE &&
-			IS_DNODE(folio) && is_cold_node(folio))
+	if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && !sync &&
+			IS_DNODE(sbi, entry) && is_cold_node(sbi, entry))
 		goto redirty_out;
 
-	/* get old block addr of this node page */
-	nid = nid_of_node(folio);
+	/* get old block addr of this node cache */
+	nid = nid_of_node(sbi, entry);
 
-	if (f2fs_sanity_check_node_footer(sbi, folio, folio->index,
+	if (f2fs_sanity_check_node_footer(sbi, entry, entry->index,
 					NODE_TYPE_REGULAR, false)) {
 		fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS);
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID);
@@ -1808,12 +1798,14 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync,
 
 	f2fs_down_read_trace(&sbi->node_write, &lc);
 
-	/* This page is already truncated */
+	/* This cache is already truncated */
 	if (unlikely(ni.blk_addr == NULL_ADDR)) {
-		folio_clear_uptodate(folio);
-		dec_page_count(sbi, F2FS_DIRTY_NODES);
+		f2fs_cache_clear_uptodate(entry);
+		f2fs_cache_update_tag(entry, F2FS_CACHE_TAG_DIRTY,
+					F2FS_CACHE_TAG_NONE);
+		dec_cache_count(entry->cache->sbi, F2FS_DIRTY_NODES);
 		f2fs_up_read_trace(&sbi->node_write, &lc);
-		folio_unlock(folio);
+		f2fs_unlock_cache(entry);
 		return true;
 	}
 
@@ -1827,28 +1819,28 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync,
 	if (atomic && !test_opt(sbi, NOBARRIER))
 		fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
 
-	set_dentry_mark(folio, false);
-	set_fsync_mark(folio, do_fsync);
-	if (IS_INODE(folio) && (atomic || is_fsync_dnode(folio)))
-		set_dentry_mark(folio,
-				f2fs_need_dentry_mark(sbi, ino_of_node(folio)));
+	set_dentry_mark(sbi, entry, false);
+	set_fsync_mark(sbi, entry, do_fsync);
+	if (IS_INODE(sbi, entry) && (atomic || is_fsync_dnode(sbi, entry)))
+		set_dentry_mark(sbi, entry,
+			f2fs_need_dentry_mark(sbi, ino_of_node(sbi, entry)));
 
-	/* should add to global list before clearing PAGECACHE status */
-	if (f2fs_in_warm_node_list(folio)) {
-		seq = f2fs_add_fsync_node_entry(sbi, folio);
+	/* should add to global list before clearing CACHE status */
+	if (f2fs_in_warm_node_list(sbi, entry)) {
+		seq = f2fs_add_fsync_node_entry(sbi, entry);
 		if (seq_id)
 			*seq_id = seq;
 	}
 
-	folio_start_writeback(folio);
+	f2fs_start_cache_writeback(entry);
 
 	fio.old_blkaddr = ni.blk_addr;
-	f2fs_do_write_node_page(nid, &fio);
-	set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(folio));
-	dec_page_count(sbi, F2FS_DIRTY_NODES);
+	f2fs_do_write_node_cache(nid, &fio);
+	set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(sbi, entry));
+	dec_cache_count(sbi, F2FS_DIRTY_NODES);
 	f2fs_up_read_trace(&sbi->node_write, &lc);
 
-	folio_unlock(folio);
+	f2fs_unlock_cache(entry);
 
 	if (unlikely(f2fs_cp_error(sbi))) {
 		f2fs_submit_merged_write(sbi, NODE);
@@ -1862,176 +1854,171 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync,
 	return true;
 
 redirty_out:
-	folio_redirty_for_writepage(wbc, folio);
-	folio_unlock(folio);
+	f2fs_cache_set_dirty(entry);
+	f2fs_unlock_cache(entry);
 	return false;
 }
 
-int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode,
+int f2fs_write_node_cache(struct f2fs_cached_block *node_entry, int sync_mode,
 			bool mark_dirty, enum iostat_type io_type)
 {
 	int err = 0;
-	struct writeback_control wbc = {
-		.sync_mode = WB_SYNC_ALL,
-		.nr_to_write = 1,
-	};
 
 	if (!sync_mode) {
-		/* set page dirty and write it */
-		if (!folio_test_writeback(node_folio))
-			folio_mark_dirty(node_folio);
-		goto out_folio;
+		/* set cache dirty and write it */
+		if (!f2fs_cache_test_writeback(node_entry))
+			f2fs_mark_cache_dirty(node_entry);
+		goto out_entry;
 	}
 
-	f2fs_folio_wait_writeback(node_folio, NODE, true, true);
+	f2fs_cache_wait_writeback(node_entry);
 
 	if (mark_dirty)
-		folio_mark_dirty(node_folio);
-	else if (!folio_test_dirty(node_folio))
-		goto out_folio;
+		f2fs_mark_cache_dirty(node_entry);
+	else if (!f2fs_cache_test_dirty(node_entry))
+		goto out_entry;
 
-	if (!folio_clear_dirty_for_io(node_folio)) {
+	if (!f2fs_cache_test_and_clear_dirty(node_entry)) {
 		err = -EAGAIN;
-		goto out_folio;
+		goto out_entry;
 	}
 
-	if (!__write_node_folio(node_folio, false, false, NULL,
-				&wbc, false, io_type, NULL))
+	if (!__write_node_cache(node_entry, false, false, NULL,
+				true, false, io_type, NULL))
 		err = -EAGAIN;
-	goto release_folio;
-out_folio:
-	folio_unlock(node_folio);
-release_folio:
-	f2fs_folio_put(node_folio, false);
+	goto release_entry;
+out_entry:
+	f2fs_unlock_cache(node_entry);
+release_entry:
+	f2fs_put_cache(node_entry, false);
 	return err;
 }
 
-int f2fs_move_node_folio(struct folio *node_folio, int gc_type)
+int f2fs_move_node_cache(struct f2fs_cached_block *entry, int gc_type)
 {
-	return f2fs_write_single_node_folio(node_folio, gc_type == FG_GC,
+	return f2fs_write_node_cache(entry, gc_type == FG_GC,
 			true, FS_GC_NODE_IO);
 }
 
-int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
-			struct writeback_control *wbc, bool atomic,
-			unsigned int *seq_id)
+int f2fs_fsync_node_caches(struct f2fs_sb_info *sbi, struct inode *inode,
+					bool atomic, unsigned int *seq_id)
 {
 	pgoff_t index;
-	struct folio_batch fbatch;
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
 	int ret = 0;
-	struct folio *last_folio = NULL;
+	struct f2fs_cached_block *last_entry = NULL;
 	bool marked = false;
 	nid_t ino = inode->i_ino;
-	int nr_folios;
+	int nr;
 	int nwritten = 0;
 
 	if (atomic) {
-		last_folio = last_fsync_dnode(sbi, ino);
-		if (IS_ERR_OR_NULL(last_folio))
-			return PTR_ERR_OR_ZERO(last_folio);
+		last_entry = last_fsync_dnode(sbi, ino);
+		if (IS_ERR_OR_NULL(last_entry))
+			return PTR_ERR_OR_ZERO(last_entry);
 	}
 retry:
-	folio_batch_init(&fbatch);
 	index = 0;
 
-	while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index,
-					(pgoff_t)-1, PAGECACHE_TAG_DIRTY,
-					&fbatch))) {
+	while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi),
+				entries, &index, F2FS_ONSTACK_CACHES,
+				F2FS_CACHE_TAG_DIRTY))) {
 		int i;
 
-		for (i = 0; i < nr_folios; i++) {
-			struct folio *folio = fbatch.folios[i];
+		for (i = 0; i < nr; i++) {
+			struct f2fs_cached_block *entry = entries[i];
 			bool submitted = false;
 			bool do_fsync = false;
 
 			if (unlikely(f2fs_cp_error(sbi))) {
-				f2fs_folio_put(last_folio, false);
-				folio_batch_release(&fbatch);
+				f2fs_put_cache(last_entry, false);
+				f2fs_cache_gang_release(entries, nr);
 				ret = -EIO;
 				goto out;
 			}
 
-			if (!IS_DNODE(folio) || !is_cold_node(folio))
+			if (!IS_DNODE(sbi, entry) || !is_cold_node(sbi, entry))
 				continue;
-			if (ino_of_node(folio) != ino)
+			if (ino_of_node(sbi, entry) != ino)
 				continue;
 
-			folio_lock(folio);
+			f2fs_lock_cache(entry);
 
-			if (unlikely(!is_node_folio(folio))) {
+			if (unlikely(!f2fs_is_node_cache(entry))) {
 continue_unlock:
-				folio_unlock(folio);
+				f2fs_unlock_cache(entry);
 				continue;
 			}
-			if (ino_of_node(folio) != ino)
+			if (ino_of_node(sbi, entry) != ino)
 				goto continue_unlock;
 
-			if (!folio_test_dirty(folio) && folio != last_folio) {
+			if (!f2fs_cache_test_dirty(entry) && entry != last_entry) {
 				/* someone wrote it for us */
 				goto continue_unlock;
 			}
 
-			f2fs_folio_wait_writeback(folio, NODE, true, true);
+			f2fs_cache_wait_writeback(entry);
 
-			if (!atomic || folio == last_folio) {
+			if (!atomic || entry == last_entry) {
 				do_fsync = true;
 				percpu_counter_inc(&sbi->rf_node_block_count);
-				if (IS_INODE(folio)) {
+				if (IS_INODE(sbi, entry)) {
 					if (is_inode_flag_set(inode,
 								FI_DIRTY_INODE))
-						f2fs_update_inode(inode, folio);
+						f2fs_update_inode(inode, entry);
 				}
 				/* may be written by other thread */
-				if (!folio_test_dirty(folio))
-					folio_mark_dirty(folio);
+				if (!f2fs_cache_test_dirty(entry))
+					f2fs_mark_cache_dirty(entry);
 			}
 
-			if (!folio_clear_dirty_for_io(folio))
+			if (!f2fs_cache_test_and_clear_dirty(entry))
 				goto continue_unlock;
 
-			if (!__write_node_folio(folio, atomic &&
-						folio == last_folio,
+			if (!__write_node_cache(entry, atomic &&
+						entry == last_entry,
 						do_fsync, &submitted,
-						wbc, true, FS_NODE_IO,
+						true, true, FS_NODE_IO,
 						seq_id)) {
-				f2fs_folio_put(last_folio, false);
-				folio_batch_release(&fbatch);
+				f2fs_put_cache(last_entry, false);
+				f2fs_cache_gang_release(entries, nr);
 				ret = -EIO;
 				goto out;
 			}
 			if (submitted)
 				nwritten++;
 
-			if (folio == last_folio) {
-				f2fs_folio_put(folio, false);
-				folio_batch_release(&fbatch);
+			if (entry == last_entry) {
+				f2fs_put_cache(entry, false);
+				f2fs_cache_gang_release(entries, nr);
 				marked = true;
 				goto out;
 			}
 		}
-		folio_batch_release(&fbatch);
+		f2fs_cache_gang_release(entries, nr);
 		cond_resched();
 	}
 	if (atomic && !marked) {
 		f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx",
-			   ino, last_folio->index);
-		folio_lock(last_folio);
-		if (unlikely(!is_node_folio(last_folio))) {
-			f2fs_folio_put(last_folio, true);
+			   ino, last_entry->index);
+		f2fs_lock_cache(last_entry);
+		if (unlikely(!f2fs_is_node_cache(last_entry))) {
+			f2fs_put_cache(last_entry, true);
 			ret = -EAGAIN;
 			goto out;
 		}
-		f2fs_folio_wait_writeback(last_folio, NODE, true, true);
-		folio_mark_dirty(last_folio);
-		folio_unlock(last_folio);
+		f2fs_cache_wait_writeback(last_entry);
+		f2fs_mark_cache_dirty(last_entry);
+		f2fs_unlock_cache(last_entry);
 		goto retry;
 	}
 out:
 	if (nwritten)
-		f2fs_submit_merged_write_cond(sbi, NULL, NULL, ino, NODE);
+		f2fs_submit_merged_write_cache(sbi, NULL, ino, NODE);
 	return ret;
 }
 
+
 static int f2fs_match_ino(struct inode *inode, u64 ino, void *data)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
@@ -2056,18 +2043,18 @@ static int f2fs_match_ino(struct inode *inode, u64 ino, void *data)
 	return 1;
 }
 
-static bool flush_dirty_inode(struct folio *folio)
+static bool flush_dirty_cache_inode(struct f2fs_cached_block *entry)
 {
-	struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
+	struct f2fs_sb_info *sbi = entry->cache->sbi;
 	struct inode *inode;
-	nid_t ino = ino_of_node(folio);
+	nid_t ino = ino_of_node(sbi, entry);
 
 	inode = find_inode_nowait(sbi->sb, ino, f2fs_match_ino, NULL);
 	if (!inode)
 		return false;
 
-	f2fs_update_inode(inode, folio);
-	folio_unlock(folio);
+	f2fs_update_inode(inode, entry);
+	f2fs_unlock_cache(entry);
 
 	iput(inode);
 	return true;
@@ -2076,72 +2063,74 @@ static bool flush_dirty_inode(struct folio *folio)
 void f2fs_flush_inline_data(struct f2fs_sb_info *sbi)
 {
 	pgoff_t index = 0;
-	struct folio_batch fbatch;
-	int nr_folios;
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
+	unsigned int nr;
 
-	folio_batch_init(&fbatch);
-
-	while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index,
-					(pgoff_t)-1, PAGECACHE_TAG_DIRTY,
-					&fbatch))) {
+	while ((nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi),
+				entries, &index, F2FS_ONSTACK_CACHES,
+				F2FS_CACHE_TAG_DIRTY))) {
 		int i;
 
-		for (i = 0; i < nr_folios; i++) {
-			struct folio *folio = fbatch.folios[i];
-
-			if (!IS_INODE(folio))
-				continue;
+		for (i = 0; i < nr; i++) {
+			struct f2fs_cached_block *entry = entries[i];
 
-			folio_lock(folio);
+			f2fs_lock_cache(entry);
 
-			if (unlikely(!is_node_folio(folio)))
+			if (unlikely(!f2fs_is_node_cache(entry)))
+				goto unlock;
+			if (!IS_INODE(sbi, entry))
 				goto unlock;
-			if (!folio_test_dirty(folio))
+			if (!f2fs_cache_test_dirty(entry))
 				goto unlock;
 
 			/* flush inline_data, if it's async context. */
-			if (folio_test_f2fs_inline(folio)) {
-				folio_clear_f2fs_inline(folio);
-				folio_unlock(folio);
-				flush_inline_data(sbi, ino_of_node(folio));
+			if (f2fs_cache_test_inline(entry)) {
+				nid_t ino = ino_of_node(sbi, entry);
+
+				f2fs_cache_clear_inline(entry);
+				f2fs_unlock_cache(entry);
+				flush_inline_data(sbi, ino);
 				continue;
 			}
 unlock:
-			folio_unlock(folio);
+			f2fs_unlock_cache(entry);
 		}
-		folio_batch_release(&fbatch);
+		f2fs_cache_gang_release(entries, nr);
 		cond_resched();
 	}
 }
 
-int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
-				struct writeback_control *wbc,
-				bool do_balance, enum iostat_type io_type)
+int f2fs_writeback_node_caches(struct f2fs_sb_info *sbi, long nr_to_write,
+			bool sync, bool do_balance, enum iostat_type io_type)
 {
 	pgoff_t index;
-	struct folio_batch fbatch;
+	struct f2fs_cached_block *entries[F2FS_ONSTACK_CACHES];
 	int step = 0;
 	int nwritten = 0;
 	int ret = 0;
-	int nr_folios, done = 0;
+	int nr, done = 0;
 
-	folio_batch_init(&fbatch);
+	trace_f2fs_write_caches(sbi, nr_to_write, 0, NODE);
 
 next_step:
 	index = 0;
 
-	while (!done && (nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi),
-				&index, (pgoff_t)-1, PAGECACHE_TAG_DIRTY,
-				&fbatch))) {
+	while (!done && (nr = f2fs_cache_gang_lookup_tag(NODE_CACHE(sbi),
+				entries, &index, F2FS_ONSTACK_CACHES,
+				F2FS_CACHE_TAG_DIRTY))) {
 		int i;
 
-		for (i = 0; i < nr_folios; i++) {
-			struct folio *folio = fbatch.folios[i];
+		for (i = 0; i < nr; i++) {
+			struct f2fs_cached_block *entry = entries[i];
 			bool submitted = false;
 
+			if (!sync && unlikely(freezing(current))) {
+				done = 1;
+				break;
+			}
+
 			/* give a priority to WB_SYNC threads */
-			if (atomic_read(&sbi->wb_sync_req[NODE]) &&
-					wbc->sync_mode == WB_SYNC_NONE) {
+			if (atomic_read(&sbi->wb_sync_req[NODE]) && !sync) {
 				done = 1;
 				break;
 			}
@@ -2152,27 +2141,27 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
 			 * 1. dentry dnodes
 			 * 2. file dnodes
 			 */
-			if (step == 0 && IS_DNODE(folio))
+			if (step == 0 && IS_DNODE(sbi, entry))
 				continue;
-			if (step == 1 && (!IS_DNODE(folio) ||
-						is_cold_node(folio)))
+			if (step == 1 && (!IS_DNODE(sbi, entry) ||
+						is_cold_node(sbi, entry)))
 				continue;
-			if (step == 2 && (!IS_DNODE(folio) ||
-						!is_cold_node(folio)))
+			if (step == 2 && (!IS_DNODE(sbi, entry) ||
+						!is_cold_node(sbi, entry)))
 				continue;
 lock_node:
-			if (wbc->sync_mode == WB_SYNC_ALL)
-				folio_lock(folio);
-			else if (!folio_trylock(folio))
+			if (sync)
+				f2fs_lock_cache(entry);
+			else if (!f2fs_trylock_cache(entry))
 				continue;
 
-			if (unlikely(!is_node_folio(folio))) {
+			if (unlikely(!f2fs_is_node_cache(entry))) {
 continue_unlock:
-				folio_unlock(folio);
+				f2fs_unlock_cache(entry);
 				continue;
 			}
 
-			if (!folio_test_dirty(folio)) {
+			if (!f2fs_cache_test_dirty(entry)) {
 				/* someone wrote it for us */
 				goto continue_unlock;
 			}
@@ -2182,38 +2171,38 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
 				goto write_node;
 
 			/* flush inline_data */
-			if (folio_test_f2fs_inline(folio)) {
-				folio_clear_f2fs_inline(folio);
-				folio_unlock(folio);
-				flush_inline_data(sbi, ino_of_node(folio));
+			if (f2fs_cache_test_inline(entry)) {
+				f2fs_cache_clear_inline(entry);
+				f2fs_unlock_cache(entry);
+				flush_inline_data(sbi, ino_of_node(sbi, entry));
 				goto lock_node;
 			}
 
 			/* flush dirty inode */
-			if (IS_INODE(folio) && flush_dirty_inode(folio))
+			if (IS_INODE(sbi, entry) && flush_dirty_cache_inode(entry))
 				goto lock_node;
 write_node:
-			f2fs_folio_wait_writeback(folio, NODE, true, true);
+			f2fs_cache_wait_writeback(entry);
 
-			if (!folio_clear_dirty_for_io(folio))
+			if (!f2fs_cache_test_and_clear_dirty(entry))
 				goto continue_unlock;
 
-			if (!__write_node_folio(folio, false, false, &submitted,
-					wbc, do_balance, io_type, NULL)) {
-				folio_batch_release(&fbatch);
+			if (!__write_node_cache(entry, false, false, &submitted,
+					sync, do_balance, io_type, NULL)) {
+				f2fs_cache_gang_release(entries, nr);
 				ret = -EIO;
 				goto out;
 			}
 			if (submitted)
 				nwritten++;
 
-			if (--wbc->nr_to_write == 0)
+			if (nwritten >= nr_to_write)
 				break;
 		}
-		folio_batch_release(&fbatch);
+		f2fs_cache_gang_release(entries, nr);
 		cond_resched();
 
-		if (wbc->nr_to_write == 0) {
+		if (nwritten >= nr_to_write) {
 			step = 2;
 			break;
 		}
@@ -2221,7 +2210,7 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
 
 	if (step < 2) {
 		if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
-				wbc->sync_mode == WB_SYNC_NONE && step == 1)
+				!sync && step == 1)
 			goto out;
 		step++;
 		goto next_step;
@@ -2230,12 +2219,14 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
 	if (nwritten)
 		f2fs_submit_merged_write(sbi, NODE);
 
+	trace_f2fs_write_caches(sbi, nr_to_write, nwritten, NODE);
+
 	if (unlikely(f2fs_cp_error(sbi)))
 		return -EIO;
 	return ret;
 }
 
-int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
+int f2fs_wait_on_node_caches_writeback(struct f2fs_sb_info *sbi,
 						unsigned int seq_id)
 {
 	struct fsync_node_entry *fn;
@@ -2244,7 +2235,7 @@ int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
 	unsigned int cur_seq_id = 0;
 
 	while (seq_id && cur_seq_id < seq_id) {
-		struct folio *folio;
+		struct f2fs_cached_block *entry;
 
 		spin_lock_irqsave(&sbi->fsync_node_lock, flags);
 		if (list_empty(head)) {
@@ -2257,94 +2248,49 @@ int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
 			break;
 		}
 		cur_seq_id = fn->seq_id;
-		folio = fn->folio;
-		folio_get(folio);
+		entry = fn->entry;
+		f2fs_cache_get(entry);
 		spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
 
-		f2fs_folio_wait_writeback(folio, NODE, true, false);
+		f2fs_lock_cache(entry);
+		f2fs_cache_wait_writeback(entry);
 
-		folio_put(folio);
+		f2fs_put_cache(entry, true);
 	}
 
-	return filemap_check_errors(NODE_MAPPING(sbi));
+	return f2fs_cp_error(sbi) ? -EIO : 0;
 }
 
-static int f2fs_write_node_pages(struct address_space *mapping,
-			    struct writeback_control *wbc)
+int f2fs_write_node_caches(struct f2fs_sb_info *sbi)
 {
-	struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
 	struct blk_plug plug;
-	long diff;
+	long nr_to_write = LONG_MAX;
 
 	if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
-		goto skip_write;
+		return -EAGAIN;
 
 	/* balancing f2fs's metadata in background */
 	f2fs_balance_fs_bg(sbi, true);
 
-	/* collect a number of dirty node pages and write together */
-	if (wbc->sync_mode != WB_SYNC_ALL &&
-			get_pages(sbi, F2FS_DIRTY_NODES) <
-					nr_pages_to_skip(sbi, NODE))
-		goto skip_write;
+	/* collect a number of dirty node caches and write together */
+	if (get_nr_caches(sbi, F2FS_DIRTY_NODES) <
+					nr_caches_to_skip(sbi, NODE))
+		return -EAGAIN;
 
-	if (wbc->sync_mode == WB_SYNC_ALL)
-		atomic_inc(&sbi->wb_sync_req[NODE]);
-	else if (atomic_read(&sbi->wb_sync_req[NODE])) {
+	if (atomic_read(&sbi->wb_sync_req[NODE])) {
 		/* to avoid potential deadlock */
 		if (current->plug)
 			blk_finish_plug(current->plug);
-		goto skip_write;
+		return -EAGAIN;
 	}
 
-	trace_f2fs_writepages(mapping->host, wbc, NODE);
-
-	diff = nr_pages_to_write(sbi, NODE, wbc);
+	nr_to_write = adjust_flush_cache_number(sbi, NODE);
 	blk_start_plug(&plug);
-	f2fs_sync_node_pages(sbi, wbc, true, FS_NODE_IO);
+	f2fs_writeback_node_caches(sbi, nr_to_write, false, true, FS_NODE_IO);
 	blk_finish_plug(&plug);
-	wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
-
-	if (wbc->sync_mode == WB_SYNC_ALL)
-		atomic_dec(&sbi->wb_sync_req[NODE]);
 	return 0;
-
-skip_write:
-	wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES);
-	trace_f2fs_writepages(mapping->host, wbc, NODE);
-	return 0;
-}
-
-static bool f2fs_dirty_node_folio(struct address_space *mapping,
-		struct folio *folio)
-{
-	trace_f2fs_set_page_dirty(folio, NODE);
-
-	if (!folio_test_uptodate(folio))
-		folio_mark_uptodate(folio);
-#ifdef CONFIG_F2FS_CHECK_FS
-	if (IS_INODE(folio))
-		f2fs_inode_chksum_set(F2FS_M_SB(mapping), folio);
-#endif
-	if (filemap_dirty_folio(mapping, folio)) {
-		inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES);
-		folio_set_f2fs_reference(folio);
-		return true;
-	}
-	return false;
 }
 
-/*
- * Structure of the f2fs node operations
- */
-const struct address_space_operations f2fs_node_aops = {
-	.writepages	= f2fs_write_node_pages,
-	.dirty_folio	= f2fs_dirty_node_folio,
-	.invalidate_folio = f2fs_invalidate_folio,
-	.release_folio	= f2fs_release_folio,
-	.migrate_folio	= filemap_migrate_folio,
-};
-
 static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,
 						nid_t n)
 {
@@ -2403,8 +2349,8 @@ static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid,
 							bool set, bool build)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
-	unsigned int nat_ofs = NAT_BLOCK_OFFSET(nid);
-	unsigned int nid_ofs = nid - START_NID(nid);
+	unsigned int nat_ofs = NAT_BLOCK_OFFSET(sbi, nid);
+	unsigned int nid_ofs = nid - f2fs_start_nid(sbi, nid);
 
 	if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap))
 		return;
@@ -2461,7 +2407,7 @@ static bool add_free_nid(struct f2fs_sb_info *sbi,
 		 *                     - f2fs_balance_fs_bg
 		 *                      - f2fs_build_free_nids
 		 *                       - __f2fs_build_free_nids
-		 *                        - scan_nat_page
+		 *                        - scan_nat_blocks
 		 *                         - add_free_nid
 		 *                          - __lookup_nat_cache
 		 *  - f2fs_add_link
@@ -2519,19 +2465,19 @@ static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid)
 		kmem_cache_free(free_nid_slab, i);
 }
 
-static int scan_nat_page(struct f2fs_sb_info *sbi,
+static int scan_nat_block(struct f2fs_sb_info *sbi,
 			struct f2fs_nat_block *nat_blk, nid_t start_nid)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	block_t blk_addr;
-	unsigned int nat_ofs = NAT_BLOCK_OFFSET(start_nid);
+	unsigned int nat_ofs = NAT_BLOCK_OFFSET(sbi, start_nid);
 	int i;
 
 	__set_bit_le(nat_ofs, nm_i->nat_block_bitmap);
 
-	i = start_nid % NAT_ENTRY_PER_BLOCK;
+	i = start_nid % NAT_ENTRY_PER_BLOCK(sbi);
 
-	for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) {
+	for (; i < NAT_ENTRY_PER_BLOCK(sbi); i++, start_nid++) {
 		if (unlikely(start_nid >= nm_i->max_nid))
 			break;
 
@@ -2587,16 +2533,16 @@ static void scan_free_nid_bits(struct f2fs_sb_info *sbi)
 			continue;
 		if (!nm_i->free_nid_count[i])
 			continue;
-		for (idx = 0; idx < NAT_ENTRY_PER_BLOCK; idx++) {
+		for (idx = 0; idx < NAT_ENTRY_PER_BLOCK(sbi); idx++) {
 			idx = find_next_bit_le(nm_i->free_nid_bitmap[i],
-						NAT_ENTRY_PER_BLOCK, idx);
-			if (idx >= NAT_ENTRY_PER_BLOCK)
+						NAT_ENTRY_PER_BLOCK(sbi), idx);
+			if (idx >= NAT_ENTRY_PER_BLOCK(sbi))
 				break;
 
-			nid = i * NAT_ENTRY_PER_BLOCK + idx;
+			nid = i * NAT_ENTRY_PER_BLOCK(sbi) + idx;
 			add_free_nid(sbi, nid, true, false);
 
-			if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS)
+			if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS(sbi))
 				goto out;
 		}
 	}
@@ -2610,6 +2556,7 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi,
 						bool sync, bool mount)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
+	struct f2fs_cached_block *entry = NULL;
 	int i = 0, ret;
 	nid_t nid = nm_i->next_scan_nid;
 	struct f2fs_lock_context lc;
@@ -2617,11 +2564,11 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi,
 	if (unlikely(nid >= nm_i->max_nid))
 		nid = 0;
 
-	if (unlikely(nid % NAT_ENTRY_PER_BLOCK))
-		nid = NAT_BLOCK_OFFSET(nid) * NAT_ENTRY_PER_BLOCK;
+	if (unlikely(nid % NAT_ENTRY_PER_BLOCK(sbi)))
+		nid = NAT_BLOCK_OFFSET(sbi, nid) * NAT_ENTRY_PER_BLOCK(sbi);
 
 	/* Enough entries */
-	if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK)
+	if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK(sbi))
 		return 0;
 
 	if (!sync && !f2fs_available_free_memory(sbi, FREE_NIDS))
@@ -2631,27 +2578,27 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi,
 		/* try to find free nids in free_nid_bitmap */
 		scan_free_nid_bits(sbi);
 
-		if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK)
+		if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK(sbi))
 			return 0;
 	}
 
-	/* readahead nat pages to be scanned */
-	f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES,
+	/* readahead nat blocks to be scanned */
+	f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nid), FREE_NID_BLOCKS,
 							META_NAT, true);
 
 	f2fs_down_read_trace(&nm_i->nat_tree_lock, &lc);
 
 	while (1) {
-		if (!test_bit_le(NAT_BLOCK_OFFSET(nid),
+		if (!test_bit_le(NAT_BLOCK_OFFSET(sbi, nid),
 						nm_i->nat_block_bitmap)) {
-			struct folio *folio = get_current_nat_folio(sbi, nid);
+			entry = get_current_nat_cache(sbi, nid);
 
-			if (IS_ERR(folio)) {
-				ret = PTR_ERR(folio);
+			if (IS_ERR(entry)) {
+				ret = PTR_ERR(entry);
 			} else {
-				ret = scan_nat_page(sbi, folio_address(folio),
+				ret = scan_nat_block(sbi, cache_address(entry),
 						nid);
-				f2fs_folio_put(folio, true);
+				f2fs_put_cache(entry, true);
 			}
 
 			if (ret) {
@@ -2668,24 +2615,25 @@ static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi,
 			}
 		}
 
-		nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
+		nid += NAT_ENTRY_PER_BLOCK(sbi) -
+			(nid % NAT_ENTRY_PER_BLOCK(sbi));
 		if (unlikely(nid >= nm_i->max_nid))
 			nid = 0;
 
-		if (++i >= FREE_NID_PAGES)
+		if (++i >= FREE_NID_BLOCKS)
 			break;
 	}
 
-	/* go to the next free nat pages to find free nids abundantly */
+	/* go to the next free nat blocks to find free nids abundantly */
 	nm_i->next_scan_nid = nid;
 
-	/* find free nids from current sum_pages */
+	/* find free nids from current sum_blocks */
 	scan_curseg_cache(sbi);
 
 	f2fs_up_read_trace(&nm_i->nat_tree_lock, &lc);
 
-	f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid),
-					nm_i->ra_nid_pages, META_NAT, false);
+	f2fs_ra_meta_caches(sbi, NAT_BLOCK_OFFSET(sbi, nm_i->next_scan_nid),
+					nm_i->ra_nid_blocks, META_NAT, false);
 
 	return 0;
 }
@@ -2750,7 +2698,7 @@ bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
 	}
 	spin_unlock(&nm_i->nid_list_lock);
 
-	/* Let's scan nat pages and its caches to get free nids */
+	/* Let's scan nat blocks and its caches to get free nids */
 	if (!f2fs_build_free_nids(sbi, true, false))
 		goto retry;
 	return false;
@@ -2811,20 +2759,20 @@ int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink)
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	int nr = nr_shrink;
 
-	if (nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS)
+	if (nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS(sbi))
 		return 0;
 
 	if (!mutex_trylock(&nm_i->build_lock))
 		return 0;
 
-	while (nr_shrink && nm_i->nid_cnt[FREE_NID] > MAX_FREE_NIDS) {
+	while (nr_shrink && nm_i->nid_cnt[FREE_NID] > MAX_FREE_NIDS(sbi)) {
 		struct free_nid *i, *next;
 		unsigned int batch = SHRINK_NID_BATCH_SIZE;
 
 		spin_lock(&nm_i->nid_list_lock);
 		list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) {
 			if (!nr_shrink || !batch ||
-				nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS)
+				nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS(sbi))
 				break;
 			__remove_free_nid(sbi, i, FREE_NID);
 			kmem_cache_free(free_nid_slab, i);
@@ -2839,18 +2787,18 @@ int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink)
 	return nr - nr_shrink;
 }
 
-int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio)
+int f2fs_recover_inline_xattr(struct inode *inode, struct f2fs_cached_block *entry)
 {
 	void *src_addr, *dst_addr;
 	size_t inline_size;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	struct f2fs_inode *ri;
 
-	ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
-	if (IS_ERR(ifolio))
-		return PTR_ERR(ifolio);
+	ientry = f2fs_get_inode_cache(F2FS_I_SB(inode), inode->i_ino);
+	if (IS_ERR(ientry))
+		return PTR_ERR(ientry);
 
-	ri = F2FS_INODE(folio);
+	ri = &CACHED_NODE(entry)->i;
 	if (ri->i_inline & F2FS_INLINE_XATTR) {
 		if (!f2fs_has_inline_xattr(inode)) {
 			set_inode_flag(inode, FI_INLINE_XATTR);
@@ -2864,26 +2812,26 @@ int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio)
 		goto update_inode;
 	}
 
-	dst_addr = inline_xattr_addr(inode, ifolio);
-	src_addr = inline_xattr_addr(inode, folio);
+	dst_addr = inline_xattr_addr(inode, ientry);
+	src_addr = inline_xattr_addr(inode, entry);
 	inline_size = inline_xattr_size(inode);
 
-	f2fs_folio_wait_writeback(ifolio, NODE, true, true);
+	f2fs_cache_wait_writeback(ientry);
 	memcpy(dst_addr, src_addr, inline_size);
 update_inode:
-	f2fs_update_inode(inode, ifolio);
-	f2fs_folio_put(ifolio, true);
+	f2fs_update_inode(inode, ientry);
+	f2fs_put_cache(ientry, true);
 	return 0;
 }
 
-int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio)
+int f2fs_recover_xattr_data(struct inode *inode, struct f2fs_cached_block *entry)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid;
 	nid_t new_xnid;
 	struct dnode_of_data dn;
 	struct node_info ni;
-	struct folio *xfolio;
+	struct f2fs_cached_block *xentry;
 	int err;
 
 	if (!prev_xnid)
@@ -2904,32 +2852,33 @@ int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio)
 		return -ENOSPC;
 
 	set_new_dnode(&dn, inode, NULL, NULL, new_xnid);
-	xfolio = f2fs_new_node_folio(&dn, XATTR_NODE_OFFSET);
-	if (IS_ERR(xfolio)) {
+	xentry = f2fs_new_node_cache(&dn, XATTR_NODE_OFFSET);
+	if (IS_ERR(xentry)) {
 		f2fs_alloc_nid_failed(sbi, new_xnid);
-		return PTR_ERR(xfolio);
+		return PTR_ERR(xentry);
 	}
 
 	f2fs_alloc_nid_done(sbi, new_xnid);
-	f2fs_update_inode_page(inode);
+	f2fs_update_inode_cache(inode);
 
-	/* 3: update and set xattr node page dirty */
-	if (folio) {
-		memcpy(F2FS_NODE(xfolio), F2FS_NODE(folio),
-				VALID_XATTR_BLOCK_SIZE);
-		folio_mark_dirty(xfolio);
+	/* 3: update and set xattr node cache dirty */
+	if (entry) {
+		memcpy(CACHED_NODE(xentry), CACHED_NODE(entry),
+				VALID_XATTR_BLOCK_SIZE(inode));
+		f2fs_mark_cache_dirty(xentry);
 	}
-	f2fs_folio_put(xfolio, true);
+	f2fs_put_cache(xentry, true);
 
 	return 0;
 }
 
-int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio)
+int f2fs_recover_inode_cache(struct f2fs_sb_info *sbi,
+				struct f2fs_cached_block *entry)
 {
 	struct f2fs_inode *src, *dst;
-	nid_t ino = ino_of_node(folio);
+	nid_t ino = ino_of_node(sbi, entry);
 	struct node_info old_ni, new_ni;
-	struct folio *ifolio;
+	struct f2fs_cached_block *ientry;
 	int err;
 
 	err = f2fs_get_node_info(sbi, ino, &old_ni, false);
@@ -2939,8 +2888,8 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio)
 	if (unlikely(old_ni.blk_addr != NULL_ADDR))
 		return -EINVAL;
 retry:
-	ifolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), ino, false);
-	if (IS_ERR(ifolio)) {
+	ientry = f2fs_grab_node_cache(sbi, ino);
+	if (IS_ERR(ientry)) {
 		memalloc_retry_wait(GFP_NOFS);
 		goto retry;
 	}
@@ -2948,13 +2897,12 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio)
 	/* Should not use this inode from free nid list */
 	remove_free_nid(sbi, ino);
 
-	if (!folio_test_uptodate(ifolio))
-		folio_mark_uptodate(ifolio);
-	fill_node_footer(ifolio, ino, ino, 0, true);
-	set_cold_node(ifolio, false);
+	f2fs_cache_set_uptodate(ientry);
+	fill_node_footer(sbi, ientry, ino, ino, 0, true);
+	set_cold_node(sbi, ientry, false);
 
-	src = F2FS_INODE(folio);
-	dst = F2FS_INODE(ifolio);
+	src = &CACHED_NODE(entry)->i;
+	dst = F2FS_INODE(ientry);
 
 	memcpy(dst, src, offsetof(struct f2fs_inode, i_ext));
 	dst->i_size = 0;
@@ -2990,46 +2938,46 @@ int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio)
 		WARN_ON(1);
 	set_node_addr(sbi, &new_ni, NEW_ADDR, false);
 	inc_valid_inode_count(sbi);
-	folio_mark_dirty(ifolio);
-	f2fs_folio_put(ifolio, true);
+	f2fs_mark_cache_dirty(ientry);
+	f2fs_put_cache(ientry, true);
 	return 0;
 }
 
 int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
 			unsigned int segno, struct f2fs_summary_block *sum)
 {
-	struct f2fs_node *rn;
 	struct f2fs_summary *sum_entry;
 	block_t addr;
-	int i, idx, last_offset, nrpages;
+	int i, idx, last_offset, nrblocks;
 
 	/* scan the node segment */
 	last_offset = BLKS_PER_SEG(sbi);
 	addr = START_BLOCK(sbi, segno);
 	sum_entry = sum_entries(sum);
 
-	for (i = 0; i < last_offset; i += nrpages, addr += nrpages) {
-		nrpages = bio_max_segs(last_offset - i);
+	for (i = 0; i < last_offset; i += nrblocks, addr += nrblocks) {
+		nrblocks = bio_max_segs(last_offset - i);
 
-		/* readahead node pages */
-		f2fs_ra_meta_pages(sbi, addr, nrpages, META_POR, true);
+		/* readahead node blocks */
+		f2fs_ra_meta_caches(sbi, addr, nrblocks, META_POR, true);
 
-		for (idx = addr; idx < addr + nrpages; idx++) {
-			struct folio *folio = f2fs_get_tmp_folio(sbi, idx);
+		for (idx = addr; idx < addr + nrblocks; idx++) {
+			struct f2fs_cached_block *entry =
+				f2fs_get_tmp_cache(sbi, idx);
 
-			if (IS_ERR(folio))
-				return PTR_ERR(folio);
+			if (IS_ERR(entry))
+				return PTR_ERR(entry);
 
-			rn = F2FS_NODE(folio);
-			sum_entry->nid = rn->footer.nid;
+			struct node_footer *footer = (struct node_footer *)(cache_address(entry) +
+				F2FS_BLKSIZE(sbi) - sizeof(struct node_footer));
+			sum_entry->nid = footer->nid;
 			sum_entry->version = 0;
 			sum_entry->ofs_in_node = 0;
 			sum_entry++;
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 		}
 
-		invalidate_mapping_pages(META_MAPPING(sbi), addr,
-							addr + nrpages);
+		f2fs_truncate_meta_caches(sbi, addr, nrblocks);
 	}
 	return 0;
 }
@@ -3074,7 +3022,7 @@ static void remove_nats_in_journal(struct f2fs_sb_info *sbi)
 			spin_unlock(&nm_i->nid_list_lock);
 		}
 
-		__set_nat_cache_dirty(nm_i, ne, init_dirty);
+		__set_nat_cache_dirty(sbi, nm_i, ne, init_dirty);
 	}
 	update_nats_in_cursum(journal, -i);
 	up_write(&curseg->journal_rwsem);
@@ -3102,7 +3050,7 @@ static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid,
 		const struct f2fs_nat_block *nat_blk)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
-	unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK;
+	unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK(sbi);
 	int valid = 0;
 	int i = 0;
 
@@ -3113,7 +3061,7 @@ static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid,
 		valid = 1;
 		i = 1;
 	}
-	for (; i < NAT_ENTRY_PER_BLOCK; i++) {
+	for (; i < NAT_ENTRY_PER_BLOCK(sbi); i++) {
 		if (le32_to_cpu(nat_blk->entries[i].block_addr) != NULL_ADDR)
 			valid++;
 	}
@@ -3124,7 +3072,7 @@ static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid,
 	}
 
 	__clear_bit_le(nat_index, nm_i->empty_nat_bits);
-	if (valid == NAT_ENTRY_PER_BLOCK)
+	if (valid == NAT_ENTRY_PER_BLOCK(sbi))
 		__set_bit_le(nat_index, nm_i->full_nat_bits);
 	else
 		__clear_bit_le(nat_index, nm_i->full_nat_bits);
@@ -3135,16 +3083,16 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
 {
 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
 	struct f2fs_journal *journal = curseg->journal;
-	nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;
+	nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK(sbi);
 	bool to_journal = true;
 	struct f2fs_nat_block *nat_blk;
 	struct nat_entry *ne, *cur;
-	struct folio *folio = NULL;
+	struct f2fs_cached_block *entry = NULL;
 
 	/*
 	 * there are two steps to flush nat entries:
 	 * #1, flush nat entries to journal in current hot data summary block.
-	 * #2, flush nat entries to nat page.
+	 * #2, flush nat entries to nat block.
 	 */
 	if (enabled_nat_bits(sbi, cpc) ||
 		!__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL))
@@ -3153,11 +3101,11 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
 	if (to_journal) {
 		down_write(&curseg->journal_rwsem);
 	} else {
-		folio = get_next_nat_folio(sbi, start_nid);
-		if (IS_ERR(folio))
-			return PTR_ERR(folio);
+		entry = get_next_nat_cache(sbi, start_nid);
+		if (IS_ERR(entry))
+			return PTR_ERR(entry);
 
-		nat_blk = folio_address(folio);
+		nat_blk = cache_address(entry);
 		f2fs_bug_on(sbi, !nat_blk);
 	}
 
@@ -3194,7 +3142,7 @@ static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
 		up_write(&curseg->journal_rwsem);
 	} else {
 		__update_nat_bits(sbi, start_nid, nat_blk);
-		f2fs_folio_put(folio, true);
+		f2fs_put_cache(entry, true);
 	}
 
 	/* Allow dirty nats by node block allocation in write_begin */
@@ -3266,7 +3214,7 @@ int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 			__has_cursum_space(sbi, journal,
 					entry_count, NAT_JOURNAL))
 			continue;
-		f2fs_ra_meta_pages(sbi, set->set, 1, META_NAT, true);
+		f2fs_ra_meta_caches(sbi, set->set, 1, META_NAT, true);
 	}
 	/* flush dirty nats in nat entry set */
 	list_for_each_entry_safe(set, tmp, &sets, set_list) {
@@ -3293,24 +3241,25 @@ static int __get_nat_bitmaps(struct f2fs_sb_info *sbi)
 	if (!enabled_nat_bits(sbi, NULL))
 		return 0;
 
-	nm_i->nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8);
+	nm_i->nat_bits_blocks = F2FS_BLK_ALIGN(sbi,
+					       (nat_bits_bytes << 1) + 8);
 	nm_i->nat_bits = f2fs_kvzalloc(sbi,
-			F2FS_BLK_TO_BYTES(nm_i->nat_bits_blocks), GFP_KERNEL);
+			F2FS_BLK_TO_BYTES(sbi, nm_i->nat_bits_blocks), GFP_KERNEL);
 	if (!nm_i->nat_bits)
 		return -ENOMEM;
 
 	nat_bits_addr = __start_cp_addr(sbi) + BLKS_PER_SEG(sbi) -
 						nm_i->nat_bits_blocks;
 	for (i = 0; i < nm_i->nat_bits_blocks; i++) {
-		struct folio *folio;
+		struct f2fs_cached_block *entry;
 
-		folio = f2fs_get_meta_folio(sbi, nat_bits_addr++);
-		if (IS_ERR(folio))
-			return PTR_ERR(folio);
+		entry = f2fs_get_meta_cache(sbi, nat_bits_addr++);
+		if (IS_ERR(entry))
+			return PTR_ERR(entry);
 
-		memcpy(nm_i->nat_bits + F2FS_BLK_TO_BYTES(i),
-					folio_address(folio), F2FS_BLKSIZE);
-		f2fs_folio_put(folio, true);
+		memcpy(nm_i->nat_bits + F2FS_BLK_TO_BYTES(sbi, i),
+					cache_address(entry), F2FS_BLKSIZE(sbi));
+		f2fs_put_cache(entry, true);
 	}
 
 	cp_ver |= (cur_cp_crc(ckpt) << 32);
@@ -3342,8 +3291,8 @@ static inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi)
 
 		__set_bit_le(i, nm_i->nat_block_bitmap);
 
-		nid = i * NAT_ENTRY_PER_BLOCK;
-		last_nid = nid + NAT_ENTRY_PER_BLOCK;
+		nid = i * NAT_ENTRY_PER_BLOCK(sbi);
+		last_nid = nid + NAT_ENTRY_PER_BLOCK(sbi);
 
 		spin_lock(&NM_I(sbi)->nid_list_lock);
 		for (; nid < last_nid; nid++)
@@ -3373,7 +3322,7 @@ static int init_node_manager(struct f2fs_sb_info *sbi)
 	/* segment_count_nat includes pair segment so divide to 2. */
 	nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1;
 	nm_i->nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg);
-	nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nm_i->nat_blocks;
+	nm_i->max_nid = NAT_ENTRY_PER_BLOCK(sbi) * nm_i->nat_blocks;
 
 	/* not used nids: 0, node, meta, (and root counted as valid node) */
 	nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count -
@@ -3381,7 +3330,7 @@ static int init_node_manager(struct f2fs_sb_info *sbi)
 	nm_i->nid_cnt[FREE_NID] = 0;
 	nm_i->nid_cnt[PREALLOC_NID] = 0;
 	nm_i->ram_thresh = DEF_RAM_THRESHOLD;
-	nm_i->ra_nid_pages = DEF_RA_NID_PAGES;
+	nm_i->ra_nid_blocks = DEF_RA_NID_BLOCKS;
 	nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD;
 	nm_i->max_rf_node_blocks = DEF_RF_NODE_BLOCKS;
 
@@ -3436,7 +3385,7 @@ static int init_free_nid_cache(struct f2fs_sb_info *sbi)
 
 	for (i = 0; i < nm_i->nat_blocks; i++) {
 		nm_i->free_nid_bitmap[i] = f2fs_kvzalloc(sbi,
-			f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK), GFP_KERNEL);
+			f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK(sbi)), GFP_KERNEL);
 		if (!nm_i->free_nid_bitmap[i])
 			return -ENOMEM;
 	}
diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h
index 5e114f3520998d..2704a5c6a54d4b 100644
--- a/fs/f2fs/node.h
+++ b/fs/f2fs/node.h
@@ -6,19 +6,26 @@
  *             http://www.samsung.com/
  */
 /* start node id of a node block dedicated to the given node id */
-#define	START_NID(nid) (((nid) / NAT_ENTRY_PER_BLOCK) * NAT_ENTRY_PER_BLOCK)
+static inline nid_t f2fs_start_nid(struct f2fs_sb_info *sbi, nid_t nid)
+{
+	unsigned int entries = NAT_ENTRY_PER_BLOCK(sbi);
+
+	return (nid / entries) * entries;
+}
 
 /* node block offset on the NAT area dedicated to the given start node id */
-#define	NAT_BLOCK_OFFSET(start_nid) ((start_nid) / NAT_ENTRY_PER_BLOCK)
+#define	NAT_BLOCK_OFFSET(sbi, start_nid) \
+	((start_nid) / NAT_ENTRY_PER_BLOCK(sbi))
 
-/* # of pages to perform synchronous readahead before building free nids */
-#define FREE_NID_PAGES	8
-#define MAX_FREE_NIDS	(NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES)
+/* # of blocks to perform synchronous readahead before building free nids */
+#define FREE_NID_BLOCKS	8
+#define MAX_FREE_NIDS(sbi)	((unsigned long)NAT_ENTRY_PER_BLOCK(sbi) * \
+				 FREE_NID_BLOCKS)
 
 /* size of free nid batch when shrinking */
 #define SHRINK_NID_BATCH_SIZE	8
 
-#define DEF_RA_NID_PAGES	0	/* # of nid pages to be readaheaded */
+#define DEF_RA_NID_BLOCKS	0	/* # of nid blocks to be readaheaded */
 
 /* maximum readahead size for node during getting data blocks */
 #define MAX_RA_NODE		128
@@ -37,8 +44,8 @@
 /* vector size for gang look-up from nat cache that consists of radix tree */
 #define NAT_VEC_SIZE	32
 
-/* return value for read_node_page */
-#define LOCKED_PAGE	1
+/* return value for read_node_cache */
+#define LOCKED_CACHE	1
 
 /* check pinned file's alignment status of physical blocks */
 #define FILE_NOT_ALIGNED	1
@@ -150,7 +157,7 @@ enum mem_type {
 	READ_EXTENT_CACHE,	/* indicates read extent cache */
 	AGE_EXTENT_CACHE,	/* indicates age extent cache */
 	DISCARD_CACHE,	/* indicates memory of cached discard cmds */
-	COMPRESS_PAGE,	/* indicates memory of cached compressed pages */
+	COMPRESS_BLOCK,	/* indicates memory of cached compressed blocks */
 	BASE_CHECK,	/* check kernel status */
 };
 
@@ -208,7 +215,7 @@ static inline pgoff_t current_nat_addr(struct f2fs_sb_info *sbi, nid_t start)
 	 * OLD = (segment_off * 512) * 2 + off_in_segment
 	 * NEW = 2 * (segment_off * 512 + off_in_segment) - off_in_segment
 	 */
-	block_off = NAT_BLOCK_OFFSET(start);
+	block_off = NAT_BLOCK_OFFSET(sbi, start);
 
 	block_addr = (pgoff_t)(nm_i->nat_blkaddr +
 		(block_off << 1) -
@@ -230,9 +237,10 @@ static inline pgoff_t next_nat_addr(struct f2fs_sb_info *sbi,
 	return block_addr + nm_i->nat_blkaddr;
 }
 
-static inline void set_to_next_nat(struct f2fs_nm_info *nm_i, nid_t start_nid)
+static inline void set_to_next_nat(struct f2fs_sb_info *sbi,
+				   struct f2fs_nm_info *nm_i, nid_t start_nid)
 {
-	unsigned int block_off = NAT_BLOCK_OFFSET(start_nid);
+	unsigned int block_off = NAT_BLOCK_OFFSET(sbi, start_nid);
 
 	f2fs_change_bit(block_off, nm_i->nat_bitmap);
 #ifdef CONFIG_F2FS_CHECK_FS
@@ -240,90 +248,95 @@ static inline void set_to_next_nat(struct f2fs_nm_info *nm_i, nid_t start_nid)
 #endif
 }
 
-static inline nid_t ino_of_node(const struct folio *node_folio)
+static inline nid_t ino_of_node(struct f2fs_sb_info *sbi,
+				const struct f2fs_cached_block *entry)
 {
-	struct f2fs_node *rn = F2FS_NODE(node_folio);
-	return le32_to_cpu(rn->footer.ino);
+	return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->ino);
 }
 
-static inline nid_t nid_of_node(const struct folio *node_folio)
+static inline nid_t nid_of_node(struct f2fs_sb_info *sbi,
+				const struct f2fs_cached_block *entry)
 {
-	struct f2fs_node *rn = F2FS_NODE(node_folio);
-	return le32_to_cpu(rn->footer.nid);
+	return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->nid);
 }
 
-static inline unsigned int ofs_of_node(const struct folio *node_folio)
+static inline unsigned int ofs_of_node(struct f2fs_sb_info *sbi,
+				const struct f2fs_cached_block *entry)
 {
-	struct f2fs_node *rn = F2FS_NODE(node_folio);
-	unsigned flag = le32_to_cpu(rn->footer.flag);
+	unsigned int flag = le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->flag);
 	return flag >> OFFSET_BIT_SHIFT;
 }
 
-static inline __u64 cpver_of_node(const struct folio *node_folio)
+static inline __u64 cpver_of_node(struct f2fs_sb_info *sbi,
+				const struct f2fs_cached_block *entry)
 {
-	struct f2fs_node *rn = F2FS_NODE(node_folio);
-	return le64_to_cpu(rn->footer.cp_ver);
+	return le64_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->cp_ver);
 }
 
-static inline block_t next_blkaddr_of_node(const struct folio *node_folio)
+static inline block_t next_blkaddr_of_node(struct f2fs_sb_info *sbi,
+				const struct f2fs_cached_block *entry)
 {
-	struct f2fs_node *rn = F2FS_NODE(node_folio);
-	return le32_to_cpu(rn->footer.next_blkaddr);
+	return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->next_blkaddr);
 }
 
-static inline void fill_node_footer(const struct folio *folio, nid_t nid,
+static inline void fill_node_footer(struct f2fs_sb_info *sbi,
+				struct f2fs_cached_block *entry, nid_t nid,
 				nid_t ino, unsigned int ofs, bool reset)
 {
-	struct f2fs_node *rn = F2FS_NODE(folio);
+	struct f2fs_node *rn = CACHED_NODE(entry);
+	struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry);
 	unsigned int old_flag = 0;
 
 	if (reset)
-		memset(rn, 0, sizeof(*rn));
+		memset(rn, 0, F2FS_BLKSIZE(sbi));
 	else
-		old_flag = le32_to_cpu(rn->footer.flag);
+		old_flag = le32_to_cpu(footer->flag);
 
-	rn->footer.nid = cpu_to_le32(nid);
-	rn->footer.ino = cpu_to_le32(ino);
+	memset(footer, 0, sizeof(*footer));
+	footer->nid = cpu_to_le32(nid);
+	footer->ino = cpu_to_le32(ino);
 
 	/* should remain old flag bits such as COLD_BIT_SHIFT */
-	rn->footer.flag = cpu_to_le32((ofs << OFFSET_BIT_SHIFT) |
+	footer->flag = cpu_to_le32((ofs << OFFSET_BIT_SHIFT) |
 					(old_flag & OFFSET_BIT_MASK));
 }
 
-static inline void copy_node_footer(const struct folio *dst,
-		const struct folio *src)
+static inline void copy_node_footer(struct f2fs_sb_info *sbi,
+		struct f2fs_cached_block *dst,
+		const struct f2fs_cached_block *src)
 {
-	struct f2fs_node *src_rn = F2FS_NODE(src);
-	struct f2fs_node *dst_rn = F2FS_NODE(dst);
-	memcpy(&dst_rn->footer, &src_rn->footer, sizeof(struct node_footer));
+	memcpy(F2FS_NODE_FOOTER(sbi, dst), F2FS_NODE_FOOTER(sbi, src),
+	       sizeof(struct node_footer));
 }
 
-static inline void fill_node_footer_blkaddr(struct folio *folio, block_t blkaddr)
+static inline void fill_node_footer_blkaddr(struct f2fs_sb_info *sbi,
+				struct f2fs_cached_block *entry, block_t blkaddr)
 {
-	struct f2fs_checkpoint *ckpt = F2FS_CKPT(F2FS_F_SB(folio));
-	struct f2fs_node *rn = F2FS_NODE(folio);
+	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
+	struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry);
 	__u64 cp_ver = cur_cp_version(ckpt);
 
 	if (__is_set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG))
 		cp_ver |= (cur_cp_crc(ckpt) << 32);
 
-	rn->footer.cp_ver = cpu_to_le64(cp_ver);
-	rn->footer.next_blkaddr = cpu_to_le32(blkaddr);
+	footer->cp_ver = cpu_to_le64(cp_ver);
+	footer->next_blkaddr = cpu_to_le32(blkaddr);
 }
 
-static inline bool is_recoverable_dnode(const struct folio *folio)
+static inline bool is_recoverable_dnode(struct f2fs_sb_info *sbi,
+				const struct f2fs_cached_block *entry)
 {
-	struct f2fs_checkpoint *ckpt = F2FS_CKPT(F2FS_F_SB(folio));
+	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
 	__u64 cp_ver = cur_cp_version(ckpt);
 
 	/* Don't care crc part, if fsck.f2fs sets it. */
 	if (__is_set_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG))
-		return (cp_ver << 32) == (cpver_of_node(folio) << 32);
+		return (cp_ver << 32) == (cpver_of_node(sbi, entry) << 32);
 
 	if (__is_set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG))
 		cp_ver |= (cur_cp_crc(ckpt) << 32);
 
-	return cp_ver == cpver_of_node(folio);
+	return cp_ver == cpver_of_node(sbi, entry);
 }
 
 /*
@@ -347,43 +360,50 @@ static inline bool is_recoverable_dnode(const struct folio *folio)
  *                 `- indirect node ((6 + 2N) + (N - 1)(N + 1))
  *                       `- direct node
  */
-static inline bool IS_DNODE(const struct folio *node_folio)
+static inline bool IS_DNODE(struct f2fs_sb_info *sbi,
+				const struct f2fs_cached_block *entry)
 {
-	unsigned int ofs = ofs_of_node(node_folio);
+	unsigned int ofs = ofs_of_node(sbi, entry);
 
 	if (f2fs_has_xattr_block(ofs))
 		return true;
 
-	if (ofs == 3 || ofs == 4 + NIDS_PER_BLOCK ||
-			ofs == 5 + 2 * NIDS_PER_BLOCK)
+	if (ofs == 3 || ofs == 4 + NIDS_PER_BLOCK(sbi) ||
+	    ofs == 5 + 2 * NIDS_PER_BLOCK(sbi))
 		return false;
-	if (ofs >= 6 + 2 * NIDS_PER_BLOCK) {
-		ofs -= 6 + 2 * NIDS_PER_BLOCK;
-		if (!((long int)ofs % (NIDS_PER_BLOCK + 1)))
+	if (ofs >= 6 + 2 * NIDS_PER_BLOCK(sbi)) {
+		ofs -= 6 + 2 * NIDS_PER_BLOCK(sbi);
+		if (!((long)ofs % (NIDS_PER_BLOCK(sbi) + 1)))
 			return false;
 	}
 	return true;
 }
 
-static inline int set_nid(struct folio *folio, int off, nid_t nid, bool i)
+static inline bool set_nid(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry, int off, nid_t nid, bool i)
 {
-	struct f2fs_node *rn = F2FS_NODE(folio);
+	struct f2fs_node *rn = CACHED_NODE(entry);
+	__le32 *inode_nids = F2FS_INODE_NIDS(sbi, entry);
+	__le32 *addr = i ? &inode_nids[off - NODE_DIR1_BLOCK(sbi)] : &rn->in.nid[off];
 
-	f2fs_folio_wait_writeback(folio, NODE, true, true);
+	f2fs_cache_wait_writeback(entry);
+	if (*addr == cpu_to_le32(nid))
+		return false;
 
-	if (i)
-		rn->i.i_nid[off - NODE_DIR1_BLOCK] = cpu_to_le32(nid);
-	else
-		rn->in.nid[off] = cpu_to_le32(nid);
-	return folio_mark_dirty(folio);
+	*addr = cpu_to_le32(nid);
+	f2fs_mark_cache_dirty(entry);
+	return true;
 }
 
-static inline nid_t get_nid(const struct folio *folio, int off, bool i)
+static inline nid_t get_nid(struct f2fs_sb_info *sbi,
+			const struct f2fs_cached_block *entry, int off, bool i)
 {
-	struct f2fs_node *rn = F2FS_NODE(folio);
+	struct f2fs_node *rn = CACHED_NODE(entry);
+	const __le32 *inode_nids = F2FS_INODE_NIDS(sbi, entry);
+	int nid_index = off - NODE_DIR1_BLOCK(sbi);
 
 	if (i)
-		return le32_to_cpu(rn->i.i_nid[off - NODE_DIR1_BLOCK]);
+		return le32_to_cpu(inode_nids[nid_index]);
 	return le32_to_cpu(rn->in.nid[off]);
 }
 
@@ -394,40 +414,43 @@ static inline nid_t get_nid(const struct folio *folio, int off, bool i)
  *  - Mark cold data pages in page cache
  */
 
-static inline int is_node(const struct folio *folio, int type)
+static inline int is_node(struct f2fs_sb_info *sbi,
+			const struct f2fs_cached_block *entry, int type)
 {
-	struct f2fs_node *rn = F2FS_NODE(folio);
-	return le32_to_cpu(rn->footer.flag) & BIT(type);
+	return le32_to_cpu(F2FS_NODE_FOOTER(sbi, entry)->flag) & BIT(type);
 }
 
-#define is_cold_node(folio)	is_node(folio, COLD_BIT_SHIFT)
-#define is_fsync_dnode(folio)	is_node(folio, FSYNC_BIT_SHIFT)
-#define is_dent_dnode(folio)	is_node(folio, DENT_BIT_SHIFT)
+#define is_cold_node(sbi, entry)	is_node(sbi, entry, COLD_BIT_SHIFT)
+#define is_fsync_dnode(sbi, entry)	is_node(sbi, entry, FSYNC_BIT_SHIFT)
+#define is_dent_dnode(sbi, entry)	is_node(sbi, entry, DENT_BIT_SHIFT)
 
-static inline void __set_mark(const struct folio *folio, bool mark, int type)
+static inline void __set_mark(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry, bool mark, int type)
 {
-	struct f2fs_node *rn = F2FS_NODE(folio);
-	unsigned int flag = le32_to_cpu(rn->footer.flag);
+	struct node_footer *footer = F2FS_NODE_FOOTER(sbi, entry);
+	unsigned int flag = le32_to_cpu(footer->flag);
 
 	if (mark)
 		flag |= BIT(type);
 	else
 		flag &= ~BIT(type);
-	rn->footer.flag = cpu_to_le32(flag);
+	footer->flag = cpu_to_le32(flag);
 }
 
-static inline void set_cold_node(const struct folio *folio, bool is_dir)
+static inline void set_cold_node(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry, bool is_dir)
 {
-	__set_mark(folio, !is_dir, COLD_BIT_SHIFT);
+	__set_mark(sbi, entry, !is_dir, COLD_BIT_SHIFT);
 }
 
-static inline void set_mark(struct folio *folio, bool mark, int type)
+static inline void set_mark(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry, bool mark, int type)
 {
-	__set_mark(folio, mark, type);
-
+	__set_mark(sbi, entry, mark, type);
 #ifdef CONFIG_F2FS_CHECK_FS
-	f2fs_inode_chksum_set(F2FS_F_SB(folio), folio);
+	f2fs_inode_chksum_set(sbi, entry);
 #endif
 }
-#define set_dentry_mark(folio, mark)	set_mark(folio, mark, DENT_BIT_SHIFT)
-#define set_fsync_mark(folio, mark)	set_mark(folio, mark, FSYNC_BIT_SHIFT)
+
+#define set_dentry_mark(sbi, entry, mark)	set_mark(sbi, entry, mark, DENT_BIT_SHIFT)
+#define set_fsync_mark(sbi, entry, mark)	set_mark(sbi, entry, mark, FSYNC_BIT_SHIFT)
diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c
index aaa5227739c892..54b5246609de36 100644
--- a/fs/f2fs/recovery.c
+++ b/fs/f2fs/recovery.c
@@ -182,38 +182,39 @@ static const char *recover_printable_name(struct inode *inode,
 	return raw->i_name;
 }
 
-static int recover_dentry(struct inode *inode, struct folio *ifolio,
+static int recover_dentry(struct inode *inode, struct f2fs_cached_block *entry,
 						struct list_head *dir_list)
 {
-	struct f2fs_inode *raw_inode = F2FS_INODE(ifolio);
+	struct f2fs_inode *raw_inode = &CACHED_NODE(entry)->i;
 	nid_t pino = le32_to_cpu(raw_inode->i_pino);
 	struct f2fs_dir_entry *de;
 	struct f2fs_filename fname;
 	struct qstr usr_fname;
-	struct folio *folio;
+	void *dentry_blk = NULL;
 	struct inode *dir, *einode;
-	struct fsync_inode_entry *entry;
+	struct fsync_inode_entry *fsync_entry;
 	int err = 0;
 	const char *name;
 	int name_len;
 
-	entry = get_fsync_inode(dir_list, pino);
-	if (!entry) {
-		entry = add_fsync_inode(F2FS_I_SB(inode), dir_list,
+	fsync_entry = get_fsync_inode(dir_list, pino);
+	if (!fsync_entry) {
+		fsync_entry = add_fsync_inode(F2FS_I_SB(inode), dir_list,
 							pino, false);
-		if (IS_ERR(entry)) {
-			dir = ERR_CAST(entry);
-			err = PTR_ERR(entry);
+		if (IS_ERR(fsync_entry)) {
+			dir = ERR_CAST(fsync_entry);
+			err = PTR_ERR(fsync_entry);
 			goto out;
 		}
 	}
 
-	dir = entry->inode;
+	dir = fsync_entry->inode;
 	err = init_recovered_filename(dir, inode, raw_inode, &fname, &usr_fname);
 	if (err)
 		goto out;
 retry:
-	de = __f2fs_find_entry(dir, &fname, &folio);
+	dentry_blk = NULL;
+	de = __f2fs_find_entry(dir, &fname, &dentry_blk);
 	if (de && inode->i_ino == le32_to_cpu(de->ino))
 		goto out_put;
 
@@ -238,11 +239,11 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio,
 			iput(einode);
 			goto out_put;
 		}
-		f2fs_delete_entry(de, folio, dir, einode);
+		f2fs_delete_entry(de, dentry_blk, dir, einode);
 		iput(einode);
 		goto retry;
-	} else if (IS_ERR(folio)) {
-		err = PTR_ERR(folio);
+	} else if (IS_ERR(dentry_blk)) {
+		err = PTR_ERR(dentry_blk);
 	} else {
 		err = f2fs_add_dentry(dir, &fname, inode,
 					inode->i_ino, inode->i_mode);
@@ -252,18 +253,18 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio,
 	goto out;
 
 out_put:
-	f2fs_folio_put(folio, false);
+	f2fs_put_dentry_block(dentry_blk, false);
 out:
 	name = recover_printable_name(inode, raw_inode, &name_len);
 	f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, dir = %llu, err = %d",
-		    __func__, ino_of_node(ifolio), name_len, name,
+		    __func__, ino_of_node(F2FS_I_SB(inode), entry), name_len, name,
 		    IS_ERR(dir) ? 0 : dir->i_ino, err);
 	return err;
 }
 
-static int recover_quota_data(struct inode *inode, struct folio *folio)
+static int recover_quota_data(struct inode *inode, struct f2fs_cached_block *entry)
 {
-	struct f2fs_inode *raw = F2FS_INODE(folio);
+	struct f2fs_inode *raw = &CACHED_NODE(entry)->i;
 	struct iattr attr;
 	uid_t i_uid = le32_to_cpu(raw->i_uid);
 	gid_t i_gid = le32_to_cpu(raw->i_gid);
@@ -300,9 +301,9 @@ static void recover_inline_flags(struct inode *inode, struct f2fs_inode *ri)
 		clear_inode_flag(inode, FI_DATA_EXIST);
 }
 
-static int recover_inode(struct inode *inode, struct folio *folio)
+static int recover_inode(struct inode *inode, struct f2fs_cached_block *entry)
 {
-	struct f2fs_inode *raw = F2FS_INODE(folio);
+	struct f2fs_inode *raw = &CACHED_NODE(entry)->i;
 	struct f2fs_inode_info *fi = F2FS_I(inode);
 	const char *name;
 	int name_len;
@@ -310,7 +311,7 @@ static int recover_inode(struct inode *inode, struct folio *folio)
 
 	inode->i_mode = le16_to_cpu(raw->i_mode);
 
-	err = recover_quota_data(inode, folio);
+	err = recover_quota_data(inode, entry);
 	if (err)
 		return err;
 
@@ -357,7 +358,7 @@ static int recover_inode(struct inode *inode, struct folio *folio)
 	name = recover_printable_name(inode, raw, &name_len);
 
 	f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, inline = %x",
-		    __func__, ino_of_node(folio), name_len, name,
+		    __func__, ino_of_node(F2FS_I_SB(inode), entry), name_len, name,
 		    raw->i_inline);
 	return 0;
 }
@@ -386,30 +387,30 @@ static int sanity_check_node_chain(struct f2fs_sb_info *sbi, block_t blkaddr,
 		return 0;
 
 	for (i = 0; i < 2; i++) {
-		struct folio *folio;
+		struct f2fs_cached_block *entry;
 
 		if (!f2fs_is_valid_blkaddr(sbi, *blkaddr_fast, META_POR)) {
 			*is_detecting = false;
 			return 0;
 		}
 
-		folio = f2fs_get_tmp_folio(sbi, *blkaddr_fast);
-		if (IS_ERR(folio))
-			return PTR_ERR(folio);
+		entry = f2fs_get_tmp_cache(sbi, *blkaddr_fast);
+		if (IS_ERR(entry))
+			return PTR_ERR(entry);
 
-		if (!is_recoverable_dnode(folio)) {
-			f2fs_folio_put(folio, true);
+		if (!is_recoverable_dnode(sbi, entry)) {
+			f2fs_put_cache(entry, true);
 			*is_detecting = false;
 			return 0;
 		}
 
 		ra_blocks = adjust_por_ra_blocks(sbi, ra_blocks, *blkaddr_fast,
-					next_blkaddr_of_node(folio));
+					next_blkaddr_of_node(sbi, entry));
 
-		*blkaddr_fast = next_blkaddr_of_node(folio);
-		f2fs_folio_put(folio, true);
+		*blkaddr_fast = next_blkaddr_of_node(sbi, entry);
+		f2fs_put_cache(entry, true);
 
-		f2fs_ra_meta_pages_cond(sbi, *blkaddr_fast, ra_blocks);
+		f2fs_ra_meta_caches_cond(sbi, *blkaddr_fast, ra_blocks);
 	}
 
 	if (*blkaddr_fast == blkaddr) {
@@ -434,45 +435,45 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head,
 	blkaddr_fast = blkaddr;
 
 	while (1) {
-		struct fsync_inode_entry *entry;
-		struct folio *folio;
+		struct fsync_inode_entry *fsync_entry;
+		struct f2fs_cached_block *entry;
 
 		if (!f2fs_is_valid_blkaddr(sbi, blkaddr, META_POR))
 			return 0;
 
-		folio = f2fs_get_tmp_folio(sbi, blkaddr);
-		if (IS_ERR(folio)) {
-			err = PTR_ERR(folio);
+		entry = f2fs_get_tmp_cache(sbi, blkaddr);
+		if (IS_ERR(entry)) {
+			err = PTR_ERR(entry);
 			break;
 		}
 
-		if (!is_recoverable_dnode(folio)) {
-			f2fs_folio_put(folio, true);
+		if (!is_recoverable_dnode(sbi, entry)) {
+			f2fs_put_cache(entry, true);
 			break;
 		}
 
-		if (!is_fsync_dnode(folio))
+		if (!is_fsync_dnode(sbi, entry))
 			goto next;
 
-		entry = get_fsync_inode(head, ino_of_node(folio));
-		if (!entry) {
+		fsync_entry = get_fsync_inode(head, ino_of_node(sbi, entry));
+		if (!fsync_entry) {
 			bool quota_inode = false;
 
 			if (!check_only &&
-					IS_INODE(folio) &&
-					is_dent_dnode(folio)) {
-				err = f2fs_recover_inode_page(sbi, folio);
+					IS_INODE(sbi, entry) &&
+					is_dent_dnode(sbi, entry)) {
+				err = f2fs_recover_inode_cache(sbi, entry);
 				if (err) {
-					f2fs_folio_put(folio, true);
+					f2fs_put_cache(entry, true);
 					break;
 				}
 				quota_inode = true;
 			}
 
-			entry = add_fsync_inode(sbi, head, ino_of_node(folio),
+			fsync_entry = add_fsync_inode(sbi, head, ino_of_node(sbi, entry),
 								quota_inode);
-			if (IS_ERR(entry)) {
-				err = PTR_ERR(entry);
+			if (IS_ERR(fsync_entry)) {
+				err = PTR_ERR(fsync_entry);
 				/*
 				 * CP | dnode(F) | inode(DF)
 				 * For this case, we should not give up now.
@@ -482,18 +483,18 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head,
 						*new_inode = true;
 					goto next;
 				}
-				f2fs_folio_put(folio, true);
+				f2fs_put_cache(entry, true);
 				break;
 			}
 		}
-		entry->blkaddr = blkaddr;
+		fsync_entry->blkaddr = blkaddr;
 
-		if (IS_INODE(folio) && is_dent_dnode(folio))
-			entry->last_dentry = blkaddr;
+		if (IS_INODE(sbi, entry) && is_dent_dnode(sbi, entry))
+			fsync_entry->last_dentry = blkaddr;
 next:
 		/* check next segment */
-		blkaddr = next_blkaddr_of_node(folio);
-		f2fs_folio_put(folio, true);
+		blkaddr = next_blkaddr_of_node(sbi, entry);
+		f2fs_put_cache(entry, true);
 
 		err = sanity_check_node_chain(sbi, blkaddr, &blkaddr_fast,
 				&is_detecting);
@@ -519,7 +520,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi,
 	unsigned short blkoff = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 	struct f2fs_summary_block *sum_node;
 	struct f2fs_summary sum;
-	struct folio *sum_folio, *node_folio;
+	struct f2fs_cached_block *entry = NULL;
+	struct f2fs_cached_block *node_entry;
 	struct dnode_of_data tdn = *dn;
 	nid_t ino, nid;
 	struct inode *inode;
@@ -541,18 +543,18 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi,
 		}
 	}
 
-	sum_folio = f2fs_get_sum_folio(sbi, segno);
-	if (IS_ERR(sum_folio))
-		return PTR_ERR(sum_folio);
-	sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, segno);
+	entry = f2fs_get_sum_cache(sbi, segno);
+	if (IS_ERR(entry))
+		return PTR_ERR(entry);
+	sum_node = SUM_BLK_ENTRY_ADDR(sbi, entry, segno);
 	sum = sum_entries(sum_node)[blkoff];
-	f2fs_folio_put(sum_folio, true);
+	f2fs_put_cache(entry, true);
 got_it:
 	/* Use the locked dnode page and inode */
 	nid = le32_to_cpu(sum.nid);
 	ofs_in_node = le16_to_cpu(sum.ofs_in_node);
 
-	max_addrs = ADDRS_PER_PAGE(dn->node_folio, dn->inode);
+	max_addrs = ADDRS_PER_PAGE(dn->node_entry, dn->inode);
 	if (ofs_in_node >= max_addrs) {
 		f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%llu, nid:%u, max:%u",
 			ofs_in_node, dn->inode->i_ino, nid, max_addrs);
@@ -562,9 +564,9 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi,
 
 	if (dn->inode->i_ino == nid) {
 		tdn.nid = nid;
-		if (!dn->inode_folio_locked)
-			folio_lock(dn->inode_folio);
-		tdn.node_folio = dn->inode_folio;
+		if (!dn->inode_entry_locked)
+			f2fs_lock_cache(dn->inode_entry);
+		tdn.node_entry = dn->inode_entry;
 		tdn.ofs_in_node = ofs_in_node;
 		goto truncate_out;
 	} else if (dn->nid == nid) {
@@ -573,13 +575,13 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi,
 	}
 
 	/* Get the node page */
-	node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR);
-	if (IS_ERR(node_folio))
-		return PTR_ERR(node_folio);
+	node_entry = f2fs_get_node_cache(sbi, nid, NODE_TYPE_REGULAR);
+	if (IS_ERR(node_entry))
+		return PTR_ERR(node_entry);
 
-	offset = ofs_of_node(node_folio);
-	ino = ino_of_node(node_folio);
-	f2fs_folio_put(node_folio, true);
+	offset = ofs_of_node(sbi, node_entry);
+	ino = ino_of_node(sbi, node_entry);
+	f2fs_put_cache(node_entry, true);
 
 	if (ino != dn->inode->i_ino) {
 		int ret;
@@ -605,8 +607,8 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi,
 	 * if inode page is locked, unlock temporarily, but its reference
 	 * count keeps alive.
 	 */
-	if (ino == dn->inode->i_ino && dn->inode_folio_locked)
-		folio_unlock(dn->inode_folio);
+	if (ino == dn->inode->i_ino && dn->inode_entry_locked)
+		f2fs_unlock_cache(dn->inode_entry);
 
 	set_new_dnode(&tdn, inode, NULL, NULL, 0);
 	if (f2fs_get_dnode_of_data(&tdn, bidx, LOOKUP_NODE))
@@ -619,15 +621,15 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi,
 out:
 	if (ino != dn->inode->i_ino)
 		iput(inode);
-	else if (dn->inode_folio_locked)
-		folio_lock(dn->inode_folio);
+	else if (dn->inode_entry_locked)
+		f2fs_lock_cache(dn->inode_entry);
 	return 0;
 
 truncate_out:
 	if (f2fs_data_blkaddr(&tdn) == blkaddr)
 		f2fs_truncate_data_blocks_range(&tdn, 1);
-	if (dn->inode->i_ino == nid && !dn->inode_folio_locked)
-		folio_unlock(dn->inode_folio);
+	if (dn->inode->i_ino == nid && !dn->inode_entry_locked)
+		f2fs_unlock_cache(dn->inode_entry);
 	return 0;
 }
 
@@ -645,7 +647,7 @@ static int f2fs_reserve_new_block_retry(struct dnode_of_data *dn)
 }
 
 static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
-					struct folio *folio)
+					struct f2fs_cached_block *entry)
 {
 	struct dnode_of_data dn;
 	struct node_info ni;
@@ -653,19 +655,19 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 	int err = 0, recovered = 0;
 
 	/* step 1: recover xattr */
-	if (IS_INODE(folio)) {
-		err = f2fs_recover_inline_xattr(inode, folio);
+	if (IS_INODE(sbi, entry)) {
+		err = f2fs_recover_inline_xattr(inode, entry);
 		if (err)
 			goto out;
-	} else if (f2fs_has_xattr_block(ofs_of_node(folio))) {
-		err = f2fs_recover_xattr_data(inode, folio);
+	} else if (f2fs_has_xattr_block(ofs_of_node(sbi, entry))) {
+		err = f2fs_recover_xattr_data(inode, entry);
 		if (!err)
 			recovered++;
 		goto out;
 	}
 
 	/* step 2: recover inline data */
-	err = f2fs_recover_inline_data(inode, folio);
+	err = f2fs_recover_inline_data(inode, entry);
 	if (err) {
 		if (err == 1)
 			err = 0;
@@ -673,8 +675,8 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 	}
 
 	/* step 3: recover data indices */
-	start = f2fs_start_bidx_of_node(ofs_of_node(folio), inode);
-	end = start + ADDRS_PER_PAGE(folio, inode);
+	start = f2fs_start_bidx_of_node(ofs_of_node(sbi, entry), inode);
+	end = start + addrs_per_page(inode, IS_INODE(sbi, entry));
 
 	set_new_dnode(&dn, inode, NULL, NULL, 0);
 retry_dn:
@@ -687,18 +689,18 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 		goto out;
 	}
 
-	f2fs_folio_wait_writeback(dn.node_folio, NODE, true, true);
+	f2fs_cache_wait_writeback(dn.node_entry);
 
 	err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
 	if (err)
 		goto err;
 
-	f2fs_bug_on(sbi, ni.ino != ino_of_node(folio));
+	f2fs_bug_on(sbi, ni.ino != ino_of_node(sbi, entry));
 
-	if (ofs_of_node(dn.node_folio) != ofs_of_node(folio)) {
+	if (ofs_of_node(sbi, dn.node_entry) != ofs_of_node(sbi, entry)) {
 		f2fs_warn(sbi, "Inconsistent ofs_of_node, ino:%llu, ofs:%u, %u",
-			  inode->i_ino, ofs_of_node(dn.node_folio),
-			  ofs_of_node(folio));
+			  inode->i_ino, ofs_of_node(sbi, dn.node_entry),
+			  ofs_of_node(sbi, entry));
 		err = -EFSCORRUPTED;
 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER);
 		fserror_report_file_metadata(dn.inode, err, GFP_NOFS);
@@ -709,7 +711,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 		block_t src, dest;
 
 		src = f2fs_data_blkaddr(&dn);
-		dest = data_blkaddr(dn.inode, folio, dn.ofs_in_node);
+		dest = data_blkaddr(dn.inode, entry, dn.ofs_in_node);
 
 		if (__is_valid_data_blkaddr(src) &&
 			!f2fs_is_valid_blkaddr(sbi, src, META_POR)) {
@@ -734,9 +736,9 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 		}
 
 		if (!file_keep_isize(inode) &&
-			(i_size_read(inode) <= ((loff_t)index << PAGE_SHIFT)))
+			(i_size_read(inode) <= F2FS_BLK_TO_BYTES(sbi, index)))
 			f2fs_i_size_write(inode,
-				(loff_t)(index + 1) << PAGE_SHIFT);
+				F2FS_BLK_TO_BYTES(sbi, index + 1));
 
 		/*
 		 * dest is reserved block, invalidate src block
@@ -784,16 +786,16 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 		}
 	}
 
-	copy_node_footer(dn.node_folio, folio);
-	fill_node_footer(dn.node_folio, dn.nid, ni.ino,
-					ofs_of_node(folio), false);
-	folio_mark_dirty(dn.node_folio);
+	copy_node_footer(sbi, dn.node_entry, entry);
+	fill_node_footer(sbi, dn.node_entry, dn.nid, ni.ino,
+					ofs_of_node(sbi, entry), false);
+	f2fs_mark_cache_dirty(dn.node_entry);
 err:
 	f2fs_put_dnode(&dn);
 out:
 	f2fs_notice(sbi, "recover_data: ino = %llx, nid = %x (i_size: %s), "
 		    "range (%u, %u), recovered = %d, err = %d",
-		    inode->i_ino, nid_of_node(folio),
+		    inode->i_ino, nid_of_node(sbi, entry),
 		    file_keep_isize(inode) ? "keep" : "recover",
 		    start, end, recovered, err);
 	return err;
@@ -820,26 +822,26 @@ static int recover_data(struct f2fs_sb_info *sbi, struct list_head *inode_list,
 	blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
 
 	while (1) {
-		struct fsync_inode_entry *entry;
-		struct folio *folio;
+		struct fsync_inode_entry *fsync_entry;
+		struct f2fs_cached_block *entry;
 
 		if (!f2fs_is_valid_blkaddr(sbi, blkaddr, META_POR))
 			break;
 
-		folio = f2fs_get_tmp_folio(sbi, blkaddr);
-		if (IS_ERR(folio)) {
-			err = PTR_ERR(folio);
+		entry = f2fs_get_tmp_cache(sbi, blkaddr);
+		if (IS_ERR(entry)) {
+			err = PTR_ERR(entry);
 			break;
 		}
 
-		if (!is_recoverable_dnode(folio)) {
-			f2fs_folio_put(folio, true);
+		if (!is_recoverable_dnode(sbi, entry)) {
+			f2fs_put_cache(entry, true);
 			break;
 		}
 		recoverable_dnode++;
 
-		entry = get_fsync_inode(inode_list, ino_of_node(folio));
-		if (!entry)
+		fsync_entry = get_fsync_inode(inode_list, ino_of_node(sbi, entry));
+		if (!fsync_entry)
 			goto next;
 		fsynced_dnode++;
 		/*
@@ -847,40 +849,40 @@ static int recover_data(struct f2fs_sb_info *sbi, struct list_head *inode_list,
 		 * In this case, we can lose the latest inode(x).
 		 * So, call recover_inode for the inode update.
 		 */
-		if (IS_INODE(folio)) {
-			err = recover_inode(entry->inode, folio);
+		if (IS_INODE(sbi, entry)) {
+			err = recover_inode(fsync_entry->inode, entry);
 			if (err) {
-				f2fs_folio_put(folio, true);
+				f2fs_put_cache(entry, true);
 				break;
 			}
 			recovered_inode++;
 		}
-		if (entry->last_dentry == blkaddr) {
-			err = recover_dentry(entry->inode, folio, dir_list);
+		if (fsync_entry->last_dentry == blkaddr) {
+			err = recover_dentry(fsync_entry->inode, entry, dir_list);
 			if (err) {
-				f2fs_folio_put(folio, true);
+				f2fs_put_cache(entry, true);
 				break;
 			}
 			recovered_dentry++;
 		}
-		err = do_recover_data(sbi, entry->inode, folio);
+		err = do_recover_data(sbi, fsync_entry->inode, entry);
 		if (err) {
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 			break;
 		}
 		recovered_dnode++;
 
-		if (entry->blkaddr == blkaddr)
-			list_move_tail(&entry->list, tmp_inode_list);
+		if (fsync_entry->blkaddr == blkaddr)
+			list_move_tail(&fsync_entry->list, tmp_inode_list);
 next:
 		ra_blocks = adjust_por_ra_blocks(sbi, ra_blocks, blkaddr,
-					next_blkaddr_of_node(folio));
+					next_blkaddr_of_node(sbi, entry));
 
 		/* check next segment */
-		blkaddr = next_blkaddr_of_node(folio);
-		f2fs_folio_put(folio, true);
+		blkaddr = next_blkaddr_of_node(sbi, entry);
+		f2fs_put_cache(entry, true);
 
-		f2fs_ra_meta_pages_cond(sbi, blkaddr, ra_blocks);
+		f2fs_ra_meta_caches_cond(sbi, blkaddr, ra_blocks);
 		total_dnode++;
 	}
 	if (!err)
@@ -937,12 +939,11 @@ int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only)
 	destroy_fsync_dnodes(&tmp_inode_list, err);
 
 	/* truncate meta pages to be used by the recovery */
-	truncate_inode_pages_range(META_MAPPING(sbi),
-			(loff_t)MAIN_BLKADDR(sbi) << PAGE_SHIFT, -1);
-
+	f2fs_truncate_meta_caches(sbi, MAIN_BLKADDR(sbi),
+				MAX_BLKADDR(sbi) - MAIN_BLKADDR(sbi));
 	if (err) {
-		truncate_inode_pages_final(NODE_MAPPING(sbi));
-		truncate_inode_pages_final(META_MAPPING(sbi));
+		f2fs_truncate_node_caches(sbi, 0, ULONG_MAX);
+		f2fs_truncate_meta_caches(sbi, 0, ULONG_MAX);
 	}
 
 	/*
diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index 63b712d3d599ed..b09ffe91664264 100644
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -335,8 +335,8 @@ static int __f2fs_commit_atomic_write(struct inode *inode)
 			goto next;
 		}
 
-		blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, cow_inode),
-				len);
+		blen = min((pgoff_t)addrs_per_page(cow_inode,
+				IS_INODE(F2FS_I_SB(cow_inode), dn.node_entry)), len);
 		index = off;
 		for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) {
 			blkaddr = f2fs_data_blkaddr(&dn);
@@ -479,11 +479,11 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
 {
 	int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
-	unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
-	unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
-	unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
-	unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
-	unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
+	unsigned int dents = get_nr_caches(sbi, F2FS_DIRTY_DENTS);
+	unsigned int qdata = get_nr_caches(sbi, F2FS_DIRTY_QDATA);
+	unsigned int nodes = get_nr_caches(sbi, F2FS_DIRTY_NODES);
+	unsigned int meta = get_nr_caches(sbi, F2FS_DIRTY_META);
+	unsigned int imeta = get_nr_caches(sbi, F2FS_DIRTY_IMETA);
 	unsigned int threshold =
 		SEGS_TO_BLKS(sbi, (factor * DEFAULT_DIRTY_THRESHOLD));
 	unsigned int global_threshold = threshold * 3 / 2;
@@ -512,10 +512,10 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
 
 	/* check the # of cached NAT entries */
 	if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
-		f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
+		f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK(sbi));
 
 	if (!f2fs_available_free_memory(sbi, FREE_NIDS))
-		f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
+		f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS(sbi));
 	else
 		f2fs_build_free_nids(sbi, false, false);
 
@@ -1310,13 +1310,14 @@ static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi,
 	/* sanity check on discard range */
 	__check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len);
 
-	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(dc->di.start);
+	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, dc->di.start);
 	bio->bi_private = dc;
 	bio->bi_end_io = f2fs_submit_discard_endio;
 	submit_bio(bio);
 
 	atomic_inc(&dcc->issued_discard);
-	f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE);
+	f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO,
+			   dc->di.len * F2FS_BLKSIZE(sbi));
 }
 #endif
 
@@ -1327,7 +1328,7 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
 {
 	struct block_device *bdev = dc->bdev;
 	unsigned int max_discard_blocks =
-			SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
+			SECTOR_TO_BLOCK(sbi, bdev_max_discard_sectors(bdev));
 	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 	struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
 					&(dcc->fstrim_list) : &(dcc->wait_list);
@@ -1389,8 +1390,8 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
 
 		dc->di.len += len;
 
-		__blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(start),
-				SECTOR_FROM_BLOCK(len), GFP_NOFS, &bio);
+		__blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(sbi, start),
+				SECTOR_FROM_BLOCK(sbi, len), GFP_NOFS, &bio);
 		f2fs_bug_on(sbi, !bio);
 
 		/*
@@ -1419,7 +1420,8 @@ static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
 
 		atomic_inc(&dcc->issued_discard);
 
-		f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE);
+		f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO,
+			   len * F2FS_BLKSIZE(sbi));
 
 		lstart += len;
 		start += len;
@@ -1518,7 +1520,7 @@ static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
 	struct discard_info di = {0};
 	struct rb_node **insert_p = NULL, *insert_parent = NULL;
 	unsigned int max_discard_blocks =
-			SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
+			SECTOR_TO_BLOCK(sbi, bdev_max_discard_sectors(bdev));
 	block_t end = lstart + len;
 
 	dc = __lookup_discard_cmd_ret(&dcc->root, lstart,
@@ -1871,6 +1873,60 @@ static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
 	return discard_blks;
 }
 
+void f2fs_drop_discard_cmd_range(struct f2fs_sb_info *sbi,
+					block_t start, block_t len)
+{
+	struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
+	struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
+	struct rb_node **insert_p = NULL, *insert_parent = NULL;
+	struct discard_cmd *dc, *wait_dc;
+	u64 cur = start;
+	u64 end = (u64)start + len;
+	int count;
+
+	if (!f2fs_realtime_discard_enable(sbi))
+		return;
+
+next:
+	count = 0;
+	wait_dc = NULL;
+
+	mutex_lock(&dcc->cmd_lock);
+	while (cur < end) {
+		dc = __lookup_discard_cmd_ret(&dcc->root, cur,
+				&prev_dc, &next_dc, &insert_p, &insert_parent);
+		if (!dc)
+			dc = next_dc;
+
+		if (!dc || (u64)dc->di.lstart >= end)
+			break;
+
+		if (dc->state == D_PREP) {
+			cur = (u64)dc->di.lstart + dc->di.len;
+			__remove_discard_cmd(sbi, dc);
+			if (++count >= MAX_DISCARD_DROP_COUNT)
+				break;
+			continue;
+		}
+
+		dc->ref++;
+		cur = (u64)dc->di.lstart + dc->di.len;
+		wait_dc = dc;
+		break;
+	}
+	mutex_unlock(&dcc->cmd_lock);
+
+	if (wait_dc) {
+		__wait_one_discard_bio(sbi, wait_dc);
+		goto next;
+	}
+
+	if (count >= MAX_DISCARD_DROP_COUNT) {
+		cond_resched();
+		goto next;
+	}
+}
+
 /* This should be covered by global mutex, &sit_i->sentry_lock */
 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
 {
@@ -2041,8 +2097,8 @@ static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
 
 	/* For sequential zones, reset the zone write pointer */
 	if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
-		sector = SECTOR_FROM_BLOCK(blkstart);
-		nr_sects = SECTOR_FROM_BLOCK(blklen);
+		sector = SECTOR_FROM_BLOCK(sbi, blkstart);
+		nr_sects = SECTOR_FROM_BLOCK(sbi, blklen);
 		div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder);
 
 		if (remainder || nr_sects != bdev_zone_sectors(bdev)) {
@@ -2749,12 +2805,12 @@ static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int typ
 }
 
 /*
- * Calculate the number of current summary pages for writing
+ * Calculate the number of current summary blocks for writing
  */
-int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
+int f2fs_nblocks_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
 {
 	int valid_sum_count = 0;
-	int i, sum_in_page;
+	int i, sum_in_block;
 
 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
 		if (sbi->ckpt->alloc_type[i] != SSR && for_ra)
@@ -2764,73 +2820,72 @@ int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
 			valid_sum_count += f2fs_curseg_valid_blocks(sbi, i);
 	}
 
-	sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size -
+	sum_in_block = (sbi->blocksize - 2 * sbi->sum_journal_size -
 			SUM_FOOTER_SIZE) / SUMMARY_SIZE;
-	if (valid_sum_count <= sum_in_page)
+	if (valid_sum_count <= sum_in_block)
 		return 1;
-	else if ((valid_sum_count - sum_in_page) <=
+	else if ((valid_sum_count - sum_in_block) <=
 		(sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
 		return 2;
 	return 3;
 }
 
-/*
- * Caller should put this summary folio
- */
-struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno)
+
+struct f2fs_cached_block *f2fs_get_sum_cache(struct f2fs_sb_info *sbi,
+		unsigned int segno)
 {
 	if (unlikely(f2fs_cp_error(sbi)))
 		return ERR_PTR(-EIO);
-	return f2fs_get_meta_folio_retry(sbi, GET_SUM_BLOCK(sbi, segno));
+	return f2fs_get_meta_cache_retry(sbi, GET_SUM_BLOCK(sbi, segno));
 }
 
-void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
+void f2fs_update_meta_block(struct f2fs_sb_info *sbi,
 					void *src, block_t blk_addr)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 
 	if (!f2fs_sb_has_packed_ssa(sbi))
-		folio = f2fs_grab_meta_folio(sbi, blk_addr);
+		entry = f2fs_grab_meta_cache(sbi, blk_addr);
 	else
-		folio = f2fs_get_meta_folio_retry(sbi, blk_addr);
+		entry = f2fs_get_meta_cache_retry(sbi, blk_addr);
 
-	if (IS_ERR(folio))
+	if (IS_ERR(entry))
 		return;
 
-	memcpy(folio_address(folio), src, PAGE_SIZE);
-	folio_mark_dirty(folio);
-	f2fs_folio_put(folio, true);
+	memcpy(cache_address(entry), src, F2FS_BLKSIZE(sbi));
+	f2fs_mark_cache_dirty(entry);
+	f2fs_put_cache(entry, true);
 }
 
-static void write_sum_page(struct f2fs_sb_info *sbi,
+static void write_sum_block(struct f2fs_sb_info *sbi,
 		struct f2fs_summary_block *sum_blk, unsigned int segno)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 
 	if (!f2fs_sb_has_packed_ssa(sbi))
-		return f2fs_update_meta_page(sbi, (void *)sum_blk,
+		return f2fs_update_meta_block(sbi, (void *)sum_blk,
 				GET_SUM_BLOCK(sbi, segno));
 
-	folio = f2fs_get_sum_folio(sbi, segno);
-	if (IS_ERR(folio))
+	entry = f2fs_get_sum_cache(sbi, segno);
+	if (IS_ERR(entry))
 		return;
 
-	memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk,
+	memcpy(SUM_BLK_ENTRY_ADDR(sbi, entry, segno), sum_blk,
 			sbi->sum_blocksize);
-	folio_mark_dirty(folio);
-	f2fs_folio_put(folio, true);
+	f2fs_mark_cache_dirty(entry);
+	f2fs_put_cache(entry, true);
 }
 
-static void write_current_sum_page(struct f2fs_sb_info *sbi,
+static void write_current_sum_block(struct f2fs_sb_info *sbi,
 						int type, block_t blk_addr)
 {
 	struct curseg_info *curseg = CURSEG_I(sbi, type);
-	struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr);
+	struct f2fs_cached_block *entry = f2fs_grab_meta_cache(sbi, blk_addr);
 	struct f2fs_summary_block *src = curseg->sum_blk;
 	struct f2fs_summary_block *dst;
 
-	dst = folio_address(folio);
-	memset(dst, 0, PAGE_SIZE);
+	dst = cache_address(entry);
+	memset(dst, 0, sbi->blocksize);
 
 	mutex_lock(&curseg->curseg_mutex);
 
@@ -2843,8 +2898,8 @@ static void write_current_sum_page(struct f2fs_sb_info *sbi,
 
 	mutex_unlock(&curseg->curseg_mutex);
 
-	folio_mark_dirty(folio);
-	f2fs_folio_put(folio, true);
+	f2fs_mark_cache_dirty(entry);
+	f2fs_put_cache(entry, true);
 }
 
 static int is_next_segment_free(struct f2fs_sb_info *sbi,
@@ -3104,7 +3159,7 @@ static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
 	int ret;
 
 	if (curseg->inited)
-		write_sum_page(sbi, curseg->sum_blk, segno);
+		write_sum_block(sbi, curseg->sum_blk, segno);
 
 	segno = __get_next_segno(sbi, type);
 	ret = get_new_segment(sbi, &segno, new_sec, pinning);
@@ -3160,10 +3215,10 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type)
 	struct curseg_info *curseg = CURSEG_I(sbi, type);
 	unsigned int new_segno = curseg->next_segno;
 	struct f2fs_summary_block *sum_node;
-	struct folio *sum_folio;
+	struct f2fs_cached_block *entry = NULL;
 
 	if (curseg->inited)
-		write_sum_page(sbi, curseg->sum_blk, curseg->segno);
+		write_sum_block(sbi, curseg->sum_blk, curseg->segno);
 
 	__set_test_and_inuse(sbi, new_segno);
 
@@ -3176,15 +3231,15 @@ static int change_curseg(struct f2fs_sb_info *sbi, int type)
 	curseg->alloc_type = SSR;
 	curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
 
-	sum_folio = f2fs_get_sum_folio(sbi, new_segno);
-	if (IS_ERR(sum_folio)) {
-		/* GC won't be able to use stale summary pages by cp_error */
+	entry = f2fs_get_sum_cache(sbi, new_segno);
+	if (IS_ERR(entry)) {
+		/* GC won't be able to use stale summary blocks by cp_error */
 		memset(curseg->sum_blk, 0, sbi->sum_entry_size);
-		return PTR_ERR(sum_folio);
+		return PTR_ERR(entry);
 	}
-	sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno);
+	sum_node = SUM_BLK_ENTRY_ADDR(sbi, entry, new_segno);
 	memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size);
-	f2fs_folio_put(sum_folio, true);
+	f2fs_put_cache(entry, true);
 	return 0;
 }
 
@@ -3271,7 +3326,7 @@ static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
 		goto out;
 
 	if (get_valid_blocks(sbi, curseg->segno, false)) {
-		write_sum_page(sbi, curseg->sum_blk, curseg->segno);
+		write_sum_block(sbi, curseg->sum_blk, curseg->segno);
 	} else {
 		mutex_lock(&DIRTY_I(sbi)->seglist_lock);
 		__set_test_and_free(sbi, curseg->segno, true);
@@ -3592,8 +3647,8 @@ static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
 
 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
 {
-	__u64 start = F2FS_BYTES_TO_BLK(range->start);
-	__u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
+	__u64 start = F2FS_BYTES_TO_BLK(sbi, range->start);
+	__u64 end = start + F2FS_BYTES_TO_BLK(sbi, range->len) - 1;
 	unsigned int start_segno, end_segno;
 	block_t start_block, end_block;
 	struct cp_control cpc;
@@ -3624,7 +3679,8 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
 	}
 
 	cpc.reason = CP_DISCARD;
-	cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
+	cpc.trim_minlen = max_t(__u64, 1,
+				F2FS_BYTES_TO_BLK(sbi, range->minlen));
 	cpc.trim_start = start_segno;
 	cpc.trim_end = end_segno;
 
@@ -3658,7 +3714,7 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
 					start_block, end_block);
 out:
 	if (!err)
-		range->len = F2FS_BLK_TO_BYTES(trimmed);
+		range->len = F2FS_BLK_TO_BYTES(sbi, trimmed);
 	return err;
 }
 
@@ -3770,7 +3826,8 @@ static int __get_segment_type_4(struct f2fs_io_info *fio)
 		else
 			return CURSEG_COLD_DATA;
 	} else {
-		if (IS_DNODE(fio->folio) && is_cold_node(fio->folio))
+		f2fs_bug_on(fio->sbi, !fio->is_cache);
+		if (IS_DNODE(fio->sbi, fio->cache_entry) && is_cold_node(fio->sbi, fio->cache_entry))
 			return CURSEG_WARM_NODE;
 		else
 			return CURSEG_COLD_NODE;
@@ -3803,7 +3860,7 @@ static int __get_segment_type_6(struct f2fs_io_info *fio)
 		if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
 			return CURSEG_COLD_DATA_PINNED;
 
-		if (page_private_gcing(fio->page)) {
+		if (folio_test_f2fs_gcing(fio->folio)) {
 			if (fio->sbi->am.atgc_enabled &&
 				(fio->io_type == FS_DATA_IO) &&
 				(fio->sbi->gc_mode != GC_URGENT_HIGH) &&
@@ -3828,9 +3885,9 @@ static int __get_segment_type_6(struct f2fs_io_info *fio)
 		return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode),
 						inode->i_write_hint);
 	} else {
-		if (IS_DNODE(fio->folio))
-			return is_cold_node(fio->folio) ? CURSEG_WARM_NODE :
-						CURSEG_HOT_NODE;
+		f2fs_bug_on(fio->sbi, !fio->is_cache);
+		if (IS_DNODE(fio->sbi, fio->cache_entry))
+			return is_cold_node(fio->sbi, fio->cache_entry) ? CURSEG_WARM_NODE : CURSEG_HOT_NODE;
 		return CURSEG_COLD_NODE;
 	}
 }
@@ -3897,7 +3954,7 @@ static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,
 		get_random_u32_inclusive(1, sbi->max_fragment_hole);
 }
 
-int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio,
+int f2fs_allocate_data_block(struct f2fs_sb_info *sbi,
 		block_t old_blkaddr, block_t *new_blkaddr,
 		struct f2fs_summary *sum, int type,
 		struct f2fs_io_info *fio)
@@ -3966,7 +4023,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio,
 	if (segment_full) {
 		if (type == CURSEG_COLD_DATA_PINNED &&
 		    !((curseg->segno + 1) % sbi->segs_per_sec)) {
-			write_sum_page(sbi, curseg->sum_blk, curseg->segno);
+			write_sum_block(sbi, curseg->sum_blk, curseg->segno);
 			reset_curseg_fields(curseg);
 			goto skip_new_segment;
 		}
@@ -4005,10 +4062,10 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio,
 
 	up_write(&sit_i->sentry_lock);
 
-	if (folio && IS_NODESEG(curseg->seg_type)) {
-		fill_node_footer_blkaddr(folio, NEXT_FREE_BLKADDR(sbi, curseg));
-
-		f2fs_inode_chksum_set(sbi, folio);
+	if (fio && fio->is_cache && IS_NODESEG(curseg->seg_type)) {
+		fill_node_footer_blkaddr(sbi, fio->cache_entry,
+				NEXT_FREE_BLKADDR(sbi, curseg));
+		f2fs_inode_chksum_set(sbi, fio->cache_entry);
 	}
 
 	if (fio) {
@@ -4084,7 +4141,7 @@ static int log_type_to_seg_type(enum log_type type)
 	return seg_type;
 }
 
-static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
+static void do_write_block(struct f2fs_summary *sum, struct f2fs_io_info *fio)
 {
 	struct folio *folio = fio->folio;
 	enum log_type type = __get_segment_type(fio);
@@ -4096,16 +4153,24 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
 	if (keep_order)
 		f2fs_down_read(&fio->sbi->io_order_lock);
 
-	err = f2fs_allocate_data_block(fio->sbi, folio, fio->old_blkaddr,
+	err = f2fs_allocate_data_block(fio->sbi, fio->old_blkaddr,
 			&fio->new_blkaddr, sum, type, fio);
 	if (unlikely(err)) {
-		f2fs_err_ratelimited(fio->sbi,
-			"%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d",
-			__func__, fio->ino, folio->index, type,
-			fio->old_blkaddr, fio->new_blkaddr, err);
-		folio_end_writeback(folio);
-		if (f2fs_in_warm_node_list(folio))
-			f2fs_del_fsync_node_entry(fio->sbi, folio);
+		if (fio->is_cache) {
+			f2fs_err_ratelimited(fio->sbi,
+				"%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d",
+				__func__, fio->ino, fio->cache_entry->index, type,
+				fio->old_blkaddr, fio->new_blkaddr, err);
+			if (f2fs_in_warm_node_list(fio->sbi, fio->cache_entry))
+				f2fs_del_fsync_node_entry(fio->sbi, fio->cache_entry);
+			f2fs_end_cache_writeback(fio->cache_entry);
+		} else {
+			f2fs_err_ratelimited(fio->sbi,
+				"%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d",
+				__func__, fio->ino, folio->index, type,
+				fio->old_blkaddr, fio->new_blkaddr, err);
+			folio_end_writeback(folio);
+		}
 		f2fs_bug_on(fio->sbi, !is_set_ckpt_flags(fio->sbi,
 							CP_ERROR_FLAG));
 		goto out;
@@ -4118,7 +4183,10 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
 		f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr, 1);
 
 	/* writeout dirty page into bdev */
-	f2fs_submit_page_write(fio);
+	if (fio->is_cache)
+		f2fs_submit_cache_write(fio);
+	else
+		f2fs_submit_page_write(fio);
 
 	f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
 out:
@@ -4126,8 +4194,9 @@ static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
 		f2fs_up_read(&fio->sbi->io_order_lock);
 }
 
-void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio,
-					enum iostat_type io_type)
+void f2fs_do_write_meta_cache(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry,
+			enum iostat_type io_type)
 {
 	struct f2fs_io_info fio = {
 		.sbi = sbi,
@@ -4135,31 +4204,36 @@ void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio,
 		.temp = HOT,
 		.op = REQ_OP_WRITE,
 		.op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
-		.old_blkaddr = folio->index,
-		.new_blkaddr = folio->index,
-		.folio = folio,
+		.old_blkaddr = entry->index,
+		.new_blkaddr = entry->index,
 		.encrypted_page = NULL,
 		.in_list = 0,
+		.cache_entry = entry,
+		.is_cache = 1,
 	};
 
-	if (unlikely(folio->index >= MAIN_BLKADDR(sbi)))
+	if (unlikely(entry->index >= MAIN_BLKADDR(sbi)))
 		fio.op_flags &= ~REQ_META;
 
-	folio_start_writeback(folio);
-	f2fs_submit_page_write(&fio);
+	f2fs_start_cache_writeback(entry);
+	f2fs_submit_cache_write(&fio);
 
-	stat_inc_meta_count(sbi, folio->index);
-	f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);
+	stat_inc_meta_count(sbi, entry->index);
+	f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE(sbi));
 }
 
-void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
+void f2fs_do_write_node_cache(unsigned int nid, struct f2fs_io_info *fio)
 {
 	struct f2fs_summary sum;
 
+	if (fio->is_cache)
+		trace_f2fs_write_cache(fio->cache_entry, NODE);
+
 	set_summary(&sum, nid, 0, 0);
-	do_write_page(&sum, fio);
+	do_write_block(&sum, fio);
 
-	f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);
+	f2fs_update_iostat(fio->sbi, NULL, fio->io_type,
+					F2FS_BLKSIZE(fio->sbi));
 }
 
 void f2fs_outplace_write_data(struct dnode_of_data *dn,
@@ -4172,10 +4246,11 @@ void f2fs_outplace_write_data(struct dnode_of_data *dn,
 	if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO)
 		f2fs_update_age_extent_cache(dn);
 	set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
-	do_write_page(&sum, fio);
+	do_write_block(&sum, fio);
 	f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
 
-	f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);
+	f2fs_update_iostat(sbi, dn->inode, fio->io_type,
+					F2FS_BLKSIZE(sbi));
 }
 
 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
@@ -4205,7 +4280,7 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio)
 	}
 
 	if (fio->meta_gc)
-		f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1);
+		f2fs_truncate_meta_caches(sbi, fio->new_blkaddr, 1);
 
 	stat_inc_inplace_blocks(fio->sbi);
 
@@ -4217,7 +4292,7 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio)
 		f2fs_update_device_state(fio->sbi, fio->ino,
 						fio->new_blkaddr, 1);
 		f2fs_update_iostat(fio->sbi, fio_inode(fio),
-						fio->io_type, F2FS_BLKSIZE);
+						fio->io_type, F2FS_BLKSIZE(fio->sbi));
 	}
 
 	return err;
@@ -4349,14 +4424,13 @@ void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
 	f2fs_update_data_blkaddr(dn, new_addr);
 }
 
-void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type,
-		bool ordered, bool locked)
+void f2fs_folio_wait_writeback(struct folio *folio, bool ordered, bool locked)
 {
 	if (folio_test_writeback(folio)) {
 		struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
 
 		/* submit cached LFS IO */
-		f2fs_submit_merged_write_folio(sbi, folio, type);
+		f2fs_submit_merged_write_folio(sbi, folio);
 		/* submit cached IPU IO */
 		f2fs_submit_merged_ipu_write(sbi, NULL, folio);
 		if (ordered) {
@@ -4371,7 +4445,7 @@ void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type,
 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-	struct folio *cfolio;
+	struct f2fs_cached_block *entry;
 
 	if (!f2fs_meta_inode_gc_required(inode))
 		return;
@@ -4379,11 +4453,12 @@ void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
 	if (!__is_valid_data_blkaddr(blkaddr))
 		return;
 
-	cfolio = filemap_lock_folio(META_MAPPING(sbi), blkaddr);
-	if (!IS_ERR(cfolio)) {
-		f2fs_folio_wait_writeback(cfolio, DATA, true, true);
-		f2fs_folio_put(cfolio, true);
-	}
+	entry = f2fs_find_cache(META_CACHE(sbi), blkaddr, 0);
+	if (IS_ERR(entry))
+		return;
+	f2fs_lock_cache(entry);
+	f2fs_cache_wait_writeback_cond(entry, DATA);
+	f2fs_put_cache(entry, true);
 }
 
 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
@@ -4398,7 +4473,7 @@ void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
 	for (i = 0; i < len; i++)
 		f2fs_wait_on_block_writeback(inode, blkaddr + i);
 
-	f2fs_truncate_meta_inode_pages(sbi, blkaddr, len);
+	f2fs_truncate_meta_caches(sbi, blkaddr, len);
 }
 
 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
@@ -4406,16 +4481,16 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi)
 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
 	struct curseg_info *seg_i;
 	unsigned char *kaddr;
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	block_t start;
 	int i, j, offset;
 
 	start = start_sum_block(sbi);
 
-	folio = f2fs_get_meta_folio(sbi, start++);
-	if (IS_ERR(folio))
-		return PTR_ERR(folio);
-	kaddr = folio_address(folio);
+	entry = f2fs_get_meta_cache(sbi, start++);
+	if (IS_ERR(entry))
+		return PTR_ERR(entry);
+	kaddr = cache_address(entry);
 
 	/* Step 1: restore nat cache */
 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
@@ -4452,16 +4527,16 @@ static int read_compacted_summaries(struct f2fs_sb_info *sbi)
 						SUM_FOOTER_SIZE)
 				continue;
 
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 
-			folio = f2fs_get_meta_folio(sbi, start++);
-			if (IS_ERR(folio))
-				return PTR_ERR(folio);
-			kaddr = folio_address(folio);
+			entry = f2fs_get_meta_cache(sbi, start++);
+			if (IS_ERR(entry))
+				return PTR_ERR(entry);
+			kaddr = cache_address(entry);
 			offset = 0;
 		}
 	}
-	f2fs_folio_put(folio, true);
+	f2fs_put_cache(entry, true);
 	return 0;
 }
 
@@ -4470,7 +4545,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
 	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
 	struct f2fs_summary_block *sum;
 	struct curseg_info *curseg;
-	struct folio *new;
+	struct f2fs_cached_block *entry;
 	unsigned short blk_off;
 	unsigned int segno = 0;
 	block_t blk_addr = 0;
@@ -4497,10 +4572,10 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
 			blk_addr = GET_SUM_BLOCK(sbi, segno);
 	}
 
-	new = f2fs_get_meta_folio(sbi, blk_addr);
-	if (IS_ERR(new))
-		return PTR_ERR(new);
-	sum = folio_address(new);
+	entry = f2fs_get_meta_cache(sbi, blk_addr);
+	if (IS_ERR(entry))
+		return PTR_ERR(entry);
+	sum = cache_address(entry);
 
 	if (IS_NODESEG(type)) {
 		if (__exist_node_summaries(sbi)) {
@@ -4537,7 +4612,7 @@ static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
 	curseg->next_blkoff = blk_off;
 	mutex_unlock(&curseg->curseg_mutex);
 out:
-	f2fs_folio_put(new, true);
+	f2fs_put_cache(entry, true);
 	return err;
 }
 
@@ -4549,11 +4624,11 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
 	int err;
 
 	if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
-		int npages = f2fs_npages_for_summary_flush(sbi, true);
+		int nblocks = f2fs_nblocks_for_summary_flush(sbi, true);
 
-		if (npages >= 2)
-			f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
-							META_CP, true);
+		if (nblocks >= 2)
+			f2fs_ra_meta_caches(sbi, start_sum_block(sbi),
+					nblocks, META_CP, true);
 
 		/* restore for compacted data summary */
 		err = read_compacted_summaries(sbi);
@@ -4563,7 +4638,7 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
 	}
 
 	if (__exist_node_summaries(sbi))
-		f2fs_ra_meta_pages(sbi,
+		f2fs_ra_meta_caches(sbi,
 				sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
 				NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
 
@@ -4586,16 +4661,16 @@ static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
 
 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
 {
-	struct folio *folio;
+	struct f2fs_cached_block *entry = NULL;
 	unsigned char *kaddr;
 	struct f2fs_summary *summary;
 	struct curseg_info *seg_i;
 	int written_size = 0;
 	int i, j;
 
-	folio = f2fs_grab_meta_folio(sbi, blkaddr++);
-	kaddr = folio_address(folio);
-	memset(kaddr, 0, PAGE_SIZE);
+	entry = f2fs_grab_meta_cache(sbi, blkaddr++);
+	kaddr = cache_address(entry);
+	memset(kaddr, 0, sbi->blocksize);
 
 	/* Step 1: write nat cache */
 	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
@@ -4611,10 +4686,10 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
 	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
 		seg_i = CURSEG_I(sbi, i);
 		for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) {
-			if (!folio) {
-				folio = f2fs_grab_meta_folio(sbi, blkaddr++);
-				kaddr = folio_address(folio);
-				memset(kaddr, 0, PAGE_SIZE);
+			if (!entry) {
+				entry = f2fs_grab_meta_cache(sbi, blkaddr++);
+				kaddr = cache_address(entry);
+				memset(kaddr, 0, sbi->blocksize);
 				written_size = 0;
 			}
 			summary = (struct f2fs_summary *)(kaddr + written_size);
@@ -4625,14 +4700,14 @@ static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
 							SUM_FOOTER_SIZE)
 				continue;
 
-			folio_mark_dirty(folio);
-			f2fs_folio_put(folio, true);
-			folio = NULL;
+			f2fs_mark_cache_dirty(entry);
+			f2fs_put_cache(entry, true);
+			entry = NULL;
 		}
 	}
-	if (folio) {
-		folio_mark_dirty(folio);
-		f2fs_folio_put(folio, true);
+	if (entry) {
+		f2fs_mark_cache_dirty(entry);
+		f2fs_put_cache(entry, true);
 	}
 }
 
@@ -4647,7 +4722,7 @@ static void write_normal_summaries(struct f2fs_sb_info *sbi,
 		end = type + NR_CURSEG_NODE_TYPE;
 
 	for (i = type; i < end; i++)
-		write_current_sum_page(sbi, i, blkaddr + (i - type));
+		write_current_sum_block(sbi, i, blkaddr + (i - type));
 }
 
 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
@@ -4686,29 +4761,29 @@ int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi,
 	return -1;
 }
 
-static struct folio *get_current_sit_folio(struct f2fs_sb_info *sbi,
+static struct f2fs_cached_block *get_current_sit_cache(struct f2fs_sb_info *sbi,
 					unsigned int segno)
 {
-	return f2fs_get_meta_folio(sbi, current_sit_addr(sbi, segno));
+	return f2fs_get_meta_cache(sbi, current_sit_addr(sbi, segno));
 }
 
-static struct folio *get_next_sit_folio(struct f2fs_sb_info *sbi,
+static struct f2fs_cached_block *get_next_sit_cache(struct f2fs_sb_info *sbi,
 					unsigned int start)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
-	struct folio *folio;
+	struct f2fs_cached_block *entry;
 	pgoff_t src_off, dst_off;
 
 	src_off = current_sit_addr(sbi, start);
 	dst_off = next_sit_addr(sbi, src_off);
 
-	folio = f2fs_grab_meta_folio(sbi, dst_off);
-	seg_info_to_sit_folio(sbi, folio, start);
+	entry = f2fs_grab_meta_cache(sbi, dst_off);
+	seg_info_to_sit_block(sbi, entry, start);
 
-	folio_mark_dirty(folio);
-	set_to_next_sit(sit_i, start);
+	f2fs_mark_cache_dirty(entry);
+	set_to_next_sit(sbi, sit_i, start);
 
-	return folio;
+	return entry;
 }
 
 static struct sit_entry_set *grab_sit_entry_set(void)
@@ -4745,10 +4820,11 @@ static void adjust_sit_entry_set(struct sit_entry_set *ses,
 	list_move_tail(&ses->set_list, head);
 }
 
-static void add_sit_entry(unsigned int segno, struct list_head *head)
+static void add_sit_entry(struct f2fs_sb_info *sbi, unsigned int segno,
+		struct list_head *head)
 {
 	struct sit_entry_set *ses;
-	unsigned int start_segno = START_SEGNO(segno);
+	unsigned int start_segno = f2fs_start_segno(sbi, segno);
 
 	list_for_each_entry(ses, head, set_list) {
 		if (ses->start_segno == start_segno) {
@@ -4773,7 +4849,7 @@ static void add_sits_in_set(struct f2fs_sb_info *sbi)
 	unsigned int segno;
 
 	for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
-		add_sit_entry(segno, set_list);
+		add_sit_entry(sbi, segno, set_list);
 }
 
 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
@@ -4791,7 +4867,7 @@ static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
 		dirtied = __mark_sit_entry_dirty(sbi, segno);
 
 		if (!dirtied)
-			add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
+			add_sit_entry(sbi, segno, &SM_I(sbi)->sit_entry_set);
 	}
 	update_sits_in_cursum(journal, -i);
 	up_write(&curseg->journal_rwsem);
@@ -4838,10 +4914,11 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 	 * #2, flush sit entries to sit page.
 	 */
 	list_for_each_entry_safe(ses, tmp, head, set_list) {
-		struct folio *folio = NULL;
+		struct f2fs_cached_block *entry = NULL;
 		struct f2fs_sit_block *raw_sit = NULL;
+
 		unsigned int start_segno = ses->start_segno;
-		unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
+		unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK(sbi),
 						(unsigned long)MAIN_SEGS(sbi));
 		unsigned int segno = start_segno;
 
@@ -4853,8 +4930,8 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 		if (to_journal) {
 			down_write(&curseg->journal_rwsem);
 		} else {
-			folio = get_next_sit_folio(sbi, start_segno);
-			raw_sit = folio_address(folio);
+			entry = get_next_sit_cache(sbi, start_segno);
+			raw_sit = cache_address(entry);
 		}
 
 		/* flush dirty sit entries in region of current sit set */
@@ -4899,7 +4976,7 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
 		if (to_journal)
 			up_write(&curseg->journal_rwsem);
 		else
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 
 		f2fs_bug_on(sbi, ses->entry_cnt);
 		release_sit_entry_set(ses);
@@ -5006,7 +5083,7 @@ static int build_sit_info(struct f2fs_sb_info *sbi)
 	sit_i->written_valid_blocks = 0;
 	sit_i->bitmap_size = sit_bitmap_size;
 	sit_i->dirty_sentries = 0;
-	sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
+	sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK(sbi);
 	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
 	sit_i->mounted_time = ktime_get_boottime_seconds();
 	init_rwsem(&sit_i->sentry_lock);
@@ -5090,7 +5167,7 @@ static int build_sit_entries(struct f2fs_sb_info *sbi)
 	block_t sit_valid_blocks[2] = {0, 0};
 
 	do {
-		readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
+		readed = f2fs_ra_meta_caches(sbi, start_blk, BIO_MAX_VECS,
 							META_SIT, true);
 
 		start = start_blk * sit_i->sents_per_block;
@@ -5098,15 +5175,15 @@ static int build_sit_entries(struct f2fs_sb_info *sbi)
 
 		for (; start < end && start < MAIN_SEGS(sbi); start++) {
 			struct f2fs_sit_block *sit_blk;
-			struct folio *folio;
+			struct f2fs_cached_block *entry;
 
 			se = &sit_i->sentries[start];
-			folio = get_current_sit_folio(sbi, start);
-			if (IS_ERR(folio))
-				return PTR_ERR(folio);
-			sit_blk = folio_address(folio);
+			entry = get_current_sit_cache(sbi, start);
+			if (IS_ERR(entry))
+				return PTR_ERR(entry);
+			sit_blk = cache_address(entry);
 			sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
-			f2fs_folio_put(folio, true);
+			f2fs_put_cache(entry, true);
 
 			err = check_block_count(sbi, start, &sit);
 			if (err)
diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h
index 5949aa5200ac16..526764ba31ae5b 100644
--- a/fs/f2fs/segment.h
+++ b/fs/f2fs/segment.h
@@ -96,25 +96,30 @@ static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi,
 #define GET_SUM_BLKOFF(sbi, segno) (segno % (sbi)->sums_per_block)
 #define SUM_BLK_PAGE_ADDR(sbi, folio, segno)	\
 	(folio_address(folio) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize)
+#define SUM_BLK_ENTRY_ADDR(sbi, entry, segno)	\
+	(cache_address(entry) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize)
 
 #define GET_SUM_TYPE(footer) ((footer)->entry_type)
 #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type))
 
 #define SIT_ENTRY_OFFSET(sit_i, segno)					\
 	((segno) % (sit_i)->sents_per_block)
-#define SIT_BLOCK_OFFSET(segno)					\
-	((segno) / SIT_ENTRY_PER_BLOCK)
-#define	START_SEGNO(segno)		\
-	(SIT_BLOCK_OFFSET(segno) * SIT_ENTRY_PER_BLOCK)
+#define SIT_BLOCK_OFFSET(sbi, segno)				\
+	((segno) / SIT_ENTRY_PER_BLOCK(sbi))
+static inline unsigned int
+f2fs_start_segno(struct f2fs_sb_info *sbi, unsigned int segno)
+{
+	return SIT_BLOCK_OFFSET(sbi, segno) * SIT_ENTRY_PER_BLOCK(sbi);
+}
 #define SIT_BLK_CNT(sbi)			\
-	DIV_ROUND_UP(MAIN_SEGS(sbi), SIT_ENTRY_PER_BLOCK)
+	DIV_ROUND_UP(MAIN_SEGS(sbi), SIT_ENTRY_PER_BLOCK(sbi))
 #define f2fs_bitmap_size(nr)			\
 	(BITS_TO_LONGS(nr) * sizeof(unsigned long))
 
-#define SECTOR_FROM_BLOCK(blk_addr)					\
-	(((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK)
-#define SECTOR_TO_BLOCK(sectors)					\
-	((sectors) >> F2FS_LOG_SECTORS_PER_BLOCK)
+#define SECTOR_FROM_BLOCK(sbi, blk_addr)				\
+	(((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK(sbi))
+#define SECTOR_TO_BLOCK(sbi, sectors)					\
+	((sectors) >> F2FS_LOG_SECTORS_PER_BLOCK(sbi))
 
 /*
  * In the victim_sel_policy->alloc_mode, there are three block allocation modes.
@@ -418,18 +423,18 @@ static inline void __seg_info_to_raw_sit(struct seg_entry *se,
 	rs->mtime = cpu_to_le64(se->mtime);
 }
 
-static inline void seg_info_to_sit_folio(struct f2fs_sb_info *sbi,
-				struct folio *folio, unsigned int start)
+static inline void seg_info_to_sit_block(struct f2fs_sb_info *sbi,
+			struct f2fs_cached_block *entry, unsigned int start)
 {
 	struct f2fs_sit_block *raw_sit;
 	struct seg_entry *se;
 	struct f2fs_sit_entry *rs;
-	unsigned int end = min(start + SIT_ENTRY_PER_BLOCK,
+	unsigned int end = min(start + SIT_ENTRY_PER_BLOCK(sbi),
 					(unsigned long)MAIN_SEGS(sbi));
 	int i;
 
-	raw_sit = folio_address(folio);
-	memset(raw_sit, 0, PAGE_SIZE);
+	raw_sit = cache_address(entry);
+	memset(raw_sit, 0, F2FS_BLKSIZE(sbi));
 	for (i = 0; i < end - start; i++) {
 		rs = &raw_sit->entries[i];
 		se = get_seg_entry(sbi, start + i);
@@ -651,15 +656,15 @@ static inline void get_additional_blocks_required(struct f2fs_sb_info *sbi,
  */
 static inline int __get_secs_required(struct f2fs_sb_info *sbi)
 {
-	unsigned int total_node_blocks = get_pages(sbi, F2FS_DIRTY_NODES) +
-					get_pages(sbi, F2FS_DIRTY_DENTS) +
-					get_pages(sbi, F2FS_DIRTY_IMETA);
-	unsigned int total_dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS);
+	unsigned int total_node_blocks = get_nr_caches(sbi, F2FS_DIRTY_NODES) +
+					get_nr_caches(sbi, F2FS_DIRTY_DENTS) +
+					get_nr_caches(sbi, F2FS_DIRTY_IMETA);
+	unsigned int total_dent_blocks = get_nr_caches(sbi, F2FS_DIRTY_DENTS);
 	unsigned int total_data_blocks = 0;
 	bool separate_dent = true;
 
 	if (f2fs_lfs_mode(sbi))
-		total_data_blocks = get_pages(sbi, F2FS_DIRTY_DATA);
+		total_data_blocks = get_nr_caches(sbi, F2FS_DIRTY_DATA);
 
 	/*
 	 * When active_logs != 4, dentry blocks and data blocks can be
@@ -869,7 +874,7 @@ static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi,
 						unsigned int start)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
-	unsigned int offset = SIT_BLOCK_OFFSET(start);
+	unsigned int offset = SIT_BLOCK_OFFSET(sbi, start);
 	block_t blk_addr = sit_i->sit_base_addr + offset;
 
 	f2fs_bug_on(sbi, !valid_main_segno(sbi, start));
@@ -894,9 +899,10 @@ static inline pgoff_t next_sit_addr(struct f2fs_sb_info *sbi,
 	return block_addr + sit_i->sit_base_addr;
 }
 
-static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start)
+static inline void set_to_next_sit(struct f2fs_sb_info *sbi,
+				   struct sit_info *sit_i, unsigned int start)
 {
-	unsigned int block_off = SIT_BLOCK_OFFSET(start);
+	unsigned int block_off = SIT_BLOCK_OFFSET(sbi, start);
 
 	f2fs_change_bit(block_off, sit_i->sit_bitmap);
 }
@@ -971,13 +977,13 @@ static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno)
 }
 
 /*
- * It is very important to gather dirty pages and write at once, so that we can
+ * It is very important to gather dirty blocks and write at once, so that we can
  * submit a big bio without interfering other data writes.
- * By default, 512 pages for directory data,
- * 512 pages (2MB) * 8 for nodes, and
- * 256 pages * 8 for meta are set.
+ * By default, 512 blocks for directory data,
+ * 512 blocks (2MB) * 8 for nodes, and
+ * 256 blocks * 8 for meta are set.
  */
-static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type)
+static inline int nr_caches_to_skip(struct f2fs_sb_info *sbi, int type)
 {
 	if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi))
 		return 0;
@@ -993,23 +999,24 @@ static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type)
 }
 
 /*
- * When writing pages, it'd better align nr_to_write for segment size.
+ * When writing cache asynchronously, align nr_to_write to BIO_MAX_VECS.
  */
-static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type,
-					struct writeback_control *wbc)
+static inline long adjust_flush_cache_number(struct f2fs_sb_info *sbi, int type)
 {
-	long nr_to_write, desired;
-
-	if (wbc->sync_mode != WB_SYNC_NONE)
+	long nr_to_write;
+
+	switch (type) {
+	case META:
+		nr_to_write = BIO_MAX_VECS;
+		break;
+	case NODE:
+		nr_to_write = BIO_MAX_VECS << 1;
+		break;
+	default:
+		f2fs_bug_on(sbi, 1);
 		return 0;
-
-	nr_to_write = wbc->nr_to_write;
-	desired = BIO_MAX_VECS;
-	if (type == NODE)
-		desired <<= 1;
-
-	wbc->nr_to_write = desired;
-	return desired - nr_to_write;
+	}
+	return nr_to_write;
 }
 
 static inline void wake_up_discard_thread(struct f2fs_sb_info *sbi, bool force)
diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c
index 4f6bf5926de4fb..29f48853149256 100644
--- a/fs/f2fs/shrinker.c
+++ b/fs/f2fs/shrinker.c
@@ -23,7 +23,7 @@ static unsigned long __count_nat_entries(struct f2fs_sb_info *sbi)
 
 static unsigned long __count_free_nids(struct f2fs_sb_info *sbi)
 {
-	long count = NM_I(sbi)->nid_cnt[FREE_NID] - MAX_FREE_NIDS;
+	long count = NM_I(sbi)->nid_cnt[FREE_NID] - MAX_FREE_NIDS(sbi);
 
 	return count > 0 ? count : 0;
 }
@@ -37,6 +37,13 @@ static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi,
 				atomic_read(&eti->total_ext_node);
 }
 
+static unsigned long __count_cache(struct f2fs_sb_info *sbi)
+{
+	return META_CACHE(sbi)->num_entries +
+		NODE_CACHE(sbi)->num_entries +
+		COMPRESS_CACHE(sbi)->num_entries;
+}
+
 unsigned long f2fs_shrink_count(struct shrinker *shrink,
 				struct shrink_control *sc)
 {
@@ -68,6 +75,9 @@ unsigned long f2fs_shrink_count(struct shrinker *shrink,
 		/* count free nids cache entries */
 		count += __count_free_nids(sbi);
 
+		/* count generic cache entries */
+		count += __count_cache(sbi);
+
 		spin_lock(&f2fs_list_lock);
 		p = p->next;
 		mutex_unlock(&sbi->umount_mutex);
@@ -120,6 +130,10 @@ unsigned long f2fs_shrink_scan(struct shrinker *shrink,
 		if (freed < nr)
 			freed += f2fs_try_to_free_nids(sbi, nr - freed);
 
+		/* shrink generic cache entries */
+		if (freed < nr)
+			freed += f2fs_shrink_cache(sbi, nr - freed);
+
 		spin_lock(&f2fs_list_lock);
 		p = p->next;
 		list_move_tail(&sbi->s_list, &f2fs_list);
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 1314b6ccced989..294f6f2c28a5f3 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -855,9 +855,11 @@ static int f2fs_parse_param(struct fs_context *fc, struct fs_parameter *param)
 		break;
 	case Opt_inline_xattr_size:
 		if (result.int_32 < MIN_INLINE_XATTR_SIZE ||
-			result.int_32 > MAX_INLINE_XATTR_SIZE) {
+			result.int_32 >
+			MAX_INLINE_XATTR_SIZE(F2FS_MAX_BLKSIZE)) {
 			f2fs_err(NULL, "inline xattr size is out of range: %u ~ %u",
-				 (u32)MIN_INLINE_XATTR_SIZE, (u32)MAX_INLINE_XATTR_SIZE);
+				 (u32)MIN_INLINE_XATTR_SIZE,
+				 (u32)MAX_INLINE_XATTR_SIZE(F2FS_MAX_BLKSIZE));
 			return -EINVAL;
 		}
 		ctx_set_opt(ctx, F2FS_MOUNT_INLINE_XATTR_SIZE);
@@ -1596,6 +1598,8 @@ static int f2fs_check_opt_consistency(struct fs_context *fc,
 	}
 
 	if (ctx_test_opt(ctx, F2FS_MOUNT_INLINE_XATTR_SIZE)) {
+		int min_size, max_size;
+
 		if (!f2fs_sb_has_extra_attr(sbi) ||
 			!f2fs_sb_has_flexible_inline_xattr(sbi)) {
 			f2fs_err(sbi, "extra_attr or flexible_inline_xattr feature is off");
@@ -1605,6 +1609,15 @@ static int f2fs_check_opt_consistency(struct fs_context *fc,
 			f2fs_err(sbi, "inline_xattr_size option should be set with inline_xattr option");
 			return -EINVAL;
 		}
+		min_size = MIN_INLINE_XATTR_SIZE;
+		max_size = MAX_INLINE_XATTR_SIZE(F2FS_BLKSIZE(sbi));
+
+		if (F2FS_OPTION(sbi).inline_xattr_size < min_size ||
+				F2FS_OPTION(sbi).inline_xattr_size > max_size) {
+			f2fs_err(sbi, "inline xattr size is out of range: %d ~ %d",
+				 min_size, max_size);
+			return -EINVAL;
+		}
 	}
 
 	if (ctx_test_opt(ctx, F2FS_MOUNT_ATGC) &&
@@ -1859,21 +1872,8 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb)
 
 static int f2fs_drop_inode(struct inode *inode)
 {
-	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	int ret;
 
-	/*
-	 * during filesystem shutdown, if checkpoint is disabled,
-	 * drop useless meta/node dirty pages.
-	 */
-	if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
-		if (inode->i_ino == F2FS_NODE_INO(sbi) ||
-			inode->i_ino == F2FS_META_INO(sbi)) {
-			trace_f2fs_drop_inode(inode, 1);
-			return 1;
-		}
-	}
-
 	/*
 	 * This is to avoid a deadlock condition like below.
 	 * writeback_single_inode(inode)
@@ -1894,7 +1894,7 @@ static int f2fs_drop_inode(struct inode *inode)
 			f2fs_i_size_write(inode, 0);
 
 			f2fs_submit_merged_write_cond(F2FS_I_SB(inode),
-					inode, NULL, 0, DATA);
+							inode, NULL);
 			truncate_inode_pages_final(inode->i_mapping);
 
 			if (F2FS_HAS_BLOCKS(inode))
@@ -1930,7 +1930,7 @@ int f2fs_inode_dirtied(struct inode *inode, bool sync)
 	if (sync && list_empty(&F2FS_I(inode)->gdirty_list)) {
 		list_add_tail(&F2FS_I(inode)->gdirty_list,
 				&sbi->inode_list[DIRTY_META]);
-		inc_page_count(sbi, F2FS_DIRTY_IMETA);
+		inc_cache_count(sbi, F2FS_DIRTY_IMETA);
 	}
 	spin_unlock(&sbi->inode_lock[DIRTY_META]);
 
@@ -1953,7 +1953,7 @@ void f2fs_inode_synced(struct inode *inode)
 	}
 	if (!list_empty(&F2FS_I(inode)->gdirty_list)) {
 		list_del_init(&F2FS_I(inode)->gdirty_list);
-		dec_page_count(sbi, F2FS_DIRTY_IMETA);
+		dec_cache_count(sbi, F2FS_DIRTY_IMETA);
 	}
 	clear_inode_flag(inode, FI_DIRTY_INODE);
 	clear_inode_flag(inode, FI_AUTO_RECOVER);
@@ -1968,12 +1968,6 @@ void f2fs_inode_synced(struct inode *inode)
  */
 static void f2fs_dirty_inode(struct inode *inode, int flags)
 {
-	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-
-	if (inode->i_ino == F2FS_NODE_INO(sbi) ||
-			inode->i_ino == F2FS_META_INO(sbi))
-		return;
-
 	if (is_inode_flag_set(inode, FI_AUTO_RECOVER))
 		clear_inode_flag(inode, FI_AUTO_RECOVER);
 
@@ -2026,6 +2020,7 @@ static void f2fs_put_super(struct super_block *sb)
 	 * flush all issued checkpoints and stop checkpoint issue thread.
 	 * after then, all checkpoints should be done by each process context.
 	 */
+	f2fs_stop_cache_wb_thread(sbi);
 	f2fs_stop_ckpt_thread(sbi);
 
 	/*
@@ -2064,30 +2059,27 @@ static void f2fs_put_super(struct super_block *sb)
 	/* our cp_error case, we can wait for any writeback page */
 	f2fs_flush_merged_writes(sbi);
 
-	f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
+	f2fs_sync_dirty_data(sbi, F2FS_WB_CP_DATA);
 
-	if (err || f2fs_cp_error(sbi)) {
-		truncate_inode_pages_final(NODE_MAPPING(sbi));
-		truncate_inode_pages_final(META_MAPPING(sbi));
+	if (err || f2fs_cp_error(sbi) ||
+		unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
+		f2fs_truncate_node_caches(sbi, 0, ULONG_MAX);
+		f2fs_truncate_meta_caches(sbi, 0, ULONG_MAX);
 	}
 
 	f2fs_bug_on(sbi, sbi->fsync_node_num);
 
-	f2fs_destroy_compress_inode(sbi);
-
-	iput(sbi->node_inode);
-	sbi->node_inode = NULL;
-
-	iput(sbi->meta_inode);
-	sbi->meta_inode = NULL;
+	f2fs_destroy_cache(COMPRESS_CACHE(sbi));
+	f2fs_destroy_cache(NODE_CACHE(sbi));
+	f2fs_destroy_cache(META_CACHE(sbi));
 
 	/* Should check the page counts after dropping all node/meta pages */
 	for (i = 0; i < NR_COUNT_TYPE; i++) {
-		if (!get_pages(sbi, i))
+		if (!get_nr_caches(sbi, i))
 			continue;
 		f2fs_err(sbi, "detect filesystem reference count leak during "
 			"umount, type: %d, count: %lld, err: %d, cp_err: %d",
-			i, get_pages(sbi, i), err, f2fs_cp_error(sbi));
+			i, get_nr_caches(sbi, i), err, f2fs_cp_error(sbi));
 		f2fs_bug_on(sbi, 1);
 	}
 
@@ -2701,9 +2693,9 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi)
 
 	f2fs_info(sbi, "%s: call sync_filesystem() to persist meta: %lld, node: %lld, data: %lld",
 			__func__,
-			get_pages(sbi, F2FS_DIRTY_META),
-			get_pages(sbi, F2FS_DIRTY_NODES),
-			get_pages(sbi, F2FS_DIRTY_DATA));
+			get_nr_caches(sbi, F2FS_DIRTY_META),
+			get_nr_caches(sbi, F2FS_DIRTY_NODES),
+			get_nr_caches(sbi, F2FS_DIRTY_DATA));
 
 	ret = sync_filesystem(sbi->sb);
 	if (ret || err) {
@@ -2720,9 +2712,9 @@ static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi)
 skip_gc:
 	f2fs_info(sbi, "%s: call f2fs_write_checkpoint(), meta: %lld, node: %lld, data: %lld",
 			__func__,
-			get_pages(sbi, F2FS_DIRTY_META),
-			get_pages(sbi, F2FS_DIRTY_NODES),
-			get_pages(sbi, F2FS_DIRTY_DATA));
+			get_nr_caches(sbi, F2FS_DIRTY_META),
+			get_nr_caches(sbi, F2FS_DIRTY_NODES),
+			get_nr_caches(sbi, F2FS_DIRTY_DATA));
 
 	f2fs_down_write_trace(&sbi->gc_lock, &lc);
 	cpc.reason = CP_PAUSE;
@@ -2754,9 +2746,9 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi)
 	long long skipped_write, dirty_data;
 
 	f2fs_info(sbi, "f2fs_enable_checkpoint() starts, meta: %lld, node: %lld, data: %lld",
-					get_pages(sbi, F2FS_DIRTY_META),
-					get_pages(sbi, F2FS_DIRTY_NODES),
-					get_pages(sbi, F2FS_DIRTY_DATA));
+					get_nr_caches(sbi, F2FS_DIRTY_META),
+					get_nr_caches(sbi, F2FS_DIRTY_NODES),
+					get_nr_caches(sbi, F2FS_DIRTY_DATA));
 
 	start = ktime_get();
 
@@ -2764,26 +2756,26 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi)
 
 	/* we should flush all the data to keep data consistency */
 	do {
-		skipped_write = get_pages(sbi, F2FS_SKIPPED_WRITE);
-		dirty_data = get_pages(sbi, F2FS_DIRTY_DATA);
+		skipped_write = get_nr_caches(sbi, F2FS_SKIPPED_WRITE);
+		dirty_data = get_nr_caches(sbi, F2FS_DIRTY_DATA);
 
 		sync_inodes_sb(sbi->sb);
 		f2fs_io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT);
 
 		f2fs_info(sbi, "sync_inode_sb done, dirty_data: %lld, %lld, "
 				"skipped write: %lld, %lld, retry: %d",
-				get_pages(sbi, F2FS_DIRTY_DATA),
+				get_nr_caches(sbi, F2FS_DIRTY_DATA),
 				dirty_data,
-				get_pages(sbi, F2FS_SKIPPED_WRITE),
+				get_nr_caches(sbi, F2FS_SKIPPED_WRITE),
 				skipped_write, retry);
 
 		/*
 		 * sync_inodes_sb() has retry logic, so let's check dirty_data
 		 * in prior to skipped_write in case there is no dirty data.
 		 */
-		if (!get_pages(sbi, F2FS_DIRTY_DATA))
+		if (!get_nr_caches(sbi, F2FS_DIRTY_DATA))
 			break;
-		if (get_pages(sbi, F2FS_SKIPPED_WRITE) == skipped_write)
+		if (get_nr_caches(sbi, F2FS_SKIPPED_WRITE) == skipped_write)
 			break;
 	} while (retry--);
 
@@ -2791,14 +2783,14 @@ static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi)
 
 	writeback = ktime_get();
 
-	if (unlikely(get_pages(sbi, F2FS_DIRTY_DATA) ||
-			get_pages(sbi, F2FS_SKIPPED_WRITE)))
+	if (unlikely(get_nr_caches(sbi, F2FS_DIRTY_DATA) ||
+			get_nr_caches(sbi, F2FS_SKIPPED_WRITE)))
 		f2fs_warn(sbi, "checkpoint=enable unwritten data: %lld, skipped data: %lld, retry: %d",
-				get_pages(sbi, F2FS_DIRTY_DATA),
-				get_pages(sbi, F2FS_SKIPPED_WRITE), retry);
+				get_nr_caches(sbi, F2FS_DIRTY_DATA),
+				get_nr_caches(sbi, F2FS_SKIPPED_WRITE), retry);
 
-	if (get_pages(sbi, F2FS_SKIPPED_WRITE))
-		atomic_set(&sbi->nr_pages[F2FS_SKIPPED_WRITE], 0);
+	if (get_nr_caches(sbi, F2FS_SKIPPED_WRITE))
+		atomic_set(&sbi->nr_caches[F2FS_SKIPPED_WRITE], 0);
 
 	f2fs_down_write_trace(&sbi->gc_lock, &lc);
 	f2fs_dirty_to_prefree(sbi);
@@ -2830,6 +2822,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 	unsigned int flags = fc->sb_flags;
 	int err;
 	bool need_restart_gc = false, need_stop_gc = false;
+	bool need_restart_wb = false, need_stop_wb = false;
 	bool need_restart_flush = false, need_stop_flush = false;
 	bool need_restart_discard = false, need_stop_discard = false;
 	bool need_enable_checkpoint = false, need_disable_checkpoint = false;
@@ -2864,7 +2857,8 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 			if (!org_mount_opt.s_qf_names[i]) {
 				for (j = 0; j < i; j++)
 					kfree(org_mount_opt.s_qf_names[j]);
-				return -ENOMEM;
+				err = -ENOMEM;
+				goto restore_holder;
 			}
 		} else {
 			org_mount_opt.s_qf_names[i] = NULL;
@@ -2988,6 +2982,18 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 		need_stop_gc = true;
 	}
 
+	if (flags & SB_RDONLY) {
+		if (sbi->cache_thread.cache_wb_task) {
+			f2fs_stop_cache_wb_thread(sbi);
+			need_restart_wb = true;
+		}
+	} else if (!sbi->cache_thread.cache_wb_task) {
+		err = f2fs_start_cache_wb_thread(sbi);
+		if (err)
+			goto restore_gc;
+		need_stop_wb = true;
+	}
+
 	if (flags & SB_RDONLY) {
 		sync_inodes_sb(sb);
 
@@ -2995,7 +3001,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 		set_sbi_flag(sbi, SBI_IS_CLOSE);
 		err = f2fs_sync_fs(sb, 1);
 		if (err)
-			goto restore_gc;
+			goto restore_wb;
 		clear_sbi_flag(sbi, SBI_IS_CLOSE);
 	}
 
@@ -3010,7 +3016,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 	} else {
 		err = f2fs_create_flush_cmd_control(sbi);
 		if (err)
-			goto restore_gc;
+			goto restore_wb;
 		need_stop_flush = true;
 	}
 
@@ -3107,6 +3113,13 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 		clear_opt(sbi, FLUSH_MERGE);
 		f2fs_destroy_flush_cmd_control(sbi, false);
 	}
+restore_wb:
+	if (need_restart_wb) {
+		if (f2fs_start_cache_wb_thread(sbi))
+			f2fs_warn(sbi, "background cache writeback thread has stopped");
+	} else if (need_stop_wb) {
+		f2fs_stop_cache_wb_thread(sbi);
+	}
 restore_gc:
 	if (need_restart_gc) {
 		if (f2fs_start_gc_thread(sbi))
@@ -3125,6 +3138,7 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb)
 	sbi->mount_opt = org_mount_opt;
 	sb->s_flags = old_sb_flags;
 
+restore_holder:
 	sbi->umount_lock_holder = NULL;
 	return err;
 }
@@ -3871,13 +3885,13 @@ static const struct export_operations f2fs_export_ops = {
 	.get_parent = f2fs_get_parent,
 };
 
-loff_t max_file_blocks(struct inode *inode)
+loff_t max_file_blocks(struct f2fs_sb_info *sbi, struct inode *inode)
 {
 	loff_t result = 0;
 	loff_t leaf_count;
 
 	/*
-	 * note: previously, result is equal to (DEF_ADDRS_PER_INODE -
+	 * note: previously, result is equal to (DEF_ADDRS_PER_INODE(sbi) -
 	 * DEFAULT_INLINE_XATTR_ADDRS), but now f2fs try to reserve more
 	 * space in inode.i_addr, it will be more safe to reassign
 	 * result as zero.
@@ -3886,17 +3900,17 @@ loff_t max_file_blocks(struct inode *inode)
 	if (inode && f2fs_compressed_file(inode))
 		leaf_count = ADDRS_PER_BLOCK(inode);
 	else
-		leaf_count = DEF_ADDRS_PER_BLOCK;
+		leaf_count = DEF_ADDRS_PER_BLOCK(sbi);
 
 	/* two direct node blocks */
 	result += (leaf_count * 2);
 
 	/* two indirect node blocks */
-	leaf_count *= NIDS_PER_BLOCK;
+	leaf_count *= NIDS_PER_BLOCK(sbi);
 	result += (leaf_count * 2);
 
 	/* one double indirect node block */
-	leaf_count *= NIDS_PER_BLOCK;
+	leaf_count *= NIDS_PER_BLOCK(sbi);
 	result += leaf_count;
 
 	/*
@@ -3905,7 +3919,8 @@ loff_t max_file_blocks(struct inode *inode)
 	 * fit within U32_MAX + 1 data units.
 	 */
 
-	result = umin(result, F2FS_BYTES_TO_BLK(((loff_t)U32_MAX + 1) * 4096));
+	result = umin(result, F2FS_BYTES_TO_BLK(sbi,
+						((loff_t)U32_MAX + 1) * 4096));
 
 	return result;
 }
@@ -3931,7 +3946,7 @@ static int __f2fs_commit_super(struct f2fs_sb_info *sbi, struct folio *folio,
 	bio = bio_alloc(sbi->sb->s_bdev, 1, opf, GFP_NOFS);
 
 	/* it doesn't need to set crypto context for superblock update */
-	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(folio->index);
+	bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, folio->index);
 
 	if (!bio_add_folio(bio, folio, folio_size(folio), 0))
 		f2fs_bug_on(sbi, 1);
@@ -4065,10 +4080,10 @@ static int sanity_check_raw_super(struct f2fs_sb_info *sbi,
 	}
 
 	/* only support block_size equals to PAGE_SIZE */
-	if (le32_to_cpu(raw_super->log_blocksize) != F2FS_BLKSIZE_BITS) {
+	if (le32_to_cpu(raw_super->log_blocksize) != PAGE_SHIFT) {
 		f2fs_info(sbi, "Invalid log_blocksize (%u), supports only %u",
 			  le32_to_cpu(raw_super->log_blocksize),
-			  F2FS_BLKSIZE_BITS);
+			  PAGE_SHIFT);
 		return -EFSCORRUPTED;
 	}
 
@@ -4329,7 +4344,7 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi)
 		return 1;
 	}
 
-	sit_blk_cnt = DIV_ROUND_UP(main_segs, SIT_ENTRY_PER_BLOCK);
+	sit_blk_cnt = DIV_ROUND_UP(main_segs, SIT_ENTRY_PER_BLOCK(sbi));
 	if (sit_bitmap_size * 8 < sit_blk_cnt) {
 		f2fs_err(sbi, "Wrong bitmap size: sit: %u, sit_blk_cnt:%u",
 			 sit_bitmap_size, sit_blk_cnt);
@@ -4357,7 +4372,7 @@ int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi)
 
 	nat_blocks = nat_segs << log_blocks_per_seg;
 	nat_bits_bytes = nat_blocks / BITS_PER_BYTE;
-	nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8);
+	nat_bits_blocks = F2FS_BLK_ALIGN(sbi, (nat_bits_bytes << 1) + 8);
 	if (__is_set_ckpt_flags(ckpt, CP_NAT_BITS_FLAG) &&
 		(cp_payload + F2FS_CP_PACKS +
 		NR_CURSEG_PERSIST_TYPE + nat_bits_blocks >= blocks_per_seg)) {
@@ -4382,6 +4397,22 @@ static void init_sb_info(struct f2fs_sb_info *sbi)
 		le32_to_cpu(raw_super->log_sectors_per_block);
 	sbi->log_blocksize = le32_to_cpu(raw_super->log_blocksize);
 	sbi->blocksize = BIT(sbi->log_blocksize);
+	sbi->nat_entries_per_block = sbi->blocksize /
+		sizeof(struct f2fs_nat_entry);
+	sbi->addrs_per_inode = F2FS_DEF_ADDRS_PER_INODE(sbi->blocksize);
+	sbi->addrs_per_block = (sbi->blocksize -
+		sizeof(struct node_footer)) / sizeof(__le32);
+	sbi->nids_per_block = sbi->addrs_per_block;
+	sbi->sit_entries_per_block = sbi->blocksize /
+		sizeof(struct f2fs_sit_entry);
+	sbi->orphans_per_block = (sbi->blocksize -
+		sizeof(struct f2fs_orphan_footer)) / sizeof(__le32);
+	sbi->dentries_per_block = (BITS_PER_BYTE * sbi->blocksize) /
+		((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1);
+	sbi->dentry_bitmap_size = DIV_ROUND_UP(sbi->dentries_per_block,
+		BITS_PER_BYTE);
+	sbi->dentry_reserved_size = sbi->blocksize - sbi->dentry_bitmap_size -
+		(SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * sbi->dentries_per_block;
 	sbi->log_blocks_per_seg = le32_to_cpu(raw_super->log_blocks_per_seg);
 	sbi->blocks_per_seg = BIT(sbi->log_blocks_per_seg);
 	sbi->segs_per_sec = le32_to_cpu(raw_super->segs_per_sec);
@@ -4389,12 +4420,10 @@ static void init_sb_info(struct f2fs_sb_info *sbi)
 	sbi->total_sections = le32_to_cpu(raw_super->section_count);
 	sbi->total_node_count = SEGS_TO_BLKS(sbi,
 			((le32_to_cpu(raw_super->segment_count_nat) / 2) *
-			NAT_ENTRY_PER_BLOCK));
+			NAT_ENTRY_PER_BLOCK(sbi)));
 	sbi->allocate_section_hint = le32_to_cpu(raw_super->section_count);
 	sbi->allocate_section_policy = ALLOCATE_FORWARD_NOHINT;
 	F2FS_ROOT_INO(sbi) = le32_to_cpu(raw_super->root_ino);
-	F2FS_NODE_INO(sbi) = le32_to_cpu(raw_super->node_ino);
-	F2FS_META_INO(sbi) = le32_to_cpu(raw_super->meta_ino);
 	sbi->cur_victim_sec = NULL_SECNO;
 	sbi->gc_mode = GC_NORMAL;
 	sbi->next_victim_seg[BG_GC] = NULL_SEGNO;
@@ -4436,7 +4465,7 @@ static void init_sb_info(struct f2fs_sb_info *sbi)
 	clear_sbi_flag(sbi, SBI_NEED_FSCK);
 
 	for (i = 0; i < NR_COUNT_TYPE; i++)
-		atomic_set(&sbi->nr_pages[i], 0);
+		atomic_set(&sbi->nr_caches[i], 0);
 
 	for (i = 0; i < META; i++)
 		atomic_set(&sbi->wb_sync_req[i], 0);
@@ -4490,7 +4519,7 @@ static int f2fs_report_zone_cb(struct blk_zone *zone, unsigned int idx,
 {
 	struct f2fs_report_zones_args *rz_args = data;
 	block_t unusable_blocks = (zone->len - zone->capacity) >>
-					F2FS_LOG_SECTORS_PER_BLOCK;
+					F2FS_LOG_SECTORS_PER_BLOCK(rz_args->sbi);
 
 	if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL)
 		return 0;
@@ -4533,10 +4562,10 @@ static int init_blkz_info(struct f2fs_sb_info *sbi, int devi)
 
 	zone_sectors = bdev_zone_sectors(bdev);
 	if (sbi->blocks_per_blkz && sbi->blocks_per_blkz !=
-				SECTOR_TO_BLOCK(zone_sectors))
+				SECTOR_TO_BLOCK(sbi, zone_sectors))
 		return -EINVAL;
-	sbi->blocks_per_blkz = SECTOR_TO_BLOCK(zone_sectors);
-	FDEV(devi).nr_blkz = div_u64(SECTOR_TO_BLOCK(nr_sectors),
+	sbi->blocks_per_blkz = SECTOR_TO_BLOCK(sbi, zone_sectors);
+	FDEV(devi).nr_blkz = div_u64(SECTOR_TO_BLOCK(sbi, nr_sectors),
 					sbi->blocks_per_blkz);
 	if (nr_sectors & (zone_sectors - 1))
 		FDEV(devi).nr_blkz++;
@@ -5045,7 +5074,7 @@ static void f2fs_restore_device_alias(struct f2fs_sb_info *sbi)
 {
 	struct inode *root = d_inode(sbi->sb->s_root);
 	struct f2fs_dir_entry *de;
-	struct folio *folio;
+	void *dentry_block = NULL;
 	int i;
 
 	if (!f2fs_sb_has_device_alias(sbi))
@@ -5060,7 +5089,7 @@ static void f2fs_restore_device_alias(struct f2fs_sb_info *sbi)
 		qstr.name = name;
 		qstr.len = strlen(name);
 
-		de = f2fs_find_entry(root, &qstr, &folio);
+		de = f2fs_find_entry(root, &qstr, &dentry_block);
 		if (!de)
 			continue;
 
@@ -5070,7 +5099,7 @@ static void f2fs_restore_device_alias(struct f2fs_sb_info *sbi)
 				FDEV(i).has_alias = true;
 			iput(inode);
 		}
-		f2fs_folio_put(folio, 0);
+		f2fs_put_dentry_block(dentry_block, false);
 	}
 }
 
@@ -5125,12 +5154,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 	}
 	mutex_init(&sbi->flush_lock);
 
-	/* set a block size */
-	if (unlikely(!sb_set_blocksize(sb, F2FS_BLKSIZE))) {
-		f2fs_err(sbi, "unable to set blocksize");
-		goto free_sbi;
-	}
-
 	err = read_raw_super_block(sbi, &raw_super, &valid_super_block,
 								&recovery);
 	if (err)
@@ -5138,6 +5161,14 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 
 	sb->s_fs_info = sbi;
 	sbi->raw_super = raw_super;
+	init_sb_info(sbi);
+
+	/* set a block size */
+	if (unlikely(!sb_set_blocksize(sb, sbi->blocksize))) {
+		f2fs_err(sbi, "unable to set blocksize %u", sbi->blocksize);
+		err = -EINVAL;
+		goto free_sb_buf;
+	}
 	sbi->max_atc_write_bio_size = UINT_MAX;
 
 	INIT_WORK(&sbi->s_error_work, f2fs_record_error_work);
@@ -5161,7 +5192,7 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 	if (err)
 		goto free_options;
 
-	sb->s_maxbytes = max_file_blocks(NULL) <<
+	sb->s_maxbytes = max_file_blocks(sbi, NULL) <<
 				le32_to_cpu(raw_super->log_blocksize);
 	sb->s_max_links = F2FS_LINK_MAX;
 
@@ -5217,8 +5248,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 	if (err)
 		goto free_bio_info;
 
-	init_sb_info(sbi);
-
 	err = f2fs_init_iostat(sbi);
 	if (err)
 		goto free_bio_info;
@@ -5231,18 +5260,14 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 	if (err)
 		goto free_percpu;
 
-	/* get an inode for meta space */
-	sbi->meta_inode = f2fs_iget(sb, F2FS_META_INO(sbi));
-	if (IS_ERR(sbi->meta_inode)) {
-		f2fs_err(sbi, "Failed to read F2FS meta data inode");
-		err = PTR_ERR(sbi->meta_inode);
-		goto free_page_array_cache;
-	}
+	f2fs_init_cache(sbi, META_CACHE(sbi), F2FS_META_CACHE);
+	f2fs_init_cache(sbi, NODE_CACHE(sbi), F2FS_NODE_CACHE);
+	f2fs_init_cache(sbi, COMPRESS_CACHE(sbi), F2FS_COMPRESS_CACHE);
 
 	err = f2fs_get_valid_checkpoint(sbi);
 	if (err) {
 		f2fs_err(sbi, "Failed to get valid F2FS checkpoint");
-		goto free_meta_inode;
+		goto free_compress_cache;
 	}
 
 	if (__is_set_ckpt_flags(F2FS_CKPT(sbi), CP_QUOTA_NEED_FSCK_FLAG))
@@ -5288,6 +5313,8 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 
 	f2fs_init_fsync_node_info(sbi);
 
+	f2fs_init_compress_cache_context(sbi);
+
 	/* setup checkpoint request control and start checkpoint issue thread */
 	f2fs_init_ckpt_req_control(sbi);
 	if (!f2fs_readonly(sb) && !test_opt(sbi, DISABLE_CHECKPOINT) &&
@@ -5339,42 +5366,30 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 	if (err)
 		goto free_nm;
 
-	/* get an inode for node space */
-	sbi->node_inode = f2fs_iget(sb, F2FS_NODE_INO(sbi));
-	if (IS_ERR(sbi->node_inode)) {
-		f2fs_err(sbi, "Failed to read node inode");
-		err = PTR_ERR(sbi->node_inode);
-		goto free_stats;
-	}
-
 	/* read root inode and dentry */
 	root = f2fs_iget(sb, F2FS_ROOT_INO(sbi));
 	if (IS_ERR(root)) {
 		f2fs_err(sbi, "Failed to read root inode");
 		err = PTR_ERR(root);
-		goto free_node_inode;
+		goto free_ino_entry;
 	}
 	if (!S_ISDIR(root->i_mode) || !root->i_blocks ||
 			!root->i_size || !root->i_nlink) {
 		iput(root);
 		err = -EINVAL;
-		goto free_node_inode;
+		goto free_ino_entry;
 	}
 
 	generic_set_sb_d_ops(sb);
 	sb->s_root = d_make_root(root); /* allocate root dentry */
 	if (!sb->s_root) {
 		err = -ENOMEM;
-		goto free_node_inode;
+		goto free_ino_entry;
 	}
 
-	err = f2fs_init_compress_inode(sbi);
-	if (err)
-		goto free_root_inode;
-
 	err = f2fs_register_sysfs(sbi);
 	if (err)
-		goto free_compress_inode;
+		goto free_root_inode;
 
 	sbi->umount_lock_holder = current;
 #ifdef CONFIG_QUOTA
@@ -5491,6 +5506,12 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 			goto sync_free_meta;
 	}
 
+	if (!f2fs_readonly(sb)) {
+		err = f2fs_start_cache_wb_thread(sbi);
+		if (err)
+			goto stop_gc_thread;
+	}
+
 	/* recover broken superblock */
 	if (recovery) {
 		err = f2fs_commit_super(sbi, true);
@@ -5513,6 +5534,8 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 	sbi->umount_lock_holder = NULL;
 	return 0;
 
+stop_gc_thread:
+	f2fs_stop_gc_thread(sbi);
 sync_free_meta:
 	/* safe to flush all the data */
 	sync_filesystem(sbi->sb);
@@ -5528,23 +5551,18 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 	 * Some dirty meta pages can be produced by f2fs_recover_orphan_inodes()
 	 * failed by EIO. Then, iput(node_inode) can trigger balance_fs_bg()
 	 * followed by f2fs_write_checkpoint() through f2fs_write_node_pages(), which
-	 * falls into an infinite loop in f2fs_sync_meta_pages().
+	 * falls into an infinite loop in f2fs_sync_meta_caches().
 	 */
-	truncate_inode_pages_final(META_MAPPING(sbi));
+	f2fs_truncate_meta_caches(sbi, 0, ULONG_MAX);
 	/* evict some inodes being cached by GC */
 	evict_inodes(sb);
 	f2fs_unregister_sysfs(sbi);
-free_compress_inode:
-	f2fs_destroy_compress_inode(sbi);
 free_root_inode:
 	dput(sb->s_root);
 	sb->s_root = NULL;
-free_node_inode:
+free_ino_entry:
 	f2fs_release_ino_entry(sbi, true);
-	truncate_inode_pages_final(NODE_MAPPING(sbi));
-	iput(sbi->node_inode);
-	sbi->node_inode = NULL;
-free_stats:
+	f2fs_truncate_node_caches(sbi, 0, ULONG_MAX);
 	f2fs_destroy_stats(sbi);
 free_nm:
 	/* stop discard thread before destroying node manager */
@@ -5560,11 +5578,10 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 free_devices:
 	destroy_device_list(sbi);
 	kvfree(sbi->ckpt);
-free_meta_inode:
-	make_bad_inode(sbi->meta_inode);
-	iput(sbi->meta_inode);
-	sbi->meta_inode = NULL;
-free_page_array_cache:
+free_compress_cache:
+	f2fs_destroy_cache(COMPRESS_CACHE(sbi));
+	f2fs_destroy_cache(NODE_CACHE(sbi));
+	f2fs_destroy_cache(META_CACHE(sbi));
 	f2fs_destroy_page_array_cache(sbi);
 free_percpu:
 	destroy_percpu_info(sbi);
@@ -5653,7 +5670,8 @@ static void kill_f2fs_super(struct super_block *sb)
 		 * compress inode cache.
 		 */
 		if (test_opt(sbi, COMPRESS_CACHE))
-			truncate_inode_pages_final(COMPRESS_MAPPING(sbi));
+			f2fs_invalidate_compress_pages_range(sbi,
+					0, UINT_MAX);
 #endif
 
 		if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) ||
diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c
index 811e350a143020..9749da70089a61 100644
--- a/fs/f2fs/sysfs.c
+++ b/fs/f2fs/sysfs.c
@@ -40,6 +40,7 @@ enum {
 	RESERVED_BLOCKS,	/* struct f2fs_sb_info */
 	CPRC_INFO,	/* struct ckpt_req_control */
 	ATGC_INFO,	/* struct atgc_management */
+	WB_THREAD,	/* struct f2fs_cache_kthread */
 };
 
 static const char *gc_mode_names[MAX_GC_MODE] = {
@@ -98,6 +99,8 @@ static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type)
 		return (unsigned char *)&sbi->cprc_info;
 	else if (struct_type == ATGC_INFO)
 		return (unsigned char *)&sbi->am;
+	else if (struct_type == WB_THREAD)
+		return (unsigned char *)&sbi->cache_thread;
 	return NULL;
 }
 
@@ -995,6 +998,13 @@ static ssize_t __sbi_store(struct f2fs_attr *a,
 		return count;
 	}
 
+	if (!strcmp(a->attr.name, "cache_wb_interval")) {
+		if (t < MIN_DIRTY_CACHE_TIMEOUT || t > MAX_DIRTY_CACHE_TIMEOUT)
+			return -EINVAL;
+		sbi->cache_thread.cache_wb_interval = t;
+		return count;
+	}
+
 	__sbi_store_value(a, sbi, ptr + a->offset, t);
 
 	return count;
@@ -1008,7 +1018,8 @@ static ssize_t f2fs_sbi_store(struct f2fs_attr *a,
 	bool gc_entry = (!strcmp(a->attr.name, "gc_urgent") ||
 					a->struct_type == GC_THREAD);
 	bool thread_entry = !strcmp(a->attr.name, "ckpt_thread_ioprio") ||
-			!strcmp(a->attr.name, "critical_task_priority");
+			!strcmp(a->attr.name, "critical_task_priority") ||
+			!strcmp(a->attr.name, "cache_wb_interval");
 
 	if (gc_entry || thread_entry) {
 		if (!down_read_trylock(&sbi->sb->s_umount))
@@ -1218,6 +1229,9 @@ static struct f2fs_attr f2fs_attr_##name = __ATTR(name, 0444, name##_show, NULL)
 #define ATGC_INFO_RW_ATTR(name, elname)				\
 	F2FS_RW_ATTR(ATGC_INFO, atgc_management, name, elname)
 
+#define WB_THREAD_RW_ATTR(name, elname)				\
+	F2FS_RW_ATTR(WB_THREAD, f2fs_cache_kthread, name, elname)
+
 /* GC_THREAD ATTR */
 GC_THREAD_RW_ATTR(gc_urgent_sleep_time, urgent_sleep_time);
 GC_THREAD_RW_ATTR(gc_min_sleep_time, min_sleep_time);
@@ -1254,7 +1268,7 @@ DCC_INFO_GENERAL_RW_ATTR(discard_io_aware);
 /* NM_INFO ATTR */
 NM_INFO_RW_ATTR(max_roll_forward_node_blocks, max_rf_node_blocks);
 NM_INFO_GENERAL_RW_ATTR(ram_thresh);
-NM_INFO_GENERAL_RW_ATTR(ra_nid_pages);
+NM_INFO_RW_ATTR(ra_nid_pages, ra_nid_blocks);
 NM_INFO_GENERAL_RW_ATTR(dirty_nats_ratio);
 
 /* F2FS_SBI ATTR */
@@ -1347,6 +1361,9 @@ ATGC_INFO_RW_ATTR(atgc_candidate_count, max_candidate_count);
 ATGC_INFO_RW_ATTR(atgc_age_weight, age_weight);
 ATGC_INFO_RW_ATTR(atgc_age_threshold, age_threshold);
 
+/* WB_THREAD ATTR */
+WB_THREAD_RW_ATTR(cache_wb_interval, cache_wb_interval);
+
 F2FS_GENERAL_RO_ATTR(dirty_segments);
 F2FS_GENERAL_RO_ATTR(free_segments);
 F2FS_GENERAL_RO_ATTR(ovp_segments);
@@ -1533,6 +1550,7 @@ static struct attribute *f2fs_attrs[] = {
 	ATTR_LIST(lock_duration_priority),
 	ATTR_LIST(adjust_lock_priority),
 	ATTR_LIST(critical_task_priority),
+	ATTR_LIST(cache_wb_interval),
 	NULL,
 };
 ATTRIBUTE_GROUPS(f2fs);
@@ -1871,8 +1889,8 @@ static int __maybe_unused disk_map_seq_show(struct seq_file *seq,
 	struct f2fs_sb_info *sbi = F2FS_SB(sb);
 	int i;
 
-	seq_printf(seq, "Address Layout   : %5luB Block address (# of Segments)\n",
-					F2FS_BLKSIZE);
+	seq_printf(seq, "Address Layout   : %5uB Block address (# of Segments)\n",
+		   F2FS_BLKSIZE(sbi));
 	seq_printf(seq, " SB            : %12s\n", "0/1024B");
 	seq_printf(seq, " seg0_blkaddr  : 0x%010x\n", SEG0_BLKADDR(sbi));
 	seq_printf(seq, " Checkpoint    : 0x%010x (%10d)\n",
@@ -1889,13 +1907,13 @@ static int __maybe_unused disk_map_seq_show(struct seq_file *seq,
 	seq_printf(seq, " Main          : 0x%010x (%10d)\n",
 			SM_I(sbi)->main_blkaddr,
 			le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_main));
-	seq_printf(seq, " Block size    : %12lu KB\n", F2FS_BLKSIZE >> 10);
+	seq_printf(seq, " Block size    : %12u KB\n", F2FS_BLKSIZE(sbi) >> 10);
 	seq_printf(seq, " Segment size  : %12d MB\n",
-			(BLKS_PER_SEG(sbi) << (F2FS_BLKSIZE_BITS - 10)) >> 10);
+			(BLKS_PER_SEG(sbi) << (F2FS_BLKSIZE_BITS(sbi) - 10)) >> 10);
 	seq_printf(seq, " Segs/Sections : %12d\n",
 			SEGS_PER_SEC(sbi));
 	seq_printf(seq, " Section size  : %12d MB\n",
-			(BLKS_PER_SEC(sbi) << (F2FS_BLKSIZE_BITS - 10)) >> 10);
+			(BLKS_PER_SEC(sbi) << (F2FS_BLKSIZE_BITS(sbi) - 10)) >> 10);
 	seq_printf(seq, " # of Sections : %12d\n",
 			le32_to_cpu(F2FS_RAW_SUPER(sbi)->section_count));
 
diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c
index 39f4825154452a..ee838f880c4a47 100644
--- a/fs/f2fs/verity.c
+++ b/fs/f2fs/verity.c
@@ -75,7 +75,8 @@ static int pagecache_write(struct inode *inode, const void *buf, size_t count,
 	struct address_space *mapping = inode->i_mapping;
 	const struct address_space_operations *aops = mapping->a_ops;
 
-	if (pos + count > F2FS_BLK_TO_BYTES(max_file_blocks(inode)))
+	if (pos + count > F2FS_BLK_TO_BYTES(F2FS_I_SB(inode),
+			max_file_blocks(F2FS_I_SB(inode), inode)))
 		return -EFBIG;
 
 	while (count) {
@@ -239,7 +240,8 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf,
 
 	/* Get the descriptor */
 	if (pos + size < pos ||
-	    pos + size > F2FS_BLK_TO_BYTES(max_file_blocks(inode)) ||
+	    pos + size > F2FS_BLK_TO_BYTES(F2FS_I_SB(inode),
+			max_file_blocks(F2FS_I_SB(inode), inode)) ||
 	    pos < f2fs_verity_metadata_pos(inode) || size > INT_MAX) {
 		f2fs_warn(F2FS_I_SB(inode), "invalid verity xattr");
 		f2fs_handle_error(F2FS_I_SB(inode),
diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c
index 6728d1488cadb9..0ed879fc307657 100644
--- a/fs/f2fs/xattr.c
+++ b/fs/f2fs/xattr.c
@@ -67,7 +67,7 @@ static int f2fs_xattr_generic_get(const struct xattr_handler *handler,
 }
 
 static int f2fs_xattr_generic_set(const struct xattr_handler *handler,
-		struct mnt_idmap *idmap,
+		const struct mnt_idmap *idmap,
 		struct dentry *unused, struct inode *inode,
 		const char *name, const void *value,
 		size_t size, int flags)
@@ -111,7 +111,7 @@ static int f2fs_xattr_advise_get(const struct xattr_handler *handler,
 }
 
 static int f2fs_xattr_advise_set(const struct xattr_handler *handler,
-		struct mnt_idmap *idmap,
+		const struct mnt_idmap *idmap,
 		struct dentry *unused, struct inode *inode,
 		const char *name, const void *value,
 		size_t size, int flags)
@@ -138,7 +138,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler,
 
 #ifdef CONFIG_F2FS_FS_SECURITY
 static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array,
-		void *folio)
+		void *fs_data)
 {
 	const struct xattr *xattr;
 	int err = 0;
@@ -146,7 +146,7 @@ static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array,
 	for (xattr = xattr_array; xattr->name != NULL; xattr++) {
 		err = f2fs_setxattr(inode, F2FS_XATTR_INDEX_SECURITY,
 				xattr->name, xattr->value,
-				xattr->value_len, folio, 0);
+				xattr->value_len, fs_data, 0);
 		if (err < 0)
 			break;
 	}
@@ -154,10 +154,10 @@ static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array,
 }
 
 int f2fs_init_security(struct inode *inode, struct inode *dir,
-				const struct qstr *qstr, struct folio *ifolio)
+				const struct qstr *qstr, struct f2fs_cached_block *ientry)
 {
 	return security_inode_init_security(inode, dir, qstr,
-				f2fs_initxattrs, ifolio);
+				f2fs_initxattrs, ientry);
 }
 #endif
 
@@ -273,25 +273,25 @@ static struct f2fs_xattr_entry *__find_inline_xattr(struct inode *inode,
 	return entry;
 }
 
-static int read_inline_xattr(struct inode *inode, struct folio *ifolio,
+static int read_inline_xattr(struct inode *inode, struct f2fs_cached_block *ientry,
 							void *txattr_addr)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	unsigned int inline_size = inline_xattr_size(inode);
-	struct folio *folio = NULL;
+	struct f2fs_cached_block *in_entry = NULL;
 	void *inline_addr;
 
-	if (ifolio) {
-		inline_addr = inline_xattr_addr(inode, ifolio);
+	if (ientry) {
+		inline_addr = inline_xattr_addr(inode, ientry);
 	} else {
-		folio = f2fs_get_inode_folio(sbi, inode->i_ino);
-		if (IS_ERR(folio))
-			return PTR_ERR(folio);
+		in_entry = f2fs_get_inode_cache(sbi, inode->i_ino);
+		if (IS_ERR(in_entry))
+			return PTR_ERR(in_entry);
 
-		inline_addr = inline_xattr_addr(inode, folio);
+		inline_addr = inline_xattr_addr(inode, in_entry);
 	}
 	memcpy(txattr_addr, inline_addr, inline_size);
-	f2fs_folio_put(folio, true);
+	f2fs_put_cache(in_entry, true);
 
 	return 0;
 }
@@ -301,22 +301,23 @@ static int read_xattr_block(struct inode *inode, void *txattr_addr)
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	nid_t xnid = F2FS_I(inode)->i_xattr_nid;
 	unsigned int inline_size = inline_xattr_size(inode);
-	struct folio *xfolio;
+	struct f2fs_cached_block *xentry;
 	void *xattr_addr;
 
 	/* The inode already has an extended attribute block. */
-	xfolio = f2fs_get_xnode_folio(sbi, xnid);
-	if (IS_ERR(xfolio))
-		return PTR_ERR(xfolio);
+	xentry = f2fs_get_xnode_cache(sbi, xnid);
+	if (IS_ERR(xentry))
+		return PTR_ERR(xentry);
 
-	xattr_addr = folio_address(xfolio);
-	memcpy(txattr_addr + inline_size, xattr_addr, VALID_XATTR_BLOCK_SIZE);
-	f2fs_folio_put(xfolio, true);
+	xattr_addr = cache_address(xentry);
+	memcpy(txattr_addr + inline_size, xattr_addr,
+	       VALID_XATTR_BLOCK_SIZE(inode));
+	f2fs_put_cache(xentry, true);
 
 	return 0;
 }
 
-static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio,
+static int lookup_all_xattrs(struct inode *inode, struct f2fs_cached_block *ientry,
 				unsigned int index, unsigned int len,
 				const char *name, struct f2fs_xattr_entry **xe,
 				void **base_addr, int *base_size,
@@ -340,7 +341,7 @@ static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio,
 
 	/* read from inline xattr */
 	if (inline_size) {
-		err = read_inline_xattr(inode, ifolio, txattr_addr);
+		err = read_inline_xattr(inode, ientry, txattr_addr);
 		if (err)
 			goto out;
 
@@ -388,12 +389,12 @@ static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio,
 	return err;
 }
 
-static int read_all_xattrs(struct inode *inode, struct folio *ifolio,
+static int read_all_xattrs(struct inode *inode, struct f2fs_cached_block *ientry,
 							void **base_addr)
 {
 	struct f2fs_xattr_header *header;
 	nid_t xnid = F2FS_I(inode)->i_xattr_nid;
-	unsigned int size = VALID_XATTR_BLOCK_SIZE;
+	unsigned int size = VALID_XATTR_BLOCK_SIZE(inode);
 	unsigned int inline_size = inline_xattr_size(inode);
 	void *txattr_addr;
 	int err;
@@ -405,7 +406,7 @@ static int read_all_xattrs(struct inode *inode, struct folio *ifolio,
 
 	/* read from inline xattr */
 	if (inline_size) {
-		err = read_inline_xattr(inode, ifolio, txattr_addr);
+		err = read_inline_xattr(inode, ientry, txattr_addr);
 		if (err)
 			goto fail;
 	}
@@ -432,14 +433,14 @@ static int read_all_xattrs(struct inode *inode, struct folio *ifolio,
 }
 
 static inline int write_all_xattrs(struct inode *inode, __u32 hsize,
-				void *txattr_addr, struct folio *ifolio)
+				void *txattr_addr, struct f2fs_cached_block *ientry)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	size_t inline_size = inline_xattr_size(inode);
-	struct folio *in_folio = NULL;
+	struct f2fs_cached_block *in_entry = NULL;
 	void *xattr_addr;
 	void *inline_addr = NULL;
-	struct folio *xfolio;
+	struct f2fs_cached_block *xentry;
 	nid_t new_nid = 0;
 	int err = 0;
 
@@ -449,75 +450,75 @@ static inline int write_all_xattrs(struct inode *inode, __u32 hsize,
 
 	/* write to inline xattr */
 	if (inline_size) {
-		if (ifolio) {
-			inline_addr = inline_xattr_addr(inode, ifolio);
+		if (ientry) {
+			inline_addr = inline_xattr_addr(inode, ientry);
 		} else {
-			in_folio = f2fs_get_inode_folio(sbi, inode->i_ino);
-			if (IS_ERR(in_folio)) {
+			in_entry = f2fs_get_inode_cache(sbi, inode->i_ino);
+			if (IS_ERR(in_entry)) {
 				f2fs_alloc_nid_failed(sbi, new_nid);
-				return PTR_ERR(in_folio);
+				return PTR_ERR(in_entry);
 			}
-			inline_addr = inline_xattr_addr(inode, in_folio);
+			inline_addr = inline_xattr_addr(inode, in_entry);
 		}
 
-		f2fs_folio_wait_writeback(ifolio ? ifolio : in_folio,
-							NODE, true, true);
+		f2fs_cache_wait_writeback(ientry ? ientry : in_entry);
 		/* no need to use xattr node block */
 		if (hsize <= inline_size) {
 			err = f2fs_truncate_xattr_node(inode);
 			f2fs_alloc_nid_failed(sbi, new_nid);
 			if (err) {
-				f2fs_folio_put(in_folio, true);
+				f2fs_put_cache(in_entry, true);
 				return err;
 			}
 			memcpy(inline_addr, txattr_addr, inline_size);
-			folio_mark_dirty(ifolio ? ifolio : in_folio);
+			f2fs_mark_cache_dirty(ientry ? ientry : in_entry);
 			goto in_page_out;
 		}
 	}
 
 	/* write to xattr node block */
 	if (F2FS_I(inode)->i_xattr_nid) {
-		xfolio = f2fs_get_xnode_folio(sbi, F2FS_I(inode)->i_xattr_nid);
-		if (IS_ERR(xfolio)) {
-			err = PTR_ERR(xfolio);
+		xentry = f2fs_get_xnode_cache(sbi, F2FS_I(inode)->i_xattr_nid);
+		if (IS_ERR(xentry)) {
+			err = PTR_ERR(xentry);
 			f2fs_alloc_nid_failed(sbi, new_nid);
 			goto in_page_out;
 		}
 		f2fs_bug_on(sbi, new_nid);
-		f2fs_folio_wait_writeback(xfolio, NODE, true, true);
+		f2fs_cache_wait_writeback(xentry);
 	} else {
 		struct dnode_of_data dn;
 
 		set_new_dnode(&dn, inode, NULL, NULL, new_nid);
-		xfolio = f2fs_new_node_folio(&dn, XATTR_NODE_OFFSET);
-		if (IS_ERR(xfolio)) {
-			err = PTR_ERR(xfolio);
+		xentry = f2fs_new_node_cache(&dn, XATTR_NODE_OFFSET);
+		if (IS_ERR(xentry)) {
+			err = PTR_ERR(xentry);
 			f2fs_alloc_nid_failed(sbi, new_nid);
 			goto in_page_out;
 		}
 		f2fs_alloc_nid_done(sbi, new_nid);
 	}
-	xattr_addr = folio_address(xfolio);
+	xattr_addr = cache_address(xentry);
 
 	if (inline_size)
 		memcpy(inline_addr, txattr_addr, inline_size);
-	memcpy(xattr_addr, txattr_addr + inline_size, VALID_XATTR_BLOCK_SIZE);
+	memcpy(xattr_addr, txattr_addr + inline_size,
+	       VALID_XATTR_BLOCK_SIZE(inode));
 
 	if (inline_size)
-		folio_mark_dirty(ifolio ? ifolio : in_folio);
-	folio_mark_dirty(xfolio);
+		f2fs_mark_cache_dirty(ientry ? ientry : in_entry);
+	f2fs_mark_cache_dirty(xentry);
 
-	f2fs_folio_put(xfolio, true);
+	f2fs_put_cache(xentry, true);
 in_page_out:
-	f2fs_folio_put(in_folio, true);
+	f2fs_put_cache(in_entry, true);
 	return err;
 }
 
 int f2fs_getxattr(struct inode *inode, int index, const char *name,
-		void *buffer, size_t buffer_size, struct folio *ifolio)
+		void *buffer, size_t buffer_size, struct f2fs_cached_block *ientry)
 {
-	struct f2fs_xattr_entry *entry = NULL;
+	struct f2fs_xattr_entry *xe = NULL;
 	int error;
 	unsigned int size, len;
 	void *base_addr = NULL;
@@ -531,16 +532,16 @@ int f2fs_getxattr(struct inode *inode, int index, const char *name,
 	if (len > F2FS_NAME_LEN)
 		return -ERANGE;
 
-	if (!ifolio)
+	if (!ientry)
 		f2fs_down_read(&F2FS_I(inode)->i_xattr_sem);
-	error = lookup_all_xattrs(inode, ifolio, index, len, name,
-				&entry, &base_addr, &base_size, &is_inline);
-	if (!ifolio)
+	error = lookup_all_xattrs(inode, ientry, index, len, name,
+				&xe, &base_addr, &base_size, &is_inline);
+	if (!ientry)
 		f2fs_up_read(&F2FS_I(inode)->i_xattr_sem);
 	if (error)
 		return error;
 
-	size = le16_to_cpu(entry->e_value_size);
+	size = le16_to_cpu(xe->e_value_size);
 
 	if (buffer && size > buffer_size) {
 		error = -ERANGE;
@@ -548,7 +549,7 @@ int f2fs_getxattr(struct inode *inode, int index, const char *name,
 	}
 
 	if (buffer) {
-		char *pval = entry->e_name + entry->e_name_len;
+		char *pval = xe->e_name + xe->e_name_len;
 
 		if (base_size - (pval - (char *)base_addr) < size) {
 			error = -ERANGE;
@@ -632,7 +633,7 @@ static bool f2fs_xattr_value_same(struct f2fs_xattr_entry *entry,
 
 static int __f2fs_setxattr(struct inode *inode, int index,
 			const char *name, const void *value, size_t size,
-			struct folio *ifolio, int flags)
+			struct f2fs_cached_block *ientry, int flags)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_xattr_entry *here, *last;
@@ -656,7 +657,7 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 	if (size > MAX_VALUE_LEN(inode))
 		return -E2BIG;
 retry:
-	error = read_all_xattrs(inode, ifolio, &base_addr);
+	error = read_all_xattrs(inode, ientry, &base_addr);
 	if (error)
 		return error;
 
@@ -773,7 +774,7 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 		*(u32 *)((u8 *)last + newsize) = 0;
 	}
 
-	error = write_all_xattrs(inode, new_hsize, base_addr, ifolio);
+	error = write_all_xattrs(inode, new_hsize, base_addr, ientry);
 	if (error)
 		goto exit;
 
@@ -807,7 +808,7 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 
 int f2fs_setxattr(struct inode *inode, int index, const char *name,
 				const void *value, size_t size,
-				struct folio *ifolio, int flags)
+				struct f2fs_cached_block *ientry, int flags)
 {
 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 	struct f2fs_lock_context lc;
@@ -823,9 +824,9 @@ int f2fs_setxattr(struct inode *inode, int index, const char *name,
 		return err;
 
 	/* this case is only from f2fs_init_inode_metadata */
-	if (ifolio)
+	if (ientry)
 		return __f2fs_setxattr(inode, index, name, value,
-						size, ifolio, flags);
+						size, ientry, flags);
 	f2fs_balance_fs(sbi, true);
 
 	f2fs_lock_op(sbi, &lc);
@@ -848,4 +849,4 @@ int __init f2fs_init_xattr_cache(void)
 void f2fs_destroy_xattr_cache(void)
 {
 	kmem_cache_destroy(inline_xattr_slab);
-}
\ No newline at end of file
+}
diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h
index bce3d93e4755b5..870524b667ed1d 100644
--- a/fs/f2fs/xattr.h
+++ b/fs/f2fs/xattr.h
@@ -71,21 +71,22 @@ struct f2fs_xattr_entry {
 		for (entry = XATTR_FIRST_ENTRY(addr);\
 				!IS_XATTR_LAST_ENTRY(entry);\
 				entry = XATTR_NEXT_ENTRY(entry))
-#define VALID_XATTR_BLOCK_SIZE	(PAGE_SIZE - sizeof(struct node_footer))
+#define VALID_XATTR_BLOCK_SIZE(i)	(i_blocksize(i) - \
+					sizeof(struct node_footer))
 #define XATTR_PADDING_SIZE	(sizeof(__u32))
 #define XATTR_SIZE(i)		((F2FS_I(i)->i_xattr_nid ?		\
-					VALID_XATTR_BLOCK_SIZE : 0) +	\
+					VALID_XATTR_BLOCK_SIZE(i) : 0) +	\
 						(inline_xattr_size(i)))
 #define MIN_OFFSET(i)		XATTR_ALIGN(inline_xattr_size(i) +	\
-						VALID_XATTR_BLOCK_SIZE)
+						VALID_XATTR_BLOCK_SIZE(i))
 
 #define MAX_VALUE_LEN(i)	(MIN_OFFSET(i) -			\
 				sizeof(struct f2fs_xattr_header) -	\
 				sizeof(struct f2fs_xattr_entry))
 
 #define MIN_INLINE_XATTR_SIZE (sizeof(struct f2fs_xattr_header) / sizeof(__le32))
-#define MAX_INLINE_XATTR_SIZE						\
-			(DEF_ADDRS_PER_INODE -				\
+#define MAX_INLINE_XATTR_SIZE(blocksize)				\
+			(F2FS_DEF_ADDRS_PER_INODE(blocksize) -		\
 			F2FS_TOTAL_EXTRA_ATTR_SIZE / sizeof(__le32) -	\
 			DEF_INLINE_RESERVED_SIZE -			\
 			MIN_INLINE_DENTRY_SIZE / sizeof(__le32))
@@ -130,9 +131,9 @@ extern const struct xattr_handler f2fs_xattr_security_handler;
 extern const struct xattr_handler * const f2fs_xattr_handlers[];
 
 int f2fs_setxattr(struct inode *, int, const char *, const void *,
-		size_t, struct folio *, int);
+		size_t, struct f2fs_cached_block *, int);
 int f2fs_getxattr(struct inode *, int, const char *, void *,
-		size_t, struct folio *);
+		size_t, struct f2fs_cached_block *);
 ssize_t f2fs_listxattr(struct dentry *, char *, size_t);
 int __init f2fs_init_xattr_cache(void);
 void f2fs_destroy_xattr_cache(void);
@@ -142,13 +143,13 @@ void f2fs_destroy_xattr_cache(void);
 #define f2fs_listxattr		NULL
 static inline int f2fs_setxattr(struct inode *inode, int index,
 		const char *name, const void *value, size_t size,
-		struct folio *folio, int flags)
+		struct f2fs_cached_block *ientry, int flags)
 {
 	return -EOPNOTSUPP;
 }
 static inline int f2fs_getxattr(struct inode *inode, int index,
 			const char *name, void *buffer,
-			size_t buffer_size, struct folio *dfolio)
+			size_t buffer_size, struct f2fs_cached_block *ientry)
 {
 	return -EOPNOTSUPP;
 }
@@ -158,10 +159,10 @@ static inline void f2fs_destroy_xattr_cache(void) { }
 
 #ifdef CONFIG_F2FS_FS_SECURITY
 int f2fs_init_security(struct inode *, struct inode *,
-				const struct qstr *, struct folio *);
+				const struct qstr *, struct f2fs_cached_block *);
 #else
 static inline int f2fs_init_security(struct inode *inode, struct inode *dir,
-				const struct qstr *qstr, struct folio *ifolio)
+				const struct qstr *qstr, struct f2fs_cached_block *ientry)
 {
 	return 0;
 }
diff --git a/fs/failfs.c b/fs/failfs.c
index 66a36da3d236d6..437cdc981c2ded 100644
--- a/fs/failfs.c
+++ b/fs/failfs.c
@@ -22,7 +22,7 @@ bool failfs_mnt(const struct vfsmount *mnt)
 	return mnt->mnt_sb == failfs_root_path.mnt->mnt_sb;
 }
 
-static int failfs_permission(struct mnt_idmap *idmap, struct inode *inode,
+static int failfs_permission(const struct mnt_idmap *idmap, struct inode *inode,
 			     int mask)
 {
 	return -EOPNOTSUPP;
@@ -35,7 +35,7 @@ static struct dentry *failfs_lookup(struct inode *dir, struct dentry *dentry,
 	return ERR_PTR(-EOPNOTSUPP);
 }
 
-static int failfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+static int failfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 			  struct kstat *stat, u32 request_mask,
 			  unsigned int query_flags)
 {
diff --git a/fs/fat/fat.h b/fs/fat/fat.h
index 61338413d9f3e4..a4a8958fcfc7b2 100644
--- a/fs/fat/fat.h
+++ b/fs/fat/fat.h
@@ -392,7 +392,7 @@ extern void fat_ent_access_init(struct super_block *sb);
 extern int fat_ent_read(struct inode *inode, struct fat_entry *fatent,
 			int entry);
 extern int fat_ent_write(struct inode *inode, struct fat_entry *fatent,
-			 int new, int wait);
+			 int new, int old, int wait);
 extern int fat_alloc_clusters(struct inode *inode, int *cluster,
 			      int nr_cluster);
 extern int fat_free_clusters(struct inode *inode, int cluster);
@@ -404,10 +404,10 @@ extern long fat_generic_ioctl(struct file *filp, unsigned int cmd,
 			      unsigned long arg);
 extern const struct file_operations fat_file_operations;
 extern const struct inode_operations fat_file_inode_operations;
-extern int fat_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+extern int fat_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct iattr *attr);
 extern void fat_truncate_blocks(struct inode *inode, loff_t offset);
-extern int fat_getattr(struct mnt_idmap *idmap,
+extern int fat_getattr(const struct mnt_idmap *idmap,
 		       const struct path *path, struct kstat *stat,
 		       u32 request_mask, unsigned int flags);
 int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
diff --git a/fs/fat/fatent.c b/fs/fat/fatent.c
index f0801d99dd62ae..df23fc85f31307 100644
--- a/fs/fat/fatent.c
+++ b/fs/fat/fatent.c
@@ -413,7 +413,7 @@ static int fat_mirror_bhs(struct super_block *sb, struct buffer_head **bhs,
 }
 
 int fat_ent_write(struct inode *inode, struct fat_entry *fatent,
-		  int new, int wait)
+		  int new, int old, int wait)
 {
 	struct super_block *sb = inode->i_sb;
 	const struct fatent_operations *ops = MSDOS_SB(sb)->fatent_ops;
@@ -422,10 +422,25 @@ int fat_ent_write(struct inode *inode, struct fat_entry *fatent,
 	ops->ent_put(fatent, new);
 	if (wait) {
 		err = fat_sync_bhs(fatent->bhs, fatent->nr_bhs);
-		if (err)
+		if (err) {
+			/*
+			 * bhs are not uptodate after I/O error. So we
+			 * can't simply re-dirty to revert. And it
+			 * would not have value to write again on I/O
+			 * error.
+			 */
 			return err;
+		}
 	}
-	return fat_mirror_bhs(sb, fatent->bhs, fatent->nr_bhs);
+
+	err = fat_mirror_bhs(sb, fatent->bhs, fatent->nr_bhs);
+	if (err) {
+		/* Try to revert if got the error on mirror FAT */
+		ops->ent_put(fatent, old);
+		if (wait)
+			fat_sync_bhs(fatent->bhs, fatent->nr_bhs);
+	}
+	return err;
 }
 
 static inline int fat_ent_next(struct msdos_sb_info *sbi,
diff --git a/fs/fat/file.c b/fs/fat/file.c
index 1c835ca5f21a51..19bb6b3cab12a4 100644
--- a/fs/fat/file.c
+++ b/fs/fat/file.c
@@ -363,7 +363,8 @@ static int fat_free(struct inode *inode, int skip)
 				     __func__, MSDOS_I(inode)->i_pos);
 			ret = -EIO;
 		} else if (ret > 0) {
-			err = fat_ent_write(inode, &fatent, FAT_ENT_EOF, wait);
+			err = fat_ent_write(inode, &fatent, FAT_ENT_EOF, ret,
+					    wait);
 			if (err)
 				ret = err;
 		}
@@ -432,7 +433,7 @@ int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 }
 EXPORT_SYMBOL_GPL(fat_fileattr_get);
 
-int fat_getattr(struct mnt_idmap *idmap, const struct path *path,
+int fat_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -493,7 +494,7 @@ static int fat_sanitize_mode(const struct msdos_sb_info *sbi,
 	return 0;
 }
 
-static int fat_allow_set_time(struct mnt_idmap *idmap,
+static int fat_allow_set_time(const struct mnt_idmap *idmap,
 			      struct msdos_sb_info *sbi, struct inode *inode)
 {
 	umode_t allow_utime = sbi->options.allow_utime;
@@ -514,7 +515,7 @@ static int fat_allow_set_time(struct mnt_idmap *idmap,
 /* valid file mode bits */
 #define FAT_VALID_MODE	(S_IFREG | S_IFDIR | S_IRWXUGO)
 
-int fat_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int fat_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *attr)
 {
 	struct msdos_sb_info *sbi = MSDOS_SB(dentry->d_sb);
diff --git a/fs/fat/inode.c b/fs/fat/inode.c
index f775a004cae1e2..0b0bbe777842da 100644
--- a/fs/fat/inode.c
+++ b/fs/fat/inode.c
@@ -20,6 +20,7 @@
 #include <linux/blkdev.h>
 #include <linux/backing-dev.h>
 #include <linux/unaligned.h>
+#include <linux/overflow.h>
 #include <linux/random.h>
 #include <linux/iversion.h>
 #include <linux/fs_struct.h>
@@ -1577,6 +1578,7 @@ int fat_fill_super(struct super_block *sb, struct fs_context *fc,
 	struct msdos_sb_info *sbi;
 	u16 logical_sector_size;
 	u32 total_sectors, total_clusters, fat_clusters, rootdir_sectors;
+	u32 dir_start, data_start;
 	long error;
 	char buf[50];
 	struct timespec64 ts;
@@ -1752,7 +1754,6 @@ int fat_fill_super(struct super_block *sb, struct fs_context *fc,
 	sbi->dir_per_block = sb->s_blocksize / sizeof(struct msdos_dir_entry);
 	sbi->dir_per_block_bits = ffs(sbi->dir_per_block) - 1;
 
-	sbi->dir_start = sbi->fat_start + sbi->fats * sbi->fat_length;
 	sbi->dir_entries = bpb.fat_dir_entries;
 	if (sbi->dir_entries & (sbi->dir_per_block - 1)) {
 		if (!silent)
@@ -1763,20 +1764,30 @@ int fat_fill_super(struct super_block *sb, struct fs_context *fc,
 
 	rootdir_sectors = sbi->dir_entries
 		* sizeof(struct msdos_dir_entry) / sb->s_blocksize;
-	sbi->data_start = sbi->dir_start + rootdir_sectors;
+	if (check_mul_overflow(sbi->fats, sbi->fat_length, &dir_start) ||
+	    check_add_overflow(sbi->fat_start, dir_start, &dir_start) ||
+	    check_add_overflow(dir_start, rootdir_sectors, &data_start)) {
+		if (!silent)
+			fat_msg(sb, KERN_ERR,
+				"overflow of root dir or data layout");
+		goto out_invalid;
+	}
+
 	total_sectors = bpb.fat_sectors;
 	if (total_sectors == 0)
 		total_sectors = bpb.fat_total_sect;
 
-	if (total_sectors < sbi->data_start) {
+	if (total_sectors < data_start) {
 		if (!silent)
 			fat_msg(sb, KERN_ERR,
-				"data area starts beyond volume (%lu > %u)",
-				sbi->data_start, total_sectors);
+				"data area starts beyond volume (%u > %u)",
+				data_start, total_sectors);
 		goto out_invalid;
 	}
 
-	total_clusters = (total_sectors - sbi->data_start) / sbi->sec_per_clus;
+	sbi->dir_start = dir_start;
+	sbi->data_start = data_start;
+	total_clusters = (total_sectors - data_start) / sbi->sec_per_clus;
 
 	if (!is_fat32(sbi))
 		sbi->fat_bits = (total_clusters > MAX_FAT12) ? 16 : 12;
diff --git a/fs/fat/misc.c b/fs/fat/misc.c
index e79762cf19754d..9a76ff8c60f061 100644
--- a/fs/fat/misc.c
+++ b/fs/fat/misc.c
@@ -133,11 +133,9 @@ int fat_chain_add(struct inode *inode, int new_dclus, int nr_cluster)
 		ret = fat_ent_read(inode, &fatent, last);
 		if (ret >= 0) {
 			int wait = inode_needs_sync(inode);
-			int old = ret;
 
-			ret = fat_ent_write(inode, &fatent, new_dclus, wait);
-			if (ret < 0)
-				fat_ent_write(inode, &fatent, old, wait);
+			ret = fat_ent_write(inode, &fatent, new_dclus, ret,
+					    wait);
 			fatent_brelse(&fatent);
 		}
 		if (ret < 0)
@@ -360,7 +358,7 @@ int fat_sync_bhs(struct buffer_head **bhs, int nr_bhs)
 
 	for (i = 0; i < nr_bhs; i++) {
 		wait_on_buffer(bhs[i]);
-		if (!err && !buffer_uptodate(bhs[i]))
+		if (!err && buffer_write_io_error(bhs[i]))
 			err = -EIO;
 	}
 	return err;
diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c
index d46d1a3851f25f..dde4215616f943 100644
--- a/fs/fat/namei_msdos.c
+++ b/fs/fat/namei_msdos.c
@@ -263,7 +263,7 @@ static int msdos_add_entry(struct inode *dir, const unsigned char *name,
 }
 
 /***** Create a file */
-static int msdos_create(struct mnt_idmap *idmap, struct inode *dir,
+static int msdos_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -345,7 +345,7 @@ static int msdos_rmdir(struct inode *dir, struct dentry *dentry)
 }
 
 /***** Make a directory */
-static struct dentry *msdos_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *msdos_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -600,7 +600,7 @@ static int do_msdos_rename(struct inode *old_dir, unsigned char *old_name,
 }
 
 /***** Rename, a wrapper for rename_same_dir & rename_diff_dir */
-static int msdos_rename(struct mnt_idmap *idmap,
+static int msdos_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c
index da3e89c0b16ac8..3dc063ba0a738c 100644
--- a/fs/fat/namei_vfat.c
+++ b/fs/fat/namei_vfat.c
@@ -753,7 +753,7 @@ static struct dentry *vfat_lookup(struct inode *dir, struct dentry *dentry,
 	return ERR_PTR(err);
 }
 
-static int vfat_create(struct mnt_idmap *idmap, struct inode *dir,
+static int vfat_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -846,7 +846,7 @@ static int vfat_unlink(struct inode *dir, struct dentry *dentry)
 	return err;
 }
 
-static struct dentry *vfat_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *vfat_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -1160,7 +1160,7 @@ static int vfat_rename_exchange(struct inode *old_dir, struct dentry *old_dentry
 	goto out;
 }
 
-static int vfat_rename2(struct mnt_idmap *idmap, struct inode *old_dir,
+static int vfat_rename2(const struct mnt_idmap *idmap, struct inode *old_dir,
 			struct dentry *old_dentry, struct inode *new_dir,
 			struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/fhandle.c b/fs/fhandle.c
index f8829231e3d758..2aa55b8a878a90 100644
--- a/fs/fhandle.c
+++ b/fs/fhandle.c
@@ -201,7 +201,7 @@ static int vfs_dentry_acceptable(void *context, struct dentry *dentry)
 	struct handle_to_path_ctx *ctx = context;
 	struct user_namespace *user_ns = current_user_ns();
 	struct dentry *d, *root = ctx->root.dentry;
-	struct mnt_idmap *idmap = mnt_idmap(ctx->root.mnt);
+	const struct mnt_idmap *idmap = mnt_idmap(ctx->root.mnt);
 	int retval = 0;
 
 	if (!root)
diff --git a/fs/file.c b/fs/file.c
index 628ca07dc4b179..88b7a53408150e 100644
--- a/fs/file.c
+++ b/fs/file.c
@@ -352,24 +352,78 @@ static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt)
 	return test_bit(fd, fdt->open_fds);
 }
 
+/* Bits of [range->from, range->to] that fall into word @i of a bitmap. */
+static unsigned long fd_range_word(struct fd_range *range, unsigned int i)
+{
+	unsigned int first = i * BITS_PER_LONG;
+	unsigned int last = first + BITS_PER_LONG - 1;
+
+	if (range->to < first || range->from > last)
+		return 0;
+	return GENMASK(min(range->to, last) - first,
+		       max(range->from, first) - first);
+}
+
+/* Bits of word @i that dup_fd() leaves behind and __range_close() closes. */
+static unsigned long dup_fd_dropped_word(struct fdtable *fdt, unsigned int i,
+					 struct fd_range *range)
+{
+	unsigned long dropped;
+
+	if (!range)
+		return 0;
+	dropped = fd_range_word(range, i);
+	if (range->flags & FD_RANGE_EXCEPT)
+		dropped = ~dropped;
+	if (range->flags & FD_RANGE_CLOEXEC_ONLY)
+		dropped &= fdt->close_on_exec[i];
+	return dropped;
+}
+
 /*
  * Note that a sane fdtable size always has to be a multiple of
  * BITS_PER_LONG, since we have bitmaps that are sized by this.
  *
- * punch_hole is optional - when close_range() is asked to unshare
- * and close, we don't need to copy descriptors in that range, so
- * a smaller cloned descriptor table might suffice if the last
- * currently opened descriptor falls into that range.
+ * range is optional. When close_range() is asked to unshare dup_fd()
+ * will leave any files behind according to the range and its flags. The
+ * cloned table only has to reach the last open descriptor that is
+ * carried over.
  */
-static unsigned int sane_fdtable_size(struct fdtable *fdt, struct fd_range *punch_hole)
+static unsigned int sane_fdtable_size(struct fdtable *fdt, struct fd_range *range)
 {
 	unsigned int last = find_last_bit(fdt->open_fds, fdt->max_fds);
+	unsigned int i;
 
 	if (last == fdt->max_fds)
 		return NR_OPEN_DEFAULT;
-	if (punch_hole && punch_hole->to >= last && punch_hole->from <= last) {
-		last = find_last_bit(fdt->open_fds, punch_hole->from);
-		if (last == punch_hole->from)
+	if (!range)
+		return ALIGN(last + 1, BITS_PER_LONG);
+
+	if (range->flags & FD_RANGE_CLOEXEC_ONLY) {
+		/* The close-on-exec bits decide what is dropped, walk the words. */
+		i = last / BITS_PER_LONG + 1;
+		while (i--) {
+			unsigned long dropped = dup_fd_dropped_word(fdt, i, range);
+
+			if (fdt->open_fds[i] & ~dropped)
+				return (i + 1) * BITS_PER_LONG;
+		}
+		return NR_OPEN_DEFAULT;
+	}
+
+	if (range->flags & FD_RANGE_EXCEPT) {
+		/* Only the range is carried over. */
+		if (last > range->to) {
+			last = find_last_bit(fdt->open_fds, range->to + 1);
+			if (last > range->to)
+				return NR_OPEN_DEFAULT;
+		}
+		if (last < range->from)
+			return NR_OPEN_DEFAULT;
+	} else if (last >= range->from && last <= range->to) {
+		/* The last open descriptor goes, the kept ones sit below the range. */
+		last = find_last_bit(fdt->open_fds, range->from);
+		if (last == range->from)
 			return NR_OPEN_DEFAULT;
 	}
 	return ALIGN(last + 1, BITS_PER_LONG);
@@ -378,13 +432,14 @@ static unsigned int sane_fdtable_size(struct fdtable *fdt, struct fd_range *punc
 /*
  * Allocate a new descriptor table and copy contents from the passed in
  * instance.  Returns a pointer to cloned table on success, ERR_PTR()
- * on failure.  For 'punch_hole' see sane_fdtable_size().
+ * on failure.  For 'range' see sane_fdtable_size().
  */
-struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_hole)
+struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *range)
 {
 	struct files_struct *newf;
 	struct file **old_fds, **new_fds;
-	unsigned int open_files, i;
+	unsigned int open_files, fd;
+	unsigned long dropped = 0;
 	struct fdtable *old_fdt, *new_fdt;
 
 	newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
@@ -406,7 +461,7 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho
 
 	spin_lock(&oldf->file_lock);
 	old_fdt = files_fdtable(oldf);
-	open_files = sane_fdtable_size(old_fdt, punch_hole);
+	open_files = sane_fdtable_size(old_fdt, range);
 
 	/*
 	 * Check whether we need to allocate a larger fd array and fd set.
@@ -430,7 +485,7 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho
 		 */
 		spin_lock(&oldf->file_lock);
 		old_fdt = files_fdtable(oldf);
-		open_files = sane_fdtable_size(old_fdt, punch_hole);
+		open_files = sane_fdtable_size(old_fdt, range);
 	}
 
 	copy_fd_bitmaps(new_fdt, old_fdt, open_files / BITS_PER_LONG);
@@ -451,13 +506,18 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho
 	 *
 	 * Instead of trying to placate userspace racing with itself, we
 	 * ref the file if we see it and mark the fd slot as unused otherwise.
+	 * Descriptors dup_fd() is asked to leave behind get the same treatment.
 	 */
-	for (i = open_files; i != 0; i--) {
+	for (fd = 0; fd < open_files; fd++) {
 		struct file *f = rcu_dereference_raw(*old_fds++);
-		if (f) {
+
+		if (!(fd % BITS_PER_LONG))
+			dropped = dup_fd_dropped_word(old_fdt, fd / BITS_PER_LONG, range);
+		if (f && !(dropped & BIT_MASK(fd))) {
 			get_file(f);
 		} else {
-			__clear_open_fd(open_files - i, new_fdt);
+			f = NULL;
+			__clear_open_fd(fd, new_fdt);
 		}
 		rcu_assign_pointer(*new_fds++, f);
 	}
@@ -471,7 +531,25 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_ho
 	return newf;
 }
 
-static struct fdtable *close_files(struct files_struct * files)
+/*
+ * Unshare file descriptor table if it is being shared
+ */
+int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
+{
+	struct files_struct *fd = current->files;
+
+	if ((unshare_flags & CLONE_FILES) &&
+	    (fd && atomic_read(&fd->count) > 1)) {
+		fd = dup_fd(fd, NULL);
+		if (IS_ERR(fd))
+			return PTR_ERR(fd);
+		*new_fdp = fd;
+	}
+
+	return 0;
+}
+
+static struct fdtable *close_files(struct files_struct *files)
 {
 	/*
 	 * It is safe to dereference the fd table without RCU or
@@ -479,24 +557,21 @@ static struct fdtable *close_files(struct files_struct * files)
 	 * files structure.
 	 */
 	struct fdtable *fdt = rcu_dereference_raw(files->fdt);
-	unsigned int i, j = 0;
+	unsigned int j = fdt->max_fds / BITS_PER_LONG;
+
+	/* Highest fd first, the order the deferred puts ran in. */
+	while (j--) {
+		unsigned long set = fdt->open_fds[j];
 
-	for (;;) {
-		unsigned long set;
-		i = j * BITS_PER_LONG;
-		if (i >= fdt->max_fds)
-			break;
-		set = fdt->open_fds[j++];
 		while (set) {
-			if (set & 1) {
-				struct file *file = fdt->fd[i];
-				if (file) {
-					filp_close(file, files);
-					cond_resched();
-				}
+			unsigned int bit = __fls(set);
+			struct file *file = fdt->fd[j * BITS_PER_LONG + bit];
+
+			set ^= 1UL << bit;
+			if (file) {
+				filp_close_sync(file, files);
+				cond_resched();
 			}
-			i++;
-			set >>= 1;
 		}
 	}
 
@@ -515,16 +590,18 @@ void put_files_struct(struct files_struct *files)
 	}
 }
 
-void exit_files(struct task_struct *tsk)
+/* Install @files on @tsk, consuming the reference, and put the old table. */
+void switch_files_struct(struct task_struct *tsk, struct files_struct *files)
 {
-	struct files_struct * files = tsk->files;
+	scoped_guard(task_lock, tsk)
+		swap(tsk->files, files);
+	put_files_struct(files);
+}
 
-	if (files) {
-		task_lock(tsk);
-		tsk->files = NULL;
-		task_unlock(tsk);
-		put_files_struct(files);
-	}
+void exit_files(struct task_struct *tsk)
+{
+	if (tsk->files)
+		switch_files_struct(tsk, NULL);
 }
 
 struct files_struct init_files = {
@@ -732,16 +809,13 @@ struct file *file_close_fd_locked(struct files_struct *files, unsigned fd)
 
 int close_fd(unsigned fd)
 {
-	struct files_struct *files = current->files;
 	struct file *file;
 
-	spin_lock(&files->file_lock);
-	file = file_close_fd_locked(files, fd);
-	spin_unlock(&files->file_lock);
+	file = file_close_fd(fd);
 	if (!file)
 		return -EBADF;
 
-	return filp_close(file, files);
+	return filp_close(file, current->files);
 }
 EXPORT_SYMBOL(close_fd);
 
@@ -759,38 +833,90 @@ static inline unsigned last_fd(struct fdtable *fdt)
 }
 
 static inline void __range_cloexec(struct files_struct *cur_fds,
-				   unsigned int fd, unsigned int max_fd)
+				   struct fd_range *range)
 {
 	struct fdtable *fdt;
+	unsigned int last;
 
-	/* make sure we're using the correct maximum value */
 	spin_lock(&cur_fds->file_lock);
 	fdt = files_fdtable(cur_fds);
-	max_fd = min(last_fd(fdt), max_fd);
-	if (fd <= max_fd)
-		bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);
+	/* make sure we're using the correct maximum value */
+	last = last_fd(fdt);
+	if (!(range->flags & FD_RANGE_EXCEPT)) {
+		if (range->from <= last)
+			bitmap_set(fdt->close_on_exec, range->from,
+				   min(range->to, last) - range->from + 1);
+	} else {
+		if (range->from > 0)
+			bitmap_set(fdt->close_on_exec, 0,
+				   min(range->from - 1, last) + 1);
+		if (range->to < last)
+			bitmap_set(fdt->close_on_exec, range->to + 1,
+				   last - range->to);
+	}
 	spin_unlock(&cur_fds->file_lock);
 }
 
-static inline void __range_close(struct files_struct *files, unsigned int fd,
-				 unsigned int max_fd)
+/* Highest open descriptor below @n that @range selects, or @n. */
+static inline unsigned int last_fd_to_close(struct fdtable *fdt, unsigned int n,
+					    struct fd_range *range)
+{
+	unsigned int i, lo = 0;
+
+	if (!(range->flags & FD_RANGE_EXCEPT))
+		lo = range->from / BITS_PER_LONG;
+	for (i = n ? (n - 1) / BITS_PER_LONG + 1 : 0; i-- > lo; ) {
+		unsigned long set = fdt->open_fds[i];
+
+		if (!set) {
+			/* Skip the empty stretch at find_last_bit() speed. */
+			unsigned int last = find_last_bit(fdt->open_fds, i * BITS_PER_LONG);
+
+			if (last >= i * BITS_PER_LONG)
+				break;
+			i = last / BITS_PER_LONG + 1;
+			continue;
+		}
+		/* Hop below the kept window in one step. */
+		if ((range->flags & FD_RANGE_EXCEPT) &&
+		    i * BITS_PER_LONG >= range->from &&
+		    i * BITS_PER_LONG + BITS_PER_LONG - 1 <= range->to) {
+			if (!range->from)
+				break;
+			i = (range->from - 1) / BITS_PER_LONG + 1;
+			continue;
+		}
+		set &= dup_fd_dropped_word(fdt, i, range);
+		if (i == (n - 1) / BITS_PER_LONG)
+			set &= BITMAP_LAST_WORD_MASK(n);
+		if (set)
+			return i * BITS_PER_LONG + __fls(set);
+	}
+	return n;
+}
+
+static inline void __range_close(struct files_struct *files,
+				 struct fd_range *range)
 {
 	struct file *file;
 	struct fdtable *fdt;
-	unsigned n;
+	unsigned int fd, n;
 
 	spin_lock(&files->file_lock);
 	fdt = files_fdtable(files);
-	n = last_fd(fdt);
-	max_fd = min(max_fd, n);
+	if (range->flags & FD_RANGE_EXCEPT)
+		/* Outside of the range means the whole table. */
+		n = fdt->max_fds;
+	else
+		n = min(range->to, last_fd(fdt)) + 1;
 
-	for (fd = find_next_bit(fdt->open_fds, max_fd + 1, fd);
-	     fd <= max_fd;
-	     fd = find_next_bit(fdt->open_fds, max_fd + 1, fd + 1)) {
+	/* Highest fd first, see close_files(). */
+	while ((fd = last_fd_to_close(fdt, n, range)) < n) {
+		n = fd;
 		file = file_close_fd_locked(files, fd);
 		if (file) {
 			spin_unlock(&files->file_lock);
-			filp_close(file, files);
+			filp_close_sync(file, files);
 			cond_resched();
 			spin_lock(&files->file_lock);
 			fdt = files_fdtable(files);
@@ -814,21 +940,43 @@ static inline void __range_close(struct files_struct *files, unsigned int fd,
  * This closes a range of file descriptors. All file descriptors
  * from @fd up to and including @max_fd are closed.
  * Currently, errors to close a given file descriptor are ignored.
+ *
+ * With CLOSE_RANGE_EXCEPT the range names what to leave alone instead:
+ * every open file descriptor outside of [@fd, @max_fd] is closed, or
+ * marked close-on-exec with CLOSE_RANGE_CLOEXEC.
+ *
+ * With CLOSE_RANGE_CLOEXEC_ONLY only file descriptors that have
+ * close-on-exec set are closed. Together with CLOSE_RANGE_EXCEPT the
+ * range names the close-on-exec file descriptors to keep. To keep none
+ * of them, name a range that cannot hold an open file descriptor, e.g.
+ * close_range(~0U, ~0U, ...).
  */
 SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd,
 		unsigned int, flags)
 {
 	struct task_struct *me = current;
 	struct files_struct *cur_fds = me->files, *fds = NULL;
+	struct fd_range range = {fd, max_fd};
+
+	if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC |
+		      CLOSE_RANGE_EXCEPT | CLOSE_RANGE_CLOEXEC_ONLY))
+		return -EINVAL;
 
-	if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC))
+	/* One marks close-on-exec, the other closes what is marked. */
+	if (hweight32(flags & (CLOSE_RANGE_CLOEXEC |
+			       CLOSE_RANGE_CLOEXEC_ONLY)) > 1)
 		return -EINVAL;
 
 	if (fd > max_fd)
 		return -EINVAL;
 
+	if (flags & CLOSE_RANGE_EXCEPT)
+		range.flags |= FD_RANGE_EXCEPT;
+	if (flags & CLOSE_RANGE_CLOEXEC_ONLY)
+		range.flags |= FD_RANGE_CLOEXEC_ONLY;
+
 	if ((flags & CLOSE_RANGE_UNSHARE) && atomic_read(&cur_fds->count) > 1) {
-		struct fd_range range = {fd, max_fd}, *punch_hole = &range;
+		struct fd_range *drop = &range;
 
 		/*
 		 * If the caller requested all fds to be made cloexec we always
@@ -836,9 +984,9 @@ SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd,
 		 * use them.
 		 */
 		if (flags & CLOSE_RANGE_CLOEXEC)
-			punch_hole = NULL;
+			drop = NULL;
 
-		fds = dup_fd(cur_fds, punch_hole);
+		fds = dup_fd(cur_fds, drop);
 		if (IS_ERR(fds))
 			return PTR_ERR(fds);
 		/*
@@ -848,20 +996,19 @@ SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd,
 		swap(cur_fds, fds);
 	}
 
-	if (flags & CLOSE_RANGE_CLOEXEC)
-		__range_cloexec(cur_fds, fd, max_fd);
-	else
-		__range_close(cur_fds, fd, max_fd);
+	if (flags & CLOSE_RANGE_CLOEXEC) {
+		__range_cloexec(cur_fds, &range);
+	} else if (!fds) {
+		/* If we unshared, dup_fd() already left behind what we'd close. */
+		__range_close(cur_fds, &range);
+	}
 
 	if (fds) {
 		/*
 		 * We're done closing the files we were supposed to. Time to install
 		 * the new file descriptor table and drop the old one.
 		 */
-		task_lock(me);
-		me->files = cur_fds;
-		task_unlock(me);
-		put_files_struct(fds);
+		switch_files_struct(me, cur_fds);
 	}
 
 	return 0;
@@ -887,34 +1034,36 @@ struct file *file_close_fd(unsigned int fd)
 	return file;
 }
 
-void do_close_on_exec(struct files_struct *files)
+void close_cloexec_files(struct files_struct *files)
 {
 	unsigned i;
 	struct fdtable *fdt;
 
 	/* exec unshares first */
 	spin_lock(&files->file_lock);
-	for (i = 0; ; i++) {
+	fdt = files_fdtable(files);
+	/* Highest fd first, see close_files(). */
+	for (i = fdt->max_fds / BITS_PER_LONG; i--; ) {
 		unsigned long set;
-		unsigned fd = i * BITS_PER_LONG;
+
 		fdt = files_fdtable(files);
-		if (fd >= fdt->max_fds)
-			break;
 		set = fdt->close_on_exec[i];
 		if (!set)
 			continue;
 		fdt->close_on_exec[i] = 0;
-		for ( ; set ; fd++, set >>= 1) {
+		while (set) {
+			unsigned int bit = __fls(set);
+			unsigned fd = i * BITS_PER_LONG + bit;
 			struct file *file;
-			if (!(set & 1))
-				continue;
+
+			set ^= 1UL << bit;
 			file = fdt->fd[fd];
 			if (!file)
 				continue;
 			rcu_assign_pointer(fdt->fd[fd], NULL);
 			__put_unused_fd(files, fd);
 			spin_unlock(&files->file_lock);
-			filp_close(file, files);
+			filp_close_sync(file, files);
 			cond_resched();
 			spin_lock(&files->file_lock);
 		}
@@ -1391,17 +1540,17 @@ int receive_fd(struct file *file, int __user *ufd, unsigned int o_flags)
 		return error;
 
 	FD_PREPARE(fdf, o_flags, file);
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 	get_file(file);
 
 	if (ufd) {
-		error = put_user(fd_prepare_fd(fdf), ufd);
+		error = put_user(fdf->fd, ufd);
 		if (error)
 			return error;
 	}
 
-	__receive_sock(fd_prepare_file(fdf));
+	__receive_sock(fdf->file);
 	return fd_publish(fdf);
 }
 EXPORT_SYMBOL_GPL(receive_fd);
@@ -1529,3 +1678,7 @@ int iterate_fd(struct files_struct *files, unsigned n,
 	return res;
 }
 EXPORT_SYMBOL(iterate_fd);
+
+#ifdef CONFIG_FDTABLE_KUNIT_TEST
+#include "tests/fdtable_kunit.c"
+#endif
diff --git a/fs/file_attr.c b/fs/file_attr.c
index bfb00d256dd560..81af4364e33ae7 100644
--- a/fs/file_attr.c
+++ b/fs/file_attr.c
@@ -265,7 +265,7 @@ static int fileattr_set_prepare(struct inode *inode,
  *
  * Return: 0 on success, or a negative error on failure.
  */
-int vfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry,
+int vfs_fileattr_set(const struct mnt_idmap *idmap, struct dentry *dentry,
 		     struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
@@ -323,7 +323,7 @@ int ioctl_getflags(struct file *file, unsigned int __user *argp)
 
 int ioctl_setflags(struct file *file, unsigned int __user *argp)
 {
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct dentry *dentry = file->f_path.dentry;
 	struct file_kattr fa = {};
 	unsigned int flags;
@@ -355,7 +355,7 @@ int ioctl_fsgetxattr(struct file *file, void __user *argp)
 
 int ioctl_fssetxattr(struct file *file, void __user *argp)
 {
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct dentry *dentry = file->f_path.dentry;
 	struct file_kattr fa = {};
 	int err;
diff --git a/fs/file_table.c b/fs/file_table.c
index c68b8c0a4097ae..31983471956fe4 100644
--- a/fs/file_table.c
+++ b/fs/file_table.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/fs.h>
 #include <linux/filelock.h>
+#include <linux/pagemap.h>
 #include <linux/security.h>
 #include <linux/cred.h>
 #include <linux/eventpoll.h>
@@ -122,7 +123,7 @@ unsigned long get_max_files(void)
 }
 EXPORT_SYMBOL_GPL(get_max_files);
 
-#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
+#ifdef CONFIG_SYSCTL
 
 /*
  * Handle nr_files sysctl
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index e744f9f9d43fae..58a6403780aa46 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -233,7 +233,7 @@ void wb_wait_for_completion(struct wb_completion *done)
  * Parameters for foreign inode detection, see wbc_detach_inode() to see
  * how they're used.
  *
- * These paramters are inherently heuristical as the detection target
+ * These parameters are inherently heuristical as the detection target
  * itself is fuzzy.  All we want to do is detaching an inode from the
  * current owner if it's being written to by some other cgroups too much.
  *
@@ -248,7 +248,7 @@ void wb_wait_for_completion(struct wb_completion *done)
  * to 16 slots.  To avoid tiny writes from swinging the decision too much,
  * writes smaller than 1/8 of avg size are ignored.
  */
-#define WB_FRN_TIME_SHIFT	13	/* 1s = 2^13, upto 8 secs w/ 16bit */
+#define WB_FRN_TIME_SHIFT	13	/* 1s = 2^13, up to 8 secs w/ 16bit */
 #define WB_FRN_TIME_AVG_SHIFT	3	/* avg = avg * 7/8 + new * 1/8 */
 #define WB_FRN_TIME_CUT_DIV	8	/* ignore rounds < avg / 8 */
 #define WB_FRN_TIME_PERIOD	(2 * (1 << WB_FRN_TIME_SHIFT))	/* 2s */
@@ -259,7 +259,7 @@ void wb_wait_for_completion(struct wb_completion *done)
 #define WB_FRN_HIST_THR_SLOTS	(WB_FRN_HIST_SLOTS / 2)
 					/* if foreign slots >= 8, switch */
 #define WB_FRN_HIST_MAX_SLOTS	(WB_FRN_HIST_THR_SLOTS / 2 + 1)
-					/* one round can affect upto 5 slots */
+					/* one round can affect up to 5 slots */
 #define WB_FRN_MAX_IN_FLIGHT	1024	/* don't queue too many concurrently */
 
 /*
@@ -727,19 +727,34 @@ static bool isw_prepare_wbs_switch(struct bdi_writeback *new_wb,
 				   struct inode_switch_wbs_context *isw,
 				   struct list_head *list, int *nr)
 {
-	struct inode *inode;
+	struct inode *inode, *tmp;
+	LIST_HEAD(scanned);
+	bool full = false;
+
+	/*
+	 * Walk from the oldest end and move scanned inodes to the newest
+	 * end, so the next scan resumes at unscanned inodes instead of
+	 * re-walking an ever-growing run of prepared and skipped ones.
+	 * For b_dirty_time this keeps the oldest unscanned inode at the
+	 * end move_expired_inodes() picks from; b_attached is unordered.
+	 */
+	list_for_each_entry_safe_reverse(inode, tmp, list, i_io_list) {
+		list_move(&inode->i_io_list, &scanned);
 
-	list_for_each_entry(inode, list, i_io_list) {
 		if (!inode_prepare_wbs_switch(inode, new_wb))
 			continue;
 
 		isw->inodes[*nr] = inode;
 		(*nr)++;
 
-		if (*nr >= WB_MAX_INODES_PER_ISW - 1)
-			return true;
+		if (*nr >= WB_MAX_INODES_PER_ISW - 1) {
+			full = true;
+			break;
+		}
 	}
-	return false;
+	list_splice(&scanned, list);
+
+	return full;
 }
 
 /**
@@ -1166,7 +1181,7 @@ int cgroup_writeback_by_id(u64 bdi_id, int memcg_id,
 	struct cgroup_subsys_state *memcg_css;
 	struct bdi_writeback *wb;
 	struct wb_writeback_work *work;
-	unsigned long dirty;
+	long dirty;
 	int ret;
 
 	/* lookup bdi and memcg */
@@ -1195,16 +1210,13 @@ int cgroup_writeback_by_id(u64 bdi_id, int memcg_id,
 	}
 
 	/*
-	 * The caller is attempting to write out most of
-	 * the currently dirty pages.  Let's take the current dirty page
-	 * count and inflate it by 25% which should be large enough to
-	 * flush out most dirty pages while avoiding getting livelocked by
-	 * concurrent dirtiers.
-	 *
-	 * BTW the memcg stats are flushed periodically and this is best-effort
-	 * estimation, so some potential error is ok.
+	 * The caller is attempting to write out most of the target wb's
+	 * currently dirty pages.  Size the work from the wb's reclaimable pages
+	 * and inflate the count by 25%, which should be large enough to flush
+	 * out most dirty pages while avoiding getting livelocked by concurrent
+	 * dirtiers.
 	 */
-	dirty = memcg_page_state(mem_cgroup_from_css(memcg_css), NR_FILE_DIRTY);
+	dirty = wb_stat_sum(wb, WB_RECLAIMABLE);
 	dirty = dirty * 10 / 8;
 
 	/* issue the writeback work */
diff --git a/fs/fs_pin.c b/fs/fs_pin.c
index 47ef3c71ce9077..1a508f2167e0a3 100644
--- a/fs/fs_pin.c
+++ b/fs/fs_pin.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/fs.h>
+#include <linux/rculist.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include "internal.h"
@@ -22,8 +23,8 @@ void pin_remove(struct fs_pin *pin)
 void pin_insert(struct fs_pin *pin, struct vfsmount *m)
 {
 	spin_lock(&pin_lock);
-	hlist_add_head(&pin->s_list, &m->mnt_sb->s_pins);
-	hlist_add_head(&pin->m_list, &real_mount(m)->mnt_pins);
+	hlist_add_head_rcu(&pin->s_list, &m->mnt_sb->s_pins);
+	hlist_add_head_rcu(&pin->m_list, &real_mount(m)->mnt_pins);
 	spin_unlock(&pin_lock);
 }
 
@@ -73,7 +74,7 @@ void mnt_pin_kill(struct mount *m)
 	while (1) {
 		struct hlist_node *p;
 		rcu_read_lock();
-		p = READ_ONCE(m->mnt_pins.first);
+		p = rcu_dereference(hlist_first_rcu(&m->mnt_pins));
 		if (!p) {
 			rcu_read_unlock();
 			break;
@@ -87,7 +88,7 @@ void group_pin_kill(struct hlist_head *p)
 	while (1) {
 		struct hlist_node *q;
 		rcu_read_lock();
-		q = READ_ONCE(p->first);
+		q = rcu_dereference(hlist_first_rcu(p));
 		if (!q) {
 			rcu_read_unlock();
 			break;
diff --git a/fs/fuse/Kconfig b/fs/fuse/Kconfig
index 3a4ae632c94aa8..8c41838b1c3d9a 100644
--- a/fs/fuse/Kconfig
+++ b/fs/fuse/Kconfig
@@ -40,9 +40,9 @@ config VIRTIO_FS
 	  If you want to share files between guests or with the host, answer Y
 	  or M.
 
-config FUSE_DAX
+config FUSE_VDAX
 	bool "Virtio Filesystem Direct Host Memory Access support"
-	default y
+	default FUSE_DAX
 	select INTERVAL_TREE
 	depends on VIRTIO_FS
 	depends on FS_DAX
@@ -54,6 +54,10 @@ config FUSE_DAX
 	  If you want to allow mounting a Virtio Filesystem with the "dax"
 	  option, answer Y.
 
+config FUSE_DAX
+    bool
+    transitional
+
 config FUSE_PASSTHROUGH
 	bool "FUSE passthrough operations support"
 	default y
diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile
index 245e67852b03e8..5858feafa91647 100644
--- a/fs/fuse/Makefile
+++ b/fs/fuse/Makefile
@@ -14,7 +14,7 @@ fuse-y := trace.o	# put trace.o first so we see ftrace errors sooner
 fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o
 fuse-y += poll.o notify.o
 fuse-y += iomode.o
-fuse-$(CONFIG_FUSE_DAX) += dax.o
+fuse-$(CONFIG_FUSE_VDAX) += dax.o
 fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o
 fuse-$(CONFIG_SYSCTL) += sysctl.o
 fuse-$(CONFIG_FUSE_IO_URING) += dev_uring.o
diff --git a/fs/fuse/acl.c b/fs/fuse/acl.c
index 31fb50e16aed7b..738abed9a81667 100644
--- a/fs/fuse/acl.c
+++ b/fs/fuse/acl.c
@@ -62,7 +62,7 @@ static inline bool fuse_no_acl(const struct fuse_conn *fc,
 	return !fc->posix_acl && (i_user_ns(inode) != &init_user_ns);
 }
 
-struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *fuse_get_acl(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, int type)
 {
 	struct inode *inode = d_inode(dentry);
@@ -90,7 +90,7 @@ struct posix_acl *fuse_get_inode_acl(struct inode *inode, int type, bool rcu)
 	return __fuse_get_acl(fc,  inode, type, rcu);
 }
 
-int fuse_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int fuse_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/fuse/backing.c b/fs/fuse/backing.c
index 472b6afa7dfff1..433fa3098d71fa 100644
--- a/fs/fuse/backing.c
+++ b/fs/fuse/backing.c
@@ -10,7 +10,7 @@
 
 #include <linux/file.h>
 
-struct fuse_backing *fuse_backing_get(struct fuse_backing *fb)
+static struct fuse_backing *fuse_backing_get(struct fuse_backing *fb)
 {
 	if (fb && refcount_inc_not_zero(&fb->count))
 		return fb;
diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c
index 85cdf0199bc0b8..a2bb9c041a4cbc 100644
--- a/fs/fuse/dax.c
+++ b/fs/fuse/dax.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * dax: direct host memory access
+ * dax: direct host memory access for virtiofs
  * Copyright (C) 2020 Red Hat, Inc.
  */
 
@@ -59,7 +59,7 @@ struct fuse_dax_mapping {
 };
 
 /* Per-inode dax map */
-struct fuse_inode_dax {
+struct fuse_inode_vdax {
 	/* Semaphore to protect modifications to the dmap tree */
 	struct rw_semaphore sem;
 
@@ -68,7 +68,7 @@ struct fuse_inode_dax {
 	unsigned long nr;
 };
 
-struct fuse_conn_dax {
+struct fuse_conn_vdax {
 	/* DAX device */
 	struct dax_device *dev;
 
@@ -102,10 +102,10 @@ node_to_dmap(struct interval_tree_node *node)
 }
 
 static struct fuse_dax_mapping *
-alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode);
+alloc_dax_mapping_reclaim(struct fuse_conn_vdax *fcd, struct inode *inode);
 
 static void
-__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
+__kick_dmap_free_worker(struct fuse_conn_vdax *fcd, unsigned long delay_ms)
 {
 	unsigned long free_threshold;
 
@@ -117,7 +117,7 @@ __kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
 				   msecs_to_jiffies(delay_ms));
 }
 
-static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,
+static void kick_dmap_free_worker(struct fuse_conn_vdax *fcd,
 				  unsigned long delay_ms)
 {
 	spin_lock(&fcd->lock);
@@ -125,7 +125,7 @@ static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,
 	spin_unlock(&fcd->lock);
 }
 
-static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)
+static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_vdax *fcd)
 {
 	struct fuse_dax_mapping *dmap;
 
@@ -144,7 +144,7 @@ static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)
 }
 
 /* This assumes fcd->lock is held */
-static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,
+static void __dmap_remove_busy_list(struct fuse_conn_vdax *fcd,
 				    struct fuse_dax_mapping *dmap)
 {
 	list_del_init(&dmap->busy_list);
@@ -152,7 +152,7 @@ static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,
 	fcd->nr_busy_ranges--;
 }
 
-static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,
+static void dmap_remove_busy_list(struct fuse_conn_vdax *fcd,
 				  struct fuse_dax_mapping *dmap)
 {
 	spin_lock(&fcd->lock);
@@ -161,7 +161,7 @@ static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,
 }
 
 /* This assumes fcd->lock is held */
-static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
+static void __dmap_add_to_free_pool(struct fuse_conn_vdax *fcd,
 				struct fuse_dax_mapping *dmap)
 {
 	list_add_tail(&dmap->list, &fcd->free_ranges);
@@ -169,7 +169,7 @@ static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
 	wake_up(&fcd->range_waitq);
 }
 
-static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
+static void dmap_add_to_free_pool(struct fuse_conn_vdax *fcd,
 				struct fuse_dax_mapping *dmap)
 {
 	/* Return fuse_dax_mapping to free list */
@@ -183,7 +183,7 @@ static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx,
 				  bool upgrade)
 {
 	struct fuse_mount *fm = get_fuse_mount(inode);
-	struct fuse_conn_dax *fcd = fm->fc->dax;
+	struct fuse_conn_vdax *fcd = fm->fc->vdax;
 	struct fuse_inode *fi = get_fuse_inode(inode);
 	struct fuse_setupmapping_in inarg;
 	loff_t offset = start_idx << FUSE_DAX_SHIFT;
@@ -218,9 +218,9 @@ static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx,
 		 */
 		dmap->inode = inode;
 		dmap->itn.start = dmap->itn.last = start_idx;
-		/* Protected by fi->dax->sem */
-		interval_tree_insert(&dmap->itn, &fi->dax->tree);
-		fi->dax->nr++;
+		/* Protected by fi->vdax->sem */
+		interval_tree_insert(&dmap->itn, &fi->vdax->tree);
+		fi->vdax->nr++;
 		spin_lock(&fcd->lock);
 		list_add_tail(&dmap->busy_list, &fcd->busy_ranges);
 		fcd->nr_busy_ranges++;
@@ -288,7 +288,7 @@ static int dmap_removemapping_list(struct inode *inode, unsigned int num,
  * Cleanup dmap entry and add back to free list. This should be called with
  * fcd->lock held.
  */
-static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,
+static void dmap_reinit_add_to_free_pool(struct fuse_conn_vdax *fcd,
 					    struct fuse_dax_mapping *dmap)
 {
 	pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n",
@@ -306,7 +306,7 @@ static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,
  * called from evict_inode() path where we know all dmap entries can be
  * reclaimed.
  */
-static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
+static void inode_reclaim_dmap_range(struct fuse_conn_vdax *fcd,
 				     struct inode *inode,
 				     loff_t start, loff_t end)
 {
@@ -319,14 +319,14 @@ static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
 	struct interval_tree_node *node;
 
 	while (1) {
-		node = interval_tree_iter_first(&fi->dax->tree, start_idx,
+		node = interval_tree_iter_first(&fi->vdax->tree, start_idx,
 						end_idx);
 		if (!node)
 			break;
 		dmap = node_to_dmap(node);
 		/* inode is going away. There should not be any users of dmap */
 		WARN_ON(refcount_read(&dmap->refcnt) > 1);
-		interval_tree_remove(&dmap->itn, &fi->dax->tree);
+		interval_tree_remove(&dmap->itn, &fi->vdax->tree);
 		num++;
 		list_add(&dmap->list, &to_remove);
 	}
@@ -335,8 +335,8 @@ static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
 	if (list_empty(&to_remove))
 		return;
 
-	WARN_ON(fi->dax->nr < num);
-	fi->dax->nr -= num;
+	WARN_ON(fi->vdax->nr < num);
+	fi->vdax->nr -= num;
 	err = dmap_removemapping_list(inode, num, &to_remove);
 	if (err && err != -ENOTCONN) {
 		pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n",
@@ -367,11 +367,11 @@ static int dmap_removemapping_one(struct inode *inode,
 
 /*
  * It is called from evict_inode() and by that time inode is going away. So
- * this function does not take any locks like fi->dax->sem for traversing
+ * this function does not take any locks like fi->vdax->sem for traversing
  * that fuse inode interval tree. If that lock is taken then lock validator
  * complains of deadlock situation w.r.t fs_reclaim lock.
  */
-void fuse_dax_inode_cleanup(struct inode *inode)
+void fuse_vdax_inode_cleanup(struct inode *inode)
 {
 	struct fuse_conn *fc = get_fuse_conn(inode);
 	struct fuse_inode *fi = get_fuse_inode(inode);
@@ -381,8 +381,8 @@ void fuse_dax_inode_cleanup(struct inode *inode)
 	 * before we arrive here. So we should not have to worry about any
 	 * pages/exception entries still associated with inode.
 	 */
-	inode_reclaim_dmap_range(fc->dax, inode, 0, -1);
-	WARN_ON(fi->dax->nr);
+	inode_reclaim_dmap_range(fc->vdax, inode, 0, -1);
+	WARN_ON(fi->vdax->nr);
 }
 
 static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length)
@@ -414,7 +414,7 @@ static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length,
 		iomap->type = IOMAP_MAPPED;
 		/*
 		 * increace refcnt so that reclaim code knows this dmap is in
-		 * use. This assumes fi->dax->sem mutex is held either
+		 * use. This assumes fi->vdax->sem mutex is held either
 		 * shared/exclusive.
 		 */
 		refcount_inc(&dmap->refcnt);
@@ -434,7 +434,7 @@ static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
 {
 	struct fuse_inode *fi = get_fuse_inode(inode);
 	struct fuse_conn *fc = get_fuse_conn(inode);
-	struct fuse_conn_dax *fcd = fc->dax;
+	struct fuse_conn_vdax *fcd = fc->vdax;
 	struct fuse_dax_mapping *dmap, *alloc_dmap = NULL;
 	int ret;
 	bool writable = flags & IOMAP_WRITE;
@@ -469,17 +469,17 @@ static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
 	 * Take write lock so that only one caller can try to setup mapping
 	 * and other waits.
 	 */
-	down_write(&fi->dax->sem);
+	down_write(&fi->vdax->sem);
 	/*
 	 * We dropped lock. Check again if somebody else setup
 	 * mapping already.
 	 */
-	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
+	node = interval_tree_iter_first(&fi->vdax->tree, start_idx, start_idx);
 	if (node) {
 		dmap = node_to_dmap(node);
 		fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
 		dmap_add_to_free_pool(fcd, alloc_dmap);
-		up_write(&fi->dax->sem);
+		up_write(&fi->vdax->sem);
 		return 0;
 	}
 
@@ -488,11 +488,11 @@ static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
 				     writable, false);
 	if (ret < 0) {
 		dmap_add_to_free_pool(fcd, alloc_dmap);
-		up_write(&fi->dax->sem);
+		up_write(&fi->vdax->sem);
 		return ret;
 	}
 	fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags);
-	up_write(&fi->dax->sem);
+	up_write(&fi->vdax->sem);
 	return 0;
 }
 
@@ -510,14 +510,14 @@ static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
 	 * Take exclusive lock so that only one caller can try to setup
 	 * mapping and others wait.
 	 */
-	down_write(&fi->dax->sem);
-	node = interval_tree_iter_first(&fi->dax->tree, idx, idx);
+	down_write(&fi->vdax->sem);
+	node = interval_tree_iter_first(&fi->vdax->tree, idx, idx);
 
 	/* We are holding either inode lock or invalidate_lock, and that should
 	 * ensure that dmap can't be truncated. We are holding a reference
 	 * on dmap and that should make sure it can't be reclaimed. So dmap
 	 * should still be there in tree despite the fact we dropped and
-	 * re-acquired the fi->dax->sem lock.
+	 * re-acquired the fi->vdax->sem lock.
 	 */
 	ret = -EIO;
 	if (WARN_ON(!node))
@@ -526,7 +526,7 @@ static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
 	dmap = node_to_dmap(node);
 
 	/* We took an extra reference on dmap to make sure its not reclaimd.
-	 * Now we hold fi->dax->sem lock and that reference is not needed
+	 * Now we hold fi->vdax->sem lock and that reference is not needed
 	 * anymore. Drop it.
 	 */
 	if (refcount_dec_and_test(&dmap->refcnt)) {
@@ -551,7 +551,7 @@ static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
 out_fill_iomap:
 	fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
 out_err:
-	up_write(&fi->dax->sem);
+	up_write(&fi->vdax->sem);
 	return ret;
 }
 
@@ -576,7 +576,7 @@ static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
 	iomap->offset = pos;
 	iomap->flags = 0;
 	iomap->bdev = NULL;
-	iomap->dax_dev = fc->dax->dev;
+	iomap->dax_dev = fc->vdax->dev;
 
 	/*
 	 * Both read/write and mmap path can race here. So we need something
@@ -585,33 +585,33 @@ static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
 	 * For now, use a semaphore for this. It probably needs to be
 	 * optimized later.
 	 */
-	down_read(&fi->dax->sem);
-	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
+	down_read(&fi->vdax->sem);
+	node = interval_tree_iter_first(&fi->vdax->tree, start_idx, start_idx);
 	if (node) {
 		dmap = node_to_dmap(node);
 		if (writable && !dmap->writable) {
 			/* Upgrade read-only mapping to read-write. This will
-			 * require exclusive fi->dax->sem lock as we don't want
+			 * require exclusive fi->vdax->sem lock as we don't want
 			 * two threads to be trying to this simultaneously
 			 * for same dmap. So drop shared lock and acquire
 			 * exclusive lock.
 			 *
-			 * Before dropping fi->dax->sem lock, take reference
+			 * Before dropping fi->vdax->sem lock, take reference
 			 * on dmap so that its not freed by range reclaim.
 			 */
 			refcount_inc(&dmap->refcnt);
-			up_read(&fi->dax->sem);
+			up_read(&fi->vdax->sem);
 			pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n",
 				 __func__, pos, length);
 			return fuse_upgrade_dax_mapping(inode, pos, length,
 							flags, iomap);
 		} else {
 			fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
-			up_read(&fi->dax->sem);
+			up_read(&fi->vdax->sem);
 			return 0;
 		}
 	} else {
-		up_read(&fi->dax->sem);
+		up_read(&fi->vdax->sem);
 		pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n",
 				__func__, pos, length);
 		if (pos >= i_size_read(inode))
@@ -668,14 +668,14 @@ static void fuse_wait_dax_page(struct inode *inode)
 }
 
 /* Should be called with mapping->invalidate_lock held exclusively. */
-int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start,
+int fuse_vdax_break_layouts(struct inode *inode, u64 dmap_start,
 				  u64 dmap_end)
 {
 	return dax_break_layout(inode, dmap_start, dmap_end,
 				fuse_wait_dax_page);
 }
 
-ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
+ssize_t fuse_vdax_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	struct inode *inode = file_inode(iocb->ki_filp);
 	ssize_t ret;
@@ -715,7 +715,7 @@ static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from)
 	return ret;
 }
 
-ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
+ssize_t fuse_vdax_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct inode *inode = file_inode(iocb->ki_filp);
 	ssize_t ret;
@@ -761,7 +761,7 @@ static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order,
 	unsigned long pfn;
 	int error = 0;
 	struct fuse_conn *fc = get_fuse_conn(inode);
-	struct fuse_conn_dax *fcd = fc->dax;
+	struct fuse_conn_vdax *fcd = fc->vdax;
 	bool retry = false;
 
 	if (write)
@@ -822,11 +822,12 @@ static const struct vm_operations_struct fuse_dax_vm_ops = {
 	.pfn_mkwrite	= fuse_dax_pfn_mkwrite,
 };
 
-int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma)
+int fuse_vdax_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	file_accessed(file);
 	vma->vm_ops = &fuse_dax_vm_ops;
-	vm_flags_set(vma, VM_MIXEDMAP | VM_HUGEPAGE);
+	vma_set_flags(vma, VMA_HUGEPAGE_BIT);
+	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
 	return 0;
 }
 
@@ -869,8 +870,8 @@ static int reclaim_one_dmap_locked(struct inode *inode,
 		return ret;
 
 	/* Remove dax mapping from inode interval tree now */
-	interval_tree_remove(&dmap->itn, &fi->dax->tree);
-	fi->dax->nr--;
+	interval_tree_remove(&dmap->itn, &fi->vdax->tree);
+	fi->vdax->nr--;
 
 	/* It is possible that umount/shutdown has killed the fuse connection
 	 * and worker thread is trying to reclaim memory in parallel.  Don't
@@ -885,7 +886,7 @@ static int reclaim_one_dmap_locked(struct inode *inode,
 }
 
 /* Find first mapped dmap for an inode and return file offset. Caller needs
- * to hold fi->dax->sem lock either shared or exclusive.
+ * to hold fi->vdax->sem lock either shared or exclusive.
  */
 static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
 {
@@ -893,7 +894,7 @@ static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
 	struct fuse_dax_mapping *dmap;
 	struct interval_tree_node *node;
 
-	for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node;
+	for (node = interval_tree_iter_first(&fi->vdax->tree, 0, -1); node;
 	     node = interval_tree_iter_next(node, 0, -1)) {
 		dmap = node_to_dmap(node);
 		/* still in use. */
@@ -911,7 +912,7 @@ static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
  * it back to free pool.
  */
 static struct fuse_dax_mapping *
-inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
+inode_inline_reclaim_one_dmap(struct fuse_conn_vdax *fcd, struct inode *inode,
 			      bool *retry)
 {
 	struct fuse_inode *fi = get_fuse_inode(inode);
@@ -924,14 +925,14 @@ inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
 	filemap_invalidate_lock(inode->i_mapping);
 
 	/* Lookup a dmap and corresponding file offset to reclaim. */
-	down_read(&fi->dax->sem);
+	down_read(&fi->vdax->sem);
 	dmap = inode_lookup_first_dmap(inode);
 	if (dmap) {
 		start_idx = dmap->itn.start;
 		dmap_start = start_idx << FUSE_DAX_SHIFT;
 		dmap_end = dmap_start + FUSE_DAX_SZ - 1;
 	}
-	up_read(&fi->dax->sem);
+	up_read(&fi->vdax->sem);
 
 	if (!dmap)
 		goto out_mmap_sem;
@@ -939,16 +940,16 @@ inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
 	 * Make sure there are no references to inode pages using
 	 * get_user_pages()
 	 */
-	ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
+	ret = fuse_vdax_break_layouts(inode, dmap_start, dmap_end);
 	if (ret) {
-		pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n",
+		pr_debug("fuse: fuse_vdax_break_layouts() failed. err=%d\n",
 			 ret);
 		dmap = ERR_PTR(ret);
 		goto out_mmap_sem;
 	}
 
-	down_write(&fi->dax->sem);
-	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
+	down_write(&fi->vdax->sem);
+	node = interval_tree_iter_first(&fi->vdax->tree, start_idx, start_idx);
 	/* Range already got reclaimed by somebody else */
 	if (!node) {
 		if (retry)
@@ -980,14 +981,14 @@ inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
 		 __func__, inode, dmap->window_offset, dmap->length);
 
 out_write_dmap_sem:
-	up_write(&fi->dax->sem);
+	up_write(&fi->vdax->sem);
 out_mmap_sem:
 	filemap_invalidate_unlock(inode->i_mapping);
 	return dmap;
 }
 
 static struct fuse_dax_mapping *
-alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
+alloc_dax_mapping_reclaim(struct fuse_conn_vdax *fcd, struct inode *inode)
 {
 	struct fuse_dax_mapping *dmap;
 	struct fuse_inode *fi = get_fuse_inode(inode);
@@ -1014,18 +1015,18 @@ alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
 		 * if a deadlock is possible if we sleep with
 		 * mapping->invalidate_lock held and worker to free memory
 		 * can't make progress due to unavailability of
-		 * mapping->invalidate_lock.  So sleep only if fi->dax->nr=0
+		 * mapping->invalidate_lock.  So sleep only if fi->vdax->nr=0
 		 */
 		if (retry)
 			continue;
 		/*
 		 * There are no mappings which can be reclaimed. Wait for one.
-		 * We are not holding fi->dax->sem. So it is possible
+		 * We are not holding fi->vdax->sem. So it is possible
 		 * that range gets added now. But as we are not holding
 		 * mapping->invalidate_lock, worker should still be able to
 		 * free up a range and wake us up.
 		 */
-		if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) {
+		if (!fi->vdax->nr && !(fcd->nr_free_ranges > 0)) {
 			if (wait_event_killable_exclusive(fcd->range_waitq,
 					(fcd->nr_free_ranges > 0))) {
 				return ERR_PTR(-EINTR);
@@ -1034,7 +1035,7 @@ alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
 	}
 }
 
-static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
+static int lookup_and_reclaim_dmap_locked(struct fuse_conn_vdax *fcd,
 					  struct inode *inode,
 					  unsigned long start_idx)
 {
@@ -1044,7 +1045,7 @@ static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
 	struct interval_tree_node *node;
 
 	/* Find fuse dax mapping at file offset inode. */
-	node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
+	node = interval_tree_iter_first(&fi->vdax->tree, start_idx, start_idx);
 
 	/* Range already got cleaned up by somebody else */
 	if (!node)
@@ -1070,10 +1071,10 @@ static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
  * Free a range of memory.
  * Locking:
  * 1. Take mapping->invalidate_lock to block dax faults.
- * 2. Take fi->dax->sem to protect interval tree and also to make sure
+ * 2. Take fi->vdax->sem to protect interval tree and also to make sure
  *    read/write can not reuse a dmap which we might be freeing.
  */
-static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,
+static int lookup_and_reclaim_dmap(struct fuse_conn_vdax *fcd,
 				   struct inode *inode,
 				   unsigned long start_idx,
 				   unsigned long end_idx)
@@ -1084,22 +1085,22 @@ static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,
 	loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1;
 
 	filemap_invalidate_lock(inode->i_mapping);
-	ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
+	ret = fuse_vdax_break_layouts(inode, dmap_start, dmap_end);
 	if (ret) {
-		pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n",
+		pr_debug("virtio_fs: fuse_vdax_break_layouts() failed. err=%d\n",
 			 ret);
 		goto out_mmap_sem;
 	}
 
-	down_write(&fi->dax->sem);
+	down_write(&fi->vdax->sem);
 	ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx);
-	up_write(&fi->dax->sem);
+	up_write(&fi->vdax->sem);
 out_mmap_sem:
 	filemap_invalidate_unlock(inode->i_mapping);
 	return ret;
 }
 
-static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,
+static int try_to_free_dmap_chunks(struct fuse_conn_vdax *fcd,
 				   unsigned long nr_to_free)
 {
 	struct fuse_dax_mapping *dmap, *pos, *temp;
@@ -1160,7 +1161,7 @@ static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,
 static void fuse_dax_free_mem_worker(struct work_struct *work)
 {
 	int ret;
-	struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax,
+	struct fuse_conn_vdax *fcd = container_of(work, struct fuse_conn_vdax,
 						 free_work.work);
 	ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK);
 	if (ret) {
@@ -1185,16 +1186,16 @@ static void fuse_free_dax_mem_ranges(struct list_head *mem_list)
 	}
 }
 
-void fuse_dax_conn_free(struct fuse_conn *fc)
+void fuse_vdax_conn_free(struct fuse_conn *fc)
 {
-	if (fc->dax) {
-		fuse_free_dax_mem_ranges(&fc->dax->free_ranges);
-		kfree(fc->dax);
-		fc->dax = NULL;
+	if (fc->vdax) {
+		fuse_free_dax_mem_ranges(&fc->vdax->free_ranges);
+		kfree(fc->vdax);
+		fc->vdax = NULL;
 	}
 }
 
-static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)
+static int fuse_dax_mem_range_init(struct fuse_conn_vdax *fcd)
 {
 	long nr_pages, nr_ranges;
 	struct fuse_dax_mapping *range;
@@ -1246,13 +1247,13 @@ static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)
 	return ret;
 }
 
-int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode dax_mode,
+int fuse_vdax_conn_alloc(struct fuse_conn *fc, enum fuse_vdax_mode dax_mode,
 			struct dax_device *dax_dev)
 {
-	struct fuse_conn_dax *fcd;
+	struct fuse_conn_vdax *fcd;
 	int err;
 
-	fc->dax_mode = dax_mode;
+	fc->vdax_mode = dax_mode;
 
 	if (!dax_dev)
 		return 0;
@@ -1269,22 +1270,22 @@ int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode dax_mode,
 		return err;
 	}
 
-	fc->dax = fcd;
+	fc->vdax = fcd;
 	return 0;
 }
 
-bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi)
+bool fuse_vdax_inode_alloc(struct super_block *sb, struct fuse_inode *fi)
 {
 	struct fuse_conn *fc = get_fuse_conn_super(sb);
 
-	fi->dax = NULL;
-	if (fc->dax) {
-		fi->dax = kzalloc_obj(*fi->dax, GFP_KERNEL_ACCOUNT);
-		if (!fi->dax)
+	fi->vdax = NULL;
+	if (fc->vdax) {
+		fi->vdax = kzalloc_obj(*fi->vdax, GFP_KERNEL_ACCOUNT);
+		if (!fi->vdax)
 			return false;
 
-		init_rwsem(&fi->dax->sem);
-		fi->dax->tree = RB_ROOT_CACHED;
+		init_rwsem(&fi->vdax->sem);
+		fi->vdax->tree = RB_ROOT_CACHED;
 	}
 
 	return true;
@@ -1298,26 +1299,26 @@ static const struct address_space_operations fuse_dax_file_aops  = {
 static bool fuse_should_enable_dax(struct inode *inode, unsigned int flags)
 {
 	struct fuse_conn *fc = get_fuse_conn(inode);
-	enum fuse_dax_mode dax_mode = fc->dax_mode;
+	enum fuse_vdax_mode dax_mode = fc->vdax_mode;
 
-	if (dax_mode == FUSE_DAX_NEVER)
+	if (dax_mode == FUSE_VDAX_NEVER)
 		return false;
 
 	/*
-	 * fc->dax may be NULL in 'inode' mode when filesystem device doesn't
+	 * fc->vdax may be NULL in 'inode' mode when filesystem device doesn't
 	 * support DAX, in which case it will silently fallback to 'never' mode.
 	 */
-	if (!fc->dax)
+	if (!fc->vdax)
 		return false;
 
-	if (dax_mode == FUSE_DAX_ALWAYS)
+	if (dax_mode == FUSE_VDAX_ALWAYS)
 		return true;
 
 	/* dax_mode is FUSE_DAX_INODE* */
-	return fc->inode_dax && (flags & FUSE_ATTR_DAX);
+	return fc->inode_vdax && (flags & FUSE_ATTR_DAX);
 }
 
-void fuse_dax_inode_init(struct inode *inode, unsigned int flags)
+void fuse_vdax_inode_init(struct inode *inode, unsigned int flags)
 {
 	if (!fuse_should_enable_dax(inode, flags))
 		return;
@@ -1326,18 +1327,18 @@ void fuse_dax_inode_init(struct inode *inode, unsigned int flags)
 	inode->i_data.a_ops = &fuse_dax_file_aops;
 }
 
-void fuse_dax_dontcache(struct inode *inode, unsigned int flags)
+void fuse_vdax_dontcache(struct inode *inode, unsigned int flags)
 {
 	struct fuse_conn *fc = get_fuse_conn(inode);
 
-	if (fuse_is_inode_dax_mode(fc->dax_mode) &&
+	if (fuse_is_inode_vdax_mode(fc->vdax_mode) &&
 	    ((bool) IS_DAX(inode) != (bool) (flags & FUSE_ATTR_DAX)))
 		d_mark_dontcache(inode);
 }
 
-bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
+bool fuse_vdax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
 {
-	if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) {
+	if (fc->vdax && (map_alignment > FUSE_DAX_SHIFT)) {
 		pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n",
 			map_alignment, FUSE_DAX_SZ);
 		return false;
@@ -1345,12 +1346,12 @@ bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
 	return true;
 }
 
-void fuse_dax_cancel_work(struct fuse_conn *fc)
+void fuse_vdax_cancel_work(struct fuse_conn *fc)
 {
-	struct fuse_conn_dax *fcd = fc->dax;
+	struct fuse_conn_vdax *fcd = fc->vdax;
 
 	if (fcd)
 		cancel_delayed_work_sync(&fcd->free_work);
 
 }
-EXPORT_SYMBOL_GPL(fuse_dax_cancel_work);
+EXPORT_SYMBOL_GPL(fuse_vdax_cancel_work);
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 4fec31fc0b8450..2d7ee5498f1c3c 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -1061,7 +1061,8 @@ static int fuse_copy_fill(struct fuse_copy_state *cs)
 		err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off);
 		if (err < 0)
 			return err;
-		BUG_ON(!err);
+		if (!err)
+			return -EIO;
 		cs->len = err;
 		cs->offset = off;
 		cs->pg = page;
@@ -1252,31 +1253,10 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio,
  * done atomically
  */
 int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
-		    unsigned offset, unsigned count, int zeroing)
+		    unsigned offset, unsigned count)
 {
 	int err;
 	struct folio *folio = *foliop;
-	size_t size;
-
-	if (folio) {
-		size = folio_size(folio);
-		if (zeroing && count < size) {
-			/*
-			 * When the copy is skipped the folio already holds the
-			 * payload, so only the bytes outside [offset, offset +
-			 * count) may be zeroed.
-			 *
-			 * Otherwise, the whole folio is cleared first so that a
-			 * failed copy leaves zeros rather than stale folio
-			 * contents.
-			 */
-			if (cs->skip_folio_copy)
-				folio_zero_segments(folio, 0, offset,
-						    offset + count, size);
-			else
-				folio_zero_range(folio, 0, size);
-		}
-	}
 
 	while (!cs->skip_folio_copy && count) {
 		if (cs->write && cs->pipebufs && folio) {
@@ -1293,7 +1273,7 @@ int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
 			}
 		} else if (!cs->len) {
 			if (cs->move_folios && folio &&
-			    offset == 0 && count == size) {
+			    offset == 0 && count == folio_size(folio)) {
 				err = fuse_try_move_folio(cs, foliop);
 				if (err <= 0)
 					return err;
@@ -1309,7 +1289,8 @@ int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
 			unsigned int copy = count;
 			unsigned int bytes_copied;
 
-			if (folio_test_highmem(folio) && count > PAGE_SIZE - offset_in_page(offset))
+			if (folio_test_partial_kmap(folio) &&
+			    count > PAGE_SIZE - offset_in_page(offset))
 				copy = PAGE_SIZE - offset_in_page(offset);
 
 			bytes_copied = fuse_copy_do(cs, &buf, &copy);
@@ -1334,10 +1315,25 @@ static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes,
 
 	for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) {
 		int err;
+		struct folio *folio = ap->folios[i];
 		unsigned int offset = ap->descs[i].offset;
-		unsigned int count = min(nbytes, ap->descs[i].length);
+		unsigned int length = ap->descs[i].length;
+		unsigned int count = min(nbytes, length);
+
+		/*
+		 * The reply may be shorter than what was asked for. The full
+		 * descs[i].length is reported as read, so the tail bytes the
+		 * server did not send are about to be marked uptodate and need
+		 * to be zeroed.
+		 *
+		 * Only [offset, offset + length) can be touched since the
+		 * rest of the folio can hold blocks that are already uptodate
+		 * or dirty, and clearing those would lose data.
+		 */
+		if (folio && zeroing && count < length)
+			folio_zero_range(folio, offset + count, length - count);
 
-		err = fuse_copy_folio(cs, &ap->folios[i], offset, count, zeroing);
+		err = fuse_copy_folio(cs, &ap->folios[i], offset, count);
 		if (err)
 			return err;
 
diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h
index 8d25378c091830..f6c47ae0395bba 100644
--- a/fs/fuse/dev.h
+++ b/fs/fuse/dev.h
@@ -90,7 +90,7 @@ int fuse_backing_close(struct fuse_conn *fc, int backing_id);
 
 int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size);
 int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
-		    unsigned offset, unsigned count, int zeroing);
+		    unsigned offset, unsigned count);
 void fuse_copy_finish(struct fuse_copy_state *cs);
 
 #ifdef CONFIG_FUSE_IO_URING
diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index c6dd420c4034a6..460776cd69cdc6 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -893,6 +893,12 @@ static int fuse_uring_args_to_ring(struct fuse_req *req,
 	fuse_copy_finish(&cs);
 	if (err) {
 		pr_info_ratelimited("%s fuse_copy_args failed\n", __func__);
+		/*
+		 * fuse-uring does not use pipe buffers, so -EIO here can only
+		 * mean the in-args did not fit ent->payload
+		 */
+		if (err == -EIO && args->opcode == FUSE_SETXATTR)
+			err = -E2BIG;
 		return err;
 	}
 
diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
index e49b4e874b15f8..d11e2aabbfd59b 100644
--- a/fs/fuse/dir.c
+++ b/fs/fuse/dir.c
@@ -751,7 +751,7 @@ static u32 fuse_ext_size(size_t size)
 /*
  * This adds just a single supplementary group that matches the parent's group.
  */
-static int get_create_supp_group(struct mnt_idmap *idmap,
+static int get_create_supp_group(const struct mnt_idmap *idmap,
 				 struct inode *dir,
 				 struct fuse_in_arg *ext)
 {
@@ -782,7 +782,7 @@ static int get_create_supp_group(struct mnt_idmap *idmap,
 	return 0;
 }
 
-static int get_create_ext(struct mnt_idmap *idmap,
+static int get_create_ext(const struct mnt_idmap *idmap,
 			  struct fuse_args *args,
 			  struct inode *dir, struct dentry *dentry,
 			  umode_t mode)
@@ -820,7 +820,7 @@ static void free_ext_value(struct fuse_args *args)
  * If the filesystem doesn't support this, then fall back to separate
  * 'mknod' + 'open' requests.
  */
-static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir,
+static int fuse_create_open(const struct mnt_idmap *idmap, struct inode *dir,
 			    struct dentry *entry, struct file *file,
 			    unsigned int flags, umode_t mode, u32 opcode)
 {
@@ -934,14 +934,14 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int fuse_mknod(struct mnt_idmap *, struct inode *, struct dentry *,
+static int fuse_mknod(const struct mnt_idmap *, struct inode *, struct dentry *,
 		      umode_t, dev_t);
 static int fuse_atomic_open(struct inode *dir, struct dentry *entry,
 			    struct file *file, unsigned flags,
 			    umode_t mode)
 {
 	int err;
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct fuse_conn *fc = get_fuse_conn(dir);
 
 	if (fuse_is_bad(dir))
@@ -980,7 +980,7 @@ static int fuse_atomic_open(struct inode *dir, struct dentry *entry,
 /*
  * Code shared between mknod, mkdir, symlink and link
  */
-static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm,
+static struct dentry *create_new_entry(const struct mnt_idmap *idmap, struct fuse_mount *fm,
 				       struct fuse_args *args, struct inode *dir,
 				       struct dentry *entry, umode_t mode)
 {
@@ -1053,7 +1053,7 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun
 	return ERR_PTR(err);
 }
 
-static int create_new_nondir(struct mnt_idmap *idmap, struct fuse_mount *fm,
+static int create_new_nondir(const struct mnt_idmap *idmap, struct fuse_mount *fm,
 			     struct fuse_args *args, struct inode *dir,
 			     struct dentry *entry, umode_t mode)
 {
@@ -1069,7 +1069,7 @@ static int create_new_nondir(struct mnt_idmap *idmap, struct fuse_mount *fm,
 	return PTR_ERR(create_new_entry(idmap, fm, args, dir, entry, mode));
 }
 
-static int fuse_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int fuse_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *entry, umode_t mode, dev_t rdev)
 {
 	struct fuse_mknod_in inarg;
@@ -1092,13 +1092,13 @@ static int fuse_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return create_new_nondir(idmap, fm, &args, dir, entry, mode);
 }
 
-static int fuse_create(struct mnt_idmap *idmap, struct inode *dir,
+static int fuse_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *entry, umode_t mode)
 {
 	return fuse_mknod(idmap, dir, entry, mode, 0);
 }
 
-static int fuse_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int fuse_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			struct file *file, umode_t mode)
 {
 	struct fuse_conn *fc = get_fuse_conn(dir);
@@ -1116,7 +1116,7 @@ static int fuse_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static struct dentry *fuse_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *fuse_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *entry, umode_t mode)
 {
 	struct fuse_mkdir_in inarg;
@@ -1146,7 +1146,7 @@ static struct dentry *fuse_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	return create_new_entry(idmap, fm, &args, dir, entry, S_IFDIR);
 }
 
-static int fuse_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int fuse_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *entry, const char *link)
 {
 	struct fuse_mount *fm = get_fuse_mount(dir);
@@ -1256,9 +1256,10 @@ static int fuse_rmdir(struct inode *dir, struct dentry *entry)
 	return err;
 }
 
-static int fuse_rename_common(struct mnt_idmap *idmap, struct inode *olddir, struct dentry *oldent,
-			      struct inode *newdir, struct dentry *newent,
-			      unsigned int flags, int opcode, size_t argsize)
+static int fuse_rename_common(const struct mnt_idmap *idmap, struct inode *olddir,
+			      struct dentry *oldent, struct inode *newdir,
+			      struct dentry *newent, unsigned int flags,
+			      int opcode, size_t argsize)
 {
 	int err;
 	struct fuse_rename2_in inarg;
@@ -1306,7 +1307,7 @@ static int fuse_rename_common(struct mnt_idmap *idmap, struct inode *olddir, str
 	return err;
 }
 
-static int fuse_rename2(struct mnt_idmap *idmap, struct inode *olddir,
+static int fuse_rename2(const struct mnt_idmap *idmap, struct inode *olddir,
 			struct dentry *oldent, struct inode *newdir,
 			struct dentry *newent, unsigned int flags)
 {
@@ -1375,7 +1376,7 @@ static int fuse_link(struct dentry *entry, struct inode *newdir,
 	return err;
 }
 
-static void fuse_fillattr(struct mnt_idmap *idmap, struct inode *inode,
+static void fuse_fillattr(const struct mnt_idmap *idmap, struct inode *inode,
 			  struct fuse_attr *attr, struct kstat *stat)
 {
 	unsigned int blkbits;
@@ -1429,7 +1430,7 @@ static void fuse_statx_to_attr(struct fuse_statx *sx, struct fuse_attr *attr)
 	attr->blksize = sx->blksize;
 }
 
-static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode,
+static int fuse_do_statx(const struct mnt_idmap *idmap, struct inode *inode,
 			 struct file *file, struct kstat *stat)
 {
 	int err;
@@ -1490,7 +1491,7 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode,
 	return 0;
 }
 
-static int fuse_do_getattr(struct mnt_idmap *idmap, struct inode *inode,
+static int fuse_do_getattr(const struct mnt_idmap *idmap, struct inode *inode,
 			   struct kstat *stat, struct file *file)
 {
 	int err;
@@ -1536,7 +1537,7 @@ static int fuse_do_getattr(struct mnt_idmap *idmap, struct inode *inode,
 	return err;
 }
 
-static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode,
+static int fuse_update_get_attr(const struct mnt_idmap *idmap, struct inode *inode,
 				struct file *file, struct kstat *stat,
 				u32 request_mask, unsigned int flags)
 {
@@ -1762,7 +1763,7 @@ static int fuse_perm_getattr(struct inode *inode, int mask)
  * access request is sent.  Execute permission is still checked
  * locally based on file mode.
  */
-static int fuse_permission(struct mnt_idmap *idmap,
+static int fuse_permission(const struct mnt_idmap *idmap,
 			   struct inode *inode, int mask)
 {
 	struct fuse_conn *fc = get_fuse_conn(inode);
@@ -2000,7 +2001,7 @@ static bool update_mtime(unsigned ivalid, bool trust_local_mtime)
 	return true;
 }
 
-static void iattr_to_fattr(struct mnt_idmap *idmap, struct fuse_conn *fc,
+static void iattr_to_fattr(const struct mnt_idmap *idmap, struct fuse_conn *fc,
 			   struct iattr *iattr, struct fuse_setattr_in *arg,
 			   bool trust_local_cmtime)
 {
@@ -2142,7 +2143,7 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff)
  * vmtruncate() doesn't allow for this case, so do the rlimit checking
  * and the actual truncation by hand.
  */
-int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int fuse_do_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		    struct iattr *attr, struct file *file)
 {
 	struct inode *inode = d_inode(dentry);
@@ -2174,10 +2175,10 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		is_truncate = true;
 	}
 
-	if (FUSE_IS_DAX(inode) && is_truncate) {
+	if (FUSE_IS_VDAX(inode) && is_truncate) {
 		filemap_invalidate_lock(mapping);
 		fault_blocked = true;
-		err = fuse_dax_break_layouts(inode, 0, -1);
+		err = fuse_vdax_break_layouts(inode, 0, -1);
 		if (err)
 			goto unlock;
 	}
@@ -2323,7 +2324,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	return err;
 }
 
-static int fuse_setattr(struct mnt_idmap *idmap, struct dentry *entry,
+static int fuse_setattr(const struct mnt_idmap *idmap, struct dentry *entry,
 			struct iattr *attr)
 {
 	struct inode *inode = d_inode(entry);
@@ -2386,7 +2387,7 @@ static int fuse_setattr(struct mnt_idmap *idmap, struct dentry *entry,
 	return ret;
 }
 
-static int fuse_getattr(struct mnt_idmap *idmap,
+static int fuse_getattr(const struct mnt_idmap *idmap,
 			const struct path *path, struct kstat *stat,
 			u32 request_mask, unsigned int flags)
 {
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index 9a36d0329e221a..3d209e2b71ba74 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -119,7 +119,7 @@ static void fuse_file_put(struct fuse_file *ff, bool sync)
 			 * DAX inodes may need to issue a number of synchronous
 			 * request for clearing the mappings.
 			 */
-			if (ra && ra->inode && FUSE_IS_DAX(ra->inode))
+			if (ra && ra->inode && FUSE_IS_VDAX(ra->inode))
 				args->may_block = true;
 			args->end = fuse_release_end;
 			if (fuse_simple_background(ff->fm, args,
@@ -256,7 +256,7 @@ static int fuse_open(struct inode *inode, struct file *file)
 	int err;
 	bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc;
 	bool is_wb_truncate = is_truncate && fc->writeback_cache;
-	bool dax_truncate = is_truncate && FUSE_IS_DAX(inode);
+	bool vdax_truncate = is_truncate && FUSE_IS_VDAX(inode);
 
 	if (fuse_is_bad(inode))
 		return -EIO;
@@ -265,17 +265,17 @@ static int fuse_open(struct inode *inode, struct file *file)
 	if (err)
 		return err;
 
-	if (is_wb_truncate || dax_truncate)
+	if (is_wb_truncate || vdax_truncate)
 		inode_lock(inode);
 
-	if (dax_truncate) {
+	if (vdax_truncate) {
 		filemap_invalidate_lock(inode->i_mapping);
-		err = fuse_dax_break_layouts(inode, 0, -1);
+		err = fuse_vdax_break_layouts(inode, 0, -1);
 		if (err)
 			goto out_unlock;
 	}
 
-	if (is_wb_truncate || dax_truncate)
+	if (is_wb_truncate || vdax_truncate)
 		fuse_set_nowrite(inode);
 
 	err = fuse_do_open(fm, get_node_id(inode), file, false);
@@ -288,7 +288,7 @@ static int fuse_open(struct inode *inode, struct file *file)
 			fuse_truncate_update_attr(inode, file);
 	}
 
-	if (is_wb_truncate || dax_truncate)
+	if (is_wb_truncate || vdax_truncate)
 		fuse_release_nowrite(inode);
 	if (!err) {
 		if (is_truncate)
@@ -297,9 +297,9 @@ static int fuse_open(struct inode *inode, struct file *file)
 			invalidate_inode_pages2(inode->i_mapping);
 	}
 out_unlock:
-	if (dax_truncate)
+	if (vdax_truncate)
 		filemap_invalidate_unlock(inode->i_mapping);
-	if (is_wb_truncate || dax_truncate)
+	if (is_wb_truncate || vdax_truncate)
 		inode_unlock(inode);
 
 	return err;
@@ -311,7 +311,7 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
 	struct fuse_conn *fc = ff->fm->fc;
 	struct fuse_release_args *ra = &ff->args->release_args;
 
-	if (fuse_file_passthrough(ff))
+	if (fuse_is_passthrough(ff))
 		fuse_passthrough_release(ff, fuse_inode_backing(fi));
 
 	/* Inode is NULL on error path of fuse_create_open() */
@@ -689,7 +689,7 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
 		struct address_space *mapping = io->iocb->ki_filp->f_mapping;
 		ssize_t res = fuse_get_res_by_io(io);
 
-		if (res >= 0) {
+		if (res >= 0 && io->write) {
 			struct fuse_conn *fc = get_fuse_conn(inode);
 			struct fuse_inode *fi = get_fuse_inode(inode);
 
@@ -864,18 +864,29 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio,
 
 	attr_ver = fuse_get_attr_version(fm->fc);
 
-	/* Don't overflow end offset */
-	if (pos + (desc.length - 1) == LLONG_MAX)
-		desc.length--;
+	/*
+	 * Don't overflow end offset.
+	 *
+	 * Ask the server for len - 1 bytes. desc.length still holds the full
+	 * length. When the reply comes back, it will be one byte shorter than
+	 * desc.length and fuse_copy_folios() will zero that last byte.
+	 *
+	 * For this reason, desc.length must not be decremented too. The caller
+	 * reports the full length to iomap_finish_folio_read(), which marks
+	 * every block it covers uptodate. Shortening the descriptor would
+	 * suppress zeroing and leave the last byte holding stale data.
+	 */
+	if (pos + (len - 1) == LLONG_MAX)
+		len--;
 
-	fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
+	fuse_read_args_fill(&ia, file, pos, len, FUSE_READ);
 	res = fuse_simple_request(fm, &ia.ap.args);
 	if (res < 0)
 		return res;
 	/*
 	 * Short read means EOF.  If file size is larger, truncate it
 	 */
-	if (res < desc.length)
+	if (res < len)
 		fuse_short_read(inode, attr_ver, res, &ia.ap);
 
 	return 0;
@@ -1068,11 +1079,20 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
 	ap->args.page_zeroing = true;
 	ap->args.page_replace = true;
 
-	/* Don't overflow end offset */
-	if (pos + (count - 1) == LLONG_MAX) {
+	/*
+	 * Don't overflow end offset.
+	 *
+	 * Ask the server for count - 1 bytes. The reply is then one byte
+	 * shorter than what the descriptor lengths add up to, so
+	 * fuse_copy_folios() zeroes the last byte when it walks the folios.
+	 *
+	 * ap->descs[] must not be decremented here. It is what
+	 * fuse_readpages_end() reports back to iomap_finish_folio_read(), and
+	 * iomap has already accounted the full descriptor length, so shortening
+	 * it would leave ifs->read_bytes_pending nonzero and the folio locked.
+	 */
+	if (pos + (count - 1) == LLONG_MAX)
 		count--;
-		ap->descs[ap->num_folios - 1].length--;
-	}
 	WARN_ON((loff_t) (pos + count) < 0);
 
 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
@@ -1487,7 +1507,7 @@ static const struct iomap_write_ops fuse_iomap_write_ops = {
 static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct file *file = iocb->ki_filp;
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct address_space *mapping = file->f_mapping;
 	ssize_t written = 0;
 	struct inode *inode = mapping->host;
@@ -1835,13 +1855,13 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	if (fuse_is_bad(inode))
 		return -EIO;
 
-	if (FUSE_IS_DAX(inode))
-		return fuse_dax_read_iter(iocb, to);
+	if (FUSE_IS_VDAX(inode))
+		return fuse_vdax_read_iter(iocb, to);
 
 	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
 	if (ff->open_flags & FOPEN_DIRECT_IO)
 		return fuse_direct_read_iter(iocb, to);
-	else if (fuse_file_passthrough(ff))
+	else if (fuse_is_passthrough(ff))
 		return fuse_passthrough_read_iter(iocb, to);
 	else
 		return fuse_cache_read_iter(iocb, to);
@@ -1856,13 +1876,13 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	if (fuse_is_bad(inode))
 		return -EIO;
 
-	if (FUSE_IS_DAX(inode))
-		return fuse_dax_write_iter(iocb, from);
+	if (FUSE_IS_VDAX(inode))
+		return fuse_vdax_write_iter(iocb, from);
 
 	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
 	if (ff->open_flags & FOPEN_DIRECT_IO)
 		return fuse_direct_write_iter(iocb, from);
-	else if (fuse_file_passthrough(ff))
+	else if (fuse_is_passthrough(ff))
 		return fuse_passthrough_write_iter(iocb, from);
 	else
 		return fuse_cache_write_iter(iocb, from);
@@ -1875,7 +1895,10 @@ static ssize_t fuse_splice_read(struct file *in, loff_t *ppos,
 	struct fuse_file *ff = in->private_data;
 
 	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
-	if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
+
+	if (ff->open_flags & FOPEN_DIRECT_IO)
+		return copy_splice_read(in, ppos, pipe, len, flags);
+	else if (fuse_is_passthrough(ff))
 		return fuse_passthrough_splice_read(in, ppos, pipe, len, flags);
 	else
 		return filemap_splice_read(in, ppos, pipe, len, flags);
@@ -1887,7 +1910,7 @@ static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out,
 	struct fuse_file *ff = out->private_data;
 
 	/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
-	if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
+	if (fuse_is_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO))
 		return fuse_passthrough_splice_write(pipe, out, ppos, len, flags);
 	else
 		return iter_file_splice_write(pipe, out, ppos, len, flags);
@@ -2394,15 +2417,15 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
 	int rc;
 
 	/* DAX mmap is superior to direct_io mmap */
-	if (FUSE_IS_DAX(inode))
-		return fuse_dax_mmap(file, vma);
+	if (FUSE_IS_VDAX(inode))
+		return fuse_vdax_mmap(file, vma);
 
 	/*
 	 * If inode is in passthrough io mode, because it has some file open
 	 * in passthrough mode, either mmap to backing file or fail mmap,
 	 * because mixing cached mmap and passthrough io mode is not allowed.
 	 */
-	if (fuse_file_passthrough(ff))
+	if (fuse_is_passthrough(ff))
 		return fuse_passthrough_mmap(file, vma);
 	else if (fuse_inode_backing(get_fuse_inode(inode)))
 		return -ENODEV;
@@ -2844,7 +2867,7 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
 		.mode = mode
 	};
 	int err;
-	bool block_faults = FUSE_IS_DAX(inode) &&
+	bool block_faults = FUSE_IS_VDAX(inode) &&
 		(!(mode & FALLOC_FL_KEEP_SIZE) ||
 		 (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)));
 
@@ -2858,7 +2881,7 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
 	inode_lock(inode);
 	if (block_faults) {
 		filemap_invalidate_lock(inode->i_mapping);
-		err = fuse_dax_break_layouts(inode, 0, -1);
+		err = fuse_vdax_break_layouts(inode, 0, -1);
 		if (err)
 			goto out;
 	}
@@ -2978,14 +3001,16 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
 
 	/*
 	 * Write out dirty pages in the destination file before sending the COPY
-	 * request to userspace.  After the request is completed, truncate off
-	 * pages (including partial ones) from the cache that have been copied,
-	 * since these contain stale data at that point.
+	 * request to userspace.  After the request is completed, drop the
+	 * folios covering the copied range from the cache, since these contain
+	 * stale data at that point.
 	 *
-	 * This should be mostly correct, but if the COPY writes to partial
-	 * pages (at the start or end) and the parts not covered by the COPY are
+	 * This should be mostly correct, but if the COPY writes to a partial
+	 * folio (at the start or end) and the parts not covered by the COPY are
 	 * written through a memory map after calling fuse_writeback_range(),
-	 * then these partial page modifications will be lost on truncation.
+	 * then that folio is laundered before it is dropped, so the memory map
+	 * modifications are written back over the range the server just copied
+	 * into and the copied data is lost.
 	 *
 	 * It is unlikely that someone would rely on such mixed style
 	 * modifications.  Yet this does give less guarantees than if the
@@ -3038,9 +3063,10 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
 		goto out;
 	}
 
-	truncate_inode_pages_range(inode_out->i_mapping,
-				   ALIGN_DOWN(pos_out, PAGE_SIZE),
-				   ALIGN(pos_out + bytes_copied, PAGE_SIZE) - 1);
+	if (bytes_copied)
+		invalidate_inode_pages2_range(inode_out->i_mapping,
+					      pos_out >> PAGE_SHIFT,
+					      (pos_out + bytes_copied - 1) >> PAGE_SHIFT);
 
 	file_update_time(file_out);
 	fuse_write_update_attr(inode_out, pos_out + bytes_copied, bytes_copied);
@@ -3113,6 +3139,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags)
 {
 	struct fuse_inode *fi = get_fuse_inode(inode);
 	struct fuse_conn *fc = get_fuse_conn(inode);
+	unsigned int max_folio_pages;
 
 	inode->i_fop = &fuse_file_operations;
 	inode->i_data.a_ops = &fuse_file_aops;
@@ -3126,6 +3153,20 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags)
 	init_waitqueue_head(&fi->page_waitq);
 	init_waitqueue_head(&fi->direct_io_waitq);
 
-	if (IS_ENABLED(CONFIG_FUSE_DAX))
-		fuse_dax_inode_init(inode, flags);
+	if (IS_ENABLED(CONFIG_FUSE_VDAX))
+		fuse_vdax_inode_init(inode, flags);
+
+	if (FUSE_IS_VDAX(inode))
+		return;
+
+	/*
+	 * A folio is never split across requests so cap the order so that one
+	 * always fits in a single request.
+	 */
+	max_folio_pages = min3(fc->max_write >> PAGE_SHIFT,
+			       fc->max_read >> PAGE_SHIFT, fc->max_pages);
+
+	if (max_folio_pages)
+		mapping_set_folio_order_range(inode->i_mapping, 0,
+					      ilog2(max_folio_pages));
 }
diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index c8d4c5f3af7e8c..87e2bd9d4bb11f 100644
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -218,11 +218,11 @@ struct fuse_inode {
 	/** @lock: Lock to protect write-related fields */
 	spinlock_t lock;
 
-#ifdef CONFIG_FUSE_DAX
+#ifdef CONFIG_FUSE_VDAX
 	/**
-	 * @dax: Dax specific inode data
+	 * @vdax: Virtiofs DAX specific inode data
 	 */
-	struct fuse_inode_dax *dax;
+	struct fuse_inode_vdax *vdax;
 #endif
 	/** @submount_lookup: Submount specific lookup tracking */
 	struct fuse_submount_lookup *submount_lookup;
@@ -364,16 +364,16 @@ struct fuse_io_priv {
 	.iocb = i,			\
 }
 
-enum fuse_dax_mode {
-	FUSE_DAX_INODE_DEFAULT,	/* default */
-	FUSE_DAX_ALWAYS,	/* "-o dax=always" */
-	FUSE_DAX_NEVER,		/* "-o dax=never" */
-	FUSE_DAX_INODE_USER,	/* "-o dax=inode" */
+enum fuse_vdax_mode {
+	FUSE_VDAX_INODE_DEFAULT,	/* default */
+	FUSE_VDAX_ALWAYS,		/* "-o dax=always" */
+	FUSE_VDAX_NEVER,		/* "-o dax=never" */
+	FUSE_VDAX_INODE_USER,		/* "-o dax=inode" */
 };
 
-static inline bool fuse_is_inode_dax_mode(enum fuse_dax_mode mode)
+static inline bool fuse_is_inode_vdax_mode(enum fuse_vdax_mode mode)
 {
-	return mode == FUSE_DAX_INODE_DEFAULT || mode == FUSE_DAX_INODE_USER;
+	return mode == FUSE_VDAX_INODE_DEFAULT || mode == FUSE_VDAX_INODE_USER;
 }
 
 struct fuse_fs_context {
@@ -391,13 +391,14 @@ struct fuse_fs_context {
 	bool no_control:1;
 	bool no_force_umount:1;
 	bool legacy_opts_show:1;
-	enum fuse_dax_mode dax_mode;
+	bool syncfs_capable:1;
+	enum fuse_vdax_mode vdax_mode;
 	unsigned int max_read;
 	unsigned int blksize;
 	const char *subtype;
 
-	/* DAX device, may be NULL */
-	struct dax_device *dax_dev;
+	/* Virtiofs DAX device, may be NULL */
+	struct dax_device *vdax_dev;
 };
 
 struct fuse_sync_bucket {
@@ -675,6 +676,14 @@ struct fuse_conn {
 	/** @sync_fs: Propagate syncfs() to server */
 	unsigned int sync_fs:1;
 
+	/**
+	 * @syncfs_capable: the privilege required to honor FUSE_HAS_SYNCFS was
+	 * present when /dev/fuse was opened (CAP_SYS_ADMIN in the initial user
+	 * namespace), i.e. the same privilege that mounting virtiofs/fuseblk
+	 * requires.
+	 */
+	unsigned int syncfs_capable:1;
+
 	/** @init_security: Initialize security xattrs when creating a new inode */
 	unsigned int init_security:1;
 
@@ -684,8 +693,8 @@ struct fuse_conn {
 	 */
 	unsigned int create_supp_group:1;
 
-	/** @inode_dax: Does the filesystem support per inode DAX? */
-	unsigned int inode_dax:1;
+	/** @inode_vdax: Does the filesystem support per inode virtiofs DAX? */
+	unsigned int inode_vdax:1;
 
 	/** @no_tmpfile: Is tmpfile not implemented by fs? */
 	unsigned int no_tmpfile:1;
@@ -744,12 +753,12 @@ struct fuse_conn {
 	 */
 	struct rw_semaphore killsb;
 
-#ifdef CONFIG_FUSE_DAX
-	/** @dax_mode: Dax mode */
-	enum fuse_dax_mode dax_mode;
+#ifdef CONFIG_FUSE_VDAX
+	/** @vdax_mode: Virtiofs DAX mode */
+	enum fuse_vdax_mode vdax_mode;
 
-	/** @dax: Dax specific conn data, non-NULL if DAX is enabled */
-	struct fuse_conn_dax *dax;
+	/** @dax: Dax specific conn data, non-NULL if virtiofs DAX is enabled */
+	struct fuse_conn_vdax *vdax;
 #endif
 
 	/** @mounts: List of filesystems using this connection */
@@ -1003,7 +1012,7 @@ void __exit fuse_ctl_cleanup(void);
 /*
  * Simple request sending that does request allocation and freeing
  */
-ssize_t __fuse_simple_request(struct mnt_idmap *idmap,
+ssize_t __fuse_simple_request(const struct mnt_idmap *idmap,
 			      struct fuse_mount *fm,
 			      struct fuse_args *args);
 
@@ -1012,7 +1021,7 @@ static inline ssize_t fuse_simple_request(struct fuse_mount *fm, struct fuse_arg
 	return __fuse_simple_request(&invalid_mnt_idmap, fm, args);
 }
 
-static inline ssize_t fuse_simple_idmap_request(struct mnt_idmap *idmap,
+static inline ssize_t fuse_simple_idmap_request(const struct mnt_idmap *idmap,
 						struct fuse_mount *fm,
 						struct fuse_args *args)
 {
@@ -1189,7 +1198,7 @@ bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written);
 int fuse_flush_times(struct inode *inode, struct fuse_file *ff);
 int fuse_write_inode(struct inode *inode, struct writeback_control *wbc);
 
-int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int fuse_do_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		    struct iattr *attr, struct file *file);
 
 void fuse_unlock_inode(struct inode *inode, bool locked);
@@ -1205,9 +1214,9 @@ extern const struct xattr_handler * const fuse_xattr_handlers[];
 
 struct posix_acl;
 struct posix_acl *fuse_get_inode_acl(struct inode *inode, int type, bool rcu);
-struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *fuse_get_acl(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, int type);
-int fuse_set_acl(struct mnt_idmap *, struct dentry *dentry,
+int fuse_set_acl(const struct mnt_idmap *, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
 
 /* readdir.c */
@@ -1217,28 +1226,28 @@ void fuse_free_conn(struct fuse_conn *fc);
 
 /* dax.c */
 
-#define FUSE_IS_DAX(inode) (IS_ENABLED(CONFIG_FUSE_DAX) && IS_DAX(inode))
-
-ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to);
-ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from);
-int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma);
-int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start, u64 dmap_end);
-int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode mode,
-			struct dax_device *dax_dev);
-void fuse_dax_conn_free(struct fuse_conn *fc);
-bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi);
-void fuse_dax_inode_init(struct inode *inode, unsigned int flags);
-void fuse_dax_inode_cleanup(struct inode *inode);
-void fuse_dax_dontcache(struct inode *inode, unsigned int flags);
-bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment);
-void fuse_dax_cancel_work(struct fuse_conn *fc);
+#define FUSE_IS_VDAX(inode) (IS_ENABLED(CONFIG_FUSE_VDAX) && IS_DAX(inode))
+
+ssize_t fuse_vdax_read_iter(struct kiocb *iocb, struct iov_iter *to);
+ssize_t fuse_vdax_write_iter(struct kiocb *iocb, struct iov_iter *from);
+int fuse_vdax_mmap(struct file *file, struct vm_area_struct *vma);
+int fuse_vdax_break_layouts(struct inode *inode, u64 dmap_start, u64 dmap_end);
+int fuse_vdax_conn_alloc(struct fuse_conn *fc, enum fuse_vdax_mode mode,
+			struct dax_device *vdax_dev);
+void fuse_vdax_conn_free(struct fuse_conn *fc);
+bool fuse_vdax_inode_alloc(struct super_block *sb, struct fuse_inode *fi);
+void fuse_vdax_inode_init(struct inode *inode, unsigned int flags);
+void fuse_vdax_inode_cleanup(struct inode *inode);
+void fuse_vdax_dontcache(struct inode *inode, unsigned int flags);
+bool fuse_vdax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment);
+void fuse_vdax_cancel_work(struct fuse_conn *fc);
 
 /* ioctl.c */
 long fuse_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
 			    unsigned long arg);
 int fuse_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int fuse_fileattr_set(struct mnt_idmap *idmap,
+int fuse_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa);
 
 /* iomode.c */
@@ -1258,24 +1267,13 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff,
 
 /* backing.c */
 #ifdef CONFIG_FUSE_PASSTHROUGH
-struct fuse_backing *fuse_backing_get(struct fuse_backing *fb);
 void fuse_backing_put(struct fuse_backing *fb);
 struct fuse_backing *fuse_backing_lookup(struct fuse_conn *fc, int backing_id);
 #else
 
-static inline struct fuse_backing *fuse_backing_get(struct fuse_backing *fb)
-{
-	return NULL;
-}
-
 static inline void fuse_backing_put(struct fuse_backing *fb)
 {
 }
-static inline struct fuse_backing *fuse_backing_lookup(struct fuse_conn *fc,
-						       int backing_id)
-{
-	return NULL;
-}
 #endif
 
 void fuse_backing_files_init(struct fuse_conn *fc);
@@ -1304,13 +1302,9 @@ static inline struct fuse_backing *fuse_inode_backing_set(struct fuse_inode *fi,
 struct fuse_backing *fuse_passthrough_open(struct file *file, int backing_id);
 void fuse_passthrough_release(struct fuse_file *ff, struct fuse_backing *fb);
 
-static inline struct file *fuse_file_passthrough(struct fuse_file *ff)
+static inline bool fuse_is_passthrough(struct fuse_file *ff)
 {
-#ifdef CONFIG_FUSE_PASSTHROUGH
-	return ff->passthrough;
-#else
-	return NULL;
-#endif
+	return IS_ENABLED(CONFIG_FUSE_PASSTHROUGH) && (ff->open_flags & FOPEN_PASSTHROUGH);
 }
 
 ssize_t fuse_passthrough_read_iter(struct kiocb *iocb, struct iov_iter *iter);
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index e9552be3637bc9..cbb10e19e7e862 100644
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -100,7 +100,7 @@ static struct inode *fuse_alloc_inode(struct super_block *sb)
 	if (!fi->forget)
 		goto out_free;
 
-	if (IS_ENABLED(CONFIG_FUSE_DAX) && !fuse_dax_inode_alloc(sb, fi))
+	if (IS_ENABLED(CONFIG_FUSE_VDAX) && !fuse_vdax_inode_alloc(sb, fi))
 		goto out_free_forget;
 
 	if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
@@ -121,8 +121,8 @@ static void fuse_free_inode(struct inode *inode)
 
 	mutex_destroy(&fi->mutex);
 	kfree(fi->forget);
-#ifdef CONFIG_FUSE_DAX
-	kfree(fi->dax);
+#ifdef CONFIG_FUSE_VDAX
+	kfree(fi->vdax);
 #endif
 	if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
 		fuse_backing_put(fuse_inode_backing(fi));
@@ -148,7 +148,7 @@ static void fuse_evict_inode(struct inode *inode)
 	/* Will write inode on close/munmap and in all other dirtiers */
 	WARN_ON(inode_state_read_once(inode) & I_DIRTY_INODE);
 
-	if (FUSE_IS_DAX(inode))
+	if (FUSE_IS_VDAX(inode))
 		dax_break_layout_final(inode);
 
 	truncate_inode_pages_final(&inode->i_data);
@@ -156,8 +156,8 @@ static void fuse_evict_inode(struct inode *inode)
 	if (inode->i_sb->s_flags & SB_ACTIVE) {
 		struct fuse_conn *fc = get_fuse_conn(inode);
 
-		if (FUSE_IS_DAX(inode))
-			fuse_dax_inode_cleanup(inode);
+		if (FUSE_IS_VDAX(inode))
+			fuse_vdax_inode_cleanup(inode);
 		if (fi->nlookup) {
 			fuse_chan_queue_forget(fc->chan, fi->forget, fi->nodeid,
 					       fi->nlookup);
@@ -385,8 +385,8 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr
 			invalidate_inode_pages2(inode->i_mapping);
 	}
 
-	if (IS_ENABLED(CONFIG_FUSE_DAX))
-		fuse_dax_dontcache(inode, attr->flags);
+	if (IS_ENABLED(CONFIG_FUSE_VDAX))
+		fuse_vdax_dontcache(inode, attr->flags);
 }
 
 void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr,
@@ -803,6 +803,15 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file)
 	if (file->f_cred->user_ns != fsc->user_ns)
 		return invalfc(fsc, "wrong user namespace for fuse device");
 
+	/*
+	 * Record whether the server opened /dev/fuse with CAP_SYS_ADMIN in the
+	 * initial user namespace -- the same privilege that mounting virtiofs
+	 * or fuseblk requires.  Only such servers are trusted to receive
+	 * FUSE_SYNCFS (see fuse_syncfs_enable()).
+	 */
+	ctx->syncfs_capable = file_ns_capable(file, &init_user_ns,
+					      CAP_SYS_ADMIN);
+
 	ctx->fud = fuse_dev_grab(file);
 
 	return 0;
@@ -944,12 +953,12 @@ static int fuse_show_options(struct seq_file *m, struct dentry *root)
 		if (sb->s_bdev && sb->s_blocksize != FUSE_DEFAULT_BLKSIZE)
 			seq_printf(m, ",blksize=%lu", sb->s_blocksize);
 	}
-#ifdef CONFIG_FUSE_DAX
-	if (fc->dax_mode == FUSE_DAX_ALWAYS)
+#ifdef CONFIG_FUSE_VDAX
+	if (fc->vdax_mode == FUSE_VDAX_ALWAYS)
 		seq_puts(m, ",dax=always");
-	else if (fc->dax_mode == FUSE_DAX_NEVER)
+	else if (fc->vdax_mode == FUSE_VDAX_NEVER)
 		seq_puts(m, ",dax=never");
-	else if (fc->dax_mode == FUSE_DAX_INODE_USER)
+	else if (fc->vdax_mode == FUSE_VDAX_INODE_USER)
 		seq_puts(m, ",dax=inode");
 #endif
 
@@ -1007,8 +1016,8 @@ void fuse_conn_put(struct fuse_conn *fc)
 	if (!refcount_dec_and_test(&fc->count))
 		return;
 
-	if (IS_ENABLED(CONFIG_FUSE_DAX))
-		fuse_dax_conn_free(fc);
+	if (IS_ENABLED(CONFIG_FUSE_VDAX))
+		fuse_vdax_conn_free(fc);
 	cancel_work_sync(&fc->epoch_work);
 	fuse_chan_release(fc->chan);
 	put_pid_ns(fc->pid_ns);
@@ -1269,6 +1278,16 @@ struct fuse_init_args {
 	struct fuse_mount *fm;
 };
 
+/*
+ * A server can stall syncfs()/sync(), so only honor FUSE_HAS_SYNCFS for
+ * servers that opened /dev/fuse with CAP_SYS_ADMIN in the initial user
+ * namespace -- the same privilege required to mount virtiofs or fuseblk.
+ */
+static bool fuse_syncfs_enable(struct fuse_conn *fc, u64 flags)
+{
+	return (flags & FUSE_HAS_SYNCFS) && fc->syncfs_capable;
+}
+
 static void process_init_reply(struct fuse_args *args, int error)
 {
 	struct fuse_init_args *ia = container_of(args, typeof(*ia), args);
@@ -1354,13 +1373,13 @@ static void process_init_reply(struct fuse_args *args, int error)
 				if (fc->max_pages > 1)
 					fc->name_max = FUSE_NAME_MAX;
 			}
-			if (IS_ENABLED(CONFIG_FUSE_DAX)) {
+			if (IS_ENABLED(CONFIG_FUSE_VDAX)) {
 				if (flags & FUSE_MAP_ALIGNMENT &&
-				    !fuse_dax_check_alignment(fc, arg->map_alignment)) {
+				    !fuse_vdax_check_alignment(fc, arg->map_alignment)) {
 					ok = false;
 				}
 				if (flags & FUSE_HAS_INODE_DAX)
-					fc->inode_dax = 1;
+					fc->inode_vdax = 1;
 			}
 			if (flags & FUSE_HANDLE_KILLPRIV_V2) {
 				fc->handle_killpriv_v2 = 1;
@@ -1410,6 +1429,9 @@ static void process_init_reply(struct fuse_args *args, int error)
 
 			if (flags & FUSE_REQUEST_TIMEOUT)
 				timeout = arg->request_timeout;
+
+			if (fuse_syncfs_enable(fc, flags))
+				fc->sync_fs = 1;
 		} else {
 			ra_pages = fc->max_read / PAGE_SIZE;
 			fc->no_lock = 1;
@@ -1469,16 +1491,21 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm)
 		FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP |
 		FUSE_NO_EXPORT_SUPPORT | FUSE_HAS_RESEND | FUSE_ALLOW_IDMAP |
 		FUSE_REQUEST_TIMEOUT;
-#ifdef CONFIG_FUSE_DAX
-	if (fm->fc->dax)
+#ifdef CONFIG_FUSE_VDAX
+	if (fm->fc->vdax)
 		flags |= FUSE_MAP_ALIGNMENT;
-	if (fuse_is_inode_dax_mode(fm->fc->dax_mode))
+	if (fuse_is_inode_vdax_mode(fm->fc->vdax_mode))
 		flags |= FUSE_HAS_INODE_DAX;
 #endif
 	if (fm->fc->auto_submounts)
 		flags |= FUSE_SUBMOUNTS;
 	if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
 		flags |= FUSE_PASSTHROUGH;
+	/* Only offered to sufficiently privileged servers; see
+	 * fuse_syncfs_enable().
+	 */
+	if (fm->fc->syncfs_capable)
+		flags |= FUSE_HAS_SYNCFS;
 
 	if (fuse_uring_enabled())
 		flags |= FUSE_OVER_IO_URING | FUSE_HAS_IO_URING_BUFPOOL;
@@ -1751,8 +1778,8 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
 
 	sb->s_subtype = ctx->subtype;
 	ctx->subtype = NULL;
-	if (IS_ENABLED(CONFIG_FUSE_DAX)) {
-		err = fuse_dax_conn_alloc(fc, ctx->dax_mode, ctx->dax_dev);
+	if (IS_ENABLED(CONFIG_FUSE_VDAX)) {
+		err = fuse_vdax_conn_alloc(fc, ctx->vdax_mode, ctx->vdax_dev);
 		if (err)
 			goto err;
 	}
@@ -1761,7 +1788,7 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
 	fm->sb = sb;
 	err = fuse_bdi_init(fc, sb);
 	if (err)
-		goto err_free_dax;
+		goto err_free_vdax;
 
 	/* Handle umasking inside the fuse code */
 	if (sb->s_flags & SB_POSIXACL)
@@ -1770,6 +1797,7 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
 
 	fc->default_permissions = ctx->default_permissions;
 	fc->allow_other = ctx->allow_other;
+	fc->syncfs_capable = ctx->syncfs_capable;
 	fc->user_id = ctx->user_id;
 	fc->group_id = ctx->group_id;
 	fc->legacy_opts_show = ctx->legacy_opts_show;
@@ -1783,7 +1811,7 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
 	set_default_d_op(sb, &fuse_dentry_operations);
 	root_dentry = d_make_root(root);
 	if (!root_dentry)
-		goto err_free_dax;
+		goto err_free_vdax;
 
 	mutex_lock(&fuse_mutex);
 	err = -EINVAL;
@@ -1809,9 +1837,9 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
  err_unlock:
 	mutex_unlock(&fuse_mutex);
 	dput(root_dentry);
- err_free_dax:
-	if (IS_ENABLED(CONFIG_FUSE_DAX))
-		fuse_dax_conn_free(fc);
+ err_free_vdax:
+	if (IS_ENABLED(CONFIG_FUSE_VDAX))
+		fuse_vdax_conn_free(fc);
  err:
 	return err;
 }
diff --git a/fs/fuse/ioctl.c b/fs/fuse/ioctl.c
index 3614ea60391398..ce1807704da6cc 100644
--- a/fs/fuse/ioctl.c
+++ b/fs/fuse/ioctl.c
@@ -130,9 +130,6 @@ static int fuse_setup_measure_verity(unsigned long arg, struct iovec *iov)
 	if (copy_from_user(&digest_size, &uarg->digest_size, sizeof(digest_size)))
 		return -EFAULT;
 
-	if (digest_size > SIZE_MAX - sizeof(struct fsverity_digest))
-		return -EINVAL;
-
 	iov->iov_len = sizeof(struct fsverity_digest) + digest_size;
 
 	return 0;
@@ -540,7 +537,7 @@ int fuse_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return err;
 }
 
-int fuse_fileattr_set(struct mnt_idmap *idmap,
+int fuse_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/fuse/iomode.c b/fs/fuse/iomode.c
index 3728933188f307..79637c09e88397 100644
--- a/fs/fuse/iomode.c
+++ b/fs/fuse/iomode.c
@@ -200,10 +200,10 @@ int fuse_file_io_open(struct file *file, struct inode *inode)
 	int err;
 
 	/*
-	 * io modes are not relevant with DAX and with server that does not
+	 * io modes are not relevant with virtiofs DAX and with server that does not
 	 * implement open.
 	 */
-	if (FUSE_IS_DAX(inode) || !ff->args)
+	if (FUSE_IS_VDAX(inode) || !ff->args)
 		return 0;
 
 	/*
diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c
index 1ba763705d9119..c0428e03d13815 100644
--- a/fs/fuse/notify.c
+++ b/fs/fuse/notify.c
@@ -155,7 +155,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
 
 	nodeid = outarg.nodeid;
 	pos = outarg.offset;
-	num = min(outarg.size, MAX_LFS_FILESIZE - pos);
+	num = umin(outarg.size, MAX_LFS_FILESIZE - pos);
 
 	down_read(&fc->killsb);
 
@@ -190,7 +190,7 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
 		folio_offset = offset_in_folio(folio, pos);
 		nr_bytes = min(num, folio_size(folio) - folio_offset);
 
-		err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0);
+		err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes);
 		if (!folio_test_uptodate(folio) && !err && folio_offset == 0 &&
 		    (nr_bytes == folio_size(folio) || file_size == end)) {
 			folio_zero_segment(folio, nr_bytes, folio_size(folio));
diff --git a/fs/fuse/passthrough.c b/fs/fuse/passthrough.c
index f2d08ac2459b7e..b43d3e0f70817d 100644
--- a/fs/fuse/passthrough.c
+++ b/fs/fuse/passthrough.c
@@ -11,6 +11,10 @@
 #include <linux/backing-file.h>
 #include <linux/splice.h>
 
+static inline struct file *fuse_file_passthrough(struct fuse_file *ff)
+{
+	return ff->passthrough;
+}
 static void fuse_file_accessed(struct file *file)
 {
 	struct inode *inode = file_inode(file);
@@ -187,10 +191,15 @@ struct fuse_backing *fuse_passthrough_open(struct file *file, int backing_id)
 
 void fuse_passthrough_release(struct fuse_file *ff, struct fuse_backing *fb)
 {
+	struct file *backing_file = fuse_file_passthrough(ff);
+
 	pr_debug("%s: fb=0x%p, backing_file=0x%p\n", __func__,
-		 fb, ff->passthrough);
+		 fb, backing_file);
+
+	if (!backing_file)
+		return;
 
-	fput(ff->passthrough);
+	fput(backing_file);
 	ff->passthrough = NULL;
 	put_cred(ff->cred);
 	ff->cred = NULL;
diff --git a/fs/fuse/req.c b/fs/fuse/req.c
index a01ee743d31e97..6274a23af445c1 100644
--- a/fs/fuse/req.c
+++ b/fs/fuse/req.c
@@ -3,19 +3,20 @@
 #include "dev.h"
 #include "fuse_i.h"
 
-static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap)
+static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args,
+			   const struct mnt_idmap *idmap)
 {
 	struct fuse_conn *fc = fm->fc;
 	bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP);
 	kuid_t fsuid = mapped_fsuid(idmap, fc->user_ns);
 	kgid_t fsgid = mapped_fsgid(idmap, fc->user_ns);
 
+	if (args->nocreds)
+		return 0;
+
 	args->pid = pid_nr_ns(task_pid(current), fc->pid_ns);
 
 	if (args->force) {
-		if (args->nocreds)
-			return 0;
-
 		if (no_idmap) {
 			args->uid = from_kuid_munged(fc->user_ns, current_fsuid());
 			args->gid = from_kgid_munged(fc->user_ns, current_fsgid());
@@ -26,7 +27,6 @@ static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args, struct
 		return 0;
 	}
 
-	WARN_ON(args->nocreds);
 	/*
 	 * Keep the old behavior when idmappings support was not
 	 * declared by a FUSE server.
@@ -49,7 +49,8 @@ static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args, struct
 	return 0;
 }
 
-static int fuse_req_prep(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap)
+static int fuse_req_prep(struct fuse_mount *fm, struct fuse_args *args,
+			 const struct mnt_idmap *idmap)
 {
 	if (!args->force && fm->fc->conn_error)
 		return -ECONNREFUSED;
@@ -57,7 +58,7 @@ static int fuse_req_prep(struct fuse_mount *fm, struct fuse_args *args, struct m
 	return fuse_fill_creds(fm, args, idmap);
 }
 
-ssize_t __fuse_simple_request(struct mnt_idmap *idmap, struct fuse_mount *fm,
+ssize_t __fuse_simple_request(const struct mnt_idmap *idmap, struct fuse_mount *fm,
 			      struct fuse_args *args)
 {
 	struct fuse_conn *fc = fm->fc;
diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c
index f15e516ebcb5c7..4f334766b8c307 100644
--- a/fs/fuse/virtio_fs.c
+++ b/fs/fuse/virtio_fs.c
@@ -102,9 +102,9 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
 				 gfp_t gfp);
 
 static const struct constant_table dax_param_enums[] = {
-	{"always",	FUSE_DAX_ALWAYS },
-	{"never",	FUSE_DAX_NEVER },
-	{"inode",	FUSE_DAX_INODE_USER },
+	{"always",	FUSE_VDAX_ALWAYS },
+	{"never",	FUSE_VDAX_NEVER },
+	{"inode",	FUSE_VDAX_INODE_USER },
 	{}
 };
 
@@ -132,10 +132,10 @@ static int virtio_fs_parse_param(struct fs_context *fsc,
 
 	switch (opt) {
 	case OPT_DAX:
-		ctx->dax_mode = FUSE_DAX_ALWAYS;
+		ctx->vdax_mode = FUSE_VDAX_ALWAYS;
 		break;
 	case OPT_DAX_ENUM:
-		ctx->dax_mode = result.uint_32;
+		ctx->vdax_mode = result.uint_32;
 		break;
 	default:
 		return -EINVAL;
@@ -798,9 +798,11 @@ static void virtio_fs_request_complete(struct fuse_req *req,
 		for (i = 0; i < ap->num_folios; i++) {
 			thislen = ap->descs[i].length;
 			if (len < thislen) {
-				WARN_ON(ap->descs[i].offset);
+				unsigned int offset = ap->descs[i].offset;
+
 				folio = ap->folios[i];
-				folio_zero_segment(folio, len, thislen);
+				folio_zero_segment(folio, offset + len,
+						   offset + thislen);
 				len = 0;
 			} else {
 				len -= thislen;
@@ -1079,7 +1081,7 @@ static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs)
 	struct dev_pagemap *pgmap;
 	bool have_cache;
 
-	if (!IS_ENABLED(CONFIG_FUSE_DAX))
+	if (!IS_ENABLED(CONFIG_FUSE_VDAX))
 		return 0;
 
 	dax_dev = alloc_dax(fs, &virtio_fs_dax_ops);
@@ -1592,14 +1594,14 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
 			goto err_free_fuse_devs;
 	}
 
-	if (ctx->dax_mode != FUSE_DAX_NEVER) {
-		if (ctx->dax_mode == FUSE_DAX_ALWAYS && !fs->dax_dev) {
+	if (ctx->vdax_mode != FUSE_VDAX_NEVER) {
+		if (ctx->vdax_mode == FUSE_VDAX_ALWAYS && !fs->dax_dev) {
 			err = -EINVAL;
 			pr_err("virtio-fs: dax can't be enabled as filesystem"
 			       " device does not support it.\n");
 			goto err_free_fuse_devs;
 		}
-		ctx->dax_dev = fs->dax_dev;
+		ctx->vdax_dev = fs->dax_dev;
 	}
 	err = fuse_fill_super_common(sb, ctx);
 	if (err < 0)
@@ -1633,8 +1635,8 @@ static void virtio_fs_conn_destroy(struct fuse_mount *fm)
 	/* Stop dax worker. Soon evict_inodes() will be called which
 	 * will free all memory ranges belonging to all inodes.
 	 */
-	if (IS_ENABLED(CONFIG_FUSE_DAX))
-		fuse_dax_cancel_work(fc);
+	if (IS_ENABLED(CONFIG_FUSE_VDAX))
+		fuse_vdax_cancel_work(fc);
 
 	/* Stop forget queue. Soon destroy will be sent */
 	spin_lock(&fsvq->lock);
diff --git a/fs/fuse/xattr.c b/fs/fuse/xattr.c
index cab2685acc6528..53e3c5e6fff0dd 100644
--- a/fs/fuse/xattr.c
+++ b/fs/fuse/xattr.c
@@ -188,7 +188,7 @@ static int fuse_xattr_get(const struct xattr_handler *handler,
 }
 
 static int fuse_xattr_set(const struct xattr_handler *handler,
-			  struct mnt_idmap *idmap,
+			  const struct mnt_idmap *idmap,
 			  struct dentry *dentry, struct inode *inode,
 			  const char *name, const void *value, size_t size,
 			  int flags)
diff --git a/fs/gfs2/acl.c b/fs/gfs2/acl.c
index a5b60778b91c90..f1c6a5e392b475 100644
--- a/fs/gfs2/acl.c
+++ b/fs/gfs2/acl.c
@@ -59,7 +59,7 @@ static struct posix_acl *__gfs2_get_acl(struct inode *inode, int type)
 
 struct posix_acl *gfs2_get_acl(struct inode *inode, int type, bool rcu)
 {
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_holder gh;
 	bool need_unlock = false;
 	struct posix_acl *acl;
@@ -67,8 +67,8 @@ struct posix_acl *gfs2_get_acl(struct inode *inode, int type, bool rcu)
 	if (rcu)
 		return ERR_PTR(-ECHILD);
 
-	if (!gfs2_glock_is_locked_by_me(ip->i_gl)) {
-		int ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED,
+	if (!gfs2_glock_is_locked_by_me(gl)) {
+		int ret = gfs2_glock_nq_init(gl, LM_ST_SHARED,
 					     LM_FLAG_ANY, &gh);
 		if (ret)
 			return ERR_PTR(ret);
@@ -102,10 +102,11 @@ int __gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type)
 	return error;
 }
 
-int gfs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int gfs2_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 	struct inode *inode = d_inode(dentry);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder gh;
 	bool need_unlock = false;
@@ -119,8 +120,8 @@ int gfs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 	if (ret)
 		return ret;
 
-	if (!gfs2_glock_is_locked_by_me(ip->i_gl)) {
-		ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
+	if (!gfs2_glock_is_locked_by_me(gl)) {
+		ret = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, &gh);
 		if (ret)
 			goto out;
 		need_unlock = true;
diff --git a/fs/gfs2/acl.h b/fs/gfs2/acl.h
index 82f5b09c04e669..d19d417559367f 100644
--- a/fs/gfs2/acl.h
+++ b/fs/gfs2/acl.h
@@ -13,7 +13,7 @@
 
 struct posix_acl *gfs2_get_acl(struct inode *inode, int type, bool rcu);
 int __gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type);
-int gfs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int gfs2_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
 
 #endif /* __ACL_DOT_H__ */
diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c
index 0a7b8076af3a5e..ee11d494e47c52 100644
--- a/fs/gfs2/aops.c
+++ b/fs/gfs2/aops.c
@@ -102,7 +102,7 @@ static int __gfs2_jdata_write_folio(struct folio *folio,
 		struct writeback_control *wbc)
 {
 	struct inode *inode = folio->mapping->host;
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 
 	if (folio_test_checked(folio)) {
 		folio_clear_checked(folio);
@@ -111,7 +111,7 @@ static int __gfs2_jdata_write_folio(struct folio *folio,
 					inode->i_sb->s_blocksize,
 					BIT(BH_Dirty)|BIT(BH_Uptodate));
 		}
-		gfs2_trans_add_databufs(ip->i_gl, folio, 0, folio_size(folio));
+		gfs2_trans_add_databufs(gl, folio, 0, folio_size(folio));
 	}
 	return gfs2_write_jdata_folio(folio, wbc);
 }
@@ -126,13 +126,13 @@ static int __gfs2_jdata_write_folio(struct folio *folio,
 int gfs2_jdata_writeback(struct address_space *mapping, struct writeback_control *wbc)
 {
 	struct inode *inode = mapping->host;
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(mapping->host);
 	struct folio *folio = NULL;
 	int error;
 
 	BUG_ON(current->journal_info);
-	if (gfs2_assert_withdraw(sdp, ip->i_gl->gl_state == LM_ST_EXCLUSIVE))
+	if (gfs2_assert_withdraw(sdp, gl->gl_state == LM_ST_EXCLUSIVE))
 		return 0;
 
 	while ((folio = writeback_iter(mapping, wbc, folio, &error))) {
@@ -362,14 +362,14 @@ static int gfs2_write_cache_jdata(struct address_space *mapping,
 static int gfs2_jdata_writepages(struct address_space *mapping,
 				 struct writeback_control *wbc)
 {
-	struct gfs2_inode *ip = GFS2_I(mapping->host);
+	struct gfs2_glock *gl = gfs2_inode_glock(mapping->host);
 	struct gfs2_sbd *sdp = GFS2_SB(mapping->host);
 	int ret;
 
 	ret = gfs2_write_cache_jdata(mapping, wbc);
 	if (ret == 0 && wbc->sync_mode == WB_SYNC_ALL) {
-		gfs2_log_flush(sdp, ip->i_gl, GFS2_LOG_HEAD_FLUSH_NORMAL |
-			       GFS2_LFC_JDATA_WPAGES);
+		gfs2_log_flush(sdp, gl, GFS2_LOG_HEAD_FLUSH_NORMAL |
+					GFS2_LFC_JDATA_WPAGES);
 		ret = gfs2_write_cache_jdata(mapping, wbc);
 	}
 	return ret;
@@ -561,12 +561,13 @@ static bool gfs2_jdata_dirty_folio(struct address_space *mapping,
 
 static sector_t gfs2_bmap(struct address_space *mapping, sector_t lblock)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(mapping->host);
 	struct gfs2_inode *ip = GFS2_I(mapping->host);
 	struct gfs2_holder i_gh;
 	sector_t dblock = 0;
 	int error;
 
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh);
 	if (error)
 		return 0;
 
diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c
index 73c62697116390..264170f91c5dd8 100644
--- a/fs/gfs2/bmap.c
+++ b/fs/gfs2/bmap.c
@@ -55,6 +55,7 @@ static int gfs2_unstuffer_folio(struct gfs2_inode *ip, struct buffer_head *dibh,
 			       u64 block, struct folio *folio)
 {
 	struct inode *inode = &ip->i_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 
 	if (!folio_test_uptodate(folio)) {
 		void *kaddr = kmap_local_folio(folio, 0);
@@ -78,7 +79,7 @@ static int gfs2_unstuffer_folio(struct gfs2_inode *ip, struct buffer_head *dibh,
 			map_bh(bh, inode->i_sb, block);
 
 		set_buffer_uptodate(bh);
-		gfs2_trans_add_data(ip->i_gl, bh);
+		gfs2_trans_add_data(gl, bh);
 	} else {
 		folio_mark_dirty(folio);
 		gfs2_ordered_add_inode(ip);
@@ -89,6 +90,8 @@ static int gfs2_unstuffer_folio(struct gfs2_inode *ip, struct buffer_head *dibh,
 
 static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct folio *folio)
 {
+	struct inode *inode = &ip->i_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct buffer_head *bh, *dibh;
 	struct gfs2_dinode *di;
 	u64 block = 0;
@@ -99,7 +102,7 @@ static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct folio *folio)
 	if (error)
 		return error;
 
-	if (i_size_read(&ip->i_inode)) {
+	if (i_size_read(inode)) {
 		/* Get a free block, fill it with the stuffed data,
 		   and write it out to disk */
 
@@ -108,7 +111,7 @@ static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct folio *folio)
 		if (error)
 			goto out_brelse;
 		if (isdir) {
-			gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1);
+			gfs2_trans_remove_revoke(GFS2_SB(inode), block, 1);
 			error = gfs2_dir_get_new_buffer(ip, block, &bh);
 			if (error)
 				goto out_brelse;
@@ -124,14 +127,14 @@ static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct folio *folio)
 
 	/*  Set up the pointer to the new block  */
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	di = (struct gfs2_dinode *)dibh->b_data;
 	gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
 
-	if (i_size_read(&ip->i_inode)) {
+	if (i_size_read(inode)) {
 		*(__be64 *)(di + 1) = cpu_to_be64(block);
-		gfs2_add_inode_blocks(&ip->i_inode, 1);
-		di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(&ip->i_inode));
+		gfs2_add_inode_blocks(inode, 1);
+		di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(inode));
 	}
 
 	ip->i_height = 1;
@@ -662,6 +665,7 @@ enum alloc_state {
 static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
 			      struct metapath *mp)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct buffer_head *dibh = metapath_dibh(mp);
@@ -678,7 +682,7 @@ static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
 	BUG_ON(dibh == NULL);
 	BUG_ON(dblks < 1);
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 
 	down_write(&ip->i_rw_mutex);
 
@@ -722,7 +726,7 @@ static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
 			}
 			for (; i - 1 < mp->mp_fheight - ip->i_height && n > 0;
 			     i++, n--)
-				gfs2_indirect_init(mp, ip->i_gl, i, 0, bn++);
+				gfs2_indirect_init(mp, gl, i, 0, bn++);
 			if (i - 1 == mp->mp_fheight - ip->i_height) {
 				i--;
 				gfs2_buffer_copy_tail(mp->mp_bh[i],
@@ -748,9 +752,9 @@ static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
 			fallthrough;	/* To branching from existing tree */
 		case ALLOC_GROW_DEPTH:
 			if (i > 1 && i < mp->mp_fheight)
-				gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[i-1]);
+				gfs2_trans_add_meta(gl, mp->mp_bh[i-1]);
 			for (; i < mp->mp_fheight && n > 0; i++, n--)
-				gfs2_indirect_init(mp, ip->i_gl, i,
+				gfs2_indirect_init(mp, gl, i,
 						   mp->mp_list[i-1], bn++);
 			if (i == mp->mp_fheight)
 				state = ALLOC_DATA;
@@ -760,7 +764,7 @@ static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
 		case ALLOC_DATA:
 			BUG_ON(n > dblks);
 			BUG_ON(mp->mp_bh[end_of_metadata] == NULL);
-			gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[end_of_metadata]);
+			gfs2_trans_add_meta(gl, mp->mp_bh[end_of_metadata]);
 			dblks = n;
 			ptr = metapointer(end_of_metadata, mp);
 			iomap->addr = bn << inode->i_blkbits;
@@ -989,12 +993,12 @@ static void gfs2_iomap_put_folio(struct inode *inode, loff_t pos,
 				 unsigned copied, struct folio *folio)
 {
 	struct gfs2_trans *tr = current->journal_info;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 
 	if (gfs2_is_jdata(ip) && !gfs2_is_stuffed(ip))
-		gfs2_trans_add_databufs(ip->i_gl, folio,
-					offset_in_folio(folio, pos),
+		gfs2_trans_add_databufs(gl, folio, offset_in_folio(folio, pos),
 					copied);
 
 	folio_unlock(folio);
@@ -1150,6 +1154,7 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
 static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
 			  ssize_t written, unsigned flags, struct iomap *iomap)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 
@@ -1196,7 +1201,7 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
 
 	if (iomap->flags & IOMAP_F_SIZE_CHANGED)
 		mark_inode_dirty(inode);
-	set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
+	set_bit(GLF_DIRTY, &gl->gl_flags);
 	return 0;
 }
 
@@ -1212,7 +1217,7 @@ const struct iomap_ops gfs2_iomap_ops = {
  * @inode: The inode
  * @lblock: The logical block number
  * @bh_map: The bh to be mapped
- * @create: True if its ok to alloc blocks to satify the request
+ * @create: True if its ok to alloc blocks to satisfy the request
  *
  * The size of the requested mapping is defined in bh_map->b_size.
  *
@@ -1387,6 +1392,7 @@ static int gfs2_journaled_truncate(struct inode *inode, u64 oldsize, u64 newsize
 
 static int trunc_start(struct inode *inode, u64 newsize)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct buffer_head *dibh = NULL;
@@ -1415,7 +1421,7 @@ static int trunc_start(struct inode *inode, u64 newsize)
 	if (error)
 		goto out;
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 
 	if (gfs2_is_stuffed(ip))
 		gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode) + newsize);
@@ -1488,7 +1494,9 @@ static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
 			      struct buffer_head *bh, __be64 *start, __be64 *end,
 			      bool meta, u32 *btotal)
 {
-	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
+	struct inode *inode = &ip->i_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
+	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct gfs2_rgrpd *rgd;
 	struct gfs2_trans *tr;
 	__be64 *p;
@@ -1546,7 +1554,7 @@ static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
 
 			jblocks_rqsted = rgd->rd_length + RES_DINODE +
 				RES_INDIRECT;
-			isize_blks = gfs2_get_inode_blocks(&ip->i_inode);
+			isize_blks = gfs2_get_inode_blocks(inode);
 			if (isize_blks > atomic_read(&sdp->sd_log_thresh2))
 				jblocks_rqsted +=
 					atomic_read(&sdp->sd_log_thresh2);
@@ -1587,7 +1595,7 @@ static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
 			goto out_unlock;
 		}
 
-		gfs2_trans_add_meta(ip->i_gl, bh);
+		gfs2_trans_add_meta(gl, bh);
 		buf_in_tr = true;
 		*p = 0;
 		if (bstart + blen == bn) {
@@ -1597,7 +1605,7 @@ static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
 		if (bstart) {
 			__gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
 			(*btotal) += blen;
-			gfs2_add_inode_blocks(&ip->i_inode, -blen);
+			gfs2_add_inode_blocks(inode, -blen);
 		}
 		bstart = bn;
 		blen = 1;
@@ -1605,7 +1613,7 @@ static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
 	if (bstart) {
 		__gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
 		(*btotal) += blen;
-		gfs2_add_inode_blocks(&ip->i_inode, -blen);
+		gfs2_add_inode_blocks(inode, -blen);
 	}
 out_unlock:
 	if (!ret && blks_outside_rgrp) { /* If buffer still has non-zero blocks
@@ -1620,8 +1628,8 @@ static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
 
 			/* Every transaction boundary, we rewrite the dinode
 			   to keep its di_blocks current in case of failure. */
-			inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
-			gfs2_trans_add_meta(ip->i_gl, dibh);
+			inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
+			gfs2_trans_add_meta(gl, dibh);
 			gfs2_dinode_out(ip, dibh->b_data);
 			brelse(dibh);
 			up_write(&ip->i_rw_mutex);
@@ -1691,7 +1699,7 @@ enum dealloc_states {
 };
 
 static inline void
-metapointer_range(struct metapath *mp, int height,
+metapointer_range(struct metapath *mp, unsigned int height,
 		  __u16 *start_list, unsigned int start_aligned,
 		  __u16 *end_list, unsigned int end_aligned,
 		  __be64 **start, __be64 **end)
@@ -1746,7 +1754,9 @@ static inline bool walk_done(struct gfs2_sbd *sdp,
  */
 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
 {
-	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
+	struct inode *inode = &ip->i_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
+	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	u64 maxsize = sdp->sd_heightsize[ip->i_height];
 	struct metapath mp = {};
 	struct buffer_head *dibh, *bh;
@@ -1760,7 +1770,7 @@ static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
 	unsigned int strip_h = ip->i_height - 1;
 	u32 btotal = 0;
 	int ret, state;
-	int mp_h; /* metapath buffers are read in to this height */
+	unsigned int mp_h; /* metapath buffers are read in to this height */
 	u64 prev_bnr = 0;
 	__be64 *start, *end;
 
@@ -1833,7 +1843,7 @@ static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
 	for (mp_h = 0; mp_h < mp.mp_aheight - 1; mp_h++) {
 		metapointer_range(&mp, mp_h, start_list, start_aligned,
 				  end_list, end_aligned, &start, &end);
-		gfs2_metapath_ra(ip->i_gl, start, end);
+		gfs2_metapath_ra(gl, start, end);
 	}
 
 	if (mp.mp_aheight == ip->i_height)
@@ -1953,7 +1963,7 @@ static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
 							  start_list, start_aligned,
 							  end_list, end_aligned,
 							  &start, &end);
-					gfs2_metapath_ra(ip->i_gl, start, end);
+					gfs2_metapath_ra(gl, start, end);
 				}
 			}
 
@@ -1985,10 +1995,9 @@ static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
 			down_write(&ip->i_rw_mutex);
 		}
 		gfs2_statfs_change(sdp, 0, +btotal, 0);
-		gfs2_quota_change(ip, -(s64)btotal, ip->i_inode.i_uid,
-				  ip->i_inode.i_gid);
-		inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
-		gfs2_trans_add_meta(ip->i_gl, dibh);
+		gfs2_quota_change(ip, -(s64)btotal, inode->i_uid, inode->i_gid);
+		inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
+		gfs2_trans_add_meta(gl, dibh);
 		gfs2_dinode_out(ip, dibh->b_data);
 		up_write(&ip->i_rw_mutex);
 		gfs2_trans_end(sdp);
@@ -2010,7 +2019,9 @@ static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
 
 static int trunc_end(struct gfs2_inode *ip)
 {
-	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
+	struct inode *inode = &ip->i_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
+	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct buffer_head *dibh;
 	int error;
 
@@ -2024,16 +2035,16 @@ static int trunc_end(struct gfs2_inode *ip)
 	if (error)
 		goto out;
 
-	if (!i_size_read(&ip->i_inode)) {
+	if (!i_size_read(inode)) {
 		ip->i_height = 0;
 		ip->i_goal = ip->i_no_addr;
 		gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
 		gfs2_ordered_del_inode(ip);
 	}
-	inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
+	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
 	ip->i_diskflags &= ~GFS2_DIF_TRUNC_IN_PROG;
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	gfs2_dinode_out(ip, dibh->b_data);
 	brelse(dibh);
 
@@ -2094,6 +2105,7 @@ static int do_shrink(struct inode *inode, u64 newsize)
 
 static int do_grow(struct inode *inode, u64 size)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct gfs2_alloc_parms ap = { .target = 1, };
@@ -2138,7 +2150,7 @@ static int do_grow(struct inode *inode, u64 size)
 
 	truncate_setsize(inode, size);
 	inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	gfs2_dinode_out(ip, dibh->b_data);
 	brelse(dibh);
 
@@ -2377,6 +2389,7 @@ int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset,
 
 static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct buffer_head *dibh;
 	int error;
@@ -2389,7 +2402,7 @@ static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length)
 	error = gfs2_meta_inode_buffer(ip, &dibh);
 	if (error)
 		return error;
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	memset(dibh->b_data + sizeof(struct gfs2_dinode) + offset, 0,
 	       length);
 	brelse(dibh);
@@ -2416,7 +2429,7 @@ static int gfs2_journaled_truncate_range(struct inode *inode, loff_t offset,
 		if (offs && chunk > PAGE_SIZE)
 			chunk = offs + ((chunk - offs) & PAGE_MASK);
 
-		truncate_pagecache_range(inode, offset, chunk);
+		truncate_pagecache_range(inode, offset, offset + chunk - 1);
 		offset += chunk;
 		length -= chunk;
 
diff --git a/fs/gfs2/dentry.c b/fs/gfs2/dentry.c
index 95050e719233db..7b344461658ec3 100644
--- a/fs/gfs2/dentry.c
+++ b/fs/gfs2/dentry.c
@@ -35,8 +35,8 @@
 static int gfs2_drevalidate(struct inode *dir, const struct qstr *name,
 			    struct dentry *dentry, unsigned int flags)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(dir);
 	struct gfs2_sbd *sdp = GFS2_SB(dir);
-	struct gfs2_inode *dip = GFS2_I(dir);
 	struct inode *inode;
 	struct gfs2_holder d_gh;
 	struct gfs2_inode *ip = NULL;
@@ -57,9 +57,9 @@ static int gfs2_drevalidate(struct inode *dir, const struct qstr *name,
 	if (sdp->sd_lockstruct.ls_ops->lm_mount == NULL)
 		return 1;
 
-	had_lock = (gfs2_glock_is_locked_by_me(dip->i_gl) != NULL);
+	had_lock = (gfs2_glock_is_locked_by_me(gl) != NULL);
 	if (!had_lock) {
-		error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &d_gh);
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &d_gh);
 		if (error)
 			return 0;
 	}
diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c
index 0237b36b9eb165..6cfe335fd5909f 100644
--- a/fs/gfs2/dir.c
+++ b/fs/gfs2/dir.c
@@ -90,10 +90,11 @@ typedef int (*gfs2_dscan_t)(const struct gfs2_dirent *dent,
 int gfs2_dir_get_new_buffer(struct gfs2_inode *ip, u64 block,
 			    struct buffer_head **bhp)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct buffer_head *bh;
 
-	bh = gfs2_meta_new(ip->i_gl, block);
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	bh = gfs2_meta_new(gl, block);
+	gfs2_trans_add_meta(gl, bh);
 	gfs2_metatype_set(bh, GFS2_METATYPE_JD, GFS2_FORMAT_JD);
 	gfs2_buffer_clear_tail(bh, sizeof(struct gfs2_meta_header));
 	*bhp = bh;
@@ -103,10 +104,11 @@ int gfs2_dir_get_new_buffer(struct gfs2_inode *ip, u64 block,
 static int gfs2_dir_get_existing_buffer(struct gfs2_inode *ip, u64 block,
 					struct buffer_head **bhp)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct buffer_head *bh;
 	int error;
 
-	error = gfs2_meta_read(ip->i_gl, block, DIO_WAIT, 0, &bh);
+	error = gfs2_meta_read(gl, block, DIO_WAIT, 0, &bh);
 	if (error)
 		return error;
 	if (gfs2_metatype_check(GFS2_SB(&ip->i_inode), bh, GFS2_METATYPE_JD)) {
@@ -120,6 +122,7 @@ static int gfs2_dir_get_existing_buffer(struct gfs2_inode *ip, u64 block,
 static int gfs2_dir_write_stuffed(struct gfs2_inode *ip, const char *buf,
 				  unsigned int offset, unsigned int size)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct buffer_head *dibh;
 	int error;
 
@@ -127,7 +130,7 @@ static int gfs2_dir_write_stuffed(struct gfs2_inode *ip, const char *buf,
 	if (error)
 		return error;
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	memcpy(dibh->b_data + offset + sizeof(struct gfs2_dinode), buf, size);
 	if (ip->i_inode.i_size < offset + size)
 		i_size_write(&ip->i_inode, offset + size);
@@ -153,6 +156,7 @@ static int gfs2_dir_write_stuffed(struct gfs2_inode *ip, const char *buf,
 static int gfs2_dir_write_data(struct gfs2_inode *ip, const char *buf,
 			       u64 offset, unsigned int size)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct buffer_head *dibh;
 	u64 lblock, dblock;
@@ -208,7 +212,7 @@ static int gfs2_dir_write_data(struct gfs2_inode *ip, const char *buf,
 		if (error)
 			goto fail;
 
-		gfs2_trans_add_meta(ip->i_gl, bh);
+		gfs2_trans_add_meta(gl, bh);
 		memcpy(bh->b_data + o, buf, amount);
 		brelse(bh);
 
@@ -230,7 +234,7 @@ static int gfs2_dir_write_data(struct gfs2_inode *ip, const char *buf,
 		i_size_write(&ip->i_inode, offset + copied);
 	inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode));
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	gfs2_dinode_out(ip, dibh->b_data);
 	brelse(dibh);
 
@@ -268,6 +272,7 @@ static int gfs2_dir_read_stuffed(struct gfs2_inode *ip, __be64 *buf,
 static int gfs2_dir_read_data(struct gfs2_inode *ip, __be64 *buf,
 			      unsigned int size)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	u64 lblock, dblock;
 	u32 extlen = 0;
@@ -299,9 +304,9 @@ static int gfs2_dir_read_data(struct gfs2_inode *ip, __be64 *buf,
 			if (error || !dblock)
 				goto fail;
 			BUG_ON(extlen < 1);
-			bh = gfs2_meta_ra(ip->i_gl, dblock, extlen);
+			bh = gfs2_meta_ra(gl, dblock, extlen);
 		} else {
-			error = gfs2_meta_read(ip->i_gl, dblock, DIO_WAIT, 0, &bh);
+			error = gfs2_meta_read(gl, dblock, DIO_WAIT, 0, &bh);
 			if (error)
 				goto fail;
 		}
@@ -672,6 +677,7 @@ static int dirent_next(struct gfs2_inode *dip, struct buffer_head *bh,
 static void dirent_del(struct gfs2_inode *dip, struct buffer_head *bh,
 		       struct gfs2_dirent *prev, struct gfs2_dirent *cur)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&dip->i_inode);
 	u16 cur_rec_len, prev_rec_len;
 
 	if (gfs2_dirent_sentinel(cur)) {
@@ -679,7 +685,7 @@ static void dirent_del(struct gfs2_inode *dip, struct buffer_head *bh,
 		return;
 	}
 
-	gfs2_trans_add_meta(dip->i_gl, bh);
+	gfs2_trans_add_meta(gl, bh);
 
 	/* If there is no prev entry, this is the first entry in the block.
 	   The de_rec_len is already as big as it needs to be.  Just zero
@@ -712,13 +718,13 @@ static struct gfs2_dirent *do_init_dirent(struct inode *inode,
 					  struct buffer_head *bh,
 					  unsigned offset)
 {
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_dirent *ndent;
 	unsigned totlen;
 
 	totlen = be16_to_cpu(dent->de_rec_len);
 	BUG_ON(offset + name->len > totlen);
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	gfs2_trans_add_meta(gl, bh);
 	ndent = (struct gfs2_dirent *)((char *)dent + offset);
 	dent->de_rec_len = cpu_to_be16(offset);
 	gfs2_qstr2dirent(name, totlen - offset, ndent);
@@ -759,10 +765,13 @@ static struct gfs2_dirent *gfs2_dirent_split_alloc(struct inode *inode,
 static int get_leaf(struct gfs2_inode *dip, u64 leaf_no,
 		    struct buffer_head **bhp)
 {
+	struct inode *inode = &dip->i_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
+	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	int error;
 
-	error = gfs2_meta_read(dip->i_gl, leaf_no, DIO_WAIT, 0, bhp);
-	if (!error && gfs2_metatype_check(GFS2_SB(&dip->i_inode), *bhp, GFS2_METATYPE_LF)) {
+	error = gfs2_meta_read(gl, leaf_no, DIO_WAIT, 0, bhp);
+	if (!error && gfs2_metatype_check(sdp, *bhp, GFS2_METATYPE_LF)) {
 		/* pr_info("block num=%llu\n", leaf_no); */
 		error = -EIO;
 	}
@@ -863,6 +872,7 @@ static struct gfs2_dirent *gfs2_dirent_search(struct inode *inode,
 
 static struct gfs2_leaf *new_leaf(struct inode *inode, struct buffer_head **pbh, u16 depth)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	unsigned int n = 1;
 	u64 bn;
@@ -875,12 +885,12 @@ static struct gfs2_leaf *new_leaf(struct inode *inode, struct buffer_head **pbh,
 	error = gfs2_alloc_blocks(ip, &bn, &n, 0);
 	if (error)
 		return NULL;
-	bh = gfs2_meta_new(ip->i_gl, bn);
+	bh = gfs2_meta_new(gl, bn);
 	if (!bh)
 		return NULL;
 
 	gfs2_trans_remove_revoke(GFS2_SB(inode), bn, 1);
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	gfs2_trans_add_meta(gl, bh);
 	gfs2_metatype_set(bh, GFS2_METATYPE_LF, GFS2_FORMAT_LF);
 	leaf = (struct gfs2_leaf *)bh->b_data;
 	leaf->lf_depth = cpu_to_be16(depth);
@@ -907,6 +917,7 @@ static struct gfs2_leaf *new_leaf(struct inode *inode, struct buffer_head **pbh,
 
 static int dir_make_exhash(struct inode *inode)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *dip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct gfs2_dirent *dent;
@@ -968,7 +979,7 @@ static int dir_make_exhash(struct inode *inode)
 	/*  We're done with the new leaf block, now setup the new
 	    hash table.  */
 
-	gfs2_trans_add_meta(dip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
 
 	lp = (__be64 *)(dibh->b_data + sizeof(struct gfs2_dinode));
@@ -998,6 +1009,7 @@ static int dir_make_exhash(struct inode *inode)
 
 static int dir_split_leaf(struct inode *inode, const struct qstr *name)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *dip = GFS2_I(inode);
 	struct buffer_head *nbh, *obh, *dibh;
 	struct gfs2_leaf *nleaf, *oleaf;
@@ -1025,7 +1037,7 @@ static int dir_split_leaf(struct inode *inode, const struct qstr *name)
 		return 1; /* can't split */
 	}
 
-	gfs2_trans_add_meta(dip->i_gl, obh);
+	gfs2_trans_add_meta(gl, obh);
 
 	nleaf = new_leaf(inode, &nbh, be16_to_cpu(oleaf->lf_depth) + 1);
 	if (!nleaf) {
@@ -1118,7 +1130,7 @@ static int dir_split_leaf(struct inode *inode, const struct qstr *name)
 
 	error = gfs2_meta_inode_buffer(dip, &dibh);
 	if (!gfs2_assert_withdraw(GFS2_SB(&dip->i_inode), !error)) {
-		gfs2_trans_add_meta(dip->i_gl, dibh);
+		gfs2_trans_add_meta(gl, dibh);
 		gfs2_add_inode_blocks(&dip->i_inode, 1);
 		gfs2_dinode_out(dip, dibh->b_data);
 		brelse(dibh);
@@ -1480,8 +1492,8 @@ static int gfs2_dir_read_leaf(struct inode *inode, struct dir_context *ctx,
 static void gfs2_dir_readahead(struct inode *inode, unsigned hsize, u32 index,
 			       struct file_ra_state *f_ra)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
-	struct gfs2_glock *gl = ip->i_gl;
 	struct buffer_head *bh;
 	u64 blocknr = 0, last;
 	unsigned count;
@@ -1722,6 +1734,7 @@ int gfs2_dir_check(struct inode *dir, const struct qstr *name,
 static int dir_new_leaf(struct inode *inode, const struct qstr *name)
 {
 	struct buffer_head *bh, *obh;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_leaf *leaf, *oleaf;
 	u32 dist = 1;
@@ -1745,7 +1758,7 @@ static int dir_new_leaf(struct inode *inode, const struct qstr *name)
 			return error;
 	} while(1);
 
-	gfs2_trans_add_meta(ip->i_gl, obh);
+	gfs2_trans_add_meta(gl, obh);
 
 	leaf = new_leaf(inode, &bh, be16_to_cpu(oleaf->lf_depth));
 	if (!leaf) {
@@ -1760,7 +1773,7 @@ static int dir_new_leaf(struct inode *inode, const struct qstr *name)
 	error = gfs2_meta_inode_buffer(ip, &bh);
 	if (error)
 		return error;
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	gfs2_trans_add_meta(gl, bh);
 	gfs2_add_inode_blocks(&ip->i_inode, 1);
 	gfs2_dinode_out(ip, bh->b_data);
 	brelse(bh);
@@ -1935,6 +1948,7 @@ int gfs2_dir_del(struct gfs2_inode *dip, const struct dentry *dentry)
 int gfs2_dir_mvino(struct gfs2_inode *dip, const struct qstr *filename,
 		   const struct gfs2_inode *nip, unsigned int new_type)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&dip->i_inode);
 	struct buffer_head *bh;
 	struct gfs2_dirent *dent;
 
@@ -1946,7 +1960,7 @@ int gfs2_dir_mvino(struct gfs2_inode *dip, const struct qstr *filename,
 	if (IS_ERR(dent))
 		return PTR_ERR(dent);
 
-	gfs2_trans_add_meta(dip->i_gl, bh);
+	gfs2_trans_add_meta(gl, bh);
 	gfs2_inum_out(nip, dent);
 	dent->de_type = cpu_to_be16(new_type);
 	brelse(bh);
@@ -1972,6 +1986,7 @@ static int leaf_dealloc(struct gfs2_inode *dip, u32 index, u32 len,
 			u64 leaf_no, struct buffer_head *leaf_bh,
 			int last_dealloc)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&dip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&dip->i_inode);
 	struct gfs2_leaf *tmp_leaf;
 	struct gfs2_rgrp_list rlist;
@@ -2066,7 +2081,7 @@ static int leaf_dealloc(struct gfs2_inode *dip, u32 index, u32 len,
 	if (error)
 		goto out_end_trans;
 
-	gfs2_trans_add_meta(dip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 	/* On the last dealloc, make this a regular file in case we crash.
 	   (We don't want to free these blocks a second time.)  */
 	if (last_dealloc)
diff --git a/fs/gfs2/export.c b/fs/gfs2/export.c
index 3334c394ce9cbe..970bf2d723884b 100644
--- a/fs/gfs2/export.c
+++ b/fs/gfs2/export.c
@@ -87,7 +87,8 @@ static int gfs2_get_name(struct dentry *parent, char *name,
 {
 	struct inode *dir = d_inode(parent);
 	struct inode *inode = d_inode(child);
-	struct gfs2_inode *dip, *ip;
+	struct gfs2_glock *gl;
+	struct gfs2_inode *ip;
 	struct get_name_filldir gnfd = {
 		.ctx.actor = get_name_filldir,
 		.name = name
@@ -102,14 +103,14 @@ static int gfs2_get_name(struct dentry *parent, char *name,
 	if (!S_ISDIR(dir->i_mode) || !inode)
 		return -EINVAL;
 
-	dip = GFS2_I(dir);
+	gl = gfs2_inode_glock(dir);
 	ip = GFS2_I(inode);
 
 	*name = 0;
 	gnfd.inum.no_addr = ip->i_no_addr;
 	gnfd.inum.no_formal_ino = ip->i_no_formal_ino;
 
-	error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &gh);
 	if (error)
 		return error;
 
diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c
index b8c10de113ba7a..1efd0679badd93 100644
--- a/fs/gfs2/file.c
+++ b/fs/gfs2/file.c
@@ -57,13 +57,13 @@
 
 static loff_t gfs2_llseek(struct file *file, loff_t offset, int whence)
 {
-	struct gfs2_inode *ip = GFS2_I(file->f_mapping->host);
+	struct gfs2_glock *gl = gfs2_inode_glock(file->f_mapping->host);
 	struct gfs2_holder i_gh;
 	loff_t error;
 
 	switch (whence) {
 	case SEEK_END:
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY,
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY,
 					   &i_gh);
 		if (!error) {
 			error = generic_file_llseek(file, offset, whence);
@@ -105,11 +105,11 @@ static loff_t gfs2_llseek(struct file *file, loff_t offset, int whence)
 static int gfs2_readdir(struct file *file, struct dir_context *ctx)
 {
 	struct inode *dir = file->f_mapping->host;
-	struct gfs2_inode *dip = GFS2_I(dir);
+	struct gfs2_glock *gl = gfs2_inode_glock(dir);
 	struct gfs2_holder d_gh;
 	int error;
 
-	error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &d_gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &d_gh);
 	if (error)
 		return error;
 
@@ -158,6 +158,7 @@ static inline u32 gfs2_gfsflags_to_fsflags(struct inode *inode, u32 gfsflags)
 int gfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder gh;
 	int error;
@@ -166,7 +167,7 @@ int gfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	if (d_is_special(dentry))
 		return -ENOTTY;
 
-	gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+	gfs2_holder_init(gl, LM_ST_SHARED, 0, &gh);
 	error = gfs2_glock_nq(&gh);
 	if (error)
 		goto out_uninit;
@@ -218,6 +219,7 @@ void gfs2_set_inode_flags(struct inode *inode)
  */
 static int do_gfs2_set_flags(struct inode *inode, u32 reqflags, u32 mask)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct buffer_head *bh;
@@ -225,7 +227,7 @@ static int do_gfs2_set_flags(struct inode *inode, u32 reqflags, u32 mask)
 	int error;
 	u32 new_flags, flags;
 
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, &gh);
 	if (error)
 		return error;
 
@@ -242,7 +244,7 @@ static int do_gfs2_set_flags(struct inode *inode, u32 reqflags, u32 mask)
 	}
 	if ((flags ^ new_flags) & GFS2_DIF_JDATA) {
 		if (new_flags & GFS2_DIF_JDATA)
-			gfs2_log_flush(sdp, ip->i_gl,
+			gfs2_log_flush(sdp, gl,
 				       GFS2_LOG_HEAD_FLUSH_NORMAL |
 				       GFS2_LFC_SET_FLAGS);
 		error = filemap_fdatawrite(inode->i_mapping);
@@ -262,7 +264,7 @@ static int do_gfs2_set_flags(struct inode *inode, u32 reqflags, u32 mask)
 	if (error)
 		goto out_trans_end;
 	inode_set_ctime_current(inode);
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	gfs2_trans_add_meta(gl, bh);
 	ip->i_diskflags = new_flags;
 	gfs2_dinode_out(ip, bh->b_data);
 	brelse(bh);
@@ -275,7 +277,7 @@ static int do_gfs2_set_flags(struct inode *inode, u32 reqflags, u32 mask)
 	return error;
 }
 
-int gfs2_fileattr_set(struct mnt_idmap *idmap,
+int gfs2_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
@@ -417,6 +419,7 @@ static vm_fault_t gfs2_page_mkwrite(struct vm_fault *vmf)
 {
 	struct folio *folio = page_folio(vmf->page);
 	struct inode *inode = file_inode(vmf->vma->vm_file);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct gfs2_alloc_parms ap = {};
@@ -430,7 +433,7 @@ static vm_fault_t gfs2_page_mkwrite(struct vm_fault *vmf)
 
 	sb_start_pagefault(inode->i_sb);
 
-	gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
+	gfs2_holder_init(gl, LM_ST_EXCLUSIVE, 0, &gh);
 	err = gfs2_glock_nq(&gh);
 	if (err) {
 		ret = vmf_fs_error(err);
@@ -455,7 +458,7 @@ static vm_fault_t gfs2_page_mkwrite(struct vm_fault *vmf)
 
 	gfs2_size_hint(vmf->vma->vm_file, pos, length);
 
-	set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
+	set_bit(GLF_DIRTY, &gl->gl_flags);
 	set_bit(GIF_SW_PAGED, &ip->i_flags);
 
 	/*
@@ -552,12 +555,12 @@ static vm_fault_t gfs2_page_mkwrite(struct vm_fault *vmf)
 static vm_fault_t gfs2_fault(struct vm_fault *vmf)
 {
 	struct inode *inode = file_inode(vmf->vma->vm_file);
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_holder gh;
 	vm_fault_t ret;
 	int err;
 
-	gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+	gfs2_holder_init(gl, LM_ST_SHARED, 0, &gh);
 	err = gfs2_glock_nq(&gh);
 	if (err) {
 		ret = vmf_fs_error(err);
@@ -590,6 +593,7 @@ static const struct vm_operations_struct gfs2_vm_ops = {
 
 static int gfs2_mmap(struct file *file, struct vm_area_struct *vma)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(file->f_mapping->host);
 	struct gfs2_inode *ip = GFS2_I(file->f_mapping->host);
 
 	if (!(file->f_flags & O_NOATIME) &&
@@ -597,7 +601,7 @@ static int gfs2_mmap(struct file *file, struct vm_area_struct *vma)
 		struct gfs2_holder i_gh;
 		int error;
 
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY,
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY,
 					   &i_gh);
 		if (error)
 			return error;
@@ -674,13 +678,14 @@ int gfs2_open_common(struct inode *inode, struct file *file)
 
 static int gfs2_open(struct inode *inode, struct file *file)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder i_gh;
 	int error;
 	bool need_unlock = false;
 
 	if (S_ISREG(ip->i_inode.i_mode)) {
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY,
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY,
 					   &i_gh);
 		if (error)
 			return error;
@@ -745,6 +750,7 @@ static int gfs2_fsync(struct file *file, loff_t start, loff_t end,
 	struct address_space *mapping = file->f_mapping;
 	struct inode *inode = mapping->host;
 	int sync_state = inode_state_read_once(inode) & I_DIRTY;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	int ret = 0, ret1 = 0;
 
@@ -767,7 +773,7 @@ static int gfs2_fsync(struct file *file, loff_t start, loff_t end,
 			ret = file_write_and_wait(file);
 		if (ret)
 			return ret;
-		gfs2_ail_flush(ip->i_gl, 1);
+		gfs2_ail_flush(gl, 1);
 	}
 
 	if (mapping->nrpages)
@@ -812,7 +818,8 @@ static ssize_t gfs2_file_direct_read(struct kiocb *iocb, struct iov_iter *to,
 				     struct gfs2_holder *gh)
 {
 	struct file *file = iocb->ki_filp;
-	struct gfs2_inode *ip = GFS2_I(file->f_mapping->host);
+	struct inode *inode = file->f_mapping->host;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	size_t prev_count = 0, window_size = 0;
 	size_t read = 0;
 	ssize_t ret;
@@ -837,7 +844,7 @@ static ssize_t gfs2_file_direct_read(struct kiocb *iocb, struct iov_iter *to,
 	if (!iov_iter_count(to))
 		return 0; /* skip atime */
 
-	gfs2_holder_init(ip->i_gl, LM_ST_DEFERRED, 0, gh);
+	gfs2_holder_init(gl, LM_ST_DEFERRED, 0, gh);
 retry:
 	ret = gfs2_glock_nq(gh);
 	if (ret)
@@ -876,7 +883,7 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from,
 {
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_mapping->host;
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	size_t prev_count = 0, window_size = 0;
 	size_t written = 0;
 	bool enough_retries;
@@ -900,13 +907,13 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from,
 	 * unfortunately, have the option of only flushing a range like the
 	 * VFS does.
 	 */
-	gfs2_holder_init(ip->i_gl, LM_ST_DEFERRED, 0, gh);
+	gfs2_holder_init(gl, LM_ST_DEFERRED, 0, gh);
 retry:
 	ret = gfs2_glock_nq(gh);
 	if (ret)
 		goto out_uninit;
 	/* Silently fall back to buffered I/O when writing beyond EOF */
-	if (iocb->ki_pos + iov_iter_count(from) > i_size_read(&ip->i_inode))
+	if (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode))
 		goto out_unlock;
 
 	from->nofault = true;
@@ -948,7 +955,7 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from,
 
 static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
-	struct gfs2_inode *ip;
+	struct gfs2_glock *gl;
 	struct gfs2_holder gh;
 	size_t prev_count = 0, window_size = 0;
 	size_t read = 0;
@@ -979,8 +986,8 @@ static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		if (iocb->ki_flags & IOCB_NOWAIT)
 			return ret;
 	}
-	ip = GFS2_I(iocb->ki_filp->f_mapping->host);
-	gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+	gl = gfs2_inode_glock(iocb->ki_filp->f_mapping->host);
+	gfs2_holder_init(gl, LM_ST_SHARED, 0, &gh);
 retry:
 	ret = gfs2_glock_nq(&gh);
 	if (ret)
@@ -1013,7 +1020,7 @@ static ssize_t gfs2_file_buffered_write(struct kiocb *iocb,
 {
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = file_inode(file);
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct gfs2_holder *statfs_gh = NULL;
 	size_t prev_count = 0, window_size = 0;
@@ -1034,7 +1041,7 @@ static ssize_t gfs2_file_buffered_write(struct kiocb *iocb,
 			return -ENOMEM;
 	}
 
-	gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, gh);
+	gfs2_holder_init(gl, LM_ST_EXCLUSIVE, 0, gh);
 	if (should_fault_in_pages(from, iocb, &prev_count, &window_size)) {
 retry:
 		window_size -= fault_in_iov_iter_readable(from, window_size);
@@ -1049,9 +1056,9 @@ static ssize_t gfs2_file_buffered_write(struct kiocb *iocb,
 		goto out_uninit;
 
 	if (inode == sdp->sd_rindex) {
-		struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
+		struct gfs2_glock *m_gl = gfs2_inode_glock(sdp->sd_statfs_inode);
 
-		ret = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE,
+		ret = gfs2_glock_nq_init(m_gl, LM_ST_EXCLUSIVE,
 					 GL_NOCACHE, statfs_gh);
 		if (ret)
 			goto out_unlock;
@@ -1105,14 +1112,15 @@ static ssize_t gfs2_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 {
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = file_inode(file);
-	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder gh;
 	ssize_t ret;
 
 	gfs2_size_hint(file, iocb->ki_pos, iov_iter_count(from));
 
 	if (iocb->ki_flags & IOCB_APPEND) {
-		ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+		struct gfs2_glock *gl = gfs2_inode_glock(inode);
+
+		ret = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &gh);
 		if (ret)
 			return ret;
 		gfs2_glock_dq_uninit(&gh);
@@ -1180,6 +1188,7 @@ static ssize_t gfs2_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
 static int fallocate_chunk(struct inode *inode, loff_t offset, loff_t len)
 {
 	struct super_block *sb = inode->i_sb;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	loff_t end = offset + len;
 	struct buffer_head *dibh;
@@ -1189,7 +1198,7 @@ static int fallocate_chunk(struct inode *inode, loff_t offset, loff_t len)
 	if (unlikely(error))
 		return error;
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gl, dibh);
 
 	if (gfs2_is_stuffed(ip)) {
 		error = gfs2_unstuff_dinode(ip);
@@ -1378,6 +1387,7 @@ static long gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t le
 {
 	struct inode *inode = file_inode(file);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder gh;
 	int ret;
@@ -1390,7 +1400,7 @@ static long gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t le
 
 	inode_lock(inode);
 
-	gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
+	gfs2_holder_init(gl, LM_ST_EXCLUSIVE, 0, &gh);
 	ret = gfs2_glock_nq(&gh);
 	if (ret)
 		goto out_uninit;
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index d59ea71a84db06..d22a088c66cdab 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -329,11 +329,6 @@ static void gfs2_holder_wake(struct gfs2_holder *gh)
 	clear_bit(HIF_WAIT, &gh->gh_iflags);
 	smp_mb__after_atomic();
 	wake_up_bit(&gh->gh_iflags, HIF_WAIT);
-	if (gh->gh_flags & GL_ASYNC) {
-		struct gfs2_sbd *sdp = glock_sbd(gh->gh_gl);
-
-		wake_up(&sdp->sd_async_glock_wait);
-	}
 }
 
 /**
@@ -512,11 +507,9 @@ static void state_change(struct gfs2_glock *gl, unsigned int new_state)
 
 static void gfs2_set_demote(int nr, struct gfs2_glock *gl)
 {
-	struct gfs2_sbd *sdp = glock_sbd(gl);
-
 	set_bit(nr, &gl->gl_flags);
-	smp_mb();
-	wake_up(&sdp->sd_async_glock_wait);
+	smp_mb__after_atomic();
+	wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
 }
 
 static void gfs2_demote_wake(struct gfs2_glock *gl)
@@ -891,7 +884,9 @@ static void gfs2_try_to_evict(struct gfs2_glock *gl)
 		/* If the inode was evicted, gl->gl_object will now be NULL. */
 		ip = gfs2_grab_existing_inode(gl);
 		if (ip) {
-			gfs2_glock_poke(ip->i_gl);
+			struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
+
+			gfs2_glock_poke(gl);
 			iput(&ip->i_inode);
 		}
 	}
@@ -1270,11 +1265,14 @@ static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
 int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs,
 			  unsigned int retries)
 {
-	struct gfs2_sbd *sdp = glock_sbd(ghs[0].gh_gl);
 	unsigned long start_time = jiffies;
-	int i, ret = 0;
-	long timeout;
+	struct wait_queue_head *waitq[4];
+	struct wait_queue_entry wait[4];
+	long ret, timeout;
+	int i;
 
+	BUILD_BUG_ON(ARRAY_SIZE(waitq) != ARRAY_SIZE(wait));
+	BUG_ON(num_gh > ARRAY_SIZE(wait));
 	might_sleep();
 
 	timeout = GL_GLOCK_MIN_HOLD;
@@ -1292,14 +1290,38 @@ int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs,
 		timeout += (incr / 3) + get_random_long() % (incr / 3);
 	}
 
-	if (!wait_event_interruptible_timeout(sdp->sd_async_glock_wait,
-				!glocks_pending(num_gh, ghs), timeout)) {
-		ret = -ESTALE; /* request timed out. */
-		goto out;
+	ret = timeout;
+	for (i = 0; i < num_gh; i++) {
+		waitq[i] = bit_waitqueue(&ghs[i].gh_iflags, HIF_WAIT);
+		init_wait(wait + i);
 	}
-	if (signal_pending(current))
-		goto interrupted;
+	for (;;) {
+		for (i = 0; i < num_gh; i++)
+			prepare_to_wait(waitq[i], wait + i, TASK_INTERRUPTIBLE);
+		if (!glocks_pending(num_gh, ghs))
+			break;
+		if (signal_pending(current)) {
+			ret = -EINTR;
+			break;
+		}
+		ret = schedule_timeout(ret);
+		if (!glocks_pending(num_gh, ghs))
+			break;
+		if (!ret) {
+			ret = -ESTALE; /* request timed out. */
+			break;
+		}
+		if (signal_pending(current)) {
+			ret = -EINTR;
+			break;
+		}
+	}
+	for (i = 0; i < num_gh; i++)
+		finish_wait(waitq[i], wait + i);
+	if (ret < 0)
+		goto out;
 
+	ret = 0;
 	for (i = 0; i < num_gh; i++) {
 		struct gfs2_holder *gh = &ghs[i];
 		int ret2;
diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c
index 28f32424ee64bf..2597e1cf2315c7 100644
--- a/fs/gfs2/glops.c
+++ b/fs/gfs2/glops.c
@@ -393,6 +393,7 @@ static int gfs2_dinode_in(struct gfs2_inode *ip, const void *buf)
 	umode_t mode = be32_to_cpu(str->di_mode);
 	struct inode *inode = &ip->i_inode;
 	bool is_new = inode_state_read_once(inode) & I_NEW;
+	u64 size;
 
 	if (unlikely(ip->i_no_addr != be64_to_cpu(str->di_num.no_addr))) {
 		gfs2_consist_inode(ip);
@@ -418,7 +419,12 @@ static int gfs2_dinode_in(struct gfs2_inode *ip, const void *buf)
 	i_uid_write(inode, be32_to_cpu(str->di_uid));
 	i_gid_write(inode, be32_to_cpu(str->di_gid));
 	set_nlink(inode, be32_to_cpu(str->di_nlink));
-	i_size_write(inode, be64_to_cpu(str->di_size));
+	size = be64_to_cpu(str->di_size);
+	if (unlikely(size > inode->i_sb->s_maxbytes)) {
+		gfs2_consist_inode(ip);
+		return -EIO;
+	}
+	i_size_write(inode, size);
 	gfs2_set_inode_blocks(inode, be64_to_cpu(str->di_blocks));
 	atime.tv_sec = be64_to_cpu(str->di_atime);
 	atime.tv_nsec = be32_to_cpu(str->di_atime_nsec);
@@ -462,7 +468,7 @@ static int gfs2_dinode_in(struct gfs2_inode *ip, const void *buf)
 		return -EIO;
 	}
 
-	if (gfs2_is_stuffed(ip) && inode->i_size > gfs2_max_stuffed_size(ip)) {
+	if (gfs2_is_stuffed(ip) && size > gfs2_max_stuffed_size(ip)) {
 		gfs2_consist_inode(ip);
 		return -EIO;
 	}
@@ -602,8 +608,7 @@ static void freeze_go_callback(struct gfs2_glock *gl, bool remote)
 static int freeze_go_xmote_bh(struct gfs2_glock *gl)
 {
 	struct gfs2_sbd *sdp = glock_sbd(gl);
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_jdesc->jd_inode);
-	struct gfs2_glock *j_gl = ip->i_gl;
+	struct gfs2_glock *j_gl = gfs2_inode_glock(sdp->sd_jdesc->jd_inode);
 	struct gfs2_log_header_host head;
 	int error;
 
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index dadb4d3c9d3d61..49cc232942a293 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -391,7 +391,7 @@ struct gfs2_inode {
 	u64 i_generation;
 	u64 i_eattr;
 	unsigned long i_flags;		/* GIF_... */
-	struct gfs2_glock *i_gl;
+	struct gfs2_glock __rcu *i_gl;
 	struct gfs2_holder i_iopen_gh;
 	struct gfs2_qadata *i_qadata; /* quota allocation data */
 	struct gfs2_holder i_rgd_gh;
@@ -716,7 +716,6 @@ struct gfs2_sbd {
 	struct work_struct sd_freeze_work;
 	struct work_struct sd_withdraw_work;
 	wait_queue_head_t sd_kill_wait;
-	wait_queue_head_t sd_async_glock_wait;
 	atomic_t sd_glock_disposal;
 	struct completion sd_locking_init;
 	struct completion sd_withdraw_helper;
@@ -879,5 +878,8 @@ static inline unsigned gfs2_max_stuffed_size(const struct gfs2_inode *ip)
 	return GFS2_SB(&ip->i_inode)->sd_sb.sb_bsize - sizeof(struct gfs2_dinode);
 }
 
+static inline struct gfs2_glock *gfs2_inode_glock(struct inode *inode)
+{
+       return rcu_dereference_protected(GFS2_I(inode)->i_gl, 1);
+}
 #endif /* __INCORE_DOT_H__ */
-
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
index f361876c558335..2c2dc459c0375a 100644
--- a/fs/gfs2/inode.c
+++ b/fs/gfs2/inode.c
@@ -130,6 +130,7 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type,
 {
 	struct inode *inode;
 	struct gfs2_inode *ip;
+	struct gfs2_glock *gl = NULL;
 	struct gfs2_holder i_gh;
 	int error;
 
@@ -147,9 +148,10 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type,
 
 		gfs2_setup_inode(inode);
 		error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE,
-				       &ip->i_gl);
+				       &gl);
 		if (unlikely(error))
 			goto fail;
+		rcu_assign_pointer(ip->i_gl, gl);
 
 		error = gfs2_glock_get(sdp, no_addr, &gfs2_iopen_glops, CREATE,
 				       &io_gl);
@@ -178,14 +180,14 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type,
 			 * block.  We read the inode when instantiating it
 			 * after possibly checking the block type.
 			 */
-			error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE,
+			error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE,
 						   GL_SKIP, &i_gh);
 			if (error)
 				goto fail;
 
 			error = -ESTALE;
 			if (no_formal_ino &&
-			    gfs2_inode_already_deleted(ip->i_gl, no_formal_ino))
+			    gfs2_inode_already_deleted(gl, no_formal_ino))
 				goto fail;
 
 			if (blktype != GFS2_BLKST_FREE) {
@@ -196,20 +198,20 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type,
 			}
 		}
 
-		set_bit(GLF_INSTANTIATE_NEEDED, &ip->i_gl->gl_flags);
+		set_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags);
 
 		/* Lowest possible timestamp; will be overwritten in gfs2_dinode_in. */
 		inode_set_atime(inode,
 				1LL << (8 * sizeof(inode_get_atime_sec(inode)) - 1),
 				0);
 
-		glock_set_object(ip->i_gl, ip);
+		glock_set_object(gl, ip);
 
 		if (type == DT_UNKNOWN) {
 			/* Inode glock must be locked already */
 			error = gfs2_instantiate(&i_gh);
 			if (error) {
-				glock_clear_object(ip->i_gl, ip);
+				glock_clear_object(gl, ip);
 				goto fail;
 			}
 		} else {
@@ -240,9 +242,9 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type,
 		gfs2_glock_dq_uninit(&ip->i_iopen_gh);
 	if (gfs2_holder_initialized(&i_gh))
 		gfs2_glock_dq_uninit(&i_gh);
-	if (ip->i_gl) {
-		gfs2_glock_put(ip->i_gl);
-		ip->i_gl = NULL;
+	if (gl) {
+		gfs2_glock_put(gl);
+		rcu_assign_pointer(ip->i_gl, NULL);
 	}
 	iget_failed(inode);
 	return ERR_PTR(error);
@@ -322,8 +324,8 @@ struct inode *gfs2_lookup_meta(struct inode *dip, const char *name)
 struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name,
 			   int is_root)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(dir);
 	struct super_block *sb = dir->i_sb;
-	struct gfs2_inode *dip = GFS2_I(dir);
 	struct gfs2_holder d_gh;
 	int error = 0;
 	struct inode *inode = NULL;
@@ -339,8 +341,8 @@ struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name,
 		return dir;
 	}
 
-	if (gfs2_glock_is_locked_by_me(dip->i_gl) == NULL) {
-		error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &d_gh);
+	if (gfs2_glock_is_locked_by_me(gl) == NULL) {
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &d_gh);
 		if (error)
 			return ERR_PTR(error);
 	}
@@ -456,7 +458,7 @@ static int alloc_dinode(struct gfs2_inode *ip, u32 flags, unsigned *dblocks)
 static void gfs2_final_release_pages(struct gfs2_inode *ip)
 {
 	struct inode *inode = &ip->i_inode;
-	struct gfs2_glock *gl = ip->i_gl;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 
 	/* This can only happen during incomplete inode creation. */
 	if (unlikely(!gl))
@@ -546,12 +548,13 @@ static void gfs2_init_dir(struct buffer_head *dibh,
 
 static void gfs2_init_xattr(struct gfs2_inode *ip)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct buffer_head *bh;
 	struct gfs2_ea_header *ea;
 
-	bh = gfs2_meta_new(ip->i_gl, ip->i_eattr);
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	bh = gfs2_meta_new(gl, ip->i_eattr);
+	gfs2_trans_add_meta(gl, bh);
 	gfs2_metatype_set(bh, GFS2_METATYPE_EA, GFS2_FORMAT_EA);
 	gfs2_buffer_clear_tail(bh, sizeof(struct gfs2_meta_header));
 
@@ -574,11 +577,12 @@ static void gfs2_init_xattr(struct gfs2_inode *ip)
 static void init_dinode(struct gfs2_inode *dip, struct gfs2_inode *ip,
 			const char *symname)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_dinode *di;
 	struct buffer_head *dibh;
 
-	dibh = gfs2_meta_new(ip->i_gl, ip->i_no_addr);
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	dibh = gfs2_meta_new(gl, ip->i_no_addr);
+	gfs2_trans_add_meta(gl, dibh);
 	di = (struct gfs2_dinode *)dibh->b_data;
 	gfs2_dinode_out(ip, di);
 
@@ -708,7 +712,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	struct inode *inode = NULL;
 	struct gfs2_inode *dip = GFS2_I(dir), *ip;
 	struct gfs2_sbd *sdp = GFS2_SB(&dip->i_inode);
-	struct gfs2_glock *io_gl;
+	struct gfs2_glock *gl = NULL, *io_gl;
 	int error, dealloc_error;
 	u32 aflags = 0;
 	unsigned blocks = 1;
@@ -726,7 +730,8 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	if (error)
 		goto fail;
 
-	error = gfs2_glock_nq_init(dip->i_gl, LM_ST_EXCLUSIVE, 0, &d_gh);
+	error = gfs2_glock_nq_init(gfs2_inode_glock(dir), LM_ST_EXCLUSIVE, 0,
+				   &d_gh);
 	if (error)
 		goto fail;
 	gfs2_holder_mark_uninitialized(&gh);
@@ -735,7 +740,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	if (error)
 		goto fail_gunlock;
 
-	inode = gfs2_dir_search(dir, &dentry->d_name, !S_ISREG(mode) || excl);
+	inode = gfs2_dir_search(dir, &dentry->d_name, excl);
 	error = PTR_ERR(inode);
 	if (!IS_ERR(inode)) {
 		if (file && (file->f_flags & __O_REGULAR) &&
@@ -832,9 +837,10 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 
 	gfs2_set_inode_blocks(inode, blocks);
 
-	error = gfs2_glock_get(sdp, ip->i_no_addr, &gfs2_inode_glops, CREATE, &ip->i_gl);
+	error = gfs2_glock_get(sdp, ip->i_no_addr, &gfs2_inode_glops, CREATE, &gl);
 	if (error)
 		goto fail_dealloc_inode;
+	rcu_assign_pointer(ip->i_gl, gl);
 
 	error = gfs2_glock_get(sdp, ip->i_no_addr, &gfs2_iopen_glops, CREATE, &io_gl);
 	if (error)
@@ -854,10 +860,10 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	if (error)
 		goto fail_gunlock2;
 
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_SKIP, &gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, GL_SKIP, &gh);
 	if (error)
 		goto fail_gunlock3;
-	clear_bit(GLF_INSTANTIATE_NEEDED, &ip->i_gl->gl_flags);
+	clear_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags);
 
 	error = gfs2_trans_begin(sdp, blocks, 0);
 	if (error)
@@ -870,7 +876,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	init_dinode(dip, ip, symname);
 	gfs2_trans_end(sdp);
 
-	glock_set_object(ip->i_gl, ip);
+	glock_set_object(gl, ip);
 	glock_set_object(io_gl, ip);
 	gfs2_set_iop(inode);
 
@@ -914,7 +920,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	return error;
 
 fail_gunlock4:
-	glock_clear_object(ip->i_gl, ip);
+	glock_clear_object(gl, ip);
 	glock_clear_object(io_gl, ip);
 fail_gunlock3:
 	gfs2_glock_dq_uninit(&ip->i_iopen_gh);
@@ -932,9 +938,9 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 		fs_warn(sdp, "%s: %d\n", __func__, dealloc_error);
 	ip->i_no_addr = 0;
 fail_free_inode:
-	if (ip->i_gl) {
-		gfs2_glock_put(ip->i_gl);
-		ip->i_gl = NULL;
+	if (gl) {
+		gfs2_glock_put(gl);
+		rcu_assign_pointer(ip->i_gl, NULL);
 	}
 	gfs2_rs_deltree(&ip->i_res);
 	gfs2_qa_put(ip);
@@ -967,7 +973,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
  * Returns: errno
  */
 
-static int gfs2_create(struct mnt_idmap *idmap, struct inode *dir,
+static int gfs2_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	return gfs2_create_inode(dir, dentry, NULL, S_IFREG | mode, 0, NULL, 0, 1);
@@ -996,7 +1002,7 @@ static struct dentry *__gfs2_lookup(struct inode *dir, struct dentry *dentry,
 	if (inode == NULL || IS_ERR(inode))
 		return d_splice_alias(inode, dentry);
 
-	gl = GFS2_I(inode)->i_gl;
+	gl = gfs2_inode_glock(inode);
 	error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &gh);
 	if (error) {
 		iput(inode);
@@ -1056,8 +1062,8 @@ static int gfs2_link(struct dentry *old_dentry, struct inode *dir,
 	if (error)
 		return error;
 
-	gfs2_holder_init(dip->i_gl, LM_ST_EXCLUSIVE, 0, &d_gh);
-	gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
+	gfs2_holder_init(gfs2_inode_glock(dir), LM_ST_EXCLUSIVE, 0, &d_gh);
+	gfs2_holder_init(gfs2_inode_glock(inode), LM_ST_EXCLUSIVE, 0, &gh);
 
 	error = gfs2_glock_nq(&d_gh);
 	if (error)
@@ -1130,7 +1136,7 @@ static int gfs2_link(struct dentry *old_dentry, struct inode *dir,
 	if (error)
 		goto out_brelse;
 
-	gfs2_trans_add_meta(ip->i_gl, dibh);
+	gfs2_trans_add_meta(gfs2_inode_glock(inode), dibh);
 	inc_nlink(&ip->i_inode);
 	inode_set_ctime_current(&ip->i_inode);
 	ihold(inode);
@@ -1256,8 +1262,8 @@ static int gfs2_unlink(struct inode *dir, struct dentry *dentry)
 
 	error = -EROFS;
 
-	gfs2_holder_init(dip->i_gl, LM_ST_EXCLUSIVE, 0, &d_gh);
-	gfs2_holder_init(ip->i_gl,  LM_ST_EXCLUSIVE, 0, &gh);
+	gfs2_holder_init(gfs2_inode_glock(dir), LM_ST_EXCLUSIVE, 0, &d_gh);
+	gfs2_holder_init(gfs2_inode_glock(inode),  LM_ST_EXCLUSIVE, 0, &gh);
 
 	rgd = gfs2_blk2rgrpd(sdp, ip->i_no_addr, 1);
 	if (!rgd)
@@ -1323,7 +1329,7 @@ static int gfs2_unlink(struct inode *dir, struct dentry *dentry)
  * Returns: errno
  */
 
-static int gfs2_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int gfs2_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, const char *symname)
 {
 	unsigned int size;
@@ -1332,7 +1338,7 @@ static int gfs2_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	if (size >= gfs2_max_stuffed_size(GFS2_I(dir)))
 		return -ENAMETOOLONG;
 
-	return gfs2_create_inode(dir, dentry, NULL, S_IFLNK | S_IRWXUGO, 0, symname, size, 0);
+	return gfs2_create_inode(dir, dentry, NULL, S_IFLNK | S_IRWXUGO, 0, symname, size, 1);
 }
 
 /**
@@ -1345,12 +1351,12 @@ static int gfs2_symlink(struct mnt_idmap *idmap, struct inode *dir,
  * Returns: the dentry, or ERR_PTR(errno)
  */
 
-static struct dentry *gfs2_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *gfs2_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	unsigned dsize = gfs2_max_stuffed_size(GFS2_I(dir));
 
-	return ERR_PTR(gfs2_create_inode(dir, dentry, NULL, mode, 0, NULL, dsize, 0));
+	return ERR_PTR(gfs2_create_inode(dir, dentry, NULL, mode, 0, NULL, dsize, 1));
 }
 
 /**
@@ -1363,10 +1369,10 @@ static struct dentry *gfs2_mkdir(struct mnt_idmap *idmap, struct inode *dir,
  *
  */
 
-static int gfs2_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int gfs2_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode, dev_t dev)
 {
-	return gfs2_create_inode(dir, dentry, NULL, mode, dev, NULL, 0, 0);
+	return gfs2_create_inode(dir, dentry, NULL, mode, dev, NULL, 0, 1);
 }
 
 /**
@@ -1531,18 +1537,19 @@ static int gfs2_rename(struct inode *odir, struct dentry *odentry,
 	}
 
 	num_gh = 1;
-	gfs2_holder_init(odip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs);
+	gfs2_holder_init(gfs2_inode_glock(odir), LM_ST_EXCLUSIVE, GL_ASYNC, ghs);
 	if (odip != ndip) {
-		gfs2_holder_init(ndip->i_gl, LM_ST_EXCLUSIVE,GL_ASYNC,
+		gfs2_holder_init(gfs2_inode_glock(ndir), LM_ST_EXCLUSIVE,GL_ASYNC,
 				 ghs + num_gh);
 		num_gh++;
 	}
-	gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh);
+	gfs2_holder_init(gfs2_inode_glock(&ip->i_inode), LM_ST_EXCLUSIVE,
+			 GL_ASYNC, ghs + num_gh);
 	num_gh++;
 
 	if (nip) {
-		gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC,
-				 ghs + num_gh);
+		gfs2_holder_init(gfs2_inode_glock(&nip->i_inode),
+				 LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh);
 		num_gh++;
 	}
 
@@ -1777,16 +1784,19 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry,
 	}
 
 	num_gh = 1;
-	gfs2_holder_init(odip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs);
+	gfs2_holder_init(gfs2_inode_glock(odir), LM_ST_EXCLUSIVE, GL_ASYNC,
+			 ghs);
 	if (odip != ndip) {
-		gfs2_holder_init(ndip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC,
-				 ghs + num_gh);
+		gfs2_holder_init(gfs2_inode_glock(ndir), LM_ST_EXCLUSIVE,
+				 GL_ASYNC, ghs + num_gh);
 		num_gh++;
 	}
-	gfs2_holder_init(oip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh);
+	gfs2_holder_init(gfs2_inode_glock(&oip->i_inode), LM_ST_EXCLUSIVE,
+			 GL_ASYNC, ghs + num_gh);
 	num_gh++;
 
-	gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh);
+	gfs2_holder_init(gfs2_inode_glock(&nip->i_inode), LM_ST_EXCLUSIVE,
+			 GL_ASYNC, ghs + num_gh);
 	num_gh++;
 
 again:
@@ -1877,7 +1887,7 @@ static int gfs2_exchange(struct inode *odir, struct dentry *odentry,
 	return error;
 }
 
-static int gfs2_rename2(struct mnt_idmap *idmap, struct inode *odir,
+static int gfs2_rename2(const struct mnt_idmap *idmap, struct inode *odir,
 			struct dentry *odentry, struct inode *ndir,
 			struct dentry *ndentry, unsigned int flags)
 {
@@ -1900,13 +1910,14 @@ static int gfs2_rename2(struct mnt_idmap *idmap, struct inode *odir,
  *
  * This can handle symlinks of any size.
  *
- * Returns: 0 on success or error code
+ * Returns: the link target on success, an ERR_PTR() on failure
  */
 
 static const char *gfs2_get_link(struct dentry *dentry,
 				 struct inode *inode,
 				 struct delayed_call *done)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder i_gh;
 	struct buffer_head *dibh;
@@ -1917,7 +1928,7 @@ static const char *gfs2_get_link(struct dentry *dentry,
 	if (!dentry)
 		return ERR_PTR(-ECHILD);
 
-	gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &i_gh);
+	gfs2_holder_init(gl, LM_ST_SHARED, 0, &i_gh);
 	error = gfs2_glock_nq(&i_gh);
 	if (error) {
 		gfs2_holder_uninit(&i_gh);
@@ -1963,7 +1974,7 @@ static const char *gfs2_get_link(struct dentry *dentry,
  * Returns: errno
  */
 
-int gfs2_permission(struct mnt_idmap *idmap, struct inode *inode,
+int gfs2_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		    int mask)
 {
 	int may_not_block = mask & MAY_NOT_BLOCK;
@@ -2095,10 +2106,11 @@ static int setattr_chown(struct inode *inode, struct iattr *attr)
  * Returns: errno
  */
 
-static int gfs2_setattr(struct mnt_idmap *idmap,
+static int gfs2_setattr(const struct mnt_idmap *idmap,
 			struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder i_gh;
 	int error;
@@ -2107,7 +2119,7 @@ static int gfs2_setattr(struct mnt_idmap *idmap,
 	if (error)
 		return error;
 
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &i_gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, &i_gh);
 	if (error)
 		goto out;
 
@@ -2156,19 +2168,20 @@ static int gfs2_setattr(struct mnt_idmap *idmap,
  * Returns: errno
  */
 
-static int gfs2_getattr(struct mnt_idmap *idmap,
+static int gfs2_getattr(const struct mnt_idmap *idmap,
 			const struct path *path, struct kstat *stat,
 			u32 request_mask, unsigned int flags)
 {
 	struct inode *inode = d_inode(path->dentry);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder gh;
 	u32 gfsflags;
 	int error;
 
 	gfs2_holder_mark_uninitialized(&gh);
-	if (gfs2_glock_is_locked_by_me(ip->i_gl) == NULL) {
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &gh);
+	if (gfs2_glock_is_locked_by_me(gl) == NULL) {
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &gh);
 		if (error)
 			return error;
 	}
@@ -2204,14 +2217,14 @@ static bool fault_in_fiemap(struct fiemap_extent_info *fi)
 static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 		       u64 start, u64 len)
 {
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_holder gh;
 	int ret;
 
 	inode_lock_shared(inode);
 
 retry:
-	ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+	ret = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &gh);
 	if (ret)
 		goto out;
 
@@ -2234,12 +2247,12 @@ static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
 loff_t gfs2_seek_data(struct file *file, loff_t offset)
 {
 	struct inode *inode = file->f_mapping->host;
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_holder gh;
 	loff_t ret;
 
 	inode_lock_shared(inode);
-	ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+	ret = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &gh);
 	if (!ret)
 		ret = iomap_seek_data(inode, offset, &gfs2_iomap_ops);
 	gfs2_glock_dq_uninit(&gh);
@@ -2253,12 +2266,12 @@ loff_t gfs2_seek_data(struct file *file, loff_t offset)
 loff_t gfs2_seek_hole(struct file *file, loff_t offset)
 {
 	struct inode *inode = file->f_mapping->host;
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_holder gh;
 	loff_t ret;
 
 	inode_lock_shared(inode);
-	ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+	ret = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &gh);
 	if (!ret)
 		ret = iomap_seek_hole(inode, offset, &gfs2_iomap_ops);
 	gfs2_glock_dq_uninit(&gh);
@@ -2272,8 +2285,7 @@ loff_t gfs2_seek_hole(struct file *file, loff_t offset)
 static int gfs2_update_time(struct inode *inode, enum fs_update_time type,
 		unsigned int flags)
 {
-	struct gfs2_inode *ip = GFS2_I(inode);
-	struct gfs2_glock *gl = ip->i_gl;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_holder *gh;
 	int error;
 
diff --git a/fs/gfs2/inode.h b/fs/gfs2/inode.h
index 2fcd96dd136139..99196d3114e429 100644
--- a/fs/gfs2/inode.h
+++ b/fs/gfs2/inode.h
@@ -97,7 +97,7 @@ int gfs2_dinode_dealloc(struct gfs2_inode *ip);
 
 struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name,
 			   int is_root);
-int gfs2_permission(struct mnt_idmap *idmap,
+int gfs2_permission(const struct mnt_idmap *idmap,
 		    struct inode *inode, int mask);
 struct inode *gfs2_lookup_meta(struct inode *dip, const char *name);
 void gfs2_dinode_out(const struct gfs2_inode *ip, void *buf);
@@ -109,7 +109,7 @@ extern const struct file_operations gfs2_file_fops_nolock;
 extern const struct file_operations gfs2_dir_fops_nolock;
 
 int gfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int gfs2_fileattr_set(struct mnt_idmap *idmap,
+int gfs2_fileattr_set(const struct mnt_idmap *idmap,
 		      struct dentry *dentry, struct file_kattr *fa);
 void gfs2_set_inode_flags(struct inode *inode);
 
diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c
index cef4bdee92a827..47eec0c13025d5 100644
--- a/fs/gfs2/lock_dlm.c
+++ b/fs/gfs2/lock_dlm.c
@@ -431,7 +431,7 @@ static void gdlm_cancel(struct gfs2_glock *gl)
  * 10. gfs2_control sets control_lock lvb = new gen + bits for failed jids
  * 12. gfs2_recover does journal recoveries for failed jids identified above
  * 14. gfs2_control clears control_lock lvb bits for recovered jids
- * 15. gfs2_control checks if recover_block == recover_start (step 3 occured
+ * 15. gfs2_control checks if recover_block == recover_start (step 3 occurred
  *     again) then do nothing, otherwise if recover_start > recover_block
  *     then clear BLOCK_LOCKS.
  *
@@ -809,7 +809,7 @@ static void gfs2_control_func(struct work_struct *work)
 
 	/*
 	 * No more jid bits set in lvb, all recovery is done, unblock locks
-	 * (unless a new recover_prep callback has occured blocking locks
+	 * (unless a new recover_prep callback has occurred blocking locks
 	 * again while working above)
 	 */
 
diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
index 78bba8cc10b8fd..b55daf1e6381c7 100644
--- a/fs/gfs2/log.c
+++ b/fs/gfs2/log.c
@@ -107,7 +107,7 @@ __acquires(&sdp->sd_ail_lock)
 		gfs2_assert(sdp, bd->bd_tr == tr);
 
 		if (!buffer_busy(bh)) {
-			if (buffer_uptodate(bh)) {
+			if (!buffer_write_io_error(bh)) {
 				list_move(&bd->bd_ail_st_list,
 					  &tr->tr_ail2_list);
 				continue;
@@ -321,7 +321,7 @@ static int gfs2_ail1_empty_one(struct gfs2_sbd *sdp, struct gfs2_trans *tr,
 			active_count++;
 			continue;
 		}
-		if (!buffer_uptodate(bh) &&
+		if (buffer_write_io_error(bh) &&
 		    !cmpxchg(&sdp->sd_log_error, 0, -EIO))
 			gfs2_io_error_bh(sdp, bh);
 		/*
@@ -1038,7 +1038,6 @@ void gfs2_remove_from_journal(struct buffer_head *bh, int meta)
 			set_bit(TR_TOUCHED, &tr->tr_flags);
 		}
 		was_pinned = 1;
-		brelse(bh);
 	}
 	if (bd) {
 		if (bd->bd_tr) {
@@ -1056,6 +1055,8 @@ void gfs2_remove_from_journal(struct buffer_head *bh, int meta)
 	}
 	clear_buffer_dirty(bh);
 	clear_buffer_uptodate(bh);
+	if (was_pinned)
+		brelse(bh);
 }
 
 /**
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index 6dabe73ad790d9..88c84895ec6f02 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -48,7 +48,7 @@ void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh)
 	clear_buffer_dirty(bh);
 	if (test_set_buffer_pinned(bh))
 		gfs2_assert_withdraw(sdp, 0);
-	if (!buffer_uptodate(bh))
+	if (!buffer_uptodate(bh) || buffer_write_io_error(bh))
 		gfs2_io_error_bh(sdp, bh);
 	bd = bh->b_private;
 	/* If this buffer is in the AIL and it has already been written
@@ -179,6 +179,8 @@ static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct folio *folio,
 	do {
 		if (error)
 			mark_buffer_write_io_error(bh);
+		else
+			clear_buffer_write_io_error(bh);
 		unlock_buffer(bh);
 		next = bh->b_this_page;
 		size -= bh->b_size;
@@ -775,9 +777,8 @@ static int buf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 				struct gfs2_log_descriptor *ld, __be64 *ptr,
 				int pass)
 {
-	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(jd->jd_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
-	struct gfs2_glock *gl = ip->i_gl;
 	unsigned int blks = be32_to_cpu(ld->ld_data1);
 	struct buffer_head *bh_log, *bh_ip;
 	u64 blkno;
@@ -828,17 +829,17 @@ static int buf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 
 static void buf_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass)
 {
-	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(jd->jd_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 
 	if (error) {
-		gfs2_inode_metasync(ip->i_gl);
+		gfs2_inode_metasync(gl);
 		return;
 	}
 	if (pass != 1)
 		return;
 
-	gfs2_inode_metasync(ip->i_gl);
+	gfs2_inode_metasync(gl);
 
 	fs_info(sdp, "jid=%u: Replayed %u of %u blocks\n",
 	        jd->jd_jid, jd->jd_replayed_blocks, jd->jd_found_blocks);
@@ -1000,8 +1001,7 @@ static int databuf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 				    struct gfs2_log_descriptor *ld,
 				    __be64 *ptr, int pass)
 {
-	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
-	struct gfs2_glock *gl = ip->i_gl;
+	struct gfs2_glock *gl = gfs2_inode_glock(jd->jd_inode);
 	unsigned int blks = be32_to_cpu(ld->ld_data1);
 	struct buffer_head *bh_log, *bh_ip;
 	u64 blkno;
@@ -1048,18 +1048,18 @@ static int databuf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 
 static void databuf_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass)
 {
-	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(jd->jd_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 
 	if (error) {
-		gfs2_inode_metasync(ip->i_gl);
+		gfs2_inode_metasync(gl);
 		return;
 	}
 	if (pass != 1)
 		return;
 
 	/* data sync? */
-	gfs2_inode_metasync(ip->i_gl);
+	gfs2_inode_metasync(gl);
 
 	fs_info(sdp, "jid=%u: Replayed %u of %u data blocks\n",
 		jd->jd_jid, jd->jd_replayed_blocks, jd->jd_found_blocks);
diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c
index a87cfbf0df3870..63591a4be10a01 100644
--- a/fs/gfs2/meta_io.c
+++ b/fs/gfs2/meta_io.c
@@ -402,18 +402,19 @@ static struct buffer_head *gfs2_getjdatabuf(struct gfs2_inode *ip, u64 blkno)
 
 void gfs2_journal_wipe(struct gfs2_inode *ip, u64 bstart, u32 blen)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct buffer_head *bh;
 	int ty;
 
 	/* This can only happen during incomplete inode creation. */
-	if (!ip->i_gl)
+	if (!gl)
 		return;
 
 	gfs2_ail1_wipe(sdp, bstart, blen);
 	while (blen) {
 		ty = REMOVE_META;
-		bh = gfs2_getbuf(ip->i_gl, bstart, NO_CREATE);
+		bh = gfs2_getbuf(gl, bstart, NO_CREATE);
 		if (!bh && gfs2_is_jdata(ip)) {
 			bh = gfs2_getjdatabuf(ip, bstart);
 			ty = REMOVE_JDATA;
@@ -447,8 +448,8 @@ void gfs2_journal_wipe(struct gfs2_inode *ip, u64 bstart, u32 blen)
 int gfs2_meta_buffer(struct gfs2_inode *ip, u32 mtype, u64 num,
 		     struct buffer_head **bhp)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
-	struct gfs2_glock *gl = ip->i_gl;
 	struct buffer_head *bh;
 	int ret = 0;
 	int rahead = 0;
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index 718e0da7dfce02..6ec47376aecf25 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -90,7 +90,6 @@ static struct gfs2_sbd *init_sbd(struct super_block *sb)
 	gfs2_tune_init(&sdp->sd_tune);
 
 	init_waitqueue_head(&sdp->sd_kill_wait);
-	init_waitqueue_head(&sdp->sd_async_glock_wait);
 	atomic_set(&sdp->sd_glock_disposal, 0);
 	init_completion(&sdp->sd_locking_init);
 	init_completion(&sdp->sd_withdraw_helper);
@@ -534,7 +533,7 @@ static void gfs2_others_may_mount(struct gfs2_sbd *sdp)
 
 static int gfs2_jindex_hold(struct gfs2_sbd *sdp, struct gfs2_holder *ji_gh)
 {
-	struct gfs2_inode *dip = GFS2_I(sdp->sd_jindex);
+	struct gfs2_glock *gl = gfs2_inode_glock(sdp->sd_jindex);
 	struct qstr name;
 	char buf[20];
 	struct gfs2_jdesc *jd;
@@ -545,7 +544,7 @@ static int gfs2_jindex_hold(struct gfs2_sbd *sdp, struct gfs2_holder *ji_gh)
 	mutex_lock(&sdp->sd_jindex_mutex);
 
 	for (;;) {
-		error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, ji_gh);
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, ji_gh);
 		if (error)
 			break;
 
@@ -611,7 +610,6 @@ static int init_statfs(struct gfs2_sbd *sdp)
 	struct inode *pn = NULL;
 	char buf[30];
 	struct gfs2_jdesc *jd;
-	struct gfs2_inode *ip;
 
 	sdp->sd_statfs_inode = gfs2_lookup_meta(master, "statfs");
 	if (IS_ERR(sdp->sd_statfs_inode)) {
@@ -656,15 +654,15 @@ static int init_statfs(struct gfs2_sbd *sdp)
 
 	iput(pn);
 	pn = NULL;
-	ip = GFS2_I(sdp->sd_sc_inode);
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID,
-				   &sdp->sd_sc_gh);
+	error = gfs2_glock_nq_init(gfs2_inode_glock(sdp->sd_sc_inode),
+				   LM_ST_EXCLUSIVE, GL_NOPID, &sdp->sd_sc_gh);
 	if (error) {
 		fs_err(sdp, "can't lock local \"sc\" file: %d\n", error);
 		goto free_local;
 	}
 	/* read in the local statfs buffer - other nodes don't change it. */
-	error = gfs2_meta_inode_buffer(ip, &sdp->sd_sc_bh);
+	error = gfs2_meta_inode_buffer(GFS2_I(sdp->sd_sc_inode),
+				       &sdp->sd_sc_bh);
 	if (error) {
 		fs_err(sdp, "Cannot read in local statfs: %d\n", error);
 		goto unlock_sd_gh;
@@ -697,7 +695,7 @@ static int init_journal(struct gfs2_sbd *sdp, int undo)
 {
 	struct inode *master = d_inode(sdp->sd_master_dir);
 	struct gfs2_holder ji_gh;
-	struct gfs2_inode *ip;
+	struct gfs2_glock *gl;
 	int error = 0;
 
 	gfs2_holder_mark_uninitialized(&ji_gh);
@@ -751,8 +749,8 @@ static int init_journal(struct gfs2_sbd *sdp, int undo)
 			goto fail_jindex;
 		}
 
-		ip = GFS2_I(sdp->sd_jdesc->jd_inode);
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED,
+		gl = gfs2_inode_glock(sdp->sd_jdesc->jd_inode);
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED,
 					   LM_FLAG_RECOVER | GL_EXACT |
 					   GL_NOCACHE | GL_NOPID,
 					   &sdp->sd_jinode_gh);
@@ -893,7 +891,7 @@ static int init_per_node(struct gfs2_sbd *sdp, int undo)
 	struct inode *pn = NULL;
 	char buf[30];
 	int error = 0;
-	struct gfs2_inode *ip;
+	struct gfs2_glock *gl;
 	struct inode *master = d_inode(sdp->sd_master_dir);
 
 	if (sdp->sd_args.ar_spectator)
@@ -920,8 +918,8 @@ static int init_per_node(struct gfs2_sbd *sdp, int undo)
 	iput(pn);
 	pn = NULL;
 
-	ip = GFS2_I(sdp->sd_qc_inode);
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID,
+	gl = gfs2_inode_glock(sdp->sd_qc_inode);
+	error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, GL_NOPID,
 				   &sdp->sd_qc_gh);
 	if (error) {
 		fs_err(sdp, "can't lock local \"qc\" file: %d\n", error);
diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c
index 001c8b39ca5573..b2984fb7b35c3b 100644
--- a/fs/gfs2/quota.c
+++ b/fs/gfs2/quota.c
@@ -408,7 +408,7 @@ static int bh_get(struct gfs2_quota_data *qd)
 {
 	struct gfs2_sbd *sdp = qd->qd_sbd;
 	struct inode *inode = sdp->sd_qc_inode;
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	unsigned int block, offset;
 	struct buffer_head *bh = NULL;
 	struct iomap iomap = { };
@@ -434,7 +434,7 @@ static int bh_get(struct gfs2_quota_data *qd)
 	if (iomap.type != IOMAP_MAPPED)
 		return error;
 
-	error = gfs2_meta_read(ip->i_gl, iomap.addr >> inode->i_blkbits,
+	error = gfs2_meta_read(gl, iomap.addr >> inode->i_blkbits,
 			       DIO_WAIT, 0, &bh);
 	if (error)
 		return error;
@@ -562,6 +562,8 @@ int gfs2_qa_get(struct gfs2_inode *ip)
 
 	if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF)
 		return 0;
+	if (ip->i_diskflags & GFS2_DIF_SYSTEM)
+		return 0;
 
 	spin_lock(&inode->i_lock);
 	if (ip->i_qadata == NULL) {
@@ -605,7 +607,7 @@ int gfs2_quota_hold(struct gfs2_inode *ip, kuid_t uid, kgid_t gid)
 		return 0;
 
 	error = gfs2_qa_get(ip);
-	if (error)
+	if (error || !ip->i_qadata)
 		return error;
 
 	qd = ip->i_qadata->qa_qd;
@@ -687,12 +689,12 @@ static int sort_qd(const void *a, const void *b)
 static void do_qc(struct gfs2_quota_data *qd, s64 change)
 {
 	struct gfs2_sbd *sdp = qd->qd_sbd;
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_qc_inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(sdp->sd_qc_inode);
 	struct gfs2_quota_change *qc = qd->qd_bh_qc;
 	bool needs_put = false;
 	s64 x;
 
-	gfs2_trans_add_meta(ip->i_gl, qd->qd_bh);
+	gfs2_trans_add_meta(gl, qd->qd_bh);
 
 	/*
 	 * The QDF_CHANGE flag indicates that the slot in the quota change file
@@ -740,8 +742,8 @@ static void do_qc(struct gfs2_quota_data *qd, s64 change)
 static int gfs2_write_buf_to_page(struct gfs2_sbd *sdp, unsigned long index,
 				  unsigned off, void *buf, unsigned bytes)
 {
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode);
-	struct inode *inode = &ip->i_inode;
+	struct inode *inode = sdp->sd_quota_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct address_space *mapping = inode->i_mapping;
 	struct folio *folio;
 	struct buffer_head *bh;
@@ -780,7 +782,7 @@ static int gfs2_write_buf_to_page(struct gfs2_sbd *sdp, unsigned long index,
 			set_buffer_uptodate(bh);
 		if (bh_read(bh, REQ_META | REQ_PRIO) < 0)
 			goto unlock_out;
-		gfs2_trans_add_data(ip->i_gl, bh);
+		gfs2_trans_add_data(gl, bh);
 
 		/* If we need to write to the next block as well */
 		if (to_write > (bsize - boff)) {
@@ -908,7 +910,9 @@ static int do_sync(unsigned int num_qd, struct gfs2_quota_data **qda,
 		   u64 sync_gen)
 {
 	struct gfs2_sbd *sdp = (*qda)->qd_sbd;
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode);
+	struct inode *inode = sdp->sd_quota_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
+	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_alloc_parms ap = {};
 	unsigned int data_blocks, ind_blocks;
 	struct gfs2_holder *ghs, i_gh;
@@ -927,7 +931,7 @@ static int do_sync(unsigned int num_qd, struct gfs2_quota_data **qda,
 		return -ENOMEM;
 
 	sort(qda, num_qd, sizeof(struct gfs2_quota_data *), sort_qd, NULL);
-	inode_lock(&ip->i_inode);
+	inode_lock(inode);
 	for (qx = 0; qx < num_qd; qx++) {
 		error = gfs2_glock_nq_init(qda[qx]->qd_gl, LM_ST_EXCLUSIVE,
 					   GL_NOCACHE, &ghs[qx]);
@@ -935,7 +939,7 @@ static int do_sync(unsigned int num_qd, struct gfs2_quota_data **qda,
 			goto out_dq;
 	}
 
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &i_gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, &i_gh);
 	if (error)
 		goto out_dq;
 
@@ -991,10 +995,9 @@ static int do_sync(unsigned int num_qd, struct gfs2_quota_data **qda,
 out_dq:
 	while (qx--)
 		gfs2_glock_dq_uninit(&ghs[qx]);
-	inode_unlock(&ip->i_inode);
+	inode_unlock(inode);
 	kfree(ghs);
-	gfs2_log_flush(glock_sbd(ip->i_gl), ip->i_gl,
-		       GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_DO_SYNC);
+	gfs2_log_flush(sdp, gl, GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_DO_SYNC);
 	if (!error) {
 		for (x = 0; x < num_qd; x++) {
 			qd = qda[x];
@@ -1038,7 +1041,7 @@ static int do_glock(struct gfs2_quota_data *qd, int force_refresh,
 		    struct gfs2_holder *q_gh)
 {
 	struct gfs2_sbd *sdp = qd->qd_sbd;
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(sdp->sd_quota_inode);
 	struct gfs2_holder i_gh;
 	int error;
 
@@ -1062,7 +1065,7 @@ static int do_glock(struct gfs2_quota_data *qd, int force_refresh,
 		if (error)
 			return error;
 
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &i_gh);
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &i_gh);
 		if (error)
 			goto fail;
 
@@ -1098,6 +1101,9 @@ int gfs2_quota_lock(struct gfs2_inode *ip, kuid_t uid, kgid_t gid)
 	error = gfs2_quota_hold(ip, uid, gid);
 	if (error)
 		return error;
+	/* Meta inodes never get quota data (see gfs2_qa_get()); nothing to lock. */
+	if (!ip->i_qadata)
+		return 0;
 
 	sort(ip->i_qadata->qa_qd, ip->i_qadata->qa_qd_num,
 	     sizeof(struct gfs2_quota_data *), sort_qd, NULL);
@@ -1402,7 +1408,8 @@ int gfs2_quota_refresh(struct gfs2_sbd *sdp, struct kqid qid)
 
 int gfs2_quota_init(struct gfs2_sbd *sdp)
 {
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_qc_inode);
+	struct inode *inode = sdp->sd_qc_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	u64 size = i_size_read(sdp->sd_qc_inode);
 	unsigned int blocks = size >> sdp->sd_sb.sb_bsize_shift;
 	unsigned int x, slot = 0;
@@ -1434,12 +1441,12 @@ int gfs2_quota_init(struct gfs2_sbd *sdp)
 
 		if (!extlen) {
 			extlen = 32;
-			error = gfs2_get_extent(&ip->i_inode, x, &dblock, &extlen);
+			error = gfs2_get_extent(inode, x, &dblock, &extlen);
 			if (error)
 				goto fail;
 		}
 		error = -EIO;
-		bh = gfs2_meta_ra(ip->i_gl, dblock, extlen);
+		bh = gfs2_meta_ra(gl, dblock, extlen);
 		if (!bh)
 			goto fail;
 		if (gfs2_metatype_check(sdp, bh, GFS2_METATYPE_QC))
@@ -1473,7 +1480,7 @@ int gfs2_quota_init(struct gfs2_sbd *sdp)
 			spin_lock_bucket(hash);
 			old_qd = gfs2_qd_search_bucket_noref(hash, sdp, qc_id);
 			if (old_qd) {
-				fs_err(sdp, "Corruption found in quota_change%u"
+				fs_err(sdp, "Corruption found in quota_change%u "
 					    "file: duplicate identifier in "
 					    "slot %u\n",
 					    sdp->sd_jdesc->jd_jid, slot);
@@ -1714,7 +1721,9 @@ static int gfs2_set_dqblk(struct super_block *sb, struct kqid qid,
 			  struct qc_dqblk *fdq)
 {
 	struct gfs2_sbd *sdp = sb->s_fs_info;
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode);
+	struct inode *inode = sdp->sd_quota_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
+	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_quota_data *qd;
 	struct gfs2_holder q_gh, i_gh;
 	unsigned int data_blocks, ind_blocks;
@@ -1741,11 +1750,11 @@ static int gfs2_set_dqblk(struct super_block *sb, struct kqid qid,
 	if (error)
 		goto out_put;
 
-	inode_lock(&ip->i_inode);
+	inode_lock(inode);
 	error = gfs2_glock_nq_init(qd->qd_gl, LM_ST_EXCLUSIVE, 0, &q_gh);
 	if (error)
 		goto out_unlockput;
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &i_gh);
+	error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, &i_gh);
 	if (error)
 		goto out_q;
 
@@ -1807,7 +1816,7 @@ static int gfs2_set_dqblk(struct super_block *sb, struct kqid qid,
 	gfs2_glock_dq_uninit(&q_gh);
 out_unlockput:
 	gfs2_qa_put(ip);
-	inode_unlock(&ip->i_inode);
+	inode_unlock(inode);
 out_put:
 	qd_put(qd);
 	return error;
diff --git a/fs/gfs2/recovery.c b/fs/gfs2/recovery.c
index 616c46aa3434ae..84db5bb7dc5c36 100644
--- a/fs/gfs2/recovery.c
+++ b/fs/gfs2/recovery.c
@@ -32,14 +32,15 @@ struct workqueue_struct *gfs2_recovery_wq;
 int gfs2_replay_read_block(struct gfs2_jdesc *jd, unsigned int blk,
 			   struct buffer_head **bh)
 {
-	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
-	struct gfs2_glock *gl = ip->i_gl;
+	struct inode *inode = jd->jd_inode;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
+	struct gfs2_inode *ip = GFS2_I(inode);
 	u64 dblock;
 	u32 extlen;
 	int error;
 
 	extlen = 32;
-	error = gfs2_get_extent(&ip->i_inode, blk, &dblock, &extlen);
+	error = gfs2_get_extent(inode, blk, &dblock, &extlen);
 	if (error)
 		return error;
 	if (!dblock) {
@@ -155,7 +156,7 @@ int __get_log_header(struct gfs2_sbd *sdp, const struct gfs2_log_header *lh,
  * @blk: the block to look at
  * @head: the log header to return
  *
- * Read the log header for a given segement in a given journal.  Do a few
+ * Read the log header for a given segment in a given journal.  Do a few
  * sanity checks on it.
  *
  * Returns: 0 on success,
@@ -306,15 +307,13 @@ static int update_statfs_inode(struct gfs2_jdesc *jd,
 			       struct inode *inode)
 {
 	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
-	struct gfs2_inode *ip;
 	struct buffer_head *bh;
 	struct gfs2_statfs_change_host sc;
 	int error = 0;
 
 	BUG_ON(!inode);
-	ip = GFS2_I(inode);
 
-	error = gfs2_meta_inode_buffer(ip, &bh);
+	error = gfs2_meta_inode_buffer(GFS2_I(inode), &bh);
 	if (error)
 		goto out;
 
@@ -345,7 +344,7 @@ static int update_statfs_inode(struct gfs2_jdesc *jd,
 
 	mark_buffer_dirty(bh);
 	brelse(bh);
-	gfs2_inode_metasync(ip->i_gl);
+	gfs2_inode_metasync(gfs2_inode_glock(inode));
 
 out:
 	return error;
@@ -398,7 +397,7 @@ static void recover_local_statfs(struct gfs2_jdesc *jd,
 void gfs2_recover_func(struct work_struct *work)
 {
 	struct gfs2_jdesc *jd = container_of(work, struct gfs2_jdesc, jd_work);
-	struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(jd->jd_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
 	struct gfs2_log_header_host head;
 	struct gfs2_holder j_gh, ji_gh;
@@ -440,7 +439,7 @@ void gfs2_recover_func(struct work_struct *work)
 			goto fail;
 		}
 
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED,
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED,
 					   LM_FLAG_RECOVER | GL_NOCACHE,
 					   &ji_gh);
 		if (error)
diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c
index 5988a165a8301d..3ac751eea05505 100644
--- a/fs/gfs2/rgrp.c
+++ b/fs/gfs2/rgrp.c
@@ -745,6 +745,7 @@ void gfs2_clear_rgrpd(struct gfs2_sbd *sdp)
 
 /**
  * compute_bitstructs - Compute the bitmap sizes
+ * @sb: The superblock
  * @rgd: The resource group descriptor
  *
  * Calculates bitmap descriptors, one for each block that contains bitmap data
@@ -752,84 +753,74 @@ void gfs2_clear_rgrpd(struct gfs2_sbd *sdp)
  * Returns: errno
  */
 
-static int compute_bitstructs(struct gfs2_rgrpd *rgd)
+static int compute_bitstructs(struct super_block *sb, struct gfs2_rgrpd *rgd)
 {
 	struct gfs2_sbd *sdp = rgd->rd_sbd;
 	struct gfs2_bitmap *bi;
-	u32 length = rgd->rd_length; /* # blocks in hdr & bitmap */
+	u32 expected_length;
 	u32 bytes_left, bytes;
+	u64 data_end;
 	int x;
 
-	if (!length)
-		return -EINVAL;
+	/*
+	 * The first resource group block has a gfs2_rgrp header; the remaining
+	 * blocks have a gfs2_meta_header header.  The rest of each block is
+	 * filled with bitmap data.
+	 */
+
+	if (rgd->rd_addr <= (GFS2_SB_ADDR >> sdp->sd_fsb2bb_shift)) {
+		gfs2_consist_rgrpd(rgd);
+		return -EIO;
+	}
+	if (check_add_overflow(rgd->rd_data0, rgd->rd_data, &data_end) ||
+	    rgd->rd_data == 0 || data_end > sb_bdev_nr_blocks(sb)) {
+		gfs2_consist_rgrpd(rgd);
+		return -EIO;
+	}
+	if (rgd->rd_bitbytes != DIV_ROUND_UP(rgd->rd_data, GFS2_NBBY)) {
+		gfs2_consist_rgrpd(rgd);
+		return -EIO;
+	}
+	expected_length = DIV_ROUND_UP(rgd->rd_bitbytes +
+		sizeof(struct gfs2_rgrp) - sizeof(struct gfs2_meta_header),
+		sdp->sd_sb.sb_bsize - sizeof(struct gfs2_meta_header));
+	if (rgd->rd_length != expected_length) {
+		gfs2_consist_rgrpd(rgd);
+		return -EIO;
+	}
+	if (rgd->rd_data0 < rgd->rd_addr + rgd->rd_length) {
+		gfs2_consist_rgrpd(rgd);
+		return -EIO;
+	}
 
-	rgd->rd_bits = kzalloc_objs(struct gfs2_bitmap, length, GFP_NOFS);
+	rgd->rd_bits = kzalloc_objs(struct gfs2_bitmap, rgd->rd_length, GFP_NOFS);
 	if (!rgd->rd_bits)
 		return -ENOMEM;
 
 	bytes_left = rgd->rd_bitbytes;
 
-	for (x = 0; x < length; x++) {
+	for (x = 0; x < rgd->rd_length; x++) {
 		bi = rgd->rd_bits + x;
 
 		bi->bi_flags = 0;
-		/* small rgrp; bitmap stored completely in header block */
-		if (length == 1) {
-			bytes = bytes_left;
-			bi->bi_offset = sizeof(struct gfs2_rgrp);
+		if (x == 0) {
+			/* header block */
 			bi->bi_start = 0;
-			bi->bi_bytes = bytes;
-			bi->bi_blocks = bytes * GFS2_NBBY;
-		/* header block */
-		} else if (x == 0) {
-			bytes = sdp->sd_sb.sb_bsize - sizeof(struct gfs2_rgrp);
 			bi->bi_offset = sizeof(struct gfs2_rgrp);
-			bi->bi_start = 0;
-			bi->bi_bytes = bytes;
-			bi->bi_blocks = bytes * GFS2_NBBY;
-		/* last block */
-		} else if (x + 1 == length) {
-			bytes = bytes_left;
-			bi->bi_offset = sizeof(struct gfs2_meta_header);
-			bi->bi_start = rgd->rd_bitbytes - bytes_left;
-			bi->bi_bytes = bytes;
-			bi->bi_blocks = bytes * GFS2_NBBY;
-		/* other blocks */
 		} else {
-			bytes = sdp->sd_sb.sb_bsize -
-				sizeof(struct gfs2_meta_header);
+			/* bitmap-only block */
+			struct gfs2_bitmap *prev = bi - 1;
+
+			bi->bi_start = prev->bi_start + prev->bi_bytes;
 			bi->bi_offset = sizeof(struct gfs2_meta_header);
-			bi->bi_start = rgd->rd_bitbytes - bytes_left;
-			bi->bi_bytes = bytes;
-			bi->bi_blocks = bytes * GFS2_NBBY;
 		}
-
+		bytes = sdp->sd_sb.sb_bsize - bi->bi_offset;
+		if (bytes > bytes_left)
+			bytes = bytes_left;
+		bi->bi_bytes = bytes;
+		bi->bi_blocks = bytes * GFS2_NBBY;
 		bytes_left -= bytes;
 	}
-
-	if (bytes_left) {
-		gfs2_consist_rgrpd(rgd);
-		return -EIO;
-	}
-	bi = rgd->rd_bits + (length - 1);
-	if ((bi->bi_start + bi->bi_bytes) * GFS2_NBBY != rgd->rd_data) {
-		gfs2_lm(sdp,
-			"ri_addr=%llu "
-			"ri_length=%u "
-			"ri_data0=%llu "
-			"ri_data=%u "
-			"ri_bitbytes=%u "
-			"start=%u len=%u offset=%u\n",
-			(unsigned long long)rgd->rd_addr,
-			rgd->rd_length,
-			(unsigned long long)rgd->rd_data0,
-			rgd->rd_data,
-			rgd->rd_bitbytes,
-			bi->bi_start, bi->bi_bytes, bi->bi_offset);
-		gfs2_consist_rgrpd(rgd);
-		return -EIO;
-	}
-
 	return 0;
 }
 
@@ -864,6 +855,7 @@ static int rgd_insert(struct gfs2_rgrpd *rgd)
 {
 	struct gfs2_sbd *sdp = rgd->rd_sbd;
 	struct rb_node **newn = &sdp->sd_rindex_tree.rb_node, *parent = NULL;
+	struct rb_node *prevn;
 
 	/* Figure out where to put new node */
 	while (*newn) {
@@ -882,6 +874,19 @@ static int rgd_insert(struct gfs2_rgrpd *rgd)
 	rb_link_node(&rgd->rd_node, parent, newn);
 	rb_insert_color(&rgd->rd_node, &sdp->sd_rindex_tree);
 	sdp->sd_rgrps++;
+
+	prevn = rb_prev(&rgd->rd_node);
+	if (prevn) {
+		struct gfs2_rgrpd *prev =
+			rb_entry(prevn, struct gfs2_rgrpd, rd_node);
+
+		if (prev->rd_data0 + prev->rd_data > rgd->rd_addr) {
+			fs_err(sdp, "overlapping resource groups.\n");
+			rb_erase(&rgd->rd_node, &sdp->sd_rindex_tree);
+			return -ENOENT;
+		}
+	}
+
 	return 0;
 }
 
@@ -928,7 +933,7 @@ static int read_rindex_entry(struct gfs2_inode *ip)
 	if (error)
 		goto fail;
 
-	error = compute_bitstructs(rgd);
+	error = compute_bitstructs(sdp->sd_vfs, rgd);
 	if (error)
 		goto fail_glock;
 
@@ -944,7 +949,9 @@ static int read_rindex_entry(struct gfs2_inode *ip)
 		return 0;
 	}
 
-	error = 0; /* someone else read in the rgrp; free it and ignore it */
+	/* If someone else read in the rgrp, free it and ignore it. */
+	if (error == -EEXIST)
+		error = 0;
 fail_glock:
 	gfs2_glock_put(rgd->rd_gl);
 
@@ -1033,8 +1040,8 @@ static int gfs2_ri_update(struct gfs2_inode *ip)
 
 int gfs2_rindex_update(struct gfs2_sbd *sdp)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(sdp->sd_rindex);
 	struct gfs2_inode *ip = GFS2_I(sdp->sd_rindex);
-	struct gfs2_glock *gl = ip->i_gl;
 	struct gfs2_holder ri_gh;
 	int error = 0;
 	int unlock_required = 0;
@@ -2453,7 +2460,7 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks,
 		if (error == 0) {
 			struct gfs2_dinode *di =
 				(struct gfs2_dinode *)dibh->b_data;
-			gfs2_trans_add_meta(ip->i_gl, dibh);
+			gfs2_trans_add_meta(gfs2_inode_glock(&ip->i_inode), dibh);
 			di->di_goal_meta = di->di_goal_data =
 				cpu_to_be64(ip->i_goal);
 			brelse(dibh);
diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c
index 06302c29340f2b..04bb4cf787d4db 100644
--- a/fs/gfs2/super.c
+++ b/fs/gfs2/super.c
@@ -132,8 +132,7 @@ int gfs2_jdesc_check(struct gfs2_jdesc *jd)
 
 int gfs2_make_fs_rw(struct gfs2_sbd *sdp)
 {
-	struct gfs2_inode *ip = GFS2_I(sdp->sd_jdesc->jd_inode);
-	struct gfs2_glock *j_gl = ip->i_gl;
+	struct gfs2_glock *j_gl = gfs2_inode_glock(sdp->sd_jdesc->jd_inode);
 	int error;
 
 	j_gl->gl_ops->go_inval(j_gl, DIO_METADATA);
@@ -176,6 +175,7 @@ void gfs2_statfs_change_out(const struct gfs2_statfs_change_host *sc, void *buf)
 
 int gfs2_statfs_init(struct gfs2_sbd *sdp)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(sdp->sd_statfs_inode);
 	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
 	struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
 	struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
@@ -183,7 +183,7 @@ int gfs2_statfs_init(struct gfs2_sbd *sdp)
 	struct gfs2_holder gh;
 	int error;
 
-	error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE,
+	error = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, GL_NOCACHE,
 				   &gh);
 	if (error)
 		return error;
@@ -216,13 +216,13 @@ int gfs2_statfs_init(struct gfs2_sbd *sdp)
 void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free,
 			s64 dinodes)
 {
-	struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(sdp->sd_sc_inode);
 	struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
 	struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
 	s64 x, y;
 	int need_sync = 0;
 
-	gfs2_trans_add_meta(l_ip->i_gl, sdp->sd_sc_bh);
+	gfs2_trans_add_meta(gl, sdp->sd_sc_bh);
 
 	spin_lock(&sdp->sd_statfs_spin);
 	l_sc->sc_total += total;
@@ -244,13 +244,13 @@ void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free,
 
 void update_statfs(struct gfs2_sbd *sdp, struct buffer_head *m_bh)
 {
-	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
-	struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode);
+	struct gfs2_glock *m_gl = gfs2_inode_glock(sdp->sd_statfs_inode);
+	struct gfs2_glock *l_gl = gfs2_inode_glock(sdp->sd_sc_inode);
 	struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
 	struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
 
-	gfs2_trans_add_meta(l_ip->i_gl, sdp->sd_sc_bh);
-	gfs2_trans_add_meta(m_ip->i_gl, m_bh);
+	gfs2_trans_add_meta(l_gl, sdp->sd_sc_bh);
+	gfs2_trans_add_meta(m_gl, m_bh);
 
 	spin_lock(&sdp->sd_statfs_spin);
 	m_sc->sc_total += l_sc->sc_total;
@@ -273,8 +273,8 @@ int gfs2_statfs_sync(struct super_block *sb, int type)
 	struct buffer_head *m_bh;
 	int error;
 
-	error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE,
-				   &gh);
+	error = gfs2_glock_nq_init(gfs2_inode_glock(&m_ip->i_inode),
+				   LM_ST_EXCLUSIVE, GL_NOCACHE, &gh);
 	if (error)
 		goto out;
 
@@ -323,7 +323,6 @@ struct lfcc {
 
 static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp)
 {
-	struct gfs2_inode *ip;
 	struct gfs2_jdesc *jd;
 	struct lfcc *lfcc;
 	LIST_HEAD(list);
@@ -336,13 +335,14 @@ static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp)
 	 */
 
 	list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) {
+		struct gfs2_glock *gl = gfs2_inode_glock(jd->jd_inode);
+
 		lfcc = kmalloc_obj(struct lfcc);
 		if (!lfcc) {
 			error = -ENOMEM;
 			goto out;
 		}
-		ip = GFS2_I(jd->jd_inode);
-		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &lfcc->gh);
+		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &lfcc->gh);
 		if (error) {
 			kfree(lfcc);
 			goto out;
@@ -438,15 +438,16 @@ void gfs2_dinode_out(const struct gfs2_inode *ip, void *buf)
 
 static int gfs2_write_inode(struct inode *inode, struct writeback_control *wbc)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
-	struct address_space *metamapping = gfs2_glock2aspace(ip->i_gl);
+	struct address_space *metamapping = gfs2_glock2aspace(gl);
 	struct backing_dev_info *bdi = inode_to_bdi(metamapping->host);
 	int ret = 0;
 	bool flush_all = (wbc->sync_mode == WB_SYNC_ALL || gfs2_is_jdata(ip));
 
 	if (flush_all)
-		gfs2_log_flush(GFS2_SB(inode), ip->i_gl,
+		gfs2_log_flush(GFS2_SB(inode), gl,
 			       GFS2_LOG_HEAD_FLUSH_NORMAL |
 			       GFS2_LFC_WRITE_INODE);
 	if (bdi_wb_dirty_exceeded(bdi))
@@ -481,6 +482,7 @@ static int gfs2_write_inode(struct inode *inode, struct writeback_control *wbc)
 
 static void gfs2_dirty_inode(struct inode *inode, int flags)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct buffer_head *bh;
@@ -490,20 +492,20 @@ static void gfs2_dirty_inode(struct inode *inode, int flags)
 	int ret;
 
 	/* This can only happen during incomplete inode creation. */
-	if (unlikely(!ip->i_gl))
+	if (unlikely(!gl))
 		return;
 
 	if (gfs2_withdrawn(sdp))
 		return;
-	if (!gfs2_glock_is_locked_by_me(ip->i_gl)) {
-		ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
+	if (!gfs2_glock_is_locked_by_me(gl)) {
+		ret = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, &gh);
 		if (ret) {
 			fs_err(sdp, "dirty_inode: glock %d\n", ret);
-			gfs2_dump_glock(NULL, ip->i_gl, true);
+			gfs2_dump_glock(NULL, gl, true);
 			return;
 		}
 		need_unlock = 1;
-	} else if (WARN_ON_ONCE(ip->i_gl->gl_state != LM_ST_EXCLUSIVE))
+	} else if (WARN_ON_ONCE(gl->gl_state != LM_ST_EXCLUSIVE))
 		return;
 
 	if (current->journal_info == NULL) {
@@ -517,7 +519,7 @@ static void gfs2_dirty_inode(struct inode *inode, int flags)
 
 	ret = gfs2_meta_inode_buffer(ip, &bh);
 	if (ret == 0) {
-		gfs2_trans_add_meta(ip->i_gl, bh);
+		gfs2_trans_add_meta(gl, bh);
 		gfs2_dinode_out(ip, bh->b_data);
 		brelse(bh);
 	}
@@ -1176,9 +1178,12 @@ static void gfs2_glock_put_eventually(struct gfs2_glock *gl)
 
 static enum evict_behavior gfs2_upgrade_iopen_glock(struct inode *inode)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
-	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct gfs2_holder *gh = &ip->i_iopen_gh;
+	struct wait_queue_head *holder_waitq, *glock_waitq;
+	struct wait_queue_entry holder_wait, glock_wait;
+	long ret = 5 * HZ;
 	int error;
 
 	gh->gh_flags |= GL_NOCACHE;
@@ -1209,13 +1214,31 @@ static enum evict_behavior gfs2_upgrade_iopen_glock(struct inode *inode)
 	if (error)
 		return EVICT_SHOULD_SKIP_DELETE;
 
-	wait_event_interruptible_timeout(sdp->sd_async_glock_wait,
-		!test_bit(HIF_WAIT, &gh->gh_iflags) ||
-		glock_needs_demote(ip->i_gl),
-		5 * HZ);
+	holder_waitq = bit_waitqueue(&gh->gh_iflags, HIF_WAIT);
+	glock_waitq = bit_waitqueue(&gl->gl_flags, GLF_DEMOTE);
+	init_wait(&holder_wait);
+	init_wait(&glock_wait);
+	for (;;) {
+		prepare_to_wait(holder_waitq, &holder_wait, TASK_INTERRUPTIBLE);
+		prepare_to_wait(glock_waitq, &glock_wait, TASK_INTERRUPTIBLE);
+		if (gfs2_glock_poll(gh) || glock_needs_demote(gl))
+			break;
+		if (signal_pending(current))
+			break;
+		ret = schedule_timeout(ret);
+		if (gfs2_glock_poll(gh) || glock_needs_demote(gl))
+			break;
+		if (!ret)
+			break;
+		if (signal_pending(current))
+			break;
+	}
+	finish_wait(holder_waitq, &holder_wait);
+	finish_wait(glock_waitq, &glock_wait);
+
 	if (!test_bit(HIF_HOLDER, &gh->gh_iflags)) {
 		gfs2_glock_dq(gh);
-		if (glock_needs_demote(ip->i_gl))
+		if (glock_needs_demote(gl))
 			return EVICT_SHOULD_SKIP_DELETE;
 		return EVICT_SHOULD_DEFER_DELETE;
 	}
@@ -1238,6 +1261,7 @@ static enum evict_behavior gfs2_upgrade_iopen_glock(struct inode *inode)
 static enum evict_behavior evict_should_delete(struct inode *inode,
 					       struct gfs2_holder *gh)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct super_block *sb = inode->i_sb;
 	struct gfs2_sbd *sdp = sb->s_fs_info;
@@ -1255,11 +1279,11 @@ static enum evict_behavior evict_should_delete(struct inode *inode,
 		return EVICT_SHOULD_DEFER_DELETE;
 
 	/* Must not read inode block until block type has been verified */
-	ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_SKIP, gh);
+	ret = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, GL_SKIP, gh);
 	if (unlikely(ret))
 		return EVICT_SHOULD_SKIP_DELETE;
 
-	if (gfs2_inode_already_deleted(ip->i_gl, ip->i_no_formal_ino))
+	if (gfs2_inode_already_deleted(gl, ip->i_no_formal_ino))
 		return EVICT_SHOULD_SKIP_DELETE;
 	ret = gfs2_check_blk_type(sdp, ip->i_no_addr, GFS2_BLKST_UNLINKED);
 	if (ret)
@@ -1288,8 +1312,8 @@ static enum evict_behavior evict_should_delete(struct inode *inode,
  */
 static int evict_unlinked_inode(struct inode *inode, struct gfs2_holder *gh)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
-	struct gfs2_glock *gl = ip->i_gl;
 	int ret;
 
 	/* The inode glock must be held exclusively and be instantiated. */
@@ -1390,8 +1414,7 @@ static int evict_linked_inode(struct inode *inode, struct gfs2_holder *gh)
 {
 	struct super_block *sb = inode->i_sb;
 	struct gfs2_sbd *sdp = sb->s_fs_info;
-	struct gfs2_inode *ip = GFS2_I(inode);
-	struct gfs2_glock *gl = ip->i_gl;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct address_space *metamapping = gfs2_glock2aspace(gl);
 	int ret;
 
@@ -1446,13 +1469,14 @@ static void gfs2_evict_inode(struct inode *inode)
 {
 	struct super_block *sb = inode->i_sb;
 	struct gfs2_sbd *sdp = sb->s_fs_info;
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder gh;
 	enum evict_behavior behavior;
 	int ret;
 
 	gfs2_holder_mark_uninitialized(&gh);
-	if (sb_rdonly(sb) || !ip->i_no_addr || !ip->i_gl)
+	if (sb_rdonly(sb) || !ip->i_no_addr || !gl)
 		goto out;
 
 	/*
@@ -1505,10 +1529,10 @@ static void gfs2_evict_inode(struct inode *inode)
 		gfs2_glock_dq_uninit(&ip->i_iopen_gh);
 		gfs2_glock_put_eventually(gl);
 	}
-	if (ip->i_gl) {
-		glock_clear_object(ip->i_gl, ip);
+	if (gl) {
+		glock_clear_object(gl, ip);
 		wait_on_bit_io(&ip->i_flags, GIF_GLOP_PENDING, TASK_UNINTERRUPTIBLE);
-		gfs2_glock_put_eventually(ip->i_gl);
+		gfs2_glock_put_eventually(gl);
 		rcu_assign_pointer(ip->i_gl, NULL);
 	}
 }
diff --git a/fs/gfs2/trace_gfs2.h b/fs/gfs2/trace_gfs2.h
index bc40320ef239b1..0c60d755ba7627 100644
--- a/fs/gfs2/trace_gfs2.h
+++ b/fs/gfs2/trace_gfs2.h
@@ -457,7 +457,7 @@ TRACE_EVENT(gfs2_bmap,
 	),
 
 	TP_fast_assign(
-		__entry->dev            = glock_sbd(ip->i_gl)->sd_vfs->s_dev;
+		__entry->dev            = ip->i_inode.i_sb->s_dev;
 		__entry->lblock		= lblock;
 		__entry->pblock		= buffer_mapped(bh) ?  bh->b_blocknr : 0;
 		__entry->inum		= ip->i_no_addr;
@@ -493,7 +493,7 @@ TRACE_EVENT(gfs2_iomap_start,
 	),
 
 	TP_fast_assign(
-		__entry->dev            = glock_sbd(ip->i_gl)->sd_vfs->s_dev;
+		__entry->dev            = ip->i_inode.i_sb->s_dev;
 		__entry->inum		= ip->i_no_addr;
 		__entry->pos		= pos;
 		__entry->length		= length;
@@ -525,7 +525,7 @@ TRACE_EVENT(gfs2_iomap_end,
 	),
 
 	TP_fast_assign(
-		__entry->dev            = glock_sbd(ip->i_gl)->sd_vfs->s_dev;
+		__entry->dev            = ip->i_inode.i_sb->s_dev;
 		__entry->inum		= ip->i_no_addr;
 		__entry->offset		= iomap->offset;
 		__entry->length		= iomap->length;
diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c
index 83b8bb6446e57b..61b0668ecabc08 100644
--- a/fs/gfs2/util.c
+++ b/fs/gfs2/util.c
@@ -55,10 +55,9 @@ int check_journal_clean(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 	int error;
 	struct gfs2_holder j_gh;
 	struct gfs2_log_header_host head;
-	struct gfs2_inode *ip;
+	struct gfs2_glock *gl = gfs2_inode_glock(jd->jd_inode);
 
-	ip = GFS2_I(jd->jd_inode);
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_RECOVER |
+	error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_RECOVER |
 				   GL_EXACT | GL_NOCACHE, &j_gh);
 	if (error) {
 		if (verbose)
@@ -333,6 +332,7 @@ void gfs2_consist_i(struct gfs2_sbd *sdp, const char *function,
 void gfs2_consist_inode_i(struct gfs2_inode *ip,
 			  const char *function, char *file, unsigned int line)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 
 	gfs2_lm(sdp,
@@ -342,7 +342,7 @@ void gfs2_consist_inode_i(struct gfs2_inode *ip,
 		(unsigned long long)ip->i_no_formal_ino,
 		(unsigned long long)ip->i_no_addr,
 		function, file, line);
-	gfs2_dump_glock(NULL, ip->i_gl, 1);
+	gfs2_dump_glock(NULL, gl, 1);
 	gfs2_withdraw(sdp);
 }
 
diff --git a/fs/gfs2/xattr.c b/fs/gfs2/xattr.c
index b9f48d6f10a97a..c26f180419d952 100644
--- a/fs/gfs2/xattr.c
+++ b/fs/gfs2/xattr.c
@@ -128,11 +128,12 @@ static int ea_foreach_i(struct gfs2_inode *ip, struct buffer_head *bh,
 
 static int ea_foreach(struct gfs2_inode *ip, ea_call_t ea_call, void *data)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct buffer_head *bh, *eabh;
 	__be64 *eablk, *end;
 	int error;
 
-	error = gfs2_meta_read(ip->i_gl, ip->i_eattr, DIO_WAIT, 0, &bh);
+	error = gfs2_meta_read(gl, ip->i_eattr, DIO_WAIT, 0, &bh);
 	if (error)
 		return error;
 
@@ -156,7 +157,7 @@ static int ea_foreach(struct gfs2_inode *ip, ea_call_t ea_call, void *data)
 			break;
 		bn = be64_to_cpu(*eablk);
 
-		error = gfs2_meta_read(ip->i_gl, bn, DIO_WAIT, 0, &eabh);
+		error = gfs2_meta_read(gl, bn, DIO_WAIT, 0, &eabh);
 		if (error)
 			break;
 		error = ea_foreach_i(ip, eabh, ea_call, data);
@@ -279,7 +280,7 @@ static int ea_dealloc_unstuffed(struct gfs2_inode *ip, struct buffer_head *bh,
 	if (error)
 		goto out_gunlock;
 
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	gfs2_trans_add_meta(gfs2_inode_glock(&ip->i_inode), bh);
 
 	dataptrs = GFS2_EA2DATAPTRS(ea);
 	for (x = 0; x < ea->ea_num_ptrs; x++, dataptrs++) {
@@ -426,7 +427,8 @@ ssize_t gfs2_listxattr(struct dentry *dentry, char *buffer, size_t size)
 		er.er_data_len = size;
 	}
 
-	error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh);
+	error = gfs2_glock_nq_init(gfs2_inode_glock(&ip->i_inode), LM_ST_SHARED,
+				   LM_FLAG_ANY, &i_gh);
 	if (error)
 		return error;
 
@@ -457,6 +459,7 @@ ssize_t gfs2_listxattr(struct dentry *dentry, char *buffer, size_t size)
 static int gfs2_iter_unstuffed(struct gfs2_inode *ip, struct gfs2_ea_header *ea,
 			       const char *din, char *dout)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct buffer_head **bh;
 	unsigned int amount = GFS2_EA_DATA_LEN(ea);
@@ -472,7 +475,7 @@ static int gfs2_iter_unstuffed(struct gfs2_inode *ip, struct gfs2_ea_header *ea,
 		return -ENOMEM;
 
 	for (x = 0; x < nptrs; x++) {
-		error = gfs2_meta_read(ip->i_gl, be64_to_cpu(*dataptrs), 0, 0,
+		error = gfs2_meta_read(gl, be64_to_cpu(*dataptrs), 0, 0,
 				       bh + x);
 		if (error) {
 			while (x--)
@@ -505,7 +508,7 @@ static int gfs2_iter_unstuffed(struct gfs2_inode *ip, struct gfs2_ea_header *ea,
 		}
 
 		if (din) {
-			gfs2_trans_add_meta(ip->i_gl, bh[x]);
+			gfs2_trans_add_meta(gl, bh[x]);
 			memcpy(pos, din, cp_size);
 			din += sdp->sd_jbsize;
 		}
@@ -608,14 +611,14 @@ static int gfs2_xattr_get(const struct xattr_handler *handler,
 			  struct dentry *unused, struct inode *inode,
 			  const char *name, void *buffer, size_t size)
 {
-	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_holder gh;
 	int ret;
 
 	/* During lookup, SELinux calls this function with the glock locked. */
 
-	if (!gfs2_glock_is_locked_by_me(ip->i_gl)) {
-		ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &gh);
+	if (!gfs2_glock_is_locked_by_me(gl)) {
+		ret = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &gh);
 		if (ret)
 			return ret;
 	} else {
@@ -637,6 +640,7 @@ static int gfs2_xattr_get(const struct xattr_handler *handler,
 
 static int ea_alloc_blk(struct gfs2_inode *ip, struct buffer_head **bhp)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct gfs2_ea_header *ea;
 	unsigned int n = 1;
@@ -647,8 +651,8 @@ static int ea_alloc_blk(struct gfs2_inode *ip, struct buffer_head **bhp)
 	if (error)
 		return error;
 	gfs2_trans_remove_revoke(sdp, block, 1);
-	*bhp = gfs2_meta_new(ip->i_gl, block);
-	gfs2_trans_add_meta(ip->i_gl, *bhp);
+	*bhp = gfs2_meta_new(gl, block);
+	gfs2_trans_add_meta(gl, *bhp);
 	gfs2_metatype_set(*bhp, GFS2_METATYPE_EA, GFS2_FORMAT_EA);
 	gfs2_buffer_clear_tail(*bhp, sizeof(struct gfs2_meta_header));
 
@@ -678,6 +682,7 @@ static int ea_alloc_blk(struct gfs2_inode *ip, struct buffer_head **bhp)
 static int ea_write(struct gfs2_inode *ip, struct gfs2_ea_header *ea,
 		    struct gfs2_ea_request *er)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	int error;
 
@@ -709,8 +714,8 @@ static int ea_write(struct gfs2_inode *ip, struct gfs2_ea_header *ea,
 			if (error)
 				return error;
 			gfs2_trans_remove_revoke(sdp, block, 1);
-			bh = gfs2_meta_new(ip->i_gl, block);
-			gfs2_trans_add_meta(ip->i_gl, bh);
+			bh = gfs2_meta_new(gl, block);
+			gfs2_trans_add_meta(gl, bh);
 			gfs2_metatype_set(bh, GFS2_METATYPE_ED, GFS2_FORMAT_ED);
 
 			gfs2_add_inode_blocks(&ip->i_inode, 1);
@@ -841,11 +846,12 @@ static struct gfs2_ea_header *ea_split_ea(struct gfs2_ea_header *ea)
 static void ea_set_remove_stuffed(struct gfs2_inode *ip,
 				  struct gfs2_ea_location *el)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_ea_header *ea = el->el_ea;
 	struct gfs2_ea_header *prev = el->el_prev;
 	u32 len;
 
-	gfs2_trans_add_meta(ip->i_gl, el->el_bh);
+	gfs2_trans_add_meta(gl, el->el_bh);
 
 	if (!prev || !GFS2_EA_IS_STUFFED(ea)) {
 		ea->ea_type = GFS2_EATYPE_UNUSED;
@@ -875,6 +881,7 @@ struct ea_set {
 static int ea_set_simple_noalloc(struct gfs2_inode *ip, struct buffer_head *bh,
 				 struct gfs2_ea_header *ea, struct ea_set *es)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_ea_request *er = es->es_er;
 	int error;
 
@@ -882,7 +889,7 @@ static int ea_set_simple_noalloc(struct gfs2_inode *ip, struct buffer_head *bh,
 	if (error)
 		return error;
 
-	gfs2_trans_add_meta(ip->i_gl, bh);
+	gfs2_trans_add_meta(gl, bh);
 
 	if (es->ea_split)
 		ea = ea_split_ea(ea);
@@ -902,11 +909,12 @@ static int ea_set_simple_noalloc(struct gfs2_inode *ip, struct buffer_head *bh,
 static int ea_set_simple_alloc(struct gfs2_inode *ip,
 			       struct gfs2_ea_request *er, void *private)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct ea_set *es = private;
 	struct gfs2_ea_header *ea = es->es_ea;
 	int error;
 
-	gfs2_trans_add_meta(ip->i_gl, es->es_bh);
+	gfs2_trans_add_meta(gl, es->es_bh);
 
 	if (es->ea_split)
 		ea = ea_split_ea(ea);
@@ -971,6 +979,7 @@ static int ea_set_simple(struct gfs2_inode *ip, struct buffer_head *bh,
 static int ea_set_block(struct gfs2_inode *ip, struct gfs2_ea_request *er,
 			void *private)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct buffer_head *indbh, *newbh;
 	__be64 *eablk;
@@ -980,7 +989,7 @@ static int ea_set_block(struct gfs2_inode *ip, struct gfs2_ea_request *er,
 	if (ip->i_diskflags & GFS2_DIF_EA_INDIRECT) {
 		__be64 *end;
 
-		error = gfs2_meta_read(ip->i_gl, ip->i_eattr, DIO_WAIT, 0,
+		error = gfs2_meta_read(gl, ip->i_eattr, DIO_WAIT, 0,
 				       &indbh);
 		if (error)
 			return error;
@@ -1002,7 +1011,7 @@ static int ea_set_block(struct gfs2_inode *ip, struct gfs2_ea_request *er,
 			goto out;
 		}
 
-		gfs2_trans_add_meta(ip->i_gl, indbh);
+		gfs2_trans_add_meta(gl, indbh);
 	} else {
 		u64 blk;
 		unsigned int n = 1;
@@ -1010,8 +1019,8 @@ static int ea_set_block(struct gfs2_inode *ip, struct gfs2_ea_request *er,
 		if (error)
 			return error;
 		gfs2_trans_remove_revoke(sdp, blk, 1);
-		indbh = gfs2_meta_new(ip->i_gl, blk);
-		gfs2_trans_add_meta(ip->i_gl, indbh);
+		indbh = gfs2_meta_new(gl, blk);
+		gfs2_trans_add_meta(gl, indbh);
 		gfs2_metatype_set(indbh, GFS2_METATYPE_IN, GFS2_FORMAT_IN);
 		gfs2_buffer_clear_tail(indbh, mh_size);
 
@@ -1088,6 +1097,7 @@ static int ea_set_remove_unstuffed(struct gfs2_inode *ip,
 
 static int ea_remove_stuffed(struct gfs2_inode *ip, struct gfs2_ea_location *el)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_ea_header *ea = el->el_ea;
 	struct gfs2_ea_header *prev = el->el_prev;
 	int error;
@@ -1096,7 +1106,7 @@ static int ea_remove_stuffed(struct gfs2_inode *ip, struct gfs2_ea_location *el)
 	if (error)
 		return error;
 
-	gfs2_trans_add_meta(ip->i_gl, el->el_bh);
+	gfs2_trans_add_meta(gl, el->el_bh);
 
 	if (prev) {
 		u32 len;
@@ -1229,11 +1239,12 @@ int __gfs2_xattr_set(struct inode *inode, const char *name,
 }
 
 static int gfs2_xattr_set(const struct xattr_handler *handler,
-			  struct mnt_idmap *idmap,
+			  const struct mnt_idmap *idmap,
 			  struct dentry *unused, struct inode *inode,
 			  const char *name, const void *value,
 			  size_t size, int flags)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(inode);
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_holder gh;
 	int ret;
@@ -1244,12 +1255,12 @@ static int gfs2_xattr_set(const struct xattr_handler *handler,
 
 	/* May be called from gfs_setattr with the glock locked. */
 
-	if (!gfs2_glock_is_locked_by_me(ip->i_gl)) {
-		ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
+	if (!gfs2_glock_is_locked_by_me(gl)) {
+		ret = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, &gh);
 		if (ret)
 			goto out;
 	} else {
-		if (WARN_ON_ONCE(ip->i_gl->gl_state != LM_ST_EXCLUSIVE)) {
+		if (WARN_ON_ONCE(gl->gl_state != LM_ST_EXCLUSIVE)) {
 			ret = -EIO;
 			goto out;
 		}
@@ -1265,6 +1276,7 @@ static int gfs2_xattr_set(const struct xattr_handler *handler,
 
 static int ea_dealloc_indirect(struct gfs2_inode *ip)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct gfs2_rgrp_list rlist;
 	struct gfs2_rgrpd *rgd;
@@ -1283,7 +1295,7 @@ static int ea_dealloc_indirect(struct gfs2_inode *ip)
 
 	memset(&rlist, 0, sizeof(struct gfs2_rgrp_list));
 
-	error = gfs2_meta_read(ip->i_gl, ip->i_eattr, DIO_WAIT, 0, &indbh);
+	error = gfs2_meta_read(gl, ip->i_eattr, DIO_WAIT, 0, &indbh);
 	if (error)
 		return error;
 
@@ -1333,7 +1345,7 @@ static int ea_dealloc_indirect(struct gfs2_inode *ip)
 	if (error)
 		goto out_gunlock;
 
-	gfs2_trans_add_meta(ip->i_gl, indbh);
+	gfs2_trans_add_meta(gl, indbh);
 
 	eablk = (__be64 *)(indbh->b_data + sizeof(struct gfs2_meta_header));
 	bstart = 0;
@@ -1367,7 +1379,7 @@ static int ea_dealloc_indirect(struct gfs2_inode *ip)
 
 	error = gfs2_meta_inode_buffer(ip, &dibh);
 	if (!error) {
-		gfs2_trans_add_meta(ip->i_gl, dibh);
+		gfs2_trans_add_meta(gl, dibh);
 		gfs2_dinode_out(ip, dibh->b_data);
 		brelse(dibh);
 	}
@@ -1385,6 +1397,7 @@ static int ea_dealloc_indirect(struct gfs2_inode *ip)
 
 static int ea_dealloc_block(struct gfs2_inode *ip, bool initialized)
 {
+	struct gfs2_glock *gl = gfs2_inode_glock(&ip->i_inode);
 	struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
 	struct gfs2_rgrpd *rgd;
 	struct buffer_head *dibh;
@@ -1419,7 +1432,7 @@ static int ea_dealloc_block(struct gfs2_inode *ip, bool initialized)
 	if (initialized) {
 		error = gfs2_meta_inode_buffer(ip, &dibh);
 		if (!error) {
-			gfs2_trans_add_meta(ip->i_gl, dibh);
+			gfs2_trans_add_meta(gl, dibh);
 			gfs2_dinode_out(ip, dibh->b_data);
 			brelse(dibh);
 		}
diff --git a/fs/hfs/attr.c b/fs/hfs/attr.c
index f8395cdd1adf87..6d737a085461ca 100644
--- a/fs/hfs/attr.c
+++ b/fs/hfs/attr.c
@@ -121,7 +121,7 @@ static int hfs_xattr_get(const struct xattr_handler *handler,
 }
 
 static int hfs_xattr_set(const struct xattr_handler *handler,
-			 struct mnt_idmap *idmap,
+			 const struct mnt_idmap *idmap,
 			 struct dentry *unused, struct inode *inode,
 			 const char *name, const void *value, size_t size,
 			 int flags)
diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c
index e1f1fb3514644d..f6b97da197887d 100644
--- a/fs/hfs/dir.c
+++ b/fs/hfs/dir.c
@@ -183,7 +183,7 @@ static int hfs_dir_release(struct inode *inode, struct file *file)
  * a directory and return a corresponding inode, given the inode for
  * the directory and the name (and its length) of the new file.
  */
-static int hfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int hfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -213,7 +213,7 @@ static int hfs_create(struct mnt_idmap *idmap, struct inode *dir,
  * in a directory, given the inode for the parent directory and the
  * name (and its length) of the new directory.
  */
-static struct dentry *hfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *hfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -281,7 +281,7 @@ static int hfs_remove(struct inode *dir, struct dentry *dentry)
  * new file/directory.
  * XXX: how do you handle must_be dir?
  */
-static int hfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int hfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h
index e250f87a5e3367..fdfa5d303d5efc 100644
--- a/fs/hfs/hfs_fs.h
+++ b/fs/hfs/hfs_fs.h
@@ -212,7 +212,7 @@ extern struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name,
 extern void hfs_inode_write_fork(struct inode *inode, struct hfs_extent *ext,
 				 __be32 *log_size, __be32 *phys_size);
 extern int hfs_write_inode(struct inode *inode, struct writeback_control *wbc);
-extern int hfs_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+extern int hfs_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			     struct iattr *attr);
 extern void hfs_inode_read_fork(struct inode *inode, struct hfs_extent *ext,
 				__be32 __log_size, __be32 phys_size,
diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c
index 2aef3c36a150a2..c81314f668ac74 100644
--- a/fs/hfs/inode.c
+++ b/fs/hfs/inode.c
@@ -643,7 +643,7 @@ static int hfs_file_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-int hfs_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int hfs_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c
index 51fcba2e6d4034..b3a1193a491f5a 100644
--- a/fs/hfsplus/dir.c
+++ b/fs/hfsplus/dir.c
@@ -460,7 +460,7 @@ static int hfsplus_rmdir(struct inode *dir, struct dentry *dentry)
 	return res;
 }
 
-static int hfsplus_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int hfsplus_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			   struct dentry *dentry, const char *symname)
 {
 	struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb);
@@ -511,7 +511,7 @@ static int hfsplus_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	return res;
 }
 
-static int hfsplus_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int hfsplus_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb);
@@ -561,19 +561,19 @@ static int hfsplus_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return res;
 }
 
-static int hfsplus_create(struct mnt_idmap *idmap, struct inode *dir,
+static int hfsplus_create(const struct mnt_idmap *idmap, struct inode *dir,
 			  struct dentry *dentry, umode_t mode)
 {
 	return hfsplus_mknod(&nop_mnt_idmap, dir, dentry, mode, 0);
 }
 
-static struct dentry *hfsplus_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *hfsplus_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				    struct dentry *dentry, umode_t mode)
 {
 	return ERR_PTR(hfsplus_mknod(&nop_mnt_idmap, dir, dentry, mode, 0));
 }
 
-static int hfsplus_rename(struct mnt_idmap *idmap,
+static int hfsplus_rename(const struct mnt_idmap *idmap,
 			  struct inode *old_dir, struct dentry *old_dentry,
 			  struct inode *new_dir, struct dentry *new_dentry,
 			  unsigned int flags)
diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h
index 1e5b58e6a13fc3..d55cb16e3897f1 100644
--- a/fs/hfsplus/hfsplus_fs.h
+++ b/fs/hfsplus/hfsplus_fs.h
@@ -459,13 +459,13 @@ void hfsplus_inode_write_fork(struct inode *inode,
 			      struct hfsplus_fork_raw *fork);
 int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd);
 int hfsplus_cat_write_inode(struct inode *inode);
-int hfsplus_getattr(struct mnt_idmap *idmap, const struct path *path,
+int hfsplus_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		    struct kstat *stat, u32 request_mask,
 		    unsigned int query_flags);
 int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end,
 		       int datasync);
 int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int hfsplus_fileattr_set(struct mnt_idmap *idmap,
+int hfsplus_fileattr_set(const struct mnt_idmap *idmap,
 			 struct dentry *dentry, struct file_kattr *fa);
 
 /* ioctl.c */
diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c
index 2ce6de574fa6ef..aed0866499b26d 100644
--- a/fs/hfsplus/inode.c
+++ b/fs/hfsplus/inode.c
@@ -305,7 +305,7 @@ static int hfsplus_file_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static int hfsplus_setattr(struct mnt_idmap *idmap,
+static int hfsplus_setattr(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -335,7 +335,7 @@ static int hfsplus_setattr(struct mnt_idmap *idmap,
 	return 0;
 }
 
-int hfsplus_getattr(struct mnt_idmap *idmap, const struct path *path,
+int hfsplus_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		    struct kstat *stat, u32 request_mask,
 		    unsigned int query_flags)
 {
@@ -797,7 +797,7 @@ int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int hfsplus_fileattr_set(struct mnt_idmap *idmap,
+int hfsplus_fileattr_set(const struct mnt_idmap *idmap,
 			 struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c
index 21a1c196c71f2e..71364e093fa704 100644
--- a/fs/hfsplus/xattr.c
+++ b/fs/hfsplus/xattr.c
@@ -1008,7 +1008,7 @@ static int hfsplus_osx_getxattr(const struct xattr_handler *handler,
 }
 
 static int hfsplus_osx_setxattr(const struct xattr_handler *handler,
-				struct mnt_idmap *idmap,
+				const struct mnt_idmap *idmap,
 				struct dentry *unused, struct inode *inode,
 				const char *name, const void *buffer,
 				size_t size, int flags)
diff --git a/fs/hfsplus/xattr_security.c b/fs/hfsplus/xattr_security.c
index 90f68ec119cddd..1969919c12cb99 100644
--- a/fs/hfsplus/xattr_security.c
+++ b/fs/hfsplus/xattr_security.c
@@ -23,7 +23,7 @@ static int hfsplus_security_getxattr(const struct xattr_handler *handler,
 }
 
 static int hfsplus_security_setxattr(const struct xattr_handler *handler,
-				     struct mnt_idmap *idmap,
+				     const struct mnt_idmap *idmap,
 				     struct dentry *unused, struct inode *inode,
 				     const char *name, const void *buffer,
 				     size_t size, int flags)
diff --git a/fs/hfsplus/xattr_trusted.c b/fs/hfsplus/xattr_trusted.c
index fdbaebc1c49a01..c140a95ab3f09d 100644
--- a/fs/hfsplus/xattr_trusted.c
+++ b/fs/hfsplus/xattr_trusted.c
@@ -22,7 +22,7 @@ static int hfsplus_trusted_getxattr(const struct xattr_handler *handler,
 }
 
 static int hfsplus_trusted_setxattr(const struct xattr_handler *handler,
-				    struct mnt_idmap *idmap,
+				    const struct mnt_idmap *idmap,
 				    struct dentry *unused, struct inode *inode,
 				    const char *name, const void *buffer,
 				    size_t size, int flags)
diff --git a/fs/hfsplus/xattr_user.c b/fs/hfsplus/xattr_user.c
index 6464b6c3d58d69..7e5da15f99372a 100644
--- a/fs/hfsplus/xattr_user.c
+++ b/fs/hfsplus/xattr_user.c
@@ -22,7 +22,7 @@ static int hfsplus_user_getxattr(const struct xattr_handler *handler,
 }
 
 static int hfsplus_user_setxattr(const struct xattr_handler *handler,
-				 struct mnt_idmap *idmap,
+				 const struct mnt_idmap *idmap,
 				 struct dentry *unused, struct inode *inode,
 				 const char *name, const void *buffer,
 				 size_t size, int flags)
diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c
index 7add056d47d8f0..613146e76dec24 100644
--- a/fs/hostfs/hostfs_kern.c
+++ b/fs/hostfs/hostfs_kern.c
@@ -592,7 +592,7 @@ static struct inode *hostfs_iget(struct super_block *sb, char *name)
 	return inode;
 }
 
-static int hostfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int hostfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -673,7 +673,7 @@ static int hostfs_unlink(struct inode *ino, struct dentry *dentry)
 	return err;
 }
 
-static int hostfs_symlink(struct mnt_idmap *idmap, struct inode *ino,
+static int hostfs_symlink(const struct mnt_idmap *idmap, struct inode *ino,
 			  struct dentry *dentry, const char *to)
 {
 	char *file;
@@ -686,7 +686,7 @@ static int hostfs_symlink(struct mnt_idmap *idmap, struct inode *ino,
 	return err;
 }
 
-static struct dentry *hostfs_mkdir(struct mnt_idmap *idmap, struct inode *ino,
+static struct dentry *hostfs_mkdir(const struct mnt_idmap *idmap, struct inode *ino,
 				   struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -719,7 +719,7 @@ static int hostfs_rmdir(struct inode *ino, struct dentry *dentry)
 	return err;
 }
 
-static int hostfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int hostfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode, dev_t dev)
 {
 	struct inode *inode;
@@ -745,7 +745,7 @@ static int hostfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return 0;
 }
 
-static int hostfs_rename2(struct mnt_idmap *idmap,
+static int hostfs_rename2(const struct mnt_idmap *idmap,
 			  struct inode *old_dir, struct dentry *old_dentry,
 			  struct inode *new_dir, struct dentry *new_dentry,
 			  unsigned int flags)
@@ -774,7 +774,7 @@ static int hostfs_rename2(struct mnt_idmap *idmap,
 	return err;
 }
 
-static int hostfs_permission(struct mnt_idmap *idmap,
+static int hostfs_permission(const struct mnt_idmap *idmap,
 			     struct inode *ino, int desired)
 {
 	char *name;
@@ -801,7 +801,7 @@ static int hostfs_permission(struct mnt_idmap *idmap,
 	return err;
 }
 
-static int hostfs_setattr(struct mnt_idmap *idmap,
+static int hostfs_setattr(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/hpfs/hpfs_fn.h b/fs/hpfs/hpfs_fn.h
index 237c1c23e8550b..a398dd8bdf3055 100644
--- a/fs/hpfs/hpfs_fn.h
+++ b/fs/hpfs/hpfs_fn.h
@@ -280,7 +280,7 @@ void hpfs_init_inode(struct inode *);
 void hpfs_read_inode(struct inode *);
 void hpfs_write_inode(struct inode *);
 void hpfs_write_inode_nolock(struct inode *);
-int hpfs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *);
+int hpfs_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *);
 void hpfs_write_if_changed(struct inode *);
 void hpfs_evict_inode(struct inode *);
 
diff --git a/fs/hpfs/inode.c b/fs/hpfs/inode.c
index 1b4fcf760aadcd..396773d0b669d8 100644
--- a/fs/hpfs/inode.c
+++ b/fs/hpfs/inode.c
@@ -257,7 +257,7 @@ void hpfs_write_inode_nolock(struct inode *i)
 	brelse(bh);
 }
 
-int hpfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int hpfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/hpfs/namei.c b/fs/hpfs/namei.c
index 9446f4038874cf..ac9b5e3e83fa0b 100644
--- a/fs/hpfs/namei.c
+++ b/fs/hpfs/namei.c
@@ -19,7 +19,7 @@ static void hpfs_update_directory_times(struct inode *dir)
 	hpfs_write_inode_nolock(dir);
 }
 
-static struct dentry *hpfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *hpfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	const unsigned char *name = dentry->d_name.name;
@@ -128,7 +128,7 @@ static struct dentry *hpfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	return ERR_PTR(err);
 }
 
-static int hpfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int hpfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	const unsigned char *name = dentry->d_name.name;
@@ -215,7 +215,7 @@ static int hpfs_create(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int hpfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int hpfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	const unsigned char *name = dentry->d_name.name;
@@ -289,7 +289,7 @@ static int hpfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int hpfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int hpfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, const char *symlink)
 {
 	const unsigned char *name = dentry->d_name.name;
@@ -500,7 +500,7 @@ const struct address_space_operations hpfs_symlink_aops = {
 	.read_folio	= hpfs_symlink_read_folio
 };
 
-static int hpfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int hpfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		       struct dentry *old_dentry, struct inode *new_dir,
 		       struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 7611a8470ea265..01c4d6f44ebe0a 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -106,11 +106,10 @@ static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
 
 	/*
 	 * vma address alignment (but not the pgoff alignment) has
-	 * already been checked by prepare_hugepage_range.  If you add
-	 * any error returns here, do so after setting VM_HUGETLB, so
-	 * is_vm_hugetlb_page tests below unmap_region go the right
-	 * way when do_mmap unwinds (may be important on powerpc
-	 * and ia64).
+	 * already been checked by hugetlb_get_unmapped_area().  If you
+	 * add any error returns here, do so after setting VM_HUGETLB,
+	 * so vma_is_hugetlb tests below unmap_region go the right
+	 * way when do_mmap unwinds (may be important on powerpc).
 	 */
 	vma_set_flags(vma, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT);
 	vma->vm_ops = &hugetlb_vm_ops;
@@ -828,7 +827,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
 	return error;
 }
 
-static int hugetlbfs_setattr(struct mnt_idmap *idmap,
+static int hugetlbfs_setattr(const struct mnt_idmap *idmap,
 			     struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -892,7 +891,7 @@ static struct inode *hugetlbfs_get_root(struct super_block *sb,
 static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
 
 static struct inode *hugetlbfs_get_inode(struct super_block *sb,
-					struct mnt_idmap *idmap,
+					const struct mnt_idmap *idmap,
 					struct inode *dir,
 					umode_t mode, dev_t dev)
 {
@@ -921,6 +920,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb,
 		simple_inode_init_ts(inode);
 		info->resv_map = resv_map;
 		info->seals = F_SEAL_SEAL;
+		hugetlbfs_pmd_sharing_init(inode);
 		switch (mode & S_IFMT) {
 		default:
 			init_special_inode(inode, mode, dev);
@@ -954,7 +954,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb,
 /*
  * File creation. Allocate an inode, and we're done..
  */
-static int hugetlbfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int hugetlbfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 			   struct dentry *dentry, umode_t mode, dev_t dev)
 {
 	struct inode *inode;
@@ -967,7 +967,7 @@ static int hugetlbfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return 0;
 }
 
-static struct dentry *hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *hugetlbfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				      struct dentry *dentry, umode_t mode)
 {
 	int retval = hugetlbfs_mknod(idmap, dir, dentry,
@@ -977,14 +977,14 @@ static struct dentry *hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir
 	return ERR_PTR(retval);
 }
 
-static int hugetlbfs_create(struct mnt_idmap *idmap,
+static int hugetlbfs_create(const struct mnt_idmap *idmap,
 			    struct inode *dir, struct dentry *dentry,
 			    umode_t mode)
 {
 	return hugetlbfs_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
 }
 
-static int hugetlbfs_tmpfile(struct mnt_idmap *idmap,
+static int hugetlbfs_tmpfile(const struct mnt_idmap *idmap,
 			     struct inode *dir, struct file *file,
 			     umode_t mode)
 {
@@ -998,7 +998,7 @@ static int hugetlbfs_tmpfile(struct mnt_idmap *idmap,
 	return finish_open_simple(file, 0);
 }
 
-static int hugetlbfs_symlink(struct mnt_idmap *idmap,
+static int hugetlbfs_symlink(const struct mnt_idmap *idmap,
 			     struct inode *dir, struct dentry *dentry,
 			     const char *symname)
 {
diff --git a/fs/inode.c b/fs/inode.c
index ba7da39be4a315..702f2d5ef459cd 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -7,6 +7,7 @@
 #include <linux/fs.h>
 #include <linux/filelock.h>
 #include <linux/mm.h>
+#include <linux/pagemap.h>
 #include <linux/backing-dev.h>
 #include <linux/hash.h>
 #include <linux/swap.h>
@@ -880,7 +881,6 @@ void evict_inodes(struct super_block *sb)
 	struct inode *inode;
 	LIST_HEAD(dispose);
 
-again:
 	spin_lock(&sb->s_inode_list_lock);
 	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
 		if (icount_read_once(inode))
@@ -899,19 +899,19 @@ void evict_inodes(struct super_block *sb)
 		inode_state_set(inode, I_FREEING);
 		inode_lru_list_del(inode);
 		spin_unlock(&inode->i_lock);
-		list_add(&inode->i_lru, &dispose);
 
 		/*
-		 * We can have a ton of inodes to evict at unmount time given
-		 * enough memory, check to see if we need to go to sleep for a
-		 * bit so we don't livelock.
+		 * Keep this inode out of dispose so it stays on s_inodes while
+		 * the list lock is dropped. I_FREEING prevents new references
+		 * and leaves eviction to us, so we can resume the walk from it.
 		 */
 		if (need_resched()) {
 			spin_unlock(&sb->s_inode_list_lock);
 			cond_resched();
 			dispose_list(&dispose);
-			goto again;
+			spin_lock(&sb->s_inode_list_lock);
 		}
+		list_add(&inode->i_lru, &dispose);
 	}
 	spin_unlock(&sb->s_inode_list_lock);
 
@@ -1771,7 +1771,7 @@ EXPORT_SYMBOL(ilookup);
  * function must never block --- find_inode() can block in
  * __wait_on_freeing_inode() --- or when the caller can not increment
  * the reference count because the resulting iput() might cause an
- * inode eviction.  The tradeoff is that the @match funtion must be
+ * inode eviction.  The tradeoff is that the @match function must be
  * very carefully implemented.
  */
 struct inode *find_inode_nowait(struct super_block *sb,
@@ -2337,7 +2337,7 @@ EXPORT_SYMBOL(touch_atime);
  * response to write or truncate. Return 0 if nothing has to be changed.
  * Negative value on error (change should be denied).
  */
-int dentry_needs_remove_privs(struct mnt_idmap *idmap,
+int dentry_needs_remove_privs(const struct mnt_idmap *idmap,
 			      struct dentry *dentry)
 {
 	struct inode *inode = d_inode(dentry);
@@ -2356,7 +2356,7 @@ int dentry_needs_remove_privs(struct mnt_idmap *idmap,
 	return mask;
 }
 
-static int __remove_privs(struct mnt_idmap *idmap,
+static int __remove_privs(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, int kill)
 {
 	struct iattr newattrs;
@@ -2716,7 +2716,7 @@ EXPORT_SYMBOL(init_special_inode);
  * and initializing i_uid and i_gid. On non-idmapped mounts or if permission
  * checking is to be performed on the raw inode simply pass @nop_mnt_idmap.
  */
-void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,
+void inode_init_owner(const struct mnt_idmap *idmap, struct inode *inode,
 		      const struct inode *dir, umode_t mode)
 {
 	inode_fsuid_set(inode, idmap);
@@ -2746,7 +2746,7 @@ EXPORT_SYMBOL(inode_init_owner);
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-bool inode_owner_or_capable(struct mnt_idmap *idmap,
+bool inode_owner_or_capable(const struct mnt_idmap *idmap,
 			    const struct inode *inode)
 {
 	vfsuid_t vfsuid;
@@ -3033,7 +3033,7 @@ EXPORT_SYMBOL(inode_set_ctime_deleg);
  *
  * Return: true if the caller is sufficiently privileged, false if not.
  */
-bool in_group_or_capable(struct mnt_idmap *idmap,
+bool in_group_or_capable(const struct mnt_idmap *idmap,
 			 const struct inode *inode, vfsgid_t vfsgid)
 {
 	if (vfsgid_in_group_p(vfsgid))
@@ -3058,7 +3058,7 @@ EXPORT_SYMBOL(in_group_or_capable);
  *
  * Return: the new mode to use for the file
  */
-umode_t mode_strip_sgid(struct mnt_idmap *idmap,
+umode_t mode_strip_sgid(const struct mnt_idmap *idmap,
 			const struct inode *dir, umode_t mode)
 {
 	if ((mode & (S_ISGID | S_IXGRP)) != (S_ISGID | S_IXGRP))
diff --git a/fs/internal.h b/fs/internal.h
index c658c8a5ebd569..e833c7e6e14fbf 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -55,7 +55,7 @@ extern int filename_lookup(int dfd, struct filename *name, unsigned flags,
 			   struct path *path, const struct path *root);
 int filename_rmdir(int dfd, struct filename *name);
 int filename_unlinkat(int dfd, struct filename *name);
-int may_linkat(struct mnt_idmap *idmap, const struct path *link);
+int may_linkat(const struct mnt_idmap *idmap, const struct path *link);
 int filename_renameat2(int olddfd, struct filename *oldname, int newdfd,
 		 struct filename *newname, unsigned int flags);
 int filename_mkdirat(int dfd, struct filename *name, umode_t mode);
@@ -63,7 +63,7 @@ int filename_mknodat(int dfd, struct filename *name, umode_t mode, unsigned int
 int filename_symlinkat(struct filename *from, int newdfd, struct filename *to);
 int filename_linkat(int olddfd, struct filename *old, int newdfd,
 			struct filename *new, int flags);
-int vfs_tmpfile(struct mnt_idmap *idmap,
+int vfs_tmpfile(const struct mnt_idmap *idmap,
 		const struct path *parentpath,
 		struct file *file, umode_t mode);
 struct dentry *d_hash_and_lookup(struct dentry *, struct qstr *);
@@ -198,6 +198,7 @@ extern struct file *do_file_open_root(const struct path *,
 extern struct open_how build_open_how(int flags, umode_t mode);
 extern int build_open_flags(const struct open_how *how, struct open_flags *op);
 struct file *file_close_fd_locked(struct files_struct *files, unsigned fd);
+int filp_close_sync(struct file *filp, fl_owner_t id);
 
 int do_ftruncate(struct file *file, loff_t length, unsigned int flags);
 int chmod_common(const struct path *path, umode_t mode);
@@ -205,13 +206,14 @@ int do_fchownat(int dfd, const char __user *filename, uid_t user, gid_t group,
 		int flag);
 int chown_common(const struct path *path, uid_t user, gid_t group);
 extern int vfs_open(const struct path *, struct file *);
+int vfs_open_consume(struct path *, struct file *);
 
 /*
  * inode.c
  */
 extern long prune_icache_sb(struct super_block *sb, struct shrink_control *sc);
-int dentry_needs_remove_privs(struct mnt_idmap *, struct dentry *dentry);
-bool in_group_or_capable(struct mnt_idmap *idmap,
+int dentry_needs_remove_privs(const struct mnt_idmap *, struct dentry *dentry);
+bool in_group_or_capable(const struct mnt_idmap *idmap,
 			 const struct inode *inode, vfsgid_t vfsgid);
 
 /*
@@ -299,21 +301,21 @@ int filename_setxattr(int dfd, struct filename *filename,
 int setxattr_copy(const char __user *name, struct kernel_xattr_ctx *ctx);
 int import_xattr_name(struct xattr_name *kname, const char __user *name);
 
-int may_write_xattr(struct mnt_idmap *idmap, struct inode *inode);
+int may_write_xattr(const struct mnt_idmap *idmap, struct inode *inode);
 
 #ifdef CONFIG_FS_POSIX_ACL
-int do_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int do_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 	       const char *acl_name, const void *kvalue, size_t size);
-ssize_t do_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+ssize_t do_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   const char *acl_name, void *kvalue, size_t size);
 #else
-static inline int do_set_acl(struct mnt_idmap *idmap,
+static inline int do_set_acl(const struct mnt_idmap *idmap,
 			     struct dentry *dentry, const char *acl_name,
 			     const void *kvalue, size_t size)
 {
 	return -EOPNOTSUPP;
 }
-static inline ssize_t do_get_acl(struct mnt_idmap *idmap,
+static inline ssize_t do_get_acl(const struct mnt_idmap *idmap,
 				 struct dentry *dentry, const char *acl_name,
 				 void *kvalue, size_t size)
 {
@@ -327,8 +329,8 @@ ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *po
  * fs/attr.c
  */
 struct mnt_idmap *alloc_mnt_idmap(struct user_namespace *mnt_userns);
-struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap);
-void mnt_idmap_put(struct mnt_idmap *idmap);
+const struct mnt_idmap *mnt_idmap_get(const struct mnt_idmap *idmap);
+void mnt_idmap_put(const struct mnt_idmap *idmap);
 struct stashed_operations {
 	struct dentry *(*stash_dentry)(struct dentry **stashed,
 				       struct dentry *dentry);
@@ -354,12 +356,12 @@ static inline bool path_mounted(const struct path *path)
 }
 void file_f_owner_release(struct file *file);
 bool file_seek_cur_needs_f_lock(struct file *file);
-int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_map);
+int statmount_mnt_idmap(const struct mnt_idmap *idmap, struct seq_file *seq, bool uid_map);
 struct dentry *find_next_child(struct dentry *parent, struct dentry *prev);
-int anon_inode_getattr(struct mnt_idmap *idmap, const struct path *path,
+int anon_inode_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		       struct kstat *stat, u32 request_mask,
 		       unsigned int query_flags);
-int anon_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int anon_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct iattr *attr);
 void pidfs_get_root(struct path *path);
 void nsfs_get_root(struct path *path);
diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c
index 48100c614431c0..d46c2f8ea18ce0 100644
--- a/fs/iomap/bio.c
+++ b/fs/iomap/bio.c
@@ -169,6 +169,7 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
 {
 	const struct iomap *srcmap = iomap_iter_srcmap(iter);
 	sector_t sector = iomap_sector(srcmap, pos);
+	struct bvec_iter saved_iter;
 	struct bio_vec bvec;
 	struct bio bio;
 	int error;
@@ -178,10 +179,11 @@ int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
 	bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos));
 	if (srcmap->flags & IOMAP_F_INTEGRITY)
 		fs_bio_integrity_alloc(&bio);
+	saved_iter = bio.bi_iter;
 	error = submit_bio_wait(&bio);
 	if (bio_integrity(&bio)) {
 		if (!error)
-			error = fs_bio_integrity_verify(&bio, sector, len);
+			error = fs_bio_integrity_verify(&bio, &saved_iter);
 		fs_bio_integrity_free(&bio);
 	}
 	bio_uninit(&bio);
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index 0a5ebfda90f12e..1e0de37b241187 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -143,8 +143,8 @@ static unsigned ifs_next_clean_block(struct folio *folio,
 			blks + start_blk) - blks;
 }
 
-static unsigned ifs_find_dirty_range(struct folio *folio,
-		struct iomap_folio_state *ifs, u64 *range_start, u64 range_end)
+static unsigned ifs_find_dirty_range(struct folio *folio, u64 *range_start,
+		u64 range_end)
 {
 	struct inode *inode = folio->mapping->host;
 	unsigned start_blk =
@@ -176,7 +176,7 @@ static unsigned iomap_find_dirty_range(struct folio *folio, u64 *range_start,
 		return 0;
 
 	if (ifs)
-		return ifs_find_dirty_range(folio, ifs, range_start, range_end);
+		return ifs_find_dirty_range(folio, range_start, range_end);
 	return range_end - *range_start;
 }
 
@@ -1708,7 +1708,7 @@ static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero,
  * @iomap_flags: Flags to set on the associated iomap to track the batch.
  *
  * Returns the folio count directly. Also returns the associated control flag if
- * the the batch lookup is performed and the expected offset of a subsequent
+ * the batch lookup is performed and the expected offset of a subsequent
  * lookup via out params. The caller is responsible to set the flag on the
  * associated iomap.
  */
@@ -2077,8 +2077,7 @@ iomap_writepages(struct iomap_writepage_ctx *wpc)
 	 * Writeback from reclaim context should never happen except in the case
 	 * of a VM regression so warn about it and refuse to write the data.
 	 */
-	if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC | PF_KSWAPD)) ==
-			PF_MEMALLOC))
+	if (WARN_ON_ONCE((current->flags & PF_MEMALLOC)))
 		return -EIO;
 
 	while ((folio = writeback_iter(mapping, wpc->wbc, folio, &error))) {
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index 8b4039d16ce89f..a431ceda9ebdbd 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -76,10 +76,19 @@ static void iomap_dio_submit_bio(const struct iomap_iter *iter,
 
 	if (dio->dops && dio->dops->submit_io) {
 		dio->dops->submit_io(iter, bio, pos);
-	} else {
-		WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_ANON_WRITE);
-		blk_crypto_submit_bio(bio);
+		return;
+	}
+
+	WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_ANON_WRITE);
+
+	if (iter->iomap.flags & IOMAP_F_INTEGRITY) {
+		if (dio->flags & IOMAP_DIO_WRITE)
+			fs_bio_integrity_generate(bio);
+		else
+			fs_bio_integrity_alloc(bio);
 	}
+
+	blk_crypto_submit_bio(bio);
 }
 
 static inline enum fserror_type iomap_dio_err_type(const struct iomap_dio *dio)
@@ -246,8 +255,7 @@ static void __iomap_dio_bio_end_io(struct bio *bio, bool inline_completion)
 		fs_bio_integrity_free(bio);
 
 	if (dio->flags & IOMAP_DIO_BOUNCE) {
-		bio_iov_iter_unbounce(bio, !!dio->error,
-				dio->flags & IOMAP_DIO_USER_BACKED);
+		bio_free_folios(bio);
 		bio_put(bio);
 	} else if (dio->flags & IOMAP_DIO_USER_BACKED) {
 		bio_check_pages_dirty(bio);
@@ -336,6 +344,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
 		struct iomap_dio *dio, loff_t pos, unsigned int alignment,
 		blk_opf_t op)
 {
+	unsigned int maxsize = iomap_max_bio_size(&iter->iomap);
 	unsigned int nr_vecs;
 	struct bio *bio;
 	ssize_t ret;
@@ -353,14 +362,12 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
 	bio->bi_private = dio;
 	bio->bi_end_io = iomap_dio_bio_end_io;
 
-
 	if (dio->flags & IOMAP_DIO_BOUNCE)
-		ret = bio_iov_iter_bounce(bio, dio->submit.iter,
-				iomap_max_bio_size(&iter->iomap), alignment);
+		ret = bio_iov_iter_bounce_write(bio, dio->submit.iter, maxsize,
+				alignment);
 	else
-		ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
-					     bdev_dma_alignment(bio->bi_bdev),
-					     alignment - 1);
+		ret = bio_iov_iter_get_pages(bio, dio->submit.iter, maxsize,
+				bdev_dma_alignment(bio->bi_bdev), alignment - 1);
 	if (unlikely(ret))
 		goto out_put_bio;
 	ret = bio->bi_iter.bi_size;
@@ -374,13 +381,6 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
 		goto out_bio_release_pages;
 	}
 
-	if (iter->iomap.flags & IOMAP_F_INTEGRITY) {
-		if (dio->flags & IOMAP_DIO_WRITE)
-			fs_bio_integrity_generate(bio);
-		else
-			fs_bio_integrity_alloc(bio);
-	}
-
 	if (dio->flags & IOMAP_DIO_WRITE)
 		task_io_account_write(ret);
 	else if ((dio->flags & IOMAP_DIO_USER_BACKED) &&
@@ -397,7 +397,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
 
 out_bio_release_pages:
 	if (dio->flags & IOMAP_DIO_BOUNCE)
-		bio_iov_iter_unbounce(bio, true, false);
+		bio_free_folios(bio);
 	else
 		bio_release_pages(bio, false);
 out_put_bio:
@@ -505,7 +505,7 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
 		 * We can only do inline completion for pure overwrites that
 		 * don't require additional I/O at completion time.
 		 *
-		 * This rules out writes that need zeroing or metdata updates to
+		 * This rules out writes that need zeroing or metadata updates to
 		 * convert unwritten or shared extents.
 		 *
 		 * Writes that extend i_size are also not supported, but this is
@@ -1034,9 +1034,9 @@ ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter,
 	bio->bi_iter.bi_sector = iomap_sector(&iomi->iomap, iomi->pos);
 	bio->bi_ioprio = iocb->ki_ioprio;
 
-	ret = bio_iov_iter_get_pages(bio, iter,
-				bdev_dma_alignment(bio->bi_bdev),
-				alignment - 1);
+	ret = bio_iov_iter_get_pages(bio, iter, BIO_MAX_SIZE,
+			bdev_dma_alignment(bio->bi_bdev),
+			alignment - 1);
 	if (unlikely(ret))
 		goto out_bio_put;
 
diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c
index 7bbbb417f9152c..bbebecc3167083 100644
--- a/fs/iomap/ioend.c
+++ b/fs/iomap/ioend.c
@@ -25,6 +25,7 @@ struct iomap_ioend *iomap_init_ioend(struct inode *inode,
 	ioend->io_parent = NULL;
 	INIT_LIST_HEAD(&ioend->io_list);
 	ioend->io_flags = ioend_flags;
+	ioend->io_bvec_offset = bio->bi_iter.bi_offset;
 	ioend->io_inode = inode;
 	ioend->io_offset = file_offset;
 	ioend->io_size = bio->bi_iter.bi_size;
@@ -149,7 +150,7 @@ int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error)
 		return error;
 	}
 
-	if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
+	if (ioend->io_flags & IOMAP_IOEND_INTEGRITY)
 		fs_bio_integrity_generate(&ioend->io_bio);
 	submit_bio(&ioend->io_bio);
 	return 0;
@@ -215,7 +216,7 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
 {
 	struct iomap_ioend *ioend = wpc->wb_ctx;
 	size_t poff = offset_in_folio(folio, pos);
-	unsigned int ioend_flags = 0;
+	unsigned int ioend_flags = iomap_ioend_flags(&wpc->iomap);
 	unsigned int map_len = min_t(u64, dirty_len,
 		wpc->iomap.offset + wpc->iomap.length - pos);
 	int error;
@@ -225,20 +226,16 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
 	WARN_ON_ONCE(!folio->private && map_len < dirty_len);
 
 	switch (wpc->iomap.type) {
+	case IOMAP_HOLE:
+		return map_len;
 	case IOMAP_UNWRITTEN:
-		ioend_flags |= IOMAP_IOEND_UNWRITTEN;
-		break;
 	case IOMAP_MAPPED:
 		break;
-	case IOMAP_HOLE:
-		return map_len;
 	default:
 		WARN_ON_ONCE(1);
 		return -EIO;
 	}
 
-	if (wpc->iomap.flags & IOMAP_F_SHARED)
-		ioend_flags |= IOMAP_IOEND_SHARED;
 	if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
 		ioend_flags |= IOMAP_IOEND_BOUNDARY;
 
@@ -312,6 +309,16 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
 }
 EXPORT_SYMBOL_GPL(iomap_add_to_ioend);
 
+#ifdef CONFIG_BLK_DEV_INTEGRITY
+int iomap_ioend_integrity_verify(struct iomap_ioend *ioend)
+{
+	struct bvec_iter data_iter = BVEC_ITER_IOEND(ioend);
+
+	return fs_bio_integrity_verify(&ioend->io_bio, &data_iter);
+}
+EXPORT_SYMBOL_GPL(iomap_ioend_integrity_verify);
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
+
 static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
 {
 	if (ioend->io_parent) {
@@ -327,13 +334,6 @@ static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
 	if (!atomic_dec_and_test(&ioend->io_remaining))
 		return 0;
 
-	if (!ioend->io_error &&
-	    bio_integrity(&ioend->io_bio) &&
-	    bio_op(&ioend->io_bio) == REQ_OP_READ) {
-		ioend->io_error = fs_bio_integrity_verify(&ioend->io_bio,
-			ioend->io_sector, ioend->io_size);
-	}
-
 	if (ioend->io_flags & IOMAP_IOEND_DIRECT)
 		return iomap_finish_ioend_direct(ioend);
 	if (bio_op(&ioend->io_bio) == REQ_OP_READ)
@@ -512,6 +512,96 @@ struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
 }
 EXPORT_SYMBOL_GPL(iomap_split_ioend);
 
+void iomap_bounce_read(struct iomap_ioend *orig_ioend, unsigned int minsize,
+		void (*submit_ioend)(struct iomap_ioend *ioend))
+{
+	struct inode *inode = orig_ioend->io_inode;
+	struct bio *orig_bio = &orig_ioend->io_bio;
+	loff_t file_offset = orig_ioend->io_offset;
+	sector_t sector = orig_ioend->io_sector;
+	size_t total_len = round_up(orig_ioend->io_size, minsize);
+
+	WARN_ON_ONCE(!(orig_ioend->io_flags & IOMAP_IOEND_DIRECT));
+
+	/* We can't poll a bio that is not passed on to hardware */
+	orig_bio->bi_opf &= ~REQ_POLLED;
+
+	do {
+		struct iomap_ioend *ioend;
+		struct bio *bio;
+		int error;
+
+		bio = bio_alloc_bioset(orig_bio->bi_bdev,
+				min(total_len / minsize, BIO_MAX_VECS),
+				orig_bio->bi_opf, GFP_KERNEL,
+				&iomap_ioend_split_bioset);
+		error = bio_alloc_bounce_folios(bio, total_len, minsize);
+		if (error) {
+			bio_put(bio);
+			orig_bio->bi_status = errno_to_blk_status(error);
+			break;
+		}
+		bio->bi_ioprio = orig_bio->bi_ioprio;
+		bio->bi_write_hint = orig_bio->bi_write_hint;
+		bio->bi_write_stream = orig_bio->bi_write_stream;
+		bio->bi_iter.bi_sector = sector;
+
+		ioend = iomap_init_ioend(inode, bio, file_offset,
+				orig_ioend->io_flags);
+
+		total_len -= bio->bi_iter.bi_size;
+		file_offset += bio->bi_iter.bi_size;
+		sector += (bio->bi_iter.bi_size >> SECTOR_SHIFT);
+
+		bio->bi_private = orig_bio;
+		bio_inc_remaining(orig_bio);
+		submit_ioend(ioend);
+	} while (total_len > 0);
+
+	bio_endio(&orig_ioend->io_bio);
+}
+EXPORT_SYMBOL_GPL(iomap_bounce_read);
+
+static void iomap_ioend_unbounce(struct iomap_ioend *orig_ioend,
+		struct iomap_ioend *ioend)
+{
+	struct bio *orig_bio = &orig_ioend->io_bio;
+	struct iov_iter to;
+	struct bio_vec *bv;
+	int i;
+
+	iov_iter_bvec(&to, ITER_DEST, orig_bio->bi_io_vec, orig_bio->bi_vcnt,
+			orig_ioend->io_size);
+	to.iov_offset = orig_ioend->io_bvec_offset;
+
+	if (ioend->io_offset != orig_ioend->io_offset) {
+		WARN_ON_ONCE(ioend->io_offset < orig_ioend->io_offset);
+		iov_iter_advance(&to, ioend->io_offset - orig_ioend->io_offset);
+	}
+
+	/* copying to pinned pages should always work */
+	bio_for_each_bvec_all(bv, &ioend->io_bio, i)
+		WARN_ON_ONCE(copy_to_iter(bvec_virt(bv), bv->bv_len, &to) !=
+				bv->bv_len);
+}
+
+void iomap_bounce_read_end_io(struct iomap_ioend *ioend, struct bio *orig_bio,
+		int error)
+{
+	if (error)
+		orig_bio->bi_status = errno_to_blk_status(error);
+	else
+		iomap_ioend_unbounce(iomap_ioend_from_bio(orig_bio), ioend);
+
+	bio_free_folios(&ioend->io_bio);
+	if (bio_integrity(&ioend->io_bio))
+		fs_bio_integrity_free(&ioend->io_bio);
+	bio_put(&ioend->io_bio);
+
+	bio_endio(orig_bio);
+}
+EXPORT_SYMBOL_GPL(iomap_bounce_read_end_io);
+
 static int __init iomap_ioend_init(void)
 {
 	const unsigned int nr_mempool_entries = 4 * (PAGE_SIZE / SECTOR_SIZE);
diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c
index f9869d62b85096..2d23abaeb87421 100644
--- a/fs/isofs/compress.c
+++ b/fs/isofs/compress.c
@@ -39,7 +39,7 @@ static DEFINE_MUTEX(zisofs_zlib_lock);
  */
 static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
 				      loff_t block_end, int pcount,
-				      struct page **pages, unsigned poffset,
+				      struct folio **folios, unsigned int poffset,
 				      int *errp)
 {
 	unsigned int zisofs_block_shift = ISOFS_I(inode)->i_format_parm[1];
@@ -66,11 +66,12 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
 	if (block_size == 0) {
 		for ( i = 0 ; i < pcount ; i++ ) {
 			unsigned int off = i ? 0 : poffset;
+			struct folio *folio = folios[i];
 
-			if (!pages[i])
+			if (!folio)
 				continue;
-			memzero_page(pages[i], off, PAGE_SIZE - off);
-			SetPageUptodate(pages[i]);
+			folio_zero_range(folio, off, folio_size(folio) - off);
+			folio_mark_uptodate(folio);
 		}
 		return (((loff_t)pcount) << PAGE_SHIFT) - poffset;
 	}
@@ -119,11 +120,12 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
 
 	while (curpage < pcount && curbh < haveblocks &&
 	       zerr != Z_STREAM_END) {
+		struct folio *folio = folios[curpage];
+
 		if (!stream.avail_out) {
-			if (pages[curpage]) {
-				stream.next_out = kmap_local_page(pages[curpage])
-						+ poffset;
-				stream.avail_out = PAGE_SIZE - poffset;
+			if (folio) {
+				stream.next_out = kmap_local_folio(folio, poffset);
+				stream.avail_out = folio_size(folio) - poffset;
 				poffset = 0;
 			} else {
 				stream.next_out = (void *)&zisofs_sink_page;
@@ -173,9 +175,9 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
 
 		if (!stream.avail_out) {
 			/* This page completed */
-			if (pages[curpage]) {
-				flush_dcache_page(pages[curpage]);
-				SetPageUptodate(pages[curpage]);
+			if (folio) {
+				flush_dcache_folio(folio);
+				folio_mark_uptodate(folio);
 			}
 			if (stream.next_out != (unsigned char *)zisofs_sink_page) {
 				kunmap_local(stream.next_out);
@@ -206,7 +208,7 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start,
  * fills in other pages if we have data for them.
  */
 static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount,
-			     struct page **pages)
+			     struct folio **folios)
 {
 	loff_t start_off, end_off;
 	loff_t block_start, block_end;
@@ -221,14 +223,14 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount,
 	int err;
 	loff_t ret;
 
-	BUG_ON(!pages[full_page]);
+	BUG_ON(!folios[full_page]);
 
 	/*
 	 * We want to read at least 'full_page' page. Because we have to
 	 * uncompress the whole compression block anyway, fill the surrounding
 	 * pages with the data we have anyway...
 	 */
-	start_off = page_offset(pages[full_page]);
+	start_off = folio_pos(folios[full_page]);
 	end_off = min_t(loff_t, start_off + PAGE_SIZE, inode->i_size);
 
 	cstart_block = start_off >> zisofs_block_shift;
@@ -267,9 +269,9 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount,
 		}
 		err = 0;
 		ret = zisofs_uncompress_block(inode, block_start, block_end,
-					      pcount, pages, poffset, &err);
+					      pcount, folios, poffset, &err);
 		poffset += ret;
-		pages += poffset >> PAGE_SHIFT;
+		folios += poffset >> PAGE_SHIFT;
 		pcount -= poffset >> PAGE_SHIFT;
 		full_page -= poffset >> PAGE_SHIFT;
 		poffset &= ~PAGE_MASK;
@@ -289,9 +291,11 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount,
 		cstart_block++;
 	}
 
-	if (poffset && *pages) {
-		memzero_page(*pages, poffset, PAGE_SIZE - poffset);
-		SetPageUptodate(*pages);
+	if (poffset && *folios) {
+		struct folio *folio = *folios;
+
+		folio_zero_range(folio, poffset, folio_size(folio) - poffset);
+		folio_mark_uptodate(folio);
 	}
 	brelse(bh);
 	return 0;
@@ -312,7 +316,7 @@ static int zisofs_read_folio(struct file *file, struct folio *folio)
 	unsigned int zisofs_pages_per_cblock =
 		PAGE_SHIFT <= zisofs_block_shift ?
 		(1 << (zisofs_block_shift - PAGE_SHIFT)) : 0;
-	struct page **pages;
+	struct folio **folios;
 	pgoff_t index = folio->index, end_index;
 
 	end_index = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
@@ -336,33 +340,38 @@ static int zisofs_read_folio(struct file *file, struct folio *folio)
 		full_page = 0;
 		pcount = 1;
 	}
-	pages = kzalloc_objs(*pages,
-			     max_t(unsigned int, zisofs_pages_per_cblock, 1));
-	if (!pages) {
+	folios = kzalloc_objs(*folios,
+			      max_t(unsigned int, zisofs_pages_per_cblock, 1));
+	if (!folios) {
 		folio_unlock(folio);
 		return -ENOMEM;
 	}
-	pages[full_page] = &folio->page;
+	folios[full_page] = folio;
 
 	for (i = 0; i < pcount; i++, index++) {
-		if (i != full_page)
-			pages[i] = grab_cache_page_nowait(mapping, index);
+		if (i == full_page)
+			continue;
+		folios[i] = __filemap_get_folio(mapping, index,
+				FGP_LOCK | FGP_CREAT | FGP_NOWAIT,
+				mapping_gfp_mask(mapping));
+		if (IS_ERR(folios[i]))
+			folios[i] = NULL;
 	}
 
-	err = zisofs_fill_pages(inode, full_page, pcount, pages);
+	err = zisofs_fill_pages(inode, full_page, pcount, folios);
 
-	/* Release any residual pages, do not SetPageUptodate */
+	/* Release any residual folios, do not mark them uptodate */
 	for (i = 0; i < pcount; i++) {
-		if (pages[i]) {
-			flush_dcache_page(pages[i]);
-			unlock_page(pages[i]);
+		if (folios[i]) {
+			flush_dcache_folio(folios[i]);
+			folio_unlock(folios[i]);
 			if (i != full_page)
-				put_page(pages[i]);
+				folio_put(folios[i]);
 		}
-	}			
+	}
 
 	/* At this point, err contains 0 or -EIO depending on the "critical" page */
-	kfree(pages);
+	kfree(folios);
 	return err;
 }
 
diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c
index c7ca7603e97a12..28741251d56e36 100644
--- a/fs/isofs/dir.c
+++ b/fs/isofs/dir.c
@@ -110,25 +110,21 @@ static int do_isofs_readdir(struct inode *inode, struct file *file,
 				return 0;
 		}
 
-		de = (struct iso_directory_record *) (bh->b_data + offset);
-
-		de_len = *(unsigned char *)de;
-
+		de = (struct iso_directory_record *)(bh->b_data + offset);
 		/*
-		 * If the length byte is zero, we should move on to the next
-		 * CDROM sector.  If we are at the end of the directory, we
-		 * kick out of the while loop.
+		 * If we are at the end of a block (or at its zero-padded
+		 * tail), move on to the next CDROM sector.  If we are at the
+		 * end of the directory, we'll abort the while loop.
 		 */
-
-		if (de_len == 0) {
+		if (offset >= bufsize || de->length[0] == 0) {
 			brelse(bh);
 			bh = NULL;
-			ctx->pos = (ctx->pos + ISOFS_BLOCK_SIZE) & ~(ISOFS_BLOCK_SIZE - 1);
-			block = ctx->pos >> bufbits;
+			block++;
+			ctx->pos = (loff_t)block << bufbits;
 			offset = 0;
 			continue;
 		}
-
+		de_len = de->length[0];
 		block_saved = block;
 		offset_saved = offset;
 		offset += de_len;
diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c
index 337836a0a17046..184350d2e6ad54 100644
--- a/fs/isofs/inode.c
+++ b/fs/isofs/inode.c
@@ -821,6 +821,8 @@ static int isofs_fill_super(struct super_block *s, struct fs_context *fc)
 	if (!sb_set_blocksize(s, orig_zonesize))
 		goto out_freesbi;
 
+	sbi->s_session_start = (sector_t)vol_desc_start <<
+				(ISOFS_BLOCK_BITS - s->s_blocksize_bits);
 	sbi->s_nls_iocharset = NULL;
 
 #ifdef CONFIG_JOLIET
@@ -1174,7 +1176,6 @@ static int isofs_read_level3_size(struct inode *inode)
 	unsigned long block, offset, block_saved, offset_saved;
 	int i = 0;
 	int more_entries = 0;
-	struct iso_directory_record *tmpde = NULL;
 	struct iso_inode_info *ei = ISOFS_I(inode);
 
 	inode->i_size = 0;
@@ -1198,9 +1199,12 @@ static int isofs_read_level3_size(struct inode *inode)
 				goto out_noread;
 		}
 		de = (struct iso_directory_record *) (bh->b_data + offset);
-		de_len = *(unsigned char *) de;
 
-		if (de_len == 0) {
+		/*
+		 * If we are at the end of a block (or at its zero-padded
+		 * tail), move on to the next block.
+		 */
+		if (offset >= bufsize || de->length[0] == 0) {
 			brelse(bh);
 			bh = NULL;
 			++block;
@@ -1208,32 +1212,18 @@ static int isofs_read_level3_size(struct inode *inode)
 			continue;
 		}
 
+		if (!isofs_dir_record_valid(de, offset, bufsize)) {
+			printk(KERN_NOTICE "iso9660: Corrupted directory entry in block %lu of inode %llu\n",
+			       block, inode->i_ino);
+			brelse(bh);
+			return -EIO;
+		}
+
+		de_len = de->length[0];
 		block_saved = block;
 		offset_saved = offset;
 		offset += de_len;
 
-		/* Make sure we have a full directory entry */
-		if (offset >= bufsize) {
-			int slop = bufsize - offset + de_len;
-			if (!tmpde) {
-				tmpde = kmalloc(256, GFP_KERNEL);
-				if (!tmpde)
-					goto out_nomem;
-			}
-			memcpy(tmpde, de, slop);
-			offset &= bufsize - 1;
-			block++;
-			brelse(bh);
-			bh = NULL;
-			if (offset) {
-				bh = sb_bread(inode->i_sb, block);
-				if (!bh)
-					goto out_noread;
-				memcpy((void *)tmpde+slop, bh->b_data, offset);
-			}
-			de = tmpde;
-		}
-
 		inode->i_size += isonum_733(de->size);
 		if (i == 1) {
 			ei->i_next_section_block = block_saved;
@@ -1247,17 +1237,11 @@ static int isofs_read_level3_size(struct inode *inode)
 			goto out_toomany;
 	} while (more_entries);
 out:
-	kfree(tmpde);
 	brelse(bh);
 	return 0;
 
-out_nomem:
-	brelse(bh);
-	return -ENOMEM;
-
 out_noread:
 	printk(KERN_INFO "ISOFS: unable to read i-node block %lu\n", block);
-	kfree(tmpde);
 	return -EIO;
 
 out_toomany:
diff --git a/fs/isofs/isofs.h b/fs/isofs/isofs.h
index dacb9cdae4fd1b..79ca0256843aa5 100644
--- a/fs/isofs/isofs.h
+++ b/fs/isofs/isofs.h
@@ -35,6 +35,8 @@ struct isofs_sb_info {
 	unsigned long s_firstdatazone;
 	unsigned long s_log_zone_size;
 	unsigned long s_max_size;
+	/* Session start in filesystem block units. */
+	sector_t s_session_start;
 	
 	int           s_rock_offset; /* offset of SUSP fields within SU area */
 	s32           s_sbsector;
diff --git a/fs/isofs/namei.c b/fs/isofs/namei.c
index 010682f5901a49..4fba1bf7f0167c 100644
--- a/fs/isofs/namei.c
+++ b/fs/isofs/namei.c
@@ -74,18 +74,20 @@ isofs_find_entry(struct inode *dir, struct dentry *dentry,
 				return 0;
 		}
 
-		de = (struct iso_directory_record *) (bh->b_data + offset);
-
-		de_len = *(unsigned char *) de;
-		if (!de_len) {
+		de = (struct iso_directory_record *)(bh->b_data + offset);
+		/*
+		 * If we are at the end of the block or at its zero-padded
+		 * tail, move to the next block.
+		 */
+		if (offset >= bufsize || de->length[0] == 0) {
 			brelse(bh);
 			bh = NULL;
-			f_pos = (f_pos + ISOFS_BLOCK_SIZE) & ~(ISOFS_BLOCK_SIZE - 1);
-			block = f_pos >> bufbits;
+			block++;
+			f_pos = block << bufbits;
 			offset = 0;
 			continue;
 		}
-
+		de_len = de->length[0];
 		block_saved = bh->b_blocknr;
 		offset_saved = offset;
 		offset += de_len;
diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c
index 2628f31bd3a5da..84e0d764c2100f 100644
--- a/fs/isofs/rock.c
+++ b/fs/isofs/rock.c
@@ -9,6 +9,7 @@
 
 #include <linux/slab.h>
 #include <linux/pagemap.h>
+#include <linux/blkdev.h>
 
 #include "isofs.h"
 #include "rock.h"
@@ -84,6 +85,10 @@ static void init_rock_state(struct rock_state *rs, struct inode *inode)
  */
 static int rock_continue(struct rock_state *rs)
 {
+	struct super_block *sb = rs->inode->i_sb;
+	struct isofs_sb_info *sbi = ISOFS_SB(sb);
+	sector_t session_end = sbi->s_session_start +
+		((sector_t)sbi->s_nzones << (ISOFS_BLOCK_BITS - sb->s_blocksize_bits));
 	int ret = 1;
 	int blocksize = 1 << rs->inode->i_blkbits;
 	const int min_de_size = offsetof(struct rock_ridge, u);
@@ -101,11 +106,13 @@ static int rock_continue(struct rock_state *rs)
 		goto out;
 	}
 
-	if ((unsigned)rs->cont_extent >= ISOFS_SB(rs->inode->i_sb)->s_nzones) {
+	if (rs->cont_extent &&
+	    (rs->cont_extent < sbi->s_session_start ||
+	     rs->cont_extent >= session_end)) {
 		printk(KERN_NOTICE "rock: corrupted directory entry. "
 			"extent=%u out of volume (nzones=%lu)\n",
 			(unsigned)rs->cont_extent,
-			ISOFS_SB(rs->inode->i_sb)->s_nzones);
+			sbi->s_nzones);
 		ret = -EIO;
 		goto out;
 	}
diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c
index 3029cb6f6d640e..ebf6ba58ff4d0c 100644
--- a/fs/jbd2/commit.c
+++ b/fs/jbd2/commit.c
@@ -32,14 +32,14 @@
 static void journal_end_buffer_io_sync(struct bio *bio)
 {
 	struct buffer_head *bh;
-	bool uptodate = bio_endio_bh(bio, &bh);
+	bool success = bio_endio_bh(bio, &bh);
 	struct buffer_head *orig_bh = bh->b_private;
 
 	BUFFER_TRACE(bh, "");
-	if (uptodate)
-		set_buffer_uptodate(bh);
+	if (success)
+		clear_buffer_write_io_error(bh);
 	else
-		clear_buffer_uptodate(bh);
+		mark_buffer_write_io_error(bh);
 	if (orig_bh) {
 		clear_and_wake_up_bit(BH_Shadow, &orig_bh->b_state);
 	}
@@ -169,7 +169,7 @@ static int journal_wait_on_commit_record(journal_t *journal,
 	clear_buffer_dirty(bh);
 	wait_on_buffer(bh);
 
-	if (unlikely(!buffer_uptodate(bh)))
+	if (unlikely(buffer_write_io_error(bh)))
 		ret = -EIO;
 	put_bh(bh);            /* One for getblk() */
 
@@ -330,9 +330,9 @@ static __u32 jbd2_checksum_data(__u32 crc32_sum, struct buffer_head *bh)
 	char *addr;
 	__u32 checksum;
 
-	addr = kmap_local_folio(bh->b_folio, bh_offset(bh));
+	addr = kmap_local_bh(bh);
 	checksum = crc32_be(crc32_sum, addr, bh->b_size);
-	kunmap_local(addr);
+	kunmap_local_bh(bh, addr);
 
 	return checksum;
 }
@@ -357,10 +357,10 @@ static void jbd2_block_tag_csum_set(journal_t *j, journal_block_tag_t *tag,
 		return;
 
 	seq = cpu_to_be32(sequence);
-	addr = kmap_local_folio(bh->b_folio, bh_offset(bh));
+	addr = kmap_local_bh(bh);
 	csum32 = jbd2_chksum(j->j_csum_seed, (__u8 *)&seq, sizeof(seq));
 	csum32 = jbd2_chksum(csum32, addr, bh->b_size);
-	kunmap_local(addr);
+	kunmap_local_bh(bh, addr);
 
 	if (jbd2_has_feature_csum3(j))
 		tag3->t_checksum = cpu_to_be32(csum32);
@@ -834,7 +834,7 @@ void jbd2_journal_commit_transaction(journal_t *journal)
 		wait_on_buffer(bh);
 		cond_resched();
 
-		if (unlikely(!buffer_uptodate(bh)))
+		if (unlikely(buffer_write_io_error(bh)))
 			err = -EIO;
 		jbd2_unfile_log_bh(bh);
 		stats.run.rs_blocks_logged++;
@@ -877,7 +877,7 @@ void jbd2_journal_commit_transaction(journal_t *journal)
 		wait_on_buffer(bh);
 		cond_resched();
 
-		if (unlikely(!buffer_uptodate(bh)))
+		if (unlikely(buffer_write_io_error(bh)))
 			err = -EIO;
 
 		BUFFER_TRACE(bh, "ph5: control buffer writeout done: unfile");
diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c
index 00f5a98f3d4fe6..cda1ff8851dcc6 100644
--- a/fs/jbd2/journal.c
+++ b/fs/jbd2/journal.c
@@ -328,8 +328,6 @@ int jbd2_journal_write_metadata_buffer(transaction_t *transaction,
 {
 	int do_escape = 0;
 	struct buffer_head *new_bh;
-	struct folio *new_folio;
-	unsigned int new_offset;
 	struct buffer_head *bh_in = jh2bh(jh_in);
 	journal_t *journal = transaction->t_journal;
 
@@ -349,24 +347,31 @@ int jbd2_journal_write_metadata_buffer(transaction_t *transaction,
 	/* keep subsequent assertions sane */
 	atomic_set(&new_bh->b_count, 1);
 
+	/*
+	 * b_frozen_data is slab memory, not page cache, so when we use it the
+	 * shadow buffer gets no folio at all: b_folio stays NULL from the
+	 * allocation and b_data points straight at the copy.  Pointing it at
+	 * the slab folio instead would hand its overloaded ->mapping to
+	 * anything that goes looking for an address_space.
+	 */
+
 	spin_lock(&jh_in->b_state_lock);
 	/*
 	 * If a new transaction has already done a buffer copy-out, then
 	 * we use that version of the data for the commit.
 	 */
 	if (jh_in->b_frozen_data) {
-		new_folio = virt_to_folio(jh_in->b_frozen_data);
-		new_offset = offset_in_folio(new_folio, jh_in->b_frozen_data);
 		do_escape = jbd2_data_needs_escaping(jh_in->b_frozen_data);
 		if (do_escape)
 			jbd2_data_do_escape(jh_in->b_frozen_data);
+		new_bh->b_data = jh_in->b_frozen_data;
 	} else {
+		struct folio *folio = bh_in->b_folio;
+		unsigned int offset = offset_in_folio(folio, bh_in->b_data);
 		char *tmp;
 		char *mapped_data;
 
-		new_folio = bh_in->b_folio;
-		new_offset = offset_in_folio(new_folio, bh_in->b_data);
-		mapped_data = kmap_local_folio(new_folio, new_offset);
+		mapped_data = kmap_local_folio(folio, offset);
 		/*
 		 * Fire data frozen trigger if data already wasn't frozen. Do
 		 * this before checking for escaping, as the trigger may modify
@@ -380,8 +385,10 @@ int jbd2_journal_write_metadata_buffer(transaction_t *transaction,
 		/*
 		 * Do we need to do a data copy?
 		 */
-		if (!do_escape)
+		if (!do_escape) {
+			folio_set_bh(new_bh, folio, offset);
 			goto escape_done;
+		}
 
 		spin_unlock(&jh_in->b_state_lock);
 		tmp = kmalloc(bh_in->b_size, GFP_NOFS | __GFP_NOFAIL);
@@ -392,7 +399,7 @@ int jbd2_journal_write_metadata_buffer(transaction_t *transaction,
 		}
 
 		jh_in->b_frozen_data = tmp;
-		memcpy_from_folio(tmp, new_folio, new_offset, bh_in->b_size);
+		memcpy_from_folio(tmp, folio, offset, bh_in->b_size);
 		/*
 		 * This isn't strictly necessary, as we're using frozen
 		 * data for the escaping, but it keeps consistency with
@@ -401,13 +408,11 @@ int jbd2_journal_write_metadata_buffer(transaction_t *transaction,
 		jh_in->b_frozen_triggers = jh_in->b_triggers;
 
 copy_done:
-		new_folio = virt_to_folio(jh_in->b_frozen_data);
-		new_offset = offset_in_folio(new_folio, jh_in->b_frozen_data);
 		jbd2_data_do_escape(jh_in->b_frozen_data);
+		new_bh->b_data = jh_in->b_frozen_data;
 	}
 
 escape_done:
-	folio_set_bh(new_bh, new_folio, new_offset);
 	new_bh->b_size = bh_in->b_size;
 	new_bh->b_bdev = journal->j_dev;
 	new_bh->b_blocknr = blocknr;
@@ -882,7 +887,7 @@ int jbd2_fc_wait_bufs(journal_t *journal, int num_blks)
 		 * Update j_fc_off so jbd2_fc_release_bufs can release remain
 		 * buffer head.
 		 */
-		if (unlikely(!buffer_uptodate(bh))) {
+		if (unlikely(buffer_write_io_error(bh))) {
 			journal->j_fc_off = i + 1;
 			return -EIO;
 		}
diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c
index 5cc7d097b2ac8e..85d84d909f785c 100644
--- a/fs/jbd2/transaction.c
+++ b/fs/jbd2/transaction.c
@@ -920,7 +920,7 @@ static void jbd2_freeze_jh_data(struct journal_head *jh)
 	char *source;
 	struct buffer_head *bh = jh2bh(jh);
 
-	J_EXPECT_JH(jh, buffer_uptodate(bh), "Possible IO failure.\n");
+	J_EXPECT_JH(jh, buffer_uptodate(bh), "Buffer not uptodate!\n");
 	source = kmap_local_folio(bh->b_folio, bh_offset(bh));
 	/* Fire data frozen trigger just before we copy the data */
 	jbd2_buffer_frozen_trigger(jh, source, jh->b_triggers);
diff --git a/fs/jffs2/acl.c b/fs/jffs2/acl.c
index f0f8a4f57add0a..7548f44bf32754 100644
--- a/fs/jffs2/acl.c
+++ b/fs/jffs2/acl.c
@@ -228,7 +228,7 @@ static int __jffs2_set_acl(struct inode *inode, int xprefix, struct posix_acl *a
 	return rc;
 }
 
-int jffs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int jffs2_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct posix_acl *acl, int type)
 {
 	int rc, xprefix;
diff --git a/fs/jffs2/acl.h b/fs/jffs2/acl.h
index e976b8cb82cf91..bc5df521633ff8 100644
--- a/fs/jffs2/acl.h
+++ b/fs/jffs2/acl.h
@@ -28,7 +28,7 @@ struct jffs2_acl_header {
 #ifdef CONFIG_JFFS2_FS_POSIX_ACL
 
 struct posix_acl *jffs2_get_acl(struct inode *inode, int type, bool rcu);
-int jffs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int jffs2_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct posix_acl *acl, int type);
 extern int jffs2_init_acl_pre(struct inode *, struct inode *, umode_t *);
 extern int jffs2_init_acl_post(struct inode *);
diff --git a/fs/jffs2/dir.c b/fs/jffs2/dir.c
index 656c920864c50b..23813f1912818d 100644
--- a/fs/jffs2/dir.c
+++ b/fs/jffs2/dir.c
@@ -25,20 +25,20 @@
 
 static int jffs2_readdir (struct file *, struct dir_context *);
 
-static int jffs2_create (struct mnt_idmap *, struct inode *,
+static int jffs2_create (const struct mnt_idmap *, struct inode *,
 		         struct dentry *, umode_t);
 static struct dentry *jffs2_lookup (struct inode *,struct dentry *,
 				    unsigned int);
 static int jffs2_link (struct dentry *,struct inode *,struct dentry *);
 static int jffs2_unlink (struct inode *,struct dentry *);
-static int jffs2_symlink (struct mnt_idmap *, struct inode *,
+static int jffs2_symlink (const struct mnt_idmap *, struct inode *,
 			  struct dentry *, const char *);
-static struct dentry *jffs2_mkdir (struct mnt_idmap *, struct inode *,struct dentry *,
+static struct dentry *jffs2_mkdir (const struct mnt_idmap *, struct inode *,struct dentry *,
 				   umode_t);
 static int jffs2_rmdir (struct inode *,struct dentry *);
-static int jffs2_mknod (struct mnt_idmap *, struct inode *,struct dentry *,
+static int jffs2_mknod (const struct mnt_idmap *, struct inode *,struct dentry *,
 			umode_t,dev_t);
-static int jffs2_rename (struct mnt_idmap *, struct inode *,
+static int jffs2_rename (const struct mnt_idmap *, struct inode *,
 			 struct dentry *, struct inode *, struct dentry *,
 			 unsigned int);
 
@@ -162,7 +162,7 @@ static int jffs2_readdir(struct file *file, struct dir_context *ctx)
 /***********************************************************************/
 
 
-static int jffs2_create(struct mnt_idmap *idmap, struct inode *dir_i,
+static int jffs2_create(const struct mnt_idmap *idmap, struct inode *dir_i,
 			struct dentry *dentry, umode_t mode)
 {
 	struct jffs2_raw_inode *ri;
@@ -284,7 +284,7 @@ static int jffs2_link (struct dentry *old_dentry, struct inode *dir_i, struct de
 
 /***********************************************************************/
 
-static int jffs2_symlink (struct mnt_idmap *idmap, struct inode *dir_i,
+static int jffs2_symlink (const struct mnt_idmap *idmap, struct inode *dir_i,
 			  struct dentry *dentry, const char *target)
 {
 	struct jffs2_inode_info *f, *dir_f;
@@ -448,7 +448,7 @@ static int jffs2_symlink (struct mnt_idmap *idmap, struct inode *dir_i,
 }
 
 
-static struct dentry *jffs2_mkdir (struct mnt_idmap *idmap, struct inode *dir_i,
+static struct dentry *jffs2_mkdir (const struct mnt_idmap *idmap, struct inode *dir_i,
 				   struct dentry *dentry, umode_t mode)
 {
 	struct jffs2_inode_info *f, *dir_f;
@@ -620,7 +620,7 @@ static int jffs2_rmdir (struct inode *dir_i, struct dentry *dentry)
 	return ret;
 }
 
-static int jffs2_mknod (struct mnt_idmap *idmap, struct inode *dir_i,
+static int jffs2_mknod (const struct mnt_idmap *idmap, struct inode *dir_i,
 		        struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct jffs2_inode_info *f, *dir_f;
@@ -769,7 +769,7 @@ static int jffs2_mknod (struct mnt_idmap *idmap, struct inode *dir_i,
 	return ret;
 }
 
-static int jffs2_rename (struct mnt_idmap *idmap,
+static int jffs2_rename (const struct mnt_idmap *idmap,
 			 struct inode *old_dir_i, struct dentry *old_dentry,
 			 struct inode *new_dir_i, struct dentry *new_dentry,
 			 unsigned int flags)
diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c
index 6ada8369a76227..05cf860307c737 100644
--- a/fs/jffs2/fs.c
+++ b/fs/jffs2/fs.c
@@ -190,7 +190,7 @@ int jffs2_do_setattr (struct inode *inode, struct iattr *iattr)
 	return 0;
 }
 
-int jffs2_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int jffs2_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/jffs2/os-linux.h b/fs/jffs2/os-linux.h
index 86ab014a349c5e..bff2134d771d17 100644
--- a/fs/jffs2/os-linux.h
+++ b/fs/jffs2/os-linux.h
@@ -164,7 +164,7 @@ long jffs2_ioctl(struct file *, unsigned int, unsigned long);
 extern const struct inode_operations jffs2_symlink_inode_operations;
 
 /* fs.c */
-int jffs2_setattr (struct mnt_idmap *, struct dentry *, struct iattr *);
+int jffs2_setattr (const struct mnt_idmap *, struct dentry *, struct iattr *);
 int jffs2_do_setattr (struct inode *, struct iattr *);
 struct inode *jffs2_iget(struct super_block *, unsigned long);
 void jffs2_evict_inode (struct inode *);
diff --git a/fs/jffs2/security.c b/fs/jffs2/security.c
index 437f3a2c1b5441..67330aeb8ae8c0 100644
--- a/fs/jffs2/security.c
+++ b/fs/jffs2/security.c
@@ -57,7 +57,7 @@ static int jffs2_security_getxattr(const struct xattr_handler *handler,
 }
 
 static int jffs2_security_setxattr(const struct xattr_handler *handler,
-				   struct mnt_idmap *idmap,
+				   const struct mnt_idmap *idmap,
 				   struct dentry *unused, struct inode *inode,
 				   const char *name, const void *buffer,
 				   size_t size, int flags)
diff --git a/fs/jffs2/xattr_trusted.c b/fs/jffs2/xattr_trusted.c
index b7c5da2d89bdf1..85133ad8b449e5 100644
--- a/fs/jffs2/xattr_trusted.c
+++ b/fs/jffs2/xattr_trusted.c
@@ -25,7 +25,7 @@ static int jffs2_trusted_getxattr(const struct xattr_handler *handler,
 }
 
 static int jffs2_trusted_setxattr(const struct xattr_handler *handler,
-				  struct mnt_idmap *idmap,
+				  const struct mnt_idmap *idmap,
 				  struct dentry *unused, struct inode *inode,
 				  const char *name, const void *buffer,
 				  size_t size, int flags)
diff --git a/fs/jffs2/xattr_user.c b/fs/jffs2/xattr_user.c
index f64edce4927b2e..dcfd3caf1d8ba0 100644
--- a/fs/jffs2/xattr_user.c
+++ b/fs/jffs2/xattr_user.c
@@ -25,7 +25,7 @@ static int jffs2_user_getxattr(const struct xattr_handler *handler,
 }
 
 static int jffs2_user_setxattr(const struct xattr_handler *handler,
-			       struct mnt_idmap *idmap,
+			       const struct mnt_idmap *idmap,
 			       struct dentry *unused, struct inode *inode,
 			       const char *name, const void *buffer,
 			       size_t size, int flags)
diff --git a/fs/jfs/acl.c b/fs/jfs/acl.c
index 16b71a23ff1e33..6e0a7feb6c8089 100644
--- a/fs/jfs/acl.c
+++ b/fs/jfs/acl.c
@@ -89,7 +89,7 @@ static int __jfs_set_acl(tid_t tid, struct inode *inode, int type,
 	return rc;
 }
 
-int jfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int jfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct posix_acl *acl, int type)
 {
 	int rc;
diff --git a/fs/jfs/file.c b/fs/jfs/file.c
index 246568cb9a6ec1..2f5bb79c05916a 100644
--- a/fs/jfs/file.c
+++ b/fs/jfs/file.c
@@ -89,7 +89,7 @@ static int jfs_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-int jfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int jfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/jfs/ioctl.c b/fs/jfs/ioctl.c
index 563f148be8affb..27d39cddaca6e7 100644
--- a/fs/jfs/ioctl.c
+++ b/fs/jfs/ioctl.c
@@ -70,7 +70,7 @@ int jfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int jfs_fileattr_set(struct mnt_idmap *idmap,
+int jfs_fileattr_set(const struct mnt_idmap *idmap,
 		     struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/jfs/jfs_acl.h b/fs/jfs/jfs_acl.h
index f892e54d0fcd2d..bda26b33351962 100644
--- a/fs/jfs/jfs_acl.h
+++ b/fs/jfs/jfs_acl.h
@@ -8,7 +8,7 @@
 #ifdef CONFIG_JFS_POSIX_ACL
 
 struct posix_acl *jfs_get_acl(struct inode *inode, int type, bool rcu);
-int jfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int jfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct posix_acl *acl, int type);
 int jfs_init_acl(tid_t, struct inode *, struct inode *);
 
diff --git a/fs/jfs/jfs_inode.h b/fs/jfs/jfs_inode.h
index 2c6c81c8cb9fb7..5a118b07fbff31 100644
--- a/fs/jfs/jfs_inode.h
+++ b/fs/jfs/jfs_inode.h
@@ -10,7 +10,7 @@ struct fid;
 extern struct inode *ialloc(struct inode *, umode_t);
 extern int jfs_fsync(struct file *, loff_t, loff_t, int);
 extern int jfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-extern int jfs_fileattr_set(struct mnt_idmap *idmap,
+extern int jfs_fileattr_set(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct file_kattr *fa);
 extern long jfs_ioctl(struct file *, unsigned int, unsigned long);
 extern struct inode *jfs_iget(struct super_block *, unsigned long);
@@ -28,7 +28,7 @@ extern struct dentry *jfs_fh_to_parent(struct super_block *sb, struct fid *fid,
 	int fh_len, int fh_type);
 extern void jfs_set_inode_flags(struct inode *);
 extern int jfs_get_block(struct inode *, sector_t, struct buffer_head *, int);
-extern int jfs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *);
+extern int jfs_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *);
 
 extern const struct address_space_operations jfs_aops;
 extern const struct inode_operations jfs_dir_inode_operations;
diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c
index 8a36c218f0f7a7..8ab2e952ce16b0 100644
--- a/fs/jfs/namei.c
+++ b/fs/jfs/namei.c
@@ -60,7 +60,7 @@ static inline void free_ea_wmap(struct inode *inode)
  * RETURN:	Errors from subroutines
  *
  */
-static int jfs_create(struct mnt_idmap *idmap, struct inode *dip,
+static int jfs_create(const struct mnt_idmap *idmap, struct inode *dip,
 		      struct dentry *dentry, umode_t mode)
 {
 	int rc = 0;
@@ -193,7 +193,7 @@ static int jfs_create(struct mnt_idmap *idmap, struct inode *dip,
  * note:
  * EACCES: user needs search+write permission on the parent directory
  */
-static struct dentry *jfs_mkdir(struct mnt_idmap *idmap, struct inode *dip,
+static struct dentry *jfs_mkdir(const struct mnt_idmap *idmap, struct inode *dip,
 				struct dentry *dentry, umode_t mode)
 {
 	int rc = 0;
@@ -876,7 +876,7 @@ static int jfs_link(struct dentry *old_dentry,
  * an intermediate result whose length exceeds PATH_MAX [XPG4.2]
 */
 
-static int jfs_symlink(struct mnt_idmap *idmap, struct inode *dip,
+static int jfs_symlink(const struct mnt_idmap *idmap, struct inode *dip,
 		       struct dentry *dentry, const char *name)
 {
 	int rc;
@@ -1066,7 +1066,7 @@ static int jfs_symlink(struct mnt_idmap *idmap, struct inode *dip,
  *
  * FUNCTION:	rename a file or directory
  */
-static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int jfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags)
 {
@@ -1355,7 +1355,7 @@ static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
  *
  * FUNCTION:	Create a special file (device)
  */
-static int jfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int jfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct jfs_inode_info *jfs_ip;
diff --git a/fs/jfs/xattr.c b/fs/jfs/xattr.c
index 11d7f74d207be0..dcc4a69d44fec9 100644
--- a/fs/jfs/xattr.c
+++ b/fs/jfs/xattr.c
@@ -956,7 +956,7 @@ static int jfs_xattr_get(const struct xattr_handler *handler,
 }
 
 static int jfs_xattr_set(const struct xattr_handler *handler,
-			 struct mnt_idmap *idmap,
+			 const struct mnt_idmap *idmap,
 			 struct dentry *unused, struct inode *inode,
 			 const char *name, const void *value,
 			 size_t size, int flags)
@@ -975,7 +975,7 @@ static int jfs_xattr_get_os2(const struct xattr_handler *handler,
 }
 
 static int jfs_xattr_set_os2(const struct xattr_handler *handler,
-			     struct mnt_idmap *idmap,
+			     const struct mnt_idmap *idmap,
 			     struct dentry *unused, struct inode *inode,
 			     const char *name, const void *value,
 			     size_t size, int flags)
diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c
index 82bbaeb326aab5..324d61a00545b0 100644
--- a/fs/kernfs/dir.c
+++ b/fs/kernfs/dir.c
@@ -30,6 +30,8 @@ static char kernfs_pr_cont_buf[PATH_MAX];	/* protected by pr_cont_lock */
 
 #define rb_to_kn(X) rb_entry((X), struct kernfs_node, rb)
 
+static void kernfs_activate_one(struct kernfs_node *kn);
+
 static bool __kernfs_active(struct kernfs_node *kn)
 {
 	return atomic_read(&kn->active) >= 0;
@@ -736,13 +738,19 @@ struct kernfs_node *kernfs_new_node(struct kernfs_node *parent,
 {
 	struct kernfs_node *kn;
 
-	if (parent->mode & S_ISGID) {
+	/*
+	 * The mode and the gid below are read unlocked on purpose: they feed
+	 * a node that does not exist yet, so nothing orders a racing chmod or
+	 * chown against this creation.
+	 */
+	if (READ_ONCE(parent->mode) & S_ISGID) {
 		/* this code block imitates inode_init_owner() for
 		 * kernfs
 		 */
+		struct kernfs_iattrs *attrs = READ_ONCE(parent->iattr);
 
-		if (parent->iattr)
-			gid = parent->iattr->ia_gid;
+		if (attrs)
+			gid = READ_ONCE(attrs->ia_gid);
 
 		if (flags & KERNFS_DIR)
 			mode |= S_ISGID;
@@ -855,7 +863,6 @@ int kernfs_add_one(struct kernfs_node *kn)
 	}
 
 	up_write(&root->kernfs_iattr_rwsem);
-	up_write(&root->kernfs_rwsem);
 
 	/*
 	 * Activate the new node unless CREATE_DEACTIVATED is requested.
@@ -863,9 +870,15 @@ int kernfs_add_one(struct kernfs_node *kn)
 	 * activating the node with kernfs_activate().  A node which hasn't
 	 * been activated is not visible to userland and its removal won't
 	 * trigger deactivation.
+	 *
+	 * @kn has no children yet, so kernfs_activate() would walk only @kn.
+	 * Do it here rather than dropping the write lock and taking it again
+	 * for every new node.
 	 */
-	if (!(kernfs_root(kn)->flags & KERNFS_ROOT_CREATE_DEACTIVATED))
-		kernfs_activate(kn);
+	if (!(root->flags & KERNFS_ROOT_CREATE_DEACTIVATED))
+		kernfs_activate_one(kn);
+
+	up_write(&root->kernfs_rwsem);
 	return 0;
 
 out_unlock:
@@ -1171,23 +1184,18 @@ struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent,
 static int kernfs_dop_revalidate(struct inode *dir, const struct qstr *name,
 				 struct dentry *dentry, unsigned int flags)
 {
-	struct kernfs_node *kn, *parent;
-	struct kernfs_root *root;
+	struct kernfs_node *parent = dir->i_private;
+	struct kernfs_node *kn;
+	const char *kn_name;
 
 	if (flags & LOOKUP_RCU)
 		return -ECHILD;
 
 	/* Negative hashed dentry? */
 	if (d_really_is_negative(dentry)) {
-		/* If the kernfs parent node has changed discard and
-		 * proceed to ->lookup.
-		 *
-		 * There's nothing special needed here when getting the
-		 * dentry parent, even if a concurrent rename is in
-		 * progress. That's because the dentry is negative so
-		 * it can only be the target of the rename and it will
-		 * be doing a d_move() not a replace. Consequently the
-		 * dentry d_parent won't change over the d_move().
+		/*
+		 * If the kernfs parent node has changed discard and proceed to
+		 * ->lookup.
 		 *
 		 * Also kernfs negative dentries transitioning from
 		 * negative to positive during revalidate won't happen
@@ -1195,50 +1203,41 @@ static int kernfs_dop_revalidate(struct inode *dir, const struct qstr *name,
 		 * changes and the lookup re-done so that a new positive
 		 * dentry can be properly created.
 		 */
-		root = kernfs_root_from_sb(dentry->d_sb);
-		down_read(&root->kernfs_rwsem);
-		parent = kernfs_dentry_node(dentry->d_parent);
-		if (parent) {
-			if (kernfs_dir_changed(parent, dentry)) {
-				up_read(&root->kernfs_rwsem);
-				return 0;
-			}
-		}
-		up_read(&root->kernfs_rwsem);
-
-		/* The kernfs parent node hasn't changed, leave the
-		 * dentry negative and return success.
-		 */
-		return 1;
+		return !kernfs_dir_changed(parent, dentry);
 	}
 
 	kn = kernfs_dentry_node(dentry);
-	root = kernfs_root(kn);
-	down_read(&root->kernfs_rwsem);
+
+	guard(rcu)();
 
 	/* The kernfs node has been deactivated */
-	if (!kernfs_active(kn))
-		goto out_bad;
+	if (!__kernfs_active(kn))
+		return 0;
 
-	parent = kernfs_parent(kn);
 	/* The kernfs node has been moved? */
-	if (kernfs_dentry_node(dentry->d_parent) != parent)
-		goto out_bad;
+	if (kernfs_parent(kn) != parent)
+		return 0;
 
 	/* The kernfs node has been renamed */
-	if (strcmp(dentry->d_name.name, kernfs_rcu_name(kn)) != 0)
-		goto out_bad;
+	kn_name = kernfs_rcu_name(kn);
+	if (name->len != strlen(kn_name) ||
+	    memcmp(name->name, kn_name, name->len))
+		return 0;
 
-	/* The kernfs node has been moved to a different namespace */
-	if (parent && kernfs_ns_enabled(parent) &&
-	    kernfs_ns_id(kernfs_info(dentry->d_sb)->ns) != kernfs_ns_id(kn->ns))
-		goto out_bad;
+	/*
+	 * The kernfs node has been moved to a different namespace.
+	 *
+	 * KERNFS_NS is set by kernfs_enable_ns() while @parent still has no
+	 * children, so it cannot change while a child of @parent is being
+	 * revalidated. The other bits in that word, KERNFS_ACTIVATED and
+	 * KERNFS_REMOVING, are updated under kernfs_rwsem and are not read
+	 * here, so racing with them is intentional and harmless.
+	 */
+	if (data_race(kernfs_ns_enabled(parent)) &&
+	    kernfs_info(dir->i_sb)->ns != READ_ONCE(kn->ns))
+		return 0;
 
-	up_read(&root->kernfs_rwsem);
 	return 1;
-out_bad:
-	up_read(&root->kernfs_rwsem);
-	return 0;
 }
 
 const struct dentry_operations kernfs_dops = {
@@ -1288,7 +1287,7 @@ static struct dentry *kernfs_iop_lookup(struct inode *dir,
 	return d_splice_alias(inode, dentry);
 }
 
-static struct dentry *kernfs_iop_mkdir(struct mnt_idmap *idmap,
+static struct dentry *kernfs_iop_mkdir(const struct mnt_idmap *idmap,
 				       struct inode *dir, struct dentry *dentry,
 				       umode_t mode)
 {
@@ -1326,7 +1325,7 @@ static int kernfs_iop_rmdir(struct inode *dir, struct dentry *dentry)
 	return ret;
 }
 
-static int kernfs_iop_rename(struct mnt_idmap *idmap,
+static int kernfs_iop_rename(const struct mnt_idmap *idmap,
 			     struct inode *old_dir, struct dentry *old_dentry,
 			     struct inode *new_dir, struct dentry *new_dentry,
 			     unsigned int flags)
@@ -1820,14 +1819,20 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
 		     const char *new_name, const struct ns_common *new_ns)
 {
 	struct kernfs_node *old_parent;
+	const char *dup_name = NULL;
+	const char *put_name = NULL;
 	struct kernfs_root *root;
 	const char *old_name;
+	bool reparent;
 	int error;
 
 	/* can't move or rename root */
 	if (!rcu_access_pointer(kn->__parent))
 		return -EINVAL;
 
+	if (new_name)
+		dup_name = kstrdup_const(new_name, GFP_KERNEL);
+
 	root = kernfs_root(kn);
 	down_write(&root->kernfs_rwsem);
 
@@ -1859,9 +1864,10 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
 	/* rename kernfs_node */
 	if (strcmp(old_name, new_name) != 0) {
 		error = -ENOMEM;
-		new_name = kstrdup_const(new_name, GFP_KERNEL);
-		if (!new_name)
+		if (!dup_name)
 			goto out;
+		new_name = dup_name;
+		dup_name = NULL;
 	} else {
 		new_name = NULL;
 	}
@@ -1871,35 +1877,39 @@ int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
 	 */
 	kernfs_unlink_sibling(kn);
 
-	/* rename_lock protects ->parent accessors */
-	if (old_parent != new_parent) {
+	reparent = old_parent != new_parent;
+	if (reparent)
 		kernfs_get(new_parent);
-		write_lock_irq(&root->kernfs_rename_lock);
 
+	/*
+	 * kernfs_rename_lock protects ->__parent, ->ns and ->name, so take it
+	 * even when the parent does not change.
+	 */
+	write_lock_irq(&root->kernfs_rename_lock);
+
+	if (reparent)
 		rcu_assign_pointer(kn->__parent, new_parent);
+	WRITE_ONCE(kn->ns, new_ns);
+	if (new_name)
+		rcu_assign_pointer(kn->name, new_name);
 
-		kn->ns = new_ns;
-		if (new_name)
-			rcu_assign_pointer(kn->name, new_name);
+	write_unlock_irq(&root->kernfs_rename_lock);
 
-		write_unlock_irq(&root->kernfs_rename_lock);
+	if (reparent)
 		kernfs_put(old_parent);
-	} else {
-		/* name assignment is RCU protected, parent is the same */
-		kn->ns = new_ns;
-		if (new_name)
-			rcu_assign_pointer(kn->name, new_name);
-	}
 
 	kn->hash = kernfs_name_hash(new_name ?: old_name, kn->ns);
 	kernfs_link_sibling(kn);
 
 	if (new_name && !is_kernel_rodata((unsigned long)old_name))
-		kfree_rcu_mightsleep(old_name);
+		put_name = old_name;
 
 	error = 0;
  out:
 	up_write(&root->kernfs_rwsem);
+	kfree_const(dup_name);
+	if (put_name)
+		kfree_rcu_mightsleep(put_name);
 	return error;
 }
 
@@ -1909,33 +1919,49 @@ static int kernfs_dir_fop_release(struct inode *inode, struct file *filp)
 	return 0;
 }
 
+/*
+ * Find where a listing left off.  @resumed says whether @pos is still that
+ * entry; if not, the search falls back to @hash, keyed by @name if given.
+ */
 static struct kernfs_node *kernfs_dir_pos(const struct ns_common *ns,
-	struct kernfs_node *parent, loff_t hash, struct kernfs_node *pos)
+	struct kernfs_node *parent, loff_t hash, struct kernfs_node *pos,
+	const char *name, bool *resumed)
 {
+	if (resumed)
+		*resumed = false;
 	if (pos) {
+		/*
+		 * A rename keeps the hash if the new name hashes the same, so
+		 * check @name too.  Otherwise the caller would step over the
+		 * entry now sitting where @pos used to be.
+		 */
 		int valid = kernfs_active(pos) &&
 			rcu_access_pointer(pos->__parent) == parent &&
-			hash == pos->hash;
+			hash == pos->hash &&
+			(!name || !strcmp(name, kernfs_rcu_name(pos)));
 		kernfs_put(pos);
 		if (!valid)
 			pos = NULL;
+		else if (resumed)
+			*resumed = true;
 	}
 	if (!pos && (hash > 1) && (hash < INT_MAX)) {
 		struct rb_node *node = parent->dir.children.rb_node;
-		u64 ns_id = kernfs_ns_id(ns);
+
+		/*
+		 * Keep a node only on the way left, so the search ends on the
+		 * first entry after the key.  An empty @name sorts before all
+		 * entries sharing the hash, so it lands on the first of them.
+		 */
 		while (node) {
-			pos = rb_to_kn(node);
+			struct kernfs_node *kn = rb_to_kn(node);
 
-			if (hash < pos->hash)
+			if (kernfs_name_compare(hash, name ?: "", ns, kn) < 0) {
+				pos = kn;
 				node = node->rb_left;
-			else if (hash > pos->hash)
+			} else {
 				node = node->rb_right;
-			else if (ns_id < kernfs_ns_id(pos->ns))
-				node = node->rb_left;
-			else if (ns_id > kernfs_ns_id(pos->ns))
-				node = node->rb_right;
-			else
-				break;
+			}
 		}
 	}
 	/* Skip over entries which are dying/dead or in the wrong namespace */
@@ -1951,10 +1977,14 @@ static struct kernfs_node *kernfs_dir_pos(const struct ns_common *ns,
 }
 
 static struct kernfs_node *kernfs_dir_next_pos(const struct ns_common *ns,
-	struct kernfs_node *parent, ino_t ino, struct kernfs_node *pos)
+	struct kernfs_node *parent, loff_t hash, struct kernfs_node *pos,
+	const char *name)
 {
-	pos = kernfs_dir_pos(ns, parent, ino, pos);
-	if (pos) {
+	bool resumed;
+
+	pos = kernfs_dir_pos(ns, parent, hash, pos, name, &resumed);
+	/* Step over @pos only if it survived; @name finds the spot if not. */
+	if (pos && resumed) {
 		do {
 			struct rb_node *node = rb_next(&pos->rb);
 			if (!node)
@@ -1972,34 +2002,55 @@ static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx)
 	struct dentry *dentry = file->f_path.dentry;
 	struct kernfs_node *parent = kernfs_dentry_node(dentry);
 	struct kernfs_node *pos = file->private_data;
+	char *name __free(kfree) = NULL;
 	struct kernfs_root *root;
 	const struct ns_common *ns = NULL;
 
 	if (!dir_emit_dots(file, ctx))
 		return 0;
 
+	/*
+	 * One buffer for the call, holding the name of the entry the listing
+	 * is on.  PATH_MAX: kernfs bounds no single name.
+	 */
+	name = kmalloc(PATH_MAX, GFP_KERNEL);
+	if (!name)
+		return -ENOMEM;
+
 	root = kernfs_root(parent);
 	down_read(&root->kernfs_rwsem);
 
 	if (kernfs_ns_enabled(parent))
 		ns = kernfs_info(dentry->d_sb)->ns;
 
-	for (pos = kernfs_dir_pos(ns, parent, ctx->pos, pos);
+	for (pos = kernfs_dir_pos(ns, parent, ctx->pos, pos, NULL, NULL);
 	     pos;
-	     pos = kernfs_dir_next_pos(ns, parent, ctx->pos, pos)) {
-		const char *name = kernfs_rcu_name(pos);
+	     pos = kernfs_dir_next_pos(ns, parent, ctx->pos, pos, name)) {
 		unsigned int type = fs_umode_to_dtype(pos->mode);
-		int len = strlen(name);
 		ino_t ino = kernfs_ino(pos);
+		int len;
+
+		/*
+		 * The copy is also the resume key, so a truncated name would
+		 * resume here again.  getname() caps a path, so only an
+		 * in-kernel caller can get here; end the listing instead.
+		 */
+		len = strscpy(name, kernfs_rcu_name(pos), PATH_MAX);
+		if (WARN_ON_ONCE(len < 0))
+			break;
 
 		ctx->pos = pos->hash;
 		file->private_data = pos;
 		kernfs_get(pos);
 
-		if (!dir_emit(ctx, name, len, ino, type)) {
-			up_read(&root->kernfs_rwsem);
+		/*
+		 * dir_emit() can fault, so run it unlocked.  @pos is pinned
+		 * above and kernfs_dir_pos() rechecks it on the way back.
+		 */
+		up_read(&root->kernfs_rwsem);
+		if (!dir_emit(ctx, name, len, ino, type))
 			return 0;
-		}
+		down_read(&root->kernfs_rwsem);
 	}
 	up_read(&root->kernfs_rwsem);
 	file->private_data = NULL;
diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c
index 8e0e90c933720b..cca9f83fc9b5a9 100644
--- a/fs/kernfs/file.c
+++ b/fs/kernfs/file.c
@@ -525,18 +525,31 @@ static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma)
 static int kernfs_get_open_node(struct kernfs_node *kn,
 				struct kernfs_open_file *of)
 {
-	struct kernfs_open_node *on;
+	struct kernfs_open_node *on, *new_on = NULL;
 	struct mutex *mutex;
 
+	/*
+	 * Peek without the mutex: if nothing has this open, we will need a
+	 * node and can allocate before taking a mutex shared by every node
+	 * hashing to it.
+	 */
+	if (!rcu_access_pointer(kn->attr.open))
+		new_on = kzalloc_obj(*new_on);
+
 	mutex = kernfs_open_file_mutex_lock(kn);
 	on = kernfs_deref_open_node_locked(kn);
 
 	if (!on) {
 		/* not there, initialize a new one */
-		on = kzalloc_obj(*on);
+		on = new_on;
+		new_on = NULL;
 		if (!on) {
-			mutex_unlock(mutex);
-			return -ENOMEM;
+			/* the peek raced; rare, so allocate here */
+			on = kzalloc_obj(*on);
+			if (!on) {
+				mutex_unlock(mutex);
+				return -ENOMEM;
+			}
 		}
 		atomic_set(&on->event, 1);
 		init_waitqueue_head(&on->poll);
@@ -549,6 +562,7 @@ static int kernfs_get_open_node(struct kernfs_node *kn,
 		on->nr_to_release++;
 
 	mutex_unlock(mutex);
+	kfree(new_on);
 	return 0;
 }
 
@@ -904,9 +918,12 @@ static loff_t kernfs_fop_llseek(struct file *file, loff_t offset, int whence)
 
 static void kernfs_notify_workfn(struct work_struct *work)
 {
-	struct kernfs_node *kn;
+	char name_buf[NAME_MAX + 1];
 	struct kernfs_super_info *info;
+	struct kernfs_node *kn;
 	struct kernfs_root *root;
+	struct qstr name;
+	bool have_name;
 repeat:
 	/* pop one off the notify_list */
 	spin_lock_irq(&kernfs_notify_lock);
@@ -922,14 +939,20 @@ static void kernfs_notify_workfn(struct work_struct *work)
 	root = kernfs_root(kn);
 	/* kick fsnotify */
 
+	/*
+	 * Sample the name once so kernfs_rwsem need not be held across the
+	 * loop.  A name that does not fit is reported without one; fsnotify()
+	 * takes the name as optional, so a watcher loses the name and not the
+	 * event.
+	 */
+	have_name = kernfs_name(kn, name_buf, sizeof(name_buf)) >= 0;
+	name = QSTR(name_buf);
+
 	down_read(&root->kernfs_supers_rwsem);
-	down_read(&root->kernfs_rwsem);
-	list_for_each_entry(info, &kernfs_root(kn)->supers, node) {
+	list_for_each_entry(info, &root->supers, node) {
 		struct kernfs_node *parent;
 		struct inode *p_inode = NULL;
-		const char *kn_name;
 		struct inode *inode;
-		struct qstr name;
 
 		/*
 		 * We want fsnotify_modify() on @kn but as the
@@ -941,15 +964,14 @@ static void kernfs_notify_workfn(struct work_struct *work)
 		if (!inode)
 			continue;
 
-		kn_name = kernfs_rcu_name(kn);
-		name = QSTR(kn_name);
 		parent = kernfs_get_parent(kn);
 		if (parent) {
 			p_inode = ilookup(info->sb, kernfs_ino(parent));
 			if (p_inode) {
 				fsnotify(FS_MODIFY | FS_EVENT_ON_CHILD,
 					 inode, FSNOTIFY_EVENT_INODE,
-					 p_inode, &name, inode, 0);
+					 p_inode, have_name ? &name : NULL,
+					 inode, 0);
 				iput(p_inode);
 			}
 
@@ -962,7 +984,6 @@ static void kernfs_notify_workfn(struct work_struct *work)
 		iput(inode);
 	}
 
-	up_read(&root->kernfs_rwsem);
 	up_read(&root->kernfs_supers_rwsem);
 	kernfs_put(kn);
 	goto repeat;
diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c
index abb286bc3474b7..6630b29d7c078f 100644
--- a/fs/kernfs/inode.c
+++ b/fs/kernfs/inode.c
@@ -107,7 +107,7 @@ int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr)
 	return ret;
 }
 
-int kernfs_iop_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int kernfs_iop_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -179,7 +179,7 @@ static void kernfs_refresh_inode(struct kernfs_node *kn, struct inode *inode)
 		set_nlink(inode, kn->dir.subdirs + 2);
 }
 
-int kernfs_iop_getattr(struct mnt_idmap *idmap,
+int kernfs_iop_getattr(const struct mnt_idmap *idmap,
 		       const struct path *path, struct kstat *stat,
 		       u32 request_mask, unsigned int query_flags)
 {
@@ -270,7 +270,7 @@ void kernfs_evict_inode(struct inode *inode)
 	kernfs_put(kn);
 }
 
-int kernfs_iop_permission(struct mnt_idmap *idmap,
+int kernfs_iop_permission(const struct mnt_idmap *idmap,
 			  struct inode *inode, int mask)
 {
 	struct kernfs_node *kn;
@@ -342,7 +342,7 @@ static int kernfs_vfs_xattr_get(const struct xattr_handler *handler,
 }
 
 static int kernfs_vfs_xattr_set(const struct xattr_handler *handler,
-				struct mnt_idmap *idmap,
+				const struct mnt_idmap *idmap,
 				struct dentry *unused, struct inode *inode,
 				const char *suffix, const void *value,
 				size_t size, int flags)
@@ -354,7 +354,7 @@ static int kernfs_vfs_xattr_set(const struct xattr_handler *handler,
 }
 
 static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler,
-				     struct mnt_idmap *idmap,
+				     const struct mnt_idmap *idmap,
 				     struct dentry *unused, struct inode *inode,
 				     const char *suffix, const void *value,
 				     size_t size, int flags)
diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h
index aa784b540b3616..f1e93b09e27df8 100644
--- a/fs/kernfs/kernfs-internal.h
+++ b/fs/kernfs/kernfs-internal.h
@@ -117,7 +117,14 @@ static inline bool kernfs_rename_is_locked(const struct kernfs_node *kn)
 
 static inline const char *kernfs_rcu_name(const struct kernfs_node *kn)
 {
-	return rcu_dereference_check(kn->name, kernfs_root_is_locked(kn));
+	/*
+	 * Like kernfs_node::__parent below, the name is only replaced under
+	 * both kernfs_root::kernfs_rwsem and kernfs_root::kernfs_rename_lock,
+	 * so either one keeps it, and the string it points at, stable.
+	 */
+	return rcu_dereference_check(kn->name,
+				     kernfs_root_is_locked(kn) ||
+				     kernfs_rename_is_locked(kn));
 }
 
 static inline struct kernfs_node *kernfs_parent(const struct kernfs_node *kn)
@@ -147,20 +154,19 @@ static inline struct kernfs_node *kernfs_dentry_node(struct dentry *dentry)
 static inline void kernfs_set_rev(struct kernfs_node *parent,
 				  struct dentry *dentry)
 {
-	dentry->d_time = parent->dir.rev;
+	WRITE_ONCE(dentry->d_time, READ_ONCE(parent->dir.rev));
 }
 
 static inline void kernfs_inc_rev(struct kernfs_node *parent)
 {
-	parent->dir.rev++;
+	lockdep_assert_held_write(&parent->dir.root->kernfs_rwsem);
+	WRITE_ONCE(parent->dir.rev, parent->dir.rev + 1);
 }
 
 static inline bool kernfs_dir_changed(struct kernfs_node *parent,
 				      struct dentry *dentry)
 {
-	if (parent->dir.rev != dentry->d_time)
-		return true;
-	return false;
+	return READ_ONCE(parent->dir.rev) != READ_ONCE(dentry->d_time);
 }
 
 extern const struct super_operations kernfs_sops;
@@ -171,11 +177,11 @@ extern struct kmem_cache *kernfs_node_cache, *kernfs_iattrs_cache;
  */
 extern const struct xattr_handler * const kernfs_xattr_handlers[];
 void kernfs_evict_inode(struct inode *inode);
-int kernfs_iop_permission(struct mnt_idmap *idmap,
+int kernfs_iop_permission(const struct mnt_idmap *idmap,
 			  struct inode *inode, int mask);
-int kernfs_iop_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int kernfs_iop_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct iattr *iattr);
-int kernfs_iop_getattr(struct mnt_idmap *idmap,
+int kernfs_iop_getattr(const struct mnt_idmap *idmap,
 		       const struct path *path, struct kstat *stat,
 		       u32 request_mask, unsigned int query_flags);
 ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size);
diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c
index f183a96778b9a2..a0be784bfb0646 100644
--- a/fs/kernfs/mount.c
+++ b/fs/kernfs/mount.c
@@ -124,22 +124,32 @@ static struct dentry *__kernfs_fh_to_dentry(struct super_block *sb,
 		return NULL;
 	}
 
-	kn = kernfs_find_and_get_node_by_id(info->root, id);
-	if (!kn)
-		return ERR_PTR(-ESTALE);
+	/*
+	 * Hold kernfs_rwsem across the lookup as well as kernfs_get_inode().
+	 * __kernfs_remove() deactivates the subtree and clears i_nlink on its
+	 * inodes under the write lock, so under the read lock either
+	 * kernfs_find_and_get_node_by_id() refuses the node, or the inode is
+	 * in the inode hash before the ilookup() pass goes looking for it.
+	 */
+	scoped_guard(rwsem_read, &info->root->kernfs_rwsem) {
+		kn = kernfs_find_and_get_node_by_id(info->root, id);
+		if (!kn)
+			return ERR_PTR(-ESTALE);
 
-	if (get_parent) {
-		struct kernfs_node *parent;
+		if (get_parent) {
+			struct kernfs_node *parent;
 
-		parent = kernfs_get_parent(kn);
+			parent = kernfs_get_parent(kn);
+			kernfs_put(kn);
+			kn = parent;
+			if (!kn)
+				return ERR_PTR(-ESTALE);
+		}
+
+		inode = kernfs_get_inode(sb, kn);
 		kernfs_put(kn);
-		kn = parent;
-		if (!kn)
-			return ERR_PTR(-ESTALE);
 	}
 
-	inode = kernfs_get_inode(sb, kn);
-	kernfs_put(kn);
 	return d_obtain_alias(inode);
 }
 
@@ -434,8 +444,8 @@ void kernfs_kill_sb(struct super_block *sb)
 	up_write(&root->kernfs_supers_rwsem);
 
 	/*
-	 * Remove the superblock from fs_supers/s_instances
-	 * so we can't find it, before freeing kernfs_super_info.
+	 * Mark the superblock dead so sget_fc() can't find it,
+	 * before freeing kernfs_super_info.
 	 */
 	kill_anon_super(sb);
 	kfree(info);
diff --git a/fs/kernfs/symlink.c b/fs/kernfs/symlink.c
index 90e2b3221b83d8..3e53105d3abf14 100644
--- a/fs/kernfs/symlink.c
+++ b/fs/kernfs/symlink.c
@@ -31,9 +31,20 @@ struct kernfs_node *kernfs_create_link(struct kernfs_node *parent,
 	kuid_t uid = GLOBAL_ROOT_UID;
 	kgid_t gid = GLOBAL_ROOT_GID;
 
-	if (target->iattr) {
-		uid = target->iattr->ia_uid;
-		gid = target->iattr->ia_gid;
+	/*
+	 * A symlink takes its owner from its target, so both fields have to
+	 * come from the same moment: read them under kernfs_iattr_rwsem, or
+	 * a chown of the target racing this could leave the link with the
+	 * old uid and the new gid.  The section ends before kernfs_add_one()
+	 * takes kernfs_rwsem.
+	 */
+	scoped_guard(rwsem_read, &kernfs_root(target)->kernfs_iattr_rwsem) {
+		struct kernfs_iattrs *attrs = READ_ONCE(target->iattr);
+
+		if (attrs) {
+			uid = attrs->ia_uid;
+			gid = attrs->ia_gid;
+		}
 	}
 
 	kn = kernfs_new_node(parent, name, S_IFLNK|0777, uid, gid, KERNFS_LINK);
diff --git a/fs/libfs.c b/fs/libfs.c
index 27d7dc16fcb0a0..8e2cc627bc7f41 100644
--- a/fs/libfs.c
+++ b/fs/libfs.c
@@ -29,7 +29,7 @@
 
 #include "internal.h"
 
-int simple_getattr(struct mnt_idmap *idmap, const struct path *path,
+int simple_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		   struct kstat *stat, u32 request_mask,
 		   unsigned int query_flags)
 {
@@ -867,7 +867,7 @@ int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
 }
 EXPORT_SYMBOL_GPL(simple_rename_exchange);
 
-int simple_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+int simple_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		  struct dentry *old_dentry, struct inode *new_dir,
 		  struct dentry *new_dentry, unsigned int flags)
 {
@@ -913,7 +913,7 @@ EXPORT_SYMBOL(simple_rename);
  * on simple regular filesystems.  Anything that needs to change on-disk
  * or wire state on size changes needs its own setattr method.
  */
-int simple_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int simple_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -1727,7 +1727,7 @@ static struct dentry *empty_dir_lookup(struct inode *dir, struct dentry *dentry,
 	return ERR_PTR(-ENOENT);
 }
 
-static int empty_dir_setattr(struct mnt_idmap *idmap,
+static int empty_dir_setattr(const struct mnt_idmap *idmap,
 			     struct dentry *dentry, struct iattr *attr)
 {
 	return -EPERM;
diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c
index ee90e743064afb..f0e1a58c910666 100644
--- a/fs/lockd/svc.c
+++ b/fs/lockd/svc.c
@@ -36,7 +36,6 @@
 #include <net/ip.h>
 #include <net/addrconf.h>
 #include <net/ipv6.h>
-#include <linux/nfs.h>
 
 #include "lockd.h"
 #include "netns.h"
diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c
index e628b5d355071f..495eacb3264f90 100644
--- a/fs/lockd/svclock.c
+++ b/fs/lockd/svclock.c
@@ -295,14 +295,17 @@ void nlmsvc_traverse_blocks(struct nlm_host *host,
 	list_for_each_entry_safe(block, next, &file->f_blocks, b_flist) {
 		if (!match(block->b_host, host))
 			continue;
-		/* Do not destroy blocks that are not on
-		 * the global retry list - why? */
+		/*
+		 * nlmsvc_retry_blocked() holds f_mutex while the block
+		 * is off nlm_blocked, so a block off the list here has
+		 * been retired.
+		 */
 		if (list_empty(&block->b_list))
 			continue;
 		kref_get(&block->b_count);
 		spin_unlock(&nlm_blocked_lock);
-		mutex_unlock(&file->f_mutex);
 		nlmsvc_unlink_block(block);
+		mutex_unlock(&file->f_mutex);
 		nlmsvc_release_block(block);
 		goto restart;
 	}
@@ -1012,6 +1015,8 @@ nlmsvc_retry_blocked(struct svc_rqst *rqstp)
 {
 	unsigned long	timeout = MAX_SCHEDULE_TIMEOUT;
 	struct nlm_block *block;
+	struct nlm_file *file;
+	bool due;
 
 	spin_lock(&nlm_blocked_lock);
 	while (!list_empty(&nlm_blocked) && !svc_thread_should_stop(rqstp)) {
@@ -1023,8 +1028,28 @@ nlmsvc_retry_blocked(struct svc_rqst *rqstp)
 			timeout = block->b_when - jiffies;
 			break;
 		}
+		kref_get(&block->b_count);
 		spin_unlock(&nlm_blocked_lock);
 
+		/*
+		 * Hold f_mutex so nlmsvc_traverse_blocks() cannot scan
+		 * the file while the retry has the block off nlm_blocked.
+		 */
+		file = block->b_file;
+		mutex_lock(&file->f_mutex);
+		spin_lock(&nlm_blocked_lock);
+		due = !list_empty(&block->b_list) &&
+		      block->b_when != NLM_NEVER &&
+		      !time_after(block->b_when, jiffies);
+		spin_unlock(&nlm_blocked_lock);
+
+		if (!due) {
+			mutex_unlock(&file->f_mutex);
+			nlmsvc_release_block(block);
+			spin_lock(&nlm_blocked_lock);
+			continue;
+		}
+
 		dprintk("nlmsvc_retry_blocked(%p, when=%ld)\n",
 			block, block->b_when);
 		if (block->b_flags & B_QUEUED) {
@@ -1033,6 +1058,8 @@ nlmsvc_retry_blocked(struct svc_rqst *rqstp)
 			retry_deferred_block(block);
 		} else
 			nlmsvc_grant_blocked(block);
+		mutex_unlock(&file->f_mutex);
+		nlmsvc_release_block(block);
 		spin_lock(&nlm_blocked_lock);
 	}
 	spin_unlock(&nlm_blocked_lock);
diff --git a/fs/lockd/trace.h b/fs/lockd/trace.h
index a11d04e8c835eb..1f79955ea0f5f6 100644
--- a/fs/lockd/trace.h
+++ b/fs/lockd/trace.h
@@ -7,7 +7,6 @@
 
 #include <linux/tracepoint.h>
 #include <linux/crc32.h>
-#include <linux/nfs.h>
 
 #include "lockd.h"
 
diff --git a/fs/lockd/xdr.h b/fs/lockd/xdr.h
index a1126cca98c6ce..56b9796aa39d96 100644
--- a/fs/lockd/xdr.h
+++ b/fs/lockd/xdr.h
@@ -10,7 +10,7 @@
 
 #include <linux/fs.h>
 #include <linux/filelock.h>
-#include <linux/nfs.h>
+#include <linux/nfs_fh.h>
 #include <linux/sunrpc/xdr.h>
 
 #define SM_MAXSTRLEN		1024
diff --git a/fs/minix/file.c b/fs/minix/file.c
index 02aabbdb5deaa0..c0929fc38fbcba 100644
--- a/fs/minix/file.c
+++ b/fs/minix/file.c
@@ -23,7 +23,7 @@ const struct file_operations minix_file_operations = {
 	.splice_read	= filemap_splice_read,
 };
 
-static int minix_setattr(struct mnt_idmap *idmap,
+static int minix_setattr(const struct mnt_idmap *idmap,
 			 struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/minix/inode.c b/fs/minix/inode.c
index daf83e4ff25c81..670179173645b3 100644
--- a/fs/minix/inode.c
+++ b/fs/minix/inode.c
@@ -724,7 +724,7 @@ static int minix_sync_inode_metadata(struct inode *inode,
 	return err;
 }
 
-int minix_getattr(struct mnt_idmap *idmap, const struct path *path,
+int minix_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	struct super_block *sb = path->dentry->d_sb;
diff --git a/fs/minix/minix.h b/fs/minix/minix.h
index 78722ce22e1e19..db92cf9e0e1bf3 100644
--- a/fs/minix/minix.h
+++ b/fs/minix/minix.h
@@ -55,7 +55,7 @@ unsigned long minix_count_free_inodes(struct super_block *sb);
 int minix_new_block(struct inode *inode);
 void minix_free_block(struct inode *inode, unsigned long block);
 unsigned long minix_count_free_blocks(struct super_block *sb);
-int minix_getattr(struct mnt_idmap *, const struct path *,
+int minix_getattr(const struct mnt_idmap *, const struct path *,
 		struct kstat *, u32, unsigned int);
 int minix_prepare_chunk(struct folio *folio, loff_t pos, unsigned len);
 struct mapping_metadata_bhs *minix_get_metadata_bhs(struct inode *inode);
diff --git a/fs/minix/namei.c b/fs/minix/namei.c
index 5525ba367ed749..f450b11b98604a 100644
--- a/fs/minix/namei.c
+++ b/fs/minix/namei.c
@@ -33,7 +33,7 @@ static struct dentry *minix_lookup(struct inode * dir, struct dentry *dentry, un
 	return d_splice_alias(inode, dentry);
 }
 
-static int minix_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int minix_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct inode *inode;
@@ -50,7 +50,7 @@ static int minix_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return add_nondir(dentry, inode);
 }
 
-static int minix_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int minix_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct file *file, umode_t mode)
 {
 	struct inode *inode = minix_new_inode(dir, mode);
@@ -63,13 +63,13 @@ static int minix_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	return finish_open_simple(file, 0);
 }
 
-static int minix_create(struct mnt_idmap *idmap, struct inode *dir,
+static int minix_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	return minix_mknod(&nop_mnt_idmap, dir, dentry, mode, 0);
 }
 
-static int minix_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int minix_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, const char *symname)
 {
 	int i = strlen(symname)+1;
@@ -104,7 +104,7 @@ static int minix_link(struct dentry * old_dentry, struct inode * dir,
 	return add_nondir(dentry, inode);
 }
 
-static struct dentry *minix_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *minix_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct inode * inode;
@@ -187,7 +187,7 @@ static int minix_rmdir(struct inode * dir, struct dentry *dentry)
 	return err;
 }
 
-static int minix_rename(struct mnt_idmap *idmap,
+static int minix_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
diff --git a/fs/mnt_idmapping.c b/fs/mnt_idmapping.c
index cb61fbdb52e906..bed57094cef00a 100644
--- a/fs/mnt_idmapping.c
+++ b/fs/mnt_idmapping.c
@@ -28,7 +28,7 @@ struct mnt_idmap {
  * mapping. This means that {g,u}id 0 is mapped to {g,u}id 0, {g,u}id 1 is
  * mapped to {g,u}id 1, [...], {g,u}id 1000 to {g,u}id 1000, [...].
  */
-struct mnt_idmap nop_mnt_idmap = {
+const struct mnt_idmap nop_mnt_idmap = {
 	.count	= REFCOUNT_INIT(1),
 };
 EXPORT_SYMBOL_GPL(nop_mnt_idmap);
@@ -37,7 +37,7 @@ EXPORT_SYMBOL_GPL(nop_mnt_idmap);
  * Carries the invalid idmapping of a full 0-4294967295 {g,u}id range.
  * This means that all {g,u}ids are mapped to INVALID_VFS{G,U}ID.
  */
-struct mnt_idmap invalid_mnt_idmap = {
+const struct mnt_idmap invalid_mnt_idmap = {
 	.count	= REFCOUNT_INIT(1),
 };
 EXPORT_SYMBOL_GPL(invalid_mnt_idmap);
@@ -77,7 +77,7 @@ static inline bool initial_idmapping(const struct user_namespace *ns)
  * returned.
  */
 
-vfsuid_t make_vfsuid(struct mnt_idmap *idmap,
+vfsuid_t make_vfsuid(const struct mnt_idmap *idmap,
 		     struct user_namespace *fs_userns,
 		     kuid_t kuid)
 {
@@ -117,7 +117,7 @@ EXPORT_SYMBOL_GPL(make_vfsuid);
  * If @kgid has no mapping in either @idmap or @fs_userns INVALID_GID is
  * returned.
  */
-vfsgid_t make_vfsgid(struct mnt_idmap *idmap,
+vfsgid_t make_vfsgid(const struct mnt_idmap *idmap,
 		     struct user_namespace *fs_userns, kgid_t kgid)
 {
 	gid_t gid;
@@ -147,7 +147,7 @@ EXPORT_SYMBOL_GPL(make_vfsgid);
  *
  * Return: @vfsuid mapped into the filesystem idmapping
  */
-kuid_t from_vfsuid(struct mnt_idmap *idmap,
+kuid_t from_vfsuid(const struct mnt_idmap *idmap,
 		   struct user_namespace *fs_userns, vfsuid_t vfsuid)
 {
 	uid_t uid;
@@ -176,7 +176,7 @@ EXPORT_SYMBOL_GPL(from_vfsuid);
  *
  * Return: @vfsgid mapped into the filesystem idmapping
  */
-kgid_t from_vfsgid(struct mnt_idmap *idmap,
+kgid_t from_vfsgid(const struct mnt_idmap *idmap,
 		   struct user_namespace *fs_userns, vfsgid_t vfsgid)
 {
 	gid_t gid;
@@ -312,10 +312,12 @@ struct mnt_idmap *alloc_mnt_idmap(struct user_namespace *mnt_userns)
  *
  * Return: @idmap with reference count bumped if @not_mnt_idmap isn't passed.
  */
-struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap)
+const struct mnt_idmap *mnt_idmap_get(const struct mnt_idmap *idmap)
 {
+	struct mnt_idmap *nonconst_idmap = (struct mnt_idmap *)idmap;
+
 	if (idmap != &nop_mnt_idmap && idmap != &invalid_mnt_idmap)
-		refcount_inc(&idmap->count);
+		refcount_inc(&nonconst_idmap->count);
 
 	return idmap;
 }
@@ -328,17 +330,20 @@ EXPORT_SYMBOL_GPL(mnt_idmap_get);
  * If this is a non-initial idmapping, put the reference count when a mount is
  * released and free it if we're the last user.
  */
-void mnt_idmap_put(struct mnt_idmap *idmap)
+void mnt_idmap_put(const struct mnt_idmap *idmap)
 {
+	struct mnt_idmap *nonconst_idmap = (struct mnt_idmap *)idmap;
+
 	if (idmap != &nop_mnt_idmap && idmap != &invalid_mnt_idmap &&
-	    refcount_dec_and_test(&idmap->count))
-		free_mnt_idmap(idmap);
+	    refcount_dec_and_test(&nonconst_idmap->count))
+		free_mnt_idmap(nonconst_idmap);
 }
 EXPORT_SYMBOL_GPL(mnt_idmap_put);
 
-int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_map)
+int statmount_mnt_idmap(const struct mnt_idmap *idmap, struct seq_file *seq, bool uid_map)
 {
-	struct uid_gid_map *map, *map_up;
+	const struct uid_gid_map *map;
+	struct uid_gid_map *map_up;
 	u32 idx, nr_mappings;
 
 	if (!is_valid_mnt_idmap(idmap))
@@ -358,7 +363,7 @@ int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_
 
 	for (idx = 0, nr_mappings = 0; idx < map->nr_extents; idx++) {
 		uid_t lower;
-		struct uid_gid_extent *extent;
+		const struct uid_gid_extent *extent;
 
 		if (map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
 			extent = &map->extent[idx];
diff --git a/fs/mount.h b/fs/mount.h
index 94fcc306d21e7d..85f136786bbcc9 100644
--- a/fs/mount.h
+++ b/fs/mount.h
@@ -33,7 +33,8 @@ struct mnt_namespace {
 } __randomize_layout;
 
 struct mnt_pcp {
-	int mnt_count;
+	unsigned int mnt_gets;
+	unsigned int mnt_puts;
 	int mnt_writers;
 };
 
diff --git a/fs/namei.c b/fs/namei.c
index 20a6534ea3efff..54ad61923548dc 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -371,7 +371,7 @@ struct filename *complete_getname(struct delayed_filename *v)
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-static int check_acl(struct mnt_idmap *idmap,
+static int check_acl(const struct mnt_idmap *idmap,
 		     struct inode *inode, int mask)
 {
 #ifdef CONFIG_FS_POSIX_ACL
@@ -435,7 +435,7 @@ static inline bool no_acl_inode(struct inode *inode)
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-static int acl_permission_check(struct mnt_idmap *idmap,
+static int acl_permission_check(const struct mnt_idmap *idmap,
 				struct inode *inode, int mask)
 {
 	unsigned int mode = inode->i_mode;
@@ -518,7 +518,7 @@ static int acl_permission_check(struct mnt_idmap *idmap,
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int generic_permission(struct mnt_idmap *idmap, struct inode *inode,
+int generic_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		       int mask)
 {
 	int ret;
@@ -575,7 +575,7 @@ EXPORT_SYMBOL(generic_permission);
  * flag in inode->i_opflags, that says "this has not special
  * permission function, use the fast case".
  */
-static inline int do_inode_permission(struct mnt_idmap *idmap,
+static inline int do_inode_permission(const struct mnt_idmap *idmap,
 				      struct inode *inode, int mask)
 {
 	if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) {
@@ -625,7 +625,7 @@ static int sb_permission(struct super_block *sb, struct inode *inode, int mask)
  *
  * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask.
  */
-int inode_permission(struct mnt_idmap *idmap,
+int inode_permission(const struct mnt_idmap *idmap,
 		     struct inode *inode, int mask)
 {
 	int retval;
@@ -658,7 +658,7 @@ int inode_permission(struct mnt_idmap *idmap,
 	if (unlikely(retval))
 		return retval;
 
-	return security_inode_permission(inode, mask);
+	return security_inode_permission(idmap, inode, mask);
 }
 EXPORT_SYMBOL(inode_permission);
 
@@ -680,7 +680,7 @@ EXPORT_SYMBOL(inode_permission);
  * on IOP_FASTPERM can still get the optimization if they set IOP_FASTPERM_MAY_EXEC
  * on their directory inodes.
  */
-static __always_inline int lookup_inode_permission_may_exec(struct mnt_idmap *idmap,
+static __always_inline int lookup_inode_permission_may_exec(const struct mnt_idmap *idmap,
 	struct inode *inode, int mask)
 {
 	/* Lookup already checked this to return -ENOTDIR */
@@ -695,7 +695,7 @@ static __always_inline int lookup_inode_permission_may_exec(struct mnt_idmap *id
 	if (unlikely(((inode->i_mode & 0111) != 0111) || !no_acl_inode(inode)))
 		return inode_permission(idmap, inode, mask);
 
-	return security_inode_permission(inode, mask);
+	return security_inode_permission(idmap, inode, mask);
 }
 
 /**
@@ -1273,7 +1273,7 @@ fs_initcall(init_fs_namei_sysctls);
  */
 static inline int may_follow_link(struct nameidata *nd, const struct inode *inode)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	vfsuid_t vfsuid;
 
 	if (!sysctl_protected_symlinks)
@@ -1314,7 +1314,7 @@ static inline int may_follow_link(struct nameidata *nd, const struct inode *inod
  *
  * Otherwise returns true.
  */
-static bool safe_hardlink_source(struct mnt_idmap *idmap,
+static bool safe_hardlink_source(const struct mnt_idmap *idmap,
 				 struct inode *inode)
 {
 	umode_t mode = inode->i_mode;
@@ -1357,7 +1357,7 @@ static bool safe_hardlink_source(struct mnt_idmap *idmap,
  *
  * Returns 0 if successful, -ve on error.
  */
-int may_linkat(struct mnt_idmap *idmap, const struct path *link)
+int may_linkat(const struct mnt_idmap *idmap, const struct path *link)
 {
 	struct inode *inode = link->dentry->d_inode;
 
@@ -1407,7 +1407,7 @@ int may_linkat(struct mnt_idmap *idmap, const struct path *link)
  *
  * Returns 0 if the open is allowed, -ve on error.
  */
-static int may_create_in_sticky(struct mnt_idmap *idmap, struct nameidata *nd,
+static int may_create_in_sticky(const struct mnt_idmap *idmap, struct nameidata *nd,
 				struct inode *const inode)
 {
 	umode_t dir_mode = nd->dir_mode;
@@ -1933,7 +1933,7 @@ static noinline struct dentry *lookup_slow(const struct qstr *name,
 	struct inode *inode = dir->d_inode;
 	struct dentry *res;
 	inode_lock_shared(inode);
-	res = __lookup_slow(name, dir, flags);
+	res = __lookup_slow(name, dir, flags | LOOKUP_SHARED);
 	inode_unlock_shared(inode);
 	return res;
 }
@@ -1947,12 +1947,12 @@ static struct dentry *lookup_slow_killable(const struct qstr *name,
 
 	if (inode_lock_shared_killable(inode))
 		return ERR_PTR(-EINTR);
-	res = __lookup_slow(name, dir, flags);
+	res = __lookup_slow(name, dir, flags | LOOKUP_SHARED);
 	inode_unlock_shared(inode);
 	return res;
 }
 
-static inline int may_lookup(struct mnt_idmap *idmap,
+static inline int may_lookup(const struct mnt_idmap *idmap,
 			     struct nameidata *restrict nd)
 {
 	int err, mask;
@@ -2596,7 +2596,7 @@ static int link_path_walk(const char *name, struct nameidata *nd)
 
 	/* At this point we know we have a real path component. */
 	for(;;) {
-		struct mnt_idmap *idmap;
+		const struct mnt_idmap *idmap;
 		const char *link;
 		unsigned long lastword;
 
@@ -2946,8 +2946,8 @@ struct dentry *start_dirop(struct dentry *parent, struct qstr *name,
  * end_dirop - signal completion of a dirop
  * @de: the dentry which was returned by start_dirop or similar.
  *
- * If the de is an error, nothing happens. Otherwise any lock taken to
- * protect the dentry is dropped and the dentry itself is release (dput()).
+ * If the @de is an error, nothing happens. Otherwise any lock taken to
+ * protect the dentry is dropped and the dentry itself is released (dput()).
  */
 void end_dirop(struct dentry *de)
 {
@@ -3111,7 +3111,7 @@ int lookup_noperm_common(struct qstr *qname, struct dentry *base)
 	return 0;
 }
 
-static int lookup_one_common(struct mnt_idmap *idmap,
+static int lookup_one_common(const struct mnt_idmap *idmap,
 			     struct qstr *qname, struct dentry *base)
 {
 	int err;
@@ -3190,7 +3190,7 @@ EXPORT_SYMBOL(lookup_noperm);
  *
  * The caller must hold base->i_rwsem.
  */
-struct dentry *lookup_one(struct mnt_idmap *idmap, struct qstr *name,
+struct dentry *lookup_one(const struct mnt_idmap *idmap, struct qstr *name,
 			  struct dentry *base)
 {
 	struct dentry *dentry;
@@ -3210,7 +3210,7 @@ EXPORT_SYMBOL(lookup_one);
 /**
  * lookup_one_unlocked - lookup single pathname component
  * @idmap:	idmap of the mount the lookup is performed from
- * @name:	qstr olding pathname component to lookup
+ * @name:	qstr holding pathname component to lookup
  * @base:	base directory to lookup from
  *
  * This can be used for in-kernel filesystem clients such as file servers.
@@ -3223,7 +3223,7 @@ EXPORT_SYMBOL(lookup_one);
  *	    - ERR_PTR(-ENOENT) if parent has been removed, or
  *	    - ERR_PTR(-EACCES) if parent directory is not searchable.
  */
-struct dentry *lookup_one_unlocked(struct mnt_idmap *idmap, struct qstr *name,
+struct dentry *lookup_one_unlocked(const struct mnt_idmap *idmap, struct qstr *name,
 				   struct dentry *base)
 {
 	int err;
@@ -3243,7 +3243,7 @@ EXPORT_SYMBOL(lookup_one_unlocked);
 /**
  * lookup_one_positive_killable - lookup single pathname component
  * @idmap:	idmap of the mount the lookup is performed from
- * @name:	qstr olding pathname component to lookup
+ * @name:	qstr holding pathname component to lookup
  * @base:	base directory to lookup from
  *
  * This helper will yield ERR_PTR(-ENOENT) on negatives. The helper returns
@@ -3259,11 +3259,11 @@ EXPORT_SYMBOL(lookup_one_unlocked);
  * the i_rwsem itself if necessary.  If a fatal signal is pending or
  * delivered, it will return %-EINTR if the lock is needed.
  *
- * Returns: A dentry, possibly negative, or
+ * Returns: A positive dentry, or
  *	   - same errors as lookup_one_unlocked() or
  *	   - ERR_PTR(-EINTR) if a fatal signal is pending.
  */
-struct dentry *lookup_one_positive_killable(struct mnt_idmap *idmap,
+struct dentry *lookup_one_positive_killable(const struct mnt_idmap *idmap,
 					    struct qstr *name,
 					    struct dentry *base)
 {
@@ -3306,7 +3306,7 @@ EXPORT_SYMBOL(lookup_one_positive_killable);
  *	   - ERR_PTR(-ENOENT) if the name could not be found, or
  *	   - same errors as lookup_one_unlocked().
  */
-struct dentry *lookup_one_positive_unlocked(struct mnt_idmap *idmap,
+struct dentry *lookup_one_positive_unlocked(const struct mnt_idmap *idmap,
 					    struct qstr *name,
 					    struct dentry *base)
 {
@@ -3381,7 +3381,7 @@ struct dentry *lookup_noperm_positive_unlocked(struct qstr *name,
 EXPORT_SYMBOL(lookup_noperm_positive_unlocked);
 
 /**
- * start_creating - prepare to create a given name with permission checking
+ * start_creating - prepare to access or create a given name with permission checking
  * @idmap:  idmap of the mount
  * @parent: directory in which to prepare to create the name
  * @name:   the name to be created
@@ -3396,7 +3396,7 @@ EXPORT_SYMBOL(lookup_noperm_positive_unlocked);
  *
  * Returns: a negative or positive dentry, or an error.
  */
-struct dentry *start_creating(struct mnt_idmap *idmap, struct dentry *parent,
+struct dentry *start_creating(const struct mnt_idmap *idmap, struct dentry *parent,
 			      struct qstr *name)
 {
 	int err = lookup_one_common(idmap, name, parent);
@@ -3413,8 +3413,8 @@ EXPORT_SYMBOL(start_creating);
  * @parent: directory in which to find the name
  * @name:   the name to be removed
  *
- * Locks are taken and a lookup in performed prior to removing
- * an object from a directory.  Permission checking (MAY_EXEC) is performed
+ * Locks are taken and a lookup is performed prior to removing an object
+ * from a directory.  Permission checking (MAY_EXEC) is performed
  * against @idmap.
  *
  * If the name doesn't exist, an error is returned.
@@ -3423,7 +3423,7 @@ EXPORT_SYMBOL(start_creating);
  *
  * Returns: a positive dentry, or an error.
  */
-struct dentry *start_removing(struct mnt_idmap *idmap, struct dentry *parent,
+struct dentry *start_removing(const struct mnt_idmap *idmap, struct dentry *parent,
 			      struct qstr *name)
 {
 	int err = lookup_one_common(idmap, name, parent);
@@ -3440,7 +3440,7 @@ EXPORT_SYMBOL(start_removing);
  * @parent: directory in which to prepare to create the name
  * @name:   the name to be created
  *
- * Locks are taken and a lookup in performed prior to creating
+ * Locks are taken and a lookup is performed prior to creating
  * an object in a directory.  Permission checking (MAY_EXEC) is performed
  * against @idmap.
  *
@@ -3451,7 +3451,7 @@ EXPORT_SYMBOL(start_removing);
  *
  * Returns: a negative or positive dentry, or an error.
  */
-struct dentry *start_creating_killable(struct mnt_idmap *idmap,
+struct dentry *start_creating_killable(const struct mnt_idmap *idmap,
 				       struct dentry *parent,
 				       struct qstr *name)
 {
@@ -3469,7 +3469,7 @@ EXPORT_SYMBOL(start_creating_killable);
  * @parent: directory in which to find the name
  * @name:   the name to be removed
  *
- * Locks are taken and a lookup in performed prior to removing
+ * Locks are taken and a lookup is performed prior to removing
  * an object from a directory.  Permission checking (MAY_EXEC) is performed
  * against @idmap.
  *
@@ -3482,7 +3482,7 @@ EXPORT_SYMBOL(start_creating_killable);
  *
  * Returns: a positive dentry, or an error.
  */
-struct dentry *start_removing_killable(struct mnt_idmap *idmap,
+struct dentry *start_removing_killable(const struct mnt_idmap *idmap,
 				       struct dentry *parent,
 				       struct qstr *name)
 {
@@ -3499,7 +3499,7 @@ EXPORT_SYMBOL(start_removing_killable);
  * @parent: directory in which to prepare to create the name
  * @name:   the name to be created
  *
- * Locks are taken and a lookup in performed prior to creating
+ * Locks are taken and a lookup is performed prior to creating
  * an object in a directory.
  *
  * If the name already exists, a positive dentry is returned.
@@ -3522,7 +3522,7 @@ EXPORT_SYMBOL(start_creating_noperm);
  * @parent: directory in which to find the name
  * @name:   the name to be removed
  *
- * Locks are taken and a lookup in performed prior to removing
+ * Locks are taken and a lookup is performed prior to removing
  * an object from a directory.
  *
  * If the name doesn't exist, an error is returned.
@@ -3543,11 +3543,11 @@ struct dentry *start_removing_noperm(struct dentry *parent,
 EXPORT_SYMBOL(start_removing_noperm);
 
 /**
- * start_creating_dentry - prepare to create a given dentry
- * @parent: directory from which dentry should be removed
- * @child:  the dentry to be removed
+ * start_creating_dentry - prepare to access or create a given dentry
+ * @parent: directory of dentry
+ * @child:  the dentry to be prepared
  *
- * A lock is taken to protect the dentry again other dirops and
+ * A lock is taken to protect the dentry against other dirops and
  * the validity of the dentry is checked: correct parent and still hashed.
  *
  * If the dentry is valid and negative a reference is taken and
@@ -3580,7 +3580,7 @@ EXPORT_SYMBOL(start_creating_dentry);
  * @parent: directory from which dentry should be removed
  * @child:  the dentry to be removed
  *
- * A lock is taken to protect the dentry again other dirops and
+ * A lock is taken to protect the dentry against other dirops and
  * the validity of the dentry is checked: correct parent and still hashed.
  *
  * If the dentry is valid and positive, a reference is taken and
@@ -3642,7 +3642,7 @@ int user_path_at(int dfd, const char __user *name, unsigned flags,
 }
 EXPORT_SYMBOL(user_path_at);
 
-int __check_sticky(struct mnt_idmap *idmap, struct inode *dir,
+int __check_sticky(const struct mnt_idmap *idmap, struct inode *dir,
 		   struct inode *inode)
 {
 	kuid_t fsuid = current_fsuid();
@@ -3675,7 +3675,7 @@ EXPORT_SYMBOL(__check_sticky);
  * 11. We don't allow removal of NFS sillyrenamed files; it's handled by
  *     nfs_async_unlink().
  */
-int may_delete_dentry(struct mnt_idmap *idmap, struct inode *dir,
+int may_delete_dentry(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *victim, bool isdir)
 {
 	struct inode *inode = d_backing_inode(victim);
@@ -3728,7 +3728,7 @@ EXPORT_SYMBOL(may_delete_dentry);
  *  4. We should have write and exec permissions on dir
  *  5. We can't do it if dir is immutable (done in permission())
  */
-int may_create_dentry(struct mnt_idmap *idmap,
+int may_create_dentry(const struct mnt_idmap *idmap,
 		      struct inode *dir, struct dentry *child)
 {
 	audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE);
@@ -4142,7 +4142,7 @@ EXPORT_SYMBOL(end_renaming);
  *
  * Returns: mode to be passed to the filesystem
  */
-static inline umode_t vfs_prepare_mode(struct mnt_idmap *idmap,
+static inline umode_t vfs_prepare_mode(const struct mnt_idmap *idmap,
 				       const struct inode *dir, umode_t mode,
 				       umode_t mask_perms, umode_t type)
 {
@@ -4174,7 +4174,7 @@ static inline umode_t vfs_prepare_mode(struct mnt_idmap *idmap,
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int vfs_create(struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode,
+int vfs_create(const struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode,
 	       struct delegated_inode *di)
 {
 	struct inode *dir = d_inode(dentry->d_parent);
@@ -4188,7 +4188,7 @@ int vfs_create(struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode,
 		return -EACCES;	/* shouldn't it be ENOSYS? */
 
 	mode = vfs_prepare_mode(idmap, dir, mode, S_IALLUGO, S_IFREG);
-	error = security_inode_create(dir, dentry, mode);
+	error = security_inode_create(idmap, dir, dentry, mode);
 	if (error)
 		return error;
 	error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, di);
@@ -4212,7 +4212,7 @@ int vfs_mkobj(struct dentry *dentry, umode_t mode,
 
 	mode &= S_IALLUGO;
 	mode |= S_IFREG;
-	error = security_inode_create(dir, dentry, mode);
+	error = security_inode_create(&nop_mnt_idmap, dir, dentry, mode);
 	if (error)
 		return error;
 	error = f(dentry, mode, arg);
@@ -4228,7 +4228,7 @@ bool may_open_dev(const struct path *path)
 		!(path->mnt->mnt_sb->s_iflags & SB_I_NODEV);
 }
 
-static int may_open(struct mnt_idmap *idmap, const struct path *path,
+static int may_open(const struct mnt_idmap *idmap, const struct path *path,
 		    int acc_mode, int flag)
 {
 	struct dentry *dentry = path->dentry;
@@ -4287,7 +4287,7 @@ static int may_open(struct mnt_idmap *idmap, const struct path *path,
 	return 0;
 }
 
-static int handle_truncate(struct mnt_idmap *idmap, struct file *filp)
+static int handle_truncate(const struct mnt_idmap *idmap, struct file *filp)
 {
 	const struct path *path = &filp->f_path;
 	struct inode *inode = path->dentry->d_inode;
@@ -4312,7 +4312,7 @@ static inline int open_to_namei_flags(int flag)
 	return flag;
 }
 
-static int may_o_create(struct mnt_idmap *idmap,
+static int may_o_create(const struct mnt_idmap *idmap,
 			const struct path *dir, struct dentry *dentry,
 			umode_t mode)
 {
@@ -4328,7 +4328,7 @@ static int may_o_create(struct mnt_idmap *idmap,
 	if (error)
 		return error;
 
-	return security_inode_create(dir->dentry->d_inode, dentry, mode);
+	return security_inode_create(idmap, dir->dentry->d_inode, dentry, mode);
 }
 
 /**
@@ -4432,7 +4432,7 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 				  const struct open_flags *op)
 {
 	struct delegated_inode delegated_inode = { };
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct dentry *dir = nd->path.dentry;
 	struct inode *dir_inode = dir->d_inode;
 	int open_flag;
@@ -4440,12 +4440,14 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 	int error, create_error;
 	umode_t mode;
 	bool got_write;
+	unsigned int shared_flag;
 
 retry:
 	open_flag = op->open_flag;
 	got_write = false;
 	mode = op->mode;
 	create_error = 0;
+	shared_flag = (open_flag & O_CREAT) ? 0 : LOOKUP_SHARED;
 
 	if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) {
 		got_write = !mnt_want_write(nd->path.mnt);
@@ -4454,10 +4456,10 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 		 * a different error; we'll be dropping this one anyway.
 		 */
 	}
-	if (open_flag & O_CREAT)
-		inode_lock(dir_inode);
-	else
+	if (shared_flag)
 		inode_lock_shared(dir_inode);
+	else
+		inode_lock(dir_inode);
 
 	if (unlikely(IS_DEADDIR(dir_inode))) {
 		dentry = ERR_PTR(-ENOENT);
@@ -4526,7 +4528,7 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 
 	if (d_in_lookup(dentry)) {
 		struct dentry *res = dir_inode->i_op->lookup(dir_inode, dentry,
-							     nd->flags);
+							     nd->flags | shared_flag);
 		d_lookup_done(dentry);
 		if (unlikely(res)) {
 			if (IS_ERR(res)) {
@@ -4574,10 +4576,10 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 		if (file->f_mode & FMODE_OPENED)
 			fsnotify_open(file);
 	}
-	if ((open_flag & O_CREAT) || create_error)
-		inode_unlock(dir_inode);
-	else
+	if (shared_flag)
 		inode_unlock_shared(dir_inode);
+	else
+		inode_unlock(dir_inode);
 
 	if (got_write)
 		mnt_drop_write(nd->path.mnt);
@@ -4789,7 +4791,8 @@ static const char *open_last_lookups(struct nameidata *nd,
 static int do_open(struct nameidata *nd,
 		   struct file *file, const struct open_flags *op)
 {
-	struct mnt_idmap *idmap;
+	struct vfsmount *mnt;
+	const struct mnt_idmap *idmap;
 	int open_flag = op->open_flag;
 	bool do_truncate;
 	int acc_mode;
@@ -4830,11 +4833,17 @@ static int do_open(struct nameidata *nd,
 		error = mnt_want_write(nd->path.mnt);
 		if (error)
 			return error;
+		/*
+		 * A dedicated reference is needed because after the call to
+		 * vfs_open_consume() we no longer own the reference in nd->path.mnt
+		 * while we need to undo write acess below.
+		 */
+		mnt = mntget(nd->path.mnt);
 		do_truncate = true;
 	}
 	error = may_open(idmap, &nd->path, acc_mode, open_flag);
 	if (!error && !(file->f_mode & FMODE_OPENED))
-		error = vfs_open(&nd->path, file);
+		error = vfs_open_consume(&nd->path, file);
 	if (!error)
 		error = security_file_post_open(file, op->acc_mode);
 	if (!error && do_truncate)
@@ -4843,8 +4852,10 @@ static int do_open(struct nameidata *nd,
 		WARN_ON(1);
 		error = -EINVAL;
 	}
-	if (do_truncate)
-		mnt_drop_write(nd->path.mnt);
+	if (do_truncate) {
+		mnt_drop_write(mnt);
+		mntput(mnt);
+	}
 	return error;
 }
 
@@ -4863,7 +4874,7 @@ static int do_open(struct nameidata *nd,
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int vfs_tmpfile(struct mnt_idmap *idmap,
+int vfs_tmpfile(const struct mnt_idmap *idmap,
 		const struct path *parentpath,
 		struct file *file, umode_t mode)
 {
@@ -4921,7 +4932,7 @@ int vfs_tmpfile(struct mnt_idmap *idmap,
  * hence this is only for kernel internal use, and must not be installed into
  * file tables or such.
  */
-struct file *kernel_tmpfile_open(struct mnt_idmap *idmap,
+struct file *kernel_tmpfile_open(const struct mnt_idmap *idmap,
 				 const struct path *parentpath,
 				 umode_t mode, int open_flag,
 				 const struct cred *cred)
@@ -5169,7 +5180,7 @@ struct file *dentry_create(struct path *path, int flags, umode_t mode,
 	struct dentry *orig_dentry = dentry;
 	struct dentry *dir = dentry->d_parent;
 	struct inode *dir_inode = d_inode(dir);
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	int error, create_error;
 
 	file = alloc_empty_file(flags, cred);
@@ -5211,6 +5222,8 @@ struct file *dentry_create(struct path *path, int flags, umode_t mode,
 		error = vfs_create(mnt_idmap(path->mnt), path->dentry, mode, NULL);
 		if (!error)
 			error = vfs_open(path, file);
+		if (!error)
+			file->f_mode |= FMODE_CREATED;
 	}
 	if (unlikely(error))
 		return ERR_PTR(error);
@@ -5236,7 +5249,7 @@ EXPORT_SYMBOL(dentry_create);
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int vfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+int vfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 	      struct dentry *dentry, umode_t mode, dev_t dev,
 	      struct delegated_inode *delegated_inode)
 {
@@ -5258,7 +5271,7 @@ int vfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	if (error)
 		return error;
 
-	error = security_inode_mknod(dir, dentry, mode, dev);
+	error = security_inode_mknod(idmap, dir, dentry, mode, dev);
 	if (error)
 		return error;
 
@@ -5294,7 +5307,7 @@ int filename_mknodat(int dfd, struct filename *name, umode_t mode,
 		     unsigned int dev)
 {
 	struct delegated_inode di = { };
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct dentry *dentry;
 	struct path path;
 	int error;
@@ -5378,7 +5391,7 @@ SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, d
  *
  * In case of an error the dentry is dput() and an ERR_PTR() is returned.
  */
-struct dentry *vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+struct dentry *vfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, umode_t mode,
 			 struct delegated_inode *delegated_inode)
 {
@@ -5395,7 +5408,7 @@ struct dentry *vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 		goto err;
 
 	mode = vfs_prepare_mode(idmap, dir, mode, S_IRWXUGO | S_ISVTX, S_IFDIR);
-	error = security_inode_mkdir(dir, dentry, mode);
+	error = security_inode_mkdir(idmap, dir, dentry, mode);
 	if (error)
 		goto err;
 
@@ -5485,7 +5498,7 @@ SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode)
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int vfs_rmdir(struct mnt_idmap *idmap, struct inode *dir,
+int vfs_rmdir(const struct mnt_idmap *idmap, struct inode *dir,
 	      struct dentry *dentry, struct delegated_inode *delegated_inode)
 {
 	int error = may_delete_dentry(idmap, dir, dentry, true);
@@ -5620,7 +5633,7 @@ SYSCALL_DEFINE1(rmdir, const char __user *, pathname)
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int vfs_unlink(struct mnt_idmap *idmap, struct inode *dir,
+int vfs_unlink(const struct mnt_idmap *idmap, struct inode *dir,
 	       struct dentry *dentry, struct delegated_inode *delegated_inode)
 {
 	struct inode *target = dentry->d_inode;
@@ -5770,7 +5783,7 @@ SYSCALL_DEFINE1(unlink, const char __user *, pathname)
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int vfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+int vfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *dentry, const char *oldname,
 		struct delegated_inode *delegated_inode)
 {
@@ -5783,7 +5796,7 @@ int vfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	if (!dir->i_op->symlink)
 		return -EPERM;
 
-	error = security_inode_symlink(dir, dentry, oldname);
+	error = security_inode_symlink(idmap, dir, dentry, oldname);
 	if (error)
 		return error;
 
@@ -5872,7 +5885,7 @@ SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newn
  * On non-idmapped mounts or if permission checking is to be performed on the
  * raw inode simply pass @nop_mnt_idmap.
  */
-int vfs_link(struct dentry *old_dentry, struct mnt_idmap *idmap,
+int vfs_link(struct dentry *old_dentry, const struct mnt_idmap *idmap,
 	     struct inode *dir, struct dentry *new_dentry,
 	     struct delegated_inode *delegated_inode)
 {
@@ -5907,7 +5920,7 @@ int vfs_link(struct dentry *old_dentry, struct mnt_idmap *idmap,
 	if (S_ISDIR(inode->i_mode))
 		return -EPERM;
 
-	error = security_inode_link(old_dentry, dir, new_dentry);
+	error = security_inode_link(idmap, old_dentry, dir, new_dentry);
 	if (error)
 		return error;
 
@@ -5949,7 +5962,7 @@ EXPORT_SYMBOL(vfs_link);
 int filename_linkat(int olddfd, struct filename *old,
 		    int newdfd, struct filename *new, int flags)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct dentry *new_dentry;
 	struct path old_path, new_path;
 	struct delegated_inode delegated_inode = { };
diff --git a/fs/namespace.c b/fs/namespace.c
index ae5dc64f8b457e..d01b5402e847b9 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -109,7 +109,7 @@ struct mount_kattr {
 	unsigned int lookup_flags;
 	enum mount_kattr_flags_t kflags;
 	struct user_namespace *mnt_userns;
-	struct mnt_idmap *mnt_idmap;
+	const struct mnt_idmap *mnt_idmap;
 };
 
 /* /sys/fs */
@@ -249,16 +249,24 @@ void mnt_release_group_id(struct mount *mnt)
 	mnt->mnt_group_id = 0;
 }
 
-/*
- * vfsmount lock must be held for read
- */
-static inline void mnt_add_count(struct mount *mnt, int n)
+static inline void mnt_inc_count(struct mount *mnt)
+{
+#ifdef CONFIG_SMP
+	this_cpu_inc(mnt->mnt_pcp->mnt_gets);
+#else
+	preempt_disable();
+	mnt->mnt_count++;
+	preempt_enable();
+#endif
+}
+
+static inline void mnt_dec_count(struct mount *mnt)
 {
 #ifdef CONFIG_SMP
-	this_cpu_add(mnt->mnt_pcp->mnt_count, n);
+	this_cpu_inc(mnt->mnt_pcp->mnt_puts);
 #else
 	preempt_disable();
-	mnt->mnt_count += n;
+	mnt->mnt_count--;
 	preempt_enable();
 #endif
 }
@@ -269,14 +277,17 @@ static inline void mnt_add_count(struct mount *mnt, int n)
 int mnt_get_count(struct mount *mnt)
 {
 #ifdef CONFIG_SMP
-	int count = 0;
+	unsigned int gets = 0, puts = 0;
 	int cpu;
 
-	for_each_possible_cpu(cpu) {
-		count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_count;
-	}
+	/* puts first, so a put counted here has its get counted below */
+	for_each_possible_cpu(cpu)
+		puts += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_puts;
+	smp_mb();	/* pairs with the smp_wmb() in mntput_no_expire() */
+	for_each_possible_cpu(cpu)
+		gets += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_gets;
 
-	return count;
+	return gets - puts;
 #else
 	return mnt->mnt_count;
 #endif
@@ -305,7 +316,7 @@ static struct mount *alloc_vfsmnt(const char *name)
 		if (!mnt->mnt_pcp)
 			goto out_free_devname;
 
-		this_cpu_add(mnt->mnt_pcp->mnt_count, 1);
+		this_cpu_inc(mnt->mnt_pcp->mnt_gets);
 #else
 		mnt->mnt_count = 1;
 		mnt->mnt_writers = 0;
@@ -746,13 +757,13 @@ int __legitimize_mnt(struct vfsmount *bastard, unsigned seq)
 	if (bastard == NULL)
 		return 0;
 	mnt = real_mount(bastard);
-	mnt_add_count(mnt, 1);
-	smp_mb();		// see mntput_no_expire() and do_umount()
+	mnt_inc_count(mnt);
+	smp_mb();	/* see mntput_no_expire_slowpath() and do_umount() */
 	if (likely(!read_seqretry(&mount_lock, seq)))
 		return 0;
 	lock_mount_hash();
 	if (unlikely(bastard->mnt_flags & (MNT_SYNC_UMOUNT | MNT_DOOMED))) {
-		mnt_add_count(mnt, -1);
+		mnt_dec_count(mnt);
 		unlock_mount_hash();
 		return 1;
 	}
@@ -1254,6 +1265,7 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root,
 
 	mnt->mnt.mnt_flags = READ_ONCE(old->mnt.mnt_flags) &
 			     ~MNT_INTERNAL_FLAGS;
+	mnt->mnt_t_flags = old->mnt_t_flags & T_UNBINDABLE;
 
 	if (flag & (CL_SLAVE | CL_PRIVATE))
 		mnt->mnt_group_id = 0; /* not a peer of original */
@@ -1347,7 +1359,7 @@ static void noinline mntput_no_expire_slowpath(struct mount *mnt)
 	 * mount_lock, we'll see their refcount increment here.
 	 */
 	smp_mb();
-	mnt_add_count(mnt, -1);
+	mnt_dec_count(mnt);
 	count = mnt_get_count(mnt);
 	if (count != 0) {
 		WARN_ON(count < 0);
@@ -1404,7 +1416,8 @@ static void mntput_no_expire(struct mount *mnt)
 		 * non-NULL under rcu_read_lock(), the reference
 		 * we are dropping is not the final one.
 		 */
-		mnt_add_count(mnt, -1);
+		smp_wmb();	/* pairs with the smp_mb() in mnt_get_count() */
+		mnt_dec_count(mnt);
 		rcu_read_unlock();
 		return;
 	}
@@ -1426,7 +1439,7 @@ EXPORT_SYMBOL(mntput);
 struct vfsmount *mntget(struct vfsmount *mnt)
 {
 	if (mnt)
-		mnt_add_count(real_mount(mnt), 1);
+		mnt_inc_count(real_mount(mnt));
 	return mnt;
 }
 EXPORT_SYMBOL(mntget);
@@ -3467,7 +3480,7 @@ static int do_set_group(const struct path *from_path, const struct path *to_path
 		return -EINVAL;
 
 	/* Setting sharing groups is only allowed on private mounts */
-	if (IS_MNT_SHARED(to) || IS_MNT_SLAVE(to))
+	if (IS_MNT_SHARED(to) || IS_MNT_SLAVE(to) || IS_MNT_UNBINDABLE(to))
 		return -EINVAL;
 
 	/* From should not be private */
@@ -4115,7 +4128,7 @@ int path_mount(const char *dev_name, const struct path *path,
 	if (flags & SB_MANDLOCK)
 		warn_mandlock();
 
-	/* Default to relatime unless overriden */
+	/* Default to relatime unless overridden */
 	if (!(flags & MS_NOATIME))
 		mnt_flags |= MNT_RELATIME;
 
@@ -4193,8 +4206,7 @@ static void dec_mnt_namespaces(struct ucounts *ucounts)
 
 static void free_mnt_ns(struct mnt_namespace *ns)
 {
-	if (!is_anon_ns(ns))
-		ns_common_free(ns);
+	ns_common_free(ns);
 	dec_mnt_namespaces(ns->ucounts);
 	mnt_ns_tree_remove(ns);
 }
@@ -4247,8 +4259,6 @@ struct mnt_namespace *copy_mnt_ns(u64 flags, struct mnt_namespace *ns,
 	struct mount *new;
 	int copy_flags;
 
-	BUG_ON(!ns);
-
 	if (likely(!(flags & CLONE_NEWNS))) {
 		get_mnt_ns(ns);
 		return ns;
@@ -4544,16 +4554,16 @@ SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags,
 
 	FD_PREPARE(fdf, (flags & FSMOUNT_CLOEXEC) ? O_CLOEXEC : 0,
 		   dentry_open(&new_path, O_PATH, fc->cred));
-	if (fdf.err) {
+	if (fdf->fd < 0) {
 		dissolve_on_fput(new_path.mnt);
-		return fdf.err;
+		return fdf->fd;
 	}
 
 	/*
 	 * Attach to an apparent O_PATH fd with a note that we
 	 * need to unmount it, not just simply put it.
 	 */
-	fd_prepare_file(fdf)->f_mode |= FMODE_NEED_UNMOUNT;
+	fdf->file->f_mode |= FMODE_NEED_UNMOUNT;
 	return fd_publish(fdf);
 }
 
@@ -4898,7 +4908,7 @@ static int mount_setattr_prepare(struct mount_kattr *kattr, struct mount *mnt)
 
 static void do_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt)
 {
-	struct mnt_idmap *old_idmap;
+	const struct mnt_idmap *old_idmap;
 
 	if (!kattr->mnt_idmap)
 		return;
@@ -4941,7 +4951,7 @@ static int do_mount_setattr(const struct path *path, struct mount_kattr *kattr)
 		return -EINVAL;
 
 	if (kattr->mnt_userns) {
-		struct mnt_idmap *mnt_idmap;
+		const struct mnt_idmap *mnt_idmap;
 
 		mnt_idmap = alloc_mnt_idmap(kattr->mnt_userns);
 		if (IS_ERR(mnt_idmap))
@@ -5198,12 +5208,12 @@ SYSCALL_DEFINE5(open_tree_attr, int, dfd, const char __user *, filename,
 		return -EINVAL;
 
 	FD_PREPARE(fdf, flags, vfs_open_tree(dfd, filename, flags));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
 	if (uattr) {
 		struct mount_kattr kattr = {};
-		struct file *file = fd_prepare_file(fdf);
+		struct file *file = fdf->file;
 		int ret;
 
 		if (flags & OPEN_TREE_CLONE)
@@ -5246,7 +5256,7 @@ struct kstatmount {
 	struct statmount __user *buf;
 	size_t bufsize;
 	struct vfsmount *mnt;
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	u64 mask;
 	struct path root;
 	struct seq_file seq;
@@ -6184,6 +6194,21 @@ struct mnt_namespace init_mnt_ns = {
 	.poll		= __WAIT_QUEUE_HEAD_INITIALIZER(init_mnt_ns.poll),
 };
 
+static void __init mount_rootfs_on_nullfs(struct vfsmount *mnt,
+					  struct vfsmount *nullfs_mnt)
+{
+	struct path root = {
+		.mnt	= nullfs_mnt,
+		.dentry	= nullfs_mnt->mnt_root,
+	};
+
+	LOCK_MOUNT_EXACT(mp, &root);
+	if (unlikely(IS_ERR(mp.parent)))
+		panic("VFS: Failed to mount rootfs on nullfs");
+	scoped_guard(mount_writer)
+		attach_mnt(real_mount(mnt), mp.parent, mp.mp);
+}
+
 static void __init init_mount_tree(void)
 {
 	struct vfsmount *mnt, *nullfs_mnt;
@@ -6215,15 +6240,7 @@ static void __init init_mount_tree(void)
 	mnt_root		= real_mount(nullfs_mnt);
 	init_mnt_ns.root	= mnt_root;
 
-	/* Mount mutable rootfs on top of nullfs. */
-	root.mnt		= nullfs_mnt;
-	root.dentry		= nullfs_mnt->mnt_root;
-
-	LOCK_MOUNT_EXACT(mp, &root);
-	if (unlikely(IS_ERR(mp.parent)))
-		panic("VFS: Failed to mount rootfs on nullfs");
-	scoped_guard(mount_writer)
-		attach_mnt(real_mount(mnt), mp.parent, mp.mp);
+	mount_rootfs_on_nullfs(mnt, nullfs_mnt);
 
 	pr_info("VFS: Finished mounting rootfs on nullfs\n");
 
@@ -6294,7 +6311,7 @@ void put_mnt_ns(struct mnt_namespace *ns)
 	guard(namespace_excl)();
 	emptied_ns = ns;
 	guard(mount_writer)();
-	umount_tree(ns->root, UMOUNT_CONNECTED);
+	umount_tree(ns->root, 0);
 }
 
 struct vfsmount *kern_mount(struct file_system_type *type)
diff --git a/fs/netfs/Kconfig b/fs/netfs/Kconfig
index 7701c037c3283f..d0e7b0971fa3f0 100644
--- a/fs/netfs/Kconfig
+++ b/fs/netfs/Kconfig
@@ -22,6 +22,9 @@ config NETFS_STATS
 	  between CPUs.  On the other hand, the stats are very useful for
 	  debugging purposes.  Saying 'Y' here is recommended.
 
+config NETFS_PGPRIV2
+	bool
+
 config NETFS_DEBUG
 	bool "Enable dynamic debugging netfslib and FS-Cache"
 	depends on NETFS_SUPPORT
diff --git a/fs/netfs/Makefile b/fs/netfs/Makefile
index b43188d64bd87e..54834cde7e5686 100644
--- a/fs/netfs/Makefile
+++ b/fs/netfs/Makefile
@@ -11,7 +11,6 @@ netfs-y := \
 	misc.o \
 	objects.o \
 	read_collect.o \
-	read_pgpriv2.o \
 	read_retry.o \
 	read_single.o \
 	rolling_buffer.o \
@@ -19,6 +18,7 @@ netfs-y := \
 	write_issue.o \
 	write_retry.o
 
+netfs-$(CONFIG_NETFS_PGPRIV2) += read_pgpriv2.o
 netfs-$(CONFIG_NETFS_STATS) += stats.o
 
 netfs-$(CONFIG_FSCACHE) += \
diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c
index 424df70a5c30f1..e30bde80276a47 100644
--- a/fs/netfs/buffered_read.c
+++ b/fs/netfs/buffered_read.c
@@ -10,9 +10,9 @@
 #include "internal.h"
 
 static void netfs_cache_expand_readahead(struct netfs_io_request *rreq,
-					 unsigned long long *_start,
-					 unsigned long long *_len,
-					 unsigned long long i_size)
+					 uoff_t *_start,
+					 uoff_t *_len,
+					 uoff_t i_size)
 {
 	struct netfs_cache_resources *cres = &rreq->cache_resources;
 
@@ -137,21 +137,6 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq)
 	return subreq->len;
 }
 
-static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_request *rreq,
-						     struct netfs_io_subrequest *subreq,
-						     loff_t i_size)
-{
-	struct netfs_cache_resources *cres = &rreq->cache_resources;
-	enum netfs_io_source source;
-
-	if (!cres->ops)
-		return NETFS_DOWNLOAD_FROM_SERVER;
-	source = cres->ops->prepare_read(subreq, i_size);
-	trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
-	return source;
-
-}
-
 /*
  * Issue a read against the cache.
  * - Eats the caller's ref on subreq.
@@ -166,6 +151,19 @@ static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq,
 			netfs_cache_read_terminated, subreq);
 }
 
+int netfs_read_query_cache(struct netfs_io_request *rreq, struct fscache_occupancy *occ)
+{
+	struct netfs_cache_resources *cres = &rreq->cache_resources;
+
+	occ->granularity = PAGE_SIZE;
+	if (occ->query_from >= occ->query_to)
+		return 0;
+	if (!cres->ops)
+		return 0;
+	occ->query_from = round_up(occ->query_from, occ->granularity);
+	return cres->ops->query_occupancy(cres, occ);
+}
+
 void netfs_queue_read(struct netfs_io_request *rreq,
 		      struct netfs_io_subrequest *subreq)
 {
@@ -242,7 +240,7 @@ static void netfs_mark_copy_to_cache(struct netfs_io_request *rreq,
 
 		if (overlap > 0 && copy) {
 			folio = folioq_folio(*fq, *slot);
-			if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags))) {
+			if (netfs_using_pgpriv2(rreq)) {
 				if (!folio_test_private_2(folio))
 					folio_start_private_2(folio);
 			} else {
@@ -268,18 +266,113 @@ static void netfs_mark_copy_to_cache(struct netfs_io_request *rreq,
  */
 static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
 {
+	struct fscache_occupancy _occ = {
+		.query_from	= rreq->start,
+		.query_to	= rreq->start + rreq->len,
+		.cached_from[0]	= 0,
+		.cached_to[0]	= 0,
+		.cached_from[1]	= ULLONG_MAX,
+		.cached_to[1]	= ULLONG_MAX,
+	};
+	struct fscache_occupancy *occ = &_occ;
 	struct folio_queue *fq = rreq->buffer.tail;
-	unsigned long long start = rreq->start;
 	unsigned int offset = 0;
 	ssize_t size = rreq->len;
+	uoff_t start = rreq->start;
 	int ret = 0, slot = 0;
 
 	do {
+		int (*prepare_read)(struct netfs_io_subrequest *subreq) = NULL;
 		struct netfs_io_subrequest *subreq;
-		enum netfs_io_source source = NETFS_SOURCE_UNKNOWN;
+		enum netfs_io_source source;
 		ssize_t slice;
+		uoff_t hole_to, cache_to;
+		size_t len = size;
+		bool copy = false;
+
+		/* If we don't have any, find out the next couple of data
+		 * extents from the cache, containing of following the
+		 * specified start offset.  Holes have to be fetched from the
+		 * server; data regions from the cache.
+		 */
+		hole_to = occ->cached_from[0];
+		cache_to = occ->cached_to[0];
+		if (start >= cache_to) {
+			/* Extent exhausted; shuffle down. */
+			int i;
+
+			for (i = 0; i < ARRAY_SIZE(occ->cached_from) - 1; i++) {
+				occ->cached_from[i] = occ->cached_from[i + 1];
+				occ->cached_to[i]   = occ->cached_to[i + 1];
+				occ->cached_type[i] = occ->cached_type[i + 1];
+			}
+			occ->cached_from[i] = ULLONG_MAX;
+			occ->cached_to[i]   = ULLONG_MAX;
+
+			if (occ->cached_from[0] != ULLONG_MAX)
+				continue;
+
+			/* Get new extents */
+			ret = netfs_read_query_cache(rreq, occ);
+			if (ret < 0)
+				break;
+			continue;
+		}
+
+		uoff_t zero_point = netfs_read_zero_point(rreq->inode);
+		uoff_t zlimit = umin(zero_point, rreq->i_size);
+
+		_debug("rsub %llx %llx-%llx", start, hole_to, cache_to);
+
+		if (start >= hole_to && start < cache_to) {
+			/* Overlap with a cached region, where the cache may
+			 * record a block of zeroes.
+			 */
+			_debug("cached s=%llx c=%llx l=%zx", start, cache_to, size);
+			len = umin(cache_to - start, size);
+			len = round_up(len, occ->granularity);
+			if (occ->cached_type[0] == FSCACHE_EXTENT_ZERO) {
+				source = NETFS_FILL_WITH_ZEROES;
+				netfs_stat(&netfs_n_rh_zero);
+			} else {
+				source = NETFS_READ_FROM_CACHE;
+				prepare_read = rreq->cache_resources.ops->prepare_read;
+			}
+		} else if (start >= zlimit && size > 0) {
+			/* If this range lies beyond the zero-point, that part
+			 * can just be cleared locally.
+			 */
+			_debug("zero %llx-%llx", start, start + size);
+			len = size;
+			source = NETFS_FILL_WITH_ZEROES;
+			if (rreq->cache_resources.ops)
+				copy = true;
+			netfs_stat(&netfs_n_rh_zero);
+		} else {
+			/* Read a cache hole from the server.  If any part of
+			 * this range lies beyond the zero-point or the EOF,
+			 * that part can just be cleared locally.
+			 */
+			uoff_t limit = min3(zlimit, start + size, hole_to);
+
+			_debug("limit %llx %llx", rreq->i_size, zero_point);
+			_debug("download %llx-%llx", start, start + size);
+			len = umin(limit - start, ULONG_MAX);
+			source = NETFS_DOWNLOAD_FROM_SERVER;
+			prepare_read = rreq->netfs_ops->prepare_read;
+			if (rreq->cache_resources.ops)
+				copy = true;
+			netfs_stat(&netfs_n_rh_download);
+		}
+
+		if (len == 0) {
+			pr_err("ZERO-LEN READ: R=%08x l=%zx/%zx s=%llx z=%llx i=%llx",
+			       rreq->debug_id, len, size,
+			       start, zero_point, rreq->i_size);
+			break;
+		}
 
-		subreq = netfs_alloc_subrequest(rreq);
+		subreq = netfs_alloc_subrequest(rreq, source);
 		if (!subreq) {
 			ret = -ENOMEM;
 			break;
@@ -287,66 +380,23 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
 
 		subreq->start	= start;
 		subreq->len	= size;
+		if (copy)
+			__set_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
 
 		netfs_queue_read(rreq, subreq);
 
-		source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size);
-		subreq->source = source;
-		if (source == NETFS_DOWNLOAD_FROM_SERVER) {
-			unsigned long long zero_point = netfs_read_zero_point(rreq->inode);
-			unsigned long long zp = umin(zero_point, rreq->i_size);
-			size_t len = subreq->len;
-
-			if (unlikely(rreq->origin == NETFS_READ_SINGLE))
-				zp = rreq->i_size;
-			if (subreq->start >= zp) {
-				subreq->source = source = NETFS_FILL_WITH_ZEROES;
-				goto fill_with_zeroes;
-			}
+		rreq->io_streams[0].sreq_max_len = MAX_RW_COUNT;
+		rreq->io_streams[0].sreq_max_segs = INT_MAX;
 
-			if (len > zp - subreq->start)
-				len = zp - subreq->start;
-			if (len == 0) {
-				pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx",
-				       rreq->debug_id, subreq->debug_index,
-				       subreq->len, size,
-				       subreq->start, zero_point, rreq->i_size);
+		if (prepare_read) {
+			ret = prepare_read(subreq);
+			if (ret < 0) {
 				netfs_cancel_read(subreq, ret);
 				break;
 			}
-			subreq->len = len;
-
-			netfs_stat(&netfs_n_rh_download);
-			if (rreq->netfs_ops->prepare_read) {
-				ret = rreq->netfs_ops->prepare_read(subreq);
-				if (ret < 0) {
-					netfs_cancel_read(subreq, ret);
-					break;
-				}
-				trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
-			}
-			goto issue;
-		}
-
-	fill_with_zeroes:
-		if (source == NETFS_FILL_WITH_ZEROES) {
-			subreq->source = NETFS_FILL_WITH_ZEROES;
-			trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
-			netfs_stat(&netfs_n_rh_zero);
-			goto issue;
+			trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
 		}
 
-		if (source == NETFS_READ_FROM_CACHE) {
-			trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
-			goto issue;
-		}
-
-		pr_err("Unexpected read source %u\n", source);
-		WARN_ON_ONCE(1);
-		netfs_cancel_read(subreq, ret);
-		break;
-
-	issue:
 		slice = netfs_prepare_read_iterator(subreq);
 		if (slice < 0) {
 			ret = slice;
@@ -355,18 +405,16 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
 		}
 		start += slice;
 		size -= slice;
-		if (size <= 0) {
-			smp_wmb(); /* Write lists before ALL_QUEUED. */
-			set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
-		}
+		if (size <= 0)
+			netfs_all_subreqs_queued(rreq);
 
 		if (fq) {
 			/* See if the cache indicated this should be cached. */
-			bool copy = test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
-
+			copy = test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
 			netfs_mark_copy_to_cache(rreq, &fq, &slot, &offset, slice, copy);
 		}
 
+		trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
 		netfs_issue_read(rreq, subreq);
 		netfs_maybe_bulk_drop_ra_refs(rreq);
 
@@ -378,8 +426,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
 	} while (size > 0);
 
 	if (unlikely(size > 0)) {
-		smp_wmb(); /* Write lists before ALL_QUEUED. */
-		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+		netfs_all_subreqs_queued(rreq);
 		netfs_wake_collector(rreq);
 	}
 
@@ -482,15 +529,14 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
 	struct netfs_group *group = netfs_folio_group(folio);
 	struct netfs_folio *finfo = netfs_folio_info(folio);
 	struct netfs_inode *ctx = netfs_inode(mapping->host);
-	struct folio *sink = NULL;
-	struct bio_vec *bvec;
+	struct bio_vec *bvec = NULL;
 	unsigned int from = finfo->dirty_offset;
 	unsigned int to = from + finfo->dirty_len;
-	unsigned int off = 0, i = 0;
+	unsigned int off = 0;
 	size_t flen = folio_size(folio);
 	size_t nr_bvec = flen / PAGE_SIZE + 2;
 	size_t part;
-	int ret;
+	int ret, i = 0, sink_from = -1, sink_to = -1;
 
 	_enter("%lx", folio->index);
 
@@ -515,24 +561,23 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
 	if (!bvec)
 		goto discard;
 
-	sink = folio_alloc(GFP_KERNEL, 0);
-	if (!sink) {
-		kfree(bvec);
-		goto discard;
-	}
-
 	trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
 
-	rreq->direct_bv = bvec;
-	rreq->direct_bv_count = nr_bvec;
 	if (from > 0) {
 		bvec_set_folio(&bvec[i++], folio, from, 0);
 		off = from;
 	}
+	sink_from = i;
 	while (off < to) {
+		struct folio *sink = folio_alloc(GFP_KERNEL, 0);
+
+		if (!sink)
+			goto discard;
 		part = min_t(size_t, to - off, PAGE_SIZE);
-		bvec_set_folio(&bvec[i++], sink, part, 0);
+		bvec_set_folio(&bvec[i], sink, part, 0);
 		off += part;
+		sink_to = i;
+		i++;
 	}
 	if (to < flen)
 		bvec_set_folio(&bvec[i++], folio, flen - to, to);
@@ -553,8 +598,10 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
 		folio_mark_uptodate(folio);
 	}
 
-	if (sink)
-		folio_put(sink);
+	if (sink_to >= 0)
+		for (; sink_from <= sink_to; sink_from++)
+			folio_put(bvec_folio(&bvec[sink_from]));
+	kfree(bvec);
 	folio_unlock(folio);
 	netfs_put_request(rreq, netfs_rreq_trace_put_return);
 	return ret < 0 ? ret : 0;
@@ -563,6 +610,10 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
 	netfs_put_failed_request(rreq);
 alloc_error:
 	folio_unlock(folio);
+	if (sink_to >= 0)
+		for (; sink_from <= sink_to; sink_from++)
+			folio_put(bvec_folio(&bvec[sink_from]));
+	kfree(bvec);
 	return ret;
 }
 
@@ -642,11 +693,11 @@ EXPORT_SYMBOL(netfs_read_folio);
  * If any of these criteria are met, then zero out the unwritten parts
  * of the folio and return true. Otherwise, return false.
  */
-static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len,
+static bool netfs_skip_folio_read(struct folio *folio, uoff_t pos, size_t len,
 				 bool always_fill)
 {
 	struct inode *inode = folio_inode(folio);
-	loff_t i_size = i_size_read(inode);
+	uoff_t i_size = i_size_read(inode);
 	size_t offset = offset_in_folio(folio, pos);
 	size_t plen = folio_size(folio);
 
@@ -711,7 +762,7 @@ static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len,
  */
 int netfs_write_begin(struct netfs_inode *ctx,
 		      struct file *file, struct address_space *mapping,
-		      loff_t pos, unsigned int len, struct folio **_folio,
+		      uoff_t pos, unsigned int len, struct folio **_folio,
 		      void **_fsdata)
 {
 	struct netfs_io_request *rreq;
@@ -807,7 +858,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio,
 	struct netfs_io_request *rreq;
 	struct address_space *mapping = folio->mapping;
 	struct netfs_inode *ctx = netfs_inode(mapping->host);
-	unsigned long long start = folio_pos(folio);
+	uoff_t start = folio_pos(folio);
 	size_t flen = folio_size(folio);
 	int ret;
 
diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c
index 2cdb68e6b16fff..49b47252f67502 100644
--- a/fs/netfs/buffered_write.c
+++ b/fs/netfs/buffered_write.c
@@ -17,7 +17,7 @@
  * as possible to hold as much of the remaining length as possible in one go.
  */
 static struct folio *netfs_grab_folio_for_write(struct address_space *mapping,
-						loff_t pos, size_t part)
+						uoff_t pos, size_t part)
 {
 	pgoff_t index = pos / PAGE_SIZE;
 	fgf_t fgp_flags = FGP_WRITEBEGIN;
@@ -35,9 +35,9 @@ static struct folio *netfs_grab_folio_for_write(struct address_space *mapping,
  * the values actually are.
  */
 void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
-			 loff_t pos, size_t copied)
+			 uoff_t pos, size_t copied)
 {
-	loff_t i_size, end = pos + copied;
+	uoff_t i_size, end = pos + copied;
 	blkcnt_t add;
 	size_t gap;
 
@@ -54,9 +54,6 @@ void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
 	i_size = i_size_read(inode);
 	if (end > i_size) {
 		i_size_write(inode, end);
-#if IS_ENABLED(CONFIG_FSCACHE)
-		fscache_update_cookie(ctx->cache, NULL, &end);
-#endif
 
 		gap = SECTOR_SIZE - (i_size & (SECTOR_SIZE - 1));
 		if (copied > gap) {
@@ -91,50 +88,20 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 	struct inode *inode = file_inode(file);
 	struct address_space *mapping = inode->i_mapping;
 	struct netfs_inode *ctx = netfs_inode(inode);
-	struct writeback_control wbc = {
-		.sync_mode	= WB_SYNC_NONE,
-		.for_sync	= true,
-		.nr_to_write	= LONG_MAX,
-		.range_start	= iocb->ki_pos,
-		.range_end	= iocb->ki_pos + iter->count,
-	};
-	struct netfs_io_request *wreq = NULL;
-	struct folio *folio = NULL, *writethrough = NULL;
+	struct folio *folio = NULL;
 	unsigned int bdp_flags = (iocb->ki_flags & IOCB_NOWAIT) ? BDP_ASYNC : 0;
-	ssize_t written = 0, ret, ret2;
-	loff_t pos = iocb->ki_pos;
+	ssize_t written = 0, ret;
+	uoff_t pos = iocb->ki_pos;
 	size_t max_chunk = mapping_max_folio_size(mapping);
 	bool maybe_trouble = false;
 
-	if (unlikely(iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC))
-	    ) {
-		wbc_attach_fdatawrite_inode(&wbc, mapping->host);
-
-		ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count);
-		if (ret < 0) {
-			wbc_detach_inode(&wbc);
-			goto out;
-		}
-
-		wreq = netfs_begin_writethrough(iocb, iter->count);
-		if (IS_ERR(wreq)) {
-			wbc_detach_inode(&wbc);
-			ret = PTR_ERR(wreq);
-			wreq = NULL;
-			goto out;
-		}
-		if (!is_sync_kiocb(iocb))
-			wreq->iocb = iocb;
-		netfs_stat(&netfs_n_wh_writethrough);
-	} else {
-		netfs_stat(&netfs_n_wh_buffered_write);
-	}
+	netfs_stat(&netfs_n_wh_buffered_write);
 
 	do {
 		enum netfs_folio_trace trace;
 		struct netfs_folio *finfo;
 		struct netfs_group *group;
-		unsigned long long fpos;
+		uoff_t fpos;
 		size_t flen;
 		size_t offset;	/* Offset into pagecache folio */
 		size_t part;	/* Bytes to write to folio */
@@ -390,15 +357,8 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 		pos += copied;
 		written += copied;
 
-		if (likely(!wreq)) {
-			folio_mark_dirty(folio);
-			folio_unlock(folio);
-		} else {
-			netfs_advance_writethrough(wreq, &wbc, folio, copied,
-						   offset + copied == flen,
-						   &writethrough);
-			/* Folio unlocked */
-		}
+		folio_mark_dirty(folio);
+		folio_unlock(folio);
 	retry:
 		folio_put(folio);
 		folio = NULL;
@@ -420,15 +380,6 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 			ctx->ops->post_modify(inode);
 	}
 
-	if (unlikely(wreq)) {
-		ret2 = netfs_end_writethrough(wreq, &wbc, writethrough);
-		wbc_detach_inode(&wbc);
-		if (ret2 == -EIOCBQUEUED)
-			return ret2;
-		if (ret == 0 && ret2 < 0)
-			ret = ret2;
-	}
-
 	iocb->ki_pos += written;
 	_leave(" = %zd [%zd]", written, ret);
 	return written ? written : ret;
diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c
index 6a8fb0d55e040e..8c15f307972383 100644
--- a/fs/netfs/direct_read.c
+++ b/fs/netfs/direct_read.c
@@ -47,15 +47,15 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq)
  */
 static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 {
-	unsigned long long start = rreq->start;
 	ssize_t size = rreq->len;
+	uoff_t start = rreq->start;
 	int ret;
 
 	do {
 		struct netfs_io_subrequest *subreq;
 		ssize_t slice;
 
-		subreq = netfs_alloc_subrequest(rreq);
+		subreq = netfs_alloc_subrequest(rreq, NETFS_DOWNLOAD_FROM_SERVER);
 		if (!subreq) {
 			/* Stash the error in the request if there's not
 			 * already an error set.
@@ -64,7 +64,6 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 			break;
 		}
 
-		subreq->source	= NETFS_DOWNLOAD_FROM_SERVER;
 		subreq->start	= start;
 		subreq->len	= size;
 
@@ -84,10 +83,8 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 		size -= slice;
 		start += slice;
 		rreq->submitted += slice;
-		if (size <= 0) {
-			smp_wmb(); /* Write lists before ALL_QUEUED. */
-			set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
-		}
+		if (size <= 0)
+			netfs_all_subreqs_queued(rreq);
 
 		rreq->netfs_ops->issue_read(subreq);
 
@@ -99,8 +96,7 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 	} while (size > 0);
 
 	if (unlikely(size > 0)) {
-		smp_wmb(); /* Write lists before ALL_QUEUED. */
-		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+		netfs_all_subreqs_queued(rreq);
 		netfs_wake_collector(rreq);
 	}
 }
diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c
index 2361277416c730..32200c10d2a45c 100644
--- a/fs/netfs/direct_write.c
+++ b/fs/netfs/direct_write.c
@@ -225,9 +225,9 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *
 						  struct netfs_group *netfs_group)
 {
 	struct netfs_io_request *wreq;
-	unsigned long long start = iocb->ki_pos;
-	unsigned long long end = start + iov_iter_count(iter);
 	ssize_t ret, n;
+	uoff_t start = iocb->ki_pos;
+	uoff_t end = start + iov_iter_count(iter);
 	size_t len = iov_iter_count(iter);
 	bool async = !is_sync_kiocb(iocb);
 
@@ -336,8 +336,8 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	struct inode *inode = mapping->host;
 	struct netfs_inode *ictx = netfs_inode(inode);
 	ssize_t ret;
-	loff_t pos = iocb->ki_pos;
-	unsigned long long end = pos + iov_iter_count(from) - 1;
+	uoff_t pos = iocb->ki_pos;
+	uoff_t end = pos + iov_iter_count(from) - 1;
 
 	_enter("%llx,%zx,%llx", pos, iov_iter_count(from), i_size_read(inode));
 
diff --git a/fs/netfs/fscache_cookie.c b/fs/netfs/fscache_cookie.c
index 3d56fc73435ffb..5a226f9cbdeaa0 100644
--- a/fs/netfs/fscache_cookie.c
+++ b/fs/netfs/fscache_cookie.c
@@ -327,7 +327,7 @@ static struct fscache_cookie *fscache_alloc_cookie(
 	u8 advice,
 	const void *index_key, size_t index_key_len,
 	const void *aux_data, size_t aux_data_len,
-	loff_t object_size)
+	uoff_t object_size)
 {
 	struct fscache_cookie *cookie;
 
@@ -452,7 +452,7 @@ struct fscache_cookie *__fscache_acquire_cookie(
 	u8 advice,
 	const void *index_key, size_t index_key_len,
 	const void *aux_data, size_t aux_data_len,
-	loff_t object_size)
+	uoff_t object_size)
 {
 	struct fscache_cookie *cookie;
 
@@ -663,7 +663,7 @@ static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie)
  * Stop using the cookie for I/O.
  */
 void __fscache_unuse_cookie(struct fscache_cookie *cookie,
-			    const void *aux_data, const loff_t *object_size)
+			    const void *aux_data, const uoff_t *object_size)
 {
 	unsigned int debug_id = cookie->debug_id;
 	unsigned int r = refcount_read(&cookie->ref);
@@ -1049,7 +1049,7 @@ static void fscache_perform_invalidation(struct fscache_cookie *cookie)
  * Invalidate an object.
  */
 void __fscache_invalidate(struct fscache_cookie *cookie,
-			  const void *aux_data, loff_t new_size,
+			  const void *aux_data, uoff_t new_size,
 			  unsigned int flags)
 {
 	bool is_caching;
diff --git a/fs/netfs/fscache_internal.h b/fs/netfs/fscache_internal.h
deleted file mode 100644
index a09b948fcef212..00000000000000
--- a/fs/netfs/fscache_internal.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* Internal definitions for FS-Cache
- *
- * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells (dhowells@redhat.com)
- */
-
-#include "internal.h"
-
-#ifdef pr_fmt
-#undef pr_fmt
-#endif
-
-#define pr_fmt(fmt) "FS-Cache: " fmt
diff --git a/fs/netfs/fscache_io.c b/fs/netfs/fscache_io.c
index 37f05b4d34699d..056a2bae5d99a3 100644
--- a/fs/netfs/fscache_io.c
+++ b/fs/netfs/fscache_io.c
@@ -79,7 +79,7 @@ static int fscache_begin_operation(struct netfs_cache_resources *cres,
 	cres->ops		= NULL;
 	cres->cache_priv	= cookie;
 	cres->cache_priv2	= NULL;
-	cres->debug_id		= cookie->debug_id;
+	cres->cookie_id		= cookie->debug_id;
 	cres->inval_counter	= cookie->inval_counter;
 
 	if (!fscache_begin_cookie_access(cookie, why)) {
@@ -162,7 +162,7 @@ EXPORT_SYMBOL(__fscache_begin_write_operation);
 struct fscache_write_request {
 	struct netfs_cache_resources cache_resources;
 	struct address_space	*mapping;
-	loff_t			start;
+	uoff_t			start;
 	size_t			len;
 	bool			set_bits;
 	bool			using_pgpriv2;
@@ -171,7 +171,7 @@ struct fscache_write_request {
 };
 
 void __fscache_clear_page_bits(struct address_space *mapping,
-			       loff_t start, size_t len)
+			       uoff_t start, size_t len)
 {
 	pgoff_t first = start / PAGE_SIZE;
 	pgoff_t last = (start + len - 1) / PAGE_SIZE;
@@ -208,7 +208,7 @@ static void fscache_wreq_done(void *priv, ssize_t transferred_or_error)
 
 void __fscache_write_to_cache(struct fscache_cookie *cookie,
 			      struct address_space *mapping,
-			      loff_t start, size_t len, loff_t i_size,
+			      uoff_t start, size_t len, uoff_t i_size,
 			      netfs_io_terminated_t term_func,
 			      void *term_func_priv,
 			      bool using_pgpriv2, bool cond)
@@ -267,7 +267,7 @@ EXPORT_SYMBOL(__fscache_write_to_cache);
 /*
  * Change the size of a backing object.
  */
-void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size)
+void __fscache_resize_cookie(struct fscache_cookie *cookie, uoff_t new_size)
 {
 	struct netfs_cache_resources cres;
 
diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h
index c79c8e69d60ca7..b8591abc90a985 100644
--- a/fs/netfs/internal.h
+++ b/fs/netfs/internal.h
@@ -23,6 +23,8 @@
 /*
  * buffered_read.c
  */
+int netfs_read_query_cache(struct netfs_io_request *rreq,
+			   struct fscache_occupancy *occ);
 void netfs_queue_read(struct netfs_io_request *rreq,
 		      struct netfs_io_subrequest *subreq);
 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
@@ -33,7 +35,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio,
  * buffered_write.c
  */
 void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode,
-			 loff_t pos, size_t copied);
+			 uoff_t pos, size_t copied);
 
 /*
  * main.c
@@ -86,13 +88,14 @@ void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq);
  */
 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
 					     struct file *file,
-					     loff_t start, size_t len,
+					     uoff_t start, size_t len,
 					     enum netfs_io_origin origin);
 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
 void netfs_clear_subrequests(struct netfs_io_request *rreq);
 void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what);
 void netfs_put_failed_request(struct netfs_io_request *rreq);
-struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq);
+struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq,
+						   enum netfs_io_source source);
 
 static inline void netfs_see_request(struct netfs_io_request *rreq,
 				     enum netfs_rreq_ref_trace what)
@@ -120,9 +123,37 @@ void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
 /*
  * read_pgpriv2.c
  */
+#ifdef CONFIG_NETFS_PGPRIV2
+int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq,
+				       unsigned int max_segs);
 void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
 void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq);
 bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
+static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq)
+{
+	return unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags));
+}
+#else
+static inline int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq,
+						     unsigned int max_segs)
+{
+	return -EIO;
+}
+static inline void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
+{
+}
+static inline void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq)
+{
+}
+static inline bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq)
+{
+	return true;
+}
+static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq)
+{
+	return false;
+}
+#endif
 
 /*
  * read_retry.c
@@ -157,7 +188,6 @@ extern atomic_t netfs_n_rh_write_zskip;
 extern atomic_t netfs_n_rh_retry_read_req;
 extern atomic_t netfs_n_rh_retry_read_subreq;
 extern atomic_t netfs_n_wh_buffered_write;
-extern atomic_t netfs_n_wh_writethrough;
 extern atomic_t netfs_n_wh_dio_write;
 extern atomic_t netfs_n_wh_writepages;
 extern atomic_t netfs_n_wh_copy_to_cache;
@@ -203,11 +233,11 @@ void netfs_write_collection_worker(struct work_struct *work);
  */
 struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
 						struct file *file,
-						loff_t start,
+						uoff_t start,
 						enum netfs_io_origin origin);
 void netfs_prepare_write(struct netfs_io_request *wreq,
 			 struct netfs_io_stream *stream,
-			 loff_t start);
+			 uoff_t start);
 void netfs_reissue_write(struct netfs_io_stream *stream,
 			 struct netfs_io_subrequest *subreq,
 			 struct iov_iter *source);
@@ -215,13 +245,7 @@ void netfs_issue_write(struct netfs_io_request *wreq,
 		       struct netfs_io_stream *stream);
 size_t netfs_advance_write(struct netfs_io_request *wreq,
 			   struct netfs_io_stream *stream,
-			   loff_t start, size_t len, bool to_eof);
-struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
-int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
-			       struct folio *folio, size_t copied, bool to_page_end,
-			       struct folio **writethrough_cache);
-ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
-			       struct folio *writethrough_cache);
+			   uoff_t start, size_t len, bool to_eof);
 
 /*
  * write_retry.c
@@ -320,6 +344,26 @@ static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subreque
 	return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
 }
 
+/*
+ * Indicate that we've generated and queued all the subrequests we're going to.
+ */
+static inline void netfs_all_subreqs_queued(struct netfs_io_request *rreq)
+{
+	smp_wmb(); /* Write lists before ALL_QUEUED. */
+	set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+	smp_mb__after_atomic();
+	trace_netfs_rreq(rreq, netfs_rreq_trace_all_queued);
+}
+
+/*
+ * Query if all subrequests are queued.
+ */
+static inline bool netfs_are_all_subreqs_queued(const struct netfs_io_request *rreq)
+{
+	/* Read lists after ALL_QUEUED. */
+	return test_bit_acquire(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+}
+
 /*
  * fscache-cache.c
  */
diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c
index b375567e0520ed..eb1efb17f53aaa 100644
--- a/fs/netfs/iterator.c
+++ b/fs/netfs/iterator.c
@@ -209,7 +209,7 @@ static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offse
 {
 	struct folio *folio;
 	unsigned int nsegs = 0;
-	loff_t pos = iter->xarray_start + iter->iov_offset;
+	uoff_t pos = iter->xarray_start + iter->iov_offset;
 	pgoff_t index = pos / PAGE_SIZE;
 	size_t span = 0, n = iter->count;
 
diff --git a/fs/netfs/main.c b/fs/netfs/main.c
index 927badf3989db4..609e22e8f76a06 100644
--- a/fs/netfs/main.c
+++ b/fs/netfs/main.c
@@ -44,7 +44,6 @@ static const char *netfs_origins[nr__netfs_io_origin] = {
 	[NETFS_DIO_READ]		= "DR",
 	[NETFS_WRITEBACK]		= "WB",
 	[NETFS_WRITEBACK_SINGLE]	= "W1",
-	[NETFS_WRITETHROUGH]		= "WT",
 	[NETFS_UNBUFFERED_WRITE]	= "UW",
 	[NETFS_DIO_WRITE]		= "DW",
 	[NETFS_PGPRIV2_COPY_TO_CACHE]	= "2C",
diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c
index f5c1c463f4ff7a..a3cd76d584b8b9 100644
--- a/fs/netfs/misc.c
+++ b/fs/netfs/misc.c
@@ -193,7 +193,7 @@ void netfs_clear_inode_writeback(struct inode *inode, const void *aux)
 	struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode));
 
 	if (inode_state_read_once(inode) & I_PINNING_NETFS_WB) {
-		loff_t i_size = i_size_read(inode);
+		uoff_t i_size = i_size_read(inode);
 		fscache_unuse_cookie(cookie, aux, &i_size);
 	}
 }
@@ -218,8 +218,8 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 	_enter("{%lx},%zx,%zx", folio->index, offset, length);
 
 	if (offset == 0 && length == flen) {
-		unsigned long long i_size, remote_i_size, zero_point;
-		unsigned long long fpos = folio_pos(folio), end;
+		uoff_t i_size, remote_i_size, zero_point;
+		uoff_t fpos = folio_pos(folio), end;
 
 		netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
 		end = umin(fpos + flen, i_size);
@@ -305,7 +305,7 @@ bool netfs_release_folio(struct folio *folio, gfp_t gfp)
 {
 	struct inode *inode = folio_inode(folio);
 	struct netfs_inode *ctx = netfs_inode(inode);
-	unsigned long long i_size, remote_i_size, zero_point, end;
+	uoff_t i_size, remote_i_size, zero_point, end;
 
 	if (folio_test_dirty(folio))
 		return false;
@@ -424,7 +424,7 @@ static int netfs_collect_in_app(struct netfs_io_request *rreq,
 			need_collect = true;
 			break;
 		}
-		if (subreq || !test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags))
+		if (subreq || !netfs_are_all_subreqs_queued(rreq))
 			done = false;
 	}
 
diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c
index 7f6a3e912602ee..4b8d20559b0e15 100644
--- a/fs/netfs/objects.c
+++ b/fs/netfs/objects.c
@@ -16,7 +16,7 @@ static void netfs_free_request(struct work_struct *work);
  */
 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
 					     struct file *file,
-					     loff_t start, size_t len,
+					     uoff_t start, size_t len,
 					     enum netfs_io_origin origin)
 {
 	static atomic_t debug_ids;
@@ -34,7 +34,7 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
 
 		rreq = mempool_alloc(mempool, gfp);
 	} else {
-		rreq = mempool->alloc(gfp, mempool->pool_data);
+		rreq = mempool_alloc_noreserve(mempool, gfp);
 		if (!rreq)
 			return ERR_PTR(-ENOMEM);
 	}
@@ -44,6 +44,7 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
 	rreq->gfp		= gfp;
 	rreq->start		= start;
 	rreq->collected_to	= start;
+	rreq->cache_coll_to	= start;
 	rreq->cleaned_to	= start;
 	rreq->len		= len;
 	rreq->progress_at	= 0;
@@ -207,14 +208,15 @@ void netfs_put_failed_request(struct netfs_io_request *rreq)
 /*
  * Allocate and partially initialise an I/O request structure.
  */
-struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq)
+struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq,
+						   enum netfs_io_source source)
 {
 	struct netfs_io_subrequest *subreq;
 	mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool;
 	struct kmem_cache *cache = mempool->pool_data;
 
 	if (rreq->gfp == GFP_KERNEL)
-		subreq = mempool->alloc(rreq->gfp, mempool->pool_data);
+		subreq = mempool_alloc_noreserve(mempool, rreq->gfp);
 	else
 		subreq = mempool_alloc(mempool, rreq->gfp);
 	if (!subreq)
@@ -224,6 +226,7 @@ struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq
 	INIT_WORK(&subreq->work, NULL);
 	INIT_LIST_HEAD(&subreq->rreq_link);
 	refcount_set(&subreq->ref, 2);
+	subreq->source = source;
 	subreq->rreq = rreq;
 	subreq->debug_index = atomic_inc_return(&rreq->subreq_counter);
 	netfs_get_request(rreq, netfs_rreq_trace_get_subreq);
diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c
index 5cf22087d2439d..2625efd48a9b13 100644
--- a/fs/netfs/read_collect.c
+++ b/fs/netfs/read_collect.c
@@ -38,7 +38,7 @@ static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
  */
 void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
 {
-	if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
+	if (!netfs_using_pgpriv2(rreq)) {
 		if (folio_get_private(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
 			folio_detach_private(folio);
 			trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
@@ -81,7 +81,7 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
 	if (unlikely(test_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags)))
 		netfs_cancel_copy_to_cache(rreq, folio);
 
-	if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
+	if (!netfs_using_pgpriv2(rreq)) {
 		if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE)  {
 			trace_netfs_folio(folio, netfs_folio_trace_sched_copy);
 			folio_mark_dirty(folio);
@@ -153,8 +153,8 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
 				     unsigned int *notes)
 {
 	struct folio_queue *folioq = rreq->buffer.tail;
-	unsigned long long collected_to = rreq->collected_to;
 	unsigned int slot = rreq->buffer.first_tail_slot;
+	uoff_t collected_to = rreq->collected_to;
 
 	if (rreq->cleaned_to >= rreq->collected_to)
 		return;
@@ -179,7 +179,7 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
 
 	for (;;) {
 		struct folio *folio;
-		unsigned long long fpos, fend;
+		uoff_t fpos, fend;
 		size_t fsize;
 
 		folio = folioq_folio(folioq, slot);
@@ -192,7 +192,7 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
 		fpos = folio_pos(folio);
 		fend = fpos + fsize;
 
-		trace_netfs_collect_folio(rreq, folio, fend, collected_to);
+		trace_netfs_collect_folio(rreq, folio);
 
 		/* Unlock any folio we've transferred all of. */
 		if (collected_to < fend)
@@ -435,6 +435,11 @@ static void netfs_rreq_assess_single(struct netfs_io_request *rreq)
 		netfs_single_mark_inode_dirty(rreq->inode);
 	}
 
+	/* To do DIO, the cache has to round the size up, so we need to undo
+	 * the rounding.
+	 */
+	rreq->transferred = min(rreq->transferred, rreq->i_size);
+
 	if (rreq->iocb) {
 		rreq->iocb->ki_pos += rreq->transferred;
 		if (rreq->iocb->ki_complete) {
@@ -462,10 +467,8 @@ bool netfs_read_collection(struct netfs_io_request *rreq)
 	/* We're done when the app thread has finished posting subreqs and the
 	 * queue is empty.
 	 */
-	if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags))
+	if (!netfs_are_all_subreqs_queued(rreq))
 		return false;
-	smp_rmb(); /* Read ALL_QUEUED before subreq lists. */
-
 	if (!list_empty(&stream->subrequests))
 		return false;
 
diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c
index a4b7bb88cbdb60..5280b606fda410 100644
--- a/fs/netfs/read_pgpriv2.c
+++ b/fs/netfs/read_pgpriv2.c
@@ -20,7 +20,7 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio
 {
 	struct netfs_io_stream *cache = &creq->io_streams[1];
 	size_t fsize = folio_size(folio), flen = fsize;
-	loff_t fpos = folio_pos(folio), i_size;
+	uoff_t fpos = folio_pos(folio), i_size;
 	bool to_eof = false;
 
 	_enter("");
@@ -158,8 +158,7 @@ void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq)
 		return;
 
 	netfs_issue_write(creq, &creq->io_streams[1]);
-	smp_wmb(); /* Write lists before ALL_QUEUED. */
-	set_bit(NETFS_RREQ_ALL_QUEUED, &creq->flags);
+	netfs_all_subreqs_queued(creq);
 	trace_netfs_rreq(rreq, netfs_rreq_trace_end_copy_to_cache);
 	if (list_empty_careful(&creq->io_streams[1].subrequests))
 		netfs_wake_collector(creq);
@@ -175,8 +174,8 @@ void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq)
 bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *creq)
 {
 	struct folio_queue *folioq = creq->buffer.tail;
-	unsigned long long collected_to = creq->collected_to;
 	unsigned int slot = creq->buffer.first_tail_slot;
+	uoff_t collected_to = creq->collected_to;
 	bool made_progress = false;
 
 	if (slot >= folioq_nr_slots(folioq)) {
@@ -186,7 +185,7 @@ bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *creq)
 
 	for (;;) {
 		struct folio *folio;
-		unsigned long long fpos, fend;
+		uoff_t fpos, fend;
 		size_t fsize, flen;
 
 		folio = folioq_folio(folioq, slot);
@@ -199,9 +198,9 @@ bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *creq)
 		fsize = folio_size(folio);
 		flen = fsize;
 
-		fend = min_t(unsigned long long, fpos + flen, creq->i_size);
+		fend = min_t(uoff_t, fpos + flen, creq->i_size);
 
-		trace_netfs_collect_folio(creq, folio, fend, collected_to);
+		trace_netfs_collect_folio(creq, folio);
 
 		/* Unlock any folio we've transferred all of. */
 		if (collected_to < fend)
diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c
index 4f6a36c6e214f7..5bd8dee5a83467 100644
--- a/fs/netfs/read_retry.c
+++ b/fs/netfs/read_retry.c
@@ -75,7 +75,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
 	do {
 		struct netfs_io_subrequest *from, *to, *tmp;
 		struct iov_iter source;
-		unsigned long long start, len;
+		uoff_t start, len;
 		size_t part;
 		bool boundary = false, subreq_superfluous = false;
 
@@ -195,12 +195,11 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
 		 * and insert them after.
 		 */
 		do {
-			subreq = netfs_alloc_subrequest(rreq);
+			subreq = netfs_alloc_subrequest(rreq, NETFS_DOWNLOAD_FROM_SERVER);
 			if (!subreq) {
 				subreq = to;
 				goto abandon_after;
 			}
-			subreq->source		= NETFS_DOWNLOAD_FROM_SERVER;
 			subreq->start		= start;
 			subreq->len		= len;
 			subreq->stream_nr	= stream->stream_nr;
@@ -272,6 +271,7 @@ void netfs_retry_reads(struct netfs_io_request *rreq)
 	struct netfs_io_stream *stream = &rreq->io_streams[0];
 
 	netfs_stat(&netfs_n_rh_retry_read_req);
+	trace_netfs_rreq(rreq, netfs_rreq_trace_retry_begin);
 
 	/* Wait for all outstanding I/O to quiesce before performing retries as
 	 * we may need to renegotiate the I/O sizes.
@@ -282,6 +282,7 @@ void netfs_retry_reads(struct netfs_io_request *rreq)
 
 	trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
 	netfs_retry_read_subrequests(rreq);
+	trace_netfs_rreq(rreq, netfs_rreq_trace_retry_end);
 }
 
 /*
diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c
index de67ac41548d1a..b248e34bd0c86d 100644
--- a/fs/netfs/read_single.c
+++ b/fs/netfs/read_single.c
@@ -58,20 +58,6 @@ static int netfs_single_begin_cache_read(struct netfs_io_request *rreq, struct n
 	return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));
 }
 
-static void netfs_single_cache_prepare_read(struct netfs_io_request *rreq,
-					    struct netfs_io_subrequest *subreq)
-{
-	struct netfs_cache_resources *cres = &rreq->cache_resources;
-
-	if (!cres->ops) {
-		subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
-		return;
-	}
-	subreq->source = cres->ops->prepare_read(subreq, rreq->i_size);
-	trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
-
-}
-
 static void netfs_single_read_cache(struct netfs_io_request *rreq,
 				    struct netfs_io_subrequest *subreq)
 {
@@ -89,21 +75,36 @@ static void netfs_single_read_cache(struct netfs_io_request *rreq,
  */
 static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
 {
+	struct fscache_occupancy occ = {
+		.query_from	= 0,
+		.query_to	= rreq->len,
+		.cached_from[0]	= ULLONG_MAX,
+		.cached_to[0]	= ULLONG_MAX,
+		.cached_from[1]	= ULLONG_MAX,
+		.cached_to[1]	= ULLONG_MAX,
+	};
 	struct netfs_io_subrequest *subreq;
+	enum netfs_io_source source = NETFS_DOWNLOAD_FROM_SERVER;
 	int ret = 0;
 
-	subreq = netfs_alloc_subrequest(rreq);
+	/* Try to use the cache if the cache content matches the size of the
+	 * remote file.
+	 */
+	netfs_read_query_cache(rreq, &occ);
+	if (occ.cached_from[0] == 0 &&
+	    occ.cached_to[0] >= rreq->len)
+		source = NETFS_READ_FROM_CACHE;
+
+	subreq = netfs_alloc_subrequest(rreq, source);
 	if (!subreq)
 		return -ENOMEM;
 
-	subreq->source	= NETFS_SOURCE_UNKNOWN;
 	subreq->start	= 0;
 	subreq->len	= rreq->len;
 	subreq->io_iter	= rreq->buffer.iter;
 
 	netfs_queue_read(rreq, subreq);
 
-	netfs_single_cache_prepare_read(rreq, subreq);
 	switch (subreq->source) {
 	case NETFS_DOWNLOAD_FROM_SERVER:
 		netfs_stat(&netfs_n_rh_download);
@@ -113,14 +114,18 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
 				goto cancel;
 		}
 
-		smp_wmb(); /* Write lists before ALL_QUEUED. */
-		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+		netfs_all_subreqs_queued(rreq);
 		rreq->netfs_ops->issue_read(subreq);
 		rreq->submitted += subreq->len;
 		break;
 	case NETFS_READ_FROM_CACHE:
-		smp_wmb(); /* Write lists before ALL_QUEUED. */
-		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+		if (rreq->cache_resources.ops->prepare_read) {
+			ret = rreq->cache_resources.ops->prepare_read(subreq);
+			if (ret < 0)
+				goto cancel;
+		}
+
+		netfs_all_subreqs_queued(rreq);
 		trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
 		netfs_single_read_cache(rreq, subreq);
 		rreq->submitted += subreq->len;
@@ -136,8 +141,7 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
 	return ret;
 cancel:
 	netfs_cancel_read(subreq, ret);
-	smp_wmb(); /* Write lists before ALL_QUEUED. */
-	set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+	netfs_all_subreqs_queued(rreq);
 	netfs_wake_collector(rreq);
 	return ret;
 }
diff --git a/fs/netfs/rolling_buffer.c b/fs/netfs/rolling_buffer.c
index 424e77a9a1098e..d30d5ef6d86eca 100644
--- a/fs/netfs/rolling_buffer.c
+++ b/fs/netfs/rolling_buffer.c
@@ -29,7 +29,7 @@ struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp,
 	struct folio_queue *fq;
 
 	if (gfp == GFP_KERNEL)
-		fq = netfs_folioq_pool.alloc(gfp, netfs_folioq_pool.pool_data);
+		fq = mempool_alloc_noreserve(&netfs_folioq_pool, gfp);
 	else
 		fq = mempool_alloc(&netfs_folioq_pool, gfp);
 	if (fq) {
diff --git a/fs/netfs/stats.c b/fs/netfs/stats.c
index ab6b916addc443..9a607c4e62dda5 100644
--- a/fs/netfs/stats.c
+++ b/fs/netfs/stats.c
@@ -32,7 +32,6 @@ atomic_t netfs_n_rh_write_zskip;
 atomic_t netfs_n_rh_retry_read_req;
 atomic_t netfs_n_rh_retry_read_subreq;
 atomic_t netfs_n_wh_buffered_write;
-atomic_t netfs_n_wh_writethrough;
 atomic_t netfs_n_wh_dio_write;
 atomic_t netfs_n_wh_writepages;
 atomic_t netfs_n_wh_copy_to_cache;
@@ -58,9 +57,8 @@ int netfs_stats_show(struct seq_file *m, void *v)
 		   atomic_read(&netfs_n_rh_read_single),
 		   atomic_read(&netfs_n_rh_write_begin),
 		   atomic_read(&netfs_n_rh_write_zskip));
-	seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u 2C=%u\n",
+	seq_printf(m, "Writes : BW=%u DW=%u WP=%u 2C=%u\n",
 		   atomic_read(&netfs_n_wh_buffered_write),
-		   atomic_read(&netfs_n_wh_writethrough),
 		   atomic_read(&netfs_n_wh_dio_write),
 		   atomic_read(&netfs_n_wh_writepages),
 		   atomic_read(&netfs_n_wh_copy_to_cache));
diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c
index 210eb8f3958d77..6e8ea534230df0 100644
--- a/fs/netfs/write_collect.c
+++ b/fs/netfs/write_collect.c
@@ -56,7 +56,7 @@ static void netfs_dump_request(const struct netfs_io_request *rreq)
  */
 int netfs_folio_written_back(struct folio *folio)
 {
-	enum netfs_folio_trace why = netfs_folio_trace_clear;
+	enum netfs_folio_trace why = netfs_folio_trace_endwb;
 	struct inode *inode = folio_inode(folio);
 	struct netfs_inode *ictx = netfs_inode(inode);
 	struct netfs_folio *finfo;
@@ -67,7 +67,7 @@ int netfs_folio_written_back(struct folio *folio)
 		/* Streaming writes cannot be redirtied whilst under writeback,
 		 * so discard the streaming record.
 		 */
-		unsigned long long fend;
+		uoff_t fend;
 
 		fend = folio_pos(folio) + finfo->dirty_offset + finfo->dirty_len;
 		spin_lock(&ictx->inode.i_lock);
@@ -79,13 +79,13 @@ int netfs_folio_written_back(struct folio *folio)
 		group = finfo->netfs_group;
 		gcount++;
 		kfree(finfo);
-		why = netfs_folio_trace_clear_s;
+		why = netfs_folio_trace_endwb_s;
 		goto end_wb;
 	}
 
 	if ((group = netfs_folio_group(folio))) {
 		if (group == NETFS_FOLIO_COPY_TO_CACHE) {
-			why = netfs_folio_trace_clear_cc;
+			why = netfs_folio_trace_endwb_cc;
 			folio_detach_private(folio);
 			goto end_wb;
 		}
@@ -98,7 +98,7 @@ int netfs_folio_written_back(struct folio *folio)
 		if (!folio_test_dirty(folio)) {
 			folio_detach_private(folio);
 			gcount++;
-			why = netfs_folio_trace_clear_g;
+			why = netfs_folio_trace_endwb_g;
 		}
 	}
 
@@ -115,8 +115,8 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq,
 					  unsigned int *notes)
 {
 	struct folio_queue *folioq = wreq->buffer.tail;
-	unsigned long long collected_to = wreq->collected_to;
 	unsigned int slot = wreq->buffer.first_tail_slot;
+	uoff_t collected_to = wreq->collected_to;
 
 	if (WARN_ON_ONCE(!folioq)) {
 		pr_err("[!] Writeback unlock found empty rolling buffer!\n");
@@ -140,7 +140,7 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq,
 	for (;;) {
 		struct folio *folio;
 		struct netfs_folio *finfo;
-		unsigned long long fpos, fend;
+		uoff_t fpos, fend;
 		size_t fsize, flen;
 
 		folio = folioq_folio(folioq, slot);
@@ -154,9 +154,9 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq,
 		finfo = netfs_folio_info(folio);
 		flen = finfo ? finfo->dirty_offset + finfo->dirty_len : fsize;
 
-		fend = min_t(unsigned long long, fpos + flen, wreq->i_size);
+		fend = min_t(uoff_t, fpos + flen, wreq->i_size);
 
-		trace_netfs_collect_folio(wreq, folio, fend, collected_to);
+		trace_netfs_collect_folio(wreq, folio);
 
 		/* Unlock any folio we've transferred all of. */
 		if (collected_to < fend)
@@ -188,6 +188,26 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq,
 	wreq->buffer.first_tail_slot = slot;
 }
 
+/*
+ * Collect cache results.
+ */
+static void netfs_cache_collect(struct netfs_io_request *wreq,
+				struct netfs_io_stream *stream,
+				enum netfs_cache_collect block_type)
+{
+	struct netfs_cache_resources *cres = &wreq->cache_resources;
+
+	if (stream->source != NETFS_WRITE_TO_CACHE ||
+	    wreq->cache_coll_to >= stream->collected_to)
+		return;
+
+	if (cres->ops && cres->ops->collect_write)
+		cres->ops->collect_write(wreq, wreq->cache_coll_to,
+					 stream->collected_to - wreq->cache_coll_to,
+					 block_type);
+	wreq->cache_coll_to = stream->collected_to;
+}
+
 /*
  * Collect and assess the results of various write subrequests.  We may need to
  * retry some of the results - or even do an RMW cycle for content crypto.
@@ -201,8 +221,8 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
 {
 	struct netfs_io_subrequest *front, *remove;
 	struct netfs_io_stream *stream;
-	unsigned long long collected_to, issued_to;
 	unsigned int notes;
+	uoff_t collected_to, issued_to;
 	int s;
 
 	_enter("%llx-%llx", wreq->start, wreq->start + wreq->len);
@@ -214,7 +234,6 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
 	smp_rmb();
 	collected_to = ULLONG_MAX;
 	if (wreq->origin == NETFS_WRITEBACK ||
-	    wreq->origin == NETFS_WRITETHROUGH ||
 	    wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE)
 		notes = NEED_UNLOCK;
 	else
@@ -236,13 +255,19 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
 		/* Read first subreq pointer before IN_PROGRESS flag. */
 
 		while (front) {
+			enum netfs_cache_collect cache_collect;
+
 			trace_netfs_collect_sreq(wreq, front);
 			//_debug("sreq [%x] %llx %zx/%zx",
 			//       front->debug_index, front->start, front->transferred, front->len);
 
 			if (stream->collected_to < front->start) {
 				trace_netfs_collect_gap(wreq, stream, issued_to, 'F');
+				if (stream->cache_collect != NETFS_CACHE_COLLECT_WRITE_GAP)
+					netfs_cache_collect(wreq, stream, stream->cache_collect);
 				stream->collected_to = front->start;
+				netfs_cache_collect(wreq, stream, NETFS_CACHE_COLLECT_WRITE_GAP);
+				stream->cache_collect = NETFS_CACHE_COLLECT_WRITE_GAP;
 			}
 
 			/* Stall if the front is still undergoing I/O. */
@@ -250,7 +275,6 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
 				notes |= HIT_PENDING;
 				break;
 			}
-			smp_rmb(); /* Read counters after I-P flag. */
 
 			if (stream->failed) {
 				stream->collected_to = front->start + front->len;
@@ -263,15 +287,44 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
 				stream->transferred_valid = true;
 				notes |= MADE_PROGRESS;
 			}
-			if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {
-				stream->failed = true;
-				stream->error = front->error;
-				if (stream->source == NETFS_UPLOAD_TO_SERVER)
-					mapping_set_error(wreq->mapping, front->error);
-				notes |= NEED_REASSESS | SAW_FAILURE;
+
+			/* Handle failed or cancelled subreqs.  Failure of
+			 * cache writes are handled differently to upload
+			 * failures.  Cache writes aren't fatal, provided we're
+			 * not doing disconnected operation, and so we can kind
+			 * of treat them as if they had succeeded - except that
+			 * we need to log any holes they cause.
+			 */
+			switch (stream->source) {
+			case NETFS_UPLOAD_TO_SERVER:
+				if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {
+					if (!stream->failed) {
+						stream->failed = true;
+						stream->error = front->error;
+						mapping_set_error(wreq->mapping, front->error);
+						break;
+					}
+					notes |= NEED_REASSESS | SAW_FAILURE;
+				}
+				break;
+
+			case NETFS_WRITE_TO_CACHE:
+				cache_collect = test_bit(NETFS_SREQ_CANCELLED, &front->flags) ?
+					NETFS_CACHE_COLLECT_WRITE_CANCEL :
+					NETFS_CACHE_COLLECT_WRITE_DATA;
+				if (cache_collect != stream->cache_collect &&
+				    stream->cache_collect != NETFS_CACHE_COLLECT_WRITE_GAP) {
+					trace_netfs_rreq(wreq, netfs_rreq_trace_cache_fail_collect);
+					netfs_cache_collect(wreq, stream, stream->cache_collect);
+				}
+				stream->cache_collect = cache_collect;
+				break;
+
+			default:
+				WARN_ON(1);
 				break;
 			}
-			if (front->transferred < front->len) {
+			if (test_bit(NETFS_SREQ_NEED_RETRY, &front->flags)) {
 				stream->need_retry = true;
 				notes |= NEED_RETRY | MADE_PROGRESS;
 				break;
@@ -360,6 +413,7 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
  */
 bool netfs_write_collection(struct netfs_io_request *wreq)
 {
+	struct netfs_io_stream *cstream = &wreq->io_streams[1];
 	struct netfs_inode *ictx = netfs_inode(wreq->inode);
 	size_t transferred;
 	bool transferred_valid = false;
@@ -372,9 +426,8 @@ bool netfs_write_collection(struct netfs_io_request *wreq)
 	/* We're done when the app thread has finished posting subreqs and all
 	 * the queues in all the streams are empty.
 	 */
-	if (!test_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags))
+	if (!netfs_are_all_subreqs_queued(wreq))
 		return false;
-	smp_rmb(); /* Read ALL_QUEUED before lists. */
 
 	transferred = LONG_MAX;
 	for (s = 0; s < NR_IO_STREAMS; s++) {
@@ -395,13 +448,19 @@ bool netfs_write_collection(struct netfs_io_request *wreq)
 		wreq->transferred = transferred;
 	trace_netfs_rreq(wreq, netfs_rreq_trace_write_done);
 
-	if (wreq->io_streams[1].active &&
-	    wreq->io_streams[1].failed &&
-	    ictx->ops->invalidate_cache) {
-		/* Cache write failure doesn't prevent writeback completion
-		 * unless we're in disconnected mode.
-		 */
-		ictx->ops->invalidate_cache(wreq);
+	if (cstream->active) {
+		if (test_bit(NETFS_RREQ_CACHE_ERROR, &wreq->flags)) {
+			if (ictx->ops->invalidate_cache) {
+				/* Cache write failure doesn't prevent
+				 * writeback completion unless we're in
+				 * disconnected mode.
+				 */
+				trace_netfs_rreq(wreq, netfs_rreq_trace_inval_cache);
+				ictx->ops->invalidate_cache(wreq);
+			}
+		} else if (!cstream->failed) {
+			netfs_cache_collect(wreq, cstream, cstream->cache_collect);
+		}
 	}
 
 	_debug("finished");
@@ -411,7 +470,6 @@ bool netfs_write_collection(struct netfs_io_request *wreq)
 	switch (wreq->origin) {
 	case NETFS_WRITEBACK:
 	case NETFS_WRITEBACK_SINGLE:
-	case NETFS_WRITETHROUGH:
 		netfs_wb_end(ictx);
 		break;
 	default:
@@ -486,24 +544,51 @@ void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error)
 
 	if (IS_ERR_VALUE(transferred_or_error)) {
 		subreq->error = transferred_or_error;
-		/* if need retry is set, error should not matter */
-		if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
-			set_bit(NETFS_SREQ_FAILED, &subreq->flags);
-			trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write);
-		}
 
 		switch (subreq->source) {
 		case NETFS_WRITE_TO_CACHE:
+			/* We don't mark a cache-write subreq as failed.
+			 * Instead we tell the issuer to produce dummy subreqs
+			 * instead and make a note if we need to invalidate the
+			 * cache at the end.  We also don't pause the loop that
+			 * grabs pages and launches upload subreqs.
+			 *
+			 * Note that we need to distinguish between -ENOBUFS
+			 * (no space available in the cache) and other errors.
+			 * In the former case, we can keep the data we have,
+			 * though we might have to change the way the on-disk
+			 * data is tracked.
+			 */
 			netfs_stat(&netfs_n_wh_write_failed);
+			if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
+				break;
+
+			trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write);
+			__set_bit(NETFS_SREQ_CANCELLED, &subreq->flags);
+			set_bit(NETFS_RREQ_CACHE_STOP, &wreq->flags);
+			if (transferred_or_error == -ENOBUFS)
+				trace_netfs_rreq(wreq, netfs_rreq_trace_cache_no_space);
+			else if (!test_and_set_bit(NETFS_RREQ_CACHE_ERROR, &wreq->flags))
+				trace_netfs_rreq(wreq, netfs_rreq_trace_cache_failed);
+			subreq->transferred = subreq->len;
 			break;
+
 		case NETFS_UPLOAD_TO_SERVER:
+			/* If need_retry is set, error should not matter */
+			if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
+				set_bit(NETFS_SREQ_FAILED, &subreq->flags);
+				trace_netfs_failure(wreq, subreq, transferred_or_error,
+						    netfs_fail_upload);
+			}
+
+			set_bit(NETFS_RREQ_PAUSE, &wreq->flags);
+			trace_netfs_rreq(wreq, netfs_rreq_trace_set_pause);
 			netfs_stat(&netfs_n_wh_upload_failed);
 			break;
+
 		default:
 			break;
 		}
-		trace_netfs_rreq(wreq, netfs_rreq_trace_set_pause);
-		set_bit(NETFS_RREQ_PAUSE, &wreq->flags);
 	} else {
 		if (WARN(transferred_or_error > subreq->len - subreq->transferred,
 			 "Subreq excess write: R=%x[%x] %zd > %zu - %zu",
diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c
index 851f6f93ad45ab..3989b4ec0c4b43 100644
--- a/fs/netfs/write_issue.c
+++ b/fs/netfs/write_issue.c
@@ -89,14 +89,13 @@ static void netfs_kill_dirty_pages(struct address_space *mapping,
  */
 struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
 						struct file *file,
-						loff_t start,
+						uoff_t start,
 						enum netfs_io_origin origin)
 {
 	struct netfs_io_request *wreq;
 	struct netfs_inode *ictx;
 	bool is_cacheable = (origin == NETFS_WRITEBACK ||
 			     origin == NETFS_WRITEBACK_SINGLE ||
-			     origin == NETFS_WRITETHROUGH ||
 			     origin == NETFS_PGPRIV2_COPY_TO_CACHE);
 
 	wreq = netfs_alloc_request(mapping, file, start, 0, origin);
@@ -112,6 +111,8 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping,
 		goto nomem;
 
 	wreq->cleaned_to = wreq->start;
+	if (wreq->cache_resources.dio_size > 1)
+		wreq->cache_coll_to = round_down(wreq->start, wreq->cache_resources.dio_size);
 
 	wreq->io_streams[0].stream_nr		= 0;
 	wreq->io_streams[0].source		= NETFS_UPLOAD_TO_SERVER;
@@ -156,7 +157,7 @@ EXPORT_SYMBOL(netfs_prepare_write_failed);
  */
 void netfs_prepare_write(struct netfs_io_request *wreq,
 			 struct netfs_io_stream *stream,
-			 loff_t start)
+			 uoff_t start)
 {
 	struct netfs_io_subrequest *subreq;
 	struct iov_iter *wreq_iter = &wreq->buffer.iter;
@@ -169,10 +170,9 @@ void netfs_prepare_write(struct netfs_io_request *wreq,
 	    wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq))
 		rolling_buffer_make_space(&wreq->buffer, wreq->gfp);
 
-	subreq = netfs_alloc_subrequest(wreq);
+	subreq = netfs_alloc_subrequest(wreq, stream->source);
 	if (!subreq)
 		return;
-	subreq->source		= stream->source;
 	subreq->start		= start;
 	subreq->stream_nr	= stream->stream_nr;
 	subreq->io_iter		= *wreq_iter;
@@ -233,6 +233,21 @@ static void netfs_do_issue_write(struct netfs_io_stream *stream,
 
 	_enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len);
 
+	if (stream->source == NETFS_WRITE_TO_CACHE &&
+	    unlikely(test_bit(NETFS_RREQ_CACHE_STOP, &wreq->flags))) {
+		size_t dio_size = wreq->cache_resources.dio_size;
+		size_t len, disp;
+
+		disp = subreq->start & (dio_size - 1);
+		len = round_up(subreq->len + disp, dio_size);
+
+		subreq->start -= disp;
+		subreq->len = len;
+
+		__set_bit(NETFS_SREQ_CANCELLED, &subreq->flags);
+		return netfs_write_subrequest_terminated(subreq, subreq->len);
+	}
+
 	if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
 		return netfs_write_subrequest_terminated(subreq, subreq->error);
 
@@ -266,6 +281,7 @@ void netfs_issue_write(struct netfs_io_request *wreq,
 
 	if (!subreq)
 		return;
+
 	stream->construct = NULL;
 	subreq->io_iter.count = subreq->len;
 	netfs_do_issue_write(stream, subreq);
@@ -279,7 +295,7 @@ void netfs_issue_write(struct netfs_io_request *wreq,
  */
 size_t netfs_advance_write(struct netfs_io_request *wreq,
 			   struct netfs_io_stream *stream,
-			   loff_t start, size_t len, bool to_eof)
+			   uoff_t start, size_t len, bool to_eof)
 {
 	struct netfs_io_subrequest *subreq = stream->construct;
 	size_t part;
@@ -330,7 +346,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq,
 	struct netfs_folio *finfo;
 	size_t iter_off = 0;
 	size_t fsize = folio_size(folio), flen = fsize, foff = 0;
-	loff_t fpos = folio_pos(folio), i_size;
+	uoff_t fpos = folio_pos(folio), i_size;
 	bool to_eof = false, streamw = false;
 	bool debug = false;
 
@@ -367,11 +383,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq,
 		streamw = true;
 	}
 
-	if (wreq->origin == NETFS_WRITETHROUGH) {
-		to_eof = false;
-		if (flen > i_size - fpos)
-			flen = i_size - fpos;
-	} else if (flen > i_size - fpos) {
+	if (flen > i_size - fpos) {
 		flen = i_size - fpos;
 		if (!streamw)
 			folio_zero_segment(folio, flen, fsize);
@@ -525,8 +537,7 @@ static void netfs_end_issue_write(struct netfs_io_request *wreq)
 {
 	bool needs_poke = true;
 
-	smp_wmb(); /* Write subreq lists before ALL_QUEUED. */
-	set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
+	netfs_all_subreqs_queued(wreq);
 
 	for (int s = 0; s < NR_IO_STREAMS; s++) {
 		struct netfs_io_stream *stream = &wreq->io_streams[s];
@@ -613,103 +624,6 @@ int netfs_writepages(struct address_space *mapping,
 }
 EXPORT_SYMBOL(netfs_writepages);
 
-/*
- * Begin a write operation for writing through the pagecache.
- */
-struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len)
-{
-	struct netfs_io_request *wreq = NULL;
-	struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp));
-
-	netfs_wb_begin(ictx, false);
-
-	wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp,
-				      iocb->ki_pos, NETFS_WRITETHROUGH);
-	if (IS_ERR(wreq)) {
-		netfs_wb_end(ictx);
-		return wreq;
-	}
-
-	wreq->io_streams[0].avail = true;
-	__set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags);
-	trace_netfs_write(wreq, netfs_write_trace_writethrough);
-	return wreq;
-}
-
-/*
- * Advance the state of the write operation used when writing through the
- * pagecache.  Data has been copied into the pagecache that we need to append
- * to the request.  If we've added more than wsize then we need to create a new
- * subrequest.
- */
-int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
-			       struct folio *folio, size_t copied, bool to_page_end,
-			       struct folio **writethrough_cache)
-{
-	int ret;
-
-	_enter("R=%x ic=%zu ws=%u cp=%zu tp=%u",
-	       wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end);
-
-	/* The folio is locked. */
-
-	if (*writethrough_cache != folio) {
-		if (*writethrough_cache) {
-			/* Did the folio get moved? */
-			folio_put(*writethrough_cache);
-			*writethrough_cache = NULL;
-		}
-		/* We can make multiple writes to the folio... */
-		if (wreq->len == 0)
-			trace_netfs_folio(folio, netfs_folio_trace_wthru);
-		else
-			trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
-		*writethrough_cache = folio;
-		folio_get(folio);
-	}
-
-	wreq->len += copied;
-
-	if (!to_page_end) {
-		folio_mark_dirty(folio);
-		folio_unlock(folio);
-		return 0;
-	}
-
-	ret = netfs_write_folio(wreq, wbc, folio);
-	folio_put(*writethrough_cache);
-	*writethrough_cache = NULL;
-	wreq->submitted = wreq->len;
-	return ret;
-}
-
-/*
- * End a write operation used when writing through the pagecache.
- */
-ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
-			       struct folio *writethrough_cache)
-{
-	ssize_t ret;
-
-	_enter("R=%x", wreq->debug_id);
-
-	if (writethrough_cache) {
-		folio_lock(writethrough_cache);
-		netfs_write_folio(wreq, wbc, writethrough_cache);
-		folio_put(writethrough_cache);
-		wreq->submitted = wreq->len;
-	}
-
-	netfs_end_issue_write(wreq);
-
-	if (wreq->iocb)
-		ret = -EIOCBQUEUED;
-	else
-		ret = netfs_wait_for_write(wreq);
-	netfs_put_request(wreq, netfs_rreq_trace_put_return);
-	return ret;
-}
-
 /*
  * Write some of a pending folio data back to the server and/or the cache.
  */
@@ -721,7 +635,7 @@ static int netfs_write_folio_single(struct netfs_io_request *wreq,
 	struct netfs_io_stream *stream;
 	size_t iter_off = 0;
 	size_t fsize = folio_size(folio), flen;
-	loff_t fpos = folio_pos(folio);
+	uoff_t fpos = folio_pos(folio);
 	ssize_t ret;
 	bool to_eof = false;
 	bool no_debug = false;
@@ -891,8 +805,7 @@ int netfs_writeback_single(struct address_space *mapping,
 stop:
 	for (int s = 0; s < NR_IO_STREAMS; s++)
 		netfs_issue_write(wreq, &wreq->io_streams[s]);
-	smp_wmb(); /* Write lists before ALL_QUEUED. */
-	set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
+	netfs_all_subreqs_queued(wreq);
 
 	netfs_wake_collector(wreq);
 
diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c
index 058bc7a166a59f..2f20577563e145 100644
--- a/fs/netfs/write_retry.c
+++ b/fs/netfs/write_retry.c
@@ -55,7 +55,7 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
 	do {
 		struct netfs_io_subrequest *subreq = NULL, *from, *to, *tmp;
 		struct iov_iter source;
-		unsigned long long start, len;
+		uoff_t start, len;
 		size_t part;
 		bool boundary = false;
 
@@ -149,8 +149,7 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
 		 * and insert them after.
 		 */
 		do {
-			subreq = netfs_alloc_subrequest(wreq);
-			subreq->source		= to->source;
+			subreq = netfs_alloc_subrequest(wreq, stream->source);
 			subreq->start		= start;
 			subreq->stream_nr	= to->stream_nr;
 			subreq->retry_count	= 1;
@@ -211,6 +210,7 @@ void netfs_retry_writes(struct netfs_io_request *wreq)
 	int s;
 
 	netfs_stat(&netfs_n_wh_retry_write_req);
+	trace_netfs_rreq(wreq, netfs_rreq_trace_retry_begin);
 
 	/* Wait for all outstanding I/O to quiesce before performing retries as
 	 * we may need to renegotiate the I/O sizes.
@@ -235,4 +235,6 @@ void netfs_retry_writes(struct netfs_io_request *wreq)
 			netfs_retry_write_stream(wreq, stream);
 		}
 	}
+
+	trace_netfs_rreq(wreq, netfs_rreq_trace_retry_end);
 }
diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig
index 6bb30543eff00f..fd430718d02b40 100644
--- a/fs/nfs/Kconfig
+++ b/fs/nfs/Kconfig
@@ -123,8 +123,23 @@ config PNFS_FILE_LAYOUT
 
 config PNFS_BLOCK
 	tristate
-	depends on NFS_V4 && BLK_DEV_DM
+
+config PNFS_BLOCK_LAYOUT
+	bool "NFS client support for pNFS block layouts"
+	depends on NFS_V4 && BLOCK && BLK_DEV_DM
+	select PNFS_BLOCK
+	help
+	  Enable support for the pNFS block-volume layout type (RFC 5663).
+
+	  If unsure, say N.
+
+config PNFS_SCSI_LAYOUT
+	bool "NFS client support for pNFS SCSI layouts"
+	depends on NFS_V4 && BLOCK
 	default NFS_V4
+	select PNFS_BLOCK
+	help
+	  Enable suport for the pNFS SCSI layout type (RFC 8154).
 
 config PNFS_FLEXFILE_LAYOUT
 	tristate
@@ -174,6 +189,7 @@ config NFS_FSCACHE
 	bool "Provide NFS client caching support"
 	depends on NFS_FS
 	select NETFS_SUPPORT
+	select NETFS_PGPRIV2
 	select FSCACHE
 	help
 	  Say Y here if you want NFS data to be cached locally on disc through
diff --git a/fs/nfs/blocklayout/Makefile b/fs/nfs/blocklayout/Makefile
index 7668a1bfb5fa57..3403cb7fe201c1 100644
--- a/fs/nfs/blocklayout/Makefile
+++ b/fs/nfs/blocklayout/Makefile
@@ -4,4 +4,5 @@
 #
 obj-$(CONFIG_PNFS_BLOCK) += blocklayoutdriver.o
 
-blocklayoutdriver-y += blocklayout.o dev.o extent_tree.o rpc_pipefs.o
+blocklayoutdriver-y += blocklayout.o dev.o extent_tree.o
+blocklayoutdriver-$(CONFIG_PNFS_BLOCK_LAYOUT) += rpc_pipefs.o
diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c
index d54a141a89b3bf..b7a3211c587a7f 100644
--- a/fs/nfs/blocklayout/blocklayout.c
+++ b/fs/nfs/blocklayout/blocklayout.c
@@ -34,6 +34,7 @@
 #include <linux/init.h>
 #include <linux/mount.h>
 #include <linux/namei.h>
+#include <linux/pagemap.h>
 #include <linux/bio.h>		/* struct bio */
 #include <linux/prefetch.h>
 
@@ -470,18 +471,6 @@ static struct pnfs_layout_hdr *__bl_alloc_layout_hdr(struct inode *inode,
 	return &bl->bl_layout;
 }
 
-static struct pnfs_layout_hdr *bl_alloc_layout_hdr(struct inode *inode,
-						   gfp_t gfp_flags)
-{
-	return __bl_alloc_layout_hdr(inode, gfp_flags, false);
-}
-
-static struct pnfs_layout_hdr *sl_alloc_layout_hdr(struct inode *inode,
-						   gfp_t gfp_flags)
-{
-	return __bl_alloc_layout_hdr(inode, gfp_flags, true);
-}
-
 static void bl_free_lseg(struct pnfs_layout_segment *lseg)
 {
 	dprintk("%s enter\n", __func__);
@@ -859,7 +848,8 @@ bl_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
 
 	if (pgio->pg_lseg &&
 		test_bit(NFS_LSEG_UNAVAILABLE, &pgio->pg_lseg->pls_flags)) {
-		pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg);
+		pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg,
+						  NULL);
 		pnfs_set_lo_fail(pgio->pg_lseg);
 		nfs_pageio_reset_read_mds(pgio);
 	}
@@ -921,7 +911,8 @@ bl_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
 	if (pgio->pg_lseg &&
 		test_bit(NFS_LSEG_UNAVAILABLE, &pgio->pg_lseg->pls_flags)) {
 
-		pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg);
+		pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg,
+						  NULL);
 		pnfs_set_lo_fail(pgio->pg_lseg);
 		nfs_pageio_reset_write_mds(pgio);
 	}
@@ -954,6 +945,13 @@ static const struct nfs_pageio_ops bl_pg_write_ops = {
 	.pg_cleanup = pnfs_generic_pg_cleanup,
 };
 
+#ifdef CONFIG_PNFS_BLOCK_LAYOUT
+static struct pnfs_layout_hdr *bl_alloc_layout_hdr(struct inode *inode,
+						   gfp_t gfp_flags)
+{
+	return __bl_alloc_layout_hdr(inode, gfp_flags, false);
+}
+
 static struct pnfs_layoutdriver_type blocklayout_type = {
 	.id				= LAYOUT_BLOCK_VOLUME,
 	.name				= "LAYOUT_BLOCK_VOLUME",
@@ -978,6 +976,48 @@ static struct pnfs_layoutdriver_type blocklayout_type = {
 	.sync				= pnfs_generic_sync,
 };
 
+static int __init pnfs_register_blocklayout(void)
+{
+	int ret;
+
+	ret = bl_init_pipefs();
+	if (ret)
+		return ret;
+
+	ret = pnfs_register_layoutdriver(&blocklayout_type);
+	if (ret) {
+		bl_cleanup_pipefs();
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit pnfs_unregister_blocklayout(void)
+{
+	pnfs_unregister_layoutdriver(&blocklayout_type);
+	bl_cleanup_pipefs();
+}
+
+MODULE_ALIAS("nfs-layouttype4-3");
+#else
+static int __init pnfs_register_blocklayout(void)
+{
+	return 0;
+}
+
+static void __exit pnfs_unregister_blocklayout(void)
+{
+}
+#endif /* CONFIG_PNFS_BLOCK_LAYOUT */
+
+#ifdef CONFIG_PNFS_SCSI_LAYOUT
+static struct pnfs_layout_hdr *sl_alloc_layout_hdr(struct inode *inode,
+						   gfp_t gfp_flags)
+{
+	return __bl_alloc_layout_hdr(inode, gfp_flags, true);
+}
+
 static struct pnfs_layoutdriver_type scsilayout_type = {
 	.id				= LAYOUT_SCSI,
 	.name				= "LAYOUT_SCSI",
@@ -1002,6 +1042,28 @@ static struct pnfs_layoutdriver_type scsilayout_type = {
 	.sync				= pnfs_generic_sync,
 };
 
+static int __init pnfs_register_scsilayout(void)
+{
+	return pnfs_register_layoutdriver(&scsilayout_type);
+}
+
+static void __exit pnfs_unregister_scsilayout(void)
+{
+	pnfs_unregister_layoutdriver(&scsilayout_type);
+}
+
+MODULE_ALIAS("nfs-layouttype4-5");
+#else
+static int __init pnfs_register_scsilayout(void)
+{
+	return 0;
+}
+
+static void __exit pnfs_unregister_scsilayout(void)
+{
+}
+#endif /* CONFIG_PNFS_SCSI_LAYOUT */
+
 
 static int __init nfs4blocklayout_init(void)
 {
@@ -1009,25 +1071,17 @@ static int __init nfs4blocklayout_init(void)
 
 	dprintk("%s: NFSv4 Block Layout Driver Registering...\n", __func__);
 
-	ret = bl_init_pipefs();
+	ret = pnfs_register_blocklayout();
 	if (ret)
-		goto out;
+		return ret;
 
-	ret = pnfs_register_layoutdriver(&blocklayout_type);
-	if (ret)
-		goto out_cleanup_pipe;
+	ret = pnfs_register_scsilayout();
+	if (ret) {
+		pnfs_unregister_blocklayout();
+		return ret;
+	}
 
-	ret = pnfs_register_layoutdriver(&scsilayout_type);
-	if (ret)
-		goto out_unregister_block;
 	return 0;
-
-out_unregister_block:
-	pnfs_unregister_layoutdriver(&blocklayout_type);
-out_cleanup_pipe:
-	bl_cleanup_pipefs();
-out:
-	return ret;
 }
 
 static void __exit nfs4blocklayout_exit(void)
@@ -1035,13 +1089,9 @@ static void __exit nfs4blocklayout_exit(void)
 	dprintk("%s: NFSv4 Block Layout Driver Unregistering...\n",
 	       __func__);
 
-	pnfs_unregister_layoutdriver(&scsilayout_type);
-	pnfs_unregister_layoutdriver(&blocklayout_type);
-	bl_cleanup_pipefs();
+	pnfs_unregister_scsilayout();
+	pnfs_unregister_blocklayout();
 }
 
-MODULE_ALIAS("nfs-layouttype4-3");
-MODULE_ALIAS("nfs-layouttype4-5");
-
 module_init(nfs4blocklayout_init);
 module_exit(nfs4blocklayout_exit);
diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c
index c926b7e4382791..00462e6affb12f 100644
--- a/fs/nfs/blocklayout/dev.c
+++ b/fs/nfs/blocklayout/dev.c
@@ -15,6 +15,7 @@
 
 #define NFSDBG_FACILITY		NFSDBG_PNFS_LD
 
+#ifdef CONFIG_PNFS_SCSI_LAYOUT
 static void bl_unregister_scsi(struct pnfs_block_dev *dev)
 {
 	struct block_device *bdev = file_bdev(dev->bdev_file);
@@ -45,6 +46,16 @@ static bool bl_register_scsi(struct pnfs_block_dev *dev)
 	trace_bl_pr_key_reg(bdev, dev->pr_key);
 	return true;
 }
+#else
+static void bl_unregister_scsi(struct pnfs_block_dev *dev)
+{
+}
+
+static bool bl_register_scsi(struct pnfs_block_dev *dev)
+{
+	return false;
+}
+#endif /* CONFIG_PNFS_SCSI_LAYOUT */
 
 static void bl_unregister_dev(struct pnfs_block_dev *dev)
 {
@@ -292,7 +303,7 @@ static int
 bl_parse_deviceid(struct nfs_server *server, struct pnfs_block_dev *d,
 		struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask);
 
-
+#ifdef CONFIG_PNFS_BLOCK_LAYOUT
 static int
 bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d,
 		struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask)
@@ -320,7 +331,17 @@ bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d,
 		file_bdev(bdev_file)->bd_disk->disk_name);
 	return 0;
 }
+#else
+static int
+bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d,
+                struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask)
+{
+	dprintk("unsupported volume type: %d\n", PNFS_BLOCK_VOLUME_SIMPLE);
+	return -EIO;
+}
+#endif /* CONFIG_PNFS_BLOCK_LAYOUT */
 
+#ifdef CONFIG_PNFS_SCSI_LAYOUT
 static bool
 bl_validate_designator(struct pnfs_block_volume *v)
 {
@@ -449,6 +470,15 @@ bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d,
 	d->bdev_file = NULL;
 	return error;
 }
+#else
+static int
+bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d,
+		struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask)
+{
+	dprintk("unsupported volume type: %d\n", PNFS_BLOCK_VOLUME_SCSI);
+	return -EIO;
+}
+#endif /* CONFIG_PNFS_SCSI_LAYOUT */
 
 static int
 bl_parse_slice(struct nfs_server *server, struct pnfs_block_dev *d,
diff --git a/fs/nfs/cache_lib.c b/fs/nfs/cache_lib.c
index 9738a1ae92ca6a..ca4e81d4e31550 100644
--- a/fs/nfs/cache_lib.c
+++ b/fs/nfs/cache_lib.c
@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2009 Trond Myklebust <Trond.Myklebust@netapp.com>
  */
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/mount.h>
diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 3fb10c8e427187..9d836cb72e781d 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -292,7 +292,7 @@ static u32 initiate_file_draining(struct nfs_client *clp,
 	switch (pnfs_mark_matching_lsegs_return(lo, &free_me_list,
 				&args->cbl_range,
 				be32_to_cpu(args->cbl_stateid.seqid),
-				args->cbl_layoutchanged)) {
+				args->cbl_layoutchanged, NULL)) {
 	case 0:
 	case -EBUSY:
 		/* There are layout segments that need to be returned */
@@ -391,7 +391,26 @@ __be32 nfs4_callback_devicenotify(void *argp, void *resp,
 			if (!ld)
 				continue;
 		}
-		nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id);
+		/*
+		 * Bump the epoch before touching the cache so a
+		 * GETDEVICEINFO already in flight can detect that it
+		 * predates the notification.  A referenced DELETE may be
+		 * racing revocation, so defer it to the state manager --
+		 * this thread cannot issue fore-channel RPCs.
+		 */
+		nfs4_deviceid_bump_change_epoch(cps->clp);
+		if (dev->cbd_notify_type == NOTIFY_DEVICEID4_CHANGE) {
+			nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id);
+			pnfs_layout_reresolve_deviceid_byclid(cps->clp, ld,
+							&dev->cbd_dev_id,
+							dev->cbd_immediate);
+		} else if (pnfs_layout_deviceid_referenced_byclid(cps->clp,
+						ld, &dev->cbd_dev_id)) {
+			pnfs_deviceid_delete_mark(cps->clp, ld,
+						  &dev->cbd_dev_id);
+		} else {
+			nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id);
+		}
 	}
 	pnfs_put_layoutdriver(ld);
 out:
diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 88e1af0fda01aa..5a1f054692689e 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -271,6 +271,13 @@ __be32 decode_devicenotify_args(struct svc_rqst *rqstp,
 	if (n == 0)
 		goto out;
 
+	/* sanity check the count against the remaining stream */
+	if (n > xdr_stream_remaining(xdr) /
+		((4 * sizeof(uint32_t)) + NFS4_DEVICEID4_SIZE)) {
+		status = htonl(NFS4ERR_BADXDR);
+		goto out;
+	}
+
 	args->devs = kmalloc_objs(*args->devs, n);
 	if (!args->devs) {
 		status = htonl(NFS4ERR_DELAY);
@@ -312,7 +319,7 @@ __be32 decode_devicenotify_args(struct svc_rqst *rqstp,
 		memcpy(dev->cbd_dev_id.data, p, NFS4_DEVICEID4_SIZE);
 		p += XDR_QUADLEN(NFS4_DEVICEID4_SIZE);
 
-		if (dev->cbd_layout_type == NOTIFY_DEVICEID4_CHANGE) {
+		if (dev->cbd_notify_type == NOTIFY_DEVICEID4_CHANGE) {
 			p = xdr_inline_decode(xdr, sizeof(uint32_t));
 			if (unlikely(p == NULL)) {
 				status = htonl(NFS4ERR_BADXDR);
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index 60386330aeeca2..dbb5131375f8dc 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -1295,7 +1295,8 @@ void nfs_clients_init(struct net *net)
 	INIT_LIST_HEAD(&nn->nfs_volume_list);
 #if IS_ENABLED(CONFIG_NFS_V4)
 	idr_init(&nn->cb_ident_idr);
-	INIT_LIST_HEAD(&nn->nfs4_data_server_cache);
+	for (int i = 0; i < NFS4_DS_CACHE_HASH_SIZE; i++)
+		INIT_HLIST_HEAD(&nn->nfs4_data_server_cache[i]);
 	spin_lock_init(&nn->nfs4_data_server_lock);
 #endif /* CONFIG_NFS_V4 */
 	spin_lock_init(&nn->nfs_client_lock);
@@ -1315,7 +1316,8 @@ void nfs_clients_exit(struct net *net)
 	WARN_ON_ONCE(!list_empty(&nn->nfs_client_list));
 	WARN_ON_ONCE(!list_empty(&nn->nfs_volume_list));
 #if IS_ENABLED(CONFIG_NFS_V4)
-	WARN_ON_ONCE(!list_empty(&nn->nfs4_data_server_cache));
+	for (int i = 0; i < NFS4_DS_CACHE_HASH_SIZE; i++)
+		WARN_ON_ONCE(!hlist_empty(&nn->nfs4_data_server_cache[i]));
 #endif /* CONFIG_NFS_V4 */
 }
 
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index 49394123bd0961..354f986e60c42a 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -2437,7 +2437,7 @@ static int nfs_do_create(struct inode *dir, struct dentry *dentry,
 	return error;
 }
 
-int nfs_create(struct mnt_idmap *idmap, struct inode *dir,
+int nfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 	       struct dentry *dentry, umode_t mode)
 {
 	return nfs_do_create(dir, dentry, mode, O_EXCL);
@@ -2448,7 +2448,7 @@ EXPORT_SYMBOL_GPL(nfs_create);
  * See comments for nfs_proc_create regarding failed operations.
  */
 int
-nfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+nfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 	  struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct iattr attr;
@@ -2475,7 +2475,7 @@ EXPORT_SYMBOL_GPL(nfs_mknod);
 /*
  * See comments for nfs_proc_create regarding failed operations.
  */
-struct dentry *nfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+struct dentry *nfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, umode_t mode)
 {
 	struct iattr attr;
@@ -2641,7 +2641,7 @@ EXPORT_SYMBOL_GPL(nfs_unlink);
  * now have a new file handle and can instantiate an in-core NFS inode
  * and move the raw page into its mapping.
  */
-int nfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+int nfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *dentry, const char *symname)
 {
 	struct folio *folio;
@@ -2771,7 +2771,7 @@ static bool nfs_rename_is_unsafe_cross_dir(struct dentry *old_dentry,
  * If these conditions are met, we can drop the dentries before doing
  * the rename.
  */
-int nfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+int nfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 	       struct dentry *old_dentry, struct inode *new_dir,
 	       struct dentry *new_dentry, unsigned int flags)
 {
@@ -3393,7 +3393,7 @@ static int nfs_execute_ok(struct inode *inode, int mask)
 	return ret;
 }
 
-int nfs_permission(struct mnt_idmap *idmap,
+int nfs_permission(const struct mnt_idmap *idmap,
 		   struct inode *inode,
 		   int mask)
 {
diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index ccafdc1ce64d48..a3c6e8f4ea06a6 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -145,11 +145,55 @@ static void nfs_direct_file_adjust_size_locked(struct inode *inode,
 	}
 }
 
-static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
+static void nfs_direct_release_pages(struct page **pages, unsigned int npages,
+				     bool pinned)
 {
-	unsigned int i;
-	for (i = 0; i < npages; i++)
-		put_page(pages[i]);
+	if (pinned)
+		unpin_user_pages(pages, npages);
+}
+
+static ssize_t nfs_direct_extract_pages(struct nfs_direct_req *dreq,
+					struct iov_iter *iter,
+					size_t size, loff_t *pos,
+					struct list_head *list)
+{
+	bool pinned = iov_iter_extract_will_pin(iter);
+	struct page **pagevec = NULL;
+	ssize_t result, bytes = 0;
+	int err = 0;
+	unsigned int npages, i;
+	size_t pgbase;
+
+	result = iov_iter_extract_pages(iter, &pagevec, size, ~0U, 0, &pgbase);
+	if (result <= 0)
+		return result;
+
+	npages = (result + pgbase + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	for (i = 0; i < npages; i++) {
+		struct nfs_page *req;
+		unsigned int req_len = min_t(size_t, result - bytes, PAGE_SIZE - pgbase);
+
+		req = nfs_page_create_from_page(dreq->ctx, pagevec[i],
+						pinned, pgbase, *pos,
+						req_len);
+		if (IS_ERR(req)) {
+			err = PTR_ERR(req);
+			break;
+		}
+
+		list_add_tail(&req->wb_list, list);
+		pgbase = 0;
+		bytes += req_len;
+		*pos += req_len;
+	}
+
+	if (i < npages) {
+		iov_iter_revert(iter, result - bytes);
+		nfs_direct_release_pages(pagevec + i, npages - i, pinned);
+	}
+
+	kvfree(pagevec);
+	return bytes ? bytes : err;
 }
 
 void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
@@ -288,13 +332,7 @@ static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
 
 static void nfs_read_sync_pgio_error(struct list_head *head, int error)
 {
-	struct nfs_page *req;
-
-	while (!list_empty(head)) {
-		req = nfs_list_entry(head->next);
-		nfs_list_remove_request(req);
-		nfs_release_request(req);
-	}
+	nfs_release_request_list(head);
 }
 
 static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
@@ -326,6 +364,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
 	ssize_t result = -EINVAL;
 	size_t requested_bytes = 0;
 	size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
+	LIST_HEAD(nfs_page_list);
 
 	nfs_pageio_init_read(&desc, dreq->inode, false,
 			     &nfs_direct_read_completion_ops);
@@ -334,40 +373,23 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
 	inode_dio_begin(inode);
 
 	while (iov_iter_count(iter)) {
-		struct page **pagevec;
-		size_t bytes;
-		size_t pgbase;
-		unsigned npages, i;
-
-		result = iov_iter_get_pages_alloc2(iter, &pagevec,
-						  rsize, &pgbase);
+		result = nfs_direct_extract_pages(dreq, iter, rsize, &pos, &nfs_page_list);
 		if (result < 0)
 			break;
-	
-		bytes = result;
-		npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
-		for (i = 0; i < npages; i++) {
-			struct nfs_page *req;
-			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
-			/* XXX do we need to do the eof zeroing found in async_filler? */
-			req = nfs_page_create_from_page(dreq->ctx, pagevec[i],
-							pgbase, pos, req_len);
-			if (IS_ERR(req)) {
-				result = PTR_ERR(req);
-				break;
-			}
+
+		while (!list_empty(&nfs_page_list)) {
+			struct nfs_page *req = nfs_list_entry(nfs_page_list.next);
+			size_t req_len = req->wb_bytes;
+
+			nfs_list_remove_request(req);
 			if (!nfs_pageio_add_request(&desc, req)) {
 				result = desc.pg_error;
 				nfs_release_request(req);
+				nfs_release_request_list(&nfs_page_list);
 				break;
 			}
-			pgbase = 0;
-			bytes -= req_len;
 			requested_bytes += req_len;
-			pos += req_len;
 		}
-		nfs_direct_release_pages(pagevec, npages);
-		kvfree(pagevec);
 		if (result < 0)
 			break;
 	}
@@ -858,6 +880,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
 	ssize_t result = 0;
 	size_t requested_bytes = 0;
 	size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
+	LIST_HEAD(nfs_page_list);
 	bool defer = false;
 
 	trace_nfs_direct_write_schedule_iovec(dreq);
@@ -870,53 +893,32 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
 
 	NFS_I(inode)->write_io += iov_iter_count(iter);
 	while (iov_iter_count(iter)) {
-		struct page **pagevec;
-		size_t bytes;
-		size_t pgbase;
-		unsigned npages, i;
-
-		result = iov_iter_get_pages_alloc2(iter, &pagevec,
-						  wsize, &pgbase);
+		result = nfs_direct_extract_pages(dreq, iter, wsize, &pos, &nfs_page_list);
 		if (result < 0)
 			break;
 
-		bytes = result;
-		npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
-		for (i = 0; i < npages; i++) {
-			struct nfs_page *req;
-			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
-
-			req = nfs_page_create_from_page(dreq->ctx, pagevec[i],
-							pgbase, pos, req_len);
-			if (IS_ERR(req)) {
-				result = PTR_ERR(req);
-				break;
-			}
-
-			if (desc.pg_error < 0) {
-				nfs_free_request(req);
-				result = desc.pg_error;
-				break;
-			}
-
-			pgbase = 0;
-			bytes -= req_len;
-			requested_bytes += req_len;
-			pos += req_len;
+		while (!list_empty(&nfs_page_list)) {
+			struct nfs_page *req = nfs_list_entry(nfs_page_list.next);
+			size_t req_len = req->wb_bytes;
 
+			nfs_list_remove_request(req);
 			if (defer) {
 				nfs_mark_request_commit(req, NULL, &cinfo, 0);
+				requested_bytes += req_len;
 				continue;
 			}
 
 			nfs_lock_request(req);
-			if (nfs_pageio_add_request(&desc, req))
+			if (nfs_pageio_add_request(&desc, req)) {
+				requested_bytes += req_len;
 				continue;
+			}
 
 			/* Exit on hard errors */
 			if (desc.pg_error < 0 && desc.pg_error != -EAGAIN) {
 				result = desc.pg_error;
 				nfs_unlock_and_release_request(req);
+				nfs_release_request_list(&nfs_page_list);
 				break;
 			}
 
@@ -927,11 +929,10 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
 			spin_unlock(&dreq->lock);
 			nfs_unlock_request(req);
 			nfs_mark_request_commit(req, NULL, &cinfo, 0);
+			requested_bytes += req_len;
 			desc.pg_error = 0;
 			defer = true;
 		}
-		nfs_direct_release_pages(pagevec, npages);
-		kvfree(pagevec);
 		if (result < 0)
 			break;
 	}
diff --git a/fs/nfs/file.c b/fs/nfs/file.c
index e1bdd10b35f10d..38f830a6467c9b 100644
--- a/fs/nfs/file.c
+++ b/fs/nfs/file.c
@@ -484,7 +484,7 @@ static int nfs_write_end(const struct kiocb *iocb,
  * Partially or wholly invalidate a page
  * - Release the private state associated with a page if undergoing complete
  *   page invalidation
- * - Called if either PG_private or PG_fscache is set on the page
+ * - Called if either folio->private or PG_fscache is set on the page
  * - Caller holds page lock
  */
 static void nfs_invalidate_folio(struct folio *folio, size_t offset,
@@ -555,7 +555,7 @@ static void nfs_check_dirty_writeback(struct folio *folio,
  * Attempt to clear the private state associated with a page when an error
  * occurs that requires the cached contents of an inode to be written back or
  * destroyed
- * - Called if either PG_private or fscache is set on the page
+ * - Called if either page->private or fscache is set on the page
  * - Caller holds page lock
  * - Return 0 if successful, -error otherwise
  */
diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
index 72e20b56fbc705..53808e42546a05 100644
--- a/fs/nfs/filelayout/filelayout.c
+++ b/fs/nfs/filelayout/filelayout.c
@@ -55,12 +55,13 @@ static loff_t
 filelayout_get_dense_offset(struct nfs4_filelayout_segment *flseg,
 			    loff_t offset)
 {
-	u32 stripe_width = flseg->stripe_unit * flseg->dsaddr->stripe_count;
+	u64 stripe_width = (u64)flseg->stripe_unit *
+			   flseg->dsaddr->stripe_count;
 	u64 stripe_no;
 	u32 rem;
 
 	offset -= flseg->pattern_offset;
-	stripe_no = div_u64(offset, stripe_width);
+	stripe_no = div64_u64(offset, stripe_width);
 	div_u64_rem(offset, flseg->stripe_unit, &rem);
 
 	return stripe_no * flseg->stripe_unit + rem;
@@ -186,7 +187,7 @@ static int filelayout_async_handle_error(struct rpc_task *task,
 		dprintk("%s DS connection error %d\n", __func__,
 			task->tk_status);
 		nfs4_mark_deviceid_unavailable(devid);
-		pnfs_error_mark_layout_for_return(inode, lseg);
+		pnfs_error_mark_layout_for_return(inode, lseg, NULL);
 		pnfs_set_lo_fail(lseg);
 		rpc_wake_up(&tbl->slot_tbl_waitq);
 		fallthrough;
@@ -796,7 +797,7 @@ filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
 	unsigned int size;
 	u64 p_stripe, r_stripe;
 	u32 stripe_offset;
-	u64 segment_offset = pgio->pg_lseg->pls_range.offset;
+	u64 pattern_offset = FILELAYOUT_LSEG(pgio->pg_lseg)->pattern_offset;
 	u32 stripe_unit = FILELAYOUT_LSEG(pgio->pg_lseg)->stripe_unit;
 
 	/* calls nfs_generic_pg_test */
@@ -808,8 +809,8 @@ filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
 
 	/* see if req and prev are in the same stripe */
 	if (prev) {
-		p_stripe = (u64)req_offset(prev) - segment_offset;
-		r_stripe = (u64)req_offset(req) - segment_offset;
+		p_stripe = (u64)req_offset(prev) - pattern_offset;
+		r_stripe = (u64)req_offset(req) - pattern_offset;
 		do_div(p_stripe, stripe_unit);
 		do_div(r_stripe, stripe_unit);
 
@@ -818,7 +819,7 @@ filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
 	}
 
 	/* calculate remaining bytes in the current stripe */
-	div_u64_rem((u64)req_offset(req) - segment_offset,
+	div_u64_rem((u64)req_offset(req) - pattern_offset,
 			stripe_unit,
 			&stripe_offset);
 	WARN_ON_ONCE(stripe_offset > stripe_unit);
@@ -856,7 +857,7 @@ fl_pnfs_update_layout(struct inode *ino,
 
 	status = filelayout_check_deviceid(lo, fl, gfp_flags);
 	if (status) {
-		pnfs_error_mark_layout_for_return(ino, lseg);
+		pnfs_error_mark_layout_for_return(ino, lseg, NULL);
 		pnfs_set_lo_fail(lseg);
 		pnfs_put_lseg(lseg);
 		lseg = NULL;
diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c
index 88bc79ec345968..5995f32f07eca0 100644
--- a/fs/nfs/filelayout/filelayoutdev.c
+++ b/fs/nfs/filelayout/filelayoutdev.c
@@ -177,27 +177,14 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
 		trace_fl_getdevinfo(server, &pdev->dev_id, dsaddr->ds_list[i]->ds_remotestr);
 
 		/* If DS was already in cache, free ds addrs */
-		while (!list_empty(&dsaddrs)) {
-			da = list_first_entry(&dsaddrs,
-					      struct nfs4_pnfs_ds_addr,
-					      da_node);
-			list_del_init(&da->da_node);
-			kfree(da->da_remotestr);
-			kfree(da);
-		}
+		nfs4_pnfs_ds_addr_list_free(&dsaddrs);
 	}
 
 	folio_put(scratch);
 	return dsaddr;
 
 out_err_drain_dsaddrs:
-	while (!list_empty(&dsaddrs)) {
-		da = list_first_entry(&dsaddrs, struct nfs4_pnfs_ds_addr,
-				      da_node);
-		list_del_init(&da->da_node);
-		kfree(da->da_remotestr);
-		kfree(da);
-	}
+	nfs4_pnfs_ds_addr_list_free(&dsaddrs);
 out_err_free_deviceid:
 	nfs4_fl_free_deviceid(dsaddr);
 	/* stripe_indicies was part of dsaddr */
@@ -280,7 +267,7 @@ nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
 		goto out_test_devid;
 
 	status = nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo,
-			     dataserver_retrans, 4,
+			     dataserver_retrans, 0, 4,
 			     s->nfs_client->cl_minorversion, true);
 	if (status) {
 		nfs4_mark_deviceid_unavailable(devid);
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 7fe8b91fa47c94..94cc324b591f49 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -44,10 +44,11 @@ static void ff_layout_read_record_layoutstats_done(struct rpc_task *task,
 static int
 ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
 			       struct nfs42_layoutstat_devinfo *devinfo,
+			       struct nfs4_ff_layoutstat_priv *priv,
 			       int dev_limit, enum nfs4_ff_op_type type);
 static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
 			      const struct nfs42_layoutstat_devinfo *devinfo,
-			      struct nfs4_ff_layout_ds_stripe *dss_info);
+			      struct nfs4_ff_layoutstat_priv *priv);
 
 static struct pnfs_layout_hdr *
 ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags)
@@ -285,8 +286,8 @@ static struct nfs4_ff_layout_mirror *ff_layout_alloc_mirror(u32 dss_count,
 
 	mirror->dss_count = dss_count;
 	mirror->dss =
-		kzalloc_objs(struct nfs4_ff_layout_ds_stripe, dss_count,
-			     gfp_flags);
+		kvzalloc_objs(struct nfs4_ff_layout_ds_stripe, dss_count,
+			      gfp_flags);
 	if (mirror->dss == NULL) {
 		kfree(mirror);
 		return NULL;
@@ -312,10 +313,12 @@ static void ff_layout_free_mirror(struct nfs4_ff_layout_mirror *mirror)
 		cred = rcu_access_pointer(mirror->dss[dss_id].rw_cred);
 		put_cred(cred);
 		nfs_close_local_fh(&mirror->dss[dss_id].nfl);
-		nfs4_ff_layout_put_deviceid(mirror->dss[dss_id].mirror_ds);
+		/* the last reference to the mirror is gone; no concurrency */
+		nfs4_ff_layout_put_deviceid(rcu_dereference_protected(
+				mirror->dss[dss_id].mirror_ds, 1));
 	}
 
-	kfree(mirror->dss);
+	kvfree(mirror->dss);
 	kfree(mirror);
 }
 
@@ -857,33 +860,17 @@ nfs4_ff_layout_stat_io_end_write(struct rpc_task *task,
 	spin_unlock(&mirror->lock);
 }
 
-static void
-ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id)
-{
-	struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id);
-
-	if (devid)
-		nfs4_mark_deviceid_unavailable(devid);
-}
-
-static void
-ff_layout_mark_ds_reachable(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id)
-{
-	struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id);
-
-	if (devid)
-		nfs4_mark_deviceid_available(devid);
-}
-
-static struct nfs4_pnfs_ds *
+static struct nfs4_ff_layout_ds *
 ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg,
 			     u32 start_idx, u32 *best_idx,
-			     u32 offset, u32 *dss_id,
+			     u64 offset, u32 *dss_id,
 			     bool check_device)
 {
 	struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg);
 	struct nfs4_ff_layout_mirror *mirror;
-	struct nfs4_pnfs_ds *ds = ERR_PTR(-EAGAIN);
+	struct nfs4_ff_layout_ds *mirror_ds;
+	struct nfs4_ff_layout_ds *ret = ERR_PTR(-EAGAIN);
+	struct nfs4_pnfs_ds *ds;
 	u32 idx;
 
 	/* mirrors are initially sorted by efficiency */
@@ -893,70 +880,79 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg,
 			fls->stripe_unit,
 			fls->mirror_array[idx]->dss_count,
 			offset);
-		ds = nfs4_ff_layout_prepare_ds(lseg, mirror, *dss_id, false);
-		if (IS_ERR(ds))
+		mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror,
+						    *dss_id);
+		ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds,
+					       *dss_id, OP_READ);
+		if (IS_ERR(ds)) {
+			nfs4_ff_layout_put_deviceid(mirror_ds);
+			ret = ERR_CAST(ds);
 			continue;
+		}
 
 		if (check_device &&
-		    nfs4_test_deviceid_unavailable(&mirror->dss[*dss_id].mirror_ds->id_node)) {
+		    nfs4_test_deviceid_unavailable(&mirror_ds->id_node)) {
+			nfs4_ff_layout_put_deviceid(mirror_ds);
 			// reinitialize the error state in case if this is the last iteration
-			ds = ERR_PTR(-EINVAL);
+			ret = ERR_PTR(-EINVAL);
 			continue;
 		}
 
 		*best_idx = idx;
-		break;
+		return mirror_ds;
 	}
 
-	return ds;
+	return ret;
 }
 
-static struct nfs4_pnfs_ds *
+static struct nfs4_ff_layout_ds *
 ff_layout_choose_any_ds_for_read(struct pnfs_layout_segment *lseg,
 				 u32 start_idx, u32 *best_idx,
-				 u32 offset, u32 *dss_id)
+				 u64 offset, u32 *dss_id)
 {
 	return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx,
 					    offset, dss_id, false);
 }
 
-static struct nfs4_pnfs_ds *
+static struct nfs4_ff_layout_ds *
 ff_layout_choose_valid_ds_for_read(struct pnfs_layout_segment *lseg,
 				   u32 start_idx, u32 *best_idx,
-				   u32 offset, u32 *dss_id)
+				   u64 offset, u32 *dss_id)
 {
 	return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx,
 					    offset, dss_id, true);
 }
 
-static struct nfs4_pnfs_ds *
+static struct nfs4_ff_layout_ds *
 ff_layout_choose_best_ds_for_read(struct pnfs_layout_segment *lseg,
 				  u32 start_idx, u32 *best_idx,
-				  u32 offset, u32 *dss_id)
+				  u64 offset, u32 *dss_id)
 {
-	struct nfs4_pnfs_ds *ds;
+	struct nfs4_ff_layout_ds *mirror_ds;
 
-	ds = ff_layout_choose_valid_ds_for_read(lseg, start_idx, best_idx,
-						offset, dss_id);
-	if (!IS_ERR(ds))
-		return ds;
+	mirror_ds = ff_layout_choose_valid_ds_for_read(lseg, start_idx,
+							best_idx, offset,
+							dss_id);
+	if (!IS_ERR(mirror_ds))
+		return mirror_ds;
 	return ff_layout_choose_any_ds_for_read(lseg, start_idx, best_idx,
 						offset, dss_id);
 }
 
-static struct nfs4_pnfs_ds *
+static struct nfs4_ff_layout_ds *
 ff_layout_get_ds_for_read(struct nfs_pageio_descriptor *pgio,
 			  u32 *best_idx,
-			  u32 offset,
+			  u64 offset,
 			  u32 *dss_id)
 {
 	struct pnfs_layout_segment *lseg = pgio->pg_lseg;
-	struct nfs4_pnfs_ds *ds;
+	struct nfs4_ff_layout_ds *mirror_ds;
 
-	ds = ff_layout_choose_best_ds_for_read(lseg, pgio->pg_mirror_idx,
-					       best_idx, offset, dss_id);
-	if (!IS_ERR(ds) || !pgio->pg_mirror_idx)
-		return ds;
+	mirror_ds = ff_layout_choose_best_ds_for_read(lseg,
+						      pgio->pg_mirror_idx,
+						      best_idx, offset, dss_id);
+	if (!IS_ERR(mirror_ds) || !pgio->pg_mirror_idx)
+		return mirror_ds;
 	return ff_layout_choose_best_ds_for_read(lseg, 0, best_idx,
 						 offset, dss_id);
 }
@@ -995,9 +991,8 @@ ff_layout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
 {
 	unsigned int size;
 	u64 p_stripe, r_stripe;
-	u32 stripe_offset;
-	u64 segment_offset = pgio->pg_lseg->pls_range.offset;
-	u32 stripe_unit = FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit;
+	u64 stripe_offset;
+	u64 stripe_unit = FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit;
 
 	/* calls nfs_generic_pg_test */
 	size = pnfs_generic_pg_test(pgio, prev, req);
@@ -1008,23 +1003,23 @@ ff_layout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
 
 	/* see if req and prev are in the same stripe */
 	if (prev) {
-		p_stripe = (u64)req_offset(prev) - segment_offset;
-		r_stripe = (u64)req_offset(req) - segment_offset;
-		do_div(p_stripe, stripe_unit);
-		do_div(r_stripe, stripe_unit);
+		p_stripe = (u64)req_offset(prev);
+		r_stripe = (u64)req_offset(req);
+		p_stripe = div64_u64(p_stripe, stripe_unit);
+		r_stripe = div64_u64(r_stripe, stripe_unit);
 
 		if (p_stripe != r_stripe)
 			return 0;
 	}
 
 	/* calculate remaining bytes in the current stripe */
-	div_u64_rem((u64)req_offset(req) - segment_offset,
+	div64_u64_rem((u64)req_offset(req),
 			stripe_unit,
 			&stripe_offset);
 	WARN_ON_ONCE(stripe_offset > stripe_unit);
 	if (stripe_offset >= stripe_unit)
 		return 0;
-	return min(stripe_unit - (unsigned int)stripe_offset, size);
+	return min_t(u64, stripe_unit - stripe_offset, size);
 }
 
 static void
@@ -1032,8 +1027,7 @@ ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio,
 			struct nfs_page *req)
 {
 	struct nfs_pgio_mirror *pgm;
-	struct nfs4_ff_layout_mirror *mirror;
-	struct nfs4_pnfs_ds *ds;
+	struct nfs4_ff_layout_ds *mirror_ds;
 	u32 ds_idx, dss_id;
 
 	if (NFS_SERVER(pgio->pg_inode)->flags &
@@ -1055,9 +1049,9 @@ ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio,
 	/* Reset wb_nio, since getting layout segment was successful */
 	req->wb_nio = 0;
 
-	ds = ff_layout_get_ds_for_read(pgio, &ds_idx,
-				       req_offset(req), &dss_id);
-	if (IS_ERR(ds)) {
+	mirror_ds = ff_layout_get_ds_for_read(pgio, &ds_idx,
+					      req_offset(req), &dss_id);
+	if (IS_ERR(mirror_ds)) {
 		if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg))
 			goto out_mds;
 		pnfs_generic_pg_cleanup(pgio);
@@ -1066,9 +1060,9 @@ ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio,
 		goto retry;
 	}
 
-	mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx);
 	pgm = &pgio->pg_mirrors[0];
-	pgm->pg_bsize = mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize;
+	pgm->pg_bsize = mirror_ds->ds_versions[0].rsize;
+	nfs4_ff_layout_put_deviceid(mirror_ds);
 
 	pgio->pg_mirror_idx = ds_idx;
 	return;
@@ -1103,6 +1097,7 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio,
 			struct nfs_page *req)
 {
 	struct nfs4_ff_layout_mirror *mirror;
+	struct nfs4_ff_layout_ds *mirror_ds;
 	struct nfs_pgio_mirror *pgm;
 	struct nfs4_pnfs_ds *ds;
 	u32 i, dss_id;
@@ -1134,9 +1129,12 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio,
 			FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit,
 			mirror->dss_count,
 			req_offset(req));
+		mirror_ds = ff_layout_get_mirror_ds(pgio->pg_lseg->pls_layout,
+						    mirror, dss_id);
 		ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror,
-					       dss_id, true);
+					       mirror_ds, dss_id, OP_WRITE);
 		if (IS_ERR(ds)) {
+			nfs4_ff_layout_put_deviceid(mirror_ds);
 			if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg))
 				goto out_mds;
 			pnfs_generic_pg_cleanup(pgio);
@@ -1145,7 +1143,8 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio,
 			goto retry;
 		}
 		pgm = &pgio->pg_mirrors[i];
-		pgm->pg_bsize = mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize;
+		pgm->pg_bsize = mirror_ds->ds_versions[0].wsize;
+		nfs4_ff_layout_put_deviceid(mirror_ds);
 	}
 
 	if (NFS_SERVER(pgio->pg_inode)->flags &
@@ -1277,14 +1276,16 @@ static void ff_layout_resend_pnfs_read(struct nfs_pgio_header *hdr)
 	u32 idx = hdr->pgio_mirror_idx + 1;
 	u32 new_idx = 0;
 	u32 dss_id = 0;
-	struct nfs4_pnfs_ds *ds;
+	struct nfs4_ff_layout_ds *mirror_ds;
 
-	ds = ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx,
-					      hdr->args.offset, &dss_id);
-	if (IS_ERR(ds))
-		pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg);
-	else
+	mirror_ds = ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx,
+						     hdr->args.offset, &dss_id);
+	if (IS_ERR(mirror_ds)) {
+		pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg, NULL);
+	} else {
+		nfs4_ff_layout_put_deviceid(mirror_ds);
 		ff_layout_send_layouterror(hdr->lseg);
+	}
 	pnfs_read_resend_pnfs(hdr, new_idx);
 }
 
@@ -1293,7 +1294,7 @@ static void ff_layout_reset_read(struct nfs_pgio_header *hdr)
 	struct rpc_task *task = &hdr->task;
 
 	pnfs_layoutcommit_inode(hdr->inode, false);
-	pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg);
+	pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg, NULL);
 
 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
 		dprintk("%s Reset task %5u for i/o through MDS "
@@ -1317,11 +1318,10 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task,
 					   struct nfs4_state *state,
 					   struct nfs_client *clp,
 					   struct pnfs_layout_segment *lseg,
-					   u32 idx, u32 dss_id)
+					   struct nfs4_deviceid_node *devid)
 {
 	struct pnfs_layout_hdr *lo = lseg->pls_layout;
 	struct inode *inode = lo->plh_inode;
-	struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id);
 	struct nfs4_slot_table *tbl = nfs4_has_session(clp) ?
 		&clp->cl_session->fc_slot_table : clp->cl_slot_tbl;
 
@@ -1394,8 +1394,9 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task,
 	case -ENODEV:
 		dprintk("%s DS connection error %d\n", __func__,
 			task->tk_status);
-		nfs4_delete_deviceid(devid->ld, devid->nfs_client,
-				&devid->deviceid);
+		if (devid)
+			nfs4_delete_deviceid(devid->ld, devid->nfs_client,
+					&devid->deviceid);
 		rpc_wake_up(&tbl->slot_tbl_waitq);
 		break;
 	default:
@@ -1419,9 +1420,8 @@ static int ff_layout_async_handle_error_v3(struct rpc_task *task,
 					   u32 op_status,
 					   struct nfs_client *clp,
 					   struct pnfs_layout_segment *lseg,
-					   u32 idx, u32 dss_id)
+					   struct nfs4_deviceid_node *devid)
 {
-	struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id);
 
 	switch (op_status) {
 	case NFS_OK:
@@ -1467,8 +1467,9 @@ static int ff_layout_async_handle_error_v3(struct rpc_task *task,
 	default:
 		dprintk("%s DS connection error %d\n", __func__,
 			task->tk_status);
-		nfs4_delete_deviceid(devid->ld, devid->nfs_client,
-				&devid->deviceid);
+		if (devid)
+			nfs4_delete_deviceid(devid->ld, devid->nfs_client,
+					&devid->deviceid);
 	}
 out_reset_to_pnfs:
 	/* FIXME: Need to prevent infinite looping here. */
@@ -1485,12 +1486,13 @@ static int ff_layout_async_handle_error(struct rpc_task *task,
 					struct nfs4_state *state,
 					struct nfs_client *clp,
 					struct pnfs_layout_segment *lseg,
-					u32 idx, u32 dss_id)
+					struct nfs4_deviceid_node *devid)
 {
 	int vers = clp->cl_nfs_mod->rpc_vers->number;
 
 	if (task->tk_status >= 0) {
-		ff_layout_mark_ds_reachable(lseg, idx, dss_id);
+		if (devid)
+			nfs4_mark_deviceid_available(devid);
 		return 0;
 	}
 
@@ -1501,10 +1503,10 @@ static int ff_layout_async_handle_error(struct rpc_task *task,
 	switch (vers) {
 	case 3:
 		return ff_layout_async_handle_error_v3(task, op_status, clp,
-						       lseg, idx, dss_id);
+						       lseg, devid);
 	case 4:
 		return ff_layout_async_handle_error_v4(task, op_status, state,
-						       clp, lseg, idx, dss_id);
+						       clp, lseg, devid);
 	default:
 		/* should never happen */
 		WARN_ON_ONCE(1);
@@ -1513,6 +1515,7 @@ static int ff_layout_async_handle_error(struct rpc_task *task,
 }
 
 static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
+					struct nfs4_deviceid_node *devid,
 					u32 idx, u32 dss_id, u64 offset, u64 length,
 					u32 *op_status, int opnum, int error)
 {
@@ -1562,8 +1565,8 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
 
 	mirror = FF_LAYOUT_COMP(lseg, idx);
 	err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout),
-				       mirror, dss_id, offset, length, status, opnum,
-				       nfs_io_gfp_mask());
+				       mirror, devid, dss_id, offset, length,
+				       status, opnum, nfs_io_gfp_mask());
 
 	/*
 	 * I/O we cancelled ourselves to return a recalled or revoked layout
@@ -1580,7 +1583,8 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
 	case NFS4ERR_PERM:
 		break;
 	case NFS4ERR_NXIO:
-		ff_layout_mark_ds_unreachable(lseg, idx, dss_id);
+		if (devid)
+			nfs4_mark_deviceid_unavailable(devid);
 		/*
 		 * Don't return the layout if this is a read and we still
 		 * have layouts to try
@@ -1590,7 +1594,8 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
 		fallthrough;
 	default:
 		pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode,
-						  lseg);
+						  lseg,
+						  &mirror->dss[dss_id].devid);
 	}
 
 out:
@@ -1609,7 +1614,7 @@ static int ff_layout_read_done_cb(struct rpc_task *task,
 	int err;
 
 	if (task->tk_status < 0) {
-		ff_layout_io_track_ds_error(hdr->lseg,
+		ff_layout_io_track_ds_error(hdr->lseg, hdr->ds_dev,
 					    hdr->pgio_mirror_idx, dss_id,
 					    hdr->args.offset, hdr->args.count,
 					    &hdr->res.op_status, OP_READ,
@@ -1620,8 +1625,7 @@ static int ff_layout_read_done_cb(struct rpc_task *task,
 	err = ff_layout_async_handle_error(task, hdr->res.op_status,
 					   hdr->args.context->state,
 					   hdr->ds_clp, hdr->lseg,
-					   hdr->pgio_mirror_idx,
-					   dss_id);
+					   hdr->ds_dev);
 
 	trace_nfs4_pnfs_read(hdr, err);
 	clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags);
@@ -1814,7 +1818,7 @@ static int ff_layout_write_done_cb(struct rpc_task *task,
 	int err;
 
 	if (task->tk_status < 0) {
-		ff_layout_io_track_ds_error(hdr->lseg,
+		ff_layout_io_track_ds_error(hdr->lseg, hdr->ds_dev,
 					    hdr->pgio_mirror_idx, dss_id,
 					    hdr->args.offset, hdr->args.count,
 					    &hdr->res.op_status, OP_WRITE,
@@ -1825,8 +1829,7 @@ static int ff_layout_write_done_cb(struct rpc_task *task,
 	err = ff_layout_async_handle_error(task, hdr->res.op_status,
 					   hdr->args.context->state,
 					   hdr->ds_clp, hdr->lseg,
-					   hdr->pgio_mirror_idx,
-					   dss_id);
+					   hdr->ds_dev);
 
 	trace_nfs4_pnfs_write(hdr, err);
 	clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags);
@@ -1868,7 +1871,7 @@ static int ff_layout_commit_done_cb(struct rpc_task *task,
 	u32 dss_id = calc_dss_id_from_commit(data->lseg, data->ds_commit_index);
 
 	if (task->tk_status < 0) {
-		ff_layout_io_track_ds_error(data->lseg, idx, dss_id,
+		ff_layout_io_track_ds_error(data->lseg, data->ds_dev, idx, dss_id,
 					    data->args.offset, data->args.count,
 					    &data->res.op_status, OP_COMMIT,
 					    task->tk_status);
@@ -1876,8 +1879,8 @@ static int ff_layout_commit_done_cb(struct rpc_task *task,
 	}
 
 	err = ff_layout_async_handle_error(task, data->res.op_status,
-					   NULL, data->ds_clp, data->lseg, idx,
-					   dss_id);
+					   NULL, data->ds_clp, data->lseg,
+					   data->ds_dev);
 
 	trace_nfs4_pnfs_commit_ds(data, err);
 	switch (err) {
@@ -2167,6 +2170,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 	struct rpc_clnt *ds_clnt;
 	struct nfsd_file *localio;
 	struct nfs4_ff_layout_mirror *mirror;
+	struct nfs4_ff_layout_ds *mirror_ds;
 	const struct cred *ds_cred;
 	loff_t offset = hdr->args.offset;
 	u32 idx = hdr->pgio_mirror_idx;
@@ -2184,22 +2188,25 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 		FF_LAYOUT_LSEG(lseg)->stripe_unit,
 		mirror->dss_count,
 		offset);
-	ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, false);
+	mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror, dss_id);
+	ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds, dss_id,
+				       OP_READ);
 	if (IS_ERR(ds)) {
 		ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds));
 		goto out_failed;
 	}
 
-	ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
-						   hdr->inode, dss_id);
+	ds_clnt = nfs4_ff_find_or_create_ds_client(mirror_ds, ds->ds_clp,
+						   hdr->inode);
 	if (IS_ERR(ds_clnt))
 		goto out_failed;
 
-	ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, dss_id);
+	ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred,
+					mirror_ds, dss_id);
 	if (!ds_cred)
 		goto out_failed;
 
-	vers = nfs4_ff_layout_ds_version(mirror, dss_id);
+	vers = nfs4_ff_layout_ds_version(mirror_ds);
 
 	dprintk("%s USE DS: %s cl_count %d vers %d\n", __func__,
 		ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count), vers);
@@ -2211,7 +2218,8 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 	if (fh)
 		hdr->args.fh = fh;
 
-	nfs4_ff_layout_select_ds_stateid(mirror, dss_id, &hdr->args.stateid);
+	nfs4_ff_layout_select_ds_stateid(mirror, mirror_ds, dss_id,
+					 &hdr->args.stateid);
 
 	/*
 	 * Note that if we ever decide to split across DSes,
@@ -2228,6 +2236,10 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 		ff_layout_read_record_layoutstats_start(&hdr->task, hdr);
 	}
 
+	/* Transfer the device node reference to the I/O; put on release */
+	pnfs_put_ds_dev(hdr->ds_dev);
+	hdr->ds_dev = &mirror_ds->id_node;
+
 	/* Perform an asynchronous read to ds */
 	nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,
 			  vers == 3 ? &ff_layout_read_call_ops_v3 :
@@ -2237,6 +2249,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 	return PNFS_ATTEMPTED;
 
 out_failed:
+	nfs4_ff_layout_put_deviceid(mirror_ds);
 	if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error)
 		return PNFS_TRY_AGAIN;
 	if (ff_layout_no_fallback_to_mds(lseg)) {
@@ -2244,7 +2257,8 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
 		 * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET,
 		 * never fall through to MDS I/O.
 		 */
-		pnfs_error_mark_layout_for_return(hdr->inode, lseg);
+		pnfs_error_mark_layout_for_return(hdr->inode, lseg,
+						  &mirror->dss[dss_id].devid);
 		return PNFS_TRY_AGAIN;
 	}
 	trace_pnfs_mds_fallback_read_pagelist(hdr->inode,
@@ -2262,6 +2276,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 	struct rpc_clnt *ds_clnt;
 	struct nfsd_file *localio;
 	struct nfs4_ff_layout_mirror *mirror;
+	struct nfs4_ff_layout_ds *mirror_ds;
 	const struct cred *ds_cred;
 	loff_t offset = hdr->args.offset;
 	int vers;
@@ -2275,22 +2290,25 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 		FF_LAYOUT_LSEG(lseg)->stripe_unit,
 		mirror->dss_count,
 		offset);
-	ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, true);
+	mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror, dss_id);
+	ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds, dss_id,
+				       OP_WRITE);
 	if (IS_ERR(ds)) {
 		ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds));
 		goto out_failed;
 	}
 
-	ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
-						   hdr->inode, dss_id);
+	ds_clnt = nfs4_ff_find_or_create_ds_client(mirror_ds, ds->ds_clp,
+						   hdr->inode);
 	if (IS_ERR(ds_clnt))
 		goto out_failed;
 
-	ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, dss_id);
+	ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred,
+					mirror_ds, dss_id);
 	if (!ds_cred)
 		goto out_failed;
 
-	vers = nfs4_ff_layout_ds_version(mirror, dss_id);
+	vers = nfs4_ff_layout_ds_version(mirror_ds);
 
 	dprintk("%s ino %llu sync %d req %zu@%llu DS: %s cl_count %d vers %d\n",
 		__func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,
@@ -2305,7 +2323,8 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 	if (fh)
 		hdr->args.fh = fh;
 
-	nfs4_ff_layout_select_ds_stateid(mirror, dss_id, &hdr->args.stateid);
+	nfs4_ff_layout_select_ds_stateid(mirror, mirror_ds, dss_id,
+					 &hdr->args.stateid);
 
 	/*
 	 * Note that if we ever decide to split across DSes,
@@ -2321,6 +2340,10 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 		ff_layout_write_record_layoutstats_start(&hdr->task, hdr);
 	}
 
+	/* Transfer the device node reference to the I/O; put on release */
+	pnfs_put_ds_dev(hdr->ds_dev);
+	hdr->ds_dev = &mirror_ds->id_node;
+
 	/* Perform an asynchronous write */
 	nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,
 			  vers == 3 ? &ff_layout_write_call_ops_v3 :
@@ -2330,6 +2353,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 	return PNFS_ATTEMPTED;
 
 out_failed:
+	nfs4_ff_layout_put_deviceid(mirror_ds);
 	if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error)
 		return PNFS_TRY_AGAIN;
 	if (ff_layout_no_fallback_to_mds(lseg)) {
@@ -2337,7 +2361,8 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
 		 * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET,
 		 * never fall through to MDS I/O.
 		 */
-		pnfs_error_mark_layout_for_return(hdr->inode, lseg);
+		pnfs_error_mark_layout_for_return(hdr->inode, lseg,
+						  &mirror->dss[dss_id].devid);
 		return PNFS_TRY_AGAIN;
 	}
 	trace_pnfs_mds_fallback_write_pagelist(hdr->inode,
@@ -2364,6 +2389,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
 	struct rpc_clnt *ds_clnt;
 	struct nfsd_file *localio;
 	struct nfs4_ff_layout_mirror *mirror;
+	struct nfs4_ff_layout_ds *mirror_ds = NULL;
 	const struct cred *ds_cred;
 	u32 idx, dss_id;
 	int vers, ret;
@@ -2376,20 +2402,23 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
 	idx = calc_mirror_idx_from_commit(lseg, data->ds_commit_index);
 	mirror = FF_LAYOUT_COMP(lseg, idx);
 	dss_id = calc_dss_id_from_commit(lseg, data->ds_commit_index);
-	ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, true);
+	mirror_ds = ff_layout_get_mirror_ds(lseg->pls_layout, mirror, dss_id);
+	ds = nfs4_ff_layout_prepare_ds(lseg, mirror, mirror_ds, dss_id,
+				       OP_COMMIT);
 	if (IS_ERR(ds))
 		goto out_err;
 
-	ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
-						   data->inode, dss_id);
+	ds_clnt = nfs4_ff_find_or_create_ds_client(mirror_ds, ds->ds_clp,
+						   data->inode);
 	if (IS_ERR(ds_clnt))
 		goto out_err;
 
-	ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred, dss_id);
+	ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred,
+					mirror_ds, dss_id);
 	if (!ds_cred)
 		goto out_err;
 
-	vers = nfs4_ff_layout_ds_version(mirror, dss_id);
+	vers = nfs4_ff_layout_ds_version(mirror_ds);
 
 	dprintk("%s ino %llu, how %d cl_count %d vers %d\n", __func__,
 		data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count),
@@ -2410,6 +2439,10 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
 		ff_layout_commit_record_layoutstats_start(&data->task, data);
 	}
 
+	/* Transfer the device node reference to the commit; put on release */
+	pnfs_put_ds_dev(data->ds_dev);
+	data->ds_dev = &mirror_ds->id_node;
+
 	ret = nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops,
 				   vers == 3 ? &ff_layout_commit_call_ops_v3 :
 					       &ff_layout_commit_call_ops_v4,
@@ -2417,6 +2450,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
 	put_cred(ds_cred);
 	return ret;
 out_err:
+	nfs4_ff_layout_put_deviceid(mirror_ds);
 	pnfs_generic_prepare_to_resend_writes(data);
 	pnfs_generic_commit_release(data);
 	return -EAGAIN;
@@ -2459,7 +2493,8 @@ static bool ff_layout_match_io(const struct rpc_task *task, const void *data)
 	return false;
 }
 
-static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg)
+static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg,
+				const struct nfs4_deviceid *devid)
 {
 	struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg);
 	struct nfs4_ff_layout_mirror *mirror;
@@ -2472,22 +2507,94 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg)
 	for (idx = 0; idx < flseg->mirror_array_cnt; idx++) {
 		mirror = flseg->mirror_array[idx];
 		for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) {
-			mirror_ds = mirror->dss[dss_id].mirror_ds;
-			if (IS_ERR_OR_NULL(mirror_ds))
+			if (devid && memcmp(&mirror->dss[dss_id].devid, devid,
+					    sizeof(*devid)) != 0)
 				continue;
-			ds = mirror->dss[dss_id].mirror_ds->ds;
-			if (!ds)
+			rcu_read_lock();
+			mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds);
+			if (IS_ERR_OR_NULL(mirror_ds) ||
+			    !atomic_inc_not_zero(&mirror_ds->id_node.ref)) {
+				rcu_read_unlock();
 				continue;
+			}
+			rcu_read_unlock();
+			ds = mirror_ds->ds;
+			if (!ds)
+				goto next;
 			ds_clp = ds->ds_clp;
 			if (!ds_clp)
-				continue;
+				goto next;
 			clnt = ds_clp->cl_rpcclient;
 			if (!clnt)
-				continue;
+				goto next;
 			if (!rpc_cancel_tasks(clnt, -ECANCELED,
 					      ff_layout_match_io, lseg))
-				continue;
+				goto next;
 			rpc_clnt_disconnect(clnt);
+next:
+			nfs4_ff_layout_put_deviceid(mirror_ds);
+		}
+	}
+}
+
+/* Called under @lo's inode i_lock. */
+static bool ff_layout_references_deviceid(struct pnfs_layout_hdr *lo,
+					  const struct nfs4_deviceid *id)
+{
+	struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
+	struct nfs4_ff_layout_mirror *mirror;
+	u32 dss_id;
+
+	list_for_each_entry(mirror, &flo->mirrors, mirrors)
+		for (dss_id = 0; dss_id < mirror->dss_count; dss_id++)
+			if (memcmp(&mirror->dss[dss_id].devid, id,
+				   sizeof(*id)) == 0)
+				return true;
+	return false;
+}
+
+/*
+ * Un-pin every stripe node resolved from @id: in-flight I/O drains on the
+ * old node through its own reference, the next I/O re-resolves.
+ */
+static void ff_layout_reresolve_deviceid(struct pnfs_layout_hdr *lo,
+					 const struct nfs4_deviceid *id,
+					 bool immediate,
+					 struct list_head *head)
+{
+	struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo);
+	struct nfs4_ff_layout_mirror *mirror;
+	struct nfs4_ff_layout_ds *old;
+	struct nfs4_deviceid_put *put;
+	u32 dss_id;
+
+	list_for_each_entry(mirror, &flo->mirrors, mirrors) {
+		for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) {
+			if (memcmp(&mirror->dss[dss_id].devid, id,
+				   sizeof(*id)) != 0)
+				continue;
+			/* Allocate before un-pinning: on failure the reference
+			 * stays put rather than being dropped here, where the
+			 * final put may not sleep.
+			 */
+			put = kzalloc_obj(*put, GFP_ATOMIC);
+			if (!put)
+				continue;
+			old = unrcu_pointer(
+				xchg(&mirror->dss[dss_id].mirror_ds, NULL));
+			if (IS_ERR_OR_NULL(old)) {
+				kfree(put);
+				continue;
+			}
+			/* A node still hashed was fetched after the unhash
+			 * and carries the new mapping; mark only the
+			 * superseded ones.
+			 */
+			if (immediate &&
+			    hlist_unhashed_lockless(&old->id_node.node))
+				nfs4_mark_deviceid_unavailable(&old->id_node);
+			put->dev = &old->id_node;
+			list_add(&put->node, head);
 		}
 	}
 }
@@ -2700,7 +2807,7 @@ ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args)
 
 	spin_lock(&args->inode->i_lock);
 	ff_args->num_dev = ff_layout_mirror_prepare_stats(
-		&ff_layout->generic_hdr, &ff_args->devinfo[0],
+		&ff_layout->generic_hdr, &ff_args->devinfo[0], &ff_args->priv[0],
 		ARRAY_SIZE(ff_args->devinfo), NFS4_FF_OP_LAYOUTRETURN);
 	spin_unlock(&args->inode->i_lock);
 
@@ -2875,10 +2982,11 @@ ff_layout_encode_io_latency(struct xdr_stream *xdr,
 static void
 ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
 			      const struct nfs42_layoutstat_devinfo *devinfo,
-			      struct nfs4_ff_layout_ds_stripe *dss_info)
+			      struct nfs4_ff_layoutstat_priv *priv)
 {
+	struct nfs4_ff_layout_ds_stripe *dss_info = priv->dss_info;
 	struct nfs4_pnfs_ds_addr *da;
-	struct nfs4_pnfs_ds *ds = dss_info->mirror_ds->ds;
+	struct nfs4_pnfs_ds *ds = priv->mirror_ds->ds;
 	struct nfs_fh *fh = &dss_info->fh_versions[0];
 	__be32 *p;
 
@@ -2925,10 +3033,10 @@ ff_layout_encode_layoutstats(struct xdr_stream *xdr, const void *args,
 static void
 ff_layout_free_layoutstats(struct nfs4_xdr_opaque_data *opaque)
 {
-	struct nfs4_ff_layout_ds_stripe *dss_info = opaque->data;
-	struct nfs4_ff_layout_mirror *mirror = dss_info->mirror;
+	struct nfs4_ff_layoutstat_priv *priv = opaque->data;
 
-	ff_layout_put_mirror(mirror);
+	nfs4_ff_layout_put_deviceid(priv->mirror_ds);
+	ff_layout_put_mirror(priv->dss_info->mirror);
 }
 
 static const struct nfs4_xdr_opaque_ops layoutstat_ops = {
@@ -2939,20 +3047,23 @@ static const struct nfs4_xdr_opaque_ops layoutstat_ops = {
 static int
 ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
 			       struct nfs42_layoutstat_devinfo *devinfo,
+			       struct nfs4_ff_layoutstat_priv *priv,
 			       int dev_limit, enum nfs4_ff_op_type type)
 {
 	struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo);
 	struct nfs4_ff_layout_mirror *mirror;
 	struct nfs4_ff_layout_ds_stripe *dss_info;
-	struct nfs4_deviceid_node *dev;
+	struct nfs4_ff_layout_ds *mirror_ds;
 	int i = 0, dss_id;
 
+	rcu_read_lock();
 	list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) {
 		for (dss_id = 0; dss_id < mirror->dss_count; ++dss_id) {
 			dss_info = &mirror->dss[dss_id];
 			if (i >= dev_limit)
 				break;
-			if (IS_ERR_OR_NULL(dss_info->mirror_ds))
+			mirror_ds = rcu_dereference(dss_info->mirror_ds);
+			if (IS_ERR_OR_NULL(mirror_ds))
 				continue;
 			if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL,
 						&mirror->flags) &&
@@ -2961,9 +3072,12 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
 			/* mirror refcount put in cleanup_layoutstats */
 			if (!refcount_inc_not_zero(&mirror->ref))
 				continue;
-			dev = &dss_info->mirror_ds->id_node;
+			/* The pin holds a reference; it is exchanged out only
+			 * under i_lock.  Put in ff_layout_free_layoutstats().
+			 */
+			atomic_inc(&mirror_ds->id_node.ref);
 			memcpy(&devinfo->dev_id,
-			       &dev->deviceid,
+			       &mirror_ds->id_node.deviceid,
 			       NFS4_DEVICEID4_SIZE);
 			devinfo->offset = 0;
 			devinfo->length = NFS4_MAX_UINT64;
@@ -2979,12 +3093,16 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
 			spin_unlock(&mirror->lock);
 			devinfo->layout_type = LAYOUT_FLEX_FILES;
 			devinfo->ld_private.ops = &layoutstat_ops;
-			devinfo->ld_private.data = &mirror->dss[dss_id];
+			priv->dss_info = dss_info;
+			priv->mirror_ds = mirror_ds;
+			devinfo->ld_private.data = priv;
 
 			devinfo++;
+			priv++;
 			i++;
 		}
 	}
+	rcu_read_unlock();
 	return i;
 }
 
@@ -2992,21 +3110,28 @@ static int ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args)
 {
 	struct pnfs_layout_hdr *lo;
 	struct nfs4_flexfile_layout *ff_layout;
+	struct nfs4_ff_layoutstat_priv *priv;
 	const int dev_count = PNFS_LAYOUTSTATS_MAXDEV;
 
-	/* For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics */
-	args->devinfo = kmalloc_objs(*args->devinfo, dev_count,
-				     nfs_io_gfp_mask());
+	/*
+	 * For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics.
+	 * The per-devinfo private entries are co-allocated after the
+	 * devinfo array and freed along with it.
+	 */
+	args->devinfo = kmalloc(dev_count * (sizeof(*args->devinfo) +
+					     sizeof(*priv)),
+				nfs_io_gfp_mask());
 	if (!args->devinfo)
 		return -ENOMEM;
+	priv = (struct nfs4_ff_layoutstat_priv *)&args->devinfo[dev_count];
 
 	spin_lock(&args->inode->i_lock);
 	lo = NFS_I(args->inode)->layout;
 	if (lo && pnfs_layout_is_valid(lo)) {
 		ff_layout = FF_LAYOUT_FROM_HDR(lo);
 		args->num_dev = ff_layout_mirror_prepare_stats(
-			&ff_layout->generic_hdr, &args->devinfo[0], dev_count,
-			NFS4_FF_OP_LAYOUTSTATS);
+			&ff_layout->generic_hdr, &args->devinfo[0], priv,
+			dev_count, NFS4_FF_OP_LAYOUTSTATS);
 	} else
 		args->num_dev = 0;
 	spin_unlock(&args->inode->i_lock);
@@ -3055,6 +3180,8 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = {
 	.pg_write_ops		= &ff_layout_pg_write_ops,
 	.get_ds_info		= ff_layout_get_ds_info,
 	.free_deviceid_node	= ff_layout_free_deviceid_node,
+	.reresolve_deviceid	= ff_layout_reresolve_deviceid,
+	.layout_references_deviceid = ff_layout_references_deviceid,
 	.read_pagelist		= ff_layout_read_pagelist,
 	.write_pagelist		= ff_layout_write_pagelist,
 	.alloc_deviceid_node    = ff_layout_alloc_deviceid_node,
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h
index a5bd00f69e8242..09c3cd6964fd63 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.h
+++ b/fs/nfs/flexfilelayout/flexfilelayout.h
@@ -79,7 +79,7 @@ struct nfs4_ff_layout_ds_stripe {
 	struct nfs4_ff_layout_mirror   *mirror;
 	struct nfs4_deviceid		devid;
 	u32				efficiency;
-	struct nfs4_ff_layout_ds	*mirror_ds;
+	struct nfs4_ff_layout_ds __rcu	*mirror_ds;
 	u32				fh_versions_cnt;
 	struct nfs_fh			*fh_versions;
 	nfs4_stateid			stateid;
@@ -124,9 +124,20 @@ struct nfs4_flexfile_layout {
 	unsigned long		flags;
 };
 
+/*
+ * Per-devinfo private data for a layoutstats/layoutreturn encode: the
+ * stripe the stats describe plus a reference on its device node so the
+ * node (and its DS addresses) stay valid until the XDR encode runs.
+ */
+struct nfs4_ff_layoutstat_priv {
+	struct nfs4_ff_layout_ds_stripe *dss_info;
+	struct nfs4_ff_layout_ds *mirror_ds;
+};
+
 struct nfs4_flexfile_layoutreturn_args {
 	struct list_head errors;
 	struct nfs42_layoutstat_devinfo devinfo[FF_LAYOUTSTATS_MAXDEV];
+	struct nfs4_ff_layoutstat_priv priv[FF_LAYOUTSTATS_MAXDEV];
 	unsigned int num_errors;
 	unsigned int num_dev;
 	struct page *pages[1];
@@ -162,20 +173,6 @@ FF_LAYOUT_COMP(struct pnfs_layout_segment *lseg, u32 idx)
 	return NULL;
 }
 
-static inline struct nfs4_deviceid_node *
-FF_LAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id)
-{
-	struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, idx);
-
-	if (mirror != NULL) {
-		struct nfs4_ff_layout_ds *mirror_ds = mirror->dss[dss_id].mirror_ds;
-
-		if (!IS_ERR_OR_NULL(mirror_ds))
-			return &mirror_ds->id_node;
-	}
-	return NULL;
-}
-
 static inline u32
 FF_LAYOUT_MIRROR_COUNT(struct pnfs_layout_segment *lseg)
 {
@@ -207,9 +204,9 @@ ff_layout_no_read_on_rw(struct pnfs_layout_segment *lseg)
 }
 
 static inline int
-nfs4_ff_layout_ds_version(const struct nfs4_ff_layout_mirror *mirror, u32 dss_id)
+nfs4_ff_layout_ds_version(const struct nfs4_ff_layout_ds *mirror_ds)
 {
-	return mirror->dss[dss_id].mirror_ds->ds_versions[0].version;
+	return mirror_ds->ds_versions[0].version;
 }
 
 static inline u32
@@ -220,7 +217,7 @@ nfs4_ff_layout_calc_dss_id(const u64 stripe_unit, const u32 dss_count, const lof
 	if (dss_count == 1 || stripe_unit == 0)
 		return 0;
 
-	do_div(tmp, stripe_unit);
+	tmp = div64_u64(tmp, stripe_unit);
 
 	return do_div(tmp, dss_count);
 }
@@ -232,6 +229,7 @@ void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds);
 void nfs4_ff_layout_free_deviceid(struct nfs4_ff_layout_ds *mirror_ds);
 int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
 			     struct nfs4_ff_layout_mirror *mirror,
+			     const struct nfs4_deviceid_node *devid,
 			     u32 dss_id, u64 offset, u64 length, int status,
 			     enum nfs_opnum4 opnum, gfp_t gfp_flags);
 void ff_layout_send_layouterror(struct pnfs_layout_segment *lseg);
@@ -245,23 +243,29 @@ struct nfs_fh *
 nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror, u32 dss_id);
 void
 nfs4_ff_layout_select_ds_stateid(const struct nfs4_ff_layout_mirror *mirror,
+				 const struct nfs4_ff_layout_ds *mirror_ds,
 				 u32 dss_id,
 				 nfs4_stateid *stateid);
 
+struct nfs4_ff_layout_ds *
+ff_layout_get_mirror_ds(struct pnfs_layout_hdr *lo,
+			struct nfs4_ff_layout_mirror *mirror,
+			u32 dss_id);
 struct nfs4_pnfs_ds *
 nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
 			  struct nfs4_ff_layout_mirror *mirror,
+			  struct nfs4_ff_layout_ds *mirror_ds,
 			  u32 dss_id,
-			  bool fail_return);
+			  enum nfs_opnum4 opnum);
 
 struct rpc_clnt *
-nfs4_ff_find_or_create_ds_client(struct nfs4_ff_layout_mirror *mirror,
+nfs4_ff_find_or_create_ds_client(const struct nfs4_ff_layout_ds *mirror_ds,
 				 struct nfs_client *ds_clp,
-				 struct inode *inode,
-				 u32 dss_id);
+				 struct inode *inode);
 const struct cred *ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror,
 					 const struct pnfs_layout_range *range,
 					 const struct cred *mdscred,
+					 const struct nfs4_ff_layout_ds *mirror_ds,
 					 u32 dss_id);
 bool ff_layout_avoid_mds_available_ds(struct pnfs_layout_segment *lseg);
 bool ff_layout_avoid_read_on_rw(struct pnfs_layout_segment *lseg);
diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
index 5c0216bd5fcea1..6165c41fcf62ed 100644
--- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c
+++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c
@@ -20,6 +20,7 @@
 
 static unsigned int dataserver_timeo = NFS_DEF_TCP_TIMEO;
 static unsigned int dataserver_retrans;
+static unsigned int dataserver_nconnect;
 
 static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg);
 
@@ -159,26 +160,13 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
 		goto out_err_drain_dsaddrs;
 
 	/* If DS was already in cache, free ds addrs */
-	while (!list_empty(&dsaddrs)) {
-		da = list_first_entry(&dsaddrs,
-				      struct nfs4_pnfs_ds_addr,
-				      da_node);
-		list_del_init(&da->da_node);
-		kfree(da->da_remotestr);
-		kfree(da);
-	}
+	nfs4_pnfs_ds_addr_list_free(&dsaddrs);
 
 	folio_put(scratch);
 	return new_ds;
 
 out_err_drain_dsaddrs:
-	while (!list_empty(&dsaddrs)) {
-		da = list_first_entry(&dsaddrs, struct nfs4_pnfs_ds_addr,
-				      da_node);
-		list_del_init(&da->da_node);
-		kfree(da->da_remotestr);
-		kfree(da);
-	}
+	nfs4_pnfs_ds_addr_list_free(&dsaddrs);
 
 	kfree(ds_versions);
 out_scratch:
@@ -256,6 +244,7 @@ ff_layout_add_ds_error_locked(struct nfs4_flexfile_layout *flo,
 
 int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
 			     struct nfs4_ff_layout_mirror *mirror,
+			     const struct nfs4_deviceid_node *devid,
 			     u32 dss_id, u64 offset, u64 length, int status,
 			     enum nfs_opnum4 opnum, gfp_t gfp_flags)
 {
@@ -264,7 +253,7 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
 	if (status == 0)
 		return 0;
 
-	if (IS_ERR_OR_NULL(mirror->dss[dss_id].mirror_ds))
+	if (devid == NULL)
 		return -EINVAL;
 
 	dserr = kmalloc_obj(*dserr, gfp_flags);
@@ -277,8 +266,7 @@ int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
 	dserr->status = status;
 	dserr->opnum = opnum;
 	nfs4_stateid_copy(&dserr->stateid, &mirror->dss[dss_id].stateid);
-	memcpy(&dserr->deviceid, &mirror->dss[dss_id].mirror_ds->id_node.deviceid,
-	       NFS4_DEVICEID4_SIZE);
+	memcpy(&dserr->deviceid, &devid->deviceid, NFS4_DEVICEID4_SIZE);
 
 	spin_lock(&flo->generic_hdr.plh_inode->i_lock);
 	ff_layout_add_ds_error_locked(flo, dserr);
@@ -317,50 +305,81 @@ nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror, u32 dss_id)
 
 void
 nfs4_ff_layout_select_ds_stateid(const struct nfs4_ff_layout_mirror *mirror,
+				 const struct nfs4_ff_layout_ds *mirror_ds,
 				 u32 dss_id,
 				 nfs4_stateid *stateid)
 {
-	if (nfs4_ff_layout_ds_version(mirror, dss_id) == 4)
+	if (nfs4_ff_layout_ds_version(mirror_ds) == 4)
 		nfs4_stateid_copy(stateid, &mirror->dss[dss_id].stateid);
 }
 
-static bool
-ff_layout_init_mirror_ds(struct pnfs_layout_hdr *lo,
-			 struct nfs4_ff_layout_mirror *mirror,
-			 u32 dss_id)
+/*
+ * Resolve the stripe's deviceid on first use and pin the node on the
+ * mirror.  Returns a node the caller must put, or an ERR_PTR.
+ */
+struct nfs4_ff_layout_ds *
+ff_layout_get_mirror_ds(struct pnfs_layout_hdr *lo,
+			struct nfs4_ff_layout_mirror *mirror,
+			u32 dss_id)
 {
+	struct nfs4_ff_layout_ds *mirror_ds, *old;
+	struct nfs4_deviceid_node *node;
+
 	if (mirror == NULL)
-		goto outerr;
-	if (mirror->dss[dss_id].mirror_ds == NULL) {
-		struct nfs4_deviceid_node *node;
-		struct nfs4_ff_layout_ds *mirror_ds = ERR_PTR(-ENODEV);
-
-		node = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode),
-				&mirror->dss[dss_id].devid, lo->plh_lc_cred,
-				GFP_KERNEL);
-		if (node)
-			mirror_ds = FF_LAYOUT_MIRROR_DS(node);
-
-		/* check for race with another call to this function */
-		if (cmpxchg(&mirror->dss[dss_id].mirror_ds, NULL, mirror_ds) &&
-		    mirror_ds != ERR_PTR(-ENODEV))
-			nfs4_put_deviceid_node(node);
+		return ERR_PTR(-ENODEV);
+
+retry:
+	rcu_read_lock();
+	mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds);
+	if (mirror_ds && !IS_ERR(mirror_ds) &&
+	    atomic_inc_not_zero(&mirror_ds->id_node.ref)) {
+		rcu_read_unlock();
+		return mirror_ds;
+	}
+	rcu_read_unlock();
+	if (IS_ERR(mirror_ds))
+		return mirror_ds;
+	if (mirror_ds != NULL)
+		/* raced with a reset; the field is being re-pointed */
+		goto retry;
+
+	node = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode),
+			&mirror->dss[dss_id].devid, lo->plh_lc_cred,
+			GFP_KERNEL);
+	if (node) {
+		mirror_ds = FF_LAYOUT_MIRROR_DS(node);
+		/*
+		 * Take the caller's reference before the pointer becomes
+		 * visible below, so a concurrent reset of the installed
+		 * pointer cannot drop the last reference under us.
+		 */
+		atomic_inc(&node->ref);
+	} else {
+		mirror_ds = ERR_PTR(-ENODEV);
 	}
 
-	if (IS_ERR(mirror->dss[dss_id].mirror_ds))
-		goto outerr;
+	/* check for race with another call to this function */
+	old = unrcu_pointer(cmpxchg(&mirror->dss[dss_id].mirror_ds,
+				    NULL, RCU_INITIALIZER(mirror_ds)));
+	if (old == NULL)
+		return mirror_ds;
 
-	return true;
-outerr:
-	return false;
+	/* lost the race; use the winner's node instead */
+	if (node) {
+		nfs4_put_deviceid_node(node);
+		nfs4_put_deviceid_node(node);
+	}
+	goto retry;
 }
 
 /**
  * nfs4_ff_layout_prepare_ds - prepare a DS connection for an RPC call
  * @lseg: the layout segment we're operating on
  * @mirror: layout mirror describing the DS to use
+ * @mirror_ds: referenced device node for the stripe, from
+ *	ff_layout_get_mirror_ds() (may be an ERR_PTR)
  * @dss_id: DS stripe id to select stripe to use
- * @fail_return: return layout on connect failure?
+ * @opnum: operation this connection is being prepared for
  *
  * Try to prepare a DS connection to accept an RPC call. This involves
  * selecting a mirror to use and connecting the client to it if it's not
@@ -368,16 +387,19 @@ ff_layout_init_mirror_ds(struct pnfs_layout_hdr *lo,
  *
  * Since we only need a single functioning mirror to satisfy a read, we don't
  * want to return the layout if there is one. For writes though, any down
- * mirror should result in a LAYOUTRETURN. @fail_return is how we distinguish
- * between the two cases.
+ * mirror should result in a LAYOUTRETURN. @opnum is how we distinguish
+ * between the two cases. On failure, @opnum is also reported in the tracked
+ * device error so that the server can tell which class of I/O the client
+ * was unable to send to the mirror.
  *
  * Returns a pointer to a connected DS object on success or NULL on failure.
  */
 struct nfs4_pnfs_ds *
 nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
 			  struct nfs4_ff_layout_mirror *mirror,
+			  struct nfs4_ff_layout_ds *mirror_ds,
 			  u32 dss_id,
-			  bool fail_return)
+			  enum nfs_opnum4 opnum)
 {
 	struct nfs4_pnfs_ds *ds;
 	struct inode *ino = lseg->pls_layout->plh_inode;
@@ -385,10 +407,10 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
 	unsigned int max_payload;
 	int status = -EAGAIN;
 
-	if (!ff_layout_init_mirror_ds(lseg->pls_layout, mirror, dss_id))
+	if (IS_ERR_OR_NULL(mirror_ds))
 		goto noconnect;
 
-	ds = mirror->dss[dss_id].mirror_ds->ds;
+	ds = mirror_ds->ds;
 	if (READ_ONCE(ds->ds_clp))
 		goto out;
 	/* matching smp_wmb() in _nfs4_pnfs_v3/4_ds_connect */
@@ -397,11 +419,12 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
 	/* FIXME: For now we assume the server sent only one version of NFS
 	 * to use for the DS.
 	 */
-	status = nfs4_pnfs_ds_connect(s, ds, &mirror->dss[dss_id].mirror_ds->id_node,
+	status = nfs4_pnfs_ds_connect(s, ds, &mirror_ds->id_node,
 			     dataserver_timeo, dataserver_retrans,
-			     mirror->dss[dss_id].mirror_ds->ds_versions[0].version,
-			     mirror->dss[dss_id].mirror_ds->ds_versions[0].minor_version,
-			     mirror->dss[dss_id].mirror_ds->ds_versions[0].tightly_coupled);
+			     dataserver_nconnect,
+			     mirror_ds->ds_versions[0].version,
+			     mirror_ds->ds_versions[0].minor_version,
+			     mirror_ds->ds_versions[0].tightly_coupled);
 
 	/* connect success, check rsize/wsize limit */
 	if (!status) {
@@ -414,20 +437,24 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
 		max_payload =
 			nfs_block_size(rpc_max_payload(ds->ds_clp->cl_rpcclient),
 				       NULL);
-		if (mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize > max_payload)
-			mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize = max_payload;
-		if (mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize > max_payload)
-			mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize = max_payload;
+		if (mirror_ds->ds_versions[0].rsize > max_payload)
+			mirror_ds->ds_versions[0].rsize = max_payload;
+		if (mirror_ds->ds_versions[0].wsize > max_payload)
+			mirror_ds->ds_versions[0].wsize = max_payload;
 		goto out;
 	}
 noconnect:
 	ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout),
-				 mirror, dss_id, lseg->pls_range.offset,
+				 mirror,
+				 IS_ERR_OR_NULL(mirror_ds) ?
+					NULL : &mirror_ds->id_node,
+				 dss_id, lseg->pls_range.offset,
 				 lseg->pls_range.length, NFS4ERR_NXIO,
-				 OP_ILLEGAL, GFP_NOIO);
+				 opnum, GFP_NOIO);
 	ff_layout_send_layouterror(lseg);
-	if (fail_return || !ff_layout_has_available_ds(lseg))
-		pnfs_error_mark_layout_for_return(ino, lseg);
+	if (opnum != OP_READ || !ff_layout_has_available_ds(lseg))
+		pnfs_error_mark_layout_for_return(ino, lseg,
+						  &mirror->dss[dss_id].devid);
 	ds = ERR_PTR(status);
 out:
 	return ds;
@@ -437,11 +464,12 @@ const struct cred *
 ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror,
 		      const struct pnfs_layout_range *range,
 		      const struct cred *mdscred,
+		      const struct nfs4_ff_layout_ds *mirror_ds,
 		      u32 dss_id)
 {
 	const struct cred *cred;
 
-	if (mirror && !mirror->dss[dss_id].mirror_ds->ds_versions[0].tightly_coupled) {
+	if (mirror && !mirror_ds->ds_versions[0].tightly_coupled) {
 		cred = ff_layout_get_mirror_cred(mirror, range->iomode, dss_id);
 		if (!cred)
 			cred = get_cred(mdscred);
@@ -453,20 +481,18 @@ ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror,
 
 /**
  * nfs4_ff_find_or_create_ds_client - Find or create a DS rpc client
- * @mirror: pointer to the mirror
+ * @mirror_ds: device node for the stripe
  * @ds_clp: nfs_client for the DS
  * @inode: pointer to inode
- * @dss_id: DS stripe id
  *
  * Find or create a DS rpc client with th MDS server rpc client auth flavor
  * in the nfs_client cl_ds_clients list.
  */
 struct rpc_clnt *
-nfs4_ff_find_or_create_ds_client(struct nfs4_ff_layout_mirror *mirror,
-				 struct nfs_client *ds_clp, struct inode *inode,
-				 u32 dss_id)
+nfs4_ff_find_or_create_ds_client(const struct nfs4_ff_layout_ds *mirror_ds,
+				 struct nfs_client *ds_clp, struct inode *inode)
 {
-	switch (mirror->dss[dss_id].mirror_ds->ds_versions[0].version) {
+	switch (mirror_ds->ds_versions[0].version) {
 	case 3:
 		/* For NFSv3 DS, flavor is set when creating DS connections */
 		return ds_clp->cl_rpcclient;
@@ -571,49 +597,60 @@ unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
 static bool ff_read_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 {
 	struct nfs4_ff_layout_mirror *mirror;
-	struct nfs4_deviceid_node *devid;
+	struct nfs4_ff_layout_ds *mirror_ds;
+	bool ret = false;
 	u32 idx, dss_id;
 
+	rcu_read_lock();
 	for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) {
 		mirror = FF_LAYOUT_COMP(lseg, idx);
 		if (!mirror)
 			continue;
 		for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) {
-			if (!mirror->dss[dss_id].mirror_ds)
-				return true;
-			if (IS_ERR(mirror->dss[dss_id].mirror_ds))
+			mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds);
+			if (!mirror_ds) {
+				ret = true;
+				goto out;
+			}
+			if (IS_ERR(mirror_ds))
 				continue;
-			devid = &mirror->dss[dss_id].mirror_ds->id_node;
-			if (!nfs4_test_deviceid_unavailable(devid))
-				return true;
+			if (!nfs4_test_deviceid_unavailable(&mirror_ds->id_node)) {
+				ret = true;
+				goto out;
+			}
 		}
 	}
-
-	return false;
+out:
+	rcu_read_unlock();
+	return ret;
 }
 
 static bool ff_rw_layout_has_available_ds(struct pnfs_layout_segment *lseg)
 {
 	struct nfs4_ff_layout_mirror *mirror;
-	struct nfs4_deviceid_node *devid;
+	struct nfs4_ff_layout_ds *mirror_ds;
+	bool ret = false;
 	u32 idx, dss_id;
 
+	rcu_read_lock();
 	for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) {
 		mirror = FF_LAYOUT_COMP(lseg, idx);
 		if (!mirror)
-			return false;
+			goto out;
 		for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) {
-			if (IS_ERR(mirror->dss[dss_id].mirror_ds))
-				return false;
-			if (!mirror->dss[dss_id].mirror_ds)
+			mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds);
+			if (IS_ERR(mirror_ds))
+				goto out;
+			if (!mirror_ds)
 				continue;
-			devid = &mirror->dss[dss_id].mirror_ds->id_node;
-			if (nfs4_test_deviceid_unavailable(devid))
-				return false;
+			if (nfs4_test_deviceid_unavailable(&mirror_ds->id_node))
+				goto out;
 		}
 	}
-
-	return FF_LAYOUT_MIRROR_COUNT(lseg) != 0;
+	ret = FF_LAYOUT_MIRROR_COUNT(lseg) != 0;
+out:
+	rcu_read_unlock();
+	return ret;
 }
 
 static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg)
@@ -644,3 +681,8 @@ module_param(dataserver_timeo, uint, 0644);
 MODULE_PARM_DESC(dataserver_timeo, "The time (in tenths of a second) the "
 			"NFSv4.1  client  waits for a response from a "
 			" data server before it retries an NFS request.");
+module_param(dataserver_nconnect, uint, 0644);
+MODULE_PARM_DESC(dataserver_nconnect, "The maximum number of connections "
+			"the NFSv4.1 client opens to each data server, "
+			"capping the value inherited from the MDS nconnect "
+			"mount option.  0 (default) applies no cap.");
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 3022454f7698ac..3c9b2ec4e24430 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -690,7 +690,7 @@ EXPORT_SYMBOL_GPL(nfs_update_delegated_mtime);
 #define NFS_VALID_ATTRS (ATTR_MODE|ATTR_UID|ATTR_GID|ATTR_SIZE|ATTR_ATIME|ATTR_ATIME_SET|ATTR_MTIME|ATTR_MTIME_SET|ATTR_FILE|ATTR_OPEN)
 
 int
-nfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+nfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 	    struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -955,7 +955,7 @@ static u32 nfs_get_valid_attrmask(struct inode *inode)
 	return reply_mask;
 }
 
-int nfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int nfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
 	struct inode *inode = d_inode(path->dentry);
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index abc81f5ae57802..d6ea41a3f9b43d 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -251,6 +251,7 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
 					     int ds_addrlen, int ds_proto,
 					     unsigned int ds_timeo,
 					     unsigned int ds_retrans,
+					     unsigned int ds_nconnect,
 					     u32 minor_version,
 					     bool tightly_coupled);
 extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
@@ -260,7 +261,7 @@ extern void nfs4_session_limit_xasize(struct nfs_server *server);
 extern struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
 			const struct sockaddr_storage *ds_addr, int ds_addrlen,
 			int ds_proto, unsigned int ds_timeo,
-			unsigned int ds_retrans);
+			unsigned int ds_retrans, unsigned int ds_nconnect);
 #ifdef CONFIG_PROC_FS
 extern int __init nfs_fs_proc_init(void);
 extern void nfs_fs_proc_exit(void);
@@ -396,18 +397,18 @@ extern unsigned long nfs_access_cache_scan(struct shrinker *shrink,
 					   struct shrink_control *sc);
 struct dentry *nfs_lookup(struct inode *, struct dentry *, unsigned int);
 void nfs_d_prune_case_insensitive_aliases(struct inode *inode);
-int nfs_create(struct mnt_idmap *, struct inode *, struct dentry *,
+int nfs_create(const struct mnt_idmap *, struct inode *, struct dentry *,
 	       umode_t);
-struct dentry *nfs_mkdir(struct mnt_idmap *, struct inode *, struct dentry *,
+struct dentry *nfs_mkdir(const struct mnt_idmap *, struct inode *, struct dentry *,
 			 umode_t);
 int nfs_rmdir(struct inode *, struct dentry *);
 int nfs_unlink(struct inode *, struct dentry *);
-int nfs_symlink(struct mnt_idmap *, struct inode *, struct dentry *,
+int nfs_symlink(const struct mnt_idmap *, struct inode *, struct dentry *,
 		const char *);
 int nfs_link(struct dentry *, struct inode *, struct dentry *);
-int nfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *, umode_t,
+int nfs_mknod(const struct mnt_idmap *, struct inode *, struct dentry *, umode_t,
 	      dev_t);
-int nfs_rename(struct mnt_idmap *, struct inode *, struct dentry *,
+int nfs_rename(const struct mnt_idmap *, struct inode *, struct dentry *,
 	       struct inode *, struct dentry *, unsigned int);
 
 #ifdef CONFIG_NFS_V4_2
diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c
index 6d0073c24771bc..c50d59c52c50bb 100644
--- a/fs/nfs/namespace.c
+++ b/fs/nfs/namespace.c
@@ -222,7 +222,7 @@ struct vfsmount *nfs_d_automount(struct path *path)
 }
 
 static int
-nfs_namespace_getattr(struct mnt_idmap *idmap,
+nfs_namespace_getattr(const struct mnt_idmap *idmap,
 		      const struct path *path, struct kstat *stat,
 		      u32 request_mask, unsigned int query_flags)
 {
@@ -235,7 +235,7 @@ nfs_namespace_getattr(struct mnt_idmap *idmap,
 }
 
 static int
-nfs_namespace_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+nfs_namespace_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      struct iattr *attr)
 {
 	if (NFS_FH(d_inode(dentry))->size != 0)
diff --git a/fs/nfs/netns.h b/fs/nfs/netns.h
index 36658579100dbe..560fa95726b046 100644
--- a/fs/nfs/netns.h
+++ b/fs/nfs/netns.h
@@ -31,7 +31,10 @@ struct nfs_net {
 	unsigned short nfs_callback_tcpport;
 	unsigned short nfs_callback_tcpport6;
 	int cb_users[NFS4_MAX_MINOR_VERSION + 1];
-	struct list_head nfs4_data_server_cache;
+#define NFS4_DS_CACHE_HASH_BITS 8
+#define NFS4_DS_CACHE_HASH_SIZE (1 << NFS4_DS_CACHE_HASH_BITS)
+	/* every entry is still in bucket 0 until the key is added */
+	struct hlist_head nfs4_data_server_cache[NFS4_DS_CACHE_HASH_SIZE];
 	spinlock_t nfs4_data_server_lock;
 #endif /* CONFIG_NFS_V4 */
 	struct nfs_netns_client *nfs_client;
diff --git a/fs/nfs/nfs3_fs.h b/fs/nfs/nfs3_fs.h
index b333ea119ef549..ffcabadb3546f9 100644
--- a/fs/nfs/nfs3_fs.h
+++ b/fs/nfs/nfs3_fs.h
@@ -12,7 +12,7 @@
  */
 #ifdef CONFIG_NFS_V3_ACL
 extern struct posix_acl *nfs3_get_acl(struct inode *inode, int type, bool rcu);
-extern int nfs3_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+extern int nfs3_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			struct posix_acl *acl, int type);
 extern int nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl,
 		struct posix_acl *dfacl);
diff --git a/fs/nfs/nfs3acl.c b/fs/nfs/nfs3acl.c
index a126eb31f62fae..2549a1985b9afc 100644
--- a/fs/nfs/nfs3acl.c
+++ b/fs/nfs/nfs3acl.c
@@ -254,7 +254,7 @@ int nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl,
 
 }
 
-int nfs3_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int nfs3_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 	struct posix_acl *orig = acl, *dfacl = NULL, *alloc;
diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c
index 5d97c1d38bb62d..cf2f7be4b43515 100644
--- a/fs/nfs/nfs3client.c
+++ b/fs/nfs/nfs3client.c
@@ -84,7 +84,8 @@ struct nfs_server *nfs3_clone_server(struct nfs_server *source,
  */
 struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
 		const struct sockaddr_storage *ds_addr, int ds_addrlen,
-		int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans)
+		int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
+		unsigned int ds_nconnect)
 {
 	struct rpc_timeout ds_timeout;
 	unsigned long connect_timeout = ds_timeo * (ds_retrans + 1) * HZ / 10;
@@ -124,8 +125,12 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
 		fallthrough;
 	case XPRT_TRANSPORT_RDMA:
 	case XPRT_TRANSPORT_TCP:
-		if (mds_clp->cl_nconnect > 1)
+		if (mds_clp->cl_nconnect > 1) {
 			cl_init.nconnect = mds_clp->cl_nconnect;
+			if (ds_nconnect)
+				cl_init.nconnect = min(cl_init.nconnect,
+						       ds_nconnect);
+		}
 	}
 
 	if (mds_srv->flags & NFS_MOUNT_NORESVPORT)
diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
index ab86246fc364f6..75048e9702a88d 100644
--- a/fs/nfs/nfs42proc.c
+++ b/fs/nfs/nfs42proc.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2014 Anna Schumaker <Anna.Schumaker@Netapp.com>
  */
 #include <linux/fs.h>
+#include <linux/pagemap.h>
 #include <linux/sunrpc/addr.h>
 #include <linux/sunrpc/sched.h>
 #include <linux/nfs.h>
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index b48e5b87cb2a9b..76dae699d4d781 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -52,6 +52,7 @@ enum nfs4_client_state {
 	NFS4CLNT_RECALL_ANY_LAYOUT_READ,
 	NFS4CLNT_RECALL_ANY_LAYOUT_RW,
 	NFS4CLNT_DELEGRETURN_DELAYED,
+	NFS4CLNT_DEVICEID_DELETE,
 };
 
 #define NFS4_RENEW_TIMEOUT		0x01
@@ -493,6 +494,7 @@ int nfs41_discover_server_trunking(struct nfs_client *clp,
 			struct nfs_client **, const struct cred *);
 extern void nfs4_schedule_session_recovery(struct nfs4_session *, int);
 extern void nfs41_notify_server(struct nfs_client *);
+extern void nfs4_deviceid_delete_recover_run(struct nfs_client *clp);
 bool nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1,
 			struct nfs41_server_owner *o2);
 
@@ -509,6 +511,7 @@ extern void nfs_inode_find_state_and_recover(struct inode *inode,
 		const nfs4_stateid *stateid);
 extern int nfs4_state_mark_reclaim_nograce(struct nfs_client *, struct nfs4_state *);
 extern void nfs4_schedule_lease_recovery(struct nfs_client *);
+extern void nfs4_reset_all_state(struct nfs_client *);
 extern int nfs4_wait_clnt_recover(struct nfs_client *clp);
 extern int nfs4_client_recover_expired_lease(struct nfs_client *clp);
 extern void nfs4_schedule_state_manager(struct nfs_client *);
diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
index b661f446ea49e8..fe779fb2ec726d 100644
--- a/fs/nfs/nfs4client.c
+++ b/fs/nfs/nfs4client.c
@@ -217,6 +217,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init)
 	clp->cl_last_renewal = jiffies;
 	init_waitqueue_head(&clp->cl_lock_waitq);
 	INIT_LIST_HEAD(&clp->pending_cb_stateids);
+	INIT_LIST_HEAD(&clp->cl_deviceid_deletes);
 
 	if (cl_init->minorversion != 0)
 		__set_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags);
@@ -286,6 +287,7 @@ static void nfs4_shutdown_client(struct nfs_client *clp)
 		nfs4_kill_renewd(clp);
 	clp->cl_mvops->shutdown_client(clp);
 	nfs4_destroy_callback(clp);
+	pnfs_deviceid_delete_queue_free(clp);
 	if (__test_and_clear_bit(NFS_CS_IDMAP, &clp->cl_res_state))
 		nfs_idmap_delete(clp);
 
@@ -792,7 +794,8 @@ static int nfs4_set_client(struct nfs_server *server,
 struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
 		const struct sockaddr_storage *ds_addr, int ds_addrlen,
 		int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans,
-		u32 minor_version, bool tightly_coupled)
+		unsigned int ds_nconnect, u32 minor_version,
+		bool tightly_coupled)
 {
 	struct rpc_timeout ds_timeout;
 	struct nfs_client *mds_clp = mds_srv->nfs_client;
@@ -830,6 +833,9 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
 	case XPRT_TRANSPORT_TCP:
 		if (mds_clp->cl_nconnect > 1) {
 			cl_init.nconnect = mds_clp->cl_nconnect;
+			if (ds_nconnect)
+				cl_init.nconnect = min(cl_init.nconnect,
+						       ds_nconnect);
 			cl_init.max_connect = NFS_MAX_TRANSPORTS;
 		}
 	}
diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c
index 6401f6363f7534..9a434f5dda8d36 100644
--- a/fs/nfs/nfs4file.c
+++ b/fs/nfs/nfs4file.c
@@ -402,6 +402,7 @@ static void __nfs42_ssc_close(struct file *filep)
 }
 
 static const struct nfs4_ssc_client_ops nfs4_ssc_clnt_ops_tbl = {
+	.owner = THIS_MODULE,
 	.sco_open = __nfs42_ssc_open,
 	.sco_close = __nfs42_ssc_close,
 };
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 04b1987115d5da..beb659744760f2 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -7866,7 +7866,7 @@ int nfs4_lock_delegation_recall(struct file_lock *fl, struct nfs4_state *state,
 #define XATTR_NAME_NFSV4_ACL "system.nfs4_acl"
 
 static int nfs4_xattr_set_nfs4_acl(const struct xattr_handler *handler,
-				   struct mnt_idmap *idmap,
+				   const struct mnt_idmap *idmap,
 				   struct dentry *unused, struct inode *inode,
 				   const char *key, const void *buf,
 				   size_t buflen, int flags)
@@ -7889,7 +7889,7 @@ static bool nfs4_xattr_list_nfs4_acl(struct dentry *dentry)
 #define XATTR_NAME_NFSV4_DACL "system.nfs4_dacl"
 
 static int nfs4_xattr_set_nfs4_dacl(const struct xattr_handler *handler,
-				    struct mnt_idmap *idmap,
+				    const struct mnt_idmap *idmap,
 				    struct dentry *unused, struct inode *inode,
 				    const char *key, const void *buf,
 				    size_t buflen, int flags)
@@ -7912,7 +7912,7 @@ static bool nfs4_xattr_list_nfs4_dacl(struct dentry *dentry)
 #define XATTR_NAME_NFSV4_SACL "system.nfs4_sacl"
 
 static int nfs4_xattr_set_nfs4_sacl(const struct xattr_handler *handler,
-				    struct mnt_idmap *idmap,
+				    const struct mnt_idmap *idmap,
 				    struct dentry *unused, struct inode *inode,
 				    const char *key, const void *buf,
 				    size_t buflen, int flags)
@@ -7935,7 +7935,7 @@ static bool nfs4_xattr_list_nfs4_sacl(struct dentry *dentry)
 #ifdef CONFIG_NFS_V4_SECURITY_LABEL
 
 static int nfs4_xattr_set_nfs4_label(const struct xattr_handler *handler,
-				     struct mnt_idmap *idmap,
+				     const struct mnt_idmap *idmap,
 				     struct dentry *unused, struct inode *inode,
 				     const char *key, const void *buf,
 				     size_t buflen, int flags)
@@ -7965,7 +7965,7 @@ static const struct xattr_handler nfs4_xattr_nfs4_label_handler = {
 
 #ifdef CONFIG_NFS_V4_2
 static int nfs4_xattr_set_nfs4_user(const struct xattr_handler *handler,
-				    struct mnt_idmap *idmap,
+				    const struct mnt_idmap *idmap,
 				    struct dentry *unused, struct inode *inode,
 				    const char *key, const void *buf,
 				    size_t buflen, int flags)
@@ -9680,6 +9680,14 @@ nfs4_layoutget_handle_exception(struct rpc_task *task,
 	case -NFS4ERR_BADLAYOUT:
 		status = -EOVERFLOW;
 		goto out;
+	/*
+	 * NFS4ERR_TOOSMALL means the layout for the requested range
+	 * exceeds what the client advertised in loga_maxcount (see
+	 * RFC8881 section 18.43.3).
+	 */
+	case -ETOOSMALL:
+		status = -EMSGSIZE;
+		goto out;
 	/*
 	 * NFS4ERR_LAYOUTTRYLATER is a conflict with another client
 	 * (or clients) writing to the same RAID stripe except when
@@ -10498,6 +10506,134 @@ static int nfs41_free_stateid(struct nfs_server *server,
 	return ret;
 }
 
+/*
+ * GETDEVICEINFO surfacing the raw status; nfs4_get_device_info()
+ * swallows it.  A device too large for one page fails with something
+ * other than -ENOENT, which still proves existence.
+ */
+static int nfs4_deviceid_validate(struct nfs_server *server,
+		const struct pnfs_layoutdriver_type *ld,
+		const struct nfs4_deviceid *id, const struct cred *cred)
+{
+	struct pnfs_device pdev;
+	struct page *page;
+	int status;
+
+	page = alloc_page(GFP_KERNEL);
+	if (!page)
+		return -ENOMEM;
+
+	memset(&pdev, 0, sizeof(pdev));
+	memcpy(&pdev.dev_id, id, sizeof(pdev.dev_id));
+	pdev.layout_type = ld->id;
+	pdev.pages = &page;
+	pdev.pglen = PAGE_SIZE;
+	pdev.maxcount = PAGE_SIZE - nfs41_maxgetdevinfo_overhead;
+
+	status = nfs4_proc_getdeviceinfo(server, &pdev, cred);
+	__free_page(page);
+	return status;
+}
+
+/*
+ * A DELETE for a deviceID we still hold layouts on implies the server
+ * revoked them: run the RFC 8881 Section 18.40.4 recovery.  A layout the
+ * server still calls valid leaves the revocations unable to confirm the
+ * delete, so verify it with GETDEVICEINFO.
+ */
+static void nfs4_deviceid_delete_recover(struct nfs_client *clp,
+		const struct pnfs_layoutdriver_type *ld,
+		const struct nfs4_deviceid *id)
+{
+	LIST_HEAD(layouts);
+	struct nfs4_deviceid_ref *ref, *confirm = NULL;
+	bool revoked = false;
+	bool inconclusive = false;
+	int status;
+
+	if (pnfs_layout_collect_deviceid_refs(clp, ld, id, &layouts)) {
+		/* Only a partial list -- an allocation failed, or an inode is
+		 * being evicted.  Leave the device cached and recover on a
+		 * later notification.
+		 */
+		pnfs_layout_put_deviceid_refs(&layouts);
+		return;
+	}
+
+	if (list_empty(&layouts)) {
+		nfs4_delete_deviceid(ld, clp, id);
+		return;
+	}
+
+	list_for_each_entry(ref, &layouts, node) {
+		struct pnfs_layout_hdr *lo = ref->lo;
+		struct inode *inode = ref->inode;
+		bool invalidated = false;
+		LIST_HEAD(head);
+
+		status = nfs41_test_stateid(NFS_SERVER(inode), &ref->stateid,
+					    ref->cred);
+		switch (status) {
+		case NFS_OK:
+		case -NFS4ERR_OLD_STATEID:
+			if (!confirm)
+				confirm = ref;
+			break;
+		case -NFS4ERR_ADMIN_REVOKED:
+		case -NFS4ERR_DELEG_REVOKED:
+		case -NFS4ERR_EXPIRED:
+		case -NFS4ERR_BAD_STATEID:
+			spin_lock(&inode->i_lock);
+			if (pnfs_layout_is_valid(lo) &&
+			    nfs4_stateid_match_other(&ref->stateid,
+						     &lo->plh_stateid)) {
+				pnfs_mark_layout_stateid_invalid(lo, &head);
+				revoked = true;
+				invalidated = true;
+			}
+			spin_unlock(&inode->i_lock);
+			pnfs_free_lseg_list(&head);
+			if (invalidated)
+				nfs_commit_inode(inode, 0);
+			nfs41_free_stateid(NFS_SERVER(inode), &ref->stateid,
+					   ref->cred, true);
+			break;
+		default:
+			inconclusive = true;
+			break;
+		}
+	}
+
+	if (confirm) {
+		status = nfs4_deviceid_validate(NFS_SERVER(confirm->inode),
+						ld, id, confirm->cred);
+		if (status == -ENOENT) {
+			/* Section 18.40.4 prescribes EXCHANGE_ID here;
+			 * nfs4_schedule_lease_recovery() would only renew
+			 * the existing lease.
+			 */
+			pr_warn_ratelimited("NFS: server %s deleted a deviceID referred to by a layout it still considers valid; re-establishing the client ID\n",
+					    clp->cl_hostname);
+			nfs4_reset_all_state(clp);
+			nfs4_delete_deviceid(ld, clp, id);
+		}
+	} else if (revoked && !inconclusive) {
+		nfs4_delete_deviceid(ld, clp, id);
+	}
+	pnfs_layout_put_deviceid_refs(&layouts);
+}
+
+void nfs4_deviceid_delete_recover_run(struct nfs_client *clp)
+{
+	struct nfs4_deviceid_delete *dd;
+
+	while ((dd = pnfs_deviceid_delete_dequeue(clp)) != NULL) {
+		nfs4_deviceid_delete_recover(clp, dd->ld, &dd->id);
+		pnfs_put_layoutdriver(dd->ld);
+		kfree(dd);
+	}
+}
+
 static void
 nfs41_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp)
 {
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index a5dec0473e221a..b5d6daa2c41624 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -2354,7 +2354,7 @@ void nfs41_notify_server(struct nfs_client *clp)
 	nfs4_schedule_state_manager(clp);
 }
 
-static void nfs4_reset_all_state(struct nfs_client *clp)
+void nfs4_reset_all_state(struct nfs_client *clp)
 {
 	if (test_and_set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state) == 0) {
 		set_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state);
@@ -2669,6 +2669,11 @@ static void nfs4_state_manager(struct nfs_client *clp)
 				set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state);
 			}
 			nfs4_layoutreturn_any_run(clp);
+			if (test_and_clear_bit(NFS4CLNT_DEVICEID_DELETE,
+					       &clp->cl_state)) {
+				nfs4_deviceid_delete_recover_run(clp);
+				set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state);
+			}
 			clear_bit(NFS4CLNT_RECALL_RUNNING, &clp->cl_state);
 		}
 
diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c
index fc049ce4ba8aca..8b3d96b4a0f0f2 100644
--- a/fs/nfs/nfs4xdr.c
+++ b/fs/nfs/nfs4xdr.c
@@ -6187,7 +6187,7 @@ static int decode_layoutget(struct xdr_stream *xdr, struct rpc_rqst *req,
 		dprintk("NFS: server cheating in layoutget reply: "
 				"layout len %u > recvd %u\n",
 				res->layoutp->len, recvd);
-		status = -EINVAL;
+		status = -EMSGSIZE;
 		goto out;
 	}
 
diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h
index b15c1732c86923..a75cd7f8c6d55a 100644
--- a/fs/nfs/nfstrace.h
+++ b/fs/nfs/nfstrace.h
@@ -9,6 +9,7 @@
 #define _TRACE_NFS_H
 
 #include <linux/tracepoint.h>
+#include <linux/pagemap.h>
 #include <linux/iversion.h>
 
 #include <trace/misc/fs.h>
diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c
index 7dd478ffc2fab4..71f0ce2bc4eabe 100644
--- a/fs/nfs/pagelist.c
+++ b/fs/nfs/pagelist.c
@@ -404,20 +404,28 @@ static struct nfs_page *nfs_page_create(struct nfs_lock_context *l_ctx,
 	return req;
 }
 
-static void nfs_page_assign_folio(struct nfs_page *req, struct folio *folio)
+static void nfs_page_assign_folio(struct nfs_page *req, struct folio *folio,
+				  bool pinned)
 {
 	if (folio != NULL) {
 		req->wb_folio = folio;
-		folio_get(folio);
+		if (pinned)
+			set_bit(PG_PINNED, &req->wb_flags);
+		else
+			folio_get(folio);
 		set_bit(PG_FOLIO, &req->wb_flags);
 	}
 }
 
-static void nfs_page_assign_page(struct nfs_page *req, struct page *page)
+static void nfs_page_assign_page(struct nfs_page *req, struct page *page,
+				 bool pinned)
 {
 	if (page != NULL) {
 		req->wb_page = page;
-		get_page(page);
+		if (pinned)
+			set_bit(PG_PINNED, &req->wb_flags);
+		else
+			get_page(page);
 	}
 }
 
@@ -425,6 +433,7 @@ static void nfs_page_assign_page(struct nfs_page *req, struct page *page)
  * nfs_page_create_from_page - Create an NFS read/write request.
  * @ctx: open context to use
  * @page: page to write
+ * @pinned: true if page is pinned
  * @pgbase: starting offset within the page for the write
  * @offset: file offset for the write
  * @count: number of bytes to read/write
@@ -435,6 +444,7 @@ static void nfs_page_assign_page(struct nfs_page *req, struct page *page)
  */
 struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx,
 					   struct page *page,
+					   bool pinned,
 					   unsigned int pgbase, loff_t offset,
 					   unsigned int count)
 {
@@ -446,7 +456,9 @@ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx,
 	ret = nfs_page_create(l_ctx, pgbase, offset >> PAGE_SHIFT,
 			      offset_in_page(offset), count);
 	if (!IS_ERR(ret)) {
-		nfs_page_assign_page(ret, page);
+		nfs_page_assign_page(ret, page, pinned);
+		if (pinned)
+			ret->wb_nr_pinned = 1;
 		nfs_page_group_init(ret, NULL);
 	}
 	nfs_put_lock_context(l_ctx);
@@ -457,6 +469,7 @@ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx,
  * nfs_page_create_from_folio - Create an NFS read/write request.
  * @ctx: open context to use
  * @folio: folio to write
+ * @pinned: true if folio is pinned
  * @offset: starting offset within the folio for the write
  * @count: number of bytes to read/write
  *
@@ -466,6 +479,7 @@ struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx,
  */
 struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx,
 					    struct folio *folio,
+					    bool pinned,
 					    unsigned int offset,
 					    unsigned int count)
 {
@@ -476,7 +490,10 @@ struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx,
 		return ERR_CAST(l_ctx);
 	ret = nfs_page_create(l_ctx, offset, folio->index, offset, count);
 	if (!IS_ERR(ret)) {
-		nfs_page_assign_folio(ret, folio);
+		nfs_page_assign_folio(ret, folio, pinned);
+		if (pinned)
+			ret->wb_nr_pinned = nfs_page_array_len(offset_in_page(offset),
+							      count);
 		nfs_page_group_init(ret, NULL);
 	}
 	nfs_put_lock_context(l_ctx);
@@ -498,9 +515,11 @@ nfs_create_subreq(struct nfs_page *req,
 			      offset, count);
 	if (!IS_ERR(ret)) {
 		if (folio)
-			nfs_page_assign_folio(ret, folio);
+			nfs_page_assign_folio(ret, folio,
+					      test_bit(PG_PINNED, &req->wb_flags));
 		else
-			nfs_page_assign_page(ret, page);
+			nfs_page_assign_page(ret, page,
+					     test_bit(PG_PINNED, &req->wb_flags));
 		/* find the last request */
 		for (last = req->wb_head;
 		     last->wb_this_page != req->wb_head;
@@ -552,11 +571,21 @@ static void nfs_clear_request(struct nfs_page *req)
 	struct nfs_open_context *ctx;
 
 	if (folio != NULL) {
-		folio_put(folio);
+		if (test_and_clear_bit(PG_PINNED, &req->wb_flags)) {
+			if (req->wb_nr_pinned > 0)
+				unpin_user_folio(folio, req->wb_nr_pinned);
+		} else {
+			folio_put(folio);
+		}
 		req->wb_folio = NULL;
 		clear_bit(PG_FOLIO, &req->wb_flags);
 	} else if (page != NULL) {
-		put_page(page);
+		if (test_and_clear_bit(PG_PINNED, &req->wb_flags)) {
+			if (req->wb_nr_pinned > 0)
+				unpin_user_pages(&page, req->wb_nr_pinned);
+		} else {
+			put_page(page);
+		}
 		req->wb_page = NULL;
 	}
 	if (l_ctx != NULL) {
@@ -599,6 +628,23 @@ void nfs_release_request(struct nfs_page *req)
 }
 EXPORT_SYMBOL_GPL(nfs_release_request);
 
+/*
+ * nfs_release_request_list - Release a list of NFS read/write requests
+ * @head: list of requests to release
+ *
+ * Removes each request from the list and drops it's refcount.
+ */
+void nfs_release_request_list(struct list_head *head)
+{
+	while (!list_empty(head)) {
+		struct nfs_page *req = nfs_list_entry(head->next);
+
+		nfs_list_remove_request(req);
+		nfs_release_request(req);
+	}
+}
+EXPORT_SYMBOL_GPL(nfs_release_request_list);
+
 /*
  * nfs_generic_pg_test - determine if requests can be coalesced
  * @desc: pointer to descriptor
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 4f9c0f639014f4..5d561562b59f41 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -30,6 +30,7 @@
 #include <linux/nfs_fs.h>
 #include <linux/nfs_page.h>
 #include <linux/module.h>
+#include <linux/pagemap.h>
 #include <linux/sort.h>
 #include "internal.h"
 #include "pnfs.h"
@@ -433,7 +434,7 @@ bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst,
 		}
 		/* Try to update the seqid to the most recent */
 		err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0,
-						      true);
+						      true, NULL);
 		if (err != -EBUSY) {
 			dst->seqid = lo->plh_stateid.seqid;
 			*dst_range = range;
@@ -487,7 +488,8 @@ static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo,
 		.length = NFS4_MAX_UINT64,
 	};
 
-	return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq, true);
+	return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq, true,
+					       NULL);
 }
 
 static int
@@ -525,7 +527,7 @@ pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
 
 	spin_lock(&inode->i_lock);
 	pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
-	pnfs_mark_matching_lsegs_return(lo, &head, &range, 0, true);
+	pnfs_mark_matching_lsegs_return(lo, &head, &range, 0, true, NULL);
 	spin_unlock(&inode->i_lock);
 	pnfs_free_lseg_list(&head);
 	dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
@@ -740,7 +742,7 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
 			if (mark_lseg_invalid(lseg, tmp_list))
 				continue;
 			remaining++;
-			pnfs_lseg_cancel_io(server, lseg);
+			pnfs_lseg_cancel_io(server, lseg, NULL);
 		}
 	dprintk("%s:Return %i\n", __func__, remaining);
 	return remaining;
@@ -1167,11 +1169,12 @@ pnfs_alloc_init_layoutget_args(struct inode *ino,
 	   struct nfs_open_context *ctx,
 	   const nfs4_stateid *stateid,
 	   const struct pnfs_layout_range *range,
-	   gfp_t gfp_flags)
+	   size_t min_reply_sz, gfp_t gfp_flags)
 {
 	struct nfs_server *server = pnfs_find_server(ino, ctx);
 	size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response;
-	size_t max_pages = max_response_pages(server);
+	size_t session_pages = max_response_pages(server);
+	size_t max_pages = session_pages;
 	struct nfs4_layoutget *lgp;
 
 	dprintk("--> %s\n", __func__);
@@ -1186,6 +1189,19 @@ pnfs_alloc_init_layoutget_args(struct inode *ino,
 			max_pages = npages;
 	}
 
+	/*
+	 * A previous LAYOUTGET on this layout or on this server did not
+	 * fit the reply buffer: raise the layout driver's default up to
+	 * the session's maximum response size.
+	 */
+	if (!min_reply_sz)
+		min_reply_sz = READ_ONCE(server->lg_reply_sz);
+	if (min_reply_sz) {
+		size_t npages = (min_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT;
+		if (npages > max_pages)
+			max_pages = min(npages, session_pages);
+	}
+
 	lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags);
 	if (!lgp->args.layout.pages) {
 		kfree(lgp);
@@ -1210,7 +1226,7 @@ pnfs_alloc_init_layoutget_args(struct inode *ino,
 				lgp->args.minlength = i_size - range->offset;
 		}
 	}
-	lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
+	lgp->args.maxcount = lgp->args.layout.pglen;
 	pnfs_copy_range(&lgp->args.range, range);
 	lgp->args.type = server->pnfs_curr_ld->id;
 	lgp->args.inode = ino;
@@ -1462,7 +1478,7 @@ _pnfs_return_layout(struct inode *ino)
 	}
 	valid_layout = pnfs_layout_is_valid(lo);
 	pnfs_clear_layoutcommit(ino, &tmp_list);
-	pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0, true);
+	pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0, true, NULL);
 
 
 	/* Don't send a LAYOUTRETURN if list was initially empty */
@@ -2145,6 +2161,7 @@ pnfs_update_layout(struct inode *ino,
 		.inode = ino,
 	};
 	unsigned long giveup = jiffies + (clp->cl_lease_time << 1);
+	size_t reply_sz = 0;
 	bool first;
 
 	if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
@@ -2304,7 +2321,8 @@ pnfs_update_layout(struct inode *ino,
 	if (arg.length != NFS4_MAX_UINT64)
 		arg.length = PAGE_ALIGN(arg.length);
 
-	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags);
+	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, reply_sz,
+					     gfp_flags);
 	if (!lgp) {
 		lseg = ERR_PTR(-ENOMEM);
 		trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL,
@@ -2335,6 +2353,25 @@ pnfs_update_layout(struct inode *ino,
 				lo, pnfs_iomode_to_fail_bit(iomode));
 			lseg = NULL;
 			goto out_put_layout_hdr;
+		case -EMSGSIZE: {
+			/*
+			 * The layout exceeded loga_maxcount (NFS4ERR_TOOSMALL):
+			 * retry once with the reply buffer raised to the
+			 * session's maximum response size before falling back
+			 * to I/O through the MDS.
+			 */
+			size_t max = max_response_pages(server) << PAGE_SHIFT;
+
+			if (reply_sz < max) {
+				reply_sz = max;
+				exception.retry = 1;
+				break;
+			}
+			pnfs_layout_set_fail_bit(
+				lo, pnfs_iomode_to_fail_bit(iomode));
+			lseg = NULL;
+			goto out_put_layout_hdr;
+		}
 		default:
 			if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
 				pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
@@ -2354,6 +2391,13 @@ pnfs_update_layout(struct inode *ino,
 			goto lookup_again;
 		}
 	} else {
+		/*
+		 * A LAYOUTGET that only succeeded with an escalated reply
+		 * buffer: remember the size so that future LAYOUTGETs to
+		 * this server skip the attempt at the driver's default.
+		 */
+		if (reply_sz)
+			WRITE_ONCE(server->lg_reply_sz, reply_sz);
 		pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
 	}
 
@@ -2448,7 +2492,7 @@ static void _lgopen_prepare_attached(struct nfs4_opendata *data,
 	lo = _pnfs_grab_empty_layout(ino, ctx);
 	if (!lo)
 		return;
-	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &current_stateid, &rng,
+	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &current_stateid, &rng, 0,
 					     nfs_io_gfp_mask());
 	if (!lgp) {
 		clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags);
@@ -2474,7 +2518,7 @@ static void _lgopen_prepare_floating(struct nfs4_opendata *data,
 	};
 	struct nfs4_layoutget *lgp;
 
-	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &current_stateid, &rng,
+	lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &current_stateid, &rng, 0,
 					     nfs_io_gfp_mask());
 	if (!lgp)
 		return;
@@ -2616,7 +2660,8 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
 			.iomode = IOMODE_ANY,
 			.length = NFS4_MAX_UINT64,
 		};
-		pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0, true);
+		pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0, true,
+						NULL);
 		goto out_forget;
 	} else {
 		/* We have a completely new layout */
@@ -2649,6 +2694,7 @@ pnfs_layout_process(struct nfs4_layoutget *lgp)
  * @return_range: describe layout segment ranges to be returned
  * @seq: stateid seqid to match
  * @cancel_io: signal io be cancelled
+ * @devid: only cancel io directed at this device (all devices if NULL)
  *
  * This function is mainly intended for use by layoutrecall. It attempts
  * to free the layout segment immediately, or else to mark it for return
@@ -2663,7 +2709,8 @@ int
 pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
 				struct list_head *tmp_list,
 				const struct pnfs_layout_range *return_range,
-				u32 seq, bool cancel_io)
+				u32 seq, bool cancel_io,
+				const struct nfs4_deviceid *devid)
 {
 	struct pnfs_layout_segment *lseg, *next;
 	struct nfs_server *server = NFS_SERVER(lo->plh_inode);
@@ -2690,7 +2737,7 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
 			remaining++;
 			set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
 			if (cancel_io)
-				pnfs_lseg_cancel_io(server, lseg);
+				pnfs_lseg_cancel_io(server, lseg, devid);
 		}
 
 	if (remaining) {
@@ -2708,7 +2755,8 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
 
 static void
 pnfs_mark_layout_for_return(struct inode *inode,
-			    const struct pnfs_layout_range *range)
+			    const struct pnfs_layout_range *range,
+			    const struct nfs4_deviceid *devid)
 {
 	struct pnfs_layout_hdr *lo;
 	bool return_now = false;
@@ -2726,7 +2774,7 @@ pnfs_mark_layout_for_return(struct inode *inode,
 	 * for how it works.
 	 */
 	if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0,
-					    true) != -EBUSY) {
+					    true, devid) != -EBUSY) {
 		const struct cred *cred;
 		nfs4_stateid stateid;
 		enum pnfs_iomode iomode;
@@ -2743,7 +2791,8 @@ pnfs_mark_layout_for_return(struct inode *inode,
 }
 
 void pnfs_error_mark_layout_for_return(struct inode *inode,
-				       struct pnfs_layout_segment *lseg)
+				       struct pnfs_layout_segment *lseg,
+				       const struct nfs4_deviceid *devid)
 {
 	struct pnfs_layout_range range = {
 		.iomode = lseg->pls_range.iomode,
@@ -2751,7 +2800,7 @@ void pnfs_error_mark_layout_for_return(struct inode *inode,
 		.length = NFS4_MAX_UINT64,
 	};
 
-	pnfs_mark_layout_for_return(inode, &range);
+	pnfs_mark_layout_for_return(inode, &range, devid);
 }
 EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);
 
@@ -2841,7 +2890,7 @@ static int pnfs_layout_return_unused_byserver(struct nfs_server *server,
 		pnfs_get_layout_hdr(lo);
 		pnfs_set_plh_return_info(lo, range->iomode, 0);
 		if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs,
-						    range, 0, true) != 0 ||
+						    range, 0, true, NULL) != 0 ||
 		    !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) {
 			spin_unlock(&inode->i_lock);
 			rcu_read_unlock();
@@ -2875,6 +2924,294 @@ pnfs_layout_return_unused_byclid(struct nfs_client *clp,
 			&range);
 }
 
+struct pnfs_reresolve_deviceid_args {
+	const struct pnfs_layoutdriver_type *ld;
+	const struct nfs4_deviceid *devid;
+	bool immediate;
+	struct list_head put_list;
+};
+
+static int pnfs_layout_reresolve_deviceid_byserver(struct nfs_server *server,
+						   void *data)
+{
+	struct pnfs_reresolve_deviceid_args *args = data;
+	struct pnfs_layout_hdr *lo;
+	struct inode *inode;
+
+	if (server->pnfs_curr_ld != args->ld)
+		return 0;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) {
+		inode = lo->plh_inode;
+		if (!inode)
+			continue;
+		spin_lock(&inode->i_lock);
+		args->ld->reresolve_deviceid(lo, args->devid, args->immediate,
+					     &args->put_list);
+		spin_unlock(&inode->i_lock);
+	}
+	rcu_read_unlock();
+	return 0;
+}
+
+/*
+ * Invoke @ld's reresolve_deviceid hook for @devid on every layout of @clp's
+ * servers, then drain the put_list once the locks are dropped.
+ */
+void
+pnfs_layout_reresolve_deviceid_byclid(struct nfs_client *clp,
+				      const struct pnfs_layoutdriver_type *ld,
+				      const struct nfs4_deviceid *devid,
+				      bool immediate)
+{
+	struct pnfs_reresolve_deviceid_args args = {
+		.ld = ld,
+		.devid = devid,
+		.immediate = immediate,
+		.put_list = LIST_HEAD_INIT(args.put_list),
+	};
+	struct nfs4_deviceid_put *put, *tmp;
+
+	if (!ld->reresolve_deviceid)
+		return;
+
+	nfs_client_for_each_server(clp,
+			pnfs_layout_reresolve_deviceid_byserver, &args);
+
+	list_for_each_entry_safe(put, tmp, &args.put_list, node) {
+		list_del(&put->node);
+		nfs4_put_deviceid_node(put->dev);
+		kfree(put);
+	}
+}
+
+struct pnfs_deviceid_ref_args {
+	const struct pnfs_layoutdriver_type *ld;
+	const struct nfs4_deviceid *devid;
+	struct list_head *result;
+	bool found;
+};
+
+static int pnfs_layout_deviceid_referenced_byserver(
+		struct nfs_server *server, void *data)
+{
+	struct pnfs_deviceid_ref_args *args = data;
+	struct pnfs_layout_hdr *lo;
+	struct inode *inode;
+
+	if (server->pnfs_curr_ld != args->ld)
+		return 0;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) {
+		inode = lo->plh_inode;
+		if (!inode)
+			continue;
+		spin_lock(&inode->i_lock);
+		if (NFS_I(inode)->layout == lo && pnfs_layout_is_valid(lo) &&
+		    args->ld->layout_references_deviceid(lo, args->devid))
+			args->found = true;
+		spin_unlock(&inode->i_lock);
+		if (args->found)
+			break;
+	}
+	rcu_read_unlock();
+	return args->found;
+}
+
+/*
+ * pnfs_layout_deviceid_referenced_byclid - does any live layout of
+ * @clp's servers using @ld still reference deviceid @devid?
+ */
+bool
+pnfs_layout_deviceid_referenced_byclid(struct nfs_client *clp,
+				const struct pnfs_layoutdriver_type *ld,
+				const struct nfs4_deviceid *devid)
+{
+	struct pnfs_deviceid_ref_args args = {
+		.ld = ld,
+		.devid = devid,
+	};
+
+	if (!ld->layout_references_deviceid)
+		return false;
+
+	nfs_client_for_each_server(clp,
+			pnfs_layout_deviceid_referenced_byserver, &args);
+	return args.found;
+}
+
+static int pnfs_layout_collect_deviceid_refs_byserver(
+		struct nfs_server *server, void *data)
+{
+	struct pnfs_deviceid_ref_args *args = data;
+	struct nfs4_deviceid_ref *ref, *tmp;
+	struct pnfs_layout_hdr *lo;
+	struct inode *inode;
+	LIST_HEAD(putme);
+	bool matched;
+	int ret = 0;
+
+	if (server->pnfs_curr_ld != args->ld)
+		return 0;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) {
+		inode = lo->plh_inode;
+		if (!inode)
+			continue;
+
+		spin_lock(&inode->i_lock);
+		matched = NFS_I(inode)->layout == lo &&
+			  pnfs_layout_is_valid(lo) &&
+			  args->ld->layout_references_deviceid(lo, args->devid);
+		if (!matched) {
+			spin_unlock(&inode->i_lock);
+			continue;
+		}
+		ref = kzalloc_obj(*ref, GFP_ATOMIC);
+		if (!ref) {
+			spin_unlock(&inode->i_lock);
+			ret = -ENOMEM;
+			break;
+		}
+		/* NFS_I()->layout == lo under i_lock means the refcount has
+		 * not reached zero: pnfs_put_layout_hdr() decrements to zero
+		 * and detaches in the same critical section.
+		 */
+		pnfs_get_layout_hdr(lo);
+		ref->lo = lo;
+		nfs4_stateid_copy(&ref->stateid, &lo->plh_stateid);
+		ref->cred = get_cred(lo->plh_lc_cred);
+		spin_unlock(&inode->i_lock);
+
+		/* the pinned hdr does not hold the inode: grab it (and
+		 * keep the superblock active) for use across RPCs
+		 */
+		ref->inode = nfs_igrab_and_active(inode);
+		if (!ref->inode) {
+			/* The layout may still name the deviceID, so report a
+			 * partial list rather than silently shortening it.
+			 * Defer the put: it can layoutreturn and sleep.
+			 */
+			list_add(&ref->node, &putme);
+			ret = -EAGAIN;
+			break;
+		}
+		list_add_tail(&ref->node, args->result);
+	}
+	rcu_read_unlock();
+
+	list_for_each_entry_safe(ref, tmp, &putme, node) {
+		list_del(&ref->node);
+		pnfs_put_layout_hdr(ref->lo);
+		put_cred(ref->cred);
+		kfree(ref);
+	}
+	return ret;
+}
+
+/*
+ * Collect @clp's layouts referencing @devid onto @result as entries usable
+ * across sleeping RPCs; release with pnfs_layout_put_deviceid_refs().
+ * A negative return means @result is only a partial set.
+ */
+int
+pnfs_layout_collect_deviceid_refs(struct nfs_client *clp,
+				const struct pnfs_layoutdriver_type *ld,
+				const struct nfs4_deviceid *devid,
+				struct list_head *result)
+{
+	struct pnfs_deviceid_ref_args args = {
+		.ld = ld,
+		.devid = devid,
+		.result = result,
+	};
+
+	if (!ld->layout_references_deviceid)
+		return 0;
+
+	return nfs_client_for_each_server(clp,
+			pnfs_layout_collect_deviceid_refs_byserver, &args);
+}
+
+void
+pnfs_layout_put_deviceid_refs(struct list_head *result)
+{
+	struct nfs4_deviceid_ref *ref, *tmp;
+
+	list_for_each_entry_safe(ref, tmp, result, node) {
+		list_del(&ref->node);
+		put_cred(ref->cred);
+		pnfs_put_layout_hdr(ref->lo);
+		nfs_iput_and_deactive(ref->inode);
+		kfree(ref);
+	}
+}
+
+/*
+ * Queue @id for the state manager's Section 18.40.4 recovery,
+ * dropping duplicates of an already-queued suspect.
+ */
+void pnfs_deviceid_delete_mark(struct nfs_client *clp,
+			       const struct pnfs_layoutdriver_type *ld,
+			       const struct nfs4_deviceid *id)
+{
+	struct nfs4_deviceid_delete *dd, *new;
+
+	new = kzalloc_obj(*new, GFP_KERNEL);
+	if (!new)
+		return;	/* lost notification; recovery waits for the next */
+	new->ld = pnfs_find_layoutdriver(ld->id);
+	if (!new->ld) {
+		kfree(new);
+		return;
+	}
+	memcpy(&new->id, id, sizeof(new->id));
+
+	spin_lock(&clp->cl_lock);
+	list_for_each_entry(dd, &clp->cl_deviceid_deletes, list) {
+		if (dd->ld == new->ld &&
+		    !memcmp(&dd->id, &new->id, sizeof(dd->id))) {
+			spin_unlock(&clp->cl_lock);
+			pnfs_put_layoutdriver(new->ld);
+			kfree(new);
+			return;
+		}
+	}
+	list_add_tail(&new->list, &clp->cl_deviceid_deletes);
+	spin_unlock(&clp->cl_lock);
+
+	set_bit(NFS4CLNT_DEVICEID_DELETE, &clp->cl_state);
+	nfs4_schedule_state_manager(clp);
+}
+
+struct nfs4_deviceid_delete *pnfs_deviceid_delete_dequeue(
+			       struct nfs_client *clp)
+{
+	struct nfs4_deviceid_delete *dd = NULL;
+
+	spin_lock(&clp->cl_lock);
+	if (!list_empty(&clp->cl_deviceid_deletes)) {
+		dd = list_first_entry(&clp->cl_deviceid_deletes,
+				      struct nfs4_deviceid_delete, list);
+		list_del(&dd->list);
+	}
+	spin_unlock(&clp->cl_lock);
+	return dd;
+}
+
+void pnfs_deviceid_delete_queue_free(struct nfs_client *clp)
+{
+	struct nfs4_deviceid_delete *dd;
+
+	while ((dd = pnfs_deviceid_delete_dequeue(clp)) != NULL) {
+		pnfs_put_layoutdriver(dd->ld);
+		kfree(dd);
+	}
+}
+
 /* Check if we have we have a valid layout but if there isn't an intersection
  * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away.
  */
@@ -3103,6 +3440,7 @@ pnfs_do_write(struct nfs_pageio_descriptor *desc,
 
 static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
 {
+	pnfs_put_ds_dev(hdr->ds_dev);
 	pnfs_put_lseg(hdr->lseg);
 	nfs_pgio_header_free(hdr);
 }
@@ -3248,6 +3586,7 @@ pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
 
 static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
 {
+	pnfs_put_ds_dev(hdr->ds_dev);
 	pnfs_put_lseg(hdr->lseg);
 	nfs_pgio_header_free(hdr);
 }
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 70d20f7796787d..2774d4adf4c317 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -57,7 +57,7 @@ struct nfs4_pnfs_ds_addr {
 };
 
 struct nfs4_pnfs_ds {
-	struct list_head	ds_node;  /* nfs4_pnfs_dev_hlist dev_dslist */
+	struct hlist_node	ds_node;  /* nfs_net nfs4_data_server_cache */
 	char			*ds_remotestr;	/* comma sep list of addrs */
 	struct list_head	ds_addrs;
 	const struct net	*ds_net;
@@ -171,6 +171,25 @@ struct pnfs_layoutdriver_type {
 	struct nfs4_deviceid_node * (*alloc_deviceid_node)
 			(struct nfs_server *server, struct pnfs_device *pdev,
 			gfp_t gfp_flags);
+	/*
+	 * Re-resolve @lo's references to the changed deviceid @id.  Called
+	 * under @lo's inode i_lock inside an RCU read-side critical section:
+	 * must not sleep, allocations are GFP_ATOMIC.  Rather than put the
+	 * references it gives up (the final put can sleep), the hook
+	 * allocates an nfs4_deviceid_put per reference and queues it on
+	 * @put_list for the caller to put and free.  On allocation failure
+	 * it must leave the reference in place.
+	 */
+	void (*reresolve_deviceid)(struct pnfs_layout_hdr *lo,
+				   const struct nfs4_deviceid *id,
+				   bool immediate,
+				   struct list_head *put_list);
+	/*
+	 * Does @lo hold any reference to deviceid @id?  Called under
+	 * @lo's inode i_lock; must not sleep.
+	 */
+	bool (*layout_references_deviceid)(struct pnfs_layout_hdr *lo,
+					   const struct nfs4_deviceid *id);
 
 	int (*prepare_layoutreturn) (struct nfs4_layoutreturn_args *);
 
@@ -178,7 +197,8 @@ struct pnfs_layoutdriver_type {
 	int (*prepare_layoutcommit) (struct nfs4_layoutcommit_args *args);
 	int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args);
 
-	void (*cancel_io)(struct pnfs_layout_segment *lseg);
+	void (*cancel_io)(struct pnfs_layout_segment *lseg,
+			  const struct nfs4_deviceid *devid);
 };
 
 struct pnfs_commit_ops {
@@ -301,7 +321,8 @@ int pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
 int pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
 				struct list_head *tmp_list,
 				const struct pnfs_layout_range *recall_range,
-				u32 seq, bool cancel_io);
+				u32 seq, bool cancel_io,
+				const struct nfs4_deviceid *devid);
 int pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
 		struct list_head *lseg_list);
 bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args,
@@ -351,9 +372,56 @@ int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *);
 int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *);
 struct nfs4_threshold *pnfs_mdsthreshold_alloc(void);
 void pnfs_error_mark_layout_for_return(struct inode *inode,
-				       struct pnfs_layout_segment *lseg);
+				       struct pnfs_layout_segment *lseg,
+				       const struct nfs4_deviceid *devid);
 void pnfs_layout_return_unused_byclid(struct nfs_client *clp,
 				      enum pnfs_iomode iomode);
+void pnfs_layout_reresolve_deviceid_byclid(struct nfs_client *clp,
+				const struct pnfs_layoutdriver_type *ld,
+				const struct nfs4_deviceid *devid,
+				bool immediate);
+bool pnfs_layout_deviceid_referenced_byclid(struct nfs_client *clp,
+				const struct pnfs_layoutdriver_type *ld,
+				const struct nfs4_deviceid *devid);
+
+/*
+ * One live layout referencing a deviceID, collected for the
+ * CB_NOTIFY_DEVICEID DELETE recovery: the hdr is pinned, the inode
+ * igrab'd with its superblock active, and the layout stateid and
+ * cred snapshotted for TEST_STATEID.
+ */
+struct nfs4_deviceid_ref {
+	struct list_head node;
+	struct pnfs_layout_hdr *lo;
+	struct inode *inode;
+	nfs4_stateid stateid;
+	const struct cred *cred;
+};
+
+int pnfs_layout_collect_deviceid_refs(struct nfs_client *clp,
+				const struct pnfs_layoutdriver_type *ld,
+				const struct nfs4_deviceid *devid,
+				struct list_head *result);
+void pnfs_layout_put_deviceid_refs(struct list_head *result);
+
+/*
+ * A CB_NOTIFY_DEVICEID DELETE naming a deviceID that live layouts
+ * still reference (RFC 8881 Section 18.40.4).  Queued on
+ * nfs_client.cl_deviceid_deletes under cl_lock for the state manager
+ * to resolve; holds a layoutdriver reference.
+ */
+struct nfs4_deviceid_delete {
+	struct list_head list;
+	const struct pnfs_layoutdriver_type *ld;
+	struct nfs4_deviceid id;
+};
+
+void pnfs_deviceid_delete_mark(struct nfs_client *clp,
+			       const struct pnfs_layoutdriver_type *ld,
+			       const struct nfs4_deviceid *id);
+struct nfs4_deviceid_delete *pnfs_deviceid_delete_dequeue(
+			       struct nfs_client *clp);
+void pnfs_deviceid_delete_queue_free(struct nfs_client *clp);
 int pnfs_layout_handle_reboot(struct nfs_client *clp);
 
 /* nfs4_deviceid_flags */
@@ -376,15 +444,32 @@ struct nfs4_deviceid_node {
 	atomic_t			ref;
 };
 
+/* One reference given up by reresolve_deviceid; nodes are shared, so a
+ * single pass can unpin the same node more than once.
+ */
+struct nfs4_deviceid_put {
+	struct list_head		node;
+	struct nfs4_deviceid_node	*dev;
+};
+
 struct nfs4_deviceid_node *
 nfs4_find_get_deviceid(struct nfs_server *server,
 		const struct nfs4_deviceid *id, const struct cred *cred,
 		gfp_t gfp_mask);
 void nfs4_delete_deviceid(const struct pnfs_layoutdriver_type *, const struct nfs_client *, const struct nfs4_deviceid *);
+void nfs4_deviceid_bump_change_epoch(struct nfs_client *clp);
 void nfs4_init_deviceid_node(struct nfs4_deviceid_node *, struct nfs_server *,
 			     const struct nfs4_deviceid *);
 bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *);
 void nfs4_mark_deviceid_available(struct nfs4_deviceid_node *node);
+
+/* Put the device node reference carried by an in-flight I/O, if any */
+static inline void pnfs_put_ds_dev(struct nfs4_deviceid_node *dev)
+{
+	if (dev)
+		nfs4_put_deviceid_node(dev);
+}
+
 void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node);
 bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node);
 void nfs4_deviceid_purge_client(const struct nfs_client *);
@@ -422,11 +507,13 @@ struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net,
 void nfs4_pnfs_v3_ds_connect_unload(void);
 int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
 			  struct nfs4_deviceid_node *devid, unsigned int timeo,
-			  unsigned int retrans, u32 version, u32 minor_version,
+			  unsigned int retrans, unsigned int nconnect,
+			  u32 version, u32 minor_version,
 			  bool tightly_coupled);
 struct nfs4_pnfs_ds_addr *nfs4_decode_mp_ds_addr(struct net *net,
 						 struct xdr_stream *xdr,
 						 gfp_t gfp_flags);
+void nfs4_pnfs_ds_addr_list_free(struct list_head *dsaddrs);
 void pnfs_layout_mark_request_commit(struct nfs_page *req,
 				     struct pnfs_layout_segment *lseg,
 				     struct nfs_commit_info *cinfo,
@@ -690,10 +777,11 @@ pnfs_lseg_request_intersecting(struct pnfs_layout_segment *lseg, struct nfs_page
 }
 
 static inline void pnfs_lseg_cancel_io(struct nfs_server *server,
-				       struct pnfs_layout_segment *lseg)
+				       struct pnfs_layout_segment *lseg,
+				       const struct nfs4_deviceid *devid)
 {
 	if (server->pnfs_curr_ld->cancel_io)
-		server->pnfs_curr_ld->cancel_io(lseg);
+		server->pnfs_curr_ld->cancel_io(lseg, devid);
 }
 
 extern unsigned int layoutstats_timer;
diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c
index 274abdd6d5f3cf..e8143977965cae 100644
--- a/fs/nfs/pnfs_dev.c
+++ b/fs/nfs/pnfs_dev.c
@@ -40,8 +40,11 @@
 
 /*
  * Device ID RCU cache. A device ID is unique per server and layout type.
+ *
+ * 256 buckets keeps the chains short at the 1024-or-more devices a
+ * striping deployment expects, for 2KB of BSS on 64-bit.
  */
-#define NFS4_DEVICE_ID_HASH_BITS	5
+#define NFS4_DEVICE_ID_HASH_BITS	8
 #define NFS4_DEVICE_ID_HASH_SIZE	(1 << NFS4_DEVICE_ID_HASH_BITS)
 #define NFS4_DEVICE_ID_HASH_MASK	(NFS4_DEVICE_ID_HASH_SIZE - 1)
 
@@ -181,6 +184,21 @@ __nfs4_find_get_deviceid(struct nfs_server *server,
 	return d;
 }
 
+/*
+ * Bumped before the stale entry is unhashed, so an insert serialised
+ * after the unhash by nfs4_deviceid_lock observes the new epoch.
+ */
+void
+nfs4_deviceid_bump_change_epoch(struct nfs_client *clp)
+{
+	atomic_inc(&clp->cl_deviceid_change_epoch);
+}
+
+/* Discarding a raced reply is an optimisation, not a correctness
+ * requirement, and the epoch moves at the server's rate: bound it.
+ */
+#define NFS4_DEVICEID_FETCH_RETRIES	3
+
 struct nfs4_deviceid_node *
 nfs4_find_get_deviceid(struct nfs_server *server,
 		const struct nfs4_deviceid *id, const struct cred *cred,
@@ -188,11 +206,14 @@ nfs4_find_get_deviceid(struct nfs_server *server,
 {
 	long hash = nfs4_deviceid_hash(id);
 	struct nfs4_deviceid_node *d, *new;
+	int epoch, tries = 0;
 
+retry:
 	d = __nfs4_find_get_deviceid(server, id, hash);
 	if (d)
 		goto found;
 
+	epoch = atomic_read(&server->nfs_client->cl_deviceid_change_epoch);
 	new = nfs4_get_device_info(server, id, cred, gfp_mask);
 	if (!new) {
 		trace_nfs4_find_deviceid(server, id, -ENOENT);
@@ -200,6 +221,13 @@ nfs4_find_get_deviceid(struct nfs_server *server,
 	}
 
 	spin_lock(&nfs4_deviceid_lock);
+	if (atomic_read(&server->nfs_client->cl_deviceid_change_epoch) != epoch &&
+	    ++tries <= NFS4_DEVICEID_FETCH_RETRIES) {
+		/* a mapping changed while we fetched; ours may be stale */
+		spin_unlock(&nfs4_deviceid_lock);
+		server->pnfs_curr_ld->free_deviceid_node(new);
+		goto retry;
+	}
 	d = __nfs4_find_get_deviceid(server, id, hash);
 	if (d) {
 		spin_unlock(&nfs4_deviceid_lock);
diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c
index 93d63f75a355cd..7adb6f941cf26e 100644
--- a/fs/nfs/pnfs_nfs.c
+++ b/fs/nfs/pnfs_nfs.c
@@ -15,6 +15,8 @@
 
 #include "nfs4session.h"
 #include "internal.h"
+#include <linux/hash.h>
+#include <linux/jhash.h>
 #include "pnfs.h"
 #include "netns.h"
 #include "nfs4trace.h"
@@ -55,6 +57,7 @@ void pnfs_generic_commit_release(void *calldata)
 	struct nfs_commit_data *data = calldata;
 
 	data->completion_ops->completion(data);
+	pnfs_put_ds_dev(data->ds_dev);
 	pnfs_put_lseg(data->lseg);
 	nfs_put_client(data->ds_clp);
 	nfs_commitdata_release(data);
@@ -576,8 +579,8 @@ same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2)
 }
 
 /*
- * Checks if 'dsaddrs1' contains a subset of 'dsaddrs2'. If it does,
- * declare a match.
+ * Checks if 'dsaddrs1' and 'dsaddrs2' hold the same set of addresses.
+ * If they do, declare a match.
  */
 static bool
 _same_data_server_addrs_locked(const struct list_head *dsaddrs1,
@@ -587,6 +590,10 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1,
 	struct sockaddr *sa1, *sa2;
 	bool match = false;
 
+	if (list_count_nodes((struct list_head *)dsaddrs1) !=
+	    list_count_nodes((struct list_head *)dsaddrs2))
+		return false;
+
 	list_for_each_entry(da1, dsaddrs1, da_node) {
 		sa1 = (struct sockaddr *)&da1->da_addr;
 		match = false;
@@ -602,16 +609,58 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1,
 	return match;
 }
 
+/* Hash family, address bytes, and port - as same_sockaddr() */
+static u32
+nfs4_ds_addr_hash(const struct sockaddr *sa)
+{
+	u32 h = sa->sa_family;
+
+	switch (sa->sa_family) {
+	case AF_INET: {
+		const struct sockaddr_in *a = (const struct sockaddr_in *)sa;
+
+		h = jhash(&a->sin_addr.s_addr, sizeof(a->sin_addr.s_addr), h);
+		h = jhash(&a->sin_port, sizeof(a->sin_port), h);
+		break;
+	}
+	case AF_INET6: {
+		const struct sockaddr_in6 *a = (const struct sockaddr_in6 *)sa;
+
+		h = jhash(&a->sin6_addr, sizeof(a->sin6_addr), h);
+		h = jhash(&a->sin6_port, sizeof(a->sin6_port), h);
+		break;
+	}
+	}
+	return h;
+}
+
 /*
- * Lookup DS by addresses and NFS version.  nfs4_ds_cache_lock is held
+ * Bucket index for a DS cache key.  Per-address hashes combine by
+ * addition so the multipath list order cannot change the bucket,
+ * matching the order-independent set comparison above.
+ */
+static u32
+nfs4_ds_cache_hash(const struct list_head *dsaddrs, u32 version)
+{
+	const struct nfs4_pnfs_ds_addr *da;
+	u32 h = 0;
+
+	list_for_each_entry(da, dsaddrs, da_node)
+		h += nfs4_ds_addr_hash((const struct sockaddr *)&da->da_addr);
+	return hash_32(jhash_1word(version, h), NFS4_DS_CACHE_HASH_BITS);
+}
+
+/*
+ * Lookup DS by addresses and NFS version.  nfs4_data_server_lock is held
  */
 static struct nfs4_pnfs_ds *
 _data_server_lookup_locked(const struct nfs_net *nn,
 			   const struct list_head *dsaddrs, u32 version)
 {
 	struct nfs4_pnfs_ds *ds;
+	u32 bucket = nfs4_ds_cache_hash(dsaddrs, version);
 
-	list_for_each_entry(ds, &nn->nfs4_data_server_cache, ds_node)
+	hlist_for_each_entry(ds, &nn->nfs4_data_server_cache[bucket], ds_node)
 		if (ds->ds_version == version &&
 		    _same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs))
 			return ds;
@@ -633,23 +682,28 @@ static void nfs4_pnfs_ds_addr_free(struct nfs4_pnfs_ds_addr *da)
 	kfree(da);
 }
 
-static void destroy_ds(struct nfs4_pnfs_ds *ds)
+void nfs4_pnfs_ds_addr_list_free(struct list_head *dsaddrs)
 {
 	struct nfs4_pnfs_ds_addr *da;
 
+	while (!list_empty(dsaddrs)) {
+		da = list_first_entry(dsaddrs, struct nfs4_pnfs_ds_addr,
+				      da_node);
+		list_del_init(&da->da_node);
+		nfs4_pnfs_ds_addr_free(da);
+	}
+}
+EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_addr_list_free);
+
+static void destroy_ds(struct nfs4_pnfs_ds *ds)
+{
 	dprintk("--> %s\n", __func__);
 	ifdebug(FACILITY)
 		print_ds(ds);
 
 	nfs_put_client(ds->ds_clp);
 
-	while (!list_empty(&ds->ds_addrs)) {
-		da = list_first_entry(&ds->ds_addrs,
-				      struct nfs4_pnfs_ds_addr,
-				      da_node);
-		list_del_init(&da->da_node);
-		nfs4_pnfs_ds_addr_free(da);
-	}
+	nfs4_pnfs_ds_addr_list_free(&ds->ds_addrs);
 
 	kfree(ds->ds_remotestr);
 	kfree(ds);
@@ -660,7 +714,7 @@ void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds)
 	struct nfs_net *nn = net_generic(ds->ds_net, nfs_net_id);
 
 	if (refcount_dec_and_lock(&ds->ds_count, &nn->nfs4_data_server_lock)) {
-		list_del_init(&ds->ds_node);
+		hlist_del_init(&ds->ds_node);
 		spin_unlock(&nn->nfs4_data_server_lock);
 		destroy_ds(ds);
 	}
@@ -726,6 +780,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version,
 {
 	struct nfs_net *nn = net_generic(net, nfs_net_id);
 	struct nfs4_pnfs_ds *tmp_ds, *ds = NULL;
+	struct hlist_head *bucket;
 	char *remotestr;
 
 	if (list_empty(dsaddrs)) {
@@ -739,6 +794,8 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version,
 
 	/* this is only used for debugging, so it's ok if its NULL */
 	remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags);
+	/* @dsaddrs is empty after the splice below. */
+	bucket = &nn->nfs4_data_server_cache[nfs4_ds_cache_hash(dsaddrs, version)];
 
 	spin_lock(&nn->nfs4_data_server_lock);
 	tmp_ds = _data_server_lookup_locked(nn, dsaddrs, version);
@@ -747,11 +804,11 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version,
 		list_splice_init(dsaddrs, &ds->ds_addrs);
 		ds->ds_remotestr = remotestr;
 		refcount_set(&ds->ds_count, 1);
-		INIT_LIST_HEAD(&ds->ds_node);
+		INIT_HLIST_NODE(&ds->ds_node);
 		ds->ds_net = net;
 		ds->ds_clp = NULL;
 		ds->ds_version = version;
-		list_add(&ds->ds_node, &nn->nfs4_data_server_cache);
+		hlist_add_head(&ds->ds_node, bucket);
 		dprintk("%s add new data server %s\n", __func__,
 			ds->ds_remotestr);
 	} else {
@@ -787,7 +844,8 @@ static struct nfs_client *(*get_v3_ds_connect)(
 			int ds_addrlen,
 			int ds_proto,
 			unsigned int ds_timeo,
-			unsigned int ds_retrans);
+			unsigned int ds_retrans,
+			unsigned int ds_nconnect);
 
 static bool load_v3_ds_connect(void)
 {
@@ -810,7 +868,8 @@ void nfs4_pnfs_v3_ds_connect_unload(void)
 static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
 				 struct nfs4_pnfs_ds *ds,
 				 unsigned int timeo,
-				 unsigned int retrans)
+				 unsigned int retrans,
+				 unsigned int nconnect)
 {
 	struct nfs_client *clp = ERR_PTR(-EIO);
 	struct nfs_client *mds_clp = mds_srv->nfs_client;
@@ -862,7 +921,7 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
 			ds_proto = XPRT_TRANSPORT_TCP_TLS;
 
 		clp = get_v3_ds_connect(mds_srv, &da->da_addr, da->da_addrlen,
-					ds_proto, timeo, retrans);
+					ds_proto, timeo, retrans, nconnect);
 		if (IS_ERR(clp))
 			continue;
 		clp->cl_rpcclient->cl_softerr = 0;
@@ -885,6 +944,7 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 				 struct nfs4_pnfs_ds *ds,
 				 unsigned int timeo,
 				 unsigned int retrans,
+				 unsigned int nconnect,
 				 u32 minor_version,
 				 bool tightly_coupled)
 {
@@ -976,7 +1036,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
 
 			clp = nfs4_set_ds_client(mds_srv, &da->da_addr,
 						 da->da_addrlen, ds_proto,
-						 timeo, retrans, minor_version,
+						 timeo, retrans, nconnect,
+						 minor_version,
 						 tightly_coupled);
 			if (IS_ERR(clp))
 				continue;
@@ -1011,7 +1072,8 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
  */
 int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
 			  struct nfs4_deviceid_node *devid, unsigned int timeo,
-			  unsigned int retrans, u32 version, u32 minor_version,
+			  unsigned int retrans, unsigned int nconnect,
+			  u32 version, u32 minor_version,
 			  bool tightly_coupled)
 {
 	int err;
@@ -1031,11 +1093,13 @@ int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
 
 	switch (version) {
 	case 3:
-		err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans);
+		err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans,
+					       nconnect);
 		break;
 	case 4:
 		err = _nfs4_pnfs_v4_ds_connect(mds_srv, ds, timeo, retrans,
-					       minor_version, tightly_coupled);
+					       nconnect, minor_version,
+					       tightly_coupled);
 		break;
 	default:
 		dprintk("%s: unsupported DS version %d\n", __func__, version);
diff --git a/fs/nfs/read.c b/fs/nfs/read.c
index 2b70bd2b934b9a..e7497b029d6cee 100644
--- a/fs/nfs/read.c
+++ b/fs/nfs/read.c
@@ -324,7 +324,7 @@ int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio,
 
 	aligned_len = min_t(unsigned int, ALIGN(len, rsize), fsize);
 
-	new = nfs_page_create_from_folio(ctx, folio, 0, aligned_len);
+	new = nfs_page_create_from_folio(ctx, folio, false, 0, aligned_len);
 	if (IS_ERR(new)) {
 		error = PTR_ERR(new);
 		if (nfs_netfs_folio_unlock(folio))
diff --git a/fs/nfs/super.c b/fs/nfs/super.c
index cb19f1540d9841..23292680adbd5d 100644
--- a/fs/nfs/super.c
+++ b/fs/nfs/super.c
@@ -58,7 +58,6 @@
 #include <linux/rcupdate.h>
 
 #include <linux/uaccess.h>
-#include <linux/nfs_ssc.h>
 
 #include <uapi/linux/tls.h>
 
@@ -92,12 +91,6 @@ const struct super_operations nfs_sops = {
 };
 EXPORT_SYMBOL_GPL(nfs_sops);
 
-#ifdef CONFIG_NFS_V4_2
-static const struct nfs_ssc_client_ops nfs_ssc_clnt_ops_tbl = {
-	.sco_sb_deactive = nfs_sb_deactive,
-};
-#endif
-
 #if IS_ENABLED(CONFIG_NFS_V4)
 static int __init register_nfs4_fs(void)
 {
@@ -119,18 +112,6 @@ static void unregister_nfs4_fs(void)
 }
 #endif
 
-#ifdef CONFIG_NFS_V4_2
-static void nfs_ssc_register_ops(void)
-{
-	nfs_ssc_register(&nfs_ssc_clnt_ops_tbl);
-}
-
-static void nfs_ssc_unregister_ops(void)
-{
-	nfs_ssc_unregister(&nfs_ssc_clnt_ops_tbl);
-}
-#endif /* CONFIG_NFS_V4_2 */
-
 static struct shrinker *acl_shrinker;
 
 /*
@@ -163,9 +144,6 @@ int __init register_nfs_fs(void)
 
 	shrinker_register(acl_shrinker);
 
-#ifdef CONFIG_NFS_V4_2
-	nfs_ssc_register_ops();
-#endif
 	return 0;
 error_3:
 	nfs_unregister_sysctl();
@@ -185,9 +163,6 @@ void __exit unregister_nfs_fs(void)
 	shrinker_free(acl_shrinker);
 	nfs_unregister_sysctl();
 	unregister_nfs4_fs();
-#ifdef CONFIG_NFS_V4_2
-	nfs_ssc_unregister_ops();
-#endif
 	unregister_filesystem(&nfs_fs_type);
 }
 
diff --git a/fs/nfs/unlink.c b/fs/nfs/unlink.c
index b57cfaa4d516f0..c8d712204e6493 100644
--- a/fs/nfs/unlink.c
+++ b/fs/nfs/unlink.c
@@ -67,6 +67,7 @@ static void nfs_async_unlink_release(void *calldata)
 	struct super_block *sb = dentry->d_sb;
 
 	up_read_non_owner(&NFS_I(d_inode(dentry->d_parent))->rmdir_sem);
+	d_lookup_acquire(dentry);
 	d_lookup_done(dentry);
 	nfs_free_unlinkdata(data);
 	dput(dentry);
@@ -159,6 +160,8 @@ static int nfs_call_unlink(struct dentry *dentry, struct inode *inode, struct nf
 		return ret;
 	}
 	data->dentry = alias;
+	d_lookup_release(alias);
+
 	nfs_do_call_unlink(inode, data);
 	return 1;
 }
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index 623e7ef1f73d57..0b42d31f7a7533 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -717,7 +717,6 @@ static void nfs_inode_add_request(struct nfs_page *req)
 	nfs_lock_request(req);
 	spin_lock(&mapping->i_private_lock);
 	set_bit(PG_MAPPED, &req->wb_flags);
-	folio_set_private(folio);
 	folio->private = req;
 	spin_unlock(&mapping->i_private_lock);
 	atomic_long_inc(&nfsi->nrequests);
@@ -745,7 +744,6 @@ static void nfs_inode_remove_request(struct nfs_page *req)
 
 			spin_lock(&mapping->i_private_lock);
 			folio->private = NULL;
-			folio_clear_private(folio);
 			clear_bit(PG_MAPPED, &req->wb_head->wb_flags);
 			spin_unlock(&mapping->i_private_lock);
 
@@ -1089,7 +1087,7 @@ static struct nfs_page *nfs_setup_write_request(struct nfs_open_context *ctx,
 	req = nfs_try_to_update_request(folio, offset, bytes);
 	if (req != NULL)
 		goto out;
-	req = nfs_page_create_from_folio(ctx, folio, offset, bytes);
+	req = nfs_page_create_from_folio(ctx, folio, false, offset, bytes);
 	if (IS_ERR(req))
 		goto out;
 	nfs_inode_add_request(req);
diff --git a/fs/nfs_common/nfs_ssc.c b/fs/nfs_common/nfs_ssc.c
index 832246b22c5175..e521e3c836fe7b 100644
--- a/fs/nfs_common/nfs_ssc.c
+++ b/fs/nfs_common/nfs_ssc.c
@@ -10,82 +10,112 @@
 #include <linux/module.h>
 #include <linux/fs.h>
 #include <linux/nfs_ssc.h>
-#include "../nfs/nfs4_fs.h"
+#include <linux/nfsd_ssc.h>
 
+struct nfs_ssc_client_ops_tbl {
+	const struct nfs4_ssc_client_ops __rcu *ssc_nfs4_ops;
+};
 
-struct nfs_ssc_client_ops_tbl nfs_ssc_client_tbl;
-EXPORT_SYMBOL_GPL(nfs_ssc_client_tbl);
+static struct nfs_ssc_client_ops_tbl nfs_ssc_client_tbl __read_mostly;
 
-#ifdef CONFIG_NFS_V4_2
 /**
- * nfs42_ssc_register - install the NFS_V4 client ops in the nfs_ssc_client_tbl
- * @ops: NFS_V4 ops to be installed
+ * nfsd42_ssc_open - Open a file to be used for server-to-server copy
+ * @ss_mnt: active mount point on which the source file resides
+ * @src_fh: file handle of the source file to be copied
+ * @stateid: stateid to use for COPY operation
  *
- * Return values:
- *   None
+ * Caller must close the returned file using nfsd42_ssc_close().
+ *
+ * Return: an open file, or an ERR_PTR on error
  */
-void nfs42_ssc_register(const struct nfs4_ssc_client_ops *ops)
+struct file *nfsd42_ssc_open(struct vfsmount *ss_mnt, struct nfs_fh *src_fh,
+			     nfs4_stateid *stateid)
 {
-	nfs_ssc_client_tbl.ssc_nfs4_ops = ops;
+	/*
+	 * Built under CONFIG_NFS_V4_2_SSC_HELPER, which the NFS client
+	 * enables on its own. The dispatch below is live only when the
+	 * server also sets CONFIG_NFSD_V4_2_INTER_SSC; without it the
+	 * source file cannot be opened, so callers get -EIO.
+	 */
+#if IS_ENABLED(CONFIG_NFSD_V4_2_INTER_SSC)
+	const struct nfs4_ssc_client_ops *ops;
+	struct file *res;
+
+	/*
+	 * sco_open() sleeps and must not run inside an RCU read-side
+	 * section. Pin the provider module so the open runs with the
+	 * module held; try_module_get() fails once unregister begins,
+	 * and the copy then gets -EIO.
+	 */
+	rcu_read_lock();
+	ops = rcu_dereference(nfs_ssc_client_tbl.ssc_nfs4_ops);
+	if (ops && try_module_get(ops->owner)) {
+		rcu_read_unlock();
+		res = ops->sco_open(ss_mnt, src_fh, stateid);
+		module_put(ops->owner);
+		return res;
+	}
+	rcu_read_unlock();
+#endif
+
+	return ERR_PTR(-EIO);
 }
-EXPORT_SYMBOL_GPL(nfs42_ssc_register);
+EXPORT_SYMBOL_GPL(nfsd42_ssc_open);
 
 /**
- * nfs42_ssc_unregister - uninstall the NFS_V4 client ops from
- *				the nfs_ssc_client_tbl
- * @ops: ops to be uninstalled
+ * nfsd42_ssc_close - Close a file opened with nfsd42_ssc_open()
+ * @filp: struct file to be closed
  *
- * Return values:
- *   None
+ * The real cleanup happens unconditionally in nfsd4_cleanup_inter_ssc().
+ * The client ops table is read under RCU; nfs42_ssc_unregister() calls
+ * synchronize_rcu() so unregistration cannot complete while a close is
+ * in flight.
  */
-void nfs42_ssc_unregister(const struct nfs4_ssc_client_ops *ops)
+void nfsd42_ssc_close(struct file *filp)
 {
-	if (nfs_ssc_client_tbl.ssc_nfs4_ops != ops)
-		return;
+	/* Live only under CONFIG_NFSD_V4_2_INTER_SSC; see nfsd42_ssc_open(). */
+#if IS_ENABLED(CONFIG_NFSD_V4_2_INTER_SSC)
+	const struct nfs4_ssc_client_ops *ops;
 
-	nfs_ssc_client_tbl.ssc_nfs4_ops = NULL;
+	rcu_read_lock();
+	ops = rcu_dereference(nfs_ssc_client_tbl.ssc_nfs4_ops);
+	if (ops)
+		ops->sco_close(filp);
+	rcu_read_unlock();
+#endif
 }
-EXPORT_SYMBOL_GPL(nfs42_ssc_unregister);
-#endif /* CONFIG_NFS_V4_2 */
+EXPORT_SYMBOL_GPL(nfsd42_ssc_close);
 
 #ifdef CONFIG_NFS_V4_2
 /**
- * nfs_ssc_register - install the NFS_FS client ops in the nfs_ssc_client_tbl
- * @ops: NFS_FS ops to be installed
+ * nfs42_ssc_register - install the NFS_V4 client ops in the nfs_ssc_client_tbl
+ * @ops: NFS_V4 ops to be installed
  *
  * Return values:
  *   None
  */
-void nfs_ssc_register(const struct nfs_ssc_client_ops *ops)
+void nfs42_ssc_register(const struct nfs4_ssc_client_ops *ops)
 {
-	nfs_ssc_client_tbl.ssc_nfs_ops = ops;
+	rcu_assign_pointer(nfs_ssc_client_tbl.ssc_nfs4_ops, ops);
 }
-EXPORT_SYMBOL_GPL(nfs_ssc_register);
+EXPORT_SYMBOL_GPL(nfs42_ssc_register);
 
 /**
- * nfs_ssc_unregister - uninstall the NFS_FS client ops from
+ * nfs42_ssc_unregister - uninstall the NFS_V4 client ops from
  *				the nfs_ssc_client_tbl
  * @ops: ops to be uninstalled
  *
  * Return values:
  *   None
  */
-void nfs_ssc_unregister(const struct nfs_ssc_client_ops *ops)
+void nfs42_ssc_unregister(const struct nfs4_ssc_client_ops *ops)
 {
-	if (nfs_ssc_client_tbl.ssc_nfs_ops != ops)
+	if (rcu_dereference_protected(nfs_ssc_client_tbl.ssc_nfs4_ops,
+				      true) != ops)
 		return;
-	nfs_ssc_client_tbl.ssc_nfs_ops = NULL;
-}
-EXPORT_SYMBOL_GPL(nfs_ssc_unregister);
 
-#else
-void nfs_ssc_register(const struct nfs_ssc_client_ops *ops)
-{
+	rcu_assign_pointer(nfs_ssc_client_tbl.ssc_nfs4_ops, NULL);
+	synchronize_rcu();
 }
-EXPORT_SYMBOL_GPL(nfs_ssc_register);
-
-void nfs_ssc_unregister(const struct nfs_ssc_client_ops *ops)
-{
-}
-EXPORT_SYMBOL_GPL(nfs_ssc_unregister);
+EXPORT_SYMBOL_GPL(nfs42_ssc_unregister);
 #endif /* CONFIG_NFS_V4_2 */
diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
index 5be7721c22c235..df02cf7464799c 100644
--- a/fs/nfsd/blocklayout.c
+++ b/fs/nfsd/blocklayout.c
@@ -9,6 +9,7 @@
 
 #include <linux/nfsd/debug.h>
 
+#include "nfserr.h"
 #include "blocklayoutxdr.h"
 #include "pnfs.h"
 #include "filecache.h"
diff --git a/fs/nfsd/blocklayoutxdr.c b/fs/nfsd/blocklayoutxdr.c
index f80dbc41fd5f01..a6589f5c878aca 100644
--- a/fs/nfsd/blocklayoutxdr.c
+++ b/fs/nfsd/blocklayoutxdr.c
@@ -8,11 +8,22 @@
 #include <linux/nfs4.h>
 
 #include "nfsd.h"
+#include "nfserr.h"
 #include "blocklayoutxdr.h"
 #include "vfs.h"
 
 #define NFSDDBG_FACILITY	NFSDDBG_PNFS
 
+static __be32
+nfsd4_decode_deviceid4(struct xdr_stream *xdr, struct nfsd4_deviceid *devid)
+{
+	__be32 *p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE);
+
+	if (unlikely(!p))
+		return nfserr_bad_xdr;
+	svcxdr_decode_deviceid4(p, devid);
+	return nfs_ok;
+}
 
 /**
  * nfsd4_block_encode_layoutget - encode block/scsi layout extent array
diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
index b6e0c543e02891..e5a0f1ababe627 100644
--- a/fs/nfsd/export.c
+++ b/fs/nfsd/export.c
@@ -21,6 +21,8 @@
 #include <uapi/linux/nfsd_netlink.h>
 
 #include "nfsd.h"
+#include "nfserr.h"
+#include "nfs4ctl.h"
 #include "nfsfh.h"
 #include "netns.h"
 #include "pnfs.h"
@@ -1890,21 +1892,19 @@ __be32 check_security_flavor(struct svc_export *exp, struct svc_rqst *rqstp,
  * check_nfsd_access - check if access to export is allowed.
  * @exp: svc_export that is being accessed.
  * @rqstp: svc_rqst attempting to access @exp.
- * @may_bypass_gss: reduce strictness of authorization check
  *
  * Return values:
  *   %nfs_ok if access is granted, or
  *   %nfserr_wrongsec if access is denied
  */
-__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp,
-			 bool may_bypass_gss)
+__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp)
 {
 	__be32 status;
 
 	status = check_xprtsec_policy(exp, rqstp);
 	if (status != nfs_ok)
 		return status;
-	return check_security_flavor(exp, rqstp, may_bypass_gss);
+	return check_security_flavor(exp, rqstp, false);
 }
 
 /*
diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h
index d2b09cd761453d..117fb28db1e024 100644
--- a/fs/nfsd/export.h
+++ b/fs/nfsd/export.h
@@ -104,8 +104,7 @@ int nfsexp_flags(struct svc_cred *cred, struct svc_export *exp);
 __be32 check_xprtsec_policy(struct svc_export *exp, struct svc_rqst *rqstp);
 __be32 check_security_flavor(struct svc_export *exp, struct svc_rqst *rqstp,
 			     bool may_bypass_gss);
-__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp,
-			 bool may_bypass_gss);
+__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp);
 
 /*
  * Function declarations
diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
index b9548eb17c77de..3539149cc75f77 100644
--- a/fs/nfsd/filecache.c
+++ b/fs/nfsd/filecache.c
@@ -43,6 +43,7 @@
 
 #include "vfs.h"
 #include "nfsd.h"
+#include "nfserr.h"
 #include "nfsfh.h"
 #include "netns.h"
 #include "filecache.h"
diff --git a/fs/nfsd/flexfilelayout.c b/fs/nfsd/flexfilelayout.c
index 6d531285ab439e..0deb913493a398 100644
--- a/fs/nfsd/flexfilelayout.c
+++ b/fs/nfsd/flexfilelayout.c
@@ -13,7 +13,9 @@
 
 #include <linux/sunrpc/addr.h>
 
+#include "nfserr.h"
 #include "flexfilelayoutxdr.h"
+#include "auth.h"
 #include "pnfs.h"
 #include "vfs.h"
 
@@ -23,10 +25,10 @@ static __be32
 nfsd4_ff_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode,
 		const struct svc_fh *fhp, struct nfsd4_layoutget *args)
 {
+	struct user_namespace *userns = nfsd_user_namespace(rqstp);
 	struct nfsd4_layout_seg *seg = &args->lg_seg;
 	u32 device_generation = 0;
 	int error;
-	uid_t u;
 
 	struct pnfs_ff_layout *fl;
 
@@ -49,20 +51,22 @@ nfsd4_ff_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode,
 	fl->flags = FF_FLAGS_NO_LAYOUTCOMMIT | FF_FLAGS_NO_IO_THRU_MDS |
 		    FF_FLAGS_NO_READ_IO;
 
-	/* Do not allow a IOMODE_READ segment to have write pemissions */
-	if (seg->iomode == IOMODE_READ) {
-		u = from_kuid(&init_user_ns, inode->i_uid) + 1;
-		fl->uid = make_kuid(&init_user_ns, u);
-	} else
-		fl->uid = inode->i_uid;
-	fl->gid = inode->i_gid;
+	fl->uid = from_kuid_munged(userns, inode->i_uid);
+	fl->gid = from_kgid_munged(userns, inode->i_gid);
+
+	/*
+	 * Do not allow an IOMODE_READ segment to have write permissions.
+	 * The group is left intact so group-readable files stay readable;
+	 * nfsd_setuser() squashes an unmapped uid to the export's anon ID.
+	 */
+	if (seg->iomode == IOMODE_READ)
+		fl->uid++;
 
 	error = nfsd4_set_deviceid(&fl->deviceid, fhp, device_generation);
 	if (error)
 		goto out_error;
 
-	fl->fh.size = fhp->fh_handle.fh_size;
-	memcpy(fl->fh.data, &fhp->fh_handle.fh_raw, fl->fh.size);
+	fh_copy_shallow(&fl->fh, &fhp->fh_handle);
 
 	/* Give whole file layout segments */
 	seg->offset = 0;
diff --git a/fs/nfsd/flexfilelayoutxdr.c b/fs/nfsd/flexfilelayoutxdr.c
index 374e52d3064a65..e297100a2ac3ce 100644
--- a/fs/nfsd/flexfilelayoutxdr.c
+++ b/fs/nfsd/flexfilelayoutxdr.c
@@ -6,6 +6,7 @@
 #include <linux/nfs4.h>
 
 #include "nfsd.h"
+#include "nfserr.h"
 #include "flexfilelayoutxdr.h"
 
 #define NFSDDBG_FACILITY	NFSDDBG_PNFS
@@ -30,10 +31,10 @@ nfsd4_ff_encode_layoutget(struct xdr_stream *xdr,
 	struct ff_idmap uid;
 	struct ff_idmap gid;
 
-	fh_len = 4 + xdr_align_size(fl->fh.size);
+	fh_len = 4 + xdr_align_size(fl->fh.fh_size);
 
-	uid.len = sprintf(uid.buf, "%u", from_kuid(&init_user_ns, fl->uid));
-	gid.len = sprintf(gid.buf, "%u", from_kgid(&init_user_ns, fl->gid));
+	uid.len = sprintf(uid.buf, "%u", fl->uid);
+	gid.len = sprintf(gid.buf, "%u", fl->gid);
 
 	/* data server entry: deviceid + efficiency + stateid + fh list +
 	 * user + group + flags + stats_collect_hint
@@ -68,7 +69,7 @@ nfsd4_ff_encode_layoutget(struct xdr_stream *xdr,
 				    sizeof(stateid_opaque_t));
 
 	*p++ = cpu_to_be32(1);			/* single file handle */
-	p = xdr_encode_opaque(p, fl->fh.data, fl->fh.size);
+	p = xdr_encode_opaque(p, fl->fh.fh_raw, fl->fh.fh_size);
 
 	p = xdr_encode_opaque(p, uid.buf, uid.len);
 	p = xdr_encode_opaque(p, gid.buf, gid.len);
diff --git a/fs/nfsd/flexfilelayoutxdr.h b/fs/nfsd/flexfilelayoutxdr.h
index 6d5a1066a903c1..f7d1dd0708ec46 100644
--- a/fs/nfsd/flexfilelayoutxdr.h
+++ b/fs/nfsd/flexfilelayoutxdr.h
@@ -6,6 +6,7 @@
 #define _NFSD_FLEXFILELAYOUTXDR_H 1
 
 #include <linux/inet.h>
+#include "nfsfh.h"
 #include "xdr4.h"
 
 #define FF_FLAGS_NO_LAYOUTCOMMIT 1
@@ -35,11 +36,12 @@ struct pnfs_ff_device_addr {
 struct pnfs_ff_layout {
 	u32				flags;
 	u32				stats_collect_hint;
-	kuid_t				uid;
-	kgid_t				gid;
+	/* Values to encode; nfsd4_ff_proc_layoutget() has mapped these */
+	u32				uid;
+	u32				gid;
 	struct nfsd4_deviceid		deviceid;
 	stateid_t			stateid;
-	struct nfs_fh			fh;
+	struct knfsd_fh			fh;
 };
 
 __be32 nfsd4_ff_encode_getdeviceinfo(struct xdr_stream *xdr,
diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c
index c458c01e94783a..33b56d1b3f440b 100644
--- a/fs/nfsd/localio.c
+++ b/fs/nfsd/localio.c
@@ -11,11 +11,9 @@
 #include <linux/exportfs.h>
 #include <linux/sunrpc/svcauth.h>
 #include <linux/sunrpc/clnt.h>
-#include <linux/nfs.h>
 #include <linux/nfs_common.h>
+#include <linux/nfs_fh.h>
 #include <linux/nfslocalio.h>
-#include <linux/nfs_fs.h>
-#include <linux/nfs_xdr.h>
 #include <linux/string.h>
 
 #include "nfsd.h"
@@ -55,7 +53,7 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom,
 	struct nfsd_file *localio;
 	__be32 beres;
 
-	if (nfs_fh->size > NFS4_FHSIZE)
+	if (nfs_fh->size > NFS_MAXFHSIZE)
 		return ERR_PTR(-EINVAL);
 
 	if (!nfsd_net_try_get(net))
@@ -68,7 +66,7 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom,
 		return localio;
 
 	/* nfs_fh -> svc_fh */
-	fh_init(&fh, NFS4_FHSIZE);
+	fh_init(&fh, NFSD_FHSIZE_UNSPEC);
 	fh.fh_handle.fh_size = nfs_fh->size;
 	memcpy(fh.fh_handle.fh_raw, nfs_fh->data, nfs_fh->size);
 
@@ -179,7 +177,7 @@ static bool localio_decode_uuidarg(struct svc_rqst *rqstp,
 	struct localio_uuidarg *argp = rqstp->rq_argp;
 	u8 uuid[UUID_SIZE];
 
-	if (decode_opaque_fixed(xdr, uuid, UUID_SIZE))
+	if (xdr_stream_decode_opaque_fixed(xdr, uuid, UUID_SIZE) < 0)
 		return false;
 	import_uuid(&argp->uuid, uuid);
 
diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c
index 72a5b499839d81..f24e45dc37a05b 100644
--- a/fs/nfsd/lockd.c
+++ b/fs/nfsd/lockd.c
@@ -9,7 +9,9 @@
 
 #include <linux/file.h>
 #include <linux/lockd/bind.h>
+#include <linux/nfs_fh.h>
 #include "nfsd.h"
+#include "nfserr.h"
 #include "vfs.h"
 
 #define NFSDDBG_FACILITY		NFSDDBG_LOCKD
@@ -33,8 +35,7 @@ static int nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f,
 	int		access;
 	struct svc_fh	fh;
 
-	/* must initialize before using! but maxsize doesn't matter */
-	fh_init(&fh,0);
+	fh_init(&fh, NFSD_FHSIZE_UNSPEC);
 	fh.fh_handle.fh_size = f->size;
 	memcpy(&fh.fh_handle.fh_raw, f->data, f->size);
 	fh.fh_export = NULL;
diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index 71eebfea020d50..0ce7da20aba3c1 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -238,8 +238,21 @@ struct nfsd_net {
 	int			nfs4_max_clients;
 
 	atomic_t		nfsd_courtesy_clients;
-	struct shrinker		*nfsd_client_shrinker;
-	struct work_struct	nfsd_shrinker_work;
+	/* per-namespace; num_delegations in nfs4state.c is host-wide */
+	atomic_long_t		nfsd_delegations;
+	struct shrinker		*nfsd_courtesy_shrinker;
+	struct shrinker		*nfsd_deleg_shrinker;
+	struct work_struct	nfsd_courtesy_work;
+	struct work_struct	nfsd_deleg_work;
+
+	/* courtesy scan requests the reaper has not retired yet */
+	atomic_long_t		nfsd_shrink_backlog;
+
+	/* delegation scan requests the reaper has not retired yet */
+	atomic_long_t		nfsd_deleg_backlog;
+
+	/* when deleg_reaper() last swept the client list */
+	time64_t		nfsd_last_recall_any;
 
 	/* last time an admin-revoke happened for NFSv4.0 */
 	time64_t		nfs40_last_revoke;
diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c
index 190f5a00190091..0a5c444fef99d8 100644
--- a/fs/nfsd/nfs2acl.c
+++ b/fs/nfsd/nfs2acl.c
@@ -6,15 +6,59 @@
  */
 
 #include "nfsd.h"
+#include "nfserr.h"
 /* FIXME: nfsacl.h is a broken header */
 #include <linux/nfsacl.h>
 #include <linux/gfp.h>
+#include <linux/nfs3.h>
 #include "cache.h"
 #include "xdr3.h"
 #include "vfs.h"
 
 #define NFSDDBG_FACILITY		NFSDDBG_PROC
 
+/*
+ * These maps are identical to the NFSv3 maps (nfs3proc.c). This enables
+ * the behavior of the two versions to diverge if needed.
+ */
+static const struct nfsd_access_map nfsd2_regaccess[] = {
+	{ NFS3_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS3_ACCESS_EXECUTE,	NFSD_MAY_EXEC				},
+	{ NFS3_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_TRUNC		},
+	{ NFS3_ACCESS_EXTEND,	NFSD_MAY_WRITE				},
+	{ 0,			0					}
+};
+
+static const struct nfsd_access_map nfsd2_diraccess[] = {
+	{ NFS3_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS3_ACCESS_LOOKUP,	NFSD_MAY_EXEC				},
+	{ NFS3_ACCESS_MODIFY,	NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC },
+	{ NFS3_ACCESS_EXTEND,	NFSD_MAY_EXEC|NFSD_MAY_WRITE		},
+	{ NFS3_ACCESS_DELETE,	NFSD_MAY_REMOVE				},
+	{ 0,			0					}
+};
+
+/*
+ * Some clients - Solaris 2.6 at least, make an access call to the NFS
+ * server to check for access for things like /dev/null (which really,
+ * NFSD doesn't care about).  So NFSD provides simple access checking
+ * for those objects, looking mainly at mode bits, ignoring read-only
+ * filesystem checks.
+ */
+static const struct nfsd_access_map nfsd2_otheraccess[] = {
+	{ NFS3_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS3_ACCESS_EXECUTE,	NFSD_MAY_EXEC				},
+	{ NFS3_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
+	{ NFS3_ACCESS_EXTEND,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
+	{ 0,			0					}
+};
+
+static const struct nfsd_access_maps nfsd2_access_maps = {
+	.regular	= nfsd2_regaccess,
+	.directory	= nfsd2_diraccess,
+	.other		= nfsd2_otheraccess,
+};
+
 /*
  * NULL call.
  */
@@ -180,7 +224,9 @@ static __be32 nfsacld_proc_access(struct svc_rqst *rqstp)
 
 	fh_copy(&resp->fh, &argp->fh);
 	resp->access = argp->access;
-	resp->status = nfsd_access(rqstp, &resp->fh, &resp->access, NULL);
+
+	resp->status = nfsd_access(rqstp, &resp->fh, &nfsd2_access_maps,
+				   &resp->access, NULL);
 	if (resp->status != nfs_ok)
 		goto out;
 	resp->status = fh_getattr(&resp->fh, &resp->stat);
diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c
index 6b6b289db63613..7183995182ab48 100644
--- a/fs/nfsd/nfs3acl.c
+++ b/fs/nfsd/nfs3acl.c
@@ -6,6 +6,7 @@
  */
 
 #include "nfsd.h"
+#include "nfserr.h"
 /* FIXME: nfsacl.h is a broken header */
 #include <linux/nfsacl.h>
 #include <linux/gfp.h>
diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
index 0904d953d10e07..60cd01b6a37d24 100644
--- a/fs/nfsd/nfs3proc.c
+++ b/fs/nfsd/nfs3proc.c
@@ -9,10 +9,12 @@
 #include <linux/ext2_fs.h>
 #include <linux/magic.h>
 #include <linux/namei.h>
+#include <linux/nfs3.h>
 
 #include "cache.h"
 #include "xdr3.h"
 #include "vfs.h"
+#include "nfserr.h"
 #include "filecache.h"
 #include "trace.h"
 
@@ -48,6 +50,58 @@ static bool nfsd3_time_in_range(const struct iattr *iap)
 	return true;
 }
 
+static const struct nfsd_access_map nfsd3_regaccess[] = {
+	{ NFS3_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS3_ACCESS_EXECUTE,	NFSD_MAY_EXEC				},
+	{ NFS3_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_TRUNC		},
+	{ NFS3_ACCESS_EXTEND,	NFSD_MAY_WRITE				},
+	{ 0,			0					}
+};
+
+static const struct nfsd_access_map nfsd3_diraccess[] = {
+	{ NFS3_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS3_ACCESS_LOOKUP,	NFSD_MAY_EXEC				},
+	{ NFS3_ACCESS_MODIFY,	NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC },
+	{ NFS3_ACCESS_EXTEND,	NFSD_MAY_EXEC|NFSD_MAY_WRITE		},
+	{ NFS3_ACCESS_DELETE,	NFSD_MAY_REMOVE				},
+	{ 0,			0					}
+};
+
+/*
+ * Some clients - Solaris 2.6 at least, make an access call to the NFS
+ * server to check for access for things like /dev/null (which really,
+ * NFSD doesn't care about).  So NFSD provides simple access checking
+ * for those objects, looking mainly at mode bits, ignoring read-only
+ * filesystem checks.
+ */
+static const struct nfsd_access_map nfsd3_otheraccess[] = {
+	{ NFS3_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS3_ACCESS_EXECUTE,	NFSD_MAY_EXEC				},
+	{ NFS3_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
+	{ NFS3_ACCESS_EXTEND,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
+	{ 0,			0					}
+};
+
+static const struct nfsd_access_maps nfsd3_access_maps = {
+	.regular	= nfsd3_regaccess,
+	.directory	= nfsd3_diraccess,
+	.other		= nfsd3_otheraccess,
+};
+
+static int nfsd3_iocb_flags(enum nfs3_stable_how how)
+{
+	switch (how) {
+	case NFS_FILE_SYNC:
+		/* persist data and timestamps */
+		return IOCB_DSYNC | IOCB_SYNC;
+	case NFS_DATA_SYNC:
+		/* persist data only */
+		return IOCB_DSYNC;
+	default:
+		return 0;
+	}
+}
+
 static __be32 nfsd3_map_status(__be32 status)
 {
 	switch (status) {
@@ -171,7 +225,8 @@ nfsd3_proc_access(struct svc_rqst *rqstp)
 
 	fh_copy(&resp->fh, &argp->fh);
 	resp->access = argp->access;
-	resp->status = nfsd_access(rqstp, &resp->fh, &resp->access, NULL);
+	resp->status = nfsd_access(rqstp, &resp->fh, &nfsd3_access_maps,
+				   &resp->access, NULL);
 	resp->status = nfsd3_map_status(resp->status);
 	return rpc_success;
 }
@@ -260,7 +315,8 @@ nfsd3_proc_write(struct svc_rqst *rqstp)
 	resp->committed = argp->stable;
 	resp->status = nfsd_write(rqstp, &resp->fh, argp->offset,
 				  &argp->payload, &cnt,
-				  resp->committed, resp->verf);
+				  nfsd3_iocb_flags(resp->committed),
+				  resp->verf);
 	resp->count = cnt;
 	resp->status = nfsd3_map_status(resp->status);
 	return rpc_success;
@@ -282,6 +338,7 @@ nfsd3_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	struct nfsd_attrs attrs = {
 		.na_iattr	= iap,
 	};
+	struct svc_export *exp;
 	__u32 v_mtime, v_atime;
 	struct inode *inode;
 	__be32 status;
@@ -320,7 +377,23 @@ nfsd3_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 			goto out;
 	}
 
-	status = fh_compose(resfhp, fhp->fh_export, child, fhp);
+	exp = exp_get(fhp->fh_export);
+	if (argp->createmode == NFS3_CREATE_UNCHECKED) {
+		/*
+		 * If name is already in dcache we need to check for mountpoints
+		 */
+		if (d_is_reg(child) &&
+		    unlikely(nfsd_mountpoint(child, exp))) {
+			status = nfsd_cross_mnt(rqstp, &child, &exp);
+			if (status != nfs_ok) {
+				exp_put(exp);
+				goto out;
+			}
+		}
+	}
+
+	status = fh_compose(resfhp, exp, child, fhp);
+	exp_put(exp);
 	if (status != nfs_ok)
 		goto out;
 
diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c
index e481804bb120c0..a14e829e1c4192 100644
--- a/fs/nfsd/nfs3xdr.c
+++ b/fs/nfsd/nfs3xdr.c
@@ -8,11 +8,13 @@
  */
 
 #include <linux/namei.h>
+#include <linux/nfs3.h>
 #include <linux/sunrpc/svc_xprt.h>
 #include "xdr3.h"
 #include "auth.h"
 #include "netns.h"
 #include "vfs.h"
+#include "nfserr.h"
 
 /*
  * Force construction of an empty post-op attr
@@ -556,6 +558,8 @@ nfs3svc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
 		return false;
 	if (xdr_stream_decode_u32(xdr, &args->stable) < 0)
 		return false;
+	if (args->stable > NFS_FILE_SYNC)
+		return false;
 
 	/* opaque data */
 	if (xdr_stream_decode_u32(xdr, &args->len) < 0)
diff --git a/fs/nfsd/nfs4acl.c b/fs/nfsd/nfs4acl.c
index 2c2f2fd89e8795..94f6ad381ebe5a 100644
--- a/fs/nfsd/nfs4acl.c
+++ b/fs/nfsd/nfs4acl.c
@@ -40,6 +40,7 @@
 
 #include "nfsfh.h"
 #include "nfsd.h"
+#include "nfserr.h"
 #include "acl.h"
 #include "vfs.h"
 
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 19dc337502ca86..a6b31d3f2bf66e 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -37,6 +37,7 @@
 #include <linux/sunrpc/svc_xprt.h>
 #include <linux/slab.h>
 #include "nfsd.h"
+#include "nfserr.h"
 #include "state.h"
 #include "netns.h"
 #include "stats.h"
@@ -1529,12 +1530,14 @@ void nfsd41_cb_referring_call(struct nfsd4_callback *cb,
 
 /**
  * nfsd41_cb_destroy_referring_call_list - release referring call info
- * @cb: context of a callback that has completed
+ * @cb: context of callback to release referring calls from
  *
  * Callers who allocate referring calls using nfsd41_cb_referring_call() must
  * release those resources by calling nfsd41_cb_destroy_referring_call_list.
  *
- * Caller serializes access to @cb.
+ * Caller serializes access to @cb. No CB_COMPOUND for @cb may be in
+ * flight, because encode_cb_sequence4args() walks this list as it
+ * encodes.
  */
 void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb)
 {
@@ -1556,6 +1559,7 @@ void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb)
 		list_del(&rcl->__list);
 		kfree(rcl);
 	}
+	cb->cb_nr_referring_call_list = 0;
 }
 
 static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata)
diff --git a/fs/nfsd/nfs4ctl.h b/fs/nfsd/nfs4ctl.h
new file mode 100644
index 00000000000000..bcec4c4ef1d536
--- /dev/null
+++ b/fs/nfsd/nfs4ctl.h
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Entry points by which the knfsd core drives the optional NFSv4
+ * subsystem: state lifecycle, the laundromat workqueue, the recovery
+ * directory, junctions, the CLD notifier, and leases-net setup.
+ *
+ * Separated from nfsd.h so that the many translation units that
+ * include nfsd.h but call none of these -- among them the NFSv2 and
+ * NFSv3 paths -- do not have to parse them. The CONFIG_NFSD_V4=n
+ * stubs let the version-agnostic callers invoke the routines
+ * unconditionally.
+ */
+
+#ifndef LINUX_NFSD_NFS4CTL_H
+#define LINUX_NFSD_NFS4CTL_H
+
+#include <linux/stddef.h>
+#include <linux/types.h>
+
+struct net;
+struct inode;
+struct dentry;
+struct svc_rqst;
+struct nfsd_net;
+
+#ifdef CONFIG_NFSD_V4
+extern unsigned long max_delegations;
+int nfsd4_init_slabs(void);
+void nfsd4_free_slabs(void);
+int nfs4_state_start(void);
+int nfs4_state_start_net(struct net *net);
+void nfs4_state_shutdown(void);
+void nfs4_state_shutdown_net(struct net *net);
+int nfs4_reset_recoverydir(char *recdir);
+char * nfs4_recoverydir(void);
+bool nfsd4_spo_must_allow(struct svc_rqst *rqstp);
+int nfsd4_create_laundry_wq(void);
+void nfsd4_destroy_laundry_wq(void);
+bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode);
+
+extern int nfsd4_is_junction(struct dentry *dentry);
+extern int register_cld_notifier(void);
+extern void unregister_cld_notifier(void);
+#ifdef CONFIG_NFSD_V4_2_INTER_SSC
+extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn);
+#endif
+
+extern void nfsd4_init_leases_net(struct nfsd_net *nn);
+
+#else /* CONFIG_NFSD_V4 */
+static inline int nfsd4_init_slabs(void) { return 0; }
+static inline void nfsd4_free_slabs(void) { }
+static inline int nfs4_state_start(void) { return 0; }
+static inline int nfs4_state_start_net(struct net *net) { return 0; }
+static inline void nfs4_state_shutdown(void) { }
+static inline void nfs4_state_shutdown_net(struct net *net) { }
+static inline int nfs4_reset_recoverydir(char *recdir) { return 0; }
+static inline char * nfs4_recoverydir(void) {return NULL; }
+static inline bool nfsd4_spo_must_allow(struct svc_rqst *rqstp)
+{
+	return false;
+}
+static inline int nfsd4_create_laundry_wq(void) { return 0; };
+static inline void nfsd4_destroy_laundry_wq(void) {};
+static inline bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp,
+					      struct inode *inode)
+{
+	return false;
+}
+
+static inline int nfsd4_is_junction(struct dentry *dentry)
+{
+	return 0;
+}
+
+static inline void nfsd4_init_leases_net(struct nfsd_net *nn) { };
+
+#define register_cld_notifier() 0
+#define unregister_cld_notifier() do { } while(0)
+
+#endif /* CONFIG_NFSD_V4 */
+
+#endif /* LINUX_NFSD_NFS4CTL_H */
diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c
index e9faf8b78f74a4..4e529759396342 100644
--- a/fs/nfsd/nfs4idmap.c
+++ b/fs/nfsd/nfs4idmap.c
@@ -41,6 +41,7 @@
 #include "auth.h"
 #include "idmap.h"
 #include "nfsd.h"
+#include "nfserr.h"
 #include "netns.h"
 #include "vfs.h"
 
diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
index 22bcb6d09f7037..da1816ddb0d15d 100644
--- a/fs/nfsd/nfs4layouts.c
+++ b/fs/nfsd/nfs4layouts.c
@@ -3,12 +3,13 @@
  * Copyright (c) 2014 Christoph Hellwig.
  */
 #include <linux/exportfs_block.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/file.h>
 #include <linux/jhash.h>
 #include <linux/sched.h>
 #include <linux/sunrpc/addr.h>
 
+#include "nfserr.h"
 #include "pnfs.h"
 #include "netns.h"
 #include "trace.h"
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 50c07561e31f3c..9cb9c57d7f30ee 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -35,22 +35,26 @@
 #include <linux/fs_struct.h>
 #include <linux/file.h>
 #include <linux/falloc.h>
+#include <linux/pagemap.h>
 #include <linux/slab.h>
 #include <linux/kthread.h>
 #include <linux/namei.h>
+#include <linux/pagemap.h>
 
 #include <linux/sunrpc/addr.h>
-#include <linux/nfs_ssc.h>
+#include <linux/nfsd_ssc.h>
 
 #include "attr4.h"
 #include "idmap.h"
 #include "cache.h"
 #include "xdr4.h"
+#include "nfs4ctl.h"
 #include "vfs.h"
 #include "current_stateid.h"
 #include "netns.h"
 #include "acl.h"
 #include "pnfs.h"
+#include "nfserr.h"
 #include "trace.h"
 
 static bool inter_copy_offload_enable;
@@ -69,6 +73,57 @@ MODULE_PARM_DESC(nfsd4_ssc_umount_timeout,
 
 #define NFSDDBG_FACILITY		NFSDDBG_PROC
 
+static const struct nfsd_access_map nfsd4_regaccess[] = {
+	{ NFS4_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS4_ACCESS_EXECUTE,	NFSD_MAY_EXEC				},
+	{ NFS4_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_TRUNC		},
+	{ NFS4_ACCESS_EXTEND,	NFSD_MAY_WRITE				},
+	{ NFS4_ACCESS_XAREAD,	NFSD_MAY_READ				},
+	{ NFS4_ACCESS_XAWRITE,	NFSD_MAY_WRITE				},
+	{ NFS4_ACCESS_XALIST,	NFSD_MAY_READ				},
+	{ 0,			0					}
+};
+
+static const struct nfsd_access_map nfsd4_diraccess[] = {
+	{ NFS4_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS4_ACCESS_LOOKUP,	NFSD_MAY_EXEC				},
+	{ NFS4_ACCESS_MODIFY,	NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC },
+	{ NFS4_ACCESS_EXTEND,	NFSD_MAY_EXEC|NFSD_MAY_WRITE		},
+	{ NFS4_ACCESS_DELETE,	NFSD_MAY_REMOVE				},
+	{ NFS4_ACCESS_XAREAD,	NFSD_MAY_READ				},
+	{ NFS4_ACCESS_XAWRITE,	NFSD_MAY_WRITE				},
+	{ NFS4_ACCESS_XALIST,	NFSD_MAY_READ				},
+	{ 0,			0					}
+};
+
+static const struct nfsd_access_map nfsd4_otheraccess[] = {
+	{ NFS4_ACCESS_READ,	NFSD_MAY_READ				},
+	{ NFS4_ACCESS_EXECUTE,	NFSD_MAY_EXEC				},
+	{ NFS4_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
+	{ NFS4_ACCESS_EXTEND,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
+	{ 0,			0					}
+};
+
+static const struct nfsd_access_maps nfsd4_access_maps = {
+	.regular	= nfsd4_regaccess,
+	.directory	= nfsd4_diraccess,
+	.other		= nfsd4_otheraccess,
+};
+
+static int nfsd4_iocb_flags(enum stable_how4 how)
+{
+	switch (how) {
+	case FILE_SYNC4:
+		/* persist data and timestamps */
+		return IOCB_DSYNC | IOCB_SYNC;
+	case DATA_SYNC4:
+		/* persist data only */
+		return IOCB_DSYNC;
+	default:
+		return 0;
+	}
+}
+
 static u32 nfsd_attrmask[] = {
 	NFSD_WRITEABLE_ATTRS_WORD0,
 	NFSD_WRITEABLE_ATTRS_WORD1,
@@ -169,23 +224,17 @@ do_open_permission(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfs
 	return fh_verify(rqstp, current_fh, S_IFREG, accmode);
 }
 
-static __be32 nfsd_check_obj_isreg(struct svc_fh *fh, u32 minor_version)
+static int nfsd_check_obj_isreg(struct dentry *child)
 {
-	umode_t mode = d_inode(fh->fh_dentry)->i_mode;
+	umode_t mode = d_inode(child)->i_mode;
 
 	if (S_ISREG(mode))
-		return nfs_ok;
+		return 0;
 	if (S_ISDIR(mode))
-		return nfserr_isdir;
+		return -EISDIR;
 	if (S_ISLNK(mode))
-		return nfserr_symlink;
-
-	/* RFC 7530 - 16.16.6 */
-	if (minor_version == 0)
-		return nfserr_symlink;
-	else
-		return nfserr_wrong_type;
-
+		return -ELOOP;
+	return -EFTYPE;
 }
 
 static void nfsd4_set_open_owner_reply_cache(struct nfsd4_compound_state *cstate, struct nfsd4_open *open, struct svc_fh *resfh)
@@ -202,40 +251,50 @@ static inline bool nfsd4_create_is_exclusive(int createmode)
 		createmode == NFS4_CREATE_EXCLUSIVE4_1;
 }
 
-static __be32
-nfsd4_vfs_create(struct svc_fh *fhp, struct dentry **child,
-		 struct nfsd4_open *open)
+static struct file *do_lookup_open(struct path *parent,
+				   struct qstr *name,
+				   unsigned int oflags,
+				   umode_t mode)
 {
-	struct file *filp;
+	struct file *filp = NULL;
 	struct path path;
-	int oflags;
+	struct dentry *child;
+	int want_write_err = 0;
 
-	oflags = O_CREAT | O_LARGEFILE;
-	if (nfsd4_create_is_exclusive(open->op_createmode))
-		oflags |= O_EXCL;
+	want_write_err = mnt_want_write(parent->mnt);
 
-	switch (open->op_share_access & NFS4_SHARE_ACCESS_BOTH) {
-	case NFS4_SHARE_ACCESS_WRITE:
-		oflags |= O_WRONLY;
-		break;
-	case NFS4_SHARE_ACCESS_BOTH:
-		oflags |= O_RDWR;
-		break;
-	default:
-		oflags |= O_RDONLY;
+	child = start_creating(&nop_mnt_idmap, parent->dentry, name);
+	if (IS_ERR(child)) {
+		filp = ERR_CAST(child);
+		goto out;
 	}
+	path.mnt = parent->mnt;
+	path.dentry = child;
 
-	path.mnt = fhp->fh_export->ex_path.mnt;
-	path.dentry = *child;
-	filp = dentry_create(&path, oflags, open->op_iattr.ia_mode,
-			     current_cred());
-	*child = path.dentry;
-
-	if (IS_ERR(filp))
-		return nfserrno(PTR_ERR(filp));
+	if (d_really_is_positive(child)) {
+		/*
+		 * open the file so that we consistently have a valid
+		 * op_filp and consequently a valid ->f_path.dentry.
+		 */
+		int err = nfsd_check_obj_isreg(child);
 
-	open->op_filp = filp;
-	return nfs_ok;
+		if (err)
+			filp = ERR_PTR(err);
+		else
+			filp = dentry_open(&path, oflags, current_cred());
+	} else if (!(oflags & O_CREAT)) {
+		filp = ERR_PTR(-ENOENT);
+	} else if (want_write_err) {
+		filp = ERR_PTR(want_write_err);
+	} else {
+		filp = dentry_create(&path, oflags, mode, current_cred());
+		child = path.dentry;
+	}
+	end_creating(child);
+out:
+	if (!want_write_err)
+		mnt_drop_write(parent->mnt);
+	return filp;
 }
 
 /*
@@ -254,11 +313,15 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		.na_iattr	= iap,
 		.na_seclabel	= &open->op_label,
 	};
-	struct dentry *parent, *child = ERR_PTR(-EINVAL);
+	int oflags = O_CREAT | O_LARGEFILE;
+	struct dentry *child = ERR_PTR(-EINVAL);
+	struct path parent = {
+		.mnt = fhp->fh_export->ex_path.mnt,
+		.dentry = fhp->fh_dentry,
+	};
 	__u32 v_mtime, v_atime;
-	struct inode *inode;
-	__be32 status;
-	int host_err;
+	__be32 status, create_status;
+	int want_write_err;
 
 	if (name_is_dot_dotdot(open->op_fname, open->op_fnamelen))
 		return nfserr_exist;
@@ -268,43 +331,69 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	status = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_EXEC);
 	if (status != nfs_ok)
 		return status;
-	parent = fhp->fh_dentry;
-	inode = d_inode(parent);
-
-	host_err = fh_want_write(fhp);
-	if (host_err)
-		return nfserrno(host_err);
 
-	if (open->op_acl) {
-		if (open->op_dpacl || open->op_pacl) {
-			status = nfserr_inval;
-			goto out;
-		}
-		if (is_create_with_attrs(open)) {
-			status = nfsd4_acl_to_attr(NF4REG, open->op_acl,
-						   &attrs);
-			if (status)
-				goto out;
+	if (open->op_createmode == NFS4_CREATE_UNCHECKED) {
+		/*
+		 * If name is already in dcache we need to check for mountpoints
+		 */
+		child = try_lookup_noperm(&QSTR_LEN(open->op_fname,
+						    open->op_fnamelen),
+					  parent.dentry);
+		if (child && !IS_ERR(child) && d_is_reg(child) &&
+		    unlikely(nfsd_mountpoint(child, fhp->fh_export))) {
+			struct svc_export *exp = exp_get(fhp->fh_export);
+
+			status = nfsd_cross_mnt(rqstp, &child, &exp);
+			if (status == nfs_ok)
+				status = fh_compose(resfhp, exp,
+						    child, fhp);
+			fh_fill_post_noop(fhp);
+			open->op_truncate =
+				(iap->ia_valid & ATTR_SIZE) &&
+				!iap->ia_size;
+			dput(child);
+			exp_put(exp);
+			return status;
 		}
-	} else if (is_create_with_attrs(open)) {
-		/* The dpacl and pacl will get released by nfsd_attrs_free(). */
-		attrs.na_dpacl = open->op_dpacl;
-		attrs.na_pacl = open->op_pacl;
-		open->op_dpacl = NULL;
-		open->op_pacl = NULL;
+		if (!IS_ERR(child))
+			dput(child);
 	}
 
-	child = start_creating(&nop_mnt_idmap, parent,
-			       &QSTR_LEN(open->op_fname, open->op_fnamelen));
-	if (IS_ERR(child)) {
-		status = nfserrno(PTR_ERR(child));
-		goto out;
+	if (!IS_POSIXACL(d_inode(parent.dentry)))
+		iap->ia_mode &= ~current_umask();
+
+	/*
+	 * For the EXCLUSIVE modes we do our own uniqueness tests
+	 * so don't want O_EXCL.
+	 */
+	if (open->op_createmode == NFS4_CREATE_GUARDED)
+		oflags |= O_EXCL;
+
+	switch (open->op_share_access & NFS4_SHARE_ACCESS_BOTH) {
+	case NFS4_SHARE_ACCESS_WRITE:
+		oflags |= O_WRONLY;
+		break;
+	case NFS4_SHARE_ACCESS_BOTH:
+		oflags |= O_RDWR;
+		break;
+	default:
+		oflags |= O_RDONLY;
 	}
 
-	if (d_really_is_negative(child)) {
-		status = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_CREATE);
-		if (status != nfs_ok)
-			goto out;
+	if (!is_create_with_attrs(open)) {
+		/* No attrs to check */
+	} else if (open->op_acl) {
+		if (open->op_dpacl || open->op_pacl) {
+			/* Cannot specify both NFSv4 and Posix ACLs */
+			return nfserr_inval;
+		}
+		status = nfsd4_acl_to_attr(NF4REG, open->op_acl,
+						   &attrs);
+		if (status)
+			return status;
+	} else {
+		attrs.na_dpacl = posix_acl_dup(open->op_dpacl);
+		attrs.na_pacl = posix_acl_dup(open->op_pacl);
 	}
 
 	v_mtime = 0;
@@ -322,24 +411,53 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		 */
 		v_mtime = verifier[0] & 0x7fffffff;
 		v_atime = verifier[1] & 0x7fffffff;
+
+		iap->ia_valid |= ATTR_MTIME | ATTR_ATIME |
+				 ATTR_MTIME_SET|ATTR_ATIME_SET;
+		iap->ia_mtime.tv_sec = v_mtime;
+		iap->ia_atime.tv_sec = v_atime;
+		iap->ia_mtime.tv_nsec = 0;
+		iap->ia_atime.tv_nsec = 0;
 	}
 
-	if (d_really_is_positive(child)) {
-		/* NFSv4 protocol requires change attributes even though
-		 * no change happened.
-		 */
-		status = fh_fill_both_attrs(fhp);
-		if (status != nfs_ok)
-			goto out;
+	create_status = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_CREATE);
+	if (create_status)
+		/* Might still succeed if no create is needed */
+		oflags &= ~O_CREAT;
+
+	open->op_filp = do_lookup_open(&parent,
+				       &QSTR_LEN(open->op_fname,
+						 open->op_fnamelen),
+				       oflags,
+				       open->op_iattr.ia_mode);
+	if (IS_ERR(open->op_filp)) {
+		status = nfserrno(PTR_ERR(open->op_filp));
+		open->op_filp = NULL;
+		if (status == nfserr_noent && create_status)
+			status = create_status;
+		goto out;
+	}
 
-		status = fh_compose(resfhp, fhp->fh_export, child, fhp);
-		if (status != nfs_ok)
-			goto out;
+	child = open->op_filp->f_path.dentry;
+	open->op_created = open->op_filp->f_mode & FMODE_CREATED;
 
-		switch (open->op_createmode) {
-		case NFS4_CREATE_UNCHECKED:
-			if (!d_is_reg(child))
-				break;
+	status = fh_compose(resfhp, fhp->fh_export, child, fhp);
+	if (status != nfs_ok)
+		goto out;
+
+	if (!open->op_created &&
+	    nfsd4_create_is_exclusive(open->op_createmode) &&
+	    inode_get_mtime_sec(d_inode(child)) == v_mtime &&
+	    inode_get_atime_sec(d_inode(child)) == v_atime &&
+	    d_inode(child)->i_size == 0)
+		open->op_created = true;
+
+	if (!open->op_created) {
+		if (open->op_createmode == NFS4_CREATE_UNCHECKED) {
+			/* NFSv4 protocol requires change attributes
+			 * even though no change happened.
+			 */
+			fh_fill_post_noop(fhp);
 
 			/*
 			 * In NFSv4, we don't want to truncate the file
@@ -347,63 +465,30 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 			 * some other reason. Furthermore, if the size is
 			 * nonzero, we should ignore it according to spec!
 			 */
-			open->op_truncate = (iap->ia_valid & ATTR_SIZE) &&
-						!iap->ia_size;
-			break;
-		case NFS4_CREATE_GUARDED:
+			open->op_truncate = (d_is_reg(child) &&
+					     (iap->ia_valid & ATTR_SIZE) &&
+					     !iap->ia_size);
+		} else
 			status = nfserr_exist;
-			break;
-		case NFS4_CREATE_EXCLUSIVE:
-			if (inode_get_mtime_sec(d_inode(child)) == v_mtime &&
-			    inode_get_atime_sec(d_inode(child)) == v_atime &&
-			    d_inode(child)->i_size == 0) {
-				open->op_created = true;
-				break;		/* subtle */
-			}
-			status = nfserr_exist;
-			break;
-		case NFS4_CREATE_EXCLUSIVE4_1:
-			if (inode_get_mtime_sec(d_inode(child)) == v_mtime &&
-			    inode_get_atime_sec(d_inode(child)) == v_atime &&
-			    d_inode(child)->i_size == 0) {
-				open->op_created = true;
-				goto set_attr;	/* subtle */
-			}
-			status = nfserr_exist;
-		}
 		goto out;
 	}
-
-	if (!IS_POSIXACL(inode))
-		iap->ia_mode &= ~current_umask();
-
-	status = fh_fill_pre_attrs(fhp);
-	if (status != nfs_ok)
-		goto out;
-	status = nfsd4_vfs_create(fhp, &child, open);
-	if (status != nfs_ok)
-		goto out;
-	open->op_created = true;
+	/* file was created */
 	fh_fill_post_attrs(fhp);
 
-	status = fh_compose(resfhp, fhp->fh_export, child, fhp);
-	if (status != nfs_ok)
-		goto out;
-
 	/* A newly created file already has a file size of zero. */
 	if ((iap->ia_valid & ATTR_SIZE) && (iap->ia_size == 0))
 		iap->ia_valid &= ~ATTR_SIZE;
-	if (nfsd4_create_is_exclusive(open->op_createmode)) {
-		iap->ia_valid = ATTR_MTIME | ATTR_ATIME |
-				ATTR_MTIME_SET|ATTR_ATIME_SET;
-		iap->ia_mtime.tv_sec = v_mtime;
-		iap->ia_atime.tv_sec = v_atime;
-		iap->ia_mtime.tv_nsec = 0;
-		iap->ia_atime.tv_nsec = 0;
-	}
 
-set_attr:
-	status = nfsd_create_setattr(rqstp, fhp, resfhp, &attrs);
+	/* We will need write access to set the attrs */
+	want_write_err = fh_want_write(fhp);
+	if (!want_write_err) {
+		status = nfsd_create_setattr(rqstp, fhp,
+					     resfhp, &attrs);
+		fh_drop_write(fhp);
+	} else if (nfsd_attrs_valid(&attrs)) {
+		/* Needed write access */
+		status = nfserrno(want_write_err);
+	}
 
 	if (attrs.na_labelerr)
 		open->op_bmval[2] &= ~FATTR4_WORD2_SECURITY_LABEL;
@@ -414,9 +499,7 @@ nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	if (attrs.na_paclerr)
 		open->op_bmval[2] &= ~FATTR4_WORD2_POSIX_ACCESS_ACL;
 out:
-	end_creating(child);
 	nfsd_attrs_free(&attrs);
-	fh_drop_write(fhp);
 	return status;
 }
 
@@ -465,6 +548,9 @@ do_open_lookup(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, stru
 	fh_init(*resfh, NFS4_FHSIZE);
 	open->op_truncate = false;
 
+	status = fh_fill_pre_attrs_unlocked(current_fh);
+	if (status)
+		goto out;
 	if (open->op_create) {
 		/* FIXME: check session persistence and pnfs flags.
 		 * The nfsv4.1 spec requires the following semantics:
@@ -496,15 +582,15 @@ do_open_lookup(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, stru
 	} else {
 		status = nfsd_lookup(rqstp, current_fh,
 				     open->op_fname, open->op_fnamelen, *resfh);
-		if (status == nfs_ok)
-			/* NFSv4 protocol requires change attributes even though
-			 * no change happened.
-			 */
-			status = fh_fill_both_attrs(current_fh);
+		/*
+		 * NFSv4 protocol requires change attributes even though
+		 * no change happened.
+		 */
+		fh_fill_post_noop(current_fh);
 	}
 	if (status)
 		goto out;
-	status = nfsd_check_obj_isreg(*resfh, cstate->minorversion);
+	status = nfserrno(nfsd_check_obj_isreg((*resfh)->fh_dentry));
 	if (status)
 		goto out;
 
@@ -516,6 +602,10 @@ do_open_lookup(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, stru
 	status = do_open_permission(rqstp, *resfh, open, accmode);
 	set_change_info(&open->op_cinfo, current_fh);
 out:
+	if (status == nfserr_wrong_type && cstate->minorversion == 0)
+		/* RFC 7530 - 16.16.6 */
+		return nfserr_symlink;
+
 	return status;
 }
 
@@ -703,13 +793,6 @@ static __be32 nfsd4_open_omfg(struct svc_rqst *rqstp, struct nfsd4_compound_stat
 	return nfsd4_open(rqstp, cstate, &op->u);
 }
 
-static void
-nfsd4_open_release(union nfsd4_op_u *u)
-{
-	posix_acl_release(u->open.op_dpacl);
-	posix_acl_release(u->open.op_pacl);
-}
-
 /*
  * filehandle-manipulating ops.
  */
@@ -788,17 +871,18 @@ nfsd4_access(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	struct nfsd4_access *access = &u->access;
 	u32 access_full;
 
-	access_full = NFS3_ACCESS_FULL;
+	access_full = NFS4_ACCESS_READ | NFS4_ACCESS_LOOKUP |
+		      NFS4_ACCESS_MODIFY | NFS4_ACCESS_EXTEND |
+		      NFS4_ACCESS_DELETE | NFS4_ACCESS_EXECUTE;
 	if (cstate->minorversion >= 2)
 		access_full |= NFS4_ACCESS_XALIST | NFS4_ACCESS_XAREAD |
 			       NFS4_ACCESS_XAWRITE;
 
 	if (access->ac_req_access & ~access_full)
 		return nfserr_inval;
-
 	access->ac_resp_access = access->ac_req_access;
-	return nfsd_access(rqstp, &cstate->current_fh, &access->ac_resp_access,
-			   &access->ac_supported);
+	return nfsd_access(rqstp, &cstate->current_fh, &nfsd4_access_maps,
+			   &access->ac_resp_access, &access->ac_supported);
 }
 
 static __be32
@@ -829,16 +913,13 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	struct nfsd_attrs attrs = {
 		.na_iattr	= &create->cr_iattr,
 		.na_seclabel	= &create->cr_label,
-		.na_dpacl	= create->cr_dpacl,
-		.na_pacl	= create->cr_pacl,
+		.na_dpacl	= posix_acl_dup(create->cr_dpacl),
+		.na_pacl	= posix_acl_dup(create->cr_pacl),
 	};
 	struct svc_fh resfh;
 	__be32 status;
 	dev_t rdev;
 
-	create->cr_dpacl = NULL;
-	create->cr_pacl = NULL;
-
 	fh_init(&resfh, NFS4_FHSIZE);
 
 	status = fh_verify(rqstp, &cstate->current_fh, S_IFDIR, NFSD_MAY_NOP);
@@ -1248,8 +1329,8 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	struct nfsd_attrs attrs = {
 		.na_iattr	= &setattr->sa_iattr,
 		.na_seclabel	= &setattr->sa_label,
-		.na_pacl	= setattr->sa_pacl,
-		.na_dpacl	= setattr->sa_dpacl,
+		.na_pacl	= posix_acl_dup(setattr->sa_pacl),
+		.na_dpacl	= posix_acl_dup(setattr->sa_dpacl),
 	};
 	bool save_no_wcc, deleg_attrs;
 	struct nfs4_stid *st = NULL;
@@ -1257,10 +1338,6 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	__be32 status = nfs_ok;
 	int err;
 
-	/* Transfer ownership to attrs for cleanup via nfsd_attrs_free() */
-	setattr->sa_pacl = NULL;
-	setattr->sa_dpacl = NULL;
-
 	deleg_attrs = setattr->sa_bmval[2] & (FATTR4_WORD2_TIME_DELEG_ACCESS |
 					      FATTR4_WORD2_TIME_DELEG_MODIFY);
 
@@ -1377,7 +1454,7 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	write->wr_how_written = write->wr_stable_how;
 	status = nfsd_vfs_write(rqstp, &cstate->current_fh, nf,
 				write->wr_offset, &write->wr_payload,
-				&cnt, write->wr_how_written,
+				&cnt, nfsd4_iocb_flags(write->wr_how_written),
 				(__be32 *)write->wr_verifier.data);
 	nfsd_file_put(nf);
 
@@ -1431,16 +1508,37 @@ nfsd4_clone(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 {
 	struct nfsd4_clone *clone = &u->clone;
 	struct nfsd_file *src, *dst;
+	bool sync_failed = false;
+	errseq_t since;
 	__be32 status;
+	int host_err;
 
 	status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
 				   &clone->cl_dst_stateid, &dst);
 	if (status)
 		goto out;
 
-	status = nfsd4_clone_file_range(rqstp, src, clone->cl_src_pos,
-			dst, clone->cl_dst_pos, clone->cl_count,
-			EX_ISSYNC(cstate->current_fh.fh_export));
+	host_err = nfsd_clone_file_range(src->nf_file, clone->cl_src_pos,
+					 dst->nf_file, clone->cl_dst_pos,
+					 clone->cl_count, &since);
+	if (!host_err && EX_ISSYNC(cstate->current_fh.fh_export)) {
+		host_err = nfsd_clone_sync_range(src->nf_file, dst->nf_file,
+						 clone->cl_dst_pos,
+						 clone->cl_count, since);
+		sync_failed = host_err < 0;
+	}
+	if (host_err < 0) {
+		trace_nfsd_clone_file_range_err(rqstp, &cstate->save_fh,
+				clone->cl_src_pos, &cstate->current_fh,
+				clone->cl_dst_pos, clone->cl_count, host_err);
+		if (sync_failed) {
+			struct nfsd_net *nn = net_generic(dst->nf_net,
+							  nfsd_net_id);
+
+			nfsd_maybe_reset_write_verifier(nn, rqstp, host_err);
+		}
+	}
+	status = nfserrno(host_err);
 
 	if (!status && (READ_ONCE(dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0)
 		nfsd_update_cmtime_attr(dst->nf_file, 0);
@@ -1653,13 +1751,6 @@ void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb)
 
 #ifdef CONFIG_NFSD_V4_2_INTER_SSC
 
-extern struct file *nfs42_ssc_open(struct vfsmount *ss_mnt,
-				   struct nfs_fh *src_fh,
-				   nfs4_stateid *stateid);
-extern void nfs42_ssc_close(struct file *filep);
-
-extern void nfs_sb_deactive(struct super_block *sb);
-
 #define NFSD42_INTERSSC_MOUNTOPS "vers=4.2,addr=%s,sec=sys"
 
 /*
@@ -1882,7 +1973,7 @@ nfsd4_cleanup_inter_ssc(struct nfsd4_ssc_umount_item *nsui, struct file *filp,
 	struct nfsd_net *nn = net_generic(dst->nf_net, nfsd_net_id);
 	long timeout = msecs_to_jiffies(nfsd4_ssc_umount_timeout);
 
-	nfs42_ssc_close(filp);
+	nfsd42_ssc_close(filp);
 	fput(filp);
 
 	spin_lock(&nn->nfsd_ssc_lock);
@@ -1914,12 +2005,6 @@ nfsd4_cleanup_inter_ssc(struct nfsd4_ssc_umount_item *nsui, struct file *filp,
 {
 }
 
-static struct file *nfs42_ssc_open(struct vfsmount *ss_mnt,
-				   struct nfs_fh *src_fh,
-				   nfs4_stateid *stateid)
-{
-	return NULL;
-}
 #endif /* CONFIG_NFSD_V4_2_INTER_SSC */
 
 static __be32
@@ -1974,7 +2059,7 @@ static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
 	copy->cp_res.wr_stable_how =
 		test_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags) ?
-			NFS_FILE_SYNC : NFS_UNSTABLE;
+			FILE_SYNC4 : UNSTABLE4;
 	nfsd4_copy_set_sync(copy, sync);
 }
 
@@ -2142,8 +2227,8 @@ static int nfsd4_do_async_copy(void *data)
 	if (nfsd4_ssc_is_inter(copy)) {
 		struct file *filp;
 
-		filp = nfs42_ssc_open(copy->ss_nsui->nsui_vfsmount,
-				      &copy->c_fh, &copy->stateid);
+		filp = nfsd42_ssc_open(copy->ss_nsui->nsui_vfsmount,
+				       &copy->c_fh, &copy->stateid);
 		if (IS_ERR(filp)) {
 			switch (PTR_ERR(filp)) {
 			case -EBADF:
@@ -3328,7 +3413,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
 
 			if (current_fh->fh_export &&
 					need_wrongsec_check(rqstp))
-				op->status = check_nfsd_access(current_fh->fh_export, rqstp, false);
+				op->status = check_nfsd_access(current_fh->fh_export, rqstp);
 		}
 encode_op:
 		if (op->status == nfserr_replay_me) {
@@ -3842,7 +3927,6 @@ static const struct nfsd4_operation nfsd4_ops[] = {
 	},
 	[OP_OPEN] = {
 		.op_func = nfsd4_open,
-		.op_release = nfsd4_open_release,
 		.op_flags = OP_HANDLES_WRONGSEC | OP_MODIFIES_SOMETHING,
 		.op_name = "OP_OPEN",
 		.op_rsize_bop = nfsd4_open_rsize,
diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
index d513971fb119d5..58a0098608fa44 100644
--- a/fs/nfsd/nfs4recover.c
+++ b/fs/nfsd/nfs4recover.c
@@ -41,12 +41,14 @@
 #include <linux/fs.h>
 #include <linux/hex.h>
 #include <linux/module.h>
+#include <linux/umh.h>
 #include <net/net_namespace.h>
 #include <linux/sunrpc/rpc_pipe_fs.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/nfsd/cld.h>
 
 #include "nfsd.h"
+#include "nfs4ctl.h"
 #include "state.h"
 #include "vfs.h"
 #include "netns.h"
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 9c4adf3110aea4..1de6c6d757c3ed 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -45,13 +45,15 @@
 #include <linux/string_helpers.h>
 #include <linux/fsnotify.h>
 #include <linux/rhashtable.h>
-#include <linux/nfs_ssc.h>
+#include <linux/nfsd_ssc.h>
 
 #include "xdr4.h"
+#include "nfs4ctl.h"
 #include "xdr4cb.h"
 #include "vfs.h"
 #include "current_stateid.h"
 #include "stats.h"
+#include "nfserr.h"
 
 #include "netns.h"
 #include "pnfs.h"
@@ -91,7 +93,9 @@ static void nfs4_free_ol_stateid(struct nfs4_stid *stid);
 static void nfsd4_end_grace(struct nfsd_net *nn);
 static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps);
 static void nfsd4_file_hash_remove(struct nfs4_file *fi);
-static void deleg_reaper(struct nfsd_net *nn);
+static void deleg_reaper(struct nfsd_net *nn, unsigned long backlog);
+static void nfsd4_drop_revoked_stid(struct nfs4_stid *s)
+	__releases(&s->sc_client->cl_lock);
 
 static const struct lease_manager_operations nfsd_lease_mng_ops;
 
@@ -1157,14 +1161,18 @@ static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
  */
 static void nfs4_free_deleg(struct nfs4_stid *stid)
 {
+	struct nfsd_net *nn = net_generic(stid->sc_client->net, nfsd_net_id);
 	struct nfs4_delegation *dp = delegstateid(stid);
 
 	WARN_ON_ONCE(!list_empty(&stid->sc_cp_list));
 	WARN_ON_ONCE(!list_empty(&dp->dl_perfile));
 	WARN_ON_ONCE(!list_empty(&dp->dl_perclnt));
 	WARN_ON_ONCE(!list_empty(&dp->dl_recall_lru));
+	/* The list outlives one recall, so ->release() cannot free it. */
+	nfsd41_cb_destroy_referring_call_list(&dp->dl_recall);
 	kmem_cache_free(deleg_slab, stid);
 	atomic_long_dec(&num_delegations);
+	atomic_long_dec(&nn->nfsd_delegations);
 }
 
 /*
@@ -1249,6 +1257,7 @@ __alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
 		   struct nfs4_clnt_odstate *odstate, u32 dl_type,
 		   void (*sc_free)(struct nfs4_stid *))
 {
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 	struct nfs4_delegation *dp;
 	struct nfs4_stid *stid;
 	long n;
@@ -1257,6 +1266,7 @@ __alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
 		return NULL;
 
 	n = atomic_long_inc_return(&num_delegations);
+	atomic_long_inc(&nn->nfsd_delegations);
 	if (n < 0 || n > max_delegations)
 		goto out_dec;
 
@@ -1279,6 +1289,9 @@ __alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
 	dp->dl_type = dl_type;
 	dp->dl_retries = 1;
 	dp->dl_recalled = false;
+	dp->dl_recall_rejected = false;
+	dp->dl_recall_grant.valid = false;
+	dp->dl_recall_grant.retired_at_send = false;
 	get_nfs4_file(fp);
 	dp->dl_stid.sc_file = fp;
 	nfsd4_init_cb(&dp->dl_recall, dp->dl_stid.sc_client,
@@ -1286,6 +1299,7 @@ __alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp,
 	return dp;
 out_dec:
 	atomic_long_dec(&num_delegations);
+	atomic_long_dec(&nn->nfsd_delegations);
 	return NULL;
 }
 
@@ -1314,6 +1328,7 @@ static void nfs4_free_dir_deleg(struct nfs4_stid *stid)
 	for (i = 0; i < ncn->ncn_evt_cnt; ++i)
 		nfsd_notify_event_put(ncn->ncn_evt[i]);
 	kfree(ncn->ncn_nf);
+	kfree(ncn->ncn_masks);
 	for (i = 0; i < NOTIFY4_PAGE_ARRAY_SIZE; i++) {
 		if (!ncn->ncn_pages[i])
 			break;
@@ -1346,6 +1361,11 @@ alloc_init_dir_deleg(struct nfs4_client *clp, struct nfs4_file *fp)
 		nfs4_put_stid(&dp->dl_stid);
 		return NULL;
 	}
+	ncn->ncn_masks = kcalloc(NOTIFY4_EVENT_QUEUE_SIZE, sizeof(*ncn->ncn_masks), GFP_KERNEL);
+	if (!ncn->ncn_masks) {
+		nfs4_put_stid(&dp->dl_stid);
+		return NULL;
+	}
 	spin_lock_init(&ncn->ncn_lock);
 	nfsd4_init_cb(&ncn->ncn_cb, dp->dl_stid.sc_client,
 			&nfsd4_cb_notify_ops, NFSPROC4_CLNT_CB_NOTIFY);
@@ -1512,6 +1532,7 @@ hash_delegation_locked(struct nfs4_delegation *dp, struct nfs4_file *fp)
 	dp->dl_stid.sc_type = SC_TYPE_DELEG;
 	list_add(&dp->dl_perfile, &fp->fi_delegations);
 	list_add(&dp->dl_perclnt, &clp->cl_delegations);
+	clp->cl_deleg_count++;
 	return 0;
 }
 
@@ -1543,6 +1564,7 @@ unhash_delegation_locked(struct nfs4_delegation *dp, unsigned short statusmask)
 	++dp->dl_time;
 	spin_lock(&fp->fi_lock);
 	list_del_init(&dp->dl_perclnt);
+	dp->dl_stid.sc_client->cl_deleg_count--;
 	list_del_init(&dp->dl_recall_lru);
 	list_del_init(&dp->dl_perfile);
 	spin_unlock(&fp->fi_lock);
@@ -1563,27 +1585,22 @@ static void destroy_delegation(struct nfs4_delegation *dp)
 }
 
 /**
- * revoke_delegation - perform nfs4 delegation structure cleanup
- * @dp: pointer to the delegation
+ * revoke_delegation - dispose of a delegation the server has revoked
+ * @dp: delegation to dispose of
+ *
+ * The caller holds a reference on @dp, which this function consumes.
+ * On NFSv4.1 and newer, @dp's sc_status must already carry
+ * SC_STATUS_REVOKED or SC_STATUS_ADMIN_REVOKED.
  *
- * This function assumes that it's called either from the administrative
- * interface (nfsd4_revoke_states()) that's revoking a specific delegation
- * stateid or it's called from a laundromat thread (nfsd4_landromat()) that
- * determined that this specific state has expired and needs to be revoked
- * (both mark state with the appropriate stid sc_status mode). It is also
- * assumed that a reference was taken on the @dp state. This function
- * consumes that reference.
+ * @dp is parked on the client's cl_revoked list to await a FREE_STATEID.
+ * Where none can arrive, @dp is destroyed here instead: FREE_STATEID has
+ * already freed it, or the client rejected the recall with
+ * NFS4ERR_BADHANDLE or NFS4ERR_BAD_STATEID and holds no record of the
+ * delegation. NFS4ERR_ADMIN_REVOKED still prompts one, so an
+ * administrative revoke waits on cl_revoked.
  *
- * If this function finds that the @dp state is SC_STATUS_FREED it means
- * that a FREE_STATEID operation for this stateid has been processed and
- * we can proceed to removing it from recalled list. However, if @dp state
- * isn't marked SC_STATUS_FREED, it means we need place it on the cl_revoked
- * list and wait for the FREE_STATEID to arrive from the client. At the same
- * time, we need to mark it as SC_STATUS_FREEABLE to indicate to the
- * nfsd4_free_stateid() function that this stateid has already been added
- * to the cl_revoked list and that nfsd4_free_stateid() is now responsible
- * for removing it from the list. Inspection of where the delegation state
- * in the revocation process is protected by the clp->cl_lock.
+ * Context: Takes and releases the client's cl_lock; may sleep after
+ *          dropping it.
  */
 static void revoke_delegation(struct nfs4_delegation *dp)
 {
@@ -1601,6 +1618,19 @@ static void revoke_delegation(struct nfs4_delegation *dp)
 		list_del_init(&dp->dl_recall_lru);
 		goto out;
 	}
+	if (dp->dl_recall_rejected &&
+	    !(dp->dl_stid.sc_status & SC_STATUS_ADMIN_REVOKED)) {
+		/*
+		 * SC_STATUS_CLOSED, set under cl_lock, makes a racing
+		 * FREE_STATEID bail out rather than drop this reference
+		 * too. The put releases what cl_revoked would have held.
+		 */
+		dp->dl_stid.sc_status |= SC_STATUS_CLOSED;
+		spin_unlock(&clp->cl_lock);
+		nfs4_put_stid(&dp->dl_stid);
+		destroy_unhashed_deleg(dp);
+		return;
+	}
 	list_add(&dp->dl_recall_lru, &clp->cl_revoked);
 	dp->dl_stid.sc_status |= SC_STATUS_FREEABLE;
 out:
@@ -2789,10 +2819,16 @@ void nfsd4_put_client(struct nfs4_client *clp)
 static void
 free_client(struct nfs4_client *clp)
 {
-	while (!list_empty(&clp->cl_sessions)) {
+	LIST_HEAD(reaplist);
+
+	/* client_info_show() walks cl_sessions under cl_lock */
+	spin_lock(&clp->cl_lock);
+	list_splice_init(&clp->cl_sessions, &reaplist);
+	spin_unlock(&clp->cl_lock);
+	while (!list_empty(&reaplist)) {
 		struct nfsd4_session *ses;
-		ses = list_entry(clp->cl_sessions.next, struct nfsd4_session,
-				se_perclnt);
+		ses = list_entry(reaplist.next, struct nfsd4_session,
+				 se_perclnt);
 		list_del(&ses->se_perclnt);
 		WARN_ON_ONCE(atomic_read(&ses->se_ref));
 		free_session(ses);
@@ -2889,11 +2925,18 @@ __destroy_client(struct nfs4_client *clp)
 		list_del_init(&dp->dl_recall_lru);
 		destroy_unhashed_deleg(dp);
 	}
+	/*
+	 * A CB_RECALL reply can release revoked delegations concurrently:
+	 * nfsd4_shutdown_callback() has not run yet.
+	 */
+	spin_lock(&clp->cl_lock);
 	while (!list_empty(&clp->cl_revoked)) {
 		dp = list_entry(clp->cl_revoked.next, struct nfs4_delegation, dl_recall_lru);
-		list_del_init(&dp->dl_recall_lru);
-		nfs4_put_stid(&dp->dl_stid);
+		/* this function drops ->cl_lock */
+		nfsd4_drop_revoked_stid(&dp->dl_stid);
+		spin_lock(&clp->cl_lock);
 	}
+	spin_unlock(&clp->cl_lock);
 	while (!list_empty(&clp->cl_openowners)) {
 		oo = list_entry(clp->cl_openowners.next, struct nfs4_openowner, oo_perclient);
 		nfs4_get_stateowner(&oo->oo_owner);
@@ -3737,14 +3780,9 @@ nfsd4_cb_notify_prepare(struct nfsd4_callback *cb)
 		struct nfsd_notify_event *nne = events[i];
 
 		if (!error) {
-			u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp));
+			u32 *maskp = &ncn->ncn_masks[i];
 			u8 *p;
 
-			if (!maskp) {
-				error = true;
-				goto put_event;
-			}
-
 			p = nfsd4_encode_notify_event(&stream, nne, dp, nf, maskp);
 			if (!p) {
 				pr_notice("Could not generate CB_NOTIFY from fsnotify mask 0x%x\n",
@@ -3762,13 +3800,10 @@ nfsd4_cb_notify_prepare(struct nfsd4_callback *cb)
 		nfsd_notify_event_put(nne);
 	}
 	if (!error && (dp->dl_notify_mask & BIT(NOTIFY4_CHANGE_DIR_ATTRS))) {
-		u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp));
+		u32 *maskp = &ncn->ncn_masks[count];
 		u8 *p;
 
-		if (maskp)
-			p = nfsd4_encode_dir_attr_change(&stream, dp, nf);
-		else
-			p = ERR_PTR(-ENOBUFS);
+		p = nfsd4_encode_dir_attr_change(&stream, dp, nf);
 
 		if (IS_ERR(p)) {
 			/*
@@ -4216,12 +4251,18 @@ nfsd4_set_ex_flags(struct nfs4_client *new, struct nfsd4_exchange_id *clid)
 static bool client_has_openowners(struct nfs4_client *clp)
 {
 	struct nfs4_openowner *oo;
+	bool found = false;
 
+	spin_lock(&clp->cl_lock);
 	list_for_each_entry(oo, &clp->cl_openowners, oo_perclient) {
-		if (!list_empty(&oo->oo_owner.so_stateids))
-			return true;
+		if (!list_empty(&oo->oo_owner.so_stateids)) {
+			found = true;
+			break;
+		}
 	}
-	return false;
+	spin_unlock(&clp->cl_lock);
+
+	return found;
 }
 
 static bool client_has_state(struct nfs4_client *clp)
@@ -5016,6 +5057,7 @@ __be32
 nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
 {
+	struct nfsd4_compoundargs *args = rqstp->rq_argp;
 	struct nfsd4_sequence *seq = &u->sequence;
 	struct nfsd4_compoundres *resp = rqstp->rq_resp;
 	struct xdr_stream *xdr = resp->xdr;
@@ -5025,6 +5067,7 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	struct nfsd4_conn *conn;
 	__be32 status;
 	int buflen;
+	u32 maxlen, respsize;
 	struct net *net = SVC_NET(rqstp);
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
@@ -5102,7 +5145,22 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 			session->se_fchannel.maxresp_sz;
 	status = (seq->cachethis) ? nfserr_rep_too_big_to_cache :
 				    nfserr_rep_too_big;
-	if (xdr_restrict_buflen(xdr, buflen - rqstp->rq_auth_slack))
+	if (buflen < rqstp->rq_auth_slack)
+		goto out_put_session;
+	maxlen = buflen - rqstp->rq_auth_slack;
+
+	/*
+	 * A SEQUENCE result too large for maxlen never reaches
+	 * nfsd4_encode_sequence(), so cstate.data_offset stays zero and
+	 * the reply cache overruns the slot.
+	 */
+	respsize = nfsd4_max_reply(rqstp, &args->ops[0]);
+	if (!nfsd4_last_compound_op(rqstp))
+		respsize += COMPOUND_ERR_SLACK_SPACE;
+	if (xdr->buf->len + respsize > maxlen)
+		goto out_put_session;
+
+	if (xdr_restrict_buflen(xdr, maxlen))
 		goto out_put_session;
 	svc_reserve_auth(rqstp, buflen);
 
@@ -5118,6 +5176,7 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		slot->sl_flags &= ~NFSD4_SLOT_CACHETHIS;
 
 	cstate->slot = slot;
+	cstate->slot_owned = true;
 	cstate->session = session;
 	cstate->clp = clp;
 
@@ -5192,7 +5251,11 @@ nfsd4_sequence_done(struct nfsd4_compoundres *resp)
 	struct nfsd4_compound_state *cs = &resp->cstate;
 
 	if (nfsd4_has_session(cs)) {
-		if (cs->status != nfserr_replay_cache) {
+		/*
+		 * Only the request that claimed the slot may update its
+		 * cached reply and clear NFSD4_SLOT_INUSE.
+		 */
+		if (cs->slot_owned) {
 			nfsd4_store_cache_entry(resp);
 			cs->slot->sl_flags &= ~NFSD4_SLOT_INUSE;
 		}
@@ -5526,26 +5589,110 @@ nfsd4_init_slabs(void)
 	return -ENOMEM;
 }
 
+#define NFSD_RECALL_ANY_COOLDOWN_SECS	5
+
 static unsigned long
-nfsd4_state_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
+nfsd4_courtesy_shrinker_count(struct shrinker *shrink,
+			      struct shrink_control *sc)
 {
 	struct nfsd_net *nn = shrink->private_data;
-	long count;
+	long backlog, count;
 
 	count = atomic_read(&nn->nfsd_courtesy_clients);
 	if (!count)
-		count = atomic_long_read(&num_delegations);
-	if (count)
-		queue_work(laundry_wq, &nn->nfsd_shrinker_work);
-	return (unsigned long)count;
+		return 0;
+
+	queue_work(laundry_wq, &nn->nfsd_courtesy_work);
+
+	/* Work already queued is not available to reclaim again. */
+	backlog = atomic_long_read(&nn->nfsd_shrink_backlog);
+	return count > backlog ? count - backlog : 0;
+}
+
+static unsigned long
+nfsd4_deleg_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct nfsd_net *nn = shrink->private_data;
+	time64_t elapsed;
+	long backlog, count;
+
+	count = atomic_long_read(&nn->nfsd_delegations);
+	if (!count)
+		return 0;
+
+	/*
+	 * Delegations the last sweep reached stay unreclaimable until
+	 * deleg_reaper()'s cooldown expires. CB_RECALL_ANY leaves the
+	 * choice of delegations to the client, so there is no return
+	 * to wait on instead.
+	 */
+	elapsed = ktime_get_boottime_seconds() -
+			READ_ONCE(nn->nfsd_last_recall_any);
+	if (elapsed < NFSD_RECALL_ANY_COOLDOWN_SECS)
+		return 0;
+
+	/*
+	 * Unlike the courtesy shrinker, this one queues no work.
+	 * Nothing is recalled until a scan request arrives. Subtract
+	 * the requests already recorded, or concurrent reclaimers
+	 * each see the whole namespace and stack a scan on top of it.
+	 */
+	backlog = atomic_long_read(&nn->nfsd_deleg_backlog);
+	return count > backlog ? count - backlog : 0;
 }
 
 static unsigned long
-nfsd4_state_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
+nfsd4_courtesy_shrinker_scan(struct shrinker *shrink,
+			     struct shrink_control *sc)
 {
+	struct nfsd_net *nn = shrink->private_data;
+
+	atomic_long_add(sc->nr_to_scan, &nn->nfsd_shrink_backlog);
+	queue_work(laundry_wq, &nn->nfsd_courtesy_work);
+
+	/*
+	 * The reaper runs from laundry_wq. Report no progress rather
+	 * than claim memory that is not free yet.
+	 */
 	return SHRINK_STOP;
 }
 
+static unsigned long
+nfsd4_deleg_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct nfsd_net *nn = shrink->private_data;
+
+	atomic_long_add(sc->nr_to_scan, &nn->nfsd_deleg_backlog);
+	queue_work(laundry_wq, &nn->nfsd_deleg_work);
+
+	/*
+	 * The reaper sends CB_RECALL_ANY, so nothing is free when
+	 * this returns.
+	 */
+	return SHRINK_STOP;
+}
+
+static struct shrinker *
+nfsd4_alloc_state_shrinker(struct nfsd_net *nn, const char *name,
+			   unsigned long (*count)(struct shrinker *,
+						  struct shrink_control *),
+			   unsigned long (*scan)(struct shrinker *,
+						 struct shrink_control *))
+{
+	struct shrinker *shrink;
+
+	shrink = shrinker_alloc(0, "%s:%s", name, nn->nfsd_name);
+	if (!shrink)
+		return NULL;
+
+	shrink->count_objects = count;
+	shrink->scan_objects = scan;
+	shrink->private_data = nn;
+
+	shrinker_register(shrink);
+	return shrink;
+}
+
 void
 nfsd4_init_leases_net(struct nfsd_net *nn)
 {
@@ -6063,6 +6210,80 @@ bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode)
 	return timeo > 0;
 }
 
+/*
+ * gen_sessionid() composes a sessionid from the client's clientid and a
+ * sequence counter, so the sequence alone identifies the granting session.
+ */
+static void nfsd4_recall_grant_sessionid(const struct nfs4_delegation *dp,
+					 struct nfsd4_sessionid *sid)
+{
+	sid->clientid = dp->dl_stid.sc_client->cl_clientid;
+	sid->sequence = dp->dl_recall_grant.sessionid_seq;
+	sid->reserved = 0;
+}
+
+static bool nfsd4_recall_grant_slot_retired(struct nfs4_delegation *dp)
+{
+	struct nfs4_client *clp = dp->dl_stid.sc_client;
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+	struct nfsd4_session *ses;
+	struct nfsd4_sessionid sid;
+	bool retired = false;
+	void *entry;
+
+	if (!dp->dl_recall_grant.valid)
+		return false;
+
+	nfsd4_recall_grant_sessionid(dp, &sid);
+
+	/*
+	 * A missing session does not prove the client saw the grant: a
+	 * DESTROY_SESSION unhashes its own session before the reply to
+	 * that compound is encoded.
+	 */
+	spin_lock(&nn->client_lock);
+	ses = __find_in_sessionid_hashtbl((struct nfs4_sessionid *)&sid,
+					  clp->net);
+	entry = ses ? xa_load(&ses->se_slots, dp->dl_recall_grant.slotid) : NULL;
+	if (xa_is_value(entry)) {
+		/*
+		 * A slot is freed only once the client has acknowledged
+		 * the smaller slot table, which it cannot do while a
+		 * request on that slot is outstanding.
+		 */
+		retired = true;
+	} else if (entry) {
+		struct nfsd4_slot *slot = entry;
+
+		/*
+		 * A reactivated slot was freed and rebuilt, so the same
+		 * acknowledgment applies. The seqid test errs toward
+		 * revoking: a rebuilt slot restarting at seqid 1 matches
+		 * an old grant.
+		 */
+		retired = (slot->sl_flags & NFSD4_SLOT_REUSED) ||
+			  ((slot->sl_flags & NFSD4_SLOT_INITIALIZED) &&
+			   slot->sl_seqid != dp->dl_recall_grant.seqid);
+	}
+	spin_unlock(&nn->client_lock);
+	return retired;
+}
+
+/*
+ * ->prepare does not run on every send: nfsd4_run_cb_work() skips it
+ * on a requeue, and a retry via rpc_restart_call_prepare() re-enters
+ * the RPC layer beneath it. The granting request does not change, so
+ * a send inherits a correct list. Retirement is the one transition
+ * the list has to follow.
+ */
+static void nfsd4_refresh_recall_grant(struct nfs4_delegation *dp)
+{
+	dp->dl_recall_grant.retired_at_send =
+			nfsd4_recall_grant_slot_retired(dp);
+	if (dp->dl_recall_grant.retired_at_send)
+		nfsd41_cb_destroy_referring_call_list(&dp->dl_recall);
+}
+
 static bool nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
 {
 	struct nfs4_delegation *dp = cb_to_delegation(cb);
@@ -6084,9 +6305,46 @@ static bool nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
 		list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru);
 	}
 	spin_unlock(&nn->deleg_lock);
+
+	nfsd4_refresh_recall_grant(dp);
+
+	if (dp->dl_recall_grant.valid && !dp->dl_recall_grant.retired_at_send) {
+		struct nfsd4_sessionid sid;
+
+		nfsd4_recall_grant_sessionid(dp, &sid);
+		nfsd41_cb_referring_call(&dp->dl_recall,
+					 (struct nfs4_sessionid *)&sid,
+					 dp->dl_recall_grant.slotid,
+					 dp->dl_recall_grant.seqid);
+	}
 	return true;
 }
 
+/*
+ * cl_lock orders this against a laundromat reaping @dp: either
+ * revoke_delegation() observes dl_recall_rejected and destroys @dp, or
+ * it reached cl_revoked first and @dp is released here instead.
+ */
+static void nfsd4_deleg_recall_rejected(struct nfs4_delegation *dp)
+{
+	struct nfs4_client *clp = dp->dl_stid.sc_client;
+
+	spin_lock(&clp->cl_lock);
+	if (dp->dl_stid.sc_status & (SC_STATUS_CLOSED | SC_STATUS_FREED |
+				     SC_STATUS_ADMIN_REVOKED)) {
+		spin_unlock(&clp->cl_lock);
+		return;
+	}
+	if (dp->dl_stid.sc_status & SC_STATUS_FREEABLE) {
+		dp->dl_stid.sc_status |= SC_STATUS_CLOSED;
+		/* this function drops ->cl_lock */
+		nfsd4_drop_revoked_stid(&dp->dl_stid);
+		return;
+	}
+	dp->dl_recall_rejected = true;
+	spin_unlock(&clp->cl_lock);
+}
+
 static int nfsd4_cb_recall_done(struct nfsd4_callback *cb,
 		struct rpc_task *task)
 {
@@ -6094,27 +6352,32 @@ static int nfsd4_cb_recall_done(struct nfsd4_callback *cb,
 
 	trace_nfsd_cb_recall_done(&dp->dl_stid.sc_stateid, task);
 
-	if (dp->dl_stid.sc_status)
-		/* CLOSED or REVOKED */
-		return 1;
-
 	switch (task->tk_status) {
 	case 0:
 		return 1;
 	case -NFS4ERR_DELAY:
+		if (dp->dl_stid.sc_status)
+			/* CLOSED or REVOKED */
+			return 1;
 		rpc_delay(task, 2 * HZ);
 		return 0;
 	case -EBADHANDLE:
 	case -NFS4ERR_BAD_STATEID:
 		/*
-		 * Race: client probably got cb_recall before open reply
-		 * granting delegation.
+		 * Retirement of the granting slot proves the client saw
+		 * the grant. Trust the rejection only if the slot had
+		 * retired when this recall was sent.
 		 */
-		if (dp->dl_retries--) {
+		if (dp->dl_recall_grant.retired_at_send) {
+			nfsd4_deleg_recall_rejected(dp);
+			return 1;
+		}
+		if (!dp->dl_stid.sc_status && dp->dl_retries--) {
+			nfsd4_refresh_recall_grant(dp);
 			rpc_delay(task, 2 * HZ);
 			return 0;
 		}
-		fallthrough;
+		return 1;
 	default:
 		return 1;
 	}
@@ -6708,9 +6971,25 @@ static bool nfsd4_want_deleg_timestamps(const struct nfsd4_open *open)
 	return open->op_deleg_want & OPEN4_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS;
 }
 
+static void
+nfs4_delegation_record_grant_slot(struct nfs4_delegation *dp,
+				 const struct nfsd4_compound_state *cstate)
+{
+	const struct nfsd4_sessionid *sid;
+
+	if (!cstate->session)
+		return;
+	sid = (struct nfsd4_sessionid *)cstate->session->se_sessionid.data;
+	dp->dl_recall_grant.sessionid_seq = sid->sequence;
+	dp->dl_recall_grant.slotid = cstate->slot->sl_index;
+	dp->dl_recall_grant.seqid = cstate->slot->sl_seqid;
+	dp->dl_recall_grant.valid = true;
+}
+
 static struct nfs4_delegation *
-nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp,
-		    struct svc_fh *parent)
+nfs4_set_delegation(struct nfsd4_open *open,
+		    const struct nfsd4_compound_state *cstate,
+		    struct nfs4_ol_stateid *stp, struct svc_fh *parent)
 {
 	bool deleg_ts = nfsd4_want_deleg_timestamps(open);
 	struct nfs4_client *clp = stp->st_stid.sc_client;
@@ -6800,6 +7079,14 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp,
 	dp = alloc_init_deleg(clp, fp, odstate, dl_type);
 	if (!dp)
 		goto out_delegees;
+
+	/*
+	 * Record the granting slot before kernel_setlease() makes @dp
+	 * visible to lease breakers. A conflicting open can drive
+	 * CB_RECALL to completion from that point on.
+	 */
+	nfs4_delegation_record_grant_slot(dp, cstate);
+
 	if (stp->st_stid.sc_export)
 		dp->dl_stid.sc_export = exp_get(stp->st_stid.sc_export);
 
@@ -6964,6 +7251,7 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open,
 		     struct nfs4_ol_stateid *stp, struct svc_fh *currentfh,
 		     struct svc_fh *fh)
 {
+	struct nfsd4_compoundres *resp = rqstp->rq_resp;
 	struct nfs4_openowner *oo = openowner(stp->st_stateowner);
 	bool deleg_ts = nfsd4_want_deleg_timestamps(open);
 	struct nfs4_client *clp = stp->st_stid.sc_client;
@@ -7000,7 +7288,7 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open,
 		default:
 			goto out_no_deleg;
 	}
-	dp = nfs4_set_delegation(open, stp, parent);
+	dp = nfs4_set_delegation(open, &resp->cstate, stp, parent);
 	if (IS_ERR(dp))
 		goto out_no_deleg;
 
@@ -7624,6 +7912,7 @@ nfs4_laundromat(struct nfsd_net *nn)
 	struct nfs4_cpntf_state *cps;
 	struct nfs4_client *clp;
 	copy_stateid_t *cps_t;
+	long held, host, n;
 	int i;
 
 	if (clients_still_reclaiming(nn)) {
@@ -7737,8 +8026,22 @@ nfs4_laundromat(struct nfsd_net *nn)
 	/* service the server-to-server copy delayed unmount list */
 	nfsd4_ssc_expire_umount(nn);
 #endif
-	if (atomic_long_read(&num_delegations) >= max_delegations)
-		deleg_reaper(nn);
+	/*
+	 * set_max_delegations() computes a zero max_delegations on a
+	 * server with very little memory. @host is a divisor below.
+	 */
+	host = atomic_long_read(&num_delegations);
+	if (host && host >= max_delegations) {
+		/*
+		 * max_delegations bounds the host, but the laundromat
+		 * runs once per network namespace. Requesting the whole
+		 * overage in each would multiply the request, so take
+		 * only this namespace's share.
+		 */
+		held = atomic_long_read(&nn->nfsd_delegations);
+		n = host - max_delegations + 1;
+		deleg_reaper(nn, DIV64_U64_ROUND_UP((u64)n * held, host));
+	}
 out:
 	return max_t(time64_t, lt.new_timeo, NFSD_LAUNDROMAT_MINTIMEOUT);
 }
@@ -7766,49 +8069,146 @@ courtesy_client_reaper(struct nfsd_net *nn)
 	nfs4_process_client_reaplist(&reaplist);
 }
 
+/* The two passes in deleg_reaper() must agree on which clients are asked. */
+static bool
+deleg_reaper_eligible(const struct nfs4_client *clp, time64_t now)
+{
+	if (clp->cl_minorversion == 0)
+		return false;
+	if (clp->cl_state != NFSD4_ACTIVE)
+		return false;
+	if (atomic_read(&clp->cl_delegs_in_recall))
+		return false;
+	if (test_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags))
+		return false;
+	if (now - clp->cl_ra_time < NFSD_RECALL_ANY_COOLDOWN_SECS)
+		return false;
+	if (clp->cl_cb_state != NFSD4_CB_UP)
+		return false;
+	return true;
+}
+
 static void
-deleg_reaper(struct nfsd_net *nn)
+deleg_reaper(struct nfsd_net *nn, unsigned long backlog)
 {
 	struct list_head *pos, *next;
 	struct nfs4_client *clp;
+	unsigned long remaining, share, total;
+	unsigned int count;
+	time64_t now;
+
+	/*
+	 * Recalling a delegation before it is needed costs the client
+	 * an OPEN when it next touches the file. Leave
+	 * nfsd_last_recall_any unstamped so the next sweep is not
+	 * delayed.
+	 */
+	if (!backlog)
+		return;
+	now = ktime_get_boottime_seconds();
 
 	spin_lock(&nn->client_lock);
-	list_for_each_safe(pos, next, &nn->client_lru) {
+
+	/*
+	 * Only the clients this sweep asks contribute to the
+	 * apportionment. Dividing the request among holders that are
+	 * skipped under-serves it, and the shortfall goes nowhere:
+	 * nfsd4_deleg_shrinker_worker() has already cleared
+	 * nfsd_deleg_backlog.
+	 */
+	total = 0;
+	list_for_each(pos, &nn->client_lru) {
 		clp = list_entry(pos, struct nfs4_client, cl_lru);
 
-		if (clp->cl_state != NFSD4_ACTIVE)
+		if (!deleg_reaper_eligible(clp, now))
 			continue;
-		if (list_empty(&clp->cl_delegations))
-			continue;
-		if (atomic_read(&clp->cl_delegs_in_recall))
-			continue;
-		if (ktime_get_boottime_seconds() - clp->cl_ra_time < 5)
+		/*
+		 * This read races with hash_delegation_locked() and
+		 * unhash_delegation_locked() on other CPUs. A stale
+		 * count only skews the keep value; the next
+		 * laundromat pass sees a more current one.
+		 */
+		total += data_race(READ_ONCE(clp->cl_deleg_count));
+	}
+	if (!total)
+		goto out;
+
+	/*
+	 * Reclaim asks in batches and is not bound by what the count
+	 * callback reported, so the backlog can exceed what these
+	 * clients hold. Cap it to keep each share within the client's
+	 * own count.
+	 */
+	backlog = min(backlog, total);
+	remaining = backlog;
+
+	list_for_each_safe(pos, next, &nn->client_lru) {
+		clp = list_entry(pos, struct nfs4_client, cl_lru);
+
+		if (!deleg_reaper_eligible(clp, now))
 			continue;
-		if (clp->cl_cb_state != NFSD4_CB_UP)
+		count = data_race(READ_ONCE(clp->cl_deleg_count));
+		if (!count)
 			continue;
 		if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags))
 			continue;
 
 		/* release in nfsd4_cb_recall_any_release */
 		kref_get(&clp->cl_nfsdfs.cl_ref);
-		clp->cl_ra_time = ktime_get_boottime_seconds();
-		clp->cl_ra->ra_keep = 0;
+		clp->cl_ra_time = now;
+		/*
+		 * Rounding up guarantees every holder gives up at least
+		 * one. The round-up can overshoot @backlog, so stop
+		 * once the request is met. client_lru is ordered by
+		 * last renewal, so the least active clients are asked
+		 * first.
+		 */
+		share = DIV64_U64_ROUND_UP((u64)backlog * count, total);
+		share = min(share, remaining);
+		remaining -= share;
+		clp->cl_ra->ra_keep = count - share;
 		clp->cl_ra->ra_bmval[0] = BIT(RCA4_TYPE_MASK_RDATA_DLG) |
-						BIT(RCA4_TYPE_MASK_WDATA_DLG);
+						BIT(RCA4_TYPE_MASK_WDATA_DLG) |
+						BIT(RCA4_TYPE_MASK_DIR_DLG);
 		trace_nfsd_cb_recall_any(clp->cl_ra);
 		nfsd4_run_cb(&clp->cl_ra->ra_cb);
+		if (!remaining)
+			break;
 	}
+out:
 	spin_unlock(&nn->client_lock);
+
+	/*
+	 * Stamp the sweep even when no recall went out. A sweep that
+	 * found nothing eligible finds nothing on an immediate retry.
+	 */
+	WRITE_ONCE(nn->nfsd_last_recall_any, now);
 }
 
 static void
-nfsd4_state_shrinker_worker(struct work_struct *work)
+nfsd4_courtesy_shrinker_worker(struct work_struct *work)
 {
 	struct nfsd_net *nn = container_of(work, struct nfsd_net,
-				nfsd_shrinker_work);
+				nfsd_courtesy_work);
+	long backlog;
 
+	/*
+	 * Retire only the requests sampled here, so that requests
+	 * arriving while the reaper runs are still discounted by
+	 * nfsd4_courtesy_shrinker_count().
+	 */
+	backlog = atomic_long_read(&nn->nfsd_shrink_backlog);
 	courtesy_client_reaper(nn);
-	deleg_reaper(nn);
+	atomic_long_sub(backlog, &nn->nfsd_shrink_backlog);
+}
+
+static void
+nfsd4_deleg_shrinker_worker(struct work_struct *work)
+{
+	struct nfsd_net *nn = container_of(work, struct nfsd_net,
+				nfsd_deleg_work);
+
+	deleg_reaper(nn, atomic_long_xchg(&nn->nfsd_deleg_backlog, 0));
 }
 
 static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stid *stp)
@@ -9770,21 +10170,31 @@ static int nfs4_state_create_net(struct net *net)
 	INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main);
 	/* Make sure this cannot run until client tracking is initialised */
 	disable_delayed_work(&nn->laundromat_work);
-	INIT_WORK(&nn->nfsd_shrinker_work, nfsd4_state_shrinker_worker);
+	INIT_WORK(&nn->nfsd_courtesy_work, nfsd4_courtesy_shrinker_worker);
+	INIT_WORK(&nn->nfsd_deleg_work, nfsd4_deleg_shrinker_worker);
+	atomic_long_set(&nn->nfsd_shrink_backlog, 0);
+	atomic_long_set(&nn->nfsd_deleg_backlog, 0);
+	nn->nfsd_last_recall_any = 0;
 	get_net(net);
 
-	nn->nfsd_client_shrinker = shrinker_alloc(0, "nfsd-client");
-	if (!nn->nfsd_client_shrinker)
+	nn->nfsd_courtesy_shrinker =
+		nfsd4_alloc_state_shrinker(nn, "nfsd-courtesy",
+					   nfsd4_courtesy_shrinker_count,
+					   nfsd4_courtesy_shrinker_scan);
+	if (!nn->nfsd_courtesy_shrinker)
 		goto err_shrinker;
 
-	nn->nfsd_client_shrinker->scan_objects = nfsd4_state_shrinker_scan;
-	nn->nfsd_client_shrinker->count_objects = nfsd4_state_shrinker_count;
-	nn->nfsd_client_shrinker->private_data = nn;
-
-	shrinker_register(nn->nfsd_client_shrinker);
+	nn->nfsd_deleg_shrinker =
+		nfsd4_alloc_state_shrinker(nn, "nfsd-delegation",
+					   nfsd4_deleg_shrinker_count,
+					   nfsd4_deleg_shrinker_scan);
+	if (!nn->nfsd_deleg_shrinker)
+		goto err_deleg_shrinker;
 
 	return 0;
 
+err_deleg_shrinker:
+	shrinker_free(nn->nfsd_courtesy_shrinker);
 err_shrinker:
 	put_net(net);
 	kfree(nn->sessionid_hashtbl);
@@ -9885,8 +10295,10 @@ nfs4_state_shutdown_net(struct net *net)
 	struct list_head *pos, *next, reaplist;
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
 
-	shrinker_free(nn->nfsd_client_shrinker);
-	cancel_work_sync(&nn->nfsd_shrinker_work);
+	shrinker_free(nn->nfsd_courtesy_shrinker);
+	shrinker_free(nn->nfsd_deleg_shrinker);
+	cancel_work_sync(&nn->nfsd_courtesy_work);
+	cancel_work_sync(&nn->nfsd_deleg_work);
 	disable_delayed_work_sync(&nn->laundromat_work);
 	locks_end_grace(&nn->nfsd4_manager);
 
@@ -10294,6 +10706,7 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate,
 	dp = alloc_init_dir_deleg(clp, fp);
 	if (!dp)
 		goto out_delegees;
+	nfs4_delegation_record_grant_slot(dp, cstate);
 	if (cstate->current_fh.fh_export)
 		dp->dl_stid.sc_export =
 			exp_get(cstate->current_fh.fh_export);
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 606ddcb085c027..00ddaac499c68b 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -54,6 +54,7 @@
 #include "xdr4.h"
 #include "vfs.h"
 #include "state.h"
+#include "nfserr.h"
 #include "cache.h"
 #include "netns.h"
 #include "pnfs.h"
@@ -113,27 +114,27 @@ static int zero_clientid(clientid_t *clid)
 	return (clid->cl_boot == 0) && (clid->cl_id == 0);
 }
 
-/**
- * svcxdr_tmpalloc - allocate memory to be freed after compound processing
- * @argp: NFSv4 compound argument structure
- * @len: length of buffer to allocate
- *
- * Allocates a buffer of size @len to be freed when processing the compound
- * operation described in @argp finishes.
- */
 static void *
-svcxdr_tmpalloc(struct nfsd4_compoundargs *argp, size_t len)
+svcxdr_tmpalloc_release(struct nfsd4_compoundargs *argp, size_t len,
+			void (*release)(void *))
 {
 	struct svcxdr_tmpbuf *tb;
 
 	tb = kmalloc_flex(*tb, buf, len);
 	if (!tb)
 		return NULL;
+	tb->release = release;
 	tb->next = argp->to_free;
 	argp->to_free = tb;
 	return tb->buf;
 }
 
+static void *
+svcxdr_tmpalloc(struct nfsd4_compoundargs *argp, size_t len)
+{
+	return svcxdr_tmpalloc_release(argp, len, NULL);
+}
+
 /*
  * For xdr strings that need to be passed to other kernel api's
  * as null-terminated strings.
@@ -441,10 +442,16 @@ nfsd4_decode_posixace4(struct nfsd4_compoundargs *argp,
 	return status;
 }
 
+static void svcxdr_release_pacl(void *p)
+{
+	posix_acl_release(*(struct posix_acl **)p);
+}
+
 static noinline __be32
 nfsd4_decode_posixacl(struct nfsd4_compoundargs *argp, struct posix_acl **acl)
 {
 	struct posix_acl_entry *ace;
+	struct posix_acl **slot;
 	__be32 status;
 	u32 count;
 
@@ -484,6 +491,15 @@ nfsd4_decode_posixacl(struct nfsd4_compoundargs *argp, struct posix_acl **acl)
 	if (count >= 3)
 		sort_pacl_range(*acl, 0, count - 1);
 
+	slot = svcxdr_tmpalloc_release(argp, sizeof(*slot),
+				       svcxdr_release_pacl);
+	if (!slot) {
+		posix_acl_release(*acl);
+		*acl = NULL;
+		return nfserr_jukebox;
+	}
+	*slot = *acl;
+
 	return nfs_ok;
 }
 
@@ -676,7 +692,6 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen,
 
 		status = nfsd4_decode_posixacl(argp, &pacl);
 		if (status) {
-			posix_acl_release(*dpaclp);
 			*dpaclp = NULL;
 			return status;
 		}
@@ -686,12 +701,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen,
 
 	/* request sanity: did attrlist4 contain the expected number of words? */
 	if (attrlist4_count != xdr_stream_pos(argp->xdr) - starting_pos) {
-#ifdef CONFIG_NFSD_V4_POSIX_ACLS
-		posix_acl_release(*dpaclp);
-		posix_acl_release(*paclp);
 		*dpaclp = NULL;
 		*paclp = NULL;
-#endif
 		return nfserr_bad_xdr;
 	}
 
@@ -1606,7 +1617,7 @@ nfsd4_decode_write(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u)
 		return nfserr_bad_xdr;
 	if (xdr_stream_decode_u32(argp->xdr, &write->wr_stable_how) < 0)
 		return nfserr_bad_xdr;
-	if (write->wr_stable_how > NFS_FILE_SYNC)
+	if (write->wr_stable_how > FILE_SYNC4)
 		return nfserr_bad_xdr;
 	if (xdr_stream_decode_u32(argp->xdr, &write->wr_buflen) < 0)
 		return nfserr_bad_xdr;
@@ -1919,6 +1930,17 @@ nfsd4_decode_get_dir_delegation(struct nfsd4_compoundargs *argp,
 }
 
 #ifdef CONFIG_NFSD_PNFS
+static __be32
+nfsd4_decode_deviceid4(struct xdr_stream *xdr, struct nfsd4_deviceid *devid)
+{
+	__be32 *p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE);
+
+	if (unlikely(!p))
+		return nfserr_bad_xdr;
+	svcxdr_decode_deviceid4(p, devid);
+	return nfs_ok;
+}
+
 static __be32
 nfsd4_decode_getdeviceinfo(struct nfsd4_compoundargs *argp,
 		union nfsd4_op_u *u)
@@ -2733,6 +2755,87 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
 	return true;
 }
 
+static __always_inline __be32
+nfsd4_encode_bool(struct xdr_stream *xdr, bool val)
+{
+	__be32 *p = xdr_reserve_space(xdr, XDR_UNIT);
+
+	if (unlikely(p == NULL))
+		return nfserr_resource;
+	*p = val ? xdr_one : xdr_zero;
+	return nfs_ok;
+}
+
+static __always_inline __be32
+nfsd4_encode_uint32_t(struct xdr_stream *xdr, u32 val)
+{
+	__be32 *p = xdr_reserve_space(xdr, XDR_UNIT);
+
+	if (unlikely(p == NULL))
+		return nfserr_resource;
+	*p = cpu_to_be32(val);
+	return nfs_ok;
+}
+
+#define nfsd4_encode_aceflag4(x, v)	nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_acemask4(x, v)	nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_acetype4(x, v)	nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_count4(x, v)	nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_mode4(x, v)	nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_nfs_lease4(x, v)	nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_qop4(x, v)		nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_sequenceid4(x, v)	nfsd4_encode_uint32_t(x, v)
+#define nfsd4_encode_slotid4(x, v)	nfsd4_encode_uint32_t(x, v)
+
+static __always_inline __be32
+nfsd4_encode_uint64_t(struct xdr_stream *xdr, u64 val)
+{
+	__be32 *p = xdr_reserve_space(xdr, XDR_UNIT * 2);
+
+	if (unlikely(p == NULL))
+		return nfserr_resource;
+	put_unaligned_be64(val, p);
+	return nfs_ok;
+}
+
+#define nfsd4_encode_changeid4(x, v)	nfsd4_encode_uint64_t(x, v)
+#define nfsd4_encode_nfs_cookie4(x, v)	nfsd4_encode_uint64_t(x, v)
+#define nfsd4_encode_length4(x, v)	nfsd4_encode_uint64_t(x, v)
+#define nfsd4_encode_offset4(x, v)	nfsd4_encode_uint64_t(x, v)
+
+static __always_inline __be32
+nfsd4_encode_opaque_fixed(struct xdr_stream *xdr, const void *data,
+			  size_t size)
+{
+	__be32 *p = xdr_reserve_space(xdr, xdr_align_size(size));
+	size_t pad = xdr_pad_size(size);
+
+	if (unlikely(p == NULL))
+		return nfserr_resource;
+	memcpy(p, data, size);
+	if (pad)
+		memset((char *)p + size, 0, pad);
+	return nfs_ok;
+}
+
+static __always_inline __be32
+nfsd4_encode_opaque(struct xdr_stream *xdr, const void *data, size_t size)
+{
+	size_t pad = xdr_pad_size(size);
+	__be32 *p;
+
+	p = xdr_reserve_space(xdr, XDR_UNIT + xdr_align_size(size));
+	if (unlikely(p == NULL))
+		return nfserr_resource;
+	*p++ = cpu_to_be32(size);
+	memcpy(p, data, size);
+	if (pad)
+		memset((char *)p + size, 0, pad);
+	return nfs_ok;
+}
+
+#define nfsd4_encode_component4(x, d, s)	nfsd4_encode_opaque(x, d, s)
+
 static __be32 nfsd4_encode_nfs_fh4(struct xdr_stream *xdr,
 				   const struct knfsd_fh *fh_handle)
 {
@@ -3110,7 +3213,7 @@ static __be32 fattr_handle_absent_fs(u32 *bmval0, u32 *bmval1, u32 *bmval2, u32
 	    *bmval1 & ~WORD1_ABSENT_FS_ATTRS) {
 		if (*bmval0 & FATTR4_WORD0_RDATTR_ERROR ||
 	            *bmval0 & FATTR4_WORD0_FS_LOCATIONS)
-			*rdattr_err = NFSERR_MOVED;
+			*rdattr_err = NFS4ERR_MOVED;
 		else
 			return nfserr_moved;
 	}
@@ -4285,21 +4388,16 @@ setup_notify_fhandle(struct dentry *dentry, struct nfs4_delegation *dp,
 static bool
 nfsd4_setup_notify_entry4(struct notify_entry4 *ne, struct xdr_stream *xdr,
 			  struct dentry *dentry, struct nfs4_delegation *dp,
-			  struct nfsd_file *nf, char *name, u32 namelen)
+			  struct nfsd_file *nf, char *name, u32 namelen,
+			  u32 *attrmask)
 {
 	struct path path = nf->nf_file->f_path;
 	struct nfsd4_fattr_args args = { };
 	const u32 *reqmask;
-	uint32_t *attrmask;
 	__be32 status;
 	bool parent;
 	int ret;
 
-	/* Reserve space for attrmask */
-	attrmask = xdr_reserve_space(xdr, 3 * sizeof(uint32_t));
-	if (!attrmask)
-		return false;
-
 	ne->ne_file.data = name;
 	ne->ne_file.len = namelen;
 	ne->ne_attrs.attrmask.element = attrmask;
@@ -4383,6 +4481,7 @@ u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *
 			      struct nfs4_delegation *dp, struct nfsd_file *nf,
 			      u32 *notify_mask)
 {
+	u32 attrmask[3][3] = { };
 	u8 *p = NULL;
 
 	*notify_mask = 0;
@@ -4391,7 +4490,8 @@ u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *
 		struct notify_remove4 nr = { };
 
 		if (!nfsd4_setup_notify_entry4(&nr.nrm_old_entry, xdr, nne->ne_dentry, dp,
-					       nf, nne->ne_name, nne->ne_namelen))
+					       nf, nne->ne_name, nne->ne_namelen,
+					       attrmask[0]))
 			goto out_err;
 		p = (u8 *)xdr->p;
 		if (!xdrgen_encode_notify_remove4(xdr, &nr))
@@ -4402,14 +4502,16 @@ u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *
 		struct notify_remove4 old = { };
 
 		if (!nfsd4_setup_notify_entry4(&na.nad_new_entry, xdr, nne->ne_dentry, dp,
-					       nf, nne->ne_name, nne->ne_namelen))
+					       nf, nne->ne_name, nne->ne_namelen,
+					       attrmask[0]))
 			goto out_err;
 
 		/* If a file was overwritten, report it in nad_old_entry */
 		if (nne->ne_target) {
 			if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
 						       NULL, dp, nf,
-						       nne->ne_name, nne->ne_namelen))
+						       nne->ne_name, nne->ne_namelen,
+						       attrmask[1]))
 				goto out_err;
 			na.nad_old_entry.count = 1;
 			na.nad_old_entry.element = &old;
@@ -4428,19 +4530,19 @@ u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *
 		/* Don't send any attributes in the old_entry since they're the same in new */
 		if (!nfsd4_setup_notify_entry4(&nr.nrn_old_entry.nrm_old_entry, xdr,
 					       NULL, dp, nf, nne->ne_name,
-					       nne->ne_namelen))
+					       nne->ne_namelen, attrmask[0]))
 			goto out_err;
 
 		if (!nfsd4_setup_notify_entry4(&nr.nrn_new_entry.nad_new_entry, xdr,
 					       nne->ne_dentry, dp, nf, newname,
-					       nne->ne_newnamelen))
+					       nne->ne_newnamelen, attrmask[1]))
 			goto out_err;
 
 		/* If a file was overwritten, report it in nad_old_entry */
 		if (nne->ne_target) {
 			if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr,
 						       NULL, dp, nf, newname,
-						       nne->ne_newnamelen))
+						       nne->ne_newnamelen, attrmask[2]))
 				goto out_err;
 			nr.nrn_new_entry.nad_old_entry.count = 1;
 			nr.nrn_new_entry.nad_old_entry.element = &old;
@@ -4476,11 +4578,12 @@ u8 *nfsd4_encode_dir_attr_change(struct xdr_stream *xdr, struct nfs4_delegation
 {
 	struct dentry *dentry = nf->nf_file->f_path.dentry;
 	struct notify_attr4 na = { };
+	u32 attrmask[3] = { };
 	u8 *p;
 
 	/* RFC 8881 s10.4.3: ne_file must be a zero-length string for dir attrs */
 	if (!nfsd4_setup_notify_entry4(&na.na_changed_entry, xdr,
-				       dentry, dp, nf, "", 0))
+				       dentry, dp, nf, "", 0, attrmask))
 		return ERR_PTR(-ENOBUFS);
 
 	/* No requested attributes to report; omit the event */
@@ -4574,8 +4677,6 @@ nfsd4_encode_entry4_fattr(struct nfsd4_readdir *cd, const char *name,
 	 * directly from the mountpoint dentry.
 	 */
 	if (nfsd_mountpoint(dentry, exp)) {
-		int err;
-
 		if (!(exp->ex_flags & NFSEXP_V4ROOT)
 				&& !attributes_need_mount(cd->rd_bmval)) {
 			ignore_crossmnt = 1;
@@ -4586,12 +4687,7 @@ nfsd4_encode_entry4_fattr(struct nfsd4_readdir *cd, const char *name,
 		 * Different "."/".." handling?  Something else?
 		 * At least, add a comment here to explain....
 		 */
-		err = nfsd_cross_mnt(cd->rd_rqstp, &dentry, &exp);
-		if (err) {
-			nfserr = nfserrno(err);
-			goto out_put;
-		}
-		nfserr = check_nfsd_access(exp, cd->rd_rqstp, false);
+		nfserr = nfsd_cross_mnt(cd->rd_rqstp, &dentry, &exp);
 		if (nfserr)
 			goto out_put;
 		crossed = true;
@@ -6637,11 +6733,20 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op)
 	unsigned int op_status_offset;
 	nfsd4_enc encoder;
 
-	if (xdr_stream_encode_u32(xdr, op->opnum) != XDR_UNIT)
+	/*
+	 * nfsd4_proc_compound() stops the COMPOUND early only
+	 * when op->status is set, so a header that cannot be
+	 * encoded has to report the failure here.
+	 */
+	if (xdr_stream_encode_u32(xdr, op->opnum) != XDR_UNIT) {
+		op->status = nfsd4_check_resp_size(resp, XDR_UNIT * 2);
 		goto release;
+	}
 	op_status_offset = xdr->buf->len;
-	if (!xdr_reserve_space(xdr, XDR_UNIT))
+	if (!xdr_reserve_space(xdr, XDR_UNIT)) {
+		op->status = nfsd4_check_resp_size(resp, XDR_UNIT);
 		goto release;
+	}
 
 	if (op->opnum == OP_ILLEGAL)
 		goto status;
@@ -6751,7 +6856,10 @@ void nfsd4_release_compoundargs(struct svc_rqst *rqstp)
 	}
 	while (args->to_free) {
 		struct svcxdr_tmpbuf *tb = args->to_free;
+
 		args->to_free = tb->next;
+		if (tb->release)
+			tb->release(tb->buf);
 		kfree(tb);
 	}
 }
diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c
index c7db532c852376..80364b91331a2a 100644
--- a/fs/nfsd/nfscache.c
+++ b/fs/nfsd/nfscache.c
@@ -19,6 +19,7 @@
 #include <net/checksum.h>
 
 #include "nfsd.h"
+#include "nfserr.h"
 #include "netns.h"
 #include "stats.h"
 #include "cache.h"
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 5abb2d4274c98e..3a6945b3bd8b27 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -20,9 +20,12 @@
 #include <linux/module.h>
 #include <linux/fsnotify.h>
 #include <linux/nfslocalio.h>
+#include <linux/nfs3.h>
 
 #include "idmap.h"
 #include "nfsd.h"
+#include "nfserr.h"
+#include "nfs4ctl.h"
 #include "netns.h"
 #include "stats.h"
 #include "cache.h"
@@ -477,7 +480,7 @@ static ssize_t write_pool_threads(struct file *file, char *buf, size_t size)
 	char *mesg = buf;
 	int i;
 	int rv;
-	int len;
+	size_t len;
 	int npools;
 	int *nthreads;
 	struct net *net = netns(file);
@@ -531,9 +534,13 @@ static ssize_t write_pool_threads(struct file *file, char *buf, size_t size)
 
 	mesg = buf;
 	size = SIMPLE_TRANSACTION_LIMIT;
-	for (i = 0; i < npools && size > 0; i++) {
-		snprintf(mesg, size, "%d%c", nthreads[i], (i == npools-1 ? '\n' : ' '));
-		len = strlen(mesg);
+	for (i = 0; i < npools; i++) {
+		len = snprintf(mesg, size, "%d%c", nthreads[i],
+			       (i == npools - 1 ? '\n' : ' '));
+		if (len >= size) {
+			rv = -ENAMETOOLONG;
+			goto out_free;
+		}
 		size -= len;
 		mesg += len;
 	}
@@ -1581,14 +1588,29 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
 			    rqstp->rq_proc == NFSPROC4_COMPOUND) {
 				/* NFSv4 compound */
 				struct nfsd4_compoundargs *args;
+				struct nfsd4_op *ops;
+				u32 opcnt;
 				int j;
 
 				args = rqstp->rq_argp;
-				genl_rqstp.rq_opcnt = min_t(u32, args->opcnt,
+				opcnt = READ_ONCE(args->opcnt);
+				ops = READ_ONCE(args->ops);
+
+				/*
+				 * Finish the seqcount retry before
+				 * dereferencing ops. An unchanged counter means
+				 * opcnt and ops came from the same COMPOUND,
+				 * where opcnt cannot exceed what ops holds.
+				 */
+				smp_rmb();
+				if (READ_ONCE(rqstp->rq_status_counter) !=
+				    status_counter)
+					continue;
+
+				genl_rqstp.rq_opcnt = min_t(u32, opcnt,
 							    ARRAY_SIZE(genl_rqstp.rq_opnum));
 				for (j = 0; j < genl_rqstp.rq_opcnt; j++)
-					genl_rqstp.rq_opnum[j] =
-						args->ops[j].opnum;
+					genl_rqstp.rq_opnum[j] = ops[j].opnum;
 			}
 #endif /* CONFIG_NFSD_V4 */
 
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index 76a69d9a4e7332..a145294c59c87c 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -14,8 +14,6 @@
 
 #include <linux/nfs.h>
 #include <linux/nfs2.h>
-#include <linux/nfs3.h>
-#include <linux/nfs4.h>
 #include <linux/sunrpc/svc.h>
 #include <linux/sunrpc/svc_xprt.h>
 
@@ -147,49 +145,7 @@ extern u64 nfsd_io_cache_write __read_mostly;
 
 extern int nfsd_max_blksize;
 
-static inline int nfsd_v4client(struct svc_rqst *rq)
-{
-	return rq && rq->rq_prog == NFS_PROGRAM && rq->rq_vers == 4;
-}
-
-/* 
- * NFSv4 State
- */
-#ifdef CONFIG_NFSD_V4
-extern unsigned long max_delegations;
-int nfsd4_init_slabs(void);
-void nfsd4_free_slabs(void);
-int nfs4_state_start(void);
-int nfs4_state_start_net(struct net *net);
-void nfs4_state_shutdown(void);
-void nfs4_state_shutdown_net(struct net *net);
-int nfs4_reset_recoverydir(char *recdir);
-char * nfs4_recoverydir(void);
-bool nfsd4_spo_must_allow(struct svc_rqst *rqstp);
-int nfsd4_create_laundry_wq(void);
-void nfsd4_destroy_laundry_wq(void);
-bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode);
-#else
-static inline int nfsd4_init_slabs(void) { return 0; }
-static inline void nfsd4_free_slabs(void) { }
-static inline int nfs4_state_start(void) { return 0; }
-static inline int nfs4_state_start_net(struct net *net) { return 0; }
-static inline void nfs4_state_shutdown(void) { }
-static inline void nfs4_state_shutdown_net(struct net *net) { }
-static inline int nfs4_reset_recoverydir(char *recdir) { return 0; }
-static inline char * nfs4_recoverydir(void) {return NULL; }
-static inline bool nfsd4_spo_must_allow(struct svc_rqst *rqstp)
-{
-	return false;
-}
-static inline int nfsd4_create_laundry_wq(void) { return 0; };
-static inline void nfsd4_destroy_laundry_wq(void) {};
-static inline bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp,
-					      struct inode *inode)
-{
-	return false;
-}
-#endif
+bool nfsd_v4client(struct svc_rqst *rqstp);
 
 /*
  * lockd binding
@@ -198,195 +154,4 @@ void		nfsd_lockd_init(void);
 void		nfsd_lockd_shutdown(void);
 
 
-/*
- * These macros provide pre-xdr'ed values for faster operation.
- */
-#define	nfs_ok			cpu_to_be32(NFS_OK)
-#define	nfserr_perm		cpu_to_be32(NFSERR_PERM)
-#define	nfserr_noent		cpu_to_be32(NFSERR_NOENT)
-#define	nfserr_io		cpu_to_be32(NFSERR_IO)
-#define	nfserr_nxio		cpu_to_be32(NFSERR_NXIO)
-#define	nfserr_acces		cpu_to_be32(NFSERR_ACCES)
-#define	nfserr_exist		cpu_to_be32(NFSERR_EXIST)
-#define	nfserr_xdev		cpu_to_be32(NFSERR_XDEV)
-#define	nfserr_nodev		cpu_to_be32(NFSERR_NODEV)
-#define	nfserr_notdir		cpu_to_be32(NFSERR_NOTDIR)
-#define	nfserr_isdir		cpu_to_be32(NFSERR_ISDIR)
-#define	nfserr_inval		cpu_to_be32(NFSERR_INVAL)
-#define	nfserr_fbig		cpu_to_be32(NFSERR_FBIG)
-#define	nfserr_nospc		cpu_to_be32(NFSERR_NOSPC)
-#define	nfserr_rofs		cpu_to_be32(NFSERR_ROFS)
-#define	nfserr_mlink		cpu_to_be32(NFSERR_MLINK)
-#define	nfserr_nametoolong	cpu_to_be32(NFSERR_NAMETOOLONG)
-#define	nfserr_notempty		cpu_to_be32(NFSERR_NOTEMPTY)
-#define	nfserr_dquot		cpu_to_be32(NFSERR_DQUOT)
-#define	nfserr_stale		cpu_to_be32(NFSERR_STALE)
-#define	nfserr_remote		cpu_to_be32(NFSERR_REMOTE)
-#define	nfserr_wflush		cpu_to_be32(NFSERR_WFLUSH)
-#define	nfserr_badhandle	cpu_to_be32(NFSERR_BADHANDLE)
-#define	nfserr_notsync		cpu_to_be32(NFSERR_NOT_SYNC)
-#define	nfserr_badcookie	cpu_to_be32(NFSERR_BAD_COOKIE)
-#define	nfserr_notsupp		cpu_to_be32(NFSERR_NOTSUPP)
-#define	nfserr_toosmall		cpu_to_be32(NFSERR_TOOSMALL)
-#define	nfserr_serverfault	cpu_to_be32(NFSERR_SERVERFAULT)
-#define	nfserr_badtype		cpu_to_be32(NFSERR_BADTYPE)
-#define	nfserr_jukebox		cpu_to_be32(NFSERR_JUKEBOX)
-#define	nfserr_denied		cpu_to_be32(NFSERR_DENIED)
-#define	nfserr_deadlock		cpu_to_be32(NFSERR_DEADLOCK)
-#define nfserr_expired          cpu_to_be32(NFSERR_EXPIRED)
-#define	nfserr_bad_cookie	cpu_to_be32(NFSERR_BAD_COOKIE)
-#define	nfserr_same		cpu_to_be32(NFSERR_SAME)
-#define	nfserr_clid_inuse	cpu_to_be32(NFSERR_CLID_INUSE)
-#define	nfserr_stale_clientid	cpu_to_be32(NFSERR_STALE_CLIENTID)
-#define	nfserr_resource		cpu_to_be32(NFSERR_RESOURCE)
-#define	nfserr_moved		cpu_to_be32(NFSERR_MOVED)
-#define	nfserr_nofilehandle	cpu_to_be32(NFSERR_NOFILEHANDLE)
-#define	nfserr_minor_vers_mismatch	cpu_to_be32(NFSERR_MINOR_VERS_MISMATCH)
-#define nfserr_share_denied	cpu_to_be32(NFSERR_SHARE_DENIED)
-#define nfserr_stale_stateid	cpu_to_be32(NFSERR_STALE_STATEID)
-#define nfserr_old_stateid	cpu_to_be32(NFSERR_OLD_STATEID)
-#define nfserr_bad_stateid	cpu_to_be32(NFSERR_BAD_STATEID)
-#define nfserr_bad_seqid	cpu_to_be32(NFSERR_BAD_SEQID)
-#define	nfserr_symlink		cpu_to_be32(NFSERR_SYMLINK)
-#define	nfserr_not_same		cpu_to_be32(NFSERR_NOT_SAME)
-#define nfserr_lock_range	cpu_to_be32(NFSERR_LOCK_RANGE)
-#define	nfserr_restorefh	cpu_to_be32(NFSERR_RESTOREFH)
-#define	nfserr_attrnotsupp	cpu_to_be32(NFSERR_ATTRNOTSUPP)
-#define	nfserr_bad_xdr		cpu_to_be32(NFSERR_BAD_XDR)
-#define	nfserr_openmode		cpu_to_be32(NFSERR_OPENMODE)
-#define	nfserr_badowner		cpu_to_be32(NFSERR_BADOWNER)
-#define	nfserr_locks_held	cpu_to_be32(NFSERR_LOCKS_HELD)
-#define	nfserr_op_illegal	cpu_to_be32(NFSERR_OP_ILLEGAL)
-#define	nfserr_grace		cpu_to_be32(NFSERR_GRACE)
-#define	nfserr_no_grace		cpu_to_be32(NFSERR_NO_GRACE)
-#define	nfserr_reclaim_bad	cpu_to_be32(NFSERR_RECLAIM_BAD)
-#define	nfserr_badname		cpu_to_be32(NFSERR_BADNAME)
-#define	nfserr_admin_revoked	cpu_to_be32(NFS4ERR_ADMIN_REVOKED)
-#define	nfserr_cb_path_down	cpu_to_be32(NFSERR_CB_PATH_DOWN)
-#define	nfserr_locked		cpu_to_be32(NFSERR_LOCKED)
-#define	nfserr_wrongsec		cpu_to_be32(NFSERR_WRONGSEC)
-#define nfserr_delay			cpu_to_be32(NFS4ERR_DELAY)
-#define nfserr_badiomode		cpu_to_be32(NFS4ERR_BADIOMODE)
-#define nfserr_badlayout		cpu_to_be32(NFS4ERR_BADLAYOUT)
-#define nfserr_bad_session_digest	cpu_to_be32(NFS4ERR_BAD_SESSION_DIGEST)
-#define nfserr_badsession		cpu_to_be32(NFS4ERR_BADSESSION)
-#define nfserr_badslot			cpu_to_be32(NFS4ERR_BADSLOT)
-#define nfserr_complete_already		cpu_to_be32(NFS4ERR_COMPLETE_ALREADY)
-#define nfserr_conn_not_bound_to_session cpu_to_be32(NFS4ERR_CONN_NOT_BOUND_TO_SESSION)
-#define nfserr_deleg_already_wanted	cpu_to_be32(NFS4ERR_DELEG_ALREADY_WANTED)
-#define nfserr_back_chan_busy		cpu_to_be32(NFS4ERR_BACK_CHAN_BUSY)
-#define nfserr_layouttrylater		cpu_to_be32(NFS4ERR_LAYOUTTRYLATER)
-#define nfserr_layoutunavailable	cpu_to_be32(NFS4ERR_LAYOUTUNAVAILABLE)
-#define nfserr_nomatching_layout	cpu_to_be32(NFS4ERR_NOMATCHING_LAYOUT)
-#define nfserr_recallconflict		cpu_to_be32(NFS4ERR_RECALLCONFLICT)
-#define nfserr_unknown_layouttype	cpu_to_be32(NFS4ERR_UNKNOWN_LAYOUTTYPE)
-#define nfserr_seq_misordered		cpu_to_be32(NFS4ERR_SEQ_MISORDERED)
-#define nfserr_sequence_pos		cpu_to_be32(NFS4ERR_SEQUENCE_POS)
-#define nfserr_req_too_big		cpu_to_be32(NFS4ERR_REQ_TOO_BIG)
-#define nfserr_rep_too_big		cpu_to_be32(NFS4ERR_REP_TOO_BIG)
-#define nfserr_rep_too_big_to_cache	cpu_to_be32(NFS4ERR_REP_TOO_BIG_TO_CACHE)
-#define nfserr_retry_uncached_rep	cpu_to_be32(NFS4ERR_RETRY_UNCACHED_REP)
-#define nfserr_unsafe_compound		cpu_to_be32(NFS4ERR_UNSAFE_COMPOUND)
-#define nfserr_too_many_ops		cpu_to_be32(NFS4ERR_TOO_MANY_OPS)
-#define nfserr_op_not_in_session	cpu_to_be32(NFS4ERR_OP_NOT_IN_SESSION)
-#define nfserr_hash_alg_unsupp		cpu_to_be32(NFS4ERR_HASH_ALG_UNSUPP)
-#define nfserr_clientid_busy		cpu_to_be32(NFS4ERR_CLIENTID_BUSY)
-#define nfserr_pnfs_io_hole		cpu_to_be32(NFS4ERR_PNFS_IO_HOLE)
-#define nfserr_seq_false_retry		cpu_to_be32(NFS4ERR_SEQ_FALSE_RETRY)
-#define nfserr_bad_high_slot		cpu_to_be32(NFS4ERR_BAD_HIGH_SLOT)
-#define nfserr_deadsession		cpu_to_be32(NFS4ERR_DEADSESSION)
-#define nfserr_encr_alg_unsupp		cpu_to_be32(NFS4ERR_ENCR_ALG_UNSUPP)
-#define nfserr_pnfs_no_layout		cpu_to_be32(NFS4ERR_PNFS_NO_LAYOUT)
-#define nfserr_not_only_op		cpu_to_be32(NFS4ERR_NOT_ONLY_OP)
-#define nfserr_wrong_cred		cpu_to_be32(NFS4ERR_WRONG_CRED)
-#define nfserr_wrong_type		cpu_to_be32(NFS4ERR_WRONG_TYPE)
-#define nfserr_dirdeleg_unavail		cpu_to_be32(NFS4ERR_DIRDELEG_UNAVAIL)
-#define nfserr_reject_deleg		cpu_to_be32(NFS4ERR_REJECT_DELEG)
-#define nfserr_returnconflict		cpu_to_be32(NFS4ERR_RETURNCONFLICT)
-#define nfserr_deleg_revoked		cpu_to_be32(NFS4ERR_DELEG_REVOKED)
-#define nfserr_partner_notsupp		cpu_to_be32(NFS4ERR_PARTNER_NOTSUPP)
-#define nfserr_partner_no_auth		cpu_to_be32(NFS4ERR_PARTNER_NO_AUTH)
-#define nfserr_union_notsupp		cpu_to_be32(NFS4ERR_UNION_NOTSUPP)
-#define nfserr_offload_denied		cpu_to_be32(NFS4ERR_OFFLOAD_DENIED)
-#define nfserr_wrong_lfs		cpu_to_be32(NFS4ERR_WRONG_LFS)
-#define nfserr_badlabel			cpu_to_be32(NFS4ERR_BADLABEL)
-#define nfserr_file_open		cpu_to_be32(NFS4ERR_FILE_OPEN)
-#define nfserr_xattr2big		cpu_to_be32(NFS4ERR_XATTR2BIG)
-#define nfserr_noxattr			cpu_to_be32(NFS4ERR_NOXATTR)
-
-/*
- * Error codes for internal use.  These are based at an impossible
- * nfsstat4 value so that, once converted to be32, they cannot conflict
- * with any value defined by the protocol (compare the nlm__int__* codes
- * in fs/lockd/lockd.h).
- */
-enum {
-/* end-of-file indicator in readdir */
-	NFSERR_EOF = 30000,
-#define	nfserr_eof		cpu_to_be32(NFSERR_EOF)
-
-/* replay detected */
-	NFSERR_REPLAY_ME,
-#define	nfserr_replay_me	cpu_to_be32(NFSERR_REPLAY_ME)
-
-/* nfs41 replay detected */
-	NFSERR_REPLAY_CACHE,
-#define	nfserr_replay_cache	cpu_to_be32(NFSERR_REPLAY_CACHE)
-
-/* symlink found where dir expected - handled differently to
- * other symlink found errors by NFSv3.
- */
-	NFSERR_SYMLINK_NOT_DIR,
-#define	nfserr_symlink_not_dir	cpu_to_be32(NFSERR_SYMLINK_NOT_DIR)
-};
-
-#ifdef CONFIG_NFSD_V4
-
-/* before processing a COMPOUND operation, we have to check that there
- * is enough space in the buffer for XDR encode to succeed.  otherwise,
- * we might process an operation with side effects, and be unable to
- * tell the client that the operation succeeded.
- *
- * COMPOUND_SLACK_SPACE - this is the minimum bytes of buffer space
- * needed to encode an "ordinary" _successful_ operation.  (GETATTR,
- * READ, READDIR, and READLINK have their own buffer checks.)  if we
- * fall below this level, we fail the next operation with NFS4ERR_RESOURCE.
- *
- * COMPOUND_ERR_SLACK_SPACE - this is the minimum bytes of buffer space
- * needed to encode an operation which has failed with NFS4ERR_RESOURCE.
- * care is taken to ensure that we never fall below this level for any
- * reason.
- */
-#define	COMPOUND_SLACK_SPACE		140    /* OP_GETFH */
-#define COMPOUND_ERR_SLACK_SPACE	16     /* OP_SETATTR */
-
-#define NFSD_LAUNDROMAT_MINTIMEOUT      1   /* seconds */
-#define	NFSD_COURTESY_CLIENT_TIMEOUT	(24 * 60 * 60)	/* seconds */
-#define	NFSD_CLIENT_MAX_TRIM_PER_RUN	128
-#define	NFS4_CLIENTS_PER_GB		1024
-#define NFSD_DELEGRETURN_TIMEOUT	(HZ / 34)	/* 30ms */
-#define	NFSD_CB_GETATTR_TIMEOUT		NFSD_DELEGRETURN_TIMEOUT
-
-extern int nfsd4_is_junction(struct dentry *dentry);
-extern int register_cld_notifier(void);
-extern void unregister_cld_notifier(void);
-#ifdef CONFIG_NFSD_V4_2_INTER_SSC
-extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn);
-#endif
-
-extern void nfsd4_init_leases_net(struct nfsd_net *nn);
-
-#else /* CONFIG_NFSD_V4 */
-static inline int nfsd4_is_junction(struct dentry *dentry)
-{
-	return 0;
-}
-
-static inline void nfsd4_init_leases_net(struct nfsd_net *nn) { };
-
-#define register_cld_notifier() 0
-#define unregister_cld_notifier() do { } while(0)
-
-#endif /* CONFIG_NFSD_V4 */
-
 #endif /* LINUX_NFSD_NFSD_H */
diff --git a/fs/nfsd/nfserr.h b/fs/nfsd/nfserr.h
new file mode 100644
index 00000000000000..d1c434b834b2c2
--- /dev/null
+++ b/fs/nfsd/nfserr.h
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Pre-xdr'ed nfsd error values and nfsd-internal error codes.
+ *
+ * Separated from nfsd.h so that nfsd.h itself does not have to pull
+ * in <linux/nfs4.h>: the NFS4ERR_* values used below are the only
+ * reason that include was needed.
+ */
+
+#ifndef LINUX_NFSD_NFSERR_H
+#define LINUX_NFSD_NFSERR_H
+
+#include <linux/nfs.h>
+#include <linux/nfs3.h>
+#include <linux/nfs4.h>
+
+/*
+ * These macros provide pre-xdr'ed values for faster operation.
+ */
+#define	nfs_ok				cpu_to_be32(NFS_OK)
+#define	nfserr_perm			cpu_to_be32(NFSERR_PERM)
+#define	nfserr_noent			cpu_to_be32(NFSERR_NOENT)
+#define	nfserr_io			cpu_to_be32(NFSERR_IO)
+#define	nfserr_nxio			cpu_to_be32(NFSERR_NXIO)
+#define	nfserr_acces			cpu_to_be32(NFSERR_ACCES)
+#define	nfserr_exist			cpu_to_be32(NFSERR_EXIST)
+#define	nfserr_xdev			cpu_to_be32(NFS3ERR_XDEV)
+#define	nfserr_nodev			cpu_to_be32(NFSERR_NODEV)
+#define	nfserr_notdir			cpu_to_be32(NFSERR_NOTDIR)
+#define	nfserr_isdir			cpu_to_be32(NFSERR_ISDIR)
+#define	nfserr_inval			cpu_to_be32(NFS3ERR_INVAL)
+#define	nfserr_fbig			cpu_to_be32(NFSERR_FBIG)
+#define	nfserr_nospc			cpu_to_be32(NFSERR_NOSPC)
+#define	nfserr_rofs			cpu_to_be32(NFSERR_ROFS)
+#define	nfserr_mlink			cpu_to_be32(NFS3ERR_MLINK)
+#define	nfserr_nametoolong		cpu_to_be32(NFSERR_NAMETOOLONG)
+#define	nfserr_notempty			cpu_to_be32(NFSERR_NOTEMPTY)
+#define	nfserr_dquot			cpu_to_be32(NFSERR_DQUOT)
+#define	nfserr_stale			cpu_to_be32(NFSERR_STALE)
+#define	nfserr_remote			cpu_to_be32(NFS3ERR_REMOTE)
+#define	nfserr_wflush			cpu_to_be32(NFSERR_WFLUSH)
+#define	nfserr_badhandle		cpu_to_be32(NFS3ERR_BADHANDLE)
+#define	nfserr_notsync			cpu_to_be32(NFS3ERR_NOT_SYNC)
+#define	nfserr_badcookie		cpu_to_be32(NFS3ERR_BAD_COOKIE)
+#define	nfserr_notsupp			cpu_to_be32(NFS3ERR_NOTSUPP)
+#define	nfserr_toosmall			cpu_to_be32(NFS3ERR_TOOSMALL)
+#define	nfserr_serverfault		cpu_to_be32(NFS3ERR_SERVERFAULT)
+#define	nfserr_badtype			cpu_to_be32(NFS3ERR_BADTYPE)
+#define	nfserr_jukebox			cpu_to_be32(NFS3ERR_JUKEBOX)
+#define nfserr_delay			cpu_to_be32(NFS4ERR_DELAY)
+#define	nfserr_same			cpu_to_be32(NFS4ERR_SAME)
+#define	nfserr_denied			cpu_to_be32(NFS4ERR_DENIED)
+#define	nfserr_deadlock			cpu_to_be32(NFS4ERR_DEADLOCK)
+#define	nfserr_expired			cpu_to_be32(NFS4ERR_EXPIRED)
+#define	nfserr_bad_cookie		cpu_to_be32(NFS4ERR_BAD_COOKIE)
+#define	nfserr_clid_inuse		cpu_to_be32(NFS4ERR_CLID_INUSE)
+#define	nfserr_stale_clientid		cpu_to_be32(NFS4ERR_STALE_CLIENTID)
+#define	nfserr_resource			cpu_to_be32(NFS4ERR_RESOURCE)
+#define	nfserr_moved			cpu_to_be32(NFS4ERR_MOVED)
+#define	nfserr_nofilehandle		cpu_to_be32(NFS4ERR_NOFILEHANDLE)
+#define	nfserr_minor_vers_mismatch	cpu_to_be32(NFS4ERR_MINOR_VERS_MISMATCH)
+#define nfserr_share_denied		cpu_to_be32(NFS4ERR_SHARE_DENIED)
+#define nfserr_stale_stateid		cpu_to_be32(NFS4ERR_STALE_STATEID)
+#define nfserr_old_stateid		cpu_to_be32(NFS4ERR_OLD_STATEID)
+#define nfserr_bad_stateid		cpu_to_be32(NFS4ERR_BAD_STATEID)
+#define nfserr_bad_seqid		cpu_to_be32(NFS4ERR_BAD_SEQID)
+#define	nfserr_symlink			cpu_to_be32(NFS4ERR_SYMLINK)
+#define	nfserr_not_same			cpu_to_be32(NFS4ERR_NOT_SAME)
+#define nfserr_lock_range		cpu_to_be32(NFS4ERR_LOCK_RANGE)
+#define	nfserr_restorefh		cpu_to_be32(NFS4ERR_RESTOREFH)
+#define	nfserr_attrnotsupp		cpu_to_be32(NFS4ERR_ATTRNOTSUPP)
+#define	nfserr_bad_xdr			cpu_to_be32(NFS4ERR_BADXDR)
+#define	nfserr_openmode			cpu_to_be32(NFS4ERR_OPENMODE)
+#define	nfserr_badowner			cpu_to_be32(NFS4ERR_BADOWNER)
+#define	nfserr_locks_held		cpu_to_be32(NFS4ERR_LOCKS_HELD)
+#define	nfserr_op_illegal		cpu_to_be32(NFS4ERR_OP_ILLEGAL)
+#define	nfserr_grace			cpu_to_be32(NFS4ERR_GRACE)
+#define	nfserr_no_grace			cpu_to_be32(NFS4ERR_NO_GRACE)
+#define	nfserr_reclaim_bad		cpu_to_be32(NFS4ERR_RECLAIM_BAD)
+#define	nfserr_badname			cpu_to_be32(NFS4ERR_BADNAME)
+#define	nfserr_admin_revoked		cpu_to_be32(NFS4ERR_ADMIN_REVOKED)
+#define	nfserr_cb_path_down		cpu_to_be32(NFS4ERR_CB_PATH_DOWN)
+#define	nfserr_locked			cpu_to_be32(NFS4ERR_LOCKED)
+#define	nfserr_wrongsec			cpu_to_be32(NFS4ERR_WRONGSEC)
+#define nfserr_badiomode		cpu_to_be32(NFS4ERR_BADIOMODE)
+#define nfserr_badlayout		cpu_to_be32(NFS4ERR_BADLAYOUT)
+#define nfserr_bad_session_digest	cpu_to_be32(NFS4ERR_BAD_SESSION_DIGEST)
+#define nfserr_badsession		cpu_to_be32(NFS4ERR_BADSESSION)
+#define nfserr_badslot			cpu_to_be32(NFS4ERR_BADSLOT)
+#define nfserr_complete_already		cpu_to_be32(NFS4ERR_COMPLETE_ALREADY)
+#define nfserr_conn_not_bound_to_session cpu_to_be32(NFS4ERR_CONN_NOT_BOUND_TO_SESSION)
+#define nfserr_deleg_already_wanted	cpu_to_be32(NFS4ERR_DELEG_ALREADY_WANTED)
+#define nfserr_back_chan_busy		cpu_to_be32(NFS4ERR_BACK_CHAN_BUSY)
+#define nfserr_layouttrylater		cpu_to_be32(NFS4ERR_LAYOUTTRYLATER)
+#define nfserr_layoutunavailable	cpu_to_be32(NFS4ERR_LAYOUTUNAVAILABLE)
+#define nfserr_nomatching_layout	cpu_to_be32(NFS4ERR_NOMATCHING_LAYOUT)
+#define nfserr_recallconflict		cpu_to_be32(NFS4ERR_RECALLCONFLICT)
+#define nfserr_unknown_layouttype	cpu_to_be32(NFS4ERR_UNKNOWN_LAYOUTTYPE)
+#define nfserr_seq_misordered		cpu_to_be32(NFS4ERR_SEQ_MISORDERED)
+#define nfserr_sequence_pos		cpu_to_be32(NFS4ERR_SEQUENCE_POS)
+#define nfserr_req_too_big		cpu_to_be32(NFS4ERR_REQ_TOO_BIG)
+#define nfserr_rep_too_big		cpu_to_be32(NFS4ERR_REP_TOO_BIG)
+#define nfserr_rep_too_big_to_cache	cpu_to_be32(NFS4ERR_REP_TOO_BIG_TO_CACHE)
+#define nfserr_retry_uncached_rep	cpu_to_be32(NFS4ERR_RETRY_UNCACHED_REP)
+#define nfserr_unsafe_compound		cpu_to_be32(NFS4ERR_UNSAFE_COMPOUND)
+#define nfserr_too_many_ops		cpu_to_be32(NFS4ERR_TOO_MANY_OPS)
+#define nfserr_op_not_in_session	cpu_to_be32(NFS4ERR_OP_NOT_IN_SESSION)
+#define nfserr_hash_alg_unsupp		cpu_to_be32(NFS4ERR_HASH_ALG_UNSUPP)
+#define nfserr_clientid_busy		cpu_to_be32(NFS4ERR_CLIENTID_BUSY)
+#define nfserr_pnfs_io_hole		cpu_to_be32(NFS4ERR_PNFS_IO_HOLE)
+#define nfserr_seq_false_retry		cpu_to_be32(NFS4ERR_SEQ_FALSE_RETRY)
+#define nfserr_bad_high_slot		cpu_to_be32(NFS4ERR_BAD_HIGH_SLOT)
+#define nfserr_deadsession		cpu_to_be32(NFS4ERR_DEADSESSION)
+#define nfserr_encr_alg_unsupp		cpu_to_be32(NFS4ERR_ENCR_ALG_UNSUPP)
+#define nfserr_pnfs_no_layout		cpu_to_be32(NFS4ERR_PNFS_NO_LAYOUT)
+#define nfserr_not_only_op		cpu_to_be32(NFS4ERR_NOT_ONLY_OP)
+#define nfserr_wrong_cred		cpu_to_be32(NFS4ERR_WRONG_CRED)
+#define nfserr_wrong_type		cpu_to_be32(NFS4ERR_WRONG_TYPE)
+#define nfserr_dirdeleg_unavail		cpu_to_be32(NFS4ERR_DIRDELEG_UNAVAIL)
+#define nfserr_reject_deleg		cpu_to_be32(NFS4ERR_REJECT_DELEG)
+#define nfserr_returnconflict		cpu_to_be32(NFS4ERR_RETURNCONFLICT)
+#define nfserr_deleg_revoked		cpu_to_be32(NFS4ERR_DELEG_REVOKED)
+#define nfserr_partner_notsupp		cpu_to_be32(NFS4ERR_PARTNER_NOTSUPP)
+#define nfserr_partner_no_auth		cpu_to_be32(NFS4ERR_PARTNER_NO_AUTH)
+#define nfserr_union_notsupp		cpu_to_be32(NFS4ERR_UNION_NOTSUPP)
+#define nfserr_offload_denied		cpu_to_be32(NFS4ERR_OFFLOAD_DENIED)
+#define nfserr_wrong_lfs		cpu_to_be32(NFS4ERR_WRONG_LFS)
+#define nfserr_badlabel			cpu_to_be32(NFS4ERR_BADLABEL)
+#define nfserr_file_open		cpu_to_be32(NFS4ERR_FILE_OPEN)
+#define nfserr_xattr2big		cpu_to_be32(NFS4ERR_XATTR2BIG)
+#define nfserr_noxattr			cpu_to_be32(NFS4ERR_NOXATTR)
+
+/*
+ * Error codes for internal use.  These are based at an impossible
+ * nfsstat4 value so that, once converted to be32, they cannot conflict
+ * with any value defined by the protocol (compare the nlm__int__* codes
+ * in fs/lockd/lockd.h).
+ */
+enum {
+/* end-of-file indicator in readdir */
+	NFSERR_EOF = 30000,
+#define	nfserr_eof		cpu_to_be32(NFSERR_EOF)
+
+/* replay detected */
+	NFSERR_REPLAY_ME,
+#define	nfserr_replay_me	cpu_to_be32(NFSERR_REPLAY_ME)
+
+/* nfs41 replay detected */
+	NFSERR_REPLAY_CACHE,
+#define	nfserr_replay_cache	cpu_to_be32(NFSERR_REPLAY_CACHE)
+
+/* symlink found where dir expected - handled differently to
+ * other symlink found errors by NFSv3.
+ */
+	NFSERR_SYMLINK_NOT_DIR,
+#define	nfserr_symlink_not_dir	cpu_to_be32(NFSERR_SYMLINK_NOT_DIR)
+};
+
+#endif /* LINUX_NFSD_NFSERR_H */
diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index c7c60c35bdfc0e..2bd6907f443f78 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -9,10 +9,12 @@
  */
 
 #include <linux/exportfs.h>
+#include <linux/nfs3.h>
 
 #include <linux/sunrpc/svcauth_gss.h>
 #include <crypto/utils.h>
 #include "nfsd.h"
+#include "nfserr.h"
 #include "netns.h"
 #include "stats.h"
 #include "vfs.h"
@@ -334,6 +336,8 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net,
 	}
 
 	switch (fhp->fh_maxsize) {
+	case NFSD_FHSIZE_UNSPEC:
+		break;
 	case NFS4_FHSIZE:
 		if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOATOMIC_ATTR)
 			fhp->fh_no_atomic_attr = true;
@@ -782,34 +786,53 @@ __be32 fh_getattr(const struct svc_fh *fhp, struct kstat *stat)
 				    AT_STATX_SYNC_AS_STAT));
 }
 
-/**
- * fh_fill_pre_attrs - Fill in pre-op attributes
- * @fhp: file handle to be updated
- *
- */
-__be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp)
+static __be32 __must_check __fh_fill_pre_attrs(struct svc_fh *fhp)
 {
 	bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE);
-	struct kstat stat;
 	__be32 err;
 
 	if (fhp->fh_no_wcc || fhp->fh_pre_saved)
 		return nfs_ok;
 
-	err = fh_getattr(fhp, &stat);
+	err = fh_getattr(fhp, &fhp->fh_post_attr);
 	if (err)
 		return err;
 
 	if (v4)
-		fhp->fh_pre_change = nfsd4_change_attribute(&stat);
+		fhp->fh_pre_change = fhp->fh_post_change =
+			nfsd4_change_attribute(&fhp->fh_post_attr);
 
-	fhp->fh_pre_mtime = stat.mtime;
-	fhp->fh_pre_ctime = stat.ctime;
-	fhp->fh_pre_size  = stat.size;
+	fhp->fh_pre_mtime = fhp->fh_post_attr.mtime;
+	fhp->fh_pre_ctime = fhp->fh_post_attr.ctime;
+	fhp->fh_pre_size  = fhp->fh_post_attr.size;
 	fhp->fh_pre_saved = true;
 	return nfs_ok;
 }
 
+/**
+ * fh_fill_pre_attrs - Fill in pre-op attributes
+ * @fhp: file handle to be updated
+ *
+ * Post-op attrs are filled and pre-op attrs are copied
+ * from there.  The post-op attrs can later be replaced by
+ * fh_fill_post_attrs() or activated by fh_fill_post_noop().
+ *
+ * The inode must be locked.
+ *
+ * Returns: error from vfs_getattr() which must be checked.
+ */
+__be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp)
+{
+	lockdep_assert_held_write(&fhp->fh_dentry->d_inode->i_rwsem);
+	return __fh_fill_pre_attrs(fhp);
+}
+
+__be32 __must_check fh_fill_pre_attrs_unlocked(struct svc_fh *fhp)
+{
+	fhp->fh_no_atomic_attr = true;
+	return __fh_fill_pre_attrs(fhp);
+}
+
 /**
  * fh_fill_post_attrs - Fill in post-op attributes
  * @fhp: file handle to be updated
@@ -826,6 +849,9 @@ __be32 fh_fill_post_attrs(struct svc_fh *fhp)
 	if (fhp->fh_post_saved)
 		printk("nfsd: inode locked twice during operation.\n");
 
+	if (!fhp->fh_no_atomic_attr)
+		lockdep_assert_held_write(&fhp->fh_dentry->d_inode->i_rwsem);
+
 	err = fh_getattr(fhp, &fhp->fh_post_attr);
 	if (err)
 		return err;
@@ -837,29 +863,6 @@ __be32 fh_fill_post_attrs(struct svc_fh *fhp)
 	return nfs_ok;
 }
 
-/**
- * fh_fill_both_attrs - Fill pre-op and post-op attributes
- * @fhp: file handle to be updated
- *
- * This is used when the directory wasn't changed, but wcc attributes
- * are needed anyway.
- */
-__be32 __must_check fh_fill_both_attrs(struct svc_fh *fhp)
-{
-	__be32 err;
-
-	err = fh_fill_post_attrs(fhp);
-	if (err)
-		return err;
-
-	fhp->fh_pre_change = fhp->fh_post_change;
-	fhp->fh_pre_mtime = fhp->fh_post_attr.mtime;
-	fhp->fh_pre_ctime = fhp->fh_post_attr.ctime;
-	fhp->fh_pre_size = fhp->fh_post_attr.size;
-	fhp->fh_pre_saved = true;
-	return nfs_ok;
-}
-
 /*
  * Release a file handle.
  */
diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h
index cdeb5eea65a896..7d8e3f0153073f 100644
--- a/fs/nfsd/nfsfh.h
+++ b/fs/nfsd/nfsfh.h
@@ -246,6 +246,20 @@ fh_copy_shallow(struct knfsd_fh *dst, const struct knfsd_fh *src)
 	memcpy(&dst->fh_raw, &src->fh_raw, src->fh_size);
 }
 
+#define NFSD_FHSIZE_UNSPEC	0
+
+/**
+ * fh_init - Prepare a file handle for fh_compose() or fh_verify()
+ * @fhp: File handle to initialize
+ * @maxsize: Largest file handle, in bytes, to build in @fhp
+ *
+ * @maxsize bounds the handle fh_compose() may build: NFS_FHSIZE,
+ * NFS3_FHSIZE, and NFS4_FHSIZE additionally select version-specific
+ * handling in fh_verify(). Callers that only verify an incoming
+ * handle pass NFSD_FHSIZE_UNSPEC, which cannot be composed.
+ *
+ * Return: @fhp
+ */
 static __inline__ struct svc_fh *
 fh_init(struct svc_fh *fhp, int maxsize)
 {
@@ -337,6 +351,18 @@ static inline void fh_clear_pre_post_attrs(struct svc_fh *fhp)
 
 u64 nfsd4_change_attribute(const struct kstat *stat);
 __be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp);
+__be32 __must_check fh_fill_pre_attrs_unlocked(struct svc_fh *fhp);
 __be32 fh_fill_post_attrs(struct svc_fh *fhp);
-__be32 __must_check fh_fill_both_attrs(struct svc_fh *fhp);
+
+/**
+ * fh_fill_post_noop - Copy pre attrs to post attrs
+ * @fhp: file handle to be updated
+ *
+ * This is used when the directory wasn't changed, but wcc attributes
+ * are needed anyway.
+ */
+static inline void fh_fill_post_noop(struct svc_fh *fhp)
+{
+	fhp->fh_post_saved = true;
+}
 #endif /* _LINUX_NFSD_NFSFH_H */
diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c
index e2b5f8a241bea7..09d36083982504 100644
--- a/fs/nfsd/nfsproc.c
+++ b/fs/nfsd/nfsproc.c
@@ -10,6 +10,7 @@
 #include "cache.h"
 #include "xdr.h"
 #include "vfs.h"
+#include "nfserr.h"
 #include "trace.h"
 
 #define NFSDDBG_FACILITY		NFSDDBG_PROC
@@ -39,6 +40,21 @@ static __be32 nfsd_map_status(__be32 status)
 	return status;
 }
 
+/*
+ * Because NFSv2 does not have an NFSERR_SYMLINK, Solaris returns
+ * NFSERR_ISDIR when the target of a READ or WRITE is any object
+ * that is not a regular file.
+ */
+static __be32 nfsd_map_io_status(__be32 status)
+{
+	switch (status) {
+	case nfserr_symlink:
+	case nfserr_wrong_type:
+		return nfserr_isdir;
+	}
+	return nfsd_map_status(status);
+}
+
 static __be32
 nfsd_proc_null(struct svc_rqst *rqstp)
 {
@@ -237,7 +253,7 @@ nfsd_proc_read(struct svc_rqst *rqstp)
 		resp->status = fh_getattr(&resp->fh, &resp->stat);
 	else if (resp->status == nfserr_jukebox)
 		set_bit(RQ_DROPME, &rqstp->rq_flags);
-	resp->status = nfsd_map_status(resp->status);
+	resp->status = nfsd_map_io_status(resp->status);
 	return rpc_success;
 }
 
@@ -265,12 +281,12 @@ nfsd_proc_write(struct svc_rqst *rqstp)
 
 	fh_copy(&resp->fh, &argp->fh);
 	resp->status = nfsd_write(rqstp, &resp->fh, argp->offset,
-				  &argp->payload, &cnt, NFS_DATA_SYNC, NULL);
+				  &argp->payload, &cnt, IOCB_DSYNC, NULL);
 	if (resp->status == nfs_ok)
 		resp->status = fh_getattr(&resp->fh, &resp->stat);
 	else if (resp->status == nfserr_jukebox)
 		set_bit(RQ_DROPME, &rqstp->rq_flags);
-	resp->status = nfsd_map_status(resp->status);
+	resp->status = nfsd_map_io_status(resp->status);
 	return rpc_success;
 }
 
@@ -291,6 +307,7 @@ nfsd_proc_create(struct svc_rqst *rqstp)
 	struct nfsd_attrs attrs = {
 		.na_iattr	= attr,
 	};
+	struct svc_export *exp;
 	struct inode	*inode;
 	struct dentry	*dchild;
 	int		type, mode;
@@ -319,8 +336,22 @@ nfsd_proc_create(struct svc_rqst *rqstp)
 		resp->status = nfserrno(PTR_ERR(dchild));
 		goto out_write;
 	}
+	/*
+	 * If name exists we need to check for mountpoints
+	 */
+	exp = exp_get(dirfhp->fh_export);
+	if (d_is_reg(dchild) &&
+	    unlikely(nfsd_mountpoint(dchild, exp))) {
+		resp->status = nfsd_cross_mnt(rqstp, &dchild, &exp);
+		if (resp->status != nfs_ok) {
+			exp_put(exp);
+			goto out_unlock;
+		}
+	}
+
 	fh_init(newfhp, NFS_FHSIZE);
-	resp->status = fh_compose(newfhp, dirfhp->fh_export, dchild, dirfhp);
+	resp->status = fh_compose(newfhp, exp, dchild, dirfhp);
+	exp_put(exp);
 	if (!resp->status && d_really_is_negative(dchild))
 		resp->status = nfserr_noent;
 	if (resp->status) {
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 2edf716ea022ff..c04ef9d180ceab 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -25,7 +25,10 @@
 #include <net/addrconf.h>
 #include <net/ipv6.h>
 #include <net/net_namespace.h>
+
 #include "nfsd.h"
+#include "nfserr.h"
+#include "nfs4ctl.h"
 #include "cache.h"
 #include "vfs.h"
 #include "netns.h"
@@ -204,6 +207,11 @@ int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change
 	return 0;
 }
 
+bool nfsd_v4client(struct svc_rqst *rqstp)
+{
+	return rqstp && rqstp->rq_prog == NFS_PROGRAM && rqstp->rq_vers == 4;
+}
+
 bool nfsd_net_try_get(struct net *net) __must_hold(rcu)
 {
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c
index 019f0cc971a7aa..0961c13d6ab1d7 100644
--- a/fs/nfsd/nfsxdr.c
+++ b/fs/nfsd/nfsxdr.c
@@ -8,6 +8,7 @@
 #include <linux/filelock.h>
 
 #include "vfs.h"
+#include "nfserr.h"
 #include "xdr.h"
 #include "auth.h"
 
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index 2d00a411c6634e..cd9294f024bb98 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -45,6 +45,25 @@
 #include "nfsfh.h"
 #include "nfsd.h"
 
+/*
+ * Before processing a COMPOUND operation, we have to check that there
+ * is enough space in the buffer for XDR encode to succeed.  otherwise,
+ * we might process an operation with side effects, and be unable to
+ * tell the client that the operation succeeded.
+ *
+ * COMPOUND_ERR_SLACK_SPACE - this is the minimum bytes of buffer space
+ * needed to encode an operation which has failed with NFS4ERR_RESOURCE.
+ * care is taken to ensure that we never fall below this level for any
+ * reason.
+ */
+#define COMPOUND_ERR_SLACK_SPACE	16     /* OP_SETATTR */
+
+#define NFSD_LAUNDROMAT_MINTIMEOUT      1   /* seconds */
+#define	NFSD_CLIENT_MAX_TRIM_PER_RUN	128
+#define	NFS4_CLIENTS_PER_GB		1024
+#define NFSD_DELEGRETURN_TIMEOUT	(HZ / 34)	/* 30ms */
+#define	NFSD_CB_GETATTR_TIMEOUT		NFSD_DELEGRETURN_TIMEOUT
+
 typedef struct {
 	u32             cl_boot;
 	u32             cl_id;
@@ -126,10 +145,13 @@ struct nfs4_stid {
 #define SC_TYPE_COPY		BIT(4)
 	unsigned short		sc_type;
 
-/* nn->deleg_lock protects sc_status for delegation stateids.
- * ->cl_lock protects sc_status for open and lock stateids.
- * ->st_mutex also protect sc_status for open stateids.
- * ->ls_lock protects sc_status for layout stateids.
+/*
+ * nn->deleg_lock protects sc_status for hashed delegation stateids.
+ * ->cl_lock protects the bits set as one is disposed of
+ * (SC_STATUS_CLOSED, SC_STATUS_FREEABLE, SC_STATUS_FREED) and
+ * sc_status for open and lock stateids. ->st_mutex also protects
+ * sc_status for open stateids. ->ls_lock protects sc_status for
+ * layout stateids.
  */
 /*
  * For an open stateid kept around *only* to process close replays.
@@ -249,6 +271,7 @@ struct nfsd4_cb_notify {
 	struct nfsd_notify_event	*ncn_evt[NOTIFY4_EVENT_QUEUE_SIZE]; // list of events
 	struct page			*ncn_pages[NOTIFY4_PAGE_ARRAY_SIZE]; // for encoding
 	struct notify4			*ncn_nf;	// array of notify4's to be sent
+	u32				*ncn_masks;	// host-order notify_mask backing for ncn_nf[]
 	bool				ncn_encode_err;	// did encoding fail?
 	struct nfsd4_callback		ncn_cb;		// notify4 callback
 };
@@ -270,7 +293,8 @@ struct nfsd4_cb_notify {
  * If the server attempts to recall a delegation and the client doesn't do so
  * before a timeout, the server may also revoke the delegation. In that case,
  * the object will either be destroyed (v4.0) or moved to a per-client list of
- * revoked delegations (v4.1+).
+ * revoked delegations (v4.1+). A v4.1+ client that rejects the recall holds
+ * no record of the delegation, so the object is destroyed rather than listed.
  *
  * This object is a superset of the nfs4_stid.
  */
@@ -286,9 +310,19 @@ struct nfs4_delegation {
 	int			dl_retries;
 	struct nfsd4_callback	dl_recall;
 	bool			dl_recalled;
+	bool			dl_recall_rejected;
 	bool			dl_written;
 	bool			dl_setattr;
 
+	/* Forward-channel slot that carried the granting request */
+	struct {
+		u32			sessionid_seq;
+		u32			slotid;
+		u32			seqid;
+		bool			valid;
+		bool			retired_at_send;
+	} dl_recall_grant;
+
 	union {
 		/* for CB_GETATTR */
 		struct nfs4_cb_fattr    dl_cb_fattr;
@@ -599,6 +633,8 @@ struct nfs4_client {
 
 	unsigned int		cl_state;
 	atomic_t		cl_delegs_in_recall;
+	/* Length of cl_delegations, updated under nn->deleg_lock */
+	unsigned int		cl_deleg_count;
 
 	struct nfsd4_cb_recall_any	*cl_ra;
 	time64_t		cl_ra_time;
diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h
index 7d7a1483109a00..2ae7f150a72ce9 100644
--- a/fs/nfsd/trace.h
+++ b/fs/nfsd/trace.h
@@ -13,7 +13,6 @@
 #include <linux/sunrpc/xprt.h>
 #include <trace/misc/fs.h>
 #include <trace/misc/fsnotify.h>
-#include <trace/misc/nfs.h>
 #include <trace/misc/sunrpc.h>
 
 #include "export.h"
@@ -790,6 +789,20 @@ TRACE_EVENT(nfsd_stateowner_replay,
 		__entry->opnum, __entry->status)
 );
 
+#define show_nfs4_seq4_status(x) \
+	__print_flags(x, "|", \
+		{ SEQ4_STATUS_CB_PATH_DOWN,		"CB_PATH_DOWN" }, \
+		{ SEQ4_STATUS_CB_GSS_CONTEXTS_EXPIRING,	"CB_GSS_CONTEXTS_EXPIRING" }, \
+		{ SEQ4_STATUS_CB_GSS_CONTEXTS_EXPIRED,	"CB_GSS_CONTEXTS_EXPIRED" }, \
+		{ SEQ4_STATUS_EXPIRED_ALL_STATE_REVOKED, "EXPIRED_ALL_STATE_REVOKED" }, \
+		{ SEQ4_STATUS_EXPIRED_SOME_STATE_REVOKED, "EXPIRED_SOME_STATE_REVOKED" }, \
+		{ SEQ4_STATUS_ADMIN_STATE_REVOKED,	"ADMIN_STATE_REVOKED" }, \
+		{ SEQ4_STATUS_RECALLABLE_STATE_REVOKED,	"RECALLABLE_STATE_REVOKED" }, \
+		{ SEQ4_STATUS_LEASE_MOVED,		"LEASE_MOVED" }, \
+		{ SEQ4_STATUS_RESTART_RECLAIM_NEEDED,	"RESTART_RECLAIM_NEEDED" }, \
+		{ SEQ4_STATUS_CB_PATH_DOWN_SESSION,	"CB_PATH_DOWN_SESSION" }, \
+		{ SEQ4_STATUS_BACKCHANNEL_FAULT,	"BACKCHANNEL_FAULT" })
+
 TRACE_EVENT_CONDITION(nfsd_seq4_status,
 	TP_PROTO(
 		const struct svc_rqst *rqstp,
@@ -1694,6 +1707,14 @@ TRACE_EVENT(nfsd_cb_setup_err,
 /* Not a real opcode, but there is no 0 operation. */
 #define _CB_NULL	0
 
+TRACE_DEFINE_ENUM(OP_CB_GETATTR);
+TRACE_DEFINE_ENUM(OP_CB_RECALL);
+TRACE_DEFINE_ENUM(OP_CB_LAYOUTRECALL);
+TRACE_DEFINE_ENUM(OP_CB_RECALL_ANY);
+TRACE_DEFINE_ENUM(OP_CB_NOTIFY);
+TRACE_DEFINE_ENUM(OP_CB_NOTIFY_LOCK);
+TRACE_DEFINE_ENUM(OP_CB_OFFLOAD);
+
 #define show_nfsd_cb_opcode(val)					\
 	__print_symbolic(val,						\
 		{ _CB_NULL,			"CB_NULL" },		\
@@ -1918,6 +1939,28 @@ TRACE_EVENT(nfsd_cb_offload,
 		__entry->fh_hash, __entry->count, __entry->status)
 );
 
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_RDATA_DLG);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_WDATA_DLG);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_DIR_DLG);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_FILE_LAYOUT);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_BLK_LAYOUT);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_OBJ_LAYOUT_MIN);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_OBJ_LAYOUT_MAX);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_OTHER_LAYOUT_MIN);
+TRACE_DEFINE_ENUM(RCA4_TYPE_MASK_OTHER_LAYOUT_MAX);
+
+#define show_rca_mask(x) \
+	__print_flags(x, "|", \
+		{ BIT(RCA4_TYPE_MASK_RDATA_DLG),	"RDATA_DLG" }, \
+		{ BIT(RCA4_TYPE_MASK_WDATA_DLG),	"WDATA_DLG" }, \
+		{ BIT(RCA4_TYPE_MASK_DIR_DLG),		"DIR_DLG" }, \
+		{ BIT(RCA4_TYPE_MASK_FILE_LAYOUT),	"FILE_LAYOUT" }, \
+		{ BIT(RCA4_TYPE_MASK_BLK_LAYOUT),	"BLK_LAYOUT" }, \
+		{ BIT(RCA4_TYPE_MASK_OBJ_LAYOUT_MIN),	"OBJ_LAYOUT_MIN" }, \
+		{ BIT(RCA4_TYPE_MASK_OBJ_LAYOUT_MAX),	"OBJ_LAYOUT_MAX" }, \
+		{ BIT(RCA4_TYPE_MASK_OTHER_LAYOUT_MIN),	"OTHER_LAYOUT_MIN" }, \
+		{ BIT(RCA4_TYPE_MASK_OTHER_LAYOUT_MAX),	"OTHER_LAYOUT_MAX" })
+
 TRACE_EVENT(nfsd_cb_recall_any,
 	TP_PROTO(
 		const struct nfsd4_cb_recall_any *ra
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index 8923a9910a08f2..4789f2ec2078c2 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -34,15 +34,14 @@
 #include <linux/sunrpc/xdr.h>
 #include <linux/fileattr.h>
 
-#include "xdr3.h"
-
 #ifdef CONFIG_NFSD_V4
 #include "acl.h"
 #include "idmap.h"
-#include "xdr4.h"
 #endif /* CONFIG_NFSD_V4 */
 
 #include "nfsd.h"
+#include "nfserr.h"
+#include "nfs4ctl.h"
 #include "netns.h"
 #include "stats.h"
 #include "vfs.h"
@@ -63,7 +62,7 @@ u64 nfsd_io_cache_write __read_mostly = NFSD_IO_BUFFERED;
  * it's an error we don't expect, log it once and return nfserr_io.
  */
 __be32
-nfserrno (int errno)
+nfserrno(int errno)
 {
 	static struct {
 		__be32	nfserr;
@@ -107,6 +106,8 @@ nfserrno (int errno)
 		{ nfserr_perm, -ENOKEY },
 		{ nfserr_no_grace, -ENOGRACE},
 		{ nfserr_io, -EBADMSG },
+		{ nfserr_symlink, -ELOOP },
+		{ nfserr_wrong_type, -EFTYPE },
 	};
 	int	i;
 
@@ -118,15 +119,15 @@ nfserrno (int errno)
 	return nfserr_io;
 }
 
-/* 
- * Called from nfsd_lookup and encode_dirent. Check if we have crossed 
+/*
+ * Called from nfsd_lookup and encode_dirent. Check if we have crossed
  * a mount point.
- * Returns -EAGAIN or -ETIMEDOUT leaving *dpp and *expp unchanged,
+ * Returns an nfs error leaving *dpp and *expp unchanged,
  *  or nfs_ok having possibly changed *dpp and *expp
  */
-int
-nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, 
-		        struct svc_export **expp)
+__be32
+nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
+	       struct svc_export **expp)
 {
 	struct svc_export *exp = *expp, *exp2 = NULL;
 	struct dentry *dentry = *dpp;
@@ -134,6 +135,7 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 			    .dentry = dget(dentry)};
 	unsigned int follow_flags = 0;
 	int err = 0;
+	__be32 nfserr = nfs_ok;
 
 	if (exp->ex_flags & NFSEXP_CROSSMOUNT)
 		follow_flags = LOOKUP_AUTOMOUNT;
@@ -163,23 +165,28 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
 			err = 0;
 	} else if (nfsd_v4client(rqstp) ||
 		(exp->ex_flags & NFSEXP_CROSSMOUNT) || EX_NOHIDE(exp2)) {
-		/* successfully crossed mount point */
-		/*
-		 * This is subtle: path.dentry is *not* on path.mnt
-		 * at this point.  The only reason we are safe is that
-		 * original mnt is pinned down by exp, so we should
-		 * put path *before* putting exp
-		 */
-		*dpp = path.dentry;
-		path.dentry = dentry;
-		*expp = exp2;
-		exp2 = exp;
+		nfserr = check_nfsd_access(exp, rqstp);
+		if (nfserr == nfs_ok) {
+			/* successfully crossed mount point */
+			/*
+			 * This is subtle: path.dentry is *not* on path.mnt
+			 * at this point.  The only reason we are safe is that
+			 * original mnt is pinned down by exp, so we should
+			 * put path *before* putting exp
+			 */
+			*dpp = path.dentry;
+			path.dentry = dentry;
+			*expp = exp2;
+			exp2 = exp;
+		}
 	}
 out:
 	path_put(&path);
 	if (exp2)
 		exp_put(exp2);
-	return err;
+	if (nfserr)
+		return nfserr;
+	return nfserrno(err);
 }
 
 static void follow_to_parent(struct path *path)
@@ -277,10 +284,12 @@ nfsd_lookup_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		if (IS_ERR(dentry))
 			goto out_nfserr;
 		if (nfsd_mountpoint(dentry, exp)) {
-			host_err = nfsd_cross_mnt(rqstp, &dentry, &exp);
-			if (host_err) {
+			__be32 nfserr = nfsd_cross_mnt(rqstp, &dentry, &exp);
+
+			if (nfserr) {
 				dput(dentry);
-				goto out_nfserr;
+				exp_put(exp);
+				return nfserr;
 			}
 		}
 	}
@@ -327,9 +336,6 @@ nfsd_lookup(struct svc_rqst *rqstp, struct svc_fh *fhp, const char *name,
 	err = nfsd_lookup_dentry(rqstp, fhp, name, len, &exp, &dentry);
 	if (err)
 		return err;
-	err = check_nfsd_access(exp, rqstp, false);
-	if (err)
-		goto out;
 	/*
 	 * Note: we compose the file handle now, but as the
 	 * dentry may be negative, it may need to be updated.
@@ -337,15 +343,26 @@ nfsd_lookup(struct svc_rqst *rqstp, struct svc_fh *fhp, const char *name,
 	err = fh_compose(resfh, exp, dentry, fhp);
 	if (!err && d_really_is_negative(dentry))
 		err = nfserr_noent;
-out:
+
 	dput(dentry);
 	exp_put(exp);
 	return err;
 }
 
-static void
-commit_reset_write_verifier(struct nfsd_net *nn, struct svc_rqst *rqstp,
-			    int err)
+/**
+ * nfsd_maybe_reset_write_verifier - Reset the write verifier after an I/O error
+ * @nn: nfsd namespace holding the write verifier
+ * @rqstp: RPC transaction context
+ * @err: errno reported by the failed operation
+ *
+ * A write verifier reset tells clients that unstable data the server has
+ * already acknowledged might have been lost. Client response is to resend
+ * in-flight dirty data.
+ *
+ * Context: Process context.
+ */
+void nfsd_maybe_reset_write_verifier(struct nfsd_net *nn,
+				     struct svc_rqst *rqstp, int err)
 {
 	switch (err) {
 	case -EAGAIN:
@@ -682,56 +699,67 @@ int nfsd4_is_junction(struct dentry *dentry)
 	return 1;
 }
 
-static struct nfsd4_compound_state *nfsd4_get_cstate(struct svc_rqst *rqstp)
+/**
+ * nfsd_clone_file_range - Clone a range of one file into another
+ * @src: file the range is cloned from
+ * @src_pos: offset in @src where the source range begins
+ * @dst: file the range is cloned into
+ * @dst_pos: offset in @dst where the destination range begins
+ * @count: length of the range, or zero to clone through end-of-file
+ * @since: receives @dst's writeback error state, sampled before the clone
+ *
+ * A caller that has to place the cloned data on durable storage passes
+ * @since to nfsd_clone_sync_range() once this call succeeds. Sampling
+ * happens here because a writeback error raised by the clone's own
+ * dirty pages has to fall inside the sampled interval.
+ *
+ * Context: Process context.
+ * Return: zero on success, or a negative errno
+ */
+int nfsd_clone_file_range(struct file *src, u64 src_pos, struct file *dst,
+			  u64 dst_pos, u64 count, errseq_t *since)
 {
-	return &((struct nfsd4_compoundres *)rqstp->rq_resp)->cstate;
+	loff_t cloned;
+
+	*since = READ_ONCE(dst->f_wb_err);
+	cloned = vfs_clone_file_range(src, src_pos, dst, dst_pos, count, 0);
+	if (cloned < 0)
+		return cloned;
+	if (count && cloned != count)
+		return -EINVAL;
+	return 0;
 }
 
-__be32 nfsd4_clone_file_range(struct svc_rqst *rqstp,
-		struct nfsd_file *nf_src, u64 src_pos,
-		struct nfsd_file *nf_dst, u64 dst_pos,
-		u64 count, bool sync)
+/**
+ * nfsd_clone_sync_range - Commit a cloned range to durable storage
+ * @src: file the range was cloned from, whose metadata is committed too
+ * @dst: file the range was cloned into
+ * @dst_pos: offset in @dst where the cloned range begins
+ * @count: length of the range, or zero if the clone ran to end-of-file
+ * @since: @dst's writeback error state as sampled by
+ *	   nfsd_clone_file_range()
+ *
+ * Context: Process context.
+ * Return: zero on success, or a negative errno
+ */
+int nfsd_clone_sync_range(struct file *src, struct file *dst, u64 dst_pos,
+			  u64 count, errseq_t since)
 {
-	struct file *src = nf_src->nf_file;
-	struct file *dst = nf_dst->nf_file;
-	errseq_t since;
-	loff_t cloned;
-	__be32 ret = 0;
+	loff_t dst_end = count ? dst_pos + count - 1 : LLONG_MAX;
+	int status;
 
-	since = READ_ONCE(dst->f_wb_err);
-	cloned = vfs_clone_file_range(src, src_pos, dst, dst_pos, count, 0);
-	if (cloned < 0) {
-		ret = nfserrno(cloned);
-		goto out_err;
-	}
-	if (count && cloned != count) {
-		ret = nfserrno(-EINVAL);
-		goto out_err;
-	}
-	if (sync) {
-		loff_t dst_end = count ? dst_pos + count - 1 : LLONG_MAX;
-		int status = vfs_fsync_range(dst, dst_pos, dst_end, 0);
-
-		if (!status)
-			status = filemap_check_wb_err(dst->f_mapping, since);
-		if (!status)
-			status = commit_inode_metadata(file_inode(src));
-		if (status < 0) {
-			struct nfsd_net *nn = net_generic(nf_dst->nf_net,
-							  nfsd_net_id);
-
-			trace_nfsd_clone_file_range_err(rqstp,
-					&nfsd4_get_cstate(rqstp)->save_fh,
-					src_pos,
-					&nfsd4_get_cstate(rqstp)->current_fh,
-					dst_pos,
-					count, status);
-			commit_reset_write_verifier(nn, rqstp, status);
-			ret = nfserrno(status);
-		}
+	status = vfs_fsync_range(dst, dst_pos, dst_end, 0);
+	if (!status)
+		status = filemap_check_wb_err(dst->f_mapping, since);
+	if (!status) {
+		/*
+		 * A reflink marks extents shared in the source inode too,
+		 * so the source's metadata has to reach durable storage
+		 * even though its data is untouched.
+		 */
+		status = commit_inode_metadata(file_inode(src));
 	}
-out_err:
-	return ret;
+	return status;
 }
 
 ssize_t nfsd_copy_file_range(struct file *src, u64 src_pos, struct file *dst,
@@ -773,64 +801,21 @@ __be32 nfsd4_vfs_fallocate(struct svc_rqst *rqstp, struct svc_fh *fhp,
 }
 #endif /* defined(CONFIG_NFSD_V4) */
 
-/*
- * Check server access rights to a file system object
+/**
+ * nfsd_access - Check caller's access rights to a file system object
+ * @rqstp: RPC transaction context
+ * @fhp: target NFS filehandle
+ * @maps: tables mapping on-the-wire access bits to NFSD_MAY flags
+ * @access: requested access bits on entry, permitted bits on return
+ * @supported: optional output of the access bits the server supports
+ *
+ * Return: nfs_ok on success, otherwise an nfserr status code
  */
-struct accessmap {
-	u32		access;
-	int		how;
-};
-static struct accessmap	nfs3_regaccess[] = {
-    {	NFS3_ACCESS_READ,	NFSD_MAY_READ			},
-    {	NFS3_ACCESS_EXECUTE,	NFSD_MAY_EXEC			},
-    {	NFS3_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_TRUNC	},
-    {	NFS3_ACCESS_EXTEND,	NFSD_MAY_WRITE			},
-
-#ifdef CONFIG_NFSD_V4
-    {	NFS4_ACCESS_XAREAD,	NFSD_MAY_READ			},
-    {	NFS4_ACCESS_XAWRITE,	NFSD_MAY_WRITE			},
-    {	NFS4_ACCESS_XALIST,	NFSD_MAY_READ			},
-#endif
-
-    {	0,			0				}
-};
-
-static struct accessmap	nfs3_diraccess[] = {
-    {	NFS3_ACCESS_READ,	NFSD_MAY_READ			},
-    {	NFS3_ACCESS_LOOKUP,	NFSD_MAY_EXEC			},
-    {	NFS3_ACCESS_MODIFY,	NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC},
-    {	NFS3_ACCESS_EXTEND,	NFSD_MAY_EXEC|NFSD_MAY_WRITE	},
-    {	NFS3_ACCESS_DELETE,	NFSD_MAY_REMOVE			},
-
-#ifdef CONFIG_NFSD_V4
-    {	NFS4_ACCESS_XAREAD,	NFSD_MAY_READ			},
-    {	NFS4_ACCESS_XAWRITE,	NFSD_MAY_WRITE			},
-    {	NFS4_ACCESS_XALIST,	NFSD_MAY_READ			},
-#endif
-
-    {	0,			0				}
-};
-
-static struct accessmap	nfs3_anyaccess[] = {
-	/* Some clients - Solaris 2.6 at least, make an access call
-	 * to the server to check for access for things like /dev/null
-	 * (which really, the server doesn't care about).  So
-	 * We provide simple access checking for them, looking
-	 * mainly at mode bits, and we make sure to ignore read-only
-	 * filesystem checks
-	 */
-    {	NFS3_ACCESS_READ,	NFSD_MAY_READ			},
-    {	NFS3_ACCESS_EXECUTE,	NFSD_MAY_EXEC			},
-    {	NFS3_ACCESS_MODIFY,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
-    {	NFS3_ACCESS_EXTEND,	NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS	},
-
-    {	0,			0				}
-};
-
-__be32
-nfsd_access(struct svc_rqst *rqstp, struct svc_fh *fhp, u32 *access, u32 *supported)
+__be32 nfsd_access(struct svc_rqst *rqstp, struct svc_fh *fhp,
+		   const struct nfsd_access_maps *maps,
+		   u32 *access, u32 *supported)
 {
-	struct accessmap	*map;
+	const struct nfsd_access_map *map;
 	struct svc_export	*export;
 	struct dentry		*dentry;
 	u32			query, result = 0, sresult = 0;
@@ -844,12 +829,11 @@ nfsd_access(struct svc_rqst *rqstp, struct svc_fh *fhp, u32 *access, u32 *suppor
 	dentry = fhp->fh_dentry;
 
 	if (d_is_reg(dentry))
-		map = nfs3_regaccess;
+		map = maps->regular;
 	else if (d_is_dir(dentry))
-		map = nfs3_diraccess;
+		map = maps->directory;
 	else
-		map = nfs3_anyaccess;
-
+		map = maps->other;
 
 	query = *access;
 	for  (; map->access; map++) {
@@ -859,7 +843,7 @@ nfsd_access(struct svc_rqst *rqstp, struct svc_fh *fhp, u32 *access, u32 *suppor
 			sresult |= map->access;
 
 			err2 = nfsd_permission(&rqstp->rq_cred, export,
-					       dentry, map->how);
+					       dentry, map->may);
 			switch (err2) {
 			case nfs_ok:
 				result |= map->access;
@@ -1062,7 +1046,6 @@ static __be32 nfsd_finish_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		nfsd_stats_io_read_add(nn, fhp->fh_export, host_err);
 		*eof = nfsd_eof_on_read(file, offset, host_err, *count);
 		*count = host_err;
-		fsnotify_access(file);
 		trace_nfsd_read_io_done(rqstp, fhp, offset, *count);
 		return 0;
 	} else {
@@ -1087,19 +1070,11 @@ __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
 			struct file *file, loff_t offset, unsigned long *count,
 			u32 *eof)
 {
-	struct splice_desc sd = {
-		.len		= 0,
-		.total_len	= *count,
-		.pos		= offset,
-		.u.data		= rqstp,
-	};
 	ssize_t host_err;
 
 	trace_nfsd_read_splice(rqstp, fhp, offset, *count);
-	host_err = rw_verify_area(READ, file, &offset, *count);
-	if (!host_err)
-		host_err = splice_direct_to_actor(file, &sd,
-						  nfsd_direct_splice_actor);
+	host_err = vfs_splice_to_actor(file, offset, *count,
+				       nfsd_direct_splice_actor, rqstp);
 	return nfsd_finish_read(rqstp, fhp, file, offset, count, eof, host_err);
 }
 
@@ -1424,7 +1399,7 @@ nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
  * @offset: Byte offset of start
  * @payload: xdr_buf containing the write payload
  * @cnt: IN: number of bytes to write, OUT: number of bytes actually written
- * @stable: An NFS stable_how value
+ * @iocb_flags: VFS IOCB_* flags expressing the requested write stability
  * @verf: NFS WRITE verifier
  *
  * Upon return, caller must invoke fh_put on @fhp.
@@ -1436,7 +1411,7 @@ __be32
 nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	       struct nfsd_file *nf, loff_t offset,
 	       const struct xdr_buf *payload, unsigned long *cnt,
-	       int stable, __be32 *verf)
+	       int iocb_flags, __be32 *verf)
 {
 	struct nfsd_net		*nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
 	struct file		*file = nf->nf_file;
@@ -1473,21 +1448,11 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	exp = fhp->fh_export;
 
 	if (!EX_ISSYNC(exp))
-		stable = NFS_UNSTABLE;
+		iocb_flags = 0;
 	init_sync_kiocb(&kiocb, file);
 	kiocb.ki_pos = offset;
-	if (likely(!fhp->fh_use_wgather)) {
-		switch (stable) {
-		case NFS_FILE_SYNC:
-			/* persist data and timestamps */
-			kiocb.ki_flags |= IOCB_DSYNC | IOCB_SYNC;
-			break;
-		case NFS_DATA_SYNC:
-			/* persist data only */
-			kiocb.ki_flags |= IOCB_DSYNC;
-			break;
-		}
-	}
+	if (likely(!fhp->fh_use_wgather))
+		kiocb.ki_flags |= iocb_flags;
 
 	nvecs = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, payload);
 	if (nvecs < 0) {
@@ -1517,21 +1482,21 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		break;
 	}
 	if (host_err < 0) {
-		commit_reset_write_verifier(nn, rqstp, host_err);
+		nfsd_maybe_reset_write_verifier(nn, rqstp, host_err);
 		goto out_nfserr;
 	}
 	nfsd_stats_io_write_add(nn, exp, *cnt);
 	fsnotify_modify(file);
 	host_err = filemap_check_wb_err(file->f_mapping, since);
 	if (host_err < 0) {
-		commit_reset_write_verifier(nn, rqstp, host_err);
+		nfsd_maybe_reset_write_verifier(nn, rqstp, host_err);
 		goto out_nfserr;
 	}
 
-	if (stable && fhp->fh_use_wgather) {
+	if (iocb_flags && fhp->fh_use_wgather) {
 		host_err = wait_for_concurrent_writes(file);
 		if (host_err < 0)
-			commit_reset_write_verifier(nn, rqstp, host_err);
+			nfsd_maybe_reset_write_verifier(nn, rqstp, host_err);
 	}
 
 out_nfserr:
@@ -1619,7 +1584,7 @@ __be32 nfsd_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
  * @offset: Byte offset of start
  * @payload: xdr_buf containing the write payload
  * @cnt: IN: number of bytes to write, OUT: number of bytes actually written
- * @stable: An NFS stable_how value
+ * @iocb_flags: VFS IOCB_* flags expressing the requested write stability
  * @verf: NFS WRITE verifier
  *
  * Upon return, caller must invoke fh_put on @fhp.
@@ -1629,8 +1594,8 @@ __be32 nfsd_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
  */
 __be32
 nfsd_write(struct svc_rqst *rqstp, struct svc_fh *fhp, loff_t offset,
-	   const struct xdr_buf *payload, unsigned long *cnt, int stable,
-	   __be32 *verf)
+	   const struct xdr_buf *payload, unsigned long *cnt,
+	   int iocb_flags, __be32 *verf)
 {
 	struct nfsd_file *nf;
 	__be32 err;
@@ -1642,7 +1607,7 @@ nfsd_write(struct svc_rqst *rqstp, struct svc_fh *fhp, loff_t offset,
 		goto out;
 
 	err = nfsd_vfs_write(rqstp, fhp, nf, offset, payload, cnt,
-			     stable, verf);
+			     iocb_flags, verf);
 	nfsd_file_put(nf);
 out:
 	trace_nfsd_write_done(rqstp, fhp, offset, *cnt);
@@ -1707,14 +1672,14 @@ nfsd_commit(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf,
 			err2 = filemap_check_wb_err(nf->nf_file->f_mapping,
 						    since);
 			if (err2 < 0)
-				commit_reset_write_verifier(nn, rqstp, err2);
+				nfsd_maybe_reset_write_verifier(nn, rqstp, err2);
 			err = nfserrno(err2);
 			break;
 		case -EINVAL:
 			err = nfserr_notsupp;
 			break;
 		default:
-			commit_reset_write_verifier(nn, rqstp, err2);
+			nfsd_maybe_reset_write_verifier(nn, rqstp, err2);
 			err = nfserrno(err2);
 		}
 	} else
diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h
index 4af2ff9e9dfeee..f0cb184643f2f4 100644
--- a/fs/nfsd/vfs.h
+++ b/fs/nfsd/vfs.h
@@ -37,6 +37,17 @@
 #define NFSD_MAY_CREATE		(NFSD_MAY_EXEC|NFSD_MAY_WRITE)
 #define NFSD_MAY_REMOVE		(NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC)
 
+struct nfsd_access_map {
+	u32		access;
+	int		may;
+};
+
+struct nfsd_access_maps {
+	const struct nfsd_access_map	*regular;
+	const struct nfsd_access_map	*directory;
+	const struct nfsd_access_map	*other;
+};
+
 struct nfsd_file;
 
 /*
@@ -75,9 +86,14 @@ static inline bool nfsd_attrs_valid(struct nfsd_attrs *attrs)
 		attrs->na_pacl || attrs->na_dpacl);
 }
 
+struct nfsd_net;
+
 __be32		nfserrno (int errno);
-int		nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
-		                struct svc_export **expp);
+void		nfsd_maybe_reset_write_verifier(struct nfsd_net *nn,
+						struct svc_rqst *rqstp,
+						int err);
+__be32		nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp,
+			       struct svc_export **expp);
 __be32		nfsd_lookup(struct svc_rqst *, struct svc_fh *,
 				const char *, unsigned int, struct svc_fh *);
 __be32		 nfsd_lookup_dentry(struct svc_rqst *, struct svc_fh *,
@@ -89,10 +105,11 @@ int nfsd_mountpoint(struct dentry *, struct svc_export *);
 #ifdef CONFIG_NFSD_V4
 __be32		nfsd4_vfs_fallocate(struct svc_rqst *, struct svc_fh *,
 				    struct file *, loff_t, loff_t, int);
-__be32		nfsd4_clone_file_range(struct svc_rqst *rqstp,
-				       struct nfsd_file *nf_src, u64 src_pos,
-				       struct nfsd_file *nf_dst, u64 dst_pos,
-				       u64 count, bool sync);
+int		nfsd_clone_file_range(struct file *src, u64 src_pos,
+				      struct file *dst, u64 dst_pos,
+				      u64 count, errseq_t *since);
+int		nfsd_clone_sync_range(struct file *src, struct file *dst,
+				      u64 dst_pos, u64 count, errseq_t since);
 #endif /* CONFIG_NFSD_V4 */
 __be32		nfsd_create_locked(struct svc_rqst *, struct svc_fh *,
 				struct nfsd_attrs *attrs, int type, dev_t rdev,
@@ -100,7 +117,9 @@ __be32		nfsd_create_locked(struct svc_rqst *, struct svc_fh *,
 __be32		nfsd_create(struct svc_rqst *, struct svc_fh *,
 				char *name, int len, struct nfsd_attrs *attrs,
 				int type, dev_t rdev, struct svc_fh *res);
-__be32		nfsd_access(struct svc_rqst *, struct svc_fh *, u32 *, u32 *);
+__be32		nfsd_access(struct svc_rqst *rqstp, struct svc_fh *fhp,
+				const struct nfsd_access_maps *maps,
+				u32 *access, u32 *supported);
 __be32		nfsd_create_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp,
 				struct svc_fh *resfhp, struct nfsd_attrs *iap);
 __be32		nfsd_commit(struct svc_rqst *rqst, struct svc_fh *fhp,
@@ -135,11 +154,13 @@ __be32		nfsd_read(struct svc_rqst *rqstp, struct svc_fh *fhp,
 				u32 *eof);
 __be32		nfsd_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 				loff_t offset, const struct xdr_buf *payload,
-				unsigned long *cnt, int stable, __be32 *verf);
+				unsigned long *cnt, int iocb_flags,
+				__be32 *verf);
 __be32		nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp,
 				struct nfsd_file *nf, loff_t offset,
 				const struct xdr_buf *payload,
-				unsigned long *cnt, int stable, __be32 *verf);
+				unsigned long *cnt, int iocb_flags,
+				__be32 *verf);
 __be32		nfsd_readlink(struct svc_rqst *, struct svc_fh *,
 				char *, int *);
 __be32		nfsd_symlink(struct svc_rqst *, struct svc_fh *,
diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h
index df540c940cefe4..52660b9f8dde43 100644
--- a/fs/nfsd/xdr.h
+++ b/fs/nfsd/xdr.h
@@ -1,8 +1,8 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /* XDR types for nfsd. This is mainly a typing exercise. */
 
-#ifndef LINUX_NFSD_H
-#define LINUX_NFSD_H
+#ifndef _LINUX_NFSD_XDR_H
+#define _LINUX_NFSD_XDR_H
 
 #include <linux/vfs.h>
 #include "nfsd.h"
@@ -175,4 +175,4 @@ bool svcxdr_encode_stat(struct xdr_stream *xdr, __be32 status);
 bool svcxdr_encode_fattr(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 			 const struct svc_fh *fhp, const struct kstat *stat);
 
-#endif /* LINUX_NFSD_H */
+#endif /* _LINUX_NFSD_XDR_H */
diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h
index 344203874b4c47..cad875d1423136 100644
--- a/fs/nfsd/xdr3.h
+++ b/fs/nfsd/xdr3.h
@@ -39,7 +39,7 @@ struct nfsd3_writeargs {
 	svc_fh			fh;
 	__u64			offset;
 	__u32			count;
-	int			stable;
+	__u32			stable;
 	__u32			len;
 	struct xdr_buf		payload;
 };
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index c7eda5bc833b1a..ec198993429e8b 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -37,6 +37,8 @@
 #ifndef _LINUX_NFSD_XDR4_H
 #define _LINUX_NFSD_XDR4_H
 
+#include <linux/nfs_fh.h>
+
 #include "state.h"
 #include "vfs.h"
 
@@ -50,134 +52,6 @@
 #define HAS_CSTATE_FLAG(c, f) ((c)->sid_flags & (f))
 #define CLEAR_CSTATE_FLAG(c, f) ((c)->sid_flags &= ~(f))
 
-/**
- * nfsd4_encode_bool - Encode an XDR bool type result
- * @xdr: target XDR stream
- * @val: boolean value to encode
- *
- * Return values:
- *    %nfs_ok: @val encoded; @xdr advanced to next position
- *    %nfserr_resource: stream buffer space exhausted
- */
-static __always_inline __be32
-nfsd4_encode_bool(struct xdr_stream *xdr, bool val)
-{
-	__be32 *p = xdr_reserve_space(xdr, XDR_UNIT);
-
-	if (unlikely(p == NULL))
-		return nfserr_resource;
-	*p = val ? xdr_one : xdr_zero;
-	return nfs_ok;
-}
-
-/**
- * nfsd4_encode_uint32_t - Encode an XDR uint32_t type result
- * @xdr: target XDR stream
- * @val: integer value to encode
- *
- * Return values:
- *    %nfs_ok: @val encoded; @xdr advanced to next position
- *    %nfserr_resource: stream buffer space exhausted
- */
-static __always_inline __be32
-nfsd4_encode_uint32_t(struct xdr_stream *xdr, u32 val)
-{
-	__be32 *p = xdr_reserve_space(xdr, XDR_UNIT);
-
-	if (unlikely(p == NULL))
-		return nfserr_resource;
-	*p = cpu_to_be32(val);
-	return nfs_ok;
-}
-
-#define nfsd4_encode_aceflag4(x, v)	nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_acemask4(x, v)	nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_acetype4(x, v)	nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_count4(x, v)	nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_mode4(x, v)	nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_nfs_lease4(x, v)	nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_qop4(x, v)		nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_sequenceid4(x, v)	nfsd4_encode_uint32_t(x, v)
-#define nfsd4_encode_slotid4(x, v)	nfsd4_encode_uint32_t(x, v)
-
-/**
- * nfsd4_encode_uint64_t - Encode an XDR uint64_t type result
- * @xdr: target XDR stream
- * @val: integer value to encode
- *
- * Return values:
- *    %nfs_ok: @val encoded; @xdr advanced to next position
- *    %nfserr_resource: stream buffer space exhausted
- */
-static __always_inline __be32
-nfsd4_encode_uint64_t(struct xdr_stream *xdr, u64 val)
-{
-	__be32 *p = xdr_reserve_space(xdr, XDR_UNIT * 2);
-
-	if (unlikely(p == NULL))
-		return nfserr_resource;
-	put_unaligned_be64(val, p);
-	return nfs_ok;
-}
-
-#define nfsd4_encode_changeid4(x, v)	nfsd4_encode_uint64_t(x, v)
-#define nfsd4_encode_nfs_cookie4(x, v)	nfsd4_encode_uint64_t(x, v)
-#define nfsd4_encode_length4(x, v)	nfsd4_encode_uint64_t(x, v)
-#define nfsd4_encode_offset4(x, v)	nfsd4_encode_uint64_t(x, v)
-
-/**
- * nfsd4_encode_opaque_fixed - Encode a fixed-length XDR opaque type result
- * @xdr: target XDR stream
- * @data: pointer to data
- * @size: length of data in bytes
- *
- * Return values:
- *    %nfs_ok: @data encoded; @xdr advanced to next position
- *    %nfserr_resource: stream buffer space exhausted
- */
-static __always_inline __be32
-nfsd4_encode_opaque_fixed(struct xdr_stream *xdr, const void *data,
-			  size_t size)
-{
-	__be32 *p = xdr_reserve_space(xdr, xdr_align_size(size));
-	size_t pad = xdr_pad_size(size);
-
-	if (unlikely(p == NULL))
-		return nfserr_resource;
-	memcpy(p, data, size);
-	if (pad)
-		memset((char *)p + size, 0, pad);
-	return nfs_ok;
-}
-
-/**
- * nfsd4_encode_opaque - Encode a variable-length XDR opaque type result
- * @xdr: target XDR stream
- * @data: pointer to data
- * @size: length of data in bytes
- *
- * Return values:
- *    %nfs_ok: @data encoded; @xdr advanced to next position
- *    %nfserr_resource: stream buffer space exhausted
- */
-static __always_inline __be32
-nfsd4_encode_opaque(struct xdr_stream *xdr, const void *data, size_t size)
-{
-	size_t pad = xdr_pad_size(size);
-	__be32 *p;
-
-	p = xdr_reserve_space(xdr, XDR_UNIT + xdr_align_size(size));
-	if (unlikely(p == NULL))
-		return nfserr_resource;
-	*p++ = cpu_to_be32(size);
-	memcpy(p, data, size);
-	if (pad)
-		memset((char *)p + size, 0, pad);
-	return nfs_ok;
-}
-
-#define nfsd4_encode_component4(x, d, s)	nfsd4_encode_opaque(x, d, s)
-
 struct nfsd4_compound_state {
 	struct svc_fh		current_fh;
 	struct svc_fh		save_fh;
@@ -188,6 +62,7 @@ struct nfsd4_compound_state {
 	struct nfsd4_slot	*slot;
 	int			data_offset;
 	bool                    spo_must_allowed;
+	bool			slot_owned;
 	size_t			iovlen;
 	u32			minorversion;
 	__be32			status;
@@ -642,17 +517,6 @@ svcxdr_decode_deviceid4(__be32 *p, struct nfsd4_deviceid *devid)
 	return p;
 }
 
-static inline __be32
-nfsd4_decode_deviceid4(struct xdr_stream *xdr, struct nfsd4_deviceid *devid)
-{
-	__be32 *p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE);
-
-	if (unlikely(!p))
-		return nfserr_bad_xdr;
-	svcxdr_decode_deviceid4(p, devid);
-	return nfs_ok;
-}
-
 struct nfsd4_layout_seg {
 	u32			iomode;
 	u64			offset;
@@ -736,6 +600,19 @@ struct nfsd4_cb_offload {
 	u32			co_referring_seqno;
 };
 
+struct nfsd4_ssc_umount_item {
+	struct list_head	nsui_list;
+	bool			nsui_busy;
+	/*
+	 * nsui_refcnt inited to 2, 1 on list and 1 for consumer. Entry
+	 * is removed when refcnt drops to 1 and nsui_expire expires.
+	 */
+	refcount_t		nsui_refcnt;
+	unsigned long		nsui_expire;
+	struct vfsmount		*nsui_vfsmount;
+	char			nsui_ipaddr[RPC_MAX_ADDRBUFLEN + 1];
+};
+
 struct nfsd4_copy {
 	/* request */
 	stateid_t		cp_src_stateid;
@@ -924,6 +801,7 @@ bool nfsd4_cache_this_op(struct nfsd4_op *);
  */
 struct svcxdr_tmpbuf {
 	struct svcxdr_tmpbuf *next;
+	void (*release)(void *buf);
 	char buf[];
 };
 
diff --git a/fs/nfsd/xdr4cb.h b/fs/nfsd/xdr4cb.h
index b06d0170d7c43b..838f8629821f48 100644
--- a/fs/nfsd/xdr4cb.h
+++ b/fs/nfsd/xdr4cb.h
@@ -6,29 +6,30 @@
 #define cb_compound_enc_hdr_sz		4
 #define cb_compound_dec_hdr_sz		(3 + (NFS4_MAXTAGLEN >> 2))
 #define sessionid_sz			(NFS4_MAX_SESSIONID_LEN >> 2)
+#define op_enc_sz			1
 #define enc_referring_call4_sz		(1 + 1)
 #define enc_referring_call_list4_sz	(sessionid_sz + 1 + \
 					enc_referring_call4_sz)
-#define cb_sequence_enc_sz		(sessionid_sz + 4 +             \
-					enc_referring_call_list4_sz)
+#define cb_sequence_enc_sz		(op_enc_sz + sessionid_sz + 4 + \
+					1 + enc_referring_call_list4_sz)
 #define cb_sequence_dec_sz		(op_dec_sz + sessionid_sz + 4)
 
-#define op_enc_sz			1
 #define op_dec_sz			2
 #define enc_nfs4_fh_sz			(1 + (NFS4_FHSIZE >> 2))
 #define enc_stateid_sz			(NFS4_STATEID_SIZE >> 2)
 #define NFS4_enc_cb_recall_sz		(cb_compound_enc_hdr_sz +       \
 					cb_sequence_enc_sz +            \
-					1 + enc_stateid_sz +            \
-					enc_nfs4_fh_sz)
+					op_enc_sz + enc_stateid_sz +    \
+					1 + enc_nfs4_fh_sz)
 
 #define NFS4_dec_cb_recall_sz		(cb_compound_dec_hdr_sz  +      \
 					cb_sequence_dec_sz +            \
 					op_dec_sz)
 #define NFS4_enc_cb_layout_sz		(cb_compound_enc_hdr_sz +       \
 					cb_sequence_enc_sz +            \
-					1 + 3 +                         \
-					enc_nfs4_fh_sz + 4)
+					op_enc_sz + 3 + 1 +             \
+					enc_nfs4_fh_sz + 4 +            \
+					enc_stateid_sz)
 #define NFS4_dec_cb_layout_sz		(cb_compound_dec_hdr_sz  +      \
 					cb_sequence_dec_sz +            \
 					op_dec_sz)
@@ -47,7 +48,7 @@
 
 #define NFS4_enc_cb_notify_lock_sz	(cb_compound_enc_hdr_sz +        \
 					cb_sequence_enc_sz +             \
-					2 + 1 +				 \
+					op_enc_sz + 2 + 1 +		 \
 					XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + \
 					enc_nfs4_fh_sz)
 #define NFS4_dec_cb_notify_lock_sz	(cb_compound_dec_hdr_sz  +      \
@@ -57,6 +58,7 @@
 					XDR_QUADLEN(NFS4_VERIFIER_SIZE))
 #define NFS4_enc_cb_offload_sz		(cb_compound_enc_hdr_sz +       \
 					cb_sequence_enc_sz +            \
+					op_enc_sz +			\
 					enc_nfs4_fh_sz +		\
 					enc_stateid_sz +		\
 					enc_cb_offload_info_sz)
@@ -65,7 +67,7 @@
 					op_dec_sz)
 #define NFS4_enc_cb_recall_any_sz	(cb_compound_enc_hdr_sz +       \
 					cb_sequence_enc_sz +            \
-					1 + 1 + 1)
+					op_enc_sz + 1 + 1 + 1)
 #define NFS4_dec_cb_recall_any_sz	(cb_compound_dec_hdr_sz  +      \
 					cb_sequence_dec_sz +            \
 					op_dec_sz)
diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c
index 34e6096069ad67..8953719c4950cf 100644
--- a/fs/nilfs2/inode.c
+++ b/fs/nilfs2/inode.c
@@ -904,7 +904,7 @@ void nilfs_evict_inode(struct inode *inode)
 	 */
 }
 
-int nilfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int nilfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *iattr)
 {
 	struct nilfs_transaction_info ti;
@@ -943,7 +943,7 @@ int nilfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	return err;
 }
 
-int nilfs_permission(struct mnt_idmap *idmap, struct inode *inode,
+int nilfs_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		     int mask)
 {
 	struct nilfs_root *root = NILFS_I(inode)->i_root;
diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c
index 01a04080ef7045..f56a79767c69f9 100644
--- a/fs/nilfs2/ioctl.c
+++ b/fs/nilfs2/ioctl.c
@@ -135,7 +135,7 @@ int nilfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
  *
  * Return: 0 on success, or a negative error code on failure.
  */
-int nilfs_fileattr_set(struct mnt_idmap *idmap,
+int nilfs_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c
index e037e0c6e31a31..d0ae24f378545c 100644
--- a/fs/nilfs2/namei.c
+++ b/fs/nilfs2/namei.c
@@ -85,7 +85,7 @@ nilfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
  * If the create succeeds, we fill in the inode information
  * with d_instantiate().
  */
-static int nilfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int nilfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -113,7 +113,7 @@ static int nilfs_create(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int
-nilfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+nilfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 	    struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct inode *inode;
@@ -138,7 +138,7 @@ nilfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int nilfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int nilfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, const char *symname)
 {
 	struct nilfs_transaction_info ti;
@@ -218,7 +218,7 @@ static int nilfs_link(struct dentry *old_dentry, struct inode *dir,
 	return err;
 }
 
-static struct dentry *nilfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *nilfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -358,7 +358,7 @@ static int nilfs_rmdir(struct inode *dir, struct dentry *dentry)
 	return err;
 }
 
-static int nilfs_rename(struct mnt_idmap *idmap,
+static int nilfs_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h
index 4fc42d3787a441..28d35b2fa7f7a5 100644
--- a/fs/nilfs2/nilfs.h
+++ b/fs/nilfs2/nilfs.h
@@ -270,7 +270,7 @@ extern int nilfs_sync_file(struct file *, loff_t, loff_t, int);
 
 /* ioctl.c */
 int nilfs_fileattr_get(struct dentry *dentry, struct file_kattr *m);
-int nilfs_fileattr_set(struct mnt_idmap *idmap,
+int nilfs_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa);
 long nilfs_ioctl(struct file *, unsigned int, unsigned long);
 long nilfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
@@ -299,10 +299,10 @@ struct inode *nilfs_iget_for_shadow(struct inode *inode);
 extern void nilfs_update_inode(struct inode *, struct buffer_head *, int);
 extern void nilfs_truncate(struct inode *);
 extern void nilfs_evict_inode(struct inode *);
-extern int nilfs_setattr(struct mnt_idmap *, struct dentry *,
+extern int nilfs_setattr(const struct mnt_idmap *, struct dentry *,
 			 struct iattr *);
 extern void nilfs_write_failed(struct address_space *mapping, loff_t to);
-int nilfs_permission(struct mnt_idmap *idmap, struct inode *inode,
+int nilfs_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		     int mask);
 int nilfs_load_inode_block(struct inode *inode, struct buffer_head **pbh);
 extern int nilfs_inode_dirty(struct inode *);
diff --git a/fs/nls/nls_iso8859-14.c b/fs/nls/nls_iso8859-14.c
index c789eccb8a695c..60b9400f915af2 100644
--- a/fs/nls/nls_iso8859-14.c
+++ b/fs/nls/nls_iso8859-14.c
@@ -138,24 +138,23 @@ static const unsigned char page00[256] = {
 };
 
 static const unsigned char page01[256] = {
-	0x00, 0x00, 0xa1, 0xa2, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */
-	0x00, 0x00, 0xa6, 0xab, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */
+	0x00, 0x00, 0xa4, 0xa5, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0xb1, /* 0x18-0x1f */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */
 	0xb2, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */
-	0xb4, 0xb5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0xb9, /* 0x50-0x57 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */
-	0xbb, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */
-	0x00, 0x00, 0xd7, 0xf7, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */
 	0x00, 0x00, 0x00, 0x00, 0xd0, 0xf0, 0xde, 0xfe, /* 0x70-0x77 */
 	0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */
-	
-	0xa8, 0xb8, 0xaa, 0xba, 0xbd, 0xbe, 0x00, 0x00, /* 0x80-0x87 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */
@@ -169,7 +168,7 @@ static const unsigned char page01[256] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */
-	0x00, 0x00, 0xac, 0xbc, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */
 };
 
@@ -178,7 +177,7 @@ static const unsigned char page1e[256] = {
 	0x00, 0x00, 0xa6, 0xab, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0xb1, /* 0x18-0x1f */
-	0xb2, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */
@@ -188,9 +187,8 @@ static const unsigned char page1e[256] = {
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */
 	0xbb, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */
 	0x00, 0x00, 0xd7, 0xf7, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */
-	0x00, 0x00, 0x00, 0x00, 0xd0, 0xf0, 0xde, 0xfe, /* 0x70-0x77 */
-	0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */
-
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */
 	0xa8, 0xb8, 0xaa, 0xba, 0xbd, 0xbe, 0x00, 0x00, /* 0x80-0x87 */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */
 	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */
diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c
index a32c6634d59278..63c9759fc3b06c 100644
--- a/fs/notify/fanotify/fanotify_user.c
+++ b/fs/notify/fanotify/fanotify_user.c
@@ -1837,7 +1837,7 @@ static int fanotify_events_supported(struct fsnotify_group *group,
 	/*
 	 * mount and sb marks are not allowed on kernel internal pseudo fs,
 	 * like pipe_mnt, because that would subscribe to events on all the
-	 * anonynous pipes in the system.
+	 * anonymous pipes in the system.
 	 *
 	 * SB_NOUSER covers all of the internal pseudo fs whose objects are not
 	 * exposed to user's mount namespace, but there are other SB_KERNMOUNT
diff --git a/fs/nsfs.c b/fs/nsfs.c
index c3b6ae76594a81..56ea0bb9ef3a34 100644
--- a/fs/nsfs.c
+++ b/fs/nsfs.c
@@ -348,8 +348,8 @@ static long ns_ioctl(struct file *filp, unsigned int ioctl,
 			return ret;
 
 		FD_PREPARE(fdf, O_CLOEXEC, dentry_open(&path, O_RDONLY, current_cred()));
-		if (fdf.err)
-			return fdf.err;
+		if (fdf->fd < 0)
+			return fdf->fd;
 		/*
 		 * If @uinfo is passed return all information about the
 		 * mount namespace as well.
diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c
index b4fcfbe2da4c9b..ddc201e3e2aa76 100644
--- a/fs/ntfs/ea.c
+++ b/fs/ntfs/ea.c
@@ -875,7 +875,7 @@ static int ntfs_validate_fattr(struct ntfs_inode *ni, __le32 fattr)
 }
 
 static int ntfs_setxattr(const struct xattr_handler *handler,
-		struct mnt_idmap *idmap, struct dentry *unused,
+		const struct mnt_idmap *idmap, struct dentry *unused,
 		struct inode *inode, const char *name, const void *value,
 		size_t size, int flags)
 {
@@ -977,7 +977,7 @@ const struct xattr_handler * const ntfs_xattr_handlers[] = {
 // clang-format on
 
 #ifdef CONFIG_NTFS_FS_POSIX_ACL
-struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+struct posix_acl *ntfs_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			       int type)
 {
 	struct inode *inode = d_inode(dentry);
@@ -1021,7 +1021,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 	return acl;
 }
 
-static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap,
+static noinline int ntfs_set_acl_ex(const struct mnt_idmap *idmap,
 				    struct inode *inode, struct posix_acl *acl,
 				    int type, bool init_acl)
 {
@@ -1103,13 +1103,13 @@ static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap,
 	return err;
 }
 
-int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 	return ntfs_set_acl_ex(idmap, d_inode(dentry), acl, type, false);
 }
 
-int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode,
+int ntfs_init_acl(const struct mnt_idmap *idmap, struct inode *inode,
 		  struct inode *dir)
 {
 	struct posix_acl *default_acl, *acl;
diff --git a/fs/ntfs/ea.h b/fs/ntfs/ea.h
index acb39c2a6fbc86..690fafe181fb01 100644
--- a/fs/ntfs/ea.h
+++ b/fs/ntfs/ea.h
@@ -17,11 +17,11 @@ int ntfs_ea_set_wsl_inode(struct inode *inode, dev_t rdev, __le16 *ea_size,
 ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size);
 
 #ifdef CONFIG_NTFS_FS_POSIX_ACL
-struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+struct posix_acl *ntfs_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			       int type);
-int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
-int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode,
+int ntfs_init_acl(const struct mnt_idmap *idmap, struct inode *inode,
 		  struct inode *dir);
 #else
 #define ntfs_get_acl NULL
diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c
index 3ec82715a58828..bb8641103d2681 100644
--- a/fs/ntfs/file.c
+++ b/fs/ntfs/file.c
@@ -318,7 +318,7 @@ static int ntfs_setattr_size(struct inode *vi, struct iattr *attr)
  * NOTE: Changes in inode size are not supported yet for compressed or
  * encrypted files.
  */
-int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr)
 {
 	struct inode *vi = d_inode(dentry);
@@ -387,7 +387,7 @@ int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	return err;
 }
 
-int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ntfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, unsigned int request_mask,
 		unsigned int query_flags)
 {
diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h
index ff61bd402df033..45a396c9784688 100644
--- a/fs/ntfs/inode.h
+++ b/fs/ntfs/inode.h
@@ -330,9 +330,9 @@ int ntfs_read_inode_mount(struct inode *vi);
 int ntfs_show_options(struct seq_file *sf, struct dentry *root);
 int ntfs_truncate_vfs(struct inode *vi, loff_t new_size, loff_t i_size);
 
-int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr);
-int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ntfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, unsigned int request_mask,
 		unsigned int query_flags);
 
diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c
index 75e20109652573..028efad7d9c0f4 100644
--- a/fs/ntfs/namei.c
+++ b/fs/ntfs/namei.c
@@ -391,7 +391,7 @@ static int ntfs_sd_add_everyone(struct ntfs_inode *ni)
 	return ret;
 }
 
-static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static struct ntfs_inode *__ntfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		__le16 *name, u8 name_len, mode_t mode, dev_t dev,
 		const char *target, int target_len)
 {
@@ -731,7 +731,7 @@ static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *d
 	return ERR_PTR(err);
 }
 
-static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int ntfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *dentry, umode_t mode)
 {
 	struct ntfs_volume *vol = NTFS_SB(dir->i_sb);
@@ -1046,7 +1046,7 @@ static int ntfs_unlink(struct inode *dir, struct dentry *dentry)
 	return err;
 }
 
-static struct dentry *ntfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ntfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *dentry, umode_t mode)
 {
 	struct super_block *sb = dir->i_sb;
@@ -1243,7 +1243,7 @@ static int __ntfs_link(struct ntfs_inode *ni, struct ntfs_inode *dir_ni,
 	return err;
 }
 
-static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int ntfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		struct dentry *old_dentry, struct inode *new_dir,
 		struct dentry *new_dentry, unsigned int flags)
 {
@@ -1393,7 +1393,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 	return err;
 }
 
-static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int ntfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *dentry, const char *symname)
 {
 	struct super_block *sb = dir->i_sb;
@@ -1440,7 +1440,7 @@ static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int ntfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int ntfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct super_block *sb = dir->i_sb;
diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c
index b1c315206ffafe..3b4ef15162da79 100644
--- a/fs/ntfs3/attrib.c
+++ b/fs/ntfs3/attrib.c
@@ -61,11 +61,16 @@ static int attr_load_runs(struct ATTRIB *attr, struct ntfs_inode *ni,
 			  struct runs_tree *run, const CLST *vcn)
 {
 	int err;
-	CLST svcn = le64_to_cpu(attr->nres.svcn);
-	CLST evcn = le64_to_cpu(attr->nres.evcn);
+	CLST svcn, evcn;
 	u32 asize;
 	u16 run_off;
 
+	if (!attr->non_res)
+		return -EIO;
+
+	svcn = le64_to_cpu(attr->nres.svcn);
+	evcn = le64_to_cpu(attr->nres.evcn);
+
 	if (svcn >= evcn + 1 || run_is_mapped_full(run, svcn, evcn))
 		return 0;
 
@@ -394,13 +399,15 @@ static int attr_set_size_res(struct ntfs_inode *ni, struct ATTRIB *attr,
 	char *next = Add2Ptr(attr, asize);
 	s64 dsize = ALIGN(new_size, 8) - ALIGN(rsize, 8);
 
+	if (new_size > sbi->record_size ||
+	    (dsize > 0 && used + dsize > sbi->max_bytes_per_attr)) {
+		return attr_make_nonresident(ni, attr, le, mi, new_size, run,
+					     ins_attr, NULL);
+	}
+
 	if (dsize < 0) {
 		memmove(next + dsize, next, tail);
 	} else if (dsize > 0) {
-		if (used + dsize > sbi->max_bytes_per_attr)
-			return attr_make_nonresident(ni, attr, le, mi, new_size,
-						     run, ins_attr, NULL);
-
 		memmove(next + dsize, next, tail);
 		memset(next, 0, dsize);
 	}
@@ -556,6 +563,10 @@ int attr_set_size_ex(struct ntfs_inode *ni, enum ATTR_TYPE type,
 		}
 
 next_le_1:
+		if (!attr->non_res) {
+			err = -EIO;
+			goto out;
+		}
 		svcn = le64_to_cpu(attr->nres.svcn);
 		evcn = le64_to_cpu(attr->nres.evcn);
 	}
@@ -1034,6 +1045,21 @@ int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen,
 
 	if (!attr_b->non_res) {
 		u32 data_size = le32_to_cpu(attr_b->res.data_size);
+
+		/*
+		 * A resident attribute is copied into a single page below
+		 * (alloc_page() + memcpy()) and mapped as a one-page
+		 * IOMAP_INLINE extent by ntfs_iomap_begin().  mi_enum_attr()
+		 * only bounds the resident value length against the MFT record
+		 * size, so a corrupted volume whose records are larger than a
+		 * page can report data_size > PAGE_SIZE; copying that many bytes
+		 * would overflow the single destination page.  Reject it.
+		 */
+		if (data_size > PAGE_SIZE) {
+			err = -EINVAL;
+			goto out;
+		}
+
 		*lcn = RESIDENT_LCN;
 		*len = data_size;
 		if (res) {
@@ -1456,6 +1482,9 @@ int attr_load_runs_vcn(struct ntfs_inode *ni, enum ATTR_TYPE type,
 		return -ENOENT;
 	}
 
+	if (!attr->non_res)
+		return -EIO;
+
 	svcn = le64_to_cpu(attr->nres.svcn);
 	evcn = le64_to_cpu(attr->nres.evcn);
 
@@ -1995,7 +2024,7 @@ int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size,
 
 		valid_size = le64_to_cpu(attr_b->nres.valid_size);
 		if (new_valid != valid_size) {
-			attr_b->nres.valid_size = cpu_to_le64(valid_size);
+			attr_b->nres.valid_size = cpu_to_le64(new_valid);
 			mi_b->dirty = true;
 		}
 	}
@@ -2610,8 +2639,8 @@ int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes)
 			char *data = Add2Ptr(attr_b,
 					     le16_to_cpu(attr_b->res.data_off));
 
-			memmove(data + bytes, data, bytes);
-			memset(data, 0, bytes);
+			memmove(data + vbo + bytes, data + vbo, data_size - vbo);
+			memset(data + vbo, 0, bytes);
 			goto done;
 		}
 
diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c
index eb9152e9fa22fa..328c9a29df89cb 100644
--- a/fs/ntfs3/dir.c
+++ b/fs/ntfs3/dir.c
@@ -34,6 +34,9 @@ int ntfs_utf16_to_nls(struct ntfs_sb_info *sbi, const __le16 *name, u32 len,
 		/* UTF-16 -> UTF-8 */
 		ret = utf16s_to_utf8s((wchar_t *)name, len, UTF16_LITTLE_ENDIAN,
 				      buf, buf_len);
+        if (ret >= buf_len) {
+            ret = buf_len-1;
+        }
 		buf[ret] = '\0';
 		return ret;
 	}
diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c
index 2abf334bfa0cf9..4cbdd9e4222b7f 100644
--- a/fs/ntfs3/file.c
+++ b/fs/ntfs3/file.c
@@ -129,7 +129,7 @@ int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 /*
  * ntfs_fileattr_set - inode_operations::fileattr_set
  */
-int ntfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_fileattr_set(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
@@ -183,6 +183,9 @@ static int ntfs_ioctl_set_volume_label(struct ntfs_sb_info *sbi, u8 __user *buf)
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 
+	if (sb_rdonly(sbi->sb))
+		return -EROFS;
+
 	if (copy_from_user(user, buf, FSLABEL_MAX))
 		return -EFAULT;
 
@@ -261,7 +264,7 @@ long ntfs_compat_ioctl(struct file *filp, u32 cmd, unsigned long arg)
 /*
  * ntfs_getattr - inode_operations::getattr
  */
-int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ntfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, u32 flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -703,7 +706,7 @@ static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len)
 /*
  * ntfs_setattr - inode_operations::setattr
  */
-int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c
index bead01a953f3fe..4651dd8f6464cd 100644
--- a/fs/ntfs3/frecord.c
+++ b/fs/ntfs3/frecord.c
@@ -2190,10 +2190,13 @@ int ni_decompress_file(struct ntfs_inode *ni)
 
 	/* Clear cached flag. */
 	ni->ni_flags &= ~NI_FLAG_COMPRESSED_MASK;
+	/* offs_folio is accessed under run_lock in attr_wof_frame_info(). */
+	down_write(&ni->file.run_lock);
 	if (ni->file.offs_folio) {
 		folio_put(ni->file.offs_folio);
 		ni->file.offs_folio = NULL;
 	}
+	up_write(&ni->file.run_lock);
 	mapping->a_ops = &ntfs_aops;
 
 out:
@@ -2521,6 +2524,8 @@ int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages,
 out:
 	for (i = 0; i < pages_per_frame; i++) {
 		pg = pages[i];
+		if (err)
+			clear_highpage(pg);
 		SetPageUptodate(pg);
 	}
 
diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c
index ed50c1d0c23e35..9a44b322ad60e8 100644
--- a/fs/ntfs3/fslog.c
+++ b/fs/ntfs3/fslog.c
@@ -7,6 +7,7 @@
 
 #include <linux/blkdev.h>
 #include <linux/fs.h>
+#include <linux/overflow.h>
 #include <linux/random.h>
 #include <linux/slab.h>
 
@@ -2457,7 +2458,8 @@ static int find_log_rec(struct ntfs_log *log, u64 lsn, struct lcb *lcb)
 	 * Check that the length field isn't greater than the total
 	 * available space the log file.
 	 */
-	rec_len = len + log->record_header_len;
+	if (check_add_overflow(len, log->record_header_len, &rec_len))
+		return -EINVAL;
 	if (rec_len >= log->total_avail)
 		return -EINVAL;
 
diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c
index 97c04ab2763abb..b55311ccd229a3 100644
--- a/fs/ntfs3/fsntfs.c
+++ b/fs/ntfs3/fsntfs.c
@@ -2153,6 +2153,9 @@ int ntfs_insert_security(struct ntfs_sb_info *sbi,
 		goto out;
 
 	while (e) {
+		if (le16_to_cpu(e->de.size) < SIZEOF_SDH_DIRENTRY)
+			break;
+
 		if (le32_to_cpu(e->sec_hdr.size) == new_sec_size) {
 			err = ntfs_read_run_nb(sbi, &ni->file.run,
 					       le64_to_cpu(e->sec_hdr.off),
diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c
index 689712d3463d41..37e3e16b1f8412 100644
--- a/fs/ntfs3/index.c
+++ b/fs/ntfs3/index.c
@@ -594,6 +594,8 @@ static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr,
 
 	if (!e)
 		return NULL;
+	if (size_add(used, ins_bytes) > le32_to_cpu(hdr->total))
+		return NULL;
 
 	/* Now we just make room for the inserted entries and jam it in. */
 	to_move = used - le32_to_cpu(hdr->de_off);
@@ -1801,7 +1803,10 @@ static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni,
 	}
 
 	/* Copy root entries into new buffer. */
-	hdr_insert_head(hdr, re, to_move);
+	if (!hdr_insert_head(hdr, re, to_move)) {
+		err = -EINVAL;
+		goto out_put_n;
+	}
 
 	/* Update bitmap attribute. */
 	indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits);
@@ -1955,7 +1960,11 @@ static int indx_insert_into_buffer(struct ntfs_index *indx,
 	/* Copy all the entries <= sp into the new buffer. */
 	de_t = hdr_first_de(hdr1);
 	to_copy = PtrOffset(de_t, sp);
-	hdr_insert_head(hdr2, de_t, to_copy);
+	if (!hdr_insert_head(hdr2, de_t, to_copy)) {
+		err = -EINVAL;
+		put_indx_node(n2);
+		goto out;
+	}
 
 	/* Remove all entries (sp including) from hdr1. */
 	used = used1 - to_copy - sp_size;
diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c
index 56b4f6469a280b..366af303824850 100644
--- a/fs/ntfs3/inode.c
+++ b/fs/ntfs3/inode.c
@@ -213,6 +213,11 @@ static int ntfs_read_mft(struct inode *inode, const struct cpu_str *name,
 
 		names += 1;
 		fname = Add2Ptr(attr, roff);
+
+		/* Make sure the full name fits in the resident data. */
+		if (rsize < fname_full_size(fname))
+			goto out;
+
 		if (fname->type == FILE_NAME_DOS)
 			goto next_attr;
 
@@ -280,7 +285,9 @@ static int ntfs_read_mft(struct inode *inode, const struct cpu_str *name,
 		break;
 
 	case ATTR_ROOT:
-		if (attr->non_res)
+		if (attr->non_res ||
+		    asize < sizeof(struct INDEX_ROOT) + roff ||
+		    rsize < sizeof(struct INDEX_ROOT))
 			goto out;
 
 		root = Add2Ptr(attr, roff);
@@ -981,6 +988,8 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 	if (err) {
 		return err;
 	}
+	if (!clen)
+		return -EINVAL;
 
 	if (lcn == EOF_LCN) {
 		/* request out of file. */
@@ -1016,11 +1025,6 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 		return 0;
 	}
 
-	if (!clen) {
-		/* broken file? */
-		return -EINVAL;
-	}
-
 	iomap->bdev = inode->i_sb->s_bdev;
 	iomap->offset = offset;
 	iomap->length = ((loff_t)clen << cluster_bits) - off;
@@ -1381,7 +1385,7 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname,
  *
  * NOTE: if fnd != NULL (ntfs_atomic_open) then @dir is locked
  */
-int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir,
+int ntfs_create_inode(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, const struct cpu_str *uni,
 		      umode_t mode, dev_t dev, const char *symname, u32 size,
 		      struct ntfs_fnd *fnd)
@@ -1866,10 +1870,10 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir,
 		goto out6;
 
 	/*
-	 * Call 'd_instantiate' after inode->i_op is set
+	 * Call 'd_instantiate_new' after inode->i_op is set
 	 * but before finish_open.
 	 */
-	d_instantiate(dentry, inode);
+	d_instantiate_new(dentry, inode);
 
 	/* Set original time. inode times (i_ctime) may be changed in ntfs_init_acl. */
 	inode_set_atime_to_ts(inode, ni->i_crtime);
@@ -1917,9 +1921,6 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir,
 	if (!fnd)
 		ni_unlock(dir_ni);
 
-	if (!err)
-		unlock_new_inode(inode);
-
 	return err;
 }
 
@@ -2247,6 +2248,9 @@ static noinline int ntfs_readlink_hlp(const struct dentry *link_de,
 	if (err < 0)
 		goto out;
 
+	if (err >= buflen)
+		err = buflen - 1;
+
 	/* Translate Windows '\' into Linux '/'. */
 	for (i = 0; i < err; i++) {
 		if (buffer[i] == '\\')
diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c
index ec59bbabd3c50b..ae88cb66fb7e74 100644
--- a/fs/ntfs3/namei.c
+++ b/fs/ntfs3/namei.c
@@ -111,7 +111,7 @@ static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry,
 /*
  * ntfs_create - inode_operations::create
  */
-static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int ntfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	return ntfs_create_inode(idmap, dir, dentry, NULL, S_IFREG | mode, 0,
@@ -121,7 +121,7 @@ static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir,
 /*
  * ntfs_mknod - inode_operations::mknod
  */
-static int ntfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int ntfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	return ntfs_create_inode(idmap, dir, dentry, NULL, mode, rdev, NULL, 0,
@@ -209,7 +209,7 @@ static int ntfs_unlink(struct inode *dir, struct dentry *dentry)
 /*
  * ntfs_symlink - inode_operations::symlink
  */
-static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int ntfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, const char *symname)
 {
 	u32 size = strlen(symname);
@@ -228,7 +228,7 @@ static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 /*
  * ntfs_mkdir - inode_operations::mkdir
  */
-static struct dentry *ntfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ntfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	return ERR_PTR(ntfs_create_inode(idmap, dir, dentry, NULL,
@@ -262,7 +262,7 @@ static int ntfs_rmdir(struct inode *dir, struct dentry *dentry)
 /*
  * ntfs_rename - inode_operations::rename
  */
-static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir,
+static int ntfs_rename(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, struct inode *new_dir,
 		       struct dentry *new_dentry, u32 flags)
 {
diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h
index 5811d89d67b394..ea24126c70db43 100644
--- a/fs/ntfs3/ntfs_fs.h
+++ b/fs/ntfs3/ntfs_fs.h
@@ -551,11 +551,11 @@ extern const struct file_operations ntfs_dir_operations;
 
 /* Globals from file.c */
 int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int ntfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_fileattr_set(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      struct file_kattr *fa);
-int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ntfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, u32 flags);
-int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr);
 int ntfs_file_open(struct inode *inode, struct file *file);
 int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
@@ -803,7 +803,7 @@ int ntfs_set_size(struct inode *inode, u64 new_size);
 int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc);
 int ntfs_sync_inode(struct inode *inode);
 int inode_read_data(struct inode *inode, void *data, size_t bytes);
-int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir,
+int ntfs_create_inode(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, const struct cpu_str *uni,
 		      umode_t mode, dev_t dev, const char *symname, u32 size,
 		      struct ntfs_fnd *fnd);
@@ -963,18 +963,18 @@ unsigned long ntfs_names_hash(const u16 *name, size_t len, const u16 *upcase,
 
 /* globals from xattr.c */
 #ifdef CONFIG_NTFS3_FS_POSIX_ACL
-struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+struct posix_acl *ntfs_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			       int type);
-int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
-int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode,
+int ntfs_init_acl(const struct mnt_idmap *idmap, struct inode *inode,
 		  struct inode *dir);
 #else
 #define ntfs_get_acl NULL
 #define ntfs_set_acl NULL
 #endif
 
-int ntfs_acl_chmod(struct mnt_idmap *idmap, struct dentry *dentry);
+int ntfs_acl_chmod(const struct mnt_idmap *idmap, struct dentry *dentry);
 ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size);
 extern const struct xattr_handler *const ntfs_xattr_handlers[];
 
diff --git a/fs/ntfs3/record.c b/fs/ntfs3/record.c
index 4f12ce15b03b43..d7785f9e8e092f 100644
--- a/fs/ntfs3/record.c
+++ b/fs/ntfs3/record.c
@@ -665,18 +665,32 @@ int mi_pack_runs(struct mft_inode *mi, struct ATTRIB *attr,
 	u32 run_size = asize - run_off;
 	u32 tail = used - aoff - asize;
 	u32 dsize = sbi->record_size - used;
+	u32 avail;
 
 	/* Make a maximum gap in current record. */
 	memmove(next + dsize, next, tail);
 
+	/* Leave room for the 8-byte ALIGN() to avoid OOB write */
+	avail = (run_size + dsize) & ~7u;
+
+	if (!avail) {
+		memmove(next, next + dsize, tail);
+		return -ENOSPC;
+	}
+
 	/* Pack as much as possible. */
-	err = run_pack(run, svcn, len, Add2Ptr(attr, run_off), run_size + dsize,
+	err = run_pack(run, svcn, len, Add2Ptr(attr, run_off), avail,
 		       &plen);
 	if (err < 0) {
 		memmove(next, next + dsize, tail);
 		return err;
 	}
 
+	if (!plen) {
+		memmove(next, next + dsize, tail);
+		return -ENOSPC;
+	}
+
 	new_run_size = ALIGN(err, 8);
 
 	memmove(next + new_run_size - run_size, next + dsize, tail);
diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c
index f4a42a0c73a444..d3093f72ea381f 100644
--- a/fs/ntfs3/super.c
+++ b/fs/ntfs3/super.c
@@ -1210,7 +1210,7 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size,
 
 #ifdef CONFIG_NTFS3_64BIT_CLUSTER
 	if (clusters >= (1ull << (64 - cluster_bits)))
-		sbi->maxbytes = -1;
+		sbi->maxbytes = MAX_LFS_FILESIZE;
 	sbi->maxbytes_sparse = MAX_LFS_FILESIZE;
 	sb->s_maxbytes = MAX_LFS_FILESIZE;
 #else
diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c
index 594ef6860b93bf..e9824bd8032264 100644
--- a/fs/ntfs3/xattr.c
+++ b/fs/ntfs3/xattr.c
@@ -165,6 +165,8 @@ static int ntfs_read_ea(struct ntfs_inode *ni, struct EA_FULL **ea,
 			/* ef->size must fit the list and cover the record. */
 			if (ea_size > bytes || ea_size < need)
 				goto out1;
+			if (bytes < offsetof(struct EA_FULL, name))
+				goto out1;
 			continue;
 		}
 
@@ -543,7 +545,7 @@ static noinline int ntfs_set_ea(struct inode *inode, const char *name,
 /*
  * ntfs_get_acl - inode_operations::get_acl
  */
-struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+struct posix_acl *ntfs_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			       int type)
 {
 	struct inode *inode = d_inode(dentry);
@@ -595,7 +597,7 @@ struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 	return acl;
 }
 
-static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap,
+static noinline int ntfs_set_acl_ex(const struct mnt_idmap *idmap,
 				    struct inode *inode, struct posix_acl *acl,
 				    int type, bool init_acl)
 {
@@ -677,7 +679,7 @@ static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap,
 /*
  * ntfs_set_acl - inode_operations::set_acl
  */
-int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ntfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 	return ntfs_set_acl_ex(idmap, d_inode(dentry), acl, type, false);
@@ -688,7 +690,7 @@ int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
  *
  * Called from ntfs_create_inode().
  */
-int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode,
+int ntfs_init_acl(const struct mnt_idmap *idmap, struct inode *inode,
 		  struct inode *dir)
 {
 	struct posix_acl *default_acl, *acl;
@@ -722,7 +724,7 @@ int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode,
 /*
  * ntfs_acl_chmod - Helper for ntfs_setattr().
  */
-int ntfs_acl_chmod(struct mnt_idmap *idmap, struct dentry *dentry)
+int ntfs_acl_chmod(const struct mnt_idmap *idmap, struct dentry *dentry)
 {
 	struct inode *inode = d_inode(dentry);
 	struct super_block *sb = inode->i_sb;
@@ -783,7 +785,7 @@ static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de,
 		if (!buffer) {
 			err = sizeof(u8);
 		} else if (size < sizeof(u8)) {
-			err = -ENODATA;
+			err = -ERANGE;
 		} else {
 			err = sizeof(u8);
 			*(u8 *)buffer = le32_to_cpu(ni->std_fa);
@@ -797,7 +799,7 @@ static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de,
 		if (!buffer) {
 			err = sizeof(u32);
 		} else if (size < sizeof(u32)) {
-			err = -ENODATA;
+			err = -ERANGE;
 		} else {
 			err = sizeof(u32);
 			*(u32 *)buffer = le32_to_cpu(ni->std_fa);
@@ -837,7 +839,7 @@ static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de,
 		if (!buffer) {
 			err = sd_size;
 		} else if (size < sd_size) {
-			err = -ENODATA;
+			err = -ERANGE;
 		} else {
 			err = sd_size;
 			memcpy(buffer, sd, sd_size);
@@ -863,7 +865,7 @@ static bool ntfs_is_reserved_lxattr(const char *name)
  * ntfs_setxattr - inode_operations::setxattr
  */
 static noinline int ntfs_setxattr(const struct xattr_handler *handler,
-				  struct mnt_idmap *idmap, struct dentry *de,
+				  const struct mnt_idmap *idmap, struct dentry *de,
 				  struct inode *inode, const char *name,
 				  const void *value, size_t size, int flags)
 {
@@ -976,8 +978,10 @@ static noinline int ntfs_setxattr(const struct xattr_handler *handler,
 			  NULL);
 
 out:
-	inode_set_ctime_current(inode);
-	mark_inode_dirty(inode);
+	if (!err) {
+		inode_set_ctime_current(inode);
+		mark_inode_dirty(inode);
+	}
 
 	return err;
 }
diff --git a/fs/ocfs2/acl.c b/fs/ocfs2/acl.c
index 090ec60fb576a6..801a2f56ad08c4 100644
--- a/fs/ocfs2/acl.c
+++ b/fs/ocfs2/acl.c
@@ -260,7 +260,7 @@ static int ocfs2_set_acl(handle_t *handle,
 	return ret;
 }
 
-int ocfs2_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ocfs2_iop_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      struct posix_acl *acl, int type)
 {
 	struct buffer_head *bh = NULL;
diff --git a/fs/ocfs2/acl.h b/fs/ocfs2/acl.h
index a91f9ce278d6d3..1ed05899cce13b 100644
--- a/fs/ocfs2/acl.h
+++ b/fs/ocfs2/acl.h
@@ -17,7 +17,7 @@ struct ocfs2_acl_entry {
 };
 
 struct posix_acl *ocfs2_iop_get_acl(struct inode *inode, int type, bool rcu);
-int ocfs2_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ocfs2_iop_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      struct posix_acl *acl, int type);
 extern int ocfs2_acl_chmod(struct inode *, struct buffer_head *);
 struct ocfs2_acl_state {
diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c
index be09e766ac1fc9..fc949e59462699 100644
--- a/fs/ocfs2/alloc.c
+++ b/fs/ocfs2/alloc.c
@@ -925,6 +925,33 @@ static int ocfs2_validate_extent_block(struct super_block *sb,
 		goto bail;
 	}
 
+	/*
+	 * Extent blocks are allocated from a per-slot suballocator, so the
+	 * slot must be in range.  Otherwise freeing the block passes it to
+	 * get_local_system_inode(), which hits BUG_ON() for
+	 * OCFS2_INVALID_SLOT or computes an out-of-bounds index otherwise.
+	 */
+	if ((u32)le16_to_cpu(eb->h_suballoc_slot) >= OCFS2_SB(sb)->max_slots) {
+		rc = ocfs2_error(sb,
+				 "Extent block #%llu has an invalid h_suballoc_slot of %u\n",
+				 (unsigned long long)bh->b_blocknr,
+				 le16_to_cpu(eb->h_suballoc_slot));
+		goto bail;
+	}
+
+	/*
+	 * Similarly the suballoc bit must fit in a block group bitmap.
+	 * Otherwise freeing the block will pass the oversized bit to
+	 * _ocfs2_free_suballoc_bits() and trigger ocfs2_error() there.
+	 */
+	if (le16_to_cpu(eb->h_suballoc_bit) >= ocfs2_suballoc_bits_per_block(sb)) {
+		rc = ocfs2_error(sb,
+				 "Extent block #%llu has an invalid h_suballoc_bit of %u\n",
+				 (unsigned long long)bh->b_blocknr,
+				 le16_to_cpu(eb->h_suballoc_bit));
+		goto bail;
+	}
+
 	if (le16_to_cpu(eb->h_list.l_count) != ocfs2_extent_recs_per_eb(sb)) {
 		rc = ocfs2_error(sb,
 				 "Extent block #%llu has invalid l_count %u (expected %u)\n",
@@ -1817,6 +1844,7 @@ static int __ocfs2_find_path(struct ocfs2_caching_info *ci,
 	int i, ret = 0;
 	u32 range;
 	u64 blkno;
+	u32 prev_depth = OCFS2_MAX_PATH_DEPTH;
 	struct buffer_head *bh = NULL;
 	struct ocfs2_extent_block *eb;
 	struct ocfs2_extent_list *el;
@@ -1824,14 +1852,16 @@ static int __ocfs2_find_path(struct ocfs2_caching_info *ci,
 
 	el = root_el;
 	while (el->l_tree_depth) {
-		if (unlikely(le16_to_cpu(el->l_tree_depth) >= OCFS2_MAX_PATH_DEPTH)) {
+		if (unlikely(le16_to_cpu(el->l_tree_depth) >= prev_depth)) {
 			ocfs2_error(ocfs2_metadata_cache_get_super(ci),
-				    "Owner %llu has invalid tree depth %u in extent list\n",
+				    "Owner %llu has invalid tree depth %u in extent list (max %u)\n",
 				    (unsigned long long)ocfs2_metadata_cache_owner(ci),
-				    le16_to_cpu(el->l_tree_depth));
+				    le16_to_cpu(el->l_tree_depth), prev_depth - 1);
 			ret = -EROFS;
 			goto out;
 		}
+		prev_depth = le16_to_cpu(el->l_tree_depth);
+
 		if (!el->l_next_free_rec || !el->l_count) {
 			ocfs2_error(ocfs2_metadata_cache_get_super(ci),
 				    "Owner %llu has empty extent list at depth %u\n"
diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c
index 7bfe377af2dfcf..733ceda79ca1f1 100644
--- a/fs/ocfs2/buffer_head_io.c
+++ b/fs/ocfs2/buffer_head_io.c
@@ -66,12 +66,14 @@ int ocfs2_write_block(struct ocfs2_super *osb, struct buffer_head *bh,
 
 	wait_on_buffer(bh);
 
-	if (buffer_uptodate(bh)) {
+	if (!buffer_write_io_error(bh)) {
 		ocfs2_set_buffer_uptodate(ci, bh);
 	} else {
-		/* We don't need to remove the clustered uptodate
-		 * information for this bh as it's not marked locally
-		 * uptodate. */
+		/*
+		 * The buffer still holds what we tried to write, but it did
+		 * not reach the disk, so don't advertise it to the cluster
+		 * as up to date.
+		 */
 		ret = -EIO;
 		mlog_errno(ret);
 	}
@@ -446,7 +448,7 @@ int ocfs2_write_super_or_backup(struct ocfs2_super *osb,
 
 	wait_on_buffer(bh);
 
-	if (!buffer_uptodate(bh)) {
+	if (buffer_write_io_error(bh)) {
 		ret = -EIO;
 		mlog_errno(ret);
 	}
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c
index 1c3def99bb0765..a4c8ea695f5cf1 100644
--- a/fs/ocfs2/cluster/heartbeat.c
+++ b/fs/ocfs2/cluster/heartbeat.c
@@ -1966,18 +1966,22 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,
 	atomic_set(&reg->hr_unsteady_iterations, (live_threshold * 3));
 	o2hb_set_region_stopping(reg, false);
 
-	hb_task = kthread_run(o2hb_thread, reg, "o2hb-%s",
-			      reg->hr_item.ci_name);
+	hb_task = kthread_create(o2hb_thread, reg, "o2hb-%s",
+				 reg->hr_item.ci_name);
 	if (IS_ERR(hb_task)) {
 		ret = PTR_ERR(hb_task);
 		mlog_errno(ret);
 		goto out;
 	}
+	/* The thread may exit on its own, so pin it before it can run. */
+	get_task_struct(hb_task);
 
 	spin_lock(&o2hb_live_lock);
 	reg->hr_task = hb_task;
 	spin_unlock(&o2hb_live_lock);
 
+	wake_up_process(hb_task);
+
 	ret = wait_event_interruptible(o2hb_steady_queue,
 				atomic_read(&reg->hr_steady_iterations) == 0 ||
 				reg->hr_node_deleted);
@@ -2022,7 +2026,7 @@ static ssize_t o2hb_region_dev_store(struct config_item *item,
 		spin_unlock(&o2hb_live_lock);
 
 		if (hb_task)
-			kthread_stop(hb_task);
+			kthread_stop_put(hb_task);
 
 		o2hb_unmap_slot_data(reg);
 
@@ -2208,7 +2212,7 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group,
 	spin_unlock(&o2hb_live_lock);
 
 	if (hb_task)
-		kthread_stop(hb_task);
+		kthread_stop_put(hb_task);
 
 	if (o2hb_global_heartbeat_active()) {
 		spin_lock(&o2hb_live_lock);
diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c
index 0075e1624310e8..55c4a305a2823e 100644
--- a/fs/ocfs2/dir.c
+++ b/fs/ocfs2/dir.c
@@ -605,6 +605,49 @@ static int ocfs2_validate_dx_root(struct super_block *sb,
 		goto bail;
 	}
 
+	if (le64_to_cpu(dx_root->dr_blkno) != bh->b_blocknr) {
+		ret = ocfs2_error(sb,
+				  "Dir Index Root # %llu has an invalid dr_blkno of %llu\n",
+				  (unsigned long long)bh->b_blocknr,
+				  (unsigned long long)le64_to_cpu(dx_root->dr_blkno));
+		goto bail;
+	}
+
+	if (le32_to_cpu(dx_root->dr_fs_generation) != OCFS2_SB(sb)->fs_generation) {
+		ret = ocfs2_error(sb,
+				  "Dir Index Root # %llu has an invalid dr_fs_generation of #%u\n",
+				  (unsigned long long)bh->b_blocknr,
+				  le32_to_cpu(dx_root->dr_fs_generation));
+		goto bail;
+	}
+
+	/*
+	 * Dir index root blocks are allocated from a per-slot suballocator,
+	 * so the slot must be in range.  Otherwise removing the index passes
+	 * it to get_local_system_inode(), which hits BUG_ON() for
+	 * OCFS2_INVALID_SLOT or computes an out-of-bounds index otherwise.
+	 */
+	if ((u32)le16_to_cpu(dx_root->dr_suballoc_slot) >= OCFS2_SB(sb)->max_slots) {
+		ret = ocfs2_error(sb,
+				  "Dir Index Root # %llu has invalid dr_suballoc_slot %u\n",
+				  (unsigned long long)le64_to_cpu(dx_root->dr_blkno),
+				  le16_to_cpu(dx_root->dr_suballoc_slot));
+		goto bail;
+	}
+
+	/*
+	 * Similarly the suballoc bit must fit in a block group bitmap.
+	 * Otherwise removing the index will pass the oversized bit to
+	 * _ocfs2_free_suballoc_bits() and trigger ocfs2_error() there.
+	 */
+	if (le16_to_cpu(dx_root->dr_suballoc_bit) >= ocfs2_suballoc_bits_per_block(sb)) {
+		ret = ocfs2_error(sb,
+				  "Dir Index Root # %llu has invalid dr_suballoc_bit %u\n",
+				  (unsigned long long)le64_to_cpu(dx_root->dr_blkno),
+				  le16_to_cpu(dx_root->dr_suballoc_bit));
+		goto bail;
+	}
+
 	if (!(dx_root->dr_flags & OCFS2_DX_FLAG_INLINE)) {
 		struct ocfs2_extent_list *el = &dx_root->dr_list;
 
@@ -690,6 +733,18 @@ static int ocfs2_validate_dx_leaf(struct super_block *sb,
 		return ocfs2_error(sb, "Dir Index Leaf has bad signature %.*s\n",
 				   7, dx_leaf->dl_signature);
 
+	if (le64_to_cpu(dx_leaf->dl_blkno) != bh->b_blocknr)
+		return ocfs2_error(sb,
+				   "Dir Index Leaf # %llu has an invalid dl_blkno of %llu\n",
+				   (unsigned long long)bh->b_blocknr,
+				   (unsigned long long)le64_to_cpu(dx_leaf->dl_blkno));
+
+	if (le32_to_cpu(dx_leaf->dl_fs_generation) != OCFS2_SB(sb)->fs_generation)
+		return ocfs2_error(sb,
+				   "Dir Index Leaf # %llu has an invalid dl_fs_generation of #%u\n",
+				   (unsigned long long)bh->b_blocknr,
+				   le32_to_cpu(dx_leaf->dl_fs_generation));
+
 	if (le16_to_cpu(dx_leaf->dl_list.de_count) !=
 	    ocfs2_dx_entries_per_leaf(sb))
 		return ocfs2_error(sb,
diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c
index 53df5dd10ad05f..d3bfcada3e3bf5 100644
--- a/fs/ocfs2/dlmfs/dlmfs.c
+++ b/fs/ocfs2/dlmfs/dlmfs.c
@@ -188,7 +188,7 @@ static int dlmfs_file_release(struct inode *inode,
  * We do ->setattr() just to override size changes.  Our size is the size
  * of the LVB and nothing else.
  */
-static int dlmfs_file_setattr(struct mnt_idmap *idmap,
+static int dlmfs_file_setattr(const struct mnt_idmap *idmap,
 			      struct dentry *dentry, struct iattr *attr)
 {
 	int error;
@@ -402,7 +402,7 @@ static struct inode *dlmfs_get_inode(struct inode *parent,
  * File creation. Allocate an inode, and we're done..
  */
 /* SMP-safe */
-static struct dentry *dlmfs_mkdir(struct mnt_idmap * idmap,
+static struct dentry *dlmfs_mkdir(const struct mnt_idmap * idmap,
 				  struct inode * dir,
 				  struct dentry * dentry,
 				  umode_t mode)
@@ -450,7 +450,7 @@ static struct dentry *dlmfs_mkdir(struct mnt_idmap * idmap,
 	return ERR_PTR(status);
 }
 
-static int dlmfs_create(struct mnt_idmap *idmap,
+static int dlmfs_create(const struct mnt_idmap *idmap,
 			struct inode *dir,
 			struct dentry *dentry,
 			umode_t mode)
diff --git a/fs/ocfs2/dlmglue.c b/fs/ocfs2/dlmglue.c
index a23dd8f86c8950..6bc4a21cf5f02f 100644
--- a/fs/ocfs2/dlmglue.c
+++ b/fs/ocfs2/dlmglue.c
@@ -18,6 +18,7 @@
 #include <linux/time.h>
 #include <linux/delay.h>
 #include <linux/quotaops.h>
+#include <linux/sched/mm.h>
 #include <linux/sched/signal.h>
 #include <linux/string_choices.h>
 
@@ -2861,21 +2862,34 @@ void ocfs2_rename_unlock(struct ocfs2_super *osb)
 		ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX);
 }
 
-int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex)
+int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex, unsigned int *nofs_flag)
 {
 	int status;
+	unsigned int flags;
 	struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres;
 
 	if (ocfs2_is_hard_readonly(osb))
 		return -EROFS;
 
+	/*
+	 * ocfs2_delete_inode() takes this lock from ->evict_inode(), which the
+	 * dentry shrinker reaches while holding fs_reclaim.  Anything allocated
+	 * under the lock must therefore stay out of filesystem reclaim, or
+	 * reclaim recurses back into the shrinker and tries to take this lock
+	 * again.  Cover the whole critical section, including the cluster lock
+	 * and the sysfile inode locks the callers take below it.
+	 */
+	flags = memalloc_nofs_save();
+
 	if (ex)
 		down_write(&osb->nfs_sync_rwlock);
 	else
 		down_read(&osb->nfs_sync_rwlock);
 
-	if (ocfs2_mount_local(osb))
+	if (ocfs2_mount_local(osb)) {
+		*nofs_flag = flags;
 		return 0;
+	}
 
 	status = ocfs2_cluster_lock(osb, lockres, ex ? LKM_EXMODE : LKM_PRMODE,
 				    0, 0);
@@ -2886,12 +2900,17 @@ int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex)
 			up_write(&osb->nfs_sync_rwlock);
 		else
 			up_read(&osb->nfs_sync_rwlock);
+		memalloc_nofs_restore(flags);
+		return status;
 	}
 
-	return status;
+	*nofs_flag = flags;
+
+	return 0;
 }
 
-void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex)
+void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex,
+			   unsigned int nofs_flag)
 {
 	struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres;
 
@@ -2902,6 +2921,8 @@ void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex)
 		up_write(&osb->nfs_sync_rwlock);
 	else
 		up_read(&osb->nfs_sync_rwlock);
+
+	memalloc_nofs_restore(nofs_flag);
 }
 
 int ocfs2_trim_fs_lock(struct ocfs2_super *osb,
@@ -3525,6 +3546,10 @@ void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb,
 	struct ocfs2_mask_waiter mw;
 	unsigned long flags, flags2;
 
+	/* We didn't get anywhere near actually using this lockres. */
+	if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED))
+		return;
+
 	ocfs2_init_mask_waiter(&mw);
 
 	spin_lock_irqsave(&lockres->l_lock, flags);
diff --git a/fs/ocfs2/dlmglue.h b/fs/ocfs2/dlmglue.h
index a3ebd7303ea20b..faa58ed8699b1c 100644
--- a/fs/ocfs2/dlmglue.h
+++ b/fs/ocfs2/dlmglue.h
@@ -161,8 +161,9 @@ void ocfs2_orphan_scan_unlock(struct ocfs2_super *osb, u32 seqno);
 
 int ocfs2_rename_lock(struct ocfs2_super *osb);
 void ocfs2_rename_unlock(struct ocfs2_super *osb);
-int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex);
-void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex);
+int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex, unsigned int *nofs_flag);
+void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex,
+			   unsigned int nofs_flag);
 void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb);
 void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super *osb);
 int ocfs2_trim_fs_lock(struct ocfs2_super *osb,
diff --git a/fs/ocfs2/export.c b/fs/ocfs2/export.c
index 9c2665dd24e218..90b9e510e700d5 100644
--- a/fs/ocfs2/export.c
+++ b/fs/ocfs2/export.c
@@ -37,6 +37,7 @@ static struct dentry *ocfs2_get_dentry(struct super_block *sb,
 	struct inode *inode;
 	struct ocfs2_super *osb = OCFS2_SB(sb);
 	u64 blkno = handle->ih_blkno;
+	unsigned int nofs_flag = 0;
 	int status, set;
 	struct dentry *result;
 
@@ -59,7 +60,7 @@ static struct dentry *ocfs2_get_dentry(struct super_block *sb,
 	 * This will synchronize us against ocfs2_delete_inode() on
 	 * all nodes
 	 */
-	status = ocfs2_nfs_sync_lock(osb, 1);
+	status = ocfs2_nfs_sync_lock(osb, 1, &nofs_flag);
 	if (status < 0) {
 		mlog(ML_ERROR, "getting nfs sync lock(EX) failed %d\n", status);
 		goto check_err;
@@ -90,7 +91,7 @@ static struct dentry *ocfs2_get_dentry(struct super_block *sb,
 	inode = ocfs2_iget(osb, blkno, 0, 0);
 
 unlock_nfs_sync:
-	ocfs2_nfs_sync_unlock(osb, 1);
+	ocfs2_nfs_sync_unlock(osb, 1, nofs_flag);
 
 check_err:
 	if (status < 0) {
@@ -133,12 +134,13 @@ static struct dentry *ocfs2_get_parent(struct dentry *child)
 	u64 blkno;
 	struct dentry *parent;
 	struct inode *dir = d_inode(child);
+	unsigned int nofs_flag = 0;
 	int set;
 
 	trace_ocfs2_get_parent(child, child->d_name.len, child->d_name.name,
 			       (unsigned long long)OCFS2_I(dir)->ip_blkno);
 
-	status = ocfs2_nfs_sync_lock(OCFS2_SB(dir->i_sb), 1);
+	status = ocfs2_nfs_sync_lock(OCFS2_SB(dir->i_sb), 1, &nofs_flag);
 	if (status < 0) {
 		mlog(ML_ERROR, "getting nfs sync lock(EX) failed %d\n", status);
 		parent = ERR_PTR(status);
@@ -183,7 +185,7 @@ static struct dentry *ocfs2_get_parent(struct dentry *child)
 	ocfs2_inode_unlock(dir, 0);
 
 unlock_nfs_sync:
-	ocfs2_nfs_sync_unlock(OCFS2_SB(dir->i_sb), 1);
+	ocfs2_nfs_sync_unlock(OCFS2_SB(dir->i_sb), 1, nofs_flag);
 
 bail:
 	trace_ocfs2_get_parent_end(parent);
diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c
index d6e977ba656568..62f45a1b5ca1a6 100644
--- a/fs/ocfs2/file.c
+++ b/fs/ocfs2/file.c
@@ -1117,7 +1117,7 @@ static int ocfs2_extend_file(struct inode *inode,
 	return ret;
 }
 
-int ocfs2_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ocfs2_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr)
 {
 	int status = 0, size_change;
@@ -1317,7 +1317,7 @@ int ocfs2_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	return status;
 }
 
-int ocfs2_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ocfs2_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -1349,7 +1349,7 @@ int ocfs2_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return err;
 }
 
-int ocfs2_permission(struct mnt_idmap *idmap, struct inode *inode,
+int ocfs2_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		     int mask)
 {
 	int ret, had_lock;
diff --git a/fs/ocfs2/file.h b/fs/ocfs2/file.h
index 41e65e45a9f3ff..97492ee5789e8b 100644
--- a/fs/ocfs2/file.h
+++ b/fs/ocfs2/file.h
@@ -50,11 +50,11 @@ int ocfs2_extend_no_holes(struct inode *inode, struct buffer_head *di_bh,
 			  u64 new_i_size, u64 zero_to);
 int ocfs2_zero_extend(struct inode *inode, struct buffer_head *di_bh,
 		      loff_t zero_to);
-int ocfs2_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ocfs2_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr);
-int ocfs2_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ocfs2_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, u32 request_mask, unsigned int flags);
-int ocfs2_permission(struct mnt_idmap *idmap,
+int ocfs2_permission(const struct mnt_idmap *idmap,
 		     struct inode *inode,
 		     int mask);
 
diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c
index 180107a11046c5..80c36c60a8cef9 100644
--- a/fs/ocfs2/inode.c
+++ b/fs/ocfs2/inode.c
@@ -638,14 +638,18 @@ static int ocfs2_read_locked_inode(struct inode *inode,
 	fe = (struct ocfs2_dinode *) bh->b_data;
 
 	/*
-	 * This is a code bug. Right now the caller needs to
-	 * understand whether it is asking for a system file inode or
-	 * not so the proper lock names can be built.
+	 * The caller must know whether it is asking for a system file inode
+	 * or not so the proper lock names can be built.  Since i_flags comes
+	 * from disk, a mismatch is filesystem corruption instead of a code
+	 * bug, so handle it with ocfs2_error() rather than BUG_ON().
 	 */
-	mlog_bug_on_msg(!!(fe->i_flags & cpu_to_le32(OCFS2_SYSTEM_FL)) !=
-			!!(args->fi_flags & OCFS2_FI_FLAG_SYSFILE),
-			"Inode %llu: system file state is ambiguous\n",
-			(unsigned long long)args->fi_blkno);
+	if (!!(fe->i_flags & cpu_to_le32(OCFS2_SYSTEM_FL)) !=
+	    !!(args->fi_flags & OCFS2_FI_FLAG_SYSFILE)) {
+		status = ocfs2_error(osb->sb,
+				     "Inode %llu: system file state is ambiguous\n",
+				     (unsigned long long)args->fi_blkno);
+		goto bail;
+	}
 
 	if (S_ISCHR(le16_to_cpu(fe->i_mode)) ||
 	    S_ISBLK(le16_to_cpu(fe->i_mode)))
@@ -1105,6 +1109,7 @@ static void ocfs2_delete_inode(struct inode *inode)
 {
 	int wipe, status;
 	sigset_t oldset;
+	unsigned int nofs_flag = 0;
 	struct buffer_head *di_bh = NULL;
 	struct ocfs2_dinode *di = NULL;
 
@@ -1139,7 +1144,7 @@ static void ocfs2_delete_inode(struct inode *inode)
 	 * shared mode so that all nodes can still concurrently
 	 * process deletes.
 	 */
-	status = ocfs2_nfs_sync_lock(OCFS2_SB(inode->i_sb), 0);
+	status = ocfs2_nfs_sync_lock(OCFS2_SB(inode->i_sb), 0, &nofs_flag);
 	if (status < 0) {
 		mlog(ML_ERROR, "getting nfs sync lock(PR) failed %d\n", status);
 		ocfs2_cleanup_delete_inode(inode, 0);
@@ -1211,7 +1216,7 @@ static void ocfs2_delete_inode(struct inode *inode)
 	brelse(di_bh);
 
 bail_unlock_nfs_sync:
-	ocfs2_nfs_sync_unlock(OCFS2_SB(inode->i_sb), 0);
+	ocfs2_nfs_sync_unlock(OCFS2_SB(inode->i_sb), 0, nofs_flag);
 
 bail_unblock:
 	ocfs2_unblock_signals(&oldset);
@@ -1520,14 +1525,45 @@ int ocfs2_validate_inode_block(struct super_block *sb,
 		goto bail;
 	}
 
-	if (le16_to_cpu(di->i_suballoc_slot) != (u16)OCFS2_INVALID_SLOT &&
-	    (u32)le16_to_cpu(di->i_suballoc_slot) > OCFS2_SB(sb)->max_slots - 1) {
+	/*
+	 * Only system inodes created by mkfs.ocfs2 are allocated from the
+	 * global allocator and thus legitimately carry OCFS2_INVALID_SLOT.
+	 * Regular inodes are always allocated from a per-slot suballocator.
+	 * If a regular inode with OCFS2_INVALID_SLOT was accepted here,
+	 * deleting it would pass the slot to get_local_system_inode() via
+	 * ocfs2_remove_inode() and trigger BUG_ON(slot == OCFS2_INVALID_SLOT).
+	 */
+	if (le16_to_cpu(di->i_suballoc_slot) == (u16)OCFS2_INVALID_SLOT) {
+		if (!(le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL)) {
+			rc = ocfs2_error(sb,
+					 "Invalid dinode %llu: suballoc slot %u for non-system inode\n",
+					 (unsigned long long)bh->b_blocknr,
+					 le16_to_cpu(di->i_suballoc_slot));
+			goto bail;
+		}
+	} else if ((u32)le16_to_cpu(di->i_suballoc_slot) > OCFS2_SB(sb)->max_slots - 1) {
 		rc = ocfs2_error(sb, "Invalid dinode %llu: suballoc slot %u\n",
 				 (unsigned long long)bh->b_blocknr,
 				 le16_to_cpu(di->i_suballoc_slot));
 		goto bail;
 	}
 
+	/*
+	 * A suballocator block group bitmap is contained in a single block
+	 * and starts after the group descriptor header, so a valid suballoc
+	 * bit can never exceed ocfs2_suballoc_bits_per_block().  Otherwise
+	 * deleting the inode will pass the oversized bit to
+	 * _ocfs2_free_suballoc_bits() via ocfs2_free_dinode() and trigger
+	 * BUG_ON((count + start_bit) > ocfs2_bits_per_group(cl)), since any
+	 * group holds at most ocfs2_suballoc_bits_per_block() bits.
+	 */
+	if (le16_to_cpu(di->i_suballoc_bit) >= ocfs2_suballoc_bits_per_block(sb)) {
+		rc = ocfs2_error(sb, "Invalid dinode %llu: suballoc bit %u\n",
+				 (unsigned long long)bh->b_blocknr,
+				 le16_to_cpu(di->i_suballoc_bit));
+		goto bail;
+	}
+
 	if ((le32_to_cpu(di->i_flags) & OCFS2_ORPHANED_FL) &&
 	    le16_to_cpu(di->i_orphaned_slot) >= OCFS2_SB(sb)->max_slots) {
 		rc = ocfs2_error(sb, "Invalid dinode %llu: orphaned slot %u\n",
diff --git a/fs/ocfs2/ioctl.c b/fs/ocfs2/ioctl.c
index cbe59d23166636..36c7c9ac8b5dd6 100644
--- a/fs/ocfs2/ioctl.c
+++ b/fs/ocfs2/ioctl.c
@@ -82,7 +82,7 @@ int ocfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return status;
 }
 
-int ocfs2_fileattr_set(struct mnt_idmap *idmap,
+int ocfs2_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/ocfs2/ioctl.h b/fs/ocfs2/ioctl.h
index 4a1c2313b42903..b1cb529fc5f9ac 100644
--- a/fs/ocfs2/ioctl.h
+++ b/fs/ocfs2/ioctl.h
@@ -12,7 +12,7 @@
 #define OCFS2_IOCTL_PROTO_H
 
 int ocfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int ocfs2_fileattr_set(struct mnt_idmap *idmap,
+int ocfs2_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa);
 long ocfs2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 long ocfs2_compat_ioctl(struct file *file, unsigned cmd, unsigned long arg);
diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c
index d8afbc1a76bb8a..cc60db6b97060c 100644
--- a/fs/ocfs2/journal.c
+++ b/fs/ocfs2/journal.c
@@ -156,7 +156,7 @@ static void ocfs2_queue_replay_slots(struct ocfs2_super *osb,
 	replay_map->rm_state = REPLAY_DONE;
 }
 
-void ocfs2_free_replay_slots(struct ocfs2_super *osb)
+static void ocfs2_free_replay_slots(struct ocfs2_super *osb)
 {
 	struct ocfs2_replay_map *replay_map = osb->replay_map;
 
@@ -243,6 +243,7 @@ void ocfs2_recovery_exit(struct ocfs2_super *osb)
 	/* XXX: Should we bug if there are dirty entries? */
 
 	kfree(rm);
+	ocfs2_free_replay_slots(osb);
 }
 
 static int __ocfs2_recovery_map_test(struct ocfs2_super *osb,
@@ -676,19 +677,20 @@ static int __ocfs2_journal_access(handle_t *handle,
 		mlog(ML_ERROR, "giving me a buffer that's not uptodate!\n");
 		mlog(ML_ERROR, "b_blocknr=%llu, b_state=0x%lx\n",
 		     (unsigned long long)bh->b_blocknr, bh->b_state);
-
+	}
+	/*
+	 * A previous transaction with a couple of buffer heads fail
+	 * to checkpoint, so all the bhs are marked as BH_Write_EIO.
+	 * For current transaction, the bh is just among those error
+	 * bhs which previous transaction handle. We can't just clear
+	 * its BH_Write_EIO and reuse directly, since other bhs are
+	 * not written to disk yet and that will cause metadata
+	 * inconsistency. So we should set fs read-only to avoid
+	 * further damage.
+	 */
+	if (buffer_write_io_error(bh)) {
 		lock_buffer(bh);
-		/*
-		 * A previous transaction with a couple of buffer heads fail
-		 * to checkpoint, so all the bhs are marked as BH_Write_EIO.
-		 * For current transaction, the bh is just among those error
-		 * bhs which previous transaction handle. We can't just clear
-		 * its BH_Write_EIO and reuse directly, since other bhs are
-		 * not written to disk yet and that will cause metadata
-		 * inconsistency. So we should set fs read-only to avoid
-		 * further damage.
-		 */
-		if (buffer_write_io_error(bh) && !buffer_uptodate(bh)) {
+		if (buffer_write_io_error(bh)) {
 			unlock_buffer(bh);
 			return ocfs2_error(osb->sb, "A previous attempt to "
 					"write this buffer head failed\n");
diff --git a/fs/ocfs2/journal.h b/fs/ocfs2/journal.h
index f8b3b2a3d6309e..19fc920d26b1cd 100644
--- a/fs/ocfs2/journal.h
+++ b/fs/ocfs2/journal.h
@@ -151,7 +151,6 @@ void ocfs2_recovery_exit(struct ocfs2_super *osb);
 void ocfs2_recovery_disable_quota(struct ocfs2_super *osb);
 
 int ocfs2_compute_replay_slots(struct ocfs2_super *osb);
-void ocfs2_free_replay_slots(struct ocfs2_super *osb);
 /*
  *  Journal Control:
  *  Initialize, Load, Shutdown, Wipe a journal.
diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c
index e9c7774ccf9153..fce9a31a3671da 100644
--- a/fs/ocfs2/namei.c
+++ b/fs/ocfs2/namei.c
@@ -227,7 +227,7 @@ static void ocfs2_cleanup_add_entry_failure(struct ocfs2_super *osb,
 	iput(inode);
 }
 
-static int ocfs2_mknod(struct mnt_idmap *idmap,
+static int ocfs2_mknod(const struct mnt_idmap *idmap,
 		       struct inode *dir,
 		       struct dentry *dentry,
 		       umode_t mode,
@@ -336,13 +336,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap,
 		goto leave;
 
 	/* calculate meta data/clusters for setting security and acl xattr */
-	status = ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters,
-				       &xattr_credits, &want_meta,
-				       &acl_state);
-	if (status < 0) {
-		mlog_errno(status);
-		goto leave;
-	}
+	ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, &xattr_credits,
+			      &want_meta, &acl_state);
 
 	/* Reserve a cluster if creating an extent based directory. */
 	if (S_ISDIR(mode) && !ocfs2_supports_inline_data(osb)) {
@@ -655,7 +650,7 @@ static int ocfs2_mknod_locked(struct ocfs2_super *osb,
 				    suballoc_loc, suballoc_bit);
 }
 
-static struct dentry *ocfs2_mkdir(struct mnt_idmap *idmap,
+static struct dentry *ocfs2_mkdir(const struct mnt_idmap *idmap,
 				  struct inode *dir,
 				  struct dentry *dentry,
 				  umode_t mode)
@@ -671,7 +666,7 @@ static struct dentry *ocfs2_mkdir(struct mnt_idmap *idmap,
 	return ERR_PTR(ret);
 }
 
-static int ocfs2_create(struct mnt_idmap *idmap,
+static int ocfs2_create(const struct mnt_idmap *idmap,
 			struct inode *dir,
 			struct dentry *dentry,
 			umode_t mode)
@@ -1211,7 +1206,7 @@ static void ocfs2_double_unlock(struct inode *inode1, struct inode *inode2)
 		ocfs2_inode_unlock(inode2, 1);
 }
 
-static int ocfs2_rename(struct mnt_idmap *idmap,
+static int ocfs2_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir,
 			struct dentry *old_dentry,
 			struct inode *new_dir,
@@ -1815,7 +1810,7 @@ static int ocfs2_create_symlink_data(struct ocfs2_super *osb,
 	return status;
 }
 
-static int ocfs2_symlink(struct mnt_idmap *idmap,
+static int ocfs2_symlink(const struct mnt_idmap *idmap,
 			 struct inode *dir,
 			 struct dentry *dentry,
 			 const char *symname)
diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h
index 62cad6522c7a31..e3bb3cc0b25a29 100644
--- a/fs/ocfs2/ocfs2.h
+++ b/fs/ocfs2/ocfs2.h
@@ -502,6 +502,11 @@ struct ocfs2_super
 	 */
 	struct workqueue_struct *ocfs2_wq;
 
+	/* deferred reclaim of fully freed suballocator block groups */
+	spinlock_t os_suballoc_reclaim_lock;
+	struct list_head os_suballoc_reclaim_list;
+	struct work_struct os_suballoc_reclaim_work;
+
 	/* sysfs directory per partition */
 	struct kset *osb_dev_kset;
 
@@ -588,6 +593,18 @@ static inline int ocfs2_supports_discontig_bg(struct ocfs2_super *osb)
 	return 0;
 }
 
+/*
+ * A suballocator block group bitmap starts right after the group
+ * descriptor header, so a suballoc bit can never exceed this number
+ * of bits.  Derive it from ocfs2_group_bitmap_size() which also caps
+ * it at OCFS2_MAX_BG_BITMAP_SIZE when discontig_bg is enabled.
+ */
+static inline u32 ocfs2_suballoc_bits_per_block(struct super_block *sb)
+{
+	return ocfs2_group_bitmap_size(sb, 1,
+				       OCFS2_SB(sb)->s_feature_incompat) * 8;
+}
+
 static inline unsigned int ocfs2_link_max(struct ocfs2_super *osb)
 {
 	if (ocfs2_supports_indexed_dirs(osb))
diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c
index f55810c59b1b11..816be594db411c 100644
--- a/fs/ocfs2/quota_local.c
+++ b/fs/ocfs2/quota_local.c
@@ -171,6 +171,10 @@ static int ocfs2_local_check_quota_file(struct super_block *sb, int type)
 	struct ocfs2_disk_dqheader *dqhead;
 	int status, ret = 0;
 
+	/* OCFS2 quota format is supported only for OCFS2 filesystems */
+	if (sb->s_magic != OCFS2_SUPER_MAGIC)
+		goto out_err;
+
 	/* First check whether we understand local quota file */
 	status = ocfs2_read_quota_block(linode, 0, &bh);
 	if (status) {
@@ -245,6 +249,25 @@ static void ocfs2_release_local_quota_bitmaps(struct list_head *head)
 	}
 }
 
+/* Check that the on-disk chunk and block counts match the file layout */
+static int ocfs2_check_local_quota_info(struct inode *inode,
+					unsigned int chunks,
+					unsigned int blocks)
+{
+	u64 chunk_len = ol_chunk_blocks(inode->i_sb) + 1;
+	u64 min_blocks = chunks ? 1 + chunk_len * (chunks - 1) + 1 : 1;
+	u64 max_blocks = 1 + chunk_len * chunks;
+
+	if (blocks >= min_blocks && blocks <= max_blocks &&
+	    blocks <= i_size_read(inode) >> inode->i_sb->s_blocksize_bits)
+		return 0;
+
+	return ocfs2_error(inode->i_sb,
+			   "Quota file %llu has bad info: %u chunks, %u blocks\n",
+			   (unsigned long long)OCFS2_I(inode)->ip_blkno,
+			   chunks, blocks);
+}
+
 /* Load quota bitmaps into memory */
 static int ocfs2_load_local_quota_bitmaps(struct inode *inode,
 			struct ocfs2_local_disk_dqinfo *ldinfo,
@@ -733,6 +756,11 @@ static int ocfs2_local_read_info(struct super_block *sb, int type)
 	oinfo->dqi_blocks = le32_to_cpu(ldinfo->dqi_blocks);
 	oinfo->dqi_libh = bh;
 
+	status = ocfs2_check_local_quota_info(lqinode, oinfo->dqi_chunks,
+					      oinfo->dqi_blocks);
+	if (status < 0)
+		goto out_err;
+
 	/* We crashed when using local quota file? */
 	if (!(oinfo->dqi_flags & OLQF_CLEAN)) {
 		rec = OCFS2_SB(sb)->quota_rec;
@@ -1071,10 +1099,13 @@ static struct ocfs2_quota_chunk *ocfs2_local_quota_add_chunk(
 		goto out;
 	}
 
+	if (list_empty(&oinfo->dqi_chunk))
+		chunk->qc_num = 0;
+	else
+		chunk->qc_num = list_entry(oinfo->dqi_chunk.prev,
+					   struct ocfs2_quota_chunk,
+					   qc_chunk)->qc_num + 1;
 	list_add_tail(&chunk->qc_chunk, &oinfo->dqi_chunk);
-	chunk->qc_num = list_entry(chunk->qc_chunk.prev,
-				   struct ocfs2_quota_chunk,
-				   qc_chunk)->qc_num + 1;
 	chunk->qc_headerbh = bh;
 	*offset = 0;
 	return chunk;
diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c
index d9f22b4a265461..3e9cccf06e48cb 100644
--- a/fs/ocfs2/refcounttree.c
+++ b/fs/ocfs2/refcounttree.c
@@ -117,6 +117,33 @@ static int ocfs2_validate_refcount_block(struct super_block *sb,
 		goto out;
 	}
 
+	/*
+	 * Refcount blocks are allocated from a per-slot suballocator, so the
+	 * slot must be in range.  Otherwise freeing the block passes it to
+	 * get_local_system_inode(), which hits BUG_ON() for
+	 * OCFS2_INVALID_SLOT or computes an out-of-bounds index otherwise.
+	 */
+	if ((u32)le16_to_cpu(rb->rf_suballoc_slot) >= OCFS2_SB(sb)->max_slots) {
+		rc = ocfs2_error(sb,
+				 "Refcount block #%llu has an invalid rf_suballoc_slot of %u\n",
+				 (unsigned long long)bh->b_blocknr,
+				 le16_to_cpu(rb->rf_suballoc_slot));
+		goto out;
+	}
+
+	/*
+	 * Similarly the suballoc bit must fit in a block group bitmap.
+	 * Otherwise freeing the block will pass the oversized bit to
+	 * _ocfs2_free_suballoc_bits() and trigger ocfs2_error() there.
+	 */
+	if (le16_to_cpu(rb->rf_suballoc_bit) >= ocfs2_suballoc_bits_per_block(sb)) {
+		rc = ocfs2_error(sb,
+				 "Refcount block #%llu has an invalid rf_suballoc_bit of %u\n",
+				 (unsigned long long)bh->b_blocknr,
+				 le16_to_cpu(rb->rf_suballoc_bit));
+		goto out;
+	}
+
 	/*
 	 * rf_records (rl_count/rl_used/rl_recs[]) is only meaningful when
 	 * this block is not an interior tree block (OCFS2_REFCOUNT_TREE_FL);
diff --git a/fs/ocfs2/stackglue.c b/fs/ocfs2/stackglue.c
index 741d6191d87114..0ccaab29426d24 100644
--- a/fs/ocfs2/stackglue.c
+++ b/fs/ocfs2/stackglue.c
@@ -18,6 +18,7 @@
 #include <linux/kobject.h>
 #include <linux/sysfs.h>
 #include <linux/sysctl.h>
+#include <linux/umh.h>
 
 #include "ocfs2_fs.h"
 
diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c
index 20c3aec6b9873c..624152e4f7fedf 100644
--- a/fs/ocfs2/suballoc.c
+++ b/fs/ocfs2/suballoc.c
@@ -2687,16 +2687,24 @@ static int ocfs2_block_group_clear_bits(handle_t *handle,
  * cleanup rec/alloc_inode job, then switches to the main bitmap
  * to reclaim released space.
  *
+ * Callers must hold inode_lock() and ocfs2_inode_lock() on
+ * main_bm_inode, i.e. the global bitmap inode locks must be taken
+ * before starting the transaction.
+ *
  * handle: The transaction handle
  * alloc_inode: The suballoc inode
  * alloc_bh: The buffer_head of suballoc inode
  * group_bh: The group descriptor buffer_head of suballocator managed.
- *           Caller should release the input group_bh.
+ *           This function takes ownership of it and will release it.
+ * main_bm_inode: The global bitmap inode
+ * main_bm_bh: The buffer_head of the global bitmap inode
  */
 static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 			struct inode *alloc_inode,
 			struct buffer_head *alloc_bh,
-			struct buffer_head *group_bh)
+			struct buffer_head *group_bh,
+			struct inode *main_bm_inode,
+			struct buffer_head *main_bm_bh)
 {
 	int idx, status = 0;
 	int i, next_free_rec, len = 0;
@@ -2706,8 +2714,6 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 	u64 bg_blkno, start_blk;
 	unsigned int count;
 	struct ocfs2_chain_rec *rec;
-	struct buffer_head *main_bm_bh = NULL;
-	struct inode *main_bm_inode = NULL;
 	struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb);
 	struct ocfs2_dinode *fe = (struct ocfs2_dinode *) alloc_bh->b_data;
 	struct ocfs2_chain_list *cl = &fe->id2.i_chain;
@@ -2794,24 +2800,12 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 	ocfs2_remove_from_cache(INODE_CACHE(alloc_inode), group_bh);
 	memset(group, 0, sizeof(struct ocfs2_group_desc));
 
-	/* prepare job for reclaim clusters */
-	main_bm_inode = ocfs2_get_system_file_inode(osb,
-						    GLOBAL_BITMAP_SYSTEM_INODE,
-						    OCFS2_INVALID_SLOT);
-	if (!main_bm_inode)
-		goto bail; /* ignore the error in reclaim path */
-
-	inode_lock(main_bm_inode);
-
-	status = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1);
-	if (status < 0)
-		goto free_bm_inode; /* ignore the error in reclaim path */
-
 	ocfs2_block_to_cluster_group(main_bm_inode, start_blk, &bg_blkno,
 				     &start_bit);
 	fe = (struct ocfs2_dinode *) main_bm_bh->b_data;
 	cl = &fe->id2.i_chain;
-	/* reuse group_bh, caller will release the input group_bh */
+	/* release the suballocator group descriptor before reuse */
+	brelse(group_bh);
 	group_bh = NULL;
 
 	/* reclaim clusters to global_bitmap */
@@ -2819,7 +2813,7 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 					     &group_bh);
 	if (status < 0) {
 		mlog_errno(status);
-		goto free_bm_bh;
+		goto bail;
 	}
 	group = (struct ocfs2_group_desc *) group_bh->b_data;
 
@@ -2827,7 +2821,7 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 		ocfs2_error(alloc_inode->i_sb,
 			"reclaim length (%d) beyands block group length (%d)",
 			count + start_bit, le16_to_cpu(group->bg_bits));
-		goto free_group_bh;
+		goto bail;
 	}
 
 	old_bg_contig_free_bits = group->bg_contig_free_bits;
@@ -2837,7 +2831,7 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 					      _ocfs2_clear_bit);
 	if (status < 0) {
 		mlog_errno(status);
-		goto free_group_bh;
+		goto bail;
 	}
 
 	status = ocfs2_journal_access_di(handle, INODE_CACHE(main_bm_inode),
@@ -2847,7 +2841,7 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 		ocfs2_block_group_set_bits(handle, main_bm_inode, group, group_bh,
 				start_bit, count,
 				le16_to_cpu(old_bg_contig_free_bits), 1);
-		goto free_group_bh;
+		goto bail;
 	}
 
 	idx = le16_to_cpu(group->bg_chain);
@@ -2858,19 +2852,168 @@ static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle,
 	fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - count);
 	ocfs2_journal_dirty(handle, main_bm_bh);
 
-free_group_bh:
+bail:
 	brelse(group_bh);
+	return status;
+}
+
+/*
+ * When a suballocator block group becomes fully freed, its space is
+ * reclaimed back to the global bitmap. Taking the global bitmap inode
+ * lock inside the freeing transaction would create a lock dependency
+ * of "j_trans_barrier -> global bitmap inode i_rwsem", which forms a
+ * circular dependency with paths like ocfs2_shutdown_local_alloc() that
+ * take the inode lock before starting a transaction, and can lead to a
+ * real deadlock with the ocfs2cmt journal commit thread. So queue the
+ * reclaim to the workqueue and let it run outside the freeing
+ * transaction.
+ */
+struct ocfs2_suballoc_reclaim_work {
+	struct list_head list;
+	struct inode *alloc_inode;
+	u64 bg_blkno;
+};
 
-free_bm_bh:
+static void ocfs2_queue_suballoc_reclaim(struct ocfs2_super *osb,
+					 struct inode *alloc_inode,
+					 u64 bg_blkno)
+{
+	struct ocfs2_suballoc_reclaim_work *reclaim_work;
+
+	reclaim_work = kmalloc_obj(*reclaim_work, GFP_NOFS);
+	if (!reclaim_work) {
+		/*
+		 * Reclaim is only a space return optimization. If we can't
+		 * queue it, the freed block group just stays owned by the
+		 * suballocator.
+		 */
+		return;
+	}
+
+	igrab(alloc_inode);
+	reclaim_work->alloc_inode = alloc_inode;
+	reclaim_work->bg_blkno = bg_blkno;
+
+	spin_lock(&osb->os_suballoc_reclaim_lock);
+	list_add_tail(&reclaim_work->list, &osb->os_suballoc_reclaim_list);
+	spin_unlock(&osb->os_suballoc_reclaim_lock);
+
+	queue_work(osb->ocfs2_wq, &osb->os_suballoc_reclaim_work);
+}
+
+static void ocfs2_do_suballoc_reclaim(struct ocfs2_super *osb,
+				struct ocfs2_suballoc_reclaim_work *reclaim_work)
+{
+	int status, i;
+	handle_t *handle;
+	struct inode *alloc_inode = reclaim_work->alloc_inode;
+	struct inode *main_bm_inode;
+	struct buffer_head *alloc_bh = NULL, *group_bh = NULL;
+	struct buffer_head *main_bm_bh = NULL;
+	struct ocfs2_dinode *fe;
+	struct ocfs2_chain_list *cl;
+	struct ocfs2_chain_rec *rec;
+
+	/* journal already gone, e.g. during dismount cleanup */
+	if (!osb->journal)
+		return;
+
+	inode_lock(alloc_inode);
+	status = ocfs2_inode_lock(alloc_inode, &alloc_bh, 1);
+	if (status < 0)
+		goto out_alloc;
+
+	fe = (struct ocfs2_dinode *) alloc_bh->b_data;
+	cl = &fe->id2.i_chain;
+
+	/*
+	 * The block group may have been allocated from again since the
+	 * reclaim work was queued, re-check that it is still fully freed.
+	 * A stale work item can also reference a group that is no longer
+	 * chained, whose descriptor would fail validation and trigger a
+	 * spurious ocfs2_error(), so verify chain membership first.
+	 */
+	for (i = 0; i < le16_to_cpu(cl->cl_next_free_rec); i++) {
+		rec = &cl->cl_recs[i];
+		if (le64_to_cpu(rec->c_blkno) == reclaim_work->bg_blkno)
+			break;
+	}
+	if (i == le16_to_cpu(cl->cl_next_free_rec) ||
+	    ocfs2_is_cluster_bitmap(alloc_inode) ||
+	    (le32_to_cpu(rec->c_free) != (le32_to_cpu(rec->c_total) - 1)) ||
+	    (le16_to_cpu(cl->cl_next_free_rec) == 1))
+		goto out_alloc_unlock;
+
+	status = ocfs2_read_group_descriptor(alloc_inode, fe,
+					     reclaim_work->bg_blkno, &group_bh);
+	if (status < 0)
+		goto out_alloc_unlock;
+
+	main_bm_inode = ocfs2_get_system_file_inode(osb,
+						    GLOBAL_BITMAP_SYSTEM_INODE,
+						    OCFS2_INVALID_SLOT);
+	if (!main_bm_inode)
+		goto out_group;
+
+	inode_lock(main_bm_inode);
+	status = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1);
+	if (status < 0)
+		goto out_main;
+
+	handle = ocfs2_start_trans(osb, OCFS2_SUBALLOC_FREE);
+	if (IS_ERR(handle)) {
+		status = PTR_ERR(handle);
+		mlog_errno(status);
+		goto out_main_unlock;
+	}
+
+	status = _ocfs2_reclaim_suballoc_to_main(handle, alloc_inode,
+						 alloc_bh, group_bh,
+						 main_bm_inode, main_bm_bh);
+	/* group_bh ownership passed to _ocfs2_reclaim_suballoc_to_main() */
+	group_bh = NULL;
+	if (status < 0)
+		mlog_errno(status);
+
+	ocfs2_commit_trans(osb, handle);
+
+out_main_unlock:
 	ocfs2_inode_unlock(main_bm_inode, 1);
 	brelse(main_bm_bh);
-
-free_bm_inode:
+out_main:
 	inode_unlock(main_bm_inode);
 	iput(main_bm_inode);
+out_group:
+	brelse(group_bh);
+out_alloc_unlock:
+	ocfs2_inode_unlock(alloc_inode, 1);
+	brelse(alloc_bh);
+out_alloc:
+	inode_unlock(alloc_inode);
+}
 
-bail:
-	return status;
+void ocfs2_suballoc_reclaim_worker(struct work_struct *work)
+{
+	struct ocfs2_super *osb = container_of(work, struct ocfs2_super,
+					       os_suballoc_reclaim_work);
+	struct ocfs2_suballoc_reclaim_work *reclaim_work;
+
+	while (1) {
+		spin_lock(&osb->os_suballoc_reclaim_lock);
+		if (list_empty(&osb->os_suballoc_reclaim_list)) {
+			spin_unlock(&osb->os_suballoc_reclaim_lock);
+			break;
+		}
+		reclaim_work = list_first_entry(&osb->os_suballoc_reclaim_list,
+					struct ocfs2_suballoc_reclaim_work,
+					list);
+		list_del(&reclaim_work->list);
+		spin_unlock(&osb->os_suballoc_reclaim_lock);
+
+		ocfs2_do_suballoc_reclaim(osb, reclaim_work);
+		iput(reclaim_work->alloc_inode);
+		kfree(reclaim_work);
+	}
 }
 
 /*
@@ -2897,10 +3040,22 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle,
 	/* The alloc_bh comes from ocfs2_free_dinode() or
 	 * ocfs2_free_clusters().  The callers have all locked the
 	 * allocator and gotten alloc_bh from the lock call.  This
-	 * validates the dinode buffer.  Any corruption that has happened
-	 * is a code bug. */
+	 * validates the dinode buffer. */
 	BUG_ON(!OCFS2_IS_VALID_DINODE(fe));
-	BUG_ON((count + start_bit) > ocfs2_bits_per_group(cl));
+
+	/*
+	 * ocfs2_bits_per_group() is derived from cl_cpg and cl_bpc of the
+	 * allocator dinode, which are not validated against the volume
+	 * geometry.  A corrupted image can carry a suballoc bit beyond it,
+	 * so error out instead of crashing.
+	 */
+	if ((count + start_bit) > ocfs2_bits_per_group(cl)) {
+		return ocfs2_error(alloc_inode->i_sb,
+				   "Allocator #%llu: freeing bits %u+%u exceeds bits per group %u\n",
+				   (unsigned long long)le64_to_cpu(fe->i_blkno),
+				   count, start_bit,
+				   ocfs2_bits_per_group(cl));
+	}
 
 	trace_ocfs2_free_suballoc_bits(
 		(unsigned long long)OCFS2_I(alloc_inode)->ip_blkno,
@@ -2915,7 +3070,20 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle,
 	}
 	group = (struct ocfs2_group_desc *) group_bh->b_data;
 
-	BUG_ON((count + start_bit) > le16_to_cpu(group->bg_bits));
+	/*
+	 * Group descriptor validation only guarantees bg_bits within the
+	 * physical bitmap size, so double check the freeing range here.
+	 * Otherwise ocfs2_block_group_clear_bits() would clear bits beyond
+	 * bg_bitmap.
+	 */
+	if ((count + start_bit) > le16_to_cpu(group->bg_bits)) {
+		status = ocfs2_error(alloc_inode->i_sb,
+				     "Group descriptor #%llu has %u bits, cannot free bits %u+%u\n",
+				     (unsigned long long)le64_to_cpu(group->bg_blkno),
+				     le16_to_cpu(group->bg_bits),
+				     count, start_bit);
+		goto bail;
+	}
 
 	if (ocfs2_is_cluster_bitmap(alloc_inode))
 		old_bg_contig_free_bits = group->bg_contig_free_bits;
@@ -2955,7 +3123,8 @@ static int _ocfs2_free_suballoc_bits(handle_t *handle,
 		goto bail;
 	}
 
-	_ocfs2_reclaim_suballoc_to_main(handle, alloc_inode, alloc_bh, group_bh);
+	ocfs2_queue_suballoc_reclaim(OCFS2_SB(alloc_inode->i_sb), alloc_inode,
+				     bg_blkno);
 
 bail:
 	brelse(group_bh);
diff --git a/fs/ocfs2/suballoc.h b/fs/ocfs2/suballoc.h
index bcf2ed4a86310b..6042abc032f96e 100644
--- a/fs/ocfs2/suballoc.h
+++ b/fs/ocfs2/suballoc.h
@@ -206,7 +206,7 @@ int ocfs2_lock_allocators(struct inode *inode, struct ocfs2_extent_tree *et,
 
 int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res);
 
-
+void ocfs2_suballoc_reclaim_worker(struct work_struct *work);
 
 /*
  * The following two interfaces are for ocfs2_create_inode_in_orphan().
diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c
index c62e389d4dd659..1e76b1d9fe0af4 100644
--- a/fs/ocfs2/super.c
+++ b/fs/ocfs2/super.c
@@ -1162,15 +1162,23 @@ static int ocfs2_fill_super(struct super_block *sb, struct fs_context *fc)
 out_dismount:
 	atomic_set(&osb->vol_state, VOLUME_DISABLED);
 	wake_up(&osb->osb_mount_event);
-	ocfs2_free_replay_slots(osb);
 	ocfs2_dismount_volume(sb, 1);
 	goto out;
 
 out_debugfs:
 	debugfs_remove_recursive(osb->osb_debug_root);
 out_super:
+	/*
+	 * A recovery thread launched by a node failure event may still be
+	 * waiting for the volume to be mounted.  Set VOLUME_DISABLED and
+	 * wake it up, then wait for it to exit before osb is freed,
+	 * otherwise the kthread would leak and stay blocked on the wait
+	 * queue embedded in the freed osb.
+	 */
+	atomic_set(&osb->vol_state, VOLUME_DISABLED);
+	wake_up(&osb->osb_mount_event);
 	ocfs2_release_system_inodes(osb);
-	kfree(osb->recovery_map);
+	ocfs2_recovery_exit(osb);
 	ocfs2_delete_osb(osb);
 	kfree(osb);
 out:
@@ -1767,17 +1775,18 @@ static int ocfs2_mount_volume(struct super_block *sb)
 	status = ocfs2_truncate_log_init(osb);
 	if (status < 0) {
 		mlog_errno(status);
-		goto out_check_volume;
+		goto out_system_inodes;
 	}
 
 	ocfs2_super_unlock(osb, 1);
 	return 0;
 
-out_check_volume:
-	ocfs2_free_replay_slots(osb);
 out_system_inodes:
 	if (osb->local_alloc_state == OCFS2_LA_ENABLED)
 		ocfs2_shutdown_local_alloc(osb);
+	/* Drain pending suballoc reclaim work before the journal goes away */
+	if (osb->ocfs2_wq)
+		flush_workqueue(osb->ocfs2_wq);
 	ocfs2_release_system_inodes(osb);
 	/* before journal shutdown, we should release slot_info */
 	ocfs2_free_slot_info(osb);
@@ -1848,6 +1857,14 @@ static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err)
 	if (osb->cconn)
 		ocfs2_super_unlock(osb, 1);
 
+	/*
+	 * Drain pending suballoc reclaim work while the system inodes and
+	 * the journal are still alive, since the worker needs to look up
+	 * the global bitmap inode and start a transaction.
+	 */
+	if (osb->ocfs2_wq)
+		flush_workqueue(osb->ocfs2_wq);
+
 	ocfs2_release_system_inodes(osb);
 
 	ocfs2_journal_shutdown(osb);
@@ -2134,6 +2151,11 @@ static int ocfs2_initialize_super(struct super_block *sb,
 	INIT_WORK(&osb->dquot_drop_work, ocfs2_drop_dquot_refs);
 	init_llist_head(&osb->dquot_drop_list);
 
+	spin_lock_init(&osb->os_suballoc_reclaim_lock);
+	INIT_LIST_HEAD(&osb->os_suballoc_reclaim_list);
+	INIT_WORK(&osb->os_suballoc_reclaim_work,
+		  ocfs2_suballoc_reclaim_worker);
+
 	/* get some pseudo constants for clustersize bits */
 	osb->s_clustersize_bits =
 		le32_to_cpu(di->id2.i_super.s_clustersize_bits);
diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c
index 35bcbb0ff607b2..f7c486ca67646d 100644
--- a/fs/ocfs2/xattr.c
+++ b/fs/ocfs2/xattr.c
@@ -237,6 +237,21 @@ static int namevalue_size_xe(struct ocfs2_xattr_entry *xe)
 	return namevalue_size(xe->xe_name_len, value_len);
 }
 
+static int ocfs2_validate_xattr_entry(struct super_block *sb, u64 blkno,
+				      struct ocfs2_xattr_entry *xe)
+{
+	u64 value_len = le64_to_cpu(xe->xe_value_size);
+
+	if (value_len > OCFS2_XATTR_INLINE_SIZE &&
+	    ocfs2_xattr_is_local(xe))
+		return ocfs2_error(sb,
+				   "Invalid local xattr in block %llu: value size %llu\n",
+				   (unsigned long long)blkno,
+				   (unsigned long long)value_len);
+
+	return 0;
+}
+
 
 static int ocfs2_xattr_bucket_get_name_value(struct super_block *sb,
 					     struct ocfs2_xattr_header *xh,
@@ -517,6 +532,31 @@ static int ocfs2_validate_xattr_block(struct super_block *sb,
 				   le32_to_cpu(xb->xb_fs_generation));
 	}
 
+	/*
+	 * Xattr blocks are allocated from a per-slot suballocator, so the
+	 * slot must be in range.  Otherwise freeing the block passes it to
+	 * get_local_system_inode(), which hits BUG_ON() for
+	 * OCFS2_INVALID_SLOT or computes an out-of-bounds index otherwise.
+	 */
+	if ((u32)le16_to_cpu(xb->xb_suballoc_slot) >= OCFS2_SB(sb)->max_slots) {
+		return ocfs2_error(sb,
+				   "Extended attribute block #%llu has an invalid xb_suballoc_slot of %u\n",
+				   (unsigned long long)bh->b_blocknr,
+				   le16_to_cpu(xb->xb_suballoc_slot));
+	}
+
+	/*
+	 * Similarly the suballoc bit must fit in a block group bitmap.
+	 * Otherwise freeing the block will pass the oversized bit to
+	 * _ocfs2_free_suballoc_bits() and trigger ocfs2_error() there.
+	 */
+	if (le16_to_cpu(xb->xb_suballoc_bit) >= ocfs2_suballoc_bits_per_block(sb)) {
+		return ocfs2_error(sb,
+				   "Extended attribute block #%llu has an invalid xb_suballoc_bit of %u\n",
+				   (unsigned long long)bh->b_blocknr,
+				   le16_to_cpu(xb->xb_suballoc_bit));
+	}
+
 	if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) {
 		size_t region_offset =
 			offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header);
@@ -635,12 +675,11 @@ int ocfs2_calc_security_init(struct inode *dir,
 	return ret;
 }
 
-int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode,
-			  struct ocfs2_security_xattr_info *si,
-			  int *want_clusters, int *xattr_credits,
-			  int *want_meta, struct ocfs2_acl_state *acl_state)
+void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode,
+			   struct ocfs2_security_xattr_info *si,
+			   int *want_clusters, int *xattr_credits,
+			   int *want_meta, struct ocfs2_acl_state *acl_state)
 {
-	int ret = 0;
 	struct ocfs2_super *osb = OCFS2_SB(dir->i_sb);
 	int s_size = 0, a_size = 0, acl_len = 0, new_clusters;
 
@@ -662,7 +701,7 @@ int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode,
 	}
 
 	if (!(s_size + a_size))
-		return ret;
+		return;
 
 	/*
 	 * The max space of security xattr taken inline is
@@ -728,8 +767,6 @@ int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode,
 			}
 		}
 	}
-
-	return ret;
 }
 
 static int ocfs2_xattr_extend_allocation(struct inode *inode,
@@ -992,7 +1029,7 @@ static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno,
 	size_t entries_limit = region_size;
 	size_t nv_limit = region_size;
 	size_t max_entries;
-	int i;
+	int i, ret;
 
 	if (region_size < sizeof(*xh))
 		return ocfs2_error(sb,
@@ -1012,6 +1049,11 @@ static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno,
 		struct ocfs2_xattr_entry *xe = &xh->xh_entries[i];
 		size_t name_offset = le16_to_cpu(xe->xe_name_offset);
 		size_t value_offset;
+		u64 value_len = le64_to_cpu(xe->xe_value_size);
+
+		ret = ocfs2_validate_xattr_entry(sb, blkno, xe);
+		if (ret)
+			return ret;
 
 		if (name_offset > nv_limit ||
 		    xe->xe_name_len > nv_limit - name_offset)
@@ -1026,8 +1068,7 @@ static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno,
 					   (unsigned long long)blkno, i);
 
 		if (ocfs2_xattr_is_local(xe)) {
-			if (le64_to_cpu(xe->xe_value_size) >
-			    nv_limit - value_offset)
+			if (value_len > nv_limit - value_offset)
 				return ocfs2_error(sb,
 						   "Invalid xattr in block %llu: entry %d value is out of bounds\n",
 						   (unsigned long long)blkno,
@@ -1109,10 +1150,29 @@ static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
 	struct ocfs2_xattr_header *xh = bucket_xh(bucket);
 	u16 xattr_count = le16_to_cpu(xh->xh_count);
 	size_t region_size = (size_t)sb->s_blocksize * bucket->bu_blocks;
-	size_t entries_limit = sb->s_blocksize;
+	/*
+	 * The entry array grows up from the header across the whole
+	 * bucket region, so it may extend beyond the first bucket block
+	 * when the blocksize is smaller than OCFS2_XATTR_BUCKET_SIZE.
+	 * Name/value pairs, however, always live within a single block.
+	 */
+	size_t entries_limit = region_size;
 	size_t nv_limit = sb->s_blocksize;
 	size_t max_entries;
-	int i;
+	int i, ret;
+
+	/*
+	 * The entry array is one contiguous region that may span the
+	 * bucket's buffer_heads.  Buckets are allocated within clusters,
+	 * so their first block is always aligned to
+	 * OCFS2_XATTR_BUCKET_SIZE and the whole bucket fits in one page.
+	 * A corrupted xattr tree can point a bucket at blocks straddling
+	 * a page, so reject it before touching the entry array.
+	 */
+	if (blkno & (bucket->bu_blocks - 1))
+		return ocfs2_error(sb,
+				   "Invalid xattr bucket %llu: unaligned block number\n",
+				   (unsigned long long)blkno);
 
 	if (region_size < sizeof(*xh))
 		return ocfs2_error(sb,
@@ -1140,6 +1200,11 @@ static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
 		size_t block_off = name_offset >> sb->s_blocksize_bits;
 		size_t block_offset = name_offset % nv_limit;
 		size_t value_offset;
+		u64 value_len = le64_to_cpu(xe->xe_value_size);
+
+		ret = ocfs2_validate_xattr_entry(sb, blkno, xe);
+		if (ret)
+			return ret;
 
 		if (name_offset >= region_size || block_off >= bucket->bu_blocks)
 			return ocfs2_error(sb,
@@ -1158,8 +1223,7 @@ static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket,
 					   (unsigned long long)blkno, i);
 
 		if (ocfs2_xattr_is_local(xe)) {
-			if (le64_to_cpu(xe->xe_value_size) >
-			    nv_limit - value_offset)
+			if (value_len > nv_limit - value_offset)
 				return ocfs2_error(sb,
 						   "Invalid xattr bucket %llu: entry %d value is out of bounds\n",
 						   (unsigned long long)blkno,
@@ -1307,7 +1371,7 @@ static int ocfs2_xattr_find_entry(struct inode *inode, int name_index,
 {
 	struct ocfs2_xattr_entry *entry;
 	size_t name_len;
-	int i, name_offset, cmp = 1;
+	int i, name_offset, cmp = 1, ret;
 
 	if (name == NULL)
 		return -EINVAL;
@@ -1330,6 +1394,12 @@ static int ocfs2_xattr_find_entry(struct inode *inode, int name_index,
 				return -EFSCORRUPTED;
 			}
 			cmp = memcmp(name, (xs->base + name_offset), name_len);
+			if (!cmp) {
+				ret = ocfs2_validate_xattr_entry(inode->i_sb,
+								 OCFS2_I(inode)->ip_blkno, entry);
+				if (ret)
+					return ret;
+			}
 		}
 		if (cmp == 0)
 			break;
@@ -2076,12 +2146,17 @@ static void ocfs2_xa_bucket_add_entry(struct ocfs2_xa_loc *loc, u32 name_hash)
 		}
 	}
 
+	/*
+	 * Increment xh_count before memmove() so __counted_by_le(xh_count)
+	 * includes the new entry in the destination bounds.
+	 */
+	le16_add_cpu(&xh->xh_count, 1);
+
 	if (low != count)
 		memmove(&xh->xh_entries[low + 1],
 			&xh->xh_entries[low],
 			((count - low) * sizeof(struct ocfs2_xattr_entry)));
 
-	le16_add_cpu(&xh->xh_count, 1);
 	loc->xl_entry = &xh->xh_entries[low];
 	memset(loc->xl_entry, 0, sizeof(struct ocfs2_xattr_entry));
 }
@@ -2912,6 +2987,9 @@ static int ocfs2_xattr_has_space_inline(struct inode *inode,
  *
  * Find extended attribute in inode block and
  * fill search info into struct ocfs2_xattr_search.
+ *
+ * The inline free-space check races with truncate and allocation, so
+ * callers must hold ip_alloc_sem for writing.
  */
 static int ocfs2_xattr_ibody_find(struct inode *inode,
 				  int name_index,
@@ -2923,13 +3001,13 @@ static int ocfs2_xattr_ibody_find(struct inode *inode,
 	int ret;
 	int has_space = 0;
 
+	lockdep_assert_held_write(&oi->ip_alloc_sem);
+
 	if (inode->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE)
 		return 0;
 
 	if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) {
-		down_read(&oi->ip_alloc_sem);
 		has_space = ocfs2_xattr_has_space_inline(inode, di);
-		up_read(&oi->ip_alloc_sem);
 		if (!has_space)
 			return 0;
 	}
@@ -3010,6 +3088,7 @@ static int ocfs2_xattr_ibody_init(struct inode *inode,
  *
  * Set, replace or remove an extended attribute into inode block.
  *
+ * Callers must hold ip_alloc_sem for writing.
  */
 static int ocfs2_xattr_ibody_set(struct inode *inode,
 				 struct ocfs2_xattr_info *xi,
@@ -3020,16 +3099,17 @@ static int ocfs2_xattr_ibody_set(struct inode *inode,
 	struct ocfs2_inode_info *oi = OCFS2_I(inode);
 	struct ocfs2_xa_loc loc;
 
+	lockdep_assert_held_write(&oi->ip_alloc_sem);
+
 	if (inode->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE)
 		return -ENOSPC;
 
-	down_write(&oi->ip_alloc_sem);
 	if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) {
 		ret = ocfs2_xattr_ibody_init(inode, xs->inode_bh, ctxt);
 		if (ret) {
 			if (ret != -ENOSPC)
 				mlog_errno(ret);
-			goto out;
+			return ret;
 		}
 	}
 
@@ -3039,13 +3119,10 @@ static int ocfs2_xattr_ibody_set(struct inode *inode,
 	if (ret) {
 		if (ret != -ENOSPC)
 			mlog_errno(ret);
-		goto out;
+		return ret;
 	}
 	xs->here = loc.xl_entry;
 
-out:
-	up_write(&oi->ip_alloc_sem);
-
 	return ret;
 }
 
@@ -3689,6 +3766,18 @@ static int __ocfs2_xattr_set_handle(struct inode *inode,
 	return ret;
 }
 
+/*
+ * ip_alloc_sem subclass for inodes being initialized before publication.
+ * ocfs2_xattr_set_handle() runs inside the create transaction, so taking
+ * ip_alloc_sem there adds a transaction -> ip_alloc_sem order that would
+ * form a lockdep cycle with the ip_alloc_sem -> transaction order used
+ * elsewhere, if not for this separate subclass.  The inode is unpublished
+ * so the acquisition can never contend.
+ */
+enum {
+	OCFS2_IP_ALLOC_SEM_UNPUBLISHED = 1,
+};
+
 /*
  * This helper is only for setting initial ACL or security xattrs on an inode
  * that is still unpublished, unhashed, and unattached to a dentry.
@@ -3750,6 +3839,13 @@ int ocfs2_xattr_set_handle(handle_t *handle,
 	xis.inode_bh = xbs.inode_bh = di_bh;
 	di = (struct ocfs2_dinode *)di_bh->b_data;
 
+	/*
+	 * The inode is unpublished and cannot contend, but take the
+	 * semaphore anyway so the helpers' lockdep assertions hold.
+	 */
+	down_write_nested(&OCFS2_I(inode)->ip_alloc_sem,
+			  OCFS2_IP_ALLOC_SEM_UNPUBLISHED);
+
 	ret = ocfs2_xattr_ibody_find(inode, name_index, name, &xis);
 	if (ret)
 		goto cleanup;
@@ -3762,6 +3858,7 @@ int ocfs2_xattr_set_handle(handle_t *handle,
 	ret = __ocfs2_xattr_set_handle(inode, di, &xi, &xis, &xbs, &ctxt);
 
 cleanup:
+	up_write(&OCFS2_I(inode)->ip_alloc_sem);
 	brelse(xbs.xattr_bh);
 	ocfs2_xattr_bucket_free(xbs.bucket);
 
@@ -3830,30 +3927,38 @@ int ocfs2_xattr_set(struct inode *inode,
 	di = (struct ocfs2_dinode *)di_bh->b_data;
 
 	down_write(&OCFS2_I(inode)->ip_xattr_sem);
+	/*
+	 * The allocation and truncate paths take ip_alloc_sem before
+	 * starting a transaction, so take it here before xattr
+	 * preparation, allocation reservations and ocfs2_start_trans()
+	 * to keep that order.  The xattr helpers below no longer take
+	 * it themselves.
+	 */
+	down_write(&OCFS2_I(inode)->ip_alloc_sem);
 	/*
 	 * Scan inode and external block to find the same name
 	 * extended attribute and collect search information.
 	 */
 	ret = ocfs2_xattr_ibody_find(inode, name_index, name, &xis);
 	if (ret)
-		goto cleanup;
+		goto out_free_ac;
 	if (xis.not_found) {
 		ret = ocfs2_xattr_block_find(inode, name_index, name, &xbs);
 		if (ret)
-			goto cleanup;
+			goto out_free_ac;
 	}
 
 	if (xis.not_found && xbs.not_found) {
 		ret = -ENODATA;
 		if (flags & XATTR_REPLACE)
-			goto cleanup;
+			goto out_free_ac;
 		ret = 0;
 		if (!value)
-			goto cleanup;
+			goto out_free_ac;
 	} else {
 		ret = -EEXIST;
 		if (flags & XATTR_CREATE)
-			goto cleanup;
+			goto out_free_ac;
 	}
 
 	/* Check whether the value is refcounted and do some preparation. */
@@ -3864,7 +3969,7 @@ int ocfs2_xattr_set(struct inode *inode,
 						   &ref_meta, &ref_credits);
 		if (ret) {
 			mlog_errno(ret);
-			goto cleanup;
+			goto out_free_ac;
 		}
 	}
 
@@ -3875,7 +3980,7 @@ int ocfs2_xattr_set(struct inode *inode,
 		if (ret < 0) {
 			inode_unlock(tl_inode);
 			mlog_errno(ret);
-			goto cleanup;
+			goto out_free_ac;
 		}
 	}
 	inode_unlock(tl_inode);
@@ -3884,7 +3989,7 @@ int ocfs2_xattr_set(struct inode *inode,
 					&xbs, &ctxt, ref_meta, &credits);
 	if (ret) {
 		mlog_errno(ret);
-		goto cleanup;
+		goto out_free_ac;
 	}
 
 	/* we need to update inode's ctime field, so add credit for it. */
@@ -3902,6 +4007,7 @@ int ocfs2_xattr_set(struct inode *inode,
 	ocfs2_commit_trans(osb, ctxt.handle);
 
 out_free_ac:
+	up_write(&OCFS2_I(inode)->ip_alloc_sem);
 	if (ctxt.data_ac)
 		ocfs2_free_alloc_context(ctxt.data_ac);
 	if (ctxt.meta_ac)
@@ -3910,7 +4016,6 @@ int ocfs2_xattr_set(struct inode *inode,
 		ocfs2_schedule_truncate_log_flush(osb, 1);
 	ocfs2_run_deallocs(osb, &ctxt.dealloc);
 
-cleanup:
 	if (ref_tree)
 		ocfs2_unlock_refcount_tree(osb, ref_tree, 1);
 	up_write(&OCFS2_I(inode)->ip_xattr_sem);
@@ -4041,6 +4146,10 @@ static int ocfs2_find_xe_in_bucket(struct inode *inode,
 
 		xe_name = bucket_block(bucket, block_off) + new_offset;
 		if (!memcmp(name, xe_name, name_len)) {
+			ret = ocfs2_validate_xattr_entry(inode->i_sb,
+							 OCFS2_I(inode)->ip_blkno, xe);
+			if (ret)
+				break;
 			*xe_index = i;
 			*found = 1;
 			ret = 0;
@@ -4393,7 +4502,8 @@ static int ocfs2_xattr_tree_list_index_block(struct inode *inode,
 	ret = ocfs2_iterate_xattr_index_block(inode, blk_bh,
 					      ocfs2_list_xattr_tree_rec, &xl);
 	if (ret) {
-		mlog_errno(ret);
+		if (ret != -ERANGE)
+			mlog_errno(ret);
 		goto out;
 	}
 
@@ -4511,6 +4621,10 @@ static void ocfs2_xattr_update_xattr_search(struct inode *inode,
 	xs->here = &xs->header->xh_entries[i];
 }
 
+/*
+ * Caller must hold ip_alloc_sem for writing, since a new xattr block
+ * is allocated and the xattr block header is rewritten.
+ */
 static int ocfs2_xattr_create_index_block(struct inode *inode,
 					  struct ocfs2_xattr_search *xs,
 					  struct ocfs2_xattr_set_ctxt *ctxt)
@@ -4526,19 +4640,14 @@ static int ocfs2_xattr_create_index_block(struct inode *inode,
 	struct ocfs2_xattr_tree_root *xr;
 	u16 xb_flags = le16_to_cpu(xb->xb_flags);
 
+	lockdep_assert_held_write(&oi->ip_alloc_sem);
+
 	trace_ocfs2_xattr_create_index_block_begin(
 				(unsigned long long)xb_bh->b_blocknr);
 
 	BUG_ON(xb_flags & OCFS2_XATTR_INDEXED);
 	BUG_ON(!xs->bucket);
 
-	/*
-	 * XXX:
-	 * We can use this lock for now, and maybe move to a dedicated mutex
-	 * if performance becomes a problem later.
-	 */
-	down_write(&oi->ip_alloc_sem);
-
 	ret = ocfs2_journal_access_xb(handle, INODE_CACHE(inode), xb_bh,
 				      OCFS2_JOURNAL_ACCESS_WRITE);
 	if (ret) {
@@ -4600,8 +4709,6 @@ static int ocfs2_xattr_create_index_block(struct inode *inode,
 	ocfs2_journal_dirty(handle, xb_bh);
 
 out:
-	up_write(&oi->ip_alloc_sem);
-
 	return ret;
 }
 
@@ -4681,6 +4788,9 @@ static int ocfs2_defrag_xattr_bucket(struct inode *inode,
 	xe = xh->xh_entries;
 	end = OCFS2_XATTR_BUCKET_SIZE;
 	for (i = 0; i < le16_to_cpu(xh->xh_count); i++, xe++) {
+		ret = ocfs2_validate_xattr_entry(inode->i_sb, blkno, xe);
+		if (ret)
+			goto out;
 		offset = le16_to_cpu(xe->xe_name_offset);
 		len = namevalue_size_xe(xe);
 
@@ -4697,16 +4807,22 @@ static int ocfs2_defrag_xattr_bucket(struct inode *inode,
 			memmove(bucket_buf + end - len,
 				bucket_buf + offset, len);
 			xe->xe_name_offset = cpu_to_le16(end - len);
+		} else if (end < offset + len) {
+			ret = ocfs2_error(inode->i_sb,
+					  "Defrag check failed for bucket %llu\n",
+					  (unsigned long long)blkno);
+			goto out;
 		}
 
-		mlog_bug_on_msg(end < offset + len, "Defrag check failed for "
-				"bucket %llu\n", (unsigned long long)blkno);
-
 		end -= len;
 	}
 
-	mlog_bug_on_msg(xh_free_start > end, "Defrag check failed for "
-			"bucket %llu\n", (unsigned long long)blkno);
+	if (xh_free_start > end) {
+		ret = ocfs2_error(inode->i_sb,
+				  "Defrag check failed for bucket %llu\n",
+				  (unsigned long long)blkno);
+		goto out;
+	}
 
 	if (xh_free_start == end)
 		goto out;
@@ -4963,6 +5079,9 @@ static int ocfs2_divide_xattr_bucket(struct inode *inode,
 	name_value_len = 0;
 	for (i = 0; i < start; i++) {
 		xe = &xh->xh_entries[i];
+		ret = ocfs2_validate_xattr_entry(inode->i_sb, blk, xe);
+		if (ret)
+			goto out;
 		name_value_len += namevalue_size_xe(xe);
 		if (le16_to_cpu(xe->xe_name_offset) < name_offset)
 			name_offset = le16_to_cpu(xe->xe_name_offset);
@@ -7503,7 +7622,7 @@ static int ocfs2_xattr_security_get(const struct xattr_handler *handler,
 }
 
 static int ocfs2_xattr_security_set(const struct xattr_handler *handler,
-				    struct mnt_idmap *idmap,
+				    const struct mnt_idmap *idmap,
 				    struct dentry *unused, struct inode *inode,
 				    const char *name, const void *value,
 				    size_t size, int flags)
@@ -7598,7 +7717,7 @@ static int ocfs2_xattr_trusted_get(const struct xattr_handler *handler,
 }
 
 static int ocfs2_xattr_trusted_set(const struct xattr_handler *handler,
-				   struct mnt_idmap *idmap,
+				   const struct mnt_idmap *idmap,
 				   struct dentry *unused, struct inode *inode,
 				   const char *name, const void *value,
 				   size_t size, int flags)
@@ -7629,7 +7748,7 @@ static int ocfs2_xattr_user_get(const struct xattr_handler *handler,
 }
 
 static int ocfs2_xattr_user_set(const struct xattr_handler *handler,
-				struct mnt_idmap *idmap,
+				const struct mnt_idmap *idmap,
 				struct dentry *unused, struct inode *inode,
 				const char *name, const void *value,
 				size_t size, int flags)
diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h
index 5e18513277f18b..887cc1a18b1aff 100644
--- a/fs/ocfs2/xattr.h
+++ b/fs/ocfs2/xattr.h
@@ -59,10 +59,10 @@ int ocfs2_calc_security_init(struct inode *,
 			     int *, int *, struct ocfs2_alloc_context **);
 
 struct ocfs2_acl_state;
-int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode,
-			  struct ocfs2_security_xattr_info *si,
-			  int *want_clusters, int *xattr_credits,
-			  int *want_meta, struct ocfs2_acl_state *acl_state);
+void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode,
+			   struct ocfs2_security_xattr_info *si,
+			   int *want_clusters, int *xattr_credits,
+			   int *want_meta, struct ocfs2_acl_state *acl_state);
 
 /*
  * xattrs can live inside an inode, as part of an external xattr block,
diff --git a/fs/omfs/dir.c b/fs/omfs/dir.c
index 692297cf84e745..18f4b4543cc81a 100644
--- a/fs/omfs/dir.c
+++ b/fs/omfs/dir.c
@@ -279,13 +279,13 @@ static int omfs_add_node(struct inode *dir, struct dentry *dentry, umode_t mode)
 	return err;
 }
 
-static struct dentry *omfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *omfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	return ERR_PTR(omfs_add_node(dir, dentry, mode));
 }
 
-static int omfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int omfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode)
 {
 	return omfs_add_node(dir, dentry, mode | S_IFREG);
@@ -370,7 +370,7 @@ static bool omfs_fill_chain(struct inode *dir, struct dir_context *ctx,
 	return true;
 }
 
-static int omfs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int omfs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		       struct dentry *old_dentry, struct inode *new_dir,
 		       struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/omfs/file.c b/fs/omfs/file.c
index 28f3b113340e89..79a413f1dc0dbf 100644
--- a/fs/omfs/file.c
+++ b/fs/omfs/file.c
@@ -338,7 +338,7 @@ const struct file_operations omfs_file_operations = {
 	.splice_read = filemap_splice_read,
 };
 
-static int omfs_setattr(struct mnt_idmap *idmap,
+static int omfs_setattr(const struct mnt_idmap *idmap,
 			struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/omfs/inode.c b/fs/omfs/inode.c
index 1d915ef72119fb..bc37029a4afb1f 100644
--- a/fs/omfs/inode.c
+++ b/fs/omfs/inode.c
@@ -145,7 +145,7 @@ static int __omfs_write_inode(struct inode *inode, int wait)
 	mark_buffer_dirty(bh);
 	if (wait) {
 		sync_dirty_buffer(bh);
-		if (buffer_req(bh) && !buffer_uptodate(bh))
+		if (buffer_write_io_error(bh))
 			sync_failed = 1;
 	}
 
@@ -159,7 +159,7 @@ static int __omfs_write_inode(struct inode *inode, int wait)
 		mark_buffer_dirty(bh2);
 		if (wait) {
 			sync_dirty_buffer(bh2);
-			if (buffer_req(bh2) && !buffer_uptodate(bh2))
+			if (buffer_write_io_error(bh2))
 				sync_failed = 1;
 		}
 		brelse(bh2);
diff --git a/fs/open.c b/fs/open.c
index 6b1c14e684a93b..e43f02ff64acbe 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -36,7 +36,7 @@
 
 #include "internal.h"
 
-int do_truncate(struct mnt_idmap *idmap, struct dentry *dentry,
+int do_truncate(const struct mnt_idmap *idmap, struct dentry *dentry,
 		loff_t length, unsigned int time_attrs, struct file *filp)
 {
 	int ret;
@@ -72,7 +72,7 @@ int do_truncate(struct mnt_idmap *idmap, struct dentry *dentry,
 
 int vfs_truncate(const struct path *path, loff_t length)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct inode *inode;
 	int error;
 
@@ -787,7 +787,7 @@ static inline bool setattr_vfsgid(struct iattr *attr, kgid_t kgid)
 
 int chown_common(const struct path *path, uid_t user, gid_t group)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct user_namespace *fs_userns;
 	struct inode *inode = path->dentry->d_inode;
 	struct delegated_inode delegated_inode = { };
@@ -931,6 +931,11 @@ static inline int file_get_write_access(struct file *f)
 	return error;
 }
 
+/*
+ * Populate struct file
+ *
+ * NOTE: it assumes f_path is populated and consumes the caller's reference.
+ */
 static int do_dentry_open(struct file *f,
 			  int (*open)(struct inode *, struct file *))
 {
@@ -938,7 +943,6 @@ static int do_dentry_open(struct file *f,
 	struct inode *inode = f->f_path.dentry->d_inode;
 	int error;
 
-	path_get(&f->f_path);
 	f->f_inode = inode;
 	f->f_mapping = inode->i_mapping;
 	f->f_wb_err = filemap_sample_wb_err(f->f_mapping);
@@ -1055,6 +1059,7 @@ int finish_open(struct file *file, struct dentry *dentry,
 	BUG_ON(file->f_mode & FMODE_OPENED); /* once it's opened, it's opened */
 
 	file->__f_path.dentry = dentry;
+	path_get(&file->f_path);
 	return do_dentry_open(file, open);
 }
 EXPORT_SYMBOL(finish_open);
@@ -1098,6 +1103,7 @@ int vfs_open(const struct path *path, struct file *file)
 	int ret;
 
 	file->__f_path = *path;
+	path_get(&file->f_path);
 	ret = do_dentry_open(file, NULL);
 	if (!ret) {
 		/*
@@ -1110,6 +1116,25 @@ int vfs_open(const struct path *path, struct file *file)
 	return ret;
 }
 
+/**
+ * vfs_open_consume - open the file at the given path and consume the reference
+ * @path: path to open
+ * @file: newly allocated file with f_flag initialized
+ */
+int vfs_open_consume(struct path *path, struct file *file)
+{
+	int ret;
+
+	file->__f_path = *path;
+	path->mnt = NULL;
+	path->dentry = NULL;
+	ret = do_dentry_open(file, NULL);
+	if (!ret) {
+		fsnotify_open(file);
+	}
+	return ret;
+}
+
 struct file *dentry_open(const struct path *path, int flags,
 			 const struct cred *cred)
 {
@@ -1537,6 +1562,19 @@ int filp_close(struct file *filp, fl_owner_t id)
 }
 EXPORT_SYMBOL(filp_close);
 
+/* Like filp_close() but the last reference is put right here. */
+int filp_close_sync(struct file *filp, fl_owner_t id)
+{
+	int retval;
+
+	/* Kernel threads must never put their final reference here. */
+	VFS_WARN_ON_ONCE(current->flags & PF_KTHREAD);
+	retval = filp_flush(filp, id);
+	fput_close_sync(filp);
+
+	return retval;
+}
+
 /*
  * Careful here! We test whether the file pointer is NULL before
  * releasing the fd. This ensures that one clone task can't release
@@ -1551,13 +1589,11 @@ SYSCALL_DEFINE1(close, unsigned int, fd)
 	if (!file)
 		return -EBADF;
 
-	retval = filp_flush(file, current->files);
-
 	/*
 	 * We're returning to user space. Don't bother
 	 * with any delayed fput() cases.
 	 */
-	fput_close_sync(file);
+	retval = filp_close_sync(file, current->files);
 
 	if (likely(retval == 0))
 		return 0;
diff --git a/fs/orangefs/acl.c b/fs/orangefs/acl.c
index a01ef0c1b1bf2b..f31196e4bfaae6 100644
--- a/fs/orangefs/acl.c
+++ b/fs/orangefs/acl.c
@@ -112,7 +112,7 @@ int __orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type)
 	return error;
 }
 
-int orangefs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int orangefs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		     struct posix_acl *acl, int type)
 {
 	int error;
diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c
index cd3273c88e030e..b1fed1c81a4dce 100644
--- a/fs/orangefs/inode.c
+++ b/fs/orangefs/inode.c
@@ -181,8 +181,27 @@ static int orangefs_writepages(struct address_space *mapping,
 {
 	struct orangefs_writepages *ow;
 	struct blk_plug plug;
-	int error;
+	int error = 0;
 	struct folio *folio = NULL;
+	int maxpages;
+
+	maxpages = orangefs_bufmap_size_query() / PAGE_SIZE;
+	if (maxpages < 1) {
+		/*
+		 * Probably the client is dead and there's no bufmap.
+		 * Walk writeback_iter anyway so each dirty folio is unlocked
+		 * and writeback is ended. wait_for_direct_io will fail; the
+		 * data is not written.
+		 */
+		gossip_err("%s: maxpages < 1. \n", __func__);
+		while ((folio = writeback_iter(mapping, wbc, folio, &error))) {
+			error = orangefs_writepage_locked(folio, wbc);
+			mapping_set_error(mapping, error);
+			folio_unlock(folio);
+			folio_end_writeback(folio);
+		}
+		return error;
+	}
 
 	ow = kzalloc_obj(struct orangefs_writepages);
 	if (!ow)
@@ -828,7 +847,7 @@ int __orangefs_setattr_mode(struct dentry *dentry, struct iattr *iattr)
 /*
  * Change attributes of an object referenced by dentry.
  */
-int orangefs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int orangefs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		     struct iattr *iattr)
 {
 	int ret;
@@ -848,7 +867,7 @@ int orangefs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 /*
  * Obtain attributes of an object given a dentry
  */
-int orangefs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int orangefs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		     struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	int ret;
@@ -872,7 +891,7 @@ int orangefs_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return ret;
 }
 
-int orangefs_permission(struct mnt_idmap *idmap,
+int orangefs_permission(const struct mnt_idmap *idmap,
 			struct inode *inode, int mask)
 {
 	int ret;
@@ -934,7 +953,7 @@ static int orangefs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-static int orangefs_fileattr_set(struct mnt_idmap *idmap,
+static int orangefs_fileattr_set(const struct mnt_idmap *idmap,
 				 struct dentry *dentry, struct file_kattr *fa)
 {
 	u64 val = 0;
diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c
index 8ebc34e112d514..32b7769ea49cea 100644
--- a/fs/orangefs/namei.c
+++ b/fs/orangefs/namei.c
@@ -15,7 +15,7 @@
 /*
  * Get a newly allocated inode to go with a negative dentry.
  */
-static int orangefs_create(struct mnt_idmap *idmap,
+static int orangefs_create(const struct mnt_idmap *idmap,
 			struct inode *dir,
 			struct dentry *dentry,
 			umode_t mode)
@@ -211,7 +211,7 @@ static int orangefs_unlink(struct inode *dir, struct dentry *dentry)
 	return ret;
 }
 
-static int orangefs_symlink(struct mnt_idmap *idmap,
+static int orangefs_symlink(const struct mnt_idmap *idmap,
 		         struct inode *dir,
 			 struct dentry *dentry,
 			 const char *symname)
@@ -296,7 +296,7 @@ static int orangefs_symlink(struct mnt_idmap *idmap,
 	return ret;
 }
 
-static struct dentry *orangefs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *orangefs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				     struct dentry *dentry, umode_t mode)
 {
 	struct orangefs_inode_s *parent = ORANGEFS_I(dir);
@@ -364,7 +364,7 @@ static struct dentry *orangefs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	return ret ? ERR_PTR(ret) : NULL;
 }
 
-static int orangefs_rename(struct mnt_idmap *idmap,
+static int orangefs_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir,
 			struct dentry *old_dentry,
 			struct inode *new_dir,
diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c
index 9f94919a6bc626..fec7c1e864d2cc 100644
--- a/fs/orangefs/orangefs-debugfs.c
+++ b/fs/orangefs/orangefs-debugfs.c
@@ -238,7 +238,7 @@ void orangefs_debugfs_init(int debug_mask)
 static void orangefs_kernel_debug_init(void)
 {
 	static char k_buffer[ORANGEFS_MAX_DEBUG_STRING_LEN] = { };
-	size_t len =
+	ssize_t len =
 		strscpy(k_buffer, kernel_debug_string, sizeof(k_buffer) - 1);
 
 	if (len > 0) {
@@ -246,7 +246,8 @@ static void orangefs_kernel_debug_init(void)
 		k_buffer[len + 1] = '\0';
 	} else {
 		strscpy(k_buffer, "none\n");
-		pr_info("%s: overflow 1!\n", __func__);
+		if (len <0)
+			pr_info("%s: overflow 1!\n", __func__);
 	}
 
 	debugfs_create_file_aux_num(ORANGEFS_KMOD_DEBUG_FILE, 0444, debug_dir, k_buffer,
@@ -337,7 +338,7 @@ static int help_show(struct seq_file *m, void *v)
 static void orangefs_client_debug_init(void)
 {
 	static char c_buffer[ORANGEFS_MAX_DEBUG_STRING_LEN] = { };
-	size_t len =
+	ssize_t len =
 		strscpy(c_buffer, client_debug_string, sizeof(c_buffer) - 1);
 
 	if (len > 0) {
@@ -345,7 +346,8 @@ static void orangefs_client_debug_init(void)
 		c_buffer[len + 1] = '\0';
 	} else {
 		strscpy(c_buffer, "none\n");
-		pr_info("%s: overflow! 2\n", __func__);
+		if (len <0)
+			pr_info("%s: overflow! 2\n", __func__);
 	}
 
 	client_debug_dentry = debugfs_create_file_aux_num(
diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h
index 1451fc2c1917c4..348fe340c5d5fc 100644
--- a/fs/orangefs/orangefs-kernel.h
+++ b/fs/orangefs/orangefs-kernel.h
@@ -98,7 +98,7 @@ enum orangefs_vfs_op_states {
 extern const struct xattr_handler * const orangefs_xattr_handlers[];
 
 extern struct posix_acl *orangefs_get_acl(struct inode *inode, int type, bool rcu);
-extern int orangefs_set_acl(struct mnt_idmap *idmap,
+extern int orangefs_set_acl(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct posix_acl *acl,
 			    int type);
 int __orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type);
@@ -352,12 +352,12 @@ struct inode *orangefs_new_inode(struct super_block *sb,
 
 int __orangefs_setattr(struct inode *, struct iattr *);
 int __orangefs_setattr_mode(struct dentry *dentry, struct iattr *iattr);
-int orangefs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *);
+int orangefs_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *);
 
-int orangefs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int orangefs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		     struct kstat *stat, u32 request_mask, unsigned int flags);
 
-int orangefs_permission(struct mnt_idmap *idmap,
+int orangefs_permission(const struct mnt_idmap *idmap,
 			struct inode *inode, int mask);
 
 int orangefs_update_time(struct inode *inode, enum fs_update_time type,
diff --git a/fs/orangefs/super.c b/fs/orangefs/super.c
index 4ec7329b41f613..e6eedb2faef769 100644
--- a/fs/orangefs/super.c
+++ b/fs/orangefs/super.c
@@ -383,6 +383,24 @@ static int orangefs_unmount(int id, __s32 fs_id, const char *devname)
 {
 	struct orangefs_kernel_op_s *op;
 	int r;
+
+	/*
+	 * If someone reboots linux without unmounting orangefs first,
+	 * don't bother firing off an unmount service operation that
+	 * will never complete, or the whole system shutdown
+	 * stalls for ORANGEFS_DEFAULT_OP_TIMEOUT_SECS.
+	 */
+
+	if (!__is_daemon_in_service() ||
+		system_state == SYSTEM_HALT ||
+		system_state == SYSTEM_POWER_OFF ||
+		system_state == SYSTEM_RESTART) {
+			gossip_debug(GOSSIP_SUPER_DEBUG,
+				"%s: dirty unmount.\n",
+				__func__);
+			return 0;
+	}
+
 	op = op_alloc(ORANGEFS_VFS_OP_FS_UMOUNT);
 	if (!op)
 		return -ENOMEM;
diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c
index 885fd3bd5a3da7..a49e64566e1eb9 100644
--- a/fs/orangefs/xattr.c
+++ b/fs/orangefs/xattr.c
@@ -527,7 +527,7 @@ ssize_t orangefs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 }
 
 static int orangefs_xattr_set_default(const struct xattr_handler *handler,
-				      struct mnt_idmap *idmap,
+				      const struct mnt_idmap *idmap,
 				      struct dentry *unused,
 				      struct inode *inode,
 				      const char *name,
diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c
index 7beb0af2649847..1194ccf981c37b 100644
--- a/fs/overlayfs/dir.c
+++ b/fs/overlayfs/dir.c
@@ -688,7 +688,7 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode,
 	return err;
 }
 
-static int ovl_create_object(struct mnt_idmap *idmap, struct dentry *dentry,
+static int ovl_create_object(const struct mnt_idmap *idmap, struct dentry *dentry,
 			     int mode, dev_t rdev, const char *link)
 {
 	int err;
@@ -730,19 +730,19 @@ static int ovl_create_object(struct mnt_idmap *idmap, struct dentry *dentry,
 	return err;
 }
 
-static int ovl_create(struct mnt_idmap *idmap, struct inode *dir,
+static int ovl_create(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode)
 {
 	return ovl_create_object(idmap, dentry, (mode & 07777) | S_IFREG, 0, NULL);
 }
 
-static struct dentry *ovl_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ovl_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode)
 {
 	return ERR_PTR(ovl_create_object(idmap, dentry, (mode & 07777) | S_IFDIR, 0, NULL));
 }
 
-static int ovl_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int ovl_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	/* Don't allow creation of "whiteout" on overlay */
@@ -752,7 +752,7 @@ static int ovl_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return ovl_create_object(idmap, dentry, mode, rdev, NULL);
 }
 
-static int ovl_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int ovl_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, const char *link)
 {
 	return ovl_create_object(idmap, dentry, S_IFLNK, 0, link);
@@ -1344,7 +1344,7 @@ static void ovl_rename_end(struct ovl_renamedata *ovlrd)
 		ovl_drop_write(ovlrd->old_dentry);
 }
 
-static int ovl_rename(struct mnt_idmap *idmap, struct inode *olddir,
+static int ovl_rename(const struct mnt_idmap *idmap, struct inode *olddir,
 		      struct dentry *old, struct inode *newdir,
 		      struct dentry *new, unsigned int flags)
 {
@@ -1420,7 +1420,7 @@ static int ovl_dummy_open(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static int ovl_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int ovl_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct file *file, umode_t mode)
 {
 	int err;
diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c
index f3d97eb146e85b..7433220d4ad6c4 100644
--- a/fs/overlayfs/file.c
+++ b/fs/overlayfs/file.c
@@ -30,7 +30,7 @@ static struct file *ovl_open_realfile(const struct file *file,
 {
 	struct inode *realinode = d_inode(realpath->dentry);
 	struct inode *inode = file_inode(file);
-	struct mnt_idmap *real_idmap;
+	const struct mnt_idmap *real_idmap;
 	struct file *realfile;
 	int flags = file->f_flags | OVL_OPEN_FLAGS;
 	int acc_mode = ACC_MODE(flags);
diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c
index 401cb8c75520cb..70183d516e5e96 100644
--- a/fs/overlayfs/inode.c
+++ b/fs/overlayfs/inode.c
@@ -18,7 +18,7 @@
 #include "overlayfs.h"
 
 
-int ovl_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ovl_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *attr)
 {
 	int err;
@@ -168,7 +168,7 @@ static inline int ovl_real_getattr_nosec(struct super_block *sb,
 		return vfs_getattr_nosec(path, stat, request_mask, flags);
 }
 
-int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ovl_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	struct dentry *dentry = path->dentry;
@@ -303,7 +303,7 @@ int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return err;
 }
 
-int ovl_permission(struct mnt_idmap *idmap,
+int ovl_permission(const struct mnt_idmap *idmap,
 		   struct inode *inode, int mask)
 {
 	struct inode *upperinode = ovl_inode_upper(inode);
@@ -355,7 +355,7 @@ static const char *ovl_get_link(struct dentry *dentry,
  * alter the POSIX ACLs for the underlying filesystem.
  */
 static void ovl_idmap_posix_acl(const struct inode *realinode,
-				struct mnt_idmap *idmap,
+				const struct mnt_idmap *idmap,
 				struct posix_acl *acl)
 {
 	struct user_namespace *fs_userns = i_user_ns(realinode);
@@ -406,7 +406,7 @@ struct posix_acl *ovl_get_acl_path(const struct path *path,
 				   const char *acl_name, bool noperm)
 {
 	struct posix_acl *real_acl, *clone;
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct inode *realinode = d_inode(path->dentry);
 
 	idmap = mnt_idmap(path->mnt);
@@ -447,7 +447,7 @@ struct posix_acl *ovl_get_acl_path(const struct path *path,
  *
  * This is obviously only relevant when idmapped layers are used.
  */
-struct posix_acl *do_ovl_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *do_ovl_get_acl(const struct mnt_idmap *idmap,
 				 struct inode *inode, int type,
 				 bool rcu, bool noperm)
 {
@@ -536,7 +536,7 @@ static int ovl_set_or_remove_acl(struct dentry *dentry, struct inode *inode,
 	return err;
 }
 
-int ovl_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ovl_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct posix_acl *acl, int type)
 {
 	int err;
@@ -650,7 +650,7 @@ int ovl_real_fileattr_set(const struct path *realpath, struct file_kattr *fa)
 	return vfs_fileattr_set(mnt_idmap(realpath->mnt), realpath->dentry, fa);
 }
 
-int ovl_fileattr_set(struct mnt_idmap *idmap,
+int ovl_fileattr_set(const struct mnt_idmap *idmap,
 		     struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
index e0d8c6152e9fc5..53fbbe15c31dfd 100644
--- a/fs/overlayfs/overlayfs.h
+++ b/fs/overlayfs/overlayfs.h
@@ -254,8 +254,10 @@ static inline struct dentry *ovl_do_mkdir(struct ovl_fs *ofs,
 {
 	struct dentry *ret;
 
+	/* vfs_mkdir() drops @dentry on failure and may replace it on success */
+	pr_debug("mkdir(%pd2, 0%o)\n", dentry, mode);
 	ret = vfs_mkdir(ovl_upper_mnt_idmap(ofs), dir, dentry, mode, NULL);
-	pr_debug("mkdir(%pd2, 0%o) = %i\n", dentry, mode, PTR_ERR_OR_ZERO(ret));
+	pr_debug("...mkdir = %i\n", PTR_ERR_OR_ZERO(ret));
 	return ret;
 }
 
@@ -802,11 +804,11 @@ int ovl_set_nlink_lower(struct dentry *dentry);
 unsigned int ovl_get_nlink(struct ovl_fs *ofs, struct dentry *lowerdentry,
 			   struct dentry *upperdentry,
 			   unsigned int fallback);
-int ovl_permission(struct mnt_idmap *idmap, struct inode *inode,
+int ovl_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		   int mask);
 
 #ifdef CONFIG_FS_POSIX_ACL
-struct posix_acl *do_ovl_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *do_ovl_get_acl(const struct mnt_idmap *idmap,
 				 struct inode *inode, int type,
 				 bool rcu, bool noperm);
 static inline struct posix_acl *ovl_get_inode_acl(struct inode *inode, int type,
@@ -814,12 +816,12 @@ static inline struct posix_acl *ovl_get_inode_acl(struct inode *inode, int type,
 {
 	return do_ovl_get_acl(&nop_mnt_idmap, inode, type, rcu, true);
 }
-static inline struct posix_acl *ovl_get_acl(struct mnt_idmap *idmap,
+static inline struct posix_acl *ovl_get_acl(const struct mnt_idmap *idmap,
 					    struct dentry *dentry, int type)
 {
 	return do_ovl_get_acl(idmap, d_inode(dentry), type, false, false);
 }
-int ovl_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int ovl_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct posix_acl *acl, int type);
 struct posix_acl *ovl_get_acl_path(const struct path *path,
 				   const char *acl_name, bool noperm);
@@ -914,7 +916,7 @@ extern const struct file_operations ovl_file_operations;
 int ovl_real_fileattr_get(const struct path *realpath, struct file_kattr *fa);
 int ovl_real_fileattr_set(const struct path *realpath, struct file_kattr *fa);
 int ovl_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int ovl_fileattr_set(struct mnt_idmap *idmap,
+int ovl_fileattr_set(const struct mnt_idmap *idmap,
 		     struct dentry *dentry, struct file_kattr *fa);
 struct ovl_file;
 struct ovl_file *ovl_file_alloc(struct file *realfile);
@@ -948,8 +950,8 @@ static inline bool ovl_force_readonly(struct ovl_fs *ofs)
 /* xattr.c */
 
 const struct xattr_handler * const *ovl_xattr_handlers(struct ovl_fs *ofs);
-int ovl_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ovl_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *attr);
-int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ovl_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, u32 request_mask, unsigned int flags);
 ssize_t ovl_listxattr(struct dentry *dentry, char *list, size_t size);
diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h
index 80cad4ea96a3ed..ac07e8769f9be6 100644
--- a/fs/overlayfs/ovl_entry.h
+++ b/fs/overlayfs/ovl_entry.h
@@ -105,7 +105,7 @@ static inline struct vfsmount *ovl_upper_mnt(struct ovl_fs *ofs)
 	return ofs->layers[0].mnt;
 }
 
-static inline struct mnt_idmap *ovl_upper_mnt_idmap(struct ovl_fs *ofs)
+static inline const struct mnt_idmap *ovl_upper_mnt_idmap(struct ovl_fs *ofs)
 {
 	return mnt_idmap(ovl_upper_mnt(ofs));
 }
diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c
index b41f4788e4f06b..521717209b2e44 100644
--- a/fs/overlayfs/util.c
+++ b/fs/overlayfs/util.c
@@ -657,7 +657,7 @@ bool ovl_path_is_whiteout(struct ovl_fs *ofs, const struct path *path)
 struct file *ovl_path_open(const struct path *path, int flags)
 {
 	struct inode *inode = d_inode(path->dentry);
-	struct mnt_idmap *real_idmap = mnt_idmap(path->mnt);
+	const struct mnt_idmap *real_idmap = mnt_idmap(path->mnt);
 	int err, acc_mode;
 
 	if (flags & ~(O_ACCMODE | O_LARGEFILE))
@@ -1496,7 +1496,7 @@ void ovl_copyattr(struct inode *inode)
 {
 	struct path realpath;
 	struct inode *realinode;
-	struct mnt_idmap *real_idmap;
+	const struct mnt_idmap *real_idmap;
 	vfsuid_t vfsuid;
 	vfsgid_t vfsgid;
 
diff --git a/fs/overlayfs/xattrs.c b/fs/overlayfs/xattrs.c
index 5ae44b9c879031..acc54f6138ef5c 100644
--- a/fs/overlayfs/xattrs.c
+++ b/fs/overlayfs/xattrs.c
@@ -190,7 +190,7 @@ static int ovl_own_xattr_get(const struct xattr_handler *handler,
 }
 
 static int ovl_own_xattr_set(const struct xattr_handler *handler,
-			     struct mnt_idmap *idmap,
+			     const struct mnt_idmap *idmap,
 			     struct dentry *dentry, struct inode *inode,
 			     const char *name, const void *value,
 			     size_t size, int flags)
@@ -217,7 +217,7 @@ static int ovl_other_xattr_get(const struct xattr_handler *handler,
 }
 
 static int ovl_other_xattr_set(const struct xattr_handler *handler,
-			       struct mnt_idmap *idmap,
+			       const struct mnt_idmap *idmap,
 			       struct dentry *dentry, struct inode *inode,
 			       const char *name, const void *value,
 			       size_t size, int flags)
diff --git a/fs/pidfs.c b/fs/pidfs.c
index a6a643f15d08a9..c37c17bcdbdc80 100644
--- a/fs/pidfs.c
+++ b/fs/pidfs.c
@@ -823,13 +823,13 @@ static struct vfsmount *pidfs_mnt __ro_after_init;
  * implemented. Let's reject it completely until we have a clean
  * permission concept for pidfds.
  */
-static int pidfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int pidfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			 struct iattr *attr)
 {
 	return anon_inode_setattr(idmap, dentry, attr);
 }
 
-static int pidfs_getattr(struct mnt_idmap *idmap, const struct path *path,
+static int pidfs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 			 struct kstat *stat, u32 request_mask,
 			 unsigned int query_flags)
 {
@@ -1102,7 +1102,7 @@ static int pidfs_xattr_get(const struct xattr_handler *handler,
 }
 
 static int pidfs_xattr_set(const struct xattr_handler *handler,
-			   struct mnt_idmap *idmap, struct dentry *unused,
+			   const struct mnt_idmap *idmap, struct dentry *unused,
 			   struct inode *inode, const char *suffix,
 			   const void *value, size_t size, int flags)
 {
diff --git a/fs/pipe.c b/fs/pipe.c
index 292425a834dd58..5791db016e2426 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -1433,7 +1433,7 @@ int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
 	spin_unlock_irq(&pipe->rd_wait.lock);
 
 	/* This might have made more room for writers */
-	wake_up_interruptible(&pipe->wr_wait);
+	wake_up_interruptible_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
 	return 0;
 }
 
diff --git a/fs/pnode.c b/fs/pnode.c
index 5d91c3e58d2a35..2cd667958efe1b 100644
--- a/fs/pnode.c
+++ b/fs/pnode.c
@@ -410,19 +410,99 @@ bool propagation_would_overmount(const struct mount *from,
 	return false;
 }
 
+/* Does @m receive propagation from @parent? */
+static bool receives_from(struct mount *m, struct mount *parent)
+{
+	if (m == parent)
+		return false;
+	for (; m; m = m->mnt_master)
+		if (m == parent || peers(m, parent))
+			return true;
+	return false;
+}
+
+/*
+ * Does @m receive propagation from the victim's parent as well? If so, then
+ * the mount at the victim's mountpoint inside of @m is a umount candidate as
+ * well. So it's the next candidate in the chain. Otherwise the chain ends at
+ * @m.
+ */
+static struct mount *next_candidate(struct mount *m, struct mount *victim)
+{
+	if (!receives_from(m, victim->mnt_parent))
+		return NULL;
+	return __lookup_mnt(&m->mnt, victim->mnt_mountpoint);
+}
+
+/*
+ * Would propagate_umount() pull out a mount of the chain of candidates that
+ * starts at @c, and does that mount have references beyond its own?
+ *
+ * This mirrors how trim_one(), trim_ancestors() and handle_locked() handle a
+ * synchronous umount:
+ *
+ * - single victim
+ * - without children
+ * - with MNT_LOCKED already cleared on every candidate by propagate_mount_unlock()
+ *
+ * A copy of the victim gets unmounted when each of its children is
+ * the next candidate in the chain or its overmount, unless the next
+ * unmount candidate is not its overmount and some unmount candidate further
+ * down has a child outside the chain.  Keep this in sync with
+ * Documentation/filesystems/propagate_umount.txt.
+ */
+static bool chain_busy(struct mount *c, struct mount *victim)
+{
+	struct mount *m, *n, *next, *deepest = NULL;
+	bool above;
+
+	/* the deepest candidate with a child outside the chain */
+	for (m = c; m; m = next) {
+		next = next_candidate(m, victim);
+		list_for_each_entry(n, &m->mnt_mounts, mnt_child) {
+			if (n != next && n != victim) {
+				deepest = m;
+				break;
+			}
+		}
+	}
+
+	above = deepest != NULL;	/* @deepest is at or below @m */
+	for (m = c; m; m = next) {
+		bool goes = true;
+
+		next = next_candidate(m, victim);
+		list_for_each_entry(n, &m->mnt_mounts, mnt_child) {
+			if (n != next && n != m->overmount && n != victim) {
+				goes = false;
+				break;
+			}
+		}
+		if (goes && next && next != m->overmount && above && m != deepest)
+			goes = false;
+		if (m == deepest)
+			above = false;
+		if (goes && do_refcount_check(m, 1))
+			return true;
+	}
+	return false;
+}
+
 /*
  * check if the mount 'mnt' can be unmounted successfully.
  * @mnt: the mount to be checked for unmount
  * NOTE: unmounting 'mnt' would naturally propagate to all
  * other mounts its parent propagates to.
- * Check if any of these mounts that **do not have submounts**
- * have more references than 'refcnt'. If so return busy.
+ * Check if any of the mounts that propagate_umount() would pull out
+ * along with it have more references than their own. If so return busy.
  *
  * vfsmount lock must be held for write
  */
 int propagate_mount_busy(struct mount *mnt, int refcnt)
 {
 	struct mount *parent = mnt->mnt_parent;
+	struct dentry *mp = mnt->mnt_mountpoint;
+	struct mount *m;
 
 	/*
 	 * quickly check if the current mount can be unmounted.
@@ -435,24 +515,16 @@ int propagate_mount_busy(struct mount *mnt, int refcnt)
 	if (mnt == parent)
 		return 0;
 
-	for (struct mount *m = propagation_next(parent, parent); m;
-	     		m = propagation_next(m, parent)) {
-		struct list_head *head;
-		struct mount *child = __lookup_mnt(&m->mnt, mnt->mnt_mountpoint);
+	/* the candidates are the mounts at @mp below the receivers */
+	for (m = propagation_next(parent, parent); m;
+	     m = propagation_next(m, parent)) {
+		struct mount *c = __lookup_mnt(&m->mnt, mp);
 
-		if (!child)
+		/* each chain once, from its top: skip receivers that are candidates */
+		if (!c || (mnt_has_parent(m) && m->mnt_mountpoint == mp &&
+			   receives_from(m->mnt_parent, parent)))
 			continue;
-
-		head = &child->mnt_mounts;
-		if (!list_empty(head)) {
-			/*
-			 * a mount that covers child completely wouldn't prevent
-			 * it being pulled out; any other would.
-			 */
-			if (!list_is_singular(head) || !child->overmount)
-				continue;
-		}
-		if (do_refcount_check(child, 1))
+		if (chain_busy(c, mnt))
 			return 1;
 	}
 	return 0;
diff --git a/fs/posix_acl.c b/fs/posix_acl.c
index 18b302f941743b..fe77934ea8f209 100644
--- a/fs/posix_acl.c
+++ b/fs/posix_acl.c
@@ -118,7 +118,7 @@ void forget_all_cached_acls(struct inode *inode)
 }
 EXPORT_SYMBOL(forget_all_cached_acls);
 
-static struct posix_acl *__get_acl(struct mnt_idmap *idmap,
+static struct posix_acl *__get_acl(const struct mnt_idmap *idmap,
 				   struct dentry *dentry, struct inode *inode,
 				   int type)
 {
@@ -378,7 +378,7 @@ EXPORT_SYMBOL(posix_acl_from_mode);
  * by the acl. Returns -E... otherwise.
  */
 int
-posix_acl_permission(struct mnt_idmap *idmap, struct inode *inode,
+posix_acl_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		     const struct posix_acl *acl, int want)
 {
 	const struct posix_acl_entry *pa, *pe, *mask_obj;
@@ -608,7 +608,7 @@ EXPORT_SYMBOL(__posix_acl_chmod);
  * performed on the raw inode simply pass @nop_mnt_idmap.
  */
 int
- posix_acl_chmod(struct mnt_idmap *idmap, struct dentry *dentry,
+ posix_acl_chmod(const struct mnt_idmap *idmap, struct dentry *dentry,
 		    umode_t mode)
 {
 	struct inode *inode = d_inode(dentry);
@@ -709,7 +709,7 @@ EXPORT_SYMBOL_GPL(posix_acl_create);
  *
  * Called from set_acl inode operations.
  */
-int posix_acl_update_mode(struct mnt_idmap *idmap,
+int posix_acl_update_mode(const struct mnt_idmap *idmap,
 			  struct inode *inode, umode_t *mode_p,
 			  struct posix_acl **acl)
 {
@@ -889,7 +889,7 @@ EXPORT_SYMBOL (posix_acl_to_xattr);
  * Return: On success, the size of the stored uapi posix acls, on error a
  * negative errno.
  */
-static ssize_t vfs_posix_acl_to_xattr(struct mnt_idmap *idmap,
+static ssize_t vfs_posix_acl_to_xattr(const struct mnt_idmap *idmap,
 				      struct inode *inode,
 				      const struct posix_acl *acl, void *buffer,
 				      size_t size)
@@ -937,7 +937,7 @@ static ssize_t vfs_posix_acl_to_xattr(struct mnt_idmap *idmap,
 }
 
 int
-set_posix_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+set_posix_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 	      int type, struct posix_acl *acl)
 {
 	struct inode *inode = d_inode(dentry);
@@ -1018,7 +1018,7 @@ const struct xattr_handler nop_posix_acl_default = {
 };
 EXPORT_SYMBOL_GPL(nop_posix_acl_default);
 
-int simple_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int simple_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   struct posix_acl *acl, int type)
 {
 	int error;
@@ -1057,7 +1057,7 @@ int simple_acl_create(struct inode *dir, struct inode *inode)
 	return 0;
 }
 
-static int vfs_set_acl_idmapped_mnt(struct mnt_idmap *idmap,
+static int vfs_set_acl_idmapped_mnt(const struct mnt_idmap *idmap,
 				    struct user_namespace *fs_userns,
 				    struct posix_acl *acl)
 {
@@ -1091,7 +1091,7 @@ static int vfs_set_acl_idmapped_mnt(struct mnt_idmap *idmap,
  *
  * Return: On success 0, on error negative errno.
  */
-int vfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int vfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		const char *acl_name, struct posix_acl *kacl)
 {
 	int acl_type;
@@ -1168,7 +1168,7 @@ EXPORT_SYMBOL_GPL(vfs_set_acl);
  *
  * Return: On success POSIX ACLs in VFS format, on error negative errno.
  */
-struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *vfs_get_acl(const struct mnt_idmap *idmap,
 			      struct dentry *dentry, const char *acl_name)
 {
 	struct inode *inode = d_inode(dentry);
@@ -1212,7 +1212,7 @@ EXPORT_SYMBOL_GPL(vfs_get_acl);
  *
  * Return: On success 0, on error negative errno.
  */
-int vfs_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int vfs_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   const char *acl_name)
 {
 	int acl_type;
@@ -1265,7 +1265,7 @@ int vfs_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 }
 EXPORT_SYMBOL_GPL(vfs_remove_acl);
 
-int do_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int do_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 	       const char *acl_name, const void *kvalue, size_t size)
 {
 	int error;
@@ -1286,7 +1286,7 @@ int do_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 	return error;
 }
 
-ssize_t do_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+ssize_t do_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   const char *acl_name, void *kvalue, size_t size)
 {
 	ssize_t error;
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 6a39de424f62a1..8c04fcd436d2d4 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -594,7 +594,8 @@ static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns,
 	 * exporting for a long time so userspace might depend on it.
 	 */
 	if (badness != LONG_MIN)
-		points = (1000 + badness * 1000 / (long)totalpages) * 2 / 3;
+		points = (OOM_SCORE_ADJ_MAX +
+			  badness * OOM_SCORE_ADJ_MAX / (long)totalpages) * 2 / 3;
 
 	seq_printf(m, "%lu\n", points);
 
@@ -702,7 +703,7 @@ static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns,
 /*                       Here the fs part begins                        */
 /************************************************************************/
 
-int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int proc_nochmod_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct iattr *attr)
 {
 	int error;
@@ -743,7 +744,7 @@ static bool has_pid_permissions(struct proc_fs_info *fs_info,
 }
 
 
-static int proc_pid_permission(struct mnt_idmap *idmap,
+static int proc_pid_permission(const struct mnt_idmap *idmap,
 			       struct inode *inode, int mask)
 {
 	struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb);
@@ -848,15 +849,35 @@ static int __mem_open(struct inode *inode, struct file *file, unsigned int mode)
 	return 0;
 }
 
+/* private_data for proc_mem_operations */
+struct mem_private {
+	struct mm_struct *mm;
+	/*
+	 * Was the ptrace access check on open bypassed because the opener used
+	 * the same MM (introspection)?
+	 */
+	bool opened_by_owner;
+};
+
 static int mem_open(struct inode *inode, struct file *file)
 {
+	struct mem_private *priv __free(kfree) = kmalloc_obj(struct mem_private);
+
+	if (!priv)
+		return -ENOMEM;
 	if (WARN_ON_ONCE(!(file->f_op->fop_flags & FOP_UNSIGNED_OFFSET)))
 		return -EINVAL;
-	return __mem_open(inode, file, PTRACE_MODE_ATTACH);
+	priv->mm = proc_mem_open(inode, PTRACE_MODE_ATTACH);
+	if (IS_ERR_OR_NULL(priv->mm))
+		return priv->mm ? PTR_ERR(priv->mm) : -ESRCH;
+	priv->opened_by_owner = priv->mm == current->mm;
+	file->private_data = no_free_ptr(priv);
+	return 0;
 }
 
 static bool proc_mem_foll_force(struct file *file, struct mm_struct *mm)
 {
+	struct mem_private *priv = file->private_data;
 	struct task_struct *task;
 	bool ptrace_active = false;
 
@@ -871,16 +892,20 @@ static bool proc_mem_foll_force(struct file *file, struct mm_struct *mm)
 					READ_ONCE(task->parent) == current;
 			put_task_struct(task);
 		}
-		return ptrace_active;
+		if (!ptrace_active)
+			return false;
+		break;
 	default:
-		return true;
+		break;
 	}
+	return security_mem_foll_force(file->f_cred, priv->opened_by_owner) == 0;
 }
 
 static ssize_t mem_rw(struct file *file, char __user *buf,
 			size_t count, loff_t *ppos, int write)
 {
-	struct mm_struct *mm = file->private_data;
+	struct mem_private *priv = file->private_data;
+	struct mm_struct *mm = priv->mm;
 	unsigned long addr = *ppos;
 	ssize_t copied;
 	char *page;
@@ -970,12 +995,21 @@ static int mem_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
+static int mem_release_with_private(struct inode *inode, struct file *file)
+{
+	struct mem_private *priv = file->private_data;
+
+	mmdrop(priv->mm);
+	kfree(priv);
+	return 0;
+}
+
 static const struct file_operations proc_mem_operations = {
 	.llseek		= mem_lseek,
 	.read		= mem_read,
 	.write		= mem_write,
 	.open		= mem_open,
-	.release	= mem_release,
+	.release	= mem_release_with_private,
 	.fop_flags	= FOP_UNSIGNED_OFFSET,
 };
 
@@ -1994,7 +2028,7 @@ static struct inode *proc_pid_make_base_inode(struct super_block *sb,
 	return inode;
 }
 
-int pid_getattr(struct mnt_idmap *idmap, const struct path *path,
+int pid_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -2519,17 +2553,23 @@ static int show_timer(struct seq_file *m, void *v)
 	struct k_itimer *timer = hlist_entry((struct hlist_node *)v, struct k_itimer, list);
 	struct timers_private *tp = m->private;
 	int notify = timer->it_sigev_notify;
+	pid_t nr = 0;
 
 	guard(spinlock_irq)(&timer->it_lock);
 	if (!posixtimer_valid(timer))
 		return 0;
 
+	if (timer->it_pid && pid_has_task(timer->it_pid, timer->it_pid_type))
+		nr = pid_nr_ns(timer->it_pid, tp->ns);
+	else
+		notify = SIGEV_NONE;
+
 	seq_printf(m, "ID: %d\n", timer->it_id);
 	seq_printf(m, "signal: %d/%px\n", timer->sigq.info.si_signo,
 		   timer->sigq.info.si_value.sival_ptr);
 	seq_printf(m, "notify: %s/%s.%d\n", nstr[notify & ~SIGEV_THREAD_ID],
 		   (notify & SIGEV_THREAD_ID) ? "tid" : "pid",
-		   pid_nr_ns(timer->it_pid, tp->ns));
+		   nr);
 	seq_printf(m, "ClockID: %d\n", timer->it_clock);
 
 	return 0;
@@ -3607,7 +3647,7 @@ int proc_pid_readdir(struct file *file, struct dir_context *ctx)
  * This function makes sure that the node is always accessible for members of
  * same thread group.
  */
-static int proc_tid_comm_permission(struct mnt_idmap *idmap,
+static int proc_tid_comm_permission(const struct mnt_idmap *idmap,
 				    struct inode *inode, int mask)
 {
 	bool is_same_tgroup;
@@ -3936,7 +3976,7 @@ static int proc_task_readdir(struct file *file, struct dir_context *ctx)
 	return 0;
 }
 
-static int proc_task_getattr(struct mnt_idmap *idmap,
+static int proc_task_getattr(const struct mnt_idmap *idmap,
 			     const struct path *path, struct kstat *stat,
 			     u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/proc/fd.c b/fs/proc/fd.c
index 0f9a1556f2a347..7214a749538015 100644
--- a/fs/proc/fd.c
+++ b/fs/proc/fd.c
@@ -82,7 +82,7 @@ static int seq_fdinfo_open(struct inode *inode, struct file *file)
  * that the current task has PTRACE_MODE_READ in addition to the normal
  * POSIX-like checks.
  */
-static int proc_fdinfo_permission(struct mnt_idmap *idmap, struct inode *inode,
+static int proc_fdinfo_permission(const struct mnt_idmap *idmap, struct inode *inode,
 				  int mask)
 {
 	bool allowed = false;
@@ -323,7 +323,7 @@ static struct dentry *proc_lookupfd(struct inode *dir, struct dentry *dentry,
  * /proc/pid/fd needs a special permission handler so that a process can still
  * access /proc/self/fd after it has executed a setuid().
  */
-int proc_fd_permission(struct mnt_idmap *idmap,
+int proc_fd_permission(const struct mnt_idmap *idmap,
 		       struct inode *inode, int mask)
 {
 	struct task_struct *p;
@@ -342,7 +342,7 @@ int proc_fd_permission(struct mnt_idmap *idmap,
 	return rv;
 }
 
-static int proc_fd_getattr(struct mnt_idmap *idmap,
+static int proc_fd_getattr(const struct mnt_idmap *idmap,
 			const struct path *path, struct kstat *stat,
 			u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/proc/fd.h b/fs/proc/fd.h
index 7e7265f7e06f8b..77f2e4f38592ba 100644
--- a/fs/proc/fd.h
+++ b/fs/proc/fd.h
@@ -10,7 +10,7 @@ extern const struct inode_operations proc_fd_inode_operations;
 extern const struct file_operations proc_fdinfo_operations;
 extern const struct inode_operations proc_fdinfo_inode_operations;
 
-extern int proc_fd_permission(struct mnt_idmap *idmap,
+extern int proc_fd_permission(const struct mnt_idmap *idmap,
 			      struct inode *inode, int mask);
 
 static inline unsigned int proc_fd(struct inode *inode)
diff --git a/fs/proc/generic.c b/fs/proc/generic.c
index 26086a283672bf..2b1971da4a85a8 100644
--- a/fs/proc/generic.c
+++ b/fs/proc/generic.c
@@ -117,7 +117,7 @@ static bool pde_subdir_insert(struct proc_dir_entry *dir,
 	return true;
 }
 
-static int proc_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int proc_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -135,7 +135,7 @@ static int proc_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	return 0;
 }
 
-static int proc_getattr(struct mnt_idmap *idmap,
+static int proc_getattr(const struct mnt_idmap *idmap,
 			const struct path *path, struct kstat *stat,
 			u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index 04bd6c9e65a722..df873df9afc560 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -258,9 +258,9 @@ extern int proc_pid_statm(struct seq_file *, struct pid_namespace *,
  * base.c
  */
 extern const struct dentry_operations pid_dentry_operations;
-extern int pid_getattr(struct mnt_idmap *, const struct path *,
+extern int pid_getattr(const struct mnt_idmap *, const struct path *,
 		       struct kstat *, u32, unsigned int);
-int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int proc_nochmod_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			struct iattr *attr);
 extern void proc_pid_evict_inode(struct proc_inode *);
 extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *, umode_t);
@@ -385,10 +385,8 @@ struct mem_size_stats;
 
 struct proc_maps_locking_ctx {
 	struct mm_struct *mm;
-#ifdef CONFIG_PER_VMA_LOCK
 	bool mmap_locked;
 	struct vm_area_struct *locked_vma;
-#endif
 };
 
 struct proc_maps_private {
diff --git a/fs/proc/page.c b/fs/proc/page.c
index 260772b20bd992..6aa99e42ec2221 100644
--- a/fs/proc/page.c
+++ b/fs/proc/page.c
@@ -31,10 +31,9 @@ static inline unsigned long get_max_dump_pfn(void)
 {
 #ifdef CONFIG_SPARSEMEM
 	/*
-	 * The memmap of early sections is completely populated and marked
-	 * online even if max_pfn does not fall on a section boundary -
-	 * pfn_to_online_page() will succeed on all pages. Allow inspecting
-	 * these memmaps.
+	 * If max_pfn does not fall on a section boundary, pfn_to_online_page()
+	 * can succeed on PFNs beyond max_pfn within the same section. Allow
+	 * inspection of these memmaps.
 	 */
 	return round_up(max_pfn, PAGES_PER_SECTION);
 #else
@@ -232,7 +231,6 @@ u64 stable_page_flags(const struct page *page)
 
 	u |= kpf_copy_bit(k, KPF_RESERVED,	PG_reserved);
 	u |= kpf_copy_bit(k, KPF_OWNER_2,	PG_owner_2);
-	u |= kpf_copy_bit(k, KPF_PRIVATE,	PG_private);
 	u |= kpf_copy_bit(k, KPF_PRIVATE_2,	PG_private_2);
 	u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE,	PG_owner_priv_1);
 	u |= kpf_copy_bit(k, KPF_ARCH,		PG_arch_1);
diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c
index 00cc385bce2121..b1f5eafb069a3c 100644
--- a/fs/proc/proc_net.c
+++ b/fs/proc/proc_net.c
@@ -308,7 +308,7 @@ static struct dentry *proc_tgid_net_lookup(struct inode *dir,
 	return de;
 }
 
-static int proc_tgid_net_getattr(struct mnt_idmap *idmap,
+static int proc_tgid_net_getattr(const struct mnt_idmap *idmap,
 				 const struct path *path, struct kstat *stat,
 				 u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c
index 04a382178c657b..6c0859c6656356 100644
--- a/fs/proc/proc_sysctl.c
+++ b/fs/proc/proc_sysctl.c
@@ -206,7 +206,7 @@ static void init_header(struct ctl_table_header *head,
 	head->node = node;
 	INIT_HLIST_HEAD(&head->inodes);
 	if (node) {
-		const struct ctl_table *entry;
+		const struct ctl_table *entry __maybe_unused;
 
 		list_for_each_table_entry(entry, head) {
 			node->header = head;
@@ -788,7 +788,7 @@ static int proc_sys_readdir(struct file *file, struct dir_context *ctx)
 	return 0;
 }
 
-static int proc_sys_permission(struct mnt_idmap *idmap,
+static int proc_sys_permission(const struct mnt_idmap *idmap,
 			       struct inode *inode, int mask)
 {
 	/*
@@ -817,7 +817,7 @@ static int proc_sys_permission(struct mnt_idmap *idmap,
 	return error;
 }
 
-static int proc_sys_setattr(struct mnt_idmap *idmap,
+static int proc_sys_setattr(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -834,7 +834,7 @@ static int proc_sys_setattr(struct mnt_idmap *idmap,
 	return 0;
 }
 
-static int proc_sys_getattr(struct mnt_idmap *idmap,
+static int proc_sys_getattr(const struct mnt_idmap *idmap,
 			    const struct path *path, struct kstat *stat,
 			    u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/proc/root.c b/fs/proc/root.c
index 99adddfeb4a44c..7fbbe92bf73a71 100644
--- a/fs/proc/root.c
+++ b/fs/proc/root.c
@@ -402,7 +402,7 @@ void __init proc_root_init(void)
 	register_filesystem(&proc_fs_type);
 }
 
-static int proc_root_getattr(struct mnt_idmap *idmap,
+static int proc_root_getattr(const struct mnt_idmap *idmap,
 			     const struct path *path, struct kstat *stat,
 			     u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 5c54aebe211824..44147ba0b899da 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -130,30 +130,12 @@ static void release_task_mempolicy(struct proc_maps_private *priv)
 }
 #endif
 
-#ifdef CONFIG_PER_VMA_LOCK
-
-static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
-{
-	int ret = mmap_read_lock_killable(lock_ctx->mm);
-
-	if (!ret)
-		lock_ctx->mmap_locked = true;
-
-	return ret;
-}
-
-static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
+static void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
 {
 	mmap_read_unlock(lock_ctx->mm);
 	lock_ctx->mmap_locked = false;
 }
 
-static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx)
-{
-	lock_ctx->locked_vma = NULL;
-	lock_ctx->mmap_locked = false;
-}
-
 static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx)
 {
 	if (lock_ctx->locked_vma) {
@@ -162,23 +144,10 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx)
 	}
 }
 
-static inline bool lock_vma_range(struct seq_file *m,
-				  struct proc_maps_locking_ctx *lock_ctx)
-{
-	rcu_read_lock();
-	reset_lock_ctx(lock_ctx);
-
-	return true;
-}
-
-static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx)
+static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx)
 {
-	if (lock_ctx->mmap_locked) {
-		unlock_ctx_mm(lock_ctx);
-	} else {
-		unlock_ctx_vma(lock_ctx);
-		rcu_read_unlock();
-	}
+	lock_ctx->locked_vma = NULL;
+	lock_ctx->mmap_locked = false;
 }
 
 static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,
@@ -198,8 +167,8 @@ static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,
 	return vma;
 }
 
-static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,
-					 loff_t pos)
+static bool fallback_to_mmap_lock(struct proc_maps_private *priv,
+		loff_t pos)
 {
 	struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx;
 
@@ -215,7 +184,8 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,
 	return true;
 }
 
-static inline void drop_rcu(struct proc_maps_private *priv)
+#if defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA)
+static void drop_rcu(struct proc_maps_private *priv)
 {
 	if (priv->lock_ctx.mmap_locked)
 		return;
@@ -223,7 +193,7 @@ static inline void drop_rcu(struct proc_maps_private *priv)
 	rcu_read_unlock();
 }
 
-static inline void reacquire_rcu(struct proc_maps_private *priv)
+static void reacquire_rcu(struct proc_maps_private *priv)
 {
 	if (priv->lock_ctx.mmap_locked)
 		return;
@@ -232,46 +202,7 @@ static inline void reacquire_rcu(struct proc_maps_private *priv)
 	/* Reinitialize the iterator. */
 	vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end);
 }
-
-#else /* CONFIG_PER_VMA_LOCK */
-
-static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
-{
-	return mmap_read_lock_killable(lock_ctx->mm);
-}
-
-static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
-{
-	mmap_read_unlock(lock_ctx->mm);
-}
-
-static inline bool lock_vma_range(struct seq_file *m,
-				  struct proc_maps_locking_ctx *lock_ctx)
-{
-	return lock_ctx_mm(lock_ctx) == 0;
-}
-
-static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx)
-{
-	unlock_ctx_mm(lock_ctx);
-}
-
-static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,
-					   loff_t last_pos)
-{
-	return vma_next(&priv->iter);
-}
-
-static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,
-					 loff_t pos)
-{
-	return false;
-}
-
-static inline void drop_rcu(struct proc_maps_private *priv) {}
-static inline void reacquire_rcu(struct proc_maps_private *priv) {}
-
-#endif /* CONFIG_PER_VMA_LOCK */
+#endif
 
 static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos)
 {
@@ -297,9 +228,6 @@ static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos)
 		 * found the extended vma with the same vm_start.
 		 */
 		*ppos = vma->vm_end;
-	} else {
-		*ppos = SENTINEL_VMA_GATE;
-		vma = get_gate_vma(priv->lock_ctx.mm);
 	}
 
 	return vma;
@@ -309,6 +237,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos)
 {
 	struct proc_maps_private *priv = m->private;
 	struct proc_maps_locking_ctx *lock_ctx;
+	struct vm_area_struct *vma;
 	loff_t last_addr = *ppos;
 	struct mm_struct *mm;
 
@@ -328,13 +257,8 @@ static void *m_start(struct seq_file *m, loff_t *ppos)
 		return NULL;
 	}
 
-	if (!lock_vma_range(m, lock_ctx)) {
-		mmput(mm);
-		put_task_struct(priv->task);
-		priv->task = NULL;
-		return ERR_PTR(-EINTR);
-	}
-
+	rcu_read_lock();
+	reset_lock_ctx(lock_ctx);
 	/*
 	 * Reset current position if last_addr was set before
 	 * and it's not a sentinel.
@@ -343,19 +267,39 @@ static void *m_start(struct seq_file *m, loff_t *ppos)
 		*ppos = last_addr = priv->last_pos;
 	vma_iter_init(&priv->iter, mm, (unsigned long)last_addr);
 	hold_task_mempolicy(priv);
+	/*
+	 * If seq_file had to flush its collected data right after m_next() set
+	 * position to SENTINEL_VMA_GATE, m_start() will get that sentinel and
+	 * should return gate_vma without calling proc_get_vma().
+	 */
 	if (last_addr == SENTINEL_VMA_GATE)
 		return get_gate_vma(mm);
 
-	return proc_get_vma(m, ppos);
+	vma = proc_get_vma(m, ppos);
+	if (vma)
+		return vma;
+
+	/* Return gate VMA at the end */
+	*ppos = SENTINEL_VMA_GATE;
+	return get_gate_vma(mm);
 }
 
 static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
 {
+	struct proc_maps_private *priv = m->private;
+	struct vm_area_struct *vma;
+
 	if (*ppos == SENTINEL_VMA_GATE) {
 		*ppos = SENTINEL_VMA_END;
 		return NULL;
 	}
-	return proc_get_vma(m, ppos);
+	vma = proc_get_vma(m, ppos);
+	if (vma)
+		return vma;
+
+	/* Return gate VMA at the end */
+	*ppos = SENTINEL_VMA_GATE;
+	return get_gate_vma(priv->lock_ctx.mm);
 }
 
 static void m_stop(struct seq_file *m, void *v)
@@ -367,7 +311,12 @@ static void m_stop(struct seq_file *m, void *v)
 		return;
 
 	release_task_mempolicy(priv);
-	unlock_vma_range(&priv->lock_ctx);
+	if (priv->lock_ctx.mmap_locked) {
+		unlock_ctx_mm(&priv->lock_ctx);
+	} else {
+		unlock_ctx_vma(&priv->lock_ctx);
+		rcu_read_unlock();
+	}
 	mmput(mm);
 	put_task_struct(priv->task);
 	priv->task = NULL;
@@ -560,23 +509,6 @@ static int pid_maps_open(struct inode *inode, struct file *file)
 		PROCMAP_QUERY_VMA_FLAGS				\
 )
 
-#ifdef CONFIG_PER_VMA_LOCK
-
-static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)
-{
-	reset_lock_ctx(lock_ctx);
-
-	return 0;
-}
-
-static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx)
-{
-	if (lock_ctx->mmap_locked)
-		unlock_ctx_mm(lock_ctx);
-	else
-		unlock_ctx_vma(lock_ctx);
-}
-
 static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,
 						     unsigned long addr)
 {
@@ -612,26 +544,6 @@ static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ct
 	return vma;
 }
 
-#else /* CONFIG_PER_VMA_LOCK */
-
-static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)
-{
-	return mmap_read_lock_killable(lock_ctx->mm);
-}
-
-static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx)
-{
-	mmap_read_unlock(lock_ctx->mm);
-}
-
-static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,
-						     unsigned long addr)
-{
-	return find_vma(lock_ctx->mm, addr);
-}
-
-#endif  /* CONFIG_PER_VMA_LOCK */
-
 static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx,
 						 unsigned long addr, u32 flags)
 {
@@ -717,12 +629,7 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg)
 	if (!mm || !mmget_not_zero(mm))
 		return -ESRCH;
 
-	err = query_vma_setup(&lock_ctx);
-	if (err) {
-		mmput(mm);
-		return err;
-	}
-
+	reset_lock_ctx(&lock_ctx);
 	vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags);
 	if (IS_ERR(vma)) {
 		err = PTR_ERR(vma);
@@ -796,7 +703,10 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg)
 		vm_file = get_file(vma->vm_file);
 
 	/* unlock vma or mmap_lock, and put mm_struct before copying data to user */
-	query_vma_teardown(&lock_ctx);
+	if (lock_ctx.mmap_locked)
+		unlock_ctx_mm(&lock_ctx);
+	else
+		unlock_ctx_vma(&lock_ctx);
 	mmput(mm);
 
 	if (karg.build_id_size) {
@@ -837,7 +747,10 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg)
 	return 0;
 
 out:
-	query_vma_teardown(&lock_ctx);
+	if (lock_ctx.mmap_locked)
+		unlock_ctx_mm(&lock_ctx);
+	else
+		unlock_ctx_vma(&lock_ctx);
 	mmput(mm);
 out_file:
 	if (vm_file)
@@ -1314,8 +1227,6 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = {
 	.walk_lock		= PGWALK_RDLOCK,
 };
 
-#ifdef CONFIG_PER_VMA_LOCK
-
 static const struct mm_walk_ops smaps_walk_vma_lock_ops = {
 	.pmd_entry		= smaps_pte_range,
 	.hugetlb_entry		= smaps_hugetlb_range,
@@ -1329,7 +1240,7 @@ static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = {
 	.walk_lock		= PGWALK_VMA_RDLOCK_VERIFY,
 };
 
-static inline const struct mm_walk_ops *
+static const struct mm_walk_ops *
 get_smaps_walk_ops(struct proc_maps_private *priv)
 {
 	if (priv->lock_ctx.mmap_locked)
@@ -1337,7 +1248,7 @@ get_smaps_walk_ops(struct proc_maps_private *priv)
 	return &smaps_walk_vma_lock_ops;
 }
 
-static inline const struct mm_walk_ops *
+static const struct mm_walk_ops *
 get_smaps_shmem_walk_ops(struct proc_maps_private *priv)
 {
 	if (priv->lock_ctx.mmap_locked)
@@ -1345,36 +1256,26 @@ get_smaps_shmem_walk_ops(struct proc_maps_private *priv)
 	return &smaps_shmem_walk_vma_lock_ops;
 }
 
-#else /* CONFIG_PER_VMA_LOCK */
-
-static inline const struct mm_walk_ops *
-get_smaps_walk_ops(struct proc_maps_private *priv)
-{
-	return &smaps_walk_ops;
-}
-
-static inline const struct mm_walk_ops *
-get_smaps_shmem_walk_ops(struct proc_maps_private *priv)
-{
-	return &smaps_shmem_walk_ops;
-}
-
-#endif /* CONFIG_PER_VMA_LOCK */
-
-/*
- * Gather mem stats from @vma with the indicated beginning
- * address @start, and keep them in @mss.
+/**
+ * smap_gather_stats_range() - Gather mem stats from a portion of the @vma.
+ * @priv: proc maps private state.
+ * @vma: The VMA to gather stats for.
+ * @mss: The accumulated stats.
+ * @start: The address from which to start.
  *
- * Use vm_start of @vma as the beginning address if @start is 0.
+ * This gathers stats for the portion of the VMA starting at the @start
+ * address.
  */
-static void smap_gather_stats(struct proc_maps_private *priv,
-			      struct vm_area_struct *vma,
-			      struct mem_size_stats *mss, unsigned long start)
+static void smap_gather_stats_range(struct proc_maps_private *priv,
+		struct vm_area_struct *vma,
+		struct mem_size_stats *mss,
+		unsigned long start)
 {
 	const struct mm_walk_ops *ops = get_smaps_walk_ops(priv);
+	const bool is_partial = start > vma->vm_start;
 
 	/* Invalid start */
-	if (start >= vma->vm_end)
+	if (start < vma->vm_start || start >= vma->vm_end)
 		return;
 
 	if (vma == get_gate_vma(priv->lock_ctx.mm))
@@ -1385,33 +1286,39 @@ static void smap_gather_stats(struct proc_maps_private *priv,
 
 	if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
 		/*
-		 * For shared or readonly shmem mappings we know that all
-		 * swapped out pages belong to the shmem object, and we can
-		 * obtain the swap value much more efficiently. For private
-		 * writable mappings, we might have COW pages that are
-		 * not affected by the parent swapped out pages of the shmem
-		 * object, so we have to distinguish them during the page walk.
-		 * Unless we know that the shmem object (or the part mapped by
-		 * our VMA) has no swapped out pages at all.
+		 * CoW mappings might map anon folios that do not belong to
+		 * shmem. Perform a less efficient page table walk in this
+		 * situation, unless we know that the shmem object (or the
+		 * part mapped by our VMA) has no swapped out pages at all.
 		 */
-		unsigned long shmem_swapped = shmem_swap_usage(vma);
+		const unsigned long shmem_swapped = shmem_swap_usage(vma);
+		const bool is_cow = vma_is_cow_mapping(vma);
 
-		if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
-					!(vma->vm_flags & VM_WRITE))) {
-			mss->swap += shmem_swapped;
-		} else {
+		if (is_partial || (shmem_swapped && is_cow))
 			ops = get_smaps_shmem_walk_ops(priv);
-		}
+		else
+			mss->swap += shmem_swapped;
 	}
 
-	if (!start)
-		walk_page_vma(vma, ops, mss);
-	else
-		walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
+	walk_page_range_vma(vma, start, vma->vm_end, ops, mss);
 
 	reacquire_rcu(priv);
 }
 
+/**
+ * smap_gather_stats() - Gather mem stats from the entire @vma.
+ * @priv: proc maps private state.
+ * @vma: The VMA to gather stats for.
+ * @mss: The accumulated stats.
+ *
+ * This gathers stats for the whole of the VMA.
+ */
+static void smap_gather_stats(struct proc_maps_private *priv,
+		struct vm_area_struct *vma, struct mem_size_stats *mss)
+{
+	smap_gather_stats_range(priv, vma, mss, vma->vm_start);
+}
+
 #define SEQ_PUT_DEC(str, val) \
 		seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
 
@@ -1462,7 +1369,7 @@ static int show_smap(struct seq_file *m, void *v)
 	struct vm_area_struct *vma = v;
 	struct mem_size_stats mss = {};
 
-	smap_gather_stats(priv, vma, &mss, 0);
+	smap_gather_stats(priv, vma, &mss);
 
 	show_map_vma(m, vma);
 
@@ -1487,12 +1394,14 @@ static int show_smap(struct seq_file *m, void *v)
 static int show_smaps_rollup(struct seq_file *m, void *v)
 {
 	struct proc_maps_private *priv = m->private;
+	struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx;
+	struct mm_struct *mm = lock_ctx->mm;
 	struct mem_size_stats mss = {};
-	struct mm_struct *mm = priv->lock_ctx.mm;
+	unsigned long last_vma_end = 0;
+	unsigned long vma_start = 0;
 	struct vm_area_struct *vma;
-	unsigned long vma_start = 0, last_vma_end = 0;
+	loff_t pos = 0;
 	int ret = 0;
-	VMA_ITERATOR(vmi, mm, 0);
 
 	priv->task = get_proc_task(priv->inode);
 	if (!priv->task)
@@ -1503,89 +1412,63 @@ static int show_smaps_rollup(struct seq_file *m, void *v)
 		goto out_put_task;
 	}
 
-	ret = lock_ctx_mm(&priv->lock_ctx);
-	if (ret)
-		goto out_put_mm;
-
 	hold_task_mempolicy(priv);
-	vma = vma_next(&vmi);
+	rcu_read_lock();
+	reset_lock_ctx(lock_ctx);
 
+	vma_iter_init(&priv->iter, mm, 0);
+	vma = proc_get_vma(m, &pos);
 	if (unlikely(!vma))
 		goto empty_set;
 
-	vma_start = vma->vm_start;
-	do {
-		smap_gather_stats(priv, vma, &mss, 0);
+	if (!IS_ERR(vma))
+		vma_start = vma->vm_start;
+
+	while (vma) {
+		if (IS_ERR(vma)) {
+			ret = PTR_ERR(vma);
+			goto out_unlock;
+		}
+
+		if (vma->vm_start < last_vma_end) {
+			/*
+			 * After retaking the lock, already reported VMA grew
+			 * or got merged with the next one and we found it
+			 * again. Gather stats for the remaining portion by
+			 * starting at last_vma_end.
+			 */
+			smap_gather_stats_range(priv, vma, &mss, last_vma_end);
+		} else {
+			/* Found next unreported VMA, start from its beginning */
+			smap_gather_stats(priv, vma, &mss);
+		}
 		last_vma_end = vma->vm_end;
 
 		/*
-		 * Release mmap_lock temporarily if someone wants to
-		 * access it for write request.
+		 * If the VMA lock is not taken, we hold the often contended
+		 * mmap lock. This can happen if we had to fall back to the
+		 * mmap lock.
+		 *
+		 * To relieve pressure, check if it is indeed contended, then
+		 * temporarily release it.
 		 */
-		if (mmap_lock_is_contended(mm)) {
-			vma_iter_invalidate(&vmi);
-			unlock_ctx_mm(&priv->lock_ctx);
-			ret = lock_ctx_mm(&priv->lock_ctx);
-			if (ret) {
-				release_task_mempolicy(priv);
-				goto out_put_mm;
-			}
-
+		if (lock_ctx->mmap_locked &&
+		    mmap_lock_is_contended(lock_ctx->mm)) {
+			unlock_ctx_mm(lock_ctx);
 			/*
-			 * After dropping the lock, there are four cases to
-			 * consider. See the following example for explanation.
-			 *
-			 *   +------+------+-----------+
-			 *   | VMA1 | VMA2 | VMA3      |
-			 *   +------+------+-----------+
-			 *   |      |      |           |
-			 *  4k     8k     16k         400k
-			 *
-			 * Suppose we drop the lock after reading VMA2 due to
-			 * contention, then we get:
-			 *
-			 *	last_vma_end = 16k
-			 *
-			 * 1) VMA2 is freed, but VMA3 exists:
-			 *
-			 *    vma_next(vmi) will return VMA3.
-			 *    In this case, just continue from VMA3.
-			 *
-			 * 2) VMA2 still exists:
-			 *
-			 *    vma_next(vmi) will return VMA3.
-			 *    In this case, just continue from VMA3.
-			 *
-			 * 3) No more VMAs can be found:
-			 *
-			 *    vma_next(vmi) will return NULL.
-			 *    No more things to do, just break.
-			 *
-			 * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
-			 *
-			 *    vma_next(vmi) will return VMA' whose range
-			 *    contains last_vma_end.
-			 *    Iterate VMA' from last_vma_end.
+			 * Even though we previously fell back to mmap lock,
+			 * we try taking VMA lock for the next VMA, since it
+			 * might not be under modification. In the worst case
+			 * we will fall back to mmap lock again.
 			 */
-			vma = vma_next(&vmi);
-			/* Case 3 above */
-			if (!vma)
-				break;
-
-			/* Case 1 and 2 above */
-			if (vma->vm_start >= last_vma_end) {
-				smap_gather_stats(priv, vma, &mss, 0);
-				last_vma_end = vma->vm_end;
-				continue;
-			}
-
-			/* Case 4 above */
-			if (vma->vm_end > last_vma_end) {
-				smap_gather_stats(priv, vma, &mss, last_vma_end);
-				last_vma_end = vma->vm_end;
-			}
+			rcu_read_lock();
+			reset_lock_ctx(lock_ctx);
+			/* Resume from the last position. */
+			pos = last_vma_end;
+			vma_iter_init(&priv->iter, mm, pos);
 		}
-	} for_each_vma(vmi, vma);
+		vma = proc_get_vma(m, &pos);
+	}
 
 empty_set:
 	show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0);
@@ -1594,10 +1477,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v)
 
 	__show_smap(m, &mss, true);
 
+out_unlock:
+	if (lock_ctx->mmap_locked) {
+		unlock_ctx_mm(lock_ctx);
+	} else {
+		unlock_ctx_vma(lock_ctx);
+		rcu_read_unlock();
+	}
 	release_task_mempolicy(priv);
-	unlock_ctx_mm(&priv->lock_ctx);
-
-out_put_mm:
 	mmput(mm);
 out_put_task:
 	put_task_struct(priv->task);
@@ -1687,7 +1574,7 @@ struct clear_refs_private {
 	enum clear_refs_types type;
 };
 
-static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
+static bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
 {
 	struct folio *folio;
 
@@ -1703,8 +1590,8 @@ static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr,
 	return folio_maybe_dma_pinned(folio);
 }
 
-static inline void clear_soft_dirty(struct vm_area_struct *vma,
-		unsigned long addr, pte_t *pte)
+static void clear_soft_dirty(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *pte)
 {
 	if (!pgtable_supports_soft_dirty())
 		return;
@@ -1735,7 +1622,7 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma,
 }
 
 #if defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
+static void clear_soft_dirty_pmd(struct vm_area_struct *vma,
 		unsigned long addr, pmd_t *pmdp)
 {
 	pmd_t old, pmd = *pmdp;
@@ -1761,7 +1648,7 @@ static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
 	}
 }
 #else
-static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
+static void clear_soft_dirty_pmd(struct vm_area_struct *vma,
 		unsigned long addr, pmd_t *pmdp)
 {
 }
@@ -1786,7 +1673,9 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
 		if (!pmd_present(*pmd))
 			goto out;
 
-		folio = pmd_folio(*pmd);
+		folio = vm_normal_folio_pmd(vma, addr, *pmd);
+		if (!folio)
+			goto out;
 
 		/* Clear accessed and referenced bits. */
 		pmdp_test_and_clear_young(vma, addr, pmd);
@@ -1961,7 +1850,7 @@ struct pagemapread {
 
 #define PM_END_OF_BUFFER    1
 
-static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
+static pagemap_entry_t make_pme(u64 frame, u64 flags)
 {
 	return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
 }
@@ -2106,7 +1995,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr,
 		goto populate_pagemap;
 
 	if (pmd_present(pmd)) {
-		page = pmd_page(pmd);
+		page = vm_normal_page_pmd(vma, addr, pmd);
 
 		flags |= PM_PRESENT;
 		if (pmd_soft_dirty(pmd))
@@ -3097,7 +2986,7 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end,
 	 * hugetlb differs, see pagemap_hugetlb_category().
 	 */
 	categories = p->cur_vma_category;
-	if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma))
+	if (userfaultfd_wp(vma) && !vma_is_hugetlb(vma))
 		categories |= PAGE_IS_WRITTEN;
 
 	if (!pagemap_scan_is_interesting_page(categories, p))
@@ -3110,7 +2999,7 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end,
 	if (~p->arg.flags & PM_SCAN_WP_MATCHING)
 		return ret;
 
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		err = pagemap_scan_hugetlb_hole_wp(vma, addr, end);
 	else
 		err = uffd_wp_range(vma, addr, end - addr, true);
@@ -3497,14 +3386,13 @@ static const struct mm_walk_ops show_numa_ops = {
 	.walk_lock = PGWALK_RDLOCK,
 };
 
-#ifdef CONFIG_PER_VMA_LOCK
 static const struct mm_walk_ops show_numa_vma_lock_ops = {
 	.hugetlb_entry = gather_hugetlb_stats,
 	.pmd_entry = gather_pte_stats,
 	.walk_lock = PGWALK_VMA_RDLOCK_VERIFY,
 };
 
-static inline const struct mm_walk_ops *
+static const struct mm_walk_ops *
 get_show_numa_ops(struct proc_maps_private *priv)
 {
 	if (priv->lock_ctx.mmap_locked)
@@ -3512,16 +3400,6 @@ get_show_numa_ops(struct proc_maps_private *priv)
 	return &show_numa_vma_lock_ops;
 }
 
-#else /* CONFIG_PER_VMA_LOCK */
-
-static inline const struct mm_walk_ops *
-get_show_numa_ops(struct proc_maps_private *priv)
-{
-	return &show_numa_ops;
-}
-
-#endif /* CONFIG_PER_VMA_LOCK */
-
 /*
  * Display pages allocated per node and memory policy via /proc.
  */
@@ -3563,7 +3441,7 @@ static int show_numa_map(struct seq_file *m, void *v)
 		seq_puts(m, " stack");
 	}
 
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		seq_puts(m, " huge");
 
 	/* Skip walking pages if gate VMA */
@@ -3592,7 +3470,7 @@ static int show_numa_map(struct seq_file *m, void *v)
 	if (md->swapcache)
 		seq_printf(m, " swapcache=%lu", md->swapcache);
 
-	if (md->active < md->pages && !is_vm_hugetlb_page(vma))
+	if (md->active < md->pages && !vma_is_hugetlb(vma))
 		seq_printf(m, " active=%lu", md->active);
 
 	if (md->writeback)
diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c
index 433aa947cd574d..53143c66cbe1b1 100644
--- a/fs/proc/uptime.c
+++ b/fs/proc/uptime.c
@@ -15,12 +15,8 @@ static int uptime_proc_show(struct seq_file *m, void *v)
 	struct timespec64 idle;
 	u64 idle_nsec;
 	u32 rem;
-	int i;
-
-	idle_nsec = 0;
-	for_each_possible_cpu(i)
-		idle_nsec += kcpustat_field(CPUTIME_IDLE, i);
 
+	idle_nsec = kcpustat_field_total(CPUTIME_IDLE, cpu_possible_mask);
 	ktime_get_boottime_ts64(&uptime);
 	timens_add_boottime(&uptime);
 
diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c
index 44d15436439fd9..1fe70c4ecc54c8 100644
--- a/fs/proc/vmcore.c
+++ b/fs/proc/vmcore.c
@@ -474,29 +474,30 @@ static vm_fault_t mmap_vmcore_fault(struct vm_fault *vmf)
 	pgoff_t index = vmf->pgoff;
 	struct iov_iter iter;
 	struct kvec kvec;
-	struct page *page;
+	struct folio *folio;
 	loff_t offset;
 	int rc;
 
-	page = find_or_create_page(mapping, index, GFP_KERNEL);
-	if (!page)
+	folio = __filemap_get_folio(mapping, index,
+			FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL);
+	if (IS_ERR(folio))
 		return VM_FAULT_OOM;
-	if (!PageUptodate(page)) {
-		offset = (loff_t) index << PAGE_SHIFT;
-		kvec.iov_base = page_address(page);
-		kvec.iov_len = PAGE_SIZE;
-		iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, PAGE_SIZE);
+	if (!folio_test_uptodate(folio)) {
+		offset = folio_pos(folio);
+		kvec.iov_base = folio_address(folio);
+		kvec.iov_len = folio_size(folio);
+		iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, folio_size(folio));
 
 		rc = __read_vmcore(&iter, &offset);
 		if (rc < 0) {
-			unlock_page(page);
-			put_page(page);
+			folio_unlock(folio);
+			folio_put(folio);
 			return vmf_error(rc);
 		}
-		SetPageUptodate(page);
+		folio_mark_uptodate(folio);
 	}
-	unlock_page(page);
-	vmf->page = page;
+	folio_unlock(folio);
+	vmf->page = folio_file_page(folio, index);
 	return 0;
 #else
 	return VM_FAULT_SIGBUS;
@@ -1709,6 +1710,24 @@ static void vmcore_free_device_dumps(void)
 #endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */
 }
 
+#define VMCOREINFO_OSRELEASE_KEY	"OSRELEASE="
+
+static void __init vmcore_report_crashed_release(void)
+{
+	const char *ver, *eol;
+
+	ver = strnstr(elfnotes_buf, VMCOREINFO_OSRELEASE_KEY, elfnotes_sz);
+	if (!ver)
+		return;
+
+	ver += sizeof(VMCOREINFO_OSRELEASE_KEY) - 1;
+	eol = memchr(ver, '\n', elfnotes_buf + elfnotes_sz - ver);
+	if (!eol)
+		return;
+
+	pr_notice("dump is from kernel %.*s\n", (int)(eol - ver), ver);
+}
+
 /* Init function for vmcore module. */
 static int __init vmcore_init(void)
 {
@@ -1733,6 +1752,8 @@ static int __init vmcore_init(void)
 	elfcorehdr_free(elfcorehdr_addr);
 	elfcorehdr_addr = ELFCORE_ADDR_ERR;
 
+	vmcore_report_crashed_release();
+
 	proc_vmcore = proc_create("vmcore", S_IRUSR, NULL, &vmcore_proc_ops);
 	if (proc_vmcore)
 		proc_vmcore->size = vmcore_size;
diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 1d76c9d92056a5..1ad6e40f495291 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -240,9 +240,14 @@ static void allocate_buf_for_compression(void)
 		return;
 	}
 
-	/* A non-NULL big_oops_buf indicates compression is available. */
-	big_oops_buf = buf;
 	max_compressed_size = compressed_size;
+	/*
+	 * A non-NULL big_oops_buf indicates compression is available.
+	 * pstore_dump() tests it without psinfo_lock and then reads
+	 * max_compressed_size, which is stored above. Publish the buffer
+	 * with release semantics.
+	 */
+	smp_store_release(&big_oops_buf, buf);
 
 	pr_info("Using crash dump compression: %s\n", compress);
 }
@@ -316,7 +321,8 @@ static void pstore_dump(struct kmsg_dumper *dumper,
 		record.part = part;
 		record.buf = psinfo->buf;
 
-		dst = big_oops_buf ?: psinfo->buf;
+		/* Pairs with the smp_store_release() in allocate_buf_for_compression(). */
+		dst = smp_load_acquire(&big_oops_buf) ?: psinfo->buf;
 		dst_size = max_compressed_size ?: psinfo->bufsize;
 
 		/* Write dump header. */
diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c
index 2eb0d4fa218695..aa056d0395a4d7 100644
--- a/fs/pstore/ram.c
+++ b/fs/pstore/ram.c
@@ -604,6 +604,7 @@ static int ramoops_init_prz(const char *name,
 	if (IS_ERR(*prz)) {
 		int err = PTR_ERR(*prz);
 
+		*prz = NULL;
 		dev_err(dev, "failed to request %s mem region (0x%zx@0x%llx): %d\n",
 			name, sz, (unsigned long long)*paddr, err);
 		return err;
diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c
index b4ec8009b8dc91..ae941777037b84 100644
--- a/fs/pstore/ram_core.c
+++ b/fs/pstore/ram_core.c
@@ -505,6 +505,12 @@ static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size,
 	prz->paddr = start;
 	prz->size = size;
 
+	if (size <= sizeof(struct persistent_ram_buffer)) {
+		pr_err("%s: persistent ram zone too small (%#llx bytes)\n",
+		       prz->label, (unsigned long long)size);
+		return -EINVAL;
+	}
+
 	if (pfn_valid(start >> PAGE_SHIFT))
 		prz->vaddr = persistent_ram_vmap(start, size, memtype);
 	else
diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index 1c78c695d0ddf4..3ad19d7e472f62 100644
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -809,6 +809,7 @@ static unsigned long
 dqcache_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
 {
 	struct dquot *dquot;
+	unsigned long orig_nr_to_scan = sc->nr_to_scan;
 	unsigned long freed = 0;
 
 	spin_lock(&dq_list_lock);
@@ -822,6 +823,21 @@ dqcache_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
 		freed++;
 	}
 	spin_unlock(&dq_list_lock);
+
+	sc->nr_scanned = orig_nr_to_scan - sc->nr_to_scan;
+
+	/*
+	 * DQST_FREE_DQUOTS is a percpu counter, so count_objects() can
+	 * report a stale/approximate value that is still positive even
+	 * though free_dquots is actually empty by the time we get here.
+	 * When that happens sc->nr_scanned comes back 0 and we have
+	 * nothing further to offer this reclaim pass, so tell
+	 * do_shrink_slab() to stop calling us instead of letting it burn
+	 * through its one-shot scan budget against an empty list.
+	 */
+	if (sc->nr_scanned == 0)
+		return SHRINK_STOP;
+
 	return freed;
 }
 
@@ -2079,7 +2095,7 @@ EXPORT_SYMBOL(__dquot_transfer);
 /* Wrapper for transferring ownership of an inode for uid/gid only
  * Called from FSXXX_setattr()
  */
-int dquot_transfer(struct mnt_idmap *idmap, struct inode *inode,
+int dquot_transfer(const struct mnt_idmap *idmap, struct inode *inode,
 		   struct iattr *iattr)
 {
 	struct dquot *transfer_to[MAXQUOTAS] = {};
diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c
index fb471bf88ab7a6..7ed6fba134c67e 100644
--- a/fs/ramfs/file-nommu.c
+++ b/fs/ramfs/file-nommu.c
@@ -22,7 +22,7 @@
 #include <linux/uaccess.h>
 #include "internal.h"
 
-static int ramfs_nommu_setattr(struct mnt_idmap *, struct dentry *, struct iattr *);
+static int ramfs_nommu_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *);
 static unsigned long ramfs_nommu_get_unmapped_area(struct file *file,
 						   unsigned long addr,
 						   unsigned long len,
@@ -161,7 +161,7 @@ static int ramfs_nommu_resize(struct inode *inode, loff_t newsize, loff_t size)
  * handle a change of attributes
  * - we're specifically interested in a change of size
  */
-static int ramfs_nommu_setattr(struct mnt_idmap *idmap,
+static int ramfs_nommu_setattr(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, struct iattr *ia)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c
index 0a88ede48e0ad7..ef91b933d4e610 100644
--- a/fs/ramfs/inode.c
+++ b/fs/ramfs/inode.c
@@ -95,7 +95,7 @@ struct inode *ramfs_get_inode(struct super_block *sb,
  */
 /* SMP-safe */
 static int
-ramfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+ramfs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 	    struct dentry *dentry, umode_t mode, dev_t dev)
 {
 	struct inode * inode = ramfs_get_inode(dir->i_sb, dir, mode, dev);
@@ -118,7 +118,7 @@ ramfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return error;
 }
 
-static struct dentry *ramfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ramfs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				 struct dentry *dentry, umode_t mode)
 {
 	int retval = ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode, 0);
@@ -127,13 +127,13 @@ static struct dentry *ramfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	return ERR_PTR(retval);
 }
 
-static int ramfs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int ramfs_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	return ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFREG, 0);
 }
 
-static int ramfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int ramfs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, const char *symname)
 {
 	struct inode *inode;
@@ -163,7 +163,7 @@ static int ramfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	return error;
 }
 
-static int ramfs_tmpfile(struct mnt_idmap *idmap,
+static int ramfs_tmpfile(const struct mnt_idmap *idmap,
 			 struct inode *dir, struct file *file, umode_t mode)
 {
 	struct inode *inode;
diff --git a/fs/read_write.c b/fs/read_write.c
index e8c14e2760b203..4da846a2bf17b8 100644
--- a/fs/read_write.c
+++ b/fs/read_write.c
@@ -274,7 +274,7 @@ loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t si
 EXPORT_SYMBOL(fixed_size_llseek);
 
 /**
- * no_seek_end_llseek - llseek implementation for fixed-sized devices
+ * no_seek_end_llseek - llseek implementation for files without SEEK_END
  * @file:	file structure to seek on
  * @offset:	file offset to seek to
  * @whence:	type of seek
@@ -293,7 +293,7 @@ loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
 EXPORT_SYMBOL(no_seek_end_llseek);
 
 /**
- * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
+ * no_seek_end_llseek_size - llseek implementation for files without SEEK_END
  * @file:	file structure to seek on
  * @offset:	file offset to seek to
  * @whence:	type of seek
@@ -1761,8 +1761,8 @@ EXPORT_SYMBOL(generic_write_checks_count);
  * Performs necessary checks before doing a write
  *
  * Can adjust writing position or amount of bytes to write.
- * Returns appropriate error code that caller should return or
- * zero in case that write should be allowed.
+ * Returns the number of bytes that may be written on success (which
+ * may be less than requested if truncated), or a negative error code.
  */
 ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
 {
diff --git a/fs/remap_range.c b/fs/remap_range.c
index 26afbbbfb10c2e..6eb7d845de5d4b 100644
--- a/fs/remap_range.c
+++ b/fs/remap_range.c
@@ -415,7 +415,7 @@ EXPORT_SYMBOL(vfs_clone_file_range);
 /* Check whether we are allowed to dedupe the destination file */
 static bool may_dedupe_file(struct file *file)
 {
-	struct mnt_idmap *idmap = file_mnt_idmap(file);
+	const struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct inode *inode = file_inode(file);
 
 	if (capable(CAP_SYS_ADMIN))
diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c
index 18ec9f564b5a1b..cafebdff70dca0 100644
--- a/fs/resctrl/ctrlmondata.c
+++ b/fs/resctrl/ctrlmondata.c
@@ -37,8 +37,8 @@ typedef int (ctrlval_parser_t)(struct rdt_parse_data *data,
 /*
  * Check whether MBA bandwidth percentage value is correct. The value is
  * checked against the minimum and max bandwidth values specified by the
- * hardware. The allocated bandwidth percentage is rounded to the next
- * control step available on the hardware.
+ * hardware. The allocated bandwidth percentage is converted as appropriate
+ * for consumption by the specific hardware driver.
  */
 static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r)
 {
@@ -71,7 +71,7 @@ static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r)
 		return false;
 	}
 
-	*data = roundup(bw, (unsigned long)r->membw.bw_gran);
+	*data = resctrl_arch_preconvert_bw(r, bw);
 	return true;
 }
 
diff --git a/fs/resctrl/pseudo_lock.c b/fs/resctrl/pseudo_lock.c
index dea2b4bf966fa5..56ab63f19badf1 100644
--- a/fs/resctrl/pseudo_lock.c
+++ b/fs/resctrl/pseudo_lock.c
@@ -750,17 +750,10 @@ static ssize_t pseudo_lock_measure_trigger(struct file *file,
 					   size_t count, loff_t *ppos)
 {
 	struct rdtgroup *rdtgrp = file->private_data;
-	size_t buf_size;
-	char buf[32];
 	int ret;
 	int sel;
 
-	buf_size = min(count, (sizeof(buf) - 1));
-	if (copy_from_user(buf, user_buf, buf_size))
-		return -EFAULT;
-
-	buf[buf_size] = '\0';
-	ret = kstrtoint(buf, 10, &sel);
+	ret = kstrtoint_from_user(user_buf, count, 10, &sel);
 	if (ret == 0) {
 		if (sel != 1 && sel != 2 && sel != 3)
 			return -EINVAL;
diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c
index 5dcbb0a964e815..68be9b903ac666 100644
--- a/fs/resctrl/rdtgroup.c
+++ b/fs/resctrl/rdtgroup.c
@@ -2858,7 +2858,7 @@ static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type
 {
 	struct resctrl_schema *s;
 	const char *suffix = "";
-	int ret, cl;
+	int cl;
 
 	s = kzalloc_obj(*s);
 	if (!s)
@@ -2882,14 +2882,12 @@ static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type
 		break;
 	}
 
-	ret = snprintf(s->name, sizeof(s->name), "%s%s", r->name, suffix);
-	if (ret >= sizeof(s->name)) {
+	cl = snprintf(s->name, sizeof(s->name), "%s%s", r->name, suffix);
+	if (cl >= sizeof(s->name)) {
 		kfree(s);
 		return -EINVAL;
 	}
 
-	cl = strlen(s->name);
-
 	/*
 	 * If CDP is supported by this resource, but not enabled,
 	 * include the suffix. This ensures the tabular format of the
diff --git a/fs/signalfd.c b/fs/signalfd.c
index dff53745e35241..22bc0870a824d8 100644
--- a/fs/signalfd.c
+++ b/fs/signalfd.c
@@ -48,17 +48,30 @@ static int signalfd_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
+static void refine_sigmask(struct signalfd_ctx *ctx, sigset_t *sigmask)
+{
+	struct k_sigaction *k = current->sighand->action;
+	int n;
+
+	*sigmask = ctx->sigmask;
+	for (n = 1; n <= _NSIG; ++n, ++k) {
+		if (k->sa.sa_flags & SA_IMMUTABLE)
+			sigaddset(sigmask, n);
+	}
+}
+
 static __poll_t signalfd_poll(struct file *file, poll_table *wait)
 {
 	struct signalfd_ctx *ctx = file->private_data;
 	__poll_t events = 0;
+	sigset_t sigmask;
 
 	poll_wait(file, &current->sighand->signalfd_wqh, wait);
 
 	spin_lock_irq(&current->sighand->siglock);
-	if (next_signal(&current->pending, &ctx->sigmask) ||
-	    next_signal(&current->signal->shared_pending,
-			&ctx->sigmask))
+	refine_sigmask(ctx, &sigmask);
+	if (next_signal(&current->pending, &sigmask) ||
+	    next_signal(&current->signal->shared_pending, &sigmask))
 		events |= EPOLLIN;
 	spin_unlock_irq(&current->sighand->siglock);
 
@@ -155,11 +168,13 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info
 				int nonblock)
 {
 	enum pid_type type;
-	ssize_t ret;
 	DECLARE_WAITQUEUE(wait, current);
+	sigset_t sigmask;
+	ssize_t ret;
 
 	spin_lock_irq(&current->sighand->siglock);
-	ret = dequeue_signal(&ctx->sigmask, info, &type);
+	refine_sigmask(ctx, &sigmask);
+	ret = dequeue_signal(&sigmask, info, &type);
 	switch (ret) {
 	case 0:
 		if (!nonblock)
@@ -174,7 +189,7 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info
 	add_wait_queue(&current->sighand->signalfd_wqh, &wait);
 	for (;;) {
 		set_current_state(TASK_INTERRUPTIBLE);
-		ret = dequeue_signal(&ctx->sigmask, info, &type);
+		ret = dequeue_signal(&sigmask, info, &type);
 		if (ret != 0)
 			break;
 		if (signal_pending(current)) {
@@ -184,6 +199,7 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info
 		spin_unlock_irq(&current->sighand->siglock);
 		schedule();
 		spin_lock_irq(&current->sighand->siglock);
+		refine_sigmask(ctx, &sigmask);
 	}
 	spin_unlock_irq(&current->sighand->siglock);
 
diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c
index 88d5e9a32f28b3..647fa26da4d24c 100644
--- a/fs/smb/client/cached_dir.c
+++ b/fs/smb/client/cached_dir.c
@@ -8,6 +8,7 @@
 #include <linux/namei.h>
 #include "cifsglob.h"
 #include "cifsproto.h"
+#include "../common/smb2status.h"
 #include "cifs_debug.h"
 #include "smb2proto.h"
 #include "cached_dir.h"
@@ -323,25 +324,37 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon,
 	rc = compound_send_recv(xid, ses, server,
 				flags, 2, rqst,
 				resp_buftype, rsp_iov);
-	if (rc) {
-		if (rc == -EREMCHG) {
-			tcon->need_reconnect = true;
-			pr_warn_once("server share %s deleted\n",
-				     tcon->tree_name);
-		}
-		goto oshr_free;
+	if (rc == -EREMCHG) {
+		tcon->need_reconnect = true;
+		pr_warn_once("server share %s deleted\n",
+			     tcon->tree_name);
 	}
-	cfid->is_open = true;
 
-	spin_lock(&cfids->cfid_list_lock);
+	if (!rsp_iov[0].iov_base || rsp_iov[0].iov_len < sizeof(*o_rsp)) {
+		if (!rc)
+			rc = -EIO;
+		goto oshr_free;
+	}
 
 	o_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base;
+	if (o_rsp->hdr.Status != STATUS_SUCCESS) {
+		if (!rc)
+			rc = -EIO;
+		goto oshr_free;
+	}
+
 	oparms.fid->persistent_fid = o_rsp->PersistentFileId;
 	oparms.fid->volatile_fid = o_rsp->VolatileFileId;
 #ifdef CONFIG_CIFS_DEBUG2
 	oparms.fid->mid = le64_to_cpu(o_rsp->hdr.MessageId);
 #endif /* CIFS_DEBUG2 */
+	cfid->is_open = true;
+	atomic_inc(&tcon->num_remote_opens);
 
+	if (rc)
+		goto oshr_free;
+
+	spin_lock(&cfids->cfid_list_lock);
 
 	if (o_rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) {
 		spin_unlock(&cfids->cfid_list_lock);
@@ -408,7 +421,6 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon,
 		close_cached_dir(cfid);
 	} else {
 		*ret_cfid = cfid;
-		atomic_inc(&tcon->num_remote_opens);
 	}
 	kfree(utf16_path);
 
diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c
index c5e47a835f99db..d1a92bb4d9a596 100644
--- a/fs/smb/client/cifsacl.c
+++ b/fs/smb/client/cifsacl.c
@@ -1904,7 +1904,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode,
 	return rc;
 }
 
-struct posix_acl *cifs_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *cifs_get_acl(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, int type)
 {
 #if defined(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) && defined(CONFIG_CIFS_POSIX)
@@ -1968,7 +1968,7 @@ struct posix_acl *cifs_get_acl(struct mnt_idmap *idmap,
 #endif
 }
 
-int cifs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int cifs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type)
 {
 #if defined(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) && defined(CONFIG_CIFS_POSIX)
diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c
index 7ecd70efdfea89..b1ecbcfb154e44 100644
--- a/fs/smb/client/cifsfs.c
+++ b/fs/smb/client/cifsfs.c
@@ -402,7 +402,7 @@ static long cifs_fallocate(struct file *file, int mode, loff_t off, loff_t len)
 	return rc;
 }
 
-static int cifs_permission(struct mnt_idmap *idmap,
+static int cifs_permission(const struct mnt_idmap *idmap,
 			   struct inode *inode, int mask)
 {
 	unsigned int sbflags = cifs_sb_flags(CIFS_SB(inode));
diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h
index 0c85daa8386ef8..51692e14c4dd38 100644
--- a/fs/smb/client/cifsfs.h
+++ b/fs/smb/client/cifsfs.h
@@ -53,23 +53,23 @@ void cifs_sb_deactive(struct super_block *sb);
 /* Functions related to inodes */
 extern const struct inode_operations cifs_dir_inode_ops;
 struct inode *cifs_root_iget(struct super_block *sb);
-int cifs_create(struct mnt_idmap *idmap, struct inode *dir,
+int cifs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *direntry, umode_t mode);
 int cifs_atomic_open(struct inode *dir, struct dentry *direntry,
 		     struct file *file, unsigned int oflags, umode_t mode);
-int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+int cifs_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 		 struct file *file, umode_t mode);
 struct dentry *cifs_lookup(struct inode *parent_dir_inode,
 			   struct dentry *direntry, unsigned int flags);
 int cifs_unlink(struct inode *dir, struct dentry *dentry);
 int cifs_hardlink(struct dentry *old_file, struct inode *inode,
 		  struct dentry *direntry);
-int cifs_mknod(struct mnt_idmap *idmap, struct inode *inode,
+int cifs_mknod(const struct mnt_idmap *idmap, struct inode *inode,
 	       struct dentry *direntry, umode_t mode, dev_t device_number);
-struct dentry *cifs_mkdir(struct mnt_idmap *idmap, struct inode *inode,
+struct dentry *cifs_mkdir(const struct mnt_idmap *idmap, struct inode *inode,
 			  struct dentry *direntry, umode_t mode);
 int cifs_rmdir(struct inode *inode, struct dentry *direntry);
-int cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir,
+int cifs_rename2(const struct mnt_idmap *idmap, struct inode *source_dir,
 		 struct dentry *source_dentry, struct inode *target_dir,
 		 struct dentry *target_dentry, unsigned int flags);
 int cifs_revalidate_file_attr(struct file *filp);
@@ -78,9 +78,9 @@ int cifs_revalidate_file(struct file *filp);
 int cifs_revalidate_dentry(struct dentry *dentry);
 int cifs_revalidate_mapping(struct inode *inode);
 int cifs_zap_mapping(struct inode *inode);
-int cifs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int cifs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int flags);
-int cifs_setattr(struct mnt_idmap *idmap, struct dentry *direntry,
+int cifs_setattr(const struct mnt_idmap *idmap, struct dentry *direntry,
 		 struct iattr *attrs);
 int cifs_fiemap(struct inode *inode, struct fiemap_extent_info *fei, u64 start,
 		u64 len);
@@ -129,7 +129,7 @@ struct vfsmount *cifs_d_automount(struct path *path);
 /* Functions related to symlinks */
 const char *cifs_get_link(struct dentry *dentry, struct inode *inode,
 			  struct delayed_call *done);
-int cifs_symlink(struct mnt_idmap *idmap, struct inode *inode,
+int cifs_symlink(const struct mnt_idmap *idmap, struct inode *inode,
 		 struct dentry *direntry, const char *symname);
 
 #ifdef CONFIG_CIFS_XATTR
diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h
index 00168839c12319..e6beff8aafe09d 100644
--- a/fs/smb/client/cifsproto.h
+++ b/fs/smb/client/cifsproto.h
@@ -212,9 +212,9 @@ struct smb_ntsd *get_cifs_acl(struct cifs_sb_info *cifs_sb,
 struct smb_ntsd *get_cifs_acl_by_fid(struct cifs_sb_info *cifs_sb,
 				     const struct cifs_fid *cifsfid,
 				     u32 *pacllen, u32 info);
-struct posix_acl *cifs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+struct posix_acl *cifs_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			       int type);
-int cifs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int cifs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		 struct posix_acl *acl, int type);
 int set_cifs_acl(struct smb_ntsd *pnntsd, __u32 acllen, struct inode *inode,
 		 const char *path, int aclflag);
diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c
index 29dfd75959413c..26956e33037e7c 100644
--- a/fs/smb/client/dfs_cache.c
+++ b/fs/smb/client/dfs_cache.c
@@ -798,13 +798,13 @@ static int get_targets(struct cache_entry *ce, struct dfs_cache_tgt_list *tl)
 	INIT_LIST_HEAD(head);
 
 	list_for_each_entry(t, &ce->tlist, list) {
-		it = kzalloc_obj(*it, GFP_ATOMIC);
+		it = kzalloc_obj(*it, GFP_KERNEL);
 		if (!it) {
 			rc = -ENOMEM;
 			goto err_free_it;
 		}
 
-		it->it_name = kstrdup(t->name, GFP_ATOMIC);
+		it->it_name = kstrdup(t->name, GFP_KERNEL);
 		if (!it->it_name) {
 			kfree(it);
 			rc = -ENOMEM;
diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c
index 6fa6d48fdfd30f..a2cf3d35ca5df0 100644
--- a/fs/smb/client/dir.c
+++ b/fs/smb/client/dir.c
@@ -199,7 +199,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 			    struct tcon_link *tlink, unsigned int oflags,
 			    umode_t mode, __u32 *oplock, struct cifs_fid *fid,
 			    struct cifs_open_info_data *buf,
-			    struct inode **inode)
+			    struct inode **inode, bool *opened)
 {
 	int rc = -ENOENT;
 	int create_options = CREATE_NOT_DIR;
@@ -216,6 +216,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 	__le32 lease_flags = 0;
 
 	*inode = NULL;
+	*opened = false;
 	*oplock = 0;
 	if (tcon->ses->server->oplocks)
 		*oplock = REQ_OPLOCK;
@@ -232,6 +233,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 				     oflags, oplock, &fid->netfid, xid);
 		switch (rc) {
 		case 0:
+			*opened = true;
 			if (newinode == NULL) {
 				/* query inode info */
 				goto cifs_create_get_file_info;
@@ -253,11 +255,9 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 				/*
 				 * The server may allow us to open things like
 				 * FIFOs, but the client isn't set up to deal
-				 * with that. If it's not a regular file, just
-				 * close it and proceed as if it were a normal
-				 * lookup.
+				 * with that. Keep the handle until the caller
+				 * can finish the lookup.
 				 */
-				CIFSSMBClose(xid, tcon, fid->netfid);
 				goto cifs_create_get_file_info;
 			}
 			/* success, no need to query */
@@ -384,6 +384,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 		}
 		return rc;
 	}
+	*opened = true;
 	if (rdwr_for_fscache == 2)
 		cifs_invalidate_cache(dir, FSCACHE_INVAL_DIO_WRITE);
 
@@ -475,7 +476,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 	return rc;
 
 out_err:
-	if (server->ops->close)
+	if (*opened && server->ops->close)
 		server->ops->close(xid, tcon, fid);
 	if (newinode)
 		iput(newinode);
@@ -487,7 +488,7 @@ static int cifs_do_create(struct inode *dir, struct dentry *direntry,
 			  unsigned int oflags, umode_t mode,
 			  __u32 *oplock, struct cifs_fid *fid,
 			  struct cifs_open_info_data *buf,
-			  struct inode **inode)
+			  struct inode **inode, bool *opened)
 {
 	void *page = alloc_dentry_path();
 	const char *full_path;
@@ -496,10 +497,11 @@ static int cifs_do_create(struct inode *dir, struct dentry *direntry,
 	full_path = build_path_from_dentry(direntry, page);
 	if (IS_ERR(full_path)) {
 		rc = PTR_ERR(full_path);
+		*opened = false;
 	} else {
 		rc = __cifs_do_create(dir, direntry, full_path, xid,
 				      tlink, oflags, mode, oplock,
-				      fid, buf, inode);
+				      fid, buf, inode, opened);
 	}
 	free_dentry_path(page);
 	return rc;
@@ -529,6 +531,8 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry,
 	struct inode *inode;
 	unsigned int xid;
 	__u32 oplock;
+	bool is_regular;
+	bool opened;
 	int rc;
 
 	if (unlikely(cifs_forced_shutdown(cifs_sb)))
@@ -581,12 +585,26 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry,
 	cifs_add_pending_open(&fid, tlink, &open);
 
 	rc = cifs_do_create(dir, direntry, xid, tlink, oflags, mode,
-			    &oplock, &fid, &buf, &inode);
+			    &oplock, &fid, &buf, &inode, &opened);
 	if (rc) {
 		cifs_del_pending_open(&open);
 		goto out;
 	}
 
+	is_regular = S_ISREG(inode->i_mode);
+	if (!is_regular || !opened) {
+		if (opened && server->ops->close)
+			server->ops->close(xid, tcon, &fid);
+		cifs_del_pending_open(&open);
+		if (S_ISLNK(inode->i_mode) &&
+		    (oflags & (O_NOFOLLOW | __O_REGULAR)) ==
+		    (O_NOFOLLOW | __O_REGULAR) && !(oflags & O_EXCL)) {
+			iput(inode);
+			rc = -ELOOP;
+			goto out;
+		}
+	}
+
 	if (d_in_lookup(direntry)) {
 		alias = d_splice_alias(inode, direntry);
 		if (!IS_ERR_OR_NULL(alias))
@@ -595,9 +613,15 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry,
 		d_instantiate(direntry, inode);
 	}
 
-	if ((oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL))
+	if (is_regular && opened &&
+	    (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL))
 		file->f_mode |= FMODE_CREATED;
 
+	if (!is_regular || !opened) {
+		rc = finish_no_open(file, NULL);
+		goto out;
+	}
+
 	rc = finish_open(file, direntry, generic_file_open);
 	if (rc) {
 		if (server->ops->close)
@@ -640,7 +664,7 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry,
  * The initial dentry state is hashed-negative.  On success, dentry will become
  * hashed-positive by calling d_instantiate().
  */
-int cifs_create(struct mnt_idmap *idmap, struct inode *dir,
+int cifs_create(const struct mnt_idmap *idmap, struct inode *dir,
 		struct dentry *direntry, umode_t mode)
 {
 	struct cifs_sb_info *cifs_sb = CIFS_SB(dir);
@@ -660,6 +684,7 @@ int cifs_create(struct mnt_idmap *idmap, struct inode *dir,
 	struct inode *inode;
 	struct cifs_fid fid;
 	__u32 oplock;
+	bool opened;
 	struct cifs_open_info_data buf = {};
 
 	cifs_dbg(FYI, "cifs_create parent inode = 0x%p name is: %pd and dentry = 0x%p\n",
@@ -682,10 +707,10 @@ int cifs_create(struct mnt_idmap *idmap, struct inode *dir,
 		server->ops->new_lease_key(&fid);
 
 	rc = cifs_do_create(dir, direntry, xid, tlink, oflags,
-			    mode, &oplock, &fid, &buf, &inode);
+			    mode, &oplock, &fid, &buf, &inode, &opened);
 	if (!rc) {
 		d_instantiate(direntry, inode);
-		if (server->ops->close)
+		if (opened && server->ops->close)
 			server->ops->close(xid, tcon, &fid);
 	}
 
@@ -696,7 +721,7 @@ int cifs_create(struct mnt_idmap *idmap, struct inode *dir,
 	return rc;
 }
 
-int cifs_mknod(struct mnt_idmap *idmap, struct inode *inode,
+int cifs_mknod(const struct mnt_idmap *idmap, struct inode *inode,
 	       struct dentry *direntry, umode_t mode, dev_t device_number)
 {
 	int rc = -EPERM;
@@ -1059,7 +1084,7 @@ static int set_tmpfile_attr(const unsigned int xid, unsigned int oflags,
  * The initial dentry state is unhashed-negative.  On success, dentry will
  * become unhashed-positive by calling d_instantiate().
  */
-int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+int cifs_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 		 struct file *file, umode_t mode)
 {
 	struct dentry *dentry = file->f_path.dentry;
@@ -1078,6 +1103,7 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	struct inode *inode;
 	unsigned int xid;
 	__u32 oplock;
+	bool opened;
 	int namelen;
 	int rc;
 
@@ -1116,7 +1142,7 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 		namelen = scnprintf(name, namesize, CIFS_TMPNAME_PREFIX "%x",
 				    atomic_inc_return(&cifs_tmpcounter));
 		rc = __cifs_do_create(dir, dentry, path, xid, tlink, oflags,
-				      mode, &oplock, &fid, NULL, &inode);
+				      mode, &oplock, &fid, NULL, &inode, &opened);
 		if (!rc) {
 			rc = d_mark_tmpfile_name(file, &QSTR_LEN(name, namelen));
 			if (rc) {
diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c
index 1fe0ef0a95dbde..4d7a87c7b73a8b 100644
--- a/fs/smb/client/inode.c
+++ b/fs/smb/client/inode.c
@@ -2281,7 +2281,7 @@ cifs_posix_mkdir(struct inode *inode, struct dentry *dentry, umode_t mode,
 }
 #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */
 
-struct dentry *cifs_mkdir(struct mnt_idmap *idmap, struct inode *inode,
+struct dentry *cifs_mkdir(const struct mnt_idmap *idmap, struct inode *inode,
 			  struct dentry *direntry, umode_t mode)
 {
 	int rc = 0;
@@ -2531,7 +2531,7 @@ cifs_do_rename(const unsigned int xid, struct dentry *from_dentry,
 }
 
 int
-cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir,
+cifs_rename2(const struct mnt_idmap *idmap, struct inode *source_dir,
 	     struct dentry *source_dentry, struct inode *target_dir,
 	     struct dentry *target_dentry, unsigned int flags)
 {
@@ -2937,7 +2937,7 @@ int cifs_revalidate_dentry(struct dentry *dentry)
 	return cifs_revalidate_mapping(inode);
 }
 
-int cifs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int cifs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	struct cifs_sb_info *cifs_sb = CIFS_SB(path->dentry);
@@ -3554,7 +3554,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs)
 }
 
 int
-cifs_setattr(struct mnt_idmap *idmap, struct dentry *direntry,
+cifs_setattr(const struct mnt_idmap *idmap, struct dentry *direntry,
 	     struct iattr *attrs)
 {
 	struct cifs_sb_info *cifs_sb = CIFS_SB(direntry->d_sb);
diff --git a/fs/smb/client/link.c b/fs/smb/client/link.c
index 8d5d6aca742aa1..76df31abeaeaec 100644
--- a/fs/smb/client/link.c
+++ b/fs/smb/client/link.c
@@ -533,7 +533,7 @@ cifs_hardlink(struct dentry *old_file, struct inode *inode,
 }
 
 int
-cifs_symlink(struct mnt_idmap *idmap, struct inode *inode,
+cifs_symlink(const struct mnt_idmap *idmap, struct inode *inode,
 	     struct dentry *direntry, const char *symname)
 {
 	struct cifs_sb_info *cifs_sb = CIFS_SB(inode);
diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c
index 13fe8e3b48f319..f46a62eae65912 100644
--- a/fs/smb/client/smb2inode.c
+++ b/fs/smb/client/smb2inode.c
@@ -598,8 +598,10 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon,
 		/* smb2_parse_contexts() fills idata->fi.IndexNumber */
 		rc = smb2_parse_contexts(server, &rsp_iov[0], &oparms->fid->epoch,
 					 oparms->fid->lease_key, &oplock, &idata->fi, NULL);
-		if (rc)
+		if (rc) {
 			cifs_dbg(VFS, "rc: %d parsing context of compound op\n", rc);
+			tmp_rc = rc;
+		}
 	}
 
 	for (i = 0; i < num_cmds; i++) {
@@ -1121,7 +1123,7 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name,
 	struct kvec close_iov;
 	int resp_buftype[2];
 	struct cifs_fid fid;
-	int flags = 0;
+	int flags = CIFS_CP_CREATE_CLOSE_OP;
 	__u8 oplock;
 	int rc;
 
diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c
index 0cfe60ae42c380..5e5cf92d1eb340 100644
--- a/fs/smb/client/smb2misc.c
+++ b/fs/smb/client/smb2misc.c
@@ -834,7 +834,8 @@ smb2_cancelled_close_fid(struct work_struct *work)
  */
 static int
 __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid,
-			    __u64 persistent_fid, __u64 volatile_fid)
+			    __u64 persistent_fid, __u64 volatile_fid,
+			    bool account_remote_open)
 {
 	struct close_cancelled_open *cancelled;
 
@@ -848,6 +849,8 @@ __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid,
 	cancelled->cmd = cmd;
 	cancelled->mid = mid;
 	INIT_WORK(&cancelled->work, smb2_cancelled_close_fid);
+	if (account_remote_open)
+		atomic_inc(&tcon->num_remote_opens);
 	WARN_ON(queue_work(cifsiod_wq, &cancelled->work) == false);
 
 	return 0;
@@ -884,7 +887,7 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid,
 	spin_unlock(&tcon->tc_lock);
 
 	rc = __smb2_handle_cancelled_cmd(tcon, SMB2_CLOSE_HE, 0,
-					 persistent_fid, volatile_fid);
+					 persistent_fid, volatile_fid, false);
 	if (rc)
 		cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_close);
 
@@ -912,7 +915,7 @@ smb2_handle_cancelled_mid(struct mid_q_entry *mid, struct TCP_Server_Info *serve
 					 le16_to_cpu(hdr->Command),
 					 le64_to_cpu(hdr->MessageId),
 					 rsp->PersistentFileId,
-					 rsp->VolatileFileId);
+					 rsp->VolatileFileId, true);
 	if (rc)
 		cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_mid);
 
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 3464470d329775..aa142420dae2e6 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -4572,25 +4572,37 @@ smb3_create_lease_buf(u8 *lease_key, u8 oplock, u8 *parent_lease_key, __le32 fla
 static __u8
 smb2_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key)
 {
-	struct create_lease *lc = (struct create_lease *)buf;
+	struct create_context *cc = buf;
+	struct lease_context lc;
 
 	*epoch = 0; /* not used */
-	if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE)
+	if (le32_to_cpu(cc->DataLength) != sizeof(lc))
+		return 0;
+
+	memcpy(&lc, (u8 *)cc + le16_to_cpu(cc->DataOffset), sizeof(lc));
+	if (lc.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE)
 		return SMB2_OPLOCK_LEVEL_NOCHANGE;
-	return le32_to_cpu(lc->lcontext.LeaseState);
+	return le32_to_cpu(lc.LeaseState);
 }
 
 static __u8
 smb3_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key)
 {
-	struct create_lease_v2 *lc = (struct create_lease_v2 *)buf;
+	struct create_context *cc = buf;
+	struct lease_context_v2 lc;
+
+	if (le32_to_cpu(cc->DataLength) != sizeof(lc)) {
+		*epoch = 0;
+		return 0;
+	}
 
-	*epoch = le16_to_cpu(lc->lcontext.Epoch);
-	if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE)
+	memcpy(&lc, (u8 *)cc + le16_to_cpu(cc->DataOffset), sizeof(lc));
+	*epoch = le16_to_cpu(lc.Epoch);
+	if (lc.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE)
 		return SMB2_OPLOCK_LEVEL_NOCHANGE;
 	if (lease_key)
-		memcpy(lease_key, &lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE);
-	return le32_to_cpu(lc->lcontext.LeaseState);
+		memcpy(lease_key, lc.LeaseKey, SMB2_LEASE_KEY_SIZE);
+	return le32_to_cpu(lc.LeaseState);
 }
 
 static unsigned int
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index 880ce12f50c481..3d7ead36d1a0e6 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -2379,23 +2379,32 @@ create_reconnect_durable_buf(struct cifs_fid *fid)
 static void
 parse_query_id_ctxt(struct create_context *cc, struct smb2_file_all_info *buf)
 {
-	struct create_disk_id_rsp *pdisk_id = (struct create_disk_id_rsp *)cc;
+	u16 doff = le16_to_cpu(cc->DataOffset);
+	u32 dlen = le32_to_cpu(cc->DataLength);
+	u8 *beg;
 
-	cifs_dbg(FYI, "parse query id context 0x%llx 0x%llx\n",
-		pdisk_id->DiskFileId, pdisk_id->VolumeId);
-	buf->IndexNumber = pdisk_id->DiskFileId;
+	if (dlen < sizeof(__le64))
+		return;
+
+	beg = (u8 *)cc + doff;
+	memcpy(&buf->IndexNumber, beg, sizeof(__le64));
+	cifs_dbg(FYI, "parse query id context 0x%llx\n",
+		 le64_to_cpu(buf->IndexNumber));
 }
 
 static void
 parse_posix_ctxt(struct create_context *cc, struct smb2_file_all_info *info,
 		 struct create_posix_rsp *posix)
 {
-	int sid_len;
 	u8 *beg = (u8 *)cc + le16_to_cpu(cc->DataOffset);
-	u8 *end = beg + le32_to_cpu(cc->DataLength);
+	u32 dlen = le32_to_cpu(cc->DataLength);
+	u8 *end = beg + dlen;
+	int sid_len;
 	u8 *sid;
 
 	memset(posix, 0, sizeof(*posix));
+	if (dlen < 3 * sizeof(__le32))
+		return;
 
 	posix->nlink = get_unaligned_le32(beg);
 	posix->reparse_tag = get_unaligned_le32(beg + 4);
@@ -2431,6 +2440,7 @@ int smb2_parse_contexts(struct TCP_Server_Info *server,
 	struct smb2_create_rsp *rsp = rsp_iov->iov_base;
 	struct create_context *cc;
 	size_t rem, off, len;
+	size_t cc_len;
 	size_t doff, dlen;
 	size_t noff, nlen;
 	char *name;
@@ -2453,29 +2463,41 @@ int smb2_parse_contexts(struct TCP_Server_Info *server,
 		buf->IndexNumber = 0;
 
 	while (rem >= sizeof(*cc)) {
+		off = le32_to_cpu(cc->Next);
+		if (off) {
+			if ((off & 0x7) || off >= rem || off < sizeof(*cc))
+				return -EINVAL;
+			cc_len = off;
+		} else {
+			cc_len = rem;
+		}
+
 		doff = le16_to_cpu(cc->DataOffset);
 		dlen = le32_to_cpu(cc->DataLength);
-		if (check_add_overflow(doff, dlen, &len) || len > rem)
+		if (doff < sizeof(*cc) ||
+		    check_add_overflow(doff, dlen, &len) || len > cc_len)
 			return -EINVAL;
 
 		noff = le16_to_cpu(cc->NameOffset);
 		nlen = le16_to_cpu(cc->NameLength);
-		if (noff + nlen > doff)
+		if (noff < sizeof(*cc) ||
+		    check_add_overflow(noff, nlen, &len) || len > cc_len ||
+		    (dlen && len > doff))
 			return -EINVAL;
 
 		name = (char *)cc + noff;
 		switch (nlen) {
 		case 4:
-			if (!strncmp(name, SMB2_CREATE_REQUEST_LEASE, 4)) {
+			if (dlen && !strncmp(name, SMB2_CREATE_REQUEST_LEASE, 4)) {
 				*oplock = server->ops->parse_lease_buf(cc, epoch,
 								       lease_key);
-			} else if (buf &&
+			} else if (dlen && buf &&
 				   !strncmp(name, SMB2_CREATE_QUERY_ON_DISK_ID, 4)) {
 				parse_query_id_ctxt(cc, buf);
 			}
 			break;
 		case 16:
-			if (posix && !memcmp(name, smb3_create_tag_posix, 16))
+			if (dlen && posix && !memcmp(name, smb3_create_tag_posix, 16))
 				parse_posix_ctxt(cc, buf, posix);
 			break;
 		default:
@@ -2487,13 +2509,18 @@ int smb2_parse_contexts(struct TCP_Server_Info *server,
 		}
 
 		off = le32_to_cpu(cc->Next);
-		if (!off)
+		if (!off) {
+			rem = 0;
 			break;
+		}
 		if (check_sub_overflow(rem, off, &rem))
 			return -EINVAL;
 		cc = (struct create_context *)((u8 *)cc + off);
 	}
 
+	if (rem)
+		return -EINVAL;
+
 	if (rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE)
 		*oplock = rsp->OplockLevel;
 
@@ -3389,6 +3416,9 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path,
 
 	rc = smb2_parse_contexts(server, &rsp_iov, &oparms->fid->epoch,
 				 oparms->fid->lease_key, oplock, file_info, posix);
+	if (rc)
+		SMB2_close(xid, tcon, oparms->fid->persistent_fid,
+			   oparms->fid->volatile_fid);
 
 	trace_smb3_open_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid,
 			     oparms->create_options, oparms->desired_access,
diff --git a/fs/smb/client/smb2pdu.h b/fs/smb/client/smb2pdu.h
index ab6c667bebc0ae..b37a1e3941a1d9 100644
--- a/fs/smb/client/smb2pdu.h
+++ b/fs/smb/client/smb2pdu.h
@@ -224,8 +224,7 @@ struct smb2_file_id_extd_directory_info {
 extern char smb2_padding[7];
 
 /*
- * See POSIX-SMB2 2.2.14.2.16
- * Link: https://gitlab.com/samba-team/smb3-posix-spec/-/blob/master/smb3_posix_extensions.md
+ * See POSIX-SMB2 2.1.3.2.1
  */
 struct create_posix_rsp {
 	u32 nlink;
@@ -238,6 +237,7 @@ struct create_posix_rsp {
 #define SMB2_QUERY_DIRECTORY_IOV_SIZE 2
 
 /*
+ * See POSIX-FSCC 2.2.1
  * SMB2-only POSIX info level for query dir
  *
  * See posix_info_sid_size(), posix_info_extra_size() and
@@ -256,13 +256,17 @@ struct smb2_posix_info {
 	__le64 Inode;
 	__le32 DeviceId;
 	__le32 Zero;
-	/* beginning of POSIX Create Context Response */
+	/*
+	 * Beginning of POSIX Create Context Response
+	 * See POSIX-SMB2 2.1.3.2.1
+	 */
 	__le32 HardLinks;
 	__le32 ReparseTag;
 	__le32 Mode;
 	/*
 	 * var sized owner SID
 	 * var sized group SID
+	 * End of POSIX Create Context Response
 	 * le32 filenamelength
 	 * u8  filename[]
 	 */
diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c
index e266859818a441..93ff6a4dbb3505 100644
--- a/fs/smb/client/transport.c
+++ b/fs/smb/client/transport.c
@@ -22,7 +22,6 @@
 #include <linux/mempool.h>
 #include <linux/sched/signal.h>
 #include <linux/task_io_accounting_ops.h>
-#include <linux/task_work.h>
 #include "cifsglob.h"
 #include "cifsproto.h"
 #include "cifs_debug.h"
@@ -171,15 +170,11 @@ smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg,
 		 * after the retries we will kill the socket and
 		 * reconnect which may clear the network problem.
 		 *
-		 * Even if regular signals are masked, EINTR might be
-		 * propagated from sk_stream_wait_memory() to here when
-		 * TIF_NOTIFY_SIGNAL is used for task work. For example,
-		 * certain io_uring completions will use that. Treat
-		 * having EINTR with pending task work the same as EAGAIN
-		 * to avoid unnecessary reconnects.
+		 * Task work must not abort the send, see signal_pending().
 		 */
-		rc = sock_sendmsg(ssocket, smb_msg);
-		if (rc == -EAGAIN || unlikely(rc == -EINTR && task_work_pending(current))) {
+		scoped_guard(no_notify_signal)
+			rc = sock_sendmsg(ssocket, smb_msg);
+		if (rc == -EAGAIN) {
 			retries++;
 			if (retries >= 14 ||
 			    (!server->noblocksnd && (retries > 2))) {
@@ -805,6 +800,18 @@ cifs_cancelled_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 	release_mid(server, mid);
 }
 
+static void
+cifs_mark_compound_mids_cancelled(struct mid_q_entry **mid, int count)
+{
+	int i;
+
+	for (i = 0; i < count; i++) {
+		spin_lock(&mid[i]->mid_lock);
+		mid[i]->wait_cancelled = true;
+		spin_unlock(&mid[i]->mid_lock);
+	}
+}
+
 /*
  * cifs_pick_channel - pick an eligible channel for network operations
  *
@@ -865,6 +872,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 		   int *resp_buf_type, struct kvec *resp_iov)
 {
 	int i, j, optype, rc = 0;
+	int num_processed = 0;
 	struct mid_q_entry *mid[MAX_COMPOUND];
 	bool cancelled_mid[MAX_COMPOUND] = {false};
 	struct cifs_credits credits[MAX_COMPOUND] = {
@@ -965,6 +973,10 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 	if (rc < 0) {
 		revert_current_mid(server, num_rqst);
 		server->sequence_number -= 2;
+		for (i = 0; i < num_rqst; i++) {
+			delete_mid(server, mid[i]);
+			cancelled_mid[i] = true;
+		}
 	}
 
 	cifs_server_unlock(server);
@@ -1011,6 +1023,14 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 			break;
 	}
 	if (rc != 0) {
+		/*
+		 * A completed CREATE earlier in the compound chain may have
+		 * opened a remote handle even though a later wait was
+		 * interrupted. Mark it cancelled so __release_mid() invokes
+		 * the existing unmatched-open cleanup.
+		 */
+		cifs_mark_compound_mids_cancelled(mid, i);
+
 		for (; i < num_rqst; i++) {
 			cifs_server_dbg(FYI, "Cancelling wait for mid %llu cmd: %d\n",
 				 mid[i]->mid, le16_to_cpu(mid[i]->command));
@@ -1033,6 +1053,14 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 
 		rc = cifs_sync_mid_result(mid[i], server);
 		if (rc != 0) {
+			/*
+			 * A previous CREATE may have completed before this
+			 * response failed. Mark it cancelled so its remote
+			 * handle is closed when the mid is released.
+			 */
+			cifs_mark_compound_mids_cancelled(mid, i);
+			/* Keep their response buffers for cancelled-mid cleanup. */
+			num_processed = 0;
 			/* mark this mid as cancelled to not free it below */
 			cancelled_mid[i] = true;
 			goto out;
@@ -1042,13 +1070,24 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 		    mid[i]->mid_state != MID_RESPONSE_READY) {
 			rc = smb_EIO1(smb_eio_trace_rx_mid_unready, mid[i]->mid_state);
 			cifs_dbg(FYI, "Bad MID state?\n");
+			cifs_mark_compound_mids_cancelled(mid, i);
+			num_processed = 0;
 			goto out;
 		}
 
 		rc = server->ops->check_receive(mid[i], server,
 						flags & CIFS_LOG_ERROR);
+		num_processed = i + 1;
+	}
 
-		if (resp_iov) {
+out:
+	/*
+	 * Delay moving response buffers out of their mids until response
+	 * synchronization completes. This lets cancelled-mid cleanup inspect
+	 * an earlier CREATE response if a later MID fails.
+	 */
+	if (resp_iov) {
+		for (i = 0; i < num_processed; i++) {
 			buf = (char *)mid[i]->resp_buf;
 			resp_iov[i].iov_base = buf;
 			resp_iov[i].iov_len = mid[i]->resp_buf_size;
@@ -1067,21 +1106,22 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 	/*
 	 * Compounding is never used during session establish.
 	 */
-	spin_lock(&ses->ses_lock);
-	if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) {
-		struct kvec iov = {
-			.iov_base = resp_iov[0].iov_base,
-			.iov_len = resp_iov[0].iov_len
-		};
-		spin_unlock(&ses->ses_lock);
-		cifs_server_lock(server);
-		smb311_update_preauth_hash(ses, server, &iov, 1);
-		cifs_server_unlock(server);
+	if (num_processed == num_rqst) {
 		spin_lock(&ses->ses_lock);
+		if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) {
+			struct kvec iov = {
+				.iov_base = resp_iov[0].iov_base,
+				.iov_len = resp_iov[0].iov_len
+			};
+			spin_unlock(&ses->ses_lock);
+			cifs_server_lock(server);
+			smb311_update_preauth_hash(ses, server, &iov, 1);
+			cifs_server_unlock(server);
+			spin_lock(&ses->ses_lock);
+		}
+		spin_unlock(&ses->ses_lock);
 	}
-	spin_unlock(&ses->ses_lock);
 
-out:
 	/*
 	 * This will dequeue all mids. After this it is important that the
 	 * demultiplex_thread will not process any of these mids any further.
diff --git a/fs/smb/client/xattr.c b/fs/smb/client/xattr.c
index 5091f6c0d7fef8..f6c9343016f715 100644
--- a/fs/smb/client/xattr.c
+++ b/fs/smb/client/xattr.c
@@ -91,7 +91,7 @@ static int cifs_creation_time_set(unsigned int xid, struct cifs_tcon *pTcon,
 }
 
 static int cifs_xattr_set(const struct xattr_handler *handler,
-			  struct mnt_idmap *idmap,
+			  const struct mnt_idmap *idmap,
 			  struct dentry *dentry, struct inode *inode,
 			  const char *name, const void *value,
 			  size_t size, int flags)
diff --git a/fs/smb/common/compress/compress.c b/fs/smb/common/compress/compress.c
index a4123c8f1c0ab3..4e5164b6dc3333 100644
--- a/fs/smb/common/compress/compress.c
+++ b/fs/smb/common/compress/compress.c
@@ -130,12 +130,18 @@ static int smb_decompress_chained(__le16 alg, bool allow_chained,
 		len = le32_to_cpu(payload->Length);
 
 		/*
-		 * CHAINED marks only the first payload. Requiring NONE on every
-		 * later payload rejects ambiguous or independently chained data.
+		 * Conforming chains must set CHAINED on the first payload.
+		 * Windows 11 clients leave uninitialized residual bits in
+		 * Flags on subsequent payload headers. Only reject trailing
+		 * payloads that attempt to initiate an invalid nested chain.
 		 */
-		if ((first && flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED)) ||
-		    (!first && flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE)))
-			return -EINVAL;
+		if (first) {
+			if (flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED))
+				return -EINVAL;
+		} else {
+			if (flags == cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED))
+				return -EINVAL;
+		}
 
 		src += SMB2_COMPRESSION_PAYLOAD_BASE_LEN;
 		remaining -= SMB2_COMPRESSION_PAYLOAD_BASE_LEN;
diff --git a/fs/smb/common/fscc.h b/fs/smb/common/fscc.h
index e46d3379b7796d..00f09e1a3afec4 100644
--- a/fs/smb/common/fscc.h
+++ b/fs/smb/common/fscc.h
@@ -399,7 +399,7 @@ static_assert(offsetof(struct smb2_file_rename_info, FileName) == sizeof(struct
 #define FS_OBJECT_ID_INFORMATION	8 /* Query, Set */
 #define FS_DRIVER_PATH_INFORMATION	9 /* Query */
 #define FS_SECTOR_SIZE_INFORMATION	11 /* SMB3 or later. Query */
-/* See POSIX Extensions to MS-FSCC 2.3.1.1 */
+/* See POSIX-FSCC 2.3 */
 #define FS_POSIX_INFORMATION		100 /* SMB3.1.1 POSIX. Query */
 
 /* See MS-FSCC 2.5.1 */
@@ -575,8 +575,7 @@ struct file_notify_information {
 } __packed;
 
 /*
- * See POSIX Extensions to MS-FSCC 2.3.2.1
- * Link: https://gitlab.com/samba-team/smb3-posix-spec/-/blob/master/fscc_posix_extensions.md
+ * See POSIX-FSCC 2.3.1
  */
 typedef struct {
 	/* For undefined recommended transfer size return -1 in that field */
diff --git a/fs/smb/common/smbglob.h b/fs/smb/common/smbglob.h
index d9c7e6e7af295c..fdf8408880620e 100644
--- a/fs/smb/common/smbglob.h
+++ b/fs/smb/common/smbglob.h
@@ -40,6 +40,7 @@ struct smb_version_values {
 	size_t		create_disk_id_size;
 	size_t		create_posix_size;
 	size_t		create_aapl_size;
+	size_t		create_rsp_size;
 };
 
 static inline unsigned int get_rfc1002_len(void *buf)
diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig
index 221ec9717a8341..b7665e0e49423c 100644
--- a/fs/smb/server/Kconfig
+++ b/fs/smb/server/Kconfig
@@ -71,3 +71,5 @@ config SMB_SERVER_KERBEROS5
 	bool "Support for Kerberos 5"
 	depends on SMB_SERVER
 	default y
+
+source "fs/smb/server/tests/Kconfig"
diff --git a/fs/smb/server/Makefile b/fs/smb/server/Makefile
index a3e9306055e8bd..9bc87695a53c76 100644
--- a/fs/smb/server/Makefile
+++ b/fs/smb/server/Makefile
@@ -19,3 +19,4 @@ $(obj)/ksmbd_spnego_negtokentarg.asn1.o: $(obj)/ksmbd_spnego_negtokentarg.asn1.c
 
 ksmbd-$(CONFIG_SMB_SERVER_SMBDIRECT) += transport_rdma.o
 ksmbd-$(CONFIG_PROC_FS) += proc.o
+obj-$(CONFIG_SMB_SERVER_KUNIT_TESTS) += tests/
diff --git a/fs/smb/server/compress.c b/fs/smb/server/compress.c
index 5162fb84c755ab..3fcb3e44302e95 100644
--- a/fs/smb/server/compress.c
+++ b/fs/smb/server/compress.c
@@ -65,6 +65,7 @@ static int __ksmbd_decompress_request(struct ksmbd_conn *conn,
 		return -ENOMEM;
 
 	*(__be32 *)out = cpu_to_be32(out_size);
+	out[out_size + 4] = 0;
 	rc = smb_compression_decompress(conn->compress_algorithm,
 					conn->compress_chained,
 					conn->compress_pattern,
diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c
index d211861ff86f11..d1cc725eeb0626 100644
--- a/fs/smb/server/connection.c
+++ b/fs/smb/server/connection.c
@@ -114,7 +114,7 @@ static int proc_show_clients(struct seq_file *m, void *v)
 			sessions++;
 		rcu_read_unlock();
 #if IS_ENABLED(CONFIG_IPV6)
-		if (!conn->inet_addr)
+		if (conn->is_ipv6)
 			seq_printf(m, "client:\t%pI6c\n", &conn->inet6_addr);
 		else
 #endif
@@ -752,6 +752,7 @@ int ksmbd_conn_handler_loop(void *p)
 		if (!conn->request_buf)
 			break;
 
+		conn->request_buf[pdu_size + 4] = 0;
 		memcpy(conn->request_buf, hdr_buf, sizeof(hdr_buf));
 
 		/*
diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h
index 371f17b4f02a33..2026a2afa5e47d 100644
--- a/fs/smb/server/connection.h
+++ b/fs/smb/server/connection.h
@@ -68,6 +68,9 @@ struct ksmbd_conn {
 		u8			inet6_addr[16];
 #endif
 	};
+#if IS_ENABLED(CONFIG_IPV6)
+	bool				is_ipv6;
+#endif
 	unsigned int			inet_hash;
 	char				*request_buf;
 	struct ksmbd_transport		*transport;
diff --git a/fs/smb/server/ksmbd_work.c b/fs/smb/server/ksmbd_work.c
index d307aefe0aec1e..52ea1a674a67ba 100644
--- a/fs/smb/server/ksmbd_work.c
+++ b/fs/smb/server/ksmbd_work.c
@@ -57,7 +57,6 @@ struct ksmbd_work *ksmbd_alloc_work_struct(void)
 		INIT_LIST_HEAD(&work->request_entry);
 		INIT_LIST_HEAD(&work->async_request_entry);
 		INIT_LIST_HEAD(&work->fp_entry);
-		INIT_LIST_HEAD(&work->notify_entry);
 		INIT_LIST_HEAD(&work->aux_read_list);
 		work->iov_alloc_cnt = ARRAY_SIZE(work->iov_inline);
 		work->iov = work->iov_inline;
@@ -87,8 +86,6 @@ void ksmbd_free_work_struct(struct ksmbd_work *work)
 
 	if (work->async_id)
 		ksmbd_release_id(&work->conn->async_ida, work->async_id);
-	if (work->owns_conn_ref)
-		ksmbd_conn_put(work->conn);
 	ksmbd_fd_put(work, work->request_open);
 	kmem_cache_free(work_cache, work);
 }
diff --git a/fs/smb/server/ksmbd_work.h b/fs/smb/server/ksmbd_work.h
index 0844aa929f55da..3a14e4d69aa1ef 100644
--- a/fs/smb/server/ksmbd_work.h
+++ b/fs/smb/server/ksmbd_work.h
@@ -91,8 +91,6 @@ struct ksmbd_work {
 	bool                            compress_response:1;
 	/* Is this SYNC or ASYNC ksmbd_work */
 	bool                            asynchronous:1;
-	/* Work owns a reference to @conn. */
-	bool				owns_conn_ref:1;
 	bool                            need_invalidate_rkey:1;
 	bool				request_open_chseq_tracked:1;
 	bool				session_setup_reauth:1;
@@ -115,9 +113,8 @@ struct ksmbd_work {
 	struct list_head                request_entry;
 	/* List head at conn->async_requests */
 	struct list_head                async_request_entry;
+	/* List head at ksmbd_file->blocked_works */
 	struct list_head                fp_entry;
-	/* List head at ksmbd_file->notify_pendings */
-	struct list_head                notify_entry;
 };
 
 /**
diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c
index 44dc3f800cd49c..1321cd6c84e7ef 100644
--- a/fs/smb/server/mgmt/user_session.c
+++ b/fs/smb/server/mgmt/user_session.c
@@ -130,7 +130,7 @@ static int show_proc_session(struct seq_file *m, void *v)
 		const char *name;
 
 #if IS_ENABLED(CONFIG_IPV6)
-		if (chan->conn->inet_addr)
+		if (!chan->conn->is_ipv6)
 			seq_printf(m, "client:\t%pI4\n",
 					&chan->conn->inet_addr);
 		else
@@ -231,7 +231,7 @@ static int show_proc_sessions(struct seq_file *m, void *v)
 			ksmbd_user_session_get(session);
 
 #if IS_ENABLED(CONFIG_IPV6)
-			if (!chan->conn->inet_addr)
+			if (chan->conn->is_ipv6)
 				seq_printf(m, "client:\t%pI6c\n", &chan->conn->inet6_addr);
 			else
 #endif
diff --git a/fs/smb/server/ndr.c b/fs/smb/server/ndr.c
index 58d71560f626b4..7e546c22e284b9 100644
--- a/fs/smb/server/ndr.c
+++ b/fs/smb/server/ndr.c
@@ -338,7 +338,7 @@ static int ndr_encode_posix_acl_entry(struct ndr *n, struct xattr_smb_acl *acl)
 }
 
 int ndr_encode_posix_acl(struct ndr *n,
-			 struct mnt_idmap *idmap,
+			 const struct mnt_idmap *idmap,
 			 struct inode *inode,
 			 struct xattr_smb_acl *acl,
 			 struct xattr_smb_acl *def_acl)
diff --git a/fs/smb/server/ndr.h b/fs/smb/server/ndr.h
index f3c108c8cf4dbf..646568c42e4d13 100644
--- a/fs/smb/server/ndr.h
+++ b/fs/smb/server/ndr.h
@@ -14,7 +14,7 @@ struct ndr {
 
 int ndr_encode_dos_attr(struct ndr *n, struct xattr_dos_attrib *da);
 int ndr_decode_dos_attr(struct ndr *n, struct xattr_dos_attrib *da);
-int ndr_encode_posix_acl(struct ndr *n, struct mnt_idmap *idmap,
+int ndr_encode_posix_acl(struct ndr *n, const struct mnt_idmap *idmap,
 			 struct inode *inode, struct xattr_smb_acl *acl,
 			 struct xattr_smb_acl *def_acl);
 int ndr_encode_v4_ntacl(struct ndr *n, struct xattr_ntacl *acl);
diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c
index 1b8c3482d1e4e8..d0f18ebf471b08 100644
--- a/fs/smb/server/oplock.c
+++ b/fs/smb/server/oplock.c
@@ -2270,7 +2270,7 @@ void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp)
 {
 	struct create_posix_rsp *buf;
 	struct inode *inode = file_inode(fp->filp);
-	struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
 	vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
 	vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode);
 
diff --git a/fs/smb/server/smb2ops.c b/fs/smb/server/smb2ops.c
index 4578291fb1722d..747276b171bd14 100644
--- a/fs/smb/server/smb2ops.c
+++ b/fs/smb/server/smb2ops.c
@@ -13,6 +13,35 @@
 #include "server.h"
 #include "stats.h"
 
+/*
+ * work->response_sz includes the RFC1002 length field while smb_get_msg()
+ * skips over it. Durable v1 and v2 response contexts are mutually exclusive,
+ * and POSIX CREATE contexts are only negotiated for SMB3.1.1.
+ */
+#define SMB2_CREATE_RSP_SIZE(lease_size, durable_size)			\
+	(sizeof(__be32) + offsetof(struct smb2_create_rsp, Buffer) +	\
+	 (lease_size) + (durable_size) +				\
+	 sizeof(struct create_mxac_rsp) +				\
+	 sizeof(struct create_disk_id_rsp) +				\
+	 AAPL_RSP_MAX_SIZE)
+
+#define SMB21_CREATE_RSP_SIZE						\
+	SMB2_CREATE_RSP_SIZE(sizeof(struct create_lease),		\
+			     sizeof(struct create_durable_rsp))
+
+#define SMB3_CREATE_DURABLE_RSP_SIZE					\
+	((sizeof(struct create_durable_rsp) >				\
+	  sizeof(struct create_durable_rsp_v2)) ?			\
+	 sizeof(struct create_durable_rsp) :				\
+	 sizeof(struct create_durable_rsp_v2))
+
+#define SMB3_CREATE_RSP_SIZE						\
+	SMB2_CREATE_RSP_SIZE(sizeof(struct create_lease_v2),		\
+			     SMB3_CREATE_DURABLE_RSP_SIZE)
+
+#define SMB311_CREATE_RSP_SIZE						\
+	(SMB3_CREATE_RSP_SIZE + sizeof(struct create_posix_rsp))
+
 static struct smb_version_values smb21_server_values = {
 	.version_string = SMB21_VERSION_STRING,
 	.protocol_id = SMB21_PROT_ID,
@@ -38,6 +67,7 @@ static struct smb_version_values smb21_server_values = {
 	.create_disk_id_size = sizeof(struct create_disk_id_rsp),
 	.create_posix_size = sizeof(struct create_posix_rsp),
 	.create_aapl_size = AAPL_RSP_MAX_SIZE,
+	.create_rsp_size = SMB21_CREATE_RSP_SIZE,
 };
 
 static struct smb_version_values smb30_server_values = {
@@ -66,6 +96,7 @@ static struct smb_version_values smb30_server_values = {
 	.create_disk_id_size = sizeof(struct create_disk_id_rsp),
 	.create_posix_size = sizeof(struct create_posix_rsp),
 	.create_aapl_size = AAPL_RSP_MAX_SIZE,
+	.create_rsp_size = SMB3_CREATE_RSP_SIZE,
 };
 
 static struct smb_version_values smb302_server_values = {
@@ -94,6 +125,7 @@ static struct smb_version_values smb302_server_values = {
 	.create_disk_id_size = sizeof(struct create_disk_id_rsp),
 	.create_posix_size = sizeof(struct create_posix_rsp),
 	.create_aapl_size = AAPL_RSP_MAX_SIZE,
+	.create_rsp_size = SMB3_CREATE_RSP_SIZE,
 };
 
 static struct smb_version_values smb311_server_values = {
@@ -122,6 +154,7 @@ static struct smb_version_values smb311_server_values = {
 	.create_disk_id_size = sizeof(struct create_disk_id_rsp),
 	.create_posix_size = sizeof(struct create_posix_rsp),
 	.create_aapl_size = AAPL_RSP_MAX_SIZE,
+	.create_rsp_size = SMB311_CREATE_RSP_SIZE,
 };
 
 static struct smb_version_ops smb2_0_server_ops = {
diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c
index 6b8809f67b9260..45dce9c30b6b20 100644
--- a/fs/smb/server/smb2pdu.c
+++ b/fs/smb/server/smb2pdu.c
@@ -16,6 +16,7 @@
 #include <linux/mount.h>
 #include <linux/filelock.h>
 #include <linux/fileattr.h>
+#include <linux/math.h>
 #include <linux/timekeeping.h>
 #include <linux/unaligned.h>
 
@@ -58,10 +59,6 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp)
 	}
 }
 
-static struct ksmbd_work *smb2_notify_cancel_claim(void **argv);
-static void smb2_notify_cancel_fn(void **argv);
-static void smb2_complete_notify_cancel(struct ksmbd_work *in_work);
-
 #define WORK_BUFFERS(w, rq, rs)	__wbuf((w), (void **)&(rq), (void **)&(rs))
 
 #define SMB2_CREATE_FILE_ATTRIBUTE_MASK \
@@ -858,14 +855,18 @@ static void smb2_update_lock_sequence(struct ksmbd_work *work,
 int smb2_allocate_rsp_buf(struct ksmbd_work *work)
 {
 	struct smb2_hdr *hdr = smb_get_msg(work->request_buf);
+	struct smb_version_values *vals = work->conn->vals;
 	size_t small_sz = MAX_CIFS_SMALL_BUFFER_SIZE;
-	size_t large_sz = small_sz + work->conn->vals->max_trans_size;
+	size_t large_sz = small_sz + vals->max_trans_size;
 	size_t sz = small_sz;
 	int cmd = le16_to_cpu(hdr->Command);
 
 	if (cmd == SMB2_IOCTL_HE || cmd == SMB2_QUERY_DIRECTORY_HE)
 		sz = large_sz;
 
+	if (cmd == SMB2_CREATE_HE)
+		sz = max_t(size_t, sz, vals->create_rsp_size);
+
 	if (cmd == SMB2_QUERY_INFO_HE) {
 		struct smb2_query_info_req *req;
 
@@ -1246,6 +1247,7 @@ void smb2_send_interim_resp(struct ksmbd_work *work, __le32 status)
 {
 	struct smb2_hdr *rsp_hdr;
 	struct ksmbd_work *in_work = ksmbd_alloc_work_struct();
+	u16 command;
 
 	if (!in_work)
 		return;
@@ -1269,6 +1271,23 @@ void smb2_send_interim_resp(struct ksmbd_work *work, __le32 status)
 	smb2_set_err_rsp(in_work);
 	rsp_hdr->Status = status;
 
+	/*
+	 * Async interim responses are unsigned, but final responses must
+	 * follow the normal signing rules. The synthetic work has no
+	 * request buffer, so use the original work for request signing
+	 * checks and the response header for SMB3 command selection.
+	 */
+	command = work->conn->ops->get_cmd_val(work);
+	if (status != STATUS_PENDING && !work->encrypted && work->sess &&
+	    work->conn->ops->set_sign_rsp &&
+	    (work->sess->sign ||
+	     (work->conn->ops->is_sign_req &&
+	      work->conn->ops->is_sign_req(work, command)))) {
+		in_work->sess = work->sess;
+		work->conn->ops->set_sign_rsp(in_work);
+		in_work->sess = NULL;
+	}
+
 	if (smb2_send_interim_work(in_work, work, true))
 		ksmbd_debug(SMB, "failed to send interim response\n");
 	ksmbd_free_work_struct(in_work);
@@ -3278,7 +3297,7 @@ static bool smb2_is_private_ea(const char *name, size_t name_len)
 static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len,
 		       const struct path *path, bool get_write)
 {
-	struct mnt_idmap *idmap = mnt_idmap(path->mnt);
+	const struct mnt_idmap *idmap = mnt_idmap(path->mnt);
 	char *attr_name = NULL, *value;
 	int rc = 0;
 	unsigned int next = 0;
@@ -3379,7 +3398,7 @@ static noinline int smb2_set_stream_name_xattr(const struct path *path,
 					       struct ksmbd_file *fp,
 					       char *stream_name, int s_type)
 {
-	struct mnt_idmap *idmap = mnt_idmap(path->mnt);
+	const struct mnt_idmap *idmap = mnt_idmap(path->mnt);
 	size_t xattr_stream_size;
 	char *xattr_stream_name;
 	int rc;
@@ -3419,8 +3438,9 @@ static noinline int smb2_set_stream_name_xattr(const struct path *path,
 			 * AAPL there too.
 			 */
 			static const u8 afpinfo_empty[60] = {
-				0x00, 0x05, 0x16, 0x07, /* magic  0x00051607 BE */
-				0x00, 0x02, 0x00, 0x00, /* version 0x00020000 BE */
+				'A', 'F', 'P', 0x00,	/* signature */
+				0x00, 0x00, 0x01, 0x00,	/* version */
+				[15] = 0x80,		/* backup time */
 			};
 			rc = ksmbd_vfs_setxattr(idmap, path, xattr_stream_name,
 						(void *)afpinfo_empty,
@@ -3455,7 +3475,7 @@ static loff_t ksmbd_stream_eof(struct ksmbd_file *fp)
 
 static int smb2_remove_smb_xattrs(const struct path *path)
 {
-	struct mnt_idmap *idmap = mnt_idmap(path->mnt);
+	const struct mnt_idmap *idmap = mnt_idmap(path->mnt);
 	char *name, *xattr_list = NULL;
 	ssize_t xattr_list_len;
 	int err = 0;
@@ -3647,10 +3667,11 @@ static int smb2_create_sd_buffer(struct ksmbd_work *work,
 			    le32_to_cpu(sd_buf->ccontext.DataLength), true, false);
 }
 
-static void ksmbd_acls_fattr(struct smb_fattr *fattr,
-			     struct mnt_idmap *idmap,
-			     struct inode *inode)
+static int ksmbd_acls_fattr(struct smb_fattr *fattr,
+			    const struct mnt_idmap *idmap,
+			    struct inode *inode)
 {
+	struct posix_acl *acl;
 	vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
 	vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode);
 
@@ -3661,10 +3682,28 @@ static void ksmbd_acls_fattr(struct smb_fattr *fattr,
 	fattr->cf_dacls = NULL;
 
 	if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) {
-		fattr->cf_acls = get_inode_acl(inode, ACL_TYPE_ACCESS);
-		if (S_ISDIR(inode->i_mode))
-			fattr->cf_dacls = get_inode_acl(inode, ACL_TYPE_DEFAULT);
+		acl = get_inode_acl(inode, ACL_TYPE_ACCESS);
+		if (IS_ERR(acl)) {
+			if (acl != ERR_PTR(-EOPNOTSUPP))
+				return PTR_ERR(acl);
+			acl = NULL;
+		}
+		fattr->cf_acls = acl;
+
+		if (S_ISDIR(inode->i_mode)) {
+			acl = get_inode_acl(inode, ACL_TYPE_DEFAULT);
+			if (IS_ERR(acl)) {
+				if (acl != ERR_PTR(-EOPNOTSUPP)) {
+					posix_acl_release(fattr->cf_acls);
+					return PTR_ERR(acl);
+				}
+				acl = NULL;
+			}
+			fattr->cf_dacls = acl;
+		}
 	}
+
+	return 0;
 }
 
 enum {
@@ -4132,7 +4171,7 @@ int smb2_open(struct ksmbd_work *work)
 	struct ksmbd_share_config *share = tcon->share_conf;
 	struct ksmbd_file *fp = NULL;
 	struct file *filp = NULL;
-	struct mnt_idmap *idmap = NULL;
+	const struct mnt_idmap *idmap = NULL;
 	struct kstat stat;
 	struct create_context *context;
 	struct lease_ctx_info *lc = NULL;
@@ -4791,7 +4830,10 @@ int smb2_open(struct ksmbd_work *work)
 					int pntsd_size;
 					size_t scratch_len;
 
-					ksmbd_acls_fattr(&fattr, idmap, inode);
+					rc = ksmbd_acls_fattr(&fattr, idmap, inode);
+					if (rc)
+						goto err_out;
+
 					scratch_len = smb_acl_sec_desc_scratch_len(&fattr,
 							NULL, 0,
 							OWNER_SECINFO | GROUP_SECINFO |
@@ -5797,7 +5839,7 @@ struct smb2_query_dir_private {
 
 static int process_query_dir_entries(struct smb2_query_dir_private *priv)
 {
-	struct mnt_idmap	*idmap = file_mnt_idmap(priv->dir_fp->filp);
+	const struct mnt_idmap	*idmap = file_mnt_idmap(priv->dir_fp->filp);
 	struct kstat		kstat;
 	struct ksmbd_kstat	ksmbd_kstat;
 	int			rc;
@@ -6390,7 +6432,7 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp,
 	ssize_t buf_free_len, alignment_bytes, next_offset, rsp_data_cnt = 0;
 	struct smb2_ea_info_req *ea_req = NULL;
 	const struct path *path;
-	struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
 
 	if (!(fp->daccess & FILE_READ_EA_LE)) {
 		pr_err("Not permitted to read ext attr : 0x%x\n",
@@ -6741,12 +6783,12 @@ static void get_file_alternate_info(struct ksmbd_work *work,
 				    void *rsp_org)
 {
 	struct ksmbd_conn *conn = work->conn;
-	struct smb2_file_alt_name_info *file_info;
+	struct smb2_file_name_info *file_info;
 	struct dentry *dentry = fp->filp->f_path.dentry;
 	int conv_len;
 
 	spin_lock(&dentry->d_lock);
-	file_info = (struct smb2_file_alt_name_info *)rsp->Buffer;
+	file_info = (struct smb2_file_name_info *)rsp->Buffer;
 	conv_len = ksmbd_extract_shortname(conn,
 					   dentry->d_name.name,
 					   file_info->FileName);
@@ -6793,7 +6835,7 @@ static int get_file_normalized_name_info(struct ksmbd_work *work,
 					 struct smb2_query_info_rsp *rsp,
 					 struct ksmbd_file *fp)
 {
-	struct smb2_file_alt_name_info *file_info;
+	struct smb2_file_name_info *file_info;
 	char *filename, *normalized, *stream_name;
 	int buf_free_len, conv_len, filename_len;
 
@@ -6827,7 +6869,7 @@ static int get_file_normalized_name_info(struct ksmbd_work *work,
 		return -EINVAL;
 	}
 
-	file_info = (struct smb2_file_alt_name_info *)rsp->Buffer;
+	file_info = (struct smb2_file_name_info *)rsp->Buffer;
 	conv_len = smbConvertToUTF16((__le16 *)file_info->FileName,
 				     normalized, filename_len,
 				     work->conn->local_nls, 0);
@@ -6912,17 +6954,8 @@ static int get_file_stream_info(struct ksmbd_work *work,
 		streamlen *= 2;
 		kfree(stream_buf);
 		file_info->StreamNameLength = cpu_to_le32(streamlen);
-		/*
-		 * stream_name_len is the byte length of the xattr's *name*,
-		 * not its value -- same class of bug ksmbd_stream_eof()
-		 * (smb2pdu.c) already fixes for EndOfFile/AllocationSize on
-		 * a stream handle; this enumeration path needs the same
-		 * real xattr value length, not the name length reused as a
-		 * size.
-		 */
-		slen = ksmbd_vfs_casexattr_len(file_mnt_idmap(fp->filp),
-						path->dentry, stream_name,
-						strlen(stream_name) + 1);
+		slen = ksmbd_vfs_xattr_len(file_mnt_idmap(fp->filp),
+					   path->dentry, stream_name);
 		ssize = slen < 0 ? 0 : (loff_t)slen;
 		file_info->StreamSize = cpu_to_le64(ssize);
 		file_info->StreamAllocationSize = cpu_to_le64(ssize);
@@ -7108,7 +7141,7 @@ static int find_file_posix_info(struct smb2_query_info_rsp *rsp,
 {
 	struct smb311_posix_qinfo *file_info;
 	struct inode *inode = file_inode(fp->filp);
-	struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
 	vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
 	vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode);
 	struct kstat stat;
@@ -7190,7 +7223,7 @@ static int smb2_get_info_file(struct ksmbd_work *work,
 	struct ksmbd_file *fp;
 	int fileinfoclass = 0;
 	int rc = 0;
-	unsigned int fixed_len;
+	unsigned int fixed_len, req_output_len;
 	unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID;
 
 	if (test_share_config_flag(work->tcon->share_conf,
@@ -7297,24 +7330,24 @@ static int smb2_get_info_file(struct ksmbd_work *work,
 		rc = -EOPNOTSUPP;
 	}
 	if (!rc) {
+		req_output_len = le32_to_cpu(req->OutputBufferLength);
 		fixed_len = le32_to_cpu(rsp->OutputBufferLength);
 		switch (fileinfoclass) {
 		case FILE_ALL_INFORMATION:
 			fixed_len = FILE_ALL_INFORMATION_SIZE;
 			break;
 		case FILE_ALTERNATE_NAME_INFORMATION:
-			fixed_len = FILE_ALTERNATE_NAME_INFORMATION_SIZE;
+			fixed_len = FILE_NAME_INFORMATION_SIZE;
 			break;
 		case FILE_NORMALIZED_NAME_INFORMATION:
-			fixed_len = FILE_NORMALIZED_NAME_INFORMATION_SIZE;
+			fixed_len = FILE_NAME_INFORMATION_SIZE;
+			req_output_len = round_down(req_output_len, 2);
 			break;
 		case FILE_STREAM_INFORMATION:
 			fixed_len = FILE_STREAM_INFORMATION_SIZE;
 			break;
 		}
-		rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength),
-				      fixed_len,
-				      rsp);
+		rc = buffer_check_err(req_output_len, fixed_len, rsp);
 	}
 	ksmbd_fd_put(work, fp);
 
@@ -7600,7 +7633,7 @@ static int smb2_get_info_sec(struct ksmbd_work *work,
 			     struct smb2_query_info_rsp *rsp)
 {
 	struct ksmbd_file *fp;
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct smb_ntsd *pntsd = NULL, *ppntsd = NULL;
 	struct smb_fattr fattr = {{0}};
 	struct inode *inode;
@@ -7652,7 +7685,11 @@ static int smb2_get_info_sec(struct ksmbd_work *work,
 
 	idmap = file_mnt_idmap(fp->filp);
 	inode = file_inode(fp->filp);
-	ksmbd_acls_fattr(&fattr, idmap, inode);
+	rc = ksmbd_acls_fattr(&fattr, idmap, inode);
+	if (rc) {
+		ksmbd_fd_put(work, fp);
+		return rc;
+	}
 
 	if (test_share_config_flag(work->tcon->share_conf,
 				   KSMBD_SHARE_FLAG_ACL_XATTR))
@@ -7982,9 +8019,11 @@ static int smb2_rename(struct ksmbd_work *work,
 		return PTR_ERR(new_name);
 
 	if (fp->is_posix_ctxt == false && strchr(new_name, ':')) {
-		int s_type;
+		int s_type = 0;
 		char *xattr_stream_name, *stream_name = NULL;
+		char *stream_buf = NULL;
 		size_t xattr_stream_size;
+		ssize_t stream_buf_len = 0;
 		int len;
 
 		rc = parse_stream_name(new_name, &stream_name, &s_type);
@@ -7998,6 +8037,10 @@ static int smb2_rename(struct ksmbd_work *work,
 			goto out;
 		}
 
+		/* An empty stream name is the base file's default stream. */
+		if (!stream_name || !stream_name[0])
+			goto out;
+
 		rc = ksmbd_vfs_xattr_stream_name(stream_name,
 						 &xattr_stream_name,
 						 &xattr_stream_size,
@@ -8005,15 +8048,34 @@ static int smb2_rename(struct ksmbd_work *work,
 		if (rc)
 			goto out;
 
+		/* A handle opened without a stream has no source to copy. */
+		if (ksmbd_stream_fd(fp)) {
+			if (!strcasecmp(xattr_stream_name, fp->stream.name)) {
+				kfree(xattr_stream_name);
+				goto out;
+			}
+
+			stream_buf_len = ksmbd_vfs_getcasexattr(file_mnt_idmap(fp->filp),
+								fp->filp->f_path.dentry,
+								fp->stream.name,
+								fp->stream.size,
+								&stream_buf);
+			if (stream_buf_len < 0) {
+				rc = stream_buf_len;
+				kfree(xattr_stream_name);
+				goto out;
+			}
+		}
+
 		rc = ksmbd_vfs_setxattr(file_mnt_idmap(fp->filp),
 					&fp->filp->f_path,
 					xattr_stream_name,
-					NULL, 0, 0, true);
-		if (rc < 0) {
+					stream_buf, stream_buf_len, 0, true);
+		kfree(stream_buf);
+		if (rc < 0)
 			pr_err("failed to store stream name in xattr: %d\n",
 			       rc);
-			rc = -EINVAL;
-		}
+
 		kfree(xattr_stream_name);
 		goto out;
 	}
@@ -8113,7 +8175,7 @@ static int set_file_basic_info(struct ksmbd_file *fp,
 	struct iattr attrs;
 	struct file *filp;
 	struct inode *inode;
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	__le32 attrs_mask = FILE_ATTRIBUTE_DIRECTORY_LE |
 		FILE_ATTRIBUTE_COMPRESSED_LE;
 	int rc = 0;
@@ -9276,23 +9338,25 @@ static noinline int smb2_write_pipe(struct ksmbd_work *work)
 	int err = 0, ret = 0;
 	char *data_buf;
 	size_t length;
+	unsigned int data_offset, req_len;
 
 	WORK_BUFFERS(work, req, rsp);
 
 	length = le32_to_cpu(req->Length);
 	id = req->VolatileFileId;
+	data_offset = le16_to_cpu(req->DataOffset);
+	req_len = smb2_current_req_len(work, &req->hdr);
 
-	if ((u64)le16_to_cpu(req->DataOffset) + length >
-	    get_rfc1002_len(work->request_buf)) {
-		pr_err("invalid write data offset %u, smb_len %u\n",
-		       le16_to_cpu(req->DataOffset),
-		       get_rfc1002_len(work->request_buf));
+	if (data_offset < offsetof(struct smb2_write_req, Buffer) ||
+	    data_offset > req_len || length > req_len - data_offset) {
+		pr_err("invalid write data offset %u, length %zu, req_len %u\n",
+		       data_offset, length, req_len);
 		err = -EINVAL;
 		goto out;
 	}
 
 	data_buf = (char *)(((char *)&req->hdr.ProtocolId) +
-			   le16_to_cpu(req->DataOffset));
+			   data_offset);
 
 	rpc_resp = ksmbd_rpc_write(work->sess, id, data_buf, length);
 	if (rpc_resp) {
@@ -9579,14 +9643,21 @@ int smb2_write(struct ksmbd_work *work)
 		writethrough = true;
 
 	if (is_rdma_channel == false) {
-		if (le16_to_cpu(req->DataOffset) <
-		    offsetof(struct smb2_write_req, Buffer)) {
+		unsigned int data_offset = le16_to_cpu(req->DataOffset);
+		unsigned int req_len = smb2_current_req_len(work, &req->hdr);
+
+		if (data_offset < offsetof(struct smb2_write_req, Buffer) ||
+		    data_offset > req_len ||
+		    length > req_len - data_offset) {
+			ksmbd_debug(SMB,
+				    "invalid write data offset %u, length %zu, req_len %u\n",
+				    data_offset, length, req_len);
 			err = -EINVAL;
 			goto out;
 		}
 
 		data_buf = (char *)(((char *)&req->hdr.ProtocolId) +
-				    le16_to_cpu(req->DataOffset));
+				    data_offset);
 
 		ksmbd_debug(SMB, "filename %pD, offset %lld, len %zu\n",
 			    fp->filp, offset, length);
@@ -9708,7 +9779,6 @@ int smb2_cancel(struct ksmbd_work *work)
 	struct smb2_hdr *hdr = smb_get_msg(work->request_buf);
 	struct smb2_hdr *chdr;
 	struct ksmbd_work *iter;
-	struct ksmbd_work *cancelled_notify = NULL;
 	struct list_head *command_list;
 
 	if (work->next_smb2_rcv_hdr_off)
@@ -9746,23 +9816,11 @@ int smb2_cancel(struct ksmbd_work *work)
 				    "smb2 with AsyncId %llu cancelled command = 0x%x\n",
 				    le64_to_cpu(hdr->Id.AsyncId),
 				    le16_to_cpu(chdr->Command));
-			if (iter->cancel_fn == smb2_notify_cancel_fn)
-				cancelled_notify =
-					smb2_notify_cancel_claim(iter->cancel_argv);
-			else if (iter->cancel_fn)
+			if (iter->cancel_fn)
 				iter->cancel_fn(iter->cancel_argv);
 			break;
 		}
 		spin_unlock(&conn->request_lock);
-
-		/*
-		 * Complete a cancelled notify before this CANCEL handler returns.
-		 * Deferring it to the system workqueue lets a following request and
-		 * its response overtake STATUS_CANCELLED, leaving clients waiting
-		 * for the original notify even though the cancellation was accepted.
-		 */
-		if (cancelled_notify)
-			smb2_complete_notify_cancel(cancelled_notify);
 	} else {
 		command_list = &conn->requests;
 
@@ -10466,6 +10524,27 @@ static __be32 idev_ipv4_address(struct in_device *idev)
 	return addr;
 }
 
+static struct network_interface_info_ioctl_rsp *
+ksmbd_iface_entry_init(struct smb2_ioctl_rsp *rsp, int nbytes,
+		       struct net_device *netdev, unsigned long long speed)
+{
+	struct network_interface_info_ioctl_rsp *nii_rsp;
+
+	nii_rsp = (struct network_interface_info_ioctl_rsp *)&rsp->Buffer[nbytes];
+	nii_rsp->IfIndex = cpu_to_le32(netdev->ifindex);
+	nii_rsp->Capability = 0;
+	if (netdev->real_num_tx_queues > 1)
+		nii_rsp->Capability |= RSS_CAPABLE;
+	if (ksmbd_rdma_capable_netdev(netdev))
+		nii_rsp->Capability |= RDMA_CAPABLE;
+	nii_rsp->Next = cpu_to_le32(152);
+	nii_rsp->Reserved = 0;
+	nii_rsp->LinkSpeed = cpu_to_le64(speed);
+	memset(nii_rsp->SockAddr_Storage, 0, 128);
+
+	return nii_rsp;
+}
+
 static int fsctl_query_iface_info_ioctl(struct ksmbd_conn *conn,
 					struct smb2_ioctl_rsp *rsp,
 					unsigned int out_buf_len)
@@ -10476,10 +10555,16 @@ static int fsctl_query_iface_info_ioctl(struct ksmbd_conn *conn,
 	struct sockaddr_storage_rsp *sockaddr_storage;
 	unsigned int flags;
 	unsigned long long speed;
+	struct ethtool_link_ksettings cmd;
 
 	rtnl_lock();
 	for_each_netdev(&init_net, netdev) {
-		bool ipv4_set = false;
+		struct inet6_ifaddr *ifa;
+		struct inet6_dev *idev6;
+		struct in_device *idev;
+		struct in6_addr ip6 = { };
+		bool have_ip6 = false;
+		__be32 ip4 = 0;
 
 		if (netdev->type == ARPHRD_LOOPBACK)
 			continue;
@@ -10490,87 +10575,80 @@ static int fsctl_query_iface_info_ioctl(struct ksmbd_conn *conn,
 		flags = netif_get_flags(netdev);
 		if (!(flags & IFF_RUNNING))
 			continue;
-ipv6_retry:
-		if (out_buf_len <
-		    nbytes + sizeof(struct network_interface_info_ioctl_rsp)) {
-			rtnl_unlock();
-			return -ENOSPC;
-		}
-
-		nii_rsp = (struct network_interface_info_ioctl_rsp *)
-				&rsp->Buffer[nbytes];
-		nii_rsp->IfIndex = cpu_to_le32(netdev->ifindex);
-
-		nii_rsp->Capability = 0;
-		if (netdev->real_num_tx_queues > 1)
-			nii_rsp->Capability |= RSS_CAPABLE;
-		if (ksmbd_rdma_capable_netdev(netdev))
-			nii_rsp->Capability |= RDMA_CAPABLE;
 
-		nii_rsp->Next = cpu_to_le32(152);
-		nii_rsp->Reserved = 0;
-
-		if (netdev->ethtool_ops->get_link_ksettings) {
-			struct ethtool_link_ksettings cmd;
-
-			netdev->ethtool_ops->get_link_ksettings(netdev, &cmd);
+		if (!__ethtool_get_link_ksettings(netdev, &cmd) &&
+		    cmd.base.speed && cmd.base.speed != SPEED_UNKNOWN) {
 			speed = cmd.base.speed;
 		} else {
 			ksmbd_debug(SMB, "%s %s\n", netdev->name,
 				    "speed is unknown, defaulting to 1Gb/sec");
 			speed = SPEED_1000;
 		}
-
 		speed *= 1000000;
-		nii_rsp->LinkSpeed = cpu_to_le64(speed);
 
-		sockaddr_storage = (struct sockaddr_storage_rsp *)
-					nii_rsp->SockAddr_Storage;
-		memset(sockaddr_storage, 0, 128);
+		/*
+		 * Query IPv4 and IPv6 independently; emit an entry only when a
+		 * usable address exists, so an interface missing one family is
+		 * still reported for the other and 0.0.0.0 / :: placeholders are
+		 * never advertised.
+		 */
+		idev = __in_dev_get_rtnl(netdev);
+		if (idev)
+			ip4 = idev_ipv4_address(idev);
+
+		idev6 = __in6_dev_get(netdev);
+		if (idev6) {
+			rcu_read_lock();
+			list_for_each_entry_rcu(ifa, &idev6->addr_list, if_list) {
+				if (ifa->flags & (IFA_F_TENTATIVE | IFA_F_DEPRECATED))
+					continue;
+				memcpy(&ip6, ifa->addr.s6_addr, sizeof(ip6));
+				have_ip6 = true;
+				break;
+			}
+			rcu_read_unlock();
+		}
 
-		if (!ipv4_set) {
-			struct in_device *idev;
+		if (ip4) {
+			if (out_buf_len <
+			    nbytes + sizeof(struct network_interface_info_ioctl_rsp)) {
+				rtnl_unlock();
+				return -ENOSPC;
+			}
 
+			nii_rsp = ksmbd_iface_entry_init(rsp, nbytes, netdev, speed);
+			sockaddr_storage = (struct sockaddr_storage_rsp *)nii_rsp->SockAddr_Storage;
 			sockaddr_storage->Family = INTERNETWORK;
 			sockaddr_storage->addr4.Port = 0;
-
-			idev = __in_dev_get_rtnl(netdev);
-			if (!idev)
-				continue;
-			sockaddr_storage->addr4.IPv4Address =
-						idev_ipv4_address(idev);
+			sockaddr_storage->addr4.IPv4Address = ip4;
 			nbytes += sizeof(struct network_interface_info_ioctl_rsp);
-			ipv4_set = true;
-			goto ipv6_retry;
-		} else {
-			struct inet6_dev *idev6;
-			struct inet6_ifaddr *ifa;
-			__u8 *ipv6_addr = sockaddr_storage->addr6.IPv6Address;
+		}
 
+		if (have_ip6) {
+			if (out_buf_len <
+			    nbytes + sizeof(struct network_interface_info_ioctl_rsp)) {
+				rtnl_unlock();
+				return -ENOSPC;
+			}
+
+			nii_rsp = ksmbd_iface_entry_init(rsp, nbytes, netdev, speed);
+			sockaddr_storage = (struct sockaddr_storage_rsp *)nii_rsp->SockAddr_Storage;
 			sockaddr_storage->Family = INTERNETWORKV6;
 			sockaddr_storage->addr6.Port = 0;
 			sockaddr_storage->addr6.FlowInfo = 0;
-
-			idev6 = __in6_dev_get(netdev);
-			if (!idev6)
-				continue;
-
-			list_for_each_entry(ifa, &idev6->addr_list, if_list) {
-				if (ifa->flags & (IFA_F_TENTATIVE |
-							IFA_F_DEPRECATED))
-					continue;
-				memcpy(ipv6_addr, ifa->addr.s6_addr, 16);
-				break;
-			}
+			memcpy(sockaddr_storage->addr6.IPv6Address, ip6.s6_addr, 16);
 			sockaddr_storage->addr6.ScopeId = 0;
 			nbytes += sizeof(struct network_interface_info_ioctl_rsp);
 		}
 	}
 	rtnl_unlock();
 
-	/* zero if this is last one */
-	if (nii_rsp)
+	/* Clear Next of the last committed entry to terminate the list. */
+	if (nbytes > 0) {
+		nii_rsp = (struct network_interface_info_ioctl_rsp *)
+			  &rsp->Buffer[nbytes - sizeof(*nii_rsp)];
 		nii_rsp->Next = 0;
+	}
 
 	rsp->PersistentFileId = SMB2_NO_FID;
 	rsp->VolatileFileId = SMB2_NO_FID;
@@ -10715,7 +10793,7 @@ static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id,
 				   struct file_sparse *sparse)
 {
 	struct ksmbd_file *fp;
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	int ret = 0;
 	__le32 old_fattr;
 
@@ -10857,9 +10935,28 @@ int smb2_ioctl(struct ksmbd_work *work)
 	case FSCTL_QUERY_NETWORK_INTERFACE_INFO:
 	case FSCTL_VALIDATE_NEGOTIATE_INFO:
 	case FSCTL_PIPE_WAIT:
-	case FSCTL_PIPE_TRANSCEIVE:
 		no_fileid_ioctl = true;
 		break;
+	case FSCTL_PIPE_TRANSCEIVE:
+		if (!test_share_config_flag(work->tcon->share_conf,
+					    KSMBD_SHARE_FLAG_PIPE)) {
+			ret = -EOPNOTSUPP;
+			goto out;
+		}
+
+		/* RPC pipe handles are not in the regular file table. */
+		if (has_file_id(id) && !pid) {
+			down_read(&work->sess->rpc_lock);
+			no_fileid_ioctl =
+				ksmbd_session_rpc_method(work->sess, id) != 0;
+			up_read(&work->sess->rpc_lock);
+		}
+		if (!no_fileid_ioctl) {
+			ret = -EBADF;
+			rsp->hdr.Status = STATUS_FILE_CLOSED;
+			goto out2;
+		}
+		break;
 	default:
 		break;
 	}
@@ -11018,6 +11115,8 @@ int smb2_ioctl(struct ksmbd_work *work)
 		break;
 	}
 	case FSCTL_PIPE_TRANSCEIVE:
+		rsp->PersistentFileId = pid;
+		rsp->VolatileFileId = id;
 		out_buf_len = min_t(u32, KSMBD_IPC_MAX_PAYLOAD, out_buf_len);
 		nbytes = fsctl_pipe_transceive(work, id, out_buf_len, req, rsp);
 		break;
@@ -11719,137 +11818,26 @@ int smb2_oplock_break(struct ksmbd_work *work)
 	return 0;
 }
 
-/*
- * Cancel handler for a deferred CHANGE_NOTIFY. Races against
- * __ksmbd_close_fd()'s notify_pendings drain (vfs_cache.c), which can run
- * concurrently on a different connection closing the same handle -- only
- * one of the two may claim and free in_work, so both sides check
- * list_empty() under fp->f_lock before touching it (list_del_init()
- * leaves a node empty, so whichever side removes it first is the owner;
- * the loser must not touch in_work again, since the winner may already be
- * freeing it).
- *
- * smb2_cancel() holds conn->request_lock (a spinlock) for the entire
- * time it walks conn->async_requests and calls this function -- so this
- * runs with preemption disabled and must not sleep or re-acquire that
- * same lock. release_async_work() does both (it takes conn->request_lock
- * itself, and frees things that can involve sleeping paths), so calling
- * it from here would self-deadlock the very thread processing the
- * client's CANCEL command. ksmbd_conn_write() can also sleep (it takes
- * conn's write mutex). So: do only the non-sleeping, no-relock cleanup
- * inline here. smb2_cancel() sends and frees the claimed notify after it
- * drops request_lock, preserving response order for a client CANCEL. The
- * connection teardown caller has no such post-unlock path, so its wrapper
- * defers the send and free to a workqueue.
- */
-struct notify_cancel_ctx {
-	struct work_struct	work;
-	struct ksmbd_work	*in_work;
+struct ksmbd_notify_req {
+	wait_queue_head_t wait;
 };
 
-static void smb2_send_notify_cancelled(struct ksmbd_work *work)
-{
-	struct smb2_hdr *hdr = smb_get_msg(work->response_buf);
-	struct ksmbd_conn *conn = work->conn;
-	struct ksmbd_session *sess;
-
-	sess = ksmbd_session_lookup(conn, le64_to_cpu(hdr->SessionId));
-	if (sess) {
-		work->sess = sess;
-		if (work->encrypted && sess->enc && conn->ops->encrypt_resp) {
-			conn->ops->encrypt_resp(work);
-		} else if (conn->ops->is_sign_req && conn->ops->set_sign_rsp &&
-			   conn->ops->is_sign_req(work,
-						 conn->ops->get_cmd_val(work))) {
-			conn->ops->set_sign_rsp(work);
-		}
-	}
-
-	ksmbd_conn_write(work);
-	if (sess) {
-		ksmbd_user_session_put(sess);
-		work->sess = NULL;
-	}
-}
-
-static void smb2_notify_cancel_deferred(struct work_struct *w)
-{
-	struct notify_cancel_ctx *ctx =
-		container_of(w, struct notify_cancel_ctx, work);
-	struct ksmbd_conn *conn = ctx->in_work->conn;
-
-	smb2_complete_notify_cancel(ctx->in_work);
-	kfree(ctx);
-	/*
-	 * The connection teardown waits for r_count before destroying
-	 * connection sessions and their proc entries.
-	 */
-	ksmbd_conn_r_count_dec(conn);
-}
-
-static struct ksmbd_work *smb2_notify_cancel_claim(void **argv)
-{
-	struct ksmbd_work *in_work = (struct ksmbd_work *)argv[0];
-	struct ksmbd_file *fp = (struct ksmbd_file *)argv[1];
-	bool claimed;
-
-	spin_lock(&fp->f_lock);
-	claimed = !list_empty(&in_work->notify_entry);
-	if (claimed)
-		list_del_init(&in_work->notify_entry);
-	spin_unlock(&fp->f_lock);
-
-	if (!claimed)
-		return NULL;
-
-	/* conn->request_lock is held by smb2_cancel() or connection teardown. */
-	in_work->cancel_fn = NULL;
-	kfree(in_work->cancel_argv);
-	in_work->cancel_argv = NULL;
-	return in_work;
-}
-
-static void smb2_complete_notify_cancel(struct ksmbd_work *in_work)
-{
-	struct smb2_hdr *in_hdr = smb_get_msg(in_work->response_buf);
-
-	in_hdr->Status = STATUS_CANCELLED;
-	smb2_send_notify_cancelled(in_work);
-	release_async_work(in_work);
-	ksmbd_free_work_struct(in_work);
-}
-
-static void smb2_notify_cancel_fn(void **argv)
+/*
+ * Cancel handler for a pending CHANGE_NOTIFY. Called either by
+ * smb2_cancel() (conn->request_lock held, work->state already set to
+ * KSMBD_WORK_CANCELLED by the caller) or by
+ * set_close_state_blocked_works() (vfs_cache.c, fp->f_lock held,
+ * work->state already set to KSMBD_WORK_CLOSED by the caller) -- both
+ * callers hold a spinlock across this call, so it must not sleep.
+ * wake_up() only wakes the waiter in smb2_notify(); it does not touch
+ * fp->blocked_works itself, matching smb2_remove_blocked_lock()'s same
+ * non-mutating style for the equivalent byte-range-lock wait.
+ */
+static void smb2_notify_cancel(void **argv)
 {
-	struct ksmbd_work *in_work = smb2_notify_cancel_claim(argv);
-	struct ksmbd_conn *conn;
-	struct notify_cancel_ctx *ctx;
-
-	if (!in_work)
-		return;
-	conn = in_work->conn;
+	struct ksmbd_notify_req *notify_req = argv[0];
 
-	ctx = kmalloc_obj(*ctx, GFP_ATOMIC);
-	if (!ctx) {
-		/* Can't defer the response -- free without sending one. */
-		list_del_init(&in_work->async_request_entry);
-		in_work->asynchronous = false;
-		if (in_work->async_id) {
-			ksmbd_release_id(&conn->async_ida, in_work->async_id);
-			in_work->async_id = 0;
-		}
-		ksmbd_free_work_struct(in_work);
-		return;
-	}
-	ctx->in_work = in_work;
-	INIT_WORK(&ctx->work, smb2_notify_cancel_deferred);
-	/*
-	 * This deferred work can outlive the connection handler's receive loop.
-	 * Keep teardown from destroying the connection's sessions until the
-	 * deferred response has finished using them.
-	 */
-	ksmbd_conn_r_count_inc(conn);
-	schedule_work(&ctx->work);
+	wake_up(&notify_req->wait);
 }
 
 /**
@@ -11862,9 +11850,11 @@ int smb2_notify(struct ksmbd_work *work)
 {
 	struct smb2_change_notify_req *req;
 	struct smb2_change_notify_rsp *rsp;
-	struct ksmbd_work *in_work;
-	struct smb2_hdr *in_hdr;
-	struct ksmbd_file *fp;
+	struct ksmbd_notify_req notify_req;
+	struct ksmbd_file *fp = NULL;
+	void **argv = NULL;
+	bool async_work = false;
+	int err = 0;
 
 	ksmbd_debug(SMB, "Received smb2 notify\n");
 
@@ -11875,164 +11865,83 @@ int smb2_notify(struct ksmbd_work *work)
 
 	if (work->next_smb2_rcv_hdr_off && req->hdr.NextCommand) {
 		rsp->hdr.Status = STATUS_INTERNAL_ERROR;
-		smb2_set_err_rsp(work);
-		return -EIO;
-	}
-
-	/*
-	 * macOS backupd sends CHANGE_NOTIFY with FileId=FFFF...FFFF (share-root
-	 * sentinel) to watch for changes on the share root without holding an
-	 * open handle. Respond STATUS_PENDING + STATUS_NOTIFY_CLEANUP immediately;
-	 * without this, backupd aborts Time Machine setup on STATUS_FILE_CLOSED.
-	 */
-	if (req->VolatileFileId == SMB2_NO_FID &&
-	    req->PersistentFileId == SMB2_NO_FID) {
-		in_work = ksmbd_alloc_work_struct();
-		if (!in_work || allocate_interim_rsp_buf(in_work)) {
-			if (in_work)
-				ksmbd_free_work_struct(in_work);
-			rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES;
-			smb2_set_err_rsp(work);
-			return 0;
-		}
-		if (setup_async_work(work, NULL, NULL)) {
-			ksmbd_free_work_struct(in_work);
-			rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES;
-			smb2_set_err_rsp(work);
-			return 0;
-		}
-		smb2_send_interim_resp(work, STATUS_PENDING);
-		in_work->conn = work->conn;
-		in_hdr = smb_get_msg(in_work->response_buf);
-		memcpy(in_hdr, ksmbd_resp_buf_next(work),
-		       __SMB2_HEADER_STRUCTURE_SIZE);
-		in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND;
-		in_hdr->Id.AsyncId = cpu_to_le64(work->async_id);
-		smb2_set_err_rsp(in_work);
-		in_hdr->Status = STATUS_NOTIFY_CLEANUP;
-		in_work->async_id = work->async_id;
-		work->async_id = 0;
-		release_async_work(work);
-		if (smb2_send_interim_work(in_work, work, false))
-			ksmbd_debug(SMB, "failed to send notify cleanup\n");
-		ksmbd_free_work_struct(in_work);
-		work->send_no_response = 1;
-		return 0;
+		err = -EIO;
+		goto out;
 	}
 
-	/*
-	 * KSMBD does not implement a real change-notification backend.
-	 * Genuine SMB2 servers (and macOS smbfs) never complete a
-	 * CHANGE_NOTIFY spontaneously: it is satisfied only by a real
-	 * directory change, or with STATUS_NOTIFY_CLEANUP when the watched
-	 * handle is closed. Completing it early (e.g. on a timer) makes
-	 * Finder treat the cleanup as "directory changed" and re-enumerate
-	 * the directory forever, leaving items unopenable. Returning
-	 * STATUS_NOT_IMPLEMENTED here (like stock ksmbd) makes macOS smbfs
-	 * hard-freeze on unmount, so this must stay deferred.
-	 */
 	fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId);
 	if (!fp) {
 		rsp->hdr.Status = STATUS_FILE_CLOSED;
-		smb2_set_err_rsp(work);
-		return 0;
+		err = -ENOENT;
+		goto out;
 	}
 
-	in_work = ksmbd_alloc_work_struct();
-	if (!in_work || allocate_interim_rsp_buf(in_work)) {
-		if (in_work)
-			ksmbd_free_work_struct(in_work);
-		ksmbd_fd_put(work, fp);
-		rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES;
-		smb2_set_err_rsp(work);
-		return 0;
-	}
-	/*
-	 * in_work is synthetic (not from the normal request-receiving
-	 * pipeline), so it has no request_buf of its own. It gets registered
-	 * into conn->async_requests below, and smb2_cancel() unconditionally
-	 * computes smb_get_msg(iter->request_buf) for every entry in that
-	 * list while searching for a match -- give it its own small buffer
-	 * (not an alias of response_buf: ksmbd_free_work_struct() kvfree()s
-	 * both separately, so aliasing them would double-free) so that stays
-	 * a harmless read instead of a near-NULL dereference.
-	 */
-	in_work->request_buf = kzalloc(MAX_CIFS_SMALL_BUFFER_SIZE, KSMBD_DEFAULT_GFP);
-	if (!in_work->request_buf) {
-		ksmbd_free_work_struct(in_work);
-		ksmbd_fd_put(work, fp);
+	argv = kmalloc_obj(*argv, KSMBD_DEFAULT_GFP);
+	if (!argv) {
 		rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES;
-		smb2_set_err_rsp(work);
-		return 0;
+		err = -ENOMEM;
+		goto out;
 	}
-	memcpy(smb_get_msg(in_work->request_buf), req,
-	       __SMB2_HEADER_STRUCTURE_SIZE);
+	init_waitqueue_head(&notify_req.wait);
+	argv[0] = &notify_req;
 
-	if (setup_async_work(work, NULL, NULL)) {
-		ksmbd_free_work_struct(in_work);
-		ksmbd_fd_put(work, fp);
+	err = setup_async_work(work, smb2_notify_cancel, argv);
+	if (err) {
 		rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES;
-		smb2_set_err_rsp(work);
-		return 0;
+		goto out;
 	}
-
-	smb2_send_interim_resp(work, STATUS_PENDING);
-
-	/* Keep the async IDA alive until the deferred work is released. */
-	in_work->conn = ksmbd_conn_get(work->conn);
-	in_work->owns_conn_ref = true;
-	in_work->encrypted = work->encrypted;
-	in_hdr = smb_get_msg(in_work->response_buf);
-	memcpy(in_hdr, ksmbd_resp_buf_next(work), __SMB2_HEADER_STRUCTURE_SIZE);
-	in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND;
-	in_hdr->Id.AsyncId = cpu_to_le64(work->async_id);
-	smb2_set_err_rsp(in_work);
-	in_hdr->Status = STATUS_NOTIFY_CLEANUP;
+	async_work = true;
 
 	/*
-	 * Transfer ownership of the async id to in_work; it stays reserved
-	 * until in_work is freed after the deferred response is sent on
-	 * close, so it can't be reused for an unrelated async response.
+	 * Handle close holds the file-table write lock while it marks the
+	 * handle closed and walks blocked_works.  Hold the matching read lock
+	 * across the state check and registration so close cannot finish its
+	 * walk between the lookup above and this list insertion.
 	 */
-	in_work->async_id = work->async_id;
-	work->async_id = 0;
-	release_async_work(work);
+	read_lock(&work->sess->file_table.lock);
+	if (fp->f_state != FP_INITED) {
+		read_unlock(&work->sess->file_table.lock);
+		rsp->hdr.Status = STATUS_NOTIFY_CLEANUP;
+		err = -ENOENT;
+		goto out;
+	}
+	spin_lock(&fp->f_lock);
+	list_add_tail(&work->fp_entry, &fp->blocked_works);
+	spin_unlock(&fp->f_lock);
+	read_unlock(&work->sess->file_table.lock);
 
-	/*
-	 * work itself is about to be recycled by the normal request-processing
-	 * pipeline, so it can't stay the target of a future CANCEL -- register
-	 * in_work instead, reusing the same async_id, so a client-sent CANCEL
-	 * for this notify actually finds something to cancel instead of
-	 * silently doing nothing until the handle eventually closes.
-	 */
-	in_work->asynchronous = true;
-	in_work->cancel_argv = kmalloc_array(2, sizeof(void *), KSMBD_DEFAULT_GFP);
-	if (in_work->cancel_argv) {
-		in_work->cancel_argv[0] = in_work;
-		in_work->cancel_argv[1] = fp;
-		in_work->cancel_fn = smb2_notify_cancel_fn;
-	}
-
-	if (!ksmbd_conn_link_async_request(work->conn, in_work)) {
-		kfree(in_work->cancel_argv);
-		in_work->cancel_argv = NULL;
-		in_work->cancel_fn = NULL;
-		in_work->asynchronous = false;
-		ksmbd_fd_put(work, fp);
-		if (smb2_send_interim_work(in_work, work, false))
-			ksmbd_debug(SMB, "failed to send notify cleanup\n");
-		ksmbd_free_work_struct(in_work);
-		work->send_no_response = 1;
-		return 0;
+	smb2_send_interim_resp(work, STATUS_PENDING);
+
+	err = wait_event_interruptible(notify_req.wait,
+				       READ_ONCE(work->state) != KSMBD_WORK_ACTIVE);
+	if (err && READ_ONCE(work->state) == KSMBD_WORK_ACTIVE) {
+		/*
+		 * Woken by a signal, not a real cancel/close. There is no
+		 * notification backend yet to report anything else against,
+		 * so treat this the same as a client-side cancel.
+		 */
+		WRITE_ONCE(work->state, KSMBD_WORK_CANCELLED);
 	}
 
 	spin_lock(&fp->f_lock);
-	list_add_tail(&in_work->notify_entry, &fp->notify_pendings);
+	list_del_init(&work->fp_entry);
 	spin_unlock(&fp->f_lock);
 
-	ksmbd_fd_put(work, fp);
+	rsp->hdr.Status = work->state == KSMBD_WORK_CLOSED ?
+			  STATUS_NOTIFY_CLEANUP : STATUS_CANCELLED;
+	smb2_send_interim_resp(work, rsp->hdr.Status);
 	work->send_no_response = 1;
-	return 0;
+
+out:
+	if (rsp->hdr.Status != STATUS_SUCCESS && !work->send_no_response)
+		smb2_set_err_rsp(work);
+	if (async_work)
+		release_async_work(work);
+	else
+		kfree(argv);
+	if (fp)
+		ksmbd_fd_put(work, fp);
+	return err;
 }
 
 /**
@@ -12227,11 +12136,12 @@ void smb3_set_sign_rsp(struct ksmbd_work *work)
 	struct channel *chann;
 	char signature[SMB2_CMACAES_SIZE];
 	struct kvec *iov;
-	u16 command = conn->ops->get_cmd_val(work);
+	u16 command;
 	int n_vec;
 	char *signing_key;
 
 	hdr = ksmbd_resp_buf_curr(work);
+	command = le16_to_cpu(hdr->Command);
 
 	if (command == SMB2_SESSION_SETUP_HE &&
 	    (!conn->binding || hdr->Status != STATUS_SUCCESS)) {
diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h
index ca8e27f7b712f7..40d61745ef41d9 100644
--- a/fs/smb/server/smb2pdu.h
+++ b/fs/smb/server/smb2pdu.h
@@ -156,8 +156,7 @@ struct create_durable_rsp {
 } __packed;
 
 /*
- * See POSIX-SMB2 2.2.14.2.16
- * Link: https://gitlab.com/samba-team/smb3-posix-spec/-/blob/master/smb3_posix_extensions.md
+ * See POSIX-SMB2 2.1.3.2.1
  */
 struct create_posix_rsp {
 	struct create_context_hdr ccontext;
@@ -199,7 +198,7 @@ struct file_sparse {
 #define FILE_INTERNAL_INFORMATION_SIZE        8
 #define FILE_EA_INFORMATION_SIZE              4
 #define FILE_ACCESS_INFORMATION_SIZE          4
-#define FILE_NAME_INFORMATION_SIZE            9
+#define FILE_NAME_INFORMATION_SIZE            8
 #define FILE_RENAME_INFORMATION_SIZE          10
 #define FILE_LINK_INFORMATION_SIZE            11
 #define FILE_NAMES_INFORMATION_SIZE           12
@@ -211,8 +210,6 @@ struct file_sparse {
 #define FILE_ALL_INFORMATION_SIZE             104
 #define FILE_ALLOCATION_INFORMATION_SIZE      19
 #define FILE_END_OF_FILE_INFORMATION_SIZE     20
-#define FILE_ALTERNATE_NAME_INFORMATION_SIZE  8
-#define FILE_NORMALIZED_NAME_INFORMATION_SIZE 8
 #define FILE_STREAM_INFORMATION_SIZE          32
 #define FILE_PIPE_INFORMATION_SIZE            23
 #define FILE_PIPE_LOCAL_INFORMATION_SIZE      24
@@ -259,7 +256,7 @@ struct smb2_file_alignment_info {
 	__le32 AlignmentRequirement;
 } __packed;
 
-struct smb2_file_alt_name_info {
+struct smb2_file_name_info {
 	__le32 FileNameLength;
 	char FileName[];
 } __packed;
@@ -351,6 +348,7 @@ struct create_sd_buf_req {
 	struct smb_ntsd ntsd;
 } __packed;
 
+/* See POSIX-FSCC 2.2.1 */
 struct smb2_posix_info {
 	__le32 NextEntryOffset;
 	__u32 Ignored;
@@ -364,20 +362,18 @@ struct smb2_posix_info {
 	__le64 Inode;
 	__le32 DeviceId;
 	__le32 Zero;
-	/* beginning of POSIX Create Context Response */
+	/*
+	 * Beginning of POSIX Create Context Response
+	 * See POSIX-SMB2 2.1.3.2.1
+	 */
 	__le32 HardLinks;
 	__le32 ReparseTag;
 	__le32 Mode;
 	/* SidBuffer contain two sids (UNIX user sid(16), UNIX group sid(16)) */
 	u8 SidBuffer[32];
+	/* End of POSIX Create Context Response */
 	__le32 name_len;
 	u8 name[];
-	/*
-	 * var sized owner SID
-	 * var sized group SID
-	 * le32 filenamelength
-	 * u8  filename[]
-	 */
 } __packed;
 
 /* functions */
diff --git a/fs/smb/server/smb_common.c b/fs/smb/server/smb_common.c
index 4c2da65510bcd2..7dbcfa658edd5b 100644
--- a/fs/smb/server/smb_common.c
+++ b/fs/smb/server/smb_common.c
@@ -467,7 +467,7 @@ int ksmbd_populate_dot_dotdot_entries(struct ksmbd_work *work, int info_level,
 {
 	int i, rc = 0;
 	struct ksmbd_conn *conn = work->conn;
-	struct mnt_idmap *idmap = file_mnt_idmap(dir->filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(dir->filp);
 
 	for (i = 0; i < 2; i++) {
 		struct kstat kstat;
@@ -523,8 +523,6 @@ int ksmbd_populate_dot_dotdot_entries(struct ksmbd_work *work, int info_level,
  * @shortname:	destination short filename
  *
  * Return:	shortname length or 0 when source long name is '.' or '..'
- * TODO: Though this function conforms the restriction of 8.3 Filename spec,
- * but the result is different with Windows 7's one. need to check.
  */
 int ksmbd_extract_shortname(struct ksmbd_conn *conn, const char *longname,
 			    char *shortname)
@@ -588,7 +586,7 @@ int ksmbd_extract_shortname(struct ksmbd_conn *conn, const char *longname,
 	if (dot_present)
 		memcpy(out + baselen + 4, extension, 4);
 	else
-		out[baselen + 4] = '\0';
+		out[baselen + 3] = '\0';
 	smbConvertToUTF16((__le16 *)shortname, out, PATH_MAX,
 			  conn->local_nls, 0);
 	len = strlen(out) * 2;
diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c
index 1fad6ccf3a72ab..96428df33b4389 100644
--- a/fs/smb/server/smbacl.c
+++ b/fs/smb/server/smbacl.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/fs.h>
+#include <kunit/visibility.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/mnt_idmapping.h>
@@ -257,7 +258,7 @@ void id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid)
 	ssid->num_subauth++;
 }
 
-static int sid_to_id(struct mnt_idmap *idmap,
+static int sid_to_id(const struct mnt_idmap *idmap,
 		     struct smb_sid *psid, uint sidtype,
 		     struct smb_fattr *fattr)
 {
@@ -383,7 +384,7 @@ void free_acl_state(struct posix_acl_state *state)
 	kfree(state->groups);
 }
 
-static int parse_dacl(struct mnt_idmap *idmap,
+static int parse_dacl(const struct mnt_idmap *idmap,
 		      struct smb_acl *pdacl, char *end_of_acl,
 		      struct smb_sid *pownersid, struct smb_sid *pgrpsid,
 		      struct smb_fattr *fattr)
@@ -619,7 +620,7 @@ static int parse_dacl(struct mnt_idmap *idmap,
 	return ret;
 }
 
-static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
+static void set_posix_acl_entries_dacl(const struct mnt_idmap *idmap,
 				       struct smb_ace *pndace,
 				       struct smb_fattr *fattr, u16 *num_aces,
 				       u16 *size, u16 existing_nt_aces,
@@ -750,7 +751,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 	}
 }
 
-static void set_ntacl_dacl(struct mnt_idmap *idmap,
+static void set_ntacl_dacl(const struct mnt_idmap *idmap,
 			   struct smb_acl *pndacl,
 			   struct smb_acl *nt_dacl,
 			   unsigned int aces_size,
@@ -809,7 +810,7 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap,
 	pndacl->size = cpu_to_le16(le16_to_cpu(pndacl->size) + size);
 }
 
-static void set_mode_dacl(struct mnt_idmap *idmap,
+static void set_mode_dacl(const struct mnt_idmap *idmap,
 			  struct smb_acl *pndacl, struct smb_fattr *fattr)
 {
 	struct smb_ace *pace, *pndace;
@@ -895,7 +896,7 @@ static int parse_sid(struct smb_sid *psid, char *end_of_acl)
 }
 
 /* Convert CIFS ACL to POSIX form */
-int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd,
+int parse_sec_desc(const struct mnt_idmap *idmap, struct smb_ntsd *pntsd,
 		   int acl_len, struct smb_fattr *fattr)
 {
 	int rc = 0;
@@ -1030,7 +1031,7 @@ size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr,
 }
 
 /* Convert permission bits from mode to equivalent CIFS ACL */
-int build_sec_desc(struct mnt_idmap *idmap,
+int build_sec_desc(const struct mnt_idmap *idmap,
 		   struct smb_ntsd *pntsd, struct smb_ntsd *ppntsd,
 		   int ppntsd_size, int addition_info, __u32 *secdesclen,
 		   struct smb_fattr *fattr)
@@ -1096,9 +1097,12 @@ int build_sec_desc(struct mnt_idmap *idmap,
 			struct smb_acl *ppdacl_ptr;
 			unsigned int dacl_offset = le32_to_cpu(ppntsd->dacloffset);
 			int ppdacl_size, ntacl_size = ppntsd_size - dacl_offset;
+			size_t dacl_struct_end;
 
 			if (!dacl_offset ||
-			    (dacl_offset + sizeof(struct smb_acl) > ppntsd_size))
+			    check_add_overflow(dacl_offset, sizeof(struct smb_acl),
+					       &dacl_struct_end) ||
+			    dacl_struct_end > (size_t)ppntsd_size)
 				goto out;
 
 			ppdacl_ptr = (struct smb_acl *)((char *)ppntsd + dacl_offset);
@@ -1196,7 +1200,7 @@ int smb_inherit_dacl(struct ksmbd_conn *conn,
 	struct smb_ntsd *parent_pntsd = NULL;
 	struct smb_sid owner_sid, group_sid;
 	struct dentry *parent = path->dentry->d_parent;
-	struct mnt_idmap *idmap = mnt_idmap(path->mnt);
+	const struct mnt_idmap *idmap = mnt_idmap(path->mnt);
 	int inherited_flags = 0, flags = 0, i, nt_size = 0, pdacl_size;
 	int rc = 0, pntsd_type, ppntsd_size, acl_len, aces_size;
 	unsigned int dacloffset;
@@ -1451,7 +1455,7 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 			__le32 *pdaccess, __le32 raw_daccess, int uid,
 			bool strict)
 {
-	struct mnt_idmap *idmap = mnt_idmap(path->mnt);
+	const struct mnt_idmap *idmap = mnt_idmap(path->mnt);
 	struct smb_ntsd *pntsd = NULL;
 	struct smb_acl *pdacl;
 	struct posix_acl *posix_acls;
@@ -1665,6 +1669,7 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path,
 	kfree(pntsd);
 	return rc;
 }
+EXPORT_SYMBOL_IF_KUNIT(smb_check_perm_dacl);
 
 int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon,
 		 const struct path *path, struct smb_ntsd *pntsd, int ntsd_len,
@@ -1673,7 +1678,7 @@ int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon,
 	int rc;
 	struct smb_fattr fattr = {{0}};
 	struct inode *inode = d_inode(path->dentry);
-	struct mnt_idmap *idmap = mnt_idmap(path->mnt);
+	const struct mnt_idmap *idmap = mnt_idmap(path->mnt);
 	struct iattr newattrs;
 
 	fattr.cf_uid = INVALID_UID;
diff --git a/fs/smb/server/smbacl.h b/fs/smb/server/smbacl.h
index 01810c16cc0461..28d215807faae2 100644
--- a/fs/smb/server/smbacl.h
+++ b/fs/smb/server/smbacl.h
@@ -81,9 +81,9 @@ struct posix_acl_state {
 	struct posix_ace_state_array *groups;
 };
 
-int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd,
+int parse_sec_desc(const struct mnt_idmap *idmap, struct smb_ntsd *pntsd,
 		   int acl_len, struct smb_fattr *fattr);
-int build_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd,
+int build_sec_desc(const struct mnt_idmap *idmap, struct smb_ntsd *pntsd,
 		   struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info,
 		   __u32 *secdesclen, struct smb_fattr *fattr);
 int init_acl_state(struct posix_acl_state *state, u16 cnt);
@@ -105,7 +105,7 @@ void ksmbd_init_domain(u32 *sub_auth);
 size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr,
 		struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info);
 
-static inline uid_t posix_acl_uid_translate(struct mnt_idmap *idmap,
+static inline uid_t posix_acl_uid_translate(const struct mnt_idmap *idmap,
 					    struct posix_acl_entry *pace)
 {
 	vfsuid_t vfsuid;
@@ -117,7 +117,7 @@ static inline uid_t posix_acl_uid_translate(struct mnt_idmap *idmap,
 	return from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid));
 }
 
-static inline gid_t posix_acl_gid_translate(struct mnt_idmap *idmap,
+static inline gid_t posix_acl_gid_translate(const struct mnt_idmap *idmap,
 					    struct posix_acl_entry *pace)
 {
 	vfsgid_t vfsgid;
diff --git a/fs/smb/server/tests/Kconfig b/fs/smb/server/tests/Kconfig
new file mode 100644
index 00000000000000..ad7a4e94ceaa88
--- /dev/null
+++ b/fs/smb/server/tests/Kconfig
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2026 Hang Nan <nanx95726@gmail.com>
+
+config SMB_SERVER_KUNIT_TESTS
+	tristate "KUnit tests for SMB3 server helpers" if !KUNIT_ALL_TESTS
+	depends on SMB_SERVER && SMB_KUNIT_TESTS && TMPFS_XATTR
+	default SMB_KUNIT_TESTS
+	help
+	  This builds the KUnit tests for ksmbd server helpers. The tests
+	  exercise internal server functionality and help detect regressions
+	  in server-side behavior. They are intended for kernel developers
+	  and are not suitable for production systems.
+
+	  For more information on KUnit and unit tests in the kernel,
+	  please read Documentation/dev-tools/kunit/index.rst.
diff --git a/fs/smb/server/tests/Makefile b/fs/smb/server/tests/Makefile
new file mode 100644
index 00000000000000..8738ab0b0667ba
--- /dev/null
+++ b/fs/smb/server/tests/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2026 Hang Nan <nanx95726@gmail.com>
+
+obj-$(CONFIG_SMB_SERVER_KUNIT_TESTS) += smbacl_kunit.o
diff --git a/fs/smb/server/tests/smbacl_kunit.c b/fs/smb/server/tests/smbacl_kunit.c
new file mode 100644
index 00000000000000..33496b4d31a3ec
--- /dev/null
+++ b/fs/smb/server/tests/smbacl_kunit.c
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * KUnit tests for ksmbd security descriptor (DACL) handling.
+ *
+ * Copyright (C) 2026 Hang Nan <nanx95726@gmail.com>
+ *
+ * The tests pin the DACL declared-size boundary in smb_check_perm_dacl():
+ *
+ * - ksmbd_dacl_walk_must_stop_at_declared_size: a pure semantic harness
+ *   that models the ACE walk.  Walking to the end of the enclosing
+ *   security descriptor (the pre-fix behaviour) selects an ACE that
+ *   sits beyond struct smb_acl::size; stopping at the declared DACL
+ *   size (the fixed behaviour) rejects it.
+ *
+ * - ksmbd_smb_check_perm_dacl_boundary and
+ *   ksmbd_smb_check_perm_dacl_maximal_boundary: drive the real
+ *   smb_check_perm_dacl() with a descriptor stored through ksmbd's own
+ *   NTACL xattr path on a tmpfs file, and assert that a post-boundary
+ *   ACE is not selected for either a regular or maximal access check.
+ */
+
+#include <kunit/test.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/shmem_fs.h>
+#include <linux/slab.h>
+
+#include "../smbacl.h"
+#include "../smb_common.h"
+#include "../vfs.h"
+
+struct ksmbd_acl_walk_result {
+	bool found;
+	bool allowed;
+	const struct smb_ace *selected;
+};
+
+static const struct smb_sid test_nonmatching_sid = {
+	1, 5, {0, 0, 0, 0, 0, 5},
+	{ cpu_to_le32(21), cpu_to_le32(1), cpu_to_le32(2),
+	  cpu_to_le32(3), cpu_to_le32(9999) }
+};
+
+/*
+ * S-1-22-1-0: the SID id_to_sid(0, SIDUNIX_USER) resolves to, i.e. what
+ * smb_check_perm_dacl() looks for when called with uid == 0.
+ */
+static const struct smb_sid test_owner_sid = {
+	1, 2, {0, 0, 0, 0, 0, 22},
+	{ cpu_to_le32(1), cpu_to_le32(0) }
+};
+
+static int test_compare_sids(const struct smb_sid *a, const struct smb_sid *b)
+{
+	int i;
+
+	if (a->revision != b->revision || a->num_subauth != b->num_subauth)
+		return 1;
+	for (i = 0; i < NUM_AUTHS; i++) {
+		if (a->authority[i] != b->authority[i])
+			return 1;
+	}
+	for (i = 0; i < a->num_subauth; i++) {
+		if (a->sub_auth[i] != b->sub_auth[i])
+			return 1;
+	}
+	return 0;
+}
+
+static u16 test_ace_size(const struct smb_sid *sid)
+{
+	return offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE +
+	       sid->num_subauth * sizeof(__le32);
+}
+
+static u16 fill_test_ace(struct smb_ace *ace, const struct smb_sid *sid,
+			 u32 access_req)
+{
+	u16 size = test_ace_size(sid);
+
+	ace->type = ACCESS_ALLOWED_ACE_TYPE;
+	ace->flags = 0;
+	ace->size = cpu_to_le16(size);
+	ace->access_req = cpu_to_le32(access_req);
+	memcpy(&ace->sid, sid, size - offsetof(struct smb_ace, sid));
+	return size;
+}
+
+static struct ksmbd_acl_walk_result test_walk_dacl(struct smb_acl *pdacl,
+						    int walk_boundary,
+						    const struct smb_sid *target,
+						    u32 requested)
+{
+	struct ksmbd_acl_walk_result result = {};
+	struct smb_ace *ace;
+	int aces_size;
+	int i;
+
+	ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl));
+	aces_size = walk_boundary - sizeof(struct smb_acl);
+	for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) {
+		u16 ace_size;
+
+		if (aces_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE)
+			break;
+		ace_size = le16_to_cpu(ace->size);
+		if (ace_size > aces_size ||
+		    ace_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE)
+			break;
+		aces_size -= ace_size;
+
+		if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES ||
+		    ace_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE +
+			       sizeof(__le32) * ace->sid.num_subauth)
+			break;
+
+		if (!test_compare_sids(target, &ace->sid)) {
+			result.found = true;
+			result.selected = ace;
+			result.allowed = !(requested & ~le32_to_cpu(ace->access_req));
+			return result;
+		}
+
+		ace = (struct smb_ace *)((char *)ace + ace_size);
+	}
+
+	return result;
+}
+
+static void ksmbd_dacl_walk_must_stop_at_declared_size(struct kunit *test)
+{
+	struct ksmbd_acl_walk_result declared, enclosing;
+	struct smb_acl *acl;
+	struct smb_ace *ace1, *fake;
+	u16 ace1_size, fake_size;
+	u16 pdacl_size;
+	u16 acl_size;
+
+	acl = kunit_kzalloc(test, 128, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, acl);
+
+	acl->revision = cpu_to_le16(2);
+	acl->num_aces = cpu_to_le16(2);
+
+	ace1 = (struct smb_ace *)((char *)acl + sizeof(*acl));
+	ace1_size = fill_test_ace(ace1, &test_nonmatching_sid, 0);
+	fake = (struct smb_ace *)((char *)ace1 + ace1_size);
+	fake_size = fill_test_ace(fake, &test_owner_sid, FILE_READ_DATA);
+
+	pdacl_size = sizeof(*acl) + ace1_size;
+	acl_size = pdacl_size + fake_size;
+	acl->size = cpu_to_le16(pdacl_size);
+
+	declared = test_walk_dacl(acl, pdacl_size, &test_owner_sid,
+				  FILE_READ_DATA);
+	enclosing = test_walk_dacl(acl, acl_size, &test_owner_sid,
+				   FILE_READ_DATA);
+
+	KUNIT_EXPECT_FALSE(test, declared.found);
+	KUNIT_EXPECT_FALSE(test, declared.allowed);
+
+	/* Demonstrates that the buggy acl_size boundary selects fake ACE #2. */
+	KUNIT_EXPECT_TRUE(test, enclosing.found);
+	KUNIT_EXPECT_TRUE(test, enclosing.allowed);
+}
+
+/*
+ * Build an NTSD whose DACL declares one ACE (pdacl->size) but actually
+ * contains two: the second ACE sits beyond the declared DACL boundary
+ * yet inside the enclosing security descriptor.  The trailing ACE applies
+ * to S-1-22-1-0, which smb_check_perm_dacl() looks for when uid is zero.
+ */
+static struct smb_ntsd *build_boundary_ntsd(struct kunit *test,
+					    const struct smb_sid *first_sid,
+					    u32 first_access,
+					    u32 trailing_access,
+					    int *ntsd_size)
+{
+	struct smb_ntsd *pntsd;
+	struct smb_acl *pdacl;
+	struct smb_ace *ace;
+	u16 first_size = test_ace_size(first_sid);
+	u16 trailing_size = test_ace_size(&test_owner_sid);
+
+	*ntsd_size = sizeof(struct smb_ntsd) + sizeof(struct smb_acl) +
+		     first_size + trailing_size;
+	pntsd = kunit_kzalloc(test, *ntsd_size, GFP_KERNEL);
+	if (!pntsd)
+		return NULL;
+
+	pntsd->revision = cpu_to_le16(SD_REVISION);
+	pntsd->type = cpu_to_le16(DACL_PRESENT);
+	pntsd->dacloffset = cpu_to_le32(sizeof(struct smb_ntsd));
+
+	pdacl = (struct smb_acl *)((char *)pntsd + sizeof(struct smb_ntsd));
+	pdacl->revision = cpu_to_le16(2);
+	pdacl->num_aces = cpu_to_le16(2);
+	pdacl->size = cpu_to_le16(sizeof(struct smb_acl) + first_size);
+
+	ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl));
+	fill_test_ace(ace, first_sid, first_access);
+
+	ace = (struct smb_ace *)((char *)ace + first_size);
+	fill_test_ace(ace, &test_owner_sid, trailing_access);
+
+	return pntsd;
+}
+
+static void ksmbd_smb_check_perm_dacl_boundary_test(struct kunit *test)
+{
+	struct file *file;
+	struct smb_ntsd *pntsd;
+	__le32 daccess = cpu_to_le32(FILE_READ_DATA);
+	int ntsd_size, rc;
+
+	pntsd = build_boundary_ntsd(test, &test_nonmatching_sid, 0,
+				     FILE_READ_DATA, &ntsd_size);
+	KUNIT_ASSERT_NOT_NULL(test, pntsd);
+
+	file = shmem_file_setup("ksmbd-kunit-dacl", 0,
+				mk_vma_flags(VMA_NORESERVE_BIT));
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file);
+
+	rc = ksmbd_vfs_set_sd_xattr(NULL, mnt_idmap(file->f_path.mnt),
+				    &file->f_path, pntsd, ntsd_size,
+				    false);
+	KUNIT_EXPECT_EQ(test, 0, rc);
+	if (rc)
+		goto out;
+
+	rc = smb_check_perm_dacl(NULL, &file->f_path, &daccess,
+				 cpu_to_le32(FILE_READ_DATA), 0, false);
+
+	/*
+	 * The post-boundary ACE (ACE #2, beyond pdacl->size) grants
+	 * FILE_READ_DATA to the caller's SID, but it must not be
+	 * selected: the walk stops at the declared DACL size and access
+	 * is denied.  Before the fix the walk used the enclosing
+	 * descriptor length, selected ACE #2 and returned 0.
+	 */
+	KUNIT_EXPECT_EQ(test, -EACCES, rc);
+out:
+	fput(file);
+}
+
+static void
+ksmbd_smb_check_perm_dacl_maximal_boundary_test(struct kunit *test)
+{
+	struct file *file;
+	struct smb_ntsd *pntsd;
+	__le32 daccess = FILE_MAXIMAL_ACCESS_LE;
+	int ntsd_size, rc;
+
+	/*
+	 * The in-boundary ACE grants read access.  The trailing ACE grants
+	 * write access, which must not be included in the maximal access mask.
+	 */
+	pntsd = build_boundary_ntsd(test, &test_owner_sid, FILE_READ_DATA,
+				     FILE_WRITE_DATA, &ntsd_size);
+	KUNIT_ASSERT_NOT_NULL(test, pntsd);
+
+	file = shmem_file_setup("ksmbd-kunit-dacl-maximal", 0,
+				mk_vma_flags(VMA_NORESERVE_BIT));
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file);
+
+	rc = ksmbd_vfs_set_sd_xattr(NULL, mnt_idmap(file->f_path.mnt),
+				    &file->f_path, pntsd, ntsd_size,
+				    false);
+	KUNIT_EXPECT_EQ(test, 0, rc);
+	if (rc)
+		goto out;
+
+	rc = smb_check_perm_dacl(NULL, &file->f_path, &daccess,
+				 FILE_MAXIMAL_ACCESS_LE, 0, false);
+	KUNIT_EXPECT_EQ(test, 0, rc);
+	if (rc)
+		goto out;
+
+	KUNIT_EXPECT_TRUE(test, le32_to_cpu(daccess) & FILE_READ_DATA);
+	KUNIT_EXPECT_FALSE(test, le32_to_cpu(daccess) & FILE_WRITE_DATA);
+out:
+	fput(file);
+}
+
+static struct kunit_case ksmbd_smbacl_test_cases[] = {
+	KUNIT_CASE(ksmbd_dacl_walk_must_stop_at_declared_size),
+	KUNIT_CASE(ksmbd_smb_check_perm_dacl_boundary_test),
+	KUNIT_CASE(ksmbd_smb_check_perm_dacl_maximal_boundary_test),
+	{}
+};
+
+static struct kunit_suite ksmbd_smbacl_test_suite = {
+	.name = "ksmbd-smbacl",
+	.test_cases = ksmbd_smbacl_test_cases,
+};
+
+kunit_test_suite(ksmbd_smbacl_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for ksmbd smbacl helpers");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c
index 4968cfc1a572f7..488ca113b43152 100644
--- a/fs/smb/server/transport_tcp.c
+++ b/fs/smb/server/transport_tcp.c
@@ -77,6 +77,7 @@ static struct tcp_transport *alloc_transport(struct socket *client_sk)
 	if (client_sk->sk->sk_family == AF_INET6) {
 		memcpy(&conn->inet6_addr, &client_sk->sk->sk_v6_daddr, 16);
 		conn->inet_hash = ipv6_addr_hash(&client_sk->sk->sk_v6_daddr);
+		conn->is_ipv6 = true;
 	} else {
 		conn->inet_addr = inet_sk(client_sk->sk)->inet_daddr;
 		conn->inet_hash = ipv4_addr_hash(inet_sk(client_sk->sk)->inet_daddr);
@@ -656,10 +657,15 @@ static void ksmbd_tcp_stop_listener(struct interface *iface)
 void ksmbd_tcp_destroy(void)
 {
 	struct interface *iface, *tmp;
+	LIST_HEAD(iface_list_to_free);
 
 	unregister_netdevice_notifier(&ksmbd_netdev_notifier);
 
-	list_for_each_entry_safe(iface, tmp, &iface_list, entry) {
+	rtnl_lock();
+	list_splice_init(&iface_list, &iface_list_to_free);
+	rtnl_unlock();
+
+	list_for_each_entry_safe(iface, tmp, &iface_list_to_free, entry) {
 		ksmbd_tcp_stop_listener(iface);
 		list_del(&iface->entry);
 		kfree(iface->name);
diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c
index c2c9aaa5de1b58..2e2c554bc2c1fa 100644
--- a/fs/smb/server/vfs.c
+++ b/fs/smb/server/vfs.c
@@ -5,6 +5,7 @@
  */
 
 #include <crypto/sha2.h>
+#include <kunit/visibility.h>
 #include <linux/kernel.h>
 #include <linux/fs.h>
 #include <linux/fs_struct.h>
@@ -115,7 +116,7 @@ static int ksmbd_vfs_path_lookup(struct ksmbd_share_config *share_conf,
 	return 0;
 }
 
-void ksmbd_vfs_query_maximal_access(struct mnt_idmap *idmap,
+void ksmbd_vfs_query_maximal_access(const struct mnt_idmap *idmap,
 				   struct dentry *dentry, __le32 *daccess)
 {
 	*daccess = cpu_to_le32(FILE_READ_ATTRIBUTES | READ_CONTROL);
@@ -183,7 +184,7 @@ int ksmbd_vfs_create(struct ksmbd_work *work, const char *name, umode_t mode)
  */
 int ksmbd_vfs_mkdir(struct ksmbd_work *work, const char *name, umode_t mode)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct path path;
 	struct dentry *dentry, *d;
 	int err = 0;
@@ -216,9 +217,9 @@ int ksmbd_vfs_mkdir(struct ksmbd_work *work, const char *name, umode_t mode)
 	return err;
 }
 
-static ssize_t ksmbd_vfs_getcasexattr(struct mnt_idmap *idmap,
-				      struct dentry *dentry, char *attr_name,
-				      int attr_name_len, char **attr_value)
+ssize_t ksmbd_vfs_getcasexattr(const struct mnt_idmap *idmap,
+			       struct dentry *dentry, char *attr_name,
+			       int attr_name_len, char **attr_value)
 {
 	char *name, *xattr_list = NULL;
 	ssize_t value_len = -ENOENT, xattr_list_len;
@@ -386,7 +387,7 @@ static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos,
 {
 	const struct cred *saved_cred;
 	char *stream_buf = NULL, *wbuf;
-	struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
 	size_t size;
 	ssize_t v_len;
 	int err = 0;
@@ -577,7 +578,7 @@ int ksmbd_vfs_fsync(struct ksmbd_work *work, u64 fid, u64 p_id)
  */
 int ksmbd_vfs_remove_file(struct ksmbd_work *work, const struct path *path)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct dentry *parent = path->dentry->d_parent;
 	int err;
 
@@ -841,8 +842,8 @@ ssize_t ksmbd_vfs_listxattr(struct dentry *dentry, char **list)
 	return size;
 }
 
-static ssize_t ksmbd_vfs_xattr_len(struct mnt_idmap *idmap,
-				   struct dentry *dentry, char *xattr_name)
+ssize_t ksmbd_vfs_xattr_len(const struct mnt_idmap *idmap,
+			    struct dentry *dentry, char *xattr_name)
 {
 	return vfs_getxattr(idmap, dentry, xattr_name, NULL, 0);
 }
@@ -856,7 +857,7 @@ static ssize_t ksmbd_vfs_xattr_len(struct mnt_idmap *idmap,
  *
  * Return:	read xattr value length on success, otherwise error
  */
-ssize_t ksmbd_vfs_getxattr(struct mnt_idmap *idmap,
+ssize_t ksmbd_vfs_getxattr(const struct mnt_idmap *idmap,
 			   struct dentry *dentry,
 			   char *xattr_name, char **xattr_buf)
 {
@@ -893,7 +894,7 @@ ssize_t ksmbd_vfs_getxattr(struct mnt_idmap *idmap,
  *
  * Return:	0 on success, otherwise error
  */
-int ksmbd_vfs_setxattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_setxattr(const struct mnt_idmap *idmap,
 		       const struct path *path, const char *attr_name,
 		       void *attr_value, size_t attr_size, int flags,
 		       bool get_write)
@@ -1177,7 +1178,7 @@ int ksmbd_vfs_query_allocated_ranges(struct ksmbd_file *fp, loff_t start,
 	return ret;
 }
 
-int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_remove_xattr(const struct mnt_idmap *idmap,
 			   const struct path *path, char *attr_name,
 			   bool get_write)
 {
@@ -1202,7 +1203,7 @@ int ksmbd_vfs_unlink(struct file *filp)
 	const struct cred *saved_cred;
 	int err = 0;
 	struct dentry *dir, *dentry = filp->f_path.dentry;
-	struct mnt_idmap *idmap = file_mnt_idmap(filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(filp);
 
 	saved_cred = override_creds(filp->f_cred);
 	err = mnt_want_write(filp->f_path.mnt);
@@ -1471,7 +1472,7 @@ struct dentry *ksmbd_vfs_kern_path_create(struct ksmbd_work *work,
 	return dent;
 }
 
-int ksmbd_vfs_remove_acl_xattrs(struct mnt_idmap *idmap,
+int ksmbd_vfs_remove_acl_xattrs(const struct mnt_idmap *idmap,
 				const struct path *path)
 {
 	char *name, *xattr_list = NULL;
@@ -1511,7 +1512,7 @@ int ksmbd_vfs_remove_acl_xattrs(struct mnt_idmap *idmap,
 	return err;
 }
 
-int ksmbd_vfs_remove_sd_xattrs(struct mnt_idmap *idmap, const struct path *path)
+int ksmbd_vfs_remove_sd_xattrs(const struct mnt_idmap *idmap, const struct path *path)
 {
 	char *name, *xattr_list = NULL;
 	ssize_t xattr_list_len;
@@ -1540,7 +1541,7 @@ int ksmbd_vfs_remove_sd_xattrs(struct mnt_idmap *idmap, const struct path *path)
 	return err;
 }
 
-static struct xattr_smb_acl *ksmbd_vfs_make_xattr_posix_acl(struct mnt_idmap *idmap,
+static struct xattr_smb_acl *ksmbd_vfs_make_xattr_posix_acl(const struct mnt_idmap *idmap,
 							    struct inode *inode,
 							    int acl_type)
 {
@@ -1606,7 +1607,7 @@ static struct xattr_smb_acl *ksmbd_vfs_make_xattr_posix_acl(struct mnt_idmap *id
 }
 
 int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn,
-			   struct mnt_idmap *idmap,
+			   const struct mnt_idmap *idmap,
 			   const struct path *path,
 			   struct smb_ntsd *pntsd, int len,
 			   bool get_write)
@@ -1671,9 +1672,10 @@ int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn,
 	kfree(def_smb_acl);
 	return rc;
 }
+EXPORT_SYMBOL_IF_KUNIT(ksmbd_vfs_set_sd_xattr);
 
 int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn,
-			   struct mnt_idmap *idmap,
+			   const struct mnt_idmap *idmap,
 			   struct dentry *dentry,
 			   struct smb_ntsd **pntsd)
 {
@@ -1742,7 +1744,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn,
 	return rc;
 }
 
-int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_set_dos_attrib_xattr(const struct mnt_idmap *idmap,
 				   const struct path *path,
 				   struct xattr_dos_attrib *da,
 				   bool get_write)
@@ -1764,7 +1766,7 @@ int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap,
 	return err;
 }
 
-int ksmbd_vfs_get_dos_attrib_xattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_get_dos_attrib_xattr(const struct mnt_idmap *idmap,
 				   struct dentry *dentry,
 				   struct xattr_dos_attrib *da)
 {
@@ -1820,7 +1822,7 @@ void *ksmbd_vfs_init_kstat(char **p, struct ksmbd_kstat *ksmbd_kstat)
 }
 
 int ksmbd_vfs_fill_dentry_attrs(struct ksmbd_work *work,
-				struct mnt_idmap *idmap,
+				const struct mnt_idmap *idmap,
 				struct dentry *dentry,
 				struct ksmbd_kstat *ksmbd_kstat)
 {
@@ -1895,7 +1897,7 @@ int ksmbd_vfs_fill_dentry_attrs(struct ksmbd_work *work,
 	return 0;
 }
 
-ssize_t ksmbd_vfs_casexattr_len(struct mnt_idmap *idmap,
+ssize_t ksmbd_vfs_casexattr_len(const struct mnt_idmap *idmap,
 				struct dentry *dentry, char *attr_name,
 				int attr_name_len)
 {
@@ -2210,7 +2212,7 @@ void ksmbd_vfs_posix_lock_unblock(struct file_lock *flock)
 	locks_delete_block(flock);
 }
 
-int ksmbd_vfs_set_init_posix_acl(struct mnt_idmap *idmap,
+int ksmbd_vfs_set_init_posix_acl(const struct mnt_idmap *idmap,
 				 const struct path *path)
 {
 	struct posix_acl_state acl_state;
@@ -2263,7 +2265,7 @@ int ksmbd_vfs_set_init_posix_acl(struct mnt_idmap *idmap,
 	return rc;
 }
 
-int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap,
+int ksmbd_vfs_inherit_posix_acl(const struct mnt_idmap *idmap,
 				const struct path *path, struct inode *parent_inode)
 {
 	struct posix_acl *acls;
@@ -2326,7 +2328,7 @@ static int __ksmbd_vfs_set_compression(struct ksmbd_work *work,
 	const struct cred *saved_cred = NULL;
 	struct file_kattr fa;
 	struct dentry *dentry = fp->filp->f_path.dentry;
-	struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
+	const struct mnt_idmap *idmap = file_mnt_idmap(fp->filp);
 	u32 flags;
 	__le32 old_fattr;
 	int rc;
diff --git a/fs/smb/server/vfs.h b/fs/smb/server/vfs.h
index 55d099de71f527..216c76291fc143 100644
--- a/fs/smb/server/vfs.h
+++ b/fs/smb/server/vfs.h
@@ -74,7 +74,7 @@ struct ksmbd_kstat {
 };
 
 int ksmbd_vfs_lock_parent(struct dentry *parent, struct dentry *child);
-void ksmbd_vfs_query_maximal_access(struct mnt_idmap *idmap,
+void ksmbd_vfs_query_maximal_access(const struct mnt_idmap *idmap,
 				   struct dentry *dentry, __le32 *daccess);
 int ksmbd_vfs_create(struct ksmbd_work *work, const char *name, umode_t mode);
 int ksmbd_vfs_mkdir(struct ksmbd_work *work, const char *name, umode_t mode);
@@ -104,20 +104,25 @@ int ksmbd_vfs_copy_file_ranges(struct ksmbd_work *work,
 			       unsigned int *chunk_size_written,
 			       loff_t  *total_size_written);
 ssize_t ksmbd_vfs_listxattr(struct dentry *dentry, char **list);
-ssize_t ksmbd_vfs_getxattr(struct mnt_idmap *idmap,
+ssize_t ksmbd_vfs_getxattr(const struct mnt_idmap *idmap,
 			   struct dentry *dentry,
 			   char *xattr_name,
 			   char **xattr_buf);
-ssize_t ksmbd_vfs_casexattr_len(struct mnt_idmap *idmap,
+ssize_t ksmbd_vfs_xattr_len(const struct mnt_idmap *idmap,
+			    struct dentry *dentry, char *xattr_name);
+ssize_t ksmbd_vfs_getcasexattr(const struct mnt_idmap *idmap,
+			       struct dentry *dentry, char *attr_name,
+			       int attr_name_len, char **attr_value);
+ssize_t ksmbd_vfs_casexattr_len(const struct mnt_idmap *idmap,
 				struct dentry *dentry, char *attr_name,
 				int attr_name_len);
-int ksmbd_vfs_setxattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_setxattr(const struct mnt_idmap *idmap,
 		       const struct path *path, const char *attr_name,
 		       void *attr_value, size_t attr_size, int flags,
 		       bool get_write);
 int ksmbd_vfs_xattr_stream_name(char *stream_name, char **xattr_stream_name,
 				size_t *xattr_stream_name_size, int s_type);
-int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_remove_xattr(const struct mnt_idmap *idmap,
 			   const struct path *path, char *attr_name,
 			   bool get_write);
 int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *name,
@@ -147,33 +152,33 @@ int ksmbd_vfs_query_allocated_ranges(struct ksmbd_file *fp, loff_t start,
 int ksmbd_vfs_unlink(struct file *filp);
 void *ksmbd_vfs_init_kstat(char **p, struct ksmbd_kstat *ksmbd_kstat);
 int ksmbd_vfs_fill_dentry_attrs(struct ksmbd_work *work,
-				struct mnt_idmap *idmap,
+				const struct mnt_idmap *idmap,
 				struct dentry *dentry,
 				struct ksmbd_kstat *ksmbd_kstat);
 void ksmbd_vfs_posix_lock_wait(struct file_lock *flock);
 void ksmbd_vfs_posix_lock_unblock(struct file_lock *flock);
-int ksmbd_vfs_remove_acl_xattrs(struct mnt_idmap *idmap,
+int ksmbd_vfs_remove_acl_xattrs(const struct mnt_idmap *idmap,
 				const struct path *path);
-int ksmbd_vfs_remove_sd_xattrs(struct mnt_idmap *idmap, const struct path *path);
+int ksmbd_vfs_remove_sd_xattrs(const struct mnt_idmap *idmap, const struct path *path);
 int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn,
-			   struct mnt_idmap *idmap,
+			   const struct mnt_idmap *idmap,
 			   const struct path *path,
 			   struct smb_ntsd *pntsd, int len,
 			   bool get_write);
 int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn,
-			   struct mnt_idmap *idmap,
+			   const struct mnt_idmap *idmap,
 			   struct dentry *dentry,
 			   struct smb_ntsd **pntsd);
-int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_set_dos_attrib_xattr(const struct mnt_idmap *idmap,
 				   const struct path *path,
 				   struct xattr_dos_attrib *da,
 				   bool get_write);
-int ksmbd_vfs_get_dos_attrib_xattr(struct mnt_idmap *idmap,
+int ksmbd_vfs_get_dos_attrib_xattr(const struct mnt_idmap *idmap,
 				   struct dentry *dentry,
 				   struct xattr_dos_attrib *da);
-int ksmbd_vfs_set_init_posix_acl(struct mnt_idmap *idmap,
+int ksmbd_vfs_set_init_posix_acl(const struct mnt_idmap *idmap,
 				 const struct path *path);
-int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap,
+int ksmbd_vfs_inherit_posix_acl(const struct mnt_idmap *idmap,
 				const struct path *path,
 				struct inode *parent_inode);
 void ksmbd_vfs_update_compressed_fattr(struct dentry *dentry, __le32 *fattr);
diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c
index fd2c595f048688..7bc95fae4ac006 100644
--- a/fs/smb/server/vfs_cache.c
+++ b/fs/smb/server/vfs_cache.c
@@ -617,7 +617,6 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp)
 {
 	struct file *filp;
 	struct ksmbd_lock *smb_lock, *tmp_lock;
-	struct ksmbd_work *cn_work;
 
 	fd_limit_close();
 	ksmbd_remove_durable_fd(fp);
@@ -652,52 +651,6 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp)
 		kfree(smb_lock);
 	}
 
-	/*
-	 * Complete any CHANGE_NOTIFY left pending on this handle now that
-	 * it is closed. KSMBD never completes CHANGE_NOTIFY spontaneously
-	 * (no real change-notification backend), only on close -- matching
-	 * genuine SMB2/macOS smbfs semantics and avoiding the Finder
-	 * "directory changed, re-enumerate everything" loop.
-	 *
-	 * smb2_notify() on another connection can be adding to
-	 * notify_pendings under fp->f_lock at the same time this handle is
-	 * closed, and a client-sent CANCEL can concurrently be racing to
-	 * claim the same entry via smb2_notify_cancel_fn() (smb2pdu.c).
-	 * Pop one entry at a time under the lock via list_del_init() rather
-	 * than a bulk list_splice_init(): list_del_init() leaves the node
-	 * self-linked ("empty"), which is what the cancel path checks under
-	 * the same lock to tell whether it lost the race -- a bulk splice
-	 * would instead relink every entry into a shared local list, so an
-	 * entry claimed here would still read as "not empty" to a racing
-	 * cancel_fn, and both sides could end up freeing the same work.
-	 * ksmbd_conn_write() can sleep (it takes conn's write mutex), so it
-	 * must not be called while fp->f_lock is held -- release the lock
-	 * before processing each popped entry, then reacquire it for the
-	 * next.
-	 */
-	for (;;) {
-		spin_lock(&fp->f_lock);
-		if (list_empty(&fp->notify_pendings)) {
-			spin_unlock(&fp->f_lock);
-			break;
-		}
-		cn_work = list_first_entry(&fp->notify_pendings,
-					   struct ksmbd_work, notify_entry);
-		list_del_init(&cn_work->notify_entry);
-		spin_unlock(&fp->f_lock);
-
-		ksmbd_conn_write(cn_work);
-		/*
-		 * release_async_work() removes cn_work from
-		 * conn->async_requests, frees cancel_argv, and releases+zeroes
-		 * async_id -- all needed before ksmbd_free_work_struct(), which
-		 * only releases async_id itself if still nonzero (i.e. if this
-		 * hadn't already been done).
-		 */
-		release_async_work(cn_work);
-		ksmbd_free_work_struct(cn_work);
-	}
-
 	/*
 	 * Drop fp's strong reference on conn (taken in ksmbd_open_fd() /
 	 * ksmbd_reopen_durable_fd()).  Durable fps that reached the
@@ -1266,7 +1219,6 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp)
 	INIT_LIST_HEAD(&fp->blocked_works);
 	INIT_LIST_HEAD(&fp->node);
 	INIT_LIST_HEAD(&fp->lock_list);
-	INIT_LIST_HEAD(&fp->notify_pendings);
 	spin_lock_init(&fp->f_lock);
 	mutex_init(&fp->readdir_lock);
 	atomic_set(&fp->refcount, 1);
@@ -1877,9 +1829,21 @@ int ksmbd_validate_name_reconnect(struct ksmbd_share_config *share,
 		return -EACCES;
 	}
 
-	if (name && strcmp(&ab_pathname[share->path_sz + 1], name)) {
-		ksmbd_debug(SMB, "invalid name reconnect %s\n", name);
-		ret = -EINVAL;
+	if (name) {
+		size_t len = strlen(ab_pathname);
+
+		if (len == share->path_sz && !strncmp(ab_pathname, share->path, len)) {
+			/* the durable fp is the share root itself */
+			if (name[0])
+				ret = -EINVAL;
+		} else if (len <= share->path_sz ||
+			   strncmp(ab_pathname, share->path, share->path_sz) ||
+			   ab_pathname[share->path_sz] != '/' ||
+			   strcmp(&ab_pathname[share->path_sz + 1], name)) {
+			ret = -EINVAL;
+		}
+		if (ret)
+			ksmbd_debug(SMB, "invalid name reconnect %s\n", name);
 	}
 
 	kfree(pathname);
diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h
index 1884f6deb9d0e4..732ae26dd6a7db 100644
--- a/fs/smb/server/vfs_cache.h
+++ b/fs/smb/server/vfs_cache.h
@@ -162,12 +162,6 @@ struct ksmbd_file {
 	unsigned int			outstanding_requests;
 	unsigned int			outstanding_pre_requests;
 	struct ksmbd_lock_sequence	lock_seq[KSMBD_LOCK_SEQ_ARRAY_SIZE];
-
-	/*
-	 * Pending CHANGE_NOTIFY completions for this handle, sent with
-	 * STATUS_NOTIFY_CLEANUP when the handle is closed.
-	 */
-	struct list_head		notify_pendings;
 };
 
 static inline void set_ctx_actor(struct dir_context *ctx,
diff --git a/fs/splice.c b/fs/splice.c
index 9d8f63e2fd1abe..bc243ab8dc431b 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -177,9 +177,9 @@ static const struct pipe_buf_operations user_page_pipe_buf_ops = {
 
 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
 {
-	smp_mb();
-	if (waitqueue_active(&pipe->rd_wait))
-		wake_up_interruptible(&pipe->rd_wait);
+	if (wq_has_sleeper(&pipe->rd_wait))
+		wake_up_interruptible_poll(&pipe->rd_wait,
+					   EPOLLIN | EPOLLRDNORM);
 	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 }
 
@@ -413,9 +413,9 @@ EXPORT_SYMBOL(nosteal_pipe_buf_ops);
 
 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
 {
-	smp_mb();
-	if (waitqueue_active(&pipe->wr_wait))
-		wake_up_interruptible(&pipe->wr_wait);
+	if (wq_has_sleeper(&pipe->wr_wait))
+		wake_up_interruptible_poll(&pipe->wr_wait,
+					   EPOLLOUT | EPOLLWRNORM);
 	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 }
 
@@ -1009,21 +1009,14 @@ ssize_t vfs_splice_read(struct file *in, loff_t *ppos,
 }
 EXPORT_SYMBOL_GPL(vfs_splice_read);
 
-/**
- * splice_direct_to_actor - splices data directly between two non-pipes
- * @in:		file to splice from
- * @sd:		actor information on where to splice to
- * @actor:	handles the data splicing
- *
- * Description:
- *    This is a special case helper to splice directly between two
- *    points, without requiring an explicit pipe. Internally an allocated
- *    pipe is cached in the process, and reused during the lifetime of
- *    that process.
- *
+/*
+ * This is a special case helper to splice directly between two
+ * points, without requiring an explicit pipe. Internally an allocated
+ * pipe is cached in the process, and reused during the lifetime of
+ * that process.
  */
-ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
-			       splice_direct_actor *actor)
+static ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
+				      splice_direct_actor *actor)
 {
 	struct pipe_inode_info *pipe;
 	ssize_t ret, bytes;
@@ -1147,7 +1140,42 @@ ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
 
 	goto done;
 }
-EXPORT_SYMBOL(splice_direct_to_actor);
+
+/**
+ * vfs_splice_to_actor - call an actor on data read from a file
+ * @in:		file to read from
+ * @pos:	file offset
+ * @count:	maximum number of bytes to read
+ * @actor:	callback to process a pipe's worth of data
+ * @private:	private data passed to @actor
+ *
+ * Read up to @count worth of data from @in at @pos, and call @actor
+ * when the hidden pipe used to buffer the data is full. Ensures the
+ * read is allowed using rw_verify_area() and emits fsnotify access
+ * events. @in must be seekable (FMODE_LSEEK).
+ *
+ * Return: The number of bytes spliced, or a negative errno.
+ */
+ssize_t vfs_splice_to_actor(struct file *in, loff_t pos, size_t count,
+			    splice_direct_actor *actor, void *private)
+{
+	struct splice_desc sd = {
+		.total_len	= count,
+		.pos		= pos,
+		.u.data		= private,
+	};
+	ssize_t ret;
+
+	ret = rw_verify_area(READ, in, &sd.pos, sd.total_len);
+	if (ret < 0)
+		return ret;
+
+	ret = splice_direct_to_actor(in, &sd, actor);
+	if (ret >= 0)
+		fsnotify_access(in);
+	return ret;
+}
+EXPORT_SYMBOL(vfs_splice_to_actor);
 
 static int direct_splice_actor(struct pipe_inode_info *pipe,
 			       struct splice_desc *sd)
diff --git a/fs/squashfs/fragment.c b/fs/squashfs/fragment.c
index 49602b9a42e19e..c46e946fa47447 100644
--- a/fs/squashfs/fragment.c
+++ b/fs/squashfs/fragment.c
@@ -69,9 +69,11 @@ __le64 *squashfs_read_fragment_index_table(struct super_block *sb,
 	/*
 	 * Sanity check, length bytes should not extend into the next table -
 	 * this check also traps instances where fragment_table_start is
-	 * incorrectly larger than the next table start
+	 * incorrectly larger than the next table start.  Both values are read
+	 * from the filesystem image, so compare without adding them.
 	 */
-	if (fragment_table_start + length > next_table)
+	if (fragment_table_start > next_table ||
+	    length > next_table - fragment_table_start)
 		return ERR_PTR(-EINVAL);
 
 	table = squashfs_read_table(sb, fragment_table_start, length);
diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h
index a955d9369749f2..93436c7d80c973 100644
--- a/fs/squashfs/squashfs_fs.h
+++ b/fs/squashfs/squashfs_fs.h
@@ -136,7 +136,7 @@ static inline int squashfs_block_size(__le32 raw)
 
 /* fragment and fragment table defines */
 #define SQUASHFS_FRAGMENT_BYTES(A)	\
-				((A) * sizeof(struct squashfs_fragment_entry))
+			((u64)(A) * sizeof(struct squashfs_fragment_entry))
 
 #define SQUASHFS_FRAGMENT_INDEX(A)	(SQUASHFS_FRAGMENT_BYTES(A) / \
 					SQUASHFS_METADATA_SIZE)
diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c
index 0a4ff3ec9c8cd3..6610af24144934 100644
--- a/fs/squashfs/xz_wrapper.c
+++ b/fs/squashfs/xz_wrapper.c
@@ -57,10 +57,10 @@ static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk,
 
 		opts->dict_size = le32_to_cpu(comp_opts->dictionary_size);
 
-		/* the dictionary size should be 2^n or 2^n+2^(n+1) */
+		/* the dictionary size should be positive and 2^n or 2^n+2^(n+1) */
 		n = ffs(opts->dict_size) - 1;
-		if (opts->dict_size != (1 << n) && opts->dict_size != (1 << n) +
-						(1 << (n + 1))) {
+		if (opts->dict_size <= 0 || (opts->dict_size != (1 << n) &&
+			opts->dict_size != (1 << n) + (1 << (n + 1)))) {
 			err = -EIO;
 			goto out;
 		}
diff --git a/fs/stat.c b/fs/stat.c
index c461c305423408..a9b7383d538da8 100644
--- a/fs/stat.c
+++ b/fs/stat.c
@@ -79,7 +79,7 @@ EXPORT_SYMBOL(fill_mg_cmtime);
  * uid and gid filds. On non-idmapped mounts or if permission checking is to be
  * performed on the raw inode simply pass @nop_mnt_idmap.
  */
-void generic_fillattr(struct mnt_idmap *idmap, u32 request_mask,
+void generic_fillattr(const struct mnt_idmap *idmap, u32 request_mask,
 		      struct inode *inode, struct kstat *stat)
 {
 	vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode);
@@ -181,7 +181,7 @@ EXPORT_SYMBOL_GPL(generic_fill_statx_atomic_writes);
 int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
 		      u32 request_mask, unsigned int query_flags)
 {
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct inode *inode = d_backing_inode(path->dentry);
 
 	memset(stat, 0, sizeof(*stat));
diff --git a/fs/super.c b/fs/super.c
index 9d402521352127..b1d4add11b77cc 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -419,15 +419,19 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags,
 void put_super(struct super_block *s)
 {
 	if (refcount_dec_and_test(&s->s_passive)) {
+		struct file_system_type *type = s->s_type;
 
 		spin_lock(&sb_lock);
 		list_del_init(&s->s_list);
+		hlist_del_init(&s->s_instances);
 		spin_unlock(&sb_lock);
 
 		WARN_ON(s->s_dentry_lru.node);
 		WARN_ON(s->s_inode_lru.node);
 		WARN_ON(s->s_mounts);
 		call_rcu(&s->rcu, destroy_super_rcu);
+		/* The unlink above may touch type->fs_supers, so drop it last. */
+		put_filesystem(type);
 	}
 }
 
@@ -544,17 +548,6 @@ static void kill_super_notify(struct super_block *sb)
 	if (sb->s_flags & SB_DEAD)
 		return;
 
-	/*
-	 * Remove it from @fs_supers so it isn't found by new
-	 * sget_fc() walkers anymore. Any concurrent mounter still
-	 * managing to grab a temporary reference is guaranteed to
-	 * already see SB_DYING and will wait until we notify them about
-	 * SB_DEAD.
-	 */
-	spin_lock(&sb_lock);
-	hlist_del_init(&sb->s_instances);
-	spin_unlock(&sb_lock);
-
 	/* Drop sget_fc()'s claim; a never-registered entry stays with the sb. */
 	if (sb->s_super_dev->sd_dev) {
 		super_dev_put(sb->s_super_dev);
@@ -563,11 +556,15 @@ static void kill_super_notify(struct super_block *sb)
 
 	/*
 	 * Let concurrent mounts know that this thing is really dead.
-	 * We don't need @sb->s_umount here as every concurrent caller
-	 * will see SB_DYING and either discard the superblock or wait
-	 * for SB_DEAD.
+	 * sget_fc() skips SB_DEAD superblocks and calls test() under
+	 * sb_lock, so set it under sb_lock: once we return no test()
+	 * runs on this superblock anymore and none will start. Everyone
+	 * else already saw SB_DYING and either discarded the superblock
+	 * or waits for SB_DEAD.
 	 */
+	spin_lock(&sb_lock);
 	super_wake(sb, SB_DEAD);
+	spin_unlock(&sb_lock);
 }
 
 /**
@@ -594,7 +591,6 @@ void deactivate_locked_super(struct super_block *s)
 		list_lru_destroy(&s->s_dentry_lru);
 		list_lru_destroy(&s->s_inode_lru);
 
-		put_filesystem(fs);
 		put_super(s);
 	} else {
 		super_unlock_excl(s);
@@ -781,12 +777,12 @@ void generic_shutdown_super(struct super_block *sb)
 	}
 	/*
 	 * Broadcast to everyone that grabbed a temporary reference to this
-	 * superblock before we removed it from @fs_supers that the superblock
-	 * is dying. Every walker of @fs_supers outside of sget_fc() will now
-	 * discard this superblock and treat it as dead.
+	 * superblock that it is dying. Every walker of @fs_supers outside
+	 * of sget_fc() will now discard this superblock and treat it as
+	 * dead.
 	 *
-	 * We leave the superblock on @fs_supers so it can be found by
-	 * sget_fc() until we passed sb->kill_sb().
+	 * sget_fc() keeps finding the superblock until SB_DEAD is set, so
+	 * a concurrent mounter waits until we passed sb->kill_sb().
 	 */
 	super_wake(sb, SB_DYING);
 	super_unlock_excl(sb);
@@ -865,6 +861,9 @@ struct super_block *sget_fc(struct fs_context *fc,
 	spin_lock(&sb_lock);
 	if (test) {
 		hlist_for_each_entry(old, &fc->fs_type->fs_supers, s_instances) {
+			/* Only unlinked at the last passive reference. */
+			if (super_flags(old, SB_DEAD))
+				continue;
 			if (test(old, fc))
 				goto share_extant_sb;
 		}
@@ -1375,7 +1374,17 @@ static int test_single_super(struct super_block *s, struct fs_context *fc)
 	return 1;
 }
 
-static int vfs_get_super(struct fs_context *fc,
+/**
+ * get_tree_super - Get a superblock, optionally sharing an existing one
+ * @fc: The filesystem context holding the parameters
+ * @test: Comparison function to find a matching existing superblock, or NULL
+ * @fill_super: Helper to initialise a new superblock
+ *
+ * If @test is non-NULL and matches an existing superblock, that superblock is
+ * reused; otherwise a new anonymous superblock is created and initialised with
+ * @fill_super.  Passing NULL for @test always creates a new superblock.
+ */
+int get_tree_super(struct fs_context *fc,
 		int (*test)(struct super_block *, struct fs_context *),
 		int (*fill_super)(struct super_block *sb,
 				  struct fs_context *fc))
@@ -1402,12 +1411,13 @@ static int vfs_get_super(struct fs_context *fc,
 	deactivate_locked_super(sb);
 	return err;
 }
+EXPORT_SYMBOL(get_tree_super);
 
 int get_tree_nodev(struct fs_context *fc,
 		  int (*fill_super)(struct super_block *sb,
 				    struct fs_context *fc))
 {
-	return vfs_get_super(fc, NULL, fill_super);
+	return get_tree_super(fc, NULL, fill_super);
 }
 EXPORT_SYMBOL(get_tree_nodev);
 
@@ -1415,7 +1425,7 @@ int get_tree_single(struct fs_context *fc,
 		  int (*fill_super)(struct super_block *sb,
 				    struct fs_context *fc))
 {
-	return vfs_get_super(fc, test_single_super, fill_super);
+	return get_tree_super(fc, test_single_super, fill_super);
 }
 EXPORT_SYMBOL(get_tree_single);
 
@@ -1425,7 +1435,7 @@ int get_tree_keyed(struct fs_context *fc,
 		void *key)
 {
 	fc->s_fs_info = key;
-	return vfs_get_super(fc, test_keyed_super, fill_super);
+	return get_tree_super(fc, test_keyed_super, fill_super);
 }
 EXPORT_SYMBOL(get_tree_keyed);
 
diff --git a/fs/tests/.kunitconfig b/fs/tests/.kunitconfig
new file mode 100644
index 00000000000000..de67125a9421a0
--- /dev/null
+++ b/fs/tests/.kunitconfig
@@ -0,0 +1,2 @@
+CONFIG_KUNIT=y
+CONFIG_FDTABLE_KUNIT_TEST=y
diff --git a/fs/tests/fdtable_kunit.c b/fs/tests/fdtable_kunit.c
new file mode 100644
index 00000000000000..c5b028264557d7
--- /dev/null
+++ b/fs/tests/fdtable_kunit.c
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <kunit/test.h>
+#include <linux/fdtable.h>
+#include <linux/file.h>
+
+static void test_alloc_fdtable(struct kunit *test)
+{
+	struct fdtable *fdt;
+	unsigned int slots = 64;
+
+	fdt = alloc_fdtable(slots);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fdt);
+
+	/* Check that max_fds is set correctly and is >= slots */
+	KUNIT_EXPECT_GE(test, fdt->max_fds, slots);
+
+	/* Check that fd is allocated */
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fdt->fd);
+
+	/*
+	 * Check dynamic object size of fdt->fd if compiler supports
+	 * __counted_by_ptr.
+	 */
+#ifdef CONFIG_CC_HAS_COUNTED_BY_PTR
+	KUNIT_EXPECT_EQ(test, __struct_size(fdt->fd),
+			fdt->max_fds * sizeof(struct file *));
+#endif
+
+	__free_fdtable(fdt);
+}
+
+static void test_dup_fd(struct kunit *test)
+{
+	struct files_struct *newf;
+	struct fdtable *fdt;
+
+	newf = dup_fd(&init_files, NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, newf);
+
+	fdt = rcu_dereference_raw(newf->fdt);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fdt);
+
+	/* Check that max_fds is set correctly and is >= NR_OPEN_DEFAULT */
+	KUNIT_EXPECT_GE(test, fdt->max_fds, NR_OPEN_DEFAULT);
+
+	/* Check that fd is allocated */
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fdt->fd);
+
+	/*
+	 * Check dynamic object size of fdt->fd if compiler supports
+	 * __counted_by_ptr.
+	 */
+#ifdef CONFIG_CC_HAS_COUNTED_BY_PTR
+	KUNIT_EXPECT_EQ(test, __struct_size(fdt->fd),
+			fdt->max_fds * sizeof(struct file *));
+#endif
+
+	put_files_struct(newf);
+}
+
+static struct kunit_case fdtable_test_cases[] = {
+	KUNIT_CASE(test_alloc_fdtable),
+	KUNIT_CASE(test_dup_fd),
+	{}
+};
+
+static struct kunit_suite fdtable_test_suite = {
+	.name = "fdtable",
+	.test_cases = fdtable_test_cases,
+};
+
+kunit_test_suite(fdtable_test_suite);
diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c
index 6e3513b13cfa21..6f6daac88621ed 100644
--- a/fs/tracefs/event_inode.c
+++ b/fs/tracefs/event_inode.c
@@ -182,7 +182,7 @@ static void update_attr(struct eventfs_attr *attr, struct iattr *iattr)
 	}
 }
 
-static int eventfs_set_attr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int eventfs_set_attr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			    struct iattr *iattr)
 {
 	const struct eventfs_entry *entry;
diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c
index f3d6188a3b7b24..5020365ca70487 100644
--- a/fs/tracefs/inode.c
+++ b/fs/tracefs/inode.c
@@ -94,7 +94,7 @@ static struct tracefs_dir_ops {
 	int (*rmdir)(const char *name);
 } tracefs_ops __ro_after_init;
 
-static struct dentry *tracefs_syscall_mkdir(struct mnt_idmap *idmap,
+static struct dentry *tracefs_syscall_mkdir(const struct mnt_idmap *idmap,
 					    struct inode *inode, struct dentry *dentry,
 					    umode_t mode)
 {
@@ -189,14 +189,14 @@ static void set_tracefs_inode_owner(struct inode *inode)
 		inode->i_gid = gid;
 }
 
-static int tracefs_permission(struct mnt_idmap *idmap,
+static int tracefs_permission(const struct mnt_idmap *idmap,
 			      struct inode *inode, int mask)
 {
 	set_tracefs_inode_owner(inode);
 	return generic_permission(idmap, inode, mask);
 }
 
-static int tracefs_getattr(struct mnt_idmap *idmap,
+static int tracefs_getattr(const struct mnt_idmap *idmap,
 			   const struct path *path, struct kstat *stat,
 			   u32 request_mask, unsigned int flags)
 {
@@ -207,7 +207,7 @@ static int tracefs_getattr(struct mnt_idmap *idmap,
 	return 0;
 }
 
-static int tracefs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int tracefs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			   struct iattr *attr)
 {
 	unsigned int ia_valid = attr->ia_valid;
diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c
index 23ec924162d63b..c67954f6bee008 100644
--- a/fs/ubifs/dir.c
+++ b/fs/ubifs/dir.c
@@ -302,7 +302,7 @@ static int ubifs_prepare_create(struct inode *dir, struct dentry *dentry,
 	return fscrypt_setup_filename(dir, &dentry->d_name, 0, nm);
 }
 
-static int ubifs_create(struct mnt_idmap *idmap, struct inode *dir,
+static int ubifs_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -440,7 +440,7 @@ static void unlock_2_inodes(struct inode *inode1, struct inode *inode2)
 	mutex_unlock(&ubifs_inode(inode1)->ui_mutex);
 }
 
-static int ubifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int ubifs_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct file *file, umode_t mode)
 {
 	struct dentry *dentry = file->f_path.dentry;
@@ -1002,7 +1002,7 @@ static int ubifs_rmdir(struct inode *dir, struct dentry *dentry)
 	return err;
 }
 
-static struct dentry *ubifs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ubifs_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -1077,7 +1077,7 @@ static struct dentry *ubifs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	return ERR_PTR(err);
 }
 
-static int ubifs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int ubifs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct inode *inode;
@@ -1170,7 +1170,7 @@ static int ubifs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int ubifs_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int ubifs_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, const char *symname)
 {
 	struct inode *inode;
@@ -1642,7 +1642,7 @@ static int ubifs_xrename(struct inode *old_dir, struct dentry *old_dentry,
 	return err;
 }
 
-static int ubifs_rename(struct mnt_idmap *idmap,
+static int ubifs_rename(const struct mnt_idmap *idmap,
 			struct inode *old_dir, struct dentry *old_dentry,
 			struct inode *new_dir, struct dentry *new_dentry,
 			unsigned int flags)
@@ -1667,7 +1667,7 @@ static int ubifs_rename(struct mnt_idmap *idmap,
 	return do_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
 }
 
-int ubifs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ubifs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, u32 request_mask, unsigned int flags)
 {
 	loff_t size;
diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c
index e73c28b12f97fd..99dd52715eee97 100644
--- a/fs/ubifs/file.c
+++ b/fs/ubifs/file.c
@@ -12,14 +12,14 @@
  * This file implements VFS file and inode operations for regular files, device
  * nodes and symlinks as well as address space operations.
  *
- * UBIFS uses 2 page flags: @PG_private and @PG_checked. @PG_private is set if
+ * UBIFS uses folio->private and page flag @PG_checked. folio->private is set if
  * the page is dirty and is used for optimization purposes - dirty pages are
- * not budgeted so the flag shows that 'ubifs_write_end()' should not release
+ * not budgeted so it shows that 'ubifs_write_end()' should not release
  * the budget for this page. The @PG_checked flag is set if full budgeting is
  * required for the page e.g., when it corresponds to a file hole or it is
  * beyond the file size. The budgeting is done in 'ubifs_write_begin()', because
  * it is OK to fail in this function, and the budget is released in
- * 'ubifs_write_end()'. So the @PG_private and @PG_checked flags carry
+ * 'ubifs_write_end()'. So the folio->private and the @PG_checked flag carry
  * information about how the page was budgeted, to make it possible to release
  * the budget properly.
  *
@@ -1251,7 +1251,7 @@ static int do_setattr(struct ubifs_info *c, struct inode *inode,
 	return err;
 }
 
-int ubifs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ubifs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr)
 {
 	int err;
@@ -1509,7 +1509,7 @@ static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf)
 	 *
 	 * At the moment we do not know whether the folio is dirty or not, so we
 	 * assume that it is not and budget for a new folio. We could look at
-	 * the @PG_private flag and figure this out, but we may race with write
+	 * folio->private and figure this out, but we may race with write
 	 * back and the folio state may change by the time we lock it, so this
 	 * would need additional care. We do not bother with this at the
 	 * moment, although it might be good idea to do. Instead, we allocate
@@ -1611,7 +1611,7 @@ static const char *ubifs_get_link(struct dentry *dentry,
 	return fscrypt_get_symlink(inode, ui->data, ui->data_len, done);
 }
 
-static int ubifs_symlink_getattr(struct mnt_idmap *idmap,
+static int ubifs_symlink_getattr(const struct mnt_idmap *idmap,
 				 const struct path *path, struct kstat *stat,
 				 u32 request_mask, unsigned int query_flags)
 {
diff --git a/fs/ubifs/ioctl.c b/fs/ubifs/ioctl.c
index 79536b2e3d7a1a..5c34f895bd4e12 100644
--- a/fs/ubifs/ioctl.c
+++ b/fs/ubifs/ioctl.c
@@ -144,7 +144,7 @@ int ubifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-int ubifs_fileattr_set(struct mnt_idmap *idmap,
+int ubifs_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/ubifs/ubifs.h b/fs/ubifs/ubifs.h
index 00db0d19a85e38..b85b45a0564bc0 100644
--- a/fs/ubifs/ubifs.h
+++ b/fs/ubifs/ubifs.h
@@ -2020,7 +2020,7 @@ int ubifs_calc_dark(const struct ubifs_info *c, int spc);
 
 /* file.c */
 int ubifs_fsync(struct file *file, loff_t start, loff_t end, int datasync);
-int ubifs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ubifs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr);
 int ubifs_update_time(struct inode *inode, enum fs_update_time type,
 		      unsigned int flags);
@@ -2028,7 +2028,7 @@ int ubifs_update_time(struct inode *inode, enum fs_update_time type,
 /* dir.c */
 struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir,
 			      umode_t mode, bool is_xattr);
-int ubifs_getattr(struct mnt_idmap *idmap, const struct path *path,
+int ubifs_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, u32 request_mask, unsigned int flags);
 int ubifs_check_dir_empty(struct inode *dir);
 
@@ -2083,7 +2083,7 @@ void ubifs_destroy_size_tree(struct ubifs_info *c);
 
 /* ioctl.c */
 int ubifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int ubifs_fileattr_set(struct mnt_idmap *idmap,
+int ubifs_fileattr_set(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, struct file_kattr *fa);
 long ubifs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 void ubifs_set_inode_flags(struct inode *inode);
diff --git a/fs/ubifs/xattr.c b/fs/ubifs/xattr.c
index b5a9ab9d8a10ad..3b0e8a270f5944 100644
--- a/fs/ubifs/xattr.c
+++ b/fs/ubifs/xattr.c
@@ -660,7 +660,7 @@ static int xattr_get(const struct xattr_handler *handler,
 }
 
 static int xattr_set(const struct xattr_handler *handler,
-			   struct mnt_idmap *idmap,
+			   const struct mnt_idmap *idmap,
 			   struct dentry *dentry, struct inode *inode,
 			   const char *name, const void *value,
 			   size_t size, int flags)
diff --git a/fs/udf/file.c b/fs/udf/file.c
index 57d11606a2a776..02e9314818dc64 100644
--- a/fs/udf/file.c
+++ b/fs/udf/file.c
@@ -212,7 +212,7 @@ const struct file_operations udf_file_operations = {
 	.setlease		= generic_setlease,
 };
 
-static int udf_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int udf_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/udf/inode.c b/fs/udf/inode.c
index e45e546a739a90..71386e7ac7965a 100644
--- a/fs/udf/inode.c
+++ b/fs/udf/inode.c
@@ -442,6 +442,13 @@ int udf_expand_file_adinicb(struct inode *inode)
 	err = udf_map_block(inode, &map);
 	if (err < 0)
 		goto restore;
+	/*
+	 * The block may have held metadata that is still dirty in the block
+	 * device page cache (e.g. the file entry of a deleted inode). Make
+	 * sure writeback of that buffer cannot overwrite our data.
+	 */
+	if (map.oflags & UDF_BLK_NEW)
+		clean_bdev_aliases(inode->i_sb->s_bdev, map.pblk, 1);
 
 	folio_mark_dirty(folio);
 	folio_unlock(folio);
@@ -1336,6 +1343,26 @@ int udf_setsize(struct inode *inode, loff_t newsize)
 	return err;
 }
 
+/*
+ * Verify validity of struct deviceSpec on disk. udf_get_extendedattr() has
+ * already verified the generic header and made sure attribute fits in the
+ * inode so we just have to make sure attribute space is large enough for
+ * deviceSpec struct and required impUse information.
+ */
+static bool udf_device_spec_valid(struct deviceSpec *dsea)
+{
+	u32 attr_length, imp_use_length;
+
+	attr_length = le32_to_cpu(dsea->attrLength);
+	imp_use_length = le32_to_cpu(dsea->impUseLength);
+	if (attr_length < sizeof(struct deviceSpec) ||
+	    imp_use_length < sizeof(struct regid) ||
+	    imp_use_length > attr_length - sizeof(struct deviceSpec))
+		return false;
+
+	return true;
+}
+
 /*
  * Maximum length of linked list formed by ICB hierarchy. The chosen number is
  * arbitrary - just that we hopefully don't limit any real use of rewritten
@@ -1654,13 +1681,19 @@ static int udf_read_inode(struct inode *inode, bool hidden_inode)
 	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
 		struct deviceSpec *dsea =
 			(struct deviceSpec *)udf_get_extendedattr(inode, 12, 1);
-		if (dsea) {
-			init_special_inode(inode, inode->i_mode,
+
+		if (IS_ERR(dsea)) {
+			ret = PTR_ERR(dsea);
+			goto out;
+		}
+		/* Device inodes must have a device spec attribute */
+		if (!dsea || !udf_device_spec_valid(dsea)) {
+			ret = -EFSCORRUPTED;
+			goto out;
+		}
+		init_special_inode(inode, inode->i_mode,
 				MKDEV(le32_to_cpu(dsea->majorDeviceIdent),
 				      le32_to_cpu(dsea->minorDeviceIdent)));
-			/* Developer ID ??? */
-		} else
-			goto out;
 	}
 	ret = 0;
 out:
@@ -1757,6 +1790,7 @@ int udf_write_inode(struct inode *inode, struct writeback_control *wbc)
 	struct udf_sb_info *sbi = UDF_SB(inode->i_sb);
 	unsigned char blocksize_bits = inode->i_sb->s_blocksize_bits;
 	struct udf_inode_info *iinfo = UDF_I(inode);
+	int err;
 
 	bh = sb_getblk(inode->i_sb,
 			udf_get_lb_pblock(inode->i_sb, &iinfo->i_location, 0));
@@ -1816,11 +1850,21 @@ int udf_write_inode(struct inode *inode, struct writeback_control *wbc)
 		struct regid *eid;
 		struct deviceSpec *dsea =
 			(struct deviceSpec *)udf_get_extendedattr(inode, 12, 1);
+
+		/* Validity of extended attrs was checked on load */
+		if (WARN_ON_ONCE(IS_ERR(dsea))) {
+			err = PTR_ERR(dsea);
+			goto out_unlock;
+		}
 		if (!dsea) {
 			dsea = (struct deviceSpec *)
 				udf_add_extendedattr(inode,
 						     sizeof(struct deviceSpec) +
 						     sizeof(struct regid), 12, 0x3);
+			if (IS_ERR(dsea)) {
+				err = PTR_ERR(dsea);
+				goto out_unlock;
+			}
 			dsea->attrType = cpu_to_le32(12);
 			dsea->attrSubtype = 1;
 			dsea->attrLength = cpu_to_le32(
@@ -1962,6 +2006,11 @@ int udf_write_inode(struct inode *inode, struct writeback_control *wbc)
 	set_inode_metadata_writeback(inode);
 
 	return 0;
+
+out_unlock:
+	unlock_buffer(bh);
+	brelse(bh);
+	return err;
 }
 
 struct inode *__udf_iget(struct super_block *sb, struct kernel_lb_addr *ino,
diff --git a/fs/udf/misc.c b/fs/udf/misc.c
index 6928e378fbbdcb..a2084dfbfbd66f 100644
--- a/fs/udf/misc.c
+++ b/fs/udf/misc.c
@@ -58,7 +58,7 @@ struct genericFormat *udf_add_extendedattr(struct inode *inode, uint32_t size,
 					cpu_to_le16(TAG_IDENT_EAHD) ||
 			    le32_to_cpu(eahd->descTag.tagLocation) !=
 					iinfo->i_location.logicalBlockNum)
-				return NULL;
+				return ERR_PTR(-EFSCORRUPTED);
 		} else {
 			struct udf_sb_info *sbi = UDF_SB(inode->i_sb);
 
@@ -122,7 +122,7 @@ struct genericFormat *udf_add_extendedattr(struct inode *inode, uint32_t size,
 		return (struct genericFormat *)&ea[offset];
 	}
 
-	return NULL;
+	return ERR_PTR(-ENOSPC);
 }
 
 struct genericFormat *udf_get_extendedattr(struct inode *inode, uint32_t type,
@@ -144,7 +144,7 @@ struct genericFormat *udf_get_extendedattr(struct inode *inode, uint32_t type,
 				cpu_to_le16(TAG_IDENT_EAHD) ||
 		    le32_to_cpu(eahd->descTag.tagLocation) !=
 				iinfo->i_location.logicalBlockNum)
-			return NULL;
+			return ERR_PTR(-EFSCORRUPTED);
 
 		if (type < 2048)
 			offset = sizeof(struct extendedAttrHeaderDesc);
@@ -153,16 +153,17 @@ struct genericFormat *udf_get_extendedattr(struct inode *inode, uint32_t type,
 		else
 			offset = le32_to_cpu(eahd->appAttrLocation);
 
-		while (offset + sizeof(*gaf) < iinfo->i_lenEAttr) {
+		while (offset <
+		       iinfo->i_lenEAttr - sizeof(struct genericFormat)) {
 			uint32_t attrLength;
 
 			gaf = (struct genericFormat *)&ea[offset];
 			attrLength = le32_to_cpu(gaf->attrLength);
 
 			/* Detect undersized elements and buffer overflows */
-			if ((attrLength < sizeof(*gaf)) ||
-			    (attrLength > (iinfo->i_lenEAttr - offset)))
-				break;
+			if (attrLength < sizeof(struct genericFormat) ||
+			    attrLength > iinfo->i_lenEAttr - offset)
+				return ERR_PTR(-EFSCORRUPTED);
 
 			if (le32_to_cpu(gaf->attrType) == type &&
 					gaf->attrSubtype == subtype)
diff --git a/fs/udf/namei.c b/fs/udf/namei.c
index b90841ac0a4031..42d000fe9d6bc5 100644
--- a/fs/udf/namei.c
+++ b/fs/udf/namei.c
@@ -370,7 +370,7 @@ static int udf_add_nondir(struct dentry *dentry, struct inode *inode)
 	return 0;
 }
 
-static int udf_create(struct mnt_idmap *idmap, struct inode *dir,
+static int udf_create(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode = udf_new_inode(dir, mode);
@@ -386,7 +386,7 @@ static int udf_create(struct mnt_idmap *idmap, struct inode *dir,
 	return udf_add_nondir(dentry, inode);
 }
 
-static int udf_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+static int udf_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct file *file, umode_t mode)
 {
 	struct inode *inode = udf_new_inode(dir, mode);
@@ -403,7 +403,7 @@ static int udf_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	return finish_open_simple(file, 0);
 }
 
-static int udf_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int udf_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct inode *inode;
@@ -419,7 +419,7 @@ static int udf_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return udf_add_nondir(dentry, inode);
 }
 
-static struct dentry *udf_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *udf_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -567,7 +567,7 @@ static int udf_unlink(struct inode *dir, struct dentry *dentry)
 	return ret;
 }
 
-static int udf_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int udf_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, const char *symname)
 {
 	struct inode *inode;
@@ -762,7 +762,7 @@ static int udf_link(struct dentry *old_dentry, struct inode *dir,
 /* Anybody can rename anything with this: the permission checks are left to the
  * higher-level routines.
  */
-static int udf_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int udf_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/udf/super.c b/fs/udf/super.c
index 2ba5973ef4dd45..5351755aca3ecc 100644
--- a/fs/udf/super.c
+++ b/fs/udf/super.c
@@ -1617,7 +1617,10 @@ static bool udf_lvid_valid(struct super_block *sb,
 
 	parts = le32_to_cpu(lvid->numOfPartitions);
 	impuselen = le32_to_cpu(lvid->lengthOfImpUse);
-	if (parts >= sb->s_blocksize || impuselen >= sb->s_blocksize ||
+	if (sizeof(struct logicalVolIntegrityDescImpUse) > impuselen ||
+	    impuselen >= sb->s_blocksize)
+		return false;
+	if (parts >= sb->s_blocksize ||
 	    sizeof(struct logicalVolIntegrityDesc) + impuselen +
 	    2 * parts * sizeof(u32) > sb->s_blocksize)
 		return false;
diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c
index a05d1888a2babe..df41bf5a05b1b6 100644
--- a/fs/udf/symlink.c
+++ b/fs/udf/symlink.c
@@ -133,7 +133,7 @@ static int udf_symlink_filler(struct file *file, struct folio *folio)
 	return err;
 }
 
-static int udf_symlink_getattr(struct mnt_idmap *idmap,
+static int udf_symlink_getattr(const struct mnt_idmap *idmap,
 			       const struct path *path, struct kstat *stat,
 			       u32 request_mask, unsigned int flags)
 {
diff --git a/fs/ufs/dir.c b/fs/ufs/dir.c
index ce43cf20b07c98..e96174b738b4c3 100644
--- a/fs/ufs/dir.c
+++ b/fs/ufs/dir.c
@@ -213,7 +213,7 @@ static void *ufs_get_folio(struct inode *dir, unsigned long n,
 static unsigned
 ufs_last_byte(struct inode *inode, unsigned long page_nr)
 {
-	unsigned last_byte = inode->i_size;
+	u64 last_byte = inode->i_size;
 
 	last_byte -= page_nr << PAGE_SHIFT;
 	if (last_byte > PAGE_SIZE)
diff --git a/fs/ufs/inode.c b/fs/ufs/inode.c
index 440d014cc5ed58..c9ff8673fa668b 100644
--- a/fs/ufs/inode.c
+++ b/fs/ufs/inode.c
@@ -1195,7 +1195,7 @@ static int ufs_truncate(struct inode *inode, loff_t size)
 	return err;
 }
 
-int ufs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int ufs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/fs/ufs/namei.c b/fs/ufs/namei.c
index 6703f3bcf76fc7..d45347a2574196 100644
--- a/fs/ufs/namei.c
+++ b/fs/ufs/namei.c
@@ -69,7 +69,7 @@ static struct dentry *ufs_lookup(struct inode * dir, struct dentry *dentry, unsi
  * If the create succeeds, we fill in the inode information
  * with d_instantiate(). 
  */
-static int ufs_create (struct mnt_idmap * idmap,
+static int ufs_create (const struct mnt_idmap * idmap,
 		struct inode * dir, struct dentry * dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -85,7 +85,7 @@ static int ufs_create (struct mnt_idmap * idmap,
 	return ufs_add_nondir(dentry, inode);
 }
 
-static int ufs_mknod(struct mnt_idmap *idmap, struct inode *dir,
+static int ufs_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 		     struct dentry *dentry, umode_t mode, dev_t rdev)
 {
 	struct inode *inode;
@@ -105,7 +105,7 @@ static int ufs_mknod(struct mnt_idmap *idmap, struct inode *dir,
 	return err;
 }
 
-static int ufs_symlink (struct mnt_idmap * idmap, struct inode * dir,
+static int ufs_symlink (const struct mnt_idmap * idmap, struct inode * dir,
 	struct dentry * dentry, const char * symname)
 {
 	struct super_block * sb = dir->i_sb;
@@ -165,7 +165,7 @@ static int ufs_link (struct dentry * old_dentry, struct inode * dir,
 	return error;
 }
 
-static struct dentry *ufs_mkdir(struct mnt_idmap * idmap, struct inode * dir,
+static struct dentry *ufs_mkdir(const struct mnt_idmap * idmap, struct inode * dir,
 				struct dentry * dentry, umode_t mode)
 {
 	struct inode * inode;
@@ -240,7 +240,7 @@ static int ufs_rmdir (struct inode * dir, struct dentry *dentry)
 	return err;
 }
 
-static int ufs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
+static int ufs_rename(const struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags)
 {
diff --git a/fs/ufs/ufs.h b/fs/ufs/ufs.h
index 788e025056b2bf..541566f5b5fb85 100644
--- a/fs/ufs/ufs.h
+++ b/fs/ufs/ufs.h
@@ -120,7 +120,7 @@ extern struct inode *ufs_iget(struct super_block *, unsigned long);
 extern int ufs_write_inode (struct inode *, struct writeback_control *);
 extern int ufs_sync_inode (struct inode *);
 extern void ufs_evict_inode (struct inode *);
-extern int ufs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+extern int ufs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct iattr *attr);
 
 /* namei.c */
diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c
index 0b9eab157432df..b9c46c769c4e6d 100644
--- a/fs/vboxsf/dir.c
+++ b/fs/vboxsf/dir.c
@@ -296,14 +296,14 @@ static int vboxsf_dir_create(struct inode *parent, struct dentry *dentry,
 	return err;
 }
 
-static int vboxsf_dir_mkfile(struct mnt_idmap *idmap,
+static int vboxsf_dir_mkfile(const struct mnt_idmap *idmap,
 			     struct inode *parent, struct dentry *dentry,
 			     umode_t mode)
 {
 	return vboxsf_dir_create(parent, dentry, mode, false, true, NULL);
 }
 
-static struct dentry *vboxsf_dir_mkdir(struct mnt_idmap *idmap,
+static struct dentry *vboxsf_dir_mkdir(const struct mnt_idmap *idmap,
 				       struct inode *parent, struct dentry *dentry,
 				       umode_t mode)
 {
@@ -382,7 +382,7 @@ static int vboxsf_dir_unlink(struct inode *parent, struct dentry *dentry)
 	return 0;
 }
 
-static int vboxsf_dir_rename(struct mnt_idmap *idmap,
+static int vboxsf_dir_rename(const struct mnt_idmap *idmap,
 			     struct inode *old_parent,
 			     struct dentry *old_dentry,
 			     struct inode *new_parent,
@@ -425,7 +425,7 @@ static int vboxsf_dir_rename(struct mnt_idmap *idmap,
 	return err;
 }
 
-static int vboxsf_dir_symlink(struct mnt_idmap *idmap,
+static int vboxsf_dir_symlink(const struct mnt_idmap *idmap,
 			      struct inode *parent, struct dentry *dentry,
 			      const char *symname)
 {
diff --git a/fs/vboxsf/utils.c b/fs/vboxsf/utils.c
index 298bfc93255c03..8775fbee1ce648 100644
--- a/fs/vboxsf/utils.c
+++ b/fs/vboxsf/utils.c
@@ -233,7 +233,7 @@ int vboxsf_inode_revalidate(struct dentry *dentry)
 	return 0;
 }
 
-int vboxsf_getattr(struct mnt_idmap *idmap, const struct path *path,
+int vboxsf_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		   struct kstat *kstat, u32 request_mask, unsigned int flags)
 {
 	int err;
@@ -258,7 +258,7 @@ int vboxsf_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return 0;
 }
 
-int vboxsf_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int vboxsf_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   struct iattr *iattr)
 {
 	struct vboxsf_inode *sf_i = VBOXSF_I(d_inode(dentry));
diff --git a/fs/vboxsf/vfsmod.h b/fs/vboxsf/vfsmod.h
index b61afd0ce842d2..59a4e44c400577 100644
--- a/fs/vboxsf/vfsmod.h
+++ b/fs/vboxsf/vfsmod.h
@@ -98,10 +98,10 @@ int vboxsf_stat(struct vboxsf_sbi *sbi, struct shfl_string *path,
 		struct shfl_fsobjinfo *info);
 int vboxsf_stat_dentry(struct dentry *dentry, struct shfl_fsobjinfo *info);
 int vboxsf_inode_revalidate(struct dentry *dentry);
-int vboxsf_getattr(struct mnt_idmap *idmap, const struct path *path,
+int vboxsf_getattr(const struct mnt_idmap *idmap, const struct path *path,
 		   struct kstat *kstat, u32 request_mask,
 		   unsigned int query_flags);
-int vboxsf_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+int vboxsf_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   struct iattr *iattr);
 struct shfl_string *vboxsf_path_from_dentry(struct vboxsf_sbi *sbi,
 					    struct dentry *dentry);
diff --git a/fs/xattr.c b/fs/xattr.c
index d5897911520075..d9f035610f0bf1 100644
--- a/fs/xattr.c
+++ b/fs/xattr.c
@@ -100,7 +100,7 @@ xattr_resolve_name(struct inode *inode, const char **name)
  *
  * Return: On success zero is returned. On error a negative errno is returned.
  */
-int may_write_xattr(struct mnt_idmap *idmap, struct inode *inode)
+int may_write_xattr(const struct mnt_idmap *idmap, struct inode *inode)
 {
 	if (IS_IMMUTABLE(inode))
 		return -EPERM;
@@ -123,7 +123,7 @@ static inline int xattr_permission_error(int mask)
  * because different namespaces have very different rules.
  */
 static int
-xattr_permission(struct mnt_idmap *idmap, struct inode *inode,
+xattr_permission(const struct mnt_idmap *idmap, struct inode *inode,
 		 const char *name, int mask)
 {
 	if (mask & MAY_WRITE) {
@@ -204,7 +204,7 @@ xattr_supports_user_prefix(struct inode *inode)
 EXPORT_SYMBOL(xattr_supports_user_prefix);
 
 int
-__vfs_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
+__vfs_setxattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 	       struct inode *inode, const char *name, const void *value,
 	       size_t size, int flags)
 {
@@ -242,7 +242,7 @@ EXPORT_SYMBOL(__vfs_setxattr);
  *  is executed. It also assumes that the caller will make the appropriate
  *  permission checks.
  */
-int __vfs_setxattr_noperm(struct mnt_idmap *idmap,
+int __vfs_setxattr_noperm(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, const char *name,
 			  const void *value, size_t size, int flags)
 {
@@ -295,7 +295,7 @@ int __vfs_setxattr_noperm(struct mnt_idmap *idmap,
  *  a delegation was broken on, NULL if none.
  */
 int
-__vfs_setxattr_locked(struct mnt_idmap *idmap, struct dentry *dentry,
+__vfs_setxattr_locked(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      const char *name, const void *value, size_t size,
 		      int flags, struct delegated_inode *delegated_inode)
 {
@@ -324,7 +324,7 @@ __vfs_setxattr_locked(struct mnt_idmap *idmap, struct dentry *dentry,
 EXPORT_SYMBOL_GPL(__vfs_setxattr_locked);
 
 int
-vfs_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
+vfs_setxattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 	     const char *name, const void *value, size_t size, int flags)
 {
 	struct inode *inode = dentry->d_inode;
@@ -358,7 +358,7 @@ vfs_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
 EXPORT_SYMBOL_GPL(vfs_setxattr);
 
 static ssize_t
-xattr_getsecurity(struct mnt_idmap *idmap, struct inode *inode,
+xattr_getsecurity(const struct mnt_idmap *idmap, struct inode *inode,
 		  const char *name, void *value, size_t size)
 {
 	void *buffer = NULL;
@@ -395,7 +395,7 @@ xattr_getsecurity(struct mnt_idmap *idmap, struct inode *inode,
  * Returns the result of alloc, if failed, or the getxattr operation.
  */
 int
-vfs_getxattr_alloc(struct mnt_idmap *idmap, struct dentry *dentry,
+vfs_getxattr_alloc(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   const char *name, char **xattr_value, size_t xattr_size,
 		   gfp_t flags)
 {
@@ -448,7 +448,7 @@ __vfs_getxattr(struct dentry *dentry, struct inode *inode, const char *name,
 EXPORT_SYMBOL(__vfs_getxattr);
 
 ssize_t
-vfs_getxattr(struct mnt_idmap *idmap, struct dentry *dentry,
+vfs_getxattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 	     const char *name, void *value, size_t size)
 {
 	struct inode *inode = dentry->d_inode;
@@ -527,7 +527,7 @@ vfs_listxattr(struct dentry *dentry, char *list, size_t size)
 EXPORT_SYMBOL_GPL(vfs_listxattr);
 
 int
-__vfs_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
+__vfs_removexattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		  const char *name)
 {
 	struct inode *inode = d_inode(dentry);
@@ -557,7 +557,7 @@ EXPORT_SYMBOL(__vfs_removexattr);
  *  a delegation was broken on, NULL if none.
  */
 int
-__vfs_removexattr_locked(struct mnt_idmap *idmap,
+__vfs_removexattr_locked(const struct mnt_idmap *idmap,
 			 struct dentry *dentry, const char *name,
 			 struct delegated_inode *delegated_inode)
 {
@@ -589,7 +589,7 @@ __vfs_removexattr_locked(struct mnt_idmap *idmap,
 EXPORT_SYMBOL_GPL(__vfs_removexattr_locked);
 
 int
-vfs_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
+vfs_removexattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		const char *name)
 {
 	struct inode *inode = dentry->d_inode;
@@ -652,7 +652,7 @@ int setxattr_copy(const char __user *name, struct kernel_xattr_ctx *ctx)
 	return error;
 }
 
-static int do_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int do_setxattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 		struct kernel_xattr_ctx *ctx)
 {
 	if (is_posix_acl_xattr(ctx->kname->name))
@@ -787,7 +787,7 @@ SYSCALL_DEFINE5(fsetxattr, int, fd, const char __user *, name,
  * Extended attribute GET operations
  */
 static ssize_t
-do_getxattr(struct mnt_idmap *idmap, struct dentry *d,
+do_getxattr(const struct mnt_idmap *idmap, struct dentry *d,
 	struct kernel_xattr_ctx *ctx)
 {
 	ssize_t error;
@@ -1029,7 +1029,7 @@ SYSCALL_DEFINE3(flistxattr, int, fd, char __user *, list, size_t, size)
  * Extended attribute REMOVE operations
  */
 static long
-removexattr(struct mnt_idmap *idmap, struct dentry *d, const char *name)
+removexattr(const struct mnt_idmap *idmap, struct dentry *d, const char *name)
 {
 	if (is_posix_acl_xattr(name))
 		return vfs_remove_acl(idmap, d, name);
diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h
index fd22fe59893176..ee636b66a72f6d 100644
--- a/fs/xfs/libxfs/xfs_ag.h
+++ b/fs/xfs/libxfs/xfs_ag.h
@@ -207,7 +207,7 @@ xfs_perag_next(
 }
 
 /*
- * Per-ag geometry infomation and validation
+ * Per-ag geometry information and validation
  */
 xfs_agblock_t xfs_ag_block_count(struct xfs_mount *mp, xfs_agnumber_t agno);
 void xfs_agino_range(struct xfs_mount *mp, xfs_agnumber_t agno,
diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c
index d99602bcc16ffe..f762dcce8d133d 100644
--- a/fs/xfs/libxfs/xfs_alloc.c
+++ b/fs/xfs/libxfs/xfs_alloc.c
@@ -3487,7 +3487,7 @@ xfs_alloc_read_agf(
 }
 
 /*
- * Pre-proces allocation arguments to set initial state that we don't require
+ * Pre-process allocation arguments to set initial state that we don't require
  * callers to set up correctly, as well as bounds check the allocation args
  * that are set up.
  */
@@ -3608,7 +3608,7 @@ xfs_alloc_vextent_finish(
 	 * ABBA AGF deadlocks because a future allocation attempt in this
 	 * transaction may attempt to lock a lower number AGF.
 	 *
-	 * We can't release the AGF until the transaction is commited, so at
+	 * We can't release the AGF until the transaction is committed, so at
 	 * this point we must update the "first allocation" tracker to point at
 	 * this AG if the tracker is empty or points to a lower AG. This allows
 	 * the next allocation attempt to be modified appropriately to avoid
diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c
index b3f7b2c34ad766..c4bb59633ce856 100644
--- a/fs/xfs/libxfs/xfs_attr.c
+++ b/fs/xfs/libxfs/xfs_attr.c
@@ -59,8 +59,7 @@ STATIC void xfs_attr_restore_rmt_blk(struct xfs_da_args *args);
 static int xfs_attr_node_try_addname(struct xfs_attr_intent *attr);
 STATIC int xfs_attr_node_addname_find_attr(struct xfs_attr_intent *attr);
 STATIC int xfs_attr_node_remove_attr(struct xfs_attr_intent *attr);
-STATIC int xfs_attr_node_lookup(struct xfs_da_args *args,
-		struct xfs_da_state *state);
+STATIC int xfs_attr_node_lookup(struct xfs_da_state *state);
 
 int
 xfs_inode_hasattr(
@@ -709,7 +708,7 @@ int xfs_attr_node_removename_setup(
 	int				error;
 
 	xfs_attr_item_init_da_state(attr);
-	error = xfs_attr_node_lookup(args, attr->xattri_da_state);
+	error = xfs_attr_node_lookup(attr->xattri_da_state);
 	if (error != -EEXIST)
 		goto out;
 	error = 0;
@@ -985,7 +984,7 @@ xfs_attr_lookup(
 	}
 
 	state = xfs_da_state_alloc(args);
-	error = xfs_attr_node_lookup(args, state);
+	error = xfs_attr_node_lookup(state);
 	xfs_da_state_free(state);
 	return error;
 }
@@ -1014,7 +1013,7 @@ xfs_attr_add_fork(
 	if (xfs_inode_has_attr_fork(ip))
 		goto trans_cancel;
 
-	error = xfs_bmap_add_attrfork(tp, ip, size, rsvd);
+	error = xfs_bmap_add_attrfork(tp, ip, size);
 	if (error)
 		goto trans_cancel;
 
@@ -1386,7 +1385,6 @@ xfs_attr_leaf_get(
 /* Return EEXIST if attr is found, or ENOATTR if not. */
 STATIC int
 xfs_attr_node_lookup(
-	struct xfs_da_args	*args,
 	struct xfs_da_state	*state)
 {
 	int			retval, error;
@@ -1417,7 +1415,7 @@ xfs_attr_node_addname_find_attr(
 	 * to where it should go.
 	 */
 	xfs_attr_item_init_da_state(attr);
-	error = xfs_attr_node_lookup(args, attr->xattri_da_state);
+	error = xfs_attr_node_lookup(attr->xattri_da_state);
 	switch (error) {
 	case -ENOATTR:
 		if (args->op_flags & XFS_DA_OP_REPLACE)
@@ -1588,7 +1586,7 @@ xfs_attr_node_get(
 	 * Search to see if name exists, and get back a pointer to it.
 	 */
 	state = xfs_da_state_alloc(args);
-	error = xfs_attr_node_lookup(args, state);
+	error = xfs_attr_node_lookup(state);
 	if (error != -EEXIST)
 		goto out_release;
 
diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c
index b6288395f8533d..2c80f4fd0b78de 100644
--- a/fs/xfs/libxfs/xfs_attr_leaf.c
+++ b/fs/xfs/libxfs/xfs_attr_leaf.c
@@ -1715,7 +1715,7 @@ xfs_attr3_leaf_add_work(
 			/*
 			 * This freemap entry starts at the old end of the
 			 * leaf entry array, so we need to adjust its base
-			 * upward to accomodate the larger array.
+			 * upward to accommodate the larger array.
 			 */
 			diff = sizeof(struct xfs_attr_leaf_entry);
 		} else if (ichdr->freemap[i].size > 0 &&
diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c
index d64defeda645ed..ae91f63455c5de 100644
--- a/fs/xfs/libxfs/xfs_bmap.c
+++ b/fs/xfs/libxfs/xfs_bmap.c
@@ -1026,8 +1026,7 @@ int						/* error code */
 xfs_bmap_add_attrfork(
 	struct xfs_trans	*tp,
 	struct xfs_inode	*ip,		/* incore inode pointer */
-	int			size,		/* space new attribute needs */
-	int			rsvd)		/* xact may use reserved blks */
+	int			size)		/* space new attribute needs */
 {
 	struct xfs_mount	*mp = tp->t_mountp;
 	int			logflags;	/* logging flags */
@@ -6088,7 +6087,7 @@ xfs_bmap_validate_extent_raw(
 	int			whichfork,
 	struct xfs_bmbt_irec	*irec)
 {
-	if (!xfs_verify_fileext(mp, irec->br_startoff, irec->br_blockcount))
+	if (!xfs_verify_fileext(irec->br_startoff, irec->br_blockcount))
 		return __this_address;
 
 	if (rtfile && whichfork == XFS_DATA_FORK) {
diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h
index d5f2729305fada..60f6df4ce0579a 100644
--- a/fs/xfs/libxfs/xfs_bmap.h
+++ b/fs/xfs/libxfs/xfs_bmap.h
@@ -181,7 +181,7 @@ void	xfs_trim_extent(struct xfs_bmbt_irec *irec, xfs_fileoff_t bno,
 		xfs_filblks_t len);
 unsigned int xfs_bmap_compute_attr_offset(struct xfs_mount *mp);
 int	xfs_bmap_add_attrfork(struct xfs_trans *tp, struct xfs_inode *ip,
-		int size, int rsvd);
+		int size);
 void	xfs_bmap_local_to_extents_empty(struct xfs_trans *tp,
 		struct xfs_inode *ip, int whichfork);
 int xfs_bmap_local_to_extents(struct xfs_trans *tp, struct xfs_inode *ip,
diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c
index 60ef7f08b1d300..6738d9d1511bc7 100644
--- a/fs/xfs/libxfs/xfs_btree.c
+++ b/fs/xfs/libxfs/xfs_btree.c
@@ -2994,7 +2994,6 @@ struct xfs_btree_split_args {
 	struct xfs_btree_cur	**curp;
 	int			*stat;		/* success/failure */
 	int			result;
-	bool			kswapd;	/* allocation in kswapd context */
 	struct completion	*done;
 	struct work_struct	work;
 };
@@ -3008,33 +3007,18 @@ xfs_btree_split_worker(
 {
 	struct xfs_btree_split_args	*args = container_of(work,
 						struct xfs_btree_split_args, work);
-	unsigned long		pflags;
-	unsigned long		new_pflags = 0;
-
-	/*
-	 * we are in a transaction context here, but may also be doing work
-	 * in kswapd context, and hence we may need to inherit that state
-	 * temporarily to ensure that we don't block waiting for memory reclaim
-	 * in any way.
-	 */
-	if (args->kswapd)
-		new_pflags |= PF_MEMALLOC | PF_KSWAPD;
-
-	current_set_flags_nested(&pflags, new_pflags);
 	xfs_trans_set_context(args->cur->bc_tp);
 
 	args->result = __xfs_btree_split(args->cur, args->level, args->ptrp,
 					 args->key, args->curp, args->stat);
 
 	xfs_trans_clear_context(args->cur->bc_tp);
-	current_restore_flags_nested(&pflags, new_pflags);
 
 	/*
 	 * Do not access args after complete() has run here. We don't own args
 	 * and the owner may run and free args before we return here.
 	 */
 	complete(args->done);
-
 }
 
 /*
@@ -3078,7 +3062,7 @@ xfs_btree_split(
 	args.curp = curp;
 	args.stat = stat;
 	args.done = &done;
-	args.kswapd = current_is_kswapd();
+
 	INIT_WORK_ONSTACK(&args.work, xfs_btree_split_worker);
 	queue_work(xfs_alloc_wq, &args.work);
 	wait_for_completion(&done);
diff --git a/fs/xfs/libxfs/xfs_btree_mem.c b/fs/xfs/libxfs/xfs_btree_mem.c
index 1d83a4251ceeb6..3d5c7a014d4fff 100644
--- a/fs/xfs/libxfs/xfs_btree_mem.c
+++ b/fs/xfs/libxfs/xfs_btree_mem.c
@@ -73,9 +73,7 @@ xfbtree_destroy(
 
 /* Compute the number of bytes available for records. */
 static inline unsigned int
-xfbtree_rec_bytes(
-	struct xfs_mount		*mp,
-	const struct xfs_btree_ops	*ops)
+xfbtree_rec_bytes(void)
 {
 	return XMBUF_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN;
 }
@@ -118,7 +116,7 @@ xfbtree_init(
 	const struct xfs_btree_ops	*ops)
 {
 	unsigned long long		owner = xfbt->owner;
-	unsigned int			blocklen = xfbtree_rec_bytes(mp, ops);
+	unsigned int			blocklen = xfbtree_rec_bytes();
 	unsigned int			keyptr_len;
 	int				error;
 
diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c
index 77954d1d924cc3..6c8a9afdfb1c0f 100644
--- a/fs/xfs/libxfs/xfs_dquot_buf.c
+++ b/fs/xfs/libxfs/xfs_dquot_buf.c
@@ -454,19 +454,8 @@ xfs_dqinode_metadir_link(
 		.path			= xfs_dqinode_path(type),
 		.ip			= ip,
 	};
-	int				error;
 
-	error = xfs_metadir_start_link(&upd);
-	if (error)
-		return error;
-
-	error = xfs_metadir_link(&upd);
-	if (error)
-		return error;
-
-	xfs_trans_log_inode(upd.tp, upd.ip, XFS_ILOG_CORE);
-
-	return xfs_metadir_commit(&upd);
+	return xfs_metadir_link_file(&upd);
 }
 #endif /* __KERNEL__ */
 
diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h
index 6de207fed2d892..d14aa289699f9a 100644
--- a/fs/xfs/libxfs/xfs_errortag.h
+++ b/fs/xfs/libxfs/xfs_errortag.h
@@ -75,7 +75,8 @@
 #define XFS_ERRTAG_METAFILE_RESV_CRITICAL		45
 #define XFS_ERRTAG_FORCE_ZERO_RANGE			46
 #define XFS_ERRTAG_ZONE_RESET				47
-#define XFS_ERRTAG_MAX					48
+#define XFS_ERRTAG_BOUNCE_REREAD			48
+#define XFS_ERRTAG_MAX					49
 
 /*
  * Random factors for above tags, 1 means always, 2 means 1/2 time, etc.
@@ -83,7 +84,7 @@
 #define XFS_RANDOM_DEFAULT				100
 
 /*
- * Table of errror injection knobs.  The parameters to the XFS_ERRTAG macro are:
+ * Table of error injection knobs.  The parameters to the XFS_ERRTAG macro are:
  *   1. The XFS_ERRTAG_ flag but without the prefix;
  *   2. The name of the sysfs knob; and
  *   3. The default value for the knob.
@@ -137,7 +138,8 @@ XFS_ERRTAG(WRITE_DELAY_MS,	write_delay_ms,		3000) \
 XFS_ERRTAG(EXCHMAPS_FINISH_ONE,	exchmaps_finish_one,	1) \
 XFS_ERRTAG(METAFILE_RESV_CRITICAL, metafile_resv_crit,	4) \
 XFS_ERRTAG(FORCE_ZERO_RANGE,	force_zero_range,	4) \
-XFS_ERRTAG(ZONE_RESET,		zone_reset,		1)
+XFS_ERRTAG(ZONE_RESET,		zone_reset,		1)  \
+XFS_ERRTAG(BOUNCE_REREAD,	bounce_reread,		XFS_RANDOM_DEFAULT)
 #endif /* XFS_ERRTAG */
 
 #endif /* __XFS_ERRORTAG_H_ */
diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c
index 49eda8d0994dee..ccb97da3765f22 100644
--- a/fs/xfs/libxfs/xfs_exchmaps.c
+++ b/fs/xfs/libxfs/xfs_exchmaps.c
@@ -135,7 +135,6 @@ xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi)
 /* Check all mappings to make sure we can actually exchange them. */
 int
 xfs_exchmaps_check_forks(
-	struct xfs_mount		*mp,
 	const struct xfs_exchmaps_req	*req)
 {
 	struct xfs_ifork		*ifp1, *ifp2;
@@ -395,7 +394,7 @@ xfs_exchmaps_one_step(
 	/*
 	 * Re-add both mappings.  We exchange the file offsets between the two
 	 * maps and add the opposite map, which has the effect of filling the
-	 * logical offsets we just unmapped, but with with the physical mapping
+	 * logical offsets we just unmapped, but with the physical mapping
 	 * information exchanged.
 	 */
 	swap(irec1->br_startoff, irec2->br_startoff);
@@ -959,16 +958,6 @@ xmi_can_exchange_reflink_flags(
 {
 	struct xfs_mount		*mp = req->ip1->i_mount;
 
-	/*
-	 * The INO1_WRITTEN optimization can skip exchanging hole and
-	 * unwritten mappings, which means we cannot guarantee that all
-	 * shared extents actually moved to the other file.  Clearing the
-	 * reflink flag of an inode that still holds shared extents breaks
-	 * the CoW write path, so refuse to exchange the flags in that case.
-	 */
-	if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN)
-		return false;
-
 	/*
 	 * The INO1_WRITTEN optimization can skip exchanging hole and
 	 * unwritten mappings, which means we cannot guarantee that all
diff --git a/fs/xfs/libxfs/xfs_exchmaps.h b/fs/xfs/libxfs/xfs_exchmaps.h
index fa822dff202ade..055b8dbabd5bc1 100644
--- a/fs/xfs/libxfs/xfs_exchmaps.h
+++ b/fs/xfs/libxfs/xfs_exchmaps.h
@@ -115,8 +115,7 @@ void xfs_exchmaps_upgrade_extent_counts(struct xfs_trans *tp,
 int xfs_exchmaps_finish_one(struct xfs_trans *tp,
 		struct xfs_exchmaps_intent *xmi);
 
-int xfs_exchmaps_check_forks(struct xfs_mount *mp,
-		const struct xfs_exchmaps_req *req);
+int xfs_exchmaps_check_forks(const struct xfs_exchmaps_req *req);
 
 void xfs_exchange_mappings(struct xfs_trans *tp,
 		const struct xfs_exchmaps_req *req);
diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h
index dd0ed046fbe9c9..1a7a7e60a170c4 100644
--- a/fs/xfs/libxfs/xfs_format.h
+++ b/fs/xfs/libxfs/xfs_format.h
@@ -1051,7 +1051,7 @@ enum xfs_dinode_fmt {
  * block is 1KB in size.
  *
  * With XFS_MAX_EXTCNT_DATA_FORK_SMALL representing maximum extent count and
- * with 1KB sized blocks, a file can reach upto,
+ * with 1KB sized blocks, a file can reach up to,
  * 1KB * (2^31) = 2TB
  *
  * This is much larger than the theoretical maximum size of a directory
diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c
index 58dac4d505ba63..19b513b1169269 100644
--- a/fs/xfs/libxfs/xfs_ialloc.c
+++ b/fs/xfs/libxfs/xfs_ialloc.c
@@ -615,7 +615,7 @@ xfs_inobt_insert_sprec(
 
 	trace_xfs_irec_merge_post(pag, nrec);
 
-	error = xfs_inobt_rec_check_count(mp, nrec);
+	error = xfs_inobt_rec_check_count(nrec);
 	if (error)
 		goto error;
 
diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.c b/fs/xfs/libxfs/xfs_ialloc_btree.c
index 1376e8630449af..1f0bace2f144ee 100644
--- a/fs/xfs/libxfs/xfs_ialloc_btree.c
+++ b/fs/xfs/libxfs/xfs_ialloc_btree.c
@@ -687,7 +687,6 @@ xfs_inobt_irec_to_allocmask(
  */
 int
 xfs_inobt_rec_check_count(
-	struct xfs_mount		*mp,
 	struct xfs_inobt_rec_incore	*rec)
 {
 	int				inocount = 0;
diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.h b/fs/xfs/libxfs/xfs_ialloc_btree.h
index 300edf5bc00949..e04c63c66f397f 100644
--- a/fs/xfs/libxfs/xfs_ialloc_btree.h
+++ b/fs/xfs/libxfs/xfs_ialloc_btree.h
@@ -57,10 +57,9 @@ unsigned int xfs_inobt_maxrecs(struct xfs_mount *mp, unsigned int blocklen,
 uint64_t xfs_inobt_irec_to_allocmask(const struct xfs_inobt_rec_incore *irec);
 
 #if defined(DEBUG) || defined(XFS_WARN)
-int xfs_inobt_rec_check_count(struct xfs_mount *,
-			      struct xfs_inobt_rec_incore *);
+int xfs_inobt_rec_check_count(struct xfs_inobt_rec_incore *);
 #else
-#define xfs_inobt_rec_check_count(mp, rec)	0
+#define xfs_inobt_rec_check_count(rec)	0
 #endif	/* DEBUG */
 
 int xfs_finobt_calc_reserves(struct xfs_perag *perag, struct xfs_trans *tp,
diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c
index e4c3f7b24e952f..0340e2189921db 100644
--- a/fs/xfs/libxfs/xfs_inode_buf.c
+++ b/fs/xfs/libxfs/xfs_inode_buf.c
@@ -626,7 +626,7 @@ xfs_dinode_verify(
 	 * have di_nlink track the link count, even if the actual filesystem
 	 * only supported V1 inodes (i.e. di_onlink).  When writing out the
 	 * ondisk inode, it would set both the ondisk di_nlink and di_onlink to
-	 * the the incore di_nlink value, which is why we cannot check for
+	 * the incore di_nlink value, which is why we cannot check for
 	 * di_nlink==0 on a V1 inode.  V2/3 inodes would get written out with
 	 * di_onlink==0, so we can check that.
 	 */
diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c
index 606a36526ce245..486fe7ab8b6110 100644
--- a/fs/xfs/libxfs/xfs_inode_fork.c
+++ b/fs/xfs/libxfs/xfs_inode_fork.c
@@ -683,9 +683,7 @@ xfs_ifork_verify_local_data(
 		break;
 	}
 	case S_IFLNK: {
-		struct xfs_ifork	*ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK);
-
-		fa = xfs_symlink_shortform_verify(ifp->if_data, ifp->if_bytes);
+		fa = xfs_symlink_shortform_verify(ip);
 		break;
 	}
 	default:
diff --git a/fs/xfs/libxfs/xfs_inode_util.h b/fs/xfs/libxfs/xfs_inode_util.h
index 060242998a230d..e9eac35159c372 100644
--- a/fs/xfs/libxfs/xfs_inode_util.h
+++ b/fs/xfs/libxfs/xfs_inode_util.h
@@ -27,7 +27,7 @@ prid_t		xfs_get_initial_prid(struct xfs_inode *dp);
  * idmap to NULL.  To create a tree root, set pip to NULL.
  */
 struct xfs_icreate_args {
-	struct mnt_idmap	*idmap;
+	const struct mnt_idmap	*idmap;
 	struct xfs_inode	*pip;	/* parent inode or null */
 	dev_t			rdev;
 	umode_t			mode;
diff --git a/fs/xfs/libxfs/xfs_metadir.c b/fs/xfs/libxfs/xfs_metadir.c
index 7c6b086b73db61..79b37cb272734b 100644
--- a/fs/xfs/libxfs/xfs_metadir.c
+++ b/fs/xfs/libxfs/xfs_metadir.c
@@ -163,7 +163,7 @@ xfs_metadir_teardown(
 	trace_xfs_metadir_teardown(upd, error);
 
 	if (upd->ppargs) {
-		xfs_parent_finish(upd->dp->i_mount, upd->ppargs);
+		xfs_parent_finish(upd->ppargs);
 		upd->ppargs = NULL;
 	}
 
@@ -317,7 +317,7 @@ xfs_metadir_create(
  * Begin the process of linking a metadata file by allocating transactions
  * and locking whatever resources we're going to need.
  */
-int
+static int
 xfs_metadir_start_link(
 	struct xfs_metadir_update	*upd)
 {
@@ -364,7 +364,7 @@ xfs_metadir_start_link(
  * The path (up to the final component) must already exist, but the final
  * component must not already exist.
  */
-int
+static int
 xfs_metadir_link(
 	struct xfs_metadir_update	*upd)
 {
@@ -409,7 +409,7 @@ xfs_metadir_link(
 #endif /* ! __KERNEL__ */
 
 /* Commit a metadir update and unlock/drop all resources. */
-int
+static int
 xfs_metadir_commit(
 	struct xfs_metadir_update	*upd)
 {
@@ -499,3 +499,27 @@ xfs_metadir_mkdir(
 
 	return xfs_metadir_create_file(&upd, S_IFDIR, NULL, NULL, ipp);
 }
+
+#ifndef __KERNEL__
+/* Link a metadata file into a metadata directory. */
+int
+xfs_metadir_link_file(
+	struct xfs_metadir_update	*upd)
+{
+	int				error;
+
+	error = xfs_metadir_start_link(upd);
+	if (error)
+		return error;
+
+	error = xfs_metadir_link(upd);
+	if (error) {
+		xfs_metadir_cancel(upd, error);
+		return error;
+	}
+
+	xfs_trans_log_inode(upd->tp, upd->ip, XFS_ILOG_CORE);
+
+	return xfs_metadir_commit(upd);
+}
+#endif /* ! __KERNEL__ */
diff --git a/fs/xfs/libxfs/xfs_metadir.h b/fs/xfs/libxfs/xfs_metadir.h
index e434b9d1c93200..b64f9fc5ca7867 100644
--- a/fs/xfs/libxfs/xfs_metadir.h
+++ b/fs/xfs/libxfs/xfs_metadir.h
@@ -38,10 +38,7 @@ int xfs_metadir_create_file(struct xfs_metadir_update *upd, umode_t mode,
 		xfs_metadir_createfn create, void *priv,
 		struct xfs_inode **ipp);
 
-int xfs_metadir_start_link(struct xfs_metadir_update *upd);
-int xfs_metadir_link(struct xfs_metadir_update *upd);
-
-int xfs_metadir_commit(struct xfs_metadir_update *upd);
+int xfs_metadir_link_file(struct xfs_metadir_update *upd);
 
 int xfs_metadir_mkdir(struct xfs_inode *dp, const char *path,
 		struct xfs_inode **ipp);
diff --git a/fs/xfs/libxfs/xfs_metafile.c b/fs/xfs/libxfs/xfs_metafile.c
index 71f004e9dc6451..1f54d39003c27c 100644
--- a/fs/xfs/libxfs/xfs_metafile.c
+++ b/fs/xfs/libxfs/xfs_metafile.c
@@ -297,14 +297,14 @@ xfs_metafile_resv_init(
 		goto out_unlock;
 
 	/*
-	 * Space taken by the per-AG metadata btrees are accounted on-disk as
-	 * used space.  We therefore only hide the space that is reserved but
-	 * not used by the trees.
+	 * Space taken by metadata btrees are accounted on-disk as used space.
+	 * We therefore only hide the space that is reserved but not used by
+	 * the trees.
 	 */
 	if (used > target)
 		target = used;
 	else if (target > dblocks_avail)
-		target = dblocks_avail;
+		target = max(dblocks_avail, used);
 	hidden_space = target - used;
 
 	error = xfs_dec_fdblocks(mp, hidden_space, true);
diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h
index 8eb4de9c5f1a9c..1dd3968a78b1ef 100644
--- a/fs/xfs/libxfs/xfs_parent.h
+++ b/fs/xfs/libxfs/xfs_parent.h
@@ -72,7 +72,6 @@ xfs_parent_start(
 /* Finish a parent pointer update by freeing the context object. */
 static inline void
 xfs_parent_finish(
-	struct xfs_mount	*mp,
 	struct xfs_parent_args	*ppargs)
 {
 	if (ppargs)
diff --git a/fs/xfs/libxfs/xfs_refcount_btree.c b/fs/xfs/libxfs/xfs_refcount_btree.c
index 7e5f92c1ac56fd..2c6148a979942b 100644
--- a/fs/xfs/libxfs/xfs_refcount_btree.c
+++ b/fs/xfs/libxfs/xfs_refcount_btree.c
@@ -127,7 +127,7 @@ xfs_refcountbt_get_maxrecs(
 	return cur->bc_mp->m_refc_mxr[level != 0];
 }
 
-STATIC void
+void
 xfs_refcountbt_init_key_from_rec(
 	union xfs_btree_key		*key,
 	const union xfs_btree_rec	*rec)
@@ -135,7 +135,7 @@ xfs_refcountbt_init_key_from_rec(
 	key->refc.rc_startblock = rec->refc.rc_startblock;
 }
 
-STATIC void
+void
 xfs_refcountbt_init_high_key_from_rec(
 	union xfs_btree_key		*key,
 	const union xfs_btree_rec	*rec)
@@ -147,7 +147,7 @@ xfs_refcountbt_init_high_key_from_rec(
 	key->refc.rc_startblock = cpu_to_be32(x);
 }
 
-STATIC void
+void
 xfs_refcountbt_init_rec_from_cur(
 	struct xfs_btree_cur	*cur,
 	union xfs_btree_rec	*rec)
@@ -174,7 +174,7 @@ xfs_refcountbt_init_ptr_from_cur(
 	ptr->s = agf->agf_refcount_root;
 }
 
-STATIC int
+int
 xfs_refcountbt_cmp_key_with_cur(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*key)
@@ -188,7 +188,7 @@ xfs_refcountbt_cmp_key_with_cur(
 	return cmp_int(be32_to_cpu(kp->rc_startblock), start);
 }
 
-STATIC int
+int
 xfs_refcountbt_cmp_two_keys(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*k1,
@@ -283,7 +283,7 @@ const struct xfs_buf_ops xfs_refcountbt_buf_ops = {
 	.verify_struct		= xfs_refcountbt_verify,
 };
 
-STATIC int
+int
 xfs_refcountbt_keys_inorder(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*k1,
@@ -293,7 +293,7 @@ xfs_refcountbt_keys_inorder(
 	       be32_to_cpu(k2->refc.rc_startblock);
 }
 
-STATIC int
+int
 xfs_refcountbt_recs_inorder(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_rec	*r1,
@@ -304,7 +304,7 @@ xfs_refcountbt_recs_inorder(
 		be32_to_cpu(r2->refc.rc_startblock);
 }
 
-STATIC enum xbtree_key_contig
+enum xbtree_key_contig
 xfs_refcountbt_keys_contiguous(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*key1,
diff --git a/fs/xfs/libxfs/xfs_refcount_btree.h b/fs/xfs/libxfs/xfs_refcount_btree.h
index beb93bef6a8141..40eedfae6846ec 100644
--- a/fs/xfs/libxfs/xfs_refcount_btree.h
+++ b/fs/xfs/libxfs/xfs_refcount_btree.h
@@ -15,6 +15,8 @@ struct xfs_btree_cur;
 struct xfs_mount;
 struct xfs_perag;
 struct xbtree_afakeroot;
+union xfs_btree_key;
+union xfs_btree_rec;
 
 /*
  * Btree block header size
@@ -69,4 +71,28 @@ unsigned int xfs_refcountbt_maxlevels_ondisk(void);
 int __init xfs_refcountbt_init_cur_cache(void);
 void xfs_refcountbt_destroy_cur_cache(void);
 
+/*
+ * Key and record btree ops.  The refcount on-disk key/record format is
+ * identical for the AG refcount btree and the realtime refcount btree, so
+ * these are shared by both.
+ */
+void xfs_refcountbt_init_key_from_rec(union xfs_btree_key *key,
+		const union xfs_btree_rec *rec);
+void xfs_refcountbt_init_high_key_from_rec(union xfs_btree_key *key,
+		const union xfs_btree_rec *rec);
+void xfs_refcountbt_init_rec_from_cur(struct xfs_btree_cur *cur,
+		union xfs_btree_rec *rec);
+int xfs_refcountbt_cmp_key_with_cur(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *key);
+int xfs_refcountbt_cmp_two_keys(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *k1, const union xfs_btree_key *k2,
+		const union xfs_btree_key *mask);
+int xfs_refcountbt_keys_inorder(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *k1, const union xfs_btree_key *k2);
+int xfs_refcountbt_recs_inorder(struct xfs_btree_cur *cur,
+		const union xfs_btree_rec *r1, const union xfs_btree_rec *r2);
+enum xbtree_key_contig xfs_refcountbt_keys_contiguous(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *key1, const union xfs_btree_key *key2,
+		const union xfs_btree_key *mask);
+
 #endif	/* __XFS_REFCOUNT_BTREE_H__ */
diff --git a/fs/xfs/libxfs/xfs_rmap.c b/fs/xfs/libxfs/xfs_rmap.c
index 34d218de21a956..14aef87837a9cc 100644
--- a/fs/xfs/libxfs/xfs_rmap.c
+++ b/fs/xfs/libxfs/xfs_rmap.c
@@ -260,7 +260,7 @@ xfs_rmap_check_irec(
 
 	/* Check for a valid fork offset, if applicable. */
 	if (is_inode && !is_bmbt &&
-	    !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
+	    !xfs_verify_fileext(irec->rm_offset, irec->rm_blockcount))
 		return __this_address;
 
 	return NULL;
@@ -310,7 +310,7 @@ xfs_rtrmap_check_inode_irec(
 		return __this_address;
 	if (!xfs_verify_rgbext(rtg, irec->rm_startblock, irec->rm_blockcount))
 		return __this_address;
-	if (!xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount))
+	if (!xfs_verify_fileext(irec->rm_offset, irec->rm_blockcount))
 		return __this_address;
 	return NULL;
 }
@@ -904,7 +904,6 @@ xfs_rmap_hook_enable(void)
 /* Call downstream hooks for a reverse mapping update. */
 static inline void
 xfs_rmap_update_hook(
-	struct xfs_trans		*tp,
 	struct xfs_group		*xg,
 	enum xfs_rmap_intent_type	op,
 	xfs_agblock_t			startblock,
@@ -952,7 +951,7 @@ xfs_rmap_hook_setup(
 	xfs_hook_setup(&hook->rmap_hook, mod_fn);
 }
 #else
-# define xfs_rmap_update_hook(t, p, o, s, b, u, oi)	do { } while (0)
+# define xfs_rmap_update_hook(p, o, s, b, u, oi)	do { } while (0)
 #endif /* CONFIG_XFS_LIVE_HOOKS */
 
 /*
@@ -975,7 +974,7 @@ xfs_rmap_free(
 		return 0;
 
 	cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
-	xfs_rmap_update_hook(tp, pag_group(pag), XFS_RMAP_UNMAP, bno, len,
+	xfs_rmap_update_hook(pag_group(pag), XFS_RMAP_UNMAP, bno, len,
 			false, oinfo);
 	error = xfs_rmap_unmap(cur, bno, len, false, oinfo);
 
@@ -1220,7 +1219,7 @@ xfs_rmap_alloc(
 		return 0;
 
 	cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag);
-	xfs_rmap_update_hook(tp, pag_group(pag), XFS_RMAP_MAP, bno, len, false,
+	xfs_rmap_update_hook(pag_group(pag), XFS_RMAP_MAP, bno, len, false,
 			oinfo);
 	error = xfs_rmap_map(cur, bno, len, false, oinfo);
 
@@ -2721,7 +2720,7 @@ xfs_rmap_finish_one(
 	if (error)
 		return error;
 
-	xfs_rmap_update_hook(tp, ri->ri_group, ri->ri_type, bno,
+	xfs_rmap_update_hook(ri->ri_group, ri->ri_type, bno,
 			ri->ri_bmap.br_blockcount, unwritten, &oinfo);
 	return 0;
 }
diff --git a/fs/xfs/libxfs/xfs_rmap_btree.c b/fs/xfs/libxfs/xfs_rmap_btree.c
index 10b3272238ebcb..5b283a5ddd130d 100644
--- a/fs/xfs/libxfs/xfs_rmap_btree.c
+++ b/fs/xfs/libxfs/xfs_rmap_btree.c
@@ -170,7 +170,7 @@ static inline __be64 ondisk_rec_offset_to_key(const union xfs_btree_rec *rec)
 	return rec->rmap.rm_offset & ~cpu_to_be64(XFS_RMAP_OFF_UNWRITTEN);
 }
 
-STATIC void
+void
 xfs_rmapbt_init_key_from_rec(
 	union xfs_btree_key		*key,
 	const union xfs_btree_rec	*rec)
@@ -187,7 +187,7 @@ xfs_rmapbt_init_key_from_rec(
  * the startblock for all records, and if the record is for a data/attr
  * fork mapping, we add blockcount-1 to the offset too.
  */
-STATIC void
+void
 xfs_rmapbt_init_high_key_from_rec(
 	union xfs_btree_key		*key,
 	const union xfs_btree_rec	*rec)
@@ -209,7 +209,7 @@ xfs_rmapbt_init_high_key_from_rec(
 	key->rmap.rm_offset = cpu_to_be64(off);
 }
 
-STATIC void
+void
 xfs_rmapbt_init_rec_from_cur(
 	struct xfs_btree_cur	*cur,
 	union xfs_btree_rec	*rec)
@@ -243,7 +243,7 @@ static inline uint64_t offset_keymask(uint64_t offset)
 	return offset & ~XFS_RMAP_OFF_UNWRITTEN;
 }
 
-STATIC int
+int
 xfs_rmapbt_cmp_key_with_cur(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*key)
@@ -257,7 +257,7 @@ xfs_rmapbt_cmp_key_with_cur(
 		       offset_keymask(xfs_rmap_irec_offset_pack(rec)));
 }
 
-STATIC int
+int
 xfs_rmapbt_cmp_two_keys(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*k1,
@@ -390,7 +390,7 @@ const struct xfs_buf_ops xfs_rmapbt_buf_ops = {
 	.verify_struct		= xfs_rmapbt_verify,
 };
 
-STATIC int
+int
 xfs_rmapbt_keys_inorder(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*k1,
@@ -420,7 +420,7 @@ xfs_rmapbt_keys_inorder(
 	return 0;
 }
 
-STATIC int
+int
 xfs_rmapbt_recs_inorder(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_rec	*r1,
@@ -450,7 +450,7 @@ xfs_rmapbt_recs_inorder(
 	return 0;
 }
 
-STATIC enum xbtree_key_contig
+enum xbtree_key_contig
 xfs_rmapbt_keys_contiguous(
 	struct xfs_btree_cur		*cur,
 	const union xfs_btree_key	*key1,
diff --git a/fs/xfs/libxfs/xfs_rmap_btree.h b/fs/xfs/libxfs/xfs_rmap_btree.h
index 119b1567cd0ee8..7071dac745e1b8 100644
--- a/fs/xfs/libxfs/xfs_rmap_btree.h
+++ b/fs/xfs/libxfs/xfs_rmap_btree.h
@@ -11,6 +11,8 @@ struct xfs_btree_cur;
 struct xfs_mount;
 struct xbtree_afakeroot;
 struct xfbtree;
+union xfs_btree_key;
+union xfs_btree_rec;
 
 /* rmaps only exist on crc enabled filesystems */
 #define XFS_RMAP_BLOCK_LEN	XFS_BTREE_SBLOCK_CRC_LEN
@@ -69,4 +71,28 @@ struct xfs_btree_cur *xfs_rmapbt_mem_cursor(struct xfs_perag *pag,
 int xfs_rmapbt_mem_init(struct xfs_mount *mp, struct xfbtree *xfbtree,
 		struct xfs_buftarg *btp, xfs_agnumber_t agno);
 
+/*
+ * Key and record btree ops.  The rmap on-disk key/record format is identical
+ * for the AG rmap btree and the realtime rmap btree, so these are shared by
+ * both.
+ */
+void xfs_rmapbt_init_key_from_rec(union xfs_btree_key *key,
+		const union xfs_btree_rec *rec);
+void xfs_rmapbt_init_high_key_from_rec(union xfs_btree_key *key,
+		const union xfs_btree_rec *rec);
+void xfs_rmapbt_init_rec_from_cur(struct xfs_btree_cur *cur,
+		union xfs_btree_rec *rec);
+int xfs_rmapbt_cmp_key_with_cur(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *key);
+int xfs_rmapbt_cmp_two_keys(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *k1, const union xfs_btree_key *k2,
+		const union xfs_btree_key *mask);
+int xfs_rmapbt_keys_inorder(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *k1, const union xfs_btree_key *k2);
+int xfs_rmapbt_recs_inorder(struct xfs_btree_cur *cur,
+		const union xfs_btree_rec *r1, const union xfs_btree_rec *r2);
+enum xbtree_key_contig xfs_rmapbt_keys_contiguous(struct xfs_btree_cur *cur,
+		const union xfs_btree_key *key1, const union xfs_btree_key *key2,
+		const union xfs_btree_key *mask);
+
 #endif /* __XFS_RMAP_BTREE_H__ */
diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.c b/fs/xfs/libxfs/xfs_rtrefcount_btree.c
index e2950dbe20681b..697f5e62268573 100644
--- a/fs/xfs/libxfs/xfs_rtrefcount_btree.c
+++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.c
@@ -20,6 +20,7 @@
 #include "xfs_btree_staging.h"
 #include "xfs_rtrefcount_btree.h"
 #include "xfs_refcount.h"
+#include "xfs_refcount_btree.h"
 #include "xfs_trace.h"
 #include "xfs_cksum.h"
 #include "xfs_error.h"
@@ -113,41 +114,6 @@ xfs_rtrefcountbt_get_dmaxrecs(
 	return xfs_rtrefcountbt_droot_maxrecs(cur->bc_ino.forksize, level == 0);
 }
 
-STATIC void
-xfs_rtrefcountbt_init_key_from_rec(
-	union xfs_btree_key		*key,
-	const union xfs_btree_rec	*rec)
-{
-	key->refc.rc_startblock = rec->refc.rc_startblock;
-}
-
-STATIC void
-xfs_rtrefcountbt_init_high_key_from_rec(
-	union xfs_btree_key		*key,
-	const union xfs_btree_rec	*rec)
-{
-	__u32				x;
-
-	x = be32_to_cpu(rec->refc.rc_startblock);
-	x += be32_to_cpu(rec->refc.rc_blockcount) - 1;
-	key->refc.rc_startblock = cpu_to_be32(x);
-}
-
-STATIC void
-xfs_rtrefcountbt_init_rec_from_cur(
-	struct xfs_btree_cur	*cur,
-	union xfs_btree_rec	*rec)
-{
-	const struct xfs_refcount_irec *irec = &cur->bc_rec.rc;
-	uint32_t		start;
-
-	start = xfs_refcount_encode_startblock(irec->rc_startblock,
-			irec->rc_domain);
-	rec->refc.rc_startblock = cpu_to_be32(start);
-	rec->refc.rc_blockcount = cpu_to_be32(cur->bc_rec.rc.rc_blockcount);
-	rec->refc.rc_refcount = cpu_to_be32(cur->bc_rec.rc.rc_refcount);
-}
-
 STATIC void
 xfs_rtrefcountbt_init_ptr_from_cur(
 	struct xfs_btree_cur	*cur,
@@ -156,33 +122,6 @@ xfs_rtrefcountbt_init_ptr_from_cur(
 	ptr->l = 0;
 }
 
-STATIC int
-xfs_rtrefcountbt_cmp_key_with_cur(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*key)
-{
-	const struct xfs_refcount_key	*kp = &key->refc;
-	const struct xfs_refcount_irec	*irec = &cur->bc_rec.rc;
-	uint32_t			start;
-
-	start = xfs_refcount_encode_startblock(irec->rc_startblock,
-			irec->rc_domain);
-	return cmp_int(be32_to_cpu(kp->rc_startblock), start);
-}
-
-STATIC int
-xfs_rtrefcountbt_cmp_two_keys(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*k1,
-	const union xfs_btree_key	*k2,
-	const union xfs_btree_key	*mask)
-{
-	ASSERT(!mask || mask->refc.rc_startblock);
-
-	return cmp_int(be32_to_cpu(k1->refc.rc_startblock),
-		       be32_to_cpu(k2->refc.rc_startblock));
-}
-
 static xfs_failaddr_t
 xfs_rtrefcountbt_verify(
 	struct xfs_buf		*bp)
@@ -249,40 +188,6 @@ const struct xfs_buf_ops xfs_rtrefcountbt_buf_ops = {
 	.verify_struct		= xfs_rtrefcountbt_verify,
 };
 
-STATIC int
-xfs_rtrefcountbt_keys_inorder(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*k1,
-	const union xfs_btree_key	*k2)
-{
-	return be32_to_cpu(k1->refc.rc_startblock) <
-	       be32_to_cpu(k2->refc.rc_startblock);
-}
-
-STATIC int
-xfs_rtrefcountbt_recs_inorder(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_rec	*r1,
-	const union xfs_btree_rec	*r2)
-{
-	return  be32_to_cpu(r1->refc.rc_startblock) +
-		be32_to_cpu(r1->refc.rc_blockcount) <=
-		be32_to_cpu(r2->refc.rc_startblock);
-}
-
-STATIC enum xbtree_key_contig
-xfs_rtrefcountbt_keys_contiguous(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*key1,
-	const union xfs_btree_key	*key2,
-	const union xfs_btree_key	*mask)
-{
-	ASSERT(!mask || mask->refc.rc_startblock);
-
-	return xbtree_key_contig(be32_to_cpu(key1->refc.rc_startblock),
-				 be32_to_cpu(key2->refc.rc_startblock));
-}
-
 static inline void
 xfs_rtrefcountbt_move_ptrs(
 	struct xfs_mount	*mp,
@@ -311,7 +216,7 @@ xfs_rtrefcountbt_broot_realloc(
 	unsigned int		old_size = ifp->if_broot_bytes;
 	const unsigned int	level = cur->bc_nlevels - 1;
 
-	new_size = xfs_rtrefcount_broot_space_calc(mp, level, new_numrecs);
+	new_size = xfs_rtrefcount_broot_space_calc(level, new_numrecs);
 
 	/* Handle the nop case quietly. */
 	if (new_size == old_size)
@@ -383,16 +288,16 @@ const struct xfs_btree_ops xfs_rtrefcountbt_ops = {
 	.get_minrecs		= xfs_rtrefcountbt_get_minrecs,
 	.get_maxrecs		= xfs_rtrefcountbt_get_maxrecs,
 	.get_dmaxrecs		= xfs_rtrefcountbt_get_dmaxrecs,
-	.init_key_from_rec	= xfs_rtrefcountbt_init_key_from_rec,
-	.init_high_key_from_rec	= xfs_rtrefcountbt_init_high_key_from_rec,
-	.init_rec_from_cur	= xfs_rtrefcountbt_init_rec_from_cur,
+	.init_key_from_rec	= xfs_refcountbt_init_key_from_rec,
+	.init_high_key_from_rec	= xfs_refcountbt_init_high_key_from_rec,
+	.init_rec_from_cur	= xfs_refcountbt_init_rec_from_cur,
 	.init_ptr_from_cur	= xfs_rtrefcountbt_init_ptr_from_cur,
-	.cmp_key_with_cur	= xfs_rtrefcountbt_cmp_key_with_cur,
+	.cmp_key_with_cur	= xfs_refcountbt_cmp_key_with_cur,
 	.buf_ops		= &xfs_rtrefcountbt_buf_ops,
-	.cmp_two_keys		= xfs_rtrefcountbt_cmp_two_keys,
-	.keys_inorder		= xfs_rtrefcountbt_keys_inorder,
-	.recs_inorder		= xfs_rtrefcountbt_recs_inorder,
-	.keys_contiguous	= xfs_rtrefcountbt_keys_contiguous,
+	.cmp_two_keys		= xfs_refcountbt_cmp_two_keys,
+	.keys_inorder		= xfs_refcountbt_keys_inorder,
+	.recs_inorder		= xfs_refcountbt_recs_inorder,
+	.keys_contiguous	= xfs_refcountbt_keys_contiguous,
 	.broot_realloc		= xfs_rtrefcountbt_broot_realloc,
 };
 
@@ -602,7 +507,7 @@ xfs_rtrefcountbt_from_disk(
 	unsigned int			maxrecs;
 	unsigned int			rblocklen;
 
-	rblocklen = xfs_rtrefcount_broot_space(mp, dblock);
+	rblocklen = xfs_rtrefcount_broot_space(dblock);
 
 	xfs_btree_init_block(mp, rblock, &xfs_rtrefcountbt_ops, 0, 0,
 			I_INO(ip));
@@ -617,7 +522,7 @@ xfs_rtrefcountbt_from_disk(
 		fpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs);
 		tpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen);
 		numrecs = be16_to_cpu(dblock->bb_numrecs);
-		memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs);
+		memcpy(tkp, fkp, sizeof(*fkp) * numrecs);
 		memcpy(tpp, fpp, sizeof(*fpp) * numrecs);
 	} else {
 		frp = xfs_rtrefcount_droot_rec_addr(dblock, 1);
@@ -661,7 +566,7 @@ xfs_iformat_rtrefcount(
 	}
 
 	broot = xfs_broot_alloc(xfs_ifork_ptr(ip, XFS_DATA_FORK),
-			xfs_rtrefcount_broot_space_calc(mp, level, numrecs));
+			xfs_rtrefcount_broot_space_calc(level, numrecs));
 	if (broot)
 		xfs_rtrefcountbt_from_disk(ip, dfp, dsize, broot);
 	return 0;
@@ -703,7 +608,7 @@ xfs_rtrefcountbt_to_disk(
 		fpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen);
 		tpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs);
 		numrecs = be16_to_cpu(rblock->bb_numrecs);
-		memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs);
+		memcpy(tkp, fkp, sizeof(*fkp) * numrecs);
 		memcpy(tpp, fpp, sizeof(*fpp) * numrecs);
 	} else {
 		frp = xfs_rtrefcount_rec_addr(rblock, 1);
@@ -751,7 +656,7 @@ xfs_rtrefcountbt_create(
 
 	/* Initialize the empty incore btree root. */
 	broot = xfs_broot_realloc(ifp,
-			xfs_rtrefcount_broot_space_calc(mp, 0, 0));
+			xfs_rtrefcount_broot_space_calc(0, 0));
 	if (broot)
 		xfs_btree_init_block(mp, broot, &xfs_rtrefcountbt_ops, 0, 0,
 				I_INO(ip));
diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.h b/fs/xfs/libxfs/xfs_rtrefcount_btree.h
index a99b7a8aec8659..9ab6ecf90ba515 100644
--- a/fs/xfs/libxfs/xfs_rtrefcount_btree.h
+++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.h
@@ -129,7 +129,6 @@ xfs_rtrefcount_broot_ptr_addr(
  */
 static inline size_t
 xfs_rtrefcount_broot_space_calc(
-	struct xfs_mount	*mp,
 	unsigned int		level,
 	unsigned int		nrecs)
 {
@@ -146,9 +145,9 @@ xfs_rtrefcount_broot_space_calc(
  * btree root block.
  */
 static inline size_t
-xfs_rtrefcount_broot_space(struct xfs_mount *mp, struct xfs_rtrefcount_root *bb)
+xfs_rtrefcount_broot_space(struct xfs_rtrefcount_root *bb)
 {
-	return xfs_rtrefcount_broot_space_calc(mp, be16_to_cpu(bb->bb_level),
+	return xfs_rtrefcount_broot_space_calc(be16_to_cpu(bb->bb_level),
 			be16_to_cpu(bb->bb_numrecs));
 }
 
diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.c b/fs/xfs/libxfs/xfs_rtrmap_btree.c
index a15e460a1ec7f4..2f00d0698ae17a 100644
--- a/fs/xfs/libxfs/xfs_rtrmap_btree.c
+++ b/fs/xfs/libxfs/xfs_rtrmap_btree.c
@@ -20,6 +20,7 @@
 #include "xfs_btree_staging.h"
 #include "xfs_metafile.h"
 #include "xfs_rmap.h"
+#include "xfs_rmap_btree.h"
 #include "xfs_rtrmap_btree.h"
 #include "xfs_trace.h"
 #include "xfs_cksum.h"
@@ -113,60 +114,6 @@ xfs_rtrmapbt_get_dmaxrecs(
 	return xfs_rtrmapbt_droot_maxrecs(cur->bc_ino.forksize, level == 0);
 }
 
-/*
- * Convert the ondisk record's offset field into the ondisk key's offset field.
- * Fork and bmbt are significant parts of the rmap record key, but written
- * status is merely a record attribute.
- */
-static inline __be64 ondisk_rec_offset_to_key(const union xfs_btree_rec *rec)
-{
-	return rec->rmap.rm_offset & ~cpu_to_be64(XFS_RMAP_OFF_UNWRITTEN);
-}
-
-STATIC void
-xfs_rtrmapbt_init_key_from_rec(
-	union xfs_btree_key		*key,
-	const union xfs_btree_rec	*rec)
-{
-	key->rmap.rm_startblock = rec->rmap.rm_startblock;
-	key->rmap.rm_owner = rec->rmap.rm_owner;
-	key->rmap.rm_offset = ondisk_rec_offset_to_key(rec);
-}
-
-STATIC void
-xfs_rtrmapbt_init_high_key_from_rec(
-	union xfs_btree_key		*key,
-	const union xfs_btree_rec	*rec)
-{
-	uint64_t			off;
-	int				adj;
-
-	adj = be32_to_cpu(rec->rmap.rm_blockcount) - 1;
-
-	key->rmap.rm_startblock = rec->rmap.rm_startblock;
-	be32_add_cpu(&key->rmap.rm_startblock, adj);
-	key->rmap.rm_owner = rec->rmap.rm_owner;
-	key->rmap.rm_offset = ondisk_rec_offset_to_key(rec);
-	if (XFS_RMAP_NON_INODE_OWNER(be64_to_cpu(rec->rmap.rm_owner)) ||
-	    XFS_RMAP_IS_BMBT_BLOCK(be64_to_cpu(rec->rmap.rm_offset)))
-		return;
-	off = be64_to_cpu(key->rmap.rm_offset);
-	off = (XFS_RMAP_OFF(off) + adj) | (off & ~XFS_RMAP_OFF_MASK);
-	key->rmap.rm_offset = cpu_to_be64(off);
-}
-
-STATIC void
-xfs_rtrmapbt_init_rec_from_cur(
-	struct xfs_btree_cur	*cur,
-	union xfs_btree_rec	*rec)
-{
-	rec->rmap.rm_startblock = cpu_to_be32(cur->bc_rec.r.rm_startblock);
-	rec->rmap.rm_blockcount = cpu_to_be32(cur->bc_rec.r.rm_blockcount);
-	rec->rmap.rm_owner = cpu_to_be64(cur->bc_rec.r.rm_owner);
-	rec->rmap.rm_offset = cpu_to_be64(
-			xfs_rmap_irec_offset_pack(&cur->bc_rec.r));
-}
-
 STATIC void
 xfs_rtrmapbt_init_ptr_from_cur(
 	struct xfs_btree_cur	*cur,
@@ -175,69 +122,6 @@ xfs_rtrmapbt_init_ptr_from_cur(
 	ptr->l = 0;
 }
 
-/*
- * Mask the appropriate parts of the ondisk key field for a key comparison.
- * Fork and bmbt are significant parts of the rmap record key, but written
- * status is merely a record attribute.
- */
-static inline uint64_t offset_keymask(uint64_t offset)
-{
-	return offset & ~XFS_RMAP_OFF_UNWRITTEN;
-}
-
-STATIC int
-xfs_rtrmapbt_cmp_key_with_cur(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*key)
-{
-	struct xfs_rmap_irec		*rec = &cur->bc_rec.r;
-	const struct xfs_rmap_key	*kp = &key->rmap;
-
-	return cmp_int(be32_to_cpu(kp->rm_startblock), rec->rm_startblock) ?:
-	       cmp_int(be64_to_cpu(kp->rm_owner), rec->rm_owner) ?:
-	       cmp_int(offset_keymask(be64_to_cpu(kp->rm_offset)),
-		       offset_keymask(xfs_rmap_irec_offset_pack(rec)));
-}
-
-STATIC int
-xfs_rtrmapbt_cmp_two_keys(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*k1,
-	const union xfs_btree_key	*k2,
-	const union xfs_btree_key	*mask)
-{
-	const struct xfs_rmap_key	*kp1 = &k1->rmap;
-	const struct xfs_rmap_key	*kp2 = &k2->rmap;
-	int				d;
-
-	/* Doesn't make sense to mask off the physical space part */
-	ASSERT(!mask || mask->rmap.rm_startblock);
-
-	d = cmp_int(be32_to_cpu(kp1->rm_startblock),
-		    be32_to_cpu(kp2->rm_startblock));
-	if (d)
-		return d;
-
-	if (!mask || mask->rmap.rm_owner) {
-		d = cmp_int(be64_to_cpu(kp1->rm_owner),
-			    be64_to_cpu(kp2->rm_owner));
-		if (d)
-			return d;
-	}
-
-	if (!mask || mask->rmap.rm_offset) {
-		/* Doesn't make sense to allow offset but not owner */
-		ASSERT(!mask || mask->rmap.rm_owner);
-
-		d = cmp_int(offset_keymask(be64_to_cpu(kp1->rm_offset)),
-			    offset_keymask(be64_to_cpu(kp2->rm_offset)));
-		if (d)
-			return d;
-	}
-
-	return 0;
-}
-
 static xfs_failaddr_t
 xfs_rtrmapbt_verify(
 	struct xfs_buf		*bp)
@@ -304,86 +188,6 @@ const struct xfs_buf_ops xfs_rtrmapbt_buf_ops = {
 	.verify_struct		= xfs_rtrmapbt_verify,
 };
 
-STATIC int
-xfs_rtrmapbt_keys_inorder(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*k1,
-	const union xfs_btree_key	*k2)
-{
-	uint32_t			x;
-	uint32_t			y;
-	uint64_t			a;
-	uint64_t			b;
-
-	x = be32_to_cpu(k1->rmap.rm_startblock);
-	y = be32_to_cpu(k2->rmap.rm_startblock);
-	if (x < y)
-		return 1;
-	else if (x > y)
-		return 0;
-	a = be64_to_cpu(k1->rmap.rm_owner);
-	b = be64_to_cpu(k2->rmap.rm_owner);
-	if (a < b)
-		return 1;
-	else if (a > b)
-		return 0;
-	a = offset_keymask(be64_to_cpu(k1->rmap.rm_offset));
-	b = offset_keymask(be64_to_cpu(k2->rmap.rm_offset));
-	if (a <= b)
-		return 1;
-	return 0;
-}
-
-STATIC int
-xfs_rtrmapbt_recs_inorder(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_rec	*r1,
-	const union xfs_btree_rec	*r2)
-{
-	uint32_t			x;
-	uint32_t			y;
-	uint64_t			a;
-	uint64_t			b;
-
-	x = be32_to_cpu(r1->rmap.rm_startblock);
-	y = be32_to_cpu(r2->rmap.rm_startblock);
-	if (x < y)
-		return 1;
-	else if (x > y)
-		return 0;
-	a = be64_to_cpu(r1->rmap.rm_owner);
-	b = be64_to_cpu(r2->rmap.rm_owner);
-	if (a < b)
-		return 1;
-	else if (a > b)
-		return 0;
-	a = offset_keymask(be64_to_cpu(r1->rmap.rm_offset));
-	b = offset_keymask(be64_to_cpu(r2->rmap.rm_offset));
-	if (a <= b)
-		return 1;
-	return 0;
-}
-
-STATIC enum xbtree_key_contig
-xfs_rtrmapbt_keys_contiguous(
-	struct xfs_btree_cur		*cur,
-	const union xfs_btree_key	*key1,
-	const union xfs_btree_key	*key2,
-	const union xfs_btree_key	*mask)
-{
-	ASSERT(!mask || mask->rmap.rm_startblock);
-
-	/*
-	 * We only support checking contiguity of the physical space component.
-	 * If any callers ever need more specificity than that, they'll have to
-	 * implement it here.
-	 */
-	ASSERT(!mask || (!mask->rmap.rm_owner && !mask->rmap.rm_offset));
-
-	return xbtree_key_contig(be32_to_cpu(key1->rmap.rm_startblock),
-				 be32_to_cpu(key2->rmap.rm_startblock));
-}
-
 static inline void
 xfs_rtrmapbt_move_ptrs(
 	struct xfs_mount	*mp,
@@ -412,7 +216,7 @@ xfs_rtrmapbt_broot_realloc(
 	unsigned int		old_size = ifp->if_broot_bytes;
 	const unsigned int	level = cur->bc_nlevels - 1;
 
-	new_size = xfs_rtrmap_broot_space_calc(mp, level, new_numrecs);
+	new_size = xfs_rtrmap_broot_space_calc(level, new_numrecs);
 
 	/* Handle the nop case quietly. */
 	if (new_size == old_size)
@@ -486,16 +290,16 @@ const struct xfs_btree_ops xfs_rtrmapbt_ops = {
 	.get_minrecs		= xfs_rtrmapbt_get_minrecs,
 	.get_maxrecs		= xfs_rtrmapbt_get_maxrecs,
 	.get_dmaxrecs		= xfs_rtrmapbt_get_dmaxrecs,
-	.init_key_from_rec	= xfs_rtrmapbt_init_key_from_rec,
-	.init_high_key_from_rec	= xfs_rtrmapbt_init_high_key_from_rec,
-	.init_rec_from_cur	= xfs_rtrmapbt_init_rec_from_cur,
+	.init_key_from_rec	= xfs_rmapbt_init_key_from_rec,
+	.init_high_key_from_rec	= xfs_rmapbt_init_high_key_from_rec,
+	.init_rec_from_cur	= xfs_rmapbt_init_rec_from_cur,
 	.init_ptr_from_cur	= xfs_rtrmapbt_init_ptr_from_cur,
-	.cmp_key_with_cur	= xfs_rtrmapbt_cmp_key_with_cur,
+	.cmp_key_with_cur	= xfs_rmapbt_cmp_key_with_cur,
 	.buf_ops		= &xfs_rtrmapbt_buf_ops,
-	.cmp_two_keys		= xfs_rtrmapbt_cmp_two_keys,
-	.keys_inorder		= xfs_rtrmapbt_keys_inorder,
-	.recs_inorder		= xfs_rtrmapbt_recs_inorder,
-	.keys_contiguous	= xfs_rtrmapbt_keys_contiguous,
+	.cmp_two_keys		= xfs_rmapbt_cmp_two_keys,
+	.keys_inorder		= xfs_rmapbt_keys_inorder,
+	.recs_inorder		= xfs_rmapbt_recs_inorder,
+	.keys_contiguous	= xfs_rmapbt_keys_contiguous,
 	.broot_realloc		= xfs_rtrmapbt_broot_realloc,
 };
 
@@ -595,16 +399,16 @@ const struct xfs_btree_ops xfs_rtrmapbt_mem_ops = {
 	.free_block		= xfbtree_free_block,
 	.get_minrecs		= xfbtree_get_minrecs,
 	.get_maxrecs		= xfbtree_get_maxrecs,
-	.init_key_from_rec	= xfs_rtrmapbt_init_key_from_rec,
-	.init_high_key_from_rec	= xfs_rtrmapbt_init_high_key_from_rec,
-	.init_rec_from_cur	= xfs_rtrmapbt_init_rec_from_cur,
+	.init_key_from_rec	= xfs_rmapbt_init_key_from_rec,
+	.init_high_key_from_rec	= xfs_rmapbt_init_high_key_from_rec,
+	.init_rec_from_cur	= xfs_rmapbt_init_rec_from_cur,
 	.init_ptr_from_cur	= xfbtree_init_ptr_from_cur,
-	.cmp_key_with_cur	= xfs_rtrmapbt_cmp_key_with_cur,
+	.cmp_key_with_cur	= xfs_rmapbt_cmp_key_with_cur,
 	.buf_ops		= &xfs_rtrmapbt_mem_buf_ops,
-	.cmp_two_keys		= xfs_rtrmapbt_cmp_two_keys,
-	.keys_inorder		= xfs_rtrmapbt_keys_inorder,
-	.recs_inorder		= xfs_rtrmapbt_recs_inorder,
-	.keys_contiguous	= xfs_rtrmapbt_keys_contiguous,
+	.cmp_two_keys		= xfs_rmapbt_cmp_two_keys,
+	.keys_inorder		= xfs_rmapbt_keys_inorder,
+	.recs_inorder		= xfs_rmapbt_recs_inorder,
+	.keys_contiguous	= xfs_rmapbt_keys_contiguous,
 };
 
 /* Create a cursor for an in-memory btree. */
@@ -895,7 +699,7 @@ xfs_iformat_rtrmap(
 	}
 
 	broot = xfs_broot_alloc(xfs_ifork_ptr(ip, XFS_DATA_FORK),
-			xfs_rtrmap_broot_space_calc(mp, level, numrecs));
+			xfs_rtrmap_broot_space_calc(level, numrecs));
 	if (broot)
 		xfs_rtrmapbt_from_disk(ip, dfp, dsize, broot);
 	return 0;
@@ -980,7 +784,7 @@ xfs_rtrmapbt_create(
 	ASSERT(ifp->if_bytes == 0);
 
 	/* Initialize the empty incore btree root. */
-	broot = xfs_broot_realloc(ifp, xfs_rtrmap_broot_space_calc(mp, 0, 0));
+	broot = xfs_broot_realloc(ifp, xfs_rtrmap_broot_space_calc(0, 0));
 	if (broot)
 		xfs_btree_init_block(mp, broot, &xfs_rtrmapbt_ops, 0, 0,
 				I_INO(ip));
diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.h b/fs/xfs/libxfs/xfs_rtrmap_btree.h
index e328fd62a149d1..c59a144b4bbfbe 100644
--- a/fs/xfs/libxfs/xfs_rtrmap_btree.h
+++ b/fs/xfs/libxfs/xfs_rtrmap_btree.h
@@ -140,7 +140,6 @@ xfs_rtrmap_broot_ptr_addr(
  */
 static inline size_t
 xfs_rtrmap_broot_space_calc(
-	struct xfs_mount	*mp,
 	unsigned int		level,
 	unsigned int		nrecs)
 {
@@ -159,7 +158,7 @@ xfs_rtrmap_broot_space_calc(
 static inline size_t
 xfs_rtrmap_broot_space(struct xfs_mount *mp, struct xfs_rtrmap_root *bb)
 {
-	return xfs_rtrmap_broot_space_calc(mp, be16_to_cpu(bb->bb_level),
+	return xfs_rtrmap_broot_space_calc(be16_to_cpu(bb->bb_level),
 			be16_to_cpu(bb->bb_numrecs));
 }
 
diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c
index f0341adbb8790c..d2ef7b20a49ffa 100644
--- a/fs/xfs/libxfs/xfs_sb.c
+++ b/fs/xfs/libxfs/xfs_sb.c
@@ -1460,46 +1460,6 @@ xfs_update_secondary_sbs(
 	return saved_error ? saved_error : error;
 }
 
-/*
- * Same behavior as xfs_sync_sb, except that it is always synchronous and it
- * also writes the superblock buffer to disk sector 0 immediately.
- */
-int
-xfs_sync_sb_buf(
-	struct xfs_mount	*mp,
-	bool			update_rtsb)
-{
-	struct xfs_trans	*tp;
-	int			error;
-
-	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_sb, 0, 0, 0, &tp);
-	if (error)
-		return error;
-
-	xfs_log_sb(tp);
-	if (update_rtsb)
-		xfs_log_rtsb(tp, xfs_trans_getsb(tp));
-	xfs_trans_set_sync(tp);
-	error = xfs_trans_commit(tp);
-	if (error)
-		return error;
-
-	/* Re-acquire and write the sb and rtsb to disk. */
-	xfs_buf_lock(mp->m_sb_bp);
-	error = xfs_bwrite(mp->m_sb_bp);
-	xfs_buf_unlock(mp->m_sb_bp);
-	if (error)
-		return error;
-
-	if (update_rtsb && mp->m_rtsb_bp) {
-		xfs_buf_lock(mp->m_rtsb_bp);
-		error = xfs_bwrite(mp->m_rtsb_bp);
-		xfs_buf_unlock(mp->m_rtsb_bp);
-	}
-
-	return error;
-}
-
 void
 xfs_fs_geometry(
 	struct xfs_mount	*mp,
diff --git a/fs/xfs/libxfs/xfs_sb.h b/fs/xfs/libxfs/xfs_sb.h
index 34d0dd374e9b0b..77de65922213b1 100644
--- a/fs/xfs/libxfs/xfs_sb.h
+++ b/fs/xfs/libxfs/xfs_sb.h
@@ -15,7 +15,6 @@ struct xfs_perag;
 
 extern void	xfs_log_sb(struct xfs_trans *tp);
 extern int	xfs_sync_sb(struct xfs_mount *mp, bool wait);
-extern int	xfs_sync_sb_buf(struct xfs_mount *mp, bool update_rtsb);
 extern void	xfs_sb_mount_common(struct xfs_mount *mp, struct xfs_sb *sbp);
 void		xfs_sb_mount_rextsize(struct xfs_mount *mp, struct xfs_sb *sbp);
 void		xfs_mount_sb_set_rextsize(struct xfs_mount *mp,
diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c
index b0dc3888bf1b40..0201a3d59b1a24 100644
--- a/fs/xfs/libxfs/xfs_symlink_remote.c
+++ b/fs/xfs/libxfs/xfs_symlink_remote.c
@@ -208,11 +208,15 @@ xfs_symlink_local_to_remote(
  */
 xfs_failaddr_t
 xfs_symlink_shortform_verify(
-	void			*sfp,
-	int64_t			size)
+	struct xfs_inode	*ip)
 {
+	struct xfs_ifork	*ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK);
+	char			*sfp = (char *)ifp->if_data;
+	int			size = ifp->if_bytes;
 	char			*endp = sfp + size;
 
+	ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL);
+
 	/*
 	 * Zero length symlinks should never occur in memory as they are
 	 * never allowed to exist on disk.
diff --git a/fs/xfs/libxfs/xfs_symlink_remote.h b/fs/xfs/libxfs/xfs_symlink_remote.h
index c1672fe1f17bb2..3f6590602473d0 100644
--- a/fs/xfs/libxfs/xfs_symlink_remote.h
+++ b/fs/xfs/libxfs/xfs_symlink_remote.h
@@ -18,7 +18,7 @@ bool xfs_symlink_hdr_ok(xfs_ino_t ino, uint32_t offset,
 void xfs_symlink_local_to_remote(struct xfs_trans *tp, struct xfs_buf *bp,
 				 struct xfs_inode *ip, struct xfs_ifork *ifp,
 				 void *priv);
-xfs_failaddr_t xfs_symlink_shortform_verify(void *sfp, int64_t size);
+xfs_failaddr_t xfs_symlink_shortform_verify(struct xfs_inode *ip);
 int xfs_symlink_remote_read(struct xfs_inode *ip, char *link);
 int xfs_symlink_write_target(struct xfs_trans *tp, struct xfs_inode *ip,
 		xfs_ino_t owner, const char *target_path, int pathlen,
diff --git a/fs/xfs/libxfs/xfs_trans_resv.c b/fs/xfs/libxfs/xfs_trans_resv.c
index 3151e97ca8ff6d..b09c88aec185ea 100644
--- a/fs/xfs/libxfs/xfs_trans_resv.c
+++ b/fs/xfs/libxfs/xfs_trans_resv.c
@@ -606,10 +606,10 @@ static inline unsigned int xfs_calc_pptr_replace_overhead(void)
  */
 STATIC uint
 xfs_calc_rename_reservation(
-	struct xfs_mount	*mp)
+	struct xfs_mount	*mp,
+	struct xfs_trans_resv	*resp)
 {
 	unsigned int		overhead = XFS_DQUOT_LOGRES;
-	struct xfs_trans_resv	*resp = M_RES(mp);
 	unsigned int		t1, t2, t3 = 0;
 
 	t1 = xfs_calc_inode_res(mp, 5) +
@@ -715,10 +715,10 @@ xfs_link_log_count(
  */
 STATIC uint
 xfs_calc_link_reservation(
-	struct xfs_mount	*mp)
+	struct xfs_mount	*mp,
+	struct xfs_trans_resv	*resp)
 {
 	unsigned int		overhead = XFS_DQUOT_LOGRES;
-	struct xfs_trans_resv	*resp = M_RES(mp);
 	unsigned int		t1, t2, t3 = 0;
 
 	overhead += xfs_calc_iunlink_remove_reservation(mp);
@@ -777,10 +777,10 @@ xfs_remove_log_count(
  */
 STATIC uint
 xfs_calc_remove_reservation(
-	struct xfs_mount	*mp)
+	struct xfs_mount	*mp,
+	struct xfs_trans_resv	*resp)
 {
 	unsigned int            overhead = XFS_DQUOT_LOGRES;
-	struct xfs_trans_resv   *resp = M_RES(mp);
 	unsigned int            t1, t2, t3 = 0;
 
 	overhead += xfs_calc_iunlink_add_reservation(mp);
@@ -862,9 +862,9 @@ xfs_icreate_log_count(
 
 STATIC uint
 xfs_calc_icreate_reservation(
-	struct xfs_mount	*mp)
+	struct xfs_mount	*mp,
+	struct xfs_trans_resv	*resp)
 {
-	struct xfs_trans_resv	*resp = M_RES(mp);
 	unsigned int		overhead = XFS_DQUOT_LOGRES;
 	unsigned int		t1, t2, t3 = 0;
 
@@ -911,9 +911,10 @@ xfs_mkdir_log_count(
  */
 STATIC uint
 xfs_calc_mkdir_reservation(
-	struct xfs_mount	*mp)
+	struct xfs_mount	*mp,
+	struct xfs_trans_resv	*resp)
 {
-	return xfs_calc_icreate_reservation(mp);
+	return xfs_calc_icreate_reservation(mp, resp);
 }
 
 static inline unsigned int
@@ -940,9 +941,10 @@ xfs_symlink_log_count(
  */
 STATIC uint
 xfs_calc_symlink_reservation(
-	struct xfs_mount	*mp)
+	struct xfs_mount	*mp,
+	struct xfs_trans_resv	*resp)
 {
-	return xfs_calc_icreate_reservation(mp) +
+	return xfs_calc_icreate_reservation(mp, resp) +
 	       xfs_calc_buf_res(1, XFS_SYMLINK_MAXLEN);
 }
 
@@ -1265,34 +1267,33 @@ xfs_calc_namespace_reservations(
 {
 	ASSERT(resp->tr_attrsetm.tr_logres > 0);
 
-	resp->tr_rename.tr_logres = xfs_calc_rename_reservation(mp);
+	resp->tr_rename.tr_logres = xfs_calc_rename_reservation(mp, resp);
 	resp->tr_rename.tr_logcount = xfs_rename_log_count(mp, resp);
 	resp->tr_rename.tr_logflags |= XFS_TRANS_PERM_LOG_RES;
 
-	resp->tr_link.tr_logres = xfs_calc_link_reservation(mp);
+	resp->tr_link.tr_logres = xfs_calc_link_reservation(mp, resp);
 	resp->tr_link.tr_logcount = xfs_link_log_count(mp, resp);
 	resp->tr_link.tr_logflags |= XFS_TRANS_PERM_LOG_RES;
 
-	resp->tr_remove.tr_logres = xfs_calc_remove_reservation(mp);
+	resp->tr_remove.tr_logres = xfs_calc_remove_reservation(mp, resp);
 	resp->tr_remove.tr_logcount = xfs_remove_log_count(mp, resp);
 	resp->tr_remove.tr_logflags |= XFS_TRANS_PERM_LOG_RES;
 
-	resp->tr_symlink.tr_logres = xfs_calc_symlink_reservation(mp);
+	resp->tr_symlink.tr_logres = xfs_calc_symlink_reservation(mp, resp);
 	resp->tr_symlink.tr_logcount = xfs_symlink_log_count(mp, resp);
 	resp->tr_symlink.tr_logflags |= XFS_TRANS_PERM_LOG_RES;
 
-	resp->tr_create.tr_logres = xfs_calc_icreate_reservation(mp);
+	resp->tr_create.tr_logres = xfs_calc_icreate_reservation(mp, resp);
 	resp->tr_create.tr_logcount = xfs_icreate_log_count(mp, resp);
 	resp->tr_create.tr_logflags |= XFS_TRANS_PERM_LOG_RES;
 
-	resp->tr_mkdir.tr_logres = xfs_calc_mkdir_reservation(mp);
+	resp->tr_mkdir.tr_logres = xfs_calc_mkdir_reservation(mp, resp);
 	resp->tr_mkdir.tr_logcount = xfs_mkdir_log_count(mp, resp);
 	resp->tr_mkdir.tr_logflags |= XFS_TRANS_PERM_LOG_RES;
 }
 
 STATIC void
 xfs_calc_default_atomic_ioend_reservation(
-	struct xfs_mount	*mp,
 	struct xfs_trans_resv	*resp)
 {
 	/* Pick a default that will scale reasonably for the log size. */
@@ -1398,7 +1399,7 @@ xfs_trans_resv_calc(
 	 * Now that we've finished computing the static reservations, we can
 	 * compute the dynamic reservation for atomic writes.
 	 */
-	xfs_calc_default_atomic_ioend_reservation(mp, resp);
+	xfs_calc_default_atomic_ioend_reservation(resp);
 }
 
 /*
@@ -1508,7 +1509,7 @@ xfs_calc_atomic_write_log_geometry(
 
 	ASSERT(blockcount > 0);
 
-	xfs_calc_default_atomic_ioend_reservation(mp, M_RES(mp));
+	xfs_calc_default_atomic_ioend_reservation(M_RES(mp));
 
 	per_intent = xfs_calc_atomic_write_ioend_geometry(mp, &step_size);
 
@@ -1545,7 +1546,7 @@ xfs_calc_atomic_write_reservation(
 	 * use the defaults.
 	 */
 	if (blockcount == 0) {
-		xfs_calc_default_atomic_ioend_reservation(mp, M_RES(mp));
+		xfs_calc_default_atomic_ioend_reservation(M_RES(mp));
 		return 0;
 	}
 
diff --git a/fs/xfs/libxfs/xfs_trans_space.c b/fs/xfs/libxfs/xfs_trans_space.c
index c4cd547033e584..7edd0d86f0bdb2 100644
--- a/fs/xfs/libxfs/xfs_trans_space.c
+++ b/fs/xfs/libxfs/xfs_trans_space.c
@@ -127,10 +127,10 @@ xfs_rename_space_res(
 		if (has_whiteout)
 			ret += xfs_parent_calc_space_res(mp, src_namelen);
 		ret += 2 * xfs_parent_calc_space_res(mp, target_namelen);
-	}
 
-	if (target_exists)
-		ret += xfs_parent_calc_space_res(mp, target_namelen);
+		if (target_exists)
+			ret += xfs_parent_calc_space_res(mp, target_namelen);
+	}
 
 	return ret;
 }
diff --git a/fs/xfs/libxfs/xfs_types.c b/fs/xfs/libxfs/xfs_types.c
index 67c947a47f1478..f195a04dbf666f 100644
--- a/fs/xfs/libxfs/xfs_types.c
+++ b/fs/xfs/libxfs/xfs_types.c
@@ -222,7 +222,6 @@ xfs_verify_icount(
 /* Sanity-checking of dir/attr block offsets. */
 bool
 xfs_verify_dablk(
-	struct xfs_mount	*mp,
 	xfs_fileoff_t		dabno)
 {
 	xfs_dablk_t		max_dablk = -1U;
@@ -233,7 +232,6 @@ xfs_verify_dablk(
 /* Check that a file block offset does not exceed the maximum. */
 bool
 xfs_verify_fileoff(
-	struct xfs_mount	*mp,
 	xfs_fileoff_t		off)
 {
 	return off <= XFS_MAX_FILEOFF;
@@ -242,15 +240,14 @@ xfs_verify_fileoff(
 /* Check that a range of file block offsets do not exceed the maximum. */
 bool
 xfs_verify_fileext(
-	struct xfs_mount	*mp,
 	xfs_fileoff_t		off,
 	xfs_fileoff_t		len)
 {
 	if (off + len <= off)
 		return false;
 
-	if (!xfs_verify_fileoff(mp, off))
+	if (!xfs_verify_fileoff(off))
 		return false;
 
-	return xfs_verify_fileoff(mp, off + len - 1);
+	return xfs_verify_fileoff(off + len - 1);
 }
diff --git a/fs/xfs/libxfs/xfs_types.h b/fs/xfs/libxfs/xfs_types.h
index f6f4f2d4b5dbf5..19dd5e7c8b1279 100644
--- a/fs/xfs/libxfs/xfs_types.h
+++ b/fs/xfs/libxfs/xfs_types.h
@@ -277,11 +277,10 @@ bool xfs_verify_rtbno(struct xfs_mount *mp, xfs_rtblock_t rtbno);
 bool xfs_verify_rtbext(struct xfs_mount *mp, xfs_rtblock_t rtbno,
 		xfs_filblks_t len);
 bool xfs_verify_icount(struct xfs_mount *mp, unsigned long long icount);
-bool xfs_verify_dablk(struct xfs_mount *mp, xfs_fileoff_t off);
+bool xfs_verify_dablk(xfs_fileoff_t off);
 void xfs_icount_range(struct xfs_mount *mp, unsigned long long *min,
 		unsigned long long *max);
-bool xfs_verify_fileoff(struct xfs_mount *mp, xfs_fileoff_t off);
-bool xfs_verify_fileext(struct xfs_mount *mp, xfs_fileoff_t off,
-		xfs_fileoff_t len);
+bool xfs_verify_fileoff(xfs_fileoff_t off);
+bool xfs_verify_fileext(xfs_fileoff_t off, xfs_fileoff_t len);
 
 #endif	/* __XFS_TYPES_H__ */
diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c
index a66b611588c47f..ff1b4b361cf2e0 100644
--- a/fs/xfs/scrub/agheader_repair.c
+++ b/fs/xfs/scrub/agheader_repair.c
@@ -1369,7 +1369,7 @@ xrep_iunlink_mark_ondisk(
 
 /*
  * Walk an iunlink bucket's inode list.  For each inode that should be on this
- * chain, clear its entry in in iunlink_bmp because it's ok and we don't need
+ * chain, clear its entry in iunlink_bmp because it's ok and we don't need
  * to touch it further.
  */
 STATIC int
diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c
index 95e318e4f3a6c7..6e835240536181 100644
--- a/fs/xfs/scrub/alloc_repair.c
+++ b/fs/xfs/scrub/alloc_repair.c
@@ -108,9 +108,6 @@ struct xrep_abt {
 
 	struct xfs_scrub	*sc;
 
-	/* Number of non-null records in @free_records. */
-	uint64_t		nr_real_records;
-
 	/* get_records()'s position in the free space record array. */
 	xfarray_idx_t		array_cur;
 
@@ -338,7 +335,7 @@ xrep_cntbt_extent_cmp(
 }
 
 /*
- * Sort the free extents by length so so that we can put the records into the
+ * Sort the free extents by length so that we can put the records into the
  * cntbt in the correct order.  Don't let userspace kill us if we're resorting
  * after allocating btree blocks.
  */
@@ -403,7 +400,6 @@ xrep_abt_find_freespace(
 	if (error)
 		goto err_agfl;
 
-	ra->nr_real_records = xfarray_length(ra->free_records);
 err_agfl:
 	xfs_trans_brelse(sc->tp, agfl_bp);
 err:
@@ -446,15 +442,17 @@ xrep_abt_reserve_space(
 		uint64_t		required;
 		unsigned int		desired;
 		unsigned int		len;
+		const uint64_t		nr_records =
+			xfarray_length(ra->free_records);
 
 		/* Compute how many blocks we'll need. */
 		error = xfs_btree_bload_compute_geometry(cnt_cur,
-				&ra->new_cntbt.bload, ra->nr_real_records);
+				&ra->new_cntbt.bload, nr_records);
 		if (error)
 			break;
 
 		error = xfs_btree_bload_compute_geometry(bno_cur,
-				&ra->new_bnobt.bload, ra->nr_real_records);
+				&ra->new_bnobt.bload, nr_records);
 		if (error)
 			break;
 
@@ -470,7 +468,7 @@ xrep_abt_reserve_space(
 		desired = required - allocated;
 
 		/* We need space but there's none left; bye! */
-		if (ra->nr_real_records == 0) {
+		if (nr_records == 0) {
 			error = -ENOSPC;
 			break;
 		}
@@ -517,10 +515,9 @@ xrep_abt_reserve_space(
 		 * records (but doesn't break the sorting order), so we must
 		 * go around the loop once more to re-run _bload_init.
 		 */
-		error = xfarray_unset(ra->free_records, record_nr);
+		error = xfarray_trim(ra->free_records, 1);
 		if (error)
 			break;
-		ra->nr_real_records--;
 		record_nr--;
 	} while (1);
 
diff --git a/fs/xfs/scrub/bitmap.c b/fs/xfs/scrub/bitmap.c
index c7fa908d92b25e..08f216d26ec632 100644
--- a/fs/xfs/scrub/bitmap.c
+++ b/fs/xfs/scrub/bitmap.c
@@ -122,8 +122,8 @@ xbitmap64_set(
 	uint64_t		start,
 	uint64_t		len)
 {
-	struct xbitmap64_node	*left;
-	struct xbitmap64_node	*right;
+	struct xbitmap64_node	*left = NULL;
+	struct xbitmap64_node	*right = NULL;
 	uint64_t		last = start + len - 1;
 	int			error;
 
@@ -131,6 +131,7 @@ xbitmap64_set(
 	left = xbitmap64_tree_iter_first(&bitmap->xb_root, start, last);
 	if (left && left->bn_start <= start && left->bn_last >= last)
 		return 0;
+	left = NULL;
 
 	/* Clear out everything in the range we want to set. */
 	error = xbitmap64_clear(bitmap, start, len);
@@ -138,11 +139,15 @@ xbitmap64_set(
 		return error;
 
 	/* Do we have a left-adjacent extent? */
-	left = xbitmap64_tree_iter_first(&bitmap->xb_root, start - 1, start - 1);
+	if (start > 0)
+		left = xbitmap64_tree_iter_first(&bitmap->xb_root, start - 1,
+				start - 1);
 	ASSERT(!left || left->bn_last + 1 == start);
 
 	/* Do we have a right-adjacent extent? */
-	right = xbitmap64_tree_iter_first(&bitmap->xb_root, last + 1, last + 1);
+	if (last < U64_MAX)
+		right = xbitmap64_tree_iter_first(&bitmap->xb_root, last + 1,
+				last + 1);
 	ASSERT(!right || right->bn_start == last + 1);
 
 	if (left && right) {
@@ -397,8 +402,8 @@ xbitmap32_set(
 	uint32_t		start,
 	uint32_t		len)
 {
-	struct xbitmap32_node	*left;
-	struct xbitmap32_node	*right;
+	struct xbitmap32_node	*left = NULL;
+	struct xbitmap32_node	*right = NULL;
 	uint32_t		last = start + len - 1;
 	int			error;
 
@@ -406,6 +411,7 @@ xbitmap32_set(
 	left = xbitmap32_tree_iter_first(&bitmap->xb_root, start, last);
 	if (left && left->bn_start <= start && left->bn_last >= last)
 		return 0;
+	left = NULL;
 
 	/* Clear out everything in the range we want to set. */
 	error = xbitmap32_clear(bitmap, start, len);
@@ -413,11 +419,15 @@ xbitmap32_set(
 		return error;
 
 	/* Do we have a left-adjacent extent? */
-	left = xbitmap32_tree_iter_first(&bitmap->xb_root, start - 1, start - 1);
+	if (start > 0)
+		left = xbitmap32_tree_iter_first(&bitmap->xb_root, start - 1,
+				start - 1);
 	ASSERT(!left || left->bn_last + 1 == start);
 
 	/* Do we have a right-adjacent extent? */
-	right = xbitmap32_tree_iter_first(&bitmap->xb_root, last + 1, last + 1);
+	if (last < U32_MAX)
+		right = xbitmap32_tree_iter_first(&bitmap->xb_root, last + 1,
+				last + 1);
 	ASSERT(!right || right->bn_start == last + 1);
 
 	if (left && right) {
diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c
index 4f3c7f681bd921..c190590bc56216 100644
--- a/fs/xfs/scrub/bmap.c
+++ b/fs/xfs/scrub/bmap.c
@@ -453,18 +453,17 @@ xchk_bmap_dirattr_extent(
 	struct xchk_bmap_info	*info,
 	struct xfs_bmbt_irec	*irec)
 {
-	struct xfs_mount	*mp = ip->i_mount;
 	xfs_fileoff_t		off;
 
 	if (!S_ISDIR(VFS_I(ip)->i_mode) && info->whichfork != XFS_ATTR_FORK)
 		return;
 
-	if (!xfs_verify_dablk(mp, irec->br_startoff))
+	if (!xfs_verify_dablk(irec->br_startoff))
 		xchk_fblock_set_corrupt(info->sc, info->whichfork,
 				irec->br_startoff);
 
 	off = irec->br_startoff + irec->br_blockcount - 1;
-	if (!xfs_verify_dablk(mp, off))
+	if (!xfs_verify_dablk(off))
 		xchk_fblock_set_corrupt(info->sc, info->whichfork, off);
 }
 
@@ -486,7 +485,7 @@ xchk_bmap_iextent(
 		xchk_fblock_set_corrupt(info->sc, info->whichfork,
 				irec->br_startoff);
 
-	if (!xfs_verify_fileext(mp, irec->br_startoff, irec->br_blockcount))
+	if (!xfs_verify_fileext(irec->br_startoff, irec->br_blockcount))
 		xchk_fblock_set_corrupt(info->sc, info->whichfork,
 				irec->br_startoff);
 
@@ -877,8 +876,6 @@ xchk_bmap_iextent_delalloc(
 	struct xchk_bmap_info	*info,
 	struct xfs_bmbt_irec	*irec)
 {
-	struct xfs_mount	*mp = info->sc->mp;
-
 	/*
 	 * Check for out-of-order extents.  This record could have come
 	 * from the incore list, for which there is no ordering check.
@@ -888,7 +885,7 @@ xchk_bmap_iextent_delalloc(
 		xchk_fblock_set_corrupt(info->sc, info->whichfork,
 				irec->br_startoff);
 
-	if (!xfs_verify_fileext(mp, irec->br_startoff, irec->br_blockcount))
+	if (!xfs_verify_fileext(irec->br_startoff, irec->br_blockcount))
 		xchk_fblock_set_corrupt(info->sc, info->whichfork,
 				irec->br_startoff);
 
diff --git a/fs/xfs/scrub/bmap_repair.c b/fs/xfs/scrub/bmap_repair.c
index eabffba47776e9..03af6cb92fcf5e 100644
--- a/fs/xfs/scrub/bmap_repair.c
+++ b/fs/xfs/scrub/bmap_repair.c
@@ -211,7 +211,7 @@ xrep_bmap_check_fork_rmap(
 
 	/* Check the file offset range. */
 	if (!(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) &&
-	    !xfs_verify_fileext(sc->mp, rec->rm_offset, rec->rm_blockcount))
+	    !xfs_verify_fileext(rec->rm_offset, rec->rm_blockcount))
 		return -EFSCORRUPTED;
 
 	/* No contradictory flags. */
@@ -389,7 +389,7 @@ xrep_bmap_check_rtfork_rmap(
 		return -EFSCORRUPTED;
 
 	/* Check the file offsets and physical extents. */
-	if (!xfs_verify_fileext(sc->mp, rec->rm_offset, rec->rm_blockcount))
+	if (!xfs_verify_fileext(rec->rm_offset, rec->rm_blockcount))
 		return -EFSCORRUPTED;
 
 	/* Check that this is within the rtgroup. */
diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c
index 2a037aae904d03..19d974c7e2b7a9 100644
--- a/fs/xfs/scrub/dir.c
+++ b/fs/xfs/scrub/dir.c
@@ -492,7 +492,7 @@ xchk_directory_data_bestfree(
 		goto out;
 	xchk_buffer_recheck(sc, bp);
 
-	if (xfs_has_crc(sc->mp)) {
+	if (!is_block && xfs_has_crc(sc->mp)) {
 		struct xfs_dir3_data_hdr    *hdr3 = bp->b_addr;
 
 		if (hdr3->pad)
diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c
index 9b0ab23166124e..887383d2e94104 100644
--- a/fs/xfs/scrub/dirtree.c
+++ b/fs/xfs/scrub/dirtree.c
@@ -1021,7 +1021,7 @@ xchk_dirtree(
 	return error;
 }
 
-/* Does the directory targetted by this scrub have no parents? */
+/* Does the directory targeted by this scrub have no parents? */
 bool
 xchk_dirtree_parentless(const struct xchk_dirtree *dl)
 {
diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c
index eab3ac2704befe..d921fe5a9b0c76 100644
--- a/fs/xfs/scrub/findparent.c
+++ b/fs/xfs/scrub/findparent.c
@@ -473,6 +473,9 @@ xrep_findparent_from_dcache(
 	pip = igrab(d_inode(parent));
 	dput(parent);
 
+	if (!pip)
+		goto out_dput;
+
 	if (S_ISDIR(pip->i_mode)) {
 		ret = pip->i_ino;
 		trace_xrep_findparent_from_dcache(sc->ip, ret);
diff --git a/fs/xfs/scrub/health.c b/fs/xfs/scrub/health.c
index 2171bcf0f6c1f2..487ecc5f9f3c29 100644
--- a/fs/xfs/scrub/health.c
+++ b/fs/xfs/scrub/health.c
@@ -202,9 +202,9 @@ xchk_update_health(
 	 * there's no sick flag defined for it, so we branch here ahead of the
 	 * mask check.
 	 */
-	if (sc->sm->sm_type == XFS_SCRUB_TYPE_HEALTHY &&
-	    !(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) {
-		xchk_mark_all_healthy(sc->mp);
+	if (sc->sm->sm_type == XFS_SCRUB_TYPE_HEALTHY) {
+		if (!(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT))
+			xchk_mark_all_healthy(sc->mp);
 		return;
 	}
 
diff --git a/fs/xfs/scrub/inode.c b/fs/xfs/scrub/inode.c
index 65b13e31191681..46e9bf4a43179e 100644
--- a/fs/xfs/scrub/inode.c
+++ b/fs/xfs/scrub/inode.c
@@ -607,7 +607,7 @@ xchk_dinode(
 	}
 
 	/* di_forkoff */
-	if (XFS_DFORK_BOFF(dip) >= mp->m_sb.sb_inodesize)
+	if (dip->di_forkoff >= (XFS_LITINO(mp) >> 3))
 		xchk_ino_set_corrupt(sc, ino);
 	if (naextents != 0 && dip->di_forkoff == 0)
 		xchk_ino_set_corrupt(sc, ino);
diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c
index 8bc508336aa5f4..c65912d8765904 100644
--- a/fs/xfs/scrub/inode_repair.c
+++ b/fs/xfs/scrub/inode_repair.c
@@ -933,7 +933,7 @@ xrep_dinode_bad_bmbt_fork(
 
 		fkp = xfs_bmdr_key_addr(dfp, i);
 		fileoff = be64_to_cpu(fkp->br_startoff);
-		if (!xfs_verify_fileoff(sc->mp, fileoff))
+		if (!xfs_verify_fileoff(fileoff))
 			return true;
 
 		fpp = xfs_bmdr_ptr_addr(dfp, i, dmxr);
@@ -1024,6 +1024,30 @@ xrep_dinode_bad_metabt_fork(
 	return false;
 }
 
+static xfs_failaddr_t
+xrep_symlink_shortform_verify(
+	void			*sfp,
+	int64_t			size)
+{
+	/*
+	 * Zero length symlinks should never occur in memory as they are
+	 * never allowed to exist on disk.
+	 */
+	if (!size)
+		return __this_address;
+
+	/* No negative sizes or overly long symlink targets. */
+	if (size < 0 || size > XFS_SYMLINK_MAXLEN)
+		return __this_address;
+
+	/* No NULLs in the target either. */
+	if (memchr(sfp, 0, size))
+		return __this_address;
+
+	/* ondisk symlink target isn't null terminated, unlike incore */
+	return NULL;
+}
+
 /*
  * Check the data fork for things that will fail the ifork verifiers or the
  * ifork formatters.
@@ -1099,7 +1123,7 @@ xrep_dinode_check_dfork(
 			return true;
 		/* symlink structure must pass verification. */
 		if (S_ISLNK(mode) &&
-		    xfs_symlink_shortform_verify(dfork_ptr, data_size) != NULL)
+		    xrep_symlink_shortform_verify(dfork_ptr, data_size) != NULL)
 			return true;
 		break;
 	case XFS_DINODE_FMT_EXTENTS:
@@ -1405,7 +1429,7 @@ xrep_dinode_ensure_forkoff(
 			break;
 		case XFS_METAFILE_RTREFCOUNT:
 			rcdr = XFS_DFORK_PTR(dip, XFS_DATA_FORK);
-			dfork_min = xfs_rtrefcount_broot_space(sc->mp, rcdr);
+			dfork_min = xfs_rtrefcount_broot_space(rcdr);
 			break;
 		default:
 			dfork_min = 0;
@@ -1702,7 +1726,7 @@ xrep_inode_blockcounts(
 				&acount);
 		if (error)
 			return error;
-		if (count >= sc->mp->m_sb.sb_dblocks)
+		if (acount >= sc->mp->m_sb.sb_dblocks)
 			return -EFSCORRUPTED;
 		error = xrep_ino_ensure_extent_count(sc, XFS_ATTR_FORK,
 				nextents);
@@ -1949,7 +1973,7 @@ xrep_inode_pptr(
 		return 0;
 
 	return xfs_bmap_add_attrfork(sc->tp, ip,
-			sizeof(struct xfs_attr_sf_hdr), true);
+			sizeof(struct xfs_attr_sf_hdr));
 }
 
 /* Fix COW extent size hint problems. */
diff --git a/fs/xfs/scrub/newbt.c b/fs/xfs/scrub/newbt.c
index c82f4631fd9c27..584076b2a6eed9 100644
--- a/fs/xfs/scrub/newbt.c
+++ b/fs/xfs/scrub/newbt.c
@@ -193,9 +193,11 @@ xrep_newbt_add_blocks(
 	struct xrep_newbt_resv		*resv;
 	int				error;
 
-	resv = kmalloc_obj(struct xrep_newbt_resv, XCHK_GFP_FLAGS);
-	if (!resv)
-		return -ENOMEM;
+	/*
+	 * We have no way to clean up the allocated space *and* return an
+	 * ENOMEM if we fail to allocate this control structure.
+	 */
+	resv = kmalloc_obj(struct xrep_newbt_resv, GFP_KERNEL | __GFP_NOFAIL);
 
 	INIT_LIST_HEAD(&resv->list);
 	resv->agbno = XFS_FSB_TO_AGBNO(mp, args->fsbno);
diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c
index 3aca66869b8000..d8e1457f0d5e1a 100644
--- a/fs/xfs/scrub/orphanage.c
+++ b/fs/xfs/scrub/orphanage.c
@@ -192,12 +192,16 @@ xrep_orphanage_create(
 	/* Make sure the orphanage is owned by root. */
 	error = xrep_chown_orphanage(sc, XFS_I(orphanage_inode));
 	if (error)
-		goto out_dput_orphanage;
+		goto out_rele_orphanage;
 
 	/* Stash the reference for later and bail out. */
 	sc->orphanage = XFS_I(orphanage_inode);
 	sc->orphanage_ilock_flags = 0;
+	orphanage_inode = NULL;
 
+out_rele_orphanage:
+	if (orphanage_inode)
+		xchk_irele(sc, XFS_I(orphanage_inode));
 out_dput_orphanage:
 	end_creating(orphanage_dentry);
 out_dput_root:
@@ -546,7 +550,7 @@ xrep_adoption_move(
 	if (!xfs_inode_has_attr_fork(sc->ip) && xfs_has_parent(sc->mp)) {
 		int sf_size = xrep_adoption_attr_sizeof(adopt);
 
-		error = xfs_bmap_add_attrfork(sc->tp, sc->ip, sf_size, true);
+		error = xfs_bmap_add_attrfork(sc->tp, sc->ip, sf_size);
 		if (error)
 			return error;
 	}
diff --git a/fs/xfs/scrub/quota.c b/fs/xfs/scrub/quota.c
index 222812fe202c21..8c6ba1240fd53a 100644
--- a/fs/xfs/scrub/quota.c
+++ b/fs/xfs/scrub/quota.c
@@ -89,7 +89,7 @@ xchk_quota_item_bmap(
 	int			nmaps = 1;
 	int			error;
 
-	if (!xfs_verify_fileoff(mp, offset)) {
+	if (!xfs_verify_fileoff(offset)) {
 		xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset);
 		return 0;
 	}
diff --git a/fs/xfs/scrub/quota_repair.c b/fs/xfs/scrub/quota_repair.c
index 59302e8afc7ef0..40bb85e8f90927 100644
--- a/fs/xfs/scrub/quota_repair.c
+++ b/fs/xfs/scrub/quota_repair.c
@@ -116,8 +116,8 @@ xrep_quota_item_bmap(
 	int			error;
 
 	/* The computed file offset should always be valid. */
-	if (!xfs_verify_fileoff(mp, offset)) {
-		ASSERT(xfs_verify_fileoff(mp, offset));
+	if (!xfs_verify_fileoff(offset)) {
+		ASSERT(xfs_verify_fileoff(offset));
 		return -EFSCORRUPTED;
 	}
 	dq->q_fileoffset = offset;
@@ -248,10 +248,7 @@ xrep_quota_item(
 
 	dq->q_flags |= XFS_DQFLAG_DIRTY;
 	xfs_trans_dqjoin(sc->tp, dq);
-	if (dq->q_id) {
-		xfs_qm_adjust_dqlimits(dq);
-		xfs_qm_adjust_dqtimers(dq);
-	}
+	xfs_qm_adjust_dqenforcement(dq);
 	xfs_trans_log_dquot(sc->tp, dq);
 	return xfs_trans_roll(&sc->tp);
 
diff --git a/fs/xfs/scrub/quotacheck_repair.c b/fs/xfs/scrub/quotacheck_repair.c
index dbb522e1513b0b..b8334edf380c7f 100644
--- a/fs/xfs/scrub/quotacheck_repair.c
+++ b/fs/xfs/scrub/quotacheck_repair.c
@@ -39,6 +39,54 @@
  * dquot is locked.
  */
 
+static bool
+xqcheck_dqres_force_dirty(
+	const struct xfs_dquot_res	*res,
+	const struct xfs_quota_limits	*qlim)
+{
+	/* zero limits mean that we should set the default limits */
+	if (res->softlimit == 0 && qlim->soft != 0)
+		return true;
+	if (res->hardlimit == 0 && qlim->hard != 0)
+		return true;
+
+	/* do we need to adjust the timer setting? */
+	if ((res->softlimit && res->count > res->softlimit) ||
+	    (res->hardlimit && res->count > res->hardlimit)) {
+		if (!res->timer)
+			return true;
+	} else {
+		if (res->timer)
+			return true;
+	}
+
+	return false;
+}
+
+/* Decide if we need to adjust the dquot limits or timers */
+static bool
+xqcheck_dquot_force_dirty(
+	const struct xfs_dquot	*dq)
+{
+	struct xfs_quotainfo	*qi = dq->q_mount->m_quotainfo;
+	struct xfs_def_quota	*defq;
+
+	/* root dquot does not enforce limits */
+	if (dq->q_id == 0)
+		return false;
+
+	defq = xfs_get_defquota(qi, xfs_dquot_type(dq));
+
+	if (xqcheck_dqres_force_dirty(&dq->q_blk, &defq->blk))
+		return true;
+	if (xqcheck_dqres_force_dirty(&dq->q_ino, &defq->ino))
+		return true;
+	if (xqcheck_dqres_force_dirty(&dq->q_rtb, &defq->rtb))
+		return true;
+
+	return false;
+}
+
 /* Commit new counters to a dquot. */
 static int
 xqcheck_commit_dquot(
@@ -91,6 +139,9 @@ xqcheck_commit_dquot(
 		dirty = true;
 	}
 
+	if (!dirty && xqcheck_dquot_force_dirty(dq))
+		dirty = true;
+
 	xcdq.flags |= (XQCHECK_DQUOT_REPAIR_SCANNED | XQCHECK_DQUOT_WRITTEN);
 	error = xfarray_store(counts, dq->q_id, &xcdq);
 	if (error == -EFBIG) {
@@ -110,8 +161,7 @@ xqcheck_commit_dquot(
 
 	/* Commit the dirty dquot to disk. */
 	dq->q_flags |= XFS_DQFLAG_DIRTY;
-	if (dq->q_id)
-		xfs_qm_adjust_dqtimers(dq);
+	xfs_qm_adjust_dqenforcement(dq);
 	xfs_trans_log_dquot(xqc->sc->tp, dq);
 	return xrep_trans_commit(xqc->sc);
 
diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c
index f698b9be3dd1df..0dfe61bafc6a03 100644
--- a/fs/xfs/scrub/reap.c
+++ b/fs/xfs/scrub/reap.c
@@ -172,7 +172,7 @@ static inline bool xreap_is_dirty(const struct xreap_state *rs)
 }
 
 /*
- * Decide if we need to roll the transaction to clear out the the log
+ * Decide if we need to roll the transaction to clear out the log
  * reservation that we allocated to buffer invalidations.
  */
 static inline bool xreap_want_binval_roll(const struct xreap_state *rs)
diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c
index 11697a8b2a1d13..c2a437416227a8 100644
--- a/fs/xfs/scrub/repair.c
+++ b/fs/xfs/scrub/repair.c
@@ -399,6 +399,7 @@ xrep_calc_rtgroup_resblks(
 	struct xfs_mount		*mp = sc->mp;
 	struct xfs_scrub_metadata	*sm = sc->sm;
 	uint64_t			usedlen;
+	xfs_extlen_t			refcbt_sz = 0;
 	xfs_extlen_t			rmapbt_sz = 0;
 
 	if (!(sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR))
@@ -411,13 +412,27 @@ xrep_calc_rtgroup_resblks(
 	usedlen = xfs_rtbxlen_to_blen(mp, xfs_rtgroup_extents(mp, sm->sm_agno));
 	ASSERT(usedlen <= XFS_MAX_RGBLOCKS);
 
+	if (xfs_has_reflink(mp))
+		refcbt_sz = xfs_rtrefcountbt_calc_size(mp, usedlen);
+
 	if (xfs_has_rmapbt(mp))
 		rmapbt_sz = xfs_rtrmapbt_calc_size(mp, usedlen);
 
+	/*
+	 * Guess how many blocks we need to rebuild the rmapbt.  For
+	 * non-reflink filesystems we can't have more records than used blocks.
+	 * However, with reflink it's possible to have more than one rmap
+	 * record per rtgroup block.  We don't know how many rmaps there could
+	 * be in the rtgroup, so we start off with what we hope is an generous
+	 * over-estimation.
+	 */
+	if (refcbt_sz > 0 && rmapbt_sz > 0)
+		rmapbt_sz *= 2;
+
 	trace_xrep_calc_rtgroup_resblks_btsize(mp, sm->sm_agno, usedlen,
-			rmapbt_sz);
+			rmapbt_sz, refcbt_sz);
 
-	return rmapbt_sz;
+	return max(rmapbt_sz, refcbt_sz);
 }
 #endif /* CONFIG_XFS_RT */
 
diff --git a/fs/xfs/scrub/rmap.c b/fs/xfs/scrub/rmap.c
index 0cd3eecd2ca5b4..68e2847c962bb2 100644
--- a/fs/xfs/scrub/rmap.c
+++ b/fs/xfs/scrub/rmap.c
@@ -493,11 +493,18 @@ xchk_rmapbt_walk_ag_metadata(
 	 * If there's an error, set XFAIL and disable the bitmap
 	 * cross-referencing checks, but proceed with the scrub anyway.
 	 */
-	if (error)
-		xchk_btree_xref_process_error(sc, sc->sa.rmap_cur,
-				sc->sa.rmap_cur->bc_nlevels - 1, &error);
-	else
-		cr->bitmaps_complete = true;
+	if (error) {
+		if (!xchk_btree_xref_process_error(sc, sc->sa.rmap_cur,
+				sc->sa.rmap_cur->bc_nlevels - 1, &error)) {
+			/* only set incomplete if we didn't set xfail */
+			if (error)
+				xchk_set_incomplete(sc);
+		}
+
+		return 0;
+	}
+
+	cr->bitmaps_complete = true;
 	return 0;
 }
 
@@ -567,7 +574,8 @@ xchk_rmapbt(
 	if (error)
 		goto out;
 
-	xchk_rmapbt_check_bitmaps(sc, cr);
+	if (cr->bitmaps_complete)
+		xchk_rmapbt_check_bitmaps(sc, cr);
 
 out:
 	xagb_bitmap_destroy(&cr->refcbt_owned);
diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c
index 590f9f41856ef7..725035bf490329 100644
--- a/fs/xfs/scrub/rmap_repair.c
+++ b/fs/xfs/scrub/rmap_repair.c
@@ -1109,6 +1109,7 @@ xrep_rmap_try_reserve(
 		return error;
 
 	error = xfs_agfl_walk(sc->mp, agf, agfl_bp, xrep_rmap_walk_agfl, &ra);
+	xfs_trans_brelse(sc->tp, agfl_bp);
 	if (error)
 		return error;
 
diff --git a/fs/xfs/scrub/rtrefcount_repair.c b/fs/xfs/scrub/rtrefcount_repair.c
index 2b939960c7ddcd..c78a6d2990c586 100644
--- a/fs/xfs/scrub/rtrefcount_repair.c
+++ b/fs/xfs/scrub/rtrefcount_repair.c
@@ -596,8 +596,7 @@ xrep_rtrefc_iroot_size(
 	unsigned int		nr_this_level,
 	void			*priv)
 {
-	return xfs_rtrefcount_broot_space_calc(cur->bc_mp, level,
-			nr_this_level);
+	return xfs_rtrefcount_broot_space_calc(level, nr_this_level);
 }
 
 /*
diff --git a/fs/xfs/scrub/rtrmap_repair.c b/fs/xfs/scrub/rtrmap_repair.c
index a2b72e61edf5fe..5cfa4470c57c0f 100644
--- a/fs/xfs/scrub/rtrmap_repair.c
+++ b/fs/xfs/scrub/rtrmap_repair.c
@@ -693,7 +693,7 @@ xrep_rtrmap_iroot_size(
 	unsigned int		nr_this_level,
 	void			*priv)
 {
-	return xfs_rtrmap_broot_space_calc(cur->bc_mp, level, nr_this_level);
+	return xfs_rtrmap_broot_space_calc(level, nr_this_level);
 }
 
 /*
diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h
index 737a5d6db15ff3..b093945f363148 100644
--- a/fs/xfs/scrub/scrub.h
+++ b/fs/xfs/scrub/scrub.h
@@ -40,7 +40,7 @@ static inline int xchk_maybe_relax(struct xchk_relax *widget)
 		return 0;
 	widget->resched_nr = 0;
 
-	if (unlikely(widget->next_resched <= jiffies)) {
+	if (unlikely(time_after_eq(jiffies, widget->next_resched))) {
 		cond_resched();
 		widget->next_resched = XCHK_RELAX_NEXT;
 	}
diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h
index 0f5adc293962fa..cb85f75ce10180 100644
--- a/fs/xfs/scrub/trace.h
+++ b/fs/xfs/scrub/trace.h
@@ -2376,25 +2376,29 @@ TRACE_EVENT(xrep_calc_ag_resblks_btsize,
 #ifdef CONFIG_XFS_RT
 TRACE_EVENT(xrep_calc_rtgroup_resblks_btsize,
 	TP_PROTO(struct xfs_mount *mp, xfs_rgnumber_t rgno,
-		 xfs_rgblock_t usedlen, xfs_rgblock_t rmapbt_sz),
-	TP_ARGS(mp, rgno, usedlen, rmapbt_sz),
+		 xfs_rgblock_t usedlen, xfs_rgblock_t rmapbt_sz,
+		 xfs_rgblock_t refcbt_sz),
+	TP_ARGS(mp, rgno, usedlen, rmapbt_sz, refcbt_sz),
 	TP_STRUCT__entry(
 		__field(dev_t, dev)
 		__field(xfs_rgnumber_t, rgno)
 		__field(xfs_rgblock_t, usedlen)
 		__field(xfs_rgblock_t, rmapbt_sz)
+		__field(xfs_rgblock_t, refcbt_sz)
 	),
 	TP_fast_assign(
 		__entry->dev = mp->m_super->s_dev;
 		__entry->rgno = rgno;
 		__entry->usedlen = usedlen;
 		__entry->rmapbt_sz = rmapbt_sz;
+		__entry->refcbt_sz = refcbt_sz;
 	),
-	TP_printk("dev %d:%d rgno 0x%x usedlen %u rmapbt %u",
+	TP_printk("dev %d:%d rgno 0x%x usedlen %u rmapbt %u refcountbt %u",
 		  MAJOR(__entry->dev), MINOR(__entry->dev),
 		  __entry->rgno,
 		  __entry->usedlen,
-		  __entry->rmapbt_sz)
+		  __entry->rmapbt_sz,
+		  __entry->refcbt_sz)
 );
 #endif /* CONFIG_XFS_RT */
 
diff --git a/fs/xfs/scrub/xfarray.c b/fs/xfs/scrub/xfarray.c
index 2ce24bfe4c0fab..c94f355607790e 100644
--- a/fs/xfs/scrub/xfarray.c
+++ b/fs/xfs/scrub/xfarray.c
@@ -140,55 +140,23 @@ xfarray_load(
 			xfarray_pos(array, idx));
 }
 
-/* Is this array element potentially unset? */
-static inline bool
-xfarray_is_unset(
-	struct xfarray	*array,
-	loff_t		pos)
-{
-	void		*temp = xfarray_scratch(array);
-	int		error;
-
-	if (array->unset_slots == 0)
-		return false;
-
-	error = xfile_load(array->xfile, temp, array->obj_size, pos);
-	if (!error && xfarray_element_is_null(array, temp))
-		return true;
-
-	return false;
-}
-
-/*
- * Unset an array element.  If @idx is the last element in the array, the
- * array will be truncated.  Otherwise, the entry will be zeroed.
- */
+/* Remove the elements at the end of an array. */
 int
-xfarray_unset(
-	struct xfarray	*array,
-	xfarray_idx_t	idx)
+xfarray_trim(
+	struct xfarray		*array,
+	unsigned long long	nr)
 {
-	void		*temp = xfarray_scratch(array);
-	loff_t		pos = xfarray_pos(array, idx);
-	int		error;
+	loff_t			new_eof;
 
-	if (idx >= array->nr)
+	if (nr > array->nr)
 		return -ENODATA;
 
-	if (idx == array->nr - 1) {
-		array->nr--;
-		return 0;
-	}
-
-	if (xfarray_is_unset(array, pos))
-		return 0;
-
-	memset(temp, 0, array->obj_size);
-	error = xfile_store(array->xfile, temp, array->obj_size, pos);
-	if (error)
-		return error;
+	array->nr -= nr;
+	if (!array->nr)
+		array->possibly_sparse = false;
 
-	array->unset_slots++;
+	new_eof = xfarray_pos(array, array->nr);
+	xfile_discard(array->xfile, new_eof, MAX_LFS_FILESIZE - new_eof);
 	return 0;
 }
 
@@ -214,6 +182,8 @@ xfarray_store(
 	if (ret)
 		return ret;
 
+	if (idx > array->nr)
+		array->possibly_sparse = true;
 	array->nr = max(array->nr, idx + 1);
 	return 0;
 }
@@ -227,43 +197,6 @@ xfarray_element_is_null(
 	return !memchr_inv(ptr, 0, array->obj_size);
 }
 
-/*
- * Store an element anywhere in the array that is unset.  If there are no
- * unset slots, append the element to the array.
- */
-int
-xfarray_store_anywhere(
-	struct xfarray	*array,
-	const void	*ptr)
-{
-	void		*temp = xfarray_scratch(array);
-	loff_t		endpos = xfarray_pos(array, array->nr);
-	loff_t		pos;
-	int		error;
-
-	/* Find an unset slot to put it in. */
-	for (pos = 0;
-	     pos < endpos && array->unset_slots > 0;
-	     pos += array->obj_size) {
-		error = xfile_load(array->xfile, temp, array->obj_size,
-				pos);
-		if (error || !xfarray_element_is_null(array, temp))
-			continue;
-
-		error = xfile_store(array->xfile, ptr, array->obj_size,
-				pos);
-		if (error)
-			return error;
-
-		array->unset_slots--;
-		return 0;
-	}
-
-	/* No unset slots found; attach it on the end. */
-	array->unset_slots = 0;
-	return xfarray_append(array, ptr);
-}
-
 /* Return length of array. */
 uint64_t
 xfarray_length(
@@ -677,26 +610,10 @@ xfarray_qsort_pivot(
 
 	/* Load the selected xfarray records into the pivot array. */
 	for (i = 0; i < XFARRAY_QSORT_PIVOT_NR; i++) {
-		xfarray_idx_t	idx;
-
 		recp = xfarray_pivot_array_rec(parray, pivot_rec_sz, i);
 		idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, i);
 
-		/* No unset records; load directly into the array. */
-		if (likely(si->array->unset_slots == 0)) {
-			error = xfarray_sort_load(si, *idxp, recp);
-			if (error)
-				return error;
-			continue;
-		}
-
-		/*
-		 * Load non-null records into the scratchpad without changing
-		 * the xfarray_idx_t in the pivot array.
-		 */
-		idx = *idxp;
-		xfarray_sort_bump_loads(si);
-		error = xfarray_load_next(si->array, &idx, recp);
+		error = xfarray_sort_load(si, *idxp, recp);
 		if (error)
 			return error;
 	}
@@ -765,6 +682,18 @@ xfarray_qsort_push(
 		return -EFSCORRUPTED;
 	}
 
+	/*
+	 * Avoid the integer underflow below in (lo - 1).  This shouldn't
+	 * be possible because the pivot is the median of nine distinct
+	 * filesystem metadata records, so at least four records will be less
+	 * than the pivot, which means the pivot will not be in the low end of
+	 * the range by the time we get here.
+	 */
+	if (lo == 0) {
+		ASSERT(lo != 0);
+		return -EFSCORRUPTED;
+	}
+
 	si->max_stack_used = max_t(uint8_t, si->max_stack_used,
 					    si->stack_depth + 2);
 
@@ -794,6 +723,46 @@ xfarray_sort_scan_done(
 	si->folio = NULL;
 }
 
+static int
+xfarray_sort_load_folio(
+	struct xfarray_sortinfo	*si,
+	xfarray_idx_t		idx,
+	loff_t			idx_pos)
+{
+	struct folio		*folio;
+	loff_t			next_pos;
+
+	folio = xfile_get_folio(si->array->xfile, idx_pos, si->array->obj_size,
+			XFILE_ALLOC);
+	if (IS_ERR(folio))
+		return PTR_ERR(folio);
+	si->folio = folio;
+
+	/* No folio?  Get the caller to read into the scratchpad. */
+	if (!si->folio)
+		return 0;
+
+	si->first_folio_idx = xfarray_idx(si->array,
+			folio_pos(si->folio) + si->array->obj_size - 1);
+
+	next_pos = folio_next_pos(si->folio);
+	si->last_folio_idx = xfarray_idx(si->array, next_pos - 1);
+	if (xfarray_pos(si->array, si->last_folio_idx + 1) > next_pos)
+		si->last_folio_idx--;
+
+	/*
+	 * If this folio still doesn't cover the desired element, it must cross
+	 * a folio boundary.  Get the caller to read into the scratchpad.
+	 */
+	if (idx < si->first_folio_idx || idx > si->last_folio_idx) {
+		xfarray_sort_scan_done(si);
+		return 0;
+	}
+
+	trace_xfarray_sort_scan(si, idx);
+	return 0;
+}
+
 /*
  * Cache the folio backing the start of the given array element.  If the array
  * element is contained entirely within the folio, return a pointer to the
@@ -819,33 +788,18 @@ xfarray_sort_scan(
 	    (idx < si->first_folio_idx || idx > si->last_folio_idx))
 		xfarray_sort_scan_done(si);
 
-	/* Grab the first folio that backs this array element. */
+	/* Grab the folio that backs this array element. */
 	if (!si->folio) {
-		struct folio	*folio;
-		loff_t		next_pos;
-
-		folio = xfile_get_folio(si->array->xfile, idx_pos,
-				si->array->obj_size, XFILE_ALLOC);
-		if (IS_ERR(folio))
-			return PTR_ERR(folio);
-		si->folio = folio;
-
-		si->first_folio_idx = xfarray_idx(si->array,
-				folio_pos(si->folio) + si->array->obj_size - 1);
-
-		next_pos = folio_next_pos(si->folio);
-		si->last_folio_idx = xfarray_idx(si->array, next_pos - 1);
-		if (xfarray_pos(si->array, si->last_folio_idx + 1) > next_pos)
-			si->last_folio_idx--;
-
-		trace_xfarray_sort_scan(si, idx);
+		error = xfarray_sort_load_folio(si, idx, idx_pos);
+		if (error)
+			return error;
 	}
 
 	/*
-	 * If this folio still doesn't cover the desired element, it must cross
-	 * a folio boundary.  Read into the scratchpad and we're done.
+	 * If we don't have a folio mapping the entire array element, read into
+	 * the scratchpad and we're done.
 	 */
-	if (idx < si->first_folio_idx || idx > si->last_folio_idx) {
+	if (!si->folio) {
 		void		*temp = xfarray_scratch(si->array);
 
 		error = xfile_load(si->array->xfile, temp, si->array->obj_size,
@@ -916,6 +870,14 @@ xfarray_sort(
 		return 0;
 	if (array->nr >= QSORT_MAX_RECS)
 		return -E2BIG;
+	if (array->possibly_sparse) {
+		/*
+		 * What does it mean to sort an array with holes in it?
+		 * Currently none of the users need this ability.
+		 */
+		ASSERT(!array->possibly_sparse);
+		return -EINVAL;
+	}
 
 	error = xfarray_sortinfo_alloc(array, cmp_fn, flags, &si);
 	if (error)
@@ -1069,4 +1031,5 @@ xfarray_truncate(
 {
 	xfile_discard(array->xfile, 0, MAX_LFS_FILESIZE);
 	array->nr = 0;
+	array->possibly_sparse = false;
 }
diff --git a/fs/xfs/scrub/xfarray.h b/fs/xfs/scrub/xfarray.h
index 5eeeeed13ae24a..05ff65b09fcf41 100644
--- a/fs/xfs/scrub/xfarray.h
+++ b/fs/xfs/scrub/xfarray.h
@@ -27,23 +27,22 @@ struct xfarray {
 	/* Maximum possible array size. */
 	xfarray_idx_t	max_nr;
 
-	/* Number of unset slots in the array below @nr. */
-	uint64_t	unset_slots;
-
 	/* Size of an array element. */
 	size_t		obj_size;
 
 	/* log2 of array element size, if possible. */
 	int		obj_size_log;
+
+	/* Might there be sparse holes in this array? */
+	bool		possibly_sparse;
 };
 
 int xfarray_create(const char *descr, unsigned long long required_capacity,
 		size_t obj_size, struct xfarray **arrayp);
 void xfarray_destroy(struct xfarray *array);
 int xfarray_load(struct xfarray *array, xfarray_idx_t idx, void *ptr);
-int xfarray_unset(struct xfarray *array, xfarray_idx_t idx);
+int xfarray_trim(struct xfarray *array, unsigned long long nr);
 int xfarray_store(struct xfarray *array, xfarray_idx_t idx, const void *ptr);
-int xfarray_store_anywhere(struct xfarray *array, const void *ptr);
 bool xfarray_element_is_null(struct xfarray *array, const void *ptr);
 void xfarray_truncate(struct xfarray *array);
 unsigned long long xfarray_bytes(struct xfarray *array);
diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c
index fdfca6fc75b6ea..20d87b52c4fc70 100644
--- a/fs/xfs/xfs_acl.c
+++ b/fs/xfs/xfs_acl.c
@@ -243,7 +243,7 @@ xfs_acl_set_mode(
 }
 
 int
-xfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+xfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 	    struct posix_acl *acl, int type)
 {
 	umode_t mode;
diff --git a/fs/xfs/xfs_acl.h b/fs/xfs/xfs_acl.h
index bf7f960997d36a..183526bec32c17 100644
--- a/fs/xfs/xfs_acl.h
+++ b/fs/xfs/xfs_acl.h
@@ -11,7 +11,7 @@ struct posix_acl;
 
 #ifdef CONFIG_XFS_POSIX_ACL
 extern struct posix_acl *xfs_get_acl(struct inode *inode, int type, bool rcu);
-extern int xfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+extern int xfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		       struct posix_acl *acl, int type);
 extern int __xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type);
 void xfs_forget_acl(struct inode *inode, const char *name);
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 8b6119776fb3ec..c30e688cfc9f6e 100644
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -23,7 +23,6 @@
 #include "xfs_ioend.h"
 #include "xfs_zone_alloc.h"
 #include "xfs_rtgroup.h"
-#include <linux/bio-integrity.h>
 
 struct xfs_writepage_ctx {
 	struct iomap_writepage_ctx ctx;
@@ -498,8 +497,7 @@ xfs_zoned_writeback_submit(
 		bio_endio(&ioend->io_bio);
 		return error;
 	}
-	if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
-		fs_bio_integrity_generate(&ioend->io_bio);
+
 	xfs_zone_alloc_and_submit(ioend, &XFS_ZWPC(wpc)->open_zone);
 	return 0;
 }
@@ -585,11 +583,10 @@ xfs_bio_submit_read(
 	const struct iomap_iter		*iter,
 	struct iomap_read_folio_ctx	*ctx)
 {
-	struct bio			*bio = ctx->read_ctx;
-
-	/* defer read completions to the ioend workqueue */
-	iomap_init_ioend(iter->inode, bio, ctx->read_ctx_file_offset, 0);
-	iomap_bio_submit_read_endio(iter, ctx, xfs_end_bio);
+	xfs_ioend_submit_read(iter->inode, ctx->read_ctx,
+			ctx->read_ctx_file_offset,
+			iomap_ioend_flags(&iter->iomap));
+	ctx->read_ctx = NULL;
 }
 
 static const struct iomap_read_ops xfs_iomap_read_ops = {
diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c
index 89f6e79a955f54..62be62342fea17 100644
--- a/fs/xfs/xfs_bmap_item.c
+++ b/fs/xfs/xfs_bmap_item.c
@@ -339,7 +339,7 @@ xfs_bmap_update_get_group(
 
 	/*
 	 * Bump the intent count on behalf of the deferred rmap and refcount
-	 * intent items that that we can queue when we finish this bmap work.
+	 * intent items that we can queue when we finish this bmap work.
 	 * This new intent item will bump the intent count before the bmap
 	 * intent drops the intent count, ensuring that the intent count
 	 * remains nonzero across the transaction roll.
@@ -442,7 +442,7 @@ xfs_bui_validate(
 	if (!xfs_verify_ino(mp, map->me_owner))
 		return false;
 
-	if (!xfs_verify_fileext(mp, map->me_startoff, map->me_len))
+	if (!xfs_verify_fileext(map->me_startoff, map->me_len))
 		return false;
 
 	if (map->me_flags & XFS_BMAP_EXTENT_REALTIME)
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index 8256c1d13ce20f..6c93b4f5629c1f 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -5,6 +5,7 @@
  */
 #include "xfs_platform.h"
 #include <linux/backing-dev.h>
+#include <linux/blk-integrity.h>
 #include <linux/dax.h>
 
 #include "xfs_shared.h"
@@ -1694,6 +1695,7 @@ xfs_configure_buftarg(
 	struct xfs_mount	*mp = btp->bt_mount;
 
 	if (btp->bt_bdev) {
+		struct blk_integrity *bi = bdev_get_integrity(btp->bt_bdev);
 		int		error;
 
 		error = bdev_validate_blocksize(btp->bt_bdev, sectorsize);
@@ -1706,6 +1708,15 @@ xfs_configure_buftarg(
 
 		if (bdev_can_atomic_write(btp->bt_bdev))
 			xfs_configure_buftarg_atomic_writes(btp);
+
+		if (!bi)
+			;
+		else if (btp->bt_bdev == btp->bt_mount->m_super->s_bdev)
+			xfs_info(mp, "using %s integrity profile",
+				blk_integrity_profile_name(bi));
+		else
+			xfs_info(mp, "using %s integrity profile for %pg",
+				blk_integrity_profile_name(bi), btp->bt_bdev);
 	}
 
 	btp->bt_meta_sectorsize = sectorsize;
diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c
index b4f6c594808ce9..6d22ead562da58 100644
--- a/fs/xfs/xfs_dquot.c
+++ b/fs/xfs/xfs_dquot.c
@@ -115,18 +115,15 @@ xfs_qm_dqdestroy(
  * We overwrite the dquot limits only if they are zero and this
  * is not the root dquot.
  */
-void
+static void
 xfs_qm_adjust_dqlimits(
 	struct xfs_dquot	*dq)
 {
 	struct xfs_mount	*mp = dq->q_mount;
 	struct xfs_quotainfo	*q = mp->m_quotainfo;
-	struct xfs_def_quota	*defq;
+	struct xfs_def_quota	*defq = xfs_get_defquota(q, xfs_dquot_type(dq));
 	int			prealloc = 0;
 
-	ASSERT(dq->q_id);
-	defq = xfs_get_defquota(q, xfs_dquot_type(dq));
-
 	if (!dq->q_blk.softlimit) {
 		dq->q_blk.softlimit = defq->blk.soft;
 		prealloc = 1;
@@ -139,10 +136,14 @@ xfs_qm_adjust_dqlimits(
 		dq->q_ino.softlimit = defq->ino.soft;
 	if (!dq->q_ino.hardlimit)
 		dq->q_ino.hardlimit = defq->ino.hard;
-	if (!dq->q_rtb.softlimit)
+	if (!dq->q_rtb.softlimit) {
 		dq->q_rtb.softlimit = defq->rtb.soft;
-	if (!dq->q_rtb.hardlimit)
+		prealloc = 1;
+	}
+	if (!dq->q_rtb.hardlimit) {
 		dq->q_rtb.hardlimit = defq->rtb.hard;
+		prealloc = 1;
+	}
 
 	if (prealloc)
 		xfs_dquot_set_prealloc_limits(dq);
@@ -219,6 +220,19 @@ xfs_qm_adjust_dqtimers(
 	xfs_qm_adjust_res_timer(dq->q_mount, &dq->q_rtb, &defq->rtb);
 }
 
+/* Adjust enforcement limits and timers after a change in usage. */
+void
+xfs_qm_adjust_dqenforcement(
+	struct xfs_dquot	*dq)
+{
+	if (dq->q_id == 0)
+		return;
+
+	xfs_qm_adjust_dqlimits(dq);
+	xfs_qm_adjust_dqtimers(dq);
+	dq->q_flags |= XFS_DQFLAG_DIRTY;
+}
+
 /*
  * initialize a buffer full of dquots and log the whole thing
  */
diff --git a/fs/xfs/xfs_dquot.h b/fs/xfs/xfs_dquot.h
index bbb824adca82ce..28c09704acc261 100644
--- a/fs/xfs/xfs_dquot.h
+++ b/fs/xfs/xfs_dquot.h
@@ -205,7 +205,7 @@ void		xfs_qm_dqdestroy(struct xfs_dquot *dqp);
 int		xfs_qm_dqflush(struct xfs_dquot *dqp, struct xfs_buf *bp);
 void		xfs_qm_dqunpin_wait(struct xfs_dquot *dqp);
 void		xfs_qm_adjust_dqtimers(struct xfs_dquot *d);
-void		xfs_qm_adjust_dqlimits(struct xfs_dquot *d);
+void		xfs_qm_adjust_dqenforcement(struct xfs_dquot *d);
 xfs_dqid_t	xfs_qm_id_for_quotatype(struct xfs_inode *ip,
 				xfs_dqtype_t type);
 int		xfs_qm_dqget(struct xfs_mount *mp, xfs_dqid_t id,
diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c
index dd5d92ca1010fe..dd104bf778baf8 100644
--- a/fs/xfs/xfs_exchmaps_item.c
+++ b/fs/xfs/xfs_exchmaps_item.c
@@ -341,10 +341,10 @@ xfs_xmi_validate(
 	    !xfs_verify_ino(mp, xlf->xmi_inode2))
 		return false;
 
-	if (!xfs_verify_fileext(mp, xlf->xmi_startoff1, xlf->xmi_blockcount))
+	if (!xfs_verify_fileext(xlf->xmi_startoff1, xlf->xmi_blockcount))
 		return false;
 
-	if (!xfs_verify_fileext(mp, xlf->xmi_startoff2, xlf->xmi_blockcount))
+	if (!xfs_verify_fileext(xlf->xmi_startoff2, xlf->xmi_blockcount))
 		return false;
 
 	if (xlf->xmi_flags & XFS_EXCHMAPS_SET_SIZES) {
diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c
index c69ecd6a19de4f..a1001315d4a7bf 100644
--- a/fs/xfs/xfs_exchrange.c
+++ b/fs/xfs/xfs_exchrange.c
@@ -238,7 +238,7 @@ xfs_exchrange_mappings(
 	trace_xfs_exchrange_before(ip2, 2);
 	trace_xfs_exchrange_before(ip1, 1);
 
-	error = xfs_exchmaps_check_forks(mp, &req);
+	error = xfs_exchmaps_check_forks(&req);
 	if (error)
 		goto out_trans_cancel;
 
@@ -633,6 +633,9 @@ xfs_exchrange_prep(
 	if (error)
 		return error;
 
+	if (fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN)
+		return 0;
+
 	trace_xfs_exchrange_flush(fxr, ip1, ip2);
 
 	/* Flush the relevant ranges of both files. */
@@ -709,9 +712,11 @@ xfs_exchrange_contents(
 	 * other file write would do.  This may involve turning on support for
 	 * logged xattrs if either file has security capabilities.
 	 */
-	error = xfs_exchange_range_finish(fxr);
-	if (error)
-		goto out_unlock;
+	if (!(fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN)) {
+		error = xfs_exchange_range_finish(fxr);
+		if (error)
+			goto out_unlock;
+	}
 
 out_unlock:
 	xfs_iunlock2_io_mmap(ip1, ip2);
@@ -902,7 +907,7 @@ xfs_ioc_commit_range(
 
 	if (copy_from_user(&args, argp, sizeof(args)))
 		return -EFAULT;
-	if (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS)
+	if (args.pad || (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS))
 		return -EINVAL;
 	if (kern_f->magic != XCR_FRESH_MAGIC)
 		return -EBUSY;
diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index d8202da15aca91..d164de6ff98b15 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -37,6 +37,7 @@
 #include <linux/fadvise.h>
 #include <linux/mount.h>
 #include <linux/filelock.h>
+#include <linux/bio-integrity.h>
 
 static const struct vm_operations_struct xfs_file_vm_ops;
 
@@ -222,9 +223,8 @@ xfs_dio_read_bounce_submit_io(
 	struct bio		*bio,
 	loff_t			file_offset)
 {
-	iomap_init_ioend(iter->inode, bio, file_offset, IOMAP_IOEND_DIRECT);
-	bio->bi_end_io = xfs_end_bio;
-	submit_bio(bio);
+	xfs_ioend_submit_read(iter->inode, bio, file_offset,
+			iomap_ioend_flags(&iter->iomap) | IOMAP_IOEND_DIRECT);
 }
 
 static const struct iomap_dio_ops xfs_dio_read_bounce_ops = {
@@ -252,8 +252,7 @@ xfs_file_dio_read(
 		return ret;
 	if (mapping_stable_writes(iocb->ki_filp->f_mapping)) {
 		ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops,
-				&xfs_dio_read_bounce_ops, IOMAP_DIO_BOUNCE,
-				NULL, 0);
+				&xfs_dio_read_bounce_ops, 0, NULL, 0);
 	} else {
 		ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin);
 		if (ret == -ENOTBLK)
@@ -713,7 +712,7 @@ xfs_dio_zoned_submit_io(
 
 	bio->bi_end_io = xfs_end_bio;
 	ioend = iomap_init_ioend(iter->inode, bio, file_offset,
-			IOMAP_IOEND_DIRECT);
+			iomap_ioend_flags(&iter->iomap) | IOMAP_IOEND_DIRECT);
 	xfs_zone_alloc_and_submit(ioend, &ac->open_zone);
 }
 
@@ -1872,17 +1871,21 @@ xfs_file_release(
 		return 0;
 
 	/*
-	 * If we can't get the iolock just skip truncating the blocks past EOF
-	 * because we could deadlock with the mmap_lock otherwise. We'll get
-	 * another chance to drop them once the last reference to the inode is
-	 * dropped, so we'll never leak blocks permanently.
+	 * If we can't get the iolock or if the filesystem is frozen, just skip
+	 * truncating the blocks past EOF because we could deadlock with the
+	 * mmap_lock or hang the close() call. We'll get another chance to drop
+	 * them once the last reference to the inode is dropped, so we'll never
+	 * leak blocks permanently.
 	 */
 	if (!xfs_iflags_test(ip, XFS_EOFBLOCKS_RELEASED) &&
-	    xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
-		if (xfs_can_free_eofblocks(ip) &&
-		    !xfs_iflags_test_and_set(ip, XFS_EOFBLOCKS_RELEASED))
-			xfs_free_eofblocks(ip);
-		xfs_iunlock(ip, XFS_IOLOCK_EXCL);
+	    sb_start_write_trylock(mp->m_super)) {
+		if (xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
+			if (xfs_can_free_eofblocks(ip) &&
+			    !xfs_iflags_test_and_set(ip, XFS_EOFBLOCKS_RELEASED))
+				xfs_free_eofblocks(ip);
+			xfs_iunlock(ip, XFS_IOLOCK_EXCL);
+		}
+		sb_end_write(mp->m_super);
 	}
 
 	return 0;
diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c
index 0689cade8f74c2..4924e676ae91d8 100644
--- a/fs/xfs/xfs_handle.c
+++ b/fs/xfs/xfs_handle.c
@@ -272,11 +272,11 @@ xfs_open_by_handle(
 	path.mnt = mntget(parfilp->f_path.mnt);
 
 	FD_PREPARE(fdf, 0, dentry_open(&path, hreq->oflags, cred));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
 	if (S_ISREG(inode->i_mode)) {
-		struct file *filp = fd_prepare_file(fdf);
+		struct file *filp = fdf->file;
 
 		filp->f_flags |= O_NOATIME;
 		filp->f_mode |= FMODE_NOCMTIME;
@@ -409,7 +409,7 @@ xfs_ioc_attr_list(
 	void				*buffer;
 	int				error;
 
-	if (bufsize < sizeof(struct xfs_attrlist) ||
+	if (bufsize < struct_size(alist, al_offset, 1) ||
 	    bufsize > XFS_XATTR_LIST_MAX)
 		return -EINVAL;
 
diff --git a/fs/xfs/xfs_healthmon.c b/fs/xfs/xfs_healthmon.c
index c3749675ef1921..2bdd747f1ead24 100644
--- a/fs/xfs/xfs_healthmon.c
+++ b/fs/xfs/xfs_healthmon.c
@@ -247,7 +247,9 @@ xfs_healthmon_merge_events(
 	case XFS_HEALTHMON_DIOWRITE:
 	case XFS_HEALTHMON_DATALOST:
 		/* logically adjacent file ranges can merge */
-		if (existing->fino != new->fino || existing->fgen != new->fgen)
+		if (existing->fino != new->fino ||
+		    existing->fgen != new->fgen ||
+		    existing->error != new->error)
 			return false;
 
 		if (existing->fpos + existing->flen == new->fpos) {
diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c
index 82dac88e3c4c71..de8be344e987d0 100644
--- a/fs/xfs/xfs_icache.c
+++ b/fs/xfs/xfs_icache.c
@@ -1653,7 +1653,7 @@ xfs_blockgc_free_dquots(
 		do_work = true;
 	}
 
-	if (XFS_IS_UQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) {
+	if (XFS_IS_GQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) {
 		icw.icw_gid = make_kgid(mp->m_super->s_user_ns, gdqp->q_id);
 		icw.icw_flags |= XFS_ICWALK_FLAG_GID;
 		do_work = true;
diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
index 030a7c8f2c129a..05a14da280314b 100644
--- a/fs/xfs/xfs_inode.c
+++ b/fs/xfs/xfs_inode.c
@@ -755,7 +755,7 @@ xfs_create(
 	*ipp = du.ip;
 	xfs_iunlock(du.ip, XFS_ILOCK_EXCL);
 	xfs_iunlock(dp, XFS_ILOCK_EXCL);
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
 	return 0;
 
  out_trans_cancel:
@@ -772,7 +772,7 @@ xfs_create(
 		xfs_irele(du.ip);
 	}
  out_parent:
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
  out_release_dquots:
 	xfs_qm_dqrele(udqp);
 	xfs_qm_dqrele(gdqp);
@@ -972,7 +972,7 @@ xfs_link(
 	error = xfs_trans_commit(tp);
 	xfs_iunlock(tdp, XFS_ILOCK_EXCL);
 	xfs_iunlock(sip, XFS_ILOCK_EXCL);
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
 	return error;
 
  error_return:
@@ -980,7 +980,7 @@ xfs_link(
 	xfs_iunlock(tdp, XFS_ILOCK_EXCL);
 	xfs_iunlock(sip, XFS_ILOCK_EXCL);
  out_parent:
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
  std_return:
 	if (error == -ENOSPC && nospace_error)
 		error = nospace_error;
@@ -1064,7 +1064,7 @@ xfs_itruncate_extents_flags(
 	 * the page cache can't scale that far.
 	 */
 	first_unmap_block = XFS_B_TO_FSB(mp, (xfs_ufsize_t)new_size);
-	if (!xfs_verify_fileoff(mp, first_unmap_block)) {
+	if (!xfs_verify_fileoff(first_unmap_block)) {
 		WARN_ON_ONCE(first_unmap_block > XFS_MAX_FILEOFF);
 		return 0;
 	}
@@ -1985,7 +1985,7 @@ xfs_remove(
 
 	xfs_iunlock(ip, XFS_ILOCK_EXCL);
 	xfs_iunlock(dp, XFS_ILOCK_EXCL);
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
 	return 0;
 
  out_trans_cancel:
@@ -1994,7 +1994,7 @@ xfs_remove(
 	xfs_iunlock(ip, XFS_ILOCK_EXCL);
 	xfs_iunlock(dp, XFS_ILOCK_EXCL);
  out_parent:
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
  std_return:
 	return error;
 }
@@ -2084,7 +2084,7 @@ xfs_sort_inodes(
  */
 static int
 xfs_rename_alloc_whiteout(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct xfs_name		*src_name,
 	struct xfs_inode	*dp,
 	struct xfs_inode	**wip)
@@ -2130,7 +2130,7 @@ xfs_rename_alloc_whiteout(
  */
 int
 xfs_rename(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct xfs_inode	*src_dp,
 	struct xfs_name		*src_name,
 	struct xfs_inode	*src_ip,
@@ -2357,11 +2357,11 @@ xfs_rename(
 out_unlock:
 	xfs_iunlock_rename(inodes, num_inodes);
 out_tgt_ppargs:
-	xfs_parent_finish(mp, du_tgt.ppargs);
+	xfs_parent_finish(du_tgt.ppargs);
 out_wip_ppargs:
-	xfs_parent_finish(mp, du_wip.ppargs);
+	xfs_parent_finish(du_wip.ppargs);
 out_src_ppargs:
-	xfs_parent_finish(mp, du_src.ppargs);
+	xfs_parent_finish(du_src.ppargs);
 out_release_wip:
 	if (du_wip.ip)
 		xfs_irele(du_wip.ip);
@@ -2669,7 +2669,7 @@ xfs_irele(
 }
 
 /*
- * Ensure all commited transactions touching the inode are written to the log.
+ * Ensure all committed transactions touching the inode are written to the log.
  */
 int
 xfs_log_force_inode(
diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h
index 1602027cd0aa5d..ca96ba096359aa 100644
--- a/fs/xfs/xfs_inode.h
+++ b/fs/xfs/xfs_inode.h
@@ -568,7 +568,7 @@ int		xfs_remove(struct xfs_inode *dp, struct xfs_name *name,
 			   struct xfs_inode *ip);
 int		xfs_link(struct xfs_inode *tdp, struct xfs_inode *sip,
 			 struct xfs_name *target_name);
-int		xfs_rename(struct mnt_idmap *idmap,
+int		xfs_rename(const struct mnt_idmap *idmap,
 			   struct xfs_inode *src_dp, struct xfs_name *src_name,
 			   struct xfs_inode *src_ip, struct xfs_inode *target_dp,
 			   struct xfs_name *target_name,
diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
index 96ca3e480cb9fe..f81b6e52ac403d 100644
--- a/fs/xfs/xfs_ioctl.c
+++ b/fs/xfs/xfs_ioctl.c
@@ -748,7 +748,7 @@ xfs_ioctl_setattr_check_projid(
 
 int
 xfs_fileattr_set(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct dentry		*dentry,
 	struct file_kattr	*fa)
 {
@@ -962,10 +962,10 @@ xfs_ioc_getbmap(
 }
 
 int
-xfs_ioc_swapext(
-	xfs_swapext_t	*sxp)
+xfs_swapext(
+	struct xfs_swapext	*sxp)
 {
-	xfs_inode_t     *ip, *tip;
+	struct xfs_inode	*ip, *tip;
 
 	/* Pull information for the target fd */
 	CLASS(fd, f)((int)sxp->sx_fdtarget);
@@ -1035,6 +1035,46 @@ xfs_ioc_getlabel(
 	return 0;
 }
 
+/*
+ * Same behavior as xfs_sync_sb, except that it is always synchronous and it
+ * also writes the superblock buffer to disk sector 0 immediately.
+ */
+static int
+xfs_sync_sb_buf(
+	struct xfs_mount	*mp,
+	bool			update_rtsb)
+{
+	struct xfs_trans	*tp;
+	int			error;
+
+	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_sb, 0, 0, 0, &tp);
+	if (error)
+		return error;
+
+	xfs_log_sb(tp);
+	if (update_rtsb)
+		xfs_log_rtsb(tp, xfs_trans_getsb(tp));
+	xfs_trans_set_sync(tp);
+	error = xfs_trans_commit(tp);
+	if (error)
+		return error;
+
+	/* Re-acquire and write the sb and rtsb to disk. */
+	xfs_buf_lock(mp->m_sb_bp);
+	error = xfs_bwrite(mp->m_sb_bp);
+	xfs_buf_unlock(mp->m_sb_bp);
+	if (error)
+		return error;
+
+	if (update_rtsb && mp->m_rtsb_bp) {
+		xfs_buf_lock(mp->m_rtsb_bp);
+		error = xfs_bwrite(mp->m_rtsb_bp);
+		xfs_buf_unlock(mp->m_rtsb_bp);
+	}
+
+	return error;
+}
+
 static int
 xfs_ioc_setlabel(
 	struct file		*filp,
@@ -1144,7 +1184,7 @@ xfs_fs_eofblocks_from_user(
 }
 
 static int
-xfs_ioctl_getset_resblocks(
+xfs_ioc_getset_resblocks(
 	struct file		*filp,
 	unsigned int		cmd,
 	void __user		*arg)
@@ -1183,7 +1223,7 @@ xfs_ioctl_getset_resblocks(
 }
 
 static int
-xfs_ioctl_fs_counts(
+xfs_ioc_fs_counts(
 	struct xfs_mount	*mp,
 	struct xfs_fsop_counts __user	*uarg)
 {
@@ -1200,6 +1240,202 @@ xfs_ioctl_fs_counts(
 	return 0;
 }
 
+static int
+xfs_ioc_dioinfo(
+	struct file		*file,
+	void __user		*arg)
+{
+	struct kstat		st;
+	struct dioattr		da;
+	int			error;
+
+	error = vfs_getattr(&file->f_path, &st, STATX_DIOALIGN, 0);
+	if (error)
+		return error;
+
+	/*
+	 * Some userspace directly feeds the return value to posix_memalign,
+	 * which fails for values that are smaller than the pointer size.
+	 * Round up the value to not break userspace.
+	 */
+	da.d_mem = roundup(st.dio_mem_align, sizeof(void *));
+	da.d_miniosz = st.dio_offset_align;
+	da.d_maxiosz = INT_MAX & ~(da.d_miniosz - 1);
+	if (copy_to_user(arg, &da, sizeof(da)))
+		return -EFAULT;
+	return 0;
+}
+
+static int
+xfs_ioc_find_handle(
+	unsigned int		cmd,
+	void __user		*arg)
+{
+	struct xfs_fsop_handlereq hreq;
+
+	if (copy_from_user(&hreq, arg, sizeof(hreq)))
+		return -EFAULT;
+	return xfs_find_handle(cmd, &hreq);
+}
+
+static int
+xfs_ioc_open_by_handle(
+	struct file		*file,
+	void __user		*arg)
+{
+	struct xfs_fsop_handlereq hreq;
+
+	if (copy_from_user(&hreq, arg, sizeof(hreq)))
+		return -EFAULT;
+	return xfs_open_by_handle(file, &hreq);
+}
+
+static int
+xfs_ioc_readlink_by_handle(
+	struct file		*file,
+	void __user		*arg)
+{
+	struct xfs_fsop_handlereq hreq;
+
+	if (copy_from_user(&hreq, arg, sizeof(hreq)))
+		return -EFAULT;
+	return xfs_readlink_by_handle(file, &hreq);
+}
+
+static int
+xfs_ioc_swapext(
+	struct file		*file,
+	void __user		*arg)
+{
+	struct xfs_swapext	sxp;
+	int			error;
+
+	if (copy_from_user(&sxp, arg, sizeof(sxp)))
+		return -EFAULT;
+
+	error = mnt_want_write_file(file);
+	if (error)
+		return error;
+	error = xfs_swapext(&sxp);
+	mnt_drop_write_file(file);
+	return error;
+}
+
+static int
+xfs_ioc_growfs_data(
+	struct file		*file,
+	struct xfs_mount	*mp,
+	void __user		*arg)
+{
+	struct xfs_growfs_data	in;
+	int			error;
+
+	if (copy_from_user(&in, arg, sizeof(in)))
+		return -EFAULT;
+
+	error = mnt_want_write_file(file);
+	if (error)
+		return error;
+	error = xfs_growfs_data(mp, &in);
+	mnt_drop_write_file(file);
+	return error;
+}
+
+static int
+xfs_ioc_growfs_log(
+	struct file		*file,
+	struct xfs_mount	*mp,
+	void  __user		*arg)
+{
+	struct xfs_growfs_log	in;
+	int			error;
+
+	if (copy_from_user(&in, arg, sizeof(in)))
+		return -EFAULT;
+
+	error = mnt_want_write_file(file);
+	if (error)
+		return error;
+	error = xfs_growfs_log(mp, &in);
+	mnt_drop_write_file(file);
+	return error;
+}
+
+static int
+xfs_ioc_growfs_rt(
+	struct file		*file,
+	struct xfs_mount	*mp,
+	void __user		*arg)
+{
+	struct xfs_growfs_rt	in;
+	int			error;
+
+	if (copy_from_user(&in, arg, sizeof(in)))
+		return -EFAULT;
+
+	error = mnt_want_write_file(file);
+	if (error)
+		return error;
+	error = xfs_growfs_rt(mp, &in);
+	mnt_drop_write_file(file);
+	return error;
+}
+
+static int
+xfs_ioc_goingdown(
+	struct xfs_mount	*mp,
+	uint32_t __user		*arg)
+{
+	uint32_t		in;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+	if (get_user(in, arg))
+		return -EFAULT;
+	return xfs_fs_goingdown(mp, in);
+}
+
+static int
+xfs_ioc_error_injection(
+	struct xfs_mount	*mp,
+	struct xfs_error_injection __user *arg)
+{
+	struct xfs_error_injection in;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+	if (copy_from_user(&in, arg, sizeof(in)))
+		return -EFAULT;
+	return xfs_errortag_add(mp, in.errtag);
+}
+
+static int
+xfs_ioc_free_eofblocks(
+	struct xfs_mount	*mp,
+	struct xfs_fs_eofblocks	__user *arg)
+{
+	struct xfs_fs_eofblocks	eofb;
+	struct xfs_icwalk	icw;
+	int			error;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+	if (xfs_is_readonly(mp))
+		return -EROFS;
+
+	if (copy_from_user(&eofb, arg, sizeof(eofb)))
+		return -EFAULT;
+
+	error = xfs_fs_eofblocks_from_user(&eofb, &icw);
+	if (error)
+		return error;
+
+	trace_xfs_ioc_free_eofblocks(mp, &icw, _RET_IP_);
+
+	guard(super_write)(mp->m_super);
+	return xfs_blockgc_free_space(mp, &icw);
+}
+
 /*
  * These long-unused ioctls were removed from the official ioctl API in 5.17,
  * but retain these definitions so that we can log warnings about them.
@@ -1209,12 +1445,6 @@ xfs_ioctl_fs_counts(
 #define XFS_IOC_ALLOCSP64	_IOW ('X', 36, struct xfs_flock64)
 #define XFS_IOC_FREESP64	_IOW ('X', 37, struct xfs_flock64)
 
-/*
- * Note: some of the ioctl's return positive numbers as a
- * byte count indicating success, such as readlink_by_handle.
- * So we don't "sign flip" like most other routines.  This means
- * true errors need to be returned as a negative value.
- */
 long
 xfs_file_ioctl(
 	struct file		*filp,
@@ -1225,7 +1455,6 @@ xfs_file_ioctl(
 	struct xfs_inode	*ip = XFS_I(inode);
 	struct xfs_mount	*mp = ip->i_mount;
 	void			__user *arg = (void __user *)p;
-	int			error;
 
 	trace_xfs_file_ioctl(ip);
 
@@ -1244,26 +1473,9 @@ xfs_file_ioctl(
 	"%s should use fallocate; XFS_IOC_{ALLOC,FREE}SP ioctl unsupported",
 				current->comm);
 		return -ENOTTY;
-	case XFS_IOC_DIOINFO: {
-		struct kstat		st;
-		struct dioattr		da;
 
-		error = vfs_getattr(&filp->f_path, &st, STATX_DIOALIGN, 0);
-		if (error)
-			return error;
-
-		/*
-		 * Some userspace directly feeds the return value to
-		 * posix_memalign, which fails for values that are smaller than
-		 * the pointer size.  Round up the value to not break userspace.
-		 */
-		da.d_mem = roundup(st.dio_mem_align, sizeof(void *));
-		da.d_miniosz = st.dio_offset_align;
-		da.d_maxiosz = INT_MAX & ~(da.d_miniosz - 1);
-		if (copy_to_user(arg, &da, sizeof(da)))
-			return -EFAULT;
-		return 0;
-	}
+	case XFS_IOC_DIOINFO:
+		return xfs_ioc_dioinfo(filp, arg);
 
 	case XFS_IOC_FSBULKSTAT_SINGLE:
 	case XFS_IOC_FSBULKSTAT:
@@ -1311,148 +1523,45 @@ xfs_file_ioctl(
 
 	case XFS_IOC_FD_TO_HANDLE:
 	case XFS_IOC_PATH_TO_HANDLE:
-	case XFS_IOC_PATH_TO_FSHANDLE: {
-		xfs_fsop_handlereq_t	hreq;
-
-		if (copy_from_user(&hreq, arg, sizeof(hreq)))
-			return -EFAULT;
-		return xfs_find_handle(cmd, &hreq);
-	}
-	case XFS_IOC_OPEN_BY_HANDLE: {
-		xfs_fsop_handlereq_t	hreq;
-
-		if (copy_from_user(&hreq, arg, sizeof(xfs_fsop_handlereq_t)))
-			return -EFAULT;
-		return xfs_open_by_handle(filp, &hreq);
-	}
-
-	case XFS_IOC_READLINK_BY_HANDLE: {
-		xfs_fsop_handlereq_t	hreq;
-
-		if (copy_from_user(&hreq, arg, sizeof(xfs_fsop_handlereq_t)))
-			return -EFAULT;
-		return xfs_readlink_by_handle(filp, &hreq);
-	}
+	case XFS_IOC_PATH_TO_FSHANDLE:
+		return xfs_ioc_find_handle(cmd, arg);
+	case XFS_IOC_OPEN_BY_HANDLE:
+		return xfs_ioc_open_by_handle(filp, arg);
+	case XFS_IOC_READLINK_BY_HANDLE:
+		return xfs_ioc_readlink_by_handle(filp, arg);
 	case XFS_IOC_ATTRLIST_BY_HANDLE:
 		return xfs_attrlist_by_handle(filp, arg);
-
 	case XFS_IOC_ATTRMULTI_BY_HANDLE:
 		return xfs_attrmulti_by_handle(filp, arg);
 
-	case XFS_IOC_SWAPEXT: {
-		struct xfs_swapext	sxp;
-
-		if (copy_from_user(&sxp, arg, sizeof(xfs_swapext_t)))
-			return -EFAULT;
-		error = mnt_want_write_file(filp);
-		if (error)
-			return error;
-		error = xfs_ioc_swapext(&sxp);
-		mnt_drop_write_file(filp);
-		return error;
-	}
+	case XFS_IOC_SWAPEXT:
+		return xfs_ioc_swapext(filp, arg);
 
 	case XFS_IOC_FSCOUNTS:
-		return xfs_ioctl_fs_counts(mp, arg);
+		return xfs_ioc_fs_counts(mp, arg);
 
 	case XFS_IOC_SET_RESBLKS:
 	case XFS_IOC_GET_RESBLKS:
-		return xfs_ioctl_getset_resblocks(filp, cmd, arg);
-
-	case XFS_IOC_FSGROWFSDATA: {
-		struct xfs_growfs_data in;
-
-		if (copy_from_user(&in, arg, sizeof(in)))
-			return -EFAULT;
-
-		error = mnt_want_write_file(filp);
-		if (error)
-			return error;
-		error = xfs_growfs_data(mp, &in);
-		mnt_drop_write_file(filp);
-		return error;
-	}
-
-	case XFS_IOC_FSGROWFSLOG: {
-		struct xfs_growfs_log in;
-
-		if (copy_from_user(&in, arg, sizeof(in)))
-			return -EFAULT;
-
-		error = mnt_want_write_file(filp);
-		if (error)
-			return error;
-		error = xfs_growfs_log(mp, &in);
-		mnt_drop_write_file(filp);
-		return error;
-	}
-
-	case XFS_IOC_FSGROWFSRT: {
-		xfs_growfs_rt_t in;
-
-		if (copy_from_user(&in, arg, sizeof(in)))
-			return -EFAULT;
-
-		error = mnt_want_write_file(filp);
-		if (error)
-			return error;
-		error = xfs_growfs_rt(mp, &in);
-		mnt_drop_write_file(filp);
-		return error;
-	}
-
-	case XFS_IOC_GOINGDOWN: {
-		uint32_t in;
-
-		if (!capable(CAP_SYS_ADMIN))
-			return -EPERM;
-
-		if (get_user(in, (uint32_t __user *)arg))
-			return -EFAULT;
-
-		return xfs_fs_goingdown(mp, in);
-	}
-
-	case XFS_IOC_ERROR_INJECTION: {
-		xfs_error_injection_t in;
-
-		if (!capable(CAP_SYS_ADMIN))
-			return -EPERM;
-
-		if (copy_from_user(&in, arg, sizeof(in)))
-			return -EFAULT;
-
-		return xfs_errortag_add(mp, in.errtag);
-	}
-
+		return xfs_ioc_getset_resblocks(filp, cmd, arg);
+
+	case XFS_IOC_FSGROWFSDATA:
+		return xfs_ioc_growfs_data(filp, mp, arg);
+	case XFS_IOC_FSGROWFSLOG:
+		return xfs_ioc_growfs_log(filp, mp, arg);
+	case XFS_IOC_FSGROWFSRT:
+		return xfs_ioc_growfs_rt(filp, mp, arg);
+
+	case XFS_IOC_GOINGDOWN:
+		return xfs_ioc_goingdown(mp, arg);
+	case XFS_IOC_ERROR_INJECTION:
+		return xfs_ioc_error_injection(mp, arg);
 	case XFS_IOC_ERROR_CLEARALL:
 		if (!capable(CAP_SYS_ADMIN))
 			return -EPERM;
-
 		return xfs_errortag_clearall(mp);
 
-	case XFS_IOC_FREE_EOFBLOCKS: {
-		struct xfs_fs_eofblocks	eofb;
-		struct xfs_icwalk	icw;
-
-		if (!capable(CAP_SYS_ADMIN))
-			return -EPERM;
-
-		if (xfs_is_readonly(mp))
-			return -EROFS;
-
-		if (copy_from_user(&eofb, arg, sizeof(eofb)))
-			return -EFAULT;
-
-		error = xfs_fs_eofblocks_from_user(&eofb, &icw);
-		if (error)
-			return error;
-
-		trace_xfs_ioc_free_eofblocks(mp, &icw, _RET_IP_);
-
-		guard(super_write)(mp->m_super);
-		return xfs_blockgc_free_space(mp, &icw);
-	}
+	case XFS_IOC_FREE_EOFBLOCKS:
+		return xfs_ioc_free_eofblocks(mp, arg);
 
 	case XFS_IOC_EXCHANGE_RANGE:
 		return xfs_ioc_exchange_range(filp, arg);
diff --git a/fs/xfs/xfs_ioctl.h b/fs/xfs/xfs_ioctl.h
index f5ed5cf9d3df65..6e55cc8476543d 100644
--- a/fs/xfs/xfs_ioctl.h
+++ b/fs/xfs/xfs_ioctl.h
@@ -10,9 +10,7 @@ struct xfs_bstat;
 struct xfs_ibulk;
 struct xfs_inogrp;
 
-int
-xfs_ioc_swapext(
-	xfs_swapext_t	*sxp);
+int xfs_swapext(struct xfs_swapext *sxp);
 
 extern int
 xfs_fileattr_get(
@@ -21,7 +19,7 @@ xfs_fileattr_get(
 
 extern int
 xfs_fileattr_set(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct dentry		*dentry,
 	struct file_kattr	*fa);
 
diff --git a/fs/xfs/xfs_ioctl32.c b/fs/xfs/xfs_ioctl32.c
index c66e192448a89d..f6875a2cc7065d 100644
--- a/fs/xfs/xfs_ioctl32.c
+++ b/fs/xfs/xfs_ioctl32.c
@@ -44,26 +44,46 @@ xfs_compat_ioc_fsgeometry_v1(
 	return 0;
 }
 
-STATIC int
-xfs_compat_growfs_data_copyin(
-	struct xfs_growfs_data	 *in,
-	compat_xfs_growfs_data_t __user *arg32)
+static int
+xfs_compat_ioc_growfs_data(
+	struct file		*file,
+	struct xfs_mount	*mp,
+	struct compat_xfs_growfs_data __user *arg32)
 {
-	if (get_user(in->newblocks, &arg32->newblocks) ||
-	    get_user(in->imaxpct,   &arg32->imaxpct))
+	struct xfs_growfs_data	in = { };
+	int			error;
+
+	if (get_user(in.newblocks, &arg32->newblocks) ||
+	    get_user(in.imaxpct, &arg32->imaxpct))
 		return -EFAULT;
-	return 0;
+
+	error = mnt_want_write_file(file);
+	if (error)
+		return error;
+	error = xfs_growfs_data(mp, &in);
+	mnt_drop_write_file(file);
+	return error;
 }
 
-STATIC int
-xfs_compat_growfs_rt_copyin(
-	struct xfs_growfs_rt	 *in,
-	compat_xfs_growfs_rt_t	__user *arg32)
+static int
+xfs_compat_ioc_growfs_rt(
+	struct file		*file,
+	struct xfs_mount	*mp,
+	struct compat_xfs_growfs_rt __user *arg32)
 {
-	if (get_user(in->newblocks, &arg32->newblocks) ||
-	    get_user(in->extsize,   &arg32->extsize))
+	struct xfs_growfs_rt	in = {};
+	int			error;
+
+	if (get_user(in.newblocks, &arg32->newblocks) ||
+	    get_user(in.extsize, &arg32->extsize))
 		return -EFAULT;
-	return 0;
+
+	error = mnt_want_write_file(file);
+	if (error)
+		return error;
+	error = xfs_growfs_rt(mp, &in);
+	mnt_drop_write_file(file);
+	return error;
 }
 
 STATIC int
@@ -138,6 +158,27 @@ xfs_ioctl32_bstat_copyin(
 	return 0;
 }
 
+static int
+xfs_compat_ioc_swapext(
+	struct file		*file,
+	struct compat_xfs_swapext __user *sxu)
+{
+	struct xfs_swapext	  sxp;
+	int			error;
+
+	/* Bulk copy in up to the sx_stat field, then copy bstat */
+	if (copy_from_user(&sxp, sxu, offsetof(struct xfs_swapext, sx_stat)) ||
+	    xfs_ioctl32_bstat_copyin(&sxp.sx_stat, &sxu->sx_stat))
+		return -EFAULT;
+
+	error = mnt_want_write_file(file);
+	if (error)
+		return error;
+	error = xfs_swapext(&sxp);
+	mnt_drop_write_file(file);
+	return error;
+}
+
 /* XFS_IOC_FSBULKSTAT and friends */
 
 STATIC int
@@ -338,6 +379,43 @@ xfs_compat_handlereq_to_dentry(
 			compat_ptr(hreq->ihandle), hreq->ihandlen);
 }
 
+static int
+xfs_compat_ioc_find_handle(
+	unsigned int		cmd,
+	void __user		*arg)
+{
+	struct xfs_fsop_handlereq hreq;
+
+	if (xfs_compat_handlereq_copyin(&hreq, arg))
+		return -EFAULT;
+	return xfs_find_handle(_NATIVE_IOC(cmd, struct xfs_fsop_handlereq),
+			&hreq);
+}
+
+static int
+xfs_compat_ioc_open_by_handle(
+	struct file		*file,
+	void __user		*arg)
+{
+	struct xfs_fsop_handlereq hreq;
+
+	if (xfs_compat_handlereq_copyin(&hreq, arg))
+		return -EFAULT;
+	return xfs_open_by_handle(file, &hreq);
+}
+
+static int
+xfs_compat_ioc_readlink_by_handle(
+	struct file		*file,
+	void __user		*arg)
+{
+	struct xfs_fsop_handlereq hreq;
+
+	if (xfs_compat_handlereq_copyin(&hreq, arg))
+		return -EFAULT;
+	return xfs_readlink_by_handle(file, &hreq);
+}
+
 STATIC int
 xfs_compat_attrlist_by_handle(
 	struct file		*parfilp,
@@ -426,7 +504,6 @@ xfs_file_compat_ioctl(
 	struct inode		*inode = file_inode(filp);
 	struct xfs_inode	*ip = XFS_I(inode);
 	void			__user *arg = compat_ptr(p);
-	int			error;
 
 	trace_xfs_file_compat_ioctl(ip);
 
@@ -434,85 +511,34 @@ xfs_file_compat_ioctl(
 #if defined(BROKEN_X86_ALIGNMENT)
 	case XFS_IOC_FSGEOMETRY_V1_32:
 		return xfs_compat_ioc_fsgeometry_v1(ip->i_mount, arg);
-	case XFS_IOC_FSGROWFSDATA_32: {
-		struct xfs_growfs_data	in;
-
-		if (xfs_compat_growfs_data_copyin(&in, arg))
-			return -EFAULT;
-		error = mnt_want_write_file(filp);
-		if (error)
-			return error;
-		error = xfs_growfs_data(ip->i_mount, &in);
-		mnt_drop_write_file(filp);
-		return error;
-	}
-	case XFS_IOC_FSGROWFSRT_32: {
-		struct xfs_growfs_rt	in;
-
-		if (xfs_compat_growfs_rt_copyin(&in, arg))
-			return -EFAULT;
-		error = mnt_want_write_file(filp);
-		if (error)
-			return error;
-		error = xfs_growfs_rt(ip->i_mount, &in);
-		mnt_drop_write_file(filp);
-		return error;
-	}
+	case XFS_IOC_FSGROWFSDATA_32:
+		return xfs_compat_ioc_growfs_data(filp, ip->i_mount, arg);
+	case XFS_IOC_FSGROWFSRT_32:
+		return xfs_compat_ioc_growfs_rt(filp, ip->i_mount, arg);
 #endif
-	/* long changes size, but xfs only copiese out 32 bits */
 	case XFS_IOC_GETVERSION_32:
-		cmd = _NATIVE_IOC(cmd, long);
-		return xfs_file_ioctl(filp, cmd, p);
-	case XFS_IOC_SWAPEXT_32: {
-		struct xfs_swapext	  sxp;
-		struct compat_xfs_swapext __user *sxu = arg;
-
-		/* Bulk copy in up to the sx_stat field, then copy bstat */
-		if (copy_from_user(&sxp, sxu,
-				   offsetof(struct xfs_swapext, sx_stat)) ||
-		    xfs_ioctl32_bstat_copyin(&sxp.sx_stat, &sxu->sx_stat))
-			return -EFAULT;
-		error = mnt_want_write_file(filp);
-		if (error)
-			return error;
-		error = xfs_ioc_swapext(&sxp);
-		mnt_drop_write_file(filp);
-		return error;
-	}
+		/* long changes size, but xfs only copies out 32 bits */
+		return xfs_file_ioctl(filp, _NATIVE_IOC(cmd, long), p);
+	case XFS_IOC_SWAPEXT_32:
+		return xfs_compat_ioc_swapext(filp, arg);
 	case XFS_IOC_FSBULKSTAT_32:
 	case XFS_IOC_FSBULKSTAT_SINGLE_32:
 	case XFS_IOC_FSINUMBERS_32:
 		return xfs_compat_ioc_fsbulkstat(filp, cmd, arg);
 	case XFS_IOC_FD_TO_HANDLE_32:
 	case XFS_IOC_PATH_TO_HANDLE_32:
-	case XFS_IOC_PATH_TO_FSHANDLE_32: {
-		struct xfs_fsop_handlereq	hreq;
-
-		if (xfs_compat_handlereq_copyin(&hreq, arg))
-			return -EFAULT;
-		cmd = _NATIVE_IOC(cmd, struct xfs_fsop_handlereq);
-		return xfs_find_handle(cmd, &hreq);
-	}
-	case XFS_IOC_OPEN_BY_HANDLE_32: {
-		struct xfs_fsop_handlereq	hreq;
-
-		if (xfs_compat_handlereq_copyin(&hreq, arg))
-			return -EFAULT;
-		return xfs_open_by_handle(filp, &hreq);
-	}
-	case XFS_IOC_READLINK_BY_HANDLE_32: {
-		struct xfs_fsop_handlereq	hreq;
-
-		if (xfs_compat_handlereq_copyin(&hreq, arg))
-			return -EFAULT;
-		return xfs_readlink_by_handle(filp, &hreq);
-	}
+	case XFS_IOC_PATH_TO_FSHANDLE_32:
+		return xfs_compat_ioc_find_handle(cmd, arg);
+	case XFS_IOC_OPEN_BY_HANDLE_32:
+		return xfs_compat_ioc_open_by_handle(filp, arg);
+	case XFS_IOC_READLINK_BY_HANDLE_32:
+		return xfs_compat_ioc_readlink_by_handle(filp, arg);
 	case XFS_IOC_ATTRLIST_BY_HANDLE_32:
 		return xfs_compat_attrlist_by_handle(filp, arg);
 	case XFS_IOC_ATTRMULTI_BY_HANDLE_32:
 		return xfs_compat_attrmulti_by_handle(filp, arg);
 	default:
 		/* try the native version */
-		return xfs_file_ioctl(filp, cmd, (unsigned long)arg);
+		return xfs_file_ioctl(filp, cmd, p);
 	}
 }
diff --git a/fs/xfs/xfs_ioend.c b/fs/xfs/xfs_ioend.c
index 40695d18dac081..e70be5b86f0b7a 100644
--- a/fs/xfs/xfs_ioend.c
+++ b/fs/xfs/xfs_ioend.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2016-2025 Christoph Hellwig.
+ * Copyright (c) 2016-2026 Christoph Hellwig.
  * All Rights Reserved.
  */
 #include "xfs_platform.h"
@@ -16,6 +16,135 @@
 #include "xfs_reflink.h"
 #include "xfs_zone_alloc.h"
 #include "xfs_ioend.h"
+#include "xfs_error.h"
+#include "xfs_errortag.h"
+#include <linux/bio-integrity.h>
+
+static void
+xfs_dio_bounce_end_io(
+	struct bio		*bio)
+{
+	struct iomap_ioend	*ioend = iomap_ioend_from_bio(bio);
+	int			error = blk_status_to_errno(bio->bi_status);
+	struct bio		*orig_bio = bio->bi_private;
+
+	if ((ioend->io_flags & IOMAP_IOEND_INTEGRITY) && !bio->bi_status)
+		error = iomap_ioend_integrity_verify(ioend);
+	iomap_bounce_read_end_io(ioend, orig_bio, error);
+}
+
+static void
+xfs_bounce_submit_ioend(
+	struct iomap_ioend	*ioend)
+{
+	if (ioend->io_flags & IOMAP_IOEND_INTEGRITY)
+		fs_bio_integrity_alloc(&ioend->io_bio);
+	ioend->io_bio.bi_end_io = xfs_dio_bounce_end_io;
+	bio_set_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK);
+	submit_bio(&ioend->io_bio);
+}
+
+static void
+xfs_end_bio_bounced(
+	struct bio		*bio)
+{
+	/*
+	 * Just complete the original ioends as all verification is done by the
+	 * end_io handlers for the clone bio(s).
+	 */
+	iomap_finish_ioends(iomap_ioend_from_bio(bio),
+			blk_status_to_errno(bio->bi_status));
+}
+
+static void
+xfs_read_bounce_and_resubmit(
+	struct iomap_ioend	*ioend)
+{
+	struct bio		*bio = &ioend->io_bio;
+	struct xfs_inode	*ip = XFS_I(ioend->io_inode);
+	unsigned int		nofs_flag = memalloc_nofs_save();
+
+	trace_xfs_bounce_reread(ip, ioend->io_offset, ioend->io_size);
+
+	/*
+	 * Free the bio integrity data for the original bio, as we'll allocate
+	 * a new one for each sub-I/O, which could deadlock if we keep the
+	 * integrity data for the original bio around.
+	 */
+	if (bio_integrity(bio))
+		fs_bio_integrity_free(bio);
+
+	/*
+	 * Resubmit the bio through the iomap bounce machinery.  The original
+	 * bio itself is not resubmitted to the block layer, but just used to
+	 * track I/O completion of the cloned bios.
+	 */
+	bio_prepare_reissue(bio, xfs_inode_buftarg(ip)->bt_bdev);
+	bio->bi_iter = (struct bvec_iter) {
+		.bi_sector	= ioend->io_sector,
+		.bi_size	= ioend->io_size,
+		.bi_offset	= ioend->io_bvec_offset,
+	};
+	bio->bi_end_io = xfs_end_bio_bounced;
+	iomap_bounce_read(ioend, bdev_logical_block_size(bio->bi_bdev),
+			xfs_bounce_submit_ioend);
+	memalloc_nofs_restore(nofs_flag);
+}
+
+static void
+xfs_end_io_read(
+	struct bio		*bio)
+{
+	struct iomap_ioend	*ioend = iomap_ioend_from_bio(bio);
+	struct xfs_inode	*ip = XFS_I(ioend->io_inode);
+	struct xfs_mount	*mp = ip->i_mount;
+	int			error = blk_status_to_errno(bio->bi_status);
+
+	if (!error && (ioend->io_flags & IOMAP_IOEND_INTEGRITY)) {
+		error = iomap_ioend_integrity_verify(ioend);
+		if ((ioend->io_flags & IOMAP_IOEND_DIRECT) &&
+		    READ_ONCE(mp->m_read_bounce) == XFS_READ_BOUNCE_LAZY) {
+			/*
+			 * We only really need to retry for guard tag errors,
+			 * but right now we can't distinguish them from other
+			 * (i.e, reftag) errors.
+			 */
+			if (error ||
+			    XFS_TEST_ERROR(mp, XFS_ERRTAG_BOUNCE_REREAD)) {
+				xfs_read_bounce_and_resubmit(ioend);
+				return;
+			}
+		}
+	}
+
+	iomap_finish_ioends(ioend, error);
+}
+
+void
+xfs_ioend_submit_read(
+	struct inode		*inode,
+	struct bio		*bio,
+	loff_t			file_offset,
+	u16			ioend_flags)
+{
+	struct xfs_inode	*ip = XFS_I(inode);
+	struct xfs_mount	*mp = ip->i_mount;
+	struct iomap_ioend	*ioend;
+
+	ioend = iomap_init_ioend(inode, bio, file_offset, ioend_flags);
+	if ((ioend_flags & IOMAP_IOEND_DIRECT) &&
+	    READ_ONCE(mp->m_read_bounce) == XFS_READ_BOUNCE_ALWAYS) {
+		iomap_bounce_read(ioend, bdev_logical_block_size(bio->bi_bdev),
+				xfs_bounce_submit_ioend);
+		return;
+	}
+
+	if (ioend_flags & IOMAP_IOEND_INTEGRITY)
+		fs_bio_integrity_alloc(bio);
+	bio->bi_end_io = xfs_end_io_read;
+	bio_set_flag(bio, BIO_COMPLETE_IN_TASK);
+	submit_bio(bio);
+}
 
 static void
 xfs_ioend_put_open_zones(
@@ -148,11 +277,7 @@ xfs_end_io(
 			io_list))) {
 		list_del_init(&ioend->io_list);
 		iomap_ioend_try_merge(ioend, &tmp);
-		if (bio_op(&ioend->io_bio) == REQ_OP_READ)
-			iomap_finish_ioends(ioend,
-				blk_status_to_errno(ioend->io_bio.bi_status));
-		else
-			xfs_end_ioend_write(ioend);
+		xfs_end_ioend_write(ioend);
 		cond_resched();
 	}
 }
diff --git a/fs/xfs/xfs_ioend.h b/fs/xfs/xfs_ioend.h
index 525865767fcac9..7c2a1ea3e6eddc 100644
--- a/fs/xfs/xfs_ioend.h
+++ b/fs/xfs/xfs_ioend.h
@@ -12,5 +12,7 @@ static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
 }
 
 void xfs_end_bio(struct bio *bio);
+void xfs_ioend_submit_read(struct inode *inode, struct bio *bio,
+		loff_t file_offset, u16 ioend_flags);
 
 #endif /* __XFS_IOEND_H */
diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c
index d1306e723899cd..f67a541ef35a08 100644
--- a/fs/xfs/xfs_iops.c
+++ b/fs/xfs/xfs_iops.c
@@ -169,7 +169,7 @@ xfs_create_need_xattr(
 
 STATIC int
 xfs_generic_create(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct inode		*dir,
 	struct dentry		*dentry,
 	umode_t			mode,
@@ -279,7 +279,7 @@ xfs_generic_create(
 
 STATIC int
 xfs_vn_mknod(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct inode		*dir,
 	struct dentry		*dentry,
 	umode_t			mode,
@@ -290,7 +290,7 @@ xfs_vn_mknod(
 
 STATIC int
 xfs_vn_create(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct inode		*dir,
 	struct dentry		*dentry,
 	umode_t			mode)
@@ -300,7 +300,7 @@ xfs_vn_create(
 
 STATIC struct dentry *
 xfs_vn_mkdir(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct inode		*dir,
 	struct dentry		*dentry,
 	umode_t			mode)
@@ -425,7 +425,7 @@ xfs_vn_unlink(
 
 STATIC int
 xfs_vn_symlink(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct inode		*dir,
 	struct dentry		*dentry,
 	const char		*symname)
@@ -466,7 +466,7 @@ xfs_vn_symlink(
 
 STATIC int
 xfs_vn_rename(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct inode		*odir,
 	struct dentry		*odentry,
 	struct inode		*ndir,
@@ -679,7 +679,7 @@ xfs_report_atomic_write(
 
 STATIC int
 xfs_vn_getattr(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	const struct path	*path,
 	struct kstat		*stat,
 	u32			request_mask,
@@ -754,7 +754,7 @@ xfs_vn_getattr(
 
 static int
 xfs_vn_change_ok(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct dentry		*dentry,
 	struct iattr		*iattr)
 {
@@ -777,7 +777,7 @@ xfs_vn_change_ok(
  */
 static int
 xfs_setattr_nonsize(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct dentry		*dentry,
 	struct xfs_inode	*ip,
 	struct iattr		*iattr)
@@ -903,7 +903,7 @@ xfs_setattr_nonsize(
  */
 int
 xfs_vn_setattr_size(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct dentry		*dentry,
 	struct iattr		*iattr)
 {
@@ -1130,7 +1130,7 @@ xfs_vn_setattr_size(
 
 STATIC int
 xfs_vn_setattr(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct dentry		*dentry,
 	struct iattr		*iattr)
 {
@@ -1250,7 +1250,7 @@ xfs_vn_fiemap(
 
 STATIC int
 xfs_vn_tmpfile(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct inode		*dir,
 	struct file		*file,
 	umode_t			mode)
diff --git a/fs/xfs/xfs_iops.h b/fs/xfs/xfs_iops.h
index 0896f6b8b3b8a1..328305bba19dc7 100644
--- a/fs/xfs/xfs_iops.h
+++ b/fs/xfs/xfs_iops.h
@@ -10,7 +10,7 @@ struct xfs_inode;
 
 extern ssize_t xfs_vn_listxattr(struct dentry *, char *data, size_t size);
 
-int xfs_vn_setattr_size(struct mnt_idmap *idmap,
+int xfs_vn_setattr_size(const struct mnt_idmap *idmap,
 		struct dentry *dentry, struct iattr *vap);
 
 int xfs_inode_init_security(struct inode *inode, struct inode *dir,
diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c
index 159295c63e8f73..a4cf1effa5e613 100644
--- a/fs/xfs/xfs_itable.c
+++ b/fs/xfs/xfs_itable.c
@@ -63,7 +63,7 @@ want_metadir_file(
 STATIC int
 xfs_bulkstat_one_int(
 	struct xfs_mount	*mp,
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct xfs_trans	*tp,
 	xfs_ino_t		ino,
 	struct xfs_bstat_chunk	*bc)
diff --git a/fs/xfs/xfs_itable.h b/fs/xfs/xfs_itable.h
index 2d0612f14d6e12..c0567bfc30fbf8 100644
--- a/fs/xfs/xfs_itable.h
+++ b/fs/xfs/xfs_itable.h
@@ -8,7 +8,7 @@
 /* In-memory representation of a userspace request for batch inode data. */
 struct xfs_ibulk {
 	struct xfs_mount	*mp;
-	struct mnt_idmap	*idmap;
+	const struct mnt_idmap	*idmap;
 	void __user		*ubuffer; /* user output buffer */
 	xfs_ino_t		startino; /* start with this inode */
 	unsigned int		icount;   /* number of elements in ubuffer */
diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c
index f9e07a32f60f71..9446ac44ba8851 100644
--- a/fs/xfs/xfs_log_cil.c
+++ b/fs/xfs/xfs_log_cil.c
@@ -1370,7 +1370,7 @@ xlog_cil_cleanup_whiteouts(
  * allocation context. However, we do not want to block on memory reclaim
  * recursing back into the filesystem because this push may have been triggered
  * by memory reclaim itself. Hence we really need to run under full GFP_NOFS
- * contraints here.
+ * constraints here.
  */
 static void
 xlog_cil_push_work(
diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
index e7e49529658be3..cf0d610265fe76 100644
--- a/fs/xfs/xfs_log_recover.c
+++ b/fs/xfs/xfs_log_recover.c
@@ -2736,12 +2736,13 @@ xlog_recover_iunlink_bucket(
 {
 	struct xfs_mount	*mp = pag_mount(pag);
 	struct xfs_inode	*prev_ip = NULL;
-	struct xfs_inode	*ip;
 	xfs_agino_t		prev_agino, agino;
 	int			error = 0;
 
 	agino = be32_to_cpu(agi->agi_unlinked[bucket]);
 	while (agino != NULLAGINO) {
+		struct xfs_inode	*ip;
+
 		error = xfs_iget(mp, NULL, xfs_agino_to_ino(pag, agino), 0, 0,
 				&ip);
 		if (error)
@@ -2750,11 +2751,11 @@ xlog_recover_iunlink_bucket(
 		ASSERT(VFS_I(ip)->i_nlink == 0);
 		ASSERT(VFS_I(ip)->i_mode != 0);
 		xfs_iflags_clear(ip, XFS_IRECOVERY);
-		agino = ip->i_next_unlinked;
 
 		if (prev_ip) {
 			ip->i_prev_unlinked = prev_agino;
 			xfs_irele(prev_ip);
+			prev_ip = NULL;
 
 			/*
 			 * Ensure the inode is removed from the unlinked list
@@ -2766,18 +2767,20 @@ xlog_recover_iunlink_bucket(
 			 * complete.
 			 */
 			error = xfs_inodegc_flush(mp);
-			if (error)
-				break;
+			if (error) {
+				xfs_irele(ip);
+				return error;
+			}
 		}
 
 		prev_agino = agino;
+		agino = ip->i_next_unlinked;
 		prev_ip = ip;
 	}
 
 	if (prev_ip) {
 		int	error2;
 
-		ip->i_prev_unlinked = prev_agino;
 		xfs_irele(prev_ip);
 
 		error2 = xfs_inodegc_flush(mp);
diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h
index 216a38a354e7c2..894ff2f4ecbd9a 100644
--- a/fs/xfs/xfs_mount.h
+++ b/fs/xfs/xfs_mount.h
@@ -142,6 +142,12 @@ struct xfs_freecounter {
 	uint64_t		res_saved;
 };
 
+enum xfs_read_bounce {
+	XFS_READ_BOUNCE_NEVER,
+	XFS_READ_BOUNCE_ALWAYS,
+	XFS_READ_BOUNCE_LAZY,
+};
+
 /*
  * The struct xfsmount layout is optimised to separate read-mostly variables
  * from variables that are frequently modified. We put the read-mostly variables
@@ -177,6 +183,7 @@ typedef struct xfs_mount {
 	struct workqueue_struct	*m_sync_workqueue;
 	struct workqueue_struct *m_blockgc_wq;
 	struct workqueue_struct *m_inodegc_wq;
+	enum xfs_read_bounce	m_read_bounce;
 
 	int			m_bsize;	/* fs logical block size */
 	uint8_t			m_blkbit_log;	/* blocklog + NBBY */
@@ -291,6 +298,7 @@ typedef struct xfs_mount {
 	struct xfs_zone_info	*m_zone_info;	/* zone allocator information */
 	struct dentry		*m_debugfs;	/* debugfs parent */
 	struct xfs_kobj		m_kobj;
+	struct xfs_kobj		m_csum_kobj;
 	struct xfs_kobj		m_error_kobj;
 	struct xfs_kobj		m_error_meta_kobj;
 	struct xfs_error_cfg	m_error_cfg[XFS_ERR_CLASS_MAX][XFS_ERR_ERRNO_MAX];
diff --git a/fs/xfs/xfs_platform.h b/fs/xfs/xfs_platform.h
index 5d542e95fe444e..e961e36c53fce7 100644
--- a/fs/xfs/xfs_platform.h
+++ b/fs/xfs/xfs_platform.h
@@ -115,10 +115,6 @@ typedef __u32			xfs_nlink_t;
 #define xfs_blockgc_secs	xfs_params.blockgc_timer.val
 
 #define current_cpu()		(raw_smp_processor_id())
-#define current_set_flags_nested(sp, f)		\
-		(*(sp) = current->flags, current->flags |= (f))
-#define current_restore_flags_nested(sp, f)	\
-		(current->flags = ((current->flags & ~(f)) | (*(sp) & (f))))
 
 #define NBBY		8		/* number of bits per byte */
 
@@ -153,7 +149,7 @@ static inline void delay(long ticks)
 /*
  * XFS wrapper structure for sysfs support. It depends on external data
  * structures and is embedded in various internal data structures to implement
- * the XFS sysfs object heirarchy. Define it here for broad access throughout
+ * the XFS sysfs object hierarchy. Define it here for broad access throughout
  * the codebase.
  */
 struct xfs_kobj {
diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c
index 99a82107b8e6f2..008fed8624be2c 100644
--- a/fs/xfs/xfs_qm.c
+++ b/fs/xfs/xfs_qm.c
@@ -1294,11 +1294,7 @@ xfs_qm_quotacheck_dqadjust(
 	 *
 	 * There are no timers for the default values set in the root dquot.
 	 */
-	if (dqp->q_id) {
-		xfs_qm_adjust_dqlimits(dqp);
-		xfs_qm_adjust_dqtimers(dqp);
-	}
-
+	xfs_qm_adjust_dqenforcement(dqp);
 	dqp->q_flags |= XFS_DQFLAG_DIRTY;
 out_unlock:
 	mutex_unlock(&dqp->q_qlock);
@@ -1432,16 +1428,22 @@ xfs_qm_flush_one(
 
 	error = xfs_dquot_use_attached_buf(dqp, &bp);
 	if (error)
-		goto out_unlock;
+		goto out_dqflock;
 	if (!bp) {
 		error = -EFSCORRUPTED;
-		goto out_unlock;
+		goto out_dqflock;
 	}
 
 	error = xfs_qm_dqflush(dqp, bp);
 	if (!error)
 		xfs_buf_delwri_queue(bp, buffer_list);
 	xfs_buf_relse(bp);
+	mutex_unlock(&dqp->q_qlock);
+	xfs_qm_dqrele(dqp);
+	return error;
+
+out_dqflock:
+	xfs_dqfunlock(dqp);
 out_unlock:
 	mutex_unlock(&dqp->q_qlock);
 	xfs_qm_dqrele(dqp);
diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c
index 8bccf89a77668b..682c6e1b45e3ee 100644
--- a/fs/xfs/xfs_refcount_item.c
+++ b/fs/xfs/xfs_refcount_item.c
@@ -508,6 +508,7 @@ xfs_refcount_recover_work(
 	struct xfs_cui_log_item		*cuip = CUI_ITEM(lip);
 	struct xfs_trans		*tp;
 	struct xfs_mount		*mp = lip->li_log->l_mp;
+	unsigned int			dblocks;
 	bool				isrt = xfs_cui_item_isrt(lip);
 	int				i;
 	int				error = 0;
@@ -543,8 +544,11 @@ xfs_refcount_recover_work(
 	 * full btree split on either end of the refcount range.
 	 */
 	resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate);
-	error = xfs_trans_alloc(mp, &resv, mp->m_refc_maxlevels * 2, 0,
-			XFS_TRANS_RESERVE, &tp);
+	if (isrt)
+		dblocks = mp->m_rtrefc_maxlevels * 2;
+	else
+		dblocks = mp->m_refc_maxlevels * 2;
+	error = xfs_trans_alloc(mp, &resv, dblocks, 0, XFS_TRANS_RESERVE, &tp);
 	if (error)
 		return error;
 
diff --git a/fs/xfs/xfs_reflink.h b/fs/xfs/xfs_reflink.h
index 9d1ed9bb0beed8..683c1841e64074 100644
--- a/fs/xfs/xfs_reflink.h
+++ b/fs/xfs/xfs_reflink.h
@@ -48,9 +48,6 @@ extern int xfs_reflink_end_cow(struct xfs_inode *ip, xfs_off_t offset,
 int xfs_reflink_end_atomic_cow(struct xfs_inode *ip, xfs_off_t offset,
 		xfs_off_t count);
 extern int xfs_reflink_recover_cow(struct xfs_mount *mp);
-extern loff_t xfs_reflink_remap_range(struct file *file_in, loff_t pos_in,
-		struct file *file_out, loff_t pos_out, loff_t len,
-		unsigned int remap_flags);
 extern int xfs_reflink_inode_has_shared_extents(struct xfs_trans *tp,
 		struct xfs_inode *ip, bool *has_shared);
 extern int xfs_reflink_clear_inode_flag(struct xfs_inode *ip,
diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c
index 2a3a73a8566d11..be2d5d6fe86337 100644
--- a/fs/xfs/xfs_rmap_item.c
+++ b/fs/xfs/xfs_rmap_item.c
@@ -494,7 +494,7 @@ xfs_rui_validate_map(
 	    !xfs_verify_ino(mp, map->me_owner))
 		return false;
 
-	if (!xfs_verify_fileext(mp, map->me_startoff, map->me_len))
+	if (!xfs_verify_fileext(map->me_startoff, map->me_len))
 		return false;
 
 	if (isrt)
@@ -573,6 +573,7 @@ xfs_rmap_recover_work(
 	struct xfs_rui_log_item		*ruip = RUI_ITEM(lip);
 	struct xfs_trans		*tp;
 	struct xfs_mount		*mp = lip->li_log->l_mp;
+	unsigned int			dblocks;
 	bool				isrt = xfs_rui_item_isrt(lip);
 	int				i;
 	int				error = 0;
@@ -596,8 +597,11 @@ xfs_rmap_recover_work(
 	}
 
 	resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate);
-	error = xfs_trans_alloc(mp, &resv, mp->m_rmap_maxlevels, 0,
-			XFS_TRANS_RESERVE, &tp);
+	if (isrt)
+		dblocks = mp->m_rtrmap_maxlevels;
+	else
+		dblocks = mp->m_rmap_maxlevels;
+	error = xfs_trans_alloc(mp, &resv, dblocks, 0, XFS_TRANS_RESERVE, &tp);
 	if (error)
 		return error;
 
diff --git a/fs/xfs/xfs_rtalloc.h b/fs/xfs/xfs_rtalloc.h
index 78a690b489ed59..1d5108861acdca 100644
--- a/fs/xfs/xfs_rtalloc.h
+++ b/fs/xfs/xfs_rtalloc.h
@@ -46,10 +46,34 @@ int xfs_rtalloc_reinit_frextents(struct xfs_mount *mp);
 int xfs_growfs_check_rtgeom(const struct xfs_mount *mp, xfs_rfsblock_t dblocks,
 		xfs_rfsblock_t rblocks, xfs_agblock_t rextsize);
 #else
-# define xfs_growfs_rt(mp,in)				(-ENOSYS)
-# define xfs_rtalloc_reinit_frextents(m)		(0)
-# define xfs_rtmount_readsb(mp)				(0)
-# define xfs_rtmount_freesb(mp)				((void)0)
+static inline int
+xfs_growfs_rt(
+	struct xfs_mount	*mp,
+	struct xfs_growfs_rt	*in)
+{
+	return -ENOSYS;
+}
+
+static inline int
+xfs_rtalloc_reinit_frextents(
+	struct xfs_mount	*mp)
+{
+	return 0;
+}
+
+static inline int
+xfs_rtmount_readsb(
+	struct xfs_mount	*mp)
+{
+	return 0;
+}
+
+static inline void
+xfs_rtmount_freesb(
+	struct xfs_mount	*mp)
+{
+}
+
 static inline int		/* error */
 xfs_rtmount_init(
 	xfs_mount_t	*mp)	/* file system mount structure */
@@ -60,8 +84,21 @@ xfs_rtmount_init(
 	xfs_warn(mp, "Not built with CONFIG_XFS_RT");
 	return -ENOSYS;
 }
-# define xfs_rtmount_inodes(m)  (((mp)->m_sb.sb_rblocks == 0)? 0 : (-ENOSYS))
-# define xfs_rtunmount_inodes(m)
+
+static inline int
+xfs_rtmount_inodes(
+	struct xfs_mount	*mp)
+{
+	if (mp->m_sb.sb_rblocks)
+		return -ENOSYS;
+	return 0;
+}
+
+static inline void
+xfs_rtunmount_inodes(
+	struct xfs_mount	*mp)
+{
+}
 
 static inline int
 xfs_growfs_check_rtgeom(const struct xfs_mount *mp,
diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
index b24db75eaedc5f..5a06132aa38466 100644
--- a/fs/xfs/xfs_super.c
+++ b/fs/xfs/xfs_super.c
@@ -1889,7 +1889,7 @@ xfs_fs_fill_super(
 	 * Avoid integer overflow by comparing the maximum bmbt offset to the
 	 * maximum pagecache offset in units of fs blocks.
 	 */
-	if (!xfs_verify_fileoff(mp, XFS_B_TO_FSBT(mp, MAX_LFS_FILESIZE))) {
+	if (!xfs_verify_fileoff(XFS_B_TO_FSBT(mp, MAX_LFS_FILESIZE))) {
 		xfs_warn(mp,
 "MAX_LFS_FILESIZE block offset (%llu) exceeds extent map maximum (%llu)!",
 			 XFS_B_TO_FSBT(mp, MAX_LFS_FILESIZE),
@@ -2317,6 +2317,7 @@ xfs_init_fs_context(
 	mp->m_logbufs = -1;
 	mp->m_logbsize = -1;
 	mp->m_allocsize_log = 16; /* 64k */
+	mp->m_read_bounce = XFS_READ_BOUNCE_LAZY;
 
 	xfs_hooks_init(&mp->m_dir_update_hooks);
 
diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c
index 5585ac7f4d16bf..709cd22248f821 100644
--- a/fs/xfs/xfs_symlink.c
+++ b/fs/xfs/xfs_symlink.c
@@ -82,7 +82,7 @@ xfs_readlink(
 
 int
 xfs_symlink(
-	struct mnt_idmap	*idmap,
+	const struct mnt_idmap	*idmap,
 	struct xfs_inode	*dp,
 	struct xfs_name		*link_name,
 	const char		*target_path,
@@ -219,7 +219,7 @@ xfs_symlink(
 	*ipp = du.ip;
 	xfs_iunlock(du.ip, XFS_ILOCK_EXCL);
 	xfs_iunlock(dp, XFS_ILOCK_EXCL);
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
 	return 0;
 
 out_trans_cancel:
@@ -236,7 +236,7 @@ xfs_symlink(
 		xfs_irele(du.ip);
 	}
 out_parent:
-	xfs_parent_finish(mp, du.ppargs);
+	xfs_parent_finish(du.ppargs);
 out_release_dquots:
 	xfs_qm_dqrele(udqp);
 	xfs_qm_dqrele(gdqp);
diff --git a/fs/xfs/xfs_symlink.h b/fs/xfs/xfs_symlink.h
index 0d29a50e66fdc8..3c5a969f9fc558 100644
--- a/fs/xfs/xfs_symlink.h
+++ b/fs/xfs/xfs_symlink.h
@@ -7,7 +7,7 @@
 
 /* Kernel only symlink definitions */
 
-int xfs_symlink(struct mnt_idmap *idmap, struct xfs_inode *dp,
+int xfs_symlink(const struct mnt_idmap *idmap, struct xfs_inode *dp,
 		struct xfs_name *link_name, const char *target_path,
 		umode_t mode, struct xfs_inode **ipp);
 int xfs_readlink(struct xfs_inode *ip, char *link);
diff --git a/fs/xfs/xfs_sysfs.c b/fs/xfs/xfs_sysfs.c
index b6271218732412..e77917ac179d47 100644
--- a/fs/xfs/xfs_sysfs.c
+++ b/fs/xfs/xfs_sysfs.c
@@ -392,6 +392,71 @@ const struct kobj_type xfs_stats_ktype = {
 	.default_groups = xfs_stats_groups,
 };
 
+static inline struct xfs_mount *csum_to_mp(struct kobject *kobj)
+{
+	return container_of(to_kobj(kobj), struct xfs_mount, m_csum_kobj);
+}
+
+static bool
+xfs_has_read_bounce(
+	struct xfs_mount	*mp)
+{
+	if (bdev_has_integrity_csum(mp->m_ddev_targp->bt_bdev))
+		return true;
+	if (mp->m_rtdev_targp &&
+	    bdev_has_integrity_csum(mp->m_rtdev_targp->bt_bdev))
+		return true;
+	return false;
+}
+
+static const char * const bounce_modes[] = {
+	[XFS_READ_BOUNCE_NEVER]		= "never",
+	[XFS_READ_BOUNCE_ALWAYS]	= "always",
+	[XFS_READ_BOUNCE_LAZY]		= "lazy",
+};
+
+static ssize_t
+read_bounce_show(
+	struct kobject		*kobj,
+	char			*buf)
+{
+	struct xfs_mount	*mp = csum_to_mp(kobj);
+
+	return sysfs_emit(buf, "%s\n",
+			bounce_modes[READ_ONCE(mp->m_read_bounce)]);
+}
+
+static ssize_t
+read_bounce_store(
+	struct kobject		*kobj,
+	const char		*buf,
+	size_t			count)
+{
+	struct xfs_mount	*mp = csum_to_mp(kobj);
+	int			ret;
+
+	if (!xfs_has_read_bounce(mp))
+		return -EINVAL;
+	ret = sysfs_match_string(bounce_modes, buf);
+	if (ret < 0)
+		return ret;
+	WRITE_ONCE(mp->m_read_bounce, ret);
+	return count;
+}
+XFS_SYSFS_ATTR_RW(read_bounce);
+
+static struct attribute *xfs_csum_attrs[] = {
+	ATTR_LIST(read_bounce),
+	NULL,
+};
+ATTRIBUTE_GROUPS(xfs_csum);
+
+static const struct kobj_type xfs_csum_ktype = {
+	.release = xfs_sysfs_release,
+	.sysfs_ops = &xfs_sysfs_ops,
+	.default_groups = xfs_csum_groups,
+};
+
 /* xlog */
 
 static inline struct xlog *
@@ -817,11 +882,17 @@ xfs_mount_sysfs_init(
 	if (error)
 		goto out_remove_fsdir;
 
+	/* .../xfs/<dev>/csum/ */
+	error = xfs_sysfs_init(&mp->m_csum_kobj, &xfs_csum_ktype, &mp->m_kobj,
+			"csum");
+	if (error)
+		goto out_remove_stats_dir;
+
 	/* .../xfs/<dev>/error/ */
 	error = xfs_sysfs_init(&mp->m_error_kobj, &xfs_error_ktype,
 				&mp->m_kobj, "error");
 	if (error)
-		goto out_remove_stats_dir;
+		goto out_remove_csum_dir;
 
 	/* .../xfs/<dev>/error/fail_at_unmount */
 	error = sysfs_create_file(&mp->m_error_kobj.kobject,
@@ -835,12 +906,14 @@ xfs_mount_sysfs_init(
 				"metadata", &mp->m_error_meta_kobj,
 				xfs_error_meta_init);
 	if (error)
-		goto out_remove_error_dir;
+		goto out_remove_csum_dir;
 
 	return 0;
 
 out_remove_error_dir:
 	xfs_sysfs_del(&mp->m_error_kobj);
+out_remove_csum_dir:
+	xfs_sysfs_del(&mp->m_csum_kobj);
 out_remove_stats_dir:
 	xfs_sysfs_del(&mp->m_stats.xs_kobj);
 out_remove_fsdir:
@@ -864,6 +937,7 @@ xfs_mount_sysfs_del(
 	}
 	xfs_sysfs_del(&mp->m_error_meta_kobj);
 	xfs_sysfs_del(&mp->m_error_kobj);
+	xfs_sysfs_del(&mp->m_csum_kobj);
 	xfs_sysfs_del(&mp->m_stats.xs_kobj);
 	xfs_sysfs_del(&mp->m_kobj);
 }
diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index 6aa379c2cf0cd1..28d49f158b22d5 100644
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -1896,6 +1896,7 @@ DEFINE_SIMPLE_IO_EVENT(xfs_zero_eof);
 DEFINE_SIMPLE_IO_EVENT(xfs_end_io_direct_write);
 DEFINE_SIMPLE_IO_EVENT(xfs_file_splice_read);
 DEFINE_SIMPLE_IO_EVENT(xfs_zoned_map_blocks);
+DEFINE_SIMPLE_IO_EVENT(xfs_bounce_reread);
 
 DECLARE_EVENT_CLASS(xfs_itrunc_class,
 	TP_PROTO(struct xfs_inode *ip, xfs_fsize_t new_size),
@@ -6030,32 +6031,54 @@ DEFINE_HEALTHMON_EVENT(xfs_healthmon_detach);
 DEFINE_HEALTHMON_EVENT(xfs_healthmon_report_unmount);
 
 #define XFS_HEALTHMON_TYPE_STRINGS \
+	{ XFS_HEALTHMON_RUNNING,	"run" }, \
 	{ XFS_HEALTHMON_LOST,		"lost" }, \
 	{ XFS_HEALTHMON_UNMOUNT,	"unmount" }, \
+	{ XFS_HEALTHMON_SHUTDOWN,	"shutdown" }, \
 	{ XFS_HEALTHMON_SICK,		"sick" }, \
 	{ XFS_HEALTHMON_CORRUPT,	"corrupt" }, \
 	{ XFS_HEALTHMON_HEALTHY,	"healthy" }, \
-	{ XFS_HEALTHMON_SHUTDOWN,	"shutdown" }
+	{ XFS_HEALTHMON_MEDIA_ERROR,	"media" }, \
+	{ XFS_HEALTHMON_BUFREAD,	"bufread" }, \
+	{ XFS_HEALTHMON_BUFWRITE,	"bufwrite" }, \
+	{ XFS_HEALTHMON_DIOREAD,	"dioread" }, \
+	{ XFS_HEALTHMON_DIOWRITE,	"diowrite" }, \
+	{ XFS_HEALTHMON_DATALOST,	"datalost" }
 
 #define XFS_HEALTHMON_DOMAIN_STRINGS \
 	{ XFS_HEALTHMON_MOUNT,		"mount" }, \
 	{ XFS_HEALTHMON_FS,		"fs" }, \
 	{ XFS_HEALTHMON_AG,		"ag" }, \
 	{ XFS_HEALTHMON_INODE,		"inode" }, \
-	{ XFS_HEALTHMON_RTGROUP,	"rtgroup" }
+	{ XFS_HEALTHMON_RTGROUP,	"rtgroup" }, \
+	{ XFS_HEALTHMON_DATADEV,	"datadev" }, \
+	{ XFS_HEALTHMON_RTDEV,		"rtdev" }, \
+	{ XFS_HEALTHMON_LOGDEV,		"logdev" }, \
+	{ XFS_HEALTHMON_FILERANGE,	"filerange" }
 
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_RUNNING);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_LOST);
-TRACE_DEFINE_ENUM(XFS_HEALTHMON_SHUTDOWN);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_UNMOUNT);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_SHUTDOWN);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_SICK);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_CORRUPT);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_HEALTHY);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_MEDIA_ERROR);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_BUFREAD);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_BUFWRITE);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_DIOREAD);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_DIOWRITE);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_DATALOST);
 
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_MOUNT);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_FS);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_AG);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_INODE);
 TRACE_DEFINE_ENUM(XFS_HEALTHMON_RTGROUP);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_DATADEV);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_RTDEV);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_LOGDEV);
+TRACE_DEFINE_ENUM(XFS_HEALTHMON_FILERANGE);
 
 DECLARE_EVENT_CLASS(xfs_healthmon_event_class,
 	TP_PROTO(const struct xfs_healthmon *hm,
diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c
index 1606c614f205ae..93ec876792cc76 100644
--- a/fs/xfs/xfs_trans_dquot.c
+++ b/fs/xfs/xfs_trans_dquot.c
@@ -566,11 +566,7 @@ xfs_trans_apply_dquot_deltas(
 			 * Get any default limits in use.
 			 * Start/reset the timer(s) if needed.
 			 */
-			if (dqp->q_id) {
-				xfs_qm_adjust_dqlimits(dqp);
-				xfs_qm_adjust_dqtimers(dqp);
-			}
-
+			xfs_qm_adjust_dqenforcement(dqp);
 			dqp->q_flags |= XFS_DQFLAG_DIRTY;
 			/*
 			 * add this to the list of items to get logged
diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c
index 1efe6c8139b252..b059a9714d11ab 100644
--- a/fs/xfs/xfs_xattr.c
+++ b/fs/xfs/xfs_xattr.c
@@ -169,7 +169,7 @@ xfs_xattr_flags_to_op(
 
 static int
 xfs_xattr_set(const struct xattr_handler *handler,
-	      struct mnt_idmap *idmap, struct dentry *unused,
+	      const struct mnt_idmap *idmap, struct dentry *unused,
 	      struct inode *inode, const char *name, const void *value,
 	      size_t size, int flags)
 {
diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c
index 28c1e48909fa8d..5f0af0c2c5e509 100644
--- a/fs/xfs/xfs_zone_alloc.c
+++ b/fs/xfs/xfs_zone_alloc.c
@@ -26,6 +26,7 @@
 #include "xfs_zones.h"
 #include "xfs_trace.h"
 #include "xfs_mru_cache.h"
+#include <linux/bio-integrity.h>
 
 static void
 xfs_open_zone_free_rcu(
@@ -820,7 +821,7 @@ xfs_get_cached_zone(
 		spin_unlock(&ip->i_flags_lock);
 	}
 
-	if (!atomic_inc_not_zero(&oz->oz_ref))
+	if (oz && !atomic_inc_not_zero(&oz->oz_ref))
 		oz = NULL;
 out_unlock:
 	rcu_read_unlock();
@@ -828,7 +829,7 @@ xfs_get_cached_zone(
 }
 
 /*
- * Stash our zone in the inode so that is is reused for future allocations.
+ * Stash our zone in the inode so that it is reused for future allocations.
  *
  * The open_zone structure will be pinned until either the inode is freed or
  * until the cached open zone is replaced with a different one because the
@@ -911,6 +912,9 @@ xfs_zone_alloc_and_submit(
 	if (xfs_is_shutdown(mp))
 		goto out_error;
 
+	if (ioend->io_flags & IOMAP_IOEND_INTEGRITY)
+		fs_bio_integrity_generate(&ioend->io_bio);
+
 	/*
 	 * If we don't have a locally cached zone in this write context, see if
 	 * the inode is still associated with a zone and use that if so.
diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c
index 5fdcf98a21338b..54b70ed2922f52 100644
--- a/fs/xfs/xfs_zone_gc.c
+++ b/fs/xfs/xfs_zone_gc.c
@@ -46,7 +46,7 @@
  * before remapping.
  *
  * Once a zone does not contain any valid data, be that through GC or user
- * block removal, it is queued for for a zone reset.  The reset operation
+ * block removal, it is queued for a zone reset.  The reset operation
  * carefully ensures that the RT device cache is flushed and all transactions
  * referencing the rmap have been committed to disk.
  */
diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c
index ff43d6d1ea30ae..b97f1f2b8dda24 100644
--- a/fs/zonefs/super.c
+++ b/fs/zonefs/super.c
@@ -533,7 +533,7 @@ static int zonefs_show_options(struct seq_file *seq, struct dentry *root)
 	return 0;
 }
 
-static int zonefs_inode_setattr(struct mnt_idmap *idmap,
+static int zonefs_inode_setattr(const struct mnt_idmap *idmap,
 				struct dentry *dentry, struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index a10a591c18b239..a755b15af9db97 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -9,6 +9,8 @@
 #ifndef __ACPI_BUS_H__
 #define __ACPI_BUS_H__
 
+#ifdef CONFIG_ACPI
+
 #include <linux/completion.h>
 #include <linux/container_of.h>
 #include <linux/device.h>
@@ -58,7 +60,6 @@ bool acpi_dock_match(acpi_handle handle);
 bool acpi_check_dsm(acpi_handle handle, const guid_t *guid, u64 rev, u64 funcs);
 union acpi_object *acpi_evaluate_dsm(acpi_handle handle, const guid_t *guid,
 			u64 rev, u64 func, union acpi_object *argv4);
-#ifdef CONFIG_ACPI
 bool
 acpi_get_physical_device_location(acpi_handle handle, struct acpi_pld_info **pld);
 
@@ -77,7 +78,6 @@ acpi_evaluate_dsm_typed(acpi_handle handle, const guid_t *guid, u64 rev,
 
 	return obj;
 }
-#endif
 
 #define	ACPI_INIT_DSM_ARGV4(cnt, eles)			\
 	{						\
@@ -90,8 +90,6 @@ bool acpi_dev_found(const char *hid);
 bool acpi_dev_present(const char *hid, const char *uid, s64 hrv);
 bool acpi_reduced_hardware(void);
 
-#ifdef CONFIG_ACPI
-
 struct proc_dir_entry;
 
 #define ACPI_BUS_FILE_ROOT	"acpi"
@@ -435,9 +433,9 @@ struct acpi_device_software_nodes {
 
 /* Device */
 struct acpi_device {
+	acpi_handle handle;		/* no handle for fixed hardware */
 	u32 pld_crc;
 	int device_type;
-	acpi_handle handle;		/* no handle for fixed hardware */
 	struct fwnode_handle fwnode;
 	struct list_head wakeup_list;
 	struct list_head del_list;
@@ -613,7 +611,6 @@ int acpi_bus_get_status(struct acpi_device *device);
 int acpi_bus_set_power(acpi_handle handle, int state);
 const char *acpi_power_state_string(int state);
 int acpi_device_set_power(struct acpi_device *device, int state);
-int acpi_bus_init_power(struct acpi_device *device);
 int acpi_device_fix_up_power(struct acpi_device *device);
 void acpi_device_fix_up_power_extended(struct acpi_device *adev);
 void acpi_device_fix_up_power_children(struct acpi_device *adev);
@@ -665,7 +662,7 @@ struct acpi_bus_type {
 int register_acpi_bus_type(struct acpi_bus_type *);
 int unregister_acpi_bus_type(struct acpi_bus_type *);
 int acpi_bind_one(struct device *dev, struct acpi_device *adev);
-int acpi_unbind_one(struct device *dev);
+void acpi_unbind_one(struct device *dev);
 
 enum acpi_bridge_type {
 	ACPI_BRIDGE_TYPE_PCIE = 1,
@@ -830,7 +827,15 @@ static inline bool acpi_str_uid_match(struct acpi_device *adev, const char *uid2
 {
 	const char *uid1 = acpi_device_uid(adev);
 
-	return uid1 && uid2 && !strcmp(uid1, uid2);
+	if (!uid1 || !uid2)
+		return false;
+
+	if (*uid1 == '\\' && uid1[1])
+		uid1++;
+	if (*uid2 == '\\' && uid2[1])
+		uid2++;
+
+	return !strcmp(uid1, uid2);
 }
 
 static inline bool acpi_int_uid_match(struct acpi_device *adev, u64 uid2)
@@ -857,6 +862,10 @@ static inline bool acpi_int_uid_match(struct acpi_device *adev, u64 uid2)
  *
  * Matches UID in @adev with given @uid2.
  *
+ * If both the UID in @adev and @uid2 are strings, they are compared
+ * after optionally skipping a leading backslash ('\') if the given
+ * string contains additional characters.
+ *
  * Returns: %true if matches, %false otherwise.
  */
 #define acpi_dev_uid_match(adev, uid2)					\
@@ -942,8 +951,21 @@ int acpi_wait_for_acpi_ipmi(void);
 
 int acpi_scan_add_dep(acpi_handle handle, struct acpi_handle_list *dep_devices);
 u32 arch_acpi_add_auto_dep(acpi_handle handle);
+
 #else	/* CONFIG_ACPI */
 
+static inline struct acpi_device *
+acpi_find_child_device(struct acpi_device *parent, u64 address,
+		       bool check_children)
+{
+	return NULL;
+}
+
+static inline bool acpi_has_method(acpi_handle handle, char *name)
+{
+	return false;
+}
+
 static inline struct device *acpi_bus_get_primary_device(struct acpi_device *adev)
 {
 	return NULL;
@@ -959,6 +981,11 @@ static inline const char *acpi_device_hid(struct acpi_device *device)
 	return "";
 }
 
+static inline char *acpi_device_uid(struct acpi_device *device)
+{
+	return NULL;
+}
+
 static inline bool
 acpi_get_physical_device_location(acpi_handle handle, struct acpi_pld_info **pld)
 {
diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h
index 94a6277edab273..3f0005abac6487 100644
--- a/include/acpi/cppc_acpi.h
+++ b/include/acpi/cppc_acpi.h
@@ -72,11 +72,16 @@ struct cpc_register_resource {
 		struct {
 			struct cpc_reg reg;
 			bool use_rmw_lock;
+			bool read_unsupported;
+			bool write_unsupported;
 		};
 		u64 int_value;
 	} cpc_entry;
 };
 
+struct cpc_sysmem_node;
+struct cpc_non_mmio_node;
+
 /* Container to hold the CPC details for each CPU */
 struct cpc_desc {
 	int num_entries;
@@ -84,10 +89,12 @@ struct cpc_desc {
 	int cpu_id;
 	int write_cmd_status;
 	int write_cmd_id;
-	/* Lock used for RMW operations in cpc_write() */
+	/* Serialize partial SystemMemory writes within this descriptor. */
 	raw_spinlock_t rmw_lock;
 	struct cpc_register_resource cpc_regs[MAX_CPC_REG_ENT];
 	struct acpi_psd_package domain_info;
+	struct cpc_sysmem_node *sysmem_nodes;
+	struct cpc_non_mmio_node *non_mmio_nodes;
 	struct kobject kobj;
 };
 
@@ -141,6 +148,8 @@ struct cppc_perf_ctrls {
 	u32 desired_perf;
 	u32 energy_perf;
 	bool auto_sel;
+	/* Allow an explicit zero minimum; otherwise zero omits the update. */
+	bool min_perf_valid;
 };
 
 struct cppc_perf_fb_ctrs {
@@ -188,6 +197,7 @@ extern int cppc_set_epp(int cpu, u64 epp_val);
 extern int cppc_get_auto_act_window(int cpu, u64 *auto_act_window);
 extern int cppc_set_auto_act_window(int cpu, u64 auto_act_window);
 extern int cppc_get_auto_sel(int cpu, bool *enable);
+bool cppc_auto_sel_is_immutable(int cpu);
 extern int cppc_set_auto_sel(int cpu, bool enable);
 extern int cppc_get_perf_limited(int cpu, u64 *perf_limited);
 extern int cppc_set_perf_limited(int cpu, u64 bits_to_clear);
@@ -289,6 +299,12 @@ static inline int cppc_get_auto_sel(int cpu, bool *enable)
 {
 	return -EOPNOTSUPP;
 }
+
+static inline bool cppc_auto_sel_is_immutable(int cpu)
+{
+	return false;
+}
+
 static inline int cppc_set_auto_sel(int cpu, bool enable)
 {
 	return -EOPNOTSUPP;
diff --git a/include/acpi/ghes.h b/include/acpi/ghes.h
index 8d7e5caef3f18a..7acf209061ea78 100644
--- a/include/acpi/ghes.h
+++ b/include/acpi/ghes.h
@@ -85,6 +85,10 @@ int devm_ghes_register_vendor_record_notifier(struct device *dev,
 struct list_head *ghes_get_devices(void);
 
 void ghes_estatus_pool_region_free(unsigned long addr, u32 size);
+
+struct cxl_cper_sec_prot_err;
+void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err,
+			    int severity, u32 len);
 #else
 static inline struct list_head *ghes_get_devices(void) { return NULL; }
 
diff --git a/include/acpi/processor.h b/include/acpi/processor.h
index 554be224ce7666..b5447af4d40b48 100644
--- a/include/acpi/processor.h
+++ b/include/acpi/processor.h
@@ -427,10 +427,8 @@ int acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi);
 #endif /* CONFIG_ACPI_PROCESSOR_IDLE */
 
 /* in processor_thermal.c */
-int acpi_processor_thermal_init(struct acpi_processor *pr,
-				struct acpi_device *device);
-void acpi_processor_thermal_exit(struct acpi_processor *pr,
-				 struct acpi_device *device);
+int acpi_processor_thermal_init(struct acpi_processor *pr);
+void acpi_processor_thermal_exit(struct acpi_processor *pr);
 extern const struct thermal_cooling_device_ops processor_cooling_ops;
 #ifdef CONFIG_CPU_FREQ
 void acpi_thermal_cpufreq_init(struct cpufreq_policy *policy);
diff --git a/include/asm-generic/kexec_handover.h b/include/asm-generic/kexec_handover.h
index 50839fb5ee8ed6..cf73d9b34c3c93 100644
--- a/include/asm-generic/kexec_handover.h
+++ b/include/asm-generic/kexec_handover.h
@@ -4,7 +4,7 @@
 
 #include <linux/types.h>
 
-struct kho_scratch {
+struct kho_bootmem {
 	phys_addr_t addr;
 	phys_addr_t size;
 };
diff --git a/include/asm-generic/pgtable-nop4d.h b/include/asm-generic/pgtable-nop4d.h
index 89c21f84cffbe2..1cf739ee38aa15 100644
--- a/include/asm-generic/pgtable-nop4d.h
+++ b/include/asm-generic/pgtable-nop4d.h
@@ -22,7 +22,6 @@ static inline int pgd_none(pgd_t pgd)		{ return 0; }
 static inline int pgd_bad(pgd_t pgd)		{ return 0; }
 static inline int pgd_present(pgd_t pgd)	{ return 1; }
 static inline void pgd_clear(pgd_t *pgd)	{ }
-#define p4d_ERROR(p4d)				(pgd_ERROR((p4d).pgd))
 
 #define pgd_populate(mm, pgd, p4d)		do { } while (0)
 #define pgd_populate_safe(mm, pgd, p4d)		do { } while (0)
diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h
index 36b6490ed18081..ff4235cf84d772 100644
--- a/include/asm-generic/pgtable-nopmd.h
+++ b/include/asm-generic/pgtable-nopmd.h
@@ -33,7 +33,6 @@ static inline int pud_present(pud_t pud)	{ return 1; }
 static inline int pud_user(pud_t pud)		{ return 0; }
 static inline int pud_leaf(pud_t pud)		{ return 0; }
 static inline void pud_clear(pud_t *pud)	{ }
-#define pmd_ERROR(pmd)				(pud_ERROR((pmd).pud))
 
 #define pud_populate(mm, pmd, pte)		do { } while (0)
 
diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h
index 356cbfbaab2476..eedee8e3ad68fd 100644
--- a/include/asm-generic/pgtable-nopud.h
+++ b/include/asm-generic/pgtable-nopud.h
@@ -29,7 +29,6 @@ static inline int p4d_none(p4d_t p4d)		{ return 0; }
 static inline int p4d_bad(p4d_t p4d)		{ return 0; }
 static inline int p4d_present(p4d_t p4d)	{ return 1; }
 static inline void p4d_clear(p4d_t *p4d)	{ }
-#define pud_ERROR(pud)				(p4d_ERROR((pud).p4d))
 
 #define p4d_populate(mm, p4d, pud)		do { } while (0)
 #define p4d_populate_safe(mm, p4d, pud)		do { } while (0)
diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h
index c8683c046615dd..1adddeab8545a1 100644
--- a/include/asm-generic/preempt.h
+++ b/include/asm-generic/preempt.h
@@ -96,19 +96,9 @@ static __always_inline bool should_resched(int preempt_offset)
 extern asmlinkage void preempt_schedule(void);
 extern asmlinkage void preempt_schedule_notrace(void);
 
-#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-
-void dynamic_preempt_schedule(void);
-void dynamic_preempt_schedule_notrace(void);
-#define __preempt_schedule()		dynamic_preempt_schedule()
-#define __preempt_schedule_notrace()	dynamic_preempt_schedule_notrace()
-
-#else /* !CONFIG_PREEMPT_DYNAMIC || !CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/
-
 #define __preempt_schedule() preempt_schedule()
 #define __preempt_schedule_notrace() preempt_schedule_notrace()
 
-#endif /* CONFIG_PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_KEY*/
 #endif /* CONFIG_PREEMPTION */
 
 #endif /* __ASM_PREEMPT_H */
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index bdcc2778ac64f4..fbe1114e9a443d 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -11,9 +11,9 @@
 #ifndef _ASM_GENERIC__TLB_H
 #define _ASM_GENERIC__TLB_H
 
+#include <linux/mm.h>
 #include <linux/mmu_notifier.h>
 #include <linux/swap.h>
-#include <linux/hugetlb_inline.h>
 #include <asm/tlbflush.h>
 #include <asm/cacheflush.h>
 
@@ -67,11 +67,8 @@
  *  - tlb_remove_table()
  *
  *    tlb_remove_table() is the basic primitive to free page-table directories
- *    (__p*_free_tlb()).  In it's most primitive form it is an alias for
- *    tlb_remove_page() below, for when page directories are pages and have no
- *    additional constraints.
- *
- *    See also MMU_GATHER_TABLE_FREE and MMU_GATHER_RCU_TABLE_FREE.
+ *    (__p*_free_tlb()).  Page directories are freed after an RCU grace
+ *    period - see the comment in mm/mmu_gather.c.
  *
  *  - tlb_remove_page() / tlb_remove_page_size()
  *  - __tlb_remove_folio_pages() / __tlb_remove_page_size()
@@ -151,24 +148,15 @@
  *  This might be useful if your architecture has size specific TLB
  *  invalidation instructions.
  *
- *  MMU_GATHER_TABLE_FREE
- *
- *  This provides tlb_remove_table(), to be used instead of tlb_remove_page()
- *  for page directores (__p*_free_tlb()).
- *
- *  Useful if your architecture has non-page page directories.
+ *  Page directories (__p*_free_tlb()) are always freed via tlb_remove_table(),
+ *  after an RCU grace period (see mm/mmu_gather.c).
  *
- *  When used, an architecture is expected to provide __tlb_remove_table() or
- *  use the generic __tlb_remove_table(), which does the actual freeing of these
- *  pages.
+ *  This serialises against software page-table walkers, including architectures
+ *  which do not use IPIs for remote TLB invalidates.
  *
- *  MMU_GATHER_RCU_TABLE_FREE
- *
- *  Like MMU_GATHER_TABLE_FREE, and adds semi-RCU semantics to the free (see
- *  comment below).
- *
- *  Useful if your architecture doesn't use IPIs for remote TLB invalidates
- *  and therefore doesn't naturally serialize with software page-table walkers.
+ *  An architecture is expected to provide __tlb_remove_table() (see
+ *  HAVE_ARCH_TLB_REMOVE_TABLE) or use the generic __tlb_remove_table(), which
+ *  does the actual freeing of these pages.
  *
  *  MMU_GATHER_NO_FLUSH_CACHE
  *
@@ -200,12 +188,8 @@
  *  various ptep_get_and_clear() functions.
  */
 
-#ifdef CONFIG_MMU_GATHER_TABLE_FREE
-
 struct mmu_table_batch {
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 	struct rcu_head		rcu;
-#endif
 	unsigned int		nr;
 	void			*tables[];
 };
@@ -224,23 +208,6 @@ static inline void __tlb_remove_table(void *table)
 
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 
-#else /* !CONFIG_MMU_GATHER_TABLE_FREE */
-
-static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
-/*
- * Without MMU_GATHER_TABLE_FREE the architecture is assumed to have page based
- * page directories and we can use the normal page batching to free them.
- */
-static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
-{
-	struct ptdesc *ptdesc = (struct ptdesc *)table;
-
-	pagetable_dtor(ptdesc);
-	tlb_remove_page(tlb, ptdesc_page(ptdesc));
-}
-#endif /* CONFIG_MMU_GATHER_TABLE_FREE */
-
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 /*
  * This allows an architecture that does not use the linux page-tables for
  * hardware to skip the TLBI when freeing page tables.
@@ -253,19 +220,6 @@ void tlb_remove_table_sync_one(void);
 
 void tlb_remove_table_sync_rcu(void);
 
-#else
-
-#ifdef tlb_needs_table_invalidate
-#error tlb_needs_table_invalidate() requires MMU_GATHER_RCU_TABLE_FREE
-#endif
-
-static inline void tlb_remove_table_sync_one(void) { }
-
-static inline void tlb_remove_table_sync_rcu(void) { }
-
-#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
-
-
 #ifndef CONFIG_MMU_GATHER_NO_GATHER
 /*
  * If we can't allocate a page to make a big batch of page pointers
@@ -325,9 +279,7 @@ static inline void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct
 struct mmu_gather {
 	struct mm_struct	*mm;
 
-#ifdef CONFIG_MMU_GATHER_TABLE_FREE
 	struct mmu_table_batch	*batch;
-#endif
 
 	unsigned long		start;
 	unsigned long		end;
@@ -486,7 +438,7 @@ tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma)
 	 * We rely on tlb_end_vma() to issue a flush, such that when we reset
 	 * these values the batch is empty.
 	 */
-	tlb->vma_huge = is_vm_hugetlb_page(vma);
+	tlb->vma_huge = vma_is_hugetlb(vma);
 	tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);
 
 	/*
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index b2988aa12f6645..7a3e9f00d6283f 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -855,7 +855,7 @@
 		KLP_SYMID
 
 #define MODINFO								\
-		.modinfo : { *(.modinfo) . = ALIGN(8); }
+		.modinfo (INFO) : { KEEP(*(.modinfo)) }
 
 #ifdef CONFIG_GENERIC_BUG
 #define BUG_TABLE							\
diff --git a/include/crypto/aes-ccm.h b/include/crypto/aes-ccm.h
index 8b00859ac4d6bd..c52982dd91d2c8 100644
--- a/include/crypto/aes-ccm.h
+++ b/include/crypto/aes-ccm.h
@@ -18,6 +18,15 @@ struct aes_ccm_key {
 	size_t authtag_len; /* Length of authentication tags in bytes */
 };
 
+/**
+ * aes_ccm_zeroize_key() - Zeroize an aes_ccm_key structure
+ * @key: The aes_ccm_key to zeroize
+ */
+static inline void aes_ccm_zeroize_key(struct aes_ccm_key *key)
+{
+	memzero_explicit(key, sizeof(*key));
+}
+
 /**
  * struct aes_ccm_ctx - Context for incrementally en/decrypting a message
  */
@@ -50,6 +59,15 @@ struct aes_ccm_ctx {
 	bool ad_padded;
 };
 
+/**
+ * aes_ccm_zeroize_ctx() - Zeroize an aes_ccm_ctx structure
+ * @ctx: The aes_ccm_ctx to zeroize
+ */
+static inline void aes_ccm_zeroize_ctx(struct aes_ccm_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 /**
  * aes_ccm_preparekey() - Prepare a key for AES-CCM encryption and decryption
  * @key: (output) The key structure to initialize
@@ -58,8 +76,8 @@ struct aes_ccm_ctx {
  * @authtag_len: Length of the authentication tag in bytes:
  *		 4, 6, 8, 10, 12, 14, or 16.  16 is recommended.
  *
- * Users should use memzero_explicit() to zeroize the key struct at the end of
- * its lifetime.  (But if this function fails, zeroization is unnecessary.)
+ * Users should use aes_ccm_zeroize_key() to zeroize the key struct at the end
+ * of its lifetime.  (But if this function fails, zeroization is unnecessary.)
  *
  * Context: Any context.
  * Return:
diff --git a/include/crypto/aes-gcm.h b/include/crypto/aes-gcm.h
index 2aee62f0198919..a81b00fd8e27f6 100644
--- a/include/crypto/aes-gcm.h
+++ b/include/crypto/aes-gcm.h
@@ -21,6 +21,15 @@ struct aes_gcm_key {
 	size_t authtag_len; /* Length of authentication tags in bytes */
 };
 
+/**
+ * aes_gcm_zeroize_key() - Zeroize an aes_gcm_key structure
+ * @key: The aes_gcm_key to zeroize
+ */
+static inline void aes_gcm_zeroize_key(struct aes_gcm_key *key)
+{
+	memzero_explicit(key, sizeof(*key));
+}
+
 /**
  * struct aes_gcm_ctx - Context for incrementally en/decrypting a message
  */
@@ -58,6 +67,15 @@ struct aes_gcm_ctx {
 	u64 data_len;
 };
 
+/**
+ * aes_gcm_zeroize_ctx() - Zeroize an aes_gcm_ctx structure
+ * @ctx: The aes_gcm_ctx to zeroize
+ */
+static inline void aes_gcm_zeroize_ctx(struct aes_gcm_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 /**
  * aes_gcm_preparekey() - Prepare a key for AES-GCM encryption and decryption
  * @key: (output) The key structure to initialize
@@ -66,8 +84,8 @@ struct aes_gcm_ctx {
  * @authtag_len: Length of the authentication tag in bytes:
  *		 4, 8, 12, 13, 14, 15, or 16.  16 is recommended.
  *
- * Users should use memzero_explicit() to zeroize the key struct at the end of
- * its lifetime.  (But if this function fails, zeroization is unnecessary.)
+ * Users should use aes_gcm_zeroize_key() to zeroize the key struct at the end
+ * of its lifetime.  (But if this function fails, zeroization is unnecessary.)
  *
  * Context: Any context.
  * Return:
diff --git a/include/crypto/aes-xts.h b/include/crypto/aes-xts.h
index b9e828265e58ab..3a52e1cf40b57a 100644
--- a/include/crypto/aes-xts.h
+++ b/include/crypto/aes-xts.h
@@ -22,6 +22,15 @@ struct aes_xts_key {
 	struct aes_enckey tweak_key;
 };
 
+/**
+ * aes_xts_zeroize_key() - Zeroize an aes_xts_key structure
+ * @key: The aes_xts_key to zeroize
+ */
+static inline void aes_xts_zeroize_key(struct aes_xts_key *key)
+{
+	memzero_explicit(key, sizeof(*key));
+}
+
 /**
  * aes_xts_preparekey() - Prepare a key for AES-XTS encryption and decryption
  * @key: (output) The key structure to initialize
@@ -30,8 +39,8 @@ struct aes_xts_key {
  * @flags: Optional flag XTS_FORBID_WEAK_KEYS to forbid keys whose two halves
  *	   are the same.
  *
- * Users should use memzero_explicit() to zeroize the key struct at the end of
- * its lifetime.  (But if this function fails, zeroization is unnecessary.)
+ * Users should use aes_xts_zeroize_key() to zeroize the key struct at the end
+ * of its lifetime.  (But if this function fails, zeroization is unnecessary.)
  *
  * Context: Any context.
  * Return:
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index 3279cfa5460854..fa5c3d363ac9bd 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -101,6 +101,15 @@ struct aes_enckey {
 	union aes_enckey_arch k;
 };
 
+/**
+ * aes_zeroize_enckey() - Zeroize an aes_enckey structure
+ * @key: The aes_enckey to zeroize
+ */
+static inline void aes_zeroize_enckey(struct aes_enckey *key)
+{
+	memzero_explicit(key, sizeof(*key));
+}
+
 /**
  * struct aes_key - An AES key prepared for encryption and decryption
  * @aes_enckey: Common fields and the key prepared for encryption
@@ -115,6 +124,15 @@ struct aes_key {
 	union aes_invkey_arch inv_k;
 };
 
+/**
+ * aes_zeroize_key() - Zeroize an aes_key structure
+ * @key: The aes_key to zeroize
+ */
+static inline void aes_zeroize_key(struct aes_key *key)
+{
+	memzero_explicit(key, sizeof(*key));
+}
+
 /*
  * Please ensure that the first two fields are 16-byte aligned
  * relative to the start of the structure, i.e., don't move them!
@@ -159,6 +177,19 @@ static inline int aes_check_keylen(size_t keylen)
 int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		  unsigned int key_len);
 
+/**
+ * aes_zeroize_ctx - Clear a crypto_aes_ctx structure
+ * @ctx:	The location of the context that should be zeroized
+ *
+ * Explicitly fills the crypto_aes_ctx with zeroes. This should be done
+ * once the context is not required anymore to avoid that its contents
+ * are leaked on the stack or heap (if not using kfree_sensitive()).
+ */
+static inline void aes_zeroize_ctx(struct crypto_aes_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 /*
  * The following functions are temporarily exported for use by the AES mode
  * implementations in arch/$(SRCARCH)/crypto/.  These exports will go away when
diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h
index 3bc37fd103a7ab..eda1604bce780e 100644
--- a/include/crypto/blake2b.h
+++ b/include/crypto/blake2b.h
@@ -37,6 +37,15 @@ struct blake2b_ctx {
 	unsigned int outlen;
 };
 
+/**
+ * blake2b_zeroize_ctx() - Zeroize a blake2b_ctx structure
+ * @ctx: The blake2b_ctx to zeroize
+ */
+static inline void blake2b_zeroize_ctx(struct blake2b_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 enum blake2b_iv {
 	BLAKE2B_IV0 = 0x6A09E667F3BCC908ULL,
 	BLAKE2B_IV1 = 0xBB67AE8584CAA73BULL,
diff --git a/include/crypto/blake2s.h b/include/crypto/blake2s.h
index 648cb782435887..bb4e6870ed1960 100644
--- a/include/crypto/blake2s.h
+++ b/include/crypto/blake2s.h
@@ -41,6 +41,15 @@ struct blake2s_ctx {
 	unsigned int outlen;
 };
 
+/**
+ * blake2s_zeroize_ctx() - Zeroize a blake2s_ctx structure
+ * @ctx: The blake2s_ctx to zeroize
+ */
+static inline void blake2s_zeroize_ctx(struct blake2s_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 enum blake2s_iv {
 	BLAKE2S_IV0 = 0x6A09E667UL,
 	BLAKE2S_IV1 = 0xBB67AE85UL,
diff --git a/include/crypto/md5.h b/include/crypto/md5.h
index c47aedfe67ecd0..1ed89c15b662c9 100644
--- a/include/crypto/md5.h
+++ b/include/crypto/md5.h
@@ -4,6 +4,7 @@
 
 #include <crypto/hash.h>
 #include <linux/types.h>
+#include <linux/string.h>
 
 #define MD5_DIGEST_SIZE		16
 #define MD5_HMAC_BLOCK_SIZE	64
@@ -98,6 +99,15 @@ struct hmac_md5_key {
 	struct md5_block_state ostate;
 };
 
+/**
+ * hmac_md5_zeroize_key() - Zeroize an hmac_md5_key structure
+ * @key: The hmac_md5_key to zeroize
+ */
+static inline void hmac_md5_zeroize_key(struct hmac_md5_key *key)
+{
+	memzero_explicit(key, sizeof(*key));
+}
+
 /**
  * struct hmac_md5_ctx - Context for computing HMAC-MD5 of a message
  * @hash_ctx: private
@@ -108,6 +118,15 @@ struct hmac_md5_ctx {
 	struct md5_block_state ostate;
 };
 
+/**
+ * hmac_md5_zeroize_ctx() - Zeroize an hmac_md5_ctx structure
+ * @ctx: The hmac_md5_ctx context to zeroize
+ */
+static inline void hmac_md5_zeroize_ctx(struct hmac_md5_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 /**
  * hmac_md5_preparekey() - Prepare a key for HMAC-MD5
  * @key: (output) the key structure to initialize
diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h
index 4d973e016cd696..bc0046bffeaeef 100644
--- a/include/crypto/sha1.h
+++ b/include/crypto/sha1.h
@@ -7,6 +7,7 @@
 #define _CRYPTO_SHA1_H
 
 #include <linux/types.h>
+#include <linux/string.h>
 
 #define SHA1_DIGEST_SIZE        20
 #define SHA1_BLOCK_SIZE         64
@@ -96,6 +97,15 @@ struct hmac_sha1_key {
 	struct sha1_block_state ostate;
 };
 
+/**
+ * hmac_sha1_zeroize_key() - Zeroize an hmac_sha1_key structure
+ * @key: The hmac_sha1_key to zeroize
+ */
+static inline void hmac_sha1_zeroize_key(struct hmac_sha1_key *key)
+{
+	memzero_explicit(key, sizeof(*key));
+}
+
 /**
  * struct hmac_sha1_ctx - Context for computing HMAC-SHA1 of a message
  * @sha_ctx: private
@@ -106,6 +116,15 @@ struct hmac_sha1_ctx {
 	struct sha1_block_state ostate;
 };
 
+/**
+ * hmac_sha1_zeroize_ctx() - Zeroize an hmac_sha1_ctx structure
+ * @ctx: The hmac_sha1_ctx context to zeroize
+ */
+static inline void hmac_sha1_zeroize_ctx(struct hmac_sha1_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 /**
  * hmac_sha1_preparekey() - Prepare a key for HMAC-SHA1
  * @key: (output) the key structure to initialize
diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h
index 371e8a66170546..d044ca80213ba2 100644
--- a/include/crypto/sm3.h
+++ b/include/crypto/sm3.h
@@ -11,6 +11,7 @@
 #define _CRYPTO_SM3_H
 
 #include <linux/types.h>
+#include <linux/string.h>
 
 #define SM3_DIGEST_SIZE	32
 #define SM3_BLOCK_SIZE	64
@@ -41,6 +42,15 @@ struct sm3_ctx {
 	u8 buf[SM3_BLOCK_SIZE] __aligned(__alignof__(__be64));
 };
 
+/**
+ * sm3_zeroize_ctx() - Zeroize an sm3_ctx structure
+ * @ctx: The sm3_ctx to zeroize
+ */
+static inline void sm3_zeroize_ctx(struct sm3_ctx *ctx)
+{
+	memzero_explicit(ctx, sizeof(*ctx));
+}
+
 /**
  * sm3_init() - Initialize an SM3 context for a new message
  * @ctx: the context to initialize
diff --git a/include/cxl/event.h b/include/cxl/event.h
index b5673384d93066..a9f5c2381cc9ef 100644
--- a/include/cxl/event.h
+++ b/include/cxl/event.h
@@ -312,13 +312,13 @@ static inline int cxl_cper_prot_err_kfifo_get(struct cxl_cper_prot_err_work_data
 #endif
 
 #ifdef CONFIG_ACPI_APEI_PCIEAER
-int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err);
+int cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err, u32 len);
 int cxl_cper_setup_prot_err_work_data(struct cxl_cper_prot_err_work_data *wd,
 				      struct cxl_cper_sec_prot_err *prot_err,
 				      int severity);
 #else
 static inline int
-cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err)
+cxl_cper_sec_prot_err_valid(struct cxl_cper_sec_prot_err *prot_err, u32 len)
 {
 	return -EOPNOTSUPP;
 }
@@ -331,6 +331,4 @@ cxl_cper_setup_prot_err_work_data(struct cxl_cper_prot_err_work_data *wd,
 }
 #endif
 
-void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *wd);
-
 #endif /* _LINUX_CXL_EVENT_H */
diff --git a/include/drm/bridge/analogix_dp.h b/include/drm/bridge/analogix_dp.h
index 0e0b87abee590e..6033137b651e8c 100644
--- a/include/drm/bridge/analogix_dp.h
+++ b/include/drm/bridge/analogix_dp.h
@@ -35,7 +35,6 @@ static inline bool analogix_dp_is_rockchip(enum analogix_dp_devtype type)
 
 struct analogix_dp_plat_data {
 	enum analogix_dp_devtype dev_type;
-	struct drm_panel *panel;
 	struct drm_bridge *next_bridge;
 	struct drm_encoder *encoder;
 	struct drm_connector *connector;
diff --git a/include/drm/display/drm_dp.h b/include/drm/display/drm_dp.h
index 8b0ca36083dd18..e63c7f075f54c0 100644
--- a/include/drm/display/drm_dp.h
+++ b/include/drm/display/drm_dp.h
@@ -1035,6 +1035,7 @@
 # define DP_EDP_14a                         0x04    /* eDP 1.4a */
 # define DP_EDP_14b                         0x05    /* eDP 1.4b */
 # define DP_EDP_15			    0x06    /* eDP 1.5 */
+# define DP_EDP_20			    0x07    /* eDP 2.0 */
 
 #define DP_EDP_GENERAL_CAP_1		    0x701
 # define DP_EDP_TCON_BACKLIGHT_ADJUSTMENT_CAP		(1 << 0)
diff --git a/include/drm/display/drm_dp_helper.h b/include/drm/display/drm_dp_helper.h
index ab16c1be3900ff..5b05a7fbe174a6 100644
--- a/include/drm/display/drm_dp_helper.h
+++ b/include/drm/display/drm_dp_helper.h
@@ -139,6 +139,12 @@ bool drm_dp_as_sdp_supported(struct drm_dp_aux *aux, const u8 dpcd[DP_RECEIVER_C
 
 int drm_dp_psr_setup_time(const u8 psr_cap[EDP_PSR_RECEIVER_CAP_SIZE]);
 
+static inline bool
+drm_dp_lane_count_is_valid(int lane_count)
+{
+	return lane_count == 1 || lane_count == 2 || lane_count == 4;
+}
+
 static inline int
 drm_dp_max_link_rate(const u8 dpcd[DP_RECEIVER_CAP_SIZE])
 {
diff --git a/include/drm/display/drm_hdmi_helper.h b/include/drm/display/drm_hdmi_helper.h
index 9c31ed90516b85..2cdd6e3bc090b5 100644
--- a/include/drm/display/drm_hdmi_helper.h
+++ b/include/drm/display/drm_hdmi_helper.h
@@ -8,6 +8,7 @@
 struct drm_connector;
 struct drm_connector_state;
 struct drm_display_mode;
+struct drm_modeset_acquire_ctx;
 enum drm_output_color_format;
 
 void
@@ -35,4 +36,18 @@ drm_hdmi_acr_get_n_cts(unsigned long long tmds_char_rate,
 		       unsigned int *out_n,
 		       unsigned int *out_cts);
 
+bool
+drm_hdmi_mode_needs_scrambling(const struct drm_display_mode *mode,
+			       unsigned int bpc, enum drm_output_color_format fmt);
+
+int
+drm_connector_hdmi_enable_scrambling(struct drm_connector *connector,
+				     const struct drm_connector_state *conn_state);
+int
+drm_connector_hdmi_disable_scrambling(struct drm_connector *connector);
+
+int
+drm_connector_hdmi_sync_scdc(struct drm_connector *connector, bool plugged,
+			     struct drm_modeset_acquire_ctx *ctx);
+
 #endif
diff --git a/include/drm/display/drm_hdmi_state_helper.h b/include/drm/display/drm_hdmi_state_helper.h
index 13375bd0f4ae10..965342f6bbc961 100644
--- a/include/drm/display/drm_hdmi_state_helper.h
+++ b/include/drm/display/drm_hdmi_state_helper.h
@@ -3,10 +3,12 @@
 #ifndef DRM_HDMI_STATE_HELPER_H_
 #define DRM_HDMI_STATE_HELPER_H_
 
+struct dentry;
 struct drm_atomic_commit;
 struct drm_connector;
 struct drm_connector_state;
 struct drm_display_mode;
+struct drm_modeset_acquire_ctx;
 struct hdmi_audio_infoframe;
 
 enum drm_connector_status;
@@ -22,12 +24,17 @@ int drm_atomic_helper_connector_hdmi_update_audio_infoframe(struct drm_connector
 int drm_atomic_helper_connector_hdmi_clear_audio_infoframe(struct drm_connector *connector);
 int drm_atomic_helper_connector_hdmi_update_infoframes(struct drm_connector *connector,
 						       struct drm_atomic_commit *state);
-void drm_atomic_helper_connector_hdmi_hotplug(struct drm_connector *connector,
-					      enum drm_connector_status status);
+int drm_atomic_helper_connector_hdmi_hotplug(struct drm_connector *connector,
+					     struct drm_modeset_acquire_ctx *ctx,
+					     enum drm_connector_status status);
 void drm_atomic_helper_connector_hdmi_force(struct drm_connector *connector);
+int drm_atomic_helper_connector_hdmi_force_ctx(struct drm_connector *connector,
+					       struct drm_modeset_acquire_ctx *ctx);
 
 enum drm_mode_status
 drm_hdmi_connector_mode_valid(struct drm_connector *connector,
 			      const struct drm_display_mode *mode);
+void drm_hdmi_connector_debugfs_init(struct drm_connector *connector,
+				     struct dentry *root);
 
 #endif // DRM_HDMI_STATE_HELPER_H_
diff --git a/include/drm/display/drm_scdc.h b/include/drm/display/drm_scdc.h
index 3d58f37e8ed8ed..7f0b05b2f28006 100644
--- a/include/drm/display/drm_scdc.h
+++ b/include/drm/display/drm_scdc.h
@@ -29,6 +29,8 @@
 #define SCDC_SOURCE_VERSION 0x02
 
 #define SCDC_UPDATE_0 0x10
+#define  SCDC_RSED_UPDATE (1 << 6)
+#define  SCDC_FLT_UPDATE (1 << 5)
 #define  SCDC_READ_REQUEST_TEST (1 << 2)
 #define  SCDC_CED_UPDATE (1 << 1)
 #define  SCDC_STATUS_UPDATE (1 << 0)
@@ -46,14 +48,25 @@
 #define SCDC_CONFIG_0 0x30
 #define  SCDC_READ_REQUEST_ENABLE (1 << 0)
 
+#define SCDC_CONFIG_1 0x31
+#define  SCDC_FRL_RATE   0x0f
+#define  SCDC_FFE_LEVELS 0xf0
+
 #define SCDC_STATUS_FLAGS_0 0x40
+#define  SCDC_DSC_FAIL (1 << 7)
+#define  SCDC_FLT_READY (1 << 6)
+#define  SCDC_LN3_LOCK (1 << 4)
 #define  SCDC_CH2_LOCK (1 << 3)
 #define  SCDC_CH1_LOCK (1 << 2)
 #define  SCDC_CH0_LOCK (1 << 1)
-#define  SCDC_CH_LOCK_MASK (SCDC_CH2_LOCK | SCDC_CH1_LOCK | SCDC_CH0_LOCK)
+#define  SCDC_CH_LOCK_MASK (SCDC_LN3_LOCK | SCDC_CH2_LOCK | SCDC_CH1_LOCK | \
+			    SCDC_CH0_LOCK)
 #define  SCDC_CLOCK_DETECT (1 << 0)
 
 #define SCDC_STATUS_FLAGS_1 0x41
+#define  SCDC_LN_EVEN_TRAIN_PTRN 0x0f
+#define  SCDC_LN_ODD_TRAIN_PTRN 0xf0
+#define SCDC_STATUS_FLAGS_2 0x42
 
 #define SCDC_ERR_DET_0_L 0x50
 #define SCDC_ERR_DET_0_H 0x51
@@ -65,6 +78,12 @@
 
 #define SCDC_ERR_DET_CHECKSUM 0x56
 
+#define SCDC_ERR_DET_3_L 0x57
+#define SCDC_ERR_DET_3_H 0x58
+
+#define SCDC_ERR_DET_RS_L 0x59
+#define SCDC_ERR_DET_RS_H 0x5a
+
 #define SCDC_TEST_CONFIG_0 0xc0
 #define  SCDC_TEST_READ_REQUEST (1 << 7)
 #define  SCDC_TEST_READ_REQUEST_DELAY(x) ((x) & 0x7f)
diff --git a/include/drm/display/drm_scdc_helper.h b/include/drm/display/drm_scdc_helper.h
index 34600476a1b9c9..6a8be1f16ce0a6 100644
--- a/include/drm/display/drm_scdc_helper.h
+++ b/include/drm/display/drm_scdc_helper.h
@@ -24,17 +24,108 @@
 #ifndef DRM_SCDC_HELPER_H
 #define DRM_SCDC_HELPER_H
 
+#include <linux/errno.h>
 #include <linux/types.h>
 
 #include <drm/display/drm_scdc.h>
 
 struct drm_connector;
 struct i2c_adapter;
+struct dentry;
 
-ssize_t drm_scdc_read(struct i2c_adapter *adapter, u8 offset, void *buffer,
-		      size_t size);
-ssize_t drm_scdc_write(struct i2c_adapter *adapter, u8 offset,
-		       const void *buffer, size_t size);
+struct drm_scdc_status_flags {
+	/* Status Register 0 */
+	bool clock_detected;
+	bool ch0_locked;
+	bool ch1_locked;
+	bool ch2_locked;
+	bool ln3_locked;
+	bool flt_ready;
+	bool dsc_fail;
+
+	/* Status Register 1 */
+	u8 ln0_training_pattern : 4;
+	u8 ln1_training_pattern : 4;
+
+	/* Status Register 2 */
+	u8 ln2_training_pattern : 4;
+	u8 ln3_training_pattern : 4;
+};
+
+enum drm_scdc_frl_rate {
+	SCDC_FRL_RATE_OFF = 0,
+	SCDC_FRL_RATE_3X3 = 1,
+	SCDC_FRL_RATE_6X3 = 2,
+	SCDC_FRL_RATE_6X4 = 3,
+	SCDC_FRL_RATE_8X4 = 4,
+	SCDC_FRL_RATE_10X4 = 5,
+	SCDC_FRL_RATE_12X4 = 6,
+	SCDC_FRL_RATE_RESV_7 = 7,
+	SCDC_FRL_RATE_RESV_8 = 8,
+	SCDC_FRL_RATE_RESV_9 = 9,
+	SCDC_FRL_RATE_RESV_10 = 10,
+	SCDC_FRL_RATE_RESV_11 = 11,
+	SCDC_FRL_RATE_RESV_12 = 12,
+	SCDC_FRL_RATE_RESV_13 = 13,
+	SCDC_FRL_RATE_RESV_14 = 14,
+	SCDC_FRL_RATE_RESV_15 = 15
+};
+
+/**
+ * drm_scdc_num_frl_lanes - get number of lanes for a given FRL rate
+ * @rate: one of &enum drm_scdc_frl_rate
+ *
+ * For a given @rate, return the number of lanes it uses.
+ *
+ * Returns: %-EINVAL if @rate is not a valid FRL rate, or the number of lanes
+ * for a given &enum drm_scdc_frl_rate on success (including %0 for "off")
+ */
+static inline __pure int drm_scdc_num_frl_lanes(enum drm_scdc_frl_rate rate)
+{
+	switch (rate) {
+	case SCDC_FRL_RATE_OFF:
+		return 0;
+	case SCDC_FRL_RATE_3X3:
+	case SCDC_FRL_RATE_6X3:
+		return 3;
+	case SCDC_FRL_RATE_6X4:
+	case SCDC_FRL_RATE_8X4:
+	case SCDC_FRL_RATE_10X4:
+	case SCDC_FRL_RATE_12X4:
+		return 4;
+	default:
+		return -EINVAL;
+	}
+}
+
+struct drm_scdc_state {
+	/** @stf: contents of the status flag registers */
+	struct drm_scdc_status_flags stf;
+	/** @scramling_enabled: true if TMDS scrambling is on */
+	bool scrambling_enabled;
+	/** @scrambling_detected: true if the sink actually detected scrambling */
+	bool scrambling_detected;
+	/**
+	 * @tmds_bclk_x40: true if TMDS bit period is 1/40th of the TMDS
+	 * clock period, false if it's 1/10th of the clock period.
+	 */
+	bool tmds_bclk_x40;
+	/** @rate: FRL rate set by the source */
+	enum drm_scdc_frl_rate rate : 4;
+	/** @ffe_levels: The FFE levels for @rate set by the source */
+	u8 ffe_levels : 4;
+	/** @error_count: character error counts for each channel/link */
+	u16 error_count[4];
+	/** @rs_corrections: number of Reed-Solomon Corrections */
+	u16 rs_corrections;
+	/** @scdc: raw SCDC data buffer */
+	u8 scdc[256];
+};
+
+int drm_scdc_read(struct i2c_adapter *adapter, u8 offset, void *buffer,
+		  size_t size);
+int drm_scdc_write(struct i2c_adapter *adapter, u8 offset, const void *buffer,
+		   size_t size);
 
 /**
  * drm_scdc_readb - read a single byte from SCDC
@@ -77,4 +168,10 @@ bool drm_scdc_get_scrambling_status(struct drm_connector *connector);
 bool drm_scdc_set_scrambling(struct drm_connector *connector, bool enable);
 bool drm_scdc_set_high_tmds_clock_ratio(struct drm_connector *connector, bool set);
 
+int drm_scdc_set_source_version(struct drm_connector *connector, u8 ver);
+
+int drm_scdc_read_state(struct drm_connector *connector,
+			struct drm_scdc_state *state);
+void drm_scdc_debugfs_init(struct drm_connector *connector, struct dentry *root);
+
 #endif
diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
index 88087910ab1a3a..3ae35b09c0cfb7 100644
--- a/include/drm/drm_atomic.h
+++ b/include/drm/drm_atomic.h
@@ -159,7 +159,23 @@ struct drm_crtc_commit {
 
 struct __drm_colorops_state {
 	struct drm_colorop *ptr;
-	struct drm_colorop_state *state, *old_state, *new_state;
+
+	/**
+	 * @state_to_destroy:
+	 *
+	 * Used to track the @drm_colorop_state we will need to free
+	 * when tearing down the associated &drm_atomic_commit in
+	 * $drm_mode_config_funcs.atomic_state_clear or
+	 * drm_atomic_commit_default_clear().
+	 *
+	 * Before a commit, and the call to
+	 * drm_atomic_helper_swap_state() in particular, it points to
+	 * the same state than @new_state. After a commit, it points to
+	 * the same state than @old_state.
+	 */
+	struct drm_colorop_state *state_to_destroy;
+
+	struct drm_colorop_state *old_state, *new_state;
 };
 
 struct __drm_planes_state {
@@ -951,9 +967,11 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	     (__i)++)								\
 		for_each_if ((__state)->connectors[__i].ptr &&			\
 			     ((connector) = (__state)->connectors[__i].ptr,	\
-			     (void)(connector) /* Only to avoid unused-but-set-variable warning */, \
-			     (old_connector_state) = (__state)->connectors[__i].old_state,	\
-			     (new_connector_state) = (__state)->connectors[__i].new_state, 1))
+			      (void)(connector) /* Only to avoid unused-but-set-variable warning */, \
+			      (old_connector_state) = (__state)->connectors[__i].old_state,	\
+			      (void)(old_connector_state) /* Only to avoid unused-but-set-variable warning */, \
+			      (new_connector_state) = (__state)->connectors[__i].new_state,	\
+			      (void)(new_connector_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_old_connector_in_state - iterate over all connectors in an atomic update
@@ -973,8 +991,9 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	     (__i)++)								\
 		for_each_if ((__state)->connectors[__i].ptr &&			\
 			     ((connector) = (__state)->connectors[__i].ptr,	\
-			     (void)(connector) /* Only to avoid unused-but-set-variable warning */, \
-			     (old_connector_state) = (__state)->connectors[__i].old_state, 1))
+			      (void)(connector) /* Only to avoid unused-but-set-variable warning */, \
+			      (old_connector_state) = (__state)->connectors[__i].old_state, \
+			      (void)(old_connector_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_new_connector_in_state - iterate over all connectors in an atomic update
@@ -994,9 +1013,9 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	     (__i)++)								\
 		for_each_if ((__state)->connectors[__i].ptr &&			\
 			     ((connector) = (__state)->connectors[__i].ptr,	\
-			     (void)(connector) /* Only to avoid unused-but-set-variable warning */, \
-			     (new_connector_state) = (__state)->connectors[__i].new_state, \
-			     (void)(new_connector_state) /* Only to avoid unused-but-set-variable warning */, 1))
+			      (void)(connector) /* Only to avoid unused-but-set-variable warning */, \
+			      (new_connector_state) = (__state)->connectors[__i].new_state, \
+			      (void)(new_connector_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_oldnew_crtc_in_state - iterate over all CRTCs in an atomic update
@@ -1017,10 +1036,10 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 		for_each_if ((__state)->crtcs[__i].ptr &&		\
 			     ((crtc) = (__state)->crtcs[__i].ptr,	\
 			      (void)(crtc) /* Only to avoid unused-but-set-variable warning */, \
-			     (old_crtc_state) = (__state)->crtcs[__i].old_state, \
-			     (void)(old_crtc_state) /* Only to avoid unused-but-set-variable warning */, \
-			     (new_crtc_state) = (__state)->crtcs[__i].new_state, \
-			     (void)(new_crtc_state) /* Only to avoid unused-but-set-variable warning */, 1))
+			      (old_crtc_state) = (__state)->crtcs[__i].old_state, \
+			      (void)(old_crtc_state) /* Only to avoid unused-but-set-variable warning */, \
+			      (new_crtc_state) = (__state)->crtcs[__i].new_state, \
+			      (void)(new_crtc_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_old_crtc_in_state - iterate over all CRTCs in an atomic update
@@ -1040,7 +1059,8 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 		for_each_if ((__state)->crtcs[__i].ptr &&		\
 			     ((crtc) = (__state)->crtcs[__i].ptr,	\
 			     (void)(crtc) /* Only to avoid unused-but-set-variable warning */, \
-			     (old_crtc_state) = (__state)->crtcs[__i].old_state, 1))
+			     (old_crtc_state) = (__state)->crtcs[__i].old_state, \
+			     (void)(old_crtc_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_new_crtc_in_state - iterate over all CRTCs in an atomic update
@@ -1126,8 +1146,10 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 		for_each_if ((__state)->planes[__i].ptr &&		\
 			     ((plane) = (__state)->planes[__i].ptr,	\
 			      (void)(plane) /* Only to avoid unused-but-set-variable warning */, \
-			      (old_plane_state) = (__state)->planes[__i].old_state,\
-			      (new_plane_state) = (__state)->planes[__i].new_state, 1))
+			      (old_plane_state) = (__state)->planes[__i].old_state, \
+			      (void)(old_plane_state) /* Only to avoid unused-but-set-variable warning */, \
+			      (new_plane_state) = (__state)->planes[__i].new_state, \
+			      (void)(new_plane_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_oldnew_plane_in_state_reverse - iterate over all planes in an atomic
@@ -1148,8 +1170,11 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	     (__i)--)							\
 		for_each_if ((__state)->planes[__i].ptr &&		\
 			     ((plane) = (__state)->planes[__i].ptr,	\
-			      (old_plane_state) = (__state)->planes[__i].old_state,\
-			      (new_plane_state) = (__state)->planes[__i].new_state, 1))
+			      (void)(plane) /* Only to avoid unused-but-set-variable warning */, \
+			      (old_plane_state) = (__state)->planes[__i].old_state, \
+			      (void)(old_plane_state) /* Only to avoid unused-but-set-variable warning */, \
+			      (new_plane_state) = (__state)->planes[__i].new_state, \
+			      (void)(new_plane_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_new_plane_in_state_reverse - other than only tracking new state,
@@ -1165,7 +1190,9 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	     (__i)--)							\
 		for_each_if ((__state)->planes[__i].ptr &&		\
 			     ((plane) = (__state)->planes[__i].ptr,	\
-			      (new_plane_state) = (__state)->planes[__i].new_state, 1))
+			      (void)(plane) /* Only to avoid unused-but-set-variable warning */, \
+			      (new_plane_state) = (__state)->planes[__i].new_state, \
+			      (void)(new_plane_state) /* Only to avoid unused-but-set-variable warning */, 1))
 
 /**
  * for_each_old_plane_in_state - iterate over all planes in an atomic update
@@ -1184,7 +1211,10 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	     (__i)++)							\
 		for_each_if ((__state)->planes[__i].ptr &&		\
 			     ((plane) = (__state)->planes[__i].ptr,	\
-			      (old_plane_state) = (__state)->planes[__i].old_state, 1))
+			      (void)(plane) /* Only to avoid unused-but-set-variable warning */, \
+			      (old_plane_state) = (__state)->planes[__i].old_state, \
+			      (void)(old_plane_state) /* Only to avoid unused-but-set-variable warning */, 1))
+
 /**
  * for_each_new_plane_in_state - iterate over all planes in an atomic update
  * @__state: &struct drm_atomic_commit pointer
@@ -1222,8 +1252,11 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	for ((__i) = 0; \
 	     (__i) < (__state)->num_private_objs && \
 		     ((obj) = (__state)->private_objs[__i].ptr, \
+		      (void)(obj) /* Only to avoid unused-but-set-variable warning */, \
 		      (old_obj_state) = (__state)->private_objs[__i].old_state,	\
-		      (new_obj_state) = (__state)->private_objs[__i].new_state, 1); \
+		      (void)(old_obj_state) /* Only to avoid unused-but-set-variable warning */, \
+		      (new_obj_state) = (__state)->private_objs[__i].new_state, \
+		      (void)(new_obj_state) /* Only to avoid unused-but-set-variable warning */, 1); \
 	     (__i)++)
 
 /**
@@ -1241,7 +1274,9 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	for ((__i) = 0; \
 	     (__i) < (__state)->num_private_objs && \
 		     ((obj) = (__state)->private_objs[__i].ptr, \
-		      (old_obj_state) = (__state)->private_objs[__i].old_state, 1); \
+		      (void)(obj) /* Only to avoid unused-but-set-variable warning */, \
+		      (old_obj_state) = (__state)->private_objs[__i].old_state, \
+		      (void)(old_obj_state) /* Only to avoid unused-but-set-variable warning */, 1); \
 	     (__i)++)
 
 /**
@@ -1260,7 +1295,8 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 	     (__i) < (__state)->num_private_objs && \
 		     ((obj) = (__state)->private_objs[__i].ptr, \
 		      (void)(obj) /* Only to avoid unused-but-set-variable warning */, \
-		      (new_obj_state) = (__state)->private_objs[__i].new_state, 1); \
+		      (new_obj_state) = (__state)->private_objs[__i].new_state, \
+		      (void)(new_obj_state) /* Only to avoid unused-but-set-variable warning */, 1); \
 	     (__i)++)
 
 /**
@@ -1361,11 +1397,8 @@ struct drm_bridge_state {
 	struct drm_bus_cfg output_bus_cfg;
 };
 
-static inline struct drm_bridge_state *
-drm_priv_to_bridge_state(struct drm_private_state *priv)
-{
-	return container_of(priv, struct drm_bridge_state, base);
-}
+#define drm_priv_to_bridge_state(priv)		\
+	container_of_const(priv, struct drm_bridge_state, base)
 
 struct drm_bridge_state *
 drm_atomic_get_bridge_state(struct drm_atomic_commit *state,
diff --git a/include/drm/drm_atomic_state_helper.h b/include/drm/drm_atomic_state_helper.h
index 34a599c3d86d45..4706eeab52c104 100644
--- a/include/drm/drm_atomic_state_helper.h
+++ b/include/drm/drm_atomic_state_helper.h
@@ -42,9 +42,6 @@ struct drm_device;
 
 void __drm_atomic_helper_crtc_state_init(struct drm_crtc_state *state,
 					  struct drm_crtc *crtc);
-void __drm_atomic_helper_crtc_reset(struct drm_crtc *crtc,
-				    struct drm_crtc_state *state);
-void drm_atomic_helper_crtc_reset(struct drm_crtc *crtc);
 struct drm_crtc_state *
 drm_atomic_helper_crtc_create_state(struct drm_crtc *crtc);
 void __drm_atomic_helper_crtc_duplicate_state(struct drm_crtc *crtc,
@@ -59,9 +56,6 @@ void __drm_atomic_helper_plane_state_init(struct drm_plane_state *state,
 					   struct drm_plane *plane);
 struct drm_plane_state *
 drm_atomic_helper_plane_create_state(struct drm_plane *plane);
-void __drm_atomic_helper_plane_reset(struct drm_plane *plane,
-				     struct drm_plane_state *state);
-void drm_atomic_helper_plane_reset(struct drm_plane *plane);
 void __drm_atomic_helper_plane_duplicate_state(struct drm_plane *plane,
 					       struct drm_plane_state *state);
 struct drm_plane_state *
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index 58fff047f43b6e..1981d24a700d22 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -561,6 +561,41 @@ struct drm_bridge_funcs {
 				     const struct drm_display_mode *mode,
 				     unsigned long long tmds_rate);
 
+	/**
+	 * @hdmi_scrambler_enable:
+	 *
+	 * Enable source-side scrambling and the high TMDS clock ratio for the
+	 * mode currently being applied.
+	 *
+	 * This only programs the source side. Sink-side SCDC configuration is
+	 * handled by the HDMI connector helpers.
+	 *
+	 * Mandatory for bridges that set the DRM_BRIDGE_OP_HDMI flag in their
+	 * &drm_bridge->ops and advertise HDMI_VERSION_2_0 or later via
+	 * &drm_bridge->supported_hdmi_ver; unused otherwise.
+	 *
+	 * Returns:
+	 * 0 on success, a negative error code otherwise.
+	 */
+	int (*hdmi_scrambler_enable)(struct drm_bridge *bridge);
+
+	/**
+	 * @hdmi_scrambler_disable:
+	 *
+	 * Disable source-side scrambling and the high TMDS clock ratio.
+	 *
+	 * This only programs the source side. Sink-side SCDC configuration is
+	 * handled by the HDMI connector helpers.
+	 *
+	 * Mandatory for bridges that set the DRM_BRIDGE_OP_HDMI flag in their
+	 * &drm_bridge->ops and advertise HDMI_VERSION_2_0 or later via
+	 * &drm_bridge->supported_hdmi_ver; unused otherwise.
+	 *
+	 * Returns:
+	 * 0 on success, a negative error code otherwise.
+	 */
+	int (*hdmi_scrambler_disable)(struct drm_bridge *bridge);
+
 	/**
 	 * @hdmi_clear_avi_infoframe:
 	 *
@@ -1081,6 +1116,12 @@ struct drm_bridge {
 	 */
 	const char *product;
 
+	/**
+	 * @supported_hdmi_ver: HDMI version the bridge is conformant with.
+	 * This is only relevant if @DRM_BRIDGE_OP_HDMI is set.
+	 */
+	enum hdmi_version supported_hdmi_ver;
+
 	/**
 	 * @supported_formats: Bitmask of @drm_output_color_format listing
 	 * supported output formats. This is only relevant if
@@ -1088,6 +1129,14 @@ struct drm_bridge {
 	 */
 	unsigned int supported_formats;
 
+	/**
+	 * @max_tmds_char_rate: Maximum TMDS character rate, in Hz, the HDMI
+	 * bridge supports. A value of 0 means the core should use the default
+	 * limit implied by @supported_hdmi_ver. This is only relevant if
+	 * @DRM_BRIDGE_OP_HDMI is set.
+	 */
+	unsigned long long max_tmds_char_rate;
+
 	/**
 	 * @max_bpc: Maximum bits per char the HDMI bridge supports. Allowed
 	 * values are 8, 10 and 12. This is only relevant if
@@ -1473,60 +1522,6 @@ void drm_bridge_hpd_disable(struct drm_bridge *bridge);
 void drm_bridge_hpd_notify(struct drm_bridge *bridge,
 			   enum drm_connector_status status);
 
-#ifdef CONFIG_DRM_PANEL_BRIDGE
-bool drm_bridge_is_panel(const struct drm_bridge *bridge);
-struct drm_bridge *drm_panel_bridge_add(struct drm_panel *panel);
-struct drm_bridge *drm_panel_bridge_add_typed(struct drm_panel *panel,
-					      u32 connector_type);
-void drm_panel_bridge_remove(struct drm_bridge *bridge);
-int drm_panel_bridge_set_orientation(struct drm_connector *connector,
-				     struct drm_bridge *bridge);
-struct drm_bridge *devm_drm_panel_bridge_add(struct device *dev,
-					     struct drm_panel *panel);
-struct drm_bridge *devm_drm_panel_bridge_add_typed(struct device *dev,
-						   struct drm_panel *panel,
-						   u32 connector_type);
-struct drm_bridge *drmm_panel_bridge_add(struct drm_device *drm,
-					     struct drm_panel *panel);
-struct drm_connector *drm_panel_bridge_connector(struct drm_bridge *bridge);
-#else
-static inline bool drm_bridge_is_panel(const struct drm_bridge *bridge)
-{
-	return false;
-}
-
-static inline int drm_panel_bridge_set_orientation(struct drm_connector *connector,
-						   struct drm_bridge *bridge)
-{
-	return -EINVAL;
-}
-#endif
-
-#if defined(CONFIG_OF) && defined(CONFIG_DRM_PANEL_BRIDGE)
-struct drm_bridge *devm_drm_of_get_bridge(struct device *dev, struct device_node *node,
-					  u32 port, u32 endpoint);
-struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm, struct device_node *node,
-					  u32 port, u32 endpoint);
-#else
-static inline struct drm_bridge *devm_drm_of_get_bridge(struct device *dev,
-							struct device_node *node,
-							u32 port,
-							u32 endpoint)
-{
-	return ERR_PTR(-ENODEV);
-}
-
-static inline struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm,
-						     struct device_node *node,
-						     u32 port,
-						     u32 endpoint)
-{
-	return ERR_PTR(-ENODEV);
-}
-#endif
-
-void devm_drm_put_bridge(struct device *dev, struct drm_bridge *bridge);
-
 void drm_bridge_debugfs_params(struct dentry *root);
 void drm_bridge_debugfs_encoder_params(struct dentry *root, struct drm_encoder *encoder);
 
diff --git a/include/drm/drm_client.h b/include/drm/drm_client.h
index 49a21f3dcb36fc..10a0cae3e48fcd 100644
--- a/include/drm/drm_client.h
+++ b/include/drm/drm_client.h
@@ -66,6 +66,20 @@ struct drm_client_funcs {
 	 */
 	int (*restore)(struct drm_client_dev *client, bool force);
 
+	/**
+	 * @acquire_outputs:
+	 *
+	 * Called by vga-switcheroo. Informs the client that the outputs will
+	 * be switched to its device. When @acquire_outputs runs, the outputs
+	 * have not been switched yet. The client should only prepare the software
+	 * state. After the switch happened, the client might get a hotplug
+	 * event to update the hardware state.
+	 *
+	 * This callback exists for remapping framebuffers to virtual terminals
+	 * in fbcon.
+	 */
+	void (*acquire_outputs)(struct drm_client_dev *client);
+
 	/**
 	 * @hotplug:
 	 *
diff --git a/include/drm/drm_client_event.h b/include/drm/drm_client_event.h
index 79369c755bc90d..c93f404bae1d9f 100644
--- a/include/drm/drm_client_event.h
+++ b/include/drm/drm_client_event.h
@@ -11,6 +11,7 @@ struct drm_device;
 void drm_client_dev_unregister(struct drm_device *dev);
 void drm_client_dev_hotplug(struct drm_device *dev);
 void drm_client_dev_restore(struct drm_device *dev, bool force);
+void drm_client_dev_acquire_outputs(struct drm_device *dev);
 void drm_client_dev_suspend(struct drm_device *dev);
 void drm_client_dev_resume(struct drm_device *dev);
 #else
@@ -20,6 +21,8 @@ static inline void drm_client_dev_hotplug(struct drm_device *dev)
 { }
 static inline void drm_client_dev_restore(struct drm_device *dev, bool force)
 { }
+static inline void drm_client_dev_acquire_outputs(struct drm_device *dev)
+{ }
 static inline void drm_client_dev_suspend(struct drm_device *dev)
 { }
 static inline void drm_client_dev_resume(struct drm_device *dev)
diff --git a/include/drm/drm_colorop.h b/include/drm/drm_colorop.h
index 224fae40ed2b20..26ef67c835e2e9 100644
--- a/include/drm/drm_colorop.h
+++ b/include/drm/drm_colorop.h
@@ -134,6 +134,114 @@ enum drm_colorop_curve_1d_type {
 	DRM_COLOROP_1D_CURVE_COUNT
 };
 
+/**
+ * enum drm_colorop_fixed_matrix_type - type of Fixed Matrix
+ *
+ * Describes a Fixed Matrix operation to be applied by the DRM_COLOROP_FIXED_MATRIX
+ */
+enum drm_colorop_fixed_matrix_type {
+	/**
+	 * @DRM_COLOROP_FM_YCBCR601_FULL_RGB:
+	 *
+	 * enum string "YCbCr 601 Full to RGB"
+	 *
+	 * Converts full-range YCbCr into full-range RGB using the BT.601
+	 * coefficients. Y is normalized to [0, 1] and Cb, Cr are centered
+	 * at 0 (the nominal 0.5 offset removed) before the matrix::
+	 *
+	 *   | R |   | 1.0   0.000000   1.402000 |   | Y  |
+	 *   | G | = | 1.0  -0.344136  -0.714136 | x | Cb |
+	 *   | B |   | 1.0   1.772000   0.000000 |   | Cr |
+	 */
+	DRM_COLOROP_FM_YCBCR601_FULL_RGB,
+
+	/**
+	 * @DRM_COLOROP_FM_YCBCR601_LIMITED_RGB:
+	 *
+	 * enum string "YCbCr 601 Limited to RGB"
+	 *
+	 * Converts limited- (narrow-) range YCbCr into full-range RGB using
+	 * the BT.601 coefficients. Before the matrix Y is offset by 16/255
+	 * and Cb, Cr are centered at 0, following the studio-range
+	 * convention of ITU-R BT.601::
+	 *
+	 *   | R |   | 1.164384   0.000000   1.596027 |   | Y - 16/255 |
+	 *   | G | = | 1.164384  -0.391762  -0.812968 | x | Cb         |
+	 *   | B |   | 1.164384   2.017232   0.000000 |   | Cr         |
+	 */
+	DRM_COLOROP_FM_YCBCR601_LIMITED_RGB,
+
+	/**
+	 * @DRM_COLOROP_FM_YCBCR709_FULL_RGB:
+	 *
+	 * enum string "YCbCr 709 Full to RGB"
+	 *
+	 * Converts full-range YCbCr into full-range RGB using the BT.709
+	 * coefficients. Y is normalized to [0, 1] and Cb, Cr are centered
+	 * at 0 (the nominal 0.5 offset removed) before the matrix::
+	 *
+	 *   | R |   | 1.0   0.000000   1.574800 |   | Y  |
+	 *   | G | = | 1.0  -0.187324  -0.468124 | x | Cb |
+	 *   | B |   | 1.0   1.855600   0.000000 |   | Cr |
+	 */
+	DRM_COLOROP_FM_YCBCR709_FULL_RGB,
+
+	/**
+	 * @DRM_COLOROP_FM_YCBCR709_LIMITED_RGB:
+	 *
+	 * enum string "YCbCr 709 Limited to RGB"
+	 *
+	 * Converts limited- (narrow-) range YCbCr into full-range RGB using
+	 * the BT.709 coefficients. Before the matrix Y is offset by 16/255
+	 * and Cb, Cr are centered at 0, following the studio-range
+	 * convention of ITU-R BT.709::
+	 *
+	 *   | R |   | 1.164384   0.000000   1.792741 |   | Y - 16/255 |
+	 *   | G | = | 1.164384  -0.213249  -0.532909 | x | Cb         |
+	 *   | B |   | 1.164384   2.112402   0.000000 |   | Cr         |
+	 */
+	DRM_COLOROP_FM_YCBCR709_LIMITED_RGB,
+
+	/**
+	 * @DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB:
+	 *
+	 * enum string "YCbCr 2020 NC Full to RGB"
+	 *
+	 * Converts full-range YCbCr into full-range RGB using the BT.2020
+	 * non-constant luminance coefficients. Y is normalized to [0, 1]
+	 * and Cb, Cr are centered at 0 (the nominal 0.5 offset removed)
+	 * before the matrix::
+	 *
+	 *   | R |   | 1.0   0.000000   1.474600 |   | Y  |
+	 *   | G | = | 1.0  -0.164553  -0.571353 | x | Cb |
+	 *   | B |   | 1.0   1.881400   0.000000 |   | Cr |
+	 */
+	DRM_COLOROP_FM_YCBCR2020_NC_FULL_RGB,
+
+	/**
+	 * @DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB:
+	 *
+	 * enum string "YCbCr 2020 NC Limited to RGB"
+	 *
+	 * Converts limited- (narrow-) range YCbCr into full-range RGB using
+	 * the BT.2020 non-constant luminance coefficients. Before the matrix
+	 * Y is offset by 16/255 and Cb, Cr are centered at 0, following the
+	 * studio-range convention of ITU-R BT.2020::
+	 *
+	 *   | R |   | 1.164384   0.000000   1.678674 |   | Y - 16/255 |
+	 *   | G | = | 1.164384  -0.187326  -0.650424 | x | Cb         |
+	 *   | B |   | 1.164384   2.141772   0.000000 |   | Cr         |
+	 */
+	DRM_COLOROP_FM_YCBCR2020_NC_LIMITED_RGB,
+
+	/**
+	 * @DRM_COLOROP_FM_COUNT:
+	 *
+	 * enum value denoting the size of the enum
+	 */
+	DRM_COLOROP_FM_COUNT
+};
+
 /**
  * struct drm_colorop_state - mutable colorop state
  */
@@ -197,6 +305,13 @@ struct drm_colorop_state {
 	 */
 	enum drm_colorop_lut3d_interpolation_type lut3d_interpolation;
 
+	/**
+	 * @fixed_matrix_type:
+	 *
+	 * Type of Fixed Matrix operation.
+	 */
+	enum drm_colorop_fixed_matrix_type fixed_matrix_type;
+
 	/** @state: backpointer to global drm_atomic_commit */
 	struct drm_atomic_commit *state;
 };
@@ -366,6 +481,13 @@ struct drm_colorop {
 	 */
 	struct drm_property *data_property;
 
+	/**
+	 * @fixed_matrix_type_property:
+	 *
+	 * Sub-type for DRM_COLOROP_FIXED_MATRIX type.
+	 */
+	struct drm_property *fixed_matrix_type_property;
+
 	/**
 	 * @next_property:
 	 *
@@ -422,6 +544,10 @@ int drm_plane_colorop_3dlut_init(struct drm_device *dev, struct drm_colorop *col
 				 uint32_t lut_size,
 				 enum drm_colorop_lut3d_interpolation_type interpolation,
 				 uint32_t flags);
+int drm_plane_colorop_fixed_matrix_init(struct drm_device *dev, struct drm_colorop *colorop,
+					struct drm_plane *plane,
+					const struct drm_colorop_funcs *funcs,
+					u64 supported_fm, uint32_t flags);
 
 struct drm_colorop_state *
 drm_atomic_helper_colorop_create_state(struct drm_colorop *colorop);
@@ -480,6 +606,7 @@ drm_get_colorop_lut1d_interpolation_name(enum drm_colorop_lut1d_interpolation_ty
 
 const char *
 drm_get_colorop_lut3d_interpolation_name(enum drm_colorop_lut3d_interpolation_type type);
+const char *drm_get_colorop_fixed_matrix_type_name(enum drm_colorop_fixed_matrix_type type);
 
 void drm_colorop_set_next_property(struct drm_colorop *colorop, struct drm_colorop *next);
 
diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
index a0cf0268de483a..2ee7c59329aaea 100644
--- a/include/drm/drm_connector.h
+++ b/include/drm/drm_connector.h
@@ -28,6 +28,7 @@
 #include <linux/ctype.h>
 #include <linux/hdmi.h>
 #include <linux/notifier.h>
+#include <linux/workqueue.h>
 #include <drm/drm_mode_object.h>
 #include <drm/drm_util.h>
 #include <drm/drm_property.h>
@@ -254,6 +255,44 @@ struct drm_scdc {
 	struct drm_scrambling scrambling;
 };
 
+/**
+ * struct drm_hdmi_vrr_cap - Information about VRR capabilities of a HDMI sink
+ *
+ * Describes the VRR support provided by HDMI 2.1 sink. The information is
+ * fetched fom additional HFVSDB blocks defined for HDMI 2.1.
+ */
+struct drm_hdmi_vrr_cap {
+	/** @fva: flag for Fast VActive (Quick Frame Transport) support */
+	bool fva;
+
+	/** @mcnmvrr: flag for Negative M VRR support */
+	bool cnmvrr;
+
+	/** @mcinema_vrr: flag for Cinema VRR support */
+	bool cinema_vrr;
+
+	/** @mdelta: flag for limited frame-to-frame compensation support */
+	bool mdelta;
+
+	/**
+	 * @vrr_min : minimum supported variable refresh rate in Hz.
+	 * Valid values only inide 1 - 48 range
+	 */
+	u16 vrr_min;
+
+	/**
+	 * @vrr_max : maximum supported variable refresh rate in Hz (optional).
+	 * Valid values are either 0 (max based on video mode) or >= 100
+	 */
+	u16 vrr_max;
+
+	/**
+	 * @supported: flag for vrr support based on checking for VRRmin and
+	 * VRRmax values having correct values.
+	 */
+	bool supported;
+};
+
 /**
  * struct drm_hdmi_dsc_cap - DSC capabilities of HDMI sink
  *
@@ -330,6 +369,15 @@ struct drm_hdmi_info {
 	/** @max_lanes: supported by sink */
 	u8 max_lanes;
 
+	/** @fapa_start_location: flag for the FAPA in blanking support */
+	bool fapa_start_location;
+
+	/** @allm: flag for Auto Low Latency Mode support by sink */
+	bool allm;
+
+	/** @vrr_cap: VRR capabilities of the sink */
+	struct drm_hdmi_vrr_cap vrr_cap;
+
 	/** @dsc_cap: DSC capabilities of the sink */
 	struct drm_hdmi_dsc_cap dsc_cap;
 };
@@ -765,7 +813,7 @@ enum drm_bus_flags {
  * struct drm_amd_vsdb_info - AMD-specific VSDB information
  *
  * This structure holds information parsed from the AMD Vendor-Specific Data
- * Block (VSDB) version 3.
+ * Block (VSDB) versions 1 to 3.
  */
 struct drm_amd_vsdb_info {
 	/**
@@ -773,6 +821,26 @@ struct drm_amd_vsdb_info {
 	 */
 	u8 version;
 
+	/**
+	 * @freesync_supported: FreeSync (variable refresh rate) supported by panel
+	 */
+	bool freesync_supported;
+
+	/**
+	 * @min_frame_rate: FreeSync minimum refresh rate in Hz
+	 */
+	u16 min_frame_rate;
+
+	/**
+	 * @max_frame_rate: FreeSync maximum refresh rate in Hz
+	 */
+	u16 max_frame_rate;
+
+	/**
+	 * @freesync_vcp_code: MCCS VCP code for FreeSync
+	 */
+	u8 freesync_vcp_code;
+
 	/**
 	 * @replay_mode: Panel Replay supported
 	 */
@@ -1131,6 +1199,17 @@ struct drm_connector_hdmi_state {
 	 * @tmds_char_rate: TMDS Character Rate, in Hz.
 	 */
 	unsigned long long tmds_char_rate;
+
+	/**
+	 * @scrambler_needed: Whether HDMI 2.0 SCDC scrambling is required
+	 * for the negotiated mode/bpc/format.
+	 *
+	 * Computed by drm_atomic_helper_connector_hdmi_check() according to
+	 * the HDMI 2.0 specification: scrambling is mandatory above a 340 MHz
+	 * TMDS character rate. Optional scrambling at lower rates is
+	 * deliberately not requested by the helper.
+	 */
+	bool scrambler_needed;
 };
 
 /**
@@ -1403,8 +1482,50 @@ struct drm_connector_infoframe_funcs {
 
 /**
  * struct drm_connector_hdmi_funcs - drm_hdmi_connector control functions
+ * and controller capabilities
  */
 struct drm_connector_hdmi_funcs {
+	/**
+	 * @vendor: HDMI Controller Vendor name.
+	 */
+	const char *vendor;
+
+	/**
+	 * @product: HDMI Controller Product name
+	 */
+	const char *product;
+
+	/**
+	 * @supported_hdmi_ver:
+	 *
+	 * Maximum HDMI specification version supported by the controller side
+	 * of this connector. This describes the controller capability only;
+	 * the effective link capabilities may be further restricted by the
+	 * sink, bridge, or mode validation.
+	 */
+	enum hdmi_version supported_hdmi_ver;
+
+	/**
+	 * @supported_tmds_char_rate:
+	 *
+	 * Maximum TMDS character rate supported by the controller, in Hz.
+	 * A value of 0 means the core should use the default limit implied by
+	 * @supported_hdmi_ver.
+	 */
+	unsigned long long supported_tmds_char_rate;
+
+	/**
+	 * @supported_formats:
+	 *
+	 * Bitmask of @drm_output_color_format listing supported output formats.
+	 */
+	unsigned long supported_formats;
+
+	/**
+	 * @max_bpc: Maximum bits per char the HDMI connector supports.
+	 */
+	unsigned int max_bpc;
+
 	/**
 	 * @tmds_char_rate_valid:
 	 *
@@ -1439,6 +1560,36 @@ struct drm_connector_hdmi_funcs {
 	 */
 	const struct drm_edid *(*read_edid)(struct drm_connector *connector);
 
+	/**
+	 * @scrambler_enable:
+	 *
+	 * The callback is invoked via @drm_connector_hdmi_enable_scrambling
+	 * during commit to setup SCDC scrambling and high TMDS clock ratio on
+	 * the source side.
+	 *
+	 * The @scrambler_enable callback is mandatory if HDMI 2.0 is to be
+	 * supported.
+	 *
+	 * Returns:
+	 * 0 on success, a negative error code otherwise
+	 */
+	int (*scrambler_enable)(struct drm_connector *connector);
+
+	/**
+	 * @scrambler_disable:
+	 *
+	 * The callback is invoked via @drm_connector_hdmi_disable_scrambling
+	 * during commit to tear down SCDC scrambling and high TMDS clock ratio
+	 * on the source side.
+	 *
+	 * The @scrambler_disable callback is mandatory if HDMI 2.0 is to be
+	 * supported.
+	 *
+	 * Returns:
+	 * 0 on success, a negative error code otherwise
+	 */
+	int (*scrambler_disable)(struct drm_connector *connector);
+
 	/**
 	 * @avi:
 	 *
@@ -1561,6 +1712,12 @@ struct drm_connector_funcs {
 	 * the sysfs interfaces or on the kernel cmdline. In that case the
 	 * @detect callback isn't called.
 	 *
+	 * New drivers should implement
+	 * &drm_connector_helper_funcs.force_ctx instead, which gets passed
+	 * the modeset acquire context and thus allows taking additional
+	 * locks.  It takes precedence over this callback when both are
+	 * implemented.
+	 *
 	 * FIXME:
 	 *
 	 * Note that this hook is only called by the probe helper. It's not in
@@ -2052,7 +2209,38 @@ struct drm_connector_hdmi {
 	unsigned long supported_formats;
 
 	/**
-	 * @funcs: HDMI connector Control Functions
+	 * @max_tmds_char_rate: Maximum TMDS character rate, in Hz,
+	 * supported by the controller.
+	 *
+	 * This is inferred from &drm_connector_hdmi_funcs.supported_hdmi_ver,
+	 * by default. However, controllers may support a lower rate than that
+	 * specification version would imply. If that is the case, drivers are
+	 * expected to set &drm_connector_hdmi_funcs.supported_tmds_char_rate
+	 * to a non-zero value indicating the actual limit.
+	 */
+	unsigned long long max_tmds_char_rate;
+
+	/**
+	 * @scrambler_enabled: Tracks whether HDMI 2.0 scrambler is currently enabled.
+	 */
+	bool scrambler_enabled;
+
+	/**
+	 * @scdc_work: Work item currently used to monitor sink-side scrambling
+	 * status and retry setup if the sink resets it.
+	 */
+	struct delayed_work scdc_work;
+
+	/**
+	 * @scdc_work_initialized: Tracks whether @scdc_work has been set up via
+	 * INIT_DELAYED_WORK(). The work item is initialized lazily on the first
+	 * scrambling enable, so this guards the teardown paths against touching
+	 * an uninitialized work item.
+	 */
+	bool scdc_work_initialized;
+
+	/**
+	 * @funcs: HDMI connector Control Functions and controller capabilities
 	 */
 	const struct drm_connector_hdmi_funcs *funcs;
 
@@ -2550,6 +2738,12 @@ int drmm_connector_init(struct drm_device *dev,
 			int connector_type,
 			struct i2c_adapter *ddc);
 int drmm_connector_hdmi_init(struct drm_device *dev,
+			     struct drm_connector *connector,
+			     const struct drm_connector_funcs *funcs,
+			     const struct drm_connector_hdmi_funcs *hdmi_funcs,
+			     int connector_type,
+			     struct i2c_adapter *ddc);
+int drmm_connector_hdmi_ini2(struct drm_device *dev,
 			     struct drm_connector *connector,
 			     const char *vendor, const char *product,
 			     const struct drm_connector_funcs *funcs,
@@ -2636,6 +2830,24 @@ drm_connector_is_unregistered(struct drm_connector *connector)
 		DRM_CONNECTOR_UNREGISTERED;
 }
 
+/**
+ * drm_connector_hdmi_scrambler_supported - does connector support HDMI
+ * scrambling?
+ * @connector: DRM connector
+ *
+ * Checks whether or not @connector driver supports HDMI scrambling, based on
+ * the HDMI version advertised by the controller.
+ *
+ * Returns:
+ * True if the connector supports scrambling, false otherwise.
+ */
+static inline bool
+drm_connector_hdmi_scrambler_supported(const struct drm_connector *connector)
+{
+	return connector->hdmi.funcs &&
+		connector->hdmi.funcs->supported_hdmi_ver >= HDMI_VERSION_2_0;
+}
+
 void drm_connector_oob_hotplug_event(struct fwnode_handle *connector_fwnode,
 				     enum drm_connector_status status);
 enum drm_connector_color_format
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index 152349f973e336..be5dca538d87ed 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -423,18 +423,6 @@ struct drm_crtc_state {
  * bus accessors.
  */
 struct drm_crtc_funcs {
-	/**
-	 * @reset:
-	 *
-	 * Reset CRTC hardware and software state to off. This function isn't
-	 * called by the core directly, only through drm_mode_config_reset().
-	 * It's not a helper hook only for historical reasons.
-	 *
-	 * Atomic drivers can use drm_atomic_helper_crtc_reset() to reset
-	 * atomic state using this hook.
-	 */
-	void (*reset)(struct drm_crtc *crtc);
-
 	/**
 	 * @cursor_set:
 	 *
diff --git a/include/drm/drm_device.h b/include/drm/drm_device.h
index 768a8dae83c523..75f030d027ee4c 100644
--- a/include/drm/drm_device.h
+++ b/include/drm/drm_device.h
@@ -37,6 +37,7 @@ struct pci_controller;
 #define DRM_WEDGE_RECOVERY_REBIND	BIT(1)	/* unbind + bind driver */
 #define DRM_WEDGE_RECOVERY_BUS_RESET	BIT(2)	/* unbind + reset bus device + bind */
 #define DRM_WEDGE_RECOVERY_VENDOR	BIT(3)	/* vendor specific recovery method */
+#define DRM_WEDGE_RECOVERY_COLD_RESET	BIT(4)	/* remove device + slot power cycle + rescan */
 
 /**
  * struct drm_wedge_task_info - information about the guilty task of a wedge dev
diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h
index 04f7a7f1f10826..a2617aa34edfc4 100644
--- a/include/drm/drm_edid.h
+++ b/include/drm/drm_edid.h
@@ -490,5 +490,7 @@ u32 drm_edid_get_panel_id(const struct drm_edid *drm_edid);
 bool drm_edid_match(const struct drm_edid *drm_edid,
 		    const struct drm_edid_ident *ident);
 bool drm_edid_has_quirk(struct drm_connector *connector, enum drm_edid_quirk quirk);
+int drm_edid_detect_panel_size(const struct drm_edid *drm_edid,
+			       unsigned int *width, unsigned int *height);
 
 #endif /* __DRM_EDID_H__ */
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index 885244e375d329..ed329408acbea2 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -227,6 +227,17 @@ struct drm_gem_object_funcs {
 	 */
 	size_t (*rss)(struct drm_gem_object *obj);
 
+	/**
+	 * @handle_free:
+	 *
+	 * This callback is called when the GEM handle count goes down to 0.
+	 * It is currently used by AMDGPU driver to release their exported BO
+	 * handles.
+	 *
+	 * This callback is optional.
+	 */
+	void (*handle_free)(struct drm_gem_object *obj);
+
 	/**
 	 * @vm_ops:
 	 *
@@ -616,8 +627,7 @@ drm_gem_lru_scan(struct drm_device *dev,
 		 struct drm_gem_lru *lru,
 		 unsigned int nr_to_scan,
 		 unsigned long *remaining,
-		 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
-		 struct ww_acquire_ctx *ticket);
+		 bool (*shrink)(struct drm_gem_object *obj));
 
 int drm_gem_evict_locked(struct drm_gem_object *obj);
 
diff --git a/include/drm/drm_gem_atomic_helper.h b/include/drm/drm_gem_atomic_helper.h
index 3e01c619a25e01..d41877f77b7746 100644
--- a/include/drm/drm_gem_atomic_helper.h
+++ b/include/drm/drm_gem_atomic_helper.h
@@ -91,10 +91,10 @@ to_drm_shadow_plane_state(struct drm_plane_state *state)
 void __drm_gem_duplicate_shadow_plane_state(struct drm_plane *plane,
 					    struct drm_shadow_plane_state *new_shadow_plane_state);
 void __drm_gem_destroy_shadow_plane_state(struct drm_shadow_plane_state *shadow_plane_state);
-void __drm_gem_reset_shadow_plane(struct drm_plane *plane,
-				  struct drm_shadow_plane_state *shadow_plane_state);
 
-void drm_gem_reset_shadow_plane(struct drm_plane *plane);
+void __drm_gem_shadow_plane_state_init(struct drm_plane *plane,
+				       struct drm_shadow_plane_state *shadow_plane_state);
+struct drm_plane_state *drm_gem_create_shadow_plane_state(struct drm_plane *plane);
 struct drm_plane_state *drm_gem_duplicate_shadow_plane_state(struct drm_plane *plane);
 void drm_gem_destroy_shadow_plane_state(struct drm_plane *plane,
 					struct drm_plane_state *plane_state);
@@ -107,7 +107,7 @@ void drm_gem_destroy_shadow_plane_state(struct drm_plane *plane,
  * macro initializes struct drm_plane_funcs to use the rsp helper functions.
  */
 #define DRM_GEM_SHADOW_PLANE_FUNCS \
-	.reset = drm_gem_reset_shadow_plane, \
+	.atomic_create_state = drm_gem_create_shadow_plane_state, \
 	.atomic_duplicate_state = drm_gem_duplicate_shadow_plane_state, \
 	.atomic_destroy_state = drm_gem_destroy_shadow_plane_state
 
@@ -130,7 +130,8 @@ int drm_gem_simple_kms_begin_shadow_fb_access(struct drm_simple_display_pipe *pi
 					      struct drm_plane_state *plane_state);
 void drm_gem_simple_kms_end_shadow_fb_access(struct drm_simple_display_pipe *pipe,
 					     struct drm_plane_state *plane_state);
-void drm_gem_simple_kms_reset_shadow_plane(struct drm_simple_display_pipe *pipe);
+struct drm_plane_state *
+drm_gem_simple_kms_create_shadow_plane_state(struct drm_simple_display_pipe *pipe);
 struct drm_plane_state *
 drm_gem_simple_kms_duplicate_shadow_plane_state(struct drm_simple_display_pipe *pipe);
 void drm_gem_simple_kms_destroy_shadow_plane_state(struct drm_simple_display_pipe *pipe,
@@ -147,7 +148,7 @@ void drm_gem_simple_kms_destroy_shadow_plane_state(struct drm_simple_display_pip
 #define DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS \
 	.begin_fb_access = drm_gem_simple_kms_begin_shadow_fb_access, \
 	.end_fb_access = drm_gem_simple_kms_end_shadow_fb_access, \
-	.reset_plane = drm_gem_simple_kms_reset_shadow_plane, \
+	.create_plane_state = drm_gem_simple_kms_create_shadow_plane_state, \
 	.duplicate_plane_state = drm_gem_simple_kms_duplicate_shadow_plane_state, \
 	.destroy_plane_state = drm_gem_simple_kms_destroy_shadow_plane_state
 
diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h
index b7d987bf76aaaf..9e35584812fd8d 100644
--- a/include/drm/drm_gpusvm.h
+++ b/include/drm/drm_gpusvm.h
@@ -10,6 +10,7 @@
 #include <linux/kref.h>
 #include <linux/interval_tree.h>
 #include <linux/mmu_notifier.h>
+#include <drm/drm_pagemap.h>
 
 struct dev_pagemap_ops;
 struct drm_device;
@@ -18,7 +19,6 @@ struct drm_gpusvm_notifier;
 struct drm_gpusvm_ops;
 struct drm_gpusvm_range;
 struct drm_pagemap;
-struct drm_pagemap_addr;
 
 /**
  * struct drm_gpusvm_ops - Operations structure for GPU SVM
@@ -112,6 +112,7 @@ struct drm_gpusvm_notifier {
  * @unmapped: Flag indicating if the pages has been unmapped
  * @has_devmem_pages: Flag indicating if the pages has devmem pages
  * @has_dma_mapping: Flag indicating if the pages has a DMA mapping
+ * @inline_dma_mapping: Flag indicating if the pages have an inline DMA mapping
  * @__flags: Flags for pages in u16 form (used for READ_ONCE)
  */
 struct drm_gpusvm_pages_flags {
@@ -121,6 +122,7 @@ struct drm_gpusvm_pages_flags {
 			u16 unmapped : 1;
 			u16 has_devmem_pages : 1;
 			u16 has_dma_mapping : 1;
+			u16 inline_dma_mapping : 1;
 		};
 		u16 __flags;
 	};
@@ -130,17 +132,27 @@ struct drm_gpusvm_pages_flags {
  * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages
  *
  * @drm: The DRM device that owns the dma mappings
- * @dma_addr: Device address array
+ * @dma_addr: Device address array, valid while @flags.inline_dma_mapping is
+ *            not set
+ * @inline_addr: Device address inline address, valid while
+ *               @flags.inline_dma_mapping is set
  * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping.
  *            Note this is assuming only one drm_pagemap per range is allowed.
  * @state: DMA IOVA state for mapping.
  * @state_offset: DMA IOVA offset for mapping.
  * @notifier_seq: Notifier sequence number of the range's pages
  * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags
+ *
+ * @dma_addr and @inline_addr share storage, discriminated by
+ * @flags.inline_dma_mapping. Driver should use drm_gpusvm_pages_first_dma()
+ * to access the correct DMA address.
  */
 struct drm_gpusvm_pages {
 	struct drm_device *drm;
-	struct drm_pagemap_addr *dma_addr;
+	union {
+		struct drm_pagemap_addr *dma_addr;
+		struct drm_pagemap_addr inline_addr;
+	};
 	struct drm_pagemap *dpagemap;
 	struct dma_iova_state state;
 	unsigned long state_offset;
@@ -254,6 +266,14 @@ struct drm_gpusvm {
  * @allow_mixed: Allow mixed mappings in get pages. Mixing between system and
  *               single dpagemap is supported, mixing between multiple dpagemap
  *               is unsupported.
+ * @no_dma_map: Only fault the CPU pages for the range; skip the device DMA
+ *              mapping step. Used by drivers that consume the faulted pages
+ *              without needing a DMA mapping. In this mode @drm on the
+ *              drm_gpusvm_pages is not required, no mapping state is recorded
+ *              and drm_gpusvm_pages_valid() therefore never reports these
+ *              pages as valid; the caller revalidates the snapshot itself, see
+ *              drm_gpusvm_get_pages(). @devmem_only is rejected and no page
+ *              type check is performed, so @allow_mixed has no effect.
  *
  * Context that is DRM GPUSVM is operating in (i.e. user arguments).
  */
@@ -266,6 +286,7 @@ struct drm_gpusvm_ctx {
 	unsigned int devmem_possible :1;
 	unsigned int devmem_only :1;
 	unsigned int allow_mixed :1;
+	unsigned int no_dma_map :1;
 };
 
 int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
@@ -324,6 +345,7 @@ void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
 
 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 			 struct drm_gpusvm_pages *svm_pages,
+			 unsigned int num_pages,
 			 struct mm_struct *mm,
 			 struct mmu_interval_notifier *notifier,
 			 unsigned long pages_start, unsigned long pages_end,
@@ -355,6 +377,49 @@ static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages,
 	svm_pages->notifier_seq = LONG_MAX;
 }
 
+/**
+ * drm_gpusvm_pages_first_dma() - Resolve the device address array
+ * @svm_pages: Pointer to the drm_gpusvm_pages.
+ * @contiguous: Where to store whether one entry spans the whole range, or NULL
+ *
+ * drm_gpusvm_pages use unions to optimize the storage of DMA addresses,
+ * this function abstracts the access to the first device address. The driver
+ * should use this helper instead of reading dma_addr directly to prevent
+ * array out of bounds access.
+ *
+ * @contiguous comes from the same read of the flags as the array itself, so a
+ * caller cannot see the two disagree and walk past that single entry into the
+ * fields behind it. When it is set the length comes from the range rather than
+ * from the order. The order still states what one PTE may cover: the range
+ * length for a huge page, PAGE_SIZE for an IOVA mapped range of single pages.
+ *
+ * Only get_pages() and the free path switch between the two union members.
+ * Both hold the notifier lock for read, so taking that lock does not stop
+ * them; callers need the driver lock that does, which every reader of the
+ * addresses holds anyway. The notifier never touches the union, so the
+ * pointer returned here stays good and can then be used under the notifier
+ * lock.
+ *
+ * Return: Pointer to the first device address, NULL if none is populated.
+ */
+static inline const struct drm_pagemap_addr *
+drm_gpusvm_pages_first_dma(const struct drm_gpusvm_pages *svm_pages,
+			   bool *contiguous)
+{
+	struct drm_gpusvm_pages_flags flags = {
+		/* READ_ONCE pairs with the WRITE_ONCE of the flag writers */
+		.__flags = READ_ONCE(svm_pages->flags.__flags),
+	};
+
+	if (contiguous)
+		*contiguous = flags.inline_dma_mapping;
+
+	if (flags.inline_dma_mapping)
+		return &svm_pages->inline_addr;
+
+	return READ_ONCE(svm_pages->dma_addr);
+}
+
 /**
  * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function.
  * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible.
diff --git a/include/drm/drm_mipi_dbi.h b/include/drm/drm_mipi_dbi.h
index 78ab23d844ce91..d93e25fdc2e1e7 100644
--- a/include/drm/drm_mipi_dbi.h
+++ b/include/drm/drm_mipi_dbi.h
@@ -234,7 +234,7 @@ void drm_mipi_dbi_plane_helper_atomic_update(struct drm_plane *plane,
  */
 
 #define DRM_MIPI_DBI_CRTC_FUNCS \
-	.reset = drm_atomic_helper_crtc_reset, \
+	.atomic_create_state = drm_atomic_helper_crtc_create_state, \
 	.set_config = drm_atomic_helper_set_config, \
 	.page_flip = drm_atomic_helper_page_flip, \
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, \
diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h
index d8f5b7e9673ee1..084517beb92459 100644
--- a/include/drm/drm_mode_config.h
+++ b/include/drm/drm_mode_config.h
@@ -495,8 +495,8 @@ struct drm_mode_config {
 	 *
 	 * Raw spinlock used to protect critical sections of code that access
 	 * the display hardware or modeset software state, which the panic
-	 * printing code must be protected against. See drm_panic_trylock(),
-	 * drm_panic_lock() and drm_panic_unlock().
+	 * printing code must be protected against. See the documentation on
+	 * DRM panic handling for more information.
 	 */
 	struct raw_spinlock panic_lock;
 
diff --git a/include/drm/drm_modeset_helper_vtables.h b/include/drm/drm_modeset_helper_vtables.h
index ca6268945c28d3..60b30cc2694043 100644
--- a/include/drm/drm_modeset_helper_vtables.h
+++ b/include/drm/drm_modeset_helper_vtables.h
@@ -281,6 +281,20 @@ struct drm_crtc_helper_funcs {
 	 */
 	void (*disable)(struct drm_crtc *crtc);
 
+	/**
+	 * @hw_reset:
+	 *
+	 * Optional hook for CRTC hardware reset.
+	 *
+	 * Unlike @drm_crtc_funcs.reset, which both resets hardware and
+	 * creates new software state, this hook only resets the
+	 * hardware to a known good state without touching the software
+	 * state at all.
+	 *
+	 * This hook is called by drm_mode_config_reset().
+	 */
+	void (*hw_reset)(struct drm_crtc *crtc);
+
 	/**
 	 * @atomic_check:
 	 *
@@ -921,6 +935,34 @@ struct drm_connector_helper_funcs {
 			  struct drm_modeset_acquire_ctx *ctx,
 			  bool force);
 
+	/**
+	 * @force_ctx:
+	 *
+	 * This function is called to update internal encoder state when the
+	 * connector is forced to a certain state by userspace, either through
+	 * the sysfs interfaces or on the kernel cmdline. In that case the
+	 * @detect_ctx callback isn't called.
+	 *
+	 * This is the atomic version of &drm_connector_funcs.force.  When
+	 * both are implemented, this one takes precedence and
+	 * &drm_connector_funcs.force is not called.
+	 *
+	 * To avoid races against concurrent connector state updates, the
+	 * helper libraries always call this with ctx set to a valid context,
+	 * and &drm_mode_config.connection_mutex will always be locked with
+	 * the ctx parameter set to this ctx. This allows taking additional
+	 * locks as required.
+	 *
+	 * RETURNS:
+	 *
+	 * Zero on success, or a negative error code otherwise.  This includes
+	 * -EDEADLK when a lock acquisition needs to be restarted, in which
+	 * case the helpers take care of backing off and calling this function
+	 * again.
+	 */
+	int (*force_ctx)(struct drm_connector *connector,
+			 struct drm_modeset_acquire_ctx *ctx);
+
 	/**
 	 * @mode_valid:
 	 *
@@ -1442,17 +1484,20 @@ struct drm_plane_helper_funcs {
 	/**
 	 * @get_scanout_buffer:
 	 *
-	 * Get the current scanout buffer, to display a message with drm_panic.
+	 * Get the current scanout buffer to display a message with drm_panic.
 	 * The driver should do the minimum changes to provide a buffer,
 	 * that can be used to display the panic screen. Currently only linear
 	 * buffers are supported. Non-linear buffer support is on the TODO list.
 	 * The device &dev.mode_config.panic_lock is taken before calling this
 	 * function, so you can safely access the &plane.state
 	 * It is called from a panic callback, and must follow its restrictions.
-	 * Please look the documentation at drm_panic_trylock() for an in-depth
+	 * Please look the documentation on DRM panic handling for an in-depth
 	 * discussions of what's safe and what is not allowed.
+	 *
 	 * It's a best effort mode, so it's expected that in some complex cases
-	 * the panic screen won't be displayed.
+	 * the panic screen won't be displayed. Drivers must not make any
+	 * assumptions about the actual state of the hardware.
+	 *
 	 * The returned &drm_scanout_buffer.map must be valid if no error code is
 	 * returned.
 	 *
@@ -1472,7 +1517,7 @@ struct drm_plane_helper_funcs {
 	 * It is only called if get_scanout_buffer() returned successfully, and
 	 * the &dev.mode_config.panic_lock is held during the entire sequence.
 	 * It is called from a panic callback, and must follow its restrictions.
-	 * Please look the documentation at drm_panic_trylock() for an in-depth
+	 * Please look the documentation on DRM panic handling for an in-depth
 	 * discussions of what's safe and what is not allowed.
 	 */
 	void (*panic_flush)(struct drm_plane *plane);
diff --git a/include/drm/drm_of.h b/include/drm/drm_of.h
index ebebed14c611e3..002ba1b18eb1ee 100644
--- a/include/drm/drm_of.h
+++ b/include/drm/drm_of.h
@@ -7,6 +7,7 @@
 #if IS_ENABLED(CONFIG_OF) && IS_ENABLED(CONFIG_DRM_PANEL_BRIDGE)
 #include <linux/of.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_panel.h>
 #endif
 
 struct component_master_ops;
@@ -14,8 +15,6 @@ struct component_match;
 struct device;
 struct drm_device;
 struct drm_encoder;
-struct drm_panel;
-struct drm_bridge;
 struct device_node;
 struct mipi_dsi_device_info;
 struct mipi_dsi_host;
@@ -51,10 +50,6 @@ int drm_of_encoder_active_endpoint(struct device_node *node,
 				   struct of_endpoint *endpoint);
 int drm_of_get_panel_orientation(const struct device_node *np,
 				 enum drm_panel_orientation *orientation);
-int drm_of_find_panel_or_bridge(const struct device_node *np,
-				int port, int endpoint,
-				struct drm_panel **panel,
-				struct drm_bridge **bridge);
 int drm_of_lvds_get_dual_link_pixel_order(const struct device_node *port1,
 					  const struct device_node *port2);
 int drm_of_lvds_get_dual_link_pixel_order_sink(struct device_node *port1,
@@ -112,14 +107,6 @@ static inline int drm_of_get_panel_orientation(const struct device_node *np,
 	return -EINVAL;
 }
 
-static inline int drm_of_find_panel_or_bridge(const struct device_node *np,
-					      int port, int endpoint,
-					      struct drm_panel **panel,
-					      struct drm_bridge **bridge)
-{
-	return -EINVAL;
-}
-
 static inline int
 drm_of_lvds_get_dual_link_pixel_order(const struct device_node *port1,
 				      const struct device_node *port2)
diff --git a/include/drm/drm_panel.h b/include/drm/drm_panel.h
index b87323443f49c9..58113ea7d63a07 100644
--- a/include/drm/drm_panel.h
+++ b/include/drm/drm_panel.h
@@ -30,10 +30,14 @@
 #include <linux/mutex.h>
 #include <linux/kref.h>
 
+#include <drm/drm_bridge.h>
+
 struct backlight_device;
 struct dentry;
 struct device_node;
+struct drm_bridge;
 struct drm_connector;
+struct drm_device;
 struct drm_panel_follower;
 struct drm_panel;
 struct display_timing;
@@ -228,6 +232,22 @@ struct drm_panel {
 	 */
 	const struct drm_panel_funcs *funcs;
 
+	/**
+	 * @bridge:
+	 *
+	 * Bridge to access the panel features through the drm_bridge API.
+	 */
+	struct drm_bridge bridge;
+
+	/**
+	 * @connector:
+	 *
+	 * Connector instantiated by the bridge (only for legacy code not
+	 * yet using the drm_bridge_connector and
+	 * DRM_BRIDGE_ATTACH_NO_CONNECTOR).
+	 */
+	struct drm_connector connector;
+
 	/**
 	 * @connector_type:
 	 *
@@ -281,17 +301,6 @@ struct drm_panel {
 	 * If true then the panel has been enabled.
 	 */
 	bool enabled;
-
-	/**
-	 * @container: Pointer to the private driver struct embedding this
-	 * @struct drm_panel.
-	 */
-	void *container;
-
-	/**
-	 * @refcount: reference count of users referencing this panel.
-	 */
-	struct kref refcount;
 };
 
 void *__devm_drm_panel_alloc(struct device *dev, size_t size, size_t offset,
@@ -335,16 +344,28 @@ void drm_panel_disable(struct drm_panel *panel);
 
 int drm_panel_get_modes(struct drm_panel *panel, struct drm_connector *connector);
 
-#if defined(CONFIG_OF) && defined(CONFIG_DRM_PANEL)
+#if defined(CONFIG_OF) && IS_ENABLED(CONFIG_DRM_PANEL)
 struct drm_panel *of_drm_find_panel(const struct device_node *np);
+int drm_of_find_panel_or_bridge(const struct device_node *np,
+				int port, int endpoint,
+				struct drm_panel **panel,
+				struct drm_bridge **bridge);
 #else
 static inline struct drm_panel *of_drm_find_panel(const struct device_node *np)
 {
 	return ERR_PTR(-ENODEV);
 }
+
+static inline int drm_of_find_panel_or_bridge(const struct device_node *np,
+					      int port, int endpoint,
+					      struct drm_panel **panel,
+					      struct drm_bridge **bridge)
+{
+	return -EINVAL;
+}
 #endif
 
-#if defined(CONFIG_DRM_PANEL)
+#if IS_ENABLED(CONFIG_DRM_PANEL)
 bool drm_is_panel_follower(struct device *dev);
 int drm_panel_add_follower(struct device *follower_dev,
 			   struct drm_panel_follower *follower);
@@ -381,4 +402,56 @@ static inline int drm_panel_of_backlight(struct drm_panel *panel)
 }
 #endif
 
+#ifdef CONFIG_DRM_PANEL_BRIDGE
+bool drm_bridge_is_panel(const struct drm_bridge *bridge);
+struct drm_bridge *drm_panel_bridge_add(struct drm_panel *panel);
+struct drm_bridge *drm_panel_bridge_add_typed(struct drm_panel *panel,
+					      u32 connector_type);
+void drm_panel_bridge_remove(struct drm_bridge *bridge);
+int drm_panel_bridge_set_orientation(struct drm_connector *connector,
+				     struct drm_bridge *bridge);
+struct drm_bridge *devm_drm_panel_bridge_add(struct device *dev,
+					     struct drm_panel *panel);
+struct drm_bridge *devm_drm_panel_bridge_add_typed(struct device *dev,
+						   struct drm_panel *panel,
+						   u32 connector_type);
+struct drm_bridge *drmm_panel_bridge_add(struct drm_device *drm,
+					 struct drm_panel *panel);
+struct drm_connector *drm_panel_bridge_connector(struct drm_bridge *bridge);
+#else
+static inline bool drm_bridge_is_panel(const struct drm_bridge *bridge)
+{
+	return false;
+}
+
+static inline int drm_panel_bridge_set_orientation(struct drm_connector *connector,
+						   struct drm_bridge *bridge)
+{
+	return -EINVAL;
+}
+#endif
+
+#if defined(CONFIG_OF) && defined(CONFIG_DRM_PANEL_BRIDGE)
+struct drm_bridge *devm_drm_of_get_bridge(struct device *dev, struct device_node *node,
+					  u32 port, u32 endpoint);
+struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm, struct device_node *node,
+				      u32 port, u32 endpoint);
+#else
+static inline struct drm_bridge *devm_drm_of_get_bridge(struct device *dev,
+							struct device_node *node,
+							u32 port,
+							u32 endpoint)
+{
+	return ERR_PTR(-ENODEV);
+}
+
+static inline struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm,
+						    struct device_node *node,
+						    u32 port,
+						    u32 endpoint)
+{
+	return ERR_PTR(-ENODEV);
+}
+#endif
+
 #endif
diff --git a/include/drm/drm_panic.h b/include/drm/drm_panic.h
index ac0e46b73436e9..3e628cbaffca64 100644
--- a/include/drm/drm_panic.h
+++ b/include/drm/drm_panic.h
@@ -8,13 +8,19 @@
 #ifndef __DRM_PANIC_H__
 #define __DRM_PANIC_H__
 
-#include <linux/module.h>
 #include <linux/types.h>
 #include <linux/iosys-map.h>
 
-#include <drm/drm_device.h>
 #include <drm/drm_fourcc.h>
 
+struct page;
+
+enum drm_panic_type {
+	DRM_PANIC_TYPE_KMSG,
+	DRM_PANIC_TYPE_USER,
+	DRM_PANIC_TYPE_QR,
+};
+
 /**
  * struct drm_scanout_buffer - DRM scanout buffer
  *
@@ -77,113 +83,6 @@ struct drm_scanout_buffer {
 	 * set_pixel()
 	 */
 	void *private;
-
 };
 
-#ifdef CONFIG_DRM_PANIC
-
-/**
- * drm_panic_trylock - try to enter the panic printing critical section
- * @dev: struct drm_device
- * @flags: unsigned long irq flags you need to pass to the unlock() counterpart
- *
- * This function must be called by any panic printing code. The panic printing
- * attempt must be aborted if the trylock fails.
- *
- * Panic printing code can make the following assumptions while holding the
- * panic lock:
- *
- * - Anything protected by drm_panic_lock() and drm_panic_unlock() pairs is safe
- *   to access.
- *
- * - Furthermore the panic printing code only registers in drm_dev_unregister()
- *   and gets removed in drm_dev_unregister(). This allows the panic code to
- *   safely access any state which is invariant in between these two function
- *   calls, like the list of planes &drm_mode_config.plane_list or most of the
- *   struct drm_plane structure.
- *
- * Specifically thanks to the protection around plane updates in
- * drm_atomic_helper_swap_state() the following additional guarantees hold:
- *
- * - It is safe to deference the drm_plane.state pointer.
- *
- * - Anything in struct drm_plane_state or the driver's subclass thereof which
- *   stays invariant after the atomic check code has finished is safe to access.
- *   Specifically this includes the reference counted pointers to framebuffer
- *   and buffer objects.
- *
- * - Anything set up by &drm_plane_helper_funcs.fb_prepare and cleaned up
- *   &drm_plane_helper_funcs.fb_cleanup is safe to access, as long as it stays
- *   invariant between these two calls. This also means that for drivers using
- *   dynamic buffer management the framebuffer is pinned, and therefer all
- *   relevant datastructures can be accessed without taking any further locks
- *   (which would be impossible in panic context anyway).
- *
- * - Importantly, software and hardware state set up by
- *   &drm_plane_helper_funcs.begin_fb_access and
- *   &drm_plane_helper_funcs.end_fb_access is not safe to access.
- *
- * Drivers must not make any assumptions about the actual state of the hardware,
- * unless they explicitly protected these hardware access with drm_panic_lock()
- * and drm_panic_unlock().
- *
- * Return:
- * %0 when failing to acquire the raw spinlock, nonzero on success.
- */
-#define drm_panic_trylock(dev, flags) \
-	raw_spin_trylock_irqsave(&(dev)->mode_config.panic_lock, flags)
-
-/**
- * drm_panic_lock - protect panic printing relevant state
- * @dev: struct drm_device
- * @flags: unsigned long irq flags you need to pass to the unlock() counterpart
- *
- * This function must be called to protect software and hardware state that the
- * panic printing code must be able to rely on. The protected sections must be
- * as small as possible. It uses the irqsave/irqrestore variant, and can be
- * called from irq handler. Examples include:
- *
- * - Access to peek/poke or other similar registers, if that is the way the
- *   driver prints the pixels into the scanout buffer at panic time.
- *
- * - Updates to pointers like &drm_plane.state, allowing the panic handler to
- *   safely deference these. This is done in drm_atomic_helper_swap_state().
- *
- * - An state that isn't invariant and that the driver must be able to access
- *   during panic printing.
- */
-
-#define drm_panic_lock(dev, flags) \
-	raw_spin_lock_irqsave(&(dev)->mode_config.panic_lock, flags)
-
-/**
- * drm_panic_unlock - end of the panic printing critical section
- * @dev: struct drm_device
- * @flags: irq flags that were returned when acquiring the lock
- *
- * Unlocks the raw spinlock acquired by either drm_panic_lock() or
- * drm_panic_trylock().
- */
-#define drm_panic_unlock(dev, flags) \
-	raw_spin_unlock_irqrestore(&(dev)->mode_config.panic_lock, flags)
-
-#else
-
-static inline bool drm_panic_trylock(struct drm_device *dev, unsigned long flags)
-{
-	return true;
-}
-
-static inline void drm_panic_lock(struct drm_device *dev, unsigned long flags) {}
-static inline void drm_panic_unlock(struct drm_device *dev, unsigned long flags) {}
-
-#endif
-
-#if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
-size_t drm_panic_qr_max_data_size(u8 version, size_t url_len);
-
-u8 drm_panic_qr_generate(const char *url, u8 *data, size_t data_len, size_t data_size,
-			 u8 *tmp, size_t tmp_size);
-#endif
-
 #endif /* __DRM_PANIC_H__ */
diff --git a/include/drm/drm_panic_helper.h b/include/drm/drm_panic_helper.h
new file mode 100644
index 00000000000000..fbfe51161aef4b
--- /dev/null
+++ b/include/drm/drm_panic_helper.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+
+#ifndef __DRM_DRM_PANIC_HELPER_H_
+#define __DRM_DRM_PANIC_HELPER_H_
+
+#include <linux/types.h>
+
+enum drm_panic_type;
+
+struct drm_plane;
+struct drm_scanout_buffer;
+
+/* drm_panic_helper.c */
+int drm_plane_helper_display_panic_screen(struct drm_plane *plane,
+					  const char *description,
+					  enum drm_panic_type panic_type,
+					  u32 fg_color, u32 bg_color,
+					  unsigned int qr_version);
+#if IS_ENABLED(CONFIG_KUNIT)
+int drm_panic_helper_draw_screen_user(struct drm_scanout_buffer *sb, u32 fg_color, u32 bg_color);
+int drm_panic_helper_draw_screen_kmsg(struct drm_scanout_buffer *sb, u32 fg_color, u32 bg_color);
+int drm_panic_helper_draw_screen_qr_code(struct drm_scanout_buffer *sb,
+					 u32 fg_color, u32 bg_color, unsigned int qr_version);
+void drm_panic_helper_set_description(const char *description);
+void drm_panic_helper_clear_description(void);
+#endif
+
+/* drm_panic_helper_qr.rs */
+size_t drm_panic_helper_qr_max_data_size(u8 version, size_t url_len);
+u8 drm_panic_helper_qr_generate(const char *url, u8 *data, size_t data_len, size_t data_size,
+				u8 *tmp, size_t tmp_size);
+
+#if IS_ENABLED(CONFIG_DRM_PANIC)
+#define DRM_PANIC_PLANE_FUNCS \
+	.display_panic_screen = drm_plane_helper_display_panic_screen
+#else
+#define DRM_PANIC_PLANE_FUNCS \
+	.display_panic_screen = NULL
+#endif
+
+#endif
diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h
index 2c5a5a70a71b81..410d7b02b77a49 100644
--- a/include/drm/drm_plane.h
+++ b/include/drm/drm_plane.h
@@ -32,6 +32,8 @@
 #include <drm/drm_modeset_lock.h>
 #include <drm/drm_util.h>
 
+enum drm_panic_type;
+
 struct drm_crtc;
 struct drm_plane_size_hint;
 struct drm_printer;
@@ -359,18 +361,6 @@ struct drm_plane_funcs {
 	 */
 	void (*destroy)(struct drm_plane *plane);
 
-	/**
-	 * @reset:
-	 *
-	 * Reset plane hardware and software state to off. This function isn't
-	 * called by the core directly, only through drm_mode_config_reset().
-	 * It's not a helper hook only for historical reasons.
-	 *
-	 * Atomic drivers can use drm_atomic_helper_plane_reset() to reset
-	 * atomic state using this hook.
-	 */
-	void (*reset)(struct drm_plane *plane);
-
 	/**
 	 * @set_property:
 	 *
@@ -590,6 +580,61 @@ struct drm_plane_funcs {
 	bool (*format_mod_supported_async)(struct drm_plane *plane,
 					   u32 format, u64 modifier);
 
+	/**
+	 * @display_panic_screen:
+	 *
+	 * DRM panic handling invokes this callback on panics.
+	 *
+	 * It is a panic handler, so it can't take locks, allocate memory, run tasks/irq,
+	 * or attempt to sleep. It's a best effort, and it may not be able to display
+	 * the message in all situations (like if the panic occurs in the middle of a
+	 * modesetting).
+	 *
+	 * Panic-printing code must acquire the panic lock before interacting with the
+	 * scanout buffer. Such helpers can make the following assumptions while holding
+	 * the panic lock:
+	 *
+	 * - Anything protected by drm_panic_lock() and drm_panic_unlock() pairs is safe
+	 *   to access.
+	 *
+	 * - Furthermore the panic printing code only registers in drm_dev_unregister()
+	 *   and gets removed in drm_dev_unregister(). This allows the panic code to
+	 *   safely access any state which is invariant in between these two function
+	 *   calls, like the list of planes &drm_mode_config.plane_list or most of the
+	 *   struct drm_plane structure.
+	 *
+	 * Specifically thanks to the protection around plane updates in
+	 * drm_atomic_helper_swap_state() the following additional guarantees hold:
+	 *
+	 * - It is safe to deference the drm_plane.state pointer.
+	 *
+	 * - Anything in struct drm_plane_state or the driver's subclass thereof which
+	 *   stays invariant after the atomic check code has finished is safe to access.
+	 *   Specifically this includes the reference counted pointers to framebuffer
+	 *   and buffer objects.
+	 *
+	 * - Anything set up by &drm_plane_helper_funcs.fb_prepare and cleaned up
+	 *   &drm_plane_helper_funcs.fb_cleanup is safe to access, as long as it stays
+	 *   invariant between these two calls. This also means that for drivers using
+	 *   dynamic buffer management the framebuffer is pinned, and therefer all
+	 *   relevant datastructures can be accessed without taking any further locks
+	 *   (which would be impossible in panic context anyway).
+	 *
+	 * - Importantly, software and hardware state set up by
+	 *   &drm_plane_helper_funcs.begin_fb_access and
+	 *   &drm_plane_helper_funcs.end_fb_access is not safe to access.
+	 *
+	 * It will display only one static frame, so performance optimizations are
+	 * low priority as the machine is already in an unusable state.
+	 *
+	 * This callback is optional. Planes without will not be considered
+	 * for panic output.
+	 */
+	int (*display_panic_screen)(struct drm_plane *plane,
+				    const char *description,
+				    enum drm_panic_type panic_type,
+				    u32 fg_color, u32 bg_color,
+				    unsigned int qr_version);
 };
 
 /**
diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h
index 2adc5ac688e112..efa256ef01fc83 100644
--- a/include/drm/drm_print.h
+++ b/include/drm/drm_print.h
@@ -28,6 +28,7 @@
 
 #include <linux/compiler.h>
 #include <linux/printk.h>
+#include <linux/string_choices.h>
 #include <linux/device.h>
 #include <linux/dynamic_debug.h>
 
@@ -644,6 +645,9 @@ static inline struct device *__drm_to_dev(const struct drm_device *drm)
 #define drm_err_ratelimited(drm, fmt, ...)				\
 	__drm_printk((drm), err, _ratelimited, "*ERROR* " fmt, ##__VA_ARGS__)
 
+#define drm_warn_ratelimited(drm, fmt, ...)				\
+	__drm_printk((drm), warn, _ratelimited, fmt, ##__VA_ARGS__)
+
 
 #define drm_dbg_core(drm, fmt, ...)					\
 	drm_dev_dbg(__drm_to_dev(drm), DRM_UT_CORE, fmt, ##__VA_ARGS__)
diff --git a/include/drm/drm_ras.h b/include/drm/drm_ras.h
index 0beede3ddc4e4b..1765a3130b09b1 100644
--- a/include/drm/drm_ras.h
+++ b/include/drm/drm_ras.h
@@ -71,6 +71,35 @@ struct drm_ras_node {
 	 */
 	int (*clear_error_counter)(struct drm_ras_node *node, u32 error_id);
 
+	/**
+	 * @query_error_threshold:
+	 *
+	 * This callback is used by drm-ras to query error threshold of a
+	 * specific counter.
+	 *
+	 * Driver should expect query_error_threshold() to be called with
+	 * error_id from `error_counter_range.first` to
+	 * `error_counter_range.last`.
+	 *
+	 * Returns: 0 on success, negative error code on failure.
+	 */
+	int (*query_error_threshold)(struct drm_ras_node *node, u32 error_id, const char **name,
+				     u32 *threshold);
+
+	/**
+	 * @set_error_threshold:
+	 *
+	 * This callback is used by drm-ras to set error threshold of a specific
+	 * counter.
+	 *
+	 * Driver should expect set_error_threshold() to be called with error_id
+	 * from `error_counter_range.first` to `error_counter_range.last`.
+	 * Driver is responsible for error threshold bounds checking.
+	 *
+	 * Returns: 0 on success, negative error code on failure.
+	 */
+	int (*set_error_threshold)(struct drm_ras_node *node, u32 error_id, u32 threshold);
+
 	/** @priv: Driver private data */
 	void *priv;
 };
@@ -80,9 +109,14 @@ struct drm_device;
 #if IS_ENABLED(CONFIG_DRM_RAS)
 int drm_ras_node_register(struct drm_ras_node *node);
 void drm_ras_node_unregister(struct drm_ras_node *node);
+int drm_ras_nl_error_event(struct drm_ras_node *node, u32 error_id, const char *error_name,
+			   u32 value);
 #else
 static inline int drm_ras_node_register(struct drm_ras_node *node) { return 0; }
 static inline void drm_ras_node_unregister(struct drm_ras_node *node) { }
+static inline int drm_ras_nl_error_event(struct drm_ras_node *node, u32 error_id,
+					 const char *error_name, u32 value)
+{ return 0; }
 #endif
 
 #endif
diff --git a/include/drm/drm_simple_kms_helper.h b/include/drm/drm_simple_kms_helper.h
index cb672ce0e856d6..2b31c25c57a9c1 100644
--- a/include/drm/drm_simple_kms_helper.h
+++ b/include/drm/drm_simple_kms_helper.h
@@ -39,11 +39,10 @@ struct drm_simple_display_pipe_funcs {
 			      struct drm_plane_state *plane_state);
 	int (*enable_vblank)(struct drm_simple_display_pipe *pipe);
 	void (*disable_vblank)(struct drm_simple_display_pipe *pipe);
-	void (*reset_crtc)(struct drm_simple_display_pipe *pipe);
 	struct drm_crtc_state * (*duplicate_crtc_state)(struct drm_simple_display_pipe *pipe);
 	void (*destroy_crtc_state)(struct drm_simple_display_pipe *pipe,
 				   struct drm_crtc_state *crtc_state);
-	void (*reset_plane)(struct drm_simple_display_pipe *pipe);
+	struct drm_plane_state * (*create_plane_state)(struct drm_simple_display_pipe *pipe);
 	struct drm_plane_state * (*duplicate_plane_state)(struct drm_simple_display_pipe *pipe);
 	void (*destroy_plane_state)(struct drm_simple_display_pipe *pipe,
 				    struct drm_plane_state *plane_state);
@@ -68,10 +67,6 @@ int drm_simple_display_pipe_init(struct drm_device *dev,
 			const uint64_t *format_modifiers,
 			struct drm_connector *connector);
 
-int drm_simple_encoder_init(struct drm_device *dev,
-			    struct drm_encoder *encoder,
-			    int encoder_type);
-
 void *__drmm_simple_encoder_alloc(struct drm_device *dev, size_t size,
 				  size_t offset, int encoder_type);
 
diff --git a/include/drm/drm_sysfs.h b/include/drm/drm_sysfs.h
index 96a5d858404b07..7695873a945657 100644
--- a/include/drm/drm_sysfs.h
+++ b/include/drm/drm_sysfs.h
@@ -3,13 +3,9 @@
 #define _DRM_SYSFS_H_
 
 struct drm_device;
-struct device;
 struct drm_connector;
 struct drm_property;
 
-int drm_class_device_register(struct device *dev);
-void drm_class_device_unregister(struct device *dev);
-
 void drm_sysfs_hotplug_event(struct drm_device *dev);
 void drm_sysfs_connector_hotplug_event(struct drm_connector *connector);
 void drm_sysfs_connector_property_event(struct drm_connector *connector,
diff --git a/include/drm/intel/display_parent_interface.h b/include/drm/intel/display_parent_interface.h
index de395df9ca3015..f36134e89cb6d1 100644
--- a/include/drm/intel/display_parent_interface.h
+++ b/include/drm/intel/display_parent_interface.h
@@ -179,7 +179,7 @@ struct intel_display_pc8_interface {
 };
 
 struct intel_display_pcode_interface {
-	int (*read)(struct drm_device *drm, u32 mbox, u32 *val, u32 *val1);
+	int (*read)(struct drm_device *drm, u32 mbox, u32 *val0, u32 *val1);
 	int (*write)(struct drm_device *drm, u32 mbox, u32 val, int timeout_ms);
 	int (*request)(struct drm_device *drm, u32 mbox, u32 request,
 		       u32 reply_mask, u32 reply, int timeout_base_ms);
@@ -196,6 +196,7 @@ struct intel_display_rpm_interface {
 	void (*put_unchecked)(const struct drm_device *drm);
 
 	bool (*suspended)(const struct drm_device *drm);
+	bool (*pme_enabled)(const struct drm_device *drm); /* Optional */
 	void (*assert_held)(const struct drm_device *drm);
 	void (*assert_block)(const struct drm_device *drm);
 	void (*assert_unblock)(const struct drm_device *drm);
@@ -230,6 +231,10 @@ struct intel_display_vlv_iosf_interface {
 	int (*write)(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr, u32 val);
 };
 
+struct intel_display_wa_interface {
+	bool (*wa_16023588340)(struct drm_device *drm);
+};
+
 /**
  * struct intel_display_parent_interface - services parent driver provides to display
  *
@@ -291,6 +296,12 @@ struct intel_display_parent_interface {
 	/** @vlv_iosf: VLV IOSF sideband. Optional. */
 	const struct intel_display_vlv_iosf_interface *vlv_iosf;
 
+	/**
+	 * @wa: Display workarounds query. Use only for workarounds that require
+	 * information only available to the parent driver. Optional.
+	 */
+	const struct intel_display_wa_interface *wa;
+
 	/* Generic independent functions */
 	struct {
 		/** @fence_priority_display: Set display priority. Optional. */
@@ -302,6 +313,9 @@ struct intel_display_parent_interface {
 		/** @has_fenced_regions: Support legacy fencing? Optional. */
 		bool (*has_fenced_regions)(struct drm_device *drm);
 
+		/** @transient_data_flush: Transient data flush. Optional. */
+		void (*transient_data_flush)(struct drm_device *drm);
+
 		/** @vgpu_active: Is vGPU active? Optional. */
 		bool (*vgpu_active)(struct drm_device *drm);
 	};
diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h
index dff389b56eb3bf..32c88a690ad822 100644
--- a/include/drm/intel/pciids.h
+++ b/include/drm/intel/pciids.h
@@ -880,6 +880,7 @@
 	MACRO__(0xB08F, ## __VA_ARGS__), \
 	MACRO__(0xB090, ## __VA_ARGS__), \
 	MACRO__(0xB0A0, ## __VA_ARGS__), \
+	MACRO__(0xB0A1, ## __VA_ARGS__), \
 	MACRO__(0xB0B0, ## __VA_ARGS__)
 
 /* WCL */
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index 3f2812743a7eb5..714c533622c201 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -35,7 +35,7 @@
 #include <drm/ttm/ttm_kmap_iter.h>
 
 #define TTM_MAX_BO_PRIORITY	4U
-#define TTM_NUM_MEM_TYPES 9
+#define TTM_NUM_MEM_TYPES 10
 
 struct dentry;
 struct dmem_cgroup_device;
diff --git a/include/drm/ttm/ttm_tt.h b/include/drm/ttm/ttm_tt.h
index 406437ad674bf1..58c8bf9a6cbd91 100644
--- a/include/drm/ttm/ttm_tt.h
+++ b/include/drm/ttm/ttm_tt.h
@@ -27,7 +27,6 @@
 #ifndef _TTM_TT_H_
 #define _TTM_TT_H_
 
-#include <linux/pagemap.h>
 #include <linux/types.h>
 #include <drm/ttm/ttm_caching.h>
 #include <drm/ttm/ttm_kmap_iter.h>
diff --git a/include/dt-bindings/arm/qcom,ids.h b/include/dt-bindings/arm/qcom,ids.h
index 2cf433f61d5e7e..1352970275803b 100644
--- a/include/dt-bindings/arm/qcom,ids.h
+++ b/include/dt-bindings/arm/qcom,ids.h
@@ -209,6 +209,7 @@
 #define QCOM_ID_IPQ8172			397
 #define QCOM_ID_IPQ8173			398
 #define QCOM_ID_IPQ8174			399
+#define QCOM_ID_SM7250			400
 #define QCOM_ID_IPQ6018			402
 #define QCOM_ID_IPQ6028			403
 #define QCOM_ID_SDM429W			416
@@ -300,8 +301,10 @@
 #define QCOM_ID_QCS9075			676
 #define QCOM_ID_QCS615			680
 #define QCOM_ID_SA8797P			690
+#define QCOM_ID_SM8975			707
 #define QCOM_ID_CQ7790M			731
 #define QCOM_ID_CQ7790S			732
+#define QCOM_ID_SM8950			735
 #define QCOM_ID_CQ2390M			756
 #define QCOM_ID_CQ2390S			758
 #define QCOM_ID_IQ2390S			759
diff --git a/include/dt-bindings/clock/agilex-clock.h b/include/dt-bindings/clock/agilex-clock.h
index 06feca07e08e1a..6206a45e007c66 100644
--- a/include/dt-bindings/clock/agilex-clock.h
+++ b/include/dt-bindings/clock/agilex-clock.h
@@ -44,6 +44,7 @@
 
 /* Gate clocks */
 #define AGILEX_MPU_CLK			30
+/* Legacy ABI ID; do not use in new DTs, use AGILEX_MPU_CCU_CLK */
 #define AGILEX_MPU_L2RAM_CLK		31
 #define AGILEX_MPU_PERIPH_CLK		32
 #define AGILEX_L4_MAIN_CLK		33
diff --git a/include/dt-bindings/clock/anlogic,dr1v90-cru.h b/include/dt-bindings/clock/anlogic,dr1v90-cru.h
new file mode 100644
index 00000000000000..664daf3946a7bc
--- /dev/null
+++ b/include/dt-bindings/clock/anlogic,dr1v90-cru.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (C) 2024-2025 Anlogic, Inc.
+ * Copyright (C) 2025 Junhui Liu <junhui.liu@pigmoral.tech>
+ */
+
+#ifndef _DT_BINDINGS_CLOCK_ANLOGIC_DR1V90_CRU_H_
+#define _DT_BINDINGS_CLOCK_ANLOGIC_DR1V90_CRU_H_
+
+#define CLK_OSC_DIV2		0
+#define CLK_CPU_PLL		1
+#define CLK_CPU_PLL_4X		2
+#define CLK_CPU_4X		3
+#define CLK_CPU_2X		4
+#define CLK_CPU_1X		5
+#define CLK_IO_PLL		6
+#define CLK_IO_1000M		7
+#define CLK_IO_400M		8
+#define CLK_IO_25M		9
+#define CLK_IO_80M		10
+#define CLK_IO_400M_DIV2	11
+#define CLK_IO_400M_DIV4	12
+#define CLK_IO_400M_DIV8	13
+#define CLK_IO_400M_DIV16	14
+#define CLK_QSPI		15
+#define CLK_SPI			16
+#define CLK_SMC			17
+#define CLK_SDIO		18
+#define CLK_GPIO_DB		19
+#define CLK_EFUSE		20
+#define CLK_TVS			21
+#define CLK_TRNG		22
+#define CLK_OSC_DIV		23
+#define CLK_PWM			24
+#define CLK_FCLK0		25
+#define CLK_FCLK1		26
+#define CLK_FCLK2		27
+#define CLK_FCLK3		28
+#define CLK_WDT_SEL		29
+#define CLK_EFUSE_SEL		30
+#define CLK_CAN_SEL		31
+#define CLK_CPU_SEL		32
+#define CLK_CAN0		33
+#define CLK_CAN1		34
+
+#endif /* _DT_BINDINGS_CLOCK_ANLOGIC_DR1V90_CRU_H_ */
diff --git a/include/dt-bindings/clock/qcom,eliza-cambistmclkcc.h b/include/dt-bindings/clock/qcom,eliza-cambistmclkcc.h
new file mode 100644
index 00000000000000..7b8b285f18d271
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,eliza-cambistmclkcc.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_BIST_MCLK_CC_ELIZA_H
+#define _DT_BINDINGS_CLK_QCOM_CAM_BIST_MCLK_CC_ELIZA_H
+
+/* CAM_BIST_MCLK_CC clocks */
+#define CAM_BIST_MCLK_CC_MCLK0_CLK				0
+#define CAM_BIST_MCLK_CC_MCLK0_CLK_SRC				1
+#define CAM_BIST_MCLK_CC_MCLK1_CLK				2
+#define CAM_BIST_MCLK_CC_MCLK1_CLK_SRC				3
+#define CAM_BIST_MCLK_CC_MCLK2_CLK				4
+#define CAM_BIST_MCLK_CC_MCLK2_CLK_SRC				5
+#define CAM_BIST_MCLK_CC_MCLK3_CLK				6
+#define CAM_BIST_MCLK_CC_MCLK3_CLK_SRC				7
+#define CAM_BIST_MCLK_CC_MCLK4_CLK				8
+#define CAM_BIST_MCLK_CC_MCLK4_CLK_SRC				9
+#define CAM_BIST_MCLK_CC_MCLK5_CLK				10
+#define CAM_BIST_MCLK_CC_MCLK5_CLK_SRC				11
+#define CAM_BIST_MCLK_CC_MCLK6_CLK				12
+#define CAM_BIST_MCLK_CC_MCLK6_CLK_SRC				13
+#define CAM_BIST_MCLK_CC_MCLK7_CLK				14
+#define CAM_BIST_MCLK_CC_MCLK7_CLK_SRC				15
+#define CAM_BIST_MCLK_CC_PLL0					16
+#define CAM_BIST_MCLK_CC_PLL_TEST_CLK				17
+#define CAM_BIST_MCLK_CC_PLL_TEST_DIV_CLK_SRC			18
+#define CAM_BIST_MCLK_CC_SLEEP_CLK				19
+#define CAM_BIST_MCLK_CC_SLEEP_CLK_SRC				20
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,eliza-camcc.h b/include/dt-bindings/clock/qcom,eliza-camcc.h
new file mode 100644
index 00000000000000..d85ef9777d08d1
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,eliza-camcc.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_ELIZA_H
+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_ELIZA_H
+
+/* CAM_CC clocks */
+#define CAM_CC_CAM_TOP_AHB_CLK					0
+#define CAM_CC_CAM_TOP_FAST_AHB_CLK				1
+#define CAM_CC_CAMNOC_DCD_XO_CLK				2
+#define CAM_CC_CAMNOC_NRT_AXI_CLK				3
+#define CAM_CC_CAMNOC_NRT_CRE_CLK				4
+#define CAM_CC_CAMNOC_NRT_IPE_NPS_CLK				5
+#define CAM_CC_CAMNOC_NRT_OFE_ANCHOR_CLK			6
+#define CAM_CC_CAMNOC_NRT_OFE_HDR_CLK				7
+#define CAM_CC_CAMNOC_NRT_OFE_MAIN_CLK				8
+#define CAM_CC_CAMNOC_RT_AXI_CLK				9
+#define CAM_CC_CAMNOC_RT_AXI_CLK_SRC				10
+#define CAM_CC_CAMNOC_RT_IFE_LITE_CLK				11
+#define CAM_CC_CAMNOC_RT_TFE_0_BAYER_CLK			12
+#define CAM_CC_CAMNOC_RT_TFE_0_MAIN_CLK				13
+#define CAM_CC_CAMNOC_RT_TFE_1_BAYER_CLK			14
+#define CAM_CC_CAMNOC_RT_TFE_1_MAIN_CLK				15
+#define CAM_CC_CAMNOC_RT_TFE_2_BAYER_CLK			16
+#define CAM_CC_CAMNOC_RT_TFE_2_MAIN_CLK				17
+#define CAM_CC_CAMNOC_XO_CLK					18
+#define CAM_CC_CCI_0_CLK					19
+#define CAM_CC_CCI_0_CLK_SRC					20
+#define CAM_CC_CCI_1_CLK					21
+#define CAM_CC_CCI_1_CLK_SRC					22
+#define CAM_CC_CCI_2_CLK					23
+#define CAM_CC_CCI_2_CLK_SRC					24
+#define CAM_CC_CORE_AHB_CLK					25
+#define CAM_CC_CPHY_RX_CLK_SRC					26
+#define CAM_CC_CRE_AHB_CLK					27
+#define CAM_CC_CRE_CLK						28
+#define CAM_CC_CRE_CLK_SRC					29
+#define CAM_CC_CSI0PHYTIMER_CLK					30
+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				31
+#define CAM_CC_CSI1PHYTIMER_CLK					32
+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				33
+#define CAM_CC_CSI2PHYTIMER_CLK					34
+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				35
+#define CAM_CC_CSI3PHYTIMER_CLK					36
+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				37
+#define CAM_CC_CSI4PHYTIMER_CLK					38
+#define CAM_CC_CSI4PHYTIMER_CLK_SRC				39
+#define CAM_CC_CSI5PHYTIMER_CLK					40
+#define CAM_CC_CSI5PHYTIMER_CLK_SRC				41
+#define CAM_CC_CSID_CLK						42
+#define CAM_CC_CSID_CLK_SRC					43
+#define CAM_CC_CSID_CSIPHY_RX_CLK				44
+#define CAM_CC_CSIPHY0_CLK					45
+#define CAM_CC_CSIPHY1_CLK					46
+#define CAM_CC_CSIPHY2_CLK					47
+#define CAM_CC_CSIPHY3_CLK					48
+#define CAM_CC_CSIPHY4_CLK					49
+#define CAM_CC_CSIPHY5_CLK					50
+#define CAM_CC_DRV_AHB_CLK					51
+#define CAM_CC_DRV_XO_CLK					52
+#define CAM_CC_FAST_AHB_CLK_SRC					53
+#define CAM_CC_GDSC_CLK						54
+#define CAM_CC_ICP_0_AHB_CLK					55
+#define CAM_CC_ICP_0_CLK					56
+#define CAM_CC_ICP_0_CLK_SRC					57
+#define CAM_CC_ICP_1_AHB_CLK					58
+#define CAM_CC_ICP_1_CLK					59
+#define CAM_CC_ICP_1_CLK_SRC					60
+#define CAM_CC_IFE_LITE_AHB_CLK					61
+#define CAM_CC_IFE_LITE_CLK					62
+#define CAM_CC_IFE_LITE_CLK_SRC					63
+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				64
+#define CAM_CC_IFE_LITE_CSID_CLK				65
+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				66
+#define CAM_CC_IPE_NPS_AHB_CLK					67
+#define CAM_CC_IPE_NPS_CLK					68
+#define CAM_CC_IPE_NPS_CLK_SRC					69
+#define CAM_CC_IPE_NPS_FAST_AHB_CLK				70
+#define CAM_CC_IPE_PPS_CLK					71
+#define CAM_CC_IPE_PPS_FAST_AHB_CLK				72
+#define CAM_CC_JPEG_0_CLK					73
+#define CAM_CC_JPEG_1_CLK					74
+#define CAM_CC_JPEG_CLK_SRC					75
+#define CAM_CC_OFE_AHB_CLK					76
+#define CAM_CC_OFE_ANCHOR_CLK					77
+#define CAM_CC_OFE_ANCHOR_FAST_AHB_CLK				78
+#define CAM_CC_OFE_CLK_SRC					79
+#define CAM_CC_OFE_HDR_CLK					80
+#define CAM_CC_OFE_HDR_FAST_AHB_CLK				81
+#define CAM_CC_OFE_MAIN_CLK					82
+#define CAM_CC_OFE_MAIN_FAST_AHB_CLK				83
+#define CAM_CC_PLL0						84
+#define CAM_CC_PLL0_OUT_EVEN					85
+#define CAM_CC_PLL0_OUT_ODD					86
+#define CAM_CC_PLL1						87
+#define CAM_CC_PLL1_OUT_EVEN					88
+#define CAM_CC_PLL2						89
+#define CAM_CC_PLL2_OUT_EVEN					90
+#define CAM_CC_PLL3						91
+#define CAM_CC_PLL3_OUT_EVEN					92
+#define CAM_CC_PLL4						93
+#define CAM_CC_PLL4_OUT_EVEN					94
+#define CAM_CC_PLL5						95
+#define CAM_CC_PLL5_OUT_EVEN					96
+#define CAM_CC_PLL6						97
+#define CAM_CC_PLL6_OUT_EVEN					98
+#define CAM_CC_PLL6_OUT_ODD					99
+#define CAM_CC_QDSS_DEBUG_CLK					100
+#define CAM_CC_QDSS_DEBUG_CLK_SRC				101
+#define CAM_CC_QDSS_DEBUG_XO_CLK				102
+#define CAM_CC_SLEEP_CLK					103
+#define CAM_CC_SLEEP_CLK_SRC					104
+#define CAM_CC_SLOW_AHB_CLK_SRC					105
+#define CAM_CC_TFE_0_BAYER_CLK					106
+#define CAM_CC_TFE_0_BAYER_FAST_AHB_CLK				107
+#define CAM_CC_TFE_0_CLK_SRC					108
+#define CAM_CC_TFE_0_MAIN_CLK					109
+#define CAM_CC_TFE_0_MAIN_FAST_AHB_CLK				110
+#define CAM_CC_TFE_1_BAYER_CLK					111
+#define CAM_CC_TFE_1_BAYER_FAST_AHB_CLK				112
+#define CAM_CC_TFE_1_CLK_SRC					113
+#define CAM_CC_TFE_1_MAIN_CLK					114
+#define CAM_CC_TFE_1_MAIN_FAST_AHB_CLK				115
+#define CAM_CC_TFE_2_BAYER_CLK					116
+#define CAM_CC_TFE_2_BAYER_FAST_AHB_CLK				117
+#define CAM_CC_TFE_2_CLK_SRC					118
+#define CAM_CC_TFE_2_MAIN_CLK					119
+#define CAM_CC_TFE_2_MAIN_FAST_AHB_CLK				120
+#define CAM_CC_XO_CLK_SRC					121
+
+/* CAM_CC power domains */
+#define CAM_CC_IPE_0_GDSC					0
+#define CAM_CC_OFE_GDSC						1
+#define CAM_CC_TFE_0_GDSC					2
+#define CAM_CC_TFE_1_GDSC					3
+#define CAM_CC_TFE_2_GDSC					4
+#define CAM_CC_TITAN_TOP_GDSC					5
+
+/* CAM_CC resets */
+#define CAM_CC_DRV_BCR						0
+#define CAM_CC_ICP_BCR						1
+#define CAM_CC_IPE_0_BCR					2
+#define CAM_CC_OFE_BCR						3
+#define CAM_CC_QDSS_DEBUG_BCR					4
+#define CAM_CC_TFE_0_BCR					5
+#define CAM_CC_TFE_1_BCR					6
+#define CAM_CC_TFE_2_BCR					7
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,eliza-gpucc.h b/include/dt-bindings/clock/qcom,eliza-gpucc.h
new file mode 100644
index 00000000000000..c3d9b7827325c5
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,eliza-gpucc.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_ELIZA_H
+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_ELIZA_H
+
+/* GPU_CC clocks */
+#define GPU_CC_AHB_CLK						0
+#define GPU_CC_CRC_AHB_CLK					1
+#define GPU_CC_CX_ACCU_SHIFT_CLK				2
+#define GPU_CC_CX_FF_CLK					3
+#define GPU_CC_CX_GMU_CLK					4
+#define GPU_CC_CXO_AON_CLK					5
+#define GPU_CC_CXO_CLK						6
+#define GPU_CC_DEMET_CLK					7
+#define GPU_CC_DEMET_DIV_CLK_SRC				8
+#define GPU_CC_FF_CLK_SRC					9
+#define GPU_CC_FREQ_MEASURE_CLK					10
+#define GPU_CC_GMU_CLK_SRC					11
+#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK				12
+#define GPU_CC_HUB_AON_CLK					13
+#define GPU_CC_HUB_CLK_SRC					14
+#define GPU_CC_HUB_CX_INT_CLK					15
+#define GPU_CC_MEMNOC_GFX_CLK					16
+#define GPU_CC_MND1X_0_GFX3D_CLK				17
+#define GPU_CC_MND1X_1_GFX3D_CLK				18
+#define GPU_CC_PLL0						19
+#define GPU_CC_PLL1						20
+#define GPU_CC_SLEEP_CLK					21
+#define GPU_CC_XO_CLK_SRC					22
+#define GPU_CC_XO_DIV_CLK_SRC					23
+
+/* GPU_CC power domains */
+#define GPU_CC_CX_GDSC						0
+#define GPU_CC_GX_GDSC						1
+
+/* GPU_CC resets */
+#define GPU_CC_ACD_BCR						0
+#define GPU_CC_CB_BCR						1
+#define GPU_CC_CX_BCR						2
+#define GPU_CC_FAST_HUB_BCR					3
+#define GPU_CC_FF_BCR						4
+#define GPU_CC_GFX3D_AON_BCR					5
+#define GPU_CC_GMU_BCR						6
+#define GPU_CC_GX_BCR						7
+#define GPU_CC_RBCPR_BCR					8
+#define GPU_CC_XO_BCR						9
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,eliza-videocc.h b/include/dt-bindings/clock/qcom,eliza-videocc.h
new file mode 100644
index 00000000000000..1e922250a7fae7
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,eliza-videocc.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_VIDEO_CC_ELIZA_H
+#define _DT_BINDINGS_CLK_QCOM_VIDEO_CC_ELIZA_H
+
+/* VIDEO_CC clocks */
+#define VIDEO_CC_AHB_CLK					0
+#define VIDEO_CC_AHB_CLK_SRC					1
+#define VIDEO_CC_MVS0_CLK					2
+#define VIDEO_CC_MVS0_CLK_SRC					3
+#define VIDEO_CC_MVS0_DIV_CLK_SRC				4
+#define VIDEO_CC_MVS0_SHIFT_CLK					5
+#define VIDEO_CC_MVS0C_CLK					6
+#define VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC				7
+#define VIDEO_CC_MVS0C_SHIFT_CLK				8
+#define VIDEO_CC_PLL0						9
+#define VIDEO_CC_SLEEP_CLK					10
+#define VIDEO_CC_SLEEP_CLK_SRC					11
+#define VIDEO_CC_XO_CLK						12
+#define VIDEO_CC_XO_CLK_SRC					13
+
+/* VIDEO_CC power domains */
+#define VIDEO_CC_MVS0_GDSC					0
+#define VIDEO_CC_MVS0C_GDSC					1
+
+/* VIDEO_CC resets */
+#define VIDEO_CC_INTERFACE_BCR					0
+#define VIDEO_CC_MVS0_CLK_ARES					1
+#define VIDEO_CC_MVS0_BCR					2
+#define VIDEO_CC_MVS0C_CLK_ARES					3
+#define VIDEO_CC_MVS0C_BCR					4
+#define VIDEO_CC_XO_CLK_ARES					5
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,gcc-msm8952.h b/include/dt-bindings/clock/qcom,gcc-msm8952.h
new file mode 100644
index 00000000000000..5b5c38a9f4dd10
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,gcc-msm8952.h
@@ -0,0 +1,215 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright 2020 Linaro Limited
+ * Copyright 2026 Muzaffer Kadir <muzafferkadir@mainlining.org>
+ */
+
+#ifndef _DT_BINDINGS_CLK_MSM_GCC_8952_H
+#define _DT_BINDINGS_CLK_MSM_GCC_8952_H
+
+#define GPLL0						0
+#define GPLL0_VOTE					1
+#define GPLL3						2
+#define GPLL3_EARLY					3
+#define GPLL4						4
+#define GPLL4_VOTE					5
+#define GPLL6						6
+#define GPLL6_VOTE					7
+#define CAMSS_TOP_AHB_CLK_SRC				8
+#define APSS_AHB_CLK_SRC				9
+#define CSI0_CLK_SRC					10
+#define CSI1_CLK_SRC					11
+#define GFX3D_CLK_SRC					12
+#define VFE0_CLK_SRC					13
+#define BLSP1_QUP1_I2C_APPS_CLK_SRC			14
+#define BLSP1_QUP1_SPI_APPS_CLK_SRC			15
+#define BLSP1_QUP2_I2C_APPS_CLK_SRC			16
+#define BLSP1_QUP2_SPI_APPS_CLK_SRC			17
+#define BLSP1_QUP3_I2C_APPS_CLK_SRC			18
+#define BLSP1_QUP3_SPI_APPS_CLK_SRC			19
+#define BLSP1_QUP4_I2C_APPS_CLK_SRC			20
+#define BLSP1_QUP4_SPI_APPS_CLK_SRC			21
+#define BLSP1_UART1_APPS_CLK_SRC			22
+#define BLSP1_UART2_APPS_CLK_SRC			23
+#define CCI_CLK_SRC					24
+#define CAMSS_GP0_CLK_SRC				25
+#define CAMSS_GP1_CLK_SRC				26
+#define JPEG0_CLK_SRC					27
+#define MCLK0_CLK_SRC					28
+#define MCLK1_CLK_SRC					29
+#define CSI0PHYTIMER_CLK_SRC				30
+#define CSI1PHYTIMER_CLK_SRC				31
+#define CPP_CLK_SRC					32
+#define CRYPTO_CLK_SRC					33
+#define GP1_CLK_SRC					34
+#define GP2_CLK_SRC					35
+#define GP3_CLK_SRC					36
+#define BYTE0_CLK_SRC					37
+#define ESC0_CLK_SRC					38
+#define MDP_CLK_SRC					39
+#define PCLK0_CLK_SRC					40
+#define VSYNC_CLK_SRC					41
+#define PDM2_CLK_SRC					42
+#define SDCC1_APPS_CLK_SRC				43
+#define SDCC2_APPS_CLK_SRC				44
+#define USB_HS_SYSTEM_CLK_SRC				45
+#define VCODEC0_CLK_SRC					46
+#define GCC_BLSP1_AHB_CLK				47
+#define GCC_BLSP1_QUP1_I2C_APPS_CLK			48
+#define GCC_BLSP1_QUP1_SPI_APPS_CLK			49
+#define GCC_BLSP1_QUP2_I2C_APPS_CLK			50
+#define GCC_BLSP1_QUP2_SPI_APPS_CLK			51
+#define GCC_BLSP1_QUP3_I2C_APPS_CLK			52
+#define GCC_BLSP1_QUP3_SPI_APPS_CLK			53
+#define GCC_BLSP1_QUP4_I2C_APPS_CLK			54
+#define GCC_BLSP1_QUP4_SPI_APPS_CLK			55
+#define GCC_BLSP1_UART1_APPS_CLK			56
+#define GCC_BLSP1_UART2_APPS_CLK			57
+#define GCC_BOOT_ROM_AHB_CLK				58
+#define GCC_CAMSS_CCI_AHB_CLK				59
+#define GCC_CAMSS_CCI_CLK				60
+#define GCC_CAMSS_CSI0_AHB_CLK				61
+#define GCC_CAMSS_CSI0_CLK				62
+#define GCC_CAMSS_CSI0PHY_CLK				63
+#define GCC_CAMSS_CSI0PIX_CLK				64
+#define GCC_CAMSS_CSI0RDI_CLK				65
+#define GCC_CAMSS_CSI1_AHB_CLK				66
+#define GCC_CAMSS_CSI1_CLK				67
+#define GCC_CAMSS_CSI1PHY_CLK				68
+#define GCC_CAMSS_CSI1PIX_CLK				69
+#define GCC_CAMSS_CSI1RDI_CLK				70
+#define GCC_CAMSS_CSI_VFE0_CLK				71
+#define GCC_CAMSS_GP0_CLK				72
+#define GCC_CAMSS_GP1_CLK				73
+#define GCC_CAMSS_ISPIF_AHB_CLK				74
+#define GCC_CAMSS_JPEG0_CLK				75
+#define GCC_CAMSS_JPEG_AHB_CLK				76
+#define GCC_CAMSS_JPEG_AXI_CLK				77
+#define GCC_CAMSS_MCLK0_CLK				78
+#define GCC_CAMSS_MCLK1_CLK				79
+#define GCC_CAMSS_MICRO_AHB_CLK				80
+#define GCC_CAMSS_CSI0PHYTIMER_CLK			81
+#define GCC_CAMSS_CSI1PHYTIMER_CLK			82
+#define GCC_CAMSS_AHB_CLK				83
+#define GCC_CAMSS_TOP_AHB_CLK				84
+#define GCC_CAMSS_CPP_AHB_CLK				85
+#define GCC_CAMSS_CPP_CLK				86
+#define GCC_CAMSS_VFE0_CLK				87
+#define GCC_CAMSS_VFE0_AHB_CLK				88
+#define GCC_CAMSS_VFE0_AXI_CLK				89
+#define GCC_CRYPTO_AHB_CLK				90
+#define GCC_CRYPTO_AXI_CLK				91
+#define GCC_CRYPTO_CLK					92
+#define GCC_OXILI_GMEM_CLK				93
+#define GCC_GP1_CLK					94
+#define GCC_GP2_CLK					95
+#define GCC_GP3_CLK					96
+#define GCC_MDSS_AHB_CLK				97
+#define GCC_MDSS_AXI_CLK				98
+#define GCC_MDSS_BYTE0_CLK				99
+#define GCC_MDSS_ESC0_CLK				100
+#define GCC_MDSS_MDP_CLK				101
+#define GCC_MDSS_PCLK0_CLK				102
+#define GCC_MDSS_VSYNC_CLK				103
+#define GCC_MSS_CFG_AHB_CLK				104
+#define GCC_OXILI_AHB_CLK				105
+#define GCC_OXILI_GFX3D_CLK				106
+#define GCC_PDM2_CLK					107
+#define GCC_PDM_AHB_CLK					108
+#define GCC_PRNG_AHB_CLK				109
+#define GCC_SDCC1_AHB_CLK				110
+#define GCC_SDCC1_APPS_CLK				111
+#define GCC_SDCC2_AHB_CLK				112
+#define GCC_SDCC2_APPS_CLK				113
+#define GCC_GTCU_AHB_CLK				114
+#define GCC_JPEG_TBU_CLK				115
+#define GCC_MDP_TBU_CLK					116
+#define GCC_SMMU_CFG_CLK				117
+#define GCC_VENUS_TBU_CLK				118
+#define GCC_VFE_TBU_CLK					119
+#define GCC_USB2A_PHY_SLEEP_CLK				120
+#define GCC_USB_HS_AHB_CLK				121
+#define GCC_USB_HS_SYSTEM_CLK				122
+#define GCC_VENUS0_AHB_CLK				123
+#define GCC_VENUS0_AXI_CLK				124
+#define GCC_VENUS0_VCODEC0_CLK				125
+#define GCC_APSS_TCU_CLK				126
+#define GCC_GFX_TCU_CLK					127
+#define GCC_BIMC_GFX_CLK				128
+#define GCC_BIMC_GPU_CLK				129
+#define GCC_MSS_Q6_BIMC_AXI_CLK				130
+#define GCC_GFX_TBU_CLK					131
+#define GCC_CPP_TBU_CLK					132
+#define USB_FS_SYSTEM_CLK_SRC				133
+#define USB_FS_IC_CLK_SRC				134
+#define GCC_USB_FS_AHB_CLK				135
+#define GCC_USB_FS_IC_CLK				136
+#define GCC_USB_FS_SYSTEM_CLK				137
+#define GCC_VENUS0_CORE0_VCODEC0_CLK			138
+#define GCC_VENUS0_CORE1_VCODEC0_CLK			139
+#define GCC_OXILI_TIMER_CLK				140
+#define GCC_USB_HS_PHY_CFG_AHB_CLK			141
+#define CSI2_CLK_SRC					142
+#define GCC_CAMSS_CSI2_AHB_CLK				143
+#define GCC_CAMSS_CSI2_CLK				144
+#define GCC_CAMSS_CSI2PHY_CLK				145
+#define GCC_CAMSS_CSI2PIX_CLK				146
+#define GCC_CAMSS_CSI2RDI_CLK				147
+#define SDCC1_ICE_CORE_CLK_SRC				148
+#define GCC_SDCC1_ICE_CORE_CLK				149
+#define BLSP2_QUP1_I2C_APPS_CLK_SRC			150
+#define BLSP2_QUP1_SPI_APPS_CLK_SRC			151
+#define BLSP2_QUP2_I2C_APPS_CLK_SRC			152
+#define BLSP2_QUP2_SPI_APPS_CLK_SRC			153
+#define BLSP2_QUP3_I2C_APPS_CLK_SRC			154
+#define BLSP2_QUP3_SPI_APPS_CLK_SRC			155
+#define BLSP2_QUP4_I2C_APPS_CLK_SRC			156
+#define BLSP2_QUP4_SPI_APPS_CLK_SRC			157
+#define BLSP2_UART1_APPS_CLK_SRC			158
+#define BLSP2_UART2_APPS_CLK_SRC			159
+#define GCC_BLSP2_AHB_CLK				160
+#define GCC_BLSP2_QUP1_I2C_APPS_CLK			161
+#define GCC_BLSP2_QUP1_SPI_APPS_CLK			162
+#define GCC_BLSP2_QUP2_I2C_APPS_CLK			163
+#define GCC_BLSP2_QUP2_SPI_APPS_CLK			164
+#define GCC_BLSP2_QUP3_I2C_APPS_CLK			165
+#define GCC_BLSP2_QUP3_SPI_APPS_CLK			166
+#define GCC_BLSP2_QUP4_I2C_APPS_CLK			167
+#define GCC_BLSP2_QUP4_SPI_APPS_CLK			168
+#define GCC_BLSP2_UART1_APPS_CLK			169
+#define GCC_BLSP2_UART2_APPS_CLK			170
+#define VFE1_CLK_SRC					171
+#define GCC_CAMSS_VFE1_CLK				172
+#define GCC_CAMSS_VFE1_AHB_CLK				173
+#define GCC_CAMSS_VFE1_AXI_CLK				174
+#define GCC_CAMSS_CSI_VFE1_CLK				175
+#define MCLK2_CLK_SRC					176
+#define GCC_CAMSS_MCLK2_CLK				177
+#define GCC_VFE1_TBU_CLK				178
+#define GCC_IPA_TBU_CLK					179
+#define GCC_CAMSS_CPP_AXI_CLK				180
+
+/* Indexes for GDSCs */
+#define VENUS_GDSC				0
+#define MDSS_GDSC				1
+#define JPEG_GDSC				2
+#define VFE0_GDSC				3
+#define VFE1_GDSC				4
+#define OXILI_GX_GDSC				5
+#define VENUS_CORE0_GDSC			6
+#define VENUS_CORE1_GDSC			7
+#define CPP_GDSC				8
+
+/* Indexes for Resets */
+#define GCC_CRYPTO_BCR				0
+#define GCC_USB_FS_BCR				1
+#define GCC_USB_HS_BCR				2
+#define GCC_QUSB2_PHY_BCR			3
+#define GCC_USB2_HS_PHY_ONLY_BCR		4
+#define GCC_SDCC1_BCR				5
+#define GCC_SDCC2_BCR				6
+#define GCC_MDSS_BCR				7
+#define GCC_CAMSS_MICRO_BCR			8
+#define GCC_MSS_BCR				9
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,hawi-camcc.h b/include/dt-bindings/clock/qcom,hawi-camcc.h
new file mode 100644
index 00000000000000..6ef731738e6e87
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,hawi-camcc.h
@@ -0,0 +1,165 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_HAWI_H
+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_HAWI_H
+
+/* CAM_CC clocks */
+#define CAM_CC_CAM_TOP_AHB_CLK					0
+#define CAM_CC_CAM_TOP_FAST_AHB_CLK				1
+#define CAM_CC_CAMNOC_DCD_XO_CLK				2
+#define CAM_CC_CAMNOC_NRT_AXI_CLK				3
+#define CAM_CC_CAMNOC_NRT_CRE_CLK				4
+#define CAM_CC_CAMNOC_NRT_IPE_NPS_CLK				5
+#define CAM_CC_CAMNOC_NRT_IPE_SCALERS_CLK			6
+#define CAM_CC_CAMNOC_NRT_OFE_MAIN_CLK				7
+#define CAM_CC_CAMNOC_NRT_SQPF_CLK				8
+#define CAM_CC_CAMNOC_RT_AXI_CLK				9
+#define CAM_CC_CAMNOC_RT_AXI_CLK_SRC				10
+#define CAM_CC_CAMNOC_RT_IFE_LITE_CLK				11
+#define CAM_CC_CAMNOC_RT_TFE_0_MAIN_CLK				12
+#define CAM_CC_CAMNOC_RT_TFE_1_MAIN_CLK				13
+#define CAM_CC_CAMNOC_RT_TFE_2_MAIN_CLK				14
+#define CAM_CC_CCI_0_CLK					15
+#define CAM_CC_CCI_0_CLK_SRC					16
+#define CAM_CC_CCI_1_CLK					17
+#define CAM_CC_CCI_1_CLK_SRC					18
+#define CAM_CC_CCI_2_CLK					19
+#define CAM_CC_CCI_2_CLK_SRC					20
+#define CAM_CC_CORE_AHB_CLK					21
+#define CAM_CC_CPHY_RX_CLK_SRC					22
+#define CAM_CC_CRE_AHB_CLK					23
+#define CAM_CC_CRE_CLK						24
+#define CAM_CC_CRE_CLK_SRC					25
+#define CAM_CC_CSI0PHYTIMER_CLK					26
+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				27
+#define CAM_CC_CSI1PHYTIMER_CLK					28
+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				29
+#define CAM_CC_CSI2PHYTIMER_CLK					30
+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				31
+#define CAM_CC_CSI3PHYTIMER_CLK					32
+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				33
+#define CAM_CC_CSI4PHYTIMER_CLK					34
+#define CAM_CC_CSI4PHYTIMER_CLK_SRC				35
+#define CAM_CC_CSI5PHYTIMER_CLK					36
+#define CAM_CC_CSI5PHYTIMER_CLK_SRC				37
+#define CAM_CC_CSID_CLK						38
+#define CAM_CC_CSID_CLK_SRC					39
+#define CAM_CC_CSID_CSIPHY_RX_CLK				40
+#define CAM_CC_CSIPHY0_CLK					41
+#define CAM_CC_CSIPHY1_CLK					42
+#define CAM_CC_CSIPHY2_CLK					43
+#define CAM_CC_CSIPHY3_CLK					44
+#define CAM_CC_CSIPHY4_CLK					45
+#define CAM_CC_CSIPHY5_CLK					46
+#define CAM_CC_CX_GDSC_CLK					47
+#define CAM_CC_DRV_AHB_CLK					48
+#define CAM_CC_DRV_XO_CLK					49
+#define CAM_CC_FAST_AHB_CLK_SRC					50
+#define CAM_CC_GDSC_CLK						51
+#define CAM_CC_ICP_0_AHB_CLK					52
+#define CAM_CC_ICP_0_CLK					53
+#define CAM_CC_ICP_0_CLK_SRC					54
+#define CAM_CC_ICP_1_AHB_CLK					55
+#define CAM_CC_ICP_1_CLK					56
+#define CAM_CC_ICP_1_CLK_SRC					57
+#define CAM_CC_IFE_LITE_AHB_CLK					58
+#define CAM_CC_IFE_LITE_CLK					59
+#define CAM_CC_IFE_LITE_CLK_SRC					60
+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				61
+#define CAM_CC_IFE_LITE_CSID_CLK				62
+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				63
+#define CAM_CC_IPE_NPS_AHB_CLK					64
+#define CAM_CC_IPE_NPS_CLK					65
+#define CAM_CC_IPE_NPS_CLK_SRC					66
+#define CAM_CC_IPE_NPS_FAST_AHB_CLK				67
+#define CAM_CC_IPE_PPS_CLK					68
+#define CAM_CC_IPE_PPS_FAST_AHB_CLK				69
+#define CAM_CC_IPE_SCALERS_CLK					70
+#define CAM_CC_IPE_SCALERS_FAST_AHB_CLK				71
+#define CAM_CC_JPEG_CLK						72
+#define CAM_CC_JPEG_CLK_SRC					73
+#define CAM_CC_MMSS_AHB_CLK					74
+#define CAM_CC_OFE_AHB_CLK					75
+#define CAM_CC_OFE_ANCHOR_CLK					76
+#define CAM_CC_OFE_ANCHOR_FAST_AHB_CLK				77
+#define CAM_CC_OFE_CLK_SRC					78
+#define CAM_CC_OFE_HDR_CLK					79
+#define CAM_CC_OFE_HDR_FAST_AHB_CLK				80
+#define CAM_CC_OFE_MAIN_CLK					81
+#define CAM_CC_OFE_MAIN_FAST_AHB_CLK				82
+#define CAM_CC_OFE_MP_CLK					83
+#define CAM_CC_OFE_MP_FAST_AHB_CLK				84
+#define CAM_CC_PLL0						85
+#define CAM_CC_PLL0_OUT_EVEN					86
+#define CAM_CC_PLL0_OUT_ODD					87
+#define CAM_CC_PLL1						88
+#define CAM_CC_PLL1_OUT_EVEN					89
+#define CAM_CC_PLL2						90
+#define CAM_CC_PLL2_OUT_EVEN					91
+#define CAM_CC_PLL3						92
+#define CAM_CC_PLL3_OUT_EVEN					93
+#define CAM_CC_PLL4						94
+#define CAM_CC_PLL4_OUT_EVEN					95
+#define CAM_CC_PLL5						96
+#define CAM_CC_PLL5_OUT_EVEN					97
+#define CAM_CC_PLL6						98
+#define CAM_CC_PLL6_OUT_EVEN					99
+#define CAM_CC_PLL6_OUT_ODD					100
+#define CAM_CC_PLL7						101
+#define CAM_CC_PLL8						102
+#define CAM_CC_PLL8_OUT_EVEN					103
+#define CAM_CC_QDSS_DEBUG_CLK					104
+#define CAM_CC_QDSS_DEBUG_CLK_SRC				105
+#define CAM_CC_QDSS_DEBUG_XO_CLK				106
+#define CAM_CC_SLEEP_CLK					107
+#define CAM_CC_SLOW_AHB_CLK_SRC					108
+#define CAM_CC_SQPF_AHB_CLK					109
+#define CAM_CC_SQPF_CLK						110
+#define CAM_CC_SQPF_CLK_SRC					111
+#define CAM_CC_SQPF_FAST_AHB_CLK				112
+#define CAM_CC_TFE_0_BAYER_CLK					113
+#define CAM_CC_TFE_0_BAYER_FAST_AHB_CLK				114
+#define CAM_CC_TFE_0_CLK_SRC					115
+#define CAM_CC_TFE_0_MAIN_CLK					116
+#define CAM_CC_TFE_0_MAIN_FAST_AHB_CLK				117
+#define CAM_CC_TFE_1_BAYER_CLK					118
+#define CAM_CC_TFE_1_BAYER_FAST_AHB_CLK				119
+#define CAM_CC_TFE_1_CLK_SRC					120
+#define CAM_CC_TFE_1_MAIN_CLK					121
+#define CAM_CC_TFE_1_MAIN_FAST_AHB_CLK				122
+#define CAM_CC_TFE_2_BAYER_CLK					123
+#define CAM_CC_TFE_2_BAYER_FAST_AHB_CLK				124
+#define CAM_CC_TFE_2_CLK_SRC					125
+#define CAM_CC_TFE_2_MAIN_CLK					126
+#define CAM_CC_TFE_2_MAIN_FAST_AHB_CLK				127
+#define CAM_CC_TITAN_CAMNOC_XO_CLK				128
+#define CAM_CC_TITAN_MNOC_HF_CLK				129
+#define CAM_CC_TITAN_MNOC_SF_CLK				130
+#define CAM_CC_TRACENOC_TPDM_1_CMB_CLK				131
+#define CAM_CC_XO_CLK_SRC					132
+
+/* CAM_CC power domains */
+#define CAM_CC_IPE_0_GDSC					0
+#define CAM_CC_OFE_GDSC						1
+#define CAM_CC_SQPF_GDSC					2
+#define CAM_CC_TFE_0_GDSC					3
+#define CAM_CC_TFE_1_GDSC					4
+#define CAM_CC_TFE_2_GDSC					5
+#define CAM_CC_TITAN_TOP_CX_GDSC				6
+#define CAM_CC_TITAN_TOP_GDSC					7
+
+/* CAM_CC resets */
+#define CAM_CC_DRV_BCR						0
+#define CAM_CC_ICP_BCR						1
+#define CAM_CC_IPE_0_BCR					2
+#define CAM_CC_OFE_BCR						3
+#define CAM_CC_QDSS_DEBUG_BCR					4
+#define CAM_CC_SQPF_BCR						5
+#define CAM_CC_TFE_0_BCR					6
+#define CAM_CC_TFE_1_BCR					7
+#define CAM_CC_TFE_2_BCR					8
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,hawi-gpucc.h b/include/dt-bindings/clock/qcom,hawi-gpucc.h
new file mode 100644
index 00000000000000..181534771b10aa
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,hawi-gpucc.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_HAWI_H
+#define _DT_BINDINGS_CLK_QCOM_GPU_CC_HAWI_H
+
+/* GPU_CC clocks */
+#define GPU_CC_AHB_CLK						0
+#define GPU_CC_CB_CLK						1
+#define GPU_CC_CX_ACCU_SHIFT_CLK				2
+#define GPU_CC_CX_GMU_CLK					3
+#define GPU_CC_CXO_AON_CLK					4
+#define GPU_CC_CXO_CLK						5
+#define GPU_CC_DEMET_CLK					6
+#define GPU_CC_DPM_CLK						7
+#define GPU_CC_FREQ_MEASURE_CLK					8
+#define GPU_CC_GMU_CLK_SRC					9
+#define GPU_CC_GPU_SMMU_VOTE_CLK				10
+#define GPU_CC_GX_ACCU_SHIFT_CLK				11
+#define GPU_CC_HUB_AON_CLK					12
+#define GPU_CC_HUB_CLK_SRC					13
+#define GPU_CC_HUB_CX_INT_CLK					14
+#define GPU_CC_HUB_DIV_CLK_SRC					15
+#define GPU_CC_MEMNOC_GFX_CLK					16
+#define GPU_CC_MXG_AHB_CLK					17
+#define GPU_CC_MXG_XO_CLK					18
+#define GPU_CC_PLL0						19
+#define GPU_CC_RSCC_HUB_AON_CLK					20
+#define GPU_CC_RSCC_XO_AON_CLK					21
+#define GPU_CC_SLEEP_CLK					22
+
+/* GPU_CC power domains */
+#define GPU_CC_CX_GDSC						0
+#define GPU_CC_MXG_GDSC						1
+
+/* GPU_CC resets */
+#define GPU_CC_CB_BCR						0
+#define GPU_CC_CX_BCR						1
+#define GPU_CC_FAST_HUB_BCR					2
+#define GPU_CC_GMU_BCR						3
+#define GPU_CC_GX_BCR						4
+#define GPU_CC_MXG_BCR						5
+#define GPU_CC_XO_BCR						6
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,ipq5210-cmn-pll.h b/include/dt-bindings/clock/qcom,ipq5210-cmn-pll.h
new file mode 100644
index 00000000000000..9201d1e1062c4f
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,ipq5210-cmn-pll.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_IPQ5210_CMN_PLL_H
+#define _DT_BINDINGS_CLK_QCOM_IPQ5210_CMN_PLL_H
+
+/* Parent clock */
+#define IPQ5210_CMN_PLL_CLK		0
+
+/* Fixed-rate clocks */
+#define IPQ5210_XO_24MHZ_CLK		1
+#define IPQ5210_SLEEP_32KHZ_CLK		2
+
+/* Configurable divider clocks */
+#define IPQ5210_NSS_CLK			3
+#define IPQ5210_PPE_CLK			4
+#define IPQ5210_PON_CLK			5
+#define IPQ5210_EPHY_RAW_CLK		6
+
+/* Gate clocks */
+#define IPQ5210_PCS_31P25MHZ_CLK	7
+#define IPQ5210_ETH0_50MHZ_CLK		8
+#define IPQ5210_ETH1_50MHZ_CLK		9
+#define IPQ5210_ETH2_50MHZ_CLK		10
+#define IPQ5210_EPHY_50MHZ_CLK		11
+#define IPQ5210_ETH_25MHZ_CLK		12
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,kuno-gcc.h b/include/dt-bindings/clock/qcom,kuno-gcc.h
new file mode 100644
index 00000000000000..a237b25a061dba
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,kuno-gcc.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_KUNO_H
+#define _DT_BINDINGS_CLK_QCOM_GCC_KUNO_H
+
+/* GCC clocks */
+#define GPLL0							0
+#define GPLL0_OUT_EVEN						1
+#define GPLL2							2
+#define GPLL3							3
+#define GPLL4							4
+#define GPLL4_OUT_EVEN						5
+#define GCC_EMAC0_AXI_CLK					7
+#define GCC_EMAC0_PHY_AUX_CLK					8
+#define GCC_EMAC0_PHY_AUX_CLK_SRC				9
+#define GCC_EMAC0_PTP_CLK					10
+#define GCC_EMAC0_PTP_CLK_SRC					11
+#define GCC_EMAC0_RGMII_CLK					12
+#define GCC_EMAC0_RGMII_CLK_SRC					13
+#define GCC_EMAC0_SLV_AHB_CLK					14
+#define GCC_EMAC_0_CLKREF_EN					15
+#define GCC_GP1_CLK						16
+#define GCC_GP1_CLK_SRC						17
+#define GCC_GP2_CLK						18
+#define GCC_GP2_CLK_SRC						19
+#define GCC_GP3_CLK						20
+#define GCC_GP3_CLK_SRC						21
+#define GCC_PCIE_0_CLKREF_EN					22
+#define GCC_PCIE_AUX_CLK					23
+#define GCC_PCIE_AUX_CLK_SRC					24
+#define GCC_PCIE_AUX_PHY_CLK_SRC				25
+#define GCC_PCIE_CFG_AHB_CLK					26
+#define GCC_PCIE_MSTR_AXI_CLK					27
+#define GCC_PCIE_PIPE_CLK					28
+#define GCC_PCIE_PIPE_CLK_SRC					29
+#define GCC_PCIE_RCHNG_PHY_CLK					30
+#define GCC_PCIE_RCHNG_PHY_CLK_SRC				31
+#define GCC_PCIE_SLEEP_CLK					32
+#define GCC_PCIE_SLV_AXI_CLK					33
+#define GCC_PCIE_SLV_Q2A_AXI_CLK				34
+#define GCC_PDM2_CLK						35
+#define GCC_PDM2_CLK_SRC					36
+#define GCC_PDM_AHB_CLK						37
+#define GCC_PDM_XO4_CLK						38
+#define GCC_QUPV3_WRAP0_CORE_2X_CLK				39
+#define GCC_QUPV3_WRAP0_CORE_CLK				40
+#define GCC_QUPV3_WRAP0_S0_CLK					41
+#define GCC_QUPV3_WRAP0_S0_CLK_SRC				42
+#define GCC_QUPV3_WRAP0_S1_CLK					43
+#define GCC_QUPV3_WRAP0_S1_CLK_SRC				44
+#define GCC_QUPV3_WRAP0_S2_CLK					45
+#define GCC_QUPV3_WRAP0_S2_CLK_SRC				46
+#define GCC_QUPV3_WRAP0_S3_CLK					47
+#define GCC_QUPV3_WRAP0_S3_CLK_SRC				48
+#define GCC_QUPV3_WRAP0_S4_CLK					49
+#define GCC_QUPV3_WRAP0_S4_CLK_SRC				50
+#define GCC_QUPV3_WRAP_0_M_AHB_CLK				51
+#define GCC_QUPV3_WRAP_0_S_AHB_CLK				52
+#define GCC_SDCC4_AHB_CLK					53
+#define GCC_SDCC4_APPS_CLK					54
+#define GCC_SDCC4_APPS_CLK_SRC					55
+#define GCC_SNOC_CNOC_USB3_CLK					56
+#define GCC_SYS_NOC_USB_SF_AXI_CLK				57
+#define GCC_USB20_MASTER_CLK					58
+#define GCC_USB20_MASTER_CLK_SRC				59
+#define GCC_USB20_MOCK_UTMI_CLK					60
+#define GCC_USB20_MOCK_UTMI_CLK_SRC				61
+#define GCC_USB20_MOCK_UTMI_POSTDIV_CLK_SRC			62
+#define GCC_USB20_SLEEP_CLK					63
+#define GCC_USB2_CLKREF_EN					64
+#define GCC_USB3_PRIM_CLKREF_EN					65
+#define GCC_USB_PHY_CFG_AHB2PHY_CLK				66
+#define GCC_BOOT_ROM_AHB_CLK					68
+
+/* GCC power domains */
+#define GCC_EMAC0_GDSC						0
+#define GCC_PCIE_GDSC						1
+#define GCC_USB20_GDSC						2
+
+/* GCC resets */
+#define GCC_EMAC0_BCR						0
+#define GCC_PCIE_BCR						1
+#define GCC_PCIE_LINK_DOWN_BCR					2
+#define GCC_PCIE_NOCSR_COM_PHY_BCR				3
+#define GCC_PCIE_PHY_BCR					4
+#define GCC_PCIE_PHY_CFG_AHB_BCR				5
+#define GCC_PCIE_PHY_COM_BCR					6
+#define GCC_PCIE_PHY_NOCSR_COM_PHY_BCR				7
+#define GCC_PDM_BCR						8
+#define GCC_QUPV3_WRAPPER_0_BCR					9
+#define GCC_QUSB2PHY_BCR					10
+#define GCC_SDCC4_BCR						11
+#define GCC_TCSR_PCIE_BCR					12
+#define GCC_USB20_BCR						13
+#define GCC_USB_PHY_CFG_AHB2PHY_BCR				14
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,nord-camcc.h b/include/dt-bindings/clock/qcom,nord-camcc.h
new file mode 100644
index 00000000000000..655fef2084a580
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,nord-camcc.h
@@ -0,0 +1,167 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_CAM_CC_NORD_H
+#define _DT_BINDINGS_CLK_QCOM_CAM_CC_NORD_H
+
+/* CAM_CC clocks */
+#define CAM_CC_CAMNOC_DCD_XO_CLK				0
+#define CAM_CC_CAMNOC_NRT_AXI_CLK				1
+#define CAM_CC_CAMNOC_RT_AXI_CLK				2
+#define CAM_CC_CAMNOC_RT_AXI_CLK_SRC				3
+#define CAM_CC_CAMNOC_XO_CLK					4
+#define CAM_CC_CCI_0_CLK					5
+#define CAM_CC_CCI_0_CLK_SRC					6
+#define CAM_CC_CCI_1_CLK					7
+#define CAM_CC_CCI_1_CLK_SRC					8
+#define CAM_CC_CCI_2_CLK					9
+#define CAM_CC_CCI_2_CLK_SRC					10
+#define CAM_CC_CCI_3_CLK					11
+#define CAM_CC_CCI_3_CLK_SRC					12
+#define CAM_CC_CCI_4_CLK					13
+#define CAM_CC_CCI_4_CLK_SRC					14
+#define CAM_CC_CCU_FAST_AHB_CLK					15
+#define CAM_CC_CORE_AHB_CLK					16
+#define CAM_CC_CPHY_RX_CLK_SRC					17
+#define CAM_CC_CSI0PHYTIMER_CLK					18
+#define CAM_CC_CSI0PHYTIMER_CLK_SRC				19
+#define CAM_CC_CSI1PHYTIMER_CLK					20
+#define CAM_CC_CSI1PHYTIMER_CLK_SRC				21
+#define CAM_CC_CSI2PHYTIMER_CLK					22
+#define CAM_CC_CSI2PHYTIMER_CLK_SRC				23
+#define CAM_CC_CSI3PHYTIMER_CLK					24
+#define CAM_CC_CSI3PHYTIMER_CLK_SRC				25
+#define CAM_CC_CSI4PHYTIMER_CLK					26
+#define CAM_CC_CSI4PHYTIMER_CLK_SRC				27
+#define CAM_CC_CSID_CLK						28
+#define CAM_CC_CSID_CLK_SRC					29
+#define CAM_CC_CSID_CSIPHY_RX_CLK				30
+#define CAM_CC_CSIPHY0_CLK					31
+#define CAM_CC_CSIPHY1_CLK					32
+#define CAM_CC_CSIPHY2_CLK					33
+#define CAM_CC_CSIPHY3_CLK					34
+#define CAM_CC_CSIPHY4_CLK					35
+#define CAM_CC_FAST_AHB_CLK_SRC					36
+#define CAM_CC_GDSC_CLK						37
+#define CAM_CC_ICP_0_AHB_CLK					38
+#define CAM_CC_ICP_0_CLK					39
+#define CAM_CC_ICP_0_CLK_SRC					40
+#define CAM_CC_ICP_1_AHB_CLK					41
+#define CAM_CC_ICP_1_CLK					42
+#define CAM_CC_ICP_1_CLK_SRC					43
+#define CAM_CC_IFE_0_MAIN_CLK					44
+#define CAM_CC_IFE_0_MAIN_CLK_SRC				45
+#define CAM_CC_IFE_0_MAIN_FAST_AHB_CLK				46
+#define CAM_CC_IFE_0_PCP_CLK					47
+#define CAM_CC_IFE_0_PCP_FAST_AHB_CLK				48
+#define CAM_CC_IFE_0_SCALAR_CLK					49
+#define CAM_CC_IFE_0_SCALAR_FAST_AHB_CLK			50
+#define CAM_CC_IFE_0_TMC_CLK					51
+#define CAM_CC_IFE_0_TMC_FAST_AHB_CLK				52
+#define CAM_CC_IFE_1_MAIN_CLK					53
+#define CAM_CC_IFE_1_MAIN_CLK_SRC				54
+#define CAM_CC_IFE_1_MAIN_FAST_AHB_CLK				55
+#define CAM_CC_IFE_1_PCP_CLK					56
+#define CAM_CC_IFE_1_PCP_FAST_AHB_CLK				57
+#define CAM_CC_IFE_1_SCALAR_CLK					58
+#define CAM_CC_IFE_1_SCALAR_FAST_AHB_CLK			59
+#define CAM_CC_IFE_1_TMC_CLK					60
+#define CAM_CC_IFE_1_TMC_FAST_AHB_CLK				61
+#define CAM_CC_IFE_2_MAIN_CLK					62
+#define CAM_CC_IFE_2_MAIN_CLK_SRC				63
+#define CAM_CC_IFE_2_MAIN_FAST_AHB_CLK				64
+#define CAM_CC_IFE_2_PCP_CLK					65
+#define CAM_CC_IFE_2_PCP_FAST_AHB_CLK				66
+#define CAM_CC_IFE_2_SCALAR_CLK					67
+#define CAM_CC_IFE_2_SCALAR_FAST_AHB_CLK			68
+#define CAM_CC_IFE_2_TMC_CLK					69
+#define CAM_CC_IFE_2_TMC_FAST_AHB_CLK				70
+#define CAM_CC_IFE_LITE_AHB_CLK					71
+#define CAM_CC_IFE_LITE_CLK					72
+#define CAM_CC_IFE_LITE_CLK_SRC					73
+#define CAM_CC_IFE_LITE_CPHY_RX_CLK				74
+#define CAM_CC_IFE_LITE_CSID_CLK				75
+#define CAM_CC_IFE_LITE_CSID_CLK_SRC				76
+#define CAM_CC_IPE_0_AHB_CLK					77
+#define CAM_CC_IPE_0_CLK					78
+#define CAM_CC_IPE_0_CLK_SRC					79
+#define CAM_CC_IPE_0_FAST_AHB_CLK				80
+#define CAM_CC_IPE_1_AHB_CLK					81
+#define CAM_CC_IPE_1_CLK					82
+#define CAM_CC_IPE_1_CLK_SRC					83
+#define CAM_CC_IPE_1_FAST_AHB_CLK				84
+#define CAM_CC_PLL0						85
+#define CAM_CC_PLL0_OUT_EVEN					86
+#define CAM_CC_PLL0_OUT_ODD					87
+#define CAM_CC_PLL2						88
+#define CAM_CC_PLL2_OUT_EVEN					89
+#define CAM_CC_PLL3						90
+#define CAM_CC_PLL3_OUT_EVEN					91
+#define CAM_CC_PLL4						92
+#define CAM_CC_PLL4_OUT_EVEN					93
+#define CAM_CC_PLL5						94
+#define CAM_CC_PLL5_OUT_EVEN					95
+#define CAM_CC_PLL6						96
+#define CAM_CC_PLL6_OUT_EVEN					97
+#define CAM_CC_QDSS_DEBUG_CLK					98
+#define CAM_CC_QDSS_DEBUG_CLK_SRC				99
+#define CAM_CC_QDSS_DEBUG_XO_CLK				100
+#define CAM_CC_QUP_AHBM_CLK					101
+#define CAM_CC_QUP_AHBM_CLK_SRC					102
+#define CAM_CC_QUP_CORE_2X_CLK					103
+#define CAM_CC_QUP_CORE_2X_CLK_SRC				104
+#define CAM_CC_QUP_CORE_2X_DIV_CLK_SRC				105
+#define CAM_CC_QUP_CORE_CLK					106
+#define CAM_CC_QUP_SE_CLK					107
+#define CAM_CC_QUP_SE_CLK_SRC					108
+#define CAM_CC_QUP_SLEEP_CLK					109
+#define CAM_CC_SFE_LITE_0_CLK					110
+#define CAM_CC_SFE_LITE_0_FAST_AHB_CLK				111
+#define CAM_CC_SFE_LITE_1_CLK					112
+#define CAM_CC_SFE_LITE_1_FAST_AHB_CLK				113
+#define CAM_CC_SFE_LITE_2_CLK					114
+#define CAM_CC_SFE_LITE_2_FAST_AHB_CLK				115
+#define CAM_CC_SLEEP_CLK					116
+#define CAM_CC_SLEEP_CLK_SRC					117
+#define CAM_CC_SLOW_AHB_CLK_SRC					118
+#define CAM_CC_SM_OBS_CLK					119
+#define CAM_CC_TOP_AHB_CLK					120
+#define CAM_CC_TOP_FAST_AHB_CLK					121
+#define CAM_CC_TOP_IFE_0_CLK					122
+#define CAM_CC_TOP_IFE_1_CLK					123
+#define CAM_CC_TOP_IFE_2_CLK					124
+#define CAM_CC_TOP_IFE_LITE_CLK					125
+#define CAM_CC_TOP_IPE_0_CLK					126
+#define CAM_CC_TOP_IPE_1_CLK					127
+#define CAM_CC_TOP_QUP_AHBM_CLK					128
+#define CAM_CC_TOP_SFE_LITE_0_CLK				129
+#define CAM_CC_TOP_SFE_LITE_1_CLK				130
+#define CAM_CC_TOP_SFE_LITE_2_CLK				131
+#define CAM_CC_TPG_CSIPHY_RX_CLK				132
+#define CAM_CC_XO_CLK_SRC					133
+
+/* CAM_CC power domains */
+#define CAM_CC_IFE_0_GDSC					0
+#define CAM_CC_IFE_1_GDSC					1
+#define CAM_CC_IFE_2_GDSC					2
+#define CAM_CC_IPE_0_GDSC					3
+#define CAM_CC_IPE_1_GDSC					4
+#define CAM_CC_TITAN_TOP_GDSC					5
+
+/* CAM_CC resets */
+#define CAM_CC_CCU_BCR						0
+#define CAM_CC_ICP_0_BCR					1
+#define CAM_CC_ICP_1_BCR					2
+#define CAM_CC_IFE_0_BCR					3
+#define CAM_CC_IFE_1_BCR					4
+#define CAM_CC_IFE_2_BCR					5
+#define CAM_CC_IPE_0_BCR					6
+#define CAM_CC_IPE_1_BCR					7
+#define CAM_CC_QDSS_DEBUG_BCR					8
+#define CAM_CC_SFE_LITE_0_BCR					9
+#define CAM_CC_SFE_LITE_1_BCR					10
+#define CAM_CC_SFE_LITE_2_BCR					11
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,nord-negcc.h b/include/dt-bindings/clock/qcom,nord-negcc.h
index a7024317e90b13..ad9ff389956dca 100644
--- a/include/dt-bindings/clock/qcom,nord-negcc.h
+++ b/include/dt-bindings/clock/qcom,nord-negcc.h
@@ -121,5 +121,6 @@
 #define NE_GCC_USB3PHY_PHY_PRIM_BCR				11
 #define NE_GCC_USB3PHY_PHY_SEC_BCR				12
 #define NE_GCC_QUSB2PHY_PRIM_BCR				13
+#define NE_GCC_QUSB2PHY_SEC_BCR					14
 
 #endif
diff --git a/include/dt-bindings/clock/qcom,nord-nwgcc.h b/include/dt-bindings/clock/qcom,nord-nwgcc.h
index b6253dd2aa85a3..8beaa129aa814c 100644
--- a/include/dt-bindings/clock/qcom,nord-nwgcc.h
+++ b/include/dt-bindings/clock/qcom,nord-nwgcc.h
@@ -65,5 +65,8 @@
 #define NW_GCC_GPU_2_BCR					6
 #define NW_GCC_GPU_BCR						7
 #define NW_GCC_VIDEO_BCR					8
+#define NW_GCC_VIDEO_AXI0_CLK_ARES				9
+#define NW_GCC_VIDEO_AXI0C_CLK_ARES				10
+#define NW_GCC_VIDEO_AXI1_CLK_ARES				11
 
 #endif
diff --git a/include/dt-bindings/clock/qcom,nord-segcc.h b/include/dt-bindings/clock/qcom,nord-segcc.h
index f0f7422af692d0..de4539b9998502 100644
--- a/include/dt-bindings/clock/qcom,nord-segcc.h
+++ b/include/dt-bindings/clock/qcom,nord-segcc.h
@@ -84,6 +84,8 @@
 #define SE_GCC_QUPV3_WRAP1_S6_CLK				74
 #define SE_GCC_QUPV3_WRAP1_S6_CLK_SRC				75
 #define SE_GCC_QUPV3_WRAP1_S_AHB_CLK				76
+#define SE_GCC_EMAC0_CNOC_APB_CLK				77
+#define SE_GCC_EMAC1_CNOC_APB_CLK				78
 
 /* SE_GCC power domains */
 #define SE_GCC_EMAC0_GDSC					0
diff --git a/include/dt-bindings/clock/qcom,qcs8300-gcc.h b/include/dt-bindings/clock/qcom,qcs8300-gcc.h
index a0083b1d212692..3c674c7baaba91 100644
--- a/include/dt-bindings/clock/qcom,qcs8300-gcc.h
+++ b/include/dt-bindings/clock/qcom,qcs8300-gcc.h
@@ -191,6 +191,11 @@
 #define GCC_VIDEO_AXI0_CLK					181
 #define GCC_VIDEO_AXI1_CLK					182
 #define GCC_VIDEO_XO_CLK					183
+#define GCC_GPLL5						184
+#define GCC_TSCSS_AHB_CLK					185
+#define GCC_TSCSS_CNTR_CLK_SRC					186
+#define GCC_TSCSS_ETU_CLK					187
+#define GCC_TSCSS_GLOBAL_CNTR_CLK				188
 
 /* GCC power domains */
 #define GCC_EMAC0_GDSC						0
@@ -230,5 +235,6 @@
 #define GCC_VIDEO_BCR						26
 #define GCC_VIDEO_AXI0_CLK_ARES					27
 #define GCC_VIDEO_AXI1_CLK_ARES					28
+#define GCC_TSCSS_BCR						29
 
 #endif
diff --git a/include/dt-bindings/clock/renesas,r8a774a3-cpg-mssr.h b/include/dt-bindings/clock/renesas,r8a774a3-cpg-mssr.h
new file mode 100644
index 00000000000000..09121a30e066e3
--- /dev/null
+++ b/include/dt-bindings/clock/renesas,r8a774a3-cpg-mssr.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+#ifndef __DT_BINDINGS_CLOCK_RENESAS_R8A774A3_CPG_MSSR_H__
+#define __DT_BINDINGS_CLOCK_RENESAS_R8A774A3_CPG_MSSR_H__
+
+#include <dt-bindings/clock/renesas-cpg-mssr.h>
+
+/* r8a774a3 CPG Core Clocks */
+#define R8A774A3_CLK_Z			0
+#define R8A774A3_CLK_Z2			1
+#define R8A774A3_CLK_ZG			2
+#define R8A774A3_CLK_ZTR		3
+#define R8A774A3_CLK_ZTRD2		4
+#define R8A774A3_CLK_ZT			5
+#define R8A774A3_CLK_ZX			6
+#define R8A774A3_CLK_S0D1		7
+#define R8A774A3_CLK_S0D2		8
+#define R8A774A3_CLK_S0D3		9
+#define R8A774A3_CLK_S0D4		10
+#define R8A774A3_CLK_S0D6		11
+#define R8A774A3_CLK_S0D8		12
+#define R8A774A3_CLK_S0D12		13
+#define R8A774A3_CLK_S1D2		14
+#define R8A774A3_CLK_S1D4		15
+#define R8A774A3_CLK_S2D1		16
+#define R8A774A3_CLK_S2D2		17
+#define R8A774A3_CLK_S2D4		18
+#define R8A774A3_CLK_S3D1		19
+#define R8A774A3_CLK_S3D2		20
+#define R8A774A3_CLK_S3D4		21
+#define R8A774A3_CLK_LB			22
+#define R8A774A3_CLK_CL			23
+#define R8A774A3_CLK_ZB3		24
+#define R8A774A3_CLK_ZB3D2		25
+#define R8A774A3_CLK_ZB3D4		26
+#define R8A774A3_CLK_CR			27
+#define R8A774A3_CLK_CRD2		28
+#define R8A774A3_CLK_SD0H		29
+#define R8A774A3_CLK_SD0		30
+#define R8A774A3_CLK_SD1H		31
+#define R8A774A3_CLK_SD1		32
+#define R8A774A3_CLK_SD2H		33
+#define R8A774A3_CLK_SD2		34
+#define R8A774A3_CLK_SD3H		35
+#define R8A774A3_CLK_SD3		36
+#define R8A774A3_CLK_RPC		37
+#define R8A774A3_CLK_RPCD2		38
+#define R8A774A3_CLK_MSO		39
+#define R8A774A3_CLK_CANFD		40
+#define R8A774A3_CLK_HDMI		41
+#define R8A774A3_CLK_CSI0		42
+#define R8A774A3_CLK_CP			43
+#define R8A774A3_CLK_CPEX		44
+#define R8A774A3_CLK_R			45
+#define R8A774A3_CLK_OSC		46
+
+#endif /* __DT_BINDINGS_CLOCK_RENESAS_R8A774A3_CPG_MSSR_H__ */
diff --git a/include/dt-bindings/clock/renesas,r8a78000-cpg.h b/include/dt-bindings/clock/renesas,r8a78000-cpg.h
index 7fd09a3fb353d9..14b3885f2d9933 100644
--- a/include/dt-bindings/clock/renesas,r8a78000-cpg.h
+++ b/include/dt-bindings/clock/renesas,r8a78000-cpg.h
@@ -10,5 +10,7 @@
 #define R8A78000_CPG_SGASYNCD4_PERW_BUS		0
 #define R8A78000_CPG_SGASYNCD8_PERW_BUS		1
 #define R8A78000_CPG_SGASYNCD16_PERW_BUS	2
+#define R8A78000_CPG_MSO_SCP_MAIN		3
+#define R8A78000_CPG_MSOCK_PERW_BUS		4
 
 #endif /* __DT_BINDINGS_CLOCK_RENESAS_R8A78000_CPG_H__ */
diff --git a/include/dt-bindings/clock/samsung,exynos5515-cmu.h b/include/dt-bindings/clock/samsung,exynos5515-cmu.h
new file mode 100644
index 00000000000000..2877d3681b55de
--- /dev/null
+++ b/include/dt-bindings/clock/samsung,exynos5515-cmu.h
@@ -0,0 +1,191 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) 2026 Aiden Isik <aidenisik+git@member.fsf.org>
+ * Author: Aiden Isik <aidenisik+git@member.fsf.org>
+ *
+ * Device Tree binding constants for Exynos5515 clock controller.
+ */
+
+#ifndef _DT_BINDINGS_CLOCK_EXYNOS5515_H
+#define _DT_BINDINGS_CLOCK_EXYNOS5515_H
+
+/* CMU_TOP */
+#define CLK_FOUT_AUD_PLL			1
+#define CLK_FOUT_SHARED0_PLL			2
+#define CLK_FOUT_SHARED1_PLL			3
+
+#define CLK_MOUT_CMU_APM_BUS			4
+#define CLK_MOUT_CMU_CORE_BUS			5
+#define CLK_MOUT_CMU_CORE_SSS			6
+#define CLK_MOUT_CMU_CPUCL0_BUSP		7
+#define CLK_MOUT_CMU_CPUCL0_CPU			8
+#define CLK_MOUT_CMU_CPUCL0_DBG			9
+#define CLK_MOUT_CMU_DPU_AUDIF			10
+#define CLK_MOUT_CMU_DPU_AUD_BUS		11
+#define CLK_MOUT_CMU_DPU_AUD_CPU		12
+#define CLK_MOUT_CMU_DPU_BUS			13
+#define CLK_MOUT_CMU_FSYS_BUS			14
+#define CLK_MOUT_CMU_FSYS_MMC_CARD		15
+#define CLK_MOUT_CMU_FSYS_MMC_EMBD		16
+#define CLK_MOUT_CMU_FSYS_USB20DRD		17
+#define CLK_MOUT_CMU_G3D_BUS			18
+#define CLK_MOUT_CMU_MFC_BUS			19
+#define CLK_MOUT_CMU_MIF_BUSP			20
+#define CLK_MOUT_CMU_MIF_SWITCH			21
+#define CLK_MOUT_CMU_MODEM_SHARED1		22
+#define CLK_MOUT_CMU_PERI_BUS			23
+#define CLK_MOUT_CMU_PERI_IP			24
+#define CLK_MOUT_CMU_PERI_UART			25
+#define CLK_MOUT_CMU_CMUREF			26
+
+#define CLK_DOUT_AP2CP_SHARED0_PLL		27
+#define CLK_DOUT_AP2CP_SHARED1_PLL		28
+#define CLK_DOUT_AP2CP_SHARED2_PLL		29
+#define CLK_DOUT_CMU_APM_BUS			30
+#define CLK_DOUT_CMU_CMU_BOOST			31
+#define CLK_DOUT_CMU_CORE_BUS			32
+#define CLK_DOUT_CMU_CORE_SSS			33
+#define CLK_DOUT_CMU_CPUCL0_BUSP		34
+#define CLK_DOUT_CMU_CPUCL0_CPU			35
+#define CLK_DOUT_CMU_CPUCL0_DBG			36
+#define CLK_DOUT_CMU_DPU_AUDIF			37
+#define CLK_DOUT_CMU_DPU_AUD_BUS		38
+#define CLK_DOUT_CMU_DPU_AUD_CPU		39
+#define CLK_DOUT_CMU_DPU_BUS			40
+#define CLK_DOUT_CMU_FSYS_BUS			41
+#define CLK_DOUT_CMU_FSYS_MMC_CARD		42
+#define CLK_DOUT_CMU_FSYS_MMC_EMBD		43
+#define CLK_DOUT_CMU_FSYS_USB20DRD		44
+#define CLK_DOUT_CMU_G3D_BUS			45
+#define CLK_DOUT_CMU_MFC_BUS			46
+#define CLK_DOUT_CMU_MIF_BUSP			47
+#define CLK_DOUT_CMU_PERI_BUS			48
+#define CLK_DOUT_CMU_PERI_IP			49
+#define CLK_DOUT_CMU_PERI_UART			50
+#define CLK_DOUT_CMU_CMUREF			51
+
+#define CLK_DOUT_SHARED0_DIV2			52
+#define CLK_DOUT_SHARED0_DIV3			53
+#define CLK_DOUT_SHARED0_DIV4			54
+#define CLK_DOUT_SHARED1_DIV2			55
+#define CLK_DOUT_SHARED1_DIV3			56
+#define CLK_DOUT_SHARED1_DIV4			57
+#define CLK_DOUT_AUD_DIV2			58
+#define CLK_DOUT_AUD_DIV3			59
+#define CLK_DOUT_AUD_DIV4			60
+
+#define CLK_GOUT_CMU_MIF_SWITCH			61
+#define CLK_GOUT_CMU_APM_BUS			62
+#define CLK_GOUT_CMU_CORE_BUS			63
+#define CLK_GOUT_CMU_CORE_SSS			64
+#define CLK_GOUT_CMU_CPUCL0_BUSP		65
+#define CLK_GOUT_CMU_CPUCL0_CPU			66
+#define CLK_GOUT_CMU_CPUCL0_DBG			67
+#define CLK_GOUT_CMU_DPU_AUDIF			68
+#define CLK_GOUT_CMU_DPU_AUD_BUS		69
+#define CLK_GOUT_CMU_DPU_AUD_CPU		70
+#define CLK_GOUT_CMU_DPU_BUS			71
+#define CLK_GOUT_CMU_FSYS_BUS			72
+#define CLK_GOUT_CMU_FSYS_MMC_CARD		73
+#define CLK_GOUT_CMU_FSYS_MMC_EMBD		74
+#define CLK_GOUT_CMU_FSYS_USB20DRD		75
+#define CLK_GOUT_CMU_G3D_BUS			76
+#define CLK_GOUT_CMU_MFC_BUS			77
+#define CLK_GOUT_CMU_MIF_BUSP			78
+#define CLK_GOUT_CMU_MODEM_SHARED0		79
+#define CLK_GOUT_CMU_MODEM_SHARED1		80
+#define CLK_GOUT_CMU_MODEM_SHARED2		81
+#define CLK_GOUT_CMU_PERI_BUS			82
+#define CLK_GOUT_CMU_PERI_IP			83
+#define CLK_GOUT_CMU_PERI_UART			84
+
+/* CMU_PERI */
+#define CLK_MOUT_PLL_PERI_BUS_USER		1
+#define CLK_MOUT_PLL_PERI_IP_USER		2
+#define CLK_MOUT_PLL_PERI_UART_USER		3
+#define CLK_MOUT_PERI_SPI			4
+#define CLK_MOUT_PERI_USI00_USI			5
+
+#define CLK_DOUT_PERI_OTP			6
+
+#define CLK_DOUT_PERI_I2C_0			7
+#define CLK_DOUT_PERI_I2C_1			8
+#define CLK_DOUT_PERI_I2C_2			9
+#define CLK_DOUT_PERI_I2C_3			10
+#define CLK_DOUT_PERI_SPI			11
+#define CLK_DOUT_PERI_USI00_I2C			12
+#define CLK_DOUT_PERI_USI00_USI			13
+
+#define CLK_GOUT_PERI_BUSIF_TMU_PCLK		14
+#define CLK_GOUT_PERI_D_TZPC_PCLK		15
+#define CLK_GOUT_PERI_GPIO_PCLK			16
+#define CLK_GOUT_PERI_I2C_0_IPCLK		17
+#define CLK_GOUT_PERI_I2C_0_PCLK		18
+#define CLK_GOUT_PERI_I2C_1_IPCLK		19
+#define CLK_GOUT_PERI_I2C_1_PCLK		20
+#define CLK_GOUT_PERI_I2C_2_IPCLK		21
+#define CLK_GOUT_PERI_I2C_2_PCLK		22
+#define CLK_GOUT_PERI_I2C_3_IPCLK		23
+#define CLK_GOUT_PERI_I2C_3_PCLK		24
+#define CLK_GOUT_PERI_MCT_PCLK			25
+#define CLK_GOUT_PERI_OTP_OSCCLK		26
+#define CLK_GOUT_PERI_OTP_PCLK			27
+#define CLK_GOUT_PERI_CMU_PCLK			28
+#define CLK_GOUT_PERI_PWM_MOTOR_PCLK		29
+#define CLK_GOUT_PERI_BUS_CLK			30
+#define CLK_GOUT_PERI_I2C_0_CLK			31
+#define CLK_GOUT_PERI_I2C_1_CLK			32
+#define CLK_GOUT_PERI_I2C_2_CLK			33
+#define CLK_GOUT_PERI_I2C_3_CLK			34
+#define CLK_GOUT_PERI_OSCCLK			35
+#define CLK_GOUT_PERI_SPI_CLK			36
+#define CLK_GOUT_PERI_UART_CLK			37
+#define CLK_GOUT_PERI_USI00_I2C_CLK		38
+#define CLK_GOUT_PERI_USI00_USI_CLK		39
+#define CLK_GOUT_PERI_SLH_AXI_MI_P_CLK		40
+#define CLK_GOUT_PERI_SYSREG_PCLK		41
+#define CLK_GOUT_PERI_USI00_I2C_IPCLK		42
+#define CLK_GOUT_PERI_USI00_I2C_PCLK		43
+#define CLK_GOUT_PERI_USI00_USI_IPCLK		44
+#define CLK_GOUT_PERI_USI00_USI_PCLK		45
+#define CLK_GOUT_PERI_USI_SPI_IPCLK		46
+#define CLK_GOUT_PERI_USI_SPI_PCLK		47
+#define CLK_GOUT_PERI_USI_UART_IPCLK		48
+#define CLK_GOUT_PERI_USI_UART_PCLK		49
+#define CLK_GOUT_PERI_WDT0_PCLK			50
+#define CLK_GOUT_PERI_WDT1_PCLK			51
+#define CLK_GOUT_PERI_I2C_0			52
+#define CLK_GOUT_PERI_I2C_1			53
+#define CLK_GOUT_PERI_I2C_2			54
+#define CLK_GOUT_PERI_I2C_3			55
+#define CLK_GOUT_PERI_USI00_I2C			56
+
+/* CMU_FSYS */
+#define CLK_MOUT_PLL_FSYS_BUS_USER		1
+#define CLK_MOUT_PLL_FSYS_MMC_CARD_USER		2
+#define CLK_MOUT_PLL_FSYS_MMC_EMBD_USER		3
+#define CLK_MOUT_PLL_FSYS_USB20DRD_USER		4
+
+#define CLK_GOUT_FSYS_CMU_PCLK			5
+#define CLK_GOUT_FSYS_MMC_CARD_SDCLKIN		6
+#define CLK_GOUT_FSYS_MMC_EMBD_SDCLKIN		7
+#define CLK_GOUT_FSYS_OSCCLK			8
+#define CLK_GOUT_FSYS_USB20DRD_REFCLK		9
+#define CLK_GOUT_FSYS_USB20DRD_PHY_REFCLK	10
+#define CLK_GOUT_FSYS_D_TZPC_PCLK		11
+#define CLK_GOUT_FSYS_GPIO_PCLK			12
+#define CLK_GOUT_FSYS_MMC_CARD_ACLK		13
+#define CLK_GOUT_FSYS_MMC_EMBD_ACLK		14
+#define CLK_GOUT_FSYS_PPMU_ACLK			15
+#define CLK_GOUT_FSYS_PPMU_PCLK			16
+#define CLK_GOUT_FSYS_BUS_CLK			17
+#define CLK_GOUT_FSYS_SLH_AXI_MI_P_CLK		18
+#define CLK_GOUT_FSYS_SLH_AXI_SI_D_CLK		19
+#define CLK_GOUT_FSYS_SYSREG_PCLK		20
+#define CLK_GOUT_FSYS_USB20DRD_ACLK_PHYCTRL	21
+#define CLK_GOUT_FSYS_USB20DRD_BUS_CLK_EARLY	22
+#define CLK_GOUT_FSYS_US_64TO128_ACLK		23
+#define CLK_GOUT_FSYS_XIU_D_ACLK		24
+#define CLK_GOUT_FSYS_XIU_P_ACLK		25
+
+#endif /* _DT_BINDINGS_CLOCK_EXYNOS5515_H */
diff --git a/include/dt-bindings/interconnect/qcom,ipq9650.h b/include/dt-bindings/interconnect/qcom,ipq9650.h
new file mode 100644
index 00000000000000..023a3878cc08e6
--- /dev/null
+++ b/include/dt-bindings/interconnect/qcom,ipq9650.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+#ifndef INTERCONNECT_QCOM_IPQ9650_H
+#define INTERCONNECT_QCOM_IPQ9650_H
+
+#define MASTER_ANOC_PCIE0		0
+#define SLAVE_ANOC_PCIE0		1
+#define MASTER_CNOC_PCIE0		2
+#define SLAVE_CNOC_PCIE0		3
+#define MASTER_ANOC_PCIE1		4
+#define SLAVE_ANOC_PCIE1		5
+#define MASTER_CNOC_PCIE1		6
+#define SLAVE_CNOC_PCIE1		7
+#define MASTER_ANOC_PCIE2		8
+#define SLAVE_ANOC_PCIE2		9
+#define MASTER_CNOC_PCIE2		10
+#define SLAVE_CNOC_PCIE2		11
+#define MASTER_ANOC_PCIE3		12
+#define SLAVE_ANOC_PCIE3		13
+#define MASTER_CNOC_PCIE3		14
+#define SLAVE_CNOC_PCIE3		15
+#define MASTER_ANOC_PCIE4		16
+#define SLAVE_ANOC_PCIE4		17
+#define MASTER_CNOC_PCIE4		18
+#define SLAVE_CNOC_PCIE4		19
+#define MASTER_SNOC_USB			20
+#define SLAVE_SNOC_USB			21
+
+#endif /* INTERCONNECT_QCOM_IPQ9650_H */
diff --git a/include/dt-bindings/interconnect/qcom,kuno.h b/include/dt-bindings/interconnect/qcom,kuno.h
new file mode 100644
index 00000000000000..eacbae38f85d07
--- /dev/null
+++ b/include/dt-bindings/interconnect/qcom,kuno.h
@@ -0,0 +1,89 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_KUNO_H
+#define __DT_BINDINGS_INTERCONNECT_QCOM_KUNO_H
+
+#define MASTER_SYS_TCU				0
+#define MASTER_LLCC				1
+#define MASTER_AUDIO				2
+#define MASTER_CPUCP				3
+#define MASTER_PCIE_RSCC				4
+#define MASTER_QDSS_BAM				5
+#define MASTER_QPIC				6
+#define MASTER_QUP_0				7
+#define MASTER_ANOC_SNOC				8
+#define MASTER_CNOC_DC_NOC				9
+#define MASTER_CNOC_SNOC				10
+#define MASTER_MSS_PROC				11
+#define MASTER_GEM_NOC_CFG				12
+#define MASTER_GEM_NOC_CNOC				13
+#define MASTER_GEM_NOC_PCIE_SNOC				14
+#define MASTER_ANOC_PCIE_GEM_NOC				15
+#define MASTER_SNOC_SF_MEM_NOC				16
+#define MASTER_QUP_CORE_0				17
+#define MASTER_CRYPTO				18
+#define MASTER_IPA				19
+#define MASTER_MSS_NAV				20
+#define MASTER_TME				21
+#define MASTER_APPSS_PROC				22
+#define MASTER_EMAC				23
+#define MASTER_IPA_PCIE				24
+#define MASTER_PCIE_0				25
+#define MASTER_QDSS_DAP				26
+#define MASTER_QDSS_ETR				27
+#define MASTER_QDSS_ETR_1				28
+#define MASTER_SDCC_4				29
+#define MASTER_USB3_0				30
+#define SLAVE_EBI1				31
+#define SLAVE_AHB2PHY				32
+#define SLAVE_AOSS				33
+#define SLAVE_APPSS				34
+#define SLAVE_AUDIO				35
+#define SLAVE_CLK_CTL				36
+#define SLAVE_RBCPR_CX_CFG				37
+#define SLAVE_RBCPR_MXA_CFG				38
+#define SLAVE_RBCPR_MXC_CFG				39
+#define SLAVE_CRYPTO_0_CFG				40
+#define SLAVE_EMAC_CFG				41
+#define SLAVE_IMEM_CFG				42
+#define SLAVE_IPA_CFG				43
+#define SLAVE_IPC_ROUTER_CFG				44
+#define SLAVE_LAGG_CFG				45
+#define SLAVE_MCCC_MASTER				46
+#define SLAVE_CNOC_MSS				47
+#define SLAVE_PCIE_0_CFG				48
+#define SLAVE_PCIE_RSC_CFG				49
+#define SLAVE_PDM				50
+#define SLAVE_PMU_WRAPPER_CFG				51
+#define SLAVE_PRNG				52
+#define SLAVE_QDSS_CFG				53
+#define SLAVE_QPIC				54
+#define SLAVE_QUP_0				55
+#define SLAVE_SDCC_4				56
+#define SLAVE_SPMI_VGI_COEX				57
+#define SLAVE_TCSR				58
+#define SLAVE_TLMM				59
+#define SLAVE_TME_CFG				60
+#define SLAVE_USB3				61
+#define SLAVE_VSENSE_CTRL_CFG				62
+#define SLAVE_A1NOC_CFG				63
+#define SLAVE_DDRSS_CFG				64
+#define SLAVE_LLCC				65
+#define SLAVE_GEM_NOC_CNOC				66
+#define SLAVE_SNOC_MEM_NOC_SF				67
+#define SLAVE_MEM_NOC_PCIE_SNOC				68
+#define SLAVE_ANOC_PCIE_GEM_NOC				69
+#define SLAVE_SNOC_CNOC				70
+#define SLAVE_ANOC_THROTTLE_CFG				71
+#define SLAVE_QUP_CORE_0				72
+#define SLAVE_BOOT_IMEM				73
+#define SLAVE_IMEM				74
+#define SLAVE_SERVICE_GEM_NOC				75
+#define SLAVE_PCIE_0				76
+#define SLAVE_QDSS_STM				77
+#define SLAVE_TCU				78
+
+#endif
diff --git a/include/dt-bindings/media/video-interface-devices.h b/include/dt-bindings/media/video-interface-devices.h
new file mode 100644
index 00000000000000..d2340b4572927a
--- /dev/null
+++ b/include/dt-bindings/media/video-interface-devices.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR MIT) */
+/*
+ * Copyright (C) 2026 Kieran Bingham <kieran.bingham@ideasonboard.com>
+ */
+
+#ifndef __DT_BINDINGS_MEDIA_VIDEO_INTERFACE_DEVICES_H__
+#define __DT_BINDINGS_MEDIA_VIDEO_INTERFACE_DEVICES_H__
+
+#define MEDIA_ORIENTATION_FRONT		0
+#define MEDIA_ORIENTATION_BACK		1
+#define MEDIA_ORIENTATION_EXTERNAL	2
+
+#endif /* __DT_BINDINGS_MEDIA_VIDEO_INTERFACE_DEVICES_H__ */
diff --git a/include/dt-bindings/phy/phy-qcom-qmp.h b/include/dt-bindings/phy/phy-qcom-qmp.h
index 6b43ea9e005185..1c3ce0c02b0cd3 100644
--- a/include/dt-bindings/phy/phy-qcom-qmp.h
+++ b/include/dt-bindings/phy/phy-qcom-qmp.h
@@ -16,6 +16,7 @@
 /* QMP USB4-USB3-DP PHYs */
 #define QMP_USB43DP_USB3_PHY		0
 #define QMP_USB43DP_DP_PHY		1
+#define QMP_USB43DP_USB4_PHY		2
 
 /* QMP PCIE PHYs */
 #define QMP_PCIE_PIPE_CLK		0
diff --git a/include/dt-bindings/power/qcom,rpmhpd.h b/include/dt-bindings/power/qcom,rpmhpd.h
index 07bd2a7b0150b4..92c8408dbc57df 100644
--- a/include/dt-bindings/power/qcom,rpmhpd.h
+++ b/include/dt-bindings/power/qcom,rpmhpd.h
@@ -32,6 +32,7 @@
 #define RPMHPD_GBX		22
 #define RPMHPD_NSP3		23
 #define RPMHPD_GFX1		24
+#define RPMHPD_NMXC		25
 
 /* RPMh Power Domain performance levels */
 #define RPMH_REGULATOR_LEVEL_RETENTION		16
diff --git a/include/dt-bindings/power/renesas,r8a774a3-sysc.h b/include/dt-bindings/power/renesas,r8a774a3-sysc.h
new file mode 100644
index 00000000000000..ddee6812e9a44b
--- /dev/null
+++ b/include/dt-bindings/power/renesas,r8a774a3-sysc.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ */
+#ifndef __DT_BINDINGS_POWER_RENESAS_R8A774A3_SYSC_H__
+#define __DT_BINDINGS_POWER_RENESAS_R8A774A3_SYSC_H__
+
+/*
+ * These power domain indices match the numbers of the interrupt bits
+ * representing the power areas in the various Interrupt Registers
+ * (e.g. SYSCISR, Interrupt Status Register)
+ */
+
+#define R8A774A3_PD_CA57_CPU0		 0
+#define R8A774A3_PD_CA57_CPU1		 1
+#define R8A774A3_PD_CA53_CPU0		 5
+#define R8A774A3_PD_CA53_CPU1		 6
+#define R8A774A3_PD_CA53_CPU2		 7
+#define R8A774A3_PD_CA53_CPU3		 8
+#define R8A774A3_PD_CA57_SCU		12
+#define R8A774A3_PD_A3VC		14
+#define R8A774A3_PD_3DG_A		17
+#define R8A774A3_PD_3DG_B		18
+#define R8A774A3_PD_CA53_SCU		21
+#define R8A774A3_PD_A2VC1		26
+
+/* Always-on power area */
+#define R8A774A3_PD_ALWAYS_ON		32
+
+#endif /* __DT_BINDINGS_POWER_RENESAS_R8A774A3_SYSC_H__ */
diff --git a/include/dt-bindings/reset/anlogic,dr1v90-cru.h b/include/dt-bindings/reset/anlogic,dr1v90-cru.h
new file mode 100644
index 00000000000000..c5b7aae1ab6ac7
--- /dev/null
+++ b/include/dt-bindings/reset/anlogic,dr1v90-cru.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (C) 2025 Junhui Liu <junhui.liu@pigmoral.tech>
+ */
+
+#ifndef _DT_BINDINGS_RESET_ANLOGIC_DR1V90_CRU_H_
+#define _DT_BINDINGS_RESET_ANLOGIC_DR1V90_CRU_H_
+
+#define RESET_OCM		0
+#define RESET_QSPI		1
+#define RESET_SMC		2
+#define RESET_WDT		3
+#define RESET_DMAC_AXI		4
+#define RESET_DMAC_AHB		5
+#define RESET_NPU		6
+#define RESET_JPU		7
+#define RESET_DDRBUS		8
+#define RESET_NIC_HP0		9
+#define RESET_NIC_HP1		10
+#define RESET_NIC_GP0M		11
+#define RESET_NIC_GP1M		12
+#define RESET_GPIO		13
+#define RESET_IPC		14
+#define RESET_USB0		15
+#define RESET_USB1		16
+#define RESET_GBE0		17
+#define RESET_GBE1		18
+#define RESET_SDIO0		19
+#define RESET_SDIO1		20
+#define RESET_UART0		21
+#define RESET_UART1		22
+#define RESET_SPI0		23
+#define RESET_SPI1		24
+#define RESET_CAN0		25
+#define RESET_CAN1		26
+#define RESET_TTC0		27
+#define RESET_TTC1		28
+#define RESET_I2C0		29
+#define RESET_I2C1		30
+
+#endif /* _DT_BINDINGS_RESET_ANLOGIC_DR1V90_CRU_H_ */
diff --git a/include/dt-bindings/reset/mediatek,mt8167-wdt.h b/include/dt-bindings/reset/mediatek,mt8167-wdt.h
new file mode 100644
index 00000000000000..1a24bfe079cac8
--- /dev/null
+++ b/include/dt-bindings/reset/mediatek,mt8167-wdt.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
+#ifndef _DT_BINDINGS_RESET_MEDIATEK_MT8167_WDT_H_
+#define _DT_BINDINGS_RESET_MEDIATEK_MT8167_WDT_H_
+
+/* TOPRGU reset IDs */
+#define MT8167_TOPRGU_DDRPHY_FLASH_RST		0
+#define MT8167_TOPRGU_AUD_PAD_RST		1
+#define MT8167_TOPRGU_MM_RST			2
+#define MT8167_TOPRGU_MFG_RST			3
+#define MT8167_TOPRGU_MDSYS_RST			4
+#define MT8167_TOPRGU_CONN_RST			5
+#define MT8167_TOPRGU_PAD2CAM_DIG_MIPI_RX_RST	6
+#define MT8167_TOPRGU_DIG_MIPI_TX_RST		7
+#define MT8167_TOPRGU_SPI_PAD_MACRO_RST		8
+#define MT8167_TOPRGU_APMIXED_RST		9
+#define MT8167_TOPRGU_VDEC_RST			10
+#define MT8167_TOPRGU_CONN_MCU_RST		11
+#define MT8167_TOPRGU_EFUSE_RST			12
+#define MT8167_TOPRGU_PWRAP_SPICTL_RST		13
+
+#endif  /* _DT_BINDINGS_RESET_MEDIATEK_MT8167_WDT_H_ */
diff --git a/include/dt-bindings/soc/airoha,scu-ssr.h b/include/dt-bindings/soc/airoha,scu-ssr.h
new file mode 100644
index 00000000000000..33c64844ada3f2
--- /dev/null
+++ b/include/dt-bindings/soc/airoha,scu-ssr.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+
+#ifndef __DT_BINDINGS_AIROHA_SCU_SSR_H
+#define __DT_BINDINGS_AIROHA_SCU_SSR_H
+
+#define AIROHA_SCU_SERDES_PCIE1		0
+#define AIROHA_SCU_SERDES_PCIE2		1
+#define AIROHA_SCU_SERDES_USB1		2
+#define AIROHA_SCU_SERDES_USB2		3
+
+#endif /* __DT_BINDINGS_AIROHA_SCU_SSR_H */
diff --git a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
index 3a99703dbc4a0d..e6c792d9312fc5 100644
--- a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
+++ b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
@@ -156,6 +156,58 @@
 #define LPI_MI2S_TX_5		150
 #define LPI_MI2S_RX_6		151
 #define LPI_MI2S_TX_6		152
+#define AIF_MI2S_RX_0		153
+#define AIF_MI2S_TX_0		154
+#define AIF_MI2S_RX_1		155
+#define AIF_MI2S_TX_1		156
+#define AIF_MI2S_RX_2		157
+#define AIF_MI2S_TX_2		158
+#define AIF_MI2S_RX_3		159
+#define AIF_MI2S_TX_3		160
+#define AIF_MI2S_RX_4		161
+#define AIF_MI2S_TX_4		162
+#define AIF_MI2S_RX_5		163
+#define AIF_MI2S_TX_5		164
+#define AIF_MI2S_RX_6		165
+#define AIF_MI2S_TX_6		166
+#define AIF_MI2S_RX_7		167
+#define AIF_MI2S_TX_7		168
+#define AIF_MI2S_RX_8		169
+#define AIF_MI2S_TX_8		170
+#define AIF_MI2S_RX_9		171
+#define AIF_MI2S_TX_9		172
+#define AIF_MI2S_RX_10		173
+#define AIF_MI2S_TX_10		174
+#define AIF_MI2S_RX_11		175
+#define AIF_MI2S_TX_11		176
+#define AIF_MI2S_RX_12		177
+#define AIF_MI2S_TX_12		178
+#define AIF_TDM_RX_0		179
+#define AIF_TDM_TX_0		180
+#define AIF_TDM_RX_1		181
+#define AIF_TDM_TX_1		182
+#define AIF_TDM_RX_2		183
+#define AIF_TDM_TX_2		184
+#define AIF_TDM_RX_3		185
+#define AIF_TDM_TX_3		186
+#define AIF_TDM_RX_4		187
+#define AIF_TDM_TX_4		188
+#define AIF_TDM_RX_5		189
+#define AIF_TDM_TX_5		190
+#define AIF_TDM_RX_6		191
+#define AIF_TDM_TX_6		192
+#define AIF_TDM_RX_7		193
+#define AIF_TDM_TX_7		194
+#define AIF_TDM_RX_8		195
+#define AIF_TDM_TX_8		196
+#define AIF_TDM_RX_9		197
+#define AIF_TDM_TX_9		198
+#define AIF_TDM_RX_10		199
+#define AIF_TDM_TX_10		200
+#define AIF_TDM_RX_11		201
+#define AIF_TDM_TX_11		202
+#define AIF_TDM_RX_12		203
+#define AIF_TDM_TX_12		204
 
 #define LPASS_CLK_ID_PRI_MI2S_IBIT	1
 #define LPASS_CLK_ID_PRI_MI2S_EBIT	2
diff --git a/include/dt-bindings/thermal/qcom,pas.h b/include/dt-bindings/thermal/qcom,pas.h
new file mode 100644
index 00000000000000..f603bddef9a8d2
--- /dev/null
+++ b/include/dt-bindings/thermal/qcom,pas.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
+/*
+ * Qualcomm PAS remoteproc cooling device indices
+ *
+ * These indices are used in device tree cooling-maps to reference
+ * specific TMD devices provided by PAS-managed remote processors via QMI.
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#ifndef _DT_BINDINGS_THERMAL_QCOM_PAS_H
+#define _DT_BINDINGS_THERMAL_QCOM_PAS_H
+
+/* CDSP thermal mitigation device id */
+#define QCOM_TMD_CDSP_SW	0
+
+/* Modem thermal mitigation device id */
+#define QCOM_TMD_PA	0
+#define QCOM_TMD_MODEM	1
+
+#endif /* _DT_BINDINGS_THERMAL_QCOM_PAS_H */
diff --git a/include/kunit/clk.h b/include/kunit/clk.h
index f226044cc78d11..0b5ab8e971c9bf 100644
--- a/include/kunit/clk.h
+++ b/include/kunit/clk.h
@@ -29,5 +29,7 @@ int of_clk_hw_register_kunit(struct kunit *test, struct device_node *node,
 int of_clk_add_hw_provider_kunit(struct kunit *test, struct device_node *np,
 				 struct clk_hw *(*get)(struct of_phandle_args *clkspec, void *data),
 				 void *data);
+struct device_node *clk_of_find_node_by_name_kunit(struct kunit *test, struct device_node *from,
+						   const char *name);
 
 #endif
diff --git a/include/kunit/test.h b/include/kunit/test.h
index da5312e0dfa57f..31d549578f8ec1 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -400,11 +400,10 @@ struct kunit_suite_set kunit_merge_suite_sets(struct kunit_suite_set init_suite_
 const void *kunit_array_gen_params(struct kunit *test, const void *prev, char *desc);
 
 #if IS_BUILTIN(CONFIG_KUNIT)
-int kunit_run_all_tests(void);
+void kunit_run_all_tests(void);
 #else
-static inline int kunit_run_all_tests(void)
+static inline void kunit_run_all_tests(void)
 {
-	return 0;
 }
 #endif /* IS_BUILTIN(CONFIG_KUNIT) */
 
diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h
index 086b7578e5f330..7be81b936bf10a 100644
--- a/include/kvm/arm_vgic.h
+++ b/include/kvm/arm_vgic.h
@@ -31,7 +31,12 @@
 #define VGIC_MAX_SPI		1019
 #define VGIC_MAX_RESERVED	1023
 #define VGIC_MIN_LPI		8192
-#define KVM_IRQCHIP_NUM_PINS	(1020 - 32)
+
+/*
+ * The common IRQ routing table must accommodate all GIC types. GICv2 and
+ * GICv3 need 988 pins, while GICv5 needs 1024 pins.
+ */
+#define KVM_IRQCHIP_NUM_PINS	1024
 
 /*
  * GICv5 supports 128 PPIs, but only the first 64 are architected. We only
@@ -40,6 +45,8 @@
  * in KVM for now. At a future stage, this can be bumped up to 128, if required.
  */
 #define VGIC_V5_NR_PRIVATE_IRQS	64
+#define VGIC_V5_DEFAULT_NR_SPIS	32
+#define VGIC_V5_MAX_NR_SPIS	BIT(10)
 
 #define is_v5_type(t, i)	(FIELD_GET(GICV5_HWIRQ_TYPE, (i)) == (t))
 
@@ -132,6 +139,19 @@ enum vgic_type {
 	VGIC_V5,		/* Newer, fancier GICv5 */
 };
 
+struct vgic_v5_irs_caps {
+	/* Base address of the host IRS's CONFIG_FRAME */
+	void __iomem	*irs_base;
+
+	/* Raw host IRS ID registers */
+	u32		idr2;
+	u32		idr3;
+	u32		idr4;
+
+	/* Is the IRS coherent with us, or not? */
+	bool		non_coherent;
+};
+
 /* same for all guests, as depending only on the _host's_ GIC model */
 struct vgic_global {
 	/* type of the host GIC */
@@ -159,9 +179,16 @@ struct vgic_global {
 	/* Maintenance IRQ number */
 	unsigned int		maint_irq;
 
-	/* maximum number of VCPUs allowed (GICv2 limits us to 8) */
+	/*
+	 * Maximum number of VCPUs exposed before userspace has selected a
+	 * VGIC model. Individual VGIC models can impose a lower limit
+	 * (GICv2 limits us to 8).
+	 */
 	int			max_gic_vcpus;
 
+	/* Maximum number of VCPUs allowed for a GICv5 VM. */
+	int			max_gicv5_vcpus;
+
 	/* Only needed for the legacy KVM_CREATE_IRQCHIP */
 	bool			can_emulate_gicv2;
 
@@ -182,6 +209,9 @@ struct vgic_global {
 	struct {
 		DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS);
 	} vgic_v5_ppi_caps;
+
+	/* GICv5 IRS capabilities */
+	struct vgic_v5_irs_caps vgic_v5_irs_caps;
 };
 
 extern struct vgic_global kvm_vgic_global_state;
@@ -216,6 +246,12 @@ struct irq_ops {
 	 */
 	bool (*get_input_level)(int vintid);
 
+	/*
+	 * Function pointer to directly update hardware pending state after the
+	 * VGIC shadow pending state has changed.
+	 */
+	bool (*set_pending_state)(struct kvm_vcpu *vcpu, struct vgic_irq *irq);
+
 	/*
 	 * Function pointer to override the queuing of an IRQ.
 	 */
@@ -287,7 +323,8 @@ enum iodev_type {
 	IODEV_CPUIF,
 	IODEV_DIST,
 	IODEV_REDIST,
-	IODEV_ITS
+	IODEV_ITS,
+	IODEV_GICV5_IRS
 };
 
 struct vgic_io_device {
@@ -345,7 +382,11 @@ struct vgic_redist_region {
 	struct list_head list;
 };
 
+#define VGIC_V5_VM_ID_INVAL		(-1)
+
 struct vgic_v5_vm {
+	struct kvm		*kvm;
+
 	/*
 	 * We only expose a subset of PPIs to the guest. This subset is a
 	 * combination of the PPIs that are actually implemented and what we
@@ -363,6 +404,96 @@ struct vgic_v5_vm {
 	 * convenient way to do that).
 	 */
 	DECLARE_BITMAP(vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS);
+
+	struct fwnode_handle	*fwnode;
+	struct irq_domain	*domain;
+	int			vpe_db_base;
+	u32			vm_id;
+	bool			vmte_allocated;
+};
+
+/*** GICv5 ***/
+struct vgic_v5_irs {
+	/* base addresses in guest physical address space: */
+	gpa_t vgic_v5_irs_base;
+
+	struct vgic_io_device iodev;
+	struct kvm_device *dev;
+
+	/* IRS state - used for registers etc */
+	struct {
+		u8 domain;
+		u8 pa_range;
+		bool virt;
+		bool setlpi;
+		bool mec;
+		bool mpam;
+		bool swe;
+		u16 irs_id;
+	} idr0;
+
+	struct {
+		/* PE_CNT is populated from online_vcpus at runtime */
+		u8 priority_bits;
+	} idr1;
+
+	struct {
+		u8 id_bits;
+		u8 min_lpi_id_bits;
+		bool ist_levels;
+		u8 ist_l2sz;
+		bool istmd;
+		u8 istmd_sz;
+	} idr2;
+
+	struct {
+		u32 spi_range;
+	} idr5;
+
+	struct {
+		u32 spi_irs_range;
+	} idr6;
+
+	struct {
+		u32 spi_base;
+	} idr7;
+
+	struct {
+		u8 sh;
+		u8 oc;
+		u8 ic;
+		bool ist_ra;
+		bool ist_wa;
+		bool vmt_ra;
+		bool vpet_ra;
+		bool vmd_ra;
+		bool vmd_wa;
+		bool vped_ra;
+		bool vped_wa;
+	} cr1;
+
+	struct {
+		u32 id;
+	} spi_selr;
+
+	struct {
+		u32 iaffid;
+	} pe_selr;
+
+	struct {
+		u8 lpi_id_bits;
+		u8 l2sz;
+		u8 istsz;
+		bool structure;
+	} ist_cfgr;
+
+	struct {
+		bool valid;
+		u64 addr;
+	} ist_baser;
+
+	/* A valid userspace-restored BASER still needs its host LPI IST. */
+	bool lpi_ist_restore_pending;
 };
 
 struct vgic_dist {
@@ -442,6 +573,11 @@ struct vgic_dist {
 	 * GICv5 per-VM data.
 	 */
 	struct vgic_v5_vm	gicv5_vm;
+
+	/*
+	 * GICv5 IRS data. Dynamically allocated due to the size.
+	 */
+	struct vgic_v5_irs	*vgic_v5_irs_data;
 };
 
 struct vgic_v2_cpu_if {
@@ -492,6 +628,9 @@ struct vgic_v5_cpu_if {
 	 */
 	u64	vgic_icsr;
 
+	/* The contextr used to make VPEs resident and non-resident */
+	u64	vgic_contextr;
+
 	struct gicv5_vpe gicv5_vpe;
 };
 
@@ -598,10 +737,11 @@ void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu);
 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1);
 
 /**
- * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
+ * kvm_vgic_get_max_vcpus - Get the pre-VGIC-selection VCPU limit
  *
- * The host's GIC naturally limits the maximum amount of VCPUs a guest
- * can use.
+ * Userspace can query KVM_CAP_MAX_VCPUS before selecting a VGIC model, so
+ * expose the highest model-specific limit and let kvm_vgic_create() enforce
+ * the selected model's actual limit.
  */
 static inline int kvm_vgic_get_max_vcpus(void)
 {
@@ -627,7 +767,66 @@ int vgic_v4_load(struct kvm_vcpu *vcpu);
 void vgic_v4_commit(struct kvm_vcpu *vcpu);
 int vgic_v4_put(struct kvm_vcpu *vcpu);
 
-int vgic_v5_finalize_ppi_state(struct kvm *kvm);
+static inline u8 vgic_v5_irs_ist_id_bits(const struct vgic_v5_irs_caps *caps)
+{
+	return FIELD_GET(GICV5_IRS_IDR2_ID_BITS, caps->idr2);
+}
+
+static inline u8 vgic_v5_irs_min_lpi_id_bits(const struct vgic_v5_irs_caps *caps)
+{
+	return FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, caps->idr2);
+}
+
+static inline bool vgic_v5_irs_ist_levels(const struct vgic_v5_irs_caps *caps)
+{
+	return !!(caps->idr2 & GICV5_IRS_IDR2_IST_LEVELS);
+}
+
+static inline u8 vgic_v5_irs_ist_l2sz(const struct vgic_v5_irs_caps *caps)
+{
+	return FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, caps->idr2);
+}
+
+static inline bool vgic_v5_irs_istmd(const struct vgic_v5_irs_caps *caps)
+{
+	return !!(caps->idr2 & GICV5_IRS_IDR2_ISTMD);
+}
+
+static inline u8 vgic_v5_irs_istmd_sz(const struct vgic_v5_irs_caps *caps)
+{
+	return FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, caps->idr2);
+}
+
+static inline u32 vgic_v5_irs_max_vms(const struct vgic_v5_irs_caps *caps)
+{
+	return BIT(FIELD_GET(GICV5_IRS_IDR3_VM_ID_BITS, caps->idr3));
+}
+
+static inline bool vgic_v5_irs_two_level_vmt_support(const struct vgic_v5_irs_caps *caps)
+{
+	return !!(caps->idr3 & GICV5_IRS_IDR3_VMT_LEVELS);
+}
+
+static inline u16 vgic_v5_irs_vmd_size(const struct vgic_v5_irs_caps *caps)
+{
+	if (!(caps->idr3 & GICV5_IRS_IDR3_VMD))
+		return 0;
+
+	return BIT(FIELD_GET(GICV5_IRS_IDR3_VMD_SZ, caps->idr3));
+}
+
+static inline u16 vgic_v5_irs_vped_size(const struct vgic_v5_irs_caps *caps)
+{
+	return BIT(FIELD_GET(GICV5_IRS_IDR4_VPED_SZ, caps->idr4));
+}
+
+static inline u32 vgic_v5_irs_max_vpes(const struct vgic_v5_irs_caps *caps)
+{
+	/* Field stores VPE_ID_BITS - 1 */
+	return BIT(FIELD_GET(GICV5_IRS_IDR4_VPE_ID_BITS, caps->idr4) + 1);
+}
+
+int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu);
 bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
 				  unsigned long flags);
 void vgic_v5_set_ppi_dvi(struct kvm_vcpu *vcpu, struct vgic_irq *irq, bool dvi);
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index ddacac81209473..aef0a01f4cfaef 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -25,17 +25,19 @@ struct irq_domain_ops;
 #define _LINUX
 #endif
 #include <acpi/acpi.h>
+#include <acpi/acpi_bus.h>
 #include <acpi/acpi_numa.h>
 
 #ifdef	CONFIG_ACPI
 
+DEFINE_FREE(acpi_object_free, union acpi_object *, if (_T) ACPI_FREE(_T));
+
 #include <linux/list.h>
 #include <linux/dynamic_debug.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/fw_table.h>
 
-#include <acpi/acpi_bus.h>
 #include <acpi/acpi_drivers.h>
 #include <acpi/acpi_io.h>
 #include <asm/acpi.h>
@@ -434,7 +436,7 @@ extern acpi_handle ec_get_handle(void);
 
 extern bool acpi_is_pnp_device(struct acpi_device *);
 
-#if defined(CONFIG_ACPI_WMI) || defined(CONFIG_ACPI_WMI_MODULE)
+#if IS_ENABLED(CONFIG_ACPI_WMI)
 
 typedef void (*wmi_notify_handler) (union acpi_object *data, void *context);
 
@@ -1305,34 +1307,34 @@ void __acpi_handle_debug(struct _ddebug *descriptor, acpi_handle handle, const c
 #endif
 
 /*
- * acpi_handle_<level>: Print message with ACPI prefix and object path
+ * acpi_handle_<level> - Print a message with ACPI prefix and object path
  *
- * These interfaces acquire the global namespace mutex to obtain an object
- * path.  In interrupt context, it shows the object path as <n/a>.
+ * In thread context, the global namespace mutex is acquired to obtain the
+ * object path.  In interrupt context, the object path is shown as <n/a>.
  */
 #define acpi_handle_emerg(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_EMERG, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_EMERG, handle, dev_fmt(fmt), ##__VA_ARGS__)
 #define acpi_handle_alert(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_ALERT, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_ALERT, handle, dev_fmt(fmt), ##__VA_ARGS__)
 #define acpi_handle_crit(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_CRIT, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_CRIT, handle, dev_fmt(fmt), ##__VA_ARGS__)
 #define acpi_handle_err(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_ERR, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_ERR, handle, dev_fmt(fmt), ##__VA_ARGS__)
 #define acpi_handle_warn(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_WARNING, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_WARNING, handle, dev_fmt(fmt), ##__VA_ARGS__)
 #define acpi_handle_notice(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_NOTICE, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_NOTICE, handle, dev_fmt(fmt), ##__VA_ARGS__)
 #define acpi_handle_info(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_INFO, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_INFO, handle, dev_fmt(fmt), ##__VA_ARGS__)
 
 #if defined(DEBUG)
 #define acpi_handle_debug(handle, fmt, ...)				\
-	acpi_handle_printk(KERN_DEBUG, handle, fmt, ##__VA_ARGS__)
+	acpi_handle_printk(KERN_DEBUG, handle, dev_fmt(fmt), ##__VA_ARGS__)
 #else
 #if defined(CONFIG_DYNAMIC_DEBUG)
 #define acpi_handle_debug(handle, fmt, ...)				\
 	_dynamic_func_call(fmt, __acpi_handle_debug,			\
-			   handle, pr_fmt(fmt), ##__VA_ARGS__)
+			   handle, dev_fmt(fmt), ##__VA_ARGS__)
 #else
 #define acpi_handle_debug(handle, fmt, ...)				\
 ({									\
diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h
index edcee9f5335a6f..104817e8d8977f 100644
--- a/include/linux/amd-iommu.h
+++ b/include/linux/amd-iommu.h
@@ -11,11 +11,11 @@
 #include <linux/types.h>
 
 struct amd_iommu;
+struct pci_dev;
 
 #ifdef CONFIG_AMD_IOMMU
 
 struct task_struct;
-struct pci_dev;
 
 extern void amd_iommu_detect(void);
 
@@ -76,4 +76,15 @@ static inline int amd_iommu_snp_disable(void) { return 0; }
 static inline bool amd_iommu_sev_tio_supported(void) { return false; }
 #endif
 
+#ifdef CONFIG_AMD_IOMMU
+int amd_iommu_enable_perfopt(struct pci_dev *pdev);
+void amd_iommu_disable_perfopt(struct pci_dev *pdev);
+#else
+static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev)
+{
+	return 0;
+}
+static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { }
+#endif
+
 #endif /* _ASM_X86_AMD_IOMMU_H */
diff --git a/include/linux/apple-gmux.h b/include/linux/apple-gmux.h
index 206d97ffda79ae..d7939c3a08fdc4 100644
--- a/include/linux/apple-gmux.h
+++ b/include/linux/apple-gmux.h
@@ -109,7 +109,7 @@ static inline bool apple_gmux_detect(struct pnp_dev *pnp_dev, enum apple_gmux_ty
 		if (!adev)
 			return false;
 
-		dev = get_device(acpi_get_first_physical_node(adev));
+		dev = acpi_bus_get_primary_device(adev);
 		acpi_dev_put(adev);
 		if (!dev)
 			return false;
diff --git a/include/linux/arm-rsi-cmds.h b/include/linux/arm-rsi-cmds.h
new file mode 100644
index 00000000000000..3f7a6a833993b2
--- /dev/null
+++ b/include/linux/arm-rsi-cmds.h
@@ -0,0 +1,239 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2023 ARM Ltd.
+ */
+
+#ifndef __LINUX_ARM_RSI_CMDS_H_
+#define __LINUX_ARM_RSI_CMDS_H_
+
+#include <linux/arm-smccc-rsi.h>
+#include <linux/jump_label.h>
+#include <linux/string.h>
+#include <asm/memory.h>
+
+#ifdef CONFIG_ARM_RMM_RSI
+DECLARE_STATIC_KEY_FALSE(rsi_present);
+
+void __init arm64_rsi_init(void);
+
+bool arm64_rsi_is_protected(phys_addr_t base, size_t size);
+
+static inline bool is_realm_world(void)
+{
+	return static_branch_unlikely(&rsi_present);
+}
+#else
+static inline void arm64_rsi_init(void) { }
+
+static inline bool arm64_rsi_is_protected(phys_addr_t base, size_t size)
+{
+	return false;
+}
+
+static inline bool is_realm_world(void) { return false; }
+#endif
+
+#define RSI_GRANULE_SHIFT		12
+#define RSI_GRANULE_SIZE		(_AC(1, UL) << RSI_GRANULE_SHIFT)
+
+enum ripas {
+	RSI_RIPAS_EMPTY = 0,
+	RSI_RIPAS_RAM = 1,
+	RSI_RIPAS_DESTROYED = 2,
+	RSI_RIPAS_DEV = 3,
+};
+
+static inline unsigned long rsi_request_version(unsigned long req,
+						unsigned long *out_lower,
+						unsigned long *out_higher)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_smc(SMC_RSI_ABI_VERSION, req, 0, 0, 0, 0, 0, 0, &res);
+
+	if (out_lower)
+		*out_lower = res.a1;
+	if (out_higher)
+		*out_higher = res.a2;
+
+	return res.a0;
+}
+
+static inline unsigned long rsi_get_realm_config(struct realm_config *cfg)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_smc(SMC_RSI_REALM_CONFIG, virt_to_phys(cfg),
+		      0, 0, 0, 0, 0, 0, &res);
+	return res.a0;
+}
+
+static inline unsigned long rsi_ipa_state_get(phys_addr_t start,
+					      phys_addr_t end,
+					      enum ripas *state,
+					      phys_addr_t *top)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_smc(SMC_RSI_IPA_STATE_GET,
+		      start, end, 0, 0, 0, 0, 0,
+		      &res);
+
+	if (res.a0 == RSI_SUCCESS) {
+		if (top)
+			*top = res.a1;
+		if (state)
+			*state = res.a2;
+	}
+
+	return res.a0;
+}
+
+static inline long rsi_set_addr_range_state(phys_addr_t start,
+					    phys_addr_t end,
+					    enum ripas state,
+					    unsigned long flags,
+					    phys_addr_t *top)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_smc(SMC_RSI_IPA_STATE_SET, start, end, state,
+		      flags, 0, 0, 0, &res);
+
+	if (top)
+		*top = res.a1;
+
+	if (res.a2 != RSI_ACCEPT)
+		return -EPERM;
+
+	return res.a0;
+}
+
+static inline int rsi_set_memory_range(phys_addr_t start, phys_addr_t end,
+				       enum ripas state, unsigned long flags)
+{
+	unsigned long ret;
+	phys_addr_t top;
+
+	while (start != end) {
+		ret = rsi_set_addr_range_state(start, end, state, flags, &top);
+		if (ret || top < start || top > end)
+			return -EINVAL;
+		start = top;
+	}
+
+	return 0;
+}
+
+/*
+ * Convert the specified range to RAM. Do not use this if you rely on the
+ * contents of a page that may already be in RAM state.
+ */
+static inline int rsi_set_memory_range_protected(phys_addr_t start,
+						 phys_addr_t end)
+{
+	return rsi_set_memory_range(start, end, RSI_RIPAS_RAM,
+				    RSI_CHANGE_DESTROYED);
+}
+
+/*
+ * Convert the specified range to RAM. Do not convert any pages that may have
+ * been DESTROYED, without our permission.
+ */
+static inline int rsi_set_memory_range_protected_safe(phys_addr_t start,
+						      phys_addr_t end)
+{
+	return rsi_set_memory_range(start, end, RSI_RIPAS_RAM,
+				    RSI_NO_CHANGE_DESTROYED);
+}
+
+static inline int rsi_set_memory_range_shared(phys_addr_t start,
+					      phys_addr_t end)
+{
+	return rsi_set_memory_range(start, end, RSI_RIPAS_EMPTY,
+				    RSI_CHANGE_DESTROYED);
+}
+
+#define RSI_ATTEST_CHALLENGE_MIN_SIZE	32
+#define RSI_ATTEST_CHALLENGE_MAX_SIZE	64
+
+struct rsi_attestation_token_init_args {
+	unsigned long fid;
+	u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE];
+};
+
+/**
+ * rsi_attestation_token_init - Initialise the operation to retrieve an
+ * attestation token.
+ *
+ * @challenge:	The challenge data to be used in the attestation token
+ *		generation.
+ * @size:	Size of the challenge data in bytes.
+ *
+ * Initialises the attestation token generation and returns an upper bound
+ * on the attestation token size that can be used to allocate an adequate
+ * buffer. The caller is expected to subsequently call
+ * rsi_attestation_token_continue() to retrieve the attestation token data on
+ * the same CPU.
+ *
+ * Returns:
+ *  On success, returns the upper limit of the attestation report size.
+ *  Otherwise, -EINVAL
+ */
+static inline long
+rsi_attestation_token_init(const u8 *challenge, unsigned long size)
+{
+	union {
+		struct arm_smccc_1_2_regs regs;
+		struct rsi_attestation_token_init_args init;
+	} args = { 0 };
+
+	if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE ||
+	    size > RSI_ATTEST_CHALLENGE_MAX_SIZE)
+		return -EINVAL;
+
+	args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT;
+	memcpy(args.init.challenge, challenge, size);
+	arm_smccc_1_2_smc(&args.regs, &args.regs);
+
+	if (args.regs.a0 == RSI_SUCCESS)
+		return args.regs.a1;
+
+	return -EINVAL;
+}
+
+/**
+ * rsi_attestation_token_continue - Continue the operation to retrieve an
+ * attestation token.
+ *
+ * @granule: {I}PA of the Granule to which the token will be written.
+ * @offset:  Offset within Granule to start of buffer in bytes.
+ * @size:    The size of the buffer.
+ * @len:     The number of bytes written to the buffer.
+ *
+ * Retrieves up to a RSI_GRANULE_SIZE worth of token data per call. The caller
+ * is expected to call rsi_attestation_token_init() before calling this
+ * function to retrieve the attestation token.
+ *
+ * Return:
+ * * %RSI_SUCCESS     - Attestation token retrieved successfully.
+ * * %RSI_INCOMPLETE  - Token generation is not complete.
+ * * %RSI_ERROR_INPUT - A parameter was not valid.
+ * * %RSI_ERROR_STATE - Attestation not in progress.
+ */
+static inline unsigned long rsi_attestation_token_continue(phys_addr_t granule,
+							   unsigned long offset,
+							   unsigned long size,
+							   unsigned long *len)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_ATTESTATION_TOKEN_CONTINUE,
+			     granule, offset, size, 0, &res);
+
+	if (len)
+		*len = res.a1;
+	return res.a0;
+}
+
+#endif /* __LINUX_ARM_RSI_CMDS_H_ */
diff --git a/include/linux/arm-smccc-bus.h b/include/linux/arm-smccc-bus.h
new file mode 100644
index 00000000000000..05c0c850510e05
--- /dev/null
+++ b/include/linux/arm-smccc-bus.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2026 Arm Limited
+ */
+#ifndef __LINUX_ARM_SMCCC_BUS_H
+#define __LINUX_ARM_SMCCC_BUS_H
+
+#include <linux/device.h>
+#include <linux/device-id/arm_smccc.h>
+#include <linux/module.h>
+
+struct arm_smccc_device {
+	struct device dev;
+	u32 func_id;
+};
+
+#define to_arm_smccc_device(d) container_of_const(d, struct arm_smccc_device, dev)
+
+struct arm_smccc_driver {
+	struct device_driver driver;
+
+	const char *name;
+	int (*probe)(struct arm_smccc_device *sdev);
+	void (*remove)(struct arm_smccc_device *sdev);
+	const struct arm_smccc_device_id *id_table;
+};
+
+#define to_arm_smccc_driver(d) \
+	container_of_const(d, struct arm_smccc_driver, driver)
+
+int arm_smccc_driver_register(struct arm_smccc_driver *driver,
+		struct module *owner, const char *mod_name);
+void arm_smccc_driver_unregister(struct arm_smccc_driver *driver);
+struct arm_smccc_device *arm_smccc_device_register(const char *name, u32 func_id);
+void arm_smccc_device_unregister(struct arm_smccc_device *smcc_dev);
+
+#define arm_smccc_register(driver) \
+	arm_smccc_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
+#define arm_smccc_unregister(driver) \
+	arm_smccc_driver_unregister(driver)
+
+#define module_arm_smccc_driver(__arm_smccc_driver) \
+	module_driver(__arm_smccc_driver, arm_smccc_register, \
+		      arm_smccc_unregister)
+
+extern const struct bus_type arm_smccc_bus_type;
+
+#endif /* __LINUX_ARM_SMCCC_BUS_H */
diff --git a/include/linux/arm-smccc-rsi.h b/include/linux/arm-smccc-rsi.h
new file mode 100644
index 00000000000000..fddb77986f706c
--- /dev/null
+++ b/include/linux/arm-smccc-rsi.h
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2023 ARM Ltd.
+ */
+
+#ifndef __LINUX_ARM_SMCCC_RSI_H_
+#define __LINUX_ARM_SMCCC_RSI_H_
+
+#include <linux/arm-smccc.h>
+
+/*
+ * This file describes the Realm Services Interface (RSI) Application Binary
+ * Interface (ABI) for SMC calls made from within the Realm to the RMM and
+ * serviced by the RMM.
+ */
+
+/*
+ * The major version number of the RSI implementation.  This is increased when
+ * the binary format or semantics of the SMC calls change.
+ */
+#define RSI_ABI_VERSION_MAJOR		UL(1)
+
+/*
+ * The minor version number of the RSI implementation.  This is increased when
+ * a bug is fixed, or a feature is added without breaking binary compatibility.
+ */
+#define RSI_ABI_VERSION_MINOR		UL(0)
+
+#define RSI_ABI_VERSION			((RSI_ABI_VERSION_MAJOR << 16) | \
+					 RSI_ABI_VERSION_MINOR)
+
+#define RSI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16)
+#define RSI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF)
+
+#define RSI_SUCCESS		UL(0)
+#define RSI_ERROR_INPUT		UL(1)
+#define RSI_ERROR_STATE		UL(2)
+#define RSI_INCOMPLETE		UL(3)
+#define RSI_ERROR_UNKNOWN	UL(4)
+
+#define SMC_RSI_FID(n)		ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,      \
+						   ARM_SMCCC_SMC_64,         \
+						   ARM_SMCCC_OWNER_STANDARD, \
+						   n)
+
+/*
+ * Returns RSI version.
+ *
+ * arg1 == Requested interface revision
+ * ret0 == Status / error
+ * ret1 == Lower implemented interface revision
+ * ret2 == Higher implemented interface revision
+ */
+#define SMC_RSI_ABI_VERSION	SMC_RSI_FID(0x190)
+
+/*
+ * Read feature register.
+ *
+ * arg1 == Feature register index
+ * ret0 == Status / error
+ * ret1 == Feature register value
+ */
+#define SMC_RSI_FEATURES			SMC_RSI_FID(0x191)
+
+/*
+ * Read measurement for the current Realm.
+ *
+ * arg1 == Index, which measurements slot to read
+ * ret0 == Status / error
+ * ret1 == Measurement value, bytes:  0 -  7
+ * ret2 == Measurement value, bytes:  8 - 15
+ * ret3 == Measurement value, bytes: 16 - 23
+ * ret4 == Measurement value, bytes: 24 - 31
+ * ret5 == Measurement value, bytes: 32 - 39
+ * ret6 == Measurement value, bytes: 40 - 47
+ * ret7 == Measurement value, bytes: 48 - 55
+ * ret8 == Measurement value, bytes: 56 - 63
+ */
+#define SMC_RSI_MEASUREMENT_READ		SMC_RSI_FID(0x192)
+
+/*
+ * Extend Realm Extensible Measurement (REM) value.
+ *
+ * arg1  == Index, which measurements slot to extend
+ * arg2  == Size of realm measurement in bytes, max 64 bytes
+ * arg3  == Measurement value, bytes:  0 -  7
+ * arg4  == Measurement value, bytes:  8 - 15
+ * arg5  == Measurement value, bytes: 16 - 23
+ * arg6  == Measurement value, bytes: 24 - 31
+ * arg7  == Measurement value, bytes: 32 - 39
+ * arg8  == Measurement value, bytes: 40 - 47
+ * arg9  == Measurement value, bytes: 48 - 55
+ * arg10 == Measurement value, bytes: 56 - 63
+ * ret0  == Status / error
+ */
+#define SMC_RSI_MEASUREMENT_EXTEND		SMC_RSI_FID(0x193)
+
+/*
+ * Initialize the operation to retrieve an attestation token.
+ *
+ * arg1 == Challenge value, bytes:  0 -  7
+ * arg2 == Challenge value, bytes:  8 - 15
+ * arg3 == Challenge value, bytes: 16 - 23
+ * arg4 == Challenge value, bytes: 24 - 31
+ * arg5 == Challenge value, bytes: 32 - 39
+ * arg6 == Challenge value, bytes: 40 - 47
+ * arg7 == Challenge value, bytes: 48 - 55
+ * arg8 == Challenge value, bytes: 56 - 63
+ * ret0 == Status / error
+ * ret1 == Upper bound of token size in bytes
+ */
+#define SMC_RSI_ATTESTATION_TOKEN_INIT		SMC_RSI_FID(0x194)
+
+/*
+ * Continue the operation to retrieve an attestation token.
+ *
+ * arg1 == The IPA of token buffer
+ * arg2 == Offset within the granule of the token buffer
+ * arg3 == Size of the granule buffer
+ * ret0 == Status / error
+ * ret1 == Length of token bytes copied to the granule buffer
+ */
+#define SMC_RSI_ATTESTATION_TOKEN_CONTINUE	SMC_RSI_FID(0x195)
+
+#ifndef __ASSEMBLER__
+
+struct realm_config {
+	union {
+		struct {
+			unsigned long ipa_bits; /* Width of IPA in bits */
+			unsigned long hash_algo; /* Hash algorithm */
+		};
+		u8 pad[0x200];
+	};
+	union {
+		u8 rpv[64]; /* Realm Personalization Value */
+		u8 pad2[0xe00];
+	};
+	/*
+	 * The RMM requires the configuration structure to be aligned to a 4k
+	 * boundary, ensure this happens by aligning this structure.
+	 */
+} __aligned(0x1000);
+
+#endif /* __ASSEMBLER__ */
+
+/*
+ * Read configuration for the current Realm.
+ *
+ * arg1 == struct realm_config addr
+ * ret0 == Status / error
+ */
+#define SMC_RSI_REALM_CONFIG			SMC_RSI_FID(0x196)
+
+/*
+ * Request RIPAS of a target IPA range to be changed to a specified value.
+ *
+ * arg1 == Base IPA address of target region
+ * arg2 == Top of the region
+ * arg3 == RIPAS value
+ * arg4 == flags
+ * ret0 == Status / error
+ * ret1 == Top of modified IPA range
+ * ret2 == Whether the Host accepted or rejected the request
+ */
+#define SMC_RSI_IPA_STATE_SET			SMC_RSI_FID(0x197)
+
+#define RSI_NO_CHANGE_DESTROYED			UL(0)
+#define RSI_CHANGE_DESTROYED			UL(1)
+
+#define RSI_ACCEPT				UL(0)
+#define RSI_REJECT				UL(1)
+
+/*
+ * Get RIPAS of a target IPA range.
+ *
+ * arg1 == Base IPA of target region
+ * arg2 == End of target IPA region
+ * ret0 == Status / error
+ * ret1 == Top of IPA region which has the reported RIPAS value
+ * ret2 == RIPAS value
+ */
+#define SMC_RSI_IPA_STATE_GET			SMC_RSI_FID(0x198)
+
+/*
+ * Make a Host call.
+ *
+ * arg1 == IPA of host call structure
+ * ret0 == Status / error
+ */
+#define SMC_RSI_HOST_CALL			SMC_RSI_FID(0x199)
+
+#endif /* __LINUX_ARM_SMCCC_RSI_H_ */
diff --git a/include/linux/audit.h b/include/linux/audit.h
index 45abb3722d304c..18b44a1c5397db 100644
--- a/include/linux/audit.h
+++ b/include/linux/audit.h
@@ -547,7 +547,7 @@ static inline void audit_openat2_how(struct open_how *how)
 
 static inline void audit_log_kern_module(const char *name)
 {
-	if (!audit_dummy_context())
+	if (unlikely(!audit_dummy_context()))
 		__audit_log_kern_module(name);
 }
 
@@ -563,7 +563,7 @@ static inline void audit_tk_injoffset(struct timespec64 offset)
 	if (offset.tv_sec == 0 && offset.tv_nsec == 0)
 		return;
 
-	if (!audit_dummy_context())
+	if (unlikely(!audit_dummy_context()))
 		__audit_tk_injoffset(offset);
 }
 
@@ -586,7 +586,7 @@ static inline void audit_ntp_set_new(struct audit_ntp_data *ad,
 
 static inline void audit_ntp_log(const struct audit_ntp_data *ad)
 {
-	if (!audit_dummy_context())
+	if (unlikely(!audit_dummy_context()))
 		__audit_ntp_log(ad);
 }
 
diff --git a/include/linux/balloon.h b/include/linux/balloon.h
index ca5b15150f4255..13a0af318cc296 100644
--- a/include/linux/balloon.h
+++ b/include/linux/balloon.h
@@ -36,7 +36,6 @@
  */
 #ifndef _LINUX_BALLOON_H
 #define _LINUX_BALLOON_H
-#include <linux/pagemap.h>
 #include <linux/page-flags.h>
 #include <linux/migrate.h>
 #include <linux/gfp.h>
diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
index f686a37f7a0a04..2e87faf9a8c285 100644
--- a/include/linux/binfmts.h
+++ b/include/linux/binfmts.h
@@ -128,7 +128,8 @@ struct linux_binfmt {
 	struct module *module;
 	int (*load_binary)(struct linux_binprm *);
 #ifdef CONFIG_COREDUMP
-	int (*core_dump)(struct coredump_params *cprm);
+	/* Returns true if the whole coredump was written. */
+	bool (*core_dump)(struct coredump_params *cprm);
 	unsigned long min_coredump;	/* minimal dump size */
 #endif
 } __randomize_layout;
diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h
index 0ea2a8bf7efb73..a954c97be0b336 100644
--- a/include/linux/bio-integrity.h
+++ b/include/linux/bio-integrity.h
@@ -151,7 +151,6 @@ void bio_integrity_setup_default(struct bio *bio);
 unsigned int fs_bio_integrity_alloc(struct bio *bio);
 void fs_bio_integrity_free(struct bio *bio);
 void fs_bio_integrity_generate(struct bio *bio);
-int fs_bio_integrity_verify(struct bio *bio, sector_t sector,
-		unsigned int size);
+int fs_bio_integrity_verify(struct bio *bio, struct bvec_iter *data_iter);
 
 #endif /* _LINUX_BIO_INTEGRITY_H */
diff --git a/include/linux/bio.h b/include/linux/bio.h
index bb3235497e673a..75f8086351c05b 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -256,6 +256,12 @@ static inline struct folio *bio_first_folio_all(struct bio *bio)
 	return page_folio(bio_first_page_all(bio));
 }
 
+static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
+{
+	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
+	return &bio->bi_io_vec[bio->bi_vcnt - 1];
+}
+
 /**
  * struct folio_iter - State for iterating all folios in a bio.
  * @folio: The current folio we're iterating.  NULL after the last folio.
@@ -479,6 +485,7 @@ static inline void bio_init_inline(struct bio *bio, struct block_device *bdev,
 extern void bio_uninit(struct bio *);
 void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf);
 void bio_reuse(struct bio *bio, blk_opf_t opf);
+void bio_prepare_reissue(struct bio *bio, struct block_device *bdev);
 void bio_chain(struct bio *, struct bio *);
 void bio_await(struct bio *bio, void *priv,
 	       void (*submit)(struct bio *bio, void *priv));
@@ -516,16 +523,18 @@ int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data,
 		size_t len, enum req_op op);
 
 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
-		unsigned mem_align_mask, unsigned len_align_mask);
+		unsigned maxlen, unsigned mem_align_mask,
+		unsigned len_align_mask);
 
 bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter);
 void __bio_release_pages(struct bio *bio, bool mark_dirty);
 extern void bio_set_pages_dirty(struct bio *bio);
 extern void bio_check_pages_dirty(struct bio *bio);
 
-int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen,
-		size_t minsize);
-void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty);
+int bio_alloc_bounce_folios(struct bio *bio, size_t total_len, size_t minsize);
+void bio_free_folios(struct bio *bio);
+int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter,
+		size_t maxlen, size_t minsize);
 
 extern void bio_copy_data(struct bio *dst, struct bio *src);
 extern void bio_free_pages(struct bio *bio);
diff --git a/include/linux/bitfield-fix-width.h b/include/linux/bitfield-fix-width.h
new file mode 100644
index 00000000000000..f73b8cd656010b
--- /dev/null
+++ b/include/linux/bitfield-fix-width.h
@@ -0,0 +1,241 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef _LINUX_BITFIELD_FIX_WIDTH_H
+#define _LINUX_BITFIELD_FIX_WIDTH_H
+
+#include <linux/compiler.h>
+#include <linux/types.h>
+#include <asm/byteorder.h>
+
+extern void __compiletime_error("value doesn't fit into mask")
+__field_overflow(void);
+
+extern void __compiletime_error("bad bitfield mask")
+__bad_mask(void);
+
+static __always_inline
+u64 field_multiplier(u64 field)
+{
+	if ((field | (field - 1)) & ((field | (field - 1)) + 1))
+		__bad_mask();
+	return field & -field;
+}
+
+static __always_inline
+u64 field_mask(u64 field)
+{
+	return field / field_multiplier(field);
+}
+
+#define field_max(field)	((typeof(field))field_mask(field))
+
+#define __assert_field(v, field)						\
+	do {								\
+		if (__builtin_constant_p(v) &&				\
+		    ((v) & ~field_mask(field)))				\
+			__field_overflow();				\
+	} while (0)
+
+static __always_inline __must_check
+__u8 u8_encode_bits(u8 v, u8 field)
+{
+	__assert_field(v, field);
+	return (v & field_mask(field)) * field_multiplier(field);
+}
+
+static __always_inline __must_check
+__u8 u8_replace_bits(__u8 old, u8 val, u8 field)
+{
+	return (old & ~field) | u8_encode_bits(val, field);
+}
+
+static __always_inline
+void u8p_replace_bits(__u8 *p, u8 val, u8 field)
+{
+	*p = (*p & ~field) | u8_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+u8 u8_get_bits(__u8 v, u8 field)
+{
+	return (v & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__le16 le16_encode_bits(u16 v, u16 field)
+{
+	__assert_field(v, field);
+	return cpu_to_le16((v & field_mask(field)) * field_multiplier(field));
+}
+
+static __always_inline
+void le16p_replace_bits(__le16 *p, u16 val, u16 field)
+{
+	*p = (*p & ~cpu_to_le16(field)) | le16_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+u16 le16_get_bits(__le16 v, u16 field)
+{
+	return (le16_to_cpu(v) & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__be16 be16_encode_bits(u16 v, u16 field)
+{
+	__assert_field(v, field);
+	return cpu_to_be16((v & field_mask(field)) * field_multiplier(field));
+}
+
+static __always_inline __must_check
+u16 be16_get_bits(__be16 v, u16 field)
+{
+	return (be16_to_cpu(v) & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__be16 be16_replace_bits(__be16 old, u16 val, u16 field)
+{
+	return (old & ~cpu_to_be16(field)) | be16_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+__u16 u16_encode_bits(u16 v, u16 field)
+{
+	__assert_field(v, field);
+	return (v & field_mask(field)) * field_multiplier(field);
+}
+
+static __always_inline __must_check
+__u16 u16_replace_bits(__u16 old, u16 val, u16 field)
+{
+	return (old & ~field) | u16_encode_bits(val, field);
+}
+
+static __always_inline
+void u16p_replace_bits(__u16 *p, u16 val, u16 field)
+{
+	*p = (*p & ~field) | u16_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+u16 u16_get_bits(__u16 v, u16 field)
+{
+	return (v & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__le32 le32_encode_bits(u32 v, u32 field)
+{
+	__assert_field(v, field);
+	return cpu_to_le32((v & field_mask(field)) * field_multiplier(field));
+}
+
+static __always_inline
+void le32p_replace_bits(__le32 *p, u32 val, u32 field)
+{
+	*p = (*p & ~cpu_to_le32(field)) | le32_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+u32 le32_get_bits(__le32 v, u32 field)
+{
+	return (le32_to_cpu(v) & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__be32 be32_encode_bits(u32 v, u32 field)
+{
+	__assert_field(v, field);
+	return cpu_to_be32((v & field_mask(field)) * field_multiplier(field));
+}
+
+static __always_inline
+void be32p_replace_bits(__be32 *p, u32 val, u32 field)
+{
+	*p = (*p & ~cpu_to_be32(field)) | be32_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+u32 be32_get_bits(__be32 v, u32 field)
+{
+	return (be32_to_cpu(v) & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__u32 u32_encode_bits(u32 v, u32 field)
+{
+	__assert_field(v, field);
+	return (v & field_mask(field)) * field_multiplier(field);
+}
+
+static __always_inline __must_check
+__u32 u32_replace_bits(__u32 old, u32 val, u32 field)
+{
+	return (old & ~field) | u32_encode_bits(val, field);
+}
+
+static __always_inline
+void u32p_replace_bits(__u32 *p, u32 val, u32 field)
+{
+	*p = (*p & ~field) | u32_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+u32 u32_get_bits(__u32 v, u32 field)
+{
+	return (v & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__le64 le64_encode_bits(u64 v, u64 field)
+{
+	__assert_field(v, field);
+	return cpu_to_le64((v & field_mask(field)) * field_multiplier(field));
+}
+
+static __always_inline __must_check
+u64 le64_get_bits(__le64 v, u64 field)
+{
+	return (le64_to_cpu(v) & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__be64 be64_encode_bits(u64 v, u64 field)
+{
+	__assert_field(v, field);
+	return cpu_to_be64((v & field_mask(field)) * field_multiplier(field));
+}
+
+static __always_inline __must_check
+u64 be64_get_bits(__be64 v, u64 field)
+{
+	return (be64_to_cpu(v) & field) / field_multiplier(field);
+}
+
+static __always_inline __must_check
+__u64 u64_encode_bits(u64 v, u64 field)
+{
+	__assert_field(v, field);
+	return (v & field_mask(field)) * field_multiplier(field);
+}
+
+static __always_inline __must_check
+__u64 u64_replace_bits(__u64 old, u64 val, u64 field)
+{
+	return (old & ~field) | u64_encode_bits(val, field);
+}
+
+static __always_inline
+void u64p_replace_bits(__u64 *p, u64 val, u64 field)
+{
+	*p = (*p & ~field) | u64_encode_bits(val, field);
+}
+
+static __always_inline __must_check
+u64 u64_get_bits(__u64 v, u64 field)
+{
+	return (v & field) / field_multiplier(field);
+}
+
+#endif /* _LINUX_BITFIELD_FIX_WIDTH_H */
diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h
index 14f86e455a678b..e3f7315c1d539b 100644
--- a/include/linux/bitfield.h
+++ b/include/linux/bitfield.h
@@ -10,7 +10,7 @@
 #include <linux/build_bug.h>
 #include <linux/compiler.h>
 #include <linux/typecheck.h>
-#include <asm/byteorder.h>
+#include <linux/bitfield-fix-width.h>
 
 /*
  * Bitfield access macros
@@ -211,53 +211,6 @@
 		*(_reg_p) |= (((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask));	\
 	})
 
-extern void __compiletime_error("value doesn't fit into mask")
-__field_overflow(void);
-extern void __compiletime_error("bad bitfield mask")
-__bad_mask(void);
-static __always_inline u64 field_multiplier(u64 field)
-{
-	if ((field | (field - 1)) & ((field | (field - 1)) + 1))
-		__bad_mask();
-	return field & -field;
-}
-static __always_inline u64 field_mask(u64 field)
-{
-	return field / field_multiplier(field);
-}
-#define field_max(field)	((typeof(field))field_mask(field))
-#define ____MAKE_OP(type,base,to,from)					\
-static __always_inline __##type __must_check type##_encode_bits(base v, base field)	\
-{									\
-	if (__builtin_constant_p(v) && (v & ~field_mask(field)))	\
-		__field_overflow();					\
-	return to((v & field_mask(field)) * field_multiplier(field));	\
-}									\
-static __always_inline __##type __must_check type##_replace_bits(__##type old,	\
-							base val, base field)	\
-{									\
-	return (old & ~to(field)) | type##_encode_bits(val, field);	\
-}									\
-static __always_inline void type##p_replace_bits(__##type *p,		\
-					base val, base field)		\
-{									\
-	*p = (*p & ~to(field)) | type##_encode_bits(val, field);	\
-}									\
-static __always_inline base __must_check type##_get_bits(__##type v, base field)	\
-{									\
-	return (from(v) & field)/field_multiplier(field);		\
-}
-#define __MAKE_OP(size)							\
-	____MAKE_OP(le##size,u##size,cpu_to_le##size,le##size##_to_cpu)	\
-	____MAKE_OP(be##size,u##size,cpu_to_be##size,be##size##_to_cpu)	\
-	____MAKE_OP(u##size,u##size,,)
-____MAKE_OP(u8,u8,,)
-__MAKE_OP(16)
-__MAKE_OP(32)
-__MAKE_OP(64)
-#undef __MAKE_OP
-#undef ____MAKE_OP
-
 #define __field_prep(mask, val)						\
 	({								\
 		auto __mask = (mask);					\
diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index 7df1573a409c3c..adafbcf2016bf7 100644
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -52,6 +52,7 @@ struct device;
  *  bitmap_complement(dst, src, nbits)          *dst = ~(*src)
  *  bitmap_equal(src1, src2, nbits)             Are *src1 and *src2 equal?
  *  bitmap_intersects(src1, src2, nbits)        Do *src1 and *src2 overlap?
+ *  bitmap_intersects_and(src1, src2, src3, nbits) Do *src1, *src2 and *src3 overlap?
  *  bitmap_subset(src1, src2, nbits)            Is *src1 a subset of *src2?
  *  bitmap_empty(src, nbits)                    Are all bits zero in *src?
  *  bitmap_full(src, nbits)                     Are all bits set in *src?
@@ -181,6 +182,9 @@ void __bitmap_replace(unsigned long *dst,
 		      const unsigned long *mask, unsigned int nbits);
 bool __bitmap_intersects(const unsigned long *bitmap1,
 			 const unsigned long *bitmap2, unsigned int nbits);
+bool __bitmap_intersects_and(const unsigned long *bitmap1,
+			     const unsigned long *bitmap2,
+			     const unsigned long *bitmap3, unsigned int nbits);
 bool __bitmap_subset(const unsigned long *bitmap1,
 		     const unsigned long *bitmap2, unsigned int nbits);
 unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits);
@@ -445,6 +449,16 @@ bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, uns
 		return __bitmap_intersects(src1, src2, nbits);
 }
 
+static __always_inline
+bool bitmap_intersects_and(const unsigned long *src1, const unsigned long *src2,
+			   const unsigned long *src3, unsigned int nbits)
+{
+	if (small_const_nbits(nbits))
+		return ((*src1 & *src2 & *src3) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
+	else
+		return __bitmap_intersects_and(src1, src2, src3, nbits);
+}
+
 static __always_inline
 bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
 {
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 4f7905c3412b8b..d003a9d2d1f6c7 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -191,14 +191,17 @@ struct gendisk {
 #ifdef CONFIG_BLK_DEV_ZONED
 	/*
 	 * Zoned block device information. Reads of this information must be
-	 * protected with blk_queue_enter() / blk_queue_exit(). Modifying this
-	 * information is only allowed while no requests are being processed.
-	 * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue().
+	 * protected with blk_queue_enter() / blk_queue_exit() or by holding a
+	 * lock on zone_revalidate_mutex. blk_revalidate_disk_zones() may modify
+	 * this information while no requests are being processed (disk queue
+	 * frozen with blk_mq_freeze_queue()) and while holding a lock on
+	 * zone_revalidate_mutex.
 	 */
+	struct mutex		zone_revalidate_mutex;
 	unsigned int		nr_zones;
 	unsigned int		zone_capacity;
 	unsigned int		last_zone_capacity;
-	u8 __rcu		*zones_cond;
+	u8 __rcu		*zones_state;
 	unsigned int		zone_wplugs_hash_bits;
 	atomic_t		nr_zone_wplugs;
 	spinlock_t		zone_wplugs_hash_lock;
@@ -1816,9 +1819,11 @@ static inline int bio_split_rw_at(struct bio *bio,
  */
 static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
 {
-	return min_t(unsigned int, lim->max_segment_size,
-		(BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) <<
-			lim->integrity.interval_exp);
+	u64 max_intervals;
+
+	max_intervals = BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size;
+	return min_t(u64, lim->max_segment_size,
+		max_intervals << lim->integrity.interval_exp);
 }
 
 #define DEFINE_IO_COMP_BATCH(name)	struct io_comp_batch name = { }
diff --git a/include/linux/bnge/hsi.h b/include/linux/bnge/hsi.h
index 1f7bd96415a527..d4fe662d60b10f 100644
--- a/include/linux/bnge/hsi.h
+++ b/include/linux/bnge/hsi.h
@@ -8607,6 +8607,7 @@ struct hwrm_cfa_l2_filter_alloc_input {
 	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID      0x10000UL
 	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS           0x20000UL
 	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_NUM_VLANS         0x40000UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX    0x80000UL
 	u8	l2_addr[6];
 	u8	num_vlans;
 	u8	t_num_vlans;
@@ -8663,7 +8664,8 @@ struct hwrm_cfa_l2_filter_alloc_input {
 	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN          0x4UL
 	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_LAST        CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN
 	u8	unused_5;
-	__le32	unused_6;
+	__le16	rfs_ring_tbl_idx;
+	__le16	unused_6;
 	__le64	l2_filter_id_hint;
 };
 
@@ -8900,6 +8902,95 @@ struct hwrm_cfa_tunnel_filter_free_output {
 	u8	valid;
 };
 
+/* hwrm_cfa_ntuple_filter_alloc_input (size:1024b/128B) */
+struct hwrm_cfa_ntuple_filter_alloc_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_LOOPBACK              0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP                  0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_METER                 0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_FID              0x8UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_ARP_REPLY             0x10UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX     0x20UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_NO_L2_CONTEXT         0x40UL
+	__le32	enables;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID         0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE            0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE          0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR          0x8UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE          0x10UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR           0x20UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK      0x40UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR           0x80UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK      0x100UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL          0x200UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT             0x400UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK        0x800UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT             0x1000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK        0x2000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_PRI_HINT             0x4000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_NTUPLE_FILTER_ID     0x8000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID               0x10000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID       0x20000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR          0x40000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX     0x80000UL
+	__le64	l2_filter_id;
+	u8	src_macaddr[6];
+	__be16	ethertype;
+	u8	ip_addr_type;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN 0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4    0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6    0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_LAST   CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6
+	u8	ip_protocol;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN 0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP     0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP     0x11UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_ICMP    0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_ICMPV6  0x3aUL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_RSVD    0xffUL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_LAST   CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_RSVD
+	__le16	dst_id;
+	__le16	rfs_ring_tbl_idx;
+	u8	tunnel_type;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL    0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN        0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE        0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE        0x3UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP         0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE       0x5UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS         0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT          0x7UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE        0x8UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4     0x9UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1     0xaUL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE     0xbUL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 0xcUL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE    0x10UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL    0xffUL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST        CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
+	u8	pri_hint;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER 0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_ABOVE     0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_BELOW     0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_HIGHEST   0x3UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST    0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LAST     CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST
+	__be32	src_ipaddr[4];
+	__be32	src_ipaddr_mask[4];
+	__be32	dst_ipaddr[4];
+	__be32	dst_ipaddr_mask[4];
+	__be16	src_port;
+	__be16	src_port_mask;
+	__be16	dst_port;
+	__be16	dst_port_mask;
+	__le64	ntuple_filter_id_hint;
+};
+
 /* hwrm_cfa_ntuple_filter_alloc_output (size:192b/24B) */
 struct hwrm_cfa_ntuple_filter_alloc_output {
 	__le16	error_code;
@@ -9014,6 +9105,49 @@ struct hwrm_cfa_ntuple_filter_cfg_output {
 	u8	valid;
 };
 
+/* hwrm_cfa_adv_flow_mgnt_qcaps_input (size:256b/32B) */
+struct hwrm_cfa_adv_flow_mgnt_qcaps_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	unused_0[4];
+};
+
+/* hwrm_cfa_adv_flow_mgnt_qcaps_output (size:128b/16B) */
+struct hwrm_cfa_adv_flow_mgnt_qcaps_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	flags;
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_HND_16BIT_SUPPORTED                     0x1UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_HND_64BIT_SUPPORTED                     0x2UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_BATCH_DELETE_SUPPORTED                  0x4UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_RESET_ALL_SUPPORTED                     0x8UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_DEST_FUNC_SUPPORTED              0x10UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_TX_EEM_FLOW_SUPPORTED                        0x20UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RX_EEM_FLOW_SUPPORTED                        0x40UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_COUNTER_ALLOC_SUPPORTED                 0x80UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_SUPPORTED                   0x100UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_UNTAGGED_VLAN_SUPPORTED                      0x200UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_XDP_SUPPORTED                                0x400UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_L2_HEADER_SOURCE_FIELDS_SUPPORTED            0x800UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_ARP_SUPPORTED                 0x1000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED                0x2000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_ETHERTYPE_IP_SUPPORTED        0x4000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_TRUFLOW_CAPABLE                              0x8000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_L2_FILTER_TRAFFIC_TYPE_L2_ROCE_SUPPORTED     0x10000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_LAG_SUPPORTED                                0x20000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_NO_L2CTX_SUPPORTED               0x40000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NIC_FLOW_STATS_SUPPORTED                     0x80000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED        0x100000UL
+	#define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED                0x200000UL
+	u8	unused_0[3];
+	u8	valid;
+};
+
 /* hwrm_tunnel_dst_port_alloc_input (size:192b/24B) */
 struct hwrm_tunnel_dst_port_alloc_input {
 	__le16	req_type;
diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h
index deda507500da23..fdc15b6f4d1b8c 100644
--- a/include/linux/bootconfig.h
+++ b/include/linux/bootconfig.h
@@ -291,12 +291,7 @@ int __init xbc_init(const char *buf, size_t size, const char **emsg, int *epos);
 int __init xbc_get_info(int *node_size, size_t *data_size);
 
 /* XBC cleanup data structures */
-void __init _xbc_exit(bool early);
-
-static __always_inline void xbc_exit(void)
-{
-	_xbc_exit(false);
-}
+void __init xbc_exit(void);
 
 /* XBC embedded bootconfig data in kernel */
 #ifdef CONFIG_BOOT_CONFIG_EMBED
diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h
index c9e6b26abab6d7..0147b8bec97368 100644
--- a/include/linux/bpf-cgroup-defs.h
+++ b/include/linux/bpf-cgroup-defs.h
@@ -47,6 +47,7 @@ enum cgroup_bpf_attach_type {
 	CGROUP_INET6_GETSOCKNAME,
 	CGROUP_UNIX_GETSOCKNAME,
 	CGROUP_INET_SOCK_RELEASE,
+	CGROUP_TCP_SOCK_OPS,
 	CGROUP_LSM_START,
 	CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1,
 	MAX_CGROUP_BPF_ATTACH_TYPE
diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
index 4d0cc65976a14a..8a75a6cd73096c 100644
--- a/include/linux/bpf-cgroup.h
+++ b/include/linux/bpf-cgroup.h
@@ -100,6 +100,8 @@ struct bpf_cgroup_storage {
 struct bpf_cgroup_link {
 	struct bpf_link link;
 	struct cgroup *cgroup;
+	struct bpf_map *map;
+	wait_queue_head_t wait_hup;
 };
 
 struct bpf_prog_list {
@@ -110,6 +112,18 @@ struct bpf_prog_list {
 	u32 flags;
 };
 
+#define bpf_cgroup_struct_ops_foreach(var, item, cgrp, atype)		\
+	for (item = rcu_dereference((cgrp)->bpf.effective[atype])->items;\
+	     ((var) = READ_ONCE(item->kdata));				\
+	     item++)
+
+static inline bool cgroup_bpf_is_struct_ops_atype(enum cgroup_bpf_attach_type atype)
+{
+	return atype == CGROUP_TCP_SOCK_OPS;
+}
+void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_trace);
+int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr);
+
 void __init cgroup_bpf_lifetime_notifier_init(void);
 
 int __cgroup_bpf_run_filter_skb(struct sock *sk,
@@ -479,6 +493,20 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
 	return 0;
 }
 
+static inline bool cgroup_bpf_is_struct_ops_atype(int atype)
+{
+	return false;
+}
+static inline void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs,
+						  bool mult_trace)
+{
+}
+static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map,
+					       const union bpf_attr *attr)
+{
+	return -EOPNOTSUPP;
+}
+
 #define cgroup_bpf_enabled(atype) (0)
 #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx) ({ 0; })
 #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype) ({ 0; })
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index e57af902560c36..d23a188a6f406c 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -17,6 +17,7 @@
 #include <linux/numa.h>
 #include <linux/mm_types.h>
 #include <linux/wait.h>
+#include <linux/irq_work_types.h>
 #include <linux/refcount.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
@@ -90,6 +91,7 @@ struct bpf_map_ops {
 	struct bpf_map *(*map_alloc)(union bpf_attr *attr);
 	void (*map_release)(struct bpf_map *map, struct file *map_file);
 	void (*map_free)(struct bpf_map *map);
+	void (*map_free_pre_rcu)(struct bpf_map *map);
 	int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
 	void (*map_release_uref)(struct bpf_map *map);
 	void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
@@ -214,6 +216,7 @@ enum btf_field_type {
 	BPF_UPTR       = (1 << 11),
 	BPF_RES_SPIN_LOCK = (1 << 12),
 	BPF_TASK_WORK  = (1 << 13),
+	BPF_RCU_HEAD   = (1 << 14),
 };
 
 enum bpf_cgroup_storage_type {
@@ -268,6 +271,7 @@ struct btf_record {
 	int wq_off;
 	int refcount_off;
 	int task_work_off;
+	int rcu_head_off;
 	struct btf_field fields[];
 };
 
@@ -341,8 +345,18 @@ struct bpf_map {
 	s64 __percpu *elem_count;
 	u64 cookie; /* write-once */
 	char *excl_prog_sha;
+	/*
+	 * Which programs use the map, see bpf_map_claim(): 0 - none so far,
+	 * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED
+	 * set - only that one and it stored the addresses of its functions into
+	 * the map, BPF_MAP_USER_MANY - more than one.
+	 */
+	unsigned long user;
 };
 
+#define BPF_MAP_USER_MANY	1UL
+#define BPF_MAP_USER_PATCHED	1UL
+
 static inline const char *btf_field_type_name(enum btf_field_type type)
 {
 	switch (type) {
@@ -373,6 +387,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type)
 		return "bpf_refcount";
 	case BPF_TASK_WORK:
 		return "bpf_task_work";
+	case BPF_RCU_HEAD:
+		return "bpf_rcu_head";
 	default:
 		WARN_ON_ONCE(1);
 		return "unknown";
@@ -413,6 +429,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type)
 		return sizeof(struct bpf_refcount);
 	case BPF_TASK_WORK:
 		return sizeof(struct bpf_task_work);
+	case BPF_RCU_HEAD:
+		return sizeof(struct bpf_rcu_head);
 	default:
 		WARN_ON_ONCE(1);
 		return 0;
@@ -447,6 +465,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type)
 		return __alignof__(struct bpf_refcount);
 	case BPF_TASK_WORK:
 		return __alignof__(struct bpf_task_work);
+	case BPF_RCU_HEAD:
+		return __alignof__(struct bpf_rcu_head);
 	default:
 		WARN_ON_ONCE(1);
 		return 0;
@@ -479,6 +499,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
 	case BPF_KPTR_PERCPU:
 	case BPF_UPTR:
 	case BPF_TASK_WORK:
+	case BPF_RCU_HEAD:
 		break;
 	default:
 		WARN_ON_ONCE(1);
@@ -572,7 +593,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec,
 		if (long_memcpy)
 			bpf_long_memcpy(dst, src, size);
 		else
-			memcpy(dst, src, size);
+			data_race(memcpy(dst, src, size));
 		return;
 	}
 
@@ -580,10 +601,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec,
 		u32 next_off = rec->fields[i].offset;
 		u32 sz = next_off - curr_off;
 
-		memcpy(dst + curr_off, src + curr_off, sz);
+		data_race(memcpy(dst + curr_off, src + curr_off, sz));
 		curr_off += rec->fields[i].size + sz;
 	}
-	memcpy(dst + curr_off, src + curr_off, size - curr_off);
+	data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off));
 }
 
 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
@@ -780,7 +801,10 @@ enum bpf_type_flag {
 	 */
 	PTR_UNTRUSTED		= BIT(6 + BPF_BASE_TYPE_BITS),
 
-	/* MEM can be uninitialized. */
+	/*
+	 * MEM can be uninitialized. Generic memory outputs need not be fully
+	 * initialized by the callee.
+	 */
 	MEM_UNINIT		= BIT(7 + BPF_BASE_TYPE_BITS),
 
 	/* DYNPTR points to memory local to the bpf program. */
@@ -874,7 +898,7 @@ enum bpf_type_flag {
 
 /* function argument constraints */
 enum bpf_arg_type {
-	ARG_DONTCARE = 0,	/* unused argument in helper function */
+	ARG_UNUSED = 0,		/* unused argument; terminates argument iteration */
 
 	/* the following constraints used to prototype
 	 * bpf_map_lookup/update/delete_elem() functions
@@ -909,6 +933,24 @@ enum bpf_arg_type {
 	ARG_PTR_TO_TIMER,	/* pointer to bpf_timer */
 	ARG_KPTR_XCHG_DEST,	/* pointer to destination that kptrs are bpf_kptr_xchg'd into */
 	ARG_PTR_TO_DYNPTR,      /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
+
+	ARG_CONST_SCALAR,	/* scalar known at verification time */
+	ARG_CONST_MEM_SIZE,	/* ARG_MEM_SIZE that must be constant */
+	ARG_PTR_TO_ALLOC_BTF_ID,	/* pointer to an allocated object */
+	ARG_PTR_TO_REFCOUNTED_KPTR,	/* pointer to a refcounted local kptr */
+	ARG_PTR_TO_ITER,	/* pointer to an iterator */
+	ARG_PTR_TO_LIST_HEAD,	/* pointer to bpf_list_head */
+	ARG_PTR_TO_LIST_NODE,	/* pointer to bpf_list_node */
+	ARG_PTR_TO_RB_ROOT,	/* pointer to bpf_rb_root */
+	ARG_PTR_TO_RB_NODE,	/* pointer to bpf_rb_node */
+	ARG_PTR_TO_WORKQUEUE,	/* pointer to bpf_wq */
+	ARG_PTR_TO_TASK_WORK,	/* pointer to bpf_task_work */
+	ARG_PTR_TO_RCU_HEAD,	/* pointer to bpf_rcu_head */
+	ARG_PTR_TO_IRQ_FLAG,	/* pointer to saved IRQ flags on the stack */
+	ARG_PTR_TO_RES_SPIN_LOCK,	/* pointer to bpf_res_spin_lock */
+	ARG_PTR_TO_CTX_OUT,	/* hook output argument passed through from ctx */
+	ARG_PTR_TO_PROG_AUX,	/* pointer to the caller's bpf_prog_aux */
+	ARG_IGNORE,		/* argument the verifier does not check at all */
 	__BPF_ARG_TYPE_MAX,
 
 	/* Extended arg_types. */
@@ -977,6 +1019,13 @@ static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
  */
 #define MAX_BPF_FUNC_REG_ARGS 5
 
+/* A by-value argument takes two eightbytes at most, so the maximum number of
+ * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may
+ * need that size for processing, although eventually the maximum slots will
+ * be capped at MAX_BPF_FUNC_ARGS.
+ */
+#define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS)
+
 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
  * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
  * instructions after verifying
@@ -1005,13 +1054,13 @@ struct bpf_func_proto {
 	};
 	union {
 		struct {
-			u32 *arg1_btf_id;
-			u32 *arg2_btf_id;
-			u32 *arg3_btf_id;
-			u32 *arg4_btf_id;
-			u32 *arg5_btf_id;
+			const u32 *arg1_btf_id;
+			const u32 *arg2_btf_id;
+			const u32 *arg3_btf_id;
+			const u32 *arg4_btf_id;
+			const u32 *arg5_btf_id;
 		};
-		u32 *arg_btf_id[MAX_BPF_FUNC_ARGS];
+		const u32 *arg_btf_id[MAX_BPF_FUNC_ARGS];
 		struct {
 			size_t arg1_size;
 			size_t arg2_size;
@@ -1114,6 +1163,7 @@ struct bpf_insn_access_aux {
 			u32 ref_id;
 		};
 	};
+	u32 mem_size;
 	struct bpf_verifier_log *log; /* for verbose logs */
 	bool is_retval; /* is accessing function return value ? */
 };
@@ -1194,6 +1244,9 @@ struct bpf_prog_offload {
 	u32			jited_len;
 };
 
+/* The argument is aligned to 16 bytes. */
+#define BTF_FMODEL_ALIGN16_ARG		BIT(0)
+
 /* The argument is signed. */
 #define BTF_FMODEL_SIGNED_ARG		BIT(1)
 
@@ -1211,6 +1264,11 @@ struct btf_func_model {
 	u8 arg_flags[MAX_BPF_FUNC_ARGS];
 };
 
+static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg)
+{
+	return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64);
+}
+
 /* Restore arguments before returning from trampoline to let original function
  * continue executing. This flag is used for fentry progs when there are no
  * fexit progs.
@@ -1258,11 +1316,15 @@ struct btf_func_model {
 #define BPF_TRAMP_F_INDIRECT		BIT(8)
 
 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
- * bytes on x86.
+ * bytes on x86. The trampoline image has to fit in PAGE_SIZE.
  */
 enum {
-#if defined(__s390x__)
+#if defined(__s390x__) || defined(__powerpc64__)
 	BPF_MAX_TRAMP_LINKS = 27,
+#elif defined(__x86_64__)
+	BPF_MAX_TRAMP_LINKS = 36,
+#elif defined(__aarch64__)
+	BPF_MAX_TRAMP_LINKS = 37,
 #else
 	BPF_MAX_TRAMP_LINKS = 38,
 #endif
@@ -1315,6 +1377,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 void *arch_alloc_bpf_trampoline(unsigned int size);
 void arch_free_bpf_trampoline(void *image, unsigned int size);
 int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
+int arch_bpf_trampoline_skip(void *nop, void *target);
 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 			     struct bpf_tramp_nodes *tnodes, void *func_addr);
 
@@ -1363,19 +1426,48 @@ enum bpf_tramp_prog_type {
 	BPF_TRAMP_FSESSION,
 };
 
+/*
+ * Each prog call in a trampoline image is preceded by a nop. When the prog is
+ * detached, the nop is patched to a jump to target, right after the call, so
+ * that tasks still running in the image skip the prog.
+ */
+struct bpf_tramp_skip {
+	struct bpf_prog *prog;
+	void *nop;
+	void *target;
+};
+
 struct bpf_tramp_image {
 	void *image;
 	int size;
 	struct bpf_ksym ksym;
 	struct percpu_ref pcref;
-	void *ip_after_call;
-	void *ip_epilogue;
+	bool call_orig;
+	/* entry in tr->images, the image holds a reference on tr */
+	struct bpf_trampoline *tr;
+	struct list_head list;
+	int nr_skips;
+	struct bpf_tramp_skip *skips;
 	union {
 		struct rcu_head rcu;
 		struct work_struct work;
 	};
 };
 
+static inline void bpf_tramp_image_add_skip(struct bpf_tramp_image *im, struct bpf_prog *prog,
+					    void *nop, void *target)
+{
+	struct bpf_tramp_skip *skip;
+
+	/* struct_ops trampolines and arch_bpf_trampoline_size() have no image */
+	if (!im || !im->skips)
+		return;
+	skip = &im->skips[im->nr_skips++];
+	skip->prog = prog;
+	skip->nop = nop;
+	skip->target = target;
+}
+
 struct bpf_trampoline {
 	/* hlist for trampoline_key_table */
 	struct hlist_node hlist_key;
@@ -1402,6 +1494,8 @@ struct bpf_trampoline {
 	int progs_cnt[BPF_TRAMP_MAX];
 	/* Executable image of trampoline */
 	struct bpf_tramp_image *cur_image;
+	/* Images not freed yet, cur_image and older ones still in use */
+	struct list_head images;
 	/* Used as temporary old image storage for multi_attach */
 	struct {
 		struct bpf_tramp_image *old_image;
@@ -1651,8 +1745,9 @@ static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags
 struct bpf_func_info_aux {
 	u16 linkage;
 	bool unreliable;
-	bool called : 1;
-	bool verified : 1;
+	/* Indexed by in_sleepable. */
+	bool called[2];
+	bool verified[2];
 };
 
 enum bpf_jit_poke_reason {
@@ -1684,6 +1779,7 @@ struct bpf_ctx_arg_aux {
 	struct btf *btf;
 	u32 btf_id;
 	u32 ref_id;
+	u32 mem_size;
 	bool refcounted;
 };
 
@@ -1712,12 +1808,18 @@ enum {
 };
 
 struct bpf_stream {
-	atomic_t capacity;
+	refcount_t refcnt;
+	atomic_t capacity;	/* bytes reserved against the stream limit */
+	atomic_t readable;	/* published bytes available to readers */
 	struct llist_head log;	/* list of in-flight stream elements in LIFO order */
 
 	struct mutex lock;  /* lock protecting backlog_{head,tail} */
 	struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */
 	struct llist_node *backlog_tail; /* tail of the list above */
+	wait_queue_head_t waitq;
+	struct irq_work notify_work;
+	bool notify_used;	/* notify_work was queued at least once */
+	bool dead;
 };
 
 struct bpf_stream_stage {
@@ -1736,6 +1838,7 @@ enum bpf_sig_keyring {
 	BPF_SIG_KEYRING_SECONDARY,
 	BPF_SIG_KEYRING_PLATFORM,
 	BPF_SIG_KEYRING_USER,
+	BPF_SIG_KEYRING_BPF,
 };
 
 struct bpf_prog_aux {
@@ -1769,12 +1872,12 @@ struct bpf_prog_aux {
 	bool offload_requested; /* Program is bound and offloaded to the netdev. */
 	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
 	bool attach_tracing_prog; /* true if tracing another tracing program */
+	bool tramp_linked; /* true if it was ever linked to a trampoline */
 	bool func_proto_unreliable;
 	bool tail_call_reachable;
 	bool xdp_has_frags;
 	bool exception_cb;
 	bool exception_boundary;
-	bool is_extended; /* true if extended by freplace program */
 	bool jits_use_priv_stack;
 	bool priv_stack_requested;
 	bool changes_pkt_data;
@@ -1786,7 +1889,7 @@ struct bpf_prog_aux {
 		u8 verdict;
 	} sig;
 	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
-	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
+	struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */
 	struct bpf_arena *arena;
 	void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */
 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
@@ -1818,6 +1921,7 @@ struct bpf_prog_aux {
 	char name[BPF_OBJ_NAME_LEN];
 	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
 	u16 stack_arg_sp_adjust;
+	u16 freplace_link_cnt; /* counts freplace links extending this prog */
 #ifdef CONFIG_SECURITY
 	void *security;
 #endif
@@ -1855,7 +1959,7 @@ struct bpf_prog_aux {
 		struct work_struct work;
 		struct rcu_head	rcu;
 	};
-	struct bpf_stream stream[2];
+	struct bpf_stream *stream[2];
 	struct mutex st_ops_assoc_mutex;
 	struct bpf_map __rcu *st_ops_assoc;
 };
@@ -2099,6 +2203,18 @@ struct btf_member;
  *	   unloaded while in use.
  * @name: The name of the struct bpf_struct_ops object.
  * @func_models: Func models
+ * @cgroup_atype: A value in enum cgroup_bpf_attach_type for cgroup attachment.
+ *		  0 means the struct_ops type does not support cgroup attachment.
+ *		  If cgroup_atype is non-zero, the @reg and @unreg must be NULL
+ *		  because the attachment/detachment will be handled by the bpf core.
+ * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for
+ *                           a tasks_rcu gp before freeing the struct_ops map
+ *                           and its progs. It is unnecessary if the @unreg
+ *                           has waited for the correct rcu gp or the @unreg
+ *                           has ensured all struct_ops prog has finished running.
+ * @free_after_mult_rcu_gp: Same as @free_after_tasks_rcu_gp but waiting for
+ *                          both tasks_trace_rcu and regular rcu grace period.
+ *                          It is usually needed if the struct_ops has sleepable prog.
  */
 struct bpf_struct_ops {
 	const struct bpf_verifier_ops *verifier_ops;
@@ -2117,6 +2233,9 @@ struct bpf_struct_ops {
 	struct module *owner;
 	const char *name;
 	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
+	int cgroup_atype;
+	bool free_after_tasks_rcu_gp;
+	bool free_after_mult_rcu_gp;
 };
 
 /* Every member of a struct_ops type has an instance even a member is not
@@ -2251,6 +2370,12 @@ u32 bpf_struct_ops_id(const void *kdata);
 int bpf_struct_ops_for_each_prog(const void *kdata,
 				 int (*cb)(struct bpf_prog *prog, void *data),
 				 void *data);
+void *bpf_struct_ops_map_kdata(struct bpf_map *map);
+void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map);
+bool bpf_struct_ops_valid_to_reg(struct bpf_map *map);
+int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map,
+				     struct bpf_map *expected_old_map);
+int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map);
 
 #ifdef CONFIG_NET
 /* Define it here to avoid the use of forward declaration */
@@ -2315,6 +2440,32 @@ static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struc
 static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
 {
 }
+static inline u32 bpf_struct_ops_id(void *kdata)
+{
+	return 0;
+}
+static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map)
+{
+	return NULL;
+}
+static inline int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map)
+{
+	return 0;
+}
+static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map)
+{
+	return NULL;
+}
+static inline bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
+{
+	return false;
+}
+static inline int bpf_struct_ops_link_update_check(struct bpf_map *new_map,
+						   struct bpf_map *old_map,
+						   struct bpf_map *expected_old_map)
+{
+	return -EOPNOTSUPP;
+}
 
 #endif
 
@@ -2490,7 +2641,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
  * since other cpus are walking the array of pointers in parallel.
  */
 struct bpf_prog_array_item {
-	struct bpf_prog *prog;
+	union {
+		struct bpf_prog *prog;
+		void *kdata;
+	};
 	union {
 		struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
 		u64 bpf_cookie;
@@ -2532,6 +2686,7 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array,
 			struct bpf_prog *include_prog,
 			u64 bpf_cookie,
 			struct bpf_prog_array **new_array);
+struct bpf_prog *bpf_prog_dummy(void);
 
 struct bpf_run_ctx {};
 
@@ -2550,6 +2705,7 @@ struct bpf_trace_run_ctx {
 struct bpf_tramp_run_ctx {
 	struct bpf_run_ctx run_ctx;
 	u64 bpf_cookie;
+	int retval;
 	struct bpf_run_ctx *saved_run_ctx;
 };
 
@@ -3820,6 +3976,8 @@ struct bpf_key {
 #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL)
 struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags);
 struct bpf_key *bpf_lookup_system_key(u64 id);
+struct bpf_key *bpf_lookup_keyring(void);
+bool bpf_keyring_enforced(void);
 void bpf_key_put(struct bpf_key *bkey);
 int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
 			       const struct bpf_dynptr *sig_p,
@@ -3840,6 +3998,16 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id)
 	return NULL;
 }
 
+static inline struct bpf_key *bpf_lookup_keyring(void)
+{
+	return NULL;
+}
+
+static inline bool bpf_keyring_enforced(void)
+{
+	return false;
+}
+
 static inline void bpf_key_put(struct bpf_key *bkey)
 {
 }
@@ -4100,9 +4268,10 @@ void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
 int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs);
 void bpf_put_buffers(void);
 
-void bpf_prog_stream_init(struct bpf_prog *prog);
+int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags);
 void bpf_prog_stream_free(struct bpf_prog *prog);
-int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len);
+int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len);
+int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags);
 void bpf_stream_stage_init(struct bpf_stream_stage *ss);
 void bpf_stream_stage_free(struct bpf_stream_stage *ss);
 __printf(2, 3)
@@ -4165,7 +4334,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void);
 int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog);
 int bpf_insn_array_ready(struct bpf_map *map);
 void bpf_insn_array_release(struct bpf_map *map);
-void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len);
+void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len);
 void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len);
 
 #ifdef CONFIG_BPF_SYSCALL
diff --git a/include/linux/bpf_crypto.h b/include/linux/bpf_crypto.h
deleted file mode 100644
index a41e71d4e2d9f0..00000000000000
--- a/include/linux/bpf_crypto.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
-#ifndef _BPF_CRYPTO_H
-#define _BPF_CRYPTO_H
-
-struct bpf_crypto_type {
-	void *(*alloc_tfm)(const char *algo);
-	void (*free_tfm)(void *tfm);
-	int (*has_algo)(const char *algo);
-	int (*setkey)(void *tfm, const u8 *key, unsigned int keylen);
-	int (*setauthsize)(void *tfm, unsigned int authsize);
-	int (*encrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv);
-	int (*decrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv);
-	unsigned int (*ivsize)(void *tfm);
-	unsigned int (*statesize)(void *tfm);
-	u32 (*get_flags)(void *tfm);
-	struct module *owner;
-	char name[14];
-};
-
-int bpf_crypto_register_type(const struct bpf_crypto_type *type);
-int bpf_crypto_unregister_type(const struct bpf_crypto_type *type);
-
-#endif /* _BPF_CRYPTO_H */
diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h
index dda272d78f013f..eb4a38eef87ecf 100644
--- a/include/linux/bpf_lsm.h
+++ b/include/linux/bpf_lsm.h
@@ -21,6 +21,13 @@ extern bool bpf_lsm_initialized __ro_after_init;
 #include <linux/lsm_hook_defs.h>
 #undef LSM_HOOK
 
+/*
+ * Number of xattr slots the BPF LSM reserves in the array handed to
+ * security_inode_init_security(), i.e. the maximum number of labels
+ * a policy may attach to an inode while it is being created.
+ */
+#define BPF_LSM_INODE_INIT_XATTRS	2
+
 struct bpf_storage_blob {
 	struct bpf_local_storage __rcu *storage;
 };
@@ -28,7 +35,7 @@ struct bpf_storage_blob {
 extern struct lsm_blob_sizes bpf_lsm_blob_sizes;
 
 int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
-			const struct bpf_prog *prog);
+			struct bpf_prog *prog);
 
 bool bpf_lsm_is_sleepable_hook(u32 btf_id);
 bool bpf_lsm_is_trusted(const struct bpf_prog *prog);
@@ -71,7 +78,7 @@ static inline bool bpf_lsm_is_trusted(const struct bpf_prog *prog)
 }
 
 static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
-				      const struct bpf_prog *prog)
+				      struct bpf_prog *prog)
 {
 	return -EOPNOTSUPP;
 }
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 36b65797877d09..811342e3c04199 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -19,11 +19,12 @@
  * that converting umax_value to int cannot overflow.
  */
 #define BPF_MAX_VAR_SIZ	(1 << 29)
-/* size of tmp_str_buf in bpf_verifier.
- * we need at least 306 bytes to fit full stack mask representation
- * (in the "-8,-16,...,-512" form)
+/*
+ * size of tmp_str_buf in bpf_verifier.
+ * we need at least 1399 bytes to fit full stack mask representation
+ * (in the "-8,-16,...,-2048" form)
  */
-#define TMP_STR_BUF_LEN 320
+#define TMP_STR_BUF_LEN 1408
 /* Patch buffer size */
 #define INSN_BUF_SIZE 32
 
@@ -45,18 +46,20 @@ struct bpf_reg_state {
 	union {
 		/* valid when type == PTR_TO_PACKET */
 		int range;
+		/*
+		 * Valid when type == PTR_TO_STACK. Inside the callee two registers
+		 * can be both PTR_TO_STACK like R1=fp-8 and R2=fp-8, but one of them
+		 * points to this function stack while another to the caller's stack.
+		 * To differentiate them 'frameno' is used which is an index in
+		 * bpf_verifier_state->frame[] array pointing to bpf_func_state.
+		 */
+		u8 frameno;
 
-		/* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE |
-		 *   PTR_TO_MAP_VALUE_OR_NULL
+		/*
+		 * For CONST_PTR_TO_MAP, PTR_TO_MAP_KEY, PTR_TO_MAP_VALUE and
+		 * PTR_TO_INSN.
 		 */
-		struct {
-			struct bpf_map *map_ptr;
-			/* To distinguish map lookups from outer map
-			 * the map_uid is non-zero for registers
-			 * pointing to inner maps.
-			 */
-			u32 map_uid;
-		};
+		struct bpf_map *map_ptr;
 
 		/* for PTR_TO_BTF_ID */
 		struct {
@@ -155,13 +158,12 @@ struct bpf_reg_state {
 	 * gets parent_id set to the dynptr's id.
 	 */
 	u32 parent_id;
-	/* Inside the callee two registers can be both PTR_TO_STACK like
-	 * R1=fp-8 and R2=fp-8, but one of them points to this function stack
-	 * while another to the caller's stack. To differentiate them 'frameno'
-	 * is used which is an index in bpf_verifier_state->frame[] array
-	 * pointing to bpf_func_state.
+	/*
+	 * Distinguishes inner-map lookups and their keys and values. Zero for
+	 * other registers. Kept outside the metadata union for ID remapping
+	 * during state comparisons.
 	 */
-	u32 frameno;
+	u32 map_uid;
 	/* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */
 	bool precise;
 };
@@ -242,55 +244,18 @@ enum bpf_stack_slot_type {
 
 #define BPF_REG_SIZE 8	/* size of eBPF register in bytes */
 
+/*
+ * Largest number of BPF_REG_SIZE stack slots a single frame can have, sized
+ * for the largest stack budget any JIT supports. A frame may use any part of
+ * its program's budget; check_max_stack_depth() enforces the budget on the
+ * combined depth of frames sharing the kernel stack and on each frame using
+ * a private stack.
+ */
+#define MAX_BPF_STACK_SLOTS	(MAX_BPF_STACK_JIT / BPF_REG_SIZE)
+
 /* 4-byte stack slot granularity for liveness analysis */
 #define BPF_HALF_REG_SIZE	4
 #define STACK_SLOT_SZ		4
-#define STACK_SLOTS		(MAX_BPF_STACK / BPF_HALF_REG_SIZE)	/* 128 */
-
-typedef struct {
-	u64 v[2];
-} spis_t;
-
-#define SPIS_ZERO	((spis_t){})
-#define SPIS_ALL	((spis_t){{ U64_MAX, U64_MAX }})
-
-static inline bool spis_is_zero(spis_t s)
-{
-	return s.v[0] == 0 && s.v[1] == 0;
-}
-
-static inline bool spis_equal(spis_t a, spis_t b)
-{
-	return a.v[0] == b.v[0] && a.v[1] == b.v[1];
-}
-
-static inline spis_t spis_or(spis_t a, spis_t b)
-{
-	return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }};
-}
-
-static inline spis_t spis_and(spis_t a, spis_t b)
-{
-	return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }};
-}
-
-static inline spis_t spis_not(spis_t s)
-{
-	return (spis_t){{ ~s.v[0], ~s.v[1] }};
-}
-
-static inline bool spis_test_bit(spis_t s, u32 slot)
-{
-	return s.v[slot / 64] & BIT_ULL(slot % 64);
-}
-
-static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi)
-{
-	u32 w;
-
-	for (w = lo; w <= hi && w < STACK_SLOTS; w++)
-		mask->v[w / 64] |= BIT_ULL(w % 64);
-}
 
 #define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \
 			  (1 << BPF_REG_3) | (1 << BPF_REG_4) | \
@@ -420,30 +385,36 @@ enum {
 	INSN_F_STACK_ARG_ACCESS = BIT(3),
 };
 
+/* Registers linked to one jump condition that a history entry can record */
+#define BPF_LINKED_REGS_MAX	5
+
 struct bpf_jmp_history_entry {
 	/* insn idx can't be bigger than 1 million */
 	u32 idx : 20;
 	u32 frame : 4;	/* stack access frame number */
-	u32 spi : 6;	/* stack slot index (0..63) */
-	u32 : 2;
-	u32 prev_idx : 20;
 	/* special INSN_F_xxx flags */
 	u32 flags : 4;
-	u32 : 8;
+	u32 : 4;
+	u32 prev_idx : 20;
+	u32 spi : 12;	/* stack slot index */
 	/*
-	 * additional registers that need precision tracking when this
-	 * jump is backtracked, vector of five 11-bit records
+	 * Scalar registers and spilled scalars linked to the condition of
+	 * this jump, which need precision tracking together when the jump is
+	 * backtracked. Each is packed as 4 bits of frame number, one bit
+	 * telling a register from a stack slot and 11 bits of register or
+	 * slot index, see linked_regs_pack().
 	 */
-	u64 linked_regs;
+	u16 linked_regs[BPF_LINKED_REGS_MAX];
+	u8 linked_regs_cnt;
 };
 
 static_assert(MAX_CALL_FRAMES <= (1 << 4));
-static_assert(MAX_BPF_STACK / 8 <= (1 << 6));
+static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12));
 
 /* Maximum number of bpf_reg_state objects that can exist at once */
 #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS)
-#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \
-			  MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES)
+#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \
+			 MAX_CALL_FRAMES)
 struct bpf_verifier_state {
 	/* call stack tracking */
 	struct bpf_func_state *frame[MAX_CALL_FRAMES];
@@ -529,12 +500,31 @@ struct bpf_verifier_state {
 	u32 may_goto_depth;
 };
 
+/* Number of BPF_REG_SIZE stack slots tracked for the frame so far. */
+static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame)
+{
+	return frame->allocated_stack / BPF_REG_SIZE;
+}
+
+/*
+ * Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8).
+ * The caller must ensure spi < bpf_stack_nr_slots(frame), see grow_stack_state().
+ */
+static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi)
+{
+	return &frame->stack[spi];
+}
+
 static inline struct bpf_reg_state *
 bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask)
 {
-	if (slot < frame->allocated_stack / BPF_REG_SIZE &&
-	    (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask)
-		return &frame->stack[slot].spilled_ptr;
+	struct bpf_stack_state *ss;
+
+	if (slot >= bpf_stack_nr_slots(frame))
+		return NULL;
+	ss = bpf_stack_slot(frame, slot);
+	if ((1 << ss->slot_type[BPF_REG_SIZE - 1]) & mask)
+		return &ss->spilled_ptr;
 	return NULL;
 }
 
@@ -550,7 +540,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame)
 /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */
 #define bpf_for_each_spilled_reg(iter, frame, reg, mask)			\
 	for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask);		\
-	     iter < frame->allocated_stack / BPF_REG_SIZE;		\
+	     iter < bpf_stack_nr_slots(frame);				\
 	     iter++, reg = bpf_get_spilled_reg(iter, frame, mask))
 
 /* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */
@@ -575,7 +565,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame)
 			bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \
 				if (!__reg)                              \
 					continue;                        \
-				__stack = &__state->stack[___j];         \
+				__stack = bpf_stack_slot(__state, ___j); \
 				(void)(__expr);                          \
 			}                                                \
 			__stack = NULL;                                  \
@@ -670,7 +660,6 @@ struct bpf_insn_aux_data {
 		/* remember the offset of node field within type to rewrite */
 		u64 insert_off;
 	};
-	struct bpf_iarray *jt;	/* jump table for gotox or bpf_tailcall call instruction */
 	struct btf_struct_meta *kptr_struct_meta;
 	u64 map_key_state; /* constant (32 bit) key tracking for maps */
 	int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
@@ -679,6 +668,7 @@ struct bpf_insn_aux_data {
 	bool nospec_result; /* result is unsafe under speculation, nospec must follow */
 	bool zext_dst; /* this insn zero extends dst reg */
 	bool needs_zext; /* alu op needs to clear upper bits */
+	bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */
 	bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */
 	bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */
 	bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
@@ -706,6 +696,7 @@ struct bpf_insn_aux_data {
 	 */
 	u32 calls_callback:1;
 	u32 indirect_target:1; /* if it is an indirect jump target */
+	u32 non_stack_access:1; /* instruction can access non-stack memory */
 	/* true if some jump or call instruction targets this instruction */
 	u32 jump_target:1;
 	/*
@@ -732,7 +723,11 @@ struct bpf_insn_aux_data {
 #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
 #define MAX_USED_BTFS 64 /* max number of BTFs accessed by one BPF program */
 
-#define BPF_VERIFIER_TMP_LOG_SIZE	1024
+/*
+ * Longest line the verifier log can carry: a full stack mask of
+ * MAX_BPF_STACK_SLOTS slots, see TMP_STR_BUF_LEN, plus its prefix.
+ */
+#define BPF_VERIFIER_TMP_LOG_SIZE	2048
 
 struct bpf_verifier_log {
 	/* Logical start and end positions of a "log window" of the verifier log.
@@ -785,6 +780,21 @@ int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *lo
 
 #define BPF_MAX_SUBPROGS 256
 
+/*
+ * A pointer to a static subprog in the value of a frozen read-only array map:
+ * a 64-bit value that is the offset in bytes of the first instruction of
+ * the subprog in the program.
+ */
+struct bpf_func_ptr {
+	struct bpf_map *map;
+	u32 map_off;		/* offset of the pointer in the value of the map */
+	u32 orig_off;		/* what the map has: the first instruction of the subprog */
+	u32 xlated_off;		/* the same after instructions were patched and removed */
+};
+
+/* the subprog that a bpf_func_ptr pointed to was removed as dead code */
+#define BPF_FUNC_PTR_DELETED ((u32)-1)
+
 struct bpf_subprog_arg_info {
 	enum bpf_arg_type arg_type;
 	union {
@@ -804,7 +814,12 @@ struct bpf_subprog_info {
 	u32 start; /* insn idx of function entry point */
 	u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
 	u32 postorder_start; /* The idx to the env->cfg.insn_postorder */
-	u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */
+	/*
+	 * Index of one of the BPF_EXIT instructions in this subprogram, or
+	 * U32_MAX when it has none.
+	 */
+	u32 exit_idx;
+	struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */
 	u16 stack_depth; /* max. stack depth used by this function */
 	u16 stack_extra;
 	u32 insns_total;
@@ -821,11 +836,13 @@ struct bpf_subprog_info {
 	bool is_async_cb: 1;
 	bool is_exception_cb: 1;
 	bool args_cached: 1;
+	/* true if the return value is passed in the R0:R2 register pair */
+	bool ret_reg_pair: 1;
 	/* true if bpf_fastcall stack region is used by functions that can't be inlined */
 	bool keep_fastcall_stack: 1;
 	bool changes_pkt_data: 1;
 	bool might_sleep: 1;
-	u8 arg_cnt:4;
+	u8 arg_slot_cnt:4;
 
 	enum priv_stack_mode priv_stack_mode;
 	struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS];
@@ -835,8 +852,8 @@ struct bpf_subprog_info {
 
 static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub)
 {
-	if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS)
-		return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS;
+	if (sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS)
+		return sub->arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS;
 	return 0;
 }
 
@@ -847,7 +864,7 @@ struct backtrack_state {
 	struct bpf_verifier_env *env;
 	u32 frame;
 	u32 reg_masks[MAX_CALL_FRAMES];
-	u64 stack_masks[MAX_CALL_FRAMES];
+	unsigned long stack_masks[MAX_CALL_FRAMES][BITS_TO_LONGS(MAX_BPF_STACK_SLOTS)];
 	u8 stack_arg_masks[MAX_CALL_FRAMES];
 };
 
@@ -959,9 +976,24 @@ struct bpf_verifier_env {
 	const struct bpf_line_info *prev_linfo;
 	struct bpf_verifier_log log;
 	struct bpf_diag *diag;
+	struct bpf_func_proto bpf_subprog_scratch;
 	struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */
 	/* subprog indices sorted in topological order: leaves first, callers last */
 	int subprog_topo_order[BPF_MAX_SUBPROGS + 2];
+	/*
+	 * Pointers to static subprogs found in frozen read-only maps of the
+	 * program, see resolve_func_ptrs(). Sorted by map and map_off.
+	 */
+	struct bpf_func_ptr *func_ptrs;
+	u32 func_ptr_cnt;
+	bool has_callx;
+	/*
+	 * Call graph edges created by callx instructions. A bitmap of
+	 * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee)
+	 * is set when the main verification pass sees 'caller' calling 'callee'
+	 * via callx. Allocated when the first such edge is recorded.
+	 */
+	unsigned long *callx_edges;
 	union {
 		struct bpf_idmap idmap_scratch;
 		struct bpf_idset idset_scratch;
@@ -977,11 +1009,13 @@ struct bpf_verifier_env {
 		int cur_stack;
 		/* current position in the insn_postorder vector */
 		int cur_postorder;
+		u32 gotox_edges;
+		bool subprog_jts_ready;
 	} cfg;
 	struct backtrack_state bt;
 	struct bpf_jmp_history_entry *cur_hist_ent;
-	/* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */
-	struct arg_track **callsite_at_stack;
+	/* Per-callsite snapshot of the parent's tracked spill slots for cross-frame fills. */
+	struct spill_snapshot **callsite_at_stack;
 	u32 pass_cnt; /* number of times do_check() was called */
 	u32 subprog_cnt;
 	/* number of instructions analyzed by the verifier */
@@ -990,6 +1024,8 @@ struct bpf_verifier_env {
 	u32 prev_jmps_processed, jmps_processed;
 	/* maximum combined stack depth */
 	u32 max_stack_depth;
+	/* stack budget of the program, see bpf_prog_stack_limit() */
+	u32 stack_limit;
 	/* total verification time */
 	u64 verification_time;
 	/* maximum number of verifier states kept in 'branching' instructions */
@@ -1025,7 +1061,7 @@ struct bpf_verifier_env {
 	 */
 	u32 scratched_regs;
 	/* Same as scratched_regs but for stack slots */
-	u64 scratched_stack_slots;
+	DECLARE_BITMAP(scratched_stack_slots, MAX_BPF_STACK_SLOTS);
 	u64 prev_log_pos, prev_insn_print_pos;
 	/* buffer used to temporary hold constants as scalar registers */
 	struct bpf_reg_state fake_reg[1];
@@ -1057,8 +1093,13 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env
 	return &env->subprog_info[subprog];
 }
 
+static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog)
+{
+	return subprog_info(env, subprog)->ret_reg_pair;
+}
+
 struct bpf_call_summary {
-	u8 num_params;
+	u8 arg_slot_cnt;
 	bool is_void;
 	bool fastcall;
 };
@@ -1081,6 +1122,12 @@ static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
 	       insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
 }
 
+/* callx: indirect call of a bpf subprog whose address is in insn->dst_reg */
+static inline bool bpf_is_callx(const struct bpf_insn *insn)
+{
+	return insn->code == (BPF_JMP | BPF_CALL | BPF_X);
+}
+
 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
 				      const char *fmt, va_list args);
 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
@@ -1197,6 +1244,8 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id)
 
 int bpf_prepare_btf_info(struct bpf_verifier_env *env,
 			 const union bpf_attr *attr, bpfptr_t uattr);
+int bpf_check_core_relo(struct bpf_verifier_env *env,
+			const union bpf_attr *attr, bpfptr_t uattr);
 int bpf_check_btf_info(struct bpf_verifier_env *env,
 		       const union bpf_attr *attr, bpfptr_t uattr);
 
@@ -1219,7 +1268,8 @@ struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx);
 void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self);
 void bpf_free_backedges(struct bpf_scc_visit *visit);
 int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
-			 int insn_flags, int spi, int frame, u64 linked_regs);
+			 int insn_flags, int spi, int frame, const u16 *linked_regs,
+			 u8 linked_regs_cnt);
 void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist);
 void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
 			   struct bpf_reg_state *reg);
@@ -1236,11 +1286,36 @@ static inline int bpf_get_spi(s32 off)
 	return (-off - 1) / BPF_REG_SIZE;
 }
 
+/*
+ * Stack a program may use in total: combined over the frames of a call
+ * chain on the kernel stack, or per frame on a private stack. Any single
+ * frame may reach that deep. Only a JIT that lays out such frames may go
+ * beyond MAX_BPF_STACK, the interpreter's frame size, and only one whose
+ * tail calls let the target set up its own frame: without subprogram
+ * tail calls, do_misc_fixups() gives every program with tail calls a
+ * MAX_BPF_STACK frame, which a deeper frame would overrun.
+ */
+static inline u32 bpf_prog_stack_limit(const struct bpf_prog *prog)
+{
+	/* an offloaded program never runs on the host JIT, whatever it supports */
+	if (prog->jit_requested && !bpf_prog_is_offloaded(prog->aux) &&
+	    bpf_jit_supports_large_stack() && bpf_jit_supports_subprog_tailcalls())
+		return MAX_BPF_STACK_JIT;
+	return MAX_BPF_STACK;
+}
+
+/*
+ * Return the function state a stack pointer register refers to. frameno
+ * shares storage with other pointer metadata, so return NULL for any
+ * other register type instead of indexing frame[] with aliased bytes.
+ */
 static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env,
 					      const struct bpf_reg_state *reg)
 {
 	struct bpf_verifier_state *cur = env->cur_state;
 
+	if (reg->type != PTR_TO_STACK)
+		return NULL;
 	return cur->frame[reg->frameno];
 }
 
@@ -1279,12 +1354,7 @@ static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u
 
 static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
 {
-	bt->stack_masks[frame] |= 1ull << slot;
-}
-
-static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask)
-{
-	bt->stack_masks[frame] |= mask;
+	__set_bit(slot, bt->stack_masks[frame]);
 }
 
 static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot)
@@ -1299,10 +1369,17 @@ static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u3
 
 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
 {
-	return bt->stack_masks[frame] & (1ull << slot);
+	return test_bit(slot, bt->stack_masks[frame]);
 }
 
 bool bpf_map_is_rdonly(const struct bpf_map *map);
+struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env,
+				       const struct bpf_map *map, u32 *cnt);
+struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env,
+					     const struct bpf_map *map,
+					     u64 off, u64 size, u32 *cnt);
+void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len);
+void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len);
 int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
 			bool is_ldsx);
 
@@ -1430,7 +1507,7 @@ static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
 
 static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
 {
-	env->scratched_stack_slots |= 1ULL << spi;
+	__set_bit(spi, env->scratched_stack_slots);
 }
 
 static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
@@ -1438,27 +1515,28 @@ static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
 	return (env->scratched_regs >> regno) & 1;
 }
 
-static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
+static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u32 spi)
 {
-	return (env->scratched_stack_slots >> regno) & 1;
+	return test_bit(spi, env->scratched_stack_slots);
 }
 
 static inline bool verifier_state_scratched(const struct bpf_verifier_env *env)
 {
-	return env->scratched_regs || env->scratched_stack_slots;
+	return env->scratched_regs ||
+	       !bitmap_empty(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS);
 }
 
 static inline void mark_verifier_state_clean(struct bpf_verifier_env *env)
 {
 	env->scratched_regs = 0U;
-	env->scratched_stack_slots = 0ULL;
+	bitmap_zero(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS);
 }
 
 /* Used for printing the entire verifier state. */
 static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env)
 {
 	env->scratched_regs = ~0U;
-	env->scratched_stack_slots = ~0ULL;
+	bitmap_fill(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS);
 }
 
 static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size)
@@ -1493,14 +1571,25 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en
 const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog);
 int bpf_jmp_offset(struct bpf_insn *insn);
 struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
-void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
+void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask);
 bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);
 
+/* Kinds of member a by-value struct or union may be composed of. */
+enum btf_member_kind {
+	BTF_MEMBER_SCALAR	= BIT(0), /* an int or an enum */
+	BTF_MEMBER_ARENA_PTR	= BIT(1), /* a pointer carrying the "arena" type tag */
+};
+
+bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf,
+			       const struct btf_type *t, u32 member_kinds);
+u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t);
+
 int bpf_find_subprog(struct bpf_verifier_env *env, int off);
 bool bpf_is_throw_kfunc(struct bpf_insn *insn);
 int bpf_compute_const_regs(struct bpf_verifier_env *env);
 int bpf_prune_dead_branches(struct bpf_verifier_env *env);
 int bpf_check_cfg(struct bpf_verifier_env *env);
+void bpf_free_subprog_jts(struct bpf_verifier_env *env);
 int bpf_compute_postorder(struct bpf_verifier_env *env);
 int bpf_compute_scc(struct bpf_verifier_env *env);
 
@@ -1529,12 +1618,15 @@ struct ref_obj_desc {
 };
 
 /*
- * A memory argument a call fills in. The verifier allows the stack to be uninitialized if
- * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access().
+ * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes
+ * variable-sized buffers from raw mode without losing the output annotation.
+ * size records constant ranges to mark initialized after checking all arguments,
+ * only when the caller is allowed to read uninitialized stack memory.
  */
 struct arg_raw_mem_desc {
-	u8 regno;
-	int size;
+	u16 mask;
+	u16 var_size_mask;
+	int size[MAX_BPF_FUNC_ARGS];
 };
 
 /* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */
@@ -1554,6 +1646,7 @@ struct bpf_call_arg_meta {
 	struct btf *btf;
 	u32 func_id;
 	const struct bpf_func_proto *fn;
+	const struct btf_type *func_proto;
 	u8 release_regno;
 	u32 ret_btf_id;
 	u32 subprogno;
@@ -1561,11 +1654,14 @@ struct bpf_call_arg_meta {
 	struct bpf_dynptr_desc dynptr;
 	struct ref_obj_desc ref_obj;
 	struct ret_mem_desc ret_mem;
+	struct arg_raw_mem_desc arg_raw_mem;
+
+	/* Only set by subprog */
+	bool subprog_may_change_pkt;
 
 	/* Only set by kfunc */
 	bool r0_rdonly;
 	u32 kfunc_flags;
-	const struct btf_type *func_proto;
 	const char *func_name;
 	struct arg_constant_desc arg_constant;
 
@@ -1574,7 +1670,7 @@ struct bpf_call_arg_meta {
 	 * verification logic
 	 *   bpf_obj_drop/bpf_percpu_obj_drop
 	 *     Record the local kptr type to be drop'd
-	 *   bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)
+	 *   bpf_refcount_acquire (via ARG_PTR_TO_REFCOUNTED_KPTR arg type)
 	 *     Record the local kptr type to be refcount_incr'd and use
 	 *     arg_owning_ref to determine whether refcount_acquire should be
 	 *     fallible
@@ -1582,7 +1678,6 @@ struct bpf_call_arg_meta {
 	struct btf *arg_btf;
 	u32 arg_btf_id;
 	bool arg_owning_ref;
-	bool arg_prog;
 
 	struct {
 		struct btf_field *field;
@@ -1600,7 +1695,6 @@ struct bpf_call_arg_meta {
 	s64 const_map_key;
 	struct btf *ret_btf;
 	struct btf_field *kptr_field;
-	struct arg_raw_mem_desc arg_raw_mem;
 };
 
 int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,
@@ -1669,6 +1763,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
 	return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
 }
 
+static inline bool bpf_is_mem_insn(struct bpf_insn *insn)
+{
+	if (BPF_CLASS(insn->code) != BPF_ST &&
+	    BPF_CLASS(insn->code) != BPF_STX &&
+	    BPF_CLASS(insn->code) != BPF_LDX)
+		return false;
+
+	if (insn->code == (BPF_ST | BPF_NOSPEC))
+		return false;
+
+	return (BPF_MODE(insn->code) == BPF_MEM ||
+		BPF_MODE(insn->code) == BPF_MEMSX ||
+		BPF_MODE(insn->code) == BPF_ATOMIC);
+}
+
 #define MAX_PACKET_OFF 0xffff
 #define CALLER_SAVED_REGS 6
 
@@ -1702,7 +1811,6 @@ struct bpf_kfunc_desc_tab {
 };
 
 /* Functions exported from verifier.c, used by fixups.c */
-void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len);
 void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog);
 bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env);
 bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm);
diff --git a/include/linux/btf.h b/include/linux/btf.h
index 89d5a5c4f11718..4b63bb91550a1b 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -80,6 +80,7 @@
 #define KF_ARENA_ARG2   (1 << 15) /* kfunc takes an arena pointer as its second argument */
 #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */
 #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */
+#define KF_PERFMON      (1 << 18) /* kfunc requires CAP_PERFMON */
 
 /*
  * Tag marking a kernel function as a kfunc. This is meant to minimize the
@@ -235,6 +236,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
 				    u32 field_mask, u32 value_size);
 int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec);
 bool btf_type_is_void(const struct btf_type *t);
+bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t);
 s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind);
 s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p);
 struct btf *btf_get_module_btf(const struct module *module);
@@ -260,6 +262,11 @@ const char *btf_type_str(const struct btf_type *t);
 	     i < btf_type_vlen(datasec_type);			\
 	     i++, member++)
 
+#define for_each_loc(i, locsec_type, member)			\
+	for (i = 0, member = btf_type_loc_secinfo(locsec_type);	\
+	     i < btf_type_vlen(locsec_type);			\
+	     i++, member++)
+
 static inline bool btf_type_is_ptr(const struct btf_type *t)
 {
 	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
@@ -328,6 +335,21 @@ static inline u64 btf_enum64_value(const struct btf_enum64 *e)
 	return ((u64)e->val_hi32 << 32) | e->val_lo32;
 }
 
+static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t)
+{
+	return (struct btf_loc_param *)(t + 1);
+}
+
+static inline __u32 *btf_loc_proto_params(const struct btf_type *t)
+{
+	return (__u32 *)(t + 1);
+}
+
+static inline struct btf_loc *btf_type_loc_secinfo(const struct btf_type *t)
+{
+	return (struct btf_loc *)(t + 1);
+}
+
 static inline bool btf_is_composite(const struct btf_type *t)
 {
 	u16 kind = btf_kind(t);
@@ -558,7 +580,7 @@ struct btf_field_desc {
 	/* member struct size, or zero, if no members */
 	int m_sz;
 	/* repeated per-member offsets */
-	int m_off_cnt, m_offs[1];
+	int m_off_cnt, m_offs[2];
 };
 
 struct btf_field_iter {
diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h
index fd2c7115c05427..f8782b719026f2 100644
--- a/include/linux/buffer_head.h
+++ b/include/linux/buffer_head.h
@@ -59,10 +59,7 @@ struct address_space;
 struct buffer_head {
 	unsigned long b_state;		/* buffer state bitmap (see above) */
 	struct buffer_head *b_this_page;/* circular list of page's buffers */
-	union {
-		struct page *b_page;	/* the page this bh is mapped to */
-		struct folio *b_folio;	/* the folio this bh is mapped to */
-	};
+	struct folio *b_folio;		/* the folio this bh is mapped to */
 
 	sector_t b_blocknr;		/* start block number */
 	size_t b_size;			/* size of mapping */
@@ -172,15 +169,39 @@ static __always_inline int buffer_uptodate(const struct buffer_head *bh)
 
 static inline unsigned long bh_offset(const struct buffer_head *bh)
 {
-	return (unsigned long)(bh)->b_data & (page_size(bh->b_page) - 1);
+	return (unsigned long)(bh)->b_data & (folio_size(bh->b_folio) - 1);
 }
 
-/* If we *know* page->private refers to buffer_heads */
-#define page_buffers(page)					\
-	({							\
-		BUG_ON(!PagePrivate(page));			\
-		((struct buffer_head *)page_private(page));	\
-	})
+/**
+ * kmap_local_bh - Map the data of a buffer.
+ * @bh: The buffer.
+ *
+ * Buffers usually live in the page cache, but a few are built over memory
+ * which is not.  Those carry no folio and b_data is already a kernel address
+ * which is always mapped, so there is nothing to do for them.  Pair with
+ * kunmap_local_bh().
+ *
+ * Return: A pointer to the buffer's data.
+ */
+static inline void *kmap_local_bh(const struct buffer_head *bh)
+{
+	if (!bh->b_folio)
+		return bh->b_data;
+	return kmap_local_folio(bh->b_folio, bh_offset(bh));
+}
+
+/**
+ * kunmap_local_bh - Unmap the data of a buffer.
+ * @bh: The buffer.
+ * @addr: The address returned by kmap_local_bh().
+ */
+static inline void kunmap_local_bh(const struct buffer_head *bh, void *addr)
+{
+	if (bh->b_folio)
+		kunmap_local(addr);
+}
+
+/* If we *know* folio->private refers to buffer_heads */
 #define folio_buffers(folio)		folio_get_private(folio)
 
 void buffer_check_dirty_writeback(struct folio *folio,
@@ -197,7 +218,6 @@ void folio_set_bh(struct buffer_head *bh, struct folio *folio,
 		  unsigned long offset);
 struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,
 					gfp_t gfp);
-struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size);
 struct buffer_head *create_empty_buffers(struct folio *folio,
 		unsigned long blocksize, unsigned long b_state);
 void end_buffer_read_sync(struct buffer_head *bh, int uptodate);
@@ -338,20 +358,58 @@ static inline void bforget(struct buffer_head *bh)
 		__bforget(bh);
 }
 
-static inline struct buffer_head *
-sb_bread(struct super_block *sb, sector_t block)
+/**
+ * sb_bread - Read a block.
+ * @sb: The superblock to read from.
+ * @block: Block number in units of block size.
+ *
+ * Read a specified block, and return the buffer head that refers
+ * to it.  The memory is allocated from the movable area so that it can
+ * be migrated.  The returned buffer head has its refcount increased.
+ * The caller should call brelse() when it has finished with the buffer.
+ *
+ * Context: May sleep waiting for I/O.
+ * Return: NULL if the block was unreadable.
+ */
+static inline
+struct buffer_head *sb_bread(struct super_block *sb, sector_t block)
 {
 	return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, __GFP_MOVABLE);
 }
 
-static inline struct buffer_head *
-sb_bread_unmovable(struct super_block *sb, sector_t block)
+/**
+ * sb_bread_unmovable - Read a block.
+ * @sb: The superblock to read from.
+ * @block: Block number in units of block size.
+ *
+ * Read a specified block, and return the buffer head that refers to it.
+ * The memory is allocated from the unmovable area so that pointers into
+ * it remain valid after compaction runs.  The returned buffer head has
+ * its refcount increased.  The caller should call brelse() when it has
+ * finished with the buffer.
+ *
+ * Context: May sleep waiting for I/O.
+ * Return: NULL if the block was unreadable.
+ */
+static inline
+struct buffer_head *sb_bread_unmovable(struct super_block *sb, sector_t block)
 {
 	return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, 0);
 }
 
-static inline void
-sb_breadahead(struct super_block *sb, sector_t block)
+/**
+ * sb_breadahead - Start readahead.
+ * @sb: Superblock identifying the block device.
+ * @block: The block to read.
+ *
+ * Read this block.  The I/O will be flagged as being readahead rather
+ * than immediate read, but (unlike the page cache), surrounding blocks
+ * will not be read.
+ *
+ * Context: May sleep in order to allocate memory.
+ */
+static inline
+void sb_breadahead(struct super_block *sb, sector_t block)
 {
 	__breadahead(sb->s_bdev, block, sb->s_blocksize);
 }
diff --git a/include/linux/cache.h b/include/linux/cache.h
index e69768f50d5327..b6e857b985bcaa 100644
--- a/include/linux/cache.h
+++ b/include/linux/cache.h
@@ -89,6 +89,7 @@
  */
 #ifndef INTERNODE_CACHE_SHIFT
 #define INTERNODE_CACHE_SHIFT L1_CACHE_SHIFT
+#define INTERNODE_CACHE_BYTES (1 << INTERNODE_CACHE_SHIFT)
 #endif
 
 #if !defined(____cacheline_internodealigned_in_smp)
diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h
index fc879ac4cc4f28..56fa646df0d122 100644
--- a/include/linux/cacheinfo.h
+++ b/include/linux/cacheinfo.h
@@ -82,6 +82,7 @@ struct cpu_cacheinfo {
 };
 
 struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu);
+int get_cpu_cacheinfo_id(int cpu, int level);
 int early_cache_level(unsigned int cpu);
 int init_cache_level(unsigned int cpu);
 int init_of_cache_level(unsigned int cpu);
@@ -137,17 +138,6 @@ static inline struct cacheinfo *get_cpu_cacheinfo_level(int cpu, int level)
 	return NULL;
 }
 
-/*
- * Get the id of the cache associated with @cpu at level @level.
- * cpuhp lock must be held.
- */
-static inline int get_cpu_cacheinfo_id(int cpu, int level)
-{
-	struct cacheinfo *ci = get_cpu_cacheinfo_level(cpu, level);
-
-	return ci ? ci->id : -1;
-}
-
 #if defined(CONFIG_ARM64) || defined(CONFIG_ARM)
 #define use_arch_cache_info()	(true)
 #else
diff --git a/include/linux/capability.h b/include/linux/capability.h
index f8532d92fcad69..622137f66f0928 100644
--- a/include/linux/capability.h
+++ b/include/linux/capability.h
@@ -186,9 +186,9 @@ static inline bool ns_capable_setid(struct user_namespace *ns, int cap)
 }
 #endif /* CONFIG_MULTIUSER */
 bool privileged_wrt_inode_uidgid(struct user_namespace *ns,
-				 struct mnt_idmap *idmap,
+				 const struct mnt_idmap *idmap,
 				 const struct inode *inode);
-bool capable_wrt_inode_uidgid(struct mnt_idmap *idmap,
+bool capable_wrt_inode_uidgid(const struct mnt_idmap *idmap,
 			      const struct inode *inode, int cap);
 extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap);
 extern bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns);
@@ -215,11 +215,11 @@ static inline bool checkpoint_restore_ns_capable_noaudit(struct user_namespace *
 }
 
 /* audit system wants to get cap info from files as well */
-int get_vfs_caps_from_disk(struct mnt_idmap *idmap,
+int get_vfs_caps_from_disk(const struct mnt_idmap *idmap,
 			   const struct dentry *dentry,
 			   struct cpu_vfs_cap_data *cpu_caps);
 
-int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry,
+int cap_convert_nscap(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      const void **ivalue, size_t size);
 
 #endif /* !_LINUX_CAPABILITY_H */
diff --git a/include/linux/ccp.h b/include/linux/ccp.h
index 868924dec5a17b..e6c599243f116f 100644
--- a/include/linux/ccp.h
+++ b/include/linux/ccp.h
@@ -26,7 +26,7 @@ struct ccp_cmd;
 /**
  * ccp_present - check if a CCP device is present
  *
- * Returns zero if a CCP device is present, -ENODEV otherwise.
+ * Returns: zero if a CCP device is present, -ENODEV otherwise.
  */
 int ccp_present(void);
 
@@ -38,7 +38,7 @@ int ccp_present(void);
 /**
  * ccp_version - get the version of the CCP
  *
- * Returns a positive version number, or zero if no CCP
+ * Returns: a positive version number, or zero if no CCP
  */
 unsigned int ccp_version(void);
 
@@ -61,9 +61,9 @@ unsigned int ccp_version(void);
  * will be -EINPROGRESS. Any other "err" value during callback is
  * the result of the operation.
  *
- * The cmd has been successfully queued if:
- *   the return code is -EINPROGRESS or
- *   the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set
+ * Returns: The cmd has been successfully queued if:
+ * * the return code is -EINPROGRESS or
+ * * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set
  */
 int ccp_enqueue_cmd(struct ccp_cmd *cmd);
 
@@ -89,7 +89,7 @@ static inline int ccp_enqueue_cmd(struct ccp_cmd *cmd)
 
 /***** AES engine *****/
 /**
- * ccp_aes_type - AES key size
+ * enum ccp_aes_type - AES key size
  *
  * @CCP_AES_TYPE_128: 128-bit key
  * @CCP_AES_TYPE_192: 192-bit key
@@ -99,11 +99,12 @@ enum ccp_aes_type {
 	CCP_AES_TYPE_128 = 0,
 	CCP_AES_TYPE_192,
 	CCP_AES_TYPE_256,
+	/* private: */
 	CCP_AES_TYPE__LAST,
 };
 
 /**
- * ccp_aes_mode - AES operation mode
+ * enum ccp_aes_mode - AES operation mode
  *
  * @CCP_AES_MODE_ECB: ECB mode
  * @CCP_AES_MODE_CBC: CBC mode
@@ -111,6 +112,10 @@ enum ccp_aes_type {
  * @CCP_AES_MODE_CFB: CFB mode
  * @CCP_AES_MODE_CTR: CTR mode
  * @CCP_AES_MODE_CMAC: CMAC mode
+ * @CCP_AES_MODE_GHASH: GHASH mode
+ * @CCP_AES_MODE_GCTR: GCTR mode
+ * @CCP_AES_MODE_GCM: GCM mode
+ * @CCP_AES_MODE_GMAC: GMAC mode
  */
 enum ccp_aes_mode {
 	CCP_AES_MODE_ECB = 0,
@@ -123,11 +128,12 @@ enum ccp_aes_mode {
 	CCP_AES_MODE_GCTR,
 	CCP_AES_MODE_GCM,
 	CCP_AES_MODE_GMAC,
+	/* private: */
 	CCP_AES_MODE__LAST,
 };
 
 /**
- * ccp_aes_mode - AES operation mode
+ * enum ccp_aes_action - AES operation mode
  *
  * @CCP_AES_ACTION_DECRYPT: AES decrypt operation
  * @CCP_AES_ACTION_ENCRYPT: AES encrypt operation
@@ -135,6 +141,7 @@ enum ccp_aes_mode {
 enum ccp_aes_action {
 	CCP_AES_ACTION_DECRYPT = 0,
 	CCP_AES_ACTION_ENCRYPT,
+	/* private: */
 	CCP_AES_ACTION__LAST,
 };
 /* Overloaded field */
@@ -146,6 +153,7 @@ enum ccp_aes_action {
  * @type: AES operation key size
  * @mode: AES operation mode
  * @action: AES operation (decrypt/encrypt)
+ * @authsize: AES block request size
  * @key: key to be used for this AES operation
  * @key_len: length in bytes of key
  * @iv: IV to be used for this AES operation
@@ -156,6 +164,7 @@ enum ccp_aes_action {
  * @cmac_final: indicates final operation when running in CMAC mode
  * @cmac_key: K1/K2 key used in final CMAC operation
  * @cmac_key_len: length in bytes of cmac_key
+ * @aad_len: length in bytes of Additional Authenticated Data
  *
  * Variables required to be set when calling ccp_enqueue_cmd():
  *   - type, mode, action, key, key_len, src, dst, src_len
@@ -192,7 +201,7 @@ struct ccp_aes_engine {
 
 /***** XTS-AES engine *****/
 /**
- * ccp_xts_aes_unit_size - XTS unit size
+ * enum ccp_xts_aes_unit_size - XTS unit size
  *
  * @CCP_XTS_AES_UNIT_SIZE_16: Unit size of 16 bytes
  * @CCP_XTS_AES_UNIT_SIZE_512: Unit size of 512 bytes
@@ -206,11 +215,13 @@ enum ccp_xts_aes_unit_size {
 	CCP_XTS_AES_UNIT_SIZE_1024,
 	CCP_XTS_AES_UNIT_SIZE_2048,
 	CCP_XTS_AES_UNIT_SIZE_4096,
+	/* private: */
 	CCP_XTS_AES_UNIT_SIZE__LAST,
 };
 
 /**
  * struct ccp_xts_aes_engine - CCP XTS AES operation
+ * @type: ccp_aes_type - AES key size
  * @action: AES operation (decrypt/encrypt)
  * @unit_size: unit size of the XTS operation
  * @key: key to be used for this XTS AES operation
@@ -247,11 +258,13 @@ struct ccp_xts_aes_engine {
 
 /***** SHA engine *****/
 /**
- * ccp_sha_type - type of SHA operation
+ * enum ccp_sha_type - type of SHA operation
  *
  * @CCP_SHA_TYPE_1: SHA-1 operation
  * @CCP_SHA_TYPE_224: SHA-224 operation
  * @CCP_SHA_TYPE_256: SHA-256 operation
+ * @CCP_SHA_TYPE_384: SHA-384 operation
+ * @CCP_SHA_TYPE_512: SHA-512 operation
  */
 enum ccp_sha_type {
 	CCP_SHA_TYPE_1 = 1,
@@ -259,6 +272,7 @@ enum ccp_sha_type {
 	CCP_SHA_TYPE_256,
 	CCP_SHA_TYPE_384,
 	CCP_SHA_TYPE_512,
+	/* private: */
 	CCP_SHA_TYPE__LAST,
 };
 
@@ -384,7 +398,7 @@ struct ccp_rsa_engine {
 
 /***** Passthru engine *****/
 /**
- * ccp_passthru_bitwise - type of bitwise passthru operation
+ * enum ccp_passthru_bitwise - type of bitwise passthru operation
  *
  * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed
  * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask
@@ -398,11 +412,12 @@ enum ccp_passthru_bitwise {
 	CCP_PASSTHRU_BITWISE_OR,
 	CCP_PASSTHRU_BITWISE_XOR,
 	CCP_PASSTHRU_BITWISE_MASK,
+	/* private: */
 	CCP_PASSTHRU_BITWISE__LAST,
 };
 
 /**
- * ccp_passthru_byteswap - type of byteswap passthru operation
+ * enum ccp_passthru_byteswap - type of byteswap passthru operation
  *
  * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed
  * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words
@@ -412,6 +427,7 @@ enum ccp_passthru_byteswap {
 	CCP_PASSTHRU_BYTESWAP_NOOP = 0,
 	CCP_PASSTHRU_BYTESWAP_32BIT,
 	CCP_PASSTHRU_BYTESWAP_256BIT,
+	/* private: */
 	CCP_PASSTHRU_BYTESWAP__LAST,
 };
 
@@ -450,8 +466,8 @@ struct ccp_passthru_engine {
  * @byte_swap: byteswap operation to perform
  * @mask: mask to be applied to data
  * @mask_len: length in bytes of mask
- * @src: data to be used for this operation
- * @dst: data produced by this operation
+ * @src_dma: data to be used for this operation
+ * @dst_dma: data produced by this operation
  * @src_len: length in bytes of data used for this operation
  * @final: indicate final pass-through operation
  *
@@ -478,7 +494,7 @@ struct ccp_passthru_nomap_engine {
 #define CCP_ECC_MAX_OUTPUTS	3
 
 /**
- * ccp_ecc_function - type of ECC function
+ * enum ccp_ecc_function - type of ECC function
  *
  * @CCP_ECC_FUNCTION_MMUL_384BIT: 384-bit modular multiplication
  * @CCP_ECC_FUNCTION_MADD_384BIT: 384-bit modular addition
@@ -564,8 +580,9 @@ struct ccp_ecc_point_math {
  * @function: ECC function to perform
  * @mod: ECC modulus
  * @mod_len: length in bytes of modulus
- * @mm: module math parameters
- * @pm: point math parameters
+ * @u: union for math parameters
+ * @u.mm: module math parameters
+ * @u.pm: point math parameters
  * @ecc_result: result of the ECC operation
  *
  * Variables required to be set when calling ccp_enqueue_cmd():
@@ -589,11 +606,11 @@ struct ccp_ecc_engine {
 
 
 /**
- * ccp_engine - CCP operation identifiers
+ * enum ccp_engine - CCP operation identifiers
  *
  * @CCP_ENGINE_AES: AES operation
- * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation
- * @CCP_ENGINE_RSVD1: unused
+ * @CCP_ENGINE_XTS_AES_128: 128-bit XTS AES operation
+ * @CCP_ENGINE_DES3: 3DES operation
  * @CCP_ENGINE_SHA: SHA operation
  * @CCP_ENGINE_RSA: RSA operation
  * @CCP_ENGINE_PASSTHRU: pass-through operation
@@ -609,6 +626,7 @@ enum ccp_engine {
 	CCP_ENGINE_PASSTHRU,
 	CCP_ENGINE_ZLIB_DECOMPRESS,
 	CCP_ENGINE_ECC,
+	/* private: */
 	CCP_ENGINE__LAST,
 };
 
diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
index 691e1bdece49c3..f92fdd853f1f34 100644
--- a/include/linux/ceph/libceph.h
+++ b/include/linux/ceph/libceph.h
@@ -11,7 +11,6 @@
 #include <linux/bug.h>
 #include <linux/fs.h>
 #include <linux/mempool.h>
-#include <linux/pagemap.h>
 #include <linux/wait.h>
 #include <linux/writeback.h>
 #include <linux/slab.h>
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index 5dfa915a630e8d..2afb4cb2bb4fa4 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -154,6 +154,9 @@ struct cgroup *cgroup_get_from_path(const char *path);
 struct cgroup *cgroup_get_from_fd(int fd);
 struct cgroup *cgroup_v1v2_get_from_fd(int fd);
 
+/* Was this controller blocked from v1 hierarchies by cgroup_no_v1= ? */
+bool cgroup1_ssid_disabled(int ssid);
+
 int cgroup_attach_task_all(struct task_struct *from, struct task_struct *);
 int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from);
 
diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h
index b1b5698cbf1b6f..1fb8058b897d4a 100644
--- a/include/linux/cleanup.h
+++ b/include/linux/cleanup.h
@@ -261,10 +261,6 @@ const volatile void * __must_check_fn(const volatile void *val)
  * CLASS(name, var)(args...):
  *	declare the variable @var as an instance of the named class
  *
- * CLASS_INIT(name, var, init_expr):
- *	declare the variable @var as an instance of the named class with
- *	custom initialization expression.
- *
  * Ex.
  *
  * DEFINE_CLASS(fdget, struct fd, fdput(_T), fdget(fd), int fd)
@@ -302,9 +298,6 @@ static __always_inline class_##_name##_t class_##_name##ext##_constructor(_init_
 	class_##_name##_t var __cleanup(class_##_name##_destructor) =	\
 		class_##_name##_constructor
 
-#define CLASS_INIT(_name, _var, _init_expr)                             \
-        class_##_name##_t _var __cleanup(class_##_name##_destructor) = (_init_expr)
-
 #define __scoped_class(_name, var, _label, args...)        \
 	for (CLASS(_name, var)(args); ; ({ goto _label; })) \
 		if (0) {                                   \
diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h
index dc8e161f9c2cc3..a355d1ee5189d4 100644
--- a/include/linux/clk-provider.h
+++ b/include/linux/clk-provider.h
@@ -7,6 +7,7 @@
 #define __LINUX_CLK_PROVIDER_H
 
 #include <dt-bindings/clock/clock.h>
+#include <linux/bits.h>
 #include <linux/of.h>
 #include <linux/of_clk.h>
 
@@ -50,6 +51,7 @@ struct clk;
 struct clk_hw;
 struct clk_core;
 struct dentry;
+struct module;
 
 /**
  * struct clk_rate_request - Structure encoding the clk constraints that
@@ -351,8 +353,9 @@ struct clk_init_data {
  * @core: pointer to the struct clk_core instance that points back to this
  * struct clk_hw instance
  *
- * @clk: pointer to the per-user struct clk instance that can be used to call
- * into the clk API
+ * @clk: pointer to the per-user struct clk instance. This will be removed at
+ * some point in the future, Please consider the field obsolete and do not use
+ * it. Use clk_hw_get_clk() to get a per-user struct clk from a struct clk_hw.
  *
  * @init: pointer to struct clk_init_data that contains the init data shared
  * with the common clock framework. This pointer will be set to NULL once
@@ -755,7 +758,7 @@ struct clk_divider {
 	spinlock_t	*lock;
 };
 
-#define clk_div_mask(width)	((1 << (width)) - 1)
+#define clk_div_mask(width)	GENMASK((width) - 1, 0)
 #define to_clk_divider(_hw) container_of(_hw, struct clk_divider, hw)
 
 #define CLK_DIVIDER_ONE_BASED		BIT(0)
@@ -1474,10 +1477,24 @@ static inline struct clk_hw *__clk_get_hw(struct clk *clk)
 }
 #endif
 
-struct clk *clk_hw_get_clk(struct clk_hw *hw, const char *con_id);
+struct clk *__clk_hw_get_clk(struct clk_hw *hw, const char *con_id,
+			     struct module *owner);
 struct clk *devm_clk_hw_get_clk(struct device *dev, struct clk_hw *hw,
 				const char *con_id);
 
+/**
+ * clk_hw_get_clk - get clk consumer given a clk_hw
+ * @hw: clk_hw associated with the clk being consumed
+ * @con_id: connection ID string on device
+ *
+ * Return: new clk consumer
+ * This is the function to be used by providers which need
+ * to get a consumer clk and act on the clock element
+ * Calls to this function must be balanced with calls to clk_put()
+ */
+#define clk_hw_get_clk(hw, con_id) \
+	__clk_hw_get_clk((hw), (con_id), THIS_MODULE)
+
 unsigned int clk_hw_get_num_parents(const struct clk_hw *hw);
 struct clk_hw *clk_hw_get_parent(const struct clk_hw *hw);
 struct clk_hw *clk_hw_get_parent_by_index(const struct clk_hw *hw,
diff --git a/include/linux/clk/tegra.h b/include/linux/clk/tegra.h
index 3650e926e93f44..f1034e14c1a44b 100644
--- a/include/linux/clk/tegra.h
+++ b/include/linux/clk/tegra.h
@@ -170,7 +170,7 @@ struct tegra210_clk_emc_provider {
 	struct module *owner;
 	struct device *dev;
 
-	struct tegra210_clk_emc_config *configs;
+	struct tegra210_clk_emc_config *configs __counted_by_ptr(num_configs);
 	unsigned int num_configs;
 
 	int (*set_rate)(struct device *dev,
diff --git a/include/linux/clk/ti.h b/include/linux/clk/ti.h
index 54a3fa37000477..61c4abf199959f 100644
--- a/include/linux/clk/ti.h
+++ b/include/linux/clk/ti.h
@@ -51,7 +51,7 @@ struct clk_omap_reg {
  * @autoidle_mask: mask of the DPLL autoidle mode bitfield in @autoidle_reg
  * @freqsel_mask: mask of the DPLL jitter correction bitfield in @control_reg
  * @dcc_mask: mask of the DPLL DCC correction bitfield @mult_div1_reg
- * @dcc_rate: rate atleast which DCC @dcc_mask must be set
+ * @dcc_rate: rate at least which DCC @dcc_mask must be set
  * @idlest_mask: mask of the DPLL idle status bitfield in @idlest_reg
  * @lpmode_mask: mask of the DPLL low-power mode bitfield in @control_reg
  * @m4xen_mask: mask of the DPLL M4X multiplier bitfield in @control_reg
diff --git a/include/linux/compaction.h b/include/linux/compaction.h
index 66a2f70e9e019d..691c09f0a6971d 100644
--- a/include/linux/compaction.h
+++ b/include/linux/compaction.h
@@ -81,10 +81,6 @@ static inline unsigned long compact_gap(unsigned int order)
 	return min(2UL << order, COMPACT_CLUSTER_MAX);
 }
 
-static inline int current_is_kcompactd(void)
-{
-	return current->flags & PF_KCOMPACTD;
-}
 
 #ifdef CONFIG_COMPACTION
 
@@ -103,7 +99,7 @@ extern void compaction_defer_reset(struct zone *zone, int order,
 
 bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
 					int alloc_flags, gfp_t gfp_mask);
-
+bool current_is_kcompactd(void);
 extern void __meminit kcompactd_run(int nid);
 extern void __meminit kcompactd_stop(int nid);
 extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx);
@@ -120,6 +116,11 @@ static inline bool compaction_suitable(struct zone *zone, int order,
 	return false;
 }
 
+static inline bool current_is_kcompactd(void)
+{
+	return false;
+}
+
 static inline void kcompactd_run(int nid)
 {
 }
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index cb2f6050bdf7dc..ef9036fa5413d6 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -275,6 +275,11 @@ static inline void *offset_to_ptr(const int *off)
 #define __ADDRESSABLE(sym) \
 	___ADDRESSABLE(sym, __section(".discard.addressable"))
 
+/* Enforce static storage duration. */
+#define ASSERT_STATIC_STORAGE(name) \
+	static typeof(name) * const __always_unused \
+		name##_storage_check = &(name)
+
 /*
  * This returns a constant expression while determining if an argument is
  * a constant expression, most importantly without evaluating the argument.
diff --git a/include/linux/configfs.h b/include/linux/configfs.h
index ef65c75beeaad8..5bead9173ec1ae 100644
--- a/include/linux/configfs.h
+++ b/include/linux/configfs.h
@@ -66,8 +66,11 @@ struct config_item_type {
 	struct module				*ct_owner;
 	const struct configfs_item_operations	*ct_item_ops;
 	const struct configfs_group_operations	*ct_group_ops;
-	struct configfs_attribute		**ct_attrs;
-	struct configfs_bin_attribute		**ct_bin_attrs;
+	union {
+		struct configfs_attribute		**ct_attrs;
+		const struct configfs_attribute		*const *ct_attrs_const;
+	};
+	const struct configfs_bin_attribute	*const *ct_bin_attrs;
 };
 
 /**
@@ -160,41 +163,41 @@ struct configfs_bin_attribute {
 	ssize_t (*write)(struct config_item *, const void *, size_t);
 };
 
-#define CONFIGFS_BIN_ATTR(_pfx, _name, _priv, _maxsz)		\
-static struct configfs_bin_attribute _pfx##attr_##_name = {	\
-	.cb_attr = {						\
-		.ca_name	= __stringify(_name),		\
-		.ca_mode	= S_IRUGO | S_IWUSR,		\
-		.ca_owner	= THIS_MODULE,			\
-	},							\
-	.cb_private	= _priv,				\
-	.cb_max_size	= _maxsz,				\
-	.read		= _pfx##_name##_read,			\
-	.write		= _pfx##_name##_write,			\
+#define CONFIGFS_BIN_ATTR(_pfx, _name, _priv, _maxsz)			\
+static const struct configfs_bin_attribute _pfx##attr_##_name = {	\
+	.cb_attr = {							\
+		.ca_name	= __stringify(_name),			\
+		.ca_mode	= S_IRUGO | S_IWUSR,			\
+		.ca_owner	= THIS_MODULE,				\
+	},								\
+	.cb_private	= _priv,					\
+	.cb_max_size	= _maxsz,					\
+	.read		= _pfx##_name##_read,				\
+	.write		= _pfx##_name##_write,				\
 }
 
-#define CONFIGFS_BIN_ATTR_RO(_pfx, _name, _priv, _maxsz)	\
-static struct configfs_bin_attribute _pfx##attr_##_name = {	\
-	.cb_attr = {						\
-		.ca_name	= __stringify(_name),		\
-		.ca_mode	= S_IRUGO,			\
-		.ca_owner	= THIS_MODULE,			\
-	},							\
-	.cb_private	= _priv,				\
-	.cb_max_size	= _maxsz,				\
-	.read		= _pfx##_name##_read,			\
+#define CONFIGFS_BIN_ATTR_RO(_pfx, _name, _priv, _maxsz)		\
+static const struct configfs_bin_attribute _pfx##attr_##_name = {	\
+	.cb_attr = {							\
+		.ca_name	= __stringify(_name),			\
+		.ca_mode	= S_IRUGO,				\
+		.ca_owner	= THIS_MODULE,				\
+	},								\
+	.cb_private	= _priv,					\
+	.cb_max_size	= _maxsz,					\
+	.read		= _pfx##_name##_read,				\
 }
 
-#define CONFIGFS_BIN_ATTR_WO(_pfx, _name, _priv, _maxsz)	\
-static struct configfs_bin_attribute _pfx##attr_##_name = {	\
-	.cb_attr = {						\
-		.ca_name	= __stringify(_name),		\
-		.ca_mode	= S_IWUSR,			\
-		.ca_owner	= THIS_MODULE,			\
-	},							\
-	.cb_private	= _priv,				\
-	.cb_max_size	= _maxsz,				\
-	.write		= _pfx##_name##_write,			\
+#define CONFIGFS_BIN_ATTR_WO(_pfx, _name, _priv, _maxsz)		\
+static const struct configfs_bin_attribute _pfx##attr_##_name = {	\
+	.cb_attr = {							\
+		.ca_name	= __stringify(_name),			\
+		.ca_mode	= S_IWUSR,				\
+		.ca_owner	= THIS_MODULE,				\
+	},								\
+	.cb_private	= _priv,					\
+	.cb_max_size	= _maxsz,					\
+	.write		= _pfx##_name##_write,				\
 }
 
 /*
@@ -220,8 +223,8 @@ struct configfs_group_operations {
 	struct config_group *(*make_group)(struct config_group *group, const char *name);
 	void (*disconnect_notify)(struct config_group *group, struct config_item *item);
 	void (*drop_item)(struct config_group *group, struct config_item *item);
-	bool (*is_visible)(struct config_item *item, struct configfs_attribute *attr, int n);
-	bool (*is_bin_visible)(struct config_item *item, struct configfs_bin_attribute *attr,
+	bool (*is_visible)(struct config_item *item, const struct configfs_attribute *attr, int n);
+	bool (*is_bin_visible)(struct config_item *item, const struct configfs_bin_attribute *attr,
 			       int n);
 };
 
diff --git a/include/linux/coredump.h b/include/linux/coredump.h
index 7b38ee2e7913be..74af57b9406b8e 100644
--- a/include/linux/coredump.h
+++ b/include/linux/coredump.h
@@ -6,9 +6,20 @@
 #include <linux/mm.h>
 #include <linux/fs.h>
 #include <linux/sched/coredump.h>
+#include <uapi/linux/coredump.h>
 #include <asm/siginfo.h>
 
 #ifdef CONFIG_COREDUMP
+/**
+ * enum coredump_state - what happened while the coredump was written
+ * @COREDUMP_STATE_STARTED: the dumper committed to writing a coredump
+ * @COREDUMP_STATE_TRUNCATED: the dumper stopped before it had written all of it
+ */
+enum coredump_state {
+	COREDUMP_STATE_STARTED		= (1U << 0),
+	COREDUMP_STATE_TRUNCATED	= (1U << 1),
+};
+
 struct core_vma_metadata {
 	unsigned long start, end;
 	vm_flags_t flags;
@@ -21,12 +32,20 @@ struct coredump_params {
 	const kernel_siginfo_t *siginfo;
 	struct file *file;
 	unsigned long limit;
-	/* MMF_DUMP_FILTER_* bits, snapshot of mm->flags at dump start. */
-	unsigned long mm_flags;
+	/* COREDUMP_MEMORY_* types to dump, the task's or the server's. */
+	u64 memory_types;
 	/* Snapshot of dumpable at dump start. */
 	enum task_dumpable dumpable;
 	int cpu;
+	/* COREDUMP_* options negotiated with the coredump server. */
+	u64 mask;
+	/* COREDUMP_STATE_* raised while the coredump is written. */
+	enum coredump_state state;
+	/* Record header scratch, NULL unless the coredump is a record stream. */
+	struct coredump_record_header *record_hdr;
+	/* Bytes handed to the file, record headers included. */
 	loff_t written;
+	/* Offset in the coredump, record headers excluded. */
 	loff_t pos;
 	loff_t to_skip;
 	int vma_count;
@@ -41,13 +60,13 @@ extern unsigned int core_file_note_size_limit;
  * These are the only things you should do on a core-file: use only these
  * functions to write out all the necessary info.
  */
-extern void dump_skip_to(struct coredump_params *cprm, unsigned long to);
-extern void dump_skip(struct coredump_params *cprm, size_t nr);
-extern int dump_emit(struct coredump_params *cprm, const void *addr, int nr);
-extern int dump_align(struct coredump_params *cprm, int align);
-int dump_user_range(struct coredump_params *cprm, unsigned long start,
-		    unsigned long len);
-extern void vfs_coredump(const kernel_siginfo_t *siginfo);
+void dump_skip_to(struct coredump_params *cprm, unsigned long to);
+void dump_skip(struct coredump_params *cprm, size_t nr);
+bool dump_emit(struct coredump_params *cprm, const void *addr, int nr);
+bool dump_align(struct coredump_params *cprm, int align);
+bool dump_user_range(struct coredump_params *cprm, unsigned long start,
+		     unsigned long len);
+void vfs_coredump(const kernel_siginfo_t *siginfo);
 
 /*
  * Logging for the coredump code, ratelimited.
diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h
index 35ce665edfd824..d3d0d9d02aa4a6 100644
--- a/include/linux/cpufreq.h
+++ b/include/linux/cpufreq.h
@@ -420,6 +420,9 @@ struct cpufreq_driver {
 	/* Will be called after the driver is fully initialized */
 	void		(*ready)(struct cpufreq_policy *policy);
 
+	/* Return the capacity reference frequency for policy. */
+	unsigned int	(*scale_freq_ref)(struct cpufreq_policy *policy);
+
 	struct freq_attr **attr;
 
 	/* platform specific boost support code */
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index 4c8bb6953107d1..bf89bb3f30f664 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -121,12 +121,20 @@ extern struct cpumask __cpu_enabled_mask;
 extern struct cpumask __cpu_present_mask;
 extern struct cpumask __cpu_active_mask;
 extern struct cpumask __cpu_dying_mask;
+
+#ifdef CONFIG_PREFERRED_CPU
+extern struct cpumask __cpu_preferred_mask;
+#else
+#define __cpu_preferred_mask __cpu_active_mask
+#endif
+
 #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
 #define cpu_online_mask   ((const struct cpumask *)&__cpu_online_mask)
 #define cpu_enabled_mask   ((const struct cpumask *)&__cpu_enabled_mask)
 #define cpu_present_mask  ((const struct cpumask *)&__cpu_present_mask)
 #define cpu_active_mask   ((const struct cpumask *)&__cpu_active_mask)
 #define cpu_dying_mask    ((const struct cpumask *)&__cpu_dying_mask)
+#define cpu_preferred_mask ((const struct cpumask *)&__cpu_preferred_mask)
 
 extern atomic_t __num_online_cpus;
 extern unsigned int __num_possible_cpus;
@@ -824,6 +832,24 @@ bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p
 						      small_cpumask_bits);
 }
 
+/**
+ * cpumask_intersects_and - (*src1p & *src2p & *src3p) != 0
+ * @src1p: the first input
+ * @src2p: the second input
+ * @src3p: the third input
+ *
+ * Return: true if AND of the three cpumasks is non-empty,
+ *	   otherwise false
+ */
+static __always_inline
+bool cpumask_intersects_and(const struct cpumask *src1p,
+			    const struct cpumask *src2p,
+			    const struct cpumask *src3p)
+{
+	return bitmap_intersects_and(cpumask_bits(src1p), cpumask_bits(src2p),
+				 cpumask_bits(src3p), small_cpumask_bits);
+}
+
 /**
  * cpumask_subset - (*src1p & ~*src2p) == 0
  * @src1p: the first input
@@ -1163,6 +1189,12 @@ void init_cpu_possible(const struct cpumask *src);
 #define set_cpu_active(cpu, active)	assign_cpu((cpu), &__cpu_active_mask, (active))
 #define set_cpu_dying(cpu, dying)	assign_cpu((cpu), &__cpu_dying_mask, (dying))
 
+#ifdef CONFIG_PREFERRED_CPU
+#define set_cpu_preferred(cpu, preferred) assign_cpu((cpu), &__cpu_preferred_mask, (preferred))
+#else
+#define set_cpu_preferred(cpu, preferred) do { } while (0)
+#endif
+
 void set_cpu_online(unsigned int cpu, bool online);
 void set_cpu_possible(unsigned int cpu, bool possible);
 
@@ -1257,6 +1289,11 @@ static __always_inline bool cpu_dying(unsigned int cpu)
 	return cpumask_test_cpu(cpu, cpu_dying_mask);
 }
 
+static __always_inline bool cpu_preferred(unsigned int cpu)
+{
+	return cpumask_test_cpu(cpu, cpu_preferred_mask);
+}
+
 #else
 
 #define num_online_cpus()	1U
@@ -1295,6 +1332,11 @@ static __always_inline bool cpu_dying(unsigned int cpu)
 	return false;
 }
 
+static __always_inline bool cpu_preferred(unsigned int cpu)
+{
+	return cpu == 0;
+}
+
 #endif /* NR_CPUS > 1 */
 
 #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))
diff --git a/include/linux/cred.h b/include/linux/cred.h
index 6ef1750c93e2a4..49c26af3734908 100644
--- a/include/linux/cred.h
+++ b/include/linux/cred.h
@@ -180,12 +180,18 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred)
 
 static inline const struct cred *override_creds(const struct cred *override_cred)
 {
-	return rcu_replace_pointer(current->cred, override_cred, 1);
+	const struct cred *old = current->cred;
+
+	current->cred = override_cred;
+	return old;
 }
 
 static inline const struct cred *revert_creds(const struct cred *revert_cred)
 {
-	return rcu_replace_pointer(current->cred, revert_cred, 1);
+	const struct cred *override_cred = current->cred;
+
+	current->cred = revert_cred;
+	return override_cred;
 }
 
 DEFINE_CLASS(override_creds,
@@ -293,11 +299,10 @@ DEFINE_FREE(put_cred, struct cred *, if (!IS_ERR_OR_NULL(_T)) put_cred(_T))
 /**
  * current_cred - Access the current task's subjective credentials
  *
- * Access the subjective credentials of the current task.  RCU-safe,
- * since nobody else can modify it.
+ * Access the subjective credentials of the current task.
+ * Nobody else can modify it.
  */
-#define current_cred() \
-	rcu_dereference_protected(current->cred, 1)
+#define current_cred() (current->cred)
 
 /**
  * current_real_cred - Access the current task's objective credentials
diff --git a/include/linux/damon.h b/include/linux/damon.h
index 0c8b7ddef9abb3..6a29dc2ac8db79 100644
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -117,7 +117,6 @@ struct damon_target {
  * @DAMOS_MIGRATE_HOT:  Migrate the regions prioritizing warmer regions.
  * @DAMOS_MIGRATE_COLD:	Migrate the regions prioritizing colder regions.
  * @DAMOS_STAT:		Do nothing but count the stat.
- * @NR_DAMOS_ACTIONS:	Total number of DAMOS actions
  *
  * The support of each action is up to running &struct damon_operations.
  * Refer to 'Operation Action' section of Documentation/mm/damon/design.rst for
@@ -137,7 +136,6 @@ enum damos_action {
 	DAMOS_MIGRATE_HOT,
 	DAMOS_MIGRATE_COLD,
 	DAMOS_STAT,		/* Do nothing but only record the stat */
-	NR_DAMOS_ACTIONS,
 };
 
 /**
@@ -153,9 +151,7 @@ enum damos_action {
  * @DAMOS_QUOTA_INACTIVE_MEM_BP:	Inactive to total LRU memory ratio.
  * @DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP:	Scheme-eligible memory ratio of a
  *					node in basis points (0-10000).
- * @NR_DAMOS_QUOTA_GOAL_METRICS:	Number of DAMOS quota goal metrics.
- *
- * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported.
+ * @DAMOS_QUOTA_HUGEPAGE_MEM_BP:	Huge page to total used memory ratio.
  */
 enum damos_quota_goal_metric {
 	DAMOS_QUOTA_USER_INPUT,
@@ -167,7 +163,7 @@ enum damos_quota_goal_metric {
 	DAMOS_QUOTA_ACTIVE_MEM_BP,
 	DAMOS_QUOTA_INACTIVE_MEM_BP,
 	DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP,
-	NR_DAMOS_QUOTA_GOAL_METRICS,
+	DAMOS_QUOTA_HUGEPAGE_MEM_BP,
 };
 
 /**
@@ -311,12 +307,10 @@ struct damos_quota {
  *
  * @DAMOS_WMARK_NONE:		Ignore the watermarks of the given scheme.
  * @DAMOS_WMARK_FREE_MEM_RATE:	Free memory rate of the system in [0,1000].
- * @NR_DAMOS_WMARK_METRICS:	Total number of DAMOS watermark metrics
  */
 enum damos_wmark_metric {
 	DAMOS_WMARK_NONE,
 	DAMOS_WMARK_FREE_MEM_RATE,
-	NR_DAMOS_WMARK_METRICS,
 };
 
 /**
@@ -357,7 +351,8 @@ struct damos_watermarks {
  *		Total bytes that passed ops layer-handled DAMOS filters.
  * @qt_exceeds: Total number of times the quota of the scheme has exceeded.
  * @nr_snapshots:
- *		Total number of DAMON snapshots that the scheme has tried.
+ *		Total number of DAMON snapshots that the scheme is completely
+ *		tried to be applied.
  *
  * "Tried an action to a region" in this context means the DAMOS core logic
  * determined the region as eligible to apply the action.  The access pattern
@@ -396,14 +391,14 @@ struct damos_stat {
  * @DAMOS_FILTER_TYPE_UNMAPPED:	Unmapped pages.
  * @DAMOS_FILTER_TYPE_ADDR:	Address range.
  * @DAMOS_FILTER_TYPE_TARGET:	Data Access Monitoring target.
- * @NR_DAMOS_FILTER_TYPES:	Number of filter types.
+ * @DAMOS_FILTER_TYPE_PROBE_HITS_WSUM:	probe_hits weighted sum range.
  *
- * All types except &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET
- * are handled by the underlying &struct damon_operations as a part of scheme
- * action trying, and therefore accounted as 'tried'.  In contrast,
- * &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET filters are handled
- * by the core layer before trying of the action, and therefore not accounted
- * as 'tried'.
+ * All types except &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET and
+ * &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM are handled by the underlying &struct
+ * damon_operations as a part of scheme action trying, and therefore accounted
+ * as 'tried'.  In contrast, &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET
+ * and &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM filters are handled by the core layer
+ * before trying of the action, and therefore not accounted as 'tried'.
  *
  * Support for the operations-handled filters depends on the running
  * &struct damon_operations.
@@ -417,7 +412,7 @@ enum damos_filter_type {
 	DAMOS_FILTER_TYPE_UNMAPPED,
 	DAMOS_FILTER_TYPE_ADDR,
 	DAMOS_FILTER_TYPE_TARGET,
-	NR_DAMOS_FILTER_TYPES,
+	DAMOS_FILTER_TYPE_PROBE_HITS_WSUM,
 };
 
 /**
@@ -431,6 +426,8 @@ enum damos_filter_type {
  *		&damon_ctx->adaptive_targets if @type is
  *		DAMOS_FILTER_TYPE_TARGET.
  * @sz_range:	Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE.
+ * @range_min:	Minimum value of range arguments.
+ * @range_max:	Maximum value of range arguments.
  *
  * Before applying the &damos->action to a memory region, DAMOS checks if each
  * byte of the region matches to this given condition and avoid applying the
@@ -447,6 +444,10 @@ struct damos_filter {
 		struct damon_addr_range addr_range;
 		int target_idx;
 		struct damon_size_range sz_range;
+		struct {
+			unsigned long range_min;
+			unsigned long range_max;
+		};
 	};
 /* private: */
 	/* List head for siblings. */
@@ -548,7 +549,7 @@ struct damos_migrate_dests {
  *
  * After applying the &action to each region, &stat is updated.
  *
- * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots be same to or
+ * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots equals or is
  * greater than it, the scheme is deactivated.
  */
 struct damos {
@@ -627,6 +628,7 @@ enum damon_ops_id {
  * @update:			Update operations-related data structures.
  * @prepare_access_checks:	Prepare next access check of target regions.
  * @check_accesses:		Check the accesses to target regions.
+ * @prep_probes:		Prepare applying probes for each region.
  * @apply_probes:		Apply probes for each region.
  * @get_scheme_score:		Get the score of a region for a scheme.
  * @apply_scheme:		Apply a DAMON-based operation scheme.
@@ -654,6 +656,9 @@ enum damon_ops_id {
  * last preparation and update the number of observed accesses of each region.
  * It should also return max number of observed accesses that made as a result
  * of its update.  The value will be used for regions adjustment threshold.
+ * @prep_probes should execute required &struct damon_prep for next &struct
+ * damon_probe applications to each region.  It should also set
+ * &damon_region->sampling_addr of each region if ``set_samples`` is true.
  * @apply_probes should apply the data attribute probes to each region and
  * accordingly update the probe hits counter of the region.  It should also
  * set &damon_region->sampling_addr of each region if ``set_samples`` is true.
@@ -676,6 +681,7 @@ struct damon_operations {
 	void (*update)(struct damon_ctx *context);
 	void (*prepare_access_checks)(struct damon_ctx *context);
 	unsigned int (*check_accesses)(struct damon_ctx *context);
+	void (*prep_probes)(struct damon_ctx *context, bool set_samples);
 	unsigned int (*apply_probes)(struct damon_ctx *context,
 			bool set_samples, bool return_max_wsum);
 	int (*get_scheme_score)(struct damon_ctx *context,
@@ -739,15 +745,42 @@ struct damon_intervals_goal {
 	unsigned long max_sample_us;
 };
 
+/**
+ * enum damon_prep_action - DAMON probing preparation action.
+ *
+ * @DAMON_PREP_SET_PGIDLE:	Set the probing memory as idle page.
+ */
+enum damon_prep_action {
+	DAMON_PREP_SET_PGIDLE,
+};
+
+/**
+ * struct damon_prep - DAMON probing preparation request.
+ *
+ * @action:	Action to do to the probing memory for the preparation.
+ */
+struct damon_prep {
+	enum damon_prep_action action;
+/* private: */
+	/* siblings list. */
+	struct list_head list;
+};
+
 /**
  * enum damon_filter_type - Type of &struct damon_filter
  *
- * @DAMON_FILTER_TYPE_ANON:	Anonymous pages.
- * @DAMON_FILTER_TYPE_MEMCG:	Specific memcg's pages.
+ * @DAMON_FILTER_TYPE_ANON:		Anonymous pages.
+ * @DAMON_FILTER_TYPE_MEMCG:		Specific memcg's pages.
+ * @DAMON_FILTER_TYPE_PGIDLE_UNSET:	Pgidle is unset.
+ * @DAMON_FILTER_TYPE_PGIDLE_SET:	Pgidle is set.
+ * @DAMON_FILTER_TYPE_HUGEPAGE_SIZE:	Page is part of a hugepage.
  */
 enum damon_filter_type {
 	DAMON_FILTER_TYPE_ANON,
 	DAMON_FILTER_TYPE_MEMCG,
+	DAMON_FILTER_TYPE_PGIDLE_UNSET,
+	DAMON_FILTER_TYPE_PGIDLE_SET,
+	DAMON_FILTER_TYPE_HUGEPAGE_SIZE,
 };
 
 /**
@@ -757,6 +790,8 @@ enum damon_filter_type {
  * @matching:	Whether this filter is for the type-matching ones.
  * @allow:	Whether the @type-@matching ones should pass this filter.
  * @memcg_id:	Memcg id of the question if @type is DAMON_FILTER_MEMCG.
+ * @range_min:	Minimum value of range arguments.
+ * @range_max:	Maximum value of range arguments.
  */
 struct damon_filter {
 	enum damon_filter_type type;
@@ -764,6 +799,10 @@ struct damon_filter {
 	bool allow;
 	union {
 		u64 memcg_id;
+		struct {
+			unsigned long range_min;
+			unsigned long range_max;
+		};
 	};
 /* private: */
 	/* Siblings list. */
@@ -778,6 +817,8 @@ struct damon_filter {
 struct damon_probe {
 	unsigned int weight;
 /* private: */
+	/* Preparation actions to apply to each probing memory. */
+	struct list_head preps;
 	/* Filters for assessing if a given region is for this probe. */
 	struct list_head filters;
 	/* Siblings list. */
@@ -787,7 +828,8 @@ struct damon_probe {
 /**
  * struct damon_attrs - Monitoring attributes for accuracy/overhead control.
  *
- * @sample_interval:		The time between access samplings.
+ * @sample_interval:		The time between access samplings.  Zero is
+ *				accepted.
  * @aggr_interval:		The time between monitor results aggregations.
  * @ops_update_interval:	The time between monitoring operations updates.
  * @intervals_goal:		Intervals auto-tuning goal.
@@ -957,6 +999,12 @@ static inline unsigned long damon_sz_region(struct damon_region *r)
 	return r->ar.end - r->ar.start;
 }
 
+#define damon_for_each_prep(p, probe) \
+	list_for_each_entry(p, &(probe)->preps, list)
+
+#define damon_for_each_prep_safe(p, next, probe) \
+	list_for_each_entry_safe(p, next, &(probe)->preps, list)
+
 #define damon_for_each_filter(f, p) \
 	list_for_each_entry(f, &(p)->filters, list)
 
@@ -1010,6 +1058,9 @@ static inline unsigned long damon_sz_region(struct damon_region *r)
 
 #ifdef CONFIG_DAMON
 
+struct damon_prep *damon_new_prep(enum damon_prep_action action);
+void damon_add_prep(struct damon_probe *p, struct damon_prep *prep);
+
 struct damon_filter *damon_new_filter(enum damon_filter_type type,
 		bool matching, bool allow);
 void damon_add_filter(struct damon_probe *probe, struct damon_filter *f);
@@ -1023,7 +1074,7 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r,
 		struct damon_ctx *ctx);
 unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r,
 		struct damon_ctx *ctx);
-unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last,
+unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, bool mv,
 		struct damon_ctx *ctx);
 
 int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
@@ -1054,7 +1105,7 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src);
 
 struct damon_target *damon_new_target(void);
 void damon_add_target(struct damon_ctx *ctx, struct damon_target *t);
-bool damon_targets_empty(struct damon_ctx *ctx);
+int damon_set_target_pid(struct damon_target *t, int pid);
 void damon_free_target(struct damon_target *t);
 void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx);
 unsigned int damon_nr_regions(struct damon_target *t);
@@ -1065,7 +1116,6 @@ int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs);
 void damon_set_schemes(struct damon_ctx *ctx,
 			struct damos **schemes, ssize_t nr_schemes);
 int damon_commit_ctx(struct damon_ctx *old_ctx, struct damon_ctx *new_ctx);
-int damon_nr_running_ctxs(void);
 bool damon_is_registered_ops(enum damon_ops_id id);
 int damon_register_ops(struct damon_operations *ops);
 int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id);
diff --git a/include/linux/dax.h b/include/linux/dax.h
index fe6c3ded1b50f7..f2d47975d905fe 100644
--- a/include/linux/dax.h
+++ b/include/linux/dax.h
@@ -155,8 +155,6 @@ int dax_writeback_mapping_range(struct address_space *mapping,
 		struct dax_device *dax_dev, struct writeback_control *wbc);
 int dax_folio_reset_order(struct folio *folio);
 
-struct page *dax_layout_busy_page(struct address_space *mapping);
-struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
 dax_entry_t dax_lock_folio(struct folio *folio);
 void dax_unlock_folio(struct folio *folio, dax_entry_t cookie);
 dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
@@ -173,16 +171,6 @@ static inline int fs_dax_get(struct dax_device *dax_dev, void *holder,
 {
 	return -EOPNOTSUPP;
 }
-static inline struct page *dax_layout_busy_page(struct address_space *mapping)
-{
-	return NULL;
-}
-
-static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages)
-{
-	return NULL;
-}
-
 static inline int dax_writeback_mapping_range(struct address_space *mapping,
 		struct dax_device *dax_dev, struct writeback_control *wbc)
 {
diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index 4b1ff99608e01c..adf239f8205f5c 100644
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -116,6 +116,8 @@ struct dentry {
 					 * possible!
 					 */
 
+	/* lockdep tracking of DCACHE_PAR_LOOKUP locks */
+	struct lockdep_map		lookup_map;
 	struct list_head d_lru;		/* LRU list */
 	struct hlist_node d_sib;	/* child of parent list */
 	struct hlist_head d_children;	/* our children */
@@ -236,7 +238,9 @@ enum dentry_flags {
 	DCACHE_PAR_LOOKUP		= BIT(24),	/* being looked up (with parent locked shared) */
 	DCACHE_DENTRY_CURSOR		= BIT(25),
 	DCACHE_NORCU			= BIT(26),	/* No RCU delay for freeing */
-	DCACHE_PERSISTENT		= BIT(27)
+	DCACHE_PERSISTENT		= BIT(27),
+/* 28, 29, 30 free */
+	DCACHE_PRIVATE			= BIT(31)	/* fs-specific flag */
 };
 
 #define DCACHE_MANAGED_DENTRY \
@@ -257,7 +261,9 @@ extern void d_delete(struct dentry *);
 extern struct dentry * d_alloc(struct dentry *, const struct qstr *);
 extern struct dentry * d_alloc_anon(struct super_block *);
 extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *);
+extern struct dentry * d_alloc_trylock(struct dentry *, struct qstr *);
 extern struct dentry * d_splice_alias(struct inode *, struct dentry *);
+struct dentry *d_duplicate(struct dentry *dentry);
 /* weird procfs mess; *NOT* exported */
 extern struct dentry * d_splice_alias_ops(struct inode *, struct dentry *,
 					  const struct dentry_operations *);
@@ -552,6 +558,36 @@ static inline int simple_positive(const struct dentry *dentry)
 
 unsigned long vfs_pressure_ratio(unsigned long val);
 
+/**
+ * d_lookup_release - release ownership of DCACHE_PAR_LOOKUP lock
+ * @dentry: dentry that is locked
+ *
+ * If an in-lookup dentry is to be passed to another thread which
+ * will drop the in-lookup lock, then d_lookup_release() must be called
+ * to tell lockdep that this thread no lock holds the lock.  The
+ * thread that receives the lock must call d_lookup_acquire() to
+ * acquire the lock.
+ */
+static inline void d_lookup_release(struct dentry *dentry)
+{
+	if (d_in_lookup(dentry))
+		lock_map_release(&dentry->lookup_map);
+}
+
+/**
+ * d_lookup_acquire - acquire ownership of DCACHE_PAR_LOOKUP lock
+ * @dentry: dentry that is locked
+ *
+ * If an in-lookup dentry was passed to this thread, the
+ * d_lookup_acquire() must be called to tell lockdep that this
+ * thread now owns the DCACHE_PAR_LOOKUP lock.
+ */
+static inline void d_lookup_acquire(struct dentry *dentry)
+{
+	if (d_in_lookup(dentry))
+		lock_map_acquire_try(&dentry->lookup_map);
+}
+
 /**
  * d_inode - Get the actual inode of this dentry
  * @dentry: The dentry to query
diff --git a/include/linux/device-id/apr.h b/include/linux/device-id/apr.h
deleted file mode 100644
index f282608ea018fc..00000000000000
--- a/include/linux/device-id/apr.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef LINUX_DEVICE_ID_APR_H
-#define LINUX_DEVICE_ID_APR_H
-
-#ifdef __KERNEL__
-#include <linux/types.h>
-typedef unsigned long kernel_ulong_t;
-#endif
-
-#define APR_NAME_SIZE	32
-#define APR_MODULE_PREFIX "apr:"
-
-struct apr_device_id {
-	char name[APR_NAME_SIZE];
-	__u32 domain_id;
-	__u32 svc_id;
-	__u32 svc_version;
-	kernel_ulong_t driver_data;	/* Data private to the driver */
-};
-
-#endif /* ifndef LINUX_DEVICE_ID_APR_H */
diff --git a/include/linux/device-id/arm_smccc.h b/include/linux/device-id/arm_smccc.h
new file mode 100644
index 00000000000000..a8579832fc1cbe
--- /dev/null
+++ b/include/linux/device-id/arm_smccc.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_DEVICE_ID_ARM_SMCCC_H
+#define __LINUX_DEVICE_ID_ARM_SMCCC_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#endif
+
+#define ARM_SMCCC_MODULE_PREFIX "arm_smccc:"
+
+/**
+ * struct arm_smccc_device_id - Arm SMCCC bus device identifier
+ * @func_id: SMCCC function identifier
+ */
+struct arm_smccc_device_id {
+	__u32 func_id;
+};
+
+#endif /* __LINUX_DEVICE_ID_ARM_SMCCC_H */
diff --git a/include/linux/device-id/scmi.h b/include/linux/device-id/scmi.h
new file mode 100644
index 00000000000000..1b4ccfa9dcc538
--- /dev/null
+++ b/include/linux/device-id/scmi.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef LINUX_DEVICE_ID_SCMI_H
+#define LINUX_DEVICE_ID_SCMI_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#endif
+
+#define SCMI_NAME_SIZE		32
+#define SCMI_MODULE_PREFIX	"scmi:"
+
+struct scmi_device_id {
+	__u8 protocol_id;
+	char name[SCMI_NAME_SIZE];
+};
+
+#endif /* ifndef LINUX_DEVICE_ID_SCMI_H */
diff --git a/include/linux/device.h b/include/linux/device.h
index aee79fd6b32b47..879eb758b5eec3 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -129,19 +129,19 @@ struct dev_ext_attribute {
 	void *var;
 };
 
-ssize_t device_show_ulong(struct device *dev, struct device_attribute *attr,
+ssize_t device_show_ulong(struct device *dev, const struct device_attribute *attr,
 			  char *buf);
-ssize_t device_store_ulong(struct device *dev, struct device_attribute *attr,
+ssize_t device_store_ulong(struct device *dev, const struct device_attribute *attr,
 			   const char *buf, size_t count);
-ssize_t device_show_int(struct device *dev, struct device_attribute *attr,
+ssize_t device_show_int(struct device *dev, const struct device_attribute *attr,
 			char *buf);
-ssize_t device_store_int(struct device *dev, struct device_attribute *attr,
+ssize_t device_store_int(struct device *dev, const struct device_attribute *attr,
 			 const char *buf, size_t count);
-ssize_t device_show_bool(struct device *dev, struct device_attribute *attr,
+ssize_t device_show_bool(struct device *dev, const struct device_attribute *attr,
 			char *buf);
-ssize_t device_store_bool(struct device *dev, struct device_attribute *attr,
+ssize_t device_store_bool(struct device *dev, const struct device_attribute *attr,
 			 const char *buf, size_t count);
-ssize_t device_show_string(struct device *dev, struct device_attribute *attr,
+ssize_t device_show_string(struct device *dev, const struct device_attribute *attr,
 			   char *buf);
 
 typedef ssize_t __device_show_handler_const(struct device *dev, const struct device_attribute *attr,
@@ -1121,6 +1121,7 @@ static inline void device_unlock(struct device *dev)
 
 DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T))
 DEFINE_GUARD_COND(device, _intr, device_lock_interruptible(_T), _RET == 0)
+DEFINE_GUARD_COND(device, _try, device_trylock(_T))
 
 static inline void device_lock_assert(struct device *dev)
 {
diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h
index a38f7229b8f4d0..89e156b9acb587 100644
--- a/include/linux/device/bus.h
+++ b/include/linux/device/bus.h
@@ -113,6 +113,7 @@ struct bus_type {
 	bool need_parent_lock;
 };
 
+int __must_check companion_bus_register(const struct bus_type *bus);
 int __must_check bus_register(const struct bus_type *bus);
 
 void bus_unregister(const struct bus_type *bus);
diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h
index 768a1334c0a1dd..29fbc01ef06fc3 100644
--- a/include/linux/device/driver.h
+++ b/include/linux/device/driver.h
@@ -297,4 +297,35 @@ static int __init __driver##_init(void) \
 } \
 device_initcall(__driver##_init);
 
+/**
+ * subsys_driver() - Helper macro for drivers that don't do anything special
+ * in init/exit but have to register earlier, at subsys_initcall level, when
+ * built in. This eliminates a lot of boilerplate. Each driver may only use
+ * this macro once, and calling it replaces the init/exit boilerplate.
+ *
+ * @__driver: driver name
+ * @__register: register function for this driver type
+ * @__unregister: unregister function for this driver type
+ * @...: Additional arguments to be passed to __register and __unregister.
+ *
+ * This is meant to be a direct parallel of module_driver() above, but with
+ * the init call promoted to subsys_initcall() for built-in drivers so they
+ * are available earlier during boot. When built as a module subsys_initcall()
+ * collapses to module_init() and the normal module init/exit paths are used.
+ *
+ * Use this macro to construct bus specific macros for registering drivers,
+ * and do not use it on its own.
+ */
+#define subsys_driver(__driver, __register, __unregister, ...) \
+static int __init __driver##_init(void) \
+{ \
+	return __register(&(__driver), ##__VA_ARGS__); \
+} \
+subsys_initcall(__driver##_init) \
+static void __exit __driver##_exit(void) \
+{ \
+	__unregister(&(__driver), ##__VA_ARGS__); \
+} \
+module_exit(__driver##_exit)
+
 #endif	/* _DEVICE_DRIVER_H_ */
diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
index ffa99b930843e2..dd07d128adc3b0 100644
--- a/include/linux/dma-fence.h
+++ b/include/linux/dma-fence.h
@@ -507,7 +507,7 @@ dma_fence_test_signaled_flag(struct dma_fence *fence)
  * Returns true if the fence was already signaled, false if not. Since this
  * function doesn't enable signaling, it is not guaranteed to ever return
  * true if dma_fence_add_callback(), dma_fence_wait() or
- * dma_fence_enable_sw_signaling() haven't been called before.
+ * dma_fence_enable_signaling() haven't been called before.
  *
  * This function requires &dma_fence.lock to be held.
  *
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 8fae2b7deb20de..afaf2c12189b8a 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -60,7 +60,7 @@ struct dma_map_ops {
 	int (*dma_supported)(struct device *dev, u64 mask);
 	u64 (*get_required_mask)(struct device *dev);
 	size_t (*max_mapping_size)(struct device *dev);
-	size_t (*opt_mapping_size)(void);
+	size_t (*max_opt_mapping_size)(void);
 	unsigned long (*get_merge_boundary)(struct device *dev);
 };
 
@@ -143,7 +143,7 @@ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size,
 static inline void dma_free_contiguous(struct device *dev, struct page *page,
 		size_t size)
 {
-	__free_pages(page, get_order(size));
+	free_pages_exact(page_address(page), size);
 }
 #endif /* CONFIG_DMA_CMA*/
 
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index a3e880649fa41a..efde0e5a58dc22 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -207,7 +207,7 @@ int dma_set_coherent_mask(struct device *dev, u64 mask);
 u64 dma_get_required_mask(struct device *dev);
 bool dma_addressing_limited(struct device *dev);
 size_t dma_max_mapping_size(struct device *dev);
-size_t dma_opt_mapping_size(struct device *dev);
+size_t dma_max_opt_mapping_size(struct device *dev);
 unsigned long dma_get_merge_boundary(struct device *dev);
 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
 		enum dma_data_direction dir, gfp_t gfp, unsigned long attrs);
@@ -326,7 +326,7 @@ static inline size_t dma_max_mapping_size(struct device *dev)
 {
 	return 0;
 }
-static inline size_t dma_opt_mapping_size(struct device *dev)
+static inline size_t dma_max_opt_mapping_size(struct device *dev)
 {
 	return 0;
 }
diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
index 3c8e2ef9dee0c0..affd896563a1e7 100644
--- a/include/linux/dma/edma.h
+++ b/include/linux/dma/edma.h
@@ -130,7 +130,7 @@ struct dw_edma_chip {
 
 	u16			ll_wr_cnt;
 	u16			ll_rd_cnt;
-	/* link list address */
+	/* Linked-list regions must not cross a 4 GiB address boundary. */
 	struct dw_edma_region	ll_region_wr[HDMA_MAX_WR_CH];
 	struct dw_edma_region	ll_region_rd[HDMA_MAX_RD_CH];
 
@@ -147,6 +147,7 @@ struct dw_edma_chip {
 
 	struct dw_edma		*dw;
 	bool			cfg_non_ll;
+	u32			ch_space_sz;
 };
 
 /* Export to the platform drivers */
diff --git a/include/linux/dma/imx-dma.h b/include/linux/dma/imx-dma.h
index 76a8de9ae15175..16e491f63883ce 100644
--- a/include/linux/dma/imx-dma.h
+++ b/include/linux/dma/imx-dma.h
@@ -57,11 +57,6 @@ struct imx_dma_data {
 	int priority;
 };
 
-static inline int imx_dma_is_ipu(struct dma_chan *chan)
-{
-	return !strcmp(dev_name(chan->device->dev), "ipu-core");
-}
-
 static inline int imx_dma_is_general_purpose(struct dma_chan *chan)
 {
 	return !strcmp(chan->device->dev->driver->name, "imx-sdma") ||
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index fe33a20abc6146..1b57bc4a7e72ac 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -325,7 +325,8 @@ struct dma_router {
  * @lock: protect between config and prepare transfer when driver have not
  *	  implemented callback device_prep_config_sg().
  * @chan_id: channel ID for sysfs
- * @dev: class device for sysfs
+ * @chan_dev: class channel device for sysfs, some device use it for per-channel
+ *            IOMMU mapping.
  * @name: backlink name for sysfs
  * @dbg_client_name: slave name for debugfs in format:
  *	dev_name(requester's dev):channel name, for example: "2b00000.mcasp:tx"
@@ -351,7 +352,14 @@ struct dma_chan {
 
 	/* sysfs */
 	int chan_id;
-	struct dma_chan_dev *dev;
+	union {
+		struct dma_chan_dev *chan_dev;
+		/*
+		 * Use chan_dev; dev will be removed once all users are
+		 * converted.
+		 */
+		struct dma_chan_dev *dev;
+	};
 	const char *name;
 #ifdef CONFIG_DEBUG_FS
 	char *dbg_client_name;
@@ -532,7 +540,7 @@ struct dma_slave_caps {
 
 static inline const char *dma_chan_name(struct dma_chan *chan)
 {
-	return dev_name(&chan->dev->device);
+	return dev_name(&chan->chan_dev->device);
 }
 
 /**
@@ -1760,15 +1768,6 @@ void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
 #define dma_request_channel(mask, x, y) \
 	__dma_request_channel(&(mask), x, y, NULL)
 
-/* Deprecated, please use dma_request_chan() directly */
-static inline struct dma_chan * __deprecated
-dma_request_slave_channel(struct device *dev, const char *name)
-{
-	struct dma_chan *ch = dma_request_chan(dev, name);
-
-	return IS_ERR(ch) ? NULL : ch;
-}
-
 static inline struct dma_chan
 *dma_request_slave_channel_compat(const dma_cap_mask_t mask,
 				  dma_filter_fn fn, void *fn_param,
@@ -1803,10 +1802,15 @@ dmaengine_get_direction_text(enum dma_transfer_direction dir)
 	}
 }
 
+static inline struct device *dmaengine_chan_dev(struct dma_chan *chan)
+{
+	return &chan->chan_dev->device;
+}
+
 static inline struct device *dmaengine_get_dma_device(struct dma_chan *chan)
 {
-	if (chan->dev->chan_dma_dev)
-		return &chan->dev->device;
+	if (chan->chan_dev->chan_dma_dev)
+		return dmaengine_chan_dev(chan);
 
 	return chan->device->dev;
 }
diff --git a/include/linux/efi.h b/include/linux/efi.h
index aa15ff88539bdf..ecb34be37a874f 100644
--- a/include/linux/efi.h
+++ b/include/linux/efi.h
@@ -957,6 +957,17 @@ extern int efi_status_to_err(efi_status_t status);
 #define   EFI_DEV_END_INSTANCE			0x01
 #define   EFI_DEV_END_ENTIRE			0xFF
 
+enum efi_hd_partition_format {
+	EFI_HD_PARTITION_FORMAT_MBR = 1,
+	EFI_HD_PARTITION_FORMAT_GPT,
+};
+
+enum efi_hd_signature_type {
+	EFI_HD_SIGNATURE_TYPE_NONE,
+	EFI_HD_SIGNATURE_TYPE_MBR,
+	EFI_HD_SIGNATURE_TYPE_GUID,
+};
+
 struct efi_generic_dev_path {
 	u8				type;
 	u8				sub_type;
@@ -988,6 +999,16 @@ struct efi_rel_offset_dev_path {
 	u64				ending_offset;
 } __packed;
 
+struct efi_hd_dev_path {
+	struct efi_generic_dev_path	header;
+	u32				partition_number;
+	u64				partition_start;
+	u64				partition_size;
+	efi_guid_t			signature;
+	u8				partition_format;
+	u8				signature_type;
+} __packed;
+
 struct efi_mem_mapped_dev_path {
 	struct efi_generic_dev_path	header;
 	u32				memory_type;
@@ -1007,6 +1028,7 @@ struct efi_dev_path {
 		struct efi_pci_dev_path		pci;
 		struct efi_vendor_dev_path	vendor;
 		struct efi_rel_offset_dev_path	rel_offset;
+		struct efi_hd_dev_path		hd;
 	};
 } __packed;
 
@@ -1109,6 +1131,7 @@ extern void efi_call_virt_check_flags(unsigned long flags, const void *caller);
 extern unsigned long efi_call_virt_save_flags(void);
 
 void efi_runtime_assert_lock_held(void);
+efi_status_t efi_runtime_set_enable_flag(bool enable);
 
 enum efi_secureboot_mode {
 	efi_secureboot_mode_unset,
diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h
index 253600c0eccdf9..c4c9ce0386111b 100644
--- a/include/linux/ethtool.h
+++ b/include/linux/ethtool.h
@@ -944,6 +944,7 @@ struct kernel_ethtool_ts_info {
 #define ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM	BIT(6)
 #define ETHTOOL_OP_NEEDS_RTNL_RSS		BIT(7)
 #define ETHTOOL_OP_NEEDS_RTNL_GLINK		BIT(8)
+#define ETHTOOL_OP_NEEDS_RTNL_TEST		BIT(9)
 
 /**
  * struct ethtool_ops - optional netdev operations
@@ -981,6 +982,7 @@ struct kernel_ethtool_ts_info {
  *	 - netdev_update_features()
  *	 - netif_set_real_num_tx_queues()
  *	 - ethtool_op_get_link() (syncs link watch under rtnl_lock)
+ *	 - netif_open() / netif_close() (used by @self_test)
  *
  * @get_drvinfo: Report driver/device information. Modern drivers no
  *	longer have to implement this callback. Most fields are
diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h
index bb2b6cd5d5070e..3081702b1ddba7 100644
--- a/include/linux/f2fs_fs.h
+++ b/include/linux/f2fs_fs.h
@@ -14,9 +14,9 @@
 #define F2FS_SUPER_OFFSET		1024	/* byte-size offset */
 #define F2FS_MIN_LOG_SECTOR_SIZE	9	/* 9 bits for 512 bytes */
 #define F2FS_MAX_LOG_SECTOR_SIZE	PAGE_SHIFT	/* Max is Block Size */
-#define F2FS_LOG_SECTORS_PER_BLOCK	(PAGE_SHIFT - 9) /* log number for sector/blk */
-#define F2FS_BLKSIZE			PAGE_SIZE /* support only block == page */
-#define F2FS_BLKSIZE_BITS		PAGE_SHIFT /* bits for F2FS_BLKSIZE */
+#define F2FS_MIN_LOG_BLOCKSIZE		12
+#define F2FS_MIN_BLKSIZE		4096UL
+#define F2FS_MAX_BLKSIZE		PAGE_SIZE
 #define F2FS_MAX_EXTENSION		64	/* # of extension entries */
 #define F2FS_EXTENSION_LEN		8	/* max size of extension */
 
@@ -24,19 +24,25 @@
 #define NEW_ADDR		((block_t)-1)	/* used as block_t addresses */
 #define COMPRESS_ADDR		((block_t)-2)	/* used as compressed data flag */
 
-#define F2FS_BLKSIZE_MASK		(F2FS_BLKSIZE - 1)
-#define F2FS_BYTES_TO_BLK(bytes)	((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS)
-#define F2FS_BLK_TO_BYTES(blk)		((unsigned long long)(blk) << F2FS_BLKSIZE_BITS)
-#define F2FS_BLK_END_BYTES(blk)		(F2FS_BLK_TO_BYTES(blk + 1) - 1)
-#define F2FS_BLK_ALIGN(x)		(F2FS_BYTES_TO_BLK((x) + F2FS_BLKSIZE - 1))
+#define F2FS_BLKSIZE(sbi)		((sbi)->blocksize)
+#define F2FS_BLKSIZE_BITS(sbi)		((sbi)->log_blocksize)
+#define F2FS_BLKSIZE_MASK(sbi)		(F2FS_BLKSIZE(sbi) - 1)
+#define F2FS_LOG_SECTORS_PER_BLOCK(sbi)	(F2FS_BLKSIZE_BITS(sbi) - 9)
+#define F2FS_BLKS_PER_PAGE(sbi)		(PAGE_SIZE / F2FS_BLKSIZE(sbi))
+#define F2FS_BYTES_TO_BLK(sbi, bytes)					\
+	((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS(sbi))
+#define F2FS_BLK_TO_BYTES(sbi, blk)					\
+	((unsigned long long)(blk) << F2FS_BLKSIZE_BITS(sbi))
+#define F2FS_BLK_END_BYTES(sbi, blk)					\
+	(F2FS_BLK_TO_BYTES(sbi, (blk) + 1) - 1)
+#define F2FS_BLK_ALIGN(sbi, bytes)					\
+	F2FS_BYTES_TO_BLK(sbi, (unsigned long long)(bytes) +		\
+				 F2FS_BLKSIZE(sbi) - 1)
 
 /* 0, 1(node nid), 2(meta nid) are reserved node id */
 #define F2FS_RESERVED_NODE_NUM		3
 
 #define F2FS_ROOT_INO(sbi)	((sbi)->root_ino_num)
-#define F2FS_NODE_INO(sbi)	((sbi)->node_ino_num)
-#define F2FS_META_INO(sbi)	((sbi)->meta_ino_num)
-#define F2FS_COMPRESS_INO(sbi)	(NM_I(sbi)->max_nid)
 
 #define F2FS_MAX_QUOTAS		3
 
@@ -214,20 +220,27 @@ struct f2fs_checkpoint {
 	unsigned char sit_nat_version_bitmap[];
 } __packed;
 
-#define CP_CHKSUM_OFFSET	(F2FS_BLKSIZE - sizeof(__le32))	/* default chksum offset in checkpoint */
 #define CP_MIN_CHKSUM_OFFSET						\
 	(offsetof(struct f2fs_checkpoint, sit_nat_version_bitmap))
 
 /*
  * For orphan inode management
+ *
+ * The number of inode entries in an orphan block depends on the filesystem
+ * block size. Its exact on-disk layout is:
+ *
+ * 0                           blocksize - 16             blocksize
+ * +--------------------------+--------------------------+
+ * | ino[0] ... ino[n - 1]    | struct f2fs_orphan_footer |
+ * +--------------------------+--------------------------+
+ *
+ * n = (blocksize - sizeof(struct f2fs_orphan_footer)) / sizeof(__le32)
  */
-#define F2FS_ORPHANS_PER_BLOCK	((F2FS_BLKSIZE - 4 * sizeof(__le32)) / sizeof(__le32))
-
-#define GET_ORPHAN_BLOCKS(n)	(((n) + F2FS_ORPHANS_PER_BLOCK - 1) / \
-					F2FS_ORPHANS_PER_BLOCK)
-
 struct f2fs_orphan_block {
-	__le32 ino[F2FS_ORPHANS_PER_BLOCK];	/* inode numbers */
+	DECLARE_FLEX_ARRAY(__le32, ino);
+} __packed;
+
+struct f2fs_orphan_footer {
 	__le32 reserved;	/* reserved */
 	__le16 blk_addr;	/* block index in current CP */
 	__le16 blk_count;	/* Number of orphan inode blocks in CP */
@@ -260,26 +273,14 @@ struct node_footer {
 } __packed;
 
 /* Address Pointers in an Inode */
-#define DEF_ADDRS_PER_INODE	((F2FS_BLKSIZE - OFFSET_OF_END_OF_I_EXT	\
-					- SIZE_OF_I_NID	\
-					- sizeof(struct node_footer)) / sizeof(__le32))
-#define CUR_ADDRS_PER_INODE(inode)	(DEF_ADDRS_PER_INODE - \
-					get_extra_isize(inode))
+#define F2FS_DEF_ADDRS_PER_INODE(blocksize)				\
+	(((blocksize) - OFFSET_OF_END_OF_I_EXT - SIZE_OF_I_NID -	\
+	  sizeof(struct node_footer)) / sizeof(__le32))
 #define DEF_NIDS_PER_INODE	5	/* Node IDs in an Inode */
 #define ADDRS_PER_INODE(inode)	addrs_per_page(inode, true)
 /* Address Pointers in a Direct Block */
-#define DEF_ADDRS_PER_BLOCK	((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32))
 #define ADDRS_PER_BLOCK(inode)	addrs_per_page(inode, false)
-/* Node IDs in an Indirect Block */
-#define NIDS_PER_BLOCK		((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32))
-
-#define ADDRS_PER_PAGE(folio, inode)	(addrs_per_page(inode, IS_INODE(folio)))
-
-#define	NODE_DIR1_BLOCK		(DEF_ADDRS_PER_INODE + 1)
-#define	NODE_DIR2_BLOCK		(DEF_ADDRS_PER_INODE + 2)
-#define	NODE_IND1_BLOCK		(DEF_ADDRS_PER_INODE + 3)
-#define	NODE_IND2_BLOCK		(DEF_ADDRS_PER_INODE + 4)
-#define	NODE_DIND_BLOCK		(DEF_ADDRS_PER_INODE + 5)
+#define ADDRS_PER_PAGE(folio, inode)	(addrs_per_page(inode, IS_INODE(F2FS_I_SB(inode), folio)))
 
 #define F2FS_INLINE_XATTR	0x01	/* file inline xattr flag */
 #define F2FS_INLINE_DATA	0x02	/* file inline data flag */
@@ -339,18 +340,26 @@ struct f2fs_inode {
 						 */
 			__le32 i_extra_end[0];	/* for attribute size calculation */
 		} __packed;
-		__le32 i_addr[DEF_ADDRS_PER_INODE];	/* Pointers to data blocks */
+		DECLARE_FLEX_ARRAY(__le32, i_addr); /* data block pointers */
 	};
-	__le32 i_nid[DEF_NIDS_PER_INODE];	/* direct(2), indirect(2),
-						double_indirect(1) node id */
+	/*
+	 * __le32 i_nid[DEF_NIDS_PER_INODE];
+	 * direct(2), indirect(2), double_indirect(1) node IDs
+	 *
+	 * It is stored immediately before the node footer at the end of the
+	 * filesystem block. Its offset depends on the filesystem block size, so
+	 * locate it dynamically with F2FS_INODE_NIDS().
+	 */
 } __packed;
 
 struct direct_node {
-	__le32 addr[DEF_ADDRS_PER_BLOCK];	/* array of data block address */
+	/* The address count depends on the filesystem block size. */
+	DECLARE_FLEX_ARRAY(__le32, addr); /* array of data block address */
 } __packed;
 
 struct indirect_node {
-	__le32 nid[NIDS_PER_BLOCK];	/* array of data block address */
+	/* The node ID count depends on the filesystem block size. */
+	DECLARE_FLEX_ARRAY(__le32, nid); /* array of data block address */
 } __packed;
 
 enum {
@@ -369,14 +378,18 @@ struct f2fs_node {
 		struct direct_node dn;
 		struct indirect_node in;
 	};
-	struct node_footer footer;
+	/*
+	 * struct node_footer footer;
+	 *
+	 * It is stored at the end of the filesystem block, after the inode or
+	 * direct/indirect node data. Its offset depends on the filesystem block
+	 * size, so locate it dynamically with F2FS_NODE_FOOTER().
+	 */
 } __packed;
 
 /*
  * For NAT entries
  */
-#define NAT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_nat_entry))
-
 struct f2fs_nat_entry {
 	__u8 version;		/* latest version of cached nat entry */
 	__le32 ino;		/* inode number */
@@ -384,7 +397,8 @@ struct f2fs_nat_entry {
 } __packed;
 
 struct f2fs_nat_block {
-	struct f2fs_nat_entry entries[NAT_ENTRY_PER_BLOCK];
+	/* The entry count depends on the filesystem block size. */
+	DECLARE_FLEX_ARRAY(struct f2fs_nat_entry, entries);
 } __packed;
 
 /*
@@ -396,8 +410,6 @@ struct f2fs_nat_block {
  * Not allow to change this.
  */
 #define SIT_VBLOCK_MAP_SIZE 64
-#define SIT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_sit_entry))
-
 /*
  * F2FS uses 4 bytes to represent block address. As a result, supported size of
  * disk is 16 TB for a 4K page size and 64 TB for a 16K page size and it equals
@@ -424,8 +436,13 @@ struct f2fs_sit_entry {
 	__le64 mtime;				/* segment age for cleaning */
 } __packed;
 
+/*
+ * The on-disk SIT block is a filesystem-block-sized array of SIT entries.
+ * Its entry count depends on the filesystem block size, so it must be
+ * calculated by the caller rather than implied by this C structure.
+ */
 struct f2fs_sit_block {
-	struct f2fs_sit_entry entries[SIT_ENTRY_PER_BLOCK];
+	DECLARE_FLEX_ARRAY(struct f2fs_sit_entry, entries);
 } __packed;
 
 /*
@@ -595,15 +612,7 @@ typedef __le32	f2fs_hash_t;
  * dentry, when converting inline dentry we should handle this carefully.
  */
 
-/* the number of dentry in a block */
-#define NR_DENTRY_IN_BLOCK	((BITS_PER_BYTE * F2FS_BLKSIZE) / \
-					((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1))
 #define SIZE_OF_DIR_ENTRY	11	/* by byte */
-#define SIZE_OF_DENTRY_BITMAP	((NR_DENTRY_IN_BLOCK + BITS_PER_BYTE - 1) / \
-					BITS_PER_BYTE)
-#define SIZE_OF_RESERVED	(F2FS_BLKSIZE - ((SIZE_OF_DIR_ENTRY + \
-				F2FS_SLOT_LEN) * \
-				NR_DENTRY_IN_BLOCK + SIZE_OF_DENTRY_BITMAP))
 #define MIN_INLINE_DENTRY_SIZE		40	/* just include '.' and '..' entries */
 
 /* One directory entry slot representing F2FS_SLOT_LEN-sized file name */
@@ -614,14 +623,21 @@ struct f2fs_dir_entry {
 	__u8 file_type;		/* file type */
 } __packed;
 
-/* Block-sized directory entry block */
-struct f2fs_dentry_block {
-	/* validity bitmap for directory entries in each block */
-	__u8 dentry_bitmap[SIZE_OF_DENTRY_BITMAP];
-	__u8 reserved[SIZE_OF_RESERVED];
-	struct f2fs_dir_entry dentry[NR_DENTRY_IN_BLOCK];
-	__u8 filename[NR_DENTRY_IN_BLOCK][F2FS_SLOT_LEN];
-} __packed;
+/*
+ * A dentry block is laid out as follows, where the number of entries and all
+ * offsets are determined by the filesystem block size at runtime:
+ *
+ * 0                                                         blocksize
+ * +--------+----------+-------------------+-----------------------+
+ * | bitmap | reserved | dir_entry[entries]| filename[entries][8] |
+ * +--------+----------+-------------------+-----------------------+
+ *
+ * entries = (BITS_PER_BYTE * blocksize) /
+ *           ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1)
+ * bitmap_size = DIV_ROUND_UP(entries, BITS_PER_BYTE)
+ * reserved_size = blocksize - bitmap_size -
+ *                 (SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * entries
+ */
 
 #define	F2FS_DEF_PROJID		0	/* default project ID */
 
diff --git a/include/linux/fault-inject.h b/include/linux/fault-inject.h
index 58fd14c8227080..5c74748a53f38f 100644
--- a/include/linux/fault-inject.h
+++ b/include/linux/fault-inject.h
@@ -18,6 +18,13 @@ enum fault_flags {
 #include <linux/configfs.h>
 #include <linux/ratelimit.h>
 
+/*
+ * Length of the debugfs directory name embedded in struct fault_attr.
+ * Chosen to accommodate every in-tree caller of fault_create_debugfs_attr()
+ * (the longest is "fail_dma_array_full", 19 chars) with generous headroom.
+ */
+#define FAULT_ATTR_DNAME_LEN	64
+
 /*
  * For explanation of the elements of this struct, see
  * Documentation/fault-injection/fault-injection.rst
@@ -37,7 +44,7 @@ struct fault_attr {
 
 	unsigned long count;
 	struct ratelimit_state ratelimit_state;
-	struct dentry *dname;
+	char dname[FAULT_ATTR_DNAME_LEN];
 };
 
 #define FAULT_ATTR_INITIALIZER {					\
@@ -47,7 +54,6 @@ struct fault_attr {
 		.stacktrace_depth = 32,					\
 		.ratelimit_state = RATELIMIT_STATE_INIT_DISABLED,	\
 		.verbose = 2,						\
-		.dname = NULL,						\
 	}
 
 #define DECLARE_FAULT_ATTR(name) struct fault_attr name = FAULT_ATTR_INITIALIZER
diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h
index c45306a9f00723..a4678105872959 100644
--- a/include/linux/fdtable.h
+++ b/include/linux/fdtable.h
@@ -25,7 +25,7 @@
 
 struct fdtable {
 	unsigned int max_fds;
-	struct file __rcu **fd;      /* current fd array */
+	struct file __rcu **fd __counted_by_ptr(max_fds);      /* current fd array */
 	unsigned long *close_on_exec;
 	unsigned long *open_fds;
 	unsigned long *full_fds_bits;
@@ -101,11 +101,22 @@ struct task_struct;
 
 void put_files_struct(struct files_struct *fs);
 int unshare_files(void);
+void switch_files_struct(struct task_struct *tsk, struct files_struct *files);
+int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp);
+enum fd_range_flags {
+	/* Leave behind all descriptors outside of the specified range. */
+	FD_RANGE_EXCEPT		= (1U << 0),
+
+	/* Only select descriptors that have close-on-exec set. */
+	FD_RANGE_CLOEXEC_ONLY	= (1U << 1),
+};
+
 struct fd_range {
 	unsigned int from, to;
+	enum fd_range_flags flags;
 };
 struct files_struct *dup_fd(struct files_struct *, struct fd_range *) __latent_entropy;
-void do_close_on_exec(struct files_struct *);
+void close_cloexec_files(struct files_struct *);
 int iterate_fd(struct files_struct *, unsigned,
 		int (*)(const void *, struct file *, unsigned),
 		const void *);
diff --git a/include/linux/file.h b/include/linux/file.h
index 27484b444d3155..41c3c0be106441 100644
--- a/include/linux/file.h
+++ b/include/linux/file.h
@@ -12,6 +12,7 @@
 #include <linux/errno.h>
 #include <linux/cleanup.h>
 #include <linux/err.h>
+#include <linux/vfsdebug.h>
 
 struct file;
 
@@ -129,117 +130,84 @@ extern unsigned int sysctl_nr_open_min, sysctl_nr_open_max;
 
 /*
  * fd_prepare: Combined fd + file allocation cleanup class.
- * @err: Error code to indicate if allocation succeeded.
- * @__fd: Allocated fd (may not be accessed directly)
- * @__file: Allocated struct file pointer (may not be accessed directly)
+ * @fd: Allocated fd
+ * @file: Allocated struct file pointer
  *
  * Allocates an fd and a file together. On error paths, automatically cleans
  * up whichever resource was successfully allocated. Allows flexible file
  * allocation with different functions per usage.
  *
- * Do not use directly.
+ * Do not declare directly, use FD_PREPARE().
  */
 struct fd_prepare {
-	s32 err;
-	s32 __fd; /* do not access directly */
-	struct file *__file; /* do not access directly */
+	int fd;
+	struct file *file;
 };
 
-/* Typedef for fd_prepare cleanup guards. */
-typedef struct fd_prepare class_fd_prepare_t;
-
-/*
- * Accessors for fd_prepare class members.
- * _Generic() is used for zero-cost type safety.
- */
-#define fd_prepare_fd(_fdf) \
-	(_Generic((_fdf), struct fd_prepare: (_fdf).__fd))
-
-#define fd_prepare_file(_fdf) \
-	(_Generic((_fdf), struct fd_prepare: (_fdf).__file))
-
 /* Do not use directly. */
-static inline void class_fd_prepare_destructor(const struct fd_prepare *fdf)
+static __always_inline void __fd_prepare_cleanup(const struct fd_prepare *fdf)
 {
-	if (unlikely(fdf->__fd >= 0))
-		put_unused_fd(fdf->__fd);
-	if (unlikely(!IS_ERR_OR_NULL(fdf->__file)))
-		fput(fdf->__file);
+	if (unlikely(fdf->fd >= 0)) {
+		put_unused_fd(fdf->fd);
+		fput(fdf->file);
+	}
 }
 
 /* Do not use directly. */
-static inline int class_fd_prepare_lock_err(const struct fd_prepare *fdf)
+static __always_inline struct fd_prepare __fd_prepare(int fd, struct file *file)
 {
-	if (unlikely(fdf->err))
-		return fdf->err;
-	if (unlikely(fdf->__fd < 0))
-		return fdf->__fd;
-	if (unlikely(IS_ERR(fdf->__file)))
-		return PTR_ERR(fdf->__file);
-	if (unlikely(!fdf->__file))
-		return -ENOMEM;
-	return 0;
+	if (fd >= 0 && IS_ERR_OR_NULL(file)) {
+		int err = file ? PTR_ERR(file) : -ENOMEM;
+
+		put_unused_fd(fd);
+		fd = err;
+		file = NULL;
+	}
+	return (struct fd_prepare){ .fd = fd, .file = file };
 }
 
 /*
- * __FD_PREPARE_INIT - Helper to initialize fd_prepare class.
- * @_fd_flags: flags for get_unused_fd_flags()
- * @_file_owned: expression that returns struct file *
- *
- * Returns a struct fd_prepare with fd, file, and err set.
- * If fd allocation fails, fd will be negative and err will be set. If
- * fd succeeds but file_init_expr fails, file will be ERR_PTR and err
- * will be set. The err field is the single source of truth for error
- * checking.
- */
-#define __FD_PREPARE_INIT(_fd_flags, _file_owned)                 \
-	({                                                        \
-		struct fd_prepare fdf = {                         \
-			.__fd = get_unused_fd_flags((_fd_flags)), \
-		};                                                \
-		if (likely(fdf.__fd >= 0))                        \
-			fdf.__file = (_file_owned);               \
-		fdf.err = ACQUIRE_ERR(fd_prepare, &fdf);          \
-		fdf;                                              \
-	})
-
-/*
- * FD_PREPARE - Macro to declare and initialize an fd_prepare variable.
+ * FD_PREPARE - Declare and initialize an fd_prepare instance.
  *
- * Declares and initializes an fd_prepare variable with automatic
- * cleanup. No separate scope required - cleanup happens when variable
- * goes out of scope.
+ * This allocates a new fd and only evaluates @_file_owned if the
+ * allocation succeeded. Cleanup happens when the variable goes out of
+ * scope and the guard releases whichever of the descriptor and the file
+ * was allocated. If fd_publish() was called the fd and file are
+ * published and cleanup becomes a nop.
  *
- * @_fdf: name of struct fd_prepare variable to define
+ * @_fdf: name of the const struct fd_prepare pointer to define
  * @_fd_flags: flags for get_unused_fd_flags()
  * @_file_owned: struct file to take ownership of (can be expression)
  */
+#define __FD_PREPARE(_guard, _fdf, _fd_flags, _file_owned)		\
+	struct fd_prepare _guard __cleanup(__fd_prepare_cleanup) = ({	\
+		int __fd = get_unused_fd_flags(_fd_flags);		\
+		__fd_prepare(__fd, __fd < 0 ? NULL : (_file_owned));	\
+	});								\
+	const struct fd_prepare *const _fdf = &_guard
+
 #define FD_PREPARE(_fdf, _fd_flags, _file_owned) \
-	CLASS_INIT(fd_prepare, _fdf, __FD_PREPARE_INIT(_fd_flags, _file_owned))
+	__FD_PREPARE(__UNIQUE_ID(fd_prepare), _fdf, _fd_flags, _file_owned)
 
 /*
  * fd_publish - Publish prepared fd and file to the fd table.
- * @_fdf: struct fd_prepare variable
+ * @fdf: struct fd_prepare pointer defined by FD_PREPARE()
  */
-#define fd_publish(_fdf)                                       \
-	({                                                     \
-		struct fd_prepare *fdp = &(_fdf);              \
-		VFS_WARN_ON_ONCE(fdp->err);                    \
-		VFS_WARN_ON_ONCE(fdp->__fd < 0);               \
-		VFS_WARN_ON_ONCE(IS_ERR_OR_NULL(fdp->__file)); \
-		fd_install(fdp->__fd, fdp->__file);            \
-		retain_and_null_ptr(fdp->__file);              \
-		take_fd(fdp->__fd);                            \
-	})
+static __always_inline int fd_publish(const struct fd_prepare *fdf)
+{
+	/* Callers only get a const view, the guard itself is writable. */
+	struct fd_prepare *guard = (struct fd_prepare *)fdf;
+
+	VFS_WARN_ON_ONCE(guard->fd < 0);
+	fd_install(guard->fd, guard->file);
+	return take_fd(guard->fd);
+}
 
 /* Do not use directly. */
-#define __FD_ADD(_fdf, _fd_flags, _file_owned)            \
-	({                                                \
-		FD_PREPARE(_fdf, _fd_flags, _file_owned); \
-		s32 ret = _fdf.err;                       \
-		if (likely(!ret))                         \
-			ret = fd_publish(_fdf);           \
-		ret;                                      \
+#define __FD_ADD(_fdf, _fd_flags, _file_owned)			\
+	({							\
+		FD_PREPARE(_fdf, _fd_flags, _file_owned);	\
+		_fdf->fd < 0 ? _fdf->fd : fd_publish(_fdf);	\
 	})
 
 /*
diff --git a/include/linux/fileattr.h b/include/linux/fileattr.h
index 58044b59801620..09e32b84e02acf 100644
--- a/include/linux/fileattr.h
+++ b/include/linux/fileattr.h
@@ -74,7 +74,7 @@ static inline bool fileattr_has_fsx(const struct file_kattr *fa)
 }
 
 int vfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
-int vfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry,
+int vfs_fileattr_set(const struct mnt_idmap *idmap, struct dentry *dentry,
 		     struct file_kattr *fa);
 int ioctl_getflags(struct file *file, unsigned int __user *argp);
 int ioctl_setflags(struct file *file, unsigned int __user *argp);
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 39decde7fc7305..e42eccb0990e65 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -98,6 +98,11 @@ struct ctl_table_header;
 
 /* BPF program can access up to 512 bytes of stack space. */
 #define MAX_BPF_STACK	512
+/*
+ * Stack budget of a program on a JIT that lays out frames of that size.
+ * The interpreter and JITs without such support keep MAX_BPF_STACK.
+ */
+#define MAX_BPF_STACK_JIT	2048
 
 /* Helper macros for filter block array initializers. */
 
@@ -1237,6 +1242,8 @@ bool bpf_jit_inlines_helper_call(s32 imm);
 bool bpf_jit_supports_subprog_tailcalls(void);
 bool bpf_jit_supports_percpu_insn(void);
 bool bpf_jit_supports_kfunc_call(void);
+bool bpf_jit_supports_callx(void);
+bool bpf_jit_supports_kfunc_ret_reg_pair(void);
 bool bpf_jit_supports_stack_args(void);
 bool bpf_jit_supports_arena_args(void);
 bool bpf_jit_supports_far_kfunc_call(void);
@@ -1245,8 +1252,41 @@ bool bpf_jit_supports_ptr_xchg(void);
 bool bpf_jit_supports_arena(void);
 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
 bool bpf_jit_supports_private_stack(void);
+bool bpf_jit_supports_large_stack(void);
 bool bpf_jit_supports_timed_may_goto(void);
 bool bpf_jit_supports_fsession(void);
+
+struct bpf_jit_arg_abi {
+	/* Argument registers of the kernel convention. */
+	u8 nr_arg_regs;
+	/* Round the register number up to an even one for 16-byte alignment. */
+	bool even_reg_align;
+	/* Round the stack slot up to an even one for 16-byte alignment. */
+	bool even_stack_align;
+	/* An argument may straddle the last register and the stack. */
+	bool split_at_boundary;
+	/* A later argument may reuse a register a stack-passed one skipped. */
+	bool backfill_after_stack;
+};
+
+const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void);
+u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi,
+		       const struct btf_func_model *fm, u8 *pos_of_slot);
+
+/* The JIT's scratch register, in place of an argument slot. */
+#define BPF_JIT_ARG_TMP		0xff
+
+/* Every argument slot moves at most once, and the scratch goes out and back. */
+#define BPF_JIT_MAX_ARG_MOVES	(MAX_BPF_FUNC_ARG_SLOTS + 2)
+
+struct bpf_jit_arg_move {
+	u8 dst;
+	u8 src;
+};
+
+u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi,
+			   const struct btf_func_model *fm,
+			   struct bpf_jit_arg_move *moves);
 u64 bpf_arch_uaddress_limit(void);
 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
 u64 arch_bpf_timed_may_goto(void);
@@ -1897,6 +1937,11 @@ static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 inde
 	return XDP_REDIRECT;
 }
 
+int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock,
+				void *search_res, u32 len, u64 flags);
+int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock,
+				 const void *from, u32 len, u64 flags);
+
 #ifdef CONFIG_NET
 int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len);
 int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
diff --git a/include/linux/firewire.h b/include/linux/firewire.h
index fd35a6570cd889..dbb5c825a1ccfb 100644
--- a/include/linux/firewire.h
+++ b/include/linux/firewire.h
@@ -172,10 +172,10 @@ struct fw_attribute_group {
 };
 
 enum fw_device_quirk {
-	// See afa1282a35d3 ("firewire: core: check for 1394a compliant IRM, fix inaccessibility of Sony camcorder").
+	// See 10389536742c ("firewire: core: check for 1394a compliant IRM, fix inaccessibility of Sony camcorder").
 	FW_DEVICE_QUIRK_IRM_IS_1394_1995_ONLY = BIT(0),
 
-	// See a509e43ff338 ("firewire: core: fix unstable I/O with Canon camcorder").
+	// See 6044565af458 ("firewire: core: fix unstable I/O with Canon camcorder").
 	FW_DEVICE_QUIRK_IRM_IGNORES_BUS_MANAGER = BIT(1),
 
 	// MOTU Audio Express transfers acknowledge packet with 0x10 for pending state.
@@ -224,7 +224,7 @@ struct fw_device {
 	struct mutex client_list_mutex;
 	struct list_head client_list;
 
-	const u32 *config_rom;
+	const u32 *config_rom __counted_by_ptr(config_rom_length);
 	size_t config_rom_length;
 	int config_rom_retries;
 	unsigned is_local:1;
@@ -298,16 +298,25 @@ union fw_transaction_callback {
 	fw_transaction_callback_with_tstamp_t with_tstamp;
 };
 
-/*
- * This callback handles an inbound request subaction.  It is called in
- * RCU read-side context, therefore must not sleep.
+/**
+ * typedef fw_address_callback_t - Function to handle the request of the asynchronous transaction.
+ * @card: the card instance which receives the request
+ * @request: the request instance.
+ * @tcode: the transaction code
+ * @destination: the destination node ID
+ * @source: the source node ID
+ * @generation: the bus generation in which the request was sent
+ * @offset: the destination offset in source node.
+ * @data: the request content if available.
+ * @length: the length of data.
+ * @callback_data: the data registered with this function.
  *
- * The callback should not initiate outbound request subactions directly.
- * Otherwise there is a danger of recursion of inbound and outbound
- * transactions from and to the local node.
+ * This callback handles an inbound request subaction.
  *
  * The callback is responsible that fw_send_response() is called on the @request, except for FCP
  * registers for which the core takes care of that.
+ *
+ * Context: Process context.
  */
 typedef void (*fw_address_callback_t)(struct fw_card *card,
 				      struct fw_request *request,
@@ -322,22 +331,24 @@ struct fw_packet {
 	int generation;
 	u32 header[4];
 	size_t header_length;
-	void *payload;
+	void *payload __counted_by_ptr(payload_length);
 	size_t payload_length;
 	dma_addr_t payload_bus;
 	bool payload_mapped;
 	u32 timestamp;
 
+	// Used to handle the local-to-local packets in the AT request/response contexts.
+	struct list_head link_for_local;
+
 	/*
 	 * This callback is called when the packet transmission has completed.
 	 * For successful transmission, the status code is the ack received
 	 * from the destination.  Otherwise it is one of the juju-specific
 	 * rcodes:  RCODE_SEND_ERROR, _CANCELLED, _BUSY, _GENERATION, _NO_ACK.
-	 * The callback can be called from workqueue and thus must never block.
+	 * The callback is called from a workqueue. It is not preferable to block it so long.
 	 */
 	fw_packet_callback_t callback;
 	int ack;
-	struct list_head link;
 	void *driver_data;
 };
 
@@ -359,6 +370,11 @@ struct fw_transaction {
 	union fw_transaction_callback callback;
 	bool with_tstamp;
 	void *callback_data;
+
+	// For some error cases.
+	struct work_struct error_work;
+	int rcode;
+	u32 response_timestamp;
 };
 
 struct fw_address_handler {
@@ -411,9 +427,8 @@ void __fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode
  * A variation of __fw_send_request() to generate callback for response subaction without time
  * stamp.
  *
- * The callback is invoked in the workqueue context in most cases. However, if an error is detected
- * before queueing or the destination address refers to the local node, it is invoked in the
- * current context instead.
+ * After the transaction is completed successfully or unsuccessfully, the @callback will be called
+ * in process context.
  */
 static inline void fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode,
 				   int destination_id, int generation, int speed,
@@ -444,9 +459,8 @@ static inline void fw_send_request(struct fw_card *card, struct fw_transaction *
  *
  * A variation of __fw_send_request() to generate callback for response subaction with time stamp.
  *
- * The callback is invoked in the workqueue context in most cases. However, if an error is detected
- * before queueing or the destination address refers to the local node, it is invoked in the current
- * context instead.
+ * After the transaction is completed successfully or unsuccessfully, the @callback will be called
+ * in process context.
  */
 static inline void fw_send_request_with_tstamp(struct fw_card *card, struct fw_transaction *t,
 	int tcode, int destination_id, int generation, int speed, unsigned long long offset,
diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h
index 4e3baa5570687e..6aa1e1b2b4a57f 100644
--- a/include/linux/firmware/cirrus/cs_dsp.h
+++ b/include/linux/firmware/cirrus/cs_dsp.h
@@ -96,7 +96,7 @@ struct cs_dsp_alg_region {
 struct cs_dsp_coeff_ctl {
 	struct list_head list;
 	struct cs_dsp *dsp;
-	void *cache;
+	void *cache __counted_by_ptr(len);
 	const char *fw_name;
 	/* Subname is needed to match with firmware */
 	const char *subname;
diff --git a/include/linux/firmware/imx/se_api.h b/include/linux/firmware/imx/se_api.h
new file mode 100644
index 00000000000000..b1c4c9115d7b2a
--- /dev/null
+++ b/include/linux/firmware/imx/se_api.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2025 NXP
+ */
+
+#ifndef __SE_API_H__
+#define __SE_API_H__
+
+#include <linux/types.h>
+
+#define SOC_ID_OF_IMX8ULP		0x084d
+#define SOC_ID_OF_IMX93			0x9300
+
+#endif /* __SE_API_H__ */
diff --git a/include/linux/firmware/qcom/qcom_scm.h b/include/linux/firmware/qcom/qcom_scm.h
index 5747bd191bf15e..a0a6bc0229c4b7 100644
--- a/include/linux/firmware/qcom/qcom_scm.h
+++ b/include/linux/firmware/qcom/qcom_scm.h
@@ -64,35 +64,6 @@ bool qcom_scm_is_available(void);
 int qcom_scm_set_cold_boot_addr(void *entry);
 int qcom_scm_set_warm_boot_addr(void *entry);
 void qcom_scm_cpu_power_down(u32 flags);
-int qcom_scm_set_remote_state(u32 state, u32 id);
-
-struct qcom_scm_pas_context {
-	struct device *dev;
-	u32 pas_id;
-	phys_addr_t mem_phys;
-	size_t mem_size;
-	void *ptr;
-	dma_addr_t phys;
-	ssize_t size;
-	bool use_tzmem;
-};
-
-struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev,
-							     u32 pas_id,
-							     phys_addr_t mem_phys,
-							     size_t mem_size);
-int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size,
-			    struct qcom_scm_pas_context *ctx);
-void qcom_scm_pas_metadata_release(struct qcom_scm_pas_context *ctx);
-int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size);
-int qcom_scm_pas_auth_and_reset(u32 pas_id);
-int qcom_scm_pas_shutdown(u32 pas_id);
-bool qcom_scm_pas_supported(u32 pas_id);
-struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *ctx,
-						  void *input_rt, size_t input_rt_size,
-						  size_t *output_rt_size);
-
-int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx);
 
 int qcom_scm_io_readl(phys_addr_t addr, unsigned int *val);
 int qcom_scm_io_writel(phys_addr_t addr, unsigned int val);
diff --git a/include/linux/firmware/xlnx-zynqmp-ufs.h b/include/linux/firmware/xlnx-zynqmp-ufs.h
index d3538dd5822a5d..00383dd835f242 100644
--- a/include/linux/firmware/xlnx-zynqmp-ufs.h
+++ b/include/linux/firmware/xlnx-zynqmp-ufs.h
@@ -9,17 +9,17 @@
 #define __FIRMWARE_XLNX_ZYNQMP_UFS_H__
 
 #if IS_REACHABLE(CONFIG_ZYNQMP_FIRMWARE)
-int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready);
-int zynqmp_pm_is_sram_init_done(bool *is_done);
+int zynqmp_pm_wait_mphy_tx_rx_config_ready(u32 timeout_us);
+int zynqmp_pm_wait_sram_init_done(u32 timeout_us);
 int zynqmp_pm_set_sram_bypass(void);
 int zynqmp_pm_get_ufs_calibration_values(u32 *val);
 #else
-static inline int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready)
+static inline int zynqmp_pm_wait_mphy_tx_rx_config_ready(u32 timeout_us)
 {
 	return -ENODEV;
 }
 
-static inline int zynqmp_pm_is_sram_init_done(bool *is_done)
+static inline int zynqmp_pm_wait_sram_init_done(u32 timeout_us)
 {
 	return -ENODEV;
 }
diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
index f4b3df614c7739..41a1cd810bdac7 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
+++ b/include/linux/firmware/xlnx-zynqmp.h
@@ -277,11 +277,6 @@ enum rpu_oper_mode {
 	PM_RPU_MODE_SPLIT = 1,
 };
 
-enum rpu_boot_mem {
-	PM_RPU_BOOTMEM_LOVEC = 0,
-	PM_RPU_BOOTMEM_HIVEC = 1,
-};
-
 enum rpu_tcm_comb {
 	PM_RPU_TCM_SPLIT = 0,
 	PM_RPU_TCM_COMB = 1,
diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h
index cf841dc71feffd..7e7c369e0a6cff 100644
--- a/include/linux/fortify-string.h
+++ b/include/linux/fortify-string.h
@@ -458,10 +458,8 @@ __FORTIFY_INLINE bool fortify_memset_chk(__kernel_size_t size,
  * __struct_size() vs __member_size() must be captured here to avoid
  * evaluating argument side-effects further into the macro layers.
  */
-#ifndef CONFIG_KMSAN
 #define memset(p, c, s) __fortify_memset_chk(p, c, s,			\
 		__struct_size(p), __member_size(p))
-#endif
 
 /*
  * To make sure the compiler can enforce protection against buffer overflows,
diff --git a/include/linux/fs.h b/include/linux/fs.h
index f9d1e05e8ae68e..3db90996756f2b 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1436,10 +1436,10 @@ static inline void i_gid_write(struct inode *inode, gid_t gid)
  * @idmap: idmap of the mount the inode was found from
  * @inode: inode to map
  *
- * Return: whe inode's i_uid mapped down according to @idmap.
+ * Return: the inode's i_uid mapped down according to @idmap.
  * If the inode's i_uid has no mapping INVALID_VFSUID is returned.
  */
-static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap,
+static inline vfsuid_t i_uid_into_vfsuid(const struct mnt_idmap *idmap,
 					 const struct inode *inode)
 {
 	return make_vfsuid(idmap, i_user_ns(inode), inode->i_uid);
@@ -1456,7 +1456,7 @@ static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap,
  *
  * Return: true if @inode's i_uid field needs to be updated, false if not.
  */
-static inline bool i_uid_needs_update(struct mnt_idmap *idmap,
+static inline bool i_uid_needs_update(const struct mnt_idmap *idmap,
 				      const struct iattr *attr,
 				      const struct inode *inode)
 {
@@ -1474,7 +1474,7 @@ static inline bool i_uid_needs_update(struct mnt_idmap *idmap,
  * Safely update @inode's i_uid field translating the vfsuid of any idmapped
  * mount into the filesystem kuid.
  */
-static inline void i_uid_update(struct mnt_idmap *idmap,
+static inline void i_uid_update(const struct mnt_idmap *idmap,
 				const struct iattr *attr,
 				struct inode *inode)
 {
@@ -1491,7 +1491,7 @@ static inline void i_uid_update(struct mnt_idmap *idmap,
  * Return: the inode's i_gid mapped down according to @idmap.
  * If the inode's i_gid has no mapping INVALID_VFSGID is returned.
  */
-static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap,
+static inline vfsgid_t i_gid_into_vfsgid(const struct mnt_idmap *idmap,
 					 const struct inode *inode)
 {
 	return make_vfsgid(idmap, i_user_ns(inode), inode->i_gid);
@@ -1508,7 +1508,7 @@ static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap,
  *
  * Return: true if @inode's i_gid field needs to be updated, false if not.
  */
-static inline bool i_gid_needs_update(struct mnt_idmap *idmap,
+static inline bool i_gid_needs_update(const struct mnt_idmap *idmap,
 				      const struct iattr *attr,
 				      const struct inode *inode)
 {
@@ -1526,7 +1526,7 @@ static inline bool i_gid_needs_update(struct mnt_idmap *idmap,
  * Safely update @inode's i_gid field translating the vfsgid of any idmapped
  * mount into the filesystem kgid.
  */
-static inline void i_gid_update(struct mnt_idmap *idmap,
+static inline void i_gid_update(const struct mnt_idmap *idmap,
 				const struct iattr *attr,
 				struct inode *inode)
 {
@@ -1544,7 +1544,7 @@ static inline void i_gid_update(struct mnt_idmap *idmap,
  * an idmapped mount map the caller's fsuid according to @idmap.
  */
 static inline void inode_fsuid_set(struct inode *inode,
-				   struct mnt_idmap *idmap)
+				   const struct mnt_idmap *idmap)
 {
 	inode->i_uid = mapped_fsuid(idmap, i_user_ns(inode));
 }
@@ -1558,7 +1558,7 @@ static inline void inode_fsuid_set(struct inode *inode,
  * an idmapped mount map the caller's fsgid according to @idmap.
  */
 static inline void inode_fsgid_set(struct inode *inode,
-				   struct mnt_idmap *idmap)
+				   const struct mnt_idmap *idmap)
 {
 	inode->i_gid = mapped_fsgid(idmap, i_user_ns(inode));
 }
@@ -1575,7 +1575,7 @@ static inline void inode_fsgid_set(struct inode *inode,
  * Return: true if fsuid and fsgid is mapped, false if not.
  */
 static inline bool fsuidgid_has_mapping(struct super_block *sb,
-					struct mnt_idmap *idmap)
+					const struct mnt_idmap *idmap)
 {
 	struct user_namespace *fs_userns = sb->s_user_ns;
 	kuid_t kuid;
@@ -1755,25 +1755,25 @@ static inline bool file_write_not_started(const struct file *file)
 	return sb_write_not_started(file_inode(file)->i_sb);
 }
 
-bool inode_owner_or_capable(struct mnt_idmap *idmap,
+bool inode_owner_or_capable(const struct mnt_idmap *idmap,
 			    const struct inode *inode);
 
 /*
  * VFS helper functions..
  */
-int vfs_create(struct mnt_idmap *, struct dentry *, umode_t,
+int vfs_create(const struct mnt_idmap *, struct dentry *, umode_t,
 	       struct delegated_inode *);
-struct dentry *vfs_mkdir(struct mnt_idmap *, struct inode *,
+struct dentry *vfs_mkdir(const struct mnt_idmap *, struct inode *,
 			 struct dentry *, umode_t, struct delegated_inode *);
-int vfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *,
+int vfs_mknod(const struct mnt_idmap *, struct inode *, struct dentry *,
 	      umode_t, dev_t, struct delegated_inode *);
-int vfs_symlink(struct mnt_idmap *, struct inode *,
+int vfs_symlink(const struct mnt_idmap *, struct inode *,
 		struct dentry *, const char *, struct delegated_inode *);
-int vfs_link(struct dentry *, struct mnt_idmap *, struct inode *,
+int vfs_link(struct dentry *, const struct mnt_idmap *, struct inode *,
 	     struct dentry *, struct delegated_inode *);
-int vfs_rmdir(struct mnt_idmap *, struct inode *, struct dentry *,
+int vfs_rmdir(const struct mnt_idmap *, struct inode *, struct dentry *,
 	      struct delegated_inode *);
-int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *,
+int vfs_unlink(const struct mnt_idmap *, struct inode *, struct dentry *,
 	       struct delegated_inode *);
 
 /**
@@ -1787,7 +1787,7 @@ int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *,
  * @flags:             rename flags
  */
 struct renamedata {
-	struct mnt_idmap *mnt_idmap;
+	const struct mnt_idmap *mnt_idmap;
 	struct dentry *old_parent;
 	struct dentry *old_dentry;
 	struct dentry *new_parent;
@@ -1798,14 +1798,14 @@ struct renamedata {
 
 int vfs_rename(struct renamedata *);
 
-static inline int vfs_whiteout(struct mnt_idmap *idmap,
+static inline int vfs_whiteout(const struct mnt_idmap *idmap,
 			       struct inode *dir, struct dentry *dentry)
 {
 	return vfs_mknod(idmap, dir, dentry, S_IFCHR | WHITEOUT_MODE,
 			 WHITEOUT_DEV, NULL);
 }
 
-struct file *kernel_tmpfile_open(struct mnt_idmap *idmap,
+struct file *kernel_tmpfile_open(const struct mnt_idmap *idmap,
 				 const struct path *parentpath,
 				 umode_t mode, int open_flag,
 				 const struct cred *cred);
@@ -1830,12 +1830,12 @@ extern long compat_ptr_ioctl(struct file *file, unsigned int cmd,
 /*
  * VFS file helper functions.
  */
-void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode,
+void inode_init_owner(const struct mnt_idmap *idmap, struct inode *inode,
 		      const struct inode *dir, umode_t mode);
 extern bool may_open_dev(const struct path *path);
-umode_t mode_strip_sgid(struct mnt_idmap *idmap,
+umode_t mode_strip_sgid(const struct mnt_idmap *idmap,
 			const struct inode *dir, umode_t mode);
-bool in_group_or_capable(struct mnt_idmap *idmap,
+bool in_group_or_capable(const struct mnt_idmap *idmap,
 			 const struct inode *inode, vfsgid_t vfsgid);
 
 /*
@@ -1994,26 +1994,26 @@ enum fs_update_time {
 struct inode_operations {
 	struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int);
 	const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *);
-	int (*permission) (struct mnt_idmap *, struct inode *, int);
+	int (*permission) (const struct mnt_idmap *, struct inode *, int);
 	struct posix_acl * (*get_inode_acl)(struct inode *, int, bool);
 
 	int (*readlink) (struct dentry *, char __user *,int);
 
-	int (*create) (struct mnt_idmap *, struct inode *,struct dentry *,
+	int (*create) (const struct mnt_idmap *, struct inode *,struct dentry *,
 		       umode_t);
 	int (*link) (struct dentry *,struct inode *,struct dentry *);
 	int (*unlink) (struct inode *,struct dentry *);
-	int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *,
+	int (*symlink) (const struct mnt_idmap *, struct inode *,struct dentry *,
 			const char *);
-	struct dentry *(*mkdir) (struct mnt_idmap *, struct inode *,
+	struct dentry *(*mkdir) (const struct mnt_idmap *, struct inode *,
 				 struct dentry *, umode_t);
 	int (*rmdir) (struct inode *,struct dentry *);
-	int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *,
+	int (*mknod) (const struct mnt_idmap *, struct inode *,struct dentry *,
 		      umode_t,dev_t);
-	int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *,
+	int (*rename) (const struct mnt_idmap *, struct inode *, struct dentry *,
 			struct inode *, struct dentry *, unsigned int);
-	int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *);
-	int (*getattr) (struct mnt_idmap *, const struct path *,
+	int (*setattr) (const struct mnt_idmap *, struct dentry *, struct iattr *);
+	int (*getattr) (const struct mnt_idmap *, const struct path *,
 			struct kstat *, u32, unsigned int);
 	ssize_t (*listxattr) (struct dentry *, char *, size_t);
 	int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
@@ -2024,13 +2024,13 @@ struct inode_operations {
 	int (*atomic_open)(struct inode *, struct dentry *,
 			   struct file *, unsigned open_flag,
 			   umode_t create_mode);
-	int (*tmpfile) (struct mnt_idmap *, struct inode *,
+	int (*tmpfile) (const struct mnt_idmap *, struct inode *,
 			struct file *, umode_t);
-	struct posix_acl *(*get_acl)(struct mnt_idmap *, struct dentry *,
+	struct posix_acl *(*get_acl)(const struct mnt_idmap *, struct dentry *,
 				     int);
-	int (*set_acl)(struct mnt_idmap *, struct dentry *,
+	int (*set_acl)(const struct mnt_idmap *, struct dentry *,
 		       struct posix_acl *, int);
-	int (*fileattr_set)(struct mnt_idmap *idmap,
+	int (*fileattr_set)(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, struct file_kattr *fa);
 	int (*fileattr_get)(struct dentry *dentry, struct file_kattr *fa);
 	struct offset_ctx *(*get_offset_ctx)(struct inode *inode);
@@ -2173,7 +2173,7 @@ extern loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
 				 (inode)->i_rdev == WHITEOUT_DEV)
 #define IS_ANON_FILE(inode)	((inode)->i_flags & S_ANON_INODE)
 
-static inline bool HAS_UNMAPPED_ID(struct mnt_idmap *idmap,
+static inline bool HAS_UNMAPPED_ID(const struct mnt_idmap *idmap,
 				   struct inode *inode)
 {
 	return !vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) ||
@@ -2459,7 +2459,7 @@ struct filename {
 static_assert(offsetof(struct filename, iname) % sizeof(long) == 0);
 static_assert(sizeof(struct filename) % 64 == 0);
 
-static inline struct mnt_idmap *file_mnt_idmap(const struct file *file)
+static inline const struct mnt_idmap *file_mnt_idmap(const struct file *file)
 {
 	return mnt_idmap(file->f_path.mnt);
 }
@@ -2483,7 +2483,7 @@ static inline bool is_idmapped_mnt(const struct vfsmount *mnt)
 }
 
 int vfs_truncate(const struct path *, loff_t);
-int do_truncate(struct mnt_idmap *, struct dentry *, loff_t start,
+int do_truncate(const struct mnt_idmap *, struct dentry *, loff_t start,
 		unsigned int time_attrs, struct file *filp);
 extern int vfs_fallocate(struct file *file, int mode, loff_t offset,
 			loff_t len);
@@ -2707,10 +2707,10 @@ static inline int bmap(struct inode *inode,  sector_t *block)
 }
 #endif
 
-int notify_change(struct mnt_idmap *, struct dentry *,
+int notify_change(const struct mnt_idmap *, struct dentry *,
 		  struct iattr *, struct delegated_inode *);
-int inode_permission(struct mnt_idmap *, struct inode *, int);
-int generic_permission(struct mnt_idmap *, struct inode *, int);
+int inode_permission(const struct mnt_idmap *, struct inode *, int);
+int generic_permission(const struct mnt_idmap *, struct inode *, int);
 static inline int file_permission(struct file *file, int mask)
 {
 	return inode_permission(file_mnt_idmap(file),
@@ -2721,12 +2721,12 @@ static inline int path_permission(const struct path *path, int mask)
 	return inode_permission(mnt_idmap(path->mnt),
 				d_inode(path->dentry), mask);
 }
-int __check_sticky(struct mnt_idmap *idmap, struct inode *dir,
+int __check_sticky(const struct mnt_idmap *idmap, struct inode *dir,
 		   struct inode *inode);
 
-int may_delete_dentry(struct mnt_idmap *idmap, struct inode *dir,
+int may_delete_dentry(const struct mnt_idmap *idmap, struct inode *dir,
 		      struct dentry *victim, bool isdir);
-int may_create_dentry(struct mnt_idmap *idmap,
+int may_create_dentry(const struct mnt_idmap *idmap,
 		      struct inode *dir, struct dentry *child);
 
 static inline bool execute_ok(struct inode *inode)
@@ -3045,9 +3045,9 @@ static inline struct inode *new_inode_pseudo(struct super_block *sb)
 }
 extern struct inode *new_inode(struct super_block *sb);
 extern void free_inode_nonrcu(struct inode *inode);
-extern int setattr_should_drop_suidgid(struct mnt_idmap *, struct inode *);
+extern int setattr_should_drop_suidgid(const struct mnt_idmap *, struct inode *);
 extern int file_remove_privs(struct file *);
-int setattr_should_drop_sgid(struct mnt_idmap *idmap,
+int setattr_should_drop_sgid(const struct mnt_idmap *idmap,
 			     const struct inode *inode);
 
 /*
@@ -3204,7 +3204,7 @@ extern int page_symlink(struct inode *inode, const char *symname, int len);
 extern const struct inode_operations page_symlink_inode_operations;
 extern void kfree_link(void *);
 void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode);
-void generic_fillattr(struct mnt_idmap *, u32, struct inode *, struct kstat *);
+void generic_fillattr(const struct mnt_idmap *, u32, struct inode *, struct kstat *);
 void generic_fill_statx_attr(struct inode *inode, struct kstat *stat);
 void generic_fill_statx_atomic_writes(struct kstat *stat,
 				      unsigned int unit_min,
@@ -3261,9 +3261,9 @@ extern int dcache_dir_open(struct inode *, struct file *);
 extern int dcache_dir_close(struct inode *, struct file *);
 extern loff_t dcache_dir_lseek(struct file *, loff_t, int);
 extern int dcache_readdir(struct file *, struct dir_context *);
-extern int simple_setattr(struct mnt_idmap *, struct dentry *,
+extern int simple_setattr(const struct mnt_idmap *, struct dentry *,
 			  struct iattr *);
-extern int simple_getattr(struct mnt_idmap *, const struct path *,
+extern int simple_getattr(const struct mnt_idmap *, const struct path *,
 			  struct kstat *, u32, unsigned int);
 extern int simple_statfs(struct dentry *, struct kstatfs *);
 extern int simple_open(struct inode *inode, struct file *file);
@@ -3276,7 +3276,7 @@ void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry,
 			     struct inode *new_dir, struct dentry *new_dentry);
 extern int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry,
 				  struct inode *new_dir, struct dentry *new_dentry);
-extern int simple_rename(struct mnt_idmap *, struct inode *,
+extern int simple_rename(const struct mnt_idmap *, struct inode *,
 			 struct dentry *, struct inode *, struct dentry *,
 			 unsigned int);
 extern void simple_recursive_removal(struct dentry *,
@@ -3397,11 +3397,11 @@ static inline bool generic_ci_validate_strict_name(struct inode *dir,
 }
 #endif
 
-int may_setattr(struct mnt_idmap *idmap, struct inode *inode,
+int may_setattr(const struct mnt_idmap *idmap, struct inode *inode,
 		unsigned int ia_valid);
-int setattr_prepare(struct mnt_idmap *, struct dentry *, struct iattr *);
+int setattr_prepare(const struct mnt_idmap *, struct dentry *, struct iattr *);
 extern int inode_newsize_ok(const struct inode *, loff_t offset);
-void setattr_copy(struct mnt_idmap *, struct inode *inode,
+void setattr_copy(const struct mnt_idmap *, struct inode *inode,
 		  const struct iattr *attr);
 
 extern int file_update_time(struct file *file);
@@ -3578,7 +3578,7 @@ static inline bool is_sxid(umode_t mode)
 	return mode & (S_ISUID | S_ISGID);
 }
 
-static inline int check_sticky(struct mnt_idmap *idmap,
+static inline int check_sticky(const struct mnt_idmap *idmap,
 			       struct inode *dir, struct inode *inode)
 {
 	if (!(dir->i_mode & S_ISVTX))
@@ -3653,23 +3653,6 @@ extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len,
 extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
 			   int advice);
 
-static inline bool vfs_empty_path(int dfd, const char __user *path)
-{
-	char c;
-
-	if (dfd < 0)
-		return false;
-
-	/* We now allow NULL to be used for empty path. */
-	if (!path)
-		return true;
-
-	if (unlikely(get_user(c, path)))
-		return false;
-
-	return !c;
-}
-
 int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter);
 
 static inline bool extensible_ioctl_valid(unsigned int cmd_a,
diff --git a/include/linux/fs_context.h b/include/linux/fs_context.h
index 0d6c8a6d7be21f..c920aba5177c36 100644
--- a/include/linux/fs_context.h
+++ b/include/linux/fs_context.h
@@ -150,6 +150,10 @@ extern int vfs_parse_fs_param_source(struct fs_context *fc,
 				     struct fs_parameter *param);
 extern void fc_drop_locked(struct fs_context *fc);
 
+extern int get_tree_super(struct fs_context *fc,
+			 int (*test)(struct super_block *, struct fs_context *),
+			 int (*fill_super)(struct super_block *sb,
+					   struct fs_context *fc));
 extern int get_tree_nodev(struct fs_context *fc,
 			 int (*fill_super)(struct super_block *sb,
 					   struct fs_context *fc));
diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h
index 4c91a019972b6a..ee524c863fa9be 100644
--- a/include/linux/fscache-cache.h
+++ b/include/linux/fscache-cache.h
@@ -67,7 +67,7 @@ struct fscache_cache_ops {
 
 	/* Change the size of a data object */
 	void (*resize_cookie)(struct netfs_cache_resources *cres,
-			      loff_t new_size);
+			      uoff_t new_size);
 
 	/* Invalidate an object */
 	bool (*invalidate_cookie)(struct fscache_cookie *cookie);
diff --git a/include/linux/fscache.h b/include/linux/fscache.h
index 58fdb9605425dc..f2d958bd1f480b 100644
--- a/include/linux/fscache.h
+++ b/include/linux/fscache.h
@@ -112,7 +112,7 @@ struct fscache_cookie {
 	struct list_head		proc_link;	/* Link in proc list */
 	struct list_head		commit_link;	/* Link in commit queue */
 	struct work_struct		work;		/* Commit/relinq/withdraw work */
-	loff_t				object_size;	/* Size of the netfs object */
+	uoff_t				object_size;	/* Size of the netfs object */
 	unsigned long			unused_at;	/* Time at which unused (jiffies) */
 	unsigned long			flags;
 #define FSCACHE_COOKIE_RELINQUISHED	0		/* T if cookie has been relinquished */
@@ -147,6 +147,23 @@ struct fscache_cookie {
 	};
 };
 
+enum fscache_extent_type {
+	FSCACHE_EXTENT_DATA,
+	FSCACHE_EXTENT_ZERO,
+} __mode(byte);
+
+/*
+ * Cache occupancy information.
+ */
+struct fscache_occupancy {
+	unsigned long long	query_from;	/* Point to query from */
+	unsigned long long	query_to;	/* Point to query to */
+	unsigned long long	cached_from[2];	/* Point at which cache extents start */
+	unsigned long long	cached_to[2];	/* Point at which cache extents end */
+	unsigned int		granularity;	/* Granularity desired */
+	enum fscache_extent_type cached_type[2];	/* Type of cache extent */
+};
+
 /*
  * slow-path functions for when there is actually caching available, and the
  * netfs does actually have a valid token
@@ -163,22 +180,22 @@ extern struct fscache_cookie *__fscache_acquire_cookie(
 	u8,
 	const void *, size_t,
 	const void *, size_t,
-	loff_t);
+	uoff_t);
 extern void __fscache_use_cookie(struct fscache_cookie *, bool);
-extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const loff_t *);
+extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const uoff_t *);
 extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool);
-extern void __fscache_resize_cookie(struct fscache_cookie *, loff_t);
-extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int);
+extern void __fscache_resize_cookie(struct fscache_cookie *, uoff_t);
+extern void __fscache_invalidate(struct fscache_cookie *, const void *, uoff_t, unsigned int);
 extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *);
 extern int __fscache_begin_write_operation(struct netfs_cache_resources *, struct fscache_cookie *);
 
 void __fscache_write_to_cache(struct fscache_cookie *cookie,
 			      struct address_space *mapping,
-			      loff_t start, size_t len, loff_t i_size,
+			      uoff_t start, size_t len, uoff_t i_size,
 			      netfs_io_terminated_t term_func,
 			      void *term_func_priv,
 			      bool using_pgpriv2, bool cond);
-extern void __fscache_clear_page_bits(struct address_space *, loff_t, size_t);
+extern void __fscache_clear_page_bits(struct address_space *, uoff_t, size_t);
 
 /**
  * fscache_acquire_volume - Register a volume as desiring caching services
@@ -249,7 +266,7 @@ struct fscache_cookie *fscache_acquire_cookie(struct fscache_volume *volume,
 					      size_t index_key_len,
 					      const void *aux_data,
 					      size_t aux_data_len,
-					      loff_t object_size)
+					      uoff_t object_size)
 {
 	if (!fscache_volume_valid(volume))
 		return NULL;
@@ -286,7 +303,7 @@ static inline void fscache_use_cookie(struct fscache_cookie *cookie,
  */
 static inline void fscache_unuse_cookie(struct fscache_cookie *cookie,
 					const void *aux_data,
-					const loff_t *object_size)
+					const uoff_t *object_size)
 {
 	if (fscache_cookie_valid(cookie))
 		__fscache_unuse_cookie(cookie, aux_data, object_size);
@@ -327,7 +344,7 @@ static inline void *fscache_get_aux(struct fscache_cookie *cookie)
  */
 static inline
 void fscache_update_aux(struct fscache_cookie *cookie,
-			const void *aux_data, const loff_t *object_size)
+			const void *aux_data, const uoff_t *object_size)
 {
 	void *p = fscache_get_aux(cookie);
 
@@ -343,7 +360,7 @@ extern atomic_t fscache_n_updates;
 
 static inline
 void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data,
-			     const loff_t *object_size)
+			     const uoff_t *object_size)
 {
 #ifdef CONFIG_FSCACHE_STATS
 	atomic_inc(&fscache_n_updates);
@@ -369,7 +386,7 @@ void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data
  */
 static inline
 void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data,
-			   const loff_t *object_size)
+			   const uoff_t *object_size)
 {
 	if (fscache_cookie_enabled(cookie))
 		__fscache_update_cookie(cookie, aux_data, object_size);
@@ -386,7 +403,7 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data,
  * description.
  */
 static inline
-void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size)
+void fscache_resize_cookie(struct fscache_cookie *cookie, uoff_t new_size)
 {
 	if (fscache_cookie_enabled(cookie))
 		__fscache_resize_cookie(cookie, new_size);
@@ -413,7 +430,7 @@ void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size)
  */
 static inline
 void fscache_invalidate(struct fscache_cookie *cookie,
-			const void *aux_data, loff_t size, unsigned int flags)
+			const void *aux_data, uoff_t size, unsigned int flags)
 {
 	if (fscache_cookie_enabled(cookie))
 		__fscache_invalidate(cookie, aux_data, size, flags);
@@ -502,7 +519,7 @@ static inline void fscache_end_operation(struct netfs_cache_resources *cres)
  */
 static inline
 int fscache_read(struct netfs_cache_resources *cres,
-		 loff_t start_pos,
+		 uoff_t start_pos,
 		 struct iov_iter *iter,
 		 enum netfs_read_from_hole read_hole,
 		 netfs_io_terminated_t term_func,
@@ -561,7 +578,7 @@ int fscache_begin_write_operation(struct netfs_cache_resources *cres,
  */
 static inline
 int fscache_write(struct netfs_cache_resources *cres,
-		  loff_t start_pos,
+		  uoff_t start_pos,
 		  struct iov_iter *iter,
 		  netfs_io_terminated_t term_func,
 		  void *term_func_priv)
@@ -581,7 +598,7 @@ int fscache_write(struct netfs_cache_resources *cres,
  * waiting.
  */
 static inline void fscache_clear_page_bits(struct address_space *mapping,
-					   loff_t start, size_t len,
+					   uoff_t start, size_t len,
 					   bool caching)
 {
 	if (caching)
@@ -615,7 +632,7 @@ static inline void fscache_clear_page_bits(struct address_space *mapping,
  */
 static inline void fscache_write_to_cache(struct fscache_cookie *cookie,
 					  struct address_space *mapping,
-					  loff_t start, size_t len, loff_t i_size,
+					  uoff_t start, size_t len, uoff_t i_size,
 					  netfs_io_terminated_t term_func,
 					  void *term_func_priv,
 					  bool using_pgpriv2, bool caching)
diff --git a/include/linux/fsl/bestcomm/gen_bd.h b/include/linux/fsl/bestcomm/gen_bd.h
index aeb312a1cd004a..7253d308d77a48 100644
--- a/include/linux/fsl/bestcomm/gen_bd.h
+++ b/include/linux/fsl/bestcomm/gen_bd.h
@@ -16,10 +16,6 @@ struct bcom_gen_bd {
 };
 
 
-extern struct bcom_task *
-bcom_gen_bd_rx_init(int queue_len, phys_addr_t fifo,
-			int initiator, int ipr, int maxbufsize);
-
 extern int
 bcom_gen_bd_rx_reset(struct bcom_task *tsk);
 
@@ -27,10 +23,6 @@ extern void
 bcom_gen_bd_rx_release(struct bcom_task *tsk);
 
 
-extern struct bcom_task *
-bcom_gen_bd_tx_init(int queue_len, phys_addr_t fifo,
-			int initiator, int ipr);
-
 extern int
 bcom_gen_bd_tx_reset(struct bcom_task *tsk);
 
diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h
index 07ef1c8341a4f8..1b39b27f0cfa81 100644
--- a/include/linux/generic_pt/common.h
+++ b/include/linux/generic_pt/common.h
@@ -213,4 +213,10 @@ enum {
 	PT_FEAT_X86_64_AMD_ENCRYPT_TABLES = PT_FEAT_FMT_START,
 };
 
+struct pt_bcm2712 {
+	struct pt_common common;
+	u8 bigpage_lg2;
+	u8 superpage_lg2;
+};
+
 #endif
diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h
index dd0edd02a48a24..13ebb72a67de31 100644
--- a/include/linux/generic_pt/iommu.h
+++ b/include/linux/generic_pt/iommu.h
@@ -346,6 +346,18 @@ struct pt_iommu_x86_64_hw_info {
 
 IOMMU_FORMAT(x86_64, x86_64_pt);
 
+struct pt_iommu_bcm2712_cfg {
+	struct pt_iommu_cfg common;
+	u8 bigpage_lg2;
+	u8 superpage_lg2;
+};
+
+struct pt_iommu_bcm2712_hw_info {
+	phys_addr_t pt_base;
+};
+
+IOMMU_FORMAT(bcm2712, bcm2712pt);
+
 #undef IOMMU_PROTOTYPES
 #undef IOMMU_FORMAT
 #endif
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 872bc53f32ec87..aa4601805f21af 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -329,7 +329,7 @@ void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) __alloc_size(1);
 
 void free_pages_exact(void *virt, size_t size);
 
-__meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);
+void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2);
 #define alloc_pages_exact_nid(...)					\
 	alloc_hooks(alloc_pages_exact_nid_noprof(__VA_ARGS__))
 
diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h
index 190191411009f2..bfd4c43ed77794 100644
--- a/include/linux/gfp_types.h
+++ b/include/linux/gfp_types.h
@@ -244,7 +244,7 @@ enum {
  * definitely preferable to use the flag rather than opencode endless
  * loop around allocator.
  * Allocating pages from the buddy with __GFP_NOFAIL and order > 1 is
- * not supported. Please consider using kvmalloc() instead.
+ * discouraged. Please consider using kvmalloc() instead if possible.
  */
 #define __GFP_IO	((__force gfp_t)___GFP_IO)
 #define __GFP_FS	((__force gfp_t)___GFP_FS)
diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h
index 17511434ed077d..4077dc678caed4 100644
--- a/include/linux/gpio/driver.h
+++ b/include/linux/gpio/driver.h
@@ -359,6 +359,10 @@ struct gpio_irq_chip {
  * @set_config: optional hook for all kinds of settings. Uses the same
  *	packed config format as generic pinconf. Must return 0 on success and
  *	a negative error number on failure.
+ * @get_config: optional hook to read back a setting. Takes the packed
+ *	generic pinconf parameter to query and returns its bare argument in
+ *	the same variable, like pinctrl_gpio_get_config(). Must return 0 on
+ *	success and a negative error number on failure.
  * @to_irq: optional hook supporting non-static gpiod_to_irq() mappings;
  *	implementation may not sleep
  * @dbg_show: optional routine to show contents in debugfs; default code
@@ -434,6 +438,9 @@ struct gpio_chip {
 	int			(*set_config)(struct gpio_chip *gc,
 					      unsigned int offset,
 					      unsigned long config);
+	int			(*get_config)(struct gpio_chip *gc,
+					      unsigned int offset,
+					      unsigned long *config);
 	int			(*to_irq)(struct gpio_chip *gc,
 						unsigned int offset);
 
@@ -708,6 +715,8 @@ int gpiochip_generic_request(struct gpio_chip *gc, unsigned int offset);
 void gpiochip_generic_free(struct gpio_chip *gc, unsigned int offset);
 int gpiochip_generic_config(struct gpio_chip *gc, unsigned int offset,
 			    unsigned long config);
+int gpiochip_generic_get_config(struct gpio_chip *gc, unsigned int offset,
+				unsigned long *config);
 
 /**
  * struct gpio_pin_range - pin range controlled by a gpio chip
diff --git a/include/linux/gpio/regmap.h b/include/linux/gpio/regmap.h
index 6f52c140e50d00..6fe0877717cb02 100644
--- a/include/linux/gpio/regmap.h
+++ b/include/linux/gpio/regmap.h
@@ -46,6 +46,7 @@ enum gpio_regmap_operation {
  * @parent:		The parent device
  * @regmap:		The regmap used to access the registers
  *			given, the name of the device is used
+ * @pm_dev:		(Optional) Device to use for runtime power management.
  * @fwnode:		(Optional) The firmware node.
  *			If not given, the fwnode of the parent is used.
  * @label:		(Optional) Descriptive name for GPIO controller.
@@ -117,6 +118,7 @@ enum gpio_regmap_operation {
 struct gpio_regmap_config {
 	struct device *parent;
 	struct regmap *regmap;
+	struct device *pm_dev;
 	struct fwnode_handle *fwnode;
 
 	const char *label;
diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h
index 2c36124bb69644..ddc4eca84175bb 100644
--- a/include/linux/gpu_buddy.h
+++ b/include/linux/gpu_buddy.h
@@ -43,8 +43,8 @@
 /**
  * GPU_BUDDY_CLEAR_ALLOCATION - Prefer pre-cleared (zeroed) memory
  *
- * Attempt to allocate from the clear tree first. If insufficient clear
- * memory is available, falls back to dirty memory. Useful when the
+ * Attempt to allocate outside dirty-tracked ranges first. If insufficient
+ * clear memory is available, falls back to dirty memory. Useful when the
  * caller needs zeroed memory and wants to avoid GPU clear operations.
  */
 #define GPU_BUDDY_CLEAR_ALLOCATION		BIT(3)
@@ -53,8 +53,8 @@
  * GPU_BUDDY_CLEARED - Mark returned blocks as cleared
  *
  * Used with gpu_buddy_free_list() to indicate that the memory being
- * freed has been cleared (zeroed). The blocks will be placed in the
- * clear tree for future GPU_BUDDY_CLEAR_ALLOCATION requests.
+ * freed has been cleared (zeroed). The blocks will be removed from the
+ * dirty tracker for future GPU_BUDDY_CLEAR_ALLOCATION requests.
  */
 #define GPU_BUDDY_CLEARED			BIT(4)
 
@@ -67,15 +67,17 @@
  */
 #define GPU_BUDDY_TRIM_DISABLE			BIT(5)
 
-enum gpu_buddy_free_tree {
-	GPU_BUDDY_CLEAR_TREE = 0,
-	GPU_BUDDY_DIRTY_TREE,
-	GPU_BUDDY_MAX_FREE_TREES,
+/*
+ * Clear/dirty state of a free block. Ordered so a numerically larger value
+ * is "more clear" (DIRTY < MIXED < CLEAR) which lets subtree_block_state be
+ * maintained as a simple max-augment over the per-order free tree.
+ */
+enum gpu_block_state {
+	GPU_BLOCK_DIRTY = 0,
+	GPU_BLOCK_MIXED = 1,
+	GPU_BLOCK_CLEAR = 2,
 };
 
-#define for_each_free_tree(tree) \
-	for ((tree) = 0; (tree) < GPU_BUDDY_MAX_FREE_TREES; (tree)++)
-
 /**
  * struct gpu_buddy_block - Block within a buddy allocator
  *
@@ -103,6 +105,13 @@ struct gpu_buddy_block {
 #define   GPU_BUDDY_ALLOCATED	   (1 << 10)
 #define   GPU_BUDDY_FREE	   (2 << 10)
 #define   GPU_BUDDY_SPLIT	   (3 << 10)
+/*
+ * GPU_BUDDY_HEADER_CLEAR has two roles:
+ *  - FREE state:      set when the block's full range is cleared (dirty
+ *                     tracker confirmed no overlap).
+ *  - ALLOCATED state: set when the block was served from cleared memory,
+ *                     informing the caller that no GPU clear pass is needed.
+ */
 #define GPU_BUDDY_HEADER_CLEAR  GENMASK_ULL(9, 9)
 /* Free to be used, if needed in the future */
 #define GPU_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6)
@@ -128,13 +137,44 @@ struct gpu_buddy_block {
 		struct list_head link;
 	};
 /* private: */
-	struct list_head tmp_link;
+	enum gpu_block_state subtree_block_state;
 	unsigned int subtree_max_alignment;
+	struct list_head tmp_link;
+	bool has_clear;
 };
 
 /* Order-zero must be at least SZ_4K */
 #define GPU_BUDDY_MAX_ORDER (63 - 12)
 
+/**
+ * struct gpu_dirty_extent - a contiguous dirty address range
+ *
+ * Tracks a single contiguous address range whose memory content is known
+ * to be dirty.  Extents are non-overlapping and stored in an augmented
+ * red-black tree sorted by @start.  The augmented value @subtree_max_size
+ * allows O(log N) search for an extent of at least a given size.
+ */
+struct gpu_dirty_extent {
+/* private: */
+	struct rb_node	rb;
+	u64		start;
+	u64		end;
+	u64		subtree_max_size;
+};
+
+/**
+ * struct gpu_dirty_tracker - tracks dirty address intervals
+ *
+ * Maintains a set of non-overlapping dirty extents as an augmented
+ * red-black tree.
+ */
+struct gpu_dirty_tracker {
+/* private: */
+	struct rb_root	root;
+	/* Total bytes of dirty memory currently tracked. */
+	u64		total_dirty;
+};
+
 /**
  * struct gpu_buddy - GPU binary buddy allocator
  *
@@ -152,20 +192,21 @@ struct gpu_buddy_block {
  * @chunk_size: Minimum allocation granularity in bytes. Must be at least SZ_4K.
  * @size: Total size of the address space managed by this allocator in bytes.
  * @avail: Total free space currently available for allocation in bytes.
- * @clear_avail: Free space available in the clear tree (zeroed memory) in bytes.
- *               This is a subset of @avail.
  * @lock_dep_map: Annotates gpu_buddy API with a driver provided lock.
  */
 struct gpu_buddy {
 /* private: */
+	/* Tracker of dirty address ranges (decoupled from free_tree). */
+	struct gpu_dirty_tracker dirty;
 	/*
-	 * Array of red-black trees for free block management.
-	 * Indexed as free_trees[clear/dirty][order] where:
-	 * - Index 0 (GPU_BUDDY_CLEAR_TREE): blocks with zeroed content
-	 * - Index 1 (GPU_BUDDY_DIRTY_TREE): blocks with unknown content
-	 * Each tree holds free blocks of the corresponding order.
+	 * One RB-tree per order containing all free blocks (clear and
+	 * dirty alike).  The augment field subtree_block_state (a max over
+	 * the subtree of each block's state) lets clear allocations
+	 * find the right-most fully-clear or mixed block in O(log N).
+	 * Dirty free blocks coexist here but are also indexed by the
+	 * @dirty tracker for fast dirty allocation lookups.
 	 */
-	struct rb_root **free_trees;
+	struct rb_root *free_tree;
 	/*
 	 * Array of root blocks representing the top-level blocks of the
 	 * binary tree(s). Multiple roots exist when the total size is not
@@ -194,7 +235,6 @@ struct gpu_buddy {
 	u64 chunk_size;
 	u64 size;
 	u64 avail;
-	u64 clear_avail;
 #ifdef CONFIG_LOCKDEP
 	struct lockdep_map *lock_dep_map;
 #endif
@@ -226,17 +266,32 @@ struct gpu_buddy {
  *
  * Ensure driver lock is held.
  */
-static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm)
+static inline void gpu_buddy_driver_lock_held(const struct gpu_buddy *mm)
 {
 	if (mm->lock_dep_map)
 		lockdep_assert(lock_is_held_type(mm->lock_dep_map, 0));
 }
 #else
-static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm)
+static inline void gpu_buddy_driver_lock_held(const struct gpu_buddy *mm)
 {
 }
 #endif
 
+/**
+ * gpu_buddy_clear_avail - free space that is clear (zeroed), in bytes
+ * @mm: gpu buddy allocator
+ *
+ * A subset of @mm->avail. Derived on demand as @mm->avail minus the bytes
+ * the dirty tracker records as dirty, so it is always consistent with the
+ * tracker without a cached field to keep in sync. Zero for a fresh pool,
+ * which is fully dirty.
+ */
+static inline u64 gpu_buddy_clear_avail(const struct gpu_buddy *mm)
+{
+	gpu_buddy_driver_lock_held(mm);
+	return mm->avail - mm->dirty.total_dirty;
+}
+
 static inline u64
 gpu_buddy_block_offset(const struct gpu_buddy_block *block)
 {
diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h
new file mode 100644
index 00000000000000..d1670121947a8f
--- /dev/null
+++ b/include/linux/hazptr.h
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: LGPL-2.1-or-later
+//
+// SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+
+#ifndef _LINUX_HAZPTR_H
+#define _LINUX_HAZPTR_H
+
+/*
+ * hazptr: Hazard Pointers
+ *
+ * This API provides existence guarantees of objects through hazard
+ * pointers.
+ *
+ * Its main benefit over RCU is that it allows fast reclaim of
+ * HP-protected pointers without needing to wait for a grace period.
+ *
+ * References:
+ *
+ * [1]: M. M. Michael, "Hazard pointers: safe memory reclamation for
+ *      lock-free objects," in IEEE Transactions on Parallel and
+ *      Distributed Systems, vol. 15, no. 6, pp. 491-504, June 2004
+ */
+
+#include <linux/percpu.h>
+#include <linux/types.h>
+#include <linux/cleanup.h>
+#include <linux/sched.h>
+
+/* 4 slots (each sizeof(hazptr_slot_item)) fit in a single 64-byte cache line. */
+#define NR_HAZPTR_PERCPU_SLOTS	4
+
+/* The current hazard pointer wildcard. */
+extern void *hazptr_wildcard;
+
+/*
+ * Hazard pointer slot.
+ */
+struct hazptr_slot {
+	void *addr;
+};
+
+struct hazptr_overflow_list;
+
+struct hazptr_backup_slot {
+	struct hlist_node overflow_node;
+	struct hazptr_slot slot;
+	/* Overflow list where the backup slot is added. */
+	struct hazptr_overflow_list *overflow_list;
+};
+
+struct hazptr_ctx {
+	struct hazptr_slot *slot;
+	/* Backup slot in case all per-CPU slots are used. */
+	struct hazptr_backup_slot backup_slot;
+	struct hlist_node preempt_node;
+#ifdef CONFIG_HAZPTR_DEBUG
+	bool detach_task, detach_cpu;	/* Whether the ctx has been detached from task/cpu. */
+	int acquire_pid, acquire_cpu;	/* Note the task and cpu number at acquire. */
+	unsigned long acquire_caller;	/* Acquire instruction pointer. */
+#endif
+};
+
+struct hazptr_slot_ctx {
+	struct hazptr_ctx *ctx;
+};
+
+struct hazptr_slot_item {
+	struct hazptr_slot slot;
+	struct hazptr_slot_ctx ctx;
+};
+
+struct hazptr_percpu_slots {
+	struct hazptr_slot_item items[NR_HAZPTR_PERCPU_SLOTS];
+} ____cacheline_aligned;
+
+DECLARE_PER_CPU(struct hazptr_percpu_slots, hazptr_percpu_slots);
+
+void *__hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p);
+
+/**
+ * hazptr_synchronize: Wait for release from hazard-pointer protection
+ *
+ * @addr: The address to be released from hazard-pointer protection
+ *
+ * Wait for the specified @addr to be released from protection from all
+ * hazard pointers.  The caller should make @addr inaccessible to all
+ * hazard-pointer readers before invoking this function.
+ *
+ * Must be called from preemptible context.
+ */
+void hazptr_synchronize(void *addr);
+
+/*
+ * hazptr_chain_backup_slot: Chain backup slot into overflow list.
+ *
+ * Set backup slot address to @addr, and chain it into the overflow
+ * list.
+ */
+struct hazptr_slot *hazptr_chain_backup_slot(struct hazptr_ctx *ctx);
+
+/*
+ * hazptr_unchain_backup_slot: Unchain backup slot from overflow list.
+ */
+void hazptr_unchain_backup_slot(struct hazptr_ctx *ctx);
+
+static inline
+bool hazptr_slot_is_backup(struct hazptr_ctx *ctx, struct hazptr_slot *slot)
+{
+	return slot == &ctx->backup_slot.slot;
+}
+
+/* Internal helper. */
+static inline
+void hazptr_promote_to_backup_slot(struct hazptr_ctx *ctx, struct hazptr_slot *slot)
+{
+	struct hazptr_slot *backup_slot;
+
+	backup_slot = hazptr_chain_backup_slot(ctx);
+	/*
+	 * Move hazard pointer from the per-CPU slot to the
+	 * backup slot. This requires hazard pointer
+	 * synchronize to iterate on per-CPU slots with
+	 * load-acquire before iterating on the overflow list.
+	 */
+	WRITE_ONCE(backup_slot->addr, slot->addr);
+	/*
+	 * store-release orders store to backup slot addr before
+	 * store to per-CPU slot addr.
+	 */
+	smp_store_release(&slot->addr, NULL);
+	/* Use the backup slot for context. */
+	ctx->slot = backup_slot;
+}
+
+/**
+ * hazptr_detach - Allow a hazard pointer to be released in some other context
+ *
+ * @ctx: The hazard-pointer context to be detached.
+ *
+ * By default, a given hazptr_acquire() and the corresponding
+ * hazptr_release() must run in a single execution context, for example,
+ * the context of a single task or a single interrupt handler.  When you
+ * have acquired a hazard pointer in one context and need to release it
+ * in another, you must invoke hazptr_detach() on that hazard pointer's
+ * context.  It is permissible to invoke hazptr_detach() multiple times
+ * on the same @ctx while it is protecting the same pointer, however,
+ * the first invocation absolutely must be in the same context that did
+ * the hazptr_acquire(), and must take place after the return from that
+ * hazptr_acquire().
+ *
+ * For example, if a hazard pointer is acquired by a task and released
+ * by a timer handler, that task would need to pass the hazard pointer's
+ * context to hazptr_detach() after return from the hazptr_acquire() and
+ * before arming the timer (or at least before the handler had a chance
+ * to access that hazard-pointer context).
+ */
+static inline
+void hazptr_detach(struct hazptr_ctx *ctx)
+{
+	struct hazptr_slot *slot;
+
+	guard(preempt)();
+	slot = ctx->slot;
+	if (!slot->addr)
+		return;
+#ifdef CONFIG_HAZPTR_DEBUG
+	ctx->detach_task = ctx->detach_cpu = true;
+#endif
+	if (unlikely(hazptr_slot_is_backup(ctx, slot)))
+		return;
+	hazptr_promote_to_backup_slot(ctx, slot);
+}
+
+static inline
+void hazptr_note_context_switch(void)
+{
+	struct hazptr_percpu_slots *percpu_slots = this_cpu_ptr(&hazptr_percpu_slots);
+	unsigned int idx;
+
+	for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) {
+		struct hazptr_slot_item *item = &percpu_slots->items[idx];
+		struct hazptr_slot *slot = &item->slot;
+
+		if (!slot->addr)
+			continue;
+#ifdef CONFIG_HAZPTR_DEBUG
+		item->ctx.ctx->detach_cpu = true;
+#endif
+		hazptr_promote_to_backup_slot(item->ctx.ctx, slot);
+	}
+}
+
+/**
+ * hazptr_acquire - Load pointer at address and protect with hazard pointer.
+ *
+ * @ctx: The hazard-pointer context to be passed to hazptr_release().
+ * @addr_p: Pointer to the pointer that is to be hazard-pointer protected.
+ *
+ * Load @addr_p, and protect the loaded pointer with hazard pointer.
+ * This protection is roughly similar to (but way faster than) that of a
+ * reference counter, and ends with a later call to hazptr_release().
+ *
+ * This protection is unconditional, and has limitations similar to
+ * that of unconditional reference-counter acquisition.  In particular,
+ * although holding a hazard pointer prevents a hazard-pointer-protected
+ * object from being freed, it does not prevent that object from being
+ * removed from a linked data structure, and does not prevent other
+ * hazard-pointer-protected objects referenced by this object from being
+ * both removed and freed.  At which point, invoking hazptr_acquire()
+ * on these dangling pointers would be a bug.  On the other hand, use of
+ * hazptr_acquire() is safe for immortal pointers to objects that do not
+ * themselves contain pointers to hazard-pointer-protected objects.
+ * Other (more complex) use cases are also possible.
+ *
+ * By default, the call to hazptr_release() must be running in the same
+ * execution context as the corresponding hazptr_acquire(), for example,
+ * within the same task or interrupt handler.  When it is necessary to
+ * instead call hazptr_release() from some other context, pass @ctx to
+ * hazptr_detach() in the original context after invoking hazptr_acquire()
+ * but before making the hazard pointer available to that other context.
+ *
+ * It is not permissible to invoke hazptr_acquire() twice on the same @ctx
+ * without an intervening hazptr_release().
+ *
+ * Returns a non-NULL protected address if the loaded pointer is non-NULL.
+ * Returns NULL if the loaded pointer is NULL.
+ *
+ * On success the protected hazptr slot is stored in @ctx->slot.
+ */
+static inline
+void *hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p)
+{
+	struct hazptr_percpu_slots *percpu_slots;
+	struct hazptr_slot_item *slot_item;
+	struct hazptr_slot *slot;
+	void *addr;
+
+	guard(preempt)();
+	percpu_slots = this_cpu_ptr(&hazptr_percpu_slots);
+	slot_item = &percpu_slots->items[0];
+	slot = &slot_item->slot;
+#ifdef CONFIG_HAZPTR_DEBUG
+	ctx->detach_cpu = ctx->detach_task = false;
+	ctx->acquire_pid = current->pid;
+	ctx->acquire_cpu = smp_processor_id();
+	ctx->acquire_caller = _THIS_IP_;
+#endif
+	if (unlikely(slot->addr))
+		return __hazptr_acquire(ctx, addr_p);
+	WRITE_ONCE(slot->addr, READ_ONCE(hazptr_wildcard));	/* Store B */
+
+	/* Memory ordering: Store B before Load A. */
+	smp_mb();
+
+	/*
+	 * Load @addr_p after storing wildcard to the hazard pointer slot.
+	 */
+	addr = READ_ONCE(*addr_p);	/* Load A */
+
+	/*
+	 * We don't care about ordering of Store C. It will simply
+	 * replace the wildcard by a more specific address. If addr is
+	 * NULL, we simply store NULL into the slot.
+	 */
+	WRITE_ONCE(slot->addr, addr);	/* Store C */
+	slot_item->ctx.ctx = ctx;
+	ctx->slot = slot;
+	return addr;
+}
+
+#ifdef CONFIG_HAZPTR_DEBUG
+/* Called with preemption disabled. */
+static inline
+void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr)
+{
+	int pid = current->pid, cpu = smp_processor_id();
+	bool warn_remote_cpu = !ctx->detach_cpu && ctx->acquire_cpu != cpu,
+	     warn_remote_task = !ctx->detach_task && ctx->acquire_pid != pid;
+
+	WARN_ONCE(warn_remote_cpu || warn_remote_task,
+		"Hazard Pointer (addr=%p) released on remote %s without %s. Acquire: caller=%pS, pid=%d, cpu=%d. Release: pid=%d, cpu=%d.",
+		addr,
+		warn_remote_task ? "task" : "cpu",
+		warn_remote_task ? "being detached from task" : "context switch",
+		(void *) ctx->acquire_caller, ctx->acquire_pid, ctx->acquire_cpu, pid, cpu);
+}
+#else
+static inline void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr) { }
+#endif
+
+/**
+ * hazptr_release - Release the specified hazard pointer
+ *
+ * @ctx: The hazard-pointer context that was passed to hazptr_acquire().
+ * @addr_p: The pointer that is to be hazard-pointer unprotected.
+ *
+ * Release the protected hazard pointer recorded in @ctx.
+ *
+ * By default, hazptr_release() must execute in the same execution context
+ * that invoked the corresponding hazptr_acquire(), for example, within the
+ * same task or the same interrupt handler.  However, if this restriction
+ * is problematic for your use case, please see hazptr_detach().
+ *
+ * It is permissible (though unwise from a maintainability viewpoint)
+ * to invoke hazptr_release() twice on the same @ctx without an intervening
+ * hazptr_acquire().
+ */
+static inline
+void hazptr_release(struct hazptr_ctx *ctx, void *addr)
+{
+	struct hazptr_slot *slot;
+
+	if (!addr)
+		return;
+	guard(preempt)();
+	hazptr_release_debug(ctx, addr);
+	slot = ctx->slot;
+	smp_store_release(&slot->addr, NULL);
+	if (unlikely(hazptr_slot_is_backup(ctx, slot)))
+		hazptr_unchain_backup_slot(ctx);
+}
+
+void hazptr_init(void);
+
+#endif /* _LINUX_HAZPTR_H */
diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h
index 8dab78e1f61bd9..b80a5ee63bb234 100644
--- a/include/linux/hdmi.h
+++ b/include/linux/hdmi.h
@@ -27,6 +27,18 @@
 #include <linux/types.h>
 #include <linux/device.h>
 
+enum hdmi_version {
+	HDMI_VERSION_UNKNOWN,
+	HDMI_VERSION_1_0,
+	HDMI_VERSION_1_1,
+	HDMI_VERSION_1_2,
+	HDMI_VERSION_1_3,
+	HDMI_VERSION_1_4,
+	HDMI_VERSION_2_0,
+	HDMI_VERSION_2_1,
+	HDMI_VERSION_2_2,
+};
+
 enum hdmi_packet_type {
 	HDMI_PACKET_TYPE_NULL = 0x00,
 	HDMI_PACKET_TYPE_AUDIO_CLOCK_REGEN = 0x01,
diff --git a/include/linux/hid-sensor-hub.h b/include/linux/hid-sensor-hub.h
index ab5cc8db3fbb3b..5aecf447418398 100644
--- a/include/linux/hid-sensor-hub.h
+++ b/include/linux/hid-sensor-hub.h
@@ -38,6 +38,7 @@ struct hid_sensor_hub_attribute_info {
 /**
  * struct sensor_hub_pending - Synchronous read pending information
  * @status:		Pending status true/false.
+ * @shutdown:		The device is being removed.
  * @ready:		Completion synchronization data.
  * @usage_id:		Usage id for physical device, e.g. gyro usage id.
  * @attr_usage_id:	Usage Id of a field, e.g. X-axis for a gyro.
@@ -48,6 +49,7 @@ struct hid_sensor_hub_attribute_info {
  */
 struct sensor_hub_pending {
 	bool status;
+	bool shutdown;
 	struct completion ready;
 	u32 usage_id;
 	u32 attr_usage_id;
diff --git a/include/linux/hiddev.h b/include/linux/hiddev.h
index 2164c03d2c7224..8e9f8a33e35976 100644
--- a/include/linux/hiddev.h
+++ b/include/linux/hiddev.h
@@ -13,6 +13,7 @@
 #ifndef _HIDDEV_H
 #define _HIDDEV_H
 
+#include <linux/refcount.h>
 #include <uapi/linux/hiddev.h>
 
 
@@ -24,6 +25,7 @@ struct hiddev {
 	int minor;
 	int exist;
 	int open;
+	refcount_t refcount;
 	struct mutex existancelock;
 	wait_queue_head_t wait;
 	struct hid_device *hid;
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 29072d89e5cb38..cad8482337cb7b 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -72,7 +72,7 @@ enum hrtimer_mode {
  */
 struct hrtimer_sleeper {
 	struct hrtimer timer;
-	struct task_struct *task;
+	struct task_struct *__private task;
 };
 
 static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
@@ -320,6 +320,14 @@ extern int schedule_hrtimeout_range_clock(ktime_t *expires,
 					  const enum hrtimer_mode mode,
 					  clockid_t clock_id);
 extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
+static inline struct task_struct *hrtimer_sleeper_task_get(struct hrtimer_sleeper *sl)
+{
+	return READ_ONCE(ACCESS_PRIVATE(sl, task));
+}
+static inline void hrtimer_sleeper_task_set(struct hrtimer_sleeper *sl, struct task_struct *t)
+{
+	WRITE_ONCE(ACCESS_PRIVATE(sl, task), t);
+}
 
 /* Soft interrupt function to run the hrtimer queues: */
 extern void hrtimer_run_queues(void);
diff --git a/include/linux/hrtimer_rearm.h b/include/linux/hrtimer_rearm.h
index 17a81826bd9a9b..12e42721c8e019 100644
--- a/include/linux/hrtimer_rearm.h
+++ b/include/linux/hrtimer_rearm.h
@@ -45,7 +45,7 @@ hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif
 		clear_thread_flag(TIF_HRTIMER_REARM);
 		__hrtimer_rearm_deferred();
 		/* Don't go into the loop if HRTIMER_REARM was the only flag */
-		*tif_work &= ~TIF_HRTIMER_REARM;
+		*tif_work &= ~_TIF_HRTIMER_REARM;
 		return !*tif_work;
 	}
 	return false;
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index c745f7ad22987f..8ca0fa3be2acb1 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -510,8 +510,6 @@ change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
 
 int hugepage_madvise(struct vm_area_struct *vma, vm_flags_t *vm_flags,
 		     int advice);
-int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
-		     unsigned long end, bool *lock_dropped);
 void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start,
 			   unsigned long end, struct vm_area_struct *next);
 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma);
@@ -715,13 +713,6 @@ static inline int hugepage_madvise(struct vm_area_struct *vma,
 	return -EINVAL;
 }
 
-static inline int madvise_collapse(struct vm_area_struct *vma,
-				   unsigned long start,
-				   unsigned long end, bool *lock_dropped)
-{
-	return -EINVAL;
-}
-
 static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
 					 unsigned long start,
 					 unsigned long end,
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 16c4c4caa126ce..5029c72418631b 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -7,11 +7,11 @@
 #include <linux/mm_types.h>
 #include <linux/mmdebug.h>
 #include <linux/fs.h>
-#include <linux/hugetlb_inline.h>
 #include <linux/cgroup.h>
 #include <linux/page_ref.h>
 #include <linux/list.h>
 #include <linux/kref.h>
+#include <linux/atomic.h>
 #include <linux/pgtable.h>
 #include <linux/gfp.h>
 #include <linux/userfaultfd_k.h>
@@ -171,7 +171,6 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio);
 
 extern int movable_gigantic_pages __read_mostly;
 extern int sysctl_hugetlb_shm_group __read_mostly;
-extern struct list_head huge_boot_pages[MAX_NUMNODES];
 
 void hugetlb_bootmem_struct_page_init(void);
 void hugetlb_bootmem_alloc(void);
@@ -253,14 +252,14 @@ extern void __hugetlb_zap_end(struct vm_area_struct *vma,
 static inline void hugetlb_zap_begin(struct vm_area_struct *vma,
 				     unsigned long *start, unsigned long *end)
 {
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		__hugetlb_zap_begin(vma, start, end);
 }
 
 static inline void hugetlb_zap_end(struct vm_area_struct *vma,
 				   struct zap_details *details)
 {
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		__hugetlb_zap_end(vma, details);
 }
 
@@ -509,6 +508,9 @@ struct hugetlbfs_inode_info {
 	struct inode vfs_inode;
 	struct resv_map *resv_map;
 	unsigned int seals;
+#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
+	atomic64_t pmd_sharing_count;
+#endif
 };
 
 static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
@@ -516,6 +518,39 @@ static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
 	return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
 }
 
+#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
+static inline void hugetlbfs_pmd_sharing_init(struct inode *inode)
+{
+	atomic64_set(&HUGETLBFS_I(inode)->pmd_sharing_count, 0);
+}
+
+static inline void hugetlbfs_pmd_sharing_inc(struct inode *inode)
+{
+	atomic64_inc(&HUGETLBFS_I(inode)->pmd_sharing_count);
+}
+
+static inline void hugetlbfs_pmd_sharing_dec(struct inode *inode)
+{
+	atomic64_dec(&HUGETLBFS_I(inode)->pmd_sharing_count);
+}
+
+static inline bool hugetlbfs_pmd_sharing_active(struct inode *inode)
+{
+	return atomic64_read(&HUGETLBFS_I(inode)->pmd_sharing_count) != 0;
+}
+#else
+static inline void hugetlbfs_pmd_sharing_init(struct inode *inode) {}
+
+static inline void hugetlbfs_pmd_sharing_inc(struct inode *inode) {}
+
+static inline void hugetlbfs_pmd_sharing_dec(struct inode *inode) {}
+
+static inline bool hugetlbfs_pmd_sharing_active(struct inode *inode)
+{
+	return false;
+}
+#endif
+
 extern const struct vm_operations_struct hugetlb_vm_ops;
 struct file *hugetlb_file_setup(const char *name, size_t size, vma_flags_t acct,
 				int creat_flags, int page_size_log);
@@ -676,10 +711,6 @@ struct hstate {
 	char name[HSTATE_NAME_LEN];
 };
 
-#define HUGE_BOOTMEM_HVO		0x0001
-#define HUGE_BOOTMEM_ZONES_VALID	0x0002
-#define HUGE_BOOTMEM_CMA		0x0004
-
 int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list);
 int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn);
 void wait_for_freed_hugetlb_folios(void);
@@ -699,7 +730,8 @@ enum hugetlb_alloc_flag {
 #define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT)
 
 struct folio *hugetlb_alloc_folio(struct hstate *h,
-		struct mempolicy_interpreted *mpoli, u8 alloc_flags);
+		struct mempolicy_interpreted *mpoli, struct mm_struct *mm,
+		u8 alloc_flags);
 struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 				unsigned long addr, bool cow_from_owner);
 struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h
deleted file mode 100644
index 5c29cd3223a1e4..00000000000000
--- a/include/linux/hugetlb_inline.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _LINUX_HUGETLB_INLINE_H
-#define _LINUX_HUGETLB_INLINE_H
-
-#include <linux/mm.h>
-
-#ifdef CONFIG_HUGETLB_PAGE
-
-static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
-{
-	return vma_flags_test(flags, VMA_HUGETLB_BIT);
-}
-
-#else
-
-static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
-{
-	return false;
-}
-
-#endif
-
-static inline bool is_vm_hugetlb_page(const struct vm_area_struct *vma)
-{
-	return is_vma_hugetlb_flags(&vma->flags);
-}
-
-#endif
diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h
index dd713e193d0c3a..a3a7d27f3b5ffd 100644
--- a/include/linux/hwmon.h
+++ b/include/linux/hwmon.h
@@ -134,6 +134,8 @@ enum hwmon_in_attributes {
 	hwmon_in_max,
 	hwmon_in_lcrit,
 	hwmon_in_crit,
+	hwmon_in_lemergency,
+	hwmon_in_emergency,
 	hwmon_in_average,
 	hwmon_in_lowest,
 	hwmon_in_highest,
@@ -144,6 +146,8 @@ enum hwmon_in_attributes {
 	hwmon_in_max_alarm,
 	hwmon_in_lcrit_alarm,
 	hwmon_in_crit_alarm,
+	hwmon_in_lemergency_alarm,
+	hwmon_in_emergency_alarm,
 	hwmon_in_rated_min,
 	hwmon_in_rated_max,
 	hwmon_in_beep,
@@ -156,6 +160,8 @@ enum hwmon_in_attributes {
 #define HWMON_I_MAX		BIT(hwmon_in_max)
 #define HWMON_I_LCRIT		BIT(hwmon_in_lcrit)
 #define HWMON_I_CRIT		BIT(hwmon_in_crit)
+#define HWMON_I_LEMERGENCY	BIT(hwmon_in_lemergency)
+#define HWMON_I_EMERGENCY	BIT(hwmon_in_emergency)
 #define HWMON_I_AVERAGE		BIT(hwmon_in_average)
 #define HWMON_I_LOWEST		BIT(hwmon_in_lowest)
 #define HWMON_I_HIGHEST		BIT(hwmon_in_highest)
@@ -166,6 +172,8 @@ enum hwmon_in_attributes {
 #define HWMON_I_MAX_ALARM	BIT(hwmon_in_max_alarm)
 #define HWMON_I_LCRIT_ALARM	BIT(hwmon_in_lcrit_alarm)
 #define HWMON_I_CRIT_ALARM	BIT(hwmon_in_crit_alarm)
+#define HWMON_I_LEMERGENCY_ALARM BIT(hwmon_in_lemergency_alarm)
+#define HWMON_I_EMERGENCY_ALARM	BIT(hwmon_in_emergency_alarm)
 #define HWMON_I_RATED_MIN	BIT(hwmon_in_rated_min)
 #define HWMON_I_RATED_MAX	BIT(hwmon_in_rated_max)
 #define HWMON_I_BEEP		BIT(hwmon_in_beep)
@@ -178,6 +186,7 @@ enum hwmon_curr_attributes {
 	hwmon_curr_max,
 	hwmon_curr_lcrit,
 	hwmon_curr_crit,
+	hwmon_curr_emergency,
 	hwmon_curr_average,
 	hwmon_curr_lowest,
 	hwmon_curr_highest,
@@ -188,6 +197,7 @@ enum hwmon_curr_attributes {
 	hwmon_curr_max_alarm,
 	hwmon_curr_lcrit_alarm,
 	hwmon_curr_crit_alarm,
+	hwmon_curr_emergency_alarm,
 	hwmon_curr_rated_min,
 	hwmon_curr_rated_max,
 	hwmon_curr_beep,
@@ -199,6 +209,7 @@ enum hwmon_curr_attributes {
 #define HWMON_C_MAX		BIT(hwmon_curr_max)
 #define HWMON_C_LCRIT		BIT(hwmon_curr_lcrit)
 #define HWMON_C_CRIT		BIT(hwmon_curr_crit)
+#define HWMON_C_EMERGENCY	BIT(hwmon_curr_emergency)
 #define HWMON_C_AVERAGE		BIT(hwmon_curr_average)
 #define HWMON_C_LOWEST		BIT(hwmon_curr_lowest)
 #define HWMON_C_HIGHEST		BIT(hwmon_curr_highest)
@@ -209,6 +220,7 @@ enum hwmon_curr_attributes {
 #define HWMON_C_MAX_ALARM	BIT(hwmon_curr_max_alarm)
 #define HWMON_C_LCRIT_ALARM	BIT(hwmon_curr_lcrit_alarm)
 #define HWMON_C_CRIT_ALARM	BIT(hwmon_curr_crit_alarm)
+#define HWMON_C_EMERGENCY_ALARM	BIT(hwmon_curr_emergency_alarm)
 #define HWMON_C_RATED_MIN	BIT(hwmon_curr_rated_min)
 #define HWMON_C_RATED_MAX	BIT(hwmon_curr_rated_max)
 #define HWMON_C_BEEP		BIT(hwmon_curr_beep)
diff --git a/include/linux/i2c.h b/include/linux/i2c.h
index 14ab4d3055aff6..6a847fc88d2595 100644
--- a/include/linux/i2c.h
+++ b/include/linux/i2c.h
@@ -174,8 +174,23 @@ i2c_smbus_write_word_swapped(const struct i2c_client *client,
 }
 
 /* Returns the number of read bytes */
-s32 i2c_smbus_read_block_data(const struct i2c_client *client,
-			      u8 command, u8 *values);
+s32 __i2c_smbus_read_block_data(const struct i2c_client *client,
+				u8 command, u8 length, u8 *values);
+/*
+ * This monstrosity allows to call i2c_smbus_read_block_data() with either
+ * 3 or 4 arguments and will be removed once all users have been switched
+ * to the 4 argument version.
+ */
+#define __i2c_smbus_read_block_data_3arg(client, cmd, values)			\
+	__i2c_smbus_read_block_data(client, cmd, I2C_SMBUS_BLOCK_MAX, values)
+#define __i2c_smbus_read_block_data_4arg(client, cmd, length, values)		\
+	__i2c_smbus_read_block_data(client, cmd, length, values)
+#define __i2c_smbus_read_block_data_impl(_1, _2, _3, _4, impl, ...) impl
+#define i2c_smbus_read_block_data(client, cmd, varargs...)			\
+	__i2c_smbus_read_block_data_impl(client, cmd, varargs,			\
+					 __i2c_smbus_read_block_data_4arg,	\
+					 __i2c_smbus_read_block_data_3arg)	\
+					 (client, cmd, varargs)
 s32 i2c_smbus_write_block_data(const struct i2c_client *client,
 			       u8 command, u8 length, const u8 *values);
 /* Returns the number of read bytes */
@@ -742,6 +757,9 @@ struct i2c_adapter {
 	struct rt_mutex mux_lock;
 
 	int timeout;			/* in jiffies */
+#ifdef CONFIG_I2C_DYNAMIC_TIMEOUT
+	int user_timeout;		/* I2C_TIMEOUT ioctl value in jiffies */
+#endif
 	int retries;
 	struct device dev;		/* the adapter device */
 	unsigned long locked_flags;	/* owned by the I2C core */
@@ -913,14 +931,24 @@ unsigned int i2c_adapter_depth(struct i2c_adapter *adapter);
 
 void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults);
 
+#ifdef CONFIG_I2C_DYNAMIC_TIMEOUT
+void i2c_update_timeout(struct i2c_adapter *adap, u32 bus_freq_hz,
+			size_t len, unsigned int safety_coeff,
+			unsigned int min_usec);
+#else
+static inline void i2c_update_timeout(struct i2c_adapter *adap, u32 bus_freq_hz,
+				      size_t len, unsigned int safety_coeff,
+				      unsigned int min_usec) {}
+#endif
+
 /* Return the functionality mask */
 static inline u32 i2c_get_functionality(struct i2c_adapter *adap)
 {
 	return adap->algo->functionality(adap);
 }
 
-/* Return 1 if adapter supports everything we need, 0 if not. */
-static inline int i2c_check_functionality(struct i2c_adapter *adap, u32 func)
+/* Return true if adapter supports everything we need, false if not. */
+static inline bool i2c_check_functionality(struct i2c_adapter *adap, u32 func)
 {
 	return (func & i2c_get_functionality(adap)) == func;
 }
diff --git a/include/linux/i3c/device.h b/include/linux/i3c/device.h
index 0f065b883ee05f..e30ea2cb13cb26 100644
--- a/include/linux/i3c/device.h
+++ b/include/linux/i3c/device.h
@@ -79,6 +79,10 @@ struct i3c_xfer {
 	enum i3c_error_code err;
 };
 
+/* For splitting 'cmd' from struct i3c_xfer */
+#define I3C_HDR_CMD_RNW			BIT(7)
+#define I3C_HDR_CMD_CODE		GENMASK(6, 0)
+
 /**
  * enum i3c_dcr - I3C DCR values
  * @I3C_DCR_GENERIC_DEVICE: generic I3C device
diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h
index f7ceec2b447738..7b9730e7e22f2d 100644
--- a/include/linux/i3c/master.h
+++ b/include/linux/i3c/master.h
@@ -11,6 +11,7 @@
 #include <asm/bitsperlong.h>
 
 #include <linux/bitops.h>
+#include <linux/dma-direction.h>
 #include <linux/i2c.h>
 #include <linux/i3c/ccc.h>
 #include <linux/i3c/device.h>
@@ -536,6 +537,8 @@ struct i3c_master_controller_ops {
  *		 Bit 0: SETDASA
  *		 Bit 1: SETAASA
  *		 All other bits are reserved.
+ * @instance: Zero-based instance number of the Bus Controller as defined by the
+ *            DisCo specification I3C Target Address (_ADR) Encoding
  * @wq: freezable workqueue which can be used by master
  *	drivers if they need to postpone operations that need to take place
  *	in a thread context. Typical examples are Hot Join processing which
@@ -573,6 +576,7 @@ struct i3c_master_controller {
 	} boardinfo;
 	struct i3c_bus bus;
 	u8 addr_method;
+	u8 instance;
 	struct workqueue_struct *wq;
 	struct work_struct hj_work;
 	struct work_struct reg_work;
diff --git a/include/linux/idr.h b/include/linux/idr.h
index 789e23e6744410..e2a4b6298511c7 100644
--- a/include/linux/idr.h
+++ b/include/linux/idr.h
@@ -16,6 +16,7 @@
 #include <linux/gfp.h>
 #include <linux/percpu.h>
 #include <linux/cleanup.h>
+#include <linux/compiler.h>
 
 struct idr {
 	struct radix_tree_root	idr_rt;
@@ -269,7 +270,9 @@ struct ida {
 #define IDA_INIT(name)	{						\
 	.xa = XARRAY_INIT(name, IDA_INIT_FLAGS)				\
 }
-#define DEFINE_IDA(name)	struct ida name = IDA_INIT(name)
+#define DEFINE_IDA(name)							\
+	struct ida name = IDA_INIT(name);					\
+	ASSERT_STATIC_STORAGE(name)
 
 int ida_alloc_range(struct ida *, unsigned int min, unsigned int max, gfp_t);
 void ida_free(struct ida *, unsigned int id);
diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h
index b62297a978e7f4..a699d108eee613 100644
--- a/include/linux/ieee80211-eht.h
+++ b/include/linux/ieee80211-eht.h
@@ -482,6 +482,7 @@ struct ieee80211_multi_link_elem {
 #define IEEE80211_MLC_BASIC_PRES_MLD_ID			0x0200
 #define IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP	0x0400
 #define IEEE80211_MLC_BASIC_PRES_ENH_CRIT_UPD		0x0800
+#define IEEE80211_MLC_BASIC_PRES_AGE_OF_BSS_LOAD	0x1000
 
 #define IEEE80211_MED_SYNC_DELAY_DURATION		0x00ff
 #define IEEE80211_MED_SYNC_DELAY_SYNC_OFDM_ED_THRESH	0x0f00
@@ -550,6 +551,8 @@ struct ieee80211_mle_basic_common_info {
 } __packed;
 
 #define IEEE80211_MLC_PREQ_PRES_MLD_ID			0x0010
+#define IEEE80211_MLC_PREQ_PRES_MLD_MAC_ADDR		0x0020
+#define IEEE80211_MLC_PREQ_PRES_NEIGH_AP_MLD_MAC_ADDR	0x0040
 
 struct ieee80211_mle_preq_common_info {
 	u8 len;
@@ -560,6 +563,7 @@ struct ieee80211_mle_preq_common_info {
 #define IEEE80211_MLC_RECONF_PRES_EML_CAPA		0x0020
 #define IEEE80211_MLC_RECONF_PRES_MLD_CAPA_OP		0x0040
 #define IEEE80211_MLC_RECONF_PRES_EXT_MLD_CAPA_OP	0x0080
+#define IEEE80211_MLC_RECONF_PRES_TARGET_AP_MLD_ADDR	0x0100
 
 /* no fixed fields in RECONF */
 
@@ -766,7 +770,7 @@ static inline u16 ieee80211_mle_get_mld_capa_op(const u8 *data)
 #define IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE		0x0020
 #define IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK	0x0040
 #define IEEE80211_EHT_ML_EXT_MLD_CAPA_BTM_MLD_RECO_MULTI_AP	0x0080
-/* defined by UHR Draft P802.11bn_D1.3 Figure 9-1147 */
+/* defined by UHR Draft P802.11bn_D1.5 Figure 9-1168 */
 #define IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM			0x0100
 
 /**
@@ -928,11 +932,17 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
 			common += 2;
 		if (control & IEEE80211_MLC_BASIC_PRES_ENH_CRIT_UPD)
 			common += 1;
+		if (control & IEEE80211_MLC_BASIC_PRES_AGE_OF_BSS_LOAD)
+			common += 1;
 		break;
 	case IEEE80211_ML_CONTROL_TYPE_PREQ:
 		common += sizeof(struct ieee80211_mle_preq_common_info);
 		if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID)
 			common += 1;
+		if (control & IEEE80211_MLC_PREQ_PRES_MLD_MAC_ADDR)
+			common += ETH_ALEN;
+		if (control & IEEE80211_MLC_PREQ_PRES_NEIGH_AP_MLD_MAC_ADDR)
+			common += ETH_ALEN;
 		break;
 	case IEEE80211_ML_CONTROL_TYPE_RECONF:
 		common += 1;
@@ -944,6 +954,8 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len)
 			common += 2;
 		if (control & IEEE80211_MLC_RECONF_PRES_EXT_MLD_CAPA_OP)
 			common += 2;
+		if (control & IEEE80211_MLC_RECONF_PRES_TARGET_AP_MLD_ADDR)
+			common += ETH_ALEN;
 		break;
 	case IEEE80211_ML_CONTROL_TYPE_TDLS:
 		common += sizeof(struct ieee80211_mle_tdls_common_info);
@@ -1003,6 +1015,7 @@ enum ieee80211_mle_subelems {
 #define IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT	0x0800
 #define IEEE80211_MLE_STA_CONTROL_ENH_CRIT_UPD_PRESENT		0x1000
 #define IEEE80211_MLE_STA_CONTROL_AP_CONDUCTED_TX_PWR_PRESENT	0x2000
+#define IEEE80211_MLE_STA_CONTROL_AGE_OF_BSS_LOAD_PRESENT	0x4000
 
 struct ieee80211_mle_per_sta_profile {
 	__le16 control;
@@ -1054,6 +1067,9 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data,
 	if (control & IEEE80211_MLE_STA_CONTROL_AP_CONDUCTED_TX_PWR_PRESENT)
 		info_len += 1;
 
+	if (control & IEEE80211_MLE_STA_CONTROL_AGE_OF_BSS_LOAD_PRESENT)
+		info_len += 1;
+
 	return prof->sta_info_len >= info_len &&
 	       fixed + prof->sta_info_len - 1 <= len;
 }
diff --git a/include/linux/ieee80211-uhr.h b/include/linux/ieee80211-uhr.h
index 665d4b3a5b4186..19e9260d0db80d 100644
--- a/include/linux/ieee80211-uhr.h
+++ b/include/linux/ieee80211-uhr.h
@@ -17,32 +17,32 @@
 #define IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA		0x0004
 #define IEEE80211_UHR_OPER_PARAMS_DBE_ENA		0x0008
 #define IEEE80211_UHR_OPER_PARAMS_DBE_BW		0x0070
-#define IEEE80211_UHR_OPER_PARAMS_DUO_PRES		0x0080
-#define IEEE80211_UHR_OPER_PARAMS_DPS_PRES		0x0100
-#define IEEE80211_UHR_OPER_PARAMS_NPCA_PRES		0x0200
-#define IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES		0x0400
-#define IEEE80211_UHR_OPER_PARAMS_DBE_PRES		0x0800
+#define IEEE80211_UHR_OPER_PARAMS_ELR_RX		0x0080
+#define IEEE80211_UHR_OPER_PARAMS_DUO_PRES		0x0100
+#define IEEE80211_UHR_OPER_PARAMS_DPS_PRES		0x0200
+#define IEEE80211_UHR_OPER_PARAMS_NPCA_PRES		0x0400
+#define IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES		0x0800
+#define IEEE80211_UHR_OPER_PARAMS_DBE_PRES		0x1000
 
 struct ieee80211_uhr_operation {
 	__le16 params;
-	u8 basic_mcs_nss_set[4];
 	u8 variable[];
 } __packed;
 
-#define IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN_OFFS	0x0000000F
-#define IEEE80211_UHR_NPCA_PARAMS_MIN_DUR_THRESH	0x000000F0
-#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_DELAY		0x00003F00
-#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_BACK_DELAY	0x000FC000
-#define IEEE80211_UHR_NPCA_PARAMS_INIT_QSRC		0x00300000
-#define IEEE80211_UHR_NPCA_PARAMS_MOPLEN		0x00400000
-#define IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES	0x00800000
+#define IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN		0x000000FF
+#define IEEE80211_UHR_NPCA_PARAMS_MIN_DUR_THRESH	0x00000F00
+#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_DELAY		0x0003F000
+#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_BACK_DELAY	0x00FC0000
+#define IEEE80211_UHR_NPCA_PARAMS_INIT_QSRC		0x03000000
+#define IEEE80211_UHR_NPCA_PARAMS_MOPLEN		0x04000000
+#define IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES	0x08000000
 
 /**
  * struct ieee80211_uhr_npca_info - npca operation information
  *
  * This structure is the "NPCA Operation Parameters field format" of "UHR
- * Operation Element" fields as described in P802.11bn_D1.3
- * subclause 9.4.2.353. See Figure 9-aa4.
+ * Operation Element" fields as described in P802.11bn_D1.5
+ * subclause 9.4.2.355.2, see Figure 9-aa6.
  *
  * Refer to IEEE80211_UHR_NPCA*
  * @params:
@@ -97,7 +97,7 @@ struct ieee80211_uhr_npca_info {
  * struct ieee80211_uhr_dps_info - DPS operation information
  *
  * This structure is the "DPS Operation Parameter field" of "UHR
- * Operation Element" fields as described in P802.11bn_D1.3
+ * Operation Element" fields as described in P802.11bn_D1.5
  * subclause 9.4.1.87. See Figure 9-207u.
  *
  * Refer to IEEE80211_UHR_DPS*
@@ -211,8 +211,8 @@ static inline int ieee80211_uhr_dbe_bw_mhz(enum ieee80211_uhr_dbe_oper_bw bw)
  * struct ieee80211_uhr_dbe_info - DBE operation information
  *
  * This structure is the "DBE Operation Parameters field" of
- * "UHR Operation Element" fields as described in P802.11bn_D1.3
- * subclause 9.4.2.353. See Figure 9-aa6.
+ * "UHR Operation Element" fields as described in P802.11bn_D1.5
+ * subclause 9.4.2.355.4, see Figure 9-aa9.
  *
  * Refer to IEEE80211_UHR_DBE_OPER*
  * @params:
@@ -234,25 +234,23 @@ struct ieee80211_uhr_dbe_info {
 	__le16 dis_subch_bmap[];
 } __packed;
 
-#define IEEE80211_UHR_P_EDCA_ECWMIN		0x0F
-#define IEEE80211_UHR_P_EDCA_ECWMAX		0xF0
-#define IEEE80211_UHR_P_EDCA_AIFSN		0x000F
-#define IEEE80211_UHR_P_EDCA_CW_DS		0x0030
-#define IEEE80211_UHR_P_EDCA_PSRC_THRESHOLD	0x01C0
-#define IEEE80211_UHR_P_EDCA_QSRC_THRESHOLD	0x0600
+#define IEEE80211_UHR_P_EDCA_ECWMIN		0x0007
+#define IEEE80211_UHR_P_EDCA_ECWMAX		0x0038
+#define IEEE80211_UHR_P_EDCA_AIFSN		0x01C0
+#define IEEE80211_UHR_P_EDCA_CW_DS		0x0600
+#define IEEE80211_UHR_P_EDCA_PSRC_THRESHOLD	0x3800
+#define IEEE80211_UHR_P_EDCA_QSRC_THRESHOLD	0xC000
 
 /**
  * struct ieee80211_uhr_p_edca_info - P-EDCA operation information
  *
  * This structure is the "P-EDCA Operation Parameters field" of
- * "UHR Operation Element" fields as described in P802.11bn_D1.3
- * subclause 9.4.2.353. See Figure 9-aa5.
+ * "UHR Operation Element" fields as described in P802.11bn_D1.5
+ * subclause 9.4.2.355.3, see Figure 9-aa8.
  *
  * Refer to IEEE80211_UHR_P_EDCA*
- * @p_edca_ec: P-EDCA ECWmin and ECWmax.
- *	These fields indicate the CWmin and CWmax values used by a
- *	P-EDCA STA during P-EDCA contention.
- * @params: AIFSN, CW DS, PSRC threshold, and QSRC threshold.
+ * @params: ECWmin, ECWmax, AIFSN, CW DS, PSRC threshold, and QSRC threshold.
+ *	- CWmin/CWmax values used by a P-EDCA STA during P-EDCA contention.
  *	- The AIFSN field indicates the AIFSN value used by a P-EDCA STA
  *	  during P-EDCA contention.
  *	- The CW DS field indicates the value used for randomization of the
@@ -266,7 +264,6 @@ struct ieee80211_uhr_dbe_info {
  *	  value 0 is reserved.
  */
 struct ieee80211_uhr_p_edca_info {
-	u8 p_edca_ec;
 	__le16 params;
 } __packed;
 
@@ -302,7 +299,7 @@ static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len)
 		}
 	}
 
-	/* P-EDCA Operation Parameters (fixed 3 bytes) */
+	/* P-EDCA Operation Parameters */
 	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES)) {
 		needed += sizeof(struct ieee80211_uhr_p_edca_info);
 		if (len < needed)
@@ -341,6 +338,9 @@ ieee80211_uhr_npca_info(const struct ieee80211_uhr_operation *oper)
 	if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA)))
 		return NULL;
 
+	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DUO_PRES))
+		pos += 1;
+
 	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_PRES))
 		pos += sizeof(struct ieee80211_uhr_dps_info);
 
@@ -372,6 +372,9 @@ ieee80211_uhr_oper_dbe_info(const struct ieee80211_uhr_operation *oper)
 	if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DBE_ENA)))
 		return NULL;
 
+	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DUO_PRES))
+		pos += 1;
+
 	if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_PRES))
 		pos += sizeof(struct ieee80211_uhr_dps_info);
 
@@ -415,28 +418,42 @@ ieee80211_uhr_oper_dbe_info(const struct ieee80211_uhr_operation *oper)
 #define IEEE80211_UHR_MAC_CAP2_UHR_OM_PU_TO_LOW		0xC0
 
 #define IEEE80211_UHR_MAC_CAP3_UHR_OM_PU_TO_HIGH	0x03
-#define IEEE80211_UHR_MAC_CAP3_PARAM_UPD_ADV_NOTIF_INTV	0x1C
-#define IEEE80211_UHR_MAC_CAP3_UPD_IND_TIM_INTV_LOW	0xE0
+#define IEEE80211_UHR_MAC_CAP3_PARAM_UPD_ADV_NOTIF_INTV	0x7C
+#define IEEE80211_UHR_MAC_CAP3_UPD_IND_TIM_INTV_LOW	0x80
 
-#define IEEE80211_UHR_MAC_CAP4_UPD_IND_TIM_INTV_HIGH	0x03
-#define IEEE80211_UHR_MAC_CAP4_BOUNDED_ESS		0x04
-#define IEEE80211_UHR_MAC_CAP4_BTM_ASSURANCE		0x08
-#define IEEE80211_UHR_MAC_CAP4_CO_BF_SUPP		0x10
+#define IEEE80211_UHR_MAC_CAP4_UPD_IND_TIM_INTV_HIGH	0x0F
+#define IEEE80211_UHR_MAC_CAP4_BOUNDED_ESS		0x10
+#define IEEE80211_UHR_MAC_CAP4_BTM_ASSURANCE		0x20
+#define IEEE80211_UHR_MAC_CAP4_CO_BF_SUPP		0x40
+#define IEEE80211_UHR_MAC_CAP4_CO_SR_SUPP		0x80
+
+#define IEEE80211_UHR_MAC_CAP5_MAPC_ENH_MSRMT_SUPP	0x01
 
 #define IEEE80211_UHR_MAC_CAP_DBE_MAX_BW		0x07
 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES	0x08
 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES	0x10
 
+/* struct ieee80211_uhr_cap_dbe::bwcap[].cap */
+#define IEEE80211_UHR_MAC_CAP_DBE_CAP_NUM_SND_DIMS		0x07
+#define IEEE80211_UHR_MAC_CAP_DBE_CAP_NON_OFDMA_UL_MUMIMO	0x08
+#define IEEE80211_UHR_MAC_CAP_DBE_CAP_MU_BEAMFORMER		0x10
+#define IEEE80211_UHR_MAC_CAP_DBE_CAP_BEAMFORMEE_SS		0xe0
+
 struct ieee80211_uhr_cap_dbe {
 	u8 cap;
+	u8 max_switch_time_period;
+	u8 mode_change_intvl;
 	/* present 0, 1 or 2 times depending on _PRES bits */
-	struct ieee80211_eht_mcs_nss_supp_bw eht_mcs_map[];
+	struct {
+		struct ieee80211_eht_mcs_nss_supp_bw eht_mcs_map;
+		u8 cap;
+	} __packed bwcap[];
 } __packed;
 
 /**
  * enum ieee80211_uhr_dbe_max_supported_bw - DBE Maximum Supported Bandwidth
  *
- * As per spec P802.11bn_D1.3 "Table 9-bb5—Encoding of the DBE Maximum
+ * As per spec P802.11bn_D1.5 Table 9-bb8 "Encoding of the DBE Maximum
  * Supported Bandwidth field".
  *
  * @IEEE80211_UHR_DBE_MAX_BW_40: Indicates 40 MHz DBE max supported bw
@@ -520,10 +537,10 @@ static inline bool ieee80211_uhr_capa_size_ok(const u8 *data, u8 len,
 		dbe = (const void *)cap->variable;
 
 		if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES)
-			needed += sizeof(dbe->eht_mcs_map[0]);
+			needed += sizeof(dbe->bwcap[0]);
 
 		if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES)
-			needed += sizeof(dbe->eht_mcs_map[0]);
+			needed += sizeof(dbe->bwcap[0]);
 	}
 
 	return len >= needed;
diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h
index 26e67403886542..73ac5336a17ad5 100644
--- a/include/linux/ieee80211.h
+++ b/include/linux/ieee80211.h
@@ -1828,6 +1828,7 @@ enum ieee80211_eid_ext {
 	WLAN_EID_EXT_BANDWIDTH_INDICATION = 135,
 	WLAN_EID_EXT_KNOWN_STA_IDENTIFCATION = 136,
 	WLAN_EID_EXT_NON_AP_STA_REG_CON = 137,
+	WLAN_EID_EXT_CIP_CAPA = 150,
 	WLAN_EID_EXT_UHR_OPER = 151,
 	WLAN_EID_EXT_UHR_CAPA = 152,
 	WLAN_EID_EXT_MACP = 153,
@@ -2421,8 +2422,7 @@ static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb)
 
 	/* action frame - additionally check for non-bufferable FTM */
 
-	if (mgmt->u.action.category != WLAN_CATEGORY_PUBLIC &&
-	    mgmt->u.action.category != WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION)
+	if (mgmt->u.action.category != WLAN_CATEGORY_PUBLIC)
 		return true;
 
 	if (mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_REQUEST ||
@@ -2900,4 +2900,25 @@ static inline bool ieee80211_check_tim(const struct ieee80211_tim_ie *tim,
 		     __ieee80211_check_tim(tim, tim_len, aid);
 }
 
+/**
+ * enum ieee80211_cip_cap_fields - CIP Capabilities element fields
+ * @IEEE80211_CIP_CAP_MIC_PADDING: The MIC padding subfield in the CIP
+ *	capabilities
+ * @IEEE80211_CIP_CAP_PROTECTED_CTRL_FRAME_ONLY: The MIC Padding For Protected
+ *	Control Frames Only bit in CIP capabilities
+ */
+enum ieee80211_cip_cap_fields {
+	IEEE80211_CIP_CAP_MIC_PADDING			= 0x0F,
+	IEEE80211_CIP_CAP_PROTECTED_CTRL_FRAME_ONLY	= 0x10,
+};
+
+/**
+ * struct ieee80211_cip_cap - CIP Capabilities element
+ *
+ * @v: The value of the Control Integrity Protocol Capability element
+ */
+struct ieee80211_cip_cap {
+	u8 v;
+} __packed;
+
 #endif /* LINUX_IEEE80211_H */
diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h
index 20cc16ea4e5abc..4846032bf4ffc8 100644
--- a/include/linux/if_vlan.h
+++ b/include/linux/if_vlan.h
@@ -365,6 +365,9 @@ static inline int __vlan_insert_inner_tag(struct sk_buff *skb,
 	const u8 meta_len = mac_len > ETH_TLEN ? skb_metadata_len(skb) : 0;
 	struct vlan_ethhdr *veth;
 
+	if (unlikely(!pskb_may_pull(skb, mac_len)))
+		return -EINVAL;
+
 	if (skb_cow_head(skb, meta_len + VLAN_HLEN) < 0)
 		return -ENOMEM;
 
diff --git a/include/linux/igmp.h b/include/linux/igmp.h
index a0cf0398519fd7..e075611344ef3b 100644
--- a/include/linux/igmp.h
+++ b/include/linux/igmp.h
@@ -14,6 +14,7 @@
 #include <linux/timer.h>
 #include <linux/in.h>
 #include <linux/ip.h>
+#include <linux/net.h>
 #include <linux/refcount.h>
 #include <linux/sockptr.h>
 #include <uapi/linux/igmp.h>
@@ -273,7 +274,7 @@ extern int ip_mc_source(int add, int omode, struct sock *sk,
 		struct ip_mreq_source *mreqs, int ifindex);
 extern int ip_mc_msfilter(struct sock *sk, struct ip_msfilter *msf,int ifindex);
 extern int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
-			sockptr_t optval, sockptr_t optlen);
+			sockopt_t *opt);
 extern int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf,
 			sockptr_t optval, size_t offset);
 extern int ip_mc_sf_allow(const struct sock *sk, __be32 local, __be32 rmt,
diff --git a/include/linux/iio/adc/qcom-adc5-gen3-common.h b/include/linux/iio/adc/qcom-adc5-gen3-common.h
index 39cbfcbdb101c3..b3e2c2f50e8f51 100644
--- a/include/linux/iio/adc/qcom-adc5-gen3-common.h
+++ b/include/linux/iio/adc/qcom-adc5-gen3-common.h
@@ -137,7 +137,7 @@ struct adc5_sdam_data {
  */
 struct adc5_device_data {
 	struct regmap *regmap;
-	struct adc5_sdam_data *base;
+	struct adc5_sdam_data *base __counted_by_ptr(num_sdams);
 	int num_sdams;
 };
 
diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h
index 3f95ed1fdf9ebb..9c16b695a602f8 100644
--- a/include/linux/iio/backend.h
+++ b/include/linux/iio/backend.h
@@ -133,6 +133,8 @@ enum iio_backend_capabilities {
  * @filter_type_set: Set filter type.
  * @interface_data_align: Perform the data alignment process.
  * @num_lanes_set: Set the number of lanes enabled.
+ * @crc_enable: Enable direction-specific CRC handling.
+ * @crc_disable: Disable direction-specific CRC handling.
  * @ddr_enable: Enable interface DDR (Double Data Rate) mode.
  * @ddr_disable: Disable interface DDR (Double Data Rate) mode.
  * @data_stream_enable: Enable data stream.
@@ -189,6 +191,8 @@ struct iio_backend_ops {
 			       enum iio_backend_filter_type type);
 	int (*interface_data_align)(struct iio_backend *back, u32 timeout_us);
 	int (*num_lanes_set)(struct iio_backend *back, unsigned int num_lanes);
+	int (*crc_enable)(struct iio_backend *back);
+	int (*crc_disable)(struct iio_backend *back);
 	int (*ddr_enable)(struct iio_backend *back);
 	int (*ddr_disable)(struct iio_backend *back);
 	int (*data_stream_enable)(struct iio_backend *back);
@@ -237,6 +241,8 @@ int iio_backend_filter_type_set(struct iio_backend *back,
 				enum iio_backend_filter_type type);
 int iio_backend_interface_data_align(struct iio_backend *back, u32 timeout_us);
 int iio_backend_num_lanes_set(struct iio_backend *back, unsigned int num_lanes);
+int iio_backend_crc_enable(struct iio_backend *back);
+int iio_backend_crc_disable(struct iio_backend *back);
 int iio_backend_ddr_enable(struct iio_backend *back);
 int iio_backend_ddr_disable(struct iio_backend *back);
 int iio_backend_data_stream_enable(struct iio_backend *back);
diff --git a/include/linux/iio/iio-gts-helper.h b/include/linux/iio/iio-gts-helper.h
index 66f830ab9b49b5..0c4b2e9fbe1e36 100644
--- a/include/linux/iio/iio-gts-helper.h
+++ b/include/linux/iio/iio-gts-helper.h
@@ -7,6 +7,7 @@
 #ifndef __IIO_GTS_HELPER__
 #define __IIO_GTS_HELPER__
 
+#include <linux/errno.h>
 #include <linux/types.h>
 
 struct device;
@@ -58,14 +59,14 @@ struct iio_itime_sel_mul {
 
 struct iio_gts {
 	u64 max_scale;
-	const struct iio_gain_sel_pair *hwgain_table;
+	const struct iio_gain_sel_pair *hwgain_table __counted_by_ptr(num_hwgain);
 	int num_hwgain;
-	const struct iio_itime_sel_mul *itime_table;
+	const struct iio_itime_sel_mul *itime_table __counted_by_ptr(num_itime);
 	int num_itime;
-	int **per_time_avail_scale_tables;
-	int *avail_all_scales_table;
+	int **per_time_avail_scale_tables __counted_by_ptr(num_itime);
+	int *avail_all_scales_table __counted_by_ptr(num_avail_all_scales);
 	int num_avail_all_scales;
-	int *avail_time_tables;
+	int *avail_time_tables __counted_by_ptr(num_avail_time_tables);
 	int num_avail_time_tables;
 };
 
diff --git a/include/linux/iio/iio-opaque.h b/include/linux/iio/iio-opaque.h
index b87841a355f802..98330385e08d2f 100644
--- a/include/linux/iio/iio-opaque.h
+++ b/include/linux/iio/iio-opaque.h
@@ -73,7 +73,7 @@ struct iio_dev_opaque {
 #if defined(CONFIG_DEBUG_FS)
 	struct dentry			*debugfs_dentry;
 	unsigned int			cached_reg_addr;
-	char				read_buf[20];
+	char				read_buf[24];
 	unsigned int			read_buf_len;
 #endif
 };
diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h
index 711c00f6737135..9b63785d5aec77 100644
--- a/include/linux/iio/iio.h
+++ b/include/linux/iio/iio.h
@@ -264,6 +264,10 @@ struct iio_scan_type {
  * @ext_info:		Array of extended info attributes for this channel.
  *			The array is NULL terminated, the last element should
  *			have its name field set to NULL.
+ * @parent:		Optional pointer to the parent channel spec for
+ *			hierarchical channel relationships. When set, a read-only
+ *			"parent" sysfs attribute is created containing the
+ *			parent channel's sysfs name prefix (e.g. "in_voltage0").
  * @extend_name:	Allows labeling of channel attributes with an
  *			informative name. Note this has no effect codes etc,
  *			unlike modifiers.
@@ -309,6 +313,7 @@ struct iio_chan_spec {
 	const struct iio_event_spec *event_spec;
 	unsigned int		num_event_specs;
 	const struct iio_chan_spec_ext_info *ext_info;
+	const struct iio_chan_spec *parent;
 	const char		*extend_name;
 	const char		*datasheet_name;
 	unsigned int		modified:1;
@@ -484,6 +489,10 @@ struct iio_trigger; /* forward declaration */
  * @update_scan_mode:	function to configure device and scan buffer when
  *			channels have changed
  * @debugfs_reg_access:	function to read or write register value of device
+ * @debugfs_reg64_access: function to read or write 64-bit register value of
+ *			device. debugfs_reg_access() and debugfs_reg64_access()
+ *			are mutually exclusive, but the 64-bit variant has
+ *			precedence in case both are implemented.
  * @fwnode_xlate:	fwnode based function pointer to obtain channel specifier index.
  * @hwfifo_set_watermark: function pointer to set the current hardware
  *			fifo watermark level; see hwfifo_* entries in
@@ -572,6 +581,9 @@ struct iio_info {
 	int (*debugfs_reg_access)(struct iio_dev *indio_dev,
 				  unsigned int reg, unsigned int writeval,
 				  unsigned int *readval);
+	int (*debugfs_reg64_access)(struct iio_dev *indio_dev,
+				    unsigned int reg, u64 writeval,
+				    u64 *readval);
 	int (*fwnode_xlate)(struct iio_dev *indio_dev,
 			    const struct fwnode_reference_args *iiospec);
 	int (*hwfifo_set_watermark)(struct iio_dev *indio_dev, unsigned int val);
@@ -648,7 +660,7 @@ struct iio_dev {
 	struct iio_poll_func		*pollfunc;
 	struct iio_poll_func		*pollfunc_event;
 
-	struct iio_chan_spec const	*channels;
+	struct iio_chan_spec const	*channels __counted_by_ptr(num_channels);
 	int				num_channels;
 
 	const char			*name;
@@ -789,7 +801,7 @@ DEFINE_GUARD_COND(__priv__iio_dev_mode_lock, _try_direct,
  *
  * Tries to acquire the direct mode lock with cleanup ACQUIRE() semantics and
  * automatically releases it at the end of the scope. It most be always paired
- * with IIO_DEV_ACQUIRE_ERR(), for example (notice the scope braces)::
+ * with IIO_DEV_ACQUIRE_FAILED(), for example (notice the scope braces)::
  *
  *	switch() {
  *	case IIO_CHAN_INFO_RAW: {
@@ -1083,6 +1095,15 @@ static inline unsigned int iio_get_masklength(const struct iio_dev *indio_dev)
 	return ACCESS_PRIVATE(indio_dev, masklength);
 }
 
+/**
+ * iio_scan_timestamp_enabled - Is the timestamp channel in the current scan
+ * @indio_dev: the IIO device to check
+ */
+static inline bool iio_scan_timestamp_enabled(const struct iio_dev *indio_dev)
+{
+	return ACCESS_PRIVATE(indio_dev, scan_timestamp);
+}
+
 int iio_active_scan_mask_index(struct iio_dev *indio_dev);
 
 /**
diff --git a/include/linux/input/samsung-keypad.h b/include/linux/input/samsung-keypad.h
deleted file mode 100644
index ab6b97114c089f..00000000000000
--- a/include/linux/input/samsung-keypad.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Samsung Keypad platform data definitions
- *
- * Copyright (C) 2010 Samsung Electronics Co.Ltd
- * Author: Joonyoung Shim <jy0922.shim@samsung.com>
- */
-
-#ifndef __SAMSUNG_KEYPAD_H
-#define __SAMSUNG_KEYPAD_H
-
-#include <linux/input/matrix_keypad.h>
-
-#define SAMSUNG_MAX_ROWS	8
-#define SAMSUNG_MAX_COLS	8
-
-/**
- * struct samsung_keypad_platdata - Platform device data for Samsung Keypad.
- * @keymap_data: pointer to &matrix_keymap_data.
- * @rows: number of keypad row supported.
- * @cols: number of keypad col supported.
- * @no_autorepeat: disable key autorepeat.
- * @wakeup: controls whether the device should be set up as wakeup source.
- * @cfg_gpio: configure the GPIO.
- *
- * Initialisation data specific to either the machine or the platform
- * for the device driver to use or call-back when configuring gpio.
- */
-struct samsung_keypad_platdata {
-	const struct matrix_keymap_data	*keymap_data;
-	unsigned int rows;
-	unsigned int cols;
-	bool no_autorepeat;
-	bool wakeup;
-
-	void (*cfg_gpio)(unsigned int rows, unsigned int cols);
-};
-
-#endif /* __SAMSUNG_KEYPAD_H */
diff --git a/include/linux/intel_vsec.h b/include/linux/intel_vsec.h
index 843cda8f864443..917d9397a99349 100644
--- a/include/linux/intel_vsec.h
+++ b/include/linux/intel_vsec.h
@@ -90,13 +90,25 @@ enum intel_vsec_quirks {
  * @read_telem: when specified, called by client driver to access PMT
  * data (instead of direct copy).
  * * dev:   device reference for the callback's use
- * * guid:  ID of data to acccss
+ * * guid:  ID of data to access
  * * data:  buffer for the data to be copied
  * * off:   offset into the requested buffer
  * * count: size of buffer
+ * @read_reg: when specified called by client driver to read PMT state
+ * * dev:      device reference for the callback's use
+ * * guid:     ID of data to access
+ * * reg_data: register data
+ * * offset:   offset of register to read
+ * @write_reg: when specified called by client driver to write PMT state
+ * * dev:      device reference for the callback's use
+ * * guid:     ID of data to access
+ * * reg_data: register data
+ * * offset:   offset of register to write
  */
 struct pmt_callbacks {
 	int (*read_telem)(struct device *dev, u32 guid, u64 *data, loff_t off, u32 count);
+	int (*read_reg)(struct device *dev, u32 guid, u32 *reg_data, u32 offset);
+	int (*write_reg)(struct device *dev, u32 guid, u32 reg_data, u32 offset);
 };
 
 struct vsec_feature_dependency {
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 3bf969ad8fe078..52bb684090c0da 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -573,9 +573,11 @@ enum
  * _ IRQ_POLL: irq_poll_cpu_dead() migrates the queue
  *
  * _ (HR)TIMER_SOFTIRQ: (hr)timers_dead_cpu() migrates the queue
+ *
+ * _ BLOCK_SOFTIRQ: blk_softirq_cpu_dead() completes the remaining requests
  */
-#define SOFTIRQ_HOTPLUG_SAFE_MASK (BIT(TIMER_SOFTIRQ) | BIT(IRQ_POLL_SOFTIRQ) |\
-				   BIT(HRTIMER_SOFTIRQ) | BIT(RCU_SOFTIRQ))
+#define SOFTIRQ_HOTPLUG_SAFE_MASK (BIT(TIMER_SOFTIRQ) | BIT(BLOCK_SOFTIRQ) |\
+				   BIT(IRQ_POLL_SOFTIRQ) | BIT(HRTIMER_SOFTIRQ) | BIT(RCU_SOFTIRQ))
 
 
 /* map softirq index to softirq name. update 'softirq_to_name' in
diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h
index d1aa4edfc2a59a..505caa99c62186 100644
--- a/include/linux/io_uring.h
+++ b/include/linux/io_uring.h
@@ -2,10 +2,42 @@
 #ifndef _LINUX_IO_URING_H
 #define _LINUX_IO_URING_H
 
+#include <linux/io_uring_types.h>
 #include <linux/sched.h>
 #include <linux/xarray.h>
 #include <uapi/linux/io_uring.h>
 
+static inline void req_set_fail(struct io_kiocb *req)
+{
+	req->flags |= REQ_F_FAIL;
+	if (req->flags & REQ_F_CQE_SKIP) {
+		req->flags &= ~REQ_F_CQE_SKIP;
+		req->flags |= REQ_F_SKIP_LINK_CQES;
+	}
+}
+
+static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
+{
+	req->cqe.res = res;
+	req->cqe.flags = cflags;
+}
+
+static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx)
+{
+	if (ctx->flags & IORING_SETUP_CQE_MIXED)
+		return IORING_CQE_F_32;
+	return 0;
+}
+
+static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags,
+				    __u64 extra1, __u64 extra2)
+{
+	req->cqe.res = res;
+	req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx);
+	req->big_cqe.extra1 = extra1;
+	req->big_cqe.extra2 = extra2;
+}
+
 #if defined(CONFIG_IO_URING)
 void __io_uring_cancel(bool cancel_all);
 void __io_uring_free(struct task_struct *tsk);
diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h
index 42801f0b6456e1..e18d4af62e1a2e 100644
--- a/include/linux/io_uring/cmd.h
+++ b/include/linux/io_uring/cmd.h
@@ -3,6 +3,7 @@
 #define _LINUX_IO_URING_CMD_H
 
 #include <uapi/linux/io_uring.h>
+#include <linux/io_uring.h>
 #include <linux/io_uring_types.h>
 #include <linux/blk-mq.h>
 
@@ -41,6 +42,24 @@ static inline void io_uring_cmd_private_sz_check(size_t cmd_sz)
 	((pdu_type *)&(cmd)->pdu) \
 )
 
+static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
+{
+	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+	if (ret < 0)
+		req_set_fail(req);
+	io_req_set_res(req, ret, 0);
+}
+
+static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2)
+{
+	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+	if (ret < 0)
+		req_set_fail(req);
+	io_req_set_res32(req, ret, 0, res2, 0);
+}
+
 #if defined(CONFIG_IO_URING)
 int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
 			      struct iov_iter *iter,
@@ -56,11 +75,12 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
+ * io_uring_cmd_set_res()/io_uring_cmd_set_res32() must be called first.
+ *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-			 unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);
 
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 			    io_req_tw_func_t task_work_cb,
@@ -116,8 +136,8 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 {
 	return -EOPNOTSUPP;
 }
-static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
-		u64 ret2, unsigned issue_flags, bool is_cqe32)
+static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
+				       unsigned issue_flags)
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
@@ -205,13 +225,15 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)
 static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret,
 				     unsigned issue_flags)
 {
-	return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false);
+	io_uring_cmd_set_res(ioucmd, ret);
+	__io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret,
 				       u64 res2, unsigned issue_flags)
 {
-	return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
+	io_uring_cmd_set_res32(ioucmd, ret, res2);
+	__io_uring_cmd_done(ioucmd, issue_flags);
 }
 
 #endif /* _LINUX_IO_URING_CMD_H */
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 39629ee77b911c..90fea94ad20293 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -522,6 +522,8 @@ struct io_ring_ctx {
 
 	/* protected by ->completion_lock */
 	unsigned			nr_req_allocated;
+	/* pending SEND_ZC notifications, protected by ->uring_lock */
+	unsigned			nr_notifs;
 
 #ifdef CONFIG_NET_RX_BUSY_POLL
 	struct list_head	napi_list;	/* track busy poll napi_id */
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index bc7ae6327dbf87..59718f73c15a03 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -483,13 +483,35 @@ sector_t iomap_bmap(struct address_space *mapping, sector_t bno,
 #define IOMAP_IOEND_BOUNDARY		(1U << 2)
 /* is direct I/O */
 #define IOMAP_IOEND_DIRECT		(1U << 3)
+/* generate integrity (PI) information */
+#ifdef CONFIG_BLK_DEV_INTEGRITY
+#define IOMAP_IOEND_INTEGRITY		(1U << 4)
+#else
+#define IOMAP_IOEND_INTEGRITY		0
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
 
 /*
  * Flags that if set on either ioend prevent the merge of two ioends.
  * (IOMAP_IOEND_BOUNDARY also prevents merges, but only one-way)
  */
 #define IOMAP_IOEND_NOMERGE_FLAGS \
-	(IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT)
+	(IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT | \
+	 IOMAP_IOEND_INTEGRITY)
+
+/* ioend flags directly implied by iomap flags */
+static inline u16 iomap_ioend_flags(const struct iomap *iomap)
+{
+	unsigned int flags = 0;
+
+	if (iomap->type == IOMAP_UNWRITTEN)
+		flags |= IOMAP_IOEND_UNWRITTEN;
+	if (iomap->flags & IOMAP_F_SHARED)
+		flags |= IOMAP_IOEND_SHARED;
+	if (iomap->flags & IOMAP_F_INTEGRITY)
+		flags |= IOMAP_IOEND_INTEGRITY;
+
+	return flags;
+}
 
 /*
  * Structure for writeback I/O completions.
@@ -500,6 +522,7 @@ sector_t iomap_bmap(struct address_space *mapping, sector_t bno,
 struct iomap_ioend {
 	struct list_head	io_list;	/* next ioend in chain */
 	u16			io_flags;	/* IOMAP_IOEND_* */
+	u32			io_bvec_offset;	/* offset into first bvec */
 	struct inode		*io_inode;	/* file being written to */
 	size_t			io_size;	/* size of the extent */
 	atomic_t		io_remaining;	/* completetion defer count */
@@ -517,6 +540,13 @@ static inline struct iomap_ioend *iomap_ioend_from_bio(struct bio *bio)
 	return container_of(bio, struct iomap_ioend, io_bio);
 }
 
+#define BVEC_ITER_IOEND(_ioend)				\
+{							\
+	.bi_sector	= (_ioend)->io_sector,		\
+	.bi_size	= (_ioend)->io_size,		\
+	.bi_offset	= (_ioend)->io_bvec_offset,	\
+}
+
 struct iomap_writeback_ops {
 	/*
 	 * Performs writeback on the passed in range
@@ -565,6 +595,7 @@ void iomap_finish_ioends(struct iomap_ioend *ioend, int error);
 void iomap_ioend_try_merge(struct iomap_ioend *ioend,
 		struct list_head *more_ioends);
 void iomap_sort_ioends(struct list_head *ioend_list);
+int iomap_ioend_integrity_verify(struct iomap_ioend *ioend);
 ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
 		loff_t pos, loff_t end_pos, unsigned int dirty_len);
 int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error);
@@ -577,6 +608,11 @@ void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
 int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio);
 int iomap_writepages(struct iomap_writepage_ctx *wpc);
 
+void iomap_bounce_read(struct iomap_ioend *orig_ioend, unsigned int minsize,
+		void (*submit_ioend)(struct iomap_ioend *ioend));
+void iomap_bounce_read_end_io(struct iomap_ioend *ioend, struct bio *orig_bio,
+		int error);
+
 struct iomap_read_folio_ctx {
 	const struct iomap_read_ops *ops;
 	struct folio		*cur_folio;
diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h
index 060f6e23ab3c83..fae5d50e4f2776 100644
--- a/include/linux/iommu-dma.h
+++ b/include/linux/iommu-dma.h
@@ -39,7 +39,7 @@ int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
 		unsigned long attrs);
 unsigned long iommu_dma_get_merge_boundary(struct device *dev);
-size_t iommu_dma_opt_mapping_size(void);
+size_t iommu_dma_max_opt_mapping_size(void);
 size_t iommu_dma_max_mapping_size(struct device *dev);
 void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
 		dma_addr_t handle, unsigned long attrs);
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index ac43b8b93f14aa..05f7cee1375eed 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -773,8 +773,10 @@ struct iommu_ops {
  *                         passes in the cache invalidation requests, in form
  *                         of a driver data structure. The driver must update
  *                         array->entry_num to report the number of handled
- *                         invalidation requests. The driver data structure
- *                         must be defined in include/uapi/linux/iommufd.h
+ *                         invalidation requests. A driver may handle fewer than
+ *                         the requested, in which case the core re-invokes the
+ *                         op for the remainder. The driver data structure must
+ *                         be defined in include/uapi/linux/iommufd.h
  * @iova_to_phys: translate iova to physical address
  * @enforce_cache_coherency: Prevent any kind of DMA from bypassing IOMMU_CACHE,
  *                           including no-snoop TLPs on PCIe or other platform
diff --git a/include/linux/iommufd.h b/include/linux/iommufd.h
index 6e7efe83bc5d83..3087f5b2def848 100644
--- a/include/linux/iommufd.h
+++ b/include/linux/iommufd.h
@@ -154,6 +154,8 @@ struct iommufd_hw_queue {
  *                    The @array passes in the cache invalidation requests, in
  *                    form of a driver data structure. A driver must update the
  *                    array->entry_num to report the number of handled requests.
+ *                    A driver may handle fewer than the requested entry_num, in
+ *                    which case the core re-invokes the op for the remainder.
  *                    The data structure of the array entry must be defined in
  *                    include/uapi/linux/iommufd.h
  * @vdevice_size: Size of the driver-defined vDEVICE structure per this vIOMMU
diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h
index 0bb6c03481fada..2be273bb68f00f 100644
--- a/include/linux/irq-entry-common.h
+++ b/include/linux/irq-entry-common.h
@@ -346,22 +346,7 @@ typedef struct irqentry_state {
  *
  * Conditional reschedule with additional sanity checks.
  */
-void raw_irqentry_exit_cond_resched(void);
-
-#ifdef CONFIG_PREEMPT_DYNAMIC
-#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-#define irqentry_exit_cond_resched_dynamic_enabled	raw_irqentry_exit_cond_resched
-#define irqentry_exit_cond_resched_dynamic_disabled	NULL
-DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
-#define irqentry_exit_cond_resched()	static_call(irqentry_exit_cond_resched)()
-#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
-void dynamic_irqentry_exit_cond_resched(void);
-#define irqentry_exit_cond_resched()	dynamic_irqentry_exit_cond_resched()
-#endif
-#else /* CONFIG_PREEMPT_DYNAMIC */
-#define irqentry_exit_cond_resched()	raw_irqentry_exit_cond_resched()
-#endif /* CONFIG_PREEMPT_DYNAMIC */
+void irqentry_exit_cond_resched(void);
 
 /**
  * irqentry_enter_from_kernel_mode - Establish state before invoking the irq handler
diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h
index 2c2fb39f049c3d..299e75bc95255b 100644
--- a/include/linux/irqchip/arm-gic-v5.h
+++ b/include/linux/irqchip/arm-gic-v5.h
@@ -63,20 +63,34 @@
 #define GICV5_OUTER_SHARE		0b10
 #define GICV5_INNER_SHARE		0b11
 
+#define GICV5_AIDR_COMPONENT_IRS	0b00
+#define GICV5_AIDR_COMPONENT_ITS	0b01
+#define GICV5_AIDR_COMPONENT_IWB	0b10
+
+#define GICV5_AIDR_ARCH_MAJ_REV_V5	0
+#define GICV5_AIDR_ARCH_MIN_REV_V0	0
+
 /*
  * IRS registers and tables structures
  */
 #define GICV5_IRS_IDR0			0x0000
 #define GICV5_IRS_IDR1			0x0004
 #define GICV5_IRS_IDR2			0x0008
+#define GICV5_IRS_IDR3			0x000c
+#define GICV5_IRS_IDR4			0x0010
 #define GICV5_IRS_IDR5			0x0014
 #define GICV5_IRS_IDR6			0x0018
 #define GICV5_IRS_IDR7			0x001c
+#define GICV5_IRS_IIDR			0x0040
+#define GICV5_IRS_AIDR			0x0044
 #define GICV5_IRS_CR0			0x0080
 #define GICV5_IRS_CR1			0x0084
 #define GICV5_IRS_SYNCR			0x00c0
 #define GICV5_IRS_SYNC_STATUSR		0x00c4
+#define GICV5_IRS_SPI_VMR		0x0100
 #define GICV5_IRS_SPI_SELR		0x0108
+#define GICV5_IRS_SPI_DOMAINR		0x010c
+#define GICV5_IRS_SPI_RESAMPLER		0x0110
 #define GICV5_IRS_SPI_CFGR		0x0114
 #define GICV5_IRS_SPI_STATUSR		0x0118
 #define GICV5_IRS_PE_SELR		0x0140
@@ -86,11 +100,51 @@
 #define GICV5_IRS_IST_CFGR		0x0190
 #define GICV5_IRS_IST_STATUSR		0x0194
 #define GICV5_IRS_MAP_L2_ISTR		0x01c0
-
+#define GICV5_IRS_VMT_BASER		0x0200
+#define GICV5_IRS_VMT_CFGR		0x0210
+#define GICV5_IRS_VMT_STATUSR		0x0214
+#define GICV5_IRS_VPE_SELR		0x0240
+#define GICV5_IRS_VPE_DBR		0x0248
+#define GICV5_IRS_VPE_HPPIR		0x0250
+#define GICV5_IRS_VPE_CR0		0x0258
+#define GICV5_IRS_VPE_STATUSR		0x025c
+#define GICV5_IRS_VM_DBR		0x0280
+#define GICV5_IRS_VM_SELR		0x0288
+#define GICV5_IRS_VM_STATUSR		0x028c
+#define GICV5_IRS_VMAP_L2_VMTR		0x02c0
+#define GICV5_IRS_VMAP_VMR		0x02c8
+#define GICV5_IRS_VMAP_VISTR		0x02d0
+#define GICV5_IRS_VMAP_L2_VISTR		0x02d8
+#define GICV5_IRS_VMAP_VPER		0x02e0
+#define GICV5_IRS_SAVE_VMR		0x0300
+#define GICV5_IRS_SAVE_VM_STATUSR	0x0308
+#define GICV5_IRS_MEC_IDR		0x0340
+#define GICV5_IRS_MEC_MECID_R		0x0344
+#define GICV5_IRS_MPAM_IDR		0x0380
+#define GICV5_IRS_MPAM_PARTID_R		0x0384
+#define GICV5_IRS_SWERR_STATUSR		0x03c0
+#define GICV5_IRS_SWERR_SYNDROMER0	0x03c8
+#define GICV5_IRS_SWERR_SYNDROMER1	0x03d0
+
+#define GICV5_IRS_IDR0_IRSID		GENMASK(31, 16)
+#define GICV5_IRS_IDR0_SWE		BIT(12)
+#define GICV5_IRS_IDR0_MPAM		BIT(11)
+#define GICV5_IRS_IDR0_MEC		BIT(10)
+#define GICV5_IRS_IDR0_SETLPI		BIT(9)
+#define GICV5_IRS_IDR0_VIRT_ONE_N	BIT(8)
+#define GICV5_IRS_IDR0_ONE_N		BIT(7)
 #define GICV5_IRS_IDR0_VIRT		BIT(6)
+#define GICV5_IRS_IDR0_PA_RANGE		GENMASK(5, 2)
+#define GICV5_IRS_IDR0_INT_DOM		GENMASK(1, 0)
+
+#define GICV5_IRS_IDR0_INT_DOM_SECURE		0b00
+#define GICV5_IRS_IDR0_INT_DOM_NON_SECURE	0b01
+#define GICV5_IRS_IDR0_INT_DOM_EL3		0b10
+#define GICV5_IRS_IDR0_INT_DOM_REALM		0b11
 
 #define GICV5_IRS_IDR1_PRIORITY_BITS	GENMASK(22, 20)
 #define GICV5_IRS_IDR1_IAFFID_BITS	GENMASK(19, 16)
+#define GICV5_IRS_IDR1_PE_CNT		GENMASK(15, 0)
 
 #define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS	0b000
 #define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS	0b001
@@ -106,13 +160,30 @@
 #define GICV5_IRS_IDR2_LPI		BIT(5)
 #define GICV5_IRS_IDR2_ID_BITS		GENMASK(4, 0)
 
+#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r)	FIELD_GET(BIT(0), (r))
+#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r)	FIELD_GET(BIT(1), (r))
+#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r)	FIELD_GET(BIT(2), (r))
+
+#define GICV5_IRS_IDR3_VMT_LEVELS	BIT(10)
+#define GICV5_IRS_IDR3_VM_ID_BITS	GENMASK(9, 5)
+#define GICV5_IRS_IDR3_VMD_SZ		GENMASK(4, 1)
+#define GICV5_IRS_IDR3_VMD		BIT(0)
+
+#define GICV5_IRS_IDR4_VPE_ID_BITS	GENMASK(9, 6)
+#define GICV5_IRS_IDR4_VPED_SZ		GENMASK(5, 0)
+
 #define GICV5_IRS_IDR5_SPI_RANGE	GENMASK(24, 0)
 #define GICV5_IRS_IDR6_SPI_IRS_RANGE	GENMASK(24, 0)
 #define GICV5_IRS_IDR7_SPI_BASE		GENMASK(23, 0)
 
-#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r)	FIELD_GET(BIT(11), (r))
-#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r)	FIELD_GET(BIT(12), (r))
-#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r)	FIELD_GET(BIT(13), (r))
+#define GICV5_IRS_IIDR_PRODUCT_ID	GENMASK(31, 20)
+#define GICV5_IRS_IIDR_VARIANT		GENMASK(19, 16)
+#define GICV5_IRS_IIDR_REVISION		GENMASK(15, 12)
+#define GICV5_IRS_IIDR_IMPLEMENTER	GENMASK(11, 0)
+
+#define GICV5_IRS_AIDR_COMPONENT	GENMASK(11, 8)
+#define GICV5_IRS_AIDR_ARCHMAJORREV	GENMASK(7, 4)
+#define GICV5_IRS_AIDR_ARCHMINORREV	GENMASK(3, 0)
 
 #define GICV5_IRS_CR0_IDLE		BIT(1)
 #define GICV5_IRS_CR0_IRSEN		BIT(0)
@@ -135,21 +206,39 @@
 
 #define GICV5_IRS_SYNC_STATUSR_IDLE	BIT(0)
 
-#define GICV5_IRS_SPI_STATUSR_V		BIT(1)
-#define GICV5_IRS_SPI_STATUSR_IDLE	BIT(0)
+#define GICV5_IRS_SPI_VMR_VIRT		BIT_ULL(63)
+#define GICV5_IRS_SPI_VMR_VM_ID		GENMASK_ULL(15, 0)
 
 #define GICV5_IRS_SPI_SELR_ID		GENMASK(23, 0)
 
+#define GICV5_IRS_SPI_DOMAINR_DOMAIN	GENMASK(1, 0)
+
+#define GICV5_IRS_SPI_DOMAINR_DOMAIN_SECURE	0b00
+#define GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE	0b01
+#define GICV5_IRS_SPI_DOMAINR_DOMAIN_EL3	0b10
+#define GICV5_IRS_SPI_DOMAINR_DOMAIN_REALM	0b11
+
+#define GICV5_IRS_SPI_RESAMPLER_ID	GENMASK(23, 0)
+
 #define GICV5_IRS_SPI_CFGR_TM		BIT(0)
 
+#define GICV5_IRS_SPI_CFGR_TM_EDGE	0b0
+#define GICV5_IRS_SPI_CFGR_TM_LEVEL	0b1
+
+#define GICV5_IRS_SPI_STATUSR_V		BIT(1)
+#define GICV5_IRS_SPI_STATUSR_IDLE	BIT(0)
+
 #define GICV5_IRS_PE_SELR_IAFFID	GENMASK(15, 0)
 
+#define GICV5_IRS_PE_STATUSR_ONLINE	BIT(2)
 #define GICV5_IRS_PE_STATUSR_V		BIT(1)
 #define GICV5_IRS_PE_STATUSR_IDLE	BIT(0)
 
 #define GICV5_IRS_PE_CR0_DPS		BIT(0)
 
-#define GICV5_IRS_IST_STATUSR_IDLE	BIT(0)
+#define GICV5_IRS_IST_BASER_ADDR_MASK	GENMASK_ULL(55, 6)
+#define GICV5_IRS_IST_BASER_VALID	BIT_ULL(0)
+#define GICV5_IRS_IST_BASER_ADDR_SHIFT	6ULL
 
 #define GICV5_IRS_IST_CFGR_STRUCTURE	BIT(16)
 #define GICV5_IRS_IST_CFGR_ISTSZ	GENMASK(8, 7)
@@ -167,15 +256,111 @@
 #define GICV5_IRS_IST_CFGR_L2SZ_16K	0b01
 #define GICV5_IRS_IST_CFGR_L2SZ_64K	0b10
 
-#define GICV5_IRS_IST_BASER_ADDR_MASK	GENMASK_ULL(55, 6)
-#define GICV5_IRS_IST_BASER_VALID	BIT_ULL(0)
+#define GICV5_IRS_IST_STATUSR_IDLE	BIT(0)
 
 #define GICV5_IRS_MAP_L2_ISTR_ID	GENMASK(23, 0)
 
+#define GICV5_IRS_VMT_BASER_ADDR	GENMASK_ULL(55, 3)
+#define GICV5_IRS_VMT_BASER_ADDR_SHIFT	3ULL
+#define GICV5_IRS_VMT_BASER_VALID	BIT_ULL(0)
+
+#define GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL	0b1
+#define GICV5_IRS_VMT_CFGR_STRUCTURE_LINEAR	0b0
+
+#define GICV5_IRS_VMT_CFGR_STRUCTURE	BIT(16)
+#define GICV5_IRS_VMT_CFGR_VM_ID_BITS	GENMASK(4, 0)
+
+#define GICV5_IRS_VMT_STATUSR_IDLE	BIT(0)
+
+#define GICV5_IRS_VPE_SELR_S		BIT_ULL(63)
+#define GICV5_IRS_VPE_SELR_VPE_ID	GENMASK_ULL(47, 32)
+#define GICV5_IRS_VPE_SELR_VM_ID	GENMASK_ULL(15, 0)
+
+#define GICV5_IRS_VPE_DBR_DBV		BIT_ULL(63)
+#define GICV5_IRS_VPE_DBR_REQ_DB	BIT_ULL(62)
+#define GICV5_IRS_VPE_DBR_DBPM		GENMASK_ULL(36, 32)
+#define GICV5_IRS_VPE_DBR_INTID	GENMASK_ULL(23, 0)
+
+#define GICV5_IRS_VPE_HPPIR_HPPIV	BIT_ULL(32)
+#define GICV5_IRS_VPE_HPPIR_TYPE	GENMASK_ULL(31, 29)
+#define GICV5_IRS_VPE_HPPIR_ID		GENMASK_ULL(23, 0)
+
+#define GICV5_IRS_VPE_CR0_DPS		BIT(0)
+
+#define GICV5_IRS_VPE_STATUSR_V		BIT(1)
+#define GICV5_IRS_VPE_STATUSR_IDLE	BIT(0)
+
+#define GICV5_IRS_VM_DBR_EN		BIT_ULL(63)
+#define GICV5_IRS_VM_DBR_VPE_ID		GENMASK_ULL(15, 0)
+
+#define GICV5_IRS_VM_SELR_VM_ID		GENMASK(15, 0)
+
+#define GICV5_IRS_VM_STATUSR_V		BIT(1)
+#define GICV5_IRS_VM_STATUSR_IDLE	BIT(0)
+
+#define GICV5_IRS_VMAP_L2_VMTR_M	BIT_ULL(63)
+#define GICV5_IRS_VMAP_L2_VMTR_VM_ID	GENMASK_ULL(15, 0)
+
+#define GICV5_IRS_VMAP_VMR_M		BIT_ULL(63)
+#define GICV5_IRS_VMAP_VMR_U		BIT_ULL(62)
+#define GICV5_IRS_VMAP_VMR_VM_ID	GENMASK_ULL(15, 0)
+
+#define GICV5_IRS_VMAP_VISTR_M		BIT_ULL(63)
+#define GICV5_IRS_VMAP_VISTR_U		BIT_ULL(62)
+#define GICV5_IRS_VMAP_VISTR_VM_ID	GENMASK_ULL(47, 32)
+#define GICV5_IRS_VMAP_VISTR_TYPE	GENMASK_ULL(31, 29)
+
+#define GICV5_IRS_VMAP_L2_VISTR_M	BIT_ULL(63)
+#define GICV5_IRS_VMAP_L2_VISTR_VM_ID	GENMASK_ULL(47, 32)
+#define GICV5_IRS_VMAP_L2_VISTR_TYPE	GENMASK_ULL(31, 29)
+#define GICV5_IRS_VMAP_L2_VISTR_ID	GENMASK_ULL(23, 0)
+
+#define GICV5_IRS_VMAP_VPER_M		BIT_ULL(63)
+#define GICV5_IRS_VMAP_VPER_VM_ID	GENMASK_ULL(47, 32)
+#define GICV5_IRS_VMAP_VPER_VPE_ID	GENMASK_ULL(15, 0)
+
+#define GICV5_IRS_SAVE_VMR_VM_ID	GENMASK_ULL(15, 0)
+#define GICV5_IRS_SAVE_VMR_Q		BIT_ULL(62)
+#define GICV5_IRS_SAVE_VMR_S		BIT_ULL(63)
+
+#define GICV5_IRS_SAVE_VM_STATUSR_IDLE	BIT(0)
+#define GICV5_IRS_SAVE_VM_STATUSR_Q	BIT(1)
+
+#define GICV5_IRS_MEC_IDR_MECIDSIZE	GENMASK(3, 0)
+
+#define GICV5_IRS_MEC_MECID_R_MECID	GENMASK(15, 0)
+
+#define GICV5_IRS_MPAM_IDR_HAS_MPAM_SP	BIT(24)
+#define GICV5_IRS_MPAM_IDR_PMG_MAX	GENMASK(23, 16)
+#define GICV5_IRS_MPAM_IDR_PARTID_MAX	GENMASK(15, 0)
+
+#define GICV5_IRS_MPAM_PARTID_R_IDLE	BIT(31)
+#define GICV5_IRS_MPAM_PARTID_R_MPAM_SP	GENMASK(25, 24)
+#define GICV5_IRS_MPAM_PARTID_R_PMG	GENMASK(23, 16)
+#define GICV5_IRS_MPAM_PARTID_R_PARTID	GENMASK(15, 0)
+
+#define GICV5_IRS_SWERR_STATUSR_IMP_EC	GENMASK_ULL(31, 24)
+#define GICV5_IRS_SWERR_STATUSR_EC	GENMASK_ULL(23, 16)
+#define GICV5_IRS_SWERR_STATUSR_OF	BIT_ULL(3)
+#define GICV5_IRS_SWERR_STATUSR_S1V	BIT_ULL(2)
+#define GICV5_IRS_SWERR_STATUSR_S0V	BIT_ULL(1)
+#define GICV5_IRS_SWERR_STATUSR_V	BIT_ULL(0)
+
+#define GICV5_IRS_SWERR_SYNDROMER0_VIRTUAL	BIT_ULL(63)
+#define GICV5_IRS_SWERR_SYNDROMER0_TYPE		GENMASK_ULL(62, 60)
+#define GICV5_IRS_SWERR_SYNDROMER0_ID		GENMASK_ULL(55, 32)
+#define GICV5_IRS_SWERR_SYNDROMER0_VM_ID	GENMASK_ULL(15, 0)
+
+#define GICV5_IRS_SWERR_SYNDROMER1_ADDR	GENMASK_ULL(55, 3)
+
 #define GICV5_ISTL1E_VALID		BIT_ULL(0)
+#define GICV5_IRS_ISTL1E_SIZE		8UL
 
 #define GICV5_ISTL1E_L2_ADDR_MASK	GENMASK_ULL(55, 12)
 
+#define GICV5_IRS_SETLPIR		0x0000
+#define GICV5_IRS_SETLPIR_ID		GENMASK(23, 0)
+
 /*
  * ITS registers and tables structures
  */
@@ -297,6 +482,42 @@
 #define GICV5_GSI_IWB_FRAME_ID			GENMASK(28, 16)
 #define GICV5_GSI_IWB_WIRE			GENMASK(15, 0)
 
+/*
+ * CoreSight identification registers - ordered by increasing offset.
+ */
+#define GICV5_CORESIGHT_DEVARCH		0xffbc
+#define GICV5_CORESIGHT_PIDR4		0xffd0
+#define GICV5_CORESIGHT_PIDR5		0xffd4
+#define GICV5_CORESIGHT_PIDR6		0xffd8
+#define GICV5_CORESIGHT_PIDR7		0xffdc
+#define GICV5_CORESIGHT_PIDR0		0xffe0
+#define GICV5_CORESIGHT_PIDR1		0xffe4
+#define GICV5_CORESIGHT_PIDR2		0xffe8
+#define GICV5_CORESIGHT_PIDR3		0xffec
+#define GICV5_CORESIGHT_CIDR0		0xfff0
+#define GICV5_CORESIGHT_CIDR1		0xfff4
+#define GICV5_CORESIGHT_CIDR2		0xfff8
+#define GICV5_CORESIGHT_CIDR3		0xfffc
+
+#define GICV5_CORESIGHT_DEVARCH_VAL			\
+	(FIELD_PREP(GENMASK(31, 21), 0x23b) |		\
+	 BIT(20) |					\
+	 0x5a19)
+
+#define GICV5_CORESIGHT_PIDR4_JEP106_CONT	0x04
+#define GICV5_CORESIGHT_PIDR5_RES0		0x00
+#define GICV5_CORESIGHT_PIDR6_RES0		0x00
+#define GICV5_CORESIGHT_PIDR7_RES0		0x00
+#define GICV5_CORESIGHT_PIDR0_PART_0		0x4b
+#define GICV5_CORESIGHT_PIDR1_DES_0_PART_1	0xb0
+#define GICV5_CORESIGHT_PIDR2_DES_1		0x0b
+#define GICV5_CORESIGHT_PIDR3_REVAND_CMOD	0x00
+
+#define GICV5_CORESIGHT_CIDR0_VAL		0x0d
+#define GICV5_CORESIGHT_CIDR1_VAL		0xf0
+#define GICV5_CORESIGHT_CIDR2_VAL		0x05
+#define GICV5_CORESIGHT_CIDR3_VAL		0xb1
+
 /*
  * Global Data structures and functions
  */
@@ -332,6 +553,8 @@ struct gicv5_irs_chip_data {
 	raw_spinlock_t		spi_config_lock;
 };
 
+#define IRS_FLAGS_NON_COHERENT		BIT(0)
+
 static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset,
 					     const char *reg_s, u32 mask,
 					     u32 *val)
@@ -379,6 +602,7 @@ void __init gicv5_free_lpi_domain(void);
 
 int gicv5_irs_of_probe(struct device_node *parent);
 int gicv5_irs_acpi_probe(void);
+struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void);
 void gicv5_irs_remove(void);
 int gicv5_irs_enable(void);
 void gicv5_irs_its_probe(void);
@@ -387,10 +611,13 @@ int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid);
 struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id);
 int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type);
 int gicv5_irs_iste_alloc(u32 lpi);
+unsigned int gicv5_irs_l2_sz(u32 l2sz);
 void gicv5_irs_syncr(void);
 
 /* Embedded in kvm.arch */
 struct gicv5_vpe {
+	int			db;
+	bool			db_fired;
 	bool			resident;
 };
 
@@ -429,4 +656,15 @@ void gicv5_deinit_lpis(void);
 
 void __init gicv5_its_of_probe(struct device_node *parent);
 void __init gicv5_its_acpi_probe(void);
+
+enum gicv5_vcpu_cmd {
+	VMT_L2_MAP,		/* Map in a L2 VMT - *may* happen on VM init */
+	VMTE_MAKE_VALID,	/* Make the VMTE valid */
+	VMTE_MAKE_INVALID,	/* Make the VMTE (et al.) invalid */
+	VPE_MAKE_VALID,		/* No corresponding invalid */
+	SPI_VIST_MAKE_VALID,	/* No corresponding invalid */
+	LPI_VIST_MAKE_VALID,	/* Triggered by a guest */
+	LPI_VIST_MAKE_INVALID,	/* Triggered by a guest */
+};
+
 #endif
diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h
index 67d9d960273b99..f05370e2debf44 100644
--- a/include/linux/irqchip/arm-vgic-info.h
+++ b/include/linux/irqchip/arm-vgic-info.h
@@ -38,6 +38,11 @@ struct gic_kvm_info {
 	bool		has_v4_1;
 	/* Deactivation impared, subpar stuff */
 	bool		no_hw_deactivation;
+	/* GICv5 IRS base */
+	struct {
+		void __iomem	*base;
+		bool		non_coherent;
+	}		gicv5_irs;
 };
 
 #ifdef CONFIG_KVM
diff --git a/include/linux/kdev_t.h b/include/linux/kdev_t.h
index 4856706fbfeb45..2dbbd47f1e68e1 100644
--- a/include/linux/kdev_t.h
+++ b/include/linux/kdev_t.h
@@ -9,7 +9,7 @@
 
 #define MAJOR(dev)	((unsigned int) ((dev) >> MINORBITS))
 #define MINOR(dev)	((unsigned int) ((dev) & MINORMASK))
-#define MKDEV(ma,mi)	(((ma) << MINORBITS) | (mi))
+#define MKDEV(ma, mi)	(((dev_t)(ma) << MINORBITS) | (mi))
 
 #define print_dev_t(buffer, dev)					\
 	sprintf((buffer), "%u:%u\n", MAJOR(dev), MINOR(dev))
diff --git a/include/linux/kernel-page-flags.h b/include/linux/kernel-page-flags.h
index 196778a087c4df..fe5ab6e50bd70e 100644
--- a/include/linux/kernel-page-flags.h
+++ b/include/linux/kernel-page-flags.h
@@ -11,7 +11,6 @@
 #define KPF_RESERVED		32
 #define KPF_MLOCKED		33
 #define KPF_OWNER_2		34
-#define KPF_PRIVATE		35
 #define KPF_PRIVATE_2		36
 #define KPF_OWNER_PRIVATE	37
 #define KPF_ARCH		38
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 24414c79e59abe..4b11d1dc0a6746 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -43,30 +43,10 @@ struct completion;
 struct user;
 
 #ifdef CONFIG_PREEMPT_VOLUNTARY_BUILD
-
 extern int __cond_resched(void);
 # define might_resched() __cond_resched()
-
-#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-
-extern int __cond_resched(void);
-
-DECLARE_STATIC_CALL(might_resched, __cond_resched);
-
-static __always_inline void might_resched(void)
-{
-	static_call_mod(might_resched)();
-}
-
-#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-
-extern int dynamic_might_resched(void);
-# define might_resched() dynamic_might_resched()
-
 #else
-
 # define might_resched() do { } while (0)
-
 #endif /* CONFIG_PREEMPT_* */
 
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index 9ca6c2259dfea6..c1e85550bf1256 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -196,6 +196,17 @@ static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
 }
 #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */
 
+static inline u64 kcpustat_field_total(enum cpu_usage_stat usage, const struct cpumask *cpus)
+{
+	u64 total = 0;
+	int cpu;
+
+	for_each_cpu(cpu, cpus)
+		total += kcpustat_field(usage, cpu);
+
+	return total;
+}
+
 extern void account_user_time(struct task_struct *, u64);
 extern void account_guest_time(struct task_struct *, u64);
 extern void account_system_time(struct task_struct *, int, u64);
diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h
index 351a5101c86287..73786b567ccd8e 100644
--- a/include/linux/kernfs.h
+++ b/include/linux/kernfs.h
@@ -309,7 +309,7 @@ struct kernfs_ops {
 	 * write() is bounced through kernel buffer.  If atomic_write_len
 	 * is not set, a write larger than PAGE_SIZE results in partial
 	 * operations of PAGE_SIZE chunks.  If atomic_write_len is set,
-	 * writes upto the specified size are executed atomically but
+	 * writes up to the specified size are executed atomically but
 	 * larger ones are rejected with -E2BIG.
 	 */
 	size_t atomic_write_len;
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index 0af8ae4fdd087b..f970ca2c8ce95d 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -259,21 +259,6 @@ int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len);
 extern int kexec_add_buffer(struct kexec_buf *kbuf);
 int kexec_locate_mem_hole(struct kexec_buf *kbuf);
 
-#ifndef arch_kexec_locate_mem_hole
-/**
- * arch_kexec_locate_mem_hole - Find free memory to place the segments.
- * @kbuf:                       Parameters for the memory search.
- *
- * On success, kbuf->mem will have the start address of the memory region found.
- *
- * Return: 0 on success, negative errno on error.
- */
-static inline int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf)
-{
-	return kexec_locate_mem_hole(kbuf);
-}
-#endif
-
 #ifndef arch_kexec_apply_relocations_add
 /*
  * arch_kexec_apply_relocations_add - apply relocations of type RELA
@@ -418,7 +403,7 @@ struct kimage {
 #endif
 
 	struct {
-		struct kexec_segment *scratch;
+		struct kexec_segment *bootmem;
 		phys_addr_t fdt;
 	} kho;
 
diff --git a/include/linux/kexec_handover.h b/include/linux/kexec_handover.h
index 46de86dc343e9b..ef2b188c8ed090 100644
--- a/include/linux/kexec_handover.h
+++ b/include/linux/kexec_handover.h
@@ -36,15 +36,15 @@ int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size);
 void kho_memory_init(void);
 void kho_memory_init_early(void);
 
-void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys,
-		  u64 scratch_len);
+void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t bootmem_phys,
+		  u64 bootmem_len);
 
-bool kho_scratch_overlap(phys_addr_t phys, size_t size);
+bool kho_bootmem_overlap(phys_addr_t phys, size_t size);
 
-static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
+static inline enum migratetype kho_bootmem_migratetype(unsigned long pfn,
 						       enum migratetype mt)
 {
-	if (kho_scratch_overlap(PFN_PHYS(pfn), pageblock_nr_pages << PAGE_SHIFT))
+	if (kho_bootmem_overlap(PFN_PHYS(pfn), pageblock_nr_pages << PAGE_SHIFT))
 		return MIGRATE_CMA;
 	return mt;
 }
@@ -123,16 +123,16 @@ static inline void kho_memory_init(void) { }
 static inline void kho_memory_init_early(void) { }
 
 static inline void kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
-				phys_addr_t scratch_phys, u64 scratch_len)
+				phys_addr_t bootmem_phys, u64 bootmem_len)
 {
 }
 
-static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size)
+static inline bool kho_bootmem_overlap(phys_addr_t phys, size_t size)
 {
 	return false;
 }
 
-static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
+static inline enum migratetype kho_bootmem_migratetype(unsigned long pfn,
 						       enum migratetype mt)
 {
 	return mt;
diff --git a/include/linux/key.h b/include/linux/key.h
index 81b8f05c68985c..bd10fe45819dbb 100644
--- a/include/linux/key.h
+++ b/include/linux/key.h
@@ -440,6 +440,8 @@ extern key_ref_t keyring_search(key_ref_t keyring,
 extern int keyring_restrict(key_ref_t keyring, const char *type,
 			    const char *restriction);
 
+extern void key_register_bpf_keyring(struct key *keyring);
+
 extern struct key *key_lookup(key_serial_t id);
 
 static inline key_serial_t key_serial(const struct key *key)
diff --git a/include/linux/kho/abi/pci.h b/include/linux/kho/abi/pci.h
new file mode 100644
index 00000000000000..9485ed73c35196
--- /dev/null
+++ b/include/linux/kho/abi/pci.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * Copyright (c) 2026, Google LLC.
+ * David Matlack <dmatlack@google.com>
+ */
+
+#ifndef _LINUX_KHO_ABI_PCI_H
+#define _LINUX_KHO_ABI_PCI_H
+
+#include <linux/bug.h>
+#include <linux/compiler.h>
+#include <linux/types.h>
+
+/**
+ * DOC: PCI File-Lifecycle Bound (FLB) Live Update ABI
+ *
+ * This header defines the ABI for preserving core PCI state across kexec using
+ * Live Update File-Lifecycle Bound (FLB) data.
+ *
+ * This interface is a contract. Any modification to any of the serialization
+ * structs defined here constitutes a breaking change. Such changes require
+ * incrementing the version number in the PCI_LUO_FLB_VERSION number.
+ */
+
+#define PCI_LUO_FLB_COMPATIBLE "pci"
+#define PCI_LUO_FLB_VERSION 2
+
+/**
+ * struct pci_dev_ser - Serialized state about a single PCI device.
+ *
+ * @domain: The device's PCI domain number (segment).
+ * @bdf: The device's PCI bus, device, and function number.
+ * @refcount: Reference count used by the PCI core to keep track of whether it
+ *            is done using a device's struct pci_dev_ser. The value of the
+ *            refcount is equal to the number of preserved devices at or below
+ *            it in the PCI hierarchy when the struct pci_dev_ser is in use, and
+ *            0 otherwise.
+ */
+struct pci_dev_ser {
+	u32 domain;
+	u16 bdf;
+	u16 refcount;
+} __packed;
+
+/**
+ * struct pci_ser - PCI Subsystem Live Update State
+ *
+ * This struct tracks state about all devices that are being preserved across
+ * a Live Update for the next kernel. It contains only state owned by the PCI
+ * core; the state a driver needs to resume its device is preserved separately
+ * by that driver.
+ *
+ * @version: The version of the "pci" FLB struct. This field must never be
+ *           deleted, moved, or resized, as the kernel depends on always being
+ *           able to check the struct pci_ser version number.
+ * @nr_devices: The number of devices that were preserved.
+ * @devices: Physical address of the first KHO block containing pci_dev_ser.
+ */
+struct pci_ser {
+	u32 version;
+	u32 nr_devices;
+	u64 devices;
+} __packed;
+
+#endif /* _LINUX_KHO_ABI_PCI_H */
diff --git a/include/linux/kmod.h b/include/linux/kmod.h
index 9a07c321538978..b9474a62a5682d 100644
--- a/include/linux/kmod.h
+++ b/include/linux/kmod.h
@@ -2,17 +2,9 @@
 #ifndef __LINUX_KMOD_H__
 #define __LINUX_KMOD_H__
 
-/*
- *	include/linux/kmod.h
- */
-
-#include <linux/umh.h>
-#include <linux/gfp.h>
-#include <linux/stddef.h>
+#include <linux/compiler_attributes.h>
 #include <linux/errno.h>
-#include <linux/compiler.h>
-#include <linux/workqueue.h>
-#include <linux/sysctl.h>
+#include <linux/types.h>
 
 #ifdef CONFIG_MODULES
 /* modprobe exit status on success, -ve on error.  Return value
diff --git a/include/linux/ksm.h b/include/linux/ksm.h
index d39d0d5483a23a..edfa67017560da 100644
--- a/include/linux/ksm.h
+++ b/include/linux/ksm.h
@@ -10,7 +10,6 @@
 
 #include <linux/bitops.h>
 #include <linux/mm.h>
-#include <linux/pagemap.h>
 #include <linux/rmap.h>
 #include <linux/sched.h>
 
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 03bfc92864b6e6..c0748f7e40ea25 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -722,27 +722,6 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
 }
 #endif
 
-#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
-static inline bool kvm_arch_has_private_mem(struct kvm *kvm)
-{
-	return false;
-}
-#endif
-
-#ifdef CONFIG_KVM_GUEST_MEMFD
-bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm);
-
-static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm)
-{
-	u64 flags = GUEST_MEMFD_FLAG_MMAP;
-
-	if (!kvm || kvm_arch_supports_gmem_init_shared(kvm))
-		flags |= GUEST_MEMFD_FLAG_INIT_SHARED;
-
-	return flags;
-}
-#endif
-
 #ifndef kvm_arch_has_readonly_mem
 static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm)
 {
@@ -791,7 +770,6 @@ struct kvm {
 	/* The current active memslot set for each address space */
 	struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES];
 	struct xarray vcpu_array;
-	DECLARE_BITMAP(vcpu_ids, KVM_MAX_VCPU_IDS);
 	/*
 	 * Protected by slots_lock, but can be read outside if an
 	 * incorrect answer is acceptable.
@@ -855,6 +833,8 @@ struct kvm {
 	gfn_t mmu_invalidate_range_start;
 	gfn_t mmu_invalidate_range_end;
 
+	unsigned long gpc_invalidate_seq;
+
 	struct list_head devices;
 	u64 manual_dirty_log_protect;
 	struct dentry *debugfs_dentry;
@@ -872,7 +852,7 @@ struct kvm {
 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
 	struct notifier_block pm_notifier;
 #endif
-#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
 	/* Protected by slots_lock (for writes) and RCU (for reads) */
 	struct xarray mem_attr_array;
 #endif
@@ -2324,13 +2304,18 @@ static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu)
 	return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests);
 }
 
-static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu)
+static __always_inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu)
 {
+	BUILD_BUG_ON(req == KVM_REQ_VM_DEAD);
+
 	clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests);
 }
 
-static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
+static __always_inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
 {
+	/* Once a VM is dead, it needs to stay dead. */
+	BUILD_BUG_ON(req == KVM_REQ_VM_DEAD);
+
 	if (kvm_test_request(req, vcpu)) {
 		kvm_clear_request(req, vcpu);
 
@@ -2560,48 +2545,80 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot)
 	return slot->flags & KVM_MEMSLOT_GMEM_ONLY;
 }
 
-#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
-static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn)
+#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
+static inline unsigned long kvm_get_vm_memory_attributes(struct kvm *kvm, gfn_t gfn)
 {
 	return xa_to_value(xa_load(&kvm->mem_attr_array, gfn));
 }
 
-bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
-				     unsigned long mask, unsigned long attrs);
-bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
-					struct kvm_gfn_range *range);
-bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
-					 struct kvm_gfn_range *range);
+bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
+					unsigned long mask, unsigned long attrs);
+bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm,
+					   struct kvm_gfn_range *range);
+bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm,
+					    struct kvm_gfn_range *range);
 
-static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
+static inline bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn)
 {
-	return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE;
+	return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE;
+}
+#endif  /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */
+
+#ifdef kvm_arch_has_private_mem
+extern bool gmem_in_place_conversion;
+
+typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn);
+DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t);
+
+static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn)
+{
+	return static_call(__kvm_is_private_gfn)(kvm, gfn);
 }
 #else
-static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
+#define gmem_in_place_conversion false
+
+static inline bool kvm_arch_has_private_mem(struct kvm *kvm)
+{
+	return false;
+}
+
+static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	return false;
 }
-#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
+#endif /* kvm_arch_has_private_mem */
 
 #ifdef CONFIG_KVM_GUEST_MEMFD
+bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn);
+bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm);
+
+static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm)
+{
+	u64 flags = GUEST_MEMFD_FLAG_MMAP;
+
+	if (!kvm || kvm_arch_supports_gmem_init_shared(kvm))
+		flags |= GUEST_MEMFD_FLAG_INIT_SHARED;
+
+	return flags;
+}
+
 int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
-		     gfn_t gfn, kvm_pfn_t *pfn, struct page **page,
-		     int *max_order);
+		     gfn_t gfn, kvm_pfn_t *pfn, int *max_order);
 #else
 static inline int kvm_gmem_get_pfn(struct kvm *kvm,
 				   struct kvm_memory_slot *slot, gfn_t gfn,
-				   kvm_pfn_t *pfn, struct page **page,
-				   int *max_order)
+				   kvm_pfn_t *pfn, int *max_order)
 {
 	KVM_BUG_ON(1, kvm);
 	return -EIO;
 }
 #endif /* CONFIG_KVM_GUEST_MEMFD */
 
-#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
 int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
 			       kvm_pfn_t nr_pages);
+void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
+#ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
+#define kvm_arch_has_gmem_convert() false
 #endif
 
 #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE
@@ -2643,6 +2660,7 @@ void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range
 #endif
 
 #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
+int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu);
 long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
 				    struct kvm_pre_fault_memory *range);
 #endif
diff --git a/include/linux/leds.h b/include/linux/leds.h
index a515f075c29a11..beaf2399306393 100644
--- a/include/linux/leds.h
+++ b/include/linux/leds.h
@@ -109,6 +109,7 @@ struct led_classdev {
 #define LED_INIT_DEFAULT_TRIGGER BIT(23)
 #define LED_REJECT_NAME_CONFLICT BIT(24)
 #define LED_MULTI_COLOR		BIT(25)
+#define LED_TRIG_HW_CHANGED	BIT(26)
 
 	/* set_brightness_work / blink_timer flags, atomic, private. */
 	unsigned long		work_flags;
@@ -239,6 +240,11 @@ struct led_classdev {
 	struct kernfs_node	*brightness_hw_changed_kn;
 #endif
 
+#ifdef CONFIG_LEDS_TRIGGERS_HW_CHANGED
+	bool			trigger_hw_changed;
+	struct work_struct	trigger_hw_changed_work;
+#endif
+
 	/* Ensures consistent access to the LED class device */
 	struct mutex		led_access;
 };
@@ -485,6 +491,7 @@ struct led_trigger {
 	const char	 *name;
 	int		(*activate)(struct led_classdev *led_cdev);
 	void		(*deactivate)(struct led_classdev *led_cdev);
+	bool		(*hw_offloaded)(struct led_classdev *led_cdev);
 
 	/* Brightness set by led_trigger_event */
 	enum led_brightness brightness;
@@ -533,6 +540,9 @@ void led_trigger_blink_oneshot(struct led_trigger *trigger,
 void led_trigger_set_default(struct led_classdev *led_cdev);
 int led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trigger);
 void led_trigger_remove(struct led_classdev *led_cdev);
+void led_trigger_remove_hw_control(struct led_classdev *led_cdev);
+
+bool led_trigger_is_hw_controlled(struct led_classdev *led_cdev);
 
 static inline void led_set_trigger_data(struct led_classdev *led_cdev,
 					void *trigger_data)
@@ -584,6 +594,13 @@ static inline int led_trigger_set(struct led_classdev *led_cdev,
 }
 
 static inline void led_trigger_remove(struct led_classdev *led_cdev) {}
+static inline void led_trigger_remove_hw_control(struct led_classdev *led_cdev) {}
+
+static inline bool led_trigger_is_hw_controlled(struct led_classdev *led_cdev)
+{
+	return false;
+}
+
 static inline void led_set_trigger_data(struct led_classdev *led_cdev) {}
 static inline void *led_get_trigger_data(struct led_classdev *led_cdev)
 {
@@ -598,6 +615,13 @@ led_trigger_get_brightness(const struct led_trigger *trigger)
 
 #endif /* CONFIG_LEDS_TRIGGERS */
 
+#ifdef CONFIG_LEDS_TRIGGERS_HW_CHANGED
+void led_trigger_notify_hw_control_changed(struct led_classdev *led_cdev, bool activate);
+#else
+static inline void led_trigger_notify_hw_control_changed(struct led_classdev *led_cdev,
+							 bool activate) {}
+#endif
+
 /* Trigger specific enum */
 enum led_trigger_netdev_modes {
 	TRIGGER_NETDEV_LINK = 0,
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 313e96173b1921..3bd74d79f8126a 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -76,7 +76,7 @@ enum ata_quirks {
 	__ATA_QUIRK_MAX_SEC,		/* Limit max sectors */
 	__ATA_QUIRK_MAX_TRIM_128M,	/* Limit max trim size to 128M */
 	__ATA_QUIRK_NO_NCQ_ON_ATI,	/* Disable NCQ on ATI chipset */
-	__ATA_QUIRK_NO_LPM_ON_ATI,	/* Disable LPM on ATI chipset */
+	__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD, /* Disable LPM on ATI and AMD chipsets */
 	__ATA_QUIRK_NO_ID_DEV_LOG,	/* Identify device log missing */
 	__ATA_QUIRK_NO_LOG_DIR,		/* Do not read log directory */
 	__ATA_QUIRK_NO_FUA,		/* Do not use FUA */
@@ -115,7 +115,7 @@ enum {
 	ATA_QUIRK_MAX_SEC		= BIT_ULL(__ATA_QUIRK_MAX_SEC),
 	ATA_QUIRK_MAX_TRIM_128M		= BIT_ULL(__ATA_QUIRK_MAX_TRIM_128M),
 	ATA_QUIRK_NO_NCQ_ON_ATI		= BIT_ULL(__ATA_QUIRK_NO_NCQ_ON_ATI),
-	ATA_QUIRK_NO_LPM_ON_ATI		= BIT_ULL(__ATA_QUIRK_NO_LPM_ON_ATI),
+	ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD	= BIT_ULL(__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD),
 	ATA_QUIRK_NO_ID_DEV_LOG		= BIT_ULL(__ATA_QUIRK_NO_ID_DEV_LOG),
 	ATA_QUIRK_NO_LOG_DIR		= BIT_ULL(__ATA_QUIRK_NO_LOG_DIR),
 	ATA_QUIRK_NO_FUA		= BIT_ULL(__ATA_QUIRK_NO_FUA),
@@ -1147,6 +1147,7 @@ extern struct ata_host *ata_host_alloc_pinfo(struct device *dev,
 extern void ata_host_get(struct ata_host *host);
 extern void ata_host_put(struct ata_host *host);
 extern int ata_host_start(struct ata_host *host);
+void ata_host_undo_start(struct ata_host *host);
 extern int ata_host_register(struct ata_host *host,
 			     const struct scsi_host_template *sht);
 extern int ata_host_activate(struct ata_host *host, int irq,
diff --git a/include/linux/list.h b/include/linux/list.h
index 77fb62f79928ac..e3753695e76c3c 100644
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -1172,7 +1172,7 @@ static inline void hlist_move_list(struct hlist_head *old,
 {
 	new->first = old->first;
 	if (new->first)
-		new->first->pprev = &new->first;
+		WRITE_ONCE(new->first->pprev, &new->first);
 	old->first = NULL;
 }
 
@@ -1189,10 +1189,10 @@ static inline void hlist_splice_init(struct hlist_head *from,
 				     struct hlist_head *to)
 {
 	if (to->first)
-		to->first->pprev = &last->next;
+		WRITE_ONCE(to->first->pprev, &last->next);
 	last->next = to->first;
 	to->first = from->first;
-	from->first->pprev = &to->first;
+	WRITE_ONCE(from->first->pprev, &to->first);
 	from->first = NULL;
 }
 
diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h
index ba9e3988c07c31..5f74f79c22b47d 100644
--- a/include/linux/livepatch.h
+++ b/include/linux/livepatch.h
@@ -33,6 +33,7 @@
  * @kobj:	kobject for sysfs resources
  * @node:	list node for klp_object func_list
  * @stack_node:	list node for klp_ops func_stack list
+ * @obj:	back pointer to the owning object
  * @old_size:	size of the old function
  * @new_size:	size of the new function
  * @nop:        temporary patch to use the original code again; dyn. allocated
@@ -72,6 +73,7 @@ struct klp_func {
 	struct kobject kobj;
 	struct list_head node;
 	struct list_head stack_node;
+	struct klp_object *obj;
 	unsigned long old_size, new_size;
 	bool nop;
 	bool patched;
@@ -86,6 +88,7 @@ struct klp_func {
  * @kobj:	kobject for sysfs resources
  * @func_list:	dynamic list of the function entries
  * @node:	list node for klp_patch obj_list
+ * @patch:	back pointer to the owning patch
  * @mod:	kernel module associated with the patched object
  *		(NULL for vmlinux)
  * @dynamic:    temporary object for nop functions; dynamically allocated
@@ -101,6 +104,7 @@ struct klp_object {
 	struct kobject kobj;
 	struct list_head func_list;
 	struct list_head node;
+	struct klp_patch *patch;
 	struct module *mod;
 	bool dynamic;
 	bool patched;
diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h
index 63ea5417de849a..6051abc0612cee 100644
--- a/include/linux/liveupdate.h
+++ b/include/linux/liveupdate.h
@@ -25,6 +25,7 @@ struct file;
 /**
  * struct liveupdate_file_op_args - Arguments for file operation callbacks.
  * @handler:          The file handler being called.
+ * @session:          The session this file belongs to.
  * @retrieve_status:  The retrieve status for the 'can_finish / finish'
  *                    operation. A value of 0 means the retrieve has not been
  *                    attempted, a positive value means the retrieve was
@@ -45,6 +46,7 @@ struct file;
  */
 struct liveupdate_file_op_args {
 	struct liveupdate_file_handler *handler;
+	struct liveupdate_session *session;
 	int retrieve_status;
 	struct file *file;
 	u64 serialized_data;
@@ -247,6 +249,14 @@ void liveupdate_flb_put_incoming(struct liveupdate_flb *flb);
 int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp);
 void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb);
 
+/* kernel can internally retrieve files */
+int liveupdate_get_file_incoming(struct liveupdate_session *s, u64 token,
+				 struct file **filep);
+
+/* Get a token for an outgoing file, or -ENOENT if file is not preserved */
+int liveupdate_get_token_outgoing(struct liveupdate_session *s,
+				  struct file *file, u64 *tokenp);
+
 #else /* CONFIG_LIVEUPDATE */
 
 static inline bool liveupdate_enabled(void)
@@ -299,5 +309,17 @@ static inline void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb)
 {
 }
 
+static inline int liveupdate_get_file_incoming(struct liveupdate_session *s,
+					       u64 token, struct file **filep)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int liveupdate_get_token_outgoing(struct liveupdate_session *s,
+						struct file *file, u64 *tokenp)
+{
+	return -EOPNOTSUPP;
+}
+
 #endif /* CONFIG_LIVEUPDATE */
 #endif /* _LINUX_LIVEUPDATE_H */
diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h
index 584db296e43b20..5cf0b479506571 100644
--- a/include/linux/lsm_audit.h
+++ b/include/linux/lsm_audit.h
@@ -44,7 +44,7 @@ struct lsm_network_audit {
 
 struct lsm_ioctlop_audit {
 	struct path path;
-	u16 cmd;
+	unsigned int cmd;
 };
 
 struct lsm_ibpkey_audit {
@@ -78,6 +78,7 @@ struct common_audit_data {
 #define LSM_AUDIT_DATA_NOTIFICATION 16
 #define LSM_AUDIT_DATA_ANONINODE	17
 #define LSM_AUDIT_DATA_NLMSGTYPE	18
+#define LSM_AUDIT_DATA_NS		19
 	union 	{
 		struct path path;
 		struct dentry *dentry;
@@ -100,6 +101,10 @@ struct common_audit_data {
 		int reason;
 		const char *anonclass;
 		u16 nlmsg_type;
+		struct {
+			u32 ns_type;
+			u64 ns_id;
+		} ns;
 	} u;
 	/* this union contains LSM specific data */
 	union {
diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
index 65c9609ec20770..ef16e5343e0998 100644
--- a/include/linux/lsm_hook_defs.h
+++ b/include/linux/lsm_hook_defs.h
@@ -36,6 +36,7 @@ LSM_HOOK(int, 0, binder_transfer_file, const struct cred *from,
 LSM_HOOK(int, 0, ptrace_access_check, struct task_struct *child,
 	 unsigned int mode)
 LSM_HOOK(int, 0, ptrace_traceme, struct task_struct *parent)
+LSM_HOOK(int, 0, mem_foll_force, const struct cred *subject, bool opened_by_owner)
 LSM_HOOK(int, 0, capget, const struct task_struct *target, kernel_cap_t *effective,
 	 kernel_cap_t *inheritable, kernel_cap_t *permitted)
 LSM_HOOK(int, 0, capset, struct cred *new, const struct cred *old,
@@ -94,7 +95,7 @@ LSM_HOOK(int, 0, path_mkdir, const struct path *dir, struct dentry *dentry,
 LSM_HOOK(int, 0, path_rmdir, const struct path *dir, struct dentry *dentry)
 LSM_HOOK(int, 0, path_mknod, const struct path *dir, struct dentry *dentry,
 	 umode_t mode, unsigned int dev)
-LSM_HOOK(void, LSM_RET_VOID, path_post_mknod, struct mnt_idmap *idmap,
+LSM_HOOK(void, LSM_RET_VOID, path_post_mknod, const struct mnt_idmap *idmap,
 	 struct dentry *dentry)
 LSM_HOOK(int, 0, path_truncate, const struct path *path)
 LSM_HOOK(int, 0, path_symlink, const struct path *dir, struct dentry *dentry,
@@ -120,59 +121,60 @@ LSM_HOOK(int, -EOPNOTSUPP, inode_init_security, struct inode *inode,
 	 int *xattr_count)
 LSM_HOOK(int, 0, inode_init_security_anon, struct inode *inode,
 	 const struct qstr *name, const struct inode *context_inode)
-LSM_HOOK(int, 0, inode_create, struct inode *dir, struct dentry *dentry,
-	 umode_t mode)
-LSM_HOOK(void, LSM_RET_VOID, inode_post_create_tmpfile, struct mnt_idmap *idmap,
+LSM_HOOK(int, 0, inode_create, const struct mnt_idmap *idmap, struct inode *dir,
+	 struct dentry *dentry, umode_t mode)
+LSM_HOOK(void, LSM_RET_VOID, inode_post_create_tmpfile, const struct mnt_idmap *idmap,
 	 struct inode *inode)
-LSM_HOOK(int, 0, inode_link, struct dentry *old_dentry, struct inode *dir,
-	 struct dentry *new_dentry)
+LSM_HOOK(int, 0, inode_link, const struct mnt_idmap *idmap,
+	 struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry)
 LSM_HOOK(int, 0, inode_unlink, struct inode *dir, struct dentry *dentry)
-LSM_HOOK(int, 0, inode_symlink, struct inode *dir, struct dentry *dentry,
-	 const char *old_name)
-LSM_HOOK(int, 0, inode_mkdir, struct inode *dir, struct dentry *dentry,
-	 umode_t mode)
+LSM_HOOK(int, 0, inode_symlink, const struct mnt_idmap *idmap, struct inode *dir,
+	 struct dentry *dentry, const char *old_name)
+LSM_HOOK(int, 0, inode_mkdir, const struct mnt_idmap *idmap, struct inode *dir,
+	 struct dentry *dentry, umode_t mode)
 LSM_HOOK(int, 0, inode_rmdir, struct inode *dir, struct dentry *dentry)
-LSM_HOOK(int, 0, inode_mknod, struct inode *dir, struct dentry *dentry,
-	 umode_t mode, dev_t dev)
+LSM_HOOK(int, 0, inode_mknod, const struct mnt_idmap *idmap, struct inode *dir,
+	 struct dentry *dentry, umode_t mode, dev_t dev)
 LSM_HOOK(int, 0, inode_rename, struct inode *old_dir, struct dentry *old_dentry,
 	 struct inode *new_dir, struct dentry *new_dentry)
 LSM_HOOK(int, 0, inode_readlink, struct dentry *dentry)
 LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode,
 	 bool rcu)
-LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask)
-LSM_HOOK(int, 0, inode_setattr, struct mnt_idmap *idmap, struct dentry *dentry,
+LSM_HOOK(int, 0, inode_permission, const struct mnt_idmap *idmap,
+	 struct inode *inode, int mask)
+LSM_HOOK(int, 0, inode_setattr, const struct mnt_idmap *idmap, struct dentry *dentry,
 	 struct iattr *attr)
-LSM_HOOK(void, LSM_RET_VOID, inode_post_setattr, struct mnt_idmap *idmap,
+LSM_HOOK(void, LSM_RET_VOID, inode_post_setattr, const struct mnt_idmap *idmap,
 	 struct dentry *dentry, int ia_valid)
 LSM_HOOK(int, 0, inode_getattr, const struct path *path)
 LSM_HOOK(int, 0, inode_xattr_skipcap, const char *name)
-LSM_HOOK(int, 0, inode_setxattr, struct mnt_idmap *idmap,
+LSM_HOOK(int, 0, inode_setxattr, const struct mnt_idmap *idmap,
 	 struct dentry *dentry, const char *name, const void *value,
 	 size_t size, int flags)
 LSM_HOOK(void, LSM_RET_VOID, inode_post_setxattr, struct dentry *dentry,
 	 const char *name, const void *value, size_t size, int flags)
 LSM_HOOK(int, 0, inode_getxattr, struct dentry *dentry, const char *name)
 LSM_HOOK(int, 0, inode_listxattr, struct dentry *dentry)
-LSM_HOOK(int, 0, inode_removexattr, struct mnt_idmap *idmap,
+LSM_HOOK(int, 0, inode_removexattr, const struct mnt_idmap *idmap,
 	 struct dentry *dentry, const char *name)
 LSM_HOOK(void, LSM_RET_VOID, inode_post_removexattr, struct dentry *dentry,
 	 const char *name)
 LSM_HOOK(int, 0, inode_file_setattr, struct dentry *dentry, struct file_kattr *fa)
 LSM_HOOK(int, 0, inode_file_getattr, struct dentry *dentry, struct file_kattr *fa)
-LSM_HOOK(int, 0, inode_set_acl, struct mnt_idmap *idmap,
+LSM_HOOK(int, 0, inode_set_acl, const struct mnt_idmap *idmap,
 	 struct dentry *dentry, const char *acl_name, struct posix_acl *kacl)
 LSM_HOOK(void, LSM_RET_VOID, inode_post_set_acl, struct dentry *dentry,
 	 const char *acl_name, struct posix_acl *kacl)
-LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap,
+LSM_HOOK(int, 0, inode_get_acl, const struct mnt_idmap *idmap,
 	 struct dentry *dentry, const char *acl_name)
-LSM_HOOK(int, 0, inode_remove_acl, struct mnt_idmap *idmap,
+LSM_HOOK(int, 0, inode_remove_acl, const struct mnt_idmap *idmap,
 	 struct dentry *dentry, const char *acl_name)
-LSM_HOOK(void, LSM_RET_VOID, inode_post_remove_acl, struct mnt_idmap *idmap,
+LSM_HOOK(void, LSM_RET_VOID, inode_post_remove_acl, const struct mnt_idmap *idmap,
 	 struct dentry *dentry, const char *acl_name)
 LSM_HOOK(int, 0, inode_need_killpriv, struct dentry *dentry)
-LSM_HOOK(int, 0, inode_killpriv, struct mnt_idmap *idmap,
+LSM_HOOK(int, 0, inode_killpriv, const struct mnt_idmap *idmap,
 	 struct dentry *dentry)
-LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, struct mnt_idmap *idmap,
+LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, const struct mnt_idmap *idmap,
 	 struct inode *inode, const char *name, void **buffer, bool alloc)
 LSM_HOOK(int, -EOPNOTSUPP, inode_setsecurity, struct inode *inode,
 	 const char *name, const void *value, size_t size, int flags)
@@ -187,7 +189,7 @@ LSM_HOOK(int, 0, inode_setintegrity, const struct inode *inode,
 	 enum lsm_integrity_type type, const void *value, size_t size)
 LSM_HOOK(int, 0, kernfs_init_security, struct kernfs_node *kn_dir,
 	 struct kernfs_node *kn)
-LSM_HOOK(int, 0, file_permission, struct file *file, int mask)
+LSM_HOOK(int, 0, file_permission, const struct file *file, int mask)
 LSM_HOOK(int, 0, file_alloc_security, struct file *file)
 LSM_HOOK(void, LSM_RET_VOID, file_release, struct file *file)
 LSM_HOOK(void, LSM_RET_VOID, file_free_security, struct file *file)
@@ -265,6 +267,9 @@ LSM_HOOK(int, -ENOSYS, task_prctl, int option, unsigned long arg2,
 LSM_HOOK(void, LSM_RET_VOID, task_to_inode, struct task_struct *p,
 	 struct inode *inode)
 LSM_HOOK(int, 0, userns_create, const struct cred *cred)
+LSM_HOOK(int, 0, namespace_init, struct ns_common *ns)
+LSM_HOOK(void, LSM_RET_VOID, namespace_free, struct ns_common *ns)
+LSM_HOOK(int, 0, namespace_install, const struct nsset *nsset, struct ns_common *ns)
 LSM_HOOK(int, 0, ipc_permission, struct kern_ipc_perm *ipcp, short flag)
 LSM_HOOK(void, LSM_RET_VOID, ipc_getlsmprop, struct kern_ipc_perm *ipcp,
 	 struct lsm_prop *prop)
diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h
index c4488c4a6d8a99..13621e9e233e6b 100644
--- a/include/linux/lsm_hooks.h
+++ b/include/linux/lsm_hooks.h
@@ -112,6 +112,7 @@ struct lsm_blob_sizes {
 	unsigned int lbs_ipc;
 	unsigned int lbs_key;
 	unsigned int lbs_msg_msg;
+	unsigned int lbs_ns;
 	unsigned int lbs_perf_event;
 	unsigned int lbs_task;
 	unsigned int lbs_xattr_count; /* num xattr slots in new_xattrs array */
diff --git a/include/linux/mailbox/riscv-rpmi-message.h b/include/linux/mailbox/riscv-rpmi-message.h
index e135c6564d0c80..d5362b5821f95a 100644
--- a/include/linux/mailbox/riscv-rpmi-message.h
+++ b/include/linux/mailbox/riscv-rpmi-message.h
@@ -93,6 +93,7 @@ static inline int rpmi_to_linux_error(int rpmi_error)
 /* RPMI service group IDs */
 #define RPMI_SRVGRP_SYSTEM_MSI		0x00002
 #define RPMI_SRVGRP_CLOCK		0x00008
+#define RPMI_SRVGRP_DEVICE_POWER	0x00009
 
 /* RPMI clock service IDs */
 enum rpmi_clock_service_id {
@@ -119,6 +120,16 @@ enum rpmi_sysmsi_service_id {
 	RPMI_SYSMSI_SRV_ID_MAX_COUNT
 };
 
+/* RPMI device power service IDs */
+enum rpmi_device_power_service_id {
+	RPMI_DP_SRV_ENABLE_NOTIFICATION = 0x01,
+	RPMI_DP_SRV_GET_NUM_DOMAINS  = 0x02,
+	RPMI_DP_SRV_GET_ATTRS = 0x03,
+	RPMI_DP_SRV_SET_STATE = 0x04,
+	RPMI_DP_SRV_GET_STATE = 0x05,
+	RPMI_DP_SRV_ID_MAX_COUNT,
+};
+
 /* RPMI Linux mailbox attribute IDs */
 enum rpmi_mbox_attribute_id {
 	RPMI_MBOX_ATTR_SPEC_VERSION,
diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index e595ae5cd0eed4..e30e57f5f3df9f 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -9,6 +9,7 @@
  */
 
 #include <linux/kernel.h>
+#include <linux/compiler.h>
 #include <linux/rcupdate.h>
 #include <linux/spinlock.h>
 
@@ -297,7 +298,8 @@ struct maple_tree {
 #endif
 
 #define DEFINE_MTREE(name)						\
-	struct maple_tree name = MTREE_INIT(name, 0)
+	struct maple_tree name = MTREE_INIT(name, 0);		\
+	ASSERT_STATIC_STORAGE(name)
 
 #define mtree_lock(mt)		spin_lock((&(mt)->ma_lock))
 #define mtree_lock_nested(mas, subclass) \
diff --git a/include/linux/mdio-mux.h b/include/linux/mdio-mux.h
index a5d58f22193967..e2af9cb8e0179a 100644
--- a/include/linux/mdio-mux.h
+++ b/include/linux/mdio-mux.h
@@ -1,5 +1,5 @@
 /*
- * MDIO bus multiplexer framwork.
+ * MDIO bus multiplexer framework.
  *
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file "COPYING" in the main directory of this archive
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index d62db9e776cf94..ea288cf2140084 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -46,11 +46,11 @@ extern unsigned long long max_possible_pfn;
  * @MEMBLOCK_RSRV_KERN: memory region that is reserved for kernel use,
  * either explictitly with memblock_reserve_kern() or via memblock
  * allocation APIs. All memblock allocations set this flag.
- * @MEMBLOCK_KHO_SCRATCH: memory region that kexec can pass to the next
- * kernel in handover mode. During early boot, we do not know about all
- * memory reservations yet, so we get scratch memory from the previous
- * kernel that we know is good to use. It is the only memory that
- * allocations may happen from in this phase.
+ * @MEMBLOCK_KHO_NOPRSRV: memory region with no preservation from kexec
+ * handover. During early boot, we do not know about all memory preservations
+ * yet, so we get memory with no preservations from the previous kernel that we
+ * know is good to use. It is the only memory that allocations may happen from
+ * in this phase.
  * @MEMBLOCK_RSRV_HUGETLB: memory is reserved for hugetlb pages
  */
 enum memblock_flags {
@@ -61,7 +61,7 @@ enum memblock_flags {
 	MEMBLOCK_DRIVER_MANAGED = 0x8,	/* always detected via a driver */
 	MEMBLOCK_RSRV_NOINIT	= 0x10,	/* don't initialize struct pages */
 	MEMBLOCK_RSRV_KERN	= 0x20,	/* memory reserved for kernel use */
-	MEMBLOCK_KHO_SCRATCH	= 0x40,	/* scratch memory for kexec handover */
+	MEMBLOCK_KHO_NOPRSRV	= 0x40,	/* memory with no KHO preservations */
 	MEMBLOCK_RSRV_HUGETLB	= 0x80, /* memory reserved for hugetlb pages */
 };
 
@@ -158,11 +158,10 @@ int memblock_mark_nomap(phys_addr_t base, phys_addr_t size);
 int memblock_clear_nomap(phys_addr_t base, phys_addr_t size);
 int memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size);
 int memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t size);
-int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size);
-int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size);
+int memblock_mark_kho_noprsrv(phys_addr_t base, phys_addr_t size);
+int memblock_clear_kho_noprsrv(phys_addr_t base, phys_addr_t size);
 
 void memblock_free(void *ptr, size_t size);
-void reset_all_zones_managed_pages(void);
 
 /* Low level functions */
 void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
@@ -301,9 +300,9 @@ static inline bool memblock_is_driver_managed(struct memblock_region *m)
 	return m->flags & MEMBLOCK_DRIVER_MANAGED;
 }
 
-static inline bool memblock_is_kho_scratch(struct memblock_region *m)
+static inline bool memblock_is_kho_noprsrv(struct memblock_region *m)
 {
-	return m->flags & MEMBLOCK_KHO_SCRATCH;
+	return m->flags & MEMBLOCK_KHO_NOPRSRV;
 }
 
 int memblock_search_pfn_nid(unsigned long pfn, unsigned long *start_pfn,
@@ -614,12 +613,12 @@ static inline void early_memtest(phys_addr_t start, phys_addr_t end) { }
 static inline void memtest_report_meminfo(struct seq_file *m) { }
 #endif
 
-#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH
-void memblock_set_kho_scratch_only(void);
-void memblock_clear_kho_scratch_only(void);
+#ifdef CONFIG_KEXEC_HANDOVER
+void memblock_set_kho_noprsrv_only(void);
+void memblock_clear_kho_noprsrv_only(void);
 #else
-static inline void memblock_set_kho_scratch_only(void) { }
-static inline void memblock_clear_kho_scratch_only(void) { }
+static inline void memblock_set_kho_noprsrv_only(void) { }
+static inline void memblock_clear_kho_noprsrv_only(void) { }
 #endif
 
 #endif /* _LINUX_MEMBLOCK_H */
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 7d1c0ce189a887..74110a324f9e22 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -23,6 +23,7 @@
 #include <linux/writeback.h>
 #include <linux/page-flags.h>
 #include <linux/shrinker.h>
+#include <linux/irq_work_types.h>
 
 struct mem_cgroup;
 struct obj_cgroup;
@@ -66,11 +67,6 @@ struct mem_cgroup_reclaim_cookie {
 
 #define MEM_CGROUP_ID_SHIFT	16
 
-struct mem_cgroup_private_id {
-	int id;
-	refcount_t ref;
-};
-
 struct memcg_vmstats_percpu;
 struct memcg1_events_percpu;
 struct memcg_vmstats;
@@ -87,50 +83,48 @@ struct mem_cgroup_reclaim_iter {
  * per-node information in memory controller.
  */
 struct mem_cgroup_per_node {
-	/* Keep the read-only fields at the start */
+	/* Set when the memcg is created, then only read. */
+	__cacheline_group_begin_aligned(memcg_pn_read_mostly);
 	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
 						/* use container_of	   */
 
 	struct lruvec_stats_percpu __percpu	*lruvec_stats_percpu;
 	struct lruvec_stats			*lruvec_stats;
 	struct shrinker_info __rcu	*shrinker_info;
+	struct obj_cgroup __rcu		*objcg;
+
+	__cacheline_group_end_aligned(memcg_pn_read_mostly);
 
-#ifdef CONFIG_MEMCG_V1
 	/*
-	 * Memcg-v1 only stuff in middle as buffer between read mostly fields
-	 * and update often fields to avoid false sharing. If v1 stuff is
-	 * not present, an explicit padding is needed.
+	 * Keep lruvec on its own lines. Sharing them with lru_zone_size[]
+	 * regressed, see commit f59adcf59332 ("mm: memcg: add cacheline
+	 * padding after lruvec in mem_cgroup_per_node").
 	 */
-
-	struct rb_node		tree_node;	/* RB tree node */
-	unsigned long		usage_in_excess;/* Set to the value by which */
-						/* the soft limit is exceeded*/
-	bool			on_tree;
-#else
-	CACHELINE_PADDING(_pad1_);
-#endif
-
-	/* Fields which get updated often at the end. */
+	__cacheline_group_begin_aligned(memcg_pn_lruvec);
 	struct lruvec		lruvec;
-	CACHELINE_PADDING(_pad2_);
-	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
+	__cacheline_group_end_aligned(memcg_pn_lruvec);
+
+	/* Written on every LRU update and on every reclaim iteration. */
+	__cacheline_group_begin_aligned(memcg_pn_write_hot);
+	long			lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
 	struct mem_cgroup_reclaim_iter	iter;
+#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
+	/* slab stats for nmi context */
+	atomic_t		slab_reclaimable;
+	atomic_t		slab_unreclaimable;
+#endif
+	__cacheline_group_end_aligned(memcg_pn_write_hot);
 
+	/* Touched only when the memcg is reparented or freed. */
+	__cacheline_group_begin_aligned(memcg_pn_cold);
 	/*
-	 * objcg is wiped out as a part of the objcg repaprenting process.
 	 * orig_objcg preserves a pointer (and a reference) to the original
-	 * objcg until the end of live of memcg.
+	 * objcg until the end of life of memcg.
 	 */
-	struct obj_cgroup __rcu	*objcg;
 	struct obj_cgroup	*orig_objcg;
 	/* list of inherited objcgs, protected by objcg_lock */
 	struct list_head objcg_list;
-
-#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
-	/* slab stats for nmi context */
-	atomic_t		slab_reclaimable;
-	atomic_t		slab_unreclaimable;
-#endif
+	__cacheline_group_end_aligned(memcg_pn_cold);
 };
 
 struct mem_cgroup_threshold {
@@ -186,6 +180,7 @@ struct obj_cgroup {
 	struct percpu_ref refcnt;
 	struct mem_cgroup *memcg;
 	atomic_t nr_charged_bytes;
+	refcount_t private_id_ref;
 	union {
 		struct list_head list; /* protected by objcg_lock */
 		struct rcu_head rcu;
@@ -202,9 +197,6 @@ struct obj_cgroup {
 struct mem_cgroup {
 	struct cgroup_subsys_state css;
 
-	/* Private memcg ID. Used to ID objects that outlive the cgroup */
-	struct mem_cgroup_private_id id;
-
 	/* Accounted resources */
 	struct page_counter memory;		/* Both v1 & v2 */
 
@@ -213,70 +205,59 @@ struct mem_cgroup {
 		struct page_counter memsw;	/* v1 only */
 	};
 
-	/* registered local peak watchers */
-	struct list_head memory_peaks;
-	struct list_head swap_peaks;
-	spinlock_t	 peaks_lock;
-
-	/* Range enforcement for interrupt charges */
-	struct work_struct high_work;
-
-#ifdef CONFIG_ZSWAP
-	unsigned long zswap_max;
-
+	/* Written on the charge, reclaim and socket paths. */
+	__cacheline_group_begin_aligned(memcg_write_hot);
 	/*
-	 * Prevent pages from this memcg from being written back from zswap to
-	 * swap, and from being swapped out on zswap store failures.
+	 * Hint of reclaim pressure for socket memory management. Note
+	 * that this indicator should NOT be used in legacy cgroup mode
+	 * where socket memory is accounted/charged separately.
 	 */
-	bool zswap_writeback;
+	u64			socket_pressure;
+#if BITS_PER_LONG < 64
+	seqlock_t		socket_pressure_seqlock;
 #endif
-
-	/* vmpressure notifications */
-	struct vmpressure vmpressure;
-
 	/*
-	 * Should the OOM killer kill all belonging tasks, had it kill one?
+	 * memory.events is bumped for this memcg and all its ancestors, so a
+	 * busy child dirties every ancestor.
 	 */
-	bool oom_group;
-
-	/* memory.events and memory.events.local */
-	struct cgroup_file events_file;
-	struct cgroup_file events_local_file;
-
-	/* handle for "memory.swap.events" */
-	struct cgroup_file swap_events_file;
-
-	/* memory.stat */
-	struct memcg_vmstats	*vmstats;
-
-	/* memory.events */
 	atomic_long_t		memory_events[MEMCG_NR_MEMORY_EVENTS];
 	atomic_long_t		memory_events_local[MEMCG_NR_MEMORY_EVENTS];
 
+	/* vmpressure notifications. Written on every reclaim iteration. */
+	struct vmpressure vmpressure;
+
 #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
 	/* MEMCG_KMEM for nmi context */
 	atomic_t		kmem_stat;
 #endif
+
+	/* Range enforcement for interrupt charges */
+	struct irq_work high_irq_work;
+	struct work_struct high_work;
+
+	__cacheline_group_end_aligned(memcg_write_hot);
+
 	/*
-	 * Hint of reclaim pressure for socket memroy management. Note
-	 * that this indicator should NOT be used in legacy cgroup mode
-	 * where socket memory is accounted/charged separately.
+	 * Off the charge and fault paths.  Not write free: cgwb_domain is
+	 * written on every writeout completion and mm_list on fork, exit and
+	 * MGLRU aging.  They are grouped here so those writes cannot land on
+	 * a line that the fast paths read.
 	 */
-	u64			socket_pressure;
-#if BITS_PER_LONG < 64
-	seqlock_t		socket_pressure_seqlock;
-#endif
-	int kmemcg_id;
+	__cacheline_group_begin_aligned(memcg_cold);
+	/* registered local peak watchers */
+	struct list_head memory_peaks;
+	struct list_head swap_peaks;
+	spinlock_t	 peaks_lock;
 
-#ifdef CONFIG_CGROUP_WRITEBACK
-	struct list_head cgwb_list;
-#endif
+	/* memory.events and memory.events.local */
+	struct cgroup_file events_file;
+	struct cgroup_file events_local_file;
 
-	/* Keep the hot per-CPU stats pointer away from memory event counters. */
-	struct memcg_vmstats_percpu __percpu *vmstats_percpu
-		____cacheline_aligned_in_smp;
+	/* handle for "memory.swap.events" */
+	struct cgroup_file swap_events_file;
 
 #ifdef CONFIG_CGROUP_WRITEBACK
+	struct list_head cgwb_list;
 	struct wb_domain cgwb_domain;
 	struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT];
 #endif
@@ -285,16 +266,17 @@ struct mem_cgroup {
 	/* per-memcg mm_struct list */
 	struct lru_gen_mm_list mm_list;
 #endif
+	__cacheline_group_end_aligned(memcg_cold);
 
 #ifdef CONFIG_MEMCG_V1
+	/* v1 only. Not grouped: v1 is legacy, sorting it is not worth it. */
+
 	/* Legacy consumer-oriented counters */
 	struct page_counter kmem;		/* v1 only */
 	struct page_counter tcpmem;		/* v1 only */
 
 	struct memcg1_events_percpu __percpu *events_percpu;
 
-	unsigned long soft_limit;
-
 	/* protected by memcg_oom_lock */
 	bool oom_lock;
 	int under_oom;
@@ -325,6 +307,44 @@ struct mem_cgroup {
 	int swappiness;
 #endif /* CONFIG_MEMCG_V1 */
 
+	/*
+	 * Set when the memcg is created and cleared when it is offlined.
+	 * Never written on a hot path.
+	 */
+	__cacheline_group_begin_aligned(memcg_read_mostly);
+	/* Read on every stat update */
+	struct memcg_vmstats_percpu __percpu *vmstats_percpu;
+
+	/* memory.stat */
+	struct memcg_vmstats	*vmstats;
+
+#ifdef CONFIG_ZSWAP
+	unsigned long zswap_max;
+#endif
+
+	/* The objcg holding private memcg ID. */
+	struct obj_cgroup *private_id_objcg;
+
+	/* Private memcg ID. Used to ID objects that outlive the cgroup */
+	int private_id;
+
+	int kmemcg_id;
+
+	/*
+	 * Should the OOM killer kill all belonging tasks, had it kill one?
+	 */
+	bool oom_group;
+
+#ifdef CONFIG_ZSWAP
+	/*
+	 * Prevent pages from this memcg from being written back from zswap to
+	 * swap, and from being swapped out on zswap store failures.
+	 */
+	bool zswap_writeback;
+#endif
+	/* Not padded: nodeinfo[] is read-mostly too, let it share the line. */
+	__cacheline_group_end(memcg_read_mostly);
+
 	struct mem_cgroup_per_node *nodeinfo[];
 };
 
@@ -377,10 +397,10 @@ enum objext_flags {
  *
  * The caller must ensure that the returned memcg won't be released.
  */
-static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
+static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg)
 {
 	lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex));
-	return READ_ONCE(objcg->memcg);
+	return objcg ? READ_ONCE(objcg->memcg) : NULL;
 }
 
 /*
@@ -391,7 +411,7 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg)
  * or NULL. This function assumes that the folio is known to have a
  * proper object cgroup pointer.
  */
-static inline struct obj_cgroup *folio_objcg(struct folio *folio)
+static inline struct obj_cgroup *folio_objcg(const struct folio *folio)
 {
 	unsigned long memcg_data = folio->memcg_data;
 
@@ -429,11 +449,11 @@ static inline struct obj_cgroup *folio_objcg(struct folio *folio)
  * Note: The caller should hold an rcu read lock or cgroup_mutex to protect
  * memcg associated with a folio from being released.
  */
-static inline struct mem_cgroup *folio_memcg(struct folio *folio)
+static inline struct mem_cgroup *folio_memcg(const struct folio *folio)
 {
 	struct obj_cgroup *objcg = folio_objcg(folio);
 
-	return objcg ? obj_cgroup_memcg(objcg) : NULL;
+	return obj_cgroup_memcg(objcg);
 }
 
 /*
@@ -442,7 +462,7 @@ static inline struct mem_cgroup *folio_memcg(struct folio *folio)
  *
  * Returns true if folio is charged to a memory cgroup, otherwise returns false.
  */
-static inline bool folio_memcg_charged(struct folio *folio)
+static inline bool folio_memcg_charged(const struct folio *folio)
 {
 	return folio->memcg_data != 0;
 }
@@ -462,7 +482,7 @@ static inline bool folio_memcg_charged(struct folio *folio)
  * A caller should hold an rcu read lock to protect memcg associated with a
  * page from being released.
  */
-static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
+static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio)
 {
 	/*
 	 * Because folio->memcg_data might be changed asynchronously
@@ -476,17 +496,17 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
 
 	objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
 
-	return objcg ? obj_cgroup_memcg(objcg) : NULL;
+	return obj_cgroup_memcg(objcg);
 }
 
-static inline struct mem_cgroup *page_memcg_check(struct page *page)
+static inline struct mem_cgroup *page_memcg_check(const struct page *page)
 {
 	if (PageTail(page))
 		return NULL;
-	return folio_memcg_check((struct folio *)page);
+	return folio_memcg_check((const struct folio *)page);
 }
 
-static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)
+static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg)
 {
 	struct mem_cgroup *memcg;
 
@@ -508,19 +528,19 @@ static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *ob
  * that the folio has an associated memory cgroup. It's not safe to call
  * this function against some types of folios, e.g. slab folios.
  */
-static inline bool folio_memcg_kmem(struct folio *folio)
+static inline bool folio_memcg_kmem(const struct folio *folio)
 {
 	VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page);
 	VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio);
 	return folio->memcg_data & MEMCG_DATA_KMEM;
 }
 
-static inline bool PageMemcgKmem(struct page *page)
+static inline bool PageMemcgKmem(const struct page *page)
 {
 	return folio_memcg_kmem(page_folio(page));
 }
 
-static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
+static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg)
 {
 	return (memcg == root_mem_cgroup);
 }
@@ -536,7 +556,7 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
  * and do not honour sc->memcg can use this to early-return 0 in per-memcg
  * contexts.
  */
-static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc)
+static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc)
 {
 	return !sc->memcg || mem_cgroup_is_root(sc->memcg);
 }
@@ -551,8 +571,8 @@ static inline bool mem_cgroup_disabled(void)
 	return !cgroup_subsys_enabled(memory_cgrp_subsys);
 }
 
-static inline void mem_cgroup_protection(struct mem_cgroup *root,
-					 struct mem_cgroup *memcg,
+static inline void mem_cgroup_protection(const struct mem_cgroup *root,
+					 const struct mem_cgroup *memcg,
 					 unsigned long *min,
 					 unsigned long *low,
 					 unsigned long *usage)
@@ -606,8 +626,8 @@ static inline void mem_cgroup_protection(struct mem_cgroup *root,
 void mem_cgroup_calculate_protection(struct mem_cgroup *root,
 				     struct mem_cgroup *memcg);
 
-static inline bool mem_cgroup_unprotected(struct mem_cgroup *target,
-					  struct mem_cgroup *memcg)
+static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target,
+					  const struct mem_cgroup *memcg)
 {
 	/*
 	 * The root memcg doesn't account charges, and doesn't support
@@ -618,8 +638,8 @@ static inline bool mem_cgroup_unprotected(struct mem_cgroup *target,
 		memcg == target;
 }
 
-static inline bool mem_cgroup_below_low(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+static inline bool mem_cgroup_below_low(const struct mem_cgroup *target,
+					const struct mem_cgroup *memcg)
 {
 	if (mem_cgroup_unprotected(target, memcg))
 		return false;
@@ -628,8 +648,8 @@ static inline bool mem_cgroup_below_low(struct mem_cgroup *target,
 		page_counter_read(&memcg->memory);
 }
 
-static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+static inline bool mem_cgroup_below_min(const struct mem_cgroup *target,
+					const struct mem_cgroup *memcg)
 {
 	if (mem_cgroup_unprotected(target, memcg))
 		return false;
@@ -662,7 +682,8 @@ static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm,
 	return __mem_cgroup_charge(folio, mm, gfp);
 }
 
-int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp);
+int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm,
+			      gfp_t gfp);
 
 int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id,
 				   struct mm_struct *mm, gfp_t gfp);
@@ -702,7 +723,7 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new);
  * @pgdat combination. This can be the node lruvec, if the memory
  * controller is disabled.
  */
-static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
+static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg,
 					       struct pglist_data *pgdat)
 {
 	struct mem_cgroup_per_node *mz;
@@ -743,7 +764,7 @@ static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
  * their binding is stable if the returned lruvec matches the one the caller has
  * locked. Useful for lock batching.
  */
-static inline struct lruvec *folio_lruvec(struct folio *folio)
+static inline struct lruvec *folio_lruvec(const struct folio *folio)
 {
 	struct mem_cgroup *memcg = folio_memcg(folio);
 
@@ -757,11 +778,11 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
 
 struct mem_cgroup *get_mem_cgroup_from_current(void);
 
-struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio);
+struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio);
 
-struct lruvec *folio_lruvec_lock(struct folio *folio);
-struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
-struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
+struct lruvec *folio_lruvec_lock(const struct folio *folio);
+struct lruvec *folio_lruvec_lock_irq(const struct folio *folio);
+struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio,
 						unsigned long *flags);
 
 static inline
@@ -820,16 +841,16 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
 void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
 			   int (*)(struct task_struct *, void *), void *arg);
 
-static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg)
+static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg)
 {
 	if (mem_cgroup_disabled())
 		return 0;
 
-	return memcg->id.id;
+	return memcg->private_id;
 }
 struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id);
 
-static inline u64 mem_cgroup_id(struct mem_cgroup *memcg)
+static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg)
 {
 	return memcg ? cgroup_id(memcg->css.cgroup) : 0;
 }
@@ -841,14 +862,14 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
 	return mem_cgroup_from_css(seq_css(m));
 }
 
-static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
+static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec)
 {
-	struct mem_cgroup_per_node *mz;
+	const struct mem_cgroup_per_node *mz;
 
 	if (mem_cgroup_disabled())
 		return NULL;
 
-	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
+	mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec);
 	return mz->memcg;
 }
 
@@ -858,13 +879,13 @@ static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
  *
  * Returns the parent memcg, or NULL if this is the root.
  */
-static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
+static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg)
 {
 	return mem_cgroup_from_css(memcg->css.parent);
 }
 
-static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
-			      struct mem_cgroup *root)
+static inline bool mem_cgroup_is_descendant(const struct mem_cgroup *memcg,
+			      const struct mem_cgroup *root)
 {
 	if (root == memcg)
 		return true;
@@ -872,7 +893,7 @@ static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
 }
 
 static inline bool mm_match_cgroup(struct mm_struct *mm,
-				   struct mem_cgroup *memcg)
+				   const struct mem_cgroup *memcg)
 {
 	struct mem_cgroup *task_memcg;
 	bool match = false;
@@ -885,8 +906,8 @@ static inline bool mm_match_cgroup(struct mm_struct *mm,
 	return match;
 }
 
-struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio);
-ino_t page_cgroup_ino(struct page *page);
+struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio);
+ino_t page_cgroup_ino(const struct page *page);
 
 static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
 {
@@ -899,13 +920,18 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
 		int zid, long nr_pages);
 
 static inline
-unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
+unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec,
 		enum lru_list lru, int zone_idx)
 {
-	struct mem_cgroup_per_node *mz;
+	long val;
+	const struct mem_cgroup_per_node *mz;
+
+	mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec);
+	val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]);
+	if (WARN_ON_ONCE(val < 0))
+		return 0;
 
-	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
-	return READ_ONCE(mz->lru_zone_size[zone_idx][lru]);
+	return val;
 }
 
 void __mem_cgroup_handle_over_high(gfp_t gfp_mask);
@@ -916,9 +942,9 @@ static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask)
 		__mem_cgroup_handle_over_high(gfp_mask);
 }
 
-unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg);
+unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg);
 
-void mem_cgroup_print_oom_context(struct mem_cgroup *memcg,
+void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg,
 				struct task_struct *p);
 
 void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg);
@@ -931,7 +957,7 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg);
 void mod_memcg_state(struct mem_cgroup *memcg,
 		     enum memcg_stat_item idx, int val);
 
-static inline void mod_memcg_page_state(struct page *page,
+static inline void mod_memcg_page_state(const struct page *page,
 					enum memcg_stat_item idx, int val)
 {
 	struct mem_cgroup *memcg;
@@ -946,17 +972,21 @@ static inline void mod_memcg_page_state(struct page *page,
 	rcu_read_unlock();
 }
 
-unsigned long memcg_events(struct mem_cgroup *memcg, int event);
-unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx);
-unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item);
+unsigned long memcg_events(const struct mem_cgroup *memcg, int event);
+unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx);
+unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item);
 bool memcg_stat_item_valid(int idx);
 bool memcg_vm_event_item_valid(enum vm_event_item idx);
-unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx);
-unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+unsigned long lruvec_page_state(const struct lruvec *lruvec,
+				enum node_stat_item idx);
+unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec,
 					  enum node_stat_item idx);
-unsigned long lruvec_page_state_local(struct lruvec *lruvec,
+unsigned long lruvec_page_state_local(const struct lruvec *lruvec,
 				      enum node_stat_item idx);
 
+void mod_memcg_lruvec_state(struct lruvec *lruvec,
+			    enum node_stat_item idx, int val);
+
 void mem_cgroup_flush_stats(struct mem_cgroup *memcg);
 void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg);
 
@@ -965,7 +995,7 @@ void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val);
 void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
 			unsigned long count);
 
-static inline void count_memcg_folio_events(struct folio *folio,
+static inline void count_memcg_folio_events(const struct folio *folio,
 		enum vm_event_item idx, unsigned long nr)
 {
 	struct mem_cgroup *memcg;
@@ -1050,51 +1080,56 @@ void mem_cgroup_flush_workqueue(void);
 extern int mem_cgroup_init(void);
 #else /* CONFIG_MEMCG */
 
+static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg)
+{
+	return NULL;
+}
+
 #define MEM_CGROUP_ID_SHIFT	0
 
 #define root_mem_cgroup		(NULL)
 
-static inline struct mem_cgroup *folio_memcg(struct folio *folio)
+static inline struct mem_cgroup *folio_memcg(const struct folio *folio)
 {
 	return NULL;
 }
 
-static inline bool folio_memcg_charged(struct folio *folio)
+static inline bool folio_memcg_charged(const struct folio *folio)
 {
 	return false;
 }
 
-static inline struct mem_cgroup *folio_memcg_check(struct folio *folio)
+static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio)
 {
 	return NULL;
 }
 
-static inline struct mem_cgroup *page_memcg_check(struct page *page)
+static inline struct mem_cgroup *page_memcg_check(const struct page *page)
 {
 	return NULL;
 }
 
-static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg)
+static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg)
 {
 	return NULL;
 }
 
-static inline bool folio_memcg_kmem(struct folio *folio)
+static inline bool folio_memcg_kmem(const struct folio *folio)
 {
 	return false;
 }
 
-static inline bool PageMemcgKmem(struct page *page)
+static inline bool PageMemcgKmem(const struct page *page)
 {
 	return false;
 }
 
-static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
+static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg)
 {
 	return true;
 }
 
-static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc)
+static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc)
 {
 	return true;
 }
@@ -1119,8 +1154,8 @@ static inline void memcg_memory_event_mm(struct mm_struct *mm,
 {
 }
 
-static inline void mem_cgroup_protection(struct mem_cgroup *root,
-					 struct mem_cgroup *memcg,
+static inline void mem_cgroup_protection(const struct mem_cgroup *root,
+					 const struct mem_cgroup *memcg,
 					 unsigned long *min,
 					 unsigned long *low,
 					 unsigned long *usage)
@@ -1133,19 +1168,19 @@ static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root,
 {
 }
 
-static inline bool mem_cgroup_unprotected(struct mem_cgroup *target,
-					  struct mem_cgroup *memcg)
+static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target,
+					  const struct mem_cgroup *memcg)
 {
 	return true;
 }
-static inline bool mem_cgroup_below_low(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+static inline bool mem_cgroup_below_low(const struct mem_cgroup *target,
+					const struct mem_cgroup *memcg)
 {
 	return false;
 }
 
-static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+static inline bool mem_cgroup_below_min(const struct mem_cgroup *target,
+					const struct mem_cgroup *memcg)
 {
 	return false;
 }
@@ -1156,9 +1191,10 @@ static inline int mem_cgroup_charge(struct folio *folio,
 	return 0;
 }
 
-static inline int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp)
+static inline int mem_cgroup_charge_hugetlb(struct folio *folio,
+					    struct mm_struct *mm, gfp_t gfp)
 {
-        return 0;
+	return 0;
 }
 
 static inline int mem_cgroup_swapin_charge_folio(struct folio *folio,
@@ -1184,25 +1220,25 @@ static inline void mem_cgroup_migrate(struct folio *old, struct folio *new)
 {
 }
 
-static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
+static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg,
 					       struct pglist_data *pgdat)
 {
 	return &pgdat->__lruvec;
 }
 
-static inline struct lruvec *folio_lruvec(struct folio *folio)
+static inline struct lruvec *folio_lruvec(const struct folio *folio)
 {
 	struct pglist_data *pgdat = folio_pgdat(folio);
 	return &pgdat->__lruvec;
 }
 
-static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
+static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg)
 {
 	return NULL;
 }
 
 static inline bool mm_match_cgroup(struct mm_struct *mm,
-		struct mem_cgroup *memcg)
+		const struct mem_cgroup *memcg)
 {
 	return true;
 }
@@ -1217,7 +1253,7 @@ static inline struct mem_cgroup *get_mem_cgroup_from_current(void)
 	return NULL;
 }
 
-static inline struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio)
+static inline struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio)
 {
 	return NULL;
 }
@@ -1250,7 +1286,7 @@ static inline void mem_cgroup_put(struct mem_cgroup *memcg)
 {
 }
 
-static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
+static inline struct lruvec *folio_lruvec_lock(const struct folio *folio)
 {
 	struct pglist_data *pgdat = folio_pgdat(folio);
 
@@ -1259,7 +1295,7 @@ static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
 	return &pgdat->__lruvec;
 }
 
-static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
+static inline struct lruvec *folio_lruvec_lock_irq(const struct folio *folio)
 {
 	struct pglist_data *pgdat = folio_pgdat(folio);
 
@@ -1268,7 +1304,7 @@ static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
 	return &pgdat->__lruvec;
 }
 
-static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
+static inline struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio,
 		unsigned long *flagsp)
 {
 	struct pglist_data *pgdat = folio_pgdat(folio);
@@ -1296,7 +1332,7 @@ static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
 {
 }
 
-static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg)
+static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg)
 {
 	return 0;
 }
@@ -1308,7 +1344,7 @@ static inline struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id)
 	return NULL;
 }
 
-static inline u64 mem_cgroup_id(struct mem_cgroup *memcg)
+static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg)
 {
 	return 0;
 }
@@ -1323,7 +1359,7 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
 	return NULL;
 }
 
-static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
+static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec)
 {
 	return NULL;
 }
@@ -1334,19 +1370,20 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
 }
 
 static inline
-unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
+unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec,
 		enum lru_list lru, int zone_idx)
 {
 	return 0;
 }
 
-static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
+static inline unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg)
 {
 	return 0;
 }
 
 static inline void
-mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
+mem_cgroup_print_oom_context(const struct mem_cgroup *memcg,
+			     struct task_struct *p)
 {
 }
 
@@ -1375,17 +1412,17 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg,
 {
 }
 
-static inline void mod_memcg_page_state(struct page *page,
+static inline void mod_memcg_page_state(const struct page *page,
 					enum memcg_stat_item idx, int val)
 {
 }
 
-static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
+static inline unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx)
 {
 	return 0;
 }
 
-static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item)
+static inline unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item)
 {
 	return 0;
 }
@@ -1400,24 +1437,29 @@ static inline bool memcg_vm_event_item_valid(enum vm_event_item idx)
 	return false;
 }
 
-static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
+static inline unsigned long lruvec_page_state(const struct lruvec *lruvec,
 					      enum node_stat_item idx)
 {
 	return node_page_state(lruvec_pgdat(lruvec), idx);
 }
 
-static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+static inline unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec,
 							enum node_stat_item idx)
 {
 	return node_page_state_monotonic(lruvec_pgdat(lruvec), idx);
 }
 
-static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
+static inline unsigned long lruvec_page_state_local(const struct lruvec *lruvec,
 						    enum node_stat_item idx)
 {
 	return node_page_state(lruvec_pgdat(lruvec), idx);
 }
 
+static inline void mod_memcg_lruvec_state(struct lruvec *lruvec,
+					  enum node_stat_item idx, int val)
+{
+}
+
 static inline void mem_cgroup_flush_stats(struct mem_cgroup *memcg)
 {
 }
@@ -1440,7 +1482,7 @@ static inline void count_memcg_events(struct mem_cgroup *memcg,
 {
 }
 
-static inline void count_memcg_folio_events(struct folio *folio,
+static inline void count_memcg_folio_events(const struct folio *folio,
 		enum vm_event_item idx, unsigned long nr)
 {
 }
@@ -1474,7 +1516,7 @@ static inline void mem_cgroup_flush_workqueue(void) { }
 static inline int mem_cgroup_init(void) { return 0; }
 #endif /* CONFIG_MEMCG */
 
-static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
+static inline struct lruvec *parent_lruvec(const struct lruvec *lruvec)
 {
 	struct mem_cgroup *memcg;
 
@@ -1537,15 +1579,15 @@ static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long
 }
 
 /* Test requires a stable folio->memcg binding, see folio_memcg() */
-static inline bool folio_matches_lruvec(struct folio *folio,
-		struct lruvec *lruvec)
+static inline bool folio_matches_lruvec(const struct folio *folio,
+		const struct lruvec *lruvec)
 {
 	return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&
 	       lruvec_memcg(lruvec) == folio_memcg(folio);
 }
 
 /* Don't lock again iff page's lruvec locked */
-static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
+static inline struct lruvec *folio_lruvec_relock_irq(const struct folio *folio,
 		struct lruvec *locked_lruvec)
 {
 	if (locked_lruvec) {
@@ -1559,7 +1601,7 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
 }
 
 /* Don't lock again iff folio's lruvec locked */
-static inline void folio_lruvec_relock_irqsave(struct folio *folio,
+static inline void folio_lruvec_relock_irqsave(const struct folio *folio,
 		struct lruvec **lruvecp, unsigned long *flags)
 {
 	if (*lruvecp) {
@@ -1651,7 +1693,7 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg)
 	write_sequnlock_irqrestore(&memcg->socket_pressure_seqlock, flags);
 }
 
-static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg)
+static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg)
 {
 	unsigned int seq;
 	u64 val;
@@ -1669,7 +1711,7 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg)
 	WRITE_ONCE(memcg->socket_pressure, jiffies + HZ);
 }
 
-static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg)
+static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg)
 {
 	return READ_ONCE(memcg->socket_pressure);
 }
@@ -1736,7 +1778,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order);
  * needs to be used outside of the local scope.
  */
 struct obj_cgroup *current_obj_cgroup(void);
-struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio);
+struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio);
 
 static inline struct obj_cgroup *get_obj_cgroup_from_current(void)
 {
@@ -1782,7 +1824,7 @@ static inline void memcg_kmem_uncharge_page(struct page *page, int order)
  * A helper for accessing memcg's kmem_id, used for getting
  * corresponding LRU lists.
  */
-static inline int memcg_kmem_id(struct mem_cgroup *memcg)
+static inline int memcg_kmem_id(const struct mem_cgroup *memcg)
 {
 	return memcg ? memcg->kmemcg_id : -1;
 }
@@ -1839,7 +1881,7 @@ static inline void __memcg_kmem_uncharge_page(struct page *page, int order)
 {
 }
 
-static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)
+static inline struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio)
 {
 	return NULL;
 }
@@ -1854,7 +1896,7 @@ static inline bool memcg_kmem_online(void)
 	return false;
 }
 
-static inline int memcg_kmem_id(struct mem_cgroup *memcg)
+static inline int memcg_kmem_id(const struct mem_cgroup *memcg)
 {
 	return -1;
 }
@@ -1870,7 +1912,7 @@ static inline void count_objcg_events(struct obj_cgroup *objcg,
 {
 }
 
-static inline ino_t page_cgroup_ino(struct page *page)
+static inline ino_t page_cgroup_ino(const struct page *page)
 {
 	return 0;
 }
@@ -1890,11 +1932,21 @@ static inline bool memcg_is_dying(struct mem_cgroup *memcg)
 }
 #endif /* CONFIG_MEMCG */
 
+#if defined(CONFIG_MEMCG) && defined(CONFIG_LRU_GEN)
+void mem_cgroup_calculate_protection_path(struct mem_cgroup *root,
+					  struct mem_cgroup *memcg);
+#else
+static inline void mem_cgroup_calculate_protection_path(struct mem_cgroup *root,
+							struct mem_cgroup *memcg)
+{
+}
+#endif
+
 #if defined(CONFIG_MEMCG) && defined(CONFIG_ZSWAP)
 bool obj_cgroup_may_zswap(struct obj_cgroup *objcg);
 void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size);
 void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size);
-bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg);
+bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg);
 #else
 static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg)
 {
@@ -1908,7 +1960,7 @@ static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg,
 					     size_t size)
 {
 }
-static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg)
+static inline bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg)
 {
 	/* if zswap is disabled, do not block pages going to the swapping device */
 	return true;
@@ -1919,13 +1971,9 @@ static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg)
 /* Cgroup v1-related declarations */
 
 #ifdef CONFIG_MEMCG_V1
-unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order,
-					gfp_t gfp_mask,
-					unsigned long *total_scanned);
-
 bool mem_cgroup_oom_synchronize(bool wait);
 
-static inline bool task_in_memcg_oom(struct task_struct *p)
+static inline bool task_in_memcg_oom(const struct task_struct *p)
 {
 	return p->memcg_in_oom;
 }
@@ -1943,15 +1991,7 @@ static inline void mem_cgroup_exit_user_fault(void)
 }
 
 #else /* CONFIG_MEMCG_V1 */
-static inline
-unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order,
-					gfp_t gfp_mask,
-					unsigned long *total_scanned)
-{
-	return 0;
-}
-
-static inline bool task_in_memcg_oom(struct task_struct *p)
+static inline bool task_in_memcg_oom(const struct task_struct *p)
 {
 	return false;
 }
diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h
index 65c732d440d2f4..ba262af5386337 100644
--- a/include/linux/mempolicy.h
+++ b/include/linux/mempolicy.h
@@ -13,7 +13,6 @@
 #include <linux/spinlock.h>
 #include <linux/node.h>
 #include <linux/nodemask.h>
-#include <linux/pagemap.h>
 #include <uapi/linux/mempolicy.h>
 
 struct mm_struct;
diff --git a/include/linux/mempool.h b/include/linux/mempool.h
index a0fa6d43e0dc4e..6da502aef2f718 100644
--- a/include/linux/mempool.h
+++ b/include/linux/mempool.h
@@ -70,6 +70,13 @@ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem,
 #define mempool_alloc_bulk(...)						\
 	alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__))
 
+/*
+ * Allocate a new element without dipping into the pool's reserves or
+ * waiting.  Returns NULL on failure.
+ */
+#define mempool_alloc_noreserve(_pool, _gfp)				\
+	alloc_hooks((_pool)->alloc(_gfp, (_pool)->pool_data))
+
 void *mempool_alloc_preallocated(struct mempool *pool) __malloc;
 void mempool_free(void *element, struct mempool *pool);
 unsigned int mempool_free_bulk(struct mempool *pool, void **elem,
diff --git a/include/linux/mfd/88pm886.h b/include/linux/mfd/88pm886.h
index 2c24dd3032ab3d..9e96d2cb92f570 100644
--- a/include/linux/mfd/88pm886.h
+++ b/include/linux/mfd/88pm886.h
@@ -2,6 +2,7 @@
 #ifndef __MFD_88PM886_H
 #define __MFD_88PM886_H
 
+#include <linux/bits.h>
 #include <linux/i2c.h>
 #include <linux/regmap.h>
 
@@ -130,6 +131,12 @@
 #define PM886_GPADC_INDEX_TO_BIAS_uA(i)	(1 + (i) * 5)
 
 /* Battery block register definitions */
+#define PM886_REG_BATTERY_CONFIG1	0x28
+#define PM886_REG_VBUS_EN		BIT(7)
+
+#define PM886_REG_BOOST_CONFIG1		0x6b
+#define PM886_REG_BOOST_MASK		GENMASK(2, 0)
+
 #define PM886_REG_CLS_CONFIG1		0x71
 
 struct pm886_chip {
diff --git a/include/linux/mfd/cs42l43-regs.h b/include/linux/mfd/cs42l43-regs.h
index 68831f113589df..4c00ceae8b4610 100644
--- a/include/linux/mfd/cs42l43-regs.h
+++ b/include/linux/mfd/cs42l43-regs.h
@@ -1183,6 +1183,7 @@
 
 /* CS42L43B VARIANT REGISTERS */
 #define CS42L43B_DEVID_VAL					0x0042A43B
+#define CS42L44_DEVID_VAL					0x00042A44
 
 #define CS42L43B_DECIM_VOL_CTRL_CH1_CH2				0x00008280
 #define CS42L43B_DECIM_VOL_CTRL_CH3_CH4				0x00008284
diff --git a/include/linux/mfd/da9150/core.h b/include/linux/mfd/da9150/core.h
index d116d5f3ef5612..369698036d1aa5 100644
--- a/include/linux/mfd/da9150/core.h
+++ b/include/linux/mfd/da9150/core.h
@@ -65,6 +65,7 @@ struct da9150 {
 	struct regmap_irq_chip_data *regmap_irq_data;
 	int irq;
 	int irq_base;
+	bool irq_wake_enabled;
 };
 
 /* Device I/O - Query Interface for FG and standard register access */
diff --git a/include/linux/mfd/khadas-mcu.h b/include/linux/mfd/khadas-mcu.h
index a99ba2ed0e4e0f..acd3291061b40e 100644
--- a/include/linux/mfd/khadas-mcu.h
+++ b/include/linux/mfd/khadas-mcu.h
@@ -70,6 +70,13 @@
 #define KHADAS_MCU_WOL_INIT_START_REG		0x87 /* WO */
 #define KHADAS_MCU_CMD_FAN_STATUS_CTRL_REG	0x88 /* WO */
 
+/* VIM4 specific registers */
+#define KHADAS_MCU_VIM4_REST_CONF_REG		0x2c /* WO - reset EEPROM */
+#define KHADAS_MCU_VIM4_LED_ON_RAM_REG		0x89 /* WO - LED volatile */
+#define KHADAS_MCU_VIM4_FAN_CTRL_REG		0x8a /* WO */
+#define KHADAS_MCU_VIM4_WDT_EN_REG		0x8b /* WO */
+#define KHADAS_MCU_VIM4_SYS_RST_REG		0x91 /* WO */
+
 enum {
 	KHADAS_BOARD_VIM1 = 0x1,
 	KHADAS_BOARD_VIM2,
@@ -80,7 +87,7 @@ enum {
 
 /**
  * struct khadas_mcu - Khadas MCU structure
- * @device:		device reference used for logs
+ * @dev:		device reference used for logs
  * @regmap:		register map
  */
 struct khadas_mcu {
@@ -88,4 +95,14 @@ struct khadas_mcu {
 	struct regmap *regmap;
 };
 
+/**
+ * enum khadas_mcu_type - Khadas MCU hardware variant
+ * @KHADAS_MCU_GENERIC: VIM1, VIM2, VIM3, Edge, Edge-V (shared register map)
+ * @KHADAS_MCU_VIM4: VIM4 (extended register map, distinct fan/LED/WDT regs)
+ */
+enum khadas_mcu_type {
+	KHADAS_MCU_GENERIC = 1,
+	KHADAS_MCU_VIM4,
+};
+
 #endif /* MFD_KHADAS_MCU_H */
diff --git a/include/linux/mfd/lm3533.h b/include/linux/mfd/lm3533.h
index 69059a7a2ce534..8f72dd41e8f020 100644
--- a/include/linux/mfd/lm3533.h
+++ b/include/linux/mfd/lm3533.h
@@ -15,9 +15,14 @@
 #define LM3533_ATTR_RW(_name) \
 	DEVICE_ATTR(_name, S_IRUGO | S_IWUSR , show_##_name, store_##_name)
 
+#define LM3533_MAX_CURRENT_MIN		5000
+#define LM3533_MAX_CURRENT_MAX		29800
+#define LM3533_MAX_CURRENT_STEP		800
+
 struct device;
 struct gpio_desc;
 struct regmap;
+struct regulator;
 
 struct lm3533 {
 	struct device *dev;
@@ -25,7 +30,10 @@ struct lm3533 {
 	struct regmap *regmap;
 
 	struct gpio_desc *hwen;
-	int irq;
+	struct regulator *vin_supply;
+
+	u32 boost_ovp;
+	u32 boost_freq;
 
 	unsigned have_als:1;
 	unsigned have_backlights:1;
@@ -33,67 +41,18 @@ struct lm3533 {
 };
 
 struct lm3533_ctrlbank {
-	struct lm3533 *lm3533;
+	struct regmap *regmap;
 	struct device *dev;
 	int id;
 };
 
-struct lm3533_als_platform_data {
-	unsigned pwm_mode:1;		/* PWM input mode (default analog) */
-	u8 r_select;			/* 1 - 127 (ignored in PWM-mode) */
-};
-
-struct lm3533_bl_platform_data {
-	char *name;
-	u16 max_current;		/* 5000 - 29800 uA (800 uA step) */
-	u8 default_brightness;		/* 0 - 255 */
-	u8 pwm;				/* 0 - 0x3f */
-};
-
-struct lm3533_led_platform_data {
-	char *name;
-	const char *default_trigger;
-	u16 max_current;		/* 5000 - 29800 uA (800 uA step) */
-	u8 pwm;				/* 0 - 0x3f */
-};
-
-enum lm3533_boost_freq {
-	LM3533_BOOST_FREQ_500KHZ,
-	LM3533_BOOST_FREQ_1000KHZ,
-};
-
-enum lm3533_boost_ovp {
-	LM3533_BOOST_OVP_16V,
-	LM3533_BOOST_OVP_24V,
-	LM3533_BOOST_OVP_32V,
-	LM3533_BOOST_OVP_40V,
-};
-
-struct lm3533_platform_data {
-	enum lm3533_boost_ovp boost_ovp;
-	enum lm3533_boost_freq boost_freq;
-
-	struct lm3533_als_platform_data *als;
-
-	struct lm3533_bl_platform_data *backlights;
-	int num_backlights;
-
-	struct lm3533_led_platform_data *leds;
-	int num_leds;
-};
-
-extern int lm3533_ctrlbank_enable(struct lm3533_ctrlbank *cb);
-extern int lm3533_ctrlbank_disable(struct lm3533_ctrlbank *cb);
-
-extern int lm3533_ctrlbank_set_brightness(struct lm3533_ctrlbank *cb, u8 val);
-extern int lm3533_ctrlbank_get_brightness(struct lm3533_ctrlbank *cb, u8 *val);
-extern int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb,
-								u16 imax);
-extern int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u8 val);
-extern int lm3533_ctrlbank_get_pwm(struct lm3533_ctrlbank *cb, u8 *val);
+int lm3533_ctrlbank_enable(struct lm3533_ctrlbank *cb);
+int lm3533_ctrlbank_disable(struct lm3533_ctrlbank *cb);
 
-extern int lm3533_read(struct lm3533 *lm3533, u8 reg, u8 *val);
-extern int lm3533_write(struct lm3533 *lm3533, u8 reg, u8 val);
-extern int lm3533_update(struct lm3533 *lm3533, u8 reg, u8 val, u8 mask);
+int lm3533_ctrlbank_set_brightness(struct lm3533_ctrlbank *cb, u32 val);
+int lm3533_ctrlbank_get_brightness(struct lm3533_ctrlbank *cb, u32 *val);
+int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb, u32 imax);
+int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u32 val);
+int lm3533_ctrlbank_get_pwm(struct lm3533_ctrlbank *cb, u32 *val);
 
 #endif	/* __LINUX_MFD_LM3533_H */
diff --git a/include/linux/mfd/motorola-cpcap.h b/include/linux/mfd/motorola-cpcap.h
index 981e5777deb747..bb23363eeccdfe 100644
--- a/include/linux/mfd/motorola-cpcap.h
+++ b/include/linux/mfd/motorola-cpcap.h
@@ -25,6 +25,13 @@
 #define CPCAP_REVISION_2_0	0x10
 #define CPCAP_REVISION_2_1	0x11
 
+enum cpcap_variant {
+	CPCAP_DEFAULT = 1,
+	CPCAP_MAPPHONE,
+	CPCAP_MOT,
+	CPCAP_MAX
+};
+
 /* CPCAP registers */
 #define CPCAP_REG_INT1		0x0000	/* Interrupt 1 */
 #define CPCAP_REG_INT2		0x0004	/* Interrupt 2 */
diff --git a/include/linux/mfd/rohm-bd73800.h b/include/linux/mfd/rohm-bd73800.h
new file mode 100644
index 00000000000000..c6c0d453a40d90
--- /dev/null
+++ b/include/linux/mfd/rohm-bd73800.h
@@ -0,0 +1,306 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright 2026 ROHM Semiconductors.
+ *
+ * Author: Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>
+ */
+
+#ifndef _MFD_BD73800_H
+#define _MFD_BD73800_H
+
+#include <linux/regmap.h>
+
+enum {
+	BD73800_BUCK1	=	0,
+	BD73800_BUCK2,
+	BD73800_BUCK3,
+	BD73800_BUCK4,
+	BD73800_BUCK5,
+	BD73800_BUCK6,
+	BD73800_BUCK7,
+	BD73800_BUCK8,
+	BD73800_LDO1,
+	BD73800_LDO2,
+	BD73800_LDO3,
+	BD73800_LDO4,
+};
+
+/*
+ * All regulators except BUCK 5 have full 8-bit register of valid voltage
+ * values, including 0.
+ */
+#define BD73800_NUM_VOLTS	(0xff + 1)
+/*
+ * BUCK 5 has two sets of voltage ranges, both having valid voltage selectors
+ * from 0x0 to 0x7f
+ */
+#define BD73800_BUCK5_VOLTS	(0x80 + 0x80)
+
+/* BD73800 interrupts */
+enum {
+	/* INT_STAT_1 register IRQs, ADC and RTC */
+	BD73800_INT_ADC_ACCUM_DONE,
+	BD73800_INT_ADC_ACCUM_OVF,
+	BD73800_INT_ADC_ACCUM_VAL,
+	BD73800_INT_ADC_ACCUM_TW,
+	BD73800_INT_ADC_POW_VAL,
+	BD73800_INT_RTC0,
+	BD73800_INT_RTC1,
+	BD73800_INT_RTC2,
+
+	/* BUCK reg interrupts */
+	/* INT_STAT_2 IRQs */
+	BD73800_INT_BUCK1_DVS_DONE,
+	BD73800_INT_BUCK2_DVS_DONE,
+	BD73800_INT_BUCK3_DVS_DONE,
+	BD73800_INT_BUCK4_DVS_DONE,
+	BD73800_INT_BUCK5_DVS_DONE,
+	BD73800_INT_BUCK6_DVS_DONE,
+	BD73800_INT_BUCK7_DVS_DONE,
+	BD73800_INT_BUCK8_DVS_DONE,
+	/* INT_STAT_3 IRQs */
+	BD73800_INT_BUCK1_OCP,
+	BD73800_INT_BUCK2_OCP,
+	BD73800_INT_BUCK3_OCP,
+	BD73800_INT_BUCK4_OCP,
+	BD73800_INT_BUCK5_OCP,
+	BD73800_INT_BUCK6_OCP,
+	BD73800_INT_BUCK7_OCP,
+	BD73800_INT_BUCK8_OCP,
+
+	/* INT_STAT_4 IRQs, power-button, WDG and reset */
+	BD73800_INT_PBTN_LONG_PRESS,
+	BD73800_INT_PBTN_MID_PRESS,
+	/*
+	 * The SHORT_PUSH is generated when button is first pressed (longer
+	 * than configured time limit), and then released before the MID_PRESS
+	 * time limit. The SHORT_PRESS is generated immediately when button is
+	 * pressed for longer than configured limit, whether it is released or
+	 * not.
+	 */
+	BD73800_INT_PBTN_SHORT_PUSH,
+	BD73800_INT_PBTN_SHORT_PRESS,
+	BD73800_INT_WDG,
+	BD73800_INT_SWRESET,
+	BD73800_INT_SEQ_DONE,
+
+	/* INT_STAT_5 IRQs, GPIO */
+	BD73800_INT_GPIO1,
+	BD73800_INT_GPIO2,
+	BD73800_INT_GPIO3,
+	BD73800_INT_GPIO4,
+};
+
+#define BD73800_MASK_RUN_EN		BIT(2)
+#define BD73800_MASK_SUSP_EN		BIT(1)
+#define BD73800_MASK_IDLE_EN		BIT(0)
+#define BD73800_MASK_VOLT		GENMASK(7, 0)
+#define BD73800_MASK_BUCK5_VOLT		GENMASK(6, 0)
+#define BD73800_MASK_RAMP_DELAY		GENMASK(2, 1)
+#define BD73800_BUCK5_RANGE_MASK	BIT(7)
+
+/* BD73800 registers */
+enum {
+	BD73800_REG_PRODUCT_ID	= 0x0,
+	BD73800_REG_MANUFACTURER_ID,
+	BD73800_REG_REVISION,
+	BD73800_REG_NVMVERSION,
+	BD73800_REG_POR_REASON,
+	BD73800_REG_RESET_REASON1,
+	BD73800_REG_RESET_REASON2,
+	BD73800_REG_RESET_REASON3,
+	BD73800_REG_POW_STATE,
+	BD73800_REG_WRST_SEL,
+	BD73800_REG_PS_CTRL_1,
+	BD73800_REG_PS_CTRL_2,
+	BD73800_REG_RCVCFG,
+	BD73800_REG_RCVNUM,
+	BD73800_REG_CRDCFG, /* 0x0f, followed by undocumented reg */
+
+	BD73800_REG_BUCK1_ON = 0x11,
+	BD73800_REG_BUCK1_MODE,
+	BD73800_REG_BUCK1_VOLT_RUN,
+	BD73800_REG_BUCK1_VOLT_IDLE,
+	BD73800_REG_BUCK1_VOLT_SUSP, /* 0x15, followed by undocumented reg */
+
+	BD73800_REG_BUCK2_ON = 0x17,
+	BD73800_REG_BUCK2_MODE,
+	BD73800_REG_BUCK2_VOLT_RUN,
+	BD73800_REG_BUCK2_VOLT_IDLE,
+	BD73800_REG_BUCK2_VOLT_SUSP, /* 0x1b */
+
+	BD73800_REG_BUCK3_ON = 0x1d,
+	BD73800_REG_BUCK3_MODE,
+	BD73800_REG_BUCK3_VOLT_RUN,
+	BD73800_REG_BUCK3_VOLT_IDLE,
+	BD73800_REG_BUCK3_VOLT_SUSP, /* 0x21 */
+
+	BD73800_REG_BUCK4_ON = 0x23,
+	BD73800_REG_BUCK4_MODE,
+	BD73800_REG_BUCK4_VOLT_RUN,
+	BD73800_REG_BUCK4_VOLT_IDLE,
+	BD73800_REG_BUCK4_VOLT_SUSP, /* 0x27 */
+
+	BD73800_REG_BUCK5_ON = 0x29,
+	BD73800_REG_BUCK5_MODE,
+	BD73800_REG_BUCK5_VOLT_RUN,
+	BD73800_REG_BUCK5_VOLT_IDLE,
+	BD73800_REG_BUCK5_VOLT_SUSP, /* 0x2d */
+
+	BD73800_REG_BUCK6_ON = 0x2f,
+	BD73800_REG_BUCK6_MODE,
+	BD73800_REG_BUCK6_VOLT_RUN,
+	BD73800_REG_BUCK6_VOLT_IDLE,
+	BD73800_REG_BUCK6_VOLT_SUSP, /* 0x33 */
+
+	BD73800_REG_BUCK7_ON = 0x35,
+	BD73800_REG_BUCK7_MODE,
+	BD73800_REG_BUCK7_VOLT_RUN,
+	BD73800_REG_BUCK7_VOLT_IDLE,
+	BD73800_REG_BUCK7_VOLT_SUSP, /* 0x39 */
+
+	BD73800_REG_BUCK8_ON = 0x3b,
+	BD73800_REG_BUCK8_MODE,
+	BD73800_REG_BUCK8_VOLT_RUN,
+	BD73800_REG_BUCK8_VOLT_IDLE,
+	BD73800_REG_BUCK8_VOLT_SUSP, /* 0x3f */
+
+	BD73800_REG_LDO1_ON = 0x41,
+	BD73800_REG_LDO1_VOLT,
+	BD73800_REG_LDO1_MODE,
+	BD73800_REG_LDO2_ON,
+	BD73800_REG_LDO2_VOLT,
+	BD73800_REG_LDO2_MODE,
+	BD73800_REG_LDO3_ON,
+	BD73800_REG_LDO3_VOLT,
+	BD73800_REG_LDO3_MODE,
+	BD73800_REG_LDO4_ON,
+	BD73800_REG_LDO4_VOLT,
+	BD73800_REG_LDO4_MODE, /* 0x4c */
+
+	BD73800_REG_GPO_OUT = 0x4e,
+	BD73800_REG_OUT32K = 0x50,
+	BD73800_REG_RTC_SEC,
+	BD73800_REG_RTC_MIN,
+	BD73800_REG_RTC_HOUR,
+	BD73800_REG_RTC_WEEK,
+	BD73800_REG_RTC_DAY,
+	BD73800_REG_RTC_MONTH,
+	BD73800_REG_RTC_YEAR,
+	BD73800_REG_RTC_ALM0_SEC,
+	BD73800_REG_RTC_ALM0_MIN,
+	BD73800_REG_RTC_ALM0_HOUR,
+	BD73800_REG_RTC_ALM0_WEEK,
+	BD73800_REG_RTC_ALM0_DAY,
+	BD73800_REG_RTC_ALM0_MONTH,
+	BD73800_REG_RTC_ALM0_YEAR,
+	BD73800_REG_RTC_ALM1_SEC,
+	BD73800_REG_RTC_ALM1_MIN,
+	BD73800_REG_RTC_ALM1_HOUR,
+	BD73800_REG_RTC_ALM1_WEEK,
+	BD73800_REG_RTC_ALM1_DAY,
+	BD73800_REG_RTC_ALM1_MONTH,
+	BD73800_REG_RTC_ALM1_YEAR,
+	BD73800_REG_RTC_ALM2,
+	BD73800_REG_RTC_CONF, /* 0x69 */
+
+	BD73800_REG_ADC_CTRL_1 = 0x6b,
+	BD73800_REG_ADC_CTRL_2,
+	BD73800_REG_ADC_ACCUM_NUM2,
+	BD73800_REG_ADC_ACCUM_NUM1,
+	BD73800_REG_ADC_ACCUM_NUM0,
+	BD73800_REG_ADC_ACCUM_KICK,
+	BD73800_REG_ADC_ACCUM_CNT2,
+	BD73800_REG_ADC_ACCUM_CNT1,
+	BD73800_REG_ADC_ACCUM_CNT0,
+	BD73800_REG_ADC_ACCUM_VAL2,
+	BD73800_REG_ADC_ACCUM_VAL1,
+	BD73800_REG_ADC_ACCUM_VAL0,
+	BD73800_REG_ADC_VOL_VAL1,
+	BD73800_REG_ADC_VOL_VAL0,
+	BD73800_REG_ADC_CUR_VAL1,
+	BD73800_REG_ADC_CUR_VAL0,
+	BD73800_REG_ADC_POW_VAL1,
+	BD73800_REG_ADC_POW_VAL0,
+	BD73800_REG_ADC_TEMP_VAL1,
+	BD73800_REG_ADC_TEMP_VAL0,
+	BD73800_REG_ADC_ACCUM_VAL_INT_TH4,
+	BD73800_REG_ADC_ACCUM_VAL_INT_TH3,
+	BD73800_REG_ADC_ACCUM_VAL_INT_TH2,
+	BD73800_REG_ADC_ACCUM_VAL_INT_TH1,
+	BD73800_REG_ADC_ACCUM_VAL_INT_TH0,
+	BD73800_REG_ADC_WARN_TEMP_INT_TH1,
+	BD73800_REG_ADC_WARN_TEMP_INT_TH0,
+	BD73800_REG_ADC_POW_VAL_INT_TH1,
+	BD73800_REG_ADC_POW_VAL_INT_TH0, /* 0x89 */
+
+	BD73800_REG_PBTN_CONF = 0x8b,
+
+	BD73800_REG_INT_MAIN_EN = 0x8f,
+	BD73800_REG_INT_1_EN,
+	BD73800_REG_INT_2_EN,
+	BD73800_REG_INT_3_EN,
+	BD73800_REG_INT_4_EN,
+	BD73800_REG_INT_5_EN, /* 0x94 */
+
+	BD73800_REG_INT_MAIN_STAT = 0x96,
+	BD73800_REG_INT_1_STAT,
+	BD73800_REG_INT_2_STAT,
+	BD73800_REG_INT_3_STAT,
+	BD73800_REG_INT_4_STAT,
+	BD73800_REG_INT_5_STAT, /* 0x9b */
+
+	BD73800_REG_INT_MAIN_SRC = 0x9d,
+	BD73800_REG_INT_1_SRC,
+	BD73800_REG_INT_2_SRC,
+	BD73800_REG_INT_3_SRC,
+	BD73800_REG_INT_4_SRC,
+	BD73800_REG_INT_5_SRC, /* 0xa2 */
+
+	BD73800_REG_RST_MASK = 0xaf,
+	BD73800_MAX_REGISTER,
+};
+
+#define BD73800_REG_RTC_START			BD73800_REG_RTC_SEC
+#define BD73800_REG_RTC_ALM_START		BD73800_REG_RTC_ALM0_SEC
+
+/* BD73800 IRQ register masks */
+
+#define BD73800_INT_MAIN_EN_ALL			GENMASK(4, 0)
+#define BD73800_INT_ADC_ACCUM_DONE_MASK		BIT(0)
+#define BD73800_INT_ADC_ACCUM_OVF_MASK		BIT(1)
+#define BD73800_INT_ADC_ACCUM_VAL_MASK		BIT(2)
+#define BD73800_INT_ADC_ACCUM_TW_MASK		BIT(3)
+#define BD73800_INT_ADC_POW_VAL_MASK		BIT(4)
+#define BD73800_INT_RTC0_MASK			BIT(5)
+#define BD73800_INT_RTC1_MASK			BIT(6)
+#define BD73800_INT_RTC2_MASK			BIT(7)
+#define BD73800_INT_BUCK1_DVS_DONE_MASK		BIT(0)
+#define BD73800_INT_BUCK2_DVS_DONE_MASK		BIT(1)
+#define BD73800_INT_BUCK3_DVS_DONE_MASK		BIT(2)
+#define BD73800_INT_BUCK4_DVS_DONE_MASK		BIT(3)
+#define BD73800_INT_BUCK5_DVS_DONE_MASK		BIT(4)
+#define BD73800_INT_BUCK6_DVS_DONE_MASK		BIT(5)
+#define BD73800_INT_BUCK7_DVS_DONE_MASK		BIT(6)
+#define BD73800_INT_BUCK8_DVS_DONE_MASK		BIT(7)
+#define BD73800_INT_BUCK1_OCP_MASK		BIT(0)
+#define BD73800_INT_BUCK2_OCP_MASK		BIT(1)
+#define BD73800_INT_BUCK3_OCP_MASK		BIT(2)
+#define BD73800_INT_BUCK4_OCP_MASK		BIT(3)
+#define BD73800_INT_BUCK5_OCP_MASK		BIT(4)
+#define BD73800_INT_BUCK6_OCP_MASK		BIT(5)
+#define BD73800_INT_BUCK7_OCP_MASK		BIT(6)
+#define BD73800_INT_BUCK8_OCP_MASK		BIT(7)
+#define BD73800_INT_PBTN_LONG_PRESS_MASK	BIT(0)
+#define BD73800_INT_PBTN_MID_PRESS_MASK		BIT(1)
+#define BD73800_INT_PBTN_SHORT_PUSH_MASK	BIT(2)
+#define BD73800_INT_PBTN_SHORT_PRESS_MASK	BIT(3)
+#define BD73800_INT_WDG_MASK			BIT(4)
+#define BD73800_INT_SWRESET_MASK		BIT(5)
+#define BD73800_INT_SEQ_DONE_MASK		BIT(6)
+#define BD73800_INT_GPIO1_MASK			BIT(0)
+#define BD73800_INT_GPIO2_MASK			BIT(1)
+#define BD73800_INT_GPIO3_MASK			BIT(2)
+#define BD73800_INT_GPIO4_MASK			BIT(3)
+
+#endif /* _MFD_BD73800_H */
diff --git a/include/linux/mfd/rohm-generic.h b/include/linux/mfd/rohm-generic.h
index 0a284919a6c3d0..3ec87428ee97ce 100644
--- a/include/linux/mfd/rohm-generic.h
+++ b/include/linux/mfd/rohm-generic.h
@@ -17,6 +17,7 @@ enum rohm_chip_type {
 	ROHM_CHIP_TYPE_BD71837,
 	ROHM_CHIP_TYPE_BD71847,
 	ROHM_CHIP_TYPE_BD72720,
+	ROHM_CHIP_TYPE_BD73800,
 	ROHM_CHIP_TYPE_BD96801,
 	ROHM_CHIP_TYPE_BD96802,
 	ROHM_CHIP_TYPE_BD96805,
diff --git a/include/linux/mhi.h b/include/linux/mhi.h
index 8616bacd8675d0..65e8a397cc50b8 100644
--- a/include/linux/mhi.h
+++ b/include/linux/mhi.h
@@ -687,7 +687,7 @@ int mhi_pm_resume(struct mhi_controller *mhi_cntrl);
  * Resume the device irrespective of its MHI state. As per the MHI spec, devices
  * has to be in M3 state during resume. But some devices seem to be in a
  * different MHI state other than M3 but they continue working fine if allowed.
- * This API is intented to be used for such devices.
+ * This API is intended to be used for such devices.
  *
  * Return: 0 if the resume succeeds, a negative error code otherwise
  */
diff --git a/include/linux/micrel_phy.h b/include/linux/micrel_phy.h
index 9c6f9817383f34..bb71b2510c2cec 100644
--- a/include/linux/micrel_phy.h
+++ b/include/linux/micrel_phy.h
@@ -47,11 +47,6 @@
 #define MICREL_PHY_FXEN		BIT(1)
 #define MICREL_KSZ8_P1_ERRATA	BIT(2)
 
-#define MICREL_KSZ9021_EXTREG_CTRL	0xB
-#define MICREL_KSZ9021_EXTREG_DATA_WRITE	0xC
-#define MICREL_KSZ9021_RGMII_CLK_CTRL_PAD_SCEW	0x104
-#define MICREL_KSZ9021_RGMII_RX_DATA_PAD_SCEW	0x105
-
 /* Device specific MII_BMCR (Reg 0) bits */
 /* 1 = HP Auto MDI/MDI-X mode, 0 = Microchip Auto MDI/MDI-X mode */
 #define KSZ886X_BMCR_HP_MDIX			BIT(5)
diff --git a/include/linux/minmax.h b/include/linux/minmax.h
index a0158db54a0411..5ef4d58c0c4291 100644
--- a/include/linux/minmax.h
+++ b/include/linux/minmax.h
@@ -38,9 +38,9 @@
  * Note that 'x' is the original expression, and 'ux' is the unique variable
  * that contains the value.
  *
- * We use 'ux' for pure type checking, and 'x' for when we need to look at the
- * value (but without evaluating it for side effects!
- * Careful to only ever evaluate it with sizeof() or __builtin_constant_p() etc).
+ * We use 'ux' for both the type and the value checks, so 'x' itself is only
+ * expanded twice: once to initialise 'ux', and once quoted in the error
+ * message.
  *
  * Pointers end up being checked by the normal C type rules at the actual
  * comparison, and these expressions only need to be careful to not cause
diff --git a/include/linux/mlx5/cq.h b/include/linux/mlx5/cq.h
index 9d47cdc727ad0d..a1c14479e462c2 100644
--- a/include/linux/mlx5/cq.h
+++ b/include/linux/mlx5/cq.h
@@ -81,16 +81,6 @@ enum {
 
 enum {
 	MLX5_CQE_OWNER_MASK	= 1,
-	MLX5_CQE_REQ		= 0,
-	MLX5_CQE_RESP_WR_IMM	= 1,
-	MLX5_CQE_RESP_SEND	= 2,
-	MLX5_CQE_RESP_SEND_IMM	= 3,
-	MLX5_CQE_RESP_SEND_INV	= 4,
-	MLX5_CQE_RESIZE_CQ	= 5,
-	MLX5_CQE_SIG_ERR	= 12,
-	MLX5_CQE_REQ_ERR	= 13,
-	MLX5_CQE_RESP_ERR	= 14,
-	MLX5_CQE_INVALID	= 15,
 };
 
 enum {
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 8cb321a9fb3d1d..37727b5ab78f25 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -36,6 +36,7 @@
 #include <linux/types.h>
 #include <rdma/ib_verbs.h>
 #include <linux/mlx5/mlx5_ifc.h>
+#include <linux/mlx5/mlx5_ifc_macros.h>
 #include <linux/bitfield.h>
 
 #if defined(__LITTLE_ENDIAN)
@@ -46,122 +47,6 @@
 #error Host endianness not defined
 #endif
 
-/* helper macros */
-#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0)
-#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld)
-#define __mlx5_bit_off(typ, fld) (offsetof(struct mlx5_ifc_##typ##_bits, fld))
-#define __mlx5_16_off(typ, fld) (__mlx5_bit_off(typ, fld) / 16)
-#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32)
-#define __mlx5_64_off(typ, fld) (__mlx5_bit_off(typ, fld) / 64)
-#define __mlx5_16_bit_off(typ, fld) (16 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0xf))
-#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0x1f))
-#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1))
-#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << __mlx5_dw_bit_off(typ, fld))
-#define __mlx5_mask16(typ, fld) ((u16)((1ull << __mlx5_bit_sz(typ, fld)) - 1))
-#define __mlx5_16_mask(typ, fld) (__mlx5_mask16(typ, fld) << __mlx5_16_bit_off(typ, fld))
-#define __mlx5_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits)
-
-#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8)
-#define MLX5_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8)
-#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32)
-#define MLX5_ST_SZ_QW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 64)
-#define MLX5_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8)
-#define MLX5_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32)
-#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8)
-#define MLX5_ADDR_OF(typ, p, fld) ((void *)((u8 *)(p) + MLX5_BYTE_OFF(typ, fld)))
-
-/* insert a value to a struct */
-#define MLX5_SET(typ, p, fld, v) do { \
-	u32 _v = v; \
-	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32);             \
-	*((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \
-	cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \
-		     (~__mlx5_dw_mask(typ, fld))) | (((_v) & __mlx5_mask(typ, fld)) \
-		     << __mlx5_dw_bit_off(typ, fld))); \
-} while (0)
-
-#define MLX5_ARRAY_SET(typ, p, fld, idx, v) do { \
-	BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 32); \
-	MLX5_SET(typ, p, fld[idx], v); \
-} while (0)
-
-#define MLX5_SET_TO_ONES(typ, p, fld) do { \
-	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32);             \
-	*((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \
-	cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \
-		     (~__mlx5_dw_mask(typ, fld))) | ((__mlx5_mask(typ, fld)) \
-		     << __mlx5_dw_bit_off(typ, fld))); \
-} while (0)
-
-#define MLX5_GET(typ, p, fld) ((be32_to_cpu(*((__be32 *)(p) +\
-__mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \
-__mlx5_mask(typ, fld))
-
-#define MLX5_GET_PR(typ, p, fld) ({ \
-	u32 ___t = MLX5_GET(typ, p, fld); \
-	pr_debug(#fld " = 0x%x\n", ___t); \
-	___t; \
-})
-
-#define __MLX5_SET64(typ, p, fld, v) do { \
-	BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) != 64); \
-	*((__be64 *)(p) + __mlx5_64_off(typ, fld)) = cpu_to_be64(v); \
-} while (0)
-
-#define MLX5_SET64(typ, p, fld, v) do { \
-	BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \
-	__MLX5_SET64(typ, p, fld, v); \
-} while (0)
-
-#define MLX5_ARRAY_SET64(typ, p, fld, idx, v) do { \
-	BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \
-	__MLX5_SET64(typ, p, fld[idx], v); \
-} while (0)
-
-#define MLX5_GET64(typ, p, fld) be64_to_cpu(*((__be64 *)(p) + __mlx5_64_off(typ, fld)))
-
-#define MLX5_GET64_PR(typ, p, fld) ({ \
-	u64 ___t = MLX5_GET64(typ, p, fld); \
-	pr_debug(#fld " = 0x%llx\n", ___t); \
-	___t; \
-})
-
-#define MLX5_GET16(typ, p, fld) ((be16_to_cpu(*((__be16 *)(p) +\
-__mlx5_16_off(typ, fld))) >> __mlx5_16_bit_off(typ, fld)) & \
-__mlx5_mask16(typ, fld))
-
-#define MLX5_SET16(typ, p, fld, v) do { \
-	u16 _v = v; \
-	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 16);             \
-	*((__be16 *)(p) + __mlx5_16_off(typ, fld)) = \
-	cpu_to_be16((be16_to_cpu(*((__be16 *)(p) + __mlx5_16_off(typ, fld))) & \
-		     (~__mlx5_16_mask(typ, fld))) | (((_v) & __mlx5_mask16(typ, fld)) \
-		     << __mlx5_16_bit_off(typ, fld))); \
-} while (0)
-
-/* Big endian getters */
-#define MLX5_GET64_BE(typ, p, fld) (*((__be64 *)(p) +\
-	__mlx5_64_off(typ, fld)))
-
-#define MLX5_GET_BE(type_t, typ, p, fld) ({				  \
-		type_t tmp;						  \
-		switch (sizeof(tmp)) {					  \
-		case sizeof(u8):					  \
-			tmp = (__force type_t)MLX5_GET(typ, p, fld);	  \
-			break;						  \
-		case sizeof(u16):					  \
-			tmp = (__force type_t)cpu_to_be16(MLX5_GET(typ, p, fld)); \
-			break;						  \
-		case sizeof(u32):					  \
-			tmp = (__force type_t)cpu_to_be32(MLX5_GET(typ, p, fld)); \
-			break;						  \
-		case sizeof(u64):					  \
-			tmp = (__force type_t)MLX5_GET64_BE(typ, p, fld); \
-			break;						  \
-			}						  \
-		tmp;							  \
-		})
-
 enum mlx5_inline_modes {
 	MLX5_INLINE_MODE_NONE,
 	MLX5_INLINE_MODE_L2,
@@ -172,7 +57,7 @@ enum mlx5_inline_modes {
 enum {
 	MLX5_MAX_COMMANDS		= 32,
 	MLX5_CMD_DATA_BLOCK_SIZE	= 512,
-	MLX5_PCI_CMD_XPORT		= 7,
+	MLX5_PCI_CMD_XPORT		= MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT,
 	MLX5_MKEY_BSF_OCTO_SIZE		= 4,
 	MLX5_MAX_PSVS			= 4,
 };
@@ -308,63 +193,6 @@ enum {
 	MLX5_EVENT_QUEUE_TYPE_DCT = 6,
 };
 
-/* mlx5 components can subscribe to any one of these events via
- * mlx5_eq_notifier_register API.
- */
-enum mlx5_event {
-	/* Special value to subscribe to any event */
-	MLX5_EVENT_TYPE_NOTIFY_ANY	   = 0x0,
-	/* HW events enum start: comp events are not subscribable */
-	MLX5_EVENT_TYPE_COMP		   = 0x0,
-	/* HW Async events enum start: subscribable events */
-	MLX5_EVENT_TYPE_PATH_MIG	   = 0x01,
-	MLX5_EVENT_TYPE_COMM_EST	   = 0x02,
-	MLX5_EVENT_TYPE_SQ_DRAINED	   = 0x03,
-	MLX5_EVENT_TYPE_SRQ_LAST_WQE	   = 0x13,
-	MLX5_EVENT_TYPE_SRQ_RQ_LIMIT	   = 0x14,
-
-	MLX5_EVENT_TYPE_CQ_ERROR	   = 0x04,
-	MLX5_EVENT_TYPE_WQ_CATAS_ERROR	   = 0x05,
-	MLX5_EVENT_TYPE_PATH_MIG_FAILED	   = 0x07,
-	MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,
-	MLX5_EVENT_TYPE_WQ_ACCESS_ERROR	   = 0x11,
-	MLX5_EVENT_TYPE_SRQ_CATAS_ERROR	   = 0x12,
-	MLX5_EVENT_TYPE_OBJECT_CHANGE	   = 0x27,
-
-	MLX5_EVENT_TYPE_INTERNAL_ERROR	   = 0x08,
-	MLX5_EVENT_TYPE_PORT_CHANGE	   = 0x09,
-	MLX5_EVENT_TYPE_GPIO_EVENT	   = 0x15,
-	MLX5_EVENT_TYPE_PORT_MODULE_EVENT  = 0x16,
-	MLX5_EVENT_TYPE_TEMP_WARN_EVENT    = 0x17,
-	MLX5_EVENT_TYPE_XRQ_ERROR	   = 0x18,
-	MLX5_EVENT_TYPE_REMOTE_CONFIG	   = 0x19,
-	MLX5_EVENT_TYPE_GENERAL_EVENT	   = 0x22,
-	MLX5_EVENT_TYPE_MONITOR_COUNTER    = 0x24,
-	MLX5_EVENT_TYPE_PPS_EVENT          = 0x25,
-
-	MLX5_EVENT_TYPE_DB_BF_CONGESTION   = 0x1a,
-	MLX5_EVENT_TYPE_STALL_EVENT	   = 0x1b,
-
-	MLX5_EVENT_TYPE_CMD		   = 0x0a,
-	MLX5_EVENT_TYPE_PAGE_REQUEST	   = 0xb,
-
-	MLX5_EVENT_TYPE_PAGE_FAULT	   = 0xc,
-	MLX5_EVENT_TYPE_NIC_VPORT_CHANGE   = 0xd,
-
-	MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0xe,
-	MLX5_EVENT_TYPE_VHCA_STATE_CHANGE = 0xf,
-
-	MLX5_EVENT_TYPE_DCT_DRAINED        = 0x1c,
-	MLX5_EVENT_TYPE_DCT_KEY_VIOLATION  = 0x1d,
-
-	MLX5_EVENT_TYPE_FPGA_ERROR         = 0x20,
-	MLX5_EVENT_TYPE_FPGA_QP_ERROR      = 0x21,
-
-	MLX5_EVENT_TYPE_DEVICE_TRACER      = 0x26,
-
-	MLX5_EVENT_TYPE_MAX                = 0x100,
-};
-
 enum mlx5_driver_event {
 	MLX5_DRIVER_EVENT_TYPE_TRAP = 0,
 	MLX5_DRIVER_EVENT_UPLINK_NETDEV,
@@ -420,22 +248,6 @@ enum {
 };
 
 enum {
-	MLX5_OPCODE_NOP			= 0x00,
-	MLX5_OPCODE_SEND_INVAL		= 0x01,
-	MLX5_OPCODE_RDMA_WRITE		= 0x08,
-	MLX5_OPCODE_RDMA_WRITE_IMM	= 0x09,
-	MLX5_OPCODE_SEND		= 0x0a,
-	MLX5_OPCODE_SEND_IMM		= 0x0b,
-	MLX5_OPCODE_LSO			= 0x0e,
-	MLX5_OPCODE_RDMA_READ		= 0x10,
-	MLX5_OPCODE_ATOMIC_CS		= 0x11,
-	MLX5_OPCODE_ATOMIC_FA		= 0x12,
-	MLX5_OPCODE_ATOMIC_MASKED_CS	= 0x14,
-	MLX5_OPCODE_ATOMIC_MASKED_FA	= 0x15,
-	MLX5_OPCODE_BIND_MW		= 0x18,
-	MLX5_OPCODE_CONFIG_CMD		= 0x1f,
-	MLX5_OPCODE_ENHANCED_MPSW	= 0x29,
-
 	MLX5_RECV_OPCODE_RDMA_WRITE_IMM	= 0x00,
 	MLX5_RECV_OPCODE_SEND		= 0x01,
 	MLX5_RECV_OPCODE_SEND_IMM	= 0x02,
@@ -443,19 +255,6 @@ enum {
 
 	MLX5_CQE_OPCODE_ERROR		= 0x1e,
 	MLX5_CQE_OPCODE_RESIZE		= 0x16,
-
-	MLX5_OPCODE_SET_PSV		= 0x20,
-	MLX5_OPCODE_GET_PSV		= 0x21,
-	MLX5_OPCODE_CHECK_PSV		= 0x22,
-	MLX5_OPCODE_DUMP		= 0x23,
-	MLX5_OPCODE_RGET_PSV		= 0x26,
-	MLX5_OPCODE_RCHECK_PSV		= 0x27,
-
-	MLX5_OPCODE_UMR			= 0x25,
-
-	MLX5_OPCODE_FLOW_TBL_ACCESS	= 0x2c,
-
-	MLX5_OPCODE_ACCESS_ASO		= 0x2d,
 };
 
 enum {
@@ -1224,12 +1023,6 @@ enum mlx5_flex_parser_protos {
 
 /* MLX5 DEV CAPs */
 
-/* TODO: EAT.ME */
-enum mlx5_cap_mode {
-	HCA_CAP_OPMOD_GET_MAX	= 0,
-	HCA_CAP_OPMOD_GET_CUR	= 1,
-};
-
 /* Any new cap addition must update mlx5_hca_caps_alloc() to allocate
  * capability memory.
  */
@@ -1507,26 +1300,6 @@ enum mlx5_qcam_feature_groups {
 #define MLX5_CAP_PSP(mdev, cap)\
 	MLX5_GET(psp_cap, (mdev)->caps.hca[MLX5_CAP_PSP]->cur, cap)
 
-enum {
-	MLX5_CMD_STAT_OK			= 0x0,
-	MLX5_CMD_STAT_INT_ERR			= 0x1,
-	MLX5_CMD_STAT_BAD_OP_ERR		= 0x2,
-	MLX5_CMD_STAT_BAD_PARAM_ERR		= 0x3,
-	MLX5_CMD_STAT_BAD_SYS_STATE_ERR		= 0x4,
-	MLX5_CMD_STAT_BAD_RES_ERR		= 0x5,
-	MLX5_CMD_STAT_RES_BUSY			= 0x6,
-	MLX5_CMD_STAT_NOT_READY			= 0x7,
-	MLX5_CMD_STAT_LIM_ERR			= 0x8,
-	MLX5_CMD_STAT_BAD_RES_STATE_ERR		= 0x9,
-	MLX5_CMD_STAT_IX_ERR			= 0xa,
-	MLX5_CMD_STAT_NO_RES_ERR		= 0xf,
-	MLX5_CMD_STAT_BAD_INP_LEN_ERR		= 0x50,
-	MLX5_CMD_STAT_BAD_OUTP_LEN_ERR		= 0x51,
-	MLX5_CMD_STAT_BAD_QP_STATE_ERR		= 0x10,
-	MLX5_CMD_STAT_BAD_PKT_ERR		= 0x30,
-	MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR	= 0x40,
-};
-
 enum {
 	MLX5_IEEE_802_3_COUNTERS_GROUP	      = 0x0,
 	MLX5_RFC_2863_COUNTERS_GROUP	      = 0x1,
diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h
index a0dd162baa787c..03d3620141c837 100644
--- a/include/linux/mlx5/eswitch.h
+++ b/include/linux/mlx5/eswitch.h
@@ -222,6 +222,9 @@ static inline bool is_mdev_switchdev_mode(struct mlx5_core_dev *dev)
 /* The returned number is valid only when the dev is eswitch manager. */
 static inline u16 mlx5_eswitch_manager_vport(struct mlx5_core_dev *dev)
 {
+	if (MLX5_CAP_ESW(dev, esw_manager_vport_number_valid))
+		return MLX5_CAP_ESW(dev, esw_manager_vport_number);
+
 	return mlx5_core_is_ecpf_esw_manager(dev) ?
 		MLX5_VPORT_ECPF : MLX5_VPORT_HOST_PF;
 }
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 12b9a81e562146..8247cb0f8dfd64 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -5979,6 +5979,108 @@ struct mlx5_ifc_cqe_error_syndrome_bits {
 	u8         syndrome[0x8];
 };
 
+struct mlx5_ifc_wqe_ctrl_seg_bits {
+	u8         opmod[0x8];
+	u8         wqe_index[0x10];
+	u8         opcode[0x8];
+
+	u8         qp_or_sq[0x18];
+	u8         reserved_at_38[0x2];
+	u8         ds[0x6];
+
+	u8         signature[0x8];
+	u8         reserved_at_48[0x10];
+	u8         fm[0x3];
+	u8         reserved_at_5b[0x1];
+	u8         ce[0x2];
+	u8         se[0x1];
+	u8         reserved_at_5f[0x1];
+
+	u8         imm[0x20];
+};
+
+/* Values for wqe_ctrl_seg_bits.opcode */
+enum {
+	MLX5_OPCODE_NOP			= 0x00,
+	MLX5_OPCODE_SEND_INVAL		= 0x01,
+	MLX5_OPCODE_RDMA_WRITE		= 0x08,
+	MLX5_OPCODE_RDMA_WRITE_IMM	= 0x09,
+	MLX5_OPCODE_SEND		= 0x0a,
+	MLX5_OPCODE_SEND_IMM		= 0x0b,
+	MLX5_OPCODE_LSO			= 0x0e,
+	MLX5_OPCODE_RDMA_READ		= 0x10,
+	MLX5_OPCODE_ATOMIC_CS		= 0x11,
+	MLX5_OPCODE_ATOMIC_FA		= 0x12,
+	MLX5_OPCODE_ATOMIC_MASKED_CS	= 0x14,
+	MLX5_OPCODE_ATOMIC_MASKED_FA	= 0x15,
+	MLX5_OPCODE_BIND_MW		= 0x18,
+	MLX5_OPCODE_CONFIG_CMD		= 0x1f,
+	MLX5_OPCODE_SET_PSV		= 0x20,
+	MLX5_OPCODE_GET_PSV		= 0x21,
+	MLX5_OPCODE_CHECK_PSV		= 0x22,
+	MLX5_OPCODE_DUMP		= 0x23,
+	MLX5_OPCODE_UMR			= 0x25,
+	MLX5_OPCODE_RGET_PSV		= 0x26,
+	MLX5_OPCODE_RCHECK_PSV		= 0x27,
+	MLX5_OPCODE_ENHANCED_MPSW	= 0x29,
+	MLX5_OPCODE_FLOW_TBL_ACCESS	= 0x2c,
+	MLX5_OPCODE_ACCESS_ASO		= 0x2d,
+};
+
+/* Values for wqe_ctrl_seg_bits.ce */
+enum {
+	MLX5_WQE_CE_CQE_ALWAYS		= 2,
+};
+
+struct mlx5_ifc_wqe_raddr_seg_bits {
+	u8         raddr[0x40];
+
+	u8         rkey[0x20];
+	u8         reserved_at_60[0x20];
+};
+
+struct mlx5_ifc_wqe_data_seg_bits {
+	u8         reserved_at_0[0x1];
+	u8         byte_count[0x1f];
+
+	u8         lkey[0x20];
+
+	u8         addr[0x40];
+};
+
+struct mlx5_ifc_cqe64_bits {
+	u8         reserved_at_0[0x1a0];
+
+	union {
+		u8         reserved_at_1a0[0x20];
+		struct mlx5_ifc_cqe_error_syndrome_bits error_syndrome;
+	};
+
+	u8         send_wqe_opcode[0x8];
+	u8         qpn_or_dctn_or_flow_tag[0x18];
+
+	u8         wqe_counter[0x10];
+	u8         signature[0x8];
+	u8         opcode[0x4];
+	u8         cqe_format[0x2];
+	u8         se[0x1];
+	u8         owner[0x1];
+};
+
+/* Values for cqe64_bits.opcode */
+enum {
+	MLX5_CQE_REQ		= 0,
+	MLX5_CQE_RESP_WR_IMM	= 1,
+	MLX5_CQE_RESP_SEND	= 2,
+	MLX5_CQE_RESP_SEND_IMM	= 3,
+	MLX5_CQE_RESP_SEND_INV	= 4,
+	MLX5_CQE_RESIZE_CQ	= 5,
+	MLX5_CQE_SIG_ERR	= 12,
+	MLX5_CQE_REQ_ERR	= 13,
+	MLX5_CQE_RESP_ERR	= 14,
+	MLX5_CQE_INVALID	= 15,
+};
+
 struct mlx5_ifc_qp_context_extension_bits {
 	u8         reserved_at_0[0x60];
 
@@ -6475,6 +6577,12 @@ struct mlx5_ifc_query_hca_cap_in_bits {
 	u8         reserved_at_60[0x20];
 };
 
+/* Values for query_hca_cap_in_bits.op_mod */
+enum mlx5_cap_mode {
+	HCA_CAP_OPMOD_GET_MAX	= 0,
+	HCA_CAP_OPMOD_GET_CUR	= 1,
+};
+
 struct mlx5_ifc_other_hca_cap_bits {
 	u8         roce[0x1];
 	u8         reserved_at_1[0x27f];
@@ -11270,6 +11378,55 @@ struct mlx5_ifc_eqe_bits {
 	u8         owner[0x1];
 };
 
+/* Values for eqe_bits.event_type */
+enum mlx5_event {
+	/* Special value to subscribe to any event */
+	MLX5_EVENT_TYPE_NOTIFY_ANY	   = 0x0,
+	/* HW events enum start: comp events are not subscribable */
+	MLX5_EVENT_TYPE_COMP		   = 0x0,
+	/* HW Async events enum start: subscribable events */
+	MLX5_EVENT_TYPE_PATH_MIG	   = 0x01,
+	MLX5_EVENT_TYPE_COMM_EST	   = 0x02,
+	MLX5_EVENT_TYPE_SQ_DRAINED	   = 0x03,
+	MLX5_EVENT_TYPE_SRQ_LAST_WQE	   = 0x13,
+	MLX5_EVENT_TYPE_SRQ_RQ_LIMIT	   = 0x14,
+
+	MLX5_EVENT_TYPE_CQ_ERROR	   = 0x04,
+	MLX5_EVENT_TYPE_WQ_CATAS_ERROR	   = 0x05,
+	MLX5_EVENT_TYPE_PATH_MIG_FAILED	   = 0x07,
+	MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,
+	MLX5_EVENT_TYPE_WQ_ACCESS_ERROR	   = 0x11,
+	MLX5_EVENT_TYPE_SRQ_CATAS_ERROR	   = 0x12,
+	MLX5_EVENT_TYPE_OBJECT_CHANGE	   = 0x27,
+
+	MLX5_EVENT_TYPE_INTERNAL_ERROR	   = 0x08,
+	MLX5_EVENT_TYPE_PORT_CHANGE	   = 0x09,
+	MLX5_EVENT_TYPE_CMD		   = 0x0a,
+	MLX5_EVENT_TYPE_PAGE_REQUEST	   = 0x0b,
+	MLX5_EVENT_TYPE_PAGE_FAULT	   = 0x0c,
+	MLX5_EVENT_TYPE_NIC_VPORT_CHANGE   = 0x0d,
+	MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0x0e,
+	MLX5_EVENT_TYPE_VHCA_STATE_CHANGE  = 0x0f,
+	MLX5_EVENT_TYPE_GPIO_EVENT	   = 0x15,
+	MLX5_EVENT_TYPE_PORT_MODULE_EVENT  = 0x16,
+	MLX5_EVENT_TYPE_TEMP_WARN_EVENT    = 0x17,
+	MLX5_EVENT_TYPE_XRQ_ERROR	   = 0x18,
+	MLX5_EVENT_TYPE_REMOTE_CONFIG	   = 0x19,
+	MLX5_EVENT_TYPE_DB_BF_CONGESTION   = 0x1a,
+	MLX5_EVENT_TYPE_STALL_EVENT	   = 0x1b,
+	MLX5_EVENT_TYPE_DCT_DRAINED        = 0x1c,
+	MLX5_EVENT_TYPE_DCT_KEY_VIOLATION  = 0x1d,
+	MLX5_EVENT_TYPE_FPGA_ERROR         = 0x20,
+	MLX5_EVENT_TYPE_FPGA_QP_ERROR      = 0x21,
+	MLX5_EVENT_TYPE_GENERAL_EVENT	   = 0x22,
+	MLX5_EVENT_TYPE_MONITOR_COUNTER    = 0x24,
+	MLX5_EVENT_TYPE_PPS_EVENT          = 0x25,
+	MLX5_EVENT_TYPE_DEVICE_TRACER      = 0x26,
+
+	MLX5_EVENT_TYPE_MAX                = 0x100,
+};
+
+/* Values for cmd_queue_entry_bits.type */
 enum {
 	MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT  = 0x7,
 };
@@ -11312,6 +11469,27 @@ struct mlx5_ifc_cmd_out_bits {
 	u8         command_output[0x20];
 };
 
+/* Values for cmd_out_bits.status */
+enum {
+	MLX5_CMD_STAT_OK			= 0x0,
+	MLX5_CMD_STAT_INT_ERR			= 0x1,
+	MLX5_CMD_STAT_BAD_OP_ERR		= 0x2,
+	MLX5_CMD_STAT_BAD_PARAM_ERR		= 0x3,
+	MLX5_CMD_STAT_BAD_SYS_STATE_ERR		= 0x4,
+	MLX5_CMD_STAT_BAD_RES_ERR		= 0x5,
+	MLX5_CMD_STAT_RES_BUSY			= 0x6,
+	MLX5_CMD_STAT_NOT_READY			= 0x7,
+	MLX5_CMD_STAT_LIM_ERR			= 0x8,
+	MLX5_CMD_STAT_BAD_RES_STATE_ERR		= 0x9,
+	MLX5_CMD_STAT_IX_ERR			= 0xa,
+	MLX5_CMD_STAT_NO_RES_ERR		= 0xf,
+	MLX5_CMD_STAT_BAD_QP_STATE_ERR		= 0x10,
+	MLX5_CMD_STAT_BAD_PKT_ERR		= 0x30,
+	MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR	= 0x40,
+	MLX5_CMD_STAT_BAD_INP_LEN_ERR		= 0x50,
+	MLX5_CMD_STAT_BAD_OUTP_LEN_ERR		= 0x51,
+};
+
 struct mlx5_ifc_cmd_in_bits {
 	u8         opcode[0x10];
 	u8         reserved_at_10[0x10];
diff --git a/include/linux/mlx5/mlx5_ifc_macros.h b/include/linux/mlx5/mlx5_ifc_macros.h
new file mode 100644
index 00000000000000..1b9e39a509495a
--- /dev/null
+++ b/include/linux/mlx5/mlx5_ifc_macros.h
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/*
+ * Copyright (c) 2013-2026, Mellanox Technologies. All rights reserved.
+ *
+ * Accessor macros for mlx5 IFC structures.
+ *
+ * Extracted from device.h so that code which cannot include device.h
+ * (e.g. selftests) can still use the MLX5_SET/GET family directly.
+ */
+
+#ifndef MLX5_IFC_MACROS_H
+#define MLX5_IFC_MACROS_H
+
+/* Internal helpers -- 32-bit */
+#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0)
+#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld)
+#define __mlx5_bit_off(typ, fld) (offsetof(struct mlx5_ifc_##typ##_bits, fld))
+#define __mlx5_16_off(typ, fld) (__mlx5_bit_off(typ, fld) / 16)
+#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32)
+#define __mlx5_64_off(typ, fld) (__mlx5_bit_off(typ, fld) / 64)
+#define __mlx5_16_bit_off(typ, fld) (16 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0xf))
+#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0x1f))
+#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1))
+#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << __mlx5_dw_bit_off(typ, fld))
+#define __mlx5_mask16(typ, fld) ((u16)((1ull << __mlx5_bit_sz(typ, fld)) - 1))
+#define __mlx5_16_mask(typ, fld) (__mlx5_mask16(typ, fld) << __mlx5_16_bit_off(typ, fld))
+#define __mlx5_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits)
+
+/* Size and address macros */
+#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8)
+#define MLX5_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8)
+#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32)
+#define MLX5_ST_SZ_QW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 64)
+#define MLX5_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8)
+#define MLX5_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32)
+#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8)
+#define MLX5_ADDR_OF(typ, p, fld) ((void *)((u8 *)(p) + MLX5_BYTE_OFF(typ, fld)))
+
+/* insert a value to a struct */
+#define MLX5_SET(typ, p, fld, v) do { \
+	u32 _v = v; \
+	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32);             \
+	*((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \
+	cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \
+		     (~__mlx5_dw_mask(typ, fld))) | (((_v) & __mlx5_mask(typ, fld)) \
+		     << __mlx5_dw_bit_off(typ, fld))); \
+} while (0)
+
+#define MLX5_ARRAY_SET(typ, p, fld, idx, v) do { \
+	BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 32); \
+	MLX5_SET(typ, p, fld[idx], v); \
+} while (0)
+
+#define MLX5_SET_TO_ONES(typ, p, fld) do { \
+	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32);             \
+	*((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \
+	cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \
+		     (~__mlx5_dw_mask(typ, fld))) | ((__mlx5_mask(typ, fld)) \
+		     << __mlx5_dw_bit_off(typ, fld))); \
+} while (0)
+
+#define MLX5_GET(typ, p, fld) ((be32_to_cpu(*((__be32 *)(p) +\
+__mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \
+__mlx5_mask(typ, fld))
+
+#define MLX5_GET_PR(typ, p, fld) ({ \
+	u32 ___t = MLX5_GET(typ, p, fld); \
+	pr_debug(#fld " = 0x%x\n", ___t); \
+	___t; \
+})
+
+/* 64-bit field accessors */
+#define __MLX5_SET64(typ, p, fld, v) do { \
+	BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) != 64); \
+	*((__be64 *)(p) + __mlx5_64_off(typ, fld)) = cpu_to_be64(v); \
+} while (0)
+
+#define MLX5_SET64(typ, p, fld, v) do { \
+	BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \
+	__MLX5_SET64(typ, p, fld, v); \
+} while (0)
+
+#define MLX5_ARRAY_SET64(typ, p, fld, idx, v) do { \
+	BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \
+	__MLX5_SET64(typ, p, fld[idx], v); \
+} while (0)
+
+#define MLX5_ARRAY_GET64(typ, p, fld, idx)                                   \
+	({                                                                   \
+		BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64);                 \
+		be64_to_cpu(                                                 \
+			*((__be64 *)(p) + __mlx5_64_off(typ, fld) + (idx))); \
+	})
+
+#define MLX5_GET64(typ, p, fld) be64_to_cpu(*((__be64 *)(p) + __mlx5_64_off(typ, fld)))
+
+#define MLX5_GET64_PR(typ, p, fld) ({ \
+	u64 ___t = MLX5_GET64(typ, p, fld); \
+	pr_debug(#fld " = 0x%llx\n", ___t); \
+	___t; \
+})
+
+/* 16-bit field accessors */
+#define MLX5_GET16(typ, p, fld) ((be16_to_cpu(*((__be16 *)(p) +\
+__mlx5_16_off(typ, fld))) >> __mlx5_16_bit_off(typ, fld)) & \
+__mlx5_mask16(typ, fld))
+
+#define MLX5_SET16(typ, p, fld, v) do { \
+	u16 _v = v; \
+	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 16);             \
+	*((__be16 *)(p) + __mlx5_16_off(typ, fld)) = \
+	cpu_to_be16((be16_to_cpu(*((__be16 *)(p) + __mlx5_16_off(typ, fld))) & \
+		     (~__mlx5_16_mask(typ, fld))) | (((_v) & __mlx5_mask16(typ, fld)) \
+		     << __mlx5_16_bit_off(typ, fld))); \
+} while (0)
+
+/* Big endian getters */
+#define MLX5_GET64_BE(typ, p, fld) (*((__be64 *)(p) +\
+	__mlx5_64_off(typ, fld)))
+
+#define MLX5_GET_BE(type_t, typ, p, fld) ({				  \
+		type_t tmp;						  \
+		switch (sizeof(tmp)) {					  \
+		case sizeof(u8):					  \
+			tmp = (__force type_t)MLX5_GET(typ, p, fld);	  \
+			break;						  \
+		case sizeof(u16):					  \
+			tmp = (__force type_t)cpu_to_be16(MLX5_GET(typ, p, fld)); \
+			break;						  \
+		case sizeof(u32):					  \
+			tmp = (__force type_t)cpu_to_be32(MLX5_GET(typ, p, fld)); \
+			break;						  \
+		case sizeof(u64):					  \
+			tmp = (__force type_t)MLX5_GET64_BE(typ, p, fld); \
+			break;						  \
+			}						  \
+		tmp;							  \
+		})
+
+/*
+ * Use READ_ONCE/WRITE_ONCE for a single field that hardware may read/write
+ * unpredictably, mostly owner bits. All other bits in the DW must be stable.
+ * Usually a dma_wmb() will be required before a write and a dma_rmb() after a
+ * read.
+ */
+#define MLX5_GET_ONCE(typ, p, fld)                                              \
+	((be32_to_cpu(READ_ONCE(*((__be32 *)(p) + __mlx5_dw_off(typ, fld)))) >> \
+	  __mlx5_dw_bit_off(typ, fld)) &                                        \
+	 __mlx5_mask(typ, fld))
+
+#define MLX5_SET_ONCE(typ, p, fld, v)                                      \
+	do {                                                               \
+		u32 _v = v;                                                \
+		__be32 *_dw = (__be32 *)(p) + __mlx5_dw_off(typ, fld);     \
+		BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32);                 \
+		WRITE_ONCE(*_dw,                                           \
+			   cpu_to_be32((be32_to_cpu(READ_ONCE(*_dw)) &     \
+					(~__mlx5_dw_mask(typ, fld))) |     \
+				       (((_v) & __mlx5_mask(typ, fld))     \
+					<< __mlx5_dw_bit_off(typ, fld)))); \
+	} while (0)
+
+/* Access MMIO registers, usually the init segment, using IFC structs. */
+#define MLX5_GET_MMIO(typ, p, fld)                                         \
+	((ioread32be(((__be32 __iomem *)(p) + __mlx5_dw_off(typ, fld))) >> \
+	  __mlx5_dw_bit_off(typ, fld)) &                                   \
+	 __mlx5_mask(typ, fld))
+
+/* The set is not relaxed so there is an integrated dma_wmb(). */
+#define MLX5_SET_MMIO(typ, p, fld, v)                                         \
+	do {                                                                  \
+		u32 _v = v;                                                   \
+		void __iomem *_dw =                                           \
+			((__be32 __iomem *)(p) + __mlx5_dw_off(typ, fld));    \
+		if (__mlx5_bit_sz(typ, fld) == 32)                            \
+			iowrite32be(_v, _dw);                                 \
+		else                                                          \
+			iowrite32be((ioread32be(_dw) &                        \
+				     (~__mlx5_dw_mask(typ, fld))) |           \
+					    ((_v & __mlx5_mask(typ, fld))     \
+					     << __mlx5_dw_bit_off(typ, fld)), \
+				    _dw);                                     \
+	} while (0)
+
+#endif /* MLX5_IFC_MACROS_H */
diff --git a/include/linux/mm.h b/include/linux/mm.h
index dd09c438fa23ec..f4d495561c14e4 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -38,6 +38,7 @@
 #include <linux/bitops.h>
 #include <linux/iommu-debug-pagealloc.h>
 #include <linux/kcsan-checks.h>
+#include <linux/vmemmap-optimization.h>
 
 struct mempolicy;
 struct anon_vma;
@@ -57,16 +58,6 @@ static inline unsigned long totalram_pages(void)
 	return (unsigned long)atomic_long_read(&_totalram_pages);
 }
 
-static inline void totalram_pages_inc(void)
-{
-	atomic_long_inc(&_totalram_pages);
-}
-
-static inline void totalram_pages_dec(void)
-{
-	atomic_long_dec(&_totalram_pages);
-}
-
 static inline void totalram_pages_add(long count)
 {
 	atomic_long_add(count, &_totalram_pages);
@@ -586,14 +577,6 @@ enum {
 #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
 #define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)
 
-/*
- * Special vmas that are non-mergable, non-mlock()able.
- */
-
-#define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \
-				       VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT)
-#define VM_SPECIAL vma_flags_to_legacy(VMA_SPECIAL_FLAGS)
-
 /*
  * Physically remapped pages are special. Tell the
  * rest of the world about it:
@@ -610,9 +593,6 @@ enum {
 #define VMA_REMAP_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT,	\
 				     VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)
 
-/* This mask prevents VMA from being scanned with khugepaged */
-#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
-
 /* This mask defines which mm->def_flags a process can inherit its parent */
 #define VM_INIT_DEF_MASK	VM_NOHUGEPAGE
 
@@ -928,7 +908,6 @@ static inline void vma_numab_state_free(struct vm_area_struct *vma) {}
  * These must be here rather than mmap_lock.h as dependent on vm_fault type,
  * declared in this header.
  */
-#ifdef CONFIG_PER_VMA_LOCK
 static inline void release_fault_lock(struct vm_fault *vmf)
 {
 	if (vmf->flags & FAULT_FLAG_VMA_LOCK)
@@ -944,17 +923,6 @@ static inline void assert_fault_locked(const struct vm_fault *vmf)
 	else
 		mmap_assert_locked(vmf->vma->vm_mm);
 }
-#else
-static inline void release_fault_lock(struct vm_fault *vmf)
-{
-	mmap_read_unlock(vmf->vma->vm_mm);
-}
-
-static inline void assert_fault_locked(const struct vm_fault *vmf)
-{
-	mmap_assert_locked(vmf->vma->vm_mm);
-}
-#endif /* CONFIG_PER_VMA_LOCK */
 
 static inline bool mm_flags_test(int flag, const struct mm_struct *mm)
 {
@@ -1551,11 +1519,6 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma)
 	vma->vm_ops = NULL;
 }
 
-static inline void vma_desc_set_anonymous(struct vm_area_desc *desc)
-{
-	desc->vm_ops = NULL;
-}
-
 static inline bool vma_is_anonymous(const struct vm_area_struct *vma)
 {
 	return !vma->vm_ops;
@@ -1639,6 +1602,211 @@ static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma)
 	return is_shared_maywrite(&vma->flags);
 }
 
+/**
+ * vma_flags_is_hugetlb() - Do the specified VMA flags indicate that the
+ * VMA is a hugetlb mapping?
+ * @flags: The VMA flags to test.
+ *
+ * Returns: true if the flags indicate a hugetlb mapping, false otherwise.
+ */
+static inline bool vma_flags_is_hugetlb(const vma_flags_t *flags)
+{
+	return IS_ENABLED(CONFIG_HUGETLB_PAGE) &&
+	       vma_flags_test(flags, VMA_HUGETLB_BIT);
+}
+
+/**
+ * vma_is_hugetlb() - Is @vma a hugetlb mapping?
+ * @vma: The VMA to test.
+ *
+ * Returns: true if @vma is a hugetlb mapping, false otherwise.
+ */
+static inline bool vma_is_hugetlb(const struct vm_area_struct *vma)
+{
+	return vma_flags_is_hugetlb(&vma->flags);
+}
+
+/**
+ * vma_flags_is_kernel_owned() - Do the specified VMA flags indicate that the
+ * contents of the VMA are owned by the kernel rather than the core mm?
+ * @flags: The VMA flags to test.
+ *
+ * A kernel-owned mapping is one whose contents are established and controlled
+ * by the kernel, typically a driver, rather than by the core mm's fault and
+ * rmap machinery.
+ *
+ * The mapping may be memory-mapped I/O, kernel-allocated pages or ordinary
+ * pages the owner has chosen to map itself (shmem via a PFN map, for instance).
+ *
+ * In all cases the core mm must not populate, reclaim, migrate, copy-on-write
+ * or merge it of its own accord.
+ *
+ * Pages mapped this way are not necessarily reference counted or map counted.
+ *
+ * Returns: true if the flags indicate a kernel-owned mapping.
+ */
+static inline bool vma_flags_is_kernel_owned(const vma_flags_t *flags)
+{
+	return vma_flags_test_any(flags, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}
+
+/**
+ * vma_is_kernel_owned() - Are the contents of @vma owned by the kernel?
+ * @vma: The VMA to test.
+ *
+ * See vma_flags_is_kernel_owned() for a description of this property.
+ *
+ * Returns: true if the VMA is kernel-owned.
+ */
+static inline bool vma_is_kernel_owned(const struct vm_area_struct *vma)
+{
+	return vma_flags_is_kernel_owned(&vma->flags);
+}
+
+/**
+ * vma_flags_is_fixed_mapping() - Do the specified VMA flags indicate that this
+ * is a fixed mapping that cannot be expanded or merged?
+ * @flags: The VMA flags to test.
+ *
+ * Fixed mappings are those whose size is set at the point of mmap (for
+ * instance, a kernel-owned mapping of a fixed range of memory), and thus
+ * cannot be expanded or merged.
+ *
+ * Returns: true if the flags indicate a fixed mapping.
+ */
+static inline bool vma_flags_is_fixed_mapping(const vma_flags_t *flags)
+{
+	/*
+	 * VMA_PFNMAP_BIT should imply VMA_DONTEXPAND_BIT, but some callers set
+	 * only the former.
+	 */
+	return vma_flags_test_any(flags, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT);
+}
+
+/**
+ * vma_is_fixed_mapping() - Is this VMA a fixed mapping that cannot be
+ * expanded or merged?
+ * @vma: The VMA to test.
+ *
+ * See vma_flags_is_fixed_mapping() for a description of this property.
+ *
+ * Returns: true if the VMA maps a fixed mapping.
+ */
+static inline bool vma_is_fixed_mapping(const struct vm_area_struct *vma)
+{
+	return vma_flags_is_fixed_mapping(&vma->flags);
+}
+
+/**
+ * vma_flags_can_merge() - Do the specified VMA flags permit the VMA to be
+ * merged with another?
+ * @flags: The VMA flags to test.
+ * Returns: true if the flags permit merging, false otherwise.
+ */
+static inline bool vma_flags_can_merge(const vma_flags_t *flags)
+{
+	/*
+	 * VMA merging assumes that a VMA's flags and fields completely describe
+	 * its state.
+	 *
+	 * However, kernel-owned mappings may have established state upon mapping
+	 * not embodied in any attribute of the VMA.
+	 *
+	 * Additionally, private (CoW) PFN maps encode the source PFN of the
+	 * range in vma->vm_pgoff, which may otherwise cause spurious merges.
+	 */
+	if (vma_flags_is_kernel_owned(flags))
+		return false;
+	/* VMA explicitly marked as being unmergeable. */
+	if (vma_flags_is_fixed_mapping(flags))
+		return false;
+
+	return true;
+}
+
+/**
+ * vma_can_merge() - Do @vma's flags permit it to be merged with another VMA?
+ * @vma: The VMA to test.
+ * Returns: true if the flags permit merging, otherwise false.
+ */
+static inline bool vma_can_merge(const struct vm_area_struct *vma)
+{
+	return vma_flags_can_merge(&vma->flags);
+}
+
+/**
+ * vma_flags_is_persistent() - Do the specified VMA flags imply that the VMA
+ * contains persistent data?
+ * @flags: The VMA flags to test.
+ *
+ * Persistent in the sense that - if you write bytes to the mapping - do they
+ * stay written?
+ *
+ * If the kernel or a device could write to the memory independently of
+ * userland, or the kernel could arbitrarily discard it, then it is not
+ * persistent.
+ *
+ * Returns: true if the flags imply this VMA is persistent, otherwise false.
+ */
+static inline bool vma_flags_is_persistent(const vma_flags_t *flags)
+{
+	/* hugetlb is a fixed mapping, but its contents are the user's own. */
+	if (vma_flags_is_hugetlb(flags))
+		return true;
+	/*
+	 * MMIO mappings may not store what is written and may be changed by the
+	 * device. Kernel-owned and fixed mappings may be changed by their owner
+	 * without the user having initiated it.
+	 */
+	if (vma_flags_is_kernel_owned(flags) ||
+	    vma_flags_is_fixed_mapping(flags))
+		return false;
+	/* Droppable memory is discardable by definition. */
+	return !vma_flags_test_single_mask(flags, VMA_DROPPABLE);
+}
+
+/**
+ * vma_is_persistent() - Does the VMA contain persistent data?
+ * @vma: The VMA to test.
+ *
+ * See vma_flags_is_persistent() for details.
+ *
+ * Returns: true if the VMA is persistent, otherwise false.
+ */
+static inline bool vma_is_persistent(const struct vm_area_struct *vma)
+{
+	return vma_flags_is_persistent(&vma->flags);
+}
+
+/**
+ * vma_flags_can_gup() - Do the specified VMA flags permit GUP to access the
+ * mapping's pages?
+ * @flags: The VMA flags to test.
+ *
+ * GUP cannot obtain pages from a PFN map (VMA_PFNMAP_BIT), which may have no
+ * struct pages behind it, and must not provide access to memory-mapped I/O
+ * (VMA_IO_BIT).
+ *
+ * Returns: true if GUP may access pages from the mapping, otherwise false.
+ */
+static inline bool vma_flags_can_gup(const vma_flags_t *flags)
+{
+	return !vma_flags_test_any(flags, VMA_IO_BIT, VMA_PFNMAP_BIT);
+}
+
+/**
+ * vma_can_gup() - May GUP obtain pages from @vma?
+ * @vma: The VMA to test.
+ *
+ * See vma_flags_can_gup() for details.
+ *
+ * Returns: true if GUP may access pages from the mapping, otherwise false.
+ */
+static inline bool vma_can_gup(const struct vm_area_struct *vma)
+{
+	return vma_flags_can_gup(&vma->flags);
+}
+
 /**
  * vma_kernel_pagesize - Default page size granularity for this VMA.
  * @vma: The user mapping.
@@ -2075,20 +2243,21 @@ vm_fault_t finish_fault(struct vm_fault *vmf);
  *
  * A pagecache page contains an opaque `private' member, which belongs to the
  * page's address_space. Usually, this is the address of a circular list of
- * the page's disk buffers. PG_private must be set to tell the VM to call
- * into the filesystem to release these pages.
+ * the page's disk buffers. It tells the VM to call into the filesystem to
+ * release these pages.
  *
  * A folio may belong to an inode's memory mapping. In this case,
  * folio->mapping points to the inode, and folio->index is the file
  * offset of the folio, in units of PAGE_SIZE.
  *
- * If pagecache pages are not associated with an inode, they are said to be
- * anonymous pages. These may become associated with the swapcache, and in that
- * case PG_swapcache is set, and page->private is an offset into the swapcache.
+ * If pagecache folios are not associated with an inode, they are said to be
+ * anonymous folios. These may become associated with the swapcache, and in that
+ * case PG_swapcache is set, and folio->private is an offset into the swapcache.
  *
  * In either case (swapcache or inode backed), the pagecache itself holds one
- * reference to the page. Setting PG_private should also increment the
- * refcount. The each user mapping also has a reference to the page.
+ * reference to the folio. Attaching filesystem private data via
+ * folio_attach_private() also increments the refcount. Each user mapping also
+ * has a reference to the folio.
  *
  * The pagecache pages are stored in a per-mapping radix tree, which is
  * rooted at mapping->i_pages, and indexed by offset.
@@ -2644,12 +2813,23 @@ static inline void set_page_section(struct page *page, unsigned long section)
 	page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
 }
 
+static inline void set_page_section_from_pfn(struct page *page,
+		unsigned long pfn)
+{
+	set_page_section(page, pfn_to_section_nr(pfn));
+}
+
 static inline unsigned long memdesc_section(const memdesc_flags_t *mdf)
 {
 	ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT);
 	return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
 }
 #else /* !SECTION_IN_PAGE_FLAGS */
+static inline void set_page_section_from_pfn(struct page *page,
+		unsigned long pfn)
+{
+}
+
 static inline unsigned long memdesc_section(const memdesc_flags_t *mdf)
 {
 	return 0;
@@ -2872,9 +3052,7 @@ static inline void set_page_links(struct page *page, enum zone_type zone,
 {
 	set_page_zone(page, zone);
 	set_page_node(page, node);
-#ifdef SECTION_IN_PAGE_FLAGS
-	set_page_section(page, pfn_to_section_nr(pfn));
-#endif
+	set_page_section_from_pfn(page, pfn);
 }
 
 /**
@@ -3022,9 +3200,9 @@ static inline bool folio_maybe_mapped_shared(struct folio *folio)
  * @folio: the folio
  *
  * Calculate the expected folio refcount, taking references from the pagecache,
- * swapcache, PG_private and page table mappings into account. Useful in
- * combination with folio_ref_count() to detect unexpected references (e.g.,
- * GUP or other temporary references).
+ * swapcache, private data (folio->private != NULL) and page table mappings into
+ * account. Useful in combination with folio_ref_count() to detect unexpected
+ * references (e.g., GUP or other temporary references).
  *
  * Does currently not consider references from the LRU cache. If the folio
  * was isolated from the LRU (which is the case during migration or split),
@@ -3062,10 +3240,15 @@ static inline int folio_expected_ref_count(const struct folio *folio)
 	ref_count += folio_test_swapcache(folio) << order;
 
 	if (!folio_test_anon(folio)) {
-		/* One reference per page from the pagecache. */
-		ref_count += !!folio->mapping << order;
-		/* One reference from PG_private. */
-		ref_count += folio_test_private(folio);
+		/*
+		 * One reference per page from the pagecache.
+		 * Use data_race() since folio might not be locked.
+		 */
+		ref_count += !!data_race(folio->mapping) << order;
+		/*
+		 * One reference from filesystem private data.
+		 */
+		ref_count += folio_has_attached_private(folio);
 	}
 
 	/* One reference per page table mapping. */
@@ -3325,10 +3508,10 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
 		void *buf, int len, unsigned int gup_flags);
 
-#ifdef CONFIG_BPF_SYSCALL
-extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
-			      void *buf, int len, unsigned int gup_flags);
-#endif
+int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr,
+		void *buf, int len, unsigned int gup_flags);
+int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
+		void *buf, int len, unsigned int gup_flags);
 
 long get_user_pages_remote(struct mm_struct *mm,
 			   unsigned long start, unsigned long nr_pages,
@@ -4083,12 +4266,6 @@ static inline void free_reserved_page(struct page *page)
 	free_reserved_pages(page, 0);
 }
 
-static inline void mark_page_reserved(struct page *page)
-{
-	SetPageReserved(page);
-	adjust_managed_page_count(page, -1);
-}
-
 static inline void free_reserved_ptdesc(struct ptdesc *pt)
 {
 	free_reserved_page(ptdesc_page(pt));
@@ -4181,6 +4358,8 @@ void mapping_rmap_tree_insert_after(struct vm_area_struct *vma,
 				    struct address_space *mapping);
 void mapping_rmap_tree_remove(struct vm_area_struct *vma,
 			      struct address_space *mapping);
+void mapping_rmap_tree_update_inplace(struct vm_area_struct *vma);
+
 struct vm_area_struct *
 mapping_rmap_tree_iter_first(struct address_space *mapping,
 			     pgoff_t pgoff_start, pgoff_t pgoff_last);
@@ -4198,6 +4377,7 @@ void anon_rmap_tree_insert(struct anon_vma_chain *avc,
 			   struct anon_vma *anon_vma);
 void anon_rmap_tree_remove(struct anon_vma_chain *avc,
 			   struct anon_vma *anon_vma);
+void anon_rmap_tree_update_inplace(struct anon_vma_chain *avc);
 struct anon_vma_chain *
 anon_rmap_tree_iter_first(struct anon_vma *anon_vma,
 			  pgoff_t pgoff_start, pgoff_t pgoff_last);
@@ -4408,9 +4588,8 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma)
  * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the
  * page offset within the file.
  *
- * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies
- * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap
- * regions have their page offset set to the first PFN in the range.
+ * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions
+ * have their page offset set to the first PFN in the range.
  *
  * Returns: The page offset of the start of @vma.
  */
@@ -4619,7 +4798,7 @@ static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc,
  * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped.
  * @start: The virtual address from which to map them.
  * @pages: An array of struct page pointers describing the memory to map.
- * @nr_pages: The number of entries in the @pages aray.
+ * @nr_pages: The number of entries in the @pages array.
  */
 static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc,
 		unsigned long start, struct page **pages,
@@ -4627,7 +4806,7 @@ static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc,
 {
 	struct mmap_action *action = &desc->action;
 
-	action->type = MMAP_MAP_KERNEL_PAGES;
+	action->type = MMAP_KERNEL_PAGES;
 	action->map_kernel.start = start;
 	action->map_kernel.pages = pages;
 	action->map_kernel.nr_pages = nr_pages;
@@ -4651,10 +4830,55 @@ static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc,
 				     vma_desc_pages(desc));
 }
 
+static inline
+void mmap_action_map_discontig_kernel_pages(struct vm_area_desc *desc,
+		void *init_private, const struct discontig_kernel_page_ops *ops)
+{
+	struct mmap_action *action = &desc->action;
+
+	action->type = MMAP_DISCONTIG_KERNEL_PAGES;
+	action->map_kernel_discontig.init_private = init_private;
+	action->map_kernel_discontig.ops = ops;
+}
+
 int mmap_action_prepare(struct vm_area_desc *desc);
 int mmap_action_complete(struct vm_area_struct *vma,
 			 struct mmap_action *action, bool is_compat);
 
+static inline void
+discontig_kernel_map_abort(struct discontig_kernel_page_state *state)
+{
+	state->action = DISCONTIG_KERNEL_PAGE_ABORT;
+}
+
+static inline void
+discontig_kernel_map_page(struct discontig_kernel_page_state *state,
+			  struct page *page)
+{
+	struct folio *folio = page_folio(page);
+
+	if (folio_test_large(folio)) {
+		VM_WARN_ON_ONCE(page != folio_page(folio, 0));
+		state->action = DISCONTIG_KERNEL_PAGE_MAP_COMPOUND_PAGE;
+		state->__folio = folio;
+		state->__nr_pages = min(state->nr_pages_remain,
+					folio_nr_pages(folio));
+	} else {
+		state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGE;
+		state->__page = page;
+		state->__nr_pages = 1;
+	}
+}
+
+static inline void
+discontig_kernel_map_page_range(struct discontig_kernel_page_state *state,
+				struct page **page_arr, unsigned long nr_pages)
+{
+	state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE;
+	state->__page_arr = page_arr;
+	state->__nr_pages = nr_pages;
+}
+
 /* Look up the first VMA which exactly match the interval vm_start ... vm_end */
 static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
 				unsigned long vm_start, unsigned long vm_end)
@@ -4772,9 +4996,6 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
 int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
 			struct page **pages, unsigned long *num);
-int map_kernel_pages_prepare(struct vm_area_desc *desc);
-int map_kernel_pages_complete(struct vm_area_struct *vma,
-			      struct mmap_action *action);
 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
 				unsigned long num);
 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
@@ -5140,7 +5361,6 @@ static inline void print_vma_addr(char *prefix, unsigned long rip)
 }
 #endif
 
-unsigned long section_map_size(void);
 struct page * __populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
 		struct dev_pagemap *pgmap);
@@ -5159,9 +5379,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end,
 			       int node, struct vmem_altmap *altmap);
 int vmemmap_populate(unsigned long start, unsigned long end, int node,
 		struct vmem_altmap *altmap);
-int vmemmap_populate_hvo(unsigned long start, unsigned long end,
-			 unsigned int order, struct zone *zone,
-			 unsigned long headsize);
 void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node,
 			  unsigned long headsize);
 void vmemmap_populate_print_last(void);
@@ -5196,7 +5413,6 @@ static inline void vmem_altmap_free(struct vmem_altmap *altmap,
 }
 #endif
 
-#define VMEMMAP_RESERVE_NR	2
 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
 static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap,
 					  struct dev_pagemap *pgmap)
@@ -5204,6 +5420,9 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap,
 	unsigned long nr_pages;
 	unsigned long nr_vmemmap_pages;
 
+	if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION))
+		return false;
+
 	if (!pgmap || !is_power_of_2(sizeof(struct page)))
 		return false;
 
@@ -5213,7 +5432,7 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap,
 	 * For vmemmap optimization with DAX we need minimum 2 vmemmap
 	 * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst
 	 */
-	return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR);
+	return !altmap && (nr_vmemmap_pages > VMEMMAP_OPTIMIZATION_PAGES);
 }
 /*
  * If we don't have an architecture override, use the generic rule
@@ -5254,6 +5473,8 @@ extern const struct attribute_group memory_failure_attr_group;
 extern void memory_failure_queue(unsigned long pfn, int flags);
 void num_poisoned_pages_inc(unsigned long pfn);
 void num_poisoned_pages_sub(unsigned long pfn, long i);
+phys_addr_t range_first_hwpoison(phys_addr_t start, unsigned long size);
+phys_addr_t range_last_hwpoison(phys_addr_t start, unsigned long size);
 #else
 static inline void memory_failure_queue(unsigned long pfn, int flags)
 {
@@ -5266,6 +5487,18 @@ static inline void num_poisoned_pages_inc(unsigned long pfn)
 static inline void num_poisoned_pages_sub(unsigned long pfn, long i)
 {
 }
+
+static inline phys_addr_t range_first_hwpoison(phys_addr_t start,
+					       unsigned long size)
+{
+	return PHYS_ADDR_MAX;
+}
+
+static inline phys_addr_t range_last_hwpoison(phys_addr_t start,
+					      unsigned long size)
+{
+	return PHYS_ADDR_MAX;
+}
 #endif
 
 #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 621c8653d8f7eb..ab69b9930893ff 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -30,6 +30,17 @@ static inline int folio_is_file_lru(const struct folio *folio)
 	return !folio_test_swapbacked(folio);
 }
 
+/**
+ * folio_flags_is_file_lru - Should the folio be on a file LRU or anon LRU?
+ * @flags: The folio's flags.
+ *
+ * Just like folio_is_file_lru but take the folio flags directly instead.
+ */
+static inline int folio_flags_is_file_lru(const unsigned long *flags)
+{
+	return !test_bit(PG_swapbacked, flags);
+}
+
 static __always_inline void __update_lru_size(struct lruvec *lruvec,
 				enum lru_list lru, enum zone_type zid,
 				long nr_pages)
@@ -142,10 +153,66 @@ static inline int lru_tier_from_refs(int refs, bool workingset)
 	return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs);
 }
 
-static inline int folio_lru_refs(const struct folio *folio)
+/**
+ * lru_set_gen_flags - Set the LRU generation number to specified folio flags.
+ * @flags: pointer to the folio flags
+ * @gen: generation number, between 0 and (MAX_NR_GENS - 1), inclusive.
+ */
+static inline void lru_set_gen_flags(unsigned long *flags, int gen)
+{
+	BUILD_BUG_ON(LRU_GEN_MASK & LRU_REFS_MASK);
+	VM_WARN_ON_ONCE(gen >= MAX_NR_GENS || gen < 0);
+	/* Store gen offset by 1, zero means the folio is off-list. */
+	*flags &= ~LRU_GEN_MASK;
+	*flags |= (gen + 1UL) << LRU_GEN_PGOFF;
+}
+
+/**
+ * lru_get_gen_flags - Return the LRU generation number from folio flags.
+ * @flags: folio flags
+ *
+ * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns
+ * -1 if the flags indicate the folio is off the list (e.g., isolated).
+ */
+static inline int lru_get_gen_flags(unsigned long flags)
+{
+	int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+
+	/* Exclude the legal -1 from the unsigned MAX_NR_GENS comparison */
+	VM_WARN_ON_ONCE(gen != -1 && gen >= MAX_NR_GENS);
+	return gen;
+}
+
+/**
+ * lru_set_refs_flags - Set the LRU referenced count to folio flags.
+ * @flags: pointer to the folio flags
+ * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive.
+ *
+ * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the
+ * remaining refs. For classical LRU the extra bits are not used, so it can
+ * also be seen as the refs count never exceeds 1. In both cases, refs == 1
+ * means PG_referenced is set and the extra bits are zero, and refs == 0 means
+ * PG_referenced and the extra bits are all unset.
+ */
+static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs)
 {
-	unsigned long flags = READ_ONCE(folio->flags.f);
+	VM_WARN_ON_ONCE(refs > LRU_REFS_MAX);
+	BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1);
 
+	*flags &= ~LRU_REFS_FLAGS;
+	if (!refs)
+		return;
+	*flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF));
+}
+
+/**
+ * lru_get_refs_flags - Return LRU referenced / access count from folio flags.
+ * @flags: folio flags
+ *
+ * Reads the LRU referenced count set by lru_set_refs_flags().
+ */
+static inline int lru_get_refs_flags(unsigned long flags)
+{
 	if (!(flags & BIT(PG_referenced)))
 		return 0;
 	/*
@@ -155,11 +222,24 @@ static inline int folio_lru_refs(const struct folio *folio)
 	return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1;
 }
 
-static inline int folio_lru_gen(const struct folio *folio)
+static inline int folio_lru_refs(const struct folio *folio)
 {
-	unsigned long flags = READ_ONCE(folio->flags.f);
+	return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0)));
+}
 
-	return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs)
+{
+	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+
+	do {
+		new_flags = old_flags;
+		lru_set_refs_flags(&new_flags, refs);
+	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
+}
+
+static inline int folio_lru_gen(const struct folio *folio)
+{
+	return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0)));
 }
 
 static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen)
@@ -270,7 +350,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio,
 	gen = lru_gen_from_seq(seq);
 	flags = (gen + 1UL) << LRU_GEN_PGOFF;
 	/* see the comment on MIN_NR_GENS about PG_active */
-	set_mask_bits(&folio->flags.f, LRU_GEN_MASK | BIT(PG_active), flags);
+	set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags);
 
 	lru_gen_update_size(lruvec, folio, -1, gen);
 	/* for folio_rotate_reclaimable() */
@@ -295,7 +375,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
 
 	/* for folio_migrate_flags() */
 	flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
-	flags = set_mask_bits(&folio->flags.f, LRU_GEN_MASK, flags);
+	flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags);
 	gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
 
 	lru_gen_update_size(lruvec, folio, gen, -1);
@@ -304,11 +384,19 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
 	return true;
 }
 
-static inline void folio_migrate_refs(struct folio *new, const struct folio *old)
+/**
+ * folio_migrate_lru_refs - copy the reference state to a new folio
+ * @new: the destination folio
+ * @old: the source folio
+ *
+ * Transfer the reference state to @new during migration: the MGLRU
+ * refs count, including PG_referenced, or just PG_referenced for the
+ * active/inactive LRU.
+ */
+static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
 {
-	unsigned long refs = READ_ONCE(old->flags.f) & LRU_REFS_MASK;
-
-	set_mask_bits(&new->flags.f, LRU_REFS_MASK, refs);
+	BUILD_BUG_ON(LRU_REFS_MASK & BIT(PG_referenced));
+	folio_set_lru_refs(new, folio_lru_refs(old));
 }
 #else /* !CONFIG_LRU_GEN */
 
@@ -337,9 +425,10 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio,
 	return false;
 }
 
-static inline void folio_migrate_refs(struct folio *new, const struct folio *old)
+static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old)
 {
-
+	if (folio_test_referenced(old))
+		folio_set_referenced(new);
 }
 #endif /* CONFIG_LRU_GEN */
 
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 6d815f6440c94e..6141160ec6526b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -108,7 +108,7 @@ struct page {
 			};
 			/**
 			 * @private: Mapping-private opaque data.
-			 * Usually used for buffer_heads if PagePrivate.
+			 * Usually used for buffer_heads.
 			 * Used for swp_entry_t if swapcache flag set.
 			 * Indicates order in the buddy system if PageBuddy
 			 * or on pcp_llist.
@@ -675,7 +675,7 @@ static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc)
 #define STRUCT_PAGE_MAX_SHIFT	(order_base_2(sizeof(struct page)))
 
 /*
- * page_private can be used on tail pages.  However, PagePrivate is only
+ * page_private can be used on tail pages.  However, it is only
  * checked by the VM on the head page.  So page_private on the tail pages
  * should be used for data that's ancillary to the head page (eg attaching
  * buffer heads to tail pages after attaching buffer heads to the head page)
@@ -759,7 +759,7 @@ static inline struct anon_vma_name *anon_vma_name_alloc(const char *name)
 /*
  * While __vma_enter_locked() is working to ensure are no read-locks held on a
  * VMA (either while acquiring a VMA write lock or marking a VMA detached) we
- * set the VM_REFCNT_EXCLUDE_READERS_FLAG in vma->vm_refcnt to indiciate to
+ * set the VM_REFCNT_EXCLUDE_READERS_FLAG in vma->vm_refcnt to indicate to
  * vma_start_read() that the reference count should be left alone.
  *
  * See the comment describing vm_refcnt in vm_area_struct for details as to
@@ -815,11 +815,47 @@ struct pfnmap_track_ctx {
 
 /* What action should be taken after an .mmap_prepare call is complete? */
 enum mmap_action_type {
-	MMAP_NOTHING,		/* Mapping is complete, no further action. */
-	MMAP_REMAP_PFN,		/* Remap PFN range. */
-	MMAP_IO_REMAP_PFN,	/* I/O remap PFN range. */
-	MMAP_SIMPLE_IO_REMAP,	/* I/O remap with guardrails. */
-	MMAP_MAP_KERNEL_PAGES,	/* Map kernel page range from array. */
+	MMAP_NOTHING,
+	MMAP_REMAP_PFN,
+	MMAP_IO_REMAP_PFN,
+	MMAP_SIMPLE_IO_REMAP,		/* I/O remap with guardrails. */
+	MMAP_KERNEL_PAGES,		/* Map kernel page range from array. */
+	MMAP_DISCONTIG_KERNEL_PAGES,	/* Map kernel discontig page range. */
+};
+
+enum discontig_kernel_page_action {
+	DISCONTIG_KERNEL_PAGE_ABORT,
+	DISCONTIG_KERNEL_PAGE_MAP_PAGE,
+	DISCONTIG_KERNEL_PAGE_MAP_COMPOUND_PAGE,
+	DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE,
+};
+
+struct discontig_kernel_page_state {
+	/* Map state. */
+	const unsigned long start;	/* Start address of VMA. */
+	const unsigned long end;	/* End address of VMA. */
+	unsigned long addr;		/* The current address to be mapped. */
+	pgoff_t pgoff;			/* The current pgoff to be mapped. */
+	unsigned long nr_pages_mapped;	/* The number of pages mapped. */
+	unsigned long nr_pages_remain;	/* The number of pages remaining. */
+
+	/* User-defined state. */
+	void *vm_private_data;		/* VMA private data. */
+	void *private;			/* Mapping private data. */
+
+	/* Users should not touch these, use discontig_kernel_map_*() helpers. */
+	enum discontig_kernel_page_action action;
+	union {
+		struct page *__page;
+		struct folio *__folio;
+		struct page **__page_arr;
+	};
+	unsigned long __nr_pages;
+};
+
+struct discontig_kernel_page_ops {
+	int (*init)(void *vm_private_data, void **private);
+	int (*get)(struct discontig_kernel_page_state *state);
 };
 
 /*
@@ -844,6 +880,10 @@ struct mmap_action {
 			unsigned long nr_pages;
 			pgoff_t pgoff;
 		} map_kernel;
+		struct {
+			void *init_private;
+			const struct discontig_kernel_page_ops *ops;
+		} map_kernel_discontig;
 	};
 	enum mmap_action_type type;
 
@@ -950,7 +990,6 @@ struct vm_area_struct {
 		vma_flags_t flags;
 	};
 
-#ifdef CONFIG_PER_VMA_LOCK
 	/*
 	 * Can only be written (using WRITE_ONCE()) while holding both:
 	 *  - mmap_lock (in write mode)
@@ -966,7 +1005,7 @@ struct vm_area_struct {
 	 * slowpath.
 	 */
 	unsigned int vm_lock_seq;
-#endif
+
 	/*
 	 * Low 32-bits of anonymous page offset.
 	 * See vma_start_anon_pgoff() comment for details.
@@ -1003,7 +1042,6 @@ struct vm_area_struct {
 #ifdef CONFIG_NUMA_BALANCING
 	struct vma_numab_state *numab_state;	/* NUMA Balancing state */
 #endif
-#ifdef CONFIG_PER_VMA_LOCK
 	/*
 	 * Used to keep track of firstly, whether the VMA is attached, secondly,
 	 * if attached, how many read locks are taken, and thirdly, if the
@@ -1046,7 +1084,6 @@ struct vm_area_struct {
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	struct lockdep_map vmlock_dep_map;
 #endif
-#endif
 #ifdef CONFIG_64BIT
 	/*
 	 * High 32-bits of anonymous page offset.
@@ -1226,7 +1263,7 @@ struct mm_struct {
 		struct mm_mm_cid mm_cid;
 
 		/* sched_cache related statistics */
-		struct sched_cache_stat sc_stat;
+		struct sched_cache_group *sched_cache_grp;
 #ifdef CONFIG_MMU
 		atomic_long_t pgtables_bytes;	/* size of all page tables */
 #endif
@@ -1254,7 +1291,6 @@ struct mm_struct {
 					  * init_mm.mmlist, and are protected
 					  * by mmlist_lock
 					  */
-#ifdef CONFIG_PER_VMA_LOCK
 		struct rcuwait vma_writer_wait;
 		/*
 		 * This field has lock-like semantics, meaning it is sometimes
@@ -1274,7 +1310,7 @@ struct mm_struct {
 		 * mmap_lock.
 		 */
 		seqcount_t mm_lock_seq;
-#endif
+
 		struct futex_mm_data	futex;
 
 		unsigned long hiwater_rss; /* High-watermark of RSS usage */
@@ -1624,8 +1660,9 @@ static inline unsigned int mm_cid_size(void)
 #endif /* CONFIG_SCHED_MM_CID */
 
 #ifdef CONFIG_SCHED_CACHE
-void mm_init_sched(struct mm_struct *mm,
-		   struct sched_cache_time __percpu *pcpu_sched);
+int mm_init_sched(struct mm_struct *mm,
+		  struct sched_cache_time __percpu *pcpu_sched);
+void mm_destroy_sched(struct mm_struct *mm);
 
 static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
 {
@@ -1635,17 +1672,11 @@ static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
 	if (!pcpu_sched)
 		return -ENOMEM;
 
-	mm_init_sched(mm, pcpu_sched);
-	return 0;
+	return mm_init_sched(mm, pcpu_sched);
 }
 
 #define mm_alloc_sched(...)	alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__))
 
-static inline void mm_destroy_sched(struct mm_struct *mm)
-{
-	free_percpu(mm->sc_stat.pcpu_sched);
-	mm->sc_stat.pcpu_sched = NULL;
-}
 #else /* !CONFIG_SCHED_CACHE */
 
 static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; }
@@ -1980,7 +2011,7 @@ enum {
 /*
  * MMF_HAS_PINNED: Whether this mm has pinned any pages.  This can be either
  * replaced in the future by mm.pinned_vm when it becomes stable, or grow into
- * a counter on its own. We're aggresive on this bit for now: even if the
+ * a counter on its own. We're aggressive on this bit for now: even if the
  * pinned pages were unpinned later on, we'll still keep this bit set for the
  * lifecycle of this mm, just for simplicity.
  */
diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h
index b8a13b8d36a45b..e5553f4a414cd1 100644
--- a/include/linux/mmap_lock.h
+++ b/include/linux/mmap_lock.h
@@ -76,8 +76,6 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm)
 	rwsem_assert_held_write(&mm->mmap_lock);
 }
 
-#ifdef CONFIG_PER_VMA_LOCK
-
 #ifdef CONFIG_LOCKDEP
 #define __vma_lockdep_map(vma) (&vma->vmlock_dep_map)
 #else
@@ -230,10 +228,14 @@ static inline void vma_refcount_put(struct vm_area_struct *vma)
 }
 
 /*
- * Use only while holding mmap read lock which guarantees that locking will not
- * fail (nobody can concurrently write-lock the vma). vma_start_read() should
+ * Use only while holding mmap read lock which guarantees that vma lock is not
+ * contended (nobody can concurrently write-lock the vma). vma_start_read() should
  * not be used in such cases because it might fail due to mm_lock_seq overflow.
  * This functionality is used to obtain vma read lock and drop the mmap read lock.
+ *
+ * VMA can't be detached while we are holding mmap lock, therefore in practice this
+ * function can fail only when there are so many readers that vm_refcnt overflows.
+ * The failure case is very unlikely and is already annotated as such internally.
  */
 static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass)
 {
@@ -249,22 +251,29 @@ static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int
 }
 
 /*
- * Use only while holding mmap read lock which guarantees that locking will not
- * fail (nobody can concurrently write-lock the vma). vma_start_read() should
+ * Use only while holding mmap read lock which guarantees that vma lock is not
+ * contended (nobody can concurrently write-lock the vma). vma_start_read() should
  * not be used in such cases because it might fail due to mm_lock_seq overflow.
  * This functionality is used to obtain vma read lock and drop the mmap read lock.
+ *
+ * VMA can't be detached while we are holding mmap lock, therefore in practice this
+ * function can fail only when there are so many readers that vm_refcnt overflows.
+ * The failure case is very unlikely and is already annotated as such internally.
  */
 static inline bool vma_start_read_locked(struct vm_area_struct *vma)
 {
 	return vma_start_read_locked_nested(vma, 0);
 }
 
+struct vm_area_struct *vma_start_read_unlocked(struct mm_struct *mm,
+					       unsigned long address);
+
 static inline void vma_end_read(struct vm_area_struct *vma)
 {
 	vma_refcount_put(vma);
 }
 
-static inline unsigned int __vma_raw_mm_seqnum(struct vm_area_struct *vma)
+static inline unsigned int __vma_raw_mm_seqnum(const struct vm_area_struct *vma)
 {
 	const struct mm_struct *mm = vma->vm_mm;
 
@@ -279,7 +288,7 @@ static inline unsigned int __vma_raw_mm_seqnum(struct vm_area_struct *vma)
  *
  * Returns true if write-locked, otherwise false.
  */
-static inline bool __is_vma_write_locked(struct vm_area_struct *vma)
+static inline bool __is_vma_write_locked(const struct vm_area_struct *vma)
 {
 	/*
 	 * current task is holding mmap_write_lock, both vma->vm_lock_seq and
@@ -297,6 +306,9 @@ int __vma_start_write(struct vm_area_struct *vma, int state);
  */
 static inline void vma_start_write(struct vm_area_struct *vma)
 {
+	if (!IS_ENABLED(CONFIG_MMU))
+		return;
+
 	if (__is_vma_write_locked(vma))
 		return;
 
@@ -319,6 +331,9 @@ static inline void vma_start_write(struct vm_area_struct *vma)
 static inline __must_check
 int vma_start_write_killable(struct vm_area_struct *vma)
 {
+	if (!IS_ENABLED(CONFIG_MMU))
+		return 0;
+
 	if (__is_vma_write_locked(vma))
 		return 0;
 
@@ -329,8 +344,13 @@ int vma_start_write_killable(struct vm_area_struct *vma)
  * vma_assert_write_locked() - assert that @vma holds a VMA write lock.
  * @vma: The VMA to assert.
  */
-static inline void vma_assert_write_locked(struct vm_area_struct *vma)
+static inline void vma_assert_write_locked(const struct vm_area_struct *vma)
 {
+	if (!IS_ENABLED(CONFIG_MMU)) {
+		mmap_assert_write_locked(vma->vm_mm);
+		return;
+	}
+
 	VM_WARN_ON_ONCE_VMA(!__is_vma_write_locked(vma), vma);
 }
 
@@ -339,10 +359,15 @@ static inline void vma_assert_write_locked(struct vm_area_struct *vma)
  * lock and is not detached.
  * @vma: The VMA to assert.
  */
-static inline void vma_assert_locked(struct vm_area_struct *vma)
+static inline void vma_assert_locked(const struct vm_area_struct *vma)
 {
 	unsigned int refcnt;
 
+	if (!IS_ENABLED(CONFIG_MMU)) {
+		mmap_assert_locked(vma->vm_mm);
+		return;
+	}
+
 	if (IS_ENABLED(CONFIG_LOCKDEP)) {
 		if (!lock_is_held(__vma_lockdep_map(vma)))
 			vma_assert_write_locked(vma);
@@ -385,7 +410,7 @@ static inline void vma_assert_locked(struct vm_area_struct *vma)
  * With lockdep disabled we may sometimes race with other threads acquiring the
  * mmap read lock simultaneous with our VMA read lock.
  */
-static inline void vma_assert_stabilised(struct vm_area_struct *vma)
+static inline void vma_assert_stabilised(const struct vm_area_struct *vma)
 {
 	/*
 	 * If another thread owns an mmap lock, it may go away at any time, and
@@ -420,7 +445,7 @@ static inline void vma_assert_stabilised(struct vm_area_struct *vma)
 	vma_assert_locked(vma);
 }
 
-static inline bool vma_is_attached(struct vm_area_struct *vma)
+static inline bool vma_is_attached(const struct vm_area_struct *vma)
 {
 	return refcount_read(&vma->vm_refcnt);
 }
@@ -432,6 +457,9 @@ static inline bool vma_is_attached(struct vm_area_struct *vma)
  */
 static inline void vma_assert_attached(struct vm_area_struct *vma)
 {
+	if (!IS_ENABLED(CONFIG_MMU))
+		return;
+
 	WARN_ON_ONCE(!vma_is_attached(vma));
 }
 
@@ -442,6 +470,9 @@ static inline void vma_assert_detached(struct vm_area_struct *vma)
 
 static inline void vma_mark_attached(struct vm_area_struct *vma)
 {
+	if (!IS_ENABLED(CONFIG_MMU))
+		return;
+
 	vma_assert_write_locked(vma);
 	vma_assert_detached(vma);
 	refcount_set_release(&vma->vm_refcnt, 1);
@@ -451,6 +482,9 @@ void __vma_exclude_readers_for_detach(struct vm_area_struct *vma);
 
 static inline void vma_mark_detached(struct vm_area_struct *vma)
 {
+	if (!IS_ENABLED(CONFIG_MMU))
+		return;
+
 	vma_assert_write_locked(vma);
 	vma_assert_attached(vma);
 
@@ -484,54 +518,6 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm,
 				     struct vma_iterator *iter,
 				     unsigned long address);
 
-#else /* CONFIG_PER_VMA_LOCK */
-
-static inline void mm_lock_seqcount_init(struct mm_struct *mm) {}
-static inline void mm_lock_seqcount_begin(struct mm_struct *mm) {}
-static inline void mm_lock_seqcount_end(struct mm_struct *mm) {}
-
-static inline bool mmap_lock_speculate_try_begin(struct mm_struct *mm, unsigned int *seq)
-{
-	return false;
-}
-
-static inline bool mmap_lock_speculate_retry(struct mm_struct *mm, unsigned int seq)
-{
-	return true;
-}
-static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) {}
-static inline void vma_end_read(struct vm_area_struct *vma) {}
-static inline void vma_start_write(struct vm_area_struct *vma) {}
-static inline __must_check
-int vma_start_write_killable(struct vm_area_struct *vma) { return 0; }
-static inline void vma_assert_write_locked(struct vm_area_struct *vma)
-		{ mmap_assert_write_locked(vma->vm_mm); }
-static inline bool vma_is_attached(struct vm_area_struct *vma)
-		{ return true; }
-static inline void vma_assert_attached(struct vm_area_struct *vma) {}
-static inline void vma_assert_detached(struct vm_area_struct *vma) {}
-static inline void vma_mark_attached(struct vm_area_struct *vma) {}
-static inline void vma_mark_detached(struct vm_area_struct *vma) {}
-
-static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
-		unsigned long address)
-{
-	return NULL;
-}
-
-static inline void vma_assert_locked(struct vm_area_struct *vma)
-{
-	mmap_assert_locked(vma->vm_mm);
-}
-
-static inline void vma_assert_stabilised(struct vm_area_struct *vma)
-{
-	/* If no VMA locks, then either mmap lock suffices to stabilise. */
-	mmap_assert_locked(vma->vm_mm);
-}
-
-#endif /* CONFIG_PER_VMA_LOCK */
-
 static inline void vma_assert_can_modify(struct vm_area_struct *vma)
 {
 	if (vma_is_attached(vma))
diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h
index ba84f02c2a101e..032c018650e0d1 100644
--- a/include/linux/mmc/host.h
+++ b/include/linux/mmc/host.h
@@ -23,6 +23,7 @@ struct mmc_ios {
 	unsigned int	clock;			/* clock rate */
 	unsigned short	vdd;
 	unsigned int	power_delay_ms;		/* waiting for stable power */
+	unsigned int	power_off_delay_us;	/* waiting for power discharge */
 
 /* vdd stores the bit number of the selected voltage range from below. */
 
@@ -463,6 +464,7 @@ struct mmc_host {
 #define MMC_CAP2_CRYPTO		0
 #endif
 #define MMC_CAP2_ALT_GPT_TEGRA	(1 << 28)	/* Host with eMMC that has GPT entry at a non-standard location */
+#define MMC_CAP2_CRYPTO_NO_REPROG (1 << 29)	/* Host handles inline crypto key reprogramming */
 
 	bool			uhs2_sd_tran;	/* UHS-II flag for SD_TRAN state */
 	bool			uhs2_app_cmd;	/* UHS-II flag for APP command */
@@ -583,11 +585,9 @@ struct mmc_host {
 
 struct device_node;
 
-struct mmc_host *mmc_alloc_host(int extra, struct device *);
 struct mmc_host *devm_mmc_alloc_host(struct device *dev, int extra);
 int mmc_add_host(struct mmc_host *);
 void mmc_remove_host(struct mmc_host *);
-void mmc_free_host(struct mmc_host *);
 void mmc_of_parse_clk_phase(struct device *dev,
 			    struct mmc_clk_phase_map *map);
 int mmc_of_parse(struct mmc_host *host);
@@ -753,6 +753,8 @@ static inline int mmc_card_uhs2_hd_mode(struct mmc_host *host)
 int mmc_sd_switch(struct mmc_card *card, bool mode, int group,
 		u8 value, u8 *resp);
 int mmc_send_status(struct mmc_card *card, u32 *status);
+int mmc_send_tuning_timeout(struct mmc_host *host, u32 opcode, int *cmd_error,
+			    u32 timeout_ms);
 int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error);
 int mmc_send_abort_tuning(struct mmc_host *host, u32 opcode);
 int mmc_get_ext_csd(struct mmc_card *card, u8 **new_ext_csd);
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 94f9c3ff541604..ecd1ebd49756cc 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -96,25 +96,6 @@
 
 #define MAX_FOLIO_NR_PAGES	(1UL << MAX_FOLIO_ORDER)
 
-/*
- * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to
- * be naturally aligned with regard to the folio size.
- *
- * HVO which is only active if the size of struct page is a power of 2.
- */
-#define MAX_FOLIO_VMEMMAP_ALIGN \
-	(IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && \
-	 is_power_of_2(sizeof(struct page)) ? \
-	 MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
-
-/*
- * vmemmap optimization (like HVO) is only possible for page orders that fill
- * two or more pages with struct pages.
- */
-#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page)))
-#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1)
-#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0)
-
 enum migratetype {
 	MIGRATE_UNMOVABLE,
 	MIGRATE_MOVABLE,
@@ -492,11 +473,14 @@ enum lruvec_flags {
  * folio->flags, masked by LRU_REFS_MASK.
  */
 #define MAX_NR_TIERS		4U
+#define LRU_TIER_MIN		0U
+#define LRU_TIER_MAX		(MAX_NR_TIERS - 1)
 
 #ifndef __GENERATING_BOUNDS_H
 
 #define LRU_GEN_MASK		((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
 #define LRU_REFS_MASK		((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
+#define LRU_REFS_MAX		BIT(LRU_REFS_WIDTH)
 
 /*
  * For folios accessed multiple times through file descriptors,
@@ -635,35 +619,32 @@ struct lru_gen_mm_walk {
  * For each node, memcgs are divided into two generations: the old and the
  * young. For each generation, memcgs are randomly sharded into multiple bins
  * to improve scalability. For each bin, the hlist_nulls is virtually divided
- * into three segments: the head, the tail and the default.
+ * into two segments: the tail and the default.
  *
  * An onlining memcg is added to the tail of a random bin in the old generation.
  * The eviction starts at the head of a random bin in the old generation. The
  * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes
  * the old generation, is incremented when all its bins become empty.
  *
- * There are four operations:
- * 1. MEMCG_LRU_HEAD, which moves a memcg to the head of a random bin in its
- *    current generation (old or young) and updates its "seg" to "head";
- * 2. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its
+ * There are three operations:
+ * 1. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its
  *    current generation (old or young) and updates its "seg" to "tail";
- * 3. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old
+ * 2. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old
  *    generation, updates its "gen" to "old" and resets its "seg" to "default";
- * 4. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the
+ * 3. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the
  *    young generation, updates its "gen" to "young" and resets its "seg" to
  *    "default".
  *
  * The events that trigger the above operations are:
- * 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD;
- * 2. The first attempt to reclaim a memcg below low, which triggers
+ * 1. The first attempt to reclaim a memcg below low, which triggers
  *    MEMCG_LRU_TAIL;
- * 3. The first attempt to reclaim a memcg offlined or below reclaimable size
+ * 2. The first attempt to reclaim a memcg offlined or below reclaimable size
  *    threshold, which triggers MEMCG_LRU_TAIL;
- * 4. The second attempt to reclaim a memcg offlined or below reclaimable size
+ * 3. The second attempt to reclaim a memcg offlined or below reclaimable size
  *    threshold, which triggers MEMCG_LRU_YOUNG;
- * 5. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG;
- * 6. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG;
- * 7. Offlining a memcg, which triggers MEMCG_LRU_OLD.
+ * 4. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG;
+ * 5. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG;
+ * 6. Offlining a memcg, which triggers MEMCG_LRU_OLD.
  *
  * Notes:
  * 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing
@@ -696,7 +677,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg);
 void lru_gen_online_memcg(struct mem_cgroup *memcg);
 void lru_gen_offline_memcg(struct mem_cgroup *memcg);
 void lru_gen_release_memcg(struct mem_cgroup *memcg);
-void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid);
 void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid);
 bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid);
 void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid);
@@ -737,10 +717,6 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg)
 {
 }
 
-static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)
-{
-}
-
 static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid)
 {
 }
@@ -1043,6 +1019,42 @@ struct zone {
 	 * cma pages is present pages that are assigned for CMA use
 	 * (MIGRATE_CMA).
 	 *
+	 * pages_with_online_memmap tracks pages within the zone that have
+	 * an online memory map: present pages and memory holes whose
+	 * memory map has been initialized and pfn_to_online_page()
+	 * succeeds. When spanned_pages == pages_with_online_memmap,
+	 * pfn_to_page() can be performed without further checks on any
+	 * PFN within the zone span.
+	 *
+	 * Note: this counter may temporarily undercount when pages with an
+	 * online memory map exist outside the current zone span. Such pages
+	 * are only created during boot, when initializing the memory map of
+	 * pages that do not fall into any zone span. The undercount itself
+	 * can only happen after boot, during memory hotplug, when growing
+	 * the zone to cover such pages and later shrinking it back, which
+	 * may result in a "too small" value. This is safe: it merely
+	 * prevents detecting a contiguous zone.
+	 *
+	 * Here is an example (page numbers are just for illustration
+	 * purposes):
+	 *   after boot:
+	 *       [  zone span  ]
+	 *       [   zone pages   ]
+	 *       spanned=10, initialized=15, online=10
+	 *       online == spanned  ->  contiguous
+	 *
+	 *   growing after hotplug (hotplug 5):
+	 *       [  zone span                          ]
+	 *       [   zone pages   ]   [   zone pages   ]
+	 *       spanned=30, initialized=20, online=15
+	 *       online != spanned  ->  not contiguous
+	 *
+	 *   shrinking after hotunplug (hotunplug 5 again):
+	 *       [  zone span     ]
+	 *       [   zone pages   ]
+	 *       spanned=15, initialized=15, online=10
+	 *       online != spanned  ->  not contiguous although contiguous
+	 *
 	 * So present_pages may be used by memory hotplug or memory power
 	 * management logic to figure out unmanaged pages by checking
 	 * (present_pages - managed_pages). And managed_pages should be used
@@ -1067,6 +1079,7 @@ struct zone {
 	atomic_long_t		managed_pages;
 	unsigned long		spanned_pages;
 	unsigned long		present_pages;
+	unsigned long		pages_with_online_memmap;
 #if defined(CONFIG_MEMORY_HOTPLUG)
 	unsigned long		present_early_pages;
 #endif
@@ -1101,9 +1114,6 @@ struct zone {
 #ifdef CONFIG_UNACCEPTED_MEMORY
 	/* Pages to be accepted. All pages on the list are MAX_PAGE_ORDER */
 	struct list_head	unaccepted_pages;
-
-	/* To be called once the last page in the zone is accepted */
-	struct work_struct	unaccepted_cleanup;
 #endif
 
 	/* zone flags, see below */
@@ -1157,8 +1167,8 @@ struct zone {
 	/* Zone statistics */
 	atomic_long_t		vm_stat[NR_VM_ZONE_STAT_ITEMS];
 	atomic_long_t		vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
-	struct page *vmemmap_tails[NR_VMEMMAP_TAILS];
+#ifdef CONFIG_VMEMMAP_OPTIMIZATION
+	struct page **vmemmap_tails;
 #endif
 } ____cacheline_internodealigned_in_smp;
 
@@ -1662,7 +1672,7 @@ extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn
 
 extern void lruvec_init(struct lruvec *lruvec);
 
-static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
+static inline struct pglist_data *lruvec_pgdat(const struct lruvec *lruvec)
 {
 #ifdef CONFIG_MEMCG
 	return lruvec->pgdat;
@@ -1694,6 +1704,38 @@ static inline bool zone_is_zone_device(const struct zone *zone)
 }
 #endif
 
+/**
+ * zone_is_contiguous - test whether a zone is contiguous
+ * @zone: the zone to test.
+ *
+ * In a contiguous zone, it is valid to call pfn_to_page() on any PFN in the
+ * spanned zone without requiring pfn_valid() or pfn_to_online_page() checks.
+ *
+ * Note that missing synchronization with memory offlining makes any PFN
+ * traversal prone to races.
+ *
+ * ZONE_DEVICE zones are always marked non-contiguous.
+ *
+ * Return: true if contiguous, otherwise false.
+ */
+static inline bool zone_is_contiguous(const struct zone *zone)
+{
+	return READ_ONCE(zone->contiguous);
+}
+
+static inline void set_zone_contiguous(struct zone *zone)
+{
+	if (zone_is_zone_device(zone))
+		return;
+	if (zone->spanned_pages == zone->pages_with_online_memmap)
+		WRITE_ONCE(zone->contiguous, true);
+}
+
+static inline void clear_zone_contiguous(struct zone *zone)
+{
+	WRITE_ONCE(zone->contiguous, false);
+}
+
 /*
  * Returns true if a zone has pages managed by the buddy allocator.
  * All the reclaim decisions have to use this function rather than
@@ -2021,19 +2063,23 @@ struct mem_section {
 	unsigned long section_mem_map;
 
 	struct mem_section_usage *usage;
+#ifdef CONFIG_VMEMMAP_OPTIMIZATION
+	/*
+	 * Normally, sections hold regular (order-0) pages. However, for
+	 * sections with HVO enabled, this tracks the compound page order
+	 * to enable deduplication of redundant vmemmap pages.
+	 */
+	unsigned int compound_page_order;
+#endif
 #ifdef CONFIG_PAGE_EXTENSION
 	/*
 	 * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use
 	 * section. (see page_ext.h about this.)
 	 */
 	struct page_ext *page_ext;
-	unsigned long pad;
 #endif
-	/*
-	 * WARNING: mem_section must be a power-of-2 in size for the
-	 * calculation and use of SECTION_ROOT_MASK to make sense.
-	 */
-};
+/* Sacrifice minor padding space for efficient lookup. */
+} __aligned(2 * sizeof(unsigned long));
 
 #ifdef CONFIG_SPARSEMEM_EXTREME
 #define SECTIONS_PER_ROOT       (PAGE_SIZE / sizeof (struct mem_section))
@@ -2043,7 +2089,6 @@ struct mem_section {
 
 #define SECTION_NR_TO_ROOT(sec)	((sec) / SECTIONS_PER_ROOT)
 #define NR_SECTION_ROOTS	DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
-#define SECTION_ROOT_MASK	(SECTIONS_PER_ROOT - 1)
 
 #ifdef CONFIG_SPARSEMEM_EXTREME
 extern struct mem_section **mem_section;
@@ -2067,7 +2112,7 @@ static inline struct mem_section *__nr_to_section(unsigned long nr)
 	if (!mem_section || !mem_section[root])
 		return NULL;
 #endif
-	return &mem_section[root][nr & SECTION_ROOT_MASK];
+	return &mem_section[root][nr % SECTIONS_PER_ROOT];
 }
 
 /*
@@ -2083,29 +2128,21 @@ static inline struct mem_section *__nr_to_section(unsigned long nr)
  * accommodate SECTION_MAP_LAST_BIT. We use BUILD_BUG_ON() to ensure this.
  */
 enum {
-	SECTION_MARKED_PRESENT_BIT,
 	SECTION_HAS_MEM_MAP_BIT,
 	SECTION_IS_ONLINE_BIT,
 	SECTION_IS_EARLY_BIT,
 #ifdef CONFIG_ZONE_DEVICE
 	SECTION_TAINT_ZONE_DEVICE_BIT,
-#endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-	SECTION_IS_VMEMMAP_PREINIT_BIT,
 #endif
 	SECTION_MAP_LAST_BIT,
 };
 
-#define SECTION_MARKED_PRESENT		BIT(SECTION_MARKED_PRESENT_BIT)
 #define SECTION_HAS_MEM_MAP		BIT(SECTION_HAS_MEM_MAP_BIT)
 #define SECTION_IS_ONLINE		BIT(SECTION_IS_ONLINE_BIT)
 #define SECTION_IS_EARLY		BIT(SECTION_IS_EARLY_BIT)
 #ifdef CONFIG_ZONE_DEVICE
 #define SECTION_TAINT_ZONE_DEVICE	BIT(SECTION_TAINT_ZONE_DEVICE_BIT)
 #endif
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-#define SECTION_IS_VMEMMAP_PREINIT	BIT(SECTION_IS_VMEMMAP_PREINIT_BIT)
-#endif
 #define SECTION_MAP_MASK		(~(BIT(SECTION_MAP_LAST_BIT) - 1))
 #define SECTION_NID_SHIFT		SECTION_MAP_LAST_BIT
 
@@ -2116,16 +2153,6 @@ static inline struct page *__section_mem_map_addr(struct mem_section *section)
 	return (struct page *)map;
 }
 
-static inline int present_section(const struct mem_section *section)
-{
-	return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
-}
-
-static inline int present_section_nr(unsigned long nr)
-{
-	return present_section(__nr_to_section(nr));
-}
-
 static inline int valid_section(const struct mem_section *section)
 {
 	return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
@@ -2141,6 +2168,11 @@ static inline int valid_section_nr(unsigned long nr)
 	return valid_section(__nr_to_section(nr));
 }
 
+static inline int early_section_nr(unsigned long nr)
+{
+	return early_section(__nr_to_section(nr));
+}
+
 static inline int online_section(const struct mem_section *section)
 {
 	return (section && (section->section_mem_map & SECTION_IS_ONLINE));
@@ -2153,28 +2185,20 @@ static inline int online_device_section(const struct mem_section *section)
 
 	return section && ((section->section_mem_map & flags) == flags);
 }
-#else
-static inline int online_device_section(const struct mem_section *section)
-{
-	return 0;
-}
-#endif
 
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-static inline int preinited_vmemmap_section(const struct mem_section *section)
+static inline struct zone *device_zone(int nid)
 {
-	return (section &&
-		(section->section_mem_map & SECTION_IS_VMEMMAP_PREINIT));
+	return &NODE_DATA(nid)->node_zones[ZONE_DEVICE];
 }
-
-void sparse_vmemmap_init_nid_early(int nid);
 #else
-static inline int preinited_vmemmap_section(const struct mem_section *section)
+static inline int online_device_section(const struct mem_section *section)
 {
 	return 0;
 }
-static inline void sparse_vmemmap_init_nid_early(int nid)
+
+static inline struct zone *device_zone(int nid)
 {
+	return NULL;
 }
 #endif
 
@@ -2193,7 +2217,7 @@ static inline struct mem_section *__pfn_to_section(unsigned long pfn)
 	return __nr_to_section(pfn_to_section_nr(pfn));
 }
 
-extern unsigned long __highest_present_section_nr;
+extern unsigned long __highest_used_section_nr;
 
 static inline int subsection_map_index(unsigned long pfn)
 {
@@ -2241,9 +2265,6 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long
 }
 #endif
 
-void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
-			       unsigned long flags);
-
 #ifndef CONFIG_HAVE_ARCH_PFN_VALID
 /**
  * pfn_valid - check if there is a valid memory map entry for a PFN
@@ -2295,7 +2316,7 @@ static inline unsigned long first_valid_pfn(unsigned long pfn, unsigned long end
 
 	rcu_read_lock_sched();
 
-	while (nr <= __highest_present_section_nr && pfn < end_pfn) {
+	while (nr <= __highest_used_section_nr && pfn < end_pfn) {
 		struct mem_section *ms = __pfn_to_section(pfn);
 
 		if (valid_section(ms) &&
@@ -2341,27 +2362,20 @@ static inline unsigned long next_valid_pfn(unsigned long pfn, unsigned long end_
 
 #endif
 
-static inline int pfn_in_present_section(unsigned long pfn)
-{
-	if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
-		return 0;
-	return present_section(__pfn_to_section(pfn));
-}
-
-static inline unsigned long next_present_section_nr(unsigned long section_nr)
+static inline unsigned long next_early_section_nr(unsigned long section_nr)
 {
-	while (++section_nr <= __highest_present_section_nr) {
-		if (present_section_nr(section_nr))
+	while (++section_nr <= __highest_used_section_nr) {
+		if (early_section_nr(section_nr))
 			return section_nr;
 	}
 
 	return -1;
 }
 
-#define for_each_present_section_nr(start, section_nr)		\
-	for (section_nr = next_present_section_nr(start - 1);	\
+#define for_each_early_section_nr(start, section_nr)		\
+	for (section_nr = next_early_section_nr(start - 1);	\
 	     section_nr != -1;					\
-	     section_nr = next_present_section_nr(section_nr))
+	     section_nr = next_early_section_nr(section_nr))
 
 /*
  * These are _only_ used during initialisation, therefore they
@@ -2377,10 +2391,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr)
 #else
 #define pfn_to_nid(pfn)		(0)
 #endif
-
-#else
-#define sparse_vmemmap_init_nid_early(_nid) do {} while (0)
-#define pfn_in_present_section pfn_valid
 #endif /* CONFIG_SPARSEMEM */
 
 /*
diff --git a/include/linux/mnt_idmapping.h b/include/linux/mnt_idmapping.h
index e71a6070a8f8c3..78eeef4c29965a 100644
--- a/include/linux/mnt_idmapping.h
+++ b/include/linux/mnt_idmapping.h
@@ -8,8 +8,8 @@
 struct mnt_idmap;
 struct user_namespace;
 
-extern struct mnt_idmap nop_mnt_idmap;
-extern struct mnt_idmap invalid_mnt_idmap;
+extern const struct mnt_idmap nop_mnt_idmap;
+extern const struct mnt_idmap invalid_mnt_idmap;
 extern struct user_namespace init_user_ns;
 
 typedef struct {
@@ -121,19 +121,19 @@ static inline bool vfsgid_eq_kgid(vfsgid_t vfsgid, kgid_t kgid)
 
 int vfsgid_in_group_p(vfsgid_t vfsgid);
 
-struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap);
-void mnt_idmap_put(struct mnt_idmap *idmap);
+const struct mnt_idmap *mnt_idmap_get(const struct mnt_idmap *idmap);
+void mnt_idmap_put(const struct mnt_idmap *idmap);
 
-vfsuid_t make_vfsuid(struct mnt_idmap *idmap,
+vfsuid_t make_vfsuid(const struct mnt_idmap *idmap,
 		     struct user_namespace *fs_userns, kuid_t kuid);
 
-vfsgid_t make_vfsgid(struct mnt_idmap *idmap,
+vfsgid_t make_vfsgid(const struct mnt_idmap *idmap,
 		     struct user_namespace *fs_userns, kgid_t kgid);
 
-kuid_t from_vfsuid(struct mnt_idmap *idmap,
+kuid_t from_vfsuid(const struct mnt_idmap *idmap,
 		   struct user_namespace *fs_userns, vfsuid_t vfsuid);
 
-kgid_t from_vfsgid(struct mnt_idmap *idmap,
+kgid_t from_vfsgid(const struct mnt_idmap *idmap,
 		   struct user_namespace *fs_userns, vfsgid_t vfsgid);
 
 /**
@@ -148,7 +148,7 @@ kgid_t from_vfsgid(struct mnt_idmap *idmap,
  *
  * Return: true if @vfsuid has a mapping in the filesystem, false if not.
  */
-static inline bool vfsuid_has_fsmapping(struct mnt_idmap *idmap,
+static inline bool vfsuid_has_fsmapping(const struct mnt_idmap *idmap,
 					struct user_namespace *fs_userns,
 					vfsuid_t vfsuid)
 {
@@ -186,7 +186,7 @@ static inline kuid_t vfsuid_into_kuid(vfsuid_t vfsuid)
  *
  * Return: true if @vfsgid has a mapping in the filesystem, false if not.
  */
-static inline bool vfsgid_has_fsmapping(struct mnt_idmap *idmap,
+static inline bool vfsgid_has_fsmapping(const struct mnt_idmap *idmap,
 					struct user_namespace *fs_userns,
 					vfsgid_t vfsgid)
 {
@@ -225,7 +225,7 @@ static inline kgid_t vfsgid_into_kgid(vfsgid_t vfsgid)
  *
  * Return: the caller's current fsuid mapped up according to @idmap.
  */
-static inline kuid_t mapped_fsuid(struct mnt_idmap *idmap,
+static inline kuid_t mapped_fsuid(const struct mnt_idmap *idmap,
 				  struct user_namespace *fs_userns)
 {
 	return from_vfsuid(idmap, fs_userns, VFSUIDT_INIT(current_fsuid()));
@@ -244,7 +244,7 @@ static inline kuid_t mapped_fsuid(struct mnt_idmap *idmap,
  *
  * Return: the caller's current fsgid mapped up according to @idmap.
  */
-static inline kgid_t mapped_fsgid(struct mnt_idmap *idmap,
+static inline kgid_t mapped_fsgid(const struct mnt_idmap *idmap,
 				  struct user_namespace *fs_userns)
 {
 	return from_vfsgid(idmap, fs_userns, VFSGIDT_INIT(current_fsgid()));
diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
index a397213bedace6..d1581bb647298b 100644
--- a/include/linux/mod_devicetable.h
+++ b/include/linux/mod_devicetable.h
@@ -15,7 +15,7 @@
 #include "device-id/acpi.h"
 #include "device-id/amba.h"
 #include "device-id/ap.h"
-#include "device-id/apr.h"
+#include "device-id/arm_smccc.h"
 #include "device-id/auxiliary.h"
 #include "device-id/bcma.h"
 #include "device-id/ccw.h"
diff --git a/include/linux/module.h b/include/linux/module.h
index 96cc98568eea54..f6f90cd88f151b 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -29,6 +29,7 @@
 #include <linux/srcu.h>
 #include <linux/static_call_types.h>
 #include <linux/dynamic_debug.h>
+#include <linux/panic.h>
 
 #include <linux/percpu.h>
 #include <asm/module.h>
@@ -770,6 +771,9 @@ static inline bool is_livepatch_module(struct module *mod)
 
 void module_for_each_mod(int(*func)(struct module *mod, void *data), void *data);
 
+void add_taint_module(struct module *mod, unsigned flag,
+		      enum lockdep_ok lockdep_ok);
+
 #else /* !CONFIG_MODULES... */
 
 static inline struct module *__module_address(unsigned long addr)
@@ -877,6 +881,12 @@ static inline bool module_is_coming(struct module *mod)
 static inline void module_for_each_mod(int(*func)(struct module *mod, void *data), void *data)
 {
 }
+
+static inline void add_taint_module(struct module *mod, unsigned flag,
+				    enum lockdep_ok lockdep_ok)
+{
+	add_taint(flag, lockdep_ok);
+}
 #endif /* CONFIG_MODULES */
 
 #ifdef CONFIG_SYSFS
diff --git a/include/linux/module_symbol.h b/include/linux/module_symbol.h
index 574609aced991f..698d3db2b37e1b 100644
--- a/include/linux/module_symbol.h
+++ b/include/linux/module_symbol.h
@@ -7,8 +7,11 @@ enum ksym_flags {
 	KSYM_FLAG_GPL_ONLY	= 1 << 0,
 };
 
-/* This ignores the intensely annoying "mapping symbols" found in ELF files. */
-static inline bool is_mapping_symbol(const char *str)
+/*
+ * Ignore local labels (.L*, L0*) and mapping symbols ($*). These symbols are
+ * not useful for the kernel, for example, they should not appear in kallsyms.
+ */
+static inline bool is_ignored_kernel_symbol(const char *str)
 {
 	if (str[0] == '.' && str[1] == 'L')
 		return true;
diff --git a/include/linux/mount.h b/include/linux/mount.h
index acfe7ef86a1b3f..e90ccafef281b8 100644
--- a/include/linux/mount.h
+++ b/include/linux/mount.h
@@ -59,10 +59,10 @@ struct vfsmount {
 	struct dentry *mnt_root;	/* root of the mounted tree */
 	struct super_block *mnt_sb;	/* pointer to superblock */
 	int mnt_flags;
-	struct mnt_idmap *mnt_idmap;
+	const struct mnt_idmap *mnt_idmap;
 } __randomize_layout;
 
-static inline struct mnt_idmap *mnt_idmap(const struct vfsmount *mnt)
+static inline const struct mnt_idmap *mnt_idmap(const struct vfsmount *mnt)
 {
 	/* Pairs with smp_store_release() in do_idmap_mount(). */
 	return READ_ONCE(mnt->mnt_idmap);
diff --git a/include/linux/mtd/bbm.h b/include/linux/mtd/bbm.h
index d890805f549444..67929fdf2dff5d 100644
--- a/include/linux/mtd/bbm.h
+++ b/include/linux/mtd/bbm.h
@@ -75,7 +75,7 @@ struct nand_bbt_descr {
  * with NAND_BBT_CREATE.
  */
 #define NAND_BBT_CREATE_EMPTY	0x00000400
-/* Write bbt if neccecary */
+/* Write bbt if necessary */
 #define NAND_BBT_WRITE		0x00002000
 /* Read and write back block contents when writing bbt */
 #define NAND_BBT_SAVECONTENT	0x00004000
diff --git a/include/linux/mtd/lpc32xx_mlc.h b/include/linux/mtd/lpc32xx_mlc.h
deleted file mode 100644
index 35e971be0950b6..00000000000000
--- a/include/linux/mtd/lpc32xx_mlc.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Platform data for LPC32xx SoC MLC NAND controller
- *
- * Copyright © 2012 Roland Stigge
- */
-
-#ifndef __LINUX_MTD_LPC32XX_MLC_H
-#define __LINUX_MTD_LPC32XX_MLC_H
-
-#include <linux/dmaengine.h>
-
-struct lpc32xx_mlc_platform_data {
-	dma_filter_fn dma_filter;
-};
-
-#endif  /* __LINUX_MTD_LPC32XX_MLC_H */
diff --git a/include/linux/mtd/lpc32xx_slc.h b/include/linux/mtd/lpc32xx_slc.h
deleted file mode 100644
index a044b806566b43..00000000000000
--- a/include/linux/mtd/lpc32xx_slc.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Platform data for LPC32xx SoC SLC NAND controller
- *
- * Copyright © 2012 Roland Stigge
- */
-
-#ifndef __LINUX_MTD_LPC32XX_SLC_H
-#define __LINUX_MTD_LPC32XX_SLC_H
-
-#include <linux/dmaengine.h>
-
-struct lpc32xx_slc_platform_data {
-	dma_filter_fn dma_filter;
-};
-
-#endif  /* __LINUX_MTD_LPC32XX_SLC_H */
diff --git a/include/linux/mtd/nand-qpic-common.h b/include/linux/mtd/nand-qpic-common.h
index 437448995187a6..0f40964eefe7a7 100644
--- a/include/linux/mtd/nand-qpic-common.h
+++ b/include/linux/mtd/nand-qpic-common.h
@@ -262,7 +262,6 @@ struct bam_transaction {
 		u32 tx_sgl_start;
 		u32 rx_sgl_pos;
 		u32 rx_sgl_start;
-
 	);
 };
 
diff --git a/include/linux/mtd/nand.h b/include/linux/mtd/nand.h
index 09c8c93e4dbabd..6936180b6ea584 100644
--- a/include/linux/mtd/nand.h
+++ b/include/linux/mtd/nand.h
@@ -305,6 +305,8 @@ int nand_ecc_prepare_io_req(struct nand_device *nand,
 			    struct nand_page_io_req *req);
 int nand_ecc_finish_io_req(struct nand_device *nand,
 			   struct nand_page_io_req *req);
+bool nand_ecc_is_pipelined(const struct nand_device *nand);
+
 bool nand_ecc_is_strong_enough(struct nand_device *nand);
 
 #if IS_REACHABLE(CONFIG_MTD_NAND_CORE)
diff --git a/include/linux/mtd/rawnand.h b/include/linux/mtd/rawnand.h
index 5c70e7bd3ed5ad..43fefff020251c 100644
--- a/include/linux/mtd/rawnand.h
+++ b/include/linux/mtd/rawnand.h
@@ -1592,7 +1592,7 @@ void nand_wait_ready(struct nand_chip *chip);
 
 /*
  * Free resources held by the NAND device, must be called on error after a
- * sucessful nand_scan().
+ * successful nand_scan().
  */
 void nand_cleanup(struct nand_chip *chip);
 
diff --git a/include/linux/mtd/sh_flctl.h b/include/linux/mtd/sh_flctl.h
index 78fc2d4218c808..15adf2474fa01e 100644
--- a/include/linux/mtd/sh_flctl.h
+++ b/include/linux/mtd/sh_flctl.h
@@ -60,8 +60,8 @@
  * Some hardware uses bits called PULSEx instead of FCKSEL_E and QTSEL_E
  * to control the clock divider used between the High-Speed Peripheral Clock
  * and the FLCTL internal clock. If so, use CLK_8_BIT_xxx for connecting 8 bit
- * and CLK_16_BIT_xxx for connecting 16 bit bus bandwith NAND chips. For the 16
- * bit version the divider is seperate for the pulse width of high and low
+ * and CLK_16_BIT_xxx for connecting 16 bit bus bandwidth NAND chips. For the 16
+ * bit version the divider is separate for the pulse width of high and low
  * signals.
  */
 #define PULSE3	(0x1 << 27)
diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h
index 4b92494827b151..b3e3c6b1018661 100644
--- a/include/linux/mtd/spi-nor.h
+++ b/include/linux/mtd/spi-nor.h
@@ -25,6 +25,7 @@
 #define SPINOR_OP_WRSR		0x01	/* Write status register 1 */
 #define SPINOR_OP_RDSR2		0x3f	/* Read status register 2 */
 #define SPINOR_OP_WRSR2		0x3e	/* Write status register 2 */
+#define SPINOR_OP_WRSR2_ALT	0x31	/* Write status register 2 (alternative) */
 #define SPINOR_OP_READ		0x03	/* Read data bytes (low frequency) */
 #define SPINOR_OP_READ_FAST	0x0b	/* Read data bytes (high frequency) */
 #define SPINOR_OP_READ_1_1_2	0x3b	/* Read data bytes (Dual Output SPI) */
@@ -113,20 +114,16 @@
 #define SR_E_ERR		BIT(5)
 #define SR_P_ERR		BIT(6)
 
-#define SR1_QUAD_EN_BIT6	BIT(6)
-
 #define SR_BP_SHIFT		2
 
 /* Enhanced Volatile Configuration Register bits */
 #define EVCR_QUAD_EN_MICRON	BIT(7)	/* Micron Quad I/O */
 
 /* Status Register 2 bits. */
-#define SR2_QUAD_EN_BIT1	BIT(1)
 #define SR2_LB1			BIT(3)	/* Security Register Lock Bit 1 */
 #define SR2_LB2			BIT(4)	/* Security Register Lock Bit 2 */
 #define SR2_LB3			BIT(5)	/* Security Register Lock Bit 3 */
 #define SR2_CMP_BIT6		BIT(6)
-#define SR2_QUAD_EN_BIT7	BIT(7)
 
 /* Supported SPI protocols */
 #define SNOR_PROTO_INST_MASK	GENMASK(23, 16)
@@ -365,8 +362,6 @@ struct spi_nor_flash_parameter;
  * @read_dummy:		the dummy needed by the read operation
  * @program_opcode:	the program opcode
  * @sst_write_second:	used by the SST write operation
- * @flags:		flag options for the current SPI NOR (SNOR_F_*)
- * @cmd_ext_type:	the command opcode extension type for DTR mode.
  * @read_proto:		the SPI protocol for read operations
  * @write_proto:	the SPI protocol for write operations
  * @reg_proto:		the SPI protocol for read_reg/write_reg/erase operations
@@ -398,6 +393,7 @@ struct spi_nor {
 	u8			*id;
 	const struct flash_info	*info;
 	const struct spi_nor_manufacturer *manufacturer;
+	const char		*partname;
 	u8			addr_nbytes;
 	u8			erase_opcode;
 	u8			read_opcode;
@@ -407,8 +403,6 @@ struct spi_nor {
 	enum spi_nor_protocol	write_proto;
 	enum spi_nor_protocol	reg_proto;
 	bool			sst_write_second;
-	u32			flags;
-	enum spi_nor_cmd_ext	cmd_ext_type;
 	struct sfdp		*sfdp;
 	struct dentry		*debugfs_root;
 	u8			dfs_sr_cache[2];
diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h
index 5f4c00ae72a724..d7c99fcfb85eff 100644
--- a/include/linux/mtd/spinand.h
+++ b/include/linux/mtd/spinand.h
@@ -399,7 +399,7 @@ struct spinand_devid {
 };
 
 /**
- * struct manufacurer_ops - SPI NAND manufacturer specific operations
+ * struct spinand_manufacturer_ops - SPI NAND manufacturer specific operations
  * @init: initialize a SPI NAND device
  * @cleanup: cleanup a SPI NAND device
  *
@@ -438,6 +438,7 @@ extern const struct spinand_manufacturer fmsh_spinand_manufacturer;
 extern const struct spinand_manufacturer foresee_spinand_manufacturer;
 extern const struct spinand_manufacturer gigadevice_spinand_manufacturer;
 extern const struct spinand_manufacturer heyangtek_spinand_manufacturer;
+extern const struct spinand_manufacturer issi_spinand_manufacturer;
 extern const struct spinand_manufacturer macronix_spinand_manufacturer;
 extern const struct spinand_manufacturer micron_spinand_manufacturer;
 extern const struct spinand_manufacturer paragon_spinand_manufacturer;
diff --git a/include/linux/namei.h b/include/linux/namei.h
index 86d657b24fc6a1..c4436e5c2ba654 100644
--- a/include/linux/namei.h
+++ b/include/linux/namei.h
@@ -32,8 +32,9 @@ enum { MAX_NESTED_LINKS = 8 };
 #define LOOKUP_CREATE		BIT(17)	/* ... in object creation */
 #define LOOKUP_EXCL		BIT(18)	/* ... in target must not exist */
 #define LOOKUP_RENAME_TARGET	BIT(19)	/* ... in destination of rename() */
+#define LOOKUP_SHARED		BIT(20) /* Parent lock is held shared */
 
-/* 4 spare bits for intent */
+/* 3 spare bits for intent */
 
 /* Scoping flags for lookup. */
 #define LOOKUP_NO_SYMLINKS	BIT(24) /* No symlink crossing. */
@@ -70,24 +71,24 @@ extern struct dentry *try_lookup_noperm(struct qstr *, struct dentry *);
 extern struct dentry *lookup_noperm(struct qstr *, struct dentry *);
 extern struct dentry *lookup_noperm_unlocked(struct qstr *, struct dentry *);
 extern struct dentry *lookup_noperm_positive_unlocked(struct qstr *, struct dentry *);
-struct dentry *lookup_one(struct mnt_idmap *, struct qstr *, struct dentry *);
-struct dentry *lookup_one_unlocked(struct mnt_idmap *idmap,
+struct dentry *lookup_one(const struct mnt_idmap *, struct qstr *, struct dentry *);
+struct dentry *lookup_one_unlocked(const struct mnt_idmap *idmap,
 				   struct qstr *name, struct dentry *base);
-struct dentry *lookup_one_positive_unlocked(struct mnt_idmap *idmap,
+struct dentry *lookup_one_positive_unlocked(const struct mnt_idmap *idmap,
 					    struct qstr *name,
 					    struct dentry *base);
-struct dentry *lookup_one_positive_killable(struct mnt_idmap *idmap,
+struct dentry *lookup_one_positive_killable(const struct mnt_idmap *idmap,
 					    struct qstr *name,
 					    struct dentry *base);
 
-struct dentry *start_creating(struct mnt_idmap *idmap, struct dentry *parent,
+struct dentry *start_creating(const struct mnt_idmap *idmap, struct dentry *parent,
 			      struct qstr *name);
-struct dentry *start_removing(struct mnt_idmap *idmap, struct dentry *parent,
+struct dentry *start_removing(const struct mnt_idmap *idmap, struct dentry *parent,
 			      struct qstr *name);
-struct dentry *start_creating_killable(struct mnt_idmap *idmap,
+struct dentry *start_creating_killable(const struct mnt_idmap *idmap,
 				       struct dentry *parent,
 				       struct qstr *name);
-struct dentry *start_removing_killable(struct mnt_idmap *idmap,
+struct dentry *start_removing_killable(const struct mnt_idmap *idmap,
 				       struct dentry *parent,
 				       struct qstr *name);
 struct dentry *start_creating_noperm(struct dentry *parent, struct qstr *name);
diff --git a/include/linux/net.h b/include/linux/net.h
index 3d82966e2243b0..e2a866fbcfa5ea 100644
--- a/include/linux/net.h
+++ b/include/linux/net.h
@@ -47,6 +47,9 @@ typedef struct sockopt {
 	int optlen;
 } sockopt_t;
 
+int sockptr_to_sockopt(sockopt_t *opt, sockptr_t optval, sockptr_t optlen,
+		       struct kvec *kvec);
+
 /*
  * Initialize a user-backed sockopt_t from the (optval, optlen) __user pair of
  * a getsockopt() callback. Used by transitional __user getsockopt wrappers
@@ -70,6 +73,35 @@ static inline int sockopt_init_user(sockopt_t *opt, char __user *optval,
 	return 0;
 }
 
+/*
+ * Grow optval to @size, for the options whose reply is sized by a count the
+ * caller left in optval rather than by optlen. Those write past optlen today
+ * and userspace relies on it.
+ *
+ * Call it before writing through opt->iter_out: it re-anchors the iterator at
+ * the head of optval. Only a user buffer can be longer than the optlen the
+ * caller declared, so a kernel-backed optval is refused with -EINVAL.
+ */
+static inline int sockopt_expand_out(sockopt_t *opt, size_t size)
+{
+	if (size <= (size_t)opt->optlen)
+		return 0;
+
+	if (size > INT_MAX)
+		return -EINVAL;
+
+	/* Re-anchoring reads iter_out.ubuf, so the iterator has to be a user
+	 * buffer that nothing has written through yet.
+	 */
+	if (WARN_ON_ONCE(!iter_is_ubuf(&opt->iter_out) ||
+			 iov_iter_count(&opt->iter_out) != (size_t)opt->optlen))
+		return -EINVAL;
+
+	iov_iter_ubuf(&opt->iter_out, ITER_DEST, opt->iter_out.ubuf, size);
+
+	return 0;
+}
+
 struct poll_table_struct;
 struct pipe_inode_info;
 struct inode;
@@ -166,7 +198,7 @@ struct socket {
 
 	struct file		*file;
 	struct sock		*sk;
-	const struct proto_ops	*ops; /* Might change with IPV6_ADDRFORM or MPTCP. */
+	const struct proto_ops	*ops; /* Might change with MPTCP. */
 
 	struct socket_wq	wq;
 };
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 87cafc932e9e65..d037faff7c44b6 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1135,13 +1135,16 @@ struct netdev_net_notifier {
  *				struct netdev_hw_addr_list *uc,
  *				struct netdev_hw_addr_list *mc);
  *	Async version of ndo_set_rx_mode which runs in process context
- *	with rtnl_lock and netdev_lock_ops(dev) held. The uc/mc parameters
+ *	under the netdev instance lock for "ops locked" drivers, or
+ *	rtnl_lock for all other drivers. The uc/mc parameters
  *	are snapshots of the address lists - iterate with
  *	netdev_hw_addr_list_for_each(ha, uc). Return 0 on success or a
  *	negative errno to request a retry via the core backoff.
  *
  * void (*ndo_work)(struct net_device *dev, unsigned long events);
  *	Run deferred work scheduled with netdev_work_sched(@events).
+ *	Runs in process context under the netdev instance lock for "ops
+ *	locked" drivers, or rtnl_lock for all other drivers.
  *
  * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
  *	This function  is called when the Media Access Control address
@@ -1151,11 +1154,6 @@ struct netdev_net_notifier {
  * int (*ndo_validate_addr)(struct net_device *dev);
  *	Test if Media Access Control address is valid for the device.
  *
- * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
- *	Old-style ioctl entry point. This is used internally by the
- *	ieee802154 subsystem but is no longer called by the device
- *	ioctl handler.
- *
  * int (*ndo_siocbond)(struct net_device *dev, struct ifreq *ifr, int cmd);
  *	Used by the bonding driver for its device specific ioctls:
  *	SIOCBONDENSLAVE, SIOCBONDRELEASE, SIOCBONDSETHWADDR, SIOCBONDCHANGEACTIVE,
@@ -1477,8 +1475,6 @@ struct net_device_ops {
 	int			(*ndo_set_mac_address)(struct net_device *dev,
 						       void *addr);
 	int			(*ndo_validate_addr)(struct net_device *dev);
-	int			(*ndo_do_ioctl)(struct net_device *dev,
-					        struct ifreq *ifr, int cmd);
 	int			(*ndo_eth_ioctl)(struct net_device *dev,
 						 struct ifreq *ifr, int cmd);
 	int			(*ndo_siocbond)(struct net_device *dev,
@@ -1837,6 +1833,7 @@ enum netdev_reg_state {
  *			drivers. Mainly used by logical interfaces, such as
  *			bonding and tunnels
  *	@netmem_tx:	device netmem TX mode
+ *	@pacing_offload: enable EDT pacing offload.
  *
  *	@name:	This is the first field of the "visible" part of this structure
  *		(i.e. as seen by users in the "Space.c" file).  It is the name
@@ -2167,6 +2164,7 @@ struct net_device {
 		unsigned long		priv_flags:32;
 		unsigned long		lltx:1;
 		unsigned long		netmem_tx:2;
+		unsigned long		pacing_offload:1;
 	);
 	const struct net_device_ops *netdev_ops;
 	const struct header_ops *header_ops;
@@ -3669,7 +3667,11 @@ struct page_pool_bh {
 };
 DECLARE_PER_CPU(struct page_pool_bh, system_page_pool);
 
+#ifdef CONFIG_KASAN
+#define XMIT_RECURSION_LIMIT	4
+#else
 #define XMIT_RECURSION_LIMIT	8
+#endif
 
 #ifndef CONFIG_PREEMPT_RT
 static inline int dev_recursion_level(void)
@@ -5321,7 +5323,6 @@ void netdev_lower_state_changed(struct net_device *lower_dev,
 				void *lower_state_info);
 
 #define NETDEV_RSS_KEY_LEN 256
-extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
 void netdev_rss_key_fill(void *buffer, size_t len);
 
 int skb_checksum_help(struct sk_buff *skb);
@@ -5606,12 +5607,12 @@ static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
 
 static inline bool netif_is_l3_master(const struct net_device *dev)
 {
-	return dev->priv_flags & IFF_L3MDEV_MASTER;
+	return IS_ENABLED(CONFIG_NET_VRF) && (dev->priv_flags & IFF_L3MDEV_MASTER);
 }
 
 static inline bool netif_is_l3_slave(const struct net_device *dev)
 {
-	return dev->priv_flags & IFF_L3MDEV_SLAVE;
+	return IS_ENABLED(CONFIG_NET_VRF) && (dev->priv_flags & IFF_L3MDEV_SLAVE);
 }
 
 static inline int dev_sdif(const struct net_device *dev)
diff --git a/include/linux/netfilter_netdev.h b/include/linux/netfilter_netdev.h
index 3175073a66ba6b..2a854a0bbd4ba7 100644
--- a/include/linux/netfilter_netdev.h
+++ b/include/linux/netfilter_netdev.h
@@ -133,7 +133,7 @@ static inline struct sk_buff *nf_hook_egress(struct sk_buff *skb, int *rc,
 
 static inline void nf_skip_egress(struct sk_buff *skb, bool skip)
 {
-#ifdef CONFIG_NETFILTER_SKIP_EGRESS
+#ifdef CONFIG_NET_EGRESS
 	skb->nf_skip_egress = skip;
 #endif
 }
diff --git a/include/linux/netfs.h b/include/linux/netfs.h
index b4dd32863dd45f..67e010b6994b31 100644
--- a/include/linux/netfs.h
+++ b/include/linux/netfs.h
@@ -22,6 +22,7 @@
 
 enum netfs_sreq_ref_trace;
 typedef struct mempool mempool_t;
+struct fscache_occupancy;
 struct folio_queue;
 
 /**
@@ -62,8 +63,8 @@ struct netfs_inode {
 	struct fscache_cookie	*cache;
 #endif
 	struct list_head	wb_queue;	/* Queue of processes wanting to do writeback */
-	loff_t			_remote_i_size;	/* Size of the remote file */
-	loff_t			_zero_point;	/* Size after which we assume there's no data
+	uoff_t			_remote_i_size;	/* Size of the remote file */
+	uoff_t			_zero_point;	/* Size after which we assume there's no data
 						 * on the server */
 	spinlock_t		lock;		/* Lock covering wb_queue */
 	atomic_t		io_count;	/* Number of outstanding reqs */
@@ -125,6 +126,12 @@ static inline struct netfs_group *netfs_folio_group(struct folio *folio)
 	return priv;
 }
 
+enum netfs_cache_collect {
+	NETFS_CACHE_COLLECT_WRITE_GAP,		/* Gap in collection, no state either way */
+	NETFS_CACHE_COLLECT_WRITE_DATA,		/* Currently collecting good writes */
+	NETFS_CACHE_COLLECT_WRITE_CANCEL,	/* Currently collecting cancelled writes */
+};
+
 /*
  * Stream of I/O subrequests going to a particular destination, such as the
  * server or the local cache.  This is mainly intended for writing where we may
@@ -142,7 +149,7 @@ struct netfs_io_stream {
 	void (*issue_write)(struct netfs_io_subrequest *subreq);
 	/* Collection tracking */
 	struct list_head	subrequests;	/* Contributory I/O operations */
-	unsigned long long	collected_to;	/* Position we've collected results to */
+	uoff_t			collected_to;	/* Position we've collected results to */
 	size_t			transferred;	/* The amount transferred from this stream */
 	unsigned short		error;		/* Aggregate error for the stream */
 	enum netfs_io_source	source;		/* Where to read from/write to */
@@ -152,6 +159,7 @@ struct netfs_io_stream {
 	bool			need_retry;	/* T if this stream needs retrying */
 	bool			failed;		/* T if this stream failed */
 	bool			transferred_valid; /* T is ->transferred is valid */
+	enum netfs_cache_collect cache_collect;	/* Current writeback cache collect state */
 };
 
 /*
@@ -161,8 +169,11 @@ struct netfs_cache_resources {
 	const struct netfs_cache_ops	*ops;
 	void				*cache_priv;
 	void				*cache_priv2;
-	unsigned int			debug_id;	/* Cookie debug ID */
+	uoff_t				cache_i_size;	/* Initial size of cache file */
+	unsigned int			cookie_id;	/* Cache cookie debug ID */
+	unsigned int			object_id;	/* Cache object debug ID */
 	unsigned int			inval_counter;	/* object->inval_counter at begin_op */
+	unsigned int			dio_size;	/* DIO block size */
 };
 
 /*
@@ -177,7 +188,7 @@ struct netfs_io_subrequest {
 	struct work_struct	work;
 	struct list_head	rreq_link;	/* Link in rreq->subrequests */
 	struct iov_iter		io_iter;	/* Iterator for this subrequest */
-	unsigned long long	start;		/* Where to start the I/O */
+	uoff_t			start;		/* Where to start the I/O */
 	size_t			len;		/* Size of the I/O */
 	size_t			transferred;	/* Amount of data transferred */
 	refcount_t		ref;
@@ -196,6 +207,7 @@ struct netfs_io_subrequest {
 #define NETFS_SREQ_IN_PROGRESS		8	/* Unlocked when the subrequest completes */
 #define NETFS_SREQ_NEED_RETRY		9	/* Set if the filesystem requests a retry */
 #define NETFS_SREQ_FAILED		10	/* Set if the subreq failed unretryably */
+#define NETFS_SREQ_CANCELLED		11	/* Set if the subreq was cancelled by netfslib */
 };
 
 enum netfs_io_origin {
@@ -208,7 +220,6 @@ enum netfs_io_origin {
 	NETFS_DIO_READ,			/* This is a direct I/O read */
 	NETFS_WRITEBACK,		/* This write was triggered by writepages */
 	NETFS_WRITEBACK_SINGLE,		/* This monolithic write was triggered by writepages */
-	NETFS_WRITETHROUGH,		/* This write was made by netfs_perform_write() */
 	NETFS_UNBUFFERED_WRITE,		/* This is an unbuffered write */
 	NETFS_DIO_WRITE,		/* This is a direct I/O write */
 	NETFS_PGPRIV2_COPY_TO_CACHE,	/* [DEPRECATED] This is writing read data to the cache */
@@ -243,17 +254,18 @@ struct netfs_io_request {
 	void			*netfs_priv;	/* Private data for the netfs */
 	void			*netfs_priv2;	/* Private data for the netfs */
 	struct bio_vec		*direct_bv;	/* DIO buffer list (when handling iovec-iter) */
-	unsigned long long	submitted;	/* Amount submitted for I/O so far */
-	unsigned long long	len;		/* Length of the request */
+	uoff_t			submitted;	/* Amount submitted for I/O so far */
+	uoff_t			len;		/* Length of the request */
 	size_t			transferred;	/* Amount to be indicated as transferred */
 	size_t			progress_at;	/* Report read progress when hit this much read */
 	long			error;		/* 0 or error that occurred */
-	unsigned long long	i_size;		/* Size of the file */
-	unsigned long long	start;		/* Start position */
+	uoff_t			i_size;		/* Size of the file */
+	uoff_t			start;		/* Start position */
 	atomic64_t		issued_to;	/* Write issuer folio cursor */
-	unsigned long long	collected_to;	/* Point we've collected to */
-	unsigned long long	cleaned_to;	/* Position we've cleaned folios to */
-	unsigned long long	abandon_to;	/* Position to abandon folios to */
+	uoff_t			collected_to;	/* Point we've collected to */
+	uoff_t			cache_coll_to;	/* Point the cache has collected to */
+	uoff_t			cleaned_to;	/* Position we've cleaned folios to */
+	uoff_t			abandon_to;	/* Position to abandon folios to */
 	const struct folio	*no_unlock_folio; /* Don't unlock this folio after read */
 	gfp_t			gfp;		/* GFP flags to use */
 	unsigned int		direct_bv_count; /* Number of elements in direct_bv[] */
@@ -273,14 +285,18 @@ struct netfs_io_request {
 #define NETFS_RREQ_FAILED		3	/* The request failed */
 #define NETFS_RREQ_RETRYING		4	/* Set if we're in the retry path */
 #define NETFS_RREQ_SHORT_TRANSFER	5	/* Set if we have a short transfer */
-#define NETFS_RREQ_OFFLOAD_COLLECTION	8	/* Offload collection to workqueue */
-#define NETFS_RREQ_NO_UNLOCK_FOLIO	9	/* Don't unlock no_unlock_folio on completion */
+#define NETFS_RREQ_CACHE_STOP		8	/* Set to stop caching (ENOBUFS or error) */
+#define NETFS_RREQ_CACHE_ERROR		9	/* Set if we got an error from the cache */
 #define NETFS_RREQ_CANCEL_CACHING	10	/* Set to cancel caching */
-#define NETFS_RREQ_UPLOAD_TO_SERVER	11	/* Need to write to the server */
-#define NETFS_RREQ_USE_IO_ITER		12	/* Use ->io_iter rather than ->i_pages */
+#define NETFS_RREQ_OFFLOAD_COLLECTION	12	/* Offload collection to workqueue */
+#define NETFS_RREQ_NO_UNLOCK_FOLIO	13	/* Don't unlock no_unlock_folio on completion */
+#define NETFS_RREQ_UPLOAD_TO_SERVER	14	/* Need to write to the server */
+#define NETFS_RREQ_USE_IO_ITER		15	/* Use ->io_iter rather than ->i_pages */
 #define NETFS_RREQ_NEED_PUT_RA_REFS	17	/* Need to put the folio refs RA gave us */
+#ifdef CONFIG_NETFS_PGPRIV2
 #define NETFS_RREQ_USE_PGPRIV2		31	/* [DEPRECATED] Use PG_private_2 to mark
 						 * write to cache on read */
+#endif
 	const struct netfs_request_ops *netfs_ops;
 };
 
@@ -299,12 +315,12 @@ struct netfs_request_ops {
 	int (*prepare_read)(struct netfs_io_subrequest *subreq);
 	void (*issue_read)(struct netfs_io_subrequest *subreq);
 	bool (*is_still_valid)(struct netfs_io_request *rreq);
-	int (*check_write_begin)(struct file *file, loff_t pos, unsigned len,
+	int (*check_write_begin)(struct file *file, uoff_t pos, unsigned len,
 				 struct folio **foliop, void **_fsdata);
 	void (*done)(struct netfs_io_request *rreq);
 
 	/* Modification handling */
-	void (*update_i_size)(struct inode *inode, loff_t i_size);
+	void (*update_i_size)(struct inode *inode, uoff_t i_size);
 	void (*post_modify)(struct inode *inode);
 
 	/* Write request handling */
@@ -332,7 +348,7 @@ struct netfs_cache_ops {
 
 	/* Read data from the cache */
 	int (*read)(struct netfs_cache_resources *cres,
-		    loff_t start_pos,
+		    uoff_t start_pos,
 		    struct iov_iter *iter,
 		    enum netfs_read_from_hole read_hole,
 		    netfs_io_terminated_t term_func,
@@ -340,7 +356,7 @@ struct netfs_cache_ops {
 
 	/* Write data to the cache */
 	int (*write)(struct netfs_cache_resources *cres,
-		     loff_t start_pos,
+		     uoff_t start_pos,
 		     struct iov_iter *iter,
 		     netfs_io_terminated_t term_func,
 		     void *term_func_priv);
@@ -350,15 +366,14 @@ struct netfs_cache_ops {
 
 	/* Expand readahead request */
 	void (*expand_readahead)(struct netfs_cache_resources *cres,
-				 unsigned long long *_start,
-				 unsigned long long *_len,
-				 unsigned long long i_size);
+				 uoff_t *_start,
+				 uoff_t *_len,
+				 uoff_t i_size);
 
 	/* Prepare a read operation, shortening it to a cached/uncached
 	 * boundary as appropriate.
 	 */
-	enum netfs_io_source (*prepare_read)(struct netfs_io_subrequest *subreq,
-					     unsigned long long i_size);
+	int (*prepare_read)(struct netfs_io_subrequest *subreq);
 
 	/* Prepare a write subrequest, working out if we're allowed to do it
 	 * and finding out the maximum amount of data to gather before
@@ -371,15 +386,24 @@ struct netfs_cache_ops {
 	 * actually do.
 	 */
 	int (*prepare_write)(struct netfs_cache_resources *cres,
-			     loff_t *_start, size_t *_len, size_t upper_len,
-			     loff_t i_size, bool no_space_allocated_yet);
+			     uoff_t *_start, size_t *_len, size_t upper_len,
+			     uoff_t i_size, bool no_space_allocated_yet);
 
 	/* Query the occupancy of the cache in a region, returning where the
 	 * next chunk of data starts and how long it is.
 	 */
 	int (*query_occupancy)(struct netfs_cache_resources *cres,
-			       loff_t start, size_t len, size_t granularity,
-			       loff_t *_data_start, size_t *_data_len);
+			       struct fscache_occupancy *occ);
+
+	/* Collect the result of buffered writeback to the cache.  This
+	 * includes copying a read to the cache.  block_type is one of:
+	 * - NETFS_CACHE_COLLECT_WRITE_DATA for a block of data
+	 * - NETFS_CACHE_COLLECT_WRITE_GAP if a discontiguity was skipped
+	 * - NETFS_CACHE_COLLECT_WRITE_CANCEL for a cancellation gap
+	 */
+	void (*collect_write)(struct netfs_io_request *wreq,
+			      uoff_t start, size_t len,
+			      enum netfs_cache_collect block_type);
 };
 
 /* High-level read API. */
@@ -410,7 +434,7 @@ struct readahead_control;
 void netfs_readahead(struct readahead_control *);
 int netfs_read_folio(struct file *, struct folio *);
 int netfs_write_begin(struct netfs_inode *, struct file *,
-		      struct address_space *, loff_t pos, unsigned int len,
+		      struct address_space *, uoff_t pos, unsigned int len,
 		      struct folio **, void **fsdata);
 int netfs_writepages(struct address_space *mapping,
 		     struct writeback_control *wbc);
@@ -488,10 +512,10 @@ static inline struct netfs_inode *netfs_inode(struct inode *inode)
  * cmpxchg8b without the need of the lock prefix).  For SMP compiles and 64bit
  * archs it makes no difference if preempt is enabled or not.
  */
-static inline unsigned long long netfs_read_remote_i_size(const struct inode *inode)
+static inline uoff_t netfs_read_remote_i_size(const struct inode *inode)
 {
 	const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
-	unsigned long long remote_i_size;
+	uoff_t remote_i_size;
 
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	unsigned int seq;
@@ -526,7 +550,7 @@ static inline unsigned long long netfs_read_remote_i_size(const struct inode *in
  * spinning forever.
  */
 static inline void netfs_write_remote_i_size(struct inode *inode,
-					     unsigned long long remote_i_size)
+					     uoff_t remote_i_size)
 {
 	struct netfs_inode *ictx = netfs_inode(inode);
 
@@ -563,10 +587,10 @@ static inline void netfs_write_remote_i_size(struct inode *inode,
  * cmpxchg8b without the need of the lock prefix).  For SMP compiles and 64bit
  * archs it makes no difference if preempt is enabled or not.
  */
-static inline unsigned long long netfs_read_zero_point(const struct inode *inode)
+static inline uoff_t netfs_read_zero_point(const struct inode *inode)
 {
 	struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
-	unsigned long long zero_point;
+	uoff_t zero_point;
 
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
 	unsigned int seq;
@@ -601,7 +625,7 @@ static inline unsigned long long netfs_read_zero_point(const struct inode *inode
  * forever.
  */
 static inline void netfs_write_zero_point(struct inode *inode,
-					  unsigned long long zero_point)
+					  uoff_t zero_point)
 {
 	struct netfs_inode *ictx = netfs_inode(inode);
 
@@ -642,9 +666,9 @@ static inline void netfs_write_zero_point(struct inode *inode,
  * archs it makes no difference if preempt is enabled or not.
  */
 static inline void netfs_read_sizes(const struct inode *inode,
-				    unsigned long long *i_size,
-				    unsigned long long *remote_i_size,
-				    unsigned long long *zero_point)
+				    uoff_t *i_size,
+				    uoff_t *remote_i_size,
+				    uoff_t *zero_point)
 {
 	const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
@@ -690,9 +714,9 @@ static inline void netfs_read_sizes(const struct inode *inode,
  * forever.
  */
 static inline void netfs_write_sizes(struct inode *inode,
-				     unsigned long long i_size,
-				     unsigned long long remote_i_size,
-				     unsigned long long zero_point)
+				     uoff_t i_size,
+				     uoff_t remote_i_size,
+				     uoff_t zero_point)
 {
 	struct netfs_inode *ictx = netfs_inode(inode);
 
@@ -760,7 +784,7 @@ static inline void netfs_inode_init(struct netfs_inode *ctx,
  * Inform the netfs lib that a file got resized so that it can adjust its state.
  */
 static inline void netfs_resize_file(struct netfs_inode *ictx,
-				     unsigned long long new_i_size,
+				     uoff_t new_i_size,
 				     bool changed_on_server)
 {
 #if BITS_PER_LONG==32 && defined(CONFIG_SMP)
diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h
index 1c6b1eec5efd68..ec0821a5b02dbb 100644
--- a/include/linux/netpoll.h
+++ b/include/linux/netpoll.h
@@ -16,11 +16,6 @@
 #include <linux/ip.h>
 #include <linux/udp.h>
 
-union inet_addr {
-	__be32		ip;
-	struct in6_addr	in6;
-};
-
 struct netpoll {
 	struct net_device *dev;
 	netdevice_tracker dev_tracker;
diff --git a/include/linux/nfs.h b/include/linux/nfs.h
index 0906a0b40c6aa5..8c2818db43c51f 100644
--- a/include/linux/nfs.h
+++ b/include/linux/nfs.h
@@ -11,59 +11,8 @@
 #include <linux/cred.h>
 #include <linux/sunrpc/auth.h>
 #include <linux/sunrpc/msg_prot.h>
-#include <linux/string.h>
-#include <linux/crc32.h>
-#include <uapi/linux/nfs.h>
-
-/* The LOCALIO program is entirely private to Linux and is
- * NOT part of the uapi.
- */
-#define NFS_LOCALIO_PROGRAM		400122
-#define LOCALIOPROC_NULL		0
-#define LOCALIOPROC_UUID_IS_LOCAL	1
-
-/*
- * This is the kernel NFS client file handle representation
- */
-#define NFS_MAXFHSIZE		128
-struct nfs_fh {
-	unsigned short		size;
-	unsigned char		data[NFS_MAXFHSIZE];
-};
-
-/*
- * Returns a zero iff the size and data fields match.
- * Checks only "size" bytes in the data field.
- */
-static inline int nfs_compare_fh(const struct nfs_fh *a, const struct nfs_fh *b)
-{
-	return a->size != b->size || memcmp(a->data, b->data, a->size) != 0;
-}
-
-static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *source)
-{
-	target->size = source->size;
-	memcpy(target->data, source->data, source->size);
-}
-
-enum nfs3_stable_how {
-	NFS_UNSTABLE = 0,
-	NFS_DATA_SYNC = 1,
-	NFS_FILE_SYNC = 2,
+#include <linux/nfs_fh.h>
 
-	/* used by direct.c to mark verf as invalid */
-	NFS_INVALID_STABLE_HOW = -1
-};
+#include <uapi/linux/nfs.h>
 
-/**
- * nfs_fhandle_hash - calculate the crc32 hash for the filehandle
- * @fh - pointer to filehandle
- *
- * returns a crc32 hash for the filehandle that is compatible with
- * the one displayed by "wireshark".
- */
-static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh)
-{
-	return ~crc32_le(0xFFFFFFFF, &fh->data[0], fh->size);
-}
 #endif /* _LINUX_NFS_H */
diff --git a/include/linux/nfs3.h b/include/linux/nfs3.h
index 404b8f724fc956..b6539a75edea07 100644
--- a/include/linux/nfs3.h
+++ b/include/linux/nfs3.h
@@ -7,6 +7,49 @@
 
 #include <uapi/linux/nfs3.h>
 
+/*
+ * NFSv3 error status values.
+ * See RFC 1813 Section 2.5
+ */
+enum {
+	NFS3ERR_PERM		= 1,
+	NFS3ERR_NOENT		= 2,
+	NFS3ERR_IO		= 5,
+	NFS3ERR_NXIO		= 6,
+	NFS3ERR_ACCES		= 13,
+	NFS3ERR_EXIST		= 17,
+	NFS3ERR_XDEV		= 18,
+	NFS3ERR_NODEV		= 19,
+	NFS3ERR_NOTDIR		= 20,
+	NFS3ERR_ISDIR		= 21,
+	NFS3ERR_INVAL		= 22,
+	NFS3ERR_FBIG		= 27,
+	NFS3ERR_NOSPC		= 28,
+	NFS3ERR_ROFS		= 30,
+	NFS3ERR_MLINK		= 31,
+	NFS3ERR_NAMETOOLONG	= 63,
+	NFS3ERR_NOTEMPTY	= 66,
+	NFS3ERR_DQUOT		= 69,
+	NFS3ERR_STALE		= 70,
+	NFS3ERR_REMOTE		= 71,
+	NFS3ERR_BADHANDLE	= 10001,
+	NFS3ERR_NOT_SYNC	= 10002,
+	NFS3ERR_BAD_COOKIE	= 10003,
+	NFS3ERR_NOTSUPP		= 10004,
+	NFS3ERR_TOOSMALL	= 10005,
+	NFS3ERR_SERVERFAULT	= 10006,
+	NFS3ERR_BADTYPE		= 10007,
+	NFS3ERR_JUKEBOX		= 10008,
+};
+
+enum nfs3_stable_how {
+	NFS_UNSTABLE = 0,
+	NFS_DATA_SYNC = 1,
+	NFS_FILE_SYNC = 2,
+
+	/* used to mark verf as invalid */
+	NFS_INVALID_STABLE_HOW = -1
+};
 
 /* Number of 32bit words in post_op_attr */
 #define NFS3_POST_OP_ATTR_WORDS		22
diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h
index 1a3981c26b23bf..41b7cdcc674f12 100644
--- a/include/linux/nfs4.h
+++ b/include/linux/nfs4.h
@@ -263,6 +263,12 @@ enum why_no_delegation4 { /* new to v4.1 */
 	WND4_IS_DIR = 8,
 };
 
+enum stable_how4 {
+	UNSTABLE4	= 0,
+	DATA_SYNC4	= 1,
+	FILE_SYNC4	= 2,
+};
+
 enum lock_type4 {
 	NFS4_UNLOCK_LT = 0,
 	NFS4_READ_LT = 1,
diff --git a/include/linux/nfs_fh.h b/include/linux/nfs_fh.h
new file mode 100644
index 00000000000000..49dfc5ec60fec8
--- /dev/null
+++ b/include/linux/nfs_fh.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * struct nfs_fh is an NFS version-agnostic data structure that
+ * stores an NFS file handle. It is also commonly used in NFS
+ * related APIs.
+ */
+#ifndef _LINUX_NFS_FH_H
+#define _LINUX_NFS_FH_H
+
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/crc32.h>
+
+/*
+ * The largest file handle size today is an NFSv4 file handle,
+ * which can be up to 128 octets long.
+ */
+#define NFS_MAXFHSIZE		128
+struct nfs_fh {
+	unsigned short		size;
+	unsigned char		data[NFS_MAXFHSIZE];
+};
+
+/**
+ * nfs_compare_fh - Compare two NFS file handles
+ * @a: An NFS file handle to be compared
+ * @b: An NFS file handle to be compared
+ *
+ * Checks only "size" bytes in each data field.
+ *
+ * Return: %false if the two file handles are equal, otherwise %true
+ */
+static inline bool nfs_compare_fh(const struct nfs_fh *a, const struct nfs_fh *b)
+{
+	return a->size != b->size || memcmp(a->data, b->data, a->size) != 0;
+}
+
+/**
+ * nfs_copy_fh - Copy an NFS file handle
+ * @target: Destination file handle
+ * @source: Source file handle
+ *
+ * Copies source->size bytes of file handle data into target.
+ */
+static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *source)
+{
+	target->size = source->size;
+	memcpy(target->data, source->data, source->size);
+}
+
+/**
+ * nfs_fhandle_hash - Calculate the crc32 hash for the filehandle
+ * @fh: An NFS file handle to hash
+ *
+ * Return: a crc32 hash for the filehandle that is compatible with
+ * the one displayed by "wireshark"
+ */
+static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh)
+{
+	return ~crc32_le(0xFFFFFFFF, &fh->data[0], fh->size);
+}
+
+#endif /* _LINUX_NFS_FH_H */
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index b85a73ae7919fb..9f3c0017944a26 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -21,7 +21,6 @@
 
 #include <linux/in.h>
 #include <linux/mm.h>
-#include <linux/pagemap.h>
 #include <linux/rbtree.h>
 #include <linux/refcount.h>
 #include <linux/rwsem.h>
@@ -437,11 +436,11 @@ extern int nfs_refresh_inode(struct inode *, struct nfs_fattr *);
 extern int nfs_post_op_update_inode(struct inode *inode, struct nfs_fattr *fattr);
 extern int nfs_post_op_update_inode_force_wcc(struct inode *inode, struct nfs_fattr *fattr);
 extern int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fattr *fattr);
-extern int nfs_getattr(struct mnt_idmap *, const struct path *,
+extern int nfs_getattr(const struct mnt_idmap *, const struct path *,
 		       struct kstat *, u32, unsigned int);
 extern void nfs_access_add_cache(struct inode *, struct nfs_access_entry *, const struct cred *);
 extern void nfs_access_set_mask(struct nfs_access_entry *, u32);
-extern int nfs_permission(struct mnt_idmap *, struct inode *, int);
+extern int nfs_permission(const struct mnt_idmap *, struct inode *, int);
 extern int nfs_open(struct inode *, struct file *);
 extern int nfs_attribute_cache_expired(struct inode *inode);
 extern int nfs_revalidate_inode(struct inode *inode, unsigned long flags);
@@ -450,7 +449,7 @@ extern int nfs_clear_invalid_mapping(struct address_space *mapping);
 extern bool nfs_mapping_need_revalidate_inode(struct inode *inode);
 extern int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping);
 extern int nfs_revalidate_mapping_rcu(struct inode *inode);
-extern int nfs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *);
+extern int nfs_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *);
 extern void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr, struct nfs_fattr *);
 extern void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr);
 extern struct nfs_open_context *get_nfs_open_context(struct nfs_open_context *ctx);
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index 34d294774f8cec..416c6f39f31ddb 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -74,6 +74,8 @@ struct nfs_client {
 	u64			cl_clientid;	/* constant */
 	nfs4_verifier		cl_confirm;	/* Clientid verifier */
 	unsigned long		cl_state;
+	/* bumped on each CB_NOTIFY_DEVICEID CHANGE for this client */
+	atomic_t		cl_deviceid_change_epoch;
 
 	spinlock_t		cl_lock;
 
@@ -101,6 +103,8 @@ struct nfs_client {
 	/* The flags used for obtaining the clientid during EXCHANGE_ID */
 	u32			cl_exchange_flags;
 	struct nfs4_session	*cl_session;	/* shared session */
+	/* CB_NOTIFY_DEVICEID DELETE suspects, protected by cl_lock */
+	struct list_head	cl_deviceid_deletes;
 	bool			cl_preserve_clid;
 	struct nfs41_server_owner *cl_serverowner;
 	struct nfs41_server_scope *cl_serverscope;
@@ -248,6 +252,10 @@ struct nfs_server {
 						   that are supported on this
 						   filesystem */
 	struct pnfs_layoutdriver_type  *pnfs_curr_ld; /* Active layout driver */
+	unsigned int		lg_reply_sz;	/* Learned LAYOUTGET reply
+						   buffer size, when the layout
+						   driver's default has proved
+						   too small */
 	struct rpc_wait_queue	roc_rpcwaitq;
 
 	/* the following fields are protected by nfs_client->cl_lock */
diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h
index 4b9a35dbc062da..bddcc632172cd9 100644
--- a/include/linux/nfs_page.h
+++ b/include/linux/nfs_page.h
@@ -12,7 +12,6 @@
 
 
 #include <linux/list.h>
-#include <linux/pagemap.h>
 #include <linux/wait.h>
 #include <linux/sunrpc/auth.h>
 #include <linux/nfs_xdr.h>
@@ -38,6 +37,7 @@ enum {
 	PG_REMOVE,		/* page group sync bit in write path */
 	PG_CONTENDED1,		/* Is someone waiting for a lock? */
 	PG_CONTENDED2,		/* Is someone waiting for a lock? */
+	PG_PINNED,		/* page is pinned by GUP */
 };
 
 struct nfs_inode;
@@ -58,6 +58,7 @@ struct nfs_page {
 	struct nfs_page		*wb_this_page;  /* list of reqs for this page */
 	struct nfs_page		*wb_head;       /* head pointer for req list */
 	unsigned short		wb_nio;		/* Number of I/O attempts */
+	unsigned int		wb_nr_pinned;	/* Number of pinned pages */
 };
 
 struct nfs_pgio_mirror;
@@ -125,15 +126,17 @@ struct nfs_pageio_descriptor {
 
 extern struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx,
 						  struct page *page,
+						  bool pinned,
 						  unsigned int pgbase,
 						  loff_t offset,
 						  unsigned int count);
 extern struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx,
 						   struct folio *folio,
+						   bool pinned,
 						   unsigned int offset,
 						   unsigned int count);
-extern	void nfs_release_request(struct nfs_page *);
-
+void nfs_release_request(struct nfs_page *req);
+void nfs_release_request_list(struct list_head *head);
 
 extern	void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
 			     struct inode *inode,
diff --git a/include/linux/nfs_ssc.h b/include/linux/nfs_ssc.h
index 22265b1ff08005..c199ea23e7eb77 100644
--- a/include/linux/nfs_ssc.h
+++ b/include/linux/nfs_ssc.h
@@ -2,80 +2,33 @@
 /*
  * include/linux/nfs_ssc.h
  *
+ * NFSv4.2 server-to-server copy, NFS client side APIs
+ *
  * Author: Dai Ngo <dai.ngo@oracle.com>
  *
  * Copyright (c) 2020, Oracle and/or its affiliates.
  */
 
-#include <linux/nfs_fs.h>
-#include <linux/sunrpc/svc.h>
+#ifndef _LINUX_NFS_SSC_H
+#define _LINUX_NFS_SSC_H
 
-extern struct nfs_ssc_client_ops_tbl nfs_ssc_client_tbl;
+#include <linux/nfs_fh.h>
+#include <linux/nfs4.h>
+
+struct file;
+struct vfsmount;
 
-/*
- * NFS_V4
- */
 struct nfs4_ssc_client_ops {
+	struct module *owner;
 	struct file *(*sco_open)(struct vfsmount *ss_mnt,
 		struct nfs_fh *src_fh, nfs4_stateid *stateid);
 	void (*sco_close)(struct file *filep);
 };
 
-/*
- * NFS_FS
- */
-struct nfs_ssc_client_ops {
-	void (*sco_sb_deactive)(struct super_block *sb);
-};
-
-struct nfs_ssc_client_ops_tbl {
-	const struct nfs4_ssc_client_ops *ssc_nfs4_ops;
-	const struct nfs_ssc_client_ops *ssc_nfs_ops;
-};
-
 extern void nfs42_ssc_register_ops(void);
 extern void nfs42_ssc_unregister_ops(void);
 
 extern void nfs42_ssc_register(const struct nfs4_ssc_client_ops *ops);
 extern void nfs42_ssc_unregister(const struct nfs4_ssc_client_ops *ops);
 
-#ifdef CONFIG_NFSD_V4_2_INTER_SSC
-static inline struct file *nfs42_ssc_open(struct vfsmount *ss_mnt,
-		struct nfs_fh *src_fh, nfs4_stateid *stateid)
-{
-	if (nfs_ssc_client_tbl.ssc_nfs4_ops)
-		return (*nfs_ssc_client_tbl.ssc_nfs4_ops->sco_open)(ss_mnt, src_fh, stateid);
-	return ERR_PTR(-EIO);
-}
-
-static inline void nfs42_ssc_close(struct file *filep)
-{
-	if (nfs_ssc_client_tbl.ssc_nfs4_ops)
-		(*nfs_ssc_client_tbl.ssc_nfs4_ops->sco_close)(filep);
-}
-#endif
-
-struct nfsd4_ssc_umount_item {
-	struct list_head nsui_list;
-	bool nsui_busy;
-	/*
-	 * nsui_refcnt inited to 2, 1 on list and 1 for consumer. Entry
-	 * is removed when refcnt drops to 1 and nsui_expire expires.
-	 */
-	refcount_t nsui_refcnt;
-	unsigned long nsui_expire;
-	struct vfsmount *nsui_vfsmount;
-	char nsui_ipaddr[RPC_MAX_ADDRBUFLEN + 1];
-};
-
-/*
- * NFS_FS
- */
-extern void nfs_ssc_register(const struct nfs_ssc_client_ops *ops);
-extern void nfs_ssc_unregister(const struct nfs_ssc_client_ops *ops);
-
-static inline void nfs_do_sb_deactive(struct super_block *sb)
-{
-	if (nfs_ssc_client_tbl.ssc_nfs_ops)
-		(*nfs_ssc_client_tbl.ssc_nfs_ops->sco_sb_deactive)(sb);
-}
+#endif /* _LINUX_NFS_SSC_H */
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index 7ed8fdb930d664..c0e29b4dfa6261 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -1693,6 +1693,7 @@ struct nfs_pgio_header {
 	struct nfs_client	*ds_clp;	/* pNFS data server */
 	u32			ds_commit_idx;	/* ds index if ds_clp is set */
 	u32			pgio_mirror_idx;/* mirror index in pgio layer */
+	struct nfs4_deviceid_node *ds_dev;	/* device node ref held across the I/O */
 };
 
 struct nfs_mds_commit_info {
@@ -1731,6 +1732,7 @@ struct nfs_commit_data {
 	struct nfs_open_context *context;
 	struct pnfs_layout_segment *lseg;
 	struct nfs_client	*ds_clp;	/* pNFS data server */
+	struct nfs4_deviceid_node *ds_dev;	/* device node ref held across the commit */
 	int			ds_commit_index;
 	loff_t			lwb;
 	const struct rpc_call_ops *mds_ops;
diff --git a/include/linux/nfsd_ssc.h b/include/linux/nfsd_ssc.h
new file mode 100644
index 00000000000000..7001410f01c299
--- /dev/null
+++ b/include/linux/nfsd_ssc.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * include/linux/nfsd_ssc.h
+ *
+ * NFSv4.2 server-to-server copy, NFS server side APIs
+ *
+ * Author: Dai Ngo <dai.ngo@oracle.com>
+ *
+ * Copyright (c) 2020, Oracle and/or its affiliates.
+ */
+
+#ifndef _LINUX_NFSD_SSC_H
+#define _LINUX_NFSD_SSC_H
+
+#include <linux/nfs_fh.h>
+#include <linux/nfs4.h>
+
+struct file;
+struct vfsmount;
+
+#if IS_ENABLED(CONFIG_NFS_V4_2_SSC_HELPER)
+struct file *nfsd42_ssc_open(struct vfsmount *ss_mnt, struct nfs_fh *src_fh,
+			     nfs4_stateid *stateid);
+void nfsd42_ssc_close(struct file *filp);
+#else
+static inline struct file *nfsd42_ssc_open(struct vfsmount *ss_mnt,
+					   struct nfs_fh *src_fh,
+					   nfs4_stateid *stateid)
+{
+	return ERR_PTR(-EIO);
+}
+
+static inline void nfsd42_ssc_close(struct file *filp)
+{
+}
+#endif
+
+#endif /* _LINUX_NFSD_SSC_H */
diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h
index 3d91043254e64a..8ce4d978a6367e 100644
--- a/include/linux/nfslocalio.h
+++ b/include/linux/nfslocalio.h
@@ -13,9 +13,18 @@
 #include <linux/uuid.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/svcauth.h>
-#include <linux/nfs.h>
+#include <linux/nfs_fh.h>
+
 #include <net/net_namespace.h>
 
+/*
+ * The LOCALIO program is entirely private to Linux and is NOT part of
+ * the uapi.
+ */
+#define NFS_LOCALIO_PROGRAM		400122
+#define LOCALIOPROC_NULL		0
+#define LOCALIOPROC_UUID_IS_LOCAL	1
+
 struct nfs_client;
 struct nfs_file_localio;
 
diff --git a/include/linux/notifier.h b/include/linux/notifier.h
index 01b6c9d9956f90..4eeae9741a6e85 100644
--- a/include/linux/notifier.h
+++ b/include/linux/notifier.h
@@ -46,6 +46,7 @@
  * often but notifier_blocks will seldom be removed.
  */
 
+struct device;
 struct notifier_block;
 
 typedef	int (*notifier_fn_t)(struct notifier_block *nb,
@@ -145,8 +146,14 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
 
 extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
 		struct notifier_block *nb);
+int devm_atomic_notifier_chain_register(struct device *dev,
+		struct atomic_notifier_head *nh,
+		struct notifier_block *nb);
 extern int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
 		struct notifier_block *nb);
+int devm_blocking_notifier_chain_register(struct device *dev,
+		struct blocking_notifier_head *nh,
+		struct notifier_block *nb);
 extern int raw_notifier_chain_register(struct raw_notifier_head *nh,
 		struct notifier_block *nb);
 extern int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
diff --git a/include/linux/ns/ns_common_types.h b/include/linux/ns/ns_common_types.h
index 6ed6b497831c7a..fddc62be3b626e 100644
--- a/include/linux/ns/ns_common_types.h
+++ b/include/linux/ns/ns_common_types.h
@@ -118,6 +118,9 @@ struct ns_common {
 	unsigned int inum;
 	struct ns_tree;
 	struct rcu_head ns_rcu;
+#ifdef CONFIG_SECURITY
+	void *ns_security;
+#endif
 };
 
 #define to_ns_common(__ns)                                    \
diff --git a/include/linux/omap-mailbox.h b/include/linux/omap-mailbox.h
index 3cc5c4ed7f5a67..e45f8c3841c239 100644
--- a/include/linux/omap-mailbox.h
+++ b/include/linux/omap-mailbox.h
@@ -6,8 +6,9 @@
 #ifndef OMAP_MAILBOX_H
 #define OMAP_MAILBOX_H
 
-typedef uintptr_t mbox_msg_t;
+typedef u32 mbox_msg_t;
 
-#define omap_mbox_message(data) (u32)(mbox_msg_t)(data)
+#define omap_mbox_to_message(data) ((void *)(uintptr_t)(data))
+#define omap_mbox_from_message(data) ((mbox_msg_t)(uintptr_t)(data))
 
 #endif /* OMAP_MAILBOX_H */
diff --git a/include/linux/once_lite.h b/include/linux/once_lite.h
index 236592c4eeb1a4..5e2b67039aaf01 100644
--- a/include/linux/once_lite.h
+++ b/include/linux/once_lite.h
@@ -10,19 +10,21 @@
 #define DO_ONCE_LITE(func, ...)						\
 	DO_ONCE_LITE_IF(true, func, ##__VA_ARGS__)
 
-#define __ONCE_LITE_IF(condition)					\
+#define __ONCE_LITE()							\
 	({								\
 		static bool __section(".data..once") __already_done;	\
-		bool __ret_cond = !!(condition);			\
 		bool __ret_once = false;				\
 									\
-		if (unlikely(__ret_cond) && unlikely(!__already_done)) {\
+		if (unlikely(!__already_done)) {			\
 			__already_done = true;				\
 			__ret_once = true;				\
 		}							\
 		unlikely(__ret_once);					\
 	})
 
+#define __ONCE_LITE_IF(condition)					\
+	(unlikely(condition) && __ONCE_LITE())
+
 #define DO_ONCE_LITE_IF(condition, func, ...)				\
 	({								\
 		bool __ret_do_once = !!(condition);			\
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 7a863572adce79..b0ddc652e76ccb 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -44,10 +44,6 @@
  * Consequently, PG_reserved for a page mapped into user space can indicate
  * the zero page, the vDSO, MMIO pages or device memory.
  *
- * The PG_private bitflag is set on pagecache pages if they contain filesystem
- * specific data (which is normally at page->private). It can be used by
- * private allocations for its own usage.
- *
  * During initiation of disk I/O, PG_locked is set. This bit is set before I/O
  * and cleared when writeback _starts_ or when read _completes_. PG_writeback
  * is set before writeback starts and cleared when it finishes.
@@ -105,7 +101,7 @@ enum pageflags {
 	PG_owner_2,		/* Owner use. If pagecache, fs may use */
 	PG_arch_1,
 	PG_reserved,
-	PG_private,		/* If pagecache, has fs-private data */
+	PG_folio,		/* Do not use: reserved for folio identification */
 	PG_private_2,		/* If pagecache, has fs aux data */
 	PG_reclaim,		/* To be reclaimed asap */
 	PG_swapbacked,		/* Page is backed by RAM/swap */
@@ -208,14 +204,13 @@ enum pageflags {
 static __always_inline bool compound_info_has_mask(void)
 {
 	/*
-	 * Limit mask usage to HugeTLB vmemmap optimization (HVO) where it
-	 * makes a difference.
+	 * Limit mask usage to HVO where it makes a difference.
 	 *
 	 * The approach with mask would work in the wider set of conditions,
 	 * but it requires validating that struct pages are naturally aligned
 	 * for all orders up to the MAX_FOLIO_ORDER, which can be tricky.
 	 */
-	if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP))
+	if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION))
 		return false;
 
 	return is_power_of_2(sizeof(struct page));
@@ -576,9 +571,19 @@ FOLIO_FLAG(swapbacked, FOLIO_HEAD_PAGE)
 /*
  * Private page markings that may be used by the filesystem that owns the page
  * for its own purposes.
- * - PG_private and PG_private_2 cause release_folio() and co to be invoked
+ * - folio->private and PG_private_2 cause release_folio() and co to be invoked
  */
-PAGEFLAG(Private, private, PF_ANY)
+
+static __always_inline bool folio_test_private(const struct folio *folio)
+{
+	/*
+	 * data_race() is added for readers without holding the folio lock.
+	 * Only the NULL/non-NULL answer is used and both are valid while
+	 * private is being attached or detached, so the race is benign.
+	 */
+	return data_race(folio->private);
+}
+
 FOLIO_FLAG(private_2, FOLIO_HEAD_PAGE)
 
 /* owner_2 can be set on tail pages for anon memory */
@@ -588,13 +593,12 @@ FOLIO_FLAG(owner_2, FOLIO_HEAD_PAGE)
  * Only test-and-set exist for PG_writeback.  The unconditional operators are
  * risky: they bypass page accounting.
  */
-TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
-	TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
+FOLIO_TEST_FLAG(writeback, FOLIO_HEAD_PAGE)
+	FOLIO_TEST_SET_FLAG(writeback, FOLIO_HEAD_PAGE)
 FOLIO_FLAG(mappedtodisk, FOLIO_HEAD_PAGE)
 
 /* PG_readahead is only used for reads; PG_reclaim is only for writes */
-PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
-	TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
+FOLIO_FLAG(reclaim, FOLIO_HEAD_PAGE)
 FOLIO_FLAG(readahead, FOLIO_HEAD_PAGE)
 	FOLIO_TEST_CLEAR_FLAG(readahead, FOLIO_HEAD_PAGE)
 
@@ -656,6 +660,7 @@ TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
 #define __PG_HWPOISON (1UL << PG_hwpoison)
 #else
 PAGEFLAG_FALSE(HWPoison, hwpoison)
+TESTSCFLAG_FALSE(HWPoison, hwpoison)
 #define __PG_HWPOISON 0
 #endif
 
@@ -1170,7 +1175,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
  */
 #define PAGE_FLAGS_CHECK_AT_FREE				\
 	(1UL << PG_lru		| 1UL << PG_locked	|	\
-	 1UL << PG_private	| 1UL << PG_private_2	|	\
+	 1UL << PG_private_2	|				\
 	 1UL << PG_writeback	| 1UL << PG_reserved	|	\
 	 1UL << PG_active 	|				\
 	 1UL << PG_unevictable	| __PG_MLOCKED | LRU_GEN_MASK)
@@ -1194,8 +1199,31 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
 	(0xffUL /* order */		| 1UL << PG_has_hwpoisoned |	\
 	 1UL << PG_large_rmappable	| 1UL << PG_partially_mapped)
 
-#define PAGE_FLAGS_PRIVATE				\
-	(1UL << PG_private | 1UL << PG_private_2)
+/**
+ * folio_has_attached_private - check if the folio has private data attached
+ * @folio: The folio to check.
+ *
+ * Use this in code that may encounter swapcache or hugetlb folios but only
+ * wants to detect attached private data.
+ *
+ * Return: true if the folio has private data attached.
+ */
+static inline bool folio_has_attached_private(const struct folio *folio)
+{
+	/*
+	 * Swapcache stores swp_entry_t in folio->swap, a union with
+	 * folio->private, and hugetlb stores its own flags in folio->private;
+	 * both are excluded.
+	 *
+	 * NOTE: For swapcache, folio->swap.val PG_swapcache are not set as
+	 * a whole, so folio_test_swapcache() is not reliable to exclude
+	 * swapcache. Use folio_test_swapbacked() instead, since it remains set
+	 * when a folio is added to/removed from swapcache.
+	 */
+
+	return folio_test_private(folio) && !folio_test_swapbacked(folio) &&
+	       !folio_test_hugetlb(folio);
+}
 /**
  * folio_has_private - Determine if folio has private stuff
  * @folio: The folio to be checked
@@ -1205,7 +1233,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page)
  */
 static inline int folio_has_private(const struct folio *folio)
 {
-	return !!(folio->flags.f & PAGE_FLAGS_PRIVATE);
+	return folio_has_attached_private(folio) || folio_test_private_2(folio);
 }
 
 #undef PF_ANY
diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h
index d649b6bbbc871b..2baf7a2b29b2e1 100644
--- a/include/linux/page_counter.h
+++ b/include/linux/page_counter.h
@@ -63,11 +63,12 @@ static inline void page_counter_init(struct page_counter *counter,
 	counter->track_failcnt = false;
 }
 
-static inline unsigned long page_counter_read(struct page_counter *counter)
+static inline unsigned long page_counter_read(const struct page_counter *counter)
 {
 	return atomic_long_read(&counter->usage);
 }
 
+long page_counter_margin(const struct page_counter *counter);
 void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages);
 void page_counter_charge(struct page_counter *counter, unsigned long nr_pages);
 bool page_counter_try_charge(struct page_counter *counter,
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 0adfa6605653db..73af18a3736706 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -14,7 +14,6 @@
 #include <linux/gfp.h>
 #include <linux/bitops.h>
 #include <linux/hardirq.h> /* for in_interrupt() */
-#include <linux/hugetlb_inline.h>
 
 struct folio_batch;
 
@@ -594,7 +593,6 @@ static inline void folio_attach_private(struct folio *folio, void *data)
 {
 	folio_get(folio);
 	folio->private = data;
-	folio_set_private(folio);
 }
 
 /**
@@ -629,9 +627,8 @@ static inline void *folio_detach_private(struct folio *folio)
 {
 	void *data = folio_get_private(folio);
 
-	if (!folio_test_private(folio))
+	if (!data)
 		return NULL;
-	folio_clear_private(folio);
 	folio->private = NULL;
 	folio_put(folio);
 
@@ -1128,8 +1125,7 @@ static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma,
 	const pgoff_t pgoff = __linear_anon_page_index(vma, address);
 
 	VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma));
-	/* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */
-	if (vma_is_anonymous(vma) && !vma->vm_file)
+	if (vma_is_anonymous(vma))
 		VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address));
 
 	return pgoff;
@@ -1416,6 +1412,7 @@ struct readahead_control {
 	bool dropbehind;
 	bool _workingset;
 	unsigned long _pflags;
+	bool _forward;
 };
 
 #define DEFINE_READAHEAD(ractl, f, r, m, i)				\
@@ -1480,18 +1477,29 @@ void page_cache_async_readahead(struct address_space *mapping,
 	page_cache_async_ra(&ractl, folio, req_count);
 }
 
+/*
+ * Adjust readahead_control to ensure next folio comes from
+ * [_index, _index + _nr_pages) afterwards and reset _batch_count.
+ */
+static inline void __readahead_advance(struct readahead_control *rac)
+{
+	if (rac->_forward)
+		rac->_index += rac->_batch_count;
+
+	rac->_nr_pages -= rac->_batch_count;
+	rac->_batch_count = 0;
+}
+
 static inline struct folio *__readahead_folio(struct readahead_control *ractl)
 {
 	struct folio *folio;
 
 	BUG_ON(ractl->_batch_count > ractl->_nr_pages);
-	ractl->_nr_pages -= ractl->_batch_count;
-	ractl->_index += ractl->_batch_count;
+	__readahead_advance(ractl);
+	ractl->_forward = true;
 
-	if (!ractl->_nr_pages) {
-		ractl->_batch_count = 0;
+	if (!ractl->_nr_pages)
 		return NULL;
-	}
 
 	folio = xa_load(&ractl->mapping->i_pages, ractl->_index);
 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
@@ -1517,6 +1525,39 @@ static inline struct folio *readahead_folio(struct readahead_control *ractl)
 	return folio;
 }
 
+/**
+ * readahead_folio_last - Get the next folio to read, from the tail.
+ * @ractl: The current readahead request.
+ *
+ * Like readahead_folio(), but walks the range back-to-front. The folio is
+ * returned locked with its refcount dropped; the caller unlocks it once I/O
+ * completes. Compound folios are returned once, at their head index.
+ *
+ * Context: The folio is locked.
+ * Return: A pointer to the next folio, or %NULL when done.
+ */
+static inline struct folio *readahead_folio_last(struct readahead_control *ractl)
+{
+	struct folio *folio;
+
+	/* Drop the previously returned batch from the remaining range. */
+	__readahead_advance(ractl);
+	ractl->_forward = false;
+
+	if (!ractl->_nr_pages)
+		return NULL;
+
+	/* xa_load() follows sibling entries, so a tail index returns the head */
+	folio = xa_load(&ractl->mapping->i_pages,
+			ractl->_index + ractl->_nr_pages - 1);
+	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
+
+	ractl->_batch_count = folio_nr_pages(folio);
+
+	folio_put(folio);
+	return folio;
+}
+
 static inline unsigned int __readahead_batch(struct readahead_control *rac,
 		struct page **array, unsigned int array_sz)
 {
@@ -1525,9 +1566,8 @@ static inline unsigned int __readahead_batch(struct readahead_control *rac,
 	struct folio *folio;
 
 	BUG_ON(rac->_batch_count > rac->_nr_pages);
-	rac->_nr_pages -= rac->_batch_count;
-	rac->_index += rac->_batch_count;
-	rac->_batch_count = 0;
+	__readahead_advance(rac);
+	rac->_forward = true;
 
 	xas_set(&xas, rac->_index);
 	rcu_read_lock();
diff --git a/include/linux/panic.h b/include/linux/panic.h
index f1dd417e54b294..23976b1dfdb6c8 100644
--- a/include/linux/panic.h
+++ b/include/linux/panic.h
@@ -13,7 +13,8 @@ __printf(1, 2)
 void panic(const char *fmt, ...) __noreturn __cold;
 __printf(1, 0)
 void vpanic(const char *fmt, va_list args) __noreturn __cold;
-void nmi_panic(struct pt_regs *regs, const char *msg);
+__printf(2, 3)
+void nmi_panic(struct pt_regs *regs, const char *fmt, ...);
 void check_panic_on_warn(const char *origin);
 extern void oops_enter(void);
 extern void oops_exit(void);
@@ -88,7 +89,8 @@ static inline void set_arch_panic_timeout(int timeout, int arch_default_timeout)
 #define TAINT_RANDSTRUCT		17
 #define TAINT_TEST			18
 #define TAINT_FWCTL			19
-#define TAINT_FLAGS_COUNT		20
+#define TAINT_FORCED_BIND		20
+#define TAINT_FLAGS_COUNT		21
 #define TAINT_FLAGS_MAX			((1UL << TAINT_FLAGS_COUNT) - 1)
 
 struct taint_flag {
@@ -110,4 +112,6 @@ extern void add_taint(unsigned flag, enum lockdep_ok);
 extern int test_taint(unsigned flag);
 extern unsigned long get_taint(void);
 
+void arch_do_panic(void);
+
 #endif	/* _LINUX_PANIC_H */
diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h
index 704e1dc8b30aa5..226f3e59cf60ad 100644
--- a/include/linux/pci-epf.h
+++ b/include/linux/pci-epf.h
@@ -163,7 +163,8 @@ enum pci_epf_doorbell_type {
  *       For MSI-backed doorbells this is the MSI message, while for
  *       "embedded" doorbells this represents an MMIO write that asserts
  *       an interrupt on the EP side.
- * @virq: IRQ number of this doorbell message
+ * @virq: IRQ number of this doorbell message. Multiple messages may use the
+ *        same IRQ; consumers must request each distinct IRQ only once.
  * @irq_flags: Required flags for request_irq()/request_threaded_irq().
  *             Callers may OR-in additional flags (e.g. IRQF_ONESHOT).
  * @type: Doorbell type.
diff --git a/include/linux/pci.h b/include/linux/pci.h
index d31a8d107b1ef5..d50cf9ae142de4 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -43,6 +43,7 @@
 #include <uapi/linux/pci.h>
 
 #include <linux/pci_ids.h>
+#include <linux/pci_liveupdate.h>
 
 #define PCI_STATUS_ERROR_BITS (PCI_STATUS_DETECTED_PARITY  | \
 			       PCI_STATUS_SIG_SYSTEM_ERROR | \
@@ -598,6 +599,9 @@ struct pci_dev {
 	u8		tph_mode;	/* TPH mode */
 	u8		tph_req_type;	/* TPH requester type */
 #endif
+#ifdef CONFIG_PCI_LIVEUPDATE
+	struct pci_liveupdate liveupdate;
+#endif
 };
 
 static inline struct pci_dev *pci_physfn(struct pci_dev *dev)
@@ -832,6 +836,9 @@ static inline struct pci_dev *pci_upstream_bridge(struct pci_dev *dev)
 	return dev->bus->self;
 }
 
+#define for_each_pci_dev_in_path(dev) \
+	for (; dev; dev = pci_upstream_bridge(dev))
+
 #ifdef CONFIG_PCI_MSI
 static inline bool pci_dev_msi_enabled(struct pci_dev *pci_dev)
 {
@@ -1951,9 +1958,10 @@ int pci_disable_link_state(struct pci_dev *pdev, int state);
 int pci_disable_link_state_locked(struct pci_dev *pdev, int state);
 int pci_enable_link_state(struct pci_dev *pdev, int state);
 int pci_enable_link_state_locked(struct pci_dev *pdev, int state);
+int pci_force_enable_link_state(struct pci_dev *pdev, int state);
 void pcie_no_aspm(void);
 bool pcie_aspm_support_enabled(void);
-bool pcie_aspm_enabled(struct pci_dev *pdev);
+u32 pcie_aspm_enabled(struct pci_dev *pdev);
 #else
 static inline int pci_disable_link_state(struct pci_dev *pdev, int state)
 { return 0; }
@@ -1963,9 +1971,11 @@ static inline int pci_enable_link_state(struct pci_dev *pdev, int state)
 { return 0; }
 static inline int pci_enable_link_state_locked(struct pci_dev *pdev, int state)
 { return 0; }
+static inline int pci_force_enable_link_state(struct pci_dev *pdev, int state)
+{ return 0; }
 static inline void pcie_no_aspm(void) { }
 static inline bool pcie_aspm_support_enabled(void) { return false; }
-static inline bool pcie_aspm_enabled(struct pci_dev *pdev) { return false; }
+static inline u32 pcie_aspm_enabled(struct pci_dev *pdev) { return 0; }
 #endif
 
 #ifdef CONFIG_HOTPLUG_PCI
diff --git a/include/linux/pci_liveupdate.h b/include/linux/pci_liveupdate.h
new file mode 100644
index 00000000000000..bb79348769b722
--- /dev/null
+++ b/include/linux/pci_liveupdate.h
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * PCI Live Update support (Public/Driver API)
+ *
+ * Copyright (c) 2026, Google LLC.
+ * David Matlack <dmatlack@google.com>
+ */
+#ifndef LINUX_PCI_LIVEUPDATE_H
+#define LINUX_PCI_LIVEUPDATE_H
+
+#include <linux/kho/abi/pci.h>
+#include <linux/liveupdate.h>
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+
+/**
+ * struct pci_liveupdate - PCI Live Update state for a struct pci_dev
+ * @outgoing: State preserved for the next kernel.
+ * @incoming: State preserved by the previous kernel.
+ * @was_incoming: True if this struct pci_dev was incoming-preserved when it was
+ *                set up, i.e. it was matched to state preserved by the previous
+ *                kernel. Unlike @incoming, this is never cleared, so it stays
+ *                true after the device finishes participating in Live Update.
+ * @frozen: True if the outgoing preservation status of this device is frozen
+ *          and thus cannot be changed.
+ */
+struct pci_liveupdate {
+	struct pci_dev_ser *outgoing;
+	struct pci_dev_ser *incoming;
+	bool was_incoming;
+	bool frozen;
+};
+
+struct pci_dev;
+
+#ifdef CONFIG_PCI_LIVEUPDATE
+int pci_liveupdate_register_flb(struct liveupdate_file_handler *fh);
+void pci_liveupdate_unregister_flb(struct liveupdate_file_handler *fh);
+int pci_liveupdate_preserve(struct pci_dev *dev);
+void pci_liveupdate_unpreserve(struct pci_dev *dev);
+void pci_liveupdate_finish(struct pci_dev *dev);
+bool pci_liveupdate_is_incoming(struct pci_dev *dev);
+#else
+static inline int pci_liveupdate_register_flb(struct liveupdate_file_handler *fh)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void pci_liveupdate_unregister_flb(struct liveupdate_file_handler *fh)
+{
+}
+
+static inline int pci_liveupdate_preserve(struct pci_dev *dev)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline void pci_liveupdate_unpreserve(struct pci_dev *dev)
+{
+}
+
+static inline void pci_liveupdate_finish(struct pci_dev *dev)
+{
+}
+
+static inline bool pci_liveupdate_is_incoming(struct pci_dev *dev)
+{
+	return false;
+}
+#endif
+
+#endif /* LINUX_PCI_LIVEUPDATE_H */
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 5842552294c195..7797ce2075551b 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -306,6 +306,7 @@ struct perf_event_pmu_context;
 #define PERF_PMU_CAP_AUX_PAUSE		0x0200
 #define PERF_PMU_CAP_AUX_PREFER_LARGE	0x0400
 #define PERF_PMU_CAP_MEDIATED_VPMU	0x0800
+#define PERF_PMU_CAP_SIMD_REGS		0x1000
 
 /**
  * pmu::scope
@@ -1467,23 +1468,7 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data,
 	return size;
 }
 
-/*
- * Clear all bitfields in the perf_branch_entry.
- * The to and from fields are not cleared because they are
- * systematically modified by caller.
- */
-static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br)
-{
-	br->mispred	= 0;
-	br->predicted	= 0;
-	br->in_tx	= 0;
-	br->abort	= 0;
-	br->cycles	= 0;
-	br->type	= 0;
-	br->spec	= PERF_BR_SPEC_NA;
-	br->reserved	= 0;
-}
-
+extern u64 perf_update_xregs_size(struct perf_event *event, bool intr);
 extern void perf_output_sample(struct perf_output_handle *handle,
 			       struct perf_event_header *header,
 			       struct perf_sample_data *data,
@@ -1534,6 +1519,27 @@ perf_event__output_id_sample(struct perf_event *event,
 extern void
 perf_log_lost_samples(struct perf_event *event, u64 lost);
 
+static inline bool event_has_simd_regs(struct perf_event *event)
+{
+	struct perf_event_attr *attr = &event->attr;
+
+	if (!(event->attr.sample_type &
+	      (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)))
+		return false;
+
+	return attr->sample_simd_regs_enabled != 0;
+}
+
+static inline bool event_has_extended_regs(struct perf_event *event)
+{
+	struct perf_event_attr *attr = &event->attr;
+
+	return ((attr->sample_type & PERF_SAMPLE_REGS_USER) &&
+		(attr->sample_regs_user & PERF_REG_EXTENDED_MASK)) ||
+	       ((attr->sample_type & PERF_SAMPLE_REGS_INTR) &&
+		(attr->sample_regs_intr & PERF_REG_EXTENDED_MASK));
+}
+
 static inline bool event_has_any_exclude_flag(struct perf_event *event)
 {
 	struct perf_event_attr *attr = &event->attr;
diff --git a/include/linux/perf_regs.h b/include/linux/perf_regs.h
index f632c5725f162f..09dbc2fc38592f 100644
--- a/include/linux/perf_regs.h
+++ b/include/linux/perf_regs.h
@@ -9,6 +9,16 @@ struct perf_regs {
 	struct pt_regs	*regs;
 };
 
+u64 perf_reg_value(struct pt_regs *regs, int idx);
+int perf_reg_validate(u64 mask, bool simd_enabled);
+u64 perf_reg_abi(struct task_struct *task);
+void perf_get_regs_user(struct perf_regs *regs_user,
+			struct pt_regs *regs);
+int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask,
+			   u16 pred_qwords, u32 pred_mask);
+u64 perf_simd_reg_value(struct pt_regs *regs, int idx,
+			u16 qwords_idx, bool pred);
+
 #ifdef CONFIG_HAVE_PERF_REGS
 #include <asm/perf_regs.h>
 
@@ -16,35 +26,9 @@ struct perf_regs {
 #define PERF_REG_EXTENDED_MASK	0
 #endif
 
-u64 perf_reg_value(struct pt_regs *regs, int idx);
-int perf_reg_validate(u64 mask);
-u64 perf_reg_abi(struct task_struct *task);
-void perf_get_regs_user(struct perf_regs *regs_user,
-			struct pt_regs *regs);
 #else
 
 #define PERF_REG_EXTENDED_MASK	0
 
-static inline u64 perf_reg_value(struct pt_regs *regs, int idx)
-{
-	return 0;
-}
-
-static inline int perf_reg_validate(u64 mask)
-{
-	return mask ? -ENOSYS : 0;
-}
-
-static inline u64 perf_reg_abi(struct task_struct *task)
-{
-	return PERF_SAMPLE_REGS_ABI_NONE;
-}
-
-static inline void perf_get_regs_user(struct perf_regs *regs_user,
-				      struct pt_regs *regs)
-{
-	regs_user->regs = task_pt_regs(current);
-	regs_user->abi = perf_reg_abi(current);
-}
 #endif /* CONFIG_HAVE_PERF_REGS */
 #endif /* _LINUX_PERF_REGS_H */
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 8c093c119e5a82..780fe849ff8b7e 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -490,35 +490,35 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma,
 #endif
 
 #ifndef ptep_get
-static inline pte_t ptep_get(pte_t *ptep)
+static inline pte_t ptep_get(const pte_t *ptep)
 {
 	return READ_ONCE(*ptep);
 }
 #endif
 
 #ifndef pmdp_get
-static inline pmd_t pmdp_get(pmd_t *pmdp)
+static inline pmd_t pmdp_get(const pmd_t *pmdp)
 {
 	return READ_ONCE(*pmdp);
 }
 #endif
 
 #ifndef pudp_get
-static inline pud_t pudp_get(pud_t *pudp)
+static inline pud_t pudp_get(const pud_t *pudp)
 {
 	return READ_ONCE(*pudp);
 }
 #endif
 
 #ifndef p4dp_get
-static inline p4d_t p4dp_get(p4d_t *p4dp)
+static inline p4d_t p4dp_get(const p4d_t *p4dp)
 {
 	return READ_ONCE(*p4dp);
 }
 #endif
 
 #ifndef pgdp_get
-static inline pgd_t pgdp_get(pgd_t *pgdp)
+static inline pgd_t pgdp_get(const pgd_t *pgdp)
 {
 	return READ_ONCE(*pgdp);
 }
@@ -2313,6 +2313,20 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level)
 	}
 }
 
+void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size);
+
+#define ptval_to_str(buf, val)								\
+	do {										\
+		auto __val = (val);							\
+											\
+		ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val));	\
+	} while (0)
+
+#if defined(__SIZEOF_INT128__)
+#define PTVAL_STR_MAX	(32 + 1) /* Max 128-bit value in hex + NUL */
+#else
+#define PTVAL_STR_MAX	(16 + 1) /* Max 64-bit value in hex + NUL */
+#endif
 #endif /* !__ASSEMBLER__ */
 
 #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT)
diff --git a/include/linux/phy.h b/include/linux/phy.h
index 5f8d65868e0ff5..7c5098a0dd6cd1 100644
--- a/include/linux/phy.h
+++ b/include/linux/phy.h
@@ -376,6 +376,24 @@ struct mii_bus {
 			 int regnum, u16 val);
 	/** @reset: Perform a reset of the bus */
 	int (*reset)(struct mii_bus *bus);
+	/**
+	 * @notify_phy_attach: Perform post-attach handling for MDIO bus
+	 * drivers. Optional and independent of @notify_phy_detach. Called
+	 * in phy_attach_direct() right before phy_resume(). Runs in process
+	 * context, may sleep and may be called with RTNL held. Must not
+	 * acquire or rely on RTNL. Returns 0 on success or negative errno
+	 * on failure. Must unwind its own state on error as attachment is
+	 * aborted.
+	 */
+	int (*notify_phy_attach)(struct phy_device *phydev);
+	/**
+	 * @notify_phy_detach: Perform pre-detach handling for MDIO bus
+	 * drivers. Optional and independent of @notify_phy_attach. Called
+	 * in phy_detach() right after phy_suspend(). Runs in process context,
+	 * may sleep and may be called with RTNL held. Must not acquire or
+	 * rely on RTNL.
+	 */
+	void (*notify_phy_detach)(struct phy_device *phydev);
 
 	/** @stats: Statistic counters per device on the bus */
 	struct mdio_bus_stats stats[PHY_MAX_ADDR];
@@ -1697,7 +1715,7 @@ static inline bool phy_can_wakeup(struct phy_device *phydev)
  * phy_may_wakeup() - indicate whether PHY has wakeup enabled
  * @phydev: The phy_device struct
  *
- * Returns: true/false depending on the PHY driver's device_set_wakeup_enabled()
+ * Returns: true/false depending on the PHY driver's device_set_wakeup_enable()
  * setting if using the driver model, otherwise the legacy determination.
  */
 bool phy_may_wakeup(struct phy_device *phydev);
@@ -2422,10 +2440,10 @@ int phy_get_mac_termination(struct phy_device *phydev, struct device *dev,
 void phy_resolve_pause(unsigned long *local_adv, unsigned long *partner_adv,
 		       bool *tx_pause, bool *rx_pause);
 
-int phy_register_fixup_for_id(const char *bus_id,
-			      int (*run)(struct phy_device *));
-int phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask,
-			       int (*run)(struct phy_device *));
+void __init phy_register_fixup_for_id(const char *bus_id,
+				      int (*run)(struct phy_device *));
+void __init phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask,
+				       int (*run)(struct phy_device *));
 
 int phy_eee_tx_clock_stop_capable(struct phy_device *phydev);
 int phy_eee_rx_clock_stop(struct phy_device *phydev, bool clk_stop_enable);
diff --git a/include/linux/phy/phy-thunderbolt.h b/include/linux/phy/phy-thunderbolt.h
new file mode 100644
index 00000000000000..5f48059814e199
--- /dev/null
+++ b/include/linux/phy/phy-thunderbolt.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __PHY_TBT_H_
+#define __PHY_TBT_H_
+
+enum tbt_phy_submode {
+	PHY_SUBMODE_TBT3,
+	PHY_SUBMODE_USB4,
+};
+
+#endif
diff --git a/include/linux/phy/phy.h b/include/linux/phy/phy.h
index ea47975e288aea..c480d6cb42c500 100644
--- a/include/linux/phy/phy.h
+++ b/include/linux/phy/phy.h
@@ -20,6 +20,7 @@
 #include <linux/phy/phy-hdmi.h>
 #include <linux/phy/phy-lvds.h>
 #include <linux/phy/phy-mipi-dphy.h>
+#include <linux/phy/phy-thunderbolt.h>
 
 struct phy;
 
@@ -45,6 +46,7 @@ enum phy_mode {
 	PHY_MODE_LVDS,
 	PHY_MODE_DP,
 	PHY_MODE_HDMI,
+	PHY_MODE_TBT,
 };
 
 enum phy_media {
@@ -284,6 +286,8 @@ struct phy *devm_of_phy_optional_get(struct device *dev, struct device_node *np,
 				     const char *con_id);
 struct phy *devm_of_phy_get_by_index(struct device *dev, struct device_node *np,
 				     int index);
+struct phy *phy_get_by_of_node(struct device_node *np);
+struct phy *devm_phy_get_by_of_node(struct device *dev, struct device_node *np);
 void of_phy_put(struct phy *phy);
 void phy_put(struct device *dev, struct phy *phy);
 void devm_phy_put(struct device *dev, struct phy *phy);
@@ -493,6 +497,17 @@ static inline struct phy *devm_of_phy_get_by_index(struct device *dev,
 	return ERR_PTR(-ENOSYS);
 }
 
+static inline struct phy *phy_get_by_of_node(struct device_node *np)
+{
+	return ERR_PTR(-ENOSYS);
+}
+
+static inline struct phy *devm_phy_get_by_of_node(struct device *dev,
+						  struct device_node *np)
+{
+	return ERR_PTR(-ENOSYS);
+}
+
 static inline void of_phy_put(struct phy *phy)
 {
 }
diff --git a/include/linux/phylink.h b/include/linux/phylink.h
index 1dda5c7ed5f146..3a88a69882a610 100644
--- a/include/linux/phylink.h
+++ b/include/linux/phylink.h
@@ -843,4 +843,6 @@ void phylink_replay_link_begin(struct phylink *pl);
 
 void phylink_replay_link_end(struct phylink *pl);
 
+void phylink_update_mac_pause_capabilities(struct phylink *pl, unsigned long mac_pause);
+
 #endif
diff --git a/include/linux/pinctrl/consumer.h b/include/linux/pinctrl/consumer.h
index 11b8f0b8da0c49..81e7866bd2e330 100644
--- a/include/linux/pinctrl/consumer.h
+++ b/include/linux/pinctrl/consumer.h
@@ -12,6 +12,7 @@
 #define __LINUX_PINCTRL_CONSUMER_H
 
 #include <linux/err.h>
+#include <linux/errno.h>
 #include <linux/types.h>
 
 #include <linux/pinctrl/pinctrl-state.h>
@@ -107,14 +108,14 @@ static inline int
 pinctrl_gpio_get_config(struct gpio_chip *gc, unsigned int offset,
 			unsigned long *config)
 {
-	return 0;
+	return -ENOTSUPP;
 }
 
 static inline int
 pinctrl_gpio_set_config(struct gpio_chip *gc, unsigned int offset,
 			    unsigned long config)
 {
-	return 0;
+	return -ENOTSUPP;
 }
 
 static inline struct pinctrl * __must_check pinctrl_get(struct device *dev)
diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h
index 749d985e9da2db..a7bd69d8352227 100644
--- a/include/linux/platform_data/cros_ec_commands.h
+++ b/include/linux/platform_data/cros_ec_commands.h
@@ -1344,6 +1344,26 @@ enum ec_feature_code {
 	 * The EC supports UCSI PPM.
 	 */
 	EC_FEATURE_UCSI_PPM = 54,
+	/*
+	 * The EC supports Strauss keyboard.
+	 */
+	EC_FEATURE_STRAUSS = 55,
+	/*
+	 * The EC supports PoE.
+	 */
+	EC_FEATURE_POE = 56,
+	/*
+	 * The EC supports a hybrid boost charger
+	 */
+	EC_FEATURE_CHARGER_HYBRID_POWER_BOOST = 57,
+	/*
+	 * Support signaling new console logs via host event
+	 */
+	EC_FEATURE_CONSOLE_LOG_EVENT = 58,
+	/*
+	 * The EC supports power monitoring
+	 */
+	EC_FEATURE_PWRMON = 59,
 };
 
 #define EC_FEATURE_MASK_0(event_code) BIT(event_code % 32)
@@ -3516,6 +3536,9 @@ enum ec_mkbp_event {
 	/* Peripheral device charger event */
 	EC_MKBP_EVENT_PCHG = 12,
 
+	/* Power monitor telemetry event */
+	EC_MKBP_EVENT_PWRMON = 13,
+
 	/* Number of MKBP events */
 	EC_MKBP_EVENT_COUNT,
 };
@@ -3599,6 +3622,12 @@ union __ec_align_offset1 ec_response_get_next_data_v3 {
 	uint32_t cec_events;
 
 	uint8_t cec_message[16];
+
+	struct __ec_todo_packed {
+		int64_t value;
+		uint32_t samples;
+		uint8_t channel_id;
+	} pwrmon_data;
 };
 BUILD_ASSERT(sizeof(union ec_response_get_next_data_v3) == 18);
 
@@ -6589,6 +6618,49 @@ struct ec_params_usb_pd_mux_ack {
 	uint8_t port; /* USB-C port number */
 } __ec_align1;
 
+/**
+ * Power monitoring. Used to read power consumptions on rails
+ */
+#define EC_CMD_PWRMON 0x0608
+
+enum ec_pwrmon_cmd {
+	EC_PWRMON_GET_CHANNEL_COUNT = 0,
+	EC_PWRMON_DUMP_INFO = 1,
+	EC_PWRMON_SET_RATE = 2,
+	EC_PWRMON_GET_RATE = 3,
+	EC_PWRMON_START = 4,
+	EC_PWRMON_STOP = 5,
+	EC_PWRMON_LATCH = 6,
+};
+
+struct ec_params_pwrmon {
+	uint8_t cmd;
+	union {
+		uint16_t set_rate;
+		uint8_t channel_id;
+	} __ec_align2;
+
+	/*
+	 * The following commands have no args:
+	 *
+	 * start, stop, latch
+	 *
+	 */
+} __ec_align4;
+
+struct pwrmon_dump_info {
+	uint8_t channel_id;
+	char channel_name[32];
+} __ec_align4;
+
+struct ec_response_pwrmon {
+	union {
+		uint16_t sample_rate;
+		uint8_t channel_count;
+		struct pwrmon_dump_info dump_info;
+	} __ec_align4;
+} __ec_align4;
+
 /*****************************************************************************/
 /*
  * Reserve a range of host commands for board-specific, experimental, or
diff --git a/include/linux/platform_data/microchip-ksz.h b/include/linux/platform_data/microchip-ksz.h
index 028781ad405932..411d5e164383a2 100644
--- a/include/linux/platform_data/microchip-ksz.h
+++ b/include/linux/platform_data/microchip-ksz.h
@@ -31,9 +31,11 @@ enum ksz_chip_id {
 	KSZ88X3_CHIP_ID = 0x8830,
 	KSZ8864_CHIP_ID = 0x8864,
 	KSZ8895_CHIP_ID = 0x8895,
+	KSZ8995XA_CHIP_ID = 0x8995,
 	KSZ9477_CHIP_ID = 0x00947700,
 	KSZ9896_CHIP_ID = 0x00989600,
 	KSZ9897_CHIP_ID = 0x00989700,
+	KSZ9897S_CHIP_ID = 0x00989701,
 	KSZ9893_CHIP_ID = 0x00989300,
 	KSZ9563_CHIP_ID = 0x00956300,
 	KSZ8567_CHIP_ID = 0x00856700,
diff --git a/include/linux/platform_data/mipi-i3c-hci.h b/include/linux/platform_data/mipi-i3c-hci.h
index ab7395f455f942..6f61d8a023786e 100644
--- a/include/linux/platform_data/mipi-i3c-hci.h
+++ b/include/linux/platform_data/mipi-i3c-hci.h
@@ -3,13 +3,17 @@
 #define INCLUDE_PLATFORM_DATA_MIPI_I3C_HCI_H
 
 #include <linux/compiler_types.h>
+#include <linux/types.h>
 
 /**
  * struct mipi_i3c_hci_platform_data - Platform-dependent data for mipi_i3c_hci
  * @base_regs: Register set base address (to support multi-bus instances)
+ * @instance: Zero-based instance number of the Bus Controller as defined by the
+ *            DisCo specification I3C Target Address (_ADR) Encoding
  */
 struct mipi_i3c_hci_platform_data {
 	void __iomem *base_regs;
+	u8 instance;
 };
 
 #endif
diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h
index b5ed8c83ace1c4..be4d2873ffc94f 100644
--- a/include/linux/platform_data/x86/asus-wmi.h
+++ b/include/linux/platform_data/x86/asus-wmi.h
@@ -54,6 +54,7 @@
 #define ASUS_WMI_DEVID_LED5		0x00020015
 #define ASUS_WMI_DEVID_LED6		0x00020016
 #define ASUS_WMI_DEVID_MICMUTE_LED		0x00040017
+#define ASUS_WMI_DEVID_MUTE_LED	0x0004001C
 
 /* Disable Camera LED */
 #define ASUS_WMI_DEVID_CAMERA_LED_NEG	0x00060078 /* 0 = on (unused) */
@@ -140,8 +141,9 @@
 
 #define ASUS_WMI_DEVID_APU_MEM		0x000600C1
 
-#define ASUS_WMI_DEVID_DGPU_BASE_TGP	0x00120099
+#define ASUS_WMI_DEVID_DGPU_POWER_STATE	0x00120097
 #define ASUS_WMI_DEVID_DGPU_SET_TGP	0x00120098
+#define ASUS_WMI_DEVID_DGPU_BASE_TGP	0x00120099
 
 /* gpu mux switch, 0 = dGPU, 1 = Optimus */
 #define ASUS_WMI_DEVID_GPU_MUX		0x00090016
diff --git a/include/linux/platform_data/x86/int3472.h b/include/linux/platform_data/x86/int3472.h
index a73841dfae2732..b1040e36deb865 100644
--- a/include/linux/platform_data/x86/int3472.h
+++ b/include/linux/platform_data/x86/int3472.h
@@ -25,6 +25,8 @@
 #define INT3472_GPIO_TYPE_RESET					0x00
 #define INT3472_GPIO_TYPE_POWERDOWN				0x01
 #define INT3472_GPIO_TYPE_STROBE				0x02
+#define INT3472_GPIO_TYPE_POWER0				0x07
+#define INT3472_GPIO_TYPE_POWER1				0x08
 #define INT3472_GPIO_TYPE_POWER_ENABLE				0x0b
 #define INT3472_GPIO_TYPE_CLK_ENABLE				0x0c
 #define INT3472_GPIO_TYPE_PRIVACY_LED				0x0d
diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h
index 3d5bbcbae73082..3bd0f1f0d87049 100644
--- a/include/linux/platform_device.h
+++ b/include/linux/platform_device.h
@@ -344,6 +344,19 @@ static inline void platform_set_drvdata(struct platform_device *pdev,
 #define builtin_platform_driver(__platform_driver) \
 	builtin_driver(__platform_driver, platform_driver_register)
 
+/*
+ * subsys_platform_driver() - Helper macro for drivers that don't do anything
+ * special in module init/exit but need to register earlier, at
+ * subsys_initcall level, when built in.  This eliminates a lot of
+ * boilerplate.  Each driver may only use this macro once, and calling it
+ * replaces module_init() and module_exit().  This is meant to be a parallel
+ * of module_platform_driver() above, but with the init call promoted to
+ * subsys_initcall() so built-in providers are available earlier during boot.
+ */
+#define subsys_platform_driver(__platform_driver) \
+	subsys_driver(__platform_driver, platform_driver_register, \
+			platform_driver_unregister)
+
 /* module_platform_driver_probe() - Helper macro for drivers that don't do
  * anything special in module init/exit.  This eliminates a lot of
  * boilerplate.  Each module may only use this macro once, and
@@ -376,6 +389,23 @@ static int __init __platform_driver##_init(void) \
 } \
 device_initcall(__platform_driver##_init); \
 
+/*
+ * subsys_platform_driver_probe() - Helper macro for drivers that don't do
+ * anything special in device init and have no exit, but need to register
+ * earlier, at subsys_initcall level.  This eliminates some boilerplate.  Each
+ * driver may only use this macro once, and using it replaces subsys_initcall.
+ * This is meant to be a parallel of builtin_platform_driver_probe above, but
+ * with the init call promoted to subsys_initcall so the provider is available
+ * earlier during boot.
+ */
+#define subsys_platform_driver_probe(__platform_driver, __platform_probe) \
+static int __init __platform_driver##_init(void) \
+{ \
+	return platform_driver_probe(&(__platform_driver), \
+				     __platform_probe);    \
+} \
+subsys_initcall(__platform_driver##_init) \
+
 #define platform_create_bundle(driver, probe, res, n_res, data, size) \
 	__platform_create_bundle(driver, probe, res, n_res, data, size, THIS_MODULE, KBUILD_MODNAME)
 extern struct platform_device *__platform_create_bundle(
diff --git a/include/linux/pm.h b/include/linux/pm.h
index afcaaa37a81266..ef3f1310e749c3 100644
--- a/include/linux/pm.h
+++ b/include/linux/pm.h
@@ -663,6 +663,99 @@ struct pm_subsys_data {
 #define DPM_FLAG_SMART_SUSPEND		BIT(2)
 #define DPM_FLAG_MAY_SKIP_RESUME	BIT(3)
 
+/**
+ * struct dev_pm_info - Device power management information.
+ *
+ * @power_state: Legacy power state (mostly unused in modern kernels).
+ * @can_wakeup: Device is capable of generating wakeup signals.
+ * @async_suspend: Device can be suspended and resumed asynchronously.
+ * @in_dpm_list: Device is on the dpm_list.
+ * @is_prepared: Device's ->prepare() callback has run successfully.
+ * @is_suspended: Device is suspended during a system sleep transition.
+ * @is_noirq_suspended: Device's noirq suspend callback has run successfully.
+ * @is_late_suspended: Device's late suspend callback has run successfully.
+ * @no_pm: Device does not participate in power management transitions.
+ * @early_init: Device was initialized before standard PM initialization.
+ * @direct_complete: Device can skip suspend/resume callbacks and remain
+ *   runtime-suspended during system sleep.
+ * @driver_flags: Driver flags (e.g. %DPM_FLAG_SMART_SUSPEND) set at probe time.
+ * @lock: Spinlock used for synchronizing PM state transitions and runtime PM
+ *   operations.
+ * @entry: List head for device power management lists.
+ * @completion: Completion for synchronization during asynchronous system
+ *   suspend/resume.
+ * @wakeup: Wakeup source object associated with the device.
+ * @work_in_progress: Asynchronous PM operation in progress.
+ * @wakeup_path: Device is in the wakeup path or can wake the system up.
+ * @syscore: Device participates in syscore power management operations.
+ * @no_pm_callbacks: Device has no PM callbacks; handled by parent or subsystem.
+ * @smart_suspend: Driver requested smart-suspend behavior.
+ * @must_resume: Device must be resumed during system resume.
+ * @may_skip_resume: Set by subsystems to indicate driver resume callbacks may
+ *   be skipped.
+ * @out_band_wakeup: Out-of-band wakeup is supported.
+ * @strict_midlayer: Middle layer code does not want callbacks invoked via
+ *   pm_runtime_force_suspend() / pm_runtime_force_resume().
+ * @should_wakeup: Wakeup flag when system sleep is not enabled.
+ * @suspend_timer: High-resolution timer used for scheduling delayed runtime
+ *   suspend and autosuspend requests.
+ * @timer_expires: Timer expiration time in nanoseconds monotonic time
+ *   (runtime PM).
+ * @work: Work structure used for queuing up requests into pm_wq (runtime PM).
+ * @wait_queue: Wait queue used if any helper functions need to wait for another
+ *   state change to complete (runtime PM).
+ * @wakeirq: Dedicated wakeup interrupt for the device.
+ * @usage_count: Device runtime PM usage counter.
+ * @child_count: Count of active children of the device (runtime PM).
+ * @disable_depth: Disable counter for runtime PM (runtime PM is enabled when
+ *   this is 0; initial value is 1).
+ * @idle_notification: Set if ->runtime_idle() is being executed.
+ * @request_pending: Set if a work item is queued into pm_wq (runtime PM).
+ * @deferred_resume: Set if ->runtime_resume() should run as soon as
+ *   ->runtime_suspend() completes.
+ * @needs_force_resume: Indicates the device was forced into suspend by
+ *   pm_runtime_force_suspend() and must be resumed by
+ *   pm_runtime_force_resume().
+ * @runtime_auto: User space has allowed the driver to power manage the device
+ *   at runtime via sysfs control attribute; also can be set by
+ *   pm_runtime_allow() or pm_runtime_forbid().
+ * @ignore_children: If set, the value of child_count is ignored for runtime
+ *   suspend and idle decisions.
+ * @no_callbacks: Indicates the device does not use runtime PM callbacks.
+ * @irq_safe: Indicates runtime PM callbacks will be invoked with the spinlock
+ *   held and interrupts disabled.
+ * @use_autosuspend: Indicates the device driver supports delayed runtime
+ *   autosuspend.
+ * @timer_autosuspends: Indicates the runtime PM core should attempt an
+ *   autosuspend rather than a normal suspend when the timer expires.
+ * @memalloc_noio: Indicates memory allocation during runtime PM transitions
+ *   must avoid I/O (GFP_NOIO).
+ * @links_count: Number of device links that require runtime PM coordination.
+ * @request: Type of pending runtime PM request (valid if request_pending is
+ *   set).
+ * @runtime_status: Runtime PM status of the device.
+ * @last_status: Last status captured before disabling runtime PM, or
+ *   %RPM_BLOCKED / %RPM_INVALID.
+ * @runtime_error: Fatal error code returned by a failing callback, blocking
+ *   helpers until cleared.
+ * @autosuspend_delay: Delay time in milliseconds to be used for runtime
+ *   autosuspend.
+ * @last_busy: Timestamp in nanoseconds when pm_runtime_mark_last_busy() was
+ *   last called. Used in calculating inactivity periods for autosuspend.
+ * @active_time: Accumulated time in nanoseconds spent in %RPM_ACTIVE state.
+ * @suspended_time: Accumulated time in nanoseconds spent in %RPM_SUSPENDED
+ *   state.
+ * @accounting_timestamp: Timestamp in nanoseconds of the last runtime PM state
+ *   accounting update.
+ * @subsys_data: Subsystem-specific power management data.
+ * @set_latency_tolerance: Callback for setting latency tolerance.
+ * @qos: Per-device PM Quality of Service (QoS) constraints.
+ * @detach_power_off: Indicates device should be detached from PM domain on
+ *   power off.
+ *
+ * Device power management information stored in the "power" member of struct
+ * device.
+ */
 struct dev_pm_info {
 	pm_message_t		power_state;
 	bool			can_wakeup:1;
diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h
index f925614aebdbc9..14e0e346c610a9 100644
--- a/include/linux/pm_domain.h
+++ b/include/linux/pm_domain.h
@@ -121,6 +121,14 @@ struct dev_pm_domain_list {
  *				powered-off until the ->sync_state() callback is
  *				invoked. This flag informs genpd to allow a
  *				power-off without waiting for ->sync_state().
+ *
+ * GENPD_FLAG_POWER_UNKNOWN:	Use this flag to inform	genpd that its initial
+ *				status for the PM domain is set to powered off,
+ *				which may not correctly reflect the state of the
+ *				HW, as it's unknown. If the PM domain becomes
+ *				powered on during boot, genpd will prevent it
+ *				from being powered off until the ->sync_state
+ *				callback is invoked for it.
  */
 #define GENPD_FLAG_PM_CLK	 (1U << 0)
 #define GENPD_FLAG_IRQ_SAFE	 (1U << 1)
@@ -133,6 +141,7 @@ struct dev_pm_domain_list {
 #define GENPD_FLAG_DEV_NAME_FW	 (1U << 8)
 #define GENPD_FLAG_NO_SYNC_STATE (1U << 9)
 #define GENPD_FLAG_NO_STAY_ON	 (1U << 10)
+#define GENPD_FLAG_POWER_UNKNOWN (1U << 11)
 
 enum gpd_status {
 	GENPD_STATE_ON = 0,	/* PM domain is on */
diff --git a/include/linux/pm_qos.h b/include/linux/pm_qos.h
index 6cea4455f86757..439a9e779d816b 100644
--- a/include/linux/pm_qos.h
+++ b/include/linux/pm_qos.h
@@ -37,6 +37,8 @@ enum pm_qos_flags_status {
 #define PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT	(-1)
 
 #define PM_QOS_FLAG_NO_POWER_OFF	(1 << 0)
+/* latency value applies to system-wide suspend/s2idle */
+#define PM_QOS_FLAG_LATENCY_SYS		(2 << 0)
 
 enum pm_qos_type {
 	PM_QOS_UNITIALIZED,
@@ -217,6 +219,12 @@ static inline s32 dev_pm_qos_raw_resume_latency(struct device *dev)
 		PM_QOS_RESUME_LATENCY_NO_CONSTRAINT :
 		pm_qos_read_value(&dev->power.qos->resume_latency);
 }
+
+static inline s32 dev_pm_qos_raw_flags(struct device *dev)
+{
+	return IS_ERR_OR_NULL(dev->power.qos) ?
+		0 : READ_ONCE(dev->power.qos->flags.effective_flags);
+}
 #else
 static inline enum pm_qos_flags_status __dev_pm_qos_flags(struct device *dev,
 							  s32 mask)
@@ -298,6 +306,7 @@ static inline s32 dev_pm_qos_raw_resume_latency(struct device *dev)
 {
 	return PM_QOS_RESUME_LATENCY_NO_CONSTRAINT;
 }
+static inline s32 dev_pm_qos_raw_flags(struct device *dev) { return 0; }
 #endif
 
 static inline int freq_qos_request_active(struct freq_qos_request *req)
diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h
index 64921b10ac74f4..322e3b17f98713 100644
--- a/include/linux/pm_runtime.h
+++ b/include/linux/pm_runtime.h
@@ -137,13 +137,14 @@ static inline void pm_runtime_put_noidle(struct device *dev)
  * pm_runtime_suspended - Check whether or not a device is runtime-suspended.
  * @dev: Target device.
  *
- * Return %true if runtime PM is enabled for @dev and its runtime PM status is
- * %RPM_SUSPENDED, or %false otherwise.
- *
  * Note that the return value of this function can only be trusted if it is
  * called under the runtime PM lock of @dev or under conditions in which
  * runtime PM cannot be either disabled or enabled for @dev and its runtime PM
  * status cannot change.
+ *
+ * Return:
+ * * %true: @dev has runtime PM enabled and its status is %RPM_SUSPENDED.
+ * * %false: Otherwise.
  */
 static inline bool pm_runtime_suspended(struct device *dev)
 {
@@ -155,13 +156,14 @@ static inline bool pm_runtime_suspended(struct device *dev)
  * pm_runtime_active - Check whether or not a device is runtime-active.
  * @dev: Target device.
  *
- * Return %true if runtime PM is disabled for @dev or its runtime PM status is
- * %RPM_ACTIVE, or %false otherwise.
- *
  * Note that the return value of this function can only be trusted if it is
  * called under the runtime PM lock of @dev or under conditions in which
  * runtime PM cannot be either disabled or enabled for @dev and its runtime PM
  * status cannot change.
+ *
+ * Return:
+ * * %true: Runtime PM is disabled for @dev or its status is %RPM_ACTIVE.
+ * * %false: Otherwise.
  */
 static inline bool pm_runtime_active(struct device *dev)
 {
@@ -173,12 +175,13 @@ static inline bool pm_runtime_active(struct device *dev)
  * pm_runtime_status_suspended - Check if runtime PM status is "suspended".
  * @dev: Target device.
  *
- * Return %true if the runtime PM status of @dev is %RPM_SUSPENDED, or %false
- * otherwise, regardless of whether or not runtime PM has been enabled for @dev.
- *
  * Note that the return value of this function can only be trusted if it is
  * called under the runtime PM lock of @dev or under conditions in which the
  * runtime PM status of @dev cannot change.
+ *
+ * Return:
+ * * %true: Runtime PM status of @dev is %RPM_SUSPENDED.
+ * * %false: Otherwise.
  */
 static inline bool pm_runtime_status_suspended(struct device *dev)
 {
@@ -189,11 +192,13 @@ static inline bool pm_runtime_status_suspended(struct device *dev)
  * pm_runtime_enabled - Check if runtime PM is enabled.
  * @dev: Target device.
  *
- * Return %true if runtime PM is enabled for @dev or %false otherwise.
- *
  * Note that the return value of this function can only be trusted if it is
  * called under the runtime PM lock of @dev or under conditions in which
  * runtime PM cannot be either disabled or enabled for @dev.
+ *
+ * Return:
+ * * %true: Runtime PM is enabled for @dev.
+ * * %false: Otherwise.
  */
 static inline bool pm_runtime_enabled(struct device *dev)
 {
@@ -205,6 +210,10 @@ static inline bool pm_runtime_enabled(struct device *dev)
  * @dev: Target device.
  *
  * Do not call this function outside system suspend/resume code paths.
+ *
+ * Return:
+ * * %true: Runtime PM enabling is blocked for @dev.
+ * * %false: Otherwise.
  */
 static inline bool pm_runtime_blocked(struct device *dev)
 {
@@ -215,8 +224,9 @@ static inline bool pm_runtime_blocked(struct device *dev)
  * pm_runtime_has_no_callbacks - Check if runtime PM callbacks may be present.
  * @dev: Target device.
  *
- * Return %true if @dev is a special device without runtime PM callbacks or
- * %false otherwise.
+ * Return:
+ * * %true: @dev is marked as having no runtime PM callbacks.
+ * * %false: Otherwise.
  */
 static inline bool pm_runtime_has_no_callbacks(struct device *dev)
 {
@@ -239,9 +249,11 @@ static inline void pm_runtime_mark_last_busy(struct device *dev)
  * pm_runtime_is_irq_safe - Check if runtime PM can work in interrupt context.
  * @dev: Target device.
  *
- * Return %true if @dev has been marked as an "IRQ-safe" device (with respect
- * to runtime PM), in which case its runtime PM callabcks can be expected to
- * work correctly when invoked from interrupt handlers.
+ * Return:
+ * * %true: @dev has been marked as an "IRQ-safe" device, in which case its
+ *   runtime PM callbacks can be expected to work correctly from interrupt
+ *   handlers.
+ * * %false: Otherwise.
  */
 static inline bool pm_runtime_is_irq_safe(struct device *dev)
 {
@@ -340,25 +352,25 @@ static inline int pm_runtime_force_resume(struct device *dev) { return -ENXIO; }
 #endif /* CONFIG_PM_SLEEP */
 
 /**
- * pm_runtime_idle - Conditionally set up autosuspend of a device or suspend it.
+ * pm_runtime_idle - Conditionally initiate autosuspend of a device or suspend it.
  * @dev: Target device.
  *
  * Invoke the "idle check" callback of @dev and, depending on its return value,
- * set up autosuspend of @dev or suspend it (depending on whether or not
+ * initiate autosuspend of @dev or suspend it (depending on whether or not
  * autosuspend has been enabled for it).
  *
  * Return:
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change
- *            ongoing or device not in %RPM_ACTIVE state.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -EINPROGRESS: Suspend already in progress.
- * * -ENOSYS: CONFIG_PM not enabled.
- * Other values and conditions for the above values are possible as returned by
- * Runtime PM idle and suspend callbacks.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change
+ *   ongoing or device not in %RPM_ACTIVE state.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-EINPROGRESS: Suspend already in progress.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
+ * * Other values and conditions for the above values are possible as returned
+ *   by Runtime PM idle and suspend callbacks.
  */
 static inline int pm_runtime_idle(struct device *dev)
 {
@@ -370,17 +382,17 @@ static inline int pm_runtime_idle(struct device *dev)
  * @dev: Target device.
  *
  * Return:
- * * 1: Success; device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change
- *            ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -ENOSYS: CONFIG_PM not enabled.
- * Other values and conditions for the above values are possible as returned by
- * Runtime PM suspend callbacks.
+ * * %1: Success; device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change
+ *   ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
+ * * Other values and conditions for the above values are possible as returned
+ *   by Runtime PM suspend callbacks.
  */
 static inline int pm_runtime_suspend(struct device *dev)
 {
@@ -388,26 +400,26 @@ static inline int pm_runtime_suspend(struct device *dev)
 }
 
 /**
- * pm_runtime_autosuspend - Update the last access time and set up autosuspend
+ * pm_runtime_autosuspend - Update the last access time and initiate autosuspend
  * of a device.
  * @dev: Target device.
  *
- * First update the last access time, then set up autosuspend of @dev or suspend
- * it (depending on whether or not autosuspend is enabled for it) without
- * engaging its "idle check" callback.
+ * First update the last access time, then initiate autosuspend of @dev or
+ * suspend it (depending on whether or not autosuspend is enabled for it)
+ * without engaging its "idle check" callback.
  *
  * Return:
- * * 1: Success; device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change
- *            ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -ENOSYS: CONFIG_PM not enabled.
- * Other values and conditions for the above values are possible as returned by
- * Runtime PM suspend callbacks.
+ * * %1: Success; device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change
+ *   ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
+ * * Other values and conditions for the above values are possible as returned
+ *   by Runtime PM suspend callbacks.
  */
 static inline int pm_runtime_autosuspend(struct device *dev)
 {
@@ -418,6 +430,11 @@ static inline int pm_runtime_autosuspend(struct device *dev)
 /**
  * pm_runtime_resume - Resume a device synchronously.
  * @dev: Target device.
+ *
+ * Return:
+ * * %1: Success; @dev is already %RPM_ACTIVE.
+ * * %0: Success.
+ * * Error code on failure.
  */
 static inline int pm_runtime_resume(struct device *dev)
 {
@@ -425,22 +442,22 @@ static inline int pm_runtime_resume(struct device *dev)
 }
 
 /**
- * pm_request_idle - Queue up "idle check" execution for a device.
+ * pm_request_idle - Request an asynchronous idle check for a device.
  * @dev: Target device.
  *
- * Queue up a work item to run an equivalent of pm_runtime_idle() for @dev
- * asynchronously.
+ * Asynchronously request the PM core to evaluate whether @dev can be idled
+ * or suspended, invoking its ->runtime_idle() callback if provided.
  *
  * Return:
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change
- *            ongoing or device not in %RPM_ACTIVE state.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -EINPROGRESS: Suspend already in progress.
- * * -ENOSYS: CONFIG_PM not enabled.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change
+ *   ongoing or device not in %RPM_ACTIVE state.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-EINPROGRESS: Suspend already in progress.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
  */
 static inline int pm_request_idle(struct device *dev)
 {
@@ -448,8 +465,16 @@ static inline int pm_request_idle(struct device *dev)
 }
 
 /**
- * pm_request_resume - Queue up runtime-resume of a device.
+ * pm_request_resume - Request an asynchronous runtime resume for a device.
  * @dev: Target device.
+ *
+ * Asynchronously request the PM core to resume @dev to %RPM_ACTIVE state
+ * without modifying its usage counter.
+ *
+ * Return:
+ * * %1: Success; @dev is already %RPM_ACTIVE.
+ * * %0: Success.
+ * * Error code on failure.
  */
 static inline int pm_request_resume(struct device *dev)
 {
@@ -457,24 +482,23 @@ static inline int pm_request_resume(struct device *dev)
 }
 
 /**
- * pm_request_autosuspend - Update the last access time and queue up autosuspend
- * of a device.
+ * pm_request_autosuspend - Update access time and request delayed suspension.
  * @dev: Target device.
  *
- * Update the last access time of a device and queue up a work item to run an
- * equivalent pm_runtime_autosuspend() for @dev asynchronously.
+ * Update the last access time of @dev and asynchronously request the PM core
+ * to suspend it after the autosuspend delay has elapsed.
  *
  * Return:
- * * 1: Success; device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change
- *            ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -EINPROGRESS: Suspend already in progress.
- * * -ENOSYS: CONFIG_PM not enabled.
+ * * %1: Success; device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change
+ *   ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-EINPROGRESS: Suspend already in progress.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
  */
 static inline int pm_request_autosuspend(struct device *dev)
 {
@@ -483,11 +507,16 @@ static inline int pm_request_autosuspend(struct device *dev)
 }
 
 /**
- * pm_runtime_get - Bump up usage counter and queue up resume of a device.
+ * pm_runtime_get - Increment usage counter and request asynchronous resume.
  * @dev: Target device.
  *
- * Bump up the runtime PM usage counter of @dev and queue up a work item to
- * carry out runtime-resume of it.
+ * Increment the runtime PM usage counter of @dev and, if the device is
+ * currently suspended, asynchronously request the PM core to resume it.
+ *
+ * Return:
+ * * %1: Success; @dev is already %RPM_ACTIVE.
+ * * %0: Success; runtime-resume was queued.
+ * * Error code on failure.
  */
 static inline int pm_runtime_get(struct device *dev)
 {
@@ -501,12 +530,15 @@ static inline int pm_runtime_get(struct device *dev)
  * Bump up the runtime PM usage counter of @dev and carry out runtime-resume of
  * it synchronously.
  *
- * The possible return values of this function are the same as for
- * pm_runtime_resume() and the runtime PM usage counter of @dev remains
- * incremented in all cases, even if it returns an error code.
- * Consider using pm_runtime_resume_and_get() instead of it, especially
- * if its return value is checked by the caller, as this is likely to result
- * in cleaner code.
+ * Note that the runtime PM usage counter of @dev remains incremented in all
+ * cases, even if it returns an error code. Consider using
+ * pm_runtime_resume_and_get() instead, especially if the return value is
+ * checked by the caller, as this is likely to result in cleaner code.
+ *
+ * Return:
+ * * %1: Success; @dev is already %RPM_ACTIVE.
+ * * %0: Success.
+ * * Error code on failure.
  */
 static inline int pm_runtime_get_sync(struct device *dev)
 {
@@ -531,8 +563,11 @@ static inline int pm_runtime_get_active(struct device *dev, int rpmflags)
  * @dev: Target device.
  *
  * Resume @dev synchronously and if that is successful, increment its runtime
- * PM usage counter. Return 0 if the runtime PM usage counter of @dev has been
- * incremented or a negative error code otherwise.
+ * PM usage counter.
+ *
+ * Return:
+ * * %0: Success; @dev is active and its usage counter has been incremented.
+ * * Negative error code on failure; usage counter is unchanged.
  */
 static inline int pm_runtime_resume_and_get(struct device *dev)
 {
@@ -540,11 +575,12 @@ static inline int pm_runtime_resume_and_get(struct device *dev)
 }
 
 /**
- * pm_runtime_put - Drop device usage counter and queue up "idle check" if 0.
+ * pm_runtime_put - Drop device usage counter and request asynchronous idle check.
  * @dev: Target device.
  *
- * Decrement the runtime PM usage counter of @dev and if it turns out to be
- * equal to 0, queue up a work item for @dev like in pm_request_idle().
+ * Decrement the runtime PM usage counter of @dev. If the counter reaches zero
+ * and the device has no active child dependencies, asynchronously request the
+ * PM core to idle or suspend the device.
  */
 static inline void pm_runtime_put(struct device *dev)
 {
@@ -559,16 +595,16 @@ static inline void pm_runtime_put(struct device *dev)
  * equal to 0, queue up a work item for @dev like in pm_request_autosuspend().
  *
  * Return:
- * * 1: Success. Usage counter dropped to zero, but device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
- *            change ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -EINPROGRESS: Suspend already in progress.
- * * -ENOSYS: CONFIG_PM not enabled.
+ * * %1: Success. Usage counter dropped to zero, but device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
+ *   change ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-EINPROGRESS: Suspend already in progress.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
  */
 static inline int __pm_runtime_put_autosuspend(struct device *dev)
 {
@@ -576,25 +612,25 @@ static inline int __pm_runtime_put_autosuspend(struct device *dev)
 }
 
 /**
- * pm_runtime_put_autosuspend - Update the last access time of a device, drop
- * its usage counter and queue autosuspend if the usage counter becomes 0.
+ * pm_runtime_put_autosuspend - Update the last access time, drop usage counter
+ * and request autosuspend.
  * @dev: Target device.
  *
- * Update the last access time of @dev, decrement runtime PM usage counter of
- * @dev and if it turns out to be equal to 0, queue up a work item for @dev like
- * in pm_request_autosuspend().
+ * Update the last access time of @dev and decrement its runtime PM usage
+ * counter. If the counter drops to zero, asynchronously request the PM core to
+ * suspend the device once its autosuspend delay has elapsed.
  *
  * Return:
- * * 1: Success. Usage counter dropped to zero, but device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
- *            change ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -EINPROGRESS: Suspend already in progress.
- * * -ENOSYS: CONFIG_PM not enabled.
+ * * %1: Success. Usage counter dropped to zero, but device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
+ *   change ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-EINPROGRESS: Suspend already in progress.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
  */
 static inline int pm_runtime_put_autosuspend(struct device *dev)
 {
@@ -653,26 +689,28 @@ DEFINE_GUARD_COND(pm_runtime_active_auto, _try_enabled,
  * pm_runtime_put_sync - Drop device usage counter and run "idle check" if 0.
  * @dev: Target device.
  *
- * Decrement the runtime PM usage counter of @dev and if it turns out to be
- * equal to 0, invoke the "idle check" callback of @dev and, depending on its
- * return value, set up autosuspend of @dev or suspend it (depending on whether
- * or not autosuspend has been enabled for it).
+ * Decrement the runtime PM usage counter of @dev. If the counter drops to zero,
+ * synchronously evaluate and trigger idle/suspend handling.
+ *
+ * Note that this does not update the last access time, but it does respect
+ * existing autosuspend timers. If @dev uses autosuspend, consider using
+ * pm_runtime_put_sync_autosuspend() or pm_runtime_put_sync_suspend() instead.
  *
  * The runtime PM usage counter of @dev remains decremented in all cases, even
  * if it returns an error code.
  *
  * Return:
- * * 1: Success. Usage counter dropped to zero, but device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
- *            change ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -ENOSYS: CONFIG_PM not enabled.
- * Other values and conditions for the above values are possible as returned by
- * Runtime PM suspend callbacks.
+ * * %1: Success. Usage counter dropped to zero, but device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
+ *   change ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
+ * * Other values and conditions for the above values are possible as returned
+ *   by Runtime PM suspend callbacks.
  */
 static inline int pm_runtime_put_sync(struct device *dev)
 {
@@ -683,24 +721,28 @@ static inline int pm_runtime_put_sync(struct device *dev)
  * pm_runtime_put_sync_suspend - Drop device usage counter and suspend if 0.
  * @dev: Target device.
  *
- * Decrement the runtime PM usage counter of @dev and if it turns out to be
- * equal to 0, carry out runtime-suspend of @dev synchronously.
+ * Decrement the runtime PM usage counter of @dev. If the counter drops to zero,
+ * suspend the device synchronously.
+ *
+ * This API differs from pm_runtime_put_sync() and
+ * pm_runtime_put_sync_autosuspend() in that it ignores any outstanding
+ * autosuspend delays.
  *
  * The runtime PM usage counter of @dev remains decremented in all cases, even
  * if it returns an error code.
  *
  * Return:
- * * 1: Success. Usage counter dropped to zero, but device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
- *            change ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -ENOSYS: CONFIG_PM not enabled.
- * Other values and conditions for the above values are possible as returned by
- * Runtime PM suspend callbacks.
+ * * %1: Success. Usage counter dropped to zero, but device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
+ *   change ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
+ * * Other values and conditions for the above values are possible as returned
+ *   by Runtime PM suspend callbacks.
  */
 static inline int pm_runtime_put_sync_suspend(struct device *dev)
 {
@@ -712,27 +754,28 @@ static inline int pm_runtime_put_sync_suspend(struct device *dev)
  * drop device usage counter and autosuspend if 0.
  * @dev: Target device.
  *
- * Update the last access time of @dev, decrement the runtime PM usage counter
- * of @dev and if it turns out to be equal to 0, set up autosuspend of @dev or
- * suspend it synchronously (depending on whether or not autosuspend has been
- * enabled for it).
+ * Update the last access time of @dev and decrement its runtime PM usage
+ * counter. If the counter drops to zero, synchronously suspend the device (or
+ * schedule autosuspend if the delay has not elapsed).
+ *
+ * Prefer this API over pm_runtime_put_sync() for devices that use autosuspend.
  *
  * The runtime PM usage counter of @dev remains decremented in all cases, even
  * if it returns an error code.
  *
  * Return:
- * * 1: Success. Usage counter dropped to zero, but device was already suspended.
- * * 0: Success.
- * * -EINVAL: Runtime PM error.
- * * -EACCES: Runtime PM disabled.
- * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
- *            change ongoing.
- * * -EBUSY: Runtime PM child_count non-zero.
- * * -EPERM: Device PM QoS resume latency 0.
- * * -EINPROGRESS: Suspend already in progress.
- * * -ENOSYS: CONFIG_PM not enabled.
- * Other values and conditions for the above values are possible as returned by
- * Runtime PM suspend callbacks.
+ * * %1: Success. Usage counter dropped to zero, but device was already suspended.
+ * * %0: Success.
+ * * %-EINVAL: Runtime PM error.
+ * * %-EACCES: Runtime PM disabled.
+ * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status
+ *   change ongoing.
+ * * %-EBUSY: Runtime PM child_count non-zero.
+ * * %-EPERM: Device PM QoS resume latency 0.
+ * * %-EINPROGRESS: Suspend already in progress.
+ * * %-ENOSYS: %CONFIG_PM not enabled.
+ * * Other values and conditions for the above values are possible as returned
+ *   by Runtime PM suspend callbacks.
  */
 static inline int pm_runtime_put_sync_autosuspend(struct device *dev)
 {
@@ -741,13 +784,22 @@ static inline int pm_runtime_put_sync_autosuspend(struct device *dev)
 }
 
 /**
- * pm_runtime_set_active - Set runtime PM status to "active".
+ * pm_runtime_set_active - Set runtime PM status to "active" and clear errors.
  * @dev: Target device.
  *
- * Set the runtime PM status of @dev to %RPM_ACTIVE and ensure that dependencies
- * of it will be taken into account.
+ * Set the runtime PM status of @dev to %RPM_ACTIVE and ensure that its
+ * dependencies will be taken into account. Also clear the device's error
+ * status (@dev->power.runtime_error).
+ *
+ * It is only valid to call this function if runtime PM is disabled or if
+ * @dev->power.runtime_error is set.
  *
- * It is not valid to call this function for devices with runtime PM enabled.
+ * This will fail if suppliers cannot be resumed, or if the parent is not in
+ * the correct state.
+ *
+ * Return:
+ * * %0: Success.
+ * * Error code on failure.
  */
 static inline int pm_runtime_set_active(struct device *dev)
 {
@@ -755,13 +807,19 @@ static inline int pm_runtime_set_active(struct device *dev)
 }
 
 /**
- * pm_runtime_set_suspended - Set runtime PM status to "suspended".
+ * pm_runtime_set_suspended - Set runtime PM status to "suspended" and clear errors.
  * @dev: Target device.
  *
- * Set the runtime PM status of @dev to %RPM_SUSPENDED and ensure that
- * dependencies of it will be taken into account.
+ * Set the runtime PM status of @dev to %RPM_SUSPENDED and ensure that its
+ * dependencies will be taken into account. Also clear the device's error
+ * status (@dev->power.runtime_error).
  *
- * It is not valid to call this function for devices with runtime PM enabled.
+ * It is only valid to call this function if runtime PM is disabled or if
+ * @dev->power.runtime_error is set.
+ *
+ * Return:
+ * * %0: Success.
+ * * Error code on failure.
  */
 static inline int pm_runtime_set_suspended(struct device *dev)
 {
@@ -777,9 +835,8 @@ static inline int pm_runtime_set_suspended(struct device *dev)
  *
  * If the counter is zero when this function runs and there is a pending runtime
  * resume request for @dev, it will be resumed.  If the counter is still zero at
- * that point, all of the pending runtime PM requests for @dev will be canceled
- * and all runtime PM operations in progress involving it will be waited for to
- * complete.
+ * that point, this function cancels all pending runtime PM requests for @dev
+ * and waits for its runtime PM operations to complete (if any).
  *
  * For each invocation of this function for @dev, there must be a matching
  * pm_runtime_enable() call, so that runtime PM is eventually enabled for it
diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h
index 9a1a0c61361cb4..00767acbc111e3 100644
--- a/include/linux/posix-timers.h
+++ b/include/linux/posix-timers.h
@@ -66,38 +66,6 @@ struct cpu_timer {
 	struct task_struct __rcu	*handling;
 };
 
-static inline bool cpu_timer_enqueue(struct timerqueue_head *head,
-				     struct cpu_timer *ctmr)
-{
-	ctmr->head = head;
-	return timerqueue_add(head, &ctmr->node);
-}
-
-static inline bool cpu_timer_queued(struct cpu_timer *ctmr)
-{
-	return !!ctmr->head;
-}
-
-static inline bool cpu_timer_dequeue(struct cpu_timer *ctmr)
-{
-	if (cpu_timer_queued(ctmr)) {
-		timerqueue_del(ctmr->head, &ctmr->node);
-		ctmr->head = NULL;
-		return true;
-	}
-	return false;
-}
-
-static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr)
-{
-	return ctmr->node.expires;
-}
-
-static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp)
-{
-	ctmr->node.expires = exp;
-}
-
 static inline void posix_cputimers_init(struct posix_cputimers *pct)
 {
 	memset(pct, 0, sizeof(*pct));
@@ -224,14 +192,15 @@ struct k_itimer {
 } ____cacheline_aligned_in_smp;
 
 void run_posix_cpu_timers(void);
-void posix_cpu_timers_exit(struct task_struct *task);
-void posix_cpu_timers_exit_group(struct task_struct *task);
 void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx,
 			   u64 *newval, u64 *oldval);
 
 int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new);
 
 #ifdef CONFIG_POSIX_TIMERS
+void posixtimer_exec(void);
+void posixtimer_exit(bool group_dead);
+
 static inline void posixtimer_putref(struct k_itimer *tmr)
 {
 	if (rcuref_put(&tmr->rcuref))
@@ -259,6 +228,8 @@ static inline bool posixtimer_valid(const struct k_itimer *timer)
 	return !(val & 0x1UL);
 }
 #else  /* CONFIG_POSIX_TIMERS */
+static inline void posixtimer_exec(void) { }
+static inline void posixtimer_exit(bool group_dead) { }
 static inline void posixtimer_sigqueue_getref(struct sigqueue *q) { }
 static inline void posixtimer_sigqueue_putref(struct sigqueue *q) { }
 #endif /* !CONFIG_POSIX_TIMERS */
diff --git a/include/linux/posix_acl.h b/include/linux/posix_acl.h
index 62d497763e25a5..caf500bed9933c 100644
--- a/include/linux/posix_acl.h
+++ b/include/linux/posix_acl.h
@@ -74,20 +74,20 @@ extern int __posix_acl_create(struct posix_acl **, gfp_t, umode_t *);
 extern int __posix_acl_chmod(struct posix_acl **, gfp_t, umode_t);
 
 extern struct posix_acl *get_posix_acl(struct inode *, int);
-int set_posix_acl(struct mnt_idmap *, struct dentry *, int,
+int set_posix_acl(const struct mnt_idmap *, struct dentry *, int,
 		  struct posix_acl *);
 
 struct posix_acl *get_cached_acl_rcu(struct inode *inode, int type);
 struct posix_acl *posix_acl_clone(const struct posix_acl *acl, gfp_t flags);
 
 #ifdef CONFIG_FS_POSIX_ACL
-int posix_acl_chmod(struct mnt_idmap *, struct dentry *, umode_t);
+int posix_acl_chmod(const struct mnt_idmap *, struct dentry *, umode_t);
 extern int posix_acl_create(struct inode *, umode_t *, struct posix_acl **,
 		struct posix_acl **);
-int posix_acl_update_mode(struct mnt_idmap *, struct inode *, umode_t *,
+int posix_acl_update_mode(const struct mnt_idmap *, struct inode *, umode_t *,
 			  struct posix_acl **);
 
-int simple_set_acl(struct mnt_idmap *, struct dentry *,
+int simple_set_acl(const struct mnt_idmap *, struct dentry *,
 		   struct posix_acl *, int);
 extern int simple_acl_create(struct inode *, struct inode *);
 
@@ -96,7 +96,7 @@ void set_cached_acl(struct inode *inode, int type, struct posix_acl *acl);
 void forget_cached_acl(struct inode *inode, int type);
 void forget_all_cached_acls(struct inode *inode);
 int posix_acl_valid(struct user_namespace *, const struct posix_acl *);
-int posix_acl_permission(struct mnt_idmap *, struct inode *,
+int posix_acl_permission(const struct mnt_idmap *, struct inode *,
 			 const struct posix_acl *, int);
 
 static inline void cache_no_acl(struct inode *inode)
@@ -105,16 +105,16 @@ static inline void cache_no_acl(struct inode *inode)
 	inode->i_default_acl = NULL;
 }
 
-int vfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int vfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		const char *acl_name, struct posix_acl *kacl);
-struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap,
+struct posix_acl *vfs_get_acl(const struct mnt_idmap *idmap,
 			      struct dentry *dentry, const char *acl_name);
-int vfs_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+int vfs_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 		   const char *acl_name);
 int posix_acl_listxattr(struct inode *inode, char **buffer,
 			ssize_t *remaining_size);
 #else
-static inline int posix_acl_chmod(struct mnt_idmap *idmap,
+static inline int posix_acl_chmod(const struct mnt_idmap *idmap,
 				  struct dentry *dentry, umode_t mode)
 {
 	return 0;
@@ -141,21 +141,21 @@ static inline void forget_all_cached_acls(struct inode *inode)
 {
 }
 
-static inline int vfs_set_acl(struct mnt_idmap *idmap,
+static inline int vfs_set_acl(const struct mnt_idmap *idmap,
 			      struct dentry *dentry, const char *name,
 			      struct posix_acl *acl)
 {
 	return -EOPNOTSUPP;
 }
 
-static inline struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap,
+static inline struct posix_acl *vfs_get_acl(const struct mnt_idmap *idmap,
 					    struct dentry *dentry,
 					    const char *acl_name)
 {
 	return ERR_PTR(-EOPNOTSUPP);
 }
 
-static inline int vfs_remove_acl(struct mnt_idmap *idmap,
+static inline int vfs_remove_acl(const struct mnt_idmap *idmap,
 				 struct dentry *dentry, const char *acl_name)
 {
 	return -EOPNOTSUPP;
diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h
index d56e1276aafe5e..0b833c4bb5839d 100644
--- a/include/linux/power/bq27xxx_battery.h
+++ b/include/linux/power/bq27xxx_battery.h
@@ -33,6 +33,7 @@ enum bq27xxx_chip {
 	BQ27441,
 	BQ27621,
 	BQ27Z561,
+	BQ27Z746,
 	BQ28Z610,
 	BQ34Z100,
 	BQ78Z100,
diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h
index e749d218933520..bee659427166d3 100644
--- a/include/linux/power_supply.h
+++ b/include/linux/power_supply.h
@@ -104,6 +104,14 @@ enum {
 	POWER_SUPPLY_SCOPE_DEVICE,
 };
 
+enum {
+	POWER_SUPPLY_LOAD_SWITCH_UNKNOWN = 0,
+	POWER_SUPPLY_LOAD_SWITCH_ON,
+	POWER_SUPPLY_LOAD_SWITCH_OFF,
+	POWER_SUPPLY_LOAD_SWITCH_STANDBY,
+	POWER_SUPPLY_LOAD_SWITCH_SHIP,
+};
+
 enum power_supply_property {
 	/* Properties of type `int' */
 	POWER_SUPPLY_PROP_STATUS = 0,
@@ -182,6 +190,7 @@ enum power_supply_property {
 	POWER_SUPPLY_PROP_MANUFACTURE_DAY,
 	POWER_SUPPLY_PROP_INTERNAL_RESISTANCE,
 	POWER_SUPPLY_PROP_STATE_OF_HEALTH,
+	POWER_SUPPLY_PROP_LOAD_SWITCH,
 	/* Properties of type `const char *' */
 	POWER_SUPPLY_PROP_MODEL_NAME,
 	POWER_SUPPLY_PROP_MANUFACTURER,
@@ -259,9 +268,9 @@ struct power_supply_config {
 struct power_supply_desc {
 	const char *name;
 	enum power_supply_type type;
-	u8 charge_behaviours;
 	u32 charge_types;
 	u32 usb_types;
+	u32 load_switches;
 	const enum power_supply_property *properties;
 	size_t num_properties;
 
@@ -295,14 +304,17 @@ struct power_supply_desc {
 	 */
 	int (*init)(struct power_supply *psy);
 
+	/* For APM emulation, think legacy userspace. */
+	int use_for_apm;
+
 	/*
 	 * Set if thermal zone should not be created for this power supply.
 	 * For example for virtual supplies forwarding calls to actual
 	 * sensors or other supplies.
 	 */
 	bool no_thermal;
-	/* For APM emulation, think legacy userspace. */
-	int use_for_apm;
+
+	u8 charge_behaviours;
 };
 
 struct power_supply_ext {
@@ -414,9 +426,9 @@ struct power_supply_vbat_ri_table {
  * @charge_voltage_max_uv: maintenance charging voltage that is usually a bit
  *   lower than the constant_charge_voltage_max_uv. We can apply this settings
  *   charge_current_max_ua until we get back up to this voltage.
- * @safety_timer_minutes: maintenance charging safety timer, with an expiry
- *   time in minutes. We will only use maintenance charging in this setting
- *   for a certain amount of time, then we will first move to the next
+ * @charge_safety_timer_minutes: maintenance charging safety timer, with an
+ *   expiry time in minutes. We will only use maintenance charging in this
+ *   setting for a certain amount of time, then we will first move to the next
  *   maintenance charge current and voltage pair in respective array and wait
  *   for the next safety timer timeout, or, if we reached the last maintencance
  *   charging setting, disable charging until we reach
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 2e689de7b29ab3..06ff44a4b8b6ab 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -498,21 +498,11 @@ DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_n
 
 #ifdef CONFIG_PREEMPT_DYNAMIC
 
-extern bool preempt_model_none(void);
-extern bool preempt_model_voluntary(void);
 extern bool preempt_model_full(void);
 extern bool preempt_model_lazy(void);
 
 #else
 
-static inline bool preempt_model_none(void)
-{
-	return IS_ENABLED(CONFIG_PREEMPT_NONE);
-}
-static inline bool preempt_model_voluntary(void)
-{
-	return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
-}
 static inline bool preempt_model_full(void)
 {
 	return IS_ENABLED(CONFIG_PREEMPT);
@@ -525,6 +515,16 @@ static inline bool preempt_model_lazy(void)
 
 #endif
 
+static inline bool preempt_model_none(void)
+{
+	return IS_ENABLED(CONFIG_PREEMPT_NONE);
+}
+
+static inline bool preempt_model_voluntary(void)
+{
+	return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
+}
+
 static inline bool preempt_model_rt(void)
 {
 	return IS_ENABLED(CONFIG_PREEMPT_RT);
diff --git a/include/linux/property.h b/include/linux/property.h
index 907c790a3f013f..9143fe4f5c0517 100644
--- a/include/linux/property.h
+++ b/include/linux/property.h
@@ -547,6 +547,11 @@ unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode,
 	for (child = fwnode_graph_get_next_endpoint(fwnode, NULL); child;	\
 	     child = fwnode_graph_get_next_endpoint(fwnode, child))
 
+#define fwnode_graph_for_each_endpoint_scoped(fwnode, child)			\
+	for (struct fwnode_handle *child __free(fwnode_handle) =		\
+		fwnode_graph_get_next_endpoint(fwnode, NULL);			\
+	     child; child = fwnode_graph_get_next_endpoint(fwnode, child))
+
 int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
 				struct fwnode_endpoint *endpoint);
 
diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h
index 03a79786df1d7c..fab62228f98170 100644
--- a/include/linux/psp-sev.h
+++ b/include/linux/psp-sev.h
@@ -49,7 +49,7 @@
 
 #define SEV_FW_BLOB_MAX_SIZE	0x4000	/* 16KB */
 
-/**
+/*
  * SEV platform state
  */
 enum sev_state {
@@ -60,7 +60,7 @@ enum sev_state {
 	SEV_STATE_MAX
 };
 
-/**
+/*
  * SEV platform and guest management commands
  */
 enum sev_cmd {
@@ -157,6 +157,7 @@ enum sev_cmd {
  * struct sev_data_init - INIT command parameters
  *
  * @flags: processing flags
+ * @reserved: reserved
  * @tmr_address: system physical address used for SEV-ES
  * @tmr_len: len of tmr_address
  */
@@ -174,6 +175,7 @@ struct sev_data_init {
  * @flags: processing flags
  * @tmr_address: system physical address used for SEV-ES
  * @tmr_len: len of tmr_address
+ * @reserved: reserved
  * @nv_address: system physical address used for PSP NV storage
  * @nv_len: len of nv_address
  */
@@ -201,12 +203,13 @@ struct sev_data_pek_csr {
 } __packed;
 
 /**
- * struct sev_data_cert_import - PEK_CERT_IMPORT command parameters
+ * struct sev_data_pek_cert_import - PEK_CERT_IMPORT command parameters
  *
- * @pek_address: PEK certificate chain
- * @pek_len: len of PEK certificate
- * @oca_address: OCA certificate chain
- * @oca_len: len of OCA certificate
+ * @pek_cert_address: PEK certificate chain
+ * @pek_cert_len: len of PEK certificate
+ * @reserved: reserved
+ * @oca_cert_address: OCA certificate chain
+ * @oca_cert_len: len of OCA certificate
  */
 struct sev_data_pek_cert_import {
 	u64 pek_cert_address;			/* In */
@@ -240,8 +243,9 @@ struct sev_data_get_id {
 /**
  * struct sev_data_pdh_cert_export - PDH_CERT_EXPORT command parameters
  *
- * @pdh_address: PDH certificate address
- * @pdh_len: len of PDH certificate
+ * @pdh_cert_address: PDH certificate address
+ * @pdh_cert_len: len of PDH certificate
+ * @reserved: reserved
  * @cert_chain_address: PDH certificate chain
  * @cert_chain_len: len of PDH certificate chain
  */
@@ -304,6 +308,7 @@ struct sev_data_guest_status {
  * @policy: guest launch policy
  * @dh_cert_address: physical address of DH certificate blob
  * @dh_cert_len: len of DH certificate blob
+ * @reserved: reserved
  * @session_address: physical address of session parameters
  * @session_len: len of session parameters
  */
@@ -321,6 +326,7 @@ struct sev_data_launch_start {
  * struct sev_data_launch_update_data - LAUNCH_UPDATE_DATA command parameter
  *
  * @handle: handle of the VM to update
+ * @reserved: reserved
  * @len: len of memory to be encrypted
  * @address: physical address of memory region to encrypt
  */
@@ -335,6 +341,7 @@ struct sev_data_launch_update_data {
  * struct sev_data_launch_update_vmsa - LAUNCH_UPDATE_VMSA command
  *
  * @handle: handle of the VM
+ * @reserved: reserved
  * @address: physical address of memory region to encrypt
  * @len: len of memory region to encrypt
  */
@@ -349,6 +356,7 @@ struct sev_data_launch_update_vmsa {
  * struct sev_data_launch_measure - LAUNCH_MEASURE command parameters
  *
  * @handle: handle of the VM to process
+ * @reserved: reserved
  * @address: physical address containing the measurement blob
  * @len: len of measurement blob
  */
@@ -363,10 +371,13 @@ struct sev_data_launch_measure {
  * struct sev_data_launch_secret - LAUNCH_SECRET command parameters
  *
  * @handle: handle of the VM to process
+ * @reserved1: reserved
  * @hdr_address: physical address containing the packet header
  * @hdr_len: len of packet header
+ * @reserved2: reserved
  * @guest_address: system physical address of guest memory region
  * @guest_len: len of guest_paddr
+ * @reserved3: reserved
  * @trans_address: physical address of transport memory buffer
  * @trans_len: len of transport memory buffer
  */
@@ -399,10 +410,13 @@ struct sev_data_launch_finish {
  * @policy: policy information for the VM
  * @pdh_cert_address: physical address containing PDH certificate
  * @pdh_cert_len: len of PDH certificate
+ * @reserved1: reserved
  * @plat_certs_address: physical address containing platform certificate
  * @plat_certs_len: len of platform certificate
+ * @reserved2: reserved
  * @amd_certs_address: physical address containing AMD certificate
  * @amd_certs_len: len of AMD certificate
+ * @reserved3: reserved
  * @session_address: physical address containing Session data
  * @session_len: len of session data
  */
@@ -423,13 +437,16 @@ struct sev_data_send_start {
 } __packed;
 
 /**
- * struct sev_data_send_update - SEND_UPDATE_DATA command
+ * struct sev_data_send_update_data - SEND_UPDATE_DATA command
  *
  * @handle: handle of the VM to process
+ * @reserved1: reserved
  * @hdr_address: physical address containing packet header
  * @hdr_len: len of packet header
+ * @reserved2: reserved
  * @guest_address: physical address of guest memory region to send
  * @guest_len: len of guest memory region to send
+ * @reserved3: reserved
  * @trans_address: physical address of host memory region
  * @trans_len: len of host memory region
  */
@@ -447,13 +464,15 @@ struct sev_data_send_update_data {
 } __packed;
 
 /**
- * struct sev_data_send_update - SEND_UPDATE_VMSA command
+ * struct sev_data_send_update_vmsa - SEND_UPDATE_VMSA command
  *
  * @handle: handle of the VM to process
  * @hdr_address: physical address containing packet header
  * @hdr_len: len of packet header
+ * @reserved2: reserved
  * @guest_address: physical address of guest memory region to send
  * @guest_len: len of guest memory region to send
+ * @reserved3: reserved
  * @trans_address: physical address of host memory region
  * @trans_len: len of host memory region
  */
@@ -491,8 +510,10 @@ struct sev_data_send_cancel {
  * struct sev_data_receive_start - RECEIVE_START command parameters
  *
  * @handle: handle of the VM to perform receive operation
+ * @policy: policy information for the VM
  * @pdh_cert_address: system physical address containing PDH certificate blob
  * @pdh_cert_len: len of PDH certificate blob
+ * @reserved1: reserved
  * @session_address: system physical address containing session blob
  * @session_len: len of session blob
  */
@@ -510,10 +531,13 @@ struct sev_data_receive_start {
  * struct sev_data_receive_update_data - RECEIVE_UPDATE_DATA command parameters
  *
  * @handle: handle of the VM to update
+ * @reserved1: reserved
  * @hdr_address: physical address containing packet header blob
  * @hdr_len: len of packet header
+ * @reserved2: reserved
  * @guest_address: system physical address of guest memory region
  * @guest_len: len of guest memory region
+ * @reserved3: reserved
  * @trans_address: system physical address of transport buffer
  * @trans_len: len of transport buffer
  */
@@ -534,10 +558,13 @@ struct sev_data_receive_update_data {
  * struct sev_data_receive_update_vmsa - RECEIVE_UPDATE_VMSA command parameters
  *
  * @handle: handle of the VM to update
+ * @reserved1: reserved
  * @hdr_address: physical address containing packet header blob
  * @hdr_len: len of packet header
+ * @reserved2: reserved
  * @guest_address: system physical address of guest memory region
  * @guest_len: len of guest memory region
+ * @reserved3: reserved
  * @trans_address: system physical address of transport buffer
  * @trans_len: len of transport buffer
  */
@@ -567,6 +594,7 @@ struct sev_data_receive_finish {
  * struct sev_data_dbg - DBG_ENCRYPT/DBG_DECRYPT command parameters
  *
  * @handle: handle of the VM to perform debug operation
+ * @reserved: reserved
  * @src_addr: source address of data to operate on
  * @dst_addr: destination address of data to operate on
  * @len: len of data to operate on
@@ -583,6 +611,7 @@ struct sev_data_dbg {
  * struct sev_data_attestation_report - SEV_ATTESTATION_REPORT command parameters
  *
  * @handle: handle of the VM
+ * @reserved: reserved
  * @mnonce: a random nonce that will be included in the report.
  * @address: physical address where the report will be copied.
  * @len: length of the physical buffer.
@@ -781,9 +810,14 @@ struct sev_data_snp_guest_request {
  *
  * @init_rmp: indicate that the RMP should be initialized.
  * @list_paddr_en: indicate that list_paddr is valid
+ * @rapl_dis: whether RAPL is disabled
+ * @ciphertext_hiding_en: whether ciphertext hiding is enabled
+ * @tio_en: Indicates that SNP_INIT_EX initialized the RMP for SEV-TIO
  * @rsvd: reserved
  * @rsvd1: reserved
  * @list_paddr: system physical address of range list
+ * @max_snp_asid: When non-zero, enable ciphertext hiding and specify the
+ *  maximum ASID that can be used for an SEV-SNP guest.
  * @rsvd2: reserved
  */
 struct sev_data_snp_init_ex {
@@ -841,7 +875,7 @@ struct sev_data_snp_shutdown_ex {
 } __packed;
 
 /**
- * struct sev_platform_init_args
+ * struct sev_platform_init_args - parameters for sev_platform_init()
  *
  * @error: SEV firmware error code
  * @probe: True if this is being called as part of CCP module probe, which
@@ -879,12 +913,12 @@ struct sev_data_snp_feature_info {
 } __packed;
 
 /**
- * struct feature_info - FEATURE_INFO structure
+ * struct snp_feature_info - FEATURE_INFO structure
  *
  * @eax: output of SNP_FEATURE_INFO command
  * @ebx: output of SNP_FEATURE_INFO command
  * @ecx: output of SNP_FEATURE_INFO command
- * #edx: output of SNP_FEATURE_INFO command
+ * @edx: output of SNP_FEATURE_INFO command
  */
 struct snp_feature_info {
 	u32 eax;
@@ -954,7 +988,7 @@ struct sev_data_snp_verify_mitigation_dst {
 } __packed;
 
 /**
- * struct sev_snp_tcb_version_genoa_milan
+ * struct sev_snp_tcb_version_genoa_milan - v1 SVN payload
  *
  * @boot_loader: SVN of PSP bootloader
  * @tee: SVN of PSP operating system
@@ -971,7 +1005,7 @@ struct sev_snp_tcb_version_genoa_milan {
 };
 
 /**
- * struct sev_snp_tcb_version_turin
+ * struct sev_snp_tcb_version_turin - v2 SVN payload
  *
  * @fmc: SVN of FMC firmware
  * @boot_loader: SVN of PSP bootloader
@@ -1037,9 +1071,9 @@ int sev_platform_status(struct sev_user_data_status *status, int *error);
  * behalf of userspace. The caller must pass a valid SEV file descriptor
  * so that we know that it has access to SEV device.
  *
- * @filep - SEV device file pointer
- * @cmd - command to issue
- * @data - command buffer
+ * @filep: SEV device file pointer
+ * @id: command to issue
+ * @data: command buffer
  * @error: SEV command return code
  *
  * Returns:
@@ -1056,8 +1090,8 @@ int sev_issue_cmd_external_user(struct file *filep, unsigned int id,
 /**
  * sev_guest_deactivate - perform SEV DEACTIVATE command
  *
- * @deactivate: sev_data_deactivate structure to be processed
- * @sev_ret: sev command return code
+ * @data: sev_data_deactivate structure to be processed
+ * @error: sev command return code
  *
  * Returns:
  * 0 if the sev successfully processed the command
@@ -1071,8 +1105,8 @@ int sev_guest_deactivate(struct sev_data_deactivate *data, int *error);
 /**
  * sev_guest_activate - perform SEV ACTIVATE command
  *
- * @activate: sev_data_activate structure to be processed
- * @sev_ret: sev command return code
+ * @data: sev_data_activate structure to be processed
+ * @error: sev command return code
  *
  * Returns:
  * 0 if the sev successfully processed the command
@@ -1086,7 +1120,7 @@ int sev_guest_activate(struct sev_data_activate *data, int *error);
 /**
  * sev_guest_df_flush - perform SEV DF_FLUSH command
  *
- * @sev_ret: sev command return code
+ * @error: sev command return code
  *
  * Returns:
  * 0 if the sev successfully processed the command
@@ -1100,8 +1134,8 @@ int sev_guest_df_flush(int *error);
 /**
  * sev_guest_decommission - perform SEV DECOMMISSION command
  *
- * @decommission: sev_data_decommission structure to be processed
- * @sev_ret: sev command return code
+ * @data: sev_data_decommission structure to be processed
+ * @error: sev command return code
  *
  * Returns:
  * 0 if the sev successfully processed the command
diff --git a/include/linux/ptdump.h b/include/linux/ptdump.h
index 240bd3bff18dd2..af18d1459b2f49 100644
--- a/include/linux/ptdump.h
+++ b/include/linux/ptdump.h
@@ -31,9 +31,9 @@ bool ptdump_walk_pgd_level_core(struct seq_file *m,
 void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd);
 bool ptdump_check_wx(void);
 
-static inline void debug_checkwx(void)
+static inline void pgtable_checkwx(void)
 {
-	if (IS_ENABLED(CONFIG_DEBUG_WX))
+	if (IS_ENABLED(CONFIG_CHECK_WX))
 		ptdump_check_wx();
 }
 
diff --git a/include/linux/pwrseq/consumer.h b/include/linux/pwrseq/consumer.h
index 16fad5f3e3ab56..b69117ad3fa72a 100644
--- a/include/linux/pwrseq/consumer.h
+++ b/include/linux/pwrseq/consumer.h
@@ -25,6 +25,8 @@ int pwrseq_disable(struct pwrseq_desc *desc);
 
 struct device *pwrseq_to_device(struct pwrseq_desc *desc);
 
+bool pwrseq_is_controllable(struct pwrseq_desc *desc);
+
 #else /* CONFIG_POWER_SEQUENCING */
 
 static inline struct pwrseq_desc * __must_check
@@ -58,6 +60,11 @@ static inline struct device *pwrseq_to_device(struct pwrseq_desc *desc)
 	return NULL;
 }
 
+static inline bool pwrseq_is_controllable(struct pwrseq_desc *desc)
+{
+	return false;
+}
+
 #endif /* CONFIG_POWER_SEQUENCING */
 
 #endif /* __POWER_SEQUENCING_CONSUMER_H__ */
diff --git a/include/linux/pwrseq/provider.h b/include/linux/pwrseq/provider.h
index 33b3d2c2e39dec..7285ad94a22140 100644
--- a/include/linux/pwrseq/provider.h
+++ b/include/linux/pwrseq/provider.h
@@ -6,12 +6,15 @@
 #ifndef __POWER_SEQUENCING_PROVIDER_H__
 #define __POWER_SEQUENCING_PROVIDER_H__
 
+#include <linux/types.h>
+
 struct device;
 struct module;
 struct pwrseq_device;
 
 typedef int (*pwrseq_power_state_func)(struct pwrseq_device *);
 typedef int (*pwrseq_match_func)(struct pwrseq_device *, struct device *);
+typedef bool (*pwrseq_is_controllable_func)(struct pwrseq_device *);
 
 #define PWRSEQ_NO_MATCH 0
 #define PWRSEQ_MATCH_OK 1
@@ -43,11 +46,16 @@ struct pwrseq_unit_data {
  *               the state lock has been released. It's useful for implementing
  *               boot-up delays without blocking other users from powering up
  *               using the same power sequencer.
+ * @is_controllable: Optional callback checking whether enabling/disabling this
+ *                   target actually controls power (for example when the
+ *                   controlling GPIO is wired up). If not provided, the
+ *                   target's power is assumed to be always controllable.
  */
 struct pwrseq_target_data {
 	const char *name;
 	const struct pwrseq_unit_data *unit;
 	pwrseq_power_state_func post_enable;
+	pwrseq_is_controllable_func is_controllable;
 };
 
 /**
diff --git a/include/linux/quotaops.h b/include/linux/quotaops.h
index f9c0f9d7c9d93a..0c64ca674e778a 100644
--- a/include/linux/quotaops.h
+++ b/include/linux/quotaops.h
@@ -20,7 +20,7 @@ static inline struct quota_info *sb_dqopt(struct super_block *sb)
 }
 
 /* i_rwsem must being held */
-static inline bool is_quota_modification(struct mnt_idmap *idmap,
+static inline bool is_quota_modification(const struct mnt_idmap *idmap,
 					 struct inode *inode, struct iattr *ia)
 {
 	return ((ia->ia_valid & ATTR_SIZE) ||
@@ -109,7 +109,7 @@ int dquot_set_dqblk(struct super_block *sb, struct kqid id,
 		struct qc_dqblk *di);
 
 int __dquot_transfer(struct inode *inode, struct dquot **transfer_to);
-int dquot_transfer(struct mnt_idmap *idmap, struct inode *inode,
+int dquot_transfer(const struct mnt_idmap *idmap, struct inode *inode,
 		   struct iattr *iattr);
 
 static inline struct mem_dqinfo *sb_dqinfo(struct super_block *sb, int type)
@@ -229,7 +229,7 @@ static inline void dquot_free_inode(struct inode *inode)
 {
 }
 
-static inline int dquot_transfer(struct mnt_idmap *idmap,
+static inline int dquot_transfer(const struct mnt_idmap *idmap,
 				 struct inode *inode, struct iattr *iattr)
 {
 	return 0;
diff --git a/include/linux/rcuref.h b/include/linux/rcuref.h
index 2fb2af6d982497..01fee161b67c3e 100644
--- a/include/linux/rcuref.h
+++ b/include/linux/rcuref.h
@@ -34,7 +34,7 @@ static inline void rcuref_init(rcuref_t *ref, unsigned int cnt)
  * indicate that it is safe to schedule the object, protected by this reference
  * counter, for deconstruction.
  * If you want to know if the reference counter has been marked DEAD (as
- * signaled by rcuref_put()) please use rcuread_is_dead().
+ * signaled by rcuref_put()) please use rcuref_is_dead().
  */
 static inline unsigned int rcuref_read(rcuref_t *ref)
 {
diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h
index cc6ce709ec8630..55cc519103ed8c 100644
--- a/include/linux/regulator/driver.h
+++ b/include/linux/regulator/driver.h
@@ -362,7 +362,7 @@ enum regulator_type {
  * @off_on_delay: guard time (in uS), before re-enabling a regulator
  *
  * @poll_enabled_time: The polling interval (in uS) to use while checking that
- *                     the regulator was actually enabled. Max upto enable_time.
+ *                     the regulator was actually enabled. Max up to enable_time.
  *
  * @of_map_mode: Maps a hardware mode defined in a DeviceTree to a standard mode
  */
diff --git a/include/linux/regulator/pca9450.h b/include/linux/regulator/pca9450.h
index 0df8b3c48082f9..bf94df5fafe30b 100644
--- a/include/linux/regulator/pca9450.h
+++ b/include/linux/regulator/pca9450.h
@@ -210,9 +210,11 @@ enum {
 #define LDO5L_EN_MASK			0xC0
 #define LDO5LOUT_MASK			0x0F
 
-#define LDO5H_EN_MASK			0xC0
 #define LDO5HOUT_MASK			0x0F
 
+/* LDO ENMODE value: ON in RUN, OFF while PMIC_STBY_REQ is asserted */
+#define LDO_ENMODE_ONREQ_STBYREQ	0x80
+
 /* PCA9450_REG_IRQ bits */
 #define IRQ_PWRON			0x80
 #define IRQ_WDOGB			0x40
diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h
index a44368737b39a5..ff0ad1e12c0157 100644
--- a/include/linux/remoteproc.h
+++ b/include/linux/remoteproc.h
@@ -15,6 +15,7 @@
 #include <linux/cdev.h>
 #include <linux/completion.h>
 #include <linux/idr.h>
+#include <linux/srcu.h>
 #include <linux/of.h>
 #include <linux/rsc_table.h>
 
@@ -228,6 +229,7 @@ enum rproc_features {
  * @mappings: list of iommu mappings we initiated, needed on shutdown
  * @bootaddr: address of first instruction to boot rproc with (optional)
  * @rvdevs: list of remote virtio devices
+ * @vq_srcu: SRCU domain for the virtqueue callbacks of @rvdevs
  * @subdevs: list of subdevices, to following the running state
  * @notifyids: idr for dynamically assigning rproc-wide unique notify ids
  * @index: index of this rproc device
@@ -274,6 +276,7 @@ struct rproc {
 	struct list_head mappings;
 	u64 bootaddr;
 	struct list_head rvdevs;
+	struct srcu_struct vq_srcu;
 	struct list_head subdevs;
 	struct idr notifyids;
 	int index;
diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
index dd09c2ce9a0f2d..10dfdca7f4bf5e 100644
--- a/include/linux/resctrl.h
+++ b/include/linux/resctrl.h
@@ -505,6 +505,25 @@ bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r);
  */
 int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable);
 
+/**
+ * resctrl_arch_preconvert_bw() - Prepare bandwidth control value for arch use.
+ * @r:		Resource whose schema was written.
+ * @val:	Bandwidth control value written to the schemata file by userspace.
+ *
+ * Convert the user provided bandwidth control value to an appropriate form for
+ * consumption by the hardware driver for resource @r. Converted value is stored
+ * in rdt_ctrl_domain::staged_config[] for later consumption by
+ * resctrl_arch_update_domains(). Is not called when MBA software controller is
+ * enabled.
+ *
+ * Architectures for which this pre-conversion hook is not useful should supply
+ * an implementation of this function that just returns @val unmodified.
+ *
+ * Return:
+ * The converted value.
+ */
+u32 resctrl_arch_preconvert_bw(const struct rdt_resource *r, u32 val);
+
 /*
  * Update the ctrl_val and apply this config right now.
  * Must be called on one of the domain's CPUs.
diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h
index 57c11ec9dc645c..3576b8f08aff1f 100644
--- a/include/linux/rhashtable-types.h
+++ b/include/linux/rhashtable-types.h
@@ -70,6 +70,12 @@ struct rhashtable_params {
 	rht_obj_cmpfn_t		obj_cmpfn;
 };
 
+struct rhashtable_lockdep_keys {
+	struct lock_class_key lock_key;
+	struct lock_class_key mutex_key;
+	struct lock_class_key bucket_key;
+};
+
 /**
  * struct rhashtable - Hash table handle
  * @tbl: Bucket table
@@ -97,6 +103,9 @@ struct rhashtable {
 #ifdef CONFIG_MEM_ALLOC_PROFILING
 	struct alloc_tag		*alloc_tag;
 #endif
+#ifdef CONFIG_LOCKDEP
+	struct lock_class_key		*lockdep_key;
+#endif
 };
 
 /**
@@ -138,24 +147,77 @@ struct rhashtable_iter {
 
 int __rhashtable_init_noprof(struct rhashtable *ht,
 		    const struct rhashtable_params *params,
-		    struct lock_class_key *key);
+		    struct rhashtable_lockdep_keys *keys);
 #define rhashtable_init_noprof(ht, params)				\
 ({									\
-	static struct lock_class_key __key;				\
+	static struct rhashtable_lockdep_keys __keys;			\
 									\
-	__rhashtable_init_noprof(ht, params, &__key);			\
+	__rhashtable_init_noprof(ht, params, &__keys);			\
 })
+
+/**
+ * rhashtable_init - initialize a new hash table
+ * @ht:		hash table to be initialized
+ * @params:	configuration parameters
+ *
+ * Initializes a new hash table based on the provided configuration
+ * parameters. A table can be configured either with a variable or
+ * fixed length key:
+ *
+ * Configuration Example 1: Fixed length keys
+ * struct test_obj {
+ *	int			key;
+ *	void *			my_member;
+ *	struct rhash_head	node;
+ * };
+ *
+ * struct rhashtable_params params = {
+ *	.head_offset = offsetof(struct test_obj, node),
+ *	.key_offset = offsetof(struct test_obj, key),
+ *	.key_len = sizeof(int),
+ *	.hashfn = jhash,
+ * };
+ *
+ * Configuration Example 2: Variable length keys
+ * struct test_obj {
+ *	[...]
+ *	struct rhash_head	node;
+ * };
+ *
+ * u32 my_hash_fn(const void *data, u32 len, u32 seed)
+ * {
+ *	struct test_obj *obj = data;
+ *
+ *	return [... hash ...];
+ * }
+ *
+ * struct rhashtable_params params = {
+ *	.head_offset = offsetof(struct test_obj, node),
+ *	.hashfn = jhash,
+ *	.obj_hashfn = my_hash_fn,
+ * };
+ */
 #define rhashtable_init(...)	alloc_hooks(rhashtable_init_noprof(__VA_ARGS__))
 
 int __rhltable_init_noprof(struct rhltable *hlt,
 		  const struct rhashtable_params *params,
-		  struct lock_class_key *key);
+		  struct rhashtable_lockdep_keys *keys);
 #define rhltable_init_noprof(hlt, params)				\
 ({									\
-	static struct lock_class_key __key;				\
+	static struct rhashtable_lockdep_keys __keys;			\
 									\
-	__rhltable_init_noprof(hlt, params, &__key);			\
+	__rhltable_init_noprof(hlt, params, &__keys);			\
 })
+
+/**
+ * rhltable_init - initialize a new hash list table
+ * @hlt:	hash list table to be initialized
+ * @params:	configuration parameters
+ *
+ * Initializes a new hash list table.
+ *
+ * See documentation for rhashtable_init.
+ */
 #define rhltable_init(...)	alloc_hooks(rhltable_init_noprof(__VA_ARGS__))
 
 #endif /* _LINUX_RHASHTABLE_TYPES_H */
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index 57a2a29bef0e8f..ec853c1b9af3a9 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -319,18 +319,6 @@ static inline struct rhash_lock_head __rcu **rht_bucket_insert(
  * When we write to a bucket without unlocking, we use rht_assign_locked().
  */
 
-static inline unsigned long rht_lock(struct bucket_table *tbl,
-				     struct rhash_lock_head __rcu **bkt)
-	__acquires(__bitlock(0, bkt))
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-	bit_spin_lock(0, (unsigned long *)bkt);
-	lock_map_acquire(&tbl->dep_map);
-	return flags;
-}
-
 static inline unsigned long rht_lock_nested(struct bucket_table *tbl,
 					struct rhash_lock_head __rcu **bucket,
 					unsigned int subclass)
@@ -344,6 +332,13 @@ static inline unsigned long rht_lock_nested(struct bucket_table *tbl,
 	return flags;
 }
 
+static inline unsigned long rht_lock(struct bucket_table *tbl,
+				     struct rhash_lock_head __rcu **bkt)
+	__acquires(__bitlock(0, bkt))
+{
+	return rht_lock_nested(tbl, bkt, 0);
+}
+
 static inline void rht_unlock(struct bucket_table *tbl,
 			      struct rhash_lock_head __rcu **bkt,
 			      unsigned long flags)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 0b332770abeed5..74cca0e3c72641 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -888,7 +888,7 @@ struct page_vma_mapped_walk {
 static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
 {
 	/* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
-	if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
+	if (pvmw->pte && !vma_is_hugetlb(pvmw->vma))
 		pte_unmap(pvmw->pte);
 	if (pvmw->ptl)
 		spin_unlock(pvmw->ptl);
diff --git a/include/linux/rpmsg/virtio_rpmsg.h b/include/linux/rpmsg/virtio_rpmsg.h
new file mode 100644
index 00000000000000..0872cbca4593ee
--- /dev/null
+++ b/include/linux/rpmsg/virtio_rpmsg.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Pinecone Inc. 2019
+ * Copyright (C) Xiang Xiao <xiaoxiang@pinecone.net>
+ * Copyright (C) Advanced Micro Devices, Inc. 2026
+ */
+
+#ifndef _LINUX_VIRTIO_RPMSG_H
+#define _LINUX_VIRTIO_RPMSG_H
+
+#include <linux/types.h>
+#include <linux/virtio_types.h>
+
+/* The feature bitmap for virtio rpmsg */
+#define VIRTIO_RPMSG_F_NS	0 /* RP supports name service notifications */
+#define VIRTIO_RPMSG_F_BUFSZ	1 /* RP advertises RX/TX buffer sizes from config space */
+
+/* Version of struct virtio_rpmsg_config understood by this driver */
+#define RPMSG_VDEV_CONFIG_V1	1
+
+/**
+ * struct virtio_rpmsg_config - config space for rpmsg virtio device
+ *
+ * @version:	version of this structure, currently %RPMSG_VDEV_CONFIG_V1.
+ * @size:	size of this structure in bytes.
+ * @txbuf_size:	Tx buf size from remote's view. For Linux this is rx buf size.
+ * @rxbuf_size:	Rx buf size from remote's view. For Linux this is tx buf size.
+ *
+ * This is the configuration structure shared by the device and the driver,
+ * read when %VIRTIO_RPMSG_F_BUFSZ is negotiated. The fields are laid out so
+ * the structure is naturally 32-bit aligned.
+ */
+struct virtio_rpmsg_config {
+	u8 version;
+	__virtio16 size;
+	/* The tx/rx individual buffer size (if VIRTIO_RPMSG_F_BUFSZ) */
+	__virtio32 txbuf_size;
+	__virtio32 rxbuf_size;
+} __packed;
+
+#endif /* _LINUX_VIRTIO_RPMSG_H */
diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h
index 95729339e7a548..a54ec40d095c04 100644
--- a/include/linux/rtnetlink.h
+++ b/include/linux/rtnetlink.h
@@ -186,7 +186,12 @@ void net_dec_ingress_queue(void);
 #ifdef CONFIG_NET_EGRESS
 void net_inc_egress_queue(void);
 void net_dec_egress_queue(void);
-void netdev_xmit_skip_txqueue(bool skip);
+bool netdev_xmit_skip_txqueue(bool skip);
+#else
+static inline bool netdev_xmit_skip_txqueue(bool skip)
+{
+	return false;
+}
 #endif
 
 void rtnetlink_init(void);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 705970d07614f7..d828f0fb189657 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -554,6 +554,7 @@ struct sched_statistics {
 	u64				nr_failed_migrations_running;
 	u64				nr_failed_migrations_hot;
 	u64				nr_forced_migrations;
+	u64				nr_migrations_cpu_non_preferred;
 
 	u64				nr_wakeups;
 	u64				nr_wakeups_sync;
@@ -1172,8 +1173,12 @@ struct task_struct {
 	/* Objective and real subjective task credentials (COW): */
 	const struct cred __rcu		*real_cred;
 
-	/* Effective (overridable) subjective task credentials (COW): */
-	const struct cred __rcu		*cred;
+	/*
+	 * Effective (overridable) subjective task credentials (COW).
+	 * Only accessible for the current task and during task creation/freeing.
+	 * This pointer is not managed by RCU!
+	 */
+	const struct cred		*cred;
 
 #ifdef CONFIG_KEYS
 	/* Cached requested key. */
@@ -1433,6 +1438,7 @@ struct task_struct {
 
 #ifdef CONFIG_SCHED_CACHE
 	struct callback_head		cache_work;
+	struct sched_cache_group __rcu	*sched_cache_grp;
 	int				preferred_llc;
 	/* 1: task was enqueued to its preferred LLC, 0 otherwise */
 	int				pref_llc_queued;
@@ -1808,15 +1814,15 @@ extern struct pid __rcu *cad_pid;
 #define PF_USED_MATH		0x00002000	/* If unset the fpu must be initialized before use */
 #define PF_USER_WORKER		0x00004000	/* Kernel thread cloned from userspace thread */
 #define PF_NOFREEZE		0x00008000	/* This thread should not be frozen */
-#define PF_KCOMPACTD		0x00010000	/* I am kcompactd */
-#define PF_KSWAPD		0x00020000	/* I am kswapd */
+#define PF__HOLE__00010000	0x00010000
+#define PF__HOLE__00020000	0x00020000
 #define PF_MEMALLOC_NOFS	0x00040000	/* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */
 #define PF_MEMALLOC_NOIO	0x00080000	/* All allocations inherit GFP_NOIO. See memalloc_noio_save() */
 #define PF_LOCAL_THROTTLE	0x00100000	/* Throttle writes only against the bdi I write to,
 						 * I am cleaning dirty pages from some other bdi. */
 #define PF_KTHREAD		0x00200000	/* I am a kernel thread */
 #define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */
-#define PF__HOLE__00800000	0x00800000
+#define PF_NO_NOTIFY_SIGNAL	0x00800000	/* see no_notify_signal_save() */
 #define PF__HOLE__01000000	0x01000000
 #define PF__HOLE__02000000	0x02000000
 #define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_mask */
@@ -2134,44 +2140,19 @@ static inline void set_need_resched_current(void)
  * value indicates whether a reschedule was done in fact.
  * cond_resched_lock() will drop the spinlock before scheduling,
  */
-#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
+#if !defined(CONFIG_PREEMPTION)
 extern int __cond_resched(void);
 
-#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-
-DECLARE_STATIC_CALL(cond_resched, __cond_resched);
-
-static __always_inline int _cond_resched(void)
-{
-	return static_call_mod(cond_resched)();
-}
-
-#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-
-extern int dynamic_cond_resched(void);
-
-static __always_inline int _cond_resched(void)
-{
-	return dynamic_cond_resched();
-}
-
-#else /* !CONFIG_PREEMPTION */
-
 static inline int _cond_resched(void)
 {
 	return __cond_resched();
 }
-
-#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */
-
-#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */
-
+#else
 static inline int _cond_resched(void)
 {
 	return 0;
 }
-
-#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */
+#endif
 
 #define cond_resched() ({			\
 	__might_resched(__FILE__, __LINE__, 0);	\
@@ -2405,7 +2386,7 @@ struct sched_cache_time {
 	unsigned long epoch;
 };
 
-struct sched_cache_stat {
+struct sched_cache_group {
 	struct sched_cache_time __percpu *pcpu_sched;
 	raw_spinlock_t lock;
 	unsigned long epoch;
@@ -2413,11 +2394,26 @@ struct sched_cache_stat {
 	unsigned long next_scan;
 	unsigned long footprint;
 	int cpu;
+	refcount_t refcnt;
+	struct rcu_head rcu;
 } ____cacheline_aligned_in_smp;
 
+struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp);
+struct sched_cache_group *task_cache_group_get(struct task_struct *p);
+
+void sched_cache_fork(struct task_struct *p);
+void sched_cache_fork_cleanup(struct task_struct *p);
+void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm);
+void sched_cache_exit_mm(struct task_struct *p);
+
 #else
 
-struct sched_cache_stat { };
+struct sched_cache_group { };
+
+static inline void sched_cache_fork(struct task_struct *p) { }
+static inline void sched_cache_fork_cleanup(struct task_struct *p) { }
+static inline void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) { }
+static inline void sched_cache_exit_mm(struct task_struct *p) { }
 
 #endif
 
diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h
index e90efaf6d26e90..694126411dfea6 100644
--- a/include/linux/sched/cputime.h
+++ b/include/linux/sched/cputime.h
@@ -182,9 +182,9 @@ extern unsigned long long
 task_sched_runtime(struct task_struct *task);
 
 #ifdef CONFIG_PARAVIRT
-struct static_key;
-extern struct static_key paravirt_steal_enabled;
-extern struct static_key paravirt_steal_rq_enabled;
+#include <linux/jump_label.h>
+DECLARE_STATIC_KEY_FALSE(paravirt_steal_enabled);
+DECLARE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled);
 
 #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN
 u64 dummy_steal_clock(int cpu);
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index 582d7cd4a98395..23f9e178bc5aad 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -91,7 +91,7 @@ struct scx_dispatch_q {
 	struct rhash_head	hash_node;
 	struct llist_node	free_node;
 	struct scx_sched	*sched;
-	struct scx_dsq_pcpu __percpu *pcpu;
+	struct scx_dsq_pcpu __percpu *pcpu_user;
 	struct rcu_head		rcu;
 };
 
@@ -104,6 +104,7 @@ enum scx_ent_flags {
 	SCX_TASK_SUB_INIT	= 1 << 4, /* task being initialized for a sub sched */
 	SCX_TASK_IMMED		= 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */
 	SCX_TASK_PROTECTED	= 1 << 6, /* slice and DSQ head position protected */
+	SCX_TASK_RUN_TRACKED	= 1 << 7, /* task is in an ops.running()/stopping() session */
 
 	/*
 	 * Bits 8 to 10 are used to carry task state:
@@ -136,6 +137,7 @@ enum scx_ent_flags {
 	 * IMMED	reenqueued due to failed ENQ_IMMED
 	 * PREEMPTED	preempted while running
 	 * CAP		sub-sched cap miss, see p->scx.reenq_reason_*
+	 * PROXY	proxy state prevented a remote DSQ transfer
 	 */
 	SCX_TASK_REENQ_REASON_SHIFT = 12,
 	SCX_TASK_REENQ_REASON_BITS = 3,
@@ -146,6 +148,7 @@ enum scx_ent_flags {
 	SCX_TASK_REENQ_IMMED	= 2 << SCX_TASK_REENQ_REASON_SHIFT,
 	SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT,
 	SCX_TASK_REENQ_CAP	= 4 << SCX_TASK_REENQ_REASON_SHIFT,
+	SCX_TASK_REENQ_PROXY	= 5 << SCX_TASK_REENQ_REASON_SHIFT,
 
 	/* iteration cursor, not a task */
 	SCX_TASK_CURSOR		= 1 << 31,
@@ -197,9 +200,19 @@ struct sched_ext_entity {
 	u64			ddsp_slice;
 	u64			ddsp_vtime;
 	struct scx_dsq_list_node dsq_list;	/* dispatch order */
-	struct rb_node		dsq_priq;	/* p->scx.dsq_vtime order */
 	u32			dsq_seq;
 	u32			dsq_flags;	/* protected by DSQ lock */
+
+	/*
+	 * Used to order tasks when dispatching to the vtime-ordered priority
+	 * queue of a dsq. This is usually set through
+	 * scx_bpf_dsq_insert_vtime() but can also be modified directly by the
+	 * BPF scheduler. Modifying it while a task is queued on a dsq may
+	 * mangle the ordering and is not recommended. Kept next to @dsq_priq
+	 * as rbtree insertion reads both on every visited node.
+	 */
+	u64			dsq_vtime;
+	struct rb_node		dsq_priq;	/* p->scx.dsq_vtime order */
 	u32			flags;		/* protected by rq lock */
 	u32			weight;
 	u32			reenq_cnt;	/* reenqueues since last run */
@@ -225,7 +238,7 @@ struct sched_ext_entity {
 	u64			tid;
 	struct rhash_head	tid_hash_node;	/* see SCX_OPS_TID_TO_TASK */
 
-	/* BPF scheduler modifiable fields */
+	/* BPF scheduler modifiable fields, along with @dsq_vtime above */
 
 	/*
 	 * Runtime budget in nsecs - how long the task may hold its cpu. Owned
@@ -240,15 +253,6 @@ struct sched_ext_entity {
 	 */
 	u64			slice;
 
-	/*
-	 * Used to order tasks when dispatching to the vtime-ordered priority
-	 * queue of a dsq. This is usually set through
-	 * scx_bpf_dsq_insert_vtime() but can also be modified directly by the
-	 * BPF scheduler. Modifying it while a task is queued on a dsq may
-	 * mangle the ordering and is not recommended.
-	 */
-	u64			dsq_vtime;
-
 	/*
 	 * Out-of-band slice request from scx_bpf_task_set_slice() when the
 	 * caller does not hold the rq lock, applied under the rq lock at the
@@ -278,6 +282,14 @@ struct sched_ext_entity {
 	 */
 	bool			disallow;	/* reject switching into SCX */
 
+	/*
+	 * If set, depletion of this task's slice at the scheduler tick requests
+	 * lazy instead of immediate rescheduling. Initialized from
+	 * %SCX_OPS_LAZY_RESCHED immediately before ops.enable() and may be
+	 * modified afterwards with scx_bpf_task_set_lazy_resched().
+	 */
+	bool			lazy_resched;
+
 	/* cold fields */
 #ifdef CONFIG_EXT_GROUP_SCHED
 	struct cgroup		*cgrp_moving_from;
@@ -323,7 +335,7 @@ struct scx_task_group {
 	u64			bw_period_us;
 	u64			bw_quota_us;
 	u64			bw_burst_us;
-	bool			idle;
+	bool			sched_idle;
 #endif
 };
 
diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
index d45a5476b97dec..d9419dc902f603 100644
--- a/include/linux/sched/signal.h
+++ b/include/linux/sched/signal.h
@@ -2,6 +2,7 @@
 #ifndef _LINUX_SCHED_SIGNAL_H
 #define _LINUX_SCHED_SIGNAL_H
 
+#include <linux/cleanup.h>
 #include <linux/rculist.h>
 #include <linux/signal.h>
 #include <linux/sched.h>
@@ -79,9 +80,9 @@ struct core_thread {
 };
 
 struct core_state {
-	atomic_t nr_threads;
-	struct core_thread dumper;
-	struct completion startup;
+	/* Threads the dumper still waits for. */
+	atomic_t threads_remaining;
+	struct core_thread *tasks;
 };
 
 /*
@@ -384,14 +385,36 @@ static inline int task_sigpending(struct task_struct *p)
 	return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
 }
 
+/* Prevent TIF_NOTIFY_SIGNAL from interrupting this task. */
+static inline unsigned int no_notify_signal_save(void)
+{
+	unsigned int flags = current->flags;
+
+	current->flags |= PF_NO_NOTIFY_SIGNAL;
+	return flags;
+}
+
+/* Restore the previous PF_NO_NOTIFY_SIGNAL state. */
+static inline void no_notify_signal_restore(unsigned int flags)
+{
+	current_restore_flags(flags, PF_NO_NOTIFY_SIGNAL);
+}
+
+DEFINE_LOCK_GUARD_0(no_notify_signal,
+		    _T->flags = no_notify_signal_save(),
+		    no_notify_signal_restore(_T->flags),
+		    unsigned int flags)
+
 static inline int signal_pending(struct task_struct *p)
 {
 	/*
 	 * TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same
 	 * behavior in terms of ensuring that we break out of wait loops
-	 * so that notify signal callbacks can be processed.
+	 * so that notify signal callbacks can be processed. Not for a task
+	 * that asked not to be interrupted by it, see no_notify_signal_save().
 	 */
-	if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL)))
+	if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL)) &&
+	    likely(!(READ_ONCE(p->flags) & PF_NO_NOTIFY_SIGNAL)))
 		return 1;
 	return task_sigpending(p);
 }
diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h
index e0c1ca8c6a1884..90ed5bc3c7afee 100644
--- a/include/linux/sched/task.h
+++ b/include/linux/sched/task.h
@@ -94,7 +94,6 @@ static inline void exit_thread(struct task_struct *tsk)
 extern __noreturn void do_group_exit(int);
 
 extern void exit_files(struct task_struct *);
-extern void exit_itimers(struct task_struct *);
 
 extern pid_t kernel_clone(struct kernel_clone_args *kargs);
 struct task_struct *copy_process(struct pid *pid, int trace, int node,
diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index b5d9d7c2b8add7..f96812d71c515c 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -281,9 +281,9 @@ static inline int task_node(const struct task_struct *p)
 }
 
 #ifdef CONFIG_SCHED_CACHE
-extern void sched_update_llc_bytes(unsigned int cpu);
+extern void sched_update_llc_bytes(const struct cpumask *cpus);
 #else
-static inline void sched_update_llc_bytes(unsigned int cpu) { }
+static inline void sched_update_llc_bytes(const struct cpumask *cpus) { }
 #endif
 
 #endif /* _LINUX_SCHED_TOPOLOGY_H */
diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h
index 5ab73b1ab9aa4f..3f7cc35eb861f6 100644
--- a/include/linux/scmi_protocol.h
+++ b/include/linux/scmi_protocol.h
@@ -9,6 +9,7 @@
 #define _LINUX_SCMI_PROTOCOL_H
 
 #include <linux/bitfield.h>
+#include <linux/device-id/scmi.h>
 #include <linux/device.h>
 #include <linux/notifier.h>
 #include <linux/types.h>
@@ -528,22 +529,25 @@ struct scmi_sensor_proto_ops {
 			  u32 sensor_id, u32 sensor_config);
 };
 
+struct scmi_reset_domain_info {
+	char name[SCMI_MAX_STR_SIZE];
+	u32 latency_us;
+};
+
 /**
  * struct scmi_reset_proto_ops - represents the various operations provided
  *	by SCMI Reset Protocol
  *
  * @num_domains_get: get the count of reset domains provided by SCMI
- * @name_get: gets the name of a reset domain
- * @latency_get: gets the reset latency for the specified reset domain
+ * @info_get: gets the information of the specified reset domain
  * @reset: resets the specified reset domain
  * @assert: explicitly assert reset signal of the specified reset domain
  * @deassert: explicitly deassert reset signal of the specified reset domain
  */
 struct scmi_reset_proto_ops {
 	int (*num_domains_get)(const struct scmi_protocol_handle *ph);
-	const char *(*name_get)(const struct scmi_protocol_handle *ph,
-				u32 domain);
-	int (*latency_get)(const struct scmi_protocol_handle *ph, u32 domain);
+	const struct scmi_reset_domain_info __must_check *(*info_get)
+		(const struct scmi_protocol_handle *ph, u32 domain);
 	int (*reset)(const struct scmi_protocol_handle *ph, u32 domain);
 	int (*assert)(const struct scmi_protocol_handle *ph, u32 domain);
 	int (*deassert)(const struct scmi_protocol_handle *ph, u32 domain);
@@ -930,6 +934,7 @@ enum scmi_std_protocol {
 	SCMI_PROTOCOL_VOLTAGE = 0x17,
 	SCMI_PROTOCOL_POWERCAP = 0x18,
 	SCMI_PROTOCOL_PINCTRL = 0x19,
+	SCMI_PROTOCOL_TELEMETRY = 0x1B,
 };
 
 enum scmi_system_events {
@@ -951,11 +956,6 @@ struct scmi_device {
 
 #define to_scmi_dev(d) container_of_const(d, struct scmi_device, dev)
 
-struct scmi_device_id {
-	u8 protocol_id;
-	const char *name;
-};
-
 struct scmi_driver {
 	const char *name;
 	int (*probe)(struct scmi_device *sdev);
diff --git a/include/linux/security.h b/include/linux/security.h
index 153e9043058f88..8b8d6b8b802f1a 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -67,6 +67,7 @@ enum fs_value_type;
 struct watch;
 struct watch_notification;
 struct lsm_ctx;
+struct nsset;
 
 /* Default (no) options for the capable function */
 #define CAP_OPT_NONE 0x0
@@ -80,6 +81,7 @@ struct lsm_ctx;
 
 struct ctl_table;
 struct audit_krule;
+struct ns_common;
 struct user_namespace;
 struct timezone;
 
@@ -185,11 +187,11 @@ extern int cap_capset(struct cred *new, const struct cred *old,
 extern int cap_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file);
 int cap_inode_setxattr(struct dentry *dentry, const char *name,
 		       const void *value, size_t size, int flags);
-int cap_inode_removexattr(struct mnt_idmap *idmap,
+int cap_inode_removexattr(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, const char *name);
 int cap_inode_need_killpriv(struct dentry *dentry);
-int cap_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry);
-int cap_inode_getsecurity(struct mnt_idmap *idmap,
+int cap_inode_killpriv(const struct mnt_idmap *idmap, struct dentry *dentry);
+int cap_inode_getsecurity(const struct mnt_idmap *idmap,
 			  struct inode *inode, const char *name, void **buffer,
 			  bool alloc);
 extern int cap_mmap_addr(unsigned long addr);
@@ -338,6 +340,7 @@ int security_binder_transfer_file(const struct cred *from,
 				  const struct cred *to, const struct file *file);
 int security_ptrace_access_check(struct task_struct *child, unsigned int mode);
 int security_ptrace_traceme(struct task_struct *parent);
+int security_mem_foll_force(const struct cred *subject, bool opened_by_owner);
 int security_capget(const struct task_struct *target,
 		    kernel_cap_t *effective,
 		    kernel_cap_t *inheritable,
@@ -404,49 +407,53 @@ int security_inode_init_security(struct inode *inode, struct inode *dir,
 int security_inode_init_security_anon(struct inode *inode,
 				      const struct qstr *name,
 				      const struct inode *context_inode);
-int security_inode_create(struct inode *dir, struct dentry *dentry, umode_t mode);
-void security_inode_post_create_tmpfile(struct mnt_idmap *idmap,
+int security_inode_create(const struct mnt_idmap *idmap, struct inode *dir,
+			  struct dentry *dentry, umode_t mode);
+void security_inode_post_create_tmpfile(const struct mnt_idmap *idmap,
 					struct inode *inode);
-int security_inode_link(struct dentry *old_dentry, struct inode *dir,
-			 struct dentry *new_dentry);
+int security_inode_link(const struct mnt_idmap *idmap, struct dentry *old_dentry,
+			struct inode *dir, struct dentry *new_dentry);
 int security_inode_unlink(struct inode *dir, struct dentry *dentry);
-int security_inode_symlink(struct inode *dir, struct dentry *dentry,
-			   const char *old_name);
-int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode);
+int security_inode_symlink(const struct mnt_idmap *idmap, struct inode *dir,
+			   struct dentry *dentry, const char *old_name);
+int security_inode_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
+			 struct dentry *dentry, umode_t mode);
 int security_inode_rmdir(struct inode *dir, struct dentry *dentry);
-int security_inode_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev);
+int security_inode_mknod(const struct mnt_idmap *idmap, struct inode *dir,
+			 struct dentry *dentry, umode_t mode, dev_t dev);
 int security_inode_rename(struct inode *old_dir, struct dentry *old_dentry,
 			  struct inode *new_dir, struct dentry *new_dentry,
 			  unsigned int flags);
 int security_inode_readlink(struct dentry *dentry);
 int security_inode_follow_link(struct dentry *dentry, struct inode *inode,
 			       bool rcu);
-int security_inode_permission(struct inode *inode, int mask);
-int security_inode_setattr(struct mnt_idmap *idmap,
+int security_inode_permission(const struct mnt_idmap *idmap, struct inode *inode,
+			      int mask);
+int security_inode_setattr(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, struct iattr *attr);
-void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+void security_inode_post_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 				 int ia_valid);
 int security_inode_getattr(const struct path *path);
-int security_inode_setxattr(struct mnt_idmap *idmap,
+int security_inode_setxattr(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, const char *name,
 			    const void *value, size_t size, int flags);
-int security_inode_set_acl(struct mnt_idmap *idmap,
+int security_inode_set_acl(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, const char *acl_name,
 			   struct posix_acl *kacl);
 void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name,
 				 struct posix_acl *kacl);
-int security_inode_get_acl(struct mnt_idmap *idmap,
+int security_inode_get_acl(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, const char *acl_name);
-int security_inode_remove_acl(struct mnt_idmap *idmap,
+int security_inode_remove_acl(const struct mnt_idmap *idmap,
 			      struct dentry *dentry, const char *acl_name);
-void security_inode_post_remove_acl(struct mnt_idmap *idmap,
+void security_inode_post_remove_acl(const struct mnt_idmap *idmap,
 				    struct dentry *dentry,
 				    const char *acl_name);
 void security_inode_post_setxattr(struct dentry *dentry, const char *name,
 				  const void *value, size_t size, int flags);
 int security_inode_getxattr(struct dentry *dentry, const char *name);
 int security_inode_listxattr(struct dentry *dentry);
-int security_inode_removexattr(struct mnt_idmap *idmap,
+int security_inode_removexattr(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, const char *name);
 void security_inode_post_removexattr(struct dentry *dentry, const char *name);
 int security_inode_file_setattr(struct dentry *dentry,
@@ -454,8 +461,8 @@ int security_inode_file_setattr(struct dentry *dentry,
 int security_inode_file_getattr(struct dentry *dentry,
 			      struct file_kattr *fa);
 int security_inode_need_killpriv(struct dentry *dentry);
-int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry);
-int security_inode_getsecurity(struct mnt_idmap *idmap,
+int security_inode_killpriv(const struct mnt_idmap *idmap, struct dentry *dentry);
+int security_inode_getsecurity(const struct mnt_idmap *idmap,
 			       struct inode *inode, const char *name,
 			       void **buffer, bool alloc);
 int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags);
@@ -468,7 +475,7 @@ int security_inode_setintegrity(const struct inode *inode,
 				size_t size);
 int security_kernfs_init_security(struct kernfs_node *kn_dir,
 				  struct kernfs_node *kn);
-int security_file_permission(struct file *file, int mask);
+int security_file_permission(const struct file *file, int mask);
 int security_file_alloc(struct file *file);
 void security_file_release(struct file *file);
 void security_file_free(struct file *file);
@@ -540,6 +547,9 @@ int security_task_prctl(int option, unsigned long arg2, unsigned long arg3,
 			unsigned long arg4, unsigned long arg5);
 void security_task_to_inode(struct task_struct *p, struct inode *inode);
 int security_create_user_ns(const struct cred *cred);
+int security_namespace_init(struct ns_common *ns);
+void security_namespace_free(struct ns_common *ns);
+int security_namespace_install(const struct nsset *nsset, struct ns_common *ns);
 int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag);
 void security_ipc_getlsmprop(struct kern_ipc_perm *ipcp, struct lsm_prop *prop);
 int security_msg_msg_alloc(struct msg_msg *msg);
@@ -676,6 +686,12 @@ static inline int security_ptrace_traceme(struct task_struct *parent)
 	return cap_ptrace_traceme(parent);
 }
 
+static inline int security_mem_foll_force(const struct cred *subject,
+					  bool opened_by_owner)
+{
+	return 0;
+}
+
 static inline int security_capget(const struct task_struct *target,
 				   kernel_cap_t *effective,
 				   kernel_cap_t *inheritable,
@@ -902,20 +918,22 @@ static inline int security_inode_init_security_anon(struct inode *inode,
 	return 0;
 }
 
-static inline int security_inode_create(struct inode *dir,
-					 struct dentry *dentry,
-					 umode_t mode)
+static inline int security_inode_create(const struct mnt_idmap *idmap,
+					struct inode *dir,
+					struct dentry *dentry,
+					umode_t mode)
 {
 	return 0;
 }
 
 static inline void
-security_inode_post_create_tmpfile(struct mnt_idmap *idmap, struct inode *inode)
+security_inode_post_create_tmpfile(const struct mnt_idmap *idmap, struct inode *inode)
 { }
 
-static inline int security_inode_link(struct dentry *old_dentry,
-				       struct inode *dir,
-				       struct dentry *new_dentry)
+static inline int security_inode_link(const struct mnt_idmap *idmap,
+				      struct dentry *old_dentry,
+				      struct inode *dir,
+				      struct dentry *new_dentry)
 {
 	return 0;
 }
@@ -926,16 +944,18 @@ static inline int security_inode_unlink(struct inode *dir,
 	return 0;
 }
 
-static inline int security_inode_symlink(struct inode *dir,
-					  struct dentry *dentry,
-					  const char *old_name)
+static inline int security_inode_symlink(const struct mnt_idmap *idmap,
+					 struct inode *dir,
+					 struct dentry *dentry,
+					 const char *old_name)
 {
 	return 0;
 }
 
-static inline int security_inode_mkdir(struct inode *dir,
-					struct dentry *dentry,
-					int mode)
+static inline int security_inode_mkdir(const struct mnt_idmap *idmap,
+				       struct inode *dir,
+				       struct dentry *dentry,
+				       int mode)
 {
 	return 0;
 }
@@ -946,9 +966,10 @@ static inline int security_inode_rmdir(struct inode *dir,
 	return 0;
 }
 
-static inline int security_inode_mknod(struct inode *dir,
-					struct dentry *dentry,
-					int mode, dev_t dev)
+static inline int security_inode_mknod(const struct mnt_idmap *idmap,
+				       struct inode *dir,
+				       struct dentry *dentry,
+				       int mode, dev_t dev)
 {
 	return 0;
 }
@@ -974,12 +995,13 @@ static inline int security_inode_follow_link(struct dentry *dentry,
 	return 0;
 }
 
-static inline int security_inode_permission(struct inode *inode, int mask)
+static inline int security_inode_permission(const struct mnt_idmap *idmap,
+					    struct inode *inode, int mask)
 {
 	return 0;
 }
 
-static inline int security_inode_setattr(struct mnt_idmap *idmap,
+static inline int security_inode_setattr(const struct mnt_idmap *idmap,
 					 struct dentry *dentry,
 					 struct iattr *attr)
 {
@@ -987,7 +1009,7 @@ static inline int security_inode_setattr(struct mnt_idmap *idmap,
 }
 
 static inline void
-security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+security_inode_post_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			    int ia_valid)
 { }
 
@@ -996,14 +1018,14 @@ static inline int security_inode_getattr(const struct path *path)
 	return 0;
 }
 
-static inline int security_inode_setxattr(struct mnt_idmap *idmap,
+static inline int security_inode_setxattr(const struct mnt_idmap *idmap,
 		struct dentry *dentry, const char *name, const void *value,
 		size_t size, int flags)
 {
 	return cap_inode_setxattr(dentry, name, value, size, flags);
 }
 
-static inline int security_inode_set_acl(struct mnt_idmap *idmap,
+static inline int security_inode_set_acl(const struct mnt_idmap *idmap,
 					 struct dentry *dentry,
 					 const char *acl_name,
 					 struct posix_acl *kacl)
@@ -1016,21 +1038,21 @@ static inline void security_inode_post_set_acl(struct dentry *dentry,
 					       struct posix_acl *kacl)
 { }
 
-static inline int security_inode_get_acl(struct mnt_idmap *idmap,
+static inline int security_inode_get_acl(const struct mnt_idmap *idmap,
 					 struct dentry *dentry,
 					 const char *acl_name)
 {
 	return 0;
 }
 
-static inline int security_inode_remove_acl(struct mnt_idmap *idmap,
+static inline int security_inode_remove_acl(const struct mnt_idmap *idmap,
 					    struct dentry *dentry,
 					    const char *acl_name)
 {
 	return 0;
 }
 
-static inline void security_inode_post_remove_acl(struct mnt_idmap *idmap,
+static inline void security_inode_post_remove_acl(const struct mnt_idmap *idmap,
 						  struct dentry *dentry,
 						  const char *acl_name)
 { }
@@ -1050,7 +1072,7 @@ static inline int security_inode_listxattr(struct dentry *dentry)
 	return 0;
 }
 
-static inline int security_inode_removexattr(struct mnt_idmap *idmap,
+static inline int security_inode_removexattr(const struct mnt_idmap *idmap,
 					     struct dentry *dentry,
 					     const char *name)
 {
@@ -1078,13 +1100,13 @@ static inline int security_inode_need_killpriv(struct dentry *dentry)
 	return cap_inode_need_killpriv(dentry);
 }
 
-static inline int security_inode_killpriv(struct mnt_idmap *idmap,
+static inline int security_inode_killpriv(const struct mnt_idmap *idmap,
 					  struct dentry *dentry)
 {
 	return cap_inode_killpriv(idmap, dentry);
 }
 
-static inline int security_inode_getsecurity(struct mnt_idmap *idmap,
+static inline int security_inode_getsecurity(const struct mnt_idmap *idmap,
 					     struct inode *inode,
 					     const char *name, void **buffer,
 					     bool alloc)
@@ -1132,7 +1154,7 @@ static inline int security_inode_copy_up_xattr(struct dentry *src, const char *n
 	return -EOPNOTSUPP;
 }
 
-static inline int security_file_permission(struct file *file, int mask)
+static inline int security_file_permission(const struct file *file, int mask)
 {
 	return 0;
 }
@@ -1431,6 +1453,21 @@ static inline int security_create_user_ns(const struct cred *cred)
 	return 0;
 }
 
+static inline int security_namespace_init(struct ns_common *ns)
+{
+	return 0;
+}
+
+static inline void security_namespace_free(struct ns_common *ns)
+{
+}
+
+static inline int security_namespace_install(const struct nsset *nsset,
+					     struct ns_common *ns)
+{
+	return 0;
+}
+
 static inline int security_ipc_permission(struct kern_ipc_perm *ipcp,
 					  short flag)
 {
@@ -2085,7 +2122,7 @@ int security_path_mkdir(const struct path *dir, struct dentry *dentry, umode_t m
 int security_path_rmdir(const struct path *dir, struct dentry *dentry);
 int security_path_mknod(const struct path *dir, struct dentry *dentry, umode_t mode,
 			unsigned int dev);
-void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry);
+void security_path_post_mknod(const struct mnt_idmap *idmap, struct dentry *dentry);
 int security_path_truncate(const struct path *path);
 int security_path_symlink(const struct path *dir, struct dentry *dentry,
 			  const char *old_name);
@@ -2120,7 +2157,7 @@ static inline int security_path_mknod(const struct path *dir, struct dentry *den
 	return 0;
 }
 
-static inline void security_path_post_mknod(struct mnt_idmap *idmap,
+static inline void security_path_post_mknod(const struct mnt_idmap *idmap,
 					    struct dentry *dentry)
 { }
 
diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h
index 904760876ca84a..4eeb8253752aa0 100644
--- a/include/linux/serial_core.h
+++ b/include/linux/serial_core.h
@@ -1140,8 +1140,6 @@ int uart_add_one_port(struct uart_driver *reg, struct uart_port *port);
 void uart_remove_one_port(struct uart_driver *reg, struct uart_port *port);
 int uart_read_port_properties(struct uart_port *port);
 int uart_read_and_validate_port_properties(struct uart_port *port);
-bool uart_match_port(const struct uart_port *port1,
-		const struct uart_port *port2);
 
 /*
  * Power Management
diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h
index 3030d9245f5ac8..27c32fbabfec90 100644
--- a/include/linux/set_memory.h
+++ b/include/linux/set_memory.h
@@ -5,16 +5,92 @@
 #ifndef _LINUX_SET_MEMORY_H_
 #define _LINUX_SET_MEMORY_H_
 
+/**
+ * DOC: Kernel page table permissions
+ *
+ * The set_memory() and set_direct_map() APIs update permissions of existing
+ * kernel mappings.
+ *
+ * The set_memory() functions operate on a range of kernel virtual addresses,
+ * the set_direct_map() functions operate on the direct map.
+ *
+ * The updates are not atomic: when a call fails, an arbitrary prefix of the
+ * range may have been updated already and there is no automatic rollback.
+ * A caller must restore the required permissions before reusing or freeing
+ * the memory.
+ *
+ * When an architecture does not implement these APIs they succeed without
+ * doing anything, so a return value of 0 does not mean that the permissions
+ * were actually changed.
+ *
+ * Callers that depend on the permissions being applied must ensure that the
+ * architecture supports the required operation for the target addresses. The
+ * Kconfig symbols alone do not guarantee this.
+ *
+ * See Documentation/mm/kernel-page-tables.rst for the details and for the
+ * differences between the architecture implementations.
+ */
+
 #ifdef CONFIG_ARCH_HAS_SET_MEMORY
 #include <asm/set_memory.h>
 #else
+/**
+ * set_memory_ro - make a kernel mapping read-only
+ * @addr: page aligned start of the kernel virtual address range
+ * @numpages: number of pages in the range
+ *
+ * Flushes the TLB for the range.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
 static inline int __must_check set_memory_ro(unsigned long addr, int numpages) { return 0; }
+
+/**
+ * set_memory_rw - make a kernel mapping writable
+ * @addr: page aligned start of the kernel virtual address range
+ * @numpages: number of pages in the range
+ *
+ * Flushes the TLB for the range.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
 static inline int __must_check set_memory_rw(unsigned long addr, int numpages) { return 0; }
+
+/**
+ * set_memory_x - make a kernel mapping executable
+ * @addr: page aligned start of the kernel virtual address range
+ * @numpages: number of pages in the range
+ *
+ * Flushes the TLB for the range.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
 static inline int __must_check set_memory_x(unsigned long addr,  int numpages) { return 0; }
+
+/**
+ * set_memory_nx - make a kernel mapping non-executable
+ * @addr: page aligned start of the kernel virtual address range
+ * @numpages: number of pages in the range
+ *
+ * Flushes the TLB for the range.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
 static inline int __must_check set_memory_nx(unsigned long addr, int numpages) { return 0; }
 #endif
 
 #ifndef set_memory_rox
+/**
+ * set_memory_rox - make a kernel mapping read-only and executable
+ * @addr: page aligned start of the kernel virtual address range
+ * @numpages: number of pages in the range
+ *
+ * A failure may leave the range read-only but not executable.
+ *
+ * Flushes the TLB for the range.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
 static inline int set_memory_rox(unsigned long addr, int numpages)
 {
 	int ret = set_memory_ro(addr, numpages);
@@ -25,17 +101,35 @@ static inline int set_memory_rox(unsigned long addr, int numpages)
 #endif
 
 #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP
-static inline int set_direct_map_invalid_noflush(struct page *page)
-{
-	return 0;
-}
-static inline int set_direct_map_default_noflush(struct page *page)
+/**
+ * set_direct_map_invalid_noflush - remove pages from the direct map
+ * @page: first page to update
+ * @nr: number of pages to update
+ *
+ * Makes the direct mapping of @nr pages starting at @page not present.
+ * The caller is responsible for any required TLB flushing.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+static inline int set_direct_map_invalid_noflush(struct page *page,
+						 unsigned int nr)
 {
 	return 0;
 }
 
-static inline int set_direct_map_valid_noflush(struct page *page,
-					       unsigned nr, bool valid)
+/**
+ * set_direct_map_default_noflush - restore the direct map of pages
+ * @page: first page to update
+ * @nr: number of pages to update
+ *
+ * Restores the default kernel permissions of the direct mapping of @nr
+ * pages starting at @page.
+ * The caller is responsible for any required TLB flushing.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+static inline int set_direct_map_default_noflush(struct page *page,
+						 unsigned int nr)
 {
 	return 0;
 }
@@ -50,6 +144,18 @@ static inline bool kernel_page_present(struct page *page)
  * boot time. Let them overrive this query.
  */
 #ifndef can_set_direct_map
+/**
+ * can_set_direct_map - check if the direct map can be modified
+ *
+ * Available with CONFIG_ARCH_HAS_SET_DIRECT_MAP. Architectures may override
+ * this to report whether direct map updates are enabled at runtime.
+ * Even though the generic implementation returns true this does not guarantee
+ * that every address can be updated.
+ *
+ * See Documentation/mm/kernel-page-tables.rst for the details
+ *
+ * Return: true unless the architecture reports direct map updates disabled.
+ */
 static inline bool can_set_direct_map(void)
 {
 	return true;
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 5663dff53186e2..a7c7a96a7cbf90 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -11,6 +11,7 @@
 #include <linux/fs_parser.h>
 #include <linux/userfaultfd_k.h>
 #include <linux/bits.h>
+#include <linux/list_lru.h>
 
 /* inode in-kernel data */
 
@@ -54,6 +55,11 @@ struct shmem_inode_info {
 	struct dquot __rcu	*i_dquot[MAXQUOTAS];
 #endif
 	struct inode		vfs_inode;
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	struct obj_cgroup	*shrinklist_objcg;
+	int			shrinklist_nid;
+#endif
 };
 
 #define SHMEM_FL_USER_VISIBLE		(FS_FL_USER_VISIBLE | FS_CASEFOLD_FL)
@@ -83,9 +89,9 @@ struct shmem_sb_info {
 	ino_t next_ino;		    /* The next per-sb inode number to use */
 	ino_t __percpu *ino_batch;  /* The next per-cpu inode number to use */
 	struct mempolicy *mpol;     /* default memory policy for mappings */
-	spinlock_t shrinklist_lock;   /* Protects shrinklist */
-	struct list_head shrinklist;  /* List of shinkable inodes */
-	unsigned long shrinklist_len; /* Length of shrinklist */
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	struct list_lru shrinklist; /* List of shrinkable inodes */
+#endif
 	struct shmem_quota_limits qlimits; /* Default quota limits */
 	struct simple_xattr_cache xa_cache;
 };
diff --git a/include/linux/siphash.h b/include/linux/siphash.h
index 9153e77382e109..00c58bbe549a94 100644
--- a/include/linux/siphash.h
+++ b/include/linux/siphash.h
@@ -75,8 +75,10 @@ static inline u64 ___siphash_aligned(const __le64 *data, size_t len,
 /**
  * siphash - compute 64-bit siphash PRF value
  * @data: buffer to hash
- * @size: size of @data
+ * @len: size of @data
  * @key: the siphash key
+ *
+ * Returns: siphash PRF value
  */
 static inline u64 siphash(const void *data, size_t len,
 			  const siphash_key_t *key)
@@ -125,8 +127,10 @@ static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len,
 /**
  * hsiphash - compute 32-bit hsiphash PRF value
  * @data: buffer to hash
- * @size: size of @data
+ * @len: size of @data
  * @key: the hsiphash key
+ *
+ * Returns: hsiphash PRF value
  */
 static inline u32 hsiphash(const void *data, size_t len,
 			   const hsiphash_key_t *key)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 421f6fc454511e..27ec1e38c82836 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1020,7 +1020,7 @@ struct sk_buff {
 #ifdef CONFIG_NET_REDIRECT
 	__u8			from_ingress:1;
 #endif
-#ifdef CONFIG_NETFILTER_SKIP_EGRESS
+#ifdef CONFIG_NET_EGRESS
 	__u8			nf_skip_egress:1;
 #endif
 #ifdef CONFIG_SKB_DECRYPTED
@@ -1834,22 +1834,6 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
 	}
 }
 
-static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
-{
-	skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
-	skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG;
-}
-
-static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
-{
-	return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
-}
-
-static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
-{
-	return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
-}
-
 static inline void net_zcopy_put(struct ubuf_info *uarg)
 {
 	if (uarg)
@@ -1872,8 +1856,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
 	struct ubuf_info *uarg = skb_zcopy(skb);
 
 	if (uarg) {
-		if (!skb_zcopy_is_nouarg(skb))
-			uarg->ops->complete(skb, uarg, zerocopy_success);
+		uarg->ops->complete(skb, uarg, zerocopy_success);
 
 		skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
 	}
@@ -3833,9 +3816,12 @@ static inline dma_addr_t __skb_frag_dma_map(struct device *dev,
 					    size_t offset, size_t size,
 					    enum dma_data_direction dir)
 {
+	dma_addr_t addr;
+
 	if (skb_frag_is_net_iov(frag)) {
-		return netmem_to_net_iov(frag->netmem)->desc.dma_addr +
-		       offset + frag->offset;
+		addr = netmem_dma_addr_decode(
+			netmem_get_dma_addr(frag->netmem));
+		return addr + offset + frag->offset;
 	}
 	return dma_map_page(dev, skb_frag_page(frag),
 			    skb_frag_off(frag) + offset, size, dir);
@@ -4372,7 +4358,10 @@ skb_header_pointer_careful(const struct sk_buff *skb, int offset,
 static inline void * __must_check
 skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len)
 {
-	if (likely(skb_headlen(skb) - offset >= len))
+	unsigned int uoffset = (unsigned int)offset;
+
+	if (likely(uoffset <= skb_headlen(skb) &&
+		   (unsigned int)len <= skb_headlen(skb) - uoffset))
 		return skb->data + offset;
 	return NULL;
 }
diff --git a/include/linux/slimbus.h b/include/linux/slimbus.h
index ca6f1da4bdf397..1a2d3aa0e03393 100644
--- a/include/linux/slimbus.h
+++ b/include/linux/slimbus.h
@@ -98,7 +98,7 @@ struct slim_driver {
  * @start_offset: Specifies starting offset in information/value element map
  * @rbuf: buffer to read the values
  * @wbuf: buffer to write
- * @num_bytes: upto 16. This ensures that the message will fit the slicesize
+ * @num_bytes: up to 16. This ensures that the message will fit the slicesize
  *		per SLIMbus spec
  * @comp: completion for asynchronous operations, valid only if TID is
  *	  required for transaction, like REQUEST operations.
diff --git a/include/linux/soc/qcom/apr.h b/include/linux/soc/qcom/apr.h
index 909e84f84e0cfa..2e4231092088ed 100644
--- a/include/linux/soc/qcom/apr.h
+++ b/include/linux/soc/qcom/apr.h
@@ -5,7 +5,6 @@
 
 #include <linux/spinlock.h>
 #include <linux/device.h>
-#include <linux/device-id/apr.h>
 #include <dt-bindings/soc/qcom,apr.h>
 #include <dt-bindings/soc/qcom,gpr.h>
 
@@ -135,6 +134,8 @@ struct pkt_router_svc {
 
 typedef struct pkt_router_svc gpr_port_t;
 
+#define APR_NAME_SIZE   32
+
 struct apr_device {
 	struct device	dev;
 	uint16_t	svc_id;
@@ -158,7 +159,6 @@ struct apr_driver {
 			    const struct apr_resp_pkt *d);
 	gpr_port_cb gpr_callback;
 	struct device_driver		driver;
-	const struct apr_device_id	*id_table;
 };
 
 typedef struct apr_driver gpr_driver_t;
diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h
index 29a53bbc0dd495..2243005f5e2ee5 100644
--- a/include/linux/soc/qcom/geni-se.h
+++ b/include/linux/soc/qcom/geni-se.h
@@ -347,17 +347,18 @@ struct geni_se {
 #define QUP_SE_VERSION_2_5                  0x20050000
 
 /*
- * Define bandwidth thresholds that cause the underlying Core 2X interconnect
- * clock to run at the named frequency. These baseline values are recommended
- * by the hardware team, and are not dynamically scaled with GENI bandwidth
- * beyond basic on/off.
+ * ICC bandwidth values in Bps for the GENI_TO_CORE ("qup-core") path.
+ * Convert them with Bps_to_icc() before setting avg_bw. The QUP ICC provider
+ * maps each threshold to the Core 2X rate named by the macro suffix.
+ *
+ * These values are core clock votes, not GENI transfer bandwidths.
  */
-#define CORE_2X_19_2_MHZ		960
-#define CORE_2X_50_MHZ			2500
-#define CORE_2X_100_MHZ			5000
-#define CORE_2X_150_MHZ			7500
-#define CORE_2X_200_MHZ			10000
-#define CORE_2X_236_MHZ			16383
+#define CORE_2X_19_2_MHZ		9600000
+#define CORE_2X_50_MHZ			25000000
+#define CORE_2X_100_MHZ			50000000
+#define CORE_2X_150_MHZ			75000000
+#define CORE_2X_200_MHZ			100000000
+#define CORE_2X_236_MHZ			118000000
 
 #define GENI_DEFAULT_BW			Bps_to_icc(1000)
 
diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h
index f3ed63e475ab10..713cb0221b14ca 100644
--- a/include/linux/soc/qcom/llcc-qcom.h
+++ b/include/linux/soc/qcom/llcc-qcom.h
@@ -74,6 +74,7 @@
 #define LLCC_CAMSRTIP	 73
 #define LLCC_CAMRTRF	 74
 #define LLCC_CAMSRTRF	 75
+#define LLCC_GPU_LITTLE  80
 #define LLCC_OOBM_NS	 81
 #define LLCC_OOBM_S	 82
 #define LLCC_VIDEO_APV	 83
@@ -85,7 +86,12 @@
 #define LLCC_CAM_OFE_STROV	 93
 #define LLCC_CPUSS_HEU	 94
 #define LLCC_PCIE_TCU	 97
+#define LLCC_GPUHTW_LITTLE	 98
 #define LLCC_MDM_PNG_FIXED	 100
+#define LLCC_PPE_RXDESC		 101
+#define LLCC_PPE_RXFILL		 102
+#define LLCC_WLAN_5G		 103
+#define LLCC_WLAN_6G		 104
 
 /**
  * struct llcc_slice_desc - Cache slice descriptor
diff --git a/include/linux/soc/qcom/qmi.h b/include/linux/soc/qcom/qmi.h
index b9dcb437a0be54..683d27cee41321 100644
--- a/include/linux/soc/qcom/qmi.h
+++ b/include/linux/soc/qcom/qmi.h
@@ -96,6 +96,7 @@ struct qmi_elem_info {
  * Enumerate the IDs of the QMI services
  */
 #define QMI_SERVICE_ID_TEST		0x0f	/*   15 */
+#define QMI_SERVICE_ID_TMD		0x18	/*   24 */
 #define QMI_SERVICE_ID_SSCTL		0x2b	/*   43 */
 #define QMI_SERVICE_ID_IPA		0x31	/*   49 */
 #define QMI_SERVICE_ID_SERVREG_LOC	0x40	/*   64 */
diff --git a/include/linux/soc/qcom/qmi_tmd.h b/include/linux/soc/qcom/qmi_tmd.h
new file mode 100644
index 00000000000000..452f51f23731f4
--- /dev/null
+++ b/include/linux/soc/qcom/qmi_tmd.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2025, Linaro Limited
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ * QMI Thermal Mitigation Device (TMD) library header.
+ */
+
+#ifndef __QMI_TMD_H__
+#define __QMI_TMD_H__
+
+struct device;
+struct qmi_tmd_client;
+
+#if IS_ENABLED(CONFIG_QCOM_QMI_TMD)
+struct qmi_tmd_client *qmi_tmd_init(struct device *dev,
+				    unsigned int instance_id,
+				    const char * const *tmd_names,
+				    int num_tmds);
+
+void qmi_tmd_exit(struct qmi_tmd_client *tmd_cli);
+#else
+static inline struct qmi_tmd_client *qmi_tmd_init(struct device *dev,
+						  unsigned int instance_id,
+						  const char * const *tmd_names,
+						  int num_tmds)
+{
+	return NULL;
+}
+
+static inline void qmi_tmd_exit(struct qmi_tmd_client *tmd_cli)
+{
+}
+#endif
+
+#endif /* __QMI_TMD_H__ */
diff --git a/include/linux/soc/qcom/tc9563.h b/include/linux/soc/qcom/tc9563.h
new file mode 100644
index 00000000000000..086f37a40d801d
--- /dev/null
+++ b/include/linux/soc/qcom/tc9563.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Author: Lorenzo Bianconi <lorenzo.bianconi@oss.qualcomm.com>
+ */
+
+#ifndef __QCOM_TC9563_H
+#define __QCOM_TC9563_H
+
+#define TC9563_GPIO_DEV_NAME	"tc9563-gpio"
+
+#define TC9563_GPIO_IN0_OFFSET		0x801200
+#define TC9563_GPIO_EN0_OFFSET		0x801208
+#define TC9563_GPIO_OUT0_OFFSET		0x801210
+
+#define TC9563_GPIO_CONFIG		TC9563_GPIO_EN0_OFFSET
+#define TC9563_RESET_GPIO		TC9563_GPIO_OUT0_OFFSET
+
+#endif /* __QCOM_TC9563_H */
diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h
index f3a70360b177bb..b09ae05cbeae4f 100644
--- a/include/linux/soc/qcom/ubwc.h
+++ b/include/linux/soc/qcom/ubwc.h
@@ -36,6 +36,7 @@ struct qcom_ubwc_cfg_data {
 #define UBWC_4_3 0x40030000
 #define UBWC_5_0 0x50000000
 #define UBWC_6_0 0x60000000
+#define UBWC_7_0 0x70000000
 
 #if IS_ENABLED(CONFIG_QCOM_UBWC_CONFIG)
 const struct qcom_ubwc_cfg_data *qcom_ubwc_config_get_data(void);
@@ -108,6 +109,8 @@ static inline u32 qcom_ubwc_swizzle(const struct qcom_ubwc_cfg_data *cfg)
 
 static inline u32 qcom_ubwc_version_tag(const struct qcom_ubwc_cfg_data *cfg)
 {
+	if (cfg->ubwc_enc_version >= UBWC_7_0)
+		return 6;
 	if (cfg->ubwc_enc_version >= UBWC_6_0)
 		return 5;
 	if (cfg->ubwc_enc_version >= UBWC_5_0)
diff --git a/include/linux/soc/ti/k3-ringacc.h b/include/linux/soc/ti/k3-ringacc.h
index 39b022b9259867..7b7fc237c08f5a 100644
--- a/include/linux/soc/ti/k3-ringacc.h
+++ b/include/linux/soc/ti/k3-ringacc.h
@@ -57,13 +57,13 @@ struct k3_ringacc;
 struct k3_ring;
 
 /**
- * enum k3_ring_cfg - RA ring configuration structure
+ * struct k3_ring_cfg - RA ring configuration structure
  *
  * @size: Ring size, number of elements
  * @elm_size: Ring element size
  * @mode: Ring operational mode
  * @flags: Ring configuration flags. Possible values:
- *	 @K3_RINGACC_RING_SHARED: when set allows to request the same ring
+ *	 %K3_RINGACC_RING_SHARED: when set allows to request the same ring
  *	 few times. It's usable when the same ring is used as Free Host PD ring
  *	 for different flows, for example.
  *	 Note: Locking should be done by consumer if required
@@ -88,11 +88,10 @@ struct k3_ring_cfg {
 /**
  * of_k3_ringacc_get_by_phandle - find a RA by phandle property
  * @np: device node
- * @propname: property name containing phandle on RA node
+ * @property: property name containing phandle on RA node
  *
- * Returns pointer on the RA - struct k3_ringacc
- * or -ENODEV if not found,
- * or -EPROBE_DEFER if not yet registered
+ * Return: Pointer to the RA, or ERR_PTR(-ENODEV) if the phandle cannot
+ * be resolved, or ERR_PTR(-EPROBE_DEFER) if the RA is not yet registered.
  */
 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
 						const char *property);
@@ -103,10 +102,8 @@ struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
  * k3_ringacc_request_ring - request ring from ringacc
  * @ringacc: pointer on ringacc
  * @id: ring id or K3_RINGACC_RING_ID_ANY for any general purpose ring
- * @flags:
- *	@K3_RINGACC_RING_USE_PROXY: if set - proxy will be allocated and
- *		used to access ring memory. Sopported only for rings in
- *		Message/Credentials/Queue mode.
+ * @flags: Set %K3_RINGACC_RING_USE_PROXY to allocate a proxy for
+ *	accessing ring memory in Message mode.
  *
  * Returns pointer on the Ring - struct k3_ring
  * or NULL in case of failure.
@@ -126,8 +123,9 @@ int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
  */
 void k3_ringacc_ring_reset(struct k3_ring *ring);
 /**
- * k3_ringacc_ring_reset - ring reset for DMA rings
+ * k3_ringacc_ring_reset_dma - ring reset for DMA rings
  * @ring: pointer on Ring
+ * @occ: occupancy used by the DMA reset quirk, or zero to read it from hardware
  *
  * Resets ring internal state ((hw)occ, (hw)idx). Should be used for rings
  * which are read by K3 UDMA, like TX or Free Host PD rings.
@@ -217,8 +215,8 @@ int k3_ringacc_ring_push(struct k3_ring *ring, void *elem);
  * @ring: pointer on ring
  * @elem: pointer on ring element buffer
  *
- * Push one ring element from the ring head. Size of the ring element is
- * determined by ring configuration &struct k3_ring_cfg elm_size..
+ * Pop one ring element from the ring head. Size of the ring element is
+ * determined by ring configuration &struct k3_ring_cfg elm_size.
  *
  * Returns 0 on success, errno otherwise.
  */
@@ -242,7 +240,7 @@ int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem);
  * @ring: pointer on ring
  * @elem: pointer on ring element buffer
  *
- * Push one ring element from the ring tail. Size of the ring element is
+ * Pop one ring element from the ring tail. Size of the ring element is
  * determined by ring configuration &struct k3_ring_cfg elm_size.
  *
  * Returns 0 on success, errno otherwise.
@@ -256,7 +254,10 @@ u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring);
 struct ti_sci_handle;
 
 /**
- * struct struct k3_ringacc_init_data - Initialization data for DMA rings
+ * struct k3_ringacc_init_data - Initialization data for DMA rings
+ * @tisci: TI SCI firmware handle
+ * @tisci_dev_id: TI SCI device ID of the DMA controller
+ * @num_rings: Total number of available rings
  */
 struct k3_ringacc_init_data {
 	const struct ti_sci_handle *tisci;
diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h
index 707ca6bd1e23df..f710e5932b4b23 100644
--- a/include/linux/soundwire/sdw.h
+++ b/include/linux/soundwire/sdw.h
@@ -618,6 +618,7 @@ struct sdw_bus_params {
  * @update_status: Update Slave status
  * @bus_config: Update the bus config for Slave
  * @port_prep: Prepare the port with parameters
+ * @get_clk_stop_mode: Get the clock stop mode of the Slave
  * @clk_stop: handle imp-def sequences before and after prepare and de-prepare
  */
 struct sdw_slave_ops {
@@ -631,6 +632,7 @@ struct sdw_slave_ops {
 	int (*port_prep)(struct sdw_slave *slave,
 			 struct sdw_prepare_ch *prepare_ch,
 			 enum sdw_port_prep_ops pre_ops);
+	enum sdw_clk_stop_mode (*get_clk_stop_mode)(struct sdw_slave *slave);
 	int (*clk_stop)(struct sdw_slave *slave,
 			enum sdw_clk_stop_mode mode,
 			enum sdw_clk_stop_type type);
@@ -649,6 +651,7 @@ struct sdw_slave_ops {
  * @node: node for bus list
  * @port_ready: Port ready completion flag for each Slave port
  * @m_port_map: static Master port map for each Slave port
+ * @clk_stop_mode: The clock stop mode of the Slave
  * @dev_num: Current Device Number, values can be 0 or dev_num_sticky
  * @dev_num_sticky: one-time static Device Number assigned by Bus
  * @probed: boolean tracking driver state
@@ -683,6 +686,7 @@ struct sdw_slave {
 	struct list_head node;
 	struct completion port_ready[SDW_MAX_PORTS];
 	unsigned int m_port_map[SDW_MAX_PORTS];
+	enum sdw_clk_stop_mode clk_stop_mode;
 	u16 dev_num;
 	u16 dev_num_sticky;
 	bool probed;
diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h
index 9c943500953712..9710f2dc04e298 100644
--- a/include/linux/soundwire/sdw_intel.h
+++ b/include/linux/soundwire/sdw_intel.h
@@ -424,7 +424,7 @@ struct sdw_intel_hw_ops {
 	int (*start_bus)(struct sdw_intel *sdw);
 	int (*start_bus_after_reset)(struct sdw_intel *sdw);
 	int (*start_bus_after_clock_stop)(struct sdw_intel *sdw);
-	int (*stop_bus)(struct sdw_intel *sdw, bool clock_stop);
+	int (*stop_bus)(struct sdw_intel *sdw, bool clock_stop, bool wake_enable);
 
 	int (*link_power_up)(struct sdw_intel *sdw);
 	int (*link_power_down)(struct sdw_intel *sdw);
diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h
index 88d17fce02dcbf..fb4baa0d339821 100644
--- a/include/linux/spi/spi.h
+++ b/include/linux/spi/spi.h
@@ -169,6 +169,12 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg,
  * @cs_inactive: delay to be introduced by the controller after CS is
  *	deasserted. If @cs_change_delay is used from @spi_transfer, then the
  *	two delays will be added up.
+ * @rx_sample_delay_ns: Delay in nanoseconds by which the controller should
+ *	postpone sampling the incoming data, relative to the sampling point it
+ *	uses by default. Describes the board rather than the device, namely the
+ *	flight time of the clock and data signals between controller and
+ *	device, and comes from the "rx-sample-delay-ns" property. Zero when the
+ *	property is absent.
  * @chip_select: Array of physical chipselect, spi->chipselect[i] gives
  *	the corresponding physical CS for logical CS i.
  * @num_chipselect: Number of physical chipselects used.
@@ -235,6 +241,9 @@ struct spi_device {
 	struct spi_delay	cs_hold;
 	struct spi_delay	cs_inactive;
 
+	/* Additional delay before the incoming data is sampled, in ns */
+	u32			rx_sample_delay_ns;
+
 	u8			chip_select[SPI_DEVICE_CS_CNT_MAX];
 	u8			num_chipselect;
 
diff --git a/include/linux/splice.h b/include/linux/splice.h
index 9dec4861d09f62..0e6c955dc6ffed 100644
--- a/include/linux/splice.h
+++ b/include/linux/splice.h
@@ -79,8 +79,8 @@ ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf);
 ssize_t vfs_splice_read(struct file *in, loff_t *ppos,
 			struct pipe_inode_info *pipe, size_t len,
 			unsigned int flags);
-ssize_t splice_direct_to_actor(struct file *file, struct splice_desc *sd,
-			       splice_direct_actor *actor);
+ssize_t vfs_splice_to_actor(struct file *file, loff_t pos, size_t count,
+			    splice_direct_actor *actor, void *private);
 ssize_t do_splice(struct file *in, loff_t *off_in, struct file *out,
 		  loff_t *off_out, size_t len, unsigned int flags);
 ssize_t do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 7d9bc06df98d0b..1a8a465a5650dc 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -36,6 +36,8 @@ static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
 int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key);
 int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name,
 				   struct lock_class_key *key);
+int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name,
+			      struct lock_class_key *key);
 #endif // #ifndef CONFIG_TINY_SRCU
 
 #define init_srcu_struct_fast(ssp) \
@@ -52,6 +54,13 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name,
 	__init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \
 })
 
+#define init_srcu_struct_atomic(ssp) \
+({ \
+	static struct lock_class_key __srcu_key; \
+	\
+	__init_srcu_struct_atomic((ssp), #ssp, &__srcu_key); \
+})
+
 #define __SRCU_DEP_MAP_INIT(srcu_name)	.dep_map = { .name = #srcu_name },
 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
@@ -64,6 +73,7 @@ static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
 #ifndef CONFIG_TINY_SRCU
 int init_srcu_struct_fast(struct srcu_struct *ssp);
 int init_srcu_struct_fast_updown(struct srcu_struct *ssp);
+int init_srcu_struct_atomic(struct srcu_struct *ssp);
 #endif // #ifndef CONFIG_TINY_SRCU
 
 #define __SRCU_DEP_MAP_INIT(srcu_name)
@@ -77,14 +87,18 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp);
 })
 
 /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
-#define SRCU_READ_FLAVOR_NORMAL		0x1		// srcu_read_lock().
-#define SRCU_READ_FLAVOR_NMI		0x2		// srcu_read_lock_nmisafe().
-//					0x4		// SRCU-lite is no longer with us.
-#define SRCU_READ_FLAVOR_FAST		0x4		// srcu_read_lock_fast(), also NMI-safe.
-#define SRCU_READ_FLAVOR_FAST_UPDOWN	0x8		// srcu_read_lock_fast_updown().
+#define SRCU_READ_FLAVOR_NORMAL		0x01		// srcu_read_lock().
+#define SRCU_READ_FLAVOR_NMI		0x02		// srcu_read_lock_nmisafe().
+//					0x04		// SRCU-lite is no longer with us.
+#define SRCU_READ_FLAVOR_FAST		0x04		// srcu_read_lock_fast(), also NMI-safe.
+#define SRCU_READ_FLAVOR_FAST_UPDOWN	0x08		// srcu_read_lock_fast_updown().
+#define SRCU_READ_FLAVOR_ATOMIC		0x10		// srcu_read_lock_atomic().
 #define SRCU_READ_FLAVOR_ALL		(SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
-					 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
+					 SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN | \
+					 SRCU_READ_FLAVOR_ATOMIC)
 						// All of the above.
+#define SRCU_READ_FLAVOR_PREDEF		(SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_ATOMIC)
+						// Flavors special DEFINE_SRCU() flavors.
 #define SRCU_READ_FLAVOR_SLOWGP		(SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN)
 						// Flavors requiring synchronize_rcu()
 						// instead of smp_mb().
@@ -102,6 +116,7 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
 		void (*func)(struct rcu_head *head));
 void cleanup_srcu_struct(struct srcu_struct *ssp);
 void synchronize_srcu(struct srcu_struct *ssp);
+void synchronize_srcu_atomic(struct srcu_struct *ssp);
 
 #define SRCU_GET_STATE_COMPLETED 0x1
 
@@ -306,6 +321,45 @@ static inline int srcu_read_lock(struct srcu_struct *ssp)
 	return retval;
 }
 
+/**
+ * srcu_read_lock_atomic - register a new reader promising an atomic section
+ * @ssp: srcu_struct in which to register the new reader.
+ *
+ * As srcu_read_lock(), but the caller promises that the read-side
+ * critical section never sleeps and never blocks on anything which
+ * may itself depend on memory allocation to make progress. Preemption
+ * is disabled for the duration, which both enforces that promise (any
+ * sleepable call in the section will splat on every configuration)
+ * and bounds the section so that the update side may spin rather
+ * than sleep when waiting for readers: see synchronize_srcu_atomic().
+ *
+ * The lock and matching srcu_read_unlock_atomic() must be invoked on
+ * the same CPU, from the same context; passing the return value to
+ * another task is not permitted for this flavor.
+ */
+static inline int srcu_read_lock_atomic(struct srcu_struct *ssp)
+	__acquires_shared(ssp)
+{
+	int retval;
+
+	preempt_disable();
+	/*
+	 * Arm might_sleep() to catch even a *potentially* sleeping call
+	 * in the section, not just an actual schedule: the atomic-domain
+	 * promise must hold on every path, contended or not. In hardirq,
+	 * softirq, or NMI the annotation would land on the interrupted
+	 * task, and can also result in data races against that task's
+	 * own non_block_start()/non_block_end() invocations; it is also
+	 * redundant there, so skip it.
+	 */
+	if (in_task())
+		non_block_start();
+	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC);
+	retval = __srcu_read_lock(ssp);
+	srcu_lock_acquire(&ssp->dep_map);
+	return retval;
+}
+
 /**
  * srcu_read_lock_fast - register a new reader for an SRCU-protected structure.
  * @ssp: srcu_struct in which to register the new reader.
@@ -498,6 +552,23 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
 	__srcu_read_unlock(ssp, idx);
 }
 
+/**
+ * srcu_read_unlock_atomic - unregister an atomic-section reader
+ * @ssp: srcu_struct from which to unregister the old reader.
+ * @idx: return value from corresponding srcu_read_lock_atomic().
+ */
+static inline void srcu_read_unlock_atomic(struct srcu_struct *ssp, int idx)
+	__releases_shared(ssp)
+{
+	WARN_ON_ONCE(idx & ~0x1);
+	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC);
+	srcu_lock_release(&ssp->dep_map);
+	__srcu_read_unlock(ssp, idx);
+	if (in_task())
+		non_block_end();
+	preempt_enable();
+}
+
 /**
  * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure.
  * @ssp: srcu_struct in which to unregister the old reader.
diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index fbcf13bc12d15e..47a368f945e349 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -12,12 +12,14 @@
 #define _LINUX_SRCU_TINY_H
 
 #include <linux/irq_work_types.h>
+#include <linux/llist.h>
 #include <linux/swait.h>
 
 struct srcu_struct {
 	short srcu_lock_nesting[2];	/* srcu_read_lock() nesting depth. */
 	u8 srcu_gp_running;		/* GP workqueue running? */
 	u8 srcu_gp_waiting;		/* GP waiting for readers? */
+	u8 srcu_atomic_gp_flag;		/* Serialize atomic GP work.*/
 	unsigned long srcu_idx;		/* Current reader array element in bit 0x2. */
 	unsigned long srcu_idx_max;	/* Furthest future srcu_idx request. */
 	struct swait_queue_head srcu_wq;
@@ -26,6 +28,8 @@ struct srcu_struct {
 	struct rcu_head **srcu_cb_tail;	/* Pending callbacks: Tail. */
 	struct work_struct srcu_work;	/* For driving grace periods. */
 	struct irq_work srcu_irq_work;	/* Defer schedule_work() to irq work. */
+	struct llist_head defer_cbs;	/* Callbacks deferred on re-entry. */
+	struct irq_work defer_iw;	/* Re-issues defer_cbs later. */
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	struct lockdep_map dep_map;
 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
@@ -33,6 +37,7 @@ struct srcu_struct {
 
 void srcu_drive_gp(struct work_struct *wp);
 void srcu_tiny_irq_work(struct irq_work *irq_work);
+void srcu_defer_drain(struct irq_work *irq_work);
 
 #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored)	\
 {									\
@@ -40,6 +45,9 @@ void srcu_tiny_irq_work(struct irq_work *irq_work);
 	.srcu_cb_tail = &name.srcu_cb_head,				\
 	.srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp),	\
 	.srcu_irq_work = { .func = srcu_tiny_irq_work },		\
+	.defer_cbs = LLIST_HEAD_INIT(name.defer_cbs),			\
+	.defer_iw = { .node = { .u_flags = IRQ_WORK_HARD_IRQ },		\
+		      .func = srcu_defer_drain },			\
 	__SRCU_DEP_MAP_INIT(name)					\
 }
 
@@ -57,15 +65,20 @@ void srcu_tiny_irq_work(struct irq_work *irq_work);
 #define DEFINE_SRCU_FAST_UPDOWN(name) DEFINE_SRCU(name)
 #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \
 	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
+#define DEFINE_SRCU_ATOMIC(name) DEFINE_SRCU(name)
+#define DEFINE_STATIC_SRCU_ATOMIC(name) \
+	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
 
 // Dummy structure for srcu_notifier_head.
 struct srcu_usage { };
 #define __SRCU_USAGE_INIT(name) { }
 #define __init_srcu_struct_fast __init_srcu_struct
 #define __init_srcu_struct_fast_updown __init_srcu_struct
+#define __init_srcu_struct_atomic __init_srcu_struct
 #ifndef CONFIG_DEBUG_LOCK_ALLOC
 #define init_srcu_struct_fast init_srcu_struct
 #define init_srcu_struct_fast_updown init_srcu_struct
+#define init_srcu_struct_atomic init_srcu_struct
 #endif // #ifndef CONFIG_DEBUG_LOCK_ALLOC
 
 void synchronize_srcu(struct srcu_struct *ssp);
@@ -131,10 +144,7 @@ static inline void synchronize_srcu_expedited(struct srcu_struct *ssp)
 	synchronize_srcu(ssp);
 }
 
-static inline void srcu_barrier(struct srcu_struct *ssp)
-{
-	synchronize_srcu(ssp);
-}
+void srcu_barrier(struct srcu_struct *ssp);
 
 static inline void srcu_expedite_current(struct srcu_struct *ssp) { }
 #define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 75e54e4f963fac..93b76e54389432 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -13,6 +13,8 @@
 
 #include <linux/rcu_node_tree.h>
 #include <linux/completion.h>
+#include <linux/irq_work_types.h>
+#include <linux/llist.h>
 
 struct srcu_node;
 struct srcu_struct;
@@ -41,6 +43,8 @@ struct srcu_data {
 	bool srcu_cblist_invoking;		/* Invoking these CBs? */
 	struct timer_list delay_work;		/* Delay for CB invoking */
 	struct work_struct work;		/* Context for CB invoking. */
+	struct llist_head defer_cbs;		/* Callbacks deferred on re-entry. */
+	struct llist_node defer_link;		/* Links onto the per-CPU deferral drain list */
 	struct rcu_head srcu_barrier_head;	/* For srcu_barrier() use. */
 	struct rcu_head srcu_ec_head;		/* For srcu_expedite_current() use. */
 	int srcu_ec_state;			/*  State for srcu_expedite_current(). */
@@ -76,6 +80,7 @@ struct srcu_usage {
 	struct mutex srcu_cb_mutex;		/* Serialize CB preparation. */
 	raw_spinlock_t __private lock;		/* Protect counters and size state. */
 	struct mutex srcu_gp_mutex;		/* Serialize GP work. */
+	atomic_t srcu_atomic_gp_flag;		/* Serialize atomic GP work. */
 	unsigned long srcu_gp_seq;		/* Grace-period seq #. */
 	unsigned long srcu_gp_seq_needed;	/* Latest gp_seq needed. */
 	unsigned long srcu_gp_seq_needed_exp;	/* Furthest future exp GP. */
@@ -209,6 +214,12 @@ struct srcu_struct {
  * instead of smp_mb(), and given that the first (for example)
  * srcu_read_lock_fast() might race with the first synchronize_srcu(),
  * this different must be specified at initialization time.
+ *
+ * If you use any of the DEFINE_SRCU() functions within a module, the
+ * module-entry code will invoke init_srcu_struct() and the module-exit
+ * code will invoke cleanup_srcu_struct().  This means that if your module
+ * passes the resulting srcu_struct structure to call_srcu(), you will
+ * need to also pass this structure to srcu_barrier() prior to module exit.
  */
 #ifdef MODULE
 # define __DEFINE_SRCU(name, fast, is_static)							\
@@ -225,14 +236,16 @@ struct srcu_struct {
 	is_static struct srcu_struct name =							\
 		__SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data, fast)
 #endif
-#define DEFINE_SRCU(name)		__DEFINE_SRCU(name, 0, /* not static */)
+#define DEFINE_SRCU(name)		__DEFINE_SRCU(name, 0, /* !static */)
 #define DEFINE_STATIC_SRCU(name)	__DEFINE_SRCU(name, 0, static)
-#define DEFINE_SRCU_FAST(name)		__DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* not static */)
+#define DEFINE_SRCU_FAST(name)		__DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* !static */)
 #define DEFINE_STATIC_SRCU_FAST(name)	__DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, static)
 #define DEFINE_SRCU_FAST_UPDOWN(name)	__DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, \
-						      /* not static */)
+						      /* !static */)
 #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \
 					__DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, static)
+#define DEFINE_SRCU_ATOMIC(name)	__DEFINE_SRCU(name, SRCU_READ_FLAVOR_ATOMIC, /* !static */)
+#define DEFINE_STATIC_SRCU_ATOMIC(name)	__DEFINE_SRCU(name, SRCU_READ_FLAVOR_ATOMIC, static)
 
 int __srcu_read_lock(struct srcu_struct *ssp) __acquires_shared(ssp);
 void synchronize_srcu_expedited(struct srcu_struct *ssp);
diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h
index 4430b967abdeb1..00be2df63d2206 100644
--- a/include/linux/stmmac.h
+++ b/include/linux/stmmac.h
@@ -242,6 +242,8 @@ struct plat_stmmacenet_data {
 	 * that phylink uses.
 	 */
 	phy_interface_t phy_interface;
+	bool has_internal_tx_delay;
+	bool has_internal_rx_delay;
 	struct stmmac_mdio_bus_data *mdio_bus_data;
 	struct device_node *phy_node;
 	struct device_node *mdio_node;
diff --git a/include/linux/string.h b/include/linux/string.h
index 5702daca4326b7..6cb5cdd01158b2 100644
--- a/include/linux/string.h
+++ b/include/linux/string.h
@@ -278,6 +278,19 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
 }
 #endif
 
+#ifndef memcpy_nontemporal
+/*
+ * memcpy_nontemporal() requests a non-temporal copy when the
+ * architecture has a suitable backend. Architectures without a
+ * specialized backend fall back to memcpy(). Keep this as a
+ * function-like macro so the compiler can still see the original
+ * memcpy() call site and preserve the usual FORTIFY coverage when
+ * object sizes remain visible there, while keeping the API void.
+ */
+#define memcpy_nontemporal(dst, src, len) \
+	((void)memcpy(dst, src, len))
+#endif
+
 void *memchr_inv(const void *s, int c, size_t n);
 char *strreplace(char *str, char old, char new);
 
diff --git a/include/linux/string_choices.h b/include/linux/string_choices.h
index ee84087d4b26bc..a2eba454004f42 100644
--- a/include/linux/string_choices.h
+++ b/include/linux/string_choices.h
@@ -65,6 +65,12 @@ static inline const char *str_read_write(bool v)
 }
 #define str_write_read(v)		str_read_write(!(v))
 
+static inline const char *str_supported_unsupported(bool v)
+{
+	return v ? "supported" : "unsupported";
+}
+#define str_unsupported_supported(v)	str_supported_unsupported(!(v))
+
 static inline const char *str_true_false(bool v)
 {
 	return v ? "true" : "false";
diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h
index da2a2531e1106d..2af222f3ea2c2e 100644
--- a/include/linux/sunrpc/svc_xprt.h
+++ b/include/linux/sunrpc/svc_xprt.h
@@ -37,6 +37,9 @@ struct svc_xprt_class {
 	struct list_head	xcl_list;
 	u32			xcl_max_payload;
 	int			xcl_ident;
+	u32			xcl_flags;
+/* Set only on classes whose xpo_has_wspace() reads xpt_reserved */
+#define SVC_XPRT_FLAG_WSPACE_RESERVE	BIT(0)
 };
 
 /*
@@ -59,7 +62,7 @@ struct svc_xprt {
 	unsigned long		xpt_flags;
 
 	struct svc_serv		*xpt_server;	/* service for transport */
-	atomic_t    	    	xpt_reserved;	/* space on outq that is rsvd */
+	atomic_t		xpt_reserved;	/* outq space rsvd, UDP only */
 	atomic_t		xpt_nr_rqsts;	/* Number of requests */
 	struct mutex		xpt_mutex;	/* to serialize sending data */
 	spinlock_t		xpt_lock;	/* protects sk_deferred
diff --git a/include/linux/suspend.h b/include/linux/suspend.h
index b02876f1ae38ac..b1a1db042a452d 100644
--- a/include/linux/suspend.h
+++ b/include/linux/suspend.h
@@ -2,7 +2,6 @@
 #ifndef _LINUX_SUSPEND_H
 #define _LINUX_SUSPEND_H
 
-#include <linux/swap.h>
 #include <linux/notifier.h>
 #include <linux/init.h>
 #include <linux/pm.h>
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 5658a1634b85ea..517b70c46b9e6b 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -10,7 +10,6 @@
 #include <linux/sched.h>
 #include <linux/node.h>
 #include <linux/fs.h>
-#include <linux/pagemap.h>
 #include <linux/atomic.h>
 #include <linux/page-flags.h>
 #include <uapi/linux/mempolicy.h>
@@ -25,12 +24,6 @@
 #define SWAP_FLAGS_VALID	(SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \
 				 SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \
 				 SWAP_FLAG_DISCARD_PAGES)
-
-static inline int current_is_kswapd(void)
-{
-	return current->flags & PF_KSWAPD;
-}
-
 /*
  * MAX_SWAPFILES defines the maximum number of swaptypes: things which can
  * be swapped to.  The swap type and the offset into that swap type are
@@ -246,7 +239,7 @@ struct swap_info_struct {
 	struct plist_node list;		/* entry in swap_active_head */
 	signed char	type;		/* strange name for an index */
 	unsigned int	max;		/* size of this swap device */
-	struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
+	struct swap_cluster_info *cluster_info; /* array, one entry per cluster */
 	struct list_head free_clusters; /* free clusters list */
 	struct list_head full_clusters; /* full clusters list */
 	struct list_head nonfull_clusters[SWAP_NR_ORDERS];
@@ -278,15 +271,39 @@ struct swap_info_struct {
 	const struct swap_ops *ops;
 };
 
-static inline swp_entry_t page_swap_entry(struct page *page)
+/**
+ * folio_swap_entry - Return the swap entry at a page index within a folio.
+ * @folio: The folio.
+ * @idx: The index of the page within the folio.
+ *
+ * A folio in the swap cache occupies folio_nr_pages() contiguous swap
+ * entries starting at folio->swap. The caller must ensure the folio is
+ * in the swap cache and that @idx is within the folio.
+ */
+static inline
+swp_entry_t folio_swap_entry(const struct folio *folio, unsigned long idx)
 {
-	struct folio *folio = page_folio(page);
 	swp_entry_t entry = folio->swap;
 
-	entry.val += folio_page_idx(folio, page);
+	VM_WARN_ON_ONCE_FOLIO(idx >= folio_nr_pages(folio), folio);
+	entry.val += idx;
 	return entry;
 }
 
+/**
+ * folio_page_swap_entry - Return the swap entry of a page in a folio.
+ * @folio: The folio containing @page.
+ * @page: A page within @folio.
+ *
+ * The caller must ensure the folio is in the swap cache and that @page
+ * is part of @folio.
+ */
+static inline swp_entry_t folio_page_swap_entry(const struct folio *folio,
+		const struct page *page)
+{
+	return folio_swap_entry(folio, folio_page_idx(folio, page));
+}
+
 /* linux/mm/page_alloc.c */
 extern unsigned long totalreserve_pages;
 
@@ -318,6 +335,9 @@ static inline bool lru_cache_disabled(void)
 
 extern unsigned long shrink_all_memory(unsigned long nr_pages);
 long remove_mapping(struct address_space *mapping, struct folio *folio);
+long remove_mapping_set_shadow(struct address_space *mapping,
+			       struct folio *folio,
+			       struct mem_cgroup *target_memcg);
 
 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
 extern int reclaim_register_node(struct node *node);
@@ -339,6 +359,7 @@ void check_move_unevictable_folios(struct folio_batch *fbatch);
 
 extern void __meminit kswapd_run(int nid);
 extern void __meminit kswapd_stop(int nid);
+bool current_is_kswapd(void);
 
 #ifdef CONFIG_SWAP
 int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
@@ -402,6 +423,8 @@ void swap_put_entries_direct(swp_entry_t entry, int nr);
  */
 bool folio_free_swap(struct folio *folio);
 
+void swap_writeback_dropbehind_folio(struct folio *folio);
+
 /* Allocate / free (hibernation) exclusive entries */
 swp_entry_t swap_alloc_hibernation_slot(int type);
 void swap_free_hibernation_slot(swp_entry_t entry);
@@ -412,6 +435,7 @@ static inline void put_swap_device(struct swap_info_struct *si)
 }
 
 #else /* CONFIG_SWAP */
+static inline void swap_writeback_dropbehind_folio(struct folio *folio) {}
 static inline struct swap_info_struct *get_swap_device(swp_entry_t entry)
 {
 	return NULL;
@@ -508,8 +532,9 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_p
 	__mem_cgroup_uncharge_swap(id, nr_pages);
 }
 
-extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
-extern bool mem_cgroup_swap_full(struct folio *folio);
+long mem_cgroup_get_folio_swap_margin(const struct folio *folio);
+long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg);
+bool mem_cgroup_swap_full(const struct folio *folio);
 #else
 static inline int mem_cgroup_try_charge_swap(struct folio *folio)
 {
@@ -521,12 +546,17 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id,
 {
 }
 
-static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
+static inline long mem_cgroup_get_folio_swap_margin(const struct folio *folio)
+{
+	return PAGE_COUNTER_MAX;
+}
+
+static inline long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg)
 {
 	return get_nr_swap_pages();
 }
 
-static inline bool mem_cgroup_swap_full(struct folio *folio)
+static inline bool mem_cgroup_swap_full(const struct folio *folio)
 {
 	return vm_swap_full();
 }
diff --git a/include/linux/swapops.h b/include/linux/swapops.h
index e7d0d529f3e0f9..603f9e3f909cd1 100644
--- a/include/linux/swapops.h
+++ b/include/linux/swapops.h
@@ -261,11 +261,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page)
 	return swp_entry(SWP_HWPOISON, page_to_pfn(page));
 }
 
-static inline int is_hwpoison_entry(swp_entry_t entry)
-{
-	return swp_type(entry) == SWP_HWPOISON;
-}
-
 #else
 
 static inline swp_entry_t make_hwpoison_entry(struct page *page)
@@ -273,10 +268,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page)
 	return swp_entry(0, 0);
 }
 
-static inline int is_hwpoison_entry(swp_entry_t swp)
-{
-	return 0;
-}
 #endif
 
 typedef unsigned long pte_marker;
diff --git a/include/linux/thermal.h b/include/linux/thermal.h
index 083b4f5339331d..306ad17aed89c1 100644
--- a/include/linux/thermal.h
+++ b/include/linux/thermal.h
@@ -293,8 +293,9 @@ struct device *thermal_zone_device(struct thermal_zone_device *tzd);
 void thermal_zone_device_update(struct thermal_zone_device *,
 				enum thermal_notify_event);
 
-struct thermal_cooling_device *thermal_cooling_device_register(const char *,
-		void *, const struct thermal_cooling_device_ops *);
+struct thermal_cooling_device *thermal_cooling_device_create(
+			struct device *parent, const char *type, void *devdata,
+			const struct thermal_cooling_device_ops *ops);
 
 struct thermal_cooling_device *
 devm_thermal_cooling_device_register(struct device *dev, const char *type, void *devdata,
@@ -340,9 +341,9 @@ static inline void thermal_zone_device_update(struct thermal_zone_device *tz,
 					      enum thermal_notify_event event)
 { }
 
-static inline struct thermal_cooling_device *
-thermal_cooling_device_register(const char *type, void *devdata,
-	const struct thermal_cooling_device_ops *ops)
+static inline struct thermal_cooling_device *thermal_cooling_device_create(
+			struct device *parent, const char *type, void *devdata,
+			const struct thermal_cooling_device_ops *ops)
 { return ERR_PTR(-ENODEV); }
 
 static inline struct thermal_cooling_device *
@@ -391,4 +392,12 @@ static inline void thermal_pm_prepare(void) {}
 static inline void thermal_pm_complete(void) {}
 #endif /* CONFIG_THERMAL */
 
+static inline struct thermal_cooling_device *thermal_cooling_device_register(
+				const char *type, void *devdata,
+				const struct thermal_cooling_device_ops *ops)
+{
+	return thermal_cooling_device_create(NULL, type, devdata, ops);
+}
+
+
 #endif /* __THERMAL_H__ */
diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h
index d48623fda79ba4..57502da290805e 100644
--- a/include/linux/thunderbolt.h
+++ b/include/linux/thunderbolt.h
@@ -22,6 +22,7 @@ struct device;
 #include <linux/device.h>
 #include <linux/idr.h>
 #include <linux/list.h>
+#include <linux/lockdep.h>
 #include <linux/mutex.h>
 #include <linux/device-id/tb.h>
 #include <linux/pci.h>
@@ -369,7 +370,7 @@ int tb_xdomain_request(struct tb_xdomain *xd, const void *request,
  * @uuid: XDomain messages with this UUID are dispatched to this handler
  * @callback: Callback called with the XDomain message. Returning %1
  *	      here tells the XDomain core that the message was handled
- *	      by this handler and should not be forwared to other
+ *	      by this handler and should not be forwarded to other
  *	      handlers.
  * @data: Data passed with the callback
  * @list: Handlers are linked using this
@@ -506,6 +507,7 @@ void tb_service_properties_changed(struct tb_service *svc);
  * @iobase: MMIO space of the NHI
  * @tx_rings: All Tx rings available on this host controller
  * @rx_rings: All Rx rings available on this host controller
+ * @interrupt_mask: Shadow copy of the ring interrupt mask register
  * @going_away: The host controller device is about to disappear so when
  *		this flag is set, avoid touching the hardware anymore.
  * @iommu_dma_protection: An IOMMU will isolate external-facing ports.
@@ -527,6 +529,7 @@ struct tb_nhi {
 	void __iomem *iobase;
 	struct tb_ring **tx_rings;
 	struct tb_ring **rx_rings;
+	u32 *interrupt_mask;
 	bool going_away;
 	bool iommu_dma_protection;
 	struct work_struct interrupt_work;
@@ -552,6 +555,8 @@ struct tb_nhi {
  * @work: Interrupt work structure
  * @is_tx: Is the ring Tx or Rx
  * @running: Is the ring running
+ * @notify_pending: Controller has not been notified about the posted
+ *		    descriptors yet
  * @irq: MSI-X irq number if the ring uses MSI-X. %0 otherwise.
  * @vector: MSI-X vector number the ring uses (only set if @irq is > 0)
  * @flags: Ring specific flags
@@ -565,6 +570,7 @@ struct tb_nhi {
  * @interval_nsec: Interval counter if interrupt throttling is to be
  *		   used with this ring (in ns)
  * @wait: Used to signal that the ring may be empty now
+ * @lock_key: Lock validator class key per-ring
  */
 struct tb_ring {
 	spinlock_t lock;
@@ -580,6 +586,7 @@ struct tb_ring {
 	struct work_struct work;
 	bool is_tx:1;
 	bool running:1;
+	bool notify_pending:1;
 	int irq;
 	u8 vector;
 	unsigned int flags;
@@ -590,6 +597,7 @@ struct tb_ring {
 	void *poll_data;
 	unsigned int interval_nsec;
 	wait_queue_head_t wait;
+	struct lock_class_key lock_key;
 };
 
 /* Leave ring interrupt enabled on suspend */
@@ -669,7 +677,8 @@ bool tb_ring_flush(struct tb_ring *ring, unsigned int timeout_msec);
 void tb_ring_stop(struct tb_ring *ring);
 void tb_ring_free(struct tb_ring *ring);
 
-int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame);
+int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame, bool more);
+void tb_ring_notify(struct tb_ring *ring);
 
 /**
  * tb_ring_rx() - enqueue a frame on an RX ring
@@ -690,7 +699,24 @@ int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame);
 static inline int tb_ring_rx(struct tb_ring *ring, struct ring_frame *frame)
 {
 	WARN_ON(ring->is_tx);
-	return __tb_ring_enqueue(ring, frame);
+	return __tb_ring_enqueue(ring, frame, false);
+}
+
+/**
+ * tb_ring_rx_more() - enqueue a frame on an RX ring without notifying
+ * @ring: Ring to enqueue the frame
+ * @frame: Frame to enqueue
+ *
+ * Same as tb_ring_rx() but does not notify the controller about the
+ * enqueued frame. The caller must call tb_ring_notify() once it is done
+ * enqueuing frames.
+ *
+ * Return: %-ESHUTDOWN if tb_ring_stop() has been called, %0 otherwise.
+ */
+static inline int tb_ring_rx_more(struct tb_ring *ring, struct ring_frame *frame)
+{
+	WARN_ON(ring->is_tx);
+	return __tb_ring_enqueue(ring, frame, true);
 }
 
 /**
@@ -711,11 +737,27 @@ static inline int tb_ring_rx(struct tb_ring *ring, struct ring_frame *frame)
 static inline int tb_ring_tx(struct tb_ring *ring, struct ring_frame *frame)
 {
 	WARN_ON(!ring->is_tx);
-	return __tb_ring_enqueue(ring, frame);
+	return __tb_ring_enqueue(ring, frame, false);
+}
+
+/**
+ * tb_ring_tx_more() - enqueue a frame on a TX ring without notifying
+ * @ring: Ring to enqueue the frame
+ * @frame: Frame to enqueue
+ *
+ * Same as tb_ring_rx_more() but for TX ring.
+ *
+ * Return: %-ESHUTDOWN if tb_ring_stop() has been called, %0 otherwise.
+ */
+static inline int tb_ring_tx_more(struct tb_ring *ring, struct ring_frame *frame)
+{
+	WARN_ON(!ring->is_tx);
+	return __tb_ring_enqueue(ring, frame, true);
 }
 
 /* Used only when the ring is in polling mode */
 struct ring_frame *tb_ring_poll(struct tb_ring *ring);
+bool tb_ring_poll_pending(struct tb_ring *ring);
 void tb_ring_poll_complete(struct tb_ring *ring);
 
 int tb_ring_throttling(struct tb_ring *ring, unsigned int interval_nsec);
diff --git a/include/linux/torture.h b/include/linux/torture.h
index c9b47d13830295..1b6d5be641f8d5 100644
--- a/include/linux/torture.h
+++ b/include/linux/torture.h
@@ -98,6 +98,7 @@ void torture_shutdown_absorb(const char *title);
 int torture_shutdown_init(int ssecs, void (*cleanup)(void));
 
 /* Task stuttering, which forces load/no-load transitions. */
+bool stutter_will_wait(void);
 bool stutter_wait(const char *title);
 int torture_stutter_init(int s, int sgap);
 
@@ -131,8 +132,11 @@ void _torture_stop_kthread(char *m, struct task_struct **tp);
 #endif
 void torture_sched_set_normal(struct task_struct *t, int nice);
 
-#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST)
+#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_ENABLED(CONFIG_HAZPTR_TORTURE_TEST)
 long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn);
 #endif
 
+/* Atomic per-CPU counters. */
+s64 torture_sum_pcpu_atomic_long(atomic_long_t __percpu *pcp);
+
 #endif /* __LINUX_TORTURE_H */
diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h
index c8ae1e1f5e721d..e4cc2d4497bcfa 100644
--- a/include/linux/trace_remote_event.h
+++ b/include/linux/trace_remote_event.h
@@ -3,6 +3,8 @@
 #ifndef _LINUX_TRACE_REMOTE_EVENTS_H
 #define _LINUX_TRACE_REMOTE_EVENTS_H
 
+#include <linux/types.h>
+
 struct trace_remote;
 struct trace_event_fields;
 struct trace_seq;
diff --git a/include/linux/tty.h b/include/linux/tty.h
index 0a46e4054dec78..833bb7b97cebbb 100644
--- a/include/linux/tty.h
+++ b/include/linux/tty.h
@@ -168,6 +168,7 @@ struct tty_operations;
  * @write_wait: concurrent writers are waiting in this queue until they are
  *		allowed to write
  * @read_wait: readers wait for data in this queue
+ * @break_wait: wait queue for timed breaks
  * @hangup_work: normally a work to perform a hangup (do_tty_hangup()); while
  *		 freeing the tty, (re)used to release_one_tty()
  * @disc_data: pointer to @ldisc's private data (e.g. to &struct n_tty_data)
@@ -230,6 +231,7 @@ struct tty_struct {
 	struct fasync_struct *fasync;
 	wait_queue_head_t write_wait;
 	wait_queue_head_t read_wait;
+	wait_queue_head_t break_wait;
 	struct work_struct hangup_work;
 	void *disc_data;
 	void *driver_data;
diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h
index 23cad403bb8f5e..8d22c59c6f15b4 100644
--- a/include/linux/tty_port.h
+++ b/include/linux/tty_port.h
@@ -245,7 +245,8 @@ bool tty_port_carrier_raised(struct tty_port *port);
 void tty_port_raise_dtr_rts(struct tty_port *port);
 void tty_port_lower_dtr_rts(struct tty_port *port);
 void tty_port_hangup(struct tty_port *port);
-void __tty_port_tty_hangup(struct tty_port *port, bool check_clocal, bool async);
+void tty_port_tty_hangup(struct tty_port *port, bool check_clocal);
+void tty_port_tty_vhangup(struct tty_port *port);
 void tty_port_tty_wakeup(struct tty_port *port);
 int tty_port_block_til_ready(struct tty_port *port, struct tty_struct *tty,
 		struct file *filp);
@@ -264,25 +265,6 @@ static inline int tty_port_users(struct tty_port *port)
 	return port->count + port->blocked_open;
 }
 
-/**
- * tty_port_tty_hangup - helper to hang up a tty asynchronously
- * @port: tty port
- * @check_clocal: hang only ttys with %CLOCAL unset?
- */
-static inline void tty_port_tty_hangup(struct tty_port *port, bool check_clocal)
-{
-	__tty_port_tty_hangup(port, check_clocal, true);
-}
-
-/**
- * tty_port_tty_vhangup - helper to hang up a tty synchronously
- * @port: tty port
- */
-static inline void tty_port_tty_vhangup(struct tty_port *port)
-{
-	__tty_port_tty_hangup(port, false, false);
-}
-
 #ifdef CONFIG_TTY
 void tty_kref_put(struct tty_struct *tty);
 __DEFINE_CLASS_IS_CONDITIONAL(tty_port_tty, true);
diff --git a/include/linux/ucs2_string.h b/include/linux/ucs2_string.h
index c499ae809c7d99..74f23ca5a96787 100644
--- a/include/linux/ucs2_string.h
+++ b/include/linux/ucs2_string.h
@@ -14,7 +14,16 @@ ssize_t ucs2_strscpy(ucs2_char_t *dst, const ucs2_char_t *src, size_t count);
 int ucs2_strncmp(const ucs2_char_t *a, const ucs2_char_t *b, size_t len);
 
 unsigned long ucs2_utf8size(const ucs2_char_t *src);
+unsigned long
+ucs2_as_utf8_l(u8 *dest, const ucs2_char_t *src, unsigned long limit,
+	       unsigned long maxlength);
+
+static inline
 unsigned long ucs2_as_utf8(u8 *dest, const ucs2_char_t *src,
-			   unsigned long maxlength);
+			   unsigned long maxlength)
+{
+	return ucs2_as_utf8_l(dest, src, ucs2_strnlen(src, maxlength),
+			      maxlength);
+}
 
 #endif /* _LINUX_UCS2_STRING_H_ */
diff --git a/include/linux/uidgid.h b/include/linux/uidgid.h
index 2dc767e08f5447..02403629b49f1f 100644
--- a/include/linux/uidgid.h
+++ b/include/linux/uidgid.h
@@ -130,9 +130,9 @@ static inline bool kgid_has_mapping(struct user_namespace *ns, kgid_t gid)
 	return from_kgid(ns, gid) != (gid_t) -1;
 }
 
-u32 map_id_down(struct uid_gid_map *map, u32 id);
-u32 map_id_up(struct uid_gid_map *map, u32 id);
-u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count);
+u32 map_id_down(const struct uid_gid_map *map, u32 id);
+u32 map_id_up(const struct uid_gid_map *map, u32 id);
+u32 map_id_range_up(const struct uid_gid_map *map, u32 id, u32 count);
 
 #else
 
@@ -182,17 +182,17 @@ static inline bool kgid_has_mapping(struct user_namespace *ns, kgid_t gid)
 	return gid_valid(gid);
 }
 
-static inline u32 map_id_down(struct uid_gid_map *map, u32 id)
+static inline u32 map_id_down(const struct uid_gid_map *map, u32 id)
 {
 	return id;
 }
 
-static inline u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count)
+static inline u32 map_id_range_up(const struct uid_gid_map *map, u32 id, u32 count)
 {
 	return id;
 }
 
-static inline u32 map_id_up(struct uid_gid_map *map, u32 id)
+static inline u32 map_id_up(const struct uid_gid_map *map, u32 id)
 {
 	return id;
 }
diff --git a/include/linux/ulpi/driver.h b/include/linux/ulpi/driver.h
index c668d9c8d87690..c899197e0aa8c5 100644
--- a/include/linux/ulpi/driver.h
+++ b/include/linux/ulpi/driver.h
@@ -51,8 +51,8 @@ struct ulpi_driver {
 /*
  * use a macro to avoid include chaining to get THIS_MODULE
  */
-#define ulpi_register_driver(drv) __ulpi_register_driver(drv, THIS_MODULE)
-int __ulpi_register_driver(struct ulpi_driver *drv, struct module *module);
+#define ulpi_register_driver(drv) __ulpi_register_driver(drv, THIS_MODULE, KBUILD_MODNAME)
+int __ulpi_register_driver(struct ulpi_driver *drv, struct module *module, const char *mod_name);
 void ulpi_unregister_driver(struct ulpi_driver *drv);
 
 #define module_ulpi_driver(__ulpi_driver) \
diff --git a/include/linux/usb.h b/include/linux/usb.h
index 49ab8dbb885f65..4377eeae541e69 100644
--- a/include/linux/usb.h
+++ b/include/linux/usb.h
@@ -1369,8 +1369,10 @@ extern void usb_deregister(struct usb_driver *);
 	module_driver(__usb_driver, usb_register, \
 		       usb_deregister)
 
-extern int usb_register_device_driver(struct usb_device_driver *,
-			struct module *);
+#define usb_register_device_driver(new_udriver) \
+	__usb_register_device_driver(new_udriver, THIS_MODULE, KBUILD_MODNAME)
+extern int __usb_register_device_driver(struct usb_device_driver *,
+			struct module *, const char *);
 extern void usb_deregister_device_driver(struct usb_device_driver *);
 
 extern int usb_register_dev(struct usb_interface *intf,
diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h
index 534e6650e2aadf..ab0a32a90629d4 100644
--- a/include/linux/usb/serial.h
+++ b/include/linux/usb/serial.h
@@ -381,6 +381,7 @@ void usb_serial_handle_dcd_change(struct usb_serial_port *usb_port,
 
 int usb_serial_bus_register(struct usb_serial_driver *device);
 void usb_serial_bus_deregister(struct usb_serial_driver *device);
+void usb_serial_bus_remove_new_id(struct usb_serial_driver *driver);
 
 extern const struct bus_type usb_serial_bus_type;
 extern struct tty_driver *usb_serial_tty_driver;
diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h
index ef21ead551be45..92309b569c3a05 100644
--- a/include/linux/usb/typec_altmode.h
+++ b/include/linux/usb/typec_altmode.h
@@ -227,9 +227,9 @@ struct typec_altmode_driver {
  * handle all SVID specific communication.
  */
 #define typec_altmode_register_driver(drv) \
-		__typec_altmode_register_driver(drv, THIS_MODULE)
+		__typec_altmode_register_driver(drv, THIS_MODULE, KBUILD_MODNAME)
 int __typec_altmode_register_driver(struct typec_altmode_driver *drv,
-				    struct module *module);
+				    struct module *module, const char *mod_name);
 /**
  * typec_altmode_unregister_driver - unregisters a USB Type-C alternate mode
  * 				     device driver
diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h
index e38d9e60569f00..91232053775d8d 100644
--- a/include/linux/user_namespace.h
+++ b/include/linux/user_namespace.h
@@ -29,8 +29,8 @@ struct uid_gid_map { /* 64 bytes -- 1 cache line */
 			u32 nr_extents;
 		};
 		struct {
-			struct uid_gid_extent *forward;
-			struct uid_gid_extent *reverse;
+			struct uid_gid_extent *forward __counted_by_ptr(nr_extents);
+			struct uid_gid_extent *reverse __counted_by_ptr(nr_extents);
 		};
 	};
 };
@@ -207,6 +207,13 @@ extern bool in_userns(const struct user_namespace *ancestor,
 		       const struct user_namespace *child);
 extern bool current_in_userns(const struct user_namespace *target_ns);
 struct ns_common *ns_get_owner(struct ns_common *ns);
+
+#if IS_ENABLED(CONFIG_KUNIT)
+extern int uid_gid_map_insert_extent(struct uid_gid_map *map,
+				     struct uid_gid_extent *extent);
+extern int uid_gid_map_sort(struct uid_gid_map *map);
+#endif /* CONFIG_KUNIT */
+
 #else
 
 static inline struct user_namespace *get_user_ns(struct user_namespace *ns)
diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h
index a4351cffc60ce3..a14b8a9ffb7b1c 100644
--- a/include/linux/userfaultfd_k.h
+++ b/include/linux/userfaultfd_k.h
@@ -18,7 +18,6 @@
 #include <linux/swap.h>
 #include <linux/leafops.h>
 #include <asm-generic/pgtable_uffd.h>
-#include <linux/hugetlb_inline.h>
 
 /* The set of all possible UFFD-related VM flags. */
 #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \
diff --git a/include/linux/vga_switcheroo.h b/include/linux/vga_switcheroo.h
index 7e6ac0114d556b..d7eee65664abba 100644
--- a/include/linux/vga_switcheroo.h
+++ b/include/linux/vga_switcheroo.h
@@ -31,8 +31,12 @@
 #ifndef _LINUX_VGA_SWITCHEROO_H_
 #define _LINUX_VGA_SWITCHEROO_H_
 
-#include <linux/fb.h>
+#include <linux/errno.h>
+#include <linux/types.h>
 
+struct device;
+struct dev_pm_domain;
+struct fb_info;
 struct pci_dev;
 
 /**
@@ -127,23 +131,28 @@ struct vga_switcheroo_handler {
  * @set_gpu_state: do the equivalent of suspend/resume for the card.
  * 	Mandatory. This should not cut power to the discrete GPU,
  * 	which is the job of the handler
- * @reprobe: poll outputs.
- * 	Optional. This gets called after waking the GPU and switching
- * 	the outputs to it
  * @can_switch: check if the device is in a position to switch now.
  * 	Mandatory. The client should return false if a user space process
  * 	has one of its device files open
+ * @pre_switch: prepare switch
+ *	Optional. This gets called before switching the outputs to the
+ *	GPU. Allows drivers to prepare for the switch.
+ * @post_switch: completes switch
+ *	Optional. This gets called after waking the GPU and switching
+ *	the outputs to it. Allows drivers to poll the switched outputs.
  * @gpu_bound: notify the client id to audio client when the GPU is bound.
  *
  * Client callbacks. A client can be either a GPU or an audio device on a GPU.
- * The @set_gpu_state and @can_switch methods are mandatory, @reprobe may be
- * set to NULL. For audio clients, the @reprobe member is bogus.
- * OTOH, @gpu_bound is only for audio clients, and not used for GPU clients.
+ * The @set_gpu_state and @can_switch methods are mandatory, @pre_switch and
+ * @post_switch may be set to NULL. For audio clients, the @pre_switch and
+ * @post_switch members are bogus. OTOH, @gpu_bound is only for audio clients,
+ * and not used for GPU clients.
  */
 struct vga_switcheroo_client_ops {
 	void (*set_gpu_state)(struct pci_dev *dev, enum vga_switcheroo_state);
-	void (*reprobe)(struct pci_dev *dev);
 	bool (*can_switch)(struct pci_dev *dev);
+	void (*pre_switch)(struct pci_dev *dev);
+	void (*post_switch)(struct pci_dev *dev);
 	void (*gpu_bound)(struct pci_dev *dev, enum vga_switcheroo_client_id);
 };
 
@@ -156,9 +165,6 @@ int vga_switcheroo_register_audio_client(struct pci_dev *pdev,
 					 const struct vga_switcheroo_client_ops *ops,
 					 struct pci_dev *vga_dev);
 
-void vga_switcheroo_client_fb_set(struct pci_dev *dev,
-				  struct fb_info *info);
-
 int vga_switcheroo_register_handler(const struct vga_switcheroo_handler *handler,
 				    enum vga_switcheroo_handler_flags_t handler_flags);
 void vga_switcheroo_unregister_handler(void);
@@ -178,7 +184,6 @@ void vga_switcheroo_fini_domain_pm_ops(struct device *dev);
 static inline void vga_switcheroo_unregister_client(struct pci_dev *dev) {}
 static inline int vga_switcheroo_register_client(struct pci_dev *dev,
 		const struct vga_switcheroo_client_ops *ops, bool driver_power_control) { return 0; }
-static inline void vga_switcheroo_client_fb_set(struct pci_dev *dev, struct fb_info *info) {}
 static inline int vga_switcheroo_register_handler(const struct vga_switcheroo_handler *handler,
 		enum vga_switcheroo_handler_flags_t handler_flags) { return 0; }
 static inline int vga_switcheroo_register_audio_client(struct pci_dev *pdev,
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index aed121d729b013..034a693777ca05 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -3,6 +3,7 @@
 #define _LINUX_VMALLOC_H
 
 #include <linux/alloc_tag.h>
+#include <linux/cleanup.h>
 #include <linux/sched.h>
 #include <linux/spinlock.h>
 #include <linux/init.h>
@@ -38,6 +39,7 @@ struct iov_iter;		/* in uio.h */
 #define VM_DEFER_KMEMLEAK	0
 #endif
 #define VM_SPARSE		0x00001000	/* sparse vm_area. not all pages are present. */
+#define VM_REQUIRE_HUGE_VMAP	0x00002000	/* huge page mapping or nothing */
 
 /* bits [20..32] reserved for arch specific ioremap internals */
 
@@ -214,6 +216,8 @@ void *__must_check vrealloc_node_align_noprof(const void *p, size_t size,
 extern void vfree(const void *addr);
 extern void vfree_atomic(const void *addr);
 
+DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T))
+
 extern void *vmap(struct page **pages, unsigned int count,
 			unsigned long flags, pgprot_t prot);
 void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot);
diff --git a/include/linux/vmemmap-optimization.h b/include/linux/vmemmap-optimization.h
new file mode 100644
index 00000000000000..fa9e9abd66560f
--- /dev/null
+++ b/include/linux/vmemmap-optimization.h
@@ -0,0 +1,115 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * vmemmap-optimization.h
+ *
+ * Generic vmemmap optimization declarations.
+ *
+ * Author: Muchun Song <songmuchun@bytedance.com>
+ */
+#ifndef _LINUX_VMEMMAP_OPTIMIZATION_H
+#define _LINUX_VMEMMAP_OPTIMIZATION_H
+
+#include <linux/align.h>
+#include <linux/log2.h>
+#include <linux/mmdebug.h>
+#include <linux/mmzone.h>
+
+/*
+ * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to
+ * be naturally aligned with regard to the folio size.
+ *
+ * HVO which is only active if the size of struct page is a power of 2.
+ */
+#define MAX_FOLIO_VMEMMAP_ALIGN					\
+	(IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) &&		\
+	 is_power_of_2(sizeof(struct page)) ?			\
+	 MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
+
+/* The number of retained vmemmap pages with HVO enabled. */
+#define VMEMMAP_OPTIMIZATION_PAGES		1
+#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES	\
+	(VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page))
+#define VMEMMAP_OPTIMIZATION_MIN_ORDER		(ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1)
+
+#ifdef CONFIG_VMEMMAP_OPTIMIZATION
+static inline unsigned int section_compound_order(const struct mem_section *section)
+{
+	return section->compound_page_order;
+}
+
+static inline void section_set_compound_order(struct mem_section *section,
+					      unsigned int order)
+{
+	VM_WARN_ON(section_compound_order(section) && order &&
+		   section_compound_order(section) != order);
+	section->compound_page_order = order;
+}
+
+static inline void section_set_compound_order_range(unsigned long pfn,
+		unsigned long nr_pages, unsigned int order)
+{
+	unsigned long section_nr = pfn_to_section_nr(pfn);
+
+	if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))
+		return;
+
+	for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++)
+		section_set_compound_order(__nr_to_section(section_nr + i), order);
+}
+
+static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
+{
+	return section_compound_order(__pfn_to_section(pfn));
+}
+
+struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone);
+#else
+static inline unsigned int section_compound_order(const struct mem_section *section)
+{
+	return 0;
+}
+
+static inline void section_set_compound_order(struct mem_section *section,
+					      unsigned int order)
+{
+}
+
+static inline void section_set_compound_order_range(unsigned long pfn,
+		unsigned long nr_pages, unsigned int order)
+{
+}
+
+static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
+{
+	return 0;
+}
+
+static inline struct page *vmemmap_shared_tail_page(unsigned int order,
+						    struct zone *zone)
+{
+	return NULL;
+}
+#endif /* CONFIG_VMEMMAP_OPTIMIZATION */
+
+static inline bool vmemmap_optimizable_pfn(unsigned long pfn)
+{
+	const unsigned int order = pfn_to_section_compound_order(pfn);
+	const unsigned long nr_pages = 1UL << order;
+
+	if (!is_power_of_2(sizeof(struct page)))
+		return false;
+
+	return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES;
+}
+
+static inline bool vmemmap_optimizable_order(unsigned int order)
+{
+	if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION))
+		return false;
+
+	if (!is_power_of_2(sizeof(struct page)))
+		return false;
+
+	return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER;
+}
+#endif /* _LINUX_VMEMMAP_OPTIMIZATION_H */
diff --git a/include/linux/wait.h b/include/linux/wait.h
index 7e215330199cfe..c2af98b0074dcb 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -556,7 +556,7 @@ do {										\
 	}									\
 										\
 	__ret = ___wait_event(wq_head, condition, state, 0, 0,			\
-		if (!__t.task) {						\
+		if (!hrtimer_sleeper_task_get(&__t)) {				\
 			__ret = -ETIME;						\
 			break;							\
 		}								\
diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h
index 553d7b23e3adfd..af077ed4caf6a1 100644
--- a/include/linux/wait_bit.h
+++ b/include/linux/wait_bit.h
@@ -432,6 +432,32 @@ do {									\
 	__ret;								\
 })
 
+/**
+ * wait_var_event_state - wait for a variable to be updated and notified
+ * @var: the address of variable being waited on
+ * @condition: the condition to wait for
+ * @state: the task state to sleep in, %TASK_UNINTERRUPTIBLE etc.
+ *
+ * Wait for a @condition to be true, only re-checking when a wake up is
+ * received for the given @var (an arbitrary kernel address which need
+ * not be directly related to the given condition, but usually is).
+ *
+ * Returns 0 if the condition became true, or %-ERESTARTSYS if a signal
+ * arrived which @state allows to interrupt.
+ *
+ * The condition should normally use smp_load_acquire() or a similarly
+ * ordered access to ensure that any changes to memory made before the
+ * condition became true will be visible after the wait completes.
+ */
+#define wait_var_event_state(var, condition, state)					\
+({											\
+	int __ret = 0;									\
+	might_sleep();									\
+	if (!(condition))								\
+		__ret = ___wait_var_event(var, condition, (state), 0, 0, schedule());	\
+	__ret;										\
+})
+
 /**
  * wait_var_event_any_lock - wait for a variable to be updated under a lock
  * @var: the address of the variable being waited on
diff --git a/include/linux/xattr.h b/include/linux/xattr.h
index 54ac3cbc133f86..4cc4257de084e5 100644
--- a/include/linux/xattr.h
+++ b/include/linux/xattr.h
@@ -47,7 +47,7 @@ struct xattr_handler {
 		   struct inode *inode, const char *name, void *buffer,
 		   size_t size);
 	int (*set)(const struct xattr_handler *,
-		   struct mnt_idmap *idmap, struct dentry *dentry,
+		   const struct mnt_idmap *idmap, struct dentry *dentry,
 		   struct inode *inode, const char *name, const void *buffer,
 		   size_t size, int flags);
 };
@@ -77,25 +77,25 @@ struct xattr {
 };
 
 ssize_t __vfs_getxattr(struct dentry *, struct inode *, const char *, void *, size_t);
-ssize_t vfs_getxattr(struct mnt_idmap *, struct dentry *, const char *,
+ssize_t vfs_getxattr(const struct mnt_idmap *, struct dentry *, const char *,
 		     void *, size_t);
 ssize_t vfs_listxattr(struct dentry *d, char *list, size_t size);
-int __vfs_setxattr(struct mnt_idmap *, struct dentry *, struct inode *,
+int __vfs_setxattr(const struct mnt_idmap *, struct dentry *, struct inode *,
 		   const char *, const void *, size_t, int);
-int __vfs_setxattr_noperm(struct mnt_idmap *, struct dentry *,
+int __vfs_setxattr_noperm(const struct mnt_idmap *, struct dentry *,
 			  const char *, const void *, size_t, int);
-int __vfs_setxattr_locked(struct mnt_idmap *, struct dentry *,
+int __vfs_setxattr_locked(const struct mnt_idmap *, struct dentry *,
 			  const char *, const void *, size_t, int,
 			  struct delegated_inode *);
-int vfs_setxattr(struct mnt_idmap *, struct dentry *, const char *,
+int vfs_setxattr(const struct mnt_idmap *, struct dentry *, const char *,
 		 const void *, size_t, int);
-int __vfs_removexattr(struct mnt_idmap *, struct dentry *, const char *);
-int __vfs_removexattr_locked(struct mnt_idmap *, struct dentry *,
+int __vfs_removexattr(const struct mnt_idmap *, struct dentry *, const char *);
+int __vfs_removexattr_locked(const struct mnt_idmap *, struct dentry *,
 			     const char *, struct delegated_inode *);
-int vfs_removexattr(struct mnt_idmap *, struct dentry *, const char *);
+int vfs_removexattr(const struct mnt_idmap *, struct dentry *, const char *);
 
 ssize_t generic_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size);
-int vfs_getxattr_alloc(struct mnt_idmap *idmap,
+int vfs_getxattr_alloc(const struct mnt_idmap *idmap,
 		       struct dentry *dentry, const char *name,
 		       char **xattr_value, size_t size, gfp_t flags);
 
diff --git a/include/linux/zsmalloc.h b/include/linux/zsmalloc.h
index 478410c880b1fe..5b7298a5026b8d 100644
--- a/include/linux/zsmalloc.h
+++ b/include/linux/zsmalloc.h
@@ -40,10 +40,6 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size);
 
 void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats);
 
-void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle,
-			size_t mem_len, void *local_copy);
-void zs_obj_read_end(struct zs_pool *pool, unsigned long handle,
-		     size_t mem_len, void *handle_mem);
 void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle,
 			  struct scatterlist *sg, size_t mem_len);
 void zs_obj_read_sg_end(struct zs_pool *pool, unsigned long handle);
diff --git a/include/linux/zswap.h b/include/linux/zswap.h
index 30c193a1207e16..df6cafbe95dc0c 100644
--- a/include/linux/zswap.h
+++ b/include/linux/zswap.h
@@ -27,7 +27,7 @@ struct zswap_lruvec_state {
 unsigned long zswap_total_pages(void);
 bool zswap_store(struct folio *folio);
 int zswap_load(struct folio *folio);
-void zswap_invalidate(swp_entry_t swp);
+void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries);
 int zswap_swapon(int type, unsigned long nr_pages);
 void zswap_swapoff(int type);
 void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg);
@@ -49,7 +49,11 @@ static inline int zswap_load(struct folio *folio)
 	return -ENOENT;
 }
 
-static inline void zswap_invalidate(swp_entry_t swp) {}
+static inline void zswap_invalidate(int type, pgoff_t offset,
+		unsigned long nr_entries)
+{
+}
+
 static inline int zswap_swapon(int type, unsigned long nr_pages)
 {
 	return 0;
diff --git a/include/media/ipu-bridge.h b/include/media/ipu-bridge.h
index 16fac765456e81..760f0767ce58f9 100644
--- a/include/media/ipu-bridge.h
+++ b/include/media/ipu-bridge.h
@@ -17,13 +17,28 @@
 #define IPU_SENSOR_ROTATION_NORMAL		0
 #define IPU_SENSOR_ROTATION_INVERTED		1
 
-#define IPU_SENSOR_CONFIG(_HID, _NR, ...)	\
-	(const struct ipu_sensor_config) {	\
-		.hid = _HID,			\
-		.nr_link_freqs = _NR,		\
-		.link_freqs = { __VA_ARGS__ }	\
+/* Flags for struct ipu_sensor_config */
+/* The sensor's CSI-2 transmitter needs a non-continuous clock */
+#define IPU_BR_FL_CSI2_CLK_NONCONTINUOUS	BIT(0)
+
+/*
+ * Sensor config specific to one or more IPUs, identified by their PCI product
+ * IDs, with flags describing what the sensor needs there. Entries for one HID
+ * must be adjacent in ipu_supported_sensors[], with the IPU-specific ones
+ * before the generic one.
+ */
+#define IPU_SENSOR_CONFIG_MATCH_FL(_HID, _IDS, _FLAGS, _NR, ...)	\
+	(const struct ipu_sensor_config) {				\
+		.hid = _HID,						\
+		.pci_ids = _IDS,					\
+		.flags = _FLAGS,					\
+		.nr_link_freqs = _NR,					\
+		.link_freqs = { __VA_ARGS__ }				\
 	}
 
+#define IPU_SENSOR_CONFIG(_HID, _NR, ...)				\
+	IPU_SENSOR_CONFIG_MATCH_FL(_HID, NULL, 0, _NR, __VA_ARGS__)
+
 #define NODE_SENSOR(_HID, _PROPS)		\
 	(const struct software_node) {		\
 		.name = _HID,			\
@@ -64,6 +79,24 @@ enum ipu_sensor_swnodes {
 	SWNODE_COUNT
 };
 
+enum ipu_bridge_ep_props {
+	IPU_BRIDGE_EP_BUS_TYPE,
+	IPU_BRIDGE_EP_DATA_LANES,
+	IPU_BRIDGE_EP_REMOTE_EP,
+	IPU_BRIDGE_EP_LINK_FREQUENCIES,
+	IPU_BRIDGE_EP_CLOCK_NONCONTINUOUS,
+	IPU_BRIDGE_EP_NUM_OF,
+	IPU_BRIDGE_EP_NUM_ENTRIES
+};
+
+/*
+ * Get the index of the next property in a property array, with a given maximum
+ * value.
+ */
+#define IPU_BRIDGE_NEXT_PROPERTY(index, max)		\
+	(WARN_ON((index) > IPU_BRIDGE_##max) ?		\
+	 IPU_BRIDGE_##max : (index)++)
+
 /* Data representation as it is in ACPI SSDB buffer */
 struct ipu_sensor_ssdb {
 	u8 version;
@@ -115,6 +148,9 @@ struct ipu_node_names {
 
 struct ipu_sensor_config {
 	const char *hid;
+	/* Zero-terminated list of IPU PCI product IDs, NULL for any IPU */
+	const u16 *pci_ids;
+	const u32 flags;
 	const u8 nr_link_freqs;
 	const u64 link_freqs[MAX_NUM_LINK_FREQS];
 };
@@ -141,7 +177,7 @@ struct ipu_sensor {
 	const char *vcm_type;
 
 	struct ipu_property_names prop_names;
-	struct property_entry ep_properties[5];
+	struct property_entry ep_properties[IPU_BRIDGE_EP_NUM_ENTRIES];
 	struct property_entry dev_properties[5];
 	struct property_entry ipu_properties[3];
 	struct property_entry ivsc_properties[1];
@@ -160,6 +196,8 @@ typedef int (*ipu_parse_sensor_fwnode_t)(struct acpi_device *adev,
 
 struct ipu_bridge {
 	struct device *dev;
+	/* PCI product ID of the IPU, 0 if it is not a PCI device */
+	u16 pci_id;
 	ipu_parse_sensor_fwnode_t parse_sensor_fwnode;
 	char ipu_node_name[ACPI_ID_LEN];
 	struct software_node ipu_hid_node;
@@ -169,11 +207,13 @@ struct ipu_bridge {
 };
 
 #if IS_ENABLED(CONFIG_IPU_BRIDGE)
+struct pci_dev *ipu_bridge_get_ipu6(void);
 int ipu_bridge_init(struct device *dev,
 		    ipu_parse_sensor_fwnode_t parse_sensor_fwnode);
 int ipu_bridge_parse_ssdb(struct acpi_device *adev, struct ipu_sensor *sensor);
 int ipu_bridge_instantiate_vcm(struct device *sensor);
 #else
+static inline struct pci_dev *ipu_bridge_get_ipu6(void) { return NULL; }
 /* Use a define to avoid the @parse_sensor_fwnode argument getting evaluated */
 #define ipu_bridge_init(dev, parse_sensor_fwnode)	(0)
 static inline int ipu_bridge_instantiate_vcm(struct device *s) { return 0; }
diff --git a/include/media/ipu6-pci-table.h b/include/media/ipu6-pci-table.h
index 0899d9d2f97864..cacb8a3170d04f 100644
--- a/include/media/ipu6-pci-table.h
+++ b/include/media/ipu6-pci-table.h
@@ -14,6 +14,8 @@
 #define PCI_DEVICE_ID_INTEL_IPU6EP_ADLN		0x462e
 #define PCI_DEVICE_ID_INTEL_IPU6EP_RPLP		0xa75d
 #define PCI_DEVICE_ID_INTEL_IPU6EP_MTL		0x7d19
+#define PCI_DEVICE_ID_INTEL_IPU7		0x645d
+#define PCI_DEVICE_ID_INTEL_IPU7P5		0xb05d
 
 static const struct pci_device_id ipu6_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU6) },
@@ -22,6 +24,8 @@ static const struct pci_device_id ipu6_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU6EP_ADLN) },
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU6EP_RPLP) },
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU6EP_MTL) },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU7) },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IPU7P5) },
 	{ }
 };
 
diff --git a/include/media/rc-map.h b/include/media/rc-map.h
index d95ed3e96de283..f167c37179c8f6 100644
--- a/include/media/rc-map.h
+++ b/include/media/rc-map.h
@@ -148,7 +148,6 @@ struct rc_map_table {
  * @scan: pointer to struct &rc_map_table
  * @size: Max number of entries
  * @len: Number of entries that are in use
- * @alloc: size of \*scan, in bytes
  * @rc_proto: type of the remote controller protocol, as defined at
  *	     enum &rc_proto
  * @name: name of the key map table
@@ -158,7 +157,6 @@ struct rc_map {
 	struct rc_map_table	*scan;
 	unsigned int		size;
 	unsigned int		len;
-	unsigned int		alloc;
 	enum rc_proto		rc_proto;
 	const char		*name;
 	spinlock_t		lock;
diff --git a/include/media/v4l2-common.h b/include/media/v4l2-common.h
index edd416178c3333..33f5713734cb22 100644
--- a/include/media/v4l2-common.h
+++ b/include/media/v4l2-common.h
@@ -554,15 +554,80 @@ static inline bool v4l2_is_format_bayer(const struct v4l2_format_info *f)
 const struct v4l2_format_info *v4l2_format_info(u32 format);
 void v4l2_apply_frmsize_constraints(u32 *width, u32 *height,
 				    const struct v4l2_frmsize_stepwise *frmsize);
-int v4l2_fill_pixfmt(struct v4l2_pix_format *pixfmt, u32 pixelformat,
-		     u32 width, u32 height);
-int v4l2_fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt, u32 pixelformat,
-			u32 width, u32 height);
-/* @stride_alignment is a power of 2 value in bytes */
+
+/**
+ * v4l2_fill_pixfmt_aligned - Fill in a &struct v4l2_pix_format with stride
+ *	alignment requirements
+ *
+ * @pixfmt: pointer to the &struct v4l2_pix_format to be filled
+ * @pixelformat: the V4L2 pixel format (V4L2_PIX_FMT_*)
+ * @width: image width in pixels
+ * @height: image height in pixels
+ * @stride_alignment: stride alignment in bytes, must be a power of 2
+ *
+ * Fills all fields of @pixfmt for the given pixel format, dimensions, and
+ * stride alignment. Only formats stored in a single memory plane are
+ * supported; returns -EINVAL for multi-memory-plane formats.
+ *
+ * @pixfmt->bytesperline is set to the stride of the primary (plane 0) plane,
+ * rounded up to a multiple of @stride_alignment. For formats that store
+ * multiple component planes in a single memory buffer (e.g. YUV420), the
+ * alignment applied to each component plane's stride is scaled relative to
+ * @stride_alignment so that the chroma stride remains consistently derivable
+ * from the luma stride. @pixfmt->bytesperline therefore reflects only the
+ * primary plane stride.
+ *
+ * @pixfmt->sizeimage is set to the total size in bytes of all planes.
+ *
+ * Return: 0 on success, -EINVAL if @pixelformat is unknown or uses multiple
+ * memory planes.
+ */
+int v4l2_fill_pixfmt_aligned(struct v4l2_pix_format *pixfmt, u32 pixelformat,
+			     u32 width, u32 height, u8 stride_alignment);
+
+static inline int v4l2_fill_pixfmt(struct v4l2_pix_format *pixfmt,
+				   u32 pixelformat, u32 width, u32 height)
+{
+	return v4l2_fill_pixfmt_aligned(pixfmt, pixelformat, width, height, 1);
+}
+
+/**
+ * v4l2_fill_pixfmt_mp_aligned - Fill in a &struct v4l2_pix_format_mplane with
+ *	stride alignment requirements.
+ *
+ * @pixfmt: pointer to the &struct v4l2_pix_format_mplane to be filled
+ * @pixelformat: the V4L2 pixel format (V4L2_PIX_FMT_*)
+ * @width: image width in pixels
+ * @height: image height in pixels
+ * @stride_alignment: stride alignment in bytes; must be a power of 2
+ *
+ * Fills all fields of @pixfmt for the given pixel format, dimensions, and
+ * stride alignment.
+ *
+ * For formats stored in a single memory plane (mem_planes == 1), the
+ * behaviour matches v4l2_fill_pixfmt_aligned(): plane_fmt[0].bytesperline
+ * is set to the primary plane stride. The strides of all components are
+ * aligned to the @stride_alignment. To keep the chroma strides consistently
+ * derivable from the luma stride, strides may be aligned to a multiple of
+ * the @stride_alignment instead. plane_fmt[0].sizeimage covers all
+ * component planes.
+ *
+ * For formats with multiple memory planes (mem_planes > 1), each plane's
+ * bytesperline is independently rounded up to @stride_alignment, and each
+ * plane's sizeimage is set to bytesperline multiplied by the plane height.
+ *
+ * Return: 0 on success, -EINVAL if @pixelformat is unknown.
+ */
 int v4l2_fill_pixfmt_mp_aligned(struct v4l2_pix_format_mplane *pixfmt,
 				u32 pixelformat, u32 width, u32 height,
 				u8 stride_alignment);
 
+static inline int v4l2_fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt,
+				      u32 pixelformat, u32 width, u32 height)
+{
+	return v4l2_fill_pixfmt_mp_aligned(pixfmt, pixelformat, width, height, 1);
+}
+
 /**
  * v4l2_get_link_freq - Get link rate from transmitter
  *
diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h
index 327976b14d50b7..cec9217d97ac69 100644
--- a/include/media/v4l2-ctrls.h
+++ b/include/media/v4l2-ctrls.h
@@ -834,6 +834,9 @@ bool v4l2_ctrl_radio_filter(const struct v4l2_ctrl *ctrl);
  *
  * @ncontrols:	The number of controls in this cluster.
  * @controls:	The cluster control array of size @ncontrols.
+ *
+ *		If controls[0] is NULL, then this function does nothing and just
+ *		returns.
  */
 void v4l2_ctrl_cluster(unsigned int ncontrols, struct v4l2_ctrl **controls);
 
@@ -845,6 +848,9 @@ void v4l2_ctrl_cluster(unsigned int ncontrols, struct v4l2_ctrl **controls);
  * @ncontrols:	The number of controls in this cluster.
  * @controls:	The cluster control array of size @ncontrols. The first control
  *		must be the 'auto' control (e.g. autogain, autoexposure, etc.)
+ *
+ *		If controls[0] is NULL, then this function does nothing and just
+ *		returns.
  * @manual_val: The value for the first control in the cluster that equals the
  *		manual setting.
  * @set_volatile: If true, then all controls except the first auto control will
diff --git a/include/media/v4l2-dv-timings.h b/include/media/v4l2-dv-timings.h
index 2b42e5d81f9e2b..de5ef825a6c68e 100644
--- a/include/media/v4l2-dv-timings.h
+++ b/include/media/v4l2-dv-timings.h
@@ -252,6 +252,20 @@ v4l2_hdmi_rx_colorimetry(const struct hdmi_avi_infoframe *avi,
 			 const struct hdmi_vendor_infoframe *hdmi,
 			 unsigned int height);
 
+/*
+ * The time in milliseconds that the HPD should be pulled low when writing
+ * a new EDID. This will tell the HDMI source that the EDID was changed and
+ * that it has to be re-read.
+ *
+ * The source is supposed to re-read the EDID if the HPD is low for more than
+ * 100 ms, but in practice the sink should pull it low for a bit longer due
+ * to clock differences and imprecise video source implementations.
+ *
+ * Practice has shown that setting the delay to HZ / 7 (approx 143 ms) works
+ * well.
+ */
+#define V4L2_SET_EDID_HPD_LOW_JIFFIES (HZ / 7)
+
 unsigned int v4l2_num_edid_blocks(const u8 *edid, unsigned int max_blocks);
 u16 v4l2_get_edid_phys_addr(const u8 *edid, unsigned int size,
 			    unsigned int *offset);
diff --git a/include/media/v4l2-subdev.h b/include/media/v4l2-subdev.h
index d256b7ec8f848c..5796e52678b908 100644
--- a/include/media/v4l2-subdev.h
+++ b/include/media/v4l2-subdev.h
@@ -734,6 +734,14 @@ struct v4l2_subdev_state {
 	struct v4l2_subdev_stream_configs stream_configs;
 };
 
+/**
+ * struct v4l2_subdev_client_info - Sub-device client information
+ * @caps: bitmask of ``V4L2_SUBDEV_CLIENT_CAP_*``
+ */
+struct v4l2_subdev_client_info {
+	u64 caps;
+};
+
 /**
  * struct v4l2_subdev_pad_ops - v4l2-subdev pad level operations
  *
@@ -747,11 +755,14 @@ struct v4l2_subdev_state {
  *
  * @get_fmt: callback for VIDIOC_SUBDEV_G_FMT() ioctl handler code.
  *
- * @set_fmt: callback for VIDIOC_SUBDEV_S_FMT() ioctl handler code.
+ * @set_fmt: callback for VIDIOC_SUBDEV_S_FMT() ioctl handler code. The ci
+ *	     pointer may be NULL for in-kernel calls.
  *
  * @get_selection: callback for VIDIOC_SUBDEV_G_SELECTION() ioctl handler code.
+ *		   The ci pointer may be NULL for in-kernel calls.
  *
  * @set_selection: callback for VIDIOC_SUBDEV_S_SELECTION() ioctl handler code.
+ *		   The ci pointer may be NULL for in-kernel calls.
  *
  * @get_frame_interval: callback for VIDIOC_SUBDEV_G_FRAME_INTERVAL()
  *			ioctl handler code.
@@ -814,6 +825,11 @@ struct v4l2_subdev_state {
  *	V4L2_SUBDEV_CAP_STREAMS sub-device capability flag can ignore the mask
  *	argument.
  *
+ *	Due to device constraints, starting the requested streams may result in
+ *	additional streams also being started by the driver. Streams that are
+ *	started and stopped together due to the nature of the hardware are
+ *	called a stream group.
+ *
  * @disable_streams: Disable the streams defined in streams_mask on the given
  *	source pad. Subdevs that implement this operation must use the active
  *	state management provided by the subdev core (enabled through a call to
@@ -823,6 +839,9 @@ struct v4l2_subdev_state {
  *	Drivers that support only a single stream without setting the
  *	V4L2_SUBDEV_CAP_STREAMS sub-device capability flag can ignore the mask
  *	argument.
+ *
+ *	When the requested stream are part of a stream group, they will be
+ *	stopped once all streams in the group are stopped.
  */
 struct v4l2_subdev_pad_ops {
 	int (*enum_mbus_code)(struct v4l2_subdev *sd,
@@ -838,12 +857,15 @@ struct v4l2_subdev_pad_ops {
 		       struct v4l2_subdev_state *state,
 		       struct v4l2_subdev_format *format);
 	int (*set_fmt)(struct v4l2_subdev *sd,
+		       const struct v4l2_subdev_client_info *ci,
 		       struct v4l2_subdev_state *state,
 		       struct v4l2_subdev_format *format);
 	int (*get_selection)(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_selection *sel);
 	int (*set_selection)(struct v4l2_subdev *sd,
+			     const struct v4l2_subdev_client_info *ci,
 			     struct v4l2_subdev_state *state,
 			     struct v4l2_subdev_selection *sel);
 	int (*get_frame_interval)(struct v4l2_subdev *sd,
@@ -1122,14 +1144,14 @@ struct v4l2_subdev {
  * @vfh: pointer to &struct v4l2_fh
  * @state: pointer to &struct v4l2_subdev_state
  * @owner: module pointer to the owner of this file handle
- * @client_caps: bitmask of ``V4L2_SUBDEV_CLIENT_CAP_*``
+ * @ci: sub-device client info related to this file handle
  */
 struct v4l2_subdev_fh {
 	struct v4l2_fh vfh;
 	struct module *owner;
 #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API)
 	struct v4l2_subdev_state *state;
-	u64 client_caps;
+	struct v4l2_subdev_client_info ci;
 #endif
 };
 
@@ -1935,14 +1957,16 @@ extern const struct v4l2_subdev_ops v4l2_subdev_call_wrappers;
 		int __result;						\
 		if (!__sd)						\
 			__result = -ENODEV;				\
-		else if (!(__sd->ops->o && __sd->ops->o->f))		\
+		else if (!__sd->ops->o)					\
 			__result = -ENOIOCTLCMD;			\
 		else if (v4l2_subdev_call_wrappers.o &&			\
 			 v4l2_subdev_call_wrappers.o->f)		\
 			__result = v4l2_subdev_call_wrappers.o->f(	\
 							__sd, ##args);	\
-		else							\
+		else if (__sd->ops->o->f)				\
 			__result = __sd->ops->o->f(__sd, ##args);	\
+		else							\
+			__result = -ENOIOCTLCMD;			\
 		__result;						\
 	})
 
diff --git a/include/net/arp.h b/include/net/arp.h
index e8747e0713c79f..e932def63d62f7 100644
--- a/include/net/arp.h
+++ b/include/net/arp.h
@@ -7,8 +7,10 @@
 #include <linux/hash.h>
 #include <net/neighbour.h>
 
-
-extern struct neigh_table arp_tbl;
+static inline struct neigh_table *arp_table(struct net *net)
+{
+	return net->neigh_tables[NEIGH_ARP_TABLE];
+}
 
 static inline u32 arp_hashfn(const void *pkey, const struct net_device *dev, u32 *hash_rnd)
 {
@@ -21,10 +23,12 @@ static inline u32 arp_hashfn(const void *pkey, const struct net_device *dev, u32
 #ifdef CONFIG_INET
 static inline struct neighbour *__ipv4_neigh_lookup_noref(struct net_device *dev, u32 key)
 {
+	struct neigh_table *tbl = arp_table(dev_net(dev));
+
 	if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))
 		key = INADDR_ANY;
 
-	return ___neigh_lookup_noref(&arp_tbl, neigh_key_eq32, arp_hashfn, &key, dev);
+	return ___neigh_lookup_noref(tbl, neigh_key_eq32, arp_hashfn, &key, dev);
 }
 #else
 static inline
diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h
index 4105c446ca983b..fb85a0ccb250fc 100644
--- a/include/net/bluetooth/hci_core.h
+++ b/include/net/bluetooth/hci_core.h
@@ -28,6 +28,7 @@
 #include <linux/rculist.h>
 #include <linux/spinlock.h>
 #include <linux/srcu.h>
+#include <linux/uio.h>
 
 #include <net/bluetooth/hci.h>
 #include <net/bluetooth/hci_drv.h>
@@ -62,7 +63,7 @@ struct inquiry_entry {
 		NAME_PENDING,
 		NAME_KNOWN,
 	} name_state;
-	__u32			timestamp;
+	unsigned long		timestamp;
 	struct inquiry_data	data;
 };
 
@@ -78,7 +79,7 @@ struct discovery_state {
 	struct list_head	all;	/* All devices found during inquiry */
 	struct list_head	unknown;	/* Name state not known */
 	struct list_head	resolve;	/* Name needs to be resolved */
-	__u32			timestamp;
+	unsigned long		timestamp;
 	bdaddr_t		last_adv_addr;
 	u8			last_adv_addr_type;
 	s8			last_adv_rssi;
@@ -562,6 +563,7 @@ struct hci_dev {
 	struct list_head	link_keys;
 	struct list_head	long_term_keys;
 	struct list_head	identity_resolving_keys;
+	struct mutex		remote_oob_lock;
 	struct list_head	remote_oob_data;
 	struct list_head	le_accept_list;
 	struct list_head	le_resolv_list;
@@ -646,6 +648,8 @@ struct hci_dev {
 	int (*setup)(struct hci_dev *hdev);
 	int (*shutdown)(struct hci_dev *hdev);
 	int (*send)(struct hci_dev *hdev, struct sk_buff *skb);
+	/* Receive HCI_VENDOR_PKT */
+	void (*recv_vendor_pkt)(struct hci_dev *hdev, struct sk_buff *skb);
 	/* Handle HCI_EV_VENDOR; return true if handled, false otherwise */
 	bool (*handle_ev_vendor)(struct hci_dev *hdev, struct sk_buff *skb);
 	void (*notify)(struct hci_dev *hdev, unsigned int evt);
@@ -1030,6 +1034,9 @@ static inline bool hci_conn_sc_enabled(struct hci_conn *conn)
 static inline void hci_conn_hash_add(struct hci_dev *hdev, struct hci_conn *c)
 {
 	struct hci_conn_hash *h = &hdev->conn_hash;
+
+	lockdep_assert_held(&hdev->lock);
+
 	list_add_tail_rcu(&c->list, &h->list);
 	switch (c->type) {
 	case ACL_LINK:
@@ -1060,6 +1067,8 @@ static inline void hci_conn_hash_del(struct hci_dev *hdev, struct hci_conn *c)
 {
 	struct hci_conn_hash *h = &hdev->conn_hash;
 
+	lockdep_assert_held(&hdev->lock);
+
 	list_del_rcu(&c->list);
 	synchronize_rcu();
 
@@ -1088,6 +1097,15 @@ static inline void hci_conn_hash_del(struct hci_dev *hdev, struct hci_conn *c)
 	}
 }
 
+#ifdef CONFIG_PROVE_RCU
+#define HCI_CONN_HASH_LOCKDEP_CHECK(hdev)				\
+	RCU_LOCKDEP_WARN(!lockdep_is_held(&(hdev)->lock) &&		\
+			 !rcu_read_lock_held(),				\
+			 "suspicious hci_conn locking")
+#else
+#define HCI_CONN_HASH_LOCKDEP_CHECK(hdev) do { } while (0 && (hdev))
+#endif
+
 static inline unsigned int hci_conn_num(struct hci_dev *hdev, __u8 type)
 {
 	struct hci_conn_hash *h = &hdev->conn_hash;
@@ -1169,6 +1187,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_bis(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1191,6 +1211,8 @@ hci_conn_hash_lookup_create_pa_sync(struct hci_dev *hdev)
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1217,6 +1239,8 @@ hci_conn_hash_lookup_per_adv_bis(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1241,6 +1265,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_handle(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1260,6 +1286,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_ba(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1281,6 +1309,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_role(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1302,6 +1332,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_le(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1328,6 +1360,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_cis(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1360,6 +1394,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_cig(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1383,6 +1419,8 @@ static inline struct hci_conn *hci_conn_hash_lookup_big(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1407,6 +1445,8 @@ hci_conn_hash_lookup_big_sync_pend(struct hci_dev *hdev,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1431,6 +1471,8 @@ hci_conn_hash_lookup_big_state(struct hci_dev *hdev, __u8 handle, __u16 state,
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1454,6 +1496,8 @@ hci_conn_hash_lookup_pa_sync_big_handle(struct hci_dev *hdev, __u8 big)
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1477,6 +1521,8 @@ hci_conn_hash_lookup_pa_sync_handle(struct hci_dev *hdev, __u16 sync_handle)
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -1546,6 +1592,8 @@ static inline struct hci_conn *hci_lookup_le_connect(struct hci_dev *hdev)
 	struct hci_conn_hash *h = &hdev->conn_hash;
 	struct hci_conn  *c;
 
+	HCI_CONN_HASH_LOCKDEP_CHECK(hdev);
+
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -2356,6 +2404,8 @@ static inline int hci_check_conn_params(u16 min, u16 max, u16 latency,
 int hci_register_cb(struct hci_cb *hcb);
 int hci_unregister_cb(struct hci_cb *hcb);
 
+int hci_send_vendor_frame(struct hci_dev *hdev, struct iov_iter *iter);
+
 int __hci_cmd_send(struct hci_dev *hdev, u16 opcode, u32 plen,
 		   const void *param);
 
diff --git a/include/net/bluetooth/hci_h4.h b/include/net/bluetooth/hci_h4.h
new file mode 100644
index 00000000000000..a37e7df8c9ce8d
--- /dev/null
+++ b/include/net/bluetooth/hci_h4.h
@@ -0,0 +1,60 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *  Bluetooth HCI H:4 packet reassembly
+ *
+ *  Copyright (C) 2000-2001  Qualcomm Incorporated
+ *  Copyright (C) 2002-2003  Maxim Krasnyansky <maxk@qualcomm.com>
+ *  Copyright (C) 2004-2005  Marcel Holtmann <marcel@holtmann.org>
+ */
+
+#ifndef __HCI_H4_H
+#define __HCI_H4_H
+
+#include <linux/skbuff.h>
+#include <linux/types.h>
+
+struct hci_dev;
+
+struct h4_recv_pkt {
+	u8  type;	/* Packet type */
+	u8  hlen;	/* Header length */
+	u8  loff;	/* Data length offset in header */
+	u8  lsize;	/* Data length field size */
+	u16 maxlen;	/* Max overall packet length */
+	int (*recv)(struct hci_dev *hdev, struct sk_buff *skb);
+};
+
+#define H4_RECV_ACL \
+	.type = HCI_ACLDATA_PKT, \
+	.hlen = HCI_ACL_HDR_SIZE, \
+	.loff = 2, \
+	.lsize = 2, \
+	.maxlen = HCI_MAX_FRAME_SIZE \
+
+#define H4_RECV_SCO \
+	.type = HCI_SCODATA_PKT, \
+	.hlen = HCI_SCO_HDR_SIZE, \
+	.loff = 2, \
+	.lsize = 1, \
+	.maxlen = HCI_MAX_SCO_SIZE
+
+#define H4_RECV_EVENT \
+	.type = HCI_EVENT_PKT, \
+	.hlen = HCI_EVENT_HDR_SIZE, \
+	.loff = 1, \
+	.lsize = 1, \
+	.maxlen = HCI_MAX_EVENT_SIZE
+
+#define H4_RECV_ISO \
+	.type = HCI_ISODATA_PKT, \
+	.hlen = HCI_ISO_HDR_SIZE, \
+	.loff = 2, \
+	.lsize = 2, \
+	.maxlen = HCI_MAX_FRAME_SIZE \
+
+struct sk_buff *h4_recv_skb(struct hci_dev *hdev, u8 *alignment, u8 *padding,
+			    struct sk_buff *skb, const unsigned char *buffer,
+			    int count, const struct h4_recv_pkt *pkts,
+			    int pkts_count);
+
+#endif /* __HCI_H4_H */
diff --git a/include/net/bluetooth/hci_mon.h b/include/net/bluetooth/hci_mon.h
index 4b2a0af4ed58b4..7710688c0d309d 100644
--- a/include/net/bluetooth/hci_mon.h
+++ b/include/net/bluetooth/hci_mon.h
@@ -50,6 +50,8 @@ struct hci_mon_hdr {
 #define HCI_MON_ISO_RX_PKT	19
 #define HCI_MON_DRV_TX_PKT	20
 #define HCI_MON_DRV_RX_PKT	21
+#define HCI_MON_VENDOR_TX_PKT	22
+#define HCI_MON_VENDOR_RX_PKT	23
 
 struct hci_mon_new_index {
 	__u8		type;
diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h
index 3d9a32094347cd..efb9b7f422d1d8 100644
--- a/include/net/bluetooth/l2cap.h
+++ b/include/net/bluetooth/l2cap.h
@@ -614,31 +614,6 @@ struct l2cap_chan {
 	struct mutex		lock;
 };
 
-struct l2cap_ops {
-	char			*name;
-
-	int			(*new_connection)(struct l2cap_chan *chan,
-						  struct l2cap_chan *new_chan);
-	int			(*recv) (struct l2cap_chan * chan,
-					 struct sk_buff *skb);
-	void			(*teardown) (struct l2cap_chan *chan, int err);
-	void			(*close) (struct l2cap_chan *chan);
-	void			(*state_change) (struct l2cap_chan *chan,
-						 int state, int err);
-	void			(*ready) (struct l2cap_chan *chan);
-	void			(*defer) (struct l2cap_chan *chan);
-	void			(*resume) (struct l2cap_chan *chan);
-	void			(*suspend) (struct l2cap_chan *chan);
-	void			(*set_shutdown) (struct l2cap_chan *chan);
-	long			(*get_sndtimeo) (struct l2cap_chan *chan);
-	struct pid		*(*get_peer_pid) (struct l2cap_chan *chan);
-	struct sk_buff		*(*alloc_skb) (struct l2cap_chan *chan,
-					       unsigned long hdr_len,
-					       unsigned long len, int nb);
-	int			(*filter) (struct l2cap_chan * chan,
-					   struct sk_buff *skb);
-};
-
 struct l2cap_conn {
 	struct hci_conn		*hcon;
 	struct hci_chan		*hchan;
@@ -668,12 +643,45 @@ struct l2cap_conn {
 
 	struct l2cap_chan	*smp;
 
-	struct list_head	chan_l;
+	struct list_head	chan_l __guarded_by(&lock);
 	struct mutex		lock;
 	struct kref		ref;
 	struct list_head	users;
 };
 
+struct l2cap_ops {
+	char			*name;
+
+	int			(*new_connection)(struct l2cap_chan *chan,
+						  struct l2cap_chan *new_chan)
+					__must_hold(&chan->lock)
+					__must_hold(&new_chan->lock);
+	int			(*recv) (struct l2cap_chan * chan,
+					 struct sk_buff *skb);
+	void			(*teardown) (struct l2cap_chan *chan, int err)
+					__must_hold(&chan->lock);
+	void			(*close) (struct l2cap_chan *chan)
+					__must_hold(&chan->lock);
+	void			(*state_change) (struct l2cap_chan *chan,
+						 int state, int err);
+	void			(*ready) (struct l2cap_chan *chan)
+					__must_hold(&chan->lock)
+					__must_hold(&chan->conn->lock);
+	void			(*defer) (struct l2cap_chan *chan);
+	void			(*resume) (struct l2cap_chan *chan)
+					__must_hold(&chan->lock);
+	void			(*suspend) (struct l2cap_chan *chan);
+	void			(*set_shutdown) (struct l2cap_chan *chan)
+					__must_hold(&chan->lock);
+	long			(*get_sndtimeo) (struct l2cap_chan *chan);
+	struct pid		*(*get_peer_pid) (struct l2cap_chan *chan);
+	struct sk_buff		*(*alloc_skb) (struct l2cap_chan *chan,
+					       unsigned long hdr_len,
+					       unsigned long len, int nb);
+	int			(*filter) (struct l2cap_chan * chan,
+					   struct sk_buff *skb);
+};
+
 struct l2cap_user {
 	struct list_head list;
 	int (*probe) (struct l2cap_conn *conn, struct l2cap_user *user);
@@ -758,6 +766,10 @@ enum {
  * otherwise considers all channels equal and will e.g. complain about a
  * connection oriented channel triggering SMP procedures or a listening
  * channel creating and locking a child channel.
+ *
+ * Lock nesting of channels at the same nesting level is allowed if the channels
+ * have the same l2cap_chan::conn and l2cap_chan::conn.lock is taken before the
+ * nested locks. l2cap_chan_try_sibling_lock() must be used.
  */
 enum {
 	L2CAP_NESTING_SMP,
@@ -830,11 +842,13 @@ struct l2cap_chan *l2cap_chan_hold_unless_zero(struct l2cap_chan *c);
 void l2cap_chan_put(struct l2cap_chan *c);
 
 static inline void l2cap_chan_lock(struct l2cap_chan *chan)
+	__acquires(&chan->lock)
 {
 	mutex_lock_nested(&chan->lock, atomic_read(&chan->nesting));
 }
 
 static inline void l2cap_chan_unlock(struct l2cap_chan *chan)
+	__releases(&chan->lock)
 {
 	mutex_unlock(&chan->lock);
 }
@@ -845,12 +859,11 @@ static inline void l2cap_set_timer(struct l2cap_chan *chan,
 	BT_DBG("chan %p state %s timeout %ld", chan,
 	       state_to_string(chan->state), timeout);
 
-	/* If delayed work cancelled do not hold(chan)
-	   since it is already done with previous set_timer */
-	if (!cancel_delayed_work(work))
-		l2cap_chan_hold(chan);
+	l2cap_chan_hold(chan);
 
-	schedule_delayed_work(work, timeout);
+	/* put(chan) if timer was already queued so it already has a ref */
+	if (mod_delayed_work(system_percpu_wq, work, timeout))
+		l2cap_chan_put(chan);
 }
 
 static inline bool l2cap_clear_timer(struct l2cap_chan *chan,
@@ -952,14 +965,16 @@ void l2cap_cleanup_sockets(void);
 bool l2cap_is_socket(struct socket *sock);
 
 void __l2cap_le_connect_rsp_defer(struct l2cap_chan *chan);
-void __l2cap_ecred_conn_rsp_defer(struct l2cap_chan *chan);
+void __l2cap_ecred_conn_rsp_defer(struct l2cap_chan *chan)
+	__must_hold(&chan->lock) __must_hold(&chan->conn->lock);
 void __l2cap_connect_rsp_defer(struct l2cap_chan *chan);
 
 int l2cap_add_psm(struct l2cap_chan *chan, bdaddr_t *src, __le16 psm);
 int l2cap_add_scid(struct l2cap_chan *chan,  __u16 scid);
 
 struct l2cap_chan *l2cap_chan_create(void);
-void l2cap_chan_close(struct l2cap_chan *chan, int reason);
+void l2cap_chan_close_unlocked(struct l2cap_chan *chan, int reason)
+	__must_not_hold(&chan->lock);
 int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid,
 		       bdaddr_t *dst, u8 dst_type, u16 timeout);
 int l2cap_chan_reconfigure(struct l2cap_chan *chan, __u16 mtu);
@@ -971,11 +986,13 @@ int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator);
 void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan);
 int l2cap_ertm_init(struct l2cap_chan *chan);
 void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan);
-void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan);
+void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan)
+	__must_hold(&conn->lock) __must_hold(&chan->lock);
 typedef void (*l2cap_chan_func_t)(struct l2cap_chan *chan, void *data);
 void l2cap_chan_list(struct l2cap_conn *conn, l2cap_chan_func_t func,
 		     void *data);
-void l2cap_chan_del(struct l2cap_chan *chan, int err);
+void l2cap_chan_del(struct l2cap_chan *chan, int err)
+	__must_hold(&chan->lock) __must_hold(&chan->conn->lock);
 void l2cap_send_conn_req(struct l2cap_chan *chan);
 
 struct l2cap_conn *l2cap_conn_get(struct l2cap_conn *conn);
@@ -985,4 +1002,19 @@ void l2cap_conn_put(struct l2cap_conn *conn);
 int l2cap_register_user(struct l2cap_conn *conn, struct l2cap_user *user);
 void l2cap_unregister_user(struct l2cap_conn *conn, struct l2cap_user *user);
 
+bool l2cap_chan_lock_conn(struct l2cap_chan *chan)
+	__acquires(&chan->lock) __cond_acquires(true, &chan->conn->lock);
+
+/* Release macro for l2cap_chan_lock_conn, so context analysis understands it */
+#define l2cap_chan_unlock_conn(chan, conn_locked)			\
+	({								\
+		struct l2cap_chan *__chan = (chan);			\
+		struct l2cap_conn *__conn = __chan->conn;		\
+		l2cap_chan_unlock(__chan);				\
+		if (conn_locked) {					\
+			mutex_unlock(&__conn->lock);			\
+			l2cap_conn_put(__conn);				\
+		}							\
+	})
+
 #endif /* __L2CAP_H */
diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h
index 102c278e35844c..9dc5fd4cb8c815 100644
--- a/include/net/bluetooth/rfcomm.h
+++ b/include/net/bluetooth/rfcomm.h
@@ -207,6 +207,7 @@ struct rfcomm_dlc {
 #define RFCOMM_AUTH_REJECT  7
 #define RFCOMM_DEFER_SETUP  8
 #define RFCOMM_ENC_DROP     9
+#define RFCOMM_CLOSED       10
 
 /* Scheduling flags and events */
 #define RFCOMM_SCHED_WAKEUP 31
diff --git a/include/net/bond_alb.h b/include/net/bond_alb.h
index e5945427f38ddd..32f1981033e4c1 100644
--- a/include/net/bond_alb.h
+++ b/include/net/bond_alb.h
@@ -57,12 +57,12 @@ struct tlb_client_info {
 				 * packets to a Client that the Hash function
 				 * gave this entry index.
 				 */
-	u32 tx_bytes;		/* Each Client accumulates the BytesTx that
+	u64 tx_bytes;		/* Each Client accumulates the BytesTx that
 				 * were transmitted to it, and after each
 				 * CallBack the LoadHistory is divided
 				 * by the balance interval
 				 */
-	u32 load_history;	/* This field contains the amount of Bytes
+	u64 load_history;	/* This field contains the amount of Bytes
 				 * that were transmitted to this client by
 				 * the server on the previous balance
 				 * interval in Bps.
@@ -118,14 +118,20 @@ struct tlb_slave_info {
 			 * are the entries that were assigned to use this
 			 * slave for transmit.
 			 */
-	u32 load;	/* Each slave sums the loadHistory of all clients
+	u64 load;	/* Each slave sums the loadHistory of all clients
 			 * assigned to it
 			 */
 };
 
+struct unbalanced_load_stats {
+	u64_stats_t		tx_bytes;
+	struct u64_stats_sync	syncp;
+};
+
 struct alb_bond_info {
 	struct tlb_client_info	*tx_hashtbl; /* Dynamically allocated */
-	u32			unbalanced_load;
+	struct unbalanced_load_stats __percpu	*unbalanced_load;
+	u64			prev_total_unbalanced;
 	atomic_t		tx_rebalance_counter;
 	int			lp_counter;
 	/* -------- rlb parameters -------- */
diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h
index 97c16d4ff1271b..b603f1804cc26f 100644
--- a/include/net/cfg80211.h
+++ b/include/net/cfg80211.h
@@ -576,7 +576,6 @@ struct ieee80211_sta_s1g_cap {
  * @vht_cap: VHT capabilities in this band
  * @s1g_cap: S1G capabilities in this band
  * @edmg_cap: EDMG capabilities in this band
- * @s1g_cap: S1G capabilities in this band (S1G band only, of course)
  * @n_iftype_data: number of iftype data entries
  * @iftype_data: interface type data entries.  Note that the bits in
  *	@types_mask inside this structure cannot overlap (i.e. only
@@ -901,6 +900,7 @@ struct cfg80211_bitrate_mask {
 		enum nl80211_eht_gi eht_gi;
 		enum nl80211_he_ltf he_ltf;
 		enum nl80211_eht_ltf eht_ltf;
+		bool nonht_dup_6ghz;
 	} control[NUM_NL80211_BANDS];
 };
 
@@ -1821,6 +1821,8 @@ struct sta_txpwr {
  * @s1g_capa: S1G capabilities of station
  * @uhr_capa: UHR capabilities of the station
  * @uhr_capa_len: the length of the UHR capabilities
+ * @cip_cap_set: If CIP capabilities are present, required for CIP stations
+ * @cip_cap: CIP capabilities of station
  */
 struct link_station_parameters {
 	const u8 *mld_mac;
@@ -1842,6 +1844,8 @@ struct link_station_parameters {
 	const struct ieee80211_s1g_cap *s1g_capa;
 	const struct ieee80211_uhr_cap *uhr_capa;
 	u8 uhr_capa_len;
+	bool cip_cap_set;
+	u8 cip_cap;
 };
 
 /**
@@ -3402,6 +3406,7 @@ struct cfg80211_ml_reconf_req {
  *	flag is not set.
  * @ASSOC_REQ_SPP_AMSDU: SPP A-MSDUs will be used on this connection (if any)
  * @ASSOC_REQ_DISABLE_UHR: Disable UHR
+ * @ASSOC_REQ_CIP: Enable Control Integrity Protocol
  */
 enum cfg80211_assoc_req_flags {
 	ASSOC_REQ_DISABLE_HT			= BIT(0),
@@ -3413,6 +3418,7 @@ enum cfg80211_assoc_req_flags {
 	CONNECT_REQ_MLO_SUPPORT			= BIT(6),
 	ASSOC_REQ_SPP_AMSDU			= BIT(7),
 	ASSOC_REQ_DISABLE_UHR			= BIT(8),
+	ASSOC_REQ_CIP				= BIT(9),
 };
 
 /**
@@ -4009,6 +4015,7 @@ struct cfg80211_update_ft_ies_params {
  * @link_id: for MLO, the link ID to transmit on, -1 if not given; note
  *	that the link ID isn't validated (much), it's in range but the
  *	link might not exist (or be used by the receiver STA)
+ * @no_sta: set if the frame should not be transmitted using an existing STA
  */
 struct cfg80211_mgmt_tx_params {
 	struct ieee80211_channel *chan;
@@ -4021,6 +4028,7 @@ struct cfg80211_mgmt_tx_params {
 	int n_csa_offsets;
 	const u16 *csa_offsets;
 	int link_id;
+	bool no_sta;
 };
 
 /**
@@ -4256,6 +4264,21 @@ struct cfg80211_nan_local_sched {
 	struct cfg80211_nan_channel nan_channels[] __counted_by(n_channels);
 };
 
+/**
+ * struct cfg80211_nan_non_evac_channels - NAN non-evacuable channels
+ *
+ * This struct defines the set of NAN local schedule channels that must not
+ * be evacuated for concurrent operations.
+ *
+ * @n_channels: number of channel definitions in %chandefs.
+ * @chandefs: array of channel definitions that must not be evacuated. Each
+ *	must match a channel of the current local schedule.
+ */
+struct cfg80211_nan_non_evac_channels {
+	u8 n_channels;
+	struct cfg80211_chan_def chandefs[] __counted_by(n_channels);
+};
+
 /**
  * struct cfg80211_nan_peer_map - NAN peer schedule map
  *
@@ -5204,6 +5227,12 @@ struct mgmt_frame_regs {
  *	schedule, the full new schedule is provided - partial updates are not
  *	supported, and the new schedule completely replaces the previous one.
  *
+ * @nan_set_non_evac_channels: set the list of local schedule channels that
+ *	must not be evacuated for concurrent operations. The provided list
+ *	replaces the previous set; channels of the current schedule that are
+ *	not included become evacuable again. All provided channels are
+ *	guaranteed by cfg80211 to belong to the current local schedule.
+ *
  * @set_multicast_to_unicast: configure multicast to unicast conversion for BSS
  *
  * @get_txq_stats: Get TXQ stats for interface or phy. If wdev is %NULL, this
@@ -5302,14 +5331,17 @@ struct cfg80211_ops {
 				 unsigned int link_id);
 
 	int	(*add_key)(struct wiphy *wiphy, struct wireless_dev *wdev,
-			   int link_id, u8 key_index, bool pairwise,
+			   int link_id, u8 key_index,
+			   enum nl80211_key_type type,
 			   const u8 *mac_addr, struct key_params *params);
 	int	(*get_key)(struct wiphy *wiphy, struct wireless_dev *wdev,
-			   int link_id, u8 key_index, bool pairwise,
+			   int link_id, u8 key_index,
+			   enum nl80211_key_type type,
 			   const u8 *mac_addr, void *cookie,
 			   void (*callback)(void *cookie, struct key_params*));
 	int	(*del_key)(struct wiphy *wiphy, struct wireless_dev *wdev,
-			   int link_id, u8 key_index, bool pairwise,
+			   int link_id, u8 key_index,
+			   enum nl80211_key_type type,
 			   const u8 *mac_addr);
 	int	(*set_default_key)(struct wiphy *wiphy,
 				   struct net_device *netdev, int link_id,
@@ -5590,6 +5622,9 @@ struct cfg80211_ops {
 	int	(*nan_set_peer_sched)(struct wiphy *wiphy,
 				      struct wireless_dev *wdev,
 				      struct cfg80211_nan_peer_sched *sched);
+	int	(*nan_set_non_evac_channels)(struct wiphy *wiphy,
+					     struct wireless_dev *wdev,
+					     struct cfg80211_nan_non_evac_channels *channels);
 	int	(*set_multicast_to_unicast)(struct wiphy *wiphy,
 					    struct net_device *dev,
 					    const bool enabled);
@@ -6043,6 +6078,8 @@ struct wiphy_vendor_command {
  * @eml_capabilities: EML capabilities (for MLO)
  * @mld_capa_and_ops: MLD capabilities and operations (for MLO)
  * @ext_mld_capa_and_ops: Extended MLD capabilities and operations (for MLO)
+ * @cip_supported: CIP is supported and the capabilities are valid
+ * @cip_capabilities: CIP Capabilities element containing the MIC padding delay
  */
 struct wiphy_iftype_ext_capab {
 	enum nl80211_iftype iftype;
@@ -6052,6 +6089,8 @@ struct wiphy_iftype_ext_capab {
 	u16 eml_capabilities;
 	u16 mld_capa_and_ops;
 	u16 ext_mld_capa_and_ops;
+	bool cip_supported;
+	u8 cip_capabilities;
 };
 
 /**
@@ -7332,6 +7371,8 @@ struct wireless_dev {
 		union {
 			struct {
 				unsigned int beacon_interval;
+				enum ieee80211_ap_reg_power reg_power;
+				enum ieee80211_ap_reg_power csa_reg_power;
 				struct cfg80211_chan_def chandef;
 			} ap;
 			struct {
@@ -9501,6 +9542,7 @@ void cfg80211_conn_failed(struct net_device *dev, const u8 *mac_addr,
  * @flags: flags, as defined in &enum nl80211_rxmgmt_flags
  * @rx_tstamp: Hardware timestamp of frame RX in nanoseconds
  * @ack_tstamp: Hardware timestamp of ack TX in nanoseconds
+ * @no_sta: set if no station is known for the frame (relevant for MLD)
  */
 struct cfg80211_rx_info {
 	int freq;
@@ -9512,6 +9554,7 @@ struct cfg80211_rx_info {
 	u32 flags;
 	u64 rx_tstamp;
 	u64 ack_tstamp;
+	bool no_sta;
 };
 
 /**
diff --git a/include/net/cfg802154.h b/include/net/cfg802154.h
index 2e960441ea49e3..2c5dcf6a106f5b 100644
--- a/include/net/cfg802154.h
+++ b/include/net/cfg802154.h
@@ -458,15 +458,16 @@ struct ieee802154_llsec_device_key {
 	u32 frame_counter;
 };
 
-struct wpan_dev_header_ops {
+struct wpan_dev_ops {
 	/* TODO create callback currently assumes ieee802154_mac_cb inside
 	 * skb->cb. This should be changed to give these information as
 	 * parameter.
 	 */
-	int	(*create)(struct sk_buff *skb, struct net_device *dev,
+	int	(*header_create)(struct sk_buff *skb, struct net_device *dev,
 			  const struct ieee802154_addr *daddr,
 			  const struct ieee802154_addr *saddr,
 			  unsigned int len);
+	int	(*ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
 };
 
 struct wpan_dev {
@@ -477,7 +478,7 @@ struct wpan_dev {
 	struct list_head list;
 	struct net_device *netdev;
 
-	const struct wpan_dev_header_ops *header_ops;
+	const struct wpan_dev_ops *ops;
 
 	/* lowpan interface, set when the wpan_dev belongs to one lowpan_dev */
 	struct net_device *lowpan_dev;
@@ -523,7 +524,7 @@ wpan_dev_hard_header(struct sk_buff *skb, struct net_device *dev,
 {
 	struct wpan_dev *wpan_dev = dev->ieee802154_ptr;
 
-	return wpan_dev->header_ops->create(skb, dev, daddr, saddr, len);
+	return wpan_dev->ops->header_create(skb, dev, daddr, saddr, len);
 }
 #endif
 
diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h
index 2f312d1f67d698..12f90965159129 100644
--- a/include/net/dropreason-core.h
+++ b/include/net/dropreason-core.h
@@ -93,6 +93,7 @@
 	FN(IP_INVALID_SOURCE)		\
 	FN(IP_LOCALNET)			\
 	FN(IP_INVALID_DEST)		\
+	FN(IP_TTL_EXCEEDED)		\
 	FN(PKT_TOO_BIG)			\
 	FN(DUP_FRAG)			\
 	FN(FRAG_REASM_TIMEOUT)		\
@@ -474,6 +475,11 @@ enum skb_drop_reason {
 	 * 1) dest ip is 0
 	 */
 	SKB_DROP_REASON_IP_INVALID_DEST,
+	/**
+	 * @SKB_DROP_REASON_IP_TTL_EXCEEDED: IPv4 TTL or IPv6 hop limit <= 1
+	 * (see IPSTATS_MIB_INHDRERRORS)
+	 */
+	SKB_DROP_REASON_IP_TTL_EXCEEDED,
 	/**
 	 * @SKB_DROP_REASON_PKT_TOO_BIG: packet size is too big (maybe exceed the
 	 * MTU)
diff --git a/include/net/dsa.h b/include/net/dsa.h
index 7507d632e7c6df..5d12191b6f6f52 100644
--- a/include/net/dsa.h
+++ b/include/net/dsa.h
@@ -61,6 +61,7 @@ struct tc_action;
 #define DSA_TAG_PROTO_NETC_VALUE		33
 #define DSA_TAG_PROTO_KSZ8463_VALUE		34
 #define DSA_TAG_PROTO_MT7628_VALUE		35
+#define DSA_TAG_PROTO_KS8995_VALUE		36
 
 enum dsa_tag_protocol {
 	DSA_TAG_PROTO_NONE		= DSA_TAG_PROTO_NONE_VALUE,
@@ -99,6 +100,7 @@ enum dsa_tag_protocol {
 	DSA_TAG_PROTO_NETC		= DSA_TAG_PROTO_NETC_VALUE,
 	DSA_TAG_PROTO_KSZ8463		= DSA_TAG_PROTO_KSZ8463_VALUE,
 	DSA_TAG_PROTO_MT7628		= DSA_TAG_PROTO_MT7628_VALUE,
+	DSA_TAG_PROTO_KS8995		= DSA_TAG_PROTO_KS8995_VALUE,
 };
 
 struct dsa_switch;
diff --git a/include/net/dst.h b/include/net/dst.h
index 307073eae7f834..dbedfe72e1fd15 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -455,7 +455,8 @@ static inline unsigned int dst_dev_overhead(struct dst_entry *dst,
 					    struct sk_buff *skb)
 {
 	if (likely(dst))
-		return LL_RESERVED_SPACE(dst->dev);
+		return max_t(unsigned int, skb->mac_len,
+			     LL_RESERVED_SPACE(dst->dev));
 
 	return skb->mac_len;
 }
diff --git a/include/net/gso.h b/include/net/gso.h
index 29975440cad51e..0749230d414ecb 100644
--- a/include/net/gso.h
+++ b/include/net/gso.h
@@ -19,10 +19,18 @@ struct skb_gso_cb {
 	int	encap_level;
 	__wsum	csum;
 	__u16	csum_start;
+	/* Number of IPv4/IPv6 GSO handler entries for this packet. */
+	u8	recursion_counter;
 };
 #define SKB_GSO_CB_OFFSET	32
 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET))
 
+static inline bool gso_recursion_inc_test(struct sk_buff *skb,
+					  unsigned int limit)
+{
+	return ++SKB_GSO_CB(skb)->recursion_counter > limit;
+}
+
 static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
 {
 	return (skb_mac_header(inner_skb) - inner_skb->head) -
diff --git a/include/net/gue.h b/include/net/gue.h
index caefd6da86939b..d377155fd0b318 100644
--- a/include/net/gue.h
+++ b/include/net/gue.h
@@ -84,8 +84,9 @@ static inline size_t guehdr_priv_flags_len(__be32 flags)
 }
 
 /* Validate standard and private flags. Returns non-zero (meaning invalid)
- * if there is an unknown standard or private flags, or the options length for
- * the flags exceeds the options length specific in hlen of the GUE header.
+ * if there is an unknown standard or private flags, if the options length for
+ * the flags exceeds the options length specified in hlen of the GUE header, or
+ * if a private option contains invalid data.
  */
 static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen)
 {
@@ -103,8 +104,8 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen)
 		/* Private flags are last four bytes accounted in
 		 * guehdr_flags_len
 		 */
-		__be32 pflags = *(__be32 *)((void *)&guehdr[1] +
-					    len - GUE_LEN_PRIV);
+		void *data = (void *)&guehdr[1] + len;
+		__be32 pflags = *(__be32 *)(data - GUE_LEN_PRIV);
 
 		if (pflags & ~GUE_PFLAGS_ALL)
 			return 1;
@@ -112,6 +113,16 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen)
 		len += guehdr_priv_flags_len(pflags);
 		if (len > optlen)
 			return 1;
+
+		if (pflags & GUE_PFLAG_REMCSUM) {
+			__be16 *pd = data;
+
+			/* The field offset pd[1] must not be less
+			 * than the start pd[0].
+			 */
+			if (ntohs(pd[1]) < ntohs(pd[0]))
+				return 1;
+		}
 	}
 
 	return 0;
diff --git a/include/net/ieee80211_radiotap.h b/include/net/ieee80211_radiotap.h
index 8bbaf77da7cfc8..782c3dcd8b524c 100644
--- a/include/net/ieee80211_radiotap.h
+++ b/include/net/ieee80211_radiotap.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2017		Intel Deutschland GmbH
- * Copyright (c) 2018-2019, 2021-2022, 2025 Intel Corporation
+ * Copyright (c) 2018-2019, 2021-2022, 2025-2026 Intel Corporation
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -93,6 +93,7 @@ enum ieee80211_radiotap_presence {
 	IEEE80211_RADIOTAP_RADIOTAP_NAMESPACE = 29,
 	IEEE80211_RADIOTAP_VENDOR_NAMESPACE = 30,
 	IEEE80211_RADIOTAP_EXT = 31,
+	IEEE80211_RADIOTAP_S1G = 32,
 	IEEE80211_RADIOTAP_EHT_USIG = 33,
 	IEEE80211_RADIOTAP_EHT = 34,
 	IEEE80211_RADIOTAP_UHR_ELR = 37,
@@ -112,6 +113,9 @@ enum ieee80211_radiotap_flags {
 
 /* for IEEE80211_RADIOTAP_CHANNEL */
 enum ieee80211_radiotap_channel_flags {
+	IEEE80211_CHAN_700MHZ = 0x0001,
+	IEEE80211_CHAN_800MHZ = 0x0002,
+	IEEE80211_CHAN_900MHZ = 0x0004,
 	IEEE80211_CHAN_CCK = 0x0020,
 	IEEE80211_CHAN_OFDM = 0x0040,
 	IEEE80211_CHAN_2GHZ = 0x0080,
@@ -592,7 +596,7 @@ enum ieee80211_radiotap_eht_usig_mu {
 
 enum ieee80211_radiotap_eht_usig_tb {
 	/* TB-USIG-1 */
-	IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD	= 0x0000001f,
+	IEEE80211_RADIOTAP_EHT_USIG1_TB_B20_B25_DISREGARD	= 0x0000003f,
 
 	/* TB-USIG-2 */
 	IEEE80211_RADIOTAP_EHT_USIG2_TB_B0_B1_PPDU_TYPE		= 0x000000c0,
@@ -604,6 +608,49 @@ enum ieee80211_radiotap_eht_usig_tb {
 	IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL		= 0xfc000000,
 };
 
+enum ieee80211_radiotap_uhr_usig_mu {
+	/* USIG-1 */
+	IEEE80211_RADIOTAP_UHR_USIG1_MU_BSS_COLOR_2		= 0x0000003f,
+	IEEE80211_RADIOTAP_UHR_USIG1_MU_B20_B24_DISREGARD	= 0x0000001f,
+	IEEE80211_RADIOTAP_UHR_USIG1_MU_B25_VALIDATE		= 0x00000020,
+	/* USIG-2 */
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B0_B1_PPDU_TYPE		= 0x000000c0,
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B2_COBF_COSR		= 0x00000100,
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B3_B7_PUNCTURED_INFO	= 0x00003e00,
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B8_VALIDATE		= 0x00004000,
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B9_B10_SIG_MCS		= 0x00018000,
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B11_B15_UHR_SIG_SYMBOLS	= 0x003e0000,
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B16_B19_CRC		= 0x03c00000,
+	IEEE80211_RADIOTAP_UHR_USIG2_MU_B20_B25_TAIL		= 0xfc000000,
+};
+
+enum ieee80211_radiotap_uhr_usig_tb {
+	/* USIG-1 */
+	IEEE80211_RADIOTAP_UHR_USIG1_TB_B20_B25_DISREGARD	= 0x0000003f,
+
+	/* USIG-2 */
+	IEEE80211_RADIOTAP_UHR_USIG2_TB_B0_B1_PPDU_TYPE		= 0x000000c0,
+	IEEE80211_RADIOTAP_UHR_USIG2_TB_B2_VALIDATE		= 0x00000100,
+	IEEE80211_RADIOTAP_UHR_USIG2_TB_B3_B6_SPATIAL_REUSE_1	= 0x00001e00,
+	IEEE80211_RADIOTAP_UHR_USIG2_TB_B7_B10_SPATIAL_REUSE_2	= 0x0001e000,
+	IEEE80211_RADIOTAP_UHR_USIG2_TB_B11_B15_DISREGARD	= 0x003e0000,
+	IEEE80211_RADIOTAP_UHR_USIG2_TB_B16_B19_CRC		= 0x03c00000,
+	IEEE80211_RADIOTAP_UHR_USIG2_TB_B20_B25_TAIL		= 0xfc000000,
+};
+
+enum ieee80211_radiotap_uhr_usig_elr {
+	/* USIG-1 */
+	IEEE80211_RADIOTAP_UHR_USIG1_ELR_B20_B24_DISREGARD	= 0x0000001f,
+	IEEE80211_RADIOTAP_UHR_USIG1_ELR_B25_VALIDATE		= 0x00000020,
+
+	/* USIG-2 */
+	IEEE80211_RADIOTAP_UHR_USIG2_ELR_B0_B1_PPDU_TYPE	= 0x000000c0,
+	IEEE80211_RADIOTAP_UHR_USIG2_ELR_B2_B12_STA_ID		= 0x0007ff00,
+	IEEE80211_RADIOTAP_UHR_USIG2_ELR_B13_B15_VALIDATE	= 0x00380000,
+	IEEE80211_RADIOTAP_UHR_USIG2_ELR_B16_B19_CRC		= 0x03c00000,
+	IEEE80211_RADIOTAP_UHR_USIG2_ELR_B20_B25_TAIL		= 0xfc000000,
+};
+
 /*
  * ieee80211_radiotap_uhr_elr - content of UHR-ELR TLV (type 37)
  * see https://www.radiotap.org/fields/UHR-ELR for details
@@ -792,6 +839,68 @@ enum ieee80211_radiotap_uhr_user_info {
 	IEEE80211_RADIOTAP_UHR_USER_INFO_2X_LDPC		= 0x02000000,
 };
 
+/*
+ * ieee80211_radiotap_s1g - content of S1G TLV (type 32)
+ * see www.radiotap.org/fields/S1G.html for details
+ */
+struct ieee80211_radiotap_s1g {
+	__le16 known;
+	__le16 data1;
+	__le16 data2;
+} __packed;
+
+enum ieee80211_radiotap_s1g_known {
+	IEEE80211_RADIOTAP_S1G_KNOWN_PPDU_FORMAT	= 0x0001,
+	IEEE80211_RADIOTAP_S1G_KNOWN_RESPONSE_IND	= 0x0002,
+	IEEE80211_RADIOTAP_S1G_KNOWN_GI			= 0x0004,
+	IEEE80211_RADIOTAP_S1G_KNOWN_NSS		= 0x0008,
+	IEEE80211_RADIOTAP_S1G_KNOWN_BW			= 0x0010,
+	IEEE80211_RADIOTAP_S1G_KNOWN_MCS		= 0x0020,
+	IEEE80211_RADIOTAP_S1G_KNOWN_COLOR		= 0x0040,
+	IEEE80211_RADIOTAP_S1G_KNOWN_UPLINK_IND		= 0x0080,
+};
+
+enum ieee80211_radiotap_s1g_data1 {
+	IEEE80211_RADIOTAP_S1G_DATA1_PPDU_FORMAT	= 0x0003,
+	IEEE80211_RADIOTAP_S1G_DATA1_RESPONSE_IND	= 0x000c,
+	IEEE80211_RADIOTAP_S1G_DATA1_GI			= 0x0020,
+	IEEE80211_RADIOTAP_S1G_DATA1_NSS		= 0x00c0,
+	IEEE80211_RADIOTAP_S1G_DATA1_BW			= 0x0f00,
+	IEEE80211_RADIOTAP_S1G_DATA1_MCS		= 0xf000,
+};
+
+enum ieee80211_radiotap_s1g_data2 {
+	IEEE80211_RADIOTAP_S1G_DATA2_COLOR		= 0x0007,
+	IEEE80211_RADIOTAP_S1G_DATA2_UPLINK_IND		= 0x0008,
+	IEEE80211_RADIOTAP_S1G_DATA2_RSSI		= 0xff00,
+};
+
+enum ieee80211_radiotap_s1g_ppdu_format {
+	IEEE80211_RADIOTAP_S1G_PPDU_FORMAT_1M		= 0,
+	IEEE80211_RADIOTAP_S1G_PPDU_FORMAT_SHORT	= 1,
+	IEEE80211_RADIOTAP_S1G_PPDU_FORMAT_LONG		= 2,
+};
+
+enum ieee80211_radiotap_s1g_response_ind {
+	IEEE80211_RADIOTAP_S1G_RESPONSE_IND_NONE	= 0,
+	IEEE80211_RADIOTAP_S1G_RESPONSE_IND_NDP		= 1,
+	IEEE80211_RADIOTAP_S1G_RESPONSE_IND_NORMAL	= 2,
+	IEEE80211_RADIOTAP_S1G_RESPONSE_IND_LONG	= 3,
+};
+
+enum ieee80211_radiotap_s1g_gi {
+	IEEE80211_RADIOTAP_S1G_GI_LONG			= 0,
+	IEEE80211_RADIOTAP_S1G_GI_SHORT			= 1,
+};
+
+enum ieee80211_radiotap_s1g_bw {
+	IEEE80211_RADIOTAP_S1G_BW_1MHZ			= 0,
+	IEEE80211_RADIOTAP_S1G_BW_2MHZ			= 1,
+	IEEE80211_RADIOTAP_S1G_BW_4MHZ			= 2,
+	IEEE80211_RADIOTAP_S1G_BW_8MHZ			= 3,
+	IEEE80211_RADIOTAP_S1G_BW_16MHZ			= 4,
+};
+
 /**
  * ieee80211_get_radiotap_len - get radiotap header length
  * @data: pointer to the header
diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
index 433c2df23076d8..903499581db7e2 100644
--- a/include/net/inet_connection_sock.h
+++ b/include/net/inet_connection_sock.h
@@ -94,6 +94,7 @@ struct inet_connection_sock {
 	u32			  icsk_rto_max;
 	__u32                     icsk_delack_max;
 	__u32			  icsk_pmtu_cookie;
+	unsigned char		  unhashed_state;
 	const struct tcp_congestion_ops *icsk_ca_ops;
 	const struct inet_connection_sock_af_ops *icsk_af_ops;
 	const struct tcp_ulp_ops  *icsk_ulp_ops;
diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h
index b9e8d2b759e9b7..0f9b7a260d2537 100644
--- a/include/net/ip6_route.h
+++ b/include/net/ip6_route.h
@@ -101,12 +101,12 @@ static inline struct dst_entry *ip6_route_output(struct net *net,
 }
 
 /* Only conditionally release dst if flags indicates
- * !RT6_LOOKUP_F_DST_NOREF or dst is in uncached_list.
+ * !RT6_LOOKUP_F_DST_NOREF or dst is uncached.
  */
 static inline void ip6_rt_put_flags(struct rt6_info *rt, int flags)
 {
 	if (!(flags & RT6_LOOKUP_F_DST_NOREF) ||
-	    !list_empty(&rt->dst.rt_uncached))
+	    rt->dst.rt_uncached_list)
 		ip6_rt_put(rt);
 }
 
diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
index b99805ee2fd14b..d1f0a427e9c82a 100644
--- a/include/net/ip6_tunnel.h
+++ b/include/net/ip6_tunnel.h
@@ -45,6 +45,7 @@ struct __ip6_tnl_parm {
 /* IPv6 tunnel */
 struct ip6_tnl {
 	struct ip6_tnl __rcu *next;	/* next tunnel in list */
+	struct hlist_node hash_node;
 	struct net_device *dev;	/* virtual device associated with tunnel */
 	netdevice_tracker dev_tracker;
 	struct net *net;	/* netns for packet i/o */
diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h
index 7c9aadfe8fe396..7102aa11fae299 100644
--- a/include/net/ip_tunnels.h
+++ b/include/net/ip_tunnels.h
@@ -127,6 +127,7 @@ struct ip_tunnel_6rd_parm {
 	__be32			relay_prefix;
 	u16			prefixlen;
 	u16			relay_prefixlen;
+	struct rcu_head		rcu;
 };
 #endif
 
@@ -148,6 +149,7 @@ struct ip_tunnel_parm_kern {
 	__be32			o_key;
 	int			link;
 	struct iphdr		iph;
+	struct rcu_head		rcu;
 };
 
 struct ip_tunnel {
@@ -185,10 +187,11 @@ struct ip_tunnel {
 
 	/* for SIT */
 #ifdef CONFIG_IPV6_SIT_6RD
-	struct ip_tunnel_6rd_parm ip6rd;
+	struct ip_tunnel_6rd_parm __rcu *ip6rd;
 #endif
 	struct ip_tunnel_prl_entry __rcu *prl;	/* potential router list */
 	unsigned int		prl_count;	/* # of entries in PRL */
+	struct ip_tunnel_parm_kern __rcu *sit_parms;
 	unsigned int		ip_tnl_net_id;
 	struct gro_cells	gro_cells;
 	__u32			fwmark;
@@ -215,6 +218,7 @@ struct ip_tunnel_net {
 	struct net_device *fb_tunnel_dev;
 	struct rtnl_link_ops *rtnl_link_ops;
 	struct hlist_head tunnels[IP_TNL_HASH_SIZE];
+	struct mutex tunnels_lock;
 	struct ip_tunnel __rcu *collect_md_tun;
 	int type;
 };
@@ -398,7 +402,6 @@ int ip_tunnel_get_iflink(const struct net_device *dev);
 int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id,
 		       struct rtnl_link_ops *ops, char *devname);
 void ip_tunnel_delete_net(struct net *net, unsigned int id,
-			  struct rtnl_link_ops *ops,
 			  struct list_head *dev_to_kill);
 
 void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
diff --git a/include/net/mac80211.h b/include/net/mac80211.h
index ed6a5874ff9650..cb9f8e14b3b625 100644
--- a/include/net/mac80211.h
+++ b/include/net/mac80211.h
@@ -923,12 +923,14 @@ struct ieee80211_bss_conf {
  * @channel_entry: the Channel Entry blob as defined in Wi-Fi Aware
  *	(TM) 4.0 specification Table 100 (Channel Entry format for the NAN
  *	Availability attribute).
+ * @no_evacuate: if set, this channel must not be evacuated
  */
 struct ieee80211_nan_channel {
 	struct ieee80211_chan_req chanreq;
 	u8 needed_rx_chains;
 	struct ieee80211_chanctx_conf *chanctx_conf;
 	u8 channel_entry[6];
+	bool no_evacuate;
 };
 
 /**
@@ -1770,10 +1772,9 @@ enum mac80211_rx_encoding {
  * @ampdu_reference: A-MPDU reference number, must be a different value for
  *	each A-MPDU but the same for each subframe within one A-MPDU
  * @zero_length_psdu_type: radiotap type of the 0-length PSDU
- * @link_valid: if the link which is identified by @link_id is valid. This flag
- *	is set only when connection is MLO.
- * @link_id: id of the link used to receive the packet. This is used along with
- *	@link_valid.
+ * @link_id: id of the link used to receive the packet. Set and used by
+ *	mac80211 internally, it uses @freq set by the driver to identify the
+ *	correct link per vif.
  */
 struct ieee80211_rx_status {
 	u64 mactime;
@@ -1813,7 +1814,7 @@ struct ieee80211_rx_status {
 	u8 chains;
 	s8 chain_signal[IEEE80211_MAX_CHAINS];
 	u8 zero_length_psdu_type;
-	u8 link_valid:1, link_id:4;
+	u8 link_id:4;
 };
 
 static_assert(sizeof(struct ieee80211_rx_status) <= sizeof_field(struct sk_buff, cb));
@@ -2396,6 +2397,8 @@ static inline bool lockdep_vif_wiphy_mutex_held(struct ieee80211_vif *vif)
  *	number generation only
  * @IEEE80211_KEY_FLAG_SPP_AMSDU: SPP A-MSDUs can be used with this key
  *	(set by mac80211 from the sta->spp_amsdu flag)
+ * @IEEE80211_KEY_FLAG_CIP: This key is used for the Control Integrity
+ *	Protocol, and is either a pairwise key or a CIGTK.
  */
 enum ieee80211_key_flags {
 	IEEE80211_KEY_FLAG_GENERATE_IV_MGMT	= BIT(0),
@@ -2410,6 +2413,7 @@ enum ieee80211_key_flags {
 	IEEE80211_KEY_FLAG_NO_AUTO_TX		= BIT(9),
 	IEEE80211_KEY_FLAG_GENERATE_MMIE	= BIT(10),
 	IEEE80211_KEY_FLAG_SPP_AMSDU		= BIT(11),
+	IEEE80211_KEY_FLAG_CIP			= BIT(12),
 };
 
 /**
@@ -2630,6 +2634,7 @@ struct ieee80211_sta_aggregates {
  * @eht_cap: EHT capabilities of this STA
  * @uhr_cap: UHR capabilities of this STA
  * @s1g_cap: S1G capabilities of this STA
+ * @cip_cap: the CIP capabilities of this STA (or zero)
  * @agg: per-link data for multi-link aggregation
  * @bandwidth: current bandwidth the station can receive with.
  *	This is the minimum between the peer's capabilities and our own
@@ -2658,6 +2663,7 @@ struct ieee80211_link_sta {
 	struct ieee80211_sta_eht_cap eht_cap;
 	struct ieee80211_sta_uhr_cap uhr_cap;
 	struct ieee80211_sta_s1g_cap s1g_cap;
+	u8 cip_cap;
 
 	struct ieee80211_sta_aggregates agg;
 
@@ -2721,6 +2727,7 @@ struct ieee80211_link_sta {
  * @valid_links: bitmap of valid links, or 0 for non-MLO
  * @spp_amsdu: indicates whether the STA uses SPP A-MSDU or not.
  * @epp_peer: indicates that the peer is an EPP peer.
+ * @cip: indicates whether the STA uses control frame protection or not.
  * @nmi: For NDI stations, pointer to the NMI station of the peer.
  * @nan_sched: NAN peer schedule for this station. Valid only for NMI stations.
  * @ext_mld_capa_ops: the MLD's extended MLD capabilities and operations
@@ -2742,6 +2749,8 @@ struct ieee80211_sta {
 	u8 max_amsdu_subframes;
 	u16 eml_cap;
 
+	bool cip;
+
 	struct ieee80211_sta_aggregates *cur;
 
 	bool support_p2p_ps;
@@ -2919,8 +2928,8 @@ struct ieee80211_txq {
  *	autonomously manages the PS status of connected stations. When
  *	this flag is set mac80211 will not trigger PS mode for connected
  *	stations based on the PM bit of incoming frames.
- *	Use ieee80211_start_ps()/ieee8021_end_ps() to manually configure
- *	the PS mode of connected stations.
+ *	Use ieee80211_sta_ps_transition() to manually toggle the PS mode
+ *	of connected stations.
  *
  * @IEEE80211_HW_TX_AMPDU_SETUP_IN_HW: The device handles TX A-MPDU session
  *	setup strictly in HW. mac80211 should not attempt to do this in
@@ -5389,14 +5398,18 @@ void ieee80211_restart_hw(struct ieee80211_hw *hw);
  * mixed for a single hardware. Must not run concurrently with
  * ieee80211_tx_status_skb() or ieee80211_tx_status_ni().
  *
+ * For data frames, when hardware has done address translation, a link station
+ * has to be provided and the frequency information may be skipped.
+ *
  * This function must be called with BHs disabled and RCU read lock
  *
  * @hw: the hardware this frame came in on
- * @sta: the station the frame was received from, or %NULL
+ * @link_sta: the link station the data frame was received from, or %NULL
  * @skb: the buffer to receive, owned by mac80211 after this call
  * @list: the destination list
  */
-void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
+void ieee80211_rx_list(struct ieee80211_hw *hw,
+		       struct ieee80211_link_sta *link_sta,
 		       struct sk_buff *skb, struct list_head *list);
 
 /**
@@ -5414,14 +5427,18 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
  * mixed for a single hardware. Must not run concurrently with
  * ieee80211_tx_status_skb() or ieee80211_tx_status_ni().
  *
+ * For data frames, when hardware has done address translation, a link station
+ * has to be provided and the frequency information may be skipped.
+ *
  * This function must be called with BHs disabled.
  *
  * @hw: the hardware this frame came in on
- * @sta: the station the frame was received from, or %NULL
+ * @link_sta: the link station the data frame was received from, or %NULL
  * @skb: the buffer to receive, owned by mac80211 after this call
  * @napi: the NAPI context
  */
-void ieee80211_rx_napi(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
+void ieee80211_rx_napi(struct ieee80211_hw *hw,
+		       struct ieee80211_link_sta *link_sta,
 		       struct sk_buff *skb, struct napi_struct *napi);
 
 /**
@@ -5499,10 +5516,8 @@ static inline void ieee80211_rx_ni(struct ieee80211_hw *hw,
  *
  * @sta: currently connected sta
  * @start: start or stop PS
- *
- * Return: 0 on success. -EINVAL when the requested PS mode is already set.
  */
-int ieee80211_sta_ps_transition(struct ieee80211_sta *sta, bool start);
+void ieee80211_sta_ps_transition(struct ieee80211_sta *sta, bool start);
 
 /**
  * ieee80211_sta_ps_transition_ni - PS transition for connected sta
@@ -5514,19 +5529,13 @@ int ieee80211_sta_ps_transition(struct ieee80211_sta *sta, bool start);
  *
  * @sta: currently connected sta
  * @start: start or stop PS
- *
- * Return: Like ieee80211_sta_ps_transition().
  */
-static inline int ieee80211_sta_ps_transition_ni(struct ieee80211_sta *sta,
+static inline void ieee80211_sta_ps_transition_ni(struct ieee80211_sta *sta,
 						  bool start)
 {
-	int ret;
-
 	local_bh_disable();
-	ret = ieee80211_sta_ps_transition(sta, start);
+	ieee80211_sta_ps_transition(sta, start);
 	local_bh_enable();
-
-	return ret;
 }
 
 /**
@@ -6279,6 +6288,7 @@ void ieee80211_set_key_rx_seq(struct ieee80211_key_conf *keyconf,
  * @key_len: the key data. Might be bigger than the actual key length,
  *	but not smaller (for the driver convinence)
  * @link_id: the link id of the key or -1 for non-MLO
+ * @cigtk: whether this is a CIGTK
  *
  * When GTK rekeying was done while the system was suspended, (a) new
  * key(s) will be available. These will be needed by mac80211 for proper
@@ -6305,7 +6315,7 @@ void ieee80211_set_key_rx_seq(struct ieee80211_key_conf *keyconf,
 struct ieee80211_key_conf *
 ieee80211_gtk_rekey_add(struct ieee80211_vif *vif,
 			u8 idx, u8 *key_data, u8 key_len,
-			int link_id);
+			int link_id, bool cigtk);
 
 /**
  * ieee80211_gtk_rekey_notify - notify userspace supplicant of rekeying
@@ -7640,12 +7650,14 @@ bool ieee80211_tx_prepare_skb(struct ieee80211_hw *hw,
  * @dev: the &struct device of this 802.11 device
  * @chandef: the channel definition the frame will be transmitted on, or
  *	%NULL to skip the bandwidth checks
+ * @trim_fcs: trim the FCS trailer when present; validate it otherwise
  *
  * Return: %true if the radiotap header was parsed, %false otherwise
  */
 bool ieee80211_parse_tx_radiotap(struct sk_buff *skb,
 				 struct net_device *dev,
-				 const struct cfg80211_chan_def *chandef);
+				 const struct cfg80211_chan_def *chandef,
+				 bool trim_fcs);
 
 /**
  * struct ieee80211_noa_data - holds temporary data for tracking P2P NoA state
@@ -8061,25 +8073,6 @@ void
 ieee80211_obss_color_collision_notify(struct ieee80211_vif *vif,
 				      u64 color_bitmap, u8 link_id);
 
-/**
- * ieee80211_is_tx_data - check if frame is a data frame
- *
- * The function is used to check if a frame is a data frame. Frames with
- * hardware encapsulation enabled are data frames.
- *
- * @skb: the frame to be transmitted.
- *
- * Return: %true if @skb is a data frame, %false otherwise
- */
-static inline bool ieee80211_is_tx_data(struct sk_buff *skb)
-{
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-	struct ieee80211_hdr *hdr = (void *) skb->data;
-
-	return info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP ||
-	       ieee80211_is_data(hdr->frame_control);
-}
-
 /**
  * ieee80211_set_active_links - set active links in client mode
  * @vif: interface to set active links on
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 308950f9b54b04..2a28fcaa637d91 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -6,6 +6,7 @@
 
 #include <linux/dma-mapping.h>
 #include <linux/netdevice.h>
+#include <rdma/ib_verbs.h>
 
 #include "shm_channel.h"
 
@@ -54,6 +55,7 @@ enum gdma_work_request_flags {
 	GDMA_WR_NONE			= 0,
 	GDMA_WR_OOB_IN_SGL		= BIT(0),
 	GDMA_WR_PAD_BY_SGE0		= BIT(1),
+	GDMA_WR_IB_SGL			= BIT(2),
 };
 
 enum gdma_eqe_type {
@@ -212,7 +214,10 @@ struct gdma_sge {
 }; /* HW DATA */
 
 struct gdma_wqe_request {
-	struct gdma_sge *sgl;
+	union {
+		struct gdma_sge *sgl;
+		struct ib_sge *ib_sgl;
+	};
 	u32 num_sge;
 
 	u32 inline_oob_size;
@@ -552,6 +557,7 @@ struct gdma_wqe {
 
 #define INLINE_OOB_SMALL_SIZE 8
 #define INLINE_OOB_LARGE_SIZE 24
+#define INLINE_OOB_EXTRA_LARGE_SIZE 32
 
 #define MANA_MAX_TX_WQE_SGL_ENTRIES 30
 
@@ -624,6 +630,8 @@ struct gdma_eqe {
 
 struct gdma_posted_wqe_info {
 	u32 wqe_size_in_bu;
+	/* Unmasked WQE start offset in GDMA basic units. */
+	u32 wqe_offset;
 };
 
 /* GDMA_GENERATE_TEST_EQE */
@@ -1062,6 +1070,8 @@ void mana_gd_free_res_map(struct gdma_resource *r);
 
 void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
 			      struct gdma_queue *queue);
+void mana_gd_wq_ring_doorbell_ext(struct gdma_context *gc, struct gdma_queue *queue,
+				  u32 tail_ptr, u8 wqe_cnt, u8 client_offset);
 
 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
 			 struct gdma_mem_info *gmi, bool allow_scatter);
diff --git a/include/net/mana/hw_channel.h b/include/net/mana/hw_channel.h
index 16feb39616c1be..1ab2d66c891e42 100644
--- a/include/net/mana/hw_channel.h
+++ b/include/net/mana/hw_channel.h
@@ -20,8 +20,8 @@
 #define HWC_INIT_DATA_MAX_NUM_CQS	7
 #define HWC_INIT_DATA_PDID		8
 #define HWC_INIT_DATA_GPA_MKEY		9
-#define HWC_INIT_DATA_PF_DEST_RQ_ID	10
-#define HWC_INIT_DATA_PF_DEST_CQ_ID	11
+#define HWC_INIT_DATA_DEST_RQ_ID	10
+#define HWC_INIT_DATA_DEST_CQ_ID	11
 
 #define HWC_DATA_CFG_HWC_TIMEOUT 1
 #define HWC_DATA_HW_LINK_CONNECT 2
@@ -195,8 +195,8 @@ struct hw_channel_context {
 	struct semaphore sema;
 	struct gdma_resource inflight_msg_res;
 
-	u32 pf_dest_vrq_id;
-	u32 pf_dest_vrcq_id;
+	u32 dest_vrq_id;
+	u32 dest_vrcq_id;
 	u32 hwc_timeout;
 
 	struct hwc_caller_ctx *caller_ctx;
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 83b7eff4646ead..e8fda092be37fb 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -47,8 +47,8 @@ enum mana_priv_flag_bits {
 #define COMP_ENTRY_SIZE 64
 
 /* This Max value for RX buffers is derived from __alloc_page()'s max page
- * allocation calculation. It allows maximum 2^(MAX_ORDER -1) pages. RX buffer
- * size beyond this value gets rejected by __alloc_page() call.
+ * allocation calculation. It allows maximum 2^MAX_PAGE_ORDER pages. RX
+ * buffer size beyond this value gets rejected by __alloc_page() call.
  */
 #define MAX_RX_BUFFERS_PER_QUEUE 8192
 #define DEF_RX_BUFFERS_PER_QUEUE 1024
diff --git a/include/net/ndisc.h b/include/net/ndisc.h
index 9e5379ad2d8e00..96e3bb6e83afd8 100644
--- a/include/net/ndisc.h
+++ b/include/net/ndisc.h
@@ -67,6 +67,15 @@ struct prefix_info;
 
 extern struct neigh_table nd_tbl;
 
+static inline struct neigh_table *nd_table(struct net *net)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+	if (disable_ipv6_mod)
+		return &nd_tbl;
+#endif
+	return net->neigh_tables[NEIGH_ND_TABLE];
+}
+
 struct nd_msg {
         struct icmp6hdr	icmph;
         struct in6_addr	target;
@@ -354,7 +363,9 @@ static inline u32 ndisc_hashfn(const void *pkey, const struct net_device *dev, _
 
 static inline struct neighbour *__ipv6_neigh_lookup_noref(struct net_device *dev, const void *pkey)
 {
-	return ___neigh_lookup_noref(&nd_tbl, neigh_key_eq128, ndisc_hashfn, pkey, dev);
+	struct neigh_table *tbl = nd_table(dev_net(dev));
+
+	return ___neigh_lookup_noref(tbl, neigh_key_eq128, ndisc_hashfn, pkey, dev);
 }
 
 static inline struct neighbour *__ipv6_neigh_lookup(struct net_device *dev, const void *pkey)
@@ -388,8 +399,11 @@ static inline struct neighbour *ip_neigh_gw6(struct net_device *dev,
 	struct neighbour *neigh;
 
 	neigh = __ipv6_neigh_lookup_noref(dev, addr);
-	if (unlikely(!neigh))
-		neigh = __neigh_create(&nd_tbl, addr, dev, false);
+	if (unlikely(!neigh)) {
+		struct neigh_table *tbl = nd_table(dev_net(dev));
+
+		neigh = __neigh_create(tbl, addr, dev, false);
+	}
 
 	return neigh;
 #else
diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 8860cc2175fc1c..ed9d9ae6d3a653 100644
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -31,6 +31,8 @@
 #include <net/rtnetlink.h>
 #include <net/neighbour_tables.h>
 
+extern int sysctl_neigh_inherit_init_net;
+
 /*
  * NUD stands for "neighbor unreachability detection"
  */
@@ -179,7 +181,6 @@ struct neigh_ops {
 
 struct pneigh_entry {
 	struct pneigh_entry	__rcu *next;
-	possible_net_t		net;
 	struct net_device	*dev;
 	netdevice_tracker	dev_tracker;
 	union {
@@ -234,7 +235,7 @@ struct neigh_table {
 	struct delayed_work	managed_work;
 	struct timer_list 	proxy_timer;
 	struct sk_buff_head	proxy_queue;
-	atomic_t		entries;
+	refcount_t		entries;
 	atomic_t		gc_entries;
 	struct list_head	gc_list;
 	struct list_head	managed_list;
@@ -339,8 +340,8 @@ static inline void neigh_confirm(struct neighbour *n)
 	}
 }
 
-void neigh_table_init(int index, struct neigh_table *tbl);
-int neigh_table_clear(int index, struct neigh_table *tbl);
+int neigh_table_register(struct net *net, struct neigh_table *tbl, int index);
+void neigh_table_unregister(struct net *net, int index);
 struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,
 			       struct net_device *dev);
 struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,
@@ -389,24 +390,17 @@ static inline void neigh_set_reach_time(struct neigh_parms *p)
 
 void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p,
 		    struct sk_buff *skb);
-struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl, struct net *net,
+struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl,
 				   const void *key, struct net_device *dev);
-int pneigh_create(struct neigh_table *tbl, struct net *net, const void *key,
+int pneigh_create(struct neigh_table *tbl, const void *key,
 		  struct net_device *dev, u32 flags, u8 protocol,
 		  bool permanent);
-int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *key,
+int pneigh_delete(struct neigh_table *tbl, const void *key,
 		  struct net_device *dev);
 
-static inline struct net *pneigh_net(const struct pneigh_entry *pneigh)
-{
-	return read_pnet(&pneigh->net);
-}
-
 void neigh_app_ns(struct neighbour *n);
 void neigh_for_each(struct neigh_table *tbl,
 		    void (*cb)(struct neighbour *, void *), void *cookie);
-void __neigh_for_each_release(struct neigh_table *tbl,
-			      int (*cb)(struct neighbour *));
 int neigh_xmit(int fam, struct net_device *, const void *, struct sk_buff *);
 
 struct neigh_seq_state {
diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h
index e5ee673b9fcf84..46b4c67e2966fe 100644
--- a/include/net/net_namespace.h
+++ b/include/net/net_namespace.h
@@ -39,6 +39,7 @@
 #include <net/netns/mctp.h>
 #include <net/netns/vsock.h>
 #include <net/net_trackers.h>
+#include <net/neighbour_tables.h>
 #include <linux/ns_common.h>
 #include <linux/idr.h>
 #include <linux/skbuff.h>
@@ -47,6 +48,7 @@
 
 struct user_namespace;
 struct proc_dir_entry;
+struct neigh_table;
 struct net_device;
 struct sock;
 struct ctl_table_header;
@@ -103,6 +105,8 @@ struct net {
 	struct proc_dir_entry 	*proc_net;
 	struct proc_dir_entry 	*proc_net_stat;
 
+	struct neigh_table	*neigh_tables[NEIGH_NR_TABLES];
+
 #ifdef CONFIG_SYSCTL
 	struct ctl_table_set	sysctls;
 #endif
diff --git a/include/net/netdev_lock.h b/include/net/netdev_lock.h
index 9fb3e93857c3d3..edd8cc2e2b36e2 100644
--- a/include/net/netdev_lock.h
+++ b/include/net/netdev_lock.h
@@ -110,34 +110,20 @@ static inline int netdev_is_locked_ops_compat(const struct net_device *dev)
 	return lockdep_rtnl_is_held();
 }
 
-static inline int netdev_lock_cmp_fn(const struct lockdep_map *a,
-				     const struct lockdep_map *b)
-{
-	if (a == b)
-		return 0;
-
-	/* Allow locking multiple devices only under rtnl_lock,
-	 * the exact order doesn't matter.
-	 * Note that upper devices don't lock their ops, so nesting
-	 * mostly happens in batched device removal for now.
-	 */
-	return lockdep_rtnl_is_held() ? -1 : 1;
-}
+void netdev_set_instance_lock_class(struct net_device *dev);
 
 #define netdev_lockdep_set_classes(dev)				\
 {								\
 	static struct lock_class_key qdisc_tx_busylock_key;	\
 	static struct lock_class_key qdisc_xmit_lock_key;	\
 	static struct lock_class_key dev_addr_list_lock_key;	\
-	static struct lock_class_key dev_instance_lock_key;	\
 	unsigned int i;						\
 								\
 	(dev)->qdisc_tx_busylock = &qdisc_tx_busylock_key;	\
 	lockdep_set_class(&(dev)->addr_list_lock,		\
 			  &dev_addr_list_lock_key);		\
-	lockdep_set_class(&(dev)->lock,				\
-			  &dev_instance_lock_key);		\
-	lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL);	\
+	if (IS_ENABLED(CONFIG_PROVE_LOCKING))			\
+		netdev_set_instance_lock_class(dev);		\
 	for (i = 0; i < (dev)->num_tx_queues; i++)		\
 		lockdep_set_class(&(dev)->_tx[i]._xmit_lock,	\
 				  &qdisc_xmit_lock_key);	\
diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h
index bc42dd0e10e659..c39425e54d87d3 100644
--- a/include/net/netfilter/nf_conntrack.h
+++ b/include/net/netfilter/nf_conntrack.h
@@ -185,6 +185,11 @@ static inline void nf_ct_put(struct nf_conn *ct)
 		nf_ct_destroy(&ct->ct_general);
 }
 
+static inline bool nf_ct_shared(const struct nf_conn *ct)
+{
+	return refcount_read(&ct->ct_general.use) > 1;
+}
+
 /* load module; enable/disable conntrack in this namespace */
 int nf_ct_netns_get(struct net *net, u8 nfproto);
 void nf_ct_netns_put(struct net *net, u8 nfproto);
diff --git a/include/net/netfilter/nf_conntrack_seqadj.h b/include/net/netfilter/nf_conntrack_seqadj.h
index 883c414b768eef..0f5bbb14a25aa7 100644
--- a/include/net/netfilter/nf_conntrack_seqadj.h
+++ b/include/net/netfilter/nf_conntrack_seqadj.h
@@ -33,6 +33,7 @@ static inline struct nf_conn_seqadj *nfct_seqadj_ext_add(struct nf_conn *ct)
 
 int nf_ct_seqadj_init(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 		      s32 off);
+void nf_ct_seqadj_reset(struct nf_conn *ct, enum ip_conntrack_info ctinfo);
 int nf_ct_seqadj_set(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 		     __be32 seq, s32 off);
 void nf_ct_tcp_seqadj_set(struct sk_buff *skb, struct nf_conn *ct,
diff --git a/include/net/netmem.h b/include/net/netmem.h
index bccacd21b6c374..cc97611632dc8d 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -358,6 +358,33 @@ static inline bool netmem_is_pfmemalloc(netmem_ref netmem)
 	return page_is_pfmemalloc(netmem_to_page(netmem));
 }
 
+#define NETMEM_32BIT_ARCH_WITH_64BIT_DMA	\
+	(sizeof(dma_addr_t) > sizeof(unsigned long))
+
+static inline unsigned long netmem_dma_addr_encode(dma_addr_t addr)
+{
+	if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA)
+		addr >>= PAGE_SHIFT;
+
+	return addr;
+}
+
+static inline dma_addr_t netmem_dma_addr_decode(unsigned long addr)
+{
+	if (NETMEM_32BIT_ARCH_WITH_64BIT_DMA)
+		return (dma_addr_t)addr << PAGE_SHIFT;
+
+	return addr;
+}
+
+static inline bool netmem_dma_addr_fits(dma_addr_t addr)
+{
+	/* We assume page alignment to shave off bottom bits,
+	 * if this "compression" doesn't work we need to drop.
+	 */
+	return addr == netmem_dma_addr_decode(netmem_dma_addr_encode(addr));
+}
+
 static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
 {
 	return netmem_to_nmdesc(netmem)->dma_addr;
diff --git a/include/net/netns/netfilter.h b/include/net/netns/netfilter.h
index a6a0bf4a247e51..7fd78394d1e7bd 100644
--- a/include/net/netns/netfilter.h
+++ b/include/net/netns/netfilter.h
@@ -33,5 +33,7 @@ struct netns_nf {
 #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6)
 	unsigned int defrag_ipv6_users;
 #endif
+	unsigned int hook_base_seq;
+	unsigned int nat_hook_base_seq;
 };
 #endif
diff --git a/include/net/nfc/hci.h b/include/net/nfc/hci.h
index 756c11084f65fe..86ed63e5d5334a 100644
--- a/include/net/nfc/hci.h
+++ b/include/net/nfc/hci.h
@@ -144,7 +144,7 @@ struct nfc_hci_dev {
 	data_exchange_cb_t async_cb;
 	void *async_cb_context;
 
-	u8 *gb;
+	u8 gb[NFC_MAX_GT_LEN];
 	size_t gb_len;
 
 	unsigned long quirks;
diff --git a/include/net/nfc/nfc.h b/include/net/nfc/nfc.h
index c54df042db6be2..bcafab5c53e51a 100644
--- a/include/net/nfc/nfc.h
+++ b/include/net/nfc/nfc.h
@@ -273,7 +273,8 @@ struct sk_buff *nfc_alloc_recv_skb(unsigned int size, gfp_t gfp);
 
 int nfc_set_remote_general_bytes(struct nfc_dev *dev,
 				 const u8 *gt, u8 gt_len);
-u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, size_t *gb_len);
+u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, u8 *out_gb,
+				size_t gb_max_len, size_t *gb_len);
 
 int nfc_fw_download_done(struct nfc_dev *dev, const char *firmware_name,
 			 u32 result);
diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h
index e2730dd273b284..cd021832c3fa3f 100644
--- a/include/net/page_pool/helpers.h
+++ b/include/net/page_pool/helpers.h
@@ -408,16 +408,13 @@ static inline void page_pool_recycle_direct_netmem(struct page_pool *pool,
 	page_pool_put_full_netmem(pool, netmem, true);
 }
 
-#define PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA	\
-		(sizeof(dma_addr_t) > sizeof(unsigned long))
-
 /**
  * page_pool_free_va() - free a va into the page_pool
  * @pool: pool from which va was allocated
  * @va: va to be freed
  * @allow_direct: freed by the consumer, allow lockless caching
  *
- * Free a va allocated from page_pool_allo_va().
+ * Free a va allocated from page_pool_alloc_va().
  */
 static inline void page_pool_free_va(struct page_pool *pool, void *va,
 				     bool allow_direct)
@@ -427,12 +424,7 @@ static inline void page_pool_free_va(struct page_pool *pool, void *va,
 
 static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem)
 {
-	dma_addr_t ret = netmem_get_dma_addr(netmem);
-
-	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
-		ret <<= PAGE_SHIFT;
-
-	return ret;
+	return netmem_dma_addr_decode(netmem_get_dma_addr(netmem));
 }
 
 /**
diff --git a/include/net/phy/realtek_phy.h b/include/net/phy/realtek_phy.h
deleted file mode 100644
index d683bc1b0659cd..00000000000000
--- a/include/net/phy/realtek_phy.h
+++ /dev/null
@@ -1,7 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _REALTEK_PHY_H
-#define _REALTEK_PHY_H
-
-#define	PHY_ID_RTL_DUMMY_SFP	0x001ccbff
-
-#endif /* _REALTEK_PHY_H */
diff --git a/include/net/psp/functions.h b/include/net/psp/functions.h
index c5c23a54774e6d..b23c30898389c7 100644
--- a/include/net/psp/functions.h
+++ b/include/net/psp/functions.h
@@ -24,11 +24,6 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv);
 /* Kernel-facing API */
 void psp_assoc_put(struct psp_assoc *pas);
 
-static inline void *psp_assoc_drv_data(struct psp_assoc *pas)
-{
-	return pas->drv_data;
-}
-
 #if IS_ENABLED(CONFIG_INET_PSP)
 unsigned int psp_key_size(u32 version);
 void psp_sk_assoc_free(struct sock *sk);
diff --git a/include/net/psp/types.h b/include/net/psp/types.h
index 87991a1ea02d0a..b8905efbd604e5 100644
--- a/include/net/psp/types.h
+++ b/include/net/psp/types.h
@@ -219,12 +219,16 @@ struct psp_dev_ops {
 	 * @tx_key_add: add a Tx key to the device
 	 * Install an association in the device. Core will allocate space
 	 * for the driver to use at drv_data.
+	 * Can be left NULL if device does not store Tx keys and @tx_key_del
+	 * is also NULL.
 	 */
 	int (*tx_key_add)(struct psp_dev *psd, struct psp_assoc *pas,
 			  struct netlink_ext_ack *extack);
 	/**
 	 * @tx_key_del: remove a Tx key from the device
 	 * Remove an association from the device.
+	 * Can be left NULL if device does not store Tx keys and @tx_key_add
+	 * is also NULL.
 	 */
 	void (*tx_key_del)(struct psp_dev *psd, struct psp_assoc *pas);
 
diff --git a/include/net/route.h b/include/net/route.h
index 45290177a33c4a..6b55de2e4df88e 100644
--- a/include/net/route.h
+++ b/include/net/route.h
@@ -405,8 +405,11 @@ static inline struct neighbour *ip_neigh_gw4(struct net_device *dev,
 	struct neighbour *neigh;
 
 	neigh = __ipv4_neigh_lookup_noref(dev, (__force u32)daddr);
-	if (unlikely(!neigh))
-		neigh = __neigh_create(&arp_tbl, &daddr, dev, false);
+	if (unlikely(!neigh)) {
+		struct neigh_table *tbl = arp_table(dev_net(dev));
+
+		neigh = __neigh_create(tbl, &daddr, dev, false);
+	}
 
 	return neigh;
 }
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index f35bd06a6bad68..1acaadb3e2cf08 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -810,6 +810,15 @@ static inline bool skb_at_tc_ingress(const struct sk_buff *skb)
 #endif
 }
 
+static inline bool skb_at_tc_egress(const struct sk_buff *skb)
+{
+#ifdef CONFIG_NET_EGRESS
+	return skb->nf_skip_egress && !skb_at_tc_ingress(skb);
+#else
+	return false;
+#endif
+}
+
 static inline bool skb_skip_tc_classify(struct sk_buff *skb)
 {
 #ifdef CONFIG_NET_CLS_ACT
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 436495ff2271de..effc5f41bb41b4 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -823,6 +823,9 @@ unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
 unsigned int tcp_current_mss(struct sock *sk);
 u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
 
+u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
+		     int min_tso_segs);
+
 /* Bound MSS / TSO packet size with the half of the window */
 static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize)
 {
@@ -1232,9 +1235,9 @@ static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb)
 static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
 					const struct sk_buff *from)
 {
-	/* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */
 	return likely(tcp_skb_can_collapse_to(to) &&
 		      mptcp_skb_can_collapse(to, from) &&
+		      !skb_cmp_decrypted(to, from) &&
 		      skb_pure_zcopy_same(to, from) &&
 		      skb_frags_readable(to) == skb_frags_readable(from));
 }
@@ -1360,8 +1363,16 @@ struct tcp_congestion_ops {
 	/* hook for packet ack accounting (optional) */
 	void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
 
-	/* override sysctl_tcp_min_tso_segs (optional) */
-	u32 (*min_tso_segs)(struct sock *sk);
+	/* Override tcp_tso_autosize() (optional)
+	 *
+	 * If provided, this callback supplies the TSO segment target count
+	 * instead of using tcp_tso_autosize(). The returned value is
+	 * subsequently clamped to [1, sk->sk_gso_max_segs] by the caller.
+	 *
+	 * For the kernel callback path, mss_now originates from
+	 * tcp_current_mss() and should never be zero.
+	 */
+	u32 (*tso_segs)(struct sock *sk, u32 mss_now);
 
 	/* new value of cwnd after loss (required) */
 	u32  (*undo_cwnd)(struct sock *sk);
@@ -2327,7 +2338,7 @@ static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct soc
 
 static inline void tcp_write_collapse_fence(struct sock *sk)
 {
-	struct sk_buff *skb = tcp_write_queue_tail(sk);
+	struct sk_buff *skb = tcp_write_queue_tail(sk) ?: tcp_rtx_queue_tail(sk);
 
 	if (skb)
 		TCP_SKB_CB(skb)->eor = 1;
@@ -2959,12 +2970,163 @@ static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk)
 
 #endif
 
+#if defined(CONFIG_BPF_JIT) && defined(CONFIG_CGROUP_BPF)
+
+struct bpf_tcp_ops {
+	/* Should return the initial SYN (active open) or SYN-ACK (passive open)
+	 * retransmission timeout. Return the timeout in jiffies, or <= 0 for
+	 * the kernel default.
+	 *
+	 * @req: request_sock on the passive (synack) path; NULL otherwise.
+	 */
+	int (*timeout_init)(struct sock *sk, struct request_sock *req);
+
+	/* Should return the initial advertised receive window, in packets,
+	 * or < 0 for the kernel default. @req as in timeout_init().
+	 */
+	int (*rwnd_init)(struct sock *sk, struct request_sock *req);
+
+	/* Called when an active connection becomes established.
+	 * @skb is the SYNACK that completed the 3WHS, or NULL for a
+	 * TCP_REPAIR socket (tcp_finish_connect() with no skb).
+	 */
+	void (*active_established)(struct sock *sk, struct sk_buff *skb__nullable);
+
+	/* Called when a passive connection becomes established.
+	 * @skb is the ACK that completed the 3WHS.
+	 */
+	void (*passive_established)(struct sock *sk, struct sk_buff *skb);
+
+	/* Called when the retransmission timer fires. */
+	void (*rto)(struct sock *sk);
+
+	/* Called on every RTT sample.
+	 * @mrtt: the measured RTT, in microseconds.
+	 * @srtt: the updated smoothed RTT.
+	 */
+	void (*rtt)(struct sock *sk, long mrtt, u32 srtt);
+
+	/* Called when the connection changes TCP state.
+	 * @state: the new state (one of the TCP_* states).
+	 */
+	void (*set_state)(struct sock *sk, int state);
+
+	/* Called when an skb is retransmitted.
+	 * @skb: the retransmitted skb.
+	 * @err: tcp_transmit_skb() return value (0 on success).
+	 */
+	void (*retrans)(struct sock *sk, struct sk_buff *skb, int err);
+
+	/* Called right before an active connection is initialized. */
+	void (*connect)(struct sock *sk);
+
+	/* Called on listen(2), right after the socket enters TCP_LISTEN. */
+	void (*listen)(struct sock *sk);
+
+	/*
+	 * Parse the TCP header options of an incoming skb received on an
+	 * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes()
+	 * to access the options.
+	 */
+	void (*parse_hdr)(struct sock *sk, struct sk_buff *skb);
+
+	/*
+	 * Reserve space in the outgoing TCP header for options to be written
+	 * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes.
+	 *
+	 * @skb: outgoing packet. NULL when called from tcp_current_mss()
+	 *       (MSS sizing).
+	 * @req: request_sock on the synack path; NULL otherwise.
+	 * @syn_skb: incoming SYN on the synack path; NULL otherwise.
+	 * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie.
+	 * @remaining: pointer to the size of space still available; cast it
+	 *             using bpf_rdonly_cast() before dereferencing.
+	 */
+	void (*hdr_opt_len)(struct sock *sk, struct sk_buff *skb,
+			    struct request_sock *req, struct sk_buff *syn_skb,
+			    enum tcp_synack_type synack_type,
+			    unsigned int *remaining);
+
+	/*
+	 * Write header options into the space reserved earlier by hdr_opt_len().
+	 * Use bpf_store_hdr_opt() to write; it appends within the reserved window
+	 * shared with legacy SOCKOPS.
+	 *
+	 * @skb: outgoing packet.
+	 * @req: request_sock on the synack path; NULL otherwise.
+	 * @syn_skb: incoming SYN on the synack path; NULL otherwise.
+	 * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie.
+	 * @opt_off: offset in the outgoing @skb's TCP header where the
+	 *	     bpf_tcp_ops portion of the reserved window begins, i.e. after
+	 *	     the kernel and legacy options.
+	 */
+	void (*write_hdr_opt)(struct sock *sk, struct sk_buff *skb,
+			      struct request_sock *req, struct sk_buff *syn_skb,
+			      enum tcp_synack_type synack_type,
+			      u32 opt_off);
+};
+
+#define bpf_tcp_ops_call(op, sk, ...)					\
+do {									\
+	if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) {			\
+		const struct bpf_prog_array_item *item;			\
+		const struct bpf_tcp_ops *tcp_ops;			\
+		struct cgroup *cgrp;					\
+									\
+		cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);		\
+		rcu_read_lock_dont_migrate();				\
+		bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp,	\
+					      CGROUP_TCP_SOCK_OPS) {	\
+			if (tcp_ops->op)				\
+				tcp_ops->op(sk, ##__VA_ARGS__);		\
+		}							\
+		rcu_read_unlock_migrate();				\
+	}								\
+} while (0)
+
+#define bpf_tcp_ops_call_int(op, init_retval, sk, ...)			\
+({									\
+	int __retval = (init_retval);					\
+	if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) {			\
+		const struct bpf_prog_array_item *item;			\
+		const struct bpf_tcp_ops *tcp_ops;			\
+		struct bpf_tramp_run_ctx run_ctx;			\
+		struct bpf_run_ctx *old_run_ctx;			\
+		struct sock *__sk = sk_to_full_sk(sk);                  \
+		struct request_sock *req = NULL;			\
+		struct cgroup *cgrp;					\
+									\
+		if (__sk) {						\
+			run_ctx.retval = (init_retval);			\
+			cgrp = sock_cgroup_ptr(&__sk->sk_cgrp_data);	\
+			if (!sk_fullsock(sk))				\
+				req = (struct request_sock *)sk;	\
+			rcu_read_lock_dont_migrate();			\
+			old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);\
+			bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \
+						      CGROUP_TCP_SOCK_OPS) { \
+				if (tcp_ops->op)			\
+					run_ctx.retval = tcp_ops->op(__sk, req, ##__VA_ARGS__); \
+			}						\
+			bpf_reset_run_ctx(old_run_ctx);			\
+			rcu_read_unlock_migrate();			\
+			__retval = run_ctx.retval;			\
+		}							\
+	}								\
+	__retval;							\
+})
+
+#else
+#define bpf_tcp_ops_call(op, sk, ...)		do { } while (0)
+#define bpf_tcp_ops_call_int(op, init_retval, sk, ...)	(init_retval)
+#endif
+
 static inline u32 tcp_timeout_init(struct sock *sk)
 {
 	int timeout;
 
 	timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
-
+	timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk);
 	if (timeout <= 0)
 		timeout = TCP_TIMEOUT_INIT;
 	return min_t(int, timeout, TCP_RTO_MAX);
@@ -2975,7 +3137,7 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
 	int rwnd;
 
 	rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
-
+	rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk);
 	if (rwnd < 0)
 		rwnd = 0;
 	return rwnd;
@@ -2990,6 +3152,7 @@ static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt)
 {
 	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
 		tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt);
+	bpf_tcp_ops_call(rtt, sk, mrtt, srtt);
 }
 
 #if IS_ENABLED(CONFIG_SMC)
diff --git a/include/net/vxlan.h b/include/net/vxlan.h
index 7b820750552370..c3c9f2ccc3d1bf 100644
--- a/include/net/vxlan.h
+++ b/include/net/vxlan.h
@@ -204,7 +204,6 @@ struct vxlan_rdst {
 	u8			 offloaded:1;
 	__be32			 remote_vni;
 	u32			 remote_ifindex;
-	struct net_device	 *remote_dev;
 	struct list_head	 list;
 	struct rcu_head		 rcu;
 	struct dst_cache	 dst_cache;
@@ -229,6 +228,7 @@ struct vxlan_config {
 	bool				no_share;
 	enum ifla_vxlan_df		df;
 	struct vxlanhdr			reserved_bits;
+	struct rcu_head			rcu;
 };
 
 enum {
@@ -294,14 +294,15 @@ struct vxlan_dev {
 #endif
 	struct net_device *dev;
 	struct net	  *net;		/* netns for packet i/o */
-	struct vxlan_rdst default_dst;	/* default destination */
+	struct net_device *lowerdev;
 
 	struct timer_list age_timer;
 	spinlock_t	  hash_lock;
 	unsigned int	  addrcnt;
 	struct gro_cells  gro_cells;
+	unsigned long	  flags;
 
-	struct vxlan_config	cfg;
+	struct vxlan_config __rcu	*cfg;
 
 	struct vxlan_vni_group  __rcu *vnigrp;
 
@@ -313,6 +314,10 @@ struct vxlan_dev {
 	unsigned int mdb_seq;
 };
 
+enum vxlan_dev_flags {
+	VXLAN_DEV_F_MDB,
+};
+
 #define VXLAN_F_LEARN			0x01
 #define VXLAN_F_PROXY			0x02
 #define VXLAN_F_RSC			0x04
@@ -331,7 +336,6 @@ struct vxlan_dev {
 #define VXLAN_F_IPV6_LINKLOCAL		0x8000
 #define VXLAN_F_TTL_INHERIT		0x10000
 #define VXLAN_F_VNIFILTER               0x20000
-#define VXLAN_F_MDB			0x40000
 #define VXLAN_F_LOCALBYPASS		0x80000
 #define VXLAN_F_MC_ROUTE		0x100000
 
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index 1fe87fd1d769d8..dcb6454196924b 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -7,6 +7,7 @@
 #ifndef IB_UMEM_H
 #define IB_UMEM_H
 
+#include <linux/dma-direction.h>
 #include <linux/scatterlist.h>
 
 struct ib_device;
@@ -19,6 +20,7 @@ struct ib_umem {
 	size_t			length;
 	unsigned long		address;
 	unsigned long		dma_attrs;
+	enum dma_data_direction dma_dir;
 	u32 writable : 1;
 	u32 is_odp : 1;
 	u32 is_dmabuf : 1;
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index cb3b6163961b8a..d6eaa3fee66e39 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -3532,7 +3532,7 @@ static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num)
  * @port_num: Port number to check
  *
  * Similar to above, but specific to iWARP connections which have a different
- * managment protocol than InfiniBand.
+ * management protocol than InfiniBand.
  *
  * Return: true if the port supports an iWARP CM (this does not guarantee that
  * a CM is actually running however).
@@ -3577,7 +3577,7 @@ static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num)
  * to this group.  And it should leave the group only after all queue pairs
  * attached to the group have been detached.
  *
- * Return: true if the port must undertake the additional adminstrative
+ * Return: true if the port must undertake the additional administrative
  * overhead of registering/unregistering with the SM and tracking of the
  * total number of queue pairs attached to the multicast group.
  */
diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h
index 80f3ba6663d06e..1aaaadb8daf6e3 100644
--- a/include/rdma/uverbs_ioctl.h
+++ b/include/rdma/uverbs_ioctl.h
@@ -1142,7 +1142,7 @@ static inline int ib_is_udata_in_empty(struct ib_udata *udata)
  * @_udata: The system calls ib_udata struct
  *
  * Used when there is no driver response data to return. Provides forward
- * compatability by zeroing any buffer the user may have provided.
+ * compatibility by zeroing any buffer the user may have provided.
  */
 static inline int ib_respond_empty_udata(struct ib_udata *udata)
 {
diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h
index 8ecfb94049db51..2f2364c964570a 100644
--- a/include/scsi/scsi_cmnd.h
+++ b/include/scsi/scsi_cmnd.h
@@ -391,8 +391,7 @@ static inline unsigned scsi_transfer_length(struct scsi_cmnd *scmd)
 	return xfer_len;
 }
 
-extern void scsi_build_sense(struct scsi_cmnd *scmd, int desc,
-			     u8 key, u8 asc, u8 ascq);
+void scsi_set_sense(struct scsi_cmnd *scmd, int desc, u8 key, u16 code);
 
 struct request *scsi_alloc_request(struct request_queue *q, blk_opf_t opf,
 				   blk_mq_req_flags_t flags);
diff --git a/include/scsi/scsi_common.h b/include/scsi/scsi_common.h
index 00c8a16d3cd24b..88a076c1a1375e 100644
--- a/include/scsi/scsi_common.h
+++ b/include/scsi/scsi_common.h
@@ -71,8 +71,7 @@ extern u64 scsilun_to_int(struct scsi_lun *);
 struct scsi_sense_hdr {		/* See SPC-3 section 4.5 */
 	u8 response_code;	/* permit: 0x0, 0x70, 0x71, 0x72, 0x73 */
 	u8 sense_key;
-	u8 asc;
-	u8 ascq;
+	u16 sense_code;
 	u8 byte4;
 	u8 byte5;
 	u8 byte6;
@@ -87,10 +86,21 @@ static inline bool scsi_sense_valid(const struct scsi_sense_hdr *sshdr)
 	return (sshdr->response_code & 0x70) == 0x70;
 }
 
+static inline u8 scsi_sense_asc(const struct scsi_sense_hdr *sshdr)
+{
+	return scsi_sense_code_asc(sshdr->sense_code);
+}
+
+static inline u8 scsi_sense_ascq(const struct scsi_sense_hdr *sshdr)
+{
+	return scsi_sense_code_ascq(sshdr->sense_code);
+}
+
 extern bool scsi_normalize_sense(const u8 *sense_buffer, int sb_len,
 				 struct scsi_sense_hdr *sshdr);
 
-extern void scsi_build_sense_buffer(int desc, u8 *buf, u8 key, u8 asc, u8 ascq);
+void scsi_set_sense_buffer(int desc, u8 *buf, u8 key, u16 code);
+
 int scsi_set_sense_information(u8 *buf, int buf_len, u64 info);
 int scsi_set_sense_field_pointer(u8 *buf, int buf_len, u16 fp, u8 bp, bool cd);
 extern const u8 * scsi_sense_desc_find(const u8 * sense_buffer, int sb_len,
diff --git a/include/scsi/scsi_dbg.h b/include/scsi/scsi_dbg.h
index efcdc78530d50a..f21a5e3caca927 100644
--- a/include/scsi/scsi_dbg.h
+++ b/include/scsi/scsi_dbg.h
@@ -20,8 +20,8 @@ extern void scsi_print_result(struct scsi_cmnd *, const char *, int);
 #ifdef CONFIG_SCSI_CONSTANTS
 extern bool scsi_opcode_sa_name(int, int, const char **, const char **);
 extern const char *scsi_sense_key_string(unsigned char);
-extern const char *scsi_extd_sense_format(unsigned char, unsigned char,
-					  const char **);
+extern const char *scsi_extd_sense_format(const struct scsi_sense_hdr *sshdr,
+					  const char **fmt);
 extern const char *scsi_mlreturn_string(int);
 extern const char *scsi_hostbyte_string(int);
 #else
@@ -57,7 +57,7 @@ scsi_sense_key_string(unsigned char key)
 }
 
 static inline const char *
-scsi_extd_sense_format(unsigned char asc, unsigned char ascq, const char **fmt)
+scsi_extd_sense_format(const struct scsi_sense_hdr *sshdr, const char **fmt)
 {
 	*fmt = NULL;
 	return NULL;
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index 8694eeadd753e5..e6fd6bc3064434 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -514,20 +514,20 @@ extern void scsi_sanitize_inquiry_string(unsigned char *s, int len);
  */
 #define SCMD_FAILURE_STAT_ANY	0xff
 /*
- * The following can be set to the scsi_failure sense, asc and ascq fields to
- * match on any sense, ASC, or ASCQ value.
+ * The following can be set to the scsi_failure sense key, asc and ascq fields
+ * to match any sense key, ASC, and ASCQ value.
  */
-#define SCMD_FAILURE_SENSE_ANY	0xff
-#define SCMD_FAILURE_ASC_ANY	0xff
-#define SCMD_FAILURE_ASCQ_ANY	0xff
+#define SCMD_FAILURE_SENSE_KEY_ANY	0xff
+#define SCMD_FAILURE_SENSE_CODE_ANY	0xffff
+#define SCMD_FAILURE_ASC_ANY		0xff
+#define SCMD_FAILURE_ASCQ_ANY		0xff
 /* Always retry a matching failure. */
 #define SCMD_FAILURE_NO_LIMIT	-1
 
 struct scsi_failure {
 	int result;
-	u8 sense;
-	u8 asc;
-	u8 ascq;
+	u8 sense_key;
+	u16 sense_code;
 	/*
 	 * Number of times scsi_execute_cmd will retry the failure. It does
 	 * not count for the total_allowed.
@@ -537,6 +537,16 @@ struct scsi_failure {
 	s8 retries;
 };
 
+static inline u8 scsi_failure_asc(const struct scsi_failure *failure)
+{
+	return scsi_sense_code_asc(failure->sense_code);
+}
+
+static inline u8 scsi_failure_ascq(const struct scsi_failure *failure)
+{
+	return scsi_sense_code_ascq(failure->sense_code);
+}
+
 struct scsi_failures {
 	/*
 	 * If a scsi_failure does not have a retry limit setup this limit will
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index 16243ec376cd97..daaf1247673ff8 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -534,9 +534,9 @@ struct scsi_host_template {
 		unsigned long irq_flags;				\
 		enum scsi_qc_status rc;					\
 									\
-		spin_lock_irqsave(shost->host_lock, irq_flags);		\
+		spin_lock_irqsave(&shost->host_lock, irq_flags);	\
 		rc = func_name##_lck(cmd);				\
-		spin_unlock_irqrestore(shost->host_lock, irq_flags);	\
+		spin_unlock_irqrestore(&shost->host_lock, irq_flags);	\
 		return rc;						\
 	}
 
@@ -570,8 +570,7 @@ struct Scsi_Host {
 	
 	struct list_head	starved_list;
 
-	spinlock_t		default_lock;
-	spinlock_t		*host_lock;
+	spinlock_t		host_lock;
 
 	struct mutex		scan_mutex;/* serialize scanning activity */
 
@@ -953,6 +952,6 @@ static inline unsigned char scsi_host_get_guard(struct Scsi_Host *shost)
 }
 
 int scsi_host_set_state(struct Scsi_Host *shost, enum scsi_host_state state)
-	__must_hold(shost->host_lock);
+	__must_hold(&shost->host_lock);
 
 #endif /* _SCSI_SCSI_HOST_H */
diff --git a/include/scsi/scsi_proto.h b/include/scsi/scsi_proto.h
index c6b4a4dc8d9cad..9c7d31aa6bae59 100644
--- a/include/scsi/scsi_proto.h
+++ b/include/scsi/scsi_proto.h
@@ -12,6 +12,7 @@
 
 #include <linux/build_bug.h>
 #include <linux/types.h>
+#include <scsi/scsi_sense.h>
 
 /*
  *      SCSI opcodes
@@ -218,76 +219,6 @@ enum sam_status {
 
 #define STATUS_MASK         0xfe
 
-/*
- *  SENSE KEYS
- */
-#define NO_SENSE            0x00
-#define RECOVERED_ERROR     0x01
-#define NOT_READY           0x02
-#define MEDIUM_ERROR        0x03
-#define HARDWARE_ERROR      0x04
-#define ILLEGAL_REQUEST     0x05
-#define UNIT_ATTENTION      0x06
-#define DATA_PROTECT        0x07
-#define BLANK_CHECK         0x08
-#define VENDOR_SPECIFIC     0x09
-#define COPY_ABORTED        0x0a
-#define ABORTED_COMMAND     0x0b
-#define VOLUME_OVERFLOW     0x0d
-#define MISCOMPARE          0x0e
-#define COMPLETED	    0x0f
-
-/*
- * Additional Sense Codes (ASC).
- */
-#define NO_ADDITIONAL_SENSE		0x00
-#define OVERLAP_ATOMIC_COMMAND_ASC	0x00
-#define LOGICAL_UNIT_NOT_READY		0x04
-#define LOGICAL_UNIT_COMMUNICATION_FAILURE 0x8
-#define WRITE_ERROR_ASC			0x0c
-#define UNRECOVERED_READ_ERR		0x11
-#define PARAMETER_LIST_LENGTH_ERR	0x1a
-#define MISCOMPARE_VERIFY_ASC		0x1d
-#define INVALID_OPCODE			0x20
-#define LBA_OUT_OF_RANGE		0x21
-#define INVALID_FIELD_IN_CDB		0x24
-#define INVALID_FIELD_IN_PARAM_LIST	0x26
-#define WRITE_PROTECTED			0x27
-#define UA_READY_ASC			0x28
-#define UA_RESET_ASC			0x29
-#define UA_CHANGED_ASC			0x2a
-#define TOO_MANY_IN_PARTITION_ASC	0x3b
-#define TARGET_CHANGED_ASC		0x3f
-#define SAVING_PARAMS_UNSUP		0x39
-#define TRANSPORT_PROBLEM		0x4b
-#define INSUFF_RES_ASC			0x55
-#define LOW_POWER_COND_ON		0x5e
-#define THRESHOLD_EXCEEDED		0x5d
-
-/*
- * Additional Sense Code Qualifiers (ASCQ).
- */
-#define POWER_ON_RESET_ASCQ		0x00
-#define MODE_CHANGED_ASCQ		0x01	/* mode parameters changed */
-#define FILEMARK_DETECTED_ASCQ		0x01
-#define POWER_ON_OCCURRED_ASCQ		0x01
-#define MICROCODE_CHANGED_ASCQ		0x01	/* with TARGET_CHANGED_ASC */
-#define BUS_RESET_ASCQ			0x02	/* scsi bus reset occurred */
-#define EOP_EOM_DETECTED_ASCQ		0x02
-#define INSUFF_RES_ASCQ			0x03
-#define BEGINNING_OF_P_M_DETECTED_ASCQ	0x04
-#define UNALIGNED_WRITE_ASCQ		0x04
-#define EOD_DETECTED_ASCQ		0x05
-#define WRITE_BOUNDARY_ASCQ		0x05
-#define READ_INVDATA_ASCQ		0x06
-#define READ_BOUNDARY_ASCQ		0x07
-#define CAPACITY_CHANGED_ASCQ		0x09
-#define ATTEMPT_ACCESS_GAP		0x09
-#define LUNS_CHANGED_ASCQ		0x0e
-#define INSUFF_ZONE_ASCQ		0x0e
-#define MICROCODE_CHANGED_WO_RESET_ASCQ 0x16
-#define OVERLAP_ATOMIC_COMMAND_ASCQ	0x23
-
 /*
  *  DEVICE TYPES
  *  Please keep them in 0x%02x format for $MODALIAS to work
diff --git a/include/scsi/scsi_sense.h b/include/scsi/scsi_sense.h
new file mode 100644
index 00000000000000..6e010a72482ef5
--- /dev/null
+++ b/include/scsi/scsi_sense.h
@@ -0,0 +1,950 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Definitions of SCSI sense keys, additional sense codes and additional sense
+ * code qualifiers.
+ */
+#ifndef _SCSI_SENSE_H_
+#define _SCSI_SENSE_H_
+
+#include <linux/types.h>
+
+/*
+ * Sense keys.
+ * See: https://www.t10.org/lists/2sensekey.htm
+ */
+enum scsi_sense_key {
+	NO_SENSE		= 0x00,
+	RECOVERED_ERROR		= 0x01,
+	NOT_READY		= 0x02,
+	MEDIUM_ERROR		= 0x03,
+	HARDWARE_ERROR		= 0x04,
+	ILLEGAL_REQUEST		= 0x05,
+	UNIT_ATTENTION		= 0x06,
+	DATA_PROTECT		= 0x07,
+	BLANK_CHECK		= 0x08,
+	VENDOR_SPECIFIC		= 0x09,
+	COPY_ABORTED		= 0x0a,
+	ABORTED_COMMAND		= 0x0b,
+	VOLUME_OVERFLOW		= 0x0d,
+	MISCOMPARE		= 0x0e,
+	COMPLETED		= 0x0f,
+};
+
+#define SCSI_ASC_DEF(name)	ASC_##name
+#define SCSI_ASC(name, code)				\
+	SCSI_ASC_DEF(name)	= (code),		\
+	name			= ((code) << 8) | 0x00
+
+/*
+ * SCSI additional sense codes and sense codes qualifiers.
+ * See: https://www.t10.org/lists/asc-num.txt
+ * Additional sense codes and their qualifiers are combined as a single value
+ * equal to (ASC << 8) | ASCQ.
+ *
+ * The definitions are sorted in increasing sense code value order with a naming
+ * following as close as possible the exact naming used in T10. To avoid
+ * excessively long code lines when using these macros, some long ASC/ASCQ
+ * combination names are shortened (E.g. LOGICAL_UNIT_xxx -> LU_xxx).
+ * Very generic ASC names that potentially may clash with other macros elsewhere
+ * are prefixed with "SSC_" (e.g. WARNING and WRITE_ERROR).
+ */
+enum scsi_sense_code {
+	SCSI_ASC(NO_ADDITIONAL_SENSE_INFORMATION,		0x00),
+	FILEMARK_DETECTED					= 0x0001,
+	END_OF_PARTITION_MEDIUM_DETECTED			= 0x0002,
+	SETMARK_DETECTED					= 0x0003,
+	BEGINNING_OF_PARTITION_MEDIUM_DETECTED			= 0x0004,
+	END_OF_DATA_DETECTED					= 0x0005,
+	IO_PROCESS_TERMINATED					= 0x0006,
+	PROGRAMMABLE_EARLY_WARNING_DETECTED			= 0x0007,
+	AUDIO_PLAY_OP_IN_PROGRESS				= 0x0011,
+	AUDIO_PLAY_OP_PAUSED					= 0x0012,
+	AUDIO_PLAY_OP_SUCCESSFULLY_COMPLETED			= 0x0013,
+	AUDIO_PLAY_OP_STOPPED_DUE_TO_ERROR			= 0x0014,
+	NO_CURRENT_AUDIO_STATUS_TO_RETURN			= 0x0015,
+	OP_IN_PROGRESS						= 0x0016,
+	CLEANING_REQUESTED					= 0x0017,
+	ERASE_OP_IN_PROGRESS					= 0x0018,
+	LOCATE_OP_IN_PROGRESS					= 0x0019,
+	REWIND_OP_IN_PROGRESS					= 0x001A,
+	SET_CAPACITY_OP_IN_PROGRESS				= 0x001B,
+	VERIFY_OP_IN_PROGRESS					= 0x001C,
+	ATA_PASS_THROUGH_INFORMATION_AVAILABLE			= 0x001D,
+	CONFLICTING_SA_CREATION_REQUEST				= 0x001E,
+	LU_TRANSITIONING_TO_ANOTHER_POWER_CONDITION		= 0x001F,
+	EXTENDED_COPY_INFORMATION_AVAILABLE			= 0x0020,
+	ATOMIC_COMMAND_ABORTED_DUE_TO_ACA			= 0x0021,
+	DEFERRED_MICROCODE_IS_PENDING				= 0x0022,
+	OVERLAPPING_ATOMIC_COMMAND_IN_PROGRESS			= 0x0023,
+
+	SCSI_ASC(NO_INDEX_SECTOR_SIGNAL,			0x01),
+
+	SCSI_ASC(NO_SEEK_COMPLETE,				0x02),
+
+	SCSI_ASC(PERIPHERAL_DEVICE_WRITE_FAULT,			0x03),
+	NO_WRITE_CURRENT					= 0x0301,
+	EXCESSIVE_WRITE_ERRORS					= 0x0302,
+
+	SCSI_ASC(LU_NOT_READY,					0x04),
+	LU_IS_IN_PROCESS_OF_BECOMING_READY			= 0x0401,
+	LU_NOT_READY_INITIALIZING_COMMAND_REQUIRED		= 0x0402,
+	LU_NOT_READY_MANUAL_INTERVENTION_REQUIRED		= 0x0403,
+	LU_NOT_READY_FORMAT_IN_PROGRESS				= 0x0404,
+	LU_NOT_READY_REBUILD_IN_PROGRESS			= 0x0405,
+	LU_NOT_READY_RECALCULATION_IN_PROGRESS			= 0x0406,
+	LU_NOT_READY_OP_IN_PROGRESS				= 0x0407,
+	LU_NOT_READY_LONG_WRITE_IN_PROGRESS			= 0x0408,
+	LU_NOT_READY_SELFTEST_IN_PROGRESS			= 0x0409,
+	LU_NOT_ACCESSIBLE_ASYMMETRIC_ACCESS_STATE_TRANSITION	= 0x040A,
+	LU_NOT_ACCESSIBLE_TARGET_PORT_IN_STANDBY_STATE		= 0x040B,
+	LU_NOT_ACCESSIBLE_TARGET_PORT_IN_UNAVAILABLE_STATE	= 0x040C,
+	LU_NOT_READY_STRUCTURE_CHECK_REQUIRED			= 0x040D,
+	LU_NOT_READY_SECURITY_SESSION_IN_PROGRESS		= 0x040E,
+	LU_NOT_READY_AUXILIARY_MEMORY_NOT_ACCESSIBLE		= 0x0410,
+	LU_NOT_READY_NOTIFY_REQUIRED				= 0x0411,
+	LU_NOT_READY_OFFLINE					= 0x0412,
+	LU_NOT_READY_SA_CREATION_IN_PROGRESS			= 0x0413,
+	LU_NOT_READY_SPACE_ALLOCATION_IN_PROGRESS		= 0x0414,
+	LU_NOT_READY_ROBOTICS_DISABLED				= 0x0415,
+	LU_NOT_READY_CONFIG_REQUIRED				= 0x0416,
+	LU_NOT_READY_CALIBRATION_REQUIRED			= 0x0417,
+	LU_NOT_READY_A_DOOR_IS_OPEN				= 0x0418,
+	LU_NOT_READY_OPERATING_IN_SEQUENTIAL_MODE		= 0x0419,
+	LU_NOT_READY_START_STOP_UNIT_COMMAND_IN_PROGRESS	= 0x041A,
+	LU_NOT_READY_SANITIZE_IN_PROGRESS			= 0x041B,
+	LU_NOT_READY_ADDITIONAL_POWER_USE_NOT_YET_GRANTED	= 0x041C,
+	LU_NOT_READY_CONFIG_IN_PROGRESS				= 0x041D,
+	LU_NOT_READY_MICROCODE_ACTIVATION_REQUIRED		= 0x041E,
+	LU_NOT_READY_MICROCODE_DOWNLOAD_REQUIRED		= 0x041F,
+	LU_NOT_READY_LU_RESET_REQUIRED				= 0x0420,
+	LU_NOT_READY_HARD_RESET_REQUIRED			= 0x0421,
+	LU_NOT_READY_POWER_CYCLE_REQUIRED			= 0x0422,
+	LU_NOT_READY_AFFILIATION_REQUIRED			= 0x0423,
+	DEPOPULATION_IN_PROGRESS				= 0x0424,
+	DEPOPULATION_RESTORATION_IN_PROGRESS			= 0x0425,
+
+	SCSI_ASC(LU_DOES_NOT_RESPOND_TO_SELECTION,		0x05),
+
+	SCSI_ASC(NO_REFERENCE_POSITION_FOUND,			0x06),
+
+	SCSI_ASC(MULTIPLE_PERIPHERAL_DEVICES_SELECTED,		0x07),
+
+	SCSI_ASC(LU_COMMUNICATION_FAILURE,			0x08),
+	LU_COMMUNICATION_TIMEOUT				= 0x0801,
+	LU_COMMUNICATION_PARITY_ERROR				= 0x0802,
+	LU_COMMUNICATION_CRC_ERROR				= 0x0803,
+	UNREACHABLE_COPY_TARGET					= 0x0804,
+
+	SCSI_ASC(TRACK_FOLLOWING_ERROR,				0x09),
+	TRACKING_SERVO_FAILURE					= 0x0901,
+	FOCUS_SERVO_FAILURE					= 0x0902,
+	SPINDLE_SERVO_FAILURE					= 0x0903,
+	HEAD_SELECT_FAULT					= 0x0904,
+	VIBRATION_INDUCED_TRACKING_ERROR			= 0x0905,
+
+	SCSI_ASC(ERROR_LOG_OVERFLOW,				0x0A),
+
+	SCSI_ASC(SSC_WARNING,					0x0B),
+	WARNING_SPECIFIED_TEMPERATURE_EXCEEDED			= 0x0B01,
+	WARNING_ENCLOSURE_DEGRADED				= 0x0B02,
+	WARNING_BACKGROUND_SELFTEST_FAILED			= 0x0B03,
+	WARNING_BACKGROUND_PRESCAN_DETECTED_MEDIUM_ERROR	= 0x0B04,
+	WARNING_BACKGROUND_MEDIUM_SCAN_DETECTED_MEDIUM_ERROR	= 0x0B05,
+	WARNING_NONVOLATILE_CACHE_NOW_VOLATILE			= 0x0B06,
+	WARNING_DEGRADED_POWER_TO_NONVOLATILE_CACHE		= 0x0B07,
+	WARNING_POWER_LOSS_EXPECTED				= 0x0B08,
+	WARNING_DEVICE_STATISTICS_NOTIFICATION_ACTIVE		= 0x0B09,
+	WARNING_HIGH_CRITICAL_TEMPERATURE_LIMIT_EXCEEDED	= 0x0B0A,
+	WARNING_LOW_CRITICAL_TEMPERATURE_LIMIT_EXCEEDED		= 0x0B0B,
+	WARNING_HIGH_OPERATING_TEMPERATURE_LIMIT_EXCEEDED	= 0x0B0C,
+	WARNING_LOW_OPERATING_TEMPERATURE_LIMIT_EXCEEDED	= 0x0B0D,
+	WARNING_HIGH_CRITICAL_HUMIDITY_LIMIT_EXCEEDED		= 0x0B0E,
+	WARNING_LOW_CRITICAL_HUMIDITY_LIMIT_EXCEEDED		= 0x0B0F,
+	WARNING_HIGH_OPERATING_HUMIDITY_LIMIT_EXCEEDED		= 0x0B10,
+	WARNING_LOW_OPERATING_HUMIDITY_LIMIT_EXCEEDED		= 0x0B11,
+	WARNING_MICROCODE_SECURITY_AT_RISK			= 0x0B12,
+	WARNING_MICROCODE_DIGITAL_SIGNATURE_VALIDATION_FAILURE	= 0x0B13,
+	WARNING_PHYSICAL_ELEMENT_STATUS_CHANGE			= 0x0B14,
+
+	/* Defined as "WRITE ERROR" in T10. */
+	SCSI_ASC(SSC_WRITE_ERROR,				0x0C),
+	WRITE_ERROR_RECOVERED_WITH_AUTO_REALLOCATION		= 0x0C01,
+	WRITE_ERROR_AUTO_REALLOCATION_FAILED			= 0x0C02,
+	WRITE_ERROR_RECOMMEND_REASSIGNMENT			= 0x0C03,
+	COMPRESSION_CHECK_MISCOMPARE_ERROR			= 0x0C04,
+	DATA_EXPANSION_OCCURRED_DURING_COMPRESSION		= 0x0C05,
+	BLOCK_NOT_COMPRESSIBLE					= 0x0C06,
+	WRITE_ERROR_RECOVERY_NEEDED				= 0x0C07,
+	WRITE_ERROR_RECOVERY_FAILED				= 0x0C08,
+	WRITE_ERROR_LOSS_OF_STREAMING				= 0x0C09,
+	WRITE_ERROR_PADDING_BLOCKS_ADDED			= 0x0C0A,
+	AUXILIARY_MEMORY_WRITE_ERROR				= 0x0C0B,
+	WRITE_ERROR_UNEXPECTED_UNSOLICITED_DATA			= 0x0C0C,
+	WRITE_ERROR_NOT_ENOUGH_UNSOLICITED_DATA			= 0x0C0D,
+	MULTIPLE_WRITE_ERRORS					= 0x0C0E,
+	DEFECTS_IN_ERROR_WINDOW					= 0x0C0F,
+	INCOMPLETE_MULTIPLE_ATOMIC_WRITE_OPERATIONS		= 0x0C10,
+	WRITE_ERROR_RECOVERY_SCAN_NEEDED			= 0x0C11,
+	WRITE_ERROR_INSUFFICIENT_ZONE_RESOURCES			= 0x0C12,
+
+	SCSI_ASC(ERROR_DETECTED_BY_THIRD_PARTY_TEMPORARY_INITIATOR, 0x0D),
+	THIRD_PARTY_DEVICE_FAILURE				= 0x0D01,
+	COPY_TARGET_DEVICE_NOT_REACHABLE			= 0x0D02,
+	INCORRECT_COPY_TARGET_DEVICE_TYPE			= 0x0D03,
+	COPY_TARGET_DEVICE_DATA_UNDERRUN			= 0x0D04,
+	COPY_TARGET_DEVICE_DATA_OVERRUN				= 0x0D05,
+
+	SCSI_ASC(INVALID_INFORMATION_UNIT,			0x0E),
+	INFORMATION_UNIT_TOO_SHORT				= 0x0E01,
+	INFORMATION_UNIT_TOO_LONG				= 0x0E02,
+	INVALID_FIELD_IN_COMMAND_INFORMATION_UNIT		= 0x0E03,
+
+	SCSI_ASC(ID_CRC_OR_ECC_ERROR,				0x10),
+	LOGICAL_BLOCK_GUARD_CHECK_FAILED			= 0x1001,
+	LOGICAL_BLOCK_APPLICATION_TAG_CHECK_FAILED		= 0x1002,
+	LOGICAL_BLOCK_REFERENCE_TAG_CHECK_FAILED		= 0x1003,
+	LOGICAL_BLOCK_PROTECTION_ERROR_ON_RECOVER_BUFFERED_DATA	= 0x1004,
+	LOGICAL_BLOCK_PROTECTION_METHOD_ERROR			= 0x1005,
+
+	SCSI_ASC(UNRECOVERED_READ_ERROR,			0x11),
+	READ_RETRIES_EXHAUSTED					= 0x1101,
+	ERROR_TOO_LONG_TO_CORRECT				= 0x1102,
+	MULTIPLE_READ_ERRORS					= 0x1103,
+	UNRECOVERED_READ_ERROR_AUTO_REALLOCATE_FAILED		= 0x1104,
+	LEC_UNCORRECTABLE_ERROR					= 0x1105,
+	CIRC_UNRECOVERED_ERROR					= 0x1106,
+	DATA_RESYNCHRONIZATION_ERROR				= 0x1107,
+	INCOMPLETE_BLOCK_READ					= 0x1108,
+	NO_GAP_FOUND						= 0x1109,
+	MISCORRECTED_ERROR					= 0x110A,
+	UNRECOVERED_READ_ERROR_RECOMMEND_REASSIGNMENT		= 0x110B,
+	UNRECOVERED_READ_ERROR_RECOMMEND_REWRITE_THE_DATA	= 0x110C,
+	DECOMPRESSION_CRC_ERROR					= 0x110D,
+	CANNOT_DECOMPRESS_USING_DECLARED_ALGORITHM		= 0x110E,
+	ERROR_READING_UPC_EAN_NUMBER				= 0x110F,
+	ERROR_READING_ISRC_NUMBER				= 0x1110,
+	READ_ERROR_LOSS_OF_STREAMING				= 0x1111,
+	AUXILIARY_MEMORY_READ_ERROR				= 0x1112,
+	READ_ERROR_FAILED_RETRANSMISSION_REQUEST		= 0x1113,
+	READ_ERROR_LBA_MARKED_BAD_BY_APPLICATION_CLIENT		= 0x1114,
+	WRITE_AFTER_SANITIZE_REQUIRED				= 0x1115,
+
+	SCSI_ASC(ADDRESS_MARK_NOT_FOUND_FOR_ID_FIELD,		0x12),
+
+	SCSI_ASC(ADDRESS_MARK_NOT_FOUND_FOR_DATA_FIELD,		0x13),
+
+	SCSI_ASC(RECORDED_ENTITY_NOT_FOUND,			0x14),
+	RECORD_NOT_FOUND					= 0x1401,
+	FILEMARK_OR_SETMARK_NOT_FOUND				= 0x1402,
+	END_OF_DATA_NOT_FOUND					= 0x1403,
+	BLOCK_SEQUENCE_ERROR					= 0x1404,
+	RECORD_NOT_FOUND_RECOMMEND_REASSIGNMENT			= 0x1405,
+	RECORD_NOT_FOUND_DATA_AUTO_REALLOCATED			= 0x1406,
+	LOCATE_OPERATION_FAILURE				= 0x1407,
+
+	SCSI_ASC(RANDOM_POSITIONING_ERROR,			0x15),
+	MECHANICAL_POSITIONING_ERROR				= 0x1501,
+	POSITIONING_ERROR_DETECTED_BY_READ_OF_MEDIUM		= 0x1502,
+
+	SCSI_ASC(DATA_SYNCHRONIZATION_MARK_ERROR,		0x16),
+	DATA_SYNC_ERROR_DATA_REWRITTEN				= 0x1601,
+	DATA_SYNC_ERROR_RECOMMEND_REWRITE			= 0x1602,
+	DATA_SYNC_ERROR_DATA_AUTO_REALLOCATED			= 0x1603,
+	DATA_SYNC_ERROR_RECOMMEND_REASSIGNMENT			= 0x1604,
+
+	SCSI_ASC(RECOVERED_DATA_WITH_NO_ERROR_CORRECTION_APPLIED, 0x17),
+	RECOVERED_DATA_WITH_RETRIES				= 0x1701,
+	RECOVERED_DATA_WITH_POSITIVE_HEAD_OFFSET		= 0x1702,
+	RECOVERED_DATA_WITH_NEGATIVE_HEAD_OFFSET		= 0x1703,
+	RECOVERED_DATA_WITH_RETRIES_AND_OR_CIRC_APPLIED		= 0x1704,
+	RECOVERED_DATA_USING_PREVIOUS_SECTOR_ID			= 0x1705,
+	RECOVERED_DATA_WITHOUT_ECC_DATA_AUTO_REALLOCATED	= 0x1706,
+	RECOVERED_DATA_WITHOUT_ECC_RECOMMEND_REASSIGNMENT	= 0x1707,
+	RECOVERED_DATA_WITHOUT_ECC_RECOMMEND_REWRITE		= 0x1708,
+	RECOVERED_DATA_WITHOUT_ECC_DATA_REWRITTEN		= 0x1709,
+
+	SCSI_ASC(RECOVERED_DATA_WITH_ERROR_CORRECTION_APPLIED,	0x18),
+	RECOVERED_DATA_WITH_ERROR_CORRECTION_RETRIES_APPLIED	= 0x1801,
+	RECOVERED_DATA_DATA_AUTO_REALLOCATED			= 0x1802,
+	RECOVERED_DATA_WITH_CIRC				= 0x1803,
+	RECOVERED_DATA_WITH_LEC					= 0x1804,
+	RECOVERED_DATA_RECOMMEND_REASSIGNMENT			= 0x1805,
+	RECOVERED_DATA_RECOMMEND_REWRITE			= 0x1806,
+	RECOVERED_DATA_WITH_ECC_DATA_REWRITTEN			= 0x1807,
+	RECOVERED_DATA_WITH_LINKING				= 0x1808,
+
+	SCSI_ASC(DEFECT_LIST_ERROR,				0x19),
+	DEFECT_LIST_NOT_AVAILABLE				= 0x1901,
+	DEFECT_LIST_ERROR_IN_PRIMARY_LIST			= 0x1902,
+	DEFECT_LIST_ERROR_IN_GROWN_LIST				= 0x1903,
+
+	SCSI_ASC(PARAMETER_LIST_LENGTH_ERROR,			0x1A),
+
+	SCSI_ASC(SYNCHRONOUS_DATA_TRANSFER_ERROR,		0x1B),
+
+	SCSI_ASC(DEFECT_LIST_NOT_FOUND,				0x1C),
+	PRIMARY_DEFECT_LIST_NOT_FOUND				= 0x1C01,
+	GROWN_DEFECT_LIST_NOT_FOUND				= 0x1C02,
+
+	SCSI_ASC(MISCOMPARE_DURING_VERIFY_OPERATION,		0x1D),
+	MISCOMPARE_VERIFY_OF_UNMAPPED_LBA			= 0x1D01,
+
+	SCSI_ASC(RECOVERED_ID_WITH_ECC_CORRECTION,		0x1E),
+
+	SCSI_ASC(PARTIAL_DEFECT_LIST_TRANSFER,			0x1F),
+
+	SCSI_ASC(INVALID_COMMAND_OP_CODE,			0x20),
+	ACCESS_DENIED_INITIATOR_PENDING_ENROLLED		= 0x2001,
+	ACCESS_DENIED_NO_ACCESS_RIGHTS				= 0x2002,
+	ACCESS_DENIED_INVALID_MGMT_ID_KEY			= 0x2003,
+	ILLEGAL_COMMAND_WHILE_IN_WRITE_CAPABLE_STATE		= 0x2004,
+	ILLEGAL_COMMAND_WHILE_IN_EXPLICIT_ADDRESS_MODE		= 0x2006,
+	ILLEGAL_COMMAND_WHILE_IN_IMPLICIT_ADDRESS_MODE		= 0x2007,
+	ACCESS_DENIED_ENROLLMENT_CONFLICT			= 0x2008,
+	ACCESS_DENIED_INVALID_LU_IDENTIFIER			= 0x2009,
+	ACCESS_DENIED_INVALID_PROXY_TOKEN			= 0x200A,
+	ACCESS_DENIED_ACL_LUN_CONFLICT				= 0x200B,
+	ILLEGAL_COMMAND_WHEN_NOT_IN_APPEND_ONLY_MODE		= 0x200C,
+	NOT_AN_ADMINISTRATIVE_LOGICAL_UNIT			= 0x200D,
+	NOT_A_SUBSIDIARY_LOGICAL_UNIT				= 0x200E,
+	NOT_A_CONGLOMERATE_LOGICAL_UNIT				= 0x200F,
+
+	SCSI_ASC(LOGICAL_BLOCK_ADDRESS_OUT_OF_RANGE,		0x21),
+	INVALID_ELEMENT_ADDRESS					= 0x2101,
+	INVALID_ADDRESS_FOR_WRITE				= 0x2102,
+	INVALID_WRITE_CROSSING_LAYER_JUMP			= 0x2103,
+	UNALIGNED_WRITE_COMMAND					= 0x2104,
+	WRITE_BOUNDARY_VIOLATION				= 0x2105,
+	ATTEMPT_TO_READ_INVALID_DATA				= 0x2106,
+	READ_BOUNDARY_VIOLATION					= 0x2107,
+	MISALIGNED_WRITE_COMMAND				= 0x2108,
+	ATTEMPT_TO_ACCESS_GAP_ZONE				= 0x2109,
+
+	SCSI_ASC(ILLEGAL_FUNCTION,				0x22),
+
+	SCSI_ASC(INVALID_TOKEN_OP,				0x23),
+	INVALID_TOKEN_OP_UNSUPPORTED_TOKEN_TYPE			= 0x2301,
+	INVALID_TOKEN_OP_REMOTE_TOKEN_USAGE_NOT_SUPPORTED	= 0x2302,
+	INVALID_TOKEN_OP_REMOTE_ROD_TOKEN_CREATION_NOT_SUPPORTED = 0x2303,
+	INVALID_TOKEN_OP_TOKEN_UNKNOWN				= 0x2304,
+	INVALID_TOKEN_OP_TOKEN_CORRUPT				= 0x2305,
+	INVALID_TOKEN_OP_TOKEN_REVOKED				= 0x2306,
+	INVALID_TOKEN_OP_TOKEN_EXPIRED				= 0x2307,
+	INVALID_TOKEN_OP_TOKEN_CANCELLED			= 0x2308,
+	INVALID_TOKEN_OP_TOKEN_DELETED				= 0x2309,
+	INVALID_TOKEN_OP_INVALID_TOKEN_LENGTH			= 0x230A,
+
+	SCSI_ASC(INVALID_FIELD_IN_CDB,				0x24),
+	CDB_DECRYPTION_ERROR					= 0x2401,
+	SECURITY_AUDIT_VALUE_FROZEN				= 0x2404,
+	SECURITY_WORKING_KEY_FROZEN				= 0x2405,
+	NONCE_NOT_UNIQUE					= 0x2406,
+	NONCE_TIMESTAMP_OUT_OF_RANGE				= 0x2407,
+	INVALID_XCDB						= 0x2408,
+	INVALID_FAST_FORMAT					= 0x2409,
+
+	SCSI_ASC(LU_NOT_SUPPORTED,				0x25),
+
+	SCSI_ASC(INVALID_FIELD_IN_PARAMETER_LIST,		0x26),
+	PARAMETER_NOT_SUPPORTED					= 0x2601,
+	PARAMETER_VALUE_INVALID					= 0x2602,
+	THRESHOLD_PARAMETERS_NOT_SUPPORTED			= 0x2603,
+	INVALID_RELEASE_OF_PERSISTENT_RESERVATION		= 0x2604,
+	DATA_DECRYPTION_ERROR					= 0x2605,
+	TOO_MANY_TARGET_DESCRIPTORS				= 0x2606,
+	UNSUPPORTED_TARGET_DESCRIPTOR_TYPE_CODE			= 0x2607,
+	TOO_MANY_SEGMENT_DESCRIPTORS				= 0x2608,
+	UNSUPPORTED_SEGMENT_DESCRIPTOR_TYPE_CODE		= 0x2609,
+	UNEXPECTED_INEXACT_SEGMENT				= 0x260A,
+	INLINE_DATA_LENGTH_EXCEEDED				= 0x260B,
+	INVALID_OP_FOR_COPY_SOURCE_OR_DESTINATION		= 0x260C,
+	COPY_SEGMENT_GRANULARITY_VIOLATION			= 0x260D,
+	INVALID_PARAMETER_WHILE_PORT_IS_ENABLED			= 0x260E,
+	INVALID_DATA_OUT_BUFFER_INTEGRITY_CHECK_VALUE		= 0x260F,
+	DATA_DECRYPTION_KEY_FAIL_LIMIT_REACHED			= 0x2610,
+	INCOMPLETE_KEY_ASSOCIATED_DATA_SET			= 0x2611,
+	VENDOR_SPECIFIC_KEY_REFERENCE_NOT_FOUND			= 0x2612,
+	APPLICATION_TAG_MODE_PAGE_IS_INVALID			= 0x2613,
+	TAPE_STREAM_MIRRORING_PREVENTED				= 0x2614,
+	COPY_SOURCE_OR_COPY_DESTINATION_NOT_AUTHORIZED		= 0x2615,
+	FAST_COPY_NOT_POSSIBLE					= 0x2616,
+
+	SCSI_ASC(WRITE_PROTECTED,				0x27),
+	HW_WRITE_PROTECTED					= 0x2701,
+	LU_SOFTWARE_WRITE_PROTECTED				= 0x2702,
+	ASSOCIATED_WRITE_PROTECT				= 0x2703,
+	PERSISTENT_WRITE_PROTECT				= 0x2704,
+	PERMANENT_WRITE_PROTECT					= 0x2705,
+	CONDITIONAL_WRITE_PROTECT				= 0x2706,
+	SPACE_ALLOCATION_FAILED_WRITE_PROTECT			= 0x2707,
+	ZONE_IS_READ_ONLY					= 0x2708,
+
+	SCSI_ASC(NOT_READY_TO_READY_CHANGE_MEDIUM_MAY_HAVE_CHANGED, 0x28),
+	IMPORT_OR_EXPORT_ELEMENT_ACCESSED			= 0x2801,
+	FORMAT_LAYER_MAY_HAVE_CHANGED				= 0x2802,
+	IMPORT_EXPORT_ELEMENT_ACCESSED_MEDIUM_CHANGED		= 0x2803,
+
+	SCSI_ASC(POWER_ON_RESET_OR_BUS_DEVICE_RESET_OCCURRED,	0x29),
+	POWER_ON_OCCURRED					= 0x2901,
+	SCSI_BUS_RESET_OCCURRED					= 0x2902,
+	BUS_DEVICE_RESET_FUNCTION_OCCURRED			= 0x2903,
+	DEVICE_INTERNAL_RESET					= 0x2904,
+	TRANSCEIVER_MODE_CHANGED_TO_SINGLE_ENDED		= 0x2905,
+	TRANSCEIVER_MODE_CHANGED_TO_LVD				= 0x2906,
+	I_T_NEXUS_LOSS_OCCURRED					= 0x2907,
+
+	SCSI_ASC(PARAMETERS_CHANGED,				0x2A),
+	MODE_PARAMETERS_CHANGED					= 0x2A01,
+	LOG_PARAMETERS_CHANGED					= 0x2A02,
+	RESERVATIONS_PREEMPTED					= 0x2A03,
+	RESERVATIONS_RELEASED					= 0x2A04,
+	REGISTRATIONS_PREEMPTED					= 0x2A05,
+	ASYMMETRIC_ACCESS_STATE_CHANGED				= 0x2A06,
+	IMPLICIT_ASYMMETRIC_ACCESS_STATE_TRANSITION_FAILED	= 0x2A07,
+	PRIORITY_CHANGED					= 0x2A08,
+	CAPACITY_DATA_HAS_CHANGED				= 0x2A09,
+	ERROR_HISTORY_I_T_NEXUS_CLEARED				= 0x2A0A,
+	ERROR_HISTORY_SNAPSHOT_RELEASED				= 0x2A0B,
+	ERROR_RECOVERY_ATTRIBUTES_HAVE_CHANGED			= 0x2A0C,
+	DATA_ENCRYPTION_CAPABILITIES_CHANGED			= 0x2A0D,
+	TIMESTAMP_CHANGED					= 0x2A10,
+	DATA_ENCRYPTION_PARAMS_CHANGED_BY_ANOTHER_I_T_NEXUS	= 0x2A11,
+	DATA_ENCRYPTION_PARAMS_CHANGED_BY_VENDOR_SPECIFIC_EVENT = 0x2A12,
+	DATA_ENCRYPTION_KEY_INSTANCE_COUNTER_HAS_CHANGED	= 0x2A13,
+	SA_CREATION_CAPABILITIES_DATA_HAS_CHANGED		= 0x2A14,
+	MEDIUM_REMOVAL_PREVENTION_PREEMPTED			= 0x2A15,
+	ZONE_RESET_WRITE_POINTER_RECOMMENDED			= 0x2A16,
+
+	SCSI_ASC(COPY_CANNOT_EXECUTE_SINCE_HOST_CANNOT_DISCONNECT, 0x2B),
+
+	SCSI_ASC(COMMAND_SEQUENCE_ERROR,			0x2C),
+	TOO_MANY_WINDOWS_SPECIFIED				= 0x2C01,
+	INVALID_COMBINATION_OF_WINDOWS_SPECIFIED		= 0x2C02,
+	CURRENT_PROGRAM_AREA_IS_NOT_EMPTY			= 0x2C03,
+	CURRENT_PROGRAM_AREA_IS_EMPTY				= 0x2C04,
+	ILLEGAL_POWER_CONDITION_REQUEST				= 0x2C05,
+	PERSISTENT_PREVENT_CONFLICT				= 0x2C06,
+	PREVIOUS_BUSY_STATUS					= 0x2C07,
+	PREVIOUS_TASK_SET_FULL_STATUS				= 0x2C08,
+	PREVIOUS_RESERVATION_CONFLICT_STATUS			= 0x2C09,
+	PARTITION_OR_COLLECTION_CONTAINS_USER_OBJECTS		= 0x2C0A,
+	NOT_RESERVED						= 0x2C0B,
+	ORWRITE_GENERATION_DOES_NOT_MATCH			= 0x2C0C,
+	RESET_WRITE_POINTER_NOT_ALLOWED				= 0x2C0D,
+	ZONE_IS_OFFLINE						= 0x2C0E,
+	STREAM_NOT_OPEN						= 0x2C0F,
+	UNWRITTEN_DATA_IN_ZONE					= 0x2C10,
+	DESCRIPTOR_FORMAT_SENSE_DATA_REQUIRED			= 0x2C11,
+	ZONE_IS_INACTIVE					= 0x2C12,
+	WELL_KNOWN_LU_ACCESS_REQUIRED				= 0x2C13,
+
+	SCSI_ASC(OVERWRITE_ERROR_ON_UPDATE_IN_PLACE,		0x2D),
+
+	SCSI_ASC(INSUFFICIENT_TIME_FOR_OPERATION,		0x2E),
+	COMMAND_TIMEOUT_BEFORE_PROCESSING			= 0x2E01,
+	COMMAND_TIMEOUT_DURING_PROCESSING			= 0x2E02,
+	COMMAND_TIMEOUT_DURING_PROCESSING_DUE_TO_ERROR_RECOVERY	= 0x2E03,
+
+	SCSI_ASC(COMMANDS_CLEARED_BY_ANOTHER_INITIATOR,		0x2F),
+	COMMANDS_CLEARED_BY_POWER_LOSS_NOTIFICATION		= 0x2F01,
+	COMMANDS_CLEARED_BY_DEVICE_SERVER			= 0x2F02,
+	SOME_COMMANDS_CLEARED_BY_QUEUING_LAYER_EVENT		= 0x2F03,
+
+	SCSI_ASC(INCOMPATIBLE_MEDIUM_INSTALLED,			0x30),
+	CANNOT_READ_MEDIUM_UNKNOWN_FORMAT			= 0x3001,
+	CANNOT_READ_MEDIUM_INCOMPATIBLE_FORMAT			= 0x3002,
+	CLEANING_CARTRIDGE_INSTALLED				= 0x3003,
+	CANNOT_WRITE_MEDIUM_UNKNOWN_FORMAT			= 0x3004,
+	CANNOT_WRITE_MEDIUM_INCOMPATIBLE_FORMAT			= 0x3005,
+	CANNOT_FORMAT_MEDIUM_INCOMPATIBLE_MEDIUM		= 0x3006,
+	CLEANING_FAILURE					= 0x3007,
+	CANNOT_WRITE_APPLICATION_CODE_MISMATCH			= 0x3008,
+	CURRENT_SESSION_NOT_FIXATED_FOR_APPEND			= 0x3009,
+	CLEANING_REQUEST_REJECTED				= 0x300A,
+	WORM_MEDIUM_OVERWRITE_ATTEMPTED				= 0x300C,
+	WORM_MEDIUM_INTEGRITY_CHECK				= 0x300D,
+	MEDIUM_NOT_FORMATTED					= 0x3010,
+	INCOMPATIBLE_VOLUME_TYPE				= 0x3011,
+	INCOMPATIBLE_VOLUME_QUALIFIER				= 0x3012,
+	CLEANING_VOLUME_EXPIRED					= 0x3013,
+
+	SCSI_ASC(MEDIUM_FORMAT_CORRUPTED,			0x31),
+	FORMAT_COMMAND_FAILED					= 0x3101,
+	ZONED_FORMATTING_FAILED_DUE_TO_SPARE_LINKING		= 0x3102,
+	SANITIZE_COMMAND_FAILED					= 0x3103,
+	DEPOPULATION_FAILED					= 0x3104,
+	DEPOPULATION_RESTORATION_FAILED				= 0x3105,
+
+	SCSI_ASC(NO_DEFECT_SPARE_LOCATION_AVAILABLE,		0x32),
+	DEFECT_LIST_UPDATE_FAILURE				= 0x3201,
+
+	SCSI_ASC(TAPE_LENGTH_ERROR,				0x33),
+
+	SCSI_ASC(ENCLOSURE_FAILURE,				0x34),
+
+	SCSI_ASC(ENCLOSURE_SERVICES_FAILURE,			0x35),
+	UNSUPPORTED_ENCLOSURE_FUNCTION				= 0x3501,
+	ENCLOSURE_SERVICES_UNAVAILABLE				= 0x3502,
+	ENCLOSURE_SERVICES_TRANSFER_FAILURE			= 0x3503,
+	ENCLOSURE_SERVICES_TRANSFER_REFUSED			= 0x3504,
+	ENCLOSURE_SERVICES_CHECKSUM_ERROR			= 0x3505,
+
+	SCSI_ASC(RIBBON_INK_OR_TONER_FAILURE,			0x36),
+
+	SCSI_ASC(ROUNDED_PARAMETER,				0x37),
+
+	SCSI_ASC(EVENT_STATUS_NOTIFICATION,			0x38),
+	ESN_POWER_MANAGEMENT_CLASS_EVENT			= 0x3802,
+	ESN_MEDIA_CLASS_EVENT					= 0x3804,
+	ESN_DEVICE_BUSY_CLASS_EVENT				= 0x3806,
+	THIN_PROVISIONING_SOFT_THRESHOLD_REACHED		= 0x3807,
+	DEPOPULATION_INTERRUPTED				= 0x3808,
+	DEPOPULATION_RESTORATION_INTERRUPTED			= 0x3809,
+
+	SCSI_ASC(SAVING_PARAMETERS_NOT_SUPPORTED,		0x39),
+
+	SCSI_ASC(MEDIUM_NOT_PRESENT,				0x3A),
+	MEDIUM_NOT_PRESENT_TRAY_CLOSED				= 0x3A01,
+	MEDIUM_NOT_PRESENT_TRAY_OPEN				= 0x3A02,
+	MEDIUM_NOT_PRESENT_LOADABLE				= 0x3A03,
+	MEDIUM_NOT_PRESENT_MEDIUM_AUXILIARY_MEMORY_ACCESSIBLE	= 0x3A04,
+
+	SCSI_ASC(SEQUENTIAL_POSITIONING_ERROR,			0x3B),
+	TAPE_POSITION_ERROR_AT_BEGINNING_OF_MEDIUM		= 0x3B01,
+	TAPE_POSITION_ERROR_AT_END_OF_MEDIUM			= 0x3B02,
+	TAPE_OR_ELECTRONIC_VERTICAL_FORMS_UNIT_NOT_READY	= 0x3B03,
+	SLEW_FAILURE						= 0x3B04,
+	PAPER_JAM						= 0x3B05,
+	FAILED_TO_SENSE_TOP_OF_FORM				= 0x3B06,
+	FAILED_TO_SENSE_BOTTOM_OF_FORM				= 0x3B07,
+	REPOSITION_ERROR					= 0x3B08,
+	READ_PAST_END_OF_MEDIUM					= 0x3B09,
+	READ_PAST_BEGINNING_OF_MEDIUM				= 0x3B0A,
+	POSITION_PAST_END_OF_MEDIUM				= 0x3B0B,
+	POSITION_PAST_BEGINNING_OF_MEDIUM			= 0x3B0C,
+	MEDIUM_DESTINATION_ELEMENT_FULL				= 0x3B0D,
+	MEDIUM_SOURCE_ELEMENT_EMPTY				= 0x3B0E,
+	END_OF_MEDIUM_REACHED					= 0x3B0F,
+	MEDIUM_MAGAZINE_NOT_ACCESSIBLE				= 0x3B11,
+	MEDIUM_MAGAZINE_REMOVED					= 0x3B12,
+	MEDIUM_MAGAZINE_INSERTED				= 0x3B13,
+	MEDIUM_MAGAZINE_LOCKED					= 0x3B14,
+	MEDIUM_MAGAZINE_UNLOCKED				= 0x3B15,
+	MECHANICAL_POSITIONING_OR_CHANGER_ERROR			= 0x3B16,
+	READ_PAST_END_OF_USER_OBJECT				= 0x3B17,
+	ELEMENT_DISABLED					= 0x3B18,
+	ELEMENT_ENABLED						= 0x3B19,
+	DATA_TRANSFER_DEVICE_REMOVED				= 0x3B1A,
+	DATA_TRANSFER_DEVICE_INSERTED				= 0x3B1B,
+	TOO_MANY_LOGICAL_OBJECTS_ON_PARTITION_TO_SUPPORT_OP	= 0x3B1C,
+	ELEMENT_STATIC_INFORMATION_CHANGED			= 0x3B20,
+
+	SCSI_ASC(INVALID_BITS_IN_IDENTIFY_MESSAGE,		0x3D),
+
+	SCSI_ASC(LU_HAS_NOT_SELF_CONFIGURED_YET,		0x3E),
+	LU_FAILURE						= 0x3E01,
+	TIMEOUT_ON_LOGICAL_UNIT					= 0x3E02,
+	LU_FAILED_SELF_TEST					= 0x3E03,
+	LU_UNABLE_TO_UPDATE_SELFTEST_LOG			= 0x3E04,
+
+	SCSI_ASC(TARGET_OPERATING_CONDITIONS_HAVE_CHANGED,	0x3F),
+	MICROCODE_HAS_BEEN_CHANGED				= 0x3F01,
+	CHANGED_OPERATING_DEFINITION				= 0x3F02,
+	INQUIRY_DATA_HAS_CHANGED				= 0x3F03,
+	COMPONENT_DEVICE_ATTACHED				= 0x3F04,
+	DEVICE_IDENTIFIER_CHANGED				= 0x3F05,
+	REDUNDANCY_GROUP_CREATED_OR_MODIFIED			= 0x3F06,
+	REDUNDANCY_GROUP_DELETED				= 0x3F07,
+	SPARE_CREATED_OR_MODIFIED				= 0x3F08,
+	SPARE_DELETED						= 0x3F09,
+	VOLUME_SET_CREATED_OR_MODIFIED				= 0x3F0A,
+	VOLUME_SET_DELETED					= 0x3F0B,
+	VOLUME_SET_DEASSIGNED					= 0x3F0C,
+	VOLUME_SET_REASSIGNED					= 0x3F0D,
+	REPORTED_LUNS_DATA_HAS_CHANGED				= 0x3F0E,
+	ECHO_BUFFER_OVERWRITTEN					= 0x3F0F,
+	MEDIUM_LOADABLE						= 0x3F10,
+	MEDIUM_AUXILIARY_MEMORY_ACCESSIBLE			= 0x3F11,
+	iSCSI_IP_ADDRESS_ADDED					= 0x3F12,
+	iSCSI_IP_ADDRESS_REMOVED				= 0x3F13,
+	iSCSI_IP_ADDRESS_CHANGED				= 0x3F14,
+	INSPECT_REFERRALS_SENSE_DESCRIPTORS			= 0x3F15,
+	MICROCODE_HAS_BEEN_CHANGED_WITHOUT_RESET		= 0x3F16,
+	ZONE_TRANSITION_TO_FULL					= 0x3F17,
+	BIND_COMPLETED						= 0x3F18,
+	BIND_REDIRECTED						= 0x3F19,
+	SUBSIDIARY_BINDING_CHANGED				= 0x3F1A,
+
+	SCSI_ASC(RAM_FAILURE,					0x40),
+	/* DIAGNOSTIC_FAILURE_ON_COMPONENT_NN			= 0x40NN */
+
+	SCSI_ASC(DATA_PATH_FAILURE,				0x41),
+
+	SCSI_ASC(POWER_ON_OR_SELFTEST_FAILURE,			0x42),
+
+	SCSI_ASC(MESSAGE_ERROR,					0x43),
+
+	SCSI_ASC(INTERNAL_TARGET_FAILURE,			0x44),
+	PERSISTENT_RESERVATION_INFORMATION_LOST			= 0x4401,
+	ATA_DEVICE_FAILED_SET_FEATURES				= 0x4471,
+
+	SCSI_ASC(SELECT_OR_RESELECT_FAILURE,			0x45),
+
+	SCSI_ASC(UNSUCCESSFUL_SOFT_RESET,			0x46),
+
+	SCSI_ASC(SCSI_PARITY_ERROR,				0x47),
+	DATA_PHASE_CRC_ERROR_DETECTED				= 0x4701,
+	SCSI_PARITY_ERROR_DETECTED_DURING_ST_DATA_PHASE		= 0x4702,
+	INFORMATION_UNIT_iuCRC_ERROR_DETECTED			= 0x4703,
+	ASYNCHRONOUS_INFORMATION_PROTECTION_ERROR_DETECTED	= 0x4704,
+	PROTOCOL_SERVICE_CRC_ERROR				= 0x4705,
+	PHY_TEST_FUNCTION_IN_PROGRESS				= 0x4706,
+	SOME_COMMANDS_CLEARED_BY_ISCSI_PROTOCOL_EVENT		= 0x477F,
+
+	SCSI_ASC(INITIATOR_DETECTED_ERROR_MESSAGE_RECEIVED,	0x48),
+
+	SCSI_ASC(INVALID_MESSAGE_ERROR,				0x49),
+
+	SCSI_ASC(COMMAND_PHASE_ERROR,				0x4A),
+
+	SCSI_ASC(DATA_PHASE_ERROR,				0x4B),
+	INVALID_TARGET_PORT_TRANSFER_TAG_RECEIVED		= 0x4B01,
+	TOO_MUCH_WRITE_DATA					= 0x4B02,
+	ACK_NAK_TIMEOUT						= 0x4B03,
+	NAK_RECEIVED						= 0x4B04,
+	DATA_OFFSET_ERROR					= 0x4B05,
+	INITIATOR_RESPONSE_TIMEOUT				= 0x4B06,
+	CONNECTION_LOST						= 0x4B07,
+	DATA_IN_BUFFER_OVERFLOW_BUFFER_SIZE			= 0x4B08,
+	DATA_IN_BUFFER_OVERFLOW_BUFFER_DESCRIPTOR_AREA		= 0x4B09,
+	DATA_IN_BUFFER_ERROR					= 0x4B0A,
+	DATA_OUT_BUFFER_OVERFLOW_BUFFER_SIZE			= 0x4B0B,
+	DATA_OUT_BUFFER_OVERFLOW_BUFFER_DESCRIPTOR_AREA		= 0x4B0C,
+	DATA_OUT_BUFFER_ERROR					= 0x4B0D,
+	PCIE_FABRIC_ERROR					= 0x4B0E,
+	PCIE_COMPLETION_TIMEOUT					= 0x4B0F,
+	PCIE_COMPLETER_ABORT					= 0x4B10,
+	PCIE_POISONED_TLP_RECEIVED				= 0x4B11,
+	PCIE_ECRC_CHECK_FAILED					= 0x4B12,
+	PCIE_UNSUPPORTED_REQUEST				= 0x4B13,
+	PCIE_ACS_VIOLATION					= 0x4B14,
+	PCIE_TLP_PREFIX_BLOCKED					= 0x4B15,
+
+	SCSI_ASC(LU_FAILED_SELF_CONFIGURATION,			0x4C),
+
+	SCSI_ASC(TAGGED_OVERLAPPED_COMMANDS,			0x4D),
+
+	SCSI_ASC(OVERLAPPED_COMMANDS_ATTEMPTED,			0x4E),
+
+	SCSI_ASC(WRITE_APPEND_ERROR,				0x50),
+	WRITE_APPEND_POSITION_ERROR				= 0x5001,
+	POSITION_ERROR_RELATED_TO_TIMING			= 0x5002,
+
+	SCSI_ASC(ERASE_FAILURE,					0x51),
+	ERASE_FAILURE_INCOMPLETE_ERASE_OP_DETECTED		= 0x5101,
+
+	SCSI_ASC(CARTRIDGE_FAULT,				0x52),
+
+	SCSI_ASC(MEDIA_LOAD_OR_EJECT_FAILED,			0x53),
+	UNLOAD_TAPE_FAILURE					= 0x5301,
+	MEDIUM_REMOVAL_PREVENTED				= 0x5302,
+	MEDIUM_REMOVAL_PREVENTED_BY_DATA_TRANSFER_ELEMENT	= 0x5303,
+	MEDIUM_THREAD_OR_UNTHREAD_FAILURE			= 0x5304,
+	VOLUME_IDENTIFIER_INVALID				= 0x5305,
+	VOLUME_IDENTIFIER_MISSING				= 0x5306,
+	DUPLICATE_VOLUME_IDENTIFIER				= 0x5307,
+	ELEMENT_STATUS_UNKNOWN					= 0x5308,
+	DATA_TRANSFER_DEVICE_ERR_LOAD_FAILED			= 0x5309,
+	DATA_TRANSFER_DEVICE_ERR_UNLOAD_FAILED			= 0x530A,
+	DATA_TRANSFER_DEVICE_ERR_UNLOAD_MISSING			= 0x530B,
+	DATA_TRANSFER_DEVICE_ERR_EJECT_FAILED			= 0x530C,
+	DATA_TRANSFER_DEVICE_ERR_LIBRARY_COMMUNICATION_FAILED	= 0x530D,
+
+	SCSI_ASC(SCSI_TO_HOST_SYSTEM_INTERFACE_FAILURE,		0x54),
+
+	SCSI_ASC(SYSTEM_RESOURCE_FAILURE,			0x55),
+	SYSTEM_BUFFER_FULL					= 0x5501,
+	INSUFFICIENT_RESERVATION_RESOURCES			= 0x5502,
+	INSUFFICIENT_RESOURCES					= 0x5503,
+	INSUFFICIENT_REGISTRATION_RESOURCES			= 0x5504,
+	INSUFFICIENT_ACCESS_CONTROL_RESOURCES			= 0x5505,
+	AUXILIARY_MEMORY_OUT_OF_SPACE				= 0x5506,
+	QUOTA_ERROR						= 0x5507,
+	MAXIMUM_NUMBER_OF_SUPPLEMENTAL_DECRYPTION_KEYS_EXCEEDED	= 0x5508,
+	MEDIUM_AUXILIARY_MEMORY_NOT_ACCESSIBLE			= 0x5509,
+	DATA_CURRENTLY_UNAVAILABLE				= 0x550A,
+	INSUFFICIENT_POWER_FOR_OPERATION			= 0x550B,
+	INSUFFICIENT_RESOURCES_TO_CREATE_ROD			= 0x550C,
+	INSUFFICIENT_RESOURCES_TO_CREATE_ROD_TOKEN		= 0x550D,
+	INSUFFICIENT_ZONE_RESOURCES				= 0x550E,
+	INSUFFICIENT_ZONE_RESOURCES_TO_COMPLETE_WRITE		= 0x550F,
+	MAXIMUM_NUMBER_OF_STREAMS_OPEN				= 0x5510,
+	INSUFFICIENT_RESOURCES_TO_BIND				= 0x5511,
+
+	SCSI_ASC(UNABLE_TO_RECOVER_TABLE_OF_CONTENTS,		0x57),
+
+	SCSI_ASC(GENERATION_DOES_NOT_EXIST,			0x58),
+
+	SCSI_ASC(UPDATED_BLOCK_READ,				0x59),
+
+	SCSI_ASC(OPERATOR_REQUEST_OR_STATE_CHANGE_INPUT,	0x5A),
+	OPERATOR_MEDIUM_REMOVAL_REQUEST				= 0x5A01,
+	OPERATOR_SELECTED_WRITE_PROTECT				= 0x5A02,
+	OPERATOR_SELECTED_WRITE_PERMIT				= 0x5A03,
+
+	SCSI_ASC(LOG_EXCEPTION,					0x5B),
+	THRESHOLD_CONDITION_MET					= 0x5B01,
+	LOG_COUNTER_AT_MAXIMUM					= 0x5B02,
+	LOG_LIST_CODES_EXHAUSTED				= 0x5B03,
+
+	SCSI_ASC(RPL_STATUS_CHANGE,				0x5C),
+	SPINDLES_SYNCHRONIZED					= 0x5C01,
+	SPINDLES_NOT_SYNCHRONIZED				= 0x5C02,
+
+	SCSI_ASC(FAILURE_PREDICTION_THRESHOLD_EXCEEDED,		0x5D),
+	MEDIA_FAILURE_PREDICTION_THRESHOLD_EXCEEDED		= 0x5D01,
+	LU_FAILURE_PREDICTION_THRESHOLD_EXCEEDED		= 0x5D02,
+	SPARE_AREA_EXHAUSTION_PREDICTION_THRESHOLD_EXCEEDED	= 0x5D03,
+	HW_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE		= 0x5D10,
+	HW_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH		= 0x5D11,
+	HW_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH		= 0x5D12,
+	HW_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH		= 0x5D13,
+	HW_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS		= 0x5D14,
+	HW_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH		= 0x5D15,
+	HW_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH		= 0x5D16,
+	HW_IMPENDING_FAILURE_CHANNEL_PARAMETRICS		= 0x5D17,
+	HW_IMPENDING_FAILURE_CONTROLLER_DETECTED		= 0x5D18,
+	HW_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE		= 0x5D19,
+	HW_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE		= 0x5D1A,
+	HW_IMPENDING_FAILURE_SPINUP_RETRY_COUNT			= 0x5D1B,
+	HW_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT	= 0x5D1C,
+	HW_IMPENDING_FAILURE_POWER_LOSS_PROTECTION_CIRCUIT	= 0x5D1D,
+	CONTROLLER_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE	= 0x5D20,
+	CONTROLLER_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH	= 0x5D21,
+	CONTROLLER_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH	= 0x5D22,
+	CONTROLLER_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH	= 0x5D23,
+	CONTROLLER_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS	= 0x5D24,
+	CONTROLLER_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH	= 0x5D25,
+	CONTROLLER_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH	= 0x5D26,
+	CONTROLLER_IMPENDING_FAILURE_CHANNEL_PARAMETRICS	= 0x5D27,
+	CONTROLLER_IMPENDING_FAILURE_CONTROLLER_DETECTED	= 0x5D28,
+	CONTROLLER_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE	= 0x5D29,
+	CONTROLLER_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE	= 0x5D2A,
+	CONTROLLER_IMPENDING_FAILURE_SPINUP_RETRY_COUNT		= 0x5D2B,
+	CONTROLLER_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT = 0x5D2C,
+	DATA_CHANNEL_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE = 0x5D30,
+	DATA_CHANNEL_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH = 0x5D31,
+	DATA_CHANNEL_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH	= 0x5D32,
+	DATA_CHANNEL_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH	= 0x5D33,
+	DATA_CHANNEL_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS	= 0x5D34,
+	DATA_CHANNEL_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH	= 0x5D35,
+	DATA_CHANNEL_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH = 0x5D36,
+	DATA_CHANNEL_IMPENDING_FAILURE_CHANNEL_PARAMETRICS	= 0x5D37,
+	DATA_CHANNEL_IMPENDING_FAILURE_CONTROLLER_DETECTED	= 0x5D38,
+	DATA_CHANNEL_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE	= 0x5D39,
+	DATA_CHANNEL_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE	= 0x5D3A,
+	DATA_CHANNEL_IMPENDING_FAILURE_SPINUP_RETRY_COUNT	= 0x5D3B,
+	DATA_CHANNEL_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT = 0x5D3C,
+	SERVO_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE	= 0x5D40,
+	SERVO_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH	= 0x5D41,
+	SERVO_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH	= 0x5D42,
+	SERVO_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH	= 0x5D43,
+	SERVO_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS	= 0x5D44,
+	SERVO_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH		= 0x5D45,
+	SERVO_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH	= 0x5D46,
+	SERVO_IMPENDING_FAILURE_CHANNEL_PARAMETRICS		= 0x5D47,
+	SERVO_IMPENDING_FAILURE_CONTROLLER_DETECTED		= 0x5D48,
+	SERVO_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE		= 0x5D49,
+	SERVO_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE		= 0x5D4A,
+	SERVO_IMPENDING_FAILURE_SPINUP_RETRY_COUNT		= 0x5D4B,
+	SERVO_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT	= 0x5D4C,
+	SPINDLE_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE	= 0x5D50,
+	SPINDLE_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH	= 0x5D51,
+	SPINDLE_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH	= 0x5D52,
+	SPINDLE_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH	= 0x5D53,
+	SPINDLE_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS	= 0x5D54,
+	SPINDLE_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH		= 0x5D55,
+	SPINDLE_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH	= 0x5D56,
+	SPINDLE_IMPENDING_FAILURE_CHANNEL_PARAMETRICS		= 0x5D57,
+	SPINDLE_IMPENDING_FAILURE_CONTROLLER_DETECTED		= 0x5D58,
+	SPINDLE_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE	= 0x5D59,
+	SPINDLE_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE		= 0x5D5A,
+	SPINDLE_IMPENDING_FAILURE_SPINUP_RETRY_COUNT		= 0x5D5B,
+	SPINDLE_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT	= 0x5D5C,
+	FW_IMPENDING_FAILURE_GENERAL_HARD_DRIVE_FAILURE		= 0x5D60,
+	FW_IMPENDING_FAILURE_DRIVE_ERROR_RATE_TOO_HIGH		= 0x5D61,
+	FW_IMPENDING_FAILURE_DATA_ERROR_RATE_TOO_HIGH		= 0x5D62,
+	FW_IMPENDING_FAILURE_SEEK_ERROR_RATE_TOO_HIGH		= 0x5D63,
+	FW_IMPENDING_FAILURE_TOO_MANY_BLOCK_REASSIGNS		= 0x5D64,
+	FW_IMPENDING_FAILURE_ACCESS_TIMES_TOO_HIGH		= 0x5D65,
+	FW_IMPENDING_FAILURE_START_UNIT_TIMES_TOO_HIGH		= 0x5D66,
+	FW_IMPENDING_FAILURE_CHANNEL_PARAMETRICS		= 0x5D67,
+	FW_IMPENDING_FAILURE_CONTROLLER_DETECTED		= 0x5D68,
+	FW_IMPENDING_FAILURE_THROUGHPUT_PERFORMANCE		= 0x5D69,
+	FW_IMPENDING_FAILURE_SEEK_TIME_PERFORMANCE		= 0x5D6A,
+	FW_IMPENDING_FAILURE_SPINUP_RETRY_COUNT			= 0x5D6B,
+	FW_IMPENDING_FAILURE_DRIVE_CALIBRATION_RETRY_COUNT	= 0x5D6C,
+	MEDIA_IMPENDING_FAILURE_ENDURANCE_LIMIT_MET		= 0x5D73,
+	FAILURE_PREDICTION_THRESHOLD_EXCEEDED_FALSE		= 0x5DFF,
+
+	SCSI_ASC(LOW_POWER_CONDITION_ON,			0x5E),
+	IDLE_CONDITION_ACTIVATED_BY_TIMER			= 0x5E01,
+	STANDBY_CONDITION_ACTIVATED_BY_TIMER			= 0x5E02,
+	IDLE_CONDITION_ACTIVATED_BY_COMMAND			= 0x5E03,
+	STANDBY_CONDITION_ACTIVATED_BY_COMMAND			= 0x5E04,
+	IDLE_B_CONDITION_ACTIVATED_BY_TIMER			= 0x5E05,
+	IDLE_B_CONDITION_ACTIVATED_BY_COMMAND			= 0x5E06,
+	IDLE_C_CONDITION_ACTIVATED_BY_TIMER			= 0x5E07,
+	IDLE_C_CONDITION_ACTIVATED_BY_COMMAND			= 0x5E08,
+	STANDBY_Y_CONDITION_ACTIVATED_BY_TIMER			= 0x5E09,
+	STANDBY_Y_CONDITION_ACTIVATED_BY_COMMAND		= 0x5E0A,
+	POWER_STATE_CHANGE_TO_ACTIVE				= 0x5E41,
+	POWER_STATE_CHANGE_TO_IDLE				= 0x5E42,
+	POWER_STATE_CHANGE_TO_STANDBY				= 0x5E43,
+	POWER_STATE_CHANGE_TO_SLEEP				= 0x5E45,
+	POWER_STATE_CHANGE_TO_DEVICE_CONTROL			= 0x5E47,
+
+	SCSI_ASC(LAMP_FAILURE,					0x60),
+
+	SCSI_ASC(VIDEO_ACQUISITION_ERROR,			0x61),
+	UNABLE_TO_ACQUIRE_VIDEO					= 0x6101,
+	OUT_OF_FOCUS						= 0x6102,
+
+	SCSI_ASC(SCAN_HEAD_POSITIONING_ERROR,			0x62),
+
+	SCSI_ASC(END_OF_USER_AREA_ENCOUNTERED_ON_THIS_TRACK,	0x63),
+	PACKET_DOES_NOT_FIT_IN_AVAILABLE_SPACE			= 0x6301,
+
+	SCSI_ASC(ILLEGAL_MODE_FOR_THIS_TRACK,			0x64),
+	INVALID_PACKET_SIZE					= 0x6401,
+
+	SCSI_ASC(VOLTAGE_FAULT,					0x65),
+
+	SCSI_ASC(AUTOMATIC_DOCUMENT_FEEDER_COVER_UP,		0x66),
+	AUTOMATIC_DOCUMENT_FEEDER_LIFT_UP			= 0x6601,
+	DOCUMENT_JAM_IN_AUTOMATIC_DOCUMENT_FEEDER		= 0x6602,
+	DOCUMENT_MISS_FEED_AUTOMATIC_IN_DOCUMENT_FEEDER		= 0x6603,
+
+	SCSI_ASC(CONFIG_FAILURE,				0x67),
+	CONFIG_OF_INCAPABLE_LOGICAL_UNITS_FAILED		= 0x6701,
+	ADD_LU_FAILED						= 0x6702,
+	MODIFICATION_OF_LU_FAILED				= 0x6703,
+	EXCHANGE_OF_LU_FAILED					= 0x6704,
+	REMOVE_OF_LU_FAILED					= 0x6705,
+	ATTACHMENT_OF_LU_FAILED					= 0x6706,
+	CREATION_OF_LU_FAILED					= 0x6707,
+	ASSIGN_FAILURE_OCCURRED					= 0x6708,
+	MULTIPLY_ASSIGNED_LOGICAL_UNIT				= 0x6709,
+	SET_TARGET_PORT_GROUPS_COMMAND_FAILED			= 0x670A,
+	ATA_DEVICE_FEATURE_NOT_ENABLED				= 0x670B,
+	COMMAND_REJECTED					= 0x670C,
+	EXPLICIT_BIND_NOT_ALLOWED				= 0x670D,
+	FEATURE_NOT_ENABLED					= 0x670E,
+
+	SCSI_ASC(LU_NOT_CONFIGURED,				0x68),
+	SUBSIDIARY_LU_NOT_CONFIGURED				= 0x6801,
+
+	SCSI_ASC(DATA_LOSS_ON_LOGICAL_UNIT,			0x69),
+	MULTIPLE_LU_FAILURES					= 0x6901,
+	PARITY_DATA_MISMATCH					= 0x6902,
+
+	SCSI_ASC(INFORMATIONAL_REFER_TO_LOG,			0x6A),
+
+	SCSI_ASC(STATE_CHANGE_HAS_OCCURRED,			0x6B),
+	REDUNDANCY_LEVEL_GOT_BETTER				= 0x6B01,
+	REDUNDANCY_LEVEL_GOT_WORSE				= 0x6B02,
+
+	SCSI_ASC(REBUILD_FAILURE_OCCURRED,			0x6C),
+
+	SCSI_ASC(RECALCULATE_FAILURE_OCCURRED,			0x6D),
+
+	SCSI_ASC(COMMAND_TO_LU_FAILED,				0x6E),
+
+	SCSI_ASC(COPY_PROTECTION_KEY_EXCHANGE_FAILURE_AUTHENTICATION_FAILURE, 0x6F),
+	COPY_PROTECTION_KEY_EXCHANGE_FAILURE_KEY_NOT_PRESENT	= 0x6F01,
+	COPY_PROTECTION_KEY_EXCHANGE_FAILURE_KEY_NOT_ESTABLISHED = 0x6F02,
+	READ_OF_SCRAMBLED_SECTOR_WITHOUT_AUTHENTICATION		= 0x6F03,
+	MEDIA_REGION_CODE_IS_MISMATCHED_TO_LU_REGION		= 0x6F04,
+	DRIVE_REGION_MUST_BE_PERMANENT_REGION_RESET_COUNT_ERROR	= 0x6F05,
+	INSUFFICIENT_BLOCK_COUNT_FOR_BINDING_NONCE_RECORDING	= 0x6F06,
+	CONFLICT_IN_BINDING_NONCE_RECORDING			= 0x6F07,
+	INSUFFICIENT_PERMISSION					= 0x6F08,
+	INVALID_DRIVE_HOST_PAIRING_SERVER			= 0x6F09,
+	DRIVE_HOST_PAIRING_SUSPENDED				= 0x6F0A,
+
+	SCSI_ASC(DECOMPRESSION_EXCEPTION_SHORT_ALGORITHM_ID,	0x70),
+
+	SCSI_ASC(DECOMPRESSION_EXCEPTION_LONG_ALGORITHM_ID,	0x71),
+
+	SCSI_ASC(SESSION_FIXATION_ERROR,			0x72),
+	SESSION_FIXATION_ERROR_WRITING_LEAD_IN			= 0x7201,
+	SESSION_FIXATION_ERROR_WRITING_LEAD_OUT			= 0x7202,
+	SESSION_FIXATION_ERROR_INCOMPLETE_TRACK_IN_SESSION	= 0x7203,
+	EMPTY_OR_PARTIALLY_WRITTEN_RESERVED_TRACK		= 0x7204,
+	NO_MORE_TRACK_RESERVATIONS_ALLOWED			= 0x7205,
+	RMZ_EXTENSION_IS_NOT_ALLOWED				= 0x7206,
+	NO_MORE_TEST_ZONE_EXTENSIONS_ARE_ALLOWED		= 0x7207,
+
+	SCSI_ASC(CD_CONTROL_ERROR,				0x73),
+	POWER_CALIBRATION_AREA_ALMOST_FULL			= 0x7301,
+	POWER_CALIBRATION_AREA_IS_FULL				= 0x7302,
+	POWER_CALIBRATION_AREA_ERROR				= 0x7303,
+	PROGRAM_MEMORY_AREA_UPDATE_FAILURE			= 0x7304,
+	PROGRAM_MEMORY_AREA_IS_FULL				= 0x7305,
+	RMA_PMA_IS_ALMOST_FULL					= 0x7306,
+	CURRENT_POWER_CALIBRATION_AREA_ALMOST_FULL		= 0x7310,
+	CURRENT_POWER_CALIBRATION_AREA_IS_FULL			= 0x7311,
+	RDZ_IS_FULL						= 0x7317,
+
+	SCSI_ASC(SECURITY_ERROR,				0x74),
+	UNABLE_TO_DECRYPT_DATA					= 0x7401,
+	UNENCRYPTED_DATA_ENCOUNTERED_WHILE_DECRYPTING		= 0x7402,
+	INCORRECT_DATA_ENCRYPTION_KEY				= 0x7403,
+	CRYPTOGRAPHIC_INTEGRITY_VALIDATION_FAILED		= 0x7404,
+	ERROR_DECRYPTING_DATA					= 0x7405,
+	UNKNOWN_SIGNATURE_VERIFICATION_KEY			= 0x7406,
+	ENCRYPTION_PARAMS_NOT_USEABLE				= 0x7407,
+	DIGITAL_SIGNATURE_VALIDATION_FAILURE			= 0x7408,
+	ENCRYPTION_MODE_MISMATCH_ON_READ			= 0x7409,
+	ENCRYPTED_BLOCK_NOT_RAW_READ_ENABLED			= 0x740A,
+	INCORRECT_ENCRYPTION_PARAMS				= 0x740B,
+	UNABLE_TO_DECRYPT_PARAMETER_LIST			= 0x740C,
+	ENCRYPTION_ALGORITHM_DISABLED				= 0x740D,
+	SA_CREATION_PARAMETER_VALUE_INVALID			= 0x7410,
+	SA_CREATION_PARAMETER_VALUE_REJECTED			= 0x7411,
+	INVALID_SA_USAGE					= 0x7412,
+	DATA_ENCRYPTION_CONFIG_PREVENTED			= 0x7421,
+	SA_CREATION_PARAMETER_NOT_SUPPORTED			= 0x7430,
+	AUTHENTICATION_FAILED					= 0x7440,
+	EXTERNAL_DATA_ENCRYPTION_KEY_MANAGER_ACCESS_ERROR	= 0x7461,
+	EXTERNAL_DATA_ENCRYPTION_KEY_MANAGER_ERROR		= 0x7462,
+	EXTERNAL_DATA_ENCRYPTION_KEY_NOT_FOUND			= 0x7463,
+	EXTERNAL_DATA_ENCRYPTION_REQUEST_NOT_AUTHORIZED		= 0x7464,
+	EXTERNAL_DATA_ENCRYPTION_CONTROL_TIMEOUT		= 0x746E,
+	EXTERNAL_DATA_ENCRYPTION_CONTROL_ERROR			= 0x746F,
+	LU_ACCESS_NOT_AUTHORIZED				= 0x7471,
+	SECURITY_CONFLICT_IN_TRANSLATED_DEVICE			= 0x7479,
+};
+
+#undef SCSI_ASC_DEF
+#undef SCSI_ASC
+
+/*
+ * Use this to build non-standard sense codes.
+ * Note: this is defined as a macro instead of an inline function so that it can
+ * be used to define SCSI failures array variables (see struct scsi_failure).
+ */
+#define scsi_sense_code(asc, ascq)	\
+	((((asc) & 0xff) << 8) | ((ascq) & 0xff))
+
+static inline u8 scsi_sense_code_asc(u16 code)
+{
+	return (code >> 8) & 0xff;
+}
+
+static inline u8 scsi_sense_code_ascq(u16 code)
+{
+	return code & 0xff;
+}
+
+#endif /* _SCSI_SENSE_H_ */
diff --git a/include/soc/fsl/qe/qe.h b/include/soc/fsl/qe/qe.h
index 8f967d15e47979..7f0e4165c3b0ca 100644
--- a/include/soc/fsl/qe/qe.h
+++ b/include/soc/fsl/qe/qe.h
@@ -24,6 +24,7 @@
 #include <linux/types.h>
 
 struct device;
+struct gpio_desc;
 
 #define QE_NUM_OF_SNUM	256	/* There are 256 serial number in QE */
 #define QE_NUM_OF_BRGS	16
@@ -193,12 +194,13 @@ static inline int par_io_data_set(u8 port, u8 pin, u8 val) { return -ENOSYS; }
  */
 struct qe_pin;
 #ifdef CONFIG_QE_GPIO
-extern struct qe_pin *qe_pin_request(struct device *dev, int index);
+extern struct qe_pin *qe_pin_request(struct device *dev, struct gpio_desc *gpiod);
 extern void qe_pin_free(struct qe_pin *qe_pin);
 extern void qe_pin_set_gpio(struct qe_pin *qe_pin);
 extern void qe_pin_set_dedicated(struct qe_pin *pin);
 #else
-static inline struct qe_pin *qe_pin_request(struct device *dev, int index)
+static inline struct qe_pin *
+qe_pin_request(struct device *dev, struct gpio_desc *gpiod)
 {
 	return ERR_PTR(-ENOSYS);
 }
diff --git a/include/soc/imx/cpu.h b/include/soc/imx/cpu.h
index 0bf610acafd061..8c53150acd76eb 100644
--- a/include/soc/imx/cpu.h
+++ b/include/soc/imx/cpu.h
@@ -30,7 +30,7 @@
 #define MXC_CPU_VF600		0x600
 #define MXC_CPU_VF610		(MXC_CPU_VF600 | MXC_CPU_VFx10)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 extern unsigned int __mxc_cpu_type;
 #endif
 
diff --git a/include/sound/control.h b/include/sound/control.h
index 909db0d0485df1..37e9c545503e5f 100644
--- a/include/sound/control.h
+++ b/include/sound/control.h
@@ -244,7 +244,7 @@ int _snd_ctl_add_follower(struct snd_kcontrol *master,
  * @follower: follower element to add
  *
  * Add a virtual follower control to the given master element created via
- * snd_ctl_create_virtual_master() beforehand.
+ * snd_ctl_make_virtual_master() beforehand.
  *
  * All followers must be the same type (returning the same information
  * via info callback).  The function doesn't check it, so it's your
diff --git a/include/sound/cs35l41.h b/include/sound/cs35l41.h
index 7542cabfa7266e..c97db581650229 100644
--- a/include/sound/cs35l41.h
+++ b/include/sound/cs35l41.h
@@ -11,6 +11,7 @@
 #define __CS35L41_H
 
 #include <linux/regmap.h>
+#include <linux/interrupt.h>
 #include <linux/firmware/cirrus/cs_dsp.h>
 
 #define CS35L41_FIRSTREG		0x00000000
diff --git a/include/sound/emu10k1.h b/include/sound/emu10k1.h
index 4f94565c9d15af..6474239a11c1e7 100644
--- a/include/sound/emu10k1.h
+++ b/include/sound/emu10k1.h
@@ -633,8 +633,8 @@ SUB_REG(PEFE, FILTERAMOUNT,	0x000000ff)	/* Filter envlope amount				*/
 #define A_ADCCR_RCHANENABLE	0x00000020
 #define A_ADCCR_LCHANENABLE	0x00000010
 
-#define A_ADCCR_SAMPLERATE_MASK 0x0000000F      /* Audigy sample rate convertor output rate		*/
-#define ADCCR_SAMPLERATE_MASK	0x00000007	/* Sample rate convertor output rate			*/
+#define A_ADCCR_SAMPLERATE_MASK 0x0000000F      /* Audigy sample rate converter output rate		*/
+#define ADCCR_SAMPLERATE_MASK	0x00000007	/* Sample rate converter output rate			*/
 #define ADCCR_SAMPLERATE_48	0x00000000	/* 48kHz sample rate					*/
 #define ADCCR_SAMPLERATE_44	0x00000001	/* 44.1kHz sample rate					*/
 #define ADCCR_SAMPLERATE_32	0x00000002	/* 32kHz sample rate					*/
diff --git a/include/sound/emux_legacy.h b/include/sound/emux_legacy.h
index 1127e30d33bbf3..55a7fbfb9bfa1a 100644
--- a/include/sound/emux_legacy.h
+++ b/include/sound/emux_legacy.h
@@ -5,7 +5,7 @@
 /*
  *  Copyright (c) 1999-2000 Takashi Iwai <tiwai@suse.de>
  *
- *  Definitions of OSS compatible headers for Emu8000 device informations
+ *  Definitions of OSS compatible headers for Emu8000 device information
  */
 
 #include <sound/seq_oss_legacy.h>
diff --git a/include/sound/omap-hdmi-audio.h b/include/sound/omap-hdmi-audio.h
index e5f82044a40496..8dca92360ee033 100644
--- a/include/sound/omap-hdmi-audio.h
+++ b/include/sound/omap-hdmi-audio.h
@@ -34,6 +34,7 @@ struct omap_hdmi_audio_pdata {
 	phys_addr_t audio_dma_addr;
 
 	const struct omap_hdmi_audio_ops *ops;
+	void (*audio_hpd)(struct device *dev, bool connected);
 };
 
 #endif /* __OMAP_HDMI_AUDIO_H__ */
diff --git a/include/sound/rawmidi.h b/include/sound/rawmidi.h
index 4154035af41420..d812f70725bdd0 100644
--- a/include/sound/rawmidi.h
+++ b/include/sound/rawmidi.h
@@ -86,6 +86,7 @@ struct snd_rawmidi_substream {
 	bool opened;			/* open flag */
 	bool append;			/* append flag (merge more streams) */
 	bool active_sensing;		/* send active sensing when close */
+	bool disconnected;		/* already disconnected */
 	unsigned int framing;		/* whether to frame input data */
 	unsigned int clock_type;	/* clock source to use for input framing */
 	int use_count;			/* use counter (for output) */
diff --git a/include/sound/sdca.h b/include/sound/sdca.h
index 2bdf4e333e0449..db2672b3aae27a 100644
--- a/include/sound/sdca.h
+++ b/include/sound/sdca.h
@@ -60,20 +60,16 @@ enum sdca_quirk {
 	SDCA_QUIRKS_SKIP_FUNC_TYPE_PATCHING,
 };
 
-#if IS_ENABLED(CONFIG_ACPI) && IS_ENABLED(CONFIG_SND_SOC_SDCA)
+#if IS_ENABLED(CONFIG_SND_SOC_SDCA)
 
-void sdca_lookup_functions(struct sdw_slave *slave);
 void sdca_lookup_swft(struct sdw_slave *slave);
-void sdca_lookup_interface_revision(struct sdw_slave *slave);
 bool sdca_device_quirk_match(struct sdw_slave *slave, enum sdca_quirk quirk);
 int sdca_dev_register_functions(struct sdw_slave *slave);
 void sdca_dev_unregister_functions(struct sdw_slave *slave);
 
 #else
 
-static inline void sdca_lookup_functions(struct sdw_slave *slave) {}
 static inline void sdca_lookup_swft(struct sdw_slave *slave) {}
-static inline void sdca_lookup_interface_revision(struct sdw_slave *slave) {}
 static inline bool sdca_device_quirk_match(struct sdw_slave *slave, enum sdca_quirk quirk)
 {
 	return false;
@@ -88,4 +84,21 @@ static inline void sdca_dev_unregister_functions(struct sdw_slave *slave) {}
 
 #endif
 
+/*
+ * Called from the SoundWire bus during peripheral enumeration; gated on
+ * ACPI to avoid a soundwire_bus <-> snd_soc_sdca module cycle on DT builds
+ * (where the bodies are stubs anyway).
+ */
+#if IS_ENABLED(CONFIG_ACPI) && IS_ENABLED(CONFIG_SND_SOC_SDCA)
+
+void sdca_lookup_functions(struct sdw_slave *slave);
+void sdca_lookup_interface_revision(struct sdw_slave *slave);
+
+#else
+
+static inline void sdca_lookup_functions(struct sdw_slave *slave) {}
+static inline void sdca_lookup_interface_revision(struct sdw_slave *slave) {}
+
+#endif
+
 #endif
diff --git a/include/sound/sdca_class.h b/include/sound/sdca_class.h
new file mode 100644
index 00000000000000..794a756472cd62
--- /dev/null
+++ b/include/sound/sdca_class.h
@@ -0,0 +1,70 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * The MIPI SDCA specification is available for public downloads at
+ * https://www.mipi.org/mipi-sdca-v1-0-download
+ *
+ * Copyright (C) 2025 Cirrus Logic, Inc. and
+ *                    Cirrus Logic International Semiconductor Ltd.
+ */
+
+#ifndef __SDCA_CLASS_H__
+#define __SDCA_CLASS_H__
+
+#include <linux/completion.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+
+struct device;
+struct regmap;
+struct sdw_slave;
+struct sdca_function_data;
+
+/**
+ * struct sdca_class_ops - optional codec-provided class callbacks
+ * @populate_function: fill @function (entities, clusters, init_table, ...)
+ *           from static tables in place of sdca_parse_function() on
+ *           DT/non-DisCo platforms.  Called once per SDCA function with
+ *           @dev pointing at that function's device.  Pure data source;
+ *           performs no bus I/O.  Return 0 on success or a negative errno.
+ */
+struct sdca_class_ops {
+	int (*populate_function)(struct device *dev,
+				 struct sdca_function_data *function);
+};
+
+struct sdca_class_drv {
+	struct device *dev;
+	struct regmap *dev_regmap;
+	struct sdw_slave *sdw;
+
+	struct sdca_interrupt_info *irq_info;
+
+	const struct sdca_class_ops *ops;
+
+	struct mutex regmap_lock;
+	/* Serialise function initialisations */
+	struct mutex init_lock;
+	struct work_struct boot_work;
+};
+
+/* Library helpers used by codec-specific SDCA SoundWire drivers. */
+int sdca_class_probe(struct sdw_slave *sdw,
+		     struct sdca_class_drv *drv,
+		     const struct sdca_class_ops *ops);
+void sdca_class_remove(struct sdca_class_drv *drv);
+
+/*
+ * PM helpers.  Codec drivers embed sdca_class_drv in their own priv,
+ * own dev_set_drvdata(), and compose these into their own dev_pm_ops:
+ *
+ *	static int wcd_runtime_suspend(struct device *dev) {
+ *		struct wcd_priv *priv = dev_get_drvdata(dev);
+ *		return sdca_class_runtime_suspend(&priv->class);
+ *	}
+ */
+int sdca_class_runtime_suspend(struct sdca_class_drv *drv);
+int sdca_class_runtime_resume(struct sdca_class_drv *drv);
+int sdca_class_system_suspend(struct sdca_class_drv *drv);
+int sdca_class_system_resume(struct sdca_class_drv *drv);
+
+#endif /* __SDCA_CLASS_H__ */
diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h
index bd8c3a03357733..68faec56970e99 100644
--- a/include/sound/simple_card_utils.h
+++ b/include/sound/simple_card_utils.h
@@ -219,12 +219,14 @@ static inline int simple_util_parse_aux_devs(struct simple_util_priv *priv, char
 int simple_util_init_jack(struct snd_soc_card *card,
 			       struct simple_util_jack *sjack,
 			       int is_hp, char *prefix, char *pin);
+void simple_util_remove_jack(struct simple_util_jack *sjack);
 int simple_util_init_aux_jacks(struct snd_soc_card *card, char *prefix);
 int simple_util_init_priv(struct simple_util_priv *priv,
 			       struct link_info *li);
 void simple_util_remove(struct platform_device *pdev);
 
 int graph_util_card_probe(struct snd_soc_card *card);
+int graph_util_card_remove(struct snd_soc_card *card);
 int graph_util_is_ports0(struct device_node *port);
 int graph_util_parse_dai(struct simple_util_priv *priv, struct device_node *ep,
 			 struct snd_soc_dai_link_component *dlc, int *is_single_link);
diff --git a/include/sound/soc-component.h b/include/sound/soc-component.h
index c49b59630101fa..6c1acc984ecb06 100644
--- a/include/sound/soc-component.h
+++ b/include/sound/soc-component.h
@@ -258,10 +258,16 @@ struct snd_soc_component {
 	void *priv;
 };
 
-#define for_each_component_dais(component, dai)\
-	list_for_each_entry(dai, &(component)->dai_list, list)
-#define for_each_component_dais_safe(component, dai, _dai)\
-	list_for_each_entry_safe(dai, _dai, &(component)->dai_list, list)
+#define for_each_component_dais(component, dai)				\
+	for (dai = snd_soc_dai_from_list((component)->dai_list.next);	\
+	     snd_soc_dai_to_list(dai) != &(component)->dai_list;	\
+	     dai = snd_soc_dai_from_list(snd_soc_dai_to_list(dai)->next))
+
+#define for_each_component_dais_safe(component, dai, _dai)			\
+	for (dai = snd_soc_dai_from_list((component)->dai_list.next),		\
+	     _dai = snd_soc_dai_from_list(snd_soc_dai_to_list(dai)->next);	\
+	     snd_soc_dai_to_list(dai) != &(component)->dai_list;		\
+	     dai = _dai, _dai = snd_soc_dai_from_list(snd_soc_dai_to_list(_dai)->next))
 
 /**
  * snd_soc_component_to_dapm() - Returns the DAPM context associated with a
diff --git a/include/sound/soc-dai.h b/include/sound/soc-dai.h
index ba3ae56c6b0620..c044909ef10d81 100644
--- a/include/sound/soc-dai.h
+++ b/include/sound/soc-dai.h
@@ -212,8 +212,11 @@ int snd_soc_dai_get_channel_map(const struct snd_soc_dai *dai,
 		unsigned int *tx_num, unsigned int *tx_slot,
 		unsigned int *rx_num, unsigned int *rx_slot);
 
+struct snd_soc_component *snd_soc_dai_to_component(const struct snd_soc_dai *dai);
+struct snd_soc_dai_driver *snd_soc_dai_to_driver(const struct snd_soc_dai *dai);
 int snd_soc_dai_is_dummy(const struct snd_soc_dai *dai);
-
+int snd_soc_dai_add_controls(struct snd_soc_dai *dai,
+			     const struct snd_kcontrol_new *controls, int num_controls);
 int snd_soc_dai_hw_params(struct snd_soc_dai *dai,
 			  struct snd_pcm_substream *substream,
 			  struct snd_pcm_hw_params *params);
@@ -228,18 +231,7 @@ void snd_soc_dai_suspend(struct snd_soc_dai *dai);
 void snd_soc_dai_resume(struct snd_soc_dai *dai);
 int snd_soc_dai_compress_new(struct snd_soc_dai *dai, struct snd_soc_pcm_runtime *rtd);
 bool snd_soc_dai_stream_valid(const struct snd_soc_dai *dai, int stream);
-void snd_soc_dai_action(struct snd_soc_dai *dai,
-			int stream, int action);
-static inline void snd_soc_dai_activate(struct snd_soc_dai *dai,
-					int stream)
-{
-	snd_soc_dai_action(dai, stream,  1);
-}
-static inline void snd_soc_dai_deactivate(struct snd_soc_dai *dai,
-					  int stream)
-{
-	snd_soc_dai_action(dai, stream, -1);
-}
+
 int snd_soc_dai_active(const struct snd_soc_dai *dai);
 
 int snd_soc_pcm_dai_probe(struct snd_soc_pcm_runtime *rtd, int order);
@@ -277,7 +269,12 @@ int snd_soc_dai_compr_get_metadata(struct snd_soc_dai *dai,
 				   struct snd_compr_stream *cstream,
 				   struct snd_compr_metadata *metadata);
 
-const char *snd_soc_dai_name_get(const struct snd_soc_dai *dai);
+int snd_soc_dai_matches_args(const struct snd_soc_dai *dai,
+			     const struct of_phandle_args *args2);
+int snd_soc_dai_matches_dlc(struct snd_soc_dai *dai,
+			    const struct snd_soc_dai_link_component *dlc);
+const char *snd_soc_dai_name(const struct snd_soc_dai *dai);
+int snd_soc_dai_id(const struct snd_soc_dai *dai);
 
 struct snd_soc_dai_ops {
 	/* DAI driver callbacks */
@@ -489,69 +486,51 @@ struct snd_soc_dai {
 	void *priv;
 };
 
+const struct snd_soc_pcm_stream *
+snd_soc_dai_pcm_stream_get_i(const struct snd_soc_dai *dai, int stream);
 static inline const struct snd_soc_pcm_stream *
-snd_soc_dai_get_pcm_stream(const struct snd_soc_dai *dai, int stream)
-{
-	return (stream == SNDRV_PCM_STREAM_PLAYBACK) ?
-		&dai->driver->playback : &dai->driver->capture;
-}
-
-#define snd_soc_dai_get_widget_playback(dai)	snd_soc_dai_get_widget(dai, SNDRV_PCM_STREAM_PLAYBACK)
-#define snd_soc_dai_get_widget_capture(dai)	snd_soc_dai_get_widget(dai, SNDRV_PCM_STREAM_CAPTURE)
-static inline
-struct snd_soc_dapm_widget *snd_soc_dai_get_widget(struct snd_soc_dai *dai, int stream)
-{
-	return dai->stream[stream].widget;
-}
-
-#define snd_soc_dai_set_widget_playback(dai, widget)	snd_soc_dai_set_widget(dai, SNDRV_PCM_STREAM_PLAYBACK, widget)
-#define snd_soc_dai_set_widget_capture(dai,  widget)	snd_soc_dai_set_widget(dai, SNDRV_PCM_STREAM_CAPTURE,  widget)
-static inline
-void snd_soc_dai_set_widget(struct snd_soc_dai *dai, int stream, struct snd_soc_dapm_widget *widget)
-{
-	dai->stream[stream].widget = widget;
-}
-
-#define snd_soc_dai_dma_data_get_playback(dai)	snd_soc_dai_dma_data_get(dai, SNDRV_PCM_STREAM_PLAYBACK)
-#define snd_soc_dai_dma_data_get_capture(dai)	snd_soc_dai_dma_data_get(dai, SNDRV_PCM_STREAM_CAPTURE)
-#define snd_soc_dai_get_dma_data(dai, ss)	snd_soc_dai_dma_data_get(dai, ss->stream)
-static inline void *snd_soc_dai_dma_data_get(const struct snd_soc_dai *dai, int stream)
-{
-	return dai->stream[stream].dma_data;
-}
-
-#define snd_soc_dai_dma_data_set_playback(dai, data)	snd_soc_dai_dma_data_set(dai, SNDRV_PCM_STREAM_PLAYBACK, data)
-#define snd_soc_dai_dma_data_set_capture(dai,  data)	snd_soc_dai_dma_data_set(dai, SNDRV_PCM_STREAM_CAPTURE,  data)
-#define snd_soc_dai_set_dma_data(dai, ss, data)		snd_soc_dai_dma_data_set(dai, ss->stream, data)
-static inline void snd_soc_dai_dma_data_set(struct snd_soc_dai *dai, int stream, void *data)
+snd_soc_dai_pcm_stream_get_s(const struct snd_soc_dai *dai, struct snd_pcm_substream *substream)
 {
-	dai->stream[stream].dma_data = data;
+	return snd_soc_dai_pcm_stream_get_i(dai, substream->stream);
 }
-
-static inline void snd_soc_dai_init_dma_data(struct snd_soc_dai *dai, void *playback, void *capture)
+#define snd_soc_dai_pcm_stream_get(dai, x) _Generic((x),		\
+	int :				snd_soc_dai_pcm_stream_get_i,	\
+	struct snd_pcm_substream * :	snd_soc_dai_pcm_stream_get_s)(dai, x)
+
+#define snd_soc_dai_stream_widget_get_playback(dai)	snd_soc_dai_stream_widget_get(dai, SNDRV_PCM_STREAM_PLAYBACK)
+#define snd_soc_dai_stream_widget_get_capture(dai)	snd_soc_dai_stream_widget_get(dai, SNDRV_PCM_STREAM_CAPTURE)
+struct snd_soc_dapm_widget *snd_soc_dai_stream_widget_get(struct snd_soc_dai *dai, int stream);
+
+#define snd_soc_dai_stream_widget_set_playback(dai, widget)	snd_soc_dai_stream_widget_set(dai, SNDRV_PCM_STREAM_PLAYBACK, widget)
+#define snd_soc_dai_stream_widget_set_capture(dai,  widget)	snd_soc_dai_stream_widget_set(dai, SNDRV_PCM_STREAM_CAPTURE,  widget)
+void snd_soc_dai_stream_widget_set(struct snd_soc_dai *dai, int stream, struct snd_soc_dapm_widget *widget);
+
+#define snd_soc_dai_stream_dma_data_get_playback(dai)	snd_soc_dai_stream_dma_data_get(dai, SNDRV_PCM_STREAM_PLAYBACK)
+#define snd_soc_dai_stream_dma_data_get_capture(dai)	snd_soc_dai_stream_dma_data_get(dai, SNDRV_PCM_STREAM_CAPTURE)
+void *snd_soc_dai_stream_dma_data_get_i(const struct snd_soc_dai *dai, int stream);
+static inline void *snd_soc_dai_stream_dma_data_get_s(const struct snd_soc_dai *dai, struct snd_pcm_substream *substream)
 {
-	snd_soc_dai_dma_data_set_playback(dai, playback);
-	snd_soc_dai_dma_data_set_capture(dai,  capture);
+	return snd_soc_dai_stream_dma_data_get_i(dai, substream->stream);
 }
-
-static inline unsigned int snd_soc_dai_tdm_mask_get(const struct snd_soc_dai *dai,
-						    int stream)
+#define snd_soc_dai_stream_dma_data_get(dai, x) _Generic((x),			\
+	int :				snd_soc_dai_stream_dma_data_get_i,	\
+	struct snd_pcm_substream * :	snd_soc_dai_stream_dma_data_get_s)(dai, x)
+
+#define snd_soc_dai_stream_dma_data_set_playback(dai, data)	snd_soc_dai_stream_dma_data_set(dai, SNDRV_PCM_STREAM_PLAYBACK, data)
+#define snd_soc_dai_stream_dma_data_set_capture(dai,  data)	snd_soc_dai_stream_dma_data_set(dai, SNDRV_PCM_STREAM_CAPTURE,  data)
+void snd_soc_dai_stream_dma_data_set_i(struct snd_soc_dai *dai, int stream, void *data);
+static inline void snd_soc_dai_stream_dma_data_set_s(struct snd_soc_dai *dai, struct snd_pcm_substream *substream, void *data)
 {
-	return dai->stream[stream].tdm_mask;
+	snd_soc_dai_stream_dma_data_set_i(dai, substream->stream, data);
 }
+#define snd_soc_dai_stream_dma_data_set(dai, x, data) _Generic((x),		\
+	int :				snd_soc_dai_stream_dma_data_set_i,	\
+	struct snd_pcm_substream * :	snd_soc_dai_stream_dma_data_set_s)(dai, x, data)
 
-static inline void snd_soc_dai_tdm_mask_set(struct snd_soc_dai *dai, int stream,
-					    unsigned int tdm_mask)
-{
-	dai->stream[stream].tdm_mask = tdm_mask;
-}
+unsigned int snd_soc_dai_stream_tdm_mask_get(const struct snd_soc_dai *dai, int stream);
+void snd_soc_dai_stream_tdm_mask_set(struct snd_soc_dai *dai, int stream, unsigned int tdm_mask);
 
-static inline unsigned int snd_soc_dai_stream_active(const struct snd_soc_dai *dai,
-						     int stream)
-{
-	/* see snd_soc_dai_action() for setup */
-	return dai->stream[stream].active;
-}
+unsigned int snd_soc_dai_stream_active(const struct snd_soc_dai *dai, int stream);
 
 static inline void snd_soc_dai_set_drvdata(struct snd_soc_dai *dai,
 		void *data)
@@ -564,45 +543,43 @@ static inline void *snd_soc_dai_get_drvdata(struct snd_soc_dai *dai)
 	return dev_get_drvdata(dai->dev);
 }
 
-/**
- * snd_soc_dai_set_stream() - Configures a DAI for stream operation
- * @dai: DAI
- * @stream: STREAM (opaque structure depending on DAI type)
- * @direction: Stream direction(Playback/Capture)
- * Some subsystems, such as SoundWire, don't have a notion of direction and we reuse
- * the ASoC stream direction to configure sink/source ports.
- * Playback maps to source ports and Capture for sink ports.
- *
- * This should be invoked with NULL to clear the stream set previously.
- * Returns 0 on success, a negative error code otherwise.
- */
-static inline int snd_soc_dai_set_stream(struct snd_soc_dai *dai,
-					 void *stream, int direction)
-{
-	if (dai->driver->ops->set_stream)
-		return dai->driver->ops->set_stream(dai, stream, direction);
-	else
-		return -ENOTSUPP;
-}
-
-/**
- * snd_soc_dai_get_stream() - Retrieves stream from DAI
- * @dai: DAI
- * @direction: Stream direction(Playback/Capture)
- *
- * This routine only retrieves that was previously configured
- * with snd_soc_dai_get_stream()
- *
- * Returns pointer to stream or an ERR_PTR value, e.g.
- * ERR_PTR(-ENOTSUPP) if callback is not supported;
- */
-static inline void *snd_soc_dai_get_stream(struct snd_soc_dai *dai,
-					   int direction)
+int snd_soc_dai_set_stream(struct snd_soc_dai *dai, void *stream, int direction);
+void *snd_soc_dai_get_stream(struct snd_soc_dai *dai, int direction);
+
+struct snd_soc_dai *snd_soc_dai_register(struct snd_soc_component *component,
+					 struct snd_soc_dai_driver *dai_drv,
+					 bool legacy_dai_naming);
+void snd_soc_dai_unregister(struct snd_soc_dai *dai);
+struct snd_soc_dai *snd_soc_dai_from_list(struct list_head *list);
+struct list_head *snd_soc_dai_to_list(struct snd_soc_dai *dai);
+unsigned int snd_soc_dai_get_symmetric_rate(struct snd_soc_dai *dai);
+unsigned int snd_soc_dai_get_symmetric_channels(struct snd_soc_dai *dai);
+unsigned int snd_soc_dai_get_symmetric_sample_bits(struct snd_soc_dai *dai);
+void snd_soc_dai_set_priv(struct snd_soc_dai *dai, void *priv);
+void *snd_soc_dai_to_priv(struct snd_soc_dai *dai);
+
+/* REMOVE ME */
+#define snd_soc_dai_get_pcm_stream			snd_soc_dai_pcm_stream_get
+#define snd_soc_dai_get_widget_playback(dai)		snd_soc_dai_stream_widget_get(dai, SNDRV_PCM_STREAM_PLAYBACK)
+#define snd_soc_dai_get_widget_capture(dai)		snd_soc_dai_stream_widget_get(dai, SNDRV_PCM_STREAM_CAPTURE)
+#define snd_soc_dai_get_widget				snd_soc_dai_stream_widget_get
+#define snd_soc_dai_set_widget_playback(dai, widget)	snd_soc_dai_stream_widget_set(dai, SNDRV_PCM_STREAM_PLAYBACK, widget)
+#define snd_soc_dai_set_widget_capture(dai,  widget)	snd_soc_dai_stream_widget_set(dai, SNDRV_PCM_STREAM_CAPTURE,  widget)
+#define snd_soc_dai_set_widget				snd_soc_dai_stream_widget_set
+#define snd_soc_dai_dma_data_get_playback(dai)		snd_soc_dai_stream_dma_data_get(dai, SNDRV_PCM_STREAM_PLAYBACK)
+#define snd_soc_dai_dma_data_get_capture(dai)		snd_soc_dai_stream_dma_data_get(dai, SNDRV_PCM_STREAM_CAPTURE)
+#define snd_soc_dai_get_dma_data			snd_soc_dai_stream_dma_data_get
+#define snd_soc_dai_dma_data_get			snd_soc_dai_stream_dma_data_get
+#define snd_soc_dai_dma_data_set_playback(dai, data)	snd_soc_dai_stream_dma_data_set(dai, SNDRV_PCM_STREAM_PLAYBACK, data)
+#define snd_soc_dai_dma_data_set_capture(dai,  data)	snd_soc_dai_stream_dma_data_set(dai, SNDRV_PCM_STREAM_CAPTURE,  data)
+#define snd_soc_dai_set_dma_data			snd_soc_dai_stream_dma_data_set
+#define snd_soc_dai_dma_data_set			snd_soc_dai_stream_dma_data_set
+#define snd_soc_dai_tdm_mask_get			snd_soc_dai_stream_tdm_mask_get
+#define snd_soc_dai_tdm_mask_set			snd_soc_dai_stream_tdm_mask_set
+static inline void snd_soc_dai_init_dma_data(struct snd_soc_dai *dai, void *playback, void *capture)
 {
-	if (dai->driver->ops->get_stream)
-		return dai->driver->ops->get_stream(dai, direction);
-	else
-		return ERR_PTR(-ENOTSUPP);
+	snd_soc_dai_stream_dma_data_set_playback(dai, playback);
+	snd_soc_dai_stream_dma_data_set_capture(dai,  capture);
 }
 
 #endif
diff --git a/include/sound/soc.h b/include/sound/soc.h
index 5afc34b147b517..553bb7e2be9840 100644
--- a/include/sound/soc.h
+++ b/include/sound/soc.h
@@ -555,8 +555,6 @@ int snd_soc_add_component_controls(struct snd_soc_component *component,
 	const struct snd_kcontrol_new *controls, unsigned int num_controls);
 int snd_soc_add_card_controls(struct snd_soc_card *soc_card,
 	const struct snd_kcontrol_new *controls, int num_controls);
-int snd_soc_add_dai_controls(struct snd_soc_dai *dai,
-	const struct snd_kcontrol_new *controls, int num_controls);
 int snd_soc_info_enum_double(struct snd_kcontrol *kcontrol,
 	struct snd_ctl_elem_info *uinfo);
 int snd_soc_get_enum_double(struct snd_kcontrol *kcontrol,
@@ -1389,19 +1387,12 @@ void snd_soc_dlc_use_cpu_as_platform(struct snd_soc_dai_link_component *platform
 struct of_phandle_args *snd_soc_copy_dai_args(struct device *dev,
 					      const struct of_phandle_args *args);
 struct snd_soc_dai *snd_soc_get_dai_via_args(const struct of_phandle_args *dai_args);
-struct snd_soc_dai *snd_soc_register_dai(struct snd_soc_component *component,
-					 struct snd_soc_dai_driver *dai_drv,
-					 bool legacy_dai_naming);
-void snd_soc_unregister_dai(struct snd_soc_dai *dai);
 
 struct snd_soc_dai *snd_soc_find_dai(
 	const struct snd_soc_dai_link_component *dlc);
 struct snd_soc_dai *snd_soc_find_dai_with_mutex(
 	const struct snd_soc_dai_link_component *dlc);
 
-void soc_pcm_set_dai_params(struct snd_soc_dai *dai,
-			    struct snd_pcm_hw_params *params);
-
 #include <sound/soc-dai.h>
 
 static inline
@@ -1551,4 +1542,11 @@ static inline void _snd_soc_dpcm_mutex_assert_held_r(struct snd_soc_pcm_runtime
 #include <sound/soc-card.h>
 #include <sound/soc-jack.h>
 
+/*
+ * REMOVE ME
+ */
+#define snd_soc_add_dai_controls			snd_soc_dai_add_controls
+#define snd_soc_register_dai				snd_soc_dai_register
+#define snd_soc_unregister_dai				snd_soc_dai_unregister
+
 #endif
diff --git a/include/sound/soc_sdw_utils.h b/include/sound/soc_sdw_utils.h
index 9fbb69b9052df4..418466438ea1cb 100644
--- a/include/sound/soc_sdw_utils.h
+++ b/include/sound/soc_sdw_utils.h
@@ -292,4 +292,24 @@ int asoc_sdw_es9356_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai
 int asoc_sdw_es9356_spk_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 int asoc_sdw_es9356_dmic_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 
+/* Senary SN624x SDCA */
+int asoc_sdw_senary_sdca_jack_init(struct snd_soc_card *card,
+				   struct snd_soc_dai_link *dai_links,
+				   struct asoc_sdw_codec_info *info,
+				   bool playback);
+int asoc_sdw_senary_sdca_jack_exit(struct snd_soc_card *card,
+				   struct snd_soc_dai_link *dai_link);
+int asoc_sdw_senary_sdca_jack_rtd_init(struct snd_soc_pcm_runtime *rtd,
+				       struct snd_soc_dai *dai);
+int asoc_sdw_senary_amp_init(struct snd_soc_card *card,
+			     struct snd_soc_dai_link *dai_links,
+			     struct asoc_sdw_codec_info *info,
+			     bool playback);
+int asoc_sdw_senary_amp_exit(struct snd_soc_card *card,
+			     struct snd_soc_dai_link *dai_link);
+int asoc_sdw_senary_sdca_spk_rtd_init(struct snd_soc_pcm_runtime *rtd,
+				      struct snd_soc_dai *dai);
+int asoc_sdw_senary_dmic_rtd_init(struct snd_soc_pcm_runtime *rtd,
+				  struct snd_soc_dai *dai);
+
 #endif
diff --git a/include/sound/tas2781-dsp.h b/include/sound/tas2781-dsp.h
index e845687decf978..3380dfd526de7c 100644
--- a/include/sound/tas2781-dsp.h
+++ b/include/sound/tas2781-dsp.h
@@ -56,6 +56,10 @@ enum tasdevice_dsp_dev_idx {
 	TASDEVICE_DSP_TAS_2781_QUAD,
 	TASDEVICE_DSP_TAS_5825_MONO,
 	TASDEVICE_DSP_TAS_5825_DUAL,
+	TASDEVICE_DSP_TAS_2573_MONO,
+	TASDEVICE_DSP_TAS_2573_DUAL,
+	TASDEVICE_DSP_TAS_2573_21,
+	TASDEVICE_DSP_TAS_2573_QUAD,
 	TASDEVICE_DSP_TAS_MAX_DEVICE
 };
 
@@ -142,6 +146,7 @@ struct tasdevice_fw {
 	struct tasdevice_calibration *calibrations;
 	struct fct_param_address fct_par_addr;
 	struct device *dev;
+	int calibration_config_id;
 };
 
 enum tasdevice_fw_state {
@@ -206,6 +211,14 @@ struct tasdevice_rca {
 	 * capture.
 	 */
 	int capture_profile_id;
+	/*
+	 * Primarily designed for speaker calibration scenarios with special
+	 * requirements. For regular use cases, when the default value -1 is
+	 * set, calibration will directly reuse the current playback
+	 * configuration; non-negative values can be customized for special
+	 * calibration demands.
+	 */
+	int calibration_profile_id;
 	/*
 	 * Since version 0x105, the keyword 'init' was introduced into the
 	 * profile, which is used for chip initialization, particularly to
diff --git a/include/sound/tas2781.h b/include/sound/tas2781.h
index b763da6137699d..7367c76959aa1a 100644
--- a/include/sound/tas2781.h
+++ b/include/sound/tas2781.h
@@ -91,6 +91,30 @@
 /* prm_TE_1_Beta1 */
 #define TAS2563_TE_DT_REG		TASDEVICE_REG(0x00, 0x0f, 0x70)
 
+#define TAS2573_FCT_INT_LATCH		TASDEVICE_REG(0x00, 0x00, 0x67)
+/* YM57 */
+#define TAS2573_FCT_OUTPUT_R0		TASDEVICE_REG(0x00, 0x65, 0x74)
+/* YM58 */
+#define TAS2573_FCT_OUTPUT_R0_LOW	TASDEVICE_REG(0x00, 0x65, 0x78)
+/* YM59 */
+#define TAS2573_FCT_OUTPUT_INV_R0	TASDEVICE_REG(0x00, 0x65, 0x7C)
+/* YM61 */
+#define TAS2573_FCT_OUTPUT_POWERTOT	TASDEVICE_REG(0x00, 0x66, 0x0C)
+/* YM64 */
+#define TAS2573_FCT_OUTPUT_F0		TASDEVICE_REG(0x00, 0x65, 0x18)
+/* YM147 */
+#define TAS2573_FCT_STATUS_CTRL		TASDEVICE_REG(0x00, 0x68, 0x74)
+/* YM148 */
+#define TAS2573_FCT_STATUS_BINNING	TASDEVICE_REG(0x00, 0x68, 0x78)
+/* YM664 */
+#define TAS2573_RE_OUT			TASDEVICE_REG(0x00, 0x7A, 0x18)
+/* YM952 */
+#define TAS2573_SILENCE_DETECTED	TASDEVICE_REG(0x00, 0x83, 0x60)
+/* YM954 */
+#define TAS2573_OPEN_CIRCUIT		TASDEVICE_REG(0x00, 0x83, 0x68)
+/* YM955 */
+#define TAS2573_SHORTCKT		TASDEVICE_REG(0x00, 0x83, 0x6C)
+
 #define TAS2781_PRM_INT_MASK_REG	TASDEVICE_REG(0x00, 0x00, 0x3b)
 #define TAS2781_PRM_CLK_CFG_REG		TASDEVICE_REG(0x00, 0x00, 0x5c)
 #define TAS2781_PRM_RSVD_REG		TASDEVICE_REG(0x00, 0x01, 0x19)
diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h
index e3101410e8b2dc..a19233e8ae78e6 100644
--- a/include/trace/events/cachefiles.h
+++ b/include/trace/events/cachefiles.h
@@ -52,6 +52,8 @@ enum cachefiles_coherency_trace {
 	cachefiles_coherency_check_ok,
 	cachefiles_coherency_check_type,
 	cachefiles_coherency_check_xattr,
+	cachefiles_coherency_discontiguous,
+	cachefiles_coherency_remove,
 	cachefiles_coherency_set_fail,
 	cachefiles_coherency_set_ok,
 	cachefiles_coherency_vol_check_cmp,
@@ -63,9 +65,11 @@ enum cachefiles_coherency_trace {
 };
 
 enum cachefiles_trunc_trace {
+	cachefiles_trunc_clear_padding,
 	cachefiles_trunc_dio_adjust,
 	cachefiles_trunc_expand_tmpfile,
 	cachefiles_trunc_shrink,
+	cachefiles_trunc_zap,
 };
 
 enum cachefiles_prepare_read_trace {
@@ -80,11 +84,14 @@ enum cachefiles_prepare_read_trace {
 };
 
 enum cachefiles_error_trace {
+	cachefiles_trace_alignment_error,
+	cachefiles_trace_create_nospace,
 	cachefiles_trace_fallocate_error,
 	cachefiles_trace_getxattr_error,
 	cachefiles_trace_link_error,
 	cachefiles_trace_lookup_error,
 	cachefiles_trace_mkdir_error,
+	cachefiles_trace_mkdir_nospace,
 	cachefiles_trace_notify_change_error,
 	cachefiles_trace_open_error,
 	cachefiles_trace_read_error,
@@ -97,6 +104,8 @@ enum cachefiles_error_trace {
 	cachefiles_trace_trunc_error,
 	cachefiles_trace_unlink_error,
 	cachefiles_trace_write_error,
+	cachefiles_trace_write_nospace,
+	cachefiles_trace_write_nospace_2,
 };
 
 #endif
@@ -136,6 +145,8 @@ enum cachefiles_error_trace {
 	EM(cachefiles_coherency_check_ok,	"OK      ")		\
 	EM(cachefiles_coherency_check_type,	"BAD type")		\
 	EM(cachefiles_coherency_check_xattr,	"BAD xatt")		\
+	EM(cachefiles_coherency_discontiguous,	"--- gap ")		\
+	EM(cachefiles_coherency_remove,		"REMOVE  ")		\
 	EM(cachefiles_coherency_set_fail,	"SET fail")		\
 	EM(cachefiles_coherency_set_ok,		"SET ok  ")		\
 	EM(cachefiles_coherency_vol_check_cmp,	"VOL BAD cmp ")		\
@@ -146,9 +157,11 @@ enum cachefiles_error_trace {
 	E_(cachefiles_coherency_vol_set_ok,	"VOL SET ok  ")
 
 #define cachefiles_trunc_traces						\
+	EM(cachefiles_trunc_clear_padding,	"CLRPAD")		\
 	EM(cachefiles_trunc_dio_adjust,		"DIOADJ")		\
 	EM(cachefiles_trunc_expand_tmpfile,	"EXPTMP")		\
-	E_(cachefiles_trunc_shrink,		"SHRINK")
+	EM(cachefiles_trunc_shrink,		"SHRINK")		\
+	E_(cachefiles_trunc_zap,		"ZAP   ")
 
 #define cachefiles_prepare_read_traces					\
 	EM(cachefiles_trace_read_after_eof,	"after-eof ")		\
@@ -161,11 +174,14 @@ enum cachefiles_error_trace {
 	E_(cachefiles_trace_read_seek_nxio,	"seek-enxio")
 
 #define cachefiles_error_traces						\
+	EM(cachefiles_trace_alignment_error,	"align")		\
+	EM(cachefiles_trace_create_nospace,	"create-nospace")	\
 	EM(cachefiles_trace_fallocate_error,	"fallocate")		\
 	EM(cachefiles_trace_getxattr_error,	"getxattr")		\
 	EM(cachefiles_trace_link_error,		"link")			\
 	EM(cachefiles_trace_lookup_error,	"lookup")		\
 	EM(cachefiles_trace_mkdir_error,	"mkdir")		\
+	EM(cachefiles_trace_mkdir_nospace,	"mkdir-nospace")	\
 	EM(cachefiles_trace_notify_change_error, "notify_change")	\
 	EM(cachefiles_trace_open_error,		"open")			\
 	EM(cachefiles_trace_read_error,		"read")			\
@@ -177,7 +193,9 @@ enum cachefiles_error_trace {
 	EM(cachefiles_trace_tmpfile_error,	"tmpfile")		\
 	EM(cachefiles_trace_trunc_error,	"trunc")		\
 	EM(cachefiles_trace_unlink_error,	"unlink")		\
-	E_(cachefiles_trace_write_error,	"write")
+	EM(cachefiles_trace_write_error,	"write")		\
+	EM(cachefiles_trace_write_nospace,	"write-nospace")	\
+	E_(cachefiles_trace_write_nospace_2,	"write-nospace-2")
 
 
 /*
@@ -371,12 +389,12 @@ TRACE_EVENT(cachefiles_rename,
 
 TRACE_EVENT(cachefiles_coherency,
 	    TP_PROTO(struct cachefiles_object *obj,
-		     ino_t ino,
+		     ino_t ino, uoff_t obj_size,
 		     const void *disk_aux,
 		     enum cachefiles_content content,
 		     enum cachefiles_coherency_trace why),
 
-	    TP_ARGS(obj, ino, disk_aux, content, why),
+	    TP_ARGS(obj, ino, obj_size, disk_aux, content, why),
 
 	    /* Note that obj may be NULL */
 	    TP_STRUCT__entry(
@@ -384,6 +402,7 @@ TRACE_EVENT(cachefiles_coherency,
 		    __field(enum cachefiles_coherency_trace,	why)
 		    __field(enum cachefiles_content,		content)
 		    __field(u64,				ino)
+		    __field(u64,				obj_size)
 		    __field(u64,				aux)
 		    __field(u64,				disk_aux)
 			     ),
@@ -398,6 +417,7 @@ TRACE_EVENT(cachefiles_coherency,
 		    __entry->why	= why;
 		    __entry->content	= content;
 		    __entry->ino	= ino;
+		    __entry->obj_size	= obj_size;
 		    __entry->aux	= be64_to_cpup((__be64 *)obj->cookie->inline_aux);
 
 		    /* cachefiles_xattr::data is 2-byte aligned but not 8-byte aligned. */
@@ -412,10 +432,11 @@ TRACE_EVENT(cachefiles_coherency,
 		    }
 			   ),
 
-	    TP_printk("o=%08x %s B=%llx c=%u aux=%llx dsk=%llx",
+	    TP_printk("o=%08x %s B=%llx oz=%llx c=%u aux=%llx dsk=%llx",
 		      __entry->obj,
 		      __print_symbolic(__entry->why, cachefiles_coherency_traces),
 		      __entry->ino,
+		      __entry->obj_size,
 		      __entry->content,
 		      __entry->aux,
 		      __entry->disk_aux)
@@ -449,7 +470,7 @@ TRACE_EVENT(cachefiles_vol_coherency,
 
 TRACE_EVENT(cachefiles_prep_read,
 	    TP_PROTO(struct cachefiles_object *obj,
-		     loff_t start,
+		     uoff_t start,
 		     size_t len,
 		     unsigned short flags,
 		     enum netfs_io_source source,
@@ -464,7 +485,7 @@ TRACE_EVENT(cachefiles_prep_read,
 		    __field(enum netfs_io_source,	source)
 		    __field(enum cachefiles_prepare_read_trace,	why)
 		    __field(size_t,			len)
-		    __field(loff_t,			start)
+		    __field(uoff_t,			start)
 		    __field(unsigned int,		netfs_inode)
 		    __field(unsigned int,		cache_inode)
 			     ),
@@ -492,16 +513,16 @@ TRACE_EVENT(cachefiles_prep_read,
 TRACE_EVENT(cachefiles_read,
 	    TP_PROTO(struct cachefiles_object *obj,
 		     struct inode *backer,
-		     loff_t start,
+		     uoff_t start,
 		     size_t len),
 
 	    TP_ARGS(obj, backer, start, len),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,			obj)
-		    __field(unsigned int,			backer)
-		    __field(size_t,				len)
-		    __field(loff_t,				start)
+		    __field(unsigned int,	obj)
+		    __field(unsigned int,	backer)
+		    __field(size_t,		len)
+		    __field(uoff_t,		start)
 			     ),
 
 	    TP_fast_assign(
@@ -521,16 +542,16 @@ TRACE_EVENT(cachefiles_read,
 TRACE_EVENT(cachefiles_write,
 	    TP_PROTO(struct cachefiles_object *obj,
 		     struct inode *backer,
-		     loff_t start,
+		     uoff_t start,
 		     size_t len),
 
 	    TP_ARGS(obj, backer, start, len),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,			obj)
-		    __field(unsigned int,			backer)
-		    __field(size_t,				len)
-		    __field(loff_t,				start)
+		    __field(unsigned int,	obj)
+		    __field(unsigned int,	backer)
+		    __field(size_t,		len)
+		    __field(uoff_t,		start)
 			     ),
 
 	    TP_fast_assign(
@@ -549,7 +570,7 @@ TRACE_EVENT(cachefiles_write,
 
 TRACE_EVENT(cachefiles_trunc,
 	    TP_PROTO(struct cachefiles_object *obj, struct inode *backer,
-		     loff_t from, loff_t to, enum cachefiles_trunc_trace why),
+		     uoff_t from, uoff_t to, enum cachefiles_trunc_trace why),
 
 	    TP_ARGS(obj, backer, from, to, why),
 
@@ -557,8 +578,8 @@ TRACE_EVENT(cachefiles_trunc,
 		    __field(unsigned int,			obj)
 		    __field(unsigned int,			backer)
 		    __field(enum cachefiles_trunc_trace,	why)
-		    __field(loff_t,				from)
-		    __field(loff_t,				to)
+		    __field(uoff_t,				from)
+		    __field(uoff_t,				to)
 			     ),
 
 	    TP_fast_assign(
@@ -694,6 +715,26 @@ TRACE_EVENT(cachefiles_io_error,
 		      __entry->error)
 	    );
 
+TRACE_EVENT(cachefiles_no_space,
+	    TP_PROTO(struct cachefiles_object *obj, enum cachefiles_error_trace trace),
+
+	    TP_ARGS(obj, trace),
+
+	    TP_STRUCT__entry(
+		    __field(unsigned int,			obj)
+		    __field(enum cachefiles_error_trace,	trace)
+			     ),
+
+	    TP_fast_assign(
+		    __entry->obj	= obj ? obj->debug_id : 0;
+		    __entry->trace	= trace;
+			   ),
+
+	    TP_printk("o=%08x %s",
+		      __entry->obj,
+		      __print_symbolic(__entry->trace, cachefiles_error_traces))
+	    );
+
 #endif /* _TRACE_CACHEFILES_H */
 
 /* This part must be outside protection */
diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h
index d53be932df01ce..df4cd346ae726d 100644
--- a/include/trace/events/f2fs.h
+++ b/include/trace/events/f2fs.h
@@ -1430,6 +1430,50 @@ DEFINE_EVENT(f2fs__folio, f2fs_set_page_dirty,
 	TP_ARGS(folio, type)
 );
 
+DECLARE_EVENT_CLASS(f2fs__cached_block,
+
+	TP_PROTO(struct f2fs_cached_block *block, int type),
+
+	TP_ARGS(block, type),
+
+	TP_STRUCT__entry(
+		__field(dev_t,	dev)
+		__field(pgoff_t, index)
+		__field(int,	type)
+		__field(int,	dirty)
+		__field(int,	uptodate)
+	),
+
+	TP_fast_assign(
+		__entry->dev	= block->cache->sbi->sb->s_dev;
+		__entry->index	= block->index;
+		__entry->type	= type;
+		__entry->dirty	= f2fs_cache_test_dirty(block);
+		__entry->uptodate = f2fs_cache_test_uptodate(block);
+	),
+
+	TP_printk("dev = (%d,%d), %s, index = %lu, dirty = %d, uptodate = %d",
+		show_dev(__entry->dev),
+		show_block_type(__entry->type),
+		(unsigned long)__entry->index,
+		__entry->dirty,
+		__entry->uptodate)
+);
+
+DEFINE_EVENT(f2fs__cached_block, f2fs_write_cache,
+
+	TP_PROTO(struct f2fs_cached_block *block, int type),
+
+	TP_ARGS(block, type)
+);
+
+DEFINE_EVENT(f2fs__cached_block, f2fs_cache_set_dirty,
+
+	TP_PROTO(struct f2fs_cached_block *block, int type),
+
+	TP_ARGS(block, type)
+);
+
 TRACE_EVENT(f2fs_replace_atomic_write_block,
 
 	TP_PROTO(struct inode *inode, struct inode *cow_inode, pgoff_t index,
@@ -1574,6 +1618,33 @@ TRACE_EVENT(f2fs_writepages,
 		__entry->for_sync)
 );
 
+TRACE_EVENT(f2fs_write_caches,
+
+	TP_PROTO(struct f2fs_sb_info *sbi, long nr_to_write, long nwritten, int type),
+
+	TP_ARGS(sbi, nr_to_write, nwritten, type),
+
+	TP_STRUCT__entry(
+		__field(dev_t,	dev)
+		__field(long,	nr_to_write)
+		__field(long,	nwritten)
+		__field(int,	type)
+	),
+
+	TP_fast_assign(
+		__entry->dev		= sbi->sb->s_dev;
+		__entry->nr_to_write	= nr_to_write;
+		__entry->nwritten	= nwritten;
+		__entry->type		= type;
+	),
+
+	TP_printk("dev = (%d,%d), %s, nr_to_write = %ld, nwritten = %ld",
+		show_dev(__entry->dev),
+		show_block_type(__entry->type),
+		__entry->nr_to_write,
+		__entry->nwritten)
+);
+
 TRACE_EVENT(f2fs_readpages,
 
 	TP_PROTO(struct inode *inode, pgoff_t start, unsigned int nrpage),
diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h
index f1a73aa83fbbfb..8735d428ebd9ba 100644
--- a/include/trace/events/fscache.h
+++ b/include/trace/events/fscache.h
@@ -460,13 +460,13 @@ TRACE_EVENT(fscache_relinquish,
 	    );
 
 TRACE_EVENT(fscache_invalidate,
-	    TP_PROTO(struct fscache_cookie *cookie, loff_t new_size),
+	    TP_PROTO(struct fscache_cookie *cookie, uoff_t new_size),
 
 	    TP_ARGS(cookie, new_size),
 
 	    TP_STRUCT__entry(
 		    __field(unsigned int,		cookie		)
-		    __field(loff_t,			new_size	)
+		    __field(uoff_t,			new_size	)
 			     ),
 
 	    TP_fast_assign(
@@ -479,14 +479,14 @@ TRACE_EVENT(fscache_invalidate,
 	    );
 
 TRACE_EVENT(fscache_resize,
-	    TP_PROTO(struct fscache_cookie *cookie, loff_t new_size),
+	    TP_PROTO(struct fscache_cookie *cookie, uoff_t new_size),
 
 	    TP_ARGS(cookie, new_size),
 
 	    TP_STRUCT__entry(
 		    __field(unsigned int,		cookie		)
-		    __field(loff_t,			old_size	)
-		    __field(loff_t,			new_size	)
+		    __field(uoff_t,			old_size	)
+		    __field(uoff_t,			new_size	)
 			     ),
 
 	    TP_fast_assign(
diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h
index 5a48c5406cce49..5fb4d92cfd8408 100644
--- a/include/trace/events/huge_memory.h
+++ b/include/trace/events/huge_memory.h
@@ -55,10 +55,10 @@ SCAN_STATUS
 
 TRACE_EVENT(mm_khugepaged_scan_pmd,
 
-	TP_PROTO(struct mm_struct *mm, struct folio *folio,
+	TP_PROTO(struct mm_struct *mm, unsigned long pfn,
 		 int referenced, int none_or_zero, int status, int unmapped),
 
-	TP_ARGS(mm, folio, referenced, none_or_zero, status, unmapped),
+	TP_ARGS(mm, pfn, referenced, none_or_zero, status, unmapped),
 
 	TP_STRUCT__entry(
 		__field(struct mm_struct *, mm)
@@ -71,7 +71,7 @@ TRACE_EVENT(mm_khugepaged_scan_pmd,
 
 	TP_fast_assign(
 		__entry->mm = mm;
-		__entry->pfn = folio ? folio_pfn(folio) : -1;
+		__entry->pfn = pfn;
 		__entry->referenced = referenced;
 		__entry->none_or_zero = none_or_zero;
 		__entry->status = status;
@@ -178,10 +178,10 @@ TRACE_EVENT(mm_collapse_huge_page_swapin,
 
 TRACE_EVENT(mm_khugepaged_scan_file,
 
-	TP_PROTO(struct mm_struct *mm, struct folio *folio, struct file *file,
+	TP_PROTO(struct mm_struct *mm, unsigned long pfn, struct file *file,
 		 int present, int swap, int result),
 
-	TP_ARGS(mm, folio, file, present, swap, result),
+	TP_ARGS(mm, pfn, file, present, swap, result),
 
 	TP_STRUCT__entry(
 		__field(struct mm_struct *, mm)
@@ -194,7 +194,7 @@ TRACE_EVENT(mm_khugepaged_scan_file,
 
 	TP_fast_assign(
 		__entry->mm = mm;
-		__entry->pfn = folio ? folio_pfn(folio) : -1;
+		__entry->pfn = pfn;
 		__assign_str(filename);
 		__entry->present = present;
 		__entry->swap = swap;
@@ -211,10 +211,10 @@ TRACE_EVENT(mm_khugepaged_scan_file,
 );
 
 TRACE_EVENT(mm_khugepaged_collapse_file,
-	TP_PROTO(struct mm_struct *mm, struct folio *new_folio, pgoff_t index,
+	TP_PROTO(struct mm_struct *mm, unsigned long new_pfn, pgoff_t index,
 			unsigned long addr, bool is_shmem, struct file *file,
 			int nr, int result),
-	TP_ARGS(mm, new_folio, index, addr, is_shmem, file, nr, result),
+	TP_ARGS(mm, new_pfn, index, addr, is_shmem, file, nr, result),
 	TP_STRUCT__entry(
 		__field(struct mm_struct *, mm)
 		__field(unsigned long, hpfn)
@@ -228,7 +228,7 @@ TRACE_EVENT(mm_khugepaged_collapse_file,
 
 	TP_fast_assign(
 		__entry->mm = mm;
-		__entry->hpfn = new_folio ? folio_pfn(new_folio) : -1;
+		__entry->hpfn = new_pfn;
 		__entry->index = index;
 		__entry->addr = addr;
 		__entry->is_shmem = is_shmem;
diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h
index b282e3a8676969..c4d514d2350cfe 100644
--- a/include/trace/events/kvm.h
+++ b/include/trace/events/kvm.h
@@ -358,13 +358,13 @@ TRACE_EVENT(kvm_dirty_ring_exit,
 	TP_printk("vcpu %d", __entry->vcpu_id)
 );
 
-#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
 /*
  * @start:	Starting address of guest memory range
  * @end:	End address of guest memory range
  * @attr:	The value of the attribute being set.
  */
-TRACE_EVENT(kvm_vm_set_mem_attributes,
+TRACE_EVENT(kvm_set_vm_mem_attributes,
 	TP_PROTO(gfn_t start, gfn_t end, unsigned long attr),
 	TP_ARGS(start, end, attr),
 
@@ -383,7 +383,7 @@ TRACE_EVENT(kvm_vm_set_mem_attributes,
 	TP_printk("%#016llx -- %#016llx [0x%lx]",
 		  __entry->start, __entry->end, __entry->attr)
 );
-#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
+#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */
 
 TRACE_EVENT(kvm_unmap_hva_range,
 	TP_PROTO(unsigned long start, unsigned long end),
diff --git a/include/trace/events/landlock.h b/include/trace/events/landlock.h
index d05253afaf5909..3a43638c9bc290 100644
--- a/include/trace/events/landlock.h
+++ b/include/trace/events/landlock.h
@@ -10,14 +10,19 @@
 #if !defined(_TRACE_LANDLOCK_H) || defined(TRACE_HEADER_MULTI_READ)
 #define _TRACE_LANDLOCK_H
 
+#include <linux/in.h>
+#include <linux/in6.h>
 #include <linux/landlock.h>
+#include <linux/socket.h>
 #include <linux/string.h>
 #include <linux/string_helpers.h>
 #include <linux/tracepoint.h>
 #include <linux/trace_seq.h>
 #include <net/af_unix.h>
 
+enum landlock_request_type;
 struct dentry;
+struct landlock_blockers;
 struct landlock_domain;
 struct landlock_hierarchy;
 struct landlock_rule;
@@ -26,6 +31,12 @@ struct path;
 struct sock;
 struct task_struct;
 
+static_assert(sizeof(access_mask_t) <= sizeof(u64));
+
+TRACE_DEFINE_ENUM(LANDLOCK_REQUEST_FS_CHANGE_TOPOLOGY);
+TRACE_DEFINE_ENUM(LANDLOCK_REQUEST_FS_ACCESS);
+TRACE_DEFINE_ENUM(LANDLOCK_REQUEST_NET_ACCESS);
+
 #ifdef CREATE_TRACE_POINTS
 
 /* About 6 KiB, leaving about 2 KiB for sibling helpers and fixed fields. */
@@ -108,7 +119,7 @@ __trace_print_untrusted_str(struct trace_seq *p, const char *src, size_t len)
  * Fills the dense per-domain-layer array layers (one access mask per layer,
  * indexed by level - 1) from rule's sparse layer stack, keeping only the
  * requested rights (access_request).  Layers with no matching rule entry get
- * a zero mask.  Shared by the check_rule_fs and check_rule_net events.
+ * a zero mask.  Shared by the check_rule_inode and check_rule_net_port events.
  *
  * rule->layers is sorted by ascending level, with levels in the domain's
  * [1, num_layers] range (see landlock_merge_ruleset()), so every entry maps
@@ -182,6 +193,9 @@ static inline const char *__trace_landlock_print_layers(
 /* Maps a shared _LANDLOCK_*_NAMES entry to a __print_flags() pair. */
 #define _LANDLOCK_NAME_ENTRY(mask, name) { mask, name }
 
+#define _LANDLOCK_FS_BLOCKER_TYPE_NAMES \
+	{ LANDLOCK_REQUEST_FS_CHANGE_TOPOLOGY, "change_topology" }
+
 /**
  * DOC: Landlock trace events
  *
@@ -192,22 +206,22 @@ static inline const char *__trace_landlock_print_layers(
  * Decision context
  * ~~~~~~~~~~~~~~~~
  *
- * A denial event, together with the lifecycle events, exposes the full
- * set of inputs the verdict consumed, so a consumer that tracked domain
- * creation (landlock_create_ruleset, landlock_create_domain) can verify
- * or reproduce the Landlock decision rather than merely observe it
- * happened.  In who/what/why terms: who is the denying domain (the domain
- * field, always the subject that enforced the policy, never the current
- * task), what is the operation and its object, and why is every other
- * input the verdict weighed.
+ * A denial event identifies the domain whose policy denied the request, the
+ * Landlock operation and policy object that were checked, and the blocker or
+ * domain relationship responsible for the denial.  When tracing starts with
+ * sandbox construction, ruleset and domain events provide the policy history
+ * needed to interpret these identifiers.  The denying domain is the subject
+ * that enforced the policy, not necessarily current.  Generic tracepoints can
+ * provide additional operational context.
  *
  * Lifecycle consistency
  * ~~~~~~~~~~~~~~~~~~~~~~
  *
- * Lifecycle events are balanced: a creation event always has a matching
- * deallocation event and vice versa, so an eBPF program can model object
- * lifetimes from the trace stream without reconciliation logic.  A creation
- * event fires while the object is still private to the calling thread
+ * Lifecycle emission is balanced: a creation event always has a matching
+ * deallocation event and vice versa.  A consumer that observes an object's
+ * complete lifetime can model it from this pair; one that attaches late or
+ * loses exported records must reconcile incomplete state.  A creation event
+ * fires while the object is still private to the calling thread
  * (landlock_create_ruleset fires before the ruleset's file descriptor is
  * installed, so it cannot race a concurrent :manpage:`close(2)`); if fd
  * installation later fails and the ruleset is freed, free_ruleset still
@@ -243,8 +257,8 @@ static inline const char *__trace_landlock_print_layers(
  * Field encoding
  * ~~~~~~~~~~~~~~
  *
- * Fields that mirror the Landlock UAPI use the same C types and endianness
- * (e.g. network ports are __u64 in host endianness, like
+ * Fields that mirror the Landlock UAPI preserve their widths and endianness
+ * (e.g. network ports are u64 in host endianness, like
  * landlock_net_port_attr.port).  Per-event details, such as where a value
  * is byte-swapped, live in the field's own kdoc.
  *
@@ -281,6 +295,16 @@ static inline const char *__trace_landlock_print_layers(
  * the two parties without kernel-internal state.  The ID is a scalar
  * snapshot, not a live domain pointer that could dangle: an optional
  * relational referent is a scalar (0 sentinel), not a nullable pointer.
+ * Nonzero IDs are unique within one boot.  For ptrace, same_exec instead
+ * describes the tracer, even for
+ * PTRACE_TRACEME, and may differ from the current task.
+ *
+ * Blocker fields
+ * ~~~~~~~~~~~~~~
+ *
+ * The filesystem and network blocker arguments identify the request type
+ * and carry its final missing access subset when applicable.  The type
+ * determines how to interpret the access value.
  */
 
 /*
@@ -319,8 +343,8 @@ TRACE_EVENT(landlock_create_ruleset,
 	TP_ARGS(ruleset),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		ruleset_id	)
-		__field(	__u32,		ruleset_version	)
+		__field(	u64,		ruleset_id	)
+		__field(	u64,		ruleset_version	)
 		__field(	access_mask_t,	handled_fs	)
 		__field(	access_mask_t,	handled_net	)
 		__field(	access_mask_t,	scoped		)
@@ -334,7 +358,7 @@ TRACE_EVENT(landlock_create_ruleset,
 		__entry->scoped		= ruleset->handled_masks.scope;
 	),
 
-	TP_printk("ruleset=%llx.%u handled_fs=%s handled_net=%s scoped=%s",
+	TP_printk("ruleset=%llx.%llu handled_fs=%s handled_net=%s scoped=%s",
 		__entry->ruleset_id, __entry->ruleset_version,
 		__print_flags(__entry->handled_fs, "|", _LANDLOCK_ACCESS_FS_NAMES),
 		__print_flags(__entry->handled_net, "|", _LANDLOCK_ACCESS_NET_NAMES),
@@ -359,8 +383,8 @@ TRACE_EVENT(landlock_free_ruleset,
 	TP_ARGS(ruleset),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		ruleset_id	)
-		__field(	__u32,		ruleset_version	)
+		__field(	u64,		ruleset_id	)
+		__field(	u64,		ruleset_version	)
 	),
 
 	TP_fast_assign(
@@ -368,17 +392,19 @@ TRACE_EVENT(landlock_free_ruleset,
 		__entry->ruleset_version = ruleset->version;
 	),
 
-	TP_printk("ruleset=%llx.%u",
+	TP_printk("ruleset=%llx.%llu",
 		__entry->ruleset_id, __entry->ruleset_version)
 );
 
 /**
- * landlock_add_rule_fs - Filesystem rule added to a ruleset
+ * landlock_add_rule_path_beneath - Path-beneath rule added to a ruleset
  *
  * @ruleset: Source ruleset (never NULL).
- * @access_rights: Effective access mask stored in the rule, not the raw
- *                 sys_landlock_add_rule() argument (unhandled rights
- *                 added).
+ * @flags: Complete validated landlock_add_rule_flags value supplied by this
+ *         successful call, not the rule's accumulated quiet state.
+ * @access_rights: Canonical per-call access mask passed to
+ *                 landlock_insert_rule() after normalization, not the raw
+ *                 sys_landlock_add_rule() argument or accumulated rule.
  * @path: Filesystem path for the rule (never NULL).
  * @pathname: Resolved absolute path string (never NULL; error placeholder
  *            on resolution failure).
@@ -387,17 +413,17 @@ TRACE_EVENT(landlock_free_ruleset,
  * the reported ruleset is a stable snapshot that no concurrent writer can
  * change.
  */
-TRACE_EVENT(landlock_add_rule_fs,
+TRACE_EVENT(landlock_add_rule_path_beneath,
 
-	TP_PROTO(const struct landlock_ruleset *ruleset,
-		 access_mask_t access_rights, const struct path *path,
+	TP_PROTO(const struct landlock_ruleset *ruleset, u32 flags,
+		 u64 access_rights, const struct path *path,
 		 const char *pathname),
 
-	TP_ARGS(ruleset, access_rights, path, pathname),
+	TP_ARGS(ruleset, flags, access_rights, path, pathname),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		ruleset_id	)
-		__field(	__u32,		ruleset_version	)
+		__field(	u64,		ruleset_id	)
+		__field(	u64,		ruleset_version	)
 		__field(	access_mask_t,	access_rights	)
 		__field(	dev_t,		dev		)
 		__field(	ino_t,		ino		)
@@ -418,7 +444,7 @@ TRACE_EVENT(landlock_add_rule_fs,
 		__assign_str(pathname);
 	),
 
-	TP_printk("ruleset=%llx.%u access_rights=%s dev=%u:%u ino=%lu path=%s",
+	TP_printk("ruleset=%llx.%llu access_rights=%s dev=%u:%u ino=%lu path=%s",
 		__entry->ruleset_id, __entry->ruleset_version,
 		__print_flags(__entry->access_rights, "|", _LANDLOCK_ACCESS_FS_NAMES),
 		MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino,
@@ -427,31 +453,33 @@ TRACE_EVENT(landlock_add_rule_fs,
 );
 
 /**
- * landlock_add_rule_net - Network port rule added to a ruleset
+ * landlock_add_rule_net_port - Network-port rule added to a ruleset
  *
  * @ruleset: Source ruleset (never NULL).
- * @access_rights: Effective access mask stored in the rule, not the raw
- *                 sys_landlock_add_rule() argument (unhandled rights
- *                 added).
- * @port: Network port, the landlock_net_port_attr.port UAPI value
- *        forwarded directly.
+ * @flags: Complete validated landlock_add_rule_flags value supplied by this
+ *         successful call, not the rule's accumulated quiet state.
+ * @access_rights: Canonical per-call access mask passed to
+ *                 landlock_insert_rule() after normalization, not the raw
+ *                 sys_landlock_add_rule() argument or accumulated rule.
+ * @port: Network port in host endianness, forwarded directly from
+ *        &landlock_net_port_attr.port.
  *
  * Emitted by sys_landlock_add_rule() under the modified ruleset's lock, so
  * the reported ruleset is a stable snapshot that no concurrent writer can
  * change.
  */
-TRACE_EVENT(landlock_add_rule_net,
+TRACE_EVENT(landlock_add_rule_net_port,
 
-	TP_PROTO(const struct landlock_ruleset *ruleset,
-		 access_mask_t access_rights, __u64 port),
+	TP_PROTO(const struct landlock_ruleset *ruleset, u32 flags,
+		 u64 access_rights, u64 port),
 
-	TP_ARGS(ruleset, access_rights, port),
+	TP_ARGS(ruleset, flags, access_rights, port),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		ruleset_id	)
-		__field(	__u32,		ruleset_version	)
+		__field(	u64,		ruleset_id	)
+		__field(	u64,		ruleset_version	)
 		__field(	access_mask_t,	access_rights	)
-		__field(	__u64,		port		)
+		__field(	u64,		port		)
 	),
 
 	TP_fast_assign(
@@ -462,7 +490,7 @@ TRACE_EVENT(landlock_add_rule_net,
 		__entry->port		= port;
 	),
 
-	TP_printk("ruleset=%llx.%u access_rights=%s port=%llu",
+	TP_printk("ruleset=%llx.%llu access_rights=%s port=%llu",
 		__entry->ruleset_id, __entry->ruleset_version,
 		__print_flags(__entry->access_rights, "|", _LANDLOCK_ACCESS_NET_NAMES),
 		__entry->port)
@@ -495,10 +523,10 @@ TRACE_EVENT(landlock_create_domain,
 	TP_ARGS(domain, ruleset),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
-		__field(	__u64,		parent_id	)
-		__field(	__u64,		ruleset_id	)
-		__field(	__u32,		ruleset_version	)
+		__field(	u64,		domain_id	)
+		__field(	u64,		parent_id	)
+		__field(	u64,		ruleset_id	)
+		__field(	u64,		ruleset_version	)
 	),
 
 	TP_fast_assign(
@@ -510,7 +538,7 @@ TRACE_EVENT(landlock_create_domain,
 		__entry->ruleset_version = ruleset->version;
 	),
 
-	TP_printk("domain=%llx parent=%llx ruleset=%llx.%u",
+	TP_printk("domain=%llx parent=%llx ruleset=%llx.%llu",
 		__entry->domain_id, __entry->parent_id,
 		__entry->ruleset_id, __entry->ruleset_version)
 );
@@ -557,7 +585,7 @@ TRACE_EVENT(landlock_enforce_domain,
 	TP_ARGS(domain, complete, process_wide, no_new_privs),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	bool,		complete	)
 		__field(	bool,		process_wide	)
 		__field(	bool,		no_new_privs	)
@@ -595,8 +623,8 @@ TRACE_EVENT(landlock_free_domain,
 	TP_ARGS(hierarchy),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
-		__field(	__u64,		denials		)
+		__field(	u64,		domain_id	)
+		__field(	u64,		denials		)
 	),
 
 	TP_fast_assign(
@@ -609,7 +637,7 @@ TRACE_EVENT(landlock_free_domain,
 );
 
 /**
- * landlock_check_rule_fs - Filesystem rule evaluated during access check
+ * landlock_check_rule_inode - Inode rule evaluated during access check
  *
  * @domain: Enforcing domain (never NULL).
  * @rule: Matching rule with per-layer access masks (never NULL).
@@ -622,16 +650,16 @@ TRACE_EVENT(landlock_free_domain,
  * domain layer.  See Documentation/trace/events-landlock.rst for how to
  * interpret it.
  */
-TRACE_EVENT(landlock_check_rule_fs,
+TRACE_EVENT(landlock_check_rule_inode,
 
 	TP_PROTO(const struct landlock_domain *domain,
 		 const struct landlock_rule *rule,
-		 access_mask_t access_request, const struct dentry *dentry),
+		 u64 access_request, const struct dentry *dentry),
 
 	TP_ARGS(domain, rule, access_request, dentry),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	access_mask_t,	access_request	)
 		__field(	dev_t,		dev		)
 		__field(	ino_t,		ino		)
@@ -648,7 +676,8 @@ TRACE_EVENT(landlock_check_rule_fs,
 		__trace_landlock_fill_layers(__get_dynamic_array(grants),
 					     __get_dynamic_array_len(grants) /
 						     sizeof(access_mask_t),
-					     rule, access_request);
+					     rule,
+					     (access_mask_t)access_request);
 	),
 
 	TP_printk("domain=%llx access_request=%s dev=%u:%u ino=%lu grants=%s",
@@ -659,7 +688,7 @@ TRACE_EVENT(landlock_check_rule_fs,
 );
 
 /**
- * landlock_check_rule_net - Network port rule evaluated during access check
+ * landlock_check_rule_net_port - Network port rule evaluated
  *
  * @domain: Enforcing domain (never NULL).
  * @rule: Matching rule with per-layer access masks (never NULL).
@@ -671,18 +700,18 @@ TRACE_EVENT(landlock_check_rule_fs,
  * layer.  See Documentation/trace/events-landlock.rst for how to
  * interpret it.
  */
-TRACE_EVENT(landlock_check_rule_net,
+TRACE_EVENT(landlock_check_rule_net_port,
 
 	TP_PROTO(const struct landlock_domain *domain,
 		 const struct landlock_rule *rule,
-		 access_mask_t access_request, __u64 port),
+		 u64 access_request, u64 port),
 
 	TP_ARGS(domain, rule, access_request, port),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	access_mask_t,	access_request	)
-		__field(	__u64,		port		)
+		__field(	u64,		port		)
 		__dynamic_array(access_mask_t,	grants,
 				domain->num_layers)
 	),
@@ -695,7 +724,8 @@ TRACE_EVENT(landlock_check_rule_net,
 		__trace_landlock_fill_layers(__get_dynamic_array(grants),
 					     __get_dynamic_array_len(grants) /
 						     sizeof(access_mask_t),
-					     rule, access_request);
+					     rule,
+					     (access_mask_t)access_request);
 	),
 
 	TP_printk("domain=%llx access_request=%s port=%llu grants=%s",
@@ -712,8 +742,7 @@ TRACE_EVENT(landlock_check_rule_net,
  *             domain field.
  * @same_exec: Whether the current task entered the denying domain itself.
  * @logged: The domain's audit-logging decision for this denial.
- * @blockers: Access mask that was blocked (zero for a mount-topology
- *            change, whose only blocker is the operation itself).
+ * @blockers: Request type and final missing access subset (never NULL).
  * @path: Filesystem path that was denied (never NULL).
  * @pathname: Resolved path string (never NULL; an error placeholder on
  *            resolution failure).
@@ -723,16 +752,17 @@ TRACE_EVENT(landlock_check_rule_net,
 TRACE_EVENT(landlock_deny_access_fs,
 
 	TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec,
-		 bool logged, access_mask_t blockers, const struct path *path,
-		 const char *pathname),
+		 bool logged, const struct landlock_blockers *blockers,
+		 const struct path *path, const char *pathname),
 
 	TP_ARGS(hierarchy, same_exec, logged, blockers, path, pathname),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	bool,		same_exec	)
 		__field(	bool,		logged		)
-		__field(	access_mask_t,	blockers	)
+		__field(	enum landlock_request_type, blockers_type	)
+		__field(	access_mask_t,	blockers_access	)
 		__field(	dev_t,		dev		)
 		__field(	ino_t,		ino		)
 		__string(	pathname,	pathname	)
@@ -744,7 +774,8 @@ TRACE_EVENT(landlock_deny_access_fs,
 		__entry->domain_id	= hierarchy->id;
 		__entry->same_exec	= same_exec;
 		__entry->logged		= logged;
-		__entry->blockers	= blockers;
+		__entry->blockers_type	= blockers->type;
+		__entry->blockers_access = blockers->access;
 		__entry->dev		= path->dentry->d_sb->s_dev;
 		/*
 		 * A negative dentry has no backing inode, so mirror the
@@ -756,12 +787,20 @@ TRACE_EVENT(landlock_deny_access_fs,
 
 	TP_printk("domain=%llx same_exec=%d logged=%d blockers=%s dev=%u:%u ino=%lu path=%s",
 		__entry->domain_id, __entry->same_exec, __entry->logged,
-		__print_flags(__entry->blockers, "|", _LANDLOCK_ACCESS_FS_NAMES),
+		__entry->blockers_type == LANDLOCK_REQUEST_FS_ACCESS ?
+			__print_flags(__entry->blockers_access, "|", _LANDLOCK_ACCESS_FS_NAMES) :
+			__print_symbolic(__entry->blockers_type,
+					 _LANDLOCK_FS_BLOCKER_TYPE_NAMES),
 		MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino,
 		__trace_print_untrusted_str(p, __get_str(pathname),
 					    __get_dynamic_array_len(pathname) - 1))
 );
 
+static_assert(offsetof(struct sockaddr_in, sin_port) ==
+	      offsetof(struct sockaddr_in6, sin6_port));
+static_assert(sizeof_field(struct sockaddr_in, sin_port) ==
+	      sizeof_field(struct sockaddr_in6, sin6_port));
+
 /**
  * landlock_deny_access_net - Network access denied
  *
@@ -769,54 +808,69 @@ TRACE_EVENT(landlock_deny_access_fs,
  *             domain field.
  * @same_exec: Whether the current task entered the denying domain itself.
  * @logged: The domain's audit-logging decision for this denial.
- * @blockers: Access mask that was blocked.
- * @sk: Socket object (never NULL), read without a socket lock, so its
- *      fields are a best-effort snapshot.  The denied endpoint is not
- *      available: the hook runs before :manpage:`bind(2)` /
- *      :manpage:`connect(2)` sets the socket addresses.
- * @sport: Source port in host endianness, set for bind denials (zero for
- *         an autobind/ephemeral port); zero for connect and send denials.
- * @dport: Destination port in host endianness, set for connect and send
- *         denials; zero for bind denials, and also zero for a UDP send to
- *         an AF_UNSPEC address on an IPv6 socket (indistinguishable from a
- *         real destination port 0).  The bind-vs-connect direction is
- *         given by @blockers, not by which port is set.
- *
- * Emitted when a Landlock domain denies a network operation.
- *
- * The port fields are converted from the socket's network byte order to
- * host endianness before emitting.
+ * @blockers: Request type and final missing access subset (never NULL).
+ * @sk: Socket object (never NULL), read without a socket lock, so its fields
+ *      are a best-effort snapshot.
+ * @socket_family: Socket-family snapshot used by the verdict.
+ * @address: Authoritative address checked by the verdict (never NULL).
+ *           The producer copies @addrlen bytes from the checked address and
+ *           zeroes the remaining storage before emission.  The
+ *           &sockaddr_in.sin_port or &sockaddr_in6.sin6_port member, when
+ *           present, remains in network endianness.
+ * @addrlen: Validated signed length of @address.
+ *
+ * Emitted when a Landlock domain denies a network operation.  The blocker
+ * identifies whether the address is a bind or connect/send policy object.
+ * The flattened port field is converted from the checked address to host
+ * endianness, or is -1 when no port was checked.  Zero is a valid checked
+ * port.
  */
 TRACE_EVENT(landlock_deny_access_net,
 
 	TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec,
-		 bool logged, access_mask_t blockers, const struct sock *sk,
-		 __u64 sport, __u64 dport),
+		 bool logged, const struct landlock_blockers *blockers,
+		 const struct sock *sk, u16 socket_family,
+		 const struct sockaddr_storage *address, int addrlen),
 
-	TP_ARGS(hierarchy, same_exec, logged, blockers, sk, sport, dport),
+	TP_ARGS(hierarchy, same_exec, logged, blockers, sk, socket_family,
+		address, addrlen),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	bool,		same_exec	)
 		__field(	bool,		logged		)
-		__field(	access_mask_t,	blockers	)
-		__field(	__u64,		sport		)
-		__field(	__u64,		dport		)
+		__field(	enum landlock_request_type, blockers_type	)
+		__field(	access_mask_t,	blockers_access	)
+		__field(	s64,		port		)
 	),
 
 	TP_fast_assign(
+		const struct sockaddr *const addr =
+			(const struct sockaddr *)address;
+		const bool has_port =
+			addrlen >= (int)offsetofend(struct sockaddr_in, sin_port) &&
+			(addr->sa_family == AF_INET ||
+			 addr->sa_family == AF_INET6 ||
+			 (addr->sa_family == AF_UNSPEC &&
+			  socket_family == AF_INET));
+
 		__entry->domain_id	= hierarchy->id;
 		__entry->same_exec	= same_exec;
 		__entry->logged		= logged;
-		__entry->blockers	= blockers;
-		__entry->sport		= sport;
-		__entry->dport		= dport;
+		__entry->blockers_type	= blockers->type;
+		__entry->blockers_access = blockers->access;
+		__entry->port		=
+			has_port ?
+				ntohs(((const struct sockaddr_in *)addr)->sin_port) :
+				-1;
 	),
 
-	TP_printk("domain=%llx same_exec=%d logged=%d blockers=%s sport=%llu dport=%llu",
-		__entry->domain_id, __entry->same_exec, __entry->logged,
-		__print_flags(__entry->blockers, "|", _LANDLOCK_ACCESS_NET_NAMES),
-		__entry->sport, __entry->dport)
+	TP_printk("domain=%llx same_exec=%d logged=%d blockers=%s port=%lld",
+		  __entry->domain_id, __entry->same_exec, __entry->logged,
+		  __entry->blockers_type == LANDLOCK_REQUEST_NET_ACCESS ?
+			  __print_flags(__entry->blockers_access, "|", _LANDLOCK_ACCESS_NET_NAMES) :
+			  "unknown",
+		  __entry->port)
 );
 
 /**
@@ -824,12 +878,14 @@ TRACE_EVENT(landlock_deny_access_net,
  *
  * @hierarchy: Denying domain's hierarchy node (never NULL); its id is the
  *             domain field.
- * @same_exec: Whether the current task entered the denying domain itself.
+ * @same_exec: Whether the tracer entered the denying domain itself.
  * @logged: The domain's audit-logging decision for this denial.
  * @tracee_domain_id: The tracee's Landlock domain ID, or 0 if the tracee
  *                    is unsandboxed.
  * @tracee: The target task ptrace acted on (never NULL).  tracee_pid is
  *          the init-namespace TGID (like audit's opid).
+ * @tracer: The tracer or proposed tracer (never NULL); for PTRACE_TRACEME
+ *          this is the parent, not the syscall caller.
  *
  * Emitted when a Landlock domain denies a ptrace operation.
  */
@@ -837,15 +893,16 @@ TRACE_EVENT(landlock_deny_ptrace,
 
 	TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec,
 		 bool logged, u64 tracee_domain_id,
-		 const struct task_struct *tracee),
+		 const struct task_struct *tracee,
+		 const struct task_struct *tracer),
 
-	TP_ARGS(hierarchy, same_exec, logged, tracee_domain_id, tracee),
+	TP_ARGS(hierarchy, same_exec, logged, tracee_domain_id, tracee, tracer),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	bool,		same_exec	)
 		__field(	bool,		logged		)
-		__field(	__u64,		tracee_domain_id)
+		__field(	u64,		tracee_domain_id)
 		__field(	pid_t,		tracee_pid	)
 		__string(	tracee_comm,	tracee->comm	)
 	),
@@ -872,12 +929,14 @@ TRACE_EVENT(landlock_deny_ptrace,
  *
  * @hierarchy: Denying domain's hierarchy node (never NULL); its id is the
  *             domain field.
- * @same_exec: Whether the current task entered the denying domain itself.
+ * @same_exec: Whether the policy subject entered the denying domain itself.
  * @logged: The domain's audit-logging decision for this denial.
  * @target_domain_id: The target's Landlock domain ID, or 0 if the target
  *                    is unsandboxed.
  * @target: The task the signal was aimed at (never NULL).  target_pid is
  *          the init-namespace TGID (like audit's opid).
+ * @signal: The signal selected by the denied check.  Zero is a permission
+ *          probe, not an absent value.
  *
  * Emitted when a Landlock domain denies signal delivery to a scoped-out
  * target.
@@ -886,15 +945,15 @@ TRACE_EVENT(landlock_deny_scope_signal,
 
 	TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec,
 		 bool logged, u64 target_domain_id,
-		 const struct task_struct *target),
+		 const struct task_struct *target, int signal),
 
-	TP_ARGS(hierarchy, same_exec, logged, target_domain_id, target),
+	TP_ARGS(hierarchy, same_exec, logged, target_domain_id, target, signal),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	bool,		same_exec	)
 		__field(	bool,		logged		)
-		__field(	__u64,		target_domain_id)
+		__field(	u64,		target_domain_id)
 		__field(	pid_t,		target_pid	)
 		__string(	target_comm,	target->comm	)
 	),
@@ -940,10 +999,10 @@ TRACE_EVENT(landlock_deny_scope_abstract_unix_socket,
 	TP_ARGS(hierarchy, same_exec, logged, peer_domain_id, peer),
 
 	TP_STRUCT__entry(
-		__field(	__u64,		domain_id	)
+		__field(	u64,		domain_id	)
 		__field(	bool,		same_exec	)
 		__field(	bool,		logged		)
-		__field(	__u64,		peer_domain_id	)
+		__field(	u64,		peer_domain_id	)
 		__field(	pid_t,		peer_pid	)
 		/*
 		 * Abstract socket names are untrusted binary data from
@@ -973,11 +1032,10 @@ TRACE_EVENT(landlock_deny_scope_abstract_unix_socket,
 		__entry->logged		= logged;
 		__entry->peer_domain_id	= peer_domain_id;
 		/*
-		 * Best-effort (0 for a datagram peer).  sk_peer_pid is
-		 * canonically guarded by sk->sk_peer_lock, but the target
-		 * peer's peercred is set once and not updated concurrently in
-		 * these hooks, so this READ_ONCE() is safe; sun_path is the
-		 * reliable identifier.
+		 * Best-effort (0 for a datagram peer).  The caller holds the
+		 * peer's AF_UNIX state lock, serializing published peercred
+		 * updates.  The peer socket keeps a reference to sk_peer_pid
+		 * through pid_nr(); sun_path is the reliable identifier.
 		 */
 		peer_pid		= READ_ONCE(peer->sk_peer_pid);
 		__entry->peer_pid	= peer_pid ? pid_nr(peer_pid) : 0;
@@ -991,6 +1049,7 @@ TRACE_EVENT(landlock_deny_scope_abstract_unix_socket,
 					    __get_dynamic_array_len(sun_path) - 1))
 );
 
+#undef _LANDLOCK_FS_BLOCKER_TYPE_NAMES
 #undef _LANDLOCK_NAME_ENTRY
 
 #endif /* _TRACE_LANDLOCK_H */
diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h
index 935893e5ea53bc..3c153b3ad84507 100644
--- a/include/trace/events/mmflags.h
+++ b/include/trace/events/mmflags.h
@@ -144,7 +144,7 @@ TRACE_DEFINE_ENUM(___GFP_LAST_BIT);
 	DEF_PAGEFLAG_NAME(owner_2),					\
 	DEF_PAGEFLAG_NAME(arch_1),					\
 	DEF_PAGEFLAG_NAME(reserved),					\
-	DEF_PAGEFLAG_NAME(private),					\
+	DEF_PAGEFLAG_NAME(folio),					\
 	DEF_PAGEFLAG_NAME(private_2),					\
 	DEF_PAGEFLAG_NAME(writeback),					\
 	DEF_PAGEFLAG_NAME(head),					\
@@ -168,6 +168,10 @@ IF_HAVE_PG_ARCH_3(arch_3)
 #define __VM_ARCH_SPECIFIC_1 {VM_SAO,     "sao"           }
 #elif defined(CONFIG_PARISC)
 #define __VM_ARCH_SPECIFIC_1 {VM_GROWSUP,	"growsup"	}
+#elif defined(CONFIG_SPARC64)
+#define __VM_ARCH_SPECIFIC_1 {VM_SPARC_ADI, "adi"		}
+#elif defined(CONFIG_ARM64)
+#define __VM_ARCH_SPECIFIC_1 {VM_ARM64_BTI, "bti"		}
 #elif !defined(CONFIG_MMU)
 #define __VM_ARCH_SPECIFIC_1 {VM_MAPPED_COPY,"mappedcopy"	}
 #else
@@ -198,6 +202,37 @@ IF_HAVE_PG_ARCH_3(arch_3)
 # define IF_HAVE_VM_DROPPABLE(flag, name)
 #endif
 
+#ifdef CONFIG_ARCH_HAS_PKEYS
+# define IF_HAVE_VM_PKEY(flag, name) {flag, name},
+#if CONFIG_ARCH_PKEY_BITS > 3
+# define IF_HAVE_VM_PKEY3(flag, name) {flag, name},
+#else
+# define IF_HAVE_VM_PKEY3(flag, name)
+#endif
+#if CONFIG_ARCH_PKEY_BITS > 4
+# define IF_HAVE_VM_PKEY4(flag, name) {flag, name},
+#else
+# define IF_HAVE_VM_PKEY4(flag, name)
+#endif
+#else
+# define IF_HAVE_VM_PKEY(flag, name)
+# define IF_HAVE_VM_PKEY3(flag, name)
+# define IF_HAVE_VM_PKEY4(flag, name)
+#endif
+
+#ifdef CONFIG_ARM64_MTE
+# define IF_HAVE_VM_MTE(flag, name) {flag, name},
+#else
+# define IF_HAVE_VM_MTE(flag, name)
+#endif
+
+#if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_RISCV_USER_CFI) || \
+	defined(CONFIG_ARM64_GCS)
+# define IF_HAVE_VM_SHADOW_STACK(flag, name) {flag, name},
+#else
+# define IF_HAVE_VM_SHADOW_STACK(flag, name)
+#endif
+
 #define __def_vmaflag_names						\
 	{VM_READ,			"read"		},		\
 	{VM_WRITE,			"write"		},		\
@@ -233,6 +268,14 @@ IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY,	"softdirty"	)		\
 	{VM_HUGEPAGE,			"hugepage"	},		\
 	{VM_NOHUGEPAGE,			"nohugepage"	},		\
 IF_HAVE_VM_DROPPABLE(VM_DROPPABLE,	"droppable"	)		\
+IF_HAVE_VM_PKEY(VM_PKEY_BIT0,		"pkey_bit0")		\
+IF_HAVE_VM_PKEY(VM_PKEY_BIT1,		"pkey_bit1")		\
+IF_HAVE_VM_PKEY(VM_PKEY_BIT2,		"pkey_bit2")		\
+IF_HAVE_VM_PKEY3(VM_PKEY_BIT3,		"pkey_bit3")		\
+IF_HAVE_VM_PKEY4(VM_PKEY_BIT4,		"pkey_bit4")		\
+IF_HAVE_VM_MTE(VM_MTE,			"mte")			\
+IF_HAVE_VM_MTE(VM_MTE_ALLOWED,		"mte_allowed")		\
+IF_HAVE_VM_SHADOW_STACK(VM_SHADOW_STACK, "shadow_stack")	\
 	{VM_MERGEABLE,			"mergeable"	}		\
 
 #define show_vma_flags(flags)						\
diff --git a/include/trace/events/module.h b/include/trace/events/module.h
index e5a006be9dc66c..19df3e39bba437 100644
--- a/include/trace/events/module.h
+++ b/include/trace/events/module.h
@@ -26,7 +26,8 @@ struct module;
 	{ (1UL << TAINT_OOT_MODULE),		"O" },		\
 	{ (1UL << TAINT_FORCED_MODULE),		"F" },		\
 	{ (1UL << TAINT_CRAP),			"C" },		\
-	{ (1UL << TAINT_UNSIGNED_MODULE),	"E" })
+	{ (1UL << TAINT_UNSIGNED_MODULE),	"E" },		\
+	{ (1UL << TAINT_FORCED_BIND),		"Y" })
 
 TRACE_EVENT(module_load,
 
diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h
index 3fec3e8f91c859..bf1e1f185b05c4 100644
--- a/include/trace/events/netfs.h
+++ b/include/trace/events/netfs.h
@@ -30,8 +30,7 @@
 	EM(netfs_write_trace_dio_write,		"DIO-WRITE")	\
 	EM(netfs_write_trace_unbuffered_write,	"UNB-WRITE")	\
 	EM(netfs_write_trace_writeback,		"WRITEBACK")	\
-	EM(netfs_write_trace_writeback_single,	"WB-SINGLE") \
-	E_(netfs_write_trace_writethrough,	"WRITETHRU")
+	E_(netfs_write_trace_writeback_single,	"WB-SINGLE")
 
 #define netfs_rreq_origins					\
 	EM(NETFS_READAHEAD,			"RA")		\
@@ -43,13 +42,17 @@
 	EM(NETFS_DIO_READ,			"DR")		\
 	EM(NETFS_WRITEBACK,			"WB")		\
 	EM(NETFS_WRITEBACK_SINGLE,		"W1")		\
-	EM(NETFS_WRITETHROUGH,			"WT")		\
 	EM(NETFS_UNBUFFERED_WRITE,		"UW")		\
 	EM(NETFS_DIO_WRITE,			"DW")		\
 	E_(NETFS_PGPRIV2_COPY_TO_CACHE,		"2C")
 
 #define netfs_rreq_traces					\
+	EM(netfs_rreq_trace_all_queued,		"ALL-Q  ")	\
 	EM(netfs_rreq_trace_assess,		"ASSESS ")	\
+	EM(netfs_rreq_trace_cache_cancelled,	"CA-CNCL")	\
+	EM(netfs_rreq_trace_cache_failed,	"CA-FAIL")	\
+	EM(netfs_rreq_trace_cache_fail_collect,	"CA-F-CO")	\
+	EM(netfs_rreq_trace_cache_no_space,	"CA-NOSP")	\
 	EM(netfs_rreq_trace_collect,		"COLLECT")	\
 	EM(netfs_rreq_trace_complete,		"COMPLET")	\
 	EM(netfs_rreq_trace_copy,		"COPY   ")	\
@@ -58,11 +61,14 @@
 	EM(netfs_rreq_trace_end_copy_to_cache,	"END-C2C")	\
 	EM(netfs_rreq_trace_free,		"FREE   ")	\
 	EM(netfs_rreq_trace_intr,		"INTR   ")	\
+	EM(netfs_rreq_trace_inval_cache,	"INVL-CA")	\
 	EM(netfs_rreq_trace_ki_complete,	"KI-CMPL")	\
 	EM(netfs_rreq_trace_ra_put_ref,		"RA-PUT ")	\
 	EM(netfs_rreq_trace_recollect,		"RECLLCT")	\
 	EM(netfs_rreq_trace_redirty,		"REDIRTY")	\
 	EM(netfs_rreq_trace_resubmit,		"RESUBMT")	\
+	EM(netfs_rreq_trace_retry_begin,	"RETRY-BEGIN")	\
+	EM(netfs_rreq_trace_retry_end,		"RETRY-END")	\
 	EM(netfs_rreq_trace_set_abandon,	"S-ABNDN")	\
 	EM(netfs_rreq_trace_set_pause,		"PAUSE  ")	\
 	EM(netfs_rreq_trace_unlock,		"UNLOCK ")	\
@@ -94,8 +100,10 @@
 	EM(netfs_sreq_trace_abandoned,		"ABNDN")	\
 	EM(netfs_sreq_trace_add_donations,	"+DON ")	\
 	EM(netfs_sreq_trace_added,		"ADD  ")	\
+	EM(netfs_sreq_trace_cache_nofile,	"CA-!F")	\
 	EM(netfs_sreq_trace_cache_nowrite,	"CA-NW")	\
 	EM(netfs_sreq_trace_cache_prepare,	"CA-PR")	\
+	EM(netfs_sreq_trace_cache_waitfail,	"CA-!W")	\
 	EM(netfs_sreq_trace_cache_write,	"CA-WR")	\
 	EM(netfs_sreq_trace_cancel,		"CANCL")	\
 	EM(netfs_sreq_trace_clear,		"CLEAR")	\
@@ -134,12 +142,12 @@
 
 #define netfs_failures							\
 	EM(netfs_fail_check_write_begin,	"check-write-begin")	\
-	EM(netfs_fail_copy_to_cache,		"copy-to-cache")	\
 	EM(netfs_fail_dio_read_short,		"dio-read-short")	\
 	EM(netfs_fail_dio_read_zero,		"dio-read-zero")	\
 	EM(netfs_fail_read,			"read")			\
 	EM(netfs_fail_short_read,		"short-read")		\
 	EM(netfs_fail_prepare_write,		"prep-write")		\
+	EM(netfs_fail_upload,			"upload")		\
 	E_(netfs_fail_write,			"write")
 
 #define netfs_rreq_ref_traces					\
@@ -194,11 +202,11 @@
 	EM(netfs_folio_trace_alloc_buffer,	"alloc-buf")	\
 	EM(netfs_folio_trace_cancel_copy,	"cancel-copy")	\
 	EM(netfs_folio_trace_cancel_store,	"cancel-store")	\
-	EM(netfs_folio_trace_clear,		"clear")	\
-	EM(netfs_folio_trace_clear_cc,		"clear-cc")	\
-	EM(netfs_folio_trace_clear_g,		"clear-g")	\
-	EM(netfs_folio_trace_clear_s,		"clear-s")	\
 	EM(netfs_folio_trace_end_copy,		"end-copy")	\
+	EM(netfs_folio_trace_endwb,		"endwb")	\
+	EM(netfs_folio_trace_endwb_cc,		"endwb-cc")	\
+	EM(netfs_folio_trace_endwb_g,		"endwb-g")	\
+	EM(netfs_folio_trace_endwb_s,		"endwb-s")	\
 	EM(netfs_folio_trace_filled_gaps,	"filled-gaps")	\
 	EM(netfs_folio_trace_invalidate_all,	"inval-all")	\
 	EM(netfs_folio_trace_invalidate_front,	"inval-front")	\
@@ -223,9 +231,7 @@
 	EM(netfs_folio_trace_sched_copy,	"sched-copy")	\
 	EM(netfs_folio_trace_store,		"store")	\
 	EM(netfs_folio_trace_store_copy,	"store-copy")	\
-	EM(netfs_folio_trace_store_plus,	"store+")	\
-	EM(netfs_folio_trace_wthru,		"wthru")	\
-	E_(netfs_folio_trace_wthru_plus,	"wthru+")
+	E_(netfs_folio_trace_store_plus,	"store+")
 
 #define netfs_collect_contig_traces				\
 	EM(netfs_contig_trace_collect,		"Collect")	\
@@ -301,7 +307,7 @@ netfs_folioq_traces;
 
 TRACE_EVENT(netfs_read,
 	    TP_PROTO(struct netfs_io_request *rreq,
-		     loff_t start, size_t len,
+		     uoff_t start, size_t len,
 		     enum netfs_read_trace what),
 
 	    TP_ARGS(rreq, start, len, what),
@@ -309,8 +315,9 @@ TRACE_EVENT(netfs_read,
 	    TP_STRUCT__entry(
 		    __field(unsigned int,		rreq)
 		    __field(unsigned int,		cookie)
-		    __field(loff_t,			i_size)
-		    __field(loff_t,			start)
+		    __field(unsigned int,		object)
+		    __field(uoff_t,			i_size)
+		    __field(uoff_t,			start)
 		    __field(size_t,			len)
 		    __field(enum netfs_read_trace,	what)
 		    __field(u64,			netfs_inode)
@@ -318,7 +325,8 @@ TRACE_EVENT(netfs_read,
 
 	    TP_fast_assign(
 		    __entry->rreq	= rreq->debug_id;
-		    __entry->cookie	= rreq->cache_resources.debug_id;
+		    __entry->cookie	= rreq->cache_resources.cookie_id;
+		    __entry->object	= rreq->cache_resources.object_id;
 		    __entry->i_size	= rreq->i_size;
 		    __entry->start	= start;
 		    __entry->len	= len;
@@ -326,10 +334,10 @@ TRACE_EVENT(netfs_read,
 		    __entry->netfs_inode = rreq->inode->i_ino;
 			   ),
 
-	    TP_printk("R=%08x %s c=%08x ni=%llx s=%llx l=%zx sz=%llx",
+	    TP_printk("R=%08x %s c=%08x o=%08x ni=%llx s=%llx l=%zx sz=%llx",
 		      __entry->rreq,
 		      __print_symbolic(__entry->what, netfs_read_traces),
-		      __entry->cookie,
+		      __entry->cookie, __entry->object,
 		      __entry->netfs_inode,
 		      __entry->start, __entry->len, __entry->i_size)
 	    );
@@ -377,7 +385,7 @@ TRACE_EVENT(netfs_sreq,
 		    __field(u8,				slot)
 		    __field(size_t,			len)
 		    __field(size_t,			transferred)
-		    __field(loff_t,			start)
+		    __field(uoff_t,			start)
 			     ),
 
 	    TP_fast_assign(
@@ -418,7 +426,7 @@ TRACE_EVENT(netfs_failure,
 		    __field(enum netfs_failure,		what)
 		    __field(size_t,			len)
 		    __field(size_t,			transferred)
-		    __field(loff_t,			start)
+		    __field(uoff_t,			start)
 			     ),
 
 	    TP_fast_assign(
@@ -501,6 +509,7 @@ TRACE_EVENT(netfs_folio,
 	    TP_STRUCT__entry(
 		    __field(u64,			ino)
 		    __field(pgoff_t,			index)
+		    __field(unsigned long,		pfn)
 		    __field(unsigned int,		nr)
 		    __field(enum netfs_folio_trace,	why)
 			     ),
@@ -511,9 +520,11 @@ TRACE_EVENT(netfs_folio,
 		    __entry->why = why;
 		    __entry->index = folio->index;
 		    __entry->nr = folio_nr_pages(folio);
+		    __entry->pfn = folio_pfn(folio);
 			   ),
 
-	    TP_printk("i=%05llx ix=%05lx-%05lx %s",
+	    TP_printk("p=%lx i=%05llx ix=%05lx-%05lx %s",
+		      __entry->pfn,
 		      __entry->ino, __entry->index, __entry->index + __entry->nr - 1,
 		      __print_symbolic(__entry->why, netfs_folio_traces))
 	    );
@@ -524,10 +535,10 @@ TRACE_EVENT(netfs_write_iter,
 	    TP_ARGS(iocb, from),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned long long,		start)
-		    __field(size_t,			len)
-		    __field(unsigned int,		flags)
-		    __field(unsigned int,		ino)
+		    __field(uoff_t,		start)
+		    __field(size_t,		len)
+		    __field(unsigned int,	flags)
+		    __field(unsigned int,	ino)
 			     ),
 
 	    TP_fast_assign(
@@ -550,27 +561,27 @@ TRACE_EVENT(netfs_write,
 	    TP_STRUCT__entry(
 		    __field(unsigned int,		wreq)
 		    __field(unsigned int,		cookie)
+		    __field(unsigned int,		object)
 		    __field(unsigned int,		ino)
 		    __field(enum netfs_write_trace,	what)
-		    __field(unsigned long long,		start)
-		    __field(unsigned long long,		len)
+		    __field(uoff_t,			start)
+		    __field(uoff_t,			len)
 			     ),
 
 	    TP_fast_assign(
-		    struct netfs_inode *__ctx = netfs_inode(wreq->inode);
-		    struct fscache_cookie *__cookie = netfs_i_cookie(__ctx);
 		    __entry->wreq	= wreq->debug_id;
-		    __entry->cookie	= __cookie ? __cookie->debug_id : 0;
+		    __entry->cookie	= wreq->cache_resources.cookie_id;
+		    __entry->object	= wreq->cache_resources.object_id;
 		    __entry->ino	= wreq->inode->i_ino;
 		    __entry->what	= what;
 		    __entry->start	= wreq->start;
 		    __entry->len	= wreq->len;
 			   ),
 
-	    TP_printk("R=%08x %s c=%08x i=%x by=%llx-%llx",
+	    TP_printk("R=%08x %s c=%08x o=%08x i=%x by=%llx-%llx",
 		      __entry->wreq,
 		      __print_symbolic(__entry->what, netfs_write_traces),
-		      __entry->cookie,
+		      __entry->cookie, __entry->object,
 		      __entry->ino,
 		      __entry->start, __entry->start + __entry->len - 1)
 	    );
@@ -582,25 +593,26 @@ TRACE_EVENT(netfs_copy2cache,
 	    TP_ARGS(rreq, creq),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		rreq)
-		    __field(unsigned int,		creq)
-		    __field(unsigned int,		cookie)
-		    __field(unsigned int,		ino)
+		    __field(unsigned int,	rreq)
+		    __field(unsigned int,	creq)
+		    __field(unsigned int,	cookie)
+		    __field(unsigned int,	object)
+		    __field(unsigned int,	ino)
 			     ),
 
 	    TP_fast_assign(
-		    struct netfs_inode *__ctx = netfs_inode(rreq->inode);
-		    struct fscache_cookie *__cookie = netfs_i_cookie(__ctx);
 		    __entry->rreq	= rreq->debug_id;
 		    __entry->creq	= creq->debug_id;
-		    __entry->cookie	= __cookie ? __cookie->debug_id : 0;
+		    __entry->cookie	= rreq->cache_resources.cookie_id;
+		    __entry->object	= rreq->cache_resources.object_id;
 		    __entry->ino	= rreq->inode->i_ino;
 			   ),
 
-	    TP_printk("R=%08x CR=%08x c=%08x i=%x ",
+	    TP_printk("R=%08x CR=%08x c=%08x o=%08x i=%x ",
 		      __entry->rreq,
 		      __entry->creq,
 		      __entry->cookie,
+		      __entry->object,
 		      __entry->ino)
 	    );
 
@@ -610,10 +622,10 @@ TRACE_EVENT(netfs_collect,
 	    TP_ARGS(wreq),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		wreq)
-		    __field(unsigned int,		len)
-		    __field(unsigned long long,		transferred)
-		    __field(unsigned long long,		start)
+		    __field(unsigned int,	wreq)
+		    __field(unsigned int,	len)
+		    __field(uoff_t,		transferred)
+		    __field(uoff_t,		start)
 			     ),
 
 	    TP_fast_assign(
@@ -636,12 +648,12 @@ TRACE_EVENT(netfs_collect_sreq,
 	    TP_ARGS(wreq, subreq),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		wreq)
-		    __field(unsigned int,		subreq)
-		    __field(unsigned int,		stream)
-		    __field(unsigned int,		len)
-		    __field(unsigned int,		transferred)
-		    __field(unsigned long long,		start)
+		    __field(unsigned int,	wreq)
+		    __field(unsigned int,	subreq)
+		    __field(unsigned int,	stream)
+		    __field(unsigned int,	len)
+		    __field(unsigned int,	transferred)
+		    __field(uoff_t,		start)
 			     ),
 
 	    TP_fast_assign(
@@ -660,37 +672,30 @@ TRACE_EVENT(netfs_collect_sreq,
 
 TRACE_EVENT(netfs_collect_folio,
 	    TP_PROTO(const struct netfs_io_request *wreq,
-		     const struct folio *folio,
-		     unsigned long long fend,
-		     unsigned long long collected_to),
+		     const struct folio *folio),
 
-	    TP_ARGS(wreq, folio, fend, collected_to),
+	    TP_ARGS(wreq, folio),
 
 	    TP_STRUCT__entry(
 		    __field(unsigned int,	wreq)
 		    __field(unsigned long,	index)
-		    __field(unsigned long long,	fend)
-		    __field(unsigned long long,	cleaned_to)
-		    __field(unsigned long long,	collected_to)
+		    __field(unsigned int,	nr)
 			     ),
 
 	    TP_fast_assign(
 		    __entry->wreq	= wreq->debug_id;
 		    __entry->index	= folio->index;
-		    __entry->fend	= fend;
-		    __entry->cleaned_to	= wreq->cleaned_to;
-		    __entry->collected_to = collected_to;
+		    __entry->nr		= folio_nr_pages(folio);
 			   ),
 
-	    TP_printk("R=%08x ix=%05lx r=%llx-%llx t=%llx/%llx",
+	    TP_printk("R=%08x ix=%05lx-%05lx",
 		      __entry->wreq, __entry->index,
-		      (unsigned long long)__entry->index * PAGE_SIZE, __entry->fend,
-		      __entry->cleaned_to, __entry->collected_to)
+		      __entry->index + __entry->nr - 1)
 	    );
 
 TRACE_EVENT(netfs_collect_state,
 	    TP_PROTO(const struct netfs_io_request *wreq,
-		     unsigned long long collected_to,
+		     uoff_t collected_to,
 		     unsigned int notes),
 
 	    TP_ARGS(wreq, collected_to, notes),
@@ -698,8 +703,8 @@ TRACE_EVENT(netfs_collect_state,
 	    TP_STRUCT__entry(
 		    __field(unsigned int,	wreq)
 		    __field(unsigned int,	notes)
-		    __field(unsigned long long,	collected_to)
-		    __field(unsigned long long,	cleaned_to)
+		    __field(uoff_t,		collected_to)
+		    __field(uoff_t,		cleaned_to)
 			     ),
 
 	    TP_fast_assign(
@@ -718,7 +723,7 @@ TRACE_EVENT(netfs_collect_state,
 TRACE_EVENT(netfs_collect_gap,
 	    TP_PROTO(const struct netfs_io_request *wreq,
 		     const struct netfs_io_stream *stream,
-		     unsigned long long jump_to, char type),
+		     uoff_t jump_to, char type),
 
 	    TP_ARGS(wreq, stream, jump_to, type),
 
@@ -726,8 +731,8 @@ TRACE_EVENT(netfs_collect_gap,
 		    __field(unsigned int,	wreq)
 		    __field(unsigned char,	stream)
 		    __field(unsigned char,	type)
-		    __field(unsigned long long,	from)
-		    __field(unsigned long long,	to)
+		    __field(uoff_t,		from)
+		    __field(uoff_t,		to)
 			     ),
 
 	    TP_fast_assign(
@@ -752,8 +757,8 @@ TRACE_EVENT(netfs_collect_stream,
 	    TP_STRUCT__entry(
 		    __field(unsigned int,	wreq)
 		    __field(unsigned char,	stream)
-		    __field(unsigned long long,	collected_to)
-		    __field(unsigned long long,	issued_to)
+		    __field(uoff_t,		collected_to)
+		    __field(uoff_t,		issued_to)
 			     ),
 
 	    TP_fast_assign(
diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h
index 36c3a90f0accad..5d47b774633a4e 100644
--- a/include/trace/events/pagemap.h
+++ b/include/trace/events/pagemap.h
@@ -22,7 +22,7 @@
 	(folio_test_swapcache(folio)	? PAGEMAP_SWAPCACHE  : 0) | \
 	(folio_test_swapbacked(folio)	? PAGEMAP_SWAPBACKED : 0) | \
 	(folio_test_mappedtodisk(folio)	? PAGEMAP_MAPPEDDISK : 0) | \
-	(folio_test_private(folio)	? PAGEMAP_BUFFERS    : 0) \
+	(folio_has_attached_private(folio)	? PAGEMAP_BUFFERS    : 0) \
 	)
 
 TRACE_EVENT(mm_lru_insertion,
diff --git a/include/trace/events/scmi.h b/include/trace/events/scmi.h
index 703b7bb68e4429..b421c253df0032 100644
--- a/include/trace/events/scmi.h
+++ b/include/trace/events/scmi.h
@@ -7,8 +7,6 @@
 
 #include <linux/tracepoint.h>
 
-#define TRACE_SCMI_MAX_TAG_LEN	6
-
 TRACE_EVENT(scmi_fc_call,
 	TP_PROTO(u8 protocol_id, u8 msg_id, u32 res_id, u32 val1, u32 val2),
 	TP_ARGS(protocol_id, msg_id, res_id, val1, val2),
@@ -156,7 +154,7 @@ TRACE_EVENT(scmi_msg_dump,
 		__field(u8, channel_id)
 		__field(u8, protocol_id)
 		__field(u8, msg_id)
-		__array(char, tag, TRACE_SCMI_MAX_TAG_LEN)
+		__string(tag, tag)
 		__field(u16, seq)
 		__field(int, status)
 		__field(size_t, len)
@@ -168,7 +166,7 @@ TRACE_EVENT(scmi_msg_dump,
 		__entry->channel_id = channel_id;
 		__entry->protocol_id = protocol_id;
 		__entry->msg_id = msg_id;
-		strscpy(__entry->tag, tag, TRACE_SCMI_MAX_TAG_LEN);
+		__assign_str(tag);
 		__entry->seq = seq;
 		__entry->status = status;
 		__entry->len = len;
@@ -177,7 +175,7 @@ TRACE_EVENT(scmi_msg_dump,
 
 	TP_printk("id=%d ch=%02X pt=%02X t=%s msg_id=%02X seq=%04X s=%d pyld=%s",
 		  __entry->id, __entry->channel_id, __entry->protocol_id,
-		  __entry->tag, __entry->msg_id, __entry->seq, __entry->status,
+		  __get_str(tag), __entry->msg_id, __entry->seq, __entry->status,
 		__print_hex_str(__get_dynamic_array(cmd), __entry->len))
 );
 #endif /* _TRACE_SCMI_H */
diff --git a/include/trace/events/scsi.h b/include/trace/events/scsi.h
index c36c72ab7f2bf6..e8328df587c3e4 100644
--- a/include/trace/events/scsi.h
+++ b/include/trace/events/scsi.h
@@ -303,8 +303,8 @@ DECLARE_EVENT_CLASS(scsi_cmd_done_timeout_template,
 		if (cmd->sense_buffer && SCSI_SENSE_VALID(cmd) &&
 		    scsi_command_normalize_sense(cmd, &sshdr)) {
 			__entry->sense_key = sshdr.sense_key;
-			__entry->asc = sshdr.asc;
-			__entry->ascq = sshdr.ascq;
+			__entry->asc = scsi_sense_asc(&sshdr);
+			__entry->ascq = scsi_sense_ascq(&sshdr);
 		} else {
 			__entry->sense_key = 0;
 			__entry->asc = 0;
diff --git a/include/trace/events/tegra_apb_dma.h b/include/trace/events/tegra_apb_dma.h
index 6d9f5075baa3e4..c38117c127ef31 100644
--- a/include/trace/events/tegra_apb_dma.h
+++ b/include/trace/events/tegra_apb_dma.h
@@ -11,7 +11,7 @@ TRACE_EVENT(tegra_dma_tx_status,
 	TP_PROTO(struct dma_chan *dc, dma_cookie_t cookie, struct dma_tx_state *state),
 	TP_ARGS(dc, cookie, state),
 	TP_STRUCT__entry(
-		__string(chan,	dev_name(&dc->dev->device))
+		__string(chan,	dma_chan_name(dc))
 		__field(dma_cookie_t, cookie)
 		__field(__u32,	residue)
 	),
@@ -28,7 +28,7 @@ TRACE_EVENT(tegra_dma_complete_cb,
 	TP_PROTO(struct dma_chan *dc, int count, void *ptr),
 	TP_ARGS(dc, count, ptr),
 	TP_STRUCT__entry(
-		__string(chan,	dev_name(&dc->dev->device))
+		__string(chan,	dma_chan_name(dc))
 		__field(int,	count)
 		__field(void *,	ptr)
 		),
@@ -45,7 +45,7 @@ TRACE_EVENT(tegra_dma_isr,
 	TP_PROTO(struct dma_chan *dc, int irq),
 	TP_ARGS(dc, irq),
 	TP_STRUCT__entry(
-		__string(chan,	dev_name(&dc->dev->device))
+		__string(chan,	dma_chan_name(dc))
 		__field(int,	irq)
 	),
 	TP_fast_assign(
diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h
index b4bf7b8def1f5f..8a872990b4bee6 100644
--- a/include/trace/events/vmscan.h
+++ b/include/trace/events/vmscan.h
@@ -214,13 +214,6 @@ DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_reclaim_be
 
 	TP_ARGS(gfp_flags, order, memcg)
 );
-
-DEFINE_EVENT(mm_vmscan_direct_reclaim_begin_template, mm_vmscan_memcg_softlimit_reclaim_begin,
-
-	TP_PROTO(gfp_t gfp_flags, int order, struct mem_cgroup *memcg),
-
-	TP_ARGS(gfp_flags, order, memcg)
-);
 #endif /* CONFIG_MEMCG */
 
 DECLARE_EVENT_CLASS(mm_vmscan_direct_reclaim_end_template,
@@ -260,13 +253,6 @@ DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_reclaim_end,
 
 	TP_ARGS(nr_reclaimed, memcg)
 );
-
-DEFINE_EVENT(mm_vmscan_direct_reclaim_end_template, mm_vmscan_memcg_softlimit_reclaim_end,
-
-	TP_PROTO(unsigned long nr_reclaimed, struct mem_cgroup *memcg),
-
-	TP_ARGS(nr_reclaimed, memcg)
-);
 #endif /* CONFIG_MEMCG */
 
 TRACE_EVENT(mm_shrink_slab_start,
diff --git a/include/trace/events/workqueue.h b/include/trace/events/workqueue.h
index b0de2bc9ed52fa..5b820ff8f81a41 100644
--- a/include/trace/events/workqueue.h
+++ b/include/trace/events/workqueue.h
@@ -9,6 +9,7 @@
 #include <linux/workqueue.h>
 
 struct pool_workqueue;
+struct worker_pool;
 
 /**
  * workqueue_queue_work - called when a work gets queued
@@ -126,7 +127,152 @@ TRACE_EVENT(workqueue_execute_end,
 	TP_printk("work struct %p: function %ps", __entry->work, __entry->function)
 );
 
+/**
+ * workqueue_cpu_intensive - called when a work item exceeds cpu_intensive threshold
+ * @pwq:	pointer to struct pool_workqueue
+ * @work:	pointer to struct work_struct
+ * @function:	pointer to worker function
+ * @duration_us: CPU time consumed in microseconds
+ *
+ * This event occurs when a concurrency-managed work item runs for longer
+ * than wq_cpu_intensive_thresh_us without sleeping and is excluded from
+ * concurrency management to prevent stalling other work items.
+ */
+TRACE_EVENT(workqueue_cpu_intensive,
+
+	TP_PROTO(struct pool_workqueue *pwq, struct work_struct *work,
+		 work_func_t function, u64 duration_us),
+
+	TP_ARGS(pwq, work, function, duration_us),
+
+	TP_STRUCT__entry(
+		__field( void *,	work		)
+		__field( void *,	function	)
+		__string( workqueue,	pwq->wq->name	)
+		__field( int,		cpu		)
+		__field( u64,		duration_us	)
+	),
+
+	TP_fast_assign(
+		__entry->work		= work;
+		__entry->function	= function;
+		__assign_str(workqueue);
+		__entry->cpu		= pwq->pool->cpu;
+		__entry->duration_us	= duration_us;
+	),
+
+	TP_printk("work struct=%p function=%ps workqueue=%s cpu=%d duration_us=%llu",
+		  __entry->work, __entry->function, __get_str(workqueue),
+		  __entry->cpu, __entry->duration_us)
+);
+
+/**
+ * workqueue_mayday - called when a pool_workqueue sends mayday to rescuer
+ * @pwq:	pointer to struct pool_workqueue
+ *
+ * This event occurs when a worker pool fails to create a new worker and
+ * requests the workqueue's rescuer thread to process pending works.
+ */
+TRACE_EVENT(workqueue_mayday,
+
+	TP_PROTO(struct pool_workqueue *pwq),
+
+	TP_ARGS(pwq),
+
+	TP_STRUCT__entry(
+		__string( workqueue,	pwq->wq->name	)
+		__field( int,		pool_id		)
+		__field( int,		cpu		)
+		__field( int,		nr_active	)
+	),
+
+	TP_fast_assign(
+		__assign_str(workqueue);
+		__entry->pool_id	= pwq->pool->id;
+		__entry->cpu		= pwq->pool->cpu;
+		__entry->nr_active	= pwq->nr_active;
+	),
+
+	TP_printk("workqueue=%s pool_id=%d cpu=%d nr_active=%d",
+		  __get_str(workqueue), __entry->pool_id, __entry->cpu,
+		  __entry->nr_active)
+);
+
+/**
+ * workqueue_rescued - called when a work item is assigned to a rescuer
+ * @pwq:	pointer to struct pool_workqueue
+ * @work:	pointer to struct work_struct
+ * @function:	pointer to worker function
+ *
+ * This event occurs when a work item is claimed by a rescuer thread
+ * to guarantee forward progress.
+ */
+TRACE_EVENT(workqueue_rescued,
+
+	TP_PROTO(struct pool_workqueue *pwq, struct work_struct *work,
+		 work_func_t function),
+
+	TP_ARGS(pwq, work, function),
+
+	TP_STRUCT__entry(
+		__field( void *,	work		)
+		__field( void *,	function	)
+		__string( workqueue,	pwq->wq->name	)
+		__field( int,		cpu		)
+	),
+
+	TP_fast_assign(
+		__entry->work		= work;
+		__entry->function	= function;
+		__assign_str(workqueue);
+		__entry->cpu		= pwq->pool->cpu;
+	),
+
+	TP_printk("work struct=%p function=%ps workqueue=%s cpu=%d",
+		  __entry->work, __entry->function, __get_str(workqueue),
+		  __entry->cpu)
+);
+
+/**
+ * workqueue_bh_budget_yield - called when a BH worker yields due to budget exhaustion
+ * @pool:	pointer to struct worker_pool
+ * @restarts:	number of restarts executed
+ * @timeout:	whether execution hit the time limit (BH_WORKER_JIFFIES)
+ * @highpri:	whether this is a high-priority BH pool
+ *
+ * This event occurs when a bottom-half (BH) worker pool running in softirq
+ * context exhausts its execution time slice or restart limit and must yield
+ * execution.
+ */
+TRACE_EVENT(workqueue_bh_budget_yield,
+
+	TP_PROTO(struct worker_pool *pool, int restarts, bool timeout, bool highpri),
+
+	TP_ARGS(pool, restarts, timeout, highpri),
+
+	TP_STRUCT__entry(
+		__field( int,	pool_id		)
+		__field( int,	cpu		)
+		__field( int,	restarts	)
+		__field( bool,	timeout		)
+		__field( bool,	highpri		)
+	),
+
+	TP_fast_assign(
+		__entry->pool_id	= pool->id;
+		__entry->cpu		= pool->cpu;
+		__entry->restarts	= restarts;
+		__entry->timeout	= timeout;
+		__entry->highpri	= highpri;
+	),
+
+	TP_printk("pool_id=%d cpu=%d restarts=%d timeout=%d highpri=%d",
+		  __entry->pool_id, __entry->cpu, __entry->restarts,
+		  __entry->timeout, __entry->highpri)
+);
+
 #endif /*  _TRACE_WORKQUEUE_H */
 
 /* This part must be outside protection */
 #include <trace/define_trace.h>
+
diff --git a/include/trace/misc/nfs.h b/include/trace/misc/nfs.h
index a394b4d38e18fa..3146813fc4fe02 100644
--- a/include/trace/misc/nfs.h
+++ b/include/trace/misc/nfs.h
@@ -8,6 +8,7 @@
  */
 
 #include <linux/nfs.h>
+#include <linux/nfs3.h>
 #include <linux/nfs4.h>
 #include <uapi/linux/nfs.h>
 
@@ -358,18 +359,6 @@ TRACE_DEFINE_ENUM(IOMODE_ANY);
 		{ IOMODE_RW,			"RW" }, \
 		{ IOMODE_ANY,			"ANY" })
 
-#define show_rca_mask(x) \
-	__print_flags(x, "|", \
-		{ BIT(RCA4_TYPE_MASK_RDATA_DLG),	"RDATA_DLG" }, \
-		{ BIT(RCA4_TYPE_MASK_WDATA_DLG),	"WDATA_DLG" }, \
-		{ BIT(RCA4_TYPE_MASK_DIR_DLG),		"DIR_DLG" }, \
-		{ BIT(RCA4_TYPE_MASK_FILE_LAYOUT),	"FILE_LAYOUT" }, \
-		{ BIT(RCA4_TYPE_MASK_BLK_LAYOUT),	"BLK_LAYOUT" }, \
-		{ BIT(RCA4_TYPE_MASK_OBJ_LAYOUT_MIN),	"OBJ_LAYOUT_MIN" }, \
-		{ BIT(RCA4_TYPE_MASK_OBJ_LAYOUT_MAX),	"OBJ_LAYOUT_MAX" }, \
-		{ BIT(RCA4_TYPE_MASK_OTHER_LAYOUT_MIN),	"OTHER_LAYOUT_MIN" }, \
-		{ BIT(RCA4_TYPE_MASK_OTHER_LAYOUT_MAX),	"OTHER_LAYOUT_MAX" })
-
 #define show_nfs4_seq4_status(x) \
 	__print_flags(x, "|", \
 		{ SEQ4_STATUS_CB_PATH_DOWN,		"CB_PATH_DOWN" }, \
diff --git a/include/uapi/asm-generic/setup.h b/include/uapi/asm-generic/setup.h
index 88ac5100df3598..b8d06e6d56bd7e 100644
--- a/include/uapi/asm-generic/setup.h
+++ b/include/uapi/asm-generic/setup.h
@@ -2,6 +2,10 @@
 #ifndef __ASM_GENERIC_SETUP_H
 #define __ASM_GENERIC_SETUP_H
 
+#ifdef __KERNEL__
+#define COMMAND_LINE_SIZE	CONFIG_COMMAND_LINE_SIZE
+#else
 #define COMMAND_LINE_SIZE	512
+#endif
 
 #endif	/* __ASM_GENERIC_SETUP_H */
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 42a5fa8ad6b0ab..cd1b8034a2fb47 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -59,6 +59,7 @@ extern "C" {
 #define DRM_AMDGPU_USERQ_WAIT		0x18
 #define DRM_AMDGPU_GEM_LIST_HANDLES	0x19
 #define DRM_AMDGPU_PROC_OPTIONS		0x1A
+#define DRM_AMDGPU_UALINK_HANDLE	0x1B
 
 #define DRM_IOCTL_AMDGPU_GEM_CREATE	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_CREATE, union drm_amdgpu_gem_create)
 #define DRM_IOCTL_AMDGPU_GEM_MMAP	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_MMAP, union drm_amdgpu_gem_mmap)
@@ -81,6 +82,7 @@ extern "C" {
 #define DRM_IOCTL_AMDGPU_USERQ_WAIT	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_USERQ_WAIT, struct drm_amdgpu_userq_wait)
 #define DRM_IOCTL_AMDGPU_GEM_LIST_HANDLES DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_GEM_LIST_HANDLES, struct drm_amdgpu_gem_list_handles)
 #define DRM_IOCTL_AMDGPU_PROC_OPTIONS	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_PROC_OPTIONS, struct drm_amdgpu_proc_options)
+#define DRM_IOCTL_AMDGPU_UALINK_HANDLE DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_UALINK_HANDLE, union drm_amdgpu_ualink_handle)
 
 /**
  * DOC: memory domains
@@ -115,6 +117,12 @@ extern "C" {
 #define AMDGPU_GEM_DOMAIN_GWS		0x10
 #define AMDGPU_GEM_DOMAIN_OA		0x20
 #define AMDGPU_GEM_DOMAIN_DOORBELL	0x40
+#define AMDGPU_GEM_DOMAIN_MMIO_REMAP	0x80
+#define AMDGPU_GEM_DOMAIN_NPA		0x100
+
+/* User-mode is not allowed to allocate NPA space. As a result,
+ * we don't add AMDGPU_GEM_DOMAIN_NPA in the DOMAIN_MASK below.
+ */
 #define AMDGPU_GEM_DOMAIN_MASK		(AMDGPU_GEM_DOMAIN_CPU | \
 					 AMDGPU_GEM_DOMAIN_GTT | \
 					 AMDGPU_GEM_DOMAIN_VRAM | \
@@ -504,6 +512,11 @@ struct drm_amdgpu_userq_signal {
 	 * @bo_write_handles.
 	 */
 	__u32	num_bo_write_handles;
+	/**
+	 * @syncobj_points: The list of syncobj points submitted by the user queue job
+	 * for the corresponding @syncobj_handles.
+	 */
+	__u64	syncobj_points;
 };
 
 struct drm_amdgpu_userq_fence_info {
@@ -1694,6 +1707,34 @@ struct drm_amdgpu_proc_options {
 	} kfd_sigbus_delay;
 };
 
+#define DRM_AMDGPU_UALINK_HANDLE_OP_EXPORT		0
+#define DRM_AMDGPU_UALINK_HANDLE_OP_IMPORT		1
+
+struct drm_amdgpu_ualink_handle_in {
+	/* Export or import */
+	__u32 op;
+	/* For future use, no flags defined so far */
+	__u32 flags;
+	union {
+		/* GEM handle of the BO to export */
+		__u32 gem_handle;
+		/* UALink handle to import */
+		__u64 import_ualink_handle[2];
+	};
+};
+
+union drm_amdgpu_ualink_handle_out {
+	/* Exported UALink handle */
+	__u64 export_ualink_handle[2];
+	/** DMABuf representing the imported UALink handle */
+	__u32 import_dmabuf_handle;
+};
+
+union drm_amdgpu_ualink_handle {
+	struct drm_amdgpu_ualink_handle_in in;
+	union drm_amdgpu_ualink_handle_out out;
+};
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index bd435effdcee65..ed1a660a3dfda0 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -955,6 +955,18 @@ enum drm_colorop_type {
 	 *         color = lut3d[index]
 	 */
 	DRM_COLOROP_3D_LUT,
+
+	/**
+	 * @DRM_COLOROP_FIXED_MATRIX:
+	 *
+	 * enum string "Fixed Matrix"
+	 *
+	 * A Colorop block that performs a pre-defined matrix operation selected
+	 * via the FIXED_MATRIX_TYPE enum property. The driver advertises the supported
+	 * operations through this property.
+	 */
+	DRM_COLOROP_FIXED_MATRIX,
+
 };
 
 /**
diff --git a/include/uapi/drm/drm_ras.h b/include/uapi/drm/drm_ras.h
index 218a3ee8680527..2f832275ee6efc 100644
--- a/include/uapi/drm/drm_ras.h
+++ b/include/uapi/drm/drm_ras.h
@@ -33,18 +33,36 @@ enum {
 	DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID,
 	DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_NAME,
 	DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_VALUE,
+	DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_THRESHOLD,
 
 	__DRM_RAS_A_ERROR_COUNTER_ATTRS_MAX,
 	DRM_RAS_A_ERROR_COUNTER_ATTRS_MAX = (__DRM_RAS_A_ERROR_COUNTER_ATTRS_MAX - 1)
 };
 
+enum {
+	DRM_RAS_A_ERROR_EVENT_ATTRS_DEVICE_NAME = 1,
+	DRM_RAS_A_ERROR_EVENT_ATTRS_NODE_ID,
+	DRM_RAS_A_ERROR_EVENT_ATTRS_NODE_NAME,
+	DRM_RAS_A_ERROR_EVENT_ATTRS_ERROR_ID,
+	DRM_RAS_A_ERROR_EVENT_ATTRS_ERROR_NAME,
+	DRM_RAS_A_ERROR_EVENT_ATTRS_ERROR_VALUE,
+
+	__DRM_RAS_A_ERROR_EVENT_ATTRS_MAX,
+	DRM_RAS_A_ERROR_EVENT_ATTRS_MAX = (__DRM_RAS_A_ERROR_EVENT_ATTRS_MAX - 1)
+};
+
 enum {
 	DRM_RAS_CMD_LIST_NODES = 1,
 	DRM_RAS_CMD_GET_ERROR_COUNTER,
 	DRM_RAS_CMD_CLEAR_ERROR_COUNTER,
+	DRM_RAS_CMD_ERROR_EVENT,
+	DRM_RAS_CMD_GET_ERROR_THRESHOLD,
+	DRM_RAS_CMD_SET_ERROR_THRESHOLD,
 
 	__DRM_RAS_CMD_MAX,
 	DRM_RAS_CMD_MAX = (__DRM_RAS_CMD_MAX - 1)
 };
 
+#define DRM_RAS_MCGRP_ERROR_REPORT	"error-report"
+
 #endif /* _UAPI_LINUX_DRM_RAS_H */
diff --git a/include/uapi/drm/ivpu_accel.h b/include/uapi/drm/ivpu_accel.h
index 264505d54f932e..9ce19d9c3131be 100644
--- a/include/uapi/drm/ivpu_accel.h
+++ b/include/uapi/drm/ivpu_accel.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #ifndef __UAPI_IVPU_DRM_H__
@@ -96,6 +96,7 @@ extern "C" {
 #define DRM_IVPU_PARAM_SKU		    12
 #define DRM_IVPU_PARAM_CAPABILITIES	    13
 #define DRM_IVPU_PARAM_PREEMPT_BUFFER_SIZE  14
+#define DRM_IVPU_PARAM_CMDQ_PRIORITY	    15
 
 #define DRM_IVPU_PLATFORM_TYPE_SILICON	    0
 
@@ -140,6 +141,15 @@ extern "C" {
  */
 #define DRM_IVPU_CAP_BO_CREATE_FROM_USERPTR	4
 
+/**
+ * DRM_IVPU_CAP_CMDQ_SET_PRIORITY
+ *
+ * Driver supports DRM_IVPU_PARAM_CMDQ_PRIORITY parameter to get/set command
+ * queue priority.
+ * This allows changing command queue priority after creation.
+ */
+#define DRM_IVPU_CAP_CMDQ_SET_PRIORITY	5
+
 /**
  * struct drm_ivpu_param - Get/Set VPU parameters
  */
@@ -192,6 +202,11 @@ struct drm_ivpu_param {
 	 *
 	 * %DRM_IVPU_PARAM_PREEMPT_BUFFER_SIZE:
 	 * Size of the preemption buffer (read-only)
+	 *
+	 * %DRM_IVPU_PARAM_CMDQ_PRIORITY:
+	 * Get/Set command queue priority (see DRM_IVPU_JOB_PRIORITY_*).
+	 * DRM_IVPU_JOB_PRIORITY_DEFAULT is treated as DRM_IVPU_JOB_PRIORITY_NORMAL.
+	 * Command queue ID is given via drm_ivpu_param.index.
 	 */
 	__u32 param;
 
diff --git a/include/uapi/drm/nova_drm.h b/include/uapi/drm/nova_drm.h
index 3ca90ed9d2bb1a..a74daba6ea94a2 100644
--- a/include/uapi/drm/nova_drm.h
+++ b/include/uapi/drm/nova_drm.h
@@ -25,6 +25,58 @@ extern "C" {
  */
 #define NOVA_GETPARAM_VRAM_BAR_SIZE	0x1
 
+/**
+ * enum drm_nova_architecture - GPU architecture identifier
+ */
+enum drm_nova_architecture {
+	NOVA_DRM_ARCHITECTURE_TURING		= 0x16,
+	NOVA_DRM_ARCHITECTURE_AMPERE		= 0x17,
+	NOVA_DRM_ARCHITECTURE_HOPPER		= 0x18,
+	NOVA_DRM_ARCHITECTURE_ADA		= 0x19,
+	NOVA_DRM_ARCHITECTURE_BLACKWELL_GB10X	= 0x1a,
+	NOVA_DRM_ARCHITECTURE_BLACKWELL_GB20X	= 0x1b,
+};
+
+/**
+ * enum drm_nova_chipid - opaque GPU chip identifier
+ *
+ * These values identify the chip a GPU is based on. They are otherwise
+ * opaque: userspace must not assume the values carry any particular meaning
+ * or encoding, only that they may be compared against this enum.
+ */
+enum drm_nova_chipid {
+	/* Turing */
+	NOVA_DRM_CHIPID_TU102			= 0x162,
+	NOVA_DRM_CHIPID_TU104			= 0x164,
+	NOVA_DRM_CHIPID_TU106			= 0x166,
+	NOVA_DRM_CHIPID_TU117			= 0x167,
+	NOVA_DRM_CHIPID_TU116			= 0x168,
+	/* Ampere */
+	NOVA_DRM_CHIPID_GA100			= 0x170,
+	NOVA_DRM_CHIPID_GA102			= 0x172,
+	NOVA_DRM_CHIPID_GA103			= 0x173,
+	NOVA_DRM_CHIPID_GA104			= 0x174,
+	NOVA_DRM_CHIPID_GA106			= 0x176,
+	NOVA_DRM_CHIPID_GA107			= 0x177,
+	/* Hopper */
+	NOVA_DRM_CHIPID_GH100			= 0x180,
+	/* Ada */
+	NOVA_DRM_CHIPID_AD102			= 0x192,
+	NOVA_DRM_CHIPID_AD103			= 0x193,
+	NOVA_DRM_CHIPID_AD104			= 0x194,
+	NOVA_DRM_CHIPID_AD106			= 0x196,
+	NOVA_DRM_CHIPID_AD107			= 0x197,
+	/* Blackwell GB10x */
+	NOVA_DRM_CHIPID_GB100			= 0x1a0,
+	NOVA_DRM_CHIPID_GB102			= 0x1a2,
+	/* Blackwell GB20x */
+	NOVA_DRM_CHIPID_GB202			= 0x1b2,
+	NOVA_DRM_CHIPID_GB203			= 0x1b3,
+	NOVA_DRM_CHIPID_GB205			= 0x1b5,
+	NOVA_DRM_CHIPID_GB206			= 0x1b6,
+	NOVA_DRM_CHIPID_GB207			= 0x1b7,
+};
+
 /**
  * struct drm_nova_getparam - query GPU and driver metadata
  */
@@ -80,9 +132,83 @@ struct drm_nova_gem_info {
 	__u64 size;
 };
 
+/**
+ * struct drm_nova_info - query device information
+ */
+struct drm_nova_info {
+	/**
+	 * @id: The identifier of the information to query.
+	 */
+	__u32 id;
+
+	/**
+	 * @size: The amount of space allocated by userspace at @data. The kernel
+	 * will return the number of bytes it wrote, or the size of the queried
+	 * information structure if @data is NULL.
+	 */
+	__u32 size;
+
+	/**
+	 * @data: Pointer to the userspace buffer into which the queried
+	 * information will be written. May be NULL, in which case nothing is
+	 * written and @size is set to the size of the information structure so
+	 * userspace can allocate a suitably sized buffer.
+	 */
+	__u64 data;
+};
+
+/**
+ * DRM_NOVA_INFO_GPU
+ *
+ * Query GPU information. The result is returned in a
+ * &struct drm_nova_info_gpu.
+ */
+#define DRM_NOVA_INFO_GPU		0x00
+
+/**
+ * struct drm_nova_info_gpu - GPU information
+ */
+struct drm_nova_info_gpu {
+	/**
+	 * @architecture: GPU architecture identifier.
+	 *
+	 * See &enum drm_nova_architecture for currently known architectures.
+	 */
+	__u32 architecture;
+
+	/**
+	 * @chipid: Opaque GPU chip identifier.
+	 *
+	 * See &enum drm_nova_chipid for currently known chips.
+	 */
+	__u32 chipid;
+
+	/**
+	 * @vram_size: Amount of usable FB, excluding GSP carveouts and protected
+	 * regions.
+	 */
+	__u64 vram_size;
+
+	/**
+	 * @gpu_name: NUL-terminated full GPU name.
+	 */
+	__u8 gpu_name[64];
+
+	/**
+	 * @gpu_short_name: NUL-terminated short GPU name.
+	 */
+	__u8 gpu_short_name[64];
+
+	/**
+	 * @gpu_gid: 16-byte SHA-1 GPU identifier supplied by GSP-RM.
+	 */
+	__u8 gpu_gid[16];
+};
+
 #define DRM_NOVA_GETPARAM		0x00
 #define DRM_NOVA_GEM_CREATE		0x01
 #define DRM_NOVA_GEM_INFO		0x02
+#define DRM_NOVA_INFO			0x03
 
 /* Note: this is an enum so that it can be resolved by Rust bindgen. */
 enum {
@@ -92,6 +218,8 @@ enum {
 						   struct drm_nova_gem_create),
 	DRM_IOCTL_NOVA_GEM_INFO		= DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_GEM_INFO,
 						   struct drm_nova_gem_info),
+	DRM_IOCTL_NOVA_INFO		= DRM_IOWR(DRM_COMMAND_BASE + DRM_NOVA_INFO,
+						   struct drm_nova_info),
 };
 
 #if defined(__cplusplus)
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index 509202a7b13eb5..578f0c1eb43e81 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -1294,8 +1294,8 @@ struct xe_vm_fault {
 #define FAULT_LEVEL_PML4	3
 #define FAULT_LEVEL_PML5	4
 	__u8 fault_level;
-	/** @pad: MBZ */
-	__u8 pad;
+	/** @srcid: ID of the faulting hardware unit */
+	__u8 srcid;
 	/** @reserved: MBZ */
 	__u64 reserved[4];
 };
@@ -1491,6 +1491,12 @@ struct drm_xe_exec_queue_destroy {
  *
  * The @property can be:
  *  - %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN
+ *
+ * For %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN, @value is a bitmask of ban reasons:
+ *  - %DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG - banned due to GPU hang/timeout
+ *  - %DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE - banned due to memory page offline
+ *
+ * A @value of 0 means the exec queue is not banned.
  */
 struct drm_xe_exec_queue_get_property {
 	/** @extensions: Pointer to the first extension struct, if any */
@@ -1503,7 +1509,17 @@ struct drm_xe_exec_queue_get_property {
 	/** @property: property to get */
 	__u32 property;
 
-	/** @value: property value */
+	/**
+	 * @value: property value
+	 *
+	 * For %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN, this is a bitmask of:
+	 *  - %DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG - banned due to GPU hang/timeout
+	 *  - %DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE - banned due to memory page offline
+	 *
+	 * Value of 0 means the exec queue is not banned.
+	 */
+#define DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG		(1 << 0)
+#define DRM_XE_EXEC_QUEUE_BAN_REASON_PAGE_OFFLINE	(1 << 1)
 	__u64 value;
 
 	/** @reserved: Reserved */
diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h
index e3ad94444864d5..7d4618bea04349 100644
--- a/include/uapi/linux/alloc_tag.h
+++ b/include/uapi/linux/alloc_tag.h
@@ -38,6 +38,7 @@ struct allocinfo_counter {
 	__u64 bytes;
 	__u64 calls;
 	__u8 accurate;
+	__u8 pad[7];
 } __attribute__((aligned(8)));
 
 struct allocinfo_tag_data {
diff --git a/include/uapi/linux/amd-pmf.h b/include/uapi/linux/amd-pmf.h
index 24cf9926dc5901..151f264cd7d6cc 100644
--- a/include/uapi/linux/amd-pmf.h
+++ b/include/uapi/linux/amd-pmf.h
@@ -199,6 +199,9 @@ struct amd_pmf_info {
 	/* BIOS parameters */
 	__u32 bios_input[AMD_PMF_BIOS_PARAMS_MAX];
 	__u32 bios_output[AMD_PMF_BIOS_PARAMS_MAX];
+
+	/* align to 64 bit boundary */
+	__u32 __pad;
 };
 
 #endif /* _UAPI_LINUX_AMD_PMF_H */
diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h
index 701cad36de43a4..96e5b0184a1b11 100644
--- a/include/uapi/linux/android/binder.h
+++ b/include/uapi/linux/android/binder.h
@@ -296,6 +296,7 @@ enum transaction_flags {
 	TF_ACCEPT_FDS	= 0x10,	/* allow replies with file descriptors */
 	TF_CLEAR_BUF	= 0x20,	/* clear buffer on txn complete */
 	TF_UPDATE_TXN	= 0x40,	/* update the outdated pending async txn */
+	TF_DEFER_COMPLETE = 0x80,	/* defer transaction complete to userspace */
 };
 
 struct binder_transaction_data {
diff --git a/include/uapi/linux/atm.h b/include/uapi/linux/atm.h
index 95ebdcf4fe8879..7a3789f39f5203 100644
--- a/include/uapi/linux/atm.h
+++ b/include/uapi/linux/atm.h
@@ -39,7 +39,7 @@
 #define ATM_MAX_VCI		 65535	/* maximum VCI */
 
 
-/* "protcol" values for the socket system call */
+/* "protocol" values for the socket system call */
 #define ATM_NO_AAL	0		/* AAL not specified */
 #define ATM_AAL0	13		/* "raw" ATM cells */
 #define ATM_AAL1	1		/* AAL1 (CBR) */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 732b35cc08d1c2..4687c331099683 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -936,6 +936,33 @@ union bpf_iter_link_info {
  * 		0 on success or -1 if an error occurred (in which case,
  * 		*errno* is set appropriately).
  *
+ * BPF_PROG_STREAM_OPEN
+ *	Description
+ *		Open a file descriptor for one of the BPF streams associated
+ *		with the program identified by *prog_fd*. The stream is selected
+ *		by *stream_id*.
+ *
+ *		The returned file descriptor supports **read**\ (2) and
+ *		**poll**\ (2). Reads block while the stream is empty unless
+ *		**BPF_F_STREAM_NONBLOCK** is specified in *flags*. A non-blocking
+ *		read of an empty stream fails with **EAGAIN**.
+ *
+ *		**poll**\ (2) reports **POLLIN** when data is available and
+ *		**POLLHUP** once the program has been freed, that is, after every
+ *		reference to it, including links and other file descriptors, has
+ *		been dropped. Hangup may lag the final release because program
+ *		teardown is deferred. Buffered data remains readable after
+ *		**POLLHUP** and a read returns zero after all such data has been
+ *		consumed.
+ *
+ *		The file descriptor is read-only and has the close-on-exec flag
+ *		set. It is not seekable and **lseek**\ (2) fails with **ESPIPE**.
+ *		*flags* may only contain **BPF_F_STREAM_NONBLOCK**.
+ *
+ *	Return
+ *		A new file descriptor (a nonnegative integer), or -1 if an
+ *		error occurred (in which case, *errno* is set appropriately).
+ *
  * NOTES
  *	eBPF objects (maps and programs) can be shared between processes.
  *
@@ -993,6 +1020,7 @@ enum bpf_cmd {
 	BPF_TOKEN_CREATE,
 	BPF_PROG_STREAM_READ_BY_FD,
 	BPF_PROG_ASSOC_STRUCT_OPS,
+	BPF_PROG_STREAM_OPEN,
 	__MAX_BPF_CMD,
 	BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */
 };
@@ -1524,6 +1552,11 @@ enum {
 	BPF_STREAM_STDERR = 2,
 };
 
+/* flags for BPF_PROG_STREAM_OPEN command */
+enum {
+	BPF_F_STREAM_NONBLOCK = (1U << 0),
+};
+
 union bpf_attr {
 	struct { /* anonymous struct used by BPF_MAP_CREATE command */
 		__u32	map_type;	/* one of enum bpf_map_type */
@@ -1756,7 +1789,7 @@ union bpf_attr {
 			__u32	prog_cnt;
 			__u32	count;
 		};
-		__u32		:32;
+		__u32		type_id;
 		/* output: per-program attach_flags.
 		 * not allowed to be set during effective query.
 		 */
@@ -1950,6 +1983,12 @@ union bpf_attr {
 		__u32		flags;
 	} prog_assoc_struct_ops;
 
+	struct {
+		__u32		prog_fd;
+		__u32		stream_id;
+		__u32		flags;
+	} prog_stream_open;
+
 } __attribute__((aligned(8)));
 
 /* The description below is an attempt at providing documentation to eBPF
@@ -4867,15 +4906,18 @@ union bpf_attr {
  * 		The non-negative copied *buf* length equal to or less than
  * 		*size* on success, or a negative error in case of failure.
  *
- * long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags)
+ * long bpf_load_hdr_opt(void *ctx, void *searchby_res, u32 len, u64 flags)
  *	Description
  *		Load header option.  Support reading a particular TCP header
- *		option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**).
+ *		option for bpf program (**BPF_PROG_TYPE_SOCK_OPS**).  For the
+ *		**bpf_tcp_ops** struct_ops, this helper can be called from the
+ *		**parse_hdr**\ () and **write_hdr_opt**\ () operators.
  *
- *		If *flags* is 0, it will search the option from the
- *		*skops*\ **->skb_data**.  The comment in **struct bpf_sock_ops**
- *		has details on what skb_data contains under different
- *		*skops*\ **->op**.
+ *		If *flags* is 0, it will search the option from the packet
+ *		associated with the current operation.  For
+ *		**BPF_PROG_TYPE_SOCK_OPS**, the comment in
+ *		**struct bpf_sock_ops** has details on what skb_data
+ *		contains under different *op*.
  *
  *		The first byte of the *searchby_res* specifies the
  *		kind that it wants to search.
@@ -4908,6 +4950,8 @@ union bpf_attr {
  *
  *		* **BPF_LOAD_HDR_OPT_TCP_SYN** to search from the
  *		  saved_syn packet or the just-received syn packet.
+ *		  Not supported by the **bpf_tcp_ops** struct_ops, which
+ *		  rejects all flags.
  *
  *	Return
  *		> 0 when found, the header option is copied to *searchby_res*.
@@ -4928,9 +4972,9 @@ union bpf_attr {
  *		packet.
  *
  *		**-EPERM** if the helper cannot be used under the current
- *		*skops*\ **->op**.
+ *		operation.
  *
- * long bpf_store_hdr_opt(struct bpf_sock_ops *skops, const void *from, u32 len, u64 flags)
+ * long bpf_store_hdr_opt(void *ctx, const void *from, u32 len, u64 flags)
  *	Description
  *		Store header option.  The data will be copied
  *		from buffer *from* with length *len* to the TCP header.
@@ -4946,7 +4990,9 @@ union bpf_attr {
  *		by searching the same option in the outgoing skb.
  *
  *		This helper can only be called during
- *		**BPF_SOCK_OPS_WRITE_HDR_OPT_CB**.
+ *		**BPF_SOCK_OPS_WRITE_HDR_OPT_CB**, or from the
+ *		**write_hdr_opt**\ () operator of the **bpf_tcp_ops**
+ *		struct_ops.
  *
  *	Return
  *		0 on success, or negative error in case of failure:
@@ -4961,9 +5007,9 @@ union bpf_attr {
  *		**-EFAULT** on failure to parse the existing header options.
  *
  *		**-EPERM** if the helper cannot be used under the current
- *		*skops*\ **->op**.
+ *		operation.
  *
- * long bpf_reserve_hdr_opt(struct bpf_sock_ops *skops, u32 len, u64 flags)
+ * long bpf_reserve_hdr_opt(void *ctx, u32 len, u64 flags)
  *	Description
  *		Reserve *len* bytes for the bpf header option.  The
  *		space will be used by **bpf_store_hdr_opt**\ () later in
@@ -4973,7 +5019,9 @@ union bpf_attr {
  *		the total number of bytes will be reserved.
  *
  *		This helper can only be called during
- *		**BPF_SOCK_OPS_HDR_OPT_LEN_CB**.
+ *		**BPF_SOCK_OPS_HDR_OPT_LEN_CB**, or from the
+ *		**hdr_opt_len**\ () operator of the **bpf_tcp_ops**
+ *		struct_ops.
  *
  *	Return
  *		0 on success, or negative error in case of failure:
@@ -4983,7 +5031,7 @@ union bpf_attr {
  *		**-ENOSPC** if there is not enough space in the header.
  *
  *		**-EPERM** if the helper cannot be used under the current
- *		*skops*\ **->op**.
+ *		operation.
  *
  * void *bpf_inode_storage_get(struct bpf_map *map, void *inode, void *value, u64 flags)
  *	Description
@@ -6890,6 +6938,8 @@ struct bpf_link_info {
 		} xdp;
 		struct {
 			__u32 map_id;
+			__u32 :32;
+			__u64 cgroup_id;
 		} struct_ops;
 		struct {
 			__u32 pf;
@@ -7600,6 +7650,10 @@ struct bpf_task_work {
 	__u64 __opaque;
 } __attribute__((aligned(8)));
 
+struct bpf_rcu_head {
+	__u64 __opaque[6];
+} __attribute__((aligned(8)));
+
 struct bpf_wq {
 	__u64 __opaque[2];
 } __attribute__((aligned(8)));
diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h
index 618167cab4e617..11d65871d163e5 100644
--- a/include/uapi/linux/btf.h
+++ b/include/uapi/linux/btf.h
@@ -54,8 +54,8 @@ struct btf_type {
 	 *             decl_tag and type_tag
 	 */
 	__u32 info;
-	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.
-	 * "size" tells the size of the type it is describing.
+	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC, ENUM64
+	 * and LOC_PARAM. "size" tells the size of the type it is describing.
 	 *
 	 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
 	 * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG.
@@ -92,7 +92,9 @@ enum {
 	BTF_KIND_DECL_TAG	= 17,	/* Decl Tag */
 	BTF_KIND_TYPE_TAG	= 18,	/* Type Tag */
 	BTF_KIND_ENUM64		= 19,	/* Enumeration up to 64-bit values */
-
+	BTF_KIND_LOC_PARAM	= 20,	/* Location parameter information */
+	BTF_KIND_LOC_PROTO	= 21,	/* Location prototype for site */
+	BTF_KIND_LOCSEC		= 22,	/* Location section */
 	NR_BTF_KINDS,
 	BTF_KIND_MAX		= NR_BTF_KINDS - 1,
 };
@@ -212,4 +214,79 @@ struct btf_enum64 {
 	__u32	val_hi32;
 };
 
+/*
+ * BTF_KIND_LOC_PARAM is followed by a single "struct btf_loc_param"
+ * that contains flags specifying the contents of the vlen-specified
+ * number of 4-byte values that follow.
+ */
+struct btf_loc_param {
+	__u32 flags;
+	__u32 values[];
+};
+
+/*
+ * The combination of size, vlen and flags gives us the means to interpret
+ * the following vlen-specified set of 4-byte values:
+ *
+ * - a BTF_LOC_PARAM_CONST is a constant value; combination
+ *   of size, vlen and _SIGNED flag determines it. If the value requires
+ *   64 bits it is stored in {lo,hi} order.
+ * - a BTF_LOC_PARAM_ADDR|BTF_LOC_PARAM_CONST is an address that should be
+ *   normalized with respect to kernel/module base address.
+ * - a BTF_LOC_PARAM_REG with vlen 1 is a simple register number;
+ *   with vlen 2 it is a multi-register parameter.  Register numbers are
+ *   numbers derived from DW_OP_reg values, i.e. 0 is DW_OP_reg0; since
+ *   DW_OP_fbreg has its own special DW_OP_value and DW_OP_regx can refer
+ *   to an arbitrary register number, we reserve BTF_LOC_PARAM_FBREG for
+ *   the frame base register to avoid collisions.
+ * - a _REG | OFFSET describes an address without dereferencing it.
+ * - a _REG | DEREF with vlen 1 dereferences the value in the register
+ *   number specified.
+ * - a REG | DEREF | OFFSET with vlen > 1 specifies the register number
+ *   in the first 4-byte value and the offset in the remainder.
+ *   In the case of REG and OFFSET combinations, the OFFSET has the width
+ *   of the value words while the type size describes the represented parameter,
+ *   so for example a REG | DEREF | OFFSET with size 8 and vlen 2 would be
+ *   an 8-byte register dereference with signed 4-byte offset, since the vlen 2
+ *   values consist of a register value and the signed value.
+ */
+enum btf_loc_param_flags {
+	BTF_LOC_PARAM_SIGNED		=	0x1,
+	BTF_LOC_PARAM_CONST		=	0x2,
+	BTF_LOC_PARAM_ADDR		=	0x4,
+	BTF_LOC_PARAM_REG		=	0x8,
+	BTF_LOC_PARAM_DEREF		=	0x10,
+	BTF_LOC_PARAM_OFFSET		=	0x20,
+};
+
+enum {
+	BTF_LOC_PARAM_FBREG		=	0xffffffff
+};
+
+/*
+ * BTF_KIND_LOC_PROTO specifies location prototypes; i.e. how locations relate
+ * to parameters; a struct btf_type of BTF_KIND_LOC_PROTO is followed by a
+ * vlen-specified number of __u32 BTF type ids which specify the associated
+ * BTF_KIND_LOC_PARAM for each function parameter associated with the
+ * location.  The type should either be 0 (no location info) or point at
+ * a BTF_KIND_LOC_PARAM.
+ */
+
+/*
+ * BTF_KIND_LOCSEC consists of vlen-specified number of "struct btf_loc"
+ * containing location site-specific information for a specific ELF section;
+ * for example locations in ".text" are in a LOCSEC named "inline.text".
+ *
+ * - function (func)
+ * - location prototype type id (loc_proto)
+ * - address offset (offset) relative to the runtime base address of the
+ *   ELF section associated with the LOCSEC
+ */
+
+struct btf_loc {
+	__u32 func;
+	__u32 loc_proto;
+	__u32 offset;
+};
+
 #endif /* _UAPI__LINUX_BTF_H__ */
diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h
index cc3b9f7dccafa2..47ee52859b4589 100644
--- a/include/uapi/linux/btrfs_tree.h
+++ b/include/uapi/linux/btrfs_tree.h
@@ -230,7 +230,7 @@
  *
  * Stored as an inline ref rather to avoid wasting space on a separate item on
  * top of the existing extent item. However, unlike the other inline refs,
- * there is one one owner ref per extent rather than one per extent.
+ * there is one owner ref per extent rather than one per extent.
  *
  * Because of this, it goes at the front of the list of inline refs, and thus
  * must have a lower type value than any other inline ref type (to satisfy the
@@ -243,7 +243,7 @@
 #define BTRFS_EXTENT_DATA_REF_KEY	178
 
 /*
- * Obsolete key. Defintion removed in 6.6, value may be reused in the future.
+ * Obsolete key. Definition removed in 6.6, value may be reused in the future.
  *
  * #define BTRFS_EXTENT_REF_V0_KEY	180
  */
@@ -1255,13 +1255,16 @@ static inline __u16 btrfs_qgroup_level(__u64 qgroupid)
 }
 
 /*
- * is subvolume quota turned on?
- */
-#define BTRFS_QGROUP_STATUS_FLAG_ON		(1ULL << 0)
-/*
- * RESCAN is set during the initialization phase
+ * The following BTRFS_QGROUP_STATUS_BIT_* are for * btrfs_qgroup_status_item::flags.
+ *
+ * Is subvolume quota turned on?
  */
-#define BTRFS_QGROUP_STATUS_FLAG_RESCAN		(1ULL << 1)
+#define BTRFS_QGROUP_STATUS_BIT_ON		(0)
+#define BTRFS_QGROUP_STATUS_FLAG_ON		(1UL << BTRFS_QGROUP_STATUS_BIT_ON)
+
+/* RESCAN is set during the initialization phase */
+#define BTRFS_QGROUP_STATUS_BIT_RESCAN		(1)
+#define BTRFS_QGROUP_STATUS_FLAG_RESCAN		(1UL << BTRFS_QGROUP_STATUS_BIT_RESCAN)
 /*
  * Some qgroup entries are known to be out of date,
  * either because the configuration has changed in a way that
@@ -1269,14 +1272,16 @@ static inline __u16 btrfs_qgroup_level(__u64 qgroupid)
  * with a non-qgroup-aware version.
  * Turning qouta off and on again makes it inconsistent, too.
  */
-#define BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT	(1ULL << 2)
+#define BTRFS_QGROUP_STATUS_BIT_INCONSISTENT	(2)
+#define BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT	(1UL << BTRFS_QGROUP_STATUS_BIT_INCONSISTENT)
 
 /*
  * Whether or not this filesystem is using simple quotas.  Not exactly the
  * incompat bit, because we support using simple quotas, disabling it, then
  * going back to full qgroup quotas.
  */
-#define BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE	(1ULL << 3)
+#define BTRFS_QGROUP_STATUS_BIT_SIMPLE_MODE	(3)
+#define BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE	(1UL << BTRFS_QGROUP_STATUS_BIT_SIMPLE_MODE)
 
 #define BTRFS_QGROUP_STATUS_FLAGS_MASK	(BTRFS_QGROUP_STATUS_FLAG_ON |		\
 					 BTRFS_QGROUP_STATUS_FLAG_RESCAN |	\
diff --git a/include/uapi/linux/close_range.h b/include/uapi/linux/close_range.h
index 2d804281554c5c..7da9ed95258a2b 100644
--- a/include/uapi/linux/close_range.h
+++ b/include/uapi/linux/close_range.h
@@ -2,11 +2,34 @@
 #ifndef _UAPI_LINUX_CLOSE_RANGE_H
 #define _UAPI_LINUX_CLOSE_RANGE_H
 
-/* Unshare the file descriptor table before closing file descriptors. */
-#define CLOSE_RANGE_UNSHARE	(1U << 1)
+/*
+ * A macro of one of these names defined before this header is parsed, by
+ * a libc or by a program's own fallback, would replace the enumerator.
+ */
+#undef CLOSE_RANGE_UNSHARE
+#undef CLOSE_RANGE_CLOEXEC
+#undef CLOSE_RANGE_EXCEPT
+#undef CLOSE_RANGE_CLOEXEC_ONLY
 
-/* Set the FD_CLOEXEC bit instead of closing the file descriptor. */
-#define CLOSE_RANGE_CLOEXEC	(1U << 2)
+enum close_range_flags {
+	/* Unshare the file descriptor table before closing file descriptors. */
+	CLOSE_RANGE_UNSHARE	= (1U << 1),
+
+	/* Set the FD_CLOEXEC bit instead of closing the file descriptor. */
+	CLOSE_RANGE_CLOEXEC	= (1U << 2),
+
+	/* Act on every file descriptor outside of the given range instead. */
+	CLOSE_RANGE_EXCEPT	= (1U << 3),
+
+	/* Only close file descriptors that have the FD_CLOEXEC bit set. */
+	CLOSE_RANGE_CLOEXEC_ONLY	= (1U << 4),
+};
+
+/* Keep #ifdef working and let glibc skip its own definitions. */
+#define CLOSE_RANGE_UNSHARE		CLOSE_RANGE_UNSHARE
+#define CLOSE_RANGE_CLOEXEC		CLOSE_RANGE_CLOEXEC
+#define CLOSE_RANGE_EXCEPT		CLOSE_RANGE_EXCEPT
+#define CLOSE_RANGE_CLOEXEC_ONLY	CLOSE_RANGE_CLOEXEC_ONLY
 
 #endif /* _UAPI_LINUX_CLOSE_RANGE_H */
 
diff --git a/include/uapi/linux/coredump.h b/include/uapi/linux/coredump.h
index dc3789b78af021..6d0c53b534ea8d 100644
--- a/include/uapi/linux/coredump.h
+++ b/include/uapi/linux/coredump.h
@@ -11,12 +11,53 @@
  * @COREDUMP_USERSPACE: userspace writes coredump
  * @COREDUMP_REJECT: don't generate coredump
  * @COREDUMP_WAIT: wait for coredump server
+ * @COREDUMP_RECORDS: send the coredump as a sequence of records instead of
+ *                    as a plain byte stream, see struct coredump_record_header;
+ *                    requires COREDUMP_KERNEL
+ * @COREDUMP_SPARSE: describe the holes in the coredump as zero records
+ *                   instead of transferring them; requires COREDUMP_RECORDS
+ * @COREDUMP_MEMORY_TYPES: dump the memory types in
+ *                          coredump_ack->memory_types instead of the ones
+ *                          the task selected; requires COREDUMP_KERNEL
  */
 enum {
 	COREDUMP_KERNEL		= (1ULL << 0),
 	COREDUMP_USERSPACE	= (1ULL << 1),
 	COREDUMP_REJECT		= (1ULL << 2),
 	COREDUMP_WAIT		= (1ULL << 3),
+	COREDUMP_RECORDS	= (1ULL << 4),
+	COREDUMP_SPARSE		= (1ULL << 5),
+	COREDUMP_MEMORY_TYPES	= (1ULL << 6),
+};
+
+/**
+ * coredump memory types
+ * @COREDUMP_MEMORY_ANON_PRIVATE: anonymous private memory
+ * @COREDUMP_MEMORY_ANON_SHARED: anonymous shared memory
+ * @COREDUMP_MEMORY_FILE_PRIVATE: file-backed private memory
+ * @COREDUMP_MEMORY_FILE_SHARED: file-backed shared memory
+ * @COREDUMP_MEMORY_ELF_HEADERS: the first page of a file-backed private
+ *                               mapping that starts an ELF file
+ * @COREDUMP_MEMORY_HUGETLB_PRIVATE: hugetlb private memory
+ * @COREDUMP_MEMORY_HUGETLB_SHARED: hugetlb shared memory
+ * @COREDUMP_MEMORY_DAX_PRIVATE: DAX private memory
+ * @COREDUMP_MEMORY_DAX_SHARED: DAX shared memory
+ *
+ * A bitmask of memory types a coredump may request to be included. New
+ * memory type bits must ensure that they do not steal memory from an
+ * existing one so a coredump server will continue to get the same
+ * coredumps even if a new bit is introduced.
+ */
+enum {
+	COREDUMP_MEMORY_ANON_PRIVATE	= (1ULL << 0),
+	COREDUMP_MEMORY_ANON_SHARED	= (1ULL << 1),
+	COREDUMP_MEMORY_FILE_PRIVATE	= (1ULL << 2),
+	COREDUMP_MEMORY_FILE_SHARED	= (1ULL << 3),
+	COREDUMP_MEMORY_ELF_HEADERS	= (1ULL << 4),
+	COREDUMP_MEMORY_HUGETLB_PRIVATE	= (1ULL << 5),
+	COREDUMP_MEMORY_HUGETLB_SHARED	= (1ULL << 6),
+	COREDUMP_MEMORY_DAX_PRIVATE	= (1ULL << 7),
+	COREDUMP_MEMORY_DAX_SHARED	= (1ULL << 8),
 };
 
 /**
@@ -24,17 +65,19 @@ enum {
  * @size: size of struct coredump_req
  * @size_ack: known size of struct coredump_ack on this kernel
  * @mask: supported features
+ * @memory_types: the memory types the task selected
+ * @memory_types_mask: the memory types this kernel knows
  *
  * When a coredump happens the kernel will connect to the coredump
  * socket and send a coredump request to the coredump server. The @size
  * member is set to the size of struct coredump_req and provides a hint
  * to userspace how much data can be read. Userspace may use MSG_PEEK to
  * peek the size of struct coredump_req and then choose to consume it in
- * one go. Userspace may also simply read a COREDUMP_ACK_SIZE_VER0
+ * one go. Userspace may also simply read a COREDUMP_REQ_SIZE_VER0
  * request. If the size the kernel sends is larger userspace simply
  * discards any remaining data.
  *
- * The coredump_req->mask member is set to the currently know features.
+ * The coredump_req->mask member is set to the currently known features.
  * Userspace may only set coredump_ack->mask to the bits raised by the
  * kernel in coredump_req->mask.
  *
@@ -42,15 +85,27 @@ enum {
  * struct coredump_ack the kernel knows. Userspace may only send up to
  * coredump_req->size_ack bytes to the kernel and must set
  * coredump_ack->size accordingly.
+ *
+ * @memory_types is set to the default memory types that are included in
+ * the coredump. This can be overridden by raising bits in
+ * coredump_ack->memory_types.
+ *
+ * @memory_types_mask contains a bitmask of all memory types the kernel
+ * knows about. A coredump server may only raise bits in
+ * coredump_ack->memory_types that are raised in
+ * coredump_req->memory_types_mask.
  */
 struct coredump_req {
 	__u32 size;
 	__u32 size_ack;
 	__u64 mask;
+	__u64 memory_types;
+	__u64 memory_types_mask;
 };
 
 enum {
 	COREDUMP_REQ_SIZE_VER0 = 16U, /* size of first published struct */
+	COREDUMP_REQ_SIZE_VER1 = 32U, /* memory_types and memory_types_mask added */
 };
 
 /**
@@ -58,6 +113,8 @@ enum {
  * @size: size of the struct
  * @spare: unused
  * @mask: features kernel is supposed to use
+ * @memory_types: memory types to dump, only with COREDUMP_MEMORY_TYPES
+ *                 in @mask
  *
  * The @size member must be set to the size of struct coredump_ack. It
  * may never exceed what the kernel returned in coredump_req->size_ack
@@ -67,15 +124,30 @@ enum {
  * The @mask member must be set to the features the coredump server
  * wants the kernel to use. Only bits the kernel returned in
  * coredump_req->mask may be set.
+ *
+ * If COREDUMP_MEMORY_TYPES is raised in @mask the kernel dumps the
+ * memory types set in the @memory_types mask. Zero is valid and dumps
+ * no memory apart from the mappings that are always dumped.
+ *
+ * Note that memory a task excluded via MADV_DONTDUMP is always left
+ * out. A coredump server wanting to add or drop memory types instead of
+ * outright replacing it should simply copy coredump_req->memory_types
+ * and then mask off or raise types as needed.
+ *
+ * Note that @memory_types must be zero if COREDUMP_MEMORY_TYPES isn't
+ * raised. COREDUMP_MEMORY_TYPES requires COREDUMP_KERNEL and an ack of
+ * at least COREDUMP_ACK_SIZE_VER1 bytes.
  */
 struct coredump_ack {
 	__u32 size;
 	__u32 spare;
 	__u64 mask;
+	__u64 memory_types;
 };
 
 enum {
 	COREDUMP_ACK_SIZE_VER0 = 16U, /* size of first published struct */
+	COREDUMP_ACK_SIZE_VER1 = 24U, /* memory_types added */
 };
 
 /**
@@ -83,11 +155,12 @@ enum {
  *
  * The kernel will place a single byte on the coredump socket. The
  * markers notify userspace whether the coredump ack succeeded or
- * failed.
+ * failed. After any marker other than COREDUMP_MARK_REQACK the kernel
+ * closes the connection and no coredump is generated.
  *
  * @COREDUMP_MARK_MINSIZE: the provided coredump_ack size was too small
  * @COREDUMP_MARK_MAXSIZE: the provided coredump_ack size was too big
- * @COREDUMP_MARK_UNSUPPORTED: the provided coredump_ack mask was invalid
+ * @COREDUMP_MARK_UNSUPPORTED: the provided coredump_ack mask or memory types were invalid
  * @COREDUMP_MARK_CONFLICTING: the provided coredump_ack mask has conflicting options
  * @COREDUMP_MARK_REQACK: the coredump request and ack was successful
  * @__COREDUMP_MARK_MAX: the maximum coredump mark value
@@ -101,4 +174,72 @@ enum coredump_mark {
 	__COREDUMP_MARK_MAX		= (1U << 31),
 };
 
+/**
+ * enum coredump_record_type - Type of a coredump record
+ *
+ * @COREDUMP_RECORD_DATA: the header is followed by ->len bytes of data
+ * @COREDUMP_RECORD_END: the coredump ends here, the header is not followed
+ *                       by any data and no further record is sent
+ * @COREDUMP_RECORD_ZERO: the header stands for ->len zero bytes and is not
+ *                        followed by any data
+ * @__COREDUMP_RECORD_TYPE_MAX: the maximum coredump record type value
+ */
+enum coredump_record_type {
+	COREDUMP_RECORD_DATA		= 0U,
+	COREDUMP_RECORD_END		= 1U,
+	COREDUMP_RECORD_ZERO		= 2U,
+	__COREDUMP_RECORD_TYPE_MAX	= (1U << 31),
+};
+
+/**
+ * struct coredump_record_header - header of a coredump record
+ * @size: size of struct coredump_record_header
+ * @type: one of enum coredump_record_type
+ * @flags: modifiers for this record
+ * @offset: offset in the coredump this record starts at
+ * @len: number of coredump bytes this record accounts for
+ *
+ * If the coredump server raises COREDUMP_RECORDS in coredump_ack->mask
+ * the kernel doesn't send the coredump as a plain byte stream. It sends
+ * a sequence of records instead. A COREDUMP_RECORD_DATA record is
+ * followed by @len bytes of actual coredump data. A
+ * COREDUMP_RECORD_ZERO record is followed by nothing and stands for
+ * @len zero bytes. A server that didn't raise COREDUMP_SPARSE never
+ * sees a zero record. Records arrive in order and leave no gaps. So
+ * @offset is the sum of the @len of all records before it.
+ *
+ * The last record is a COREDUMP_RECORD_END record. It is followed by
+ * nothing. Its @len is zero. Its @offset is the size of the coredump.
+ * The kernel only sends it once it has written the whole coredump. A
+ * server that hits end-of-file without having seen an end record must
+ * treat the coredump as incomplete.
+ *
+ * The @size member is set to the size of struct coredump_record_header
+ * the kernel knows and lets the header grow later. It comes first so it
+ * can be peeked. Userspace must consume @size bytes and discard
+ * anything beyond what it knows. It must refuse a @size smaller than
+ * COREDUMP_RECORD_HEADER_SIZE_VER0. @size covers the header alone.
+ * @offset and @len count coredump bytes.
+ *
+ * The @flags member carries modifiers that change how the record is to
+ * be interpreted. No flag is defined yet. Userspace must refuse a
+ * record carrying a flag or a type it doesn't know. Every new record
+ * type is raised in coredump_req->mask as a feature of its own. A
+ * server only ever sees the types it asked for.
+ *
+ * COREDUMP_RECORDS must be combined with COREDUMP_KERNEL, and
+ * COREDUMP_SPARSE with COREDUMP_RECORDS.
+ */
+struct coredump_record_header {
+	__u32 size;
+	__u32 type;
+	__u64 flags;
+	__u64 offset;
+	__u64 len;
+};
+
+enum {
+	COREDUMP_RECORD_HEADER_SIZE_VER0 = 32U, /* size of first published struct */
+};
+
 #endif /* _UAPI_LINUX_COREDUMP_H */
diff --git a/include/uapi/linux/dpll.h b/include/uapi/linux/dpll.h
index 85b898b1db5eea..8c71fc3f986343 100644
--- a/include/uapi/linux/dpll.h
+++ b/include/uapi/linux/dpll.h
@@ -29,8 +29,7 @@ enum dpll_mode {
 /**
  * enum dpll_lock_status - provides information of dpll device lock status,
  *   valid values for DPLL_A_LOCK_STATUS attribute
- * @DPLL_LOCK_STATUS_UNLOCKED: dpll was not yet locked to any valid input (or
- *   forced by setting DPLL_A_MODE to DPLL_MODE_DETACHED)
+ * @DPLL_LOCK_STATUS_UNLOCKED: dpll was not yet locked to any valid input
  * @DPLL_LOCK_STATUS_LOCKED: dpll is locked to a valid signal, but no holdover
  *   available
  * @DPLL_LOCK_STATUS_LOCKED_HO_ACQ: dpll is locked and holdover acquired
diff --git a/include/uapi/linux/firewire-cdev.h b/include/uapi/linux/firewire-cdev.h
index 05e3aa8fa8bcbc..4f83d6babc9218 100644
--- a/include/uapi/linux/firewire-cdev.h
+++ b/include/uapi/linux/firewire-cdev.h
@@ -293,7 +293,7 @@ struct fw_cdev_event_request3 {
 /**
  * struct fw_cdev_event_iso_interrupt - Sent when an iso packet was completed
  * @closure:	See &fw_cdev_event_common;
- *		set by %FW_CDEV_CREATE_ISO_CONTEXT ioctl
+ *		set by %FW_CDEV_IOC_CREATE_ISO_CONTEXT ioctl
  * @type:	See &fw_cdev_event_common; always %FW_CDEV_EVENT_ISO_INTERRUPT
  * @cycle:	Cycle counter of the last completed packet
  * @header_length: Total length of following headers, in bytes
@@ -353,7 +353,7 @@ struct fw_cdev_event_iso_interrupt {
 /**
  * struct fw_cdev_event_iso_interrupt_mc - An iso buffer chunk was completed
  * @closure:	See &fw_cdev_event_common;
- *		set by %FW_CDEV_CREATE_ISO_CONTEXT ioctl
+ *		set by %FW_CDEV_IOC_CREATE_ISO_CONTEXT ioctl
  * @type:	%FW_CDEV_EVENT_ISO_INTERRUPT_MULTICHANNEL
  * @completed:	Offset into the receive buffer; data before this offset is valid
  *
diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h
index 34c6f219462a59..a46c33692aa21a 100644
--- a/include/uapi/linux/fs.h
+++ b/include/uapi/linux/fs.h
@@ -88,7 +88,7 @@ struct fstrim_range {
  * We include a length field because some filesystems (vfat) have an identifier
  * that we do want to expose as a UUID, but doesn't have the standard length.
  *
- * We use a fixed size buffer beacuse this interface will, by fiat, never
+ * We use a fixed size buffer because this interface will, by fiat, never
  * support "UUIDs" longer than 16 bytes; we don't want to force all downstream
  * users to have to deal with that.
  */
diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
index 7435e09c87fee0..10a7f31c4bdf7b 100644
--- a/include/uapi/linux/fuse.h
+++ b/include/uapi/linux/fuse.h
@@ -248,6 +248,9 @@
  *  - add bufpool offset field to fuse_uring_ent_in_out struct
  *  - add FUSE_URING_ZERO_COPY, FUSE_URING_ENT_ZERO_COPY, and
  *    FOPEN_IO_URING_ZERO_COPY flag
+ *
+ *  7.47
+ *  - add FUSE_HAS_SYNCFS opt-in flag for privileged userspace servers
  */
 
 #ifndef _LINUX_FUSE_H
@@ -283,7 +286,7 @@
 #define FUSE_KERNEL_VERSION 7
 
 /** Minor version number of this interface */
-#define FUSE_KERNEL_MINOR_VERSION 46
+#define FUSE_KERNEL_MINOR_VERSION 47
 
 /** The node ID of the root inode */
 #define FUSE_ROOT_ID 1
@@ -464,6 +467,12 @@ struct fuse_file_lock {
  * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests.
  *			 init_out.request_timeout contains the timeout (in secs)
  * FUSE_HAS_IO_URING_BUFPOOL: kernel supports io-uring buffer pools
+ * FUSE_HAS_SYNCFS: server requests that syncfs()/sync() be propagated as
+ *		FUSE_SYNCFS requests.  Since an untrusted server can use this
+ *		to stall sync(), it is only honored when /dev/fuse was opened
+ *		with CAP_SYS_ADMIN in the initial user namespace (the same
+ *		privilege that mounting virtiofs or fuseblk requires).
+ *		Insufficiently privileged servers ignore it.
  */
 #define FUSE_ASYNC_READ		(1 << 0)
 #define FUSE_POSIX_LOCKS	(1 << 1)
@@ -512,6 +521,7 @@ struct fuse_file_lock {
 #define FUSE_OVER_IO_URING	(1ULL << 41)
 #define FUSE_REQUEST_TIMEOUT	(1ULL << 42)
 #define FUSE_HAS_IO_URING_BUFPOOL (1ULL << 43)
+#define FUSE_HAS_SYNCFS		(1ULL << 44)
 
 /**
  * CUSE INIT request/reply flags
diff --git a/include/uapi/linux/gpio.h b/include/uapi/linux/gpio.h
index f7cb8ae87df72c..5515b718886ec3 100644
--- a/include/uapi/linux/gpio.h
+++ b/include/uapi/linux/gpio.h
@@ -218,7 +218,7 @@ struct gpio_v2_line_request {
  * gpio_v2_line_flag, such as %GPIO_V2_LINE_FLAG_ACTIVE_LOW,
  * %GPIO_V2_LINE_FLAG_OUTPUT etc, added together
  * @attrs: the configuration attributes associated with the line
- * @padding: reserved for future use
+ * @padding: reserved for future use, must be zero filled
  */
 struct gpio_v2_line_info {
 	char name[GPIO_MAX_NAME_SIZE];
diff --git a/include/uapi/linux/handshake.h b/include/uapi/linux/handshake.h
index d7e40f5948886e..f907aa08252083 100644
--- a/include/uapi/linux/handshake.h
+++ b/include/uapi/linux/handshake.h
@@ -13,7 +13,10 @@
 enum handshake_handler_class {
 	HANDSHAKE_HANDLER_CLASS_NONE,
 	HANDSHAKE_HANDLER_CLASS_TLSHD,
-	HANDSHAKE_HANDLER_CLASS_MAX,
+
+	/* private: */
+	__HANDSHAKE_HANDLER_CLASS_MAX,
+	HANDSHAKE_HANDLER_CLASS_MAX = (__HANDSHAKE_HANDLER_CLASS_MAX - 1)
 };
 
 enum handshake_msg_type {
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index 43cecca49f017a..245b362045253a 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -381,6 +381,7 @@ enum {
 	IFLA_NETNS_IMMUTABLE,
 	IFLA_HEADROOM,
 	IFLA_TAILROOM,
+	IFLA_PACING_OFFLOAD,
 	__IFLA_MAX
 };
 
diff --git a/include/uapi/linux/iio/types.h b/include/uapi/linux/iio/types.h
index d0fb714ca0fd97..d602b40f14d08a 100644
--- a/include/uapi/linux/iio/types.h
+++ b/include/uapi/linux/iio/types.h
@@ -55,6 +55,7 @@ enum iio_chan_type {
 	IIO_ALTCURRENT,
 	IIO_COVERAGE,
 	IIO_VOLUMEFLOW,
+	IIO_FREQUENCY,
 };
 
 enum iio_modifier {
diff --git a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h
index 3528168f7c6d70..4217950e5d16a6 100644
--- a/include/uapi/linux/input-event-codes.h
+++ b/include/uapi/linux/input-event-codes.h
@@ -970,6 +970,12 @@
 
 /*
  * LEDs
+ *
+ * Do not add any new LED definitions to the input subsystem. The existing
+ * definitions are legacy and grandfathered for backwards compatibility with
+ * userspace (via evdev). Any new LEDs should be implemented using the
+ * LED subsystem (struct led_classdev). The input core provides a bridge to
+ * the LED subsystem in drivers/input/input-leds.c.
  */
 
 #define LED_NUML		0x00
diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h
index 206fa667c78281..b4c7194cb08d27 100644
--- a/include/uapi/linux/iommufd.h
+++ b/include/uapi/linux/iommufd.h
@@ -776,7 +776,7 @@ enum iommufd_hw_info_flags {
  * @out_data_type: Output the iommu hardware info type as defined in the enum
  *                 iommu_hw_info_type.
  * @out_capabilities: Output the generic iommu capability info type as defined
- *                    in the enum iommu_hw_capabilities.
+ *                    in the enum iommufd_hw_capabilities.
  * @out_max_pasid_log2: Output the width of PASIDs. 0 means no PASID support.
  *                      PCI devices turn to out_capabilities to check if the
  *                      specific capabilities is supported or not.
@@ -948,7 +948,9 @@ struct iommu_hwpt_vtd_s1_invalidate {
  *     CMDQ_OP_CFGI_CD
  *     CMDQ_OP_CFGI_CD_ALL
  *
- * -EIO will be returned if the command is not supported.
+ * User space must forward only valid commands: the kernel rejects, with
+ * -EIO, any command carrying an unsupported opcode, an unsupported field,
+ * or a field value that the underlying SMMU hardware does not implement.
  */
 struct iommu_viommu_arm_smmuv3_invalidate {
 	__aligned_le64 cmd[2];
diff --git a/include/uapi/linux/it6625.h b/include/uapi/linux/it6625.h
new file mode 100644
index 00000000000000..a895658aaf6a80
--- /dev/null
+++ b/include/uapi/linux/it6625.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
+/*
+ * Controls header for IT6625/IT6626 driver
+ */
+
+#ifndef _UAPI_LINUX_IT6625_H
+#define _UAPI_LINUX_IT6625_H
+
+#include <linux/v4l2-controls.h>
+
+/*
+ * Currently detected HDMI audio sampling rate, in Hz. Read-only.
+ * 0 means the rate is unavailable/unknown: no audio is currently
+ * present on the input, the hardware reported a sample-rate id this
+ * driver doesn't recognize, or the status read itself failed. Never a
+ * literal 0 Hz sample rate.
+ */
+#define V4L2_CID_IT6625_AUDIO_SAMPLING_RATE	(V4L2_CID_USER_IT6625_BASE + 0)
+
+/*
+ * Whether HDMI audio is currently present on the input. Read-only.
+ */
+#define V4L2_CID_IT6625_AUDIO_PRESENT		(V4L2_CID_USER_IT6625_BASE + 1)
+
+#endif /* _UAPI_LINUX_IT6625_H */
diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h
index 4c8884eea8084e..fa85b9760391a7 100644
--- a/include/uapi/linux/keyctl.h
+++ b/include/uapi/linux/keyctl.h
@@ -24,6 +24,7 @@
 #define KEY_SPEC_GROUP_KEYRING		-6	/* - key ID for GID-specific keyring */
 #define KEY_SPEC_REQKEY_AUTH_KEY	-7	/* - key ID for assumed request_key auth key */
 #define KEY_SPEC_REQUESTOR_KEYRING	-8	/* - key ID for request_key() dest keyring */
+#define KEY_SPEC_BPF_KEYRING		-9	/* - key ID for the BPF-specific keyring */
 
 /* request-key default keyrings */
 #define KEY_REQKEY_DEFL_NO_CHANGE		-1
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index ac2d77d1496356..8dff2fc1972e97 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -999,6 +999,7 @@ struct kvm_enable_cap {
 #define KVM_CAP_S390_HPAGE_2G 249
 #define KVM_CAP_PPC_COMPAT_CAPS 250
 #define KVM_CAP_ARM_PMU_V3_STRICT 251
+#define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 252
 
 struct kvm_irq_routing_irqchip {
 	__u32 irqchip;
@@ -1653,6 +1654,21 @@ struct kvm_memory_attributes {
 	__u64 flags;
 };
 
+/* Available with KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES */
+#define KVM_SET_MEMORY_ATTRIBUTES2              _IOWR(KVMIO,  0xd2, struct kvm_memory_attributes2)
+
+struct kvm_memory_attributes2 {
+	union {
+		__u64 address;
+		__u64 offset;
+	};
+	__u64 size;
+	__u64 attributes;
+	__u64 flags;
+	__u64 error_offset;
+	__u64 reserved[11];
+};
+
 #define KVM_MEMORY_ATTRIBUTE_PRIVATE           (1ULL << 3)
 
 #define KVM_CREATE_GUEST_MEMFD	_IOWR(KVMIO,  0xd4, struct kvm_create_guest_memfd)
diff --git a/include/uapi/linux/media/st/dcmipp_config.h b/include/uapi/linux/media/st/dcmipp_config.h
new file mode 100644
index 00000000000000..ad8d146820eab5
--- /dev/null
+++ b/include/uapi/linux/media/st/dcmipp_config.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * ST DCMIPP Driver - Userspace API
+ *
+ * Copyright (C) STMicroelectronics SA 2026
+ */
+
+#ifndef __UAPI_DCMIPP_CONFIG_H
+#define __UAPI_DCMIPP_CONFIG_H
+
+#include <linux/types.h>
+#include <linux/v4l2-controls.h>
+
+#define V4L2_CID_DCMIPP_PIXELPROC_GAMMA_CORRECTION_ENABLE	(V4L2_CID_USER_DCMIPP_BASE + 0x0)
+
+#endif /* __UAPI_DCMIPP_CONFIG_H */
diff --git a/include/uapi/linux/mii.h b/include/uapi/linux/mii.h
index 61d6edad4b94af..d662c10ee28246 100644
--- a/include/uapi/linux/mii.h
+++ b/include/uapi/linux/mii.h
@@ -81,7 +81,7 @@
 #define ADVERTISE_1000XPSE_ASYM	0x0100	/* Try for 1000BASE-X asym pause */
 #define ADVERTISE_100BASE4	0x0200	/* Try for 100mbps 4k packets  */
 #define ADVERTISE_PAUSE_CAP	0x0400	/* Try for pause               */
-#define ADVERTISE_PAUSE_ASYM	0x0800	/* Try for asymetric pause     */
+#define ADVERTISE_PAUSE_ASYM	0x0800	/* Try for asymmetric pause     */
 #define ADVERTISE_XNP		0x1000  /* Extended Next Page */
 #define ADVERTISE_RESV		ADVERTISE_XNP /* Used to be reserved */
 #define ADVERTISE_RFAULT	0x2000	/* Say we can detect faults    */
diff --git a/include/uapi/linux/netdev.h b/include/uapi/linux/netdev.h
index 35ff083221c719..9026e0414fbe08 100644
--- a/include/uapi/linux/netdev.h
+++ b/include/uapi/linux/netdev.h
@@ -58,7 +58,7 @@ enum netdev_xdp_rx_metadata {
  * enum netdev_xsk_flags
  * @NETDEV_XSK_FLAGS_TX_TIMESTAMP: HW timestamping egress packets is supported
  *   by the driver.
- * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L3 checksum HW offload is supported by the
+ * @NETDEV_XSK_FLAGS_TX_CHECKSUM: L4 checksum HW offload is supported by the
  *   driver.
  * @NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO: Launch time HW offload is supported
  *   by the driver.
diff --git a/include/uapi/linux/netfilter/nfnetlink_conntrack.h b/include/uapi/linux/netfilter/nfnetlink_conntrack.h
index 43233af75b9d5b..985f9c08d3e9b1 100644
--- a/include/uapi/linux/netfilter/nfnetlink_conntrack.h
+++ b/include/uapi/linux/netfilter/nfnetlink_conntrack.h
@@ -285,6 +285,7 @@ enum ctattr_filter {
 	CTA_FILTER_UNSPEC,
 	CTA_FILTER_ORIG_FLAGS,
 	CTA_FILTER_REPLY_FLAGS,
+	CTA_FILTER_ZONE,
 	__CTA_FILTER_MAX
 };
 #define CTA_FILTER_MAX (__CTA_FILTER_MAX - 1)
diff --git a/include/uapi/linux/netlink.h b/include/uapi/linux/netlink.h
index f87aaf28a6491d..82c41aeb43574b 100644
--- a/include/uapi/linux/netlink.h
+++ b/include/uapi/linux/netlink.h
@@ -329,6 +329,17 @@ enum netlink_attribute_type {
 	NL_ATTR_TYPE_UINT,
 };
 
+/**
+ * enum netlink_policy_byte_order - byte order of an integer attribute
+ * @NL_POLICY_BYTE_ORDER_BIG_ENDIAN: big endian (network byte order)
+ *
+ * Zero is left unassigned so that it keeps meaning host byte order,
+ * which is also what a missing byte order means.
+ */
+enum netlink_policy_byte_order {
+	NL_POLICY_BYTE_ORDER_BIG_ENDIAN = 1,
+};
+
 /**
  * enum netlink_policy_type_attr - policy type attributes
  * @NL_POLICY_TYPE_ATTR_UNSPEC: unused
@@ -356,6 +367,8 @@ enum netlink_attribute_type {
  *	bitfield32 type (U32)
  * @NL_POLICY_TYPE_ATTR_MASK: mask of valid bits for unsigned integers (U64)
  * @NL_POLICY_TYPE_ATTR_PAD: pad attribute for 64-bit alignment
+ * @NL_POLICY_TYPE_ATTR_BYTE_ORDER: byte order of an integer attribute,
+ *	&enum netlink_policy_byte_order, absent if host byte order (U32)
  *
  * @__NL_POLICY_TYPE_ATTR_MAX: number of attributes
  * @NL_POLICY_TYPE_ATTR_MAX: highest attribute number
@@ -374,6 +387,7 @@ enum netlink_policy_type_attr {
 	NL_POLICY_TYPE_ATTR_BITFIELD32_MASK,
 	NL_POLICY_TYPE_ATTR_PAD,
 	NL_POLICY_TYPE_ATTR_MASK,
+	NL_POLICY_TYPE_ATTR_BYTE_ORDER,
 
 	/* keep last */
 	__NL_POLICY_TYPE_ATTR_MAX,
diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h
index 020387d764122c..75d4c5d6a7a328 100644
--- a/include/uapi/linux/nl80211.h
+++ b/include/uapi/linux/nl80211.h
@@ -937,8 +937,8 @@
  *	OLBC handling in hostapd. Beacons are reported in %NL80211_CMD_FRAME
  *	messages. Note that per PHY only one application may register.
  *
- * @NL80211_CMD_SET_NOACK_MAP: sets a bitmap for the individual TIDs whether
- *      No Acknowledgement Policy should be applied.
+ * @NL80211_CMD_SET_NOACK_MAP: sets a bitmap (%NL80211_ATTR_TID_BITMAP)
+ *	indicating for which TIDs No Acknowledgment Policy should be applied.
  *
  * @NL80211_CMD_CH_SWITCH_NOTIFY: An AP or GO may decide to switch channels
  *	independently of the userspace SME, send this event indicating
@@ -1383,6 +1383,13 @@
  *	from the device to perform an announced schedule update. See
  *	%NL80211_ATTR_NAN_SCHED_DEFERRED for more details.
  *	If not set, the schedule should be applied immediately.
+ *	Setting a new schedule is always allowed and is never treated as an
+ *	evacuation, even if it removes channels that were previously marked as
+ *	non-evacuable with %NL80211_CMD_NAN_SET_NON_EVAC_CHANNELS. The
+ *	non-evacuable marking is a per-channel property of the schedule:
+ *	channels that remain in the new schedule keep their marking, channels
+ *	that are removed simply lose it, and newly added channels are
+ *	evacuable by default.
  * @NL80211_CMD_NAN_SCHED_UPDATE_DONE: Event sent to user space to notify that
  *	a deferred local NAN schedule update (requested with
  *	%NL80211_CMD_NAN_SET_LOCAL_SCHED and %NL80211_ATTR_NAN_SCHED_DEFERRED)
@@ -1399,10 +1406,12 @@
  *	be provided.
  *	Each peer channel must be compatible with at least one local channel
  *	set by %NL80211_CMD_SET_LOCAL_NAN_SCHED. Different maps must not
- *	contain compatible channels.
- *	For single-radio devices (n_radio <= 1), different maps must not
- *	schedule the same time slot, as the device cannot operate on multiple
- *	channels simultaneously.
+ *	contain compatible channels. Two maps may schedule the same time slot.
+ *	The device decides at runtime which of the channels to follow.
+ *	For example, the local schedule may place slot S on channel c1 while
+ *	the peer advertises slot S on both c1 and c2. If the device is anyway
+ *	on c2 during slot S (e.g. via the ULW mechanism for non-NAN activity),
+ *	it may use slot S on c2 to communicate with the peer.
  *	When updating an existing peer schedule, the full new schedule must be
  *	provided - partial updates are not supported. The new schedule will
  *	completely replace the previous one.
@@ -1427,6 +1436,28 @@
  * @NL80211_CMD_STOP_PD: Stop the PD operation, identified by
  *	its %NL80211_ATTR_WDEV interface.
  *
+ * @NL80211_CMD_NAN_SET_NON_EVAC_CHANNELS: Set the list of NAN local schedule
+ *	channels that must not be evacuated. NAN must be operational
+ *	(%NL80211_CMD_START_NAN was executed) and a local schedule must have
+ *	been set (%NL80211_CMD_NAN_SET_LOCAL_SCHED). The command carries zero
+ *	or more nested %NL80211_ATTR_NAN_CHANNEL attributes, each identifying a
+ *	channel (by its channel definition) of the current local schedule that
+ *	must not be evacuated for concurrent operations. The provided list
+ *	replaces the previous set of non-evacuable channels; channels of the
+ *	current schedule that are not included become evacuable again. All
+ *	provided channels must exist in the current local schedule, otherwise
+ *	the command fails. This is used to protect channels carrying NDC or
+ *	immutable schedules, whose evacuation would break existing NDP
+ *	connections.
+ *	The non-evacuable marking is a per-channel property of the current
+ *	local schedule and only affects evacuation for concurrent operations;
+ *	it does not prevent the schedule itself from being changed. Removing a
+ *	channel from the schedule with %NL80211_CMD_NAN_SET_LOCAL_SCHED is a
+ *	user-initiated change, not an evacuation, and is allowed even for a
+ *	non-evacuable channel. Across a schedule update, channels that remain
+ *	keep their non-evacuable marking, removed channels lose it, and newly
+ *	added channels are evacuable by default; issue this command again to
+ *	change the non-evacuable set.
  * @NL80211_CMD_MAX: highest used command number
  * @__NL80211_CMD_AFTER_LAST: internal use
  */
@@ -1705,6 +1736,8 @@ enum nl80211_commands {
 	NL80211_CMD_START_PD,
 	NL80211_CMD_STOP_PD,
 
+	NL80211_CMD_NAN_SET_NON_EVAC_CHANNELS,
+
 	/* add new commands above here */
 
 	/* used to define NL80211_CMD_MAX below */
@@ -2288,8 +2321,8 @@ enum nl80211_commands {
  *    abides to when initiating radiation on DFS channels. A country maps
  *    to one DFS region.
  *
- * @NL80211_ATTR_NOACK_MAP: This u16 bitmap contains the No Ack Policy of
- *      up to 16 TIDs.
+ * @NL80211_ATTR_TID_BITMAP: A TID bitmap (u16) whose meaning depends
+ *	on the command.
  *
  * @NL80211_ATTR_INACTIVITY_TIMEOUT: timeout value in seconds, this can be
  *	used by the drivers which has MLME in firmware and does not have support
@@ -3185,6 +3218,16 @@ enum nl80211_commands {
  *	The aggregated message always precedes the per-link messages for the
  *	same station within a dump sequence.
  *
+ * @NL80211_ATTR_FRAME_NO_STA: Valid for @NL80211_CMD_FRAME to denote that
+ *	the kernel had no station for a received frame or should not use a
+ *	known station to transmit a frame. This is relevant to know whether
+ *	MLD address translation happened or to disable it when sending a frame.
+ *
+ * @NL80211_ATTR_ASSOC_CIP: Enable Control Integrity Protocol for the
+ *	association
+ * @NL80211_ATTR_CIP_CAPABILITIES: The Control Integrity Protocol for the
+ *	station.
+ *
  * @NUM_NL80211_ATTR: total number of nl80211_attrs available
  * @NL80211_ATTR_MAX: highest attribute number currently defined
  * @__NL80211_ATTR_AFTER_LAST: internal use
@@ -3429,7 +3472,7 @@ enum nl80211_attrs {
 	NL80211_ATTR_DISABLE_HT,
 	NL80211_ATTR_HT_CAPABILITY_MASK,
 
-	NL80211_ATTR_NOACK_MAP,
+	NL80211_ATTR_TID_BITMAP,
 
 	NL80211_ATTR_INACTIVITY_TIMEOUT,
 
@@ -3785,6 +3828,11 @@ enum nl80211_attrs {
 
 	NL80211_ATTR_STA_DUMP_LINK_STATS,
 
+	NL80211_ATTR_FRAME_NO_STA,
+
+	NL80211_ATTR_ASSOC_CIP,
+	NL80211_ATTR_CIP_CAPABILITIES,
+
 	/* add attributes here, update the policy in nl80211.c */
 
 	__NL80211_ATTR_AFTER_LAST,
@@ -3800,6 +3848,7 @@ enum nl80211_attrs {
 #define NL80211_ATTR_CSA_C_OFF_BEACON NL80211_ATTR_CNTDWN_OFFS_BEACON
 #define NL80211_ATTR_CSA_C_OFF_PRESP NL80211_ATTR_CNTDWN_OFFS_PRESP
 #define NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS NL80211_ATTR_EXT_MLD_CAPA_AND_OPS
+#define NL80211_ATTR_NOACK_MAP NL80211_ATTR_TID_BITMAP
 
 /*
  * Allow user space programs to use #ifdef on new attributes by defining them
@@ -3934,6 +3983,7 @@ enum nl80211_iftype {
  *	that support %NL80211_FEATURE_FULL_AP_CLIENT_STATE to transition a
  *	previously added station into associated state
  * @NL80211_STA_FLAG_SPP_AMSDU: station supports SPP A-MSDUs
+ * @NL80211_STA_FLAG_CIP: station has Control Integrity Protocol (CIP) enabled
  * @NL80211_STA_FLAG_MAX: highest station flag number currently defined
  * @__NL80211_STA_FLAG_AFTER_LAST: internal use
  */
@@ -3947,6 +3997,7 @@ enum nl80211_sta_flags {
 	NL80211_STA_FLAG_TDLS_PEER,
 	NL80211_STA_FLAG_ASSOCIATED,
 	NL80211_STA_FLAG_SPP_AMSDU,
+	NL80211_STA_FLAG_CIP,
 
 	/* keep last */
 	__NL80211_STA_FLAG_AFTER_LAST,
@@ -5778,12 +5829,16 @@ enum nl80211_auth_type {
  * @NL80211_KEYTYPE_GROUP: Group (broadcast/multicast) key
  * @NL80211_KEYTYPE_PAIRWISE: Pairwise (unicast/individual) key
  * @NL80211_KEYTYPE_PEERKEY: PeerKey (DLS)
+ * @NL80211_KEYTYPE_CIGTK: Control Integrity Group Temporal Key
+ *	The cipher is GMAC-256 but passed as GCMP-256,
+ *	same as the pairwise key when used for CIP.
  * @NUM_NL80211_KEYTYPES: number of defined key types
  */
 enum nl80211_key_type {
 	NL80211_KEYTYPE_GROUP,
 	NL80211_KEYTYPE_PAIRWISE,
 	NL80211_KEYTYPE_PEERKEY,
+	NL80211_KEYTYPE_CIGTK,
 
 	NUM_NL80211_KEYTYPES
 };
@@ -5900,6 +5955,9 @@ enum nl80211_key_attributes {
  *	see &struct nl80211_txrate_eht
  * @NL80211_TXRATE_EHT_GI: configure EHT GI, (u8, see &enum nl80211_eht_gi)
  * @NL80211_TXRATE_EHT_LTF: configure EHT LTF, (u8, see &enum nl80211_eht_ltf)
+ * @NL80211_TXRATE_6GHZ_NON_HT_DUP: configure 6 GHz non-HT duplicate Beacon
+ *	transmission. This flag is applicable only in Beacon TX rate setting
+ *	and must be accompanied by a non-HT (legacy) Beacon rate.
  * @__NL80211_TXRATE_AFTER_LAST: internal
  * @NL80211_TXRATE_MAX: highest TX rate attribute
  */
@@ -5915,6 +5973,7 @@ enum nl80211_tx_rate_attributes {
 	NL80211_TXRATE_EHT,
 	NL80211_TXRATE_EHT_GI,
 	NL80211_TXRATE_EHT_LTF,
+	NL80211_TXRATE_6GHZ_NON_HT_DUP,
 
 	/* keep last */
 	__NL80211_TXRATE_AFTER_LAST,
@@ -7110,6 +7169,10 @@ enum nl80211_feature_flags {
  * @NL80211_EXT_FEATURE_PROBE_AP: Driver supports probing the associated AP
  *	in STA mode using @NL80211_CMD_PROBE_PEER.
  *
+ * @NL80211_EXT_FEATURE_FAST_ROAM_OFFLOAD: Driver supports fast roaming
+ *	offload in station mode, including Fast Transition or Opportunistic
+ *	Key Caching.
+ *
  * @NUM_NL80211_EXT_FEATURES: number of extended features.
  * @MAX_NL80211_EXT_FEATURES: highest extended feature index.
  */
@@ -7192,6 +7255,7 @@ enum nl80211_ext_feature_index {
 	NL80211_EXT_FEATURE_ROC_ADDR_FILTER,
 	NL80211_EXT_FEATURE_SET_KEY_LTF_SEED,
 	NL80211_EXT_FEATURE_PROBE_AP,
+	NL80211_EXT_FEATURE_FAST_ROAM_OFFLOAD,
 
 	/* add new features before the definition below */
 	NUM_NL80211_EXT_FEATURES,
diff --git a/include/uapi/linux/nsfs.h b/include/uapi/linux/nsfs.h
index 007fed5971b468..e5909266ec3fbc 100644
--- a/include/uapi/linux/nsfs.h
+++ b/include/uapi/linux/nsfs.h
@@ -55,6 +55,7 @@ enum init_ns_ino {
 	MNT_NS_INIT_INO		= 0xEFFFFFF8U,
 #ifdef __KERNEL__
 	MNT_NS_ANON_INO		= 0xEFFFFFF7U,
+	MNT_NS_INO_SPECIAL_MAX	= MNT_NS_ANON_INO,
 #endif
 };
 
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index fd10aa8d697f25..c49fc76292f798 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -314,8 +314,9 @@ enum {
  */
 enum perf_sample_regs_abi {
 	PERF_SAMPLE_REGS_ABI_NONE		= 0,
-	PERF_SAMPLE_REGS_ABI_32			= 1,
-	PERF_SAMPLE_REGS_ABI_64			= 2,
+	PERF_SAMPLE_REGS_ABI_32			= (1 << 0),
+	PERF_SAMPLE_REGS_ABI_64			= (1 << 1),
+	PERF_SAMPLE_REGS_ABI_SIMD		= (1 << 2),
 };
 
 /*
@@ -383,6 +384,7 @@ enum perf_event_read_format {
 #define PERF_ATTR_SIZE_VER7			128	/* Add: sig_data */
 #define PERF_ATTR_SIZE_VER8			136	/* Add: config3 */
 #define PERF_ATTR_SIZE_VER9			144	/* add: config4 */
+#define PERF_ATTR_SIZE_VER10			176	/* Add: sample_simd_{vec|pred}_reg_* */
 
 /*
  * 'struct perf_event_attr' contains various attributes that define
@@ -547,6 +549,29 @@ struct perf_event_attr {
 
 	__u64	config3; /* extension of config2 */
 	__u64	config4; /* extension of config3 */
+
+	/*
+	 * Defines the sampling SIMD/PRED(predicate) registers bitmap and
+	 * qwords (8 bytes) length.
+	 *
+	 * sample_simd_regs_enabled != 0 indicates there are SIMD/PRED
+	 * registers to be sampled, the SIMD/PRED registers bitmap and
+	 * qwords length are represented in
+	 * sample_simd_{vec|pred}_reg_{intr|user} and
+	 * sample_simd_{vec|pred}_reg_qwords fields separately.
+	 *
+	 * sample_simd_regs_enabled == 0 indicates no SIMD/PRED registers
+	 * are sampled.
+	 */
+	__u16	sample_simd_regs_enabled;
+	__u16	sample_simd_pred_reg_qwords;
+	__u16	sample_simd_vec_reg_qwords;
+	__u16	__reserved_4;
+
+	__u32	sample_simd_pred_reg_intr;
+	__u32	sample_simd_pred_reg_user;
+	__u64	sample_simd_vec_reg_intr;
+	__u64	sample_simd_vec_reg_user;
 };
 
 /*
@@ -1020,7 +1045,15 @@ enum perf_event_type {
 	 *      } && PERF_SAMPLE_BRANCH_STACK
 	 *
 	 *	{ u64			abi; # enum perf_sample_regs_abi
-	 *	  u64			regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
+	 *	  u64			regs[weight(mask)];
+	 *	  struct {
+	 *		u64 nr_vectors;		# 0 ... weight(sample_simd_vec_reg_user)
+	 *		u64 vector_qwords;	# 0 ... sample_simd_vec_reg_qwords
+	 *		u64 nr_pred;		# 0 ... weight(sample_simd_pred_reg_user)
+	 *		u64 pred_qwords;	# 0 ... sample_simd_pred_reg_qwords
+	 *		u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords];
+	 *	  } && (abi & PERF_SAMPLE_REGS_ABI_SIMD)
+	 *	} && PERF_SAMPLE_REGS_USER
 	 *
 	 *	{ u64			size;
 	 *	  char			data[size];
@@ -1047,7 +1080,15 @@ enum perf_event_type {
 	 *	{ u64			data_src; } && PERF_SAMPLE_DATA_SRC
 	 *	{ u64			transaction; } && PERF_SAMPLE_TRANSACTION
 	 *	{ u64			abi; # enum perf_sample_regs_abi
-	 *	  u64			regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR
+	 *	  u64			regs[weight(mask)];
+	 *	  struct {
+	 *		u64 nr_vectors;		# 0 ... weight(sample_simd_vec_reg_intr)
+	 *		u64 vector_qwords;	# 0 ... sample_simd_vec_reg_qwords
+	 *		u64 nr_pred;		# 0 ... weight(sample_simd_pred_reg_intr)
+	 *		u64 pred_qwords;	# 0 ... sample_simd_pred_reg_qwords
+	 *		u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords];
+	 *	  } && (abi & PERF_SAMPLE_REGS_ABI_SIMD)
+	 *	} && PERF_SAMPLE_REGS_INTR
 	 *	{ u64			phys_addr;} && PERF_SAMPLE_PHYS_ADDR
 	 *	{ u64			cgroup;} && PERF_SAMPLE_CGROUP
 	 *	{ u64			data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE
diff --git a/include/uapi/linux/psp-sev.h b/include/uapi/linux/psp-sev.h
index 52dae70b058b6a..536626abe6db18 100644
--- a/include/uapi/linux/psp-sev.h
+++ b/include/uapi/linux/psp-sev.h
@@ -15,7 +15,7 @@
 
 #include <linux/types.h>
 
-/**
+/*
  * SEV platform commands
  */
 enum {
@@ -36,7 +36,7 @@ enum {
 	SEV_MAX,
 };
 
-/**
+/*
  * SEV Firmware status code
  */
 typedef enum {
@@ -109,8 +109,8 @@ typedef enum {
 /**
  * struct sev_user_data_status - PLATFORM_STATUS command parameters
  *
- * @major: major API version
- * @minor: minor API version
+ * @api_major: major API version
+ * @api_minor: minor API version
  * @state: platform state
  * @flags: platform config flags
  * @build: firmware build id for API version
@@ -139,12 +139,12 @@ struct sev_user_data_pek_csr {
 } __packed;
 
 /**
- * struct sev_user_data_cert_import - PEK_CERT_IMPORT command parameters
+ * struct sev_user_data_pek_cert_import - PEK_CERT_IMPORT command parameters
  *
- * @pek_address: PEK certificate chain
- * @pek_len: length of PEK certificate
- * @oca_address: OCA certificate chain
- * @oca_len: length of OCA certificate
+ * @pek_cert_address: PEK certificate chain
+ * @pek_cert_len: length of PEK certificate
+ * @oca_cert_address: OCA certificate chain
+ * @oca_cert_len: length of OCA certificate
  */
 struct sev_user_data_pek_cert_import {
 	__u64 pek_cert_address;			/* In */
@@ -156,8 +156,8 @@ struct sev_user_data_pek_cert_import {
 /**
  * struct sev_user_data_pdh_cert_export - PDH_CERT_EXPORT command parameters
  *
- * @pdh_address: PDH certificate address
- * @pdh_len: length of PDH certificate
+ * @pdh_cert_address: PDH certificate address
+ * @pdh_cert_len: length of PDH certificate
  * @cert_chain_address: PDH certificate chain
  * @cert_chain_len: length of PDH certificate chain
  */
@@ -248,7 +248,7 @@ struct sev_user_data_snp_config {
 } __packed;
 
 /**
- * struct sev_data_snp_vlek_load - SNP_VLEK_LOAD structure
+ * struct sev_user_data_snp_vlek_load - SNP_VLEK_LOAD structure
  *
  * @len: length of the command buffer read by the PSP
  * @vlek_wrapped_version: version of wrapped VLEK hashstick (Must be 0h)
@@ -264,7 +264,7 @@ struct sev_user_data_snp_vlek_load {
 } __packed;
 
 /**
- * struct sev_user_data_snp_vlek_wrapped_vlek_hashstick - Wrapped VLEK data
+ * struct sev_user_data_snp_wrapped_vlek_hashstick - Wrapped VLEK data
  *
  * @data: Opaque data provided by AMD KDS (as described in SEV-SNP Firmware ABI
  *        1.54, SNP_VLEK_LOAD)
diff --git a/include/uapi/linux/random.h b/include/uapi/linux/random.h
index 1dd047ec98a121..ef6614d9c70497 100644
--- a/include/uapi/linux/random.h
+++ b/include/uapi/linux/random.h
@@ -58,7 +58,7 @@ struct rand_pool_info {
 /**
  * struct vgetrandom_opaque_params - arguments for allocating memory for vgetrandom
  *
- * @size_per_opaque_state:	Size of each state that is to be passed to vgetrandom().
+ * @size_of_opaque_state:	Size of each state passed to vgetrandom().
  * @mmap_prot:			Value of the prot argument in mmap(2).
  * @mmap_flags:			Value of the flags argument in mmap(2).
  * @reserved:			Reserved for future use.
diff --git a/include/uapi/linux/rpmsg.h b/include/uapi/linux/rpmsg.h
index f0c8da2b185b43..d553ebac3181d1 100644
--- a/include/uapi/linux/rpmsg.h
+++ b/include/uapi/linux/rpmsg.h
@@ -24,32 +24,35 @@ struct rpmsg_endpoint_info {
 };
 
 /**
- * Instantiate a new rmpsg char device endpoint.
+ * RPMSG_CREATE_EPT_IOCTL - Instantiate a new rmpsg char device endpoint.
  */
 #define RPMSG_CREATE_EPT_IOCTL	_IOW(0xb5, 0x1, struct rpmsg_endpoint_info)
 
 /**
- * Destroy a rpmsg char device endpoint created by the RPMSG_CREATE_EPT_IOCTL.
+ * RPMSG_DESTROY_EPT_IOCTL - Destroy a rpmsg char device endpoint created
+ * by the RPMSG_CREATE_EPT_IOCTL.
  */
 #define RPMSG_DESTROY_EPT_IOCTL	_IO(0xb5, 0x2)
 
 /**
- * Instantiate a new local rpmsg service device.
+ * RPMSG_CREATE_DEV_IOCTL - Instantiate a new local rpmsg service device.
  */
 #define RPMSG_CREATE_DEV_IOCTL	_IOW(0xb5, 0x3, struct rpmsg_endpoint_info)
 
 /**
- * Release a local rpmsg device.
+ * RPMSG_RELEASE_DEV_IOCTL - Release a local rpmsg device.
  */
 #define RPMSG_RELEASE_DEV_IOCTL	_IOW(0xb5, 0x4, struct rpmsg_endpoint_info)
 
 /**
- * Get the flow control state of the remote rpmsg char device.
+ * RPMSG_GET_OUTGOING_FLOWCONTROL - Get the flow control state of the
+ * remote rpmsg char device.
  */
 #define RPMSG_GET_OUTGOING_FLOWCONTROL _IOR(0xb5, 0x5, int)
 
 /**
- * Set the flow control state of the local rpmsg char device.
+ * RPMSG_SET_INCOMING_FLOWCONTROL - Set the flow control state of the
+ * local rpmsg char device.
  */
 #define RPMSG_SET_INCOMING_FLOWCONTROL _IOR(0xb5, 0x6, int)
 
diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h
index 33a4624285cd41..19ffeba894283e 100644
--- a/include/uapi/linux/sched.h
+++ b/include/uapi/linux/sched.h
@@ -53,7 +53,7 @@
  */
 #define UNSHARE_EMPTY_MNTNS 0x00100000 /* Unshare an empty mount namespace. */
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 /**
  * struct clone_args - arguments for the clone3 syscall
  * @flags:        Flags for the new process as listed above.
diff --git a/include/uapi/linux/se_ioctl.h b/include/uapi/linux/se_ioctl.h
new file mode 100644
index 00000000000000..c7ce736b5ea419
--- /dev/null
+++ b/include/uapi/linux/se_ioctl.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause*/
+/*
+ * Copyright 2025 NXP
+ */
+
+#ifndef SE_IOCTL_H
+#define SE_IOCTL_H
+
+#include <linux/types.h>
+
+#define SE_TYPE_STR_DBG			"dbg"
+#define SE_TYPE_STR_HSM			"hsm"
+#define SE_TYPE_ID_UNKWN		0x0
+#define SE_TYPE_ID_DBG			0x1
+#define SE_TYPE_ID_HSM			0x2
+/* IOCTL definitions. */
+
+struct se_ioctl_setup_iobuf {
+	__u64 user_buf;
+	__u32 length;
+	__u32 flags;
+	__u64 ele_addr;
+};
+
+struct se_ioctl_shared_mem_cfg {
+	__u32 base_offset;
+	__u32 size;
+};
+
+struct se_ioctl_get_if_info {
+	__u8 se_if_id;
+	__u8 interrupt_idx;
+	__u8 tz;
+	__u8 did;
+	__u8 cmd_tag;
+	__u8 rsp_tag;
+	__u8 success_tag;
+	__u8 base_api_ver;
+	__u8 fw_api_ver;
+};
+
+struct se_ioctl_cmd_snd_rcv_rsp_info {
+	__u64 tx_buf;
+	__u64 rx_buf;
+	__u32 tx_buf_sz;
+	__u32 rx_buf_sz;
+};
+
+struct se_ioctl_get_soc_info {
+	__u16 soc_id;
+	__u16 soc_rev;
+};
+
+/* IO Buffer Flags */
+#define SE_IO_BUF_FLAGS_IS_OUTPUT	(0x00u)
+#define SE_IO_BUF_FLAGS_IS_INPUT	(0x01u)
+#define SE_IO_BUF_FLAGS_USE_SEC_MEM	(0x02u)
+#define SE_IO_BUF_FLAGS_USE_SHORT_ADDR	(0x04u)
+#define SE_IO_BUF_FLAGS_IS_IN_OUT	(0x10u)
+
+/* IOCTLS */
+#define SE_IOCTL			0x0A /* like MISC_MAJOR. */
+
+/*
+ * ioctl to designated the current fd as logical-receiver.
+ * This ioctl is send when the nvm-daemon, a slave to the
+ * firmware is started by the user.
+ */
+#define SE_IOCTL_CHECK_CMD_RCV_REG_STATUS	_IO(SE_IOCTL, 0x01)
+
+/*
+ * ioctl to get the buffer allocated from the memory, which is shared
+ * between kernel and FW.
+ * Post allocation, the kernel tagged the allocated memory with:
+ *  Output
+ *  Input
+ *  Input-Output
+ *  Short address
+ *  Secure-memory
+ */
+#define SE_IOCTL_SETUP_IOBUF	_IOWR(SE_IOCTL, 0x03, struct se_ioctl_setup_iobuf)
+
+/*
+ * ioctl to get the mu information, that is used to exchange message
+ * with FW, from user-space.
+ */
+#define SE_IOCTL_GET_MU_INFO	_IOR(SE_IOCTL, 0x04, struct se_ioctl_get_if_info)
+/*
+ * ioctl to get SoC Info from user-space.
+ */
+#define SE_IOCTL_GET_SOC_INFO      _IOR(SE_IOCTL, 0x06, struct se_ioctl_get_soc_info)
+
+/*
+ * ioctl to send command and receive response from user-space.
+ */
+#define SE_IOCTL_CMD_SEND_RCV_RSP _IOWR(SE_IOCTL, 0x07, struct se_ioctl_cmd_snd_rcv_rsp_info)
+#endif
diff --git a/include/uapi/linux/tee.h b/include/uapi/linux/tee.h
index 5203977ed35d1d..76815b0f4b6651 100644
--- a/include/uapi/linux/tee.h
+++ b/include/uapi/linux/tee.h
@@ -384,24 +384,6 @@ struct tee_iocl_supp_send_arg {
 #define TEE_IOC_SUPPL_SEND	_IOR(TEE_IOC_MAGIC, TEE_IOC_BASE + 7, \
 				     struct tee_ioctl_buf_data)
 
-/**
- * struct tee_ioctl_shm_register_data - Shared memory register argument
- * @addr:      [in] Start address of shared memory to register
- * @length:    [in/out] Length of shared memory to register
- * @flags:     [in/out] Flags to/from registration.
- * @id:                [out] Identifier of the shared memory
- *
- * The flags field should currently be zero as input. Updated by the call
- * with actual flags as defined by TEE_IOCTL_SHM_* above.
- * This structure is used as argument for TEE_IOC_SHM_REGISTER below.
- */
-struct tee_ioctl_shm_register_data {
-	__u64 addr;
-	__u64 length;
-	__u32 flags;
-	__s32 id;
-};
-
 /**
  * struct tee_ioctl_shm_register_fd_data - Shared memory registering argument
  * @fd:		[in] File descriptor identifying dmabuf reference
@@ -426,13 +408,31 @@ struct tee_ioctl_shm_register_fd_data {
  * Returns a file descriptor on success or < 0 on failure
  *
  * The returned file descriptor refers to the shared memory object in the
- * kernel. The supplied file deccriptor can be closed if it's not needed
+ * kernel. The supplied file descriptor can be closed if it's not needed
  * for other purposes. The shared memory is freed when the descriptor is
  * closed.
  */
 #define TEE_IOC_SHM_REGISTER_FD	_IOWR(TEE_IOC_MAGIC, TEE_IOC_BASE + 8, \
 				     struct tee_ioctl_shm_register_fd_data)
 
+/**
+ * struct tee_ioctl_shm_register_data - Shared memory register argument
+ * @addr:      [in] Start address of shared memory to register
+ * @length:    [in/out] Length of shared memory to register
+ * @flags:     [in/out] Flags to/from registration.
+ * @id:                [out] Identifier of the shared memory
+ *
+ * The flags field should currently be zero as input. Updated by the call
+ * with actual flags as defined by TEE_IOCTL_SHM_* above.
+ * This structure is used as argument for TEE_IOC_SHM_REGISTER below.
+ */
+struct tee_ioctl_shm_register_data {
+	__u64 addr;
+	__u64 length;
+	__u32 flags;
+	__s32 id;
+};
+
 /**
  * TEE_IOC_SHM_REGISTER - Register shared memory argument
  *
@@ -440,7 +440,7 @@ struct tee_ioctl_shm_register_fd_data {
  *
  * Returns a file descriptor on success or < 0 on failure
  *
- * The shared memory is unregisterred when the descriptor is closed.
+ * The shared memory is unregistered when the descriptor is closed.
  */
 #define TEE_IOC_SHM_REGISTER   _IOWR(TEE_IOC_MAGIC, TEE_IOC_BASE + 9, \
 				     struct tee_ioctl_shm_register_data)
diff --git a/include/uapi/linux/thermal.h b/include/uapi/linux/thermal.h
index 46a2633d33aaa4..36f3da2d6a7caa 100644
--- a/include/uapi/linux/thermal.h
+++ b/include/uapi/linux/thermal.h
@@ -81,11 +81,11 @@ enum thermal_genl_event {
 	THERMAL_GENL_EVENT_CDEV_STATE_UPDATE,	/* Cdev state updated */
 	THERMAL_GENL_EVENT_TZ_GOV_CHANGE,	/* Governor policy changed  */
 	THERMAL_GENL_EVENT_CPU_CAPABILITY_CHANGE,	/* CPU capability changed */
-	THERMAL_GENL_EVENT_THRESHOLD_ADD,	/* A thresold has been added */
-	THERMAL_GENL_EVENT_THRESHOLD_DELETE,	/* A thresold has been deleted */
+	THERMAL_GENL_EVENT_THRESHOLD_ADD,	/* A threshold has been added */
+	THERMAL_GENL_EVENT_THRESHOLD_DELETE,	/* A threshold has been deleted */
 	THERMAL_GENL_EVENT_THRESHOLD_FLUSH,	/* All thresolds have been deleted */
-	THERMAL_GENL_EVENT_THRESHOLD_UP,	/* A thresold has been crossed the way up */
-	THERMAL_GENL_EVENT_THRESHOLD_DOWN,	/* A thresold has been crossed the way down */
+	THERMAL_GENL_EVENT_THRESHOLD_UP,	/* A threshold has been crossed the way up */
+	THERMAL_GENL_EVENT_THRESHOLD_DOWN,	/* A threshold has been crossed the way down */
 	__THERMAL_GENL_EVENT_MAX,
 };
 #define THERMAL_GENL_EVENT_MAX (__THERMAL_GENL_EVENT_MAX - 1)
diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h
index affec0ab478150..d17e41d51d2ec1 100644
--- a/include/uapi/linux/v4l2-controls.h
+++ b/include/uapi/linux/v4l2-controls.h
@@ -123,6 +123,15 @@ enum v4l2_colorfx {
 #define V4L2_CID_USER_MEYE_BASE			(V4L2_CID_USER_BASE + 0x1000)
 #endif
 
+/*
+ * The base for the vim2m driver controls.
+ * We reserve 16 controls for this driver.
+ * The vim2m control range clashed with the meye control range, which was not
+ * intended, but since the meye driver has been removed, we can just keep the
+ * vim2m control range.
+ */
+#define V4L2_CID_USER_VIM2M_BASE		(V4L2_CID_USER_BASE + 0x1000)
+
 /* The base for the bttv driver controls.
  * We reserve 32 controls for this driver. */
 #define V4L2_CID_USER_BTTV_BASE			(V4L2_CID_USER_BASE + 0x1010)
@@ -234,6 +243,18 @@ enum v4l2_colorfx {
  */
 #define V4L2_CID_USER_MALI_C55_BASE		(V4L2_CID_USER_BASE + 0x1230)
 
+/*
+ * The base for the ST DCMIPP driver controls.
+ * We reserve 16 controls for this driver
+ */
+#define V4L2_CID_USER_DCMIPP_BASE		(V4L2_CID_USER_BASE + 0x1240)
+
+/*
+ * The base for IT6625/IT6626 driver controls.
+ * We reserve 16 controls for this driver.
+ */
+#define V4L2_CID_USER_IT6625_BASE		(V4L2_CID_USER_BASE + 0x1250)
+
 /* MPEG-class control IDs */
 /* The MPEG controls are applicable to all codec controls
  * and the 'MPEG' part of the define is historical */
diff --git a/include/uapi/rdma/ionic-abi.h b/include/uapi/rdma/ionic-abi.h
index 2c70ac149c4ffa..3d8d56922a93b9 100644
--- a/include/uapi/rdma/ionic-abi.h
+++ b/include/uapi/rdma/ionic-abi.h
@@ -22,6 +22,10 @@
 #define IONIC_CMB_WC		8
 #define IONIC_CMB_UC		16
 
+enum ionic_ctx_resp_comp_mask {
+	IONIC_CTX_CMASK_IONIC_FLAGS = 1 << 0,
+};
+
 struct ionic_ctx_req {
 	__u32 rsvd[2];
 };
@@ -46,8 +50,9 @@ struct ionic_ctx_resp {
 	__u8 udma_count;
 	__u8 expdb_mask;
 	__u8 expdb_qtypes;
+	__u8 rcq_sign_bit;
 
-	__u8 rsvd2[3];
+	__u16 comp_mask;
 	__aligned_u64 phc_offset;
 };
 
@@ -85,6 +90,8 @@ struct ionic_qp_req {
 	__u8 rq_cmb;
 	__u8 udma_mask;
 	__u8 rsvd[3];
+	__u32 ionic_flags;
+	__u32 rsvd_pad;
 };
 
 struct ionic_qp_resp {
diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h
index 32cbbfc80f996c..ebc9a655b82683 100644
--- a/include/uapi/rdma/mana-abi.h
+++ b/include/uapi/rdma/mana-abi.h
@@ -48,13 +48,23 @@ struct mana_ib_create_qp_resp {
 	__u32 reserved;
 };
 
+enum mana_ib_create_rc_qp_flags {
+	MANA_IB_RC_QP_FIXED_WQE = 1 << 0,
+	MANA_IB_RC_MMQ_CREATE = 1 << 1,
+};
+
 struct mana_ib_create_rc_qp {
 	__aligned_u64 queue_buf[4];
 	__u32 queue_size[4];
+	__aligned_u64 mmq_buf;
+	__u32 mmq_size;
+	__u32 comp_mask;
 };
 
 struct mana_ib_create_rc_qp_resp {
 	__u32 queue_id[4];
+	__u32 mmq_id;
+	__u32 reserved;
 };
 
 struct mana_ib_create_uc_qp {
@@ -100,6 +110,8 @@ struct mana_ib_create_qp_rss_resp {
 
 enum mana_ib_ucontext_support {
 	MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
+	MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1,
+	MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT = 1 << 2,
 };
 
 struct mana_ib_alloc_ucontext_resp {
diff --git a/include/uapi/sound/asound.h b/include/uapi/sound/asound.h
index c11da9656e3811..767c9627324592 100644
--- a/include/uapi/sound/asound.h
+++ b/include/uapi/sound/asound.h
@@ -99,9 +99,10 @@ enum {
 	SNDRV_HWDEP_IFACE_LINE6,	/* Line6 USB processors */
 	SNDRV_HWDEP_IFACE_FW_MOTU,	/* MOTU FireWire series */
 	SNDRV_HWDEP_IFACE_FW_FIREFACE,	/* RME Fireface series */
+	SNDRV_HWDEP_IFACE_CAIAQ,	/* Native Instruments caiaq USB devices */
 
 	/* Don't forget to change the following: */
-	SNDRV_HWDEP_IFACE_LAST = SNDRV_HWDEP_IFACE_FW_FIREFACE
+	SNDRV_HWDEP_IFACE_LAST = SNDRV_HWDEP_IFACE_CAIAQ
 };
 
 struct snd_hwdep_info {
diff --git a/include/vdso/getrandom.h b/include/vdso/getrandom.h
index 6ca4d6de9e4688..1d9dbd4d76c121 100644
--- a/include/vdso/getrandom.h
+++ b/include/vdso/getrandom.h
@@ -64,7 +64,7 @@ extern void __arch_chacha20_blocks_nostack(u8 *dst_bytes, const u32 *key, u32 *c
  * @opaque_state:	Passed to __cvdso_getrandom().
  * @opaque_len:		Passed to __cvdso_getrandom();
  *
- * This function is implemented by making a single call to to __cvdso_getrandom(), whose
+ * This function is implemented by making a single call to __cvdso_getrandom(), whose
  * documentation may be consulted for more information.
  *
  * Returns:	The return value of __cvdso_getrandom().
diff --git a/include/vdso/math64.h b/include/vdso/math64.h
index 22ae212f8b28fe..55b45f5cf615ac 100644
--- a/include/vdso/math64.h
+++ b/include/vdso/math64.h
@@ -2,15 +2,36 @@
 #ifndef __VDSO_MATH64_H
 #define __VDSO_MATH64_H
 
-static __always_inline u32
-__iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder)
+static __always_inline u32 __iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder)
 {
 	u32 ret = 0;
 
 	while (dividend >= divisor) {
-		/* The following asm() prevents the compiler from
-		   optimising this loop into a modulo operation.  */
-		asm("" : "+rm"(dividend));
+		/*
+		 * Prevent the compiler from optimising this loop into a
+		 * modulo operation.
+		 */
+		OPTIMIZER_HIDE_VAR(dividend);
+
+		dividend -= divisor;
+		ret++;
+	}
+
+	*remainder = dividend;
+
+	return ret;
+}
+
+static __always_inline u32 __iter_div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder)
+{
+	u32 ret = 0;
+
+	while (dividend >= divisor) {
+		/*
+		 * Prevent the compiler from optimising this loop into a
+		 * modulo operation.
+		 */
+		OPTIMIZER_HIDE_VAR(dividend);
 
 		dividend -= divisor;
 		ret++;
diff --git a/include/video/mmp_disp.h b/include/video/mmp_disp.h
index 41354bd4989572..8f2b434da4e9a4 100644
--- a/include/video/mmp_disp.h
+++ b/include/video/mmp_disp.h
@@ -172,7 +172,7 @@ struct mmp_panel {
 	/* use node to register to list */
 	struct list_head node;
 	const char *name;
-	/* path name used to connect to proper path configed */
+	/* path name used to connect to proper path configured */
 	const char *plat_path_name;
 	struct device *dev;
 	int panel_type;
@@ -291,7 +291,7 @@ static inline int mmp_overlay_set_addr(struct mmp_overlay *overlay,
  * it defined a common interface that plat driver need to implement
  */
 struct mmp_path_info {
-	/* driver data, set when registed*/
+	/* driver data, set when registered */
 	const char *name;
 	struct device *dev;
 	int id;
@@ -309,7 +309,7 @@ extern void mmp_unregister_path(struct mmp_path *path);
 extern void mmp_register_panel(struct mmp_panel *panel);
 extern void mmp_unregister_panel(struct mmp_panel *panel);
 
-/* defintions for platform data */
+/* definitions for platform data */
 /* interface for buffer driver */
 struct mmp_buffer_driver_mach_info {
 	const char	*name;
diff --git a/init/Kconfig b/init/Kconfig
index 8583d9f06c522e..ecd71b38fc5607 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1,201 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
-config CC_VERSION_TEXT
-	string
-	default "$(CC_VERSION_TEXT)"
-	help
-	  This is used in unclear ways:
-
-	  - Re-run Kconfig when the compiler is updated
-	    The 'default' property references the environment variable,
-	    CC_VERSION_TEXT so it is recorded in include/config/auto.conf.cmd.
-	    When the compiler is updated, Kconfig will be invoked.
-
-	  - Ensure full rebuild when the compiler is updated
-	    include/linux/compiler-version.h contains this option in the comment
-	    line so fixdep adds include/config/CC_VERSION_TEXT into the
-	    auto-generated dependency. When the compiler is updated, syncconfig
-	    will touch it and then every file will be rebuilt.
-
-config CC_IS_GCC
-	def_bool $(success,test "$(cc-name)" = GCC)
-
-config GCC_VERSION
-	int
-	default $(cc-version) if CC_IS_GCC
-	default 0
-
-config CC_IS_CLANG
-	def_bool $(success,test "$(cc-name)" = Clang)
-
-config CLANG_VERSION
-	int
-	default $(cc-version) if CC_IS_CLANG
-	default 0
-
-config AS_IS_GNU
-	def_bool $(success,test "$(as-name)" = GNU)
-
-config AS_IS_LLVM
-	def_bool $(success,test "$(as-name)" = LLVM)
-
-config AS_VERSION
-	int
-	# Use clang version if this is the integrated assembler
-	default CLANG_VERSION if AS_IS_LLVM
-	default $(as-version)
-
-config LD_IS_BFD
-	def_bool $(success,test "$(ld-name)" = BFD)
-
-config LD_VERSION
-	int
-	default $(ld-version) if LD_IS_BFD
-	default 0
-
-config LD_IS_LLD
-	def_bool $(success,test "$(ld-name)" = LLD)
-
-config LLD_VERSION
-	int
-	default $(ld-version) if LD_IS_LLD
-	default 0
-
-config RUSTC_VERSION
-	int
-	default $(rustc-version)
-	help
-	  It does not depend on `RUST` since that one may need to use the version
-	  in a `depends on`.
-
-config RUST_IS_AVAILABLE
-	def_bool $(success,$(srctree)/scripts/rust_is_available.sh)
-	help
-	  This shows whether a suitable Rust toolchain is available (found).
-
-	  Please see Documentation/rust/quick-start.rst for instructions on how
-	  to satisfy the build requirements of Rust support.
-
-	  In particular, the Makefile target 'rustavailable' is useful to check
-	  why the Rust toolchain is not being detected.
-
-config RUSTC_LLVM_VERSION
-	int
-	default $(rustc-llvm-version)
-
-config RUSTC_LLVM_MAJOR_VERSION
-	int
-	default $(shell,expr $(rustc-llvm-version) / 10000)
-
-config RUSTC_CLANG_LLVM_COMPATIBLE
-	bool
-	default y if CC_IS_CLANG && RUSTC_LLVM_MAJOR_VERSION = $(shell,expr $(cc-version) / 10000)
-	help
-	  This indicates whether Rust and Clang use LLVM of the same major
-	  version.
-
-	  Operations involving handling LLVM IR or bitcode (e.g. cross-language
-	  LTO) require the same LLVM major version to work properly. For best
-	  compatibility it is recommended that the exact same LLVM is used.
-
-config ARCH_HAS_CC_CAN_LINK
-	bool
 
-config CC_CAN_LINK
-	bool
-	default ARCH_CC_CAN_LINK if ARCH_HAS_CC_CAN_LINK
-	default $(cc_can_link_user,$(m64-flag)) if 64BIT
-	default $(cc_can_link_user,$(m32-flag))
-
-# Fixed in GCC 14, 13.3, 12.4 and 11.5
-# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=113921
-config GCC_ASM_GOTO_OUTPUT_BROKEN
-	bool
-	depends on CC_IS_GCC
-	default y if GCC_VERSION < 110500
-	default y if GCC_VERSION >= 120000 && GCC_VERSION < 120400
-	default y if GCC_VERSION >= 130000 && GCC_VERSION < 130300
-
-config CC_HAS_ASM_GOTO_OUTPUT
-	def_bool y
-	depends on !GCC_ASM_GOTO_OUTPUT_BROKEN
-	depends on $(success,echo 'int foo(int x) { asm goto ("": "=r"(x) ::: bar); return x; bar: return 0; }' | $(CC) -x c - -c -o /dev/null)
-
-config CC_HAS_ASM_GOTO_TIED_OUTPUT
-	depends on CC_HAS_ASM_GOTO_OUTPUT
-	# Detect buggy gcc and clang, fixed in gcc-11 clang-14.
-	def_bool $(success,echo 'int foo(int *x) { asm goto (".long (%l[bar]) - .": "+m"(*x) ::: bar); return *x; bar: return 0; }' | $CC -x c - -c -o /dev/null)
-
-config TOOLS_SUPPORT_RELR
-	def_bool $(success,env "CC=$(CC)" "LD=$(LD)" "NM=$(NM)" "OBJCOPY=$(OBJCOPY)" $(srctree)/scripts/tools-support-relr.sh)
-
-config CC_HAS_ASM_INLINE
-	def_bool $(success,echo 'void foo(void) { asm inline (""); }' | $(CC) -x c - -c -o /dev/null)
-
-config CC_HAS_ASSUME
-	bool
-	# clang needs to be at least 19.1.0 since the meaning of the assume
-	# attribute changed:
-	# https://github.com/llvm/llvm-project/commit/c44fa3e8a9a44c2e9a575768a3c185354b9f6c17
-	default y if CC_IS_CLANG && CLANG_VERSION >= 190100
-	# supported since gcc 13.1.0
-	# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106654
-	default y if CC_IS_GCC && GCC_VERSION >= 130100
-
-config CC_HAS_NO_PROFILE_FN_ATTR
-	def_bool $(success,echo '__attribute__((no_profile_instrument_function)) int x();' | $(CC) -x c - -c -o /dev/null -Werror)
-
-config CC_HAS_COUNTED_BY
-	bool
-	# clang needs to be at least 20.1.0 to avoid potential crashes
-	# when building structures that contain __counted_by
-	# https://github.com/ClangBuiltLinux/linux/issues/2114
-	# https://github.com/llvm/llvm-project/commit/160fb1121cdf703c3ef5e61fb26c5659eb581489
-	default y if CC_IS_CLANG && CLANG_VERSION >= 200100
-	# supported since gcc 15.1.0
-	# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896
-	default y if CC_IS_GCC && GCC_VERSION >= 150100
-
-config CC_HAS_COUNTED_BY_PTR
-	bool
-	# supported since clang 22
-	default y if CC_IS_CLANG && CLANG_VERSION >= 220100
-	# supported since gcc 16.0.0
-	default y if CC_IS_GCC && GCC_VERSION >= 160000
-
-config CC_HAS_BROKEN_COUNTED_BY_REF
-	bool
-	# https://github.com/llvm/llvm-project/issues/182575
-	default y if CC_IS_CLANG && CLANG_VERSION < 220100
-
-config CC_HAS_ALLOC_TOKEN
-	def_bool $(cc-option,-falloc-token-max=123)
-
-config CC_HAS_MULTIDIMENSIONAL_NONSTRING
-	def_bool $(success,echo 'char tag[][4] __attribute__((__nonstring__)) = { };' | $(CC) $(CLANG_FLAGS) -x c - -c -o /dev/null -Werror)
-
-config LD_CAN_USE_KEEP_IN_OVERLAY
-	# ld.lld prior to 21.0.0 did not support KEEP within an overlay description
-	# https://github.com/llvm/llvm-project/pull/130661
-	def_bool LD_IS_BFD || LLD_VERSION >= 210000
-
-config RUSTC_HAS_SPAN_FILE
-	def_bool RUSTC_VERSION >= 108800
-
-config RUSTC_HAS_UNNECESSARY_TRANSMUTES
-	def_bool RUSTC_VERSION >= 108800
-
-config RUSTC_HAS_FILE_WITH_NUL
-	def_bool RUSTC_VERSION >= 108900
-
-config RUSTC_HAS_FILE_AS_C_STR
-	def_bool RUSTC_VERSION >= 109100
-
-config RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS
-	def_bool RUSTC_VERSION >= 109800
-
-config PAHOLE_VERSION
-	int
-	default "$(PAHOLE_VERSION)"
+source "scripts/Kconfig.toolchain"
 
 config CONSTRUCTORS
 	bool
@@ -231,9 +36,10 @@ config BROKEN_ON_SMP
 	default y
 
 config INIT_ENV_ARG_LIMIT
-	int
+	int "Maximum number of kernel command line arguments"
 	default 32 if !UML
 	default 128 if UML
+	range 32 4096
 	help
 	  Maximum of each of the number of arguments and environment
 	  variables passed to init from the kernel command line.
@@ -938,8 +744,6 @@ config SCHED_PROXY_EXEC
 	bool "Proxy Execution"
 	# Avoid some build failures w/ PREEMPT_RT until it can be fixed
 	depends on !PREEMPT_RT
-	# Need to investigate how to inform sched_ext of split contexts
-	depends on !SCHED_CLASS_EXT
 	# Not particularly useful until we get to multi-rq proxying
 	depends on EXPERT
 	help
@@ -1457,6 +1261,17 @@ config USER_NS
 
 	  If unsure, say N.
 
+config USER_NS_MAP_KUNIT_TEST
+	tristate "KUint test for user namespace map insertion" if !KUNIT_ALL_TESTS
+	depends on USER_NS && KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This builds the KUnit test for user namespace uid/gid map insertion.
+	  It validates map insertion, limits, dynamic allocation of the
+	  extended extents array, and mapping sorting functions.
+
+	  If unsure, say N.
+
 config PID_NS
 	bool "PID Namespaces"
 	default y
@@ -1614,6 +1429,22 @@ config CMDLINE_FROM_BOOTCONFIG
 
 	  If unsure, say N.
 
+config COMMAND_LINE_SIZE
+	int "Maximum size of kernel command line"
+	default 4096 if S390 || LOONGARCH || MIPS || UML
+	default 2048 if X86 || ARM64 || PPC || SPARC64 || RISCV
+	default 1024 if ARM || PARISC
+	default 256 if ALPHA || ARC || M68K || MICROBLAZE || SPARC32 || XTENSA
+	default 512
+	range 896 1048576 if S390
+	range 256 2048 if ARM || M68K || NIOS2 || PPC
+	range 256 3840 if SUPERH
+	range 256 256 if ALPHA
+	range 256 4096
+	help
+	  This allows you to specify the maximum length of the kernel command
+	  line.
+
 config CMDLINE_LOG_WRAP_IDEAL_LEN
 	int "Length to try to wrap the cmdline when logged at boot"
 	default 1021
diff --git a/init/Makefile b/init/Makefile
index d6f75d8907e098..9102fc34c0cc0f 100644
--- a/init/Makefile
+++ b/init/Makefile
@@ -6,10 +6,10 @@
 ccflags-y := -fno-function-sections -fno-data-sections
 
 obj-y                          := main.o version.o mounts.o
-ifneq ($(CONFIG_BLK_DEV_INITRD),y)
-obj-y                          += noinitramfs.o
+ifeq ($(CONFIG_BLK_DEV_INITRD),y)
+obj-y                          += initramfs.o
 else
-obj-$(CONFIG_BLK_DEV_INITRD)   += initramfs.o
+obj-y                          += noinitramfs.o
 endif
 obj-$(CONFIG_GENERIC_CALIBRATE_DELAY) += calibrate.o
 obj-$(CONFIG_INITRAMFS_TEST)   += initramfs_test.o
diff --git a/init/init_task.c b/init/init_task.c
index adb207cd987c53..ce7c2b07d855d2 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -160,7 +160,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 	.sibling	= LIST_HEAD_INIT(init_task.sibling),
 	.group_leader	= &init_task,
 	RCU_POINTER_INITIALIZER(real_cred, &init_cred),
-	RCU_POINTER_INITIALIZER(cred, &init_cred),
+	.cred		= &init_cred,
 	.comm		= INIT_TASK_COMM,
 	.thread		= INIT_THREAD,
 	.real_fs	= &init_fs,
diff --git a/init/initramfs.c b/init/initramfs.c
index 3cee8b50ad8281..ebddde9c8f0d91 100644
--- a/init/initramfs.c
+++ b/init/initramfs.c
@@ -80,7 +80,7 @@ static __initdata struct hash {
 	int ino, minor, major;
 	umode_t mode;
 	struct hash *next;
-	char name[N_ALIGN(PATH_MAX)];
+	char name[];
 } *head[32];
 static __initdata bool hardlink_seen;
 
@@ -92,7 +92,7 @@ static inline int hash(int major, int minor, int ino)
 }
 
 static char __init *find_link(int major, int minor, int ino,
-			      umode_t mode, char *name)
+			      umode_t mode, const char *name, size_t nlen)
 {
 	struct hash **p, *q;
 	for (p = head + hash(major, minor, ino); *p; p = &(*p)->next) {
@@ -106,14 +106,15 @@ static char __init *find_link(int major, int minor, int ino,
 			continue;
 		return (*p)->name;
 	}
-	q = kmalloc_obj(struct hash);
+
+	q = kmalloc_flex(struct hash, name, nlen);
 	if (!q)
 		panic_show_mem("can't allocate link hash entry");
 	q->major = major;
 	q->minor = minor;
 	q->ino = ino;
 	q->mode = mode;
-	strscpy(q->name, name);
+	strscpy(q->name, name, nlen);
 	q->next = NULL;
 	*p = q;
 	hardlink_seen = true;
@@ -355,7 +356,7 @@ static void __init clean_path(char *path, umode_t fmode)
 static int __init maybe_link(void)
 {
 	if (nlink >= 2) {
-		char *old = find_link(major, minor, ino, mode, collected);
+		char *old = find_link(major, minor, ino, mode, collected, name_len);
 		if (old) {
 			clean_path(collected, 0);
 			return (init_link(old, collected) < 0) ? -1 : 1;
diff --git a/init/main.c b/init/main.c
index 31f2bf54976ab9..e6f7174b05c8c8 100644
--- a/init/main.c
+++ b/init/main.c
@@ -108,6 +108,7 @@
 #include <linux/time_namespace.h>
 #include <linux/unaligned.h>
 #include <linux/vdso_datastore.h>
+#include <linux/hazptr.h>
 #include <net/net_namespace.h>
 
 #include <asm/io.h>
@@ -215,10 +216,6 @@ static bool __init obsolete_checksetup(char *line)
 				 * params and __setups of same names 8( */
 				if (line[n] == '\0' || line[n] == '=')
 					had_early_param = true;
-			} else if (!p->setup_func) {
-				pr_warn("Parameter %s is obsolete, ignored\n",
-					p->str);
-				return true;
 			} else if (p->setup_func(line + n))
 				return true;
 		}
@@ -548,12 +545,12 @@ static int __init unknown_bootoption(char *param, char *val,
 		/* Environment option */
 		unsigned int i;
 		for (i = 0; envp_init[i]; i++) {
+			if (!strncmp(param, envp_init[i], len+1))
+				break;
 			if (i == MAX_INIT_ENVS) {
 				panic_later = "env";
 				panic_param = param;
 			}
-			if (!strncmp(param, envp_init[i], len+1))
-				break;
 		}
 		envp_init[i] = param;
 	} else {
@@ -581,7 +578,7 @@ static int __init init_setup(char *str)
 	 * the shell think it should execute a script with such name.
 	 * So we ignore all arguments entered _before_ init=... [MJ]
 	 */
-	for (i = 1; i < MAX_INIT_ARGS; i++)
+	for (i = 1; i <= MAX_INIT_ARGS; i++)
 		argv_init[i] = NULL;
 	return 1;
 }
@@ -594,7 +591,7 @@ static int __init rdinit_setup(char *str)
 	ramdisk_execute_command = str;
 	ramdisk_execute_command_set = true;
 	/* See "auto" comment in init_setup */
-	for (i = 1; i < MAX_INIT_ARGS; i++)
+	for (i = 1; i <= MAX_INIT_ARGS; i++)
 		argv_init[i] = NULL;
 	return 1;
 }
@@ -1080,6 +1077,7 @@ void start_kernel(void)
 	workqueue_init_early();
 
 	rcu_init();
+	hazptr_init();
 	kvfree_rcu_init();
 
 	/* Trace events are available after this */
@@ -1535,7 +1533,7 @@ static void mark_readonly(void)
 		flush_module_init_free_work();
 		jump_label_init_ro();
 		mark_rodata_ro();
-		debug_checkwx();
+		pgtable_checkwx();
 		rodata_test();
 	} else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) {
 		pr_info("Kernel memory protection disabled.\n");
diff --git a/init/version.c b/init/version.c
index 94c96f6fbfe6a2..cbae11b6f7688b 100644
--- a/init/version.c
+++ b/init/version.c
@@ -21,13 +21,14 @@ static int __init early_hostname(char *arg)
 {
 	size_t bufsize = sizeof(init_uts_ns.name.nodename);
 	size_t maxlen  = bufsize - 1;
-	ssize_t arglen;
 
-	arglen = strscpy(init_uts_ns.name.nodename, arg, bufsize);
-	if (arglen < 0) {
-		pr_warn("hostname parameter exceeds %zd characters and will be truncated",
+	if (!arg)
+		return -EINVAL;
+
+	if (strscpy(init_uts_ns.name.nodename, arg, bufsize) < 0)
+		pr_warn("hostname parameter exceeds %zu characters and will be truncated\n",
 			maxlen);
-	}
+
 	return 0;
 }
 early_param("hostname", early_hostname);
diff --git a/io_uring/bpf_filter.c b/io_uring/bpf_filter.c
index c0037632b7af60..4a21511c481163 100644
--- a/io_uring/bpf_filter.c
+++ b/io_uring/bpf_filter.c
@@ -253,6 +253,7 @@ void io_bpf_filter_clone(struct io_restriction *dst, struct io_restriction *src)
 	 * If the src filter is going away, just ignore it.
 	 */
 	if (refcount_inc_not_zero(&src->bpf_filters->refs)) {
+		src->bpf_filters_cow = true;
 		dst->bpf_filters = src->bpf_filters;
 		dst->bpf_filters_cow = true;
 	}
diff --git a/io_uring/cancel.c b/io_uring/cancel.c
index 7d7820eab878be..911a47075064ce 100644
--- a/io_uring/cancel.c
+++ b/io_uring/cancel.c
@@ -514,8 +514,9 @@ static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
 
 __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 					 struct io_uring_task *tctx,
-					 bool cancel_all, bool is_sqpoll_thread)
+					 unsigned int flags)
 {
+	bool cancel_all = flags & IO_CANCEL_ALL;
 	struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };
 	enum io_wq_cancel cret;
 	bool ret = false;
@@ -544,7 +545,7 @@ __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 
 	/* SQPOLL thread does its own polling */
 	if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
-	    is_sqpoll_thread) {
+	    (flags & IO_CANCEL_SQPOLL)) {
 		while (!list_empty(&ctx->iopoll_list)) {
 			io_iopoll_try_reap_events(ctx);
 			ret = true;
@@ -560,7 +561,9 @@ __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 	ret |= io_poll_remove_all(ctx, tctx, cancel_all);
 	ret |= io_waitid_remove_all(ctx, tctx, cancel_all);
 	ret |= io_futex_remove_all(ctx, tctx, cancel_all);
-	ret |= io_uring_try_cancel_uring_cmd(ctx, tctx, cancel_all);
+	ret |= io_uring_try_cancel_uring_cmd(ctx, tctx);
+	if (flags & IO_CANCEL_KEEP_TIMEOUTS)
+		cancel_all = false;
 	ret |= io_kill_timeouts(ctx, tctx, cancel_all);
 	mutex_unlock(&ctx->uring_lock);
 	if (tctx)
@@ -575,6 +578,44 @@ static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
 	return percpu_counter_sum(&tctx->inflight);
 }
 
+/*
+ * If true, whole thread group is exiting, at which point no task is left that
+ * can reap completions and care about requests in-flight.
+ */
+static bool io_task_group_exiting(void)
+{
+	return current->signal->flags & SIGNAL_GROUP_EXIT;
+}
+
+/*
+ * Return a count of requests an exiting task should wait for. 
+ */
+static s64 tctx_inflight_exit(struct io_uring_task *tctx)
+{
+	struct io_tctx_node *node;
+	unsigned long index;
+	s64 inflight;
+
+	inflight = tctx_inflight(tctx, false);
+	xa_for_each(&tctx->xa, index, node) {
+		/* unlocked read is fine, the caller re-evaluates until done */
+		inflight -= data_race(node->ctx->nr_notifs);
+		/* takes ->uring_lock, we hold nothing on the group exit path */
+		inflight -= io_timeouts_armed(node->ctx, tctx);
+	}
+	return inflight;
+}
+
+static bool io_tctx_cancel_done(struct io_uring_task *tctx, bool cancel_all,
+				bool group_exit)
+{
+	if (cancel_all)
+		return !tctx_inflight(tctx, false);
+	if (tctx_inflight(tctx, true))
+		return false;
+	return !group_exit || tctx_inflight_exit(tctx) <= 0;
+}
+
 /*
  * Find any io_uring ctx that this task has registered or done IO on, and cancel
  * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
@@ -584,7 +625,9 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
 	struct io_uring_task *tctx = current->io_uring;
 	struct io_ring_ctx *ctx;
 	struct io_tctx_node *node;
+	unsigned int flags = 0;
 	unsigned long index;
+	bool group_exit;
 	s64 inflight;
 	DEFINE_WAIT(wait);
 
@@ -595,18 +638,30 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
 	if (tctx->io_wq)
 		io_wq_exit_start(tctx->io_wq);
 
+	/*
+	 * If a whole thread group is exiting, nobody will look at completions.
+	 * If a single thread is exiting, cancel only those that belong to that
+	 * thread.
+	 */
+	group_exit = !cancel_all && io_task_group_exiting();
+	if (cancel_all)
+		flags = IO_CANCEL_ALL;
+	else if (group_exit)
+		flags = IO_CANCEL_ALL | IO_CANCEL_KEEP_TIMEOUTS;
+	if (sqd)
+		flags |= IO_CANCEL_SQPOLL;
+
 	atomic_inc(&tctx->in_cancel);
 	do {
 		bool loop = false;
+		unsigned int state;
 
 		io_uring_drop_tctx_refs(current);
-		if (!tctx_inflight(tctx, !cancel_all))
+		if (io_tctx_cancel_done(tctx, cancel_all, group_exit))
 			break;
 
 		/* read completions before cancelations */
 		inflight = tctx_inflight(tctx, false);
-		if (!inflight)
-			break;
 
 		if (!sqd) {
 			xa_for_each(&tctx->xa, index, node) {
@@ -615,15 +670,13 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
 					continue;
 				loop |= io_uring_try_cancel_requests(node->ctx,
 							current->io_uring,
-							cancel_all,
-							false);
+							flags);
 			}
 		} else {
 			list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
 				loop |= io_uring_try_cancel_requests(ctx,
 								     current->io_uring,
-								     cancel_all,
-								     true);
+								     flags);
 		}
 
 		if (loop) {
@@ -631,7 +684,12 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
 			continue;
 		}
 
-		prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
+		state = TASK_INTERRUPTIBLE;
+		if (task_sigpending(current))
+			state = TASK_UNINTERRUPTIBLE;
+		if (!cancel_all)
+			state |= TASK_FREEZABLE;
+		prepare_to_wait(&tctx->wait, &wait, state);
 		io_run_task_work();
 		io_uring_drop_tctx_refs(current);
 		xa_for_each(&tctx->xa, index, node) {
@@ -646,8 +704,13 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
 		 * avoids a race where a completion comes in before we did
 		 * prepare_to_wait().
 		 */
-		if (inflight == tctx_inflight(tctx, !cancel_all))
-			schedule();
+		if (inflight == tctx_inflight(tctx, false)) {
+			unsigned long timeout = 1;
+
+			if (state & TASK_INTERRUPTIBLE)
+				timeout = MAX_SCHEDULE_TIMEOUT;
+			schedule_timeout(timeout);
+		}
 end_wait:
 		finish_wait(&tctx->wait, &wait);
 	} while (1);
diff --git a/io_uring/cancel.h b/io_uring/cancel.h
index 1b201a0943030e..e49713a1bc6691 100644
--- a/io_uring/cancel.h
+++ b/io_uring/cancel.h
@@ -30,9 +30,20 @@ bool io_cancel_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
 int io_cancel_remove(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
 		     unsigned int issue_flags, struct hlist_head *list,
 		     bool (*cancel)(struct io_kiocb *));
+
+/* io_uring_try_cancel_requests() flags */
+enum {
+	/* match all requests, not just REQ_F_INFLIGHT */
+	IO_CANCEL_ALL		= 1,
+	/* ignore timeouts */
+	IO_CANCEL_KEEP_TIMEOUTS	= 2,
+	/* called by the SQPOLL thread */
+	IO_CANCEL_SQPOLL	= 4,
+};
+
 __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 					 struct io_uring_task *tctx,
-					 bool cancel_all, bool is_sqpoll_thread);
+					 unsigned int flags);
 __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
 __cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data);
 
diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c
index 7cd411fc4f33eb..90d4ec7cc7615c 100644
--- a/io_uring/cmd_net.c
+++ b/io_uring/cmd_net.c
@@ -69,8 +69,8 @@ static inline int io_uring_cmd_setsockopt(struct socket *sock,
 				  optlen);
 }
 
-static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
-				     struct sk_buff *skb, unsigned issue_flags)
+static int io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
+				    struct sk_buff *skb, unsigned int issue_flags)
 {
 	struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
 	struct io_uring_cqe cqe[2];
@@ -83,7 +83,7 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
 
 	ret = skb_get_tx_timestamp(skb, sk, &ts);
 	if (ret < 0)
-		return false;
+		return ret;
 
 	tskey = serr->ee.ee_data;
 	tstype = serr->ee.ee_info;
@@ -98,7 +98,9 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk,
 	iots = (struct io_timespec *)&cqe[1];
 	iots->tv_sec = ts.tv_sec;
 	iots->tv_nsec = ts.tv_nsec;
-	return io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe);
+	if (!io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe))
+		return -ENOBUFS;
+	return 0;
 }
 
 static int io_uring_cmd_timestamp(struct socket *sock,
@@ -135,7 +137,8 @@ static int io_uring_cmd_timestamp(struct socket *sock,
 		skb = skb_peek(&list);
 		if (!skb)
 			break;
-		if (!io_process_timestamp_skb(cmd, sk, skb, issue_flags))
+		ret = io_process_timestamp_skb(cmd, sk, skb, issue_flags);
+		if (ret)
 			break;
 		__skb_dequeue(&list);
 		consume_skb(skb);
@@ -145,6 +148,12 @@ static int io_uring_cmd_timestamp(struct socket *sock,
 		scoped_guard(spinlock_irqsave, &q->lock)
 			skb_queue_splice(&list, q);
 	}
+	/*
+	 * Aux CQEs cannot overflow and the poll is edge triggered, so nothing
+	 * re-runs the command once the CQ drains. End the multishot instead.
+	 */
+	if (ret == -ENOBUFS)
+		return -ENOBUFS;
 	return -EAGAIN;
 }
 
diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c
index 3ae4804765b945..c362f3a81bdce8 100644
--- a/io_uring/fdinfo.c
+++ b/io_uring/fdinfo.c
@@ -155,9 +155,18 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
 	for (i = 0; i < cq_entries; i++) {
 		struct io_uring_cqe *cqe;
 		bool cqe32 = false;
+		bool is_last_cqarray_slot = (cq_head & cq_mask) == cq_mask;
 
 		cqe = &r->cqes[(cq_head & cq_mask)];
-		if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32)
+		/*
+		 * Userspace can manipulate the last cqarray slot to have
+		 * IORING_CQE_F_32 set, which would cause the second half of
+		 * that CQE to be read out of bounds.
+		 * Ignore the flag for the last cqarray slot, which can't
+		 * legitimately have the flag set.
+		 */
+		if ((cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) &&
+		    !is_last_cqarray_slot)
 			cqe32 = true;
 		seq_printf(m, "%5u: user_data:%llu, res:%d, flags:%x",
 			   cq_head & cq_mask, cqe->user_data, cqe->res,
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 2ca223e47d41f5..eecba70fd52508 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -526,7 +526,8 @@ static bool io_wait_on_hash(struct io_wq *wq, unsigned int hash)
 }
 
 static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,
-					   struct io_wq *wq)
+					   struct io_wq *wq,
+					   bool *need_wake)
 	__must_hold(acct->lock)
 {
 	struct io_wq_work_node *node, *prev;
@@ -575,8 +576,7 @@ static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,
 		raw_spin_lock(&acct->lock);
 		if (unstalled) {
 			clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
-			if (wq_has_sleeper(&wq->hash->wait))
-				wake_up(&wq->hash->wait);
+			*need_wake = wq_has_sleeper(&wq->hash->wait);
 		}
 	}
 
@@ -607,6 +607,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
 
 	do {
 		struct io_wq_work *work;
+		bool need_wake = false;
 
 		/*
 		 * If we got some work, mark us as busy. If we didn't, but
@@ -615,7 +616,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
 		 * can't make progress, any work completion or insertion will
 		 * clear the stalled flag.
 		 */
-		work = io_get_next_work(acct, wq);
+		work = io_get_next_work(acct, wq, &need_wake);
 		if (work) {
 			/*
 			 * Make sure cancelation can find this, even before
@@ -631,6 +632,9 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
 
 		raw_spin_unlock(&acct->lock);
 
+		if (need_wake)
+			wake_up(&wq->hash->wait);
+
 		if (!work)
 			break;
 
@@ -1324,6 +1328,8 @@ static bool io_task_work_match(struct callback_head *cb, void *data)
 void io_wq_exit_start(struct io_wq *wq)
 {
 	set_bit(IO_WQ_BIT_EXIT, &wq->state);
+	/* Pairs with task_work_add() in io_queue_worker_create(). */
+	smp_mb__after_atomic();
 }
 
 static void io_wq_cancel_tw_create(struct io_wq *wq)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 61053421d809a7..ecda7fe2ed9d31 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -733,7 +733,11 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
 	 * Post dummy CQE if a 32b CQE is needed and there's only room for a
 	 * 16b CQE before the ring wraps.
 	 */
-	if (cqe32 && off + 1 == ctx->cq_entries) {
+	if (cqe32 && (ctx->flags & IORING_SETUP_CQE_MIXED) &&
+	    off + 1 == ctx->cq_entries) {
+		/* one for the dummy CQE, two for the 32b CQE itself */
+		if (ctx->cq_entries - io_cqring_queued(ctx) < 3)
+			return false;
 		if (!io_fill_nop_cqe(ctx, off))
 			return false;
 		off = 0;
@@ -742,12 +746,13 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32)
 	free = ctx->cq_entries - io_cqring_queued(ctx);
 	/* we need a contiguous range, limit based on the current array offset */
 	len = min(free, ctx->cq_entries - off);
-	if (len < (cqe32 + 1))
-		return false;
-
 	if (ctx->flags & IORING_SETUP_CQE32) {
+		if (!len)
+			return false;
 		off <<= 1;
 		len <<= 1;
+	} else if (len < (cqe32 + 1)) {
+		return false;
 	}
 
 	ctx->cqe_cached = &rings->cqes[off];
@@ -781,7 +786,7 @@ static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
 		WRITE_ONCE(cqe->res, res);
 		WRITE_ONCE(cqe->flags, cflags);
 
-		if (cqe32) {
+		if (cqe32 || (ctx->flags & IORING_SETUP_CQE32)) {
 			WRITE_ONCE(cqe->big_cqe[0], 0);
 			WRITE_ONCE(cqe->big_cqe[1], 0);
 		}
@@ -907,6 +912,22 @@ bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])
 	return posted;
 }
 
+/* drop the request file before the CQE is posted, not deferred after it */
+static void io_req_put_file(struct io_kiocb *req, bool sync)
+{
+	struct file *file = req->file;
+
+	if (!file || (req->flags & (REQ_F_FIXED_FILE | REQ_F_REISSUE)))
+		return;
+
+	WRITE_ONCE(req->file, NULL);
+	/* releasing a ring may wait on other rings, keep that deferred */
+	if (sync && !io_is_uring_fops(file))
+		__fput_sync(file);
+	else
+		fput(file);
+}
+
 static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 {
 	struct io_ring_ctx *ctx = req->ctx;
@@ -919,6 +940,8 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 	if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_IOWQ)))
 		return;
 
+	io_req_put_file(req, true);
+
 	/*
 	 * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
 	 * the submitter task context, IOPOLL protects with uring_lock.
@@ -939,9 +962,11 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 		goto defer_complete;
 
 	/*
-	 * We don't free the request here because we know it's called from
-	 * io-wq only, which holds a reference, so it cannot be the last put.
+	 * Request not freed here because we know it's called from io-wq only,
+	 * which holds a reference. Hence it can't be the last put. The CQE
+	 * has been posted, last put frees it.
 	 */
+	req->flags |= REQ_F_CQE_SKIP;
 	req_ref_put(req);
 }
 
@@ -995,8 +1020,6 @@ __cold void io_free_req(struct io_kiocb *req)
 {
 	/* refs were already put, restore them for io_req_task_complete() */
 	req->flags &= ~REQ_F_REFCOUNT;
-	/* we only want to free it, don't post CQEs */
-	req->flags |= REQ_F_CQE_SKIP;
 	req->io_task_work.func = io_req_task_complete;
 	io_req_task_work_add(req);
 }
@@ -1099,6 +1122,8 @@ static void io_free_batch_list(struct io_ring_ctx *ctx,
 			}
 			if (req->flags & REQ_F_REFCOUNT) {
 				node = req->comp_list.next;
+				/* CQE posted, the last put only frees */
+				req->flags |= REQ_F_CQE_SKIP;
 				if (!req_ref_put_and_test(req))
 					continue;
 			}
@@ -1130,6 +1155,19 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 	struct io_submit_state *state = &ctx->submit_state;
 	struct io_wq_work_node *node;
 
+	/*
+	 * Drop the files before the CQEs are posted, so they're released by
+	 * the time the completions are visible. Not for requests that are
+	 * requeued or still referenced, those aren't freed below.
+	 */
+	__wq_list_for_each(node, &state->compl_reqs) {
+		struct io_kiocb *req = container_of(node, struct io_kiocb,
+					    comp_list);
+
+		if (!io_req_shared(req))
+			io_req_put_file(req, true);
+	}
+
 	__io_cq_lock(ctx);
 	__wq_list_for_each(node, &state->compl_reqs) {
 		struct io_kiocb *req = container_of(node, struct io_kiocb,
@@ -1262,7 +1300,10 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
 
 void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw)
 {
-	io_req_complete_defer(tw_req.req);
+	struct io_kiocb *req = tw_req.req;
+
+	if (io_req_complete_ready(req))
+		io_req_complete_defer(req);
 }
 
 /*
@@ -1451,7 +1492,13 @@ struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
 	struct io_kiocb *nxt = NULL;
 
 	if (req_ref_put_and_test_atomic(req)) {
-		if (req->flags & IO_REQ_LINK_FLAGS) {
+		/*
+		 * Only continue the link from here if the completion has been
+		 * posted. If it's still pending, io_free_req() completes the
+		 * request and the next link follows from there, after the CQE.
+		 */
+		if ((req->flags & IO_REQ_LINK_FLAGS) &&
+		    (req->flags & REQ_F_CQE_SKIP)) {
 			struct io_ring_ctx *ctx = req->ctx;
 
 			mutex_lock(&ctx->uring_lock);
@@ -1480,6 +1527,7 @@ void io_wq_submit_work(struct io_wq_work *work)
 	/* either cancelled or io-wq is dying, so don't touch tctx->iowq */
 	if (atomic_read(&work->flags) & IO_WQ_WORK_CANCEL) {
 fail:
+		io_req_put_file(req, false);
 		io_req_task_queue_fail(req, err);
 		return;
 	}
@@ -1555,8 +1603,10 @@ void io_wq_submit_work(struct io_wq_work *work)
 	} while (1);
 
 	/* avoid locking problems by failing it from a clean context */
-	if (ret)
+	if (ret) {
+		io_req_put_file(req, true);
 		io_req_task_queue_fail(req, ret);
+	}
 }
 
 inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
@@ -2157,8 +2207,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 	io_sq_thread_finish(ctx);
 
 	mutex_lock(&ctx->uring_lock);
-	io_sqe_buffers_unregister(ctx);
-	io_sqe_files_unregister(ctx);
 	io_unregister_zcrx(ctx);
 	io_cqring_overflow_kill(ctx);
 	io_eventfd_unregister(ctx);
@@ -2306,18 +2354,18 @@ static __cold void io_tctx_exit_cb(struct callback_head *cb)
 	complete(&work->completion);
 }
 
-static __cold void io_ring_exit_work(struct work_struct *work)
+/*
+ * Cancel what can be canceled on a dying ring and reap what has completed.
+ * Only waits on polled I/O, never anything else.
+ */
+static __cold void io_ring_ctx_cancel(struct io_ring_ctx *ctx)
 {
-	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
-	unsigned long timeout = jiffies + IO_URING_EXIT_WAIT_MAX;
-	unsigned long interval = HZ / 20;
-	struct io_tctx_exit exit;
-	struct io_tctx_node *node;
-	int ret;
+	struct io_sq_data *sqd = ctx->sq_data;
 
-	mutex_lock(&ctx->uring_lock);
-	io_terminate_zcrx(ctx);
-	mutex_unlock(&ctx->uring_lock);
+	if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) {
+		scoped_guard(mutex, &ctx->uring_lock)
+			io_cqring_overflow_kill(ctx);
+	}
 
 	/*
 	 * If we're doing polled IO and end up having requests being
@@ -2326,32 +2374,106 @@ static __cold void io_ring_exit_work(struct work_struct *work)
 	 * as nobody else will be looking for them.
 	 */
 	do {
-		if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) {
-			mutex_lock(&ctx->uring_lock);
-			io_cqring_overflow_kill(ctx);
-			mutex_unlock(&ctx->uring_lock);
+		if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+			io_cancel_local_task_work(ctx);
+		cond_resched();
+	} while (io_uring_try_cancel_requests(ctx, NULL, IO_CANCEL_ALL));
+
+	if (sqd) {
+		struct task_struct *tsk;
+
+		io_sq_thread_park(sqd);
+		tsk = sqpoll_task_locked(sqd);
+		if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
+			io_wq_cancel_cb(tsk->io_uring->io_wq,
+					io_cancel_ctx_cb, ctx, true);
+		io_sq_thread_unpark(sqd);
+	}
+
+	io_req_caches_free(ctx);
+}
+
+/* Number of requests that should be waited for */
+static __cold unsigned int io_ring_ctx_inflight(struct io_ring_ctx *ctx)
+{
+	guard(mutex)(&ctx->uring_lock);
+	__io_req_caches_free(ctx);
+	return ctx->nr_req_allocated - ctx->nr_notifs;
+}
+
+/*
+ * Run task_work completions for current. Only do so if the io_uring callback
+ * itself can get pruned first, otherwise we risk recursing.
+ */
+static __cold bool io_ring_run_own_completions(struct io_uring_task *tctx)
+{
+	unsigned int count = 0;
+
+	if (!tctx || mpscq_empty(&tctx->task_list))
+		return true;
+	if (!task_work_cancel(current, &tctx->task_work))
+		return false;
+	tctx_task_work_run(tctx, UINT_MAX, &count);
+	return true;
+}
+
+/*
+ * Requests may remain after cancelations have been run, as not all requests
+ * are cancelable. Storage I/O is an example. Wait for those so that once
+ * close(2) returns, files pinned by these requests have been released.
+ */
+static __cold void io_ring_ctx_wait_inflight(struct io_ring_ctx *ctx)
+{
+	struct io_uring_task *tctx = current->io_uring;
+	bool ran_own = true;
+
+	if (current->flags & (PF_KTHREAD | PF_EXITING))
+		return;
+	if (tctx && atomic_read(&tctx->in_cancel))
+		return;
+
+	while (io_ring_ctx_inflight(ctx) && !fatal_signal_pending(current)) {
+		unsigned int state;
+
+		if (test_thread_flag(TIF_NOTIFY_SIGNAL)) {
+			clear_notify_signal();
+			if (task_work_pending(current))
+				set_notify_resume(current);
 		}
+		state = TASK_INTERRUPTIBLE;
+		if (signal_pending(current))
+			state = TASK_KILLABLE;
+		set_current_state(state | TASK_FREEZABLE);
+		/* don't sleep on work that's already there and that we can run */
+		if (ran_own && ((tctx && !mpscq_empty(&tctx->task_list)) ||
+		    io_local_work_pending(ctx)))
+			__set_current_state(TASK_RUNNING);
+		else
+			schedule_timeout(1);
 
-		/* The SQPOLL thread never reaches this path */
-		do {
-			if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
-				io_cancel_local_task_work(ctx);
-			cond_resched();
-		} while (io_uring_try_cancel_requests(ctx, NULL, true, false));
-
-		if (ctx->sq_data) {
-			struct io_sq_data *sqd = ctx->sq_data;
-			struct task_struct *tsk;
-
-			io_sq_thread_park(sqd);
-			tsk = sqpoll_task_locked(sqd);
-			if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
-				io_wq_cancel_cb(tsk->io_uring->io_wq,
-						io_cancel_ctx_cb, ctx, true);
-			io_sq_thread_unpark(sqd);
+		/* completions may be queued behind us */
+		if (!io_ring_run_own_completions(tctx)) {
+			if (!ran_own)
+				break;
+			ran_own = false;
+		} else {
+			ran_own = true;
 		}
+		io_ring_ctx_cancel(ctx);
+	}
+}
+
+static __cold void io_ring_exit_work(struct work_struct *work)
+{
+	struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
+	unsigned long timeout = jiffies + IO_URING_EXIT_WAIT_MAX;
+	unsigned long interval = HZ / 20;
+	struct io_tctx_exit exit;
+	struct io_tctx_node *node;
+	int ret;
 
-		io_req_caches_free(ctx);
+	do {
+		io_ring_ctx_cancel(ctx);
 
 		if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
 			/* there is little hope left, don't run it too often */
@@ -2403,8 +2525,8 @@ static __cold void io_ring_exit_work(struct work_struct *work)
 	spin_lock(&ctx->completion_lock);
 	spin_unlock(&ctx->completion_lock);
 
-	/* pairs with RCU read section in io_req_local_work_add() */
-	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
+	/* pairs with the RCU read sections in the task_work add paths */
+	if (ctx->flags & (IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SQPOLL))
 		synchronize_rcu();
 
 	io_ring_ctx_free(ctx);
@@ -2419,8 +2541,23 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
 	percpu_ref_kill(&ctx->refs);
 	xa_for_each(&ctx->personalities, index, creds)
 		io_unregister_personality(ctx, index);
+	io_terminate_zcrx(ctx);
+	/* Drop these now, rather than async, to unpin the files */
+	io_sqe_buffers_unregister(ctx);
+	io_sqe_files_unregister(ctx);
 	mutex_unlock(&ctx->uring_lock);
 
+	/*
+	 * Do the first round of cancelations upfront rather than leaving it
+	 * to to exit_work. Anything cancelable is then already on its way
+	 * out, and for requests owned by the task closing the ring, this
+	 * ensures any held files are put before close(2) returns.
+	 */
+	if (!(current->flags & PF_IO_WORKER)) {
+		io_ring_ctx_cancel(ctx);
+		io_ring_ctx_wait_inflight(ctx);
+	}
+
 	INIT_WORK(&ctx->exit_work, io_ring_exit_work);
 	/*
 	 * Use system_dfl_wq to avoid spawning tons of event kworkers
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 896aab1ed026cf..109261722ca74f 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -318,37 +318,6 @@ static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx,
 	return true;
 }
 
-static inline void req_set_fail(struct io_kiocb *req)
-{
-	req->flags |= REQ_F_FAIL;
-	if (req->flags & REQ_F_CQE_SKIP) {
-		req->flags &= ~REQ_F_CQE_SKIP;
-		req->flags |= REQ_F_SKIP_LINK_CQES;
-	}
-}
-
-static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
-{
-	req->cqe.res = res;
-	req->cqe.flags = cflags;
-}
-
-static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx)
-{
-	if (ctx->flags & IORING_SETUP_CQE_MIXED)
-		return IORING_CQE_F_32;
-	return 0;
-}
-
-static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags,
-				    __u64 extra1, __u64 extra2)
-{
-	req->cqe.res = res;
-	req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx);
-	req->big_cqe.extra1 = extra1;
-	req->big_cqe.extra2 = extra2;
-}
-
 static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache,
 					      struct io_kiocb *req)
 {
diff --git a/io_uring/loop.c b/io_uring/loop.c
index bbbb6ef14e6a8e..3d472b7827fac8 100644
--- a/io_uring/loop.c
+++ b/io_uring/loop.c
@@ -2,6 +2,7 @@
 #include "io_uring.h"
 #include "wait.h"
 #include "loop.h"
+#include "tctx.h"
 
 static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
 				  const struct iou_loop_params *lp)
@@ -81,6 +82,10 @@ int io_run_loop(struct io_ring_ctx *ctx)
 {
 	int ret;
 
+	ret = io_uring_add_tctx_node(ctx);
+	if (unlikely(ret))
+		return ret;
+
 	if (!io_allowed_run_tw(ctx))
 		return -EEXIST;
 
diff --git a/io_uring/mock_file.c b/io_uring/mock_file.c
index b318ed697998e4..9f0b4d850c12f8 100644
--- a/io_uring/mock_file.c
+++ b/io_uring/mock_file.c
@@ -257,17 +257,17 @@ static int io_create_mock_file(struct io_uring_cmd *cmd, unsigned int issue_flag
 	FD_PREPARE(fdf, O_RDWR | O_CLOEXEC,
 		   anon_inode_create_getfile("[io_uring_mock]", fops, mf,
 					     O_RDWR | O_CLOEXEC, NULL));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
 	retain_and_null_ptr(mf);
-	file = fd_prepare_file(fdf);
+	file = fdf->file;
 	file->f_mode |= FMODE_READ | FMODE_CAN_READ | FMODE_WRITE |
 			FMODE_CAN_WRITE | FMODE_LSEEK;
 	if (mc.flags & IORING_MOCK_CREATE_F_SUPPORT_NOWAIT)
 		file->f_mode |= FMODE_NOWAIT;
 
-	mc.out_fd = fd_prepare_fd(fdf);
+	mc.out_fd = fdf->fd;
 	if (copy_to_user(uarg, &mc, uarg_size))
 		return -EFAULT;
 
diff --git a/io_uring/notif.c b/io_uring/notif.c
index efce8ae12eaa09..6c49aff515df23 100644
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -36,6 +36,7 @@ static void io_notif_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw)
 		}
 
 		nd = nd->next;
+		ctx->nr_notifs--;
 		io_req_task_complete((struct io_tw_req){notif}, tw);
 	} while (nd);
 }
@@ -119,6 +120,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
 
 	if (unlikely(!io_alloc_req(ctx, &notif)))
 		return NULL;
+	ctx->nr_notifs++;
 	notif->ctx = ctx;
 	notif->opcode = IORING_OP_NOP;
 	notif->flags = 0;
diff --git a/io_uring/refs.h b/io_uring/refs.h
index 0fe16b67c308e6..021bf0cd66afb4 100644
--- a/io_uring/refs.h
+++ b/io_uring/refs.h
@@ -56,6 +56,33 @@ static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)
 	}
 }
 
+/*
+ * io-wq may still hold a reference if the issue completed async. If so, the
+ * last put completes the request, so that file drop and CQE visibility are
+ * ordered. The last reference stays for the free path to put, a linked
+ * timeout may still look at the request until then.
+ */
+static inline bool io_req_complete_ready(struct io_kiocb *req)
+{
+	if (!(req->flags & REQ_F_REFCOUNT) || (req->flags & REQ_F_REISSUE))
+		return true;
+	if (atomic_read(&req->refs) == 1)
+		return true;
+	if (!req_ref_put_and_test(req))
+		return false;
+	/* the other put raced us, ours was the last after all */
+	atomic_set(&req->refs, 1);
+	return true;
+}
+
+/* io-wq still holds a reference, the request can't be completed yet */
+static inline bool io_req_shared(struct io_kiocb *req)
+{
+	if (!(req->flags & REQ_F_REFCOUNT) || (req->flags & REQ_F_REISSUE))
+		return false;
+	return atomic_read(&req->refs) > 1;
+}
+
 static inline void io_req_set_refcount(struct io_kiocb *req)
 {
 	__io_req_set_refcount(req, 1);
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 51b46e624dddf1..1efaf29514e8d0 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -174,8 +174,8 @@ static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
 		kvfree(imu);
 }
 
-static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx,
-					       struct io_mapped_ubuf *imu)
+static unsigned long io_imu_unaccount_hpages(struct io_ring_ctx *ctx,
+					     struct io_mapped_ubuf *imu)
 {
 	struct page *seen = NULL;
 	unsigned long acct = 0;
@@ -188,17 +188,14 @@ static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx,
 		struct page *page = imu->bvec[i].bv_page;
 		struct page *hpage;
 
-		if (!PageCompound(page)) {
-			acct++;
+		if (!PageCompound(page))
 			continue;
-		}
 
 		hpage = compound_head(page);
 		if (hpage == seen)
 			continue;
 		seen = hpage;
 
-		/* Unaccount on last reference */
 		if (hpage_acct_unref(ctx, hpage))
 			acct += page_size(hpage) >> PAGE_SHIFT;
 		cond_resched();
@@ -207,18 +204,38 @@ static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx,
 	return acct;
 }
 
+static unsigned long io_imu_unaccount_reg_pages(struct io_ring_ctx *ctx,
+						struct io_mapped_ubuf *imu)
+{
+	unsigned long acct = 0;
+	int i;
+
+	if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user)
+		return 0;
+
+	for (i = 0; i < imu->nr_bvecs; i++) {
+		if (!PageCompound(imu->bvec[i].bv_page))
+			acct++;
+	}
+	return acct;
+}
+
 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
 {
-	unsigned long acct_pages = 0;
+	unsigned long acct_pages;
 
-	/* Always decrement, so it works for cloned buffers too */
-	acct_pages = io_buffer_unaccount_pages(ctx, imu);
+	/* Compound hpages are accounted per-ring in ctx->hpage_acct */
+	acct_pages = io_imu_unaccount_hpages(ctx, imu);
 
 	if (unlikely(refcount_read(&imu->refs) > 1)) {
-		if (!refcount_dec_and_test(&imu->refs))
+		if (!refcount_dec_and_test(&imu->refs)) {
+			if (acct_pages)
+				io_unaccount_mem(ctx->user, ctx->mm_account, acct_pages);
 			return;
+		}
 	}
 
+	acct_pages += io_imu_unaccount_reg_pages(ctx, imu);
 	if (acct_pages)
 		io_unaccount_mem(ctx->user, ctx->mm_account, acct_pages);
 	imu->release(imu->priv);
@@ -1280,6 +1297,7 @@ static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx,
 					struct io_mapped_ubuf *imu)
 {
 	struct page *seen = NULL;
+	unsigned long acct = 0;
 	int i, ret = 0;
 
 	if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user)
@@ -1302,10 +1320,14 @@ static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx,
 		ret = hpage_acct_ref(ctx, hpage, &acct_new);
 		if (ret)
 			break;
+		if (acct_new)
+			acct += page_size(hpage) >> PAGE_SHIFT;
 
 		cond_resched();
 	}
 
+	if (!ret && acct)
+		ret = io_account_mem(ctx->user, ctx->mm_account, acct);
 	if (!ret)
 		return 0;
 
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 432820f862514c..755166e90746e8 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -16,6 +16,7 @@
 
 #include "filetable.h"
 #include "io_uring.h"
+#include "refs.h"
 #include "opdef.h"
 #include "kbuf.h"
 #include "alloc_cache.h"
@@ -1295,7 +1296,7 @@ static u64 io_hybrid_iopoll_delay(struct io_ring_ctx *ctx, struct io_kiocb *req)
 	set_current_state(TASK_INTERRUPTIBLE);
 	hrtimer_sleeper_start_expires(&timer, mode);
 
-	if (timer.task)
+	if (hrtimer_sleeper_task_get(&timer))
 		io_schedule();
 
 	hrtimer_cancel(&timer.timer);
@@ -1380,6 +1381,9 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
 		/* order with io_complete_rw_iopoll(), e.g. ->result updates */
 		if (!smp_load_acquire(&req->iopoll_completed))
 			continue;
+		/* io-wq still has a reference, reap it on the next pass */
+		if (io_req_shared(req))
+			continue;
 		list_del(&req->iopoll_node);
 		wq_list_add_tail(&req->comp_list, &ctx->submit_state.compl_reqs);
 		nr_events++;
diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index c4dd26cf342dae..9c239c0a171550 100644
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -729,6 +729,24 @@ static bool io_match_task(struct io_kiocb *head, struct io_uring_task *tctx,
 	return false;
 }
 
+__cold unsigned int io_timeouts_armed(struct io_ring_ctx *ctx,
+				      struct io_uring_task *tctx)
+{
+	struct io_timeout *timeout;
+	unsigned int nr = 0;
+
+	guard(mutex)(&ctx->uring_lock);
+	raw_spin_lock_irq(&ctx->timeout_lock);
+	list_for_each_entry(timeout, &ctx->timeout_list, list) {
+		struct io_kiocb *req = cmd_to_io_kiocb(timeout);
+
+		if (req->tctx == tctx)
+			nr += io_linked_nr(req);
+	}
+	raw_spin_unlock_irq(&ctx->timeout_lock);
+	return nr;
+}
+
 /* Returns true if we found and killed one or more timeouts */
 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
 			     bool cancel_all)
diff --git a/io_uring/timeout.h b/io_uring/timeout.h
index 1620f94dd45a81..b6cd608fb66702 100644
--- a/io_uring/timeout.h
+++ b/io_uring/timeout.h
@@ -11,6 +11,8 @@ struct io_timeout_data {
 __cold void io_flush_timeouts(struct io_ring_ctx *ctx);
 struct io_cancel_data;
 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd);
+__cold unsigned int io_timeouts_armed(struct io_ring_ctx *ctx,
+				      struct io_uring_task *tctx);
 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
 			     bool cancel_all);
 void io_queue_linked_timeout(struct io_kiocb *req);
diff --git a/io_uring/tw.c b/io_uring/tw.c
index f573bcc3af6a63..f9affd68dd263d 100644
--- a/io_uring/tw.c
+++ b/io_uring/tw.c
@@ -210,6 +210,9 @@ void io_req_normal_work_add(struct io_kiocb *req)
 	struct io_uring_task *tctx = req->tctx;
 	struct io_ring_ctx *ctx = req->ctx;
 
+	/* SQPOLL can retire the request on push, see io_ring_exit_work() */
+	guard(rcu)();
+
 	/* tw run already pending, nothing else to do */
 	if (!mpscq_push(&tctx->task_list, &req->io_task_work.node))
 		return;
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 726a659f38c388..73b606899833fc 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -49,7 +49,7 @@ void io_uring_cmd_cleanup(struct io_kiocb *req)
 }
 
 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
-				   struct io_uring_task *tctx, bool cancel_all)
+				   struct io_uring_task *tctx)
 {
 	struct hlist_node *tmp;
 	struct io_kiocb *req;
@@ -63,7 +63,7 @@ bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
 				struct io_uring_cmd);
 		struct file *file = req->file;
 
-		if (!cancel_all && req->tctx != tctx)
+		if (tctx && req->tctx != tctx)
 			continue;
 
 		if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
@@ -138,19 +138,11 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 }
 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
 
-static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
-					  u64 extra1, u64 extra2)
-{
-	req->big_cqe.extra1 = extra1;
-	req->big_cqe.extra2 = extra2;
-}
-
 /*
  * Called by consumers of io_uring_cmd, if they originally returned
  * -EIOCBQUEUED upon receiving the command.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
-		       unsigned issue_flags, bool is_cqe32)
+void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, unsigned issue_flags)
 {
 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
 
@@ -159,15 +151,6 @@ void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
 
 	io_uring_cmd_del_cancelable(ioucmd, issue_flags);
 
-	if (ret < 0)
-		req_set_fail(req);
-
-	io_req_set_res(req, ret, 0);
-	if (is_cqe32) {
-		if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
-			req->cqe.flags |= IORING_CQE_F_32;
-		io_req_set_cqe32_extra(req, res2, 0);
-	}
 	io_req_uring_cleanup(req, issue_flags);
 	if (req->flags & REQ_F_IOPOLL) {
 		/* order with io_do_iopoll() checking ->iopoll_completed */
diff --git a/io_uring/uring_cmd.h b/io_uring/uring_cmd.h
index 041aef8a8aa3f6..a5cb3f2ee1c516 100644
--- a/io_uring/uring_cmd.h
+++ b/io_uring/uring_cmd.h
@@ -14,7 +14,7 @@ void io_uring_cmd_sqe_copy(struct io_kiocb *req);
 void io_uring_cmd_cleanup(struct io_kiocb *req);
 
 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
-				   struct io_uring_task *tctx, bool cancel_all);
+				   struct io_uring_task *tctx);
 
 bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd,
 				   unsigned int issue_flags,
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 86d580d4410d9c..dcff0b995d34f1 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -374,6 +374,7 @@ struct io_zcrx_args {
 	struct io_kiocb		*req;
 	struct io_zcrx_ifq	*ifq;
 	unsigned		nr_skbs;
+	bool			stopped_early;
 };
 
 static const struct memory_provider_ops io_uring_pp_zc_ops;
@@ -1899,6 +1900,9 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
 	}
 
 out:
+	/* Bytes left in len mean an error stopped the walk early. */
+	if (len)
+		args->stopped_early = true;
 	if (offset == start_off)
 		return ret;
 	desc->count -= (offset - start_off);
@@ -1935,8 +1939,9 @@ static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
 			ret = -ENOTCONN;
 		else
 			ret = -EAGAIN;
-	} else if (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) &&
-		   (issue_flags & IO_URING_F_MULTISHOT)) {
+	} else if ((issue_flags & IO_URING_F_MULTISHOT) &&
+		   (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) ||
+		    args.stopped_early)) {
 		ret = IOU_REQUEUE;
 	} else if (sock_flag(sk, SOCK_DONE)) {
 		/* Make it to retry until it finally gets 0. */
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index d1dd36a651b0d0..322c3dd98c4477 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -528,6 +528,13 @@ static void mqueue_evict_inode(struct inode *inode)
 		list_add_tail(&msg->m_list, &tmp_msg);
 	kfree(info->node_cache);
 	spin_unlock(&info->lock);
+	/*
+	 * A shared file table can let the notification owner exit without
+	 * running ->flush(). No users of the inode remain during eviction, so
+	 * tear down any stale notification after dropping info->lock because
+	 * netlink_sendskb() may release the final socket reference.
+	 */
+	remove_notification(info);
 
 	list_for_each_entry_safe(msg, nmsg, &tmp_msg, m_list) {
 		list_del(&msg->m_list);
@@ -607,7 +614,7 @@ static int mqueue_create_attr(struct dentry *dentry, umode_t mode, void *arg)
 	return error;
 }
 
-static int mqueue_create(struct mnt_idmap *idmap, struct inode *dir,
+static int mqueue_create(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, umode_t mode)
 {
 	return mqueue_create_attr(dentry, mode, NULL);
diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec
index 15632358bcf711..a97ed9605602fb 100644
--- a/kernel/Kconfig.kexec
+++ b/kernel/Kconfig.kexec
@@ -167,7 +167,7 @@ config CRASH_MAX_MEMORY_RANGES
 	  memory regions that the elfcorehdr buffer/segment can accommodate.
 	  These regions are obtained via walk_system_ram_res(); eg. the
 	  'System RAM' entries in /proc/iomem.
-	  This value is combined with NR_CPUS_DEFAULT and multiplied by
+	  This value is combined with NR_CPUS and multiplied by
 	  sizeof(Elf64_Phdr) to determine the final elfcorehdr memory buffer/
 	  segment size.
 	  The value 8192, for example, covers a (sparsely populated) 1TiB system
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index f294dad43bd713..edc067a0c422ae 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -132,10 +132,9 @@ config PREEMPTION
 
 config PREEMPT_DYNAMIC
 	bool "Preemption behaviour defined on boot"
-	depends on HAVE_PREEMPT_DYNAMIC
-	select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY
+	depends on ARCH_HAS_PREEMPT_LAZY
 	select PREEMPT_BUILD
-	default y if HAVE_PREEMPT_DYNAMIC_CALL
+	default y
 	help
 	  This option allows to define the preemption model on the kernel
 	  command line parameter and thus override the default preemption
@@ -145,9 +144,7 @@ config PREEMPT_DYNAMIC
 	  provide a pre-built kernel binary to reduce the number of kernel
 	  flavors they offer while still offering different usecases.
 
-	  The runtime overhead is negligible with HAVE_STATIC_CALL_INLINE enabled
-	  but if runtime patching is not available for the specific architecture
-	  then the potential overhead should be considered.
+	  The runtime overhead is negligible.
 
 	  Interesting if you want the same pre-built kernel should be used for
 	  both Server and Desktop workloads.
@@ -197,3 +194,7 @@ config SCHED_CLASS_EXT
 	  For more information:
 	    Documentation/scheduler/sched-ext.rst
 	    https://github.com/sched-ext/scx
+
+config PREFERRED_CPU
+	bool
+	depends on SMP && PARAVIRT
diff --git a/kernel/Makefile b/kernel/Makefile
index 1e1a31673577d6..2944487af0364e 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -7,7 +7,7 @@ obj-y     = fork.o exec_domain.o exec_state.o panic.o \
 	    cpu.o exit.o softirq.o resource.o \
 	    sysctl.o capability.o ptrace.o user.o \
 	    signal.o sys.o umh.o workqueue.o pid.o task_work.o \
-	    extable.o params.o \
+	    extable.o params.o hazptr.o \
 	    kthread.o sys_ni.o nsproxy.o nstree.o nscommon.o \
 	    notifier.o ksysfs.o cred.o reboot.o \
 	    async.o range.o smpboot.o ucount.o regset.o ksyms_common.o
@@ -141,6 +141,7 @@ obj-$(CONFIG_WATCH_QUEUE) += watch_queue.o
 
 obj-$(CONFIG_RESOURCE_KUNIT_TEST) += resource_kunit.o
 obj-$(CONFIG_SYSCTL_KUNIT_TEST) += sysctl-test.o
+obj-$(CONFIG_USER_NS_MAP_KUNIT_TEST) += tests/user_ns_map_kunit.o
 
 CFLAGS_kstack_erase.o += $(DISABLE_KSTACK_ERASE)
 CFLAGS_kstack_erase.o += $(call cc-option,-mgeneral-regs-only)
@@ -153,9 +154,13 @@ obj-$(CONFIG_SCF_TORTURE_TEST) += scftorture.o
 
 $(obj)/configs.o: $(obj)/config_data.gz
 
+# .config is small, so it is compressed with gzip rather than the image compressor.
+quiet_cmd_gzip_config = GZIP    $@
+      cmd_gzip_config = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
+
 targets += config_data config_data.gz
 $(obj)/config_data.gz: $(obj)/config_data FORCE
-	$(call if_changed,gzip)
+	$(call if_changed,gzip_config)
 
 filechk_cat = cat $<
 
diff --git a/kernel/audit.c b/kernel/audit.c
index 9412af9144bc9a..631d6d4a23cfde 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -871,7 +871,7 @@ static void kauditd_send_multicast_skb(struct sk_buff *skb)
 	if (!copy)
 		return;
 	nlh = nlmsg_hdr(copy);
-	nlh->nlmsg_len = skb->len;
+	nlh->nlmsg_len = copy->len;
 
 	nlmsg_multicast(sock, copy, 0, AUDIT_NLGRP_READLOG, GFP_KERNEL);
 }
diff --git a/kernel/audit.h b/kernel/audit.h
index 92d5e723d570b6..7640d2c0fba450 100644
--- a/kernel/audit.h
+++ b/kernel/audit.h
@@ -53,15 +53,18 @@ struct audit_entry {
 	struct audit_krule	rule;
 };
 
+struct audit_file_caps {
+	kernel_cap_t		permitted;
+	kernel_cap_t		inheritable;
+	unsigned int		fE;		/* effective bit of file cap */
+	kuid_t			rootid;
+};
+
 struct audit_cap_data {
 	kernel_cap_t		permitted;
 	kernel_cap_t		inheritable;
-	union {
-		unsigned int	fE;		/* effective bit of file cap */
-		kernel_cap_t	effective;	/* effective set of process */
-	};
+	kernel_cap_t		effective;
 	kernel_cap_t		ambient;
-	kuid_t			rootid;
 };
 
 /* When fs/namei.c:getname() is called, we store the pointer in name and bump
@@ -83,7 +86,7 @@ struct audit_names {
 	kgid_t			gid;
 	dev_t			rdev;
 	struct lsm_prop		oprop;
-	struct audit_cap_data	fcap;
+	struct audit_file_caps	fcap;
 	unsigned int		fcap_ver;
 	unsigned char		type;		/* record type */
 	/*
@@ -272,6 +275,9 @@ extern void audit_put_tty(struct tty_struct *tty);
 /* audit watch/mark/tree functions */
 extern unsigned int audit_serial(void);
 #ifdef CONFIG_AUDITSYSCALL
+void audit_rule_account(const struct audit_krule *rule);
+void audit_rule_unaccount(const struct audit_krule *rule);
+
 extern int auditsc_get_stamp(struct audit_context *ctx,
 			     struct audit_stamp *stamp);
 
@@ -315,6 +321,8 @@ extern void audit_filter_inodes(struct task_struct *tsk,
 				struct audit_context *ctx);
 extern struct list_head *audit_killed_trees(void);
 #else /* CONFIG_AUDITSYSCALL */
+#define audit_rule_account(...) do { } while (0)
+#define audit_rule_unaccount(...) do { } while (0)
 #define auditsc_get_stamp(c, s) 0
 #define audit_put_watch(w) do { } while (0)
 #define audit_get_watch(w) do { } while (0)
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 1ed19b7759129a..e83b9fef048ff3 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -545,22 +545,39 @@ static void kill_rules(struct audit_context *context, struct audit_tree *tree)
 {
 	struct audit_krule *rule, *next;
 	struct audit_entry *entry;
+	bool need_sync = false;
 
 	list_for_each_entry_safe(rule, next, &tree->rules, rlist) {
 		entry = container_of(rule, struct audit_entry, rule);
 
-		list_del_init(&rule->rlist);
 		if (rule->tree) {
 			/* not a half-baked one */
 			audit_tree_log_remove_rule(context, rule);
-			if (entry->rule.exe)
-				audit_remove_mark(entry->rule.exe);
 			rule->tree = NULL;
 			list_del_rcu(&entry->list);
 			list_del(&entry->rule.list);
-			call_rcu(&entry->rcu, audit_free_rule_rcu);
+			if (entry->rule.exe)
+				need_sync = true;
+			audit_rule_unaccount(rule);
+		} else {
+			list_del_init(&rule->rlist);
 		}
 	}
+
+	if (list_empty(&tree->rules))
+		return;
+
+	if (need_sync)
+		synchronize_rcu();
+
+	list_for_each_entry_safe(rule, next, &tree->rules, rlist) {
+		entry = container_of(rule, struct audit_entry, rule);
+
+		list_del_init(&rule->rlist);
+		if (entry->rule.exe)
+			audit_remove_mark(entry->rule.exe);
+		call_rcu(&entry->rcu, audit_free_rule_rcu);
+	}
 }
 
 /*
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c
index 4ac8a91e9ba8ff..28fab822ca0c66 100644
--- a/kernel/audit_watch.c
+++ b/kernel/audit_watch.c
@@ -284,6 +284,7 @@ static void audit_update_watch(struct audit_parent *parent,
 			nentry = audit_dupe_rule(&oentry->rule, ctx);
 			if (IS_ERR(nentry)) {
 				list_del(&oentry->rule.list);
+				audit_rule_unaccount(r);
 				audit_panic("error updating watch, removing");
 			} else {
 				int h = audit_hash_ino(ino);
@@ -336,6 +337,7 @@ static void audit_remove_parent_watches(struct audit_parent *parent)
 			list_del(&r->rlist);
 			list_del(&r->list);
 			list_del_rcu(&e->list);
+			audit_rule_unaccount(r);
 			call_rcu(&e->rcu, audit_free_rule_rcu);
 		}
 		audit_remove_watch(w);
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c
index f526456252148f..cf91ba6f3617b1 100644
--- a/kernel/auditfilter.c
+++ b/kernel/auditfilter.c
@@ -196,7 +196,7 @@ int audit_match_class(int class, unsigned int syscall)
 }
 
 #ifdef CONFIG_AUDITSYSCALL
-static inline int audit_match_class_bits(int class, u32 *mask)
+static inline int audit_match_class_bits(int class, const u32 *mask)
 {
 	int i;
 
@@ -208,30 +208,62 @@ static inline int audit_match_class_bits(int class, u32 *mask)
 	return 1;
 }
 
-static int audit_match_signal(struct audit_entry *entry)
+static int audit_match_signal(const struct audit_krule *rule)
 {
-	struct audit_field *arch = entry->rule.arch_f;
+	struct audit_field *arch = rule->arch_f;
 
 	if (!arch) {
 		/* When arch is unspecified, we must check both masks on biarch
 		 * as syscall number alone is ambiguous. */
 		return (audit_match_class_bits(AUDIT_CLASS_SIGNAL,
-					       entry->rule.mask) &&
+					       rule->mask) &&
 			audit_match_class_bits(AUDIT_CLASS_SIGNAL_32,
-					       entry->rule.mask));
+					       rule->mask));
 	}
 
 	switch (audit_classify_arch(arch->val)) {
 	case 0: /* native */
 		return (audit_match_class_bits(AUDIT_CLASS_SIGNAL,
-					       entry->rule.mask));
+					       rule->mask));
 	case 1: /* 32bit on biarch */
 		return (audit_match_class_bits(AUDIT_CLASS_SIGNAL_32,
-					       entry->rule.mask));
+					       rule->mask));
 	default:
 		return 1;
 	}
 }
+
+static bool audit_rule_counts_syscalls(const struct audit_krule *rule)
+{
+	switch (rule->listnr) {
+	case AUDIT_FILTER_USER:
+	case AUDIT_FILTER_EXCLUDE:
+	case AUDIT_FILTER_FS:
+		return false;
+	default:
+		return true;
+	}
+}
+
+void audit_rule_account(const struct audit_krule *rule)
+{
+	lockdep_assert_held(&audit_filter_mutex);
+
+	if (audit_rule_counts_syscalls(rule))
+		audit_n_rules++;
+	if (!audit_match_signal(rule))
+		audit_signals++;
+}
+
+void audit_rule_unaccount(const struct audit_krule *rule)
+{
+	lockdep_assert_held(&audit_filter_mutex);
+
+	if (audit_rule_counts_syscalls(rule))
+		audit_n_rules--;
+	if (!audit_match_signal(rule))
+		audit_signals--;
+}
 #endif
 
 /* Common user-space to kernel rule translation. */
@@ -943,17 +975,6 @@ static inline int audit_add_rule(struct audit_entry *entry)
 	struct audit_tree *tree = entry->rule.tree;
 	struct list_head *list;
 	int err = 0;
-#ifdef CONFIG_AUDITSYSCALL
-	int dont_count = 0;
-
-	/* If any of these, don't count towards total */
-	switch (entry->rule.listnr) {
-	case AUDIT_FILTER_USER:
-	case AUDIT_FILTER_EXCLUDE:
-	case AUDIT_FILTER_FS:
-		dont_count = 1;
-	}
-#endif
 
 	mutex_lock(&audit_filter_mutex);
 	e = audit_find_rule(entry, &list);
@@ -1007,13 +1028,7 @@ static inline int audit_add_rule(struct audit_entry *entry)
 			      &audit_rules_list[entry->rule.listnr]);
 		list_add_tail_rcu(&entry->list, list);
 	}
-#ifdef CONFIG_AUDITSYSCALL
-	if (!dont_count)
-		audit_n_rules++;
-
-	if (!audit_match_signal(entry))
-		audit_signals++;
-#endif
+	audit_rule_account(&entry->rule);
 	mutex_unlock(&audit_filter_mutex);
 
 	return err;
@@ -1025,17 +1040,6 @@ int audit_del_rule(struct audit_entry *entry)
 	struct audit_entry  *e;
 	struct list_head *list;
 	int ret = 0;
-#ifdef CONFIG_AUDITSYSCALL
-	int dont_count = 0;
-
-	/* If any of these, don't count towards total */
-	switch (entry->rule.listnr) {
-	case AUDIT_FILTER_USER:
-	case AUDIT_FILTER_EXCLUDE:
-	case AUDIT_FILTER_FS:
-		dont_count = 1;
-	}
-#endif
 
 	mutex_lock(&audit_filter_mutex);
 	e = audit_find_rule(entry, &list);
@@ -1057,14 +1061,7 @@ int audit_del_rule(struct audit_entry *entry)
 	if (e->rule.exe)
 		audit_remove_mark_rule(&e->rule);
 
-#ifdef CONFIG_AUDITSYSCALL
-	if (!dont_count)
-		audit_n_rules--;
-
-	if (!audit_match_signal(entry))
-		audit_signals--;
-#endif
-
+	audit_rule_unaccount(&e->rule);
 	call_rcu(&e->rcu, audit_free_rule_rcu);
 
 out:
@@ -1427,6 +1424,7 @@ static int update_lsm_rule(struct audit_krule *r)
 			list_del(&r->rlist);
 		list_del_rcu(&entry->list);
 		list_del(&r->list);
+		audit_rule_unaccount(r);
 	} else {
 		if (r->watch || r->tree)
 			list_replace_init(&r->rlist, &nentry->rule.rlist);
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index 2b9ce0b525111f..24d06420b02acb 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -107,7 +107,7 @@ struct audit_aux_data_pids {
 
 struct audit_aux_data_bprm_fcaps {
 	struct audit_aux_data	d;
-	struct audit_cap_data	fcap;
+	struct audit_file_caps	fcap;
 	unsigned int		fcap_ver;
 	struct audit_cap_data	old_pcap;
 	struct audit_cap_data	new_pcap;
@@ -459,7 +459,7 @@ static int audit_field_compare(struct task_struct *tsk,
  *
  * If task_creation is true, this is an explicit indication that we are
  * filtering a task rule at task creation time.  This and tsk == current are
- * the only situations where tsk->cred may be accessed without an rcu read lock.
+ * the only situations where tsk->cred may be accessed.
  */
 static int audit_filter_rules(struct task_struct *tsk,
 			      struct audit_krule *rule,
@@ -476,7 +476,8 @@ static int audit_filter_rules(struct task_struct *tsk,
 	if (ctx && rule->prio <= ctx->prio)
 		return 0;
 
-	cred = rcu_dereference_check(tsk->cred, tsk == current || task_creation);
+	WARN_ON(tsk != current && !task_creation);
+	cred = tsk->cred;
 
 	for (i = 0; i < rule->field_count; i++) {
 		struct audit_field *f = &rule->fields[i];
@@ -2601,6 +2602,8 @@ void __audit_bprm(struct linux_binprm *bprm)
 {
 	struct audit_context *context = audit_context();
 
+	/* clear proctitle in audit context to allow replacement */
+	audit_proctitle_free(context);
 	context->type = AUDIT_EXECVE;
 	context->execve.argc = bprm->argc;
 }
diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig
index eb3de35734f092..98493e32db2ada 100644
--- a/kernel/bpf/Kconfig
+++ b/kernel/bpf/Kconfig
@@ -17,6 +17,10 @@ config HAVE_CBPF_JIT
 config HAVE_EBPF_JIT
 	bool
 
+# KASAN support for JIT compiler
+config HAVE_EBPF_JIT_KASAN
+	bool
+
 # Used by archs to tell that they want the BPF JIT compiler enabled by
 # default for kernels that were compiled with BPF JIT support.
 config ARCH_WANT_DEFAULT_BPF_JIT
@@ -87,6 +91,15 @@ config BPF_UNPRIV_DEFAULT_OFF
 
 	  If you are unsure how to answer this question, answer Y.
 
+config BPF_CRYPTO
+	def_bool y
+	depends on BPF_SYSCALL
+	select CRYPTO_LIB_AES_CBC
+	select CRYPTO_LIB_AES_ECB
+	help
+	  Provide the kfuncs needed for BPF programs to encrypt and decrypt
+	  data. The supported algorithms are AES-CBC and AES-ECB.
+
 source "kernel/bpf/preload/Kconfig"
 
 config BPF_LSM
@@ -101,4 +114,17 @@ config BPF_LSM
 
 	  If you are unsure how to answer this question, answer N.
 
+config BPF_JIT_KASAN
+	bool
+	depends on HAVE_EBPF_JIT_KASAN
+	depends on KASAN_GENERIC
+	depends on KASAN_VMALLOC
+	depends on BPF_JIT
+	default y
+	help
+	  Makes JIT compiler insert generic outline KASAN checks in BPF
+	  programs when they are inserted in the kernel. This feature is
+	  automatically enabled if the needed set of KASAN and BPF
+	  configuration options is enabled.
+
 endmenu # "BPF subsystem"
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 90255d80e5be6d..c1f9b0d3468d3a 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o
 obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o
 obj-$(CONFIG_BPF_SYSCALL) += tcx.o
 endif
+ifeq ($(CONFIG_KEYS),y)
+obj-$(CONFIG_BPF_SYSCALL) += keys.o
+endif
 ifeq ($(CONFIG_PERF_EVENTS),y)
 obj-$(CONFIG_BPF_SYSCALL) += stackmap.o
 endif
@@ -55,9 +58,7 @@ obj-$(CONFIG_BPF_SYSCALL) += cpumask.o
 # semantics within pahole are revisited accordingly.
 obj-${CONFIG_BPF_LSM} += bpf_lsm_proto.o bpf_lsm.o
 endif
-ifneq ($(CONFIG_CRYPTO),)
-obj-$(CONFIG_BPF_SYSCALL) += crypto.o
-endif
+obj-$(CONFIG_BPF_CRYPTO) += crypto.o
 obj-$(CONFIG_BPF_PRELOAD) += preload/
 
 obj-$(CONFIG_BPF_SYSCALL) += relo_core.o
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 7b6847200b4312..ee937fcdb236d9 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -481,7 +481,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
 	struct bpf_map *map = vmf->vma->vm_file->private_data;
 	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
 	struct mem_cgroup *new_memcg, *old_memcg;
-	struct page *page;
+	struct page *page, *new_page = NULL;
+	vm_fault_t fault_ret;
 	long kbase, kaddr;
 	unsigned long flags;
 	int ret;
@@ -489,59 +490,101 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)
 	kbase = bpf_arena_get_kern_vm_start(arena);
 	kaddr = kbase + (u32)(vmf->address);
 
-	if (raw_res_spin_lock_irqsave(&arena->spinlock, flags))
+	page = vmalloc_to_page((void *)kaddr);
+	if (!page && !(arena->map.map_flags & BPF_F_SEGV_ON_FAULT)) {
+		/*
+		 * Preallocate outside the lock so the allocation can reclaim;
+		 * __GFP_RETRY_MAYFAIL keeps the OOM killer out of it.
+		 */
+		bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
+		new_page = alloc_pages_node(map->numa_node,
+					    GFP_KERNEL | __GFP_ZERO |
+					    __GFP_ACCOUNT | __GFP_NOWARN |
+					    __GFP_RETRY_MAYFAIL, 0);
+		bpf_map_memcg_exit(old_memcg, new_memcg);
+	}
+
+	if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) {
 		/*
 		 * A failed lock means a possible deadlock was detected. Don't
 		 * return VM_FAULT_RETRY: this handler never took mmap_lock, but
 		 * the fault path would re-take it on retry and deadlock. Fail.
 		 */
+		if (new_page)
+			free_pages_nolock(new_page, 0);
 		return VM_FAULT_SIGBUS;
+	}
 
 	page = vmalloc_to_page((void *)kaddr);
 	if (page) {
-		if (page == arena->scratch_page)
+		if (page == arena->scratch_page) {
 			/* BPF triggered scratch here; don't lazy-alloc over it */
-			goto out_sigsegv;
+			fault_ret = (arena->map.map_flags & BPF_F_SEGV_ON_FAULT)
+				    ? VM_FAULT_SIGSEGV : VM_FAULT_SIGBUS;
+			goto out_err_locked;
+		}
 		/* already have a page vmap-ed */
 		goto out;
 	}
 
+	if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT) {
+		/*
+		 * User space requested to segfault when page is not allocated
+		 * by bpf prog
+		 */
+		fault_ret = VM_FAULT_SIGSEGV;
+		goto out_err_locked;
+	}
+
 	bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg);
 
-	if (arena->map.map_flags & BPF_F_SEGV_ON_FAULT)
-		/* User space requested to segfault when page is not allocated by bpf prog */
-		goto out_sigsegv_memcg;
+	if (!new_page) {
+		/*
+		 * The probed page was freed meanwhile or preallocation failed;
+		 * try the non-blocking allocator, we cannot sleep here.
+		 */
+		ret = bpf_map_alloc_pages(map, map->numa_node, 1, &new_page);
+		if (ret) {
+			fault_ret = VM_FAULT_SIGBUS;
+			goto out_err_locked_memcg;
+		}
+	}
 
 	ret = range_tree_clear(&arena->rt, vmf->pgoff, 1);
-	if (ret)
-		goto out_sigsegv_memcg;
-
-	struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 };
-	/* Account into memcg of the process that created bpf_arena */
-	ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page);
 	if (ret) {
-		range_tree_set(&arena->rt, vmf->pgoff, 1);
-		goto out_sigsegv_memcg;
+		fault_ret = VM_FAULT_SIGBUS;
+		goto out_err_locked_memcg;
 	}
+	struct apply_range_data data = {
+		.arena = arena, .pages = &new_page, .i = 0
+	};
 
 	ret = apply_to_page_range(&init_mm, kaddr, PAGE_SIZE, apply_range_set_cb, &data);
 	if (ret) {
 		range_tree_set(&arena->rt, vmf->pgoff, 1);
-		free_pages_nolock(page, 0);
-		goto out_sigsegv_memcg;
+		fault_ret = VM_FAULT_SIGBUS;
+		goto out_err_locked_memcg;
 	}
 	flush_vmap_cache(kaddr, PAGE_SIZE);
 	bpf_map_memcg_exit(old_memcg, new_memcg);
+	/* new_page was consumed */
+	page = new_page;
+	new_page = NULL;
 out:
 	page_ref_add(page, 1);
 	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
+	if (new_page)
+		free_pages_nolock(new_page, 0);
 	vmf->page = page;
 	return 0;
-out_sigsegv_memcg:
+
+out_err_locked_memcg:
 	bpf_map_memcg_exit(old_memcg, new_memcg);
-out_sigsegv:
+out_err_locked:
 	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
-	return VM_FAULT_SIGSEGV;
+	if (new_page)
+		free_pages_nolock(new_page, 0);
+	return fault_ret;
 }
 
 static const struct vm_operations_struct arena_vm_ops = {
@@ -620,8 +663,9 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
 	 * clears VM_MAYEXEC. Set VM_DONTEXPAND to avoid potential change
 	 * of user_vm_start. Set VM_DONTCOPY to prevent arena VMA from
 	 * being copied into the child process on fork.
+	 * This is a kernel page so set VM_MIXEDMAP.
 	 */
-	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY);
+	vm_flags_set(vma, VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTCOPY);
 	vma->vm_ops = &arena_vm_ops;
 	return 0;
 }
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 0ce26b538075e1..0fe9afd4a591e3 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -1035,7 +1035,7 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map,
 	}
 
 	mutex_lock(&prog->aux->ext_mutex);
-	is_extended = prog->aux->is_extended;
+	is_extended = prog->aux->freplace_link_cnt > 0;
 	if (!is_extended)
 		prog->aux->prog_array_member_cnt++;
 	mutex_unlock(&prog->aux->ext_mutex);
diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c
index 47282ffeeaf948..0e38b9575328c3 100644
--- a/kernel/bpf/backtrack.c
+++ b/kernel/bpf/backtrack.c
@@ -9,7 +9,8 @@
 
 /* for any branch, call, exit record the history of jmps in the given state */
 int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
-			 int insn_flags, int spi, int frame, u64 linked_regs)
+			 int insn_flags, int spi, int frame, const u16 *linked_regs,
+			 u8 linked_regs_cnt)
 {
 	u32 cnt = cur->jmp_history_cnt;
 	struct bpf_jmp_history_entry *p;
@@ -27,10 +28,13 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state
 		env->cur_hist_ent->flags |= insn_flags;
 		env->cur_hist_ent->spi = spi;
 		env->cur_hist_ent->frame = frame;
-		verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
-				"insn history: insn_idx %d linked_regs: %#llx",
-				env->insn_idx, env->cur_hist_ent->linked_regs);
-		env->cur_hist_ent->linked_regs = linked_regs;
+		verifier_bug_if(env->cur_hist_ent->linked_regs_cnt != 0, env,
+				"insn history: insn_idx %d has %u linked regs",
+				env->insn_idx, env->cur_hist_ent->linked_regs_cnt);
+		if (linked_regs_cnt)
+			memcpy(env->cur_hist_ent->linked_regs, linked_regs,
+			       linked_regs_cnt * sizeof(*linked_regs));
+		env->cur_hist_ent->linked_regs_cnt = linked_regs_cnt;
 		return 0;
 	}
 
@@ -47,7 +51,9 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state
 	p->flags = insn_flags;
 	p->spi = spi;
 	p->frame = frame;
-	p->linked_regs = linked_regs;
+	if (linked_regs_cnt)
+		memcpy(p->linked_regs, linked_regs, linked_regs_cnt * sizeof(*linked_regs));
+	p->linked_regs_cnt = linked_regs_cnt;
 	cur->jmp_history_cnt = cnt;
 	env->cur_hist_ent = p;
 
@@ -123,13 +129,26 @@ static inline void bt_reset(struct backtrack_state *bt)
 	bt->env = env;
 }
 
-static inline u32 bt_empty(struct backtrack_state *bt)
+static inline bool bt_frame_stack_empty(struct backtrack_state *bt, u32 frame)
 {
-	u64 mask = 0;
+	return bitmap_empty(bt->stack_masks[frame], MAX_BPF_STACK_SLOTS);
+}
+
+static inline bool bt_stack_empty(struct backtrack_state *bt)
+{
+	return bt_frame_stack_empty(bt, bt->frame);
+}
+
+static inline bool bt_empty(struct backtrack_state *bt)
+{
+	u32 mask = 0;
 	int i;
 
-	for (i = 0; i <= bt->frame; i++)
-		mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i];
+	for (i = 0; i <= bt->frame; i++) {
+		mask |= bt->reg_masks[i] | bt->stack_arg_masks[i];
+		if (!bt_frame_stack_empty(bt, i))
+			return false;
+	}
 
 	return mask == 0;
 }
@@ -181,7 +200,7 @@ static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
 
 static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
 {
-	bt->stack_masks[frame] &= ~(1ull << slot);
+	__clear_bit(slot, bt->stack_masks[frame]);
 }
 
 static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
@@ -194,14 +213,14 @@ static inline u32 bt_reg_mask(struct backtrack_state *bt)
 	return bt->reg_masks[bt->frame];
 }
 
-static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
+static inline unsigned long *bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
 {
 	return bt->stack_masks[frame];
 }
 
-static inline u64 bt_stack_mask(struct backtrack_state *bt)
+static inline unsigned long *bt_stack_mask(struct backtrack_state *bt)
 {
-	return bt->stack_masks[bt->frame];
+	return bt_frame_stack_mask(bt, bt->frame);
 }
 
 static inline u8 bt_stack_arg_mask(struct backtrack_state *bt)
@@ -233,17 +252,16 @@ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
 			break;
 	}
 }
-/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
-void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
+
+/* format stack slots bitmask, e.g., "-8,-24,-40" for slots 0, 2 and 4 */
+void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask)
 {
-	DECLARE_BITMAP(mask, 64);
 	bool first = true;
 	int i, n;
 
 	buf[0] = '\0';
 
-	bitmap_from_u64(mask, stack_mask);
-	for_each_set_bit(i, mask, 64) {
+	for_each_set_bit(i, stack_mask, MAX_BPF_STACK_SLOTS) {
 		n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
 		first = false;
 		buf += n;
@@ -406,15 +424,18 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 		if (class == BPF_STX)
 			bt_set_reg(bt, sreg);
 	} else if (class == BPF_JMP || class == BPF_JMP32) {
-		if (bpf_pseudo_call(insn)) {
-			int subprog_insn_idx, subprog;
+		if (bpf_pseudo_call(insn) || bpf_is_callx(insn)) {
+			int subprog_insn_idx, subprog = -1;
 
-			subprog_insn_idx = idx + insn->imm + 1;
-			subprog = bpf_find_subprog(env, subprog_insn_idx);
-			if (subprog < 0)
-				return -EFAULT;
+			if (bpf_pseudo_call(insn)) {
+				subprog_insn_idx = idx + insn->imm + 1;
+				subprog = bpf_find_subprog(env, subprog_insn_idx);
+				if (subprog < 0)
+					return -EFAULT;
+			}
 
-			if (bpf_subprog_is_global(env, subprog)) {
+			/* callx calls static subprogs only */
+			if (subprog >= 0 && bpf_subprog_is_global(env, subprog)) {
 				/* check that jump history doesn't have any
 				 * extra instructions from subprog; the next
 				 * instruction after call to global subprog
@@ -423,6 +444,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 				 */
 				verifier_bug_if(idx + 1 != subseq_idx, env,
 						"extra insn from subprog");
+				/* global subprog always sets R0 */
+				bt_clear_reg(bt, BPF_REG_0);
+				/* and if it does not set R2, main pass would catch it */
+				bt_clear_reg(bt, BPF_REG_2);
 				/* r1-r5 are invalidated after subprog call,
 				 * so for global func call it shouldn't be set
 				 * anymore
@@ -432,8 +457,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 						     bt_reg_mask(bt));
 					return -EFAULT;
 				}
-				/* global subprog always sets R0 */
-				bt_clear_reg(bt, BPF_REG_0);
 				return 0;
 			} else {
 				/* static subprog call instruction, which
@@ -450,10 +473,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 				/* we are now tracking register spills correctly,
 				 * so any instance of leftover slots is a bug
 				 */
-				if (bt_stack_mask(bt) != 0) {
-					verifier_bug(env,
-						     "static subprog leftover stack slots %llx",
-						     bt_stack_mask(bt));
+				if (!bt_stack_empty(bt)) {
+					bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
+							   bt_stack_mask(bt));
+					verifier_bug(env, "static subprog leftover stack slots %s",
+						     env->tmp_str_buf);
 					return -EFAULT;
 				}
 				/* propagate r1-r5 to the caller */
@@ -486,9 +510,11 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 					     bt_reg_mask(bt));
 				return -EFAULT;
 			}
-			if (bt_stack_mask(bt) != 0) {
-				verifier_bug(env, "callback leftover stack slots %llx",
-					     bt_stack_mask(bt));
+			if (!bt_stack_empty(bt)) {
+				bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
+						   bt_stack_mask(bt));
+				verifier_bug(env, "callback leftover stack slots %s",
+					     env->tmp_str_buf);
 				return -EFAULT;
 			}
 			/* clear r1-r5 in callback subprog's mask */
@@ -506,6 +532,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 				return -ENOTSUPP;
 			/* regular helper call sets R0 */
 			bt_clear_reg(bt, BPF_REG_0);
+			/* kfunc might also set R2 */
+			bt_clear_reg(bt, BPF_REG_2);
 			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
 				/* if backtracking was looking for registers R1-R5
 				 * they should have been found already.
@@ -520,40 +548,51 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 					return -EFAULT;
 			}
 		} else if (opcode == BPF_EXIT) {
-			bool from_subprog_call, r0_precise;
+			bool from_subprog_call, r0_precise, r2_precise;
 
 			/* BPF_EXIT in subprog or callback always returns
 			 * right after the call instruction, so by checking
 			 * whether the instruction at subseq_idx-1 is subprog
 			 * call or not we can distinguish actual exit from
 			 * *subprog* from exit from *callback*. In the former
-			 * case, we need to propagate r0 precision, if
-			 * necessary. In the former we never do that.
+			 * case, we need to propagate the precision of the
+			 * return registers, if necessary. In the latter we
+			 * never do that.
 			 */
 			from_subprog_call = subseq_idx - 1 >= 0 &&
-					    bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]);
+					    (bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) ||
+					     bpf_is_callx(&env->prog->insnsi[subseq_idx - 1]));
 
 			r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0);
+			r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2);
 
 			/* Backtracking to a nested function call, 'idx' is a part of
 			 * the inner frame 'subseq_idx' is a part of the outer frame.
-			 * In case of a regular function call, instructions giving
-			 * precision to registers R1-R5 should have been found already.
+			 * In case of a regular function call, the callee defines the
+			 * return registers R0 and R2, so clear them before checking
+			 * that instructions giving precision to registers R1-R5 have
+			 * been found already. R2 is also an argument register, hence
+			 * it has to be cleared before that check.
 			 * In case of a callback from bpf_loop(), R{1,4} in the calling
-			 * frame would be set as precise and that is correct.
+			 * frame would be set as precise and that is correct, and R2
+			 * might be a precise helper argument as well, so leave it set.
 			 */
+			bt_clear_reg(bt, BPF_REG_0);
+			if (from_subprog_call)
+				bt_clear_reg(bt, BPF_REG_2);
 			if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) {
 				verifier_bug(env, "backtracking exit unexpected regs %x",
 					     bt_reg_mask(bt));
 				return -EFAULT;
 			}
 
-			bt_clear_reg(bt, BPF_REG_0);
 			if (bt_subprog_enter(bt))
 				return -EFAULT;
 
 			if (r0_precise)
 				bt_set_reg(bt, BPF_REG_0);
+			if (r2_precise)
+				bt_set_reg(bt, BPF_REG_2);
 			/* r6-r9 and stack slots will stay set in caller frame
 			 * bitmasks until we return back from callee(s)
 			 */
@@ -693,10 +732,12 @@ void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,
 						i, j);
 				}
 			}
-			for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
-				if (!bpf_is_spilled_reg(&func->stack[j]))
+			for (j = 0; j < bpf_stack_nr_slots(func); j++) {
+				struct bpf_stack_state *ss = bpf_stack_slot(func, j);
+
+				if (!bpf_is_spilled_reg(ss))
 					continue;
-				reg = &func->stack[j].spilled_ptr;
+				reg = &ss->spilled_ptr;
 				if (reg->type != SCALAR_VALUE || reg->precise)
 					continue;
 				reg->precise = true;
@@ -808,6 +849,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env,
 	int subseq_idx = -1;
 	struct bpf_func_state *func;
 	bool tmp, skip_first = true;
+	struct bpf_stack_state *ss;
 	struct bpf_reg_state *reg;
 	int i, fr, err;
 
@@ -854,7 +896,7 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env,
 			if (st->curframe == 0 &&
 			    st->frame[0]->subprogno > 0 &&
 			    st->frame[0]->callsite == BPF_MAIN_FUNC &&
-			    bt_stack_mask(bt) == 0 &&
+			    bt_stack_empty(bt) &&
 			    (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
 				bitmap_from_u64(mask, bt_reg_mask(bt));
 				for_each_set_bit(i, mask, 32) {
@@ -868,8 +910,9 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env,
 				return 0;
 			}
 
-			verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",
-				     st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
+			bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
+			verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %s",
+				     st->frame[0]->subprogno, bt_reg_mask(bt), env->tmp_str_buf);
 			return -EFAULT;
 		}
 
@@ -930,18 +973,18 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env,
 				}
 			}
 
-			bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
-			for_each_set_bit(i, mask, 64) {
-				if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE,
+			for_each_set_bit(i, bt_frame_stack_mask(bt, fr), MAX_BPF_STACK_SLOTS) {
+				if (verifier_bug_if(i >= bpf_stack_nr_slots(func),
 						    env, "stack slot %d, total slots %d",
-						    i, func->allocated_stack / BPF_REG_SIZE))
+						    i, bpf_stack_nr_slots(func)))
 					return -EFAULT;
 
-				if (!bpf_is_spilled_scalar_reg(&func->stack[i])) {
+				ss = bpf_stack_slot(func, i);
+				if (!bpf_is_spilled_scalar_reg(ss)) {
 					bt_clear_frame_slot(bt, fr, i);
 					continue;
 				}
-				reg = &func->stack[i].spilled_ptr;
+				reg = &ss->spilled_ptr;
 				if (reg->precise) {
 					bt_clear_frame_slot(bt, fr, i);
 				} else {
diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c
index a2f84afe6f7c65..92bbb71576ae37 100644
--- a/kernel/bpf/bpf_insn_array.c
+++ b/kernel/bpf/bpf_insn_array.c
@@ -199,12 +199,16 @@ int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog)
 		return -EBUSY;
 
 	/*
-	 * Reset all the map indexes to the original values.  This is needed,
-	 * e.g., when a replay of verification with different log level should
-	 * be performed.
+	 * Reset the map to its pre-verification state. The xlated and jitted
+	 * offsets and the jitted target pointers are recomputed by the verifier
+	 * and the JIT for this program, so any values left by a previous owner
+	 * must be cleared here.
 	 */
-	for (i = 0; i < map->max_entries; i++)
+	for (i = 0; i < map->max_entries; i++) {
 		values[i].xlated_off = values[i].orig_off;
+		values[i].jitted_off = 0;
+		insn_array->ips[i] = 0;
+	}
 
 	return 0;
 }
@@ -228,10 +232,11 @@ void bpf_insn_array_release(struct bpf_map *map)
 {
 	struct bpf_insn_array *insn_array = cast_insn_array(map);
 
-	atomic_set(&insn_array->used, 0);
+	/* Paired with atomic_xchg() in bpf_insn_array_init(). */
+	atomic_set_release(&insn_array->used, 0);
 }
 
-void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len)
+void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len)
 {
 	struct bpf_insn_array *insn_array = cast_insn_array(map);
 	int i;
@@ -240,7 +245,7 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len)
 		return;
 
 	for (i = 0; i < map->max_entries; i++) {
-		if (insn_array->values[i].xlated_off <= off)
+		if (insn_array->values[i].xlated_off < first)
 			continue;
 		if (insn_array->values[i].xlated_off == INSN_DELETED)
 			continue;
diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c
index b40eb404adab97..d9191df5de2239 100644
--- a/kernel/bpf/bpf_iter.c
+++ b/kernel/bpf/bpf_iter.c
@@ -643,11 +643,11 @@ int bpf_iter_new_fd(struct bpf_link *link)
 	flags = O_RDONLY | O_CLOEXEC;
 
 	FD_PREPARE(fdf, flags, anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
 	iter_link = container_of(link, struct bpf_iter_link, link);
-	err = prepare_seq_file(fd_prepare_file(fdf), iter_link);
+	err = prepare_seq_file(fdf->file, iter_link);
 	if (err)
 		return err; /* Automatic cleanup handles fput */
 
diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 6fc6a4b672b556..4642da062f0f80 100644
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -240,24 +240,26 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
 	hlist_add_head_rcu(&selem->snode, &local_storage->list);
 }
 
+/* Must be called with the owning local_storage->lock held. */
 static int bpf_selem_unlink_map(struct bpf_local_storage_elem *selem)
 {
 	struct bpf_local_storage *local_storage;
 	struct bpf_local_storage_map *smap;
 	struct bpf_local_storage_map_bucket *b;
-	unsigned long flags;
 	int err;
 
+	lockdep_assert_irqs_disabled();
+
 	local_storage = rcu_dereference_check(selem->local_storage,
 					      bpf_rcu_lock_held());
 	smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
 	b = select_bucket(smap, local_storage);
-	err = raw_res_spin_lock_irqsave(&b->lock, flags);
+	err = raw_res_spin_lock(&b->lock);
 	if (err)
 		return err;
 
 	hlist_del_init_rcu(&selem->map_node);
-	raw_res_spin_unlock_irqrestore(&b->lock, flags);
+	raw_res_spin_unlock(&b->lock);
 
 	return 0;
 }
@@ -552,7 +554,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
 	struct bpf_local_storage *local_storage;
 	struct bpf_local_storage_map_bucket *b;
 	HLIST_HEAD(old_selem_free_list);
-	unsigned long flags, b_flags;
+	unsigned long flags;
 	int err;
 
 	/* BPF_EXIST and BPF_NOEXIST cannot be both set */
@@ -637,7 +639,8 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
 
 	b = select_bucket(smap, local_storage);
 
-	err = raw_res_spin_lock_irqsave(&b->lock, b_flags);
+	/* local_storage->lock is held, so IRQs are already disabled. */
+	err = raw_res_spin_lock(&b->lock);
 	if (err)
 		goto unlock;
 
@@ -655,7 +658,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
 						&old_selem_free_list);
 	}
 
-	raw_res_spin_unlock_irqrestore(&b->lock, b_flags);
+	raw_res_spin_unlock(&b->lock);
 unlock:
 	raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
 free_selem:
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index 82c5988417a0cf..f58dce888ff4a7 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -116,8 +116,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog *prog,
 }
 #endif
 
+BTF_ID_LIST_SINGLE(bpf_lsm_inode_init_security_btf_id, func,
+		   bpf_lsm_inode_init_security)
+
 int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
-			const struct bpf_prog *prog)
+			struct bpf_prog *prog)
 {
 	u32 btf_id = prog->aux->attach_btf_id;
 	const char *func_name = prog->aux->attach_func_name;
@@ -140,6 +143,23 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
 		return -EINVAL;
 	}
 
+	if (btf_id == bpf_lsm_inode_init_security_btf_id[0]) {
+		/*
+		 * inode, dir, qstr, xattrs, xattr_count
+		 *
+		 * The trusted pointer this hands the program is only as
+		 * trustworthy as the context it is loaded from, which
+		 * lsm_verifier_ops keeps read-only.
+		 */
+		static const struct bpf_ctx_arg_aux xattr_count_arg_info = {
+			.offset		= 4 * sizeof(u64),
+			.reg_type	= PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED,
+			.mem_size	= sizeof(int),
+		};
+
+		return bpf_prog_ctx_arg_info_init(prog, &xattr_count_arg_info, 1);
+	}
+
 	return 0;
 }
 
@@ -414,8 +434,8 @@ const struct bpf_prog_ops lsm_prog_ops = {
 };
 
 const struct bpf_verifier_ops lsm_verifier_ops = {
-	.get_func_proto = bpf_lsm_func_proto,
-	.is_valid_access = btf_ctx_access,
+	.get_func_proto		= bpf_lsm_func_proto,
+	.is_valid_access	= bpf_tracing_btf_ctx_access,
 };
 
 /* hooks return 0 or 1 */
diff --git a/kernel/bpf/bpf_lsm_proto.c b/kernel/bpf/bpf_lsm_proto.c
index 44a54fd8045e83..b81179ae2e086b 100644
--- a/kernel/bpf/bpf_lsm_proto.c
+++ b/kernel/bpf/bpf_lsm_proto.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/fs.h>
+#include <linux/xattr.h>
 #include <linux/bpf_lsm.h>
 
 /*
@@ -12,8 +13,22 @@
  * explicitly enforces that BPF LSM programs check for NULL before attempting to
  * dereference it.
  */
-int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot,
-		      unsigned long prot, unsigned long flags)
+noinline int bpf_lsm_mmap_file(struct file *file__nullable, unsigned long reqprot,
+			       unsigned long prot, unsigned long flags)
 {
 	return 0;
 }
+
+/*
+ * Strong definition of the inode_init_security() BPF LSM hook. Both the
+ * qstr and the xattr array are NULL for some callers, so the __nullable
+ * suffix marks it as PTR_MAYBE_NULL. BPF LSM programs have to check before
+ * dereferencing them or handing them to bpf_inode_init_xattr().
+ */
+noinline int bpf_lsm_inode_init_security(struct inode *inode, struct inode *dir,
+					 const struct qstr *qstr__nullable,
+					 struct xattr *xattrs__nullable,
+					 int *xattr_count)
+{
+	return -EOPNOTSUPP;
+}
diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c
index d7c3030bc63b01..1178acd72296ec 100644
--- a/kernel/bpf/bpf_struct_ops.c
+++ b/kernel/bpf/bpf_struct_ops.c
@@ -13,6 +13,7 @@
 #include <linux/btf_ids.h>
 #include <linux/rcupdate_wait.h>
 #include <linux/poll.h>
+#include <linux/bpf-cgroup.h>
 
 struct bpf_struct_ops_value {
 	struct bpf_struct_ops_common_value common;
@@ -57,7 +58,7 @@ struct bpf_struct_ops_map {
 
 struct bpf_struct_ops_link {
 	struct bpf_link link;
-	struct bpf_map __rcu *map;
+	struct bpf_map *map;
 	wait_queue_head_t wait_hup;
 };
 
@@ -1024,9 +1025,18 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map)
 	bpf_map_area_free(st_map);
 }
 
+static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map)
+{
+	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
+
+	bpf_struct_ops_map_del_ksyms(st_map);
+}
+
 static void bpf_struct_ops_map_free(struct bpf_map *map)
 {
 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
+	struct bpf_struct_ops *st_ops = st_map->st_ops_desc->st_ops;
+	bool tasks_rcu = st_ops->free_after_tasks_rcu_gp;
 
 	/* st_ops->owner was acquired during map_alloc to implicitly holds
 	 * the btf's refcnt. The acquire was only done when btf_is_module()
@@ -1037,24 +1047,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map)
 
 	bpf_struct_ops_map_dissoc_progs(st_map);
 
-	bpf_struct_ops_map_del_ksyms(st_map);
-
-	/* The struct_ops's function may switch to another struct_ops.
-	 *
-	 * For example, bpf_tcp_cc_x->init() may switch to
-	 * another tcp_cc_y by calling
-	 * setsockopt(TCP_CONGESTION, "tcp_cc_y").
-	 * During the switch,  bpf_struct_ops_put(tcp_cc_x) is called
-	 * and its refcount may reach 0 which then free its
-	 * trampoline image while tcp_cc_x is still running.
-	 *
-	 * A vanilla rcu gp is to wait for all bpf-tcp-cc prog
-	 * to finish. bpf-tcp-cc prog is non sleepable.
-	 * A rcu_tasks gp is to wait for the last few insn
-	 * in the tramopline image to finish before releasing
-	 * the trampoline image.
-	 */
-	synchronize_rcu_mult(call_rcu, call_rcu_tasks);
+	if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU))
+		synchronize_rcu_tasks();
 
 	__bpf_struct_ops_map_free(map);
 }
@@ -1123,6 +1117,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
 		goto errout;
 	}
 
+	if (st_ops_desc->st_ops->cgroup_atype && !(attr->map_flags & BPF_F_LINK)) {
+		ret = -EOPNOTSUPP;
+		goto errout;
+	}
+
 	vt = st_ops_desc->value_type;
 	if (attr->value_size != vt->size) {
 		ret = -EINVAL;
@@ -1163,6 +1162,8 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
 
 	mutex_init(&st_map->lock);
 	bpf_map_init_from_attr(map, attr);
+	map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp;
+	map->free_after_rcu_gp = true;
 
 	return map;
 
@@ -1195,6 +1196,7 @@ const struct bpf_map_ops bpf_struct_ops_map_ops = {
 	.map_alloc_check = bpf_struct_ops_map_alloc_check,
 	.map_alloc = bpf_struct_ops_map_alloc,
 	.map_free = bpf_struct_ops_map_free,
+	.map_free_pre_rcu = bpf_struct_ops_map_free_pre_rcu,
 	.map_get_next_key = bpf_struct_ops_map_get_next_key,
 	.map_lookup_elem = bpf_struct_ops_map_lookup_elem,
 	.map_delete_elem = bpf_struct_ops_map_delete_elem,
@@ -1281,7 +1283,31 @@ int bpf_struct_ops_for_each_prog(const void *kdata,
 }
 EXPORT_SYMBOL_GPL(bpf_struct_ops_for_each_prog);
 
-static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
+void *bpf_struct_ops_map_kdata(struct bpf_map *map)
+{
+	struct bpf_struct_ops_map *st_map;
+
+	st_map = container_of(map, struct bpf_struct_ops_map, map);
+	return st_map->kvalue.data;
+}
+
+int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map)
+{
+	struct bpf_struct_ops_map *st_map;
+
+	st_map = container_of(map, struct bpf_struct_ops_map, map);
+	return st_map->st_ops_desc->st_ops->cgroup_atype;
+}
+
+void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map)
+{
+	struct bpf_struct_ops_map *st_map;
+
+	st_map = container_of(map, struct bpf_struct_ops_map, map);
+	return st_map->st_ops_desc->st_ops->cfi_stubs;
+}
+
+bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
 {
 	struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
 
@@ -1297,8 +1323,7 @@ static void bpf_struct_ops_map_link_dealloc(struct bpf_link *link)
 	struct bpf_struct_ops_map *st_map;
 
 	st_link = container_of(link, struct bpf_struct_ops_link, link);
-	st_map = (struct bpf_struct_ops_map *)
-		rcu_dereference_protected(st_link->map, true);
+	st_map = (struct bpf_struct_ops_map *)st_link->map;
 	if (st_map) {
 		st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);
 		bpf_map_put(&st_map->map);
@@ -1313,11 +1338,11 @@ static void bpf_struct_ops_map_link_show_fdinfo(const struct bpf_link *link,
 	struct bpf_map *map;
 
 	st_link = container_of(link, struct bpf_struct_ops_link, link);
-	rcu_read_lock();
-	map = rcu_dereference(st_link->map);
+	mutex_lock(&update_mutex);
+	map = st_link->map;
 	if (map)
 		seq_printf(seq, "map_id:\t%d\n", map->id);
-	rcu_read_unlock();
+	mutex_unlock(&update_mutex);
 }
 
 static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link,
@@ -1327,11 +1352,31 @@ static int bpf_struct_ops_map_link_fill_link_info(const struct bpf_link *link,
 	struct bpf_map *map;
 
 	st_link = container_of(link, struct bpf_struct_ops_link, link);
-	rcu_read_lock();
-	map = rcu_dereference(st_link->map);
+	mutex_lock(&update_mutex);
+	map = st_link->map;
 	if (map)
 		info->struct_ops.map_id = map->id;
-	rcu_read_unlock();
+	mutex_unlock(&update_mutex);
+	return 0;
+}
+
+int bpf_struct_ops_link_update_check(struct bpf_map *new_map,
+				     struct bpf_map *old_map,
+				     struct bpf_map *expected_old_map)
+{
+	struct bpf_struct_ops_map *st_map, *old_st_map;
+
+	if (!old_map)
+		return -ENOLINK;
+	if (expected_old_map && old_map != expected_old_map)
+		return -EPERM;
+
+	st_map = container_of(new_map, struct bpf_struct_ops_map, map);
+	old_st_map = container_of(old_map, struct bpf_struct_ops_map, map);
+	/* The new and old struct_ops must be the same type. */
+	if (st_map->st_ops_desc != old_st_map->st_ops_desc)
+		return -EINVAL;
+
 	return 0;
 }
 
@@ -1353,30 +1398,19 @@ static int bpf_struct_ops_map_link_update(struct bpf_link *link, struct bpf_map
 		return -EOPNOTSUPP;
 
 	mutex_lock(&update_mutex);
-
-	old_map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
-	if (!old_map) {
-		err = -ENOLINK;
-		goto err_out;
-	}
-	if (expected_old_map && old_map != expected_old_map) {
-		err = -EPERM;
+	old_map = st_link->map;
+	err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map);
+	if (err)
 		goto err_out;
-	}
 
 	old_st_map = container_of(old_map, struct bpf_struct_ops_map, map);
-	/* The new and old struct_ops must be the same type. */
-	if (st_map->st_ops_desc != old_st_map->st_ops_desc) {
-		err = -EINVAL;
-		goto err_out;
-	}
 
 	err = st_map->st_ops_desc->st_ops->update(st_map->kvalue.data, old_st_map->kvalue.data, link);
 	if (err)
 		goto err_out;
 
 	bpf_map_inc(new_map);
-	rcu_assign_pointer(st_link->map, new_map);
+	WRITE_ONCE(st_link->map, new_map);
 	bpf_map_put(old_map);
 
 err_out:
@@ -1393,7 +1427,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link)
 
 	mutex_lock(&update_mutex);
 
-	map = rcu_dereference_protected(st_link->map, lockdep_is_held(&update_mutex));
+	map = st_link->map;
 	if (!map) {
 		mutex_unlock(&update_mutex);
 		return 0;
@@ -1402,7 +1436,7 @@ static int bpf_struct_ops_map_link_detach(struct bpf_link *link)
 
 	st_map->st_ops_desc->st_ops->unreg(&st_map->kvalue.data, link);
 
-	RCU_INIT_POINTER(st_link->map, NULL);
+	WRITE_ONCE(st_link->map, NULL);
 	/* Pair with bpf_map_get() in bpf_struct_ops_link_create() or
 	 * bpf_map_inc() in bpf_struct_ops_map_link_update().
 	 */
@@ -1422,7 +1456,7 @@ static __poll_t bpf_struct_ops_map_link_poll(struct file *file,
 
 	poll_wait(file, &st_link->wait_hup, pts);
 
-	return rcu_access_pointer(st_link->map) ? 0 : EPOLLHUP;
+	return READ_ONCE(st_link->map) ? 0 : EPOLLHUP;
 }
 
 static const struct bpf_link_ops bpf_struct_ops_map_lops = {
@@ -1440,6 +1474,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr)
 	struct bpf_link_primer link_primer;
 	struct bpf_struct_ops_map *st_map;
 	struct bpf_map *map;
+	int cgroup_atype;
 	int err;
 
 	map = bpf_map_get(attr->link_create.map_fd);
@@ -1453,6 +1488,19 @@ int bpf_struct_ops_link_create(union bpf_attr *attr)
 		goto err_out;
 	}
 
+	cgroup_atype = st_map->st_ops_desc->st_ops->cgroup_atype;
+	if (cgroup_atype) {
+		err = cgroup_bpf_struct_ops_attach(map, attr);
+		bpf_map_put(map);
+		return err;
+	}
+
+	if (memchr_inv(&attr->link_create.cgroup, 0, sizeof(attr->link_create.cgroup)) ||
+	    attr->link_create.target_fd) {
+		err = -EINVAL;
+		goto err_out;
+	}
+
 	link = kzalloc_obj(*link, GFP_USER);
 	if (!link) {
 		err = -ENOMEM;
@@ -1478,7 +1526,7 @@ int bpf_struct_ops_link_create(union bpf_attr *attr)
 		link = NULL;
 		goto err_out;
 	}
-	RCU_INIT_POINTER(link->map, map);
+	link->map = map;
 	mutex_unlock(&update_mutex);
 
 	return bpf_link_settle(&link_primer);
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 9f33e95d57410c..8cc17a1cd25c62 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -20,6 +20,7 @@
 #include <linux/btf.h>
 #include <linux/btf_ids.h>
 #include <linux/bpf.h>
+#include <linux/bpf-cgroup.h>
 #include <linux/bpf_lsm.h>
 #include <linux/skmsg.h>
 #include <linux/perf_event.h>
@@ -345,6 +346,9 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = {
 	[BTF_KIND_DECL_TAG]	= "DECL_TAG",
 	[BTF_KIND_TYPE_TAG]	= "TYPE_TAG",
 	[BTF_KIND_ENUM64]	= "ENUM64",
+	[BTF_KIND_LOC_PARAM]	= "LOC_PARAM",
+	[BTF_KIND_LOC_PROTO]	= "LOC_PROTO",
+	[BTF_KIND_LOCSEC]	= "LOCSEC",
 };
 
 const char *btf_type_str(const struct btf_type *t)
@@ -517,11 +521,27 @@ static bool btf_type_is_decl_tag(const struct btf_type *t)
 	return BTF_INFO_KIND(t->info) == BTF_KIND_DECL_TAG;
 }
 
+static bool btf_type_is_loc_param(const struct btf_type *t)
+{
+	return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PARAM;
+}
+
+static bool btf_type_is_loc_proto(const struct btf_type *t)
+{
+	return BTF_INFO_KIND(t->info) == BTF_KIND_LOC_PROTO;
+}
+
+static bool btf_type_is_locsec(const struct btf_type *t)
+{
+	return BTF_INFO_KIND(t->info) == BTF_KIND_LOCSEC;
+}
+
 static bool btf_type_nosize(const struct btf_type *t)
 {
 	return btf_type_is_void(t) || btf_type_is_fwd(t) ||
 	       btf_type_is_func(t) || btf_type_is_func_proto(t) ||
-	       btf_type_is_decl_tag(t);
+	       btf_type_is_decl_tag(t) || btf_type_is_loc_param(t) ||
+	       btf_type_is_loc_proto(t) || btf_type_is_locsec(t);
 }
 
 static bool btf_type_nosize_or_null(const struct btf_type *t)
@@ -768,7 +788,9 @@ static bool btf_type_is_resolve_source_only(const struct btf_type *t)
 {
 	return btf_type_is_var(t) ||
 	       btf_type_is_decl_tag(t) ||
-	       btf_type_is_datasec(t);
+	       btf_type_is_datasec(t) ||
+	       btf_type_is_loc_proto(t) ||
+	       btf_type_is_locsec(t);
 }
 
 /* What types need to be resolved?
@@ -796,7 +818,9 @@ static bool btf_type_needs_resolve(const struct btf_type *t)
 	       btf_type_is_var(t) ||
 	       btf_type_is_func(t) ||
 	       btf_type_is_decl_tag(t) ||
-	       btf_type_is_datasec(t);
+	       btf_type_is_datasec(t) ||
+	       btf_type_is_loc_proto(t) ||
+	       btf_type_is_locsec(t);
 }
 
 /* t->size can be used */
@@ -3538,6 +3562,22 @@ static int btf_type_tag_walk(const struct btf *btf,
 	return 0;
 }
 
+bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t)
+{
+	if (!btf_type_is_ptr(t))
+		return false;
+
+	for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t);
+	     t = btf_type_by_id(btf, t->type)) {
+		if (!btf_type_is_type_tag(t) || btf_type_kflag(t))
+			continue;
+		if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena"))
+			return true;
+	}
+
+	return false;
+}
+
 static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,
 			 u32 off, int sz, struct btf_field_info *info, u32 field_mask)
 {
@@ -3679,6 +3719,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_
 		{ BPF_TIMER, "bpf_timer", true },
 		{ BPF_WORKQUEUE, "bpf_wq", true },
 		{ BPF_TASK_WORK, "bpf_task_work", true },
+		{ BPF_RCU_HEAD, "bpf_rcu_head", true },
 		{ BPF_LIST_HEAD, "bpf_list_head", false },
 		{ BPF_LIST_NODE, "bpf_list_node", false },
 		{ BPF_RB_ROOT, "bpf_rb_root", false },
@@ -3864,6 +3905,7 @@ static int btf_find_field_one(const struct btf *btf,
 	case BPF_RB_NODE:
 	case BPF_REFCOUNT:
 	case BPF_TASK_WORK:
+	case BPF_RCU_HEAD:
 		ret = btf_find_struct(btf, var_type, off, sz, field_type,
 				      info_cnt ? &info[0] : &tmp);
 		if (ret < 0)
@@ -4159,6 +4201,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
 	rec->wq_off = -EINVAL;
 	rec->refcount_off = -EINVAL;
 	rec->task_work_off = -EINVAL;
+	rec->rcu_head_off = -EINVAL;
 	for (i = 0; i < cnt; i++) {
 		field_type_size = btf_field_type_size(info_arr[i].type);
 		if (info_arr[i].off + field_type_size > value_size) {
@@ -4202,6 +4245,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
 			WARN_ON_ONCE(rec->task_work_off >= 0);
 			rec->task_work_off = rec->fields[i].offset;
 			break;
+		case BPF_RCU_HEAD:
+			WARN_ON_ONCE(rec->rcu_head_off >= 0);
+			rec->rcu_head_off = rec->fields[i].offset;
+			break;
 		case BPF_REFCOUNT:
 			WARN_ON_ONCE(rec->refcount_off >= 0);
 			/* Cache offset for faster lookup at runtime */
@@ -4268,13 +4315,10 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
 {
 	int i;
 
-	/* There are three types that signify ownership of some other type:
-	 *  kptr_ref, bpf_list_head, bpf_rb_root.
-	 * kptr_ref only supports storing kernel types, which can't store
-	 * references to program allocated local types.
-	 *
-	 * Hence we only need to ensure that bpf_{list_head,rb_root} ownership
-	 * does not form cycles.
+	/*
+	 * Check fields which require the complete BTF and initialize runtime
+	 * metadata. Ownership relationships are validated after every record has
+	 * been fixed up.
 	 */
 	if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR)))
 		return 0;
@@ -4305,51 +4349,88 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
 		if (!meta)
 			return -EFAULT;
 		rec->fields[i].graph_root.value_rec = meta->record;
+	}
+	return 0;
+}
 
-		/* We need to set value_rec for all root types, but no need
-		 * to check ownership cycle for a type unless it's also a
-		 * node type.
-		 */
-		if (!(rec->field_mask & BPF_GRAPH_NODE))
+static int btf_owned_type_idx(const struct btf *btf, struct btf_struct_metas *tab,
+			      const struct btf_field *field)
+{
+	struct btf_struct_meta *meta;
+	u32 btf_id;
+
+	if (field->type & BPF_GRAPH_ROOT) {
+		btf_id = field->graph_root.value_btf_id;
+	} else if (field->type == BPF_KPTR_REF || field->type == BPF_KPTR_PERCPU) {
+		if (btf_is_kernel(field->kptr.btf))
+			return -ENOENT;
+		btf_id = field->kptr.btf_id;
+	} else {
+		return -ENOENT;
+	}
+
+	meta = btf_find_struct_meta(btf, btf_id);
+	if (!meta)
+		return field->type & BPF_GRAPH_ROOT ? -EFAULT : -ENOENT;
+	return meta - tab->types;
+}
+
+/*
+ * Each ownership edge adds kernel frames through bpf_obj_free_fields() and
+ * __bpf_obj_drop_impl(). Keep the bound deliberately small because object
+ * destruction can itself run below a BPF call chain. A final pointee without
+ * special fields is not present in the struct metadata table and adds only a
+ * non-recursing drop.
+ */
+#define BTF_MAX_OWNERSHIP_DEPTH 8
+
+static int btf_ownership_depth(const struct btf *btf,
+			       struct btf_struct_metas *tab, u8 *depth,
+			       int idx, int depth_left)
+{
+	const struct btf_record *rec = tab->types[idx].record;
+	int i, ret, max_depth = 0;
+
+	if (!depth_left)
+		return -ELOOP;
+	if (depth[idx])
+		goto done;
+
+	for (i = 0; i < rec->cnt; i++) {
+		ret = btf_owned_type_idx(btf, tab, &rec->fields[i]);
+		if (ret == -ENOENT)
 			continue;
+		if (ret < 0)
+			return ret;
+		ret = btf_ownership_depth(btf, tab, depth, ret, depth_left - 1);
+		if (ret < 0)
+			return ret;
+		max_depth = max(max_depth, ret);
+	}
+	depth[idx] = max_depth + 1;
+done:
+	return depth[idx] > depth_left ? -ELOOP : depth[idx];
+}
 
-		/* We need to ensure ownership acyclicity among all types. The
-		 * proper way to do it would be to topologically sort all BTF
-		 * IDs based on the ownership edges, since there can be multiple
-		 * bpf_{list_head,rb_node} in a type. Instead, we use the
-		 * following resaoning:
-		 *
-		 * - A type can only be owned by another type in user BTF if it
-		 *   has a bpf_{list,rb}_node. Let's call these node types.
-		 * - A type can only _own_ another type in user BTF if it has a
-		 *   bpf_{list_head,rb_root}. Let's call these root types.
-		 *
-		 * We ensure that if a type is both a root and node, its
-		 * element types cannot be root types.
-		 *
-		 * To ensure acyclicity:
-		 *
-		 * When A is an root type but not a node, its ownership
-		 * chain can be:
-		 *	A -> B -> C
-		 * Where:
-		 * - A is an root, e.g. has bpf_rb_root.
-		 * - B is both a root and node, e.g. has bpf_rb_node and
-		 *   bpf_list_head.
-		 * - C is only an root, e.g. has bpf_list_node
-		 *
-		 * When A is both a root and node, some other type already
-		 * owns it in the BTF domain, hence it can not own
-		 * another root type through any of the ownership edges.
-		 *	A -> B
-		 * Where:
-		 * - A is both an root and node.
-		 * - B is only an node.
-		 */
-		if (meta->record->field_mask & BPF_GRAPH_ROOT)
-			return -ELOOP;
+static int btf_check_ownership_depth(const struct btf *btf,
+				     struct btf_struct_metas *tab)
+{
+	u8 *depth;
+	int i, ret = 0;
+
+	depth = kvcalloc(tab->cnt, sizeof(*depth), GFP_KERNEL | __GFP_NOWARN);
+	if (!depth)
+		return -ENOMEM;
+
+	for (i = 0; i < tab->cnt; i++) {
+		ret = btf_ownership_depth(btf, tab, depth, i,
+					  BTF_MAX_OWNERSHIP_DEPTH);
+		if (ret < 0)
+			break;
+		ret = 0;
 	}
-	return 0;
+	kvfree(depth);
+	return ret;
 }
 
 static void __btf_struct_show(const struct btf *btf, const struct btf_type *t,
@@ -4713,6 +4794,336 @@ static const struct btf_kind_operations enum64_ops = {
 	.show = btf_enum64_show,
 };
 
+static s32 btf_loc_param_check_meta(struct btf_verifier_env *env,
+				    const struct btf_type *t,
+				    u32 meta_left)
+{
+	const struct btf_loc_param *p = btf_loc_param(t);
+	u32 size, meta_needed, vlen = btf_vlen(t);
+
+	meta_needed = sizeof(*p) + sizeof(__u32) * vlen;
+	if (meta_left < meta_needed) {
+		btf_verifier_log_basic(env, t,
+				       "meta_left:%u meta_needed:%u",
+				      meta_left, meta_needed);
+		return -EINVAL;
+	}
+
+	if (t->name_off) {
+		btf_verifier_log_type(env, t, "Invalid name");
+		return -EINVAL;
+	}
+	size = t->size;
+	if (!size || size > 16) {
+		btf_verifier_log_type(env, t, "Unexpected size");
+		return -EINVAL;
+	}
+
+	if (btf_type_kflag(t)) {
+		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
+		return -EINVAL;
+	}
+	/* All LOC_PARAMs have vlen of at least 1 */
+	if (vlen < 1)
+		goto invalid_vlen;
+
+	switch (p->flags) {
+	case BTF_LOC_PARAM_CONST:
+	case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_SIGNED:
+	case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR:
+	case BTF_LOC_PARAM_CONST | BTF_LOC_PARAM_ADDR | BTF_LOC_PARAM_SIGNED:
+		if (vlen > 2)
+			goto invalid_vlen;
+		/* Ensure size/vlen are consistent */
+		if (size > sizeof(__u32) && vlen != 2)
+			goto invalid_vlen;
+		if (size > sizeof(__u64))
+			goto invalid_vlen;
+		break;
+	case BTF_LOC_PARAM_REG:
+		/* Reg or reg/reg pair */
+		if (vlen > 2)
+			goto invalid_vlen;
+		break;
+	case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF:
+		/* A reg deref without offset can only have vlen 1 */
+		if (vlen != 1)
+			goto invalid_vlen;
+		break;
+	case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET:
+	case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_OFFSET | BTF_LOC_PARAM_SIGNED:
+	case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET:
+	case BTF_LOC_PARAM_REG | BTF_LOC_PARAM_DEREF | BTF_LOC_PARAM_OFFSET |
+	     BTF_LOC_PARAM_SIGNED:
+		if (vlen < 2 || vlen > 3)
+			goto invalid_vlen;
+		break;
+	default:
+		btf_verifier_log_type(env, t, "Invalid flags");
+		return -EINVAL;
+	}
+
+	btf_verifier_log_type(env, t, NULL);
+
+	return meta_needed;
+
+invalid_vlen:
+	btf_verifier_log_type(env, t, "Invalid vlen");
+	return -EINVAL;
+}
+
+static void btf_loc_param_log(struct btf_verifier_env *env,
+			 const struct btf_type *t)
+{
+	const struct btf_loc_param *p = btf_loc_param(t);
+	u32 i, vlen = btf_vlen(t);
+
+	btf_verifier_log(env, "size=%u vlen=%u flags=0x%x", t->size, vlen, p->flags);
+	for (i = 0; i < vlen; i++)
+		btf_verifier_log(env, ", %u", p->values[i]);
+}
+
+static const struct btf_kind_operations loc_param_ops = {
+	.check_meta = btf_loc_param_check_meta,
+	.resolve = btf_df_resolve,
+	.check_member = btf_df_check_member,
+	.check_kflag_member = btf_df_check_kflag_member,
+	.log_details = btf_loc_param_log,
+	.show = btf_df_show,
+};
+
+static s32 btf_loc_proto_check_meta(struct btf_verifier_env *env,
+				    const struct btf_type *t,
+				    u32 meta_left)
+{
+	u32 meta_needed;
+
+	meta_needed = sizeof(__u32) * btf_type_vlen(t);
+
+	if (meta_left < meta_needed) {
+		btf_verifier_log_basic(env, t,
+				       "meta_left:%u meta_needed:%u",
+				      meta_left, meta_needed);
+		return -EINVAL;
+	}
+
+	if (t->name_off) {
+		btf_verifier_log_type(env, t, "Invalid name");
+		return -EINVAL;
+	}
+
+	if (btf_type_kflag(t)) {
+		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
+		return -EINVAL;
+	}
+
+	btf_verifier_log_type(env, t, NULL);
+
+	return meta_needed;
+}
+
+static void btf_loc_proto_log(struct btf_verifier_env *env,
+			      const struct btf_type *t)
+{
+	const __u32 *params = btf_loc_proto_params(t);
+	u32 i, nr_params = btf_type_vlen(t);
+
+	btf_verifier_log(env, "vlen=%u", nr_params);
+	for (i = 0; i < nr_params; i++, params++)
+		btf_verifier_log(env, ", %u", *params);
+}
+
+static int btf_loc_proto_resolve(struct btf_verifier_env *env,
+				 const struct resolve_vertex *v)
+{
+	const struct btf_type *t = v->t;
+	const __u32 *params = btf_loc_proto_params(t);
+	u32 i, nr_params = btf_type_vlen(t);
+	struct btf *btf = env->btf;
+
+	if (t->type) {
+		btf_verifier_log_type(env, t, "Invalid loc_proto type");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < nr_params; i++) {
+		const struct btf_type *param_type;
+		u32 param_type_id = params[i];
+
+		if (!param_type_id)
+			continue;
+
+		param_type = btf_type_by_id(btf, param_type_id);
+		if (!param_type || !btf_type_is_loc_param(param_type)) {
+			btf_verifier_log_type(env, t,
+					      "Invalid loc_param#%u", i + 1);
+			return -EINVAL;
+		}
+	}
+
+	env_stack_pop_resolved(env, 0, 0);
+	return 0;
+}
+
+static const struct btf_kind_operations loc_proto_ops = {
+	.check_meta = btf_loc_proto_check_meta,
+	.resolve = btf_loc_proto_resolve,
+	.check_member = btf_df_check_member,
+	.check_kflag_member = btf_df_check_kflag_member,
+	.log_details = btf_loc_proto_log,
+	.show = btf_df_show,
+};
+
+__printf(4, 5)
+static void btf_verifier_log_loc(struct btf_verifier_env *env,
+				 const struct btf_type *locsec_type,
+				 const struct btf_loc *loc,
+				 const char *fmt, ...)
+{
+	struct bpf_verifier_log *log = &env->log;
+	va_list args;
+
+	if (!bpf_verifier_log_needed(log))
+		return;
+	if (log->level == BPF_LOG_KERNEL && !fmt)
+		return;
+	if (env->phase != CHECK_META)
+		btf_verifier_log_type(env, locsec_type, NULL);
+
+	__btf_verifier_log(log, "\t func=%u loc_proto=%u offset=%u",
+			   loc->func, loc->loc_proto, loc->offset);
+	if (fmt && *fmt) {
+		__btf_verifier_log(log, " ");
+		va_start(args, fmt);
+		bpf_verifier_vlog(log, fmt, args);
+		va_end(args);
+	}
+
+	__btf_verifier_log(log, "\n");
+}
+
+static s32 btf_locsec_check_meta(struct btf_verifier_env *env,
+				 const struct btf_type *t,
+				 u32 meta_left)
+{
+	u32 i, meta_needed, vlen = btf_type_vlen(t);
+	const struct btf_loc *loc;
+
+	meta_needed = sizeof(struct btf_loc) * vlen;
+
+	if (meta_left < meta_needed) {
+		btf_verifier_log_basic(env, t,
+				       "meta_left:%u meta_needed:%u",
+				       meta_left, meta_needed);
+		return -EINVAL;
+	}
+
+	if (btf_type_kflag(t)) {
+		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
+		return -EINVAL;
+	}
+
+	if (!t->name_off ||
+	    !btf_name_valid_section(env->btf, t->name_off)) {
+		btf_verifier_log_type(env, t, "Invalid name");
+		return -EINVAL;
+	}
+
+	for_each_loc(i, t, loc) {
+		/* A loc func, loc proto cannot be in type void */
+		if (!loc->func || !BTF_TYPE_ID_VALID(loc->func)) {
+			btf_verifier_log_loc(env, t, loc, "Invalid func");
+			return -EINVAL;
+		}
+		if (!loc->loc_proto || !BTF_TYPE_ID_VALID(loc->loc_proto)) {
+			btf_verifier_log_loc(env, t, loc, "Invalid loc_proto");
+			return -EINVAL;
+		}
+		btf_verifier_log_loc(env, t, loc, NULL);
+	}
+
+	return meta_needed;
+}
+
+static void btf_locsec_log(struct btf_verifier_env *env,
+			   const struct btf_type *t)
+{
+	btf_verifier_log(env, "vlen=%u", btf_type_vlen(t));
+}
+
+static int btf_locsec_resolve(struct btf_verifier_env *env,
+			      const struct resolve_vertex *v)
+{
+	const struct btf_type *t = v->t;
+	const struct btf_loc *loc;
+	struct btf *btf = env->btf;
+	u32 i;
+
+	if (t->type) {
+		btf_verifier_log_type(env, t, "Invalid locsec type");
+		return -EINVAL;
+	}
+
+	env->resolve_mode = RESOLVE_TBD;
+	for (i = v->next_member, loc = btf_type_loc_secinfo(t) + i;
+	     i < btf_type_vlen(t); i++, loc++) {
+		const struct btf_type *func_type, *func_proto_type;
+		const struct btf_type *loc_proto_type;
+		u32 func_type_id = loc->func;
+		u32 loc_proto_type_id = loc->loc_proto;
+		u32 proto_vlen;
+
+		func_type = btf_type_by_id(btf, func_type_id);
+		if (!func_type || !btf_type_is_func(func_type)) {
+			btf_verifier_log_type(env, t,
+					      "Invalid func#%u", i + 1);
+			return -EINVAL;
+		}
+		func_proto_type = btf_type_by_id(btf, func_type->type);
+		if (!func_proto_type || !btf_type_is_func_proto(func_proto_type)) {
+			btf_verifier_log_type(env, t,
+					      "Invalid func#%u", i + 1);
+			return -EINVAL;
+		}
+		proto_vlen = btf_vlen(func_proto_type);
+
+		if (!env_type_is_resolved(env, func_type_id)) {
+			env_stack_set_next_member(env, i);
+			return env_stack_push(env, func_type, func_type_id);
+		}
+
+		loc_proto_type = btf_type_by_id(btf, loc_proto_type_id);
+		if (!loc_proto_type || !btf_type_is_loc_proto(loc_proto_type)) {
+			btf_verifier_log_type(env, t,
+					      "Invalid loc_proto#%u", i + 1);
+			return -EINVAL;
+		}
+		if (proto_vlen != btf_vlen(loc_proto_type)) {
+			btf_verifier_log_type(env, t,
+					      "Mismatched vlen for loc_proto#%u", i + 1);
+			return -EINVAL;
+		}
+
+		if (!env_type_is_resolved(env, loc_proto_type_id)) {
+			env_stack_set_next_member(env, i + 1);
+			return env_stack_push(env, loc_proto_type,
+					      loc_proto_type_id);
+		}
+	}
+
+	env_stack_pop_resolved(env, 0, 0);
+	return 0;
+}
+
+static const struct btf_kind_operations locsec_ops = {
+	.check_meta = btf_locsec_check_meta,
+	.resolve = btf_locsec_resolve,
+	.check_member = btf_df_check_member,
+	.check_kflag_member = btf_df_check_kflag_member,
+	.log_details = btf_locsec_log,
+	.show = btf_df_show,
+};
+
 static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
 				     const struct btf_type *t,
 				     u32 meta_left)
@@ -5382,6 +5793,9 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
 	[BTF_KIND_DECL_TAG] = &decl_tag_ops,
 	[BTF_KIND_TYPE_TAG] = &modifier_ops,
 	[BTF_KIND_ENUM64] = &enum64_ops,
+	[BTF_KIND_LOC_PARAM] = &loc_param_ops,
+	[BTF_KIND_LOC_PROTO] = &loc_proto_ops,
+	[BTF_KIND_LOCSEC] = &locsec_ops,
 };
 
 static s32 btf_check_meta(struct btf_verifier_env *env,
@@ -5456,7 +5870,8 @@ static bool btf_resolve_valid(struct btf_verifier_env *env,
 	if (!env_type_is_resolved(env, type_id))
 		return false;
 
-	if (btf_type_is_struct(t) || btf_type_is_datasec(t))
+	if (btf_type_is_struct(t) || btf_type_is_datasec(t) ||
+	    btf_type_is_loc_proto(t) || btf_type_is_locsec(t))
 		return !btf_resolved_type_id(btf, type_id) &&
 		       !btf_resolved_type_size(btf, type_id);
 
@@ -5781,7 +6196,7 @@ static int btf_parse_hdr(struct btf_verifier_env *env)
 	return btf_check_sec_info(env, btf_data_size);
 }
 
-static const char *alloc_obj_fields[] = {
+static const char * const alloc_obj_fields[] = {
 	"bpf_spin_lock",
 	"bpf_list_head",
 	"bpf_list_node",
@@ -6044,6 +6459,10 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr,
 			if (err < 0)
 				goto errout_meta;
 		}
+
+		err = btf_check_ownership_depth(btf, struct_meta_tab);
+		if (err < 0)
+			goto errout_meta;
 	}
 
 	err = bpf_log_attr_finalize(attr_log, &env->log);
@@ -6978,8 +7397,9 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 	}
 
 	/*
-	 * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or
-	 * PTR_TO_ARENA (both nullable and non-nullable cases).
+	 * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL,
+	 * PTR_TO_ARENA (both nullable and non-nullable cases) or fixed-size
+	 * PTR_TO_MEM.
 	 */
 	for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
 		const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
@@ -6989,8 +7409,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 		flag = type_flag(ctx_arg_info->reg_type);
 		if (ctx_arg_info->offset == off &&
 		    (type == PTR_TO_ARENA ||
+		     type == PTR_TO_MEM ||
 		     (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) {
 			info->reg_type = ctx_arg_info->reg_type;
+			info->mem_size = ctx_arg_info->mem_size;
 			return true;
 		}
 	}
@@ -7187,7 +7609,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf,
 		if (btf_type_is_int(t))
 			return WALK_SCALAR;
 
-		if (!btf_type_is_struct(t))
+		if (!btf_type_is_struct(t) || !t->size)
 			goto error;
 
 		off = (off - moff) % t->size;
@@ -7563,6 +7985,9 @@ static u8 __get_arg_fmodel_flags(const struct btf *btf,
 {
 	u8 flags = __get_type_fmodel_flags(t);
 
+	if (btf_func_arg_align(btf, t) > sizeof(u64))
+		flags |= BTF_FMODEL_ALIGN16_ARG;
+
 	if (btf_param_match_suffix(btf, arg, "__arena__nullable"))
 		flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG;
 	else if (btf_param_match_suffix(btf, arg, "__arena"))
@@ -7606,7 +8031,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
 		return -EINVAL;
 	}
 	ret = __get_type_size(btf, func->type, &t);
-	if (ret < 0 || btf_type_is_struct(t)) {
+	if (ret < 0 || ret > 16) {
 		bpf_log(log,
 			"The function %s return type %s is unsupported.\n",
 			tname, btf_type_str(t));
@@ -7699,6 +8124,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log,
 			btf_type_str(t2), fn2);
 		return -EINVAL;
 	}
+	if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) {
+		bpf_log(log,
+			"Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n",
+			fn1, t1->size, fn2, t2->size);
+		return -EINVAL;
+	}
 
 	for (i = 0; i < nargs1; i++) {
 		t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL);
@@ -7943,51 +8374,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env,
 	return 0;
 }
 
-static int btf_scan_type_tags(struct bpf_verifier_env *env,
-			      const struct btf *btf, u32 type_id,
-			      u32 *tags)
+static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags)
 {
-	static const struct btf_type_tag_match func_type_tags[] = {
-		{ "arena", ARG_TAG_ARENA },
-	};
-	struct btf_type_tag_walk_ctx ctx;
-	const struct btf_type *t;
-	int err;
-
 	/* Find the first pointer type in the chain. */
-	t = btf_type_skip_modifiers(btf, type_id, NULL);
+	const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL);
 
-	/*
-	 * We currently reject type tags on non-pointer types,
-	 * which neither LLVM nor GCC support anyway.
-	 */
-	if (!t || !btf_type_is_ptr(t))
-		return 0;
-
-	ctx.t = t;
-	err = btf_type_tag_walk(btf, &ctx, func_type_tags,
-				ARRAY_SIZE(func_type_tags));
-	if (err) {
-		bpf_log(&env->log,
-			"function signature member has multiple type tags\n");
-		return err;
-	}
-	*tags |= ctx.res;
-
-	return 0;
+	if (btf_type_is_arena_ptr(btf, t))
+		*tags |= ARG_TAG_ARENA;
 }
 
 /* Check whether the type is a valid return type. */
 static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf,
-		const struct btf_type *t, int subprog)
+		const struct btf_type *t, int subprog, bool is_global)
 {
-	u32 tags = 0;
-	int err;
-
-	err = btf_scan_type_tags(env, btf, t->type, &tags);
-	if (err)
-		return err;
-
 	t = btf_type_skip_modifiers(btf, t->type, NULL);
 
 	/*
@@ -7995,16 +8394,62 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt
 	 * General arena variables are not allowed, since it makes no sense to return by value
 	 * a variable that's on the heap in the first place.
 	 */
-	if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t))
+	if (subprog && btf_type_is_arena_ptr(btf, t))
 		return 0;
 
 	/* We always accept void or scalars. */
 	if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t))
 		return 0;
 
+	if (btf_type_is_struct(t) && t->size <= 16) {
+		/*
+		 * A global function's caller models the return as an opaque
+		 * scalar pair, so a pointer member would be laundered into a
+		 * scalar and escape provenance and reference tracking. Only
+		 * scalars and arena pointers are allowed: an arena pointer has
+		 * no provenance to lose, since a program may already derive one
+		 * from any scalar with addr_space_cast(), which confines the
+		 * result to the arena. The main program is the exception: it
+		 * returns to the kernel, which has no arena to cast the address
+		 * back into. A local function is verified inline, so its caller
+		 * receives the real register state and any member is fine.
+		 */
+		bool local_func = subprog && !is_global;
+		u32 member_kinds = BTF_MEMBER_SCALAR;
+
+		if (subprog)
+			member_kinds |= BTF_MEMBER_ARENA_PTR;
+
+		if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds))
+			return 0;
+	}
+
 	return -EOPNOTSUPP;
 }
 
+static int btf_check_arg_slots(struct bpf_verifier_log *log, const char *tname,
+			       bool is_global, u32 slot_cnt,
+			       struct bpf_subprog_info *sub)
+{
+	if (slot_cnt <= MAX_BPF_FUNC_REG_ARGS)
+		return 0;
+
+	if (is_global) {
+		bpf_log(log,
+			"global function %s() needs %d > %d argument slots, "
+			"stack args not supported\n",
+			tname, slot_cnt, MAX_BPF_FUNC_REG_ARGS);
+		return -EINVAL;
+	}
+	if (!bpf_jit_supports_stack_args()) {
+		bpf_log(log, "JIT does not support function %s() with %d argument slots\n",
+			tname, slot_cnt);
+		return -EFAULT;
+	}
+	sub->stack_arg_cnt = slot_cnt - MAX_BPF_FUNC_REG_ARGS;
+	return 0;
+}
+
 /* Process BTF of a function to produce high-level expectation of function
  * arguments (like ARG_PTR_TO_CTX, or ARG_PTR_TO_MEM, etc). This information
  * is cached in subprog info for reuse.
@@ -8024,7 +8469,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 	const struct btf_param *args;
 	const struct btf_type *t, *ref_t, *fn_t;
 	int err;
-	u32 i, nargs, btf_id;
+	u32 i, slots_used, nargs, btf_id;
 	const char *tname;
 
 	if (sub->args_cached)
@@ -8068,34 +8513,29 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 	}
 	args = (const struct btf_param *)(t + 1);
 	nargs = btf_type_vlen(t);
-	sub->arg_cnt = nargs;
+	sub->arg_slot_cnt = nargs;
 	if (nargs > MAX_BPF_FUNC_ARGS) {
 		bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n",
 			MAX_BPF_FUNC_ARGS, tname, nargs);
 		return -EFAULT;
 	}
-	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
-		if (!bpf_jit_supports_stack_args()) {
-			bpf_log(log, "JIT does not support function %s() with %d args\n",
-				tname, nargs);
-			return -EFAULT;
-		}
-		sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS;
-	}
-
-	if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) {
-		bpf_log(log, "global function %s has %d > %d args, stack args not supported\n",
-			tname, nargs, MAX_BPF_FUNC_REG_ARGS);
-		return -EINVAL;
-	}
+	err = btf_check_arg_slots(log, tname, is_global, nargs, sub);
+	if (err)
+		return err;
 
-	err = btf_validate_return_type(env, btf, t, subprog);
+	err = btf_validate_return_type(env, btf, t, subprog, is_global);
 	if (err) {
 		if (is_global) {
+			/* Only a subprogram may return arena pointers. */
+			const char *supported = subprog ?
+				"void, scalar, arena pointer, or a struct/union of "
+				"scalars and arena pointers" :
+				"void, scalar, or a scalar-only struct/union";
+
 			bpf_log(log,
-				"Global function %s() return value not void or scalar. "
-				"Only those are supported.\n",
-				tname);
+				"Global function %s() has unsupported return type. "
+				"Only %s up to 16 bytes is supported.\n",
+				tname, supported);
 		}
 		return err;
 	}
@@ -8103,15 +8543,17 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 	/* Convert BTF function arguments into verifier types.
 	 * Only PTR_TO_CTX and SCALAR are supported atm.
 	 */
-	for (i = 0; i < nargs; i++) {
+	for (i = 0, slots_used = 0; i < nargs; i++) {
 		u32 tags = 0;
+
+		if (slots_used >= MAX_BPF_FUNC_ARGS)
+			goto too_many_slots;
+
 		err = btf_scan_decl_tags(env, btf, fn_t, i, is_global, &tags);
 		if (err)
 			return err;
 
-		err = btf_scan_type_tags(env, btf, args[i].type, &tags);
-		if (err)
-			return err;
+		btf_scan_type_tags(btf, args[i].type, &tags);
 
 		t = btf_type_by_id(btf, args[i].type);
 		while (btf_type_is_modifier(t))
@@ -8128,7 +8570,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 			    btf_validate_prog_ctx_type(log, btf, t, i, prog_type,
 						       prog->expected_attach_type))
 				return -EINVAL;
-			sub->args[i].arg_type = ARG_PTR_TO_CTX;
+			sub->args[slots_used++].arg_type = ARG_PTR_TO_CTX;
 			continue;
 		}
 		if (btf_is_dynptr_ptr(btf, t)) {
@@ -8136,7 +8578,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 				bpf_log(log, "arg#%d has invalid combination of tags\n", i);
 				return -EINVAL;
 			}
-			sub->args[i].arg_type = ARG_PTR_TO_DYNPTR;
+			sub->args[slots_used++].arg_type = ARG_PTR_TO_DYNPTR;
 			continue;
 		}
 		if (tags & ARG_TAG_TRUSTED) {
@@ -8151,10 +8593,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 			if (kern_type_id < 0)
 				return kern_type_id;
 
-			sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED;
+			sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_TRUSTED;
 			if (tags & ARG_TAG_NULLABLE)
-				sub->args[i].arg_type |= PTR_MAYBE_NULL;
-			sub->args[i].btf_id = kern_type_id;
+				sub->args[slots_used].arg_type |= PTR_MAYBE_NULL;
+			sub->args[slots_used].btf_id = kern_type_id;
+			slots_used++;
 			continue;
 		}
 		if (tags & ARG_TAG_UNTRUSTED) {
@@ -8168,8 +8611,10 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 
 			ref_t = btf_type_skip_modifiers(btf, t->type, NULL);
 			if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) {
-				sub->args[i].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED;
-				sub->args[i].mem_size = 0;
+				sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY |
+								 PTR_UNTRUSTED;
+				sub->args[slots_used].mem_size = 0;
+				slots_used++;
 				continue;
 			}
 
@@ -8185,8 +8630,9 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 					i, btf_type_str(ref_t), tname);
 				return -EINVAL;
 			}
-			sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED;
-			sub->args[i].btf_id = kern_type_id;
+			sub->args[slots_used].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED;
+			sub->args[slots_used].btf_id = kern_type_id;
+			slots_used++;
 			continue;
 		}
 		if (tags & ARG_TAG_ARENA) {
@@ -8194,7 +8640,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 				bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i);
 				return -EINVAL;
 			}
-			sub->args[i].arg_type = ARG_PTR_TO_ARENA;
+			sub->args[slots_used++].arg_type = ARG_PTR_TO_ARENA;
 			continue;
 		}
 		if (is_global) { /* generic user data pointer */
@@ -8214,10 +8660,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 				return -EINVAL;
 			}
 
-			sub->args[i].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL;
+			sub->args[slots_used].arg_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL;
 			if (tags & ARG_TAG_NONNULL)
-				sub->args[i].arg_type &= ~PTR_MAYBE_NULL;
-			sub->args[i].mem_size = mem_size;
+				sub->args[slots_used].arg_type &= ~PTR_MAYBE_NULL;
+			sub->args[slots_used].mem_size = mem_size;
+			slots_used++;
 			continue;
 		}
 
@@ -8226,8 +8673,32 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 			bpf_log(log, "arg#%d has pointer tag, but is not a pointer type\n", i);
 			return -EINVAL;
 		}
-		if (btf_type_is_int(t) || btf_is_any_enum(t)) {
-			sub->args[i].arg_type = ARG_ANYTHING;
+		if (btf_type_is_int(t) || btf_is_any_enum(t) || btf_type_is_struct(t)) {
+			u32 nslots;
+
+			if (!t->size || t->size > 2 * BPF_REG_SIZE) {
+				if (!is_global)
+					return -EINVAL;
+				bpf_log(log,
+					"Arg#%d type %s in %s() has size %u, only 1 to %d bytes "
+					"can be passed by value\n",
+					i, btf_type_str(t), tname, t->size, 2 * BPF_REG_SIZE);
+				return -EINVAL;
+			}
+			if (btf_type_is_struct(t) &&
+			    !btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) {
+				if (!is_global)
+					return -EINVAL;
+				bpf_log(log, "Arg#%d type %s in %s() is not composed of scalars\n",
+					i, btf_type_str(t), tname);
+				return -EINVAL;
+			}
+
+			nslots = (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE;
+			if (slots_used + nslots > MAX_BPF_FUNC_ARGS)
+				goto too_many_slots;
+			while (nslots--)
+				sub->args[slots_used++].arg_type = ARG_SCALAR;
 			continue;
 		}
 		if (!is_global)
@@ -8237,9 +8708,21 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 		return -EINVAL;
 	}
 
+	err = btf_check_arg_slots(log, tname, is_global, slots_used, sub);
+	if (err)
+		return err;
+	sub->arg_slot_cnt = slots_used;
+
 	sub->args_cached = true;
 
 	return 0;
+
+too_many_slots:
+	if (!is_global)
+		return -EINVAL;
+	bpf_log(log, "Arguments of %s() need more than %d argument slots\n",
+		tname, MAX_BPF_FUNC_ARGS);
+	return -EINVAL;
 }
 
 static void btf_type_show(const struct btf *btf, u32 type_id, void *obj,
@@ -9915,6 +10398,7 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops,
 		   struct bpf_verifier_log *log)
 {
 	struct btf_struct_ops_tab *tab, *new_tab;
+	int cgroup_atype;
 	int i, err;
 
 	tab = btf->struct_ops_tab;
@@ -9926,8 +10410,10 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops,
 		btf->struct_ops_tab = tab;
 	}
 
+	cgroup_atype = st_ops->cgroup_atype;
 	for (i = 0; i < tab->cnt; i++)
-		if (tab->ops[i].st_ops == st_ops)
+		if (tab->ops[i].st_ops == st_ops ||
+		    (cgroup_atype && cgroup_atype == tab->ops[i].st_ops->cgroup_atype))
 			return -EEXIST;
 
 	if (tab->cnt == tab->capacity) {
@@ -9947,6 +10433,31 @@ btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops,
 	if (err)
 		return err;
 
+	if (cgroup_atype) {
+		/*
+		 * Cgroup struct_ops callers hold the RCU read lock around the
+		 * entire trampoline call, including its trailing instructions.
+		 * A regular RCU grace period therefore protects both the kdata
+		 * and the trampoline image, so a tasks RCU grace period is not
+		 * needed.
+		 */
+		if (!cgroup_bpf_is_struct_ops_atype(cgroup_atype) ||
+		    st_ops->reg || st_ops->unreg || st_ops->free_after_tasks_rcu_gp) {
+			bpf_struct_ops_desc_release(&tab->ops[btf->struct_ops_tab->cnt]);
+			return -EINVAL;
+		}
+
+		/*
+		 * There is no need to unregister from the cgroup when btf_free()
+		 * runs. No struct_ops map or cgroup link can be created once its
+		 * BTF is gone.
+		 */
+		cgroup_bpf_struct_ops_register(cgroup_atype,
+					       tab->ops[btf->struct_ops_tab->cnt].type_id,
+					       st_ops->cfi_stubs,
+					       st_ops->free_after_mult_rcu_gp);
+	}
+
 	btf->struct_ops_tab->cnt++;
 
 	return 0;
diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c
index 842c7d1eabcccc..b0bd9ba951dfba 100644
--- a/kernel/bpf/cfg.c
+++ b/kernel/bpf/cfg.c
@@ -9,6 +9,8 @@
 
 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
 
+#define BPF_MAX_GOTOX_EDGES	BPF_COMPLEXITY_LIMIT_INSNS
+
 /* non-recursive DFS pseudo code
  * 1  procedure DFS-iterative(G,v):
  * 2      label v as discovered
@@ -284,15 +286,17 @@ static struct bpf_iarray *jt_from_map(struct bpf_map *map)
 }
 
 /*
- * Find and collect all maps which fit in the subprog. Return the result as one
- * combined jump table in jt->items (allocated with kvcalloc)
+ * Collect the jump table of every subprogram that has one, as the combined
+ * table of all maps whose targets land inside that subprogram. All gotox
+ * instructions of a subprogram share the same table, so this is done in a
+ * single pass over the maps rather than once per gotox.
  */
-static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,
-					  int subprog_start, int subprog_end)
+static int __compute_subprog_jts(struct bpf_verifier_env *env)
 {
-	struct bpf_iarray *jt = NULL;
+	struct bpf_subprog_info *subprog;
+	struct bpf_iarray *jt, *jt_cur;
 	struct bpf_map *map;
-	struct bpf_iarray *jt_cur;
+	u32 old_cnt;
 	int i;
 
 	for (i = 0; i < env->insn_array_map_cnt; i++) {
@@ -303,73 +307,100 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,
 		map = env->insn_array_maps[i];
 
 		jt_cur = jt_from_map(map);
-		if (IS_ERR(jt_cur)) {
-			kvfree(jt);
-			return jt_cur;
+		if (IS_ERR(jt_cur))
+			return PTR_ERR(jt_cur);
+
+		subprog = bpf_find_containing_subprog(env, jt_cur->items[0]);
+		if (verifier_bug_if(!subprog, env, "no subprog contains insn %u of an insn array map",
+				    jt_cur->items[0])) {
+			kvfree(jt_cur);
+			return -EFAULT;
+		}
+		if (jt_cur->items[jt_cur->cnt - 1] >= (subprog + 1)->start) {
+			verbose(env, "jump table of subprog starting at %u spans multiple subprogs, map id=%d\n",
+				subprog->start, map->id);
+			bpf_diag_program_structure(
+				env, subprog->start, "jump table spans subprograms",
+				"Keep every entry of a jump table inside one subprogram.",
+				"The jump table in map id %d for the subprogram that starts at instruction %u reaches past its end at instruction %u.",
+				map->id, subprog->start, (subprog + 1)->start);
+			kvfree(jt_cur);
+			return -EINVAL;
 		}
 
-		/*
-		 * This is enough to check one element. The full table is
-		 * checked to fit inside the subprog later in create_jt()
-		 */
-		if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) {
-			u32 old_cnt = jt ? jt->cnt : 0;
-			jt = bpf_iarray_realloc(jt, old_cnt + jt_cur->cnt);
-			if (!jt) {
-				kvfree(jt_cur);
-				return ERR_PTR(-ENOMEM);
-			}
-			memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2);
+		old_cnt = subprog->jt ? subprog->jt->cnt : 0;
+		jt = bpf_iarray_realloc(subprog->jt, old_cnt + jt_cur->cnt);
+		if (!jt) {
+			subprog->jt = NULL;
+			kvfree(jt_cur);
+			return -ENOMEM;
 		}
+		memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2);
+		subprog->jt = jt;
 
 		kvfree(jt_cur);
 	}
 
-	if (!jt) {
-		verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);
-		bpf_diag_program_structure(
-			env, subprog_start, "missing jump table",
-			"Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.",
-			"No jump table was found for the subprogram that starts at instruction %u.",
-			subprog_start);
-		return ERR_PTR(-EINVAL);
+	for (i = 0; i < env->subprog_cnt; i++) {
+		jt = env->subprog_info[i].jt;
+		if (jt)
+			jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt);
 	}
 
-	jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt);
-	return jt;
+	env->cfg.subprog_jts_ready = true;
+	return 0;
+}
+
+/* Leave nothing behind on failure. */
+static int compute_subprog_jts(struct bpf_verifier_env *env)
+{
+	int err;
+
+	err = __compute_subprog_jts(env);
+	if (err)
+		bpf_free_subprog_jts(env);
+	return err;
+}
+
+void bpf_free_subprog_jts(struct bpf_verifier_env *env)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(env->subprog_info); i++) {
+		kvfree(env->subprog_info[i].jt);
+		env->subprog_info[i].jt = NULL;
+	}
+	env->cfg.subprog_jts_ready = false;
 }
 
 static struct bpf_iarray *
-create_jt(int t, struct bpf_verifier_env *env)
+subprog_jt(int t, struct bpf_verifier_env *env)
 {
 	struct bpf_subprog_info *subprog;
-	int subprog_start, subprog_end;
-	struct bpf_iarray *jt;
-	int i;
+	int subprog_start, err;
+
+	if (!env->cfg.subprog_jts_ready) {
+		err = compute_subprog_jts(env);
+		if (err)
+			return ERR_PTR(err);
+	}
 
 	subprog = bpf_find_containing_subprog(env, t);
+	if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", t))
+		return ERR_PTR(-EFAULT);
 	subprog_start = subprog->start;
-	subprog_end = (subprog + 1)->start;
-	jt = jt_from_subprog(env, subprog_start, subprog_end);
-	if (IS_ERR(jt))
-		return jt;
 
-	/* Check that the every element of the jump table fits within the given subprogram */
-	for (i = 0; i < jt->cnt; i++) {
-		if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) {
-			verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n",
-					t, subprog_start, subprog_end);
-			bpf_diag_program_structure(
-				env, t, "jump table target out of range",
-				"Keep every jump-table target inside the same subprogram.",
-				"The jump table for instruction %d points outside subprogram range [%u,%u).",
-				t, subprog_start, subprog_end);
-			kvfree(jt);
-			return ERR_PTR(-EINVAL);
-		}
+	if (!subprog->jt) {
+		verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);
+		bpf_diag_program_structure(
+			env, subprog_start, "missing jump table",
+			"Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.",
+			"No jump table was found for the subprogram that starts at instruction %u.",
+			subprog_start);
+		return ERR_PTR(-EINVAL);
 	}
 
-	return jt;
+	return subprog->jt;
 }
 
 /* "conditional jump with N edges" */
@@ -381,27 +412,39 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env)
 	struct bpf_iarray *jt;
 	int i, w;
 
-	jt = env->insn_aux_data[t].jt;
-	if (!jt) {
-		jt = create_jt(t, env);
-		if (IS_ERR(jt))
-			return PTR_ERR(jt);
+	/*
+	 * The first visit pushes every target that still needs exploring, and
+	 * those are all done by the time this insn is revisited, so there is
+	 * nothing left to walk the jump table for.
+	 */
+	if (insn_state[t] & BRANCH)
+		return DONE_EXPLORING;
+
+	jt = subprog_jt(t, env);
+	if (IS_ERR(jt))
+		return PTR_ERR(jt);
+
+	insn_state[t] |= BRANCH;
 
-		env->insn_aux_data[t].jt = jt;
+	if (check_add_overflow(env->cfg.gotox_edges, jt->cnt,
+			       &env->cfg.gotox_edges) ||
+	    env->cfg.gotox_edges > BPF_MAX_GOTOX_EDGES) {
+		verbose(env, "number of indirect jump edges in the program exceeds %u\n",
+			BPF_MAX_GOTOX_EDGES);
+		bpf_diag_program_structure(
+			env, t, "too many indirect jump edges",
+			"Reduce the number of indirect jumps, or the number of distinct targets they can reach.",
+			"The program has more than %u indirect jump edges in total, counted over every gotox instruction.",
+			BPF_MAX_GOTOX_EDGES);
+		return -E2BIG;
 	}
 
 	mark_prune_point(env, t);
 	for (i = 0; i < jt->cnt; i++) {
 		w = jt->items[i];
-		if (w < 0 || w >= env->prog->len) {
-			verbose(env, "indirect jump out of range from insn %d to %d\n", t, w);
-			bpf_diag_program_structure(
-				env, t, "indirect jump out of range",
-				"Keep indirect jump targets inside the program.",
-				"Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.",
-				t, w, env->prog->len - 1);
-			return -EINVAL;
-		}
+		if (verifier_bug_if(w < 0 || w >= env->prog->len, env,
+				    "indirect jump from insn %d to %d out of range", t, w))
+			return -EFAULT;
 
 		mark_jmp_point(env, w);
 		mark_jump_target(env, w);
@@ -422,27 +465,73 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env)
 }
 
 /*
- * Instructions that can abnormally return from a subprog (tail_call
- * upon success, ld_{abs,ind} upon load failure) have a hidden exit
- * that the verifier must account for.
+ * Return pointers to functions in the read-only map that ld_imm64 instruction
+ * 't' loads the address of, or of its value, if there are any.
  */
-static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t)
+static struct bpf_func_ptr *insn_func_ptrs(struct bpf_verifier_env *env, int t, u32 *cnt)
 {
-	struct bpf_subprog_info *subprog;
-	struct bpf_iarray *jt;
+	struct bpf_insn *insn = &env->prog->insnsi[t];
 
-	if (env->insn_aux_data[t].jt)
-		return 0;
+	*cnt = 0;
+	if (!env->func_ptr_cnt || !bpf_is_ldimm64(insn))
+		return NULL;
+	if (insn->src_reg != BPF_PSEUDO_MAP_VALUE &&
+	    insn->src_reg != BPF_PSEUDO_MAP_IDX_VALUE &&
+	    insn->src_reg != BPF_PSEUDO_MAP_FD &&
+	    insn->src_reg != BPF_PSEUDO_MAP_IDX)
+		return NULL;
 
-	jt = bpf_iarray_realloc(NULL, 2);
-	if (!jt)
-		return -ENOMEM;
+	return bpf_map_func_ptrs(env, env->used_maps[env->insn_aux_data[t].map_index], cnt);
+}
 
-	subprog = bpf_find_containing_subprog(env, t);
-	jt->items[0] = t + 1;
-	jt->items[1] = subprog->exit_idx;
-	env->insn_aux_data[t].jt = jt;
-	return 0;
+/*
+ * ld_imm64 that loads the address of a map that has pointers to functions
+ * is similar to ld_imm64 with BPF_PSEUDO_FUNC that loads the address of one
+ * function: any of them may be read from the map and called via callx later.
+ * Treat it as a call of all of them.
+ */
+static int visit_func_ptrs_insn(int t, struct bpf_verifier_env *env,
+				struct bpf_func_ptr *ptrs, u32 cnt)
+{
+	int *insn_stack = env->cfg.insn_stack;
+	int *insn_state = env->cfg.insn_state;
+	int ret, w;
+	u32 i;
+
+	ret = push_insn(t, t + 2, FALLTHROUGH, env);
+	if (ret)
+		return ret;
+
+	/*
+	 * Explore one function at a time like push_insn() does, so that
+	 * DISCOVERED function is a caller of 't' and not the one that waits in
+	 * insn_stack[], which merge_callee_effects() relies on.
+	 */
+	for (i = 0; i < cnt; i++) {
+		w = ptrs[i].xlated_off;
+		if (insn_state[w])
+			continue;
+
+		if (env->cfg.cur_stack >= env->prog->len)
+			return -E2BIG;
+
+		insn_stack[env->cfg.cur_stack++] = w;
+		insn_state[w] = DISCOVERED;
+		return KEEP_EXPLORING;
+	}
+
+	/* all functions are explored, their effects are known */
+	mark_prune_point(env, t);
+	for (i = 0; i < cnt; i++) {
+		w = ptrs[i].xlated_off;
+		merge_callee_effects(env, t, w);
+
+		/* the same marks as push_insn() leaves on a branch target */
+		mark_prune_point(env, w);
+		mark_jmp_point(env, w);
+		mark_jump_target(env, w);
+	}
+	return DONE_EXPLORING;
 }
 
 /* Visits the instruction at index t and returns one of the following:
@@ -453,21 +542,20 @@ static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t)
 static int visit_insn(int t, struct bpf_verifier_env *env)
 {
 	struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];
+	struct bpf_func_ptr *ptrs;
 	int ret, off, insn_sz;
+	u32 cnt;
 
 	if (bpf_pseudo_func(insn))
 		return visit_func_call_insn(t, insns, env, true);
 
+	ptrs = insn_func_ptrs(env, t, &cnt);
+	if (ptrs)
+		return visit_func_ptrs_insn(t, env, ptrs, cnt);
+
 	/* All non-branch instructions have a single fall-through edge. */
 	if (BPF_CLASS(insn->code) != BPF_JMP &&
 	    BPF_CLASS(insn->code) != BPF_JMP32) {
-		if (BPF_CLASS(insn->code) == BPF_LD &&
-		    (BPF_MODE(insn->code) == BPF_ABS ||
-		     BPF_MODE(insn->code) == BPF_IND)) {
-			ret = visit_abnormal_return_insn(env, t);
-			if (ret)
-				return ret;
-		}
 		insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
 		return push_insn(t, t + insn_sz, FALLTHROUGH, env);
 	}
@@ -512,11 +600,6 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
 				mark_subprog_might_sleep(env, t);
 			if (bpf_helper_changes_pkt_data(insn->imm))
 				mark_subprog_changes_pkt_data(env, t);
-			if (insn->imm == BPF_FUNC_tail_call) {
-				ret = visit_abnormal_return_insn(env, t);
-				if (ret)
-					return ret;
-			}
 		} else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
 			struct bpf_call_arg_meta meta;
 
@@ -749,7 +832,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env)
 	struct bpf_insn_aux_data *aux = env->insn_aux_data;
 	const u32 insn_cnt = env->prog->len;
 	int stack_sz, dfs_sz, err = 0;
-	u32 *stack, *pre, *low, *dfs;
+	u32 *stack, *pre, *low, *dfs, *dfs_pos;
 	u32 i, j, t, w;
 	u32 next_preorder_num;
 	u32 next_scc_id;
@@ -762,13 +845,16 @@ int bpf_compute_scc(struct bpf_verifier_env *env)
 	 * - 'stack' accumulates vertices in DFS order, see invariant comment below;
 	 * - 'pre[t] == p' => preorder number of vertex 't' is 'p';
 	 * - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n';
-	 * - 'dfs' DFS traversal stack, used to emulate explicit recursion.
+	 * - 'dfs' DFS traversal stack, used to emulate explicit recursion;
+	 * - 'dfs_pos[k] == j' => the frame 'dfs[k]' resumes visiting its
+	 *   successors at index 'j'.
 	 */
 	stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
 	pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
 	low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
 	dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT);
-	if (!stack || !pre || !low || !dfs) {
+	dfs_pos = kvcalloc(insn_cnt, sizeof(*dfs_pos), GFP_KERNEL_ACCOUNT);
+	if (!stack || !pre || !low || !dfs || !dfs_pos) {
 		err = -ENOMEM;
 		goto exit;
 	}
@@ -851,6 +937,7 @@ int bpf_compute_scc(struct bpf_verifier_env *env)
 		stack_sz = 0;
 		dfs_sz = 1;
 		dfs[0] = i;
+		dfs_pos[0] = 0;
 dfs_continue:
 		while (dfs_sz) {
 			w = dfs[dfs_sz - 1];
@@ -860,13 +947,22 @@ int bpf_compute_scc(struct bpf_verifier_env *env)
 				next_preorder_num++;
 				stack[stack_sz++] = w;
 			}
-			/* Visit 'w' successors */
+			/* Visit remaining 'w' successors */
 			succ = bpf_insn_successors(env, w);
-			for (j = 0; j < succ->cnt; ++j) {
+			for (j = dfs_pos[dfs_sz - 1]; j < succ->cnt; ++j) {
 				if (pre[succ->items[j]]) {
 					low[w] = min(low[w], low[succ->items[j]]);
 				} else {
-					dfs[dfs_sz++] = succ->items[j];
+					/*
+					 * Once DFS for succ->items[j] is complete,
+					 * pre[succ->items[j]] is non-zero, hence
+					 * resuming at 'j' allows to follow the
+					 * low[w] = min(...) update branch above.
+					 */
+					dfs_pos[dfs_sz - 1] = j;
+					dfs_pos[dfs_sz] = 0;
+					dfs[dfs_sz] = succ->items[j];
+					dfs_sz++;
 					goto dfs_continue;
 				}
 			}
@@ -916,5 +1012,6 @@ int bpf_compute_scc(struct bpf_verifier_env *env)
 	kvfree(pre);
 	kvfree(low);
 	kvfree(dfs);
+	kvfree(dfs_pos);
 	return err;
 }
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 149672c76c49ce..2bbe77de89f006 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -24,6 +24,33 @@
 DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_CGROUP_BPF_ATTACH_TYPE);
 EXPORT_SYMBOL(cgroup_bpf_enabled_key);
 
+struct cgroup_struct_ops {
+	u32 type_id;
+	void *cfi_stubs;
+	bool mult_rcu;
+};
+
+static struct cgroup_struct_ops cgroup_struct_ops[MAX_CGROUP_BPF_ATTACH_TYPE];
+
+void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_rcu)
+{
+	cgroup_struct_ops[atype].type_id = type_id;
+	cgroup_struct_ops[atype].cfi_stubs = cfi_stubs;
+	cgroup_struct_ops[atype].mult_rcu = mult_rcu;
+}
+
+static enum cgroup_bpf_attach_type find_atype_by_struct_ops_id(u32 type_id)
+{
+	enum cgroup_bpf_attach_type atype;
+
+	for (atype = 0; atype < MAX_CGROUP_BPF_ATTACH_TYPE; atype++) {
+		if (cgroup_bpf_is_struct_ops_atype(atype) &&
+		    cgroup_struct_ops[atype].type_id == type_id)
+			return atype;
+	}
+	return CGROUP_BPF_ATTACH_TYPE_INVALID;
+}
+
 /*
  * cgroup bpf destruction makes heavy use of work items and there can be a lot
  * of concurrent destructions.  Use a separate workqueue so that cgroup bpf
@@ -306,6 +333,19 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[],
 		bpf_cgroup_storage_link(storages[stype], cgrp, attach_type);
 }
 
+static void cgroup_struct_ops_link_detach_wake(struct bpf_cgroup_link *link, bool wake_poll)
+{
+	cgroup_put(link->cgroup);
+	link->cgroup = NULL;
+
+	bpf_map_put(link->map);
+	/* READ_ONCE in cgroup_struct_ops_link_poll */
+	WRITE_ONCE(link->map, NULL);
+
+	if (wake_poll)
+		wake_up_interruptible_poll(&link->wait_hup, EPOLLHUP);
+}
+
 /* Called when bpf_cgroup_link is auto-detached from dying cgroup.
  * It drops cgroup and bpf_prog refcounts, and marks bpf_link as defunct. It
  * doesn't free link memory, which will eventually be done by bpf_link's
@@ -313,10 +353,96 @@ static void bpf_cgroup_storages_link(struct bpf_cgroup_storage *storages[],
  */
 static void bpf_cgroup_link_auto_detach(struct bpf_cgroup_link *link)
 {
+	if (link->map) {
+		cgroup_struct_ops_link_detach_wake(link, true);
+		return;
+	}
+
+	if (link->link.prog->expected_attach_type == BPF_LSM_CGROUP)
+		bpf_trampoline_unlink_cgroup_shim(link->link.prog);
 	cgroup_put(link->cgroup);
 	link->cgroup = NULL;
 }
 
+static void bpf_cgroup_array_free_rcu(struct rcu_head *rcu)
+{
+	kfree(container_of(rcu, struct bpf_prog_array, rcu));
+}
+
+static void bpf_cgroup_array_free(struct bpf_prog_array *array,
+				  enum cgroup_bpf_attach_type atype)
+{
+	if (!array || array == &bpf_empty_prog_array)
+		return;
+	if (cgroup_struct_ops[atype].mult_rcu)
+		/* RCU tasks trace grace period implies RCU grace period. */
+		call_rcu_tasks_trace(&array->rcu, bpf_cgroup_array_free_rcu);
+	else
+		kfree_rcu(array, rcu);
+}
+
+static void *bpf_cgroup_array_dummy(enum cgroup_bpf_attach_type atype)
+{
+	if (cgroup_bpf_is_struct_ops_atype(atype))
+		return cgroup_struct_ops[atype].cfi_stubs;
+	return bpf_prog_dummy();
+}
+
+static int bpf_cgroup_array_length(struct bpf_prog_array *array,
+				   enum cgroup_bpf_attach_type atype)
+{
+	struct bpf_prog_array_item *item;
+	int cnt = 0;
+
+	for (item = array->items; item->prog; item++) {
+		if (item->prog != bpf_cgroup_array_dummy(atype))
+			cnt++;
+	}
+
+	return cnt;
+}
+
+static int bpf_cgroup_array_copy_to_user(struct bpf_prog_array *array,
+					 __u32 __user *prog_ids, int cnt,
+					 enum cgroup_bpf_attach_type atype)
+{
+	struct bpf_prog_array_item *item;
+	int i = 0;
+	u32 id;
+
+	for (item = array->items; item->prog && i < cnt; item++) {
+		if (item->prog == bpf_cgroup_array_dummy(atype))
+			continue;
+
+		if (cgroup_bpf_is_struct_ops_atype(atype))
+			id = bpf_struct_ops_id(item->kdata);
+		else
+			id = item->prog->aux->id;
+
+		if (copy_to_user(prog_ids + i, &id, sizeof(id)))
+			return -EFAULT;
+		i++;
+	}
+	return item->prog ? -ENOSPC : 0;
+}
+
+static int bpf_cgroup_array_delete_safe_at(struct bpf_prog_array *array,
+					   int index, enum cgroup_bpf_attach_type atype)
+{
+	struct bpf_prog_array_item *item;
+
+	for (item = array->items; item->prog; item++) {
+		if (item->prog == bpf_cgroup_array_dummy(atype))
+			continue;
+		if (!index) {
+			WRITE_ONCE(item->prog, bpf_cgroup_array_dummy(atype));
+			return 0;
+		}
+		index--;
+	}
+	return -ENOENT;
+}
+
 /**
  * cgroup_bpf_release() - put references of all bpf programs and
  *                        release all cgroup bpf data
@@ -346,18 +472,15 @@ static void cgroup_bpf_release(struct work_struct *work)
 					bpf_trampoline_unlink_cgroup_shim(pl->prog);
 				bpf_prog_put(pl->prog);
 			}
-			if (pl->link) {
-				if (pl->link->link.prog->expected_attach_type == BPF_LSM_CGROUP)
-					bpf_trampoline_unlink_cgroup_shim(pl->link->link.prog);
+			if (pl->link)
 				bpf_cgroup_link_auto_detach(pl->link);
-			}
 			kfree(pl);
 			static_branch_dec(&cgroup_bpf_enabled_key[atype]);
 		}
 		old_array = rcu_dereference_protected(
 				cgrp->bpf.effective[atype],
 				lockdep_is_held(&cgroup_mutex));
-		bpf_prog_array_free(old_array);
+		bpf_cgroup_array_free(old_array, atype);
 	}
 
 	list_for_each_entry_safe(storage, stmp, storages, list_cg) {
@@ -399,6 +522,37 @@ static struct bpf_prog *prog_list_prog(struct bpf_prog_list *pl)
 	return NULL;
 }
 
+static bool prog_list_is_struct_ops(const struct bpf_prog_list *pl)
+{
+	return pl->link && pl->link->map;
+}
+
+static void prog_list_init_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item)
+{
+	if (prog_list_is_struct_ops(pl)) {
+		item->kdata = bpf_struct_ops_map_kdata(pl->link->map);
+		return;
+	}
+
+	item->prog = prog_list_prog(pl);
+	bpf_cgroup_storages_assign(item->cgroup_storage, pl->storage);
+}
+
+static void prog_list_replace_item(struct bpf_prog_list *pl, struct bpf_prog_array_item *item)
+{
+	if (prog_list_is_struct_ops(pl))
+		WRITE_ONCE(item->kdata, bpf_struct_ops_map_kdata(pl->link->map));
+	else
+		WRITE_ONCE(item->prog, pl->link->link.prog);
+}
+
+static u32 prog_list_id(struct bpf_prog_list *pl)
+{
+	if (prog_list_is_struct_ops(pl))
+		return pl->link->map->id;
+	return prog_list_prog(pl)->aux->id;
+}
+
 /* count number of elements in the list.
  * it's slow but the list cannot be long
  */
@@ -408,7 +562,7 @@ static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt)
 	u32 cnt = 0;
 
 	hlist_for_each_entry(pl, head, node) {
-		if (!prog_list_prog(pl))
+		if (!pl->prog && !pl->link)
 			continue;
 		if (preorder_cnt && (pl->flags & BPF_F_PREORDER))
 			(*preorder_cnt)++;
@@ -482,7 +636,7 @@ static int compute_effective_progs(struct cgroup *cgrp,
 
 		init_bstart = bstart;
 		hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
-			if (!prog_list_prog(pl))
+			if (!pl->prog && !pl->link)
 				continue;
 
 			if (pl->flags & BPF_F_PREORDER) {
@@ -492,9 +646,7 @@ static int compute_effective_progs(struct cgroup *cgrp,
 				item = &progs->items[fstart];
 				fstart++;
 			}
-			item->prog = prog_list_prog(pl);
-			bpf_cgroup_storages_assign(item->cgroup_storage,
-						   pl->storage);
+			prog_list_init_item(pl, item);
 			cnt++;
 		}
 
@@ -517,7 +669,7 @@ static void activate_effective_progs(struct cgroup *cgrp,
 	/* free prog array after grace period, since __cgroup_bpf_run_*()
 	 * might be still walking the array
 	 */
-	bpf_prog_array_free(old_array);
+	bpf_cgroup_array_free(old_array, atype);
 }
 
 /**
@@ -557,7 +709,7 @@ static int cgroup_bpf_inherit(struct cgroup *cgrp)
 	return 0;
 cleanup:
 	for (i = 0; i < NR; i++)
-		bpf_prog_array_free(arrays[i]);
+		bpf_cgroup_array_free(arrays[i], i);
 
 	for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p))
 		cgroup_bpf_put(p);
@@ -612,7 +764,7 @@ static int update_effective_progs(struct cgroup *cgrp,
 
 		if (percpu_ref_is_zero(&desc->bpf.refcnt)) {
 			if (unlikely(desc->bpf.inactive)) {
-				bpf_prog_array_free(desc->bpf.inactive);
+				bpf_cgroup_array_free(desc->bpf.inactive, atype);
 				desc->bpf.inactive = NULL;
 			}
 			continue;
@@ -631,7 +783,7 @@ static int update_effective_progs(struct cgroup *cgrp,
 	css_for_each_descendant_pre(css, &cgrp->self) {
 		struct cgroup *desc = container_of(css, struct cgroup, self);
 
-		bpf_prog_array_free(desc->bpf.inactive);
+		bpf_cgroup_array_free(desc->bpf.inactive, atype);
 		desc->bpf.inactive = NULL;
 	}
 
@@ -870,7 +1022,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
 		old_pl_flags = pl->flags;
 		bpf_cgroup_storages_assign(old_storage, pl->storage);
 	} else {
-		pl = kmalloc_obj(*pl);
+		pl = kzalloc_obj(*pl);
 		if (!pl) {
 			bpf_cgroup_storages_free(new_storage);
 			return -ENOMEM;
@@ -973,9 +1125,10 @@ static int effective_prog_pos(struct cgroup *cgrp,
 
 		init_bstart = bstart;
 		hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
-			if (!prog_list_prog(pl))
-				continue;
-
+			/*
+			 * No detaching pl (NULL prog and link) is visible to the callers,
+			 * so skip the check compute_effective_progs() needs.
+			 */
 			if (pl->flags & BPF_F_PREORDER) {
 				if (pl == target_pl)
 					pos = bstart;
@@ -1022,7 +1175,7 @@ static void replace_effective_prog(struct cgroup *cgrp,
 				desc->bpf.effective[atype],
 				lockdep_is_held(&cgroup_mutex));
 		item = &progs->items[pos];
-		WRITE_ONCE(item->prog, pl->link->link.prog);
+		prog_list_replace_item(pl, item);
 	}
 }
 
@@ -1177,7 +1330,7 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog_list *pl,
 				lockdep_is_held(&cgroup_mutex));
 
 		/* Remove the program from the array */
-		WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos),
+		WARN_ONCE(bpf_cgroup_array_delete_safe_at(progs, pos, atype),
 			  "Failed to purge a prog from array at index %d", pos);
 	}
 }
@@ -1287,7 +1440,17 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 	if (effective_query && prog_attach_flags)
 		return -EINVAL;
 
-	if (type == BPF_LSM_CGROUP) {
+	if (type == BPF_STRUCT_OPS) {
+		u32 type_id = attr->query.type_id;
+
+		atype = find_atype_by_struct_ops_id(type_id);
+		if (atype == CGROUP_BPF_ATTACH_TYPE_INVALID)
+			return -ENOENT;
+		from_atype = to_atype = atype;
+		flags = 0;
+		if (!cgroup_bpf_enabled(atype))
+			goto skip_count;
+	} else if (type == BPF_LSM_CGROUP) {
 		if (!effective_query && attr->query.prog_cnt &&
 		    prog_ids && !prog_attach_flags)
 			return -EINVAL;
@@ -1307,12 +1470,13 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 		if (effective_query) {
 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
 							      lockdep_is_held(&cgroup_mutex));
-			total_cnt += bpf_prog_array_length(effective);
+			total_cnt += bpf_cgroup_array_length(effective, atype);
 		} else {
 			total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
 		}
 	}
 
+skip_count:
 	/* always output uattr->query.attach_flags as 0 during effective query */
 	flags = effective_query ? 0 : flags;
 	if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
@@ -1337,35 +1501,33 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 		if (effective_query) {
 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
 							      lockdep_is_held(&cgroup_mutex));
-			cnt = min_t(int, bpf_prog_array_length(effective), total_cnt);
-			ret = bpf_prog_array_copy_to_user(effective, prog_ids, cnt);
+			cnt = min_t(int, bpf_cgroup_array_length(effective, atype), total_cnt);
+			ret = bpf_cgroup_array_copy_to_user(effective, prog_ids, cnt, atype);
 		} else {
 			struct hlist_head *progs;
 			struct bpf_prog_list *pl;
-			struct bpf_prog *prog;
 			u32 id;
 
 			progs = &cgrp->bpf.progs[atype];
 			cnt = min_t(int, prog_list_length(progs, NULL), total_cnt);
 			i = 0;
 			hlist_for_each_entry(pl, progs, node) {
-				prog = prog_list_prog(pl);
-				id = prog->aux->id;
+				id = prog_list_id(pl);
 				if (copy_to_user(prog_ids + i, &id, sizeof(id)))
 					return -EFAULT;
+				if (prog_attach_flags) {
+					flags = cgrp->bpf.flags[atype] |
+						(pl->flags & BPF_F_PREORDER);
+					if (copy_to_user(prog_attach_flags + i,
+							 &flags, sizeof(flags)))
+						return -EFAULT;
+				}
 				if (++i == cnt)
 					break;
 			}
 
-			if (prog_attach_flags) {
-				flags = cgrp->bpf.flags[atype];
-
-				for (i = 0; i < cnt; i++)
-					if (copy_to_user(prog_attach_flags + i,
-							 &flags, sizeof(flags)))
-						return -EFAULT;
+			if (prog_attach_flags)
 				prog_attach_flags += cnt;
-			}
 		}
 
 		prog_ids += cnt;
@@ -2329,7 +2491,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = {
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_CTX,
-	.arg2_type	= ARG_PTR_TO_MEM | MEM_WRITE,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
 	.arg3_type	= ARG_MEM_SIZE,
 	.arg4_type	= ARG_ANYTHING,
 };
@@ -2772,6 +2934,274 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = {
 const struct bpf_prog_ops cg_sockopt_prog_ops = {
 };
 
+static int __cgroup_struct_ops_link_detach(struct bpf_link *link, bool wake_poll)
+{
+	struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link);
+	enum cgroup_bpf_attach_type atype;
+	struct bpf_prog_list *pl;
+	struct bpf_map *map;
+	struct cgroup *cgrp;
+
+	cgroup_lock();
+
+	cgrp = cg_link->cgroup;
+	if (!cgrp) {
+		cgroup_unlock();
+		return 0;
+	}
+
+	map = cg_link->map;
+	atype = bpf_struct_ops_map_cgroup_atype(map);
+
+	hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) {
+		if (pl->link == cg_link)
+			break;
+	}
+	if (WARN_ON_ONCE(!pl)) {
+		cgroup_unlock();
+		return -ENOENT;
+	}
+
+	/* mark deleted so compute_effective_progs() skips it */
+	pl->link = NULL;
+	if (update_effective_progs(cgrp, atype)) {
+		pl->link = cg_link;
+		purge_effective_progs(cgrp, pl, atype);
+	}
+
+	hlist_del(&pl->node);
+	cgroup_struct_ops_link_detach_wake(cg_link, wake_poll);
+	cgrp->bpf.revisions[atype]++;
+
+	kfree(pl);
+	static_branch_dec(&cgroup_bpf_enabled_key[atype]);
+
+	cgroup_unlock();
+
+	return 0;
+}
+
+static int cgroup_struct_ops_link_detach(struct bpf_link *link)
+{
+	return __cgroup_struct_ops_link_detach(link, true);
+}
+
+static void cgroup_struct_ops_link_dealloc(struct bpf_link *link)
+{
+	struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link);
+
+	__cgroup_struct_ops_link_detach(link, false);
+	kfree(cg_link);
+}
+
+static void cgroup_struct_ops_link_show_fdinfo(const struct bpf_link *link, struct seq_file *seq)
+{
+	struct bpf_cgroup_link *cg_link =
+		container_of(link, struct bpf_cgroup_link, link);
+
+	cgroup_lock();
+	if (!cg_link->cgroup) {
+		cgroup_unlock();
+		return;
+	}
+
+	seq_printf(seq, "map_id:\t%u\n", cg_link->map->id);
+	seq_printf(seq, "cgroup_id:\t%llu\n", cgroup_id(cg_link->cgroup));
+	cgroup_unlock();
+}
+
+static int cgroup_struct_ops_link_fill_link_info(const struct bpf_link *link,
+						 struct bpf_link_info *info)
+{
+	struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link);
+
+	cgroup_lock();
+	if (!cg_link->cgroup) {
+		cgroup_unlock();
+		return 0;
+	}
+
+	info->struct_ops.map_id = cg_link->map->id;
+	info->struct_ops.cgroup_id = cgroup_id(cg_link->cgroup);
+	cgroup_unlock();
+	return 0;
+}
+
+static int cgroup_struct_ops_link_update(struct bpf_link *link, struct bpf_map *new_map,
+					 struct bpf_map *expected_old_map)
+{
+	struct bpf_cgroup_link *cg_link = container_of(link, struct bpf_cgroup_link, link);
+	enum cgroup_bpf_attach_type atype;
+	struct bpf_prog_list *pl;
+	struct bpf_map *old_map;
+	struct cgroup *cgrp;
+	bool found = false;
+	int err;
+
+	if (!bpf_struct_ops_valid_to_reg(new_map))
+		return -EINVAL;
+
+	cgroup_lock();
+
+	cgrp = cg_link->cgroup;
+	if (!cgrp) {
+		err = -ENOLINK;
+		goto out;
+	}
+
+	old_map = cg_link->map;
+	err = bpf_struct_ops_link_update_check(new_map, old_map, expected_old_map);
+	if (err)
+		goto out;
+
+	atype = bpf_struct_ops_map_cgroup_atype(new_map);
+
+	hlist_for_each_entry(pl, &cgrp->bpf.progs[atype], node) {
+		if (pl->link == cg_link) {
+			found = true;
+			break;
+		}
+	}
+	if (!found) {
+		err = -ENOENT;
+		goto out;
+	}
+
+	bpf_map_inc(new_map);
+	WRITE_ONCE(cg_link->map, new_map);
+	replace_effective_prog(cgrp, atype, pl);
+	bpf_map_put(old_map);
+	cgrp->bpf.revisions[atype]++;
+
+out:
+	cgroup_unlock();
+	return err;
+}
+
+static __poll_t cgroup_struct_ops_link_poll(struct file *file, struct poll_table_struct *pts)
+{
+	struct bpf_cgroup_link *link = file->private_data;
+
+	poll_wait(file, &link->wait_hup, pts);
+
+	return READ_ONCE(link->map) ? 0 : EPOLLHUP;
+}
+
+static const struct bpf_link_ops cgroup_struct_ops_link_ops = {
+	.dealloc = cgroup_struct_ops_link_dealloc,
+	.detach = cgroup_struct_ops_link_detach,
+	.show_fdinfo = cgroup_struct_ops_link_show_fdinfo,
+	.fill_link_info = cgroup_struct_ops_link_fill_link_info,
+	.update_map = cgroup_struct_ops_link_update,
+	.poll = cgroup_struct_ops_link_poll,
+};
+
+int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr)
+{
+	u32 flags = attr->link_create.flags;
+	u32 pl_flags = (flags & BPF_F_PREORDER) | BPF_F_ALLOW_MULTI;
+	enum cgroup_bpf_attach_type atype;
+	struct bpf_link_primer link_primer;
+	struct bpf_cgroup_link *link;
+	struct bpf_prog_list *pl = NULL;
+	struct hlist_head *progs;
+	struct cgroup *cgrp;
+	int err;
+
+	if (flags & ~BPF_F_LINK_ATTACH_MASK)
+		return -EINVAL;
+
+	/*
+	 * Attaching struct_ops to cgroup is through link only. All relative
+	 * position must be corresponding to a link id or fd.
+	 */
+	if (attr->link_create.cgroup.relative_fd && !(flags & BPF_F_LINK))
+		return -EINVAL;
+
+	link = kzalloc_obj(*link, GFP_USER);
+	if (!link)
+		return -ENOMEM;
+
+	bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
+		      &cgroup_struct_ops_link_ops, NULL,
+		      attr->link_create.attach_type);
+
+	err = bpf_link_prime(&link->link, &link_primer);
+	if (err) {
+		kfree(link);
+		return err;
+	}
+
+	cgrp = cgroup_get_from_fd(attr->link_create.target_fd);
+	if (IS_ERR(cgrp)) {
+		err = PTR_ERR(cgrp);
+		goto cleanup;
+	}
+
+	bpf_map_inc(map);
+	link->map = map;
+	link->cgroup = cgrp;
+	init_waitqueue_head(&link->wait_hup);
+
+	atype = bpf_struct_ops_map_cgroup_atype(map);
+	progs = &cgrp->bpf.progs[atype];
+
+	cgroup_lock();
+
+	if (attr->link_create.cgroup.expected_revision &&
+	    attr->link_create.cgroup.expected_revision != cgrp->bpf.revisions[atype]) {
+		err = -ESTALE;
+		goto unlock;
+	}
+
+	if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS) {
+		err = -E2BIG;
+		goto unlock;
+	}
+
+	pl = kzalloc_obj(*pl);
+	if (!pl) {
+		err = -ENOMEM;
+		goto unlock;
+	}
+
+	pl->link = link;
+	pl->flags = pl_flags;
+	cgrp->bpf.flags[atype] = BPF_F_ALLOW_MULTI;
+
+	err = insert_pl_to_hlist(pl, progs, NULL, link,
+				 flags | BPF_F_ALLOW_MULTI, attr->link_create.cgroup.relative_fd);
+	if (err)
+		goto unlock;
+
+	err = update_effective_progs(cgrp, atype);
+	if (err) {
+		hlist_del(&pl->node);
+		goto unlock;
+	}
+
+	cgrp->bpf.revisions[atype]++;
+	static_branch_inc(&cgroup_bpf_enabled_key[atype]);
+
+	cgroup_unlock();
+
+	return bpf_link_settle(&link_primer);
+
+unlock:
+	cgroup_unlock();
+
+cleanup:
+	kfree(pl);
+	if (link->cgroup) {
+		cgroup_put(link->cgroup);
+		link->cgroup = NULL;
+		bpf_map_put(link->map);
+		link->map = NULL;
+	}
+	bpf_link_cleanup(&link_primer);
+	return err;
+}
+
 /* Common helpers for cgroup hooks. */
 const struct bpf_func_proto *
 cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c
index 0e8b3ccc7a5b94..4c1ed842f661fe 100644
--- a/kernel/bpf/check_btf.c
+++ b/kernel/bpf/check_btf.c
@@ -338,9 +338,9 @@ static int check_btf_line(struct bpf_verifier_env *env,
 #define MIN_CORE_RELO_SIZE	sizeof(struct bpf_core_relo)
 #define MAX_CORE_RELO_SIZE	MAX_FUNCINFO_REC_SIZE
 
-static int check_core_relo(struct bpf_verifier_env *env,
-			   const union bpf_attr *attr,
-			   bpfptr_t uattr)
+int bpf_check_core_relo(struct bpf_verifier_env *env,
+			const union bpf_attr *attr,
+			bpfptr_t uattr)
 {
 	u32 i, nr_core_relo, ncopy, expected_size, rec_size;
 	struct bpf_core_relo core_relo = {};
@@ -414,7 +414,7 @@ int bpf_prepare_btf_info(struct bpf_verifier_env *env,
 	struct btf *btf;
 	int err;
 
-	if (!attr->func_info_cnt && !attr->line_info_cnt) {
+	if (!attr->func_info_cnt && !attr->line_info_cnt && !attr->core_relo_cnt) {
 		if (check_abnormal_return(env))
 			return -EINVAL;
 		return 0;
@@ -455,9 +455,5 @@ int bpf_check_btf_info(struct bpf_verifier_env *env,
 	if (err)
 		return err;
 
-	err = check_core_relo(env, attr, uattr);
-	if (err)
-		return err;
-
 	return 0;
 }
diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h
index a90e317e35780f..30685e43de0492 100644
--- a/kernel/bpf/cnum_defs.h
+++ b/kernel/bpf/cnum_defs.h
@@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max)
 	return (struct cnum_t){ .base = base, .size = size };
 }
 
-/* True if this cnum represents two unsigned ranges. */
+/*
+ * True if this cnum represents two unsigned ranges.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * this function.
+ */
 static inline bool FN(urange_overflow)(struct cnum_t cnum)
 {
 	/* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */
@@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum)
  * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum.
  * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range
  * [0..UT_MAX] is returned.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * cnum{T}_umin / cnum{T}_umax.
  */
 ut FN(umin)(struct cnum_t cnum)
 {
@@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum)
 }
 EXPORT_SYMBOL_GPL(FN(umax));
 
-/* True if this cnum represents two signed ranges. */
+/*
+ * True if this cnum represents two signed ranges.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * this function.
+ */
 static inline bool FN(srange_overflow)(struct cnum_t cnum)
 {
 	return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN);
@@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum)
  * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum.
  * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range
  * [ST_MIN..ST_MAX] is returned.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * cnum{T}_smin / cnum{T}_smax.
  */
 st FN(smin)(struct cnum_t cnum)
 {
-	return FN(srange_overflow)(cnum)
-	       ? ST_MIN
-	       : min((st)cnum.base, (st)(cnum.base + cnum.size));
+	return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base;
 }
 
 st FN(smax)(struct cnum_t cnum)
 {
-	return FN(srange_overflow)(cnum)
-	       ? ST_MAX
-	       : max((st)cnum.base, (st)(cnum.base + cnum.size));
+	return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size);
 }
 
 /*
@@ -181,7 +193,7 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max)
 
 static inline struct cnum_t FN(normalize)(struct cnum_t cnum)
 {
-	if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX)
+	if (cnum.size == UT_MAX)
 		cnum.base = 0;
 	return cnum;
 }
@@ -210,12 +222,7 @@ bool FN(is_empty)(struct cnum_t cnum)
 
 bool FN(contains)(struct cnum_t cnum, ut v)
 {
-	if (FN(is_empty)(cnum))
-		return false;
-	if (FN(urange_overflow)(cnum))
-		return v >= cnum.base || v <= (ut)cnum.base + cnum.size;
-	else
-		return v >= cnum.base && v <= (ut)cnum.base + cnum.size;
+	return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size;
 }
 
 bool FN(is_const)(struct cnum_t cnum)
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
index 7f1b30059cc879..b1528adbeb79d3 100644
--- a/kernel/bpf/const_fold.c
+++ b/kernel/bpf/const_fold.c
@@ -180,9 +180,12 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *
 		bool is_ldsx = mode == BPF_MEMSX;
 		int off = src->val + insn->off;
 		u64 val = 0;
+		u32 cnt;
 
 		if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr ||
 		    off < 0 || off + size > map->value_size ||
+		    /* so are the addresses of functions that the map points to */
+		    bpf_map_range_func_ptrs(env, map, off, size, &cnt) ||
 		    bpf_map_direct_read(map, off, size, &val, is_ldsx)) {
 			*dst = unknown;
 			break;
@@ -191,7 +194,8 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *
 		dst->val = val;
 		break;
 	case BPF_JMP:
-		if (opcode != BPF_CALL)
+		/* both 'call imm' and 'callx reg' clobber caller saved registers */
+		if (BPF_OP(insn->code) != BPF_CALL)
 			break;
 process_call:
 		for (r = BPF_REG_0; r <= BPF_REG_5; r++)
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 8b294dfc1ad4de..a67c895ce7344a 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -19,6 +19,7 @@
 
 #include <uapi/linux/btf.h>
 #include <linux/filter.h>
+#include <linux/sched/signal.h>
 #include <linux/skbuff.h>
 #include <linux/static_call.h>
 #include <linux/vmalloc.h>
@@ -141,10 +142,6 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
 	mutex_init(&fp->aux->dst_mutex);
 	mutex_init(&fp->aux->st_ops_assoc_mutex);
 
-#ifdef CONFIG_BPF_SYSCALL
-	bpf_prog_stream_init(fp);
-#endif
-
 	return fp;
 }
 
@@ -288,6 +285,9 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
 void __bpf_prog_free(struct bpf_prog *fp)
 {
 	if (fp->aux) {
+#ifdef CONFIG_BPF_SYSCALL
+		bpf_prog_stream_free(fp);
+#endif
 		mutex_destroy(&fp->aux->used_maps_mutex);
 		mutex_destroy(&fp->aux->dst_mutex);
 		mutex_destroy(&fp->aux->st_ops_assoc_mutex);
@@ -1361,7 +1361,7 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
 {
 	struct bpf_insn *to = to_buff;
 	u32 imm_rnd = get_random_u32();
-	s16 off;
+	int off;
 
 	BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG);
 	BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG);
@@ -1437,6 +1437,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
 		off = from->off;
 		if (off < 0)
 			off -= 2;
+		if (off < S16_MIN)
+			return -ERANGE;
 		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
 		*to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
 		*to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off);
@@ -1457,6 +1459,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
 		off = from->off;
 		if (off < 0)
 			off -= 2;
+		if (off < S16_MIN)
+			return -ERANGE;
 		*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm);
 		*to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd);
 		*to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX,
@@ -1605,7 +1609,9 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
 		if (!rewritten)
 			continue;
 
-		if (env)
+		if (rewritten < 0)
+			tmp = ERR_PTR(rewritten);
+		else if (env)
 			tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
 		else
 			tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
@@ -1619,6 +1625,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp
 			 * fix it up here on error.
 			 */
 			bpf_jit_prog_release_other(prog, clone);
+			if (env && fatal_signal_pending(current))
+				return ERR_PTR(-EINTR);
 			return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM);
 		}
 
@@ -1831,6 +1839,7 @@ bool bpf_opcode_in_insntable(u8 code)
 		[BPF_LD | BPF_IND | BPF_H] = true,
 		[BPF_LD | BPF_IND | BPF_W] = true,
 		[BPF_JMP | BPF_JA | BPF_X] = true,
+		[BPF_JMP | BPF_CALL | BPF_X] = true,
 		[BPF_JMP | BPF_JCOND] = true,
 	};
 #undef BPF_INSN_3_TBL
@@ -2636,11 +2645,14 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc
 	orig_prog = prog;
 	prog = bpf_jit_blind_constants(env, prog);
 	/*
-	 * If blinding was requested and we failed during blinding, we must fall
-	 * back to the interpreter.
+	 * Fall back to the interpreter after blinding failures, except when
+	 * the loader was killed.
 	 */
-	if (IS_ERR(prog))
+	if (IS_ERR(prog)) {
+		if (PTR_ERR(prog) == -EINTR)
+			return prog;
 		goto out_restore;
+	}
 
 	prog = bpf_int_jit_compile(env, prog);
 	if (prog->jited) {
@@ -2659,6 +2671,8 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc
 struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
 					   int *err)
 {
+	struct bpf_prog *jit_prog;
+
 	/* In case of BPF to BPF calls, verifier did all the prep
 	 * work with regards to JITing, etc.
 	 */
@@ -2681,7 +2695,12 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct
 		if (*err)
 			return fp;
 
-		fp = bpf_prog_jit_compile(env, fp);
+		jit_prog = bpf_prog_jit_compile(env, fp);
+		if (IS_ERR(jit_prog)) {
+			*err = PTR_ERR(jit_prog);
+			return fp;
+		}
+		fp = jit_prog;
 		bpf_prog_jit_attempt_done(fp);
 		if (!fp->jited && jit_needed) {
 			*err = -ENOTSUPP;
@@ -2784,6 +2803,11 @@ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs)
 	call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb);
 }
 
+struct bpf_prog *bpf_prog_dummy(void)
+{
+	return &dummy_bpf_prog.prog;
+}
+
 int bpf_prog_array_length(struct bpf_prog_array *array)
 {
 	struct bpf_prog_array_item *item;
@@ -3023,6 +3047,11 @@ void __bpf_free_used_maps(struct bpf_prog_aux *aux,
 			map->ops->map_poke_untrack(map, aux);
 		if (sleepable)
 			atomic64_dec(&map->sleepable_refcnt);
+		/*
+		 * The program that didn't load is not a user of the map. libbpf
+		 * loads the program again to get the log of the verifier.
+		 */
+		cmpxchg(&map->user, (unsigned long)aux, 0);
 		bpf_map_put(map);
 	}
 }
@@ -3062,7 +3091,6 @@ static void bpf_prog_free_deferred(struct work_struct *work)
 	aux = container_of(work, struct bpf_prog_aux, work);
 #ifdef CONFIG_BPF_SYSCALL
 	bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab);
-	bpf_prog_stream_free(aux->prog);
 #endif
 #ifdef CONFIG_CGROUP_BPF
 	if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID)
@@ -3282,6 +3310,111 @@ bool __weak bpf_jit_supports_kfunc_call(void)
 	return false;
 }
 
+/* Return TRUE if the JIT backend supports callx (indirect call) instruction. */
+bool __weak bpf_jit_supports_callx(void)
+{
+	return false;
+}
+
+bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void)
+{
+	return false;
+}
+
+/*
+ * How this arch places a by-value kfunc argument, or NULL for one that has
+ * not opted in and so only takes an argument of a single eightbyte, which
+ * every convention places in slot order.
+ */
+const struct bpf_jit_arg_abi * __weak bpf_jit_arg_abi(void)
+{
+	return NULL;
+}
+
+u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi,
+		       const struct btf_func_model *fm, u8 *pos_of_slot)
+{
+	u32 i, k, nslots, slot = 0, nregs_used = 0, stack_off = 0;
+	bool on_stack = false;
+
+	for (i = 0; i < fm->nr_args; i++) {
+		bool align16 = fm->arg_flags[i] & BTF_FMODEL_ALIGN16_ARG;
+		u32 pos;
+
+		nslots = btf_func_model_arg_slots(fm, i);
+
+		if (align16 && abi->even_reg_align)
+			nregs_used = round_up(nregs_used, 2);
+
+		if (!on_stack && nregs_used + nslots <= abi->nr_arg_regs) {
+			/* wholly in registers */
+			pos = nregs_used;
+			nregs_used += nslots;
+		} else if (!on_stack && abi->split_at_boundary) {
+			/* the last registers hold what fits, the stack the rest */
+			pos = nregs_used;
+			stack_off = (nregs_used + nslots - abi->nr_arg_regs) * BPF_REG_SIZE;
+			nregs_used = abi->nr_arg_regs;
+			on_stack = true;
+		} else {
+			/* wholly on the stack */
+			if (align16 && abi->even_stack_align)
+				stack_off = round_up(stack_off, 2 * BPF_REG_SIZE);
+			pos = abi->nr_arg_regs + stack_off / BPF_REG_SIZE;
+			stack_off += nslots * BPF_REG_SIZE;
+			if (!abi->backfill_after_stack)
+				on_stack = true;
+		}
+
+		for (k = 0; k < nslots; k++)
+			pos_of_slot[slot + k] = pos + k;
+		slot += nslots;
+	}
+
+	return slot;
+}
+
+u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi,
+			   const struct btf_func_model *fm,
+			   struct bpf_jit_arg_move *moves)
+{
+	u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS];
+	u32 nslots, n = 0, s, back;
+
+	nslots = bpf_jit_place_args(abi, fm, pos_of_slot);
+	back = nslots;
+
+	/*
+	 * An argument is two eightbytes at most, so it frees one register at
+	 * most and only one argument ever moves down. Its destination is
+	 * still in use, so carry it in the scratch. Only a lower slot can
+	 * take the one it leaves, so the walk reaches it first.
+	 */
+	for (s = nslots; s > 0; s--) {
+		u8 slot = s - 1, pos = pos_of_slot[slot];
+
+		if (pos == slot)
+			continue;
+
+		if (pos < slot) {
+			moves[n].dst = BPF_JIT_ARG_TMP;
+			back = slot;
+		} else {
+			moves[n].dst = pos;
+		}
+		moves[n].src = slot;
+		n++;
+	}
+
+	if (back < nslots) {
+		moves[n].dst = pos_of_slot[back];
+		moves[n].src = BPF_JIT_ARG_TMP;
+		n++;
+	}
+
+	return n;
+}
+
 bool __weak bpf_jit_supports_stack_args(void)
 {
 	return false;
@@ -3367,6 +3500,19 @@ bool __weak bpf_jit_supports_private_stack(void)
 	return false;
 }
 
+/*
+ * Return TRUE if the JIT lays out frames of up to MAX_BPF_STACK_JIT bytes.
+ * Its prologue, epilogue and tail call sequences must encode such frame
+ * sizes and a private stack must be sized from the program's depth. The
+ * budget is only granted alongside bpf_jit_supports_subprog_tailcalls(),
+ * whose tail calls land before the target sets up its own frame; see
+ * bpf_prog_stack_limit().
+ */
+bool __weak bpf_jit_supports_large_stack(void)
+{
+	return false;
+}
+
 void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
 {
 }
diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c
index 51f89cecefb4dc..69bc29d0a4d4d4 100644
--- a/kernel/bpf/crypto.c
+++ b/kernel/bpf/crypto.c
@@ -1,19 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright (c) 2024 Meta, Inc */
 #include <linux/bpf.h>
-#include <linux/bpf_crypto.h>
 #include <linux/bpf_mem_alloc.h>
 #include <linux/btf.h>
 #include <linux/btf_ids.h>
 #include <linux/filter.h>
-#include <linux/scatterlist.h>
 #include <linux/skbuff.h>
-#include <crypto/skcipher.h>
-
-struct bpf_crypto_type_list {
-	const struct bpf_crypto_type *type;
-	struct list_head list;
-};
+#include <crypto/aes-cbc.h>
+#include <crypto/aes-ecb.h>
 
 /* BPF crypto initialization parameters struct */
 /**
@@ -36,94 +30,53 @@ struct bpf_crypto_params {
 	u32 authsize;
 };
 
-static LIST_HEAD(bpf_crypto_types);
-static DECLARE_RWSEM(bpf_crypto_types_sem);
+enum bpf_crypto_algo_id {
+	BPF_ALGO_AES_CBC,
+	BPF_ALGO_AES_ECB,
+};
+
+static const struct {
+	const char *type_name;
+	const char *algo_name;
+	enum bpf_crypto_algo_id algo;
+} bpf_crypto_algos[] = {
+	{ "skcipher", "cbc(aes)", BPF_ALGO_AES_CBC },
+	{ "skcipher", "ecb(aes)", BPF_ALGO_AES_ECB },
+};
+
+static bool bpf_crypto_find_algo(const struct bpf_crypto_params *params,
+				 enum bpf_crypto_algo_id *id_ret)
+{
+	for (size_t i = 0; i < ARRAY_SIZE(bpf_crypto_algos); i++) {
+		if (strncmp(bpf_crypto_algos[i].type_name, params->type,
+			    sizeof(params->type)) == 0 &&
+		    strncmp(bpf_crypto_algos[i].algo_name, params->algo,
+			    sizeof(params->algo)) == 0) {
+			*id_ret = bpf_crypto_algos[i].algo;
+			return true;
+		}
+	}
+	return false;
+}
 
 /**
  * struct bpf_crypto_ctx - refcounted BPF crypto context structure
- * @type:	The pointer to bpf crypto type
- * @tfm:	The pointer to instance of crypto API struct.
- * @siv_len:    Size of IV and state storage for cipher
+ * @key:	The crypto key
+ * @algo:	The crypto algorithm ID
  * @rcu:	The RCU head used to free the crypto context with RCU safety.
  * @usage:	Object reference counter. When the refcount goes to 0, the
  *		memory is released back to the BPF allocator, which provides
  *		RCU safety.
  */
 struct bpf_crypto_ctx {
-	const struct bpf_crypto_type *type;
-	void *tfm;
-	u32 siv_len;
+	union {
+		struct aes_key aes;
+	} key;
+	enum bpf_crypto_algo_id algo;
 	struct rcu_head rcu;
 	refcount_t usage;
 };
 
-int bpf_crypto_register_type(const struct bpf_crypto_type *type)
-{
-	struct bpf_crypto_type_list *node;
-	int err = -EBUSY;
-
-	down_write(&bpf_crypto_types_sem);
-	list_for_each_entry(node, &bpf_crypto_types, list) {
-		if (!strcmp(node->type->name, type->name))
-			goto unlock;
-	}
-
-	node = kmalloc_obj(*node);
-	err = -ENOMEM;
-	if (!node)
-		goto unlock;
-
-	node->type = type;
-	list_add(&node->list, &bpf_crypto_types);
-	err = 0;
-
-unlock:
-	up_write(&bpf_crypto_types_sem);
-
-	return err;
-}
-EXPORT_SYMBOL_GPL(bpf_crypto_register_type);
-
-int bpf_crypto_unregister_type(const struct bpf_crypto_type *type)
-{
-	struct bpf_crypto_type_list *node;
-	int err = -ENOENT;
-
-	down_write(&bpf_crypto_types_sem);
-	list_for_each_entry(node, &bpf_crypto_types, list) {
-		if (strcmp(node->type->name, type->name))
-			continue;
-
-		list_del(&node->list);
-		kfree(node);
-		err = 0;
-		break;
-	}
-	up_write(&bpf_crypto_types_sem);
-
-	return err;
-}
-EXPORT_SYMBOL_GPL(bpf_crypto_unregister_type);
-
-static const struct bpf_crypto_type *bpf_crypto_get_type(const char *name)
-{
-	const struct bpf_crypto_type *type = ERR_PTR(-ENOENT);
-	struct bpf_crypto_type_list *node;
-
-	down_read(&bpf_crypto_types_sem);
-	list_for_each_entry(node, &bpf_crypto_types, list) {
-		if (strcmp(node->type->name, name))
-			continue;
-
-		if (try_module_get(node->type->owner))
-			type = node->type;
-		break;
-	}
-	up_read(&bpf_crypto_types_sem);
-
-	return type;
-}
-
 __bpf_kfunc_start_defs();
 
 /**
@@ -132,92 +85,70 @@ __bpf_kfunc_start_defs();
  * Allocates a crypto context that can be used, acquired, and released by
  * a BPF program. The crypto context returned by this function must either
  * be embedded in a map as a kptr, or freed with bpf_crypto_ctx_release().
- * As crypto API functions use GFP_KERNEL allocations, this function can
- * only be used in sleepable BPF programs.
+ * As this uses a GFP_KERNEL allocation, this function can only be used in
+ * sleepable BPF programs.
  *
  * bpf_crypto_ctx_create() allocates memory for crypto context.
  * It may return NULL if no memory is available.
  * @params:	pointer to struct bpf_crypto_params which contains all the
  *		details needed to initialise crypto context.
- * @params__sz:	size of steuct bpf_crypto_params usef by bpf program
- * @err:	integer to store error code when NULL is returned.
+ * @params__sz:	size of struct bpf_crypto_params used by bpf program
+ * @err_ret:	integer to store error code when NULL is returned.
  */
 __bpf_kfunc struct bpf_crypto_ctx *
 bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz,
-		      int *err)
+		      int *err_ret)
 {
-	const struct bpf_crypto_type *type;
 	struct bpf_crypto_ctx *ctx;
+	int err;
 
-	if (!params || params->reserved[0] || params->reserved[1] ||
-	    params__sz != sizeof(struct bpf_crypto_params)) {
-		*err = -EINVAL;
-		return NULL;
-	}
-
-	type = bpf_crypto_get_type(params->type);
-	if (IS_ERR(type)) {
-		*err = PTR_ERR(type);
+	if (!params ||
+	    params__sz != sizeof(struct bpf_crypto_params) ||
+	    params->reserved[0] || params->reserved[1]) {
+		*err_ret = -EINVAL;
 		return NULL;
 	}
 
-	if (!type->has_algo(params->algo)) {
-		*err = -EOPNOTSUPP;
-		goto err_module_put;
-	}
-
-	if (!!params->authsize ^ !!type->setauthsize) {
-		*err = -EOPNOTSUPP;
-		goto err_module_put;
-	}
-
 	if (!params->key_len || params->key_len > sizeof(params->key)) {
-		*err = -EINVAL;
-		goto err_module_put;
+		*err_ret = -EINVAL;
+		return NULL;
 	}
 
 	ctx = kzalloc_obj(*ctx);
 	if (!ctx) {
-		*err = -ENOMEM;
-		goto err_module_put;
+		*err_ret = -ENOMEM;
+		return NULL;
 	}
 
-	ctx->type = type;
-	ctx->tfm = type->alloc_tfm(params->algo);
-	if (IS_ERR(ctx->tfm)) {
-		*err = PTR_ERR(ctx->tfm);
-		goto err_free_ctx;
+	if (!bpf_crypto_find_algo(params, &ctx->algo)) {
+		err = -EOPNOTSUPP;
+		goto out;
 	}
 
-	if (params->authsize) {
-		*err = type->setauthsize(ctx->tfm, params->authsize);
-		if (*err)
-			goto err_free_tfm;
+	switch (ctx->algo) {
+	case BPF_ALGO_AES_CBC:
+	case BPF_ALGO_AES_ECB:
+		if (params->authsize)
+			err = -EOPNOTSUPP;
+		else
+			err = aes_preparekey(&ctx->key.aes, params->key,
+					     params->key_len);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		err = -EOPNOTSUPP;
+		break;
 	}
 
-	*err = type->setkey(ctx->tfm, params->key, params->key_len);
-	if (*err)
-		goto err_free_tfm;
-
-	if (type->get_flags(ctx->tfm) & CRYPTO_TFM_NEED_KEY) {
-		*err = -EINVAL;
-		goto err_free_tfm;
+out:
+	if (err) {
+		kfree_sensitive(ctx);
+		*err_ret = err;
+		return NULL;
 	}
-
-	ctx->siv_len = type->ivsize(ctx->tfm) + type->statesize(ctx->tfm);
-
 	refcount_set(&ctx->usage, 1);
-
+	*err_ret = 0;
 	return ctx;
-
-err_free_tfm:
-	type->free_tfm(ctx->tfm);
-err_free_ctx:
-	kfree(ctx);
-err_module_put:
-	module_put(type->owner);
-
-	return NULL;
 }
 
 static void crypto_free_cb(struct rcu_head *head)
@@ -225,9 +156,7 @@ static void crypto_free_cb(struct rcu_head *head)
 	struct bpf_crypto_ctx *ctx;
 
 	ctx = container_of(head, struct bpf_crypto_ctx, rcu);
-	ctx->type->free_tfm(ctx->tfm);
-	module_put(ctx->type->owner);
-	kfree(ctx);
+	kfree_sensitive(ctx);
 }
 
 /**
@@ -267,27 +196,53 @@ __bpf_kfunc void bpf_crypto_ctx_release_dtor(void *ctx)
 }
 CFI_NOSEAL(bpf_crypto_ctx_release_dtor);
 
+static int bpf_aes_cbc_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len,
+			     u8 *iv, u32 iv_len,
+			     const struct bpf_crypto_ctx *ctx, bool decrypt)
+{
+	if (iv_len != AES_BLOCK_SIZE)
+		return -EINVAL;
+	if (src_len % AES_BLOCK_SIZE || dst_len < src_len)
+		return -EINVAL;
+	if (decrypt)
+		aes_cbc_decrypt(dst, src, src_len, iv, &ctx->key.aes);
+	else
+		aes_cbc_encrypt(dst, src, src_len, iv, &ctx->key.aes);
+	return 0;
+}
+
+static int bpf_aes_ecb_crypt(u8 *dst, u32 dst_len, const u8 *src, u32 src_len,
+			     u8 *iv, u32 iv_len,
+			     const struct bpf_crypto_ctx *ctx, bool decrypt)
+{
+	if (iv_len != 0)
+		return -EINVAL;
+	if (src_len % AES_BLOCK_SIZE || dst_len < src_len)
+		return -EINVAL;
+	if (decrypt)
+		aes_ecb_decrypt(dst, src, src_len, &ctx->key.aes);
+	else
+		aes_ecb_encrypt(dst, src, src_len, &ctx->key.aes);
+	return 0;
+}
+
 static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx,
 			    const struct bpf_dynptr_kern *src,
 			    const struct bpf_dynptr_kern *dst,
-			    const struct bpf_dynptr_kern *siv,
+			    const struct bpf_dynptr_kern *iv,
 			    bool decrypt)
 {
-	u32 src_len, dst_len, siv_len;
+	u32 src_len, dst_len, iv_len;
 	const u8 *psrc;
 	u8 *pdst, *piv;
-	int err;
 
 	if (__bpf_dynptr_is_rdonly(dst))
 		return -EINVAL;
 
-	siv_len = siv ? __bpf_dynptr_size(siv) : 0;
+	iv_len = iv ? __bpf_dynptr_size(iv) : 0;
 	src_len = __bpf_dynptr_size(src);
 	dst_len = __bpf_dynptr_size(dst);
-	if (!src_len || !dst_len || src_len > dst_len)
-		return -EINVAL;
-
-	if (siv_len != ctx->siv_len)
+	if (!src_len || !dst_len)
 		return -EINVAL;
 
 	psrc = __bpf_dynptr_data(src, src_len);
@@ -297,14 +252,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx,
 	if (!pdst)
 		return -EINVAL;
 
-	piv = siv_len ? __bpf_dynptr_data_rw(siv, siv_len) : NULL;
-	if (siv_len && !piv)
+	piv = iv_len ? __bpf_dynptr_data_rw(iv, iv_len) : NULL;
+	if (iv_len && !piv)
 		return -EINVAL;
 
-	err = decrypt ? ctx->type->decrypt(ctx->tfm, psrc, pdst, src_len, piv)
-		      : ctx->type->encrypt(ctx->tfm, psrc, pdst, src_len, piv);
-
-	return err;
+	switch (ctx->algo) {
+	case BPF_ALGO_AES_CBC:
+		return bpf_aes_cbc_crypt(pdst, dst_len, psrc, src_len, piv,
+					 iv_len, ctx, decrypt);
+	case BPF_ALGO_AES_ECB:
+		return bpf_aes_ecb_crypt(pdst, dst_len, psrc, src_len, piv,
+					 iv_len, ctx, decrypt);
+	default:
+		return -EINVAL;
+	}
 }
 
 /**
@@ -312,20 +273,20 @@ static int bpf_crypto_crypt(const struct bpf_crypto_ctx *ctx,
  * @ctx:		The crypto context being used. The ctx must be a trusted pointer.
  * @src:		bpf_dynptr to the encrypted data. Must be a trusted pointer.
  * @dst:		bpf_dynptr to the buffer where to store the result. Must be a trusted pointer.
- * @siv__nullable:	bpf_dynptr to IV data and state data to be used by decryptor. May be NULL.
+ * @iv__nullable:	bpf_dynptr to the initialization vector. May be NULL.
  *
  * Decrypts provided buffer using IV data and the crypto context. Crypto context must be configured.
  */
 __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx,
 				   const struct bpf_dynptr *src,
 				   const struct bpf_dynptr *dst,
-				   const struct bpf_dynptr *siv__nullable)
+				   const struct bpf_dynptr *iv__nullable)
 {
 	const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src;
 	const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst;
-	const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable;
+	const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable;
 
-	return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, true);
+	return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, true);
 }
 
 /**
@@ -333,20 +294,20 @@ __bpf_kfunc int bpf_crypto_decrypt(struct bpf_crypto_ctx *ctx,
  * @ctx:		The crypto context being used. The ctx must be a trusted pointer.
  * @src:		bpf_dynptr to the plain data. Must be a trusted pointer.
  * @dst:		bpf_dynptr to the buffer where to store the result. Must be a trusted pointer.
- * @siv__nullable:	bpf_dynptr to IV data and state data to be used by decryptor. May be NULL.
+ * @iv__nullable:	bpf_dynptr to the initialization vector. May be NULL.
  *
  * Encrypts provided buffer using IV data and the crypto context. Crypto context must be configured.
  */
 __bpf_kfunc int bpf_crypto_encrypt(struct bpf_crypto_ctx *ctx,
 				   const struct bpf_dynptr *src,
 				   const struct bpf_dynptr *dst,
-				   const struct bpf_dynptr *siv__nullable)
+				   const struct bpf_dynptr *iv__nullable)
 {
 	const struct bpf_dynptr_kern *src_kern = (struct bpf_dynptr_kern *)src;
 	const struct bpf_dynptr_kern *dst_kern = (struct bpf_dynptr_kern *)dst;
-	const struct bpf_dynptr_kern *siv_kern = (struct bpf_dynptr_kern *)siv__nullable;
+	const struct bpf_dynptr_kern *iv_kern = (struct bpf_dynptr_kern *)iv__nullable;
 
-	return bpf_crypto_crypt(ctx, src_kern, dst_kern, siv_kern, false);
+	return bpf_crypto_crypt(ctx, src_kern, dst_kern, iv_kern, false);
 }
 
 __bpf_kfunc_end_defs();
diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c
index 0abbbe177e317d..857b668212fb3a 100644
--- a/kernel/bpf/diagnostics.c
+++ b/kernel/bpf/diagnostics.c
@@ -794,8 +794,9 @@ static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx,
 	}
 }
 
-static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label,
-			    const char *fmt, ...)
+static __printf(4, 5) void bpf_diag_source(struct bpf_verifier_env *env,
+					   u32 insn_idx, const char *label,
+					   const char *fmt, ...)
 {
 	struct bpf_diag_scratch *scratch;
 	struct bpf_linfo_source *source_lines;
@@ -960,6 +961,37 @@ const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_t
 	}
 }
 
+const char *bpf_diag_arg_type_plain(enum bpf_arg_type type)
+{
+	switch (base_type(type)) {
+	case ARG_MEM_SIZE:
+	case ARG_CONST_MEM_SIZE:
+		return "an integer scalar length for this memory argument";
+	case ARG_PTR_TO_CTX:
+		return "the original program context pointer or preserve it before modifying registers";
+	case ARG_SCALAR:
+	case ARG_CONST_SCALAR:
+	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
+		return "an integer scalar value for this argument, not a pointer or resource object";
+	case ARG_PTR_TO_CONST_STR:
+		return "a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value";
+	case ARG_PTR_TO_DYNPTR:
+		return "the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path";
+	case ARG_PTR_TO_ALLOC_BTF_ID:
+		return "a pointer returned by the matching BPF object allocation path";
+	case ARG_PTR_TO_REFCOUNTED_KPTR:
+		return "an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field";
+	case ARG_PTR_TO_ITER:
+		return "the address of a stack iterator object for iterator new, next, and destroy calls";
+	case ARG_PTR_TO_IRQ_FLAG:
+		return "the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave()";
+	case ARG_PTR_TO_CTX_OUT:
+		return "the attach hook's own output argument, loaded directly from the program context";
+	default:
+		return "a value with one of the accepted pointer or scalar types for this call";
+	}
+}
+
 static const char *diag_arg_ordinal(int argno)
 {
 	switch (argno) {
@@ -1087,16 +1119,17 @@ void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx,
 	const char *constraint, *context;
 	u32 depth;
 
-	if (env->cur_state->active_rcu_locks)
-		ctx_kind = BPF_DIAG_CONTEXT_RCU;
-	else if (env->cur_state->active_preempt_locks)
-		ctx_kind = BPF_DIAG_CONTEXT_PREEMPT;
-	else if (env->cur_state->active_irq_id)
-		ctx_kind = BPF_DIAG_CONTEXT_IRQ;
-	else if (env->cur_state->active_locks)
-		ctx_kind = BPF_DIAG_CONTEXT_LOCK;
-	else
-		ctx_kind = BPF_DIAG_CONTEXT_NONE;
+	ctx_kind = BPF_DIAG_CONTEXT_NONE;
+	if (env->cur_state->in_sleepable) {
+		if (env->cur_state->active_rcu_locks)
+			ctx_kind = BPF_DIAG_CONTEXT_RCU;
+		else if (env->cur_state->active_preempt_locks)
+			ctx_kind = BPF_DIAG_CONTEXT_PREEMPT;
+		else if (env->cur_state->active_irq_id)
+			ctx_kind = BPF_DIAG_CONTEXT_IRQ;
+		else if (env->cur_state->active_locks)
+			ctx_kind = BPF_DIAG_CONTEXT_LOCK;
+	}
 
 	depth = diag_context_depth(env, ctx_kind);
 	opts = (struct bpf_diag_history_opts) {
@@ -1535,8 +1568,7 @@ static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx,
 	} else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) {
 		event.mod.origin_valid = true;
 	}
-	if (old_reg && new_reg &&
-	    (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) &&
+	if (old_reg && new_reg && reason == BPF_DIAG_MOD_WRITE &&
 	    !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) &&
 	    !event.mod.origin_valid &&
 	    diag_mod_keeps_lineage(env, &event))
@@ -1568,9 +1600,9 @@ static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env,
 			return NULL;
 		return &state->stack_arg_regs[target->stack_arg];
 	case BPF_DIAG_MOD_TARGET_STACK_SLOT:
-		if (target->spi >= state->allocated_stack / BPF_REG_SIZE)
+		if (target->spi >= bpf_stack_nr_slots(state))
 			return NULL;
-		return &state->stack[target->spi].spilled_ptr;
+		return &bpf_stack_slot(state, target->spi)->spilled_ptr;
 	default:
 		return NULL;
 	}
@@ -1586,7 +1618,7 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta
 	for (frame = 0; frame <= vstate->curframe; frame++) {
 		struct bpf_func_state *state = vstate->frame[frame];
 		unsigned long start, end;
-		u32 nslots = state->allocated_stack / BPF_REG_SIZE;
+		u32 nslots = bpf_stack_nr_slots(state);
 		int spi;
 
 		start = (unsigned long)state->regs;
@@ -1605,6 +1637,11 @@ static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_sta
 			return true;
 		}
 
+		/*
+		 * Map the pointer back to its slot by address, which relies on
+		 * the slots forming one contiguous array as bpf_stack_slot()
+		 * indexes it.
+		 */
 		start = (unsigned long)state->stack;
 		end = (unsigned long)(state->stack + nslots);
 		if (nslots && addr >= start && addr < end) {
diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h
index d1b79945008a89..a4102fb049ece4 100644
--- a/kernel/bpf/diagnostics.h
+++ b/kernel/bpf/diagnostics.h
@@ -51,6 +51,7 @@ const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list
 const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3);
 const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id);
 const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type);
+const char *bpf_diag_arg_type_plain(enum bpf_arg_type type);
 u64 bpf_diag_event_log_save(struct bpf_verifier_env *env);
 void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos);
 u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state);
diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c
index 3ce8d74b0e4009..36d3228d774545 100644
--- a/kernel/bpf/disasm.c
+++ b/kernel/bpf/disasm.c
@@ -350,7 +350,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
 		if (opcode == BPF_CALL) {
 			char tmp[64];
 
-			if (insn->src_reg == BPF_PSEUDO_CALL) {
+			if (BPF_SRC(insn->code) == BPF_X) {
+				verbose(cbs->private_data, "(%02x) callx r%d",
+					insn->code, insn->dst_reg);
+			} else if (insn->src_reg == BPF_PSEUDO_CALL) {
 				verbose(cbs->private_data, "(%02x) call pc%s",
 					insn->code,
 					__func_get_name(cbs, insn,
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 52d3cec336727f..37cf130ebb57b8 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -8,6 +8,7 @@
 #include <linux/bsearch.h>
 #include <linux/sort.h>
 #include <linux/perf_event.h>
+#include <linux/sched/signal.h>
 #include <net/xdp.h>
 #include "disasm.h"
 
@@ -213,12 +214,19 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env,
 }
 #endif
 
+enum bpf_patch_mode {
+	BPF_PATCH_KEEP_TARGET,	/* targets of 'off' keep naming 'off', the first new insn */
+	BPF_PATCH_MOVE_TARGET,	/* targets of 'off' follow the original insn to 'off + len - 1' */
+};
+
 /* single env->prog->insni[off] instruction was replaced with the range
  * insni[off, off + cnt).  Adjust corresponding insn_aux_data by copying
  * [0, off) and [off, end) to new locations, so the patched range stays zero
  */
 static void adjust_insn_aux_data(struct bpf_verifier_env *env,
-				 struct bpf_prog *new_prog, u32 off, u32 cnt)
+				 struct bpf_prog *new_prog, u32 off, u32 cnt,
+				 struct bpf_insn *original_insn,
+				 enum bpf_patch_mode mode)
 {
 	struct bpf_insn_aux_data *data = env->insn_aux_data;
 	struct bpf_insn *insn = new_prog->insnsi;
@@ -232,8 +240,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
 	 */
 	data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0;
 
-	if (cnt == 1)
+	if (cnt == 1) {
+		/*
+		 * A non-memory accessing insn could have been replaced by a
+		 * memory accessing insn, systematically mark it for non-stack
+		 * access
+		 */
+		data[off].non_stack_access = bpf_is_mem_insn(insn + off);
 		return;
+	}
 	prog_len = new_prog->len;
 	env->insn_aux_data_len = prog_len;
 
@@ -244,16 +259,39 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
 		/* Expand insni[off]'s seen count to the patched range. */
 		data[i].seen = old_seen;
 		data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0;
+		/*
+		 * In BPF_PATCH_MOVE_TARGET mode the original insn is the last
+		 * slot by construction.
+		 */
+		if (mode == BPF_PATCH_KEEP_TARGET &&
+		    !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) {
+			data[i].non_stack_access =
+				data[off + cnt - 1].non_stack_access;
+			data[off + cnt - 1].non_stack_access = false;
+		} else if (bpf_is_mem_insn(insn + i)) {
+			data[i].non_stack_access = true;
+		}
 	}
 
+	/*
+	 * Last slot instruction could be a newly generated
+	 * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access
+	 * if it is not the original instruction, otherwise keep the
+	 * original marking
+	 */
+	if (bpf_is_mem_insn(insn + off + cnt - 1) &&
+	    memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn)))
+		data[off + cnt - 1].non_stack_access = true;
+
 	/*
 	 * The indirect_target flag of the original instruction was moved to the last of the
 	 * new instructions by the above memmove and memset, but the indirect jump target is
 	 * actually the first instruction, so move it back. This also matches with the behavior
 	 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new
-	 * instruction.
+	 * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one,
+	 * so the flag already sits where needed.
 	 */
-	if (data[off + cnt - 1].indirect_target) {
+	if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) {
 		data[off].indirect_target = 1;
 		data[off + cnt - 1].indirect_target = 0;
 	}
@@ -273,7 +311,7 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len
 	}
 }
 
-static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)
+static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len)
 {
 	int i;
 
@@ -281,7 +319,7 @@ static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)
 		return;
 
 	for (i = 0; i < env->insn_array_map_cnt; i++)
-		bpf_insn_array_adjust(env->insn_array_maps[i], off, len);
+		bpf_insn_array_adjust(env->insn_array_maps[i], first, len);
 }
 
 static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)
@@ -292,7 +330,7 @@ static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 of
 		bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len);
 }
 
-static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
+static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len)
 {
 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
 	int i, sz = prog->aux->size_poke_tab;
@@ -300,17 +338,37 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
 
 	for (i = 0; i < sz; i++) {
 		desc = &tab[i];
-		if (desc->insn_idx <= off)
+		if (desc->insn_idx < first)
 			continue;
 		desc->insn_idx += len - 1;
 	}
 }
 
-struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
-				     const struct bpf_insn *patch, u32 len)
+/*
+ * Some post-verification instruction rewriting passes require an
+ * O(prog->len) operation per instruction. Keep their shared primitives
+ * killable and preemptible.
+ */
+static bool bpf_rewrite_must_abort(void)
+{
+	if (fatal_signal_pending(current))
+		return true;
+	cond_resched();
+	return false;
+}
+
+static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
+					      const struct bpf_insn *patch, u32 len,
+					      enum bpf_patch_mode mode)
 {
 	struct bpf_prog *new_prog;
 	struct bpf_insn_aux_data *new_data = NULL;
+	/* The first slot whose targets move. */
+	u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1;
+	struct bpf_insn original_insn;
+
+	if (bpf_rewrite_must_abort())
+		return NULL;
 
 	if (len > 1) {
 		new_data = vrealloc(env->insn_aux_data,
@@ -323,6 +381,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
 		env->insn_aux_data = new_data;
 	}
 
+	memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn));
 	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
 	if (IS_ERR(new_prog)) {
 		if (PTR_ERR(new_prog) == -ERANGE)
@@ -331,13 +390,58 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
 				env->insn_aux_data[off].orig_idx);
 		return NULL;
 	}
-	adjust_insn_aux_data(env, new_prog, off, len);
+	adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode);
 	adjust_subprog_starts(env, off, len);
-	adjust_insn_arrays(env, off, len);
-	adjust_poke_descs(new_prog, off, len);
+	adjust_insn_arrays(env, first, len);
+	bpf_adjust_func_ptrs(env, off, len);
+	adjust_poke_descs(new_prog, first, len);
 	return new_prog;
 }
 
+struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
+				     const struct bpf_insn *patch, u32 len)
+{
+	return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET);
+}
+
+/*
+ * insn was moved down by delta insns inside its own patch. Operands relative
+ * to the pc that point in front of the old position did not move with it.
+ */
+static int bpf_adj_moved_insn(struct bpf_insn *insn, u32 delta)
+{
+	u8 class = BPF_CLASS(insn->code), op = BPF_OP(insn->code);
+	s64 off = insn->imm, off_min = S32_MIN;
+	bool is_imm = true;
+
+	if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) {
+		/* subprog that started at the old position starts with the patch */
+		if (off >= 0)
+			return 0;
+	} else if ((class == BPF_JMP || class == BPF_JMP32) &&
+		   op != BPF_CALL && op != BPF_EXIT) {
+		if (insn->code != (BPF_JMP32 | BPF_JA)) {
+			off = insn->off;
+			off_min = S16_MIN;
+			is_imm = false;
+		}
+		/* jump to itself stays */
+		if (off >= -1)
+			return 0;
+	} else {
+		return 0;
+	}
+
+	off -= delta;
+	if (off < off_min)
+		return -ERANGE;
+	if (is_imm)
+		insn->imm = off;
+	else
+		insn->off = off;
+	return 0;
+}
+
 /*
  * For all jmp insns in a given 'prog' that point to 'tgt_idx' insn adjust the
  * jump offset by 'delta'.
@@ -397,7 +501,11 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
 
 	if (j > i) {
 		struct bpf_prog_aux *aux = env->prog->aux;
-		int move;
+		int move, k;
+
+		/* The removed subprogs own their jump tables. */
+		for (k = i; k < j; k++)
+			kvfree(env->subprog_info[k].jt);
 
 		/* move fake 'exit' subprog as well */
 		move = env->subprog_cnt + 1 - j;
@@ -405,6 +513,9 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
 		memmove(env->subprog_info + i,
 			env->subprog_info + j,
 			sizeof(*env->subprog_info) * move);
+		/* The vacated tail still holds copies of the moved entries. */
+		memset(env->subprog_info + i + move, 0,
+		       sizeof(*env->subprog_info) * (j - i));
 		env->subprog_cnt -= j - i;
 
 		/* remove func_info and its aux */
@@ -423,6 +534,9 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
 			 * in adjust_btf_func() - no need to adjust
 			 */
 		}
+		/* The exception callback is live, so it cannot sit inside the removed range. */
+		if (env->exception_callback_subprog >= j)
+			env->exception_callback_subprog -= j - i;
 	} else {
 		/* convert i from "first prog to remove" to "first to adjust" */
 		if (env->subprog_info[i].start == off)
@@ -500,38 +614,25 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
 	return 0;
 }
 
-/*
- * Clean up dynamically allocated fields of aux data for instructions [start, ...]
- */
-void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len)
-{
-	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
-	int end = start + len;
-	int i;
-
-	for (i = start; i < end; i++) {
-		if (aux_data[i].jt) {
-			kvfree(aux_data[i].jt);
-			aux_data[i].jt = NULL;
-		}
-	}
-}
-
 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
 {
 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
 	unsigned int orig_prog_len = env->prog->len;
 	int err;
 
+	if (bpf_rewrite_must_abort())
+		return -EINTR;
+
 	if (bpf_prog_is_offloaded(env->prog->aux))
 		bpf_prog_offload_remove_insns(env, off, cnt);
 
-	bpf_clear_insn_aux_data(env, off, cnt);
-
 	err = bpf_remove_insns(env->prog, off, cnt);
 	if (err)
 		return err;
 
+	/* before subprogs are adjusted, since it looks at them */
+	bpf_adjust_func_ptrs_after_remove(env, off, cnt);
+
 	err = adjust_subprog_starts_after_remove(env, off, cnt);
 	if (err)
 		return err;
@@ -592,12 +693,41 @@ void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
 	}
 }
 
+/*
+ * The address of a function might be taken by the code that is not dead,
+ * while the function itself is dead code, since nothing calls it. Keep the first
+ * instruction of such function, so that the address remains valid and differs
+ * from the addresses of other functions. Make it a trap like
+ * sanitize_dead_code() does.
+ */
+static void keep_funcs_with_addr_taken(struct bpf_verifier_env *env)
+{
+	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
+	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
+	struct bpf_insn *insn = env->prog->insnsi;
+	int insn_cnt = env->prog->len;
+	int i, t;
+
+	for (i = 0; i < insn_cnt; i++) {
+		if (!aux_data[i].seen || !bpf_pseudo_func(insn + i))
+			continue;
+		t = i + insn[i].imm + 1;
+		if (aux_data[t].seen)
+			continue;
+		memcpy(insn + t, &trap, sizeof(trap));
+		aux_data[t].zext_dst = false;
+		aux_data[t].seen = env->pass_cnt;
+	}
+}
+
 int bpf_opt_remove_dead_code(struct bpf_verifier_env *env)
 {
 	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
 	int insn_cnt = env->prog->len;
 	int i, err;
 
+	keep_funcs_with_addr_taken(env);
+
 	for (i = 0; i < insn_cnt; i++) {
 		int j;
 
@@ -618,6 +748,7 @@ int bpf_opt_remove_dead_code(struct bpf_verifier_env *env)
 
 int bpf_opt_remove_nops(struct bpf_verifier_env *env)
 {
+	struct bpf_insn_aux_data *aux = env->insn_aux_data;
 	struct bpf_insn *insn = env->prog->insnsi;
 	int insn_cnt = env->prog->len;
 	bool is_may_goto_0, is_ja;
@@ -629,6 +760,8 @@ int bpf_opt_remove_nops(struct bpf_verifier_env *env)
 
 		if (!is_may_goto_0 && !is_ja)
 			continue;
+		if (aux[i].indirect_target)
+			continue;
 
 		err = verifier_remove_insns(env, i, 1);
 		if (err)
@@ -771,7 +904,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
 			insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
 						      -subprogs[0].stack_depth);
 			insn_buf[cnt++] = env->prog->insnsi[0];
-			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
+			new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
+							 BPF_PATCH_MOVE_TARGET);
 			if (!new_prog)
 				return -ENOMEM;
 			env->prog = new_prog;
@@ -794,7 +928,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
 			verifier_bug(env, "prologue is too long");
 			return -EFAULT;
 		} else if (cnt) {
-			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
+			new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
+							 BPF_PATCH_MOVE_TARGET);
 			if (!new_prog)
 				return -ENOMEM;
 
@@ -825,7 +960,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
 			struct bpf_insn *patch = insn_buf;
 
 			*patch++ = BPF_ST_NOSPEC();
-			*patch++ = *insn;
+			*patch = *insn;
+			ret = bpf_adj_moved_insn(patch++, 1);
+			if (ret) {
+				verbose(env, "insn %d cannot be patched due to 16-bit range\n",
+					env->insn_aux_data[i + delta].orig_idx);
+				return ret;
+			}
 			cnt = patch - insn_buf;
 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 			if (!new_prog)
@@ -1068,6 +1209,59 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s
 	env->subprog_info[env->subprog_cnt].start = env->prog->len;
 }
 
+/*
+ * Replace the offsets of functions with their addresses in the maps of
+ * the program, see resolve_func_ptrs() in verifier.c. The program must be
+ * the only user of such map. From now on no other program can use it, see
+ * bpf_map_claim().
+ */
+static int resolve_func_ptrs(struct bpf_verifier_env *env, struct bpf_prog *prog,
+			     struct bpf_prog **func)
+{
+	unsigned long me = (unsigned long)prog->aux;
+	struct bpf_func_ptr *ptr;
+	int i, err, subprog;
+	u64 addr, new, *slot;
+
+	for (i = 0; i < env->func_ptr_cnt; i++) {
+		ptr = &env->func_ptrs[i];
+
+		/* pointers are sorted by map */
+		if ((!i || ptr->map != ptr[-1].map) &&
+		    cmpxchg(&ptr->map->user, me, me | BPF_MAP_USER_PATCHED) != me) {
+			verbose(env, "map '%s' is used by another program\n", ptr->map->name);
+			return -EBUSY;
+		}
+
+		/* it's the address of the value of the map whatever the offset is */
+		err = ptr->map->ops->map_direct_value_addr(ptr->map, &addr, 0);
+		if (verifier_bug_if(err, env, "no value of map '%s'", ptr->map->name))
+			return -EFAULT;
+		slot = (u64 *)(unsigned long)(addr + ptr->map_off);
+
+		/*
+		 * The function that is dead code is removed and the pointer is
+		 * NULL. The program may read it, but callx of it is dead code
+		 * too, see check_func_callx().
+		 */
+		new = 0;
+		if (ptr->xlated_off != BPF_FUNC_PTR_DELETED) {
+			subprog = bpf_find_subprog(env, ptr->xlated_off);
+			if (verifier_bug_if(!func || subprog <= 0, env, "no function at insn %u",
+					    ptr->xlated_off))
+				return -EFAULT;
+			new = (unsigned long)func[subprog]->bpf_func;
+		}
+
+		/* the map is frozen and nothing else uses it */
+		if (verifier_bug_if(*slot != (u64)ptr->orig_off * sizeof(struct bpf_insn), env,
+				    "map '%s' offset %u changed", ptr->map->name, ptr->map_off))
+			return -EFAULT;
+		WRITE_ONCE(*slot, new);
+	}
+	return 0;
+}
+
 static int jit_subprogs(struct bpf_verifier_env *env)
 {
 	struct bpf_prog *prog = env->prog, **func, *tmp;
@@ -1258,6 +1452,11 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 		cond_resched();
 	}
 
+	/* the addresses of all functions are final */
+	err = resolve_func_ptrs(env, prog, func);
+	if (err)
+		goto out_free;
+
 	/*
 	 * Cleanup func[i]->aux fields which aren't required
 	 * or can become invalid in future
@@ -1344,8 +1543,9 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 	struct bpf_prog *prog, *orig_prog;
 	u32 *orig_subprog_starts;
 
+	/* all functions that the maps point to are removed as dead code */
 	if (env->subprog_cnt <= 1)
-		return 0;
+		return resolve_func_ptrs(env, env->prog, NULL);
 
 	prog = orig_prog = env->prog;
 	if (bpf_prog_need_blind(prog)) {
@@ -1356,7 +1556,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
 		}
 		prog = bpf_jit_blind_constants(env, prog);
 		if (IS_ERR(prog)) {
-			err = -ENOMEM;
+			err = PTR_ERR(prog);
 			prog = orig_prog;
 			goto out_restore;
 		}
@@ -1433,7 +1633,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 		err = bpf_jit_subprogs(env);
 		if (err == 0)
 			return 0;
-		if (err == -EFAULT)
+		if (err == -EFAULT || err == -EINTR)
 			return err;
 	}
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
@@ -1502,11 +1702,37 @@ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *pat
 	env->prog = prog;
 	info[cnt + 1].start = info[cnt].start;
 	info[cnt].start = prog->len - len + 1;
+	/* The patch ends with the exit. */
+	info[cnt].exit_idx = prog->len - 1;
 	env->subprog_cnt++;
 	env->hidden_subprog_cnt++;
 	return 0;
 }
 
+/*
+ * Expand may_goto: load the count from the stack, jump to the target of
+ * may_goto when it is zero, then the tail that updates the count.
+ * Use gotol when the target is too far for 16-bit offset of a conditional jump.
+ */
+static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off,
+			   const struct bpf_insn *tail, int tail_cnt)
+{
+	int cnt = 0;
+
+	/* Forward jump has to step over the tail */
+	off = off >= 0 ? off + tail_cnt : off - 1;
+
+	insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off);
+	if (off == (s16)off) {
+		insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off);
+	} else {
+		insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1);
+		insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1);
+	}
+	memcpy(insn_buf + cnt, tail, tail_cnt * sizeof(*tail));
+	return cnt + tail_cnt;
+}
+
 /* Do various post-verification rewrites in a single program pass.
  * These rewrites simplify JIT and interpreter implementations.
  */
@@ -1784,6 +2010,18 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 
 		if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) {
 			int stack_off_cnt = -stack_depth - 16;
+			/*
+			 * AX is used as an argument to pass in stack_off_cnt
+			 * (to add to r10/fp), and also as the return value of
+			 * the call to arch_bpf_timed_may_goto.
+			 */
+			struct bpf_insn tail[] = {
+				BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1),
+				BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2),
+				BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt),
+				BPF_EMIT_CALL(arch_bpf_timed_may_goto),
+				BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt),
+			};
 
 			/*
 			 * Two 8 byte slots, depth-16 stores the count, and
@@ -1800,22 +2038,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 			 * after subtraction, rinse and repeat.
 			 */
 			stack_depth_extra = 16;
-			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt);
-			if (insn->off >= 0)
-				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5);
-			else
-				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
-			insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
-			insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2);
-			/*
-			 * AX is used as an argument to pass in stack_off_cnt
-			 * (to add to r10/fp), and also as the return value of
-			 * the call to arch_bpf_timed_may_goto.
-			 */
-			insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt);
-			insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto);
-			insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt);
-			cnt = 7;
+			cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt,
+					      tail, ARRAY_SIZE(tail));
 
 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 			if (!new_prog)
@@ -1827,16 +2051,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 			goto next_insn;
 		} else if (bpf_is_may_goto_insn(insn)) {
 			int stack_off = -stack_depth - 8;
+			struct bpf_insn tail[] = {
+				BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1),
+				BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off),
+			};
 
 			stack_depth_extra = 8;
-			insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off);
-			if (insn->off >= 0)
-				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2);
-			else
-				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
-			insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
-			insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off);
-			cnt = 4;
+			cnt = may_goto_expand(insn_buf, insn->off, stack_off,
+					      tail, ARRAY_SIZE(tail));
 
 			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 			if (!new_prog)
@@ -1993,7 +2215,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 			goto next_insn;
 		}
 
-		if (insn->imm == BPF_FUNC_timer_set_callback) {
+		aux = &env->insn_aux_data[i + delta];
+		if (aux->arg_prog) {
 			/* The verifier will process callback_fn as many times as necessary
 			 * with different maps and the register states prepared by
 			 * set_timer_callback_state will be accurate.
@@ -2008,7 +2231,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 			 *     bpf_timer_set_callback-ed will return -EINVAL.
 			 */
 			struct bpf_insn ld_addrs[2] = {
-				BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
+				BPF_LD_IMM64(aux->arg_prog, (long)prog->aux),
 			};
 
 			insn_buf[0] = ld_addrs[0];
@@ -2457,9 +2680,17 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
 						     BPF_MAX_LOOPS);
 		}
 		/* Copy first actual insn to preserve it */
-		insn_buf[cnt++] = env->prog->insnsi[subprog_start];
+		insn_buf[cnt] = env->prog->insnsi[subprog_start];
+		ret = bpf_adj_moved_insn(&insn_buf[cnt], cnt);
+		if (ret) {
+			verbose(env, "insn %d cannot be patched due to 16-bit range\n",
+				env->insn_aux_data[subprog_start].orig_idx);
+			return ret;
+		}
+		cnt++;
 
-		new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);
+		new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt,
+						 BPF_PATCH_MOVE_TARGET);
 		if (!new_prog)
 			return -ENOMEM;
 		env->prog = prog = new_prog;
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 6f331c80130d76..fb612f9aaca278 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -128,6 +128,7 @@ struct htab_elem {
 
 struct htab_btf_record {
 	struct btf_record *record;
+	struct btf *btf;
 	u32 key_size;
 };
 
@@ -497,8 +498,13 @@ static void htab_dtor_ctx_free(void *ctx)
 {
 	struct htab_btf_record *hrec = ctx;
 
+	/*
+	 * The duplicated record still points into the map BTF, so free it
+	 * before dropping the reference that keeps that BTF alive.
+	 */
 	btf_record_free(hrec->record);
-	kfree(ctx);
+	btf_put(hrec->btf);
+	kfree(hrec);
 }
 
 static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
@@ -521,6 +527,15 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
 		kfree(hrec);
 		return err;
 	}
+	/*
+	 * btf_record_dup() only acquires kernel and module BTF. Fields whose
+	 * types live in the map BTF keep pointing into it: kptrs to local
+	 * types refer to map->btf, and graph roots carry a value record owned
+	 * by its struct meta table. The context can outlive the map when the
+	 * allocator defers its teardown, so hold a reference of our own.
+	 */
+	hrec->btf = map->btf;
+	btf_get(hrec->btf);
 	bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec);
 	return 0;
 }
@@ -677,11 +692,18 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 	return ERR_PTR(err);
 }
 
-static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd)
+static __always_inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd)
 {
-	if (likely(key_len % 4 == 0))
+	const u32 *k = key;
+
+	if (key_len == sizeof(u32))
+		return jhash_1word(k[0], hashrnd);
+	else if (key_len == sizeof(u64))
+		return jhash_2words(k[0], k[1], hashrnd);
+	else if (likely(key_len % 4 == 0))
 		return jhash2(key, key_len / 4, hashrnd);
-	return jhash(key, key_len, hashrnd);
+	else
+		return jhash(key, key_len, hashrnd);
 }
 
 static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash)
@@ -712,9 +734,9 @@ static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash
  * the unlikely event when elements moved from one bucket into another
  * while link list is being walked
  */
-static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
-					       u32 hash, void *key,
-					       u32 key_size, u32 n_buckets)
+static __always_inline struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
+							       u32 hash, void *key,
+							       u32 key_size, u32 n_buckets)
 {
 	struct hlist_nulls_node *n;
 	struct htab_elem *l;
@@ -730,22 +752,15 @@ static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
 	return NULL;
 }
 
-/* Called from syscall or from eBPF program directly, so
- * arguments have to match bpf_map_lookup_elem() exactly.
- * The return value is adjusted by BPF instructions
- * in htab_map_gen_lookup().
- */
-static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
+static __always_inline void *__htab_lookup(struct bpf_map *map, void *key, u32 key_size)
 {
 	struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
 	struct hlist_nulls_head *head;
 	struct htab_elem *l;
-	u32 hash, key_size;
+	u32 hash;
 
 	WARN_ON_ONCE(!bpf_rcu_lock_held());
 
-	key_size = map->key_size;
-
 	hash = htab_map_hash(key, key_size, htab->hashrnd);
 
 	head = select_bucket(htab, hash);
@@ -755,6 +770,28 @@ static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
 	return l;
 }
 
+/*
+ * Called from syscall or from eBPF program directly, so
+ * arguments have to match bpf_map_lookup_elem() exactly.
+ * The return value is adjusted by BPF instructions
+ * in htab_map_gen_lookup(). u32/u64 specializations
+ * allow utilizing compile-time key_size for optimization.
+ */
+static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
+{
+	return __htab_lookup(map, key, map->key_size);
+}
+
+static void *__htab_map_lookup_elem_u32(struct bpf_map *map, void *key)
+{
+	return __htab_lookup(map, key, sizeof(u32));
+}
+
+static void *__htab_map_lookup_elem_u64(struct bpf_map *map, void *key)
+{
+	return __htab_lookup(map, key, sizeof(u64));
+}
+
 static void *htab_map_lookup_elem(struct bpf_map *map, void *key)
 {
 	struct htab_elem *l = __htab_map_lookup_elem(map, key);
@@ -783,7 +820,16 @@ static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 
 	BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem,
 		     (void *(*)(struct bpf_map *map, void *key))NULL));
-	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
+	BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u32,
+				  (void *(*)(struct bpf_map *map, void *key))NULL));
+	BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem_u64,
+				  (void *(*)(struct bpf_map *map, void *key))NULL));
+	if (map->key_size == sizeof(u32))
+		*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u32);
+	else if (map->key_size == sizeof(u64))
+		*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem_u64);
+	else
+		*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
 				offsetof(struct htab_elem, key) +
@@ -1054,14 +1100,17 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
 	/* When not setting the initial value on all cpus, zero-fill element
 	 * values for other cpus. Otherwise, bpf program has no way to ensure
 	 * known initial values for cpus other than current one
-	 * (onallcpus=false always when coming from bpf prog).
+	 * (onallcpus=false always when coming from bpf prog,
+	 *  map_flags & BPF_F_CPU when coming from syscall but setting
+	 *  only one cpu).
 	 */
-	if (!onallcpus) {
-		int current_cpu = raw_smp_processor_id();
+	if (!onallcpus || (map_flags & BPF_F_CPU)) {
+		int init_cpu = (map_flags & BPF_F_CPU) ? map_flags >> 32 :
+			       raw_smp_processor_id();
 		int cpu;
 
 		for_each_possible_cpu(cpu) {
-			if (cpu == current_cpu)
+			if (cpu == init_cpu)
 				copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value);
 			else /* Since elem is preallocated, we cannot touch special fields */
 				zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu));
@@ -1772,6 +1821,12 @@ static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map,
 						 flags);
 }
 
+/*
+ * Max consecutive empty buckets to walk in one RCU +
+ * instrumentation-disabled section before rescheduling.
+ */
+#define HTAB_BATCH_EMPTY_RESCHED 64
+
 static int
 __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 				   const union bpf_attr *attr,
@@ -1793,6 +1848,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 	unsigned long flags = 0;
 	bool locked = false;
 	struct htab_elem *l;
+	u32 empty_cnt = 0;
 	struct bucket *b;
 	int ret = 0;
 
@@ -1971,30 +2027,41 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
 	}
 
 next_batch:
-	/* If we are not copying data, we can go to next bucket and avoid
-	 * unlocking the rcu.
+	/*
+	 * If we are not copying data, we can go to next bucket and avoid
+	 * unlocking the rcu. Bound the walk though: after
+	 * HTAB_BATCH_EMPTY_RESCHED consecutive empty buckets, fully exit
+	 * the critical section (no locks are held here) and reschedule.
 	 */
 	if (!bucket_cnt && (batch + 1 < htab->n_buckets)) {
 		batch++;
-		goto again_nocopy;
+		if (++empty_cnt < HTAB_BATCH_EMPTY_RESCHED)
+			goto again_nocopy;
+		empty_cnt = 0;
+		rcu_read_unlock();
+		bpf_enable_instrumentation();
+		cond_resched_tasks_rcu_qs();
+		goto again;
 	}
 
 	rcu_read_unlock();
 	bpf_enable_instrumentation();
-	if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys,
-	    key_size * bucket_cnt) ||
-	    copy_to_user(uvalues + total * value_size, values,
-	    value_size * bucket_cnt))) {
+	if (bucket_cnt && (copy_to_user(ukeys + (size_t)total * key_size, keys,
+	    (size_t)key_size * bucket_cnt) ||
+	    copy_to_user(uvalues + (size_t)total * value_size, values,
+	    (size_t)value_size * bucket_cnt))) {
 		ret = -EFAULT;
 		goto after_loop;
 	}
 
 	total += bucket_cnt;
+	empty_cnt = 0;
 	batch++;
 	if (batch >= htab->n_buckets) {
 		ret = -ENOENT;
 		goto after_loop;
 	}
+	cond_resched_tasks_rcu_qs();
 	goto again;
 
 after_loop:
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index b3cc5c8fc87566..a284f20c97d5bd 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -679,9 +679,44 @@ const struct bpf_func_proto bpf_copy_from_user_proto = {
 	.arg3_type	= ARG_ANYTHING,
 };
 
+static int __bpf_copy_from_user_mm(void *dst, u32 size,
+				   const void __user *user_ptr,
+				   struct mm_struct *mm)
+{
+	int ret;
+
+	ret = access_remote_vm(mm, (unsigned long)user_ptr, dst, size, 0);
+	if (ret == size)
+		return 0;
+
+	memset(dst, 0, size);
+	/* Return -EFAULT for partial read */
+	return ret < 0 ? ret : -EFAULT;
+}
+
+static int __bpf_copy_from_user_mm_str(void *dst, u32 size,
+				       const void __user *user_ptr,
+				       struct mm_struct *mm, u64 flags)
+{
+	int ret;
+
+	ret = copy_remote_mm_str(mm, (unsigned long)user_ptr, dst, size, 0);
+	if (ret < 0) {
+		if (flags & BPF_F_PAD_ZEROS)
+			memset(dst, 0, size);
+		return ret;
+	}
+
+	if (flags & BPF_F_PAD_ZEROS)
+		memset(dst + ret, 0, size - ret);
+
+	return ret + 1;
+}
+
 BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size,
 	   const void __user *, user_ptr, struct task_struct *, tsk, u64, flags)
 {
+	struct mm_struct *mm;
 	int ret;
 
 	/* flags is not used yet */
@@ -691,13 +726,16 @@ BPF_CALL_5(bpf_copy_from_user_task, void *, dst, u32, size,
 	if (unlikely(!size))
 		return 0;
 
-	ret = access_process_vm(tsk, (unsigned long)user_ptr, dst, size, 0);
-	if (ret == size)
-		return 0;
+	mm = get_task_mm(tsk);
+	if (!mm) {
+		memset(dst, 0, size);
+		return -EFAULT;
+	}
 
-	memset(dst, 0, size);
-	/* Return -EFAULT for partial read */
-	return ret < 0 ? ret : -EFAULT;
+	ret = __bpf_copy_from_user_mm(dst, size, user_ptr, mm);
+	mmput(mm);
+
+	return ret;
 }
 
 const struct bpf_func_proto bpf_copy_from_user_task_proto = {
@@ -1092,7 +1130,7 @@ const struct bpf_func_proto bpf_snprintf_proto = {
 	.func		= bpf_snprintf,
 	.gpl_only	= true,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE,
+	.arg1_type	= ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE,
 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
 	.arg3_type	= ARG_PTR_TO_CONST_STR,
 	.arg4_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
@@ -1510,6 +1548,7 @@ static const struct bpf_func_proto bpf_timer_set_callback_proto = {
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_TIMER,
 	.arg2_type	= ARG_PTR_TO_FUNC,
+	.arg3_type	= ARG_PTR_TO_PROG_AUX,
 };
 
 static bool defer_timer_wq_op(void)
@@ -3658,6 +3697,92 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user
 	return ret + 1;
 }
 
+/**
+ * bpf_copy_from_user_mm() - Copy data from an address space
+ * @dst:             Destination address, in kernel space
+ * @dst__sz:         Number of bytes to copy
+ * @unsafe_ptr__ign: Source address in the address space
+ * @mm:              Address space to copy from
+ * @flags:           Reserved for future use; must be zero
+ *
+ * Copies data from the user address space associated with @mm. The destination
+ * is zeroed if an attempted copy cannot be completed in full. Unsupported
+ * flags return -EINVAL without modifying @dst.
+ *
+ * Return: 0 on success, -EINVAL if @flags is non-zero, or -EFAULT if @mm is no
+ * longer active or the copy fails or is partial.
+ */
+__bpf_kfunc int bpf_copy_from_user_mm(void *dst, u32 dst__sz,
+				      const void __user *unsafe_ptr__ign,
+				      struct mm_struct *mm, u64 flags)
+{
+	int ret;
+
+	if (unlikely(flags))
+		return -EINVAL;
+
+	if (unlikely(!dst__sz))
+		return 0;
+
+	if (unlikely(!mmget_not_zero(mm))) {
+		memset(dst, 0, dst__sz);
+		return -EFAULT;
+	}
+
+	ret = __bpf_copy_from_user_mm(dst, dst__sz, unsafe_ptr__ign, mm);
+	mmput(mm);
+
+	return ret;
+}
+
+/**
+ * bpf_copy_from_user_mm_str() - Copy a string from an address space
+ * @dst:             Destination address, in kernel space. This buffer must be
+ *                   at least @dst__sz bytes long
+ * @dst__sz:         Maximum number of bytes to copy, including the trailing NUL
+ * @unsafe_ptr__ign: Source address in the address space
+ * @mm:              Address space to copy from
+ * @flags:           The only supported flag is BPF_F_PAD_ZEROS
+ *
+ * Copies a NUL-terminated string from the user address space associated with
+ * @mm. If the string is too long, @dst is still NUL-terminated unless @dst__sz
+ * is zero.
+ *
+ * If the flags are valid and BPF_F_PAD_ZEROS is set, the unused portion of
+ * @dst is cleared on success and all of @dst is cleared on a copy failure.
+ * Unsupported flags return -EINVAL without modifying @dst.
+ *
+ * Return: The number of copied bytes including the NUL terminator on success,
+ * -EINVAL if @flags is invalid, or -EFAULT if @mm is no longer active or the
+ * copy fails.
+ */
+__bpf_kfunc int bpf_copy_from_user_mm_str(void *dst, u32 dst__sz,
+					  const void __user *unsafe_ptr__ign,
+					  struct mm_struct *mm, u64 flags)
+{
+	int ret;
+
+	if (unlikely(flags & ~BPF_F_PAD_ZEROS))
+		return -EINVAL;
+
+	if (unlikely(dst__sz == 0))
+		return 0;
+
+	if (unlikely(!mmget_not_zero(mm))) {
+		if (flags & BPF_F_PAD_ZEROS)
+			memset(dst, 0, dst__sz);
+		else
+			*(char *)dst = '\0';
+		return -EFAULT;
+	}
+
+	ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign,
+					  mm, flags);
+	mmput(mm);
+
+	return ret;
+}
+
 /**
  * bpf_copy_from_user_task_str() - Copy a string from an task's address space
  * @dst:             Destination address, in kernel space.  This buffer must be
@@ -3681,6 +3806,7 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz,
 					    const void __user *unsafe_ptr__ign,
 					    struct task_struct *tsk, u64 flags)
 {
+	struct mm_struct *mm;
 	int ret;
 
 	if (unlikely(flags & ~BPF_F_PAD_ZEROS))
@@ -3689,17 +3815,20 @@ __bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz,
 	if (unlikely(dst__sz == 0))
 		return 0;
 
-	ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0);
-	if (ret < 0) {
+	mm = get_task_mm(tsk);
+	if (!mm) {
 		if (flags & BPF_F_PAD_ZEROS)
 			memset(dst, 0, dst__sz);
-		return ret;
+		else
+			*(char *)dst = '\0';
+		return -EFAULT;
 	}
 
-	if (flags & BPF_F_PAD_ZEROS)
-		memset(dst + ret, 0, dst__sz - ret);
+	ret = __bpf_copy_from_user_mm_str(dst, dst__sz, unsafe_ptr__ign,
+					  mm, flags);
+	mmput(mm);
 
-	return ret + 1;
+	return ret;
 }
 
 /* Keep unsigned long in prototype so that kfunc is usable when emitted to
@@ -4676,6 +4805,106 @@ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct b
 	return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME);
 }
 
+typedef int (*bpf_rcu_callback_t)(struct bpf_map *map, void *key, void *value);
+
+/* Actual type for struct bpf_rcu_head */
+struct bpf_rcu_head_kern {
+	struct rcu_head rcu;
+	bpf_callback_t callback_fn;
+	struct bpf_map *map;
+	struct bpf_prog *prog;
+	u32 armed;
+} __aligned(8);
+
+static void bpf_rcu_run_callback(struct rcu_head *rcu)
+{
+	struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu);
+	bpf_callback_t callback_fn = rh->callback_fn;
+	struct bpf_prog *prog = rh->prog;
+	struct bpf_map *map = rh->map;
+	void *value, *key;
+	u32 idx;
+
+	value = (void *)rh - map->record->rcu_head_off;
+	key = map_key_from_value(map, value, &idx);
+
+	/* Pairs with the arming cmpxchg(): rh may be re-armed as soon as this store lands. */
+	smp_store_release(&rh->armed, 0);
+
+	rcu_read_lock_dont_migrate();
+	callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0);
+	rcu_read_unlock_migrate();
+
+	bpf_prog_put(prog);
+}
+
+static int __bpf_call_rcu(struct bpf_rcu_head *rh, struct bpf_map *map, void *callback,
+			  struct bpf_prog_aux *aux, bool trace)
+{
+	struct bpf_rcu_head_kern *rhk = (void *)rh;
+	struct bpf_prog *prog;
+
+	BUILD_BUG_ON(sizeof(struct bpf_rcu_head_kern) > sizeof(struct bpf_rcu_head));
+	BUILD_BUG_ON(__alignof__(struct bpf_rcu_head_kern) != __alignof__(struct bpf_rcu_head));
+	BTF_TYPE_EMIT(struct bpf_rcu_head);
+
+	/* A queued callback cannot be cancelled, so a self-rearming one would pin prog and map. */
+	if (!atomic64_read(&map->usercnt))
+		return -EPERM;
+
+	if (cmpxchg(&rhk->armed, 0, 1))
+		return -EBUSY;
+
+	prog = bpf_prog_inc_not_zero(aux->prog);
+	if (IS_ERR(prog)) {
+		WRITE_ONCE(rhk->armed, 0);
+		return -EBADF;
+	}
+
+	rhk->callback_fn = (bpf_callback_t)callback;
+	rhk->map = map;
+	rhk->prog = prog;
+	if (trace)
+		call_rcu_tasks_trace(&rhk->rcu, bpf_rcu_run_callback);
+	else
+		call_rcu(&rhk->rcu, bpf_rcu_run_callback);
+	return 0;
+}
+
+/**
+ * bpf_call_rcu - Invoke a BPF callback after an RCU grace period
+ * @rh: struct bpf_rcu_head in a BPF map value
+ * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values
+ * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh
+ * @aux: bpf_prog_aux of the caller, implicitly set by the verifier
+ *
+ * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process
+ * nor bpffs, or -EBADF if the calling program is going away.
+ */
+__bpf_kfunc int bpf_call_rcu(struct bpf_rcu_head *rh, void *map__const_map,
+			     bpf_rcu_callback_t callback, struct bpf_prog_aux *aux)
+{
+	return __bpf_call_rcu(rh, map__const_map, callback, aux, false);
+}
+
+/**
+ * bpf_call_rcu_tasks_trace - Invoke a BPF callback after an RCU tasks trace grace period
+ * @rh: struct bpf_rcu_head in a BPF map value
+ * @map__const_map: bpf_map that embeds struct bpf_rcu_head in the values
+ * @callback: BPF subprogram, invoked as callback(map, key, value) for the value holding @rh
+ * @aux: bpf_prog_aux of the caller, implicitly set by the verifier
+ *
+ * Waits for sleepable BPF programs too.  The callback itself is not sleepable either way.
+ *
+ * Return: 0, -EBUSY if @rh is already queued, -EPERM if @map is held by neither a process
+ * nor bpffs, or -EBADF if the calling program is going away.
+ */
+__bpf_kfunc int bpf_call_rcu_tasks_trace(struct bpf_rcu_head *rh, void *map__const_map,
+					 bpf_rcu_callback_t callback, struct bpf_prog_aux *aux)
+{
+	return __bpf_call_rcu(rh, map__const_map, callback, aux, true);
+}
+
 static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep,
 			    struct bpf_dynptr_kern *ptr)
 {
@@ -4883,7 +5112,7 @@ BTF_ID(func, bpf_cgroup_release_dtor)
 
 BTF_KFUNCS_START(common_btf_ids)
 BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx, KF_FASTCALL)
-BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL)
+BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL | KF_PERFMON)
 BTF_ID_FLAGS(func, bpf_rcu_read_lock)
 BTF_ID_FLAGS(func, bpf_rcu_read_unlock)
 BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL)
@@ -4920,26 +5149,28 @@ BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, bpf_wq_start)
 BTF_ID_FLAGS(func, bpf_preempt_disable)
 BTF_ID_FLAGS(func, bpf_preempt_enable)
-BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW)
+BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW | KF_PERFMON)
 BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)
-BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)
-BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE)
-BTF_ID_FLAGS(func, bpf_get_kmem_cache)
+BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_copy_from_user_mm, KF_SLEEPABLE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_copy_from_user_mm_str, KF_SLEEPABLE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
 BTF_ID_FLAGS(func, bpf_local_irq_save)
 BTF_ID_FLAGS(func, bpf_local_irq_restore)
 #ifdef CONFIG_BPF_EVENTS
-BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr)
-BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr)
-BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr)
-BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr)
-BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE)
-BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE)
-BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE)
-BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE)
+BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr, KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr, KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr, KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr, KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE | KF_PERFMON)
 #endif
 #ifdef CONFIG_DMA_SHARED_BUFFER
 BTF_ID_FLAGS(func, bpf_iter_dmabuf_new, KF_ITER_NEW | KF_SLEEPABLE)
@@ -4947,28 +5178,30 @@ BTF_ID_FLAGS(func, bpf_iter_dmabuf_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPAB
 BTF_ID_FLAGS(func, bpf_iter_dmabuf_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
 #endif
 BTF_ID_FLAGS(func, __bpf_trap)
-BTF_ID_FLAGS(func, bpf_strcmp);
-BTF_ID_FLAGS(func, bpf_strcasecmp);
-BTF_ID_FLAGS(func, bpf_strncasecmp);
-BTF_ID_FLAGS(func, bpf_strchr);
-BTF_ID_FLAGS(func, bpf_strchrnul);
-BTF_ID_FLAGS(func, bpf_strnchr);
-BTF_ID_FLAGS(func, bpf_strrchr);
-BTF_ID_FLAGS(func, bpf_strlen);
-BTF_ID_FLAGS(func, bpf_strnlen);
-BTF_ID_FLAGS(func, bpf_strspn);
-BTF_ID_FLAGS(func, bpf_strcspn);
-BTF_ID_FLAGS(func, bpf_strstr);
-BTF_ID_FLAGS(func, bpf_strcasestr);
-BTF_ID_FLAGS(func, bpf_strnstr);
-BTF_ID_FLAGS(func, bpf_strncasestr);
+BTF_ID_FLAGS(func, bpf_strcmp, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strcasecmp, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strncasecmp, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strchr, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strchrnul, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strnchr, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strrchr, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strlen, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strnlen, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strspn, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strcspn, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strstr, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strcasestr, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strnstr, KF_PERFMON);
+BTF_ID_FLAGS(func, bpf_strncasestr, KF_PERFMON);
 #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS)
 BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU)
 #endif
-BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE)
-BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE)
+BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON)
+BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON)
 BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, bpf_call_rcu, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, bpf_call_rcu_tasks_trace, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, bpf_dynptr_from_file)
 BTF_ID_FLAGS(func, bpf_dynptr_file_discard, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_timer_cancel_async)
diff --git a/kernel/bpf/inode.c b/kernel/bpf/inode.c
index 7837968c0842cd..c6f328e4752e1d 100644
--- a/kernel/bpf/inode.c
+++ b/kernel/bpf/inode.c
@@ -176,7 +176,7 @@ static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode,
 	inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
 }
 
-static struct dentry *bpf_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *bpf_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				struct dentry *dentry, umode_t mode)
 {
 	struct inode *inode;
@@ -424,7 +424,7 @@ bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags)
 	return simple_lookup(dir, dentry, flags);
 }
 
-static int bpf_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int bpf_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, const char *target)
 {
 	struct inode *inode;
@@ -874,7 +874,7 @@ enum {
 };
 
 static int bpf_fs_xattr_set(const struct xattr_handler *handler,
-			    struct mnt_idmap *idmap, struct dentry *unused,
+			    const struct mnt_idmap *idmap, struct dentry *unused,
 			    struct inode *inode, const char *name,
 			    const void *value, size_t size, int flags)
 {
diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c
new file mode 100644
index 00000000000000..bffc356839ac56
--- /dev/null
+++ b/kernel/bpf/keys.c
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2026 Isovalent */
+
+#include <linux/bpf.h>
+#include <linux/cred.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/key.h>
+#include <linux/moduleparam.h>
+#include <linux/slab.h>
+
+#undef MODULE_PARAM_PREFIX
+#define MODULE_PARAM_PREFIX "bpf."
+
+static struct key *bpf_keyring;
+
+static bool bpf_keyring_unsealed __ro_after_init;
+module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444);
+MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed");
+
+bool bpf_keyring_enforced(void)
+{
+	return bpf_keyring_unsealed;
+}
+
+struct bpf_key *bpf_lookup_keyring(void)
+{
+	struct bpf_key *bkey;
+
+	if (!bpf_keyring)
+		return NULL;
+	if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) ||
+	    !READ_ONCE(bpf_keyring->restrict_link))
+		return NULL;
+
+	bkey = kmalloc_obj(*bkey);
+	if (!bkey)
+		return NULL;
+
+	bkey->key = bpf_keyring;
+	bkey->has_ref = false;
+	return bkey;
+}
+
+static int __init bpf_keyring_init(void)
+{
+	struct key *keyring;
+
+	keyring = keyring_alloc(".bpf",
+				GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
+				current_cred(), KEY_POS_SEARCH |
+				KEY_USR_VIEW | KEY_USR_READ |
+				KEY_USR_WRITE | KEY_USR_SEARCH |
+				KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA,
+				NULL, NULL);
+	if (IS_ERR(keyring)) {
+		pr_err("bpf: cannot allocate bpf keyring: %ld\n",
+		       PTR_ERR(keyring));
+		return 0;
+	}
+	if (!bpf_keyring_unsealed &&
+	    keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) {
+		pr_err("bpf: cannot seal bpf keyring\n");
+		key_revoke(keyring);
+		key_put(keyring);
+		return 0;
+	}
+
+	bpf_keyring = keyring;
+	key_register_bpf_keyring(keyring);
+	return 0;
+}
+late_initcall(bpf_keyring_init);
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 301fc60bddc430..cd9523f69298ef 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -10,10 +10,39 @@
 
 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
 
-struct per_frame_masks {
-	spis_t may_read;	/* stack slots that may be read by this instruction */
-	spis_t must_write;	/* stack slots written by this instruction */
-	spis_t live_before;	/* stack slots that may be read by this insn and its successors */
+/*
+ * Stack liveness is tracked with a 4-byte (half register) granularity.
+ * Half-slot 0 covers [fp-4, fp), half-slot 1 covers [fp-8, fp-4), and so on,
+ * hence FRAME_HALF_SPIS - 1 is the deepest half-slot a frame can have.
+ */
+#define FRAME_HALF_SPIS		(MAX_BPF_STACK_JIT / BPF_HALF_REG_SIZE)
+#define FRAME_MAX_WORDS		BITS_TO_LONGS(FRAME_HALF_SPIS)
+
+/* Masks tracked for each instruction of a frame */
+enum {
+	FM_MAY_READ,	/* stack slots that may be read by this instruction */
+	FM_MUST_WRITE,	/* stack slots written by this instruction */
+	FM_LIVE_BEFORE,	/* stack slots that may be read by this insn and its successors */
+	FM_MASK_CNT,
+};
+
+/*
+ * Per instruction stack masks for one frame of a function instance.
+ *
+ * Most frames use only a fraction of the stack budget, so instead of masks
+ * wide enough for every half-slot of the largest possible frame, all masks
+ * of one array share the width @words, and the marking functions widen the
+ * array as deeper half-slots are recorded. Mask @kind of the instruction at
+ * relative index @i is at &bits[(i * FM_MASK_CNT + kind) * words]. A
+ * half-slot at or past @words * BITS_PER_LONG is never read by this frame,
+ * hence never live. An instruction that may read the whole frame, such as a
+ * call passing a frame pointer to another subprog, widens the array to the
+ * program's stack budget, the deepest an accepted program can reach, so
+ * that the read cannot lose half-slots to a later widening.
+ */
+struct frame_masks {
+	u32 words;
+	unsigned long bits[];
 };
 
 /*
@@ -29,7 +58,7 @@ struct func_instance {
 	u32 subprog_start;	/* cached env->subprog_info[subprog].start */
 	u32 insn_cnt;		/* cached number of insns in the function */
 	/* Per frame, per instruction masks, frames allocated lazily. */
-	struct per_frame_masks *frames[MAX_CALL_FRAMES];
+	struct frame_masks *frames[MAX_CALL_FRAMES];
 	bool must_write_initialized;
 };
 
@@ -151,50 +180,123 @@ static int relative_idx(struct func_instance *instance, u32 insn_idx)
 	return insn_idx - instance->subprog_start;
 }
 
-static struct per_frame_masks *get_frame_masks(struct func_instance *instance,
-					       u32 frame, u32 insn_idx)
+static u32 frame_mask_bits(struct frame_masks *fm)
 {
-	if (!instance->frames[frame])
-		return NULL;
+	return fm->words * BITS_PER_LONG;
+}
 
-	return &instance->frames[frame][relative_idx(instance, insn_idx)];
+static size_t frame_mask_words(struct func_instance *instance, u32 words)
+{
+	return (size_t)instance->insn_cnt * FM_MASK_CNT * words;
 }
 
-static struct per_frame_masks *alloc_frame_masks(struct func_instance *instance,
-						 u32 frame, u32 insn_idx)
+/* Mask @kind of the instruction at relative index @rel */
+static unsigned long *rel_mask(struct frame_masks *fm, u32 rel, u32 kind)
 {
-	struct per_frame_masks *arr;
+	return fm->bits + ((size_t)rel * FM_MASK_CNT + kind) * fm->words;
+}
 
-	if (!instance->frames[frame]) {
-		arr = kvzalloc_objs(*arr, instance->insn_cnt,
-				    GFP_KERNEL_ACCOUNT);
-		instance->frames[frame] = arr;
-		if (!arr)
-			return ERR_PTR(-ENOMEM);
+/*
+ * Make sure @frame has a mask array at least @words wide, allocating it or
+ * copying the existing masks into the wider stride as needed.
+ * @words must be in range [1, FRAME_MAX_WORDS].
+ */
+static struct frame_masks *widen_frame_masks(struct func_instance *instance,
+					     u32 frame, u32 words)
+{
+	struct frame_masks *old = instance->frames[frame], *new;
+	u32 i, kind;
+
+	if (old && old->words >= words)
+		return old;
+	new = kvzalloc_flex(*new, bits, frame_mask_words(instance, words), GFP_KERNEL_ACCOUNT);
+	if (!new)
+		return NULL;
+	new->words = words;
+	if (old) {
+		for (i = 0; i < instance->insn_cnt; i++)
+			for (kind = 0; kind < FM_MASK_CNT; kind++)
+				memcpy(rel_mask(new, i, kind), rel_mask(old, i, kind),
+				       old->words * sizeof(*old->bits));
+		kvfree(old);
 	}
-	return get_frame_masks(instance, frame, insn_idx);
+	instance->frames[frame] = new;
+	return new;
 }
 
-/* Accumulate may_read masks for @frame at @insn_idx */
-static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask)
+/*
+ * Set the inclusive half-slot range [lo, hi] in mask @kind of @frame at @insn_idx.
+ * An empty range, including one with a negative @hi as computed for a write
+ * that does not fully cover any half-slot, marks nothing.
+ */
+static int mark_stack_range(struct func_instance *instance, u32 frame, u32 insn_idx,
+			    u32 kind, s32 lo, s32 hi)
 {
-	struct per_frame_masks *masks;
+	struct frame_masks *fm;
 
-	masks = alloc_frame_masks(instance, frame, insn_idx);
-	if (IS_ERR(masks))
-		return PTR_ERR(masks);
-	masks->may_read = spis_or(masks->may_read, mask);
+	/*
+	 * An access past the frame bottom is rejected by the main verifier
+	 * pass later, liveness only has to avoid running off the masks.
+	 */
+	hi = min_t(s32, hi, FRAME_HALF_SPIS - 1);
+	if (lo > hi)
+		return 0;
+	fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(hi + 1));
+	if (!fm)
+		return -ENOMEM;
+	bitmap_set(rel_mask(fm, relative_idx(instance, insn_idx), kind), lo, hi - lo + 1);
 	return 0;
 }
 
-static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx, spis_t mask)
+/* Accumulate may_read for half-slots [lo, hi] of @frame at @insn_idx */
+static int mark_stack_read(struct func_instance *instance, u32 frame, u32 insn_idx,
+			   s32 lo, s32 hi)
+{
+	return mark_stack_range(instance, frame, insn_idx, FM_MAY_READ, lo, hi);
+}
+
+/* Accumulate must_write for half-slots [lo, hi] of @frame at @insn_idx */
+static int mark_stack_write(struct func_instance *instance, u32 frame, u32 insn_idx,
+			    s32 lo, s32 hi)
+{
+	return mark_stack_range(instance, frame, insn_idx, FM_MUST_WRITE, lo, hi);
+}
+
+/*
+ * Mark every half-slot of @frame as possibly read by @insn_idx. This widens
+ * the masks to the program's stack budget: a full read recorded at a narrower
+ * width would leave the bits added by a later widening clear and lose part of
+ * it. No verifier state holds a slot past the budget, so no liveness query
+ * reaches beyond it; an access past the budget in unreachable code may still
+ * widen a frame's masks further, which is harmless.
+ */
+static int mark_stack_read_all(struct bpf_verifier_env *env, struct func_instance *instance,
+			       u32 frame, u32 insn_idx)
+{
+	return mark_stack_read(instance, frame, insn_idx, 0,
+			       env->stack_limit / BPF_HALF_REG_SIZE - 1);
+}
+
+/* Accumulate @src, a mask @src_words wide, into may_read of @frame at @insn_idx */
+static int mark_stack_read_mask(struct func_instance *instance, u32 frame, u32 insn_idx,
+				const unsigned long *src, u32 src_words)
 {
-	struct per_frame_masks *masks;
+	u32 nbits = src_words * BITS_PER_LONG;
+	struct frame_masks *fm;
+	unsigned long *dst;
+	u32 last, w;
 
-	masks = alloc_frame_masks(instance, frame, insn_idx);
-	if (IS_ERR(masks))
-		return PTR_ERR(masks);
-	masks->must_write = spis_or(masks->must_write, mask);
+	last = find_last_bit(src, nbits);
+	if (last == nbits)
+		return 0;
+	fm = widen_frame_masks(instance, frame, BITS_TO_LONGS(last + 1));
+	if (!fm)
+		return -ENOMEM;
+	dst = rel_mask(fm, relative_idx(instance, insn_idx), FM_MAY_READ);
+	/* @src has no bits set past @last, hence none past @fm->words either */
+	src_words = min(src_words, fm->words);
+	for (w = 0; w < src_words; w++)
+		dst[w] |= src[w];
 	return 0;
 }
 
@@ -248,14 +350,18 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)
 	struct bpf_iarray *succ, *jt;
 	int insn_sz;
 
-	jt = env->insn_aux_data[idx].jt;
-	if (unlikely(jt))
-		return jt;
-
 	/* pre-allocated array of size up to 2; reset cnt, as it may have been used already */
 	succ = env->succ;
 	succ->cnt = 0;
 
+	/* All gotox of a subprogram share its jump table, see compute_subprog_jts() */
+	if (unlikely(insn_is_gotox(insn))) {
+		jt = bpf_find_containing_subprog(env, idx)->jt;
+		if (verifier_bug_if(!jt, env, "no jump table for gotox insn %u", idx))
+			return succ;
+		return jt;
+	}
+
 	opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)];
 	insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
 	if (opcode_info->can_fallthrough)
@@ -264,6 +370,20 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)
 	if (opcode_info->can_jump)
 		succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1;
 
+	/*
+	 * tail_call upon success and ld_{abs,ind} upon load failure return
+	 * from the subprogram, which is a hidden edge to its exit. A subprogram
+	 * without an exit insn has no exit to model.
+	 */
+	if (unlikely((bpf_helper_call(insn) && insn->imm == BPF_FUNC_tail_call) ||
+		     (BPF_CLASS(insn->code) == BPF_LD &&
+		      (BPF_MODE(insn->code) == BPF_ABS || BPF_MODE(insn->code) == BPF_IND)))) {
+		u32 exit_idx = bpf_find_containing_subprog(env, idx)->exit_idx;
+
+		if (exit_idx != U32_MAX)
+			succ->items[succ->cnt++] = exit_idx;
+	}
+
 	return succ;
 }
 
@@ -272,33 +392,42 @@ __diag_pop();
 static inline bool update_insn(struct bpf_verifier_env *env,
 			       struct func_instance *instance, u32 frame, u32 insn_idx)
 {
-	spis_t new_before, new_after;
-	struct per_frame_masks *insn, *succ_insn;
+	unsigned long new_after[FRAME_MAX_WORDS] = {};
+	unsigned long *may_read, *must_write, *live_before;
+	struct frame_masks *fm = instance->frames[frame];
+	u32 rel = relative_idx(instance, insn_idx);
 	struct bpf_iarray *succ;
-	u32 s;
-	bool changed;
+	bool changed = false;
+	u32 s, w;
 
 	succ = bpf_insn_successors(env, insn_idx);
 	if (succ->cnt == 0)
 		return false;
 
-	changed = false;
-	insn = get_frame_masks(instance, frame, insn_idx);
-	new_before = SPIS_ZERO;
-	new_after = SPIS_ZERO;
+	/* All instructions of one frame array share the same mask width */
 	for (s = 0; s < succ->cnt; ++s) {
-		succ_insn = get_frame_masks(instance, frame, succ->items[s]);
-		new_after = spis_or(new_after, succ_insn->live_before);
+		unsigned long *succ_live;
+
+		succ_live = rel_mask(fm, relative_idx(instance, succ->items[s]), FM_LIVE_BEFORE);
+		for (w = 0; w < fm->words; w++)
+			new_after[w] |= succ_live[w];
 	}
+	may_read = rel_mask(fm, rel, FM_MAY_READ);
+	must_write = rel_mask(fm, rel, FM_MUST_WRITE);
+	live_before = rel_mask(fm, rel, FM_LIVE_BEFORE);
 	/*
 	 * New "live_before" is a union of all "live_before" of successors
 	 * minus slots written by instruction plus slots read by instruction.
 	 * new_before = (new_after & ~insn->must_write) | insn->may_read
 	 */
-	new_before = spis_or(spis_and(new_after, spis_not(insn->must_write)),
-			     insn->may_read);
-	changed |= !spis_equal(new_before, insn->live_before);
-	insn->live_before = new_before;
+	for (w = 0; w < fm->words; w++) {
+		unsigned long new_before = (new_after[w] & ~must_write[w]) | may_read[w];
+
+		if (new_before != live_before[w]) {
+			live_before[w] = new_before;
+			changed = true;
+		}
+	}
 	return changed;
 }
 
@@ -329,10 +458,12 @@ static void update_instance(struct bpf_verifier_env *env, struct func_instance *
 
 static bool is_live_before(struct func_instance *instance, u32 insn_idx, u32 frameno, u32 half_spi)
 {
-	struct per_frame_masks *masks;
+	struct frame_masks *fm = instance->frames[frameno];
 
-	masks = get_frame_masks(instance, frameno, insn_idx);
-	return masks && spis_test_bit(masks->live_before, half_spi);
+	/* No recorded access reaches past the masks, so nothing there is live */
+	if (!fm || half_spi >= frame_mask_bits(fm))
+		return false;
+	return test_bit(half_spi, rel_mask(fm, relative_idx(instance, insn_idx), FM_LIVE_BEFORE));
 }
 
 int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
@@ -356,12 +487,29 @@ int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_
 	return 0;
 }
 
+/*
+ * Stack slots of outer frames that the callee may read are accounted as read
+ * by the @callsite insn itself, i.e. they are live before @callsite and may be
+ * dead after it, when:
+ * - @callsite calls a callback, which may be called several times;
+ * - @callsite is callx. Its callee is not known when stack liveness is
+ *   computed, so there is no func instance for it at @callsite. While
+ *   the verifier is in the callee lookup_instance() finds its standalone
+ *   instance that knows nothing about outer frames. What the callee may read
+ *   there is recorded by record_call_access().
+ */
+static bool callee_stack_access_at_callsite(struct bpf_verifier_env *env, u32 callsite)
+{
+	return bpf_calls_callback(env, callsite) ||
+	       bpf_is_callx(&env->prog->insnsi[callsite]);
+}
+
 bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_spi)
 {
 	/*
 	 * Slot is alive if it is read before q->insn_idx in current func instance,
 	 * or if for some outer func instance:
-	 * - alive before callsite if callsite calls callback, otherwise
+	 * - alive before callsite if callsite calls callback or is callx, otherwise
 	 * - alive after callsite
 	 */
 	struct live_stack_query *q = &env->liveness->live_stack_query;
@@ -394,7 +542,7 @@ bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 half_sp
 		/* Get callsite from verifier state, not from instance callchain */
 		callsite = q->callsites[i];
 
-		alive = bpf_calls_callback(env, callsite)
+		alive = callee_stack_access_at_callsite(env, callsite)
 			? is_live_before(instance, callsite, rel, half_spi)
 			: is_live_before(instance, callsite + 1, rel, half_spi);
 		if (alive)
@@ -430,17 +578,19 @@ static int spi_off(int spi)
  * When only one half is set, print as "-4h","-8h",...
  * Runs of 3+ consecutive fully-set SPIs are collapsed: "fp0-8..-24"
  */
-static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first, spis_t spis)
+static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first,
+			   const unsigned long *spis, u32 words)
 {
 	int buf_sz = sizeof(env->tmp_str_buf);
+	int spi_cnt = words * BITS_PER_LONG / 2;
 	char *buf = env->tmp_str_buf;
 	int spi, n, run_start;
 
 	buf[0] = '\0';
 
-	for (spi = 0; spi < STACK_SLOTS / 2 && buf_sz > 0; spi++) {
-		bool lo = spis_test_bit(spis, spi * 2);
-		bool hi = spis_test_bit(spis, spi * 2 + 1);
+	for (spi = 0; spi < spi_cnt && buf_sz > 0; spi++) {
+		bool lo = test_bit(spi * 2, spis);
+		bool hi = test_bit(spi * 2 + 1, spis);
 		const char *space = first ? "" : " ";
 
 		if (!lo && !hi)
@@ -450,16 +600,16 @@ static char *fmt_spis_mask(struct bpf_verifier_env *env, int frame, bool first,
 			/* half-spi */
 			n = scnprintf(buf, buf_sz, "%sfp%d%d%s",
 				      space, frame, spi_off(spi) + (lo ? STACK_SLOT_SZ : 0), "h");
-		} else if (spi + 2 < STACK_SLOTS / 2 &&
-			   spis_test_bit(spis, spi * 2 + 2) &&
-			   spis_test_bit(spis, spi * 2 + 3) &&
-			   spis_test_bit(spis, spi * 2 + 4) &&
-			   spis_test_bit(spis, spi * 2 + 5)) {
+		} else if (spi + 2 < spi_cnt &&
+			   test_bit(spi * 2 + 2, spis) &&
+			   test_bit(spi * 2 + 3, spis) &&
+			   test_bit(spi * 2 + 4, spis) &&
+			   test_bit(spi * 2 + 5, spis)) {
 			/* 3+ consecutive full spis */
 			run_start = spi;
-			while (spi + 1 < STACK_SLOTS / 2 &&
-			       spis_test_bit(spis, (spi + 1) * 2) &&
-			       spis_test_bit(spis, (spi + 1) * 2 + 1))
+			while (spi + 1 < spi_cnt &&
+			       test_bit((spi + 1) * 2, spis) &&
+			       test_bit((spi + 1) * 2 + 1, spis))
 				spi++;
 			n = scnprintf(buf, buf_sz, "%sfp%d%d..%d",
 				      space, frame, spi_off(run_start), spi_off(spi));
@@ -478,7 +628,8 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i
 {
 	int start = env->subprog_info[instance->subprog].start;
 	struct bpf_insn *insns = env->prog->insnsi;
-	struct per_frame_masks *masks;
+	struct frame_masks *fm;
+	unsigned long *mask;
 	int len = instance->insn_cnt;
 	int insn_idx, frame, i;
 	bool has_use, has_def;
@@ -501,10 +652,13 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i
 		pos = env->log.end_pos;
 		verbose(env, " use: ");
 		for (frame = instance->depth; frame >= 0; --frame) {
-			masks = get_frame_masks(instance, frame, insn_idx);
-			if (!masks || spis_is_zero(masks->may_read))
+			fm = instance->frames[frame];
+			if (!fm)
+				continue;
+			mask = rel_mask(fm, i, FM_MAY_READ);
+			if (bitmap_empty(mask, frame_mask_bits(fm)))
 				continue;
-			verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, masks->may_read));
+			verbose(env, "%s", fmt_spis_mask(env, frame, !has_use, mask, fm->words));
 			has_use = true;
 		}
 		if (!has_use)
@@ -512,10 +666,13 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i
 		pos = env->log.end_pos;
 		verbose(env, " def: ");
 		for (frame = instance->depth; frame >= 0; --frame) {
-			masks = get_frame_masks(instance, frame, insn_idx);
-			if (!masks || spis_is_zero(masks->must_write))
+			fm = instance->frames[frame];
+			if (!fm)
 				continue;
-			verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, masks->must_write));
+			mask = rel_mask(fm, i, FM_MUST_WRITE);
+			if (bitmap_empty(mask, frame_mask_bits(fm)))
+				continue;
+			verbose(env, "%s", fmt_spis_mask(env, frame, !has_def, mask, fm->words));
 			has_def = true;
 		}
 		if (!has_def)
@@ -584,9 +741,9 @@ static int print_instances(struct bpf_verifier_env *env)
  *   - same frame + different offset -> offset-imprecise
  *   - different frames          -> fully-imprecise (bitmask OR)
  *
- * At memory access sites (LDX/STX/ST), offset-imprecise marks only
- * the known frame's access mask as SPIS_ALL, while fully-imprecise
- * iterates bits in the bitmask and routes each frame to its target.
+ * At memory access sites (LDX/STX/ST), offset-imprecise marks the known
+ * frame as fully read, while fully-imprecise iterates bits in the bitmask
+ * and routes each frame to its target.
  */
 #define MAX_ARG_OFFSETS 4
 
@@ -605,8 +762,15 @@ enum arg_track_state {
 	ARG_IMPRECISE	= -3,	/* lost identity; .mask is arg bitmask */
 };
 
-/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */
-#define MAX_ARG_SPILL_SLOTS 64
+/*
+ * The tracker follows as many spill slots as a subprog addresses directly
+ * through R10, see subprog_spill_slots(). A frame's slots at a call site are
+ * kept in a snapshot for the callees that receive pointers into that frame.
+ */
+struct spill_snapshot {
+	u32 slots;
+	struct arg_track at[];
+};
 
 /*
  * Combined register + stack arg tracking: R0-R10 at indices 0-10,
@@ -881,13 +1045,60 @@ static void arg_padd(struct arg_track *at, s64 delta)
 }
 
 /*
- * Convert a byte offset from FP to a callee stack slot index.
+ * Slots the spill tracker may follow for a subprog of @len instructions
+ * without its per-instruction tables costing more than they could for the
+ * largest program while every frame stayed within MAX_BPF_STACK: as many
+ * entries as 64 slots need for BPF_COMPLEXITY_LIMIT_INSNS instructions.
+ */
+static u32 spill_slots_affordable(int len)
+{
+	u32 base = MAX_BPF_STACK / BPF_REG_SIZE;
+
+	return max_t(u32, base, base * BPF_COMPLEXITY_LIMIT_INSNS / len);
+}
+
+/*
+ * Number of 8-byte spill slots to track for the instructions in [@start, @end):
+ * the 64 slots of a MAX_BPF_STACK frame, which the tracker has always
+ * followed, or the deepest 8-byte stack access made directly through R10 when
+ * that reaches further, up to the program's stack budget. Spills and fills are
+ * compiled as direct R10 accesses, so the count covers them whatever the frame
+ * size; a fill through a derived pointer that reaches past it is treated as
+ * imprecise, as one past the frame always was.
+ */
+static u32 subprog_spill_slots(struct bpf_verifier_env *env, int start, int end)
+{
+	struct bpf_insn *insns = env->prog->insnsi;
+	int deepest = 0;
+
+	for (int i = start; i < end; i++) {
+		struct bpf_insn *insn = &insns[i];
+		u8 class = BPF_CLASS(insn->code);
+		u8 base;
+
+		if (BPF_SIZE(insn->code) != BPF_DW)
+			continue;
+		if (class == BPF_LDX)
+			base = insn->src_reg;
+		else if (class == BPF_STX || class == BPF_ST)
+			base = insn->dst_reg;
+		else
+			continue;
+		if (base == BPF_REG_FP && insn->off < 0)
+			deepest = max(deepest, -insn->off);
+	}
+	return clamp_t(u32, DIV_ROUND_UP(deepest, 8), MAX_BPF_STACK / BPF_REG_SIZE,
+		       env->stack_limit / BPF_REG_SIZE);
+}
+
+/*
+ * Convert a byte offset from FP to one of the @nslots tracked stack slots.
  * Returns -1 if out of range or not 8-byte aligned.
  * Slot 0 = fp-8, slot 1 = fp-16, ..., slot 7 = fp-64, ....
  */
-static int fp_off_to_slot(s16 off)
+static int fp_off_to_slot(s16 off, u32 nslots)
 {
-	if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8))
+	if (off >= 0 || off < -(int)(nslots * 8))
 		return -1;
 	if (off % 8)
 		return -1;
@@ -896,7 +1107,7 @@ static int fp_off_to_slot(s16 off)
 
 static struct arg_track fill_from_stack(struct bpf_insn *insn,
 					struct arg_track *at_out, int reg,
-					struct arg_track *at_stack_out,
+					struct arg_track *at_stack_out, u32 nslots,
 					int depth)
 {
 	struct arg_track imp = {
@@ -907,7 +1118,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn,
 	int cnt, i;
 
 	if (reg == BPF_REG_FP) {
-		int slot = fp_off_to_slot(insn->off);
+		int slot = fp_off_to_slot(insn->off, nslots);
 
 		return slot >= 0 ? at_stack_out[slot] : imp;
 	}
@@ -920,7 +1131,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn,
 
 		if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
 			return imp;
-		slot = fp_off_to_slot(fp_off);
+		slot = fp_off_to_slot(fp_off, nslots);
 		if (slot < 0)
 			return imp;
 		result = __arg_track_join(result, at_stack_out[slot]);
@@ -935,7 +1146,7 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn,
  * When exact offset is unknown conservatively add reg values to all slots in at_stack_out.
  */
 static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out,
-			   int reg, struct arg_track *at_stack_out,
+			   int reg, struct arg_track *at_stack_out, u32 nslots,
 			   struct arg_track *val, u32 sz)
 {
 	struct arg_track none = { .frame = ARG_NONE };
@@ -943,7 +1154,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out,
 	int cnt, i;
 
 	if (reg == BPF_REG_FP) {
-		int slot = fp_off_to_slot(insn->off);
+		int slot = fp_off_to_slot(insn->off, nslots);
 
 		if (slot >= 0)
 			at_stack_out[slot] = new_val;
@@ -951,7 +1162,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out,
 	}
 	cnt = at_out[reg].off_cnt;
 	if (cnt == 0) {
-		for (int slot = 0; slot < MAX_ARG_SPILL_SLOTS; slot++)
+		for (int slot = 0; slot < nslots; slot++)
 			at_stack_out[slot] = __arg_track_join(at_stack_out[slot], new_val);
 		return;
 	}
@@ -961,7 +1172,7 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out,
 
 		if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
 			continue;
-		slot = fp_off_to_slot(fp_off);
+		slot = fp_off_to_slot(fp_off, nslots);
 		if (slot < 0)
 			continue;
 		if (cnt == 1)
@@ -975,16 +1186,17 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out,
  * Clear all tracked callee stack slots overlapping the byte range
  * [off, off+sz-1] where off is a negative FP-relative offset.
  */
-static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt)
+static void clear_overlapping_stack_slots(struct arg_track *at_stack, u32 nslots, s16 off,
+					  u32 sz, int cnt)
 {
 	struct arg_track none = { .frame = ARG_NONE };
 
 	if (cnt == 0) {
-		for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
+		for (int i = 0; i < nslots; i++)
 			at_stack[i] = __arg_track_join(at_stack[i], none);
 		return;
 	}
-	for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
+	for (int i = 0; i < nslots; i++) {
 		int slot_start = -((i + 1) * 8);
 		int slot_end = slot_start + 8;
 
@@ -1002,33 +1214,33 @@ static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u
  */
 static void clear_stack_for_all_offs(struct bpf_insn *insn,
 				     struct arg_track *at_out, int reg,
-				     struct arg_track *at_stack_out, u32 sz)
+				     struct arg_track *at_stack_out, u32 nslots, u32 sz)
 {
 	int cnt, i;
 
 	if (reg == BPF_REG_FP) {
-		clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1);
+		clear_overlapping_stack_slots(at_stack_out, nslots, insn->off, sz, 1);
 		return;
 	}
 	cnt = at_out[reg].off_cnt;
 	if (cnt == 0) {
-		clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt);
+		clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, cnt);
 		return;
 	}
 	for (i = 0; i < cnt; i++) {
 		s16 fp_off;
 
 		if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) {
-			clear_overlapping_stack_slots(at_stack_out, 0, sz, 0);
+			clear_overlapping_stack_slots(at_stack_out, nslots, 0, sz, 0);
 			break;
 		}
-		clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt);
+		clear_overlapping_stack_slots(at_stack_out, nslots, fp_off, sz, cnt);
 	}
 }
 
 static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, int idx,
 			  struct arg_track *at_in, struct arg_track *at_stack_in,
-			  struct arg_track *at_out, struct arg_track *at_stack_out)
+			  struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots)
 {
 	bool printed = false;
 	int i;
@@ -1060,7 +1272,7 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i
 		verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]);
 		verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]);
 	}
-	for (i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
+	for (i = 0; i < nslots; i++) {
 		if (arg_track_eq(&at_stack_out[i], &at_stack_in[i]))
 			continue;
 		if (!printed) {
@@ -1088,7 +1300,7 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at)
  */
 static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			   int insn_idx,
-			   struct arg_track *at_out, struct arg_track *at_stack_out,
+			   struct arg_track *at_out, struct arg_track *at_stack_out, u32 nslots,
 			   const struct arg_track *at_stack_arg_entry,
 			   struct func_instance *instance,
 			   u32 *callsites)
@@ -1172,14 +1384,15 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		 * ARG_IMPRECISE (off_cnt == 0).
 		 */
 		if (src_is_local_fp && BPF_MODE(insn->code) == BPF_MEM && sz == 8) {
-			*dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, depth);
+			*dst = fill_from_stack(insn, at_out, insn->src_reg, at_stack_out, nslots,
+					       depth);
 		} else if (src->frame >= 0 && src->frame < depth &&
 			   BPF_MODE(insn->code) == BPF_MEM && sz == 8) {
-			struct arg_track *parent_stack =
+			struct spill_snapshot *parent =
 				env->callsite_at_stack[callsites[src->frame]];
 
 			*dst = fill_from_stack(insn, at_out, insn->src_reg,
-					       parent_stack, src->frame);
+					       parent->at, parent->slots, src->frame);
 		} else if (src->frame == ARG_IMPRECISE &&
 			   !(src->mask & BIT(depth)) && src->mask &&
 			   BPF_MODE(insn->code) == BPF_MEM && sz == 8) {
@@ -1201,12 +1414,12 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
 		if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM)
 			spill_to_stack(insn, at_out, insn->dst_reg,
-				       at_stack_out, src, sz);
+				       at_stack_out, nslots, src, sz);
 
 		if (BPF_MODE(insn->code) == BPF_ATOMIC) {
 			if (dst_is_local_fp && insn->imm != BPF_LOAD_ACQ)
 				clear_stack_for_all_offs(insn, at_out, insn->dst_reg,
-							 at_stack_out, sz);
+							 at_stack_out, nslots, sz);
 
 			r = bpf_atomic_load_reg(insn);
 			if (r >= 0)
@@ -1219,7 +1432,7 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		/* BPF_ST to FP-derived dst: clear overlapping stack slots */
 		if (dst_is_local_fp)
 			clear_stack_for_all_offs(insn, at_out, insn->dst_reg,
-						 at_stack_out, sz);
+						 at_stack_out, nslots, sz);
 	}
 }
 
@@ -1235,7 +1448,6 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off,
 				   s64 access_bytes, u32 frame, u32 insn_idx)
 {
 	s32 slot_hi, slot_lo;
-	spis_t mask;
 
 	if (fp_off >= 0)
 		/*
@@ -1247,27 +1459,19 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off,
 	if (access_bytes == S64_MIN) {
 		/* helper/kfunc read unknown amount of bytes from fp_off until fp+0 */
 		slot_hi = (-fp_off - 1) / STACK_SLOT_SZ;
-		mask = SPIS_ZERO;
-		spis_or_range(&mask, 0, slot_hi);
-		return mark_stack_read(instance, frame, insn_idx, mask);
+		return mark_stack_read(instance, frame, insn_idx, 0, slot_hi);
 	}
 	if (access_bytes > 0) {
 		/* Mark any touched slot as use */
 		slot_hi = (-fp_off - 1) / STACK_SLOT_SZ;
 		slot_lo = max_t(s32, (-fp_off - access_bytes) / STACK_SLOT_SZ, 0);
-		mask = SPIS_ZERO;
-		spis_or_range(&mask, slot_lo, slot_hi);
-		return mark_stack_read(instance, frame, insn_idx, mask);
+		return mark_stack_read(instance, frame, insn_idx, slot_lo, slot_hi);
 	} else if (access_bytes < 0) {
 		/* Mark only fully covered slots as def */
 		access_bytes = -access_bytes;
 		slot_hi = (-fp_off) / STACK_SLOT_SZ - 1;
 		slot_lo = max_t(s32, (-fp_off - access_bytes + STACK_SLOT_SZ - 1) / STACK_SLOT_SZ, 0);
-		if (slot_lo <= slot_hi) {
-			mask = SPIS_ZERO;
-			spis_or_range(&mask, slot_lo, slot_hi);
-			return mark_stack_write(instance, frame, insn_idx, mask);
-		}
+		return mark_stack_write(instance, frame, insn_idx, slot_lo, slot_hi);
 	}
 	return 0;
 }
@@ -1276,7 +1480,7 @@ static int record_stack_access_off(struct func_instance *instance, s64 fp_off,
  * 'arg' is FP-derived argument to helper/kfunc or load/store that
  * reads (positive) or writes (negative) 'access_bytes' into 'use' or 'def'.
  */
-static int record_stack_access(struct func_instance *instance,
+static int record_stack_access(struct bpf_verifier_env *env, struct func_instance *instance,
 			       const struct arg_track *arg,
 			       s64 access_bytes, u32 frame, u32 insn_idx)
 {
@@ -1286,7 +1490,7 @@ static int record_stack_access(struct func_instance *instance,
 		return 0;
 	if (arg->off_cnt == 0) {
 		if (access_bytes > 0 || access_bytes == S64_MIN)
-			return mark_stack_read(instance, frame, insn_idx, SPIS_ALL);
+			return mark_stack_read_all(env, instance, frame, insn_idx);
 		return 0;
 	}
 	if (access_bytes != S64_MIN && access_bytes < 0 && arg->off_cnt != 1)
@@ -1305,7 +1509,8 @@ static int record_stack_access(struct func_instance *instance,
  * When a pointer is ARG_IMPRECISE, conservatively mark every frame in
  * the bitmask as fully used.
  */
-static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_idx)
+static int record_imprecise(struct bpf_verifier_env *env, struct func_instance *instance,
+			    u32 mask, u32 insn_idx)
 {
 	int depth = instance->depth;
 	int f, err;
@@ -1314,7 +1519,7 @@ static int record_imprecise(struct func_instance *instance, u32 mask, u32 insn_i
 		if (!(mask & 1))
 			continue;
 		if (f <= depth) {
-			err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
+			err = mark_stack_read_all(env, instance, f, insn_idx);
 			if (err)
 				return err;
 		}
@@ -1383,9 +1588,9 @@ static int record_load_store_access(struct bpf_verifier_env *env,
 	}
 
 	if (ptr->frame >= 0 && ptr->frame <= depth)
-		return record_stack_access(instance, ptr, sz, ptr->frame, insn_idx);
+		return record_stack_access(env, instance, ptr, sz, ptr->frame, insn_idx);
 	if (ptr->frame == ARG_IMPRECISE)
-		return record_imprecise(instance, ptr->mask, insn_idx);
+		return record_imprecise(env, instance, ptr->mask, insn_idx);
 	/* ARG_NONE: not derived from any frame pointer, skip */
 	return 0;
 }
@@ -1410,7 +1615,7 @@ static int record_arg_access(struct bpf_verifier_env *env,
 		bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx);
 	} else {
 		for (int f = 0; f <= depth; f++) {
-			err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
+			err = mark_stack_read_all(env, instance, f, insn_idx);
 			if (err)
 				return err;
 		}
@@ -1420,9 +1625,9 @@ static int record_arg_access(struct bpf_verifier_env *env,
 		return 0;
 
 	if (frame >= 0 && frame <= depth)
-		err = record_stack_access(instance, at, bytes, frame, insn_idx);
+		err = record_stack_access(env, instance, at, bytes, frame, insn_idx);
 	else if (frame == ARG_IMPRECISE)
-		err = record_imprecise(instance, at->mask, insn_idx);
+		err = record_imprecise(env, instance, at->mask, insn_idx);
 	return err;
 }
 
@@ -1434,21 +1639,29 @@ static int record_call_access(struct bpf_verifier_env *env,
 {
 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
 	struct bpf_call_summary cs;
-	int r, err, num_params = 5;
+	int r, err, arg_slot_cnt = 5;
 
 	if (bpf_pseudo_call(insn))
 		return 0;
 
-	if (bpf_get_call_summary(env, insn, &cs))
-		num_params = cs.num_params;
+	if (bpf_is_callx(insn))
+		/*
+		 * The callee is not known statically. Assume that all arg
+		 * slots are passed and let record_arg_access() conservatively
+		 * mark the stack of all frames as read if any of them is
+		 * derived from a frame pointer.
+		 */
+		arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS + MAX_STACK_ARG_SLOTS;
+	else if (bpf_get_call_summary(env, insn, &cs))
+		arg_slot_cnt = cs.arg_slot_cnt;
 
-	for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) {
+	for (r = BPF_REG_1; r < BPF_REG_1 + min(arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); r++) {
 		err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx);
 		if (err)
 			return err;
 	}
 
-	for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) {
+	for (r = 0; r < MAX_STACK_ARG_SLOTS && r < arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; r++) {
 		err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r],
 					r + MAX_BPF_FUNC_REG_ARGS, insn_idx);
 		if (err)
@@ -1517,7 +1730,7 @@ struct subprog_at_info {
 static void print_subprog_arg_access(struct bpf_verifier_env *env,
 				     int subprog,
 				     struct subprog_at_info *info,
-				     struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS])
+				     struct arg_track *at_stack_in, u32 nslots)
 {
 	struct bpf_insn *insns = env->prog->insnsi;
 	int start = env->subprog_info[subprog].start;
@@ -1533,7 +1746,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env,
 		bool has_extra = false;
 		u8 cls = BPF_CLASS(insns[idx].code);
 		bool is_ldx_stx_call = cls == BPF_LDX || cls == BPF_STX ||
-				       insns[idx].code == (BPF_JMP | BPF_CALL);
+				       (cls == BPF_JMP && BPF_OP(insns[idx].code) == BPF_CALL);
 
 		verbose(env, "%3d: ", idx);
 		bpf_verbose_insn(env, &insns[idx]);
@@ -1550,8 +1763,8 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env,
 					has_extra = true;
 		}
 		if (is_ldx_stx_call) {
-			for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++)
-				if (arg_is_fp(&at_stack_in[i][r]))
+			for (r = 0; r < nslots; r++)
+				if (arg_is_fp(&at_stack_in[(size_t)i * nslots + r]))
 					has_extra = true;
 		}
 
@@ -1581,11 +1794,11 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env,
 		}
 
 		if (is_ldx_stx_call) {
-			for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++) {
-				if (!arg_is_fp(&at_stack_in[i][r]))
+			for (r = 0; r < nslots; r++) {
+				if (!arg_is_fp(&at_stack_in[(size_t)i * nslots + r]))
 					continue;
 				verbose(env, " fp%+d=", -(r + 1) * 8);
-				verbose_arg_track(env, &at_stack_in[i][r]);
+				verbose_arg_track(env, &at_stack_in[(size_t)i * nslots + r]);
 			}
 		}
 
@@ -1618,9 +1831,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 	int end = env->subprog_info[subprog + 1].start;
 	int po_end = env->subprog_info[subprog + 1].postorder_start;
 	int len = end - start;
+	u32 nslots = min(subprog_spill_slots(env, start, end), spill_slots_affordable(len));
 	struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL;
 	struct arg_track at_out[MAX_AT_TRACK_REGS];
-	struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS] = NULL;
+	struct arg_track *at_stack_in = NULL;
 	struct arg_track *at_stack_out = NULL;
 	struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS];
 	struct arg_track unvisited = { .frame = ARG_UNVISITED };
@@ -1632,19 +1846,19 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 	if (!at_in)
 		goto err_free;
 
-	at_stack_in = kvmalloc_objs(*at_stack_in, len, GFP_KERNEL_ACCOUNT);
+	at_stack_in = kvmalloc_objs(*at_stack_in, (size_t)len * nslots, GFP_KERNEL_ACCOUNT);
 	if (!at_stack_in)
 		goto err_free;
 
-	at_stack_out = kvmalloc_objs(*at_stack_out, MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT);
+	at_stack_out = kvmalloc_objs(*at_stack_out, nslots, GFP_KERNEL_ACCOUNT);
 	if (!at_stack_out)
 		goto err_free;
 
 	for (i = 0; i < len; i++) {
 		for (r = 0; r < MAX_AT_TRACK_REGS; r++)
 			at_in[i][r] = unvisited;
-		for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++)
-			at_stack_in[i][r] = unvisited;
+		for (r = 0; r < nslots; r++)
+			at_stack_in[(size_t)i * nslots + r] = unvisited;
 	}
 
 	for (r = 0; r < MAX_AT_TRACK_REGS; r++)
@@ -1660,8 +1874,8 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 	}
 
 	/* Entry: all stack slots are ARG_NONE */
-	for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++)
-		at_stack_in[0][r] = none;
+	for (r = 0; r < nslots; r++)
+		at_stack_in[r] = none;
 
 	/* Entry: incoming stack args from caller, or ARG_NONE for main */
 	for (r = 0; r < MAX_STACK_ARG_SLOTS; r++)
@@ -1683,11 +1897,12 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 			continue;
 
 		memcpy(at_out, at_in[i], sizeof(at_out));
-		memcpy(at_stack_out, at_stack_in[i], MAX_ARG_SPILL_SLOTS * sizeof(*at_stack_out));
+		memcpy(at_stack_out, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*at_stack_out));
 
-		arg_track_xfer(env, insn, idx, at_out, at_stack_out,
+		arg_track_xfer(env, insn, idx, at_out, at_stack_out, nslots,
 			       at_stack_arg_entry, instance, callsites);
-		arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out);
+		arg_track_log(env, insn, idx, at_in[i], &at_stack_in[(size_t)i * nslots], at_out,
+			      at_stack_out, nslots);
 
 		/* Propagate to successors within this subprogram */
 		succ = bpf_insn_successors(env, idx);
@@ -1704,9 +1919,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 				changed |= arg_track_join(env, idx, target, r,
 							  &at_in[ti][r], at_out[r]);
 
-			for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++)
+			for (r = 0; r < nslots; r++)
 				changed |= arg_track_join(env, idx, target, -r - 1,
-							  &at_stack_in[ti][r], at_stack_out[r]);
+							  &at_stack_in[(size_t)ti * nslots + r],
+							  at_stack_out[r]);
 		}
 	}
 	if (changed)
@@ -1722,30 +1938,31 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 		if (err)
 			goto err_free;
 
-		if (insn->code == (BPF_JMP | BPF_CALL)) {
+		if (BPF_CLASS(insn->code) == BPF_JMP && BPF_OP(insn->code) == BPF_CALL) {
 			err = record_call_access(env, instance, at_in[i], idx);
 			if (err)
 				goto err_free;
 		}
 
 		if (bpf_pseudo_call(insn) || bpf_calls_callback(env, idx)) {
+			struct spill_snapshot *snap;
+
 			kvfree(env->callsite_at_stack[idx]);
-			env->callsite_at_stack[idx] =
-				kvmalloc_objs(*env->callsite_at_stack[idx],
-					      MAX_ARG_SPILL_SLOTS, GFP_KERNEL_ACCOUNT);
-			if (!env->callsite_at_stack[idx]) {
+			env->callsite_at_stack[idx] = kvmalloc_flex(*snap, at, nslots, GFP_KERNEL_ACCOUNT);
+			snap = env->callsite_at_stack[idx];
+			if (!snap) {
 				err = -ENOMEM;
 				goto err_free;
 			}
-			memcpy(env->callsite_at_stack[idx],
-			       at_stack_in[i], sizeof(struct arg_track) * MAX_ARG_SPILL_SLOTS);
+			snap->slots = nslots;
+			memcpy(snap->at, &at_stack_in[(size_t)i * nslots], nslots * sizeof(*snap->at));
 		}
 	}
 
 	info->at_in = at_in;
 	at_in = NULL;
 	info->len = len;
-	print_subprog_arg_access(env, subprog, info, at_stack_in);
+	print_subprog_arg_access(env, subprog, info, at_stack_in, nslots);
 	err = 0;
 
 err_free:
@@ -1772,36 +1989,52 @@ static bool has_fp_args(struct arg_track *args)
  * may_read: union (any pass might read the slot).
  * must_write: intersection (only slots written on ALL passes are guaranteed).
  * live_before is recomputed by a subsequent update_instance() on @dst.
+ *
+ * The two instances may have settled on different mask widths for the same
+ * frame, so @dst is widened to cover @src first. A word only @dst has counts
+ * as zero on the @src side: it unions into may_read as a no-op and intersects
+ * must_write to empty.
  */
-static void merge_instances(struct func_instance *dst, struct func_instance *src)
+static int merge_instances(struct func_instance *dst, struct func_instance *src)
 {
-	int f, i;
+	struct frame_masks *d, *s;
+	u32 f, i, w;
 
 	for (f = 0; f <= dst->depth; f++) {
-		if (!src->frames[f]) {
+		s = src->frames[f];
+		d = dst->frames[f];
+		if (!s) {
 			/* This pass didn't touch frame f — must_write intersects with empty. */
-			if (dst->frames[f])
+			if (d)
 				for (i = 0; i < dst->insn_cnt; i++)
-					dst->frames[f][i].must_write = SPIS_ZERO;
+					bitmap_zero(rel_mask(d, i, FM_MUST_WRITE),
+						    frame_mask_bits(d));
 			continue;
 		}
-		if (!dst->frames[f]) {
+		if (!d) {
 			/* Previous pass didn't touch frame f — take src, zero must_write. */
-			dst->frames[f] = src->frames[f];
+			dst->frames[f] = s;
 			src->frames[f] = NULL;
 			for (i = 0; i < dst->insn_cnt; i++)
-				dst->frames[f][i].must_write = SPIS_ZERO;
+				bitmap_zero(rel_mask(s, i, FM_MUST_WRITE), frame_mask_bits(s));
 			continue;
 		}
+		d = widen_frame_masks(dst, f, s->words);
+		if (!d)
+			return -ENOMEM;
 		for (i = 0; i < dst->insn_cnt; i++) {
-			dst->frames[f][i].may_read =
-				spis_or(dst->frames[f][i].may_read,
-					src->frames[f][i].may_read);
-			dst->frames[f][i].must_write =
-				spis_and(dst->frames[f][i].must_write,
-					 src->frames[f][i].must_write);
+			unsigned long *dst_read = rel_mask(d, i, FM_MAY_READ);
+			unsigned long *dst_write = rel_mask(d, i, FM_MUST_WRITE);
+			unsigned long *src_read = rel_mask(s, i, FM_MAY_READ);
+			unsigned long *src_write = rel_mask(s, i, FM_MUST_WRITE);
+
+			for (w = 0; w < d->words; w++) {
+				dst_read[w] |= w < s->words ? src_read[w] : 0;
+				dst_write[w] &= w < s->words ? src_write[w] : 0;
+			}
 		}
 	}
+	return 0;
 }
 
 static struct func_instance *fresh_instance(struct func_instance *src)
@@ -1916,7 +2149,7 @@ static int analyze_subprog(struct bpf_verifier_env *env,
 				if (info[subprog].at_in[j][caller_reg].frame == ARG_NONE)
 					continue;
 				for (int f = 0; f <= depth; f++) {
-					err = mark_stack_read(instance, f, idx, SPIS_ALL);
+					err = mark_stack_read_all(env, instance, f, idx);
 					if (err)
 						goto out_free;
 				}
@@ -1955,13 +2188,18 @@ static int analyze_subprog(struct bpf_verifier_env *env,
 		/* Pull callee's entry liveness back to caller's callsite */
 		{
 			u32 callee_start = callee_instance->subprog_start;
-			struct per_frame_masks *entry;
+			struct frame_masks *callee_fm;
 
 			for (int f = 0; f < callee_instance->depth; f++) {
-				entry = get_frame_masks(callee_instance, f, callee_start);
-				if (!entry)
+				callee_fm = callee_instance->frames[f];
+				if (!callee_fm)
 					continue;
-				err = mark_stack_read(instance, f, idx, entry->live_before);
+				err = mark_stack_read_mask(instance, f, idx,
+							   rel_mask(callee_fm,
+								    relative_idx(callee_instance,
+										 callee_start),
+								    FM_LIVE_BEFORE),
+							   callee_fm->words);
 				if (err)
 					goto out_free;
 			}
@@ -1969,9 +2207,11 @@ static int analyze_subprog(struct bpf_verifier_env *env,
 	}
 
 	if (prev_instance) {
-		merge_instances(prev_instance, instance);
+		err = merge_instances(prev_instance, instance);
 		free_instance(instance);
 		instance = prev_instance;
+		if (err)
+			return err;
 	}
 	update_instance(env, instance);
 	return 0;
@@ -2060,7 +2300,8 @@ static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); }
 /* Compute info->{use,def} fields for the instruction */
 static void compute_insn_live_regs(struct bpf_verifier_env *env,
 				   struct bpf_insn *insn,
-				   struct insn_live_regs *info)
+				   struct insn_live_regs *info,
+				   bool ret_reg_pair)
 {
 	struct bpf_call_summary cs;
 	const u8 class = BPF_CLASS(insn->code);
@@ -2072,6 +2313,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
 	const u32 src32 = mask_lo(src);
 	const u32 dst32 = mask_lo(dst);
 	const u32 r0  = reg64_mask(0);
+	const u32 r2  = reg64_mask(BPF_REG_2);
 	u32 def = 0;
 	u32 use = U32_MAX;
 
@@ -2191,15 +2433,18 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
 			break;
 		case BPF_EXIT:
 			def = 0;
-			use = r0;
+			use = ret_reg_pair ? (r0 | r2) : r0;
 			break;
 		case BPF_CALL:
 			def = ALL_CALLER_SAVED_REGS;
 			use = def & ~BIT(BPF_REG_0);
 			if (bpf_get_call_summary(env, insn, &cs))
-				use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1);
+				use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1);
 			def = mask_widen(def);
 			use = mask_widen(use);
+			/* callx reads the address of the callee from dst_reg */
+			if (bpf_is_callx(insn))
+				use |= dst;
 			break;
 		default:
 			def = 0;
@@ -2228,8 +2473,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
 	struct insn_live_regs *state;
 	int insn_cnt = env->prog->len;
 	u64 pos, insn_pos;
-	int err = 0, i, j;
-	bool changed;
+	int err = 0, i, j, subprog, start, end;
+	bool changed, ret_reg_pair;
 
 	/* Use the following algorithm:
 	 * - define the following:
@@ -2256,8 +2501,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
 		goto out;
 	}
 
-	for (i = 0; i < insn_cnt; ++i)
-		compute_insn_live_regs(env, &insns[i], &state[i]);
+	for (subprog = 0; subprog < env->subprog_cnt; subprog++) {
+		start = env->subprog_info[subprog].start;
+		end = env->subprog_info[subprog + 1].start;
+		ret_reg_pair = bpf_ret_reg_pair(env, subprog);
+
+		for (i = start; i < end; ++i)
+			compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair);
+	}
 
 	/* Forward pass: resolve stack access through FP-derived pointers */
 	err = bpf_compute_subprog_arg_access(env);
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index fb032dfdc0dee4..d850a7863d2edf 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -716,7 +716,8 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
 		verbose(env, "=");
 		print_reg_state(env, state, reg);
 	}
-	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
+	for (i = 0; i < bpf_stack_nr_slots(state); i++) {
+		struct bpf_stack_state *slot = bpf_stack_slot(state, i);
 		char types_buf[BPF_REG_SIZE + 1];
 		const char *sep = "";
 		bool valid = false;
@@ -727,7 +728,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
 			continue;
 
 		for (j = 0; j < BPF_REG_SIZE; j++) {
-			slot_type = state->stack[i].slot_type[j];
+			slot_type = slot->slot_type[j];
 			if (slot_type != STACK_INVALID && slot_type != STACK_POISON)
 				valid = true;
 			types_buf[j] = slot_type_char[slot_type];
@@ -736,12 +737,12 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
 		if (!valid)
 			continue;
 
-		reg = &state->stack[i].spilled_ptr;
-		switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) {
+		reg = &slot->spilled_ptr;
+		switch (slot->slot_type[BPF_REG_SIZE - 1]) {
 		case STACK_SPILL:
 			/* print MISC/ZERO/INVALID slots above subreg spill */
 			for (j = 0; j < BPF_REG_SIZE; j++)
-				if (state->stack[i].slot_type[j] == STACK_SPILL)
+				if (slot->slot_type[j] == STACK_SPILL)
 					break;
 			types_buf[j] = '\0';
 
@@ -751,7 +752,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
 		case STACK_DYNPTR:
 			/* skip to main dynptr slot */
 			i += BPF_DYNPTR_NR_SLOTS - 1;
-			reg = &state->stack[i].spilled_ptr;
+			reg = &bpf_stack_slot(state, i)->spilled_ptr;
 
 			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
 			verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type));
diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index d2cbab4bdf644a..5ee8aae7435ba5 100644
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -25,6 +25,10 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
 	if (!inner_map->ops->map_meta_equal)
 		return ERR_PTR(-ENOTSUPP);
 
+	/* An inner map has no used_maps reference to hold it under a queued callback. */
+	if (btf_record_has_field(inner_map->record, BPF_RCU_HEAD))
+		return ERR_PTR(-EOPNOTSUPP);
+
 	inner_map_meta_size = sizeof(*inner_map_meta);
 	/* In some cases verifier needs to access beyond just base map. */
 	if (inner_map->ops == &array_map_ops || inner_map->ops == &percpu_array_map_ops)
diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c
index c19b360bad9eaa..2077d46d167c75 100644
--- a/kernel/bpf/map_iter.c
+++ b/kernel/bpf/map_iter.c
@@ -117,6 +117,12 @@ static int bpf_iter_attach_map(struct bpf_prog *prog,
 		goto put_map;
 	}
 
+	/* The value ctx arg is writable and aliases the live element. */
+	if (btf_record_has_field(map->record, BPF_RCU_HEAD)) {
+		err = -EOPNOTSUPP;
+		goto put_map;
+	}
+
 	if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 	    map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 	    map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
index e9662db7198fe0..8a8f088e83e6f9 100644
--- a/kernel/bpf/memalloc.c
+++ b/kernel/bpf/memalloc.c
@@ -119,6 +119,7 @@ struct bpf_mem_cache {
 	struct llist_head waiting_for_gp_ttrace;
 	struct rcu_head rcu_ttrace;
 	atomic_t call_rcu_ttrace_in_progress;
+	raw_spinlock_t lock;
 };
 
 struct bpf_mem_caches {
@@ -214,25 +215,24 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic)
 	gfp = __GFP_NOWARN | __GFP_ACCOUNT;
 	gfp |= atomic ? GFP_NOWAIT : GFP_KERNEL;
 
-	for (i = 0; i < cnt; i++) {
-		/*
-		 * For every 'c' llist_del_first(&c->free_by_rcu_ttrace); is
-		 * done only by one CPU == current CPU. Other CPUs might
-		 * llist_add() and llist_del_all() in parallel.
-		 */
-		obj = llist_del_first(&c->free_by_rcu_ttrace);
-		if (!obj)
-			break;
-		add_obj_to_free_list(c, obj);
-	}
-	if (i >= cnt)
-		return;
+	/*
+	 * c->lock serializes concurrent llist_del_first() against
+	 * llist_del_all() in __free_rcu() and do_call_rcu_ttrace().
+	 */
+	scoped_guard(raw_spinlock_irqsave, &c->lock) {
+		for (i = 0; i < cnt; i++) {
+			obj = llist_del_first(&c->free_by_rcu_ttrace);
+			if (!obj)
+				break;
+			add_obj_to_free_list(c, obj);
+		}
 
-	for (; i < cnt; i++) {
-		obj = llist_del_first(&c->waiting_for_gp_ttrace);
-		if (!obj)
-			break;
-		add_obj_to_free_list(c, obj);
+		for (; i < cnt; i++) {
+			obj = llist_del_first(&c->waiting_for_gp_ttrace);
+			if (!obj)
+				break;
+			add_obj_to_free_list(c, obj);
+		}
 	}
 	if (i >= cnt)
 		return;
@@ -279,8 +279,12 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per
 static void __free_rcu(struct rcu_head *head)
 {
 	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace);
+	struct llist_node *llnode;
+
+	scoped_guard(raw_spinlock_irqsave, &c->lock)
+		llnode = llist_del_all(&c->waiting_for_gp_ttrace);
 
-	free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size);
+	free_all(c, llnode, !!c->percpu_size);
 	atomic_set(&c->call_rcu_ttrace_in_progress, 0);
 }
 
@@ -300,7 +304,8 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c)
 
 	if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) {
 		if (unlikely(READ_ONCE(c->draining))) {
-			llnode = llist_del_all(&c->free_by_rcu_ttrace);
+			scoped_guard(raw_spinlock_irqsave, &c->lock)
+				llnode = llist_del_all(&c->free_by_rcu_ttrace);
 			free_all(c, llnode, !!c->percpu_size);
 		}
 		return;
@@ -535,6 +540,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
 			c->objcg = objcg;
 			c->percpu_size = percpu_size;
 			c->tgt = c;
+			raw_spin_lock_init(&c->lock);
 			init_refill_work(c);
 			prefill_mem_cache(c, cpu);
 		}
@@ -557,7 +563,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
 			c->objcg = objcg;
 			c->percpu_size = percpu_size;
 			c->tgt = c;
-
+			raw_spin_lock_init(&c->lock);
 			init_refill_work(c);
 			prefill_mem_cache(c, cpu);
 		}
@@ -609,7 +615,7 @@ int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size)
 		c->objcg = objcg;
 		c->percpu_size = percpu_size;
 		c->tgt = c;
-
+		raw_spin_lock_init(&c->lock);
 		init_refill_work(c);
 		prefill_mem_cache(c, cpu);
 	}
diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c
index 0d6f5569588c35..d855399812eec3 100644
--- a/kernel/bpf/offload.c
+++ b/kernel/bpf/offload.c
@@ -698,6 +698,8 @@ static bool __bpf_offload_dev_match(struct bpf_prog *prog,
 		return false;
 	if (offload->netdev == netdev)
 		return true;
+	if (!bpf_prog_is_offloaded(prog->aux))
+		return false;
 
 	ondev1 = bpf_offload_find_netdev(offload->netdev);
 	ondev2 = bpf_offload_find_netdev(netdev);
diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c
index 6e8b18c32a106a..1cee2ba6e332b7 100644
--- a/kernel/bpf/queue_stack_maps.c
+++ b/kernel/bpf/queue_stack_maps.c
@@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr)
 		 */
 		return -E2BIG;
 
+	if (attr->max_entries >= U32_MAX / attr->value_size)
+		return -E2BIG;
+
 	return 0;
 }
 
diff --git a/kernel/bpf/range_tree.c b/kernel/bpf/range_tree.c
index 2f28886f3ff757..2460e78d6a554e 100644
--- a/kernel/bpf/range_tree.c
+++ b/kernel/bpf/range_tree.c
@@ -143,16 +143,22 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len)
 		if (rn->rn_start < start && rn->rn_last > last) {
 			u32 old_last = rn->rn_last;
 
+			/*
+			 * Pre-allocate the right-half node before modifying
+			 * the tree. If allocation fails we return -ENOMEM
+			 * without altering the range tree.
+			 */
+			new_rn = kmalloc_nolock(sizeof(struct range_node),
+						__GFP_ACCOUNT, NUMA_NO_NODE);
+			if (!new_rn)
+				return -ENOMEM;
+
 			/* Overlaps with the entire clearing range */
 			range_it_remove(rn, rt);
 			rn->rn_last = start - 1;
 			range_it_insert(rn, rt);
 
-			/* Add a range */
-			new_rn = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT,
-						NUMA_NO_NODE);
-			if (!new_rn)
-				return -ENOMEM;
+			/* Add right-half range */
 			new_rn->rn_start = last + 1;
 			new_rn->rn_last = old_last;
 			range_it_insert(new_rn, rt);
@@ -193,6 +199,7 @@ int is_range_tree_set(struct range_tree *rt, u32 start, u32 len)
 int range_tree_set(struct range_tree *rt, u32 start, u32 len)
 {
 	u32 last = start + len - 1;
+	struct range_node *new_rn = NULL;
 	struct range_node *right;
 	struct range_node *left;
 	int err;
@@ -202,20 +209,19 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len)
 	if (left && left->rn_start <= start && left->rn_last >= last)
 		return 0;
 
+	left = range_it_iter_first(rt, start - 1, start - 1);
+	right = range_it_iter_first(rt, last + 1, last + 1);
+	if (!left && !right) {
+		new_rn = kmalloc_nolock(sizeof(struct range_node),
+					__GFP_ACCOUNT, NUMA_NO_NODE);
+		if (!new_rn)
+			return -ENOMEM;
+	}
+
 	/* Clear out everything in the range we want to set. */
 	err = range_tree_clear(rt, start, len);
 	if (err)
-		return err;
-
-	/* Do we have a left-adjacent range ? */
-	left = range_it_iter_first(rt, start - 1, start - 1);
-	if (left && left->rn_last + 1 != start)
-		return -EFAULT;
-
-	/* Do we have a right-adjacent range ? */
-	right = range_it_iter_first(rt, last + 1, last + 1);
-	if (right && right->rn_start != last + 1)
-		return -EFAULT;
+		goto out_free_new;
 
 	if (left && right) {
 		/* Combine left and right adjacent ranges */
@@ -235,14 +241,16 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len)
 		right->rn_start = start;
 		range_it_insert(right, rt);
 	} else {
-		left = kmalloc_nolock(sizeof(struct range_node), __GFP_ACCOUNT, NUMA_NO_NODE);
-		if (!left)
-			return -ENOMEM;
-		left->rn_start = start;
-		left->rn_last = last;
-		range_it_insert(left, rt);
+		/* No adjacent ranges; use the pre-allocated node */
+		new_rn->rn_start = start;
+		new_rn->rn_last = last;
+		range_it_insert(new_rn, rt);
 	}
 	return 0;
+
+out_free_new:
+	kfree_nolock(new_rn);
+	return err;
 }
 
 void range_tree_destroy(struct range_tree *rt)
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index d09d4c3fe547c6..f7e8d766d12834 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -272,13 +272,10 @@ struct stack_map_vma_lock {
 /*
  * Acquire a stable read-side reference on the VMA covering @ip.
  *
- * With CONFIG_PER_VMA_LOCK=y this returns a VMA with its per-VMA read
- * lock held and mmap_lock dropped, so the caller may sleep.
- *
- * With CONFIG_PER_VMA_LOCK=n it returns a VMA with mmap_lock still
- * held; the caller must snapshot any fields it needs and pin vm_file
- * with get_file() before stack_map_unlock_vma() drops mmap_lock, as
- * the VMA may be split, merged, or freed after that.
+ * On NOMMU configurations, returns with the mmap_lock held.  If the MMU
+ * is enabled, the per-VMA lock will be held instead.  The lock
+ * should be released with stack_map_unlock_vma() which will release the
+ * appropriate lock.  Once the lock is released, the VMA may be freed.
  *
  * Returns NULL on failure, in which case no lock is held.
  */
@@ -288,7 +285,6 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip)
 	struct mm_struct *mm = lock->mm;
 	struct vm_area_struct *vma;
 
-	/* noop under !CONFIG_PER_VMA_LOCK */
 	vma = lock_vma_under_rcu(mm, ip);
 	if (vma) {
 		lock->vma = vma;
@@ -308,21 +304,20 @@ stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip)
 		return NULL;
 	}
 
-#ifdef CONFIG_PER_VMA_LOCK
+#ifdef CONFIG_MMU
 	if (!vma_start_read_locked(vma)) {
 		mmap_read_unlock(mm);
 		return NULL;
 	}
 	mmap_read_unlock(mm);
 #endif
-
 	lock->vma = vma;
 	return vma;
 }
 
 static void stack_map_unlock_vma(struct stack_map_vma_lock *lock)
 {
-#ifdef CONFIG_PER_VMA_LOCK
+#ifdef CONFIG_MMU
 	vma_end_read(lock->vma);
 #else
 	mmap_read_unlock(lock->mm);
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 66fb11b6c6a765..18bf7b660c2f62 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -415,14 +415,14 @@ static void __clean_func_state(struct bpf_verifier_env *env,
 	 * half_spi 2*i   → lower half: slot_type[0..3] (closer to FP)
 	 * half_spi 2*i+1 → upper half: slot_type[4..7] (farther from FP)
 	 */
-	for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
+	for (i = 0; i < bpf_stack_nr_slots(st); i++) {
 		bool lo_live = bpf_stack_slot_alive(env, frame, i * 2);
 		bool hi_live = bpf_stack_slot_alive(env, frame, i * 2 + 1);
 
 		if (!hi_live || !lo_live) {
 			int start = !lo_live ? 0 : BPF_REG_SIZE / 2;
 			int end = !hi_live ? BPF_REG_SIZE : BPF_REG_SIZE / 2;
-			u8 stype = st->stack[i].slot_type[7];
+			u8 stype = bpf_stack_slot(st, i)->slot_type[7];
 
 			/*
 			 * Don't clear special slots.
@@ -442,7 +442,7 @@ static void __clean_func_state(struct bpf_verifier_env *env,
 			 * rejecting as non-scalar register fills.
 			 */
 			if (!hi_live) {
-				struct bpf_reg_state *spill = &st->stack[i].spilled_ptr;
+				struct bpf_reg_state *spill = &bpf_stack_slot(st, i)->spilled_ptr;
 
 				if (lo_live && stype == STACK_SPILL) {
 					if (spill->type != SCALAR_VALUE)
@@ -454,7 +454,7 @@ static void __clean_func_state(struct bpf_verifier_env *env,
 					if (bpf_register_is_null(spill))
 						continue;
 					for (j = 0; j < 4; j++) {
-						u8 *t = &st->stack[i].slot_type[j];
+						u8 *t = &bpf_stack_slot(st, i)->slot_type[j];
 
 						if (*t == STACK_SPILL)
 							*t = STACK_MISC;
@@ -463,7 +463,7 @@ static void __clean_func_state(struct bpf_verifier_env *env,
 				bpf_mark_reg_not_init(env, spill);
 			}
 			for (j = start; j < end; j++)
-				st->stack[i].slot_type[j] = STACK_POISON;
+				bpf_stack_slot(st, i)->slot_type[j] = STACK_POISON;
 		}
 	}
 }
@@ -491,7 +491,8 @@ static bool regs_exact(const struct bpf_reg_state *rold,
 {
 	return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
 	       check_ids(rold->id, rcur->id, idmap) &&
-	       check_ids(rold->parent_id, rcur->parent_id, idmap);
+	       check_ids(rold->parent_id, rcur->parent_id, idmap) &&
+	       check_ids(rold->map_uid, rcur->map_uid, idmap);
 }
 
 enum exact_level {
@@ -616,7 +617,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
 		       range_within(rold, rcur) &&
 		       tnum_in(rold->var_off, rcur->var_off) &&
 		       check_ids(rold->id, rcur->id, idmap) &&
-		       check_ids(rold->parent_id, rcur->parent_id, idmap);
+		       check_ids(rold->parent_id, rcur->parent_id, idmap) &&
+		       check_ids(rold->map_uid, rcur->map_uid, idmap);
 	case PTR_TO_PACKET_META:
 	case PTR_TO_PACKET:
 		/* We must have at least as much range as the old ptr
@@ -635,14 +637,14 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
 		/* id relations must be preserved */
 		if (!check_ids(rold->id, rcur->id, idmap))
 			return false;
+		/* Preserve displacements between pointers sharing an ID. */
+		if (rold->id && rold->r64.base != rcur->r64.base)
+			return false;
 		/* new val must satisfy old val knowledge */
 		return range_within(rold, rcur) &&
 		       tnum_in(rold->var_off, rcur->var_off);
 	case PTR_TO_STACK:
-		/* two stack pointers are equal only if they're pointing to
-		 * the same stack frame, since fp-8 in foo != fp-8 in bar
-		 */
-		return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno;
+		return regs_exact(rold, rcur, idmap);
 	case PTR_TO_ARENA:
 		return true;
 	case PTR_TO_INSN:
@@ -707,37 +709,38 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
 	 * didn't use them
 	 */
 	for (i = 0; i < old->allocated_stack; i++) {
+		struct bpf_stack_state *old_slot, *cur_slot;
 		struct bpf_reg_state *old_reg, *cur_reg;
 		int im = i % BPF_REG_SIZE;
+		u8 old_type;
 
 		spi = i / BPF_REG_SIZE;
+		old_slot = bpf_stack_slot(old, spi);
+		old_type = old_slot->slot_type[im];
+		cur_slot = i < cur->allocated_stack ? bpf_stack_slot(cur, spi) : NULL;
 
 		if (exact == EXACT) {
-			u8 old_type = old->stack[spi].slot_type[i % BPF_REG_SIZE];
-			u8 cur_type = i < cur->allocated_stack ?
-				      cur->stack[spi].slot_type[i % BPF_REG_SIZE] : STACK_INVALID;
+			u8 cur_type = cur_slot ? cur_slot->slot_type[im] : STACK_INVALID;
 
 			/* STACK_INVALID and STACK_POISON are equivalent for pruning */
 			if (old_type == STACK_POISON)
 				old_type = STACK_INVALID;
 			if (cur_type == STACK_POISON)
 				cur_type = STACK_INVALID;
-			if (i >= cur->allocated_stack || old_type != cur_type)
+			if (!cur_slot || old_type != cur_type)
 				return false;
 		}
 
-		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID ||
-		    old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_POISON)
+		if (old_type == STACK_INVALID || old_type == STACK_POISON)
 			continue;
 
-		if (env->allow_uninit_stack &&
-		    old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC)
+		if (env->allow_uninit_stack && old_type == STACK_MISC)
 			continue;
 
 		/* explored stack has more populated slots than current stack
 		 * and these slots were used
 		 */
-		if (i >= cur->allocated_stack)
+		if (!cur_slot)
 			return false;
 
 		/*
@@ -747,8 +750,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
 		 * regsafe() to ensure scalar ids are compared.
 		 */
 		if (im == 0 || im == 4) {
-			old_reg = scalar_reg_for_stack(env, &old->stack[spi], im);
-			cur_reg = scalar_reg_for_stack(env, &cur->stack[spi], im);
+			old_reg = scalar_reg_for_stack(env, old_slot, im);
+			cur_reg = scalar_reg_for_stack(env, cur_slot, im);
 			if (old_reg && cur_reg) {
 				if (!regsafe(env, old_reg, cur_reg, idmap, exact))
 					return false;
@@ -761,21 +764,19 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
 		 * it will be safe with zero-initialized stack.
 		 * The opposite is not true
 		 */
-		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
-		    cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
+		if (old_type == STACK_MISC && cur_slot->slot_type[im] == STACK_ZERO)
 			continue;
-		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
-		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
+		if (old_type != cur_slot->slot_type[im])
 			/* Ex: old explored (safe) state has STACK_SPILL in
 			 * this stack slot, but current has STACK_MISC ->
 			 * this verifier states are not equivalent,
 			 * return false to continue verification of this path
 			 */
 			return false;
-		if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1)
+		if (im != BPF_REG_SIZE - 1)
 			continue;
 		/* Both old and cur are having same slot_type */
-		switch (old->stack[spi].slot_type[BPF_REG_SIZE - 1]) {
+		switch (old_type) {
 		case STACK_SPILL:
 			/* when explored and current stack slot are both storing
 			 * spilled registers, check that stored pointers types
@@ -787,13 +788,13 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
 			 * such verifier states are not equivalent.
 			 * return false to continue verification of this path
 			 */
-			if (!regsafe(env, &old->stack[spi].spilled_ptr,
-				     &cur->stack[spi].spilled_ptr, idmap, exact))
+			if (!regsafe(env, &old_slot->spilled_ptr, &cur_slot->spilled_ptr,
+				     idmap, exact))
 				return false;
 			break;
 		case STACK_DYNPTR:
-			old_reg = &old->stack[spi].spilled_ptr;
-			cur_reg = &cur->stack[spi].spilled_ptr;
+			old_reg = &old_slot->spilled_ptr;
+			cur_reg = &cur_slot->spilled_ptr;
 			if (old_reg->dynptr.type != cur_reg->dynptr.type ||
 			    old_reg->dynptr.first_slot != cur_reg->dynptr.first_slot ||
 			    !check_ids(old_reg->id, cur_reg->id, idmap) ||
@@ -801,8 +802,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
 				return false;
 			break;
 		case STACK_ITER:
-			old_reg = &old->stack[spi].spilled_ptr;
-			cur_reg = &cur->stack[spi].spilled_ptr;
+			old_reg = &old_slot->spilled_ptr;
+			cur_reg = &cur_slot->spilled_ptr;
 			/* iter.depth is not compared between states as it
 			 * doesn't matter for correctness and would otherwise
 			 * prevent convergence; we maintain it only to prevent
@@ -818,8 +819,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
 				return false;
 			break;
 		case STACK_IRQ_FLAG:
-			old_reg = &old->stack[spi].spilled_ptr;
-			cur_reg = &cur->stack[spi].spilled_ptr;
+			old_reg = &old_slot->spilled_ptr;
+			cur_reg = &cur_slot->spilled_ptr;
 			if (!check_ids(old_reg->id, cur_reg->id, idmap) ||
 			    old_reg->irq.kfunc_class != cur_reg->irq.kfunc_class)
 				return false;
@@ -1043,10 +1044,12 @@ static int propagate_precision(struct bpf_verifier_env *env,
 			first = false;
 		}
 
-		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
-			if (!bpf_is_spilled_reg(&state->stack[i]))
+		for (i = 0; i < bpf_stack_nr_slots(state); i++) {
+			struct bpf_stack_state *ss = bpf_stack_slot(state, i);
+
+			if (!bpf_is_spilled_reg(ss))
 				continue;
-			state_reg = &state->stack[i].spilled_ptr;
+			state_reg = &ss->spilled_ptr;
 			if (state_reg->type != SCALAR_VALUE ||
 			    !state_reg->precise)
 				continue;
@@ -1121,7 +1124,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old,
 	fcur = cur->frame[fr];
 	for (i = 0; i < MAX_BPF_REG; i++)
 		if (memcmp(&fold->regs[i], &fcur->regs[i],
-			   offsetof(struct bpf_reg_state, frameno)))
+			   offsetof(struct bpf_reg_state, precise)))
 			return false;
 	return true;
 }
@@ -1192,15 +1195,17 @@ static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf
 
 	for (fr = old->curframe; fr >= 0; fr--) {
 		state = old->frame[fr];
-		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
-			if (state->stack[i].slot_type[0] != STACK_ITER)
+		for (i = 0; i < bpf_stack_nr_slots(state); i++) {
+			struct bpf_stack_state *ss = bpf_stack_slot(state, i);
+
+			if (ss->slot_type[0] != STACK_ITER)
 				continue;
 
-			slot = &state->stack[i].spilled_ptr;
+			slot = &ss->spilled_ptr;
 			if (slot->iter.state != BPF_ITER_STATE_ACTIVE)
 				continue;
 
-			cur_slot = &cur->frame[fr]->stack[i].spilled_ptr;
+			cur_slot = &bpf_stack_slot(cur->frame[fr], i)->spilled_ptr;
 			if (cur_slot->iter.depth != slot->iter.depth)
 				return true;
 		}
@@ -1222,10 +1227,12 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_
 				continue;
 			reg->precise = false;
 		}
-		for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
-			if (!bpf_is_spilled_reg(&func->stack[j]))
+		for (j = 0; j < bpf_stack_nr_slots(func); j++) {
+			struct bpf_stack_state *ss = bpf_stack_slot(func, j);
+
+			if (!bpf_is_spilled_reg(ss))
 				continue;
-			reg = &func->stack[j].spilled_ptr;
+			reg = &ss->spilled_ptr;
 			if (reg->type != SCALAR_VALUE)
 				continue;
 			reg->precise = false;
@@ -1273,10 +1280,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 			continue;
 
 		if (sl->state.branches) {
-			struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
+			struct bpf_func_state *frame = sl->state.frame[0];
 
 			if (frame->in_async_callback_fn &&
-			    frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) {
+			    frame->async_entry_cnt != cur->frame[0]->async_entry_cnt) {
 				/* Different async_entry_cnt means that the verifier is
 				 * processing another entry into async callback.
 				 * Seeing the same state is not an indication of infinite
@@ -1328,8 +1335,9 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 			 */
 			if (is_iter_next_insn(env, insn_idx)) {
 				if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
-					struct bpf_func_state *cur_frame;
+					struct bpf_func_state *cur_frame, *iter_frame, *old_frame;
 					struct bpf_reg_state *iter_state, *iter_reg;
+					struct bpf_reg_state *old_iter;
 					int spi;
 
 					cur_frame = cur->frame[cur->curframe];
@@ -1342,8 +1350,18 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 					 * no need for extra (re-)validations
 					 */
 					spi = bpf_get_spi(iter_reg->var_off.value);
-					iter_state = &bpf_func(env, iter_reg)->stack[spi].spilled_ptr;
-					if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
+					iter_frame = bpf_func(env, iter_reg);
+					iter_state = &bpf_stack_slot(iter_frame, spi)->spilled_ptr;
+					old_frame = sl->state.frame[iter_reg->frameno];
+					old_iter = &bpf_stack_slot(old_frame, spi)->spilled_ptr;
+					/*
+					 * states_equal() matches ids through idmap.
+					 * The loop converged only if it's the same
+					 * iterator. The one that was destroyed and
+					 * created again starts from the beginning.
+					 */
+					if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE &&
+					    iter_state->id == old_iter->id) {
 						loop = true;
 						goto hit;
 					}
@@ -1410,7 +1428,7 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 			 */
 			err = 0;
 			if (bpf_is_jmp_point(env, env->insn_idx))
-				err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0);
+				err = bpf_push_jmp_history(env, cur, 0, 0, 0, NULL, 0);
 			err = err ? : propagate_precision(env, &sl->state, cur, NULL);
 			if (err)
 				return err;
diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c
index be9ce98e946925..83e803ea54ef24 100644
--- a/kernel/bpf/stream.c
+++ b/kernel/bpf/stream.c
@@ -2,11 +2,15 @@
 /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
 
 #include <linux/bpf.h>
+#include <linux/anon_inodes.h>
 #include <linux/filter.h>
 #include <linux/bpf_mem_alloc.h>
 #include <linux/gfp.h>
+#include <linux/irq_work.h>
 #include <linux/memory.h>
 #include <linux/mutex.h>
+#include <linux/poll.h>
+#include <linux/refcount.h>
 
 static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len)
 {
@@ -22,11 +26,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len)
 	size_t alloc_size;
 
 	/*
-	 * Length denotes the amount of data to be written as part of stream element,
-	 * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can
-	 * accomodate, therefore deny allocations that won't fit into them.
+	 * Length is the payload pushed into the stream, excluding the
+	 * trailing NUL of the bprintf buffer. Reject anything that cannot
+	 * fit without copying that NUL into the stream element.
 	 */
-	if (len < 0 || len > max_len)
+	if (len < 0 || len >= max_len)
 		return NULL;
 
 	alloc_size = offsetof(struct bpf_stream_elem, str[len]);
@@ -68,25 +72,82 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len)
 	return 0;
 }
 
-static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem)
+static void bpf_stream_release_capacity(struct bpf_stream *stream, int len)
 {
-	int len = elem->total_len;
-
 	atomic_sub(len, &stream->capacity);
 }
 
+static void bpf_stream_notify(struct irq_work *work)
+{
+	struct bpf_stream *stream = container_of(work, struct bpf_stream, notify_work);
+
+	/*
+	 * Writers run in arbitrary program contexts, including NMI and regions
+	 * that already hold wait queue or epoll locks. Wake waiters from
+	 * irq_work instead, where taking those locks is safe.
+	 */
+	wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM);
+}
+
+static void bpf_stream_queue_notify(struct bpf_stream *stream)
+{
+	/*
+	 * Record that the work has been used so that teardown only pays for
+	 * irq_work_sync(), which may wait for an RCU grace period, when a
+	 * callback could actually be in flight.
+	 */
+	if (!READ_ONCE(stream->notify_used))
+		WRITE_ONCE(stream->notify_used, true);
+	irq_work_queue(&stream->notify_work);
+}
+
+static int bpf_stream_readable_bytes(struct bpf_stream *stream)
+{
+	return atomic_read_acquire(&stream->readable);
+}
+
+static void bpf_stream_publish(struct bpf_stream *stream, int len)
+{
+	/*
+	 * Pairs with atomic_read_acquire() in bpf_stream_readable_bytes().
+	 *
+	 * Notify only when the stream turns from empty to readable. Readers
+	 * block and pollers wait only after finding it empty, and a read never
+	 * takes the count below zero, so the publication that makes it positive
+	 * is the one they wait for. Publishing into a stream that already holds
+	 * data would only raise an interrupt for waiters that are being woken
+	 * already, or for nobody at all.
+	 */
+	if (atomic_add_return_release(len, &stream->readable) == len)
+		bpf_stream_queue_notify(stream);
+}
+
 static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len)
 {
-	int ret = bpf_stream_consume_capacity(stream, len);
+	int ret;
+
+	/* Nothing to publish; do not allocate an element for it. */
+	if (!len)
+		return 0;
+
+	ret = bpf_stream_consume_capacity(stream, len);
+	if (ret)
+		return ret;
+
+	ret = __bpf_stream_push_str(&stream->log, str, len);
+	if (ret)
+		bpf_stream_release_capacity(stream, len);
+	else
+		bpf_stream_publish(stream, len);
 
-	return ret ?: __bpf_stream_push_str(&stream->log, str, len);
+	return ret;
 }
 
 static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux)
 {
 	if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR)
 		return NULL;
-	return &aux->stream[stream_id - 1];
+	return aux->stream[stream_id - 1];
 }
 
 static void bpf_stream_free_elem(struct bpf_stream_elem *elem)
@@ -154,14 +215,17 @@ static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len)
 
 static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
 {
-	int rem_len = len, cons_len, ret = 0;
+	int read_len, rem_len, cons_len, ret = 0;
 	struct bpf_stream_elem *elem = NULL;
 	struct llist_node *node;
 
 	mutex_lock(&stream->lock);
+	read_len = min(len, bpf_stream_readable_bytes(stream));
+	rem_len = read_len;
 
 	while (rem_len) {
-		int pos = len - rem_len;
+		int pos = read_len - rem_len;
+		int chunk, n;
 		bool cont;
 
 		node = bpf_stream_backlog_peek(stream);
@@ -175,37 +239,146 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
 
 		cons_len = elem->consumed_len;
 		cont = bpf_stream_consume_elem(elem, &rem_len) == false;
-
-		ret = copy_to_user(buf + pos, elem->str + cons_len,
-				   elem->consumed_len - cons_len);
-		/* Restore in case of error. */
-		if (ret) {
-			ret = -EFAULT;
-			elem->consumed_len = cons_len;
+		chunk = elem->consumed_len - cons_len;
+
+		n = copy_to_user(buf + pos, elem->str + cons_len, chunk);
+		if (n) {
+			/* Keep any successfully copied bytes; -EFAULT only if none. */
+			elem->consumed_len -= n;
+			rem_len += n;
+			ret = (read_len == rem_len) ? -EFAULT : 0;
 			break;
 		}
 
 		if (cont)
 			continue;
 		bpf_stream_backlog_pop(stream);
-		bpf_stream_release_capacity(stream, elem);
+		bpf_stream_release_capacity(stream, elem->total_len);
 		bpf_stream_free_elem(elem);
 	}
 
+	atomic_sub(read_len - rem_len, &stream->readable);
 	mutex_unlock(&stream->lock);
-	return ret ? ret : len - rem_len;
+	return ret ? ret : read_len - rem_len;
 }
 
-int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len)
+int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len)
 {
 	struct bpf_stream *stream;
 
 	stream = bpf_stream_get(stream_id, prog->aux);
 	if (!stream)
 		return -ENOENT;
+	if (len > INT_MAX)
+		return -EINVAL;
 	return bpf_stream_read(stream, buf, len);
 }
 
+static bool bpf_stream_has_data(struct bpf_stream *stream)
+{
+	return bpf_stream_readable_bytes(stream) > 0;
+}
+
+static void bpf_stream_put(struct bpf_stream *stream)
+{
+	if (refcount_dec_and_test(&stream->refcnt)) {
+		struct llist_node *list;
+
+		/* Only a stream that ever queued its work can have a callback in flight. */
+		if (READ_ONCE(stream->notify_used))
+			irq_work_sync(&stream->notify_work);
+		list = llist_del_all(&stream->log);
+		bpf_stream_free_list(list);
+		bpf_stream_free_list(stream->backlog_head);
+		mutex_destroy(&stream->lock);
+		kfree(stream);
+	}
+}
+
+static int bpf_stream_release(struct inode *inode, struct file *file)
+{
+	bpf_stream_put(file->private_data);
+	return 0;
+}
+
+static ssize_t bpf_stream_file_read(struct file *file, char __user *buf, size_t len,
+				    loff_t *ppos)
+{
+	struct bpf_stream *stream = file->private_data;
+	bool dead;
+	int ret;
+
+	if (!len)
+		return 0;
+
+	for (;;) {
+		/*
+		 * Sample teardown state before looking for data. Nothing is
+		 * published once the program is gone, so finding the stream
+		 * empty after observing dead means EOF. The opposite order could
+		 * report EOF while data published just before teardown is still
+		 * buffered.
+		 */
+		dead = smp_load_acquire(&stream->dead);
+		ret = bpf_stream_read(stream, buf, len);
+		if (ret)
+			return ret;
+		if (dead)
+			return 0;
+		if (file->f_flags & O_NONBLOCK)
+			return -EAGAIN;
+
+		ret = wait_event_interruptible(stream->waitq,
+					       bpf_stream_has_data(stream) ||
+					       READ_ONCE(stream->dead));
+		if (ret)
+			return ret;
+	}
+}
+
+static __poll_t bpf_stream_poll(struct file *file, struct poll_table_struct *pts)
+{
+	struct bpf_stream *stream = file->private_data;
+	__poll_t events = 0;
+
+	/*
+	 * poll_wait() only registers the wait queue callback. Register before
+	 * checking persistent state so a concurrent publication or teardown is
+	 * observed either by the callback or by the checks below.
+	 */
+	poll_wait(file, &stream->waitq, pts);
+	if (bpf_stream_has_data(stream))
+		events |= EPOLLIN | EPOLLRDNORM;
+	if (READ_ONCE(stream->dead))
+		events |= EPOLLHUP;
+	return events;
+}
+
+static const struct file_operations bpf_stream_fops = {
+	.release = bpf_stream_release,
+	.read = bpf_stream_file_read,
+	.poll = bpf_stream_poll,
+};
+
+int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags)
+{
+	struct bpf_stream *stream;
+	int fd_flags = O_RDONLY | O_CLOEXEC;
+	int fd;
+
+	stream = bpf_stream_get(stream_id, prog->aux);
+	if (!stream)
+		return -ENOENT;
+	if (flags & BPF_F_STREAM_NONBLOCK)
+		fd_flags |= O_NONBLOCK;
+
+	refcount_inc(&stream->refcnt);
+	fd = anon_inode_getfd("bpf-stream", &bpf_stream_fops, stream, fd_flags);
+	if (fd < 0)
+		bpf_stream_put(stream);
+	return fd;
+}
+
 __bpf_kfunc_start_defs();
 
 /*
@@ -238,6 +411,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo
 		return ret;
 
 	ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args);
+	/* Truncation: reject before capacity charge (not -ENOMEM). */
+	if (ret >= MAX_BPRINTF_BUF) {
+		bpf_bprintf_cleanup(&data);
+		return -E2BIG;
+	}
 	/* Exclude NULL byte during push. */
 	ret = bpf_stream_push_str(stream, data.buf, ret);
 	bpf_bprintf_cleanup(&data);
@@ -268,28 +446,47 @@ __bpf_kfunc_end_defs();
 
 /* Added kfunc to common_btf_ids */
 
-void bpf_prog_stream_init(struct bpf_prog *prog)
+int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags)
 {
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
-		atomic_set(&prog->aux->stream[i].capacity, 0);
-		init_llist_head(&prog->aux->stream[i].log);
-		mutex_init(&prog->aux->stream[i].lock);
-		prog->aux->stream[i].backlog_head = NULL;
-		prog->aux->stream[i].backlog_tail = NULL;
+		struct bpf_stream *stream;
+
+		/* On failure, bpf_prog_stream_free() releases the streams allocated so far. */
+		stream = kzalloc_obj(*stream,
+				     bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags));
+		if (!stream)
+			return -ENOMEM;
+
+		refcount_set(&stream->refcnt, 1);
+		init_llist_head(&stream->log);
+		mutex_init(&stream->lock);
+		init_waitqueue_head(&stream->waitq);
+		init_irq_work(&stream->notify_work, bpf_stream_notify);
+		prog->aux->stream[i] = stream;
 	}
+	return 0;
 }
 
 void bpf_prog_stream_free(struct bpf_prog *prog)
 {
-	struct llist_node *list;
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
-		list = llist_del_all(&prog->aux->stream[i].log);
-		bpf_stream_free_list(list);
-		bpf_stream_free_list(prog->aux->stream[i].backlog_head);
+		struct bpf_stream *stream = prog->aux->stream[i];
+
+		if (!stream)
+			continue;
+		/*
+		 * Pairs with smp_load_acquire() in bpf_stream_file_read(): every
+		 * publication precedes the dead flag, so a reader that observes
+		 * it also observes all buffered data.
+		 */
+		smp_store_release(&stream->dead, true);
+		wake_up_interruptible_poll(&stream->waitq, EPOLLHUP);
+		bpf_stream_put(stream);
+		prog->aux->stream[i] = NULL;
 	}
 }
 
@@ -311,17 +508,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...)
 {
 	struct bpf_bprintf_buffers *buf;
 	va_list args;
-	int ret;
+	int len, ret;
 
 	if (bpf_try_get_buffers(&buf))
 		return -EBUSY;
 
 	va_start(args, fmt);
-	ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args);
+	len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args);
 	va_end(args);
-	ss->len += ret;
-	/* Exclude NULL byte during push. */
-	ret = __bpf_stream_push_str(&ss->log, buf->buf, ret);
+	/* Exclude NULL byte during push; skip empty output entirely. */
+	ret = len ? __bpf_stream_push_str(&ss->log, buf->buf, len) : 0;
+	if (!ret)
+		ss->len += len;
 	bpf_put_buffers();
 	return ret;
 }
@@ -351,6 +549,7 @@ int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
 		list = tail;
 	}
 	llist_add_batch(head, tail, &stream->log);
+	bpf_stream_publish(stream, ss->len);
 	return 0;
 }
 
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index c7bc9ba9b331f0..855c58dcf10a5e 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -687,6 +687,7 @@ void btf_record_free(struct btf_record *rec)
 		case BPF_REFCOUNT:
 		case BPF_WORKQUEUE:
 		case BPF_TASK_WORK:
+		case BPF_RCU_HEAD:
 			/* Nothing to release */
 			break;
 		default:
@@ -741,6 +742,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec)
 		case BPF_REFCOUNT:
 		case BPF_WORKQUEUE:
 		case BPF_TASK_WORK:
+		case BPF_RCU_HEAD:
 			/* Nothing to acquire */
 			break;
 		default:
@@ -874,6 +876,7 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
 		case BPF_LIST_NODE:
 		case BPF_RB_NODE:
 		case BPF_REFCOUNT:
+		case BPF_RCU_HEAD:
 			break;
 		default:
 			WARN_ON_ONCE(1);
@@ -954,6 +957,9 @@ void bpf_map_put(struct bpf_map *map)
 		/* bpf_map_free_id() must be called first */
 		bpf_map_free_id(map);
 
+		if (map->ops->map_free_pre_rcu)
+			map->ops->map_free_pre_rcu(map);
+
 		WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));
 		/* RCU tasks trace grace period implies RCU grace period. */
 		if (READ_ONCE(map->free_after_mult_rcu_gp))
@@ -1277,7 +1283,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
 	map->record = btf_parse_fields(btf, value_type,
 				       BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
 				       BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR |
-				       BPF_TASK_WORK,
+				       BPF_TASK_WORK | BPF_RCU_HEAD,
 				       map->value_size);
 	if (!IS_ERR_OR_NULL(map->record)) {
 		int i;
@@ -1319,6 +1325,12 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
 					goto free_map_tab;
 				}
 				break;
+			case BPF_RCU_HEAD:
+				if (map->map_type != BPF_MAP_TYPE_ARRAY) {
+					ret = -EOPNOTSUPP;
+					goto free_map_tab;
+				}
+				break;
 			case BPF_KPTR_UNREF:
 			case BPF_KPTR_REF:
 			case BPF_KPTR_PERCPU:
@@ -2036,7 +2048,7 @@ int generic_map_delete_batch(struct bpf_map *map,
 
 	for (cp = 0; cp < max_count; cp++) {
 		err = -EFAULT;
-		if (copy_from_user(key, keys + cp * map->key_size,
+		if (copy_from_user(key, keys + (size_t)cp * map->key_size,
 				   map->key_size))
 			break;
 
@@ -2098,9 +2110,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
 
 	for (cp = 0; cp < max_count; cp++) {
 		err = -EFAULT;
-		if (copy_from_user(key, keys + cp * map->key_size,
+		if (copy_from_user(key, keys + (size_t)cp * map->key_size,
 		    map->key_size) ||
-		    copy_from_user(value, values + cp * value_size, value_size))
+		    copy_from_user(value, values + (size_t)cp * value_size, value_size))
 			break;
 
 		err = bpf_map_update_value(map, map_file, key, value,
@@ -2179,12 +2191,12 @@ int generic_map_lookup_batch(struct bpf_map *map,
 		if (err)
 			goto free_buf;
 
-		if (copy_to_user(keys + cp * map->key_size, key,
+		if (copy_to_user(keys + (size_t)cp * map->key_size, key,
 				 map->key_size)) {
 			err = -EFAULT;
 			goto free_buf;
 		}
-		if (copy_to_user(values + cp * value_size, value, value_size)) {
+		if (copy_to_user(values + (size_t)cp * value_size, value, value_size)) {
 			err = -EFAULT;
 			goto free_buf;
 		}
@@ -2441,6 +2453,7 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu)
 {
 	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
 
+	btf_put(aux->btf);
 	kvfree(aux->func_info);
 	kfree(aux->func_info_aux);
 	free_uid(aux->user);
@@ -2448,10 +2461,24 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu)
 	bpf_prog_free(aux->prog);
 }
 
+/*
+ * Progs called from a trampoline can also be reached by a task that was
+ * preempted in the trampoline before the prog's enter helper took its RCU
+ * read lock, wait for those first.
+ */
+static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu)
+{
+	struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog;
+
+	if (prog->sleepable)
+		call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu);
+	else
+		call_rcu(rcu, __bpf_prog_put_rcu);
+}
+
 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
 {
 	bpf_prog_kallsyms_del_all(prog);
-	btf_put(prog->aux->btf);
 	module_put(prog->aux->mod);
 	kvfree(prog->aux->jited_linfo);
 	kvfree(prog->aux->linfo);
@@ -2461,7 +2488,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
 		btf_put(prog->aux->attach_btf);
 
 	if (deferred) {
-		if (prog->sleepable)
+		if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked)
+			call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks);
+		else if (prog->sleepable)
 			call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu);
 		else
 			call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
@@ -3068,6 +3097,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at
 	prog->aux->user = get_current_user();
 	prog->len = attr->insn_cnt;
 
+	err = bpf_prog_stream_init(prog, GFP_USER);
+	if (err)
+		goto free_prog;
+
 	err = -EFAULT;
 	if (copy_from_bpfptr(prog->insns,
 			     make_bpfptr(attr->insns, uattr.is_kernel),
@@ -4755,6 +4788,7 @@ static int bpf_prog_query(const union bpf_attr *attr,
 	case BPF_CGROUP_GETSOCKOPT:
 	case BPF_CGROUP_SETSOCKOPT:
 	case BPF_LSM_CGROUP:
+	case BPF_STRUCT_OPS:
 		return cgroup_bpf_prog_query(attr, uattr, uattr_size);
 	case BPF_LIRC_MODE2:
 		return lirc_prog_query(attr, uattr);
@@ -6042,7 +6076,10 @@ struct bpf_link *bpf_link_get_curr_or_next(u32 *id)
 again:
 	link = idr_get_next(&link_idr, id);
 	if (link) {
-		link = bpf_link_inc_not_zero(link);
+		if (link->id)
+			link = bpf_link_inc_not_zero(link);
+		else
+			link = ERR_PTR(-EAGAIN);
 		if (IS_ERR(link)) {
 			(*id)++;
 			goto again;
@@ -6300,6 +6337,28 @@ static int prog_assoc_struct_ops(union bpf_attr *attr)
 	return ret;
 }
 
+#define BPF_PROG_STREAM_OPEN_LAST_FIELD prog_stream_open.flags
+
+static int prog_stream_open(union bpf_attr *attr)
+{
+	struct bpf_prog *prog;
+	u32 flags = attr->prog_stream_open.flags;
+	int ret;
+
+	if (CHECK_ATTR(BPF_PROG_STREAM_OPEN))
+		return -EINVAL;
+	if (flags & ~BPF_F_STREAM_NONBLOCK)
+		return -EINVAL;
+
+	prog = bpf_prog_get(attr->prog_stream_open.prog_fd);
+	if (IS_ERR(prog))
+		return PTR_ERR(prog);
+
+	ret = bpf_prog_stream_new_fd(prog, attr->prog_stream_open.stream_id, flags);
+	bpf_prog_put(prog);
+	return ret;
+}
+
 static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size,
 		     bpfptr_t uattr_common, unsigned int size_common)
 {
@@ -6472,6 +6531,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size,
 	case BPF_PROG_ASSOC_STRUCT_OPS:
 		err = prog_assoc_struct_ops(&attr);
 		break;
+	case BPF_PROG_STREAM_OPEN:
+		err = prog_stream_open(&attr);
+		break;
 	default:
 		err = -EINVAL;
 		break;
diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c
index 13e1aabe6f8868..c65ba1dcd86672 100644
--- a/kernel/bpf/task_iter.c
+++ b/kernel/bpf/task_iter.c
@@ -869,7 +869,7 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it,
 	BUILD_BUG_ON(sizeof(struct bpf_iter_task_vma_kern) != sizeof(struct bpf_iter_task_vma));
 	BUILD_BUG_ON(__alignof__(struct bpf_iter_task_vma_kern) != __alignof__(struct bpf_iter_task_vma));
 
-	if (!IS_ENABLED(CONFIG_PER_VMA_LOCK)) {
+	if (!IS_ENABLED(CONFIG_MMU)) {
 		kit->data = NULL;
 		return -EOPNOTSUPP;
 	}
diff --git a/kernel/bpf/token.c b/kernel/bpf/token.c
index e85a179523f02a..da915a4f972b17 100644
--- a/kernel/bpf/token.c
+++ b/kernel/bpf/token.c
@@ -169,8 +169,8 @@ int bpf_token_create(union bpf_attr *attr)
 	FD_PREPARE(fdf, O_CLOEXEC,
 		   alloc_file_pseudo(inode, path.mnt, BPF_TOKEN_INODE_NAME,
 				     O_RDWR, &bpf_token_fops));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
 	token = kzalloc_obj(*token, GFP_USER);
 	if (!token)
@@ -190,7 +190,7 @@ int bpf_token_create(union bpf_attr *attr)
 		return err;
 
 	get_user_ns(token->userns);
-	fd_prepare_file(fdf)->private_data = no_free_ptr(token);
+	fdf->file->private_data = no_free_ptr(token);
 	return fd_publish(fdf);
 }
 
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index 90b70ea0d37092..d880ad6913b387 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -401,6 +401,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip)
 	head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)];
 	hlist_add_head(&tr->hlist_ip, head);
 	refcount_set(&tr->refcnt, 1);
+	INIT_LIST_HEAD(&tr->images);
 	for (i = 0; i < BPF_TRAMP_MAX; i++)
 		INIT_HLIST_HEAD(&tr->progs_hlist[i]);
 out:
@@ -565,15 +566,22 @@ static void bpf_tramp_image_free(struct bpf_tramp_image *im)
 	arch_free_bpf_trampoline(im->image, im->size);
 	bpf_jit_uncharge_modmem(im->size);
 	percpu_ref_exit(&im->pcref);
+	kfree(im->skips);
 	kfree_rcu(im, rcu);
 }
 
 static void __bpf_tramp_image_put_deferred(struct work_struct *work)
 {
 	struct bpf_tramp_image *im;
+	struct bpf_trampoline *tr;
 
 	im = container_of(work, struct bpf_tramp_image, work);
+	tr = im->tr;
+	trampoline_lock(tr);
+	list_del(&im->list);
+	trampoline_unlock(tr);
 	bpf_tramp_image_free(im);
+	bpf_trampoline_put(tr);
 }
 
 /* callback, fexit step 3 or fentry step 2 */
@@ -601,7 +609,7 @@ static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu)
 	struct bpf_tramp_image *im;
 
 	im = container_of(rcu, struct bpf_tramp_image, rcu);
-	if (im->ip_after_call)
+	if (im->call_orig)
 		/* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */
 		percpu_ref_kill(&im->pcref);
 	else
@@ -621,9 +629,9 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
 	 *
 	 * The trampoline is unreachable before bpf_tramp_image_put().
 	 *
-	 * First, patch the trampoline to avoid calling into fexit progs.
-	 * The progs will be freed even if the original function is still
-	 * executing or sleeping.
+	 * Progs are patched out of the image when they are detached, see
+	 * bpf_trampoline_skip_prog(), so they can be freed even if a task is
+	 * still in the image.
 	 * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on
 	 * first few asm instructions to execute and call into
 	 * __bpf_tramp_enter->percpu_ref_get.
@@ -637,11 +645,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
 	 * percpu_ref_kill will be waiting for. Hence the first
 	 * call_rcu_tasks() is not necessary.
 	 */
-	if (im->ip_after_call) {
-		int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP,
-					     BPF_MOD_JUMP, NULL,
-					     im->ip_epilogue);
-		WARN_ON(err);
+	if (im->call_orig) {
 		if (IS_ENABLED(CONFIG_TASKS_RCU))
 			call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
 		else
@@ -658,7 +662,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im)
 	call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks);
 }
 
-static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
+static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs)
 {
 	struct bpf_tramp_image *im;
 	struct bpf_ksym *ksym;
@@ -669,6 +673,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
 	if (!im)
 		goto out;
 
+	im->skips = kzalloc_objs(*im->skips, nr_progs);
+	if (!im->skips)
+		goto out_free_im;
+
 	err = bpf_jit_charge_modmem(size);
 	if (err)
 		goto out_free_im;
@@ -695,6 +703,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size)
 out_uncharge:
 	bpf_jit_uncharge_modmem(size);
 out_free_im:
+	kfree(im->skips);
 	kfree(im);
 out:
 	return ERR_PTR(err);
@@ -771,11 +780,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
 		goto out;
 	}
 
-	im = bpf_tramp_image_alloc(tr->key, size);
+	im = bpf_tramp_image_alloc(tr->key, size, total);
 	if (IS_ERR(im)) {
 		err = PTR_ERR(im);
 		goto out;
 	}
+	im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG;
 
 	err = arch_prepare_bpf_trampoline(im, im->image, im->image + size,
 					  &tr->func.model, tr->flags, tnodes,
@@ -806,8 +816,14 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
 #endif
 
 out_free:
-	if (err)
+	if (err) {
 		bpf_tramp_image_free(im);
+	} else {
+		/* track the image until it is freed, for bpf_trampoline_skip_prog() */
+		refcount_inc(&tr->refcnt);
+		im->tr = tr;
+		list_add(&im->list, &tr->images);
+	}
 out:
 	/* If any error happens, restore previous flags */
 	if (err)
@@ -843,7 +859,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
 	}
 }
 
-static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
+static int bpf_freplace_link_tgt_prog(struct bpf_prog *tgt_prog)
 {
 	struct bpf_prog_aux *aux = tgt_prog->aux;
 
@@ -857,7 +873,7 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
 		 */
 		return -EBUSY;
 
-	aux->is_extended = true;
+	aux->freplace_link_cnt++;
 	return 0;
 }
 
@@ -907,6 +923,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr,
 	}
 
 	hlist_add_head(&node->tramp_hlist, prog_list);
+	node->link->prog->aux->tramp_linked = true;
 	if (kind == BPF_TRAMP_FSESSION) {
 		tr->progs_cnt[BPF_TRAMP_FENTRY]++;
 		fexit = fsession_exit(node);
@@ -920,6 +937,41 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr,
 	return 0;
 }
 
+/*
+ * Patch the nop in front of a prog call to a jump over it. A task can be
+ * preempted anywhere in the image, so archs that need several instructions for
+ * a jump of any range patch a single near branch here instead.
+ */
+int __weak arch_bpf_trampoline_skip(void *nop, void *target)
+{
+	return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target);
+}
+
+/*
+ * prog was detached and can be freed, but tasks may still be running in images
+ * that call it, sleeping in an earlier prog for example. They can be in any
+ * image that is not freed yet, not only in cur_image, so patch all of them to
+ * jump over prog.
+ */
+static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog)
+{
+	struct bpf_tramp_image *im;
+	int i, err;
+
+	list_for_each_entry(im, &tr->images, list) {
+		for (i = 0; i < im->nr_skips; i++) {
+			struct bpf_tramp_skip *skip = &im->skips[i];
+
+			if (skip->prog != prog)
+				continue;
+			err = arch_bpf_trampoline_skip(skip->nop, skip->target);
+			WARN_ON_ONCE(err);
+			/* not a nop anymore, and prog's address can be reused */
+			skip->prog = NULL;
+		}
+	}
+}
+
 static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr,
 				       struct bpf_tramp_node *node)
 {
@@ -937,6 +989,7 @@ static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr,
 	}
 	hlist_del_init(&node->tramp_hlist);
 	tr->progs_cnt[kind]--;
+	bpf_trampoline_skip_prog(tr, node->link->prog);
 }
 
 static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node,
@@ -970,7 +1023,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node,
 		/* Cannot attach extension if fentry/fexit are in use. */
 		if (cnt)
 			return -EBUSY;
-		err = bpf_freplace_check_tgt_prog(tgt_prog);
+		err = bpf_freplace_link_tgt_prog(tgt_prog);
 		if (err)
 			return err;
 		tr->extension_prog = node->link->prog;
@@ -1016,7 +1069,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
 					 tr->extension_prog->bpf_func, NULL);
 		tr->extension_prog = NULL;
 		guard(mutex)(&tgt_prog->aux->ext_mutex);
-		tgt_prog->aux->is_extended = false;
+		tgt_prog->aux->freplace_link_cnt--;
 		return err;
 	}
 	bpf_trampoline_remove_prog(tr, node);
@@ -1245,11 +1298,9 @@ void bpf_trampoline_put(struct bpf_trampoline *tr)
 		if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i])))
 			goto out;
 
-	/* This code will be executed even when the last bpf_tramp_image
-	 * is alive. All progs are detached from the trampoline and the
-	 * trampoline image is patched with jmp into epilogue to skip
-	 * fexit progs. The fentry-only trampoline will be freed via
-	 * multiple rcu callbacks.
+	/*
+	 * All progs are detached and the last image has been freed, images
+	 * hold a reference on the trampoline until then.
 	 */
 	hlist_del(&tr->hlist_key);
 	hlist_del(&tr->hlist_ip);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 72a3f5998dd27f..1599bac1bac93a 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -24,6 +24,7 @@
 #include <linux/bpf_lsm.h>
 #include <linux/security.h>
 #include <linux/verification.h>
+#include <linux/keyctl.h>
 #include <linux/btf_ids.h>
 #include <linux/poison.h>
 #include <linux/module.h>
@@ -301,6 +302,12 @@ static int arg_idx_from_argno(argno_t a)
 	return arg_from_argno(a) - 1;
 }
 
+/* Normalize helper register numbers and kfunc argument numbers to ABI slots. */
+static u32 arg_slot_from_argno(argno_t a)
+{
+	return abs(a.argno) - 1;
+}
+
 static const char *btf_type_name(const struct btf *btf, u32 id)
 {
 	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
@@ -394,27 +401,69 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog)
 	return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
 }
 
-static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
+static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog)
 {
-	const struct btf_type *type, *func, *func_proto;
+	const struct btf_type *func, *func_proto;
 	const struct btf *btf = env->prog->aux->btf;
 	u32 btf_id;
 
+	if (!btf || !env->prog->aux->func_info)
+		return NULL;
+
 	btf_id = env->prog->aux->func_info[subprog].type_id;
 
+	/* Both already validated by prepare_btf_func() at prog load. */
 	func = btf_type_by_id(btf, btf_id);
-	if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id))
-		return false;
-
 	func_proto = btf_type_by_id(btf, func->type);
-	if (!func_proto)
-		return false;
 
-	type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
-	if (!type)
-		return false;
+	return btf_type_skip_modifiers(btf, func_proto->type, NULL);
+}
+
+static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
+{
+	const struct btf_type *type = subprog_ret_type(env, subprog);
+
+	return type && btf_type_is_void(type);
+}
+
+static u32 ret_regs_cnt(u32 size)
+{
+	return size > 8 && size <= 16 ? 2 : 1;
+}
+
+/* Registers holding a function return value, in order. See ret_regs_cnt(). */
+static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 };
+
+static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env)
+{
+	const struct btf *btf = env->prog->aux->btf;
+	const struct btf_type *type;
+	int subprog;
+	u32 size;
+
+	if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux))
+		return 0;
+
+	/*
+	 * Skip the main program: its return value is the program's exit code,
+	 * read out of R0, so it never uses the register pair. An extension does
+	 * have a real prototype for subprog 0, but btf_check_func_type_match()
+	 * refuses to replace a function returning more than 8 bytes.
+	 */
+	for (subprog = 1; subprog < env->subprog_cnt; subprog++) {
+		type = subprog_ret_type(env, subprog);
+		if (!type || btf_type_is_void(type))
+			continue;
+		if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env,
+				    "cannot size return type of subprog %d", subprog))
+			return -EFAULT;
+		if (ret_regs_cnt(size) > 1) {
+			subprog_info(env, subprog)->ret_reg_pair = true;
+			env->prog->jit_required = 1;
+		}
+	}
 
-	return btf_type_is_void(type);
+	return 0;
 }
 
 const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog)
@@ -486,6 +535,7 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id)
 
 static bool is_sync_callback_calling_kfunc(u32 btf_id);
 static bool is_async_callback_calling_kfunc(u32 btf_id);
+static bool is_call_rcu_kfunc(u32 btf_id);
 static bool is_callback_calling_kfunc(u32 btf_id);
 
 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);
@@ -528,6 +578,10 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn
 	if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback)
 		return false;
 
+	/* bpf_call_rcu and bpf_call_rcu_tasks_trace callbacks are never sleepable. */
+	if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && is_call_rcu_kfunc(insn->imm))
+		return false;
+
 	/* bpf_wq and bpf_task_work callbacks are always sleepable. */
 	if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
 	    (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm)))
@@ -544,16 +598,17 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn)
 
 static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
 {
-       int allocated_slots = state->allocated_stack / BPF_REG_SIZE;
+	int allocated_slots = bpf_stack_nr_slots(state);
 
-       /* We need to check that slots between [spi - nr_slots + 1, spi] are
-	* within [0, allocated_stack).
-	*
-	* Please note that the spi grows downwards. For example, a dynptr
-	* takes the size of two stack slots; the first slot will be at
-	* spi and the second slot will be at spi - 1.
-	*/
-       return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
+	/*
+	 * We need to check that slots between [spi - nr_slots + 1, spi] are
+	 * within [0, allocated_stack).
+	 *
+	 * Please note that the spi grows downwards. For example, a dynptr
+	 * takes the size of two stack slots; the first slot will be at
+	 * spi and the second slot will be at spi - 1.
+	 */
+	return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
 }
 
 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
@@ -567,7 +622,7 @@ static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_s
 	}
 
 	off = reg->var_off.value;
-	if (off % BPF_REG_SIZE) {
+	if (off >= 0 || off % BPF_REG_SIZE) {
 		verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off);
 		return -EINVAL;
 	}
@@ -697,8 +752,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_
 		return err;
 
 	for (i = 0; i < BPF_REG_SIZE; i++) {
-		state->stack[spi].slot_type[i] = STACK_DYNPTR;
-		state->stack[spi - 1].slot_type[i] = STACK_DYNPTR;
+		bpf_stack_slot(state, spi)->slot_type[i] = STACK_DYNPTR;
+		bpf_stack_slot(state, spi - 1)->slot_type[i] = STACK_DYNPTR;
 	}
 
 	type = arg_to_dynptr_type(arg_type);
@@ -731,8 +786,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_
 		parent_id = dynptr->parent_id;
 	}
 
-	mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,
-			       &state->stack[spi - 1].spilled_ptr, type, parent_id);
+	mark_dynptr_stack_regs(env, &bpf_stack_slot(state, spi)->spilled_ptr,
+			       &bpf_stack_slot(state, spi - 1)->spilled_ptr, type, parent_id);
 
 	return 0;
 }
@@ -764,7 +819,7 @@ static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_re
 	 * all clones and derived slices. For non-referenced dynptr, only
 	 * the dynptr and slices derived from it will be invalidated.
 	 */
-	reg = &state->stack[spi].spilled_ptr;
+	reg = &bpf_stack_slot(state, spi)->spilled_ptr;
 	return release_reference(env, dynptr_type_referenced(reg->dynptr.type)
 				      ? reg->parent_id
 				      : reg->id);
@@ -803,6 +858,7 @@ static int dynptr_ref_cnt(struct bpf_verifier_env *env, int v_parent_id)
 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
 				        struct bpf_func_state *state, int spi)
 {
+	struct bpf_stack_state *slot = bpf_stack_slot(state, spi);
 	int err = 0;
 
 	/* We always ensure that STACK_DYNPTR is never set partially,
@@ -810,20 +866,22 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
 	 * different for STACK_SPILL, where it may be only set for
 	 * 1 byte, so code has to use is_spilled_reg.
 	 */
-	if (state->stack[spi].slot_type[0] != STACK_DYNPTR)
+	if (slot->slot_type[0] != STACK_DYNPTR)
 		return 0;
 
 	/* Reposition spi to first slot */
-	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
+	if (!slot->spilled_ptr.dynptr.first_slot) {
 		spi = spi + 1;
+		slot = bpf_stack_slot(state, spi);
+	}
 
 	/*
 	 * A referenced dynptr can be overwritten only if there is at
 	 * least one other dynptr sharing the same virtual ref parent,
 	 * ensuring the reference can still be properly released.
 	 */
-	if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) &&
-	    dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) {
+	if (dynptr_type_referenced(slot->spilled_ptr.dynptr.type) &&
+	    dynptr_ref_cnt(env, slot->spilled_ptr.parent_id) <= 1) {
 		verbose(env, "cannot overwrite referenced dynptr\n");
 		bpf_diag_res(
 			env, env->insn_idx, "referenced dynptr overwrite",
@@ -833,7 +891,7 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
 	}
 
 	/* Invalidate the dynptr and any derived slices */
-	err = release_reference(env, state->stack[spi].spilled_ptr.id);
+	err = release_reference(env, slot->spilled_ptr.id);
 	if (!err) {
 		mark_stack_slot_scratched(env, spi);
 		mark_stack_slot_scratched(env, spi - 1);
@@ -873,6 +931,7 @@ static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_
 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
 {
 	struct bpf_func_state *state = bpf_func(env, reg);
+	struct bpf_stack_state *slot;
 	int i, spi;
 
 	/* This already represents first slot of initialized bpf_dynptr.
@@ -887,12 +946,13 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re
 	spi = dynptr_get_spi(env, reg);
 	if (spi < 0)
 		return false;
-	if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
+	slot = bpf_stack_slot(state, spi);
+	if (!slot->spilled_ptr.dynptr.first_slot)
 		return false;
 
 	for (i = 0; i < BPF_REG_SIZE; i++) {
-		if (state->stack[spi].slot_type[i] != STACK_DYNPTR ||
-		    state->stack[spi - 1].slot_type[i] != STACK_DYNPTR)
+		if (slot->slot_type[i] != STACK_DYNPTR ||
+		    bpf_stack_slot(state, spi - 1)->slot_type[i] != STACK_DYNPTR)
 			return false;
 	}
 
@@ -911,14 +971,18 @@ static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct
 	if (spi < 0)
 		return BPF_DYNPTR_TYPE_INVALID;
 	state = bpf_func(env, reg);
-	return state->stack[spi].spilled_ptr.dynptr.type;
+	return bpf_stack_slot(state, spi)->spilled_ptr.dynptr.type;
 }
 
 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 				    enum bpf_arg_type arg_type)
 {
-	/* ARG_PTR_TO_DYNPTR takes any type of dynptr */
-	if (arg_type == ARG_PTR_TO_DYNPTR)
+	/*
+	 * ARG_PTR_TO_DYNPTR without a type flag takes any type of dynptr.
+	 * Test the flags rather than the whole arg_type, which may carry
+	 * unrelated ones such as PTR_MAYBE_NULL.
+	 */
+	if (!(arg_type & DYNPTR_TYPE_FLAG_MASK))
 		return true;
 
 	return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type);
@@ -947,7 +1011,7 @@ static int mark_stack_slots_iter(struct bpf_verifier_env *env,
 		return id;
 
 	for (i = 0; i < nr_slots; i++) {
-		struct bpf_stack_state *slot = &state->stack[spi - i];
+		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
 		struct bpf_reg_state *st = &slot->spilled_ptr;
 
 		__mark_reg_known_zero(st);
@@ -984,7 +1048,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env,
 		return spi;
 
 	for (i = 0; i < nr_slots; i++) {
-		struct bpf_stack_state *slot = &state->stack[spi - i];
+		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
 		struct bpf_reg_state *st = &slot->spilled_ptr;
 
 		if (i == 0)
@@ -1018,7 +1082,7 @@ static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env,
 		return false;
 
 	for (i = 0; i < nr_slots; i++) {
-		struct bpf_stack_state *slot = &state->stack[spi - i];
+		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
 
 		for (j = 0; j < BPF_REG_SIZE; j++)
 			if (slot->slot_type[j] == STACK_ITER)
@@ -1039,7 +1103,7 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s
 		return -EINVAL;
 
 	for (i = 0; i < nr_slots; i++) {
-		struct bpf_stack_state *slot = &state->stack[spi - i];
+		struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i);
 		struct bpf_reg_state *st = &slot->spilled_ptr;
 
 		if (st->type & PTR_UNTRUSTED)
@@ -1081,7 +1145,7 @@ static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env,
 	if (id < 0)
 		return id;
 
-	slot = &state->stack[spi];
+	slot = bpf_stack_slot(state, spi);
 	st = &slot->spilled_ptr;
 
 	__mark_reg_known_zero(st);
@@ -1108,7 +1172,7 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r
 	if (spi < 0)
 		return spi;
 
-	slot = &state->stack[spi];
+	slot = bpf_stack_slot(state, spi);
 	st = &slot->spilled_ptr;
 
 	if (st->irq.kfunc_class != kfunc_class) {
@@ -1177,7 +1241,7 @@ static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bp
 	if (spi < 0)
 		return false;
 
-	slot = &state->stack[spi];
+	slot = bpf_stack_slot(state, spi);
 
 	for (i = 0; i < BPF_REG_SIZE; i++)
 		if (slot->slot_type[i] == STACK_IRQ_FLAG)
@@ -1196,7 +1260,7 @@ static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_r
 	if (spi < 0)
 		return -EINVAL;
 
-	slot = &state->stack[spi];
+	slot = bpf_stack_slot(state, spi);
 	st = &slot->spilled_ptr;
 
 	if (!st->id)
@@ -1343,7 +1407,7 @@ static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf
 
 static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
 {
-	size_t n = src->allocated_stack / BPF_REG_SIZE;
+	size_t n = bpf_stack_nr_slots(src);
 
 	dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
 				GFP_KERNEL_ACCOUNT);
@@ -1382,7 +1446,7 @@ static int resize_reference_state(struct bpf_verifier_state *state, size_t n)
  */
 static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size)
 {
-	size_t old_n = state->allocated_stack / BPF_REG_SIZE, n;
+	size_t old_n = bpf_stack_nr_slots(state), n;
 
 	/* The stack size is always a multiple of BPF_REG_SIZE. */
 	size = round_up(size, BPF_REG_SIZE);
@@ -1864,6 +1928,7 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
 	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
 	reg->id = 0;
 	reg->parent_id = 0;
+	reg->map_uid = 0;
 	___mark_reg_known(reg, imm);
 }
 
@@ -1925,17 +1990,18 @@ static void refine_map_lookup_value(struct bpf_reg_state *reg)
 	if (map->inner_map_meta) {
 		reg->type = CONST_PTR_TO_MAP | maybe_null;
 		reg->map_ptr = map->inner_map_meta;
-		/* transfer reg's id which is unique for every map_lookup_elem
+		/*
+		 * transfer reg's id which is unique for every map_lookup_elem
 		 * as UID of the inner map.
 		 */
-		if (btf_record_has_field(map->inner_map_meta->record,
-					 BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK))
-			reg->map_uid = reg->id;
+		reg->map_uid = reg->id;
 	} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
 		reg->type = PTR_TO_XDP_SOCK | maybe_null;
+		reg->map_uid = 0;
 	} else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
 		   map->map_type == BPF_MAP_TYPE_SOCKHASH) {
 		reg->type = PTR_TO_SOCKET | maybe_null;
+		reg->map_uid = 0;
 	}
 }
 
@@ -2404,6 +2470,7 @@ static int add_subprog(struct bpf_verifier_env *env, int off)
 		return -E2BIG;
 	}
 	/* determine subprog starts. The end is one before the next starts */
+	env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX;
 	env->subprog_info[env->subprog_cnt++].start = off;
 	sort(env->subprog_info, env->subprog_cnt,
 	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
@@ -2797,7 +2864,7 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env,
 }
 
 static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
-			       struct bpf_func_proto *proto);
+			       const struct btf_func_model *fm, struct bpf_func_proto *proto);
 
 int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
 {
@@ -2886,6 +2953,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
 	err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model);
 	if (err)
 		return err;
+	if (func_model.ret_size > 8) {
+		if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) {
+			verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n",
+				kfunc.name);
+			return -EOPNOTSUPP;
+		}
+		if (!bpf_jit_supports_kfunc_ret_reg_pair()) {
+			verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n",
+				kfunc.name);
+			return -EOPNOTSUPP;
+		}
+	}
 
 	memset(&meta, 0, sizeof(meta));
 	meta.btf = kfunc.btf;
@@ -2902,7 +2981,7 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
 	desc = &tab->descs[tab->nr_descs];
 	memset(desc, 0, sizeof(*desc));
 
-	err = gen_kfunc_arg_proto(env, &meta, &desc->proto);
+	err = gen_kfunc_arg_proto(env, &meta, &func_model, &desc->proto);
 	if (err)
 		return err;
 
@@ -2953,6 +3032,22 @@ static int add_subprogs(struct bpf_verifier_env *env)
 			return ret;
 	}
 
+	/*
+	 * func_info describes all functions of the program. Those that are
+	 * referenced only from data, e.g. from a table of functions to be
+	 * called via callx, are not seen by the loop above. They are possible
+	 * callees that have to be known upfront as well.
+	 */
+	if (env->bpf_capable) {
+		struct bpf_prog_aux *aux = env->prog->aux;
+
+		for (i = 1; i < aux->func_info_cnt; i++) {
+			ret = add_subprog(env, aux->func_info[i].insn_off);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
 	ret = bpf_find_exception_callback_insn_off(env);
 	if (ret < 0)
 		return ret;
@@ -3034,6 +3129,8 @@ static int check_subprogs(struct bpf_verifier_env *env)
 		if (BPF_CLASS(code) == BPF_LD &&
 		    (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
 			subprog[cur_subprog].has_ld_abs = true;
+		if (bpf_is_callx(&insn[i]))
+			env->has_callx = true;
 		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
 			goto next;
 		if (BPF_OP(code) == BPF_CALL)
@@ -3042,6 +3139,8 @@ static int check_subprogs(struct bpf_verifier_env *env)
 			subprog[cur_subprog].exit_idx = i;
 			goto next;
 		}
+		if (insn_is_gotox(&insn[i]))
+			goto next;
 		off = i + bpf_jmp_offset(&insn[i]) + 1;
 		if (off < subprog_start || off >= subprog_end) {
 			verbose(env, "jump out of range from insn %d to %d\n", i, off);
@@ -3061,7 +3160,8 @@ static int check_subprogs(struct bpf_verifier_env *env)
 			 */
 			if (code != (BPF_JMP | BPF_EXIT) &&
 			    code != (BPF_JMP32 | BPF_JA) &&
-			    code != (BPF_JMP | BPF_JA)) {
+			    code != (BPF_JMP | BPF_JA) &&
+			    !insn_is_gotox(&insn[i])) {
 				verbose(env, "last insn is not an exit or jmp\n");
 				bpf_diag_program_structure(
 					env, i, "subprogram can fall through",
@@ -3079,12 +3179,50 @@ static int check_subprogs(struct bpf_verifier_env *env)
 	return 0;
 }
 
+/*
+ * The callee of callx is known to the main verification pass only, which
+ * records the 'caller' -> 'callee' edge of the call graph for the checks
+ * that follow it: absence of recursion and the maximum stack depth.
+ */
+static int record_callx_edge(struct bpf_verifier_env *env, int caller, int callee)
+{
+	u32 cnt = env->subprog_cnt;
+
+	if (!env->callx_edges) {
+		env->callx_edges = kvcalloc(BITS_TO_LONGS(cnt * cnt), sizeof(long),
+					    GFP_KERNEL_ACCOUNT);
+		if (!env->callx_edges)
+			return -ENOMEM;
+	}
+	__set_bit(caller * cnt + callee, env->callx_edges);
+	return 0;
+}
+
+/*
+ * Return the first subprog with the number >= 'from' that 'caller' calls
+ * via callx, or -1 when there is none.
+ */
+static int next_callx_callee(struct bpf_verifier_env *env, int caller, int from)
+{
+	u32 cnt = env->subprog_cnt;
+	unsigned long bit, end = (caller + 1) * cnt;
+
+	if (!env->callx_edges || from >= cnt)
+		return -1;
+	bit = find_next_bit(env->callx_edges, end, caller * cnt + from);
+	return bit < end ? bit - caller * cnt : -1;
+}
+
 /*
  * Sort subprogs in topological order so that leaf subprogs come first and
  * their callers come later. This is a DFS post-order traversal of the call
  * graph. Scan only reachable instructions (those in the computed postorder) of
  * the current subprog to discover callees (direct subprogs and sync
  * callbacks).
+ *
+ * The callees of callx are not known before the main verification pass.
+ * When callx is used the sort is repeated after it with the recorded callx
+ * edges added to the call graph to reject recursion through indirect calls.
  */
 static int sort_subprogs_topo(struct bpf_verifier_env *env)
 {
@@ -3125,12 +3263,22 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env)
 				int idx = insn_postorder[j];
 				int callee;
 
-				if (!bpf_pseudo_call(&insn[idx]) && !bpf_pseudo_func(&insn[idx]))
+				if (bpf_is_callx(&insn[idx])) {
+					/* find a callee that is not explored yet */
+					callee = -1;
+					do {
+						callee = next_callx_callee(env, cur, callee + 1);
+					} while (callee >= 0 && color[callee] == 2);
+					if (callee < 0)
+						continue;
+				} else if (bpf_pseudo_call(&insn[idx]) || bpf_pseudo_func(&insn[idx])) {
+					callee = bpf_find_subprog(env, idx + insn[idx].imm + 1);
+					if (callee < 0) {
+						ret = -EFAULT;
+						goto out;
+					}
+				} else {
 					continue;
-				callee = bpf_find_subprog(env, idx + insn[idx].imm + 1);
-				if (callee < 0) {
-					ret = -EFAULT;
-					goto out;
 				}
 				if (color[callee] == 2)
 					continue;
@@ -3227,26 +3375,30 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
 	env->insn_aux_data[idx].indirect_target = true;
 }
 
+static void mark_non_stack_access(struct bpf_verifier_env *env, int idx)
+{
+	env->insn_aux_data[idx].non_stack_access = true;
+}
+
+/* Layout of one packed linked register in the jump history, see linked_regs_pack() */
 #define LR_FRAMENO_BITS	4
-#define LR_SPI_BITS	6
-#define LR_ENTRY_BITS	(LR_SPI_BITS + LR_FRAMENO_BITS + 1)
-#define LR_SIZE_BITS	4
-#define LR_FRAMENO_MASK	((1ull << LR_FRAMENO_BITS) - 1)
-#define LR_SPI_MASK	((1ull << LR_SPI_BITS)     - 1)
-#define LR_SIZE_MASK	((1ull << LR_SIZE_BITS)    - 1)
-#define LR_SPI_OFF	LR_FRAMENO_BITS
-#define LR_IS_REG_OFF	(LR_SPI_BITS + LR_FRAMENO_BITS)
-#define LINKED_REGS_MAX	5
+#define LR_INDEX_BITS	11
+#define LR_FRAMENO_MASK	((1u << LR_FRAMENO_BITS) - 1)
+#define LR_IS_REG	BIT(LR_FRAMENO_BITS)
+#define LR_INDEX_OFF	(LR_FRAMENO_BITS + 1)
+#define LR_INDEX_MASK	((1u << LR_INDEX_BITS) - 1)
+#define LINKED_REGS_MAX	BPF_LINKED_REGS_MAX
 
 static_assert(MAX_CALL_FRAMES <= (1 << LR_FRAMENO_BITS));
-static_assert(LINKED_REGS_MAX < (1 << LR_SIZE_BITS));
-static_assert(LINKED_REGS_MAX * LR_ENTRY_BITS + LR_SIZE_BITS <= 64);
+static_assert(MAX_BPF_REG <= (1 << LR_INDEX_BITS));
+static_assert(MAX_BPF_STACK_SLOTS <= (1 << LR_INDEX_BITS));
+static_assert(LR_INDEX_OFF + LR_INDEX_BITS <= 16);
 
 struct linked_reg {
 	u8 frameno;
 	union {
-		u8 spi;
-		u8 regno;
+		u16 spi;
+		u16 regno;
 	};
 	bool is_reg;
 };
@@ -3265,48 +3417,34 @@ static struct linked_reg *linked_regs_push(struct linked_regs *s)
 }
 
 /*
- * Use u64 as a vector of 5 11-bit values, use first 4-bits to track
- * number of elements currently in stack.
- * Pack one history entry for linked registers as 11 bits in the following format:
- * - 4-bits frameno
- * - 6-bits spi_or_reg
- * - 1-bit  is_reg
+ * Pack linked registers for a jump history entry, one u16 each:
+ * - 4 bits frameno
+ * - 1 bit  is_reg
+ * - 11 bits register or stack slot index
  */
-static u64 linked_regs_pack(struct linked_regs *s)
+static void linked_regs_pack(const struct linked_regs *s, u16 *packed)
 {
-	u64 val = 0;
 	int i;
 
 	for (i = 0; i < s->cnt; ++i) {
-		struct linked_reg *e = &s->entries[i];
-		u64 tmp = 0;
+		const struct linked_reg *e = &s->entries[i];
 
-		tmp |= e->frameno;
-		tmp |= e->spi << LR_SPI_OFF;
-		tmp |= (e->is_reg ? 1 : 0) << LR_IS_REG_OFF;
-
-		val <<= LR_ENTRY_BITS;
-		val |= tmp;
+		packed[i] = e->frameno | (e->is_reg ? LR_IS_REG : 0) | (e->spi << LR_INDEX_OFF);
 	}
-	val <<= LR_SIZE_BITS;
-	val |= s->cnt;
-	return val;
 }
 
-static void linked_regs_unpack(u64 val, struct linked_regs *s)
+static void linked_regs_unpack(const struct bpf_jmp_history_entry *hist, struct linked_regs *s)
 {
 	int i;
 
-	s->cnt = val & LR_SIZE_MASK;
-	val >>= LR_SIZE_BITS;
-
+	s->cnt = hist->linked_regs_cnt;
 	for (i = 0; i < s->cnt; ++i) {
 		struct linked_reg *e = &s->entries[i];
+		u16 packed = hist->linked_regs[i];
 
-		e->frameno =  val & LR_FRAMENO_MASK;
-		e->spi     = (val >> LR_SPI_OFF) & LR_SPI_MASK;
-		e->is_reg  = (val >> LR_IS_REG_OFF) & 0x1;
-		val >>= LR_ENTRY_BITS;
+		e->frameno = packed & LR_FRAMENO_MASK;
+		e->is_reg  = packed & LR_IS_REG;
+		e->spi     = (packed >> LR_INDEX_OFF) & LR_INDEX_MASK;
 	}
 }
 
@@ -3348,10 +3486,10 @@ void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_
 	bool some_precise = false;
 	int i;
 
-	if (!hist || hist->linked_regs == 0)
+	if (!hist || !hist->linked_regs_cnt)
 		return;
 
-	linked_regs_unpack(hist->linked_regs, &linked_regs);
+	linked_regs_unpack(hist, &linked_regs);
 	for (i = 0; i < linked_regs.cnt; ++i) {
 		struct linked_reg *e = &linked_regs.entries[i];
 
@@ -3448,17 +3586,18 @@ static void save_register_state(struct bpf_verifier_env *env,
 				int spi, struct bpf_reg_state *reg,
 				int size)
 {
+	struct bpf_stack_state *slot = bpf_stack_slot(state, spi);
 	int i;
 
-	bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL);
-	state->stack[spi].spilled_ptr = *reg;
+	bpf_diag_mod_begin(env, &slot->spilled_ptr, reg, BPF_DIAG_MOD_SPILL);
+	slot->spilled_ptr = *reg;
 
 	for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
-		state->stack[spi].slot_type[i - 1] = STACK_SPILL;
+		slot->slot_type[i - 1] = STACK_SPILL;
 
 	/* size < 8 bytes spill */
 	for (; i; i--)
-		mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]);
+		mark_stack_slot_misc(env, &slot->slot_type[i - 1]);
 
 	bpf_diag_mod_end(env);
 }
@@ -3500,14 +3639,15 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
 
 static void scrub_special_slot(struct bpf_func_state *state, int spi)
 {
+	struct bpf_stack_state *slot = bpf_stack_slot(state, spi);
 	int i;
 
 	/* regular write of data into stack destroys any spilled ptr */
-	state->stack[spi].spilled_ptr.type = NOT_INIT;
+	slot->spilled_ptr.type = NOT_INIT;
 	/* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */
-	if (is_stack_slot_special(&state->stack[spi]))
+	if (is_stack_slot_special(slot))
 		for (i = 0; i < BPF_REG_SIZE; i++)
-			scrub_spilled_slot(&state->stack[spi].slot_type[i]);
+			scrub_spilled_slot(&slot->slot_type[i]);
 }
 
 /* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
@@ -3525,13 +3665,15 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 	struct bpf_reg_state *reg = NULL;
 	int insn_flags = INSN_F_STACK_ACCESS;
 	int hist_spi = spi, hist_frame = state->frameno;
+	struct bpf_stack_state *ss = bpf_stack_slot(state, spi);
 
-	/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
+	/*
+	 * caller checked that off % size == 0 and -env->stack_limit <= off < 0,
 	 * so it's aligned access and [off, off + size) are within stack limits
 	 */
 	if (!env->allow_ptr_leaks &&
-	    bpf_is_spilled_reg(&state->stack[spi]) &&
-	    !bpf_is_spilled_scalar_reg(&state->stack[spi]) &&
+	    bpf_is_spilled_reg(ss) &&
+	    !bpf_is_spilled_scalar_reg(ss) &&
 	    size != BPF_REG_SIZE) {
 		const char *reason;
 
@@ -3553,7 +3695,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 		bool sanitize = reg && is_pointer_regtype(reg->type);
 
 		for (i = 0; i < size; i++) {
-			u8 type = state->stack[spi].slot_type[(slot - i) %
+			u8 type = ss->slot_type[(slot - i) %
 							      BPF_REG_SIZE];
 
 			if (type != STACK_MISC && type != STACK_ZERO) {
@@ -3570,7 +3712,6 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 	if (err)
 		return err;
 
-	check_fastcall_stack_contract(env, state, insn_idx, off);
 	mark_stack_slot_scratched(env, spi);
 	if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
 		bool reg_value_fits;
@@ -3582,7 +3723,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 		save_register_state(env, state, spi, reg, size);
 		/* Break the relation on a narrowing spill. */
 		if (!reg_value_fits)
-			state->stack[spi].spilled_ptr.id = 0;
+			clear_scalar_id(&ss->spilled_ptr);
 	} else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) &&
 		   env->bpf_capable) {
 		struct bpf_reg_state *tmp_reg = &env->fake_reg[0];
@@ -3606,8 +3747,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 	} else {
 		u8 type = STACK_MISC;
 
-		if (bpf_is_spilled_reg(&state->stack[spi]))
-			bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr,
+		if (bpf_is_spilled_reg(ss))
+			bpf_diag_record_scrub(env, &ss->spilled_ptr,
 					      BPF_DIAG_MOD_WRITE);
 		scrub_special_slot(state, spi);
 
@@ -3628,13 +3769,13 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 
 		/* Mark slots affected by this stack write. */
 		for (i = 0; i < size; i++)
-			state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = type;
+			ss->slot_type[(slot - i) % BPF_REG_SIZE] = type;
 		insn_flags = 0; /* not a register spill */
 	}
 
 	if (insn_flags)
 		return bpf_push_jmp_history(env, env->cur_state, insn_flags,
-					    hist_spi, hist_frame, 0);
+					    hist_spi, hist_frame, NULL, 0);
 	return 0;
 }
 
@@ -3691,15 +3832,16 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
 			return err;
 	}
 
-	check_fastcall_stack_contract(env, state, insn_idx, min_off);
 	/* Variable offset writes destroy any spilled pointers in range. */
 	for (i = min_off; i < max_off; i++) {
+		struct bpf_stack_state *ss;
 		u8 new_type, *stype;
 		int slot, spi;
 
 		slot = -i - 1;
 		spi = slot / BPF_REG_SIZE;
-		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
+		ss = bpf_stack_slot(state, spi);
+		stype = &ss->slot_type[slot % BPF_REG_SIZE];
 		mark_stack_slot_scratched(env, spi);
 
 		if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {
@@ -3722,9 +3864,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
 		/* If writing_zero and the spi slot contains a spill of value 0,
 		 * maintain the spill type.
 		 */
-		if (writing_zero && *stype == STACK_SPILL &&
-		    bpf_is_spilled_scalar_reg(&state->stack[spi])) {
-			struct bpf_reg_state *spill_reg = &state->stack[spi].spilled_ptr;
+		if (writing_zero && *stype == STACK_SPILL && bpf_is_spilled_scalar_reg(ss)) {
+			struct bpf_reg_state *spill_reg = &ss->spilled_ptr;
 
 			if (tnum_is_const(spill_reg->var_off) && spill_reg->var_off.value == 0) {
 				zero_used = true;
@@ -3795,23 +3936,22 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env,
 {
 	struct bpf_verifier_state *vstate = env->cur_state;
 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
-	u64 zero_spill_mask = 0;
 	int i, slot, spi;
 	u8 *stype;
-	int zeros = 0;
+	int zeros = 0, zero_spills = 0;
 
 	for (i = min_off; i < max_off; i++) {
 		slot = -i - 1;
 		spi = slot / BPF_REG_SIZE;
 		mark_stack_slot_scratched(env, spi);
-		stype = ptr_state->stack[spi].slot_type;
+		stype = bpf_stack_slot(ptr_state, spi)->slot_type;
 		if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) {
 			zeros++;
 			continue;
 		}
 		if (stype[slot % BPF_REG_SIZE] == STACK_SPILL &&
-		    bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) {
-			zero_spill_mask |= 1ull << spi;
+		    bpf_register_is_null(&bpf_stack_slot(ptr_state, spi)->spilled_ptr)) {
+			zero_spills++;
 			zeros++;
 			continue;
 		}
@@ -3822,8 +3962,14 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env,
 		 * so the whole register == const_zero.
 		 */
 		__mark_reg_const_zero(env, &state->regs[dst_regno]);
-		if (zero_spill_mask) {
-			bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask);
+		if (zero_spills) {
+			for (i = min_off; i < max_off; i++) {
+				slot = -i - 1;
+				spi = slot / BPF_REG_SIZE;
+				stype = bpf_stack_slot(ptr_state, spi)->slot_type;
+				if (stype[slot % BPF_REG_SIZE] == STACK_SPILL)
+					bpf_bt_set_frame_slot(&env->bt, ptr_state->frameno, spi);
+			}
 			return mark_chain_precision_batch(env, env->cur_state);
 		}
 	} else {
@@ -3834,19 +3980,46 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env,
 	return 0;
 }
 
-static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i,
-				       int size)
+static void bpf_diag_stack_read_invalid(struct bpf_verifier_env *env, int off, int i, int size,
+					enum bpf_stack_slot_type type)
 {
-	const char *reason;
+	const char *problem, *reason, *suggestion, *kind;
+
+	if (type == STACK_INVALID) {
+		reason = bpf_diag_fmt(
+			env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. "
+			"Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.",
+			size, off, i);
+		bpf_diag_memory(
+			env, env->insn_idx, "uninitialized stack read", reason,
+			"Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, "
+			"or load with CAP_PERFMON if uninitialized stack reads are intended.");
+		return;
+	}
+
+	switch (type) {
+	case STACK_DYNPTR:
+		kind = "dynptr";
+		suggestion = "Use dynptr helpers or kfuncs to access the object represented by the dynptr instead of reading the dynptr state directly.";
+		break;
+	case STACK_ITER:
+		kind = "iterator";
+		suggestion = "Use iterator kfuncs to advance or destroy the iterator instead of reading its state directly.";
+		break;
+	case STACK_IRQ_FLAG:
+		kind = "IRQ flag";
+		suggestion = "Pass the saved IRQ flag to the matching restore kfunc instead of reading its state directly.";
+		break;
+	default:
+		return;
+	}
 
-	reason = bpf_diag_fmt(env,
-			      "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. "
-		"Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.",
-		size, off, i);
-	bpf_diag_memory(
-		env, env->insn_idx, "uninitialized stack read", reason,
-		"Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, "
-		"or load with CAP_PERFMON if uninitialized stack reads are intended.");
+	problem = bpf_diag_fmt(env, "direct read of %s stack state", kind);
+	reason = bpf_diag_fmt(
+		env, "This rejected read uses %d bytes at stack offset %d, but byte %d in that range belongs to verifier-managed %s state. "
+		"This state has an opaque representation that BPF programs cannot read directly.",
+		size, off, i, kind);
+	bpf_diag_memory(env, env->insn_idx, problem, reason, suggestion);
 }
 
 /* Read the stack at 'off' and put the results into the register indicated by
@@ -3871,12 +4044,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 	int err;
 	int insn_flags = INSN_F_STACK_ACCESS;
 	int hist_spi = spi, hist_frame = reg_state->frameno;
+	struct bpf_stack_state *ss = bpf_stack_slot(reg_state, spi);
 
-	stype = reg_state->stack[spi].slot_type;
-	reg = &reg_state->stack[spi].spilled_ptr;
+	stype = ss->slot_type;
+	reg = &ss->spilled_ptr;
 
 	mark_stack_slot_scratched(env, spi);
-	check_fastcall_stack_contract(env, state, env->insn_idx, off);
 
 	/*
 	 * Refine the in-progress load record's origin to the source stack slot.
@@ -3884,7 +4057,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 	if (dst_regno >= 0)
 		bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE);
 
-	if (bpf_is_spilled_reg(&reg_state->stack[spi])) {
+	if (bpf_is_spilled_reg(ss)) {
 		u8 spill_size = 1;
 
 		for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--)
@@ -3897,11 +4070,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 				return -EACCES;
 			}
 
-			if (dst_regno < 0)
-				return 0;
-
 			if (size <= spill_size &&
 			    bpf_stack_narrow_access_ok(off, size, spill_size)) {
+				if (dst_regno < 0)
+					return 0;
+
 				if (env->bpf_capable && size == 4 && spill_size == 4 &&
 				    get_reg_width(reg) <= 32)
 					/* Ensure stack slot has an ID to build a relation
@@ -3938,11 +4111,14 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 					} else {
 						verbose(env, "invalid read from stack off %d+%d size %d\n",
 							off, i, size);
-						bpf_diag_stack_read_uninit(env, off, i, size);
+						bpf_diag_stack_read_invalid(env, off, i, size, type);
 					}
 					return -EACCES;
 				}
 
+				if (dst_regno < 0)
+					return 0;
+
 				if (spill_cnt == size &&
 				    tnum_is_const(reg->var_off) && reg->var_off.value == 0) {
 					__mark_reg_const_zero(env, &state->regs[dst_regno]);
@@ -3997,7 +4173,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 			} else {
 				verbose(env, "invalid read from stack off %d+%d size %d\n",
 					off, i, size);
-				bpf_diag_stack_read_uninit(env, off, i, size);
+				bpf_diag_stack_read_invalid(env, off, i, size, type);
 			}
 			return -EACCES;
 		}
@@ -4010,7 +4186,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 	}
 	if (insn_flags)
 		return bpf_push_jmp_history(env, env->cur_state, insn_flags,
-					    hist_spi, hist_frame, 0);
+					    hist_spi, hist_frame, NULL, 0);
 	return 0;
 }
 
@@ -4063,7 +4239,6 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg
 				  dst_regno);
 	if (err)
 		return err;
-	check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
 	return 0;
 }
 
@@ -4096,13 +4271,13 @@ static int check_stack_read(struct bpf_verifier_env *env,
 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 		verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
 			tn_buf, off, size);
-		reason = bpf_diag_fmt(env,
-				      "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. "
-			"Helper stack memory arguments require a constant stack offset and a precise initialized range.",
+		reason = bpf_diag_fmt(
+			env, "The instruction would access the stack through variable offset %s plus fixed offset %d and size %d. "
+			"This stack access requires a constant stack offset and a precise initialized range.",
 			tn_buf, off, size);
 		bpf_diag_memory(
-			env, env->insn_idx, "variable stack access", reason,
-			"Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first.");
+			env, env->insn_idx, "variable-offset stack access", reason,
+			"Use a fixed stack offset for the instruction, selecting the target stack slot on separate control-flow paths if necessary.");
 		return -EACCES;
 	}
 	/* Variable offset is prohibited for unprivileged mode for simplicity
@@ -4200,7 +4375,7 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s
 	bpf_diag_mod_end(env);
 	state->no_stack_arg_load = true;
 	return bpf_push_jmp_history(env, env->cur_state,
-				    INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
+				    INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0);
 }
 
 /*
@@ -4234,7 +4409,7 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st
 	cur->regs[dst_regno] = *arg;
 	bpf_diag_mod_end(env);
 	return bpf_push_jmp_history(env, env->cur_state,
-				    INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
+				    INSN_F_STACK_ARG_ACCESS, spi, 0, NULL, 0);
 }
 
 static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx)
@@ -4594,7 +4769,7 @@ BTF_ID(struct, cgroup)
 BTF_ID(struct, bpf_cpumask)
 #endif
 BTF_ID(struct, task_struct)
-#ifdef CONFIG_CRYPTO
+#ifdef CONFIG_BPF_CRYPTO
 BTF_ID(struct, bpf_crypto_ctx)
 #endif
 #ifdef CONFIG_INET
@@ -4817,7 +4992,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *
 }
 
 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
-			       const struct bpf_func_proto *fn,
+			       const struct bpf_call_arg_meta *meta,
 			       enum bpf_access_type t)
 {
 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
@@ -4841,10 +5016,11 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
 	case BPF_PROG_TYPE_LWT_XMIT:
 	case BPF_PROG_TYPE_SK_SKB:
 	case BPF_PROG_TYPE_SK_MSG:
-		if (fn)
-			return fn->pkt_access;
+		if (meta && !meta->btf && meta->func_id)
+			return meta->fn->pkt_access;
 
-		env->seen_direct_write = true;
+		if (t == BPF_WRITE)
+			env->seen_direct_write = true;
 		return true;
 
 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
@@ -5102,18 +5278,6 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
 	[CONST_PTR_TO_MAP] = btf_bpf_map_id,
 };
 
-static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id)
-{
-	enum bpf_reg_type type;
-
-	for (type = 0; type < __BPF_REG_TYPE_MAX; type++) {
-		if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id)
-			return type;
-	}
-
-	return NOT_INIT;
-}
-
 static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
 {
 	/* A referenced register is always trusted. */
@@ -5308,6 +5472,9 @@ struct bpf_subprog_call_depth_info {
 	int ret_insn; /* caller instruction where we return to. */
 	int caller; /* caller subprogram idx */
 	int frame; /* # of consecutive static call stack frames on top of stack */
+	int callx_insn; /* callx instruction whose callees are being walked */
+	int callx_next; /* next callee of callx_insn to walk */
+	bool via_callx; /* the subprogram is entered via callx */
 };
 
 /* starting from main bpf function walk all instructions of the function
@@ -5327,11 +5494,13 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 
 	/* no caller idx */
 	dinfo[idx].caller = -1;
+	dinfo[idx].via_callx = false;
 
 	i = subprog[idx].start;
 	if (!priv_stack_supported)
 		subprog[idx].priv_stack_mode = NO_PRIV_STACK;
 process_func:
+	dinfo[idx].callx_insn = -1;
 	if (subprog[idx].has_ld_abs) {
 		for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
 			if (subprog[tmp].is_cb) {
@@ -5384,7 +5553,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
 		if (subprog_depth > env->max_stack_depth)
 			env->max_stack_depth = subprog_depth;
-		if (subprog_depth > MAX_BPF_STACK) {
+		if (subprog_depth > env->stack_limit) {
 			verbose(env, "stack size of subprog %d is %d. Too large\n",
 				idx, subprog_depth);
 			return -EACCES;
@@ -5393,7 +5562,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 		depth += subprog_depth;
 		if (depth > env->max_stack_depth)
 			env->max_stack_depth = depth;
-		if (depth > MAX_BPF_STACK) {
+		if (depth > env->stack_limit) {
 			total = 0;
 			for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller)
 				total++;
@@ -5408,11 +5577,10 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	for (; i < subprog_end; i++) {
 		int next_insn, sidx;
 
-		if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
+		if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off &&
+		    bpf_is_throw_kfunc(insn + i)) {
 			bool err = false;
 
-			if (!bpf_is_throw_kfunc(insn + i))
-				continue;
 			for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
 				if (subprog[tmp].is_cb) {
 					err = true;
@@ -5427,31 +5595,70 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 			return -EINVAL;
 		}
 
-		if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
+		if (bpf_is_callx(insn + i) || bpf_calls_callback(env, i)) {
+			/*
+			 * Walk the callees and the callbacks recorded by
+			 * the main verification pass one by one, returning to
+			 * this insn after each.
+			 */
+			if (dinfo[idx].callx_insn != i) {
+				dinfo[idx].callx_insn = i;
+				dinfo[idx].callx_next = 0;
+			}
+			sidx = next_callx_callee(env, idx, dinfo[idx].callx_next);
+			if (sidx < 0)
+				continue;
+			dinfo[idx].callx_next = sidx + 1;
+			dinfo[idx].ret_insn = i;
+			next_insn = subprog[sidx].start;
+		} else if (bpf_pseudo_call(insn + i) || bpf_pseudo_func(insn + i)) {
+			/* find the callee */
+			next_insn = i + insn[i].imm + 1;
+			sidx = bpf_find_subprog(env, next_insn);
+			if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))
+				return -EFAULT;
+			if (subprog[sidx].is_async_cb) {
+				/* async callbacks don't increase bpf prog stack size unless called directly */
+				if (!bpf_pseudo_call(insn + i))
+					continue;
+				if (subprog[sidx].is_exception_cb) {
+					verbose(env, "insn %d cannot call exception cb directly", i);
+					return -EINVAL;
+				}
+			}
+			/* remember insn to return to */
+			dinfo[idx].ret_insn = i + 1;
+		} else {
 			continue;
-		/* remember insn and function to return to */
+		}
 
-		/* find the callee */
-		next_insn = i + insn[i].imm + 1;
-		sidx = bpf_find_subprog(env, next_insn);
-		if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))
-			return -EFAULT;
-		if (subprog[sidx].is_async_cb) {
-			/* async callbacks don't increase bpf prog stack size unless called directly */
-			if (!bpf_pseudo_call(insn + i))
+		/*
+		 * sort_subprogs_topo() tolerates cycles in the call graph that
+		 * go through the address of a function being taken, since it
+		 * doesn't know what it is taken for. Such cycle is a recursion
+		 * unless it's an async callback, which are skipped above.
+		 * The main verification pass limits the depth of the recursion,
+		 * but it doesn't follow calls of global functions.
+		 */
+		for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
+			if (tmp != sidx)
 				continue;
-			if (subprog[sidx].is_exception_cb) {
-				verbose(env, "insn %d cannot call exception cb directly", i);
-				return -EINVAL;
-			}
+			verbose(env, "recursive call from %s() to %s()\n",
+				bpf_subprog_name(env, idx), bpf_subprog_name(env, sidx));
+			bpf_diag_program_structure(
+				env, i, "recursive subprogram call",
+				"Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.",
+				"This call or callback would make the subprogram call graph recursive. "
+				"The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis.");
+			return -EINVAL;
 		}
 
 		/* store caller info for after we return from callee */
 		dinfo[idx].frame = frame;
-		dinfo[idx].ret_insn = i + 1;
 
 		/* push caller idx into callee's dinfo */
 		dinfo[sidx].caller = idx;
+		dinfo[sidx].via_callx = bpf_is_callx(insn + i);
 
 		i = next_insn;
 
@@ -5485,6 +5692,14 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 				verbose(env, "tail_calls are not allowed in programs with stack args\n");
 				return -EINVAL;
 			}
+			/*
+			 * JITs pass tail call counter in a register that is
+			 * not available when the callee is called via callx.
+			 */
+			if (dinfo[tmp].via_callx) {
+				verbose(env, "tail_calls are not allowed in functions called via callx\n");
+				return -EINVAL;
+			}
 			subprog[tmp].tail_call_reachable = true;
 		}
 	} else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) {
@@ -5849,6 +6064,110 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
 	return 0;
 }
 
+static int cmp_func_ptrs(const void *_a, const void *_b)
+{
+	const struct bpf_func_ptr *a = _a, *b = _b;
+
+	if (a->map != b->map)
+		return a->map < b->map ? -1 : 1;
+	if (a->map_off != b->map_off)
+		return a->map_off < b->map_off ? -1 : 1;
+	return 0;
+}
+
+/* Find the first pointer to a function at or after 'off' in the value of 'map' */
+static u32 func_ptr_lower_bound(struct bpf_verifier_env *env, const struct bpf_map *map, u64 off)
+{
+	u32 l = 0, r = env->func_ptr_cnt, m;
+	struct bpf_func_ptr *p;
+
+	while (l < r) {
+		m = l + (r - l) / 2;
+		p = &env->func_ptrs[m];
+		if (p->map < map || (p->map == map && p->map_off < off))
+			l = m + 1;
+		else
+			r = m;
+	}
+	return l;
+}
+
+/*
+ * Return pointers to functions that overlap with 'size' bytes at offset 'off'
+ * of the value of 'map' and their number in 'cnt'.
+ */
+struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env,
+					     const struct bpf_map *map,
+					     u64 off, u64 size, u32 *cnt)
+{
+	u32 first, last;
+
+	*cnt = 0;
+	if (!env->func_ptr_cnt || !size)
+		return NULL;
+
+	/* a pointer that starts up to 7 bytes before 'off' overlaps too */
+	first = func_ptr_lower_bound(env, map, off >= sizeof(u64) ? off - sizeof(u64) + 1 : 0);
+	last = func_ptr_lower_bound(env, map, off + size);
+	if (first >= last)
+		return NULL;
+
+	*cnt = last - first;
+	return &env->func_ptrs[first];
+}
+
+/* Return all pointers to functions in the value of 'map' */
+struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env,
+				       const struct bpf_map *map, u32 *cnt)
+{
+	return bpf_map_range_func_ptrs(env, map, 0, (u64)map->value_size, cnt);
+}
+
+/* instructions [off, off + len) replaced the instruction at 'off' */
+void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len)
+{
+	struct bpf_func_ptr *p;
+	u32 i;
+
+	if (len <= 1)
+		return;
+
+	for (i = 0; i < env->func_ptr_cnt; i++) {
+		p = &env->func_ptrs[i];
+		if (p->xlated_off <= off || p->xlated_off == BPF_FUNC_PTR_DELETED)
+			continue;
+		p->xlated_off += len - 1;
+	}
+}
+
+/*
+ * Instructions [off, off + len) are about to be removed. It's called before
+ * the starts of subprogs are adjusted. A subprog is gone when all of its
+ * instructions are. Otherwise, e.g. when its first instruction is a nop,
+ * it starts where the removed instructions did.
+ */
+void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)
+{
+	struct bpf_func_ptr *p;
+	int subprog;
+	u32 i;
+
+	for (i = 0; i < env->func_ptr_cnt; i++) {
+		p = &env->func_ptrs[i];
+		if (p->xlated_off < off || p->xlated_off == BPF_FUNC_PTR_DELETED)
+			continue;
+		if (p->xlated_off >= off + len) {
+			p->xlated_off -= len;
+			continue;
+		}
+		subprog = bpf_find_subprog(env, p->xlated_off);
+		if (subprog > 0 && env->subprog_info[subprog + 1].start > off + len)
+			p->xlated_off = off;
+		else
+			p->xlated_off = BPF_FUNC_PTR_DELETED;
+	}
+}
+
 #define BTF_TYPE_SAFE_RCU(__type)  __PASTE(__type, __safe_rcu)
 #define BTF_TYPE_SAFE_RCU_OR_NULL(__type)  __PASTE(__type, __safe_rcu_or_null)
 #define BTF_TYPE_SAFE_TRUSTED(__type)  __PASTE(__type, __safe_trusted)
@@ -5918,10 +6237,36 @@ BTF_TYPE_SAFE_TRUSTED(struct file) {
 	struct inode *f_inode;
 };
 
+/*
+ * The pointer fields in the sched_ext ops argument containers are pinned by the
+ * callers for the duration of the ops calls and are never NULL.
+ */
+BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) {
+#ifdef CONFIG_EXT_GROUP_SCHED
+	struct cgroup *cgroup;
+#endif
+};
+
+BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) {
+	struct task_struct *task;
+};
+
+BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) {
+	struct sched_ext_ops *ops;
+};
+
+BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) {
+	struct sched_ext_ops *ops;
+};
+
 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) {
 	struct inode *d_inode;
 };
 
+BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm) {
+	struct mm_struct *mm;
+};
+
 BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket) {
 	struct sock *sk;
 };
@@ -5962,6 +6307,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env,
 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task));
 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm));
 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
+	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args));
+	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args));
+	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args));
+	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args));
 
 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");
 }
@@ -5972,6 +6321,7 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env,
 {
 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct socket));
 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry));
+	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct linux_binprm));
 	BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct vm_area_struct));
 
 	return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id,
@@ -6213,10 +6563,11 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env,
 	return 0;
 }
 
-/* Check that the stack access at the given offset is within bounds. The
+/*
+ * Check that the stack access at the given offset is within bounds. The
  * maximum valid offset is -1.
  *
- * The minimum valid offset is -MAX_BPF_STACK for writes, and
+ * The minimum valid offset is -env->stack_limit for writes, and
  * -state->allocated_stack for reads.
  */
 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
@@ -6227,7 +6578,7 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
 	int min_valid_off;
 
 	if (t == BPF_WRITE || env->allow_uninit_stack)
-		min_valid_off = -MAX_BPF_STACK;
+		min_valid_off = -(int)env->stack_limit;
 	else
 		min_valid_off = -state->allocated_stack;
 
@@ -6293,6 +6644,15 @@ static int check_stack_access_within_bounds(
 		return err;
 	}
 
+	/*
+	 * Every stack access passes through here, including buffers passed to
+	 * helpers and kfuncs and accesses into a caller's frame, so check the
+	 * fastcall contract of the frame that owns the slots once for all of
+	 * them. Zero-sized accesses touch no stack and keep the rewrite enabled.
+	 */
+	if (access_size)
+		check_fastcall_stack_contract(env, state, env->insn_idx, min_off);
+
 	/* Note that there is no stack access with offset zero, so the needed stack
 	 * size is -min_off, not -min_off+1.
 	 */
@@ -6327,54 +6687,194 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val)
 	reg_bounds_sync(dst_reg);
 }
 
-static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off,
-			      int bpf_size, int value_regno, bool is_ldsx)
+/* there is no support for callx in the interpreter */
+static int require_callx_jit(struct bpf_verifier_env *env)
 {
-	struct bpf_reg_state *regs = cur_regs(env);
-	int size = bpf_size_to_bytes(bpf_size);
-	struct bpf_map *map = reg->map_ptr;
-
-	switch (map->map_type) {
-	case BPF_MAP_TYPE_INSN_ARRAY:
-		if (bpf_size != BPF_DW) {
-			verbose(env, "Invalid read of %d bytes from insn_array\n", size);
-			return -EACCES;
-		}
-		regs[value_regno] = *reg;
-		add_scalar_to_reg(&regs[value_regno], off);
-		regs[value_regno].type = PTR_TO_INSN;
-		return 0;
-	case BPF_MAP_TYPE_PERCPU_ARRAY:
-		goto reg_unknown;
-	default:
-		break;
+	if (!env->prog->jit_requested) {
+		verbose(env, "JIT is required to use callx\n");
+		return -EOPNOTSUPP;
 	}
-
-	/* If map is read-only, track its contents as scalars. */
-	if (tnum_is_const(reg->var_off) &&
-	    bpf_map_is_rdonly(map) &&
-	    map->ops->map_direct_value_addr) {
-		int map_off = off + reg->var_off.value;
-		u64 val = 0;
-		int err;
-
-		err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx);
-		if (err)
-			return err;
-
-		regs[value_regno].type = SCALAR_VALUE;
-		__mark_reg_known(&regs[value_regno], val);
-		return 0;
+	if (!bpf_jit_supports_callx()) {
+		verbose(env, "JIT doesn't support callx\n");
+		return -EOPNOTSUPP;
 	}
-
-reg_unknown:
-	mark_reg_unknown(env, regs, value_regno);
+	env->prog->jit_required = true;
 	return 0;
 }
 
-/* check whether memory at (regno + off) is accessible for t = (read | write)
- * if t==write, value_regno is a register which value is stored into memory
- * if t==read, value_regno is a register which will receive the value from memory
+static void mark_reg_func_ptr(struct bpf_verifier_env *env, struct bpf_reg_state *regs,
+			      int regno, int subprog)
+{
+	mark_reg_known_zero(env, regs, regno);
+	regs[regno].type = PTR_TO_FUNC;
+	regs[regno].subprogno = subprog;
+}
+
+/* a read from a table of functions branches into that many states at most */
+#define BPF_MAX_FUNC_PTR_TARGETS 64
+/* and that many pointers at most are next to what is read at variable offset */
+#define BPF_MAX_FUNC_PTR_RANGE 4096
+
+/* Count up to 'limit' offsets in [min_off, max_off] that the read is possible at */
+static u32 count_read_offsets(struct tnum offs, u64 min_off, u64 max_off, u32 limit)
+{
+	u64 o = min_off, next;
+	u32 cnt = 0;
+
+	if (!tnum_in(offs, tnum_const(o))) {
+		next = tnum_step(offs, o);
+		if (next <= o)
+			return 0;
+		o = next;
+	}
+	while (o <= max_off && cnt < limit) {
+		cnt++;
+		next = tnum_step(offs, o);
+		if (next <= o)
+			break;
+		o = next;
+	}
+	return cnt;
+}
+
+/*
+ * A read from a frozen read-only map that has pointers to functions, see
+ * resolve_func_ptrs(). The bounds and var_off of the offset tell where the read
+ * is possible at, e.g. var_off is the stride of the elements of an array.
+ * A read of exactly one pointer yields PTR_TO_FUNC. When the offset is variable
+ * and only pointers can be read, which is how an element of a table of
+ * functions is loaded, the verification continues with each of them. Other
+ * reads that overlap with a pointer are rejected, because their result is not
+ * known until the program is jitted.
+ *
+ * Return -ENOENT if there are no pointers to functions in the bytes that are read.
+ */
+static int check_func_ptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off,
+			       int size, int value_regno)
+{
+	u64 min_off = reg_umin(reg) + off, max_off = reg_umax(reg) + off;
+	struct tnum offs = tnum_add(reg->var_off, tnum_const(off));
+	struct bpf_reg_state *regs = cur_regs(env);
+	struct bpf_map *map = reg->map_ptr;
+	struct bpf_verifier_state *branch;
+	int subprog, targets[BPF_MAX_FUNC_PTR_TARGETS];
+	struct bpf_func_ptr *ptrs;
+	u32 i, cnt, n = 0;
+	u64 o, lo, hi;
+	int err;
+
+	ptrs = bpf_map_range_func_ptrs(env, map, min_off, max_off - min_off + size, &cnt);
+	if (!ptrs)
+		return -ENOENT;
+	if (cnt > BPF_MAX_FUNC_PTR_RANGE)
+		goto overlap;
+
+	for (i = 0; i < cnt; i++) {
+		/* the offsets at which the read overlaps with the pointer */
+		lo = ptrs[i].map_off >= size ? ptrs[i].map_off - size + 1 : 0;
+		lo = max(lo, min_off);
+		hi = min(ptrs[i].map_off + sizeof(u64) - 1, max_off);
+		for (o = lo; o <= hi; o++) {
+			if (!tnum_in(offs, tnum_const(o)))
+				continue;
+			if (o != ptrs[i].map_off || size != sizeof(u64) || value_regno < 0)
+				goto overlap;
+			if (n == BPF_MAX_FUNC_PTR_TARGETS) {
+				verbose(env, "read from map '%s' may yield more than %d pointers to functions\n",
+					map->name, BPF_MAX_FUNC_PTR_TARGETS);
+				return -E2BIG;
+			}
+			subprog = bpf_find_subprog(env, ptrs[i].xlated_off);
+			if (verifier_bug_if(subprog <= 0, env,
+					    "no function at insn %u for map '%s' offset %u",
+					    ptrs[i].xlated_off, map->name, ptrs[i].map_off))
+				return -EFAULT;
+			targets[n++] = subprog;
+		}
+	}
+	/* pointers are next to what is read, e.g. the data of an array of vtables */
+	if (!n)
+		return -ENOENT;
+	/* either every offset that the read is possible at is a pointer or none */
+	if (count_read_offsets(offs, min_off, max_off, n + 1) != n)
+		goto overlap;
+
+	/* the map has the offset of the function until the program is jitted */
+	err = require_callx_jit(env);
+	if (err)
+		return err;
+
+	for (i = 0; i < n - 1; i++) {
+		branch = push_stack(env, env->insn_idx + 1, env->insn_idx,
+				    env->cur_state->speculative);
+		if (IS_ERR(branch))
+			return PTR_ERR(branch);
+		mark_reg_func_ptr(env, branch->frame[branch->curframe]->regs, value_regno,
+				  targets[i]);
+	}
+	mark_reg_func_ptr(env, regs, value_regno, targets[n - 1]);
+	return 0;
+
+overlap:
+	verbose(env, "read of %d bytes at offset [%llu,%llu] of map '%s' overlaps with a pointer to a function\n",
+		size, min_off, max_off, map->name);
+	return -EACCES;
+}
+
+static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off,
+			      int bpf_size, int value_regno, bool is_ldsx)
+{
+	struct bpf_reg_state *regs = cur_regs(env);
+	int size = bpf_size_to_bytes(bpf_size);
+	struct bpf_map *map = reg->map_ptr;
+	int err;
+
+	switch (map->map_type) {
+	case BPF_MAP_TYPE_INSN_ARRAY:
+		if (bpf_size != BPF_DW) {
+			verbose(env, "Invalid read of %d bytes from insn_array\n", size);
+			return -EACCES;
+		}
+		regs[value_regno] = *reg;
+		add_scalar_to_reg(&regs[value_regno], off);
+		regs[value_regno].type = PTR_TO_INSN;
+		return 0;
+	case BPF_MAP_TYPE_PERCPU_ARRAY:
+		goto reg_unknown;
+	default:
+		break;
+	}
+
+	if (env->func_ptr_cnt) {
+		err = check_func_ptr_read(env, reg, off, size, value_regno);
+		if (err != -ENOENT)
+			return err;
+	}
+
+	/* If map is read-only, track its contents as scalars. */
+	if (tnum_is_const(reg->var_off) &&
+	    bpf_map_is_rdonly(map) &&
+	    map->ops->map_direct_value_addr) {
+		int map_off = off + reg->var_off.value;
+		u64 val = 0;
+
+		err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx);
+		if (err)
+			return err;
+
+		regs[value_regno].type = SCALAR_VALUE;
+		__mark_reg_known(&regs[value_regno], val);
+		return 0;
+	}
+
+reg_unknown:
+	mark_reg_unknown(env, regs, value_regno);
+	return 0;
+}
+
+/* check whether memory at (regno + off) is accessible for t = (read | write)
+ * if t==write, value_regno is a register which value is stored into memory
+ * if t==read, value_regno is a register which will receive the value from memory
  * if t==write && value_regno==-1, some unknown value is stored into memory
  * if t==read && value_regno==-1, don't care what we read from memory
  */
@@ -6383,6 +6883,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
 			    int value_regno, bool strict_alignment_once, bool is_ldsx)
 {
 	struct bpf_reg_state *regs = cur_regs(env);
+	enum bpf_reg_type ptr_type = reg->type;
 	int size, err = 0;
 
 	size = bpf_size_to_bytes(bpf_size);
@@ -6453,6 +6954,15 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
 			return -EACCES;
 		}
 
+		if (rdonly_untrusted && !env->allow_ptr_leaks) {
+			verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
+				reg_type_str(env, reg->type));
+			bpf_diag_policy(env, insn_idx, "read from untrusted read-only memory",
+					"the access requires CAP_PERFMON",
+					"Load the program with CAP_PERFMON, or avoid dereferencing untrusted pointers.");
+			return -EPERM;
+		}
+
 		/*
 		 * Accesses to untrusted PTR_TO_MEM are done through probe
 		 * instructions, hence no need to check bounds in that case.
@@ -6499,6 +7009,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
 					regs[value_regno].btf = info.btf;
 					regs[value_regno].btf_id = info.btf_id;
 					regs[value_regno].id = info.ref_id;
+				} else if (base_type(info.reg_type) == PTR_TO_MEM) {
+					regs[value_regno].mem_size = info.mem_size;
 				}
 				if (type_may_be_null(info.reg_type) && !regs[value_regno].id)
 					regs[value_regno].id = ++env->id_gen;
@@ -6621,6 +7133,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
 				clear_scalar_id(&regs[value_regno]);
 		}
 	}
+
+	if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK)
+		mark_non_stack_access(env, insn_idx);
+
 	return err;
 }
 
@@ -6896,8 +7412,11 @@ static int check_stack_range_initialized(
 	 * but BTF based global subprog validation isn't accurate enough.
 	 */
 	bool allow_poison = access_size < 0 || clobber;
-	/* The call will initialize the memory; uninitialized stack allowed */
-	bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno);
+	u32 arg_slot = arg_slot_from_argno(argno);
+	bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS &&
+		      (meta->arg_raw_mem.mask & BIT(arg_slot));
+	bool raw_mode = uninit && env->allow_uninit_stack &&
+			!(meta->arg_raw_mem.var_size_mask & BIT(arg_slot));
 
 	access_size = abs(access_size);
 
@@ -6926,11 +7445,9 @@ static int check_stack_range_initialized(
 				reg_arg_name(env, argno), tn_buf);
 			return -EACCES;
 		}
-		/* Only initialized buffer on stack is allowed to be accessed
-		 * with variable offset. With uninitialized buffer it's hard to
-		 * guarantee that whole memory is marked as initialized on
-		 * helper return since specific bounds are unknown what may
-		 * cause uninitialized stack leaking.
+		/*
+		 * The call may touch any byte in the possible range, but does not
+		 * definitely initialize all of it. Fall back to ordinary stack checks.
 		 */
 		raw_mode = false;
 
@@ -6938,12 +7455,14 @@ static int check_stack_range_initialized(
 		max_off = reg_smax(reg) + off;
 	}
 
+	/* Unprivileged outputs retain each byte's initialization state. */
 	if (raw_mode) {
-		meta->arg_raw_mem.size = access_size;
+		meta->arg_raw_mem.size[arg_slot] = access_size;
 		return 0;
 	}
 
 	for (i = min_off; i < max_off + access_size; i++) {
+		struct bpf_stack_state *ss;
 		u8 *stype;
 
 		slot = -i - 1;
@@ -6953,32 +7472,41 @@ static int check_stack_range_initialized(
 			return -EFAULT;
 		}
 
-		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
+		ss = bpf_stack_slot(state, spi);
+		stype = &ss->slot_type[slot % BPF_REG_SIZE];
 		if (*stype == STACK_MISC)
 			goto mark;
 		if ((*stype == STACK_ZERO) ||
-		    (*stype == STACK_INVALID && env->allow_uninit_stack)) {
-			if (clobber) {
+		    (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) {
+			if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) {
 				/* helper can write anything into the stack */
 				*stype = STACK_MISC;
 			}
 			goto mark;
 		}
 
-		if (bpf_is_spilled_reg(&state->stack[spi]) &&
-		    (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
+		/*
+		 * Only the bytes marked STACK_SPILL hold the spilled register.
+		 * The rest of a narrowly spilled slot keeps its previous type
+		 * and must be initialized on its own.
+		 */
+		if (*stype == STACK_SPILL &&
+		    (ss->spilled_ptr.type == SCALAR_VALUE ||
 		     env->allow_ptr_leaks)) {
 			if (clobber) {
-				__mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
+				__mark_reg_unknown(env, &ss->spilled_ptr);
 				for (j = 0; j < BPF_REG_SIZE; j++)
-					scrub_spilled_slot(&state->stack[spi].slot_type[j]);
+					scrub_spilled_slot(&ss->slot_type[j]);
 			}
 			goto mark;
 		}
 
 		if (*stype == STACK_POISON) {
-			if (allow_poison)
+			if (allow_poison) {
+				if (uninit && env->allow_uninit_stack)
+					*stype = STACK_MISC;
 				goto mark;
+			}
 			verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n",
 				reg_arg_name(env, argno), min_off, i - min_off, access_size);
 		} else if (tnum_is_const(reg->var_off)) {
@@ -7012,6 +7540,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_
 	switch (base_type(reg->type)) {
 	case PTR_TO_PACKET:
 	case PTR_TO_PACKET_META:
+		if (!may_access_direct_pkt_data(env, meta, access_type)) {
+			verbose(env, "function access to the packet is not allowed\n");
+			return -EACCES;
+		}
 		return check_packet_access(env, reg, argno, 0, access_size,
 					   zero_size_allowed);
 	case PTR_TO_MAP_KEY:
@@ -7122,13 +7654,13 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
 	 */
 	meta->msize_max_value = reg_umax(size_reg);
 
-	/* The register is SCALAR_VALUE; the access check happens using
-	 * its boundaries. For unprivileged variable accesses, disable
-	 * raw mode so that the program is required to initialize all
-	 * the memory that the helper could just partially fill up.
+	/*
+	 * Check variable ranges byte by byte instead of using raw mode. Keep the
+	 * MEM_UNINIT annotation so invalid bytes are accepted without marking them
+	 * initialized when the caller cannot read uninitialized stack memory.
 	 */
 	if (!tnum_is_const(size_reg->var_off))
-		meta = NULL;
+		meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno));
 
 	if (reg_smin(size_reg) < 0) {
 		verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n",
@@ -7171,6 +7703,11 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
 	return err;
 }
 
+static bool is_subprog(const struct bpf_call_arg_meta *meta)
+{
+	return meta->btf && !meta->func_id;
+}
+
 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 			 argno_t argno, u32 mem_size, enum bpf_access_type access_type,
 			 struct bpf_call_arg_meta *meta, bool *known_memory)
@@ -7189,10 +7726,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg
 	}
 
 	/*
-	 * Only a global subprog (meta == NULL) may read poisoned stack slots:
+	 * Only a global subprog may read poisoned stack slots:
 	 * its static stack liveness proved the callee body skips them.
 	 */
-	size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size;
+	size = (is_subprog(meta) &&
+		base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size;
 
 	if (access_type & BPF_READ)
 		err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta,
@@ -7484,6 +8022,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_
 	case BPF_TASK_WORK:
 		field_off = map->record->task_work_off;
 		break;
+	case BPF_RCU_HEAD:
+		field_off = map->record->rcu_head_off;
+		break;
 	case BPF_WORKQUEUE:
 		field_off = map->record->wq_off;
 		break;
@@ -7572,11 +8113,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u
 /*
  * Validate dynptr arguments for helper, kfunc and subprog.
  *
- * @dynptr is both input and output. It is populated when the argument is
- * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed)
- * and consumed when the argument is expecting to be an initialized dynptr.
- * @parent_id is used to track the referenced parent object (e.g., file or skb in
- * qdisc program) when constructing a dynptr.
+ * @meta carries the dynptr and referenced-object state. The dynptr is populated
+ * when the argument is tagged with MEM_UNINIT (i.e., the dynptr argument that
+ * will be constructed) and consumed when the argument is expected to be an
+ * initialized dynptr. The reference tracks the parent object (e.g., file or skb
+ * in qdisc program) when constructing a dynptr.
  *
  * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
  * which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR.
@@ -7593,24 +8134,11 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u
  * and checked dynamically during runtime.
  */
 static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
-			       argno_t argno, int insn_idx, const char *call_name,
-			       enum bpf_arg_type arg_type,
-			       struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr)
+			       argno_t argno, int insn_idx, enum bpf_arg_type arg_type,
+			       struct bpf_call_arg_meta *meta)
 {
 	int spi, err = 0;
 
-	if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) {
-		verbose(env,
-			"%s expected pointer to stack or const struct bpf_dynptr\n",
-			reg_arg_name(env, argno));
-		bpf_diag_call_arg_fmt(
-			env, insn_idx, argno, call_name,
-			"Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.",
-			"a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s",
-			reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type));
-		return -EINVAL;
-	}
-
 	/*  MEM_UNINIT - Points to memory that is an appropriate candidate for
 	 *		 constructing a mutable bpf_dynptr object.
 	 *
@@ -7632,7 +8160,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat
 			verbose(env, "Dynptr has to be an uninitialized dynptr\n");
 			bpf_diag_res(
 				env, insn_idx, "dynptr is already initialized",
-				"This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.",
+				"This function constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.",
 				"Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot.");
 			return -EINVAL;
 		}
@@ -7645,7 +8173,8 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat
 				return err;
 		}
 
-		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, ref_obj, dynptr);
+		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx,
+					      &meta->ref_obj, &meta->dynptr);
 	} else /* OBJ_RELEASE and None case from above */ {
 		/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
 		if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) {
@@ -7675,7 +8204,7 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat
 			verbose(env, "Expected a dynptr of type %s as %s\n",
 				dynptr_type_str(expected_type), reg_arg_name(env, argno));
 			bpf_diag_call_arg_fmt(
-				env, insn_idx, argno, call_name,
+				env, insn_idx, argno, meta->func_name,
 				"Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.",
 				"the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s",
 				dynptr_type_str(actual_type), dynptr_type_str(expected_type));
@@ -7691,14 +8220,12 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat
 
 			mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS);
 
-			reg = &state->stack[spi].spilled_ptr;
+			reg = &bpf_stack_slot(state, spi)->spilled_ptr;
 		}
 
-		if (dynptr) {
-			dynptr->type = reg->dynptr.type;
-			dynptr->id = reg->id;
-			dynptr->parent_id = reg->parent_id;
-		}
+		meta->dynptr.type = reg->dynptr.type;
+		meta->dynptr.id = reg->id;
+		meta->dynptr.parent_id = reg->parent_id;
 	}
 	return err;
 }
@@ -7731,12 +8258,12 @@ static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx,
 	return btf_param_match_suffix(meta->btf, arg, "__iter");
 }
 
-static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx,
+static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+			    u32 arg, argno_t argno, int insn_idx,
 			    struct bpf_call_arg_meta *meta)
 {
 	struct bpf_func_state *state = bpf_func(env, reg);
 	const struct btf_type *t;
-	u32 arg_idx = arg_idx_from_argno(argno);
 	int spi, err, i, nr_slots, btf_id;
 
 	if (reg->type != PTR_TO_STACK) {
@@ -7756,14 +8283,14 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *
 	 * to any kfunc, if arg has "__iter" suffix, we need to be a bit more
 	 * conservative here.
 	 */
-	btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx);
+	btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg);
 	if (btf_id < 0) {
 		verbose(env, "expected valid iter pointer as %s\n",
 			reg_arg_name(env, argno));
 		bpf_diag_call_arg(
 			env, insn_idx, argno, meta->func_name,
-			"the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type",
-			"Pass the exact iterator state type expected by this kfunc.");
+			"the function expects a recognized iterator state pointer, but this argument does not match a valid iterator type",
+			"Pass the exact iterator state type expected by this function.");
 		return -EINVAL;
 	}
 	t = btf_type_by_id(meta->btf, btf_id);
@@ -7826,7 +8353,7 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *
 		/* remember meta->iter info for process_iter_next_call() */
 		meta->iter.spi = spi;
 		meta->iter.frameno = reg->frameno;
-		update_ref_obj(&meta->ref_obj, &state->stack[spi].spilled_ptr);
+		update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr);
 
 		if (is_iter_destroy_kfunc(meta)) {
 			err = unmark_stack_slots_iter(env, reg, nr_slots);
@@ -7903,16 +8430,15 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env,
 					&fold->regs[i],
 					&fcur->regs[i]);
 
-		num_slots = min(fold->allocated_stack / BPF_REG_SIZE,
-				fcur->allocated_stack / BPF_REG_SIZE);
+		num_slots = min(bpf_stack_nr_slots(fold), bpf_stack_nr_slots(fcur));
 		for (i = 0; i < num_slots; i++) {
-			if (!bpf_is_spilled_reg(&fold->stack[i]) ||
-			    !bpf_is_spilled_reg(&fcur->stack[i]))
+			struct bpf_stack_state *old_ss = bpf_stack_slot(fold, i);
+			struct bpf_stack_state *cur_ss = bpf_stack_slot(fcur, i);
+
+			if (!bpf_is_spilled_reg(old_ss) || !bpf_is_spilled_reg(cur_ss))
 				continue;
 
-			maybe_widen_reg(env,
-					&fold->stack[i].spilled_ptr,
-					&fcur->stack[i].spilled_ptr);
+			maybe_widen_reg(env, &old_ss->spilled_ptr, &cur_ss->spilled_ptr);
 		}
 	}
 	return 0;
@@ -7924,7 +8450,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_
 	int iter_frameno = meta->iter.frameno;
 	int iter_spi = meta->iter.spi;
 
-	return &cur_st->frame[iter_frameno]->stack[iter_spi].spilled_ptr;
+	return &bpf_stack_slot(cur_st->frame[iter_frameno], iter_spi)->spilled_ptr;
 }
 
 /* process_iter_next_call() is called when verifier gets to iterator's next
@@ -8087,9 +8613,52 @@ static bool arg_type_is_dynptr(enum bpf_arg_type type)
 	return base_type(type) == ARG_PTR_TO_DYNPTR;
 }
 
+/*
+ * An argument that only ever takes a scalar, so a zero register passed to it
+ * is a value rather than a NULL pointer.
+ */
+static bool arg_type_is_scalar(enum bpf_arg_type type)
+{
+	switch (base_type(type)) {
+	case ARG_SCALAR:
+	case ARG_CONST_SCALAR:
+	case ARG_MEM_SIZE:
+	case ARG_MEM_SIZE_OR_ZERO:
+	case ARG_CONST_MEM_SIZE:
+	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/*
+ * A helper has no BTF and a nonzero function ID. A kfunc has both, while a
+ * BPF subprogram has BTF and a zero function ID.
+ */
+static bool is_helper(const struct bpf_call_arg_meta *meta)
+{
+	return !meta->btf && meta->func_id;
+}
+
+static bool is_helper_call(const struct bpf_call_arg_meta *meta, enum bpf_func_id func_id)
+{
+	return is_helper(meta) && meta->func_id == func_id;
+}
+
+static bool is_kfunc(const struct bpf_call_arg_meta *meta)
+{
+	return meta->btf && meta->func_id;
+}
+
+static bool is_kfunc_call(const struct bpf_call_arg_meta *meta, u32 btf_id)
+{
+	return is_kfunc(meta) && meta->func_id == btf_id;
+}
+
 static int resolve_map_arg_type(struct bpf_verifier_env *env,
-				 const struct bpf_call_arg_meta *meta,
-				 enum bpf_arg_type *arg_type)
+				const struct bpf_call_arg_meta *meta,
+				enum bpf_arg_type *arg_type)
 {
 	if (!meta->map.ptr) {
 		/* kernel subsystem misconfigured verifier */
@@ -8108,7 +8677,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env,
 		}
 		break;
 	case BPF_MAP_TYPE_BLOOM_FILTER:
-		if (meta->func_id == BPF_FUNC_map_peek_elem)
+		if (is_helper_call(meta, BPF_FUNC_map_peek_elem))
 			*arg_type = ARG_PTR_TO_MAP_VALUE;
 		break;
 	default:
@@ -8117,6 +8686,48 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env,
 	return 0;
 }
 
+static int resolve_func_arg_type(struct bpf_verifier_env *env,
+				 struct bpf_reg_state *reg, u32 arg, argno_t argno,
+				 struct bpf_call_arg_meta *meta,
+				 enum bpf_arg_type *arg_type, u32 *arg_size);
+static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+				     argno_t argno, enum bpf_arg_type arg_type,
+				     const struct btf *arg_btf, u32 arg_btf_id,
+				     struct bpf_call_arg_meta *meta, int insn_idx);
+static bool is_kfunc_arg_nonown_allowed(const struct btf *btf,
+					const struct btf_param *arg);
+static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
+					  const struct btf_param *arg,
+					  const char *name);
+static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta);
+static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta);
+static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta);
+static bool is_bpf_obj_drop_kfunc(u32 func_id);
+static bool is_bpf_percpu_obj_drop_kfunc(u32 func_id);
+static bool is_bpf_rbtree_add_kfunc(u32 func_id);
+static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta);
+static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env);
+static int process_irq_flag(struct bpf_verifier_env *env,
+			    struct bpf_reg_state *reg, argno_t argno,
+			    struct bpf_call_arg_meta *meta);
+static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
+					   struct bpf_reg_state *reg,
+					   argno_t argno,
+					   struct bpf_call_arg_meta *meta);
+static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
+					     struct bpf_reg_state *reg,
+					     argno_t argno,
+					     struct bpf_call_arg_meta *meta);
+static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
+					   struct bpf_reg_state *reg,
+					   argno_t argno,
+					   struct bpf_call_arg_meta *meta);
+static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
+					     struct bpf_reg_state *reg,
+					     argno_t argno,
+					     struct bpf_call_arg_meta *meta);
+static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env);
+
 struct bpf_reg_types {
 	const enum bpf_reg_type types[10];
 	u32 *btf_id;
@@ -8160,7 +8771,7 @@ static const struct bpf_reg_types mem_types = {
 	},
 };
 
-static const struct bpf_reg_types spin_lock_types = {
+static const struct bpf_reg_types map_value_or_alloc_obj_types = {
 	.types = {
 		PTR_TO_MAP_VALUE,
 		PTR_TO_BTF_ID | MEM_ALLOC,
@@ -8189,7 +8800,33 @@ static const struct bpf_reg_types percpu_btf_ptr_types = {
 static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
 static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
-static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
+static const struct bpf_reg_types map_value_types = { .types = { PTR_TO_MAP_VALUE } };
+static const struct bpf_reg_types arena_types = {
+	.types = {
+		PTR_TO_ARENA,
+		SCALAR_VALUE,
+	}
+};
+static const struct bpf_reg_types ctx_out_types = {
+	.types = { PTR_TO_MEM | MEM_RDONLY | PTR_TRUSTED },
+};
+
+static const struct bpf_reg_types alloc_obj_drop_types = {
+	.types = {
+		PTR_TO_BTF_ID | MEM_ALLOC,
+		PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU,
+	}
+};
+
+static const struct bpf_reg_types alloc_obj_types = {
+	.types = {
+		PTR_TO_BTF_ID | MEM_ALLOC,
+		PTR_TO_BTF_ID | MEM_ALLOC | MEM_RCU,
+		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF,
+		PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU,
+	}
+};
+
 static const struct bpf_reg_types kptr_xchg_dest_types = {
 	.types = {
 		PTR_TO_MAP_VALUE,
@@ -8220,16 +8857,32 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
 #endif
 	[ARG_PTR_TO_SOCKET]		= &fullsock_types,
 	[ARG_PTR_TO_BTF_ID]		= &btf_ptr_types,
-	[ARG_PTR_TO_SPIN_LOCK]		= &spin_lock_types,
+	[ARG_PTR_TO_SPIN_LOCK]		= &map_value_or_alloc_obj_types,
 	[ARG_PTR_TO_MEM]		= &mem_types,
 	[ARG_PTR_TO_RINGBUF_MEM]	= &ringbuf_mem_types,
 	[ARG_PTR_TO_PERCPU_BTF_ID]	= &percpu_btf_ptr_types,
 	[ARG_PTR_TO_FUNC]		= &func_ptr_types,
 	[ARG_PTR_TO_STACK]		= &stack_ptr_types,
 	[ARG_PTR_TO_CONST_STR]		= &const_str_ptr_types,
-	[ARG_PTR_TO_TIMER]		= &timer_types,
+	[ARG_PTR_TO_TIMER]		= &map_value_types,
 	[ARG_KPTR_XCHG_DEST]		= &kptr_xchg_dest_types,
 	[ARG_PTR_TO_DYNPTR]		= &dynptr_types,
+	[ARG_CONST_SCALAR]		= &scalar_types,
+	[ARG_CONST_MEM_SIZE]		= &scalar_types,
+	[ARG_PTR_TO_ALLOC_BTF_ID]	= &alloc_obj_drop_types,
+	[ARG_PTR_TO_REFCOUNTED_KPTR]	= &alloc_obj_types,
+	[ARG_PTR_TO_ITER]		= &stack_ptr_types,
+	[ARG_PTR_TO_LIST_HEAD]		= &map_value_or_alloc_obj_types,
+	[ARG_PTR_TO_LIST_NODE]		= &alloc_obj_types,
+	[ARG_PTR_TO_RB_ROOT]		= &map_value_or_alloc_obj_types,
+	[ARG_PTR_TO_RB_NODE]		= &alloc_obj_types,
+	[ARG_PTR_TO_RES_SPIN_LOCK]	= &map_value_or_alloc_obj_types,
+	[ARG_PTR_TO_WORKQUEUE]		= &map_value_types,
+	[ARG_PTR_TO_TASK_WORK]		= &map_value_types,
+	[ARG_PTR_TO_RCU_HEAD]		= &map_value_types,
+	[ARG_PTR_TO_IRQ_FLAG]		= &stack_ptr_types,
+	[ARG_PTR_TO_ARENA]		= &arena_types,
+	[ARG_PTR_TO_CTX_OUT]		= &ctx_out_types,
 };
 
 static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno,
@@ -8269,6 +8922,71 @@ __printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u
 	bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion);
 }
 
+static int check_func_arg_nullability(struct bpf_verifier_env *env,
+				      struct bpf_reg_state *reg, u32 arg, argno_t argno,
+				      enum bpf_arg_type arg_type,
+				      struct bpf_call_arg_meta *meta, int insn_idx)
+{
+	const char *expected_type = "pointer";
+
+	if (arg_type_is_scalar(arg_type) || type_may_be_null(arg_type) ||
+	    (!bpf_register_is_null(reg) && !type_may_be_null(reg->type)))
+		return 0;
+
+	if (meta->btf) {
+		u32 arg_btf_id;
+
+		arg_btf_id = btf_params(meta->func_proto)[arg].type;
+		expected_type = bpf_diag_fmt(env, "value of type %s",
+					     bpf_diag_fmt_btf_type(env, meta->btf, arg_btf_id));
+	}
+
+	verbose(env, "Possibly NULL pointer passed to trusted %s\n",
+		reg_arg_name(env, argno));
+	bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+			      "Add a NULL check and make the call only on the non-NULL path.",
+			      "the pointer may be NULL, but this call requires a non-NULL %s",
+			      expected_type);
+	return -EACCES;
+}
+
+static int check_func_arg_release(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+				  u32 arg, argno_t argno, enum bpf_arg_type arg_type,
+				  struct bpf_call_arg_meta *meta, int insn_idx)
+{
+	const char *expected_type = "pointer";
+
+	if (!arg_type_is_release(arg_type))
+		return 0;
+
+	if (arg_type_is_dynptr(arg_type) || reg_is_referenced(env, reg) ||
+	    bpf_register_is_null(reg))
+		return 0;
+
+	verbose(env, "release function %s expects referenced PTR_TO_BTF_ID passed to %s\n",
+		meta->func_name, reg_arg_name(env, argno));
+
+	if (is_kfunc(meta)) {
+		const struct btf_param *btf_arg;
+		const struct btf_type *t;
+		u32 ref_id;
+
+		btf_arg = &btf_params(meta->func_proto)[arg];
+		ref_id = btf_arg->type;
+		t = btf_type_skip_modifiers(meta->btf, btf_arg->type, NULL);
+		if (btf_type_is_ptr(t))
+			btf_type_skip_modifiers(meta->btf, t->type, &ref_id);
+		expected_type = bpf_diag_fmt(env, "value of type %s",
+					     bpf_diag_fmt_btf_type(env, meta->btf, ref_id));
+	}
+
+	bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+			      bpf_diag_fmt(env, "Pass the resource-owning %s returned by the matching acquire call, or avoid the release function after ownership has already been transferred or released.",
+					   expected_type),
+			      "release functions require a value that owns a live resource returned by a matching acquire function");
+	return -EINVAL;
+}
+
 static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env,
 					       const enum bpf_reg_type *types, int count)
 {
@@ -8290,19 +9008,21 @@ static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env,
 }
 
 static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
-			  enum bpf_arg_type arg_type, const u32 *arg_btf_id,
-			  struct bpf_call_arg_meta *meta, const char *call_name)
+			  enum bpf_arg_type arg_type, struct bpf_call_arg_meta *meta)
 {
 	enum bpf_reg_type expected, type = reg->type;
 	const struct bpf_reg_types *compatible;
 	const char *actual, *accepted;
-	int i, j, err;
+	int i, j;
 
 	compatible = compatible_reg_types[base_type(arg_type)];
 	if (!compatible) {
 		verifier_bug(env, "unsupported arg type %d", arg_type);
 		return -EFAULT;
 	}
+	if (is_kfunc(meta) && base_type(arg_type) == ARG_PTR_TO_BTF_ID &&
+	    (base_type(type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(type)]))
+		goto found;
 
 	/* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
 	 * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
@@ -8322,9 +9042,14 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re
 		type &= ~PTR_MAYBE_NULL;
 	if (base_type(arg_type) == ARG_PTR_TO_MEM)
 		type &= ~DYNPTR_TYPE_FLAG_MASK;
+	/* Allow allocated memory for BTF-defined ARG_PTR_TO_MEM but not helpers. */
+	if (!is_helper(meta) && base_type(arg_type) == ARG_PTR_TO_MEM &&
+	    type_is_ptr_alloc_obj(type))
+		type = PTR_TO_MEM;
 
 	/* Local kptr types are allowed as the source argument of bpf_kptr_xchg */
-	if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) {
+	if (is_helper_call(meta, BPF_FUNC_kptr_xchg) && type_is_alloc(type) &&
+	    reg_from_argno(argno) == BPF_REG_2) {
 		type &= ~MEM_ALLOC;
 		type &= ~MEM_PERCPU;
 	}
@@ -8344,115 +9069,13 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re
 	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
 	actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type));
 	accepted = bpf_diag_expected_reg_types(env, compatible->types, i);
-	bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name,
-			      "Pass a value with one of the accepted pointer or scalar types for this call.",
+	bpf_diag_call_arg_fmt(env, env->insn_idx, argno, meta->func_name,
+			      bpf_diag_fmt(env, "Pass %s.", bpf_diag_arg_type_plain(arg_type)),
 			      "it has type %s, but this argument accepts %s",
 			      actual, accepted);
 	return -EACCES;
 
 found:
-	if (base_type(reg->type) != PTR_TO_BTF_ID)
-		return 0;
-
-	if (compatible == &mem_types) {
-		if (!(arg_type & MEM_RDONLY)) {
-			verbose(env,
-				"%s() may write into memory pointed by %s type=%s\n",
-				func_id_name(meta->func_id),
-				reg_arg_name(env, argno), reg_type_str(env, reg->type));
-			return -EACCES;
-		}
-		return 0;
-	}
-
-	switch ((int)reg->type) {
-	case PTR_TO_BTF_ID:
-	case PTR_TO_BTF_ID | PTR_TRUSTED:
-	case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL:
-	case PTR_TO_BTF_ID | MEM_RCU:
-	case PTR_TO_BTF_ID | PTR_MAYBE_NULL:
-	case PTR_TO_BTF_ID | PTR_MAYBE_NULL | MEM_RCU:
-	{
-		/* For bpf_sk_release, it needs to match against first member
-		 * 'struct sock_common', hence make an exception for it. This
-		 * allows bpf_sk_release to work for multiple socket types.
-		 */
-		bool strict_type_match = arg_type_is_release(arg_type) &&
-					 meta->func_id != BPF_FUNC_sk_release;
-
-		if (type_may_be_null(reg->type) &&
-		    (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) {
-			verbose(env, "Possibly NULL pointer passed to helper %s\n",
-				reg_arg_name(env, argno));
-			bpf_diag_call_arg(
-				env, env->insn_idx, argno, call_name,
-				"the pointer may be NULL, but this call requires a non-NULL pointer",
-				"Add a NULL check and make the call only on the non-NULL path.");
-			return -EACCES;
-		}
-
-		if (!arg_btf_id) {
-			if (!compatible->btf_id) {
-				verifier_bug(env, "missing arg compatible BTF ID");
-				return -EFAULT;
-			}
-			arg_btf_id = compatible->btf_id;
-		}
-
-		if (meta->func_id == BPF_FUNC_kptr_xchg) {
-			if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno)))
-				return -EACCES;
-		} else {
-			if (arg_btf_id == BPF_PTR_POISON) {
-				verbose(env, "verifier internal error:");
-				verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n",
-					reg_arg_name(env, argno));
-				return -EACCES;
-			}
-
-			err = __check_ptr_off_reg(env, reg, argno, true);
-			if (err)
-				return err;
-
-			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id,
-						  reg->var_off.value, btf_vmlinux, *arg_btf_id,
-						  strict_type_match, !type_is_alloc(reg->type))) {
-				verbose(env, "%s is of type %s but %s is expected\n",
-					reg_arg_name(env, argno),
-					btf_type_name(reg->btf, reg->btf_id),
-					btf_type_name(btf_vmlinux, *arg_btf_id));
-				return -EACCES;
-			}
-		}
-		break;
-	}
-	case PTR_TO_BTF_ID | MEM_ALLOC:
-	case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC:
-	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
-	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
-		if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&
-		    meta->func_id != BPF_FUNC_kptr_xchg) {
-			verifier_bug(env, "unimplemented handling of MEM_ALLOC");
-			return -EFAULT;
-		}
-		/* Check if local kptr in src arg matches kptr in dst arg */
-		if (meta->func_id == BPF_FUNC_kptr_xchg) {
-			int regno = reg_from_argno(argno);
-
-			if (regno == BPF_REG_2 &&
-			    map_kptr_match_type(env, meta->kptr_field, reg, regno))
-				return -EACCES;
-		}
-		break;
-	case PTR_TO_BTF_ID | MEM_PERCPU:
-	case PTR_TO_BTF_ID | MEM_PERCPU | MEM_RCU:
-	case PTR_TO_BTF_ID | MEM_PERCPU | PTR_TRUSTED:
-		/* Handled by helper specific checks */
-		break;
-	default:
-		verifier_bug(env, "invalid PTR_TO_BTF_ID register for type match");
-		return -EFAULT;
-	}
 	return 0;
 }
 
@@ -8473,12 +9096,11 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
 	return field;
 }
 
-static int __check_func_arg_reg_off(struct bpf_verifier_env *env,
-				    const struct bpf_reg_state *reg, argno_t argno,
-				    enum bpf_arg_type arg_type,
-				    bool btf_id_fixed_off_ok)
-{
-	u32 type = reg->type;
+static int check_func_arg_reg_off(struct bpf_verifier_env *env,
+				  const struct bpf_reg_state *reg, argno_t argno,
+				  enum bpf_arg_type arg_type)
+{
+	u32 type = reg->type;
 
 	/* When referenced register is passed to release function, its fixed
 	 * offset must be 0.
@@ -8532,12 +9154,15 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env,
 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
 		/* When referenced PTR_TO_BTF_ID is passed to release function,
-		 * its fixed offset must be 0. In the other cases, fixed offset
-		 * can be non-zero unless the caller requires otherwise.
-		 * var_off always must be 0 for PTR_TO_BTF_ID, hence we still
-		 * need to do checks instead of returning.
+		 * its fixed offset must be 0. bpf_refcount_acquire() returns the
+		 * pointer it was given while incrementing the refcount at the
+		 * refcount field offset, so it needs a zero offset too. In the
+		 * other cases, fixed offset can be non-zero. var_off always must
+		 * be 0 for PTR_TO_BTF_ID, hence we still need to do checks
+		 * instead of returning.
 		 */
-		return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok);
+		return __check_ptr_off_reg(env, reg, argno,
+					   base_type(arg_type) != ARG_PTR_TO_REFCOUNTED_KPTR);
 	case PTR_TO_CTX:
 		/*
 		 * Allow fixed and variable offsets for syscall context, but
@@ -8545,7 +9170,7 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env,
 		 * otherwise we may get modified ctx in tail called programs and
 		 * global subprogs (that may act as extension prog hooks).
 		 */
-		if (arg_type != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog))
+		if (base_type(arg_type) != ARG_PTR_TO_CTX && is_var_ctx_off_allowed(env->prog))
 			return 0;
 		fallthrough;
 	default:
@@ -8553,13 +9178,6 @@ static int __check_func_arg_reg_off(struct bpf_verifier_env *env,
 	}
 }
 
-static int check_func_arg_reg_off(struct bpf_verifier_env *env,
-				  const struct bpf_reg_state *reg, argno_t argno,
-				  enum bpf_arg_type arg_type)
-{
-	return __check_func_arg_reg_off(env, reg, argno, arg_type, true);
-}
-
 static int check_arg_const_str(struct bpf_verifier_env *env,
 			       struct bpf_reg_state *reg, argno_t argno)
 {
@@ -8568,6 +9186,7 @@ static int check_arg_const_str(struct bpf_verifier_env *env,
 	int map_off;
 	u64 map_addr;
 	char *str_ptr;
+	u32 cnt;
 
 	if (reg->type != PTR_TO_MAP_VALUE)
 		return -EINVAL;
@@ -8617,6 +9236,11 @@ static int check_arg_const_str(struct bpf_verifier_env *env,
 		verbose(env, "string is not zero-terminated\n");
 		return -EINVAL;
 	}
+	/* the bytes of a pointer to a function are not known until the program is jitted */
+	if (bpf_map_range_func_ptrs(env, map, map_off, strlen(str_ptr + map_off) + 1, &cnt)) {
+		verbose(env, "string overlaps with a pointer to a function\n");
+		return -EACCES;
+	}
 	return 0;
 }
 
@@ -8627,6 +9251,7 @@ static int get_constant_map_key(struct bpf_verifier_env *env,
 				s64 *value)
 {
 	struct bpf_func_state *state = bpf_func(env, key);
+	struct bpf_stack_state *ss;
 	struct bpf_reg_state *reg;
 	int slot, spi, off;
 	int spill_size = 0;
@@ -8646,7 +9271,8 @@ static int get_constant_map_key(struct bpf_verifier_env *env,
 	slot = -stack_off - 1;
 	spi = slot / BPF_REG_SIZE;
 	off = slot % BPF_REG_SIZE;
-	stype = state->stack[spi].slot_type;
+	ss = bpf_stack_slot(state, spi);
+	stype = ss->slot_type;
 
 	/* First handle precisely tracked STACK_ZERO */
 	for (i = off; i >= 0 && stype[i] == STACK_ZERO; i--)
@@ -8657,14 +9283,14 @@ static int get_constant_map_key(struct bpf_verifier_env *env,
 	}
 
 	/* Check that stack contains a scalar spill of expected size */
-	if (!bpf_is_spilled_scalar_reg(&state->stack[spi]))
+	if (!bpf_is_spilled_scalar_reg(ss))
 		return -EOPNOTSUPP;
 	for (i = off; i >= 0 && stype[i] == STACK_SPILL; i--)
 		spill_size++;
 	if (spill_size != key_size)
 		return -EOPNOTSUPP;
 
-	reg = &state->stack[spi].spilled_ptr;
+	reg = &ss->spilled_ptr;
 	if (!tnum_is_const(reg->var_off))
 		/* Stack value not statically known */
 		return -EOPNOTSUPP;
@@ -8708,6 +9334,8 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat
 			obj_name = "timer";
 		else if (rec->task_work_off >= 0)
 			obj_name = "bpf_task_work";
+		else if (rec->rcu_head_off >= 0)
+			obj_name = "bpf_rcu_head";
 
 		verbose(env, "%s pointer in %s map_uid=%d ",
 			obj_name, reg_arg_name(env, obj_argno), meta->map.uid);
@@ -8721,65 +9349,79 @@ static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_stat
 	return 0;
 }
 
-static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
+/*
+ * MEM_WRITE alone denotes an input/output buffer. MEM_UNINIT marks an output
+ * whose incoming contents are not read.
+ */
+static enum bpf_access_type func_arg_access_type(enum bpf_arg_type arg_type)
+{
+	if (!(arg_type & MEM_WRITE))
+		return BPF_READ;
+	if (arg_type & MEM_UNINIT)
+		return BPF_WRITE;
+	return BPF_READ | BPF_WRITE;
+}
+
+static int check_func_arg(struct bpf_verifier_env *env, u32 arg, u32 slot, u32 prev_slot,
 			  struct bpf_call_arg_meta *meta,
 			  int insn_idx)
 {
+	const struct btf_param *btf_arg = is_kfunc(meta) ?
+					  &btf_params(meta->func_proto)[arg] : NULL;
 	const struct bpf_func_proto *fn = meta->fn;
-	u32 regno = BPF_REG_1 + arg;
-	struct bpf_reg_state *reg = reg_state(env, regno);
+	struct bpf_func_state *caller = cur_func(env);
+	struct bpf_reg_state *regs = cur_regs(env);
+	argno_t argno = argno_from_arg(slot + 1);
+	struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, slot);
 	enum bpf_arg_type arg_type = fn->arg_type[arg];
-	argno_t argno = argno_from_reg(regno);
-	enum bpf_reg_type type = reg->type;
-	u32 *arg_btf_id = NULL;
+	int regno = reg_from_argno(argno);
+	u32 arg_size = arg_type & MEM_FIXED_SIZE ? fn->arg_size[arg] : 0;
 	u32 key_size;
 	int err = 0;
 
-	if (arg_type == ARG_DONTCARE)
+	if (arg_type == ARG_PTR_TO_PROG_AUX) {
+		cur_aux(env)->arg_prog = regno;
 		return 0;
+	}
 
-	err = check_reg_arg(env, regno, SRC_OP);
-	if (err)
-		return err;
+	if (arg_type == ARG_IGNORE)
+		return 0;
+
+	if (regno >= 0) {
+		err = check_reg_arg(env, regno, SRC_OP);
+		if (err)
+			return err;
+	}
 
+	/* Preserve the legacy helper behavior for privileged pointer leaks. */
 	if (arg_type == ARG_ANYTHING) {
-		if (is_pointer_value(env, regno)) {
-			verbose(env, "R%d leaks addr into helper function\n",
-				regno);
+		if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
+			verbose(env, "%s leaks addr into helper function\n",
+				reg_arg_name(env, argno));
 			return -EACCES;
 		}
 		return 0;
 	}
 
-	if (type_is_pkt_pointer(type) &&
-	    !may_access_direct_pkt_data(env, fn, BPF_READ)) {
-		verbose(env, "helper access to the packet is not allowed\n");
-		return -EACCES;
-	}
+	err = resolve_func_arg_type(env, reg, arg, argno, meta, &arg_type, &arg_size);
+	if (err)
+		return err;
 
-	if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) {
-		err = resolve_map_arg_type(env, meta, &arg_type);
-		if (err)
-			return err;
-	}
+	if (arg_type_is_raw_mem(arg_type))
+		meta->arg_raw_mem.mask |= BIT(slot);
 
 	if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) {
-		/* A NULL register has a SCALAR_VALUE type, so skip
-		 * type checking.
-		 */
-		err = mark_chain_precision(env, regno);
+		err = mark_arg_precision(env, argno);
 		if (err)
 			return err;
-		goto skip_type_check;
+		return 0;
 	}
 
-	/* arg_btf_id and arg_size are in a union. */
-	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID ||
-	    base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
-		arg_btf_id = fn->arg_btf_id[arg];
+	err = check_func_arg_nullability(env, reg, arg, argno, arg_type, meta, insn_idx);
+	if (err)
+		return err;
 
-	err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta,
-			     func_id_name(meta->func_id));
+	err = check_reg_type(env, reg, argno, arg_type, meta);
 	if (err)
 		return err;
 
@@ -8787,22 +9429,27 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 	if (err)
 		return err;
 
-skip_type_check:
-	if (arg_type_is_release(arg_type) && !arg_type_is_dynptr(arg_type) &&
-	    !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) {
-		verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n",
-			func_id_name(meta->func_id), reg_arg_name(env, argno));
-		bpf_diag_call_arg(
-			env, insn_idx, argno, func_id_name(meta->func_id),
-			"release helpers require a value that owns a live resource returned by a matching acquire helper",
-			"Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released.");
-		return -EINVAL;
-	}
+	err = check_func_arg_release(env, reg, arg, argno, arg_type, meta, insn_idx);
+	if (err)
+		return err;
 
 	if (reg_is_referenced(env, reg))
 		update_ref_obj(&meta->ref_obj, reg);
 
 	switch (base_type(arg_type)) {
+	case ARG_CONST_SCALAR:
+		err = process_const_arg(env, reg, argno, meta);
+		if (err < 0) {
+			if (err == -EINVAL)
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+						      "Pass a compile-time constant or a value the verifier can prove is constant at this call.",
+						      "the function requires this scalar argument to be a verifier-known constant, but %s is variable on this path",
+						      reg_arg_name(env, argno));
+			return err;
+		}
+		break;
+	case ARG_SCALAR:
+		break;
 	case ARG_CONST_MAP_PTR:
 		/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
 		err = process_map_ptr_arg(env, reg, argno, meta);
@@ -8824,7 +9471,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			return -EFAULT;
 		}
 		key_size = meta->map.ptr->key_size;
-		err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL,
+		err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, meta,
 					      NULL);
 		if (err)
 			return err;
@@ -8852,21 +9499,83 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			return -EFAULT;
 		}
 
-		/*
-		 * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads
-		 * the value buffer as an input rather than filling it.
-		 */
-		if (meta->func_id == BPF_FUNC_map_peek_elem &&
-		    meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER)
-			meta->arg_raw_mem.regno = 0;
-
 		err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size,
 					      arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
 					      false, meta, NULL);
 		break;
+	case ARG_PTR_TO_BTF_ID:
+	case ARG_PTR_TO_BTF_ID_SOCK_COMMON:
+	{
+		const u32 *arg_btf_id = fn->arg_btf_id[arg];
+		const struct btf *arg_btf = is_kfunc(meta) ? meta->btf : btf_vmlinux;
+
+		if (!meta->btf) {
+			const struct bpf_reg_types *compatible;
+
+			if (base_type(reg->type) != PTR_TO_BTF_ID)
+				break;
+
+			if (is_helper_call(meta, BPF_FUNC_kptr_xchg))
+				return map_kptr_match_type(env, meta->kptr_field, reg, regno) ?
+				       -EACCES : 0;
+
+			if (!arg_btf_id) {
+				compatible = compatible_reg_types[base_type(arg_type)];
+				if (!compatible->btf_id) {
+					verifier_bug(env, "missing arg compatible BTF ID");
+					return -EFAULT;
+				}
+				arg_btf_id = compatible->btf_id;
+			}
+			if (arg_btf_id == BPF_PTR_POISON) {
+				verbose(env, "verifier internal error:");
+				verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n",
+					reg_arg_name(env, argno));
+				return -EACCES;
+			}
+		}
+
+		if (is_kfunc(meta) && (!is_trusted_reg(env, reg) ||
+				       bpf_type_has_unsafe_modifiers(reg->type))) {
+			if (!(arg_type & MEM_RCU)) {
+				const char *actual_type, *arg_name, *expected_type;
+
+				expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id);
+				verbose(env, "%s must be referenced or trusted\n",
+					reg_arg_name(env, argno));
+				arg_name = reg_arg_name(env, argno);
+				actual_type = bpf_diag_reg_type_plain(env, reg->type);
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+						      "Pass a pointer acquired from a verifier-tracked source, or call this function only inside the required protection if it accepts RCU pointers.",
+						      "the function requires a trusted or resource-owning pointer to %s, but %s is %s",
+						      expected_type, arg_name, actual_type);
+				return -EINVAL;
+			}
+			if (!is_rcu_reg(reg)) {
+				const char *actual_type, *arg_name, *expected_type;
+
+				expected_type = bpf_diag_fmt_btf_type(env, arg_btf, *arg_btf_id);
+				verbose(env, "%s must be a rcu pointer\n",
+					reg_arg_name(env, argno));
+				arg_name = reg_arg_name(env, argno);
+				actual_type = bpf_diag_reg_type_plain(env, reg->type);
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+						      "Use this function with a pointer that is valid in an RCU read lock region.",
+						      "the function requires an RCU-protected pointer to %s, but %s is %s",
+						      expected_type, arg_name, actual_type);
+				return -EINVAL;
+			}
+		}
+
+		err = process_arg_ptr_to_btf_id(env, reg, argno, arg_type, arg_btf,
+						*arg_btf_id, meta, insn_idx);
+		if (err < 0)
+			return err;
+		break;
+	}
 	case ARG_PTR_TO_PERCPU_BTF_ID:
 		if (!reg->btf_id) {
-			verbose(env, "Helper has invalid btf_id in R%d\n", regno);
+			verbose(env, "Helper has invalid btf_id in %s\n", reg_arg_name(env, argno));
 			return -EACCES;
 		}
 		meta->ret_btf = reg->btf;
@@ -8877,11 +9586,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			verbose(env, "can't spin_{lock,unlock} in rbtree cb\n");
 			return -EACCES;
 		}
-		if (meta->func_id == BPF_FUNC_spin_lock) {
+		if (is_helper_call(meta, BPF_FUNC_spin_lock)) {
 			err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK);
 			if (err)
 				return err;
-		} else if (meta->func_id == BPF_FUNC_spin_unlock) {
+		} else if (is_helper_call(meta, BPF_FUNC_spin_unlock)) {
 			err = process_spin_lock(env, reg, argno, 0);
 			if (err)
 				return err;
@@ -8895,45 +9604,281 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 		if (err)
 			return err;
 		break;
+	case ARG_PTR_TO_CTX:
+		if (is_bpf_cast_to_kern_ctx_kfunc(meta)) {
+			err = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));
+			if (err < 0)
+				return -EINVAL;
+			meta->ret_btf_id = err;
+		}
+		break;
+	case ARG_PTR_TO_ARENA:
+		break;
+	case ARG_PTR_TO_ALLOC_BTF_ID:
+		if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) {
+			if (!is_bpf_obj_drop_kfunc(meta->func_id)) {
+				verbose(env, "%s expected for bpf_obj_drop()\n",
+					reg_arg_name(env, argno));
+				return -EINVAL;
+			}
+		} else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) {
+			if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) {
+				verbose(env, "%s expected for bpf_percpu_obj_drop()\n",
+					reg_arg_name(env, argno));
+				return -EINVAL;
+			}
+		}
+		if (!reg_is_referenced(env, reg)) {
+			verbose(env, "allocated object must be referenced\n");
+			bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+					      "Pass the owned object pointer before it is released or transferred.",
+					      "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource",
+					      reg_arg_name(env, argno));
+			return -EINVAL;
+		}
+		if (meta->btf == btf_vmlinux) {
+			meta->arg_btf = reg->btf;
+			meta->arg_btf_id = reg->btf_id;
+		}
+		break;
 	case ARG_PTR_TO_FUNC:
 		meta->subprogno = reg->subprogno;
 		break;
 	case ARG_PTR_TO_MEM:
+	{
+		enum bpf_access_type access_type;
+		bool known_memory;
+
 		/* The access to this pointer is only checked when we hit the
 		 * next is_mem_size argument below.
 		 */
-		if (arg_type & MEM_FIXED_SIZE) {
-			err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg],
-					    arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL);
-			if (err)
-				return err;
-			if (arg_type & MEM_ALIGNED)
-				err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true);
+		if (!(arg_type & MEM_FIXED_SIZE))
+			break;
+
+		access_type = func_arg_access_type(arg_type);
+
+		err = check_mem_reg(env, reg, argno, arg_size, access_type, meta, &known_memory);
+		if (err < 0) {
+			if (known_memory)
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+					"Pass memory with at least the required number of accessible bytes and suitable read or write access.",
+					"the function expects %u bytes of memory, but the verifier cannot prove that %s provides a range of that size with the required read or write access",
+					arg_size,
+					bpf_diag_reg_type_plain(env, reg->type));
+			else
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+					"Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.",
+					"the function expects %u bytes of memory, but it is %s and not verifier-known memory",
+					arg_size,
+					bpf_diag_reg_type_plain(env, reg->type));
+			return err;
+		}
+		/*
+		 * PTR_TO_PACKET gets passed as PTR_TO_MEM, preventing us from adjusting
+		 * bounds tracking information.
+		 */
+		if (is_subprog(meta) && meta->subprog_may_change_pkt &&
+		    (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg))) {
+			verbose(env,
+				"cannot pass packet pointer %s to %s(): function may change packet data\n",
+				reg_arg_name(env, argno), meta->func_name);
+			return -EINVAL;
 		}
+		if (arg_type & MEM_ALIGNED)
+			err = check_ptr_alignment(env, reg, 0, arg_size, true);
 		break;
+	}
+	case ARG_CONST_MEM_SIZE:
+		err = process_const_arg(env, reg, argno, meta);
+		if (err < 0) {
+			if (err == -EINVAL)
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+						      "Pass a compile-time constant or a value the verifier can prove is constant at this call.",
+						      "the function requires this memory size to be a verifier-known constant, but %s is variable on this path",
+						      reg_arg_name(env, argno));
+			return err;
+		}
+		fallthrough;
 	case ARG_MEM_SIZE:
-		err = check_mem_size_reg(env, reg_state(env, regno - 1), reg,
-					 argno_from_reg(regno - 1), argno,
-					 fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ,
-					 false, meta, NULL);
-		break;
 	case ARG_MEM_SIZE_OR_ZERO:
-		err = check_mem_size_reg(env, reg_state(env, regno - 1), reg,
-					 argno_from_reg(regno - 1), argno,
-					 fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ,
-					 true, meta, NULL);
+	{
+		struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, prev_slot);
+		argno_t buff_argno = argno_from_arg(prev_slot + 1);
+		enum bpf_mem_size_failure failure;
+		const char *buff_arg, *size_arg;
+		bool zero_size_allowed;
+		u32 access_type;
+
+		if (meta->btf && bpf_register_is_null(buff_reg))
+			break;
+
+		access_type = func_arg_access_type(fn->arg_type[arg - 1]);
+
+		zero_size_allowed = meta->btf || base_type(arg_type) == ARG_MEM_SIZE_OR_ZERO;
+
+		err = check_mem_size_reg(env, buff_reg, reg, buff_argno, argno,
+					 access_type, zero_size_allowed, meta, &failure);
+		if (!err)
+			break;
+
+		buff_arg = bpf_diag_arg_name(env, buff_argno);
+		size_arg = bpf_diag_arg_name(env, argno);
+		verbose(env, "%s and ", reg_arg_name(env, buff_argno));
+		verbose(env, "%s memory, len pair leads to invalid memory access\n",
+			reg_arg_name(env, argno));
+		if (failure == BPF_MEM_SIZE_FAIL_MEMORY) {
+			bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, meta->func_name,
+					      "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.",
+					      "it is the memory pointer in a memory/length pair with %s, but %s does not provide a verifier-accessible range of the requested length",
+					      size_arg, buff_arg);
+		} else if (failure == BPF_MEM_SIZE_FAIL_SIZE) {
+			if (reg_smin(reg) < 0)
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+					"Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.",
+					"the memory size in %s may be negative because its signed minimum is %lld",
+					size_arg, reg_smin(reg));
+			else if (!zero_size_allowed && reg_umin(reg) == 0)
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+					"Ensure the memory size is non-zero before this call.",
+					"the memory size in %s may be zero, but the function requires a non-zero size",
+					size_arg);
+			else
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+					"Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.",
+					"the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes",
+					size_arg, reg_umax(reg), BPF_MAX_VAR_SIZ);
+		}
 		break;
-	case ARG_PTR_TO_DYNPTR:
-		err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id),
-					  arg_type, &meta->ref_obj, &meta->dynptr);
+	}
+	case ARG_PTR_TO_DYNPTR: {
+		if (is_bpf_dynptr_clone_kfunc(meta) &&
+		    (arg_type & MEM_UNINIT)) {
+			enum bpf_dynptr_type parent_type = meta->dynptr.type;
+
+			if (parent_type == BPF_DYNPTR_TYPE_INVALID) {
+				verifier_bug(env, "no dynptr type for parent of clone");
+				return -EFAULT;
+			}
+
+			arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
+		}
+
+		err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, meta);
 		if (err)
 			return err;
 		break;
+	}
+	case ARG_PTR_TO_ITER:
+		if (is_bpf_iter_css_task_new_kfunc(meta) &&
+		    !check_css_task_iter_allowlist(env)) {
+			verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");
+			return -EINVAL;
+		}
+		err = process_iter_arg(env, reg, arg, argno, insn_idx, meta);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_LIST_HEAD:
+		if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
+		    !reg_is_referenced(env, reg)) {
+			verbose(env, "allocated object must be referenced\n");
+			return -EINVAL;
+		}
+		err = process_kf_arg_ptr_to_list_head(env, reg, argno, meta);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_RB_ROOT:
+		if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
+		    !reg_is_referenced(env, reg)) {
+			verbose(env, "allocated object must be referenced\n");
+			return -EINVAL;
+		}
+		err = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_LIST_NODE:
+		if (!(is_kfunc_arg_nonown_allowed(meta->btf, btf_arg) &&
+		      type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg))) {
+			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
+				verbose(env, "%s expected pointer to allocated object\n",
+					reg_arg_name(env, argno));
+				return -EINVAL;
+			}
+			if (!reg_is_referenced(env, reg)) {
+				verbose(env, "allocated object must be referenced\n");
+				return -EINVAL;
+			}
+		}
+		err = process_kf_arg_ptr_to_list_node(env, reg, argno, meta);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_RB_NODE:
+		if (is_bpf_rbtree_add_kfunc(meta->func_id)) {
+			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
+				verbose(env, "%s expected pointer to allocated object\n",
+					reg_arg_name(env, argno));
+				return -EINVAL;
+			}
+			if (!reg_is_referenced(env, reg)) {
+				verbose(env, "allocated object must be referenced\n");
+				return -EINVAL;
+			}
+		} else {
+			if (!type_is_non_owning_ref(reg->type) &&
+			    !reg_is_referenced(env, reg)) {
+				verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n",
+					meta->func_name);
+				return -EINVAL;
+			}
+			if (in_rbtree_lock_required_cb(env)) {
+				verbose(env, "%s not allowed in rbtree cb\n", meta->func_name);
+				return -EINVAL;
+			}
+		}
+		err = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta);
+		if (err < 0)
+			return err;
+		break;
 	case ARG_CONST_ALLOC_SIZE_OR_ZERO:
+		if (meta->btf && is_kfunc_arg_scalar_with_name(meta->btf, btf_arg,
+							       "rdonly_buf_size"))
+			meta->r0_rdonly = true;
 		err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem);
-		if (err)
+		if (err < 0) {
+			if (meta->btf && err == -EINVAL)
+				bpf_diag_call_arg_fmt(env, insn_idx, argno, meta->func_name,
+						      "Pass a verifier-known constant size for this function's buffer argument.",
+						      "the function uses this argument as a return-buffer size, but %s is invalid or variable on this path",
+						      reg_arg_name(env, argno));
 			return err;
+		}
+		break;
+	case ARG_PTR_TO_REFCOUNTED_KPTR:
+	{
+		struct btf_record *rec;
+
+		if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg))
+			meta->arg_owning_ref = true;
+
+		rec = reg_btf_record(reg);
+		if (!rec) {
+			verifier_bug(env, "Couldn't find btf_record");
+			return -EFAULT;
+		}
+
+		if (rec->refcount_off < 0) {
+			verbose(env, "%s doesn't point to a type with bpf_refcount field\n",
+				reg_arg_name(env, argno));
+			return -EINVAL;
+		}
+
+		meta->arg_btf = reg->btf;
+		meta->arg_btf_id = reg->btf_id;
 		break;
+	}
 	case ARG_PTR_TO_CONST_STR:
 	{
 		err = check_arg_const_str(env, reg, argno);
@@ -8941,6 +9886,50 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			return err;
 		break;
 	}
+	case ARG_PTR_TO_WORKQUEUE:
+		err = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_TASK_WORK:
+		err = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_RCU_HEAD:
+		err = check_map_field_pointer(env, reg, argno, BPF_RCU_HEAD, &meta->map);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_IRQ_FLAG:
+		err = process_irq_flag(env, reg, argno, meta);
+		if (err < 0)
+			return err;
+		break;
+	case ARG_PTR_TO_CTX_OUT:
+		if (reg->mem_size != arg_size) {
+			verbose(env, "%s expected %u bytes of ctx-provided memory, got %u\n",
+				reg_arg_name(env, argno), arg_size, reg->mem_size);
+			return -EINVAL;
+		}
+		break;
+	case ARG_PTR_TO_RES_SPIN_LOCK:
+	{
+		int flags;
+
+		if (in_rbtree_lock_required_cb(env)) {
+			verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n");
+			return -EACCES;
+		}
+
+		flags = get_bpf_res_spin_lock_kfunc_flags(meta);
+		if (!flags)
+			return -EFAULT;
+		err = process_spin_lock(env, reg, argno, flags);
+		if (err < 0)
+			return err;
+		break;
+	}
 	case ARG_KPTR_XCHG_DEST:
 		err = process_kptr_func(env, regno, meta);
 		if (err)
@@ -8951,19 +9940,152 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 	return err;
 }
 
-static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
+/*
+ * The slots a parameter takes, from its BTF type. This has to agree with
+ * btf_func_model_arg_slots(), which answers the same from the size the
+ * func model recorded, or the verifier would check an argument at a slot
+ * the JIT does not place it at.
+ */
+static u32 btf_arg_slots(const struct btf_type *t)
 {
-	enum bpf_attach_type eatype = env->prog->expected_attach_type;
-	enum bpf_prog_type type = resolve_prog_type(env->prog);
+	if (btf_type_is_int(t) || btf_type_is_struct(t))
+		return (t->size + BPF_REG_SIZE - 1) / BPF_REG_SIZE;
+	return 1;
+}
 
-	if (func_id != BPF_FUNC_map_update_elem &&
-	    func_id != BPF_FUNC_map_delete_elem)
-		return false;
+static u32 btf_proto_slots(const struct btf *btf, const struct btf_type *func_proto)
+{
+	const struct btf_param *args = btf_params(func_proto);
+	u32 i, nargs = btf_type_vlen(func_proto), slots_used = 0;
 
-	/* It's not possible to get access to a locked struct sock in these
-	 * contexts, so updating is safe.
-	 */
-	switch (type) {
+	for (i = 0; i < nargs; i++)
+		slots_used += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL));
+
+	return slots_used;
+}
+
+/* The argument slot @slot holds an eightbyte of a by-value argument. */
+static int check_arg_extra_slot(struct bpf_verifier_env *env, struct bpf_func_state *caller,
+				u32 slot, struct bpf_call_arg_meta *meta)
+{
+	struct bpf_reg_state *reg = get_func_arg_reg(caller, cur_regs(env), slot);
+	argno_t argno = argno_from_arg(slot + 1);
+	int regno = reg_from_argno(argno);
+	int err;
+
+	if (regno >= 0) {
+		err = check_reg_arg(env, regno, SRC_OP);
+		if (err)
+			return err;
+	}
+
+	return check_reg_type(env, reg, argno, ARG_SCALAR, meta);
+}
+
+static int check_func_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
+			   int insn_idx)
+{
+	u32 slot = 0, prev_slot = 0, proto_slots, nslots;
+	struct bpf_func_state *caller = cur_func(env);
+	const struct btf_param *args = NULL;
+	u32 arg, nargs = MAX_BPF_FUNC_REG_ARGS;
+	int err;
+
+	if (meta->btf) {
+		args = btf_params(meta->func_proto);
+		nargs = btf_type_vlen(meta->func_proto);
+	}
+
+	/*
+	 * A by-value argument of more than one eightbyte takes a slot per
+	 * eightbyte, so the slots a call occupies are no longer its parameter
+	 * count. Only a proto whose parameters take a slot each can name the
+	 * argument a stack slot belongs to.
+	 */
+	proto_slots = meta->btf ? btf_proto_slots(meta->btf, meta->func_proto) : nargs;
+
+	if (proto_slots > MAX_BPF_FUNC_REG_ARGS) {
+		err = check_outgoing_stack_args(env, caller, proto_slots, meta->func_name,
+						meta->btf, proto_slots == nargs ? args : NULL);
+		if (err)
+			return err;
+	}
+
+	for (arg = 0; arg < nargs; arg++, prev_slot = slot, slot += nslots) {
+		const struct btf_type *t;
+		u32 k;
+
+		nslots = 1;
+		if (args) {
+			t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL);
+			nslots = btf_arg_slots(t);
+		}
+
+		if (meta->fn->arg_type[arg] == ARG_UNUSED)
+			break;
+		err = check_func_arg(env, arg, slot, prev_slot, meta, insn_idx);
+		if (err)
+			return err;
+
+		/*
+		 * check_func_arg() took the first slot. A parameter of more
+		 * than one eightbyte is always a scalar, so every slot it
+		 * takes beyond the first holds one too.
+		 */
+		for (k = 1; k < nslots; k++) {
+			err = check_arg_extra_slot(env, caller, slot + k, meta);
+			if (err)
+				return err;
+		}
+	}
+
+	return 0;
+}
+
+static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
+			  int insn_idx)
+{
+	struct bpf_func_state *caller = cur_func(env);
+	u32 slot;
+	int i, err;
+
+	/*
+	 * Validate every argument before initializing outputs: an input argument
+	 * may alias an output buffer. Use the normal stack-write checks to discard
+	 * stale spills and preserve the rules for special stack objects.
+	 */
+	for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) {
+		struct bpf_reg_state *reg;
+		argno_t argno = argno_from_arg(slot + 1);
+
+		if (!meta->arg_raw_mem.size[slot])
+			continue;
+		reg = get_func_arg_reg(caller, cur_regs(env), slot);
+
+		for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) {
+			err = check_mem_access(env, insn_idx, reg, argno, i, BPF_B,
+					       BPF_WRITE, -1, false, false);
+			if (err)
+				return err;
+		}
+	}
+
+	return 0;
+}
+
+static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
+{
+	enum bpf_attach_type eatype = env->prog->expected_attach_type;
+	enum bpf_prog_type type = resolve_prog_type(env->prog);
+
+	if (func_id != BPF_FUNC_map_update_elem &&
+	    func_id != BPF_FUNC_map_delete_elem)
+		return false;
+
+	/* It's not possible to get access to a locked struct sock in these
+	 * contexts, so updating is safe.
+	 */
+	switch (type) {
 	case BPF_PROG_TYPE_TRACING:
 		if (eatype == BPF_TRACE_ITER)
 			return true;
@@ -9243,23 +10365,6 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
 	return -EINVAL;
 }
 
-static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
-		if (fn->arg_type[i] == ARG_DONTCARE)
-			break;
-		if (!arg_type_is_raw_mem(fn->arg_type[i]))
-			continue;
-		if (meta->arg_raw_mem.regno)
-			return false;
-		meta->arg_raw_mem.regno = i + 1;
-	}
-
-	return true;
-}
-
 static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
 {
 	bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
@@ -9298,7 +10403,7 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
-		if (fn->arg_type[i] == ARG_DONTCARE)
+		if (fn->arg_type[i] == ARG_UNUSED)
 			break;
 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID)
 			return !!fn->arg_btf_id[i];
@@ -9321,7 +10426,7 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn)
 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
 		enum bpf_arg_type arg_type = fn->arg_type[i];
 
-		if (arg_type == ARG_DONTCARE)
+		if (arg_type == ARG_UNUSED)
 			break;
 		if (base_type(arg_type) != ARG_PTR_TO_MEM)
 			continue;
@@ -9339,7 +10444,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_
 	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
 		enum bpf_arg_type arg_type = fn->arg_type[i];
 
-		if (arg_type == ARG_DONTCARE)
+		if (arg_type == ARG_UNUSED)
 			break;
 		if (arg_type_is_release(arg_type)) {
 			if (meta->release_regno)
@@ -9351,9 +10456,41 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_
 	return true;
 }
 
-static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta)
+static bool check_arg_prog_aux(struct bpf_verifier_env *env,
+			       const struct bpf_func_proto *proto)
+{
+	bool seen = false;
+	argno_t argno;
+	u32 i;
+
+	for (i = 0; i < ARRAY_SIZE(proto->arg_type); i++) {
+		if (proto->arg_type[i] == ARG_UNUSED)
+			break;
+		if (proto->arg_type[i] != ARG_PTR_TO_PROG_AUX)
+			continue;
+
+		if (seen) {
+			verifier_bug(env, "Only 1 prog->aux argument supported");
+			return false;
+		}
+
+		argno = argno_from_arg(i + 1);
+		if (reg_from_argno(argno) < 0) {
+			verbose(env, "%s prog->aux cannot be a stack argument\n",
+				reg_arg_name(env, argno));
+			return false;
+		}
+
+		seen = true;
+	}
+
+	return true;
+}
+
+static int check_func_proto(struct bpf_verifier_env *env, const struct bpf_func_proto *fn,
+			    struct bpf_call_arg_meta *meta)
 {
-	return check_raw_mode_ok(fn, meta) &&
+	return check_arg_prog_aux(env, fn) &&
 	       check_arg_pair_ok(fn) &&
 	       check_mem_arg_rw_flag_ok(fn) &&
 	       check_proto_release_reg(fn, meta) &&
@@ -9573,7 +10710,8 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id)
 			continue;
 		if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {
 			bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE);
-			reg->id = 0;
+			if (!type_may_be_null(reg->type))
+				reg->id = 0;
 			reg->type &= ~MEM_ALLOC;
 			reg->type |= MEM_RCU;
 			bpf_diag_mod_end(env);
@@ -9665,23 +10803,56 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls
 	return err;
 }
 
-static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
-				    const struct btf *btf,
-				    struct bpf_reg_state *regs)
+static void gen_subprog_arg_proto(const struct bpf_subprog_info *sub, const struct btf *btf,
+				  const struct btf_type *func_proto, struct bpf_func_proto *proto)
+{
+	const struct btf_param *args = btf_params(func_proto);
+	u32 arg, slot = 0;
+
+	memset(proto, 0, sizeof(*proto));
+	for (arg = 0; arg < btf_type_vlen(func_proto); arg++) {
+		enum bpf_arg_type arg_type = sub->args[slot].arg_type;
+		const struct btf_type *t;
+
+		if (arg_type & PTR_UNTRUSTED) {
+			/*
+			 * An __arg_untrusted argument accepts any caller value. The
+			 * callee treats it as read-only and uses probe-read instructions
+			 * to protect against invalid memory access.
+			 */
+			arg_type = ARG_IGNORE;
+		} else if (base_type(arg_type) == ARG_PTR_TO_ARENA) {
+			arg_type |= PTR_MAYBE_NULL;
+		} else if (base_type(arg_type) == ARG_PTR_TO_MEM) {
+			proto->arg_size[arg] = sub->args[slot].mem_size;
+			arg_type |= MEM_FIXED_SIZE | MEM_WRITE;
+		} else if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) {
+			proto->arg_btf_id[arg] = &sub->args[slot].btf_id;
+		}
+		proto->arg_type[arg] = arg_type;
+		t = btf_type_skip_modifiers(btf, args[arg].type, NULL);
+		slot += btf_arg_slots(t);
+	}
+}
+
+static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct btf *btf)
 {
 	struct bpf_subprog_info *sub = subprog_info(env, subprog);
 	struct bpf_func_state *caller = cur_func(env);
-	struct bpf_verifier_log *log = &env->log;
-	struct ref_obj_desc ref_obj = {};
-	const struct btf_param *args;
 	const struct btf_type *func, *func_proto;
-	u32 i;
+	struct bpf_call_arg_meta meta;
+	struct bpf_func_proto *fn;
 	int ret, err;
 
+	memset(&meta, 0, sizeof(meta));
+	meta.btf = btf;
+	meta.subprog_may_change_pkt = sub->changes_pkt_data;
+	meta.func_name = bpf_subprog_name(env, subprog);
+
 	ret = btf_prepare_func_args(env, subprog);
 	if (ret) {
 		if (bpf_in_stack_arg_cnt(sub) > 0) {
-			err = check_outgoing_stack_args(env, caller, sub->arg_cnt,
+			err = check_outgoing_stack_args(env, caller, sub->arg_slot_cnt,
 							bpf_subprog_name(env, subprog),
 							NULL, NULL);
 			if (err)
@@ -9692,115 +10863,24 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
 
 	func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id);
 	func_proto = btf_type_by_id(btf, func->type);
-	args = btf_params(func_proto);
-	ret = check_outgoing_stack_args(env, caller, sub->arg_cnt,
-					bpf_subprog_name(env, subprog), btf, args);
-	if (ret)
-		return ret;
 
-	/* check that BTF function arguments match actual types that the
-	 * verifier sees.
-	 */
-	for (i = 0; i < sub->arg_cnt; i++) {
-		argno_t argno = argno_from_arg(i + 1);
-		struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i);
-		struct bpf_subprog_arg_info *arg = &sub->args[i];
-
-		if (arg->arg_type == ARG_ANYTHING) {
-			if (reg->type != SCALAR_VALUE) {
-				bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-		} else if (arg->arg_type & PTR_UNTRUSTED) {
-			/*
-			 * Anything is allowed for untrusted arguments, as these are
-			 * read-only and probe read instructions would protect against
-			 * invalid memory access.
-			 */
-		} else if (arg->arg_type == ARG_PTR_TO_CTX) {
-			ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX);
-			if (ret < 0)
-				return ret;
-			/* If function expects ctx type in BTF check that caller
-			 * is passing PTR_TO_CTX.
-			 */
-			if (reg->type != PTR_TO_CTX) {
-				bpf_log(log, "%s expects pointer to ctx\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-		} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
-			ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE);
-			if (ret < 0)
-				return ret;
-			if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL,
-					  NULL))
-				return -EINVAL;
-			if (!(arg->arg_type & PTR_MAYBE_NULL) &&
-			    (type_may_be_null(reg->type) || bpf_register_is_null(reg))) {
-				bpf_log(log, "%s is expected to be non-NULL\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-		} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
-			/*
-			 * Can pass any value and the kernel won't crash, but
-			 * only PTR_TO_ARENA or SCALAR make sense. Everything
-			 * else is a bug in the bpf program. Point it out to
-			 * the user at the verification time instead of
-			 * run-time debug nightmare.
-			 */
-			if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) {
-				bpf_log(log, "%s is not a pointer to arena or scalar.\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-		} else if (arg->arg_type == ARG_PTR_TO_DYNPTR) {
-			ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR);
-			if (ret)
-				return ret;
-
-			ret = process_dynptr_func(env, reg, argno, env->insn_idx,
-						  bpf_subprog_name(env, subprog), arg->arg_type,
-						  &ref_obj, NULL);
-			if (ret)
-				return ret;
-		} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
-			struct bpf_call_arg_meta meta;
-			int err;
-
-			if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) {
-				err = mark_arg_precision(env, argno);
-				if (err)
-					return err;
-				continue;
-			}
-
-			memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
-			err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta,
-					     bpf_subprog_name(env, subprog));
-			err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type);
-			if (err)
-				return err;
-		} else {
-			verifier_bug(env, "unrecognized %s type %d",
-				     reg_arg_name(env, argno), arg->arg_type);
-			return -EFAULT;
-		}
-	}
+	fn = &env->bpf_subprog_scratch;
+	gen_subprog_arg_proto(sub, btf, func_proto, fn);
+	meta.fn = fn;
+	meta.func_proto = func_proto;
 
-	return 0;
+	return check_func_args(env, &meta, env->insn_idx);
 }
 
-/* Compare BTF of a function call with given bpf_reg_state.
+/*
+ * Check that call-site argument states match a subprog's BTF signature.
+ *
  * Returns:
  * EFAULT - there is a verifier bug. Abort verification.
- * EINVAL - there is a type mismatch or BTF is not available.
+ * Other errors - there is a type mismatch or BTF is not available.
  * 0 - BTF matches with what bpf_reg_state expects.
- * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
  */
-static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
-				  struct bpf_reg_state *regs)
+static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog)
 {
 	struct bpf_prog *prog = env->prog;
 	struct btf *btf = prog->aux->btf;
@@ -9817,7 +10897,7 @@ static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
 	if (prog->aux->func_info_aux[subprog].unreliable)
 		return -EINVAL;
 
-	err = btf_check_func_arg_match(env, subprog, btf, regs);
+	err = btf_check_func_arg_match(env, subprog, btf);
 	/* Compiler optimizations can remove arguments from static functions
 	 * or mismatched type can be passed into a global function.
 	 * In such cases mark the function as unreliable from BTF point of view.
@@ -9836,7 +10916,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 	int err;
 
 	caller = state->frame[state->curframe];
-	err = btf_check_subprog_call(env, subprog, caller->regs);
+	err = btf_check_subprog_call(env, subprog);
 	if (err == -EFAULT)
 		return err;
 
@@ -9868,7 +10948,7 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 		if (IS_ERR(async_cb))
 			return PTR_ERR(async_cb);
 		callee = async_cb->frame[0];
-		callee->async_entry_cnt = caller->async_entry_cnt + 1;
+		callee->async_entry_cnt = state->frame[0]->async_entry_cnt + 1;
 
 		/* Convert bpf_timer_set_callback() args into timer callback args */
 		err = set_callee_state_cb(env, caller, callee, insn_idx);
@@ -9878,6 +10958,15 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 		return 0;
 	}
 
+	/*
+	 * The address of the callback might be taken in another function or
+	 * read from a map, see check_func_ptr_read(). Like with callx it's
+	 * known here what is called.
+	 */
+	err = record_callx_edge(env, caller->subprogno, subprog);
+	if (err)
+		return err;
+
 	/* for callback functions enqueue entry to callback and
 	 * proceed with next instruction within current frame.
 	 */
@@ -9899,14 +10988,63 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 static int process_bpf_exit_full(struct bpf_verifier_env *env,
 				 bool *do_print_state, bool exception_exit);
 
-static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
-			   int *insn_idx)
+/*
+ * Call of a static subprog. The callee is verified in the context of
+ * the caller, hence set up a new frame and continue from the first
+ * instruction of the callee.
+ */
+static int check_static_func_call(struct bpf_verifier_env *env, int subprog,
+				  int *insn_idx)
 {
 	struct bpf_verifier_state *state = env->cur_state;
 	struct bpf_subprog_info *caller_info;
 	u16 callee_incoming, stack_arg_cnt;
 	struct bpf_func_state *caller;
+	int err;
+
+	caller = state->frame[state->curframe];
+
+	/*
+	 * Track caller's total stack arg count (incoming + max outgoing).
+	 * This is needed so the JIT knows how much stack arg space to allocate.
+	 */
+	caller_info = &env->subprog_info[caller->subprogno];
+	callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]);
+	stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming;
+	if (stack_arg_cnt > caller_info->stack_arg_cnt)
+		caller_info->stack_arg_cnt = stack_arg_cnt;
+
+	/*
+	 * For regular function entry setup new frame and continue
+	 * from that frame.
+	 */
+	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
+	if (err)
+		return err;
+
+	bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
+	clear_caller_saved_regs(env, caller->regs);
+
+	/* and go analyze first insn of the callee */
+	*insn_idx = env->subprog_info[subprog].start - 1;
+
+	if (env->log.level & BPF_LOG_LEVEL) {
+		verbose(env, "caller:\n");
+		print_verifier_state(env, state, caller->frameno, true);
+		verbose(env, "callee:\n");
+		print_verifier_state(env, state, state->curframe, true);
+	}
+
+	return 0;
+}
+
+static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
+			   int *insn_idx)
+{
+	struct bpf_verifier_state *state = env->cur_state;
+	struct bpf_func_state *caller;
 	int err, subprog, target_insn;
+	u32 i, nregs;
 
 	target_insn = *insn_idx + insn->imm + 1;
 	subprog = bpf_find_subprog(env, target_insn);
@@ -9915,10 +11053,11 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		return -EFAULT;
 
 	caller = state->frame[state->curframe];
-	err = btf_check_subprog_call(env, subprog, caller->regs);
+	err = btf_check_subprog_call(env, subprog);
 	if (err == -EFAULT)
 		return err;
 	if (bpf_subprog_is_global(env, subprog)) {
+		struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog);
 		const char *sub_name = bpf_subprog_name(env, subprog);
 		const char *operation;
 		bool returns_void;
@@ -9933,11 +11072,16 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 
 		if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) {
+			const char *suggestion;
+
 			verbose(env, "sleepable global function %s() called in %s\n",
 				sub_name, non_sleepable_context_description(env));
+			if (in_sleepable(env))
+				suggestion = "Move the call outside the critical section, or use a non-sleepable function.";
+			else
+				suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable function.";
 			operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name);
-			bpf_diag_ctx_forbidden(env, *insn_idx, operation,
-				"Move the call outside the critical section, or use a non-sleepable function.");
+			bpf_diag_ctx_forbidden(env, *insn_idx, operation, suggestion);
 			return -EINVAL;
 		}
 
@@ -9950,11 +11094,10 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		if (env->log.level & BPF_LOG_LEVEL)
 			verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
 				subprog, sub_name);
+		sub_aux->called[in_sleepable_context(env)] = true;
 		returns_void = subprog_returns_void(env, subprog);
 		if (env->subprog_info[subprog].changes_pkt_data)
 			clear_all_pkt_pointers(env);
-		/* mark global subprog for verifying after main prog */
-		subprog_aux(env, subprog)->called = true;
 		if (returns_void)
 			bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
 		else
@@ -9962,10 +11105,20 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		clear_caller_saved_regs(env, caller->regs);
 		invalidate_outgoing_stack_args(env, cur_func(env));
 
-		/* All non-void global functions return a 64-bit SCALAR_VALUE. */
+		/*
+		 * A non-void global function returns a 64-bit SCALAR_VALUE in
+		 * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair.
+		 */
 		if (!returns_void) {
-			mark_reg_unknown(env, caller->regs, BPF_REG_0);
+			nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1;
+			mark_reg_unknown(env, caller->regs, ret_regs[0]);
 			bpf_diag_mod_end(env);
+			for (i = 1; i < nregs; i++) {
+				bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL,
+						   BPF_DIAG_MOD_WRITE);
+				mark_reg_unknown(env, caller->regs, ret_regs[i]);
+				bpf_diag_mod_end(env);
+			}
 		}
 
 		if (env->subprog_info[subprog].might_throw) {
@@ -9983,37 +11136,69 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		return 0;
 	}
 
-	/*
-	 * Track caller's total stack arg count (incoming + max outgoing).
-	 * This is needed so the JIT knows how much stack arg space to allocate.
-	 */
-	caller_info = &env->subprog_info[caller->subprogno];
-	callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]);
-	stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming;
-	if (stack_arg_cnt > caller_info->stack_arg_cnt)
-		caller_info->stack_arg_cnt = stack_arg_cnt;
+	return check_static_func_call(env, subprog, insn_idx);
+}
 
-	/* for regular function entry setup new frame and continue
-	 * from that frame.
-	 */
-	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
+/*
+ * callx dst_reg: call a bpf subprog whose address is in dst_reg.
+ *
+ * The address of a subprog is either loaded into a register by ld_imm64 with
+ * src_reg == BPF_PSEUDO_FUNC, or it is read from a frozen read-only map, see
+ * resolve_func_ptrs(). Both are possible for static subprogs only. Hence all
+ * possible callees of callx are discovered by add_subprogs() and are reachable
+ * in the control flow graph before the main verification pass begins.
+ * PTR_TO_FUNC register identifies the callee, so from here on callx is verified
+ * as a direct call of that static subprog.
+ */
+static int check_func_callx(struct bpf_verifier_env *env, struct bpf_insn *insn,
+			    int *insn_idx)
+{
+	struct bpf_func_state *caller = cur_func(env);
+	struct bpf_reg_state *reg;
+	const char *reason;
+	int err, subprog;
+
+	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
 	if (err)
 		return err;
 
-	bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED);
-	clear_caller_saved_regs(env, caller->regs);
-
-	/* and go analyze first insn of the callee */
-	*insn_idx = env->subprog_info[subprog].start - 1;
-
-	if (env->log.level & BPF_LOG_LEVEL) {
-		verbose(env, "caller:\n");
-		print_verifier_state(env, state, caller->frameno, true);
-		verbose(env, "callee:\n");
-		print_verifier_state(env, state, state->curframe, true);
+	reg = reg_state(env, insn->dst_reg);
+	if (reg->type != PTR_TO_FUNC) {
+		verbose(env, "R%d has type %s, expected func\n", insn->dst_reg,
+			reg_type_str(env, reg->type));
+		reason = bpf_diag_fmt(
+			env, "R%d holds %s, but callx can only call through the address of a static BPF function.",
+			insn->dst_reg, bpf_diag_reg_type_plain(env, reg->type));
+		bpf_diag_register_type(
+			env, *insn_idx, insn->dst_reg, "indirect call through a non-function pointer", reason,
+			"Load the address of a static BPF function into the register before callx.");
+		return -EACCES;
 	}
 
-	return 0;
+	/*
+	 * Arithmetic on PTR_TO_FUNC is allowed, but only unmodified address
+	 * of a subprog can be called.
+	 */
+	err = check_ptr_off_reg(env, reg, insn->dst_reg);
+	if (err)
+		return err;
+
+	/* the callx is valid, but only a JIT can execute it */
+	err = require_callx_jit(env);
+	if (err)
+		return err;
+
+	/* PTR_TO_FUNC is a pointer to a static subprog */
+	subprog = reg->subprogno;
+	err = btf_check_subprog_call(env, subprog);
+	if (err == -EFAULT)
+		return err;
+
+	err = record_callx_edge(env, caller->subprogno, subprog);
+	if (err)
+		return err;
+
+	return check_static_func_call(env, subprog, insn_idx);
 }
 
 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
@@ -10036,6 +11221,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,
 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
 	callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
 	callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid;
+	callee->regs[BPF_REG_3].id = ++env->id_gen;
 
 	/* pointer to stack or null */
 	callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
@@ -10132,6 +11318,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env,
 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
 	callee->regs[BPF_REG_3].map_uid = map_uid;
+	callee->regs[BPF_REG_3].id = ++env->id_gen;
 
 	/* unused */
 	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
@@ -10246,6 +11433,41 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
 	callee->regs[BPF_REG_2].map_ptr = map_ptr;
 	callee->regs[BPF_REG_2].map_uid = map_uid;
 
+	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
+	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
+	callee->regs[BPF_REG_3].map_ptr = map_ptr;
+	callee->regs[BPF_REG_3].map_uid = map_uid;
+	callee->regs[BPF_REG_3].id = ++env->id_gen;
+
+	/* unused */
+	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
+	bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
+	callee->in_async_callback_fn = true;
+	callee->callback_ret_range = retval_range(S32_MIN, S32_MAX);
+	return 0;
+}
+
+static int set_rcu_callback_state(struct bpf_verifier_env *env,
+				  struct bpf_func_state *caller,
+				  struct bpf_func_state *callee,
+				  int insn_idx)
+{
+	struct bpf_map *map_ptr = caller->regs[BPF_REG_2].map_ptr;
+	u32 map_uid = caller->regs[BPF_REG_2].map_uid;
+
+	/*
+	 * callback_fn(struct bpf_map *map, void *key, void *value);
+	 */
+	callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
+	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
+	callee->regs[BPF_REG_1].map_ptr = map_ptr;
+	callee->regs[BPF_REG_1].map_uid = map_uid;
+
+	callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
+	__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
+	callee->regs[BPF_REG_2].map_ptr = map_ptr;
+	callee->regs[BPF_REG_2].map_uid = map_uid;
+
 	callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
 	__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
 	callee->regs[BPF_REG_3].map_ptr = map_ptr;
@@ -10337,11 +11559,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 	struct bpf_func_state *caller, *callee;
 	struct bpf_reg_state *r0;
 	bool in_callback_fn;
+	u32 i, nregs;
 	int err;
 
 	callee = state->frame[state->curframe];
 	r0 = &callee->regs[BPF_REG_0];
-	if (r0->type == PTR_TO_STACK) {
+	nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1;
+	for (i = 0; i < nregs; i++) {
+		if (callee->regs[ret_regs[i]].type != PTR_TO_STACK)
+			continue;
 		/* technically it's ok to return caller's stack pointer
 		 * (or caller's caller's pointer) back to the caller,
 		 * since these pointers are valid. Only current stack
@@ -10376,10 +11602,18 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 			return -EFAULT;
 		}
 	} else {
-		/* return to the caller whatever r0 had in the callee */
-		bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE);
-		caller->regs[BPF_REG_0] = *r0;
-		bpf_diag_mod_end(env);
+		/*
+		 * return to the caller whatever the callee had in the
+		 * return register(s)
+		 */
+		for (i = 0; i < nregs; i++) {
+			u32 regno = ret_regs[i];
+
+			bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno],
+					   BPF_DIAG_MOD_WRITE);
+			caller->regs[regno] = callee->regs[regno];
+			bpf_diag_mod_end(env);
+		}
 	}
 
 	/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
@@ -10772,23 +12006,21 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,
 /* Check if we're in a sleepable context. */
 static inline bool in_sleepable_context(struct bpf_verifier_env *env)
 {
-	return !env->cur_state->active_rcu_locks &&
-	       !env->cur_state->active_preempt_locks &&
-	       !env->cur_state->active_locks &&
-	       !env->cur_state->active_irq_id &&
-	       in_sleepable(env);
+	return !in_rcu_cs(env);
 }
 
 static const char *non_sleepable_context_description(struct bpf_verifier_env *env)
 {
-	if (env->cur_state->active_rcu_locks)
-		return "rcu_read_lock region";
-	if (env->cur_state->active_preempt_locks)
-		return "non-preemptible region";
-	if (env->cur_state->active_irq_id)
-		return "IRQ-disabled region";
-	if (env->cur_state->active_locks)
-		return "lock region";
+	if (in_sleepable(env)) {
+		if (env->cur_state->active_rcu_locks)
+			return "rcu_read_lock region";
+		if (env->cur_state->active_preempt_locks)
+			return "non-preemptible region";
+		if (env->cur_state->active_irq_id)
+			return "IRQ-disabled region";
+		if (env->cur_state->active_locks)
+			return "lock region";
+	}
 	return "non-sleepable prog";
 }
 
@@ -10873,19 +12105,24 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 
 	memset(&meta, 0, sizeof(meta));
 
-	err = check_func_proto(fn, &meta);
+	err = check_func_proto(env, fn, &meta);
 	if (err) {
 		verifier_bug(env, "incorrect func proto %s#%d", func_id_name(func_id), func_id);
 		return err;
 	}
 
 	if (fn->might_sleep && !in_sleepable_context(env)) {
+		const char *suggestion;
+
 		verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id,
 			non_sleepable_context_description(env));
+		if (in_sleepable(env))
+			suggestion = "Move the helper call outside the critical section, or use a non-sleepable helper.";
+		else
+			suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable helper.";
 		operation = bpf_diag_fmt(env, "sleepable helper %s#%d",
 					 func_id_name(func_id), func_id);
-		bpf_diag_ctx_forbidden(env, insn_idx, operation,
-			"Move the helper call outside the critical section, or use a non-sleepable helper.");
+		bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion);
 		return -EINVAL;
 	}
 
@@ -10894,13 +12131,11 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		env->insn_aux_data[insn_idx].non_sleepable = true;
 
 	meta.func_id = func_id;
+	meta.func_name = func_id_name(func_id);
 	meta.fn = fn;
-	/* check args */
-	for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
-		err = check_func_arg(env, i, &meta, insn_idx);
-		if (err)
-			return err;
-	}
+	err = check_func_args(env, &meta, insn_idx);
+	if (err)
+		return err;
 
 	err = record_func_map(env, &meta, func_id, insn_idx);
 	if (err)
@@ -10912,16 +12147,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 
 	regs = cur_regs(env);
 
-	/* Mark slots with STACK_MISC in case of raw mode, stack offset
-	 * is inferred from register state.
-	 */
-	for (i = 0; i < meta.arg_raw_mem.size; i++) {
-		err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno,
-				       argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B,
-				       BPF_WRITE, -1, false, false);
-		if (err)
-			return err;
-	}
+	err = mark_raw_stack(env, &meta, insn_idx);
+	if (err)
+		return err;
 
 	if (meta.release_regno) {
 		struct bpf_reg_state *reg = &regs[meta.release_regno];
@@ -11341,6 +12569,37 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 	return 0;
 }
 
+/*
+ * Mark the register(s) holding a @size byte kfunc return value as unknown
+ * scalars and return how many registers that took. Both halves of a register
+ * pair are treated the same way.
+ *
+ * R0 is written in the diagnostic modification scope the caller opened before
+ * the caller-saved scrub, since R0 gets no scrub record of its own and that
+ * scope is what carries its pre-call value into the history. A scope tracks a
+ * single register and does not nest, so close it before recording the rest of
+ * the return value. A value that fits in R0 leaves the caller's scope open for
+ * the caller to close, so a later write to R0 still lands in it.
+ */
+static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env,
+			       struct bpf_reg_state *regs, u32 size)
+{
+	u32 i, nregs = ret_regs_cnt(size);
+
+	mark_reg_unknown(env, regs, ret_regs[0]);
+	if (nregs == 1)
+		return nregs;
+
+	bpf_diag_mod_end(env);
+	for (i = 1; i < nregs; i++) {
+		bpf_diag_mod_begin(env, &regs[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE);
+		mark_reg_unknown(env, regs, ret_regs[i]);
+		bpf_diag_mod_end(env);
+	}
+
+	return nregs;
+}
+
 static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta)
 {
 	return meta->kfunc_flags & KF_ACQUIRE;
@@ -11356,6 +12615,11 @@ static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta)
 	return meta->kfunc_flags & KF_DESTRUCTIVE;
 }
 
+static bool is_kfunc_perfmon(struct bpf_call_arg_meta *meta)
+{
+	return meta->kfunc_flags & KF_PERFMON;
+}
+
 static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta)
 {
 	return meta->kfunc_flags & KF_RCU;
@@ -11446,6 +12710,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param
 	return btf_param_match_suffix(btf, arg, "__irq_flag");
 }
 
+static bool is_kfunc_arg_ctx_out(const struct btf *btf, const struct btf_param *arg)
+{
+	return btf_param_match_suffix(btf, arg, "__ctx_out");
+}
+
 static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg)
 {
 	return btf_param_match_suffix(btf, arg, "__arena__nullable") ||
@@ -11481,7 +12750,8 @@ enum {
 	KF_ARG_RES_SPIN_LOCK_ID,
 	KF_ARG_TASK_WORK_ID,
 	KF_ARG_PROG_AUX_ID,
-	KF_ARG_TIMER_ID
+	KF_ARG_TIMER_ID,
+	KF_ARG_RCU_HEAD_ID
 };
 
 BTF_ID_LIST(kf_arg_btf_ids)
@@ -11495,6 +12765,7 @@ BTF_ID(struct, bpf_res_spin_lock)
 BTF_ID(struct, bpf_task_work)
 BTF_ID(struct, bpf_prog_aux)
 BTF_ID(struct, bpf_timer)
+BTF_ID(struct, bpf_rcu_head)
 
 static bool __is_kfunc_ptr_arg_type(const struct btf *btf,
 				    const struct btf_param *arg, int type)
@@ -11553,6 +12824,11 @@ static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param
 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID);
 }
 
+static bool is_kfunc_arg_rcu_head(const struct btf *btf, const struct btf_param *arg)
+{
+	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RCU_HEAD_ID);
+}
+
 static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg)
 {
 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);
@@ -11607,13 +12883,39 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_
 	return argn <= arg_idx;
 }
 
-/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
-static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
-					const struct btf *btf,
-					const struct btf_type *t, int rec)
+#define BTF_MEMBER_MAX_DEPTH	4
+#define BTF_MEMBER_PATH_LEN	64
+
+struct btf_member_path {
+	const struct btf_member *member[BTF_MEMBER_MAX_DEPTH];
+	bool in_array[BTF_MEMBER_MAX_DEPTH];
+	const struct btf_type *bad_type;
+	int depth;
+	bool too_deep;
+};
+
+static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t,
+				    u32 member_kinds)
+{
+	if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t))
+		return true;
+	if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t))
+		return true;
+	return false;
+}
+
+/*
+ * Returns true if every member of struct @t is of a kind listed in
+ * @member_kinds, 4 levels of nesting allowed. An array member counts as its
+ * element type.
+ */
+static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf,
+				   const struct btf_type *t, u32 member_kinds, int rec,
+				   struct btf_member_path *path)
 {
 	const struct btf_type *member_type;
 	const struct btf_member *member;
+	bool in_array;
 	u32 i;
 
 	if (!btf_type_is_struct(t))
@@ -11622,58 +12924,128 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
 	for_each_member(i, t, member) {
 		const struct btf_array *array;
 
+		in_array = false;
 		member_type = btf_type_skip_modifiers(btf, member->type, NULL);
+		/*
+		 * Every element of an array is laid out in the value being
+		 * returned, so an array counts as its element type however many
+		 * dimensions deep that is.
+		 */
+		while (btf_type_is_array(member_type)) {
+			array = btf_array(member_type);
+			if (!array->nelems) {
+				member_type = NULL;
+				goto bad_member;
+			}
+			member_type = btf_type_skip_modifiers(btf, array->type, NULL);
+			in_array = true;
+		}
 		if (btf_type_is_struct(member_type)) {
-			if (rec >= 3) {
+			if (rec >= BTF_MEMBER_MAX_DEPTH - 1) {
 				verbose(env, "max struct nesting depth exceeded\n");
+				if (path)
+					path->too_deep = true;
 				return false;
 			}
-			if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))
-				return false;
-			continue;
-		}
-		if (btf_type_is_array(member_type)) {
-			array = btf_array(member_type);
-			if (!array->nelems)
-				return false;
-			member_type = btf_type_skip_modifiers(btf, array->type, NULL);
-			if (!btf_type_is_scalar(member_type))
-				return false;
+			if (!btf_struct_member_walk(env, btf, member_type, member_kinds,
+						    rec + 1, path))
+				goto bad_path;
 			continue;
 		}
-		if (!btf_type_is_scalar(member_type))
-			return false;
+		if (!btf_member_kind_allowed(btf, member_type, member_kinds))
+			goto bad_member;
 	}
 	return true;
+
+bad_member:
+	if (path) {
+		path->depth = rec + 1;
+		path->bad_type = member_type;
+	}
+bad_path:
+	if (path && path->depth) {
+		path->member[rec] = member;
+		path->in_array[rec] = in_array;
+	}
+	return false;
 }
 
-enum kfunc_ptr_arg_type {
-	KF_ARG_CONST_MEM_SIZE,
-	KF_ARG_MEM_SIZE,
-	KF_ARG_CONST,
-	KF_ARG_CONST_ALLOC_SIZE_OR_ZERO,
-	KF_ARG_ANYTHING,
-	KF_ARG_PTR_TO_CTX,
-	KF_ARG_PTR_TO_ALLOC_BTF_ID,    /* Allocated object */
-	KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */
-	KF_ARG_PTR_TO_DYNPTR,
-	KF_ARG_PTR_TO_ITER,
-	KF_ARG_PTR_TO_LIST_HEAD,
-	KF_ARG_PTR_TO_LIST_NODE,
-	KF_ARG_PTR_TO_BTF_ID,	       /* Also covers reg2btf_ids conversions */
-	KF_ARG_PTR_TO_MEM,
-	KF_ARG_PTR_TO_CALLBACK,
-	KF_ARG_PTR_TO_RB_ROOT,
-	KF_ARG_PTR_TO_RB_NODE,
-	KF_ARG_PTR_TO_CONST_STR,
-	KF_ARG_CONST_MAP_PTR,
-	KF_ARG_PTR_TO_TIMER,
-	KF_ARG_PTR_TO_WORKQUEUE,
-	KF_ARG_PTR_TO_IRQ_FLAG,
-	KF_ARG_PTR_TO_RES_SPIN_LOCK,
-	KF_ARG_PTR_TO_TASK_WORK,
-	KF_ARG_PTR_TO_ARENA,
-};
+bool btf_struct_is_composed_of(struct bpf_verifier_env *env,
+			       const struct btf *btf,
+			       const struct btf_type *t, u32 member_kinds)
+{
+	return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL);
+}
+
+static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env,
+				      const struct btf *btf,
+				      const struct btf_type *t)
+{
+	return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR);
+}
+
+static int resolve_func_arg_type(struct bpf_verifier_env *env,
+				 struct bpf_reg_state *reg, u32 arg, argno_t argno,
+				 struct bpf_call_arg_meta *meta,
+				 enum bpf_arg_type *arg_type, u32 *arg_size)
+{
+	const struct btf_param *args;
+	const struct btf_type *ref_t, *resolve_ret;
+	const struct btf *btf;
+	const char *ref_tname;
+	u32 ref_id;
+
+	if (base_type(*arg_type) == ARG_PTR_TO_MAP_VALUE)
+		return resolve_map_arg_type(env, meta, arg_type);
+
+	if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID)
+		return 0;
+
+	if (!is_kfunc(meta) || arg_type_is_release(*arg_type) ||
+	    base_type(reg->type) == PTR_TO_BTF_ID ||
+	    reg2btf_ids[base_type(reg->type)])
+		return 0;
+
+	args = btf_params(meta->func_proto);
+	ref_id = *meta->fn->arg_btf_id[arg];
+	btf = is_kfunc_arg_map(meta->btf, &args[arg]) ? btf_vmlinux : meta->btf;
+	ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id);
+	ref_tname = btf_name_by_offset(btf, ref_t->name_off);
+
+	if (!btf_type_is_scalar_struct(env, btf, ref_t))
+		return 0;
+
+	resolve_ret = btf_resolve_size(btf, ref_t, arg_size);
+	if (IS_ERR(resolve_ret)) {
+		verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n",
+			reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname,
+			PTR_ERR(resolve_ret));
+		return -EINVAL;
+	}
+	*arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE |
+		    (*arg_type & (PTR_MAYBE_NULL | MEM_UNINIT));
+
+	return 0;
+}
+
+static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path,
+				char *buf, size_t buf_sz)
+{
+	size_t len = 0;
+	int i;
+
+	buf[0] = '\0';
+	for (i = 0; i < path->depth; i++) {
+		const char *name = btf_name_by_offset(btf, path->member[i]->name_off);
+
+		/* An anonymous struct or union has no name to spell. */
+		if (!name || !name[0])
+			continue;
+		len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name);
+		if (path->in_array[i])
+			len += scnprintf(buf + len, buf_sz - len, "[]");
+	}
+}
 
 enum special_kfunc_type {
 	KF_bpf_obj_new_impl,
@@ -11740,9 +13112,14 @@ enum special_kfunc_type {
 	KF___bpf_trap,
 	KF_bpf_task_work_schedule_signal,
 	KF_bpf_task_work_schedule_resume,
+	KF_bpf_call_rcu,
+	KF_bpf_call_rcu_tasks_trace,
 	KF_bpf_arena_alloc_pages,
 	KF_bpf_arena_free_pages,
+	KF_bpf_arena_reserve_pages,
 	KF_bpf_session_is_return,
+	KF_bpf_stream_vprintk,
+	KF_bpf_stream_print_stack,
 };
 
 BTF_ID_LIST(special_kfunc_list)
@@ -11830,13 +13207,33 @@ BTF_ID(func, bpf_dynptr_file_discard)
 BTF_ID(func, __bpf_trap)
 BTF_ID(func, bpf_task_work_schedule_signal)
 BTF_ID(func, bpf_task_work_schedule_resume)
+BTF_ID(func, bpf_call_rcu)
+BTF_ID(func, bpf_call_rcu_tasks_trace)
 BTF_ID(func, bpf_arena_alloc_pages)
 BTF_ID(func, bpf_arena_free_pages)
+BTF_ID(func, bpf_arena_reserve_pages)
 #ifdef CONFIG_BPF_EVENTS
 BTF_ID(func, bpf_session_is_return)
 #else
 BTF_ID_UNUSED
 #endif
+BTF_ID(func, bpf_stream_vprintk)
+BTF_ID(func, bpf_stream_print_stack)
+
+static bool is_bpf_cast_to_kern_ctx_kfunc(const struct bpf_call_arg_meta *meta)
+{
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]);
+}
+
+static bool is_bpf_dynptr_clone_kfunc(const struct bpf_call_arg_meta *meta)
+{
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_clone]);
+}
+
+static bool is_bpf_iter_css_task_new_kfunc(const struct bpf_call_arg_meta *meta)
+{
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_iter_css_task_new]);
+}
 
 static bool is_bpf_obj_new_kfunc(u32 func_id)
 {
@@ -11883,6 +13280,12 @@ static bool is_bpf_rbtree_add_kfunc(u32 func_id)
 	       func_id == special_kfunc_list[KF_bpf_rbtree_add_impl];
 }
 
+static bool is_call_rcu_kfunc(u32 func_id)
+{
+	return func_id == special_kfunc_list[KF_bpf_call_rcu] ||
+	       func_id == special_kfunc_list[KF_bpf_call_rcu_tasks_trace];
+}
+
 static bool is_task_work_add_kfunc(u32 func_id)
 {
 	return func_id == special_kfunc_list[KF_bpf_task_work_schedule_signal] ||
@@ -11899,52 +13302,132 @@ static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta)
 
 static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta)
 {
-	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_lock]);
 }
 
 static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta)
 {
-	return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_rcu_read_unlock]);
 }
 
 static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta)
 {
-	return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable];
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_disable]);
 }
 
 static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta)
 {
-	return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable];
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_preempt_enable]);
 }
 
 bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta)
 {
-	return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data];
+	return is_kfunc_call(meta, special_kfunc_list[KF_bpf_xdp_pull_data]);
+}
+
+static u32 kfunc_abi_slots(const struct btf_func_model *fm)
+{
+	const struct bpf_jit_arg_abi *abi = bpf_jit_arg_abi();
+	u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS];
+	u32 i, nslots, slots = 0;
+
+	for (i = 0; i < fm->nr_args; i++)
+		slots += btf_func_model_arg_slots(fm, i);
+
+	if (!abi)
+		return slots;
+
+	nslots = bpf_jit_place_args(abi, fm, pos_of_slot);
+	for (i = 0; i < nslots; i++)
+		if (pos_of_slot[i] + 1 > slots)
+			slots = pos_of_slot[i] + 1;
+
+	return slots;
+}
+
+static u32 __btf_func_arg_align(const struct btf *btf, const struct btf_type *t, int rec)
+{
+	const struct btf_member *member;
+	const struct btf_type *mt;
+	u32 align, i;
+
+	while (btf_type_is_array(t))
+		t = btf_type_skip_modifiers(btf, btf_array(t)->type, NULL);
+
+	if (btf_type_is_int(t))
+		return t->size > BPF_REG_SIZE ? t->size : BPF_REG_SIZE;
+	if (!btf_type_is_struct(t))
+		return BPF_REG_SIZE;
+	if (rec >= BTF_MEMBER_MAX_DEPTH)
+		return 0;
+
+	for_each_member(i, t, member) {
+		mt = btf_type_skip_modifiers(btf, member->type, NULL);
+		align = __btf_func_arg_align(btf, mt, rec + 1);
+		if (!align)
+			return 0;
+		if (align > BPF_REG_SIZE)
+			return 2 * BPF_REG_SIZE;
+	}
+	return BPF_REG_SIZE;
+}
+
+u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t)
+{
+	return __btf_func_arg_align(btf, t, 0);
 }
 
 static int
 get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
-		   const struct btf_param *args, int arg, int nargs)
+		   const struct btf_param *args, int arg, int nargs, u32 slot,
+		   struct bpf_func_proto *proto)
 {
-	const struct btf_type *t, *ref_t = NULL;
-	argno_t argno = argno_from_arg(arg + 1);
+	const struct btf_type *t, *ref_t = NULL, *resolve_ret;
+	const u32 *ref_id_ptr = NULL;
+	argno_t argno = argno_from_arg(slot + 1);
 	const char *ref_tname = NULL;
+	u32 ref_id, type_size;
 	int arg_type;
 
+	proto->arg_btf_id[arg] = NULL;
+
+	if (is_kfunc_arg_prog_aux(meta->btf, &args[arg]))
+		return ARG_PTR_TO_PROG_AUX;
+
+	if (is_kfunc_arg_ignore(meta->btf, &args[arg]) || is_kfunc_arg_implicit(meta, arg))
+		return ARG_IGNORE;
+
 	t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL);
 
 	/* Scalar arguments are classified from their BTF suffix/name alone. */
 	if (btf_type_is_scalar(t)) {
 		if (is_kfunc_arg_constant(meta->btf, &args[arg]))
-			return KF_ARG_CONST;
+			return ARG_CONST_SCALAR;
 		if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg]))
-			return KF_ARG_CONST_MEM_SIZE;
+			return ARG_CONST_MEM_SIZE;
 		if (is_kfunc_arg_mem_size(meta->btf, &args[arg]))
-			return KF_ARG_MEM_SIZE;
+			return ARG_MEM_SIZE;
 		if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") ||
 		    is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size"))
-			return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO;
-		return KF_ARG_ANYTHING;
+			return ARG_CONST_ALLOC_SIZE_OR_ZERO;
+		return ARG_SCALAR;
+	}
+
+	if (btf_type_is_struct(t)) {
+		if (!t->size || t->size > 2 * BPF_REG_SIZE) {
+			verbose(env,
+				"%s type %s has size %u, only 1 to %d bytes "
+				"can be passed by value\n",
+				reg_arg_name(env, argno), btf_type_str(t), t->size,
+				2 * BPF_REG_SIZE);
+			return -EINVAL;
+		}
+		if (!btf_type_is_scalar_struct(env, meta->btf, t)) {
+			verbose(env, "%s type %s is not composed of scalars\n",
+				reg_arg_name(env, argno), btf_type_str(t));
+			return -EINVAL;
+		}
+		return ARG_SCALAR;
 	}
 
 	if (!btf_type_is_ptr(t)) {
@@ -11952,54 +13435,87 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
 			reg_arg_name(env, argno), btf_type_str(t));
 		return -EINVAL;
 	}
-	ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL);
+	/* Keep a pointer to the BTF field containing the resolved referent ID. */
+	ref_id_ptr = &t->type;
+	ref_t = btf_type_skip_modifiers(meta->btf, *ref_id_ptr, &ref_id);
+	while (*ref_id_ptr != ref_id)
+		ref_id_ptr = &btf_type_by_id(meta->btf, *ref_id_ptr)->type;
 	ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off);
 
 	/* In this function, we verify the kfunc's BTF as per the argument type,
 	 * leaving the rest of the verification with respect to the register
 	 * type to our caller. When a set of conditions hold in the BTF type of
-	 * arguments, we resolve it to a known kfunc_ptr_arg_type.
+	 * arguments, we resolve it to a known bpf_arg_type.
 	 */
-	if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
-	    meta->func_id == special_kfunc_list[KF_bpf_session_is_return] ||
-	    meta->func_id == special_kfunc_list[KF_bpf_session_cookie])
-		arg_type = KF_ARG_PTR_TO_CTX;
+	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx]) ||
+	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_is_return]) ||
+	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_session_cookie]))
+		arg_type = ARG_PTR_TO_CTX;
 	else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg))
-		arg_type = KF_ARG_PTR_TO_CTX;
+		arg_type = ARG_PTR_TO_CTX;
 	else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID;
+		arg_type = ARG_PTR_TO_ALLOC_BTF_ID;
 	else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR;
-	else if (is_kfunc_arg_dynptr(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_DYNPTR;
-	else if (is_kfunc_arg_iter(meta, arg, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_ITER;
+		arg_type = ARG_PTR_TO_REFCOUNTED_KPTR;
+	else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) {
+		arg_type = ARG_PTR_TO_DYNPTR;
+
+		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb]))
+			arg_type |= DYNPTR_TYPE_SKB;
+		else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_xdp]))
+			arg_type |= DYNPTR_TYPE_XDP;
+		else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_skb_meta]))
+			arg_type |= DYNPTR_TYPE_SKB_META;
+		else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_from_file]) ||
+			 is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_file_discard]))
+			/* OBJ_RELEASE for the latter comes from KF_RELEASE below */
+			arg_type |= DYNPTR_TYPE_FILE;
+	} else if (is_kfunc_arg_iter(meta, arg, &args[arg]))
+		arg_type = ARG_PTR_TO_ITER;
 	else if (is_kfunc_arg_list_head(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_LIST_HEAD;
+		arg_type = ARG_PTR_TO_LIST_HEAD;
 	else if (is_kfunc_arg_list_node(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_LIST_NODE;
+		arg_type = ARG_PTR_TO_LIST_NODE;
 	else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_RB_ROOT;
+		arg_type = ARG_PTR_TO_RB_ROOT;
 	else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_RB_NODE;
+		arg_type = ARG_PTR_TO_RB_NODE;
 	else if (is_kfunc_arg_const_str(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_CONST_STR;
+		arg_type = ARG_PTR_TO_CONST_STR;
 	else if (is_kfunc_arg_const_map(meta->btf, &args[arg]))
-		arg_type = KF_ARG_CONST_MAP_PTR;
+		arg_type = ARG_CONST_MAP_PTR;
 	else if (is_kfunc_arg_map(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_BTF_ID;
+		arg_type = ARG_PTR_TO_BTF_ID;
 	else if (is_kfunc_arg_wq(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_WORKQUEUE;
+		arg_type = ARG_PTR_TO_WORKQUEUE;
 	else if (is_kfunc_arg_timer(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_TIMER;
+		arg_type = ARG_PTR_TO_TIMER;
 	else if (is_kfunc_arg_task_work(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_TASK_WORK;
+		arg_type = ARG_PTR_TO_TASK_WORK;
+	else if (is_kfunc_arg_rcu_head(meta->btf, &args[arg]))
+		arg_type = ARG_PTR_TO_RCU_HEAD;
 	else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_IRQ_FLAG;
+		arg_type = ARG_PTR_TO_IRQ_FLAG;
 	else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK;
-	else if (is_kfunc_arg_callback(env, meta->btf, &args[arg]))
-		arg_type = KF_ARG_PTR_TO_CALLBACK;
+		arg_type = ARG_PTR_TO_RES_SPIN_LOCK;
+	else if (is_kfunc_arg_ctx_out(meta->btf, &args[arg])) {
+		if (!btf_type_is_scalar(ref_t)) {
+			verbose(env, "%s __ctx_out argument must point to a scalar\n",
+				reg_arg_name(env, argno));
+			return -EINVAL;
+		}
+		resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size);
+		if (IS_ERR(resolve_ret)) {
+			verbose(env,
+				"%s reference type('%s %s') size cannot be determined: %ld\n",
+				reg_arg_name(env, argno), btf_type_str(ref_t),
+				ref_tname, PTR_ERR(resolve_ret));
+			return -EINVAL;
+		}
+		proto->arg_size[arg] = type_size;
+		arg_type = ARG_PTR_TO_CTX_OUT | MEM_FIXED_SIZE;
+	} else if (is_kfunc_arg_callback(env, meta->btf, &args[arg]))
+		arg_type = ARG_PTR_TO_FUNC;
 	else if (is_kfunc_arg_arena(meta->btf, &args[arg])) {
 		if (!bpf_jit_supports_arena_args()) {
 			verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n",
@@ -12022,20 +13538,20 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
 		 * whether the JIT rebases it to the arena base or preserves NULL.
 		 * The common nullable path below records that verifier property.
 		 */
-		arg_type = KF_ARG_PTR_TO_ARENA;
+		arg_type = ARG_PTR_TO_ARENA;
 	} else if (arg + 1 < nargs &&
 		 (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) ||
 		  is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) {
 		if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) &&
-		    !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+		    !btf_type_is_scalar_struct(env, meta->btf, ref_t)) {
 			verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n",
 				reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
 			return -EINVAL;
 		}
-		arg_type = KF_ARG_PTR_TO_MEM;
+		arg_type = ARG_PTR_TO_MEM | MEM_WRITE;
 	} else if (btf_type_is_struct(ref_t))
-		/* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */
-		arg_type = KF_ARG_PTR_TO_BTF_ID;
+		/* A pointer to a struct without a size argument is classified as ARG_PTR_TO_BTF_ID */
+		arg_type = ARG_PTR_TO_BTF_ID;
 	else {
 		/*
 		 * Otherwise this is a fixed-size memory buffer supported by
@@ -12043,26 +13559,62 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
 		 * scalars. The access size is derived from the pointed-to BTF type.
 		 */
 		if (!btf_type_is_scalar(ref_t) &&
-		    !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+		    !btf_type_is_scalar_struct(env, meta->btf, ref_t)) {
 			verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n",
 				reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
 			return -EINVAL;
 		}
-		arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE;
+		resolve_ret = btf_resolve_size(meta->btf, ref_t, &type_size);
+		if (IS_ERR(resolve_ret)) {
+			verbose(env,
+				"%s reference type('%s %s') size cannot be determined: %ld\n",
+				reg_arg_name(env, argno), btf_type_str(ref_t),
+				ref_tname, PTR_ERR(resolve_ret));
+			return -EINVAL;
+		}
+		proto->arg_size[arg] = type_size;
+		arg_type = ARG_PTR_TO_MEM | MEM_FIXED_SIZE | MEM_WRITE;
 	}
 
+	if (is_kfunc_arg_uninit(meta->btf, &args[arg]))
+		arg_type |= MEM_UNINIT;
+
 	if (is_kfunc_arg_nullable(meta->btf, &args[arg]))
 		arg_type |= PTR_MAYBE_NULL;
 
+	/*
+	 * Only the first argument of a KF_RELEASE kfunc releases anything, and
+	 * bpf_fetch_kfunc_arg_meta() only ever records BPF_REG_1 for it.
+	 */
+	if (is_kfunc_release(meta) && arg == 0)
+		arg_type |= OBJ_RELEASE;
+
+	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID) {
+		if (is_kfunc_arg_map(meta->btf, &args[arg]))
+			proto->arg_btf_id[arg] = reg2btf_ids[CONST_PTR_TO_MAP];
+		else
+			proto->arg_btf_id[arg] = ref_id_ptr;
+
+		/*
+		 * A KF_RCU kfunc accepts an RCU-protected pointer where it would
+		 * otherwise demand a referenced or trusted one. Other argument kinds
+		 * have their own provenance requirements and must not inherit MEM_RCU.
+		 */
+		if (is_kfunc_rcu(meta))
+			arg_type |= MEM_RCU;
+	}
+
 	return arg_type;
 }
 
 static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
-			       struct bpf_func_proto *proto)
+			       const struct btf_func_model *fm, struct bpf_func_proto *proto)
 {
+	const struct bpf_jit_arg_abi *abi;
 	const struct btf *btf = meta->btf;
 	const struct btf_param *args;
-	u32 i, nargs;
+	const struct btf_type *t;
+	u32 i, nargs, slots_used;
 	int arg_type;
 
 	args = (const struct btf_param *)(meta->func_proto + 1);
@@ -12072,54 +13624,84 @@ static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg
 			nargs, MAX_BPF_FUNC_ARGS);
 		return -EINVAL;
 	}
-	if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) {
-		verbose(env, "JIT does not support kfunc %s() with %d args\n",
-			meta->func_name, nargs);
-		return -ENOTSUPP;
+
+	for (i = 0, slots_used = 0; i < nargs; i++) {
+		u32 nslots = btf_func_model_arg_slots(fm, i);
+
+		if (nslots > 1) {
+			t = btf_type_skip_modifiers(btf, args[i].type, NULL);
+			if (!btf_func_arg_align(btf, t)) {
+				verbose(env,
+					"Function %s arg#%d type %s nests structs more than "
+					"%d levels deep\n",
+					meta->func_name, i, btf_type_str(t),
+					BTF_MEMBER_MAX_DEPTH);
+				return -EINVAL;
+			}
+		}
+		slots_used += nslots;
 	}
 
-	for (i = 0; i < nargs; i++) {
-		if (is_kfunc_arg_prog_aux(btf, &args[i]) ||
-		    is_kfunc_arg_ignore(btf, &args[i]) ||
-		    is_kfunc_arg_implicit(meta, i))
-			continue;
+	if (slots_used > MAX_BPF_FUNC_ARGS) {
+		verbose(env, "Function %s needs %d > %d argument slots\n", meta->func_name,
+			slots_used, MAX_BPF_FUNC_ARGS);
+		return -EINVAL;
+	}
 
-		arg_type = get_kfunc_arg_type(env, meta, args, i, nargs);
+	abi = bpf_jit_arg_abi();
+
+	for (i = 0, slots_used = 0; i < nargs; i++) {
+		u32 nslots = btf_func_model_arg_slots(fm, i);
+
+		if (!abi && nslots > 1) {
+			t = btf_type_skip_modifiers(btf, args[i].type, NULL);
+			verbose(env,
+				"Function %s arg#%d type %s cannot be passed at "
+				"argument slot %d on this architecture\n",
+				meta->func_name, i, btf_type_str(t), slots_used);
+			return -EINVAL;
+		}
+		slots_used += nslots;
+
+		arg_type = get_kfunc_arg_type(env, meta, args, i, nargs,
+					      slots_used - nslots, proto);
 		if (arg_type < 0)
 			return arg_type;
 
 		proto->arg_type[i] = arg_type;
 	}
 
-	return 0;
+	if (slots_used > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) {
+		verbose(env, "JIT does not support kfunc %s() with %d argument slots\n",
+			meta->func_name, slots_used);
+		return -ENOTSUPP;
+	}
+
+	return check_arg_prog_aux(env, proto) ? 0 : -EINVAL;
 }
 
-static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
-					struct bpf_reg_state *reg,
-					const struct btf_type *ref_t,
-					const char *ref_tname, u32 ref_id,
-					struct bpf_call_arg_meta *meta,
-					int arg, argno_t argno)
+static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+				     argno_t argno, enum bpf_arg_type arg_type,
+				     const struct btf *arg_btf, u32 arg_btf_id,
+				     struct bpf_call_arg_meta *meta, int insn_idx)
 {
-	const struct btf_type *reg_ref_t;
-	bool strict_type_match = false;
+	bool taking_projection, struct_same, strict_type_match = false;
+	const struct btf_type *arg_t, *reg_t;
+	const char *arg_tname, *reg_tname;
 	const struct btf *reg_btf;
-	const char *reg_ref_tname;
-	bool taking_projection;
-	bool struct_same;
-	u32 reg_ref_id;
+	u32 reg_btf_id;
 
 	if (base_type(reg->type) == PTR_TO_BTF_ID) {
 		reg_btf = reg->btf;
-		reg_ref_id = reg->btf_id;
+		reg_btf_id = reg->btf_id;
 	} else {
 		reg_btf = btf_vmlinux;
-		reg_ref_id = *reg2btf_ids[base_type(reg->type)];
+		reg_btf_id = *reg2btf_ids[base_type(reg->type)];
 	}
 
-	/* Enforce strict type matching for calls to kfuncs that are acquiring
-	 * or releasing a reference, or are no-cast aliases. We do _not_
-	 * enforce strict matching for kfuncs by default,
+	/*
+	 * Enforce strict type matching for arguments that release a reference,
+	 * or are no-cast aliases. We do _not_ enforce strict matching by default,
 	 * as we want to enable BPF programs to pass types that are bitwise
 	 * equivalent without forcing them to explicitly cast with something
 	 * like bpf_cast_to_kern_ctx().
@@ -12141,27 +13723,30 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
 	 * btf_struct_ids_match() to walk the struct at the 0th offset, and
 	 * resolve types.
 	 */
-	if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) ||
-	    btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))
+	if ((arg_type_is_release(arg_type) && !is_helper_call(meta, BPF_FUNC_sk_release)) ||
+	    (is_kfunc(meta) && btf_type_ids_nocast_alias(&env->log, reg_btf, reg_btf_id,
+						    arg_btf, arg_btf_id)))
 		strict_type_match = true;
 
-	WARN_ON_ONCE(is_kfunc_release(meta) && !tnum_is_const(reg->var_off));
+	arg_t = btf_type_skip_modifiers(arg_btf, arg_btf_id, &arg_btf_id);
+	arg_tname = btf_name_by_offset(arg_btf, arg_t->name_off);
+	reg_t = btf_type_skip_modifiers(reg_btf, reg_btf_id, &reg_btf_id);
+	reg_tname = btf_name_by_offset(reg_btf, reg_t->name_off);
+
+	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_btf_id,
+					  reg->var_off.value, arg_btf, arg_btf_id,
+					  strict_type_match, !type_is_alloc(reg->type));
 
-	reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
-	reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
-	struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value,
-					   meta->btf, ref_id, strict_type_match,
-					   !type_is_alloc(reg->type));
 	/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
 	 * actually use it -- it must cast to the underlying type. So we allow
 	 * caller to pass in the underlying type.
 	 */
-	taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname);
+	taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname);
 	if (!taking_projection && !struct_same) {
-		verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n",
+		verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n",
 			meta->func_name, reg_arg_name(env, argno),
-			btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno),
-			btf_type_str(reg_ref_t), reg_ref_tname);
+			btf_type_str(arg_t), arg_tname,
+			reg_arg_name(env, argno), btf_type_str(reg_t), reg_tname);
 		return -EINVAL;
 	}
 	return 0;
@@ -12173,15 +13758,15 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *
 	int err, spi, kfunc_class = IRQ_NATIVE_KFUNC;
 	bool irq_save;
 
-	if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_save] ||
-	    meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) {
+	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_save]) ||
+	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) {
 		irq_save = true;
-		if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])
+		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
 			kfunc_class = IRQ_LOCK_KFUNC;
-	} else if (meta->func_id == special_kfunc_list[KF_bpf_local_irq_restore] ||
-		   meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]) {
+	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_local_irq_restore]) ||
+		   is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])) {
 		irq_save = false;
-		if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])
+		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]))
 			kfunc_class = IRQ_LOCK_KFUNC;
 	} else {
 		verifier_bug(env, "unknown irq flags kfunc");
@@ -12374,12 +13959,20 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id)
 	       btf_id == special_kfunc_list[KF_bpf_rbtree_right];
 }
 
-static bool is_bpf_res_spin_lock_kfunc(u32 btf_id)
+static int get_bpf_res_spin_lock_kfunc_flags(const struct bpf_call_arg_meta *meta)
 {
-	return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
-	       btf_id == special_kfunc_list[KF_bpf_res_spin_unlock] ||
-	       btf_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] ||
-	       btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore];
+	int flags = PROCESS_RES_LOCK;
+
+	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock]) ||
+	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
+		flags |= PROCESS_SPIN_LOCK;
+	else if (!is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock]) &&
+		 !is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]))
+		return 0;
+	if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_lock_irqsave]) ||
+	    is_kfunc_call(meta, special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]))
+		flags |= PROCESS_LOCK_IRQ;
+	return flags;
 }
 
 static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset)
@@ -12402,7 +13995,8 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id)
 static bool is_async_callback_calling_kfunc(u32 btf_id)
 {
 	return is_bpf_wq_set_callback_kfunc(btf_id) ||
-	       is_task_work_add_kfunc(btf_id);
+	       is_task_work_add_kfunc(btf_id) ||
+	       is_call_rcu_kfunc(btf_id);
 }
 
 bool bpf_is_throw_kfunc(struct bpf_insn *insn)
@@ -12595,768 +14189,67 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
 				  !type_is_alloc(reg->type))) {
 		verbose(env, "operation on %s expects arg#1 %s at offset=%d "
 			"in struct %s, but arg is at offset=%d in struct %s\n",
-			btf_field_type_name(head_field_type),
-			btf_field_type_name(node_field_type),
-			field->graph_root.node_offset,
-			btf_name_by_offset(field->graph_root.btf, et->name_off),
-			node_off, btf_name_by_offset(reg->btf, t->name_off));
-		return -EINVAL;
-	}
-	meta->arg_btf = reg->btf;
-	meta->arg_btf_id = reg->btf_id;
-
-	if (node_off != field->graph_root.node_offset) {
-		verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",
-			node_off, btf_field_type_name(node_field_type),
-			field->graph_root.node_offset,
-			btf_name_by_offset(field->graph_root.btf, et->name_off));
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
-					   struct bpf_reg_state *reg, argno_t argno,
-					   struct bpf_call_arg_meta *meta)
-{
-	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
-						  BPF_LIST_HEAD, BPF_LIST_NODE,
-						  &meta->arg_list_head.field);
-}
-
-static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
-					     struct bpf_reg_state *reg, argno_t argno,
-					     struct bpf_call_arg_meta *meta)
-{
-	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
-						  BPF_RB_ROOT, BPF_RB_NODE,
-						  &meta->arg_rbtree_root.field);
-}
-
-/*
- * css_task iter allowlist is needed to avoid dead locking on css_set_lock.
- * LSM hooks and iters (both sleepable and non-sleepable) are safe.
- * Any sleepable progs are also safe since bpf_check_attach_target() enforce
- * them can only be attached to some specific hook points.
- */
-static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)
-{
-	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
-
-	switch (prog_type) {
-	case BPF_PROG_TYPE_LSM:
-		return true;
-	case BPF_PROG_TYPE_TRACING:
-		if (env->prog->expected_attach_type == BPF_TRACE_ITER)
-			return true;
-		fallthrough;
-	default:
-		return in_sleepable(env);
-	}
-}
-
-static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
-			    int insn_idx)
-{
-	const char *func_name = meta->func_name, *ref_tname;
-	struct bpf_func_state *caller = cur_func(env);
-	struct bpf_reg_state *regs = cur_regs(env);
-	const struct btf *btf = meta->btf;
-	const struct btf_param *args;
-	struct btf_record *rec;
-	u32 i, nargs;
-	int ret;
-
-	args = (const struct btf_param *)(meta->func_proto + 1);
-	nargs = btf_type_vlen(meta->func_proto);
-
-	ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args);
-	if (ret)
-		return ret;
-
-	/* Check that BTF function arguments match actual types that the
-	 * verifier sees.
-	 */
-	for (i = 0; i < nargs; i++) {
-		struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i);
-		const struct btf_type *t, *ref_t, *resolve_ret;
-		enum bpf_arg_type arg_type = ARG_DONTCARE;
-		argno_t argno = argno_from_arg(i + 1);
-		int regno = reg_from_argno(argno);
-		bool btf_id_fixed_off_ok = true;
-		u32 ref_id = args[i].type, type_size;
-		int kf_arg_type = meta->fn->arg_type[i];
-
-		if (is_kfunc_arg_prog_aux(btf, &args[i])) {
-			/* Reject repeated use bpf_prog_aux */
-			if (meta->arg_prog) {
-				verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc");
-				return -EFAULT;
-			}
-			if (regno < 0) {
-				verbose(env, "%s prog->aux cannot be a stack argument\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			meta->arg_prog = true;
-			cur_aux(env)->arg_prog = regno;
-			continue;
-		}
-
-		if (is_kfunc_arg_ignore(btf, &args[i]) || is_kfunc_arg_implicit(meta, i))
-			continue;
-
-		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
-
-		if (btf_type_is_ptr(t)) {
-			ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
-			ref_tname = btf_name_by_offset(btf, ref_t->name_off);
-		}
-
-		if (btf_type_is_ptr(t) &&
-		    (bpf_register_is_null(reg) || type_may_be_null(reg->type)) &&
-		    !type_may_be_null(kf_arg_type)) {
-			const char *expected_type;
-
-			expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type);
-			verbose(env, "Possibly NULL pointer passed to trusted %s\n",
-				reg_arg_name(env, argno));
-			bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-					      "Add a NULL check and call the kfunc only on the non-NULL path.",
-					      "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s",
-					      expected_type);
-			return -EACCES;
-		}
-
-		if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) &&
-		    !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) {
-			const char *expected_type;
-
-			expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id);
-			verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n",
-				func_name, reg_arg_name(env, argno));
-			bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-					      "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.",
-					      "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc",
-					      expected_type);
-			return -EINVAL;
-		}
-
-		if (reg_is_referenced(env, reg))
-			update_ref_obj(&meta->ref_obj, reg);
-
-		if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) {
-			ret = mark_arg_precision(env, argno);
-			if (ret)
-				return ret;
-			continue;
-		}
-
-		if (is_kfunc_arg_map(btf, &args[i])) {
-			ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
-			ref_t = btf_type_by_id(btf_vmlinux, ref_id);
-			ref_tname = btf_name_by_offset(btf, ref_t->name_off);
-		}
-
-		switch (base_type(kf_arg_type)) {
-		case KF_ARG_CONST:
-		case KF_ARG_CONST_MEM_SIZE:
-		case KF_ARG_MEM_SIZE:
-		case KF_ARG_ANYTHING:
-		case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO:
-		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
-		case KF_ARG_PTR_TO_BTF_ID:
-		case KF_ARG_CONST_MAP_PTR:
-		case KF_ARG_PTR_TO_ITER:
-		case KF_ARG_PTR_TO_LIST_HEAD:
-		case KF_ARG_PTR_TO_LIST_NODE:
-		case KF_ARG_PTR_TO_RB_ROOT:
-		case KF_ARG_PTR_TO_RB_NODE:
-		case KF_ARG_PTR_TO_MEM:
-		case KF_ARG_PTR_TO_CALLBACK:
-		case KF_ARG_PTR_TO_CONST_STR:
-		case KF_ARG_PTR_TO_WORKQUEUE:
-		case KF_ARG_PTR_TO_TIMER:
-		case KF_ARG_PTR_TO_TASK_WORK:
-		case KF_ARG_PTR_TO_IRQ_FLAG:
-		case KF_ARG_PTR_TO_RES_SPIN_LOCK:
-		case KF_ARG_PTR_TO_ARENA:
-			break;
-		case KF_ARG_PTR_TO_DYNPTR:
-			arg_type = ARG_PTR_TO_DYNPTR;
-			break;
-		case KF_ARG_PTR_TO_CTX:
-			arg_type = ARG_PTR_TO_CTX;
-			break;
-		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
-			arg_type = ARG_PTR_TO_BTF_ID;
-			btf_id_fixed_off_ok = false;
-			break;
-		default:
-			verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type);
-			return -EFAULT;
-		}
-
-		if (regno == meta->release_regno)
-			arg_type |= OBJ_RELEASE;
-		ret = __check_func_arg_reg_off(env, reg, argno, arg_type,
-					       btf_id_fixed_off_ok);
-		if (ret < 0)
-			return ret;
-
-		switch (base_type(kf_arg_type)) {
-		case KF_ARG_CONST:
-			if (reg->type != SCALAR_VALUE) {
-				verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass an integer scalar value for this argument, not a pointer or resource object.",
-						      "the kfunc expects an integer scalar, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-
-			ret = process_const_arg(env, reg, argno, meta);
-			if (ret < 0) {
-				if (ret == -EINVAL)
-					bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-							      "Pass a compile-time constant or a value the verifier can prove is constant at this call.",
-							      "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path",
-							      reg_arg_name(env, argno));
-				return ret;
-			}
-			break;
-		case KF_ARG_ANYTHING:
-			if (reg->type != SCALAR_VALUE) {
-				verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass an integer scalar value for this argument, not a pointer or resource object.",
-						      "the kfunc expects an integer scalar, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-			break;
-		case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO:
-			if (reg->type != SCALAR_VALUE) {
-				verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass an integer scalar value for this argument, not a pointer or resource object.",
-						      "the kfunc expects an integer scalar, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-
-			if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size"))
-				meta->r0_rdonly = true;
-			ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem);
-			if (ret < 0) {
-				if (ret == -EINVAL)
-					bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-							      "Pass a verifier-known constant size for this kfunc buffer argument.",
-							      "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path",
-							      reg_arg_name(env, argno));
-				return ret;
-			}
-			break;
-		case KF_ARG_PTR_TO_CTX:
-			if (reg->type != PTR_TO_CTX) {
-				verbose(env, "%s expected pointer to ctx, but got %s\n",
-					reg_arg_name(env, argno), reg_type_str(env, reg->type));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass the original program context pointer or preserve it before modifying registers.",
-						      "the kfunc expects a context pointer, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-
-			if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
-				ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog));
-				if (ret < 0)
-					return -EINVAL;
-				meta->ret_btf_id  = ret;
-			}
-			break;
-		case KF_ARG_PTR_TO_ARENA:
-			if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) {
-				verbose(env, "%s is not a pointer to arena or scalar\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			break;
-		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
-			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				if (!is_bpf_obj_drop_kfunc(meta->func_id)) {
-					verbose(env, "%s expected for bpf_obj_drop()\n",
-						reg_arg_name(env, argno));
-					return -EINVAL;
-				}
-			} else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) {
-				if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) {
-					verbose(env, "%s expected for bpf_percpu_obj_drop()\n",
-						reg_arg_name(env, argno));
-					return -EINVAL;
-				}
-			} else {
-				verbose(env, "%s expected pointer to allocated object\n",
-					reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass a pointer returned by the matching BPF object allocation path.",
-						      "the kfunc expects an allocated object pointer, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-			if (!reg_is_referenced(env, reg)) {
-				verbose(env, "allocated object must be referenced\n");
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass the owned object pointer before it is released or transferred.",
-						      "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource",
-						      reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			if (meta->btf == btf_vmlinux) {
-				meta->arg_btf = reg->btf;
-				meta->arg_btf_id = reg->btf_id;
-			}
-			break;
-		case KF_ARG_PTR_TO_DYNPTR:
-		{
-			enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
-
-			if (is_kfunc_arg_uninit(btf, &args[i]))
-				dynptr_arg_type |= MEM_UNINIT;
-
-			if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb]) {
-				dynptr_arg_type |= DYNPTR_TYPE_SKB;
-			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_xdp]) {
-				dynptr_arg_type |= DYNPTR_TYPE_XDP;
-			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_skb_meta]) {
-				dynptr_arg_type |= DYNPTR_TYPE_SKB_META;
-			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) {
-				dynptr_arg_type |= DYNPTR_TYPE_FILE;
-			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) {
-				dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE;
-			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&
-				   (dynptr_arg_type & MEM_UNINIT)) {
-				enum bpf_dynptr_type parent_type = meta->dynptr.type;
-
-				if (parent_type == BPF_DYNPTR_TYPE_INVALID) {
-					verifier_bug(env, "no dynptr type for parent of clone");
-					return -EFAULT;
-				}
-
-				dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
-			}
-
-			ret = process_dynptr_func(env, reg, argno, insn_idx, func_name,
-						  dynptr_arg_type, &meta->ref_obj, &meta->dynptr);
-			if (ret < 0)
-				return ret;
-			break;
-		}
-		case KF_ARG_PTR_TO_ITER:
-			if (meta->func_id == special_kfunc_list[KF_bpf_iter_css_task_new]) {
-				if (!check_css_task_iter_allowlist(env)) {
-					verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n");
-					return -EINVAL;
-				}
-			}
-			ret = process_iter_arg(env, reg, argno, insn_idx, meta);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_LIST_HEAD:
-			if (reg->type != PTR_TO_MAP_VALUE &&
-			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "%s expected pointer to map value or allocated object\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
-			    !reg_is_referenced(env, reg)) {
-				verbose(env, "allocated object must be referenced\n");
-				return -EINVAL;
-			}
-			ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_RB_ROOT:
-			if (reg->type != PTR_TO_MAP_VALUE &&
-			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "%s expected pointer to map value or allocated object\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
-			    !reg_is_referenced(env, reg)) {
-				verbose(env, "allocated object must be referenced\n");
-				return -EINVAL;
-			}
-			ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_LIST_NODE:
-			if (is_kfunc_arg_nonown_allowed(btf, &args[i]) &&
-			    type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) {
-				/* Allow bpf_list_front/back return value for
-				 * __nonown_allowed list-node arguments.
-				 */
-				goto check_ok;
-			}
-			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "%s expected pointer to allocated object\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			if (!reg_is_referenced(env, reg)) {
-				verbose(env, "allocated object must be referenced\n");
-				return -EINVAL;
-			}
-check_ok:
-			ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_RB_NODE:
-			if (is_bpf_rbtree_add_kfunc(meta->func_id)) {
-				if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-					verbose(env, "%s expected pointer to allocated object\n",
-						reg_arg_name(env, argno));
-					return -EINVAL;
-				}
-				if (!reg_is_referenced(env, reg)) {
-					verbose(env, "allocated object must be referenced\n");
-					return -EINVAL;
-				}
-			} else {
-				if (!type_is_non_owning_ref(reg->type) &&
-				    !reg_is_referenced(env, reg)) {
-					verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name);
-					return -EINVAL;
-				}
-				if (in_rbtree_lock_required_cb(env)) {
-					verbose(env, "%s not allowed in rbtree cb\n", func_name);
-					return -EINVAL;
-				}
-			}
-
-			ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_CONST_MAP_PTR:
-			if (base_type(reg->type) != CONST_PTR_TO_MAP ||
-			    type_may_be_null(reg->type)) {
-				verbose(env, "pointer in %s isn't map pointer\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			ret = process_map_ptr_arg(env, reg, argno, meta);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_BTF_ID:
-			/* Only base_type is checked, further checks are done here */
-			if (base_type(reg->type) == PTR_TO_BTF_ID ||
-			    reg2btf_ids[base_type(reg->type)]) {
-				if (!is_trusted_reg(env, reg) ||
-				    bpf_type_has_unsafe_modifiers(reg->type)) {
-					if (!is_kfunc_rcu(meta)) {
-						const char *expected_type;
-
-						expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id);
-						verbose(env, "%s must be referenced or trusted\n",
-							reg_arg_name(env, argno));
-						bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-								      "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.",
-								      "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s",
-								      expected_type,
-								      reg_arg_name(env, argno),
-								      bpf_diag_reg_type_plain(env, reg->type));
-						return -EINVAL;
-					}
-					if (!is_rcu_reg(reg)) {
-						const char *expected_type;
-
-						expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id);
-						verbose(env, "%s must be a rcu pointer\n",
-							reg_arg_name(env, argno));
-						bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-								      "Use this kfunc with a pointer that is valid in an RCU read lock region.",
-								      "the kfunc requires an RCU-protected pointer to %s, but %s is %s",
-								      expected_type,
-								      reg_arg_name(env, argno),
-								      bpf_diag_reg_type_plain(env, reg->type));
-						return -EINVAL;
-					}
-				}
-
-				ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno);
-				if (ret < 0)
-					return ret;
-				break;
-			}
-
-			if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
-				enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id);
-				const char *expected_type;
-
-				verbose(env, "%s is %s expected %s %s",
-					reg_arg_name(env, argno), reg_type_str(env, reg->type),
-					btf_type_str(ref_t), ref_tname);
-				if (reg2btf_type != NOT_INIT)
-					verbose(env, " or %s", reg_type_str(env, reg2btf_type));
-				verbose(env, "\n");
-				expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id);
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.",
-						      "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer",
-						      expected_type,
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-
-			/*
-			 * If the register does not contain btf id but the argument type is a pointer to
-			 * scalar-only struct, allow verifying it as a fixed size memory.
-			 */
-			kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE;
-			fallthrough;
-		case KF_ARG_PTR_TO_MEM:
-			if (kf_arg_type & MEM_FIXED_SIZE) {
-				bool known_memory;
-
-				resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
-				if (IS_ERR(resolve_ret)) {
-					verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n",
-						reg_arg_name(env, argno), btf_type_str(ref_t),
-						ref_tname, PTR_ERR(resolve_ret));
-					return -EINVAL;
-				}
-				ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE,
-						    meta, &known_memory);
-				if (ret < 0) {
-					const char *expected_type;
-
-					expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id);
-					if (known_memory)
-						bpf_diag_call_arg_fmt(
-							env, insn_idx, argno, func_name,
-							"Pass memory with at least the required number of accessible bytes and suitable read and write access.",
-							"the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size",
-							type_size, expected_type,
-							bpf_diag_reg_type_plain(env, reg->type));
-					else
-						bpf_diag_call_arg_fmt(
-							env, insn_idx, argno, func_name,
-							"Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.",
-							"the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory",
-							type_size, expected_type,
-							bpf_diag_reg_type_plain(env, reg->type));
-					return ret;
-				}
-			}
-			break;
-		case KF_ARG_CONST_MEM_SIZE:
-			ret = process_const_arg(env, reg, argno, meta);
-			if (ret < 0) {
-				if (ret == -EINVAL)
-					bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-							      "Pass a compile-time constant or a value the verifier can prove is constant at this call.",
-							      "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path",
-							      reg_arg_name(env, argno));
-				return ret;
-			}
-			fallthrough;
-		case KF_ARG_MEM_SIZE:
-		{
-			struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1);
-			struct bpf_reg_state *size_reg = reg;
-			argno_t buff_argno = argno_from_arg(i);
-			enum bpf_mem_size_failure failure;
-
-			if (reg->type != SCALAR_VALUE) {
-				verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass an integer scalar length for this memory argument.",
-						      "the kfunc expects a scalar memory size, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-
-			if (bpf_register_is_null(buff_reg))
-				break;
-
-			ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno,
-						 BPF_READ | BPF_WRITE, true, meta, &failure);
-			if (ret < 0) {
-				const char *buff_arg, *size_arg;
-
-				buff_arg = bpf_diag_arg_name(env, buff_argno);
-				size_arg = bpf_diag_arg_name(env, argno);
-				verbose(env, "%s and ", reg_arg_name(env, buff_argno));
-				verbose(env, "%s memory, len pair leads to invalid memory access\n",
-					reg_arg_name(env, argno));
-				if (failure == BPF_MEM_SIZE_FAIL_MEMORY) {
-					bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name,
-							      "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.",
-							      "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length",
-							      size_arg, buff_arg);
-				} else if (failure == BPF_MEM_SIZE_FAIL_SIZE) {
-					if (reg_smin(size_reg) < 0)
-						bpf_diag_call_arg_fmt(
-							env, insn_idx, argno, func_name,
-							"Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.",
-							"the memory size in %s may be negative because its signed minimum is %lld",
-							size_arg, reg_smin(size_reg));
-					else
-						bpf_diag_call_arg_fmt(
-							env, insn_idx, argno, func_name,
-							"Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.",
-							"the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes",
-							size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ);
-				}
-				return ret;
-			}
-			break;
-		}
-		case KF_ARG_PTR_TO_CALLBACK:
-			if (reg->type != PTR_TO_FUNC) {
-				verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			meta->subprogno = reg->subprogno;
-			break;
-		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
-			if (!type_is_ptr_alloc_obj(reg->type)) {
-				verbose(env, "%s is neither owning or non-owning ref\n",
-					reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.",
-						      "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-			if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg))
-				meta->arg_owning_ref = true;
-
-			rec = reg_btf_record(reg);
-			if (!rec) {
-				verifier_bug(env, "Couldn't find btf_record");
-				return -EFAULT;
-			}
-
-			if (rec->refcount_off < 0) {
-				verbose(env, "%s doesn't point to a type with bpf_refcount field\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-
-			meta->arg_btf = reg->btf;
-			meta->arg_btf_id = reg->btf_id;
-			break;
-		case KF_ARG_PTR_TO_CONST_STR:
-			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "%s doesn't point to a const string\n",
-					reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.",
-						      "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-			ret = check_arg_const_str(env, reg, argno);
-			if (ret)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_WORKQUEUE:
-			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "%s doesn't point to a map value\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_TIMER:
-			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "%s doesn't point to a map value\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			ret = process_timer_func(env, reg, argno, &meta->map);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_TASK_WORK:
-			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "%s doesn't point to a map value\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
-			ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_IRQ_FLAG:
-			if (reg->type != PTR_TO_STACK) {
-				verbose(env, "%s doesn't point to an irq flag on stack\n",
-					reg_arg_name(env, argno));
-				bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name,
-						      "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().",
-						      "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s",
-						      reg_arg_name(env, argno),
-						      bpf_diag_reg_type_plain(env, reg->type));
-				return -EINVAL;
-			}
-			ret = process_irq_flag(env, reg, argno, meta);
-			if (ret < 0)
-				return ret;
-			break;
-		case KF_ARG_PTR_TO_RES_SPIN_LOCK:
-		{
-			int flags = PROCESS_RES_LOCK;
-
-			if (in_rbtree_lock_required_cb(env)) {
-				verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n");
-				return -EACCES;
-			}
-
-			if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "%s doesn't point to map value or allocated object\n",
-					reg_arg_name(env, argno));
-				return -EINVAL;
-			}
+			btf_field_type_name(head_field_type),
+			btf_field_type_name(node_field_type),
+			field->graph_root.node_offset,
+			btf_name_by_offset(field->graph_root.btf, et->name_off),
+			node_off, btf_name_by_offset(reg->btf, t->name_off));
+		return -EINVAL;
+	}
+	meta->arg_btf = reg->btf;
+	meta->arg_btf_id = reg->btf_id;
 
-			if (!is_bpf_res_spin_lock_kfunc(meta->func_id))
-				return -EFAULT;
-			if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
-			    meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])
-				flags |= PROCESS_SPIN_LOCK;
-			if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] ||
-			    meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])
-				flags |= PROCESS_LOCK_IRQ;
-			ret = process_spin_lock(env, reg, argno, flags);
-			if (ret < 0)
-				return ret;
-			break;
-		}
-		}
+	if (node_off != field->graph_root.node_offset) {
+		verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n",
+			node_off, btf_field_type_name(node_field_type),
+			field->graph_root.node_offset,
+			btf_name_by_offset(field->graph_root.btf, et->name_off));
+		return -EINVAL;
 	}
 
 	return 0;
 }
 
+static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
+					   struct bpf_reg_state *reg, argno_t argno,
+					   struct bpf_call_arg_meta *meta)
+{
+	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
+						  BPF_LIST_HEAD, BPF_LIST_NODE,
+						  &meta->arg_list_head.field);
+}
+
+static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
+					     struct bpf_reg_state *reg, argno_t argno,
+					     struct bpf_call_arg_meta *meta)
+{
+	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
+						  BPF_RB_ROOT, BPF_RB_NODE,
+						  &meta->arg_rbtree_root.field);
+}
+
+/*
+ * css_task iter allowlist is needed to avoid dead locking on css_set_lock.
+ * LSM hooks and iters (both sleepable and non-sleepable) are safe.
+ * Any sleepable progs are also safe since bpf_check_attach_target() enforce
+ * them can only be attached to some specific hook points.
+ */
+static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)
+{
+	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
+
+	switch (prog_type) {
+	case BPF_PROG_TYPE_LSM:
+		return true;
+	case BPF_PROG_TYPE_TRACING:
+		if (env->prog->expected_attach_type == BPF_TRACE_ITER)
+			return true;
+		fallthrough;
+	default:
+		return in_sleepable(env);
+	}
+}
+
 int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env,
 			     s32 func_id,
 			     s16 offset,
@@ -13404,12 +14297,20 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn
 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
 	const struct bpf_func_proto *fn;
 	enum bpf_arg_type at;
+	bool full_write;
 	s64 size;
 
 	if (bpf_get_helper_proto(env, insn->imm, &fn) < 0)
 		return S64_MIN;
 
 	at = fn->arg_type[arg];
+	/*
+	 * Generic outputs may leave bytes untouched. Keep prior initialization
+	 * live when the caller cannot read uninitialized bytes. Constructors of
+	 * special objects, such as dynptrs, still define their storage.
+	 */
+	full_write = (at & MEM_UNINIT) &&
+		     (!arg_type_is_raw_mem(at) || env->allow_uninit_stack);
 
 	switch (base_type(at)) {
 	case ARG_PTR_TO_MAP_KEY:
@@ -13476,11 +14377,11 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn
 			}
 			/*
 			 * Size arg is const on each path but differs across merged
-			 * paths. MAX_BPF_STACK is a safe upper bound for reads.
+			 * paths. Reads may extend anywhere up to the frame top.
 			 */
-			if (at & MEM_UNINIT)
+			if (full_write)
 				return 0;
-			return MAX_BPF_STACK;
+			return S64_MIN;
 		}
 		return S64_MIN;
 	case ARG_PTR_TO_DYNPTR:
@@ -13498,10 +14399,10 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn
 	}
 out:
 	/*
-	 * MEM_UNINIT args are write-only: the helper initializes the
-	 * buffer without reading it.
+	 * Other accesses keep the previous state live, including untouched bytes
+	 * of an unprivileged generic output.
 	 */
-	if (at & MEM_UNINIT)
+	if (full_write)
 		return -size;
 	return size;
 }
@@ -13524,7 +14425,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *
 	const struct btf_param *args;
 	const struct btf_type *t, *ref_t;
 	const struct btf *btf;
-	u32 nargs, type_size;
+	u32 i, slot, nargs, type_size;
 	s64 size;
 
 	if (bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta) < 0)
@@ -13533,30 +14434,41 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *
 	btf = meta.btf;
 	args = btf_params(meta.func_proto);
 	nargs = btf_type_vlen(meta.func_proto);
-	if (arg >= nargs)
+
+	/*
+	 * @arg is an argument slot and a 16-byte parameter takes two of them,
+	 * so walk the parameters to find the one that starts at this slot. A
+	 * slot holding the upper eightbyte of such a parameter belongs to no
+	 * pointer, and neither does a slot past the last parameter.
+	 */
+	for (i = 0, slot = 0; i < nargs && slot < arg; i++)
+		slot += btf_arg_slots(btf_type_skip_modifiers(btf, args[i].type, NULL));
+	if (i >= nargs || slot != arg)
 		return 0;
 
-	t = btf_type_skip_modifiers(btf, args[arg].type, NULL);
+	t = btf_type_skip_modifiers(btf, args[i].type, NULL);
 	if (!btf_type_is_ptr(t))
 		return 0;
 
 	/* dynptr: fixed 16-byte on-stack representation */
-	if (is_kfunc_arg_dynptr(btf, &args[arg])) {
+	if (is_kfunc_arg_dynptr(btf, &args[i])) {
 		size = BPF_DYNPTR_SIZE;
 		goto out;
 	}
 
-	/* ptr + __sz/__szk pair: size is in the next register */
-	if (arg + 1 < nargs &&
-	    (btf_param_match_suffix(btf, &args[arg + 1], "__sz") ||
-	     btf_param_match_suffix(btf, &args[arg + 1], "__szk"))) {
+	/* ptr + __sz/__szk pair: the size follows the pointer */
+	if (i + 1 < nargs &&
+	    (btf_param_match_suffix(btf, &args[i + 1], "__sz") ||
+	     btf_param_match_suffix(btf, &args[i + 1], "__szk"))) {
 		int size_reg = BPF_REG_1 + arg + 1;
 
-		if (aux->const_reg_mask & BIT(size_reg)) {
+		if (size_reg <= MAX_BPF_FUNC_REG_ARGS &&
+		    (aux->const_reg_mask & BIT(size_reg))) {
 			size = (s64)aux->const_reg_vals[size_reg];
 			goto out;
 		}
-		return MAX_BPF_STACK;
+		/* Unknown size: the read may extend anywhere up to the frame top. */
+		return S64_MIN;
 	}
 
 	/* fixed-size pointed-to type: resolve via BTF */
@@ -13571,7 +14483,8 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *
 	/* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */
 	if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW)
 		return -size;
-	if (is_kfunc_arg_uninit(btf, &args[arg]))
+	if (is_kfunc_arg_uninit(btf, &args[i]) &&
+	    (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack))
 		return -size;
 	return size;
 }
@@ -13650,7 +14563,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
 
 		struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
 		if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) {
-			if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {
+			if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) {
 				verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
 				return -EINVAL;
 			}
@@ -13687,12 +14600,12 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
 		struct btf_field *field = meta->arg_rbtree_root.field;
 
 		mark_reg_graph_node(regs, BPF_REG_0, &field->graph_root);
-	} else if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx]) {
+	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_cast_to_kern_ctx])) {
 		mark_reg_known_zero(env, regs, BPF_REG_0);
 		regs[BPF_REG_0].type = PTR_TO_BTF_ID | PTR_TRUSTED;
 		regs[BPF_REG_0].btf = desc_btf;
 		regs[BPF_REG_0].btf_id = meta->ret_btf_id;
-	} else if (meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
+	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_rdonly_cast])) {
 		ret_t = btf_type_by_id(desc_btf, meta->arg_constant.value);
 		if (!ret_t) {
 			verbose(env, "Unknown type ID %lld passed to kfunc bpf_rdonly_cast\n",
@@ -13712,8 +14625,8 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
 				"kfunc bpf_rdonly_cast type ID argument must be of a struct or void\n");
 			return -EINVAL;
 		}
-	} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice] ||
-		   meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice_rdwr]) {
+	} else if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice]) ||
+		   is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) {
 		enum bpf_type_flag type_flag = get_dynptr_type_flag(meta->dynptr.type);
 
 		mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -13728,7 +14641,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
 		/* PTR_MAYBE_NULL will be added when is_kfunc_ret_null is checked */
 		regs[BPF_REG_0].type = PTR_TO_MEM | type_flag;
 
-		if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_slice]) {
+		if (is_kfunc_call(meta, special_kfunc_list[KF_bpf_dynptr_slice])) {
 			regs[BPF_REG_0].type |= MEM_RDONLY;
 		} else {
 			/* this will set env->seen_direct_write to true */
@@ -13764,7 +14677,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	struct bpf_insn_aux_data *insn_aux;
 	const char *operation;
 	int err, insn_idx = *insn_idx_p;
-	u32 i, nargs, ptr_type_id;
+	u32 i, proto_slots, ptr_type_id, ret_nregs = 1;
 	struct bpf_kfunc_desc *desc;
 	struct btf *desc_btf;
 	int id;
@@ -13834,12 +14747,30 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		return -EACCES;
 	}
 
+	if (is_kfunc_perfmon(&meta) && !env->allow_ptr_leaks) {
+		verbose(env, "%s is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
+			func_name);
+		operation = bpf_diag_fmt(env, "kfunc %s", func_name);
+		bpf_diag_policy(env, insn_idx, operation, "the kfunc requires CAP_PERFMON",
+				"Load the program with CAP_PERFMON, or avoid the kfunc.");
+		return -EPERM;
+	}
+
 	sleepable = bpf_is_kfunc_sleepable(&meta);
-	if (sleepable && !in_sleepable(env)) {
-		verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name);
+	if (sleepable && !in_sleepable_context(env)) {
+		const char *suggestion;
+
+		if (in_sleepable(env)) {
+			verbose(env, "kernel func %s is sleepable within %s\n",
+				func_name, non_sleepable_context_description(env));
+			suggestion = "Move the kfunc call outside the critical section, or use a non-sleepable kfunc.";
+		} else {
+			verbose(env, "program must be sleepable to call sleepable kfunc %s\n",
+				func_name);
+			suggestion = "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc.";
+		}
 		operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name);
-		bpf_diag_ctx_forbidden(env, insn_idx, operation,
-			"Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc.");
+		bpf_diag_ctx_forbidden(env, insn_idx, operation, suggestion);
 		return -EACCES;
 	}
 
@@ -13848,10 +14779,14 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		insn_aux->non_sleepable = true;
 
 	/* Check the arguments */
-	err = check_kfunc_args(env, &meta, insn_idx);
+	err = check_func_args(env, &meta, insn_idx);
 	if (err < 0)
 		return err;
 
+	err = mark_raw_stack(env, &meta, insn_idx);
+	if (err)
+		return err;
+
 	if ((is_bpf_obj_drop_kfunc(meta.func_id) ||
 	     is_bpf_percpu_obj_drop_kfunc(meta.func_id)) && (is_tracing_prog_type(prog_type) ||
 	     /* is_tracing_prog_type() for now doesn't cover non-iterator tracing progs. */
@@ -13897,6 +14832,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 	}
 
+	if (is_call_rcu_kfunc(meta.func_id)) {
+		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+					 set_rcu_callback_state);
+		if (err) {
+			verbose(env, "kfunc %s#%d failed callback verification\n",
+				func_name, meta.func_id);
+			return err;
+		}
+	}
+
 	rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta);
 	rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);
 
@@ -13939,15 +14884,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			invalidate_rcu_protected_refs(env);
 	}
 
-	if (sleepable && !in_sleepable_context(env)) {
-		verbose(env, "kernel func %s is sleepable within %s\n",
-			func_name, non_sleepable_context_description(env));
-		operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name);
-		bpf_diag_ctx_forbidden(env, insn_idx, operation,
-			"Move the kfunc call outside the critical section, or use a non-sleepable kfunc.");
-		return -EACCES;
-	}
-
 	if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) {
 		verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n");
 		return -EACCES;
@@ -14018,10 +14954,63 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	}
 
 	if (btf_type_is_scalar(t)) {
-		mark_reg_unknown(env, regs, BPF_REG_0);
+		ret_nregs = mark_kfunc_ret_regs(env, regs, t->size);
 		if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
 		    meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
 			__mark_reg_const_zero(env, &regs[BPF_REG_0]);
+	} else if (btf_type_is_struct(t)) {
+		struct btf_member_path path = {};
+		const char *member_note = "";
+
+		/*
+		 * The returned struct comes back as raw register bits modeled
+		 * as an unknown scalar, so a pointer member would be laundered
+		 * into a scalar and escape provenance and reference tracking.
+		 * Only scalars and arena pointers are allowed; see
+		 * btf_validate_return_type() in btf.c for why an arena pointer
+		 * is safe here.
+		 */
+		if (!btf_struct_member_walk(env, desc_btf, t,
+					    BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) {
+			verbose(env,
+				"kernel function %s returns %s %s that is not composed of scalars or arena pointers\n",
+				func_name, btf_type_str(t),
+				btf_name_by_offset(desc_btf, t->name_off));
+			if (path.too_deep) {
+				member_note = bpf_diag_fmt(
+					env, " It nests structs more than %d levels deep.",
+					BTF_MEMBER_MAX_DEPTH);
+			} else if (path.depth) {
+				char bad_name[BTF_MEMBER_PATH_LEN];
+
+				btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name));
+				if (!path.bad_type) {
+					verbose(env, "member '%s' is a zero-length array\n",
+						bad_name);
+					member_note = bpf_diag_fmt(
+						env, " Its member '%s' is a zero-length array.",
+						bad_name);
+				} else {
+					verbose(env, "member '%s' has type %s\n", bad_name,
+						btf_type_str(path.bad_type));
+					member_note = bpf_diag_fmt(
+						env,
+						" Its member '%s' is %s, not a scalar or an arena pointer.",
+						bad_name, btf_type_str(path.bad_type));
+				}
+			}
+			bpf_diag_program_structure(
+				env, insn_idx, "unsupported kernel function return type",
+				"Call a kernel function that returns only scalars or arena pointers by value.",
+				"%s() returns %s %s by value.%s "
+				"Only kfuncs returning scalar values or arena pointers, or "
+				"structures composed of scalar values or arena pointers are "
+				"supported.",
+				func_name, btf_type_str(t),
+				btf_name_by_offset(desc_btf, t->name_off), member_note);
+			return -EINVAL;
+		}
+		ret_nregs = mark_kfunc_ret_regs(env, regs, t->size);
 	} else if (btf_type_is_ptr(t)) {
 		ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
 		err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);
@@ -14137,11 +15126,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	if (bpf_is_kfunc_pkt_changing(&meta))
 		clear_all_pkt_pointers(env);
 
-	nargs = btf_type_vlen(meta.func_proto);
-	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
+	proto_slots = kfunc_abi_slots(&desc->func_model);
+	if (proto_slots > MAX_BPF_FUNC_REG_ARGS) {
 		struct bpf_func_state *caller = cur_func(env);
 		struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno];
-		u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS;
+		u16 out_stack_arg_cnt = proto_slots - MAX_BPF_FUNC_REG_ARGS;
 		u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt;
 
 		if (stack_arg_cnt > caller_info->stack_arg_cnt)
@@ -14152,7 +15141,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	 * Record R0 before process_iter_next_call() snapshots the alternate
 	 * iterator path's diagnostic position.
 	 */
-	bpf_diag_mod_end(env);
+	if (ret_nregs == 1)
+		bpf_diag_mod_end(env);
 
 	if (bpf_is_iter_next_kfunc(&meta)) {
 		err = process_iter_next_call(env, insn_idx, &meta);
@@ -14229,7 +15219,8 @@ enum {
 	REASON_STACK	= -5,
 };
 
-static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
+static int retrieve_ptr_limit(const struct bpf_verifier_env *env,
+			      const struct bpf_reg_state *ptr_reg,
 			      u32 *alu_limit, bool mask_to_left)
 {
 	u32 max = 0, ptr_limit = 0;
@@ -14241,7 +15232,7 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
 		 * offset where we would need to deal with min/max bounds is
 		 * currently prohibited for unprivileged.
 		 */
-		max = MAX_BPF_STACK + mask_to_left;
+		max = env->stack_limit + mask_to_left;
 		ptr_limit = -ptr_reg->var_off.value;
 		break;
 	case PTR_TO_MAP_VALUE:
@@ -14361,7 +15352,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
 				     (opcode == BPF_SUB && !off_is_neg);
 	}
 
-	err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
+	err = retrieve_ptr_limit(env, ptr_reg, &alu_limit, info->mask_to_left);
 	if (err < 0)
 		return err;
 
@@ -14491,7 +15482,7 @@ static int check_stack_access_for_ptr_arithmetic(
 		return -EACCES;
 	}
 
-	if (off >= 0 || off < -MAX_BPF_STACK) {
+	if (off >= 0 || off < -(int)env->stack_limit) {
 		verbose(env, "R%d stack pointer arithmetic goes out of range, "
 			"prohibited for !root; off=%d\n", regno, off);
 		return -EACCES;
@@ -15704,6 +16695,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
 	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
 	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
 	bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
+	struct bpf_insn_aux_data *aux = cur_aux(env);
 	u8 opcode = BPF_OP(insn->code);
 	int err;
 
@@ -15715,12 +16707,23 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
 
 	/* Case where at least one operand is an arena. */
 	if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) {
-		struct bpf_insn_aux_data *aux = cur_aux(env);
 
 		if (dst_reg->type != PTR_TO_ARENA)
 			*dst_reg = *src_reg;
 
 		if (BPF_CLASS(insn->code) == BPF_ALU64) {
+			/*
+			 * Only arena pointers set needs_zext, but doing so
+			 * modifies the instruction at fixup time to an ALU32
+			 * and makes it unsuitable for 64-bit scalar args. We
+			 * prevent zext from being set if the instruction has
+			 * been previously called with non-arena registers.
+			 */
+			if (aux->prevent_zext) {
+				verbose(env, "same insn cannot be used with and without arena pointer\n");
+				return -EINVAL;
+			}
+
 			/*
 			 * 32-bit operations zero upper bits automatically.
 			 * 64-bit operations need to be converted to 32.
@@ -15733,6 +16736,16 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
 		return 0;
 	}
 
+	/* Prevent the instruction from being used with arena pointers (see above). */
+	if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) {
+		if (aux->needs_zext) {
+			verbose(env, "same insn cannot be used with and without arena pointer\n");
+			return -EINVAL;
+		}
+
+		aux->prevent_zext = true;
+	}
+
 	if (dst_reg->type != SCALAR_VALUE)
 		ptr_reg = dst_reg;
 
@@ -16826,10 +17839,12 @@ static void collect_linked_regs(struct bpf_verifier_env *env,
 			reg = &func->regs[j];
 			__collect_linked_regs(linked_regs, reg, id, i, j, true);
 		}
-		for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
-			if (!bpf_is_spilled_reg(&func->stack[j]))
+		for (j = 0; j < bpf_stack_nr_slots(func); j++) {
+			struct bpf_stack_state *ss = bpf_stack_slot(func, j);
+
+			if (!bpf_is_spilled_reg(ss))
 				continue;
-			reg = &func->stack[j].spilled_ptr;
+			reg = &ss->spilled_ptr;
 			__collect_linked_regs(linked_regs, reg, id, i, j, false);
 		}
 	}
@@ -16849,7 +17864,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s
 	for (i = 0; i < linked_regs->cnt; ++i) {
 		e = &linked_regs->entries[i];
 		reg = e->is_reg ? &vstate->frame[e->frameno]->regs[e->regno]
-				: &vstate->frame[e->frameno]->stack[e->spi].spilled_ptr;
+				: &bpf_stack_slot(vstate->frame[e->frameno], e->spi)->spilled_ptr;
 		if (reg->type != SCALAR_VALUE || reg == known_reg)
 			continue;
 		if ((reg->id & ~BPF_ADD_CONST) != (known_reg->id & ~BPF_ADD_CONST))
@@ -16967,7 +17982,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 	}
 
 	if (insn_flags) {
-		err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0);
+		err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, NULL, 0);
 		if (err)
 			return err;
 	}
@@ -17037,7 +18052,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 	 * if parent state is created.
 	 */
 	if (linked_regs.cnt > 1) {
-		err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs));
+		u16 packed[LINKED_REGS_MAX];
+
+		linked_regs_pack(&linked_regs, packed);
+		err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, packed, linked_regs.cnt);
 		if (err)
 			return err;
 	}
@@ -17602,37 +18620,54 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
 	return 0;
 }
 
-static int check_global_subprog_return_code(struct bpf_verifier_env *env)
+static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno)
 {
-	struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);
-	struct bpf_func_state *cur_frame = cur_func(env);
+	struct bpf_reg_state *reg;
 	int err;
 
-	if (subprog_returns_void(env, cur_frame->subprogno))
-		return 0;
-
-	err = check_reg_arg(env, BPF_REG_0, SRC_OP);
+	err = check_reg_arg(env, regno, SRC_OP);
 	if (err)
 		return err;
 
 	/* Pointers to arena are safe to pass between subprograms. */
-	if (is_arena_reg(env, BPF_REG_0))
+	if (is_arena_reg(env, regno))
 		return 0;
 
-	if (is_pointer_value(env, BPF_REG_0)) {
-		verbose(env, "R%d leaks addr as return value\n", BPF_REG_0);
+	if (is_pointer_value(env, regno)) {
+		verbose(env, "R%d leaks addr as return value\n", regno);
 		return -EACCES;
 	}
 
+	reg = reg_state(env, regno);
 	if (reg->type != SCALAR_VALUE) {
-		verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
-			reg_type_str(env, reg->type));
+		verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",
+			regno, reg_type_str(env, reg->type));
 		return -EINVAL;
 	}
 
 	return 0;
 }
 
+static int check_global_subprog_return_code(struct bpf_verifier_env *env)
+{
+	struct bpf_func_state *cur_frame = cur_func(env);
+	u32 subprog = cur_frame->subprogno;
+	u32 i, nregs;
+	int err;
+
+	if (subprog_returns_void(env, subprog))
+		return 0;
+
+	nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1;
+	for (i = 0; i < nregs; i++) {
+		err = check_global_ret_scalar_reg(env, ret_regs[i]);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
 /* Bitmask with 1s for all caller saved registers */
 #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
 
@@ -17674,11 +18709,11 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
 			       (bpf_verifier_inlines_helper_call(env, call->imm) ||
 				bpf_jit_inlines_helper_call(call->imm));
 		cs->is_void = fn->ret_type == RET_VOID;
-		cs->num_params = 0;
+		cs->arg_slot_cnt = 0;
 		for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) {
-			if (fn->arg_type[i] == ARG_DONTCARE)
+			if (fn->arg_type[i] == ARG_UNUSED)
 				break;
-			cs->num_params++;
+			cs->arg_slot_cnt++;
 		}
 		return true;
 	}
@@ -17690,7 +18725,7 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
 		if (err < 0)
 			/* error would be reported later */
 			return false;
-		cs->num_params = btf_type_vlen(meta.func_proto);
+		cs->arg_slot_cnt = btf_proto_slots(meta.btf, meta.func_proto);
 		cs->fastcall = meta.kfunc_flags & KF_FASTCALL;
 		cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type));
 		return true;
@@ -17799,7 +18834,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
 	 * - includes R1-R5 if corresponding parameter has is described
 	 *   in the function prototype.
 	 */
-	clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0);
+	clobbered_regs_mask = GENMASK(cs.arg_slot_cnt, cs.is_void ? 1 : 0);
 	/* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */
 	expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;
 
@@ -18165,10 +19200,25 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env,
 	return 0;
 }
 
+static int reject_gotox_out_of_subprog(struct bpf_verifier_env *env, u32 target,
+				       u32 subprog_start, u32 subprog_end)
+{
+	verbose(env, "indirect jump from insn %d to %u leaves the subprog [%u,%u)\n",
+		env->insn_idx, target, subprog_start, subprog_end);
+	bpf_diag_program_structure(
+		env, env->insn_idx, "indirect jump leaves subprogram",
+		"Keep every reachable jump-table target inside the subprogram of the indirect jump.",
+		"Instruction %d can jump indirectly to instruction %u, which is outside its own subprogram [%u,%u).",
+		env->insn_idx, target, subprog_start, subprog_end);
+	return -EINVAL;
+}
+
 /* gotox *dst_reg */
 static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *insn)
 {
 	struct bpf_verifier_state *other_branch;
+	struct bpf_subprog_info *subprog;
+	u32 subprog_start, subprog_end;
 	struct bpf_reg_state *dst_reg;
 	struct bpf_map *map;
 	u32 min_index, max_index;
@@ -18212,6 +19262,24 @@ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *in
 		return -EINVAL;
 	}
 
+	subprog = bpf_find_containing_subprog(env, env->insn_idx);
+	if (verifier_bug_if(!subprog, env, "no subprog contains insn %d", env->insn_idx))
+		return -EFAULT;
+	subprog_start = subprog->start;
+	subprog_end = (subprog + 1)->start;
+
+	/*
+	 * Maps are confined to a subprog, see compute_subprog_jts(), and the
+	 * targets are sorted and free of duplicates, see sort_insn_array_uniq(),
+	 * so either all of them belong to this subprog or none of them does.
+	 */
+	if (env->gotox_tmp_buf->items[0] < subprog_start)
+		return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[0],
+						   subprog_start, subprog_end);
+	if (env->gotox_tmp_buf->items[n - 1] >= subprog_end)
+		return reject_gotox_out_of_subprog(env, env->gotox_tmp_buf->items[n - 1],
+						   subprog_start, subprog_end);
+
 	for (i = 0; i < n - 1; i++) {
 		mark_indirect_target(env, env->gotox_tmp_buf->items[i]);
 		other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
@@ -18277,11 +19345,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 		env->jmps_processed++;
 		if (opcode == BPF_CALL) {
 			if (env->cur_state->active_locks) {
-				if ((insn->src_reg == BPF_REG_0 &&
-				     insn->imm != BPF_FUNC_spin_unlock &&
-				     insn->imm != BPF_FUNC_kptr_xchg) ||
-				    (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
-				     !kfunc_spin_allowed(env, insn->imm, insn->off))) {
+				/* similar to static subprog calls callx is allowed under a lock */
+				if (!bpf_is_callx(insn) &&
+				    ((insn->src_reg == BPF_REG_0 &&
+				      insn->imm != BPF_FUNC_spin_unlock &&
+				      insn->imm != BPF_FUNC_kptr_xchg) ||
+				     (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
+				      !kfunc_spin_allowed(env, insn->imm, insn->off)))) {
 					verbose(env,
 						"function calls are not allowed while holding a lock\n");
 					bpf_diag_ctx_active(
@@ -18294,6 +19364,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 			mark_reg_scratched(env, BPF_REG_0);
 			if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno]))
 				cur_func(env)->no_stack_arg_load = true;
+			if (bpf_is_callx(insn))
+				return check_func_callx(env, insn, &env->insn_idx);
 			if (insn->src_reg == BPF_PSEUDO_CALL)
 				return check_func_call(env, insn, &env->insn_idx);
 			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
@@ -18414,7 +19486,7 @@ static int do_check(struct bpf_verifier_env *env)
 		}
 
 		if (bpf_is_jmp_point(env, env->insn_idx)) {
-			err = bpf_push_jmp_history(env, state, 0, 0, 0, 0);
+			err = bpf_push_jmp_history(env, state, 0, 0, 0, NULL, 0);
 			if (err)
 				return err;
 		}
@@ -18869,6 +19941,30 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
 	return 0;
 }
 
+/*
+ * Keep track of whether the map is used by one program only. Such program may
+ * store the addresses of its functions into the map when it's frozen, see
+ * resolve_func_ptrs(), since nothing else relies on what the map has. After
+ * that the map is not available to other programs.
+ */
+static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map)
+{
+	unsigned long me = (unsigned long)env->prog->aux, old;
+
+	for (;;) {
+		old = READ_ONCE(map->user);
+		if (old == me || old == BPF_MAP_USER_MANY)
+			return 0;
+		if (old & BPF_MAP_USER_PATCHED) {
+			verbose(env, "map '%s' has addresses of functions of another program\n",
+				map->name);
+			return -EBUSY;
+		}
+		if (cmpxchg(&map->user, old, old ? BPF_MAP_USER_MANY : me) == old)
+			return 0;
+	}
+}
+
 static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
 {
 	int i, err;
@@ -18906,6 +20002,10 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
 
 	env->used_maps[env->used_map_cnt++] = map;
 
+	err = bpf_map_claim(env, map);
+	if (err)
+		return err;
+
 	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) {
 		err = bpf_insn_array_init(map, env->prog);
 		if (err) {
@@ -19060,6 +20160,14 @@ static int check_jmp_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
 
 	switch (opcode) {
 	case BPF_CALL:
+		if (bpf_is_callx(insn)) {
+			/* callx dst_reg */
+			if (insn->src_reg != BPF_REG_0 || insn->imm != 0 || insn->off != 0) {
+				verbose(env, "BPF_CALL|BPF_X uses reserved fields\n");
+				return -EINVAL;
+			}
+			return 0;
+		}
 		if (BPF_SRC(insn->code) != BPF_K ||
 		    (insn->src_reg != BPF_PSEUDO_KFUNC_CALL && insn->off != 0) ||
 		    (insn->src_reg != BPF_REG_0 && insn->src_reg != BPF_PSEUDO_CALL &&
@@ -19314,6 +20422,104 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env)
 	return 0;
 }
 
+/* that many pointers to functions are recognized in the maps of a program at most */
+#define BPF_MAX_FUNC_PTRS (1 << 20)
+
+static int add_func_ptr(struct bpf_verifier_env *env, struct bpf_map *map, u32 map_off,
+			u32 xlated_off)
+{
+	struct bpf_func_ptr *ptrs;
+
+	if (env->func_ptr_cnt == BPF_MAX_FUNC_PTRS) {
+		verbose(env, "too many pointers to functions in maps, the limit is %d\n",
+			BPF_MAX_FUNC_PTRS);
+		return -E2BIG;
+	}
+
+	/* grow by doubling, the array is sorted and searched later */
+	if (!(env->func_ptr_cnt & (env->func_ptr_cnt - 1))) {
+		ptrs = kvrealloc(env->func_ptrs,
+				 array_size(max(2 * env->func_ptr_cnt, 16U), sizeof(*ptrs)),
+				 GFP_KERNEL_ACCOUNT);
+		if (!ptrs)
+			return -ENOMEM;
+		env->func_ptrs = ptrs;
+	}
+	env->func_ptrs[env->func_ptr_cnt++] = (struct bpf_func_ptr){
+		.map = map,
+		.map_off = map_off,
+		.orig_off = xlated_off,
+		.xlated_off = xlated_off,
+	};
+	return 0;
+}
+
+/*
+ * Compilers put pointers to functions into read-only data: tables of functions,
+ * structures of operations, vtables, where they are mixed with other data.
+ * The loader stores such data in a frozen read-only array map and resolves
+ * a pointer to a static function to the offset in bytes of its first
+ * instruction in the program: the address of the function in the program.
+ *
+ * Find 64-bit values that look like that in the maps of a program that uses
+ * callx. It's a guess. When the value is not a pointer, the program either
+ * fails to load, because it does with a pointer what can be done with
+ * a number only, or it sees the address of a function instead of the number.
+ * It's known before the control flow graph of the program is built and the main
+ * verification pass begins which functions may be called via callx.
+ *
+ * When the program is jitted the offsets are replaced with the addresses of
+ * the functions in the map itself, see jit_subprogs(). Hence the program has to
+ * be the only user of the map, see bpf_map_claim(): nothing else may rely on
+ * what the map had.
+ *
+ * The program reads the addresses of its functions from there like any other
+ * data, so it has to be allowed to leak pointers.
+ */
+static int resolve_func_ptrs(struct bpf_verifier_env *env)
+{
+	int insn_cnt = env->prog->len;
+	int i, err, subprog;
+	struct bpf_map *map;
+	u64 addr, val;
+	u32 off;
+
+	if (!env->has_callx || !env->allow_ptr_leaks)
+		return 0;
+
+	for (i = 0; i < env->used_map_cnt; i++) {
+		map = env->used_maps[i];
+		/* coincidences in maps that are shared with other programs don't matter */
+		if (READ_ONCE(map->user) != (unsigned long)env->prog->aux)
+			continue;
+		if (map->map_type != BPF_MAP_TYPE_ARRAY || map->max_entries != 1 ||
+		    !bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr ||
+		    !IS_ERR_OR_NULL(map->record))
+			continue;
+		if (map->ops->map_direct_value_addr(map, &addr, 0))
+			continue;
+
+		for (off = 0; off + sizeof(u64) <= map->value_size; off += sizeof(u64)) {
+			if (!(off % SZ_64K))
+				cond_resched();
+			val = *(u64 *)(unsigned long)(addr + off);
+			if (!val || val % sizeof(struct bpf_insn) ||
+			    val / sizeof(struct bpf_insn) >= insn_cnt)
+				continue;
+			subprog = bpf_find_subprog(env, val / sizeof(struct bpf_insn));
+			if (subprog <= 0 || bpf_subprog_is_global(env, subprog))
+				continue;
+			err = add_func_ptr(env, map, off, val / sizeof(struct bpf_insn));
+			if (err)
+				return err;
+		}
+	}
+	if (env->func_ptr_cnt)
+		sort(env->func_ptrs, env->func_ptr_cnt, sizeof(*env->func_ptrs),
+		     cmp_func_ptrs, NULL);
+	return 0;
+}
+
 /* drop refcnt of maps used by the rejected program */
 static void release_maps(struct bpf_verifier_env *env)
 {
@@ -19421,13 +20627,14 @@ static void free_states(struct bpf_verifier_env *env)
 	}
 }
 
-static int do_check_common(struct bpf_verifier_env *env, int subprog)
+static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable)
 {
 	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
 	struct bpf_subprog_info *sub = subprog_info(env, subprog);
 	struct bpf_prog_aux *aux = env->prog->aux;
 	struct bpf_verifier_state *state;
 	struct bpf_reg_state *regs;
+	u32 old_insns_total = sub->insns_total;
 	u32 insn_processed = env->insn_processed;
 	int ret, i;
 
@@ -19440,7 +20647,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 	state->curframe = 0;
 	state->speculative = false;
 	state->branches = 1;
-	state->in_sleepable = env->prog->sleepable;
+	state->in_sleepable = is_sleepable;
 	state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT);
 	if (!state->frame[0]) {
 		kfree(state);
@@ -19480,20 +20687,21 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 			}
 
 			/* Also ensure the callback only has a single scalar argument. */
-			if (sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_ANYTHING) {
+			if (sub->arg_slot_cnt != 1 || sub->args[0].arg_type != ARG_SCALAR) {
 				verbose(env, "exception cb only supports single integer argument\n");
 				ret = -EINVAL;
 				goto out;
 			}
 		}
-		for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) {
+		for (i = BPF_REG_1;
+		     i <= min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++) {
 			arg = &sub->args[i - BPF_REG_1];
 			reg = &regs[i];
 
 			if (arg->arg_type == ARG_PTR_TO_CTX) {
 				reg->type = PTR_TO_CTX;
 				mark_reg_known_zero(env, regs, i);
-			} else if (arg->arg_type == ARG_ANYTHING) {
+			} else if (arg->arg_type == ARG_SCALAR) {
 				reg->type = SCALAR_VALUE;
 				mark_reg_unknown(env, regs, i);
 			} else if (arg->arg_type == ARG_PTR_TO_DYNPTR) {
@@ -19529,7 +20737,8 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 				goto out;
 			}
 		}
-		if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) {
+		if (env->prog->type == BPF_PROG_TYPE_EXT &&
+		    sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) {
 			verbose(env, "freplace programs with >%d args not supported yet\n",
 				MAX_BPF_FUNC_REG_ARGS);
 			ret = -EINVAL;
@@ -19542,9 +20751,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 		 */
 		if (env->prog->aux->func_info_aux) {
 			ret = btf_prepare_func_args(env, 0);
-			if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) {
+			if (ret || sub->arg_slot_cnt != 1 ||
+			    sub->args[0].arg_type != ARG_PTR_TO_CTX) {
 				env->prog->aux->func_info_aux[0].unreliable = true;
-				sub->arg_cnt = 1;
+				sub->arg_slot_cnt = 1;
 				sub->stack_arg_cnt = 0;
 			}
 		}
@@ -19581,8 +20791,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 	 * not accounted as callees by account_current_path().
 	 * Accumulate their total counts as total counts of the main or
 	 * global subprog hosting the async call.
+	 * Start from the saved total of earlier contexts: adding to the current
+	 * total would count this pass's synchronous paths twice.
 	 */
-	env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed;
+	sub->insns_total = old_insns_total + (env->insn_processed - insn_processed);
 	return ret;
 }
 
@@ -19610,14 +20822,19 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
 {
 	struct bpf_prog_aux *aux = env->prog->aux;
 	struct bpf_func_info_aux *sub_aux;
-	int i, ret, new_cnt;
+	int context, i, ret, new_cnt;
 
 	if (!aux->func_info)
 		return 0;
 
-	/* exception callback is presumed to be always called */
-	if (env->exception_callback_subprog)
-		subprog_aux(env, env->exception_callback_subprog)->called = true;
+	/*
+	 * Callbacks cannot throw, so the exception callback always runs in the
+	 * main program's context. It is presumed to be always called.
+	 */
+	if (env->exception_callback_subprog) {
+		sub_aux = subprog_aux(env, env->exception_callback_subprog);
+		sub_aux->called[env->prog->sleepable] = true;
+	}
 
 again:
 	new_cnt = 0;
@@ -19626,29 +20843,28 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
 			continue;
 
 		sub_aux = subprog_aux(env, i);
-		if (!sub_aux->called || sub_aux->verified)
-			continue;
+		for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) {
+			if (!sub_aux->called[context] || sub_aux->verified[context])
+				continue;
 
-		env->insn_idx = env->subprog_info[i].start;
-		WARN_ON_ONCE(env->insn_idx == 0);
-		ret = do_check_common(env, i);
-		if (ret) {
-			return ret;
-		} else if (env->log.level & BPF_LOG_LEVEL) {
-			verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n",
-				i, bpf_subprog_name(env, i));
-		}
+			env->insn_idx = env->subprog_info[i].start;
+			WARN_ON_ONCE(env->insn_idx == 0);
+			ret = do_check_common(env, i, context);
+			if (ret)
+				return ret;
+			if (env->log.level & BPF_LOG_LEVEL)
+				verbose(env, "Func#%d ('%s') is safe for any args "
+					"that match its prototype\n",
+					i, bpf_subprog_name(env, i));
 
-		/* We verified new global subprog, it might have called some
-		 * more global subprogs that we haven't verified yet, so we
-		 * need to do another pass over subprogs to verify those.
-		 */
-		sub_aux->verified = true;
-		new_cnt++;
+			sub_aux->verified[context] = true;
+			new_cnt++;
+		}
 	}
 
-	/* We can't loop forever as we verify at least one global subprog on
-	 * each pass.
+	/*
+	 * We can't loop forever as each pass verifies at least one new context,
+	 * and there are only two contexts per global subprog.
 	 */
 	if (new_cnt)
 		goto again;
@@ -19661,7 +20877,7 @@ static int do_check_main(struct bpf_verifier_env *env)
 	int ret;
 
 	env->insn_idx = 0;
-	ret = do_check_common(env, 0);
+	ret = do_check_common(env, 0, env->prog->sleepable);
 	if (!ret)
 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
 	return ret;
@@ -20989,6 +22205,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	return 0;
 }
 
+/*
+ * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably
+ * above the largest signature the kernel can verify, and far below anything
+ * that would make rejecting a load expensive. Deliberately a fixed number so
+ * that what the syscall accepts does not depend on PAGE_SIZE.
+ */
+#define BPF_PROG_MAX_SIGNATURE_SIZE	(64 * 1024)
+
 static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
 {
 	switch (keyring_id) {
@@ -20998,6 +22222,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
 		return BPF_SIG_KEYRING_SECONDARY;
 	case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING:
 		return BPF_SIG_KEYRING_PLATFORM;
+	case KEY_SPEC_BPF_KEYRING:
+		return BPF_SIG_KEYRING_BPF;
 	default:
 		return BPF_SIG_KEYRING_USER;
 	}
@@ -21026,18 +22252,25 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env,
 	u64 data_sz;
 	int err = 0;
 
-	/*
-	 * Don't attempt to use kmalloc_large or vmalloc for signatures.
-	 * Practical signature for BPF program should be below this limit.
-	 */
 	if (!attr->signature_size ||
-	    attr->signature_size > KMALLOC_MAX_CACHE_SIZE)
+	    attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE)
 		return -EINVAL;
-	if (system_keyring_id_check(attr->keyring_id) == 0)
+
+	if (system_keyring_id_check(attr->keyring_id) == 0) {
 		key = bpf_lookup_system_key(attr->keyring_id);
-	else
+	} else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) {
+		key = bpf_lookup_keyring();
+	} else if (bpf_keyring_enforced()) {
+		verbose(env, "caller-supplied keyring refused, use bpf keyring\n");
+		return -EPERM;
+	} else {
 		key = bpf_lookup_user_key(attr->keyring_id, 0);
+	}
 	if (!key) {
+		if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) {
+			verbose(env, "the bpf keyring is empty or has not been restricted\n");
+			return -ENOKEY;
+		}
 		verbose(env, "cannot resolve signing keyring with keyring_id %d\n",
 			attr->keyring_id);
 		return -EINVAL;
@@ -21129,6 +22362,26 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env,
 	return err;
 }
 
+/*
+ * bpf_clone_redirect() transmits from inside the program, so a tc egress or
+ * lwt_xmit program that redirects to its own device runs again on top of its
+ * own frame, and again from there, until the datapath's recursion limit drops
+ * the packet: ten frames deep. Ten MAX_BPF_STACK frames fit the kernel stack
+ * as they always did; ten MAX_BPF_STACK_JIT frames would not, so a program
+ * calling it keeps the smaller budget. The redirect helpers that transmit
+ * after the program has returned leave no frame behind.
+ */
+static bool bpf_prog_reenters_datapath(const struct bpf_prog *prog)
+{
+	const struct bpf_insn *insn = prog->insnsi;
+	int i;
+
+	for (i = 0; i < prog->len; i++, insn++)
+		if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_clone_redirect)
+			return true;
+	return false;
+}
+
 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	      struct bpf_log_attr *attr_log)
 {
@@ -21153,6 +22406,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	env->bt.env = env;
 	env->prog = *prog;
 	env->ops = bpf_verifier_ops[env->prog->type];
+	env->stack_limit = bpf_prog_stack_limit(env->prog);
+	if (bpf_prog_reenters_datapath(env->prog))
+		env->stack_limit = MAX_BPF_STACK;
 
 	env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token);
 	env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token);
@@ -21170,6 +22426,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	ret = bpf_diag_init(env);
 	if (ret)
 		goto err_prep;
+	if (env->prog->insnsi[env->prog->len - 1].code == (BPF_LD | BPF_IMM | BPF_DW)) {
+		verbose(env, "invalid bpf_ld_imm64 insn\n");
+		ret = -EINVAL;
+		goto err_prep;
+	}
 	if (env->signature) {
 		ret = bpf_prog_calc_tag(env->prog);
 		if (ret < 0)
@@ -21245,6 +22506,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	/* Apply CO-RE before validating the program's instruction layout. */
+	ret = bpf_check_core_relo(env, attr, uattr);
+	if (ret < 0)
+		goto skip_full_check;
+
 	/* Discover all subprograms before validating their layout and BTF. */
 	ret = add_subprogs(env);
 	if (ret < 0)
@@ -21254,7 +22520,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
-	/* Validate BTF against the complete subprogram layout and apply CO-RE. */
+	/* Validate BTF against the complete subprogram layout. */
 	ret = bpf_check_btf_info(env, attr, uattr);
 	if (ret < 0)
 		goto skip_full_check;
@@ -21264,6 +22530,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	/* Find pointers to functions in the read-only maps of the program. */
+	ret = resolve_func_ptrs(env);
+	if (ret < 0)
+		goto skip_full_check;
+
 	/* Build kfunc prototypes after resolving program resources. */
 	ret = add_kfuncs(env);
 	if (ret < 0)
@@ -21307,6 +22578,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	/* must precede the first bpf_ret_reg_pair() user below */
+	ret = bpf_compute_subprog_ret_regs(env);
+	if (ret < 0)
+		goto skip_full_check;
+
 	ret = bpf_compute_live_registers(env);
 	if (ret < 0)
 		goto skip_full_check;
@@ -21318,6 +22594,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	ret = do_check_main(env);
 	ret = ret ?: do_check_subprogs(env);
 
+	/* reject recursion through the callx edges found by the main pass */
+	if (ret == 0 && env->callx_edges)
+		ret = sort_subprogs_topo(env);
+
 	if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux))
 		ret = bpf_prog_offload_finalize(env);
 
@@ -21451,8 +22731,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	release_maps(env);
 	release_btfs(env);
 err_free_env:
-	if (env->insn_aux_data)
-		bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len);
+	bpf_free_subprog_jts(env);
 	vfree(env->insn_aux_data);
 	kvfree(env->fd_array);
 	bpf_stack_liveness_free(env);
@@ -21460,6 +22739,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	kvfree(env->scc_info);
 	kvfree(env->succ);
 	kvfree(env->gotox_tmp_buf);
+	kvfree(env->callx_edges);
+	kvfree(env->func_ptrs);
 	bpf_diag_free(env);
 	kvfree(env);
 	return ret;
diff --git a/kernel/capability.c b/kernel/capability.c
index 90e6ab62f6db5c..a689dae590ffbf 100644
--- a/kernel/capability.c
+++ b/kernel/capability.c
@@ -470,7 +470,7 @@ EXPORT_SYMBOL(file_ns_capable);
  * Return true if the inode uid and gid are within the namespace.
  */
 bool privileged_wrt_inode_uidgid(struct user_namespace *ns,
-				 struct mnt_idmap *idmap,
+				 const struct mnt_idmap *idmap,
 				 const struct inode *inode)
 {
 	return vfsuid_has_mapping(ns, i_uid_into_vfsuid(idmap, inode)) &&
@@ -487,7 +487,7 @@ bool privileged_wrt_inode_uidgid(struct user_namespace *ns,
  * its own user namespace and that the given inode's uid and gid are
  * mapped into the current user namespace.
  */
-bool capable_wrt_inode_uidgid(struct mnt_idmap *idmap,
+bool capable_wrt_inode_uidgid(const struct mnt_idmap *idmap,
 			      const struct inode *inode, int cap)
 {
 	struct user_namespace *ns = current_user_ns();
diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
index 58797123b752f3..233fb8b45d2c33 100644
--- a/kernel/cgroup/cgroup-internal.h
+++ b/kernel/cgroup/cgroup-internal.h
@@ -254,7 +254,7 @@ void cgroup_procs_write_finish(struct task_struct *task,
 			       enum cgroup_attach_lock_mode lock_mode)
 	__releases(&cgroup_threadgroup_rwsem);
 
-void cgroup_lock_and_drain_offline(struct cgroup *cgrp);
+int cgroup_lock_and_drain_offline(struct cgroup *cgrp);
 
 int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode);
 int cgroup_rmdir(struct kernfs_node *kn);
@@ -285,7 +285,6 @@ extern struct kernfs_syscall_ops cgroup1_kf_syscall_ops;
 extern const struct fs_parameter_spec cgroup1_fs_parameters[];
 
 int proc_cgroupstats_show(struct seq_file *m, void *v);
-bool cgroup1_ssid_disabled(int ssid);
 void cgroup1_pidlist_destroy_all(struct cgroup *cgrp);
 void cgroup1_release_agent(struct work_struct *work);
 void cgroup1_check_for_release(struct cgroup *cgrp);
diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index a4337c9b528786..7f55a7bc840283 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -2,7 +2,7 @@
 #include "cgroup-internal.h"
 
 #include <linux/ctype.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/sort.h>
 #include <linux/delay.h>
 #include <linux/mm.h>
@@ -506,8 +506,8 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of,
 	enum cgroup_attach_lock_mode lock_mode;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENODEV;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	task = cgroup_procs_write_start(buf, threadgroup, &lock_mode);
 	ret = PTR_ERR_OR_ZERO(task);
@@ -569,8 +569,8 @@ static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of,
 		return -EPERM;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENODEV;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 	spin_lock(&release_agent_path_lock);
 	strscpy(cgrp->root->release_agent_path, strstrip(buf),
 		sizeof(cgrp->root->release_agent_path));
@@ -605,10 +605,7 @@ static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css,
 static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css,
 					  struct cftype *cft, u64 val)
 {
-	if (val)
-		set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
-	else
-		clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags);
+	assign_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags, val);
 	return 0;
 }
 
@@ -621,10 +618,7 @@ static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css,
 static int cgroup_clone_children_write(struct cgroup_subsys_state *css,
 				       struct cftype *cft, u64 val)
 {
-	if (val)
-		set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
-	else
-		clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags);
+	assign_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags, val);
 	return 0;
 }
 
@@ -1097,7 +1091,9 @@ int cgroup1_reconfigure(struct fs_context *fc)
 	int ret = 0;
 	u32 added_mask, removed_mask;
 
-	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
+	ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
+	if (unlikely(ret))
+		return ret;
 
 	/* See what subsystems are wanted */
 	ret = check_cgroupfs_options(fc);
@@ -1262,7 +1258,9 @@ int cgroup1_get_tree(struct fs_context *fc)
 	if (!ns_capable(ctx->ns->user_ns, CAP_SYS_ADMIN))
 		return -EPERM;
 
-	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
+	ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
+	if (unlikely(ret))
+		return ret;
 
 	ret = cgroup1_root_to_use(fc);
 	if (!ret && !percpu_ref_tryget_live(&ctx->root->cgrp.self.refcnt))
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 227d09704ca591..804318ae160e64 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -1379,7 +1379,10 @@ static void cgroup_destroy_root(struct cgroup_root *root)
 
 	trace_cgroup_destroy_root(root);
 
-	cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
+	/* runs off a workqueue, no signal can interrupt the drain */
+	ret = cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp);
+	if (WARN_ON_ONCE(ret))
+		cgroup_lock();
 
 	BUG_ON(atomic_read(&root->nr_cgrps));
 	BUG_ON(!list_empty(&cgrp->self.children));
@@ -1685,9 +1688,10 @@ void cgroup_kn_unlock(struct kernfs_node *kn)
  * This helper is to be used by a cgroup kernfs method currently servicing
  * @kn.  It breaks the active protection, performs cgroup locking and
  * verifies that the associated cgroup is alive.  Returns the cgroup if
- * alive; otherwise, %NULL.  A successful return should be undone by a
- * matching cgroup_kn_unlock() invocation.  If @drain_offline is %true, the
- * cgroup is drained of offlining csses before return.
+ * alive; otherwise, an ERR_PTR value.  A successful return should be undone by
+ * a matching cgroup_kn_unlock() invocation.  If @drain_offline is %true, the
+ * cgroup is drained of offlining csses before return, and an interrupted drain
+ * fails with -ERESTARTSYS.
  *
  * Any cgroup kernfs method implementation which requires locking the
  * associated cgroup should use this helper.  It avoids nesting cgroup
@@ -1697,6 +1701,7 @@ void cgroup_kn_unlock(struct kernfs_node *kn)
 struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline)
 {
 	struct cgroup *cgrp;
+	int ret;
 
 	if (kernfs_type(kn) == KERNFS_DIR)
 		cgrp = kn->priv;
@@ -1710,19 +1715,25 @@ struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, bool drain_offline)
 	 * break the active_ref protection.
 	 */
 	if (!cgroup_tryget(cgrp))
-		return NULL;
+		return ERR_PTR(-ENODEV);
 	kernfs_break_active_protection(kn);
 
-	if (drain_offline)
-		cgroup_lock_and_drain_offline(cgrp);
-	else
+	if (drain_offline) {
+		ret = cgroup_lock_and_drain_offline(cgrp);
+		if (unlikely(ret)) {
+			kernfs_unbreak_active_protection(kn);
+			cgroup_put(cgrp);
+			return ERR_PTR(ret);
+		}
+	} else {
 		cgroup_lock();
+	}
 
 	if (!cgroup_is_dead(cgrp))
 		return cgrp;
 
 	cgroup_kn_unlock(kn);
-	return NULL;
+	return ERR_PTR(-ENODEV);
 }
 
 static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
@@ -3320,16 +3331,20 @@ static int cgroup_update_dfl_csses(struct cgroup *cgrp)
  * @cgrp: root of the target subtree
  *
  * Because css offlining is asynchronous, userland may try to re-enable a
- * controller while the previous css is still around.  This function grabs
- * cgroup_mutex and drains the previous css instances of @cgrp's subtree.
+ * controller while the previous css is still around. This function grabs
+ * cgroup_mutex and waits until no css in @cgrp's subtree is dying. A dying css
+ * offlines only after every task that still pins it has finished exiting, which
+ * can take arbitrarily long, so the wait is interruptible.
+ *
+ * Returns 0 with cgroup_mutex held once the subtree is drained, or -ERESTARTSYS
+ * without it if interrupted by a signal.
  */
-void cgroup_lock_and_drain_offline(struct cgroup *cgrp)
-	__acquires(&cgroup_mutex)
+int cgroup_lock_and_drain_offline(struct cgroup *cgrp)
 {
 	struct cgroup *dsct;
 	struct cgroup_subsys_state *d_css;
 	struct cgroup_subsys *ss;
-	int ssid;
+	int ssid, ret;
 
 restart:
 	cgroup_lock();
@@ -3343,17 +3358,20 @@ void cgroup_lock_and_drain_offline(struct cgroup *cgrp)
 				continue;
 
 			cgroup_get_live(dsct);
-			prepare_to_wait(&dsct->offline_waitq, &wait,
-					TASK_UNINTERRUPTIBLE);
-
+			ret = prepare_to_wait_event(&dsct->offline_waitq, &wait,
+						    TASK_INTERRUPTIBLE);
 			cgroup_unlock();
-			schedule();
+			if (!ret)
+				schedule();
 			finish_wait(&dsct->offline_waitq, &wait);
-
 			cgroup_put(dsct);
+			if (unlikely(ret))
+				return ret;
 			goto restart;
 		}
 	}
+
+	return 0;
 }
 
 /**
@@ -3650,8 +3668,8 @@ static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of,
 	}
 
 	cgrp = cgroup_kn_lock_live(of->kn, true);
-	if (!cgrp)
-		return -ENODEV;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	for_each_subsys(ss, ssid) {
 		if (enable & (1 << ssid)) {
@@ -3790,8 +3808,8 @@ static ssize_t cgroup_type_write(struct kernfs_open_file *of, char *buf,
 
 	/* drain dying csses before we re-apply (threaded) subtree control */
 	cgrp = cgroup_kn_lock_live(of->kn, true);
-	if (!cgrp)
-		return -ENOENT;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	/* threaded can only be enabled */
 	ret = cgroup_enable_threaded(cgrp);
@@ -3833,8 +3851,8 @@ static ssize_t cgroup_max_descendants_write(struct kernfs_open_file *of,
 		return -ERANGE;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENOENT;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	WRITE_ONCE(cgrp->max_descendants, descendants);
 
@@ -3876,8 +3894,8 @@ static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of,
 		return -ERANGE;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENOENT;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	WRITE_ONCE(cgrp->max_depth, depth);
 
@@ -4075,8 +4093,8 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
 	ssize_t ret = 0;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENODEV;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	ctx = of->priv;
 	if (!ctx) {
@@ -4192,8 +4210,8 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of,
 		return -ERANGE;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENOENT;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	psi = cgroup_psi(cgrp);
 	if (psi->enabled != enable) {
@@ -4268,8 +4286,8 @@ static ssize_t cgroup_freeze_write(struct kernfs_open_file *of,
 		return -ERANGE;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENOENT;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	cgroup_freeze(cgrp, freeze);
 
@@ -4330,8 +4348,8 @@ static ssize_t cgroup_kill_write(struct kernfs_open_file *of, char *buf,
 		return -ERANGE;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENOENT;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	/*
 	 * Killing is a process directed operation, i.e. the whole thread-group
@@ -5488,8 +5506,8 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf,
 	enum cgroup_attach_lock_mode lock_mode;
 
 	dst_cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!dst_cgrp)
-		return -ENODEV;
+	if (IS_ERR(dst_cgrp))
+		return PTR_ERR(dst_cgrp);
 
 	task = cgroup_procs_write_start(buf, threadgroup, &lock_mode);
 	ret = PTR_ERR_OR_ZERO(task);
@@ -6123,8 +6141,8 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
 		return -EINVAL;
 
 	parent = cgroup_kn_lock_live(parent_kn, false);
-	if (!parent)
-		return -ENODEV;
+	if (IS_ERR(parent))
+		return PTR_ERR(parent);
 
 	if (!cgroup_check_hierarchy_limits(parent)) {
 		ret = -EAGAIN;
@@ -6400,7 +6418,7 @@ int cgroup_rmdir(struct kernfs_node *kn)
 	int ret = 0;
 
 	cgrp = cgroup_kn_lock_live(kn, false);
-	if (!cgrp)
+	if (IS_ERR(cgrp))
 		return 0;
 
 	ret = cgroup_destroy_locked(cgrp);
@@ -7197,15 +7215,19 @@ static void do_cgroup_task_dead(struct task_struct *tsk)
  * the cgroup and task_struct can be pinned indefinitely. Bounce through lazy
  * irq_work to allow batching while ensuring timely completion.
  */
-static DEFINE_PER_CPU(struct llist_head, cgrp_dead_tasks);
-static DEFINE_PER_CPU(struct irq_work, cgrp_dead_tasks_iwork);
+struct cgroup_dead {
+	struct irq_work		iwork;
+	struct llist_head	tasks;
+};
+static DEFINE_PER_CPU(struct cgroup_dead, cgroup_dead);
 
 static void cgrp_dead_tasks_iwork_fn(struct irq_work *iwork)
 {
+	struct cgroup_dead *cgrp_dead = container_of(iwork, struct cgroup_dead, iwork);
 	struct llist_node *lnode;
 	struct task_struct *task, *next;
 
-	lnode = llist_del_all(this_cpu_ptr(&cgrp_dead_tasks));
+	lnode = llist_del_all(&cgrp_dead->tasks);
 	llist_for_each_entry_safe(task, next, lnode, cg_dead_lnode) {
 		do_cgroup_task_dead(task);
 		put_task_struct(task);
@@ -7217,17 +7239,20 @@ static void __init cgroup_rt_init(void)
 	int cpu;
 
 	for_each_possible_cpu(cpu) {
-		init_llist_head(per_cpu_ptr(&cgrp_dead_tasks, cpu));
-		per_cpu(cgrp_dead_tasks_iwork, cpu) =
-			IRQ_WORK_INIT_LAZY(cgrp_dead_tasks_iwork_fn);
+		struct cgroup_dead *cgrp_dead = per_cpu_ptr(&cgroup_dead, cpu);
+
+		init_llist_head(&cgrp_dead->tasks);
+		cgrp_dead->iwork = IRQ_WORK_INIT_LAZY(cgrp_dead_tasks_iwork_fn);
 	}
 }
 
 void cgroup_task_dead(struct task_struct *task)
 {
+	struct cgroup_dead *cgrp_dead = this_cpu_ptr(&cgroup_dead);
+
 	get_task_struct(task);
-	llist_add(&task->cg_dead_lnode, this_cpu_ptr(&cgrp_dead_tasks));
-	irq_work_queue(this_cpu_ptr(&cgrp_dead_tasks_iwork));
+	llist_add(&task->cg_dead_lnode, &cgrp_dead->tasks);
+	irq_work_queue(&cgrp_dead->iwork);
 }
 #else	/* CONFIG_PREEMPT_RT */
 static void __init cgroup_rt_init(void) {}
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 2538faac9aba8f..cf55b0719a2db0 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -771,8 +771,8 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial)
 	 * For v1, effective_cpus == cpus_allowed & user_xcpus() returns
 	 * cpus_allowed.
 	 *
-	 * For v2, is_cpu_exclusive() & is_sched_load_balance() are true only
-	 * for non-isolated partition root. At this point, the target
+	 * For v2, is_partition_valid(cur) & is_sched_load_balance() are true
+	 * only for non-isolated partition root. At this point, the target
 	 * effective_cpus isn't computed yet. user_xcpus() is the best
 	 * approximation.
 	 *
@@ -781,7 +781,8 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial)
 	 * becomes an issue.
 	 */
 	ret = -EBUSY;
-	if (is_cpu_exclusive(cur) && is_sched_load_balance(cur) &&
+	if ((is_partition_valid(cur) || (!cpuset_v2() && is_cpu_exclusive(cur))) &&
+	    is_sched_load_balance(cur) &&
 	    !cpuset_cpumask_can_shrink(cur->effective_cpus, user_xcpus(trial)))
 		goto out;
 
@@ -1173,25 +1174,6 @@ enum partition_cmd {
 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
 				    struct tmpmasks *tmp);
 
-/*
- * Update partition exclusive flag
- *
- * Return: 0 if successful, an error code otherwise
- */
-static int update_partition_exclusive_flag(struct cpuset *cs, int new_prs)
-{
-	bool exclusive = (new_prs > PRS_MEMBER);
-
-	if (exclusive && !is_cpu_exclusive(cs)) {
-		if (cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 1))
-			return PERR_NOTEXCL;
-	} else if (!exclusive && is_cpu_exclusive(cs)) {
-		/* Turning off CS_CPU_EXCLUSIVE will not return error */
-		cpuset_update_flag(CS_CPU_EXCLUSIVE, cs, 0);
-	}
-	return 0;
-}
-
 /*
  * Update partition load balance flag and/or rebuild sched domain
  *
@@ -1216,10 +1198,7 @@ static void update_partition_sd_lb(struct cpuset *cs, int old_prs)
 	}
 	if (new_lb != !!is_sched_load_balance(cs)) {
 		rebuild_domains = true;
-		if (new_lb)
-			set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
-		else
-			clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
+		assign_bit(CS_SCHED_LOAD_BALANCE, &cs->flags, new_lb);
 	}
 
 	if (rebuild_domains)
@@ -1250,11 +1229,9 @@ static void reset_partition_data(struct cpuset *cs)
 
 	lockdep_assert_held(&callback_lock);
 
-	if (cpumask_empty(cs->exclusive_cpus)) {
+	if (cpumask_empty(cs->exclusive_cpus))
 		cpumask_clear(cs->effective_xcpus);
-		if (is_cpu_exclusive(cs))
-			clear_bit(CS_CPU_EXCLUSIVE, &cs->flags);
-	}
+
 	if (!cpumask_and(cs->effective_cpus, parent->effective_cpus, cs->cpus_allowed))
 		cpumask_copy(cs->effective_cpus, parent->effective_cpus);
 }
@@ -1591,10 +1568,11 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs,
 	 * above it or remote partition root underneath it is not allowed.
 	 */
 	compute_excpus(cs, tmp->new_cpus);
-	WARN_ON_ONCE(cpumask_intersects(tmp->new_cpus, subpartitions_cpus));
 	if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) ||
 	    cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus))
 		return PERR_INVCPUS;
+	if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus))
+		return PERR_NOCPUS;
 	if (((new_prs == PRS_ISOLATED) &&
 	     !isolated_cpus_can_update(tmp->new_cpus, NULL)) ||
 	    prstate_housekeeping_conflict(new_prs, tmp->new_cpus))
@@ -2030,19 +2008,6 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
 	if (!adding && !deleting && (new_prs == old_prs))
 		return 0;
 
-	/*
-	 * Transitioning between invalid to valid or vice versa may require
-	 * changing CS_CPU_EXCLUSIVE. In the case of partcmd_update,
-	 * validate_change() has already been successfully called and
-	 * CPU lists in cs haven't been updated yet. So defer it to later.
-	 */
-	if ((old_prs != new_prs) && (cmd != partcmd_update))  {
-		int err = update_partition_exclusive_flag(cs, new_prs);
-
-		if (err)
-			return err;
-	}
-
 	/*
 	 * Change the parent's effective_cpus & effective_xcpus (top cpuset
 	 * only).
@@ -2065,9 +2030,6 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
 
 	spin_unlock_irq(&callback_lock);
 
-	if ((old_prs != new_prs) && (cmd == partcmd_update))
-		update_partition_exclusive_flag(cs, new_prs);
-
 	if (adding || deleting) {
 		cpuset_update_tasks_cpumask(parent, tmp->addmask);
 		update_sibling_cpumasks(parent, cs, tmp);
@@ -2308,10 +2270,8 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
 		 */
 		if (cpuset_v2() && !is_partition_valid(cp) &&
 		    (is_sched_load_balance(parent) != is_sched_load_balance(cp))) {
-			if (is_sched_load_balance(parent))
-				set_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
-			else
-				clear_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
+			assign_bit(CS_SCHED_LOAD_BALANCE, &cp->flags,
+				   is_sched_load_balance(parent));
 		}
 
 		/*
@@ -2890,10 +2850,7 @@ int cpuset_update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
 	if (!trialcs)
 		return -ENOMEM;
 
-	if (turning_on)
-		set_bit(bit, &trialcs->flags);
-	else
-		clear_bit(bit, &trialcs->flags);
+	assign_bit(bit, &trialcs->flags, turning_on);
 
 	err = validate_change(cs, trialcs);
 	if (err < 0)
@@ -2949,10 +2906,6 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 	if (alloc_tmpmasks(&tmpmask))
 		return -ENOMEM;
 
-	err = update_partition_exclusive_flag(cs, new_prs);
-	if (err)
-		goto out;
-
 	if (!old_prs) {
 		/*
 		 * cpus_allowed and exclusive_cpus cannot be both empty.
@@ -3016,13 +2969,10 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 	}
 out:
 	/*
-	 * Make partition invalid & disable CS_CPU_EXCLUSIVE if an error
-	 * happens.
+	 * Make partition invalid if an error happens.
 	 */
-	if (err) {
+	if (err)
 		new_prs = -new_prs;
-		update_partition_exclusive_flag(cs, new_prs);
-	}
 
 	spin_lock_irq(&callback_lock);
 	cs->partition_root_state = new_prs;
@@ -3044,8 +2994,6 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 	update_partition_sd_lb(cs, old_prs);
 
 	notify_partition_change(cs, old_prs);
-	if (force_sd_rebuild)
-		rebuild_sched_domains_locked();
 	free_tmpmasks(&tmpmask);
 	return 0;
 }
diff --git a/kernel/cgroup/debug.c b/kernel/cgroup/debug.c
index 883347b87842d6..aba243b5fd5f58 100644
--- a/kernel/cgroup/debug.c
+++ b/kernel/cgroup/debug.c
@@ -45,13 +45,13 @@ static int current_css_set_read(struct seq_file *seq, void *v)
 	struct cgroup_subsys_state *css;
 	int i, refcnt;
 
-	if (!cgroup_kn_lock_live(of->kn, false))
+	if (IS_ERR(cgroup_kn_lock_live(of->kn, false)))
 		return -ENODEV;
 
 	spin_lock_irq(&css_set_lock);
 	cset = task_css_set(current);
 	refcnt = refcount_read(&cset->refcount);
-	seq_printf(seq, "css_set %pK %d", cset, refcnt);
+	seq_printf(seq, "css_set %p %d", cset, refcnt);
 	if (refcnt > cset->nr_tasks)
 		seq_printf(seq, " +%d", refcnt - cset->nr_tasks);
 	seq_puts(seq, "\n");
@@ -125,10 +125,10 @@ static int cgroup_css_links_read(struct seq_file *seq, void *v)
 		 * Print out the proc_cset and threaded_cset relationship
 		 * and highlight difference between refcount and task_count.
 		 */
-		seq_printf(seq, "css_set %pK", cset);
+		seq_printf(seq, "css_set %p", cset);
 		if (rcu_dereference_protected(cset->dom_cset, 1) != cset) {
 			threaded_csets++;
-			seq_printf(seq, "=>%pK", cset->dom_cset);
+			seq_printf(seq, "=>%p", cset->dom_cset);
 		}
 		if (!list_empty(&cset->threaded_csets)) {
 			struct css_set *tcset;
@@ -137,7 +137,7 @@ static int cgroup_css_links_read(struct seq_file *seq, void *v)
 			list_for_each_entry(tcset, &cset->threaded_csets,
 					    threaded_csets_node) {
 				seq_puts(seq, idx ? "," : "<=");
-				seq_printf(seq, "%pK", tcset);
+				seq_printf(seq, "%p", tcset);
 				idx++;
 			}
 		} else {
@@ -206,8 +206,8 @@ static int cgroup_subsys_states_read(struct seq_file *seq, void *v)
 	int i;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENODEV;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	for_each_subsys(ss, i) {
 		css = rcu_dereference_check(cgrp->subsys[ss->id], true);
@@ -254,8 +254,8 @@ static int cgroup_masks_read(struct seq_file *seq, void *v)
 	struct cgroup *cgrp;
 
 	cgrp = cgroup_kn_lock_live(of->kn, false);
-	if (!cgrp)
-		return -ENODEV;
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
 
 	cgroup_masks_read_one(seq, "subtree_control", cgrp->subtree_control);
 	cgroup_masks_read_one(seq, "subtree_ss_mask", cgrp->subtree_ss_mask);
diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
index 4683f3d6802268..d49ed5d4a92377 100644
--- a/kernel/cgroup/dmem.c
+++ b/kernel/cgroup/dmem.c
@@ -28,7 +28,7 @@ DEFINE_STATIC_SRCU(dmemcg_srcu);
 struct dmem_cgroup_region {
 	/**
 	 * @ref: References keeping the region alive.
-	 * Keeps the region reference alive after a succesful RCU lookup.
+	 * Keeps the region reference alive after a successful RCU lookup.
 	 */
 	struct kref ref;
 
@@ -696,7 +696,7 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge);
  * dmem_cgroup_try_charge() - Try charging a new allocation to a region.
  * @region: dmem region to charge
  * @size: Size (in bytes) to charge.
- * @ret_pool: On succesfull allocation, the pool that is charged.
+ * @ret_pool: On successful allocation, the pool that is charged.
  * @ret_limit_pool: On a failed allocation, the limiting pool.
  *
  * This function charges the @region region for a size of @size bytes.
@@ -706,8 +706,8 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge);
  *
  * When this function fails with -EAGAIN and @ret_limit_pool is non-null, it
  * will be set to the pool for which the limit is hit. This can be used for
- * eviction as argument to dmem_cgroup_evict_valuable(). This reference must be freed
- * with @dmem_cgroup_pool_state_put().
+ * eviction as argument to dmem_cgroup_state_evict_valuable(). This reference
+ * must be freed with @dmem_cgroup_pool_state_put().
  *
  * Return: 0 on success, -EAGAIN on hitting a limit, or a negative errno on failure.
  */
diff --git a/kernel/cgroup/freezer.c b/kernel/cgroup/freezer.c
index 6c18854bff3485..35112d62318a78 100644
--- a/kernel/cgroup/freezer.c
+++ b/kernel/cgroup/freezer.c
@@ -20,10 +20,7 @@ static bool cgroup_update_frozen_flag(struct cgroup *cgrp, bool frozen)
 	if (test_bit(CGRP_FROZEN, &cgrp->flags) == frozen)
 		return false;
 
-	if (frozen)
-		set_bit(CGRP_FROZEN, &cgrp->flags);
-	else
-		clear_bit(CGRP_FROZEN, &cgrp->flags);
+	assign_bit(CGRP_FROZEN, &cgrp->flags, frozen);
 
 	cgroup_file_notify(&cgrp->events_file);
 	TRACE_CGROUP_PATH(notify_frozen, cgrp, frozen);
diff --git a/kernel/cgroup/namespace.c b/kernel/cgroup/namespace.c
index ea4ee13936be9c..76660332ec6441 100644
--- a/kernel/cgroup/namespace.c
+++ b/kernel/cgroup/namespace.c
@@ -53,8 +53,6 @@ struct cgroup_namespace *copy_cgroup_ns(u64 flags,
 	struct ucounts *ucounts;
 	struct css_set *cset;
 
-	BUG_ON(!old_ns);
-
 	if (!(flags & CLONE_NEWCGROUP)) {
 		get_cgroup_ns(old_ns);
 		return old_ns;
diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c
index ecbb839d2acbae..78cdc0558d0c54 100644
--- a/kernel/cgroup/pids.c
+++ b/kernel/cgroup/pids.c
@@ -253,6 +253,11 @@ static void pids_event(struct pids_cgroup *pids_forking,
 	}
 	if (!cgroup_subsys_on_dfl(pids_cgrp_subsys) ||
 	    cgrp_dfl_root.flags & CGRP_ROOT_PIDS_LOCAL_EVENTS) {
+		/*
+		 * pids.events reports the local counter on legacy hierarchies
+		 * and when pids_localevents is enabled.
+		 */
+		cgroup_file_notify(&p->events_file);
 		cgroup_file_notify(&p->events_local_file);
 		return;
 	}
diff --git a/kernel/configs/debug.config b/kernel/configs/debug.config
index 307c97ac5fa9c3..ac878669c19365 100644
--- a/kernel/configs/debug.config
+++ b/kernel/configs/debug.config
@@ -50,7 +50,7 @@ CONFIG_DEBUG_NET=y
 # CONFIG_DEBUG_PAGEALLOC is not set
 # CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF is not set
 # CONFIG_DEBUG_RODATA_TEST is not set
-# CONFIG_DEBUG_WX is not set
+# CONFIG_CHECK_WX is not set
 # CONFIG_KFENCE is not set
 # CONFIG_PAGE_POISONING is not set
 # CONFIG_SLUB_STATS is not set
diff --git a/kernel/cpu.c b/kernel/cpu.c
index b3c8553d7bd6ca..376d297a6292b6 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3103,6 +3103,11 @@ EXPORT_SYMBOL(__cpu_dying_mask);
 atomic_t __num_online_cpus __read_mostly;
 EXPORT_SYMBOL(__num_online_cpus);
 
+#ifdef CONFIG_PREFERRED_CPU
+struct cpumask __cpu_preferred_mask __read_mostly;
+EXPORT_SYMBOL_GPL(__cpu_preferred_mask);
+#endif
+
 void init_cpu_present(const struct cpumask *src)
 {
 	cpumask_copy(&__cpu_present_mask, src);
@@ -3160,6 +3165,7 @@ void __init boot_cpu_init(void)
 	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
 	set_cpu_online(cpu, true);
 	set_cpu_active(cpu, true);
+	set_cpu_preferred(cpu, true);
 	set_cpu_present(cpu, true);
 	set_cpu_possible(cpu, true);
 
diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c
index 7481fbb947d32e..a2a598ad6e6d50 100644
--- a/kernel/cpu_pm.c
+++ b/kernel/cpu_pm.c
@@ -10,6 +10,7 @@
 #include <linux/cpu_pm.h>
 #include <linux/module.h>
 #include <linux/notifier.h>
+#include <linux/rcupdate.h>
 #include <linux/spinlock.h>
 #include <linux/syscore_ops.h>
 
@@ -76,7 +77,8 @@ EXPORT_SYMBOL_GPL(cpu_pm_register_notifier);
  *
  * Remove a driver from the CPU PM notifier list.
  *
- * This function has the same return conditions as raw_notifier_chain_unregister.
+ * This function may sleep, and has the same return conditions as
+ * raw_notifier_chain_unregister.
  */
 int cpu_pm_unregister_notifier(struct notifier_block *nb)
 {
@@ -86,6 +88,11 @@ int cpu_pm_unregister_notifier(struct notifier_block *nb)
 	raw_spin_lock_irqsave(&cpu_pm_notifier.lock, flags);
 	ret = raw_notifier_chain_unregister(&cpu_pm_notifier.chain, nb);
 	raw_spin_unlock_irqrestore(&cpu_pm_notifier.lock, flags);
+
+	/* Wait for the rcu_read_lock() walkers in cpu_pm_notify(). */
+	if (!ret)
+		synchronize_rcu();
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(cpu_pm_unregister_notifier);
diff --git a/kernel/crash_core.c b/kernel/crash_core.c
index 2b36aa9fade064..d0bd2d0cf8990b 100644
--- a/kernel/crash_core.c
+++ b/kernel/crash_core.c
@@ -648,7 +648,7 @@ int crash_check_hotplug_support(void)
  * new list of CPUs and memory. To make changes to the elfcorehdr, it
  * should be large enough to permit a growing number of CPU and Memory
  * resources. One can estimate the elfcorehdr memory size based on
- * NR_CPUS_DEFAULT and CRASH_MAX_MEMORY_RANGES. The elfcorehdr is
+ * NR_CPUS and CRASH_MAX_MEMORY_RANGES. The elfcorehdr is
  * excluded from SHA verification by default if the architecture
  * supports crash hotplug.
  */
diff --git a/kernel/cred.c b/kernel/cred.c
index 3df4e15bd67ffe..d01df40a9a3861 100644
--- a/kernel/cred.c
+++ b/kernel/cred.c
@@ -414,7 +414,7 @@ int commit_creds(struct cred *new)
 		inc_rlimit_ucounts(new->ucounts, UCOUNT_RLIMIT_NPROC, 1);
 
 	rcu_assign_pointer(task->real_cred, new);
-	rcu_assign_pointer(task->cred, new);
+	task->cred = new;
 	if (new->user != old->user || new->user_ns != old->user_ns)
 		dec_rlimit_ucounts(old->ucounts, UCOUNT_RLIMIT_NPROC, 1);
 	if (new->user_ns != old->user_ns)
@@ -537,7 +537,8 @@ void __init cred_init(void)
 {
 	/* allocate a slab in which we can store credentials */
 	cred_jar = KMEM_CACHE(cred,
-			      SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT);
+			      SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT |
+			      SLAB_NO_MERGE);
 }
 
 /**
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c
index 66093460584e8c..90937dc7c627d7 100644
--- a/kernel/dma/contiguous.c
+++ b/kernel/dma/contiguous.c
@@ -451,8 +451,8 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
  *
  * This function releases memory allocated by dma_alloc_contiguous(). As the
  * cma_release returns false when provided pages do not belong to contiguous
- * area and true otherwise, this function then does a fallback __free_pages()
- * upon a false-return.
+ * area and true otherwise, this function then does a fallback
+ * free_pages_exact() upon a false-return.
  */
 void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
 {
@@ -476,7 +476,7 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
 	}
 
 	/* not in any cma, free from buddy */
-	__free_pages(page, get_order(size));
+	free_pages_exact(page_address(page), size);
 }
 
 /*
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index da665ca22d5c01..ea64453d7370a3 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -120,6 +120,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
 	int node = dev_to_node(dev);
 	struct page *page;
 	u64 phys_limit;
+	void *va;
 
 	WARN_ON_ONCE(!PAGE_ALIGNED(size));
 
@@ -133,9 +134,9 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
 		dma_free_contiguous(dev, page, size);
 	}
 
-	while ((page = alloc_pages_node(node, gfp, get_order(size)))
-	       && !dma_coherent_ok(dev, page_to_phys(page), size)) {
-		__free_pages(page, get_order(size));
+	while ((va = alloc_pages_exact_nid(node, size, gfp)) &&
+	       !dma_coherent_ok(dev, virt_to_phys(va), size)) {
+		free_pages_exact(va, size);
 
 		if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
 		    phys_limit < DMA_BIT_MASK(64) &&
@@ -146,6 +147,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
 		else
 			return NULL;
 	}
+	page = va ? virt_to_page(va) : NULL;
 
 	return page;
 }
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index bf2651a70b7c2c..82e701648d6046 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -1012,19 +1012,19 @@ size_t dma_max_mapping_size(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dma_max_mapping_size);
 
-size_t dma_opt_mapping_size(struct device *dev)
+size_t dma_max_opt_mapping_size(struct device *dev)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 	size_t size = SIZE_MAX;
 
 	if (use_dma_iommu(dev))
-		size = iommu_dma_opt_mapping_size();
-	else if (ops && ops->opt_mapping_size)
-		size = ops->opt_mapping_size();
+		size = iommu_dma_max_opt_mapping_size();
+	else if (ops && ops->max_opt_mapping_size)
+		size = ops->max_opt_mapping_size();
 
 	return min(dma_max_mapping_size(dev), size);
 }
-EXPORT_SYMBOL_GPL(dma_opt_mapping_size);
+EXPORT_SYMBOL_GPL(dma_max_opt_mapping_size);
 
 unsigned long dma_get_merge_boundary(struct device *dev)
 {
diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c
index 6b5f9208d31c07..5cdf3f60cf479a 100644
--- a/kernel/dma/ops_helpers.c
+++ b/kernel/dma/ops_helpers.c
@@ -67,10 +67,13 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size,
 	phys_addr_t phys;
 
 	page = dma_alloc_contiguous(dev, size, gfp);
-	if (!page)
-		page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size));
-	if (!page)
-		return NULL;
+	if (!page) {
+		void *va = alloc_pages_exact_nid(dev_to_node(dev), size, gfp);
+
+		if (!va)
+			return NULL;
+		page = virt_to_page(va);
+	}
 
 	phys = page_to_phys(page);
 	if (use_dma_iommu(dev))
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index e3d381fd3d2514..e234b04373feaf 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -123,7 +123,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
 /**
  * arch_irqentry_exit_need_resched - Architecture specific need resched function
  *
- * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed.
+ * Invoked from irqentry_exit_cond_resched() to check if resched is needed.
  * Defaults return true.
  *
  * The main purpose is to permit arch to avoid preemption of a task from an IRQ.
@@ -134,7 +134,7 @@ static inline bool arch_irqentry_exit_need_resched(void);
 static inline bool arch_irqentry_exit_need_resched(void) { return true; }
 #endif
 
-void raw_irqentry_exit_cond_resched(void)
+void irqentry_exit_cond_resched(void)
 {
 	if (!preempt_count()) {
 		/* Sanity check RCU and thread stack */
@@ -145,19 +145,6 @@ void raw_irqentry_exit_cond_resched(void)
 			preempt_schedule_irq();
 	}
 }
-#ifdef CONFIG_PREEMPT_DYNAMIC
-#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
-#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
-void dynamic_irqentry_exit_cond_resched(void)
-{
-	if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
-		return;
-	raw_irqentry_exit_cond_resched();
-}
-#endif
-#endif
 
 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
 {
diff --git a/kernel/events/core.c b/kernel/events/core.c
index fe33fe15689d07..a34ff4cb410dab 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -3764,6 +3764,9 @@ static void perf_ctx_sched_task_cb(struct perf_event_context *ctx,
 	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
 		cpc = this_cpc(pmu_ctx->pmu);
 
+		if (cpc->task_epc != pmu_ctx)
+			continue;
+
 		if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)
 			pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in);
 	}
@@ -3914,7 +3917,7 @@ static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc,
 	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
 	perf_pmu_disable(pmu);
 
-	pmu->sched_task(cpc->task_epc, task, sched_in);
+	pmu->sched_task(&cpc->epc, task, sched_in);
 
 	perf_pmu_enable(pmu);
 	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
@@ -3924,15 +3927,17 @@ static void perf_pmu_sched_task(struct task_struct *prev,
 				struct task_struct *next,
 				bool sched_in)
 {
-	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 	struct perf_cpu_pmu_context *cpc, *cpc2;
 
-	/* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */
-	if (prev == next || cpuctx->task_ctx)
+	if (prev == next)
 		return;
 
-	list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry)
+	list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
+		if (cpc->task_epc)
+			continue;
+
 		__perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in);
+	}
 }
 
 static void perf_event_switch(struct task_struct *task,
@@ -5454,6 +5459,8 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
 		}
 
 		if (refcount_inc_not_zero(&old->refcount)) {
+			if (global)
+				old->global = true;
 			free_perf_ctx_data(cd); /* unused */
 			return 0;
 		}
@@ -6350,6 +6357,9 @@ static DEFINE_MUTEX(perf_mediated_pmu_mutex);
 /* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */
 static inline bool is_include_guest_event(struct perf_event *event)
 {
+	if (!event->pmu)
+		return false;
+
 	if ((event->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) &&
 	    !event->attr.exclude_guest)
 		return true;
@@ -7818,22 +7828,82 @@ unsigned long perf_instruction_pointer(struct perf_event *event,
 		0 : perf_arch_instruction_pointer(regs);
 }
 
+u64 __weak perf_reg_value(struct pt_regs *regs, int idx)
+{
+	return 0;
+}
+
+int __weak perf_reg_validate(u64 mask, bool simd_enabled)
+{
+	return mask ? -ENOSYS : 0;
+}
+
+u64 __weak perf_reg_abi(struct task_struct *task)
+{
+	return PERF_SAMPLE_REGS_ABI_NONE;
+}
+
+void __weak perf_get_regs_user(struct perf_regs *regs_user,
+			       struct pt_regs *regs)
+{
+	regs_user->regs = task_pt_regs(current);
+	regs_user->abi = perf_reg_abi(current);
+}
+
+#define word_for_each_set_bit(bit, val)			\
+	for (unsigned long long __v = (val);		\
+	     __v && ((bit = __builtin_ctzll(__v)), 1);	\
+	     __v &= __v - 1)
+
 static void
 perf_output_sample_regs(struct perf_output_handle *handle,
 			struct pt_regs *regs, u64 mask)
 {
 	int bit;
-	DECLARE_BITMAP(_mask, 64);
-
-	bitmap_from_u64(_mask, mask);
-	for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) {
-		u64 val;
 
-		val = perf_reg_value(regs, bit);
+	word_for_each_set_bit(bit, mask) {
+		u64 val = perf_reg_value(regs, bit);
 		perf_output_put(handle, val);
 	}
 }
 
+static void
+perf_output_sample_simd_regs(struct perf_output_handle *handle,
+			     struct perf_event *event,
+			     struct pt_regs *regs,
+			     u64 mask, u32 pred_mask)
+{
+	u64 pred_qwords = event->attr.sample_simd_pred_reg_qwords;
+	u64 vec_qwords = event->attr.sample_simd_vec_reg_qwords;
+	u64 nr_vectors = hweight64(mask);
+	u64 nr_pred = hweight32(pred_mask);
+	int bit;
+
+	perf_output_put(handle, nr_vectors);
+	perf_output_put(handle, vec_qwords);
+	perf_output_put(handle, nr_pred);
+	perf_output_put(handle, pred_qwords);
+
+	if (nr_vectors) {
+		word_for_each_set_bit(bit, mask) {
+			for (int i = 0; i < vec_qwords; i++) {
+				u64 val = perf_simd_reg_value(regs, bit,
+							      i, false);
+				perf_output_put(handle, val);
+			}
+		}
+	}
+	if (nr_pred) {
+		word_for_each_set_bit(bit, pred_mask) {
+			for (int i = 0; i < pred_qwords; i++) {
+				u64 val = perf_simd_reg_value(regs, bit,
+							      i, true);
+				perf_output_put(handle, val);
+			}
+		}
+	}
+}
+
 static void perf_sample_regs_user(struct perf_regs *regs_user,
 				  struct pt_regs *regs)
 {
@@ -7867,6 +7937,17 @@ static void perf_sample_regs_intr(struct perf_regs *regs_intr,
 	}
 }
 
+int __weak perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask,
+				  u16 pred_qwords, u32 pred_mask)
+{
+	return -EINVAL;
+}
+
+u64 __weak perf_simd_reg_value(struct pt_regs *regs, int idx,
+			       u16 qwords_idx, bool pred)
+{
+	return 0;
+}
 
 /*
  * Get remaining task size from user stack pointer.
@@ -8397,10 +8478,17 @@ void perf_output_sample(struct perf_output_handle *handle,
 		perf_output_put(handle, abi);
 
 		if (abi) {
-			u64 mask = event->attr.sample_regs_user;
+			struct perf_event_attr *attr = &event->attr;
+			u64 mask = attr->sample_regs_user;
 			perf_output_sample_regs(handle,
 						data->regs_user.regs,
 						mask);
+			if (abi & PERF_SAMPLE_REGS_ABI_SIMD) {
+				perf_output_sample_simd_regs(handle, event,
+							     data->regs_user.regs,
+							     attr->sample_simd_vec_reg_user,
+							     attr->sample_simd_pred_reg_user);
+			}
 		}
 	}
 
@@ -8428,11 +8516,18 @@ void perf_output_sample(struct perf_output_handle *handle,
 		perf_output_put(handle, abi);
 
 		if (abi) {
-			u64 mask = event->attr.sample_regs_intr;
+			struct perf_event_attr *attr = &event->attr;
+			u64 mask = attr->sample_regs_intr;
 
 			perf_output_sample_regs(handle,
 						data->regs_intr.regs,
 						mask);
+			if (abi & PERF_SAMPLE_REGS_ABI_SIMD) {
+				perf_output_sample_simd_regs(handle, event,
+							     data->regs_intr.regs,
+							     attr->sample_simd_vec_reg_intr,
+							     attr->sample_simd_pred_reg_intr);
+			}
 		}
 	}
 
@@ -8635,6 +8730,29 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
 	return d * !!(flags & s);
 }
 
+u64 perf_update_xregs_size(struct perf_event *event, bool intr)
+{
+	u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords;
+	u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords;
+	u64 pred_mask;
+	u64 mask;
+	int size;
+
+	if (intr) {
+		mask = event->attr.sample_simd_vec_reg_intr;
+		pred_mask = event->attr.sample_simd_pred_reg_intr;
+	} else {
+		mask = event->attr.sample_simd_vec_reg_user;
+		pred_mask = event->attr.sample_simd_pred_reg_user;
+	}
+
+	size = sizeof(u64) * 4;
+	size += (hweight64(mask) * vec_qwords +
+		 hweight64(pred_mask) * pred_qwords) * sizeof(u64);
+
+	return size;
+}
+
 void perf_prepare_sample(struct perf_sample_data *data,
 			 struct perf_event *event,
 			 struct pt_regs *regs)
@@ -8697,7 +8815,12 @@ void perf_prepare_sample(struct perf_sample_data *data,
 
 		if (data->regs_user.regs) {
 			u64 mask = event->attr.sample_regs_user;
+
 			size += hweight64(mask) * sizeof(u64);
+			if (event_has_simd_regs(event)) {
+				size += perf_update_xregs_size(event, false);
+				data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
+			}
 		}
 
 		data->dyn_size += size;
@@ -8762,6 +8885,10 @@ void perf_prepare_sample(struct perf_sample_data *data,
 			u64 mask = event->attr.sample_regs_intr;
 
 			size += hweight64(mask) * sizeof(u64);
+			if (event_has_simd_regs(event)) {
+				size += perf_update_xregs_size(event, true);
+				data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD;
+			}
 		}
 
 		data->dyn_size += size;
@@ -9808,7 +9935,7 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
 
 	if (vma->vm_flags & VM_LOCKED)
 		flags |= MAP_LOCKED;
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		flags |= MAP_HUGETLB;
 
 	if (file) {
@@ -13002,6 +13129,7 @@ static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event,
 	exclusive_event_destroy(event);
 	module_put(pmu->module);
 
+	mediated_pmu_unaccount_event(event);
 	event->pmu = NULL; /* force fault instead of UAF */
 }
 
@@ -13105,12 +13233,6 @@ int perf_pmu_unregister(struct pmu *pmu)
 }
 EXPORT_SYMBOL_GPL(perf_pmu_unregister);
 
-static inline bool has_extended_regs(struct perf_event *event)
-{
-	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||
-	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);
-}
-
 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
 {
 	struct perf_event_context *ctx = NULL;
@@ -13144,8 +13266,14 @@ static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)
 	if (ret)
 		goto err_pmu;
 
+	if (!(pmu->capabilities & PERF_PMU_CAP_SIMD_REGS) &&
+	    event_has_simd_regs(event)) {
+		ret = -EOPNOTSUPP;
+		goto err_destroy;
+	}
+
 	if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) &&
-	    has_extended_regs(event)) {
+	    event_has_extended_regs(event)) {
 		ret = -EOPNOTSUPP;
 		goto err_destroy;
 	}
@@ -13639,7 +13767,8 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
 
 	attr->size = size;
 
-	if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3)
+	if (attr->__reserved_1 || attr->__reserved_2 ||
+	    attr->__reserved_3 || attr->__reserved_4)
 		return -EINVAL;
 
 	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
@@ -13685,9 +13814,18 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
 	}
 
 	if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
-		ret = perf_reg_validate(attr->sample_regs_user);
+		ret = perf_reg_validate(attr->sample_regs_user,
+					attr->sample_simd_regs_enabled);
 		if (ret)
 			return ret;
+		if (attr->sample_simd_regs_enabled) {
+			ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords,
+						     attr->sample_simd_vec_reg_user,
+						     attr->sample_simd_pred_reg_qwords,
+						     attr->sample_simd_pred_reg_user);
+			if (ret)
+				return ret;
+		}
 	}
 
 	if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
@@ -13708,8 +13846,20 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
 	if (!attr->sample_max_stack)
 		attr->sample_max_stack = sysctl_perf_event_max_stack;
 
-	if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
-		ret = perf_reg_validate(attr->sample_regs_intr);
+	if (attr->sample_type & PERF_SAMPLE_REGS_INTR) {
+		ret = perf_reg_validate(attr->sample_regs_intr,
+					attr->sample_simd_regs_enabled);
+		if (ret)
+			return ret;
+		if (attr->sample_simd_regs_enabled) {
+			ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords,
+						     attr->sample_simd_vec_reg_intr,
+						     attr->sample_simd_pred_reg_qwords,
+						     attr->sample_simd_pred_reg_intr);
+			if (ret)
+				return ret;
+		}
+	}
 
 #ifndef CONFIG_CGROUP_PERF
 	if (attr->sample_type & PERF_SAMPLE_CGROUP)
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 1b1ffe0533e58a..9d3d324f512720 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -635,11 +635,10 @@ static struct page *rb_alloc_aux_page(int node, int order)
 		/*
 		 * Communicate the allocation size to the driver:
 		 * if we managed to secure a high-order allocation,
-		 * set its first page's private to this order;
-		 * !PagePrivate(page) means it's just a normal page.
+		 * set its first page's private to this order, otherwise page's
+		 * private remains zero.
 		 */
 		split_page(page, order);
-		SetPagePrivate(page);
 		set_page_private(page, order);
 	}
 
@@ -650,7 +649,7 @@ static void rb_free_aux_page(struct perf_buffer *rb, int idx)
 {
 	struct page *page = virt_to_page(rb->aux_pages[idx]);
 
-	ClearPagePrivate(page);
+	set_page_private(page, 0);
 	__free_page(page);
 }
 
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 7709ea88247785..b89cc5cee00274 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -1726,8 +1726,8 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)
 	}
 
 	vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,
-				VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO|
-				VM_SEALED_SYSMAP,
+				VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|
+				VM_MIXEDMAP|VM_SEALED_SYSMAP,
 				&xol_mapping);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
diff --git a/kernel/exit.c b/kernel/exit.c
index 4e028f15759788..9ff1fa7b30ea01 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -17,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/capability.h>
 #include <linux/completion.h>
+#include <linux/wait_bit.h>
 #include <linux/personality.h>
 #include <linux/tty.h>
 #include <linux/iocontext.h>
@@ -167,12 +168,6 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts
 					lockdep_tasklist_lock_is_held());
 	spin_lock(&sighand->siglock);
 
-#ifdef CONFIG_POSIX_TIMERS
-	posix_cpu_timers_exit(tsk);
-	if (group_dead)
-		posix_cpu_timers_exit_group(tsk);
-#endif
-
 	if (group_dead) {
 		tty = sig->tty;
 		sig->tty = NULL;
@@ -302,12 +297,13 @@ void release_task(struct task_struct *p)
 	free_pids(post.pids);
 	release_thread(p);
 	/*
-	 * This task was already removed from the process/thread/pid lists
-	 * and lock_task_sighand(p) can't succeed. Nobody else can touch
-	 * ->pending or, if group dead, signal->shared_pending. We can call
-	 * flush_sigqueue() lockless.
+	 * This task was already removed from the process/thread/pid lists and
+	 * lock_task_sighand(p) can't succeed. If it's the group leader then
+	 * flush tsk->signal->shared_pending. tsk->pending has been flushed
+	 * already in exit_signals(). Nothing else can touch
+	 * signal->shared_pending anymore, so flush_sigqueue() can be invoked
+	 * lockless.
 	 */
-	flush_sigqueue(&p->pending);
 	if (thread_group_leader(p))
 		flush_sigqueue(&p->signal->shared_pending);
 
@@ -435,15 +431,14 @@ static void coredump_task_exit(struct task_struct *tsk,
 
 	self.task = tsk;
 	if (self.task->flags & PF_SIGNALED)
-		self.next = xchg(&core_state->dumper.next, &self);
+		self.next = xchg(&core_state->tasks, &self);
 	else
 		self.task = NULL;
 	/*
 	 * Implies mb(), the result of xchg() must be visible
-	 * to core_state->dumper.
+	 * to the dumper.
 	 */
-	if (atomic_dec_and_test(&core_state->nr_threads))
-		complete(&core_state->startup);
+	atomic_dec_and_wake_up(&core_state->threads_remaining);
 
 	for (;;) {
 		set_current_state(TASK_IDLE|TASK_FREEZABLE);
@@ -550,32 +545,6 @@ void mm_update_next_owner(struct mm_struct *mm)
 }
 #endif /* CONFIG_MEMCG */
 
-#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)
-/*
- * Subtract the memory footprint of the current task from
- * mm.
- */
-static void exit_mm_sched_cache(struct mm_struct *mm)
-{
-	unsigned long fp, sub;
-
-	if (!current->total_numa_faults)
-		return;
-	/*
-	 * No lock protection due to performance considerations.
-	 * Make sure mm->sc_stat.footprint does not become
-	 * negative.
-	 */
-	fp = READ_ONCE(mm->sc_stat.footprint);
-	sub = min(fp, current->total_numa_faults);
-	WRITE_ONCE(mm->sc_stat.footprint, fp - sub);
-}
-#else
-static inline void exit_mm_sched_cache(struct mm_struct *mm)
-{
-}
-#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */
-
 /*
  * Turn us into a lazy TLB process if we
  * aren't already..
@@ -588,7 +557,7 @@ static void exit_mm(void)
 	if (!mm)
 		return;
 
-	exit_mm_sched_cache(mm);
+	sched_cache_exit_mm(current);
 
 	mmap_read_lock(mm);
 	mmgrab_lazy_tlb(mm);
@@ -917,7 +886,7 @@ static void synchronize_group_exit(struct task_struct *tsk, long code)
 	 * Serialize with any possible pending coredump.
 	 * We must hold siglock around checking core_state
 	 * and setting PF_POSTCOREDUMP.  The core-inducing thread
-	 * will increment ->nr_threads for each thread in the
+	 * will increment ->threads_remaining for each thread in the
 	 * group without PF_POSTCOREDUMP set.
 	 */
 	tsk->flags |= PF_POSTCOREDUMP;
@@ -965,13 +934,12 @@ void __noreturn do_exit(long code)
 			panic("Attempted to kill init! exitcode=0x%08x\n",
 				tsk->signal->group_exit_code ?: (int)code);
 
-#ifdef CONFIG_POSIX_TIMERS
-		hrtimer_cancel(&tsk->signal->real_timer);
-		exit_itimers(tsk);
-#endif
 		if (tsk->mm)
 			setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);
 	}
+
+	posixtimer_exit(group_dead);
+
 	acct_collect(code, group_dead);
 	if (group_dead)
 		tty_audit_exit();
@@ -1003,10 +971,11 @@ void __noreturn do_exit(long code)
 
 	exit_sem(tsk);
 	exit_shm(tsk);
-	exit_files(tsk);
-	exit_fs(tsk);
+	/* Hang the tty up before the last close of it can clear the session. */
 	if (group_dead)
 		disassociate_ctty(1);
+	exit_files(tsk);
+	exit_fs(tsk);
 	exit_nsproxy_namespaces(tsk);
 	exit_task_work(tsk);
 	exit_thread(tsk);
diff --git a/kernel/fork.c b/kernel/fork.c
index a5934a3176346b..d458d85d7a6567 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -858,7 +858,8 @@ void __init fork_init(void)
 #ifndef ARCH_MIN_TASKALIGN
 #define ARCH_MIN_TASKALIGN	0
 #endif
-	int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
+	int align = max3(L1_CACHE_BYTES, ARCH_MIN_TASKALIGN,
+			 __alignof__(struct task_struct));
 	unsigned long useroffset, usersize;
 
 	/* create a slab on which task_structs can be allocated */
@@ -1083,9 +1084,7 @@ static void mmap_init_lock(struct mm_struct *mm)
 {
 	init_rwsem(&mm->mmap_lock);
 	mm_lock_seqcount_init(mm);
-#ifdef CONFIG_PER_VMA_LOCK
 	rcuwait_init(&mm->vma_writer_wait);
-#endif
 }
 
 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
@@ -1599,6 +1598,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk)
 
 	tsk->mm = mm;
 	tsk->active_mm = mm;
+	sched_cache_fork(tsk);
 	return 0;
 }
 
@@ -1676,6 +1676,7 @@ static int copy_files(u64 clone_flags, struct task_struct *tsk,
 
 	if (clone_flags & CLONE_FILES) {
 		atomic_inc(&oldf->count);
+		tsk->files = oldf;
 		return 0;
 	}
 
@@ -1996,9 +1997,9 @@ static bool need_futex_hash_allocate_default(u64 clone_flags)
 {
 	/*
 	 * Allocate a default futex hash for any sibling that will
-	 * share the parent's mm, except vfork.
+	 * share the parent's mm.
 	 */
-	return (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM;
+	return clone_flags & CLONE_VM;
 }
 
 /*
@@ -2145,12 +2146,9 @@ __latent_entropy struct task_struct *copy_process(
 	if (args->kthread)
 		p->flags |= PF_KTHREAD;
 	if (args->user_worker) {
-		/*
-		 * Mark us a user worker, and block any signal that isn't
-		 * fatal or STOP
-		 */
+		/* A user worker takes only the signals nobody can block. */
 		p->flags |= PF_USER_WORKER;
-		siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP));
+		siginitsetinv(&p->blocked, SIG_KERNEL_ONLY_MASK);
 	}
 	if (args->io_thread)
 		p->flags |= PF_IO_WORKER;
@@ -2199,6 +2197,8 @@ __latent_entropy struct task_struct *copy_process(
 	INIT_LIST_HEAD(&p->sibling);
 	rcu_copy_process(p);
 	p->vfork_done = NULL;
+	/* Set by copy_files(), exit_files() on the error path skips NULL. */
+	p->files = NULL;
 	spin_lock_init(&p->alloc_lock);
 
 	init_sigpending(&p->pending);
@@ -2300,7 +2300,7 @@ __latent_entropy struct task_struct *copy_process(
 		goto bad_fork_cleanup_semundo;
 	retval = copy_fs(clone_flags, p, args->umh);
 	if (retval)
-		goto bad_fork_cleanup_files;
+		goto bad_fork_cleanup_semundo;
 	retval = copy_sighand(clone_flags, p);
 	if (retval)
 		goto bad_fork_cleanup_fs;
@@ -2602,6 +2602,7 @@ __latent_entropy struct task_struct *copy_process(
 bad_fork_cleanup_namespaces:
 	exit_nsproxy_namespaces(p);
 bad_fork_cleanup_mm:
+	sched_cache_fork_cleanup(p);
 	if (p->mm) {
 		mm_clear_owner(p->mm, p);
 		mmput(p->mm);
@@ -2613,8 +2614,6 @@ __latent_entropy struct task_struct *copy_process(
 	__cleanup_sighand(p->sighand);
 bad_fork_cleanup_fs:
 	exit_fs(p); /* blocking */
-bad_fork_cleanup_files:
-	exit_files(p); /* blocking */
 bad_fork_cleanup_semundo:
 	exit_sem(p);
 bad_fork_cleanup_security:
@@ -2625,6 +2624,8 @@ __latent_entropy struct task_struct *copy_process(
 	perf_event_free_task(p);
 bad_fork_sched_cancel_fork:
 	sched_cancel_fork(p);
+	/* ->release() of a file may need scx_fork_rwsem for write. */
+	exit_files(p); /* blocking */
 bad_fork_cleanup_policy:
 	lockdep_free_task(p);
 #ifdef CONFIG_NUMA
@@ -2703,6 +2704,10 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
 		.user_worker	= 1,
 	};
 
+	/* A creator past its fatal signal or its coredump point gets no thread. */
+	if (current->flags & (PF_SIGNALED | PF_POSTCOREDUMP))
+		return ERR_PTR(-EINTR);
+
 	return copy_process(NULL, 0, node, &args);
 }
 
@@ -3210,24 +3215,6 @@ static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
 	return 0;
 }
 
-/*
- * Unshare file descriptor table if it is being shared
- */
-static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
-{
-	struct files_struct *fd = current->files;
-
-	if ((unshare_flags & CLONE_FILES) &&
-	    (fd && atomic_read(&fd->count) > 1)) {
-		fd = dup_fd(fd, NULL);
-		if (IS_ERR(fd))
-			return PTR_ERR(fd);
-		*new_fdp = fd;
-	}
-
-	return 0;
-}
-
 /*
  * unshare allows a process to 'unshare' part of the process
  * context which was originally shared using clone.  copy_*
@@ -3323,10 +3310,8 @@ int ksys_unshare(unsigned long unshare_flags)
 		if (new_fs)
 			new_fs = switch_fs_struct(new_fs);
 
-		if (new_fd) {
-			guard(task_lock)(current);
-			swap(current->files, new_fd);
-		}
+		if (new_fd)
+			switch_files_struct(current, no_free_ptr(new_fd));
 
 		if (new_cred) {
 			/* Install the new user namespace */
@@ -3359,30 +3344,6 @@ SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
 	return ksys_unshare(unshare_flags);
 }
 
-/*
- *	Helper to unshare the files of the current task.
- *	We don't want to expose copy_files internals to
- *	the exec layer of the kernel.
- */
-
-int unshare_files(void)
-{
-	struct task_struct *task = current;
-	struct files_struct *old, *copy = NULL;
-	int error;
-
-	error = unshare_fd(CLONE_FILES, &copy);
-	if (error || !copy)
-		return error;
-
-	old = task->files;
-	task_lock(task);
-	task->files = copy;
-	task_unlock(task);
-	put_files_struct(old);
-	return 0;
-}
-
 static int sysctl_max_threads(const struct ctl_table *table, int write,
 		       void *buffer, size_t *lenp, loff_t *ppos)
 {
diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c
index b3f4a4bccb1263..842d852302ddfb 100644
--- a/kernel/futex/requeue.c
+++ b/kernel/futex/requeue.c
@@ -744,7 +744,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
 
 	/* Handle spurious wakeups gracefully */
 	ret = -EWOULDBLOCK;
-	if (timeout && !timeout->task)
+	if (timeout && !hrtimer_sleeper_task_get(timeout))
 		ret = -ETIMEDOUT;
 	else if (signal_pending(current))
 		ret = -ERESTARTNOINTR;
diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c
index d4483d15d30a5d..cf18309e577030 100644
--- a/kernel/futex/waitwake.c
+++ b/kernel/futex/waitwake.c
@@ -383,7 +383,7 @@ void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout)
 		 * flagged for rescheduling. Only call schedule if there
 		 * is no timeout, or if it has yet to expire.
 		 */
-		if (!timeout || timeout->task)
+		if (!timeout || hrtimer_sleeper_task_get(timeout))
 			schedule();
 	}
 	__set_current_state(TASK_RUNNING);
@@ -539,7 +539,7 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken)
 static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count,
 				 struct hrtimer_sleeper *to)
 {
-	if (to && !to->task)
+	if (to && !hrtimer_sleeper_task_get(to))
 		return;
 
 	for (; count; count--, vs++) {
@@ -590,7 +590,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
 		if (ret >= 0)
 			return ret;
 
-		if (to && !to->task)
+		if (to && !hrtimer_sleeper_task_get(to))
 			return -ETIMEDOUT;
 		else if (signal_pending(current))
 			return -ERESTARTSYS;
@@ -725,7 +725,7 @@ int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
 	if (!futex_unqueue(&q))
 		return 0;
 
-	if (to && !to->task)
+	if (to && !hrtimer_sleeper_task_get(to))
 		return -ETIMEDOUT;
 
 	/*
diff --git a/kernel/gcov/fs.c b/kernel/gcov/fs.c
index 1d19b1be207a7d..764918570de13d 100644
--- a/kernel/gcov/fs.c
+++ b/kernel/gcov/fs.c
@@ -529,7 +529,7 @@ static void init_node(struct gcov_node *node, struct gcov_info *info,
 	}
 	node->parent = parent;
 	if (name)
-		strcpy(node->name, name);
+		strscpy(node->name, name, strlen(name) + 1);
 }
 
 /*
diff --git a/kernel/hazptr.c b/kernel/hazptr.c
new file mode 100644
index 00000000000000..d3d1050d92cf0f
--- /dev/null
+++ b/kernel/hazptr.c
@@ -0,0 +1,284 @@
+// SPDX-License-Identifier: LGPL-2.1-or-later
+//
+// SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+
+/*
+ * hazptr: Hazard Pointers
+ */
+
+#include <linux/hazptr.h>
+#include <linux/percpu.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/export.h>
+
+/*
+ * The current hazard pointer wildcard. Flips between 1UL and 2UL to guarantee
+ * hazptr_synchronize forward progress even with a steady stream of readers.
+ * This wildcard value is used by acquire to temporarily tag the per-CPU slots.
+ * This also affects the overflow list selection: the current list used by
+ * readers is array[(unsigned long) hazptr_wildcard - 1].
+ */
+static DEFINE_MUTEX(hazptr_wildcard_lock);	/* Protect the wildcard flip. */
+void *hazptr_wildcard = (void *) 1UL;
+EXPORT_SYMBOL_GPL(hazptr_wildcard);
+
+struct hazptr_overflow_list {
+	raw_spinlock_t lock;		/* Lock protecting overflow list and list generation. */
+	struct hlist_head head;		/* Overflow list head. */
+	uint64_t gen;			/* Overflow list generation. */
+};
+
+/*
+ * Flip between two lists to guarantee list scan forward progress even
+ * with frequent generation counter increments. The list additions are
+ * always done on a different list than the one used for scan. The scan
+ * successively iterates on both lists. Therefore, only list removals
+ * can cause the iteration to retry, and the number of removals is
+ * limited to the number of list elements.
+ */
+struct hazptr_overflow_list_flip {
+	struct hazptr_overflow_list array[2];
+};
+
+static DEFINE_PER_CPU(struct hazptr_overflow_list_flip, percpu_overflow_list_flip);
+
+DEFINE_PER_CPU(struct hazptr_percpu_slots, hazptr_percpu_slots);
+EXPORT_PER_CPU_SYMBOL_GPL(hazptr_percpu_slots);
+
+static
+void *flip_wildcard(void *wildcard)
+{
+	return ((unsigned long) wildcard == 1UL) ? (void *) 2UL : (void *) 1UL;
+}
+
+static
+bool is_wildcard(void *addr)
+{
+	if ((unsigned long) addr == 1UL || (unsigned long) addr == 2UL)
+		return true;
+	return false;
+}
+
+static
+struct hazptr_slot *hazptr_get_free_percpu_slot(struct hazptr_ctx *ctx)
+{
+	struct hazptr_percpu_slots *percpu_slots = this_cpu_ptr(&hazptr_percpu_slots);
+	unsigned int idx;
+
+	for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) {
+		struct hazptr_slot_item *item = &percpu_slots->items[idx];
+		struct hazptr_slot *slot = &item->slot;
+
+		if (!slot->addr) {
+			item->ctx.ctx = ctx;
+			return slot;
+		}
+	}
+	/* All slots are in use. */
+	return NULL;
+}
+
+/*
+ * Hazard pointer acquire slow path.
+ * Called with preemption disabled.
+ */
+void *__hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p)
+{
+	struct hazptr_slot *slot = hazptr_get_free_percpu_slot(ctx);
+	void *addr;
+
+	/*
+	 * If all the per-CPU slots are already in use, fallback
+	 * to the backup slot.
+	 */
+	if (unlikely(!slot))
+		slot = hazptr_chain_backup_slot(ctx);
+	WRITE_ONCE(slot->addr, READ_ONCE(hazptr_wildcard));	/* Store B */
+
+	/* Memory ordering: Store B before Load A. */
+	smp_mb();
+
+	/*
+	 * Load @addr_p after storing wildcard to the hazard pointer slot.
+	 */
+	addr = READ_ONCE(*addr_p);	/* Load A */
+
+	/*
+	 * We don't care about ordering of Store C. It will simply
+	 * replace the wildcard by a more specific address. If addr is
+	 * NULL, we simply store NULL into the slot.
+	 */
+	WRITE_ONCE(slot->addr, addr);	/* Store C */
+	ctx->slot = slot;
+	if (!addr && hazptr_slot_is_backup(ctx, slot))
+		hazptr_unchain_backup_slot(ctx);
+	return addr;
+}
+EXPORT_SYMBOL_GPL(__hazptr_acquire);
+
+/*
+ * Perform piecewise iteration on overflow list waiting until "addr" is
+ * not present. Raw spinlock is released and taken between each list
+ * item and busy loop iteration. The overflow list generation is checked
+ * each time the lock is taken to validate that the list has not changed
+ * before resuming iteration or busy wait. If the generation has
+ * changed, retry the entire list traversal.
+ */
+static
+void hazptr_synchronize_overflow_list(struct hazptr_overflow_list *overflow_list, void *addr)
+{
+	struct hazptr_backup_slot *backup_slot;
+	uint64_t snapshot_gen;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&overflow_list->lock, flags);
+retry:
+	snapshot_gen = overflow_list->gen;
+	hlist_for_each_entry(backup_slot, &overflow_list->head, overflow_node) {
+		/* Busy-wait if node is found. */
+		for (;;) {
+			void *load_addr = smp_load_acquire(&backup_slot->slot.addr);	/* Load B */
+
+			/* We don't expect wildcards in overflow list. */
+			WARN_ON_ONCE(is_wildcard(load_addr));
+			if (load_addr != addr)
+				break;
+			raw_spin_unlock_irqrestore(&overflow_list->lock, flags);
+			cpu_relax();
+			raw_spin_lock_irqsave(&overflow_list->lock, flags);
+			if (overflow_list->gen != snapshot_gen)
+				goto retry;
+		}
+		raw_spin_unlock_irqrestore(&overflow_list->lock, flags);
+		/*
+		 * Release raw spinlock, validate generation after
+		 * re-acquiring the lock.
+		 */
+		raw_spin_lock_irqsave(&overflow_list->lock, flags);
+		if (overflow_list->gen != snapshot_gen)
+			goto retry;
+	}
+	raw_spin_unlock_irqrestore(&overflow_list->lock, flags);
+}
+
+static
+void hazptr_synchronize_cpu_slots(int cpu, void *addr, void *scan_wildcard)
+{
+	struct hazptr_percpu_slots *percpu_slots = per_cpu_ptr(&hazptr_percpu_slots, cpu);
+	unsigned int idx;
+
+	for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) {
+		struct hazptr_slot_item *item = &percpu_slots->items[idx];
+
+		/* Busy-wait if node is found. */
+		smp_cond_load_acquire(&item->slot.addr, VAL != addr && VAL != scan_wildcard); /* Load B */
+	}
+}
+
+static
+void hazptr_scan_period(void *addr, void *scan_wildcard)
+{
+	unsigned int scan_idx = (unsigned long) scan_wildcard - 1;
+	int cpu;
+
+	/* Scan all CPUs slots. */
+	for_each_possible_cpu(cpu) {
+		struct hazptr_overflow_list_flip *overflow_list_flip = per_cpu_ptr(&percpu_overflow_list_flip, cpu);
+
+		/*
+		 * Scan CPU slots.
+		 * Forward progress against recurring wildcards is guaranteed
+		 * by scanning for one wildcard while new elements use the
+		 * other wildcard value (1UL vs 2UL).
+		 * Forward progress against recurring single hazard pointer
+		 * values is guaranteed by the fact that a hazard pointer
+		 * is not reclaimed nor reused until the scan for that hazard
+		 * pointer completes, which prevents a steady flow of readers
+		 * to acquire that same hazard pointer value.
+		 */
+		hazptr_synchronize_cpu_slots(cpu, addr, scan_wildcard);
+
+		/*
+		 * Scan backup slots in percpu overflow lists.
+		 * Forward progress is guaranteed by scanning one list
+		 * while new elements are added into the other list.
+		 */
+		hazptr_synchronize_overflow_list(&overflow_list_flip->array[scan_idx], addr);
+	}
+}
+
+/*
+ * hazptr_synchronize: Wait until @addr is released from all slots.
+ *
+ * Wait to observe that each slot contains a value that differs from
+ * @addr before returning.
+ * Should be called from preemptible context.
+ */
+void hazptr_synchronize(void *addr)
+{
+	void *scan_wildcard;
+
+	/*
+	 * Busy-wait should only be done from preemptible context.
+	 */
+	lockdep_assert_preemption_enabled();
+
+	/*
+	 * Store A precedes hazptr_scan(): it unpublishes addr (sets it to
+	 * NULL or to a different value), and thus hides it from hazard
+	 * pointer readers.
+	 */
+	if (!addr)
+		return;
+	/* Memory ordering: Store A before Load B. */
+	smp_mb();
+
+	guard(mutex)(&hazptr_wildcard_lock);
+	scan_wildcard = flip_wildcard(hazptr_wildcard);
+	hazptr_scan_period(addr, scan_wildcard);
+	WRITE_ONCE(hazptr_wildcard, scan_wildcard);	/* Flip the current wildcard. */
+	hazptr_scan_period(addr, flip_wildcard(scan_wildcard));
+}
+EXPORT_SYMBOL_GPL(hazptr_synchronize);
+
+struct hazptr_slot *hazptr_chain_backup_slot(struct hazptr_ctx *ctx)
+{
+	struct hazptr_overflow_list_flip *overflow_list_flip = this_cpu_ptr(&percpu_overflow_list_flip);
+	unsigned int list_idx = (unsigned long) READ_ONCE(hazptr_wildcard) - 1;
+	struct hazptr_overflow_list *overflow_list = &overflow_list_flip->array[list_idx];
+	struct hazptr_slot *slot = &ctx->backup_slot.slot;
+
+	slot->addr = NULL;
+	guard(raw_spinlock_irqsave)(&overflow_list->lock);
+	overflow_list->gen++;
+	hlist_add_head(&ctx->backup_slot.overflow_node, &overflow_list->head);
+	ctx->backup_slot.overflow_list = overflow_list;
+	return slot;
+}
+EXPORT_SYMBOL_GPL(hazptr_chain_backup_slot);
+
+void hazptr_unchain_backup_slot(struct hazptr_ctx *ctx)
+{
+	struct hazptr_overflow_list *overflow_list = ctx->backup_slot.overflow_list;
+
+	guard(raw_spinlock_irqsave)(&overflow_list->lock);
+	overflow_list->gen++;
+	hlist_del(&ctx->backup_slot.overflow_node);
+}
+EXPORT_SYMBOL_GPL(hazptr_unchain_backup_slot);
+
+void __init hazptr_init(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct hazptr_overflow_list_flip *overflow_list_flip = per_cpu_ptr(&percpu_overflow_list_flip, cpu);
+
+		for (int i = 0; i < 2; i++) {
+			raw_spin_lock_init(&overflow_list_flip->array[i].lock);
+			INIT_HLIST_HEAD(&overflow_list_flip->array[i].head);
+		}
+	}
+}
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 6fcc94ce4ca9d2..49d47ae475ab1c 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -57,8 +57,20 @@ unsigned long __read_mostly sysctl_hung_task_timeout_secs = CONFIG_DEFAULT_HUNG_
  */
 static unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
 
+/*
+ * Limit the number of printed hung tasks to prevent printing
+ * the same or similar backtraces repeatedly.
+ */
 static int __read_mostly sysctl_hung_task_warnings = 10;
 
+/*
+ * The number of hung tasks which still can be reported.
+ * The budget gets restored to the original limit when
+ * the previous stall is resolved.
+ */
+static int hung_task_warnings_budget = 10;
+static atomic_t reset_hung_task_warnings = ATOMIC_INIT(0);
+
 static int __read_mostly did_panic;
 static bool hung_task_call_panic;
 
@@ -245,11 +257,11 @@ static void hung_task_info(struct task_struct *t, unsigned long timeout,
 	/*
 	 * The given task did not get scheduled for more than
 	 * CONFIG_DEFAULT_HUNG_TASK_TIMEOUT. Therefore, complain
-	 * accordingly
+	 * accordingly with full details if the budget is not exhausted.
 	 */
-	if (sysctl_hung_task_warnings || hung_task_call_panic) {
-		if (sysctl_hung_task_warnings > 0)
-			sysctl_hung_task_warnings--;
+	if (hung_task_warnings_budget || hung_task_call_panic) {
+		if (hung_task_warnings_budget > 0)
+			hung_task_warnings_budget--;
 		pr_err("INFO: task %s:%d blocked%s for more than %ld seconds.\n",
 		       t->comm, t->pid, t->in_iowait ? " in I/O wait" : "",
 		       (jiffies - t->last_switch_time) / HZ);
@@ -264,8 +276,8 @@ static void hung_task_info(struct task_struct *t, unsigned long timeout,
 		sched_show_task(t);
 		debug_show_blocker(t, timeout);
 
-		if (!sysctl_hung_task_warnings)
-			pr_info("Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings\n");
+		if (!hung_task_warnings_budget)
+			pr_info("hung_task: further per-task details suppressed until warning budget is reset or panic is triggered (see sysctl kernel.hung_task_warnings)\n");
 	}
 
 	touch_nmi_watchdog();
@@ -304,7 +316,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	unsigned long last_break = jiffies;
 	struct task_struct *g, *t;
 	unsigned long this_round_count;
-	int need_warning = sysctl_hung_task_warnings;
+	int need_warning;
 	unsigned long si_mask = hung_task_si_mask;
 
 	/*
@@ -314,6 +326,11 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
 	if (test_taint(TAINT_DIE) || did_panic)
 		return;
 
+	if (atomic_xchg_acquire(&reset_hung_task_warnings, 0))
+		hung_task_warnings_budget =
+			READ_ONCE(sysctl_hung_task_warnings);
+	need_warning = hung_task_warnings_budget;
+
 	this_round_count = 0;
 	rcu_read_lock();
 	for_each_process_thread(g, t) {
@@ -340,8 +357,15 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout)
  unlock:
 	rcu_read_unlock();
 
-	if (!this_round_count)
+	if (!this_round_count) {
+		hung_task_warnings_budget =
+			READ_ONCE(sysctl_hung_task_warnings);
 		return;
+	}
+
+	if (!hung_task_warnings_budget && !hung_task_call_panic)
+		pr_info("hung_task: %lu hung tasks detected, warning budget exhausted\n",
+			this_round_count);
 
 	if (need_warning || hung_task_call_panic) {
 		si_mask |= SYS_INFO_LOCKS;
@@ -425,6 +449,19 @@ static int proc_dohung_task_timeout_secs(const struct ctl_table *table, int writ
 	return ret;
 }
 
+static int proc_dohung_task_warnings(const struct ctl_table *table, int write,
+				     void *buffer,
+				     size_t *lenp, loff_t *ppos)
+{
+	int ret;
+
+	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+	if (!ret && write)
+		atomic_set_release(&reset_hung_task_warnings, 1);
+
+	return ret;
+}
+
 /*
  * This is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs
  * and hung_task_check_interval_secs
@@ -480,7 +517,7 @@ static const struct ctl_table hung_task_sysctls[] = {
 		.data		= &sysctl_hung_task_warnings,
 		.maxlen		= sizeof(int),
 		.mode		= 0644,
-		.proc_handler	= proc_dointvec_minmax,
+		.proc_handler	= proc_dohung_task_warnings,
 		.extra1		= SYSCTL_NEG_ONE,
 	},
 	{
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 3a818f07a1013a..7cbcc3fb9e6d4c 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -28,7 +28,15 @@ static struct lock_class_key irq_desc_lock_class;
 static int __init irq_affinity_setup(char *str)
 {
 	alloc_bootmem_cpumask_var(&irq_default_affinity);
-	cpulist_parse(str, irq_default_affinity);
+	if (cpulist_parse(str, irq_default_affinity) < 0) {
+		/*
+		 * Clear the mask so that init_irq_default_affinity()
+		 * defaults it later.
+		 */
+		cpumask_clear(irq_default_affinity);
+		pr_warn("irqaffinity: incorrect CPU range, using default\n");
+		return 1;
+	}
 	/*
 	 * Set at least the boot cpu. We don't want to end up with
 	 * bugreports caused by random commandline masks
diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c
index 57c819da30c219..4fdcb6df53065e 100644
--- a/kernel/irq/irqdomain.c
+++ b/kernel/irq/irqdomain.c
@@ -344,6 +344,7 @@ static struct irq_domain *__irq_domain_instantiate(const struct irq_domain_info
 		err = irq_domain_alloc_generic_chips(domain, info->dgc_info);
 		if (err)
 			goto err_domain_free;
+		domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC;
 	}
 
 	if (info->init) {
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index aec2f06858afdb..52e41879c24bda 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -34,6 +34,21 @@
 
 #include "kallsyms_internal.h"
 
+/*
+ * Get the compressed symbol length and data pointer.
+ */
+static inline const u8 *get_symbol_data(unsigned int off, unsigned int *len)
+{
+	const u8 *p = &kallsyms_names[off];
+	unsigned int l = *p++;
+
+	if (unlikely(l & 0x80))
+		l = (l & 0x7F) | (*p++ << 7);
+	*len = l;
+
+	return p;
+}
+
 /*
  * Expand a compressed symbol data into the resulting uncompressed string,
  * if uncompressed string is too long (>= maxlen), it will be truncated,
@@ -42,28 +57,12 @@
 static unsigned int kallsyms_expand_symbol(unsigned int off,
 					   char *result, size_t maxlen)
 {
-	int len, skipped_first = 0;
+	int skipped_first = 0;
 	const char *tptr;
-	const u8 *data;
+	unsigned int len;
+	const u8 *data = get_symbol_data(off, &len);
 
-	/* Get the compressed symbol length from the first symbol byte. */
-	data = &kallsyms_names[off];
-	len = *data;
-	data++;
-	off++;
-
-	/* If MSB is 1, it is a "big" symbol, so needs an additional byte. */
-	if ((len & 0x80) != 0) {
-		len = (len & 0x7F) | (*data << 7);
-		data++;
-		off++;
-	}
-
-	/*
-	 * Update the offset to return the offset for the next symbol on
-	 * the compressed stream.
-	 */
-	off += len;
+	off = (data - kallsyms_names) + len;
 
 	/*
 	 * For every byte on the compressed symbol data, copy the table
@@ -101,14 +100,43 @@ static unsigned int kallsyms_expand_symbol(unsigned int off,
  */
 static char kallsyms_get_symbol_type(unsigned int off)
 {
-	/*
-	 * Get just the first code, look it up in the token table,
-	 * and return the first char from this token. If MSB of length
-	 * is 1, it is a "big" symbol, so needs an additional byte.
-	 */
-	if (kallsyms_names[off] & 0x80)
-		off++;
-	return kallsyms_token_table[kallsyms_token_index[kallsyms_names[off + 1]]];
+	unsigned int len;
+	const u8 *data = get_symbol_data(off, &len);
+
+	return kallsyms_token_table[kallsyms_token_index[*data]];
+}
+
+/*
+ * Compare an uncompressed ASCII string against a compressed symbol table entry.
+ * Returns negative if name < sym, positive if name > sym, 0 if equal.
+ * Exits immediately on the first mismatched character without decompressing
+ * the rest of the symbol name.
+ */
+static int kallsyms_strcmp_symbol(unsigned int off, const char *name)
+{
+	const char *tptr;
+	unsigned int len;
+	const u8 *data = get_symbol_data(off, &len);
+
+	tptr = &kallsyms_token_table[kallsyms_token_index[*data++]] + 1;
+	while (*tptr) {
+		int diff = (unsigned char)*name++ - (unsigned char)*tptr++;
+
+		if (diff)
+			return diff;
+	}
+
+	while (--len) {
+		tptr = &kallsyms_token_table[kallsyms_token_index[*data++]];
+		do {
+			int diff = (unsigned char)*name++ - (unsigned char)*tptr++;
+
+			if (diff)
+				return diff;
+		} while (*tptr);
+	}
+
+	return (unsigned char)*name;
 }
 
 
@@ -122,10 +150,10 @@ static unsigned int get_symbol_offset(unsigned long pos)
 	int i, len;
 
 	/*
-	 * Use the closest marker we have. We have markers every 256 positions,
-	 * so that should be close enough.
+	 * Use the closest marker we have. We have markers every
+	 * (1 << KALLSYMS_MARKER_SHIFT) positions, so that should be close enough.
 	 */
-	name = &kallsyms_names[kallsyms_markers[pos >> 8]];
+	name = &kallsyms_names[kallsyms_markers[pos >> KALLSYMS_MARKER_SHIFT]];
 
 	/*
 	 * Sequentially scan all the symbols up to the point we're searching
@@ -133,7 +161,7 @@ static unsigned int get_symbol_offset(unsigned long pos)
 	 * so we just need to add the len to the current pointer for every
 	 * symbol we wish to skip.
 	 */
-	for (i = 0; i < (pos & 0xFF); i++) {
+	for (i = 0; i < (pos & KALLSYMS_MARKER_MASK); i++) {
 		len = *name;
 
 		/*
@@ -157,14 +185,11 @@ unsigned long kallsyms_sym_address(int idx)
 	return (unsigned long)offset_to_ptr(kallsyms_offsets + idx);
 }
 
-static unsigned int get_symbol_seq(int index)
+static inline unsigned int get_symbol_seq(int index)
 {
-	unsigned int i, seq = 0;
+	const u8 *p = &kallsyms_seqs_of_names[3 * index];
 
-	for (i = 0; i < 3; i++)
-		seq = (seq << 8) | kallsyms_seqs_of_names[3 * index + i];
-
-	return seq;
+	return (p[0] << 16) | (p[1] << 8) | p[2];
 }
 
 static int kallsyms_lookup_names(const char *name,
@@ -174,7 +199,6 @@ static int kallsyms_lookup_names(const char *name,
 	int ret;
 	int low, mid, high;
 	unsigned int seq, off;
-	char namebuf[KSYM_NAME_LEN];
 
 	low = 0;
 	high = kallsyms_num_syms - 1;
@@ -183,8 +207,7 @@ static int kallsyms_lookup_names(const char *name,
 		mid = low + (high - low) / 2;
 		seq = get_symbol_seq(mid);
 		off = get_symbol_offset(seq);
-		kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
-		ret = strcmp(name, namebuf);
+		ret = kallsyms_strcmp_symbol(off, name);
 		if (ret > 0)
 			low = mid + 1;
 		else if (ret < 0)
@@ -200,8 +223,7 @@ static int kallsyms_lookup_names(const char *name,
 	while (low) {
 		seq = get_symbol_seq(low - 1);
 		off = get_symbol_offset(seq);
-		kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
-		if (strcmp(name, namebuf))
+		if (kallsyms_strcmp_symbol(off, name) != 0)
 			break;
 		low--;
 	}
@@ -212,8 +234,7 @@ static int kallsyms_lookup_names(const char *name,
 		while (high < kallsyms_num_syms - 1) {
 			seq = get_symbol_seq(high + 1);
 			off = get_symbol_offset(seq);
-			kallsyms_expand_symbol(off, namebuf, ARRAY_SIZE(namebuf));
-			if (strcmp(name, namebuf))
+			if (kallsyms_strcmp_symbol(off, name) != 0)
 				break;
 			high++;
 		}
diff --git a/kernel/kallsyms_internal.h b/kernel/kallsyms_internal.h
index 81a867dbe57d48..6a781e4cc77feb 100644
--- a/kernel/kallsyms_internal.h
+++ b/kernel/kallsyms_internal.h
@@ -2,6 +2,16 @@
 #ifndef LINUX_KALLSYMS_INTERNAL_H_
 #define LINUX_KALLSYMS_INTERNAL_H_
 
+/*
+ * Provide compile-constants for scripts/kallsyms.c
+ * so it can build the corresponding kallsyms_marker[] table.
+ * and wrap the rest in __KERNEL__
+ */
+#define KALLSYMS_MARKER_SHIFT 4
+#define KALLSYMS_MARKER_SIZE  (1U << KALLSYMS_MARKER_SHIFT)
+#define KALLSYMS_MARKER_MASK  (KALLSYMS_MARKER_SIZE - 1U)
+
+#ifdef __KERNEL__
 #include <linux/types.h>
 
 extern const int kallsyms_offsets[];
@@ -14,5 +24,6 @@ extern const u16 kallsyms_token_index[];
 
 extern const unsigned int kallsyms_markers[];
 extern const u8 kallsyms_seqs_of_names[];
+#endif /* __KERNEL__ */
 
 #endif // LINUX_KALLSYMS_INTERNAL_H_
diff --git a/kernel/kcov.c b/kernel/kcov.c
index 35420f0ac524d4..54eaae97bfd67d 100644
--- a/kernel/kcov.c
+++ b/kernel/kcov.c
@@ -253,6 +253,14 @@ static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
 
 	count = READ_ONCE(area[0]);
 
+	/*
+	 * area[0] is writable by the collecting process, so count cannot be
+	 * trusted. Bound it to the records that fit, as kcov_move_area()
+	 * does, so the end_pos multiply below cannot wrap past its check.
+	 */
+	if (count >= max_pos / (sizeof(u64) * KCOV_WORDS_PER_CMP))
+		return;
+
 	/* Every record is KCOV_WORDS_PER_CMP 64-bit words. */
 	start_index = 1 + count * KCOV_WORDS_PER_CMP;
 	end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index dc770b9a6d0539..7ee8c9f078f6bc 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -212,6 +212,16 @@ int sanity_check_segment_list(struct kimage *image)
 	}
 #endif
 
+	/*
+	 * Reject destinations that land on hardware-poisoned memory: the
+	 * relocation copy would machine-check on the bad frame.
+	 */
+	for (i = 0; i < nr_segments; i++) {
+		if (range_first_hwpoison(image->segment[i].mem,
+					 image->segment[i].memsz) != PHYS_ADDR_MAX)
+			return -EHWPOISON;
+	}
+
 	/*
 	 * The destination addresses are searched from system RAM rather than
 	 * being allocated from the buddy allocator, so they are not guaranteed
diff --git a/kernel/kexec_elf.c b/kernel/kexec_elf.c
index 3a5c25b2adc94d..89a444a0069300 100644
--- a/kernel/kexec_elf.c
+++ b/kernel/kexec_elf.c
@@ -172,7 +172,7 @@ static int elf_read_ehdr(const char *buf, size_t len, struct elfhdr *ehdr)
 
 	default:
 		pr_debug("Unknown ELF class.\n");
-		return -EINVAL;
+		return -ENOEXEC;
 	}
 
 	return elf_is_ehdr_sane(ehdr, len) ? 0 : -ENOEXEC;
@@ -236,7 +236,7 @@ static int elf_read_phdr(const char *buf, size_t len,
 
 	default:
 		pr_debug("Unknown ELF class.\n");
-		return -EINVAL;
+		return -ENOEXEC;
 	}
 
 	return elf_is_phdr_sane(phdr, len) ? 0 : -ENOEXEC;
diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c
index 59fb9d71e9d862..b668ce8d68eed2 100644
--- a/kernel/kexec_file.c
+++ b/kernel/kexec_file.c
@@ -68,17 +68,19 @@ int kexec_image_probe_default(struct kimage *image, void *buf,
 			      unsigned long buf_len)
 {
 	const struct kexec_file_ops * const *fops;
-	int ret = -ENOEXEC;
 
 	for (fops = &kexec_file_loaders[0]; *fops && (*fops)->probe; ++fops) {
-		ret = (*fops)->probe(buf, buf_len);
-		if (!ret) {
+		int err = (*fops)->probe(buf, buf_len);
+
+		if (!err) {
 			image->fops = *fops;
-			return ret;
+			return 0;
 		}
+		if (err != -ENOEXEC)
+			return err;
 	}
 
-	return ret;
+	return -ENOEXEC;
 }
 
 static void *kexec_image_load_default(struct kimage *image)
@@ -475,6 +477,7 @@ static int locate_mem_hole_top_down(unsigned long start, unsigned long end,
 {
 	struct kimage *image = kbuf->image;
 	unsigned long temp_start, temp_end;
+	phys_addr_t poison;
 
 	temp_end = min(end, kbuf->buf_max);
 	temp_start = temp_end - kbuf->memsz + 1;
@@ -484,7 +487,9 @@ static int locate_mem_hole_top_down(unsigned long start, unsigned long end,
 		/* align down start */
 		temp_start = ALIGN_DOWN(temp_start, kbuf->buf_align);
 
-		if (temp_start < start || temp_start < kbuf->buf_min)
+		/* A candidate above the range means the walk wrapped around */
+		if (temp_start < start || temp_start < kbuf->buf_min ||
+		    temp_start > end)
 			return 0;
 
 		temp_end = temp_start + kbuf->memsz - 1;
@@ -504,6 +509,13 @@ static int locate_mem_hole_top_down(unsigned long start, unsigned long end,
 			continue;
 		}
 
+		poison = range_first_hwpoison(temp_start, kbuf->memsz);
+		if (poison != PHYS_ADDR_MAX) {
+			/* we hit a poisoned page */
+			temp_start = poison - kbuf->memsz;
+			continue;
+		}
+
 		/* We found a suitable memory range */
 		break;
 	} while (1);
@@ -520,6 +532,7 @@ static int locate_mem_hole_bottom_up(unsigned long start, unsigned long end,
 {
 	struct kimage *image = kbuf->image;
 	unsigned long temp_start, temp_end;
+	phys_addr_t poison;
 
 	temp_start = max(start, kbuf->buf_min);
 
@@ -546,6 +559,13 @@ static int locate_mem_hole_bottom_up(unsigned long start, unsigned long end,
 			continue;
 		}
 
+		poison = range_last_hwpoison(temp_start, kbuf->memsz);
+		if (poison != PHYS_ADDR_MAX) {
+			/* we hit a poisoned page */
+			temp_start = poison + PAGE_SIZE;
+			continue;
+		}
+
 		/* We found a suitable memory range */
 		break;
 	} while (1);
@@ -722,8 +742,8 @@ int kexec_locate_mem_hole(struct kexec_buf *kbuf)
 		return 0;
 
 	/*
-	 * If KHO is active, only use KHO scratch memory. All other memory
-	 * could potentially be handed over.
+	 * If KHO is active, only use KHO bootmem. All other memory could
+	 * potentially be handed over.
 	 */
 	ret = kho_locate_mem_hole(kbuf, locate_mem_hole_callback);
 	if (ret <= 0)
@@ -784,7 +804,7 @@ int kexec_add_buffer(struct kexec_buf *kbuf)
 	kbuf->cma = NULL;
 
 	/* Walk the RAM ranges and allocate a suitable range for the buffer */
-	ret = arch_kexec_locate_mem_hole(kbuf);
+	ret = kexec_locate_mem_hole(kbuf);
 	if (ret)
 		return ret;
 
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 6337da5cab9e7e..4edd8ca5c65782 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -42,6 +42,7 @@
 #include <linux/execmem.h>
 #include <linux/cleanup.h>
 #include <linux/wait.h>
+#include <linux/wait_bit.h>
 
 #include <asm/sections.h>
 #include <asm/cacheflush.h>
@@ -526,7 +527,8 @@ enum {
 	OPTIMIZER_ST_FLUSHING = 2,
 };
 
-static DECLARE_COMPLETION(optimizer_completion);
+/* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */
+static unsigned long optimizer_passes;
 
 #define OPTIMIZE_DELAY 5
 
@@ -654,9 +656,9 @@ static void kprobe_optimizer(void)
 		do_free_cleaned_kprobes();
 	}
 
-	/* Step 5: Kick optimizer again if needed. But if there is a flush requested, */
-	if (completion_done(&optimizer_completion))
-		complete(&optimizer_completion);
+	/* Step 5: Wake up flushers, and kick optimizer again if needed. */
+	optimizer_passes++;
+	wake_up_var_locked(&optimizer_passes, &kprobe_mutex);
 
 	if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
 		kick_kprobe_optimizer();	/*normal kick*/
@@ -708,7 +710,8 @@ static void wait_for_kprobe_optimizer_locked(void)
 	lockdep_assert_held(&kprobe_mutex);
 
 	while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
-		init_completion(&optimizer_completion);
+		unsigned long passes = optimizer_passes;
+
 		/*
 		 * Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's
 		 * idle. If it's already kicked, it will see the state change.
@@ -717,9 +720,12 @@ static void wait_for_kprobe_optimizer_locked(void)
 			OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING)
 			wake_up(&kprobe_optimizer_wait);
 
-		mutex_unlock(&kprobe_mutex);
-		wait_for_completion(&optimizer_completion);
-		mutex_lock(&kprobe_mutex);
+		/*
+		 * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which
+		 * this drops while sleeping, so a new count means a full pass ran.
+		 */
+		wait_var_event_mutex(&optimizer_passes,
+				     optimizer_passes != passes, &kprobe_mutex);
 	}
 }
 
diff --git a/kernel/kthread.c b/kernel/kthread.c
index a3f95c90456bd4..cc8cb5d3eab78d 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -81,7 +81,7 @@ enum KTHREAD_BITS {
 
 static inline struct kthread *to_kthread(struct task_struct *k)
 {
-	WARN_ON(!(k->flags & PF_KTHREAD));
+	WARN_ON(!(READ_ONCE(k->flags) & PF_KTHREAD));
 	return k->worker_private;
 }
 
diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c
index a240d1144e89c3..517fe427ff92d9 100644
--- a/kernel/livepatch/core.c
+++ b/kernel/livepatch/core.c
@@ -646,12 +646,15 @@ static const struct kobj_type klp_ktype_patch = {
 
 static void klp_kobj_release_object(struct kobject *kobj)
 {
+	struct klp_patch *patch;
 	struct klp_object *obj;
 
 	obj = container_of(kobj, struct klp_object, kobj);
+	patch = obj->patch;
 
 	if (obj->dynamic)
 		klp_free_object_dynamic(obj);
+	kobject_put(&patch->kobj);
 }
 
 static const struct kobj_type klp_ktype_object = {
@@ -662,12 +665,15 @@ static const struct kobj_type klp_ktype_object = {
 
 static void klp_kobj_release_func(struct kobject *kobj)
 {
+	struct klp_object *obj;
 	struct klp_func *func;
 
 	func = container_of(kobj, struct klp_func, kobj);
+	obj = func->obj;
 
 	if (func->nop)
 		klp_free_func_nop(func);
+	kobject_put(&obj->kobj);
 }
 
 static const struct kobj_type klp_ktype_func = {
@@ -943,7 +949,9 @@ static void klp_init_func_early(struct klp_object *obj,
 				struct klp_func *func)
 {
 	kobject_init(&func->kobj, &klp_ktype_func);
+	kobject_get(&obj->kobj);
 	list_add_tail(&func->node, &obj->func_list);
+	func->obj = obj;
 }
 
 static void klp_init_object_early(struct klp_patch *patch,
@@ -951,7 +959,9 @@ static void klp_init_object_early(struct klp_patch *patch,
 {
 	INIT_LIST_HEAD(&obj->func_list);
 	kobject_init(&obj->kobj, &klp_ktype_object);
+	kobject_get(&patch->kobj);
 	list_add_tail(&obj->node, &patch->obj_list);
+	obj->patch = patch;
 }
 
 static void klp_init_patch_early(struct klp_patch *patch)
diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig
index c13af38ba23ab8..f50daee8eff07c 100644
--- a/kernel/liveupdate/Kconfig
+++ b/kernel/liveupdate/Kconfig
@@ -5,7 +5,6 @@ menu "Live Update and Kexec HandOver"
 config KEXEC_HANDOVER
 	bool "kexec handover"
 	depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE
-	select MEMBLOCK_KHO_SCRATCH
 	select KEXEC_FILE
 	select LIBFDT
 	select CMA
diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c
index 7c4d86daf86d57..7018e25e4eda19 100644
--- a/kernel/liveupdate/kexec_handover.c
+++ b/kernel/liveupdate/kexec_handover.c
@@ -12,6 +12,7 @@
 
 #include <linux/cleanup.h>
 #include <linux/cma.h>
+#include <linux/crash_dump.h>
 #include <linux/kmemleak.h>
 #include <linux/count_zeros.h>
 #include <linux/kasan.h>
@@ -42,12 +43,12 @@
 /*
  * This is the minimal alignment required by deferred struct page init.
  * deferred_init_memmap_chunk frees memory to the buddy allocator, which looks
- * at the neighboring pages (up to MAX_PAGE_ORDER) to merge them.
- * If KHO scratch is not aligned to that value, buddy can access uninitialized
- * struct pages, which can cause a crash.
+ * at the neighboring pages (up to MAX_PAGE_ORDER) to merge them. If KHO bootmem
+ * is not aligned to that value, buddy can access uninitialized struct pages,
+ * which can cause a crash.
  */
-#define SCRATCH_ALIGNMENT_BYTES (PAGE_SIZE * MAX_ORDER_NR_PAGES)
-static_assert(SCRATCH_ALIGNMENT_BYTES >= CMA_MIN_ALIGNMENT_BYTES);
+#define BOOTMEM_ALIGNMENT_BYTES (PAGE_SIZE * MAX_ORDER_NR_PAGES)
+static_assert(BOOTMEM_ALIGNMENT_BYTES >= CMA_MIN_ALIGNMENT_BYTES);
 
 /* The magic token for preserved pages */
 #define KHO_PAGE_MAGIC 0x4b484f50U /* ASCII for 'KHOP' */
@@ -97,7 +98,7 @@ static struct kho_out kho_out = {
 
 struct kho_in {
 	phys_addr_t fdt_phys;
-	phys_addr_t scratch_phys;
+	phys_addr_t bootmem_phys;
 	char previous_release[__NEW_UTS_LEN + 1];
 	u32 kexec_count;
 	struct kho_debugfs dbg;
@@ -657,34 +658,34 @@ static void __init *kho_get_mem_map(const void *fdt)
 
 /*
  * With KHO enabled, memory can become fragmented because KHO regions may
- * be anywhere in physical address space. The scratch regions give us a
+ * be anywhere in physical address space. The bootmem regions give us a
  * safe zones that we will never see KHO allocations from. This is where we
- * can later safely load our new kexec images into and then use the scratch
+ * can later safely load our new kexec images into and then use the bootmem
  * area for early allocations that happen before page allocator is
  * initialized.
  */
-struct kho_scratch *kho_scratch;
-unsigned int kho_scratch_cnt;
+struct kho_bootmem *kho_bootmem;
+unsigned int kho_bootmem_cnt;
 
 /*
- * The scratch areas are scaled by default as percent of memory allocated from
+ * The bootmem areas are scaled by default as percent of memory allocated from
  * memblock. A user can override the scale with command line parameter:
  *
- * kho_scratch=N%
+ * kho_bootmem=N%
  *
  * It is also possible to explicitly define size for a lowmem, a global and
- * per-node scratch areas:
+ * per-node bootmem areas:
  *
- * kho_scratch=l[KMG],n[KMG],m[KMG]
+ * kho_bootmem=l[KMG],n[KMG],m[KMG]
  *
  * The explicit size definition takes precedence over scale definition.
  */
-static unsigned int scratch_scale __initdata = 200;
-static phys_addr_t scratch_size_global __initdata;
-static phys_addr_t scratch_size_pernode __initdata;
-static phys_addr_t scratch_size_lowmem __initdata;
+static unsigned int bootmem_scale __initdata = 200;
+static phys_addr_t bootmem_size_global __initdata;
+static phys_addr_t bootmem_size_pernode __initdata;
+static phys_addr_t bootmem_size_lowmem __initdata;
 
-static int __init kho_parse_scratch_size(char *p)
+static int __init kho_parse_bootmem_size(char *p)
 {
 	size_t len;
 	unsigned long sizes[3];
@@ -708,9 +709,9 @@ static int __init kho_parse_scratch_size(char *p)
 			return -EINVAL;
 
 		memcpy(s_scale, p, len - 1);
-		ret = kstrtouint(s_scale, 10, &scratch_scale);
+		ret = kstrtouint(s_scale, 10, &bootmem_scale);
 		if (!ret)
-			pr_notice("scratch scale is %d%%\n", scratch_scale);
+			pr_notice("bootmem scale is %d%%\n", bootmem_scale);
 		return ret;
 	}
 
@@ -738,81 +739,81 @@ static int __init kho_parse_scratch_size(char *p)
 	if (*p)
 		return -EINVAL;
 
-	scratch_size_lowmem = sizes[0];
-	scratch_size_global = sizes[1];
-	scratch_size_pernode = sizes[2];
-	scratch_scale = 0;
+	bootmem_size_lowmem = sizes[0];
+	bootmem_size_global = sizes[1];
+	bootmem_size_pernode = sizes[2];
+	bootmem_scale = 0;
 
-	pr_notice("scratch areas: lowmem: %lluMiB global: %lluMiB pernode: %lldMiB\n",
-		  (u64)(scratch_size_lowmem >> 20),
-		  (u64)(scratch_size_global >> 20),
-		  (u64)(scratch_size_pernode >> 20));
+	pr_notice("bootmem areas: lowmem: %lluMiB global: %lluMiB pernode: %lldMiB\n",
+		  (u64)(bootmem_size_lowmem >> 20),
+		  (u64)(bootmem_size_global >> 20),
+		  (u64)(bootmem_size_pernode >> 20));
 
 	return 0;
 }
-early_param("kho_scratch", kho_parse_scratch_size);
+early_param("kho_bootmem", kho_parse_bootmem_size);
 
-static void __init scratch_size_update(void)
+static void __init bootmem_size_update(void)
 {
 	/*
 	 * If fixed sizes are not provided via command line, calculate them now.
 	 * Remove HugeTLB allocations from it because they never get allocated
-	 * from scratch.
+	 * from bootmem.
 	 */
-	if (scratch_scale) {
+	if (bootmem_scale) {
 		phys_addr_t size;
 
 		size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT,
 						   NUMA_NO_NODE);
 		size -= memblock_reserved_hugetlb_size(ARCH_LOW_ADDRESS_LIMIT,
 						       NUMA_NO_NODE);
-		size = size * scratch_scale / 100;
-		scratch_size_lowmem = size;
+		size = size * bootmem_scale / 100;
+		bootmem_size_lowmem = size;
 
 		size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
 						   NUMA_NO_NODE);
 		size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE,
 						       NUMA_NO_NODE);
-		size = size * scratch_scale / 100 - scratch_size_lowmem;
-		scratch_size_global = size;
+		size = size * bootmem_scale / 100 - bootmem_size_lowmem;
+		bootmem_size_global = size;
 	}
 
 	/*
-	 * Scratch areas are released as MIGRATE_CMA. Round them up to the right
+	 * bootmem areas are released as MIGRATE_CMA. Round them up to the right
 	 * size.
 	 */
-	scratch_size_lowmem = round_up(scratch_size_lowmem, SCRATCH_ALIGNMENT_BYTES);
-	scratch_size_global = round_up(scratch_size_global, SCRATCH_ALIGNMENT_BYTES);
+	bootmem_size_lowmem = round_up(bootmem_size_lowmem, BOOTMEM_ALIGNMENT_BYTES);
+	bootmem_size_global = round_up(bootmem_size_global, BOOTMEM_ALIGNMENT_BYTES);
 }
 
-static phys_addr_t __init scratch_size_node(int nid)
+static phys_addr_t __init bootmem_size_node(int nid)
 {
 	phys_addr_t size;
 
-	if (scratch_scale) {
+	if (bootmem_scale) {
 		size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE,
 						   nid);
 		/* Do not count HugeTLB pages. */
 		size -= memblock_reserved_hugetlb_size(MEMBLOCK_ALLOC_ANYWHERE,
 						       nid);
-		size = size * scratch_scale / 100;
+		size = size * bootmem_scale / 100;
 	} else {
-		size = scratch_size_pernode;
+		size = bootmem_size_pernode;
 	}
 
-	return round_up(size, SCRATCH_ALIGNMENT_BYTES);
+	return round_up(size, BOOTMEM_ALIGNMENT_BYTES);
 }
 
-bool kho_scratch_overlap(phys_addr_t phys, size_t size)
+bool kho_bootmem_overlap(phys_addr_t phys, size_t size)
 {
-	phys_addr_t scratch_start, scratch_end;
+	phys_addr_t bootmem_start, bootmem_end;
 	unsigned int i;
 
-	for (i = 0; i < kho_scratch_cnt; i++) {
-		scratch_start = kho_scratch[i].addr;
-		scratch_end = kho_scratch[i].addr + kho_scratch[i].size;
+	for (i = 0; i < kho_bootmem_cnt; i++) {
+		bootmem_start = kho_bootmem[i].addr;
+		bootmem_end = kho_bootmem[i].addr + kho_bootmem[i].size;
 
-		if (phys < scratch_end && (phys + size) > scratch_start)
+		if (phys < bootmem_end && (phys + size) > bootmem_start)
 			return true;
 	}
 
@@ -820,7 +821,7 @@ bool kho_scratch_overlap(phys_addr_t phys, size_t size)
 }
 
 /**
- * kho_reserve_scratch - Reserve a contiguous chunk of memory for kexec
+ * kho_reserve_bootmem - Reserve a contiguous chunk of memory for kexec
  *
  * With KHO we can preserve arbitrary pages in the system. To ensure we still
  * have a large contiguous region of memory when we search the physical address
@@ -828,7 +829,7 @@ bool kho_scratch_overlap(phys_addr_t phys, size_t size)
  * active. This CMA region will only be used for movable pages which are not a
  * problem for us during KHO because we can just move them somewhere else.
  */
-static void __init kho_reserve_scratch(void)
+static void __init kho_reserve_bootmem(void)
 {
 	phys_addr_t addr, size;
 	int nid, i = 0;
@@ -836,73 +837,73 @@ static void __init kho_reserve_scratch(void)
 	if (!kho_enable)
 		return;
 
-	scratch_size_update();
+	bootmem_size_update();
 
 	/* FIXME: deal with node hot-plug/remove */
-	kho_scratch_cnt = nodes_weight(node_states[N_MEMORY]) + 2;
-	size = kho_scratch_cnt * sizeof(*kho_scratch);
-	kho_scratch = memblock_alloc(size, PAGE_SIZE);
-	if (!kho_scratch) {
-		pr_err("Failed to reserve scratch array\n");
+	kho_bootmem_cnt = nodes_weight(node_states[N_MEMORY]) + 2;
+	size = kho_bootmem_cnt * sizeof(*kho_bootmem);
+	kho_bootmem = memblock_alloc(size, PAGE_SIZE);
+	if (!kho_bootmem) {
+		pr_err("Failed to reserve bootmem array\n");
 		goto err_disable_kho;
 	}
 
 	/*
-	 * reserve scratch area in low memory for lowmem allocations in the
+	 * reserve bootmem area in low memory for lowmem allocations in the
 	 * next kernel
 	 */
-	size = scratch_size_lowmem;
-	addr = memblock_phys_alloc_range(size, SCRATCH_ALIGNMENT_BYTES, 0,
+	size = bootmem_size_lowmem;
+	addr = memblock_phys_alloc_range(size, BOOTMEM_ALIGNMENT_BYTES, 0,
 					 ARCH_LOW_ADDRESS_LIMIT);
 	if (!addr) {
-		pr_err("Failed to reserve lowmem scratch buffer\n");
-		goto err_free_scratch_desc;
+		pr_err("Failed to reserve lowmem bootmem\n");
+		goto err_free_bootmem_desc;
 	}
 
-	kho_scratch[i].addr = addr;
-	kho_scratch[i].size = size;
+	kho_bootmem[i].addr = addr;
+	kho_bootmem[i].size = size;
 	i++;
 
 	/* reserve large contiguous area for allocations without nid */
-	size = scratch_size_global;
-	addr = memblock_phys_alloc(size, SCRATCH_ALIGNMENT_BYTES);
+	size = bootmem_size_global;
+	addr = memblock_phys_alloc(size, BOOTMEM_ALIGNMENT_BYTES);
 	if (!addr) {
-		pr_err("Failed to reserve global scratch buffer\n");
-		goto err_free_scratch_areas;
+		pr_err("Failed to reserve global bootmem\n");
+		goto err_free_bootmem_areas;
 	}
 
-	kho_scratch[i].addr = addr;
-	kho_scratch[i].size = size;
+	kho_bootmem[i].addr = addr;
+	kho_bootmem[i].size = size;
 	i++;
 
 	/*
 	 * Loop over nodes that have both memory and are online. Skip
-	 * memoryless nodes, as we can not allocate scratch areas there.
+	 * memoryless nodes, as we can not allocate bootmem areas there.
 	 */
 	for_each_node_state(nid, N_MEMORY) {
-		size = scratch_size_node(nid);
-		addr = memblock_alloc_range_nid(size, SCRATCH_ALIGNMENT_BYTES,
+		size = bootmem_size_node(nid);
+		addr = memblock_alloc_range_nid(size, BOOTMEM_ALIGNMENT_BYTES,
 						0, MEMBLOCK_ALLOC_ACCESSIBLE,
 						nid, true);
 		if (!addr) {
-			pr_err("Failed to reserve nid %d scratch buffer\n", nid);
-			goto err_free_scratch_areas;
+			pr_err("Failed to reserve nid %d bootmem\n", nid);
+			goto err_free_bootmem_areas;
 		}
 
-		kho_scratch[i].addr = addr;
-		kho_scratch[i].size = size;
+		kho_bootmem[i].addr = addr;
+		kho_bootmem[i].size = size;
 		i++;
 	}
 
 	return;
 
-err_free_scratch_areas:
+err_free_bootmem_areas:
 	for (i--; i >= 0; i--)
-		memblock_phys_free(kho_scratch[i].addr, kho_scratch[i].size);
-err_free_scratch_desc:
-	memblock_free(kho_scratch, kho_scratch_cnt * sizeof(*kho_scratch));
+		memblock_phys_free(kho_bootmem[i].addr, kho_bootmem[i].size);
+err_free_bootmem_desc:
+	memblock_free(kho_bootmem, kho_bootmem_cnt * sizeof(*kho_bootmem));
 err_disable_kho:
-	pr_warn("Failed to reserve scratch area, disabling kexec handover\n");
+	pr_warn("Failed to reserve bootmem, disabling kexec handover\n");
 	kho_enable = false;
 }
 
@@ -912,11 +913,11 @@ static void __init kho_reserve_scratch(void)
  * on smaller systems. The algorithm itself doesn't depend on the actual value,
  * so it can be changed to a different heuristic later if needed.
  */
-#define KHO_SCRATCH_EXT_BLKSIZE		SZ_1G
-#define KHO_SCRATCH_EXT_BLKSHIFT	const_ilog2(KHO_SCRATCH_EXT_BLKSIZE)
+#define KHO_DISCOVER_BLKSIZE		SZ_1G
+#define KHO_DISCOVER_BLKSHIFT	const_ilog2(KHO_DISCOVER_BLKSIZE)
 
 /* Called for the KHO preserved memory radix tree. */
-static int __init kho_ext_walk_leaf(unsigned long key, void *data)
+static int __init kho_discover_walk_leaf(unsigned long key, void *data)
 {
 	struct kho_radix_tree *busy_blocks = data;
 	phys_addr_t start, end;
@@ -931,54 +932,57 @@ static int __init kho_ext_walk_leaf(unsigned long key, void *data)
 	end = start + (1UL << (order + PAGE_SHIFT));
 
 	while (start < end) {
-		err = kho_radix_add_key(busy_blocks, start >> KHO_SCRATCH_EXT_BLKSHIFT);
+		err = kho_radix_add_key(busy_blocks, start >> KHO_DISCOVER_BLKSHIFT);
 		if (err)
 			return err;
 
-		start += (1UL << KHO_SCRATCH_EXT_BLKSHIFT);
+		start += (1UL << KHO_DISCOVER_BLKSHIFT);
 	}
 
 	return 0;
 }
 
 /* Called for the KHO preserved memory radix tree. */
-static int __init kho_ext_walk_node(phys_addr_t phys, void *data)
+static int __init kho_discover_walk_node(phys_addr_t phys, void *data)
 {
 	struct kho_radix_tree *busy_blocks = data;
 
-	return kho_radix_add_key(busy_blocks, phys >> KHO_SCRATCH_EXT_BLKSHIFT);
+	return kho_radix_add_key(busy_blocks, phys >> KHO_DISCOVER_BLKSHIFT);
 }
 
 /* Called for the busy block radix tree. */
-static int __init kho_ext_mark_scratch(unsigned long key, void *data)
+static int __init kho_discover_mark_noprsrv(unsigned long key, void *data)
 {
 	phys_addr_t *prev_end = data;
-	phys_addr_t start = key << KHO_SCRATCH_EXT_BLKSHIFT;
+	phys_addr_t start = key << KHO_DISCOVER_BLKSHIFT;
 	int err;
 
 	if (start > *prev_end) {
-		err = memblock_mark_kho_scratch(*prev_end, start - *prev_end);
+		err = memblock_mark_kho_noprsrv(*prev_end, start - *prev_end);
 		if (err)
 			return err;
 	}
 
-	*prev_end = start + (1UL << KHO_SCRATCH_EXT_BLKSHIFT);
+	*prev_end = start + (1UL << KHO_DISCOVER_BLKSHIFT);
 	return 0;
 }
 
 /*
- * kho_extend_scratch - Extend the scratch regions
+ * kho_discover_noprsrv - Discover memory with no preservations
+ *
+ * Discovers memory ranges with no preserved memory and marks it as NOPRSRV.
+ * This lets memblock allocate memory from areas outside KHO bootmem.
  *
  * The KHO preserved memory radix tree mixes both physical address and order
  * into a single key. This makes it hard to look for free ranges directly. This
  * function first walks the radix tree and digests it down into another radix
- * tree, whose keys identify blocks of size KHO_SCRATCH_EXT_BLKSIZE which
- * contain preserved memory.
+ * tree, whose keys identify blocks of size KHO_DISCOVER_BLKSIZE which contain
+ * preserved memory.
  *
  * Then it walks the digested radix tree and marks everything that doesn't have
- * preserved memory as scratch.
+ * preserved memory as NOPRSRV.
  *
- * NOTE: This function allocates memory so it should be called when scratch has
+ * NOTE: This function allocates memory so it should be called when bootmem has
  * available space.
  *
  * NOTE: The pages of the KHO preserved memory radix tree tables are not marked
@@ -987,18 +991,18 @@ static int __init kho_ext_mark_scratch(unsigned long key, void *data)
  * them to be "preserved memory" and marks their blocks as busy.
  *
  * NOTE: efi_init()::reserve_regions() removes all regions except
- * MEMBLOCK_KHO_SCRATCH. This function adds such regions but they are not KHO
- * scratch memory, so they should not be removed. This function should always be
+ * MEMBLOCK_KHO_NOPRSRV. This function adds such regions but they are not KHO
+ * boot memory, so they should not be removed. This function should always be
  * called after reserve_regions().
  */
-static void __init kho_extend_scratch(void)
+static void __init kho_discover_noprsrv(void)
 {
 	const struct kho_radix_walk_cb kho_cb = {
-		.leaf = kho_ext_walk_leaf,
-		.node = kho_ext_walk_node,
+		.leaf = kho_discover_walk_leaf,
+		.node = kho_discover_walk_node,
 	};
 	const struct kho_radix_walk_cb ext_cb = {
-		.leaf = kho_ext_mark_scratch,
+		.leaf = kho_discover_mark_noprsrv,
 	};
 	static struct lock_class_key busy_radix_class;
 	struct kho_radix_tree busy_blocks;
@@ -1023,21 +1027,21 @@ static void __init kho_extend_scratch(void)
 	if (err)
 		goto out;
 
-	/* Walk the busy blocks and mark everything between keys as scratch. */
+	/* Walk the busy blocks and mark everything between keys as noprsrv. */
 	err = kho_radix_walk_tree(&busy_blocks, &ext_cb, &prev_end);
 	if (err)
 		goto out;
 
 	/* Mark everything from last busy block to end of DRAM. */
 	if (prev_end < memblock_end_of_DRAM())
-		err = memblock_mark_kho_scratch(prev_end, memblock_end_of_DRAM() - prev_end);
+		err = memblock_mark_kho_noprsrv(prev_end, memblock_end_of_DRAM() - prev_end);
 
 	/* fallthrough */
 out:
 	kho_radix_destroy_tree(&busy_blocks);
 print:
 	if (err)
-		pr_err("Failed to extend scratch: %pe\n", ERR_PTR(err));
+		pr_err("Failed to discover extra allocatable memory: %pe\n", ERR_PTR(err));
 }
 
 /**
@@ -1151,7 +1155,7 @@ int kho_preserve_folio(struct folio *folio)
 	const unsigned int order = folio_order(folio);
 
 	if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) &&
-	    WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
+	    WARN_ON(kho_bootmem_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order)))
 		return -EINVAL;
 
 	return kho_radix_add_key(tree, kho_encode_radix_key(PFN_PHYS(pfn),
@@ -1229,7 +1233,7 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages)
 	int err = 0;
 
 	if (IS_ENABLED(CONFIG_KEXEC_HANDOVER_DEBUG) &&
-	    WARN_ON(kho_scratch_overlap(start_pfn << PAGE_SHIFT,
+	    WARN_ON(kho_bootmem_overlap(start_pfn << PAGE_SHIFT,
 					nr_pages << PAGE_SHIFT))) {
 		return -EINVAL;
 	}
@@ -1826,7 +1830,7 @@ static __init int kho_init(void)
 
 	err = kho_radix_init_tree(tree, NULL);
 	if (err)
-		goto err_free_scratch;
+		goto err_free_bootmem;
 
 	kho_out.fdt = kho_alloc_preserve(PAGE_SIZE);
 	if (IS_ERR(kho_out.fdt)) {
@@ -1855,9 +1859,9 @@ static __init int kho_init(void)
 		return 0;
 	}
 
-	for (int i = 0; i < kho_scratch_cnt; i++) {
-		unsigned long base_pfn = PHYS_PFN(kho_scratch[i].addr);
-		unsigned long count = kho_scratch[i].size >> PAGE_SHIFT;
+	for (int i = 0; i < kho_bootmem_cnt; i++) {
+		unsigned long base_pfn = PHYS_PFN(kho_bootmem[i].addr);
+		unsigned long count = kho_bootmem[i].size >> PAGE_SHIFT;
 		unsigned long pfn;
 
 		/*
@@ -1865,10 +1869,10 @@ static __init int kho_init(void)
 		 * corresponding PRESENT bit in the kernel page table.
 		 * Subsequent kmemleak scans of these pages cause the
 		 * non-PRESENT page faults.
-		 * Mark scratch areas with kmemleak_ignore_phys() to exclude
+		 * Mark bootmem areas with kmemleak_ignore_phys() to exclude
 		 * them from kmemleak scanning.
 		 */
-		kmemleak_ignore_phys(kho_scratch[i].addr);
+		kmemleak_ignore_phys(kho_bootmem[i].addr);
 		for (pfn = base_pfn; pfn < base_pfn + count;
 		     pfn += pageblock_nr_pages)
 			init_cma_reserved_pageblock(pfn_to_page(pfn));
@@ -1884,11 +1888,11 @@ static __init int kho_init(void)
 	kho_unpreserve_free(kho_out.fdt);
 err_free_kho_radix_tree:
 	kho_radix_destroy_tree(tree);
-err_free_scratch:
+err_free_bootmem:
 	kho_out.fdt = NULL;
-	for (int i = 0; i < kho_scratch_cnt; i++) {
-		void *start = __va(kho_scratch[i].addr);
-		void *end = start + kho_scratch[i].size;
+	for (int i = 0; i < kho_bootmem_cnt; i++) {
+		void *start = __va(kho_bootmem[i].addr);
+		void *end = start + kho_bootmem[i].size;
 
 		free_reserved_area(start, end, -1, "");
 	}
@@ -1907,7 +1911,7 @@ void __init kho_memory_init_early(void)
 
 	/*
 	 * kho_get_mem_map() should always succeed. If it fails, kho_populate()
-	 * catches that and never sets kho_in.scratch_phys, which stops memory
+	 * catches that and never sets kho_in.bootmem_phys, which stops memory
 	 * retrieval.
 	 */
 	mem_map = kho_get_mem_map(fdt);
@@ -1915,41 +1919,46 @@ void __init kho_memory_init_early(void)
 		goto err;
 
 	/*
-	 * kho_scratch_overlap() needs kho_scratch to be initialized. It
-	 * is used by free_area_init() on KHO boots, so initialize it
-	 * early.
+	 * kho_bootmem_overlap() needs kho_bootmem to be initialized. It is used
+	 * by free_area_init() on KHO boots, so initialize it early.
 	 */
-	kho_scratch = phys_to_virt(kho_in.scratch_phys);
+	kho_bootmem = phys_to_virt(kho_in.bootmem_phys);
 
 	if (kho_radix_init_tree(&kho_in.radix_tree, mem_map))
 		goto err;
 
-	kho_extend_scratch();
+	kho_discover_noprsrv();
 
 	return;
 
 err:
 	/*
 	 * Failed to initialize preserved memory radix tree. Clear FDT
-	 * and scratch so KHO users don't treat it as a KHO boot.
+	 * and bootmem so KHO users don't treat it as a KHO boot.
 	 */
 	kho_in.fdt_phys = 0;
-	kho_in.scratch_phys = 0;
+	kho_in.bootmem_phys = 0;
 }
 
 void __init kho_memory_init(void)
 {
-	if (kho_in.scratch_phys)
+	if (is_kdump_kernel()) {
+		kho_enable = false;
+		pr_info("disabled in the kdump kernel\n");
+		return;
+	}
+
+	if (kho_in.bootmem_phys)
 		kho_mem_retrieve();
 	else
-		kho_reserve_scratch();
+		kho_reserve_bootmem();
 }
 
 void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
-			 phys_addr_t scratch_phys, u64 scratch_len)
+			 phys_addr_t bootmem_phys, u64 bootmem_len)
 {
-	unsigned int scratch_cnt = scratch_len / sizeof(*kho_scratch);
-	struct kho_scratch *scratch = NULL;
+	unsigned int bootmem_cnt = bootmem_len / sizeof(*kho_bootmem);
+	struct kho_bootmem *bootmem = NULL;
 	phys_addr_t mem_map_phys;
 	void *fdt = NULL;
 	bool populated = false;
@@ -1978,52 +1987,52 @@ void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
 	if (!mem_map_phys)
 		goto unmap_fdt;
 
-	scratch = early_memremap(scratch_phys, scratch_len);
-	if (!scratch) {
-		pr_warn("setup: failed to memremap scratch (phys=0x%llx, len=%lld)\n",
-			scratch_phys, scratch_len);
+	bootmem = early_memremap(bootmem_phys, bootmem_len);
+	if (!bootmem) {
+		pr_warn("setup: failed to memremap bootmem (phys=0x%llx, len=%lld)\n",
+			bootmem_phys, bootmem_len);
 		goto unmap_fdt;
 	}
 
 	/*
-	 * We pass a safe contiguous blocks of memory to use for early boot
-	 * purporses from the previous kernel so that we can resize the
-	 * memblock array as needed.
+	 * We pass a safe contiguous blocks of memory with no preservations to
+	 * use for early boot purporses from the previous kernel so that we can
+	 * resize the memblock array as needed.
 	 */
-	for (int i = 0; i < scratch_cnt; i++) {
-		struct kho_scratch *area = &scratch[i];
+	for (int i = 0; i < bootmem_cnt; i++) {
+		struct kho_bootmem *area = &bootmem[i];
 		u64 size = area->size;
 
 		memblock_add(area->addr, size);
-		err = memblock_mark_kho_scratch(area->addr, size);
+		err = memblock_mark_kho_noprsrv(area->addr, size);
 		if (err) {
-			pr_warn("failed to mark the scratch region 0x%pa+0x%pa: %pe",
+			pr_warn("failed to mark the bootmem region 0x%pa+0x%pa: %pe",
 				&area->addr, &size, ERR_PTR(err));
-			goto unmap_scratch;
+			goto unmap_bootmem;
 		}
-		pr_debug("Marked 0x%pa+0x%pa as scratch", &area->addr, &size);
+		pr_debug("Marked 0x%pa+0x%pa as bootmem", &area->addr, &size);
 	}
 
-	memblock_reserve(scratch_phys, scratch_len);
+	memblock_reserve(bootmem_phys, bootmem_len);
 
 	/*
-	 * Now that we have a viable region of scratch memory, let's tell
+	 * Now that we have a viable region of boot memory, let's tell
 	 * the memblocks allocator to only use that for any allocations.
 	 * That way we ensure that nothing scribbles over in use data while
 	 * we initialize the page tables which we will need to ingest all
 	 * memory reservations from the previous kernel.
 	 */
-	memblock_set_kho_scratch_only();
+	memblock_set_kho_noprsrv_only();
 
 	kho_in.fdt_phys = fdt_phys;
-	kho_in.scratch_phys = scratch_phys;
-	kho_scratch_cnt = scratch_cnt;
+	kho_in.bootmem_phys = bootmem_phys;
+	kho_bootmem_cnt = bootmem_cnt;
 
 	populated = true;
 	pr_info("found kexec handover data.\n");
 
-unmap_scratch:
-	early_memunmap(scratch, scratch_len);
+unmap_bootmem:
+	early_memunmap(bootmem, bootmem_len);
 unmap_fdt:
 	early_memunmap(fdt, fdt_len);
 report:
@@ -2035,47 +2044,47 @@ void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len,
 
 int kho_fill_kimage(struct kimage *image)
 {
-	ssize_t scratch_size;
+	ssize_t bootmem_size;
 	int err = 0;
-	struct kexec_buf scratch;
+	struct kexec_buf bootmem;
 
 	if (!kho_enable || image->type == KEXEC_TYPE_CRASH)
 		return 0;
 
 	image->kho.fdt = virt_to_phys(kho_out.fdt);
 
-	scratch_size = sizeof(*kho_scratch) * kho_scratch_cnt;
-	scratch = (struct kexec_buf){
+	bootmem_size = sizeof(*kho_bootmem) * kho_bootmem_cnt;
+	bootmem = (struct kexec_buf){
 		.image = image,
-		.buffer = kho_scratch,
-		.bufsz = scratch_size,
+		.buffer = kho_bootmem,
+		.bufsz = bootmem_size,
 		.mem = KEXEC_BUF_MEM_UNKNOWN,
-		.memsz = scratch_size,
+		.memsz = bootmem_size,
 		.buf_align = SZ_64K, /* Makes it easier to map */
 		.buf_max = ULONG_MAX,
 		.top_down = true,
 	};
-	err = kexec_add_buffer(&scratch);
+	err = kexec_add_buffer(&bootmem);
 	if (err)
 		return err;
-	image->kho.scratch = &image->segment[image->nr_segments - 1];
+	image->kho.bootmem = &image->segment[image->nr_segments - 1];
 
 	return 0;
 }
 
-static int kho_walk_scratch(struct kexec_buf *kbuf,
+static int kho_walk_bootmem(struct kexec_buf *kbuf,
 			    int (*func)(struct resource *, void *))
 {
 	int ret = 0;
 	int i;
 
-	for (i = 0; i < kho_scratch_cnt; i++) {
+	for (i = 0; i < kho_bootmem_cnt; i++) {
 		struct resource res = {
-			.start = kho_scratch[i].addr,
-			.end = kho_scratch[i].addr + kho_scratch[i].size - 1,
+			.start = kho_bootmem[i].addr,
+			.end = kho_bootmem[i].addr + kho_bootmem[i].size - 1,
 		};
 
-		/* Try to fit the kimage into our KHO scratch region */
+		/* Try to fit the kimage into our KHO bootmem region */
 		ret = func(&res, kbuf);
 		if (ret)
 			break;
@@ -2092,7 +2101,7 @@ int kho_locate_mem_hole(struct kexec_buf *kbuf,
 	if (!kho_enable || kbuf->image->type == KEXEC_TYPE_CRASH)
 		return 1;
 
-	ret = kho_walk_scratch(kbuf, func);
+	ret = kho_walk_bootmem(kbuf, func);
 
 	return ret == 1 ? 0 : -EADDRNOTAVAIL;
 }
diff --git a/kernel/liveupdate/kexec_handover_debugfs.c b/kernel/liveupdate/kexec_handover_debugfs.c
index 257ee8a52be665..7d74477da0b799 100644
--- a/kernel/liveupdate/kexec_handover_debugfs.c
+++ b/kernel/liveupdate/kexec_handover_debugfs.c
@@ -77,23 +77,23 @@ void kho_debugfs_blob_remove(struct kho_debugfs *dbg, void *blob)
 	}
 }
 
-static int scratch_phys_show(struct seq_file *m, void *v)
+static int bootmem_phys_show(struct seq_file *m, void *v)
 {
-	for (int i = 0; i < kho_scratch_cnt; i++)
-		seq_printf(m, "0x%llx\n", kho_scratch[i].addr);
+	for (int i = 0; i < kho_bootmem_cnt; i++)
+		seq_printf(m, "0x%llx\n", kho_bootmem[i].addr);
 
 	return 0;
 }
-DEFINE_SHOW_ATTRIBUTE(scratch_phys);
+DEFINE_SHOW_ATTRIBUTE(bootmem_phys);
 
-static int scratch_len_show(struct seq_file *m, void *v)
+static int bootmem_len_show(struct seq_file *m, void *v)
 {
-	for (int i = 0; i < kho_scratch_cnt; i++)
-		seq_printf(m, "0x%llx\n", kho_scratch[i].size);
+	for (int i = 0; i < kho_bootmem_cnt; i++)
+		seq_printf(m, "0x%llx\n", kho_bootmem[i].size);
 
 	return 0;
 }
-DEFINE_SHOW_ATTRIBUTE(scratch_len);
+DEFINE_SHOW_ATTRIBUTE(bootmem_len);
 
 __init void kho_in_debugfs_init(struct kho_debugfs *dbg, const void *fdt)
 {
@@ -186,13 +186,13 @@ __init int kho_out_debugfs_init(struct kho_debugfs *dbg)
 	if (IS_ERR(sub_fdt_dir))
 		goto err_rmdir;
 
-	f = debugfs_create_file("scratch_phys", 0400, dir, NULL,
-				&scratch_phys_fops);
+	f = debugfs_create_file("bootmem_phys", 0400, dir, NULL,
+				&bootmem_phys_fops);
 	if (IS_ERR(f))
 		goto err_rmdir;
 
-	f = debugfs_create_file("scratch_len", 0400, dir, NULL,
-				&scratch_len_fops);
+	f = debugfs_create_file("bootmem_len", 0400, dir, NULL,
+				&bootmem_len_fops);
 	if (IS_ERR(f))
 		goto err_rmdir;
 
diff --git a/kernel/liveupdate/kexec_handover_internal.h b/kernel/liveupdate/kexec_handover_internal.h
index 805d2a76c38818..e9685e1ef6d78c 100644
--- a/kernel/liveupdate/kexec_handover_internal.h
+++ b/kernel/liveupdate/kexec_handover_internal.h
@@ -19,8 +19,8 @@ struct kho_debugfs {
 struct kho_debugfs {};
 #endif
 
-extern struct kho_scratch *kho_scratch;
-extern unsigned int kho_scratch_cnt;
+extern struct kho_bootmem *kho_bootmem;
+extern unsigned int kho_bootmem_cnt;
 
 #ifdef CONFIG_KEXEC_HANDOVER_DEBUGFS
 int kho_debugfs_init(void);
diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c
index c39f96961a85be..a39992da504fbc 100644
--- a/kernel/liveupdate/luo_file.c
+++ b/kernel/liveupdate/luo_file.c
@@ -91,6 +91,21 @@
  *    again later. If the issue cannot be resolved, these resources will be held
  *    by LUO until the next live update cycle, at which point they will be
  *    discarded.
+ *
+ * Dependencies:
+ *
+ * Preserved files may have dependencies on other files that also need to be
+ * preserved during a live update. A file handler can resolve these dependencies
+ * by querying the token of the dependency. That token can then be serialized
+ * and used to fetch the dependency during the .retrieve() callback in the next
+ * kernel.
+ *
+ * Some of these dependencies can be resolved late during file handler's
+ * .freeze() callback, but some of these might need resolution during file
+ * handler's .preserve() callback based on constraints such as immutability and
+ * performance. This introduces an order of preservation. Note that the order of
+ * preservation of dependencies can be safely relaxed later, but it cannot be
+ * enforced later.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -284,6 +299,7 @@ int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd)
 	mutex_init(&luo_file->mutex);
 
 	args.handler = fh;
+	args.session = luo_session_from_file_set(file_set);
 	args.file = file;
 	err = fh->ops->preserve(&args);
 	if (err)
@@ -341,6 +357,7 @@ void luo_file_unpreserve_files(struct luo_file_set *file_set)
 					   struct luo_file, list);
 
 		args.handler = luo_file->fh;
+		args.session = luo_session_from_file_set(file_set);
 		args.file = luo_file->file;
 		args.serialized_data = luo_file->serialized_data;
 		args.private_data = luo_file->private_data;
@@ -374,6 +391,7 @@ static int luo_file_freeze_one(struct luo_file_set *file_set,
 		struct liveupdate_file_op_args args = {0};
 
 		args.handler = luo_file->fh;
+		args.session = luo_session_from_file_set(file_set);
 		args.file = luo_file->file;
 		args.serialized_data = luo_file->serialized_data;
 		args.private_data = luo_file->private_data;
@@ -395,6 +413,7 @@ static void luo_file_unfreeze_one(struct luo_file_set *file_set,
 		struct liveupdate_file_op_args args = {0};
 
 		args.handler = luo_file->fh;
+		args.session = luo_session_from_file_set(file_set);
 		args.file = luo_file->file;
 		args.serialized_data = luo_file->serialized_data;
 		args.private_data = luo_file->private_data;
@@ -587,6 +606,7 @@ int luo_retrieve_file(struct luo_file_set *file_set, u64 token,
 	}
 
 	args.handler = luo_file->fh;
+	args.session = luo_session_from_file_set(file_set);
 	args.serialized_data = luo_file->serialized_data;
 	err = luo_file->fh->ops->retrieve(&args);
 	if (err) {
@@ -620,6 +640,7 @@ static int luo_file_can_finish_one(struct luo_file_set *file_set,
 		struct liveupdate_file_op_args args = {0};
 
 		args.handler = luo_file->fh;
+		args.session = luo_session_from_file_set(file_set);
 		args.file = luo_file->file;
 		args.serialized_data = luo_file->serialized_data;
 		args.retrieve_status = luo_file->retrieve_status;
@@ -637,6 +658,7 @@ static void luo_file_finish_one(struct luo_file_set *file_set,
 	guard(mutex)(&luo_file->mutex);
 
 	args.handler = luo_file->fh;
+	args.session = luo_session_from_file_set(file_set);
 	args.file = luo_file->file;
 	args.serialized_data = luo_file->serialized_data;
 	args.retrieve_status = luo_file->retrieve_status;
@@ -926,3 +948,65 @@ void liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh)
 	luo_flb_unregister_all(fh);
 	list_del(&ACCESS_PRIVATE(fh, list));
 }
+
+/**
+ * liveupdate_get_token_outgoing - Get the token for a preserved file.
+ * @s:      The outgoing liveupdate session.
+ * @file:   The file object to search for.
+ * @tokenp: Output parameter for the found token.
+ *
+ * Searches the list of preserved files in an outgoing session for a matching
+ * file object. If found, the corresponding user-provided token is returned.
+ *
+ * This function is intended for in-kernel callers that need to correlate a
+ * file with its liveupdate token.
+ *
+ * Context: It must be called with session mutex acquired.
+ * Return: 0 on success, -ENOENT if the file is not preserved in this session.
+ */
+int liveupdate_get_token_outgoing(struct liveupdate_session *s,
+				  struct file *file, u64 *tokenp)
+{
+	struct luo_file_set *file_set = luo_file_set_from_session_locked(s);
+	struct luo_file *luo_file;
+	int err = -ENOENT;
+
+	list_for_each_entry(luo_file, &file_set->files_list, list) {
+		if (luo_file->file == file) {
+			if (tokenp)
+				*tokenp = luo_file->token;
+			err = 0;
+			break;
+		}
+	}
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(liveupdate_get_token_outgoing);
+
+/**
+ * liveupdate_get_file_incoming - Retrieves a preserved file for in-kernel use.
+ * @s:      The incoming liveupdate session (restored from the previous kernel).
+ * @token:  The unique token identifying the file to retrieve.
+ * @filep:  On success, this will be populated with a pointer to the retrieved
+ *          'struct file'.
+ *
+ * Provides a kernel-internal API for other subsystems to retrieve their
+ * preserved files after a live update. This function is a simple wrapper
+ * around luo_retrieve_file(), allowing callers to find a file by its token.
+ *
+ * The caller receives a new reference to the file and must call fput() when it
+ * is no longer needed. The file's lifetime is managed by LUO and any userspace
+ * file descriptors.
+ *
+ * Context: It must be called with session mutex acquired of a restored session.
+ * Return: 0 on success. Returns -ENOENT if no file with the matching token is
+ *         found, or any other negative errno on failure.
+ */
+int liveupdate_get_file_incoming(struct liveupdate_session *s, u64 token,
+				 struct file **filep)
+{
+	return luo_retrieve_file(luo_file_set_from_session_locked(s),
+				 token, filep);
+}
+EXPORT_SYMBOL_GPL(liveupdate_get_file_incoming);
diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h
index 64879ffe73781c..dac6644bfb18aa 100644
--- a/kernel/liveupdate/luo_internal.h
+++ b/kernel/liveupdate/luo_internal.h
@@ -77,6 +77,23 @@ struct luo_session {
 
 extern struct rw_semaphore luo_register_rwlock;
 
+static inline struct liveupdate_session *luo_session_from_file_set(struct luo_file_set *file_set)
+{
+	struct luo_session *session;
+
+	session = container_of(file_set, struct luo_session, file_set);
+
+	return (struct liveupdate_session *)session;
+}
+
+static inline struct luo_file_set *luo_file_set_from_session_locked(struct liveupdate_session *s)
+{
+	struct luo_session *session = (struct luo_session *)s;
+
+	lockdep_assert_held(&session->mutex);
+	return &session->file_set;
+}
+
 int luo_session_create(const char *name, struct file **filep);
 int luo_session_retrieve(const char *name, struct file **filep);
 void __init luo_session_setup_outgoing(u64 *sessions_pa);
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 4728631ae71940..5a9534c715b845 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1644,7 +1644,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
 			break;
 		}
 
-		if (timeout && !timeout->task) {
+		if (timeout && !hrtimer_sleeper_task_get(timeout)) {
 			ret = -ETIMEDOUT;
 			break;
 		}
diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c
index f23126d804b25a..1ff8ce857d9258 100644
--- a/kernel/module/kallsyms.c
+++ b/kernel/module/kallsyms.c
@@ -75,23 +75,29 @@ static char elf_type(const Elf_Sym *sym, const struct load_info *info)
 	return '?';
 }
 
-static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
-			   unsigned int shnum, unsigned int pcpundx)
+static bool is_core_symbol(struct module *mod, const struct load_info *info,
+			   unsigned int symnum, const Elf_Sym *src)
 {
 	const Elf_Shdr *sec;
 	enum mod_mem_type type;
 
+	if (symnum == 0 || is_livepatch_module(mod))
+		return true;
+
 	if (src->st_shndx == SHN_UNDEF ||
-	    src->st_shndx >= shnum ||
+	    src->st_shndx >= info->hdr->e_shnum ||
 	    !src->st_name)
 		return false;
 
+	if (is_ignored_kernel_symbol(&info->strtab[src->st_name]))
+		return false;
+
 #ifdef CONFIG_KALLSYMS_ALL
-	if (src->st_shndx == pcpundx)
+	if (src->st_shndx == info->index.pcpu)
 		return true;
 #endif
 
-	sec = sechdrs + src->st_shndx;
+	sec = info->sechdrs + src->st_shndx;
 	type = sec->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
 	if (!(sec->sh_flags & SHF_ALLOC)
 #ifndef CONFIG_KALLSYMS_ALL
@@ -130,12 +136,11 @@ void layout_symtab(struct module *mod, struct load_info *info)
 
 	/* Compute total space required for the core symbols' strtab. */
 	for (ndst = i = 0; i < nsrc; i++) {
-		if (i == 0 || is_livepatch_module(mod) ||
-		    is_core_symbol(src + i, info->sechdrs, info->hdr->e_shnum,
-				   info->index.pcpu)) {
-			strtab_size += strlen(&info->strtab[src[i].st_name]) + 1;
-			ndst++;
-		}
+		if (!is_core_symbol(mod, info, i, src + i))
+			continue;
+
+		strtab_size += strlen(&info->strtab[src[i].st_name]) + 1;
+		ndst++;
 	}
 
 	/* Append room for core symbols at end of core part. */
@@ -197,23 +202,21 @@ void add_kallsyms(struct module *mod, const struct load_info *info)
 	strtab_size = info->core_typeoffs - info->stroffs;
 	src = kallsyms->symtab;
 	for (ndst = i = 0; i < kallsyms->num_symtab; i++) {
+		ssize_t ret;
+
 		kallsyms->typetab[i] = elf_type(src + i, info);
-		if (i == 0 || is_livepatch_module(mod) ||
-		    is_core_symbol(src + i, info->sechdrs, info->hdr->e_shnum,
-				   info->index.pcpu)) {
-			ssize_t ret;
-
-			mod->core_kallsyms.typetab[ndst] =
-				kallsyms->typetab[i];
-			dst[ndst] = src[i];
-			dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
-			ret = strscpy(s, &kallsyms->strtab[src[i].st_name],
-				      strtab_size);
-			if (ret < 0)
-				break;
-			s += ret + 1;
-			strtab_size -= ret + 1;
-		}
+
+		if (!is_core_symbol(mod, info, i, src + i))
+			continue;
+
+		mod->core_kallsyms.typetab[ndst] = kallsyms->typetab[i];
+		dst[ndst] = src[i];
+		dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
+		ret = strscpy(s, &kallsyms->strtab[src[i].st_name], strtab_size);
+		if (ret < 0)
+			break;
+		s += ret + 1;
+		strtab_size -= ret + 1;
 	}
 
 	/* Set up to point into init section. */
@@ -294,7 +297,7 @@ static const char *find_kallsyms_symbol(struct module *mod,
 		 * and inserted at a whim.
 		 */
 		if (*kallsyms_symbol_name(kallsyms, i) == '\0' ||
-		    is_mapping_symbol(kallsyms_symbol_name(kallsyms, i)))
+		    is_ignored_kernel_symbol(kallsyms_symbol_name(kallsyms, i)))
 			continue;
 
 		if (thisval <= addr && thisval > bestval) {
diff --git a/kernel/module/kmod.c b/kernel/module/kmod.c
index a25dccdf7aa75c..24091210741e08 100644
--- a/kernel/module/kmod.c
+++ b/kernel/module/kmod.c
@@ -28,6 +28,7 @@
 #include <linux/ptrace.h>
 #include <linux/async.h>
 #include <linux/uaccess.h>
+#include <linux/umh.h>
 
 #include <trace/events/module.h>
 #include "internal.h"
@@ -81,7 +82,7 @@ static int call_modprobe(char *orig_module_name, int wait)
 	char *module_name;
 	int ret;
 
-	char **argv = kmalloc(sizeof(char *[5]), GFP_KERNEL);
+	char **argv = kmalloc_array(5, sizeof(*argv), GFP_KERNEL);
 	if (!argv)
 		goto out;
 
diff --git a/kernel/module/main.c b/kernel/module/main.c
index d0e1e0bd2ad06b..0d720b33d648c2 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -191,11 +191,22 @@ static inline int strong_try_module_get(struct module *mod)
 		return -ENOENT;
 }
 
-static inline void add_taint_module(struct module *mod, unsigned flag,
-				    enum lockdep_ok lockdep_ok)
+/**
+ * add_taint_module: add a taint flag if not already set for a specific module
+ * @mod: pointer to the module that caused the problem
+ * @flag: one of the TAINT_* constants.
+ * @lockdep_ok: whether lock debugging is still OK.
+ *
+ * If something bad has gone wrong, you'll want
+ * @lockdep_ok = LOCKDEP_NOW_UNRELIABLE, but for some
+ * noteworthy-but-not-corrupting cases, it can be set to LOCKDEP_STILL_OK.
+ */
+void add_taint_module(struct module *mod, unsigned flag,
+		      enum lockdep_ok lockdep_ok)
 {
 	add_taint(flag, lockdep_ok);
-	set_bit(flag, &mod->taints);
+	if (mod)
+		set_bit(flag, &mod->taints);
 }
 
 /*
@@ -3581,8 +3592,9 @@ static int load_module(struct load_info *info, const char __user *uargs,
 			goto sysfs_cleanup;
 	}
 
-	if (codetag_load_module(mod))
-		goto sysfs_cleanup;
+	err = codetag_load_module(mod);
+	if (err)
+		goto livepatch_cleanup;
 
 	/* Get rid of temporary copy. */
 	free_copy(info, flags);
@@ -3592,6 +3604,9 @@ static int load_module(struct load_info *info, const char __user *uargs,
 
 	return do_init_module(mod);
 
+ livepatch_cleanup:
+	if (is_livepatch_module(mod))
+		free_module_elf(mod);
  sysfs_cleanup:
 	mod_sysfs_teardown(mod);
  coming_cleanup:
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 2f9fe7c30287f4..55fbd2be7407bb 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
+#include <linux/device/devres.h>
 #include <linux/kdebug.h>
 #include <linux/kprobes.h>
 #include <linux/export.h>
@@ -197,6 +198,58 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
 }
 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
 
+struct atomic_notifier_chain_devres {
+	struct atomic_notifier_head *nh;
+	struct notifier_block *nb;
+};
+
+static void devm_atomic_notifier_chain_unregister(struct device *dev, void *res)
+{
+	struct atomic_notifier_chain_devres *dr = res;
+
+	atomic_notifier_chain_unregister(dr->nh, dr->nb);
+}
+
+/**
+ *	devm_atomic_notifier_chain_register - Device-managed atomic notifier registration
+ *	@dev: Device to tie the notifier lifetime to
+ *	@nh: Pointer to head of the atomic notifier chain
+ *	@nb: New entry in notifier chain
+ *
+ *	Adds a notifier to an atomic notifier chain and registers a cleanup
+ *	action to automatically unregister it when @dev is unbound.
+ *
+ *	Must be called in process context.
+ *
+ *	Return:
+ *	0 on success, negative errno on error.
+ */
+int devm_atomic_notifier_chain_register(struct device *dev,
+					struct atomic_notifier_head *nh,
+					struct notifier_block *nb)
+{
+	struct atomic_notifier_chain_devres *dr;
+	int ret;
+
+	dr = devres_alloc(devm_atomic_notifier_chain_unregister,
+			  sizeof(*dr), GFP_KERNEL);
+	if (!dr)
+		return -ENOMEM;
+
+	ret = atomic_notifier_chain_register(nh, nb);
+	if (ret) {
+		devres_free(dr);
+		return ret;
+	}
+
+	dr->nh = nh;
+	dr->nb = nb;
+	devres_add(dev, dr);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(devm_atomic_notifier_chain_register);
+
 /**
  *	atomic_notifier_call_chain - Call functions in an atomic notifier chain
  *	@nh: Pointer to head of the atomic notifier chain
@@ -349,6 +402,58 @@ int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
 }
 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
 
+struct blocking_notifier_chain_devres {
+	struct blocking_notifier_head *nh;
+	struct notifier_block *nb;
+};
+
+static void devm_blocking_notifier_chain_unregister(struct device *dev,
+						    void *res)
+{
+	struct blocking_notifier_chain_devres *dr = res;
+
+	blocking_notifier_chain_unregister(dr->nh, dr->nb);
+}
+
+/**
+ *	devm_blocking_notifier_chain_register - Device-managed blocking notifier registration
+ *	@dev: Device to tie the notifier lifetime to
+ *	@nh: Pointer to head of the blocking notifier chain
+ *	@nb: New entry in notifier chain
+ *
+ *	Adds a notifier to a blocking notifier chain and registers a cleanup
+ *	action to automatically unregister it when @dev is unbound.
+ *	Must be called in process context.
+ *
+ *	Return:
+ *	0 on success, negative errno on error.
+ */
+int devm_blocking_notifier_chain_register(struct device *dev,
+					  struct blocking_notifier_head *nh,
+					  struct notifier_block *nb)
+{
+	struct blocking_notifier_chain_devres *dr;
+	int ret;
+
+	dr = devres_alloc(devm_blocking_notifier_chain_unregister,
+			  sizeof(*dr), GFP_KERNEL);
+	if (!dr)
+		return -ENOMEM;
+
+	ret = blocking_notifier_chain_register(nh, nb);
+	if (ret) {
+		devres_free(dr);
+		return ret;
+	}
+
+	dr->nh = nh;
+	dr->nb = nb;
+	devres_add(dev, dr);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(devm_blocking_notifier_chain_register);
+
 /**
  *	blocking_notifier_call_chain - Call functions in a blocking notifier chain
  *	@nh: Pointer to head of the blocking notifier chain
diff --git a/kernel/nscommon.c b/kernel/nscommon.c
index 3166c1fd844afd..e72426bba29ab1 100644
--- a/kernel/nscommon.c
+++ b/kernel/nscommon.c
@@ -4,6 +4,7 @@
 #include <linux/ns_common.h>
 #include <linux/nstree.h>
 #include <linux/proc_ns.h>
+#include <linux/security.h>
 #include <linux/user_namespace.h>
 #include <linux/vfsdebug.h>
 
@@ -59,6 +60,9 @@ int __ns_common_init(struct ns_common *ns, u32 ns_type, const struct proc_ns_ope
 
 	refcount_set(&ns->__ns_ref, 1);
 	ns->stashed = NULL;
+#ifdef CONFIG_SECURITY
+	ns->ns_security = NULL;
+#endif
 	ns->ops = ops;
 	ns->ns_id = 0;
 	ns->ns_type = ns_type;
@@ -77,6 +81,14 @@ int __ns_common_init(struct ns_common *ns, u32 ns_type, const struct proc_ns_ope
 		ret = proc_alloc_inum(&ns->inum);
 	if (ret)
 		return ret;
+
+	ret = security_namespace_init(ns);
+	if (ret) {
+		if (!inum)
+			proc_free_inum(ns->inum);
+		return ret;
+	}
+
 	/*
 	 * Tree ref starts at 0. It's incremented when namespace enters
 	 * active use (installed in nsproxy) and decremented when all
@@ -91,7 +103,10 @@ int __ns_common_init(struct ns_common *ns, u32 ns_type, const struct proc_ns_ope
 
 void __ns_common_free(struct ns_common *ns)
 {
-	proc_free_inum(ns->inum);
+	security_namespace_free(ns);
+
+	if (ns->inum > MNT_NS_INO_SPECIAL_MAX)
+		proc_free_inum(ns->inum);
 }
 
 struct ns_common *__must_check ns_owner(struct ns_common *ns)
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c
index d9d3d5973bf523..0f1b208d8eeff1 100644
--- a/kernel/nsproxy.c
+++ b/kernel/nsproxy.c
@@ -385,6 +385,12 @@ static int prepare_nsset(unsigned flags, struct nsset *nsset)
 
 static inline int validate_ns(struct nsset *nsset, struct ns_common *ns)
 {
+	int ret;
+
+	ret = security_namespace_install(nsset, ns);
+	if (ret)
+		return ret;
+
 	return ns->ops->install(nsset, ns);
 }
 
diff --git a/kernel/padata.c b/kernel/padata.c
index 0d3ea1b68b1f70..5a113c667d4cbb 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -534,7 +534,7 @@ static void padata_init_reorder_list(struct parallel_data *pd)
 	}
 }
 
-/* Allocate and initialize the internal cpumask dependend resources. */
+/* Allocate and initialize the internal cpumask dependent resources. */
 static struct parallel_data *padata_alloc_pd(struct padata_shell *ps,
 					     int offlining_cpu)
 {
diff --git a/kernel/panic.c b/kernel/panic.c
index 213725b612aa11..b824b68fcb084e 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -307,7 +307,7 @@ atomic_t panic_cpu = ATOMIC_INIT(PANIC_CPU_INVALID);
 atomic_t panic_redirect_cpu = ATOMIC_INIT(PANIC_CPU_INVALID);
 
 #if defined(CONFIG_SMP) && defined(CONFIG_CRASH_DUMP)
-static char *panic_force_buf;
+static char panic_force_buf[PANIC_MSG_BUFSZ];
 
 static int __init panic_force_cpu_setup(char *str)
 {
@@ -326,17 +326,6 @@ static int __init panic_force_cpu_setup(char *str)
 }
 early_param("panic_force_cpu", panic_force_cpu_setup);
 
-static int __init panic_force_cpu_late_init(void)
-{
-	if (panic_force_cpu < 0)
-		return 0;
-
-	panic_force_buf = kmalloc(PANIC_MSG_BUFSZ, GFP_KERNEL);
-
-	return 0;
-}
-late_initcall(panic_force_cpu_late_init);
-
 static void do_panic_on_target_cpu(void *info)
 {
 	panic("%s", (char *)info);
@@ -371,15 +360,16 @@ int __weak panic_smp_redirect_cpu(int target_cpu, void *msg)
  * for the crash kernel to function correctly. This function redirects
  * panic handling to the CPU specified via the panic_force_cpu= boot parameter.
  *
- * Returns false if panic should proceed on current CPU.
- * Returns true if panic was redirected.
+ * Returns true when this CPU must stop: the panic was redirected or is
+ * already running on another CPU.
+ * Returns false when panic() should proceed on this CPU.
  */
 __printf(1, 0)
 static bool panic_try_force_cpu(const char *fmt, va_list args)
 {
 	int this_cpu = raw_smp_processor_id();
 	int old_cpu = PANIC_CPU_INVALID;
-	const char *msg;
+	va_list ap;
 
 	/* Feature not enabled via boot parameter */
 	if (panic_force_cpu < 0)
@@ -396,27 +386,27 @@ static bool panic_try_force_cpu(const char *fmt, va_list args)
 		return false;
 	}
 
-	/* Another panic already in progress */
+	/*
+	 * Don't redirect when a panic is already in progress. Stop this
+	 * CPU when it's another one, proceed when it's this one.
+	 */
 	if (panic_in_progress())
-		return false;
+		return panic_on_other_cpu();
 
 	/*
-	 * Only one CPU can do the redirect. Use atomic cmpxchg to ensure
-	 * we don't race with another CPU also trying to redirect.
+	 * Only one CPU can do the redirection. Others should go offline.
+	 * Continue with panic() when we already tried the redirection
+	 * from this CPU before, for example via nmi_panic().
 	 */
 	if (!atomic_try_cmpxchg(&panic_redirect_cpu, &old_cpu, this_cpu))
-		return false;
+		return old_cpu != this_cpu;
 
 	/*
-	 * Use dynamically allocated buffer if available, otherwise
-	 * fall back to static message for early boot panics or allocation failure.
+	 * Do not consume args, the caller reuses them if we fail.
 	 */
-	if (panic_force_buf) {
-		vsnprintf(panic_force_buf, PANIC_MSG_BUFSZ, fmt, args);
-		msg = panic_force_buf;
-	} else {
-		msg = "Redirected panic (buffer unavailable)";
-	}
+	va_copy(ap, args);
+	vsnprintf(panic_force_buf, PANIC_MSG_BUFSZ, fmt, ap);
+	va_end(ap);
 
 	console_verbose();
 	bust_spinlocks(1);
@@ -431,7 +421,7 @@ static bool panic_try_force_cpu(const char *fmt, va_list args)
 		dump_stack();
 	}
 
-	if (panic_smp_redirect_cpu(panic_force_cpu, (void *)msg) != 0) {
+	if (panic_smp_redirect_cpu(panic_force_cpu, panic_force_buf) != 0) {
 		atomic_set(&panic_redirect_cpu, PANIC_CPU_INVALID);
 		pr_warn("panic: failed to redirect to CPU %d, continuing on CPU %d\n",
 			panic_force_cpu, this_cpu);
@@ -503,17 +493,35 @@ bool panic_on_other_cpu(void)
 EXPORT_SYMBOL(panic_on_other_cpu);
 
 /*
- * A variant of panic() called from NMI context. We return if we've already
- * panicked on this CPU. If another CPU already panicked, loop in
- * nmi_panic_self_stop() which can provide architecture dependent code such
- * as saving register state for crash dump.
+ * A variant of panic() called from NMI context. The panic is first
+ * redirected to the CPU requested via panic_force_cpu=, when configured.
+ * We return if we've already panicked on this CPU. If another CPU already
+ * panicked, loop in nmi_panic_self_stop() which can provide architecture
+ * dependent code for saving register state for crash dump.
  */
-void nmi_panic(struct pt_regs *regs, const char *msg)
+void nmi_panic(struct pt_regs *regs, const char *fmt, ...)
 {
+	va_list args;
+
+	va_start(args, fmt);
+
+	/* Try to redirect to the requested CPU before claiming panic_cpu. */
+	if (panic_try_force_cpu(fmt, args)) {
+		/*
+		 * Mark ourselves offline so panic_other_cpus_shutdown() won't
+		 * wait for us on architectures that check num_online_cpus().
+		 */
+		set_cpu_online(raw_smp_processor_id(), false);
+		nmi_panic_self_stop(regs);
+	}
+
 	if (panic_try_start())
-		panic("%s", msg);
-	else if (panic_on_other_cpu())
+		vpanic(fmt, args);
+
+	if (panic_on_other_cpu())
 		nmi_panic_self_stop(regs);
+
+	va_end(args);
 }
 EXPORT_SYMBOL(nmi_panic);
 
@@ -567,6 +575,8 @@ static void panic_other_cpus_shutdown(bool crash_kexec)
 		crash_smp_send_stop();
 }
 
+void __weak arch_do_panic(void) {}
+
 /**
  * vpanic - halt the system
  * @fmt: The text string to print
@@ -742,20 +752,9 @@ void vpanic(const char *fmt, va_list args)
 			reboot_mode = panic_reboot_mode;
 		emergency_restart();
 	}
-#ifdef __sparc__
-	{
-		extern int stop_a_enabled;
-		/* Make sure the user can actually press Stop-A (L1-A) */
-		stop_a_enabled = 1;
-		pr_emerg("Press Stop-A (L1-A) from sun keyboard or send break\n"
-			 "twice on console to return to the boot prom\n");
-	}
-#endif
-#if defined(CONFIG_S390)
-	disabled_wait();
-#endif
 	pr_emerg("---[ end Kernel panic - not syncing: %s ]---\n", buf);
 
+	arch_do_panic();
 	/* Do not scroll important messages printed above */
 	suppress_printk = 1;
 
@@ -826,6 +825,7 @@ const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = {
 	TAINT_FLAG(RANDSTRUCT,			'T', ' '),
 	TAINT_FLAG(TEST,			'N', ' '),
 	TAINT_FLAG(FWCTL,			'J', ' '),
+	TAINT_FLAG(FORCED_BIND,			'Y', ' '),
 };
 
 #undef TAINT_FLAG
@@ -862,9 +862,9 @@ static void print_tainted_seq(struct seq_buf *s, bool verbose)
  * exact size is allocated dynamically; the initial buffer remains
  * as a fallback if allocation fails.
  *
- * The verbose taint string currently requires up to 327 characters.
+ * The verbose taint string currently requires up to 344 characters.
  */
-#define INIT_TAINT_BUF_MAX 350
+#define INIT_TAINT_BUF_MAX 370
 
 static char init_taint_buf[INIT_TAINT_BUF_MAX] __initdata;
 static char *taint_buf __refdata = init_taint_buf;
@@ -948,8 +948,9 @@ unsigned long get_taint(void)
  * @flag: one of the TAINT_* constants.
  * @lockdep_ok: whether lock debugging is still OK.
  *
- * If something bad has gone wrong, you'll want @lockdebug_ok = false, but for
- * some notewortht-but-not-corrupting cases, it can be set to true.
+ * If something bad has gone wrong, you'll want
+ * @lockdep_ok = LOCKDEP_NOW_UNRELIABLE, but for some
+ * noteworthy-but-not-corrupting cases, it can be set to LOCKDEP_STILL_OK.
  */
 void add_taint(unsigned flag, enum lockdep_ok lockdep_ok)
 {
@@ -1225,14 +1226,9 @@ static int panic_print_set(const char *val, const struct kernel_param *kp)
 	return  param_set_ulong(val, kp);
 }
 
-static int panic_print_get(char *val, const struct kernel_param *kp)
-{
-	return  param_get_ulong(val, kp);
-}
-
 static const struct kernel_param_ops panic_print_ops = {
 	.set	= panic_print_set,
-	.get	= panic_print_get,
+	.get	= param_get_ulong,
 };
 __core_param_cb(panic_print, &panic_print_ops, &panic_print, 0644);
 
diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c
index d36afc58ee1db7..8bc9edb40b7822 100644
--- a/kernel/pid_namespace.c
+++ b/kernel/pid_namespace.c
@@ -238,9 +238,10 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns)
 	 * kernel_wait4() will also block until our children traced from the
 	 * parent namespace are detached and become EXIT_DEAD.
 	 */
+	/* Task work must not busy-loop the reaper, see signal_pending(). */
+	guard(no_notify_signal)();
 	do {
 		clear_thread_flag(TIF_SIGPENDING);
-		clear_thread_flag(TIF_NOTIFY_SIGNAL);
 		rc = kernel_wait4(-1, NULL, __WALL, NULL);
 	} while (rc != -ECHILD);
 
diff --git a/kernel/power/em_netlink.c b/kernel/power/em_netlink.c
index 4d4fd29bd2be9c..1f68d456a713c0 100644
--- a/kernel/power/em_netlink.c
+++ b/kernel/power/em_netlink.c
@@ -95,41 +95,62 @@ static int __em_nl_get_pd_for_dump(struct em_perf_domain *pd, void *data)
 	return ret;
 }
 
+struct em_nl_doit_ctx {
+	struct genl_info *info;
+	int cmd;
+	struct sk_buff *msg;
+};
+
+static int __em_nl_get_pd_doit_fill(struct em_perf_domain *pd, void *data)
+{
+	struct em_nl_doit_ctx *ctx = data;
+	void *hdr;
+
+	hdr = genlmsg_put_reply(ctx->msg, ctx->info, &dev_energymodel_nl_family, 0,
+				ctx->cmd);
+	if (!hdr)
+		return -EMSGSIZE;
+
+	if (__em_nl_get_pd(pd, ctx->msg)) {
+		genlmsg_cancel(ctx->msg, hdr);
+		return -EMSGSIZE;
+	}
+
+	genlmsg_end(ctx->msg, hdr);
+	return 0;
+}
+
 int dev_energymodel_nl_get_perf_domains_doit(struct sk_buff *skb,
-					      struct genl_info *info)
+					     struct genl_info *info)
 {
-	int id, ret = -EMSGSIZE, msg_sz = 0;
-	int cmd = info->genlhdr->cmd;
-	struct em_perf_domain *pd;
+	struct em_nl_doit_ctx ctx = {
+		.info = info,
+		.cmd = info->genlhdr->cmd,
+	};
 	struct sk_buff *msg;
-	void *hdr;
+	int id, ret, msg_sz = 0;
 
 	if (!info->attrs[DEV_ENERGYMODEL_A_PERF_DOMAIN_PERF_DOMAIN_ID])
 		return -EINVAL;
 
 	id = nla_get_u32(info->attrs[DEV_ENERGYMODEL_A_PERF_DOMAIN_PERF_DOMAIN_ID]);
-	pd = em_perf_domain_get_by_id(id);
-	if (!pd)
-		return -EINVAL;
 
-	__em_nl_get_pd_size(pd, &msg_sz);
+	/* Encode under em_pd_list_mutex, like the dumpit path. */
+	ret = em_perf_domain_for_id(id, __em_nl_get_pd_size, &msg_sz);
+	if (ret)
+		return ret;
+
 	msg = genlmsg_new(msg_sz, GFP_KERNEL);
 	if (!msg)
 		return -ENOMEM;
 
-	hdr = genlmsg_put_reply(msg, info, &dev_energymodel_nl_family, 0, cmd);
-	if (!hdr)
-		goto out_free_msg;
-
-	ret = __em_nl_get_pd(pd, msg);
+	ctx.msg = msg;
+	ret = em_perf_domain_for_id(id, __em_nl_get_pd_doit_fill, &ctx);
 	if (ret)
-		goto out_cancel_msg;
-	genlmsg_end(msg, hdr);
+		goto out_free_msg;
 
 	return genlmsg_reply(msg, info);
 
-out_cancel_msg:
-	genlmsg_cancel(msg, hdr);
 out_free_msg:
 	nlmsg_free(msg);
 	return ret;
@@ -148,19 +169,6 @@ int dev_energymodel_nl_get_perf_domains_dumpit(struct sk_buff *skb,
 	return for_each_em_perf_domain(__em_nl_get_pd_for_dump, &ctx);
 }
 
-static struct em_perf_domain *__em_nl_get_pd_table_id(struct nlattr **attrs)
-{
-	struct em_perf_domain *pd;
-	int id;
-
-	if (!attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID])
-		return NULL;
-
-	id = nla_get_u32(attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID]);
-	pd = em_perf_domain_get_by_id(id);
-	return pd;
-}
-
 static int __em_nl_get_pd_table_size(const struct em_perf_domain *pd)
 {
 	int id_sz, ps_sz;
@@ -245,34 +253,58 @@ int __em_nl_get_pd_table(struct sk_buff *msg, const struct em_perf_domain *pd)
 	return -EMSGSIZE;
 }
 
+static int __em_nl_get_pd_table_size_cb(struct em_perf_domain *pd, void *data)
+{
+	*(int *)data = __em_nl_get_pd_table_size(pd);
+	return 0;
+}
+
+static int __em_nl_get_pd_table_doit_fill(struct em_perf_domain *pd,
+					  void *data)
+{
+	struct em_nl_doit_ctx *ctx = data;
+	void *hdr;
+
+	hdr = genlmsg_put_reply(ctx->msg, ctx->info, &dev_energymodel_nl_family, 0,
+				ctx->cmd);
+	if (!hdr)
+		return -EMSGSIZE;
+
+	if (__em_nl_get_pd_table(ctx->msg, pd))
+		return -EMSGSIZE;
+
+	genlmsg_end(ctx->msg, hdr);
+	return 0;
+}
+
 int dev_energymodel_nl_get_perf_table_doit(struct sk_buff *skb,
-					    struct genl_info *info)
+					   struct genl_info *info)
 {
-	int cmd = info->genlhdr->cmd;
-	int msg_sz, ret = -EMSGSIZE;
-	struct em_perf_domain *pd;
+	struct em_nl_doit_ctx ctx = {
+		.info = info,
+		.cmd = info->genlhdr->cmd,
+	};
 	struct sk_buff *msg;
-	void *hdr;
+	int id, ret, msg_sz;
 
-	pd = __em_nl_get_pd_table_id(info->attrs);
-	if (!pd)
+	if (!info->attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID])
 		return -EINVAL;
 
-	msg_sz = __em_nl_get_pd_table_size(pd);
+	id = nla_get_u32(info->attrs[DEV_ENERGYMODEL_A_PERF_TABLE_PERF_DOMAIN_ID]);
+
+	ret = em_perf_domain_for_id(id, __em_nl_get_pd_table_size_cb, &msg_sz);
+	if (ret)
+		return ret;
 
 	msg = genlmsg_new(msg_sz, GFP_KERNEL);
 	if (!msg)
 		return -ENOMEM;
 
-	hdr = genlmsg_put_reply(msg, info, &dev_energymodel_nl_family, 0, cmd);
-	if (!hdr)
-		goto out_free_msg;
-
-	ret = __em_nl_get_pd_table(msg, pd);
+	ctx.msg = msg;
+	ret = em_perf_domain_for_id(id, __em_nl_get_pd_table_doit_fill, &ctx);
 	if (ret)
 		goto out_free_msg;
 
-	genlmsg_end(msg, hdr);
 	return genlmsg_reply(msg, info);
 
 out_free_msg:
diff --git a/kernel/power/em_netlink.h b/kernel/power/em_netlink.h
index 583d7f1c39390d..bc98a3c278b9fe 100644
--- a/kernel/power/em_netlink.h
+++ b/kernel/power/em_netlink.h
@@ -12,7 +12,8 @@
 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_NET)
 int for_each_em_perf_domain(int (*cb)(struct em_perf_domain*, void *),
 			    void *data);
-struct em_perf_domain *em_perf_domain_get_by_id(int id);
+int em_perf_domain_for_id(int id, int (*cb)(struct em_perf_domain *, void *),
+			  void *data);
 void em_notify_pd_created(const struct em_perf_domain *pd);
 void em_notify_pd_deleted(const struct em_perf_domain *pd);
 void em_notify_pd_updated(const struct em_perf_domain *pd);
@@ -24,9 +25,10 @@ int for_each_em_perf_domain(int (*cb)(struct em_perf_domain*, void *),
 	return -EINVAL;
 }
 static inline
-struct em_perf_domain *em_perf_domain_get_by_id(int id)
+int em_perf_domain_for_id(int id, int (*cb)(struct em_perf_domain *, void *),
+			  void *data)
 {
-	return NULL;
+	return -EINVAL;
 }
 
 static inline void em_notify_pd_created(const struct em_perf_domain *pd) {}
diff --git a/kernel/power/energy_model.c b/kernel/power/energy_model.c
index e610cf8e9a06c7..a76089e2ce8c44 100644
--- a/kernel/power/energy_model.c
+++ b/kernel/power/energy_model.c
@@ -1031,7 +1031,9 @@ int for_each_em_perf_domain(int (*cb)(struct em_perf_domain*, void *),
 	return 0;
 }
 
-struct em_perf_domain *em_perf_domain_get_by_id(int id)
+/* Run @cb on the matching domain with em_pd_list_mutex held. */
+int em_perf_domain_for_id(int id, int (*cb)(struct em_perf_domain *, void *),
+			  void *data)
 {
 	struct em_perf_domain *pd;
 
@@ -1040,9 +1042,9 @@ struct em_perf_domain *em_perf_domain_get_by_id(int id)
 
 	list_for_each_entry(pd, &em_pd_list, node) {
 		if (pd->id == id)
-			return pd;
+			return cb(pd, data);
 	}
 
-	return NULL;
+	return -EINVAL;
 }
 #endif
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c
index d2479c69d71a4c..c13f68ab7f6e05 100644
--- a/kernel/power/hibernate.c
+++ b/kernel/power/hibernate.c
@@ -408,9 +408,18 @@ int hibernation_snapshot(int platform_mode)
 	if (error)
 		goto Close;
 
+	error = dpm_prepare(PMSG_FREEZE);
+	if (error)
+		goto Complete;
+
+	/* Preallocate image memory before freezing kernel threads and shutting down devices. */
+	error = hibernate_preallocate_memory();
+	if (error)
+		goto Complete;
+
 	error = freeze_kernel_threads();
 	if (error)
-		goto Close;
+		goto Cleanup;
 
 	if (hibernation_test(TEST_FREEZER)) {
 
@@ -422,15 +431,6 @@ int hibernation_snapshot(int platform_mode)
 		goto Thaw;
 	}
 
-	error = dpm_prepare(PMSG_FREEZE);
-	if (error)
-		goto Complete;
-
-	/* Preallocate image memory before shutting down devices. */
-	error = hibernate_preallocate_memory();
-	if (error)
-		goto Complete;
-
 	console_suspend_all();
 	pm_restrict_gfp_mask();
 
@@ -464,10 +464,12 @@ int hibernation_snapshot(int platform_mode)
 	platform_end(platform_mode);
 	return error;
 
- Complete:
-	dpm_complete(PMSG_RECOVER);
  Thaw:
 	thaw_kernel_threads();
+ Cleanup:
+	swsusp_free();
+ Complete:
+	dpm_complete(PMSG_RECOVER);
 	goto Close;
 }
 
diff --git a/kernel/power/process.c b/kernel/power/process.c
index dc0dfc349f22b4..295904ec9a8207 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -16,7 +16,7 @@
 #include <linux/freezer.h>
 #include <linux/delay.h>
 #include <linux/workqueue.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <trace/events/power.h>
 #include <linux/cpuset.h>
 
diff --git a/kernel/power/qos.c b/kernel/power/qos.c
index 1944dbeb0d4c95..c241da5bccf690 100644
--- a/kernel/power/qos.c
+++ b/kernel/power/qos.c
@@ -159,7 +159,7 @@ static void pm_qos_flags_remove_req(struct pm_qos_flags *pqf,
 	list_for_each_entry(req, &pqf->list, node)
 		val |= req->flags;
 
-	pqf->effective_flags = val;
+	WRITE_ONCE(pqf->effective_flags, val);
 }
 
 /**
@@ -193,7 +193,7 @@ bool pm_qos_update_flags(struct pm_qos_flags *pqf,
 		req->flags = val;
 		INIT_LIST_HEAD(&req->node);
 		list_add_tail(&req->node, &pqf->list);
-		pqf->effective_flags |= val;
+		WRITE_ONCE(pqf->effective_flags, pqf->effective_flags | val);
 		break;
 	default:
 		/* no action */
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index b209712cb2c3ac..423b41c968e54b 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -13,6 +13,7 @@
 #include <linux/version.h>
 #include <linux/module.h>
 #include <linux/mm.h>
+#include <linux/swap.h>
 #include <linux/suspend.h>
 #include <linux/delay.h>
 #include <linux/bitops.h>
@@ -88,7 +89,7 @@ static inline int hibernate_restore_unprotect_page(void *page_address) {return 0
 static inline void hibernate_map_page(struct page *page)
 {
 	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {
-		int ret = set_direct_map_default_noflush(page);
+		int ret = set_direct_map_default_noflush(page, 1);
 
 		if (ret)
 			pr_warn_once("Failed to remap page\n");
@@ -101,7 +102,7 @@ static inline void hibernate_unmap_page(struct page *page)
 {
 	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {
 		unsigned long addr = (unsigned long)page_address(page);
-		int ret  = set_direct_map_invalid_noflush(page);
+		int ret  = set_direct_map_invalid_noflush(page, 1);
 
 		if (ret)
 			pr_warn_once("Failed to remap page\n");
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 6d3d18a50da741..6c8bacb173b7f2 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -1085,9 +1085,9 @@ static void __init log_buf_add_cpu(void)
 	if (num_possible_cpus() == 1)
 		return;
 
-	cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN;
+	cpu_extra = num_possible_cpus() * __LOG_CPU_MAX_BUF_LEN;
 
-	/* by default this will only continue through for large > 64 CPUs */
+	/* by default this will only continue through for more than 16 CPUs */
 	if (cpu_extra <= __LOG_BUF_LEN / 2)
 		return;
 
@@ -4424,36 +4424,51 @@ void __init console_init(void)
  * be a brief interval in which no messages are logged to the console, which
  * makes it difficult to diagnose problems that occur during this time.
  *
- * To mitigate this problem somewhat, only unregister consoles whose memory
- * intersects with the init section. Note that all other boot consoles will
- * get unregistered when the real preferred console is registered.
+ * If a real console has already registered, remove all remaining boot consoles.
+ * Otherwise, mitigate the no-console interval by removing only boot consoles
+ * whose memory intersects with the init section.
  */
 static int __init printk_late_init(void)
 {
+	bool have_real_console = false;
 	struct hlist_node *tmp;
 	struct console *con;
 	int ret;
 
 	console_list_lock();
+	for_each_console(con) {
+		if (!(con->flags & CON_BOOT)) {
+			have_real_console = true;
+			break;
+		}
+	}
+
 	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
 		if (!(con->flags & CON_BOOT))
 			continue;
 
-		/* Check addresses that might be used for enabled consoles. */
-		if (init_section_intersects(con, sizeof(*con)) ||
-		    init_section_contains(con->write, 0) ||
-		    init_section_contains(con->read, 0) ||
-		    init_section_contains(con->device, 0) ||
-		    init_section_contains(con->unblank, 0) ||
-		    init_section_contains(con->data, 0)) {
+		/*
+		 * Keep the boot console when requested or as a fallback
+		 * unless it is using an init section.
+		 */
+		if (keep_bootcon || !have_real_console) {
+			if (!init_section_intersects(con, sizeof(*con)) &&
+			    !init_section_contains(con->write, 0) &&
+			    !init_section_contains(con->read, 0) &&
+			    !init_section_contains(con->device, 0) &&
+			    !init_section_contains(con->unblank, 0) &&
+			    !init_section_contains(con->data, 0))
+				continue;
+
 			/*
 			 * Please, consider moving the reported consoles out
 			 * of the init section.
 			 */
 			pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n",
 				con->name, con->index);
-			unregister_console_locked(con);
 		}
+
+		unregister_console_locked(con);
 	}
 	console_list_unlock();
 
@@ -4593,31 +4608,26 @@ bool pr_flush(int timeout_ms, bool reset_on_progress)
 /*
  * Delayed printk version, for scheduler-internal messages:
  */
-#define PRINTK_PENDING_WAKEUP	0x01
-#define PRINTK_PENDING_OUTPUT	0x02
-
-static DEFINE_PER_CPU(int, printk_pending);
+#define PRINTK_PENDING_OUTPUT	0x01
 
-static void wake_up_klogd_work_func(struct irq_work *irq_work)
+static void pending_wake_fn(struct irq_work *irq_work)
 {
-	int pending = this_cpu_xchg(printk_pending, 0);
+	wake_up_interruptible(&log_wait);
+}
 
-	if (pending & PRINTK_PENDING_OUTPUT) {
-		if (force_legacy_kthread()) {
-			if (printk_legacy_kthread)
-				wake_up_interruptible(&legacy_wait);
-		} else {
-			if (console_trylock())
-				console_unlock();
-		}
+static void pending_output_fn(struct irq_work *irq_work)
+{
+	if (force_legacy_kthread()) {
+		if (printk_legacy_kthread)
+			wake_up_interruptible(&legacy_wait);
+	} else {
+		if (console_trylock())
+			console_unlock();
 	}
-
-	if (pending & PRINTK_PENDING_WAKEUP)
-		wake_up_interruptible(&log_wait);
 }
 
-static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
-	IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);
+static struct irq_work pending_wakeup_work = IRQ_WORK_INIT_LAZY(pending_wake_fn);
+static struct irq_work pending_output_work = IRQ_WORK_INIT_LAZY(pending_output_fn);
 
 static void __wake_up_klogd(int val)
 {
@@ -4631,7 +4641,6 @@ static void __wake_up_klogd(int val)
 	if (WARN_ON_ONCE(console_irqwork_blocked))
 		return;
 
-	preempt_disable();
 	/*
 	 * Guarantee any new records can be seen by tasks preparing to wait
 	 * before this context checks if the wait queue is empty.
@@ -4643,12 +4652,11 @@ static void __wake_up_klogd(int val)
 	 *
 	 * This pairs with devkmsg_read:A and syslog_print:A.
 	 */
-	if (wq_has_sleeper(&log_wait) || /* LMM(__wake_up_klogd:A) */
-	    (val & PRINTK_PENDING_OUTPUT)) {
-		this_cpu_or(printk_pending, val);
-		irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
-	}
-	preempt_enable();
+	if (wq_has_sleeper(&log_wait)) /* LMM(__wake_up_klogd:A) */
+		irq_work_queue(&pending_wakeup_work);
+
+	if (val & PRINTK_PENDING_OUTPUT)
+		irq_work_queue(&pending_output_work);
 }
 
 /**
@@ -4663,7 +4671,7 @@ static void __wake_up_klogd(int val)
  */
 void wake_up_klogd(void)
 {
-	__wake_up_klogd(PRINTK_PENDING_WAKEUP);
+	__wake_up_klogd(0);
 }
 
 /**
@@ -4684,7 +4692,7 @@ void defer_console_output(void)
 	 * New messages may have been added directly to the ringbuffer
 	 * using vprintk_store(), so wake any waiters as well.
 	 */
-	__wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
+	__wake_up_klogd(PRINTK_PENDING_OUTPUT);
 }
 
 /**
diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c
index 85c0c854b3ce2c..d38e7c2f688760 100644
--- a/kernel/printk/printk_ringbuffer.c
+++ b/kernel/printk/printk_ringbuffer.c
@@ -439,13 +439,12 @@ static enum desc_state get_desc_state(unsigned long id,
 }
 
 /*
- * Get a copy of a specified descriptor and return its queried state. If the
- * descriptor is in an inconsistent state (miss or reserved), the caller can
- * only expect the descriptor's @state_var field to be valid.
+ * Get the state of a specified descriptor.
  *
- * The sequence number and caller_id can be optionally retrieved. Like all
- * non-state_var data, they are only valid if the descriptor is in a
- * consistent state.
+ * Optionally, a copy of the descriptor, sequence number and/or caller ID can
+ * be retrieved. However, (with the exception of @desc_out->state_var) all
+ * retrieved values are invalid if the descriptor is in an inconsistent state
+ * (miss or reserved).
  */
 static enum desc_state desc_read(struct prb_desc_ring *desc_ring,
 				 unsigned long id, struct prb_desc *desc_out,
@@ -1335,7 +1334,6 @@ static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring,
 {
 	unsigned long prev_state_val;
 	enum desc_state d_state;
-	struct prb_desc desc;
 	struct prb_desc *d;
 	unsigned long id;
 	u32 cid;
@@ -1346,7 +1344,7 @@ static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring,
 	 * To reduce unnecessarily reopening, first check if the descriptor
 	 * state and caller ID are correct.
 	 */
-	d_state = desc_read(desc_ring, id, &desc, NULL, &cid);
+	d_state = desc_read(desc_ring, id, NULL, NULL, &cid);
 	if (d_state != desc_committed || cid != caller_id)
 		return NULL;
 
@@ -2007,14 +2005,13 @@ u64 prb_first_seq(struct printk_ringbuffer *rb)
 {
 	struct prb_desc_ring *desc_ring = &rb->desc_ring;
 	enum desc_state d_state;
-	struct prb_desc desc;
 	unsigned long id;
 	u64 seq;
 
 	for (;;) {
 		id = atomic_long_read(&rb->desc_ring.tail_id); /* LMM(prb_first_seq:A) */
 
-		d_state = desc_read(desc_ring, id, &desc, &seq, NULL); /* LMM(prb_first_seq:B) */
+		d_state = desc_read(desc_ring, id, NULL, &seq, NULL); /* LMM(prb_first_seq:B) */
 
 		/*
 		 * This loop will not be infinite because the tail is
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index d041645d9d17da..4e9822a87aabbc 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -1227,6 +1227,12 @@ int ptrace_request(struct task_struct *child, long request,
 	case PTRACE_SETSIGMASK: {
 		sigset_t new_set;
 
+		/* A user worker only ever takes SIGKILL and SIGSTOP. */
+		if (child->flags & PF_USER_WORKER) {
+			ret = -EPERM;
+			break;
+		}
+
 		if (addr != sizeof(sigset_t)) {
 			ret = -EINVAL;
 			break;
diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig
index 332df7a7a6347c..bc9483c0b5d84f 100644
--- a/kernel/rcu/Kconfig
+++ b/kernel/rcu/Kconfig
@@ -175,6 +175,12 @@ config RCU_STALL_COMMON
 config RCU_NEED_SEGCBLIST
 	def_bool ( TREE_RCU || TREE_SRCU || TASKS_RCU_GENERIC )
 
+# The deferral (and the IRQ_WORK it uses) is only needed where call_rcu() /
+# call_srcu() can be invoked while a callback-list operation is in flight.
+config RCU_DEFER
+	def_bool HAVE_NMI || KPROBES || FUNCTION_TRACER || TRACEPOINTS
+	select IRQ_WORK
+
 config RCU_FANOUT
 	int "Tree-based hierarchical RCU fanout value"
 	range 2 64 if 64BIT
diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug
index e078e988773d6a..7c3c6017b2660e 100644
--- a/kernel/rcu/Kconfig.debug
+++ b/kernel/rcu/Kconfig.debug
@@ -113,6 +113,18 @@ config RCU_REF_SCALE_TEST
 	  Say M if you want to build it as a module instead.
 	  Say N if you are unsure.
 
+config HAZPTR_TORTURE_TEST
+	tristate "Torture tests for hazard pointers"
+	depends on DEBUG_KERNEL
+	select TORTURE_TEST
+	default n
+	help
+	  This option provides in-kernel hazard-pointer stress tests.
+
+	  Say Y here if you want hazard-pointer testing built into the kernel.
+	  Say M if you want to build them as a module instead.
+	  Say N if you are unsure.
+
 config RCU_CPU_STALL_TIMEOUT
 	int "RCU CPU stall timeout in seconds"
 	depends on RCU_STALL_COMMON
@@ -239,4 +251,18 @@ config TRIVIAL_PREEMPT_RCU
 
 	  This has no value for production and is only for testing.
 
+config HAZPTR_DEBUG
+	bool "Provide debugging asserts for Hazard Pointers"
+	depends on DEBUG_KERNEL
+	default n
+	help
+	  This option provides consistency checks for hazard pointers.
+	  These checks detect cases where a hazard pointer is acquired in
+	  one context and released in another, but without the required
+	  call to hazptr_detach() before exiting the context in which
+	  the hazard pointer was acquired.  For more detail, please see
+	  the hazptr_detach() kernel-doc header.
+
+	  Say Y here if you want to enable those assert, N otherwise.
+
 endmenu # "RCU Debugging"
diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
index 0cfb009a99b9f7..478ef2237fee6e 100644
--- a/kernel/rcu/Makefile
+++ b/kernel/rcu/Makefile
@@ -10,6 +10,7 @@ endif
 obj-y += update.o sync.o
 obj-$(CONFIG_TREE_SRCU) += srcutree.o
 obj-$(CONFIG_TINY_SRCU) += srcutiny.o
+obj-$(CONFIG_HAZPTR_TORTURE_TEST) += hazptrtorture.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
 obj-$(CONFIG_RCU_SCALE_TEST) += rcuscale.o
 obj-$(CONFIG_RCU_REF_SCALE_TEST) += refscale.o
diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c
new file mode 100644
index 00000000000000..7c8b5899fb0111
--- /dev/null
+++ b/kernel/rcu/hazptrtorture.c
@@ -0,0 +1,946 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Hazard-pointer module-based torture test facility
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ *
+ * Author: Paul E. McKenney <paulmck@kernel.org>
+ */
+
+#define pr_fmt(fmt) fmt
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/sched/debug.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/reboot.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/torture.h>
+#include <linux/hazptr.h>
+#include <linux/rcupdate.h>
+
+#include "rcu.h"
+
+MODULE_DESCRIPTION("Hazard-pointer module-based torture test facility");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Paul E. McKenney <paulmckrcu@meta.com>");
+
+torture_param(int, defer_modulus, -1, "Defer once per specified # of hazptr ops, zero to disable");
+torture_param(int, irq_acquire, -1,
+	      "Acquire hazard pointers from irq handlers once per specified #, zero to disable");
+torture_param(int, irq_release, -1,
+	      "Release hazard pointers from irq handlers once per specified #, zero to disable");
+torture_param(int, kthread_do_pending_ms, -1,
+	      "Delay between cleanups for deferred hazard pointers (ms), zero to disable");
+torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads");
+torture_param(bool, nwriters, 1, "Number of hazard-pointer writer threads, 0 or 1");
+torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
+torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable");
+torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable");
+torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)");
+torture_param(int, reader_sleep_us, 0, "Reader sleep duration (us)");
+torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
+torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
+torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s");
+torture_param(int, stutter, 5, "Number of seconds to run/halt test");
+torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
+
+static char *torture_type = "hazptr";
+module_param(torture_type, charp, 0444);
+MODULE_PARM_DESC(torture_type, "Type of hazard pointers to torture (hazptr, ...)");
+
+static int nrealreaders;
+static struct task_struct *writer_task;
+static struct task_struct *preempt_task;
+static struct task_struct **reader_tasks;
+static struct task_struct *do_pending_task;
+static struct task_struct *stats_task;
+
+#define HAZPTR_TORTURE_PIPE_LEN 10
+
+// Update-side data structure used to check RCU readers.
+struct hazptr_torture {
+	void *obj_hazptr;
+	int htort_pipe_count;
+	struct list_head htort_free;
+};
+
+static LIST_HEAD(hazptr_torture_freelist);
+static struct hazptr_torture *hazptr_torture_current;
+static unsigned long hazptr_torture_current_version;
+static struct hazptr_torture hazptr_tortures[10 * HAZPTR_TORTURE_PIPE_LEN];
+static DEFINE_SPINLOCK(hazptr_torture_lock);
+static DEFINE_PER_CPU(long [HAZPTR_TORTURE_PIPE_LEN + 1], hazptr_torture_count);
+static atomic_t hazptr_torture_wcount[HAZPTR_TORTURE_PIPE_LEN + 1];
+static atomic_t n_hazptr_torture_alloc;
+static atomic_t n_hazptr_torture_alloc_fail;
+static atomic_t n_hazptr_torture_free;
+static atomic_t n_hazptr_torture_error;
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_acquires);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_acquires_irq);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_irq);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_defer);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_undefer);
+static struct list_head hazptr_torture_removed;
+
+// State for a deferred (AKA pending) hazard pointer
+struct hazptr_pending {
+	struct llist_node hpp_node;
+	struct hazptr_ctx hpp_hc;
+	struct hazptr_torture *hpp_htp;
+};
+static DEFINE_PER_CPU(struct llist_head, hazptr_pending);
+
+static int hazptr_torture_writer_state;
+#define HTWS_FIXED_DELAY	0
+#define HTWS_DELAY		1
+#define HTWS_REPLACE		2
+#define HTWS_SYNC		3
+#define HTWS_STUTTER		4
+#define HTWS_STOPPING		5
+static const char * const hazptr_torture_writer_state_names[] = {
+	"HTWS_FIXED_DELAY",
+	"HTWS_DELAY",
+	"HTWS_REPLACE",
+	"HTWS_SYNC",
+	"HTWS_STUTTER",
+	"HTWS_STOPPING",
+};
+
+static const char *hazptr_torture_writer_state_getname(void)
+{
+	unsigned int i = READ_ONCE(hazptr_torture_writer_state);
+
+	if (i >= ARRAY_SIZE(hazptr_torture_writer_state_names))
+		return "???";
+	return hazptr_torture_writer_state_names[i];
+}
+
+/*
+ * Allocate an element from the hazptr_tortures pool.
+ */
+static struct hazptr_torture *hazptr_torture_alloc(void)
+{
+	struct list_head *p;
+
+	spin_lock_bh(&hazptr_torture_lock);
+	if (list_empty(&hazptr_torture_freelist)) {
+		atomic_inc(&n_hazptr_torture_alloc_fail);
+		spin_unlock_bh(&hazptr_torture_lock);
+		return NULL;
+	}
+	atomic_inc(&n_hazptr_torture_alloc);
+	p = hazptr_torture_freelist.next;
+	list_del_init(p);
+	spin_unlock_bh(&hazptr_torture_lock);
+	return container_of(p, struct hazptr_torture, htort_free);
+}
+
+/*
+ * Free an element to the hazptr_tortures pool.
+ */
+static void
+hazptr_torture_free(struct hazptr_torture *p)
+{
+	atomic_inc(&n_hazptr_torture_free);
+	spin_lock_bh(&hazptr_torture_lock);
+	list_add_tail(&p->htort_free, &hazptr_torture_freelist);
+	spin_unlock_bh(&hazptr_torture_lock);
+}
+
+/*
+ * Update object in the pipe.  This should be invoked after a suitable time.
+ */
+static bool
+hazptr_torture_pipe_update_one(struct hazptr_torture *rp)
+{
+	int i;
+
+	i = rp->htort_pipe_count;
+	if (i > HAZPTR_TORTURE_PIPE_LEN)
+		i = HAZPTR_TORTURE_PIPE_LEN;
+	atomic_inc(&hazptr_torture_wcount[i]);
+	WRITE_ONCE(rp->htort_pipe_count, i + 1);
+	ASSERT_EXCLUSIVE_WRITER(rp->htort_pipe_count);
+	if (i + 1 >= HAZPTR_TORTURE_PIPE_LEN)
+		return true;
+	return false;
+}
+
+/*
+ * Update all callbacks in the pipe each time period.
+ */
+static void
+hazptr_torture_pipe_update(struct hazptr_torture *old_rp)
+{
+	struct hazptr_torture *rp;
+	struct hazptr_torture *rp1;
+
+	if (old_rp)
+		list_add(&old_rp->htort_free, &hazptr_torture_removed);
+	list_for_each_entry_safe(rp, rp1, &hazptr_torture_removed, htort_free) {
+		if (hazptr_torture_pipe_update_one(rp)) {
+			list_del(&rp->htort_free);
+			hazptr_torture_free(rp);
+		}
+	}
+}
+
+/*
+ * Operations vector for selecting different types of tests.
+ */
+
+struct hazptr_torture_ops {
+	void (*init)(void);
+	void (*cleanup)(void);
+	struct hazptr_torture *((*readlock)(struct hazptr_ctx *hcpp));
+	void (*read_delay)(struct torture_random_state *rrsp);
+	void (*readunlock)(struct hazptr_ctx *hcp, struct hazptr_torture *htp);
+	void (*sync)(void *htp);
+	int irq_capable;
+	int onstack_ctx;
+	const char *name;
+};
+
+static struct hazptr_torture_ops *cur_ops;
+
+/*
+ * Definitions for hazard-pointer torture testing using per-CPU hazptr_ctx
+ * structures.
+ */
+
+static struct hazptr_torture *hazptr_torture_read_lock(struct hazptr_ctx *hcpp)
+{
+	struct hazptr_torture *htp;
+
+	htp = (struct hazptr_torture *)hazptr_acquire(hcpp, (void *)&hazptr_torture_current);
+	return htp;
+}
+
+static void hazptr_read_delay(struct torture_random_state *rrsp)
+{
+	const bool can_sleep = !preempt_count() && !irqs_disabled();
+	const unsigned long shortdelay_us = 200;
+	unsigned long longdelay_ms = 300;
+	const bool short_spin = irqs_disabled() || irq_count();
+
+	 // We want a short delay sometimes to make a reader delay the grace
+	 // period, and we want a long delay occasionally to trigger
+	 // force_quiescent_state.
+
+	if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
+		if (short_spin)
+			longdelay_ms = 5; /* Avoid triggering BH limits. */
+		mdelay(longdelay_ms);
+	}
+	if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
+		udelay(shortdelay_us);
+	if (can_sleep && !(torture_random(rrsp) % (nrealreaders * 500)))
+		torture_preempt_schedule();  /* QS only if preemptible. */
+	if (can_sleep && reader_sleep_us > 0)
+		torture_hrtimeout_us(reader_sleep_us, 0, NULL);
+}
+
+static void hazptr_torture_read_unlock(struct hazptr_ctx *hcp, struct hazptr_torture *htp)
+{
+	if (hcp) {
+		hazptr_release(hcp, htp);
+	}
+}
+
+static void hazptr_sync_torture_init(void)
+{
+	INIT_LIST_HEAD(&hazptr_torture_removed);
+}
+
+static struct hazptr_torture_ops hazptr_ops = {
+	.init			= hazptr_sync_torture_init,
+	.readlock		= hazptr_torture_read_lock,
+	.read_delay		= hazptr_read_delay,
+	.readunlock		= hazptr_torture_read_unlock,
+	.sync			= hazptr_synchronize,
+	.irq_capable		= 1,
+	.name			= "hazptr"
+};
+
+/*
+ * Definitions for hazard-pointer torture testing using on-stack
+ * hazptr_ctx structures.
+ */
+
+static struct hazptr_torture_ops hazptr_stack_ops = {
+	.init			= hazptr_sync_torture_init,
+	.readlock		= hazptr_torture_read_lock,
+	.read_delay		= hazptr_read_delay,
+	.readunlock		= hazptr_torture_read_unlock,
+	.sync			= hazptr_synchronize,
+	.irq_capable		= 1,
+	.onstack_ctx		= 1,
+	.name			= "hazptr-stack"
+};
+
+/*
+ * Hazard-pointer torture writer kthread.  Repeatedly substitutes a new
+ * structure for that pointed to by hazptr_torture_current, freeing the
+ * old structure after a series of timeouts (the "pipeline").
+ */
+static int
+hazptr_torture_writer(void *arg)
+{
+	bool booting_still = false;
+	int i;
+	unsigned long j;
+	int oldnice = task_nice(current);
+	struct hazptr_torture *rp;
+	struct hazptr_torture *old_rp;
+	static DEFINE_TORTURE_RANDOM(rand);
+	bool stutter_waited;
+
+	VERBOSE_TOROUT_STRING("hazptr_torture_writer task started");
+	// If the system is still booting, let it finish.
+	j = jiffies;
+	while (!torture_must_stop() && !rcu_inkernel_boot_has_ended()) {
+		booting_still = true;
+		schedule_timeout_interruptible(HZ);
+	}
+	if (booting_still)
+		pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n",
+			 torture_type, jiffies - j);
+
+	do {
+		hazptr_torture_writer_state = HTWS_FIXED_DELAY;
+		torture_hrtimeout_us(500, 1000, &rand);
+		rp = hazptr_torture_alloc();
+		if (rp == NULL)
+			continue;
+		rp->htort_pipe_count = 0;
+		ASSERT_EXCLUSIVE_WRITER(rp->htort_pipe_count);
+		hazptr_torture_writer_state = HTWS_DELAY;
+		udelay(torture_random(&rand) & 0x3ff);
+		hazptr_torture_writer_state = HTWS_REPLACE;
+		old_rp = READ_ONCE(hazptr_torture_current);
+		smp_store_release(&hazptr_torture_current, rp); // Publish new structure.
+		smp_wmb(); /* Mods to old_rp must follow smp_store_release() */
+		if (old_rp) {
+			i = old_rp->htort_pipe_count;
+			if (i > HAZPTR_TORTURE_PIPE_LEN)
+				i = HAZPTR_TORTURE_PIPE_LEN;
+			atomic_inc(&hazptr_torture_wcount[i]);
+			WRITE_ONCE(old_rp->htort_pipe_count,
+				   old_rp->htort_pipe_count + 1);
+			ASSERT_EXCLUSIVE_WRITER(old_rp->htort_pipe_count);
+
+			hazptr_torture_writer_state = HTWS_SYNC;
+			cur_ops->sync((void *)old_rp);
+			hazptr_torture_pipe_update(old_rp);
+		}
+
+		WRITE_ONCE(hazptr_torture_current_version, hazptr_torture_current_version + 1);
+		hazptr_torture_writer_state = HTWS_STUTTER;
+		stutter_waited = stutter_wait("hazptr_torture_writer");
+		if (stutter_waited && !torture_must_stop())
+			for (i = 0; i < ARRAY_SIZE(hazptr_tortures); i++)
+				if (list_empty(&hazptr_tortures[i].htort_free) &&
+				    READ_ONCE(hazptr_torture_current) != &hazptr_tortures[i]) {
+					tracing_off();
+					WARN(1, "%s: htort_pipe_count: %d\n", __func__, hazptr_tortures[i].htort_pipe_count);
+					rcu_ftrace_dump(DUMP_ALL);
+					break;
+				}
+		if (stutter_waited)
+			sched_set_normal(current, oldnice);
+	} while (!torture_must_stop());
+	hazptr_torture_current = NULL;  // Let stats task know that we are done.
+	hazptr_torture_writer_state = HTWS_STOPPING;
+	torture_kthread_stopping("hazptr_torture_writer");
+	return 0;
+}
+
+/*
+ * Acquire a hazard pointer from an smp_call_function handler.
+ */
+static void hazptr_torture_acquire(void *hppp_in)
+{
+	struct hazptr_pending *hppp = hppp_in;
+
+	hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc);
+	/*
+	 * Acquiring a hazard pointer from a remote CPU.
+	 * Detach hazptr from its task so it can be released by another task.
+	 */
+	hazptr_detach(&hppp->hpp_hc);
+	atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires_irq, raw_smp_processor_id()));
+}
+
+/*
+ * Release a hazard pointer from an smp_call_function handler.
+ */
+static void hazptr_torture_release(void *hppp_in)
+{
+	struct hazptr_pending *hppp = hppp_in;
+
+	cur_ops->readunlock(&hppp->hpp_hc, hppp->hpp_htp);
+	atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_irq, raw_smp_processor_id()));
+}
+
+/*
+ * Do the delay, the accounting, and the release.  This in intended to
+ * be invoked from hazptr_torture_reader, but also for hazard pointers
+ * sent off to interrupt handlers and the like.
+ */
+static void
+hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_state *trsp)
+{
+	int cpu;
+	struct hazptr_ctx *hcp = &hppp->hpp_hc;
+	struct hazptr_torture *htp = hppp->hpp_htp;
+	int pipe_count;
+
+	cur_ops->read_delay(trsp);
+	preempt_disable();
+	pipe_count = READ_ONCE(htp->htort_pipe_count);
+	if (pipe_count > HAZPTR_TORTURE_PIPE_LEN) {
+		// Should not happen in a correct hazptr implementation,
+		// happens quite often for TBD torture_type=busted.
+		pipe_count = HAZPTR_TORTURE_PIPE_LEN;
+	}
+	if (pipe_count > 1)
+		rcu_ftrace_dump(DUMP_ALL);
+	__this_cpu_inc(hazptr_torture_count[pipe_count]);
+	preempt_enable();
+	if (irq_release && !(torture_random(trsp) % irq_release)) {
+		guard(preempt)();
+		/*
+		 * Handling release from a remote CPU.
+		 * Detach hazptr from its task so it can be released by another task.
+		 */
+		hazptr_detach(&hppp->hpp_hc);
+		cpu = cpumask_next_wrap(smp_processor_id(), cpu_online_mask);
+		smp_call_function_single(cpu, hazptr_torture_release, hppp, 1);
+	} else {
+		cur_ops->readunlock(hcp, htp);
+		atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases, raw_smp_processor_id()));
+	}
+}
+
+/*
+ * Defer the specified hazard pointer to some other context.
+ */
+static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_random_state *trsp)
+{
+	int cpu = torture_random(trsp) % nr_cpu_ids;
+	struct llist_head *llhp;
+
+	guard(preempt)();
+	/*
+	 * Handling release from a remote CPU.
+	 * Detach hazptr from its task so it can be released by another task.
+	 */
+	hazptr_detach(&hppp->hpp_hc);
+	cpu = cpumask_next_wrap(cpu, cpu_online_mask);
+	llhp = per_cpu_ptr(&hazptr_pending, cpu);
+	llist_add(&hppp->hpp_node, llhp);
+	atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_defer, raw_smp_processor_id()));
+}
+
+/*
+ * Hazard-pointer torture reader kthread.  Repeatedly dereferences
+ * hazptr_torture_current, incrementing the corresponding element of the
+ * pipeline array.  The counter in the element should never be greater
+ * than 1, otherwise, the hazard-pointer implementation is broken.
+ */
+static int hazptr_torture_reader(void *arg)
+{
+	bool can_defer = !cur_ops->onstack_ctx && kthread_do_pending_ms && defer_modulus;
+	int cpu = 0;
+	struct hazptr_pending hpp;
+	struct hazptr_pending *hppp = cur_ops->onstack_ctx ? &hpp : NULL;
+	unsigned long lastsleep = jiffies;
+	long myid = (long)arg;
+	int mynumonline = myid % nr_cpu_ids;
+	DEFINE_TORTURE_RANDOM(rand);
+
+	VERBOSE_TOROUT_STRING("hazptr_torture_reader task started");
+	set_user_nice(current, MAX_NICE);
+	do {
+		if (!hppp) {
+			hppp = kmalloc_obj(*hppp, GFP_KERNEL);
+			if (!hppp) {
+				// Allocation failure, so get out of the way.
+				schedule_timeout_interruptible(HZ / 10);
+				continue;
+			}
+		}
+		if (irq_acquire && !(torture_random(&rand) % irq_acquire)) {
+			guard(preempt)();
+			cpu = cpumask_next_wrap(cpu, cpu_online_mask);
+			if (cpu != smp_processor_id()) {
+				smp_call_function_single(cpu, hazptr_torture_acquire, hppp, 1);
+			} else {
+				hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc);
+				atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires,
+							    raw_smp_processor_id()));
+			}
+		} else {
+			hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc);
+			atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires,
+						    raw_smp_processor_id()));
+		}
+		if (!hppp->hpp_htp) {
+			// Still starting up, so get out of the way.
+			schedule_timeout_interruptible(HZ / 10);
+			continue;
+		}
+		if (time_after(jiffies, lastsleep) && !torture_must_stop()) {
+			torture_hrtimeout_us(500, 1000, &rand);
+			lastsleep = jiffies + 10;
+		}
+		if (can_defer && !(torture_random(&rand) % defer_modulus)) {
+			hazptr_torture_defer(hppp, &rand);
+			hppp = NULL;
+		} else {
+			hazptr_torture_reader_tail(hppp, &rand);
+		}
+		while (!torture_must_stop() &&
+		       (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended()))
+			schedule_timeout_interruptible(HZ / 5);
+		stutter_wait("hazptr_torture_reader");
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_reader");
+	return 0;
+}
+
+/*
+ * Release the specified CPU's set of deferred/pending hazard pointers.
+ */
+static void hazptr_torture_do_one_pending(int cpu, struct torture_random_state *trsp)
+{
+	struct hazptr_pending *hppp;
+	struct hazptr_pending *hppp1;
+	struct llist_head *llhp;
+	struct llist_node *llnp;
+
+	llhp = per_cpu_ptr(&hazptr_pending, cpu);
+	llnp = llist_del_all(llhp);
+	if (!llnp)
+		return;
+	llist_for_each_entry_safe(hppp, hppp1, llnp, hpp_node) {
+		hazptr_torture_reader_tail(hppp, trsp);
+		atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_undefer,
+					    raw_smp_processor_id()));
+		kfree(hppp);
+	}
+}
+
+/*
+ * Hazard-pointer release of deferred/pending hazard pointers.
+ */
+static int hazptr_torture_do_pending(void *arg)
+{
+	int cpu = 0;
+	DEFINE_TORTURE_RANDOM(rand);
+
+	VERBOSE_TOROUT_STRING("hazptr_torture_do_pending task started");
+	do {
+		if (stutter_will_wait()) {
+			for_each_possible_cpu(cpu)
+				hazptr_torture_do_one_pending(cpu, &rand);
+		} else {
+			cpu = cpumask_next_wrap(cpu, cpu_possible_mask);
+			hazptr_torture_do_one_pending(cpu, &rand);
+		}
+		if (!torture_must_stop())
+			torture_hrtimeout_ms(kthread_do_pending_ms, USEC_PER_MSEC, &rand);
+		// Omit stutter_wait() because this function needs to do cleanup.
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_do_pending");
+	return 0;
+}
+
+/*
+ * Spawn hazptr_torture_do_pending() if there is something for it to do.
+ */
+static int hazptr_torture_do_pending_init(void)
+{
+	if (kthread_do_pending_ms == -1)
+		kthread_do_pending_ms = (cur_ops->onstack_ctx || defer_modulus == 0) ? 0 : 3;
+	if (defer_modulus == -1)
+		defer_modulus = (cur_ops->onstack_ctx ||
+				 kthread_do_pending_ms == 0) ? 0 : 1000 * nr_cpu_ids;
+	if (kthread_do_pending_ms < 0) {
+		pr_alert("Cannot have negative kthread_do_pending_ms, disabling deferral.\n");
+		goto err_out;
+	}
+	if (cur_ops->onstack_ctx && kthread_do_pending_ms) {
+		pr_alert("Cannot defer onstack hazptr_ctx, disabling deferral.\n");
+		goto err_out;
+	}
+	if (defer_modulus < 0) {
+		pr_alert("Cannot have negative defer_modulus (%d), disabling deferral.\n",
+			 defer_modulus);
+		goto err_out;
+	}
+	if (!kthread_do_pending_ms != !defer_modulus) {
+		pr_alert("Pending kthread (%d) & deferral (%d) don't match, disabling deferral.\n",
+			 kthread_do_pending_ms, defer_modulus);
+		goto err_out;
+	}
+	if (!kthread_do_pending_ms)
+		return 0;
+	return torture_create_kthread(hazptr_torture_do_pending, NULL, do_pending_task);
+
+err_out:
+	WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
+	kthread_do_pending_ms = 0;
+	defer_modulus = 0;
+	return 0;
+}
+
+/*
+ * Print torture statistics.  Caller must ensure that there is only one
+ * call to this function at a given time!!!  This is normally accomplished
+ * by relying on the module system to only have one copy of the module
+ * loaded, and then by giving the hazptr_torture_stats kthread full control
+ * (or the init/cleanup functions when hazptr_torture_stats thread is
+ * not running).
+ */
+static void
+hazptr_torture_stats_print(void)
+{
+	const char *cp = hazptr_torture_writer_state_getname();
+	int cpu;
+	int i;
+	long pipesummary[HAZPTR_TORTURE_PIPE_LEN + 1] = { 0 };
+	long batchsummary[HAZPTR_TORTURE_PIPE_LEN + 1] = { 0 };
+	struct hazptr_torture *rtcp;
+	static unsigned long rtcv_snap = ULONG_MAX;
+	static bool splatted;
+	struct task_struct *wtp;
+
+	for_each_possible_cpu(cpu)
+		for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+			pipesummary[i] += READ_ONCE(per_cpu(hazptr_torture_count, cpu)[i]);
+	for (i = HAZPTR_TORTURE_PIPE_LEN; i >= 0; i--) {
+		if (pipesummary[i] != 0)
+			break;
+	} // The value of variable "i" is used later, so don't clobber it!
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	rtcp = READ_ONCE(hazptr_torture_current);
+	pr_cont("rtc: %p %s: %lu %s tfle: %d rta: %d rtaf: %d rtf: %d ",
+		rtcp,
+		rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
+		hazptr_torture_current_version,
+		cp,
+		list_empty(&hazptr_torture_freelist),
+		atomic_read(&n_hazptr_torture_alloc),
+		atomic_read(&n_hazptr_torture_alloc_fail),
+		atomic_read(&n_hazptr_torture_free));
+	torture_onoff_stats();
+	pr_cont("acq: %lld rel: %lld acqirq: %lld relirq: %lld reldefer: %lld relundefer %lld\n",
+		torture_sum_pcpu_atomic_long(&hazptr_torture_acquires),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_acquires_irq),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases_irq),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases_defer),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases_undefer));
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	if (i > 1) {
+		pr_cont("%s", "!!! ");
+		atomic_inc(&n_hazptr_torture_error);
+		WARN_ON_ONCE(i > 1); // Too-short grace period
+	}
+	pr_cont("Reader Pipe: ");
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		pr_cont(" %ld", pipesummary[i]);
+	pr_cont("\n");
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	pr_cont("Reader Batch: ");
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		pr_cont(" %ld", batchsummary[i]);
+	pr_cont("\n");
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	pr_cont("Free-Block Circulation: ");
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		pr_cont(" %d", atomic_read(&hazptr_torture_wcount[i]));
+	pr_cont("\n");
+
+	if (rtcv_snap == hazptr_torture_current_version &&
+	    READ_ONCE(hazptr_torture_current) &&
+	    rcu_inkernel_boot_has_ended()) {
+		int __maybe_unused flags = 0;
+		unsigned long __maybe_unused gp_seq = 0;
+
+		wtp = READ_ONCE(writer_task);
+		pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n",
+			 hazptr_torture_writer_state_getname(),
+			 hazptr_torture_writer_state, gp_seq, flags,
+			 wtp == NULL ? '?' : task_state_to_char(wtp),
+			 wtp == NULL ? -1 : (int)task_cpu(wtp));
+		if (!splatted && wtp) {
+			sched_show_task(wtp);
+			splatted = true;
+		}
+		rcu_ftrace_dump(DUMP_ALL);
+	}
+	rtcv_snap = hazptr_torture_current_version;
+}
+
+/*
+ * Periodically prints torture statistics, if periodic statistics printing
+ * was specified via the stat_interval module parameter.
+ */
+static int
+hazptr_torture_stats(void *arg)
+{
+	VERBOSE_TOROUT_STRING("hazptr_torture_stats task started");
+	do {
+		schedule_timeout_interruptible(stat_interval * HZ);
+		hazptr_torture_stats_print();
+		torture_shutdown_absorb("hazptr_torture_stats");
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_stats");
+	return 0;
+}
+
+static void
+hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char *tag)
+{
+	pr_alert("%s" TORTURE_FLAG
+		 "--- %s: nreaders=%d nwriters=%d "
+		 "defer_modulus=%d irq_acquire=%d irq_release=%d kthread_do_pending_ms=%d "
+		 "onoff_interval=%d onoff_holdoff=%d "
+		 "preempt_duration=%d preempt_interval=%d "
+		 "reader_sleep_us=%d "
+		 "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d "
+		 "verbose=%d\n",
+		 torture_type, tag, nrealreaders, nwriters,
+		 defer_modulus, irq_acquire, irq_release, kthread_do_pending_ms,
+		 onoff_interval, onoff_holdoff,
+		 preempt_duration, preempt_interval,
+		 reader_sleep_us,
+		 shuffle_interval, shutdown_secs, stat_interval, stutter,
+		 verbose);
+}
+
+// Randomly preempt online CPUs.
+static int hazptr_torture_preempt(void *unused)
+{
+	int cpu = -1;
+	DEFINE_TORTURE_RANDOM(rand);
+
+	schedule_timeout_idle(onoff_holdoff * HZ);
+	do {
+		// Wait for preempt_interval ms with up to 100us fuzz.
+		torture_hrtimeout_ms(preempt_interval, 100, &rand);
+		// Select online CPU.
+		cpu = cpumask_next(cpu, cpu_online_mask);
+		if (cpu >= nr_cpu_ids)
+			cpu = cpumask_next(-1, cpu_online_mask);
+		WARN_ON_ONCE(cpu >= nr_cpu_ids);
+		// Move to that CPU, if can't do so, retry later.
+		if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false))
+			continue;
+		// Preempt at high-ish priority, then reset to normal.
+		sched_set_fifo(current);
+		torture_sched_setaffinity(current->pid, cpu_present_mask, true);
+		mdelay(preempt_duration);
+		sched_set_normal(current, 0);
+		stutter_wait("hazptr_torture_preempt");
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_preempt");
+	return 0;
+}
+
+static void
+hazptr_torture_cleanup(void)
+{
+	int i;
+
+	if (torture_cleanup_begin())
+		return;
+	if (!cur_ops) {
+		torture_cleanup_end();
+		return;
+	}
+
+	torture_stop_kthread(hazptr_torture_do_pending, do_pending_task);
+	torture_stop_kthread(hazptr_torture_preempt, preempt_task);
+	torture_stop_kthread(hazptr_torture_writer, writer_task);
+
+	if (reader_tasks) {
+		for (i = 0; i < nrealreaders; i++)
+			torture_stop_kthread(hazptr_torture_reader,
+					     reader_tasks[i]);
+		kfree(reader_tasks);
+		reader_tasks = NULL;
+	}
+
+	torture_stop_kthread(hazptr_torture_stats, stats_task);
+
+	/* Do torture-type-specific cleanup operations.  */
+	if (cur_ops->cleanup != NULL)
+		cur_ops->cleanup();
+
+	hazptr_torture_stats_print();  /* -After- the stats thread is stopped! */
+	if (atomic_read(&n_hazptr_torture_error))
+		hazptr_torture_print_module_parms(cur_ops, "End of test: FAILURE");
+	else if (torture_onoff_failures())
+		hazptr_torture_print_module_parms(cur_ops, "End of test: HAZPTR_HOTPLUG");
+	else
+		hazptr_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
+	torture_cleanup_end();
+}
+
+static int __init hazptr_torture_init(void)
+{
+	long i;
+	int cpu;
+	int firsterr = 0;
+	static struct hazptr_torture_ops *torture_ops[] = { &hazptr_ops, &hazptr_stack_ops, };
+
+	if (!torture_init_begin(torture_type, verbose))
+		return -EBUSY;
+
+	/* Process args and tell the world that the torturer is on the job. */
+	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
+		cur_ops = torture_ops[i];
+		if (strcmp(torture_type, cur_ops->name) == 0)
+			break;
+	}
+	if (i == ARRAY_SIZE(torture_ops)) {
+		pr_alert("hazptr-torture: invalid torture type: \"%s\"\n", torture_type);
+		pr_alert("hazptr-torture types:");
+		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
+			pr_cont(" %s", torture_ops[i]->name);
+		pr_cont("\n");
+		firsterr = -EINVAL;
+		cur_ops = NULL;
+		goto unwind;
+	}
+
+	if (cur_ops->init)
+		cur_ops->init();
+
+	if (nreaders >= 0) {
+		nrealreaders = nreaders;
+	} else {
+		nrealreaders = num_online_cpus() * -nreaders;
+		if (nrealreaders <= 0)
+			nrealreaders = 1;
+	}
+	hazptr_torture_print_module_parms(cur_ops, "Start of test");
+
+	/* Set up the freelist. */
+	INIT_LIST_HEAD(&hazptr_torture_freelist);
+	for (i = 0; i < ARRAY_SIZE(hazptr_tortures); i++)
+		list_add_tail(&hazptr_tortures[i].htort_free, &hazptr_torture_freelist);
+
+	/* Initialize the statistics so that each run gets its own numbers. */
+
+	hazptr_torture_current = NULL;
+	hazptr_torture_current_version = 0;
+	atomic_set(&n_hazptr_torture_alloc, 0);
+	atomic_set(&n_hazptr_torture_alloc_fail, 0);
+	atomic_set(&n_hazptr_torture_free, 0);
+	atomic_set(&n_hazptr_torture_error, 0);
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		atomic_set(&hazptr_torture_wcount[i], 0);
+	for_each_possible_cpu(cpu) {
+		for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+			per_cpu(hazptr_torture_count, cpu)[i] = 0;
+	}
+
+	/* Start up the kthreads. */
+
+	// This must be before the readers in order to set up the module
+	// parameters used by the readers.
+	firsterr = hazptr_torture_do_pending_init();
+	if (torture_init_error(firsterr))
+		goto unwind;
+
+	if (irq_acquire == -1) {
+		irq_acquire = 1000 * nr_cpu_ids;
+	} else if (irq_acquire < 0) {
+		pr_alert("Cannot have irq_acquire (%d) < -1, disabling.\n", irq_acquire);
+		WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
+		irq_acquire = 0;
+	}
+	if (irq_release == -1) {
+		irq_release = 1000 * nr_cpu_ids;
+	} else if (irq_release < 0) {
+		pr_alert("Cannot have irq_release (%d) < -1, disabling.\n", irq_release);
+		WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
+		irq_release = 0;
+	}
+	reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders);
+	for (i = 0; i < nrealreaders; i++) {
+		firsterr = torture_create_kthread(hazptr_torture_reader, (void *)i,
+						  reader_tasks[i]);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+
+	if (nwriters) {
+		firsterr = torture_create_kthread(hazptr_torture_writer, NULL, writer_task);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+
+	firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL);
+	if (torture_init_error(firsterr))
+		goto unwind;
+
+	if (stat_interval > 0) {
+		firsterr = torture_create_kthread(hazptr_torture_stats, NULL, stats_task);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+	if (shuffle_interval > 0) {
+		firsterr = torture_shuffle_init(shuffle_interval * HZ);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+	if (stutter < 0)
+		stutter = 0;
+	if (stutter) {
+		int t;
+
+		t = stutter * HZ;
+		firsterr = torture_stutter_init(stutter * HZ, t);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+	firsterr = torture_shutdown_init(shutdown_secs, hazptr_torture_cleanup);
+	if (torture_init_error(firsterr))
+		goto unwind;
+	if (preempt_duration > 0) {
+		firsterr = torture_create_kthread(hazptr_torture_preempt, NULL, preempt_task);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+
+	torture_init_end();
+	return 0;
+
+unwind:
+	torture_init_end();
+	hazptr_torture_cleanup();
+	if (shutdown_secs) {
+		WARN_ON(!IS_MODULE(CONFIG_HAZPTR_TORTURE_TEST));
+		kernel_power_off();
+	}
+	return firsterr;
+}
+
+module_init(hazptr_torture_init);
+module_exit(hazptr_torture_cleanup);
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 39a9f6fa9a7b29..d60444bf3a02dd 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -572,6 +572,20 @@ static inline void tasks_cblist_init_generic(void) { }
 #define RCU_SCHEDULER_INIT	1
 #define RCU_SCHEDULER_RUNNING	2
 
+/*
+ * Defer whenever interrupts are disabled, since a callback-list operation may
+ * be in flight on this CPU.  Not before the scheduler is up: irq_work is not
+ * usable that early, and rcu_init() itself calls call_rcu().
+ */
+static inline bool should_rcu_defer(void)
+{
+	return IS_ENABLED(CONFIG_RCU_DEFER) && irqs_disabled() &&
+	       rcu_scheduler_active != RCU_SCHEDULER_INACTIVE;
+}
+
+/* Drain an outgoing CPU's deferred SRCU callbacks; see rcutree_migrate_callbacks(). */
+void srcu_offline_drain(int cpu);
+
 enum rcutorture_type {
 	RCU_FLAVOR,
 	RCU_TASKS_FLAVOR,
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 794937e13e7c32..51ed35f5aab171 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -48,6 +48,7 @@
 #include <linux/tick.h>
 #include <linux/rcupdate_trace.h>
 #include <linux/nmi.h>
+#include <linux/perf_event.h>
 
 #include "rcu.h"
 
@@ -115,6 +116,7 @@ torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
 torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing");
 torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers");
 torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
+torture_param(bool, nmi_calls, true, "Exercise ->call() from NMI on nmi_capable flavors");
 torture_param(int, nreaders, -1, "Number of RCU reader threads");
 torture_param(bool, nwriters, 1, "Number of RCU writer threads (0 or 1)");
 torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing");
@@ -216,6 +218,8 @@ static long n_rcu_torture_boost_failure;
 static long n_rcu_torture_boosts;
 static atomic_long_t n_rcu_torture_timers;
 static atomic_long_t n_rcu_torture_irqs;
+static atomic_long_t n_rcu_torture_nmi_call;
+static atomic_long_t n_rcu_torture_nmi_cb;
 static long n_barrier_attempts;
 static long n_barrier_successes; /* did rcu_barrier test succeed? */
 static unsigned long n_read_exits;
@@ -433,6 +437,7 @@ struct rcu_torture_ops {
 	bool (*is_task_rcu_boosted)(void);
 	long cbflood_max;
 	int irq_capable;
+	int nmi_capable;
 	int can_boost;
 	int extendables;
 	int slow_gps;
@@ -440,6 +445,7 @@ struct rcu_torture_ops {
 	int debug_objects;
 	int start_poll_irqsoff;
 	int have_up_down;
+	int rdrs_handle_load;
 	const char *name;
 };
 
@@ -648,6 +654,8 @@ static struct rcu_torture_ops rcu_ops = {
 	.extendables		= RCUTORTURE_MAX_EXTEND,
 	.debug_objects		= 1,
 	.start_poll_irqsoff	= 1,
+	.rdrs_handle_load	= !IS_ENABLED(CONFIG_PREEMPT_RCU) || IS_ENABLED(CONFIG_RCU_BOOST),
+	.nmi_capable		= 1,
 	.name			= "rcu"
 };
 
@@ -702,10 +710,21 @@ static struct rcu_torture_ops rcu_busted_ops = {
 DEFINE_STATIC_SRCU(srcu_ctl);
 DEFINE_STATIC_SRCU_FAST(srcu_ctlf);
 DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_ctlfud);
+DEFINE_STATIC_SRCU_ATOMIC(srcu_ctla);
 static struct srcu_struct srcu_ctld;
 static struct srcu_struct *srcu_ctlp = &srcu_ctl;
 static struct rcu_torture_ops srcud_ops;
 
+// Restrict APIs permitted for atomic SRCU.
+static void srcu_torture_init_forbidden_apis(void)
+{
+	cur_ops->call = NULL;
+	cur_ops->cb_barrier = NULL;
+	cur_ops->deferred_free = NULL;
+	cur_ops->exp_current = NULL;
+	cur_ops->start_gp_poll = NULL;
+}
+
 static void srcu_torture_init(void)
 {
 	rcu_sync_torture_init();
@@ -721,6 +740,11 @@ static void srcu_torture_init(void)
 		srcu_ctlp = &srcu_ctlfud;
 		VERBOSE_TOROUT_STRING("srcu_torture_init fast-up/down SRCU");
 	}
+	if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) {
+		srcu_ctlp = &srcu_ctla;
+		VERBOSE_TOROUT_STRING("srcu_torture_init atomic SRCU");
+		srcu_torture_init_forbidden_apis();
+	}
 }
 
 static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)
@@ -758,6 +782,11 @@ static int srcu_torture_read_lock(void)
 		WARN_ON_ONCE(idx & ~0x1);
 		ret += idx << 3;
 	}
+	if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) {
+		idx = srcu_read_lock_atomic(srcu_ctlp);
+		WARN_ON_ONCE(idx & ~0x1);
+		ret += idx << 4;
+	}
 	return ret;
 }
 
@@ -777,7 +806,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
 
 	delay = torture_random(rrsp) %
 		(nrealreaders * 2 * longdelay * uspertick);
-	if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled()) {
+	if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled() &&
+	    !(reader_flavor & SRCU_READ_FLAVOR_ATOMIC)) {
 		schedule_timeout_interruptible(longdelay);
 		rtrsp->rt_delay_jiffies = longdelay;
 	} else {
@@ -788,6 +818,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
 static void srcu_torture_read_unlock(int idx)
 {
 	WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));
+	if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC)
+		srcu_read_unlock_atomic(srcu_ctlp, (idx & 0x10) >> 4);
 	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)
 		srcu_read_unlock_fast_updown(srcu_ctlp,
 					     __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));
@@ -867,7 +899,10 @@ static void srcu_torture_deferred_free(struct rcu_torture *rp)
 
 static void srcu_torture_synchronize(void)
 {
-	synchronize_srcu(srcu_ctlp);
+	if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC)
+		synchronize_srcu_atomic(srcu_ctlp);
+	else
+		synchronize_srcu(srcu_ctlp);
 }
 
 static unsigned long srcu_torture_get_gp_state(void)
@@ -903,7 +938,10 @@ static void srcu_torture_stats(void)
 
 static void srcu_torture_synchronize_expedited(void)
 {
-	synchronize_srcu_expedited(srcu_ctlp);
+	if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC)
+		synchronize_srcu_atomic(srcu_ctlp);
+	else
+		synchronize_srcu_expedited(srcu_ctlp);
 }
 
 static void srcu_torture_expedite_current(void)
@@ -942,6 +980,7 @@ static struct rcu_torture_ops srcu_ops = {
 	.debug_objects	= 1,
 	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)
 				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
+	.nmi_capable	= 1,
 	.name		= "srcu"
 };
 
@@ -960,6 +999,10 @@ static void srcud_torture_init(void)
 	} else if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {
 		WARN_ON(init_srcu_struct_fast_updown(&srcu_ctld));
 		VERBOSE_TOROUT_STRING("srcud_torture_init fast-up/down SRCU");
+	} else if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) {
+		WARN_ON(init_srcu_struct_atomic(&srcu_ctld));
+		VERBOSE_TOROUT_STRING("srcud_torture_init atomic SRCU");
+		srcu_torture_init_forbidden_apis();
 	} else {
 		WARN_ON(init_srcu_struct(&srcu_ctld));
 	}
@@ -1005,6 +1048,7 @@ static struct rcu_torture_ops srcud_ops = {
 	.debug_objects	= 1,
 	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)
 				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
+	.nmi_capable	= 1,
 	.name		= "srcud"
 };
 
@@ -1269,6 +1313,7 @@ static struct rcu_torture_ops tasks_tracing_ops = {
 	.cbflood_max	= 50000,
 	.irq_capable	= 1,
 	.slow_gps	= 1,
+	.nmi_capable	= 1,
 	.name		= "tasks-tracing"
 };
 
@@ -1738,7 +1783,7 @@ rcu_torture_writer(void *arg)
 		pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n",
 			 torture_type, jiffies - j);
 
-	if (IS_ENABLED(CONFIG_RCU_LAZY))
+	if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call)
 		INIT_WORK_ONSTACK(&lazy_work, rcu_torture_writer_work);
 
 	do {
@@ -1933,7 +1978,7 @@ rcu_torture_writer(void *arg)
 				       !rcu_gp_is_normal();
 		}
 		rcu_torture_writer_state = RTWS_STUTTER;
-		if (IS_ENABLED(CONFIG_RCU_LAZY))
+		if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call)
 			queue_work(system_percpu_wq, &lazy_work);
 		stutter_waited = stutter_wait("rcu_torture_writer");
 		if (stutter_waited &&
@@ -1966,7 +2011,7 @@ rcu_torture_writer(void *arg)
 			 " Dynamic grace-period expediting was disabled.\n",
 			 torture_type);
 
-	if (IS_ENABLED(CONFIG_RCU_LAZY)) {
+	if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) {
 		cancel_work_sync(&lazy_work);
 		destroy_work_on_stack(&lazy_work);
 	}
@@ -2534,8 +2579,10 @@ static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp
 	rtorsp->p = rcu_dereference_check(rcu_torture_current,
 					  !cur_ops->readlock_held || cur_ops->readlock_held() ||
 					  (rtorsp->readstate & RCUTORTURE_RDR_UPDOWN));
-	if (rtorsp->p == NULL) {
-		/* Wait for rcu_torture_writer to get underway */
+	if ((!cur_ops->rdrs_handle_load && atomic_read(&rcu_fwd_cb_nodelay)) || rtorsp->p == NULL) {
+		// Wait for rcu_torture_writer to get underway and
+		// (if readers cannot handle heavy loads) for any
+		// forward-progress testing to complete.
 		rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp);
 		return false;
 	}
@@ -2659,6 +2706,124 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 
 static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand);
 
+/*
+ * Exercise ->call() from NMI context for flavors that set ->nmi_capable.  A
+ * per-CPU hardware perf counter overflows into an NMI, and its handler submits
+ * a preallocated callback via ->call().  One callback per CPU is in flight at a
+ * time (guarded by an atomic) to avoid allocating in NMI.
+ */
+#ifdef CONFIG_PERF_EVENTS
+static struct perf_event_attr rcu_torture_nmi_attr = {
+	.type		= PERF_TYPE_HARDWARE,
+	.config		= PERF_COUNT_HW_CPU_CYCLES,
+	.size		= sizeof(struct perf_event_attr),
+	.pinned		= 1,
+	.disabled	= 1,
+	/*
+	 * A fixed period rather than .freq: a frequency-based event bumps
+	 * nr_freq_events, which sets TICK_DEP_BIT_PERF_EVENTS and would pin the
+	 * tick for the whole run on NO_HZ_FULL kernels.
+	 */
+	.sample_period	= 20 * 1000 * 1000,
+};
+
+/* One in-flight callback per CPU; ->inuse is released by the callback. */
+struct rcu_torture_nmi_cb {
+	struct rcu_head rh;
+	atomic_t inuse;
+};
+
+static struct perf_event **rcu_torture_nmi_events;
+static int rcu_torture_nmi_hp_state;
+static DEFINE_PER_CPU(struct rcu_torture_nmi_cb, rcu_torture_nmi_cb);
+
+static void rcu_torture_nmi_invoked(struct rcu_head *rhp)
+{
+	struct rcu_torture_nmi_cb *rtncp = container_of(rhp, struct rcu_torture_nmi_cb, rh);
+
+	atomic_long_inc(&n_rcu_torture_nmi_cb);
+	atomic_set(&rtncp->inuse, 0);
+}
+
+static void rcu_torture_nmi_overflow(struct perf_event *event,
+				     struct perf_sample_data *data,
+				     struct pt_regs *regs)
+{
+	struct rcu_torture_nmi_cb *rtncp = this_cpu_ptr(&rcu_torture_nmi_cb);
+
+	if (!in_nmi())
+		return;
+	if (cur_ops->call && !atomic_xchg(&rtncp->inuse, 1)) {
+		atomic_long_inc(&n_rcu_torture_nmi_call);
+		cur_ops->call(&rtncp->rh, rcu_torture_nmi_invoked);
+	}
+}
+
+static int rcu_torture_nmi_online(unsigned int cpu)
+{
+	struct perf_event *event;
+
+	event = perf_event_create_kernel_counter(&rcu_torture_nmi_attr, cpu, NULL,
+						 rcu_torture_nmi_overflow, NULL);
+	if (IS_ERR(event))
+		return 0;
+	rcu_torture_nmi_events[cpu] = event;
+	perf_event_enable(event);
+	return 0;
+}
+
+static int rcu_torture_nmi_offline(unsigned int cpu)
+{
+	struct perf_event *event = rcu_torture_nmi_events[cpu];
+
+	if (event) {
+		rcu_torture_nmi_events[cpu] = NULL;
+		perf_event_disable(event);
+		perf_event_release_kernel(event);
+	}
+	return 0;
+}
+
+/* Drive the counters from hotplug callbacks so coverage survives onoff. */
+static void rcu_torture_nmi_init(void)
+{
+	int ret;
+
+	if (!nmi_calls || !cur_ops->nmi_capable || !cur_ops->call)
+		return;
+	rcu_torture_nmi_events = kcalloc(nr_cpu_ids, sizeof(*rcu_torture_nmi_events),
+					 GFP_KERNEL);
+	if (!rcu_torture_nmi_events)
+		return;
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "rcutorture/nmi:online",
+				rcu_torture_nmi_online, rcu_torture_nmi_offline);
+	if (ret < 0) {
+		kfree(rcu_torture_nmi_events);
+		rcu_torture_nmi_events = NULL;
+		return;
+	}
+	rcu_torture_nmi_hp_state = ret;
+}
+
+static void rcu_torture_nmi_cleanup(void)
+{
+	if (!rcu_torture_nmi_events)
+		return;
+	if (rcu_torture_nmi_hp_state > 0) {
+		cpuhp_remove_state(rcu_torture_nmi_hp_state);
+		rcu_torture_nmi_hp_state = 0;
+	}
+	kfree(rcu_torture_nmi_events);
+	rcu_torture_nmi_events = NULL;
+	if (!atomic_long_read(&n_rcu_torture_nmi_call))
+		pr_alert("%s: nmi_calls set but no ->call() ever issued from NMI, so NMI ->call() went untested (no PMU, or NMIs unavailable here).\n",
+			 __func__);
+}
+#else /* #ifdef CONFIG_PERF_EVENTS */
+static void rcu_torture_nmi_init(void) { }
+static void rcu_torture_nmi_cleanup(void) { }
+#endif /* #else #ifdef CONFIG_PERF_EVENTS */
+
 /*
  * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
  * incrementing the corresponding element of the pipeline array.  The
@@ -3047,6 +3212,9 @@ rcu_torture_stats_print(void)
 		data_race(n_barrier_attempts),
 		data_race(n_rcu_torture_barrier_error));
 	pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic.
+	pr_cont("nmi-calls: %ld nmi-cbs: %ld ",
+		atomic_long_read(&n_rcu_torture_nmi_call),
+		atomic_long_read(&n_rcu_torture_nmi_cb));
 	pr_cont("nocb-toggles: %ld:%ld ",
 		atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload));
 	pr_cont("gpwraps: %ld\n", n_gpwraps);
@@ -3195,7 +3363,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
 		 "read_exit_delay=%d read_exit_burst=%d "
 		 "reader_flavor=%x "
 		 "nocbs_nthreads=%d nocbs_toggle=%d "
-		 "test_nmis=%d "
+		 "test_nmis=%d nmi_calls=%d "
 		 "preempt_duration=%d preempt_interval=%d n_up_down=%d\n",
 		 torture_type, tag, nrealreaders, nwriters, nrealfakewriters,
 		 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
@@ -3209,7 +3377,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
 		 read_exit_delay, read_exit_burst,
 		 reader_flavor,
 		 nocbs_nthreads, nocbs_toggle,
-		 test_nmis,
+		 test_nmis, nmi_calls,
 		 preempt_duration, preempt_interval, n_up_down);
 }
 
@@ -3856,7 +4024,7 @@ static int __init rcu_torture_fwd_prog_init(void)
 	}
 	if (fwd_progress_holdoff <= 0)
 		fwd_progress_holdoff = 1;
-	if (fwd_progress_div <= 0)
+	if (fwd_progress_div < 2)
 		fwd_progress_div = 4;
 	rfp = kzalloc_objs(*rfp, fwd_progress);
 	fwd_prog_tasks = kzalloc_objs(*fwd_prog_tasks, fwd_progress);
@@ -4283,6 +4451,7 @@ rcu_torture_cleanup(void)
 	int i;
 
 	if (torture_cleanup_begin()) {
+		rcu_torture_nmi_cleanup();
 		if (cur_ops->cb_barrier != NULL) {
 			pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
 			cur_ops->cb_barrier();
@@ -4322,9 +4491,12 @@ rcu_torture_cleanup(void)
 		for (i = 0; i < nrealreaders; i++)
 			torture_stop_kthread(rcu_torture_reader,
 					     reader_tasks[i]);
+		if (irqreader && cur_ops->irq_capable)
+			kick_all_cpus_sync();
 		kfree(reader_tasks);
 		reader_tasks = NULL;
 	}
+	rcu_torture_nmi_cleanup();
 	kfree(rcu_torture_reader_mbchk);
 	rcu_torture_reader_mbchk = NULL;
 
@@ -4354,6 +4526,19 @@ rcu_torture_cleanup(void)
 		pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
 		cur_ops->cb_barrier();
 	}
+
+	/*
+	 * cb_barrier() above drained every deferred callback, so the count
+	 * issued from NMI must equal the count invoked.
+	 */
+	if (atomic_long_read(&n_rcu_torture_nmi_call) !=
+	    atomic_long_read(&n_rcu_torture_nmi_cb)) {
+		pr_alert("%s: NMI ->call() lost a callback: issued %ld invoked %ld\n",
+			 __func__, atomic_long_read(&n_rcu_torture_nmi_call),
+			 atomic_long_read(&n_rcu_torture_nmi_cb));
+		atomic_inc(&n_rcu_torture_error);
+	}
+
 	if (cur_ops->cleanup != NULL)
 		cur_ops->cleanup();
 
@@ -4786,6 +4971,7 @@ rcu_torture_init(void)
 		firsterr = -ENOMEM;
 		goto unwind;
 	}
+	rcu_torture_nmi_init();
 	for (i = 0; i < nrealreaders; i++) {
 		rcu_torture_reader_mbchk[i].rtc_chkrdr = -1;
 		firsterr = torture_create_kthread(rcu_torture_reader, (void *)i,
diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index a2d9d75d88a10d..2320b3fedec736 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -29,6 +29,7 @@
 #include <linux/reboot.h>
 #include <linux/sched.h>
 #include <linux/seq_buf.h>
+#include <linux/hazptr.h>
 #include <linux/spinlock.h>
 #include <linux/smp.h>
 #include <linux/stat.h>
@@ -1200,6 +1201,47 @@ static const struct ref_scale_ops typesafe_seqlock_ops = {
 	.name		= "typesafe_seqlock"
 };
 
+static void ref_hazptr_read_section(const int nloops)
+{
+	static void *ref_hazptr_read_section_ptr = ref_hazptr_read_section;
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		struct hazptr_ctx ctx;
+		void *addr;
+
+		addr = hazptr_acquire(&ctx, &ref_hazptr_read_section_ptr);
+		hazptr_release(&ctx, addr);
+	}
+}
+
+static void ref_hazptr_delay_section(const int nloops, const int udl, const int ndl)
+{
+	static void *ref_hazptr_delay_section_ptr = ref_hazptr_delay_section;
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		struct hazptr_ctx ctx;
+		void *addr;
+
+		addr = hazptr_acquire(&ctx, &ref_hazptr_delay_section_ptr);
+		un_delay(udl, ndl);
+		hazptr_release(&ctx, addr);
+	}
+}
+
+static bool ref_hazptr_init(void)
+{
+	return true;
+}
+
+static const struct ref_scale_ops hazptr_ops = {
+	.init		= ref_hazptr_init,
+	.readsection	= ref_hazptr_read_section,
+	.delaysection	= ref_hazptr_delay_section,
+	.name		= "hazptr"
+};
+
 static void rcu_scale_one_reader(void)
 {
 	if (readdelay <= 0)
@@ -1504,6 +1546,7 @@ ref_scale_init(void)
 		&sched_clock_ops, &clock_ops, &jiffies_ops,
 		&preempt_ops, &bh_ops, &irq_ops, &irqsave_ops,
 		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
+		&hazptr_ops,
 	};
 
 	if (!torture_init_begin(scale_type, verbose))
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index 558ba8d316db6c..99f8bfd98b045e 100644
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -10,6 +10,7 @@
 
 #include <linux/export.h>
 #include <linux/irq_work.h>
+#include <linux/llist.h>
 #include <linux/mutex.h>
 #include <linux/preempt.h>
 #include <linux/rcupdate_wait.h>
@@ -29,6 +30,8 @@ extern int rcu_scheduler_active;
 static LIST_HEAD(srcu_boot_list);
 static bool srcu_init_done;
 
+static void __srcu_defer_drain(struct srcu_struct *ssp);
+
 static int init_srcu_struct_fields(struct srcu_struct *ssp)
 {
 	ssp->srcu_lock_nesting[0] = 0;
@@ -38,11 +41,14 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp)
 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;
 	ssp->srcu_gp_running = false;
 	ssp->srcu_gp_waiting = false;
+	ssp->srcu_atomic_gp_flag = 0;
 	ssp->srcu_idx = 0;
 	ssp->srcu_idx_max = 0;
 	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
 	INIT_LIST_HEAD(&ssp->srcu_work.entry);
 	init_irq_work(&ssp->srcu_irq_work, srcu_tiny_irq_work);
+	init_llist_head(&ssp->defer_cbs);
+	ssp->defer_iw = IRQ_WORK_INIT_HARD(srcu_defer_drain);
 	return 0;
 }
 
@@ -86,6 +92,16 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic);
 void cleanup_srcu_struct(struct srcu_struct *ssp)
 {
 	WARN_ON(srcu_readers_active(ssp));
+	/*
+	 * Re-issue any deferred callbacks, then wait out ->defer_iw before it is
+	 * freed.  Skipped entirely with CONFIG_RCU_DEFER=n: irq_work_sync() ends
+	 * in an unconditional synchronize_rcu() wherever
+	 * arch_irq_work_has_interrupt() is false, which is every !SMP target.
+	 */
+	if (IS_ENABLED(CONFIG_RCU_DEFER)) {
+		__srcu_defer_drain(ssp);
+		irq_work_sync(&ssp->defer_iw);
+	}
 	irq_work_sync(&ssp->srcu_irq_work);
 	flush_work(&ssp->srcu_work);
 	WARN_ON(ssp->srcu_gp_running);
@@ -144,6 +160,8 @@ void srcu_drive_gp(struct work_struct *wp)
 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
 	WRITE_ONCE(ssp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */
 	preempt_enable();
+	if (IS_ENABLED(CONFIG_PREEMPTION))
+		synchronize_rcu(); // Needed for RCU Tasks Trace to imply RCU grace period
 	do {
 		// Deadlock issues prevent __srcu_read_unlock() from
 		// doing an unconditional wakeup, so polling is required.
@@ -213,11 +231,11 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
 }
 
 /*
- * Enqueue an SRCU callback on the specified srcu_struct structure,
- * initiating grace-period processing if it is not already running.
+ * Also called by __srcu_defer_drain() to re-issue a deferred callback, so it
+ * must not re-check the deferral condition.
  */
-void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
-	       rcu_callback_t func)
+static void srcu_do_enqueue(struct srcu_struct *ssp, struct rcu_head *rhp,
+			    rcu_callback_t func)
 {
 	unsigned long flags;
 
@@ -231,6 +249,68 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
 	srcu_gp_start_if_needed(ssp);
 	preempt_enable();
 }
+
+/*
+ * Set only by the irq_work drain, the one drain its own re-issue can re-feed;
+ * a callback staged during a direct drain is taken by ->defer_iw afterwards.
+ * Global rather than per-srcu_struct: a re-entrant call_srcu(B) inside a drain
+ * of A raises B's own ->defer_iw, whose drain can stage back onto A.
+ */
+static bool srcu_defer_draining;
+
+static void __srcu_defer_drain(struct srcu_struct *ssp)
+{
+	struct llist_node *node, *next;
+	unsigned long flags;
+
+	if (!IS_ENABLED(CONFIG_RCU_DEFER))
+		return;
+
+	/* Re-issued newest-first; nothing depends on call_srcu() ordering. */
+	local_irq_save(flags);
+	llist_for_each_safe(node, next, llist_del_all(&ssp->defer_cbs)) {
+		struct rcu_head *rhp = (struct rcu_head *)node;
+
+		srcu_do_enqueue(ssp, rhp, rhp->func);
+	}
+	local_irq_restore(flags);
+}
+
+/* Only the irq_work drain can be re-fed by its own re-issue; see Tree SRCU. */
+void srcu_defer_drain(struct irq_work *iw)
+{
+	struct srcu_struct *ssp = container_of(iw, struct srcu_struct, defer_iw);
+
+	WRITE_ONCE(srcu_defer_draining, true);
+	__srcu_defer_drain(ssp);
+	WRITE_ONCE(srcu_defer_draining, false);
+}
+EXPORT_SYMBOL_GPL(srcu_defer_drain);
+
+void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
+	       rcu_callback_t func)
+{
+	if (should_rcu_defer()) {
+		/* A re-entrant call_srcu() during the drain would livelock it. */
+		if (READ_ONCE(srcu_defer_draining) && !in_nmi()) {
+			WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU),
+				  "call_srcu() re-entered during callback drain; leaking callback\n");
+			return;
+		}
+		rhp->func = func;
+		if (llist_add((struct llist_node *)rhp, &ssp->defer_cbs))
+			irq_work_queue(&ssp->defer_iw);
+		return;
+	}
+
+	/*
+	 * Only reachable from an NMI when deferral is off: before the scheduler
+	 * is up, or with CONFIG_RCU_DEFER=n.  The enqueue can then race.
+	 */
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
+
+	srcu_do_enqueue(ssp, rhp, func);
+}
 EXPORT_SYMBOL_GPL(call_srcu);
 
 /*
@@ -260,6 +340,95 @@ void synchronize_srcu(struct srcu_struct *ssp)
 }
 EXPORT_SYMBOL_GPL(synchronize_srcu);
 
+/*
+ * synchronize_srcu_atomic - spinning grace period for atomic-reader domains
+ * @ssp: srcu_struct with which to synchronize.
+ *
+ * On !SMP this cannot spin: a reader observed mid-section is preempted
+ * or interrupted-out, and can only finish if we yield the CPU. But it
+ * is also never needed: an atomic-flavor reader (preemption disabled)
+ * cannot be observed mid-section from process context on the sole CPU.
+ * So a reader observed here has broken the atomic-domain promise, and
+ * the only correct wait for it is a real grace period.
+ *
+ * (Actual kernel-doc header is in Tree SRCU.)
+ */
+void synchronize_srcu_atomic(struct srcu_struct *ssp)
+{
+	int idx;
+	bool ret;
+	unsigned long srcu_state = get_state_synchronize_srcu(ssp);
+
+	srcu_lock_sync(&ssp->dep_map);
+
+	if (IS_ENABLED(CONFIG_PREEMPTION))
+		synchronize_rcu(); // Needed for RCU Tasks Trace to imply RCU grace period.
+				   // And in Tiny RCU, it is near zero cost and doesn't block.
+
+	// Usually, there will be no readers.
+	preempt_disable();  // Guard against lazy preemption and some other grace period.
+	ret = !READ_ONCE(ssp->srcu_lock_nesting[0]) && !READ_ONCE(ssp->srcu_lock_nesting[1]);
+	if (ret) {
+		WRITE_ONCE(ssp->srcu_idx_max, ssp->srcu_idx + 2);
+		WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 2);
+		preempt_enable();
+		return;
+	}
+
+	// Because readers disable preemption, we should never get here.
+	// However, a splat and some spinning is usually preferable to
+	// memory corruption due to a too-short grace period.  There is
+	// the possibility that this will hang if the preempted reader is
+	// not looked upon favorably by the scheduler, but this is still
+	// preferable to memory corruption.
+	WARN_ON_ONCE(1);
+
+	// Wait to drive a grace period or for someone else to do it
+	// for us while we are lazily preempted.
+	while (ssp->srcu_atomic_gp_flag) {
+		if (poll_state_synchronize_srcu(ssp, srcu_state)) {
+			preempt_enable();
+			return;
+		}
+		preempt_enable();
+		cpu_relax();
+		cond_resched_tasks_rcu_qs();
+		preempt_disable();
+	}
+	ssp->srcu_atomic_gp_flag = 1;
+	preempt_enable();
+
+	// We get here if a reader has been lazily preempted.
+	// First, wait for old readers, which are quite unlikely.
+	WRITE_ONCE(ssp->srcu_idx_max, get_state_synchronize_srcu(ssp));
+	idx = !(((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1);
+	while (READ_ONCE(ssp->srcu_lock_nesting[idx])) {
+		cond_resched_tasks_rcu_qs();
+		cpu_relax();
+	}
+
+	// Next, flip the index and wait for the other group of readers.
+	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
+	idx = !idx;
+	while (READ_ONCE(ssp->srcu_lock_nesting[idx])) {
+		cond_resched_tasks_rcu_qs();
+		cpu_relax();
+	}
+
+	// Finally, flip the index again for poll_state_synchronize_srcu().
+	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
+	WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state));
+}
+EXPORT_SYMBOL_GPL(synchronize_srcu_atomic);
+
+/* Register any deferred callbacks, then wait for all in-flight ones. */
+void srcu_barrier(struct srcu_struct *ssp)
+{
+	__srcu_defer_drain(ssp);
+	synchronize_srcu(ssp);
+}
+EXPORT_SYMBOL_GPL(srcu_barrier);
+
 /*
  * get_state_synchronize_srcu - Provide an end-of-grace-period cookie
  */
@@ -280,6 +449,11 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
  * The difference between this and get_state_synchronize_srcu() is that
  * this function ensures that the poll_state_synchronize_srcu() will
  * eventually return the value true.
+ *
+ * This function cannot be used with atomic SRCU, which only has
+ * atomic grace periods.  Doing so will silently corrupt internal
+ * SRCU state.  Tree SRCU has appropriate checking with splats,
+ * so please test with CONFIG_SMP=y as well as CONFIG_SMP=n.
  */
 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)
 {
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index ed204b3f4b8440..402718d08588cf 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -20,6 +20,7 @@
 #include <linux/percpu.h>
 #include <linux/preempt.h>
 #include <linux/irq_work.h>
+#include <linux/llist.h>
 #include <linux/rcupdate_wait.h>
 #include <linux/sched.h>
 #include <linux/smp.h>
@@ -79,6 +80,38 @@ static void process_srcu(struct work_struct *work);
 static void srcu_irq_work(struct irq_work *work);
 static void srcu_delay_timer(struct timer_list *t);
 
+struct srcu_defer;
+static void srcu_defer_drain(struct irq_work *iw);
+static void __srcu_defer_drain(struct srcu_defer *sndp);
+
+/*
+ * Per-CPU call_srcu() deferral state, shared by every srcu_struct.  A deferred
+ * callback is staged on its srcu_data's ->defer_cbs; that srcu_data is chained
+ * via ->defer_link onto ->list, which the irq_work walks.
+ */
+struct srcu_defer {
+	struct llist_head	list;
+	struct irq_work		iw;
+	raw_spinlock_t		lock;
+	bool			draining;
+};
+
+static DEFINE_PER_CPU(struct srcu_defer, srcu_defer) = {
+	.lock = __RAW_SPIN_LOCK_UNLOCKED(srcu_defer.lock),
+	.iw = IRQ_WORK_INIT_HARD(srcu_defer_drain),
+};
+
+/*
+ * Flush pending deferred callbacks so a following srcu_barrier() waits for them.
+ */
+static void srcu_defer_flush(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		__srcu_defer_drain(&per_cpu(srcu_defer, cpu));
+}
+
 /*
  * Initialize SRCU per-CPU data.  Note that statically allocated
  * srcu_struct structures might already have srcu_read_lock() and
@@ -107,6 +140,11 @@ static void init_srcu_struct_data(struct srcu_struct *ssp)
 		sdp->cpu = cpu;
 		INIT_WORK(&sdp->work, srcu_invoke_callbacks);
 		timer_setup(&sdp->delay_work, srcu_delay_timer, 0);
+		/*
+		 * ->defer_cbs and ->defer_link are valid when zeroed and are not
+		 * reinitialized here: that would clobber callbacks a reentrant
+		 * call_srcu() already staged.  See __call_srcu().
+		 */
 		sdp->ssp = ssp;
 	}
 }
@@ -200,8 +238,10 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
  * Initialize non-compile-time initialized fields, including the
  * associated srcu_node and srcu_data structures.  The is_static parameter
  * tells us that ->sda has already been wired up to srcu_data.
+ * The is_atomic parameter tells us that there is no reason to
+ * ever transition to big.
  */
-static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
+static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static, bool is_atomic)
 {
 	if (!is_static)
 		ssp->srcu_sup = kzalloc_obj(*ssp->srcu_sup);
@@ -213,12 +253,18 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
 	ssp->srcu_sup->node = NULL;
 	mutex_init(&ssp->srcu_sup->srcu_cb_mutex);
 	mutex_init(&ssp->srcu_sup->srcu_gp_mutex);
+	atomic_set(&ssp->srcu_sup->srcu_atomic_gp_flag, 0);
 	ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL;
 	ssp->srcu_sup->srcu_barrier_seq = 0;
 	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);
 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);
 	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);
-	init_irq_work(&ssp->srcu_sup->irq_work, srcu_irq_work);
+	/*
+	 * trace events started on the command line require SRCU before
+	 * the irq_work kthread starts. Since all it does is a simple
+	 * wakeup, having it as a hard irq, even on PREEMPT_RT is fine.
+	 */
+	ssp->srcu_sup->irq_work = IRQ_WORK_INIT_HARD(srcu_irq_work);
 	ssp->srcu_sup->sda_is_static = is_static;
 	if (!is_static) {
 		ssp->sda = alloc_percpu(struct srcu_data);
@@ -229,7 +275,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
 	init_srcu_struct_data(ssp);
 	ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL;
 	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();
-	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {
+	if (!is_atomic &&
+	    READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {
 		if (!preemptible())
 			WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);
 		else if (init_srcu_struct_nodes(ssp, GFP_KERNEL))
@@ -258,26 +305,27 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 
 static int
-__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
+__init_srcu_struct_common(struct srcu_struct *ssp, const char *name,
+			  struct lock_class_key *key, bool is_atomic)
 {
 	/* Don't re-initialize a lock while it is held. */
 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
 	lockdep_init_map(&ssp->dep_map, name, key, 0);
-	return init_srcu_struct_fields(ssp, false);
+	return init_srcu_struct_fields(ssp, false, is_atomic);
 }
 
 int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name,
 			     struct lock_class_key *key)
 {
 	ssp->srcu_reader_flavor = 0;
-	return __init_srcu_struct_common(ssp, name, key);
+	return __init_srcu_struct_common(ssp, name, key, false);
 }
 EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep);
 
 int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
 {
 	ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST;
-	return __init_srcu_struct_common(ssp, name, key);
+	return __init_srcu_struct_common(ssp, name, key, false);
 }
 EXPORT_SYMBOL_GPL(__init_srcu_struct_fast);
 
@@ -285,10 +333,17 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name,
 				   struct lock_class_key *key)
 {
 	ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN;
-	return __init_srcu_struct_common(ssp, name, key);
+	return __init_srcu_struct_common(ssp, name, key, false);
 }
 EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown);
 
+int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, struct lock_class_key *key)
+{
+	ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC;
+	return __init_srcu_struct_common(ssp, name, key, true);
+}
+EXPORT_SYMBOL_GPL(__init_srcu_struct_atomic);
+
 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
 /**
@@ -305,7 +360,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown);
 int init_srcu_struct_generic(struct srcu_struct *ssp)
 {
 	ssp->srcu_reader_flavor = 0;
-	return init_srcu_struct_fields(ssp, false);
+	return init_srcu_struct_fields(ssp, false, false);
 }
 EXPORT_SYMBOL_GPL(init_srcu_struct_generic);
 
@@ -322,7 +377,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic);
 int init_srcu_struct_fast(struct srcu_struct *ssp)
 {
 	ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST;
-	return init_srcu_struct_fields(ssp, false);
+	return init_srcu_struct_fields(ssp, false, false);
 }
 EXPORT_SYMBOL_GPL(init_srcu_struct_fast);
 
@@ -340,10 +395,30 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_fast);
 int init_srcu_struct_fast_updown(struct srcu_struct *ssp)
 {
 	ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN;
-	return init_srcu_struct_fields(ssp, false);
+	return init_srcu_struct_fields(ssp, false, false);
 }
 EXPORT_SYMBOL_GPL(init_srcu_struct_fast_updown);
 
+/**
+ * init_srcu_struct_atomic - initialize an atomic sleep-RCU structure
+ * @ssp: structure to initialize.
+ *
+ * Use this in place of DEFINE_SRCU_ATOMIC() and DEFINE_STATIC_SRCU_ATOMIC()
+ * for non-static srcu_struct structures that are to be passed to
+ * srcu_read_lock_atomic() and friends.  It is necessary to invoke this on a
+ * given srcu_struct before passing that srcu_struct to any other function.
+ * Each srcu_struct represents a separate domain of SRCU protection.
+ *
+ * And yes, we really are defining a sleepable RCU implementation that
+ * cannot sleep.  Strange universe we live in, isn't it?
+ */
+int init_srcu_struct_atomic(struct srcu_struct *ssp)
+{
+	ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC;
+	return init_srcu_struct_fields(ssp, false, true);
+}
+EXPORT_SYMBOL_GPL(init_srcu_struct_atomic);
+
 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 
 /*
@@ -432,9 +507,11 @@ static void raw_spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsign
  * done with compile-time initialization, so this check is added
  * to each update-side SRCU primitive.  Use ssp->lock, which -is-
  * compile-time initialized, to resolve races involving multiple
- * CPUs trying to garner first-use privileges.
+ * CPUs trying to garner first-use privileges.  The is_atomic
+ * parameter tells us that there will never be a reason to
+ * transition to big.
  */
-static void check_init_srcu_struct(struct srcu_struct *ssp)
+static void check_init_srcu_struct(struct srcu_struct *ssp, bool is_atomic)
 {
 	unsigned long flags;
 
@@ -446,7 +523,7 @@ static void check_init_srcu_struct(struct srcu_struct *ssp)
 		raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
 		return;
 	}
-	init_srcu_struct_fields(ssp, true);
+	init_srcu_struct_fields(ssp, true, is_atomic);
 	raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
 }
 
@@ -624,7 +701,7 @@ module_param(srcu_retry_check_delay, ulong, 0444);
 #define SRCU_UL_CLAMP_LO(val, low)	((val) > (low) ? (val) : (low))
 #define SRCU_UL_CLAMP_HI(val, high)	((val) < (high) ? (val) : (high))
 #define SRCU_UL_CLAMP(val, low, high)	SRCU_UL_CLAMP_HI(SRCU_UL_CLAMP_LO((val), (low)), (high))
-// per-GP-phase no-delay instances adjusted to allow non-sleeping poll upto
+// per-GP-phase no-delay instances adjusted to allow non-sleeping poll up to
 // one jiffies time duration. Mult by 2 is done to factor in the srcu_get_delay()
 // called from process_srcu().
 #define SRCU_DEFAULT_MAX_NODELAY_PHASE_ADJUSTED	\
@@ -688,6 +765,14 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
 	unsigned long delay;
 	struct srcu_usage *sup = ssp->srcu_sup;
 
+	/*
+	 * Drain before the early returns below: they leak the srcu_struct, but
+	 * srcu_module_going() frees ->sda regardless, and a staged srcu_data
+	 * left chained on a per-CPU list would then dangle.  Draining first also
+	 * has to precede the ->irq_work sync, since re-issuing a callback can
+	 * start a grace period and re-queue ->irq_work, which schedules ->work.
+	 */
+	srcu_defer_flush();
 	raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
 	delay = srcu_get_delay(ssp);
 	raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
@@ -695,7 +780,6 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
 		return; /* Just leak it! */
 	if (WARN_ON(srcu_readers_active(ssp)))
 		return; /* Just leak it! */
-	/* Wait for irq_work to finish first as it may queue a new work. */
 	irq_work_sync(&sup->irq_work);
 	flush_delayed_work(&sup->work);
 	for_each_possible_cpu(cpu) {
@@ -704,7 +788,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
 		// Call srcu_barrier() before this cleanup_srcu_struct()
 		// to avoid triggering this WARN_ON().
 		if (WARN_ON(timer_delete_sync(&sdp->delay_work) &&
-			    rcu_segcblist_n_cbs(&sdp->srcu_cblist)) &&
+			    !rcu_segcblist_empty(&sdp->srcu_cblist)) &&
 		    rcu_cpu_beenfullyonline(sdp->cpu))
 			queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work);
 		flush_work(&sdp->work);
@@ -752,7 +836,10 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
 	WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor);
 	WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor &&
 		     old_read_flavor != ssp->srcu_reader_flavor);
-	WARN_ON_ONCE(read_flavor == SRCU_READ_FLAVOR_FAST && !ssp->srcu_reader_flavor);
+	WARN_ON_ONCE(!!(read_flavor & SRCU_READ_FLAVOR_PREDEF) &&
+		     read_flavor != ssp->srcu_reader_flavor);
+	WARN_ON_ONCE(!!(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_PREDEF) &&
+		     read_flavor != ssp->srcu_reader_flavor);
 	if (!old_read_flavor) {
 		old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor);
 		if (!old_read_flavor)
@@ -892,7 +979,7 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp
  * are initiating callback invocation.  This allows the ->srcu_have_cbs[]
  * array to have a finite number of elements.
  */
-static void srcu_gp_end(struct srcu_struct *ssp)
+static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic)
 {
 	unsigned long cbdelay = 1;
 	bool cbs;
@@ -908,7 +995,8 @@ static void srcu_gp_end(struct srcu_struct *ssp)
 	struct srcu_usage *sup = ssp->srcu_sup;
 
 	/* Prevent more than one additional grace period. */
-	mutex_lock(&sup->srcu_cb_mutex);
+	if (!is_atomic)
+		mutex_lock(&sup->srcu_cb_mutex);
 
 	/* End the current grace period. */
 	raw_spin_lock_irq_rcu_node(sup);
@@ -923,15 +1011,16 @@ static void srcu_gp_end(struct srcu_struct *ssp)
 	if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq))
 		WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq);
 	raw_spin_unlock_irq_rcu_node(sup);
-	mutex_unlock(&sup->srcu_gp_mutex);
+	if (!is_atomic)
+		mutex_unlock(&sup->srcu_gp_mutex);
 	/* A new grace period can start at this point.  But only one. */
 
 	/* Initiate callback invocation as needed. */
 	ss_state = smp_load_acquire(&sup->srcu_size_state);
-	if (ss_state < SRCU_SIZE_WAIT_BARRIER) {
+	if (!is_atomic && ss_state < SRCU_SIZE_WAIT_BARRIER) {
 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()),
 					cbdelay);
-	} else {
+	} else if (!is_atomic) {
 		idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs);
 		srcu_for_each_node_breadth_first(ssp, snp) {
 			raw_spin_lock_irq_rcu_node(snp);
@@ -968,13 +1057,15 @@ static void srcu_gp_end(struct srcu_struct *ssp)
 		}
 
 	/* Callback initiation done, allow grace periods after next. */
-	mutex_unlock(&sup->srcu_cb_mutex);
+	if (!is_atomic)
+		mutex_unlock(&sup->srcu_cb_mutex);
 
 	/* Start a new grace period if needed. */
 	raw_spin_lock_irq_rcu_node(sup);
 	gpseq = rcu_seq_current(&sup->srcu_gp_seq);
 	if (!rcu_seq_state(gpseq) &&
 	    ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) {
+		WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC);
 		srcu_gp_start(ssp);
 		raw_spin_unlock_irq_rcu_node(sup);
 		srcu_reschedule(ssp, 0);
@@ -982,8 +1073,10 @@ static void srcu_gp_end(struct srcu_struct *ssp)
 		raw_spin_unlock_irq_rcu_node(sup);
 	}
 
-	/* Transition to big if needed. */
-	if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) {
+	/* Transition to big if needed, but never for atomic SRCU. */
+	if (ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC && ss_state != SRCU_SIZE_SMALL) {
+		WARN_ON_ONCE(1);
+	} else if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) {
 		if (ss_state == SRCU_SIZE_ALLOC)
 			init_srcu_struct_nodes(ssp, GFP_KERNEL);
 		else
@@ -1098,6 +1191,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
 	/* If grace period not already in progress, start it. */
 	if (!WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) &&
 	    rcu_seq_state(sup->srcu_gp_seq) == SRCU_STATE_IDLE) {
+		WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC);
 		srcu_gp_start(ssp);
 
 		// And how can that list_add() in the "else" clause
@@ -1234,7 +1328,7 @@ static bool srcu_should_expedite(struct srcu_struct *ssp)
 	unsigned long t;
 	unsigned long tlast;
 
-	check_init_srcu_struct(ssp);
+	check_init_srcu_struct(ssp, false);
 	/* If _lite() readers, don't do unsolicited expediting. */
 	if (this_cpu_read(ssp->sda->srcu_reader_flavor) & SRCU_READ_FLAVOR_SLOWGP)
 		return false;
@@ -1293,7 +1387,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,
 	struct srcu_node *sdp_mynode;
 	int ss_state;
 
-	check_init_srcu_struct(ssp);
+	check_init_srcu_struct(ssp, false);
 	/*
 	 * While starting a new grace period, make sure we are in an
 	 * SRCU read-side critical section so that the grace-period
@@ -1411,8 +1505,8 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,
  * srcu_read_lock(), and srcu_read_unlock() that are all passed the same
  * srcu_struct structure.
  */
-static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
-			rcu_callback_t func, bool do_norm)
+static void srcu_do_enqueue(struct srcu_struct *ssp, struct rcu_head *rhp,
+			    rcu_callback_t func, bool do_norm)
 {
 	if (debug_rcu_head_queue(rhp)) {
 		/* Probable double call_srcu(), so leak the callback. */
@@ -1424,6 +1518,110 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
 	(void)srcu_gp_start_if_needed(ssp, rhp, do_norm);
 }
 
+/*
+ * The srcu_cblist and srcu_node tree are only accessed with interrupts
+ * disabled, so defer when interrupts are already off rather than enqueue into
+ * an operation that may be in flight on this CPU.
+ */
+static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
+			rcu_callback_t func, bool do_norm)
+{
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC))
+		return; // Leak the callback rather than corrupt SRCU state.
+	if (should_rcu_defer()) {
+		struct srcu_defer *sndp = this_cpu_ptr(&srcu_defer);
+		struct srcu_data *sdp;
+
+		/*
+		 * Instrumentation on the enqueue path can re-enter here from
+		 * inside the drain.  Re-queuing would livelock it, so drop the
+		 * callback; an NMI cannot loop, so let it through.
+		 */
+		if (READ_ONCE(sndp->draining) && !in_nmi()) {
+			WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU),
+				  "call_srcu() re-entered during callback drain; leaking callback\n");
+			return;
+		}
+		sdp = this_cpu_ptr(ssp->sda);
+		rhp->func = func;
+		if (llist_add((struct llist_node *)rhp, &sdp->defer_cbs)) {
+			/*
+			 * Chain this srcu_data for the drain.  ->ssp must be
+			 * published here: deferral skips
+			 * check_init_srcu_struct(), so on a never-initialized
+			 * static srcu_struct the srcu_data are still zeroed and
+			 * the drain would read a NULL ->ssp.
+			 */
+			sdp->ssp = ssp;
+			if (llist_add(&sdp->defer_link, &sndp->list))
+				irq_work_queue(&sndp->iw);
+		}
+		return;
+	}
+
+	/*
+	 * Only reachable from an NMI when deferral is off: before the scheduler
+	 * is up, or with CONFIG_RCU_DEFER=n.  The enqueue can then race.
+	 */
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
+
+	srcu_do_enqueue(ssp, rhp, func, do_norm);
+}
+
+/*
+ * Re-issue deferred callbacks straight to srcu_do_enqueue() so they cannot defer
+ * again.  ->lock serializes the drainers: the irq_work, srcu_defer_flush() and
+ * srcu_offline_drain().
+ */
+static void __srcu_defer_drain(struct srcu_defer *sndp)
+{
+	struct llist_node *snode, *snext;
+	unsigned long flags;
+
+	if (!IS_ENABLED(CONFIG_RCU_DEFER))
+		return;
+
+	raw_spin_lock_irqsave(&sndp->lock, flags);
+	llist_for_each_safe(snode, snext, llist_del_all(&sndp->list)) {
+		struct srcu_data *sdp = container_of(snode, struct srcu_data, defer_link);
+		struct srcu_struct *ssp = sdp->ssp;
+		struct llist_node *cnode, *cnext;
+
+		cnode = llist_del_all(&sdp->defer_cbs);
+		llist_for_each_safe(cnode, cnext, cnode) {
+			struct rcu_head *rhp = (struct rcu_head *)cnode;
+
+			/* Bounds a node self-linked by a double call_srcu(). */
+			rhp->next = NULL;
+			srcu_do_enqueue(ssp, rhp, rhp->func, true);
+		}
+	}
+	raw_spin_unlock_irqrestore(&sndp->lock, flags);
+}
+
+/*
+ * Only the irq_work drain can be re-fed by its own re-issue, so only it sets
+ * ->draining.  A direct drain re-issues onto this CPU, and anything staged
+ * during it is picked up by that CPU's own irq_work.
+ */
+static void srcu_defer_drain(struct irq_work *iw)
+{
+	struct srcu_defer *sndp = container_of(iw, struct srcu_defer, iw);
+
+	WRITE_ONCE(sndp->draining, true);
+	__srcu_defer_drain(sndp);
+	WRITE_ONCE(sndp->draining, false);
+}
+
+/*
+ * Drain @cpu's deferred call_srcu() callbacks once @cpu is dead.  One pass
+ * covers every srcu_struct; the re-issue lands on the current CPU.
+ */
+void srcu_offline_drain(int cpu)
+{
+	__srcu_defer_drain(&per_cpu(srcu_defer, cpu));
+}
+
 /**
  * call_srcu() - Queue a callback for invocation after an SRCU grace period
  * @ssp: srcu_struct in queue the callback
@@ -1469,8 +1667,13 @@ static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm)
 
 	if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
 		return;
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) {
+		// This works, and exposes a possible bug.
+		synchronize_srcu_atomic(ssp);
+		return;
+	}
 	might_sleep();
-	check_init_srcu_struct(ssp);
+	check_init_srcu_struct(ssp, false);
 	init_completion(&rcu.completion);
 	init_rcu_head_on_stack(&rcu.head);
 	__call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm);
@@ -1589,10 +1792,21 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
  * period has elapsed in the meantime.  Unlike get_state_synchronize_srcu(),
  * this function also ensures that any needed SRCU grace period will be
  * started.  This convenience does come at a cost in terms of CPU overhead.
+ *
+ * This function cannot be used with atomic SRCU, which only has
+ * atomic grace periods.  Give a warning if someone tries, and return
+ * the same cookie that would have been returned, but refrain from
+ * messing up state by starting a grace period.  If someone somewhere
+ * somehow invokes synchronize_srcu_atomic(), passing this cookie to
+ * poll_state_synchronize_srcu() will return true.  If no one ever invokes
+ * synchronize_srcu_atomic(), too bad.
  */
 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)
 {
-	return srcu_gp_start_if_needed(ssp, NULL, true);
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC))
+		return get_state_synchronize_srcu(ssp);
+	else
+		return srcu_gp_start_if_needed(ssp, NULL, true);
 }
 EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
 
@@ -1678,9 +1892,21 @@ void srcu_barrier(struct srcu_struct *ssp)
 {
 	int cpu;
 	int idx;
-	unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);
+	unsigned long s;
+
+	check_init_srcu_struct(ssp, false);
+	// Atomic SRCU has no callbacks, so there is nothing to wait on.
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC))
+		return;
+
+	/*
+	 * Register any deferred callbacks before snapshotting the sequence.  The
+	 * staging list is per-CPU, not per-srcu_struct, so this also drains
+	 * other srcu_structs'.
+	 */
+	srcu_defer_flush();
 
-	check_init_srcu_struct(ssp);
+	s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);
 	mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex);
 	if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) {
 		smp_mb(); /* Force ordering following return. */
@@ -1743,6 +1969,9 @@ static void srcu_expedite_current_cb(struct rcu_head *rhp)
  * no current grace period, one might be created.  If the current grace
  * period is currently sleeping, that sleep will complete before expediting
  * will take effect.
+ *
+ * This function must not be invoked on srcu_struct structures that are
+ * used with srcu_read_lock_atomic() and synchronize_srcu_atomic().
  */
 void srcu_expedite_current(struct srcu_struct *ssp)
 {
@@ -1750,6 +1979,9 @@ void srcu_expedite_current(struct srcu_struct *ssp)
 	bool needcb = false;
 	struct srcu_data *sdp;
 
+	// Atomic SRCU has no callbacks, so there is nothing to expedite.
+	if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC))
+		return;
 	migrate_disable();
 	sdp = this_cpu_ptr(ssp->sda);
 	raw_spin_lock_irqsave_sdp_contention(sdp, &flags);
@@ -1785,13 +2017,16 @@ EXPORT_SYMBOL_GPL(srcu_batches_completed);
 /*
  * Core SRCU state machine.  Push state bits of ->srcu_gp_seq
  * to SRCU_STATE_SCAN2, and invoke srcu_gp_end() when scan has
- * completed in that state.
+ * completed in that state.  Set is_atomic to indicate that
+ * the caller is excluding other calls so that ->srcu_gp_mutex
+ * is not needed, and to indicate that blocking is forbidden.
  */
-static void srcu_advance_state(struct srcu_struct *ssp)
+static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic)
 {
 	int idx;
 
-	mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);
+	if (!is_atomic)
+		mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);
 
 	/*
 	 * Because readers might be delayed for an extended period after
@@ -1809,15 +2044,19 @@ static void srcu_advance_state(struct srcu_struct *ssp)
 		if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {
 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));
 			raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
-			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
+			if (!is_atomic)
+				mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
 			return;
 		}
 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));
-		if (idx == SRCU_STATE_IDLE)
+		if (idx == SRCU_STATE_IDLE) {
+			WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC);
 			srcu_gp_start(ssp);
+		}
 		raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
 		if (idx != SRCU_STATE_IDLE) {
-			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
+			if (!is_atomic)
+				mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
 			return; /* Someone else started the grace period. */
 		}
 	}
@@ -1825,7 +2064,8 @@ static void srcu_advance_state(struct srcu_struct *ssp)
 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {
 		idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]);
 		if (!try_check_zero(ssp, idx, 1)) {
-			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
+			if (!is_atomic)
+				mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
 			return; /* readers present, retry later. */
 		}
 		srcu_flip(ssp);
@@ -1843,14 +2083,137 @@ static void srcu_advance_state(struct srcu_struct *ssp)
 		 */
 		idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]);
 		if (!try_check_zero(ssp, idx, 2)) {
-			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
+			if (!is_atomic)
+				mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
 			return; /* readers present, retry later. */
 		}
 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;
-		srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */
+		srcu_gp_end(ssp, is_atomic);  /* Releases ->srcu_gp_mutex. */
 	}
 }
 
+/**
+ * synchronize_srcu_atomic - spin for prior SRCU read-side critical-section completion
+ * @ssp: srcu_struct with which to synchronize.
+ *
+ * Similar to synchronize_srcu(), but spins rather than blocking.
+ * Use only with srcu_read_lock_atomic() and srcu_read_unlock_atomic(),
+ * which are forbidden from voluntarily context switching.  If
+ * synchronize_srcu_atomic() is invoked from a more restrictive context
+ * (for example, interrupts disabled) for a given srcu_struct structure,
+ * then for that structure, all calls to both srcu_read_lock_atomic()
+ * and srcu_read_unlock_atomic() must be invoked from that same context,
+ * or one that is even more strict.
+ *
+ * If synchronize_srcu_atomic() is invoked on a given srcu_struct
+ * structure, then none of call_srcu(), synchronize_srcu(),
+ * synchronize_srcu_expedited(), start_poll_synchronize_srcu(),
+ * srcu_barrier(), or srcu_expedite_current() may be invoked on that
+ * same structure.
+ *
+ * Because synchronize_srcu_atomic() is even more expedited than is
+ * synchronize_srcu_expedited(), there is no expedited counterpart to
+ * this function.
+ */
+void synchronize_srcu_atomic(struct srcu_struct *ssp)
+{
+	unsigned long srcu_state;
+	struct srcu_usage *sup = ssp->srcu_sup;
+	unsigned long rdm0, rdm1;
+	unsigned long unlocks0, unlocks1;
+
+	// Initialize.	Either init_srcu_struct() was invoked or
+	// DEFINE_SRCU() or similar was used.  Therefore, no allocation
+	// will be done here.
+	check_init_srcu_struct(ssp, true);
+	srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC);
+
+	// Perhaps others will do our work for us.
+	srcu_state = get_state_synchronize_srcu(ssp);
+	preempt_disable();
+	while (atomic_read(&sup->srcu_atomic_gp_flag) ||
+	       atomic_xchg(&sup->srcu_atomic_gp_flag, 1)) {
+		preempt_enable();
+		if (poll_state_synchronize_srcu(ssp, srcu_state))
+			return;
+		cpu_relax();
+		preempt_disable();
+	}
+
+	// One last check for others doing our work for us under the lock.
+	raw_spin_lock_irq_rcu_node(sup);
+	if (poll_state_synchronize_srcu(ssp, srcu_state)) {
+		raw_spin_unlock_irq_rcu_node(sup);
+		atomic_set(&sup->srcu_atomic_gp_flag, 0);
+		preempt_enable();
+		return;
+	}
+
+	// OK, we really have to do it ourselves.  Start the grace period.
+	non_block_start();  // We must not voluntarily block!
+	smp_store_release(&sup->srcu_gp_seq_needed, srcu_state); // See srcu_funnel_gp_start().
+	ASSERT_EXCLUSIVE_WRITER(ssp->srcu_sup->srcu_gp_seq);
+	srcu_gp_start(ssp);
+	raw_spin_unlock_irq_rcu_node(sup);
+
+	//
+	// Fastpath:  If there are no readers at all, neither grace-period
+	// scan need wait, so both can be satisfied at once without doing
+	// the index flip.  The counter-sum proof is the same as that of
+	// srcu_readers_active_idx_check(), but spanning both indices.
+	// Atomic SRCU guarantees that all readers are of
+	// SRCU_READ_FLAVOR_ATOMIC, so the SLOWGP check never triggers and
+	// the ->srcu_reader_flavor masks returned by
+	// srcu_readers_unlock_idx() are unused.
+	//
+	// This proof must follow the grace-period anchor written by the
+	// srcu_gp_start() above, never precede it.  With the anchor first,
+	// a reader whose lock increment is missed by the sums below cannot
+	// have incremented its lock counter before the anchor, and therefore
+	// cannot be a pre-existing reader of this grace period.  Placing the
+	// proof before the anchor would let this grace period miss a
+	// pre-existing reader and return without waiting for it.
+	//
+	// The smp_mb() pairs with the smp_mb() in __srcu_read_lock()
+	// (store-buffering pattern), which guarantees that a lock is always
+	// counted if the corresponding unlock is counted, the same
+	// memory-ordering guarantee as is provided by
+	// srcu_readers_active_idx_check().
+	//
+	unlocks0 = srcu_readers_unlock_idx(ssp, 0, &rdm0);
+	unlocks1 = srcu_readers_unlock_idx(ssp, 1, &rdm1);
+	smp_mb(); /* A */
+	if (srcu_readers_lock_idx(ssp, 0, false, unlocks0) &&
+	    srcu_readers_lock_idx(ssp, 1, false, unlocks1)) {
+		// No readers, so end this grace period manually, skipping
+		// the index flip.  Advancing the sequence number via
+		// rcu_seq_start() in srcu_gp_start() above and rcu_seq_end()
+		// below keeps get_state_synchronize_srcu() and
+		// poll_state_synchronize_srcu() working, all under ->lock
+		// and ->srcu_atomic_gp_flag, which excludes concurrent
+		// sequence-number updates.
+		raw_spin_lock_irq_rcu_node(sup);
+		rcu_seq_end(&sup->srcu_gp_seq);
+		raw_spin_unlock_irq_rcu_node(sup);
+		WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state));
+		atomic_set_release(&sup->srcu_atomic_gp_flag, 0);
+		preempt_enable();
+		non_block_end();
+		return;
+	}
+
+	// Wait for it to complete, helping it along.
+	while (!poll_state_synchronize_srcu(ssp, srcu_state)) {
+		cpu_relax();
+		srcu_advance_state(ssp, true);
+	}
+	ASSERT_EXCLUSIVE_WRITER(sup->srcu_atomic_gp_flag);
+	atomic_set_release(&sup->srcu_atomic_gp_flag, 0);
+	preempt_enable();
+	non_block_end();
+}
+EXPORT_SYMBOL_GPL(synchronize_srcu_atomic);
+
 /*
  * Invoke a limited number of SRCU callbacks that have passed through
  * their grace period.  If there are more to do, SRCU will reschedule
@@ -1930,6 +2293,7 @@ static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)
 		}
 	} else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) {
 		/* Outstanding request and no GP.  Start one. */
+		WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC);
 		srcu_gp_start(ssp);
 	}
 	raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
@@ -1951,7 +2315,7 @@ static void process_srcu(struct work_struct *work)
 	sup = container_of(work, struct srcu_usage, work.work);
 	ssp = sup->srcu_ssp;
 
-	srcu_advance_state(ssp);
+	srcu_advance_state(ssp, false);
 	raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
 	curdelay = srcu_get_delay(ssp);
 	raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
@@ -2058,8 +2422,10 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)
 		}
 		pr_cont(" T(%ld,%ld)\n", s0, s1);
 	}
-	if (SRCU_SIZING_IS_TORTURE())
-		srcu_transition_to_big(ssp);
+	if (SRCU_SIZING_IS_TORTURE()) {
+		if (!WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC))
+			srcu_transition_to_big(ssp);
+	}
 }
 EXPORT_SYMBOL_GPL(srcu_torture_stats_print);
 
@@ -2135,6 +2501,12 @@ static void srcu_module_going(struct module *mod)
 	struct srcu_struct *ssp;
 	struct srcu_struct **sspp = mod->srcu_struct_ptrs;
 
+	/*
+	 * Deferral skips check_init_srcu_struct(), so cleanup_srcu_struct()
+	 * below can be skipped for an srcu_struct that has staged callbacks.
+	 * Drain them before any ->sda is freed.
+	 */
+	srcu_defer_flush();
 	for (i = 0; i < mod->num_srcu_structs; i++) {
 		ssp = *(sspp++);
 		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 627295396cd91d..fcac7361ec51ec 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -178,6 +178,8 @@ static int rcu_task_contend_lim __read_mostly = 100;
 module_param(rcu_task_contend_lim, int, 0444);
 static int rcu_task_collapse_lim __read_mostly = 10;
 module_param(rcu_task_collapse_lim, int, 0444);
+static bool rcu_task_collapse_debug __read_mostly;
+module_param(rcu_task_collapse_debug, bool, 0644);
 static int rcu_task_lazy_lim __read_mostly = 32;
 module_param(rcu_task_lazy_lim, int, 0444);
 
@@ -390,7 +392,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,
 			WRITE_ONCE(rtp->percpu_enqueue_shift, 0);
 			WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids);
 			smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids);
-			pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
+			if (data_race(rcu_task_collapse_debug))
+				pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
 		}
 		raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
 	}
@@ -511,7 +514,9 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)
 			smp_store_release(&rtp->percpu_enqueue_lim, 1);
 			rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
 			gpdone = false;
-			pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name);
+			if (data_race(rcu_task_collapse_debug))
+				pr_info("Starting switch %s to CPU-0 callback queuing.\n",
+					rtp->name);
 		}
 		raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
 	}
@@ -519,7 +524,9 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)
 		raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
 		if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) {
 			WRITE_ONCE(rtp->percpu_dequeue_lim, 1);
-			pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name);
+			if (data_race(rcu_task_collapse_debug))
+				pr_info("Completing switch %s to CPU-0 callback queuing.\n",
+					rtp->name);
 		}
 		if (rtp->percpu_dequeue_lim == 1) {
 			for (cpu = rtp->percpu_dequeue_lim; cpu < rcu_task_cpu_ids; cpu++) {
@@ -704,6 +711,8 @@ static void __init rcu_tasks_bootup_oddness(void)
 		pr_info("\tTasks-RCU CPU stall info multiplier clamped to %d (rcu_task_stall_info_mult).\n", rtsimc);
 		rcu_task_stall_info_mult = rtsimc;
 	}
+	if (rcu_task_collapse_debug)
+		pr_info("\tTasks-RCU callback contend/collapse debug enabled.\n");
 #endif /* #ifdef CONFIG_TASKS_RCU */
 #ifdef CONFIG_TASKS_RCU
 	pr_info("\tTrampoline variant of Tasks RCU enabled.\n");
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index dccccd6be94111..656b6a682e31a0 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -11,6 +11,8 @@
  */
 #include <linux/completion.h>
 #include <linux/interrupt.h>
+#include <linux/irq_work.h>
+#include <linux/llist.h>
 #include <linux/notifier.h>
 #include <linux/rcupdate_wait.h>
 #include <linux/kernel.h>
@@ -42,8 +44,100 @@ static struct rcu_ctrlblk rcu_ctrlblk = {
 	.gp_seq		= 0 - 300UL,
 };
 
+/*
+ * The callback list is only accessed with interrupts disabled, so a call_rcu()
+ * that arrives with interrupts off stages the callback on a lockless list that
+ * an irq_work re-issues later.  One global list and irq_work suffice, as Tiny
+ * RCU is uniprocessor.
+ */
+static void rcu_defer_drain(struct irq_work *iw);
+static LLIST_HEAD(rcu_defer_list);
+static struct irq_work rcu_defer_iw = IRQ_WORK_INIT_HARD(rcu_defer_drain);
+static bool rcu_defer_draining;
+
+/*
+ * Also called by __rcu_defer_drain() to re-issue a deferred callback, so it
+ * must not re-check the deferral condition.
+ */
+static void rcu_do_enqueue(struct rcu_head *head, rcu_callback_t func)
+{
+	static atomic_t doublefrees;
+	unsigned long flags;
+
+	if (debug_rcu_head_queue(head)) {
+		if (atomic_inc_return(&doublefrees) < 4) {
+			pr_err("%s(): Double-freed CB %p->%pS()!!!  ", __func__, head, head->func);
+			mem_dump_obj(head);
+		}
+		return;
+	}
+
+	head->func = func;
+	head->next = NULL;
+
+	local_irq_save(flags);
+	*rcu_ctrlblk.curtail = head;
+	rcu_ctrlblk.curtail = &head->next;
+	local_irq_restore(flags);
+}
+
+/* Force scheduling for rcu_qs() when enqueuing from the idle task. */
+static void rcu_resched_if_idle(void)
+{
+	if (unlikely(is_idle_task(current)))
+		resched_cpu(0);
+}
+
+static void __rcu_defer_drain(void)
+{
+	struct llist_node *node, *next;
+	bool drained = false;
+	unsigned long flags;
+
+	if (!IS_ENABLED(CONFIG_RCU_DEFER))
+		return;
+
+	/* Re-issued newest-first; nothing depends on call_rcu() ordering. */
+	local_irq_save(flags);
+	llist_for_each_safe(node, next, llist_del_all(&rcu_defer_list)) {
+		struct rcu_head *head = (struct rcu_head *)node;
+
+		/* Bounds a node self-linked by a double call_rcu(). */
+		head->next = NULL;
+		rcu_do_enqueue(head, head->func);
+		drained = true;
+	}
+	local_irq_restore(flags);
+
+	if (drained)
+		rcu_resched_if_idle();
+}
+
+/* Only the irq_work drain can be re-fed by its own re-issue; see Tree RCU. */
+static void rcu_defer_drain(struct irq_work *iw)
+{
+	WRITE_ONCE(rcu_defer_draining, true);
+	__rcu_defer_drain();
+	WRITE_ONCE(rcu_defer_draining, false);
+}
+
+static void call_rcu_defer(struct rcu_head *head, rcu_callback_t func)
+{
+	/* A re-entrant call_rcu() during the drain would livelock it; drop it. */
+	if (READ_ONCE(rcu_defer_draining) && !in_nmi()) {
+		WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU),
+			  "call_rcu() re-entered during callback drain; leaking callback\n");
+		return;
+	}
+	head->func = func;
+	if (llist_add((struct llist_node *)head, &rcu_defer_list))
+		irq_work_queue(&rcu_defer_iw);
+}
+
 void rcu_barrier(void)
 {
+	/* Register any deferred callbacks so the wait below covers them. */
+	__rcu_defer_drain();
 	wait_rcu_gp(call_rcu_hurry);
 }
 EXPORT_SYMBOL(rcu_barrier);
@@ -157,29 +251,19 @@ EXPORT_SYMBOL_GPL(synchronize_rcu);
  */
 void call_rcu(struct rcu_head *head, rcu_callback_t func)
 {
-	static atomic_t doublefrees;
-	unsigned long flags;
-
-	if (debug_rcu_head_queue(head)) {
-		if (atomic_inc_return(&doublefrees) < 4) {
-			pr_err("%s(): Double-freed CB %p->%pS()!!!  ", __func__, head, head->func);
-			mem_dump_obj(head);
-		}
+	if (should_rcu_defer()) {
+		call_rcu_defer(head, func);
 		return;
 	}
 
-	head->func = func;
-	head->next = NULL;
+	/*
+	 * Only reachable from an NMI when deferral is off: before the scheduler
+	 * is up, or with CONFIG_RCU_DEFER=n.  The enqueue can then race.
+	 */
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
 
-	local_irq_save(flags);
-	*rcu_ctrlblk.curtail = head;
-	rcu_ctrlblk.curtail = &head->next;
-	local_irq_restore(flags);
-
-	if (unlikely(is_idle_task(current))) {
-		/* force scheduling for rcu_qs() */
-		resched_cpu(0);
-	}
+	rcu_do_enqueue(head, func);
+	rcu_resched_if_idle();
 }
 EXPORT_SYMBOL_GPL(call_rcu);
 
@@ -211,10 +295,7 @@ unsigned long start_poll_synchronize_rcu(void)
 {
 	unsigned long gp_seq = get_state_synchronize_rcu();
 
-	if (unlikely(is_idle_task(current))) {
-		/* force scheduling for rcu_qs() */
-		resched_cpu(0);
-	}
+	rcu_resched_if_idle();
 	return gp_seq;
 }
 EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu);
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 96848fc1f02b8f..aea3aef7397ecd 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -24,6 +24,7 @@
 #include <linux/smp.h>
 #include <linux/rcupdate_wait.h>
 #include <linux/interrupt.h>
+#include <linux/llist.h>
 #include <linux/sched.h>
 #include <linux/sched/debug.h>
 #include <linux/nmi.h>
@@ -64,7 +65,6 @@
 #include <linux/mm.h>
 #include <linux/kasan.h>
 #include <linux/context_tracking.h>
-#include "../time/tick-internal.h"
 
 #include "tree.h"
 #include "rcu.h"
@@ -3148,21 +3148,19 @@ static void check_cb_ovld(struct rcu_data *rdp)
 	raw_spin_unlock_rcu_node(rnp);
 }
 
-static void
-__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in)
+/*
+ * Also called by __rcu_defer_drain() to re-issue a deferred callback, so it
+ * must not re-check the deferral condition.  Either caller may have interrupts
+ * already disabled, and a drain of a remote CPU re-issues onto the draining
+ * CPU.
+ */
+static void rcu_do_enqueue(struct rcu_head *head, rcu_callback_t func, bool lazy_in)
 {
 	static atomic_t doublefrees;
 	unsigned long flags;
 	bool lazy;
 	struct rcu_data *rdp;
 
-	/* Misaligned rcu_head! */
-	WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
-
-	/* Avoid NULL dereference if callback is NULL. */
-	if (WARN_ON_ONCE(!func))
-		return;
-
 	if (debug_rcu_head_queue(head)) {
 		/*
 		 * Probable double call_rcu(), so leak the callback.
@@ -3178,7 +3176,6 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in)
 	}
 	head->func = func;
 	head->next = NULL;
-	kasan_record_aux_stack(head);
 
 	local_irq_save(flags);
 	rdp = this_cpu_ptr(&rcu_data);
@@ -3206,6 +3203,103 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in)
 	local_irq_restore(flags);
 }
 
+/*
+ * Re-issue deferred callbacks straight to the enqueue so they cannot defer
+ * again.  ->defer_lock serializes the drainers: this CPU's irq_work,
+ * rcu_defer_flush() and rcutree_migrate_callbacks().
+ */
+static void __rcu_defer_drain(struct rcu_data *rdp)
+{
+	struct llist_node *node, *next;
+	unsigned long flags;
+
+	if (!IS_ENABLED(CONFIG_RCU_DEFER))
+		return;
+
+	raw_spin_lock_irqsave(&rdp->defer_lock, flags);
+	llist_for_each_safe(node, next, llist_del_all(&rdp->defer_head)) {
+		struct rcu_head *head = (struct rcu_head *)node;
+
+		/* Bounds a node self-linked by a double call_rcu(). */
+		head->next = NULL;
+		rcu_do_enqueue(head, head->func, false);
+	}
+	raw_spin_unlock_irqrestore(&rdp->defer_lock, flags);
+}
+
+/*
+ * Only the irq_work drain can be re-fed by its own re-issue, so only it sets
+ * ->defer_draining.  Anything staged during a direct drain is picked up by the
+ * staging CPU's own irq_work.  Every caller of irq_work_run_list() has
+ * interrupts disabled, so the flag is never visible with them enabled.
+ */
+static void rcu_defer_drain(struct irq_work *iw)
+{
+	struct rcu_data *rdp = container_of(iw, struct rcu_data, defer_work);
+
+	WRITE_ONCE(rdp->defer_draining, true);
+	__rcu_defer_drain(rdp);
+	WRITE_ONCE(rdp->defer_draining, false);
+}
+
+/*
+ * Stage @head for this CPU's irq_work to re-issue once interrupts are on.  Only
+ * the drain side takes a lock, so this stays safe from NMI.
+ */
+static void call_rcu_defer(struct rcu_head *head, rcu_callback_t func)
+{
+	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
+
+	/*
+	 * Instrumentation on the enqueue path can re-enter here from inside the
+	 * drain.  Re-queuing would livelock it, so drop the callback; an NMI
+	 * cannot loop, so let it through.
+	 */
+	if (READ_ONCE(rdp->defer_draining) && !in_nmi()) {
+		WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU),
+			  "call_rcu() re-entered during callback drain; leaking callback\n");
+		return;
+	}
+	head->func = func;
+	if (llist_add((struct llist_node *)head, &rdp->defer_head))
+		irq_work_queue(&rdp->defer_work);
+}
+
+static void rcu_defer_flush(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu)
+		__rcu_defer_drain(per_cpu_ptr(&rcu_data, cpu));
+}
+
+static void
+__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in)
+{
+	/* Misaligned rcu_head! */
+	WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
+
+	/* Avoid NULL dereference if callback is NULL. */
+	if (WARN_ON_ONCE(!func))
+		return;
+
+	/* Record the caller: the irq_work's stack says nothing about it. */
+	kasan_record_aux_stack(head);
+
+	if (should_rcu_defer()) {
+		call_rcu_defer(head, func);
+		return;
+	}
+
+	/*
+	 * Only reachable from an NMI when deferral is off: before the scheduler
+	 * is up, or with CONFIG_RCU_DEFER=n.  The enqueue can then race.
+	 */
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
+
+	rcu_do_enqueue(head, func, lazy_in);
+}
+
 #ifdef CONFIG_RCU_LAZY
 static bool enable_rcu_lazy __read_mostly = !IS_ENABLED(CONFIG_RCU_LAZY_DEFAULT_OFF);
 module_param(enable_rcu_lazy, bool, 0444);
@@ -3896,8 +3990,12 @@ void rcu_barrier(void)
 	unsigned long flags;
 	unsigned long gseq;
 	struct rcu_data *rdp;
-	unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
+	unsigned long s;
 
+	/* Register any deferred callbacks before snapshotting the sequence. */
+	rcu_defer_flush();
+
+	s = rcu_seq_snap(&rcu_state.barrier_sequence);
 	rcu_barrier_trace(TPS("Begin"), -1, s);
 
 	/* Take mutex to serialize concurrent rcu_barrier() requests. */
@@ -3989,12 +4087,12 @@ EXPORT_SYMBOL_GPL(rcu_barrier);
 static unsigned long rcu_barrier_last_throttle;
 
 /**
- * rcu_barrier_throttled - Do rcu_barrier(), but limit to one per second
+ * rcu_barrier_throttled - Drain deferred RCU frees, but rate-limit starts
  *
- * This can be thought of as guard rails around rcu_barrier() that
- * permits unrestricted userspace use, at least assuming the hardware's
- * try_cmpxchg() is robust.  There will be at most one call per second to
- * rcu_barrier() system-wide from use of this function, which means that
+ * This can be thought of as guard rails around the deferred-free barriers
+ * that permit unrestricted userspace use, at least assuming the hardware's
+ * try_cmpxchg() is robust.  There will be at most one drain operation started
+ * per sixteenth of a second from use of this function, which means that
  * callers might needlessly wait a second or three.
  *
  * This is intended for use by test suites to avoid OOM by flushing RCU
@@ -4016,14 +4114,24 @@ static void rcu_barrier_throttled(void)
 	while (time_in_range(j, old, old + HZ / 16) ||
 	       !try_cmpxchg(&rcu_barrier_last_throttle, &old, j)) {
 		schedule_timeout_idle(HZ / 16);
-		if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
-			smp_mb(); /* caller's subsequent code after above check. */
-			return;
-		}
 		j = jiffies;
 		old = READ_ONCE(rcu_barrier_last_throttle);
 	}
-	rcu_barrier();
+	/*
+	 * kfree_rcu() can retain objects outside the ordinary callback lists in
+	 * per-CPU SLUB sheaves and kvfree_rcu batches.  Always drain those queues:
+	 * an ordinary barrier does not establish that this work was drained.
+	 */
+	kvfree_rcu_barrier();
+	/*
+	 * A completed barrier can still cover ordinary callbacks queued before
+	 * our entry snapshot.  Otherwise, retain an explicit ordinary barrier
+	 * without depending on the implementation of kvfree_rcu_barrier().
+	 */
+	if (rcu_seq_done(&rcu_state.barrier_sequence, s))
+		smp_mb(); /* caller's subsequent code after above check. */
+	else
+		rcu_barrier();
 }
 
 /*
@@ -4231,6 +4339,9 @@ rcu_boot_init_percpu_data(int cpu)
 	rdp->rcu_onl_gp_state = RCU_GP_CLEANED;
 	rdp->last_sched_clock = jiffies;
 	rdp->cpu = cpu;
+	init_llist_head(&rdp->defer_head);
+	raw_spin_lock_init(&rdp->defer_lock);
+	rdp->defer_work = IRQ_WORK_INIT_HARD(rcu_defer_drain);
 	rcu_boot_init_nocb_percpu_data(rdp);
 }
 
@@ -4528,6 +4639,15 @@ void rcutree_migrate_callbacks(int cpu)
 	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
 	bool needwake;
 
+	/*
+	 * Callbacks deferred past the point the outgoing CPU's irq_work can run
+	 * sit on ->defer_head, which the ->cblist migration below does not
+	 * cover.  Drain them here, before the early returns.
+	 */
+	__rcu_defer_drain(rdp);
+	/* Likewise for the outgoing CPU's deferred call_srcu() callbacks. */
+	srcu_offline_drain(cpu);
+
 	if (rcu_rdp_is_offloaded(rdp))
 		return;
 
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index eedfa43059e802..b7cac7a13b4f22 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -229,6 +229,12 @@ struct rcu_data {
 	struct rcu_head barrier_head;
 	int exp_watching_snap;		/* Double-check need for IPI. */
 
+	/* Deferral of an NMI/reentrant call_rcu(); see __call_rcu_common(). */
+	struct llist_head defer_head;
+	struct irq_work defer_work;
+	raw_spinlock_t defer_lock;
+	bool defer_draining;
+
 	/* 5) Callback offloading. */
 #ifdef CONFIG_RCU_NOCB_CPU
 	struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 81d42b6e2e312d..01bdbd7e28cca7 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -678,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void)
 
 		nbcon_cpu_emergency_exit();
 
-		panic_on_rcu_stall(&rcu_exp_stall_cpumask);
+		panic_on_rcu_stall(&rcu_exp_stall_cpumask, NULL, 0);
 	}
 }
 
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index 19bb42672baf8c..b6cfee4e503997 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -1332,9 +1332,8 @@ lazy_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
 		rcu_nocb_try_flush_bypass(rdp, jiffies);
 		rcu_nocb_unlock_irqrestore(rdp, flags);
 		wake_nocb_gp(rdp);
-		sc->nr_to_scan -= _count;
 		count += _count;
-		if (sc->nr_to_scan <= 0)
+		if (count >= sc->nr_to_scan)
 			break;
 	}
 
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index 091e7850ab6ecc..5dded1e8919759 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -11,6 +11,7 @@
 #include <linux/kvm_para.h>
 #include <linux/rcu_notifier.h>
 #include <linux/smp.h>
+#include <linux/rtmutex.h>
 
 //////////////////////////////////////////////////////////////////////////////
 //
@@ -159,7 +160,8 @@ static int __init check_cpu_stall_init(void)
 early_initcall(check_cpu_stall_init);
 
 /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */
-static void panic_on_rcu_stall(const struct cpumask *stalled_mask)
+static void panic_on_rcu_stall(const struct cpumask *stalled_mask,
+			       unsigned long *gsp, unsigned long gp_seq)
 {
 	static int cpu_stall;
 
@@ -170,6 +172,11 @@ static void panic_on_rcu_stall(const struct cpumask *stalled_mask)
 	if (scx_rcu_cpu_stall(stalled_mask))
 		return;
 
+	if (gsp && rcu_seq_current(gsp) != gp_seq) {
+		pr_err("INFO: Stall ended before panic check.\n");
+		return;
+	}
+
 	if (++cpu_stall < sysctl_max_rcu_stall_to_panic)
 		return;
 
@@ -299,6 +306,8 @@ struct rcu_stall_chk_rdr {
 	int nesting;
 	union rcu_special rs;
 	bool on_blkd_list;
+	bool rcu_rdr_running;
+	int rcu_rdr_boosted;
 };
 
 /*
@@ -307,6 +316,7 @@ struct rcu_stall_chk_rdr {
  */
 static int check_slow_task(struct task_struct *t, void *arg)
 {
+	struct rcu_node *rnp;
 	struct rcu_stall_chk_rdr *rscrp = arg;
 
 	if (task_curr(t))
@@ -314,6 +324,19 @@ static int check_slow_task(struct task_struct *t, void *arg)
 	rscrp->nesting = t->rcu_read_lock_nesting;
 	rscrp->rs = t->rcu_read_unlock_special;
 	rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry);
+	rscrp->rcu_rdr_running = task_curr(t);
+	rscrp->rcu_rdr_boosted = 0;
+	if (rscrp->on_blkd_list) {
+		rnp = READ_ONCE(t->rcu_blocked_node);
+		raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
+		if (rnp == READ_ONCE(t->rcu_blocked_node)) {
+			if (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t)
+				rscrp->rcu_rdr_boosted = 1;
+		} else {
+			rscrp->rcu_rdr_boosted = 2;
+		}
+		raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
+	}
 	return 0;
 }
 
@@ -351,12 +374,14 @@ static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags)
 		if (task_call_func(t, check_slow_task, &rscr))
 			pr_cont(" P%d", t->pid);
 		else
-			pr_cont(" P%d/%d:%c%c%c%c",
+			pr_cont(" P%d/%d:%c%c%c%c%c%c",
 				t->pid, rscr.nesting,
 				".b"[rscr.rs.b.blocked],
 				".q"[rscr.rs.b.need_qs],
 				".e"[rscr.rs.b.exp_hint],
-				".l"[rscr.on_blkd_list]);
+				".l"[rscr.on_blkd_list],
+				".R"[rscr.rcu_rdr_running],
+				".B?"[rscr.rcu_rdr_boosted]);
 		lockdep_assert_irqs_disabled();
 		put_task_struct(t);
 		ndetected++;
@@ -703,7 +728,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps)
 
 	nbcon_cpu_emergency_exit();
 
-	panic_on_rcu_stall(&rcu_stall_cpumask);
+	panic_on_rcu_stall(&rcu_stall_cpumask, &rcu_state.gp_seq, gp_seq);
 
 	rcu_force_quiescent_state();  /* Kick them all. */
 }
@@ -758,7 +783,7 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps)
 
 	cpumask_clear(&rcu_stall_cpumask);
 	cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask);
-	panic_on_rcu_stall(&rcu_stall_cpumask);
+	panic_on_rcu_stall(&rcu_stall_cpumask, &rcu_state.gp_seq, gp_seq);
 
 	/*
 	 * Attempt to revive the RCU machinery by forcing a context switch.
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index b62735a6788423..2a778b8ab4ad78 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -44,6 +44,7 @@
 #include <linux/slab.h>
 #include <linux/irq_work.h>
 #include <linux/rcupdate_trace.h>
+#include <linux/torture.h>
 
 #define CREATE_TRACE_POINTS
 
@@ -525,7 +526,7 @@ EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
 	do { } while (0)
 #endif
 
-#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST)
+#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_ENABLED(CONFIG_HAZPTR_TORTURE_TEST)
 /* Get rcutorture access to sched_setaffinity(). */
 long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn)
 {
diff --git a/kernel/reboot.c b/kernel/reboot.c
index d177d89fcc33e2..535d9d5b2d220b 100644
--- a/kernel/reboot.c
+++ b/kernel/reboot.c
@@ -11,7 +11,6 @@
 #include <linux/ctype.h>
 #include <linux/export.h>
 #include <linux/kexec.h>
-#include <linux/kmod.h>
 #include <linux/kmsg_dump.h>
 #include <linux/rcupdate.h>
 #include <linux/reboot.h>
@@ -20,6 +19,7 @@
 #include <linux/syscalls.h>
 #include <linux/syscore_ops.h>
 #include <linux/uaccess.h>
+#include <linux/umh.h>
 
 /*
  * this indicates whether you can reboot with ctrl-alt-del: the default is yes
diff --git a/kernel/resource.c b/kernel/resource.c
index e60539a55541df..cfc1a00e86aa03 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -1350,8 +1350,8 @@ static int __request_region_locked(struct resource *res, struct resource *parent
 		}
 		if (conflict->flags & flags & IORESOURCE_MUXED) {
 			add_wait_queue(&muxed_resource_wait, &wait);
-			write_unlock(&resource_lock);
 			set_current_state(TASK_UNINTERRUPTIBLE);
+			write_unlock(&resource_lock);
 			schedule();
 			remove_wait_queue(&muxed_resource_wait, &wait);
 			write_lock(&resource_lock);
@@ -1833,7 +1833,7 @@ __setup("reserve=", reserve_setup);
  */
 int iomem_map_sanity_check(resource_size_t addr, unsigned long size)
 {
-	resource_size_t end = addr + size - 1;
+	struct resource mem = DEFINE_RES_MEM(addr, size);
 	struct resource *p;
 	int err = 0;
 
@@ -1843,12 +1843,10 @@ int iomem_map_sanity_check(resource_size_t addr, unsigned long size)
 		 * We can probably skip the resources without
 		 * IORESOURCE_IO attribute?
 		 */
-		if (p->start > end)
-			continue;
-		if (p->end < addr)
+		if (!resource_overlaps(p, &mem))
 			continue;
-		if (PFN_DOWN(p->start) <= PFN_DOWN(addr) &&
-		    PFN_DOWN(p->end) >= PFN_DOWN(end))
+		if (PFN_DOWN(p->start) <= PFN_DOWN(mem.start) &&
+		    PFN_DOWN(p->end) >= PFN_DOWN(mem.end))
 			continue;
 		/*
 		 * if a resource is "BUSY", it's not a hardware resource
@@ -1859,8 +1857,8 @@ int iomem_map_sanity_check(resource_size_t addr, unsigned long size)
 		if (p->flags & IORESOURCE_BUSY)
 			continue;
 
-		pr_debug("resource sanity check: requesting [mem %pa-%pa], which spans more than %s %pR\n",
-			&addr, &end, p->name, p);
+		pr_debug("resource sanity check: requesting %pR, which spans more than %s %pR\n",
+			 &mem, p->name, p);
 		err = -1;
 		break;
 	}
diff --git a/kernel/resource_kunit.c b/kernel/resource_kunit.c
index 42785796f1dbc5..9eeb2b1a85c01c 100644
--- a/kernel/resource_kunit.c
+++ b/kernel/resource_kunit.c
@@ -225,6 +225,9 @@ static void resource_test_region_intersects(struct kunit *test)
 	struct resource *parent;
 	resource_size_t start;
 
+	if (!IS_ENABLED(CONFIG_GET_FREE_REGION))
+		kunit_skip(test, "CONFIG_GET_FREE_REGION is disabled");
+
 	/* Find an iomem_resource hole to hold test resources */
 	parent = alloc_free_mem_region(&iomem_resource, RES_TEST_TOTAL_SIZE, SZ_1M,
 				       "test resources");
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 7885ff76e69f28..d230a3b1974b09 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -59,6 +59,7 @@
 #include <linux/profile.h>
 #include <linux/psi.h>
 #include <linux/rcuwait_api.h>
+#include <linux/hazptr.h>
 #include <linux/rseq.h>
 #include <linux/sched/wake_q.h>
 #include <linux/scs.h>
@@ -805,7 +806,7 @@ struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf)
 
 /* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */
 #ifdef CONFIG_PARAVIRT
-struct static_key paravirt_steal_rq_enabled;
+DEFINE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled);
 #endif
 
 static void update_rq_clock_task(struct rq *rq, s64 delta)
@@ -844,7 +845,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
 	}
 #endif
 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
-	if (static_key_false((&paravirt_steal_rq_enabled))) {
+	if (static_branch_unlikely(&paravirt_steal_rq_enabled)) {
 		u64 prev_steal;
 
 		steal = prev_steal = paravirt_steal_clock(cpu_of(rq));
@@ -2252,7 +2253,8 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 	dequeue_task(rq, p, flags);
 }
 
-static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
+static bool dequeue_block_task(struct rq *rq, struct task_struct *p,
+			       unsigned long task_state)
 {
 	int flags = DEQUEUE_NOCLOCK;
 
@@ -2273,9 +2275,15 @@ static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_
 	 *
 	 * Where __schedule() and ttwu() have matching control dependencies.
 	 *
-	 * After this, schedule() must not care about p->state any more.
+	 * Once the caller invokes __block_task(), schedule() must not care about
+	 * p->state any more.
 	 */
-	if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags))
+	return dequeue_task(rq, p, DEQUEUE_SLEEP | flags);
+}
+
+static void block_task(struct rq *rq, struct task_struct *p, unsigned long task_state)
+{
+	if (dequeue_block_task(rq, p, task_state))
 		__block_task(rq, p);
 }
 
@@ -2504,6 +2512,24 @@ static inline bool rq_has_pinned_tasks(struct rq *rq)
 	return rq->nr_pinned;
 }
 
+static inline bool task_can_migrate_to_preferred(struct task_struct *p, int cpu)
+{
+	/* No need to migrate from a preferred CPU */
+	if (cpu_preferred(cpu))
+		return false;
+
+	/* Only FAIR tasks honor preferred CPU state */
+	if (unlikely(p->sched_class != &fair_sched_class))
+		return false;
+
+	/* Ignore preferred state if task affinity is changing */
+	if (unlikely(!cpumask_test_cpu(task_cpu(p), p->cpus_ptr)))
+		return false;
+
+	return cpumask_intersects_and(p->cpus_ptr, cpu_preferred_mask,
+				      task_cpu_possible_mask(p));
+}
+
 /*
  * Per-CPU kthreads are allowed to run on !active && online CPUs, see
  * __set_cpus_allowed_ptr() and select_fallback_rq().
@@ -2519,8 +2545,12 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
 		return cpu_online(cpu);
 
 	/* Non kernel threads are not allowed during either online or offline. */
-	if (!(p->flags & PF_KTHREAD))
+	if (!(p->flags & PF_KTHREAD)) {
+		/* Try to use preferred CPU if task's affinity allows */
+		if (task_can_migrate_to_preferred(p, cpu))
+			return false;
 		return cpu_active(cpu);
+	}
 
 	/* KTHREAD_IS_PER_CPU is always allowed. */
 	if (kthread_is_per_cpu(p))
@@ -2530,7 +2560,11 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
 	if (cpu_dying(cpu))
 		return false;
 
-	/* But are allowed during online. */
+	/* Try to keep unbound kthreads on a preferred CPU if possible. */
+	if (task_can_migrate_to_preferred(p, cpu))
+		return false;
+
+	/* Otherwise, they are allowed to run on online CPU. */
 	return cpu_online(cpu);
 }
 
@@ -3351,6 +3385,8 @@ void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 {
 	unsigned int state = READ_ONCE(p->__state);
+	bool proxy_migrated = sched_proxy_exec() && p->is_blocked &&
+			      task_cpu(p) != p->wake_cpu;
 
 	/*
 	 * We should never call set_task_cpu() on a blocked task,
@@ -3386,7 +3422,12 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 	 */
 	WARN_ON_ONCE(!cpu_online(new_cpu));
 
-	WARN_ON_ONCE(is_migration_disabled(p));
+	/*
+	 * Proxy execution can move a blocked task's scheduling context to any
+	 * CPU without moving its migration-disabled execution context. The
+	 * wakeup path will return the task to a CPU where it can execute.
+	 */
+	WARN_ON_ONCE(is_migration_disabled(p) && !proxy_migrated);
 
 	trace_sched_migrate_task(p, new_cpu);
 
@@ -3766,6 +3807,7 @@ static inline void proxy_reset_donor(struct rq *rq)
 	WARN_ON_ONCE(rq->donor == rq->curr);
 
 	put_prev_set_next_task(rq, rq->donor, rq->curr);
+	rq->next_class = rq->curr->sched_class;
 	rq_set_donor(rq, rq->curr);
 	zap_balance_callbacks(rq);
 	resched_curr(rq);
@@ -3780,6 +3822,8 @@ static inline void proxy_reset_donor(struct rq *rq)
  */
 static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
 {
+	bool dequeued;
+
 	/*
 	 * Typically per __set_task_cpu(), task_cpu(p) == p->wake_cpu.
 	 *
@@ -3802,12 +3846,23 @@ static inline bool proxy_needs_return(struct rq *rq, struct task_struct *p)
 		/* If already current, don't need to return migrate */
 		if (task_current(rq, p))
 			return false;
-
-		/* If we're return migrating the rq->donor, switch it out for idle */
-		if (task_current_donor(rq, p))
-			proxy_reset_donor(rq);
 	}
-	block_task(rq, p, TASK_WAKING);
+
+	dequeued = dequeue_block_task(rq, p, TASK_WAKING);
+
+	/*
+	 * Dequeue @p from its scheduling class before resetting rq->donor.
+	 * In particular, sched_ext needs to end the donor's running session
+	 * and clear SCX_TASK_QUEUED before put_prev_task_scx() is called by
+	 * proxy_reset_donor(); otherwise it would reenqueue the blocked donor.
+	 *
+	 * Keep on_rq set until all donor references have been replaced.
+	 */
+	if (task_current_donor(rq, p))
+		proxy_reset_donor(rq);
+
+	if (dequeued)
+		__block_task(rq, p);
 	return true;
 }
 #else /* !CONFIG_SCHED_PROXY_EXEC */
@@ -3898,7 +3953,7 @@ static int ttwu_runnable(struct task_struct *p, int wake_flags)
 		 * When on_rq && !on_cpu the task is preempted, see if
 		 * it should preempt the task that is current now.
 		 */
-		wakeup_preempt(rq, p, wake_flags);
+		wakeup_preempt(rq, p, wake_flags | WF_TTWU_RQ);
 	}
 	ttwu_do_wakeup(p);
 	return 1;
@@ -5142,7 +5197,7 @@ static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)
 	lockdep_assert_rq_held(rq);
 
 	while (head) {
-		func = (void (*)(struct rq *))head->func;
+		func = head->func;
 		next = head->next;
 		head->next = NULL;
 		head = next;
@@ -5782,6 +5837,9 @@ void sched_tick(void)
 	unsigned long hw_pressure;
 	u64 resched_latency;
 
+	if (!cpu_preferred(cpu))
+		sched_push_current_non_preferred_cpu(rq);
+
 	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
 		arch_scale_freq_tick();
 
@@ -5791,7 +5849,7 @@ void sched_tick(void)
 	curr = rq->curr;
 	donor = rq->donor;
 
-	psi_account_irqtime(rq, donor, NULL);
+	psi_account_irqtime(rq, curr, NULL);
 
 	update_rq_clock(rq);
 	hw_pressure = arch_scale_hw_pressure(cpu_of(rq));
@@ -6276,10 +6334,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
 	 * selection. In this case, do a core-wide selection.
 	 */
 	if (rq->core->core_pick_seq == rq->core->core_task_seq &&
-	    rq->core->core_pick_seq != rq->core_sched_seq &&
 	    rq->core_pick) {
-		WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);
-
 		next = rq->core_pick;
 		rq->dl_server = rq->core_dl_server;
 		rq->core_pick = NULL;
@@ -6311,11 +6366,13 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
 	}
 
 	/*
-	 * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq
+	 * core->core_task_seq, core->core_pick_seq
 	 *
 	 * @task_seq guards the task state ({en,de}queues)
 	 * @pick_seq is the @task_seq we did a selection on
-	 * @sched_seq is the @pick_seq we scheduled
+	 *
+	 * Once a core-wide selection is committed, a non-NULL core_pick denotes
+	 * a pick which still needs to be consumed on this CPU.
 	 *
 	 * However, preemptions can cause multiple picks on the same task set.
 	 * 'Fix' this by also increasing @task_seq for every pick.
@@ -6422,7 +6479,6 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
 
 	rq->core->core_pick_seq = rq->core->core_task_seq;
 	next = rq->core_pick;
-	rq->core_sched_seq = rq->core->core_pick_seq;
 
 	/* Something should have been selected for current CPU */
 	WARN_ON_ONCE(!next);
@@ -6510,7 +6566,10 @@ static bool try_steal_cookie(int this, int that)
 		return false;
 
 	do {
-		if (p == src->core_pick || p == src->curr)
+		if (p == src->core_pick || p == src->curr || p == src->donor)
+			goto next;
+
+		if (task_is_blocked(p))
 			goto next;
 
 		if (!is_cpu_allowed(p, this))
@@ -6813,6 +6872,34 @@ static void proxy_deactivate(struct rq *rq, struct task_struct *donor)
 	block_task(rq, donor, state);
 }
 
+/*
+ * Remove a retained proxy donor before changing its scheduler ownership.
+ * The caller holds p->pi_lock, so p cannot wake and migrate if block_task()
+ * drops it from the runqueue. If DELAY_DEQUEUE keeps a blocked fair task
+ * queued, switching_from_fair() completes the dequeue in the immediately
+ * following sched_change_begin().
+ */
+void sched_proxy_block_task(struct rq *rq, struct task_struct *p)
+{
+	unsigned long state = READ_ONCE(p->__state);
+
+	lockdep_assert_held(&p->pi_lock);
+	lockdep_assert_rq_held(rq);
+
+	if (!p->is_blocked || !task_on_rq_queued(p))
+		return;
+	if (WARN_ON_ONCE(state == TASK_RUNNING))
+		return;
+
+	if (task_current_donor(rq, p))
+		proxy_reset_donor(rq);
+
+	if (!p->se.sched_delayed)
+		block_task(rq, p, state);
+
+	WARN_ON_ONCE(task_on_rq_queued(p) && !p->se.sched_delayed);
+}
+
 static inline void proxy_release_rq_lock(struct rq *rq, struct rq_flags *rf)
 	__releases(__rq_lockp(rq))
 {
@@ -6858,9 +6945,9 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
 	__must_hold(__rq_lockp(rq))
 {
 	struct rq *target_rq = cpu_rq(target_cpu);
+	LIST_HEAD(migrate_list);
 
 	lockdep_assert_rq_held(rq);
-	WARN_ON(p == rq->curr);
 	/*
 	 * Since we are migrating a blocked donor, it could be rq->donor,
 	 * and we want to make sure there aren't any references from this
@@ -6873,13 +6960,20 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf,
 	 * before we release the lock.
 	 */
 	proxy_resched_idle(rq);
-
-	deactivate_task(rq, p, DEQUEUE_NOCLOCK);
-	proxy_set_task_cpu(p, target_cpu);
-
+	for (; p; p = p->blocked_donor) {
+		WARN_ON(p == rq->curr);
+		deactivate_task(rq, p, DEQUEUE_NOCLOCK);
+		proxy_set_task_cpu(p, target_cpu);
+		/*
+		 * We can re-use se.group_node to migrate the thing,
+		 * because @p is deactivated (won't be balanced) and
+		 * we hold the rq_lock.
+		 */
+		list_add(&p->se.group_node, &migrate_list);
+	}
 	proxy_release_rq_lock(rq, rf);
 
-	attach_one_task(target_rq, p);
+	__attach_tasks(target_rq, &migrate_list);
 
 	proxy_reacquire_rq_lock(rq, rf);
 }
@@ -6972,7 +7066,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
 		if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
 			/* XXX Don't handle blocked owners/delayed dequeue yet */
 			if (curr_in_chain)
-				return proxy_resched_idle(rq);
+				goto resched_idle;
 			__clear_task_blocked_on(p, NULL);
 			goto deactivate;
 		}
@@ -6984,7 +7078,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
 			 * and leave that CPU to sort things out.
 			 */
 			if (curr_in_chain)
-				return proxy_resched_idle(rq);
+				goto resched_idle;
 			goto migrate_task;
 		}
 
@@ -6997,7 +7091,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
 			 * case we should end up back in find_proxy_task(), this time
 			 * hopefully with all relevant tasks already enqueued.
 			 */
-			return proxy_resched_idle(rq);
+			goto resched_idle;
 		}
 
 		/*
@@ -7034,7 +7128,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
 			 * So schedule rq->idle so that ttwu_runnable() can get the rq
 			 * lock and mark owner as running.
 			 */
-			return proxy_resched_idle(rq);
+			goto resched_idle;
 		}
 		/*
 		 * OK, now we're absolutely sure @owner is on this
@@ -7044,8 +7138,18 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
 		owner->blocked_donor = p;
 	}
 	WARN_ON_ONCE(owner && !owner->on_rq);
+
+	if (owner && !sched_cpu_cookie_match(rq, owner)) {
+		if (curr_in_chain)
+			return proxy_resched_idle(rq);
+		p = donor; /* Deactivate the donor, not the runnable owner */
+		clear_task_blocked_on(p, NULL);
+		goto deactivate;
+	}
 	return owner;
 
+resched_idle:
+	return proxy_resched_idle(rq);
 deactivate:
 	proxy_deactivate(rq, p);
 	return NULL;
@@ -7130,6 +7234,7 @@ static void __sched notrace __schedule(int sched_mode)
 	local_irq_disable();
 	rcu_note_context_switch(preempt);
 	migrate_disable_switch(rq, prev);
+	hazptr_note_context_switch();
 
 	/*
 	 * Make sure that signal_pending_state()->signal_pending() below
@@ -7177,13 +7282,12 @@ static void __sched notrace __schedule(int sched_mode)
 		}
 	} else if (!preempt && prev_state) {
 		/*
-		 * We pass task_is_blocked() as the should_block arg
-		 * in order to keep mutex-blocked tasks on the runqueue
-		 * for slection with proxy-exec (without proxy-exec
-		 * task_is_blocked() will always be false).
+		 * Keep mutex-blocked tasks on the runqueue for proxy execution
+		 * only when their scheduling class allows it. Without proxy
+		 * execution, task_is_blocked() always returns false.
 		 */
 		try_to_block_task(rq, prev, &prev_state,
-				  !task_is_blocked(prev));
+				  !task_is_blocked(prev) || !scx_allow_proxy_exec(prev));
 		switch_count = &prev->nvcsw;
 	}
 
@@ -7204,6 +7308,7 @@ static void __sched notrace __schedule(int sched_mode)
 			}
 			if (next == rq->idle) {
 				zap_balance_callbacks(rq);
+				scx_proxy_reenqueue_retry(rq, next);
 				goto keep_resched;
 			}
 		}
@@ -7222,8 +7327,10 @@ static void __sched notrace __schedule(int sched_mode)
 			 * on_cpu.
 			 */
 			donor->sched_class->put_prev_task(rq, donor, donor);
-			donor->sched_class->set_next_task(rq, donor, true);
+			donor->sched_class->set_next_task(rq, donor, SNT_PICK);
 		}
+		scx_proxy_donor_start(rq);
+		scx_proxy_reenqueue_retry(rq, next);
 	} else {
 		rq_set_donor(rq, next);
 	}
@@ -7482,27 +7589,6 @@ asmlinkage __visible void __sched notrace preempt_schedule(void)
 NOKPROBE_SYMBOL(preempt_schedule);
 EXPORT_SYMBOL(preempt_schedule);
 
-#ifdef CONFIG_PREEMPT_DYNAMIC
-# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
-#  ifndef preempt_schedule_dynamic_enabled
-#   define preempt_schedule_dynamic_enabled	preempt_schedule
-#   define preempt_schedule_dynamic_disabled	NULL
-#  endif
-DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
-EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule);
-void __sched notrace dynamic_preempt_schedule(void)
-{
-	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule))
-		return;
-	preempt_schedule();
-}
-NOKPROBE_SYMBOL(dynamic_preempt_schedule);
-EXPORT_SYMBOL(dynamic_preempt_schedule);
-# endif
-#endif /* CONFIG_PREEMPT_DYNAMIC */
-
 /**
  * preempt_schedule_notrace - preempt_schedule called by tracing
  *
@@ -7555,27 +7641,6 @@ asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
 }
 EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
 
-#ifdef CONFIG_PREEMPT_DYNAMIC
-# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-#  ifndef preempt_schedule_notrace_dynamic_enabled
-#   define preempt_schedule_notrace_dynamic_enabled	preempt_schedule_notrace
-#   define preempt_schedule_notrace_dynamic_disabled	NULL
-#  endif
-DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
-EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace);
-void __sched notrace dynamic_preempt_schedule_notrace(void)
-{
-	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace))
-		return;
-	preempt_schedule_notrace();
-}
-NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace);
-EXPORT_SYMBOL(dynamic_preempt_schedule_notrace);
-# endif
-#endif
-
 #endif /* CONFIG_PREEMPTION */
 
 /*
@@ -7792,7 +7857,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
 }
 #endif /* CONFIG_RT_MUTEXES */
 
-#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
+#if !defined(CONFIG_PREEMPTION)
 int __sched __cond_resched(void)
 {
 	if (should_resched(0) && !irqs_disabled()) {
@@ -7820,38 +7885,6 @@ int __sched __cond_resched(void)
 EXPORT_SYMBOL(__cond_resched);
 #endif
 
-#ifdef CONFIG_PREEMPT_DYNAMIC
-# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
-#  define cond_resched_dynamic_enabled	__cond_resched
-#  define cond_resched_dynamic_disabled	((void *)&__static_call_return0)
-DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched);
-EXPORT_STATIC_CALL_TRAMP(cond_resched);
-
-#  define might_resched_dynamic_enabled	__cond_resched
-#  define might_resched_dynamic_disabled ((void *)&__static_call_return0)
-DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched);
-EXPORT_STATIC_CALL_TRAMP(might_resched);
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched);
-int __sched dynamic_cond_resched(void)
-{
-	if (!static_branch_unlikely(&sk_dynamic_cond_resched))
-		return 0;
-	return __cond_resched();
-}
-EXPORT_SYMBOL(dynamic_cond_resched);
-
-static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched);
-int __sched dynamic_might_resched(void)
-{
-	if (!static_branch_unlikely(&sk_dynamic_might_resched))
-		return 0;
-	return __cond_resched();
-}
-EXPORT_SYMBOL(dynamic_might_resched);
-# endif
-#endif /* CONFIG_PREEMPT_DYNAMIC */
-
 /*
  * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
  * call schedule, and on return reacquire the lock.
@@ -7921,50 +7954,21 @@ EXPORT_SYMBOL(__cond_resched_rwlock_write);
 # endif
 
 /*
- * SC:cond_resched
- * SC:might_resched
- * SC:preempt_schedule
- * SC:preempt_schedule_notrace
- * SC:irqentry_exit_cond_resched
- *
- *
  * NONE:
- *   cond_resched               <- __cond_resched
- *   might_resched              <- RET0
- *   preempt_schedule           <- NOP
- *   preempt_schedule_notrace   <- NOP
- *   irqentry_exit_cond_resched <- NOP
- *   dynamic_preempt_lazy       <- false
+ *   (unselectable)
  *
  * VOLUNTARY:
- *   cond_resched               <- __cond_resched
- *   might_resched              <- __cond_resched
- *   preempt_schedule           <- NOP
- *   preempt_schedule_notrace   <- NOP
- *   irqentry_exit_cond_resched <- NOP
- *   dynamic_preempt_lazy       <- false
+ *   (unselectable)
  *
  * FULL:
- *   cond_resched               <- RET0
- *   might_resched              <- RET0
- *   preempt_schedule           <- preempt_schedule
- *   preempt_schedule_notrace   <- preempt_schedule_notrace
- *   irqentry_exit_cond_resched <- irqentry_exit_cond_resched
  *   dynamic_preempt_lazy       <- false
  *
  * LAZY:
- *   cond_resched               <- RET0
- *   might_resched              <- RET0
- *   preempt_schedule           <- preempt_schedule
- *   preempt_schedule_notrace   <- preempt_schedule_notrace
- *   irqentry_exit_cond_resched <- irqentry_exit_cond_resched
  *   dynamic_preempt_lazy       <- true
  */
 
 enum {
 	preempt_dynamic_undefined = -1,
-	preempt_dynamic_none,
-	preempt_dynamic_voluntary,
 	preempt_dynamic_full,
 	preempt_dynamic_lazy,
 };
@@ -7973,21 +7977,11 @@ int preempt_dynamic_mode = preempt_dynamic_undefined;
 
 int sched_dynamic_mode(const char *str)
 {
-# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY))
-	if (!strcmp(str, "none"))
-		return preempt_dynamic_none;
-
-	if (!strcmp(str, "voluntary"))
-		return preempt_dynamic_voluntary;
-# endif
-
 	if (!strcmp(str, "full"))
 		return preempt_dynamic_full;
 
-# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY
 	if (!strcmp(str, "lazy"))
 		return preempt_dynamic_lazy;
-# endif
 
 	return -EINVAL;
 }
@@ -7995,71 +7989,18 @@ int sched_dynamic_mode(const char *str)
 # define preempt_dynamic_key_enable(f)	static_key_enable(&sk_dynamic_##f.key)
 # define preempt_dynamic_key_disable(f)	static_key_disable(&sk_dynamic_##f.key)
 
-# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
-#  define preempt_dynamic_enable(f)	static_call_update(f, f##_dynamic_enabled)
-#  define preempt_dynamic_disable(f)	static_call_update(f, f##_dynamic_disabled)
-# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
-#  define preempt_dynamic_enable(f)	preempt_dynamic_key_enable(f)
-#  define preempt_dynamic_disable(f)	preempt_dynamic_key_disable(f)
-# else
-#  error "Unsupported PREEMPT_DYNAMIC mechanism"
-# endif
-
 static DEFINE_MUTEX(sched_dynamic_mutex);
 
 static void __sched_dynamic_update(int mode)
 {
-	/*
-	 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
-	 * the ZERO state, which is invalid.
-	 */
-	preempt_dynamic_enable(cond_resched);
-	preempt_dynamic_enable(might_resched);
-	preempt_dynamic_enable(preempt_schedule);
-	preempt_dynamic_enable(preempt_schedule_notrace);
-	preempt_dynamic_enable(irqentry_exit_cond_resched);
-	preempt_dynamic_key_disable(preempt_lazy);
-
 	switch (mode) {
-	case preempt_dynamic_none:
-		preempt_dynamic_enable(cond_resched);
-		preempt_dynamic_disable(might_resched);
-		preempt_dynamic_disable(preempt_schedule);
-		preempt_dynamic_disable(preempt_schedule_notrace);
-		preempt_dynamic_disable(irqentry_exit_cond_resched);
-		preempt_dynamic_key_disable(preempt_lazy);
-		if (mode != preempt_dynamic_mode)
-			pr_info("Dynamic Preempt: none\n");
-		break;
-
-	case preempt_dynamic_voluntary:
-		preempt_dynamic_enable(cond_resched);
-		preempt_dynamic_enable(might_resched);
-		preempt_dynamic_disable(preempt_schedule);
-		preempt_dynamic_disable(preempt_schedule_notrace);
-		preempt_dynamic_disable(irqentry_exit_cond_resched);
-		preempt_dynamic_key_disable(preempt_lazy);
-		if (mode != preempt_dynamic_mode)
-			pr_info("Dynamic Preempt: voluntary\n");
-		break;
-
 	case preempt_dynamic_full:
-		preempt_dynamic_disable(cond_resched);
-		preempt_dynamic_disable(might_resched);
-		preempt_dynamic_enable(preempt_schedule);
-		preempt_dynamic_enable(preempt_schedule_notrace);
-		preempt_dynamic_enable(irqentry_exit_cond_resched);
 		preempt_dynamic_key_disable(preempt_lazy);
 		if (mode != preempt_dynamic_mode)
 			pr_info("Dynamic Preempt: full\n");
 		break;
 
 	case preempt_dynamic_lazy:
-		preempt_dynamic_disable(cond_resched);
-		preempt_dynamic_disable(might_resched);
-		preempt_dynamic_enable(preempt_schedule);
-		preempt_dynamic_enable(preempt_schedule_notrace);
-		preempt_dynamic_enable(irqentry_exit_cond_resched);
 		preempt_dynamic_key_enable(preempt_lazy);
 		if (mode != preempt_dynamic_mode)
 			pr_info("Dynamic Preempt: lazy\n");
@@ -8092,11 +8033,7 @@ __setup("preempt=", setup_preempt_mode);
 static void __init preempt_dynamic_init(void)
 {
 	if (preempt_dynamic_mode == preempt_dynamic_undefined) {
-		if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
-			sched_dynamic_update(preempt_dynamic_none);
-		} else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
-			sched_dynamic_update(preempt_dynamic_voluntary);
-		} else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
+		if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
 			sched_dynamic_update(preempt_dynamic_lazy);
 		} else {
 			/* Default static call setting, nothing to do */
@@ -8116,8 +8053,6 @@ static void __init preempt_dynamic_init(void)
 	}								\
 	EXPORT_SYMBOL_GPL(preempt_model_##mode)
 
-PREEMPT_MODEL_ACCESSOR(none);
-PREEMPT_MODEL_ACCESSOR(voluntary);
 PREEMPT_MODEL_ACCESSOR(full);
 PREEMPT_MODEL_ACCESSOR(lazy);
 
@@ -8130,7 +8065,7 @@ static inline void preempt_dynamic_init(void) { }
 #endif /* CONFIG_PREEMPT_DYNAMIC */
 
 const char *preempt_modes[] = {
-	"none", "voluntary", "full", "lazy", NULL,
+	"full", "lazy", NULL,
 };
 
 const char *preempt_model_str(void)
@@ -8752,6 +8687,9 @@ int sched_cpu_activate(unsigned int cpu)
 	 */
 	sched_set_rq_online(rq, cpu);
 
+	/* preferred is subset of active and follows its state */
+	set_cpu_preferred(cpu, true);
+
 	return 0;
 }
 
@@ -8765,6 +8703,8 @@ int sched_cpu_deactivate(unsigned int cpu)
 	if (ret)
 		return ret;
 
+	set_cpu_preferred(cpu, false);
+
 	/*
 	 * Remove CPU from nohz.idle_cpus_mask to prevent participating in
 	 * load balancing when not active
@@ -11342,3 +11282,88 @@ void sched_change_end(struct sched_change_ctx *ctx)
 		p->sched_class->prio_changed(rq, p, ctx->prio);
 	}
 }
+
+#ifdef CONFIG_PREFERRED_CPU
+static DEFINE_PER_CPU(struct cpu_stop_work, npc_push_task_work);
+
+static int sched_non_preferred_cpu_push_stop(void *arg)
+{
+	struct task_struct *p = arg;
+	struct rq *rq = this_rq();
+	struct rq_flags rf;
+	int cpu;
+
+	if (cpu_preferred(rq->cpu)) {
+		scoped_guard(rq_lock_irqsave, rq)
+			rq->npc_push_work_pending = false;
+		put_task_struct(p);
+		return 0;
+	}
+
+	scoped_guard (raw_spinlock_irq, &p->pi_lock) {
+		/*
+		 * select_fallback_rq() may acquire the rq lock in case of
+		 * fallback. So call it before grabbing rq lock. If the task
+		 * migrates to another CPU before the rq lock is acquired,
+		 * subsequent validation of task's current rq will help to
+		 * safely bail out.
+		 */
+		cpu = select_fallback_rq(rq->cpu, p);
+		rq_lock(rq, &rf);
+		rq->npc_push_work_pending = false;
+		update_rq_clock(rq);
+		context_unsafe_alias(rq);
+
+		if (task_rq(p) == rq && task_on_rq_queued(p)) {
+			struct rq *dest_rq = __migrate_task(rq, &rf, p, cpu);
+
+			if (rq != dest_rq)
+				schedstat_inc(p->stats.nr_migrations_cpu_non_preferred);
+			rq = dest_rq;
+		}
+		rq_unlock(rq, &rf);
+	}
+
+	put_task_struct(p);
+	return 0;
+}
+
+/*
+ * Push the current task running on non-preferred CPU(npc).
+ * Using this non preferred CPU will lead to more contention
+ * in the host. So it is better not to use this CPU.
+ *
+ * Since task is running, call a stopper to push the task out. This is
+ * similar to how task moves during hotplug. In select_fallback_rq() a
+ * preferred CPU will be chosen and henceforth task shouldn't come back to
+ * this CPU again.
+ *
+ * Works for FAIR class only.
+ *
+ * If task is affined only on non-preferred CPUs, no point in moving it out.
+ */
+void sched_push_current_non_preferred_cpu(struct rq *rq)
+{
+	struct task_struct *push_task = rq->curr;
+
+	scoped_guard(rq_lock, rq) {
+		/* Push the task if its explicit affinity allows */
+		if (!task_can_migrate_to_preferred(push_task, rq->cpu))
+			return;
+
+		/* There is already a stopper thread. Don't race with it. */
+		if (rq->npc_push_work_pending)
+			return;
+
+		if (is_migration_disabled(push_task))
+			return;
+
+		rq->npc_push_work_pending = true;
+	}
+
+	/* sched_tick runs with interrupts disabled. */
+	get_task_struct(push_task);
+	stop_one_cpu_nowait(rq->cpu, sched_non_preferred_cpu_push_stop,
+			    push_task, this_cpu_ptr(&npc_push_task_work));
+}
+#endif
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index 06bddaa738e520..f16970ca81d08a 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -255,7 +255,7 @@ void __account_forceidle_time(struct task_struct *p, u64 delta)
  * occasion account more time than the calling functions think elapsed.
  */
 #ifdef CONFIG_PARAVIRT
-struct static_key paravirt_steal_enabled;
+DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled);
 
 #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN
 static u64 native_steal_clock(int cpu)
@@ -270,7 +270,7 @@ DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
 static __always_inline u64 steal_account_process_time(u64 maxtime)
 {
 #ifdef CONFIG_PARAVIRT
-	if (static_key_false(&paravirt_steal_enabled)) {
+	if (static_branch_unlikely(&paravirt_steal_enabled)) {
 		u64 steal;
 
 		steal = paravirt_steal_clock(smp_processor_id());
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 0663c00c41c042..c0ebdcde5fe5d4 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1097,7 +1097,7 @@ static int start_dl_timer(struct sched_dl_entity *dl_se)
 	 * chosen as the deadline is too small, don't even try to
 	 * start the timer in the past!
 	 */
-	if (ktime_us_delta(act, now) < 0)
+	if (ktime_before(act, now))
 		return 0;
 
 	/*
@@ -2773,11 +2773,14 @@ static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se)
  * DL keeps current in tree, because ->deadline is not typically changed while
  * a task is runnable.
  */
-static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
+static void set_next_task_dl(struct rq *rq, struct task_struct *p, enum snt_e type)
 {
 	struct sched_dl_entity *dl_se = &p->dl;
 	struct dl_rq *dl_rq = &rq->dl;
 
+	if (type == SNT_REPICK)
+		return;
+
 	p->se.exec_start = rq_clock_task(rq);
 	if (on_dl_rq(&p->dl))
 		update_stats_wait_end_dl(dl_rq, dl_se);
@@ -2788,7 +2791,7 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
 	WARN_ON_ONCE(dl_rq->curr);
 	dl_rq->curr = dl_se;
 
-	if (!first)
+	if (type != SNT_PICK)
 		return;
 
 	if (rq->donor->sched_class != &dl_sched_class)
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 72236db67983b3..e6a3b516c703ba 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -73,13 +73,13 @@ static int sched_feat_show(struct seq_file *m, void *v)
 
 #ifdef CONFIG_JUMP_LABEL
 
-#define jump_label_key__true  STATIC_KEY_INIT_TRUE
-#define jump_label_key__false STATIC_KEY_INIT_FALSE
+#define jump_label_key__true  { .key_true  = STATIC_KEY_TRUE_INIT  }
+#define jump_label_key__false { .key_false = STATIC_KEY_FALSE_INIT }
 
 #define SCHED_FEAT(name, enabled)	\
 	jump_label_key__##enabled ,
 
-struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
+union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR] = {
 #include "features.h"
 };
 
@@ -87,12 +87,12 @@ struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
 
 static void sched_feat_disable(int i)
 {
-	static_key_disable_cpuslocked(&sched_feat_keys[i]);
+	static_branch_disable_cpuslocked(&sched_feat_keys[i].key_true);
 }
 
 static void sched_feat_enable(int i)
 {
-	static_key_enable_cpuslocked(&sched_feat_keys[i]);
+	static_branch_enable_cpuslocked(&sched_feat_keys[i].key_false);
 }
 #else /* !CONFIG_JUMP_LABEL: */
 static void sched_feat_disable(int i) { };
@@ -280,16 +280,10 @@ static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
 
 static int sched_dynamic_show(struct seq_file *m, void *v)
 {
-	int i = (IS_ENABLED(CONFIG_PREEMPT_RT) || IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY)) * 2;
 	int mode = READ_ONCE(preempt_dynamic_mode);
-	int j;
 
-	/* Count entries in NULL terminated preempt_modes */
-	for (j = 0; preempt_modes[j]; j++)
-		;
-	j -= !IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY);
-
-	for (; i < j; i++) {
+	/* Stop at NULL terminator */
+	for (int i = 0; preempt_modes[i]; i++) {
 		if (mode == i)
 			seq_puts(m, "(");
 		seq_puts(m, preempt_modes[i]);
@@ -1446,6 +1440,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
 		P_SCHEDSTAT(nr_failed_migrations_running);
 		P_SCHEDSTAT(nr_failed_migrations_hot);
 		P_SCHEDSTAT(nr_forced_migrations);
+		P_SCHEDSTAT(nr_migrations_cpu_non_preferred);
 		P_SCHEDSTAT(nr_wakeups);
 		P_SCHEDSTAT(nr_wakeups_sync);
 		P_SCHEDSTAT(nr_wakeups_migrate);
diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c
index bc4eee5bb4cbf6..a5330b760ecdb5 100644
--- a/kernel/sched/ext/cid.c
+++ b/kernel/sched/ext/cid.c
@@ -31,6 +31,7 @@ static struct scx_cid_tables *scx_cid_tables;	/* used only during alloc/free */
 #define SCX_CID_TOPO_NEG	(struct scx_cid_topo) {				\
 	.core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1,		\
 	.node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1,	\
+	.cluster_cid = -1, .cluster_idx = -1,					\
 }
 
 /*
@@ -182,12 +183,14 @@ s32 scx_cid_init(struct scx_sched *sch)
 	cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	cpumask_var_t cluster_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	struct scx_cid_tables *tbls;
 	u32 next_cid = 0;
 	s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
+	s32 next_cluster_idx = 0;
 	s32 next_shard_idx = 0;
 	u32 shard_size, max_cids;
 	u32 notopo_in_shard;
@@ -215,6 +218,7 @@ s32 scx_cid_init(struct scx_sched *sch)
 	if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) ||
 	    !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) ||
 	    !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) ||
+	    !zalloc_cpumask_var(&cluster_scratch, GFP_KERNEL) ||
 	    !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) ||
 	    !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) ||
 	    !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL))
@@ -256,6 +260,7 @@ s32 scx_cid_init(struct scx_sched *sch)
 			u32 cores_per_shard, nr_large;
 			u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0;
 			s32 shard_cid, shard_idx;
+			s32 cluster_cid = -1, cluster_idx = -1;
 
 			/* llc_scratch = node_scratch & this llc */
 			cpumask_and(llc_scratch, node_scratch, llc_mask);
@@ -268,15 +273,32 @@ s32 scx_cid_init(struct scx_sched *sch)
 			tbls->shard_node[shard_idx] = nid;
 
 			while (!cpumask_empty(llc_scratch)) {
-				s32 lcpu = cpumask_first(llc_scratch);
-				const struct cpumask *sib = topology_sibling_cpumask(lcpu);
-				s32 core_cid = next_cid;
-				s32 core_idx = next_core_idx++;
-				s32 ccpu;
+				const struct cpumask *sib;
+				s32 core_cid, core_idx, lcpu, ccpu;
 				u32 max_cores, cids_in_core;
 
-				/* core_scratch = llc_scratch & this core */
-				cpumask_and(core_scratch, llc_scratch, sib);
+				/*
+				 * Take the cores of one cluster before moving
+				 * on, so that a cluster is a contiguous cid
+				 * range like the core, LLC and node levels.
+				 */
+				if (cpumask_empty(cluster_scratch)) {
+					s32 xcpu = cpumask_first(llc_scratch);
+
+					cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu),
+						   topology_sibling_cpumask(xcpu));
+					cpumask_and(cluster_scratch, cluster_scratch, llc_scratch);
+					cluster_cid = next_cid;
+					cluster_idx = next_cluster_idx++;
+				}
+
+				lcpu = cpumask_first(cluster_scratch);
+				sib = topology_sibling_cpumask(lcpu);
+				core_cid = next_cid;
+				core_idx = next_core_idx++;
+
+				/* core_scratch = cluster_scratch & this core */
+				cpumask_and(core_scratch, cluster_scratch, sib);
 				if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
 					return -EINVAL;
 
@@ -316,8 +338,11 @@ s32 scx_cid_init(struct scx_sched *sch)
 						.node_idx = node_idx,
 						.shard_cid = shard_cid,
 						.shard_idx = shard_idx,
+						.cluster_cid = cluster_cid,
+						.cluster_idx = cluster_idx,
 					};
 
+					cpumask_clear_cpu(ccpu, cluster_scratch);
 					cpumask_clear_cpu(ccpu, llc_scratch);
 					cpumask_clear_cpu(ccpu, node_scratch);
 					cpumask_clear_cpu(ccpu, to_walk);
@@ -461,8 +486,8 @@ __bpf_kfunc_start_defs();
  * starts must be strictly increasing with the first entry 0 and all values <
  * num_possible_cpus(). The last shard extends to num_possible_cpus() and no
  * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info
- * (core/LLC/node) is cleared and the shard layout is set from the input. On
- * invalid input, abort the scheduler.
+ * (core/cluster/LLC/node) is cleared and the shard layout is set from the
+ * input. On invalid input, abort the scheduler.
  */
 __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt,
 				      const s32 *shard_start__arena, u32 shard_start_cnt,
@@ -640,6 +665,29 @@ __bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux)
 	return scx_cid_to_cpu(sch, cid);
 }
 
+/**
+ * scx_bpf_cid_node - Return the NUMA node the given @cid belongs to
+ * @cid: cid to look up
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return NUMA_NO_NODE if @cid is invalid or there is no scheduler. Unlike
+ * scx_bpf_cpu_node(), this kfunc is available only with CID support, so BPF
+ * schedulers can use its presence to detect NUMA-aware CID lookups.
+ */
+__bpf_kfunc s32 scx_bpf_cid_node(s32 cid, const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+	s32 cpu;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return NUMA_NO_NODE;
+	cpu = scx_cid_to_cpu(sch, cid);
+	return cpu < 0 ? NUMA_NO_NODE : cpu_to_node(cpu);
+}
+
 /**
  * scx_bpf_cpu_to_cid - Return the cid for @cpu
  * @cpu: cpu to look up
@@ -912,30 +960,36 @@ bool scx_cmask_empty(const struct scx_cmask *m)
 /**
  * scx_bpf_cid_topo - Copy out per-cid topology info
  * @cid: cid to look up
- * @out__uninit: where to copy the topology info; fully written by this call
+ * @out: where to copy the topology info
+ * @out__sz: size of @out, the program's sizeof(struct scx_cid_topo)
  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
- * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if
- * @cid is out of range. If @cid is valid but in the no-topo section, all fields
- * are set to -1. All fields are also set to -1 when no cid tables have been
- * published yet, which a program may observe while racing the root enable.
+ * Fill @out with the topology info for @cid. Trigger scx_error() if @cid is out
+ * of range. If @cid is valid but in the no-topo section, all fields are set to
+ * -1. All fields are also set to -1 when no cid tables have been published yet,
+ * which a program may observe while racing the root enable.
+ *
+ * The program's struct may be older or newer than the kernel's. The smaller of
+ * @out__sz and the kernel's size is copied and the rest of @out is set to -1.
  */
-__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit,
+__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz,
 				  const struct bpf_prog_aux *aux)
 {
+	size_t len = min(out__sz, sizeof(*out));
 	struct scx_cid_topo *topo;
 	struct scx_sched *sch;
 
+	/* the error cases and fields the kernel lacks read as -1 */
+	memset(out, 0xff, out__sz);
+
 	guard(rcu)();
 
 	sch = scx_prog_sched(aux);
 	topo = rcu_dereference(scx_cid_topo);
-	if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) {
-		*out__uninit = SCX_CID_TOPO_NEG;
+	if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo))
 		return;
-	}
 
-	*out__uninit = topo[cid];
+	memcpy(out, &topo[cid], len);
 }
 
 __bpf_kfunc_end_defs();
@@ -952,6 +1006,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = {
 
 BTF_KFUNCS_START(scx_kfunc_ids_cid)
 BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cid_node, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS)
 BTF_KFUNCS_END(scx_kfunc_ids_cid)
diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h
index 2fe2311a0f9951..709afbfb97c6f2 100644
--- a/kernel/sched/ext/cid.h
+++ b/kernel/sched/ext/cid.h
@@ -13,11 +13,12 @@
  * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences
  * for every op.
  *
- * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or
- * NUMA node get contiguous cid ranges, so a topology unit becomes a (start,
- * length) slice of cid space. Communication can pass a slice instead of a
- * cpumask, and BPF code can process, for example, a u64 word's worth of cids at
- * a time.
+ * cids give every cpu a dense, topology-ordered id. CPUs in each core,
+ * cluster, LLC or NUMA node get contiguous cid ranges, so a topology unit
+ * becomes a (start, length) slice of cid space. A core without a wider cluster
+ * level forms a cluster of its own. Communication can pass a slice
+ * instead of a cpumask, and BPF code can process, for example, a u64 word's
+ * worth of cids at a time.
  *
  * The mapping is built once at root scheduler enable time by walking the
  * topology of online cpus only. Going by online cpus is out of necessity:
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 3219f0da0fe4e7..129d9c192e9873 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -24,6 +24,32 @@
 
 DEFINE_RAW_SPINLOCK(scx_sched_lock);
 
+bool __scx_allow_proxy_exec(const struct task_struct *p)
+{
+	struct scx_sched *sch;
+
+	if (p->sched_class != &ext_sched_class)
+		return true;
+
+	sch = scx_task_sched(p);
+	return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED);
+}
+
+/*
+ * End retained proxy execution before changing @p's BPF scheduler ownership.
+ * Called with @p's pi and rq locks held immediately before
+ * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock
+ * is updated only once.
+ */
+void scx_prepare_task_sched_change(struct task_struct *p)
+{
+	lockdep_assert_held(&p->pi_lock);
+	lockdep_assert_rq_held(task_rq(p));
+
+	update_rq_clock(task_rq(p));
+	sched_proxy_block_task(task_rq(p), p);
+}
+
 /*
  * NOTE: sched_ext is in the process of growing multiple scheduler support and
  * scx_root usage is in a transitional state. Naked dereferences are safe if the
@@ -41,10 +67,21 @@ struct scx_sched __rcu *scx_root;
 LIST_HEAD(scx_sched_all);
 
 #ifdef CONFIG_EXT_SUB_SCHED
+static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg,
+					   const void *ptr)
+{
+	const struct scx_sched *sch = ptr;
+
+	BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64));
+
+	return sch->ops.sub_cgroup_id != *(const u64 *)arg->key;
+}
+
 const struct rhashtable_params scx_sched_hash_params = {
 	.key_len		= sizeof_field(struct scx_sched, ops.sub_cgroup_id),
 	.key_offset		= offsetof(struct scx_sched, ops.sub_cgroup_id),
 	.head_offset		= offsetof(struct scx_sched, hash_node),
+	.obj_cmpfn		= scx_sched_cmpfn,
 	.insecure_elasticity	= true,	/* inserted under scx_sched_lock */
 };
 
@@ -52,10 +89,20 @@ struct rhashtable scx_sched_hash;
 #endif
 
 /* see SCX_OPS_TID_TO_TASK */
+static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
+{
+	const struct sched_ext_entity *scx = ptr;
+
+	BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64));
+
+	return scx->tid != *(const u64 *)arg->key;
+}
+
 static const struct rhashtable_params scx_tid_hash_params = {
 	.key_len		= sizeof_field(struct sched_ext_entity, tid),
 	.key_offset		= offsetof(struct sched_ext_entity, tid),
 	.head_offset		= offsetof(struct sched_ext_entity, tid_hash_node),
+	.obj_cmpfn		= scx_tid_cmpfn,
 	.insecure_elasticity	= true,	/* inserted/removed under scx_tasks_lock */
 };
 static struct rhashtable scx_tid_hash;
@@ -189,10 +236,20 @@ static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc);
  */
 static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
 
+static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr)
+{
+	const struct scx_dispatch_q *dsq = ptr;
+
+	BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64));
+
+	return dsq->id != *(const u64 *)arg->key;
+}
+
 static const struct rhashtable_params dsq_hash_params = {
 	.key_len		= sizeof_field(struct scx_dispatch_q, id),
 	.key_offset		= offsetof(struct scx_dispatch_q, id),
 	.head_offset		= offsetof(struct scx_dispatch_q, hash_node),
+	.obj_cmpfn		= dsq_cmpfn,
 };
 
 static LLIST_HEAD(dsqs_to_free);
@@ -374,28 +431,28 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
 	 */
 
 	/*
-	 * If we're in the dispatch path holding rq lock, $curr may or may not
+	 * If we're in the dispatch path holding rq lock, $donor may or may not
 	 * be ready depending on whether the on-going dispatch decides to extend
-	 * $curr's slice. We say yes here and resolve it at the end of dispatch.
+	 * $donor's slice. We say yes here and resolve it at the end of dispatch.
 	 * See dispatch_one().
 	 */
 	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
 		return true;
 
 	/*
-	 * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch,
+	 * The preemption flags clear $donor's slice if on SCX and kick dispatch,
 	 * so allow it to avoid spuriously triggering reenq on a combined
 	 * PREEMPT|IMMED insertion.
 	 */
-	if (enq_flags & SCX_ENQ_PREEMPT) {
-		struct task_struct *curr = rq->curr;
+	if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) {
+		struct task_struct *donor = rq->donor;
 
 		/*
 		 * A protected slice refuses the preemption and the cpu stays
 		 * occupied. See rq_owned_post_enq().
 		 */
-		return curr->sched_class != &ext_sched_class ||
-			likely(!(curr->scx.flags & SCX_TASK_PROTECTED));
+		return donor->sched_class != &ext_sched_class ||
+			likely(!(donor->scx.flags & SCX_TASK_PROTECTED));
 	}
 
 	/*
@@ -447,37 +504,45 @@ static void switch_rq_lock(struct rq *from, struct rq *to)
 DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
 
 /**
- * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task
+ * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask
+ * @sch: scx_sched whose scratch to fill
+ * @cpumask: cpus to translate into cids
+ *
+ * The scratch lives in BPF-writable arena memory and its header can't be
+ * trusted, so it is rewritten from kernel geometry rather than read. Caller
+ * must hold an rq lock so this cpu is the sole kernel writer for as long as the
+ * returned address is in use.
+ */
+static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch,
+						const struct cpumask *cpumask)
+{
+	struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
+	struct scx_cmask_ref ref;
+
+	scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
+	scx_cmask_ref_from_cpumask(&ref, cpumask);
+	return kern_va;
+}
+
+/**
+ * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task
  * @sch: scx_sched being invoked
  * @rq: rq to update as the currently-locked rq, or NULL
  * @task: task whose affinity is changing
  * @cpumask: new cpumask
  *
- * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu
- * scratch in cid.c and dispatch through the ops_cid union view. Caller
- * must hold @rq's rq lock so this_cpu_ptr is stable across the call.
+ * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch
+ * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock.
  */
 static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq,
 					   struct task_struct *task,
 					   const struct cpumask *cpumask)
 {
-	if (scx_is_cid_type()) {
-		struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
-		struct scx_cmask_ref ref;
-
-		/*
-		 * Build the per-cpu arena cmask from kernel geometry via @ref,
-		 * never reading its BPF-writable header. set_cmask()'s __arena
-		 * argument takes the kernel address and the struct_ops
-		 * trampoline rebases it into BPF's arena pointer form. The rq
-		 * lock makes this cpu the sole kernel writer.
-		 */
-		scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
-		scx_cmask_ref_from_cpumask(&ref, cpumask);
-		SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va);
-	} else {
+	if (scx_is_cid_type())
+		SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task,
+				     scx_fill_cmask_scratch(sch, cpumask));
+	else
 		SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask);
-	}
 }
 
 enum scx_dsq_iter_flags {
@@ -1079,6 +1144,57 @@ static void schedule_deferred_locked(struct rq *rq)
 	schedule_deferred(rq);
 }
 
+#ifdef CONFIG_NO_HZ_FULL
+static void scx_proxy_tick_bal_cb(struct rq *rq)
+{
+	sched_update_tick_dependency(rq);
+}
+
+static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next)
+{
+	bool proxy = next != rq->donor;
+	bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK;
+
+	if (likely(proxy == was_proxy))
+		return;
+
+	if (proxy) {
+		/* Keep proxy execution tick-driven for now. */
+		rq->scx.flags |= SCX_RQ_PROXY_TICK;
+		tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
+	} else {
+		rq->scx.flags &= ~SCX_RQ_PROXY_TICK;
+		/*
+		 * The selected donor is already visible, but rq->curr still
+		 * identifies the outgoing execution context. Reevaluate after
+		 * context_switch() updates rq->curr so sched_can_stop_tick() sees
+		 * the complete selection.
+		 */
+		queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb,
+				       scx_proxy_tick_bal_cb);
+	}
+}
+#endif
+
+/*
+ * Complete sched_ext bookkeeping after proxy resolution and retry tasks which
+ * couldn't be reenqueued by an earlier reject DSQ drain.
+ */
+void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next)
+{
+	lockdep_assert_rq_held(rq);
+
+#ifdef CONFIG_NO_HZ_FULL
+	if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq)))
+		scx_proxy_update_tick(rq, next);
+#endif
+
+	if (rq->scx.flags & SCX_RQ_PROXY_RETRY) {
+		rq->scx.flags &= ~SCX_RQ_PROXY_RETRY;
+		schedule_deferred_locked(rq);
+	}
+}
+
 void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 			u64 reenq_flags, struct rq *locked_rq)
 {
@@ -1127,7 +1243,7 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 	} else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) {
 		rq = this_rq();
 
-		struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq));
+		struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq));
 		struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user;
 
 		/*
@@ -1390,20 +1506,27 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e
 
 static void update_curr_scx(struct rq *rq)
 {
-	struct task_struct *curr = rq->curr;
+	struct task_struct *donor;
 	s64 delta_exec;
 
+	/*
+	 * update_curr_scx() is selected through rq->donor->sched_class, not
+	 * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT
+	 * owner executes for a FAIR donor, FAIR's update_curr() runs instead.
+	 */
+	donor = rq->donor;
+
 	/* apply even on 0 delta_exec, callers may still act on the slice */
-	apply_task_slice_oob(rq, curr);
+	apply_task_slice_oob(rq, donor);
 
 	delta_exec = update_curr_common(rq);
 	if (unlikely(delta_exec <= 0))
 		return;
 
-	if (curr->scx.slice != SCX_SLICE_INF)
-		curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec);
+	if (donor->scx.slice != SCX_SLICE_INF)
+		donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec);
 
-	if (unlikely(curr == scx_rescuee(rq)))
+	if (unlikely(donor == scx_rescuee(rq)))
 		scx_rescue_charge(rq, delta_exec);
 
 	dl_server_update(&rq->ext_server, delta_exec);
@@ -1499,31 +1622,48 @@ static inline bool task_scx_migrating(struct task_struct *p)
 	return p->scx.sticky_cpu >= 0;
 }
 
-/*
- * Call ops.dequeue() if the task is in BPF custody and not migrating.
- * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked.
- */
-static void call_task_dequeue(struct scx_sched *sch, struct rq *rq,
-			      struct task_struct *p, u64 deq_flags)
+/* Must be called under the lock serializing @p's custody transfers. */
+static bool task_leave_custody(struct task_struct *p)
 {
 	if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p))
-		return;
-
-	if (SCX_HAS_OP(sch, dequeue))
-		SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
+		return false;
 
 	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
+	return true;
+}
+
+/*
+ * A task with an infinite slice may be running with its tick stopped. Lazy
+ * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee
+ * that the lazy request is promoted by a real scheduler tick. Set the lazy
+ * request first so that the dependency update's IPI can also serve it on return
+ * to user space.
+ */
+static void scx_resched_curr_lazy(struct rq *rq)
+{
+	resched_curr_lazy(rq);
+
+	if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) {
+		rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
+		if (rq->clock_update_flags < RQCF_UPDATED)
+			update_rq_clock(rq);
+		update_other_load_avgs(rq);
+		sched_update_tick_dependency(rq);
+	}
 }
 
 static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
 			      struct scx_dispatch_q *dsq, struct task_struct *p,
 			      u64 enq_flags)
 {
-	call_task_dequeue(sch, rq, p, 0);
+	if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
+		SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
 
 	/*
-	 * Only local inserts get the wakeup treatment below. Rejects kick the
-	 * deferred reenq and rescue parks are paced by the rescue timer.
+	 * Only local inserts get the wakeup treatment below. Rejects kick a
+	 * deferred reenq and rescue parks are paced by the rescue timer. Proxy
+	 * rejects which aren't ready when drained request a later retry from
+	 * scx_proxy_reenqueue_retry().
 	 */
 	if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
 		if (dsq->id == SCX_DSQ_REJECT)
@@ -1577,12 +1717,16 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
 	if (rq->scx.flags & SCX_RQ_IN_DISPATCH)
 		return;
 
-	if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
-	    rq->curr->sched_class == &ext_sched_class) {
-		if (likely(scx_set_task_slice(rq->curr, 0)))
-			resched_curr(rq);
-		else
+	if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) &&
+	    p != rq->donor && rq->donor->sched_class == &ext_sched_class) {
+		if (likely(scx_set_task_slice(rq->donor, 0))) {
+			if (enq_flags & SCX_ENQ_PREEMPT)
+				resched_curr(rq);
+			else
+				scx_resched_curr_lazy(rq);
+		} else {
 			__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
+		}
 	}
 }
 
@@ -1590,12 +1734,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
 				 struct scx_dispatch_q *dsq, struct task_struct *p,
 				 u64 slice, u64 vtime, u64 enq_flags)
 {
-	bool is_rq_owned = false;
+	bool is_rq_owned = dsq_is_rq_owned(dsq);
 
-	if (dsq->id == SCX_DSQ_LOCAL) {
+	if (dsq->id == SCX_DSQ_LOCAL)
 		dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
-		is_rq_owned = true;
-	}
 
 	WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
 	WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
@@ -1672,7 +1814,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
 			scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks",
 				  dsq->id);
 
-		if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) {
+		if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) {
 			/* new task inserted at head - use fastpath */
 			if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
 				rcu_assign_pointer(dsq->first_task, p);
@@ -1705,20 +1847,28 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
 	if (is_rq_owned) {
 		rq_owned_post_enq(sch, rq, dsq, p, enq_flags);
 	} else {
+		bool call_dequeue = false;
+
 		/*
 		 * Global and bypass DSQs are terminal - the task leaves the
-		 * scheduler's custody, so ops.dequeue() fires here. It can run
+		 * scheduler's custody, so ops.dequeue() fires. It can run
 		 * without @p's rq lock (finish_dispatch() passes the dispatch
 		 * rq); that's safe because dequeue_task_scx() waits on
 		 * SCX_OPSS_DISPATCHING (see the ops_state note above) and so
 		 * can't race it. A non-terminal DSQ keeps the task in custody.
+		 * The custody transfer happens under @dsq->lock so that later
+		 * consumers see the flag clear; the callback runs after
+		 * @dsq->lock is dropped because it may lock a DSQ itself.
 		 */
 		if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS)
-			call_task_dequeue(sch, rq, p, 0);
+			call_dequeue = task_leave_custody(p);
 		else
 			p->scx.flags |= SCX_TASK_IN_CUSTODY;
 
 		raw_spin_unlock(&dsq->lock);
+
+		if (call_dequeue && SCX_HAS_OP(sch, dequeue))
+			SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0);
 	}
 
 	/*
@@ -1766,10 +1916,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
 			list_del_init(&p->scx.dsq_list.node);
 
 		/*
-		 * When dispatching directly from the BPF scheduler to a local
-		 * DSQ, the task isn't associated with any DSQ but
-		 * @p->scx.holding_cpu may be set under the protection of
-		 * %SCX_OPSS_DISPATCHING.
+		 * When dispatch_to_local_dsq() or remote consumption moves a
+		 * task to a local DSQ, the task isn't associated with any DSQ
+		 * but @p->scx.holding_cpu may be set. Clearing holding_cpu
+		 * tells dispatch_to_local_dsq() that it lost to a dequeue.
 		 */
 		if (p->scx.holding_cpu >= 0)
 			p->scx.holding_cpu = -1;
@@ -1789,10 +1939,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
 		scx_task_unlink_from_dsq(p, dsq);
 	} else {
 		/*
-		 * We're racing against dispatch_to_local_dsq() which already
-		 * removed @p from @dsq and set @p->scx.holding_cpu. Clear the
-		 * holding_cpu which tells dispatch_to_local_dsq() that it lost
-		 * the race.
+		 * We're racing against unlink_dsq_and_switch_rq_lock(),
+		 * which already removed @p from @dsq and set
+		 * @p->scx.holding_cpu. Clear holding_cpu to tell
+		 * unlink_dsq_and_switch_rq_lock() that it lost the race.
 		 */
 		WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
 		p->scx.holding_cpu = -1;
@@ -1975,6 +2125,12 @@ bool scx_rq_online(struct rq *rq)
 void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 			 int sticky_cpu)
 {
+	enum {
+		ENQ_ACTION_NONE,
+		ENQ_ACTION_DIRECT,
+		ENQ_ACTION_LOCAL_NOREFILL,
+		ENQ_ACTION_ENQUEUE,
+	} action = ENQ_ACTION_NONE;
 	struct scx_sched *sch = scx_task_sched(p);
 	struct task_struct **ddsp_taskp;
 	struct scx_dispatch_q *dsq;
@@ -2008,7 +2164,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 			__scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq),
 				   "%s[%d] reenqueued %u times without running",
 				   p->comm, p->pid, p->scx.reenq_cnt);
-			return;
+			goto out;
 		}
 	}
 
@@ -2028,19 +2184,28 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
 		goto direct;
 
-	/* see %SCX_OPS_ENQ_EXITING */
-	if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) &&
-	    unlikely(p->flags & PF_EXITING)) {
-		__scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1);
-		enq_flags |= SCX_ENQ_RESCUE;	/* avoid looping on cap rejection */
-		goto local;
-	}
+	/*
+	 * A full wakeup enqueue happens before is_blocked is cleared. Don't
+	 * report it as another blocked-donor admission.
+	 */
+	if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked &&
+	    !(enq_flags & SCX_ENQ_WAKEUP)) {
+		enq_flags |= SCX_ENQ_BLOCKED;
+	} else {
+		/* see %SCX_OPS_ENQ_EXITING */
+		if (unlikely(p->flags & PF_EXITING) &&
+		    !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) {
+			__scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1);
+			enq_flags |= SCX_ENQ_RESCUE;	/* avoid looping on cap rejection */
+			goto local;
+		}
 
-	/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
-	if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
-	    is_migration_disabled(p)) {
-		__scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
-		goto local;
+		/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
+		if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
+		    is_migration_disabled(p)) {
+			__scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
+			goto local;
+		}
 	}
 
 	if (unlikely(!SCX_HAS_OP(sch, enqueue)))
@@ -2073,14 +2238,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	 * dequeue may be waiting. The store_release matches their load_acquire.
 	 */
 	atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq);
-	return;
+	goto out;
 
 direct:
-	direct_dispatch(sch, p, enq_flags);
-	return;
+	action = ENQ_ACTION_DIRECT;
+	goto out;
 local_norefill:
-	scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags);
-	return;
+	action = ENQ_ACTION_LOCAL_NOREFILL;
+	goto out;
 local:
 	dsq = &rq->scx.local_dsq;
 	goto enqueue;
@@ -2092,9 +2257,27 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 	goto enqueue;
 
 enqueue:
-	refill_task_slice_dfl(sch, p);
-	clear_direct_dispatch(p);
-	scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
+	action = ENQ_ACTION_ENQUEUE;
+out:
+	/* The reason is input to ops.enqueue(), not to the resulting placement. */
+	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+
+	switch (action) {
+	case ENQ_ACTION_DIRECT:
+		direct_dispatch(sch, p, enq_flags);
+		break;
+	case ENQ_ACTION_LOCAL_NOREFILL:
+		scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
+				     enq_flags);
+		break;
+	case ENQ_ACTION_ENQUEUE:
+		refill_task_slice_dfl(sch, p);
+		clear_direct_dispatch(p);
+		scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
+		break;
+	case ENQ_ACTION_NONE:
+		break;
+	}
 }
 
 static bool task_runnable(const struct task_struct *p)
@@ -2112,8 +2295,8 @@ static void set_task_runnable(struct rq *rq, struct task_struct *p)
 	}
 
 	/*
-	 * list_add_tail() must be used. scx_bypass() depends on tasks being
-	 * appended to the runnable_list.
+	 * list_add_tail() must be used. scx_bypass() and rq_offline_scx()
+	 * depend on tasks being appended to the runnable_list.
 	 */
 	list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
 
@@ -2141,17 +2324,23 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_
 	int sticky_cpu = p->scx.sticky_cpu;
 	u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags;
 
-	if (enq_flags & ENQUEUE_WAKEUP)
+	/*
+	 * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue
+	 * returns. Only the core's wakeup path delivers one. The flags stashed
+	 * for a remote activation may carry the wakeup bit without it.
+	 */
+	if (core_enq_flags & ENQUEUE_WAKEUP)
 		rq->scx.flags |= SCX_RQ_IN_WAKEUP;
 
 	/*
-	 * Restoring a running task will be immediately followed by
-	 * set_next_task_scx() which expects the task to not be on the BPF
+	 * Restoring the current scheduling context will be immediately followed
+	 * by set_next_task_scx() which expects the task to not be on the BPF
 	 * scheduler as tasks can only start running through local DSQs. Force
 	 * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark
 	 * IGNORE_CAPS to force entry into the local DSQ.
 	 */
-	if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) {
+	if (unlikely(enq_flags & ENQUEUE_RESTORE) &&
+	    task_current_donor(rq, p)) {
 		sticky_cpu = cpu_of(rq);
 		enq_flags |= SCX_ENQ_IGNORE_CAPS;
 	}
@@ -2210,7 +2399,7 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
 		/*
 		 * A queued task must always be in BPF scheduler's custody. If
 		 * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another
-		 * CPU has already passed call_task_dequeue() (which clears the
+		 * CPU has already passed task_leave_custody() (which clears the
 		 * flag), but has not yet written SCX_OPSS_NONE. That final
 		 * store does not require this rq's lock, so retrying with
 		 * cpu_relax() is bounded: we will observe NONE (or DISPATCHING,
@@ -2258,7 +2447,8 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags)
 	 * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until
 	 * it is enqueued on the destination.
 	 */
-	call_task_dequeue(sch, rq, p, deq_flags);
+	if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue))
+		SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags);
 }
 
 static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags)
@@ -2281,10 +2471,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
 	ops_dequeue(rq, p, deq_flags);
 
 	/*
-	 * A currently running task which is going off @rq first gets dequeued
-	 * and then stops running. As we want running <-> stopping transitions
-	 * to be contained within runnable <-> quiescent transitions, trigger
-	 * ->stopping() early here instead of in put_prev_task_scx().
+	 * A current scheduling context which is going off @rq first gets
+	 * dequeued and then stops running. As we want running <-> stopping
+	 * transitions to be contained within runnable <-> quiescent transitions,
+	 * trigger ->stopping() early here instead of in put_prev_task_scx().
 	 *
 	 * @p may go through multiple stopping <-> running transitions between
 	 * here and put_prev_task_scx() if task attribute changes occur while
@@ -2292,11 +2482,13 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
 	 * information meaningful to the BPF scheduler and can be suppressed by
 	 * skipping the callbacks if the task is !QUEUED.
 	 */
-	if (task_current(rq, p) &&
-	    (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) {
-		update_curr_scx(rq);
-		if (SCX_HAS_OP(sch, stopping))
-			SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
+	if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) {
+		if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) {
+			update_curr_scx(rq);
+			if (SCX_HAS_OP(sch, stopping))
+				SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
+		}
+		p->scx.flags &= ~SCX_TASK_RUN_TRACKED;
 	}
 
 	if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p))
@@ -2312,9 +2504,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
 	sub_nr_running(rq, 1);
 
 	scx_dispatch_dequeue(rq, p);
+	p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 
 	/* see scx_task_slice_ended() for the save/restore exception */
-	if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p)))
+	if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p)))
 		scx_task_slice_ended(rq, p);
 
 	clear_direct_dispatch(p);
@@ -2354,10 +2547,23 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl
 	/*
 	 * Preemption between SCX tasks is implemented by resetting the victim
 	 * task's slice to 0 and triggering reschedule on the target CPU.
-	 * Nothing to do.
-	 */
-	if (p->sched_class == &ext_sched_class)
+	 *
+	 * A retained proxy donor can wake through ttwu_runnable() without another
+	 * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so
+	 * that ops.dispatch() can reconsider the task after ttwu_runnable() clears
+	 * is_blocked. A full wakeup activation has already enqueued the task and
+	 * doesn't need the additional reschedule.
+	 */
+	if (p->sched_class == &ext_sched_class) {
+		if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) &&
+		    p->is_blocked) {
+			struct scx_sched *sch = scx_task_sched(p);
+
+			if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED))
+				resched_curr(rq);
+		}
 		return;
+	}
 
 	/*
 	 * Getting preempted by a higher-priority class. Reenqueue IMMED tasks.
@@ -2374,19 +2580,15 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl
 }
 
 void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,
-				      u64 enq_flags, struct scx_dispatch_q *src_dsq,
-				      struct rq *dst_rq)
+				      u64 enq_flags, struct rq *dst_rq)
 {
 	struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
 
-	/* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */
-	if (!dsq_is_rq_owned(src_dsq))
-		lockdep_assert_held(&src_dsq->lock);
 	lockdep_assert_rq_held(dst_rq);
 
 	WARN_ON_ONCE(p->scx.holding_cpu >= 0);
 
-	if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
+	if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY))
 		dsq_insert_head(dst_dsq, p);
 	else
 		list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
@@ -2456,8 +2658,10 @@ static void move_remote_task_to_local_dsq(struct scx_sched *sch,
  * - The BPF scheduler is bypassed while the rq is offline and we can always say
  *   no to the BPF scheduler initiated migrations while offline.
  *
- * The caller must ensure that @p and @rq are on different CPUs.
- * If enforce == true, caller must hold @p's rq lock.
+ * The caller must ensure that @p and @rq are on different CPUs. If @enforce is
+ * true, report violations attributable to BPF-directed migrations. The caller
+ * must hold @p's rq lock to avoid reporting a transient race as a scheduler
+ * error.
  */
 static bool task_can_run_on_remote_rq(struct scx_sched *sch,
 				      struct task_struct *p, struct rq *rq,
@@ -2465,16 +2669,26 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
 {
 	s32 cpu = cpu_of(rq);
 
-	/*
-	 * To prevent races with @p still running on its old CPU while switching
-	 * out, make sure we're holding @p's rq lock so as not to risk
-	 * erroneously killing the BPF scheduler.
-	 */
 	if (enforce)
 		lockdep_assert_rq_held(task_rq(p));
 
 	WARN_ON_ONCE(task_cpu(p) == cpu);
 
+	/*
+	 * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling
+	 * context to its lock owner's rq. For example, if BPF places donor D on
+	 * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1
+	 * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared
+	 * DSQ, CPU0 could otherwise consume it and move it back, only for proxy
+	 * execution to return it to CPU1 again.
+	 *
+	 * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and
+	 * BPF may still choose its placement. An actively donating task is
+	 * rejected separately by task_proxy_running_or_donating().
+	 */
+	if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu)
+		return false;
+
 	/*
 	 * If @p has migration disabled, @p->cpus_ptr is updated to contain only
 	 * the pinned CPU in migrate_disable_switch() while @p is being switched
@@ -2516,6 +2730,64 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
 	return true;
 }
 
+/*
+ * Proxy execution can change @p's execution and migration-disabled state
+ * without touching its DSQ entry or clearing holding_cpu. Check those states
+ * with @p's rq locked. Without proxy execution, the holding_cpu handshake is
+ * sufficient and this must not affect the existing migration path.
+ *
+ * A BPF-directed transfer to a remote local DSQ performs a normal task
+ * migration and thus cannot move a migration-disabled task. In contrast,
+ * proxy_migrate_task() moves only a blocked donor's scheduling context towards
+ * the mutex owner and preserves its execution home in wake_cpu. The latter is
+ * therefore allowed even when the donor is migration-disabled.
+ */
+static bool task_proxy_running_or_donating(struct task_struct *p)
+{
+	struct rq *src_rq = task_rq(p);
+
+	lockdep_assert_rq_held(src_rq);
+
+	if (!sched_proxy_exec())
+		return false;
+
+	/* @p may be rq->curr under another task's scheduling context. */
+	if (task_on_cpu(src_rq, p))
+		return true;
+
+	/* Don't move an active scheduling context off its source rq. */
+	if (task_current_donor(src_rq, p))
+		return true;
+
+	return false;
+}
+
+static bool task_proxy_unsafe_to_move(struct task_struct *p)
+{
+	if (!sched_proxy_exec())
+		return false;
+
+	return task_proxy_running_or_donating(p) || is_migration_disabled(p);
+}
+
+/*
+ * Park a task whose remote transfer raced with proxy execution. Reenqueueing
+ * from the source rq makes the task's owning scheduler choose its placement
+ * again and preserves sub-scheduler containment.
+ */
+static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq,
+				  struct task_struct *p, u64 enq_flags)
+{
+	lockdep_assert_rq_held(rq);
+	WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
+
+	p->scx.holding_cpu = -1;
+	p->scx.flags |= SCX_TASK_REENQ_PROXY;
+	scx_divert_strip_flags(p, &enq_flags);
+
+	scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags);
+}
+
 /**
  * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock
  * @p: target task
@@ -2544,7 +2816,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch,
  * points to this CPU. See scx_dispatch_dequeue() for the counterpart.
  *
  * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is
- * still valid. %false if lost to dequeue.
+ * still valid and detached from @dsq. %false if lost to dequeue.
  */
 static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
 					  struct scx_dispatch_q *dsq,
@@ -2564,8 +2836,18 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
 	switch_rq_lock(locked_rq, src_rq);
 
 	/* task_rq couldn't have changed if we're still the holding cpu */
-	return likely(p->scx.holding_cpu == cpu) &&
-		!WARN_ON_ONCE(src_rq != task_rq(p));
+	if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) {
+		/*
+		 * Keep ->dsq set until we own @src_rq so that a racing dequeue
+		 * can find @dsq and clear holding_cpu. Once ownership is
+		 * confirmed, clear it under @src_rq so that deactivate_task()
+		 * takes the !dsq path instead of retaking @dsq->lock.
+		 */
+		p->scx.dsq = NULL;
+		return true;
+	}
+
+	return false;
 }
 
 static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
@@ -2573,6 +2855,19 @@ static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
 				struct scx_dispatch_q *dsq, struct rq *src_rq)
 {
 	if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) {
+		/*
+		 * Proxy execution may have changed @p's running or
+		 * migration-disabled state while switching rq locks without
+		 * clearing holding_cpu. Park it on the source rq and let its
+		 * owning scheduler choose its placement again.
+		 */
+		if (unlikely(task_proxy_unsafe_to_move(p))) {
+			p->scx.dsq = NULL;
+			scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
+			switch_rq_lock(src_rq, this_rq);
+			return false;
+		}
+
 		move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq);
 		return true;
 	} else {
@@ -2610,6 +2905,19 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
 
 	if (dst_dsq->id == SCX_DSQ_LOCAL) {
 		dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
+		/*
+		 * Unlike the rq-lock handoff paths, @src_rq has been locked
+		 * throughout this operation. Only active proxy state can race the
+		 * move here; let the enforcing check below diagnose an ordinary
+		 * migration-disabled task.
+		 */
+		if (src_rq != dst_rq &&
+		    unlikely(task_proxy_running_or_donating(p))) {
+			dispatch_dequeue_locked(p, src_dsq);
+			raw_spin_unlock(&src_dsq->lock);
+			scx_proxy_reject_task(sch, src_rq, p, enq_flags);
+			return src_rq;
+		}
 		if (src_rq != dst_rq &&
 		    unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
 			dst_dsq = find_global_dsq(sch, task_cpu(p));
@@ -2629,9 +2937,10 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
 		/* @p is going from a non-local DSQ to a local DSQ */
 		if (src_rq == dst_rq) {
 			scx_task_unlink_from_dsq(p, src_dsq);
-			scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq);
 			raw_spin_unlock(&src_dsq->lock);
+			scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq);
 		} else {
+			dispatch_dequeue_locked(p, src_dsq);
 			raw_spin_unlock(&src_dsq->lock);
 			move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
 		}
@@ -2680,8 +2989,8 @@ bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
 
 		if (rq == task_rq) {
 			scx_task_unlink_from_dsq(p, dsq);
-			scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
 			raw_spin_unlock(&dsq->lock);
+			scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq);
 			return true;
 		}
 
@@ -2766,6 +3075,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 	if (likely(p->scx.holding_cpu == raw_smp_processor_id()) &&
 	    !WARN_ON_ONCE(src_rq != task_rq(p))) {
 		bool fallback = false;
+
 		/*
 		 * If @p is staying on the same rq, there's no need to go
 		 * through the full deactivate/activate cycle. Optimize by
@@ -2775,6 +3085,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 			p->scx.holding_cpu = -1;
 			scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
 					     slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE);
+		} else if (unlikely(task_proxy_unsafe_to_move(p))) {
+			fallback = true;
+			scx_proxy_reject_task(sch, src_rq, p, enq_flags);
 		} else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
 			p->scx.holding_cpu = -1;
 			fallback = true;
@@ -2790,7 +3103,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
 		}
 
 		/* if the destination CPU is idle, wake it up */
-		if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class))
+		if (!fallback && sched_class_above(p->sched_class,
+						      dst_rq->donor->sched_class))
 			resched_curr(dst_rq);
 	}
 
@@ -3008,10 +3322,27 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev
 	return verdict;
 }
 
-static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
+static void scx_start_task_running(struct rq *rq, struct task_struct *p)
 {
 	struct scx_sched *sch = scx_task_sched(p);
 
+	if (p->scx.flags & SCX_TASK_RUN_TRACKED)
+		return;
+
+	if (SCX_HAS_OP(sch, running))
+		SCX_CALL_OP_TASK(sch, running, rq, p);
+
+	p->scx.flags |= SCX_TASK_RUN_TRACKED;
+}
+
+static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type)
+{
+	bool first = type == SNT_PICK;
+	bool can_stop_tick;
+
+	if (type == SNT_REPICK)
+		return;
+
 	if (p->scx.flags & SCX_TASK_QUEUED) {
 		/*
 		 * Core-sched might decide to execute @p before it is
@@ -3019,18 +3350,31 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
 		 */
 		ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC);
 		scx_dispatch_dequeue(rq, p);
+		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 	}
 
 	p->se.exec_start = rq_clock_task(rq);
 
-	/* see dequeue_task_scx() on why we skip when !QUEUED */
-	if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED))
-		SCX_CALL_OP_TASK(sch, running, rq, p);
+	/*
+	 * See dequeue_task_scx() for why we skip when !QUEUED.
+	 *
+	 * During a normal switch (@first), a blocked task is only a provisional
+	 * donor. Proxy resolution may fail or migrate the donor to another CPU,
+	 * so defer ops.running() until scx_proxy_donor_start() confirms that
+	 * resolution succeeded.
+	 *
+	 * !@first denotes restoration after a SAVE/RESTORE cycle. The matching
+	 * dequeue already issued ops.stopping(), so restart the session here
+	 * regardless of the donor state.
+	 */
+	if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first))
+		scx_start_task_running(rq, p);
 
 	clr_task_runnable(p, true);
 
 	/* apply any pending out-of-band slice request before the tick decision */
 	apply_task_slice_oob(rq, p);
+	can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked;
 
 	/*
 	 * @p is getting newly scheduled or got kicked after someone updated its
@@ -3041,7 +3385,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
 	 * nohz. In the future, we might want to add a mechanism to update
 	 * load_avgs periodically on tick-stopped CPUs.
 	 */
-	if (p->scx.slice == SCX_SLICE_INF) {
+	if (can_stop_tick) {
 		if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
 			/*
 			 * Bypass mode always assigns finite slices, so @p
@@ -3062,13 +3406,24 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
 
 		/*
 		 * @rq still references the outgoing scheduling context. A finite
-		 * slice is sufficient by itself to require the tick.
+		 * slice or a blocked proxy donor is sufficient by itself to require
+		 * the tick.
 		 */
 		if (tick_nohz_full_cpu(cpu_of(rq)))
 			tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED);
 	}
 }
 
+void scx_proxy_donor_start(struct rq *rq)
+{
+	struct task_struct *donor = rq->donor;
+
+	lockdep_assert_rq_held(rq);
+
+	if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED))
+		scx_start_task_running(rq, donor);
+}
+
 static enum scx_cpu_preempt_reason
 preempt_reason_from_class(const struct sched_class *class)
 {
@@ -3142,13 +3497,39 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
 			scx_task_slice_ended(rq, p);
 	}
 
-	/* see dequeue_task_scx() on why we skip when !QUEUED */
-	if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED))
-		SCX_CALL_OP_TASK(sch, stopping, rq, p, true);
+	/*
+	 * Preserve the running session when proxy execution refreshes the same
+	 * donor around an execution-context switch on this rq.
+	 */
+	if (next != p && (p->scx.flags & SCX_TASK_QUEUED) &&
+	    (p->scx.flags & SCX_TASK_RUN_TRACKED)) {
+		if (SCX_HAS_OP(sch, stopping))
+			SCX_CALL_OP_TASK(sch, stopping, rq, p, true);
+
+		p->scx.flags &= ~SCX_TASK_RUN_TRACKED;
+	}
 
 	if (p->scx.flags & SCX_TASK_QUEUED) {
 		set_task_runnable(rq, p);
 
+		/* Delegate retained donor admission to its owning BPF scheduler. */
+		if (p->is_blocked) {
+			/*
+			 * If the donor is the same and only the mutex owner
+			 * changes, avoid triggering another ops.enqueue(): the
+			 * BPF scheduler has already admitted the donor, so it
+			 * can continue running.
+			 */
+			if (next == p)
+				goto switch_class;
+
+			if (WARN_ON_ONCE(!sch))
+				goto switch_class;
+			WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_BLOCKED));
+			scx_do_enqueue_task(rq, p, 0, -1);
+			goto switch_class;
+		}
+
 		/*
 		 * If @p has slice left and is being put, @p is getting
 		 * preempted by a higher priority scheduler class or core-sched
@@ -3164,7 +3545,6 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
 			if (p->scx.flags & SCX_TASK_IMMED) {
 				p->scx.flags |= SCX_TASK_REENQ_PREEMPTED;
 				scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
-				p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 			} else {
 				u64 enq_flags = 0;
 
@@ -3364,7 +3744,7 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r
 static struct task_struct *
 do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
 {
-	struct task_struct *prev = rq->curr;
+	struct task_struct *prev = rq->donor;
 	enum scx_dsp_verdict verdict;
 	struct task_struct *p;
 
@@ -3629,8 +4009,12 @@ static void set_cpus_allowed_scx(struct task_struct *p,
 	 *
 	 * Fine-grained memory write control is enforced by BPF making the const
 	 * designation pointless. Cast it away when calling the operation.
+	 *
+	 * The cid form receives the initial mask when the task is enabled and
+	 * hears about changes only afterwards, see struct scx_enable_args.
 	 */
-	if (SCX_HAS_OP(sch, set_cpumask))
+	if (SCX_HAS_OP(sch, set_cpumask) &&
+	    (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED))
 		scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
 }
 
@@ -3714,8 +4098,26 @@ static void rq_online_scx(struct rq *rq)
 
 static void rq_offline_scx(struct rq *rq)
 {
+	struct task_struct *p, *n;
+
 	rq->scx.flags &= ~SCX_RQ_ONLINE;
+
+	/* sched domain rebuilds call rq_offline with the CPU staying alive */
+	if (cpu_active(cpu_of(rq)))
+		return;
+
 	scx_rescue_flush(rq);
+
+	/*
+	 * An offline CPU no longer calls ops.dispatch(). Re-enqueue its tasks
+	 * onto the local DSQ so that they run here and balance_push() moves
+	 * them off.
+	 */
+	list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) {
+		if (p->scx.dsq == &rq->scx.local_dsq)
+			continue;
+		guard(sched_change)(p, DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK);
+	}
 }
 
 static bool check_rq_for_timeouts(struct rq *rq)
@@ -3806,9 +4208,9 @@ void scx_tick(struct rq *rq)
 	update_other_load_avgs(rq);
 }
 
-static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
+static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued)
 {
-	struct scx_sched *sch = scx_task_sched(curr);
+	struct scx_sched *sch = scx_task_sched(donor);
 
 	update_curr_scx(rq);
 
@@ -3817,12 +4219,18 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
 	 * management.
 	 */
 	if (scx_bypassing(sch, cpu_of(rq)))
-		scx_set_task_slice(curr, 0);
+		scx_set_task_slice(donor, 0);
 	else if (SCX_HAS_OP(sch, tick))
-		SCX_CALL_OP_TASK(sch, tick, rq, curr);
+		SCX_CALL_OP_TASK(sch, tick, rq, donor);
 
-	if (!curr->scx.slice)
-		resched_curr(rq);
+	if (!donor->scx.slice) {
+		/* the slice can't be trusted while bypassing */
+		if (READ_ONCE(donor->scx.lazy_resched) &&
+		    !scx_bypassing(sch, cpu_of(rq)))
+			scx_resched_curr_lazy(rq);
+		else
+			resched_curr(rq);
+	}
 }
 
 #ifdef CONFIG_EXT_GROUP_SCHED
@@ -3938,9 +4346,29 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p)
 		weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
 
 	p->scx.weight = sched_weight_to_cgroup(weight);
+	p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED;
 
-	if (SCX_HAS_OP(sch, enable))
-		SCX_CALL_OP_TASK(sch, enable, rq, p);
+	if (SCX_HAS_OP(sch, enable)) {
+		if (scx_is_cid_type()) {
+			struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr);
+			struct scx_enable_args args = {
+				.cmask_arena_addr = scx_kaddr_to_arena(sch, cmask),
+			};
+
+			SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args);
+		} else {
+			SCX_CALL_OP_TASK(sch, enable, rq, p);
+		}
+	}
+
+	/*
+	 * The initial mask also goes out through set_cmask() so a scheduler can
+	 * track affinity there alone, and before set_weight() so that the mask
+	 * is in place when weight-dependent state is derived, see struct
+	 * scx_enable_args.
+	 */
+	if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
+		scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr);
 
 	if (SCX_HAS_OP(sch, set_weight))
 		SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight);
@@ -4283,9 +4711,10 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p)
 
 	/*
 	 * set_cpus_allowed_scx() is not called while @p is associated with a
-	 * different scheduler class. Keep the BPF scheduler up-to-date.
+	 * different scheduler class. Keep the BPF scheduler up-to-date. The cid
+	 * form gets its mask from scx_enable_task().
 	 */
-	if (SCX_HAS_OP(sch, set_cpumask))
+	if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask))
 		scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr);
 }
 
@@ -4313,7 +4742,7 @@ int scx_check_setscheduler(struct task_struct *p, int policy)
 {
 	lockdep_assert_rq_held(task_rq(p));
 
-	/* if disallow, reject transitioning into SCX */
+	/* If disallow, reject transitioning into SCX. */
 	if (scx_enabled() && READ_ONCE(p->scx.disallow) &&
 	    p->policy != policy && policy == SCHED_EXT)
 		return -EACCES;
@@ -4321,6 +4750,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy)
 	return 0;
 }
 
+/*
+ * Don't carry a donor retained by another class into sched_ext. The caller
+ * has updated the rq clock and invokes this immediately before
+ * sched_change_begin() records the task's queued state.
+ */
+void scx_prepare_setscheduler(struct task_struct *p, int policy)
+{
+	lockdep_assert_held(&p->pi_lock);
+	lockdep_assert_rq_held(task_rq(p));
+
+	if (scx_enabled() && p->policy != policy && policy == SCHED_EXT)
+		sched_proxy_block_task(task_rq(p), p);
+}
+
 static void process_ddsp_deferred_locals(struct rq *rq)
 {
 	struct task_struct *p;
@@ -4404,6 +4847,17 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p,
 	return *reenq_flags & SCX_REENQ_ANY;
 }
 
+/*
+ * The dispatcher stores the final ops_state after dropping the DSQ lock, so @p
+ * can be found on a DSQ while still %SCX_OPSS_DISPATCHING. Reenqueueing @p
+ * before that store lands would have it clobber the new %SCX_OPSS_QUEUED.
+ */
+void scx_reenq_wait_dispatching(struct task_struct *p)
+{
+	if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING))
+		wait_ops_state(p, SCX_OPSS_DISPATCHING);
+}
+
 static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
 {
 	LIST_HEAD(tasks);
@@ -4447,10 +4901,10 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
 		if (!local_task_should_reenq(rq, p, &reenq_flags, &reason))
 			continue;
 
+		scx_reenq_wait_dispatching(p);
 		scx_dispatch_dequeue(rq, p);
 
-		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
-			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+		WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
 		p->scx.flags |= reason;
 
 		list_add_tail(&p->scx.dsq_list.node, &tasks);
@@ -4461,21 +4915,20 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
 
 		scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
 
-		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
 		nr_enqueued++;
 	}
 
 	/*
-	 * The revoke that scheduled this scan may have raced the pick: curr
+	 * The revoke that scheduled this scan may have raced the pick: donor
 	 * may be a now-capless task, either one that kept running or one
 	 * promoted off the local DSQ between the ecaps sync and this scan.
 	 * Zero the slice to evict it. The enqueue gate blocks new capless
 	 * inserts, so no later pick can slip through after the scan.
 	 */
 	if ((reenq_flags & SCX_REENQ_CAP_REVOKE) &&
-	    rq->curr->sched_class == &ext_sched_class &&
-	    scx_task_reenq_on_cap_revoke(rq, rq->curr)) {
-		scx_set_task_slice(rq->curr, 0);
+	    rq->donor->sched_class == &ext_sched_class &&
+	    scx_task_reenq_on_cap_revoke(rq, rq->donor)) {
+		scx_set_task_slice(rq->donor, 0);
 		resched_curr(rq);
 	}
 
@@ -4570,17 +5023,15 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag
 		}
 
 		/* @p is on @dsq, its rq and @dsq are locked */
+		scx_reenq_wait_dispatching(p);
 		dispatch_dequeue_locked(p, dsq);
 		raw_spin_unlock(&dsq->lock);
 
-		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
-			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
+		WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
 		p->scx.flags |= reason;
 
 		scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1);
 
-		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
-
 		if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) {
 			scx_rq_lock_drop(locked_rq);
 			raw_spin_rq_unlock(locked_rq);
@@ -4642,6 +5093,60 @@ static void process_deferred_reenq_users(struct rq *rq)
 	}
 }
 
+/*
+ * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their
+ * owning BPF schedulers choose placement again. Proxy-active tasks remain
+ * parked and rearm the retry notification for a later proxy resolution.
+ *
+ * A task can be re-rejected repeatedly. Reenqueues are bounded per task by
+ * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning
+ * scheduler.
+ */
+static void scx_reenq_reject(struct rq *rq)
+{
+	LIST_HEAD(tasks);
+	struct task_struct *p, *n;
+
+	lockdep_assert_rq_held(rq);
+
+	if (list_empty(&rq->scx.reject_dsq.list))
+		return;
+
+	/*
+	 * Move ready tasks to a private list so a task re-rejected by
+	 * scx_do_enqueue_task() below isn't revisited this round.
+	 */
+	list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
+		u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK;
+
+		WARN_ON_ONCE(!reason);
+
+		if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) {
+			/* Affinity machinery will dequeue and reactivate @p. */
+			if (p->migration_pending)
+				continue;
+
+			if (task_on_cpu(rq, p) || task_current_donor(rq, p)) {
+				rq->scx.flags |= SCX_RQ_PROXY_RETRY;
+				continue;
+			}
+		} else {
+			WARN_ON_ONCE(p->migration_pending);
+		}
+
+		scx_reenq_wait_dispatching(p);
+		scx_dispatch_dequeue(rq, p);
+
+		list_add_tail(&p->scx.dsq_list.node, &tasks);
+	}
+
+	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
+		list_del_init(&p->scx.dsq_list.node);
+
+		scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
+	}
+}
+
 static void run_deferred(struct rq *rq)
 {
 	process_ddsp_deferred_locals(rq);
@@ -4658,14 +5163,18 @@ static void run_deferred(struct rq *rq)
 #ifdef CONFIG_NO_HZ_FULL
 bool scx_can_stop_tick(struct rq *rq)
 {
-	struct task_struct *p = rq->curr;
+	struct task_struct *p = rq->donor;
 	struct scx_sched *sch = scx_task_sched(p);
 
+	/* Proxy execution is conservatively tick-driven for now. */
+	if (rq->scx.flags & SCX_RQ_PROXY_TICK)
+		return false;
+
 	if (p->sched_class != &ext_sched_class)
 		return true;
 
 	/*
-	 * @rq->curr may still reference an outgoing EXT task after it has been
+	 * @rq->donor may still reference an outgoing EXT task after it has been
 	 * dequeued. If no EXT tasks are accounted on @rq, ignore its stale
 	 * slice state. If another task is dispatched from a DSQ,
 	 * set_next_task_scx() will update the dependency for the incoming task.
@@ -4686,7 +5195,8 @@ bool scx_can_stop_tick(struct rq *rq)
 	/*
 	 * @rq can dispatch from different DSQs, so we can't tell whether it
 	 * needs the tick or not by looking at nr_running. Allow stopping ticks
-	 * iff the BPF scheduler indicated so. See set_next_task_scx().
+	 * iff set_next_task_scx() determined that the selected scheduling context
+	 * can run tickless.
 	 */
 	return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;
 }
@@ -4701,7 +5211,7 @@ void scx_tg_init(struct task_group *tg)
 	tg->scx.weight = CGROUP_WEIGHT_DFL;
 	tg->scx.bw_period_us = default_bw_period_us();
 	tg->scx.bw_quota_us = RUNTIME_INF;
-	tg->scx.idle = false;
+	tg->scx.sched_idle = false;
 }
 
 /**
@@ -4784,7 +5294,7 @@ int scx_tg_online(struct task_group *tg)
 				  .bw_period_us = tg->scx.bw_period_us,
 				  .bw_quota_us = tg->scx.bw_quota_us,
 				  .bw_burst_us = tg->scx.bw_burst_us,
-				  .sched_idle = tg->scx.idle };
+				  .sched_idle = tg->scx.sched_idle };
 
 			ret = SCX_CALL_OP_RET(sch, cgroup_init,
 					      NULL, tg->css.cgroup, &args);
@@ -4951,11 +5461,11 @@ void scx_group_set_idle(struct task_group *tg, bool idle)
 	sch = scx_tg_knob_sched(tg);
 
 	if (scx_cgroup_enabled && sch && SCX_HAS_OP(sch, cgroup_set_idle) &&
-	    tg->scx.idle != idle)
+	    tg->scx.sched_idle != idle)
 		SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
 
 	/* Update the task group's idle state */
-	tg->scx.idle = idle;
+	tg->scx.sched_idle = idle;
 
 	percpu_up_read(&scx_cgroup_ops_rwsem);
 }
@@ -5069,12 +5579,16 @@ s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch)
 	dsq->id = dsq_id;
 	dsq->sched = sch;
 
-	dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu);
-	if (!dsq->pcpu)
+	/* per-DSQ deferred reenq state is only needed for user DSQs */
+	if (dsq_id & SCX_DSQ_FLAG_BUILTIN)
+		return 0;
+
+	dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu);
+	if (!dsq->pcpu_user)
 		return -ENOMEM;
 
 	for_each_possible_cpu(cpu) {
-		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
+		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
 
 		pcpu->dsq = dsq;
 		INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node);
@@ -5087,8 +5601,11 @@ static void exit_dsq(struct scx_dispatch_q *dsq)
 {
 	s32 cpu;
 
+	if (!dsq->pcpu_user)
+		return;
+
 	for_each_possible_cpu(cpu) {
-		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu);
+		struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu);
 		struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user;
 		struct rq *rq = cpu_rq(cpu);
 
@@ -5102,7 +5619,7 @@ static void exit_dsq(struct scx_dispatch_q *dsq)
 		}
 	}
 
-	free_percpu(dsq->pcpu);
+	free_percpu(dsq->pcpu_user);
 }
 
 static void free_dsq_rcufn(struct rcu_head *rcu)
@@ -5206,7 +5723,7 @@ static int scx_cgroup_init(struct scx_sched *sch)
 				.bw_period_us = tg->scx.bw_period_us,
 				.bw_quota_us = tg->scx.bw_quota_us,
 				.bw_burst_us = tg->scx.bw_burst_us,
-				.sched_idle = tg->scx.idle,
+				.sched_idle = tg->scx.sched_idle,
 			};
 
 			ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args);
@@ -5400,6 +5917,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
 		free_cpumask_var(pcpu->cpus_to_kick);
 		free_cpumask_var(pcpu->cpus_to_kick_if_idle);
 		free_cpumask_var(pcpu->cpus_to_preempt);
+		free_cpumask_var(pcpu->cpus_to_preempt_lazy);
 		free_cpumask_var(pcpu->cpus_to_wait);
 
 		exit_dsq(scx_bypass_dsq(sch, cpu));
@@ -5878,7 +6396,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node)
 
 	/*
 	 * We don't want CPUs to have more than $nr_donor_target tasks and
-	 * balancing to fill donee CPUs upto $nr_target. Once targets are
+	 * balancing to fill donee CPUs up to $nr_target. Once targets are
 	 * calculated, find the donee CPUs.
 	 */
 	nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus);
@@ -6193,9 +6711,9 @@ void scx_bypass(struct scx_sched *sch, bool bypass)
 
 			/*
 			 * Bypass trumps protection. Cycling clears for queued
-			 * tasks but current task needs explicit stripping.
+			 * tasks but the current donor needs explicit stripping.
 			 */
-			if (bypass && task_current(rq, p))
+			if (bypass && task_current_donor(rq, p))
 				scx_task_slice_ended(rq, p);
 
 			/* cycling deq/enq is enough, see the function comment */
@@ -6903,6 +7421,8 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
 	scx_rescue_dump(&ns, rq);
 	scx_dump_line(&ns, "          curr=%s[%d] class=%ps",
 		      rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
+	scx_dump_line(&ns, "          donor=%s[%d] class=%ps",
+		      rq->donor->comm, rq->donor->pid, rq->donor->sched_class);
 	if (!cpumask_empty(pcpu->cpus_to_kick))
 		scx_dump_line(&ns, "  cpus_to_kick   : %*pb",
 			      cpumask_pr_args(pcpu->cpus_to_kick));
@@ -6912,6 +7432,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
 	if (!cpumask_empty(pcpu->cpus_to_preempt))
 		scx_dump_line(&ns, "  cpus_to_preempt: %*pb",
 			      cpumask_pr_args(pcpu->cpus_to_preempt));
+	if (!cpumask_empty(pcpu->cpus_to_preempt_lazy))
+		scx_dump_line(&ns, "  preempt_lazy   : %*pb",
+			      cpumask_pr_args(pcpu->cpus_to_preempt_lazy));
 	if (!cpumask_empty(pcpu->cpus_to_wait))
 		scx_dump_line(&ns, "  cpus_to_wait   : %*pb",
 			      cpumask_pr_args(pcpu->cpus_to_wait));
@@ -6946,6 +7469,10 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
 	if (rq->curr->sched_class == &ext_sched_class &&
 	    (dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
 		scx_dump_task(sch, s, dctx, rq, rq->curr, '*');
+	if (rq->donor != rq->curr &&
+	    rq->donor->sched_class == &ext_sched_class &&
+	    (dump_all_tasks || scx_task_on_sched(sch, rq->donor)))
+		scx_dump_task(sch, s, dctx, rq, rq->donor, ' ');
 
 	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
 		if (dump_all_tasks || scx_task_on_sched(sch, p))
@@ -7229,6 +7756,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 		if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) ||
 		    !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) ||
 		    !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) ||
+		    !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) ||
 		    !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) {
 			ret = -ENOMEM;
 			goto err_free_pcpu;
@@ -7364,6 +7892,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 		free_cpumask_var(pcpu->cpus_to_kick);
 		free_cpumask_var(pcpu->cpus_to_kick_if_idle);
 		free_cpumask_var(pcpu->cpus_to_preempt);
+		free_cpumask_var(pcpu->cpus_to_preempt_lazy);
 		free_cpumask_var(pcpu->cpus_to_wait);
 	}
 	for_each_possible_cpu(cpu) {
@@ -7454,6 +7983,11 @@ int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
 		return -EINVAL;
 	}
 
+	if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) {
+		scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented");
+		return -EINVAL;
+	}
+
 	/*
 	 * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched
 	 * may set it to declare a dependency; reject if the root hasn't
@@ -7843,6 +8377,10 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 
 		if (old_class != new_class)
 			queue_flags |= DEQUEUE_CLASS;
+		if (new_class == &ext_sched_class) {
+			scx_prepare_task_sched_change(p);
+			queue_flags |= DEQUEUE_NOCLOCK;
+		}
 
 		scoped_guard (sched_change, p, queue_flags) {
 			scx_set_task_slice(p, READ_ONCE(sch->slice_dfl));
@@ -8356,10 +8894,11 @@ static struct bpf_struct_ops bpf_sched_ext_ops = {
 /*
  * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types
  * identical, only param names differ across structs) are reused. Some need
- * fresh stubs, set_cmask due to an argument type difference and the sub-sched
- * notifiers because no cpu-form stub exists to reuse.
+ * fresh stubs, set_cmask and enable due to argument differences and the
+ * sub-sched notifiers because no cpu-form stub exists to reuse.
  */
 static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {}
+static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {}
 static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {}
 static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {}
 
@@ -8380,7 +8919,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
 	.update_idle		= sched_ext_ops__update_idle,
 	.init_task		= sched_ext_ops__init_task,
 	.exit_task		= sched_ext_ops__exit_task,
-	.enable			= sched_ext_ops__enable,
+	.enable			= sched_ext_ops_cid__enable,
 	.disable		= sched_ext_ops__disable,
 #ifdef CONFIG_EXT_GROUP_SCHED
 	.cpuctl_init		= sched_ext_ops__cgroup_init,
@@ -8490,13 +9029,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
 {
 	struct rq *rq = cpu_rq(cpu);
 	struct scx_rq *this_scx = &this_rq->scx;
+	struct rq_flags rf;
 	const struct sched_class *cur_class;
 	bool should_wait = false;
 	bool kickable;
-	unsigned long flags;
+	bool preempt, preempt_lazy, wait, immediate;
 
-	raw_spin_rq_lock_irqsave(rq, flags);
-	cur_class = rq->curr->sched_class;
+	rq_lock_irqsave(rq, &rf);
+	cur_class = rq->donor->sched_class;
+	preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt);
+	preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy);
+	wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait);
+	/*
+	 * Immediate preemption, waiting and a plain kick take precedence over
+	 * lazy preemption. The lazy request still clears the slice, so all
+	 * accumulated requests are served.
+	 */
+	immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick);
 
 	/*
 	 * During CPU hotplug, a CPU may depend on kicking itself to make
@@ -8510,19 +9059,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
 		   !sched_class_above(cur_class, &ext_sched_class);
 
 	if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) {
-		if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) {
+		if (preempt || preempt_lazy) {
 			if (cur_class == &ext_sched_class) {
 				u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
 
-				if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps)))
+				if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) {
 					__scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1);
-				else if (unlikely(!scx_set_task_slice(rq->curr, 0)))
+					/* degrade to a plain, immediate kick */
+					immediate = true;
+				} else if (unlikely(!scx_set_task_slice(rq->donor, 0))) {
 					__scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1);
+				}
 			}
 			cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
+			cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy);
 		}
 
-		if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) {
+		if (wait) {
 			if (cur_class == &ext_sched_class) {
 				cpumask_set_cpu(cpu, this_scx->cpus_to_sync);
 				ksyncs[cpu] = rq->scx.kick_sync;
@@ -8531,17 +9084,21 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
 			cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
 		}
 
-		resched_curr(rq);
+		if (immediate)
+			resched_curr(rq);
+		else
+			scx_resched_curr_lazy(rq);
 	} else {
 		/* a kickable cpu was skipped solely for the missing caps */
 		if (kickable)
 			__scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1);
 		cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
+		cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy);
 		cpumask_clear_cpu(cpu, pcpu->cpus_to_wait);
 	}
 
 	scx_rq_lock_drop(rq);
-	raw_spin_rq_unlock_irqrestore(rq, flags);
+	rq_unlock_irqrestore(rq, &rf);
 
 	return should_wait;
 }
@@ -8591,7 +9148,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work)
 	list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) {
 		list_del_init(&pcpu->to_kick_node);
 
-		for_each_cpu(cpu, pcpu->cpus_to_kick) {
+		for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) {
 			should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs);
 			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick);
 			cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle);
@@ -8722,8 +9279,8 @@ void __init init_sched_ext_class(void)
 
 		/* local_dsq's sch will be set during scx_root_enable() */
 		BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
-#ifdef CONFIG_EXT_SUB_SCHED
 		BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
+#ifdef CONFIG_EXT_SUB_SCHED
 		scx_rescue_init(rq);
 #endif
 
@@ -8856,7 +9413,7 @@ __bpf_kfunc_start_defs();
  * task is inserted.
  *
  * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
- * and this function can be called upto ops.dispatch_max_batch times to insert
+ * and this function can be called up to ops.dispatch_max_batch times to insert
  * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
  * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the
  * counter.
@@ -8941,7 +9498,9 @@ struct scx_bpf_dsq_insert_vtime_args {
  *
  * @args->vtime ordering is according to time_before64() which considers
  * wrapping. A numerically larger vtime may indicate an earlier position in the
- * ordering and vice-versa.
+ * ordering and vice-versa. vtime is a rolling cursor and values used for
+ * ordering within a given DSQ should stay less than 2^63 apart for
+ * time_before64() ordering to remain well-defined.
  *
  * A DSQ can only be used as a FIFO or priority queue at any given time and this
  * function must not be called on a DSQ which already has one or more FIFO tasks
@@ -9511,8 +10070,10 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
 		return false;
 
 	/*
-	 * Directly write only when we hold the lock of the rq @p is queued or
-	 * running on. See the write rules above.
+	 * Directly write only when we hold the lock of the rq @p is queued on or
+	 * provides the current scheduling context for. Under proxy execution,
+	 * rq->donor owns and consumes the slice while rq->curr executes on its
+	 * behalf. See the slice write rules above.
 	 *
 	 * While @p is queued on a user DSQ or in the BPF scheduler,
 	 * synchronization is the scheduler's responsibility. This write can
@@ -9526,7 +10087,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
 	locked_rq = scx_locked_rq();
 	if (!locked_rq ||
 	    (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) &&
-	     !task_current(locked_rq, p))) {
+	     !task_current_donor(locked_rq, p))) {
 		set_task_slice_oob(sch, p, slice);
 		return true;
 	}
@@ -9567,18 +10128,56 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime,
 	return true;
 }
 
+/**
+ * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode
+ * @p: task of interest
+ * @lazy: whether slice expiry should request lazy rescheduling
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Choose whether depletion of @p's slice at the scheduler tick requests lazy or
+ * immediate rescheduling. This is a persistent per-task policy and does not
+ * itself clear @p's slice or request rescheduling, unlike the
+ * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations.
+ *
+ * With lazy rescheduling enabled, a task interrupted in user space still
+ * reschedules before returning to user space. A task interrupted in the kernel
+ * may continue until its next return to user space or the next scheduler tick,
+ * which promotes the request. If lazy preemption is disabled at runtime, the
+ * request behaves like an immediate reschedule.
+ *
+ * @p must be on the calling scheduler.
+ *
+ * Return %true on success, %false if @p is not on the calling scheduler.
+ */
+__bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy,
+					       const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+
+	/*
+	 * clang's register locations for @lazy make pahole 1.32 drop this kfunc
+	 * from BTF. Keep @lazy in memory so its location names no register.
+	 */
+	barrier_data(&lazy);
+	guard(rcu)();
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch || !scx_task_on_sched(sch, p)))
+		return false;
+
+	WRITE_ONCE(p->scx.lazy_resched, lazy);
+	return true;
+}
+
 void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 {
 	struct scx_sched_pcpu *pcpu;
 	struct rq *this_rq;
 	unsigned long irq_flags;
 
-	/*
-	 * The per-cpu kick list is guarded only by local_irq_save(), which does
-	 * not mask NMIs, so kicking from NMI could corrupt it and is unsupported.
-	 */
-	if (unlikely(in_nmi())) {
-		scx_error(sch, "scx_bpf_kick_cpu() called from NMI");
+	if (!scx_kf_allowed_ctx(sch))
+		return;
+	if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) {
+		scx_error(sch, "invalid kick flags 0x%llx", flags);
 		return;
 	}
 
@@ -9606,7 +10205,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 	if (flags & SCX_KICK_IDLE) {
 		struct rq *target_rq = cpu_rq(cpu);
 
-		if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT)))
+		if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY |
+				     SCX_KICK_WAIT)))
 			scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE");
 
 		if (raw_spin_rq_trylock(target_rq)) {
@@ -9620,12 +10220,17 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 		}
 		cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle);
 	} else {
-		cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
-
+		/*
+		 * Accumulate requests and resolve their precedence at delivery.
+		 */
 		if (flags & SCX_KICK_PREEMPT)
 			cpumask_set_cpu(cpu, pcpu->cpus_to_preempt);
+		if (flags & SCX_KICK_PREEMPT_LAZY)
+			cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy);
 		if (flags & SCX_KICK_WAIT)
 			cpumask_set_cpu(cpu, pcpu->cpus_to_wait);
+		if (!(flags & SCX_KICK_PREEMPT_LAZY))
+			cpumask_set_cpu(cpu, pcpu->cpus_to_kick);
 	}
 
 	if (list_empty(&pcpu->to_kick_node))
@@ -9665,8 +10270,9 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux
  * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the
  * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a
  * kick is dropped if the caller lacks baseline access on @cid, and a
- * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks
- * %SCX_CAP_PREEMPT for a task outside its subtree.
+ * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain
+ * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its
+ * subtree.
  */
 __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux)
 {
@@ -9748,8 +10354,13 @@ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
 
 	guard(rcu)();
 	sch = scx_prog_sched(aux);
-	if (sch)
-		destroy_dsq(sch, dsq_id);
+	if (unlikely(!sch))
+		return;
+
+	if (!scx_kf_allowed_ctx(sch))
+		return;
+
+	destroy_dsq(sch, dsq_id);
 }
 
 /**
@@ -9786,6 +10397,9 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
 	if (unlikely(!sch))
 		return -ENODEV;
 
+	if (!scx_kf_allowed_ctx(sch))
+		return -EDEADLK;
+
 	if (flags & ~__SCX_DSQ_ITER_USER_FLAGS)
 		return -EINVAL;
 
@@ -9912,6 +10526,9 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags,
 		return;
 	}
 
+	if (!scx_kf_allowed_ctx(sch))
+		return;
+
 	/* not specifying any filter bits is the same as %SCX_REENQ_ANY */
 	if (!(reenq_flags & __SCX_REENQ_FILTER_MASK))
 		reenq_flags |= SCX_REENQ_ANY;
@@ -10299,6 +10916,9 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au
 	if (unlikely(!sch))
 		return;
 
+	if (!scx_kf_allowed_ctx(sch))
+		return;
+
 	scx_cpuperf_set(sch, cpu, perf);
 }
 
@@ -10324,6 +10944,10 @@ __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf,
 	sch = scx_prog_sched(aux);
 	if (unlikely(!sch))
 		return -ENODEV;
+
+	if (!scx_kf_allowed_ctx(sch))
+		return -EDEADLK;
+
 	cpu = scx_cid_to_cpu(sch, cid);
 	if (cpu < 0)
 		return cpu;
@@ -10403,8 +11027,7 @@ __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux)
 	if (unlikely(!online))
 		return NULL;
 
-	/* BPF rebases by the low 32 bits, like __arena callback args */
-	return (void *)((unsigned long)online - sch->arena_kern_base);
+	return (void *)scx_kaddr_to_arena(sch, online);
 }
 
 /**
@@ -10457,12 +11080,17 @@ __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask)
 }
 
 /**
- * scx_bpf_task_running - Is task currently running?
+ * scx_bpf_task_running - Is task the current scheduling context?
  * @p: task of interest
+ *
+ * Under proxy execution, this reports the donor rather than the task whose
+ * code is physically executing. The physical execution context is intentionally
+ * not exposed to the BPF scheduler, which continues to observe the donor as the
+ * running scheduling context.
  */
 __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p)
 {
-	return task_rq(p)->curr == p;
+	return rcu_access_pointer(task_rq(p)->donor) == p;
 }
 
 /**
@@ -10523,10 +11151,15 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux)
 }
 
 /**
- * scx_bpf_cpu_curr - Return remote CPU's curr task
+ * scx_bpf_cpu_curr - Return remote CPU's current scheduling context
  * @cpu: CPU of interest
  * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
  *
+ * Under proxy execution, this returns the donor, which supplies the scheduling
+ * policy and runtime budget, rather than the task whose code is physically
+ * executing. The physical execution context is intentionally not exposed to
+ * the BPF scheduler.
+ *
  * Callers must hold RCU read lock (KF_RCU).
  */
 __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux)
@@ -10542,7 +11175,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_
 	if (!scx_cpu_valid(sch, cpu, NULL))
 		return NULL;
 
-	return rcu_dereference(cpu_rq(cpu)->curr);
+	return rcu_dereference(cpu_rq(cpu)->donor);
 }
 
 /**
@@ -10566,7 +11199,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_
 	cpu = scx_cid_to_cpu(sch, cid);
 	if (cpu < 0)
 		return NULL;
-	return rcu_dereference(cpu_rq(cpu)->curr);
+	return rcu_dereference(cpu_rq(cpu)->donor);
 }
 
 /**
@@ -10747,6 +11380,7 @@ __bpf_kfunc_end_defs();
 BTF_KFUNCS_START(scx_kfunc_ids_any)
 BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU);
 BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU);
+BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU);
 BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h
index 0b7fc46aee08c8..d32bc01071b01a 100644
--- a/kernel/sched/ext/ext.h
+++ b/kernel/sched/ext/ext.h
@@ -18,9 +18,28 @@ bool scx_can_stop_tick(struct rq *rq);
 void scx_rq_activate(struct rq *rq);
 void scx_rq_deactivate(struct rq *rq);
 int scx_check_setscheduler(struct task_struct *p, int policy);
+void scx_prepare_setscheduler(struct task_struct *p, int policy);
 bool task_should_scx(int policy);
 bool scx_allow_ttwu_queue(const struct task_struct *p);
+bool __scx_allow_proxy_exec(const struct task_struct *p);
+void scx_proxy_donor_start(struct rq *rq);
+void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next);
 void init_sched_ext_class(void);
+void __scx_update_idle(struct rq *rq, bool idle, bool do_notify);
+
+static inline bool scx_allow_proxy_exec(const struct task_struct *p)
+{
+	if (scx_enabled())
+		return __scx_allow_proxy_exec(p);
+
+	return true;
+}
+
+static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify)
+{
+	if (scx_enabled())
+		__scx_update_idle(rq, idle, do_notify);
+}
 
 static inline u32 scx_cpuperf_target(s32 cpu)
 {
@@ -52,24 +71,18 @@ static inline bool scx_can_stop_tick(struct rq *rq) { return true; }
 static inline void scx_rq_activate(struct rq *rq) {}
 static inline void scx_rq_deactivate(struct rq *rq) {}
 static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; }
+static inline void scx_prepare_setscheduler(struct task_struct *p, int policy) {}
 static inline bool task_on_scx(const struct task_struct *p) { return false; }
 static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; }
+static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; }
+static inline void scx_proxy_donor_start(struct rq *rq) {}
+static inline void scx_proxy_reenqueue_retry(struct rq *rq,
+					     struct task_struct *next) {}
 static inline void init_sched_ext_class(void) {}
+static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {}
 
 #endif	/* CONFIG_SCHED_CLASS_EXT */
 
-#ifdef CONFIG_SCHED_CLASS_EXT
-void __scx_update_idle(struct rq *rq, bool idle, bool do_notify);
-
-static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify)
-{
-	if (scx_enabled())
-		__scx_update_idle(rq, idle, do_notify);
-}
-#else
-static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {}
-#endif
-
 #ifdef CONFIG_CGROUP_SCHED
 #ifdef CONFIG_EXT_GROUP_SCHED
 void scx_tg_init(struct task_group *tg);
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index aa9fb6de0ad60b..de5ec21384b0e7 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -153,7 +153,18 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i
 	nodemask_t *unvisited;
 	s32 cpu = -EBUSY;
 
-	preempt_disable();
+	/*
+	 * @per_cpu_unvisited is per-CPU scratch and the idle kfuncs can be
+	 * called from IRQ-enabled contexts, so mask IRQs to keep a nested
+	 * invocation from clobbering the mask an outer invocation is still
+	 * iterating.
+	 *
+	 * NMI nesting is not handled: there is no legitimate reason to call
+	 * pick_idle from NMI and doing so poses no crash risk, so such a
+	 * caller is on its own.
+	 */
+	guard(irqsave)();
+
 	unvisited = this_cpu_ptr(&per_cpu_unvisited);
 
 	/*
@@ -183,7 +194,6 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i
 		if (cpu >= 0)
 			break;
 	}
-	preempt_enable();
 
 	return cpu;
 }
diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h
index ed423bcc26b87c..8de9d8fb4b25fa 100644
--- a/kernel/sched/ext/inlines.h
+++ b/kernel/sched/ext/inlines.h
@@ -70,7 +70,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
 #endif	/* CONFIG_EXT_SUB_SCHED */
 	}
 
-	if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
+	/*
+	 * scx_rq_online() can't be used. Its cpu_active() test goes false
+	 * before CPU hotplug waits for an RCU grace period, and
+	 * rq_offline_scx() moves this CPU's tasks to the local DSQ only after
+	 * the wait. The grace period can depend on those tasks running.
+	 */
+	if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !(rq->scx.flags & SCX_RQ_ONLINE))
 		return SCX_DSP_NONE;
 
 	dspc->rq = rq;
@@ -129,8 +135,10 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
 	 * scheduler's ops.dispatch() doesn't yield any tasks.
 	 */
 	if (scx_bypass_dsp_enabled(sch) &&
-	    scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
+	    scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
+		__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
 		return SCX_DSP_LOCAL;
+	}
 
 	return SCX_DSP_NONE;
 }
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 3464e0f113c163..d65fec631bdf7f 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -215,6 +215,34 @@ enum scx_ops_flags {
 	 */
 	SCX_OPS_TID_TO_TASK		= 1LLU << 8,
 
+	/*
+	 * If set, tasks default to requesting lazy rescheduling when their
+	 * slice runs out at the tick, the way fair.c expires a slice from
+	 * update_curr(). The default is copied to p->scx.lazy_resched
+	 * immediately before ops.enable(), after which
+	 * scx_bpf_task_set_lazy_resched() may override it per task.
+	 *
+	 * A task in user space still reschedules on the way back from the tick;
+	 * a task in the kernel runs on to its next return to user space or to
+	 * the next tick, which promotes the request. When lazy preemption is
+	 * disabled at runtime, the request behaves like an immediate
+	 * reschedule. Rescheduling while bypassing stays immediate.
+	 */
+	SCX_OPS_LAZY_RESCHED		= 1LLU << 9,
+
+	/*
+	 * If set, mutex-blocked tasks remain runnable as proxy donors and are
+	 * passed to ops.enqueue() with %SCX_ENQ_BLOCKED. The BPF scheduler controls
+	 * when donors are dispatched and whether they should preempt other work.
+	 *
+	 * If clear, mutex-blocked tasks are removed from the runqueue normally
+	 * and cannot donate their scheduling context through proxy execution.
+	 *
+	 * For blocked donors, this flag takes precedence over
+	 * %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED.
+	 */
+	SCX_OPS_ENQ_BLOCKED		= 1LLU << 10,
+
 	SCX_OPS_ALL_FLAGS		= SCX_OPS_KEEP_BUILTIN_IDLE |
 					  SCX_OPS_ENQ_LAST |
 					  SCX_OPS_ENQ_EXITING |
@@ -223,7 +251,9 @@ enum scx_ops_flags {
 					  SCX_OPS_SWITCH_PARTIAL |
 					  SCX_OPS_BUILTIN_IDLE_PER_NODE |
 					  SCX_OPS_ALWAYS_ENQ_IMMED |
-					  SCX_OPS_TID_TO_TASK,
+					  SCX_OPS_TID_TO_TASK |
+					  SCX_OPS_LAZY_RESCHED |
+					  SCX_OPS_ENQ_BLOCKED,
 
 	/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */
 	__SCX_OPS_INTERNAL_MASK		= 0xffLLU << 56,
@@ -250,6 +280,31 @@ struct scx_exit_task_args {
 	bool cancelled;
 };
 
+/**
+ * struct scx_enable_args - Argument container for cid-form ops.enable()
+ * @cmask_arena_addr: BPF arena address of the cmask of cids the task may run on
+ *
+ * @cmask_arena_addr is the task's affinity as it enters the scheduler.
+ * set_cmask() delivers the same mask right after enable(), before set_weight()
+ * and the first enqueue, then every affinity change afterwards, and is never
+ * called before enable(). A scheduler may therefore track affinity in
+ * set_cmask() alone.
+ *
+ * The kernel builds the mask in the scheduler arena from its own geometry, so
+ * the header is valid regardless of what the scheduler last wrote there. The
+ * memory is per-cpu scratch reused once the callback returns: copy the bits
+ * out, don't keep the address. The set_cmask() argument follows the same rules.
+ *
+ * The address is a plain value rather than a typed pointer because BTF can't
+ * mark a struct member as an arena pointer yet and a pointer member would reach
+ * the program typed as a kernel pointer. Cast it to struct scx_cmask __arena *
+ * before use. Once arena members can be typed, a typed alias will join this
+ * field in an anonymous union at the same offset.
+ */
+struct scx_enable_args {
+	u64	cmask_arena_addr;
+};
+
 /* argument container for ops.cgroup_init() */
 struct scx_cgroup_init_args {
 	/* the weight of the cgroup [1..10000] */
@@ -452,6 +507,12 @@ struct sched_ext_ops {
 	 * Therefore, always use scx_bpf_task_cpu(@p) to determine the
 	 * target CPU the task is going to use.
 	 *
+	 * Under proxy execution, the BPF scheduler continues to observe the
+	 * donor as the current scheduling context. A blocked donor enters a
+	 * ->running()/->stopping() session while its scheduling context drives
+	 * the lock owner. The lock owner executing on its behalf is intentionally
+	 * not reported through these callbacks.
+	 *
 	 * See ->runnable() for explanation on the task state notifiers.
 	 */
 	void (*running)(struct task_struct *p);
@@ -1037,6 +1098,7 @@ struct sched_ext_ops {
  *   - dispatch         -> dispatch (cpu arg is now cid)
  *   - update_idle      -> update_idle (cpu arg is now cid)
  *   - set_cpumask      -> set_cmask (cmask instead of cpumask)
+ *   - enable           -> enable (takes struct scx_enable_args)
  *   - cpu_online       -> cid_online
  *   - cpu_offline      -> cid_offline
  *   - dump_cpu         -> dump_cid
@@ -1070,7 +1132,7 @@ struct sched_ext_ops_cid {
 			  struct scx_init_task_args *args);
 	void (*exit_task)(struct task_struct *p,
 			   struct scx_exit_task_args *args);
-	void (*enable)(struct task_struct *p);
+	void (*enable)(struct task_struct *p, struct scx_enable_args *args);
 	void (*disable)(struct task_struct *p);
 	void (*dump)(struct scx_dump_ctx *ctx);
 	void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle);
@@ -1333,6 +1395,7 @@ struct scx_sched_pcpu {
 	cpumask_var_t		cpus_to_kick;
 	cpumask_var_t		cpus_to_kick_if_idle;
 	cpumask_var_t		cpus_to_preempt;
+	cpumask_var_t		cpus_to_preempt_lazy;
 	cpumask_var_t		cpus_to_wait;
 	struct list_head	to_kick_node;
 
@@ -1413,15 +1476,16 @@ struct scx_sched_pnode {
  * the allocation pattern.
  *
  * ENQ_IMMED  insert an IMMED task onto the cid's local DSQ
- *            - kick the cid's cpu (except SCX_KICK_PREEMPT)
+ *            - kick the cid's cpu (except SCX_KICK_PREEMPT and
+ *              SCX_KICK_PREEMPT_LAZY)
  *
  * ENQ        insert any task onto the cid's local DSQ (implies ENQ_IMMED)
  *
  * PREEMPT    preempt any task running on the cid regardless of the owning
  *            sched (implies ENQ). Preempting a task in the sched's own subtree
  *            doesn't require any cap.
- *            - SCX_ENQ_PREEMPT inserts
- *            - SCX_KICK_PREEMPT kicks
+ *            - SCX_ENQ_PREEMPT and SCX_ENQ_PREEMPT_LAZY insert
+ *            - SCX_KICK_PREEMPT and SCX_KICK_PREEMPT_LAZY kick
  *
  * PERF       control the cid's cpu power/perf management state, currently the
  *            cpufreq target set through scx_bpf_cidperf_set(). Hardware
@@ -1533,7 +1597,8 @@ struct scx_sched {
 	 * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel
 	 * access either view of the same storage without function-pointer
 	 * casts: use .ops for cpu-form and shared fields, .ops_cid for the
-	 * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...).
+	 * callbacks whose cid-form signature differs (set_cmask, enable,
+	 * select_cid, cid_online, ...).
 	 */
 	union {
 		struct sched_ext_ops		ops;
@@ -1556,9 +1621,9 @@ struct scx_sched {
 	uintptr_t		arena_kern_base;
 
 	/*
-	 * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask
-	 * to ops_cid.set_cmask(). The kernel writes through the stored kern_va
-	 * and passes it to the callback's __arena argument.
+	 * Per-CPU arena cmask the kernel fills from a task's cpumask and hands
+	 * to ops_cid.enable() and ops_cid.set_cmask(). The stored pointers are
+	 * the kernel addresses.
 	 */
 	struct scx_cmask * __percpu *set_cmask_scratch;
 	struct scx_cmask *online_cmask;
@@ -1669,6 +1734,19 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt
 	return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr);
 }
 
+/**
+ * scx_kaddr_to_arena - Translate a kernel arena address to the BPF form
+ * @sch: scheduler whose arena hosts @kaddr
+ * @kaddr: kernel address inside @sch's arena
+ *
+ * __arena callback arguments need no translation. Addresses handed to BPF any
+ * other way, such as struct fields and kfunc return values, go through this.
+ */
+static inline uintptr_t scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr)
+{
+	return (uintptr_t)kaddr - sch->arena_kern_base;
+}
+
 enum scx_wake_flags {
 	/* expose select WF_* flags as enums */
 	SCX_WAKE_FORK		= WF_FORK,
@@ -1692,6 +1770,15 @@ enum scx_enq_flags {
 	 */
 	SCX_ENQ_PREEMPT		= 1LLU << 32,
 
+	/*
+	 * Like %SCX_ENQ_PREEMPT, but request lazy rescheduling. The current
+	 * task's slice is still cleared immediately so that the next scheduling
+	 * boundary observes the new ordering. %SCX_ENQ_PREEMPT takes precedence
+	 * if both are specified. Implies %SCX_ENQ_HEAD, which is all it means
+	 * on a non-local DSQ, as with %SCX_ENQ_PREEMPT.
+	 */
+	SCX_ENQ_PREEMPT_LAZY	= 1LLU << 35,
+
 	/*
 	 * Only allowed on local DSQs. Guarantees that the task either gets
 	 * on the CPU immediately and stays on it, or gets reenqueued back
@@ -1739,6 +1826,12 @@ enum scx_enq_flags {
 	 */
 	SCX_ENQ_LAST		= 1LLU << 41,
 
+	/*
+	 * The task is blocked on a mutex and is being kept runnable as a proxy
+	 * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set.
+	 */
+	SCX_ENQ_BLOCKED		= 1LLU << 42,
+
 	/* high 8 bits are internal */
 	__SCX_ENQ_INTERNAL_MASK	= 0xffLLU << 56,
 
@@ -1751,6 +1844,15 @@ enum scx_enq_flags {
 	SCX_ENQ_SLICE_DFL	= 1LLU << 62,	/* carried slice is a default refill */
 };
 
+/* Strip priority and carried slice state when diverting from a local DSQ. */
+static inline void scx_divert_strip_flags(struct task_struct *p, u64 *enq_flags)
+{
+	*enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT |
+			SCX_ENQ_PREEMPT_LAZY | SCX_ENQ_HEAD |
+			SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL);
+	p->scx.flags &= ~SCX_TASK_IMMED;
+}
+
 enum scx_deq_flags {
 	/* expose select DEQUEUE_* flags as enums */
 	SCX_DEQ_SLEEP		= DEQUEUE_SLEEP,
@@ -1818,6 +1920,15 @@ enum scx_kick_flags {
 	 * is not on SCX.
 	 */
 	SCX_KICK_WAIT		= 1LLU << 2,
+
+	/*
+	 * Like %SCX_KICK_PREEMPT, but request lazy rescheduling. If combined
+	 * with %SCX_KICK_PREEMPT or %SCX_KICK_WAIT, rescheduling is immediate.
+	 */
+	SCX_KICK_PREEMPT_LAZY	= 1LLU << 3,
+
+	SCX_KICK_ALL_FLAGS	= SCX_KICK_IDLE | SCX_KICK_PREEMPT |
+				  SCX_KICK_WAIT | SCX_KICK_PREEMPT_LAZY,
 };
 
 enum scx_tg_flags {
@@ -2061,12 +2172,12 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter);
 bool scx_set_task_slice(struct task_struct *p, u64 slice);
 void scx_task_slice_ended(struct rq *rq, struct task_struct *p);
 void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq);
+void scx_prepare_task_sched_change(struct task_struct *p);
 void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p);
 void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
 			 int sticky_cpu);
 void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,
-				      u64 enq_flags, struct scx_dispatch_q *src_dsq,
-				      struct rq *dst_rq);
+				      u64 enq_flags, struct rq *dst_rq);
 bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
 			    struct scx_dispatch_q *dsq, u64 enq_flags);
 bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq);
@@ -2078,6 +2189,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
 u64 __scx_bpf_now(struct rq *rq);
 void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
 			u64 reenq_flags, struct rq *locked_rq);
+void scx_reenq_wait_dispatching(struct task_struct *p);
 int __scx_init_task(struct scx_sched *sch, struct task_struct *p,
 		    struct cgroup *cgrp, bool fork);
 void scx_enable_task(struct scx_sched *sch, struct task_struct *p);
@@ -2122,6 +2234,22 @@ extern struct scx_sched *scx_enabling_sub_sched;
 #define scx_error(sch, fmt, args...)						\
 	scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args)
 
+/*
+ * Tracing progs can call kfuncs from NMI. Kfuncs that take scheduler locks or
+ * touch the kick lists, which are only protected by irq masking, can't run
+ * there, so abort the scheduler instead. scx_error() is NMI-safe.
+ */
+static __always_inline bool __scx_kf_allowed_ctx(struct scx_sched *sch, const char *who)
+{
+	if (unlikely(in_nmi())) {
+		scx_error(sch, "%s called from NMI", who);
+		return false;
+	}
+	return true;
+}
+
+#define scx_kf_allowed_ctx(sch)	__scx_kf_allowed_ctx((sch), __func__)
+
 /**
  * scx_root_protected_live - Root sched for paths that only run while live
  *
@@ -2212,6 +2340,15 @@ static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags)
  */
 static inline struct rq *scx_locked_rq(void)
 {
+	/*
+	 * Tracing progs can call kfuncs from NMI. scx_locked_rq_state tracks
+	 * the rq locked by the interrupted context, so a non-NULL read from
+	 * NMI would falsely claim its lock. Return NULL from NMI so that
+	 * callers take their unlocked paths.
+	 */
+	if (unlikely(in_nmi()))
+		return NULL;
+
 	return __this_cpu_read(scx_locked_rq_state);
 }
 
@@ -2302,9 +2439,9 @@ do {										\
 } while (0)
 
 /*
- * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs
- * this: it takes an arena cmask address instead of a cpumask, so it cannot be
- * invoked via its cpu-form set_cpumask() slot.
+ * Dispatch a task op through the cid-form ops_cid table, for the ops whose
+ * cid-form signature differs from the cpu-form slot: set_cmask() takes an arena
+ * cmask instead of a cpumask and enable() takes scx_enable_args.
  */
 #define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...)			\
 	__SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args)
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 34e642a1a40344..6b964743cab9b5 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -285,7 +285,8 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec)
 	rq->scx.rescue.budget -= delta_exec;
 
 	/* per-cpu usage average feeds the overload victim pick */
-	pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq));
+	pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu,
+			   cpu_of(rq));
 	pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec;
 
 	if (!scx_rescue_slice_remaining(rq))
@@ -555,9 +556,9 @@ static void scx_rescue_timerfn(struct timer_list *timer)
 				     scx.dsq_list.node);
 		scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
 		scx_rescue_admit(rq, p, slice);
-		scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS,
-						 &rq->scx.rescue.dsq, rq);
-		if (sched_class_above(&ext_sched_class, rq->curr->sched_class))
+		scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
+						 SCX_ENQ_IGNORE_CAPS, rq);
+		if (sched_class_above(&ext_sched_class, rq->donor->sched_class))
 			resched_curr(rq);
 	} else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) {
 		/*
@@ -572,7 +573,7 @@ static void scx_rescue_timerfn(struct timer_list *timer)
 		scx_task_unlink_from_dsq(p, &rq->scx.local_dsq);
 		scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
 					SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS,
-					&rq->scx.local_dsq, rq);
+					rq);
 	}
 out_arm:
 	scx_rescue_timer_arm(rq);
@@ -585,10 +586,6 @@ void scx_rescue_flush(struct rq *rq)
 
 	lockdep_assert_rq_held(rq);
 
-	/* sched domain rebuilds call rq_offline with the CPU staying alive */
-	if (cpu_active(cpu_of(rq)))
-		return;
-
 	/* end the current rescue */
 	if (rq->scx.rescue.curr)
 		scx_task_slice_ended(rq, rq->scx.rescue.curr);
@@ -596,8 +593,8 @@ void scx_rescue_flush(struct rq *rq)
 	/* and flush out all pending ones */
 	list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) {
 		scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
-		scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS,
-						 &rq->scx.rescue.dsq, rq);
+		scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
+						 SCX_ENQ_IGNORE_CAPS, rq);
 	}
 
 	timer_delete(&rq->scx.rescue.timer);
@@ -675,7 +672,7 @@ void scx_rescue_init(struct rq *rq)
 }
 
 /**
- * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
+ * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
  * @sch: enqueuing sub-sched
  * @rq: rq whose local DSQ @p targets
  * @p: task being inserted
@@ -686,20 +683,18 @@ void scx_rescue_init(struct rq *rq)
  * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on
  * @p.
  *
- * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting
- * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback
- * migration-disabled admission.
+ * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT, %SCX_ENQ_PREEMPT_LAZY and %SCX_ENQ_HEAD are
+ * cleared when diverting to rescue or reject. %SCX_ENQ_PREEMPT and
+ * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled
+ * admission.
  *
  * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued
  * to and run by its nearest non-bypassing ancestor. If root is bypassing, it
  * always holds all caps.
  */
-struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
-					     struct task_struct *p, u64 *enq_flags)
+struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+					       struct task_struct *p, u64 *enq_flags)
 {
-	if (!scx_has_subs())
-		return &rq->scx.local_dsq;
-
 	s32 cid = __scx_cpu_to_cid(cpu_of(rq));
 	struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
 	u64 needed = scx_caps_for_enq(*enq_flags);
@@ -709,7 +704,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
 	 * On a remote activation the scheduling sched (@asch) differs from
 	 * @p's owner (@sch). Check caps against the scheduling sched.
 	 */
-	if (*enq_flags & SCX_ENQ_PREEMPT)
+	if (*enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY))
 		needed |= scx_caps_for_preempt(asch, rq, *enq_flags);
 	missing = scx_missing_caps(asch, cpu_of(rq), needed);
 
@@ -726,7 +721,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
 	if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) ||
 		     p->migration_pending)) {
 		__scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1);
-		*enq_flags &= ~SCX_ENQ_PREEMPT;
+		*enq_flags &= ~(SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY);
 		return &rq->scx.local_dsq;
 	}
 
@@ -735,9 +730,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
 	 * or HEAD - a diversion has no priority and IMMED is not allowed on
 	 * non-local DSQs. Strip the enq and task flags along with the slice.
 	 */
-	*enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD |
-			SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL);
-	p->scx.flags &= ~SCX_TASK_IMMED;
+	scx_divert_strip_flags(p, enq_flags);
 
 	/* the enqueuer opted for rescue instead of rejection and reenqueue */
 	if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) {
@@ -752,6 +745,8 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
 
 	p->scx.reenq_reason_caps = missing;
 	p->scx.reenq_reason_cid = cid;
+	WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK);
+	p->scx.flags |= SCX_TASK_REENQ_CAP;
 
 	return &rq->scx.reject_dsq;
 }
@@ -774,51 +769,6 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
 	return true;
 }
 
-/*
- * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides
- * from p->scx.reenq_reason_*.
- *
- * A task can be re-rejected repeatedly. The reenqueue is bounded per task in
- * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT.
- * Rejection can't happen for root.
- */
-void scx_reenq_reject(struct rq *rq)
-{
-	LIST_HEAD(tasks);
-	struct task_struct *p, *n;
-
-	lockdep_assert_rq_held(rq);
-
-	if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
-		return;
-
-	/*
-	 * Move to a private list so a task re-rejected by the
-	 * scx_do_enqueue_task() below isn't revisited this round.
-	 */
-	list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) {
-		/* migration_pending tasks should have bypassed to local DSQ */
-		if (WARN_ON_ONCE(p->migration_pending))
-			continue;
-
-		scx_dispatch_dequeue(rq, p);
-
-		if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK))
-			p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
-		p->scx.flags |= SCX_TASK_REENQ_CAP;
-
-		list_add_tail(&p->scx.dsq_list.node, &tasks);
-	}
-
-	list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
-		list_del_init(&p->scx.dsq_list.node);
-
-		scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
-
-		p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
-	}
-}
-
 /* record a caps change, see struct scx_caps_updated */
 static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps,
 				struct list_head *to_deliver)
@@ -924,7 +874,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid)
 	struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
 
 	/*
-	 * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check
+	 * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check
 	 * below sees the node off the list and queues it, or the in-flight sync
 	 * sees the caps[] update made before this call.
 	 */
@@ -949,7 +899,7 @@ static void discard_queued_syncs(struct rq *rq)
 }
 
 /**
- * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
+ * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[]
  * @rq: the cid's cpu rq
  * @prev: @rq's previous task from the in-progress dispatch
  *
@@ -961,16 +911,14 @@ static void discard_queued_syncs(struct rq *rq)
  * learns the cid's idle state. Such a gain arms the per-rq
  * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it.
  */
-void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
+void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
 {
 	s32 cpu = cpu_of(rq);
 	s32 cid, shard;
 	struct llist_node *batch, *pos, *tmp;
 	u64 lost_all = 0;
 
-	lockdep_assert_rq_held(rq);
-
-	if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
+	if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
 		return;
 
 	/*
@@ -1255,7 +1203,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p)
 	lockdep_assert_held(&p->pi_lock);
 	lockdep_assert_rq_held(task_rq(p));
 
-	scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+	scx_prepare_task_sched_change(p);
+	scoped_guard (sched_change, p,
+		      DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
 		scx_disable_and_exit_task(scx_task_sched(p), p);
 		scx_set_task_state(p, SCX_TASK_INIT_BEGIN);
 		scx_set_task_state(p, SCX_TASK_INIT);
@@ -1285,7 +1235,9 @@ static void scx_punt_task(struct scx_sched *to, struct task_struct *p)
 	lockdep_assert_rq_held(task_rq(p));
 	WARN_ON_ONCE(!READ_ONCE(to->bypass_depth));
 
-	scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+	scx_prepare_task_sched_change(p);
+	scoped_guard (sched_change, p,
+		      DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
 		scx_disable_and_exit_task(scx_task_sched(p), p);
 		scx_set_task_sched(p, to);
 	}
@@ -1361,7 +1313,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch)
 			.bw_period_us = tg->scx.bw_period_us,
 			.bw_quota_us = tg->scx.bw_quota_us,
 			.bw_burst_us = tg->scx.bw_burst_us,
-			.sched_idle = tg->scx.idle,
+			.sched_idle = tg->scx.sched_idle,
 		};
 
 		if (tg->scx.sched != parent ||
@@ -1465,7 +1417,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch)
 			.bw_period_us = tg->scx.bw_period_us,
 			.bw_quota_us = tg->scx.bw_quota_us,
 			.bw_burst_us = tg->scx.bw_burst_us,
-			.sched_idle = tg->scx.idle,
+			.sched_idle = tg->scx.sched_idle,
 		};
 
 		/* the first pass must have transferred everything */
@@ -1943,7 +1895,9 @@ void scx_sub_enable_workfn(struct kthread_work *work)
 		if (!(p->scx.flags & SCX_TASK_SUB_INIT))
 			continue;
 
-		scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
+		scx_prepare_task_sched_change(p);
+		scoped_guard (sched_change, p,
+			      DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) {
 			/*
 			 * $p must be either READY or ENABLED. If ENABLED,
 			 * __scx_disabled_and_exit_task() first disables and
@@ -2269,6 +2223,9 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux *
 	if (unlikely(!parent))
 		return -ENODEV;
 
+	if (!scx_kf_allowed_ctx(parent))
+		return -EDEADLK;
+
 	if (!scx_is_cid_type()) {
 		scx_error(parent, "sub-cap kfuncs require a cid-form scheduler");
 		return -EOPNOTSUPP;
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index f7bcdfda8dd85e..c0174e523f2ada 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -28,16 +28,15 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
 void scx_free_pshards(struct scx_sched *sch);
 s32 scx_alloc_pshards(struct scx_sched *sch);
 void scx_init_root_caps(struct scx_sched *sch);
-void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
+void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
 void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
 void scx_online_ecaps(struct rq *rq);
 void scx_offline_ecaps(struct rq *rq);
 void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
 void scx_discard_stale_ecaps_syncs(void);
-struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
-					     struct task_struct *p, u64 *enq_flags);
+struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+					       struct task_struct *p, u64 *enq_flags);
 bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
-void scx_reenq_reject(struct rq *rq);
 void scx_rescue_charge(struct rq *rq, s64 delta_exec);
 void scx_rescue_end(struct rq *rq);
 bool scx_rescue_keep(struct rq *rq, struct task_struct *p);
@@ -72,6 +71,25 @@ static inline void scx_dec_has_subs(struct scx_sched *sch)
 		static_branch_dec(&__scx_has_subs);
 }
 
+/* hot-path hooks, gated inline so that a root-only system skips the calls */
+static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
+{
+	lockdep_assert_rq_held(rq);
+
+	if (scx_has_subs())
+		__scx_process_sync_ecaps(rq, prev);
+}
+
+static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch,
+							   struct rq *rq,
+							   struct task_struct *p,
+							   u64 *enq_flags)
+{
+	if (!scx_has_subs())
+		return &rq->scx.local_dsq;
+	return __scx_resolve_local_dsq(sch, rq, p, enq_flags);
+}
+
 #else	/* CONFIG_EXT_SUB_SCHED */
 
 static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
@@ -93,7 +111,6 @@ static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcp
 static inline void scx_discard_stale_ecaps_syncs(void) {}
 static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
 static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
-static inline void scx_reenq_reject(struct rq *rq) {}
 static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {}
 static inline void scx_rescue_end(struct rq *rq) {}
 static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; }
@@ -169,17 +186,18 @@ static inline u64 scx_caps_for_task(struct task_struct *p)
 	return SCX_CAP_ENQ;
 }
 
-/* the cap @sch needs to preempt @rq's current task, 0 if none */
-static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags)
+/* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */
+static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq,
+				       u64 enq_flags)
 {
-	struct task_struct *curr = rq->curr;
+	struct task_struct *donor = rq->donor;
 
 	/* a kernel-forced placement preempts regardless of caps */
 	if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
 		return 0;
 	/* a non-ext task can't be preempted by ext, own-subtree needs no cap */
-	if (curr->sched_class != &ext_sched_class ||
-	    scx_is_descendant(scx_task_sched(curr), sch))
+	if (donor->sched_class != &ext_sched_class ||
+	    scx_is_descendant(scx_task_sched(donor), sch))
 		return 0;
 	return SCX_CAP_PREEMPT;
 }
diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h
index 943d8d429a2c9a..38514d6bf14957 100644
--- a/kernel/sched/ext/types.h
+++ b/kernel/sched/ext/types.h
@@ -59,17 +59,35 @@ enum scx_consts {
 };
 
 /*
- * Per-cid topology info. For each topology level (core, LLC, node) and shard,
- * records the first cid in the unit and its global index. Global indices are
- * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over
- * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields
- * set to -1 but always have valid shard assignments.
+ * Per-cid topology info. For each topology level (core, cluster, LLC, node) and
+ * shard, records the first cid in the unit and its global index. Global indices
+ * are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges
+ * over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/
+ * node fields set to -1 but always have valid shard assignments.
+ *
+ * A cluster is the cache-sharing level between the core and the LLC. Where
+ * this level is absent, each core forms a cluster of its own. Where the cache
+ * level spans the LLC, the whole LLC forms one cluster. Each cluster has a
+ * unique cluster_cid and a dense cluster_idx, and its cids form a contiguous
+ * range.
+ *
+ * Every cid therefore has a cluster, so cluster_cid alone does not say whether
+ * the machine has the level: the first cluster of an LLC starts at the LLC's
+ * own base cid and a core-wide one at the core's.
  *
  * Shards are contiguous CID ranges used as scalable locking/work domains for
  * sub-scheduler operations. By default each LLC becomes one shard, split into
  * smaller shards if the LLC exceeds the target size. No-topo cids are packed
  * into their own max-sized shards.
  *
+ * Shards are cut on core boundaries only, so a shard doesn't need to contain a
+ * whole cluster: a cluster may straddle two shards. Only core, cluster, LLC
+ * and node are nested, in that order.
+ *
+ * New fields are appended, never inserted: scx_bpf_cid_topo() copies this
+ * struct out sized by the program's own layout, and an older program's copy
+ * must stay a prefix of the kernel's.
+ *
  * @core_cid: first cid of this cid's core (smt-sibling group)
  * @core_idx: global index of that core, in [0, nr_cores_at_init)
  * @llc_cid: first cid of this cid's LLC
@@ -78,6 +96,8 @@ enum scx_consts {
  * @node_idx: global index of that node, in [0, nr_nodes_at_init)
  * @shard_cid: first cid of this cid's shard
  * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards)
+ * @cluster_cid: first cid of this cid's cluster
+ * @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init)
  */
 struct scx_cid_topo {
 	s32 core_cid;
@@ -88,6 +108,8 @@ struct scx_cid_topo {
 	s32 node_idx;
 	s32 shard_cid;
 	s32 shard_idx;
+	s32 cluster_cid;
+	s32 cluster_idx;
 };
 
 enum scx_cid_consts {
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7455a83a6a9907..56f4ab6d9ada3a 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -22,8 +22,8 @@
  */
 #include <linux/energy_model.h>
 #include <linux/mmap_lock.h>
-#include <linux/hugetlb_inline.h>
 #include <linux/jiffies.h>
+#include <linux/math.h>
 #include <linux/mm_api.h>
 #include <linux/highmem.h>
 #include <linux/hrtimer.h>
@@ -820,12 +820,6 @@ static u64 ineligible_vruntime(struct cfs_rq *cfs_rq)
 	if (curr && !curr->on_rq)
 		curr = NULL;
 
-	/*
-	 * This is called from set_next_task_fair(.first=true) /
-	 * set_protect_slice() so curr had better be set and on_rq.
-	 */
-	WARN_ON_ONCE(!curr);
-
 	if (weight) {
 		s64 runtime = cfs_rq->sum_w_vruntime;
 
@@ -1137,10 +1131,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity
 
 	/* If there are shorter slices than se's one */
 	if (slice != se->slice) {
+		vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se));
 		if (sched_feat(PREEMPT_SHORT))
 			vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq));
-		else
-			vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se));
 	}
 
 	se->vprot = vprot;
@@ -1148,10 +1141,19 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity
 
 static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-	u64 slice = cfs_rq_min_slice(cfs_rq);
 	u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq));
+	u64 slice = normalized_sysctl_sched_base_slice;
+	u64 vprot;
+
+	if (sched_feat(RUN_TO_PARITY))
+		slice = cfs_rq_min_slice(cfs_rq);
+
+	vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se));
 
-	se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se));
+	if (sched_feat(PREEMPT_SHORT) && slice != se->slice)
+		vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq));
+
+	se->vprot = vprot;
 }
 
 static inline bool protect_slice(struct sched_entity *se)
@@ -1478,7 +1480,7 @@ static inline int get_sched_cache_scale(int mul)
 	return (1 + (tol - 1) * mul);
 }
 
-static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
+static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu)
 {
 #ifdef CONFIG_NUMA_BALANCING
 	unsigned long llc, footprint;
@@ -1497,7 +1499,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
 		 * excluded.
 		 */
 		llc = sd->llc_bytes;
-		footprint = READ_ONCE(mm->sc_stat.footprint);
+		footprint = READ_ONCE(grp->footprint);
 
 		/*
 		 * Scale the LLC size by 256*llc_aggr_tolerance
@@ -1526,7 +1528,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
 	return false;
 }
 
-static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
+static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p,
 			   int cpu)
 {
 	int scale;
@@ -1542,10 +1544,32 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
 	if (scale == INT_MAX)
 		return false;
 
-	return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads),
+	return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads),
 			(scale * per_cpu(sd_llc_size, cpu)));
 }
 
+/*
+ * A task counts in nr_pref_llc_running while it is queued on its preferred
+ * LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in
+ * the runnable domain so alb_break_llc() can compare it with h_nr_runnable.
+ */
+static bool task_pref_llc_runnable(struct task_struct *p)
+{
+	return p->pref_llc_queued && !p->se.sched_delayed;
+}
+
+static void pref_llc_running_inc(struct rq *rq, struct task_struct *p)
+{
+	if (task_pref_llc_runnable(p))
+		rq->nr_pref_llc_running++;
+}
+
+static void pref_llc_running_dec(struct rq *rq, struct task_struct *p)
+{
+	if (task_pref_llc_runnable(p))
+		rq->nr_pref_llc_running--;
+}
+
 static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
 {
 	int pref_llc, pref_llc_queued;
@@ -1557,7 +1581,6 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
 
 	pref_llc_queued = (pref_llc == task_llc(p));
 	rq->nr_llc_running++;
-	rq->nr_pref_llc_running += pref_llc_queued;
 
 	/*
 	 * Record whether p is enqueued on its preferred
@@ -1575,6 +1598,9 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
 	 */
 	p->pref_llc_queued = pref_llc_queued;
 
+	/* Skipped while delayed; clear_delayed() adds it back on wake. */
+	pref_llc_running_inc(rq, p);
+
 	sd = rcu_dereference_all(rq->sd);
 	if (sd && (unsigned int)pref_llc < sd->llc_max)
 		sd->llc_counts[pref_llc]++;
@@ -1591,7 +1617,12 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
 
 	rq->nr_llc_running--;
 	if (p->pref_llc_queued) {
-		rq->nr_pref_llc_running--;
+		/*
+		 * Skipped if still delayed (set_delayed() already removed it);
+		 * clearing pref_llc_queued below also stops clear_delayed()
+		 * from re-adding it.
+		 */
+		pref_llc_running_dec(rq, p);
 		/*
 		 * Update the status in case
 		 * other logic might query
@@ -1619,12 +1650,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
 	}
 }
 
-void mm_init_sched(struct mm_struct *mm,
-		   struct sched_cache_time __percpu *_pcpu_sched)
+int mm_init_sched(struct mm_struct *mm,
+		  struct sched_cache_time __percpu *_pcpu_sched)
 {
+	struct sched_cache_group *grp;
 	unsigned long epoch = 0;
 	int i;
 
+	grp = kzalloc_obj(*grp);
+	if (!grp) {
+		free_percpu(_pcpu_sched);
+		mm->sched_cache_grp = NULL;
+		return -ENOMEM;
+	}
+
 	for_each_possible_cpu(i) {
 		struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i);
 		struct rq *rq = cpu_rq(i);
@@ -1635,18 +1674,141 @@ void mm_init_sched(struct mm_struct *mm,
 		epoch = rq->cpu_epoch;
 	}
 
-	raw_spin_lock_init(&mm->sc_stat.lock);
-	mm->sc_stat.epoch = epoch;
-	mm->sc_stat.cpu = -1;
-	mm->sc_stat.next_scan = jiffies;
-	mm->sc_stat.nr_running_avg = 0;
-	mm->sc_stat.footprint = 0;
+	raw_spin_lock_init(&grp->lock);
+	grp->epoch = epoch;
+	grp->cpu = -1;
+	grp->next_scan = jiffies;
+	grp->nr_running_avg = 0;
+	grp->footprint = 0;
+	refcount_set(&grp->refcnt, 1);
 	/*
-	 * The update to mm->sc_stat should not be reordered
-	 * before initialization to mm's other fields, in case
+	 * The update to grp->pcpu_sched should not be reordered
+	 * before initialization to grp's other fields, in case
 	 * the readers may get invalid mm_sched_epoch, etc.
 	 */
-	smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched);
+	smp_store_release(&grp->pcpu_sched, _pcpu_sched);
+	/*
+	 * Publish the group last.  Not every reader qualifies it by
+	 * grp->pcpu_sched - can_migrate_llc_task() only checks that the
+	 * pointer is non-NULL before reading grp->footprint and
+	 * grp->nr_running_avg - so a reachable group must already be
+	 * fully initialized.
+	 */
+	smp_store_release(&mm->sched_cache_grp, grp);
+	return 0;
+}
+
+static void sched_cache_group_free_rcu(struct rcu_head *rcu)
+{
+	struct sched_cache_group *grp =
+		container_of(rcu, struct sched_cache_group, rcu);
+
+	free_percpu(grp->pcpu_sched);
+	kfree(grp);
+}
+
+static void sched_cache_group_put(struct sched_cache_group *grp)
+{
+	if (!grp || !refcount_dec_and_test(&grp->refcnt))
+		return;
+
+	call_rcu(&grp->rcu, sched_cache_group_free_rcu);
+}
+
+DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *,
+	    sched_cache_group_put(_T));
+
+#define rcu_deref_sched_cache_grp(tsk) \
+	rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current)
+
+static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p,
+							 struct sched_cache_group *new)
+{
+	struct sched_cache_group *old;
+
+	old = rcu_deref_sched_cache_grp(p);
+	rcu_assign_pointer(p->sched_cache_grp, new);
+
+	return old;
+}
+
+struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp)
+{
+	/*
+	 * refcount_inc_not_zero() is the acquire primitive for lockless
+	 * (RCU) lookups; plain refcount_inc() would scribble the count if
+	 * it already reached zero. Return NULL in that case.
+	 */
+	if (grp && !refcount_inc_not_zero(&grp->refcnt))
+		grp = NULL;
+
+	return grp;
+}
+
+struct sched_cache_group *task_cache_group_get(struct task_struct *p)
+{
+	guard(rcu)();
+	return sched_cache_group_get(rcu_dereference(p->sched_cache_grp));
+}
+
+void sched_cache_fork(struct task_struct *p)
+{
+	/*
+	 * The child takes its own reference on the mm's cache group, separate
+	 * from the reference held by the mm. @p is not yet visible to readers,
+	 * so a plain initializing store is enough.
+	 */
+	RCU_INIT_POINTER(p->sched_cache_grp,
+			 sched_cache_group_get(p->mm->sched_cache_grp));
+}
+
+void sched_cache_fork_cleanup(struct task_struct *p)
+{
+	/*
+	 * A fork that fails after sched_cache_fork() never reaches exit_mm(),
+	 * so drop the reference here. @p never became visible, so there are no
+	 * concurrent readers and the reference we hold keeps the group alive.
+	 */
+	sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp));
+	RCU_INIT_POINTER(p->sched_cache_grp, NULL);
+}
+
+void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm)
+{
+	struct sched_cache_group *old;
+
+	/*
+	 * Acquire the new reference before publishing the pointer, then drop
+	 * the old one. @p is current and the only writer of its own pointer.
+	 */
+	old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp));
+	sched_cache_group_put(old);
+}
+
+void sched_cache_exit_mm(struct task_struct *p)
+{
+	struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL);
+
+#ifdef CONFIG_NUMA_BALANCING
+	/*
+	 * Subtract this task's footprint from the group before dropping the
+	 * reference, so the group footprint converges as its threads exit.
+	 * Unlocked for performance; clamp to avoid underflow.
+	 */
+	if (grp && p->total_numa_faults) {
+		unsigned long fp = READ_ONCE(grp->footprint);
+		unsigned long sub = min(fp, p->total_numa_faults);
+
+		WRITE_ONCE(grp->footprint, fp - sub);
+	}
+#endif
+	sched_cache_group_put(grp);
+}
+
+void mm_destroy_sched(struct mm_struct *mm)
+{
+	sched_cache_group_put(mm->sched_cache_grp);
+	mm->sched_cache_grp = NULL;
 }
 
 /* because why would C be fully specified */
@@ -1697,14 +1859,14 @@ static unsigned long fraction_mm_sched(struct rq *rq,
 	return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
 }
 
-static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
+static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp)
 {
 	int mm_sched_llc = -1, mm_sched_cpu;
 
-	if (!mm)
+	if (!grp)
 		return -1;
 
-	mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu);
+	mm_sched_cpu = READ_ONCE(grp->cpu);
 	if (mm_sched_cpu != -1) {
 		mm_sched_llc = llc_id(mm_sched_cpu);
 
@@ -1734,8 +1896,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
 static inline
 void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 {
+	struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
 	struct sched_cache_time *pcpu_sched;
-	struct mm_struct *mm = p->mm;
 	int mm_sched_llc = -1;
 	unsigned long epoch;
 
@@ -1746,12 +1908,18 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 		return;
 	/*
 	 * init_task, kthreads and user thread created
-	 * by user_mode_thread() don't have mm.
+	 * by user_mode_thread() don't have a cache group.
+	 * In theory a kernel thread does not have any valid
+	 * cache group, because sched_cache_fork() is not
+	 * invoked for a kernel thread - !grp should gate the
+	 * kernel thread. Use the PF_KTHREAD check explicitly
+	 * here for safety reasons, to guard against future
+	 * modifications and to pair with task_tick_cache().
 	 */
-	if (!mm || !mm->sc_stat.pcpu_sched)
+	if (p->flags & PF_KTHREAD || !grp || !grp->pcpu_sched)
 		return;
 
-	pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq));
+	pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq));
 
 	scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) {
 		__update_mm_sched(rq, pcpu_sched);
@@ -1764,14 +1932,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 	 * If this process hasn't hit task_cache_work() for a while invalidate
 	 * its preferred state.
 	 */
-	if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout ||
-	    invalid_llc_nr(mm, p, cpu_of(rq)) ||
-	    exceed_llc_capacity(mm, cpu_of(rq))) {
-		if (READ_ONCE(mm->sc_stat.cpu) != -1)
-			WRITE_ONCE(mm->sc_stat.cpu, -1);
+	if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout ||
+	    invalid_llc_nr(grp, p, cpu_of(rq)) ||
+	    exceed_llc_capacity(grp, cpu_of(rq))) {
+		if (READ_ONCE(grp->cpu) != -1)
+			WRITE_ONCE(grp->cpu, -1);
 	}
 
-	mm_sched_llc = get_pref_llc(p, mm);
+	mm_sched_llc = get_pref_llc(p, grp);
 
 	/* task not on rq accounted later in account_entity_enqueue() */
 	if (task_running_on_cpu(rq->cpu, p) &&
@@ -1784,31 +1952,32 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
 
 static void task_tick_cache(struct rq *rq, struct task_struct *p)
 {
+	struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp);
 	struct callback_head *work = &p->cache_work;
-	struct mm_struct *mm = p->mm;
 	unsigned long epoch;
 
 	if (!sched_cache_enabled())
 		return;
 
-	if (!mm || p->flags & PF_KTHREAD ||
-	    !mm->sc_stat.pcpu_sched)
+	if (!grp || p->flags & PF_KTHREAD ||
+	    !grp->pcpu_sched)
 		return;
 
 	epoch = rq->cpu_epoch;
 	/* avoid moving backwards */
-	if (time_after_eq(mm->sc_stat.epoch, epoch))
+	if (time_after_eq(grp->epoch, epoch))
 		return;
 
-	guard(raw_spinlock)(&mm->sc_stat.lock);
+	guard(raw_spinlock)(&grp->lock);
 
 	if (work->next == work) {
 		task_work_add(p, work, TWA_RESUME);
-		WRITE_ONCE(mm->sc_stat.epoch, epoch);
+		WRITE_ONCE(grp->epoch, epoch);
 	}
 }
 
-static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
+static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p,
+			      struct sched_cache_group *grp)
 {
 #ifdef CONFIG_NUMA_BALANCING
 	int cpu, curr_cpu, nid, pref_nid;
@@ -1816,7 +1985,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
 	if (!static_branch_likely(&sched_numa_balancing))
 		goto out;
 
-	cpu = READ_ONCE(p->mm->sc_stat.cpu);
+	cpu = READ_ONCE(grp->cpu);
 	if (cpu != -1)
 		nid = cpu_to_node(cpu);
 	curr_cpu = task_cpu(p);
@@ -1873,13 +2042,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample)
 
 static void task_cache_work(struct callback_head *work)
 {
+	struct sched_cache_group *grp __free(sched_cache_group_put) = NULL;
+	cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL;
 	int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
 	unsigned long next_scan, now = jiffies;
 	struct task_struct *p = current, *cur;
 	unsigned long curr_m_a_occ = 0;
-	struct mm_struct *mm = p->mm;
 	unsigned long m_a_occ = 0;
-	cpumask_var_t cpus;
 
 	WARN_ON_ONCE(work != &p->cache_work);
 
@@ -1888,21 +2057,30 @@ static void task_cache_work(struct callback_head *work)
 	if (p->flags & PF_EXITING)
 		return;
 
-	next_scan = READ_ONCE(mm->sc_stat.next_scan);
+	/*
+	 * A reference makes sure grp is not released by others. The rcu
+	 * lock can not be held till after zalloc_cpumask_var() below,
+	 * because the latter might sleep.
+	 */
+	grp = task_cache_group_get(p);
+	if (!grp)
+		return;
+
+	next_scan = READ_ONCE(grp->next_scan);
 	if (time_before(now, next_scan))
 		return;
 
 	/* only 1 thread is allowed to scan */
-	if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan,
+	if (!try_cmpxchg(&grp->next_scan, &next_scan,
 			 now + max_t(unsigned long,
 				     READ_ONCE(llc_epoch_period), 1)))
 		return;
 
 	curr_cpu = task_cpu(p);
-	if (invalid_llc_nr(mm, p, curr_cpu) ||
-	    exceed_llc_capacity(mm, curr_cpu)) {
-		if (READ_ONCE(mm->sc_stat.cpu) != -1)
-			WRITE_ONCE(mm->sc_stat.cpu, -1);
+	if (invalid_llc_nr(grp, p, curr_cpu) ||
+	    exceed_llc_capacity(grp, curr_cpu)) {
+		if (READ_ONCE(grp->cpu) != -1)
+			WRITE_ONCE(grp->cpu, -1);
 
 		return;
 	}
@@ -1913,7 +2091,7 @@ static void task_cache_work(struct callback_head *work)
 	scoped_guard (cpus_read_lock) {
 		guard(rcu)();
 
-		get_scan_cpumasks(cpus, p);
+		get_scan_cpumasks(cpus, p, grp);
 
 		for_each_cpu(cpu, cpus) {
 			/* XXX sched_cluster_active */
@@ -1926,16 +2104,20 @@ static void task_cache_work(struct callback_head *work)
 
 			for_each_cpu(i, sched_domain_span(sd)) {
 				occ = fraction_mm_sched(cpu_rq(i),
-							per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
+							per_cpu_ptr(grp->pcpu_sched, i));
 				a_occ += occ;
 				if (occ > m_occ) {
 					m_occ = occ;
 					m_cpu = i;
 				}
 
+				/*
+				 * rcu_access_pointer() is used because the
+				 * pointer is only compared, never dereferenced.
+				 */
 				cur = rcu_dereference_all(cpu_rq(i)->curr);
 				if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) &&
-				    cur->mm == mm)
+				    rcu_access_pointer(cur->sched_cache_grp) == grp)
 					nr_running++;
 			}
 
@@ -1959,7 +2141,7 @@ static void task_cache_work(struct callback_head *work)
 				m_a_cpu = m_cpu;
 			}
 
-			if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu)))
+			if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu)))
 				curr_m_a_occ = a_occ;
 
 			cpumask_andnot(cpus, cpus, sched_domain_span(sd));
@@ -1968,7 +2150,7 @@ static void task_cache_work(struct callback_head *work)
 
 	if (m_a_occ > (2 * curr_m_a_occ)) {
 		/*
-		 * Avoid switching sc_stat.cpu too fast.
+		 * Avoid switching sched_cache_grp->cpu too fast.
 		 * The reason to choose 2X is because:
 		 * 1. It is better to keep the preferred LLC stable,
 		 *    rather than changing it frequently and cause migrations
@@ -1977,11 +2159,10 @@ static void task_cache_work(struct callback_head *work)
 		 * 3. 2X is chosen based on test results, as it delivers
 		 *    the optimal performance gain so far.
 		 */
-		WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu);
+		WRITE_ONCE(grp->cpu, m_a_cpu);
 	}
 
-	update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running);
-	free_cpumask_var(cpus);
+	update_avg_scale(&grp->nr_running_avg, nr_running);
 }
 
 void init_sched_mm(struct task_struct *p)
@@ -1990,11 +2171,19 @@ void init_sched_mm(struct task_struct *p)
 
 	init_task_work(work, task_cache_work);
 	work->next = work;
+	/*
+	 * dup_task_struct() copies the parent's task_struct, including its
+	 * sched_cache_grp, for which the child holds no reference.  Clear it
+	 * here - before copy_mm() runs - so the child never carries a
+	 * borrowed pointer that the fork error path would put.
+	 */
+	RCU_INIT_POINTER(p->sched_cache_grp, NULL);
 	/*
 	 * Reset new task's preference to avoid
 	 * polluting account_llc_enqueue().
 	 */
 	p->preferred_llc = -1;
+	p->pref_llc_queued = 0;
 }
 
 #else /* CONFIG_SCHED_CACHE */
@@ -2016,6 +2205,10 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {}
 
 static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {}
 
+static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {}
+
+static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {}
+
 #endif /* CONFIG_SCHED_CACHE */
 
 /*
@@ -3522,7 +3715,7 @@ static void update_task_scan_period(struct task_struct *p,
 		p->mm->numa_next_scan = jiffies +
 			msecs_to_jiffies(p->numa_scan_period);
 
-		return;
+		goto out;
 	}
 
 	/*
@@ -3566,7 +3759,10 @@ static void update_task_scan_period(struct task_struct *p,
 
 	p->numa_scan_period = clamp(p->numa_scan_period + diff,
 			task_scan_min(p), task_scan_max(p));
-	memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
+
+out:
+	memset(p->numa_faults_locality, 0,
+	       sizeof(p->numa_faults_locality));
 }
 
 /*
@@ -3692,6 +3888,7 @@ static int preferred_group_nid(struct task_struct *p, int nid)
 static void task_numa_placement(struct task_struct *p)
 	__context_unsafe(/* conditional locking */)
 {
+	struct sched_cache_group __maybe_unused *grp;
 	int seq, nid, max_nid = NUMA_NO_NODE;
 	unsigned long max_faults = 0;
 	unsigned long fault_types[2] = { 0, 0 };
@@ -3784,19 +3981,24 @@ static void task_numa_placement(struct task_struct *p)
 			 * heuristic and occasional lost updates are tolerable.
 			 *
 			 * If a task exits, its corresponding footprint must
-			 * be subtracted from the mm->sc_stat.footprint, otherwise
-			 * the mm->sc_stat.footprint will not converge:
-			 * the exiting thread's footprint remains unchanged/undecayed
-			 * in mm->sc_stat.footprint. See exit_mm().
+			 * be subtracted from p->sched_cache_grp->footprint,
+			 * otherwise the footprint will not converge: the
+			 * exiting thread's footprint remains unchanged/undecayed.
+			 * See exit_mm().
 			 *
 			 * Lost updates and unsynchronized subtraction
 			 * in exit_mm() can cause footprint + diff to
 			 * go negative. Clamp to zero to prevent the
 			 * unsigned footprint from wrapping.
 			 */
-			new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff;
-			WRITE_ONCE(p->mm->sc_stat.footprint,
-				   max(new_fp, 0L));
+			scoped_guard(rcu) {
+				grp = rcu_dereference(p->sched_cache_grp);
+
+				if (grp) {
+					new_fp = (long)READ_ONCE(grp->footprint) + diff;
+					WRITE_ONCE(grp->footprint, max(new_fp, 0L));
+				}
+			}
 #endif
 		}
 
@@ -4220,7 +4422,7 @@ static void task_numa_work(struct callback_head *work)
 
 	for (; vma; vma = vma_next(&vmi)) {
 		if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
-			is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
+			vma_is_hugetlb(vma) || vma_is_kernel_owned(vma)) {
 			trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE);
 			continue;
 		}
@@ -6390,15 +6592,27 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
 
 static void set_delayed(struct sched_entity *se)
 {
-	se->sched_delayed = 1;
-
 	/*
 	 * Delayed se of cfs_rq have no tasks queued on them.
 	 * Do not adjust h_nr_runnable since __dequeue_task()
 	 * will account it for blocked tasks.
+	 *
+	 * This check can be removed because when flat pick
+	 * patches get merged as only task can get delayed,
+	 * same for clear_delayed().
 	 */
-	if (!entity_is_task(se))
+	if (!entity_is_task(se)) {
+		se->sched_delayed = 1;
 		return;
+	}
+
+	/*
+	 * Drop a task leaving the runnable set.
+	 * Needs to be called before sched_delayed is set.
+	 * clear_delayed() mirrors this after clearing the flag.
+	 */
+	pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se));
+	se->sched_delayed = 1;
 
 	for_each_sched_entity(se) {
 		struct cfs_rq *cfs_rq = cfs_rq_of(se);
@@ -6420,6 +6634,13 @@ static void clear_delayed(struct sched_entity *se)
 	if (!entity_is_task(se))
 		return;
 
+	/*
+	 * Re-add on wake, after sched_delayed is cleared. On a final delayed
+	 * dequeue account_llc_dequeue() already cleared pref_llc_queued, so
+	 * this does nothing.
+	 */
+	pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se));
+
 	for_each_sched_entity(se) {
 		struct cfs_rq *cfs_rq = cfs_rq_of(se);
 
@@ -7993,7 +8214,6 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	struct sched_entity *se = &p->se;
 	struct cfs_rq *cfs_rq = &rq->cfs;
 	unsigned long weight;
-	bool curr;
 
 	if (task_is_throttled(p) && enqueue_throttled_task(p))
 		return;
@@ -8022,23 +8242,14 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	if (p->in_iowait)
 		cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
 
-	/*
-	 * XXX comment on the curr thing
-	 */
-	curr = (cfs_rq->curr == se);
-	if (curr)
-		place_entity(cfs_rq, se, flags);
 
 	if (se->on_rq && se->sched_delayed)
 		requeue_delayed_entity(cfs_rq, se);
 
 	weight = enqueue_hierarchy(p, flags);
-
-	if (!curr) {
-		reweight_eevdf(cfs_rq, se, weight, false);
-		place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED);
-		__enqueue_entity(cfs_rq, se);
-	}
+	reweight_eevdf(cfs_rq, se, weight, false);
+	place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED);
+	__enqueue_entity(cfs_rq, se);
 
 	if (!rq_h_nr_queued && rq->cfs.h_nr_queued)
 		dl_server_start(&rq->fair_server);
@@ -8458,8 +8669,8 @@ static int
 sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
 {
 	unsigned long load, min_load = ULONG_MAX;
-	unsigned int min_exit_latency = UINT_MAX;
-	u64 latest_idle_timestamp = 0;
+	u64 min_exit_latency = U64_MAX;
+	unsigned int nr_candidates = 0;
 	int least_loaded_cpu = this_cpu;
 	int shallowest_idle_cpu = -1;
 	int i;
@@ -8480,24 +8691,16 @@ sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *
 
 		if (available_idle_cpu(i)) {
 			struct cpuidle_state *idle = idle_get_state(rq);
-			if (idle && idle->exit_latency < min_exit_latency) {
-				/*
-				 * We give priority to a CPU whose idle state
-				 * has the smallest exit latency irrespective
-				 * of any idle timestamp.
-				 */
-				min_exit_latency = idle->exit_latency;
-				latest_idle_timestamp = rq->idle_stamp;
-				shallowest_idle_cpu = i;
-			} else if ((!idle || idle->exit_latency == min_exit_latency) &&
-				   rq->idle_stamp > latest_idle_timestamp) {
-				/*
-				 * If equal or no active idle state, then
-				 * the most recently idled CPU might have
-				 * a warmer cache.
-				 */
-				latest_idle_timestamp = rq->idle_stamp;
+			u64 exit_latency = idle ? idle->exit_latency : U64_MAX;
+
+			if (shallowest_idle_cpu == -1 || exit_latency < min_exit_latency) {
+				min_exit_latency = exit_latency;
 				shallowest_idle_cpu = i;
+				nr_candidates = 1;
+			} else if (exit_latency == min_exit_latency) {
+				nr_candidates++;
+				if (!reciprocal_scale(sched_rng(), nr_candidates))
+					shallowest_idle_cpu = i;
 			}
 		} else if (shallowest_idle_cpu == -1) {
 			load = cpu_load(cpu_rq(i));
@@ -9856,8 +10059,14 @@ static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity
 
 static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse)
 {
-	if (cfs_rq->next && cfs_rq->next->slice < pse->slice)
-		return false;
+	if (cfs_rq->next) {
+		if (cfs_rq->next->slice < pse->slice)
+			return false;
+
+		if (cfs_rq->next->slice == pse->slice &&
+		    entity_before(cfs_rq->next, pse))
+			return false;
+	}
 
 	set_next_buddy(cfs_rq, pse);
 	return true;
@@ -10395,6 +10604,7 @@ enum migration_type {
 #define LBF_SOME_PINNED	0x08
 #define LBF_ACTIVE_LB	0x10
 #define LBF_LLC_PINNED	0x20
+#define LBF_ACTIVE_LB_LLC	0x40
 
 struct lb_env {
 	struct sched_domain	*sd;
@@ -10724,7 +10934,7 @@ static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct
 static enum llc_mig can_migrate_llc_task(struct lb_env *env,
 					 struct task_struct *p)
 {
-	struct mm_struct *mm;
+	struct sched_cache_group *grp;
 	bool to_pref;
 	int cpu, src_cpu, dst_cpu;
 
@@ -10733,19 +10943,19 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env,
 
 	src_cpu = env->src_cpu;
 	dst_cpu = env->dst_cpu;
-	mm = p->mm;
-	if (!mm)
+	grp = rcu_dereference_all(p->sched_cache_grp);
+	if (!grp)
 		return mig_unrestricted;
 
-	cpu = READ_ONCE(mm->sc_stat.cpu);
+	cpu = READ_ONCE(grp->cpu);
 	if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu))
 		return mig_unrestricted;
 
 	/* skip cache aware load balance for too many threads */
-	if (invalid_llc_nr(mm, p, dst_cpu) ||
-	    exceed_llc_capacity(mm, dst_cpu)) {
-		if (READ_ONCE(mm->sc_stat.cpu) != -1)
-			WRITE_ONCE(mm->sc_stat.cpu, -1);
+	if (invalid_llc_nr(grp, p, dst_cpu) ||
+	    exceed_llc_capacity(grp, dst_cpu)) {
+		if (READ_ONCE(grp->cpu) != -1)
+			WRITE_ONCE(grp->cpu, -1);
 		return mig_unrestricted;
 	}
 
@@ -10813,6 +11023,21 @@ alb_break_llc(struct lb_env *env)
 	return false;
 }
 
+/*
+ * Returns true if p's preferred LLC does not match the destination CPU
+ * under migrate_llc_task semantics. Passive LB passes migrate_llc_task
+ * in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in
+ * env->flags to avoid overwriting env->migration_type.
+ */
+static inline bool
+migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env)
+{
+	return sched_cache_enabled() &&
+	       (env->migration_type == migrate_llc_task ||
+		env->flags & LBF_ACTIVE_LB_LLC) &&
+	       READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu);
+}
+
 /*
  * Check if migrating task p from env->src_cpu to
  * env->dst_cpu breaks LLC localiy.
@@ -10841,8 +11066,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
 	 * run on env->dst_cpu, skip the tasks do not prefer
 	 * env->dst_cpu, and find the one that prefers.
 	 */
-	if (env->migration_type == migrate_llc_task &&
-	    READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu))
+	if (migrate_llc_task_wrong_dst(p, env))
 		return true;
 
 	if (can_migrate_llc_task(env, p) != mig_forbid)
@@ -10864,6 +11088,12 @@ alb_break_llc(struct lb_env *env)
 	return false;
 }
 
+static inline bool
+migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env)
+{
+	return false;
+}
+
 static inline bool
 migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
 {
@@ -10963,7 +11193,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 	 * 4) too many balance attempts have failed.
 	 */
 	if (env->flags & LBF_ACTIVE_LB)
-		return 1;
+		return !migrate_llc_task_wrong_dst(p, env);
 
 	degrades = migrate_degrades_locality(p, env);
 	if (!degrades) {
@@ -11202,21 +11432,7 @@ static int detach_tasks(struct lb_env *env)
  */
 static void attach_tasks(struct lb_env *env)
 {
-	struct list_head *tasks = &env->tasks;
-	struct task_struct *p;
-	struct rq_flags rf;
-
-	rq_lock(env->dst_rq, &rf);
-	update_rq_clock(env->dst_rq);
-
-	while (!list_empty(tasks)) {
-		p = list_first_entry(tasks, struct task_struct, se.group_node);
-		list_del_init(&p->se.group_node);
-
-		attach_task(env->dst_rq, p);
-	}
-
-	rq_unlock(env->dst_rq, &rf);
+	__attach_tasks(env->dst_rq, &env->tasks);
 }
 
 #ifdef CONFIG_NO_HZ_COMMON
@@ -13362,6 +13578,20 @@ static int need_active_balance(struct lb_env *env)
 }
 
 static int active_load_balance_cpu_stop(void *data);
+static int active_load_balance_llc_cpu_stop(void *data);
+
+/*
+ * migration_type is checked elsewhere to decide migration policy, so
+ * it shouldn't be repurposed just to flag an LLC-directed active
+ * balance across the stopper. Pick the callback here instead.
+ */
+static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env)
+{
+	if (env->migration_type == migrate_llc_task)
+		return active_load_balance_llc_cpu_stop;
+
+	return active_load_balance_cpu_stop;
+}
 
 static int should_we_balance(struct lb_env *env)
 {
@@ -13495,7 +13725,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 	};
 	bool need_unlock = false;
 
-	cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask);
+	cpumask_and(cpus, sched_domain_span(sd), cpu_preferred_mask);
 
 	schedstat_inc(sd->lb_count[idle]);
 
@@ -13707,7 +13937,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 	}
 	if (active_balance) {
 		stop_one_cpu_nowait(cpu_of(busiest),
-				    active_load_balance_cpu_stop, busiest,
+				    alb_stop_fn(&env), busiest,
 				    &busiest->active_balance_work);
 	}
 	preempt_enable();
@@ -13812,7 +14042,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
  * least 1 task to be running on each physical CPU where possible, and
  * avoids physical / logical imbalances.
  */
-static int active_load_balance_cpu_stop(void *data)
+static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags)
 {
 	struct rq *busiest_rq = data;
 	int busiest_cpu = cpu_of(busiest_rq);
@@ -13862,7 +14092,7 @@ static int active_load_balance_cpu_stop(void *data)
 			.src_cpu	= busiest_rq->cpu,
 			.src_rq		= busiest_rq,
 			.idle		= CPU_IDLE,
-			.flags		= LBF_ACTIVE_LB,
+			.flags		= LBF_ACTIVE_LB | lb_flags,
 		};
 
 		schedstat_inc(sd->alb_count);
@@ -13890,6 +14120,16 @@ static int active_load_balance_cpu_stop(void *data)
 	return 0;
 }
 
+static int active_load_balance_cpu_stop(void *data)
+{
+	return __active_load_balance_cpu_stop(data, 0);
+}
+
+static int active_load_balance_llc_cpu_stop(void *data)
+{
+	return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC);
+}
+
 /*
  * Scale the max sched_balance_rq interval with the number of CPUs in the system.
  * This trades load-balance latency on larger machines for less cross talk.
@@ -14610,10 +14850,8 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
 	 */
 	this_rq->idle_stamp = rq_clock(this_rq);
 
-	/*
-	 * Do not pull tasks towards !active CPUs...
-	 */
-	if (!cpu_active(this_cpu))
+	/* Do not pull tasks towards !preferred CPUs */
+	if (!cpu_preferred(this_cpu))
 		return 0;
 
 	/*
@@ -15253,14 +15491,18 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p)
 	}
 }
 
-static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
+static void set_next_task_fair(struct rq *rq, struct task_struct *p, enum snt_e type)
 {
 	struct sched_entity *se = &p->se;
-	bool throttled = false;
 	struct cfs_rq *cfs_rq = &rq->cfs;
 	unsigned long weight = NICE_0_LOAD;
+	bool first = type == SNT_PICK;
+	bool throttled = false;
 	bool on_rq = se->on_rq;
 
+	if (type == SNT_REPICK)
+		goto repick;
+
 	clear_buddies(cfs_rq, se);
 
 	if (on_rq)
@@ -15304,11 +15546,18 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
 
 	WARN_ON_ONCE(se->sched_delayed);
 
-	if (hrtick_enabled_fair(rq))
-		hrtick_start_fair(rq, p);
-
 	update_misfit_status(p, rq);
 	sched_fair_update_stop_tick(rq, p);
+
+repick:
+	/*
+	 * A same-task repick skips put_prev_task_fair(), but
+	 * pick_task_fair() refreshed the entity hrtick_start_fair() reads
+	 * before selecting it again. rq->cfs.curr identifies that entity,
+	 * including with group scheduling.
+	 */
+	if (hrtick_enabled_fair(rq))
+		hrtick_start_fair(rq, p);
 }
 
 void init_cfs_rq(struct cfs_rq *cfs_rq)
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index eb73b65ce6c471..76f3c84ca6846a 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -487,8 +487,11 @@ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct t
 	update_rq_avg_idle(rq);
 }
 
-static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first)
+static void set_next_task_idle(struct rq *rq, struct task_struct *next, enum snt_e type)
 {
+	if (type == SNT_REPICK)
+		return;
+
 	update_idle_core(rq);
 	scx_update_idle(rq, true, true);
 	schedstat_inc(rq->sched_goidle);
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 85303add726df4..1535046a23ffa0 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1654,11 +1654,14 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags)
 		check_preempt_equal_prio(rq, p);
 }
 
-static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
+static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, enum snt_e type)
 {
 	struct sched_rt_entity *rt_se = &p->rt;
 	struct rt_rq *rt_rq = &rq->rt;
 
+	if (type == SNT_REPICK)
+		return;
+
 	p->se.exec_start = rq_clock_task(rq);
 	if (on_rt_rq(&p->rt))
 		update_stats_wait_end_rt(rt_rq, rt_se);
@@ -1666,7 +1669,7 @@ static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool f
 	/* The running task is never eligible for pushing */
 	dequeue_pushable_task(rq, p);
 
-	if (!first)
+	if (type != SNT_PICK)
 		return;
 
 	/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index e656c7059bf864..b733aa4c759e4c 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -791,6 +791,8 @@ enum scx_rq_flags {
 	SCX_RQ_BAL_CB_PENDING	= 1 << 6, /* must queue a cb after dispatching */
 	SCX_RQ_SUB_IDLE_RENOTIFY	= 1 << 7, /* sub-scheds are owed update_idle() */
 	SCX_RQ_ROOT_IDLE_RENOTIFY	= 1 << 8, /* the root is owed update_idle() */
+	SCX_RQ_PROXY_RETRY	= 1 << 9, /* proxy-rejected tasks need retry */
+	SCX_RQ_PROXY_TICK	= 1 << 10, /* proxy execution requires the tick */
 
 	SCX_RQ_IN_WAKEUP	= 1 << 16,
 	SCX_RQ_IN_DISPATCH	= 1 << 17,
@@ -810,8 +812,8 @@ struct scx_rq_rescue {
 
 struct scx_rq {
 	struct scx_dispatch_q	local_dsq;
+	struct scx_dispatch_q	reject_dsq;		/* staging for rejected tasks */
 #ifdef CONFIG_EXT_SUB_SCHED
-	struct scx_dispatch_q	reject_dsq;		/* staging for cap-rejected tasks */
 	struct scx_rq_rescue	rescue;
 #endif
 	struct list_head	runnable_list;		/* runnable tasks on this rq */
@@ -845,6 +847,9 @@ struct scx_rq {
 	struct list_head	deferred_reenq_users;	/* user DSQs requesting reenq */
 	struct balance_callback	deferred_bal_cb;
 	struct balance_callback	kick_sync_bal_cb;
+#ifdef CONFIG_NO_HZ_FULL
+	struct balance_callback	proxy_tick_bal_cb;
+#endif
 	struct irq_work		deferred_irq_work;
 	struct irq_work		kick_cpus_irq_work;
 };
@@ -1326,6 +1331,9 @@ struct rq {
 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
 	u64			prev_steal_time_rq;
 #endif
+#ifdef CONFIG_PREFERRED_CPU
+	bool			npc_push_work_pending;
+#endif
 
 	/* calc_load related fields */
 	unsigned long		calc_load_update;
@@ -1371,7 +1379,6 @@ struct rq {
 	struct task_struct	*core_pick;
 	struct sched_dl_entity	*core_dl_server;
 	unsigned int		core_enabled;
-	unsigned int		core_sched_seq;
 	struct rb_root		core_tree;
 
 	/* shared state -- careful with sched_core_cpu_deactivate() */
@@ -2447,16 +2454,25 @@ extern __read_mostly unsigned int sysctl_sched_features;
 
 #ifdef CONFIG_JUMP_LABEL
 
-#define SCHED_FEAT(name, enabled)					\
-static __always_inline bool static_branch_##name(struct static_key *key) \
-{									\
-	return static_key_##enabled(key);				\
+union sched_feat_key {
+	struct static_key_true	key_true;
+	struct static_key_false	key_false;
+};
+
+#define sched_feat_branch_true(key)	static_branch_likely(&(key)->key_true)
+#define sched_feat_branch_false(key)	static_branch_unlikely(&(key)->key_false)
+
+#define SCHED_FEAT(name, enabled)			\
+static __always_inline bool				\
+static_branch_##name(union sched_feat_key *key) 	\
+{							\
+	return sched_feat_branch_##enabled(key);	\
 }
 
 #include "features.h"
 #undef SCHED_FEAT
 
-extern struct static_key sched_feat_keys[__SCHED_FEAT_NR];
+extern union sched_feat_key sched_feat_keys[__SCHED_FEAT_NR];
 #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
 
 #else /* !CONFIG_JUMP_LABEL: */
@@ -2508,6 +2524,12 @@ static inline bool task_is_blocked(struct task_struct *p)
 	return !!p->blocked_on;
 }
 
+#ifdef CONFIG_SCHED_PROXY_EXEC
+void sched_proxy_block_task(struct rq *rq, struct task_struct *p);
+#else
+static inline void sched_proxy_block_task(struct rq *rq, struct task_struct *p) {}
+#endif
+
 static inline int task_on_cpu(struct rq *rq, struct task_struct *p)
 {
 	return p->on_cpu;
@@ -2527,11 +2549,17 @@ static inline int task_on_rq_migrating(struct task_struct *p)
 #define WF_EXEC			0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */
 #define WF_FORK			0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */
 #define WF_TTWU			0x08 /* Wakeup;            maps to SD_BALANCE_WAKE */
-
-#define WF_SYNC			0x10 /* Waker goes to sleep after wakeup */
+/*
+ * Hint that the caller expects the waker to sleep soon.
+ * Scheduler classes may use it for placement or preemption.
+ * Callers must not rely on it to prevent migration,
+ * preserve CPU locality or make the wakee run next.
+ */
+#define WF_SYNC			0x10
 #define WF_MIGRATED		0x20 /* Internal use, task got migrated */
 #define WF_CURRENT_CPU		0x40 /* Prefer to move the wakee to the current CPU. */
 #define WF_RQ_SELECTED		0x80 /* ->select_task_rq() was called */
+#define WF_TTWU_RQ		0x100 /* Wakeup completed through ttwu_runnable() */
 
 static_assert(WF_EXEC == SD_BALANCE_EXEC);
 static_assert(WF_FORK == SD_BALANCE_FORK);
@@ -2621,6 +2649,12 @@ struct affinity_context {
 
 extern s64 update_curr_common(struct rq *rq);
 
+enum snt_e {
+	SNT_NORMAL,	/* set_next_task() */
+	SNT_PICK,	/* put_prev_set_next_task(): prev != next */
+	SNT_REPICK,	/* put_prev_set_next_task(): prev == next */
+};
+
 struct sched_class {
 
 #ifdef CONFIG_UCLAMP_TASK
@@ -2678,7 +2712,7 @@ struct sched_class {
 	 * __schedule: rq->lock
 	 */
 	void (*put_prev_task)(struct rq *rq, struct task_struct *p, struct task_struct *next);
-	void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first);
+	void (*set_next_task)(struct rq *rq, struct task_struct *p, enum snt_e type);
 
 	/*
 	 * select_task_rq: p->pi_lock
@@ -2781,7 +2815,7 @@ static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
 
 static inline void set_next_task(struct rq *rq, struct task_struct *next)
 {
-	next->sched_class->set_next_task(rq, next, false);
+	next->sched_class->set_next_task(rq, next, SNT_NORMAL);
 }
 
 static inline void
@@ -2802,11 +2836,13 @@ static inline void put_prev_set_next_task(struct rq *rq,
 
 	__put_prev_set_next_dl_server(rq, prev, next);
 
-	if (next == prev)
+	if (next == prev) {
+		next->sched_class->set_next_task(rq, next, SNT_REPICK);
 		return;
+	}
 
 	prev->sched_class->put_prev_task(rq, prev, next);
-	next->sched_class->set_next_task(rq, next, true);
+	next->sched_class->set_next_task(rq, next, SNT_PICK);
 }
 
 /*
@@ -3139,6 +3175,25 @@ static inline void attach_one_task(struct rq *rq, struct task_struct *p)
 	attach_task(rq, p);
 }
 
+/*
+ * __attach_tasks() - attaches a list of tasks (using se.group_node) to
+ * the new rq
+ */
+static inline void __attach_tasks(struct rq *rq, struct list_head *tasks)
+{
+	guard(rq_lock)(rq);
+	update_rq_clock(rq);
+
+	while (!list_empty(tasks)) {
+		struct task_struct *p;
+
+		p = list_first_entry(tasks, struct task_struct, se.group_node);
+		list_del_init(&p->se.group_node);
+
+		attach_task(rq, p);
+	}
+}
+
 #ifdef CONFIG_PREEMPT_RT
 # define SCHED_NR_MIGRATE_BREAK 8
 #else
@@ -4222,8 +4277,9 @@ extern void balance_callbacks(struct rq *rq, struct balance_callback *head);
  * after which it is enqueued again.
  *
  * Typically this must be called while holding task_rq_lock, since most/all
- * properties are serialized under those locks. There is currently one
- * exception to this rule in sched/ext which only holds rq->lock.
+ * properties are serialized under those locks. There are currently two
+ * exceptions to this rule in sched/ext which only hold rq->lock: scx_bypass()
+ * and rq_offline_scx().
  */
 
 /*
@@ -4252,4 +4308,10 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change)
 
 #include "ext/ext.h"
 
+#ifdef CONFIG_PREFERRED_CPU
+void sched_push_current_non_preferred_cpu(struct rq *rq);
+#else	/* !CONFIG_PREFERRED_CPU */
+static inline void sched_push_current_non_preferred_cpu(struct rq *rq) { }
+#endif
+
 #endif /* _KERNEL_SCHED_SCHED_H */
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index c909ca0d8c87c1..1e0109ec36b3f9 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -27,8 +27,11 @@ wakeup_preempt_stop(struct rq *rq, struct task_struct *p, int flags)
 	/* we're never preempted */
 }
 
-static void set_next_task_stop(struct rq *rq, struct task_struct *stop, bool first)
+static void set_next_task_stop(struct rq *rq, struct task_struct *stop, enum snt_e type)
 {
+	if (type == SNT_REPICK)
+		return;
+
 	stop->se.exec_start = rq_clock_task(rq);
 }
 
diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c
index b215b0ead9a60b..45a127deeadd97 100644
--- a/kernel/sched/syscalls.c
+++ b/kernel/sched/syscalls.c
@@ -678,6 +678,8 @@ int __sched_setscheduler(struct task_struct *p,
 	if (prev_class != next_class)
 		queue_flags |= DEQUEUE_CLASS;
 
+	scx_prepare_setscheduler(p, policy);
+
 	scoped_guard (sched_change, p, queue_flags) {
 
 		if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) {
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 0248227d983a71..3dab0253976fb4 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -985,8 +985,8 @@ void sched_cache_active_set(void)
 }
 
 /*
- * Update the bottom sched_domain's llc_bytes for @cpu and all its
- * LLC siblings. Called from cacheinfo_cpu_online() or
+ * Update the bottom sched_domain's llc_bytes for @cpus sharing a physical
+ * LLC. Called from cacheinfo_cpu_online() or
  * cacheinfo_cpu_pre_down() with cpu hotplug lock held.
  *
  * Note: get_effective_llc_bytes() returns 0 on PowerPC.
@@ -996,17 +996,13 @@ void sched_cache_active_set(void)
  * and does not populates the per-CPU struct cpu_cacheinfo array
  * that get_cpu_cacheinfo_llc() reads.
  */
-void sched_update_llc_bytes(unsigned int cpu)
+void sched_update_llc_bytes(const struct cpumask *cpus)
 {
 	struct sched_domain *sd, *sdp;
 	unsigned int i;
 
 	sched_domains_mutex_lock();
 
-	sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu));
-	if (!sdp)
-		goto unlock;
-
 	/*
 	 * ci->shared_cpu_map is built incrementally as CPUs come
 	 * online, so the first CPU in an LLC initially sees
@@ -1014,14 +1010,22 @@ void sched_update_llc_bytes(unsigned int cpu)
 	 * get_effective_llc_bytes().  Re-evaluating every LLC
 	 * sibling on each online event corrects this once the full
 	 * shared_cpu_map is known.
+	 *
+	 * The departing CPU's domains have already been detached when
+	 * cacheinfo removes it. Use the surviving cache siblings instead.
+	 * They may belong to different cpuset partitions, so use each CPU's
+	 * own LLC domain to scale its share of the physical cache.
 	 */
-	for_each_cpu(i, sched_domain_span(sdp)) {
+	for_each_cpu(i, cpus) {
+		sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i));
+		if (!sdp)
+			continue;
+
 		sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);
 		if (sd)
 			sd->llc_bytes = get_effective_llc_bytes(i, sdp);
 	}
 
-unlock:
 	sched_domains_mutex_unlock();
 }
 
diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index d033f600f48c6f..477e4bf9c01e19 100644
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -174,15 +174,11 @@ EXPORT_SYMBOL_GPL(__wake_up_locked_key);
  * @mode: which threads
  * @key: opaque value to be passed to wakeup targets
  *
- * The sync wakeup differs that the waker knows that it will schedule
- * away soon, so while the target thread will be woken up, it will not
- * be migrated to another CPU - ie. the two threads are 'synchronized'
- * with each other. This can prevent needless bouncing between CPUs.
+ * Passes WF_SYNC to waitqueue wake functions. The default wake function
+ * forwards it to the scheduler; see WF_SYNC for the hint's semantics.
  *
- * On UP it can prevent extra preemption.
- *
- * If this function wakes up a task, it executes a full memory barrier before
- * accessing the task state.
+ * If this function wakes up a task, it executes a full memory barrier
+ * before accessing the task state.
  */
 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode,
 			void *key)
@@ -200,15 +196,7 @@ EXPORT_SYMBOL_GPL(__wake_up_sync_key);
  * @mode: which threads
  * @key: opaque value to be passed to wakeup targets
  *
- * The sync wakeup differs in that the waker knows that it will schedule
- * away soon, so while the target thread will be woken up, it will not
- * be migrated to another CPU - ie. the two threads are 'synchronized'
- * with each other. This can prevent needless bouncing between CPUs.
- *
- * On UP it can prevent extra preemption.
- *
- * If this function wakes up a task, it executes a full memory barrier before
- * accessing the task state.
+ * Same as __wake_up_sync_key(), but called with @wq_head->lock held.
  */
 void __wake_up_locked_sync_key(struct wait_queue_head *wq_head,
 			       unsigned int mode, void *key)
diff --git a/kernel/seccomp.c b/kernel/seccomp.c
index 86cf4460d69eaa..94c7ba80a8f744 100644
--- a/kernel/seccomp.c
+++ b/kernel/seccomp.c
@@ -1808,10 +1808,13 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter,
 	 * We need to check again if the addfd request has been handled,
 	 * and if not, we will remove it from the queue.
 	 */
-	if (list_empty(&kaddfd.list))
+	if (list_empty(&kaddfd.list)) {
 		ret = kaddfd.ret;
-	else
+	} else {
 		list_del(&kaddfd.list);
+		if (addfd.flags & SECCOMP_ADDFD_FLAG_SEND)
+			knotif->state = SECCOMP_NOTIFY_SENT;
+	}
 
 out_unlock:
 	mutex_unlock(&filter->notify_lock);
diff --git a/kernel/signal.c b/kernel/signal.c
index ec30550951ecfa..e433de93b43030 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -457,18 +457,42 @@ static void __sigqueue_free(struct sigqueue *q)
 	kmem_cache_free(sigqueue_cachep, q);
 }
 
-void flush_sigqueue(struct sigpending *queue)
+/*
+ * flush_sigqueue_list() can only be invoked without holding sighand::siglock in
+ * the following cases:
+ *
+ *  1) When flushing task::pending _after_ setting task::flags PF_EXITING
+ *
+ *     All functions which try to send a signal to @task will observe PF_EXITING
+ *     and drop the signal.
+ *
+ *  2) When flushing task::signal::shared_pending _after_ the last task in a
+ *     thread group was unhashed and task::sighand is NULL.
+ *
+ *     Nothing can queue a signal anymore because sighand is NULL.
+ */
+static void flush_sigqueue_list(struct list_head *head)
 {
-	struct sigqueue *q;
+	struct sigqueue *q, *tmp;
 
-	sigemptyset(&queue->signal);
-	while (!list_empty(&queue->list)) {
-		q = list_entry(queue->list.next, struct sigqueue , list);
+	list_for_each_entry_safe(q, tmp, head, list) {
 		list_del_init(&q->list);
 		__sigqueue_free(q);
 	}
 }
 
+void flush_sigqueue(struct sigpending *queue)
+{
+	sigemptyset(&queue->signal);
+	flush_sigqueue_list(&queue->list);
+}
+
+static void sigqueue_dequeue_pending(struct sigpending *queue, struct list_head *head)
+{
+	sigemptyset(&queue->signal);
+	list_splice_init(&queue->list, head);
+}
+
 /*
  * Flush all pending signals for this kthread.
  */
@@ -1019,6 +1043,21 @@ static inline bool legacy_queue(struct sigpending *signals, int sig)
 	return (sig < SIGRTMIN) && sigismember(&signals->signal, sig);
 }
 
+/*
+ * When PF_EXITING is set the task is on the way out and has t::pending
+ * flushed already. Prevent queueing of PIDTYPE_PID signals as they would
+ * be leaked.
+ */
+static inline bool task_can_queue_signal(struct task_struct *t, enum pid_type type)
+{
+	lockdep_assert_held(&t->sighand->siglock);
+
+	if (!(t->flags & PF_EXITING))
+		return true;
+
+	return type != PIDTYPE_PID;
+}
+
 static int __send_signal_locked(int sig, struct kernel_siginfo *info,
 				struct task_struct *t, enum pid_type type, bool force)
 {
@@ -1030,6 +1069,10 @@ static int __send_signal_locked(int sig, struct kernel_siginfo *info,
 	lockdep_assert_held(&t->sighand->siglock);
 
 	result = TRACE_SIGNAL_IGNORED;
+
+	if (!task_can_queue_signal(t, type))
+		goto ret;
+
 	if (!prepare_signal(sig, t, force))
 		goto ret;
 
@@ -1980,11 +2023,25 @@ static inline struct task_struct *posixtimer_get_target(struct k_itimer *tmr)
 	struct task_struct *t = pid_task(tmr->it_pid, tmr->it_pid_type);
 
 	if (t && tmr->it_pid_type != PIDTYPE_PID &&
-	    same_thread_group(t, current) && !current->exit_state)
+	    same_thread_group(t, current) && !(current->flags & PF_EXITING))
 		t = current;
 	return t;
 }
 
+/*
+ * Find the target task for the POSIX timer signal and prevent that a
+ * PIDTYPE_PID signal is queued on a task which has PF_EXITING set.
+ */
+static inline struct task_struct *posixtimer_get_unignore_target(struct k_itimer *tmr)
+{
+	struct task_struct *t = posixtimer_get_target(tmr);
+
+	if (t && task_can_queue_signal(t, tmr->it_pid_type))
+		return t;
+
+	return NULL;
+}
+
 void posixtimer_send_sigqueue(struct k_itimer *tmr)
 {
 	struct sigqueue *q = &tmr->sigq;
@@ -2002,6 +2059,9 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr)
 	if (!likely(lock_task_sighand(t, &flags)))
 		return;
 
+	if (!task_can_queue_signal(t, tmr->it_pid_type))
+		goto unlock;
+
 	/*
 	 * Update @tmr::sigqueue_seq for posix timer signals with sighand
 	 * locked to prevent a race against dequeue_signal().
@@ -2093,6 +2153,7 @@ void posixtimer_send_sigqueue(struct k_itimer *tmr)
 	result = TRACE_SIGNAL_DELIVERED;
 out:
 	trace_signal_generate(sig, &q->info, t, tmr->it_pid_type != PIDTYPE_PID, result);
+unlock:
 	unlock_task_sighand(t, &flags);
 }
 
@@ -2148,7 +2209,7 @@ static void posixtimer_sig_unignore(struct task_struct *tsk, int sig)
 		 * has exited by now, drop the reference count.
 		 */
 		guard(rcu)();
-		target = posixtimer_get_target(tmr);
+		target = posixtimer_get_unignore_target(tmr);
 		if (target)
 			posixtimer_queue_sigqueue(&tmr->sigq, target, tmr->it_pid_type);
 		else
@@ -3109,6 +3170,10 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)
 	sigset_t retarget;
 	struct task_struct *t;
 
+	/* Nobody dequeues them in a dying group, see get_signal(). */
+	if (tsk->signal->flags & SIGNAL_GROUP_EXIT)
+		return;
+
 	sigandsets(&retarget, &tsk->signal->shared_pending.signal, which);
 	if (sigisemptyset(&retarget))
 		return;
@@ -3132,42 +3197,36 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)
 
 void exit_signals(struct task_struct *tsk)
 {
+	LIST_HEAD(sigq_list);
 	int group_stop = 0;
-	sigset_t unblocked;
 
 	/*
 	 * @tsk is about to have PF_EXITING set - lock out users which
-	 * expect stable threadgroup.
+	 * expect a stable threadgroup.
 	 */
 	cgroup_threadgroup_change_begin(tsk);
 
-	if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
+	scoped_guard(spinlock_irq, &tsk->sighand->siglock) {
 		tsk->flags |= PF_EXITING;
-		cgroup_threadgroup_change_end(tsk);
-		return;
-	}
 
-	spin_lock_irq(&tsk->sighand->siglock);
-	/*
-	 * From now this task is not visible for group-wide signals,
-	 * see wants_signal(), do_signal_stop().
-	 */
-	tsk->flags |= PF_EXITING;
+		sigqueue_dequeue_pending(&tsk->pending, &sigq_list);
 
-	cgroup_threadgroup_change_end(tsk);
+		if (task_sigpending(tsk) && !thread_group_empty(tsk) &&
+		    !(tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
+			sigset_t unblocked = tsk->blocked;
 
-	if (!task_sigpending(tsk))
-		goto out;
+			signotset(&unblocked);
+			retarget_shared_pending(tsk, &unblocked);
+
+			if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) &&
+			    task_participate_group_stop(tsk))
+				group_stop = CLD_STOPPED;
+		}
+	}
 
-	unblocked = tsk->blocked;
-	signotset(&unblocked);
-	retarget_shared_pending(tsk, &unblocked);
+	cgroup_threadgroup_change_end(tsk);
 
-	if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) &&
-	    task_participate_group_stop(tsk))
-		group_stop = CLD_STOPPED;
-out:
-	spin_unlock_irq(&tsk->sighand->siglock);
+	flush_sigqueue_list(&sigq_list);
 
 	/*
 	 * If group stop has completed, deliver the notification.  This
@@ -3200,6 +3259,16 @@ long do_no_restart_syscall(struct restart_block *param)
 
 static void __set_task_blocked(struct task_struct *tsk, const sigset_t *newset)
 {
+	sigset_t floor, floored;
+
+	/* A user worker never unblocks anything but SIGKILL and SIGSTOP. */
+	if (unlikely(tsk->flags & PF_USER_WORKER)) {
+		siginitsetinv(&floor, SIG_KERNEL_ONLY_MASK);
+		sigorsets(&floored, newset, &floor);
+		WARN_ON_ONCE(!sigequalsets(&floored, newset));
+		newset = &floored;
+	}
+
 	if (task_sigpending(tsk) && !thread_group_empty(tsk)) {
 		sigset_t newblocked;
 		/* A set of now blocked but previously unblocked signals. */
diff --git a/kernel/smp.c b/kernel/smp.c
index b696bcc60c08fc..161d09cb0b3462 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -236,50 +236,76 @@ bool csd_lock_is_stuck(void)
 	return !!atomic_read(&n_csd_lock_stuck);
 }
 
+/* State that csd_lock_wait_toolong() carries across the __csd_lock_wait() loop. */
+struct csd_wait_state {
+	u64 ts_start;		/* When the wait began. */
+	u64 ts_report;		/* When the last complaint was printed. */
+	u64 ts_resend;		/* When the last IPI was re-sent, 0 if never. */
+	int bug_id;
+	unsigned long nmessages;
+};
+
+/*
+ * Report a CSD lock that came back, @ts_unstuck being when the release was
+ * noticed.  Only mention the re-send delta if an IPI was actually re-sent.
+ */
+static void csd_lock_print_unstuck(struct csd_wait_state *state, int cpu, u64 ts_unstuck)
+{
+	if (state->ts_resend)
+		pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock after %lld ns, %lld ns after the last IPI re-send.\n",
+			 state->bug_id, raw_smp_processor_id(), cpu,
+			 (s64)(ts_unstuck - state->ts_start),
+			 (s64)(ts_unstuck - state->ts_resend));
+	else
+		pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock after %lld ns.\n",
+			 state->bug_id, raw_smp_processor_id(), cpu,
+			 (s64)(ts_unstuck - state->ts_start));
+}
+
 /*
  * Complain if too much time spent waiting.  Note that only
  * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
  * so waiting on other types gets much less information.
  */
-static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id, unsigned long *nmessages)
+static bool csd_lock_wait_toolong(call_single_data_t *csd, struct csd_wait_state *state)
 {
 	int cpu = -1;
 	int cpux;
 	bool firsttime;
-	u64 ts2, ts_delta;
+	u64 ts_now, ts_delta;
 	call_single_data_t *cpu_cur_csd;
 	unsigned int flags = READ_ONCE(csd->node.u_flags);
 	unsigned long long csd_lock_timeout_ns = csd_lock_timeout * NSEC_PER_MSEC;
 
 	if (!(flags & CSD_FLAG_LOCK)) {
-		if (!unlikely(*bug_id))
+		if (!unlikely(state->bug_id))
 			return true;
+		ts_now = ktime_get_mono_fast_ns();
 		cpu = csd_lock_wait_getcpu(csd);
-		pr_alert("csd: CSD lock (#%d) got unstuck on CPU#%02d, CPU#%02d released the lock.\n",
-			 *bug_id, raw_smp_processor_id(), cpu);
+		csd_lock_print_unstuck(state, cpu, ts_now);
 		atomic_dec(&n_csd_lock_stuck);
 		return true;
 	}
 
-	ts2 = ktime_get_mono_fast_ns();
+	ts_now = ktime_get_mono_fast_ns();
 	/* How long since we last checked for a stuck CSD lock.*/
-	ts_delta = ts2 - *ts1;
-	if (likely(ts_delta <= csd_lock_timeout_ns * (*nmessages + 1) *
-			       (!*nmessages ? 1 : (ilog2(num_online_cpus()) / 2 + 1)) ||
+	ts_delta = ts_now - state->ts_report;
+	if (likely(ts_delta <= csd_lock_timeout_ns * (state->nmessages + 1) *
+			       (!state->nmessages ? 1 : (ilog2(num_online_cpus()) / 2 + 1)) ||
 		   csd_lock_timeout_ns == 0))
 		return false;
 
-	if (ts0 > ts2) {
+	if (state->ts_start > ts_now) {
 		/* Our own sched_clock went backward; don't blame another CPU. */
-		ts_delta = ts0 - ts2;
+		ts_delta = state->ts_start - ts_now;
 		pr_alert("sched_clock on CPU %d went backward by %llu ns\n", raw_smp_processor_id(), ts_delta);
-		*ts1 = ts2;
+		state->ts_report = ts_now;
 		return false;
 	}
 
-	firsttime = !*bug_id;
+	firsttime = !state->bug_id;
 	if (firsttime)
-		*bug_id = atomic_inc_return(&csd_bug_count);
+		state->bug_id = atomic_inc_return(&csd_bug_count);
 	cpu = csd_lock_wait_getcpu(csd);
 	if (WARN_ONCE(cpu < 0 || cpu >= nr_cpu_ids, "%s: cpu = %d\n", __func__, cpu))
 		cpux = 0;
@@ -293,11 +319,11 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in
 	 */
 	cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux));
 	/* How long since this CSD lock was stuck. */
-	ts_delta = ts2 - ts0;
+	ts_delta = ts_now - state->ts_start;
 	pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n",
-		 firsttime ? "Detected" : "Continued", *bug_id, raw_smp_processor_id(), (s64)ts_delta,
+		 firsttime ? "Detected" : "Continued", state->bug_id, raw_smp_processor_id(), (s64)ts_delta,
 		 cpu, csd->func, csd->info);
-	(*nmessages)++;
+	state->nmessages++;
 	if (firsttime)
 		atomic_inc(&n_csd_lock_stuck);
 	/*
@@ -308,23 +334,24 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in
 	BUG_ON(panic_on_ipistall > 0 && (s64)ts_delta > ((s64)panic_on_ipistall * NSEC_PER_MSEC));
 	if (cpu_cur_csd && csd != cpu_cur_csd) {
 		pr_alert("\tcsd: CSD lock (#%d) handling prior %pS(%ps) request.\n",
-			 *bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)),
+			 state->bug_id, READ_ONCE(per_cpu(cur_csd_func, cpux)),
 			 READ_ONCE(per_cpu(cur_csd_info, cpux)));
 	} else {
 		pr_alert("\tcsd: CSD lock (#%d) %s.\n",
-			 *bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request");
+			 state->bug_id, !cpu_cur_csd ? "unresponsive" : "handling this request");
 	}
 	if (cpu >= 0) {
 		if (atomic_cmpxchg_acquire(&per_cpu(trigger_backtrace, cpu), 1, 0))
 			dump_cpu_task(cpu);
 		if (!cpu_cur_csd) {
-			pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
+			pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", state->bug_id, raw_smp_processor_id(), cpu);
 			arch_send_call_function_single_ipi(cpu);
+			state->ts_resend = ktime_get_mono_fast_ns();
 		}
 	}
 	if (firsttime)
 		dump_stack();
-	*ts1 = ts2;
+	state->ts_report = ts_now;
 
 	return false;
 }
@@ -338,15 +365,13 @@ static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, in
  */
 static void __csd_lock_wait(call_single_data_t *csd)
 {
-	unsigned long nmessages = 0;
-	int bug_id = 0;
-	u64 ts0, ts1;
+	struct csd_wait_state state = {};
 
 	guard(preempt)();
 
-	ts1 = ts0 = ktime_get_mono_fast_ns();
+	state.ts_report = state.ts_start = ktime_get_mono_fast_ns();
 	for (;;) {
-		if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages))
+		if (csd_lock_wait_toolong(csd, &state))
 			break;
 		cpu_relax();
 	}
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 5d02c36c40e392..c3729c5b284b05 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -645,8 +645,8 @@ static void handle_softirqs(bool ksirqd)
 		h->action();
 		trace_softirq_exit(vec_nr);
 		if (unlikely(prev_count != preempt_count())) {
-			pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
-			       vec_nr, softirq_to_name[vec_nr], h->action,
+			pr_err("huh, entered softirq %u %s with preempt_count %08x, exited with %08x?\n",
+			       vec_nr, softirq_to_name[vec_nr],
 			       prev_count, preempt_count());
 			preempt_count_set(prev_count);
 		}
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index f7b75985d5420a..f5145b32f6be6f 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -29,14 +29,12 @@ EXPORT_SYMBOL(sysctl_vals);
 const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX };
 EXPORT_SYMBOL_GPL(sysctl_long_vals);
 
-#if defined(CONFIG_SYSCTL)
+#ifdef CONFIG_SYSCTL
 
 /* Constants used for minimum and maximum */
 static const int ngroups_max = NGROUPS_MAX;
 static const int cap_last_cap = CAP_LAST_CAP;
 
-#ifdef CONFIG_SYSCTL
-
 /**
  * enum sysctl_writes_mode - supported sysctl write modes
  *
@@ -64,14 +62,6 @@ enum sysctl_writes_mode {
 };
 
 static enum sysctl_writes_mode sysctl_writes_strict = SYSCTL_WRITES_STRICT;
-#endif /* CONFIG_SYSCTL */
-#endif /* CONFIG_SYSCTL */
-
-/*
- * /proc/sys support
- */
-
-#ifdef CONFIG_SYSCTL
 
 static int _proc_do_string(char *data, int maxlen, int dir,
 		char *buffer, size_t *lenp, loff_t *ppos)
@@ -483,7 +473,7 @@ int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp,
 
 	if (val < 0) {
 		*negp = true;
-		*u_ptr = k_ptr_op ? -k_ptr_op((ulong)val) : -(ulong)val;
+		*u_ptr = k_ptr_op ? k_ptr_op(-(ulong)val) : -(ulong)val;
 	} else {
 		*negp = false;
 		*u_ptr = k_ptr_op ? k_ptr_op((ulong)val) : (ulong) val;
@@ -595,7 +585,7 @@ static int do_proc_int_conv_minmax(bool *negp, unsigned long *u_ptr, int *k_ptr,
 static const char proc_wspace_sep[] = { ' ', '\t', '\n' };
 
 /*
- * Element type processed by do_proc_vec(). The tag selects the element size
+ * Element type processed by proc_vec(). The tag selects the element size
  * and signedness, and it selects which member of union proc_vec_conv is live.
  */
 enum proc_vec_type {
@@ -605,7 +595,7 @@ enum proc_vec_type {
 };
 
 /*
- * Converter passed to do_proc_vec(). Only the member matching the
+ * Converter passed to proc_vec(). Only the member matching the
  * enum proc_vec_type tag is ever read, so every dispatch stays fully typed and
  * no void * converter pointer is needed.
  */
@@ -637,101 +627,177 @@ static int proc_vec_conv(enum proc_vec_type type, union proc_vec_conv conv,
 	return -EINVAL;
 }
 
-/*
- * Read/write a vector of @type elements. The element size and signedness are
- * derived from @type, so a single runtime function replaces the per-type
- * variants. table->data is walked as raw bytes (@i) advanced by @size; the
- * converter performs the actual typed load/store.
- */
-static int do_proc_vec(const struct ctl_table *table, int dir,
-		       void *buffer, size_t *lenp, loff_t *ppos,
-		       enum proc_vec_type type, union proc_vec_conv conv)
+static int commit_conv_vec(const enum proc_vec_type data_type, void *dst,
+			    const void *src, size_t nr)
 {
-	int vleft, first = 1, err = 0;
-	size_t left, size;
-	bool is_unsigned;
-	char *i, *p;
+	size_t i;
 
-	switch (type) {
+	switch (data_type) {
 	case PROC_VEC_INT:
-		size = sizeof(int);
-		is_unsigned = false;
-		break;
-	case PROC_VEC_UINT:
-		size = sizeof(uint);
-		is_unsigned = true;
-		break;
+		for (i = 0; i < nr; i++)
+			WRITE_ONCE(((int *)dst)[i], ((const int *)src)[i]);
+		return 0;
+
 	case PROC_VEC_ULONG:
-		size = sizeof(ulong);
-		is_unsigned = true;
-		break;
+		for (i = 0; i < nr; i++)
+			WRITE_ONCE(((ulong *)dst)[i], ((const ulong *)src)[i]);
+		return 0;
 	default:
 		return -EINVAL;
 	}
+}
 
-	if (!table->data || !table->maxlen || !*lenp ||
-	    (*ppos && SYSCTL_KERN_TO_USER(dir))) {
-		*lenp = 0;
-		return 0;
-	}
-
-	i = table->data;
-	vleft = table->maxlen / size;
-	left = *lenp;
-
-	/* uint arrays are not supported, *Do not* add support for them. */
-	if (type == PROC_VEC_UINT && vleft != 1)
-		return -EINVAL;
-
-	if (SYSCTL_USER_TO_KERN(dir)) {
-		if (proc_first_pos_non_zero_ignore(ppos, table))
-			goto out;
-
-		if (left > PAGE_SIZE - 1)
-			left = PAGE_SIZE - 1;
-		p = buffer;
+/**
+ * apply_conv_on_vec - Apply converter function on data vector
+ *
+ * @conv: The converter to be applied
+ * @table: The sysctl table
+ * @data_type: Type used in converter appliation (INT, UINT or ULONG)
+ * @data_size: Number of data elements.
+ * @conv_dir: %TRUE if this is a write to the sysctl file
+ * @buf_nbyte: Number of bytes for buf
+ * @buf: The user buffer
+ * @buf_left_final: Number of outstanding (non converted) bytes.
+ *
+ * Element signedness is derived from @data_type. table->data is walked
+ * as raw bytes (@data) advanced by @data_size; the converter performs
+ * the actual typed load/store. Sets buf_left_final to the number of
+ * bytes that where left outstanding after conversion. Can be > 0.
+ *
+ * Returns: %0 on success. Non-zero on error.
+ */
+static int apply_conv_on_vec(const union proc_vec_conv conv,
+			     const struct ctl_table *table,
+			     const enum proc_vec_type data_type,
+			     const size_t data_size, const int conv_dir,
+			     const size_t buf_nbyte, void *buf,
+			     size_t *buf_left_final)
+{
+	int vec_left, err = 0;
+	size_t buf_left, nr_conv = 0;
+	char *data, *data_stage = NULL, *p;
+	bool is_unsigned = data_type == PROC_VEC_UINT || data_type == PROC_VEC_ULONG;
+
+	buf_left = buf_nbyte;
+
+	data = table->data;
+	if (SYSCTL_USER_TO_KERN(conv_dir)) {
+		if (buf_left > PAGE_SIZE - 1)
+			buf_left = PAGE_SIZE - 1;
+		p = buf;
+
+		if (table->maxlen > data_size) {
+			data_stage = kmemdup(table->data, table->maxlen, GFP_KERNEL);
+			if (!data_stage) {
+				err = -ENOMEM;
+				goto out;
+			}
+			data = data_stage;
+		}
 	}
 
-	for (; left && vleft--; i += size, first = 0) {
+	vec_left = table->maxlen / data_size;
+	for (; buf_left && vec_left--; data += data_size, nr_conv++) {
 		unsigned long lval;
 		bool neg = false;
 
-		if (SYSCTL_USER_TO_KERN(dir)) {
-			proc_skip_spaces(&p, &left);
+		if (SYSCTL_USER_TO_KERN(conv_dir)) {
+			proc_skip_spaces(&p, &buf_left);
 
-			if (!left)
+			if (!buf_left)
 				break;
-			err = proc_get_long(&p, &left, &lval, &neg,
+			err = proc_get_long(&p, &buf_left, &lval, &neg,
 					    proc_wspace_sep,
 					    sizeof(proc_wspace_sep), NULL);
 			if (!err && neg && is_unsigned)
 				err = -EINVAL;
 			if (err)
 				break;
-			if (proc_vec_conv(type, conv, &neg, &lval, i, dir, table)) {
+			if (proc_vec_conv(data_type, conv, &neg, &lval, data, conv_dir, table)) {
 				err = -EINVAL;
 				break;
 			}
 		} else {
-			if (proc_vec_conv(type, conv, &neg, &lval, i, dir, table)) {
+			if (proc_vec_conv(data_type, conv, &neg, &lval, data, conv_dir, table)) {
 				err = -EINVAL;
 				break;
 			}
-			if (!first)
-				proc_put_char(&buffer, &left, '\t');
-			proc_put_long(&buffer, &left, lval, neg);
+			if (nr_conv)
+				proc_put_char(&buf, &buf_left, '\t');
+			proc_put_long(&buf, &buf_left, lval, neg);
 		}
 	}
 
-	if (SYSCTL_KERN_TO_USER(dir) && !first && left && !err)
-		proc_put_char(&buffer, &left, '\n');
-	if (SYSCTL_USER_TO_KERN(dir) && !err && left)
-		proc_skip_spaces(&p, &left);
-	if (SYSCTL_USER_TO_KERN(dir) && first)
-		return err ? : -EINVAL;
-	*lenp -= left;
+	if (SYSCTL_USER_TO_KERN(conv_dir)) {
+		if (!err && buf_left)
+			proc_skip_spaces(&p, &buf_left);
+		if (!nr_conv) {
+			err = err ? : -EINVAL;
+			goto out;
+		}
+		if (!err && data_stage)
+			err = commit_conv_vec(data_type, table->data, data_stage, nr_conv);
+	} else {
+		if (nr_conv && buf_left && !err)
+			proc_put_char(&buf, &buf_left, '\n');
+	}
+
+	*buf_left_final = buf_left;
+
+out:
+	kfree(data_stage);
+	return err;
+}
+
+/* Read/write a vector of @type elements. */
+static int proc_vec(const struct ctl_table *table, int dir, void *buffer,
+		    size_t *lenp, loff_t *ppos, enum proc_vec_type type,
+		    union proc_vec_conv conv)
+{
+	int err = 0;
+	size_t data_size, left_nbyte = SIZE_MAX;
+
+	switch (type) {
+	case PROC_VEC_INT:
+		data_size = sizeof(int);
+		break;
+	case PROC_VEC_UINT:
+		data_size = sizeof(uint);
+		break;
+	case PROC_VEC_ULONG:
+		data_size = sizeof(ulong);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (!table->data || !table->maxlen || !*lenp ||
+	    (*ppos && SYSCTL_KERN_TO_USER(dir))) {
+		*lenp = 0;
+		return 0;
+	}
+
+	if (SYSCTL_USER_TO_KERN(dir)) {
+		if (proc_first_pos_non_zero_ignore(ppos, table))
+			goto out;
+	}
+
+	err = apply_conv_on_vec(conv, table, type, data_size, dir, *lenp, buffer,
+				&left_nbyte);
+
+	/*
+	 * An unchanged left_nbyte signals a write with no parsed element; which
+	 * is an error. Using SIZE_MAX to detect this error is possible because:
+	 * 1. lenp is bounded by KMALLOC_MAX_SIZE in proc_sys_call_handler
+	 * 2. lenp could never be SIZE_MAX as it is a "ridiculous" (exabyte) allocation.
+	 */
+	if (left_nbyte == SIZE_MAX)
+		return err;
+
+	*lenp -= left_nbyte;
 out:
 	*ppos += *lenp;
+
 	return err;
 }
 
@@ -756,12 +822,15 @@ int proc_douintvec_conv(const struct ctl_table *table, int dir, void *buffer,
 			int (*conv)(bool *negp, ulong *u_ptr, uint *k_ptr,
 				    int dir, const struct ctl_table *table))
 {
+	/* uint arrays are not supported, *Do not* add support for them. */
+	if (table->maxlen && (table->maxlen / sizeof(uint)) != 1)
+		return -EINVAL;
 
 	if (!conv)
 		conv = do_proc_uint_conv;
 
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT,
-			   (union proc_vec_conv){ .uint_conv = conv });
+	return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT,
+			(union proc_vec_conv){ .uint_conv = conv });
 }
 
 /**
@@ -820,8 +889,8 @@ int proc_dobool(const struct ctl_table *table, int dir, void *buffer,
 int proc_dointvec(const struct ctl_table *table, int dir, void *buffer,
 		  size_t *lenp, loff_t *ppos)
 {
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
-			   (union proc_vec_conv){ .int_conv = do_proc_int_conv });
+	return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
+			(union proc_vec_conv){ .int_conv = do_proc_int_conv });
 }
 
 /**
@@ -840,8 +909,7 @@ int proc_dointvec(const struct ctl_table *table, int dir, void *buffer,
 int proc_douintvec(const struct ctl_table *table, int dir, void *buffer,
 		size_t *lenp, loff_t *ppos)
 {
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT,
-			   (union proc_vec_conv){ .uint_conv = do_proc_uint_conv });
+	return proc_douintvec_conv(table, dir, buffer, lenp, ppos, do_proc_uint_conv);
 }
 
 /**
@@ -864,8 +932,8 @@ int proc_douintvec(const struct ctl_table *table, int dir, void *buffer,
 int proc_dointvec_minmax(const struct ctl_table *table, int dir,
 		  void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
-			   (union proc_vec_conv){ .int_conv = do_proc_int_conv_minmax });
+	return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
+			(union proc_vec_conv){ .int_conv = do_proc_int_conv_minmax });
 }
 
 /**
@@ -891,8 +959,8 @@ int proc_dointvec_minmax(const struct ctl_table *table, int dir,
 int proc_douintvec_minmax(const struct ctl_table *table, int dir,
 			  void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_UINT,
-			   (union proc_vec_conv){ .uint_conv = do_proc_uint_conv_minmax });
+	return proc_douintvec_conv(table, dir, buffer, lenp, ppos,
+				   do_proc_uint_conv_minmax);
 }
 
 /**
@@ -935,8 +1003,7 @@ int proc_dou8vec_minmax(const struct ctl_table *table, int dir,
 		tmp.extra2 = (unsigned int *) &max;
 
 	val = READ_ONCE(*data);
-	res = do_proc_vec(&tmp, dir, buffer, lenp, ppos, PROC_VEC_UINT,
-			  (union proc_vec_conv){ .uint_conv = do_proc_uint_conv_minmax });
+	res = proc_douintvec_minmax(&tmp, dir, buffer, lenp, ppos);
 	if (res)
 		return res;
 	if (SYSCTL_USER_TO_KERN(dir))
@@ -1066,8 +1133,8 @@ int proc_doulongvec_conv(const struct ctl_table *table, int dir,
 				int (*conv)(bool *negp, ulong *u_ptr, ulong *k_ptr,
 					    int dir, const struct ctl_table *table))
 {
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG,
-			   (union proc_vec_conv){ .ulong_conv = conv });
+	return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG,
+			(union proc_vec_conv){ .ulong_conv = conv });
 }
 
 /**
@@ -1087,10 +1154,10 @@ int proc_doulongvec_conv(const struct ctl_table *table, int dir,
  * Returns: %0 on success.
  */
 int proc_doulongvec_minmax(const struct ctl_table *table, int dir,
-			   void *buffer, size_t *lenp, loff_t *ppos)
+		void *buffer, size_t *lenp, loff_t *ppos)
 {
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG,
-			   (union proc_vec_conv){ .ulong_conv = do_proc_ulong_conv });
+	return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_ULONG,
+			(union proc_vec_conv){ .ulong_conv = do_proc_ulong_conv });
 }
 
 /**
@@ -1114,8 +1181,8 @@ int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer,
 {
 	if (!conv)
 		conv = do_proc_int_conv;
-	return do_proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
-			   (union proc_vec_conv){ .int_conv = conv });
+	return proc_vec(table, dir, buffer, lenp, ppos, PROC_VEC_INT,
+			(union proc_vec_conv){ .int_conv = conv });
 }
 
 /**
@@ -1265,6 +1332,87 @@ int proc_do_large_bitmap(const struct ctl_table *table, int dir,
 	return err;
 }
 
+int proc_do_static_key(const struct ctl_table *table, int dir,
+		       void *buffer, size_t *lenp, loff_t *ppos)
+{
+	struct static_key *key = (struct static_key *)table->data;
+	static DEFINE_MUTEX(static_key_mutex);
+	int val, ret;
+	struct ctl_table tmp = {
+		.data   = &val,
+		.maxlen = sizeof(val),
+		.mode   = table->mode,
+		.extra1 = SYSCTL_ZERO,
+		.extra2 = SYSCTL_ONE,
+	};
+
+	if (SYSCTL_USER_TO_KERN(dir) && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	mutex_lock(&static_key_mutex);
+	val = static_key_enabled(key);
+	ret = proc_dointvec_minmax(&tmp, dir, buffer, lenp, ppos);
+	if (SYSCTL_USER_TO_KERN(dir) && !ret) {
+		if (val)
+			static_key_enable(key);
+		else
+			static_key_disable(key);
+	}
+	mutex_unlock(&static_key_mutex);
+	return ret;
+}
+
+static const struct ctl_table sysctl_subsys_table[] = {
+	{
+		.procname	= "sysctl_writes_strict",
+		.data		= &sysctl_writes_strict,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_NEG_ONE,
+		.extra2		= SYSCTL_ONE,
+	},
+	{
+		.procname	= "ngroups_max",
+		.data		= (void *)&ngroups_max,
+		.maxlen		= sizeof (int),
+		.mode		= 0444,
+		.proc_handler	= proc_dointvec,
+	},
+	{
+		.procname	= "cap_last_cap",
+		.data		= (void *)&cap_last_cap,
+		.maxlen		= sizeof(int),
+		.mode		= 0444,
+		.proc_handler	= proc_dointvec,
+	},
+#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_ALLOW
+	{
+		.procname	= "unaligned-trap",
+		.data		= &unaligned_enabled,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
+#endif
+#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_NO_WARN
+	{
+		.procname	= "ignore-unaligned-usertrap",
+		.data		= &no_unaligned_warning,
+		.maxlen		= sizeof (int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
+#endif
+};
+
+int __init sysctl_init_bases(void)
+{
+	register_sysctl_init("kernel", sysctl_subsys_table);
+
+	return 0;
+}
+
 #else /* CONFIG_SYSCTL */
 
 int proc_dostring(const struct ctl_table *table, int dir,
@@ -1364,91 +1512,12 @@ int proc_do_large_bitmap(const struct ctl_table *table, int dir,
 	return -ENOSYS;
 }
 
-#endif /* CONFIG_SYSCTL */
-
-#if defined(CONFIG_SYSCTL)
 int proc_do_static_key(const struct ctl_table *table, int dir,
 		       void *buffer, size_t *lenp, loff_t *ppos)
 {
-	struct static_key *key = (struct static_key *)table->data;
-	static DEFINE_MUTEX(static_key_mutex);
-	int val, ret;
-	struct ctl_table tmp = {
-		.data   = &val,
-		.maxlen = sizeof(val),
-		.mode   = table->mode,
-		.extra1 = SYSCTL_ZERO,
-		.extra2 = SYSCTL_ONE,
-	};
-
-	if (SYSCTL_USER_TO_KERN(dir) && !capable(CAP_SYS_ADMIN))
-		return -EPERM;
-
-	mutex_lock(&static_key_mutex);
-	val = static_key_enabled(key);
-	ret = proc_dointvec_minmax(&tmp, dir, buffer, lenp, ppos);
-	if (SYSCTL_USER_TO_KERN(dir) && !ret) {
-		if (val)
-			static_key_enable(key);
-		else
-			static_key_disable(key);
-	}
-	mutex_unlock(&static_key_mutex);
-	return ret;
+	return -ENOSYS;
 }
 
-static const struct ctl_table sysctl_subsys_table[] = {
-#ifdef CONFIG_SYSCTL
-	{
-		.procname	= "sysctl_writes_strict",
-		.data		= &sysctl_writes_strict,
-		.maxlen		= sizeof(int),
-		.mode		= 0644,
-		.proc_handler	= proc_dointvec_minmax,
-		.extra1		= SYSCTL_NEG_ONE,
-		.extra2		= SYSCTL_ONE,
-	},
-#endif
-	{
-		.procname	= "ngroups_max",
-		.data		= (void *)&ngroups_max,
-		.maxlen		= sizeof (int),
-		.mode		= 0444,
-		.proc_handler	= proc_dointvec,
-	},
-	{
-		.procname	= "cap_last_cap",
-		.data		= (void *)&cap_last_cap,
-		.maxlen		= sizeof(int),
-		.mode		= 0444,
-		.proc_handler	= proc_dointvec,
-	},
-#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_ALLOW
-	{
-		.procname	= "unaligned-trap",
-		.data		= &unaligned_enabled,
-		.maxlen		= sizeof(int),
-		.mode		= 0644,
-		.proc_handler	= proc_dointvec,
-	},
-#endif
-#ifdef CONFIG_SYSCTL_ARCH_UNALIGN_NO_WARN
-	{
-		.procname	= "ignore-unaligned-usertrap",
-		.data		= &no_unaligned_warning,
-		.maxlen		= sizeof (int),
-		.mode		= 0644,
-		.proc_handler	= proc_dointvec,
-	},
-#endif
-};
-
-int __init sysctl_init_bases(void)
-{
-	register_sysctl_init("kernel", sysctl_subsys_table);
-
-	return 0;
-}
 #endif /* CONFIG_SYSCTL */
 /*
  * No sense putting this after each symbol definition, twice,
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index f31df72f0e9df1..3997dbb5aeadc1 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -59,8 +59,7 @@ static DEFINE_PER_CPU(struct listener_list, listener_array);
 
 enum actions {
 	REGISTER,
-	DEREGISTER,
-	CPU_DONT_CARE
+	DEREGISTER
 };
 
 static int prepare_reply(struct genl_info *info, u8 cmd, struct sk_buff **skbp,
@@ -473,7 +472,8 @@ static size_t taskstats_packet_size(void)
 	return size;
 }
 
-static int cmd_attr_pid(struct genl_info *info)
+static int cmd_attr_pid_tgid(struct genl_info *info, int attr,
+			     int (*fill)(pid_t, struct taskstats *))
 {
 	struct taskstats *stats;
 	struct sk_buff *rep_skb;
@@ -488,41 +488,15 @@ static int cmd_attr_pid(struct genl_info *info)
 		return rc;
 
 	rc = -EINVAL;
-	pid = nla_get_u32(info->attrs[TASKSTATS_CMD_ATTR_PID]);
-	stats = mk_reply(rep_skb, TASKSTATS_TYPE_PID, pid);
+	pid = nla_get_u32(info->attrs[attr]);
+	stats = mk_reply(rep_skb,
+			 attr == TASKSTATS_CMD_ATTR_PID
+				? TASKSTATS_TYPE_PID : TASKSTATS_TYPE_TGID,
+			 pid);
 	if (!stats)
 		goto err;
 
-	rc = fill_stats_for_pid(pid, stats);
-	if (rc < 0)
-		goto err;
-	return send_reply(rep_skb, info);
-err:
-	nlmsg_free(rep_skb);
-	return rc;
-}
-
-static int cmd_attr_tgid(struct genl_info *info)
-{
-	struct taskstats *stats;
-	struct sk_buff *rep_skb;
-	size_t size;
-	u32 tgid;
-	int rc;
-
-	size = taskstats_packet_size();
-
-	rc = prepare_reply(info, TASKSTATS_CMD_NEW, &rep_skb, size);
-	if (rc < 0)
-		return rc;
-
-	rc = -EINVAL;
-	tgid = nla_get_u32(info->attrs[TASKSTATS_CMD_ATTR_TGID]);
-	stats = mk_reply(rep_skb, TASKSTATS_TYPE_TGID, tgid);
-	if (!stats)
-		goto err;
-
-	rc = fill_stats_for_tgid(tgid, stats);
+	rc = fill(pid, stats);
 	if (rc < 0)
 		goto err;
 	return send_reply(rep_skb, info);
@@ -542,9 +516,11 @@ static int taskstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
 					TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK,
 					DEREGISTER);
 	else if (info->attrs[TASKSTATS_CMD_ATTR_PID])
-		return cmd_attr_pid(info);
+		return cmd_attr_pid_tgid(info, TASKSTATS_CMD_ATTR_PID,
+					 fill_stats_for_pid);
 	else if (info->attrs[TASKSTATS_CMD_ATTR_TGID])
-		return cmd_attr_tgid(info);
+		return cmd_attr_pid_tgid(info, TASKSTATS_CMD_ATTR_TGID,
+					 fill_stats_for_tgid);
 	else
 		return -EINVAL;
 }
@@ -590,6 +566,7 @@ void taskstats_exit(struct task_struct *tsk, int group_dead)
 	struct sk_buff *rep_skb;
 	size_t size;
 	int is_thread_group;
+	unsigned long flags;
 
 	if (!family_registered)
 		return;
@@ -635,7 +612,10 @@ void taskstats_exit(struct task_struct *tsk, int group_dead)
 	if (!stats)
 		goto err;
 
+	/* This was racy before, copy the stats under siglock. */
+	spin_lock_irqsave(&tsk->sighand->siglock, flags);
 	memcpy(stats, tsk->signal->stats, sizeof(*stats));
+	spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
 	stats->version = TASKSTATS_VERSION;
 
 send:
diff --git a/kernel/tests/.kunitconfig b/kernel/tests/.kunitconfig
new file mode 100644
index 00000000000000..b3d1206fd81abd
--- /dev/null
+++ b/kernel/tests/.kunitconfig
@@ -0,0 +1,4 @@
+CONFIG_KUNIT=y
+CONFIG_NAMESPACES=y
+CONFIG_USER_NS=y
+CONFIG_USER_NS_MAP_KUNIT_TEST=y
diff --git a/kernel/tests/user_ns_map_kunit.c b/kernel/tests/user_ns_map_kunit.c
new file mode 100644
index 00000000000000..033dccc6a535bb
--- /dev/null
+++ b/kernel/tests/user_ns_map_kunit.c
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit test for user namespace map insertion and sorting.
+ */
+
+#define pr_fmt(fmt) "user_namespace: " fmt
+
+#include <kunit/test.h>
+#include <linux/user_namespace.h>
+
+#define NR_EXTENTS (UID_GID_MAP_MAX_BASE_EXTENTS + 5)
+
+static void user_ns_map_insert(struct kunit *test)
+{
+	struct uid_gid_map map;
+	struct uid_gid_extent extent;
+	int i, ret;
+
+	memset(&map, 0, sizeof(map));
+
+	/* Insert up to UID_GID_MAP_MAX_BASE_EXTENTS elements */
+	for (i = 0; i < UID_GID_MAP_MAX_BASE_EXTENTS; i++) {
+		extent.first = i * 10;
+		extent.lower_first = i * 100;
+		extent.count = 5;
+
+		ret = uid_gid_map_insert_extent(&map, &extent);
+		KUNIT_ASSERT_EQ(test, ret, 0);
+	}
+
+	KUNIT_EXPECT_EQ(test, map.nr_extents, UID_GID_MAP_MAX_BASE_EXTENTS);
+
+	/* Verify the elements ended up in the 'extent' array */
+	for (i = 0; i < UID_GID_MAP_MAX_BASE_EXTENTS; i++) {
+		KUNIT_EXPECT_EQ(test, map.extent[i].first, i * 10);
+		KUNIT_EXPECT_EQ(test, map.extent[i].lower_first, i * 100);
+		KUNIT_EXPECT_EQ(test, map.extent[i].count, 5);
+	}
+}
+
+static void user_ns_map_insert_extended(struct kunit *test)
+{
+	struct uid_gid_map map;
+	struct uid_gid_extent extent;
+	int i, ret;
+
+	memset(&map, 0, sizeof(map));
+
+	/* Insert more than UID_GID_MAP_MAX_BASE_EXTENTS elements */
+	for (i = 0; i < NR_EXTENTS; i++) {
+		int value = 9 - i;
+
+		extent.first = value * 10;
+		extent.lower_first = value * 100;
+		extent.count = 5;
+
+		ret = uid_gid_map_insert_extent(&map, &extent);
+		KUNIT_ASSERT_EQ(test, ret, 0);
+	}
+
+	KUNIT_EXPECT_EQ(test, map.nr_extents, NR_EXTENTS);
+
+	/* Now sort the map to set up reverse mapping */
+	ret = uid_gid_map_sort(&map);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, map.reverse);
+
+	/* Verify the elements are in 'forward' and that sorting is correct */
+	for (i = 0; i < map.nr_extents; i++) {
+		KUNIT_EXPECT_EQ(test, map.forward[i].first, i * 10);
+		KUNIT_EXPECT_EQ(test, map.forward[i].lower_first, i * 100);
+		KUNIT_EXPECT_EQ(test, map.forward[i].count, 5);
+
+		KUNIT_EXPECT_EQ(test, map.reverse[i].first, i * 10);
+		KUNIT_EXPECT_EQ(test, map.reverse[i].lower_first, i * 100);
+		KUNIT_EXPECT_EQ(test, map.reverse[i].count, 5);
+	}
+
+	kfree(map.forward);
+	kfree(map.reverse);
+}
+
+static struct kunit_case user_ns_map_test_cases[] = {
+	KUNIT_CASE(user_ns_map_insert),
+	KUNIT_CASE(user_ns_map_insert_extended),
+	{}
+};
+
+static struct kunit_suite user_ns_map_test_suite = {
+	.name = "user_ns_map",
+	.test_cases = user_ns_map_test_cases,
+};
+
+kunit_test_suite(user_ns_map_test_suite);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("KUnit test for user namespace map insertion");
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index cbf1693c86b387..17dd38a6cee7a8 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -780,7 +780,8 @@ static void hrtimer_switch_to_hres(void)
 		return;
 	}
 	base->hres_active = true;
-	hrtimer_resolution = HIGH_RES_NSEC;
+	if (hrtimer_resolution != HIGH_RES_NSEC)
+		hrtimer_resolution = HIGH_RES_NSEC;
 
 	tick_setup_sched_timer(true);
 	/* "Retrigger" the interrupt to get things going */
@@ -2003,7 +2004,7 @@ bool hrtimer_active(const struct hrtimer *timer)
 		base = READ_ONCE(timer->base);
 		seq = raw_read_seqcount_begin(&base->seq);
 
-		if (timer->is_queued || base->running == timer)
+		if (timer->is_queued || READ_ONCE(base->running) == timer)
 			return true;
 
 	} while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base));
@@ -2040,7 +2041,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc
 	lockdep_assert_held(&cpu_base->lock);
 
 	debug_hrtimer_deactivate(timer);
-	base->running = timer;
+	WRITE_ONCE(base->running, timer);
 
 	/*
 	 * Separate the ->running assignment from the ->is_queued assignment.
@@ -2099,7 +2100,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc
 	raw_write_seqcount_barrier(&base->seq);
 
 	WARN_ON_ONCE(base->running != timer);
-	base->running = NULL;
+	WRITE_ONCE(base->running, NULL);
 }
 
 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
@@ -2323,9 +2324,9 @@ void hrtimer_run_queues(void)
 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
 {
 	struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer);
-	struct task_struct *task = t->task;
+	struct task_struct *task = hrtimer_sleeper_task_get(t);
 
-	t->task = NULL;
+	hrtimer_sleeper_task_set(t, NULL);
 	if (task)
 		wake_up_process(task);
 
@@ -2354,7 +2355,7 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode
 
 	/* If already expired, clear the task pointer and set current state to running */
 	if (!hrtimer_start_expires_user(&sl->timer, mode)) {
-		sl->task = NULL;
+		hrtimer_sleeper_task_set(sl, NULL);
 		__set_current_state(TASK_RUNNING);
 	}
 }
@@ -2388,7 +2389,7 @@ static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_
 	}
 
 	__hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode);
-	sl->task = current;
+	hrtimer_sleeper_task_set(sl, current);
 }
 
 /**
@@ -2432,17 +2433,17 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod
 		set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
 		hrtimer_sleeper_start_expires(t, mode);
 
-		if (likely(t->task))
+		if (likely(hrtimer_sleeper_task_get(t)))
 			schedule();
 
 		hrtimer_cancel(&t->timer);
 		mode = HRTIMER_MODE_ABS;
 
-	} while (t->task && !signal_pending(current));
+	} while (hrtimer_sleeper_task_get(t) && !signal_pending(current));
 
 	__set_current_state(TASK_RUNNING);
 
-	if (!t->task)
+	if (!hrtimer_sleeper_task_get(t))
 		return 0;
 
 	restart = &current->restart_block;
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index 80c35481153841..af1c69bf0c23db 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -8,6 +8,7 @@
 #include <linux/jiffies.h>
 #include <linux/module.h>
 #include <linux/init.h>
+#include <linux/sysctl.h>
 
 #include "timekeeping.h"
 #include "tick-internal.h"
@@ -101,6 +102,8 @@ void __init register_refined_jiffies(long cycles_per_second)
 #ifdef CONFIG_SYSCTL
 static ulong mult_hz(const ulong val)
 {
+	if (val >= ULONG_MAX / HZ)
+		return ULONG_MAX;
 	return val * HZ;
 }
 
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index d73d31c7994fe1..cd75d4bb5b646c 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -408,6 +408,7 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer)
 
 	new_timer->kclock = &clock_posix_cpu;
 	timerqueue_init(&new_timer->it.cpu.node);
+	INIT_LIST_HEAD(&new_timer->it.cpu.elist);
 	new_timer->it.cpu.pid = get_pid(pid);
 	rcu_read_unlock();
 	return 0;
@@ -438,6 +439,38 @@ static void trigger_base_recalc_expires(struct k_itimer *timer,
 	base->nextevt = 0;
 }
 
+static inline bool cpu_timer_enqueue(struct timerqueue_head *head,
+				     struct cpu_timer *ctmr)
+{
+	ctmr->head = head;
+	return timerqueue_add(head, &ctmr->node);
+}
+
+static inline bool cpu_timer_queued(struct cpu_timer *ctmr)
+{
+	return !!ctmr->head;
+}
+
+static inline bool cpu_timer_dequeue(struct cpu_timer *ctmr)
+{
+	if (cpu_timer_queued(ctmr)) {
+		timerqueue_del(ctmr->head, &ctmr->node);
+		ctmr->head = NULL;
+		return true;
+	}
+	return false;
+}
+
+static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr)
+{
+	return ctmr->node.expires;
+}
+
+static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp)
+{
+	ctmr->node.expires = exp;
+}
+
 /*
  * Dequeue the timer and reset the base if it was its earliest expiration.
  * It makes sure the next tick recalculates the base next expiration so we
@@ -565,6 +598,24 @@ static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned l
 	return NULL;
 }
 
+/*
+ * If the timer is queued on the expiry list, then it cannot be dequeued because
+ * the firing list is not protected by sighand->lock. The delivery path is
+ * waiting for the timer lock. So go back, unlock and retry.
+ */
+static bool posix_cpu_timer_on_expiry_list(struct k_itimer *timer)
+{
+	if (list_empty(&timer->it.cpu.elist))
+		return false;
+
+	/*
+	 * Prevent signal delivery as there is no point in delivering a signal
+	 * which is made obsolete right away.
+	 */
+	timer->it.cpu.firing = false;
+	return true;
+}
+
 /*
  * Clean up a CPU-clock timer that is about to be destroyed.
  * This is called from timer deletion with the timer already locked.
@@ -580,22 +631,15 @@ static int posix_cpu_timer_del(struct k_itimer *timer)
 	p = timer_lock_sighand(timer, &flags);
 
 	if (likely(p)) {
-		if (timer->it.cpu.firing) {
-			/*
-			 * Prevent signal delivery. The timer cannot be dequeued
-			 * because it is on the firing list which is not protected
-			 * by sighand->lock. The delivery path is waiting for
-			 * the timer lock. So go back, unlock and retry.
-			 */
-			timer->it.cpu.firing = false;
+		if (posix_cpu_timer_on_expiry_list(timer))
 			ret = TIMER_RETRY;
-		} else {
+		else
 			disarm_timer(timer, p);
-		}
 		unlock_task_sighand(p, &flags);
 	}
 
 	if (!ret) {
+		WARN_ON_ONCE(cpu_timer_queued(&timer->it.cpu));
 		put_pid(timer->it.cpu.pid);
 		timer->it_status = POSIX_TIMER_DISARMED;
 	}
@@ -628,18 +672,50 @@ static void cleanup_timers(struct posix_cputimers *pct)
 	cleanup_timerqueue(&pct->bases[CPUCLOCK_SCHED].tqhead);
 }
 
+static inline void posix_cpu_timers_exit_work(void);
+
 /*
- * These are both called with the siglock held, when the current thread
- * is being reaped.  When the final (leader) thread in the group is reaped,
- * posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit.
+ * Invoked from posixtimer_exit_task() after PF_EXITING was set in tsk::flags or
+ * from posixtimer_exec_cleanup().
  */
-void posix_cpu_timers_exit(struct task_struct *tsk)
+void posix_cpu_timers_exit_task(void)
 {
-	cleanup_timers(&tsk->posix_cputimers);
+	posix_cpu_timers_exit_work();
+
+	guard(spinlock_irq)(&current->sighand->siglock);
+	cleanup_timers(&current->posix_cputimers);
 }
-void posix_cpu_timers_exit_group(struct task_struct *tsk)
+
+/*
+ * Invoked from posixtimer_exit_group() after PF_EXITING was set in tsk::flags.
+ */
+void posix_cpu_timers_exit_group(void)
 {
-	cleanup_timers(&tsk->signal->posix_cputimers);
+	posix_cpu_timers_exit_task();
+
+	guard(spinlock_irq)(&current->sighand->siglock);
+	cleanup_timers(&current->signal->posix_cputimers);
+}
+
+/*
+ * This function validates that POSIX CPU timers can be safely enqueued on the
+ * target task.
+ *
+ * Enqueue is allowed when PF_EXITING is not set. If set then it is only allowed
+ * for process shared timers (type = PIDTYPE_TGID) as long as tsk::signal::flags
+ * does not have SIGNAL_GROUP_EXIT set. PIDTYPE_PID targets are not allowed at
+ * all when the task has PF_EXITING set.
+ *
+ * This guarantees that after the POSIX timer cleanup in posixtimer_exit() no
+ * POSIX CPU timers are queued on the task or in case of a group exit on the
+ * process.
+ */
+static inline bool task_can_enqueue_timer(struct task_struct *tsk, enum pid_type type)
+{
+	if (likely(!(tsk->flags & PF_EXITING)))
+		return true;
+
+	return type == PIDTYPE_TGID && !(tsk->signal->flags & SIGNAL_GROUP_EXIT);
 }
 
 /*
@@ -652,7 +728,13 @@ static void arm_timer(struct k_itimer *timer, struct task_struct *p)
 	struct cpu_timer *ctmr = &timer->it.cpu;
 	u64 newexp = cpu_timer_getexpires(ctmr);
 
+	lockdep_assert_held(&p->sighand->siglock);
+
 	timer->it_status = POSIX_TIMER_ARMED;
+
+	if (unlikely(!task_can_enqueue_timer(p, clock_pid_type(timer->it_clock))))
+		return;
+
 	if (!cpu_timer_enqueue(&base->tqhead, ctmr))
 		return;
 
@@ -731,14 +813,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
 	/* Retrieve the current expiry time before disarming the timer */
 	old_expires = cpu_timer_getexpires(ctmr);
 
-	if (unlikely(timer->it.cpu.firing)) {
-		/*
-		 * Prevent signal delivery. The timer cannot be dequeued
-		 * because it is on the firing list which is not protected
-		 * by sighand->lock. The delivery path is waiting for
-		 * the timer lock. So go back, unlock and retry.
-		 */
-		timer->it.cpu.firing = false;
+	if (posix_cpu_timer_on_expiry_list(timer)) {
 		ret = TIMER_RETRY;
 	} else {
 		cpu_timer_dequeue(ctmr);
@@ -1197,6 +1272,20 @@ static void posix_cpu_timers_work(struct callback_head *work)
 	mutex_unlock(&cw->mutex);
 }
 
+static inline void posix_cpu_timers_exit_work(void)
+{
+	/* Canceling the work is only valid for exit() but not for exec() */
+	if (!(current->flags & PF_EXITING))
+		return;
+	/*
+	 * current->flags has PF_EXITING set so this can be done lockless and
+	 * with interrupts enabled as PF_EXITING prevents the interrupt from
+	 * scheduling the work.
+	 */
+	if (current->posix_cputimers_work.scheduled)
+		task_work_cancel(current, &current->posix_cputimers_work.work);
+}
+
 /*
  * Invoked from the posix-timer core when a cancel operation failed because
  * the timer is marked firing. The caller holds rcu_read_lock(), which
@@ -1327,6 +1416,8 @@ static inline void __run_posix_cpu_timers(struct task_struct *tsk)
 	lockdep_posixtimer_exit();
 }
 
+static inline void posix_cpu_timers_exit_work(void) { }
+
 static void posix_cpu_timer_wait_running(struct k_itimer *timr)
 {
 	cpu_relax();
@@ -1473,7 +1564,7 @@ void run_posix_cpu_timers(void)
 	 * posix_cpu_timer_del() may fail to lock_task_sighand(tsk) and
 	 * miss timer->it.cpu.firing != 0.
 	 */
-	if (tsk->exit_state)
+	if (tsk->flags & PF_EXITING)
 		return;
 
 	/*
diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c
index 436ba794cc0b58..188dbedbffcac4 100644
--- a/kernel/time/posix-timers.c
+++ b/kernel/time/posix-timers.c
@@ -1077,13 +1077,9 @@ SYSCALL_DEFINE1(timer_delete, timer_t, timer_id)
 	return 0;
 }
 
-/*
- * Invoked from do_exit() when the last thread of a thread group exits.
- * At that point no other task can access the timers of the dying
- * task anymore.
- */
-void exit_itimers(struct task_struct *tsk)
+static void posixtimer_delete_timers(void)
 {
+	struct task_struct *tsk = current;
 	struct hlist_head timers;
 	struct hlist_node *next;
 	struct k_itimer *timer;
@@ -1120,6 +1116,24 @@ void exit_itimers(struct task_struct *tsk)
 	}
 }
 
+void posixtimer_exit(bool group_dead)
+{
+	if (group_dead) {
+		hrtimer_cancel(&current->signal->real_timer);
+		posix_cpu_timers_exit_group();
+		posixtimer_delete_timers();
+	} else {
+		posix_cpu_timers_exit_task();
+	}
+}
+
+void posixtimer_exec(void)
+{
+	posix_cpu_timers_exit_task();
+	posixtimer_delete_timers();
+	flush_itimer_signals();
+}
+
 SYSCALL_DEFINE2(clock_settime, const clockid_t, which_clock,
 		const struct __kernel_timespec __user *, tp)
 {
diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h
index 4ea9611dd7165c..79fd7ea7104695 100644
--- a/kernel/time/posix-timers.h
+++ b/kernel/time/posix-timers.h
@@ -51,3 +51,6 @@ int common_timer_set(struct k_itimer *timr, int flags,
 		     struct itimerspec64 *old_setting);
 void posix_timer_set_common(struct k_itimer *timer, struct itimerspec64 *new_setting);
 int common_timer_del(struct k_itimer *timer);
+
+void posix_cpu_timers_exit_task(void);
+void posix_cpu_timers_exit_group(void);
diff --git a/kernel/time/sleep_timeout.c b/kernel/time/sleep_timeout.c
index 3c90574bd904f1..ad8c415851ae19 100644
--- a/kernel/time/sleep_timeout.c
+++ b/kernel/time/sleep_timeout.c
@@ -212,7 +212,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
 	hrtimer_sleeper_start_expires(&t, mode);
 
-	if (likely(t.task))
+	if (likely(hrtimer_sleeper_task_get(&t)))
 		schedule();
 
 	hrtimer_cancel(&t.timer);
@@ -220,7 +220,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
 
 	__set_current_state(TASK_RUNNING);
 
-	return !t.task ? 0 : -EINTR;
+	return !hrtimer_sleeper_task_get(&t) ? 0 : -EINTR;
 }
 EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
 
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 6c3fea38671397..a7893a079a83f3 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -738,14 +738,11 @@ bool tick_nohz_tick_stopped_cpu(int cpu)
  */
 static void tick_nohz_update_jiffies(ktime_t now)
 {
-	unsigned long flags;
+	/* Reached only from irq_enter_rcu(), i.e. hard interrupt entry. */
+	lockdep_assert_irqs_disabled();
 
 	__this_cpu_write(tick_cpu_sched.idle_waketime, now);
-
-	local_irq_save(flags);
 	tick_do_update_jiffies64(now);
-	local_irq_restore(flags);
-
 	touch_softlockup_watchdog_sched();
 }
 
@@ -819,7 +816,7 @@ u64 get_jiffies_update(unsigned long *basej)
  */
 static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 {
-	u64 basemono, next_tick, delta, expires;
+	u64 basemono, next_tick, expires;
 	unsigned long basejiff;
 	int tick_cpu;
 
@@ -859,8 +856,7 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 	 * If the tick is due in the next period, keep it ticking or
 	 * force prod the timer.
 	 */
-	delta = next_tick - basemono;
-	if (delta <= (u64)TICK_NSEC) {
+	if (next_tick - basemono <= (u64)TICK_NSEC) {
 		/*
 		 * We've not stopped the tick yet, and there's a timer in the
 		 * next period, so no point in stopping it either, bail.
@@ -876,17 +872,19 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
 	 * the sleep time to the timekeeping 'max_deferment' value.
 	 * Otherwise we can sleep as long as we want.
 	 */
-	delta = timekeeping_max_deferment();
 	tick_cpu = READ_ONCE(tick_do_timer_cpu);
 	if (tick_cpu != cpu &&
-	    (tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST)))
-		delta = KTIME_MAX;
-
-	/* Calculate the next expiry time */
-	if (delta < (KTIME_MAX - basemono))
-		expires = basemono + delta;
-	else
+	    (tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST))) {
 		expires = KTIME_MAX;
+	} else {
+		expires = timekeeping_max_deferment();
+
+		/* Calculate the next expiry time */
+		if (expires < (KTIME_MAX - basemono))
+			expires += basemono;
+		else
+			expires = KTIME_MAX;
+	}
 
 	ts->timer_expires = min_t(u64, expires, next_tick);
 
diff --git a/kernel/time/time_test.c b/kernel/time/time_test.c
index 1b99180da28818..8b718767b3bafd 100644
--- a/kernel/time/time_test.c
+++ b/kernel/time/time_test.c
@@ -87,8 +87,24 @@ static void time64_to_tm_test_date_range(struct kunit *test)
 	}
 }
 
+static void time64_to_tm_test_wide_day_count(struct kunit *test)
+{
+	/* 2^31 days: the first count that does not fit in a 32-bit long. */
+	time64_t timestamp = (1LL << 31) * 86400;
+	struct tm result;
+
+	time64_to_tm(timestamp, 0, &result);
+
+	KUNIT_EXPECT_EQ(test, result.tm_year, 5879680);
+	KUNIT_EXPECT_EQ(test, result.tm_mon, 6);
+	KUNIT_EXPECT_EQ(test, result.tm_mday, 12);
+	KUNIT_EXPECT_EQ(test, result.tm_yday, 193);
+	KUNIT_EXPECT_EQ(test, result.tm_wday, 6);
+}
+
 static struct kunit_case time_test_cases[] = {
 	KUNIT_CASE_SLOW(time64_to_tm_test_date_range),
+	KUNIT_CASE(time64_to_tm_test_wide_day_count),
 	{}
 };
 
diff --git a/kernel/time/timeconv.c b/kernel/time/timeconv.c
index 59b922c826e771..aed3af950fa0b2 100644
--- a/kernel/time/timeconv.c
+++ b/kernel/time/timeconv.c
@@ -49,8 +49,9 @@ void time64_to_tm(time64_t totalsecs, int offset, struct tm *result)
 	u32 u32tmp, day_of_century, year_of_century, day_of_year, month, day;
 	u64 u64tmp, udays, century, year;
 	bool is_Jan_or_Feb, is_leap_year;
-	long days, rem;
 	int remainder;
+	long rem;
+	s64 days;
 
 	days = div_s64_rem(totalsecs, SECS_PER_DAY, &remainder);
 	rem = remainder;
@@ -70,7 +71,8 @@ void time64_to_tm(time64_t totalsecs, int offset, struct tm *result)
 	result->tm_sec = rem % 60;
 
 	/* January 1, 1970 was a Thursday. */
-	result->tm_wday = (4 + days) % 7;
+	div_s64_rem(days + 4, 7, &remainder);
+	result->tm_wday = remainder;
 	if (result->tm_wday < 0)
 		result->tm_wday += 7;
 
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index ea2e6e55f37bfd..d54c4d303db628 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -861,8 +861,10 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
 	 *
 	 * Write xtime_sec first so that even if the memcpy() tears the store
 	 * data integrity is provided for ktime_get_real_seconds().
+	 * The same goes for ktime_sec and ktime_get_seconds().
 	 */
 	WRITE_ONCE(tkd->timekeeper.xtime_sec, tk->xtime_sec);
+	WRITE_ONCE(tkd->timekeeper.ktime_sec, tk->ktime_sec);
 	memcpy(&tkd->timekeeper, tk, sizeof(*tk));
 	write_seqcount_end(&tkd->seq);
 }
@@ -1169,7 +1171,7 @@ time64_t ktime_get_seconds(void)
 	struct timekeeper *tk = &tk_core.timekeeper;
 
 	WARN_ON(timekeeping_suspended);
-	return tk->ktime_sec;
+	return READ_ONCE(tk->ktime_sec);
 }
 EXPORT_SYMBOL_GPL(ktime_get_seconds);
 
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index ae9abf14688e01..42afdcb229d8bd 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -890,7 +890,7 @@ static inline void detach_timer(struct timer_list *timer, bool clear_pending)
 
 	__hlist_del(entry);
 	if (clear_pending)
-		entry->pprev = NULL;
+		WRITE_ONCE(entry->pprev, NULL);
 	entry->next = LIST_POISON2;
 }
 
diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c
index 059d43355e650f..f920e73fff5112 100644
--- a/kernel/time/timer_migration.c
+++ b/kernel/time/timer_migration.c
@@ -715,7 +715,7 @@ static void __tmigr_cpu_activate(struct tmigr_cpu *tmc)
 
 	trace_tmigr_cpu_active(tmc);
 
-	tmc->cpuevt.ignore = true;
+	WRITE_ONCE(tmc->cpuevt.ignore, true);
 	WRITE_ONCE(tmc->wakeup, KTIME_MAX);
 
 	walk_groups(&tmigr_active_up, &data, tmc);
@@ -1258,7 +1258,7 @@ u64 tmigr_cpu_new_timer(u64 nextexp)
 	ret = READ_ONCE(tmc->wakeup);
 	if (nextexp != KTIME_MAX) {
 		if (nextexp != tmc->cpuevt.nextevt.expires ||
-		    tmc->cpuevt.ignore) {
+		    READ_ONCE(tmc->cpuevt.ignore)) {
 			ret = tmigr_new_timer(tmc, nextexp);
 			/*
 			 * Make sure the reevaluation of timers in idle path
@@ -1362,7 +1362,7 @@ static u64 __tmigr_cpu_deactivate(struct tmigr_cpu *tmc, u64 nextexp)
 	 * or CPU goes offline.
 	 */
 	if (nextexp != KTIME_MAX)
-		tmc->cpuevt.ignore = false;
+		WRITE_ONCE(tmc->cpuevt.ignore, false);
 
 	walk_groups(&tmigr_inactive_up, &data, tmc);
 	return data.firstexp;
diff --git a/kernel/time/vsyscall.c b/kernel/time/vsyscall.c
index aa59919b8f2c23..0e4b499328c082 100644
--- a/kernel/time/vsyscall.c
+++ b/kernel/time/vsyscall.c
@@ -41,14 +41,12 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct ti
 
 	nsec = tk->tkr_mono.xtime_nsec;
 	nsec += ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift);
-	while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
-		nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift);
-		vdso_ts->sec++;
-	}
-	vdso_ts->nsec	= nsec;
+	vdso_ts->sec	+= __iter_div64_u64_rem(nsec, (u64)NSEC_PER_SEC << tk->tkr_mono.shift,
+						&vdso_ts->nsec);
 
 	/* Copy MONOTONIC time for BOOTTIME */
 	sec	= vdso_ts->sec;
+	nsec	= vdso_ts->nsec;
 	/* Add the boot offset */
 	sec	+= tk->monotonic_to_boot.tv_sec;
 	nsec	+= (u64)tk->monotonic_to_boot.tv_nsec << tk->tkr_mono.shift;
@@ -56,12 +54,8 @@ static inline void update_vdso_time_data(struct vdso_time_data *vdata, struct ti
 	/* CLOCK_BOOTTIME */
 	vdso_ts		= &vc[CS_HRES_COARSE].basetime[CLOCK_BOOTTIME];
 	vdso_ts->sec	= sec;
-
-	while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
-		nsec -= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift);
-		vdso_ts->sec++;
-	}
-	vdso_ts->nsec	= nsec;
+	vdso_ts->sec	+= __iter_div64_u64_rem(nsec, (u64)NSEC_PER_SEC << tk->tkr_mono.shift,
+						&vdso_ts->nsec);
 
 	/* CLOCK_MONOTONIC_RAW */
 	vdso_ts		= &vc[CS_RAW].basetime[CLOCK_MONOTONIC_RAW];
@@ -161,11 +155,11 @@ void vdso_time_update_aux(struct timekeeper *tk)
 
 		vdso_ts->sec = tk->xtime_sec + tk->monotonic_to_aux.tv_sec;
 
-		nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
-		nsec += tk->monotonic_to_aux.tv_nsec;
-		vdso_ts->sec += __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
-		nsec = nsec << tk->tkr_mono.shift;
-		vdso_ts->nsec = nsec;
+		nsec = tk->tkr_mono.xtime_nsec;
+		nsec += (u64)tk->monotonic_to_aux.tv_nsec << tk->tkr_mono.shift;
+		vdso_ts->sec += __iter_div64_u64_rem(nsec,
+						     (u64)NSEC_PER_SEC << tk->tkr_mono.shift,
+						     &vdso_ts->nsec);
 	}
 
 	__arch_update_vdso_clock(vc);
diff --git a/kernel/torture.c b/kernel/torture.c
index 8c4e6b2fe8babb..1f5e4463b1e107 100644
--- a/kernel/torture.c
+++ b/kernel/torture.c
@@ -732,6 +732,26 @@ static ktime_t stutter_till_abs_time;
 static int stutter;
 static int stutter_gap;
 
+static bool _stutter_will_wait(ktime_t *till_ns)
+{
+	*till_ns = READ_ONCE(stutter_till_abs_time);
+	if (*till_ns && ktime_before(ktime_get(), *till_ns))
+		return true;
+	return false;
+}
+
+/*
+ * Will stutter_wait() actually stutter?  The returned result is of
+ * course immediately stale due to the passage of time.
+ */
+bool stutter_will_wait(void)
+{
+	ktime_t till_ns;
+
+	return _stutter_will_wait(&till_ns);
+}
+EXPORT_SYMBOL_GPL(stutter_will_wait);
+
 /*
  * Block until the stutter interval ends.  This must be called periodically
  * by all running kthreads that need to be subject to stuttering.
@@ -742,8 +762,7 @@ bool stutter_wait(const char *title)
 	ktime_t till_ns;
 
 	cond_resched_tasks_rcu_qs();
-	till_ns = READ_ONCE(stutter_till_abs_time);
-	if (till_ns && ktime_before(ktime_get(), till_ns)) {
+	if (_stutter_will_wait(&till_ns)) {
 		torture_hrtimeout_ns(till_ns, 0, HRTIMER_MODE_ABS, NULL);
 		ret = true;
 	}
@@ -993,3 +1012,18 @@ void torture_sched_set_normal(struct task_struct *t, int nice)
 	sched_set_normal(t, realnice);
 }
 EXPORT_SYMBOL_GPL(torture_sched_set_normal);
+
+/*
+ * Sum the specified per-CPU atomic_long_t variable.
+ */
+s64 torture_sum_pcpu_atomic_long(atomic_long_t __percpu *pcp)
+{
+	int cpu;
+	s64 sum = 0;
+
+	for_each_possible_cpu(cpu) {
+		sum += atomic_long_read(per_cpu_ptr(pcp, cpu));
+	}
+	return sum;
+}
+EXPORT_SYMBOL_GPL(torture_sum_pcpu_atomic_long);
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 29260951aa871c..195f78db9bdafe 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -995,7 +995,7 @@ static const struct bpf_func_proto bpf_d_path_proto = {
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_BTF_ID,
 	.arg1_btf_id	= &bpf_d_path_btf_ids[0],
-	.arg2_type	= ARG_PTR_TO_MEM | MEM_WRITE,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
 	.allowed	= bpf_d_path_allowed,
 };
@@ -1052,7 +1052,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = {
 	.func		= bpf_snprintf_btf,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_MEM | MEM_WRITE,
+	.arg1_type	= ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
 	.arg2_type	= ARG_MEM_SIZE,
 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg4_type	= ARG_MEM_SIZE,
@@ -1565,7 +1565,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = {
 	.gpl_only       = true,
 	.ret_type       = RET_INTEGER,
 	.arg1_type      = ARG_PTR_TO_CTX,
-	.arg2_type      = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE,
+	.arg2_type      = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE,
 	.arg3_type      = ARG_MEM_SIZE_OR_ZERO,
 	.arg4_type      = ARG_ANYTHING,
 };
diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c
index 1e9b00997ff274..9f2d98181779af 100644
--- a/kernel/trace/fprobe.c
+++ b/kernel/trace/fprobe.c
@@ -171,6 +171,11 @@ static inline bool write_fprobe_header(unsigned long *stack,
 static inline void read_fprobe_header(unsigned long *stack,
 					struct fprobe **fp, unsigned int *size_words)
 {
+	if (!*stack) {
+		*fp = NULL;
+		*size_words = 0;
+		return;
+	}
 	*fp = arch_decode_fprobe_header_fp(*stack);
 	*size_words = arch_decode_fprobe_header_size(*stack);
 }
@@ -203,6 +208,12 @@ static inline void read_fprobe_header(unsigned long *stack,
 {
 	struct __fprobe_header *fph = (struct __fprobe_header *)stack;
 
+	if (!*stack) {
+		*fp = NULL;
+		*size_words = 0;
+		return;
+	}
+
 	*fp = fph->fp;
 	*size_words = fph->size_words;
 }
@@ -635,6 +646,10 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops
 		}
 	}
 
+	/* Terminate the list, fgraph_reserve_data() does not clear it. */
+	if (used && used < reserved_words)
+		fgraph_data[used] = 0;
+
 	/* If any exit_handler is set, data must be used. */
 	return used != 0;
 }
diff --git a/kernel/umh.c b/kernel/umh.c
index 6e2c7bb315c68f..512dc018a50bcf 100644
--- a/kernel/umh.c
+++ b/kernel/umh.c
@@ -8,7 +8,7 @@
 #include <linux/binfmts.h>
 #include <linux/syscalls.h>
 #include <linux/unistd.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/slab.h>
 #include <linux/completion.h>
 #include <linux/cred.h>
diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
index 0bed462e9b2a24..1b23d819d39836 100644
--- a/kernel/user_namespace.c
+++ b/kernel/user_namespace.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+#include <kunit/visibility.h>
 #include <linux/export.h>
 #include <linux/nsproxy.h>
 #include <linux/slab.h>
@@ -162,9 +163,6 @@ int create_user_ns(struct cred *new)
 	ns_tree_add(ns);
 	return 0;
 fail_keyring:
-#ifdef CONFIG_PERSISTENT_KEYRINGS
-	key_put(ns->persistent_keyring_register);
-#endif
 	ns_common_free(ns);
 fail_free:
 	kmem_cache_free(user_ns_cachep, ns);
@@ -278,8 +276,8 @@ static int cmp_map_id(const void *k, const void *e)
  * map_id_range_down_max - Find idmap via binary search in ordered idmap array.
  * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
  */
-static struct uid_gid_extent *
-map_id_range_down_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 count)
+static const struct uid_gid_extent *
+map_id_range_down_max(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count)
 {
 	struct idmap_key key;
 
@@ -296,8 +294,8 @@ map_id_range_down_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 cou
  * Can only be called if number of mappings is equal or less than
  * UID_GID_MAP_MAX_BASE_EXTENTS.
  */
-static struct uid_gid_extent *
-map_id_range_down_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 count)
+static const struct uid_gid_extent *
+map_id_range_down_base(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count)
 {
 	unsigned idx;
 	u32 first, last, id2;
@@ -315,9 +313,9 @@ map_id_range_down_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 co
 	return NULL;
 }
 
-static u32 map_id_range_down(struct uid_gid_map *map, u32 id, u32 count)
+static u32 map_id_range_down(const struct uid_gid_map *map, u32 id, u32 count)
 {
-	struct uid_gid_extent *extent;
+	const struct uid_gid_extent *extent;
 	unsigned extents = map->nr_extents;
 	smp_rmb();
 
@@ -335,7 +333,7 @@ static u32 map_id_range_down(struct uid_gid_map *map, u32 id, u32 count)
 	return id;
 }
 
-u32 map_id_down(struct uid_gid_map *map, u32 id)
+u32 map_id_down(const struct uid_gid_map *map, u32 id)
 {
 	return map_id_range_down(map, id, 1);
 }
@@ -345,8 +343,8 @@ u32 map_id_down(struct uid_gid_map *map, u32 id)
  * Can only be called if number of mappings is equal or less than
  * UID_GID_MAP_MAX_BASE_EXTENTS.
  */
-static struct uid_gid_extent *
-map_id_range_up_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 count)
+static const struct uid_gid_extent *
+map_id_range_up_base(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count)
 {
 	unsigned idx;
 	u32 first, last, id2;
@@ -368,8 +366,8 @@ map_id_range_up_base(unsigned extents, struct uid_gid_map *map, u32 id, u32 coun
  * map_id_up_max - Find idmap via binary search in ordered idmap array.
  * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
  */
-static struct uid_gid_extent *
-map_id_range_up_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 count)
+static const struct uid_gid_extent *
+map_id_range_up_max(unsigned extents, const struct uid_gid_map *map, u32 id, u32 count)
 {
 	struct idmap_key key;
 
@@ -381,9 +379,9 @@ map_id_range_up_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 count
 		       sizeof(struct uid_gid_extent), cmp_map_id);
 }
 
-u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count)
+u32 map_id_range_up(const struct uid_gid_map *map, u32 id, u32 count)
 {
-	struct uid_gid_extent *extent;
+	const struct uid_gid_extent *extent;
 	unsigned extents = map->nr_extents;
 	smp_rmb();
 
@@ -401,7 +399,7 @@ u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count)
 	return id;
 }
 
-u32 map_id_up(struct uid_gid_map *map, u32 id)
+u32 map_id_up(const struct uid_gid_map *map, u32 id)
 {
 	return map_id_range_up(map, id, 1);
 }
@@ -782,11 +780,13 @@ static bool mappings_overlap(struct uid_gid_map *new_map,
 }
 
 /*
- * insert_extent - Safely insert a new idmap extent into struct uid_gid_map.
+ * uid_gid_map_insert_extent - Safely insert a new idmap extent into
+ * struct uid_gid_map.
  * Takes care to allocate a 4K block of memory if the number of mappings exceeds
  * UID_GID_MAP_MAX_BASE_EXTENTS.
  */
-static int insert_extent(struct uid_gid_map *map, struct uid_gid_extent *extent)
+VISIBLE_IF_KUNIT int uid_gid_map_insert_extent(struct uid_gid_map *map,
+					       struct uid_gid_extent *extent)
 {
 	struct uid_gid_extent *dest;
 
@@ -809,15 +809,20 @@ static int insert_extent(struct uid_gid_map *map, struct uid_gid_extent *extent)
 		map->reverse = NULL;
 	}
 
-	if (map->nr_extents < UID_GID_MAP_MAX_BASE_EXTENTS)
-		dest = &map->extent[map->nr_extents];
+	/*
+	 * nr_extents must be updated before the extent and forward arrays are
+	 * accessed, otherwise KSAN will assert an out-of-bounds error.
+	 */
+	map->nr_extents++;
+	if (map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
+		dest = &map->extent[map->nr_extents - 1];
 	else
-		dest = &map->forward[map->nr_extents];
+		dest = &map->forward[map->nr_extents - 1];
 
 	*dest = *extent;
-	map->nr_extents++;
 	return 0;
 }
+EXPORT_SYMBOL_IF_KUNIT(uid_gid_map_insert_extent);
 
 /* cmp function to sort() forward mappings */
 static int cmp_extents_forward(const void *a, const void *b)
@@ -850,10 +855,10 @@ static int cmp_extents_reverse(const void *a, const void *b)
 }
 
 /*
- * sort_idmaps - Sorts an array of idmap entries.
+ * uid_gid_map_sort - Sorts an array of idmap entries.
  * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
  */
-static int sort_idmaps(struct uid_gid_map *map)
+VISIBLE_IF_KUNIT int uid_gid_map_sort(struct uid_gid_map *map)
 {
 	if (map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS)
 		return 0;
@@ -874,6 +879,7 @@ static int sort_idmaps(struct uid_gid_map *map)
 
 	return 0;
 }
+EXPORT_SYMBOL_IF_KUNIT(uid_gid_map_sort);
 
 /**
  * verify_root_map() - check the uid 0 mapping
@@ -1042,7 +1048,7 @@ static ssize_t map_write(struct file *file, const char __user *buf,
 		    (next_line != NULL))
 			goto out;
 
-		ret = insert_extent(&new_map, &extent);
+		ret = uid_gid_map_insert_extent(&new_map, &extent);
 		if (ret < 0)
 			goto out;
 		ret = -EINVAL;
@@ -1086,7 +1092,7 @@ static ssize_t map_write(struct file *file, const char __user *buf,
 	 * If we want to use binary search for lookup, this clones the extent
 	 * array and sorts both copies.
 	 */
-	ret = sort_idmaps(&new_map);
+	ret = uid_gid_map_sort(&new_map);
 	if (ret < 0)
 		goto out;
 
diff --git a/kernel/utsname.c b/kernel/utsname.c
index ebbfc578a9d308..1ebf87e24607a3 100644
--- a/kernel/utsname.c
+++ b/kernel/utsname.c
@@ -81,7 +81,6 @@ struct uts_namespace *copy_utsname(u64 flags,
 {
 	struct uts_namespace *new_ns;
 
-	BUG_ON(!old_ns);
 	get_uts_ns(old_ns);
 
 	if (!(flags & CLONE_NEWUTS))
diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c
index 8614430ca212ae..5a417f8a922abd 100644
--- a/kernel/vmcore_info.c
+++ b/kernel/vmcore_info.c
@@ -216,7 +216,6 @@ static int __init crash_save_vmcoreinfo_init(void)
 	VMCOREINFO_LENGTH(free_area.free_list, MIGRATE_TYPES);
 	VMCOREINFO_NUMBER(NR_FREE_PAGES);
 	VMCOREINFO_NUMBER(PG_lru);
-	VMCOREINFO_NUMBER(PG_private);
 	VMCOREINFO_NUMBER(PG_swapcache);
 	VMCOREINFO_NUMBER(PG_swapbacked);
 #define PAGE_SLAB_MAPCOUNT_VALUE	(PGTY_slab << 24)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 1ae3732a2c5171..f0f10fa8796cca 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -56,6 +56,7 @@
 #include <linux/kvm_para.h>
 #include <linux/delay.h>
 #include <linux/irq_work.h>
+#include <linux/math64.h>
 
 #include "workqueue_internal.h"
 
@@ -161,6 +162,9 @@ struct llc_shard_layout {
  * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
  *     reads.
  *
+ * FN: wq_flush_pnode->lock protected. list_empty(&pwq->flush_node) can also be
+ *     tested with only pool->lock held.
+ *
  * K: Only modified by worker while holding pool->lock. Can be safely read by
  *    self, while holding pool->lock or from IRQ context if %current is the
  *    kworker.
@@ -274,8 +278,10 @@ struct pool_workqueue {
 	int			work_color;	/* L: current color */
 	int			flush_color;	/* L: flushing color */
 	int			refcnt;		/* L: reference count */
+	int			total_in_flight; /* L: sum of nr_in_flight[] */
 	int			nr_in_flight[WORK_NR_COLORS];
 						/* L: nr of in_flight works */
+	struct list_head	flush_node;	/* FN: node on wq_flush_pnode->pwqs */
 	bool			plugged;	/* L: execution suspended */
 
 	/*
@@ -344,6 +350,21 @@ struct wq_node_nr_active {
 	struct list_head	pending_pwqs;	/* LN: pwqs with inactive works */
 };
 
+/*
+ * Per-node list of pwqs active since the last flush so that flushes visit only
+ * those. One per possible node plus the NUMA_NO_NODE fallback at nr_node_ids,
+ * each allocated on its node. A pwq is added when a work item is queued to it
+ * while off the list and removed by a flush which finds it with nothing in
+ * flight. ->work_color mirrors wq->work_color. Reading it and adding the pwq in
+ * one ->lock section keeps queueing coherent against flushing: a work item is
+ * either stamped with the new color or its pwq is visible to the flusher.
+ */
+struct wq_flush_pnode {
+	raw_spinlock_t		lock;		/* nests inside pool locks */
+	int			work_color;	/* FN: mirrors wq->work_color */
+	struct list_head	pwqs;		/* FN: pwqs active since last flush */
+};
+
 /*
  * The externally visible workqueue.  It relays the issued work items to
  * the appropriate worker_pool through its pool_workqueues.
@@ -359,6 +380,7 @@ struct workqueue_struct {
 	struct wq_flusher	*first_flusher;	/* WQ: first flusher */
 	struct list_head	flusher_queue;	/* WQ: flush waiters */
 	struct list_head	flusher_overflow; /* WQ: flush overflow list */
+	struct wq_flush_pnode	**flush_pnodes; /* I: per-node flush membership */
 
 	struct list_head	maydays;	/* MD: pwqs requesting rescue */
 	struct worker		*rescuer;	/* MD: rescue worker */
@@ -367,6 +389,7 @@ struct workqueue_struct {
 
 	/* See alloc_workqueue() function comment for info on min/max_active */
 	int			max_active;	/* WO: max active works */
+	int			percpu_max_active; /* WO: max active works per cpu */
 	int			min_active;	/* WO: min active works */
 	int			saved_max_active; /* WQ: saved max_active */
 	int			saved_min_active; /* WQ: saved min_active */
@@ -510,12 +533,6 @@ static DEFINE_IDR(worker_pool_idr);	/* PR: idr of all pools */
 /* PL: hash of all unbound pools keyed by pool->attrs */
 static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
 
-/* I: attributes used when instantiating standard unbound pools on demand */
-static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
-
-/* I: attributes used when instantiating ordered pools on demand */
-static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
-
 /*
  * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
  * process context while holding a pool lock. Bounce to a dedicated kthread
@@ -619,6 +636,26 @@ static void show_one_worker_pool(struct worker_pool *pool);
 	list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,		\
 				 lockdep_is_held(&(wq->mutex)))
 
+/*
+ * Per-node arrays in this file carry an extra slot at nr_node_ids serving
+ * NUMA_NO_NODE. Return the slot after all nodes.
+ */
+static int next_node_with_fallback(int node)
+{
+	if (node >= nr_node_ids)
+		return nr_node_ids + 1;
+
+	node = next_node(node, node_possible_map);
+	if (node >= nr_node_ids)
+		return nr_node_ids;
+	return node;
+}
+
+#define for_each_node_with_fallback(node)				\
+	for ((node) = first_node(node_possible_map);			\
+	     (node) <= nr_node_ids;					\
+	     (node) = next_node_with_fallback(node))
+
 #ifdef CONFIG_DEBUG_OBJECTS_WORK
 
 static const struct debug_obj_descr work_debug_descr;
@@ -739,7 +776,7 @@ static int worker_pool_assign_id(struct worker_pool *pool)
 }
 
 static struct pool_workqueue __rcu **
-unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
+pwq_slot(struct workqueue_struct *wq, int cpu)
 {
        if (cpu >= 0)
                return per_cpu_ptr(wq->cpu_pwq, cpu);
@@ -748,9 +785,9 @@ unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
 }
 
 /* @cpu < 0 for dfl_pwq */
-static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
+static struct pool_workqueue *installed_pwq(struct workqueue_struct *wq, int cpu)
 {
-	return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
+	return rcu_dereference_check(*pwq_slot(wq, cpu),
 				     lockdep_is_held(&wq_pool_mutex) ||
 				     lockdep_is_held(&wq->mutex));
 }
@@ -765,7 +802,7 @@ static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
  */
 static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq)
 {
-	return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
+	return installed_pwq(wq, -1)->pool->attrs->__pod_cpumask;
 }
 
 static unsigned int work_color_to_flags(int color)
@@ -1532,6 +1569,7 @@ void wq_worker_tick(struct task_struct *task)
 	struct worker *worker = kthread_data(task);
 	struct pool_workqueue *pwq = worker->current_pwq;
 	struct worker_pool *pool = worker->pool;
+	u64 delta;
 
 	if (!pwq)
 		return;
@@ -1572,6 +1610,11 @@ void wq_worker_tick(struct task_struct *task)
 		pwq->stats[PWQ_STAT_CM_WAKEUP]++;
 
 	raw_spin_unlock(&pool->lock);
+
+	delta = READ_ONCE(worker->task->se.sum_exec_runtime) - worker->current_at;
+	trace_workqueue_cpu_intensive(pwq, worker->current_work,
+				      worker->current_func,
+				      div_u64(delta, NSEC_PER_USEC));
 }
 
 /**
@@ -1634,6 +1677,22 @@ static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
 	return wq->node_nr_active[node];
 }
 
+/**
+ * wq_flush_pnode - Determine wq_flush_pnode to use
+ * @wq: workqueue of interest
+ * @node: NUMA node, can be %NUMA_NO_NODE
+ *
+ * Return @wq's wq_flush_pnode for @node, the nr_node_ids fallback if @node is
+ * %NUMA_NO_NODE.
+ */
+static struct wq_flush_pnode *wq_flush_pnode(struct workqueue_struct *wq, int node)
+{
+	if (node == NUMA_NO_NODE)
+		node = nr_node_ids;
+
+	return wq->flush_pnodes[node];
+}
+
 /**
  * wq_update_node_max_active - Update per-node max_actives to use
  * @wq: workqueue to update
@@ -1712,6 +1771,14 @@ static void put_pwq(struct pool_workqueue *pwq)
 	lockdep_assert_held(&pwq->pool->lock);
 	if (likely(--pwq->refcnt))
 		return;
+
+	/*
+	 * pwq_release_worker is only created in workqueue_init(). Getting
+	 * here before that means an early workqueue allocation failed, which
+	 * is fatal anyway.
+	 */
+	BUG_ON(!pwq_release_worker);
+
 	/*
 	 * @pwq can't be released under pool->lock, bounce to a dedicated
 	 * kthread_worker to avoid A-A deadlocks.
@@ -1788,10 +1855,10 @@ static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
 
 	/*
 	 * A concurrency-managed per-cpu pool accounts nr_active per pwq, so
-	 * pwq->nr_active against wq->max_active is sufficient.
+	 * pwq->nr_active against wq->percpu_max_active is sufficient.
 	 */
 	if (is_percpu_pool(pool)) {
-		obtained = pwq->nr_active < READ_ONCE(wq->max_active);
+		obtained = pwq->nr_active < READ_ONCE(wq->percpu_max_active);
 		goto out;
 	}
 
@@ -2059,6 +2126,22 @@ static void pwq_dec_nr_active(struct pool_workqueue *pwq)
 		node_activate_pending_pwq(nna, pool);
 }
 
+/**
+ * pwq_inc_nr_in_flight - increment pwq's nr_in_flight
+ * @pwq: pwq of interest
+ * @work_color: color of the work item being queued
+ *
+ * A work item or a barrier with @work_color is being queued to @pwq.
+ *
+ * CONTEXT:
+ * raw_spin_lock_irq(pool->lock).
+ */
+static void pwq_inc_nr_in_flight(struct pool_workqueue *pwq, int work_color)
+{
+	pwq->nr_in_flight[work_color]++;
+	pwq->total_in_flight++;
+}
+
 /**
  * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
  * @pwq: pwq of interest
@@ -2083,6 +2166,7 @@ static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_
 		pwq_dec_nr_active(pwq);
 
 	pwq->nr_in_flight[color]--;
+	pwq->total_in_flight--;
 
 	/* is flush in progress and are we at the flushing tip? */
 	if (likely(pwq->flush_color != color))
@@ -2290,7 +2374,7 @@ static bool is_chained_work(struct workqueue_struct *wq)
 	 * Return %true iff I'm a worker executing a work item on @wq.  If
 	 * I'm @worker, it's safe to dereference it without locking.
 	 */
-	return worker && worker->current_pwq->wq == wq;
+	return worker && worker->current_pwq && worker->current_pwq->wq == wq;
 }
 
 /*
@@ -2432,7 +2516,21 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
 	if (WARN_ON(!list_empty(&work->entry)))
 		goto out;
 
-	pwq->nr_in_flight[pwq->work_color]++;
+	/*
+	 * Add @pwq to its wq_flush_pnode list if off it. Syncing ->work_color
+	 * in the same fpn->lock section is what keeps a concurrent flush from
+	 * missing @work, see wq_flush_pnode.
+	 */
+	if (unlikely(list_empty(&pwq->flush_node))) {
+		struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node);
+
+		raw_spin_lock(&fpn->lock);
+		pwq->work_color = fpn->work_color;
+		list_add_tail(&pwq->flush_node, &fpn->pwqs);
+		raw_spin_unlock(&fpn->lock);
+	}
+
+	pwq_inc_nr_in_flight(pwq, pwq->work_color);
 	work_flags = work_color_to_flags(pwq->work_color);
 
 	/*
@@ -3117,6 +3215,7 @@ static void send_mayday(struct pool_workqueue *pwq)
 		list_add_tail(&pwq->mayday_node, &wq->maydays);
 		wake_up_process(wq->rescuer->task);
 		pwq->stats[PWQ_STAT_MAYDAY]++;
+		trace_workqueue_mayday(pwq);
 	}
 }
 
@@ -3623,6 +3722,7 @@ static bool assign_rescuer_work(struct pool_workqueue *pwq, struct worker *rescu
 	list_for_each_entry_safe_from(work, n, &pool->worklist, entry) {
 		if (get_work_pwq(work) == pwq && assign_work(work, rescuer, &n)) {
 			pwq->stats[PWQ_STAT_RESCUED]++;
+			trace_workqueue_rescued(pwq, work, work->func);
 			/* put the cursor for next search */
 			list_move_tail(&cursor->entry, &n->entry);
 			return true;
@@ -3758,6 +3858,9 @@ static void bh_worker(struct worker *worker)
 	struct worker_pool *pool = worker->pool;
 	int nr_restarts = BH_WORKER_RESTARTS;
 	unsigned long end = jiffies + BH_WORKER_JIFFIES;
+	bool budget_exhausted = false;
+	bool timeout = false;
+	int restarts = 0;
 
 	worker_lock_callback(pool);
 	raw_spin_lock_irq(&pool->lock);
@@ -3780,8 +3883,23 @@ static void bh_worker(struct worker *worker)
 
 		if (assign_work(work, worker, NULL))
 			process_scheduled_works(worker);
-	} while (keep_working(pool) &&
-		 --nr_restarts && time_before(jiffies, end));
+
+		if (!keep_working(pool))
+			break;
+
+		if (!--nr_restarts) {
+			budget_exhausted = true;
+			break;
+		}
+
+		if (!time_before(jiffies, end)) {
+			budget_exhausted = true;
+			timeout = true;
+			break;
+		}
+
+		restarts++;
+	} while (1);
 
 	worker_set_flags(worker, WORKER_PREP);
 done:
@@ -3789,6 +3907,10 @@ static void bh_worker(struct worker *worker)
 	kick_pool(pool);
 	raw_spin_unlock_irq(&pool->lock);
 	worker_unlock_callback(pool);
+
+	if (budget_exhausted)
+		trace_workqueue_bh_budget_yield(pool, restarts, timeout,
+						pool->attrs->nice == HIGHPRI_NICE_LEVEL);
 }
 
 /*
@@ -3918,7 +4040,7 @@ static void check_flush_dependency(struct workqueue_struct *target_wq,
 	WARN_ONCE(current->flags & PF_MEMALLOC,
 		  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
 		  current->pid, current->comm, target_wq->name, target_func);
-	WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
+	WARN_ONCE(worker && worker->current_pwq && ((worker->current_pwq->wq->flags &
 			      (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
 		  "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
 		  worker->current_pwq->wq->name, worker->current_func,
@@ -4008,7 +4130,15 @@ static void insert_wq_barrier(struct pool_workqueue *pwq,
 		__set_bit(WORK_STRUCT_LINKED_BIT, bits);
 	}
 
-	pwq->nr_in_flight[work_color]++;
+	/*
+	 * Flushes must wait for barriers too as a barrier pins @pwq and a
+	 * worker until it runs and destroy_workqueue()'s drain relies on that.
+	 * @target being in flight keeps @pwq on its wq_flush_pnode list, so no
+	 * need to add it here.
+	 */
+	WARN_ON_ONCE(list_empty(&pwq->flush_node));
+
+	pwq_inc_nr_in_flight(pwq, work_color);
 	work_flags |= work_color_to_flags(work_color);
 
 	insert_work(pwq, &barr->work, head, work_flags);
@@ -4020,23 +4150,23 @@ static void insert_wq_barrier(struct pool_workqueue *pwq,
  * @flush_color: new flush color, < 0 for no-op
  * @work_color: new work color, < 0 for no-op
  *
- * Prepare pwqs for workqueue flushing.
+ * Prepare pwqs for workqueue flushing. Only the pwqs on @wq's wq_flush_pnode
+ * lists, which have been active since the last flush, are visited.
  *
- * If @flush_color is non-negative, flush_color on all pwqs should be
- * -1.  If no pwq has in-flight commands at the specified color, all
- * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
- * has in flight commands, its pwq->flush_color is set to
- * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
- * wakeup logic is armed and %true is returned.
+ * If @flush_color >= 0, flush_color on all visited pwqs should be -1. If no
+ * visited pwq has in-flight work items at the specified color, all
+ * pwq->flush_color's stay at -1 and return %false. If any visited pwq has
+ * in-flight work items, set its pwq->flush_color to @flush_color, update
+ * @wq->nr_pwqs_to_flush accordingly, arm pwq wakeup logic and return %true.
  *
- * The caller should have initialized @wq->first_flusher prior to
- * calling this function with non-negative @flush_color.  If
- * @flush_color is negative, no flush color update is done and %false
- * is returned.
+ * The caller should have initialized @wq->first_flusher prior to calling this
+ * function with non-negative @flush_color. If @flush_color < 0, no flush color
+ * update is done and %false is returned.
  *
- * If @work_color is non-negative, all pwqs should have the same
- * work_color which is previous to @work_color and all will be
- * advanced to @work_color.
+ * If @work_color >= 0, all visited pwqs and every wq_flush_pnode's work color
+ * mirror are advanced to @work_color. The mirrors of nodes with no active pwqs
+ * must be advanced too as a pwq added to a list later stamps the color it reads
+ * from the mirror.
  *
  * CONTEXT:
  * mutex_lock(wq->mutex).
@@ -4048,50 +4178,81 @@ static void insert_wq_barrier(struct pool_workqueue *pwq,
 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
 				      int flush_color, int work_color)
 {
+	struct pool_workqueue *pwq, *next;
 	bool wait = false;
-	struct pool_workqueue *pwq;
-	struct worker_pool *current_pool = NULL;
+	int node;
 
 	if (flush_color >= 0) {
 		WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
 		atomic_set(&wq->nr_pwqs_to_flush, 1);
 	}
 
-	/*
-	 * For unbound workqueue, pwqs will map to only a few pools.
-	 * Most of the time, pwqs within the same pool will be linked
-	 * sequentially to wq->pwqs by cpu index. So in the majority
-	 * of pwq iters, the pool is the same, only doing lock/unlock
-	 * if the pool has changed. This can largely reduce expensive
-	 * lock operations.
-	 */
-	for_each_pwq(pwq, wq) {
-		if (current_pool != pwq->pool) {
-			if (likely(current_pool))
-				raw_spin_unlock_irq(&current_pool->lock);
-			current_pool = pwq->pool;
-			raw_spin_lock_irq(&current_pool->lock);
+	for_each_node_with_fallback(node) {
+		struct wq_flush_pnode *fpn = wq->flush_pnodes[node];
+		struct worker_pool *current_pool = NULL;
+		LIST_HEAD(to_visit);
+		LIST_HEAD(to_keep);
+
+		/*
+		 * Advance the node's color mirror and take its active pwqs.
+		 * Work items queued to off-list pwqs afterwards carry the new
+		 * color and don't need visiting.
+		 */
+		raw_spin_lock_irq(&fpn->lock);
+		if (work_color >= 0) {
+			WARN_ON_ONCE(work_color != work_next_color(fpn->work_color));
+			fpn->work_color = work_color;
 		}
+		list_splice_init(&fpn->pwqs, &to_visit);
+		raw_spin_unlock_irq(&fpn->lock);
+
+		/*
+		 * Unbound workqueues map multiple pwqs to one pool. Only cycle
+		 * pool->lock when the pool changes.
+		 */
+		list_for_each_entry_safe(pwq, next, &to_visit, flush_node) {
+			if (current_pool != pwq->pool) {
+				if (likely(current_pool))
+					raw_spin_unlock_irq(&current_pool->lock);
+				current_pool = pwq->pool;
+				raw_spin_lock_irq(&current_pool->lock);
+			}
 
-		if (flush_color >= 0) {
-			WARN_ON_ONCE(pwq->flush_color != -1);
+			if (flush_color >= 0) {
+				WARN_ON_ONCE(pwq->flush_color != -1);
 
-			if (pwq->nr_in_flight[flush_color]) {
-				pwq->flush_color = flush_color;
-				atomic_inc(&wq->nr_pwqs_to_flush);
-				wait = true;
+				if (pwq->nr_in_flight[flush_color]) {
+					pwq->flush_color = flush_color;
+					atomic_inc(&wq->nr_pwqs_to_flush);
+					wait = true;
+				}
 			}
-		}
 
-		if (work_color >= 0) {
-			WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
-			pwq->work_color = work_color;
+			if (work_color >= 0) {
+				WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
+				pwq->work_color = work_color;
+			}
+
+			/*
+			 * A cascade can arm a color retired several advances
+			 * ago, so keep a pwq listed while any color has work
+			 * items in flight.
+			 */
+			if (pwq->total_in_flight)
+				list_move_tail(&pwq->flush_node, &to_keep);
+			else
+				list_del_init(&pwq->flush_node);
 		}
 
-	}
+		if (current_pool)
+			raw_spin_unlock_irq(&current_pool->lock);
 
-	if (current_pool)
-		raw_spin_unlock_irq(&current_pool->lock);
+		if (!list_empty(&to_keep)) {
+			raw_spin_lock_irq(&fpn->lock);
+			list_splice_tail(&to_keep, &fpn->pwqs);
+			raw_spin_unlock_irq(&fpn->lock);
+		}
+	}
 
 	if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
 		complete(&wq->first_flusher->done);
@@ -5049,13 +5210,10 @@ static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
 {
 	int node;
 
-	for_each_node(node) {
+	for_each_node_with_fallback(node) {
 		kfree(nna_ar[node]);
 		nna_ar[node] = NULL;
 	}
-
-	kfree(nna_ar[nr_node_ids]);
-	nna_ar[nr_node_ids] = NULL;
 }
 
 static void init_node_nr_active(struct wq_node_nr_active *nna)
@@ -5075,21 +5233,15 @@ static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
 	struct wq_node_nr_active *nna;
 	int node;
 
-	for_each_node(node) {
-		nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
+	for_each_node_with_fallback(node) {
+		nna = kzalloc_node(sizeof(*nna), GFP_KERNEL,
+				   node < nr_node_ids ? node : NUMA_NO_NODE);
 		if (!nna)
 			goto err_free;
 		init_node_nr_active(nna);
 		nna_ar[node] = nna;
 	}
 
-	/* [nr_node_ids] is used as the fallback */
-	nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
-	if (!nna)
-		goto err_free;
-	init_node_nr_active(nna);
-	nna_ar[nr_node_ids] = nna;
-
 	return 0;
 
 err_free:
@@ -5097,6 +5249,47 @@ static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
 	return -ENOMEM;
 }
 
+static void free_flush_pnodes(struct workqueue_struct *wq)
+{
+	int node;
+
+	if (!wq->flush_pnodes)
+		return;
+
+	for_each_node_with_fallback(node)
+		kfree(wq->flush_pnodes[node]);
+	kfree(wq->flush_pnodes);
+	wq->flush_pnodes = NULL;
+}
+
+static int alloc_flush_pnodes(struct workqueue_struct *wq)
+{
+	struct wq_flush_pnode *fpn;
+	int node;
+
+	wq->flush_pnodes = kcalloc(nr_node_ids + 1, sizeof(*wq->flush_pnodes), GFP_KERNEL);
+	if (!wq->flush_pnodes)
+		return -ENOMEM;
+
+	for_each_node_with_fallback(node) {
+		fpn = kzalloc_node(sizeof(*fpn), GFP_KERNEL,
+				   node < nr_node_ids ? node : NUMA_NO_NODE);
+		if (!fpn)
+			goto err_free;
+
+		raw_spin_lock_init(&fpn->lock);
+		fpn->work_color = wq->work_color;
+		INIT_LIST_HEAD(&fpn->pwqs);
+		wq->flush_pnodes[node] = fpn;
+	}
+
+	return 0;
+
+err_free:
+	free_flush_pnodes(wq);
+	return -ENOMEM;
+}
+
 static void rcu_free_wq(struct rcu_head *rcu)
 {
 	struct workqueue_struct *wq =
@@ -5105,6 +5298,7 @@ static void rcu_free_wq(struct rcu_head *rcu)
 	if (wq->flags & WQ_UNBOUND)
 		free_node_nr_active(wq->node_nr_active);
 
+	free_flush_pnodes(wq);
 	wq_free_lockdep(wq);
 	free_percpu(wq->cpu_pwq);
 	free_workqueue_attrs(wq->attrs);
@@ -5285,6 +5479,19 @@ static void pwq_release_workfn(struct kthread_work *work)
 		list_del_rcu(&pwq->pwqs_node);
 		is_last = list_empty(&wq->pwqs);
 
+		/*
+		 * An idle pwq can linger on its wq_flush_pnode list until the
+		 * next flush. wq->mutex excludes flushers, which never drop it
+		 * with pwqs on their private lists.
+		 */
+		if (!list_empty(&pwq->flush_node)) {
+			struct wq_flush_pnode *fpn = wq_flush_pnode(wq, pool->node);
+
+			raw_spin_lock_irq(&fpn->lock);
+			list_del_init(&pwq->flush_node);
+			raw_spin_unlock_irq(&fpn->lock);
+		}
+
 		/*
 		 * For ordered workqueue with a plugged dfl_pwq, restart it now.
 		 */
@@ -5335,6 +5542,7 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
 	pwq->refcnt = 1;
 	INIT_LIST_HEAD(&pwq->inactive_works);
 	INIT_LIST_HEAD(&pwq->pending_node);
+	INIT_LIST_HEAD(&pwq->flush_node);
 	INIT_LIST_HEAD(&pwq->pwqs_node);
 	INIT_LIST_HEAD(&pwq->mayday_node);
 	kthread_init_work(&pwq->release_work, pwq_release_workfn);
@@ -5385,22 +5593,34 @@ static struct worker_pool *get_percpu_pool(struct workqueue_struct *wq, int cpu)
 	return &per_cpu_ptr(pools, cpu)[highpri];
 }
 
-/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
+/*
+ * Obtain the pool backing @wq on @cpu and create a pwq associating the two.
+ * A WQ_PERCPU workqueue is backed by the static per-cpu pool of @cpu,
+ * everything else by a pool matching @attrs. @cpu < 0 is always unbound.
+ */
 static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq,
-					const struct workqueue_attrs *attrs)
+					const struct workqueue_attrs *attrs,
+					int cpu)
 {
 	struct worker_pool *pool;
 	struct pool_workqueue *pwq;
 
 	lockdep_assert_held(&wq_pool_mutex);
 
-	pool = get_unbound_pool(attrs);
-	if (!pool)
-		return NULL;
+	WARN_ON_ONCE((wq->flags & WQ_PERCPU) && cpu < 0);
+
+	if (cpu >= 0 && (wq->flags & WQ_PERCPU)) {
+		pool = get_percpu_pool(wq, cpu);
+	} else {
+		pool = get_unbound_pool(attrs);
+		if (!pool)
+			return NULL;
+	}
 
 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
 	if (!pwq) {
-		put_unbound_pool(pool);
+		if (!is_percpu_pool(pool))
+			put_unbound_pool(pool);
 		return NULL;
 	}
 
@@ -5408,6 +5628,13 @@ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq,
 	return pwq;
 }
 
+/* create a pwq backed by an unbound pool matching @attrs */
+static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
+						const struct workqueue_attrs *attrs)
+{
+	return alloc_pwq(wq, attrs, -1);
+}
+
 /**
  * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
  * @attrs: the wq_attrs of the default pwq of the target workqueue
@@ -5437,10 +5664,10 @@ static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu)
 }
 
 /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
-static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
-					int cpu, struct pool_workqueue *pwq)
+static struct pool_workqueue *install_pwq(struct workqueue_struct *wq,
+					  int cpu, struct pool_workqueue *pwq)
 {
-	struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
+	struct pool_workqueue __rcu **slot = pwq_slot(wq, cpu);
 	struct pool_workqueue *old_pwq;
 
 	lockdep_assert_held(&wq_pool_mutex);
@@ -5503,15 +5730,18 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
 
 	/*
 	 * If something goes wrong during CPU up/down, we'll fall back to
-	 * the default pwq covering whole @attrs->cpumask.  Always create
-	 * it even if we don't use it immediately.
+	 * the default pwq covering whole @attrs->cpumask.  Create it even
+	 * if we don't use it immediately.  A percpu workqueue has a pwq on
+	 * every possible CPU and never falls back, so it has no default.
 	 */
 	copy_workqueue_attrs(new_attrs, attrs);
 	wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
 	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
-	ctx->dfl_pwq = alloc_pwq(wq, new_attrs);
-	if (!ctx->dfl_pwq)
-		goto out_free;
+	if (!(wq->flags & WQ_PERCPU)) {
+		ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
+		if (!ctx->dfl_pwq)
+			goto out_free;
+	}
 
 	for_each_possible_cpu(cpu) {
 		if (new_attrs->ordered) {
@@ -5519,7 +5749,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
 			ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
 		} else {
 			wq_calc_pod_cpumask(new_attrs, cpu);
-			ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs);
+			ctx->pwq_tbl[cpu] = alloc_pwq(wq, new_attrs, cpu);
 			if (!ctx->pwq_tbl[cpu])
 				goto out_free;
 		}
@@ -5561,9 +5791,10 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
 
 	/* save the previous pwqs and install the new ones */
 	for_each_possible_cpu(cpu)
-		ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
-							ctx->pwq_tbl[cpu]);
-	ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);
+		ctx->pwq_tbl[cpu] = install_pwq(ctx->wq, cpu,
+						ctx->pwq_tbl[cpu]);
+	if (ctx->dfl_pwq)
+		ctx->dfl_pwq = install_pwq(ctx->wq, -1, ctx->dfl_pwq);
 
 	/* update node_nr_active->max, which only unbound workqueues have */
 	if (ctx->wq->flags & WQ_UNBOUND)
@@ -5577,10 +5808,6 @@ static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
 {
 	struct apply_wqattrs_ctx *ctx;
 
-	/* only unbound workqueues can change attributes */
-	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
-		return -EINVAL;
-
 	ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
@@ -5612,6 +5839,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
 {
 	int ret;
 
+	/* only unbound workqueues can change attributes */
+	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
+		return -EINVAL;
+
 	mutex_lock(&wq_pool_mutex);
 	ret = apply_workqueue_attrs_locked(wq, attrs);
 	mutex_unlock(&wq_pool_mutex);
@@ -5660,11 +5891,11 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 
 	/* nothing to do if the target cpumask matches the current pwq */
 	wq_calc_pod_cpumask(target_attrs, cpu);
-	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
+	if (wqattrs_equal(target_attrs, installed_pwq(wq, cpu)->pool->attrs))
 		return;
 
 	/* create a new pwq */
-	pwq = alloc_pwq(wq, target_attrs);
+	pwq = alloc_unbound_pwq(wq, target_attrs);
 	if (!pwq) {
 		pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
 			wq->name);
@@ -5673,49 +5904,43 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 
 	/* Install the new pwq. */
 	mutex_lock(&wq->mutex);
-	old_pwq = install_unbound_pwq(wq, cpu, pwq);
+	old_pwq = install_pwq(wq, cpu, pwq);
 	goto out_unlock;
 
 use_dfl_pwq:
 	mutex_lock(&wq->mutex);
-	pwq = unbound_pwq(wq, -1);
+	pwq = installed_pwq(wq, -1);
 	raw_spin_lock_irq(&pwq->pool->lock);
 	get_pwq(pwq);
 	raw_spin_unlock_irq(&pwq->pool->lock);
-	old_pwq = install_unbound_pwq(wq, cpu, pwq);
+	old_pwq = install_pwq(wq, cpu, pwq);
 out_unlock:
 	mutex_unlock(&wq->mutex);
 	put_pwq_unlocked(old_pwq);
 }
 
-static int alloc_and_link_percpu_pwqs(struct workqueue_struct *wq)
+/* the attrs @wq asked for, as spelled by its flags */
+static struct workqueue_attrs *alloc_wq_std_attrs(struct workqueue_struct *wq)
 {
-	struct pool_workqueue *pwq;
-	int cpu;
-
-	for_each_possible_cpu(cpu) {
-		struct worker_pool *pool = get_percpu_pool(wq, cpu);
-
-		pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
-		if (!pwq)
-			return -ENOMEM;
+	struct workqueue_attrs *attrs;
 
-		init_pwq(pwq, wq, pool);
+	attrs = alloc_workqueue_attrs();
+	if (!attrs)
+		return NULL;
 
-		mutex_lock(&wq->mutex);
-		link_pwq(pwq);
-		mutex_unlock(&wq->mutex);
+	if (wq->flags & WQ_HIGHPRI)
+		attrs->nice = HIGHPRI_NICE_LEVEL;
 
-		rcu_assign_pointer(*per_cpu_ptr(wq->cpu_pwq, cpu), pwq);
-	}
+	if (wq->flags & __WQ_ORDERED)
+		attrs->ordered = true;
 
-	return 0;
+	return attrs;
 }
 
 static int alloc_and_link_pwqs(struct workqueue_struct *wq)
 {
-	bool highpri = wq->flags & WQ_HIGHPRI;
-	int cpu, ret;
+	struct workqueue_attrs *attrs;
+	int ret;
 
 	lockdep_assert_held(&wq_pool_mutex);
 
@@ -5723,47 +5948,29 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq)
 	if (!wq->cpu_pwq)
 		goto enomem;
 
-	if (!(wq->flags & WQ_UNBOUND)) {
-		ret = alloc_and_link_percpu_pwqs(wq);
-	} else if (wq->flags & __WQ_ORDERED) {
-		struct pool_workqueue *dfl_pwq;
+	attrs = alloc_wq_std_attrs(wq);
+	if (!attrs)
+		goto enomem;
+
+	ret = apply_workqueue_attrs_locked(wq, attrs);
+	free_workqueue_attrs(attrs);
+	if (ret)
+		goto enomem;
+
+	if (wq->flags & __WQ_ORDERED) {
+		struct pool_workqueue *dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
 
-		ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);
 		/* there should only be single pwq for ordering guarantee */
-		dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
-		WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
-			      wq->pwqs.prev != &dfl_pwq->pwqs_node),
+		WARN(wq->pwqs.next != &dfl_pwq->pwqs_node ||
+		     wq->pwqs.prev != &dfl_pwq->pwqs_node,
 		     "ordering guarantee broken for workqueue %s\n", wq->name);
-	} else {
-		ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);
 	}
 
-	if (ret)
-		goto enomem;
 	return 0;
 
 enomem:
-	if (wq->cpu_pwq) {
-		for_each_possible_cpu(cpu) {
-			struct pool_workqueue __rcu **slot;
-			struct pool_workqueue *pwq;
-
-			slot = per_cpu_ptr(wq->cpu_pwq, cpu);
-			pwq = rcu_access_pointer(*slot);
-			if (pwq) {
-				/*
-				 * Unlink pwq from wq->pwqs since link_pwq()
-				 * may have already added it. wq->mutex is not
-				 * needed as the wq has not been published yet.
-				 */
-				if (!list_empty(&pwq->pwqs_node))
-					list_del_rcu(&pwq->pwqs_node);
-				kmem_cache_free(pwq_cache, pwq);
-			}
-		}
-		free_percpu(wq->cpu_pwq);
-		wq->cpu_pwq = NULL;
-	}
+	free_percpu(wq->cpu_pwq);
+	wq->cpu_pwq = NULL;
 	return -ENOMEM;
 }
 
@@ -5828,9 +6035,9 @@ static int init_rescuer(struct workqueue_struct *wq)
  * wq_adjust_max_active - update a wq's max_active to the current setting
  * @wq: target workqueue
  *
- * If @wq isn't freezing, set @wq->max_active to the saved_max_active and
- * activate inactive work items accordingly. If @wq is freezing, clear
- * @wq->max_active to zero.
+ * If @wq isn't freezing, set the limit that applies to @wq's backing to the
+ * saved_max_active and activate inactive work items accordingly. If @wq is
+ * freezing, clear it to zero.
  */
 static void wq_adjust_max_active(struct workqueue_struct *wq)
 {
@@ -5847,20 +6054,25 @@ static void wq_adjust_max_active(struct workqueue_struct *wq)
 		new_min = wq->saved_min_active;
 	}
 
-	if (wq->max_active == new_max && wq->min_active == new_min)
-		return;
-
 	/*
-	 * Update @wq->max/min_active and then kick inactive work items if more
-	 * active work items are allowed. This doesn't break work item ordering
+	 * Update the limit and then kick inactive work items if more active
+	 * work items are allowed. This doesn't break work item ordering
 	 * because new work items are always queued behind existing inactive
 	 * work items if there are any.
 	 */
-	WRITE_ONCE(wq->max_active, new_max);
-	WRITE_ONCE(wq->min_active, new_min);
+	if (wq->flags & WQ_UNBOUND) {
+		if (wq->max_active == new_max && wq->min_active == new_min)
+			return;
 
-	if (wq->flags & WQ_UNBOUND)
+		WRITE_ONCE(wq->max_active, new_max);
+		WRITE_ONCE(wq->min_active, new_min);
 		wq_update_node_max_active(wq, -1);
+	} else {
+		if (wq->percpu_max_active == new_max)
+			return;
+
+		WRITE_ONCE(wq->percpu_max_active, new_max);
+	}
 
 	if (new_max == 0)
 		return;
@@ -5957,10 +6169,14 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 
 	/* init wq */
 	wq->flags = flags;
-	wq->max_active = max_active;
-	wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
-	wq->saved_max_active = wq->max_active;
-	wq->saved_min_active = wq->min_active;
+	if (flags & WQ_UNBOUND) {
+		wq->max_active = max_active;
+		wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
+		wq->saved_min_active = wq->min_active;
+	} else {
+		wq->percpu_max_active = max_active;
+	}
+	wq->saved_max_active = max_active;
 	mutex_init(&wq->mutex);
 	atomic_set(&wq->nr_pwqs_to_flush, 0);
 	INIT_LIST_HEAD(&wq->pwqs);
@@ -5970,6 +6186,9 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 
 	INIT_LIST_HEAD(&wq->list);
 
+	if (alloc_flush_pnodes(wq) < 0)
+		goto err_free_wq;
+
 	if (flags & WQ_UNBOUND) {
 		if (alloc_node_nr_active(wq->node_nr_active) < 0)
 			goto err_free_wq;
@@ -6007,12 +6226,13 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 	 * flushing the pwq_release_worker ensures that the pwq_release_workfn()
 	 * completes before calling kfree(wq).
 	 */
-	if (wq->flags & WQ_UNBOUND) {
+	if (pwq_release_worker)
 		kthread_flush_worker(pwq_release_worker);
+	if (wq->flags & WQ_UNBOUND)
 		free_node_nr_active(wq->node_nr_active);
-	}
 err_free_wq:
 	free_workqueue_attrs(wq->attrs);
+	free_flush_pnodes(wq);
 	kfree(wq);
 	return NULL;
 err_unlock_destroy:
@@ -6104,11 +6324,8 @@ EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map);
 
 static bool pwq_busy(struct pool_workqueue *pwq)
 {
-	int i;
-
-	for (i = 0; i < WORK_NR_COLORS; i++)
-		if (pwq->nr_in_flight[i])
-			return true;
+	if (pwq->total_in_flight)
+		return true;
 
 	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
 		return true;
@@ -6199,12 +6416,12 @@ void destroy_workqueue(struct workqueue_struct *wq)
 	rcu_read_lock();
 
 	for_each_possible_cpu(cpu) {
-		put_pwq_unlocked(unbound_pwq(wq, cpu));
-		RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
+		put_pwq_unlocked(installed_pwq(wq, cpu));
+		RCU_INIT_POINTER(*pwq_slot(wq, cpu), NULL);
 	}
 
-	put_pwq_unlocked(unbound_pwq(wq, -1));
-	RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);
+	put_pwq_unlocked(installed_pwq(wq, -1));
+	RCU_INIT_POINTER(*pwq_slot(wq, -1), NULL);
 
 	rcu_read_unlock();
 }
@@ -8203,24 +8420,6 @@ void __init workqueue_init_early(void)
 			init_cpu_worker_pool(pool, cpu, std_nice[i++]);
 	}
 
-	/* create default unbound and ordered wq attrs */
-	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
-		struct workqueue_attrs *attrs;
-
-		BUG_ON(!(attrs = alloc_workqueue_attrs()));
-		attrs->nice = std_nice[i];
-		unbound_std_wq_attrs[i] = attrs;
-
-		/*
-		 * An ordered wq should have only one pwq as ordering is
-		 * guaranteed by max_active which is enforced by pwqs.
-		 */
-		BUG_ON(!(attrs = alloc_workqueue_attrs()));
-		attrs->nice = std_nice[i];
-		attrs->ordered = true;
-		ordered_wq_attrs[i] = attrs;
-	}
-
 	system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0);
 	system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0);
 	system_highpri_wq = alloc_workqueue("events_highpri",
@@ -8264,7 +8463,7 @@ static void __init wq_cpu_intensive_thresh_init(void)
 	 * lot slower CPUs including microcontrollers where the threshold is way
 	 * too low.
 	 *
-	 * Let's scale up the threshold upto 1 second if BogoMips is below 4000.
+	 * Let's scale up the threshold up to 1 second if BogoMips is below 4000.
 	 * This is by no means accurate but it doesn't have to be. The mechanism
 	 * is still useful even when the threshold is fully scaled up. Also, as
 	 * the reports would usually be applicable to everyone, some machines
diff --git a/lib/842/842_decompress.c b/lib/842/842_decompress.c
index 582085ef8b49c0..45a9815abd637f 100644
--- a/lib/842/842_decompress.c
+++ b/lib/842/842_decompress.c
@@ -165,6 +165,9 @@ static int __do_index(struct sw842_param *p, u8 size, u8 bits, u64 fsize)
 	u64 index, offset, total = round_down(p->out - p->ostart, 8);
 	int ret;
 
+	if (size > p->olen)
+		return -ENOSPC;
+
 	ret = next_bits(p, &index, bits);
 	if (ret)
 		return ret;
@@ -306,7 +309,7 @@ int sw842_decompress(const u8 *in, unsigned int ilen,
 			if (ret)
 				return ret;
 
-			if (p.out == out) /* no previous bytes */
+			if (p.out - p.ostart < 8) /* no complete previous block */
 				return -EINVAL;
 
 			/* copy rep + 1 */
@@ -344,6 +347,8 @@ int sw842_decompress(const u8 *in, unsigned int ilen,
 
 			if (!bytes || bytes > SHORT_DATA_BITS_MAX)
 				return -EINVAL;
+			if (bytes > p.olen)
+				return -ENOSPC;
 
 			while (bytes-- > 0) {
 				ret = next_bits(&p, &tmp, 8);
diff --git a/lib/Kconfig b/lib/Kconfig
index 4e6b34c3346d56..1e42f5167c68bc 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -219,29 +219,29 @@ source "lib/xz/Kconfig"
 #
 config DECOMPRESS_GZIP
 	select ZLIB_INFLATE
-	tristate
+	bool
 
 config DECOMPRESS_BZIP2
-	tristate
+	bool
 
 config DECOMPRESS_LZMA
-	tristate
+	bool
 
 config DECOMPRESS_XZ
 	select XZ_DEC
-	tristate
+	bool
 
 config DECOMPRESS_LZO
 	select LZO_DECOMPRESS
-	tristate
+	bool
 
 config DECOMPRESS_LZ4
 	select LZ4_DECOMPRESS
-	tristate
+	bool
 
 config DECOMPRESS_ZSTD
 	select ZSTD_DECOMPRESS
-	tristate
+	bool
 
 #
 # Generic allocator support is selected if needed
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 134b15a44625e8..b924befcaccdaa 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -649,7 +649,7 @@ menu "Generic Kernel Debugging Instruments"
 
 config MAGIC_SYSRQ
 	bool "Magic SysRq key"
-	depends on !UML
+	depends on !UML || MCONSOLE
 	help
 	  If you say Y here, you will have some control over the system even
 	  if the system crashes for example during kernel debugging (e.g., you
@@ -657,9 +657,11 @@ config MAGIC_SYSRQ
 	  immediately or dump some status information). This is accomplished
 	  by pressing various keys while holding SysRq (Alt+PrintScreen). It
 	  also works on a serial console (on PC hardware at least), if you
-	  send a BREAK and then within 5 seconds a command keypress. The
-	  keys are documented in <file:Documentation/admin-guide/sysrq.rst>.
-	  Don't say Y unless you really know what this hack does.
+	  send a BREAK and then within 5 seconds a command keypress. On UML,
+	  this is accomplished by sending a "sysrq" command with mconsole,
+	  followed by the letter for the requested command. The keys are
+	  documented in <file:Documentation/admin-guide/sysrq.rst>. Don't say
+	  Y unless you really know what this hack does.
 
 config MAGIC_SYSRQ_DEFAULT_ENABLE
 	hex "Enable magic SysRq key functions by default"
@@ -1329,6 +1331,18 @@ config WQ_CPU_INTENSIVE_REPORT
 	  triggering likely indicates that the work item should be switched
 	  to use an unbound workqueue.
 
+config TEST_CSD_LOCK
+	tristate "Test module to stall a CPU on a CSD lock"
+	depends on m
+	depends on CSD_LOCK_WAIT_DEBUG
+	help
+	  This builds the "test_csd_lock" module, which keeps one CPU from
+	  answering an IPI for as long as its stall_ms parameter says, so
+	  that the CSD-lock debug code has a stall to report.  It needs
+	  csdlock_debug=1 to be of any use.
+
+	  If unsure, say N.
+
 config TEST_LOCKUP
 	tristate "Test module to generate lockups"
 	depends on m
@@ -2225,6 +2239,21 @@ menuconfig RUNTIME_TESTING_MENU
 
 if RUNTIME_TESTING_MENU
 
+config 842_DECOMPRESS_KUNIT_TEST
+	tristate "KUnit tests for the 842 decompressor" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	depends on 842_DECOMPRESS
+	default KUNIT_ALL_TESTS
+	help
+	  Enable stream-validation and boundary tests for the software 842
+	  decompressor. The tests exercise indexed copies, short data and
+	  repeat operations at valid and invalid output or history boundaries.
+
+	  For more information on KUnit and unit tests in general, refer to
+	  Documentation/dev-tools/kunit/.
+
+	  If unsure, say N.
+
 config TEST_DHRY
 	tristate "Dhrystone benchmark test"
 	help
@@ -2776,7 +2805,6 @@ config RESOURCE_KUNIT_TEST
 	tristate "KUnit test for resource API" if !KUNIT_ALL_TESTS
 	depends on KUNIT
 	default KUNIT_ALL_TESTS
-	select GET_FREE_REGION
 	help
 	  This builds the resource API unit test.
 	  Tests the logic of API provided by resource.c and ioport.h.
@@ -2797,6 +2825,21 @@ config SYSCTL_KUNIT_TEST
 
 	  If unsure, say N.
 
+config ERRSEQ_KUNIT_TEST
+	tristate "KUnit test for errseq" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  This builds the errseq KUnit test suite.
+	  It tests the documented semantics of the errseq_t error-tracking
+	  infrastructure (lib/errseq.c), which underpins writeback error
+	  reporting.
+
+	  For more information on KUnit and unit tests in general please refer
+	  to the KUnit documentation in Documentation/dev-tools/kunit/.
+
+	  If unsure, say N.
+
 config KFIFO_KUNIT_TEST
 	tristate "KUnit Test for the generic kernel FIFO implementation" if !KUNIT_ALL_TESTS
 	depends on KUNIT
diff --git a/lib/Makefile b/lib/Makefile
index dfab958327c5cc..43421c39d21bbf 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -100,6 +100,7 @@ obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o
 obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o
 obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o
 obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o
+obj-$(CONFIG_TEST_CSD_LOCK) += test_csd_lock.o
 obj-$(CONFIG_TEST_LOCKUP) += test_lockup.o
 obj-$(CONFIG_TEST_HMM) += test_hmm.o
 obj-$(CONFIG_TEST_FREE_PAGES) += test_free_pages.o
diff --git a/lib/assoc_array.c b/lib/assoc_array.c
index b6c9723e12ced5..841dfe07dc962a 100644
--- a/lib/assoc_array.c
+++ b/lib/assoc_array.c
@@ -1210,8 +1210,22 @@ struct assoc_array_edit *assoc_array_delete(struct assoc_array *array,
 				goto enomem;
 			edit->new_meta[0] = assoc_array_node_to_ptr(new_n0);
 
-			new_n0->back_pointer = node->back_pointer;
-			new_n0->parent_slot = node->parent_slot;
+			/* A shortcut above a leaf-only node is redundant.  Drop it as
+			 * GC does so that a later split can't create two shortcuts in a row.
+			 */
+			ptr = node->back_pointer;
+			if (assoc_array_ptr_is_shortcut(ptr)) {
+				struct assoc_array_shortcut *s =
+					assoc_array_ptr_to_shortcut(ptr);
+
+				new_n0->back_pointer = s->back_pointer;
+				new_n0->parent_slot = s->parent_slot;
+				edit->excised_subtree = ptr;
+			} else {
+				new_n0->back_pointer = ptr;
+				new_n0->parent_slot = node->parent_slot;
+				edit->excised_subtree = assoc_array_node_to_ptr(node);
+			}
 			new_n0->nr_leaves_on_branch = node->nr_leaves_on_branch;
 			edit->adjust_count_on = new_n0;
 
@@ -1225,21 +1239,15 @@ struct assoc_array_edit *assoc_array_delete(struct assoc_array *array,
 			pr_devel("collapsed %d,%lu\n", collapse.slot, new_n0->nr_leaves_on_branch);
 			BUG_ON(collapse.slot != new_n0->nr_leaves_on_branch - 1);
 
-			if (!node->back_pointer) {
+			if (!new_n0->back_pointer) {
 				edit->set[1].ptr = &array->root;
-			} else if (assoc_array_ptr_is_leaf(node->back_pointer)) {
-				BUG();
-			} else if (assoc_array_ptr_is_node(node->back_pointer)) {
-				struct assoc_array_node *p =
-					assoc_array_ptr_to_node(node->back_pointer);
-				edit->set[1].ptr = &p->slots[node->parent_slot];
-			} else if (assoc_array_ptr_is_shortcut(node->back_pointer)) {
-				struct assoc_array_shortcut *s =
-					assoc_array_ptr_to_shortcut(node->back_pointer);
-				edit->set[1].ptr = &s->next_node;
+			} else {
+				struct assoc_array_node *p;
+
+				p = assoc_array_ptr_to_node(new_n0->back_pointer);
+				edit->set[1].ptr = &p->slots[new_n0->parent_slot];
 			}
 			edit->set[1].to = assoc_array_node_to_ptr(new_n0);
-			edit->excised_subtree = assoc_array_node_to_ptr(node);
 		}
 	}
 
diff --git a/lib/bitmap-str.c b/lib/bitmap-str.c
index dd9aa0635fa561..cafd6388892f06 100644
--- a/lib/bitmap-str.c
+++ b/lib/bitmap-str.c
@@ -8,6 +8,7 @@
 #include <linux/hex.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
+#include <linux/overflow.h>
 #include <linux/string.h>
 
 #include "kstrtox.h"
@@ -186,10 +187,12 @@ struct region {
 
 static void bitmap_set_region(const struct region *r, unsigned long *bitmap)
 {
-	unsigned int start;
+	unsigned int start = r->start;
 
-	for (start = r->start; start <= r->end; start += r->group_len)
+	do {
 		bitmap_set(bitmap, start, min(r->end - start + 1, r->off));
+	} while (!check_add_overflow(start, r->group_len, &start) &&
+		 start <= r->end);
 }
 
 static int bitmap_check_region(const struct region *r)
@@ -299,7 +302,14 @@ static const char *bitmap_parse_region(const char *str, struct region *r)
 	if (*str != '/')
 		return ERR_PTR(-EINVAL);
 
-	return bitmap_getnum(str + 1, &r->group_len, lastbit);
+	str = bitmap_getnum(str + 1, &r->group_len, lastbit);
+	if (IS_ERR(str))
+		return str;
+
+	if (!end_of_region(*str))
+		return ERR_PTR(-EINVAL);
+
+	return end_of_str(*str) ? NULL : str;
 
 no_end:
 	r->end = r->start;
@@ -414,8 +424,12 @@ static const char *bitmap_get_x32_reverse(const char *start,
 			goto out;
 	}
 
-	if (hex_to_bin(*end--) >= 0)
-		return ERR_PTR(-EOVERFLOW);
+	/*
+	 * Eight digits have been consumed and the next character is not a
+	 * separator: another hex digit means the chunk does not fit in 32
+	 * bits, anything else is an illegal character.
+	 */
+	return ERR_PTR(hex_to_bin(*end) >= 0 ? -EOVERFLOW : -EINVAL);
 out:
 	*num = ret;
 	return end;
diff --git a/lib/bitmap.c b/lib/bitmap.c
index ed685127a1073b..d1cb8a507c6017 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -308,6 +308,23 @@ bool __bitmap_intersects(const unsigned long *bitmap1,
 }
 EXPORT_SYMBOL(__bitmap_intersects);
 
+bool __bitmap_intersects_and(const unsigned long *bitmap1,
+			     const unsigned long *bitmap2,
+			     const unsigned long *bitmap3, unsigned int bits)
+{
+	unsigned int k, lim = bits / BITS_PER_LONG;
+
+	for (k = 0; k < lim; ++k)
+		if (bitmap1[k] & bitmap2[k] & bitmap3[k])
+			return true;
+
+	if (bits % BITS_PER_LONG)
+		if ((bitmap1[k] & bitmap2[k] & bitmap3[k]) & BITMAP_LAST_WORD_MASK(bits))
+			return true;
+	return false;
+}
+EXPORT_SYMBOL(__bitmap_intersects_and);
+
 bool __bitmap_subset(const unsigned long *bitmap1,
 		     const unsigned long *bitmap2, unsigned int bits)
 {
diff --git a/lib/bootconfig.c b/lib/bootconfig.c
index 89c88e359179f0..e4650bee8f49c2 100644
--- a/lib/bootconfig.c
+++ b/lib/bootconfig.c
@@ -187,12 +187,9 @@ static inline void * __init xbc_alloc_mem(size_t size)
 	return memblock_alloc(size, SMP_CACHE_BYTES);
 }
 
-static inline void __init xbc_free_mem(void *addr, size_t size, bool early)
+static inline void __init xbc_free_mem(void *addr, size_t size)
 {
-	if (early)
-		memblock_free(addr, size);
-	else if (addr)
-		memblock_free(addr, size);
+	memblock_free(addr, size);
 }
 
 #else /* !__KERNEL__ */
@@ -202,7 +199,7 @@ static inline void *xbc_alloc_mem(size_t size)
 	return calloc(1, size);
 }
 
-static inline void xbc_free_mem(void *addr, size_t size, bool early)
+static inline void xbc_free_mem(void *addr, size_t size)
 {
 	free(addr);
 }
@@ -836,7 +833,6 @@ static int __init xbc_parse_array(char **__v)
 			return -ENOMEM;
 		*__v = next;
 	} while (c == ',');
-	node->child = 0;
 
 	return c;
 }
@@ -1003,9 +999,30 @@ static int __init xbc_close_brace(char **k, char *n)
 	return __xbc_close_brace(n - 1);
 }
 
+#ifndef __KERNEL__
+/* Sanity check for regression: node indices must be within bounds */
+static int __init xbc_sanity_check_tree(void)
+{
+	int i;
+
+	for (i = 0; i < xbc_node_num; i++) {
+		if (xbc_nodes[i].next >= xbc_node_num) {
+			return xbc_parse_error("No closing brace",
+				xbc_node_get_data(xbc_nodes + i));
+		}
+		if (xbc_nodes[i].child >= xbc_node_num) {
+			return xbc_parse_error("Broken child node",
+				xbc_node_get_data(xbc_nodes + i));
+		}
+	}
+
+	return 0;
+}
+#endif
+
 static int __init xbc_verify_tree(void)
 {
-	int i, depth;
+	int depth;
 	size_t len, wlen;
 	struct xbc_node *n, *m;
 
@@ -1022,17 +1039,6 @@ static int __init xbc_verify_tree(void)
 		return -ENOENT;
 	}
 
-	for (i = 0; i < xbc_node_num; i++) {
-		if (xbc_nodes[i].next >= xbc_node_num) {
-			return xbc_parse_error("No closing brace",
-				xbc_node_get_data(xbc_nodes + i));
-		}
-		if (xbc_nodes[i].child >= xbc_node_num) {
-			return xbc_parse_error("Broken child node",
-				xbc_node_get_data(xbc_nodes + i));
-		}
-	}
-
 	/* Key tree limitation check */
 	n = &xbc_nodes[0];
 	depth = 1;
@@ -1119,24 +1125,30 @@ static int __init xbc_parse_tree(void)
 		}
 	} while (!ret);
 
+	if (!ret) {
+		while (p < xbc_data + xbc_data_size - 1 && *p == '\0')
+			p++;
+		if (p < xbc_data + xbc_data_size - 1)
+			ret = xbc_parse_error("Unexpected data after null character", p);
+	}
+
 	return ret;
 }
 
 /**
- * _xbc_exit() - Clean up all parsed bootconfig
- * @early: Set true if this is called before budy system is initialized.
+ * xbc_exit() - Clean up all parsed bootconfig
  *
  * This clears all data structures of parsed bootconfig on memory.
  * If you need to reuse xbc_init() with new boot config, you can
  * use this.
  */
-void __init _xbc_exit(bool early)
+void __init xbc_exit(void)
 {
-	xbc_free_mem(xbc_data, xbc_data_size, early);
+	xbc_free_mem(xbc_data, xbc_data_size);
 	xbc_data = NULL;
 	xbc_data_size = 0;
 	xbc_node_num = 0;
-	xbc_free_mem(xbc_nodes, sizeof(struct xbc_node) * XBC_NODE_MAX, early);
+	xbc_free_mem(xbc_nodes, sizeof(struct xbc_node) * XBC_NODE_MAX);
 	xbc_nodes = NULL;
 	brace_index = 0;
 }
@@ -1189,20 +1201,24 @@ int __init xbc_init(const char *data, size_t size, const char **emsg, int *epos)
 	if (!xbc_nodes) {
 		if (emsg)
 			*emsg = "Failed to allocate bootconfig nodes";
-		_xbc_exit(true);
+		xbc_exit();
 		return -ENOMEM;
 	}
 
 	ret = xbc_parse_tree();
 	if (!ret)
 		ret = xbc_verify_tree();
+#ifndef __KERNEL__
+	if (!ret)
+		ret = xbc_sanity_check_tree();
+#endif
 
 	if (ret < 0) {
 		if (epos)
 			*epos = xbc_err_pos;
 		if (emsg)
 			*emsg = xbc_err_msg;
-		_xbc_exit(true);
+		xbc_exit();
 	} else {
 		ret = xbc_node_num;
 	}
diff --git a/lib/crc/x86/crc-pclmul-template.h b/lib/crc/x86/crc-pclmul-template.h
index 02744831c6fac0..893119bb7c0771 100644
--- a/lib/crc/x86/crc-pclmul-template.h
+++ b/lib/crc/x86/crc-pclmul-template.h
@@ -27,16 +27,14 @@ DEFINE_STATIC_CALL(prefix##_pclmul, prefix##_pclmul_sse)
 static inline bool have_vpclmul(void)
 {
 	return boot_cpu_has(X86_FEATURE_VPCLMULQDQ) &&
-	       boot_cpu_has(X86_FEATURE_AVX2) &&
-	       cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL);
+	       boot_cpu_has(X86_FEATURE_AVX2);
 }
 
 static inline bool have_avx512(void)
 {
 	return boot_cpu_has(X86_FEATURE_AVX512BW) &&
 	       boot_cpu_has(X86_FEATURE_AVX512VL) &&
-	       !boot_cpu_has(X86_FEATURE_PREFER_YMM) &&
-	       cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL);
+	       !boot_cpu_has(X86_FEATURE_PREFER_YMM);
 }
 
 /*
diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c
index f1549839b3de0c..2d29adca795328 100644
--- a/lib/crypto/aes.c
+++ b/lib/crypto/aes.c
@@ -539,7 +539,7 @@ static void __init aes_fips_test(void)
 	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
 		panic("aes: FIPS self-test failed (wrong plaintext)\n");
 
-	memzero_explicit(&key, sizeof(key));
+	aes_zeroize_key(&key);
 }
 
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
@@ -827,7 +827,7 @@ static void __init aes_ecb_fips_test(void)
 	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
 		panic("aes: ECB FIPS self-test failed (wrong plaintext)\n");
 
-	memzero_explicit(&key, sizeof(key));
+	aes_zeroize_key(&key);
 }
 #else /* CONFIG_CRYPTO_LIB_AES_ECB */
 static inline void aes_ecb_fips_test(void)
@@ -1040,7 +1040,7 @@ static void __init aes_cbc_fips_test(void)
 	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
 		panic("aes: CBC FIPS self-test failed (wrong plaintext)\n");
 
-	memzero_explicit(&key, sizeof(key));
+	aes_zeroize_key(&key);
 }
 
 /* FIPS cryptographic algorithm self-test for AES-CBC-CTS */
@@ -1069,7 +1069,7 @@ static void __init aes_cbc_cts_fips_test(void)
 	if (memcmp(ptext, data, data_len) != 0)
 		panic("aes: CBC-CTS FIPS self-test failed (wrong plaintext)\n");
 
-	memzero_explicit(&key, sizeof(key));
+	aes_zeroize_key(&key);
 }
 #else /* CONFIG_CRYPTO_LIB_AES_CBC */
 static inline void aes_cbc_fips_test(void)
@@ -1194,7 +1194,7 @@ static void __init aes_ctr_fips_test(void)
 	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
 		panic("aes: CTR FIPS self-test failed (wrong plaintext)\n");
 
-	memzero_explicit(&key, sizeof(key));
+	aes_zeroize_enckey(&key);
 }
 #else /* CONFIG_CRYPTO_LIB_AES_CTR */
 static inline void aes_ctr_fips_test(void)
@@ -1223,7 +1223,7 @@ int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key,
 	return 0;
 
 out_zeroize:
-	memzero_explicit(key, sizeof(*key));
+	aes_xts_zeroize_key(key);
 	return err;
 }
 EXPORT_SYMBOL_GPL(aes_xts_preparekey);
@@ -1670,7 +1670,7 @@ void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag)
 	ghash_final(&ctx->ghash, ctx->ctr); /* Use ctr as temp buffer */
 
 	crypto_xor_cpy(authtag, ctx->ctr, ctx->j0_enc, ctx->key->authtag_len);
-	memzero_explicit(ctx, sizeof(*ctx));
+	aes_gcm_zeroize_ctx(ctx);
 }
 EXPORT_SYMBOL_GPL(aes_gcm_encrypt_final);
 
@@ -1697,7 +1697,7 @@ int aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, const u8 *authtag)
 		      -EBADMSG :
 		      0;
 out:
-	memzero_explicit(ctx, sizeof(*ctx));
+	aes_gcm_zeroize_ctx(ctx);
 	return err;
 }
 EXPORT_SYMBOL_GPL(aes_gcm_decrypt_final);
@@ -1742,7 +1742,7 @@ static void __init aes_gcm_fips_test(void)
 {
 	const size_t data_len = sizeof(fips_test_data);
 	u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE];
-	struct aes_gcm_key key;
+	struct aes_gcm_key key __cleanup(aes_gcm_zeroize_key);
 	int err;
 
 	if (aes_gcm_preparekey(&key, fips_test_key, sizeof(fips_test_key),
@@ -1760,8 +1760,6 @@ static void __init aes_gcm_fips_test(void)
 		panic("aes: GCM FIPS self-test failed (decryption failed)\n");
 	if (memcmp(fips_test_data, buf, data_len) != 0)
 		panic("aes: GCM FIPS self-test failed (wrong plaintext)\n");
-
-	memzero_explicit(&key, sizeof(key));
 }
 #else /* CONFIG_CRYPTO_LIB_AES_GCM */
 static inline void aes_gcm_fips_test(void)
@@ -2013,7 +2011,7 @@ void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag)
 	if (ctx->partial_len)
 		aes_encrypt(&ctx->key->aes, ctx->mac, ctx->mac);
 	crypto_xor_cpy(authtag, ctx->mac, ctx->s0, ctx->key->authtag_len);
-	memzero_explicit(ctx, sizeof(*ctx));
+	aes_ccm_zeroize_ctx(ctx);
 }
 EXPORT_SYMBOL_GPL(aes_ccm_encrypt_final);
 
@@ -2034,7 +2032,7 @@ int aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, const u8 *authtag)
 		      -EBADMSG :
 		      0;
 out:
-	memzero_explicit(ctx, sizeof(*ctx));
+	aes_ccm_zeroize_ctx(ctx);
 	return err;
 }
 EXPORT_SYMBOL_GPL(aes_ccm_decrypt_final);
@@ -2086,7 +2084,7 @@ static void __init aes_ccm_fips_test(void)
 	const size_t data_len = sizeof(fips_test_data);
 	const size_t nonce_len = 13;
 	u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE];
-	struct aes_ccm_key key;
+	struct aes_ccm_key key __cleanup(aes_ccm_zeroize_key);
 	int err;
 
 	if (aes_ccm_preparekey(&key, fips_test_key, sizeof(fips_test_key),
@@ -2108,8 +2106,6 @@ static void __init aes_ccm_fips_test(void)
 		panic("aes: CCM FIPS self-test failed (decryption failed)\n");
 	if (memcmp(fips_test_data, buf, data_len) != 0)
 		panic("aes: CCM FIPS self-test failed (wrong plaintext)\n");
-
-	memzero_explicit(&key, sizeof(key));
 }
 #else /* CONFIG_CRYPTO_LIB_AES_CCM */
 static inline void aes_ccm_fips_test(void)
diff --git a/lib/crypto/blake2b.c b/lib/crypto/blake2b.c
index 581b7f8486fae8..55d6c437f311ff 100644
--- a/lib/crypto/blake2b.c
+++ b/lib/crypto/blake2b.c
@@ -148,7 +148,7 @@ void blake2b_final(struct blake2b_ctx *ctx, u8 *out)
 	blake2b_compress(ctx, ctx->buf, 1, ctx->buflen);
 	cpu_to_le64_array(ctx->h, ARRAY_SIZE(ctx->h));
 	memcpy(out, ctx->h, ctx->outlen);
-	memzero_explicit(ctx, sizeof(*ctx));
+	blake2b_zeroize_ctx(ctx);
 }
 EXPORT_SYMBOL(blake2b_final);
 
diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c
index 71578a08474233..24f7f34334b010 100644
--- a/lib/crypto/blake2s.c
+++ b/lib/crypto/blake2s.c
@@ -142,7 +142,7 @@ void blake2s_final(struct blake2s_ctx *ctx, u8 *out)
 	blake2s_compress(ctx, ctx->buf, 1, ctx->buflen);
 	cpu_to_le32_array(ctx->h, ARRAY_SIZE(ctx->h));
 	memcpy(out, ctx->h, ctx->outlen);
-	memzero_explicit(ctx, sizeof(*ctx));
+	blake2s_zeroize_ctx(ctx);
 }
 EXPORT_SYMBOL(blake2s_final);
 
diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c
index 3d2b017a0525ae..a8ee57600012d5 100644
--- a/lib/crypto/md5.c
+++ b/lib/crypto/md5.c
@@ -271,7 +271,7 @@ void hmac_md5_final(struct hmac_md5_ctx *ctx, u8 out[MD5_DIGEST_SIZE])
 	cpu_to_le32_array(ctx->ostate.h, ARRAY_SIZE(ctx->ostate.h));
 	memcpy(out, ctx->ostate.h, MD5_DIGEST_SIZE);
 
-	memzero_explicit(ctx, sizeof(*ctx));
+	hmac_md5_zeroize_ctx(ctx);
 }
 EXPORT_SYMBOL_GPL(hmac_md5_final);
 
diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c
index b687b89d97cb44..c4361ef77166ef 100644
--- a/lib/crypto/sha1.c
+++ b/lib/crypto/sha1.c
@@ -275,7 +275,7 @@ void hmac_sha1_final(struct hmac_sha1_ctx *ctx, u8 out[SHA1_DIGEST_SIZE])
 	for (size_t i = 0; i < SHA1_DIGEST_SIZE; i += 4)
 		put_unaligned_be32(ctx->ostate.h[i / 4], out + i);
 
-	memzero_explicit(ctx, sizeof(*ctx));
+	hmac_sha1_zeroize_ctx(ctx);
 }
 EXPORT_SYMBOL_GPL(hmac_sha1_final);
 
diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c
index b02b8a247adf23..23059347b4493d 100644
--- a/lib/crypto/sm3.c
+++ b/lib/crypto/sm3.c
@@ -258,7 +258,7 @@ static void __sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE])
 void sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE])
 {
 	__sm3_final(ctx, out);
-	memzero_explicit(ctx, sizeof(*ctx));
+	sm3_zeroize_ctx(ctx);
 }
 EXPORT_SYMBOL_GPL(sm3_final);
 
diff --git a/lib/crypto/x86/blake2s.h b/lib/crypto/x86/blake2s.h
index f8eed6cb042e4b..0f7c51f055c8fc 100644
--- a/lib/crypto/x86/blake2s.h
+++ b/lib/crypto/x86/blake2s.h
@@ -55,8 +55,6 @@ static void blake2s_mod_init_arch(void)
 	if (boot_cpu_has(X86_FEATURE_AVX) &&
 	    boot_cpu_has(X86_FEATURE_AVX2) &&
 	    boot_cpu_has(X86_FEATURE_AVX512F) &&
-	    boot_cpu_has(X86_FEATURE_AVX512VL) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
-			      XFEATURE_MASK_AVX512, NULL))
+	    boot_cpu_has(X86_FEATURE_AVX512VL))
 		static_branch_enable(&blake2s_use_avx512);
 }
diff --git a/lib/crypto/x86/chacha.h b/lib/crypto/x86/chacha.h
index 10cf8f1c569dca..c79562aac56b67 100644
--- a/lib/crypto/x86/chacha.h
+++ b/lib/crypto/x86/chacha.h
@@ -165,8 +165,7 @@ static void chacha_mod_init_arch(void)
 	static_branch_enable(&chacha_use_simd);
 
 	if (boot_cpu_has(X86_FEATURE_AVX) &&
-	    boot_cpu_has(X86_FEATURE_AVX2) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
+	    boot_cpu_has(X86_FEATURE_AVX2)) {
 		static_branch_enable(&chacha_use_avx2);
 
 		if (boot_cpu_has(X86_FEATURE_AVX512VL) &&
diff --git a/lib/crypto/x86/nh.h b/lib/crypto/x86/nh.h
index 83361c2e97838d..342636dcb750f4 100644
--- a/lib/crypto/x86/nh.h
+++ b/lib/crypto/x86/nh.h
@@ -37,9 +37,7 @@ static void nh_mod_init_arch(void)
 {
 	if (boot_cpu_has(X86_FEATURE_XMM2)) {
 		static_branch_enable(&have_sse2);
-		if (boot_cpu_has(X86_FEATURE_AVX2) &&
-		    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				      NULL))
+		if (boot_cpu_has(X86_FEATURE_AVX2))
 			static_branch_enable(&have_avx2);
 	}
 }
diff --git a/lib/crypto/x86/poly1305.h b/lib/crypto/x86/poly1305.h
index ee92e3740a7876..b061b9926fa5da 100644
--- a/lib/crypto/x86/poly1305.h
+++ b/lib/crypto/x86/poly1305.h
@@ -143,15 +143,12 @@ static void poly1305_emit(const struct poly1305_state *ctx,
 #define poly1305_mod_init_arch poly1305_mod_init_arch
 static void poly1305_mod_init_arch(void)
 {
-	if (boot_cpu_has(X86_FEATURE_AVX) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
+	if (boot_cpu_has(X86_FEATURE_AVX))
 		static_branch_enable(&poly1305_use_avx);
-	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
+	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2))
 		static_branch_enable(&poly1305_use_avx2);
 	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) &&
 	    boot_cpu_has(X86_FEATURE_AVX512F) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) &&
 	    /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */
 	    boot_cpu_data.x86_vfm != INTEL_SKYLAKE_X)
 		static_branch_enable(&poly1305_use_avx512);
diff --git a/lib/crypto/x86/sha1.h b/lib/crypto/x86/sha1.h
index c48a0131fd12c9..6aff433466e7e7 100644
--- a/lib/crypto/x86/sha1.h
+++ b/lib/crypto/x86/sha1.h
@@ -59,9 +59,7 @@ static void sha1_mod_init_arch(void)
 {
 	if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
 		static_call_update(sha1_blocks_x86, sha1_blocks_ni);
-	} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				     NULL) &&
-		   boot_cpu_has(X86_FEATURE_AVX)) {
+	} else if (boot_cpu_has(X86_FEATURE_AVX)) {
 		if (boot_cpu_has(X86_FEATURE_AVX2) &&
 		    boot_cpu_has(X86_FEATURE_BMI1) &&
 		    boot_cpu_has(X86_FEATURE_BMI2))
diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h
index 0ee69d8e39fe85..e98ffdaf4b14f0 100644
--- a/lib/crypto/x86/sha256.h
+++ b/lib/crypto/x86/sha256.h
@@ -104,9 +104,7 @@ static void sha256_mod_init_arch(void)
 		   boot_cpu_has(X86_FEATURE_PHE_EN) &&
 		   boot_cpu_data.x86 >= 0x07) {
 		static_call_update(sha256_blocks_x86, sha256_blocks_phe);
-	} else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
-				     NULL) &&
-		   boot_cpu_has(X86_FEATURE_AVX)) {
+	} else if (boot_cpu_has(X86_FEATURE_AVX)) {
 		if (boot_cpu_has(X86_FEATURE_AVX2) &&
 		    boot_cpu_has(X86_FEATURE_BMI2))
 			static_call_update(sha256_blocks_x86,
diff --git a/lib/crypto/x86/sha512.h b/lib/crypto/x86/sha512.h
index 0213c70cedd01c..4e177b4606bd2f 100644
--- a/lib/crypto/x86/sha512.h
+++ b/lib/crypto/x86/sha512.h
@@ -37,8 +37,7 @@ static void sha512_blocks(struct sha512_block_state *state,
 #define sha512_mod_init_arch sha512_mod_init_arch
 static void sha512_mod_init_arch(void)
 {
-	if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) &&
-	    boot_cpu_has(X86_FEATURE_AVX)) {
+	if (boot_cpu_has(X86_FEATURE_AVX)) {
 		if (boot_cpu_has(X86_FEATURE_AVX2) &&
 		    boot_cpu_has(X86_FEATURE_BMI2))
 			static_call_update(sha512_blocks_x86,
diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h
index 3834780f2f6a36..e06d4a22e4fa76 100644
--- a/lib/crypto/x86/sm3.h
+++ b/lib/crypto/x86/sm3.h
@@ -33,7 +33,6 @@ static void sm3_blocks(struct sm3_block_state *state,
 #define sm3_mod_init_arch sm3_mod_init_arch
 static void sm3_mod_init_arch(void)
 {
-	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) &&
-	    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
+	if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2))
 		static_call_update(sm3_blocks_x86, sm3_blocks_avx);
 }
diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c
index 1288f146661f1c..a9236d3e9678de 100644
--- a/lib/decompress_bunzip2.c
+++ b/lib/decompress_bunzip2.c
@@ -428,6 +428,10 @@ static int INIT get_next_block(struct bunzip_data *bd)
 			t += (runPos << nextSym);
 			/* +runPos if RUNA; +2*runPos if RUNB */
 
+			/* Bound the run so t and runPos cannot overflow. */
+			if (t >= dbufSize)
+				return RETVAL_DATA_ERROR;
+
 			runPos <<= 1;
 			continue;
 		}
@@ -439,7 +443,7 @@ static int INIT get_next_block(struct bunzip_data *bd)
 		   array.) */
 		if (runPos) {
 			runPos = 0;
-			if (dbufCount+t >= dbufSize)
+			if (dbufCount+t > dbufSize)
 				return RETVAL_DATA_ERROR;
 
 			uc = symToByte[mtfSymbol[0]];
diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index c0dbb3cea915eb..86e9aaec04f6d3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -139,6 +139,10 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
 		if (!fill) {
 			inp += 4;
 			size -= 4;
+			if (chunksize > size) {
+				error("data corrupted");
+				goto exit_2;
+			}
 		} else {
 			if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
 				error("chunk length is longer than allocated");
diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c
index 05d5cb490a44e7..9ccded9934c667 100644
--- a/lib/decompress_unxz.c
+++ b/lib/decompress_unxz.c
@@ -342,13 +342,13 @@ STATIC int INIT unxz(unsigned char *in, long in_size,
 				b.out_pos = 0;
 			}
 		} while (ret == XZ_OK);
+	}
 
-		if (must_free_in)
-			free(in);
+	if (must_free_in)
+		free(in);
 
-		if (flush != NULL)
-			free(b.out);
-	}
+	if (flush != NULL)
+		free(b.out);
 
 	if (in_used != NULL)
 		*in_used += b.in_pos;
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index 18a71a9108d3e5..49334d1aa4b3af 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -1442,26 +1442,29 @@ static int __init dynamic_debug_init(void)
 	i = mod_sites = mod_ct = 0;
 
 	for (; iter < __stop___dyndbg; iter++, i++, mod_sites++) {
-
 		if (strcmp(modname, iter->modname)) {
-			mod_ct++;
-			di.num_descs = mod_sites;
 			di.descs = iter_mod_start;
+			di.num_descs = mod_sites;
 			ret = ddebug_add_module(&di, modname);
 			if (ret)
 				goto out_err;
 
-			mod_sites = 0;
-			modname = iter->modname;
+			mod_ct++;
+
 			iter_mod_start = iter;
+			modname = iter->modname;
+			mod_sites = 0;
 		}
 	}
-	di.num_descs = mod_sites;
+
 	di.descs = iter_mod_start;
+	di.num_descs = mod_sites;
 	ret = ddebug_add_module(&di, modname);
 	if (ret)
 		goto out_err;
 
+	mod_ct++;
+
 	ddebug_init_success = 1;
 	vpr_info("%d prdebugs in %d modules, %d KiB in ddebug tables, %d kiB in __dyndbg section\n",
 		 i, mod_ct, (int)((mod_ct * sizeof(struct ddebug_table)) >> 10),
diff --git a/lib/fault-inject.c b/lib/fault-inject.c
index 999053fa133e3f..02916ef2761c60 100644
--- a/lib/fault-inject.c
+++ b/lib/fault-inject.c
@@ -5,6 +5,7 @@
 #include <linux/debugfs.h>
 #include <linux/sched.h>
 #include <linux/stat.h>
+#include <linux/string.h>
 #include <linux/types.h>
 #include <linux/fs.h>
 #include <linux/export.h>
@@ -64,7 +65,7 @@ static void fail_dump(struct fault_attr *attr)
 {
 	if (attr->verbose > 0 && __ratelimit(&attr->ratelimit_state)) {
 		printk(KERN_NOTICE "FAULT_INJECTION: forcing a failure.\n"
-		       "name %pd, interval %lu, probability %lu, "
+		       "name %s, interval %lu, probability %lu, "
 		       "space %d, times %d\n", attr->dname,
 		       attr->interval, attr->probability,
 		       atomic_read(&attr->space),
@@ -261,7 +262,9 @@ struct dentry *fault_create_debugfs_attr(const char *name,
 	debugfs_create_xul("reject-end", mode, dir, &attr->reject_end);
 #endif /* CONFIG_FAULT_INJECTION_STACKTRACE_FILTER */
 
-	attr->dname = dget(dir);
+	if (strscpy(attr->dname, name, sizeof(attr->dname)) == -E2BIG)
+		pr_warn("FAULT_INJECTION: name '%s' truncated to '%s'\n",
+			name, attr->dname);
 	return dir;
 }
 EXPORT_SYMBOL_GPL(fault_create_debugfs_attr);
diff --git a/lib/group_cpus.c b/lib/group_cpus.c
index e6e18d7a49bba6..3c2229feb9c3d2 100644
--- a/lib/group_cpus.c
+++ b/lib/group_cpus.c
@@ -6,6 +6,7 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/cpu.h>
+#include <linux/mutex.h>
 #include <linux/sort.h>
 #include <linux/group_cpus.h>
 
@@ -286,6 +287,77 @@ static void assign_cpus_to_groups(unsigned int ncpus,
 	}
 }
 
+/*
+ * topology_cluster_cpumask() only lists the cluster siblings that are online,
+ * so group_cpus_evenly() would compute a different managed-IRQ partition when
+ * recomputed with CPUs offline (e.g. an NVMe reset across s2idle), steering a
+ * queue's IRQ away from the CPU it serves.
+ *
+ * Snapshot the cluster masks once, on the first spread seen with every
+ * present CPU online, and reuse it so the grouping stays stable. If no
+ * snapshot exists (partial boot via maxcpus=/nosmp, or allocation failure)
+ * the cluster path is skipped and the plain present/possible spread is
+ * used. Only the cluster path is stabilised; grp_spread_init_one()'s
+ * sibling mask is unchanged. The snapshot lives for the system lifetime
+ * and is not refreshed for CPUs hot-added after boot.
+ */
+static cpumask_var_t *cluster_snapshot;
+static bool cluster_snapshot_ready;
+static DEFINE_MUTEX(cluster_snapshot_lock);
+
+static void capture_cluster_snapshot(void)
+{
+	cpumask_var_t *snapshot;
+	unsigned int cpu;
+
+	/* Pairs with the smp_store_release() below. */
+	if (smp_load_acquire(&cluster_snapshot_ready))
+		return;
+
+	/* Only capture when all present CPUs are online. */
+	if (!data_race(cpumask_equal(cpu_present_mask, cpu_online_mask)))
+		return;
+
+	mutex_lock(&cluster_snapshot_lock);
+	if (cluster_snapshot_ready)
+		goto out;
+
+	snapshot = kcalloc(nr_cpu_ids, sizeof(*snapshot), GFP_KERNEL);
+	if (!snapshot)
+		goto out;
+
+	for_each_possible_cpu(cpu)
+		if (!zalloc_cpumask_var(&snapshot[cpu], GFP_KERNEL))
+			goto free_snapshot;
+
+	/* Trylock: a caller may hold a lock the hotplug writer needs. */
+	if (!cpus_read_trylock())
+		goto free_snapshot;
+
+	/* Recheck under the lock, which also pins the cluster masks. */
+	if (!data_race(cpumask_equal(cpu_present_mask, cpu_online_mask))) {
+		cpus_read_unlock();
+		goto free_snapshot;
+	}
+
+	for_each_possible_cpu(cpu)
+		cpumask_copy(snapshot[cpu], topology_cluster_cpumask(cpu));
+	cpus_read_unlock();
+
+	cluster_snapshot = snapshot;
+	/* Publish the filled snapshot before the ready flag. */
+	smp_store_release(&cluster_snapshot_ready, true);
+	goto out;
+
+free_snapshot:
+	/* Unallocated entries are NULL, which free_cpumask_var() ignores. */
+	for_each_possible_cpu(cpu)
+		free_cpumask_var(snapshot[cpu]);
+	kfree(snapshot);
+out:
+	mutex_unlock(&cluster_snapshot_lock);
+}
+
 static int alloc_cluster_groups(unsigned int ncpus,
 				unsigned int ngroups,
 				struct cpumask *node_cpumask,
@@ -299,6 +371,17 @@ static int alloc_cluster_groups(unsigned int ncpus,
 	const struct cpumask **clusters;
 	struct node_groups *cluster_groups;
 
+	/*
+	 * Capture on the first spread with every present CPU online (normally
+	 * the first device probe); later spreads reuse it. Sample the ready
+	 * flag once so both loops below use one consistent source.
+	 */
+	capture_cluster_snapshot();
+
+	/* Pairs with the smp_store_release() in capture_cluster_snapshot(). */
+	if (!smp_load_acquire(&cluster_snapshot_ready))
+		goto no_cluster;
+
 	cpumask_copy(msk, node_cpumask);
 
 	/* Probe how many clusters in this node. */
@@ -307,7 +390,7 @@ static int alloc_cluster_groups(unsigned int ncpus,
 		if (cpu >= nr_cpu_ids)
 			break;
 
-		cluster_mask = topology_cluster_cpumask(cpu);
+		cluster_mask = cluster_snapshot[cpu];
 		if (!cpumask_weight(cluster_mask))
 			goto no_cluster;
 		/* Clean out CPUs on the same cluster. */
@@ -331,7 +414,7 @@ static int alloc_cluster_groups(unsigned int ncpus,
 	cpumask_copy(msk, node_cpumask);
 	for (n = 0; n < ncluster; n++) {
 		cpu = cpumask_first(msk);
-		cluster_mask = topology_cluster_cpumask(cpu);
+		cluster_mask = cluster_snapshot[cpu];
 		nc = cpumask_weight_and(cluster_mask, node_cpumask);
 		clusters[n] = cluster_mask;
 		cluster_groups[n].id = n;
diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c
index b0b07270ce7c3c..06f77fb3179fc8 100644
--- a/lib/interval_tree_test.c
+++ b/lib/interval_tree_test.c
@@ -244,7 +244,7 @@ static int span_iteration_check(void)
 	unsigned long start, last;
 	struct interval_tree_span_iter span, mas_span;
 
-	DEFINE_MTREE(tree);
+	struct maple_tree tree;
 
 	MA_STATE(mas, &tree, 0, 0);
 
diff --git a/lib/klist.c b/lib/klist.c
index 332a4fbf18ff08..f133740b1c2cb7 100644
--- a/lib/klist.c
+++ b/lib/klist.c
@@ -36,6 +36,7 @@
 #include <linux/klist.h>
 #include <linux/export.h>
 #include <linux/sched.h>
+#include <linux/sched/task.h>
 
 /*
  * Use the lowest bit of n_klist to mark deleted nodes and exclude
@@ -187,18 +188,24 @@ static void klist_release(struct kref *kref)
 
 	WARN_ON(!knode_dead(n));
 	list_del(&n->n_node);
+	knode_set_klist(n, NULL);
 	spin_lock(&klist_remove_lock);
 	list_for_each_entry_safe(waiter, tmp, &klist_remove_waiters, list) {
+		struct task_struct *p;
+
 		if (waiter->node != n)
 			continue;
 
+		p = waiter->process;
+		get_task_struct(p);
 		list_del(&waiter->list);
-		waiter->woken = 1;
+		/* Publish only after the final waiter and n accesses */
+		smp_store_release(&waiter->woken, 1);
 		mb();
-		wake_up_process(waiter->process);
+		wake_up_process(p);
+		put_task_struct(p);
 	}
 	spin_unlock(&klist_remove_lock);
-	knode_set_klist(n, NULL);
 }
 
 static int klist_dec_and_del(struct klist_node *n)
@@ -250,7 +257,8 @@ void klist_remove(struct klist_node *n)
 
 	for (;;) {
 		set_current_state(TASK_UNINTERRUPTIBLE);
-		if (waiter.woken)
+		/* Pairs with the release store in klist_release() */
+		if (smp_load_acquire(&waiter.woken))
 			break;
 		schedule();
 	}
diff --git a/lib/kobject_uevent.c b/lib/kobject_uevent.c
index ddbc4d7482d243..1480e2c27ed098 100644
--- a/lib/kobject_uevent.c
+++ b/lib/kobject_uevent.c
@@ -17,7 +17,7 @@
 #include <linux/string.h>
 #include <linux/kobject.h>
 #include <linux/export.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/slab.h>
 #include <linux/socket.h>
 #include <linux/skbuff.h>
@@ -83,7 +83,7 @@ static int kobject_action_type(const char *buf, size_t count,
 	for (action = 0; action < ARRAY_SIZE(kobject_actions); action++) {
 		if (strncmp(kobject_actions[action], buf, count_first) != 0)
 			continue;
-		if (kobject_actions[action][count_first] != '\0')
+		if (strlen(kobject_actions[action]) != count_first)
 			continue;
 		if (args)
 			*args = args_start;
diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig
index 94ff8e4089bfbd..cbda6189a308a0 100644
--- a/lib/kunit/Kconfig
+++ b/lib/kunit/Kconfig
@@ -133,7 +133,7 @@ config KUNIT_DEFAULT_TIMEOUT
 
 config KUNIT_UML_PCI
 	bool "KUnit UML PCI Support"
-	depends on UML
+	depends on UML && MMU
 	select UML_PCI
 	help
 	  Enables the PCI subsystem on UML for use by KUnit tests.
diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
index b0f8a41d61d367..c798a02470d854 100644
--- a/lib/kunit/executor.c
+++ b/lib/kunit/executor.c
@@ -371,7 +371,7 @@ static void kunit_handle_shutdown(void)
 
 }
 
-int kunit_run_all_tests(void)
+void kunit_run_all_tests(void)
 {
 	struct kunit_suite_set suite_set = {NULL, NULL};
 	struct kunit_suite_set filtered_suite_set = {NULL, NULL};
@@ -382,7 +382,7 @@ int kunit_run_all_tests(void)
 		__kunit_suites_start, __kunit_suites_end,
 	};
 	size_t init_num_suites = init_suite_set.end - init_suite_set.start;
-	int err = 0;
+	int err;
 
 	if (init_num_suites > 0) {
 		suite_set = kunit_merge_suite_sets(init_suite_set, normal_suite_set);
@@ -432,7 +432,6 @@ int kunit_run_all_tests(void)
 
 out:
 	kunit_handle_shutdown();
-	return err;
 }
 
 #if IS_BUILTIN(CONFIG_KUNIT_TEST)
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 2891f94a11c654..87b261b87b0bc8 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -328,6 +328,7 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
 				  s64 limit, s64 amount, s32 batch)
 {
 	s64 count;
+	s64 gcount;
 	s64 unknown;
 	unsigned long flags;
 	bool good = false;
@@ -338,11 +339,16 @@ bool __percpu_counter_limited_add(struct percpu_counter *fbc,
 	local_irq_save(flags);
 	unknown = batch * num_online_cpus();
 	count = __this_cpu_read(*fbc->counters);
+	/*
+	 * Lockless on purpose: gcount may be stale, so this is only an
+	 * approximation, bounded by the per-cpu slack ("unknown").
+	 */
+	gcount = data_race(READ_ONCE(fbc->count));
 
 	/* Skip taking the lock when safe */
 	if (abs(count + amount) <= batch &&
-	    ((amount > 0 && fbc->count + unknown <= limit) ||
-	     (amount < 0 && fbc->count - unknown >= limit))) {
+	    ((amount > 0 && gcount + unknown <= limit) ||
+	     (amount < 0 && gcount - unknown >= limit))) {
 		this_cpu_add(*fbc->counters, amount);
 		local_irq_restore(flags);
 		return true;
diff --git a/lib/plist.c b/lib/plist.c
index a5bef38add431d..b0273533c04d2a 100644
--- a/lib/plist.c
+++ b/lib/plist.c
@@ -174,8 +174,15 @@ void plist_requeue(struct plist_node *node, struct plist_head *head)
 	/*
 	 * After plist_del(), iter is the replacement of the node.  If the node
 	 * was on prio_list, take shortcut to find node_next instead of looping.
+	 *
+	 * prio_list is a headless ring, so from the LAST bucket ->next wraps
+	 * round to the first one; in that case node_next is the list head.
 	 */
 	if (!list_empty(&iter->prio_list)) {
+		struct plist_node *first = plist_first(head);
+
+		if (iter->prio_list.next == &first->prio_list)
+			goto queue;
 		iter = list_entry(iter->prio_list.next, struct plist_node,
 				  prio_list);
 		node_next = &iter->node_list;
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
index 01f007b2522cfc..563a178aa930c4 100644
--- a/lib/raid/Kconfig
+++ b/lib/raid/Kconfig
@@ -32,6 +32,7 @@ config XOR_KUNIT_TEST
 config XOR_BENCHMARK
 	bool "Benchmark for xor_gen"
 	depends on XOR_KUNIT_TEST
+	default y if KUNIT_ALL_TESTS=m
 	help
 	  Include benchmarks in the KUnit test suite for xor_gen.
 
@@ -63,6 +64,7 @@ config RAID6_PQ_KUNIT_TEST
 config RAID6_PQ_KUNIT_BENCHMARK
 	bool "Benchmark for RAID6 PQ"
 	depends on RAID6_PQ_KUNIT_TEST
+	default y if KUNIT_ALL_TESTS=m
 	help
 	  Include benchmarks in the KUnit test suite for raid P/Q generation.
 
diff --git a/lib/raid/raid6/x86/avx2.c b/lib/raid/raid6/x86/avx2.c
index 7d829c669ea795..3cc2fe7ac42c57 100644
--- a/lib/raid/raid6/x86/avx2.c
+++ b/lib/raid/raid6/x86/avx2.c
@@ -67,6 +67,7 @@ static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -115,6 +116,7 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -175,6 +177,7 @@ static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -243,6 +246,7 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -334,6 +338,7 @@ static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -444,6 +449,7 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
 		asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
 	}
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
diff --git a/lib/raid/raid6/x86/avx512.c b/lib/raid/raid6/x86/avx512.c
index e671eb5bde63e4..772bfc4af6dfd7 100644
--- a/lib/raid/raid6/x86/avx512.c
+++ b/lib/raid/raid6/x86/avx512.c
@@ -78,6 +78,7 @@ static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -137,6 +138,7 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -208,6 +210,7 @@ static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -292,6 +295,7 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -396,6 +400,7 @@ static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
 	}
 
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -529,6 +534,7 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
 			       "m" (q[d+128]), "m" (q[d+192]));
 	}
 	asm volatile("sfence" : : : "memory");
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 const struct raid6_calls raid6_avx512x4 = {
diff --git a/lib/raid/raid6/x86/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
index a714a780a2d8f6..820871046e3058 100644
--- a/lib/raid/raid6/x86/recov_avx2.c
+++ b/lib/raid/raid6/x86/recov_avx2.c
@@ -176,6 +176,7 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -293,6 +294,7 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
diff --git a/lib/raid/raid6/x86/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
index ec72d5a30c01ef..299a3f044d6162 100644
--- a/lib/raid/raid6/x86/recov_avx512.c
+++ b/lib/raid/raid6/x86/recov_avx512.c
@@ -211,6 +211,7 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
@@ -353,6 +354,7 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
 #endif
 	}
 
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 9b0fad459cdb60..e1e3455c219dee 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -31,7 +31,7 @@ xor-$(CONFIG_SPARC32)		+= sparc/xor-sparc32.o
 xor-$(CONFIG_SPARC64)		+= sparc/xor-sparc64.o sparc/xor-sparc64-glue.o
 xor-$(CONFIG_S390)		+= s390/xor.o
 xor-$(CONFIG_X86_32)		+= x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o
-xor-$(CONFIG_X86_64)		+= x86/xor-avx.o x86/xor-sse.o
+xor-$(CONFIG_X86_64)		+= x86/xor-avx.o x86/xor-sse.o x86/xor-avx512.o
 obj-y				+= tests/
 
 CFLAGS_xor-neon.o		+= $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH)
diff --git a/lib/raid/xor/x86/xor-avx.c b/lib/raid/xor/x86/xor-avx.c
index f7777d7aa269bd..95b21e7225e8d7 100644
--- a/lib/raid/xor/x86/xor-avx.c
+++ b/lib/raid/xor/x86/xor-avx.c
@@ -147,6 +147,7 @@ static void xor_gen_avx(void *dest, void **srcs, unsigned int src_cnt,
 {
 	kernel_fpu_begin();
 	xor_gen_avx_inner(dest, srcs, src_cnt, bytes);
+	asm volatile("vzeroupper");
 	kernel_fpu_end();
 }
 
diff --git a/lib/raid/xor/x86/xor-avx512.c b/lib/raid/xor/x86/xor-avx512.c
new file mode 100644
index 00000000000000..c11d83441875c1
--- /dev/null
+++ b/lib/raid/xor/x86/xor-avx512.c
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * AVX-512 optimized implementation of xor_gen()
+ *
+ * Copyright 2026 Google LLC
+ */
+
+#include <linux/types.h>
+#include <asm/fpu/api.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
+
+/*
+ * Implementation notes:
+ *
+ * Unrolling by the number of buffers (2-5) is very important.
+ *
+ * Unrolling by length is less important, especially when using register-indexed
+ * addressing with negative indices from the end of the buffers.  That approach
+ * results in just two loop control instructions being needed per iteration,
+ * regardless of the number of buffers.
+ *
+ * In fact, benchmarks showed that the 2 and 3 buffer cases require only 2x
+ * unrolling by length, while the 4 and 5 buffer cases don't require any
+ * unrolling by length.  Benchmarks also showed that the register-indexed
+ * addressing isn't a bottleneck either; i.e., we can't do any better by
+ * incrementing the pointers as we go along, even with more unrolling.
+ */
+
+static void xor_avx512_2(long bytes, u8 *p1, const u8 *p2)
+{
+	long i = -bytes;
+
+	asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
+		     "vmovdqa64 64(%1,%0), %%zmm1\n"
+		     "vpxorq (%2,%0), %%zmm0, %%zmm0\n"
+		     "vpxorq 64(%2,%0), %%zmm1, %%zmm1\n"
+		     "vmovdqa64 %%zmm0, (%1,%0)\n"
+		     "vmovdqa64 %%zmm1, 64(%1,%0)\n"
+		     "add $128, %0\n"
+		     "jnz 1b\n"
+		     : "+&r"(i)
+		     : "r"(p1 + bytes), "r"(p2 + bytes)
+		     : "memory", "cc");
+}
+
+static void xor_avx512_3(long bytes, u8 *p1, const u8 *p2, const u8 *p3)
+{
+	long i = -bytes;
+
+	asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
+		     "vmovdqa64 64(%1,%0), %%zmm1\n"
+		     "vmovdqa64 (%2,%0), %%zmm2\n"
+		     "vmovdqa64 64(%2,%0), %%zmm3\n"
+		     "vpternlogq $0x96, (%3,%0), %%zmm2, %%zmm0\n"
+		     "vpternlogq $0x96, 64(%3,%0), %%zmm3, %%zmm1\n"
+		     "vmovdqa64 %%zmm0, (%1,%0)\n"
+		     "vmovdqa64 %%zmm1, 64(%1,%0)\n"
+		     "add $128, %0\n"
+		     "jnz 1b\n"
+		     : "+&r"(i)
+		     : "r"(p1 + bytes), "r"(p2 + bytes), "r"(p3 + bytes)
+		     : "memory", "cc");
+}
+
+static void xor_avx512_4(long bytes, u8 *p1, const u8 *p2, const u8 *p3,
+			 const u8 *p4)
+{
+	long i = -bytes;
+
+	asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
+		     "vmovdqa64 (%2,%0), %%zmm1\n"
+		     "vpxorq (%3,%0), %%zmm0, %%zmm0\n"
+		     "vpternlogq $0x96, (%4,%0), %%zmm1, %%zmm0\n"
+		     "vmovdqa64 %%zmm0, (%1,%0)\n"
+		     "add $64, %0\n"
+		     "jnz 1b\n"
+		     : "+&r"(i)
+		     : "r"(p1 + bytes), "r"(p2 + bytes), "r"(p3 + bytes),
+		       "r"(p4 + bytes)
+		     : "memory", "cc");
+}
+
+static void xor_avx512_5(long bytes, u8 *p1, const u8 *p2, const u8 *p3,
+			 const u8 *p4, const u8 *p5)
+{
+	long i = -bytes;
+
+	asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
+		     "vmovdqa64 (%2,%0), %%zmm1\n"
+		     "vpternlogq $0x96, (%3,%0), %%zmm1, %%zmm0\n"
+		     "vmovdqa64 (%4,%0), %%zmm1\n"
+		     "vpternlogq $0x96, (%5,%0), %%zmm1, %%zmm0\n"
+		     "vmovdqa64 %%zmm0, (%1,%0)\n"
+		     "add $64, %0\n"
+		     "jnz 1b\n"
+		     : "+&r"(i)
+		     : "r"(p1 + bytes), "r"(p2 + bytes), "r"(p3 + bytes),
+		       "r"(p4 + bytes), "r"(p5 + bytes)
+		     : "memory", "cc");
+}
+
+DO_XOR_BLOCKS(avx512_inner, xor_avx512_2, xor_avx512_3, xor_avx512_4,
+	      xor_avx512_5);
+
+/*
+ * Preconditions: bytes is a nonzero multiple of 512, and all buffers are
+ * 64-byte aligned.
+ */
+static void xor_gen_avx512(void *dest, void **srcs, unsigned int src_cnt,
+			   unsigned int bytes)
+{
+	kernel_fpu_begin();
+	xor_gen_avx512_inner(dest, srcs, src_cnt, bytes);
+	asm volatile("vzeroupper");
+	kernel_fpu_end();
+}
+
+struct xor_block_template xor_block_avx512 = {
+	.name = "avx512",
+	.xor_gen = xor_gen_avx512,
+};
diff --git a/lib/raid/xor/x86/xor_arch.h b/lib/raid/xor/x86/xor_arch.h
index 99fe85a213c669..ed5921d2e2aadd 100644
--- a/lib/raid/xor/x86/xor_arch.h
+++ b/lib/raid/xor/x86/xor_arch.h
@@ -6,22 +6,31 @@ extern struct xor_block_template xor_block_p5_mmx;
 extern struct xor_block_template xor_block_sse;
 extern struct xor_block_template xor_block_sse_pf64;
 extern struct xor_block_template xor_block_avx;
+extern struct xor_block_template xor_block_avx512;
 
-/*
- * When SSE is available, use it as it can write around L2.  We may also be able
- * to load into the L1 only depending on how the cpu deals with a load to a line
- * that is being prefetched.
- *
- * When AVX2 is available, force using it as it is better by all measures.
- *
- * 32-bit without MMX can fall back to the generic routines.
- */
 static __always_inline void __init arch_xor_init(void)
 {
-	if (boot_cpu_has(X86_FEATURE_AVX) &&
-	    boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+	if (IS_ENABLED(CONFIG_X86_64) && boot_cpu_has(X86_FEATURE_AVX512F) &&
+	    !boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
+		/*
+		 * Use the AVX-512 code on CPUs that support AVX-512 without
+		 * overly-eager downclocking.  On such CPUs the AVX-512 code
+		 * should always work at least as well as the AVX code, so
+		 * runtime selection is unnecessary.
+		 *
+		 * The AVX-512 code can work on X86_32.  However, due to lack of
+		 * use case for that, for now it's built only for X86_64.
+		 */
+		xor_force(&xor_block_avx512);
+	} else if (boot_cpu_has(X86_FEATURE_AVX)) {
+		/* AVX will be the best; no need to try others. */
 		xor_force(&xor_block_avx);
 	} else if (IS_ENABLED(CONFIG_X86_64) || boot_cpu_has(X86_FEATURE_XMM)) {
+		/*
+		 * When SSE is available, use it as it can write around L2.  We
+		 * may also be able to load into the L1 only depending on how
+		 * the cpu deals with a load to a line that is being prefetched.
+		 */
 		xor_register(&xor_block_sse);
 		xor_register(&xor_block_sse_pf64);
 	} else if (boot_cpu_has(X86_FEATURE_MMX)) {
diff --git a/lib/region_alloc_benchmark.c b/lib/region_alloc_benchmark.c
index e88b4cf55c629c..a644f3d5431a0c 100644
--- a/lib/region_alloc_benchmark.c
+++ b/lib/region_alloc_benchmark.c
@@ -78,11 +78,13 @@ static size_t __init ida_size(unsigned long nr_ids)
 
 static unsigned long __init benchmark_ida(unsigned long cap)
 {
-	struct ida ida = IDA_INIT(ida);
+	struct ida ida;
 	unsigned long cnt, idx, off, nr_ids = 0;
 	ktime_t alloc_time, free_time;
 	int id = -ENOSPC;
 
+	ida_init(&ida);
+
 	alloc_time = ktime_get();
 	for (cnt = 0; cnt <= cap; cnt++) {
 		for (off = 0; off < reg_sz[cnt]; off++) {
@@ -125,12 +127,14 @@ static unsigned long __init benchmark_ida(unsigned long cap)
 
 static unsigned long __init benchmark_maple_tree(unsigned long cap)
 {
-	struct maple_tree mt = MTREE_INIT(mt, MT_FLAGS_ALLOC_RANGE);
+	struct maple_tree mt;
 	unsigned long cnt, idx;
 	ktime_t alloc_time, free_time;
 	size_t sz;
 	int ret;
 
+	mt_init_flags(&mt, MT_FLAGS_ALLOC_RANGE);
+
 	alloc_time = ktime_get();
 	for (cnt = 0; cnt <= cap; cnt++) {
 		ret = mtree_alloc_range(&mt, &idx, xa_mk_value(cnt + 1),
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 6362896e4f0999..a3a4a1f7751e03 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -191,7 +191,6 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
 	struct bucket_table *tbl = NULL;
 	size_t size;
 	int i;
-	static struct lock_class_key __key;
 
 	tbl = alloc_hooks_tag(ht->alloc_tag,
 			kvmalloc_node_align_noprof(struct_size(tbl, buckets, nbuckets),
@@ -207,7 +206,9 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
 	if (tbl == NULL)
 		return NULL;
 
-	lockdep_init_map(&tbl->dep_map, "rhashtable_bucket", &__key, 0);
+#ifdef CONFIG_LOCKDEP
+	lockdep_init_map(&tbl->dep_map, "rhashtable_bucket", ht->lockdep_key, 0);
+#endif
 
 	tbl->size = size;
 
@@ -1120,51 +1121,9 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed)
 	return jhash2(key, length, seed);
 }
 
-/**
- * rhashtable_init - initialize a new hash table
- * @ht:		hash table to be initialized
- * @params:	configuration parameters
- *
- * Initializes a new hash table based on the provided configuration
- * parameters. A table can be configured either with a variable or
- * fixed length key:
- *
- * Configuration Example 1: Fixed length keys
- * struct test_obj {
- *	int			key;
- *	void *			my_member;
- *	struct rhash_head	node;
- * };
- *
- * struct rhashtable_params params = {
- *	.head_offset = offsetof(struct test_obj, node),
- *	.key_offset = offsetof(struct test_obj, key),
- *	.key_len = sizeof(int),
- *	.hashfn = jhash,
- * };
- *
- * Configuration Example 2: Variable length keys
- * struct test_obj {
- *	[...]
- *	struct rhash_head	node;
- * };
- *
- * u32 my_hash_fn(const void *data, u32 len, u32 seed)
- * {
- *	struct test_obj *obj = data;
- *
- *	return [... hash ...];
- * }
- *
- * struct rhashtable_params params = {
- *	.head_offset = offsetof(struct test_obj, node),
- *	.hashfn = jhash,
- *	.obj_hashfn = my_hash_fn,
- * };
- */
 int __rhashtable_init_noprof(struct rhashtable *ht,
-		    const struct rhashtable_params *params,
-		    struct lock_class_key *key)
+			     const struct rhashtable_params *params,
+			     struct rhashtable_lockdep_keys *keys)
 {
 	struct bucket_table *tbl;
 	size_t size;
@@ -1174,8 +1133,12 @@ int __rhashtable_init_noprof(struct rhashtable *ht,
 		return -EINVAL;
 
 	memset(ht, 0, sizeof(*ht));
-	mutex_init_with_key(&ht->mutex, key);
+	mutex_init_with_key(&ht->mutex, &keys->mutex_key);
 	spin_lock_init(&ht->lock);
+	lockdep_set_class(&ht->lock, &keys->lock_key);
+#ifdef CONFIG_LOCKDEP
+	ht->lockdep_key = &keys->bucket_key;
+#endif
 	memcpy(&ht->p, params, sizeof(*params));
 
 	alloc_tag_record(ht->alloc_tag);
@@ -1228,22 +1191,13 @@ int __rhashtable_init_noprof(struct rhashtable *ht,
 }
 EXPORT_SYMBOL_GPL(__rhashtable_init_noprof);
 
-/**
- * rhltable_init - initialize a new hash list table
- * @hlt:	hash list table to be initialized
- * @params:	configuration parameters
- *
- * Initializes a new hash list table.
- *
- * See documentation for rhashtable_init.
- */
 int __rhltable_init_noprof(struct rhltable *hlt,
 			   const struct rhashtable_params *params,
-			   struct lock_class_key *key)
+			   struct rhashtable_lockdep_keys *keys)
 {
 	int err;
 
-	err = __rhashtable_init_noprof(&hlt->ht, params, key);
+	err = __rhashtable_init_noprof(&hlt->ht, params, keys);
 	hlt->ht.rhlist = true;
 	return err;
 }
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index 98d6ed0eaab7e9..4a621f884bde3f 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -329,9 +329,8 @@ int string_unescape(char *src, char *dst, size_t size, unsigned int flags)
 		size = SIZE_MAX;
 
 	while (*src && --size) {
-		if (src[0] == '\\' && src[1] != '\0' && size > 1) {
+		if (src[0] == '\\' && src[1] != '\0') {
 			src++;
-			size--;
 
 			if (flags & UNESCAPE_SPACE &&
 					unescape_space(&src, &out))
@@ -350,6 +349,8 @@ int string_unescape(char *src, char *dst, size_t size, unsigned int flags)
 				continue;
 
 			*out++ = '\\';
+			if (!--size)
+				break;
 		}
 		*out++ = *src++;
 	}
diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c
index 56bd23059b26ab..ba09d057f47f42 100644
--- a/lib/test_bitmap.c
+++ b/lib/test_bitmap.c
@@ -502,6 +502,8 @@ static const struct test_bitmap_parselist parselist_tests[] __initconst = {
 	{0, "0-N:1/3,1-N:1/3,2-N:1/3",		&exp1[8 * step], 32, 0},
 	{0, "0-31:1/3,1-31:1/3,2-31:1/3",	&exp1[8 * step], 32, 0},
 	{0, "1-10:8/12,8-31:24/29,0-31:0/3",	&exp1[9 * step], 32, 0},
+	{0, "1-1:1/4294967295",			&exp1[1 * step], 32, 0},
+	{0, "15-15:1/4294967281",		&exp1[13 * step], 32, 0},
 
 	{0,	  "all",		&exp1[8 * step], 32, 0},
 	{0,	  "0, 1, all,  ",	&exp1[8 * step], 32, 0},
@@ -528,6 +530,9 @@ static const struct test_bitmap_parselist parselist_tests[] __initconst = {
 	{-EINVAL, "a-31:10/1", NULL, 8, 0},
 	{-EINVAL, "0-31:a/1", NULL, 8, 0},
 	{-EINVAL, "0-\n", NULL, 8, 0},
+	{-EINVAL, "0-7:1/2N", NULL, 8, 0},
+	{-EINVAL, "0-7:1/2all", NULL, 8, 0},
+	{-EINVAL, "all:1/2N", NULL, 8, 0},
 
 };
 
@@ -643,6 +648,10 @@ static const struct test_bitmap_parselist parse_tests[] __initconst = {
 	{-EOVERFLOW, "badf00d,deadbeef,1,0",	NULL, 90, 0},
 	{-EOVERFLOW, "fbadf00d,deadbeef,1,0",	NULL, 95, 0},
 	{-EOVERFLOW, "badf00d,deadbeef,1,0",	NULL, 100, 0},
+
+	{-EINVAL,    "x12345678",		NULL, 32, 0},
+	{-EINVAL,    "1g12345678",		NULL, 64, 0},
+	{-EINVAL,    "0x0000000f",		NULL, 64, 0},
 #undef step
 };
 
diff --git a/lib/test_csd_lock.c b/lib/test_csd_lock.c
new file mode 100644
index 00000000000000..30c6c3332c3fc0
--- /dev/null
+++ b/lib/test_csd_lock.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Keep one CPU from answering an IPI, so that the CSD-lock debug code in
+ * kernel/smp.c has a stall to report.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates
+ * Copyright (c) 2026 Breno Leitao <leitao@debian.org>
+ *
+ * The target either spins with interrupts disabled, which leaves it idle as
+ * far as the debug code can tell and gets the IPI re-sent, or spins inside a
+ * CSD handler, which does not.  The recovery message differs between the two.
+ *
+ * Loading the module runs one stall, then fails the load with -EAGAIN so
+ * that nothing is left loaded afterwards:
+ *
+ *	echo 500 > /sys/module/smp/parameters/csd_lock_timeout
+ *	modprobe test_csd_lock stall_ms=1000 in_handler=0
+ *
+ * csd_lock_timeout has to be below stall_ms for the stall to be reported at
+ * all, and the report has to come out before the CPU answers, so leave it
+ * some room.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/completion.h>
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/ktime.h>
+#include <linux/module.h>
+#include <linux/smp.h>
+#include <linux/workqueue.h>
+
+#define STALL_MS_MAX	10000
+
+static unsigned int stall_ms = 1000;
+module_param(stall_ms, uint, 0444);
+MODULE_PARM_DESC(stall_ms, "Time the target CPU ignores the IPI, in milliseconds.");
+
+static int stall_cpu = -1;
+module_param(stall_cpu, int, 0444);
+MODULE_PARM_DESC(stall_cpu, "CPU to stall, or -1 for the first online one.");
+
+static bool in_handler;
+module_param(in_handler, bool, 0444);
+MODULE_PARM_DESC(in_handler, "Stall inside a CSD handler instead of with interrupts disabled.");
+
+static int target_cpu;
+static bool target_stalling;
+static bool hog_launched;
+static struct work_struct irqoff_work;
+static struct work_struct sender_work;
+static call_single_data_t hog_csd;
+static DECLARE_COMPLETION(hog_done);
+
+static void csd_test_nop(void *unused)
+{
+}
+
+static void csd_test_spin(void)
+{
+	u64 end = ktime_get_mono_fast_ns() + (u64)stall_ms * NSEC_PER_MSEC;
+
+	while (ktime_get_mono_fast_ns() < end)
+		cpu_relax();
+}
+
+/* Nothing is running for the target while interrupts are off, so it gets a new IPI. */
+static void csd_test_irqoff_fn(struct work_struct *work)
+{
+	local_irq_disable();
+	/* Pairs with the load in csd_test_sender_fn(), which waits for this. */
+	smp_store_release(&target_stalling, true);
+	csd_test_spin();
+	local_irq_enable();
+}
+
+/* Here cur_csd stays set on the target, which suppresses the re-send. */
+static void csd_test_hog_fn(void *unused)
+{
+	/* Pairs with the load in csd_test_sender_fn(), which waits for this. */
+	smp_store_release(&target_stalling, true);
+	csd_test_spin();
+	complete(&hog_done);
+}
+
+/*
+ * Start the stall from here rather than from module init, so that however
+ * long this work item waits to be scheduled comes off before the target
+ * stops answering, not out of the middle of the stall.
+ */
+static void csd_test_sender_fn(struct work_struct *work)
+{
+	u64 deadline, ts;
+	int err;
+
+	if (in_handler) {
+		hog_csd.func = csd_test_hog_fn;
+		err = smp_call_function_single_async(target_cpu, &hog_csd);
+		if (err) {
+			pr_err("cannot queue the CSD handler on CPU%d: %d\n", target_cpu, err);
+			return;
+		}
+	} else {
+		queue_work_on(target_cpu, system_highpri_wq, &irqoff_work);
+	}
+	WRITE_ONCE(hog_launched, true);
+
+	deadline = ktime_get_mono_fast_ns() + (u64)STALL_MS_MAX * NSEC_PER_MSEC;
+	/* Pairs with the store in the stall functions: send once it is stuck. */
+	while (!smp_load_acquire(&target_stalling)) {
+		if (ktime_get_mono_fast_ns() > deadline) {
+			pr_err("CPU%d never stopped answering\n", target_cpu);
+			return;
+		}
+		cpu_relax();
+	}
+
+	ts = ktime_get_mono_fast_ns();
+	smp_call_function_single(target_cpu, csd_test_nop, NULL, 1);
+	pr_info("CPU%d answered after %llu ns\n", target_cpu,
+		ktime_get_mono_fast_ns() - ts);
+}
+
+static int __init test_csd_lock_init(void)
+{
+	int sender_cpu;
+	int ret = 0;
+
+	if (!stall_ms || stall_ms > STALL_MS_MAX) {
+		pr_err("stall_ms must be between 1 and %d\n", STALL_MS_MAX);
+		return -EINVAL;
+	}
+
+	INIT_WORK(&irqoff_work, csd_test_irqoff_fn);
+	INIT_WORK(&sender_work, csd_test_sender_fn);
+
+	cpus_read_lock();
+
+	target_cpu = stall_cpu < 0 ? cpumask_first(cpu_online_mask) : stall_cpu;
+	sender_cpu = nr_cpu_ids;
+	if (target_cpu < nr_cpu_ids && cpu_online(target_cpu))
+		sender_cpu = cpumask_any_but(cpu_online_mask, target_cpu);
+	if (sender_cpu >= nr_cpu_ids) {
+		pr_err("need CPU%d and one other CPU online\n", target_cpu);
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	pr_info("stalling CPU%d for %u ms %s, IPI from CPU%d\n", target_cpu, stall_ms,
+		in_handler ? "inside a CSD handler" : "with interrupts disabled", sender_cpu);
+
+	queue_work_on(sender_cpu, system_highpri_wq, &sender_work);
+	flush_work(&sender_work);
+	flush_work(&irqoff_work);
+
+	/* The CSD has to be idle again before this module goes away. */
+	if (in_handler && READ_ONCE(hog_launched) &&
+	    !wait_for_completion_timeout(&hog_done, msecs_to_jiffies(2 * STALL_MS_MAX)))
+		pr_err("CSD handler on CPU%d never finished\n", target_cpu);
+
+	/* The stall is over and there is nothing left to hold, so go away. */
+	ret = -EAGAIN;
+unlock:
+	cpus_read_unlock();
+
+	return ret;
+}
+module_init(test_csd_lock_init);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Breno Leitao <leitao@debian.org>");
+MODULE_DESCRIPTION("Test module to stall a CPU on a CSD lock");
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index 6911daa9f8543c..b409c42bfe6fc4 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -1624,6 +1624,8 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp,
 
 	if (cmd.addr & ~PAGE_MASK)
 		return -EINVAL;
+	if (cmd.npages > ULONG_MAX >> PAGE_SHIFT)
+		return -EINVAL;
 	if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
 		return -EINVAL;
 
diff --git a/lib/test_sysctl.c b/lib/test_sysctl.c
index 909cfcf76dbfc6..65ce4234617071 100644
--- a/lib/test_sysctl.c
+++ b/lib/test_sysctl.c
@@ -15,6 +15,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/init.h>
+#include <linux/jiffies.h>
 #include <linux/list.h>
 #include <linux/module.h>
 #include <linux/printk.h>
@@ -48,6 +49,10 @@ struct test_sysctl_data {
 
 	int boot_int;
 
+	int int_jiffies;
+	int int_ms_jiffies;
+	int int_userhz_jiffies;
+
 	unsigned int uint_0001;
 
 	char string_0001[65];
@@ -67,6 +72,11 @@ static struct test_sysctl_data test_data = {
 
 	.boot_int = 0,
 
+	/* One second in each converter's unit; all three store HZ jiffies. */
+	.int_jiffies = HZ,
+	.int_ms_jiffies = HZ,
+	.int_userhz_jiffies = HZ,
+
 	.uint_0001 = 314,
 
 	.string_0001 = "(none)",
@@ -113,6 +123,27 @@ static const struct ctl_table test_table[] = {
 		.extra1		= SYSCTL_ZERO,
 		.extra2         = SYSCTL_ONE,
 	},
+	{
+		.procname	= "int_jiffies",
+		.data		= &test_data.int_jiffies,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_jiffies,
+	},
+	{
+		.procname	= "int_ms_jiffies",
+		.data		= &test_data.int_ms_jiffies,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_ms_jiffies,
+	},
+	{
+		.procname	= "int_userhz_jiffies",
+		.data		= &test_data.int_userhz_jiffies,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_userhz_jiffies,
+	},
 	{
 		.procname	= "uint_0001",
 		.data		= &test_data.uint_0001,
@@ -321,11 +352,11 @@ module_init(test_sysctl_init);
 
 static void __exit test_sysctl_exit(void)
 {
-	kfree(test_data.bitmap_0001);
 	for (int i = 0; i < TEST_H_SIZE; i++) {
 		if (ctl_headers[i])
 			unregister_sysctl_table(ctl_headers[i]);
 	}
+	kfree(test_data.bitmap_0001);
 }
 
 module_exit(test_sysctl_exit);
diff --git a/lib/test_xarray.c b/lib/test_xarray.c
index 5ca0aefee9aa5f..615f7730a5bd4a 100644
--- a/lib/test_xarray.c
+++ b/lib/test_xarray.c
@@ -1247,6 +1247,75 @@ static noinline void check_multi_find_3(struct xarray *xa)
 	}
 }
 
+static noinline void check_multi_find_4(struct xarray *xa)
+{
+#ifdef CONFIG_XARRAY_MULTI
+	unsigned int order = XA_CHUNK_SHIFT + 1;
+	unsigned long next = 1UL << order;
+	unsigned long start = next - 1;
+	XA_STATE(xas, xa, start);
+	XA_STATE_ORDER(split, xa, 0, 0);
+	void *entry;
+	unsigned long i;
+
+	/* Multi-index entry in two slots of a non-leaf node. */
+	xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL);
+	XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL);
+
+	rcu_read_lock();
+	entry = xas_find(&xas, ULONG_MAX);
+	XA_BUG_ON(xa, entry != xa_mk_index(0));
+	XA_BUG_ON(xa, xas.xa_index != start);
+
+	entry = xas_find(&xas, ULONG_MAX);
+	XA_BUG_ON(xa, entry != xa_mk_index(next));
+	XA_BUG_ON(xa, xas.xa_index != next);
+
+	entry = xas_find(&xas, ULONG_MAX);
+	XA_BUG_ON(xa, entry != NULL);
+	rcu_read_unlock();
+
+	xa_erase_index(xa, next);
+	xa_erase_index(xa, 0);
+	XA_BUG_ON(xa, !xa_empty(xa));
+
+	/* Split the multi-index entry after a lookup begins inside it. */
+	xa_store_order(xa, 0, order, xa_mk_index(0), GFP_KERNEL);
+	XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL) != NULL);
+
+	xas_set(&xas, start);
+	rcu_read_lock();
+	entry = xas_find(&xas, ULONG_MAX);
+	XA_BUG_ON(xa, entry != xa_mk_index(0));
+	XA_BUG_ON(xa, xas.xa_index != start);
+	rcu_read_unlock();
+
+	xas_split_alloc(&split, xa_mk_index(0), order, GFP_KERNEL);
+	if (xas_error(&split)) {
+		XA_BUG_ON(xa, true);
+		goto out;
+	}
+	xas_lock(&split);
+	xas_split(&split, xa_mk_index(0), order);
+	for (i = 0; i < next; i++)
+		__xa_store(xa, i, xa_mk_index(i), 0);
+	xas_unlock(&split);
+
+	rcu_read_lock();
+	entry = xas_find(&xas, ULONG_MAX);
+	XA_BUG_ON(xa, entry != xa_mk_index(next));
+	XA_BUG_ON(xa, xas.xa_index != next);
+
+	entry = xas_find(&xas, ULONG_MAX);
+	XA_BUG_ON(xa, entry != NULL);
+	rcu_read_unlock();
+
+out:
+	xa_destroy(xa);
+	XA_BUG_ON(xa, !xa_empty(xa));
+#endif
+}
+
 static noinline void check_find_1(struct xarray *xa)
 {
 	unsigned long i, j, k;
@@ -1370,6 +1439,7 @@ static noinline void check_find(struct xarray *xa)
 		check_multi_find_1(xa, i);
 	check_multi_find_2(xa);
 	check_multi_find_3(xa);
+	check_multi_find_4(xa);
 }
 
 /* See find_swap_entry() in mm/shmem.c */
diff --git a/lib/tests/842_decompress_kunit.c b/lib/tests/842_decompress_kunit.c
new file mode 100644
index 00000000000000..11202ba9539bc9
--- /dev/null
+++ b/lib/tests/842_decompress_kunit.c
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <kunit/test.h>
+#include <linux/sw842.h>
+
+#define SW842_GUARD_SIZE	8
+#define SW842_MAX_OUTPUT	16
+#define SW842_GUARD_BYTE	0x42
+#define SW842_OUTPUT_POISON	0x5a
+
+/* ZEROS, I8(0), END, CRC32. */
+static const u8 sw842_index_exact_fit[] = {
+	0xe6, 0x40, 0x3c, 0x00, 0x00, 0x00, 0x00,
+};
+
+/* D8(eight zero bytes), I8(0), END, CRC32. */
+static const u8 sw842_index_output_overflow[] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x06, 0x40, 0x3c, 0x00, 0x00, 0x00, 0x00,
+};
+
+/* ZEROS, SHORT_DATA(01 02 03 04 05), END, CRC32. */
+static const u8 sw842_short_data_exact_fit[] = {
+	0xe7, 0x68, 0x08, 0x10, 0x18, 0x20,
+	0x2f, 0xa9, 0x67, 0xfc, 0x07, 0xc0,
+};
+
+/* ZEROS, SHORT_DATA(five zero bytes), END, CRC32. */
+static const u8 sw842_short_data_output_overflow[] = {
+	0xe7, 0x48, 0x07, 0x80, 0x00, 0x00, 0x00, 0x00,
+};
+
+/* ZEROS, REPEAT(one block), END, CRC32. */
+static const u8 sw842_repeat_exact_history[] = {
+	0xe6, 0xc0, 0xf0, 0x00, 0x00, 0x00, 0x00,
+};
+
+/*
+ * SHORT_DATA(seven zero bytes), REPEAT(one block), END, CRC32.
+ * CRC32 includes SW842_GUARD_BYTE copied from before output into byte 7.
+ */
+static const u8 sw842_repeat_without_full_history[] = {
+	0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0xd8, 0x1e, 0x09, 0xa3, 0x1d, 0xd6,
+};
+
+static const u8 sw842_zero_output[SW842_MAX_OUTPUT];
+static const u8 sw842_short_data_output[] = {
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x01, 0x02, 0x03, 0x04, 0x05,
+};
+
+struct sw842_decompress_test_case {
+	const char *name;
+	const u8 *compressed;
+	unsigned int compressed_len;
+	const u8 *expected_output;
+	unsigned int output_capacity;
+	int expected_ret;
+};
+
+#define SW842_DECOMPRESS_CASE(_name, _compressed, _compressed_len, _expected, \
+			      _capacity, _ret) \
+	{ \
+		.name = _name, \
+		.compressed = _compressed, \
+		.compressed_len = _compressed_len, \
+		.expected_output = _expected, \
+		.output_capacity = _capacity, \
+		.expected_ret = _ret, \
+	}
+
+static const struct sw842_decompress_test_case sw842_decompress_cases[] = {
+	SW842_DECOMPRESS_CASE("index_exact_fit", sw842_index_exact_fit,
+			      ARRAY_SIZE(sw842_index_exact_fit),
+			      sw842_zero_output, 16, 0),
+	SW842_DECOMPRESS_CASE("index_output_overflow",
+			      sw842_index_output_overflow,
+			      ARRAY_SIZE(sw842_index_output_overflow),
+			      sw842_zero_output, 8, -ENOSPC),
+	SW842_DECOMPRESS_CASE("short_data_exact_fit",
+			      sw842_short_data_exact_fit,
+			      ARRAY_SIZE(sw842_short_data_exact_fit),
+			      sw842_short_data_output,
+			      sizeof(sw842_short_data_output), 0),
+	SW842_DECOMPRESS_CASE("short_data_output_overflow",
+			      sw842_short_data_output_overflow,
+			      ARRAY_SIZE(sw842_short_data_output_overflow),
+			      sw842_zero_output, 8, -ENOSPC),
+	SW842_DECOMPRESS_CASE("repeat_exact_history",
+			      sw842_repeat_exact_history,
+			      ARRAY_SIZE(sw842_repeat_exact_history),
+			      sw842_zero_output, 16, 0),
+	SW842_DECOMPRESS_CASE("repeat_without_full_history",
+			      sw842_repeat_without_full_history,
+			      ARRAY_SIZE(sw842_repeat_without_full_history),
+			      sw842_zero_output, 15, -EINVAL),
+};
+
+KUNIT_ARRAY_PARAM_DESC(sw842_decompress, sw842_decompress_cases, name);
+
+static void sw842_decompress_test(struct kunit *test)
+{
+	const struct sw842_decompress_test_case *test_case = test->param_value;
+	u8 storage[SW842_GUARD_SIZE + SW842_MAX_OUTPUT + SW842_GUARD_SIZE];
+	u8 expected_guard[SW842_GUARD_SIZE];
+	u8 *output = storage + SW842_GUARD_SIZE;
+	unsigned int output_len = test_case->output_capacity;
+	int ret;
+
+	KUNIT_ASSERT_LE(test, test_case->output_capacity, SW842_MAX_OUTPUT);
+	memset(storage, SW842_GUARD_BYTE, sizeof(storage));
+	memset(expected_guard, SW842_GUARD_BYTE, sizeof(expected_guard));
+	memset(output, SW842_OUTPUT_POISON, test_case->output_capacity);
+
+	ret = sw842_decompress(test_case->compressed,
+			       test_case->compressed_len, output, &output_len);
+
+	KUNIT_EXPECT_EQ(test, ret, test_case->expected_ret);
+	/* Every successful case is an exact-fit boundary test. */
+	KUNIT_EXPECT_EQ(test, output_len,
+			test_case->expected_ret ? 0U : test_case->output_capacity);
+	if (!test_case->expected_ret)
+		KUNIT_EXPECT_MEMEQ(test, output, test_case->expected_output,
+				   test_case->output_capacity);
+	KUNIT_EXPECT_MEMEQ(test, storage, expected_guard, SW842_GUARD_SIZE);
+	KUNIT_EXPECT_MEMEQ(test, output + test_case->output_capacity,
+			   expected_guard, SW842_GUARD_SIZE);
+}
+
+static struct kunit_case sw842_decompress_test_cases[] = {
+	KUNIT_CASE_PARAM(sw842_decompress_test, sw842_decompress_gen_params),
+	{}
+};
+
+static struct kunit_suite sw842_decompress_test_suite = {
+	.name = "842-decompress",
+	.test_cases = sw842_decompress_test_cases,
+};
+
+kunit_test_suite(sw842_decompress_test_suite);
+
+MODULE_DESCRIPTION("Software 842 decompressor KUnit tests");
+MODULE_LICENSE("GPL");
diff --git a/lib/tests/Makefile b/lib/tests/Makefile
index 3cac3b63a7522c..64bb6d4e4a44f2 100644
--- a/lib/tests/Makefile
+++ b/lib/tests/Makefile
@@ -4,6 +4,7 @@
 
 # KUnit tests
 CFLAGS_bitfield_kunit.o := $(DISABLE_STRUCTLEAK_PLUGIN)
+obj-$(CONFIG_842_DECOMPRESS_KUNIT_TEST) += 842_decompress_kunit.o
 obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o
 obj-$(CONFIG_BITOPS_KUNIT) += bitops_kunit.o
 obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o
@@ -13,6 +14,7 @@ obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o
 obj-$(CONFIG_CHECKSUM_KUNIT) += checksum_kunit.o
 obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o
 obj-$(CONFIG_CPUMASK_KUNIT_TEST) += cpumask_kunit.o
+obj-$(CONFIG_ERRSEQ_KUNIT_TEST) += errseq_kunit.o
 obj-$(CONFIG_FFS_KUNIT_TEST) += ffs_kunit.o
 CFLAGS_fortify_kunit.o += $(call cc-disable-warning, unsequenced)
 CFLAGS_fortify_kunit.o += $(call cc-disable-warning, stringop-overread)
diff --git a/lib/tests/errseq_kunit.c b/lib/tests/errseq_kunit.c
new file mode 100644
index 00000000000000..8f39ebc4a2488e
--- /dev/null
+++ b/lib/tests/errseq_kunit.c
@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit tests for the errseq_t error-tracking infrastructure.
+ *
+ * These exercise the documented single-threaded semantics of the errseq
+ * API (see Documentation/core-api/errseq.rst and lib/errseq.c): error
+ * recording and overwriting, the "seen" handoff between errseq_sample()
+ * and errseq_check_and_advance(), and the re-reporting of an error that
+ * is recorded again after it has been seen.
+ *
+ * The lockless properties of errseq_t under concurrent updates are
+ * outside the scope of these deterministic tests, as is the WARN path
+ * for invalid error values.
+ */
+#include <kunit/test.h>
+
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/errseq.h>
+
+/*
+ * A zeroed errseq_t is the "no error has ever occurred" epoch: it
+ * samples as zero and no check against it reports anything.
+ */
+static void errseq_test_zero_epoch_reports_no_error(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t since = 0;
+
+	KUNIT_EXPECT_EQ(test, errseq_sample(&eseq), 0);
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, 0), 0);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), 0);
+	KUNIT_EXPECT_EQ(test, since, 0);
+}
+
+static void errseq_test_set_records_error(struct kunit *test)
+{
+	errseq_t eseq = 0;
+
+	/* errseq_set() returns the previous value; the epoch is zero. */
+	KUNIT_EXPECT_EQ(test, errseq_set(&eseq, -EIO), 0);
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, 0), -EIO);
+}
+
+/* Any error set always overwrites an existing error. */
+static void errseq_test_set_overwrites_error(struct kunit *test)
+{
+	errseq_t eseq = 0;
+
+	errseq_set(&eseq, -EIO);
+	errseq_set(&eseq, -ENOSPC);
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, 0), -ENOSPC);
+}
+
+/* Both ends of the valid error range are recorded exactly. */
+static void errseq_test_errno_range_extremes(struct kunit *test)
+{
+	errseq_t lo = 0;
+	errseq_t hi = 0;
+
+	errseq_set(&lo, -1);
+	KUNIT_EXPECT_EQ(test, errseq_check(&lo, 0), -1);
+
+	errseq_set(&hi, -MAX_ERRNO);
+	KUNIT_EXPECT_EQ(test, errseq_check(&hi, 0), -MAX_ERRNO);
+}
+
+/*
+ * An error nobody has seen yet samples as zero, so that a check against
+ * the sample still reports it (see commit b4678df184b3 ("errseq: Always
+ * report a writeback error once")).
+ */
+static void errseq_test_sample_of_unseen_error_is_zero(struct kunit *test)
+{
+	errseq_t eseq = 0;
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_sample(&eseq), 0);
+}
+
+static void errseq_test_new_sampler_sees_unseen_error(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t since;
+
+	errseq_set(&eseq, -EIO);
+	since = errseq_sample(&eseq);
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, since), -EIO);
+}
+
+/* A given error is reported exactly once per advancing cursor. */
+static void errseq_test_check_and_advance_reports_once(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t since = errseq_sample(&eseq);
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), 0);
+}
+
+/*
+ * Once an error has been seen, a fresh sample is non-zero and checking
+ * against it reports nothing: handled errors do not reach new samplers.
+ */
+static void errseq_test_sample_after_seen_is_current(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t since = 0;
+	errseq_t sample;
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), -EIO);
+
+	sample = errseq_sample(&eseq);
+	KUNIT_EXPECT_NE(test, sample, 0);
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, sample), 0);
+}
+
+static void errseq_test_new_error_after_advance(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t since = 0;
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), -EIO);
+
+	errseq_set(&eseq, -ENOSPC);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), -ENOSPC);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), 0);
+}
+
+/*
+ * Recording the same error again after it has been seen must bump the
+ * sequence, so cursors that consumed the first occurrence see the
+ * second one too.
+ */
+static void errseq_test_same_error_reported_again_after_seen(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t since = 0;
+	errseq_t seen_cursor;
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), -EIO);
+
+	seen_cursor = since;
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, since), -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), -EIO);
+	/* The repeat must advance the sequence, not just re-toggle "seen". */
+	KUNIT_EXPECT_NE(test, since, seen_cursor);
+}
+
+/*
+ * A cursor that consumed an error must still observe a repeat of that
+ * error even when another cursor has already marked the repeat seen:
+ * recording over a seen value must advance the sequence.
+ */
+static void errseq_test_repeat_error_visible_to_all_cursors(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t cursor_a = 0;
+	errseq_t cursor_b = 0;
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_a), -EIO);
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_b), -EIO);
+
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, cursor_a), -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_a), -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_a), 0);
+}
+
+/* An advance with no new error reports nothing and leaves the cursor put. */
+static void errseq_test_advance_stable_when_unchanged(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t since = 0;
+	errseq_t cursor;
+
+	errseq_set(&eseq, -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), -EIO);
+
+	cursor = since;
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &since), 0);
+	KUNIT_EXPECT_EQ(test, since, cursor);
+}
+
+/*
+ * Cursors are independent: one subscriber consuming an error does not
+ * consume it for another, and each subscriber sees each error once.
+ */
+static void errseq_test_two_subscribers_independent(struct kunit *test)
+{
+	errseq_t eseq = 0;
+	errseq_t cursor_a = errseq_sample(&eseq);
+	errseq_t cursor_b = errseq_sample(&eseq);
+
+	errseq_set(&eseq, -EIO);
+
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_a), -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check(&eseq, cursor_b), -EIO);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_b), -EIO);
+
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_a), 0);
+	KUNIT_EXPECT_EQ(test, errseq_check_and_advance(&eseq, &cursor_b), 0);
+}
+
+static struct kunit_case errseq_test_cases[] = {
+	KUNIT_CASE(errseq_test_zero_epoch_reports_no_error),
+	KUNIT_CASE(errseq_test_set_records_error),
+	KUNIT_CASE(errseq_test_set_overwrites_error),
+	KUNIT_CASE(errseq_test_errno_range_extremes),
+	KUNIT_CASE(errseq_test_sample_of_unseen_error_is_zero),
+	KUNIT_CASE(errseq_test_new_sampler_sees_unseen_error),
+	KUNIT_CASE(errseq_test_check_and_advance_reports_once),
+	KUNIT_CASE(errseq_test_sample_after_seen_is_current),
+	KUNIT_CASE(errseq_test_new_error_after_advance),
+	KUNIT_CASE(errseq_test_same_error_reported_again_after_seen),
+	KUNIT_CASE(errseq_test_repeat_error_visible_to_all_cursors),
+	KUNIT_CASE(errseq_test_advance_stable_when_unchanged),
+	KUNIT_CASE(errseq_test_two_subscribers_independent),
+	{}
+};
+
+static struct kunit_suite errseq_test_suite = {
+	.name = "errseq",
+	.test_cases = errseq_test_cases,
+};
+
+kunit_test_suite(errseq_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for the errseq infrastructure");
+MODULE_LICENSE("GPL");
diff --git a/lib/tests/string_helpers_kunit.c b/lib/tests/string_helpers_kunit.c
index 9fbe91079c7ed0..10763a01be83c2 100644
--- a/lib/tests/string_helpers_kunit.c
+++ b/lib/tests/string_helpers_kunit.c
@@ -18,11 +18,10 @@
 
 static void test_string_check_buf(struct kunit *test,
 				  const char *name, unsigned int flags,
-				  char *in, size_t p,
 				  char *out_real, size_t q_real,
 				  char *out_test, size_t q_test)
 {
-	KUNIT_ASSERT_EQ_MSG(test, q_real, q_test, "name:%s", name);
+	KUNIT_EXPECT_EQ_MSG(test, q_real, q_test, "name:%s", name);
 	KUNIT_EXPECT_MEMEQ_MSG(test, out_test, out_real, q_test,
 			       "name:%s", name);
 }
@@ -56,9 +55,9 @@ static const struct test_string strings[] = {
 	},
 };
 
-static void test_string_unescape(struct kunit *test,
-				 const char *name, unsigned int flags,
-				 bool inplace)
+static void test_string_unescape_combined(struct kunit *test,
+					  const char *name, unsigned int flags,
+					  bool inplace)
 {
 	int q_real = 256;
 	char *in = kunit_kzalloc(test, q_real, GFP_KERNEL);
@@ -101,8 +100,8 @@ static void test_string_unescape(struct kunit *test,
 		q_real = string_unescape(in, out_real, q_real, flags);
 	}
 
-	test_string_check_buf(test, name, flags, in, p - 1, out_real, q_real,
-			      out_test, q_test);
+	test_string_check_buf(test, name, flags, out_real, q_real, out_test, q_test);
+	KUNIT_EXPECT_EQ_MSG(test, out_real[q_real], '\0', "name:%s", name);
 }
 
 struct test_string_1 {
@@ -456,8 +455,7 @@ static void test_string_escape(struct kunit *test, const char *name,
 
 	q_real = string_escape_mem(in, p, out_real, out_size, flags, esc);
 
-	test_string_check_buf(test, name, flags, in, p, out_real, q_real, out_test,
-			      q_test);
+	test_string_check_buf(test, name, flags, out_real, q_real, out_test, q_test);
 
 	test_string_escape_overflow(test, in, p, flags, esc, q_test, name);
 }
@@ -598,14 +596,38 @@ static void test_upper_lower(struct kunit *test)
 	}
 }
 
+static void test_string_unescape_one(struct kunit *test,
+				     const char *name, unsigned int flags,
+				     char *src, size_t len,
+				     char *out_test, size_t q_test)
+{
+	char *out_real = kunit_kzalloc(test, len, GFP_KERNEL);
+	int q_real;
+
+	q_real = string_unescape(src, out_real, len, flags);
+	test_string_check_buf(test, name, flags, out_real, q_real, out_test, q_test);
+}
+
 static void test_unescape(struct kunit *test)
 {
 	unsigned int i;
 
 	for (i = 0; i < UNESCAPE_ALL_MASK + 1; i++)
-		test_string_unescape(test, "unescape", i, false);
-	test_string_unescape(test, "unescape inplace",
-			     get_random_u32_below(UNESCAPE_ALL_MASK + 1), true);
+		test_string_unescape_combined(test, "unescape", i, false);
+	test_string_unescape_combined(test, "unescape inplace",
+				      get_random_u32_below(UNESCAPE_ALL_MASK + 1), true);
+
+	test_string_unescape_one(test, "simple case", UNESCAPE_HEX | UNESCAPE_SPECIAL, "ABC", 6, "ABC", 3);
+	test_string_unescape_one(test, "single escape", UNESCAPE_HEX | UNESCAPE_SPECIAL, "A\\x42C", 6, "ABC", 3);
+	test_string_unescape_one(test, "escape before end", UNESCAPE_HEX, "B\\qX", 4, "B\\q", 3);
+	test_string_unescape_one(test, "escape at end", UNESCAPE_HEX, "a\\qX", 3, "a\\", 2);
+	test_string_unescape_one(test, "backslash before escape", UNESCAPE_HEX, "\\\\x41B", 12, "\\\\x41B", 6);
+	test_string_unescape_one(test, "backslash escape", UNESCAPE_HEX | UNESCAPE_SPECIAL, "\\\\x41B", 16, "\\x41B", 5);
+
+	test_string_unescape_one(test, "short buffer", UNESCAPE_HEX, "\\x41\\x41B", 4, "AAB", 3);
+	test_string_unescape_one(test, "unrecognized escape at end", UNESCAPE_HEX, "B\\qX", 4, "B\\q", 3);
+
+	test_string_unescape_one(test, "end of buffer", UNESCAPE_HEX, "B\\x41", 3, "BA", 2);
 }
 
 static void test_escape(struct kunit *test)
diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c
index 1f7dd4eb640a87..6c067b4280b9e4 100644
--- a/lib/ucs2_string.c
+++ b/lib/ucs2_string.c
@@ -57,7 +57,7 @@ ssize_t ucs2_strscpy(ucs2_char_t *dst, const ucs2_char_t *src, size_t count)
 	 * Ensure that we have a valid amount of space. We need to store at
 	 * least one NUL-character.
 	 */
-	if (count == 0 || WARN_ON_ONCE(count > INT_MAX / sizeof(*dst)))
+	if (count == 0)
 		return -E2BIG;
 
 	/*
@@ -125,18 +125,19 @@ ucs2_utf8size(const ucs2_char_t *src)
 EXPORT_SYMBOL(ucs2_utf8size);
 
 /*
- * copy at most maxlength bytes of whole utf8 characters to dest from the
- * ucs2 string src.
+ * Copy at most @limit whole utf8 characters to @dest from the ucs2 string
+ * @src, using no more than @maxlength bytes of buffer space.
  *
- * The return value is the number of characters copied, not including the
- * final NUL character.
+ * The return value is the number of bytes copied, not including the final NUL
+ * character. No NUL character will be appended if the output length equals
+ * @maxlength.
  */
 unsigned long
-ucs2_as_utf8(u8 *dest, const ucs2_char_t *src, unsigned long maxlength)
+ucs2_as_utf8_l(u8 *dest, const ucs2_char_t *src, unsigned long limit,
+	       unsigned long maxlength)
 {
 	unsigned int i;
 	unsigned long j = 0;
-	unsigned long limit = ucs2_strnlen(src, maxlength);
 
 	for (i = 0; maxlength && i < limit; i++) {
 		u16 c = src[i];
@@ -163,7 +164,9 @@ ucs2_as_utf8(u8 *dest, const ucs2_char_t *src, unsigned long maxlength)
 		dest[j] = '\0';
 	return j;
 }
-EXPORT_SYMBOL(ucs2_as_utf8);
+EXPORT_SYMBOL(ucs2_as_utf8_l);
 
+#ifndef __DISABLE_EXPORTS
 MODULE_DESCRIPTION("UCS2 string handling");
 MODULE_LICENSE("GPL v2");
+#endif
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 1d95ab9314bf7d..90d0a724eadd84 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -762,9 +762,8 @@ static int fill_ptr_key(struct notifier_block *nb, unsigned long action, void *d
 {
 	get_random_bytes(&ptr_key, sizeof(ptr_key));
 
-	/* Pairs with smp_rmb() before reading ptr_key. */
-	smp_wmb();
-	WRITE_ONCE(filled_random_ptr_key, true);
+	/* Pairs with smp_load_acquire() before reading ptr_key. */
+	smp_store_release(&filled_random_ptr_key, true);
 	return NOTIFY_DONE;
 }
 
@@ -781,12 +780,10 @@ static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
 {
 	unsigned long hashval;
 
-	if (!READ_ONCE(filled_random_ptr_key))
+	/* Pairs with smp_store_release() after writing ptr_key. */
+	if (!smp_load_acquire(&filled_random_ptr_key))
 		return -EBUSY;
 
-	/* Pairs with smp_wmb() after writing ptr_key. */
-	smp_rmb();
-
 #ifdef CONFIG_64BIT
 	hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
 	/*
diff --git a/lib/xarray.c b/lib/xarray.c
index bfe7bef80f34eb..3913c8d6486bb1 100644
--- a/lib/xarray.c
+++ b/lib/xarray.c
@@ -1409,9 +1409,11 @@ void *xas_find(struct xa_state *xas, unsigned long max)
 		entry = xas_load(xas);
 		if (entry || xas_not_node(xas->xa_node))
 			return entry;
-	} else if (!xas->xa_node->shift &&
-		    xas->xa_offset != (xas->xa_index & XA_CHUNK_MASK)) {
-		xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
+	} else if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node)) {
+		if (!xas->xa_node->shift)
+			xas->xa_offset = ((xas->xa_index - 1) & XA_CHUNK_MASK) + 1;
+		else
+			xas->xa_offset = get_offset(xas->xa_index, xas->xa_node);
 	}
 
 	xas_next_offset(xas);
diff --git a/localversion-next b/localversion-next
new file mode 100644
index 00000000000000..00c02a32fcdc86
--- /dev/null
+++ b/localversion-next
@@ -0,0 +1 @@
+-next-20260930
diff --git a/mm/Kconfig b/mm/Kconfig
index 604c58199acbf8..afff1b25b54387 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -461,8 +461,10 @@ config SPARSEMEM_VMEMMAP
 	  pfn_to_page and page_to_pfn operations.  This is the most
 	  efficient option when sufficient kernel resources are available.
 
-config SPARSEMEM_VMEMMAP_PREINIT
+config VMEMMAP_OPTIMIZATION
 	bool
+	depends on SPARSEMEM_VMEMMAP
+
 #
 # Select this config option from the architecture Kconfig, if it is preferred
 # to enable the feature of HugeTLB/dev_dax vmemmap optimization.
@@ -473,9 +475,6 @@ config ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
 config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
 	bool
 
-config ARCH_WANT_HUGETLB_VMEMMAP_PREINIT
-	bool
-
 config HAVE_MEMBLOCK_PHYS_MAP
 	bool
 
@@ -483,10 +482,6 @@ config HAVE_GUP_FAST
 	depends on MMU
 	bool
 
-# Enable memblock support for scratch memory which is needed for kexec handover
-config MEMBLOCK_KHO_SCRATCH
-	bool
-
 # Don't discard allocated memory used to track "memory" and "reserved" memblocks
 # after early boot, so it can still be used to test for validity of memory.
 # Also, memblocks are updated with memory hot(un)plug.
@@ -1225,6 +1220,7 @@ config ZONE_DMA32
 config ZONE_DEVICE
 	bool "Device memory (pmem, HMM, etc...) hotplug support"
 	depends on MEMORY_HOTREMOVE
+	select VMEMMAP_OPTIMIZATION if ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
 	select XARRAY_MULTI
 
 	help
@@ -1292,24 +1288,19 @@ config PERCPU_STATS
 	  be used to help understand percpu memory usage.
 
 config GUP_TEST
-	bool "Enable infrastructure for get_user_pages()-related unit tests"
+	bool "Enable infrastructure for get_user_pages()-related unit tests and benchmarks"
 	depends on DEBUG_FS
 	help
 	  Provides /sys/kernel/debug/gup_test, which in turn provides a way
-	  to make ioctl calls that can launch kernel-based unit tests for
-	  the get_user_pages*() and pin_user_pages*() family of API calls.
+	  to make ioctl calls that can launch kernel-based unit tests and
+	  benchmarks for the get_user_pages*() and pin_user_pages*() families
+	  of API calls.
 
-	  These tests include benchmark testing of the _fast variants of
-	  get_user_pages*() and pin_user_pages*(), as well as smoke tests of
+	  These include benchmark testing of the _fast variants of
+	  get_user_pages*() and pin_user_pages*(), as well as tests of
 	  the non-_fast variants.
 
-	  There is also a sub-test that allows running dump_page() on any
-	  of up to eight pages (selected by command line args) within the
-	  range of user-space addresses. These pages are either pinned via
-	  pin_user_pages*(), or pinned via get_user_pages*(), as specified
-	  by other command line arguments.
-
-	  See tools/testing/selftests/mm/gup_test.c
+	  See tools/testing/selftests/mm/gup.c and tools/mm/gup_bench.c.
 
 comment "GUP_TEST needs to have DEBUG_FS enabled"
 	depends on !GUP_TEST && !DEBUG_FS
@@ -1430,18 +1421,6 @@ config LRU_GEN_WALKS_MMU
 	depends on LRU_GEN && ARCH_HAS_HW_PTE_YOUNG
 # }
 
-config ARCH_SUPPORTS_PER_VMA_LOCK
-       def_bool n
-
-config PER_VMA_LOCK
-	def_bool y
-	depends on ARCH_SUPPORTS_PER_VMA_LOCK && MMU && SMP
-	help
-	  Allow per-vma locking during page fault handling.
-
-	  This feature allows locking each virtual memory area separately when
-	  handling page faults instead of taking mmap_lock.
-
 config LOCK_MM_AND_FIND_VMA
 	bool
 	depends on !STACK_GROWSUP
@@ -1482,7 +1461,7 @@ config HAVE_ARCH_TLB_REMOVE_TABLE
 
 config PT_RECLAIM
 	def_bool y
-	depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE
+	depends on MMU && !HAVE_ARCH_TLB_REMOVE_TABLE
 	help
 	  Try to reclaim empty user page table pages in paths other than munmap
 	  and exit_mmap path.
@@ -1511,6 +1490,47 @@ config LAZY_MMU_MODE_KUNIT_TEST
 
 	  If unsure, say N.
 
+config ARCH_HAS_CHECK_WX
+	bool
+
+config CHECK_WX
+	bool "Warn on W+X mappings at boot"
+	default STRICT_KERNEL_RWX
+	depends on ARCH_HAS_CHECK_WX
+	depends on ARCH_HAS_PTDUMP
+	depends on MMU
+	select PTDUMP
+	help
+	  Generate a warning if any W+X mappings are found at boot.
+
+	  This is useful for discovering cases where the kernel is leaving W+X
+	  mappings after applying NX, as such mappings are a security risk.
+
+	  Look for a message in dmesg output like this:
+
+	    <arch>/mm: Checked W+X mappings: passed, no W+X pages found.
+
+	  or like this, if the check failed:
+
+	    <arch>/mm: Checked W+X mappings: failed, <N> W+X pages found.
+
+	  Note that even if the check fails, your kernel is possibly
+	  still fine, as W+X mappings are not a security hole in
+	  themselves, what they do is that they make the exploitation
+	  of other unfixed kernel bugs easier.
+
+	  There is no runtime or memory usage effect of this option
+	  once the kernel has booted up - it's a one time check.
+
+	  If in doubt, say "Y".
+
+config ARCH_HAS_PTDUMP
+	bool
+
+config PTDUMP
+	bool
+
+
 source "mm/damon/Kconfig"
 
 endmenu
diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug
index 15dca19dd07da9..9ae75229c76da1 100644
--- a/mm/Kconfig.debug
+++ b/mm/Kconfig.debug
@@ -180,45 +180,6 @@ config DEBUG_RODATA_TEST
 	help
 	  This option enables a testcase for the setting rodata read-only.
 
-config ARCH_HAS_DEBUG_WX
-	bool
-
-config DEBUG_WX
-	bool "Warn on W+X mappings at boot"
-	depends on ARCH_HAS_DEBUG_WX
-	depends on ARCH_HAS_PTDUMP
-	depends on MMU
-	select PTDUMP
-	help
-	  Generate a warning if any W+X mappings are found at boot.
-
-	  This is useful for discovering cases where the kernel is leaving W+X
-	  mappings after applying NX, as such mappings are a security risk.
-
-	  Look for a message in dmesg output like this:
-
-	    <arch>/mm: Checked W+X mappings: passed, no W+X pages found.
-
-	  or like this, if the check failed:
-
-	    <arch>/mm: Checked W+X mappings: failed, <N> W+X pages found.
-
-	  Note that even if the check fails, your kernel is possibly
-	  still fine, as W+X mappings are not a security hole in
-	  themselves, what they do is that they make the exploitation
-	  of other unfixed kernel bugs easier.
-
-	  There is no runtime or memory usage effect of this option
-	  once the kernel has booted up - it's a one time check.
-
-	  If in doubt, say "Y".
-
-config ARCH_HAS_PTDUMP
-	bool
-
-config PTDUMP
-	bool
-
 config PTDUMP_DEBUGFS
 	bool "Export kernel pagetable layout to userspace via debugfs"
 	depends on DEBUG_KERNEL
@@ -310,7 +271,6 @@ config DEBUG_KMEMLEAK_VERBOSE
 
 config PER_VMA_LOCK_STATS
 	bool "Statistics for per-vma locks"
-	depends on PER_VMA_LOCK
 	help
 	  Say Y here to enable success, retry and failure counters of page
 	  faults handled under protection of per-vma locks. When enabled, the
diff --git a/mm/Makefile b/mm/Makefile
index e7245cb88c6651..2a3ec53d62eef6 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -55,7 +55,8 @@ obj-y			:= filemap.o mempool.o oom_kill.o fadvise.o \
 			   mm_init.o percpu.o slab_common.o \
 			   compaction.o show_mem.o \
 			   interval_tree.o list_lru.o workingset.o \
-			   debug.o gup.o mmap_lock.o vma_init.o $(mmu-y)
+			   debug.o gup.o mmap_lock.o vma_init.o char-mem.o \
+			   $(mmu-y)
 
 # Give 'page_alloc' its own module-parameter namespace
 page-alloc-y := page_alloc.o
diff --git a/mm/alloc_tag.c b/mm/alloc_tag.c
index b3341031047796..82e2c3448dcf2c 100644
--- a/mm/alloc_tag.c
+++ b/mm/alloc_tag.c
@@ -621,7 +621,7 @@ void __init alloc_tag_sec_init(void)
 	kernel_tags.count = last_codetag - kernel_tags.first_tag;
 
 	/* Check if kernel tags fit into page flags */
-	if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
+	if (CODETAG_ID_FIRST + kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
 		shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
 		pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
 			kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index cecbcf9060a653..18e999053bae0b 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -910,8 +910,9 @@ static void cleanup_offline_cgwbs_workfn(struct work_struct *work)
 			continue;
 
 		spin_unlock_irq(&cgwb_lock);
-		while (cleanup_offline_cgwb(wb))
-			cond_resched();
+		do {
+			cond_resched_tasks_rcu_qs();
+		} while (cleanup_offline_cgwb(wb));
 		spin_lock_irq(&cgwb_lock);
 
 		wb_put(wb);
diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c
index 716df49d76477d..d8f579c285600d 100644
--- a/mm/bpf_memcontrol.c
+++ b/mm/bpf_memcontrol.c
@@ -8,6 +8,8 @@
 #include <linux/memcontrol.h>
 #include <linux/bpf.h>
 
+#include "internal.h"
+
 __bpf_kfunc_start_defs();
 
 /**
@@ -159,6 +161,51 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg)
 	mem_cgroup_flush_stats(memcg);
 }
 
+/**
+ * bpf_proactive_reclaim - proactively reclaim memory from a memory cgroup
+ * @memcg: the target memory cgroup to reclaim from.
+ * @size: the amount of memory to reclaim, in bytes, clamped to
+ *        MEMCG_CHARGE_BATCH.
+ * @swappiness: the reclaim swappiness, in the range [MIN_SWAPPINESS,
+ *              MAX_SWAPPINESS], or one of the special modes: -1 to use
+ *              the memcg's own swappiness, or SWAPPINESS_ANON_ONLY to
+ *              reclaim only anon folios.
+ *
+ * Performs one proactive reclaim pass on @memcg, like a write to
+ * memory.reclaim but without retrying until @size is reached. Call it
+ * repeatedly to reclaim more than one batch.
+ *
+ * Only available to BPF_PROG_TYPE_SYSCALL, because other sleepable programs
+ * may run with filesystem locks held, which the reclaim path can deadlock
+ * on via filesystem shrinkers.
+ *
+ * Return: The amount of memory reclaimed, in bytes, or a negative error.
+ */
+__bpf_kfunc long bpf_proactive_reclaim(struct mem_cgroup *memcg,
+				       unsigned long size,
+				       int swappiness)
+{
+	unsigned long nr_reclaimed;
+	unsigned long nr_pages;
+
+	if (swappiness != -1 && swappiness != SWAPPINESS_ANON_ONLY) {
+		if (swappiness < MIN_SWAPPINESS || swappiness > MAX_SWAPPINESS)
+			return -EINVAL;
+	}
+
+	if (size < PAGE_SIZE)
+		return -EINVAL;
+
+	nr_pages = min(size / PAGE_SIZE, (unsigned long)MEMCG_CHARGE_BATCH);
+
+	nr_reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages, GFP_KERNEL,
+						    MEMCG_RECLAIM_MAY_SWAP |
+						    MEMCG_RECLAIM_PROACTIVE,
+						    swappiness == -1 ? NULL : &swappiness);
+
+	return nr_reclaimed * PAGE_SIZE;
+}
+
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(bpf_memcontrol_kfuncs)
@@ -174,19 +221,35 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE)
 
 BTF_KFUNCS_END(bpf_memcontrol_kfuncs)
 
+BTF_KFUNCS_START(bpf_memcontrol_reclaim_kfuncs)
+BTF_ID_FLAGS(func, bpf_proactive_reclaim, KF_SLEEPABLE)
+BTF_KFUNCS_END(bpf_memcontrol_reclaim_kfuncs)
+
 static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = {
 	.owner          = THIS_MODULE,
 	.set            = &bpf_memcontrol_kfuncs,
 };
 
+static const struct btf_kfunc_id_set bpf_memcontrol_reclaim_kfunc_set = {
+	.owner          = THIS_MODULE,
+	.set            = &bpf_memcontrol_reclaim_kfuncs,
+};
+
 static int __init bpf_memcontrol_init(void)
 {
 	int err;
 
 	err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC,
 					&bpf_memcontrol_kfunc_set);
-	if (err)
+	if (err) {
 		pr_warn("error while registering bpf memcontrol kfuncs: %d", err);
+		return err;
+	}
+
+	err = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
+					&bpf_memcontrol_reclaim_kfunc_set);
+	if (err)
+		pr_warn("error registering bpf reclaim kfuncs: %d\n", err);
 
 	return err;
 }
diff --git a/mm/char-mem.c b/mm/char-mem.c
new file mode 100644
index 00000000000000..e9f94d2b5133ef
--- /dev/null
+++ b/mm/char-mem.c
@@ -0,0 +1,787 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  mm/char-mem.c
+ *
+ *  Copyright (C) 1991, 1992  Linus Torvalds
+ *
+ *  Added devfs support.
+ *    Jan-11-1998, C. Scott Ananian <cananian@alumni.princeton.edu>
+ *  Shared /dev/zero mmapping support, Feb 2000, Kanoj Sarcar <kanoj@sgi.com>
+ */
+
+#include <linux/mm.h>
+#include <linux/miscdevice.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mman.h>
+#include <linux/random.h>
+#include <linux/init.h>
+#include <linux/tty.h>
+#include <linux/capability.h>
+#include <linux/ptrace.h>
+#include <linux/device.h>
+#include <linux/highmem.h>
+#include <linux/backing-dev.h>
+#include <linux/shmem_fs.h>
+#include <linux/splice.h>
+#include <linux/pfn.h>
+#include <linux/export.h>
+#include <linux/io.h>
+#include <linux/uio.h>
+#include <linux/uaccess.h>
+#include <linux/security.h>
+
+#include "internal.h"
+
+#define DEVMEM_MINOR	1
+#define DEVPORT_MINOR	4
+
+static inline unsigned long size_inside_page(unsigned long start,
+					     unsigned long size)
+{
+	unsigned long sz;
+
+	sz = PAGE_SIZE - (start & (PAGE_SIZE - 1));
+
+	return min(sz, size);
+}
+
+#ifndef ARCH_HAS_VALID_PHYS_ADDR_RANGE
+static inline int valid_phys_addr_range(phys_addr_t addr, size_t count)
+{
+	return addr + count <= __pa(high_memory);
+}
+
+static inline int valid_mmap_phys_addr_range(unsigned long pfn, size_t size)
+{
+	return 1;
+}
+#endif
+
+#ifdef CONFIG_STRICT_DEVMEM
+static inline int page_is_allowed(unsigned long pfn)
+{
+	return devmem_is_allowed(pfn);
+}
+#else
+static inline int page_is_allowed(unsigned long pfn)
+{
+	return 1;
+}
+#endif
+
+static inline bool should_stop_iteration(void)
+{
+	if (need_resched())
+		cond_resched();
+	return signal_pending(current);
+}
+
+/*
+ * This funcion reads the *physical* memory. The f_pos points directly to the
+ * memory location.
+ */
+static ssize_t read_mem(struct file *file, char __user *buf,
+			size_t count, loff_t *ppos)
+{
+	phys_addr_t p = *ppos;
+	ssize_t read, sz;
+	void *ptr;
+	char *bounce;
+	int err;
+
+	if (p != *ppos)
+		return 0;
+
+	if (!valid_phys_addr_range(p, count))
+		return -EFAULT;
+	read = 0;
+#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPED
+	/* we don't have page 0 mapped on sparc and m68k.. */
+	if (p < PAGE_SIZE) {
+		sz = size_inside_page(p, count);
+		if (sz > 0) {
+			if (clear_user(buf, sz))
+				return -EFAULT;
+			buf += sz;
+			p += sz;
+			count -= sz;
+			read += sz;
+		}
+	}
+#endif
+
+	bounce = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!bounce)
+		return -ENOMEM;
+
+	while (count > 0) {
+		unsigned long remaining;
+		int allowed, probe;
+
+		sz = size_inside_page(p, count);
+
+		err = -EPERM;
+		allowed = page_is_allowed(p >> PAGE_SHIFT);
+		if (!allowed)
+			goto failed;
+
+		err = -EFAULT;
+		if (allowed == 2) {
+			/* Show zeros for restricted memory. */
+			remaining = clear_user(buf, sz);
+		} else {
+			/*
+			 * On ia64 if a page has been mapped somewhere as
+			 * uncached, then it must also be accessed uncached
+			 * by the kernel or data corruption may occur.
+			 */
+			ptr = xlate_dev_mem_ptr(p);
+			if (!ptr)
+				goto failed;
+
+			probe = copy_from_kernel_nofault(bounce, ptr, sz);
+			unxlate_dev_mem_ptr(p, ptr);
+			if (probe)
+				goto failed;
+
+			remaining = copy_to_user(buf, bounce, sz);
+		}
+
+		if (remaining)
+			goto failed;
+
+		buf += sz;
+		p += sz;
+		count -= sz;
+		read += sz;
+		if (should_stop_iteration())
+			break;
+	}
+	kfree(bounce);
+
+	*ppos += read;
+	return read;
+
+failed:
+	kfree(bounce);
+	return err;
+}
+
+static ssize_t write_mem(struct file *file, const char __user *buf,
+			 size_t count, loff_t *ppos)
+{
+	phys_addr_t p = *ppos;
+	ssize_t written, sz;
+	unsigned long copied;
+	void *ptr;
+
+	if (p != *ppos)
+		return -EFBIG;
+
+	if (!valid_phys_addr_range(p, count))
+		return -EFAULT;
+
+	written = 0;
+
+#ifdef __ARCH_HAS_NO_PAGE_ZERO_MAPPED
+	/* we don't have page 0 mapped on sparc and m68k.. */
+	if (p < PAGE_SIZE) {
+		sz = size_inside_page(p, count);
+		/* Hmm. Do something? */
+		buf += sz;
+		p += sz;
+		count -= sz;
+		written += sz;
+	}
+#endif
+
+	while (count > 0) {
+		int allowed;
+
+		sz = size_inside_page(p, count);
+
+		allowed = page_is_allowed(p >> PAGE_SHIFT);
+		if (!allowed)
+			return -EPERM;
+
+		/* Skip actual writing when a page is marked as restricted. */
+		if (allowed == 1) {
+			/*
+			 * On ia64 if a page has been mapped somewhere as
+			 * uncached, then it must also be accessed uncached
+			 * by the kernel or data corruption may occur.
+			 */
+			ptr = xlate_dev_mem_ptr(p);
+			if (!ptr) {
+				if (written)
+					break;
+				return -EFAULT;
+			}
+
+			copied = copy_from_user(ptr, buf, sz);
+			unxlate_dev_mem_ptr(p, ptr);
+			if (copied) {
+				written += sz - copied;
+				if (written)
+					break;
+				return -EFAULT;
+			}
+		}
+
+		buf += sz;
+		p += sz;
+		count -= sz;
+		written += sz;
+		if (should_stop_iteration())
+			break;
+	}
+
+	*ppos += written;
+	return written;
+}
+
+int __weak phys_mem_access_prot_allowed(struct file *file,
+	unsigned long pfn, unsigned long size, pgprot_t *vma_prot)
+{
+	return 1;
+}
+
+#ifndef __HAVE_PHYS_MEM_ACCESS_PROT
+
+/*
+ * Architectures vary in how they handle caching for addresses
+ * outside of main memory.
+ *
+ */
+#ifdef pgprot_noncached
+static int uncached_access(struct file *file, phys_addr_t addr)
+{
+	/*
+	 * Accessing memory above the top the kernel knows about or through a
+	 * file pointer
+	 * that was marked O_DSYNC will be done non-cached.
+	 */
+	if (file->f_flags & O_DSYNC)
+		return 1;
+	return addr >= __pa(high_memory);
+}
+#endif
+
+static pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
+				     unsigned long size, pgprot_t vma_prot)
+{
+#ifdef pgprot_noncached
+	phys_addr_t offset = pfn << PAGE_SHIFT;
+
+	if (uncached_access(file, offset))
+		return pgprot_noncached(vma_prot);
+#endif
+	return vma_prot;
+}
+#endif
+
+#ifndef CONFIG_MMU
+static unsigned long get_unmapped_area_mem(struct file *file,
+					   unsigned long addr,
+					   unsigned long len,
+					   unsigned long pgoff,
+					   unsigned long flags)
+{
+	if (!valid_mmap_phys_addr_range(pgoff, len))
+		return (unsigned long) -EINVAL;
+	return pgoff << PAGE_SHIFT;
+}
+
+/* permit direct mmap, for read, write or exec */
+static unsigned memory_mmap_capabilities(struct file *file)
+{
+	return NOMMU_MAP_DIRECT |
+		NOMMU_MAP_READ | NOMMU_MAP_WRITE | NOMMU_MAP_EXEC;
+}
+
+static unsigned zero_mmap_capabilities(struct file *file)
+{
+	return NOMMU_MAP_COPY;
+}
+
+/* can't do an in-place private mapping if there's no MMU */
+static inline int private_mapping_ok(struct vm_area_desc *desc)
+{
+	return is_nommu_shared_vma_flags(&desc->vma_flags);
+}
+#else
+
+static inline int private_mapping_ok(struct vm_area_desc *desc)
+{
+	return 1;
+}
+#endif
+
+static const struct vm_operations_struct mmap_mem_ops = {
+#ifdef CONFIG_HAVE_IOREMAP_PROT
+	.access = generic_access_phys
+#endif
+};
+
+static int mmap_mem_prepare(struct vm_area_desc *desc)
+{
+	struct file *file = desc->file;
+	const size_t size = vma_desc_size(desc);
+	const phys_addr_t offset = (phys_addr_t)desc->pgoff << PAGE_SHIFT;
+
+	/* Does it even fit in phys_addr_t? */
+	if (offset >> PAGE_SHIFT != desc->pgoff)
+		return -EINVAL;
+
+	/* It's illegal to wrap around the end of the physical address space. */
+	if (offset + (phys_addr_t)size - 1 < offset)
+		return -EINVAL;
+
+	if (!valid_mmap_phys_addr_range(desc->pgoff, size))
+		return -EINVAL;
+
+	if (!private_mapping_ok(desc))
+		return -ENOSYS;
+
+	if (!range_is_allowed(desc->pgoff, size))
+		return -EPERM;
+
+	if (!phys_mem_access_prot_allowed(file, desc->pgoff, size,
+					  &desc->page_prot))
+		return -EINVAL;
+
+	desc->page_prot = phys_mem_access_prot(file, desc->pgoff,
+					       size,
+					       desc->page_prot);
+
+	desc->vm_ops = &mmap_mem_ops;
+
+	/* Remap-pfn-range will mark the range with the I/O flag. */
+	mmap_action_remap_full(desc, desc->pgoff);
+	desc->action.error_override = -EAGAIN;
+
+	return 0;
+}
+
+#ifdef CONFIG_DEVPORT
+static ssize_t read_port(struct file *file, char __user *buf,
+			 size_t count, loff_t *ppos)
+{
+	unsigned long i = *ppos;
+	char __user *tmp = buf;
+
+	if (!access_ok(buf, count))
+		return -EFAULT;
+	while (count-- > 0 && i < 65536) {
+		if (__put_user(inb(i), tmp) < 0)
+			return -EFAULT;
+		i++;
+		tmp++;
+	}
+	*ppos = i;
+	return tmp-buf;
+}
+
+static ssize_t write_port(struct file *file, const char __user *buf,
+			  size_t count, loff_t *ppos)
+{
+	unsigned long i = *ppos;
+	const char __user *tmp = buf;
+
+	if (!access_ok(buf, count))
+		return -EFAULT;
+	while (count-- > 0 && i < 65536) {
+		char c;
+
+		if (__get_user(c, tmp)) {
+			if (tmp > buf)
+				break;
+			return -EFAULT;
+		}
+		outb(c, i);
+		i++;
+		tmp++;
+	}
+	*ppos = i;
+	return tmp-buf;
+}
+#endif
+
+static ssize_t read_null(struct file *file, char __user *buf,
+			 size_t count, loff_t *ppos)
+{
+	return 0;
+}
+
+static ssize_t write_null(struct file *file, const char __user *buf,
+			  size_t count, loff_t *ppos)
+{
+	return count;
+}
+
+static ssize_t read_iter_null(struct kiocb *iocb, struct iov_iter *to)
+{
+	return 0;
+}
+
+static ssize_t write_iter_null(struct kiocb *iocb, struct iov_iter *from)
+{
+	size_t count = iov_iter_count(from);
+	iov_iter_advance(from, count);
+	return count;
+}
+
+static int pipe_to_null(struct pipe_inode_info *info, struct pipe_buffer *buf,
+			struct splice_desc *sd)
+{
+	return sd->len;
+}
+
+static ssize_t splice_write_null(struct pipe_inode_info *pipe, struct file *out,
+				 loff_t *ppos, size_t len, unsigned int flags)
+{
+	return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_null);
+}
+
+static int uring_cmd_null(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
+{
+	return 0;
+}
+
+static ssize_t read_iter_zero(struct kiocb *iocb, struct iov_iter *iter)
+{
+	size_t written = 0;
+
+	while (iov_iter_count(iter)) {
+		size_t chunk = iov_iter_count(iter), n;
+
+		if (chunk > PAGE_SIZE)
+			chunk = PAGE_SIZE;	/* Just for latency reasons */
+		n = iov_iter_zero(chunk, iter);
+		if (!n && iov_iter_count(iter))
+			return written ? written : -EFAULT;
+		written += n;
+		if (signal_pending(current))
+			return written ? written : -ERESTARTSYS;
+		if (!need_resched())
+			continue;
+		if (iocb->ki_flags & IOCB_NOWAIT)
+			return written ? written : -EAGAIN;
+		cond_resched();
+	}
+	return written;
+}
+
+static ssize_t read_zero(struct file *file, char __user *buf,
+			 size_t count, loff_t *ppos)
+{
+	size_t cleared = 0;
+
+	while (count) {
+		size_t chunk = min_t(size_t, count, PAGE_SIZE);
+		size_t left;
+
+		left = clear_user(buf + cleared, chunk);
+		if (unlikely(left)) {
+			cleared += (chunk - left);
+			if (!cleared)
+				return -EFAULT;
+			break;
+		}
+		cleared += chunk;
+		count -= chunk;
+
+		if (signal_pending(current))
+			break;
+		cond_resched();
+	}
+
+	return cleared;
+}
+
+static int mmap_zero_prepare(struct vm_area_desc *desc)
+{
+#ifndef CONFIG_MMU
+	return -ENOSYS;
+#endif
+	if (vma_desc_test(desc, VMA_MAYSHARE_BIT))
+		return shmem_zero_setup_desc(desc);
+
+	/* MAP_PRIVATE semantics are taken care of for us by core mm. */
+	return 0;
+}
+
+#ifndef CONFIG_MMU
+static unsigned long get_unmapped_area_zero(struct file *file,
+				unsigned long addr, unsigned long len,
+				unsigned long pgoff, unsigned long flags)
+{
+	return -ENOSYS;
+}
+#else
+static unsigned long get_unmapped_area_zero(struct file *file,
+				unsigned long addr, unsigned long len,
+				unsigned long pgoff, unsigned long flags)
+{
+	if (flags & MAP_SHARED) {
+		/*
+		 * mmap_zero_prepare() will call shmem_zero_setup() to create a
+		 * file, so use shmem's get_unmapped_area in case it can be
+		 * huge; and pass NULL for file as in mmap.c's
+		 * get_unmapped_area(), so as not to confuse shmem with our
+		 * handle on "/dev/zero".
+		 */
+		return shmem_get_unmapped_area(NULL, addr, len, pgoff, flags);
+	}
+
+	/*
+	 * Otherwise flags & MAP_PRIVATE: with no shmem object beneath it,
+	 * attempt to map aligned to huge page size if possible, otherwise we
+	 * fall back to system page size mappings.
+	 */
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	return thp_get_unmapped_area(file, addr, len, pgoff, flags);
+#else
+	return mm_get_unmapped_area(file, addr, len, pgoff, flags);
+#endif
+}
+#endif /* CONFIG_MMU */
+
+static ssize_t write_full(struct file *file, const char __user *buf,
+			  size_t count, loff_t *ppos)
+{
+	return -ENOSPC;
+}
+
+/*
+ * Special lseek() function for /dev/null and /dev/zero.  Most notably, you
+ * can fopen() both devices with "a" now.  This was previously impossible.
+ * -- SRB.
+ */
+static loff_t null_lseek(struct file *file, loff_t offset, int orig)
+{
+	return file->f_pos = 0;
+}
+
+/*
+ * The memory devices use the full 32/64 bits of the offset, and so we cannot
+ * check against negative addresses: they are ok. The return value is weird,
+ * though, in that case (0).
+ *
+ * also note that seeking relative to the "end of file" isn't supported:
+ * it has no meaning, so it returns -EINVAL.
+ */
+static loff_t memory_lseek(struct file *file, loff_t offset, int orig)
+{
+	loff_t ret;
+
+	inode_lock(file_inode(file));
+	switch (orig) {
+	case SEEK_CUR:
+		offset += file->f_pos;
+		fallthrough;
+	case SEEK_SET:
+		/* to avoid userland mistaking f_pos=-9 as -EBADF=-9 */
+		if ((unsigned long long)offset >= -MAX_ERRNO) {
+			ret = -EOVERFLOW;
+			break;
+		}
+		file->f_pos = offset;
+		ret = file->f_pos;
+		force_successful_syscall_return();
+		break;
+	default:
+		ret = -EINVAL;
+	}
+	inode_unlock(file_inode(file));
+	return ret;
+}
+
+static int open_port(struct inode *inode, struct file *filp)
+{
+	int rc;
+
+	if (!capable(CAP_SYS_RAWIO))
+		return -EPERM;
+
+	rc = security_locked_down(LOCKDOWN_DEV_MEM);
+	if (rc)
+		return rc;
+
+	if (iminor(inode) != DEVMEM_MINOR)
+		return 0;
+
+	/*
+	 * Use a unified address space to have a single point to manage
+	 * revocations when drivers want to take over a /dev/mem mapped
+	 * range.
+	 */
+	filp->f_mapping = iomem_get_mapping();
+
+	return 0;
+}
+
+#define zero_lseek	null_lseek
+#define full_lseek      null_lseek
+#define write_zero	write_null
+#define write_iter_zero	write_iter_null
+#define splice_write_zero	splice_write_null
+#define open_mem	open_port
+
+static const struct file_operations __maybe_unused mem_fops = {
+	.llseek		= memory_lseek,
+	.read		= read_mem,
+	.write		= write_mem,
+	.mmap_prepare	= mmap_mem_prepare,
+	.open		= open_mem,
+#ifndef CONFIG_MMU
+	.get_unmapped_area = get_unmapped_area_mem,
+	.mmap_capabilities = memory_mmap_capabilities,
+#endif
+	.fop_flags	= FOP_UNSIGNED_OFFSET,
+};
+
+static const struct file_operations null_fops = {
+	.llseek		= null_lseek,
+	.read		= read_null,
+	.write		= write_null,
+	.read_iter	= read_iter_null,
+	.write_iter	= write_iter_null,
+	.splice_write	= splice_write_null,
+	.uring_cmd	= uring_cmd_null,
+};
+
+#ifdef CONFIG_DEVPORT
+static const struct file_operations port_fops = {
+	.llseek		= memory_lseek,
+	.read		= read_port,
+	.write		= write_port,
+	.open		= open_port,
+};
+#endif
+
+static const struct file_operations zero_fops = {
+	.llseek		= zero_lseek,
+	.write		= write_zero,
+	.read_iter	= read_iter_zero,
+	.read		= read_zero,
+	.write_iter	= write_iter_zero,
+	.splice_read	= copy_splice_read,
+	.splice_write	= splice_write_zero,
+	.mmap_prepare	= mmap_zero_prepare,
+	.get_unmapped_area = get_unmapped_area_zero,
+#ifndef CONFIG_MMU
+	.mmap_capabilities = zero_mmap_capabilities,
+#endif
+};
+
+static const struct file_operations full_fops = {
+	.llseek		= full_lseek,
+	.read_iter	= read_iter_zero,
+	.write		= write_full,
+	.splice_read	= copy_splice_read,
+};
+
+static const struct memdev {
+	const char *name;
+	const struct file_operations *fops;
+	fmode_t fmode;
+	umode_t mode;
+} devlist[] = {
+#ifdef CONFIG_DEVMEM
+	[DEVMEM_MINOR] = { "mem", &mem_fops, 0, 0 },
+#endif
+	[3] = { "null", &null_fops, FMODE_NOWAIT, 0666 },
+#ifdef CONFIG_DEVPORT
+	[4] = { "port", &port_fops, 0, 0 },
+#endif
+	[5] = { "zero", &zero_fops, FMODE_NOWAIT, 0666 },
+	[7] = { "full", &full_fops, 0, 0666 },
+	[8] = { "random", &random_fops, FMODE_NOWAIT, 0666 },
+	[9] = { "urandom", &urandom_fops, FMODE_NOWAIT, 0666 },
+#ifdef CONFIG_PRINTK
+	[11] = { "kmsg", &kmsg_fops, 0, 0644 },
+#endif
+};
+
+/**
+ * file_is_dev_zero() - is the specified @file associated with the /dev/zero
+ * driver?
+ * @file: File to test.
+ * Returns: true if it is, false otherwise.
+ */
+bool file_is_dev_zero(const struct file *file)
+{
+	return file && file->f_op == &zero_fops;
+}
+
+static int memory_open(struct inode *inode, struct file *filp)
+{
+	int minor;
+	const struct memdev *dev;
+
+	minor = iminor(inode);
+	if (minor >= ARRAY_SIZE(devlist))
+		return -ENXIO;
+
+	dev = &devlist[minor];
+	if (!dev->fops)
+		return -ENXIO;
+
+	filp->f_op = dev->fops;
+	filp->f_mode |= dev->fmode;
+
+	if (dev->fops->open)
+		return dev->fops->open(inode, filp);
+
+	return 0;
+}
+
+static const struct file_operations memory_fops = {
+	.open = memory_open,
+	.llseek = noop_llseek,
+};
+
+static char *mem_devnode(const struct device *dev, umode_t *mode)
+{
+	if (mode && devlist[MINOR(dev->devt)].mode)
+		*mode = devlist[MINOR(dev->devt)].mode;
+	return NULL;
+}
+
+static const struct class mem_class = {
+	.name		= "mem",
+	.devnode	= mem_devnode,
+};
+
+static int __init chr_dev_init(void)
+{
+	int retval;
+	int minor;
+
+	if (register_chrdev(MEM_MAJOR, "mem", &memory_fops))
+		printk("unable to get major %d for memory devs\n", MEM_MAJOR);
+
+	retval = class_register(&mem_class);
+	if (retval)
+		return retval;
+
+	for (minor = 1; minor < ARRAY_SIZE(devlist); minor++) {
+		if (!devlist[minor].name)
+			continue;
+
+		/*
+		 * Create /dev/port?
+		 */
+		if ((minor == DEVPORT_MINOR) && !arch_has_dev_port())
+			continue;
+
+		device_create(&mem_class, NULL, MKDEV(MEM_MAJOR, minor),
+			      NULL, devlist[minor].name);
+	}
+
+	return tty_init();
+}
+
+fs_initcall(chr_dev_init);
diff --git a/mm/cma.h b/mm/cma.h
index ab6d39898ea52e..68e574b0f95187 100644
--- a/mm/cma.h
+++ b/mm/cma.h
@@ -2,14 +2,24 @@
 #ifndef __MM_CMA_H__
 #define __MM_CMA_H__
 
+#include <linux/atomic.h>
 #include <linux/cma.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/spinlock.h>
+
+#ifdef CONFIG_CMA_DEBUGFS
 #include <linux/debugfs.h>
+#endif
+
+#ifdef CONFIG_CMA_SYSFS
 #include <linux/kobject.h>
 
 struct cma_kobject {
 	struct kobject kobj;
 	struct cma *cma;
 };
+#endif
 
 /*
  * Multi-range support. This can be useful if the size of the allocation
@@ -38,10 +48,10 @@ struct cma_memrange {
 #define CMA_MAX_RANGES 8
 
 struct cma {
-	unsigned long   count;
-	unsigned long	available_count;
+	unsigned long count;
+	unsigned long available_count;
 	unsigned int order_per_bit; /* Order of pages represented by one bit */
-	spinlock_t	lock;
+	spinlock_t lock;
 	struct mutex alloc_mutex;
 #ifdef CONFIG_CMA_DEBUGFS
 	struct hlist_head mem_head;
@@ -87,10 +97,11 @@ void cma_sysfs_account_fail_pages(struct cma *cma, unsigned long nr_pages);
 void cma_sysfs_account_release_pages(struct cma *cma, unsigned long nr_pages);
 #else
 static inline void cma_sysfs_account_success_pages(struct cma *cma,
-						   unsigned long nr_pages) {};
+						   unsigned long nr_pages) {}
 static inline void cma_sysfs_account_fail_pages(struct cma *cma,
-						unsigned long nr_pages) {};
+						unsigned long nr_pages) {}
 static inline void cma_sysfs_account_release_pages(struct cma *cma,
-						   unsigned long nr_pages) {};
+						   unsigned long nr_pages) {}
 #endif
+
 #endif
diff --git a/mm/collapse.h b/mm/collapse.h
new file mode 100644
index 00000000000000..b19351fbe19b94
--- /dev/null
+++ b/mm/collapse.h
@@ -0,0 +1,162 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __MM_COLLAPSE_H
+#define __MM_COLLAPSE_H
+
+#include <linux/mm.h>
+#include <linux/nodemask.h>
+#include <linux/pgtable.h>
+#include <linux/types.h>
+
+#define COLLAPSE_MAX_PTES_LIMIT		(HPAGE_PMD_NR - 1)
+#define COLLAPSE_MIN_MTHP_ORDER		2
+
+enum scan_result {
+	SCAN_FAIL,
+	SCAN_SUCCEED,
+	SCAN_NO_PTE_TABLE,
+	SCAN_PMD_MAPPED,
+	SCAN_EXCEED_NONE_PTE,
+	SCAN_EXCEED_SWAP_PTE,
+	SCAN_EXCEED_SHARED_PTE,
+	SCAN_PTE_NON_PRESENT,
+	SCAN_PTE_UFFD,
+	SCAN_PTE_MAPPED_HUGEPAGE,
+	SCAN_LACK_REFERENCED_PAGE,
+	SCAN_PAGE_NULL,
+	SCAN_SCAN_ABORT,
+	SCAN_PAGE_COUNT,
+	SCAN_PAGE_LRU,
+	SCAN_PAGE_LOCK,
+	SCAN_PAGE_ANON,
+	SCAN_PAGE_LAZYFREE,
+	SCAN_PAGE_COMPOUND,
+	SCAN_ANY_PROCESS,
+	SCAN_VMA_NULL,
+	SCAN_VMA_CHECK,
+	SCAN_ADDRESS_RANGE,
+	SCAN_DEL_PAGE_LRU,
+	SCAN_ALLOC_HUGE_PAGE_FAIL,
+	SCAN_CGROUP_CHARGE_FAIL,
+	SCAN_TRUNCATED,
+	SCAN_PAGE_HAS_PRIVATE,
+	SCAN_STORE_FAILED,
+	SCAN_COPY_MC,
+	SCAN_PAGE_FILLED,
+	SCAN_PAGE_DIRTY_OR_WRITEBACK,
+};
+
+/* How many PTEs of a window may be missing, swapped out or shared */
+struct collapse_limits {
+	/* Counted over a PMD-sized window; HPAGE_PMD_NR means "no limit" */
+	unsigned int max_ptes_none;
+	unsigned int max_ptes_swap;
+	unsigned int max_ptes_shared;
+};
+
+/* What a collapse is allowed to do, decided by the caller that asks for it */
+struct collapse_policy {
+	/* Limits for a PMD-sized window */
+	struct collapse_limits pmd;
+
+	/*
+	 * Limits for a smaller window.  Its max_ptes_none is either 0 or
+	 * COLLAPSE_MAX_PTES_LIMIT, the latter meaning all but one PTE of the
+	 * window whatever its order; any other value counts as 0.
+	 */
+	struct collapse_limits sub_pmd;
+
+	/* Leave clean lazyfree folios to reclaim rather than collapse them */
+	bool anon_skip_lazyfree;
+
+	/* Refuse an anonymous range with no sign of use */
+	bool anon_require_referenced;
+
+	/* Map the PMD over a file collapse instead of leaving it to a fault */
+	bool file_install_pmd;
+
+	/* Write dirty pages back and retry once instead of refusing them */
+	bool file_writeback_dirty;
+
+	/* How hard to try for a destination folio */
+	gfp_t gfp;
+
+	/* Which VMAs are eligible, as thp_vma_allowable_orders() spells it */
+	enum tva_type tva_type;
+};
+
+struct collapse_control {
+	struct collapse_policy policy;
+
+	/* Num pages scanned per node */
+	u32 node_load[MAX_NUMNODES];
+
+	/* Num pages scanned (see khugepaged_pages_to_scan) */
+	unsigned int progress;
+
+	/* nodemask for allocation fallback */
+	nodemask_t alloc_nmask;
+
+	/* Each bit marks a PTE the scan accepted as a collapse source */
+	DECLARE_BITMAP(eligible_ptes, MAX_PTRS_PER_PTE);
+
+	/*
+	 * What a scan found and the run after it needs.  Live only between the
+	 * two, and read by nobody else.
+	 *
+	 * The file side takes a reference while it still has the VMA, since a
+	 * file collapse works on the page cache and never sees one; the run is
+	 * what gives it back.
+	 */
+	unsigned long scan_orders;
+	int scan_referenced;
+	int scan_unmapped;
+	struct file *scan_file;
+	pgoff_t scan_pgoff;
+};
+
+/* Which orders a VMA may collapse to, zero when it may not collapse at all */
+unsigned long collapse_possible_orders(struct vm_area_struct *vma,
+		vm_flags_t vm_flags, enum tva_type tva_flags);
+
+/*
+ * A caller states what it allows in cc->policy and then hands over one PTE
+ * table's worth of a VMA at a time:
+ *
+ *     collapse_control_init(cc)              once, before the first table
+ *     collapse_scan_pmd(vma, addr, ...)      per table
+ *     collapse_run_pmd(mm, addr, result, cc) when a scan found work
+ *
+ * The caller holds mmap_lock for reading over the scan and passes an address
+ * within @vma, aligned to the PTE table to scan.
+ *
+ * The scan returns with that lock still held.  Almost every table it is
+ * offered has nothing to collapse, so a caller walks a whole VMA under the one
+ * lock it took to get there.  SCAN_SUCCEED means there is
+ * something to collapse.  SCAN_PTE_MAPPED_HUGEPAGE means the page cache
+ * already holds the PMD folio and only the PTE table is left to retract.
+ * Both are work for the run, which is handed what the scan returned; anything
+ * else is why there is nothing to do.
+ *
+ * The run is called without the lock and returns without it, taking what it
+ * needs in between: what it does -- allocate, isolate, copy, flush -- is slow
+ * enough that a writer would wait behind it.  The caller gives the lock up
+ * first, and with it @vma and anything derived under it, so a caller carrying
+ * on has to look up again with collapse_vma_revalidate().  The run revalidates
+ * for itself rather than trusting what the scan saw.
+ *
+ * A scan that found something has to be run: the file side takes a reference on
+ * the file while it still has the VMA to take it from, and the run is what
+ * gives it back.
+ */
+void collapse_control_init(struct collapse_control *cc);
+enum scan_result collapse_scan_pmd(struct vm_area_struct *vma,
+		unsigned long addr, struct collapse_control *cc,
+		unsigned long orders);
+enum scan_result collapse_run_pmd(struct mm_struct *mm, unsigned long addr,
+		enum scan_result result, struct collapse_control *cc);
+enum scan_result collapse_vma_revalidate(struct mm_struct *mm,
+		unsigned long address, bool expect_anon,
+		struct vm_area_struct **vmap, struct collapse_control *cc,
+		unsigned int order);
+
+#endif	/* __MM_COLLAPSE_H */
diff --git a/mm/compaction.c b/mm/compaction.c
index a049415512c672..f1b2060eb20168 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -216,7 +216,7 @@ static unsigned long skip_offline_sections(unsigned long start_pfn)
 	if (online_section_nr(start_nr))
 		return 0;
 
-	while (++start_nr <= __highest_present_section_nr) {
+	while (++start_nr <= __highest_used_section_nr) {
 		if (online_section_nr(start_nr))
 			return section_nr_to_pfn(start_nr);
 	}
@@ -3197,7 +3197,6 @@ static int kcompactd(void *p)
 	long default_timeout = msecs_to_jiffies(HPAGE_FRAG_CHECK_INTERVAL_MSEC);
 	long timeout = default_timeout;
 
-	current->flags |= PF_KCOMPACTD;
 	set_freezable();
 
 	pgdat->kcompactd_max_order = 0;
@@ -3254,11 +3253,15 @@ static int kcompactd(void *p)
 			pgdat->proactive_compact_trigger = false;
 	}
 
-	current->flags &= ~PF_KCOMPACTD;
-
 	return 0;
 }
 
+bool current_is_kcompactd(void)
+{
+	return kthread_func(current) == kcompactd;
+}
+EXPORT_SYMBOL_GPL(current_is_kcompactd);
+
 /*
  * This kcompactd start function will be called by init and node-hot-add.
  * On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 644daf5a165606..5ecbea5d71e1dc 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -10,11 +10,11 @@
 #include <linux/kthread.h>
 #include <linux/memcontrol.h>
 #include <linux/mm.h>
+#include <linux/pid.h>
 #include <linux/psi.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/string.h>
-#include <linux/string_choices.h>
 
 /* for damon_get_folio() used by node eligible memory metrics */
 #include "ops-common.h"
@@ -111,6 +111,63 @@ int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id)
 	return err;
 }
 
+struct damon_prep *damon_new_prep(enum damon_prep_action action)
+{
+	struct damon_prep *prep;
+
+	prep = kmalloc_obj(*prep);
+	if (!prep)
+		return NULL;
+	prep->action = action;
+	INIT_LIST_HEAD(&prep->list);
+	return prep;
+}
+
+void damon_add_prep(struct damon_probe *p, struct damon_prep *prep)
+{
+	list_add_tail(&prep->list, &p->preps);
+}
+
+static void damon_del_prep(struct damon_prep *p)
+{
+	list_del(&p->list);
+}
+
+static void damon_free_prep(struct damon_prep *p)
+{
+	kfree(p);
+}
+
+static void damon_destroy_prep(struct damon_prep *p)
+{
+	damon_del_prep(p);
+	damon_free_prep(p);
+}
+
+static struct damon_prep *damon_nth_prep(int n, struct damon_probe *p)
+{
+	struct damon_prep *prep;
+	int i = 0;
+
+	damon_for_each_prep(prep, p) {
+		if (i++ == n)
+			return prep;
+	}
+	return NULL;
+}
+
+static bool damon_has_prep(struct damon_ctx *c)
+{
+	struct damon_prep *prep;
+	struct damon_probe *probe;
+
+	damon_for_each_probe(probe, c) {
+		damon_for_each_prep(prep, probe)
+			return true;
+	}
+	return false;
+}
+
 struct damon_filter *damon_new_filter(enum damon_filter_type type,
 		bool matching, bool allow)
 {
@@ -167,6 +224,7 @@ struct damon_probe *damon_new_probe(void)
 	if (!p)
 		return NULL;
 	p->weight = 0;
+	INIT_LIST_HEAD(&p->preps);
 	INIT_LIST_HEAD(&p->filters);
 	INIT_LIST_HEAD(&p->list);
 	return p;
@@ -184,8 +242,11 @@ static void damon_del_probe(struct damon_probe *p)
 
 static void damon_free_probe(struct damon_probe *p)
 {
+	struct damon_prep *prep, *prep_next;
 	struct damon_filter *f, *next;
 
+	damon_for_each_prep_safe(prep, prep_next, p)
+		damon_free_prep(prep);
 	damon_for_each_filter_safe(f, next, p)
 		damon_free_filter(f);
 	kfree(p);
@@ -408,11 +469,12 @@ static bool damon_is_last_region(struct damon_region *r,
  * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region.
  * @r:		region to get the weighted sum of.
  * @last:	if the request is for last-window aggregated probe hits.
+ * @mv:		use moving sum.
  * @ctx:	context of &r.
  *
  * Return: the weighted sum of probe hits of the region.
  */
-unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last,
+unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, bool mv,
 		struct damon_ctx *ctx)
 {
 	struct damon_probe *probe;
@@ -422,6 +484,9 @@ unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last,
 	damon_for_each_probe(probe, ctx) {
 		if (last)
 			sum += r->last_probe_hits[i++] * probe->weight;
+		else if (mv)
+			sum += damon_probe_hits_mvsum(i++, r, ctx) *
+				probe->weight;
 		else
 			sum += r->probe_hits[i++] * probe->weight;
 	}
@@ -593,6 +658,7 @@ bool damos_filter_for_ops(enum damos_filter_type type)
 	switch (type) {
 	case DAMOS_FILTER_TYPE_ADDR:
 	case DAMOS_FILTER_TYPE_TARGET:
+	case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM:
 		return false;
 	default:
 		break;
@@ -635,6 +701,8 @@ struct damos_quota_goal *damos_new_quota_goal(
 		return NULL;
 	goal->metric = metric;
 	goal->target_value = target_value;
+	if (metric == DAMOS_QUOTA_SOME_MEM_PSI_US)
+		goal->last_psi_total = U64_MAX;
 	INIT_LIST_HEAD(&goal->list);
 	return goal;
 }
@@ -666,7 +734,7 @@ static bool damos_quota_goals_empty(struct damos_quota *q)
 }
 
 /* initialize fields of @quota that normally API users wouldn't set */
-static struct damos_quota *damos_quota_init(struct damos_quota *quota)
+static void damos_quota_init(struct damos_quota *quota)
 {
 	quota->esz = 0;
 	quota->total_charged_sz = 0;
@@ -676,7 +744,6 @@ static struct damos_quota *damos_quota_init(struct damos_quota *quota)
 	quota->charge_target_from = NULL;
 	quota->charge_addr_from = 0;
 	quota->esz_bp = 0;
-	return quota;
 }
 
 struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
@@ -708,7 +775,8 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
 	scheme->last_applied = NULL;
 	INIT_LIST_HEAD(&scheme->list);
 
-	scheme->quota = *(damos_quota_init(quota));
+	scheme->quota = *quota;
+	damos_quota_init(&scheme->quota);
 	/* quota.goals should be separately set by caller */
 	INIT_LIST_HEAD(&scheme->quota.goals);
 
@@ -795,9 +863,15 @@ void damon_add_target(struct damon_ctx *ctx, struct damon_target *t)
 	list_add_tail(&t->list, &ctx->adaptive_targets);
 }
 
-bool damon_targets_empty(struct damon_ctx *ctx)
+/*
+ * Assign the struct pid of the given pid number to the given target.
+ */
+int damon_set_target_pid(struct damon_target *t, int pid)
 {
-	return list_empty(&ctx->adaptive_targets);
+	t->pid = find_get_pid(pid);
+	if (!t->pid)
+		return -EINVAL;
+	return 0;
 }
 
 static void damon_del_target(struct damon_target *t)
@@ -864,6 +938,8 @@ struct damon_ctx *damon_new_ctx(void)
 	INIT_LIST_HEAD(&ctx->adaptive_targets);
 	INIT_LIST_HEAD(&ctx->schemes);
 
+	ctx->call_controls_obsolete = true;
+	ctx->walk_control_obsolete = true;
 	prandom_seed_state(&ctx->rnd_state, get_random_u64());
 
 	return ctx;
@@ -1122,6 +1198,9 @@ static void damos_commit_quota_goal_union(
 		struct damos_quota_goal *dst, struct damos_quota_goal *src)
 {
 	switch (dst->metric) {
+	case DAMOS_QUOTA_SOME_MEM_PSI_US:
+		dst->last_psi_total = U64_MAX;
+		break;
 	case DAMOS_QUOTA_NODE_MEM_USED_BP:
 	case DAMOS_QUOTA_NODE_MEM_FREE_BP:
 		dst->nid = src->nid;
@@ -1131,20 +1210,25 @@ static void damos_commit_quota_goal_union(
 		dst->nid = src->nid;
 		dst->memcg_id = src->memcg_id;
 		break;
+	case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP:
+		dst->nid = src->nid;
+		break;
 	default:
 		break;
 	}
 }
 
-static void damos_commit_quota_goal(
+static int damos_commit_quota_goal(
 		struct damos_quota_goal *dst, struct damos_quota_goal *src)
 {
+	if (!src->target_value)
+		return  -EINVAL;
 	dst->metric = src->metric;
 	dst->target_value = src->target_value;
 	if (dst->metric == DAMOS_QUOTA_USER_INPUT)
 		dst->current_value = src->current_value;
-	/* keep last_psi_total as is, since it will be updated in next cycle */
 	damos_commit_quota_goal_union(dst, src);
+	return 0;
 }
 
 /**
@@ -1162,14 +1246,17 @@ static void damos_commit_quota_goal(
 int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src)
 {
 	struct damos_quota_goal *dst_goal, *next, *src_goal, *new_goal;
-	int i = 0, j = 0;
+	int i = 0, j = 0, err;
 
 	damos_for_each_quota_goal_safe(dst_goal, next, dst) {
 		src_goal = damos_nth_quota_goal(i++, src);
-		if (src_goal)
-			damos_commit_quota_goal(dst_goal, src_goal);
-		else
+		if (src_goal) {
+			err = damos_commit_quota_goal(dst_goal, src_goal);
+			if (err)
+				return err;
+		} else {
 			damos_destroy_quota_goal(dst_goal);
+		}
 	}
 	damos_for_each_quota_goal_safe(src_goal, next, src) {
 		if (j++ < i)
@@ -1178,7 +1265,11 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src)
 				src_goal->metric, src_goal->target_value);
 		if (!new_goal)
 			return -ENOMEM;
-		damos_commit_quota_goal(new_goal, src_goal);
+		err = damos_commit_quota_goal(new_goal, src_goal);
+		if (err) {
+			damos_free_quota_goal(new_goal);
+			return err;
+		}
 		damos_add_quota_goal(dst, new_goal);
 	}
 	return 0;
@@ -1227,7 +1318,7 @@ static struct damos_filter *damos_nth_ops_filter(int n, struct damos *s)
 	return NULL;
 }
 
-static void damos_commit_filter_arg(
+static int damos_commit_filter_arg(
 		struct damos_filter *dst, struct damos_filter *src)
 {
 	switch (dst->type) {
@@ -1235,39 +1326,53 @@ static void damos_commit_filter_arg(
 		dst->memcg_id = src->memcg_id;
 		break;
 	case DAMOS_FILTER_TYPE_ADDR:
+		if (src->addr_range.end < src->addr_range.start)
+			return -EINVAL;
 		dst->addr_range = src->addr_range;
 		break;
 	case DAMOS_FILTER_TYPE_TARGET:
 		dst->target_idx = src->target_idx;
 		break;
 	case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE:
+		if (src->sz_range.max < src->sz_range.min)
+			return -EINVAL;
 		dst->sz_range = src->sz_range;
 		break;
+	case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM:
+		if (src->range_max < src->range_min)
+			return -EINVAL;
+		dst->range_min = src->range_min;
+		dst->range_max = src->range_max;
+		break;
 	default:
 		break;
 	}
+	return 0;
 }
 
-static void damos_commit_filter(
+static int damos_commit_filter(
 		struct damos_filter *dst, struct damos_filter *src)
 {
 	dst->type = src->type;
 	dst->matching = src->matching;
 	dst->allow = src->allow;
-	damos_commit_filter_arg(dst, src);
+	return damos_commit_filter_arg(dst, src);
 }
 
 static int damos_commit_core_filters(struct damos *dst, struct damos *src)
 {
 	struct damos_filter *dst_filter, *next, *src_filter, *new_filter;
-	int i = 0, j = 0;
+	int i = 0, j = 0, err;
 
 	damos_for_each_core_filter_safe(dst_filter, next, dst) {
 		src_filter = damos_nth_core_filter(i++, src);
-		if (src_filter)
-			damos_commit_filter(dst_filter, src_filter);
-		else
+		if (src_filter) {
+			err = damos_commit_filter(dst_filter, src_filter);
+			if (err)
+				return err;
+		} else {
 			damos_destroy_filter(dst_filter);
+		}
 	}
 
 	damos_for_each_core_filter_safe(src_filter, next, src) {
@@ -1279,7 +1384,11 @@ static int damos_commit_core_filters(struct damos *dst, struct damos *src)
 				src_filter->allow);
 		if (!new_filter)
 			return -ENOMEM;
-		damos_commit_filter_arg(new_filter, src_filter);
+		err = damos_commit_filter_arg(new_filter, src_filter);
+		if (err) {
+			damos_destroy_filter(new_filter);
+			return err;
+		}
 		damos_add_filter(dst, new_filter);
 	}
 	return 0;
@@ -1288,14 +1397,17 @@ static int damos_commit_core_filters(struct damos *dst, struct damos *src)
 static int damos_commit_ops_filters(struct damos *dst, struct damos *src)
 {
 	struct damos_filter *dst_filter, *next, *src_filter, *new_filter;
-	int i = 0, j = 0;
+	int i = 0, j = 0, err;
 
 	damos_for_each_ops_filter_safe(dst_filter, next, dst) {
 		src_filter = damos_nth_ops_filter(i++, src);
-		if (src_filter)
-			damos_commit_filter(dst_filter, src_filter);
-		else
+		if (src_filter) {
+			err = damos_commit_filter(dst_filter, src_filter);
+			if (err)
+				return err;
+		} else {
 			damos_destroy_filter(dst_filter);
+		}
 	}
 
 	damos_for_each_ops_filter_safe(src_filter, next, src) {
@@ -1307,7 +1419,11 @@ static int damos_commit_ops_filters(struct damos *dst, struct damos *src)
 				src_filter->allow);
 		if (!new_filter)
 			return -ENOMEM;
-		damos_commit_filter_arg(new_filter, src_filter);
+		err = damos_commit_filter_arg(new_filter, src_filter);
+		if (err) {
+			damos_destroy_filter(new_filter);
+			return err;
+		}
 		damos_add_filter(dst, new_filter);
 	}
 	return 0;
@@ -1345,6 +1461,13 @@ static bool damon_valid_probe_params(struct damon_ctx *ctx)
 	unsigned char max_probe_hits;
 	struct damon_probe *probe;
 	unsigned int wsum, wsum_to_add;
+	int nr_probes;
+
+	nr_probes = 0;
+	damon_for_each_probe(probe, ctx)
+		nr_probes++;
+	if (nr_probes > DAMON_MAX_PROBES)
+		return false;
 
 	if (!damon_has_probe_weights(ctx))
 		return true;
@@ -1666,7 +1789,37 @@ static int damon_commit_targets(
 	return err;
 }
 
-static void damon_commit_filter(struct damon_filter *dst,
+static void damon_commit_prep(struct damon_prep *dst, struct damon_prep *src)
+{
+	dst->action = src->action;
+}
+
+static int damon_commit_preps(struct damon_probe *dst, struct damon_probe *src)
+{
+	struct damon_prep *dst_prep, *next, *src_prep, *new_prep;
+	int i = 0, j = 0;
+
+	damon_for_each_prep_safe(dst_prep, next, dst) {
+		src_prep = damon_nth_prep(i++, src);
+		if (src_prep)
+			damon_commit_prep(dst_prep, src_prep);
+		else
+			damon_destroy_prep(dst_prep);
+	}
+
+	damon_for_each_prep_safe(src_prep, next, src) {
+		if (j++ < i)
+			continue;
+
+		new_prep = damon_new_prep(src_prep->action);
+		if (!new_prep)
+			return -ENOMEM;
+		damon_add_prep(dst, new_prep);
+	}
+	return 0;
+}
+
+static int damon_commit_filter(struct damon_filter *dst,
 		struct damon_filter *src)
 {
 	dst->type = src->type;
@@ -1676,23 +1829,33 @@ static void damon_commit_filter(struct damon_filter *dst,
 	case DAMON_FILTER_TYPE_MEMCG:
 		dst->memcg_id = src->memcg_id;
 		break;
+	case DAMON_FILTER_TYPE_HUGEPAGE_SIZE:
+		if (src->range_max < src->range_min)
+			return -EINVAL;
+		dst->range_min = src->range_min;
+		dst->range_max = src->range_max;
+		break;
 	default:
 		break;
 	}
+	return 0;
 }
 
 static int damon_commit_filters(struct damon_probe *dst,
 		struct damon_probe *src)
 {
 	struct damon_filter *dst_filter, *next, *src_filter, *new_filter;
-	int i = 0, j = 0;
+	int i = 0, j = 0, err;
 
 	damon_for_each_filter_safe(dst_filter, next, dst) {
 		src_filter = damon_nth_filter(i++, src);
-		if (src_filter)
-			damon_commit_filter(dst_filter, src_filter);
-		else
+		if (src_filter) {
+			err = damon_commit_filter(dst_filter, src_filter);
+			if (err)
+				return err;
+		} else {
 			damon_destroy_filter(dst_filter);
+		}
 	}
 
 	damon_for_each_filter_safe(src_filter, next, src) {
@@ -1707,6 +1870,14 @@ static int damon_commit_filters(struct damon_probe *dst,
 		case DAMON_FILTER_TYPE_MEMCG:
 			new_filter->memcg_id = src_filter->memcg_id;
 			break;
+		case DAMON_FILTER_TYPE_HUGEPAGE_SIZE:
+			if (src_filter->range_max < src_filter->range_min) {
+				damon_destroy_filter(new_filter);
+				return -EINVAL;
+			}
+			new_filter->range_min = src_filter->range_min;
+			new_filter->range_max = src_filter->range_max;
+			break;
 		default:
 			break;
 		}
@@ -1724,6 +1895,9 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src)
 		src_probe = damon_nth_probe(i++, src);
 		if (src_probe) {
 			dst_probe->weight = src_probe->weight;
+			err = damon_commit_preps(dst_probe, src_probe);
+			if (err)
+				return err;
 			err = damon_commit_filters(dst_probe, src_probe);
 			if (err)
 				return err;
@@ -1741,6 +1915,9 @@ static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src)
 			return -ENOMEM;
 		damon_add_probe(dst, new_probe);
 		new_probe->weight = src_probe->weight;
+		err = damon_commit_preps(new_probe, src_probe);
+		if (err)
+			return err;
 		err = damon_commit_filters(new_probe, src_probe);
 		if (err)
 			return err;
@@ -1852,20 +2029,6 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
 	return err;
 }
 
-/**
- * damon_nr_running_ctxs() - Return number of currently running contexts.
- */
-int damon_nr_running_ctxs(void)
-{
-	int nr_ctxs;
-
-	mutex_lock(&damon_lock);
-	nr_ctxs = nr_running_ctxs;
-	mutex_unlock(&damon_lock);
-
-	return nr_ctxs;
-}
-
 /* Returns the size upper limit for each monitoring region */
 static unsigned long damon_region_sz_limit(struct damon_ctx *ctx)
 {
@@ -1954,8 +2117,6 @@ static int __damon_start(struct damon_ctx *ctx)
 	return err;
 }
 
-static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src);
-
 /**
  * damon_start() - Starts the monitorings for a given group of contexts.
  * @ctxs:	an array of the pointers for contexts to start monitoring
@@ -2095,16 +2256,17 @@ int damon_kdamond_pid(struct damon_ctx *ctx)
  * synchronization.  The return value of the function will be saved in
  * &damon_call_control->return_code.
  *
- * Note that this function should be called only after damon_start() with the
- * @ctx has succeeded.  Otherwise, this function could fall into an indefinite
- * wait.
- *
  * When this function is failed, the @ctx is guaranteed to be stopped.
  *
+ * This function should not be called in parallel to damon_start() for the
+ * @ctx.  In the case, this function could indefinitely hang.
+ *
  * Return: 0 on success, negative error code otherwise.
  */
 int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
 {
+	if (!ctx)
+		return -EINVAL;
 	if (!control->repeat)
 		init_completion(&control->completion);
 	control->canceled = false;
@@ -2150,10 +2312,6 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
  * passed at least one &damos->apply_interval_us, kdamond marks the request as
  * completed so that damos_walk() can wakeup and return.
  *
- * Note that this function should be called only after damon_start() with the
- * @ctx has succeeded.  Otherwise, this function could fall into an indefinite
- * wait.
- *
  * Return: 0 on success, negative error code otherwise.
  */
 int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control)
@@ -2328,7 +2486,7 @@ static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r,
  * This function checks if a given region should be skipped or not for the
  * reason.  If only the starting part of the region has previously charged,
  * this function splits the region into two so that the second one covers the
- * area that not charged in the previous charge widnow, and return true.  The
+ * area that not charged in the previous charge window, and return true.  The
  * caller can see the second one on the next iteration of the region walk.
  * Note that this means the caller should use damon_for_each_region() instead
  * of damon_for_each_region_safe().  If damon_for_each_region_safe() is used,
@@ -2342,36 +2500,39 @@ static bool damos_skip_charged_region(struct damon_target *t,
 {
 	struct damos_quota *quota = &s->quota;
 	unsigned long sz_to_skip;
+	bool skip = false;
 
 	/* Skip previously charged regions */
 	if (quota->charge_target_from) {
 		if (t != quota->charge_target_from)
 			return true;
-		if (r == damon_last_region(t)) {
-			quota->charge_target_from = NULL;
-			quota->charge_addr_from = 0;
-			return true;
-		}
 		if (quota->charge_addr_from &&
-				r->ar.end <= quota->charge_addr_from)
-			return true;
+				r->ar.end <= quota->charge_addr_from) {
+			skip = true;
+			goto out;
+		}
 
 		if (quota->charge_addr_from && r->ar.start <
 				quota->charge_addr_from) {
 			sz_to_skip = ALIGN_DOWN(quota->charge_addr_from -
 					r->ar.start, min_region_sz);
 			if (!sz_to_skip) {
-				if (damon_sz_region(r) <= min_region_sz)
-					return true;
+				if (damon_sz_region(r) <= min_region_sz) {
+					skip = true;
+					goto out;
+				}
 				sz_to_skip = min_region_sz;
 			}
 			damon_split_region_at(t, r, sz_to_skip);
-			return true;
+			skip = true;
 		}
+	}
+out:
+	if (r == damon_last_region(t)) {
 		quota->charge_target_from = NULL;
 		quota->charge_addr_from = 0;
 	}
-	return false;
+	return skip;
 }
 
 static void damos_update_stat(struct damos *s,
@@ -2393,7 +2554,7 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t,
 	bool matched = false;
 	struct damon_target *ti;
 	int target_idx = 0;
-	unsigned long start, end;
+	unsigned long start, end, wsum;
 
 	switch (filter->type) {
 	case DAMOS_FILTER_TYPE_TARGET:
@@ -2428,6 +2589,11 @@ static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t,
 		damon_split_region_at(t, r, end - r->ar.start);
 		matched = true;
 		break;
+	case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM:
+		wsum = damon_probe_hits_wsum(r, false, true, ctx);
+		matched = filter->range_min <= wsum &&
+			wsum <= filter->range_max;
+		break;
 	default:
 		return false;
 	}
@@ -2486,7 +2652,7 @@ static void damos_walk_call_walk(struct damon_ctx *ctx, struct damon_target *t,
  * This function is called when kdamond finished applying the action of a DAMOS
  * scheme to all regions that eligible for the given &damos->apply_interval_us.
  * If every scheme of @ctx including @s now finished walking for at least one
- * &damos->apply_interval_us, this function makrs the handling of the given
+ * &damos->apply_interval_us, this function marks the handling of the given
  * DAMOS walk request is done, so that damos_walk() can wake up and return.
  */
 static void damos_walk_complete(struct damon_ctx *ctx, struct damos *s)
@@ -2613,10 +2779,12 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
 					c->min_region_sz);
 			if (!sz)
 				goto update_stat;
-			damon_split_region_at(t, r, sz);
+			if (damon_split_region_at(t, r, sz))
+				goto update_stat;
 		}
 		if (damos_core_filter_out(c, t, r, s))
 			return;
+		sz = damon_sz_region(r);
 		ktime_get_coarse_ts64(&begin);
 		trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses,
 				damon_nr_regions(t), do_trace);
@@ -2759,15 +2927,8 @@ static unsigned long damon_feed_loop_next_input(unsigned long last_input,
 	if (score >= goal * 2)
 		return min_input;
 
-	if (over_achieving)
-		score_goal_diff = score - goal;
-	else
-		score_goal_diff = goal - score;
-
-	if (last_input < ULONG_MAX / score_goal_diff)
-		compensation = last_input * score_goal_diff / goal;
-	else
-		compensation = last_input / goal * score_goal_diff;
+	score_goal_diff = abs_diff(score, goal);
+	compensation = mult_frac(last_input, score_goal_diff, goal);
 
 	if (over_achieving)
 		return max(last_input - compensation, min_input);
@@ -2795,6 +2956,28 @@ static inline u64 damos_get_some_mem_psi_total(void)
 
 #endif	/* CONFIG_PSI */
 
+static void damos_set_psi_current_val(u64 now_psi_total,
+		struct damos_quota_goal *goal, struct damos *s)
+{
+	u64 last_psi_total = goal->last_psi_total;
+
+	goal->last_psi_total = now_psi_total;
+	if (last_psi_total != U64_MAX) {
+		goal->current_value = now_psi_total - last_psi_total;
+		return;
+	}
+	/* uninitialized last_psi_total; make no effect this round */
+	if (s->quota.goal_tuner == DAMOS_QUOTA_GOAL_TUNER_CONSIST) {
+		goal->current_value = goal->target_value;
+		return;
+	}
+	/* let temporal tuner show the same achievement as in the last round */
+	if (!s->quota.esz)
+		goal->current_value = goal->target_value;
+	else
+		goal->current_value = 0;
+}
+
 #ifdef CONFIG_NUMA
 static bool invalid_mem_node(int nid)
 {
@@ -2815,6 +2998,13 @@ static __kernel_ulong_t damos_get_node_mem_bp(
 	}
 
 	si_meminfo_node(&i, goal->nid);
+	if (!i.totalram || i.totalram < i.freeram) {
+		if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP)
+			return 10000;
+		else	/* DAMOS_QUOTA_NODE_MEM_FREE_BP */
+			return 0;
+	}
+
 	if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP)
 		numerator = i.totalram - i.freeram;
 	else	/* DAMOS_QUOTA_NODE_MEM_FREE_BP */
@@ -2855,6 +3045,12 @@ static unsigned long damos_get_node_memcg_used_bp(
 	mem_cgroup_put(memcg);
 
 	si_meminfo_node(&i, goal->nid);
+	if (!i.totalram || i.totalram < used_pages) {
+		if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP)
+			return 10000;
+		else	/* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */
+			return 0;
+	}
 	if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP)
 		numerator = used_pages;
 	else	/* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */
@@ -3001,12 +3197,28 @@ static unsigned int damos_get_in_active_mem_bp(bool active_ratio)
 		global_node_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE);
 	inactive = global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_ANON) +
 		global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE);
-	total = active + inactive;
+	total = max(active + inactive, 1);
 	if (active_ratio)
 		return mult_frac(active, 10000, total);
 	return mult_frac(inactive, 10000, total);
 }
 
+static unsigned int damos_hugepage_mem_bp(void)
+{
+	unsigned long thp, total_pages, free_pages;
+
+	total_pages = totalram_pages();
+	free_pages = global_zone_page_state(NR_FREE_PAGES);
+
+	if (total_pages <= free_pages)
+		return 10000;
+
+	thp = global_node_page_state(NR_ANON_THPS) +
+				global_node_page_state(NR_SHMEM_THPS) +
+				global_node_page_state(NR_FILE_THPS);
+	return mult_frac(thp, 10000, total_pages - free_pages);
+}
+
 static void damos_set_quota_goal_current_value(struct damon_ctx *c,
 		struct damos *s, struct damos_quota_goal *goal)
 {
@@ -3018,8 +3230,7 @@ static void damos_set_quota_goal_current_value(struct damon_ctx *c,
 		break;
 	case DAMOS_QUOTA_SOME_MEM_PSI_US:
 		now_psi_total = damos_get_some_mem_psi_total();
-		goal->current_value = now_psi_total - goal->last_psi_total;
-		goal->last_psi_total = now_psi_total;
+		damos_set_psi_current_val(now_psi_total, goal, s);
 		break;
 	case DAMOS_QUOTA_NODE_MEM_USED_BP:
 	case DAMOS_QUOTA_NODE_MEM_FREE_BP:
@@ -3038,6 +3249,9 @@ static void damos_set_quota_goal_current_value(struct damon_ctx *c,
 		goal->current_value = damos_get_node_eligible_mem_bp(c, s,
 				goal->nid);
 		break;
+	case DAMOS_QUOTA_HUGEPAGE_MEM_BP:
+		goal->current_value = damos_hugepage_mem_bp();
+		break;
 	default:
 		break;
 	}
@@ -3077,7 +3291,7 @@ static void damos_goal_tune_esz_bp_temporal(struct damon_ctx *c,
 
 	if (score >= 10000)
 		quota->esz_bp = 0;
-	else if (quota->sz)
+	else if (quota->sz && quota->sz <= ULONG_MAX / 10000)
 		quota->esz_bp = quota->sz * 10000;
 	else
 		quota->esz_bp = ULONG_MAX;
@@ -3091,6 +3305,7 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s)
 	struct damos_quota *quota = &s->quota;
 	unsigned long throughput;
 	unsigned long esz = ULONG_MAX;
+	unsigned long esz_time;
 
 	if (!quota->ms && list_empty(&quota->goals)) {
 		quota->esz = quota->sz;
@@ -3111,8 +3326,8 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s)
 					1000000, quota->total_charged_ns);
 		else
 			throughput = PAGE_SIZE * 1024;
-		esz = min(throughput * quota->ms, esz);
-		esz = max(ctx->min_region_sz, esz);
+		esz_time = max(throughput * quota->ms, ctx->min_region_sz);
+		esz = min(esz_time, esz);
 	}
 
 	if (quota->sz && quota->sz < esz)
@@ -3143,7 +3358,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s)
 	unsigned long cumulated_sz, cached_esz;
 	unsigned int score, max_score = 0;
 
-	if (!quota->ms && !quota->sz && list_empty(&quota->goals))
+	if (!damos_quota_is_set(quota))
 		return;
 
 	/* First charge window */
@@ -3170,6 +3385,8 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s)
 			damos_trace_esz(c, s, quota);
 	}
 
+	if (damos_quota_is_full(quota, c->min_region_sz))
+		return;
 	if (!c->ops.get_scheme_score)
 		return;
 
@@ -3216,7 +3433,7 @@ static void damos_trace_stat(struct damon_ctx *c, struct damos *s)
 	trace_call__damos_stat_after_apply_interval(cidx, sidx, &s->stat);
 }
 
-static void kdamond_apply_schemes(struct damon_ctx *c)
+static noinline_for_stack void kdamond_apply_schemes(struct damon_ctx *c)
 {
 	struct damon_target *t;
 	struct damos *s;
@@ -3241,8 +3458,15 @@ static void kdamond_apply_schemes(struct damon_ctx *c)
 	max_region_sz = damon_region_sz_limit(c);
 	mutex_lock(&c->walk_control_lock);
 	damon_for_each_target(t, c) {
-		if (c->ops.target_valid && c->ops.target_valid(t) == false)
+		if (c->ops.target_valid && c->ops.target_valid(t) == false) {
+			damon_for_each_scheme(s, c) {
+				if (s->quota.charge_target_from != t)
+					continue;
+				s->quota.charge_target_from = NULL;
+				s->quota.charge_addr_from = 0;
+			}
 			continue;
+		}
 		damos_apply_target(c, t, max_region_sz);
 	}
 
@@ -3297,7 +3521,7 @@ static unsigned int damon_merge_score(struct damon_region *r, bool last,
 		struct damon_ctx *ctx, bool use_probe_hits)
 {
 	if (use_probe_hits)
-		return damon_probe_hits_wsum(r, last, ctx);
+		return damon_probe_hits_wsum(r, last, false, ctx);
 	if (last)
 		return r->last_nr_accesses;
 	return r->nr_accesses;
@@ -3365,16 +3589,16 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
  * while DAMON is running.  For such a case, repeat merging until the limit is
  * met while increasing @threshold up to possible maximum level.
  */
-static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
-				  unsigned long sz_limit)
+static noinline_for_stack void kdamond_merge_regions(struct damon_ctx *c,
+						     unsigned int threshold,
+						     unsigned long sz_limit)
 {
 	struct damon_target *t;
 	unsigned int nr_regions;
 	unsigned int max_thres;
 	bool count_age = true;
 
-	max_thres = c->attrs.aggr_interval /
-		(c->attrs.sample_interval ?  c->attrs.sample_interval : 1);
+	max_thres = damon_nr_samples_per_aggr(&c->attrs);
 	while (true) {
 		nr_regions = 0;
 		damon_for_each_target(t, c) {
@@ -3511,7 +3735,7 @@ static void damon_split_some_regions(struct damon_ctx *ctx,
  * split was unnecessarily made, later 'kdamond_merge_regions()' will revert
  * it.
  */
-static void kdamond_split_regions(struct damon_ctx *ctx)
+static noinline_for_stack void kdamond_split_regions(struct damon_ctx *ctx)
 {
 	struct damon_target *t;
 	unsigned long nr_regions = 0;
@@ -3597,10 +3821,6 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme)
 
 	/* higher than high watermark or lower than low watermark */
 	if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) {
-		if (scheme->wmarks.activated)
-			pr_debug("deactivate a scheme (%d) for %s wmark\n",
-				 scheme->action,
-				 str_high_low(metric > scheme->wmarks.high));
 		scheme->wmarks.activated = false;
 		return scheme->wmarks.interval;
 	}
@@ -3610,8 +3830,6 @@ static unsigned long damos_wmark_wait_us(struct damos *scheme)
 			!scheme->wmarks.activated)
 		return scheme->wmarks.interval;
 
-	if (!scheme->wmarks.activated)
-		pr_debug("activate a scheme (%d)\n", scheme->action);
 	scheme->wmarks.activated = true;
 	return 0;
 }
@@ -3754,8 +3972,6 @@ static int kdamond_fn(void *data)
 	struct damon_ctx *ctx = data;
 	unsigned long sz_limit = 0;
 
-	pr_debug("kdamond (%d) starts\n", current->pid);
-
 	mutex_lock(&ctx->call_controls_lock);
 	ctx->call_controls_obsolete = false;
 	mutex_unlock(&ctx->call_controls_lock);
@@ -3785,14 +4001,19 @@ static int kdamond_fn(void *data)
 		unsigned long next_ops_update_sis = ctx->next_ops_update_sis;
 		unsigned long sample_interval = ctx->attrs.sample_interval;
 		bool access_check_disabled = damon_has_probe_weights(ctx);
+		bool do_prep;
 		unsigned int max_merge_score = 0, max_wsum;
 		bool get_max_wsum;
 
 		if (kdamond_wait_activation(ctx))
 			break;
 
+		do_prep = ctx->ops.prep_probes && damon_has_prep(ctx);
+
 		if (!access_check_disabled && ctx->ops.prepare_access_checks)
 			ctx->ops.prepare_access_checks(ctx);
+		if (do_prep)
+			ctx->ops.prep_probes(ctx, access_check_disabled);
 
 		kdamond_usleep(sample_interval);
 		ctx->passed_sample_intervals++;
@@ -3807,7 +4028,8 @@ static int kdamond_fn(void *data)
 			else
 				get_max_wsum = false;
 			max_wsum = ctx->ops.apply_probes(ctx,
-					access_check_disabled, get_max_wsum);
+					access_check_disabled && !do_prep,
+					get_max_wsum);
 			if (get_max_wsum)
 				max_merge_score = max_wsum;
 		}
@@ -3903,7 +4125,6 @@ static int kdamond_fn(void *data)
 	mutex_unlock(&ctx->walk_control_lock);
 	damos_walk_cancel(ctx);
 
-	pr_debug("kdamond (%d) finishes\n", current->pid);
 	mutex_lock(&ctx->kdamond_lock);
 	ctx->kdamond = NULL;
 	mutex_unlock(&ctx->kdamond_lock);
@@ -3974,7 +4195,7 @@ static bool damon_find_system_rams_range(unsigned long *start,
  * This function sets the region of @t as requested by @start and @end.  If the
  * values of @start and @end are zero, however, this function finds 'System
  * RAM' resources and sets the region to cover all the resource.  In the latter
- * case, this function saves the start and the end addresseses of the first and
+ * case, this function saves the start and the end addresses of the first and
  * the last resources in @start and @end, respectively.
  *
  * Return: 0 on success, negative error code otherwise.
diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c
index bd847829a99072..273efa3c913ed4 100644
--- a/mm/damon/lru_sort.c
+++ b/mm/damon/lru_sort.c
@@ -233,8 +233,6 @@ static int damon_lru_sort_add_quota_goals(struct damos *hot_scheme,
 
 	if (!active_mem_bp)
 		return 0;
-	if (10000 < active_mem_bp)
-		return -EINVAL;
 	goal = damos_new_quota_goal(DAMOS_QUOTA_ACTIVE_MEM_BP, active_mem_bp);
 	if (!goal)
 		return -ENOMEM;
@@ -262,7 +260,7 @@ static int damon_lru_sort_add_filters(struct damos *hot_scheme,
 		return -ENOMEM;
 	damos_add_filter(hot_scheme, filter);
 
-	/* disabllow de-prioritizing young pages */
+	/* disallow de-prioritizing young pages */
 	filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, false);
 	if (!filter)
 		return -ENOMEM;
@@ -346,8 +344,6 @@ static int damon_lru_sort_commit_inputs_fn(void *arg)
 	return damon_lru_sort_apply_parameters();
 }
 
-static bool damon_lru_sort_damon_has_started;
-
 static int damon_lru_sort_commit_inputs_store(const char *val,
 					      const struct kernel_param *kp)
 {
@@ -368,10 +364,6 @@ static int damon_lru_sort_commit_inputs_store(const char *val,
 	if (!commit_inputs_request)
 		return 0;
 
-	/* Skip damon_call() if ctx has not successfully started. */
-	if (!damon_lru_sort_damon_has_started)
-		return -EINVAL;
-
 	err = damon_call(ctx, &control);
 
 	return err ? err : control.return_code;
@@ -422,8 +414,6 @@ static int damon_lru_sort_turn(bool on)
 	err = damon_start(&ctx, 1, true);
 	if (err)
 		return err;
-	if (!damon_lru_sort_damon_has_started)
-		damon_lru_sort_damon_has_started = true;
 	return damon_call(ctx, &call_control);
 }
 
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index fbda70d8ea4d05..77366f42b3e5bf 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -3,6 +3,7 @@
  * Common Code for Data Access Monitoring
  */
 
+#include <linux/hugetlb.h>
 #include <linux/migrate.h>
 #include <linux/mmu_notifier.h>
 #include <linux/page_idle.h>
@@ -14,14 +15,20 @@
 #include "../internal.h"
 #include "ops-common.h"
 
+static bool damon_folio_acceptable(struct folio *folio, bool monitor)
+{
+	return folio_test_lru(folio) ||
+		(monitor && folio_test_hugetlb(folio));
+}
+
 /*
- * Get an online page for a pfn if it's in the LRU list.  Otherwise, returns
- * NULL.
+ * Get an online page for a pfn if it's in the LRU list, or a hugetlb folio if
+ * @monitor is set.  Otherwise, returns NULL.
  *
  * The body of this function is stolen from the 'page_idle_get_folio()'.  We
  * steal rather than reuse it because the code is quite simple.
  */
-struct folio *damon_get_folio(unsigned long pfn)
+static struct folio *__damon_get_folio(unsigned long pfn, bool monitor)
 {
 	struct page *page = pfn_to_online_page(pfn);
 	struct folio *folio;
@@ -32,13 +39,24 @@ struct folio *damon_get_folio(unsigned long pfn)
 	folio = page_folio(page);
 	if (!folio_try_get(folio))
 		return NULL;
-	if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) {
+	if (unlikely(page_folio(page) != folio) ||
+			!damon_folio_acceptable(folio, monitor)) {
 		folio_put(folio);
 		folio = NULL;
 	}
 	return folio;
 }
 
+struct folio *damon_get_folio(unsigned long pfn)
+{
+	return __damon_get_folio(pfn, false);
+}
+
+struct folio *damon_get_monitor_folio(unsigned long pfn)
+{
+	return __damon_get_folio(pfn, true);
+}
+
 void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)
 {
 	pte_t pteval = ptep_get(pte);
@@ -61,7 +79,12 @@ void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr
 	 * device aspects.
 	 */
 	if (likely(pte_present(pteval)))
-		young |= ptep_test_and_clear_young(vma, addr, pte);
+		/*
+		 * Arch implementation of ptep_test_and_clear_young() may
+		 * require aligned @addr
+		 */
+		young |= ptep_test_and_clear_young(vma, PAGE_ALIGN_DOWN(addr),
+				pte);
 	young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE);
 	if (young)
 		folio_set_young(folio);
@@ -98,6 +121,42 @@ void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 }
 
+#ifdef CONFIG_HUGETLB_PAGE
+static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm,
+		struct vm_area_struct *vma, unsigned long addr, pte_t *entry)
+{
+	unsigned long psize = huge_page_size(hstate_vma(vma));
+
+	if (!pte_young(*entry))
+		return false;
+	*entry = huge_ptep_get_and_clear(mm, addr, pte, psize);
+	*entry = pte_mkold(*entry);
+	set_huge_pte_at(mm, addr, pte, *entry, psize);
+	return true;
+}
+
+void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm,
+		struct vm_area_struct *vma, unsigned long addr)
+{
+	bool referenced = false;
+	pte_t entry = huge_ptep_get(mm, addr, pte);
+	struct folio *folio = pfn_folio(pte_pfn(entry));
+
+	folio_get(folio);
+
+	referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry);
+	if (mmu_notifier_clear_young(mm, addr,
+				     addr + huge_page_size(hstate_vma(vma))))
+		referenced = true;
+
+	if (referenced)
+		folio_set_young(folio);
+
+	folio_set_idle(folio);
+	folio_put(folio);
+}
+#endif	/* CONFIG_HUGETLB_PAGE */
+
 #define DAMON_MAX_SUBSCORE	(100)
 #define DAMON_MAX_AGE_IN_LOG	(32)
 
@@ -163,10 +222,15 @@ static bool damon_folio_mkold_one(struct folio *folio,
 
 	while (page_vma_mapped_walk(&pvmw)) {
 		addr = pvmw.address;
-		if (pvmw.pte)
-			damon_ptep_mkold(pvmw.pte, vma, addr);
-		else
+		if (pvmw.pte) {
+			if (folio_test_hugetlb(folio))
+				damon_hugetlb_mkold(pvmw.pte, vma->vm_mm, vma,
+						addr);
+			else
+				damon_ptep_mkold(pvmw.pte, vma, addr);
+		} else {
 			damon_pmdp_mkold(pvmw.pmd, vma, addr);
+		}
 	}
 	return true;
 }
@@ -191,27 +255,55 @@ void damon_folio_mkold(struct folio *folio)
 
 }
 
+#ifdef CONFIG_HUGETLB_PAGE
+static bool damon_hugetlb_young(pte_t *pte, struct vm_area_struct *vma,
+		unsigned long addr, struct folio *folio)
+{
+	pte_t entry = huge_ptep_get(vma->vm_mm, addr, pte);
+
+	return (pte_present(entry) && pte_young(entry)) ||
+		!folio_test_idle(folio) ||
+		mmu_notifier_test_young(vma->vm_mm, addr);
+}
+#else
+static bool damon_hugetlb_young(pte_t *pte, struct vm_area_struct *vma,
+		unsigned long addr, struct folio *folio)
+{
+	return false;
+}
+#endif	/* CONFIG_HUGETLB_PAGE */
+
+static bool damon_pte_young(pte_t *pte, struct vm_area_struct *vma,
+		unsigned long addr, struct folio *folio)
+{
+	pte_t entry = ptep_get(pte);
+
+	/*
+	 * PFN swap PTEs, such as device-exclusive ones, that actually map
+	 * pages are "old" from a CPU perspective. The MMU notifier takes care
+	 * of any device aspects.
+	 */
+	return (pte_present(entry) && pte_young(entry)) ||
+		!folio_test_idle(folio) ||
+		mmu_notifier_test_young(vma->vm_mm, addr);
+}
+
 static bool damon_folio_young_one(struct folio *folio,
 		struct vm_area_struct *vma, unsigned long addr, void *arg)
 {
 	bool *accessed = arg;
 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
-	pte_t pte;
 
 	*accessed = false;
 	while (page_vma_mapped_walk(&pvmw)) {
 		addr = pvmw.address;
 		if (pvmw.pte) {
-			pte = ptep_get(pvmw.pte);
-
-			/*
-			 * PFN swap PTEs, such as device-exclusive ones, that
-			 * actually map pages are "old" from a CPU perspective.
-			 * The MMU notifier takes care of any device aspects.
-			 */
-			*accessed = (pte_present(pte) && pte_young(pte)) ||
-				!folio_test_idle(folio) ||
-				mmu_notifier_test_young(vma->vm_mm, addr);
+			if (folio_test_hugetlb(folio))
+				*accessed = damon_hugetlb_young(pvmw.pte, vma,
+						addr, folio);
+			else
+				*accessed = damon_pte_young(pvmw.pte, vma,
+						addr, folio);
 		} else {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 			pmd_t pmd = pmdp_get(pvmw.pmd);
@@ -330,6 +422,19 @@ static unsigned int __damon_migrate_folio_list(
 	return nr_succeeded;
 }
 
+static void damon_putback_folio_list(struct list_head *folio_list)
+{
+	struct folio *folio;
+
+	while (!list_empty(folio_list)) {
+		folio = lru_to_folio(folio_list);
+		list_del(&folio->lru);
+		node_stat_sub_folio(folio, NR_ISOLATED_ANON +
+				folio_is_file_lru(folio));
+		folio_putback_lru(folio);
+	}
+}
+
 static unsigned int damon_migrate_folio_list(struct list_head *folio_list,
 						struct pglist_data *pgdat,
 						int target_nid)
@@ -371,13 +476,7 @@ static unsigned int damon_migrate_folio_list(struct list_head *folio_list,
 
 	list_splice(&ret_folios, folio_list);
 
-	while (!list_empty(folio_list)) {
-		folio = lru_to_folio(folio_list);
-		list_del(&folio->lru);
-		node_stat_sub_folio(folio, NR_ISOLATED_ANON +
-				folio_is_file_lru(folio));
-		folio_putback_lru(folio);
-	}
+	damon_putback_folio_list(folio_list);
 
 	return nr_migrated;
 }
@@ -394,14 +493,7 @@ unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid)
 
 	if (target_nid < 0 || target_nid >= MAX_NUMNODES ||
 			!node_state(target_nid, N_MEMORY)) {
-		while (!list_empty(folio_list)) {
-			struct folio *folio = lru_to_folio(folio_list);
-
-			list_del(&folio->lru);
-			node_stat_sub_folio(folio, NR_ISOLATED_ANON +
-					folio_is_file_lru(folio));
-			folio_putback_lru(folio);
-		}
+		damon_putback_folio_list(folio_list);
 		return nr_migrated;
 	}
 
@@ -439,3 +531,45 @@ bool damos_ops_has_filter(struct damos *s)
 		return true;
 	return false;
 }
+
+bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio)
+{
+	bool matched = false;
+	struct mem_cgroup *memcg;
+	size_t folio_sz;
+
+	switch (filter->type) {
+	case DAMON_FILTER_TYPE_ANON:
+		if (!folio) {
+			matched = false;
+			break;
+		}
+		matched = folio_test_anon(folio);
+		break;
+	case DAMON_FILTER_TYPE_MEMCG:
+		if (!folio) {
+			matched = false;
+			break;
+		}
+		rcu_read_lock();
+		memcg = folio_memcg_check(folio);
+		if (!memcg)
+			matched = false;
+		else
+			matched = filter->memcg_id == mem_cgroup_id(memcg);
+		rcu_read_unlock();
+		break;
+	case DAMON_FILTER_TYPE_HUGEPAGE_SIZE:
+		if (!folio) {
+			matched = false;
+			break;
+		}
+		folio_sz = folio_size(folio);
+		matched = filter->range_min <= folio_sz &&
+			folio_sz <= filter->range_max;
+		break;
+	default:
+		break;
+	}
+	return matched == filter->matching;
+}
diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h
index 38d295488fa181..ee27058251005b 100644
--- a/mm/damon/ops-common.h
+++ b/mm/damon/ops-common.h
@@ -6,9 +6,19 @@
 #include <linux/damon.h>
 
 struct folio *damon_get_folio(unsigned long pfn);
+struct folio *damon_get_monitor_folio(unsigned long pfn);
 
 void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr);
 void damon_pmdp_mkold(pmd_t *pmd, struct vm_area_struct *vma, unsigned long addr);
+#ifdef CONFIG_HUGETLB_PAGE
+void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm,
+		struct vm_area_struct *vma, unsigned long addr);
+#else
+static inline void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm,
+		struct vm_area_struct *vma, unsigned long addr)
+{
+}
+#endif	/* CONFIG_HUGETLB_PAGE */
 void damon_folio_mkold(struct folio *folio);
 bool damon_folio_young(struct folio *folio);
 
@@ -21,3 +31,5 @@ bool damos_folio_filter_match(struct damos_filter *filter, struct folio *folio);
 unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid);
 
 bool damos_ops_has_filter(struct damos *s);
+
+bool damon_ops_filter_match(struct damon_filter *filter, struct folio *folio);
diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index 5c6c3a597fd0bf..2cfdc356b41572 100644
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -37,7 +37,7 @@ static unsigned long damon_pa_core_addr(
 
 static void damon_pa_mkold(phys_addr_t paddr)
 {
-	struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
+	struct folio *folio = damon_get_monitor_folio(PHYS_PFN(paddr));
 
 	if (!folio)
 		return;
@@ -67,7 +67,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
 
 static bool damon_pa_young(phys_addr_t paddr)
 {
-	struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
+	struct folio *folio = damon_get_monitor_folio(PHYS_PFN(paddr));
 	bool accessed;
 
 	if (!folio)
@@ -105,41 +105,65 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
 	return max_nr_accesses;
 }
 
+static void damon_pa_prep_probes_region(struct damon_region *r,
+		struct damon_probe *probe, struct damon_ctx *ctx)
+{
+	struct damon_prep *p;
+
+	damon_for_each_prep(p, probe) {
+		switch (p->action) {
+		case DAMON_PREP_SET_PGIDLE:
+			damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr,
+						ctx->addr_unit));
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+static void damon_pa_prep_probes(struct damon_ctx *ctx, bool set_samples)
+{
+	struct damon_target *t;
+	struct damon_region *r;
+	struct damon_probe *p;
+
+	damon_for_each_target(t, ctx) {
+		damon_for_each_region(r, t) {
+			if (set_samples)
+				r->sampling_addr = damon_rand(ctx, r->ar.start,
+						r->ar.end);
+			damon_for_each_probe(p, ctx)
+				damon_pa_prep_probes_region(r, p, ctx);
+		}
+	}
+}
+
 static bool damon_pa_filter_match(struct damon_filter *filter,
 		struct folio *folio)
 {
 	bool matched = false;
-	struct mem_cgroup *memcg;
 
 	switch (filter->type) {
-	case DAMON_FILTER_TYPE_ANON:
-		if (!folio) {
+	case DAMON_FILTER_TYPE_PGIDLE_UNSET:
+		if (!folio)
 			matched = false;
-			break;
-		}
-		matched = folio_test_anon(folio);
+		else
+			matched = damon_folio_young(folio);
 		break;
-	case DAMON_FILTER_TYPE_MEMCG:
-		if (!folio) {
-			matched = false;
-			break;
-		}
-		rcu_read_lock();
-		memcg = folio_memcg_check(folio);
-		if (!memcg)
+	case DAMON_FILTER_TYPE_PGIDLE_SET:
+		if (!folio)
 			matched = false;
 		else
-			matched = filter->memcg_id == mem_cgroup_id(memcg);
-		rcu_read_unlock();
+			matched = damon_folio_young(folio) == false;
 		break;
 	default:
-		break;
+		return damon_ops_filter_match(filter, folio);
 	}
 	return matched == filter->matching;
 }
 
-static bool damon_pa_filter_pass(phys_addr_t pa, struct folio *folio,
-		struct damon_probe *p)
+static bool damon_pa_filter_pass(struct folio *folio, struct damon_probe *p)
 {
 	struct damon_filter *f;
 	bool pass = true;
@@ -175,7 +199,7 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx,
 					ctx->addr_unit);
 			folio = damon_get_folio(PHYS_PFN(pa));
 			damon_for_each_probe(p, ctx) {
-				if (damon_pa_filter_pass(pa, folio, p))
+				if (damon_pa_filter_pass(folio, p))
 					r->probe_hits[i]++;
 				i++;
 			}
@@ -183,7 +207,7 @@ static unsigned int damon_pa_apply_probes(struct damon_ctx *ctx,
 				folio_put(folio);
 			if (return_max_wsum)
 				max_wsum = max(damon_probe_hits_wsum(r, false,
-							ctx), max_wsum);
+							false, ctx), max_wsum);
 		}
 	}
 	return max_wsum;
@@ -264,7 +288,7 @@ static unsigned long damon_pa_pageout(struct damon_region *r,
 		else
 			list_add(&folio->lru, &folio_list);
 put_folio:
-		addr += folio_size(folio);
+		addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio);
 		folio_put(folio);
 	}
 	if (install_young_filter)
@@ -302,7 +326,7 @@ static inline unsigned long damon_pa_de_activate(
 			folio_deactivate(folio);
 		applied += folio_nr_pages(folio);
 put_folio:
-		addr += folio_size(folio);
+		addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio);
 		folio_put(folio);
 	}
 	s->last_applied = folio;
@@ -350,7 +374,7 @@ static unsigned long damon_pa_migrate(struct damon_region *r,
 				folio_is_file_lru(folio));
 		list_add(&folio->lru, &folio_list);
 put_folio:
-		addr += folio_size(folio);
+		addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio);
 		folio_put(folio);
 	}
 	applied = damon_migrate_pages(&folio_list, s->target_nid);
@@ -379,7 +403,7 @@ static unsigned long damon_pa_stat(struct damon_region *r,
 
 		if (!damos_pa_filter_out(s, folio))
 			*sz_filter_passed += folio_size(folio) / addr_unit;
-		addr += folio_size(folio);
+		addr = PFN_PHYS(folio_pfn(folio)) + folio_size(folio);
 		folio_put(folio);
 	}
 	s->last_applied = folio;
@@ -442,6 +466,7 @@ static int __init damon_pa_initcall(void)
 		.update = NULL,
 		.prepare_access_checks = damon_pa_prepare_access_checks,
 		.check_accesses = damon_pa_check_accesses,
+		.prep_probes = damon_pa_prep_probes,
 		.apply_probes = damon_pa_apply_probes,
 		.target_valid = NULL,
 		.apply_scheme = damon_pa_apply_scheme,
diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c
index 45d5557cc575a0..42a2c9cb134310 100644
--- a/mm/damon/reclaim.c
+++ b/mm/damon/reclaim.c
@@ -271,8 +271,6 @@ static int damon_reclaim_commit_inputs_fn(void *arg)
 	return damon_reclaim_apply_parameters();
 }
 
-static bool damon_reclaim_damon_has_started;
-
 static int damon_reclaim_commit_inputs_store(const char *val,
 					     const struct kernel_param *kp)
 {
@@ -293,10 +291,6 @@ static int damon_reclaim_commit_inputs_store(const char *val,
 	if (!commit_inputs_request)
 		return 0;
 
-	/* Skip damon_call() if ctx has not successfully started. */
-	if (!damon_reclaim_damon_has_started)
-		return -EINVAL;
-
 	err = damon_call(ctx, &control);
 
 	return err ? err : control.return_code;
@@ -343,8 +337,6 @@ static int damon_reclaim_turn(bool on)
 	err = damon_start(&ctx, 1, true);
 	if (err)
 		return err;
-	if (!damon_reclaim_damon_has_started)
-		damon_reclaim_damon_has_started = true;
 	return damon_call(ctx, &call_control);
 }
 
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index 32f495a96b17a8..bfb6f0bc3f2138 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -534,7 +534,8 @@ struct damon_sysfs_scheme_filter {
 	bool allow;
 	char *memcg_path;
 	struct damon_addr_range addr_range;
-	struct damon_size_range sz_range;
+	unsigned long range_min;
+	unsigned long range_max;
 	int target_idx;
 };
 
@@ -588,6 +589,10 @@ damos_sysfs_filter_type_names[] = {
 		.type = DAMOS_FILTER_TYPE_TARGET,
 		.name = "target",
 	},
+	{
+		.type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM,
+		.name = "probe_hits_wsum",
+	},
 };
 
 static ssize_t type_show(struct kobject *kobj,
@@ -778,7 +783,7 @@ static ssize_t min_show(struct kobject *kobj,
 	struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 			struct damon_sysfs_scheme_filter, kobj);
 
-	return sysfs_emit(buf, "%lu\n", filter->sz_range.min);
+	return sysfs_emit(buf, "%lu\n", filter->range_min);
 }
 
 static ssize_t min_store(struct kobject *kobj,
@@ -786,7 +791,7 @@ static ssize_t min_store(struct kobject *kobj,
 {
 	struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 			struct damon_sysfs_scheme_filter, kobj);
-	int err = kstrtoul(buf, 0, &filter->sz_range.min);
+	int err = kstrtoul(buf, 0, &filter->range_min);
 
 	return err ? err : count;
 }
@@ -797,7 +802,7 @@ static ssize_t max_show(struct kobject *kobj,
 	struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 			struct damon_sysfs_scheme_filter, kobj);
 
-	return sysfs_emit(buf, "%lu\n", filter->sz_range.max);
+	return sysfs_emit(buf, "%lu\n", filter->range_max);
 }
 
 static ssize_t max_store(struct kobject *kobj,
@@ -805,7 +810,7 @@ static ssize_t max_store(struct kobject *kobj,
 {
 	struct damon_sysfs_scheme_filter *filter = container_of(kobj,
 			struct damon_sysfs_scheme_filter, kobj);
-	int err = kstrtoul(buf, 0, &filter->sz_range.max);
+	int err = kstrtoul(buf, 0, &filter->range_max);
 
 	return err ? err : count;
 }
@@ -1269,6 +1274,10 @@ struct damos_sysfs_qgoal_metric_name damos_sysfs_qgoal_metric_names[] = {
 		.metric = DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP,
 		.name = "node_eligible_mem_bp",
 	},
+	{
+		.metric = DAMOS_QUOTA_HUGEPAGE_MEM_BP,
+		.name = "hugepage_mem_bp",
+	},
 };
 
 static ssize_t target_metric_show(struct kobject *kobj,
@@ -2813,7 +2822,8 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme,
 
 		if (!filter)
 			return -ENOMEM;
-		if (filter->type == DAMOS_FILTER_TYPE_MEMCG) {
+		switch (filter->type) {
+		case DAMOS_FILTER_TYPE_MEMCG:
 			err = damon_sysfs_memcg_path_to_id(
 					sysfs_filter->memcg_path,
 					&filter->memcg_id);
@@ -2821,22 +2831,23 @@ static int damon_sysfs_add_scheme_filters(struct damos *scheme,
 				damos_destroy_filter(filter);
 				return err;
 			}
-		} else if (filter->type == DAMOS_FILTER_TYPE_ADDR) {
-			if (sysfs_filter->addr_range.end <
-					sysfs_filter->addr_range.start) {
-				damos_destroy_filter(filter);
-				return -EINVAL;
-			}
+			break;
+		case DAMOS_FILTER_TYPE_ADDR:
 			filter->addr_range = sysfs_filter->addr_range;
-		} else if (filter->type == DAMOS_FILTER_TYPE_TARGET) {
+			break;
+		case DAMOS_FILTER_TYPE_TARGET:
 			filter->target_idx = sysfs_filter->target_idx;
-		} else if (filter->type == DAMOS_FILTER_TYPE_HUGEPAGE_SIZE) {
-			if (sysfs_filter->sz_range.min >
-					sysfs_filter->sz_range.max) {
-				damos_destroy_filter(filter);
-				return -EINVAL;
-			}
-			filter->sz_range = sysfs_filter->sz_range;
+			break;
+		case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE:
+			filter->sz_range.min = sysfs_filter->range_min;
+			filter->sz_range.max = sysfs_filter->range_max;
+			break;
+		case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM:
+			filter->range_min = sysfs_filter->range_min;
+			filter->range_max = sysfs_filter->range_max;
+			break;
+		default:
+			break;
 		}
 
 		damos_add_filter(scheme, filter);
diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index e3858ffab4b227..70a4b64fb8ea9f 100644
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -3,7 +3,6 @@
  * DAMON sysfs Interface
  */
 
-#include <linux/pid.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 
@@ -750,6 +749,222 @@ static const struct kobj_type damon_sysfs_intervals_ktype = {
 	.default_groups = damon_sysfs_intervals_groups,
 };
 
+/*
+ * prep directory
+ */
+
+struct damon_sysfs_prep {
+	struct kobject kobj;
+	enum damon_prep_action action;
+};
+
+static struct damon_sysfs_prep *damon_sysfs_prep_alloc(void)
+{
+	struct damon_sysfs_prep *prep;
+
+	prep = kzalloc_obj(struct damon_sysfs_prep);
+	if (!prep)
+		return prep;
+	prep->action = DAMON_PREP_SET_PGIDLE;
+	return prep;
+}
+
+struct damon_sysfs_prep_action_name {
+	const enum damon_prep_action action;
+	const char *name;
+};
+
+static const struct damon_sysfs_prep_action_name
+damon_sysfs_prep_action_names[] = {
+	{
+		.action = DAMON_PREP_SET_PGIDLE,
+		.name = "set_pgidle",
+	},
+};
+
+static ssize_t prep_action_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	struct damon_sysfs_prep *prep = container_of(kobj,
+			struct damon_sysfs_prep, kobj);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(damon_sysfs_prep_action_names); i++) {
+		const struct damon_sysfs_prep_action_name *action_name;
+
+		action_name = &damon_sysfs_prep_action_names[i];
+		if (action_name->action == prep->action)
+			return sysfs_emit(buf, "%s\n", action_name->name);
+	}
+	return -EINVAL;
+}
+
+static ssize_t prep_action_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct damon_sysfs_prep *prep = container_of(kobj,
+			struct damon_sysfs_prep, kobj);
+	ssize_t ret = -EINVAL;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(damon_sysfs_prep_action_names); i++) {
+		const struct damon_sysfs_prep_action_name *action_name;
+
+		action_name = &damon_sysfs_prep_action_names[i];
+		if (sysfs_streq(buf, action_name->name)) {
+			if (!mutex_trylock(&damon_sysfs_lock))
+				return -EBUSY;
+			prep->action = action_name->action;
+			mutex_unlock(&damon_sysfs_lock);
+			ret = count;
+			break;
+		}
+	}
+	return ret;
+}
+
+static void damon_sysfs_prep_release(struct kobject *kobj)
+{
+	struct damon_sysfs_prep *prep = container_of(kobj,
+			struct damon_sysfs_prep, kobj);
+
+	kfree(prep);
+}
+
+static struct kobj_attribute damon_sysfs_prep_prep_action_attr =
+		__ATTR_RW_MODE(prep_action, 0600);
+
+static struct attribute *damon_sysfs_prep_attrs[] = {
+	&damon_sysfs_prep_prep_action_attr.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(damon_sysfs_prep);
+
+static const struct kobj_type damon_sysfs_prep_ktype = {
+	.release = damon_sysfs_prep_release,
+	.sysfs_ops = &kobj_sysfs_ops,
+	.default_groups = damon_sysfs_prep_groups,
+};
+
+/*
+ * preps directory
+ */
+
+struct damon_sysfs_preps {
+	struct kobject kobj;
+	struct damon_sysfs_prep **preps_arr;
+	int nr;
+};
+
+static struct damon_sysfs_preps *damon_sysfs_preps_alloc(void)
+{
+	return kzalloc_obj(struct damon_sysfs_preps);
+}
+
+static void damon_sysfs_preps_rm_dirs(struct damon_sysfs_preps *preps)
+{
+	struct damon_sysfs_prep **preps_arr = preps->preps_arr;
+	int i;
+
+	for (i = 0; i < preps->nr; i++) {
+		kobject_del(&preps_arr[i]->kobj);
+		kobject_put(&preps_arr[i]->kobj);
+	}
+	preps->nr = 0;
+	kfree(preps_arr);
+	preps->preps_arr = NULL;
+}
+
+static int damon_sysfs_preps_add_dirs(struct damon_sysfs_preps *preps,
+		int nr_preps)
+{
+	struct damon_sysfs_prep **preps_arr, *prep;
+	int err, i;
+
+	damon_sysfs_preps_rm_dirs(preps);
+	if (!nr_preps)
+		return 0;
+
+	preps_arr = kmalloc_objs(*preps_arr, nr_preps,
+				   GFP_KERNEL | __GFP_NOWARN);
+	if (!preps_arr)
+		return -ENOMEM;
+	preps->preps_arr = preps_arr;
+
+	for (i = 0; i < nr_preps; i++) {
+		prep = damon_sysfs_prep_alloc();
+		if (!prep) {
+			damon_sysfs_preps_rm_dirs(preps);
+			return -ENOMEM;
+		}
+
+		err = kobject_init_and_add(&prep->kobj,
+				&damon_sysfs_prep_ktype, &preps->kobj, "%d",
+				i);
+		if (err) {
+			kobject_put(&prep->kobj);
+			damon_sysfs_preps_rm_dirs(preps);
+			return err;
+		}
+
+		preps_arr[i] = prep;
+		preps->nr++;
+	}
+	return 0;
+}
+
+static ssize_t nr_preps_show(struct kobject *kobj, struct kobj_attribute *attr,
+		char *buf)
+{
+	struct damon_sysfs_preps *preps = container_of(kobj,
+			struct damon_sysfs_preps, kobj);
+
+	return sysfs_emit(buf, "%d\n", preps->nr);
+}
+
+static ssize_t nr_preps_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct damon_sysfs_preps *preps;
+	int nr, err = kstrtoint(buf, 0, &nr);
+
+	if (err)
+		return err;
+	if (nr < 0)
+		return -EINVAL;
+
+	preps = container_of(kobj, struct damon_sysfs_preps, kobj);
+
+	if (!mutex_trylock(&damon_sysfs_lock))
+		return -EBUSY;
+	err = damon_sysfs_preps_add_dirs(preps, nr);
+	mutex_unlock(&damon_sysfs_lock);
+	if (err)
+		return err;
+
+	return count;
+}
+
+static void damon_sysfs_preps_release(struct kobject *kobj)
+{
+	kfree(container_of(kobj, struct damon_sysfs_preps, kobj));
+}
+
+static struct kobj_attribute damon_sysfs_preps_nr_attr =
+		__ATTR_RW_MODE(nr_preps, 0600);
+
+static struct attribute *damon_sysfs_preps_attrs[] = {
+	&damon_sysfs_preps_nr_attr.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(damon_sysfs_preps);
+
+static const struct kobj_type damon_sysfs_preps_ktype = {
+	.release = damon_sysfs_preps_release,
+	.sysfs_ops = &kobj_sysfs_ops,
+	.default_groups = damon_sysfs_preps_groups,
+};
+
 /*
  * filter directory
  */
@@ -760,6 +975,8 @@ struct damon_sysfs_filter {
 	bool matching;
 	bool allow;
 	char *path;
+	unsigned long range_min;
+	unsigned long range_max;
 };
 
 static struct damon_sysfs_filter *damon_sysfs_filter_alloc(void)
@@ -782,6 +999,18 @@ damon_sysfs_filter_type_names[] = {
 		.type = DAMON_FILTER_TYPE_MEMCG,
 		.name = "memcg",
 	},
+	{
+		.type = DAMON_FILTER_TYPE_PGIDLE_UNSET,
+		.name = "pgidle_unset",
+	},
+	{
+		.type = DAMON_FILTER_TYPE_PGIDLE_SET,
+		.name = "pgidle_set",
+	},
+	{
+		.type = DAMON_FILTER_TYPE_HUGEPAGE_SIZE,
+		.name = "hugepage_size",
+	},
 };
 
 static ssize_t type_show(struct kobject *kobj,
@@ -904,6 +1133,44 @@ static ssize_t path_store(struct kobject *kobj,
 	return count;
 }
 
+static ssize_t min_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	struct damon_sysfs_filter *filter = container_of(kobj,
+			struct damon_sysfs_filter, kobj);
+
+	return sysfs_emit(buf, "%lu\n", filter->range_min);
+}
+
+static ssize_t min_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct damon_sysfs_filter *filter = container_of(kobj,
+			struct damon_sysfs_filter, kobj);
+	int err = kstrtoul(buf, 0, &filter->range_min);
+
+	return err ? err : count;
+}
+
+static ssize_t max_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	struct damon_sysfs_filter *filter = container_of(kobj,
+			struct damon_sysfs_filter, kobj);
+
+	return sysfs_emit(buf, "%lu\n", filter->range_max);
+}
+
+static ssize_t max_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct damon_sysfs_filter *filter = container_of(kobj,
+			struct damon_sysfs_filter, kobj);
+	int err = kstrtoul(buf, 0, &filter->range_max);
+
+	return err ? err : count;
+}
+
 static void damon_sysfs_filter_release(struct kobject *kobj)
 {
 	struct damon_sysfs_filter *filter = container_of(kobj,
@@ -925,11 +1192,19 @@ static struct kobj_attribute damon_sysfs_filter_allow_attr =
 static struct kobj_attribute damon_sysfs_filter_path_attr =
 		__ATTR_RW_MODE(path, 0600);
 
+static struct kobj_attribute damon_sysfs_filter_min_attr =
+		__ATTR_RW_MODE(min, 0600);
+
+static struct kobj_attribute damon_sysfs_filter_max_attr =
+		__ATTR_RW_MODE(max, 0600);
+
 static struct attribute *damon_sysfs_filter_attrs[] = {
 	&damon_sysfs_filter_type_attr.attr,
 	&damon_sysfs_filter_matching_attr.attr,
 	&damon_sysfs_filter_allow_attr.attr,
 	&damon_sysfs_filter_path_attr.attr,
+	&damon_sysfs_filter_min_attr.attr,
+	&damon_sysfs_filter_max_attr.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(damon_sysfs_filter);
@@ -1066,6 +1341,7 @@ static const struct kobj_type damon_sysfs_filters_ktype = {
 struct damon_sysfs_probe {
 	struct kobject kobj;
 	unsigned int weight;
+	struct damon_sysfs_preps *preps;
 	struct damon_sysfs_filters *filters;
 };
 
@@ -1076,25 +1352,50 @@ static struct damon_sysfs_probe *damon_sysfs_probe_alloc(void)
 
 static int damon_sysfs_probe_add_dirs(struct damon_sysfs_probe *probe)
 {
+	struct damon_sysfs_preps *preps;
 	struct damon_sysfs_filters *filters;
 	int err;
 
-	filters = damon_sysfs_filters_alloc();
-	if (!filters)
+	preps = damon_sysfs_preps_alloc();
+	if (!preps)
 		return -ENOMEM;
+	probe->preps = preps;
+
+	err = kobject_init_and_add(&preps->kobj, &damon_sysfs_preps_ktype,
+			&probe->kobj, "preps");
+	if (err)
+		goto put_preps_out;
+
+	filters = damon_sysfs_filters_alloc();
+	if (!filters) {
+		err = -ENOMEM;
+		goto del_preps_out;
+	}
 	probe->filters = filters;
 
 	err = kobject_init_and_add(&filters->kobj, &damon_sysfs_filters_ktype,
 			&probe->kobj, "filters");
-	if (err) {
-		kobject_put(&filters->kobj);
-		probe->filters = NULL;
-	}
+	if (err)
+		goto put_filters_out;
+	return err;
+
+put_filters_out:
+	kobject_put(&filters->kobj);
+	probe->filters = NULL;
+del_preps_out:
+	kobject_del(&preps->kobj);
+put_preps_out:
+	kobject_put(&preps->kobj);
+	probe->preps = NULL;
 	return err;
 }
 
 static void damon_sysfs_probe_rm_dirs(struct damon_sysfs_probe *probe)
 {
+	if (probe->preps) {
+		damon_sysfs_preps_rm_dirs(probe->preps);
+		kobject_put(&probe->preps->kobj);
+	}
 	if (probe->filters) {
 		damon_sysfs_filters_rm_dirs(probe->filters);
 		kobject_put(&probe->filters->kobj);
@@ -1234,7 +1535,7 @@ static ssize_t nr_probes_store(struct kobject *kobj,
 
 	if (err)
 		return err;
-	if (nr < 0 || nr > DAMON_MAX_PROBES)
+	if (nr < 0)
 		return -EINVAL;
 
 	probes = container_of(kobj, struct damon_sysfs_probes, kobj);
@@ -1789,6 +2090,7 @@ struct damon_sysfs_kdamond {
 	struct damon_sysfs_contexts *contexts;
 	struct damon_ctx *damon_ctx;
 	unsigned int refresh_ms;
+	unsigned long next_refresh_jiffies;
 };
 
 static struct damon_sysfs_kdamond *damon_sysfs_kdamond_alloc(void)
@@ -1933,6 +2235,23 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx,
 	return damon_set_attrs(ctx, &attrs);
 }
 
+static int damon_sysfs_set_preps(struct damon_probe *probe,
+		struct damon_sysfs_preps *sys_preps)
+{
+	int i;
+
+	for (i = 0; i < sys_preps->nr; i++) {
+		struct damon_sysfs_prep *sys_prep = sys_preps->preps_arr[i];
+		struct damon_prep *prep;
+
+		prep = damon_new_prep(sys_prep->action);
+		if (!prep)
+			return -ENOMEM;
+		damon_add_prep(probe, prep);
+	}
+	return 0;
+}
+
 static int damon_sysfs_set_filters(struct damon_probe *probe,
 		struct damon_sysfs_filters *sys_filters)
 {
@@ -1958,6 +2277,9 @@ static int damon_sysfs_set_filters(struct damon_probe *probe,
 				damon_destroy_filter(filter);
 				return err;
 			}
+		} else if (filter->type == DAMON_FILTER_TYPE_HUGEPAGE_SIZE) {
+			filter->range_min = sys_filter->range_min;
+			filter->range_max = sys_filter->range_max;
 		}
 		damon_add_filter(probe, filter);
 	}
@@ -1968,7 +2290,15 @@ static int damon_sysfs_set_probe(struct damon_probe *probe,
 		struct damon_sysfs_probe *sys_probe)
 {
 	struct damon_sysfs_filters *sys_filters;
+	struct damon_sysfs_preps *sys_preps;
+	int err;
 
+	sys_preps = sys_probe->preps;
+	if (sys_preps) {
+		err = damon_sysfs_set_preps(probe, sys_preps);
+		if (err)
+			return err;
+	}
 	sys_filters = sys_probe->filters;
 	if (!sys_filters)
 		return 0;
@@ -2035,9 +2365,8 @@ static int damon_sysfs_add_target(struct damon_sysfs_target *sys_target,
 		return -ENOMEM;
 	damon_add_target(ctx, t);
 	if (damon_target_has_pid(ctx)) {
-		t->pid = find_get_pid(sys_target->pid);
-		if (!t->pid)
-			/* caller will destroy targets */
+		/* caller will destroy targets */
+		if (damon_set_target_pid(t, sys_target->pid))
 			return -EINVAL;
 	}
 	t->obsolete = sys_target->obsolete;
@@ -2215,17 +2544,15 @@ static struct damon_ctx *damon_sysfs_build_ctx(
 	return ctx;
 }
 
-static unsigned long damon_sysfs_next_update_jiffies;
-
 static int damon_sysfs_repeat_call_fn(void *data)
 {
 	struct damon_sysfs_kdamond *sysfs_kdamond = data;
 
 	if (!sysfs_kdamond->refresh_ms)
 		return 0;
-	if (time_before(jiffies, damon_sysfs_next_update_jiffies))
+	if (time_before(jiffies, sysfs_kdamond->next_refresh_jiffies))
 		return 0;
-	damon_sysfs_next_update_jiffies = jiffies +
+	sysfs_kdamond->next_refresh_jiffies = jiffies +
 		msecs_to_jiffies(sysfs_kdamond->refresh_ms);
 
 	if (!mutex_trylock(&damon_sysfs_lock))
@@ -2273,14 +2600,15 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond)
 	}
 	kdamond->damon_ctx = ctx;
 
-	damon_sysfs_next_update_jiffies =
-		jiffies + msecs_to_jiffies(kdamond->refresh_ms);
+	kdamond->next_refresh_jiffies = jiffies +
+		msecs_to_jiffies(kdamond->refresh_ms);
 
 	repeat_call_control->fn = damon_sysfs_repeat_call_fn;
 	repeat_call_control->data = kdamond;
 	repeat_call_control->repeat = true;
 	repeat_call_control->dealloc_on_cancel = true;
-	if (damon_call(ctx, repeat_call_control))
+	err = damon_call(ctx, repeat_call_control);
+	if (err)
 		kfree(repeat_call_control);
 	return err;
 }
diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h
index 4a536d41cdb2d0..2111faa581532a 100644
--- a/mm/damon/tests/core-kunit.h
+++ b/mm/damon/tests/core-kunit.h
@@ -152,6 +152,8 @@ static void damon_test_split_at(struct kunit *test)
 	}
 	r->nr_accesses = 42;
 	r->last_nr_accesses = 15;
+	r->probe_hits[0] = 7;
+	r->last_probe_hits[0] = 3;
 	r->age = 10;
 	damon_add_region(r, t);
 	damon_split_region_at(t, r, 25);
@@ -168,6 +170,8 @@ static void damon_test_split_at(struct kunit *test)
 
 	KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses);
 	KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses);
+	KUNIT_EXPECT_EQ(test, r->probe_hits[0], r_new->probe_hits[0]);
+	KUNIT_EXPECT_EQ(test, r->last_probe_hits[0], r_new->last_probe_hits[0]);
 	KUNIT_EXPECT_EQ(test, r->age, r_new->age);
 
 out:
@@ -189,6 +193,7 @@ static void damon_test_merge_two(struct kunit *test)
 		kunit_skip(test, "region alloc fail");
 	}
 	r->nr_accesses = 10;
+	r->probe_hits[0] = 6;
 	r->age = 9;
 	damon_add_region(r, t);
 	r2 = damon_new_region(100, 300);
@@ -197,6 +202,7 @@ static void damon_test_merge_two(struct kunit *test)
 		kunit_skip(test, "second region alloc fail");
 	}
 	r2->nr_accesses = 20;
+	r2->probe_hits[0] = 14;
 	r2->age = 21;
 	damon_add_region(r2, t);
 
@@ -204,6 +210,7 @@ static void damon_test_merge_two(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
 	KUNIT_EXPECT_EQ(test, r->ar.end, 300ul);
 	KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u);
+	KUNIT_EXPECT_EQ(test, r->probe_hits[0], 11);
 	KUNIT_EXPECT_EQ(test, r->age, 17u);
 
 	i = 0;
@@ -434,17 +441,17 @@ static void damon_test_ops_registration(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, damon_select_ops(c, NR_DAMON_OPS), -EINVAL);
 
 	/* Registration should success after unregistration */
-	mutex_lock(&damon_ops_lock);
-	bak = damon_registered_ops[DAMON_OPS_VADDR];
-	damon_registered_ops[DAMON_OPS_VADDR] = (struct damon_operations){};
-	mutex_unlock(&damon_ops_lock);
+	scoped_guard(mutex, &damon_ops_lock) {
+		bak = damon_registered_ops[DAMON_OPS_VADDR];
+		damon_registered_ops[DAMON_OPS_VADDR] =
+			(struct damon_operations){};
+	}
 
 	ops.id = DAMON_OPS_VADDR;
 	KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), 0);
 
-	mutex_lock(&damon_ops_lock);
-	damon_registered_ops[DAMON_OPS_VADDR] = bak;
-	mutex_unlock(&damon_ops_lock);
+	scoped_guard(mutex, &damon_ops_lock)
+		damon_registered_ops[DAMON_OPS_VADDR] = bak;
 
 	/* Check double-registration failure again */
 	KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), -EINVAL);
@@ -452,10 +459,9 @@ static void damon_test_ops_registration(struct kunit *test)
 	damon_destroy_ctx(c);
 
 	if (need_cleanup) {
-		mutex_lock(&damon_ops_lock);
-		damon_registered_ops[DAMON_OPS_VADDR] =
-			(struct damon_operations){};
-		mutex_unlock(&damon_ops_lock);
+		scoped_guard(mutex, &damon_ops_lock)
+			damon_registered_ops[DAMON_OPS_VADDR] =
+				(struct damon_operations){};
 	}
 }
 
@@ -463,11 +469,12 @@ static void damon_test_set_regions_for(struct kunit *test,
 		struct damon_addr_range *old_ranges, int sz_old_ranges,
 		struct damon_addr_range *new_ranges, int sz_new_ranges,
 		unsigned long min_region_sz,
-		struct damon_addr_range *expect_ranges, int sz_expect_ranges)
+		struct damon_addr_range *expect_ranges, int sz_expect_ranges,
+		int expect_err)
 {
 	struct damon_target *t;
 	struct damon_region *r;
-	int i;
+	int i, err;
 
 	t = damon_new_target();
 	if (!t)
@@ -481,7 +488,8 @@ static void damon_test_set_regions_for(struct kunit *test,
 		damon_add_region(r, t);
 	}
 
-	damon_set_regions(t, new_ranges, sz_new_ranges, min_region_sz);
+	err = damon_set_regions(t, new_ranges, sz_new_ranges, min_region_sz);
+	KUNIT_EXPECT_EQ(test, err, expect_err);
 
 	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), sz_expect_ranges);
 	if (damon_nr_regions(t) != sz_expect_ranges) {
@@ -510,7 +518,7 @@ static void damon_test_set_regions(struct kunit *test)
 			(struct damon_addr_range[]){
 			{.start = 5, .end = 15},
 			{.start = 15, .end = 25},
-			}, 2);
+			}, 2, 0);
 	/* Un-intersecting regions should be removed. */
 	damon_test_set_regions_for(test,
 			(struct damon_addr_range[]){
@@ -523,7 +531,7 @@ static void damon_test_set_regions(struct kunit *test)
 			1,
 			(struct damon_addr_range[]){
 			{.start = 18, .end = 23},
-			}, 1);
+			}, 1, 0);
 	/*
 	 * Holes should be filled up with new regions.
 	 *
@@ -544,7 +552,7 @@ static void damon_test_set_regions(struct kunit *test)
 			{.start = 8, .end = 16},
 			{.start = 16, .end = 24},
 			{.start = 24, .end = 28},
-			}, 3);
+			}, 3, 0);
 	/*
 	 * New regions should be able to be appended.
 	 *
@@ -566,7 +574,7 @@ static void damon_test_set_regions(struct kunit *test)
 			{.start = 0, .end = 4},
 			{.start = 4, .end = 15},
 			{.start = 25, .end = 40},
-			}, 3);
+			}, 3, 0);
 	/*
 	 * New regions should be able to be inserted.
 	 *
@@ -589,7 +597,64 @@ static void damon_test_set_regions(struct kunit *test)
 			{.start = 0, .end = 15},
 			{.start = 25, .end = 40},
 			{.start = 44, .end = 50},
-			}, 3);
+			}, 3, 0);
+	/* Zero size regions should return -EINVAL. */
+	damon_test_set_regions_for(test,
+			(struct damon_addr_range[]){}, 0,
+			(struct damon_addr_range[]){
+			{.start = 42, .end = 42},
+			}, 1, 1,
+			(struct damon_addr_range[]){}, 0, -EINVAL);
+	/* Negative size regions should return -EINVAL. */
+	damon_test_set_regions_for(test,
+			(struct damon_addr_range[]){}, 0,
+			(struct damon_addr_range[]){
+			{.start = 42, .end = 21},
+			}, 1, 1,
+			(struct damon_addr_range[]){}, 0, -EINVAL);
+	/*
+	 * Regions resulting in same region after alignment should return
+	 * -EINVAL.
+	 */
+	damon_test_set_regions_for(test,
+			(struct damon_addr_range[]){}, 0,
+			(struct damon_addr_range[]){
+			{.start = 10, .end = 20},
+			{.start = 20, .end = 30},
+			}, 2, 4096,
+			(struct damon_addr_range[]){}, 0, -EINVAL);
+}
+
+static void damon_test_nr_samples_per_aggr(struct kunit *test)
+{
+	struct damon_attrs attrs;
+
+	/* Zero sample interval is treated as one. */
+	attrs = (struct damon_attrs){
+		.sample_interval = 0,
+		.aggr_interval = 5000,
+	};
+	KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), 5000);
+
+	/* Zero sample and aggregation intervals cover the zero-result fallback. */
+	attrs = (struct damon_attrs){
+		.sample_interval = 0,
+		.aggr_interval = 0,
+	};
+	KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs), 1);
+
+	/*
+	 * Too large aggregation interval on 64 bit system doesn't cause
+	 * overflow
+	 */
+	if (ULONG_MAX > UINT_MAX) {
+		attrs = (struct damon_attrs){
+			.sample_interval = 1,
+			.aggr_interval = (unsigned long)UINT_MAX + 1,
+		};
+		KUNIT_EXPECT_EQ(test, damon_nr_samples_per_aggr(&attrs),
+				UINT_MAX);
+	}
 }
 
 static void damon_test_update_monitoring_result(struct kunit *test)
@@ -747,23 +812,62 @@ static void damos_test_new_filter(struct kunit *test)
 	damos_destroy_filter(filter);
 }
 
+static void damos_test_new_scheme_keeps_src_quota(struct kunit *test)
+{
+	struct damos_access_pattern pattern = {};
+	struct damon_target target = {};
+	struct damos_quota quota = {
+		.sz = SZ_64K,
+		.esz = 123,
+		.esz_bp = 456,
+		.total_charged_sz = 789,
+		.total_charged_ns = 1011,
+		.charged_sz = 12,
+		.charged_from = 13,
+		.charge_target_from = &target,
+		.charge_addr_from = 14,
+	};
+	struct damos_watermarks wmarks = {};
+	struct damos *s;
+
+	s = damon_new_scheme(&pattern, DAMOS_STAT, 0, &quota, &wmarks,
+			NUMA_NO_NODE);
+	if (!s)
+		kunit_skip(test, "scheme alloc fail");
+	KUNIT_EXPECT_EQ(test, s->quota.sz, (unsigned long)SZ_64K);
+	KUNIT_EXPECT_EQ(test, s->quota.esz, 0ul);
+	KUNIT_EXPECT_EQ(test, s->quota.esz_bp, 0ul);
+	KUNIT_EXPECT_EQ(test, s->quota.total_charged_sz, 0ul);
+	KUNIT_EXPECT_EQ(test, s->quota.total_charged_ns, 0ul);
+	KUNIT_EXPECT_EQ(test, s->quota.charged_sz, 0ul);
+	KUNIT_EXPECT_EQ(test, s->quota.charged_from, 0ul);
+	KUNIT_EXPECT_PTR_EQ(test, s->quota.charge_target_from, NULL);
+	KUNIT_EXPECT_EQ(test, s->quota.charge_addr_from, 0ul);
+	KUNIT_EXPECT_EQ(test, quota.esz, 123ul);
+	KUNIT_EXPECT_EQ(test, quota.esz_bp, 456ul);
+	KUNIT_EXPECT_EQ(test, quota.total_charged_sz, 789ul);
+	KUNIT_EXPECT_EQ(test, quota.total_charged_ns, 1011ul);
+	KUNIT_EXPECT_EQ(test, quota.charged_sz, 12ul);
+	KUNIT_EXPECT_EQ(test, quota.charged_from, 13ul);
+	KUNIT_EXPECT_PTR_EQ(test, quota.charge_target_from, &target);
+	KUNIT_EXPECT_EQ(test, quota.charge_addr_from, 14ul);
+	damon_destroy_scheme(s);
+}
+
 static void damos_test_commit_quota_goal_for(struct kunit *test,
 		struct damos_quota_goal *dst,
 		struct damos_quota_goal *src)
 {
-	u64 dst_last_psi_total = 0;
-
-	if (dst->metric == DAMOS_QUOTA_SOME_MEM_PSI_US)
-		dst_last_psi_total = dst->last_psi_total;
 	damos_commit_quota_goal(dst, src);
 
 	KUNIT_EXPECT_EQ(test, dst->metric, src->metric);
 	KUNIT_EXPECT_EQ(test, dst->target_value, src->target_value);
 	if (src->metric == DAMOS_QUOTA_USER_INPUT)
 		KUNIT_EXPECT_EQ(test, dst->current_value, src->current_value);
-	if (dst_last_psi_total && src->metric == DAMOS_QUOTA_SOME_MEM_PSI_US)
-		KUNIT_EXPECT_EQ(test, dst->last_psi_total, dst_last_psi_total);
 	switch (dst->metric) {
+	case DAMOS_QUOTA_SOME_MEM_PSI_US:
+		KUNIT_EXPECT_EQ(test, dst->last_psi_total, U64_MAX);
+		break;
 	case DAMOS_QUOTA_NODE_MEM_USED_BP:
 	case DAMOS_QUOTA_NODE_MEM_FREE_BP:
 		KUNIT_EXPECT_EQ(test, dst->nid, src->nid);
@@ -773,6 +877,9 @@ static void damos_test_commit_quota_goal_for(struct kunit *test,
 		KUNIT_EXPECT_EQ(test, dst->nid, src->nid);
 		KUNIT_EXPECT_EQ(test, dst->memcg_id, src->memcg_id);
 		break;
+	case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP:
+		KUNIT_EXPECT_EQ(test, dst->nid, src->nid);
+		break;
 	default:
 		break;
 	}
@@ -787,6 +894,13 @@ static void damos_test_commit_quota_goal(struct kunit *test)
 		.last_psi_total = 456,
 	};
 
+	damos_test_commit_quota_goal_for(test, &dst,
+			&(struct damos_quota_goal) {
+			.metric = DAMOS_QUOTA_SOME_MEM_PSI_US,
+			.target_value = 234,
+			.current_value = 345,
+			.last_psi_total = 567,
+			});
 	damos_test_commit_quota_goal_for(test, &dst,
 			&(struct damos_quota_goal){
 			.metric = DAMOS_QUOTA_USER_INPUT,
@@ -829,6 +943,55 @@ static void damos_test_commit_quota_goal(struct kunit *test)
 			.current_value = 345,
 			.last_psi_total = 567,
 			});
+	damos_test_commit_quota_goal_for(test, &dst,
+			&(struct damos_quota_goal){
+			.metric = DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP,
+			.target_value = 12,
+			.current_value = 345,
+			.nid = 6,
+			});
+}
+
+static void damos_test_set_psi_current_val(struct kunit *test)
+{
+	struct damos s = {
+		.quota.goal_tuner = DAMOS_QUOTA_GOAL_TUNER_CONSIST,
+	};
+	struct damos_quota_goal goal = {
+		.metric = DAMOS_QUOTA_SOME_MEM_PSI_US,
+		.target_value = 100,
+		.last_psi_total = U64_MAX,
+	};
+
+	/* uninitialized last_psi_total keeps the consist tuner quota */
+	damos_set_psi_current_val(1000, &goal, &s);
+	KUNIT_EXPECT_EQ(test, goal.current_value, 100ul);
+	KUNIT_EXPECT_EQ(test, goal.last_psi_total, 1000ull);
+
+	/* initialized last_psi_total gives the delta */
+	damos_set_psi_current_val(1030, &goal, &s);
+	KUNIT_EXPECT_EQ(test, goal.current_value, 30ul);
+	KUNIT_EXPECT_EQ(test, goal.last_psi_total, 1030ull);
+
+	/* temporal tuner keeps a zero quota */
+	s.quota.goal_tuner = DAMOS_QUOTA_GOAL_TUNER_TEMPORAL;
+	s.quota.esz = 0;
+	goal.last_psi_total = U64_MAX;
+	damos_set_psi_current_val(2000, &goal, &s);
+	KUNIT_EXPECT_EQ(test, goal.current_value, 100ul);
+	KUNIT_EXPECT_EQ(test, goal.last_psi_total, 2000ull);
+
+	/* temporal tuner keeps a non-zero quota */
+	s.quota.esz = SZ_64K;
+	goal.last_psi_total = U64_MAX;
+	damos_set_psi_current_val(3000, &goal, &s);
+	KUNIT_EXPECT_EQ(test, goal.current_value, 0ul);
+	KUNIT_EXPECT_EQ(test, goal.last_psi_total, 3000ull);
+
+	/* temporal tuner uses the measured PSI delta */
+	damos_set_psi_current_val(3250, &goal, &s);
+	KUNIT_EXPECT_EQ(test, goal.current_value, 250ul);
+	KUNIT_EXPECT_EQ(test, goal.last_psi_total, 3250ull);
 }
 
 static void damos_test_commit_quota_goals_for(struct kunit *test,
@@ -1058,9 +1221,15 @@ static void damos_test_commit_dests(struct kunit *test)
 }
 
 static void damos_test_commit_filter_for(struct kunit *test,
-		struct damos_filter *dst, struct damos_filter *src)
+		struct damos_filter *dst, struct damos_filter *src,
+		bool expect_fail)
 {
-	damos_commit_filter(dst, src);
+	int err;
+
+	err = damos_commit_filter(dst, src);
+	KUNIT_EXPECT_EQ(test, err != 0, expect_fail);
+	if (expect_fail)
+		return;
 	KUNIT_EXPECT_EQ(test, dst->type, src->type);
 	KUNIT_EXPECT_EQ(test, dst->matching, src->matching);
 	KUNIT_EXPECT_EQ(test, dst->allow, src->allow);
@@ -1081,6 +1250,10 @@ static void damos_test_commit_filter_for(struct kunit *test,
 		KUNIT_EXPECT_EQ(test, dst->sz_range.min, src->sz_range.min);
 		KUNIT_EXPECT_EQ(test, dst->sz_range.max, src->sz_range.max);
 		break;
+	case DAMOS_FILTER_TYPE_PROBE_HITS_WSUM:
+		KUNIT_EXPECT_EQ(test, dst->range_min, src->range_min);
+		KUNIT_EXPECT_EQ(test, dst->range_max, src->range_max);
+		break;
 	default:
 		break;
 	}
@@ -1099,47 +1272,77 @@ static void damos_test_commit_filter(struct kunit *test)
 			.type = DAMOS_FILTER_TYPE_ANON,
 			.matching = true,
 			.allow = true,
-			});
+			}, false);
 	damos_test_commit_filter_for(test, &dst,
 			&(struct damos_filter){
 			.type = DAMOS_FILTER_TYPE_MEMCG,
 			.matching = false,
 			.allow = false,
 			.memcg_id = 123,
-			});
+			}, false);
 	damos_test_commit_filter_for(test, &dst,
 			&(struct damos_filter){
 			.type = DAMOS_FILTER_TYPE_YOUNG,
 			.matching = true,
 			.allow = true,
-			});
+			}, false);
 	damos_test_commit_filter_for(test, &dst,
 			&(struct damos_filter){
 			.type = DAMOS_FILTER_TYPE_HUGEPAGE_SIZE,
 			.matching = false,
 			.allow = false,
 			.sz_range = {.min = 234, .max = 345},
-			});
+			}, false);
+	damos_test_commit_filter_for(test, &dst,
+			&(struct damos_filter){
+			.type = DAMOS_FILTER_TYPE_HUGEPAGE_SIZE,
+			.matching = false,
+			.allow = false,
+			.sz_range = {.min = 456, .max = 123},
+			}, true);
 	damos_test_commit_filter_for(test, &dst,
 			&(struct damos_filter){
 			.type = DAMOS_FILTER_TYPE_UNMAPPED,
 			.matching = true,
 			.allow = true,
-			});
+			}, false);
 	damos_test_commit_filter_for(test, &dst,
 			&(struct damos_filter){
 			.type = DAMOS_FILTER_TYPE_ADDR,
 			.matching = false,
 			.allow = false,
 			.addr_range = {.start = 456, .end = 567},
-			});
+			}, false);
+	damos_test_commit_filter_for(test, &dst,
+			&(struct damos_filter){
+			.type = DAMOS_FILTER_TYPE_ADDR,
+			.matching = false,
+			.allow = false,
+			.addr_range = {.start = 567, .end = 456},
+			}, true);
 	damos_test_commit_filter_for(test, &dst,
 			&(struct damos_filter){
 			.type = DAMOS_FILTER_TYPE_TARGET,
 			.matching = true,
 			.allow = true,
 			.target_idx = 6,
-			});
+			}, false);
+	damos_test_commit_filter_for(test, &dst,
+			&(struct damos_filter){
+			.type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM,
+			.matching = false,
+			.allow = true,
+			.range_min = 12,
+			.range_max = 34,
+			}, false);
+	damos_test_commit_filter_for(test, &dst,
+			&(struct damos_filter){
+			.type = DAMOS_FILTER_TYPE_PROBE_HITS_WSUM,
+			.matching = false,
+			.allow = true,
+			.range_min = 34,
+			.range_max = 12,
+			}, true);
 }
 
 static void damos_test_help_initailize_scheme(struct damos *scheme)
@@ -1306,6 +1509,128 @@ static void damon_test_commit_target_regions(struct kunit *test)
 			(unsigned long[][2]) {{3, 8}, {8, 10}}, 2);
 }
 
+static void damon_test_commit_filter_for(struct kunit *test,
+		struct damon_filter *dst, struct damon_filter *src)
+{
+	damon_commit_filter(dst, src);
+	KUNIT_EXPECT_EQ(test, dst->type, src->type);
+	KUNIT_EXPECT_EQ(test, dst->matching, src->matching);
+	KUNIT_EXPECT_EQ(test, dst->allow, src->allow);
+	switch (src->type) {
+	case DAMON_FILTER_TYPE_MEMCG:
+		KUNIT_EXPECT_EQ(test, dst->memcg_id, src->memcg_id);
+		break;
+	default:
+		break;
+	}
+}
+
+static void damon_test_commit_filter(struct kunit *test)
+{
+	struct damon_filter dst = {
+		.type = DAMON_FILTER_TYPE_ANON,
+		.matching = false,
+		.allow = false,
+	};
+
+	damon_test_commit_filter_for(test, &dst,
+			&(struct damon_filter){
+			.type = DAMON_FILTER_TYPE_ANON,
+			.matching = true,
+			.allow = true,
+			});
+	damon_test_commit_filter_for(test, &dst,
+			&(struct damon_filter){
+			.type = DAMON_FILTER_TYPE_MEMCG,
+			.matching = false,
+			.allow = false,
+			.memcg_id = 123,
+			});
+}
+
+static struct damon_ctx *damon_test_help_setup_probes(unsigned int weights[],
+		int nr_weights)
+{
+	struct damon_ctx *ctx;
+	struct damon_probe *probe;
+	int i;
+
+	ctx = damon_new_ctx();
+	if (!ctx)
+		return NULL;
+	for (i = 0; i < nr_weights; i++) {
+		probe = damon_new_probe();
+		if (!probe) {
+			damon_destroy_ctx(ctx);
+			return NULL;
+		}
+		probe->weight = weights[i];
+		damon_add_probe(ctx, probe);
+	}
+	return ctx;
+}
+
+static void damon_test_commit_probes_for(struct kunit *test,
+		unsigned int dst_weights[], int nr_dst_probes,
+		unsigned int src_weights[], int nr_src_probes)
+{
+	struct damon_ctx *dst, *src;
+	int err;
+	struct damon_probe *dst_probe, *src_probe;
+
+	dst = damon_test_help_setup_probes(dst_weights, nr_dst_probes);
+	if (!dst)
+		kunit_skip(test, "dst alloc fail");
+	src = damon_test_help_setup_probes(src_weights, nr_src_probes);
+	if (!src) {
+		damon_destroy_ctx(dst);
+		kunit_skip(test, "src alloc fail");
+	}
+
+	err = damon_commit_probes(dst, src);
+	KUNIT_EXPECT_EQ(test, err, 0);
+	if (err)
+		goto out;
+	nr_dst_probes = 0;
+	damon_for_each_probe(dst_probe, dst)
+		nr_dst_probes++;
+	nr_src_probes = 0;
+	damon_for_each_probe(src_probe, src)
+		nr_src_probes++;
+	KUNIT_EXPECT_EQ(test, nr_dst_probes, nr_src_probes);
+	if (nr_dst_probes != nr_src_probes)
+		goto out;
+	nr_dst_probes = 0;
+	damon_for_each_probe(dst_probe, dst) {
+		src_probe = damon_nth_probe(nr_dst_probes, src);
+		KUNIT_EXPECT_EQ(test, src_probe->weight, dst_probe->weight);
+		nr_dst_probes++;
+	}
+out:
+	damon_destroy_ctx(dst);
+	damon_destroy_ctx(src);
+}
+
+static void damon_test_commit_probes(struct kunit *test)
+{
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){}, 0, (unsigned int[]){}, 0);
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){}, 0, (unsigned int[]){1}, 1);
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){}, 0, (unsigned int[]){1, 2}, 2);
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){1}, 1, (unsigned int[]){2}, 1);
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){1}, 1, (unsigned int[]){2, 3}, 2);
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){2, 3}, 2, (unsigned int[]){1}, 1);
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){2, 3}, 2, (unsigned int[]){}, 0);
+	damon_test_commit_probes_for(test,
+			(unsigned int[]){2}, 1, (unsigned int[]){}, 0);
+}
+
 static void damon_test_commit_ctx(struct kunit *test)
 {
 	struct damon_ctx *src, *dst;
@@ -1331,6 +1656,125 @@ static void damon_test_commit_ctx(struct kunit *test)
 	damon_destroy_ctx(dst);
 }
 
+static void damon_test_commit_ctx_keeps_quota_for(struct kunit *test,
+		unsigned long min_region_sz, int expected_err)
+{
+	struct damos_access_pattern pattern = {};
+	struct damos_quota quota = {.sz = SZ_64K};
+	struct damos_watermarks wmarks = {};
+	struct damon_ctx *src, *dst;
+	struct damon_target *target;
+	struct damos *s;
+
+	dst = damon_new_ctx();
+	if (!dst)
+		kunit_skip(test, "dst alloc fail");
+	target = damon_new_target();
+	if (!target) {
+		damon_destroy_ctx(dst);
+		kunit_skip(test, "target alloc fail");
+	}
+	damon_add_target(dst, target);
+	s = damon_new_scheme(&pattern, DAMOS_STAT, 0, &quota, &wmarks,
+			NUMA_NO_NODE);
+	if (!s) {
+		damon_destroy_ctx(dst);
+		kunit_skip(test, "scheme alloc fail");
+	}
+	damon_add_scheme(dst, s);
+
+	/* Copy the parameters before populating dst's runtime quota state. */
+	src = damon_new_test_ctx(dst);
+	if (!src) {
+		damon_destroy_ctx(dst);
+		kunit_skip(test, "src alloc fail");
+	}
+	src->min_region_sz = min_region_sz;
+	s->quota.esz = 123;
+	s->quota.esz_bp = 456;
+	s->quota.total_charged_sz = 789;
+	s->quota.total_charged_ns = 1011;
+	s->quota.charged_sz = 12;
+	s->quota.charged_from = 13;
+	s->quota.charge_target_from = target;
+	s->quota.charge_addr_from = 14;
+
+	KUNIT_EXPECT_EQ(test, damon_commit_ctx(dst, src), expected_err);
+	KUNIT_EXPECT_EQ(test, s->quota.esz, 123ul);
+	KUNIT_EXPECT_EQ(test, s->quota.esz_bp, 456ul);
+	KUNIT_EXPECT_EQ(test, s->quota.total_charged_sz, 789ul);
+	KUNIT_EXPECT_EQ(test, s->quota.total_charged_ns, 1011ul);
+	KUNIT_EXPECT_EQ(test, s->quota.charged_sz, 12ul);
+	KUNIT_EXPECT_EQ(test, s->quota.charged_from, 13ul);
+	KUNIT_EXPECT_PTR_EQ(test, s->quota.charge_target_from, target);
+	KUNIT_EXPECT_EQ(test, s->quota.charge_addr_from, 14ul);
+	damon_destroy_ctx(src);
+	damon_destroy_ctx(dst);
+}
+
+static void damon_test_commit_ctx_keeps_quota(struct kunit *test)
+{
+	/* Only power of two min_region_sz is allowed. */
+	damon_test_commit_ctx_keeps_quota_for(test, 4096, 0);
+	damon_test_commit_ctx_keeps_quota_for(test, 4095, -EINVAL);
+}
+
+static void damon_test_valid_probe_params(struct kunit *test)
+{
+	struct damon_ctx *ctx;
+	struct damon_probe *probe, *probe2;
+
+	ctx = damon_new_ctx();
+	if (!ctx)
+		kunit_skip(test, "ctx alloc fail");
+	probe = damon_new_probe();
+	if (!probe) {
+		damon_destroy_ctx(ctx);
+		kunit_skip(test, "probe alloc fail");
+	}
+	damon_add_probe(ctx, probe);
+
+	/* Parameters are validated only if any probe weight is set. */
+	ctx->attrs.sample_interval = 1;
+	ctx->attrs.aggr_interval = 1000000;
+	KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx));
+
+	/* Up to U8_MAX samples per aggregation interval are allowed. */
+	probe->weight = 100;
+	ctx->attrs.aggr_interval = 255;
+	KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx));
+
+	/* More samples could overflow the probe_hits counters. */
+	ctx->attrs.aggr_interval = 256;
+	KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx));
+
+	/* The largest weight whose weighted hit count fits in unsigned int. */
+	ctx->attrs.aggr_interval = 255;
+	probe->weight = UINT_MAX / 255;
+	KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx));
+
+	/* Any larger weight could overflow its weighted hit count. */
+	probe->weight = UINT_MAX / 255 + 1;
+	KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx));
+
+	/* With one sample per aggregation, even the largest weight fits. */
+	ctx->attrs.aggr_interval = 1;
+	probe->weight = UINT_MAX;
+	KUNIT_EXPECT_TRUE(test, damon_valid_probe_params(ctx));
+
+	/* The sum of all probes' weighted hit counts could also overflow. */
+	probe2 = damon_new_probe();
+	if (!probe2) {
+		damon_destroy_ctx(ctx);
+		kunit_skip(test, "probe2 alloc fail");
+	}
+	probe2->weight = 1;
+	damon_add_probe(ctx, probe2);
+	KUNIT_EXPECT_FALSE(test, damon_valid_probe_params(ctx));
+
+	damon_destroy_ctx(ctx);
+}
+
 static void damos_test_filter_out(struct kunit *test)
 {
 	struct damon_target *t;
@@ -1409,6 +1853,319 @@ static void damos_test_filter_out(struct kunit *test)
 	damos_free_filter(f);
 }
 
+static unsigned long damos_test_apply_scheme_stub(struct damon_ctx *c,
+		struct damon_target *t, struct damon_region *r,
+		struct damos *s, unsigned long *sz_filter_passed)
+{
+	return 0;
+}
+
+static void damos_test_apply_scheme_filtered_sz(struct kunit *test)
+{
+	struct damos_access_pattern pattern = {
+		.min_sz_region = 0,
+		.max_sz_region = ULONG_MAX,
+		.min_nr_accesses = 0,
+		.max_nr_accesses = UINT_MAX,
+		.min_age_region = 0,
+		.max_age_region = UINT_MAX,
+	};
+	unsigned long min_sz = DAMON_MIN_REGION_SZ;
+	struct damos_watermarks wmarks = {};
+	struct damos_quota quota = {};
+	struct damon_ctx *c;
+	struct damon_target *t;
+	struct damon_region *r;
+	struct damos_filter *f;
+	struct damos *s;
+
+	c = damon_new_ctx();
+	if (!c)
+		kunit_skip(test, "ctx alloc fail");
+	c->ops.apply_scheme = damos_test_apply_scheme_stub;
+
+	s = damon_new_scheme(&pattern, DAMOS_STAT, 0, &quota, &wmarks,
+			NUMA_NO_NODE);
+	if (!s) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "scheme alloc fail");
+	}
+	damon_add_scheme(c, s);
+
+	t = damon_new_target();
+	if (!t) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "target alloc fail");
+	}
+	damon_add_target(c, t);
+
+	f = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false);
+	if (!f) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "filter alloc fail");
+	}
+	f->addr_range = (struct damon_addr_range){
+		.start = 2 * min_sz,
+		.end = 8 * min_sz
+	};
+	damos_add_filter(s, f);
+	damos_set_filters_default_reject(s);
+
+	/* reject filter, region starting before the range */
+	r = damon_new_region(0, 4 * min_sz);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+	damon_add_region(r, t);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+	KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz);
+
+	if (damon_nr_regions(t) != 2)
+		goto out;
+	damon_destroy_region(damon_next_region(r), t);
+	damon_destroy_region(r, t);
+	s->stat = (struct damos_stat){};
+
+	/* allow filter, region starting inside the range */
+	f->allow = true;
+	damos_set_filters_default_reject(s);
+	r = damon_new_region(2 * min_sz, 10 * min_sz);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+	damon_add_region(r, t);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2);
+	KUNIT_EXPECT_EQ(test, r->ar.end, 8 * min_sz);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 6 * min_sz);
+
+out:
+	damon_destroy_ctx(c);
+}
+
+static void damos_test_apply_scheme_filter_sz_unchanged(struct kunit *test)
+{
+	struct damos_access_pattern pattern = {
+		.min_sz_region = 0,
+		.max_sz_region = ULONG_MAX,
+		.min_nr_accesses = 0,
+		.max_nr_accesses = UINT_MAX,
+		.min_age_region = 0,
+		.max_age_region = UINT_MAX,
+	};
+	unsigned long min_sz = DAMON_MIN_REGION_SZ;
+	struct damos_watermarks wmarks = {};
+	struct damos_quota quota = {};
+	struct damos_filter *f, *f2;
+	struct damon_ctx *c;
+	struct damon_target *t;
+	struct damon_region *r;
+	struct damos *s;
+
+	c = damon_new_ctx();
+	if (!c)
+		kunit_skip(test, "ctx alloc fail");
+	c->ops.apply_scheme = damos_test_apply_scheme_stub;
+
+	s = damon_new_scheme(&pattern, DAMOS_STAT, 0, &quota, &wmarks,
+			NUMA_NO_NODE);
+	if (!s) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "scheme alloc fail");
+	}
+	damon_add_scheme(c, s);
+
+	t = damon_new_target();
+	if (!t) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "target alloc fail");
+	}
+	damon_add_target(c, t);
+
+	f = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false);
+	if (!f) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "filter alloc fail");
+	}
+	f->addr_range = (struct damon_addr_range){
+		.start = 2 * min_sz, .end = 8 * min_sz};
+	damos_add_filter(s, f);
+	damos_set_filters_default_reject(s);
+
+	/* wholly inside a reject range: not counted at all */
+	r = damon_new_region(4 * min_sz, 6 * min_sz);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+	damon_add_region(r, t);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, s->stat.nr_tried, 0);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 0);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1);
+
+	/* wholly inside an allow range: counted whole, not split */
+	f->allow = true;
+	damos_set_filters_default_reject(s);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1);
+
+	/* two filters, each trimming: the size left after both is counted */
+	damon_destroy_region(r, t);
+	s->stat = (struct damos_stat){};
+	f->allow = false;
+	f->addr_range = (struct damon_addr_range){
+		.start = 4 * min_sz, .end = 12 * min_sz};
+	f2 = damos_new_filter(DAMOS_FILTER_TYPE_ADDR, true, false);
+	if (!f2) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "filter alloc fail");
+	}
+	f2->addr_range = (struct damon_addr_range){
+		.start = 2 * min_sz, .end = 3 * min_sz};
+	damos_add_filter(s, f2);
+	damos_set_filters_default_reject(s);
+
+	r = damon_new_region(0, 8 * min_sz);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+	damon_add_region(r, t);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3);
+	KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz);
+
+	/* a core filter that never splits: counted whole */
+	damos_destroy_filter(f2);
+	damos_destroy_filter(f);
+	f = damos_new_filter(DAMOS_FILTER_TYPE_TARGET, true, true);
+	if (!f) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "filter alloc fail");
+	}
+	f->target_idx = 0;
+	damos_add_filter(s, f);
+	damos_set_filters_default_reject(s);
+	s->stat = (struct damos_stat){};
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3);
+	KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz);
+
+	damon_destroy_ctx(c);
+}
+
+static void damos_test_apply_scheme_quota_sz(struct kunit *test)
+{
+	struct damos_access_pattern pattern = {
+		.min_sz_region = 0,
+		.max_sz_region = ULONG_MAX,
+		.min_nr_accesses = 0,
+		.max_nr_accesses = UINT_MAX,
+		.min_age_region = 0,
+		.max_age_region = UINT_MAX,
+	};
+	unsigned long min_sz = DAMON_MIN_REGION_SZ;
+	struct damos_watermarks wmarks = {};
+	struct damos_quota quota = {};
+	struct damon_region *r, *next;
+	struct damon_ctx *c;
+	struct damon_target *t;
+	struct damos *s;
+
+	c = damon_new_ctx();
+	if (!c)
+		kunit_skip(test, "ctx alloc fail");
+
+	s = damon_new_scheme(&pattern, DAMOS_STAT, 0, &quota, &wmarks,
+			NUMA_NO_NODE);
+	if (!s) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "scheme alloc fail");
+	}
+	damon_add_scheme(c, s);
+	damos_set_filters_default_reject(s);
+
+	t = damon_new_target();
+	if (!t) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "target alloc fail");
+	}
+	damon_add_target(c, t);
+
+	/* no apply_scheme operation: the whole region is counted */
+	r = damon_new_region(0, 4 * min_sz);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+	damon_add_region(r, t);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 4 * min_sz);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1);
+
+	/* no filter and no quota: the whole region is counted */
+	c->ops.apply_scheme = damos_test_apply_scheme_stub;
+	s->stat = (struct damos_stat){};
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 4 * min_sz);
+	KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1);
+
+	/* the quota trims the region: the trimmed size is counted */
+	damon_for_each_region_safe(r, next, t)
+		damon_destroy_region(r, t);
+	s->stat = (struct damos_stat){};
+	s->quota.esz = 2 * min_sz;
+	s->quota.charged_sz = 0;
+
+	r = damon_new_region(0, 8 * min_sz);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+	damon_add_region(r, t);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, r->ar.end, 2 * min_sz);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 2 * min_sz);
+
+	/* quota remainder below one region: tried, but nothing counted */
+	damon_for_each_region_safe(r, next, t)
+		damon_destroy_region(r, t);
+	s->stat = (struct damos_stat){};
+	s->quota.esz = 1;
+	s->quota.charged_sz = 0;
+
+	r = damon_new_region(0, 4 * min_sz);
+	if (!r) {
+		damon_destroy_ctx(c);
+		kunit_skip(test, "region alloc fail");
+	}
+	damon_add_region(r, t);
+
+	damos_apply_scheme(c, t, r, s);
+	KUNIT_EXPECT_EQ(test, s->stat.nr_tried, 1);
+	KUNIT_EXPECT_EQ(test, s->stat.sz_tried, 0);
+	KUNIT_EXPECT_EQ(test, r->ar.end, 4 * min_sz);
+
+	damon_destroy_ctx(c);
+}
+
 static void damon_test_feed_loop_next_input(struct kunit *test)
 {
 	unsigned long last_input = 900000, current_score = 200;
@@ -1632,6 +2389,53 @@ static void damon_test_rand(struct kunit *test)
 	}
 }
 
+static void damos_test_esz_goal_temporal(struct kunit *test)
+{
+	struct damos_access_pattern pattern = {};
+	struct damos_watermarks wmarks = {};
+	struct damos_quota quota = {
+		.goal_tuner = DAMOS_QUOTA_GOAL_TUNER_TEMPORAL,
+	};
+	struct damos_quota_goal *goal;
+	struct damon_ctx *ctx;
+	struct damos *s;
+
+	ctx = damon_new_ctx();
+	KUNIT_ASSERT_NOT_NULL(test, ctx);
+
+	s = damon_new_scheme(&pattern, DAMOS_STAT, 0, &quota, &wmarks,
+			NUMA_NO_NODE);
+	if (!s) {
+		damon_destroy_ctx(ctx);
+		kunit_skip(test, "scheme alloc fail");
+	}
+	damon_add_scheme(ctx, s);
+
+	goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, 10000);
+	if (!goal) {
+		damon_destroy_ctx(ctx);
+		kunit_skip(test, "quota goal alloc fail");
+	}
+	goal->current_value = 0;
+	damos_add_quota_goal(&s->quota, goal);
+
+	/* The largest size quota the basis-point conversion can hold. */
+	s->quota.sz = ULONG_MAX / 10000;
+	damos_set_effective_quota(ctx, s);
+	KUNIT_EXPECT_EQ(test, s->quota.esz, ULONG_MAX / 10000);
+
+	/* Any larger one saturates instead of wrapping. */
+	s->quota.sz = ULONG_MAX / 10000 + 1;
+	damos_set_effective_quota(ctx, s);
+	KUNIT_EXPECT_EQ(test, s->quota.esz, ULONG_MAX / 10000);
+
+	s->quota.sz = ULONG_MAX;
+	damos_set_effective_quota(ctx, s);
+	KUNIT_EXPECT_EQ(test, s->quota.esz, ULONG_MAX / 10000);
+
+	damon_destroy_ctx(ctx);
+}
+
 static struct kunit_case damon_test_cases[] = {
 	KUNIT_CASE(damon_test_target),
 	KUNIT_CASE(damon_test_regions),
@@ -1643,12 +2447,15 @@ static struct kunit_case damon_test_cases[] = {
 	KUNIT_CASE(damon_test_split_above_half_progresses),
 	KUNIT_CASE(damon_test_ops_registration),
 	KUNIT_CASE(damon_test_set_regions),
+	KUNIT_CASE(damon_test_nr_samples_per_aggr),
 	KUNIT_CASE(damon_test_update_monitoring_result),
 	KUNIT_CASE(damon_test_set_attrs),
 	KUNIT_CASE(damon_test_mvsum),
 	KUNIT_CASE(damon_test_nr_accesses_mvsum),
 	KUNIT_CASE(damos_test_new_filter),
+	KUNIT_CASE(damos_test_new_scheme_keeps_src_quota),
 	KUNIT_CASE(damos_test_commit_quota_goal),
+	KUNIT_CASE(damos_test_set_psi_current_val),
 	KUNIT_CASE(damos_test_commit_quota_goals),
 	KUNIT_CASE(damos_test_commit_quota),
 	KUNIT_CASE(damos_test_commit_dests),
@@ -1656,14 +2463,22 @@ static struct kunit_case damon_test_cases[] = {
 	KUNIT_CASE(damos_test_commit_pageout),
 	KUNIT_CASE(damos_test_commit_migrate_hot),
 	KUNIT_CASE(damon_test_commit_target_regions),
+	KUNIT_CASE(damon_test_commit_filter),
+	KUNIT_CASE(damon_test_commit_probes),
 	KUNIT_CASE(damon_test_commit_ctx),
+	KUNIT_CASE(damon_test_commit_ctx_keeps_quota),
+	KUNIT_CASE(damon_test_valid_probe_params),
 	KUNIT_CASE(damos_test_filter_out),
+	KUNIT_CASE(damos_test_apply_scheme_filtered_sz),
+	KUNIT_CASE(damos_test_apply_scheme_filter_sz_unchanged),
+	KUNIT_CASE(damos_test_apply_scheme_quota_sz),
 	KUNIT_CASE(damon_test_feed_loop_next_input),
 	KUNIT_CASE(damon_test_set_filters_default_reject),
 	KUNIT_CASE(damon_test_apply_min_nr_regions),
 	KUNIT_CASE(damon_test_is_last_region),
 	KUNIT_CASE(damon_test_walk_control_obsolete),
 	KUNIT_CASE(damon_test_rand),
+	KUNIT_CASE(damos_test_esz_goal_temporal),
 	{},
 };
 
diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index 0648400b2d65b4..d5dde97b3cd0d0 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -189,22 +189,12 @@ static int damon_va_three_regions(struct damon_target *t,
  *   <BIG UNMAPPED REGION 2>
  *   <stack>
  */
-static void __damon_va_init_regions(struct damon_ctx *ctx,
-				     struct damon_target *t)
+static void __damon_va_init_regions(struct damon_target *t)
 {
-	struct damon_target *ti;
 	struct damon_addr_range regions[3];
-	int tidx = 0;
 
-	if (damon_va_three_regions(t, regions)) {
-		damon_for_each_target(ti, ctx) {
-			if (ti == t)
-				break;
-			tidx++;
-		}
-		pr_debug("Failed to get three regions of %dth target\n", tidx);
+	if (damon_va_three_regions(t, regions))
 		return;
-	}
 
 	damon_set_regions(t, regions, 3, DAMON_MIN_REGION_SZ);
 }
@@ -217,7 +207,7 @@ static void damon_va_init(struct damon_ctx *ctx)
 	damon_for_each_target(t, ctx) {
 		/* the user may set the target regions as they want */
 		if (!damon_nr_regions(t))
-			__damon_va_init_regions(ctx, t);
+			__damon_va_init_regions(t);
 	}
 }
 
@@ -293,33 +283,6 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
 }
 
 #ifdef CONFIG_HUGETLB_PAGE
-static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm,
-				struct vm_area_struct *vma, unsigned long addr)
-{
-	bool referenced = false;
-	pte_t entry = huge_ptep_get(mm, addr, pte);
-	struct folio *folio = pfn_folio(pte_pfn(entry));
-	unsigned long psize = huge_page_size(hstate_vma(vma));
-
-	folio_get(folio);
-
-	if (pte_young(entry)) {
-		referenced = true;
-		entry = pte_mkold(entry);
-		set_huge_pte_at(mm, addr, pte, entry, psize);
-	}
-
-	if (mmu_notifier_clear_young(mm, addr,
-				     addr + huge_page_size(hstate_vma(vma))))
-		referenced = true;
-
-	if (referenced)
-		folio_set_young(folio);
-
-	folio_set_idle(folio);
-	folio_put(folio);
-}
-
 static int damon_mkold_hugetlb_entry(pte_t *pte, unsigned long hmask,
 				     unsigned long addr, unsigned long end,
 				     struct mm_walk *walk)
@@ -520,6 +483,243 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx)
 	return max_nr_accesses;
 }
 
+static void damon_va_prep_probe_region(struct damon_ctx *ctx,
+		struct mm_struct *mm, struct damon_region *r,
+		struct damon_probe *probe)
+{
+	struct damon_prep *p;
+
+	damon_for_each_prep(p, probe) {
+		switch (p->action) {
+		case DAMON_PREP_SET_PGIDLE:
+			damon_va_mkold(mm, r->sampling_addr);
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+static void damon_va_prep_probes(struct damon_ctx *ctx, bool set_samples)
+{
+	struct damon_target *t;
+	struct mm_struct *mm;
+	struct damon_region *r;
+	struct damon_probe *p;
+
+	damon_for_each_target(t, ctx) {
+		mm = damon_get_mm(t);
+		if (!mm)
+			continue;
+		damon_for_each_region(r, t) {
+			if (set_samples)
+				r->sampling_addr = damon_rand(ctx, r->ar.start,
+						r->ar.end);
+			damon_for_each_probe(p, ctx)
+				damon_va_prep_probe_region(ctx, mm, r, p);
+		}
+		mmput(mm);
+	}
+}
+
+static bool damon_va_young_addr(struct folio *folio, pte_t *pte, pmd_t *pmd,
+		struct mm_struct *mm, unsigned long addr)
+{
+	bool young = false;
+
+	if (pte)
+		young = pte_young(*pte);
+	else if (pmd)
+		young = pmd_young(*pmd);
+	young = young || !folio_test_idle(folio) ||
+		mmu_notifier_test_young(mm, addr);
+	return young;
+}
+
+static bool damon_va_filter_match(struct damon_filter *filter,
+		struct folio *folio, pte_t *pte, pmd_t *pmd,
+		struct mm_struct *mm, unsigned long addr)
+{
+	bool matched = false;
+
+	switch (filter->type) {
+	case DAMON_FILTER_TYPE_PGIDLE_UNSET:
+		if (!folio)
+			matched = false;
+		else
+			matched = damon_va_young_addr(folio, pte, pmd, mm,
+					addr);
+		break;
+	case DAMON_FILTER_TYPE_PGIDLE_SET:
+		if (!folio)
+			matched = false;
+		else
+			matched = !damon_va_young_addr(folio, pte, pmd, mm,
+					addr);
+		break;
+	default:
+		return damon_ops_filter_match(filter, folio);
+	}
+	return matched == filter->matching;
+}
+
+static bool damon_va_filter_pass(struct folio *folio, struct damon_probe *p,
+		pte_t *pte, pmd_t *pmd, struct mm_struct *mm,
+		unsigned long addr)
+{
+	struct damon_filter *f;
+	bool pass = true;
+
+	damon_for_each_filter(f, p) {
+		if (damon_va_filter_match(f, folio, pte, pmd, mm, addr)) {
+			pass = f->allow;
+			break;
+		}
+		pass = !f->allow;
+	}
+	return pass;
+}
+
+struct damon_va_probe_walk_private {
+	struct damon_ctx *ctx;
+	struct damon_region *r;
+};
+
+static void damon_va_probe_folio(struct damon_ctx *ctx,
+		struct damon_region *r, struct folio *folio,
+		pte_t *pte, pmd_t *pmd, struct mm_struct *mm)
+{
+	struct damon_probe *probe;
+	int i = 0;
+
+	damon_for_each_probe(probe, ctx) {
+		if (damon_va_filter_pass(folio, probe, pte, pmd, mm,
+					r->sampling_addr))
+			r->probe_hits[i]++;
+		i++;
+	}
+}
+
+static int damon_va_probe_pmd_entry(pmd_t *pmd, unsigned long addr,
+		unsigned long next, struct mm_walk *walk)
+{
+	pte_t *pte;
+	pte_t ptent;
+	spinlock_t *ptl;
+	struct folio *folio;
+	struct damon_va_probe_walk_private *priv = walk->private;
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	ptl = pmd_trans_huge_lock(pmd, walk->vma);
+	if (ptl) {
+		pmd_t pmde = pmdp_get(pmd);
+
+		if (!pmd_present(pmde))
+			goto huge_out;
+		folio = vm_normal_folio_pmd(walk->vma, addr, pmde);
+		if (!folio)
+			goto huge_out;
+		damon_va_probe_folio(priv->ctx, priv->r, folio, NULL, &pmde,
+				walk->vma->vm_mm);
+
+huge_out:
+		spin_unlock(ptl);
+		return 0;
+	}
+#endif	/* CONFIG_TRANSPARENT_HUGEPAGE */
+
+	pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
+	if (!pte)
+		return 0;
+	ptent = ptep_get(pte);
+	if (!pte_present(ptent))
+		goto out;
+	folio = vm_normal_folio(walk->vma, addr, ptent);
+	if (!folio)
+		goto out;
+	damon_va_probe_folio(priv->ctx, priv->r, folio, &ptent, NULL,
+			walk->vma->vm_mm);
+
+out:
+	pte_unmap_unlock(pte, ptl);
+	return 0;
+}
+
+#ifdef CONFIG_HUGETLB_PAGE
+static int damon_va_probe_hugetlb_entry(pte_t *pte, unsigned long hmask,
+		unsigned long addr, unsigned long end, struct mm_walk *walk)
+{
+	struct damon_va_probe_walk_private *priv = walk->private;
+	struct hstate *h = hstate_vma(walk->vma);
+	struct folio *folio;
+	spinlock_t *ptl;
+	pte_t entry;
+
+	ptl = huge_pte_lock(h, walk->mm, pte);
+	entry = huge_ptep_get(walk->mm, addr, pte);
+	if (!pte_present(entry))
+		goto out;
+
+	folio = pfn_folio(pte_pfn(entry));
+	folio_get(folio);
+	damon_va_probe_folio(priv->ctx, priv->r, folio, &entry, NULL,
+			walk->vma->vm_mm);
+	folio_put(folio);
+
+out:
+	spin_unlock(ptl);
+	return 0;
+}
+#else
+#define damon_va_probe_hugetlb_entry NULL
+#endif /* CONFIG_HUGETLB_PAGE */
+
+static void __damon_va_apply_probes(struct damon_ctx *ctx,
+		struct mm_struct *mm,  struct damon_region *r)
+{
+	struct damon_va_probe_walk_private arg = {
+		.ctx = ctx,
+		.r = r,
+	};
+	struct mm_walk_ops damon_probe_walk_ops = {
+		.pmd_entry = damon_va_probe_pmd_entry,
+		.hugetlb_entry = damon_va_probe_hugetlb_entry,
+	};
+	unsigned long addr = r->sampling_addr;
+
+	if (!mm)
+		return;
+
+	damon_va_walk_page_range(mm, addr, addr + 1, &damon_probe_walk_ops,
+			&arg);
+}
+
+static unsigned int damon_va_apply_probes(struct damon_ctx *ctx,
+		bool set_samples, bool return_max_wsum)
+{
+	struct damon_target *t;
+	struct mm_struct *mm;
+	struct damon_region *r;
+	unsigned int max_wsum = 0;
+
+	damon_for_each_target(t, ctx) {
+		mm = damon_get_mm(t);
+		damon_for_each_region(r, t) {
+			if (set_samples)
+				r->sampling_addr = damon_rand(ctx, r->ar.start,
+						r->ar.end);
+			__damon_va_apply_probes(ctx, mm, r);
+			if (return_max_wsum)
+				max_wsum = max(damon_probe_hits_wsum(r, false,
+							false, ctx), max_wsum);
+		}
+		if (mm)
+			mmput(mm);
+	}
+
+	return max_wsum;
+}
+
 static bool damos_va_filter_young_match(struct damos_filter *filter,
 		struct folio *folio, struct vm_area_struct *vma,
 		unsigned long addr, pte_t *ptep, pmd_t *pmdp)
@@ -669,6 +869,8 @@ static int damos_va_migrate_pmd_entry(pmd_t *pmd, unsigned long addr,
 		return 0;
 
 	for (; addr < next; pte += nr, addr += nr * PAGE_SIZE) {
+		unsigned long page_idx;
+
 		nr = 1;
 		ptent = ptep_get(pte);
 
@@ -681,7 +883,8 @@ static int damos_va_migrate_pmd_entry(pmd_t *pmd, unsigned long addr,
 			continue;
 		damos_va_migrate_dests_add(folio, walk->vma, addr, dests,
 				migration_lists);
-		nr = folio_nr_pages(folio);
+		page_idx = folio_page_idx(folio, pte_page(ptent));
+		nr = folio_nr_pages(folio) - page_idx;
 	}
 	pte_unmap_unlock(start_pte, ptl);
 	return 0;
@@ -831,6 +1034,8 @@ static int damos_va_stat_pmd_entry(pmd_t *pmd, unsigned long addr,
 		return 0;
 
 	for (; addr < next; pte += nr, addr += nr * PAGE_SIZE) {
+		unsigned long page_idx;
+
 		nr = 1;
 		ptent = ptep_get(pte);
 
@@ -844,7 +1049,8 @@ static int damos_va_stat_pmd_entry(pmd_t *pmd, unsigned long addr,
 
 		if (!damos_va_filter_out(s, folio, vma, addr, pte, NULL))
 			*sz_filter_passed += folio_size(folio);
-		nr = folio_nr_pages(folio);
+		page_idx = folio_page_idx(folio, pte_page(ptent));
+		nr = folio_nr_pages(folio) - page_idx;
 		s->last_applied = folio;
 	}
 	pte_unmap_unlock(start_pte, ptl);
@@ -942,6 +1148,8 @@ static int __init damon_va_initcall(void)
 		.update = damon_va_update,
 		.prepare_access_checks = damon_va_prepare_access_checks,
 		.check_accesses = damon_va_check_accesses,
+		.prep_probes = damon_va_prep_probes,
+		.apply_probes = damon_va_apply_probes,
 		.target_valid = damon_va_target_valid,
 		.cleanup_target = damon_va_cleanup_target,
 		.apply_scheme = damon_va_apply_scheme,
diff --git a/mm/debug.c b/mm/debug.c
index 9a0297b3988d89..655e6bcc0e8d91 100644
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -157,17 +157,13 @@ void dump_vma(const struct vm_area_struct *vma)
 	pr_emerg("vma %px start %px end %px mm %px\n"
 		"prot %lx anon_vma %px vm_ops %px\n"
 		"pgoff %lx file %px private_data %px\n"
-#ifdef CONFIG_PER_VMA_LOCK
 		"refcnt %x\n"
-#endif
 		"flags: %#lx(%pGv)\n",
 		vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm,
 		(unsigned long)pgprot_val(vma->vm_page_prot),
 		vma->anon_vma, vma->vm_ops, vma_start_pgoff(vma),
 		vma->vm_file, vma->vm_private_data,
-#ifdef CONFIG_PER_VMA_LOCK
 		refcount_read(&vma->vm_refcnt),
-#endif
 		vma->vm_flags, &vma->vm_flags);
 }
 EXPORT_SYMBOL(dump_vma);
diff --git a/mm/execmem.c b/mm/execmem.c
index 74a178a87e7581..349cadd874863a 100644
--- a/mm/execmem.c
+++ b/mm/execmem.c
@@ -51,7 +51,8 @@ static void *execmem_vmalloc(struct execmem_range *range, size_t size,
 	}
 
 	if (!p) {
-		pr_warn_ratelimited("unable to allocate memory\n");
+		if (!(vm_flags & VM_REQUIRE_HUGE_VMAP))
+			pr_warn_ratelimited("unable to allocate memory\n");
 		return NULL;
 	}
 
@@ -113,28 +114,6 @@ static inline unsigned long mas_range_len(struct ma_state *mas)
 	return mas->last - mas->index + 1;
 }
 
-static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid)
-{
-	unsigned int nr = (1 << get_vm_area_page_order(vm));
-	unsigned int updated = 0;
-	int err = 0;
-
-	for (int i = 0; i < vm->nr_pages; i += nr) {
-		err = set_direct_map_valid_noflush(vm->pages[i], nr, valid);
-		if (err)
-			goto err_restore;
-		updated += nr;
-	}
-
-	return 0;
-
-err_restore:
-	for (int i = 0; i < updated; i += nr)
-		set_direct_map_valid_noflush(vm->pages[i], nr, !valid);
-
-	return err;
-}
-
 static int execmem_force_rw(void *ptr, size_t size)
 {
 	unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT;
@@ -159,24 +138,30 @@ int execmem_restore_rox(void *ptr, size_t size)
 static void execmem_cache_clean(struct work_struct *work)
 {
 	struct maple_tree *free_areas = &execmem_cache.free_areas;
-	struct mutex *mutex = &execmem_cache.mutex;
 	MA_STATE(mas, free_areas, 0, ULONG_MAX);
 	void *area;
 
-	mutex_lock(mutex);
+	guard(mutex)(&execmem_cache.mutex);
 	mas_for_each(&mas, area, ULONG_MAX) {
+		struct vm_struct *vm = find_vm_area(area);
 		size_t size = mas_range_len(&mas);
 
-		if (IS_ALIGNED(size, PMD_SIZE) &&
-		    IS_ALIGNED(mas.index, PMD_SIZE)) {
-			struct vm_struct *vm = find_vm_area(area);
+		if (vm && get_vm_area_size(vm) == size) {
+			VM_WARN_ON_ONCE(!IS_ALIGNED(mas.index, PMD_SIZE) ||
+					!IS_ALIGNED(size, PMD_SIZE));
 
-			execmem_set_direct_map_valid(vm, true);
-			mas_store_gfp(&mas, NULL, GFP_KERNEL);
+			/*
+			 * Preallocate to ensure mas_store does not fail
+			 * If there is no memory for the tree update, bail out,
+			 * next execmem_free() might be more lucky
+			 */
+			if (mas_preallocate(&mas, NULL, GFP_KERNEL))
+				break;
+
+			mas_store_prealloc(&mas, NULL);
 			vfree(area);
 		}
 	}
-	mutex_unlock(mutex);
 }
 
 static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean);
@@ -242,30 +227,34 @@ static void *execmem_cache_alloc_locked(struct execmem_range *range, size_t size
 	addr = mas_free.index;
 	last = mas_free.last;
 
+	mas_set_range(&mas_free, addr, addr + size - 1);
+	if (mas_preallocate(&mas_free, NULL, GFP_KERNEL))
+		return NULL;
+
 	/* insert allocated size to busy_areas at range [addr, addr + size) */
 	mas_set_range(&mas_busy, addr, addr + size - 1);
 	err = mas_store_gfp(&mas_busy, (void *)addr, GFP_KERNEL);
 	if (err)
-		return NULL;
+		goto err_destroy_mas_free;
 
-	mas_store_gfp(&mas_free, NULL, GFP_KERNEL);
+	mas_store_prealloc(&mas_free, NULL);
 	if (area_size > size) {
-		void *ptr = (void *)(addr + size);
-
 		/*
 		 * re-insert remaining free size to free_areas at range
 		 * [addr + size, last]
+		 * the range matches an existing entry, so this cannot allocate
 		 */
+		ptr = (void *)(addr + size);
 		mas_set_range(&mas_free, addr + size, last);
-		err = mas_store_gfp(&mas_free, ptr, GFP_KERNEL);
-		if (err) {
-			mas_store_gfp(&mas_busy, NULL, GFP_KERNEL);
-			return NULL;
-		}
+		mas_store_gfp(&mas_free, ptr, GFP_KERNEL);
 	}
 	ptr = (void *)addr;
 
 	return ptr;
+
+err_destroy_mas_free:
+	mas_destroy(&mas_free);
+	return NULL;
 }
 
 static void *__execmem_cache_alloc(struct execmem_range *range, size_t size)
@@ -275,58 +264,55 @@ static void *__execmem_cache_alloc(struct execmem_range *range, size_t size)
 	return execmem_cache_alloc_locked(range, size);
 }
 
-static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size)
+static void *execmem_vmalloc_rox(struct execmem_range *range, size_t size,
+				 unsigned long vm_flags)
 {
-	unsigned long vm_flags = VM_ALLOW_HUGE_VMAP;
-	struct mutex *mutex = &execmem_cache.mutex;
-	struct vm_struct *vm;
-	size_t alloc_size;
-	int err = -ENOMEM;
-	void *p;
-
-	alloc_size = round_up(size, PMD_SIZE);
-	p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
-	if (!p) {
-		alloc_size = size;
-		p = execmem_vmalloc(range, alloc_size, PAGE_KERNEL, vm_flags);
-	}
+	void *p __free(vfree) = execmem_vmalloc(range, size, PAGE_KERNEL, vm_flags);
+	int err;
 
 	if (!p)
 		return NULL;
 
-	vm = find_vm_area(p);
-	if (!vm)
-		goto err_free_mem;
-
 	/* fill memory with instructions that will trap */
-	execmem_fill_trapping_insns(p, alloc_size);
-
-	err = set_memory_rox((unsigned long)p, vm->nr_pages);
+	execmem_fill_trapping_insns(p, size);
+	set_vm_flush_reset_perms(p);
+	err = set_memory_rox((unsigned long)p, size >> PAGE_SHIFT);
 	if (err)
-		goto err_free_mem;
+		return NULL;
+
+	return no_free_ptr(p);
+}
+
+static void *execmem_cache_populate_alloc(struct execmem_range *range, size_t size)
+{
+	unsigned long vm_flags = VM_REQUIRE_HUGE_VMAP;
+	size_t alloc_size = round_up(size, PMD_SIZE);
+	void *p __free(vfree) = NULL;
+	int err;
+
+	p = execmem_vmalloc_rox(range, alloc_size, vm_flags);
+	if (!p)
+		return NULL;
 
 	/*
 	 * New memory blocks must be allocated and added to the cache
 	 * as an atomic operation, otherwise they may be consumed
 	 * by a parallel call to the execmem_cache_alloc function.
 	 */
-	mutex_lock(mutex);
+	guard(mutex)(&execmem_cache.mutex);
 	err = execmem_cache_add_locked(p, alloc_size, GFP_KERNEL);
 	if (err)
-		goto err_reset_direct_map;
-
-	p = execmem_cache_alloc_locked(range, size);
+		return NULL;
 
-	mutex_unlock(mutex);
+	/* the chunk belongs to the cache now */
+	retain_and_null_ptr(p);
 
-	return p;
+	return execmem_cache_alloc_locked(range, size);
+}
 
-err_reset_direct_map:
-	mutex_unlock(mutex);
-	execmem_set_direct_map_valid(vm, true);
-err_free_mem:
-	vfree(p);
-	return NULL;
+static void *execmem_alloc_rox(struct execmem_range *range, size_t size)
+{
+	return execmem_vmalloc_rox(range, size, 0);
 }
 
 static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
@@ -456,6 +442,11 @@ static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
 	return NULL;
 }
 
+static void *execmem_alloc_rox(struct execmem_range *range, size_t size)
+{
+	return NULL;
+}
+
 static bool execmem_cache_free(void *ptr)
 {
 	return false;
@@ -465,17 +456,20 @@ static bool execmem_cache_free(void *ptr)
 void *execmem_alloc(enum execmem_type type, size_t size)
 {
 	struct execmem_range *range = &execmem_info->ranges[type];
-	bool use_cache = range->flags & EXECMEM_ROX_CACHE;
+	bool use_rox_cache = range->flags & EXECMEM_ROX_CACHE;
 	unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
 	pgprot_t pgprot = range->pgprot;
 	void *p = NULL;
 
 	size = PAGE_ALIGN(size);
 
-	if (use_cache)
+	if (use_rox_cache) {
 		p = execmem_cache_alloc(range, size);
-	else
+		if (!p)
+			p = execmem_alloc_rox(range, size);
+	} else {
 		p = execmem_vmalloc(range, size, pgprot, vm_flags);
+	}
 
 	return kasan_reset_tag(p);
 }
diff --git a/mm/filemap.c b/mm/filemap.c
index 00fd89cf6f5509..b74bc1e5015c6f 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -918,14 +918,6 @@ noinline int __filemap_add_folio(struct address_space *mapping,
 
 		mapping->nrpages += nr;
 
-		/* hugetlb pages do not participate in page cache accounting */
-		if (!huge) {
-			lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
-			if (folio_test_pmd_mappable(folio))
-				lruvec_stat_mod_folio(folio,
-						NR_FILE_THPS, nr);
-		}
-
 unlock:
 		xas_unlock_irq(&xas);
 
@@ -942,6 +934,13 @@ noinline int __filemap_add_folio(struct address_space *mapping,
 	if (xas_error(&xas))
 		goto error;
 
+	/* hugetlb pages do not participate in page cache accounting */
+	if (!huge) {
+		lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
+		if (folio_test_pmd_mappable(folio))
+			lruvec_stat_mod_folio(folio, NR_FILE_THPS, nr);
+	}
+
 	trace_mm_filemap_add_to_page_cache(folio);
 	return 0;
 error:
@@ -1686,6 +1685,8 @@ EXPORT_SYMBOL_GPL(folio_end_writeback_no_dropbehind);
  */
 void folio_end_writeback(struct folio *folio)
 {
+	bool swap_dropbehind;
+
 	VM_BUG_ON_FOLIO(!folio_test_writeback(folio), folio);
 
 	/*
@@ -1695,7 +1696,24 @@ void folio_end_writeback(struct folio *folio)
 	 * reused before the folio_wake_bit().
 	 */
 	folio_get(folio);
+
+	/*
+	 * Sample this before folio_end_writeback_no_dropbehind() clears
+	 * PG_writeback: until then a racing swapin cannot remove the folio from
+	 * the swap cache. Afterwards it can, and the drop below then finds a
+	 * non-swapcache folio and puts it back on the LRU instead. The
+	 * reference taken above keeps the folio alive across that window.
+	 */
+	swap_dropbehind = folio_test_swapcache(folio) &&
+			  folio_test_dropbehind(folio);
+
 	folio_end_writeback_no_dropbehind(folio);
+
+	if (swap_dropbehind) {
+		swap_writeback_dropbehind_folio(folio);
+		return;
+	}
+
 	folio_end_dropbehind(folio);
 	folio_put(folio);
 }
diff --git a/mm/folio.c b/mm/folio.c
index 50a6dbe55998e7..35e242b48870b0 100644
--- a/mm/folio.c
+++ b/mm/folio.c
@@ -354,26 +354,28 @@ static void __lru_cache_activate_folio(struct folio *folio)
 
 static void lru_gen_inc_refs(struct folio *folio)
 {
-	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
+	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+	int refs;
 
 	if (folio_test_unevictable(folio))
 		return;
 
 	/* see the comment on LRU_REFS_FLAGS */
-	if (!folio_test_referenced(folio)) {
-		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
+	if (!folio_lru_refs(folio)) {
+		folio_set_lru_refs(folio, 1);
 		return;
 	}
 
 	do {
-		if ((old_flags & LRU_REFS_MASK) == LRU_REFS_MASK) {
+		new_flags = old_flags;
+		refs = lru_get_refs_flags(old_flags);
+		if (refs == LRU_REFS_MAX) {
 			if (!folio_test_workingset(folio))
 				folio_set_workingset(folio);
 			return;
 		}
-
-		new_flags = old_flags + BIT(LRU_REFS_PGOFF);
-	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+		lru_set_refs_flags(&new_flags, refs + 1);
+	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
 }
 
 static bool lru_gen_clear_refs(struct folio *folio)
@@ -385,7 +387,8 @@ static bool lru_gen_clear_refs(struct folio *folio)
 	if (gen < 0)
 		return true;
 
-	set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS | BIT(PG_workingset), 0);
+	folio_set_lru_refs(folio, 0);
+	folio_clear_workingset(folio);
 
 	rcu_read_lock();
 	seq = READ_ONCE(folio_lruvec(folio)->lrugen.min_seq[type]);
@@ -502,7 +505,7 @@ void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma)
 {
 	VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
 
-	if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED))
+	if (vma_test(vma, VMA_LOCKED_BIT))
 		mlock_new_folio(folio);
 	else
 		folio_add_lru(folio);
diff --git a/mm/gup.c b/mm/gup.c
index eb898ea1ee22e5..0f9326be920da0 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -621,7 +621,7 @@ static struct page *no_page_table(struct vm_area_struct *vma,
 	 * But we can only make this optimization where a hole would surely
 	 * be zero-filled if handle_mm_fault() actually did handle it.
 	 */
-	if (is_vm_hugetlb_page(vma)) {
+	if (vma_is_hugetlb(vma)) {
 		struct hstate *h = hstate_vma(vma);
 
 		if (!hugetlbfs_pagecache_present(h, vma, address))
@@ -1204,7 +1204,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
 	int foreign = (gup_flags & FOLL_REMOTE);
 	bool vma_anon = vma_is_anonymous(vma);
 
-	if (vm_flags & (VM_IO | VM_PFNMAP))
+	if (!vma_can_gup(vma))
 		return -EFAULT;
 
 	if ((gup_flags & FOLL_ANON) && !vma_anon)
@@ -1213,7 +1213,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
 	if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma))
 		return -EOPNOTSUPP;
 
-	if ((gup_flags & FOLL_SPLIT_PMD) && is_vm_hugetlb_page(vma))
+	if ((gup_flags & FOLL_SPLIT_PMD) && vma_is_hugetlb(vma))
 		return -EOPNOTSUPP;
 
 	if (vma_is_secretmem(vma))
@@ -1836,6 +1836,10 @@ long populate_vma_page_range(struct vm_area_struct *vma,
 	if (!vma_is_accessible(vma))
 		return -EFAULT;
 
+	/* Unreadable VMAs also cannot be faulted in. */
+	if (!vma_test(vma, VMA_MAYREAD_BIT))
+		return -EFAULT;
+
 	gup_flags = FOLL_TOUCH;
 	/*
 	 * We want to touch writable mappings with a write fault in order
@@ -1951,7 +1955,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
 		 * range with the first VMA. Also, skip undesirable VMA types.
 		 */
 		nend = min(end, vma->vm_end);
-		if (vma->vm_flags & (VM_IO | VM_PFNMAP))
+		if (!vma_can_gup(vma))
 			continue;
 		if (nstart < vma->vm_start)
 			nstart = vma->vm_start;
@@ -1983,6 +1987,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
 	struct vm_area_struct *vma;
 	bool must_unlock = false;
 	vm_flags_t vm_flags;
+	int ret, err = -EFAULT;
 	long i;
 
 	if (!nr_pages)
@@ -1999,6 +2004,10 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
 		*locked = 1;
 	}
 
+	/* See the MMU variant: support the traditional behavior. */
+	if (pages && !(foll_flags & FOLL_PIN))
+		foll_flags |= FOLL_GET;
+
 	/* calculate required read or write permissions.
 	 * If FOLL_FORCE is set, we only require the "MAY" flags.
 	 */
@@ -2013,14 +2022,19 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
 			break;
 
 		/* protect what we can, including chardevs */
-		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
-		    !(vm_flags & vma->vm_flags))
+		if (!vma_can_gup(vma) || !(vm_flags & vma->vm_flags))
 			break;
 
 		if (pages) {
 			pages[i] = virt_to_page((void *)start);
-			if (pages[i])
-				get_page(pages[i]);
+			if (!pages[i])
+				break;
+			ret = try_grab_folio(page_folio(pages[i]), 1, foll_flags);
+			if (unlikely(ret)) {
+				pages[i] = NULL;
+				err = ret;
+				break;
+			}
 		}
 
 		start = (start + PAGE_SIZE) & PAGE_MASK;
@@ -2031,7 +2045,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
 		*locked = 0;
 	}
 
-	return i ? : -EFAULT;
+	return i ? : err;
 }
 #endif /* !CONFIG_MMU */
 
@@ -2700,8 +2714,9 @@ EXPORT_SYMBOL(get_user_pages_unlocked);
  * Before activating this code, please be aware that the following assumptions
  * are currently made:
  *
- *  *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
- *  free pages containing page tables or TLB flushing requires IPI broadcast.
+ *  *) tlb_remove_table() is used to free pages containing page tables, with
+ *  the free deferred until an RCU grace period has elapsed (see
+ *  mm/mmu_gather.c).
  *
  *  *) ptes can be read atomically by the architecture.
  *
@@ -2826,11 +2841,11 @@ static bool gup_fast_folio_allowed(struct folio *folio, unsigned int flags)
  * also check pmd here to make sure pmd doesn't change (corresponds to
  * pmdp_collapse_flush() in the THP collapse code path).
  */
-static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
-		unsigned long end, unsigned int flags, struct page **pages,
-		int *nr)
+static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
+		unsigned long addr, unsigned long end,
+		unsigned int flags, struct page **pages)
 {
-	int ret = 0;
+	unsigned long nr_pages = 0;
 	pte_t *ptep, *ptem;
 
 	ptem = ptep = pte_offset_map(&pmd, addr);
@@ -2892,15 +2907,13 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
 			goto pte_unmap;
 		}
 		folio_set_referenced(folio);
-		pages[*nr] = page;
-		(*nr)++;
+		pages[nr_pages] = page;
+		nr_pages++;
 	} while (ptep++, addr += PAGE_SIZE, addr != end);
 
-	ret = 1;
-
 pte_unmap:
 	pte_unmap(ptem);
-	return ret;
+	return nr_pages;
 }
 #else
 
@@ -2913,21 +2926,25 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
  * get_user_pages_fast_only implementation that can pin pages. Thus it's still
  * useful to have gup_fast_pmd_leaf even if we can't operate on ptes.
  */
-static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
-		unsigned long end, unsigned int flags, struct page **pages,
-		int *nr)
+static unsigned long gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp,
+		unsigned long addr, unsigned long end,
+		unsigned int flags, struct page **pages)
 {
 	return 0;
 }
 #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
 
-static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
-		unsigned long end, unsigned int flags, struct page **pages,
-		int *nr)
+static unsigned long gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp,
+		unsigned long addr, unsigned long end,
+		unsigned int flags, struct page **pages)
 {
 	struct page *page;
 	struct folio *folio;
-	int refs;
+	unsigned long nr_pages, i;
+
+	/* See gup_fast_pte_range() */
+	if (pmd_protnone(orig))
+		return 0;
 
 	if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
 		return 0;
@@ -2935,42 +2952,40 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
 	if (pmd_special(orig))
 		return 0;
 
-	refs = (end - addr) >> PAGE_SHIFT;
+	nr_pages = (end - addr) >> PAGE_SHIFT;
 	page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
 
-	folio = try_grab_folio_fast(page, refs, flags);
+	folio = try_grab_folio_fast(page, nr_pages, flags);
 	if (!folio)
 		return 0;
 
 	if (unlikely(pmd_val(orig) != pmd_val(pmdp_get_lockless(pmdp)))) {
-		gup_put_folio(folio, refs, flags);
+		gup_put_folio(folio, nr_pages, flags);
 		return 0;
 	}
 
 	if (!gup_fast_folio_allowed(folio, flags)) {
-		gup_put_folio(folio, refs, flags);
+		gup_put_folio(folio, nr_pages, flags);
 		return 0;
 	}
 	if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {
-		gup_put_folio(folio, refs, flags);
+		gup_put_folio(folio, nr_pages, flags);
 		return 0;
 	}
 
-	pages += *nr;
-	*nr += refs;
-	for (; refs; refs--)
+	for (i = 0; i < nr_pages; i++)
 		*(pages++) = page++;
 	folio_set_referenced(folio);
-	return 1;
+	return nr_pages;
 }
 
-static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
-		unsigned long end, unsigned int flags, struct page **pages,
-		int *nr)
+static unsigned long gup_fast_pud_leaf(pud_t orig, pud_t *pudp,
+		unsigned long addr, unsigned long end,
+		unsigned int flags, struct page **pages)
 {
 	struct page *page;
 	struct folio *folio;
-	int refs;
+	unsigned long nr_pages, i;
 
 	if (!pud_access_permitted(orig, flags & FOLL_WRITE))
 		return 0;
@@ -2978,41 +2993,39 @@ static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
 	if (pud_special(orig))
 		return 0;
 
-	refs = (end - addr) >> PAGE_SHIFT;
+	nr_pages = (end - addr) >> PAGE_SHIFT;
 	page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
 
-	folio = try_grab_folio_fast(page, refs, flags);
+	folio = try_grab_folio_fast(page, nr_pages, flags);
 	if (!folio)
 		return 0;
 
 	if (unlikely(pud_val(orig) != pud_val(pudp_get(pudp)))) {
-		gup_put_folio(folio, refs, flags);
+		gup_put_folio(folio, nr_pages, flags);
 		return 0;
 	}
 
 	if (!gup_fast_folio_allowed(folio, flags)) {
-		gup_put_folio(folio, refs, flags);
+		gup_put_folio(folio, nr_pages, flags);
 		return 0;
 	}
 
 	if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) {
-		gup_put_folio(folio, refs, flags);
+		gup_put_folio(folio, nr_pages, flags);
 		return 0;
 	}
 
-	pages += *nr;
-	*nr += refs;
-	for (; refs; refs--)
+	for (i = 0; i < nr_pages; i++)
 		*(pages++) = page++;
 	folio_set_referenced(folio);
-	return 1;
+	return nr_pages;
 }
 
-static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
-		unsigned long end, unsigned int flags, struct page **pages,
-		int *nr)
+static unsigned long gup_fast_pmd_range(pud_t *pudp, pud_t pud,
+		unsigned long addr, unsigned long end,
+		unsigned int flags, struct page **pages)
 {
-	unsigned long next;
+	unsigned long next, nr_pages = 0, chunk_nr_pages;
 	pmd_t *pmdp;
 
 	pmdp = pmd_offset_lockless(pudp, pud, addr);
@@ -3021,30 +3034,30 @@ static int gup_fast_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
 
 		next = pmd_addr_end(addr, end);
 		if (!pmd_present(pmd))
-			return 0;
+			break;
 
 		if (unlikely(pmd_leaf(pmd))) {
-			/* See gup_fast_pte_range() */
-			if (pmd_protnone(pmd))
-				return 0;
-
-			if (!gup_fast_pmd_leaf(pmd, pmdp, addr, next, flags,
-				pages, nr))
-				return 0;
-
-		} else if (!gup_fast_pte_range(pmd, pmdp, addr, next, flags,
-					       pages, nr))
-			return 0;
+			chunk_nr_pages = gup_fast_pmd_leaf(pmd, pmdp, addr,
+							   next, flags,
+							   &pages[nr_pages]);
+
+		} else
+			chunk_nr_pages = gup_fast_pte_range(pmd, pmdp, addr,
+							    next, flags,
+							    &pages[nr_pages]);
+		nr_pages += chunk_nr_pages;
+		if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
+			break;
 	} while (pmdp++, addr = next, addr != end);
 
-	return 1;
+	return nr_pages;
 }
 
-static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr,
-		unsigned long end, unsigned int flags, struct page **pages,
-		int *nr)
+static unsigned long gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d,
+		unsigned long addr, unsigned long end,
+		unsigned int flags, struct page **pages)
 {
-	unsigned long next;
+	unsigned long next, nr_pages = 0, chunk_nr_pages;
 	pud_t *pudp;
 
 	pudp = pud_offset_lockless(p4dp, p4d, addr);
@@ -3053,24 +3066,27 @@ static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr,
 
 		next = pud_addr_end(addr, end);
 		if (unlikely(!pud_present(pud)))
-			return 0;
-		if (unlikely(pud_leaf(pud))) {
-			if (!gup_fast_pud_leaf(pud, pudp, addr, next, flags,
-					       pages, nr))
-				return 0;
-		} else if (!gup_fast_pmd_range(pudp, pud, addr, next, flags,
-					       pages, nr))
-			return 0;
+			break;
+		if (unlikely(pud_leaf(pud)))
+			chunk_nr_pages = gup_fast_pud_leaf(pud, pudp, addr,
+							   next, flags,
+							   &pages[nr_pages]);
+		else
+			chunk_nr_pages = gup_fast_pmd_range(pudp, pud, addr,
+							    next, flags,
+							    &pages[nr_pages]);
+		nr_pages += chunk_nr_pages;
+		if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
+			break;
 	} while (pudp++, addr = next, addr != end);
 
-	return 1;
+	return nr_pages;
 }
 
-static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
-		unsigned long end, unsigned int flags, struct page **pages,
-		int *nr)
+static unsigned long gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
+		unsigned long end, unsigned int flags, struct page **pages)
 {
-	unsigned long next;
+	unsigned long next, nr_pages = 0, chunk_nr_pages;
 	p4d_t *p4dp;
 
 	p4dp = p4d_offset_lockless(pgdp, pgd, addr);
@@ -3079,20 +3095,23 @@ static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,
 
 		next = p4d_addr_end(addr, end);
 		if (!p4d_present(p4d))
-			return 0;
+			break;
 		BUILD_BUG_ON(p4d_leaf(p4d));
-		if (!gup_fast_pud_range(p4dp, p4d, addr, next, flags,
-					pages, nr))
-			return 0;
+		chunk_nr_pages = gup_fast_pud_range(p4dp, p4d, addr, next,
+						    flags, &pages[nr_pages]);
+		nr_pages += chunk_nr_pages;
+		if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
+			break;
 	} while (p4dp++, addr = next, addr != end);
 
-	return 1;
+	return nr_pages;
 }
 
-static void gup_fast_pgd_range(unsigned long addr, unsigned long end,
-		unsigned int flags, struct page **pages, int *nr)
+static unsigned long gup_fast_pgd_range(unsigned long addr,
+		unsigned long end, unsigned int flags,
+		struct page **pages)
 {
-	unsigned long next;
+	unsigned long next, nr_pages = 0, chunk_nr_pages;
 	pgd_t *pgdp;
 
 	pgdp = pgd_offset(current->mm, addr);
@@ -3101,17 +3120,23 @@ static void gup_fast_pgd_range(unsigned long addr, unsigned long end,
 
 		next = pgd_addr_end(addr, end);
 		if (pgd_none(pgd))
-			return;
+			break;
 		BUILD_BUG_ON(pgd_leaf(pgd));
-		if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags,
-					pages, nr))
-			return;
+		chunk_nr_pages = gup_fast_p4d_range(pgdp, pgd, addr, next,
+						    flags, &pages[nr_pages]);
+		nr_pages += chunk_nr_pages;
+		if (chunk_nr_pages != (next - addr) >> PAGE_SHIFT)
+			break;
 	} while (pgdp++, addr = next, addr != end);
+
+	return nr_pages;
 }
 #else
-static inline void gup_fast_pgd_range(unsigned long addr, unsigned long end,
-		unsigned int flags, struct page **pages, int *nr)
+static inline unsigned long gup_fast_pgd_range(unsigned long addr,
+		unsigned long end, unsigned int flags,
+		struct page **pages)
 {
+	return 0;
 }
 #endif /* CONFIG_HAVE_GUP_FAST */
 
@@ -3129,8 +3154,7 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end)
 static unsigned long gup_fast(unsigned long start, unsigned long end,
 		unsigned int gup_flags, struct page **pages)
 {
-	unsigned long flags;
-	int nr_pinned = 0;
+	unsigned long flags, nr_pinned;
 	unsigned seq;
 
 	if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) ||
@@ -3154,7 +3178,7 @@ static unsigned long gup_fast(unsigned long start, unsigned long end,
 	 * that come from callers of tlb_remove_table_sync_one().
 	 */
 	local_irq_save(flags);
-	gup_fast_pgd_range(start, end, gup_flags, pages, &nr_pinned);
+	nr_pinned = gup_fast_pgd_range(start, end, gup_flags, pages);
 	local_irq_restore(flags);
 
 	/*
diff --git a/mm/gup_test.c b/mm/gup_test.c
index 44c1cdfb9c3717..ba74bf3f410486 100644
--- a/mm/gup_test.c
+++ b/mm/gup_test.c
@@ -116,7 +116,8 @@ static int __gup_test_ioctl(unsigned int cmd,
 	bool needs_mmap_lock =
 		cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
 
-	if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX)
+	if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX || !gup->size ||
+	    !gup->nr_pages_per_call || !PAGE_ALIGNED(gup->size))
 		return -EINVAL;
 	if (check_add_overflow((unsigned long)gup->addr,
 			       (unsigned long)gup->size, &end))
@@ -139,11 +140,9 @@ static int __gup_test_ioctl(unsigned int cmd,
 		if (nr != gup->nr_pages_per_call)
 			break;
 
+		nr = min_t(unsigned long, nr, (end - addr) / PAGE_SIZE);
+
 		next = addr + nr * PAGE_SIZE;
-		if (next > end) {
-			next = end;
-			nr = (next - addr) / PAGE_SIZE;
-		}
 
 		switch (cmd) {
 		case GUP_FAST_BENCHMARK:
@@ -188,7 +187,7 @@ static int __gup_test_ioctl(unsigned int cmd,
 	nr_pages = i;
 
 	gup->get_delta_usec = ktime_us_delta(end_time, start_time);
-	gup->size = addr - gup->addr;
+	gup->size = nr_pages * PAGE_SIZE;
 
 	/*
 	 * Take an un-benchmark-timed moment to verify DMA pinned
diff --git a/mm/hmm.c b/mm/hmm.c
index 2f1e98c6b6440b..e9569b82a1f0cc 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -595,8 +595,7 @@ static int hmm_vma_walk_test(unsigned long start, unsigned long end,
 	struct hmm_range *range = hmm_vma_walk->range;
 	struct vm_area_struct *vma = walk->vma;
 
-	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)) &&
-	    vma->vm_flags & VM_READ)
+	if (vma_can_gup(vma) && vma_test(vma, VMA_READ_BIT))
 		return 0;
 
 	/*
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 1e5d68acf62a52..ddc631a388b93e 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -110,14 +110,6 @@ static inline bool file_thp_enabled(const struct vm_area_struct *vma)
 	return S_ISREG(inode->i_mode);
 }
 
-/* If returns true, we are unable to access the VMA's folios. */
-static bool vma_is_special_huge(const struct vm_area_struct *vma)
-{
-	if (vma_is_dax(vma))
-		return false;
-	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
-}
-
 static bool vma_file_bypass_thp_tuneables(const struct vm_area_struct *vma,
 		enum tva_type type)
 {
@@ -192,7 +184,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 	/* Check the intersection of requested and supported orders. */
 	if (vma_is_anonymous(vma))
 		supported_orders = THP_ORDERS_ALL_ANON;
-	else if (vma_is_dax(vma) || vma_is_special_huge(vma))
+	else if (vma_is_dax(vma) || vma_is_kernel_owned(vma))
 		supported_orders = THP_ORDERS_ALL_SPECIAL_DAX;
 	else
 		supported_orders = THP_ORDERS_ALL_FILE_DEFAULT;
@@ -212,11 +204,14 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 		return in_pf ? orders : 0;
 
 	/*
-	 * khugepaged special VMA and hugetlb VMA.
-	 * Must be checked after dax since some dax mappings may have
-	 * VM_MIXEDMAP set.
+	 * khugepaged moves data from VMAs once collapsed, after they have been
+	 * faulted in, relying on refaulting for file-backed memory.
+	 *
+	 * Kernel-owned mappings cannot be reliably reconstructed from page
+	 * faults, and fixed mappings (including hugetlb) may not be marked as
+	 * kernel-owned - precisely the mappings which cannot be merged.
 	 */
-	if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED))
+	if (!in_pf && !smaps && !vma_can_merge(vma))
 		return 0;
 
 	/*
@@ -259,12 +254,12 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 
 	/*
 	 * THPeligible bit of smaps should show 1 for proper VMAs even
-	 * though anon_vma is not initialized yet.
+	 * though they don't have an anon rmap yet.
 	 *
-	 * Allow page fault since anon_vma may be not initialized until
-	 * the first page fault.
+	 * Allow page fault since the VMA may not have an anon rmap until the
+	 * first page fault.
 	 */
-	if (!vma->anon_vma)
+	if (!vma_has_anon_rmap(vma))
 		return (smaps || in_pf) ? orders : 0;
 
 	return orders;
@@ -1146,6 +1141,7 @@ subsys_initcall(hugepage_init);
 static int __init setup_transparent_hugepage(char *str)
 {
 	int ret = 0;
+
 	if (!str)
 		goto out;
 	if (!strcmp(str, "always")) {
@@ -1552,6 +1548,7 @@ static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm,
 		struct folio *zero_folio)
 {
 	pmd_t entry;
+
 	entry = folio_mk_pmd(zero_folio, vma->vm_page_prot);
 	entry = pmd_mkspecial(entry);
 	pgtable_trans_huge_deposit(mm, pmd, pgtable);
@@ -1725,6 +1722,9 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn,
 						(VM_PFNMAP|VM_MIXEDMAP));
 	BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma));
 
+	if (unlikely(is_huge_zero_pfn(pfn)))
+		return VM_FAULT_SIGBUS;
+
 	pfnmap_setup_cachemode_pfn(pfn, &pgprot);
 
 	return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write);
@@ -2174,7 +2174,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
 	pmd_t orig_pmd = vmf->orig_pmd;
 
 	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
-	VM_BUG_ON_VMA(!vma->anon_vma, vma);
+	VM_BUG_ON_VMA(!vma_has_anon_rmap(vma), vma);
 
 	if (is_huge_zero_pmd(orig_pmd)) {
 		vm_fault_t ret = do_huge_zero_wp_pmd(vmf);
@@ -2423,6 +2423,10 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	}
 
 	folio = pmd_folio(orig_pmd);
+
+	if (folio_is_zone_device(folio))
+		goto out;
+
 	/*
 	 * If other processes are mapping this folio, we couldn't discard
 	 * the folio unless they all do MADV_FREE so let's skip the folio.
@@ -2514,25 +2518,13 @@ static struct folio *normal_or_softleaf_folio_pmd(struct vm_area_struct *vma,
 	return pmd_to_softleaf_folio(pmdval);
 }
 
-static bool has_deposited_pgtable(struct vm_area_struct *vma, pmd_t pmdval,
-		struct folio *folio)
+static bool vma_has_deposited_pgtable(struct vm_area_struct *vma)
 {
-	/* Some architectures require unconditional depositing. */
-	if (arch_needs_pgtable_deposit())
-		return true;
-
 	/*
-	 * Huge zero always deposited except for DAX which handles itself, see
-	 * set_huge_zero_folio().
+	 * PMDs in anonymous VMAs always have a deposited page table. PMDs in
+	 * other VMAs only have one when required by the architecture.
 	 */
-	if (is_huge_zero_pmd(pmdval))
-		return !vma_is_dax(vma);
-
-	/*
-	 * Otherwise, only anonymous folios are deposited, see
-	 * __do_huge_pmd_anonymous_page().
-	 */
-	return folio && folio_test_anon(folio);
+	return arch_needs_pgtable_deposit() || vma_is_anonymous(vma);
 }
 
 /**
@@ -2572,7 +2564,7 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 
 	is_present = pmd_present(orig_pmd);
 	folio = normal_or_softleaf_folio_pmd(vma, addr, orig_pmd, is_present);
-	has_deposit = has_deposited_pgtable(vma, orig_pmd, folio);
+	has_deposit = vma_has_deposited_pgtable(vma);
 	if (folio)
 		zap_huge_pmd_folio(mm, vma, orig_pmd, folio, is_present);
 	if (has_deposit)
@@ -2657,6 +2649,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 
 		if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
 			pgtable_t pgtable;
+
 			pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
 			pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
 		}
@@ -3055,7 +3048,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm);
 	arch_check_zapped_pud(vma, orig_pud);
 	tlb_remove_pud_tlb_entry(tlb, pud, addr);
-	if (vma_is_special_huge(vma)) {
+	if (vma_is_kernel_owned(vma)) {
 		spin_unlock(ptl);
 		/* No zero page support yet */
 	} else {
@@ -3211,7 +3204,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 		 */
 		if (arch_needs_pgtable_deposit())
 			zap_deposited_table(mm, pmd);
-		if (vma_is_special_huge(vma))
+		if (vma_is_kernel_owned(vma))
 			return;
 		if (unlikely(pmd_is_migration_entry(old_pmd))) {
 			const softleaf_t old_entry = softleaf_from_pmd(old_pmd);
@@ -3422,7 +3415,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 				swp_entry = make_readable_device_private_entry(
 							page_to_pfn(page + i));
 			/*
-			 * Young and dirty bits are not progated via swp_entry
+			 * Young and dirty bits are not propagated via swp_entry
 			 */
 			entry = swp_entry_to_pte(swp_entry);
 			if (soft_dirty)
@@ -3531,20 +3524,28 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma,
 		split_huge_pmd_if_needed(next, end);
 }
 
-static void unmap_folio(struct folio *folio)
+/*
+ * A return value of 0 does not mean that unmapping succeeded. It might
+ * still have failed, but remap_anon_folio() must be called afterwards,
+ * for anon folios.
+ */
+static int unmap_folio(struct folio *folio)
 {
 	enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC |
 		TTU_BATCH_FLUSH;
 
 	VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
 
+	/* Racy check if we can split the page, before we split PMDs */
+	if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1)
+		return -EAGAIN;
+
 	if (folio_test_pmd_mappable(folio))
 		ttu_flags |= TTU_SPLIT_HUGE_PMD;
 
 	/*
 	 * Anon pages need migration entries to preserve them, but file
 	 * pages can simply be left unmapped, then faulted back on demand.
-	 * If that is ever changed (perhaps for mlock), update remap_page().
 	 */
 	if (folio_test_anon(folio))
 		try_to_migrate(folio, ttu_flags);
@@ -3552,6 +3553,8 @@ static void unmap_folio(struct folio *folio)
 		try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
 
 	try_to_unmap_flush();
+
+	return 0;
 }
 
 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma,
@@ -3629,13 +3632,17 @@ bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr,
 	return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio);
 }
 
-static void remap_page(struct folio *folio, unsigned long nr, int flags)
+static void remap_anon_folio(struct folio *folio, unsigned long nr, int flags)
 {
 	int i = 0;
 
-	/* If unmap_folio() uses try_to_migrate() on file, remove this check */
-	if (!folio_test_anon(folio))
-		return;
+	/*
+	 * unmap_folio() installs migration entries only for anon folios,
+	 * so currently only anon folios need to be remapped. File folios
+	 * stay unmapped after the split and are faulted back on demand.
+	 */
+	VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
+
 	for (;;) {
 		remove_migration_ptes(folio, folio, TTU_RMAP_LOCKED | flags);
 		i += folio_nr_pages(folio);
@@ -3719,7 +3726,7 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
 		 *
 		 * Note that for mapped sub-pages of an anonymous THP,
 		 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in
-		 * the migration entry instead from where remap_page() will restore it.
+		 * the migration entry instead from where remap_anon_folio() will restore it.
 		 * We can still have PG_anon_exclusive set on effectively unmapped and
 		 * unreferenced sub-pages of an anonymous THP: we can simply drop
 		 * PG_anon_exclusive (-> PG_mappedtodisk) for these here.
@@ -3755,6 +3762,10 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
 		 */
 		VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head);
 
+		/*
+		 * Not all folio fields are valid during a split, so open-code
+		 * the swap entry rather than using folio_swap_entry().
+		 */
 		if (folio_test_swapcache(folio))
 			new_folio->swap.val = folio->swap.val + i;
 
@@ -3799,8 +3810,8 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
 }
 
 /**
- * __split_unmapped_folio() - splits an unmapped @folio to lower order folios in
- * two ways: uniform split or non-uniform split.
+ * __split_frozen_folio() - splits a frozen @folio to lower order folios
+ * in two ways: uniform split or non-uniform split.
  * @folio: the to-be-split folio
  * @new_order: the smallest order of the after split folios (since buddy
  *             allocator like split generates folios with orders from @folio's
@@ -3808,7 +3819,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
  * @split_at: in buddy allocator like split, the folio containing @split_at
  *            will be split until its order becomes @new_order.
  * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller
- * @mapping: @folio->mapping
  * @split_type: if the split is uniform or not (buddy allocator like split)
  *
  *
@@ -3839,9 +3849,9 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
  * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be
  * split but not to @new_order, the caller needs to check)
  */
-static int __split_unmapped_folio(struct folio *folio, int new_order,
+static int __split_frozen_folio(struct folio *folio, int new_order,
 		struct page *split_at, struct xa_state *xas,
-		struct address_space *mapping, enum split_type split_type)
+		enum split_type split_type)
 {
 	const bool is_anon = folio_test_anon(folio);
 	int old_order = folio_order(folio);
@@ -3849,6 +3859,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
 	struct folio *old_folio = folio;
 	int split_order;
 
+	/* Frozen implies unmapped, callers unmap before splitting. */
+	VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio);
+
 	/*
 	 * split to new_order one order at a time. For uniform split,
 	 * folio is split to new_order directly.
@@ -3862,7 +3875,7 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
 		if (is_anon && split_order == 1)
 			continue;
 
-		if (mapping) {
+		if (xas) {
 			/*
 			 * uniform split has xas_split_alloc() called before
 			 * irq is disabled to allocate enough memory, whereas
@@ -3922,6 +3935,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
 int folio_check_splittable(struct folio *folio, unsigned int new_order,
 			   enum split_type split_type)
 {
+	const bool is_anon = folio_test_anon(folio);
+	const bool is_swapcache = folio_test_swapcache(folio);
+
 	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
 	/*
 	 * Folios that just got truncated cannot get split. Signal to the
@@ -3930,11 +3946,11 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order,
 	 * TODO: this will also currently refuse folios without a mapping in the
 	 * swapcache (shmem or to-be-anon folios).
 	 */
-	if (!folio->mapping && !folio_test_anon(folio))
+	if (!folio->mapping && !is_anon)
 		return -EBUSY;
 
 	/* order-1 is not supported for anonymous THP. */
-	if (folio_test_anon(folio) && new_order == 1)
+	if (is_anon && new_order == 1)
 		return -EINVAL;
 
 	/*
@@ -3945,9 +3961,8 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order,
 	 * swapcache folio split. Only uniform split to order-0 can be used
 	 * here.
 	 */
-	if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && folio_test_swapcache(folio)) {
+	if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && is_swapcache)
 		return -EINVAL;
-	}
 
 	if (is_huge_zero_folio(folio))
 		return -EINVAL;
@@ -3955,169 +3970,310 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order,
 	if (folio_test_writeback(folio))
 		return -EBUSY;
 
+	/*
+	 * A non-anon swapcache folio that still has a mapping can only be a
+	 * shmem folio under SWAP IO, it's removed from either swap cache or
+	 * shmem mapping afterward. There is little benefit in splitting them
+	 * hence reject it here up front before touching anything.
+	 */
+	if (!is_anon && is_swapcache && folio->mapping)
+		return -EBUSY;
+
 	return 0;
 }
 
-/* Number of folio references from the pagecache or the swapcache. */
-static unsigned int folio_cache_ref_count(const struct folio *folio)
+/* Number of folio references from the swapcache. */
+static unsigned int folio_swapcache_ref_count(const struct folio *folio)
 {
-	if (folio_test_anon(folio) && !folio_test_swapcache(folio))
+	if (!folio_test_swapcache(folio))
 		return 0;
 	return folio_nr_pages(folio);
 }
 
-static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order,
-					     struct page *split_at, struct xa_state *xas,
-					     struct address_space *mapping, bool do_lru,
-					     struct list_head *list, enum split_type split_type,
-					     pgoff_t end, int *nr_shmem_dropped)
+static void folio_reset_partially_mapped(struct folio *folio)
+{
+	/* Folio must be frozen. */
+	VM_WARN_ON_FOLIO(folio_ref_count(folio), folio);
+
+	if (!folio_test_partially_mapped(folio))
+		return;
+
+	/*
+	 * Order-1 folios have no _deferred_list. The flag is only ever set
+	 * on folios that do, so the list can be checked after the flag.
+	 */
+	VM_WARN_ON_FOLIO(!list_empty(&folio->_deferred_list), folio);
+
+	folio_clear_partially_mapped(folio);
+	mod_mthp_stat(folio_order(folio),
+		      MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
+}
+
+static int __folio_freeze_split_anon(struct folio *folio,
+		unsigned int new_order, struct page *split_at, bool do_lru,
+		struct list_head *list, enum split_type split_type)
 {
 	struct folio *end_folio = folio_next(folio);
+	struct swap_cluster_info *ci = NULL;
+	const int old_order = folio_order(folio);
 	struct folio *new_folio, *next;
-	int old_order = folio_order(folio);
-	struct list_lru_one *lru;
-	bool dequeue_deferred;
+	struct anon_vma *anon_vma = NULL;
+	enum ttu_flags ttu_flags = 0;
+	struct lruvec *lruvec;
 	int ret = 0;
 
-	VM_WARN_ON_ONCE(!mapping && end);
 	/*
-	 * If this folio can be on the deferred split queue, lock out
-	 * the shrinker before freezing the ref. If the shrinker sees
-	 * a 0-ref folio, it assumes it beat folio_put() to the list
-	 * lock and must clean up the LRU state - the same dequeue we
-	 * will do below as part of the split.
+	 * Unmap/remap needs the anon_vma, so we first take a reference on
+	 * it to prevent it from disappearing, and lock it for write here,
+	 * letting unmap_folio() walk the rmap with TTU_RMAP_LOCKED.
+	 *
+	 * folio_mapped() is not stable here, but it can only change in
+	 * one direction while the folio is locked. The mapcount can drop
+	 * to zero at any time, zap_pte_range() takes no folio lock. It
+	 * cannot go up: swapin, migration and uffd move all lock the folio
+	 * before mapping it, and fork only copies PTEs that already exist.
+	 *
+	 * So if we see the folio mapped, the worst case is an empty rmap
+	 * walk. If we see it unmapped, it stays unmapped and needs neither
+	 * the reference nor the lock. Anything else needs a reference
+	 * first and folio_ref_freeze() below catches it.
+	 *
+	 * Note that entirely swapped-out THPs are unmapped but can be split.
 	 */
-	dequeue_deferred = folio_test_anon(folio) && old_order > 1;
-	if (dequeue_deferred) {
-		struct mem_cgroup *memcg;
-
-		rcu_read_lock();
-		memcg = folio_memcg(folio);
-		lru = list_lru_lock(&deferred_split_lru,
-				    folio_nid(folio), &memcg);
+	if (folio_mapped(folio)) {
+		anon_vma = folio_get_anon_vma(folio);
+		if (!anon_vma)
+			return -EBUSY;
+		anon_vma_lock_write(anon_vma);
+		ret = unmap_folio(folio);
+		if (ret)
+			goto out_unlock;
 	}
-	if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) {
-		struct swap_cluster_info *ci = NULL;
-		struct lruvec *lruvec;
-
-		if (dequeue_deferred) {
-			__list_lru_del(&deferred_split_lru, lru,
-				       &folio->_deferred_list, folio_nid(folio));
-			if (folio_test_partially_mapped(folio)) {
-				folio_clear_partially_mapped(folio);
-				mod_mthp_stat(old_order,
-					MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
-			}
-			list_lru_unlock(lru);
-			rcu_read_unlock();
-		}
 
-		if (mapping) {
-			int nr = folio_nr_pages(folio);
-
-			if (folio_test_pmd_mappable(folio) &&
-			    new_order < HPAGE_PMD_ORDER) {
-				if (folio_test_swapbacked(folio)) {
-					lruvec_stat_mod_folio(folio,
-							NR_SHMEM_THPS, -nr);
-				} else {
-					lruvec_stat_mod_folio(folio,
-							NR_FILE_THPS, -nr);
-				}
-			}
-		}
+	local_irq_disable();
 
-		if (folio_test_swapcache(folio)) {
-			if (mapping) {
-				VM_WARN_ON_ONCE_FOLIO(mapping, folio);
-				return -EINVAL;
-			}
+	if (!folio_ref_freeze(folio, folio_swapcache_ref_count(folio) + 1)) {
+		ret = -EAGAIN;
+		goto out_no_split;
+	}
 
-			ci = swap_cluster_get_and_lock(folio);
-		}
+	/* Take off the deferred split queue while frozen and memcg set */
+	folio_unqueue_deferred_split(folio);
 
-		/* lock lru list/PageCompound, ref frozen by page_ref_freeze */
+	/*
+	 * deferred_split_scan() takes the folio off the queue before it
+	 * splits it, so the unqueue above finds an empty list and
+	 * leaves PG_partially_mapped set.
+	 * Clear it here: the flag does not survive the split.
+	 */
+	folio_reset_partially_mapped(folio);
+
+	if (folio_test_swapcache(folio))
+		ci = swap_cluster_get_and_lock(folio);
+
+	if (do_lru)
+		lruvec = folio_lruvec_lock(folio);
+
+	ret = __split_frozen_folio(folio, new_order, split_at, NULL, split_type);
+
+	/*
+	 * Unfreeze the after-split folios and put them back to the right
+	 * place. Keep the head @folio frozen until the end: sub entries
+	 * in swap cache must be updated first, so a concurrent
+	 * swap_cache_get_folio() cannot return the head folio for a sub
+	 * entry (folio_try_get() will fail on the head @folio until unfreeze).
+	 */
+	for (new_folio = folio_next(folio); new_folio != end_folio;
+	     new_folio = next) {
+		next = folio_next(new_folio);
+		zone_device_private_split_cb(folio, new_folio);
+		folio_ref_unfreeze(new_folio,
+				   folio_swapcache_ref_count(new_folio) + 1);
 		if (do_lru)
-			lruvec = folio_lruvec_lock(folio);
+			lru_add_split_folio(folio, new_folio, lruvec, list);
+		if (ci)
+			__swap_cache_replace_folio(ci, folio, new_folio);
+	}
 
-		ret = __split_unmapped_folio(folio, new_order, split_at, xas,
-					     mapping, split_type);
+	zone_device_private_split_cb(folio, NULL);
+	folio_ref_unfreeze(folio, folio_swapcache_ref_count(folio) + 1);
 
-		/*
-		 * Unfreeze after-split folios and put them back to the right
-		 * list. @folio should be kept frozon until page cache
-		 * entries are updated with all the other after-split folios
-		 * to prevent others seeing stale page cache entries.
-		 * As a result, new_folio starts from the next folio of
-		 * @folio.
-		 */
-		for (new_folio = folio_next(folio); new_folio != end_folio;
-		     new_folio = next) {
-			unsigned long nr_pages = folio_nr_pages(new_folio);
+	if (do_lru)
+		lruvec_unlock(lruvec);
+	if (ci)
+		swap_cluster_unlock(ci);
+out_no_split:
+	local_irq_enable();
+	if (anon_vma) {
+		if (!ret && !folio_is_device_private(folio))
+			ttu_flags = TTU_USE_SHARED_ZEROPAGE;
+		remap_anon_folio(folio, 1 << old_order, ttu_flags);
+	}
+out_unlock:
+	if (anon_vma) {
+		anon_vma_unlock_write(anon_vma);
+		put_anon_vma(anon_vma);
+	}
 
-			next = folio_next(new_folio);
+	return ret;
+}
 
-			zone_device_private_split_cb(folio, new_folio);
+static int __folio_freeze_split_file(struct folio *folio,
+		unsigned int new_order, struct page *split_at,
+		struct list_head *list, enum split_type split_type)
+{
+	const long old_nr_pages = folio_nr_pages(folio);
+	struct address_space *mapping = folio->mapping;
+	XA_STATE(xas, &mapping->i_pages, folio->index);
+	struct folio *end_folio = folio_next(folio);
+	struct mem_cgroup *memcg, *old_memcg;
+	struct folio *new_folio, *next;
+	int nr_shmem_dropped = 0;
+	unsigned int min_order;
+	struct lruvec *lruvec;
+	pgoff_t end;
+	gfp_t gfp;
+	int ret = 0;
 
-			folio_ref_unfreeze(new_folio,
-					   folio_cache_ref_count(new_folio) + 1);
+	min_order = mapping_min_folio_order(mapping);
+	if (new_order < min_order)
+		return -EINVAL;
 
-			if (do_lru)
-				lru_add_split_folio(folio, new_folio, lruvec, list);
+	/*
+	 * Switch to folio's memcg as xarray node allocation can happen and
+	 * needs to charge to it.
+	 */
+	memcg = get_mem_cgroup_from_folio(folio);
+	old_memcg = set_active_memcg(memcg);
 
-			/*
-			 * Anonymous folio with swap cache.
-			 * NOTE: shmem in swap cache is not supported yet.
-			 */
-			if (ci) {
-				__swap_cache_replace_folio(ci, folio, new_folio);
-				continue;
-			}
+	gfp = current_gfp_context(mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK);
+	if (!filemap_release_folio(folio, gfp)) {
+		ret = -EBUSY;
+		goto fail_free;
+	}
 
-			/* Anonymous folio without swap cache */
-			if (!mapping)
-				continue;
+	mapping_set_update(&xas, mapping);
 
-			/* Add the new folio to the page cache. */
-			if (new_folio->index < end) {
-				__xa_store(&mapping->i_pages, new_folio->index,
-					   new_folio, 0);
-				continue;
-			}
+	if (split_type == SPLIT_TYPE_UNIFORM) {
+		const int old_order = folio_order(folio);
 
-			VM_WARN_ON_ONCE(!nr_shmem_dropped);
-			/* Drop folio beyond EOF: ->index >= end */
-			if (shmem_mapping(mapping) && nr_shmem_dropped)
-				*nr_shmem_dropped += nr_pages;
-			else if (folio_test_clear_dirty(new_folio))
-				folio_account_cleaned(
-					new_folio, inode_to_wb(mapping->host));
-			__filemap_remove_folio(new_folio, NULL);
-			folio_put_refs(new_folio, nr_pages);
+		xas_set_order(&xas, folio->index, new_order);
+		xas_split_alloc(&xas, folio, old_order, gfp);
+		if (xas_error(&xas)) {
+			ret = xas_error(&xas);
+			goto fail_free;
 		}
+	}
 
-		zone_device_private_split_cb(folio, NULL);
-		/*
-		 * Unfreeze @folio only after all page cache entries, which
-		 * used to point to it, have been updated with new folios.
-		 * Otherwise, a parallel folio_try_get() can grab @folio
-		 * and its caller can see stale page cache entries.
-		 */
-		folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1);
+	i_mmap_lock_read(mapping);
 
-		if (do_lru)
-			lruvec_unlock(lruvec);
+	/* Currently device private folios can only back anonymous memory. */
+	VM_WARN_ON_ONCE_FOLIO(folio_is_device_private(folio), folio);
 
-		if (ci)
-			swap_cluster_unlock(ci);
-	} else {
-		if (dequeue_deferred) {
-			list_lru_unlock(lru);
-			rcu_read_unlock();
+	/*
+	 * The loop below may need to trim off pages beyond
+	 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe
+	 * seqlock, which cannot be nested inside the page tree lock.
+	 * So note end now: i_size itself may be changed at any moment,
+	 * but folio lock is good enough to serialize the trimming.
+	 */
+	end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
+	if (shmem_mapping(mapping))
+		end = shmem_fallocend(mapping->host, end);
+
+	ret = unmap_folio(folio);
+	if (ret)
+		goto fail_mmap_unlock;
+
+	xas_lock_irq(&xas);
+
+	/*
+	 * Check if the folio is present in page cache.
+	 * We assume all tail are present too, if folio is there.
+	 */
+	if (xas_load(&xas) != folio) {
+		ret = -EAGAIN;
+		goto fail;
+	}
+
+	if (!folio_ref_freeze(folio, old_nr_pages + 1)) {
+		ret = -EAGAIN;
+		goto fail;
+	}
+
+	if (folio_test_pmd_mappable(folio) && new_order < HPAGE_PMD_ORDER) {
+		if (folio_test_swapbacked(folio))
+			lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -old_nr_pages);
+		else
+			lruvec_stat_mod_folio(folio, NR_FILE_THPS, -old_nr_pages);
+	}
+
+	/* lock lru list/PageCompound, ref frozen by page_ref_freeze */
+	lruvec = folio_lruvec_lock(folio);
+	ret = __split_frozen_folio(folio, new_order, split_at, &xas, split_type);
+
+	/*
+	 * Unfreeze after-split folios and put them back to the right
+	 * list. @folio should be kept frozen until page cache
+	 * entries are updated with all the other after-split folios
+	 * to prevent others seeing stale page cache entries.
+	 * As a result, new_folio starts from the next folio of
+	 * @folio.
+	 */
+	for (new_folio = folio_next(folio); new_folio != end_folio;
+	     new_folio = next) {
+		unsigned long nr_pages = folio_nr_pages(new_folio);
+
+		/* compute next before the folio can be freed below */
+		next = folio_next(new_folio);
+
+		folio_ref_unfreeze(new_folio,
+				   folio_nr_pages(new_folio) + 1);
+
+		lru_add_split_folio(folio, new_folio, lruvec, list);
+
+		/* Add the new folio to the page cache. */
+		if (new_folio->index < end) {
+			__xa_store(&mapping->i_pages, new_folio->index,
+				   new_folio, 0);
+			continue;
 		}
-		return -EAGAIN;
+
+		/* Drop folio beyond EOF: ->index >= end */
+		if (shmem_mapping(mapping))
+			nr_shmem_dropped += nr_pages;
+		else if (folio_test_clear_dirty(new_folio))
+			folio_account_cleaned(
+				new_folio, inode_to_wb(mapping->host));
+		__filemap_remove_folio(new_folio, NULL);
+		folio_put_refs(new_folio, nr_pages);
 	}
 
+	/*
+	 * Unfreeze @folio only after all page cache entries, which
+	 * used to point to it, have been updated with new folios.
+	 * Otherwise, a parallel folio_try_get() can grab @folio
+	 * and its caller can see stale page cache entries.
+	 */
+	folio_ref_unfreeze(folio, folio_nr_pages(folio) + 1);
+	lruvec_unlock(lruvec);
+fail:
+	xas_unlock_irq(&xas);
+fail_mmap_unlock:
+	if (nr_shmem_dropped)
+		shmem_uncharge(mapping->host, nr_shmem_dropped);
+	/*
+	 * Drop the mapping while the inode is still pinned. @folio stays
+	 * locked and present in the page cache, so eviction cannot free
+	 * the inode yet, nothing past this point may touch the inode or
+	 * the mapping.
+	 */
+	i_mmap_unlock_read(mapping);
+fail_free:
+	/* Restore the previously active memcg */
+	set_active_memcg(old_memcg);
+	mem_cgroup_put(memcg);
+	xas_destroy(&xas);
 	return ret;
 }
 
@@ -4130,9 +4286,9 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
  * @list: after-split folios will be put on it if non NULL
  * @split_type: perform uniform split or not (non-uniform split)
  *
- * It calls __split_unmapped_folio() to perform uniform and non-uniform split.
+ * It calls __split_frozen_folio() to perform uniform and non-uniform split.
  * It is in charge of checking whether the split is supported or not and
- * preparing @folio for __split_unmapped_folio().
+ * preparing @folio for __split_frozen_folio().
  *
  * After splitting, the after-split folio containing @lock_at remains locked
  * and others are unlocked:
@@ -4146,17 +4302,11 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
 		struct page *split_at, struct page *lock_at,
 		struct list_head *list, enum split_type split_type)
 {
-	XA_STATE(xas, &folio->mapping->i_pages, folio->index);
 	struct folio *end_folio = folio_next(folio);
-	bool is_anon = folio_test_anon(folio);
-	struct mem_cgroup *memcg, *old_memcg;
-	struct address_space *mapping = NULL;
-	struct anon_vma *anon_vma = NULL;
+	const bool is_anon = folio_test_anon(folio);
+	const bool is_swapcache = folio_test_swapcache(folio);
 	int old_order = folio_order(folio);
 	struct folio *new_folio, *next;
-	int nr_shmem_dropped = 0;
-	enum ttu_flags ttu_flags = 0;
-	pgoff_t end = 0;
 	int ret;
 
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
@@ -4164,141 +4314,26 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
 
 	if (folio != page_folio(split_at) || folio != page_folio(lock_at)) {
 		ret = -EINVAL;
-		goto out_no_memcg;
+		goto out;
 	}
 
 	if (new_order >= old_order) {
 		ret = -EINVAL;
-		goto out_no_memcg;
+		goto out;
 	}
 
 	ret = folio_check_splittable(folio, new_order, split_type);
 	if (ret) {
 		VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio");
-		goto out_no_memcg;
-	}
-
-	/*
-	 * switch to folio's memcg as xarray node allocation can happen and
-	 * needs to charge to it.
-	 */
-	memcg = get_mem_cgroup_from_folio(folio);
-	old_memcg = set_active_memcg(memcg);
-
-	if (is_anon) {
-		/*
-		 * The caller does not necessarily hold an mmap_lock that would
-		 * prevent the anon_vma disappearing so we first we take a
-		 * reference to it and then lock the anon_vma for write. This
-		 * is similar to folio_lock_anon_vma_read except the write lock
-		 * is taken to serialise against parallel split or collapse
-		 * operations.
-		 */
-		anon_vma = folio_get_anon_vma(folio);
-		if (!anon_vma) {
-			ret = -EBUSY;
-			goto out;
-		}
-		anon_vma_lock_write(anon_vma);
-		mapping = NULL;
-	} else {
-		unsigned int min_order;
-		gfp_t gfp;
-
-		mapping = folio->mapping;
-		min_order = mapping_min_folio_order(mapping);
-		if (new_order < min_order) {
-			ret = -EINVAL;
-			goto out;
-		}
-
-		gfp = current_gfp_context(mapping_gfp_mask(mapping) &
-							GFP_RECLAIM_MASK);
-
-		if (!filemap_release_folio(folio, gfp)) {
-			ret = -EBUSY;
-			goto out;
-		}
-
-		mapping_set_update(&xas, mapping);
-
-		if (split_type == SPLIT_TYPE_UNIFORM) {
-			xas_set_order(&xas, folio->index, new_order);
-			xas_split_alloc(&xas, folio, old_order, gfp);
-			if (xas_error(&xas)) {
-				ret = xas_error(&xas);
-				goto out;
-			}
-		}
-
-		anon_vma = NULL;
-		i_mmap_lock_read(mapping);
-
-		/*
-		 *__split_unmapped_folio() may need to trim off pages beyond
-		 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe
-		 * seqlock, which cannot be nested inside the page tree lock.
-		 * So note end now: i_size itself may be changed at any moment,
-		 * but folio lock is good enough to serialize the trimming.
-		 */
-		end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
-		if (shmem_mapping(mapping))
-			end = shmem_fallocend(mapping->host, end);
-	}
-
-	/*
-	 * Racy check if we can split the page, before unmap_folio() will
-	 * split PMDs
-	 */
-	if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) {
-		ret = -EAGAIN;
-		goto out_unlock;
-	}
-
-	unmap_folio(folio);
-
-	/* block interrupt reentry in xa_lock and spinlock */
-	local_irq_disable();
-	if (mapping) {
-		/*
-		 * Check if the folio is present in page cache.
-		 * We assume all tail are present too, if folio is there.
-		 */
-		xas_lock(&xas);
-		xas_reset(&xas);
-		if (xas_load(&xas) != folio) {
-			ret = -EAGAIN;
-			goto fail;
-		}
+		goto out;
 	}
 
-	ret = __folio_freeze_and_split_unmapped(folio, new_order, split_at, &xas, mapping,
-						true, list, split_type, end, &nr_shmem_dropped);
-fail:
-	if (mapping)
-		xas_unlock(&xas);
-
-	local_irq_enable();
-
-	if (nr_shmem_dropped)
-		shmem_uncharge(mapping->host, nr_shmem_dropped);
-
-	if (!ret && is_anon && !folio_is_device_private(folio))
-		ttu_flags = TTU_USE_SHARED_ZEROPAGE;
-
-	remap_page(folio, 1 << old_order, ttu_flags);
-
-	/*
-	 * Drop the mapping while the inode is still pinned. @folio stays
-	 * locked and present in the page cache until the loop below, so
-	 * eviction cannot free the inode yet; @lock_at is not enough, it may
-	 * be a tail beyond EOF that the split already dropped from the page
-	 * cache. Nothing past this point may touch the inode or the mapping.
-	 */
-	if (mapping) {
-		i_mmap_unlock_read(mapping);
-		mapping = NULL;
-	}
+	if (is_anon)
+		ret = __folio_freeze_split_anon(folio, new_order, split_at,
+						true, list, split_type);
+	else
+		ret = __folio_freeze_split_file(folio, new_order, split_at,
+						list, split_type);
 
 	/*
 	 * Unlock all after-split folios except the one containing
@@ -4309,29 +4344,19 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
 		if (new_folio == page_folio(lock_at))
 			continue;
 
-		folio_unlock(new_folio);
 		/*
 		 * Subpages whose mapping has been zapped may be freed
 		 * earlier, but freeing them requires taking the
-		 * lru_lock, so we defer put_page() on tail pages until
+		 * lru_lock, so we defer folio_put() on tail pages until
 		 * after the split completes.
 		 */
-		free_folio_and_swap_cache(new_folio);
+		if (is_swapcache && !folio_mapped(new_folio))
+			folio_free_swap(new_folio);
+		folio_unlock(new_folio);
+		folio_put(new_folio);
 	}
 
-out_unlock:
-	if (anon_vma) {
-		anon_vma_unlock_write(anon_vma);
-		put_anon_vma(anon_vma);
-	}
-	if (mapping)
-		i_mmap_unlock_read(mapping);
 out:
-	/* restore to caller's old_memcg */
-	set_active_memcg(old_memcg);
-	mem_cgroup_put(memcg);
-out_no_memcg:
-	xas_destroy(&xas);
 	if (is_pmd_order(old_order))
 		count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
 	count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED);
@@ -4348,36 +4373,27 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
  * THP pages in the middle of migration, due to allocation issues on either
  * side.
  *
- * anon_vma_lock is not required to be held, mmap_read_lock() or
+ * The anon rmap lock is not required to be held, mmap_read_lock() or
  * mmap_write_lock() should be held. @folio is expected to be locked by the
  * caller. device-private and non device-private folios are supported along
  * with folios that are in the swapcache. @folio should also be unmapped and
  * isolated from LRU (if applicable)
  *
  * Upon return, the folio is not remapped, split folios are not added to LRU,
- * free_folio_and_swap_cache() is not called, and new folios remain locked.
+ * folio_free_swap() is not called, and new folios remain locked.
  *
  * Return: 0 on success, -EAGAIN if the folio cannot be split (e.g., due to
  *         insufficient reference count or extra pins).
  */
 int folio_split_unmapped(struct folio *folio, unsigned int new_order)
 {
-	int ret = 0;
-
 	VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio);
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio);
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_anon(folio), folio);
 
-	if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1)
-		return -EAGAIN;
-
-	local_irq_disable();
-	ret = __folio_freeze_and_split_unmapped(folio, new_order, &folio->page, NULL,
-						NULL, false, NULL, SPLIT_TYPE_UNIFORM,
-						0, NULL);
-	local_irq_enable();
-	return ret;
+	return __folio_freeze_split_anon(folio, new_order, &folio->page,
+					 false, NULL, SPLIT_TYPE_UNIFORM);
 }
 
 /*
@@ -4528,11 +4544,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio)
 	memcg = folio_memcg(folio);
 	lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags);
 	if (__list_lru_del(&deferred_split_lru, lru, &folio->_deferred_list, nid)) {
-		if (folio_test_partially_mapped(folio)) {
-			folio_clear_partially_mapped(folio);
-			mod_mthp_stat(folio_order(folio),
-				      MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
-		}
+		folio_reset_partially_mapped(folio);
 		unqueued = true;
 	}
 	list_lru_unlock_irqrestore(lru, &flags);
@@ -4634,22 +4646,11 @@ static enum lru_status deferred_split_isolate(struct list_head *item,
 	struct folio *folio = container_of(item, struct folio, _deferred_list);
 	struct list_head *freeable = cb_arg;
 
-	if (folio_try_get(folio)) {
-		list_lru_isolate_move(lru, item, freeable);
-		return LRU_REMOVED;
-	}
+	/* Lost race to folio_put() or the folio is under folio_ref_freeze() */
+	if (!folio_try_get(folio))
+		return LRU_SKIP;
 
-	/*
-	 * We lost race with folio_put(). Read folio state before the
-	 * isolate: folio_unqueue_deferred_split() checks list_empty()
-	 * locklessly, so once removed the folio can be freed any time.
-	 */
-	if (folio_test_partially_mapped(folio)) {
-		folio_clear_partially_mapped(folio);
-		mod_mthp_stat(folio_order(folio),
-			      MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
-	}
-	list_lru_isolate(lru, item);
+	list_lru_isolate_move(lru, item, freeable);
 	return LRU_REMOVED;
 }
 
@@ -4770,11 +4771,9 @@ static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
 {
 	if (vma_is_dax(vma))
 		return true;
-	if (vma_is_special_huge(vma))
-		return true;
-	if (vma_test(vma, VMA_IO_BIT))
+	if (vma_is_kernel_owned(vma))
 		return true;
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		return true;
 
 	return false;
@@ -4854,7 +4853,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
 		 * will try to drop it before split and then check if the folio
 		 * can be split or not. So skip the check here.
 		 */
-		if (!folio_test_private(folio) &&
+		if (!folio_has_attached_private(folio) &&
 		    folio_expected_ref_count(folio) != folio_ref_count(folio))
 			goto next;
 
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 4f6f58bf3db6c1..a1b51251103b36 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -23,6 +23,7 @@
 #include <linux/mmdebug.h>
 #include <linux/sched/signal.h>
 #include <linux/rmap.h>
+#include <linux/rcupdate.h>
 #include <linux/string_choices.h>
 #include <linux/string_helpers.h>
 #include <linux/swap.h>
@@ -38,6 +39,7 @@
 #include <linux/mm_inline.h>
 #include <linux/padata.h>
 #include <linux/pgalloc.h>
+#include <linux/vmemmap-optimization.h>
 
 #include <asm/page.h>
 #include <asm/tlb.h>
@@ -54,12 +56,20 @@
 #include "mm_init.h"
 #include <linux/page-isolation.h>
 
+#define HUGE_BOOTMEM_ZONES_VALID	BIT(0)
+
+struct huge_bootmem_page {
+	struct list_head list;
+	struct hstate *hstate;
+	unsigned long flags;
+};
+
 int hugetlb_max_hstate __read_mostly;
 unsigned int default_hstate_idx;
 struct hstate hstates[HUGE_MAX_HSTATE];
 
 __initdata nodemask_t hugetlb_bootmem_nodes;
-__initdata struct list_head huge_boot_pages[MAX_NUMNODES];
+static struct list_head huge_boot_pages[MAX_NUMNODES] __initdata;
 
 /*
  * Due to ordering constraints across the init code for various
@@ -116,6 +126,7 @@ struct mutex *hugetlb_fault_mutex_table __ro_after_init;
 
 /* Forward declaration */
 static int hugetlb_acct_memory(struct hstate *h, long delta);
+static unsigned int allowed_mems_nr(struct hstate *h);
 static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
 static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
 static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
@@ -1137,7 +1148,7 @@ static inline struct resv_map *inode_resv_map(struct inode *inode)
 
 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 {
-	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
+	VM_WARN_ON_ONCE_VMA(!vma_is_hugetlb(vma), vma);
 	if (vma->vm_flags & VM_MAYSHARE) {
 		struct address_space *mapping = vma->vm_file->f_mapping;
 		struct inode *inode = mapping->host;
@@ -1152,7 +1163,7 @@ static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 
 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 {
-	VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
+	VM_WARN_ON_ONCE_VMA(!vma_is_hugetlb(vma), vma);
 	VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
 
 	set_vma_private_data(vma, (unsigned long)map);
@@ -1160,7 +1171,7 @@ static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 
 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 {
-	VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
+	VM_WARN_ON_ONCE_VMA(!vma_is_hugetlb(vma), vma);
 	VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
 
 	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
@@ -1168,7 +1179,7 @@ static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 
 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
 {
-	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
+	VM_WARN_ON_ONCE_VMA(!vma_is_hugetlb(vma), vma);
 
 	return (get_vma_private_data(vma) & flag) != 0;
 }
@@ -1182,7 +1193,7 @@ bool __vma_private_lock(struct vm_area_struct *vma)
 
 void hugetlb_dup_vma_private(struct vm_area_struct *vma)
 {
-	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
+	VM_WARN_ON_ONCE_VMA(!vma_is_hugetlb(vma), vma);
 	/*
 	 * Clear vm_private_data
 	 * - For shared mappings this is a per-vma semaphore that may be
@@ -1387,8 +1398,11 @@ void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
 	VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);
 
 	lockdep_assert_held(&hugetlb_lock);
-	if (hstate_is_gigantic_no_runtime(h))
+	if (hstate_is_gigantic_no_runtime(h)) {
+		/* Callers must filter gigantic_no_runtime upstream. */
+		VM_WARN_ON_ONCE(1);
 		return;
+	}
 
 	list_del(&folio->lru);
 
@@ -1433,11 +1447,8 @@ void add_hugetlb_folio(struct hstate *h, struct folio *folio,
 	}
 
 	__folio_set_hugetlb(folio);
-	folio_change_private(folio, NULL);
-	/*
-	 * We have to set hugetlb_vmemmap_optimized again as above
-	 * folio_change_private(folio, NULL) cleared it.
-	 */
+	/* Clear all folio->private flags except hugetlb_vmemmap_optimized. */
+	folio->private = NULL;
 	folio_set_hugetlb_vmemmap_optimized(folio);
 
 	arch_clear_hugetlb_flags(folio);
@@ -1449,8 +1460,11 @@ static void __update_and_free_hugetlb_folio(struct hstate *h,
 {
 	bool clear_flag = folio_test_hugetlb_vmemmap_optimized(folio);
 
-	if (hstate_is_gigantic_no_runtime(h))
+	if (hstate_is_gigantic_no_runtime(h)) {
+		/* Callers must filter gigantic_no_runtime upstream. */
+		VM_WARN_ON_ONCE(1);
 		return;
+	}
 
 	/*
 	 * If we don't know which subpages are hwpoisoned, we can't free
@@ -1967,6 +1981,15 @@ int dissolve_free_hugetlb_folio(struct folio *folio)
 		struct hstate *h = folio_hstate(folio);
 		bool adjust_surplus = false;
 
+		/*
+		 * remove_hugetlb_folio()/update_and_free_hugetlb_folio() bail
+		 * for gigantic hstates without runtime support, so dissolving one
+		 * here would leave it on the free list and, on vmemmap restore
+		 * failure, the add_hugetlb_folio() rollback corrupts that list.
+		 */
+		if (hstate_is_gigantic_no_runtime(h))
+			goto out;
+
 		if (!available_huge_pages(h))
 			goto out;
 
@@ -2178,10 +2201,17 @@ struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
 
 	folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, preferred_nid,
 					       nmask);
-	if (folio)
+	if (folio) {
+		folio_set_hugetlb_restore_reserve(folio);
 		h->resv_huge_pages--;
+	}
 
 	spin_unlock_irq(&hugetlb_lock);
+
+	if (folio)
+		lruvec_stat_mod_folio(folio, NR_HUGETLB,
+				      folio_nr_pages(folio));
+
 	return folio;
 }
 
@@ -2189,24 +2219,30 @@ struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
 struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
 		nodemask_t *nmask, gfp_t gfp_mask, bool allow_alloc_fallback)
 {
-	spin_lock_irq(&hugetlb_lock);
-	if (available_huge_pages(h)) {
-		struct folio *folio;
+	struct folio *folio = NULL;
 
+	spin_lock_irq(&hugetlb_lock);
+	if (available_huge_pages(h))
 		folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask,
 						preferred_nid, nmask);
-		if (folio) {
-			spin_unlock_irq(&hugetlb_lock);
-			return folio;
-		}
-	}
 	spin_unlock_irq(&hugetlb_lock);
 
-	/* We cannot fallback to other nodes, as we could break the per-node pool. */
-	if (!allow_alloc_fallback)
-		gfp_mask |= __GFP_THISNODE;
+	if (!folio) {
+		/*
+		 * We cannot fallback to other nodes, as we could break the
+		 * per-node pool.
+		 */
+		if (!allow_alloc_fallback)
+			gfp_mask |= __GFP_THISNODE;
+
+		folio = alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid,
+						    nmask);
+	}
+
+	if (folio)
+		lruvec_stat_mod_folio(folio, NR_HUGETLB, folio_nr_pages(folio));
 
-	return alloc_migrate_hugetlb_folio(h, gfp_mask, preferred_nid, nmask);
+	return folio;
 }
 
 static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
@@ -2226,6 +2262,19 @@ static nodemask_t *policy_mbind_nodemask(gfp_t gfp)
 	return NULL;
 }
 
+/*
+ * Reservations are globally accounted, but they must also be backed by free
+ * pages on nodes allowed by the current cpuset and MPOL_BIND policy.
+ */
+static long surplus_pages_needed(struct hstate *h, long delta, long allocated)
+{
+	long global_free = (long)h->free_huge_pages + allocated;
+	long allowed_free = (long)allowed_mems_nr(h) + allocated;
+
+	return max((long)h->resv_huge_pages + delta - global_free,
+		   delta - allowed_free);
+}
+
 /*
  * Increase the hugetlb pool such that it can accommodate a reservation
  * of size 'delta'.
@@ -2248,7 +2297,7 @@ static int gather_surplus_pages(struct hstate *h, long delta)
 		alloc_nodemask = cpuset_current_mems_allowed;
 
 	lockdep_assert_held(&hugetlb_lock);
-	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
+	needed = surplus_pages_needed(h, delta, 0);
 	if (needed <= 0) {
 		h->resv_huge_pages += delta;
 		return 0;
@@ -2279,11 +2328,10 @@ static int gather_surplus_pages(struct hstate *h, long delta)
 
 	/*
 	 * After retaking hugetlb_lock, we need to recalculate 'needed'
-	 * because either resv_huge_pages or free_huge_pages may have changed.
+	 * because either resv_huge_pages or the free page counts may have changed.
 	 */
 	spin_lock_irq(&hugetlb_lock);
-	needed = (h->resv_huge_pages + delta) -
-			(h->free_huge_pages + allocated);
+	needed = surplus_pages_needed(h, delta, allocated);
 	if (needed > 0) {
 		if (alloc_ok)
 			goto retry;
@@ -2818,6 +2866,7 @@ void wait_for_freed_hugetlb_folios(void)
  * hugetlb_alloc_folio - Allocate a hugetlb folio.
  * @h: Hugetlb state control block.
  * @mpoli: Interpreted memory policy to use for allocation.
+ * @mm: Memory descriptor of the allocation target.
  * @alloc_flags: Flags controlling the allocation behavior.
  *
  * Allocates a hugetlb folio and handles cgroup charging and global hstate
@@ -2825,10 +2874,10 @@ void wait_for_freed_hugetlb_folios(void)
  *
  * Return: A pointer to the allocated folio, or an ERR_PTR on failure.
  *         -ENOSPC if cgroup charging fails or no folio is available.
- *         -ENOMEM if mem cgroup charging fails.
  */
 struct folio *hugetlb_alloc_folio(struct hstate *h,
-		struct mempolicy_interpreted *mpoli, u8 alloc_flags)
+		struct mempolicy_interpreted *mpoli, struct mm_struct *mm,
+		u8 alloc_flags)
 {
 	bool charge_hugetlb_cgroup_rsvd = alloc_flags &
 					  HUGETLB_ALLOC_CHARG_CGROUP_RSVD;
@@ -2883,7 +2932,8 @@ struct folio *hugetlb_alloc_folio(struct hstate *h,
 
 	spin_unlock_irq(&hugetlb_lock);
 
-	ret = mem_cgroup_charge_hugetlb(folio, gfp | __GFP_RETRY_MAYFAIL);
+	ret = mem_cgroup_charge_hugetlb(folio, mm,
+					gfp | __GFP_RETRY_MAYFAIL);
 	/*
 	 * Unconditionally increment NR_HUGETLB here because if
 	 * mem_cgroup_charge_hugetlb failed, freeing the page will
@@ -2892,13 +2942,17 @@ struct folio *hugetlb_alloc_folio(struct hstate *h,
 	lruvec_stat_mod_folio(folio, NR_HUGETLB, nr_pages);
 
 	if (ret == -ENOMEM) {
-		free_huge_folio(folio);
+		folio_put(folio);
 		/*
 		 * Skip uncharging hugetlb_cgroup since the charges
 		 * were committed to the folio and freeing the folio
 		 * would have cleared those up.
 		 */
-		return ERR_PTR(ret);
+		/*
+		 * Return -ENOSPC, since retrying the fault is futile:
+		 * the OOM killer is not triggered for HugeTLB.
+		 */
+		return ERR_PTR(-ENOSPC);
 	}
 
 	return folio;
@@ -3022,7 +3076,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 		.nodemask = nodemask,
 	};
 
-	folio = hugetlb_alloc_folio(h, &mpoli, alloc_flags);
+	folio = hugetlb_alloc_folio(h, &mpoli, vma->vm_mm, alloc_flags);
 
 	mpol_cond_put(mpol);
 
@@ -3061,13 +3115,24 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 	return folio;
 
 out_subpool_put:
-	/*
-	 * put page to subpool iff the quota of subpool's rsv_hpages is used
-	 * during hugepage_subpool_get_pages.
-	 */
-	if (map_chg && !gbl_chg) {
-		gbl_reserve = hugepage_subpool_put_pages(spool, 1);
-		hugetlb_acct_memory(h, -gbl_reserve);
+	if (map_chg) {
+		if (!gbl_chg) {
+			/* Full inverse when subpool_get_pages() consumed rsv_hpages. */
+			gbl_reserve = hugepage_subpool_put_pages(spool, 1);
+			hugetlb_acct_memory(h, -gbl_reserve);
+		} else if (gbl_chg > 0 && spool && spool->min_hpages == -1 &&
+			   spool->max_hpages != -1) {
+			unsigned long flags;
+
+			/*
+			 * For max-only subpools, subpool_get_pages() took only a
+			 * speculative used_hpages slot. Drop that slot directly.
+			 */
+			spin_lock_irqsave(&spool->lock, flags);
+			if (spool->used_hpages > 0)
+				spool->used_hpages--;
+			unlock_or_release_subpool(spool, flags);
+		}
 	}
 
 out_end_reservation:
@@ -3121,7 +3186,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
 	 */
 	INIT_LIST_HEAD(&m->list);
 	m->hstate = h;
-	m->flags = hugetlb_early_cma(h) ? HUGE_BOOTMEM_CMA : 0;
+	m->flags = 0;
 
 	/* CMA pages: zone-crossing is validated in hugetlb_cma_reserve(). */
 	if (!hugetlb_early_cma(h) &&
@@ -3139,6 +3204,7 @@ static bool __init alloc_bootmem_huge_page(struct hstate *h, int nid)
 	} else {
 		list_add_tail(&m->list, &huge_boot_pages[nid]);
 		m->flags |= HUGE_BOOTMEM_ZONES_VALID;
+		hugetlb_vmemmap_optimize_bootmem_page(pfn, huge_page_order(h));
 		/*
 		 * Only initialize the head struct page in memmap_init_reserved_pages,
 		 * rest of the struct pages will be initialized by the HugeTLB
@@ -3196,16 +3262,6 @@ static void __init hugetlb_folio_init_vmemmap(struct folio *folio,
 	prep_compound_head(&folio->page, huge_page_order(h));
 }
 
-static bool __init hugetlb_bootmem_page_prehvo(struct huge_bootmem_page *m)
-{
-	return m->flags & HUGE_BOOTMEM_HVO;
-}
-
-static bool __init hugetlb_bootmem_page_earlycma(struct huge_bootmem_page *m)
-{
-	return m->flags & HUGE_BOOTMEM_CMA;
-}
-
 /*
  * memblock-allocated pageblocks might not have the migrate type set
  * if marked with the 'noinit' flag. Set it to the default (MIGRATE_MOVABLE)
@@ -3275,6 +3331,7 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
 	list_for_each_entry_safe(m, tm, &huge_boot_pages[nid], list) {
 		struct page *page = virt_to_page(m);
 		struct folio *folio = (void *)page;
+		const unsigned long pfn = folio_pfn(folio);
 
 		h = m->hstate;
 		/*
@@ -3292,16 +3349,9 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
 					   HUGETLB_VMEMMAP_RESERVE_PAGES);
 		init_new_hugetlb_folio(folio);
 
-		if (hugetlb_bootmem_page_prehvo(m))
-			/*
-			 * If pre-HVO was done, just set the
-			 * flag, the HVO code will then skip
-			 * this folio.
-			 */
+		if (vmemmap_optimizable_order(pfn_to_section_compound_order(pfn)))
 			folio_set_hugetlb_vmemmap_optimized(folio);
-
-		if (hugetlb_bootmem_page_earlycma(m))
-			folio_set_hugetlb_cma(folio);
+		section_set_compound_order_range(pfn, folio_nr_pages(folio), 0);
 
 		list_add(&folio->lru, &folio_list);
 
@@ -3313,7 +3363,9 @@ static void __init gather_bootmem_prealloc_node(unsigned long nid)
 		 * For CMA pages, this is done in init_cma_pageblock
 		 * (via hugetlb_bootmem_init_migratetype), so skip it here.
 		 */
-		if (!folio_test_hugetlb_cma(folio))
+		if (hugetlb_early_cma(h))
+			folio_set_hugetlb_cma(folio);
+		else
 			adjust_managed_page_count(page, pages_per_huge_page(h));
 		cond_resched();
 	}
@@ -3342,31 +3394,6 @@ void __init hugetlb_bootmem_struct_page_init(void)
 		.max_threads	= num_node_state(N_MEMORY),
 		.numa_aware	= true,
 	};
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
-	struct zone *zone;
-
-	for_each_zone(zone) {
-		for (int i = 0; i < NR_VMEMMAP_TAILS; i++) {
-			struct page *tail, *p;
-			unsigned int order;
-
-			tail = zone->vmemmap_tails[i];
-			if (!tail)
-				continue;
-
-			order = i + VMEMMAP_TAIL_MIN_ORDER;
-			p = page_to_virt(tail);
-			/*
-			 * prep_and_add_bootmem_folios() can access pageblock
-			 * flags on bootmem HugeTLB pages, so initialize the
-			 * shared tail struct pages here before bootmem folios
-			 * start using them.
-			 */
-			for (int j = 0; j < PAGE_SIZE / sizeof(struct page); j++)
-				init_compound_tail(p + j, NULL, order, zone);
-		}
-	}
-#endif
 
 	padata_do_multithreaded(&job);
 }
@@ -3416,7 +3443,7 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
 			folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
 					&node_states[N_MEMORY], NULL);
 			if (!folio && !list_empty(&folio_list) &&
-			    hugetlb_vmemmap_optimizable_size(h)) {
+			    hugetlb_vmemmap_optimizable(h)) {
 				prep_and_add_allocated_folios(h, &folio_list);
 				INIT_LIST_HEAD(&folio_list);
 				folio = only_alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
@@ -3485,7 +3512,7 @@ static void __init hugetlb_pages_alloc_boot_node(unsigned long start, unsigned l
 	for (i = 0; i < num; ++i) {
 		struct folio *folio;
 
-		if (hugetlb_vmemmap_optimizable_size(h) &&
+		if (hugetlb_vmemmap_optimizable(h) &&
 		    (si_mem_available() == 0) && !list_empty(&folio_list)) {
 			prep_and_add_allocated_folios(h, &folio_list);
 			INIT_LIST_HEAD(&folio_list);
@@ -4012,6 +4039,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
 	struct hstate *dst;
 	long rc = 0;
 	long nr_demoted = 0;
+	long nr_persistent = 0;
 
 	lockdep_assert_held(&hugetlb_lock);
 
@@ -4024,22 +4052,60 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
 
 	for_each_node_mask_to_free(src, nr_nodes, node, nodes_allowed) {
 		LIST_HEAD(list);
+		LIST_HEAD(surplus_list);
 		struct folio *folio, *next;
+		unsigned long nr_available, nr_target;
+
+		/*
+		 * Re-check available each node batch: the previous
+		 * batch released hugetlb_lock for vmemmap restore/split,
+		 * and a new reservation could have been added in that
+		 * window, shrinking the budget.  available is global
+		 * (resv is not per-node), so 0 means no node can
+		 * contribute -- stop the whole scan.
+		 */
+		nr_available = available_huge_pages(src);
+		if (!nr_available)
+			break;
+
+		/*
+		 * Cap this batch at the current budget; expressed as a
+		 * cumulative stop point because nr_demoted is running.
+		 */
+		nr_target = nr_demoted + min_t(unsigned long,
+				nr_to_demote - nr_demoted, nr_available);
 
 		list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node], lru) {
+			bool adjust_surplus;
+
 			if (folio_test_hwpoison(folio))
 				continue;
 
-			remove_hugetlb_folio(src, folio, false);
-			list_add(&folio->lru, &list);
+			/* Surplus accounting is maintained per node, not per folio. */
+			adjust_surplus = src->surplus_huge_pages_node[node] > 0;
+			remove_hugetlb_folio(src, folio, adjust_surplus);
+			list_add(&folio->lru, adjust_surplus ? &surplus_list : &list);
+			if (!adjust_surplus)
+				nr_persistent++;
 
-			if (++nr_demoted == nr_to_demote)
+			if (++nr_demoted == nr_target)
 				break;
 		}
 
+		if (list_empty(&list) && list_empty(&surplus_list))
+			continue;
+
 		spin_unlock_irq(&hugetlb_lock);
 
-		rc = demote_free_hugetlb_folios(src, dst, &list);
+		if (!list_empty(&list))
+			rc = demote_free_hugetlb_folios(src, dst, &list);
+		if (!list_empty(&surplus_list)) {
+			long tmp_rc;
+
+			tmp_rc = demote_free_hugetlb_folios(src, dst, &surplus_list);
+			if (rc >= 0)
+				rc = tmp_rc;
+		}
 
 		spin_lock_irq(&hugetlb_lock);
 
@@ -4047,6 +4113,14 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
 			list_del(&folio->lru);
 			add_hugetlb_folio(src, folio, false);
 
+			nr_demoted--;
+			nr_persistent--;
+		}
+
+		list_for_each_entry_safe(folio, next, &surplus_list, lru) {
+			list_del(&folio->lru);
+			add_hugetlb_folio(src, folio, true);
+
 			nr_demoted--;
 		}
 
@@ -4058,7 +4132,7 @@ long demote_pool_huge_page(struct hstate *src, nodemask_t *nodes_allowed,
 	 * Not absolutely necessary, but for consistency update max_huge_pages
 	 * based on pool changes for the demoted page.
 	 */
-	src->max_huge_pages -= nr_demoted;
+	src->max_huge_pages -= nr_persistent;
 	dst->max_huge_pages += nr_demoted << (huge_page_order(src) - huge_page_order(dst));
 
 	if (rc < 0)
@@ -4268,9 +4342,9 @@ static __init void hugetlb_parse_params(void)
 
 /*
  * hugepages command line processing
- * hugepages normally follows a valid hugepagsz or default_hugepagsz
- * specification.  If not, ignore the hugepages value.  hugepages can also
- * be the first huge page command line  option in which case it implicitly
+ * hugepages normally follows a valid hugepagesz or default_hugepagesz
+ * specification. If not, ignore the hugepages value. hugepages can also
+ * be the first huge page command line option in which case it implicitly
  * specifies the number of huge pages for the default size.
  */
 static int __init hugepages_setup(char *s)
@@ -5161,18 +5235,21 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
 	hugetlb_vma_lock_write(vma);
 	i_mmap_lock_write(mapping);
 	for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
+		const unsigned long offset_to_last_entry =
+			(old_addr | last_addr_mask) - old_addr;
+
 		src_pte = hugetlb_walk(vma, old_addr, sz);
 		if (!src_pte) {
-			old_addr |= last_addr_mask;
-			new_addr |= last_addr_mask;
+			old_addr += offset_to_last_entry;
+			new_addr += offset_to_last_entry;
 			continue;
 		}
 		if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte)))
 			continue;
 
 		if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) {
-			old_addr |= last_addr_mask;
-			new_addr |= last_addr_mask;
+			old_addr += offset_to_last_entry;
+			new_addr += offset_to_last_entry;
 			continue;
 		}
 
@@ -5211,7 +5288,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	unsigned long last_addr_mask;
 
 	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
-	WARN_ON(!is_vm_hugetlb_page(vma));
+	WARN_ON(!vma_is_hugetlb(vma));
 	BUG_ON(start & ~huge_page_mask(h));
 	BUG_ON(end & ~huge_page_mask(h));
 
@@ -5331,7 +5408,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			int rc = vma_needs_reservation(h, vma, address);
 
 			if (rc < 0)
-				/* Pressumably allocate_file_region_entries failed
+				/* Presumably allocate_file_region_entries failed
 				 * to allocate a file_region struct. Clear
 				 * hugetlb_restore_reserve so that global reserve
 				 * count will not be incremented by free_huge_folio.
@@ -5361,10 +5438,12 @@ void __hugetlb_zap_begin(struct vm_area_struct *vma,
 	if (!vma->vm_file)	/* hugetlbfs_file_mmap error */
 		return;
 
-	adjust_range_if_pmd_sharing_possible(vma, start, end);
 	hugetlb_vma_lock_write(vma);
-	if (vma->vm_file)
+	if (vma->vm_file) {
 		i_mmap_lock_write(vma->vm_file->f_mapping);
+		if (hugetlbfs_pmd_sharing_active(file_inode(vma->vm_file)))
+			adjust_range_if_pmd_sharing_possible(vma, start, end);
+	}
 }
 
 void __hugetlb_zap_end(struct vm_area_struct *vma,
@@ -5403,7 +5482,10 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
 
 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
 				start, end);
-	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
+	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
+	if (hugetlbfs_pmd_sharing_active(file_inode(vma->vm_file)))
+		adjust_range_if_pmd_sharing_possible(vma, &range.start,
+						     &range.end);
 	mmu_notifier_invalidate_range_start(&range);
 	tlb_gather_mmu(&tlb, vma->vm_mm);
 
@@ -5544,7 +5626,7 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf)
 	 * In order to determine where this is a COW on a MAP_PRIVATE mapping it
 	 * is enough to check whether the old_folio is anonymous. This means that
 	 * the reserve for this address was consumed. If reserves were used, a
-	 * partial faulted mapping at the fime of fork() could consume its reserves
+	 * partial faulted mapping at the time of fork() could consume its reserves
 	 * on COW instead of the full address range.
 	 */
 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
@@ -5612,7 +5694,7 @@ static vm_fault_t hugetlb_wp(struct vm_fault *vmf)
 
 	/*
 	 * When the original hugepage is shared one, it does not have
-	 * anon_vma prepared.
+	 * an anon rmap prepared.
 	 */
 	ret = __vmf_anon_prepare(vmf);
 	if (unlikely(ret))
@@ -6810,15 +6892,6 @@ long hugetlb_reserve_pages(struct inode *inode,
 
 out_put_pages:
 	spool_resv = chg - gbl_reserve;
-	if (spool_resv) {
-		/* put sub pool's reservation back, chg - gbl_reserve */
-		gbl_resv = hugepage_subpool_put_pages(spool, spool_resv);
-		/*
-		 * subpool's reserved pages can not be put back due to race,
-		 * return to hstate.
-		 */
-		hugetlb_acct_memory(h, -gbl_resv);
-	}
 	/* Restore used_hpages for pages that failed global reservation */
 	if (gbl_reserve && spool) {
 		unsigned long flags;
@@ -6828,6 +6901,15 @@ long hugetlb_reserve_pages(struct inode *inode,
 			spool->used_hpages -= gbl_reserve;
 		unlock_or_release_subpool(spool, flags);
 	}
+	if (spool_resv) {
+		/* put sub pool's reservation back, chg - gbl_reserve */
+		gbl_resv = hugepage_subpool_put_pages(spool, spool_resv);
+		/*
+		 * subpool's reserved pages can not be put back due to race,
+		 * return to hstate.
+		 */
+		hugetlb_acct_memory(h, -gbl_resv);
+	}
 out_uncharge_cgroup:
 	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
 					    chg * pages_per_huge_page(h), h_cg);
@@ -7006,6 +7088,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 	if (pud_none(*pud)) {
 		pud_populate(mm, pud,
 				(pmd_t *)((unsigned long)spte & PAGE_MASK));
+		hugetlbfs_pmd_sharing_inc(file_inode(vma->vm_file));
 		mm_inc_nr_pmds(mm);
 	} else {
 		ptdesc_pmd_pts_dec(virt_to_ptdesc(spte));
@@ -7037,6 +7120,7 @@ static int __huge_pmd_unshare(struct mmu_gather *tlb,
 	pud_clear(pud);
 
 	tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);
+	hugetlbfs_pmd_sharing_dec(file_inode(vma->vm_file));
 
 	mm_dec_nr_pmds(mm);
 	return 1;
@@ -7302,12 +7386,36 @@ void folio_putback_hugetlb(struct folio *folio)
 	folio_put(folio);
 }
 
+static void move_hugetlb_lruvec_stat(struct folio *old_folio,
+				     struct folio *new_folio)
+{
+	struct mem_cgroup *memcg;
+	long nr_pages = folio_nr_pages(old_folio);
+	int old_nid = folio_nid(old_folio);
+	int new_nid = folio_nid(new_folio);
+
+	if (old_nid == new_nid)
+		return;
+
+	guard(rcu)();
+
+	memcg = folio_memcg(new_folio);
+	if (!memcg)
+		return;
+
+	mod_memcg_lruvec_state(mem_cgroup_lruvec(memcg, NODE_DATA(old_nid)),
+			       NR_HUGETLB, -nr_pages);
+	mod_memcg_lruvec_state(mem_cgroup_lruvec(memcg, NODE_DATA(new_nid)),
+			       NR_HUGETLB, nr_pages);
+}
+
 void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio,
 			enum migrate_reason reason)
 {
 	struct hstate *h = folio_hstate(old_folio);
 
 	hugetlb_cgroup_migrate(old_folio, new_folio);
+	move_hugetlb_lruvec_stat(old_folio, new_folio);
 	folio_set_owner_migrate_reason(new_folio, reason);
 
 	/*
@@ -7437,6 +7545,6 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
  */
 void fixup_hugetlb_reservations(struct vm_area_struct *vma)
 {
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		clear_vma_resv_huge_pages(vma);
 }
diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c
index ecb6e0b7819a0d..3fb41311e4c7dc 100644
--- a/mm/hugetlb_cgroup.c
+++ b/mm/hugetlb_cgroup.c
@@ -179,6 +179,34 @@ static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css)
 	hugetlb_cgroup_free(hugetlb_cgroup_from_css(css));
 }
 
+static void hugetlb_cgroup_move_usage(struct hugetlb_cgroup *from,
+				      struct hugetlb_cgroup *to,
+				      struct folio *from_folio,
+				      struct folio *to_folio)
+{
+	int idx = hstate_index(folio_hstate(from_folio));
+	unsigned long nr_pages = folio_nr_pages(from_folio);
+	int from_nid = folio_nid(from_folio);
+	int to_nid = folio_nid(to_folio);
+	unsigned long usage;
+
+	lockdep_assert_held(&hugetlb_lock);
+
+	if (!from || !to)
+		return;
+
+	if (from == to && from_nid == to_nid)
+		return;
+
+	usage = from->nodeinfo[from_nid]->usage[idx];
+	if (WARN_ON_ONCE(usage < nr_pages))
+		return;
+	WRITE_ONCE(from->nodeinfo[from_nid]->usage[idx], usage - nr_pages);
+
+	usage = to->nodeinfo[to_nid]->usage[idx];
+	WRITE_ONCE(to->nodeinfo[to_nid]->usage[idx], usage + nr_pages);
+}
+
 /*
  * Should be called with hugetlb_lock held.
  * Since we are holding hugetlb_lock, pages cannot get moved from
@@ -213,6 +241,8 @@ static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,
 	/* Take the pages off the local counter */
 	page_counter_cancel(counter, nr_pages);
 
+	hugetlb_cgroup_move_usage(h_cg, parent, folio, folio);
+
 	set_hugetlb_cgroup(folio, parent);
 out:
 	return;
@@ -906,6 +936,9 @@ void hugetlb_cgroup_migrate(struct folio *old_folio, struct folio *new_folio)
 	/* move the h_cg details to new cgroup */
 	set_hugetlb_cgroup(new_folio, h_cg);
 	set_hugetlb_cgroup_rsvd(new_folio, h_cg_rsvd);
+
+	hugetlb_cgroup_move_usage(h_cg, h_cg, old_folio, new_folio);
+
 	list_move(&new_folio->lru, &h->hugepage_activelist);
 	spin_unlock_irq(&hugetlb_lock);
 }
diff --git a/mm/hugetlb_sysfs.c b/mm/hugetlb_sysfs.c
index 79ece91406bfa4..326a54b4d991c9 100644
--- a/mm/hugetlb_sysfs.c
+++ b/mm/hugetlb_sysfs.c
@@ -211,15 +211,15 @@ static ssize_t demote_store(struct kobject *kobj,
 		 * Check for available pages to demote each time thorough the
 		 * loop as demote_pool_huge_page will drop hugetlb_lock.
 		 */
+		nr_available = h->free_huge_pages - h->resv_huge_pages;
 		if (nid != NUMA_NO_NODE)
-			nr_available = h->free_huge_pages_node[nid];
-		else
-			nr_available = h->free_huge_pages;
-		nr_available -= h->resv_huge_pages;
+			nr_available = min(nr_available,
+					   h->free_huge_pages_node[nid]);
 		if (!nr_available)
 			break;
 
-		rc = demote_pool_huge_page(h, n_mask, nr_demote);
+		rc = demote_pool_huge_page(h, n_mask,
+					   min(nr_demote, nr_available));
 		if (rc < 0) {
 			err = rc;
 			break;
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 917db0984143c2..0057fa2a16a891 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -15,10 +15,10 @@
 #include <linux/mmdebug.h>
 #include <linux/pagewalk.h>
 #include <linux/pgalloc.h>
+#include <linux/vmemmap-optimization.h>
 
 #include <asm/tlbflush.h>
 #include "hugetlb_vmemmap.h"
-#include "internal.h"
 
 /**
  * struct vmemmap_remap_walk - walk vmemmap page table
@@ -492,32 +492,6 @@ static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *
 	return true;
 }
 
-static struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
-{
-	const unsigned int idx = order - VMEMMAP_TAIL_MIN_ORDER;
-	struct page *tail, *p;
-	int node = zone_to_nid(zone);
-
-	tail = READ_ONCE(zone->vmemmap_tails[idx]);
-	if (likely(tail))
-		return tail;
-
-	tail = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
-	if (!tail)
-		return NULL;
-
-	p = page_to_virt(tail);
-	for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++)
-		init_compound_tail(p + i, NULL, order, zone);
-
-	if (cmpxchg(&zone->vmemmap_tails[idx], NULL, tail)) {
-		__free_page(tail);
-		tail = READ_ONCE(zone->vmemmap_tails[idx]);
-	}
-
-	return tail;
-}
-
 static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
 					    struct folio *folio,
 					    struct list_head *vmemmap_pages,
@@ -534,7 +508,7 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
 		return ret;
 
 	nid = folio_nid(folio);
-	vmemmap_tail = vmemmap_get_tail(h->order, folio_zone(folio));
+	vmemmap_tail = vmemmap_shared_tail_page(h->order, folio_zone(folio));
 	if (!vmemmap_tail)
 		return -ENOMEM;
 
@@ -705,109 +679,13 @@ void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head
 	__hugetlb_vmemmap_optimize_folios(h, folio_list, true);
 }
 
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-
-/* Return true of a bootmem allocated HugeTLB page should be pre-HVO-ed */
-static bool vmemmap_should_optimize_bootmem_page(struct huge_bootmem_page *m)
+void __init hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order)
 {
-	unsigned long section_size, psize, pmd_vmemmap_size;
-	phys_addr_t paddr;
-
-	if (!READ_ONCE(vmemmap_optimize_enabled))
-		return false;
-
-	if (!hugetlb_vmemmap_optimizable(m->hstate))
-		return false;
-
-	psize = huge_page_size(m->hstate);
-	paddr = virt_to_phys(m);
-
-	/*
-	 * Pre-HVO only works if the bootmem huge page
-	 * is aligned to the section size.
-	 */
-	section_size = (1UL << PA_SECTION_SHIFT);
-	if (!IS_ALIGNED(paddr, section_size) ||
-	    !IS_ALIGNED(psize, section_size))
-		return false;
-
-	/*
-	 * The pre-HVO code does not deal with splitting PMDS,
-	 * so the bootmem page must be aligned to the number
-	 * of base pages that can be mapped with one vmemmap PMD.
-	 */
-	pmd_vmemmap_size = (PMD_SIZE / (sizeof(struct page))) << PAGE_SHIFT;
-	if (!IS_ALIGNED(paddr, pmd_vmemmap_size) ||
-	    !IS_ALIGNED(psize, pmd_vmemmap_size))
-		return false;
-
-	return true;
-}
-
-static struct zone *pfn_to_zone(unsigned nid, unsigned long pfn)
-{
-	struct zone *zone;
-	enum zone_type zone_type;
-
-	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
-		zone = &NODE_DATA(nid)->node_zones[zone_type];
-		if (zone_spans_pfn(zone, pfn))
-			return zone;
-	}
-
-	return NULL;
-}
-
-/*
- * Initialize memmap section for a gigantic page, HVO-style.
- */
-void __init hugetlb_vmemmap_init_early(int nid)
-{
-	unsigned long psize, paddr, section_size;
-	unsigned long ns, i, pnum, pfn, nr_pages;
-	unsigned long start, end;
-	struct huge_bootmem_page *m = NULL;
-	void *map;
-
 	if (!READ_ONCE(vmemmap_optimize_enabled))
 		return;
 
-	section_size = (1UL << PA_SECTION_SHIFT);
-
-	list_for_each_entry(m, &huge_boot_pages[nid], list) {
-		struct zone *zone;
-
-		if (!vmemmap_should_optimize_bootmem_page(m))
-			continue;
-
-		nr_pages = pages_per_huge_page(m->hstate);
-		psize = nr_pages << PAGE_SHIFT;
-		paddr = virt_to_phys(m);
-		pfn = PHYS_PFN(paddr);
-		map = pfn_to_page(pfn);
-		start = (unsigned long)map;
-		end = start + hugetlb_vmemmap_size(m->hstate);
-		zone = pfn_to_zone(nid, pfn);
-
-		if (vmemmap_populate_hvo(start, end, huge_page_order(m->hstate),
-					 zone, HUGETLB_VMEMMAP_RESERVE_SIZE))
-			panic("Failed to allocate memmap for HugeTLB page\n");
-		memmap_boot_pages_add(DIV_ROUND_UP(HUGETLB_VMEMMAP_RESERVE_SIZE, PAGE_SIZE));
-
-		pnum = pfn_to_section_nr(pfn);
-		ns = psize / section_size;
-
-		for (i = 0; i < ns; i++) {
-			sparse_init_early_section(nid, map, pnum,
-					SECTION_IS_VMEMMAP_PREINIT);
-			map += section_map_size();
-			pnum++;
-		}
-
-		m->flags |= HUGE_BOOTMEM_HVO;
-	}
+	section_set_compound_order_range(pfn, 1UL << order, order);
 }
-#endif
 
 static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
 	{
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 7ac49c52457ddf..464192e32decf4 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -9,8 +9,6 @@
 #ifndef _LINUX_HUGETLB_VMEMMAP_H
 #define _LINUX_HUGETLB_VMEMMAP_H
 #include <linux/hugetlb.h>
-#include <linux/io.h>
-#include <linux/memblock.h>
 
 /*
  * Reserve one vmemmap page, all vmemmap addresses are mapped to it. See
@@ -27,10 +25,7 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h,
 void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio);
 void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);
 void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list);
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-void hugetlb_vmemmap_init_early(int nid);
-#endif
-
+void hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order);
 
 static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h)
 {
@@ -76,13 +71,13 @@ static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h,
 {
 }
 
-static inline void hugetlb_vmemmap_init_early(int nid)
+static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
 {
+	return 0;
 }
 
-static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
+static inline void hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order)
 {
-	return 0;
 }
 #endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */
 
diff --git a/mm/init-mm.c b/mm/init-mm.c
index 3e792aad762616..a1bb2c2d0284a1 100644
--- a/mm/init-mm.c
+++ b/mm/init-mm.c
@@ -39,10 +39,8 @@ struct mm_struct init_mm = {
 	.page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
 	.arg_lock	=  __SPIN_LOCK_UNLOCKED(init_mm.arg_lock),
 	.mmlist		= LIST_HEAD_INIT(init_mm.mmlist),
-#ifdef CONFIG_PER_VMA_LOCK
 	.vma_writer_wait = __RCUWAIT_INITIALIZER(init_mm.vma_writer_wait),
 	.mm_lock_seq	= SEQCNT_ZERO(init_mm.mm_lock_seq),
-#endif
 #ifdef CONFIG_SCHED_MM_CID
 	.mm_cid.lock = __RAW_SPIN_LOCK_UNLOCKED(init_mm.mm_cid.lock),
 #endif
diff --git a/mm/internal.h b/mm/internal.h
index 38b1165212c941..a665834c214cd8 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -7,6 +7,7 @@
 #ifndef __MM_INTERNAL_H
 #define __MM_INTERNAL_H
 
+#include <linux/file.h>
 #include <linux/fs.h>
 #include <linux/khugepaged.h>
 #include <linux/mm.h>
@@ -23,13 +24,9 @@
 #include "vma.h"
 
 struct folio_batch;
-struct hstate;
 
-struct huge_bootmem_page {
-	struct list_head list;
-	struct hstate *hstate;
-	unsigned long flags;
-};
+int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr,
+		void *buf, int len, unsigned int gup_flags);
 
 /* mm/workingset.c */
 bool workingset_test_recent(void *shadow, bool file, bool *workingset,
@@ -74,21 +71,19 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
 
 #define MEMCG_RECLAIM_MAY_SWAP (1 << 1)
 #define MEMCG_RECLAIM_PROACTIVE (1 << 2)
-#define MIN_SWAPPINESS 0
-#define MAX_SWAPPINESS 200
+enum {
+	MIN_SWAPPINESS = 0,
+	MAX_SWAPPINESS = 200,
 
-/* Just reclaim from anon folios in proactive memory reclaim */
-#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1)
+	/* Just reclaim from anon folios in proactive memory reclaim */
+	SWAPPINESS_ANON_ONLY = MAX_SWAPPINESS + 1,
+};
 
 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
 					   unsigned long nr_pages,
 					   gfp_t gfp_mask,
 					   unsigned int reclaim_options,
 					   int *swappiness);
-unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
-				     gfp_t gfp_mask, bool noswap,
-				     pg_data_t *pgdat,
-				     unsigned long *nr_scanned);
 
 #ifdef CONFIG_NUMA
 extern int sysctl_min_unmapped_ratio;
@@ -223,6 +218,24 @@ static inline void *folio_raw_mapping(const struct folio *folio)
 	return (void *)(mapping & ~FOLIO_MAPPING_FLAGS);
 }
 
+/*
+ * If the VMA has a close hook then close it, and since closing it might leave
+ * it in an inconsistent state which makes the use of any hooks suspect, clear
+ * them down by installing dummy empty hooks.
+ */
+static inline void vma_close(struct vm_area_struct *vma)
+{
+	if (vma->vm_ops && vma->vm_ops->close) {
+		vma->vm_ops->close(vma);
+
+		/*
+		 * The mapping is in an inconsistent state, and no further hooks
+		 * may be invoked upon it.
+		 */
+		vma->vm_ops = &vma_dummy_vm_ops;
+	}
+}
+
 /*
  * This is a file-backed mapping, and is about to be memory mapped - invoke its
  * mmap hook and safely handle error conditions. On error, VMA hooks will be
@@ -235,37 +248,35 @@ static inline void *folio_raw_mapping(const struct folio *folio)
  */
 static inline int mmap_file(struct file *file, struct vm_area_struct *vma)
 {
-	int err = vfs_mmap(file, vma);
-
-	if (likely(!err))
-		return 0;
+	const unsigned long prev_start = vma->vm_start;
+	const unsigned long prev_end = vma->vm_end;
+	const vma_flags_t prev_flags = vma->flags;
+	int err;
 
+	err = vfs_mmap(file, vma);
 	/*
-	 * OK, we tried to call the file hook for mmap(), but an error
-	 * arose. The mapping is in an inconsistent state and we must not invoke
-	 * any further hooks on it.
+	 * Either we tried to call the file hook for mmap() and an error arose
+	 * or a driver set vma->vm_ops = NULL intending there to be no VMA
+	 * operations.
+	 *
+	 * In the former case the VMA is in an inconsistent state and we mustn't
+	 * invoke any further hooks on it, in the latter case the hook actually
+	 * wanted no further hooks to be invoked, so fix both by setting dummy
+	 * VMA ops.
 	 */
-	vma->vm_ops = &vma_dummy_vm_ops;
-
-	return err;
-}
-
-/*
- * If the VMA has a close hook then close it, and since closing it might leave
- * it in an inconsistent state which makes the use of any hooks suspect, clear
- * them down by installing dummy empty hooks.
- */
-static inline void vma_close(struct vm_area_struct *vma)
-{
-	if (vma->vm_ops && vma->vm_ops->close) {
-		vma->vm_ops->close(vma);
-
-		/*
-		 * The mapping is in an inconsistent state, and no further hooks
-		 * may be invoked upon it.
-		 */
+	if (unlikely(err || !vma->vm_ops))
 		vma->vm_ops = &vma_dummy_vm_ops;
+	if (unlikely(err))
+		return err;
+
+	err = mmap_hook_validate(prev_start, prev_end, &prev_flags, vma);
+	if (unlikely(err)) {
+		vma->vm_start = prev_start;
+		vma->vm_end = prev_end;
+		vma_close(vma);
 	}
+
+	return err;
 }
 
 /* unmap_vmas is in mm/memory.c */
@@ -295,11 +306,6 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma)
 	down_write(&anon_vma->root->rwsem);
 }
 
-static inline int anon_vma_trylock_write(struct anon_vma *anon_vma)
-{
-	return down_write_trylock(&anon_vma->root->rwsem);
-}
-
 static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)
 {
 	up_write(&anon_vma->root->rwsem);
@@ -338,7 +344,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma);
 
 static inline int anon_vma_prepare(struct vm_area_struct *vma)
 {
-	if (likely(vma->anon_vma))
+	if (likely(vma_has_anon_rmap(vma)))
 		return 0;
 
 	return __anon_vma_prepare(vma);
@@ -624,8 +630,8 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
 		pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
 int truncate_inode_folio(struct address_space *mapping, struct folio *folio);
-bool truncate_inode_partial_folio(struct folio *folio, loff_t start,
-		loff_t end);
+bool truncate_inode_partial_folio(struct folio *folio, loff_t lstart,
+		loff_t lend, pgoff_t *pstart, pgoff_t *pend);
 long mapping_evict_folio(struct address_space *mapping, struct folio *folio);
 unsigned long mapping_try_invalidate(struct address_space *mapping,
 		pgoff_t start, pgoff_t end, unsigned long *nr_failed);
@@ -799,15 +805,6 @@ static inline void prep_compound_tail(struct page *tail,
 	VM_WARN_ON_ONCE(tail->private);
 }
 
-static inline void init_compound_tail(struct page *tail,
-		const struct page *head, unsigned int order, struct zone *zone)
-{
-	atomic_set(&tail->_mapcount, -1);
-	set_page_node(tail, zone_to_nid(zone));
-	set_page_zone(tail, zone_idx(zone));
-	prep_compound_tail(tail, head, order);
-}
-
 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
 
 /*
@@ -979,15 +976,7 @@ void mlock_folio(struct folio *folio);
 static inline void mlock_vma_folio(struct folio *folio,
 				struct vm_area_struct *vma)
 {
-	/*
-	 * The VM_SPECIAL check here serves two purposes.
-	 * 1) VM_IO check prevents migration from double-counting during mlock.
-	 * 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED
-	 *    is never left set on a VM_SPECIAL vma, there is an interval while
-	 *    file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may
-	 *    still be set while VM_SPECIAL bits are added: so ignore it then.
-	 */
-	if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED))
+	if (vma_test(vma, VMA_LOCKED_BIT))
 		mlock_folio(folio);
 }
 
@@ -1004,7 +993,12 @@ static inline void munlock_vma_folio(struct folio *folio,
 	 * always munlock the folio and page reclaim will correct it
 	 * if it's wrong.
 	 */
-	if (unlikely(vma->vm_flags & VM_LOCKED))
+	/*
+	 * VMA_LOCKONFAULT_BIT alone marks an mlock walk in progress, see
+	 * mlock_vma_pages_range(). An unmap racing with the walk must still
+	 * munlock folios the walk has already counted.
+	 */
+	if (unlikely(vma_test_any_mask(vma, VMA_LOCKED_MASK)))
 		munlock_folio(folio);
 }
 
@@ -1124,11 +1118,9 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
 
 static inline bool vma_supports_mlock(const struct vm_area_struct *vma)
 {
-	if (vma_test_any_mask(vma, VMA_SPECIAL_FLAGS))
-		return false;
-	if (vma_test_single_mask(vma, VMA_DROPPABLE))
+	if (!vma_is_persistent(vma))
 		return false;
-	if (vma_is_dax(vma) || is_vm_hugetlb_page(vma))
+	if (vma_is_dax(vma) || vma_is_hugetlb(vma))
 		return false;
 	return vma != get_gate_vma(current->mm);
 }
@@ -1146,7 +1138,8 @@ extern int node_reclaim_mode;
 
 extern unsigned long node_reclaim(struct pglist_data *pgdat,
 				  gfp_t gfp_mask, unsigned int order);
-extern int find_next_best_node(int node, nodemask_t *used_node_mask);
+int find_next_best_node_in(int node, nodemask_t *used_node_mask,
+		const nodemask_t *candidates);
 #else
 #define node_reclaim_mode 0
 
@@ -1155,7 +1148,8 @@ static inline unsigned long node_reclaim(struct pglist_data *pgdat,
 {
 	return 0;
 }
-static inline int find_next_best_node(int node, nodemask_t *used_node_mask)
+static inline int find_next_best_node_in(int node, nodemask_t *used_node_mask,
+		const nodemask_t *candidates)
 {
 	return NUMA_NO_NODE;
 }
@@ -1521,6 +1515,12 @@ int remap_pfn_range_prepare(struct vm_area_desc *desc);
 int remap_pfn_range_complete(struct vm_area_struct *vma,
 			     struct mmap_action *action);
 int simple_ioremap_prepare(struct vm_area_desc *desc);
+int map_kernel_pages_prepare(struct vm_area_desc *desc);
+int map_kernel_pages_complete(struct vm_area_struct *vma,
+			      struct mmap_action *action);
+int map_discontig_kernel_pages_prepare(struct vm_area_desc *desc);
+int map_discontig_kernel_pages_complete(struct vm_area_struct *vma,
+					struct mmap_action *action);
 
 static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc)
 {
@@ -1654,4 +1654,7 @@ static inline bool can_spin_trylock(void)
 	return true;
 }
 
+/* char-mem.c */
+bool file_is_dev_zero(const struct file *file);
+
 #endif	/* __MM_INTERNAL_H */
diff --git a/mm/interval_tree.c b/mm/interval_tree.c
index 7bbbf15cfbf0c7..eafde5d12ef5f0 100644
--- a/mm/interval_tree.c
+++ b/mm/interval_tree.c
@@ -64,6 +64,21 @@ void mapping_rmap_tree_remove(struct vm_area_struct *vma,
 	__mapping_rmap_tree_remove(vma, &mapping->i_mmap);
 }
 
+/**
+ * mapping_rmap_tree_update_inplace() - Update file rmap tree to reflect an
+ * in-place change in a VMA's size.
+ * @vma: The VMA whose size has changed.
+ *
+ * The file rmap lock must be held.
+ *
+ * Invalid to do so if @vma->vm_pgoff has changed.
+ */
+void mapping_rmap_tree_update_inplace(struct vm_area_struct *vma)
+{
+	/* Propagate all the way up the tree. */
+	__mapping_rmap_tree_augment.propagate(&vma->shared.rb, NULL);
+}
+
 struct vm_area_struct *
 mapping_rmap_tree_iter_first(struct address_space *mapping,
 			     pgoff_t pgoff_start, pgoff_t pgoff_last)
@@ -111,6 +126,24 @@ void anon_rmap_tree_remove(struct anon_vma_chain *avc,
 	__anon_rmap_tree_remove(avc, &anon_vma->rb_root);
 }
 
+/**
+ * anon_rmap_tree_update_inplace() - Update anon rmap tree to reflect an
+ * in-place change in the size of @avc's VMA.
+ * @avc: The anon_vma_chain whose VMA's size has changed.
+ *
+ * The anon rmap root lock must be held.
+ *
+ * Invalid to do so if the VMA's anonymous pgoff has changed.
+ */
+void anon_rmap_tree_update_inplace(struct anon_vma_chain *avc)
+{
+#ifdef CONFIG_DEBUG_VM_RB
+	avc->cached_vma_last = avc_last_pgoff(avc);
+#endif
+	/* Propagate all the way up the tree. */
+	__anon_rmap_tree_augment.propagate(&avc->rb, NULL);
+}
+
 struct anon_vma_chain *
 anon_rmap_tree_iter_first(struct anon_vma *anon_vma,
 			  pgoff_t pgoff_start, pgoff_t pgoff_last)
diff --git a/mm/kasan/common.c b/mm/kasan/common.c
index b7d05c2a6d93dd..7d21c4db68d20f 100644
--- a/mm/kasan/common.c
+++ b/mm/kasan/common.c
@@ -114,12 +114,23 @@ void kasan_unpoison_task_stack(struct task_struct *task)
 /* Unpoison the stack for the current task beyond a watermark sp value. */
 asmlinkage void kasan_unpoison_task_stack_below(const void *watermark)
 {
+	void *base;
+
+	/*
+	 * Only the generic mode needs this. In the tag-based mode the
+	 * compiler fully initializes the shadow of stack variables on
+	 * function entry, so stale tags left below the watermark are
+	 * harmless.
+	 */
+	if (!IS_ENABLED(CONFIG_KASAN_GENERIC))
+		return;
+
 	/*
 	 * Calculate the task stack base address.  Avoid using 'current'
 	 * because this function is called by early resume code which hasn't
 	 * yet set up the percpu register (%gs).
 	 */
-	void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1));
+	base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1));
 
 	kasan_unpoison(base, watermark - base, false);
 }
diff --git a/mm/kfence/core.c b/mm/kfence/core.c
index 90925c646c4c24..55d5c114eadf6c 100644
--- a/mm/kfence/core.c
+++ b/mm/kfence/core.c
@@ -156,7 +156,7 @@ atomic_t kfence_allocation_gate = ATOMIC_INIT(1);
  * allocations of the same source filling up the pool.
  *
  * Assuming a range of 15%-85% unique allocations in the pool at any point in
- * time, the below parameters provide a probablity of 0.02-0.33 for false
+ * time, the below parameters provide a probability of 0.02-0.33 for false
  * positive hits respectively:
  *
  *	P(alloc_traces) = (1 - e^(-HNUM * (alloc_traces / SIZE)) ^ HNUM
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 75639298efc271..913086eaf17ba0 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -26,44 +26,10 @@
 #include <linux/cleanup.h>
 
 #include <asm/tlb.h>
+#include "collapse.h"
 #include "internal.h"
-#include "page_alloc.h"
 #include "mm_slot.h"
-
-enum scan_result {
-	SCAN_FAIL,
-	SCAN_SUCCEED,
-	SCAN_NO_PTE_TABLE,
-	SCAN_PMD_MAPPED,
-	SCAN_EXCEED_NONE_PTE,
-	SCAN_EXCEED_SWAP_PTE,
-	SCAN_EXCEED_SHARED_PTE,
-	SCAN_PTE_NON_PRESENT,
-	SCAN_PTE_UFFD,
-	SCAN_PTE_MAPPED_HUGEPAGE,
-	SCAN_LACK_REFERENCED_PAGE,
-	SCAN_PAGE_NULL,
-	SCAN_SCAN_ABORT,
-	SCAN_PAGE_COUNT,
-	SCAN_PAGE_LRU,
-	SCAN_PAGE_LOCK,
-	SCAN_PAGE_ANON,
-	SCAN_PAGE_LAZYFREE,
-	SCAN_PAGE_COMPOUND,
-	SCAN_ANY_PROCESS,
-	SCAN_VMA_NULL,
-	SCAN_VMA_CHECK,
-	SCAN_ADDRESS_RANGE,
-	SCAN_DEL_PAGE_LRU,
-	SCAN_ALLOC_HUGE_PAGE_FAIL,
-	SCAN_CGROUP_CHARGE_FAIL,
-	SCAN_TRUNCATED,
-	SCAN_PAGE_HAS_PRIVATE,
-	SCAN_STORE_FAILED,
-	SCAN_COPY_MC,
-	SCAN_PAGE_FILLED,
-	SCAN_PAGE_DIRTY_OR_WRITEBACK,
-};
+#include "page_alloc.h"
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/huge_memory.h>
@@ -91,7 +57,6 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
  *
  * Note that these are only respected if collapse was initiated by khugepaged.
  */
-#define KHUGEPAGED_MAX_PTES_LIMIT (HPAGE_PMD_NR - 1)
 unsigned int khugepaged_max_ptes_none __read_mostly;
 static unsigned int khugepaged_max_ptes_swap __read_mostly;
 static unsigned int khugepaged_max_ptes_shared __read_mostly;
@@ -101,24 +66,6 @@ static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
 
 static struct kmem_cache *mm_slot_cache __ro_after_init;
 
-#define KHUGEPAGED_MIN_MTHP_ORDER	2
-
-struct collapse_control {
-	bool is_khugepaged;
-
-	/* Num pages scanned per node */
-	u32 node_load[MAX_NUMNODES];
-
-	/* Num pages scanned (see khugepaged_pages_to_scan) */
-	unsigned int progress;
-
-	/* nodemask for allocation fallback */
-	nodemask_t alloc_nmask;
-
-	/* Each bit represents a single occupied (!none/zero) page. */
-	DECLARE_BITMAP(mthp_present_ptes, MAX_PTRS_PER_PTE);
-};
-
 /**
  * struct khugepaged_scan - cursor for scanning
  * @mm_head: the head of the mm list to scan
@@ -267,7 +214,7 @@ static ssize_t max_ptes_none_store(struct kobject *kobj,
 	unsigned long max_ptes_none;
 
 	err = kstrtoul(buf, 10, &max_ptes_none);
-	if (err || max_ptes_none > KHUGEPAGED_MAX_PTES_LIMIT)
+	if (err || max_ptes_none > COLLAPSE_MAX_PTES_LIMIT)
 		return -EINVAL;
 
 	khugepaged_max_ptes_none = max_ptes_none;
@@ -292,7 +239,7 @@ static ssize_t max_ptes_swap_store(struct kobject *kobj,
 	unsigned long max_ptes_swap;
 
 	err  = kstrtoul(buf, 10, &max_ptes_swap);
-	if (err || max_ptes_swap > KHUGEPAGED_MAX_PTES_LIMIT)
+	if (err || max_ptes_swap > COLLAPSE_MAX_PTES_LIMIT)
 		return -EINVAL;
 
 	khugepaged_max_ptes_swap = max_ptes_swap;
@@ -318,7 +265,7 @@ static ssize_t max_ptes_shared_store(struct kobject *kobj,
 	unsigned long max_ptes_shared;
 
 	err  = kstrtoul(buf, 10, &max_ptes_shared);
-	if (err || max_ptes_shared > KHUGEPAGED_MAX_PTES_LIMIT)
+	if (err || max_ptes_shared > COLLAPSE_MAX_PTES_LIMIT)
 		return -EINVAL;
 
 	khugepaged_max_ptes_shared = max_ptes_shared;
@@ -367,30 +314,17 @@ static bool pte_none_or_zero(pte_t pte)
 static unsigned int collapse_max_ptes_none(struct collapse_control *cc,
 		struct vm_area_struct *vma, unsigned int order)
 {
-	const unsigned int max_ptes_none = khugepaged_max_ptes_none;
+	unsigned int max_ptes_none;
 
 	if (vma && userfaultfd_armed(vma))
 		return 0;
-	/* for MADV_COLLAPSE, allow any empty/shared zeropage PTEs */
-	if (!cc->is_khugepaged)
-		return HPAGE_PMD_NR;
-	/* for PMD collapse, respect the user defined maximum */
 	if (is_pmd_order(order))
-		return max_ptes_none;
-	/*
-	 * for mTHP collapse with the sysctl value set to KHUGEPAGED_MAX_PTES_LIMIT,
-	 * scale the maximum number of PTEs to the order of the collapse.
-	 */
-	if (max_ptes_none == KHUGEPAGED_MAX_PTES_LIMIT)
+		return cc->policy.pmd.max_ptes_none;
+
+	/* Below PMD order: all but one PTE of the window, or none */
+	max_ptes_none = cc->policy.sub_pmd.max_ptes_none;
+	if (max_ptes_none == COLLAPSE_MAX_PTES_LIMIT)
 		return (1 << order) - 1;
-	/*
-	 * For mTHP collapse of values other than 0 or KHUGEPAGED_MAX_PTES_LIMIT,
-	 * emit a warning and return 0.
-	 */
-	if (max_ptes_none)
-		pr_warn_once("mTHP collapse does not support max_ptes_none"
-		     " values other than 0 or %u, defaulting to 0.\n",
-		     KHUGEPAGED_MAX_PTES_LIMIT);
 	return 0;
 }
 
@@ -406,20 +340,9 @@ static unsigned int collapse_max_ptes_none(struct collapse_control *cc,
 static unsigned int collapse_max_ptes_shared(struct collapse_control *cc,
 		unsigned int order)
 {
-	/*
-	 * For MADV_COLLAPSE, do not restrict the number of PTEs that map shared
-	 * anonymous pages.
-	 */
-	if (!cc->is_khugepaged)
-		return HPAGE_PMD_NR;
-	/*
-	 * for mTHP collapse do not allow collapsing anonymous memory pages that
-	 * are shared between processes.
-	 */
-	if (!is_pmd_order(order))
-		return 0;
-	/* for PMD collapse, respect the user defined maximum */
-	return khugepaged_max_ptes_shared;
+	if (is_pmd_order(order))
+		return cc->policy.pmd.max_ptes_shared;
+	return cc->policy.sub_pmd.max_ptes_shared;
 }
 
 /**
@@ -434,17 +357,9 @@ static unsigned int collapse_max_ptes_shared(struct collapse_control *cc,
 static unsigned int collapse_max_ptes_swap(struct collapse_control *cc,
 		unsigned int order)
 {
-	/*
-	 * For MADV_COLLAPSE, do not restrict the number PTEs entries or
-	 * pagecache entries that are non-present.
-	 */
-	if (!cc->is_khugepaged)
-		return HPAGE_PMD_NR;
-	/* for mTHP collapse do not allow any non-present PTEs or pagecache entries */
-	if (!is_pmd_order(order))
-		return 0;
-	/* for PMD collapse, respect the user defined maximum */
-	return khugepaged_max_ptes_swap;
+	if (is_pmd_order(order))
+		return cc->policy.pmd.max_ptes_swap;
+	return cc->policy.sub_pmd.max_ptes_swap;
 }
 
 int hugepage_madvise(struct vm_area_struct *vma,
@@ -476,7 +391,7 @@ int __init khugepaged_init(void)
 		return -ENOMEM;
 
 	khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
-	khugepaged_max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
+	khugepaged_max_ptes_none = COLLAPSE_MAX_PTES_LIMIT;
 	khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
 	khugepaged_max_ptes_shared = HPAGE_PMD_NR / 2;
 
@@ -561,11 +476,18 @@ void __khugepaged_enter(struct mm_struct *mm)
 		wake_up_interruptible(&khugepaged_wait);
 }
 
-/*
- * Check what orders are possible based on the vma and collapse type.
- * This is used to determine if mTHP collapse is a viable option.
+/**
+ * collapse_possible_orders - which orders a VMA may collapse to
+ * @vma: the VMA
+ * @vm_flags: its flags, passed separately where they are about to change
+ * @tva_flags: who is asking, as thp_vma_allowable_orders() spells it
+ *
+ * khugepaged may collapse anonymous memory to any enabled order; everything
+ * else collapses to PMD order only.
+ *
+ * Return: the orders as a bitmask, zero when the VMA may not collapse at all.
  */
-static unsigned long collapse_possible_orders(struct vm_area_struct *vma,
+unsigned long collapse_possible_orders(struct vm_area_struct *vma,
 		vm_flags_t vm_flags, enum tva_type tva_flags)
 {
 	unsigned long orders;
@@ -579,17 +501,11 @@ static unsigned long collapse_possible_orders(struct vm_area_struct *vma,
 	return thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders);
 }
 
-static bool collapse_possible(struct vm_area_struct *vma,
-		vm_flags_t vm_flags, enum tva_type tva_flags)
-{
-	return collapse_possible_orders(vma, vm_flags, tva_flags);
-}
-
 void khugepaged_enter_vma(struct vm_area_struct *vma,
 			  vm_flags_t vm_flags)
 {
-	if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) && hugepage_enabled()
-	    && collapse_possible(vma, vm_flags, TVA_KHUGEPAGED))
+	if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) && hugepage_enabled() &&
+	    collapse_possible_orders(vma, vm_flags, TVA_KHUGEPAGED))
 		__khugepaged_enter(vma->vm_mm);
 }
 
@@ -623,11 +539,11 @@ void __khugepaged_exit(struct mm_struct *mm)
 	}
 }
 
-static void collapse_control_init_scan(struct collapse_control *cc)
+static void collapse_scan_reset(struct collapse_control *cc)
 {
 	memset(cc->node_load, 0, sizeof(cc->node_load));
 	nodes_clear(cc->alloc_nmask);
-	bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE);
+	bitmap_zero(cc->eligible_ptes, MAX_PTRS_PER_PTE);
 }
 
 static void release_pte_folio(struct folio *folio)
@@ -731,7 +647,7 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
 		 * If the vma has the VM_DROPPABLE flag, the collapse will
 		 * preserve the lazyfree property without needing to skip.
 		 */
-		if (cc->is_khugepaged && !(vma->vm_flags & VM_DROPPABLE) &&
+		if (cc->policy.anon_skip_lazyfree && !(vma->vm_flags & VM_DROPPABLE) &&
 		    folio_test_lazyfree(folio) && !pte_dirty(pteval)) {
 			result = SCAN_PAGE_LAZYFREE;
 			goto out;
@@ -820,12 +736,12 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
 		if (folio_test_large(folio))
 			list_add_tail(&folio->lru, compound_pagelist);
 next:
-		if (cc->is_khugepaged &&
+		if (cc->policy.anon_require_referenced &&
 		    folio_pte_referenced(folio, vma, addr, pteval))
 			referenced++;
 	}
 
-	if (unlikely(cc->is_khugepaged && !referenced)) {
+	if (unlikely(cc->policy.anon_require_referenced && !referenced)) {
 		result = SCAN_LACK_REFERENCED_PAGE;
 	} else {
 		result = SCAN_SUCCEED;
@@ -991,9 +907,7 @@ static void khugepaged_alloc_sleep(void)
 	remove_wait_queue(&khugepaged_wait, &wait);
 }
 
-static struct collapse_control khugepaged_collapse_control = {
-	.is_khugepaged = true,
-};
+static struct collapse_control khugepaged_collapse_control;
 
 static bool collapse_scan_abort(int nid, struct collapse_control *cc)
 {
@@ -1029,6 +943,35 @@ static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
 	return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT;
 }
 
+/* khugepaged collapses on its own initiative, so it obeys its own settings */
+static void collapse_policy_khugepaged(struct collapse_policy *p)
+{
+	p->pmd.max_ptes_none = READ_ONCE(khugepaged_max_ptes_none);
+	p->pmd.max_ptes_swap = READ_ONCE(khugepaged_max_ptes_swap);
+	p->pmd.max_ptes_shared = READ_ONCE(khugepaged_max_ptes_shared);
+
+	/*
+	 * A sub-PMD window takes no swapped-out and no shared PTE: reading
+	 * pages back or breaking CoW is not worth it for an mTHP.  Empty PTEs
+	 * it takes all or nothing, since anything in between would let one
+	 * collapse feed the next.
+	 */
+	p->sub_pmd.max_ptes_none = p->pmd.max_ptes_none;
+	if (p->sub_pmd.max_ptes_none &&
+	    p->sub_pmd.max_ptes_none != COLLAPSE_MAX_PTES_LIMIT)
+		pr_warn_once("mTHP collapse does not support max_ptes_none values other than 0 or %u, defaulting to 0.\n",
+			     COLLAPSE_MAX_PTES_LIMIT);
+	p->sub_pmd.max_ptes_swap = 0;
+	p->sub_pmd.max_ptes_shared = 0;
+
+	p->anon_skip_lazyfree = true;
+	p->anon_require_referenced = true;
+	p->file_install_pmd = false;
+	p->file_writeback_dirty = false;
+	p->gfp = alloc_hugepage_khugepaged_gfpmask();
+	p->tva_type = TVA_KHUGEPAGED;
+}
+
 #ifdef CONFIG_NUMA
 static int collapse_find_target_node(struct collapse_control *cc)
 {
@@ -1055,19 +998,27 @@ static int collapse_find_target_node(struct collapse_control *cc)
 }
 #endif
 
-/*
- * If mmap_lock temporarily dropped, revalidate vma
- * after taking the mmap_lock again.
- * Returns enum scan_result value.
+/**
+ * collapse_vma_revalidate - look a VMA up again after mmap_lock was dropped
+ * @mm: the mm
+ * @address: an address within the PTE table being collapsed
+ * @expect_anon: the collapse started on an anonymous VMA
+ * @vmap: the VMA found, if any
+ * @cc: the control, for the policy that says who is asking
+ * @order: the order the collapse is going for
+ *
+ * Called with mmap_lock held, for reading or writing, once it has been given up
+ * and taken back.  The VMA has to span the whole PMD whatever @order is; with
+ * @expect_anon it also has to be anonymous and have an anon_vma.
+ *
+ * Return: SCAN_SUCCEED, or why a collapse of @order at @address is off.
  */
-
-static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
+enum scan_result collapse_vma_revalidate(struct mm_struct *mm, unsigned long address,
 		bool expect_anon, struct vm_area_struct **vmap,
 		struct collapse_control *cc, unsigned int order)
 {
 	struct vm_area_struct *vma;
-	enum tva_type type = cc->is_khugepaged ? TVA_KHUGEPAGED :
-				 TVA_FORCED_COLLAPSE;
+	enum tva_type type = cc->policy.tva_type;
 
 	if (unlikely(collapse_test_exit_or_disable(mm)))
 		return SCAN_ANY_PROCESS;
@@ -1094,7 +1045,7 @@ static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned l
 	 * thp_vma_allowable_orders() may return true for qualified file
 	 * vmas.
 	 */
-	if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
+	if (expect_anon && (!vma_has_anon_rmap(vma) || !vma_is_anonymous(vma)))
 		return SCAN_PAGE_ANON;
 	return SCAN_SUCCEED;
 }
@@ -1250,8 +1201,7 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
 static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm,
 		struct collapse_control *cc, unsigned int order)
 {
-	gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() :
-		     GFP_TRANSHUGE);
+	gfp_t gfp = cc->policy.gfp;
 	int node = collapse_find_target_node(cc);
 	struct folio *folio;
 
@@ -1278,14 +1228,31 @@ static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_stru
 	return SCAN_SUCCEED;
 }
 
+static pgtable_t alloc_deposit_pte_table(struct mm_struct *mm)
+{
+	/*
+	 * khugepaged is run from a kernel thread, so need to manually set the
+	 * correct memcg so the allocation gets charged correctly.
+	 */
+	struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
+	struct mem_cgroup *old_memcg = set_active_memcg(memcg);
+	pgtable_t pgtable;
+
+	pgtable = pte_alloc_one(mm);
+
+	set_active_memcg(old_memcg);
+	mem_cgroup_put(memcg);
+	return pgtable;
+}
+
 /*
  * collapse_huge_page() expects the mmap_lock to be unlocked before entering and
  * will always return with the lock unlocked, to avoid holding the mmap_lock
  * while allocating a THP, as that could trigger direct reclaim/compaction.
  * Note that the VMA must be rechecked after grabbing the mmap_lock again.
  */
-static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long start_addr,
-		int referenced, int unmapped, struct collapse_control *cc,
+static enum scan_result collapse_huge_page(struct mm_struct *mm,
+		unsigned long start_addr, struct collapse_control *cc,
 		unsigned int order)
 {
 	const unsigned long pmd_addr = start_addr & HPAGE_PMD_MASK;
@@ -1293,7 +1260,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 	LIST_HEAD(compound_pagelist);
 	pmd_t *pmd, _pmd;
 	pte_t *pte = NULL;
-	pgtable_t pgtable;
+	pgtable_t pgtable = NULL;
 	struct folio *folio;
 	spinlock_t *pmd_ptl, *pte_ptl;
 	enum scan_result result = SCAN_FAIL;
@@ -1310,8 +1277,16 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 		goto out_nolock;
 	}
 
+	if (is_pmd_order(order)) {
+		pgtable = alloc_deposit_pte_table(mm);
+		if (!pgtable) {
+			result = SCAN_ALLOC_HUGE_PAGE_FAIL;
+			goto out_nolock;
+		}
+	}
+
 	mmap_read_lock(mm);
-	result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
+	result = collapse_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
 					 &vma, cc, order);
 	if (result != SCAN_SUCCEED) {
 		mmap_read_unlock(mm);
@@ -1324,14 +1299,14 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 		goto out_nolock;
 	}
 
-	if (unmapped) {
+	if (cc->scan_unmapped) {
 		/*
 		 * __collapse_huge_page_swapin() will return with mmap_lock
 		 * released when it fails. So we jump out_nolock directly in
 		 * that case.  Continuing to collapse causes inconsistency.
 		 */
 		result = __collapse_huge_page_swapin(mm, vma, start_addr, pmd,
-						     referenced, order);
+						     cc->scan_referenced, order);
 		if (result != SCAN_SUCCEED)
 			goto out_nolock;
 	}
@@ -1340,13 +1315,13 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 	/*
 	 * Prevent all access to pagetables with the exception of
 	 * gup_fast later handled by the pmdp_collapse_flush() and the VM
-	 * handled by the anon_vma lock + folio lock.
+	 * handled by the anon rmap lock + folio lock.
 	 *
 	 * UFFDIO_MOVE is prevented to race as well thanks to the
 	 * mmap_lock.
 	 */
 	mmap_write_lock(mm);
-	result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
+	result = collapse_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
 					 &vma, cc, order);
 	if (result != SCAN_SUCCEED)
 		goto out_up_write;
@@ -1407,8 +1382,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 	}
 
 	/*
-	 * For PMD collapse all pages are isolated and locked so anon_vma
-	 * rmap can't run anymore. For mTHP collapse the PMD entry has been
+	 * For PMD collapse all pages are isolated and locked so the anon
+	 * rmap walk can't run anymore. For mTHP collapse the PMD entry has been
 	 * removed and not all pages are isolated and locked, so we must hold
 	 * the lock to prevent neighboring folios from attempting to access
 	 * this PMD until its reinstalled.
@@ -1433,8 +1408,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 	spin_lock(pmd_ptl);
 	VM_WARN_ON_ONCE(!pmd_none(*pmd));
 	if (is_pmd_order(order)) {
-		pgtable = pmd_pgtable(_pmd);
 		pgtable_trans_huge_deposit(mm, pmd, pgtable);
+		pgtable = NULL;
 		map_anon_folio_pmd_nopf(folio, pmd, vma, pmd_addr);
 	} else {
 		/*
@@ -1453,6 +1428,9 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 	}
 	spin_unlock(pmd_ptl);
 
+	if (is_pmd_order(order))
+		pte_free_defer(mm, pmd_pgtable(_pmd));
+
 	folio = NULL;
 
 	result = SCAN_SUCCEED;
@@ -1463,6 +1441,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s
 		anon_vma_unlock_write(vma->anon_vma);
 	mmap_write_unlock(mm);
 out_nolock:
+	if (pgtable)
+		pte_free(mm, pgtable);
 	if (folio)
 		folio_put(folio);
 	trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result, order);
@@ -1482,24 +1462,23 @@ static unsigned int max_order_from_offset(unsigned int offset)
  * mthp_collapse() consumes the bitmap that is generated during
  * collapse_scan_pmd() to determine what regions and mTHP orders fit best.
  *
- * Each bit in cc->mthp_present_ptes represents a single occupied (!none/zero)
- * page. We start at the PMD order and check if it is eligible for collapse;
+ * Each bit in cc->eligible_ptes marks a PTE the scan accepted as a collapse
+ * source. We start at the PMD order and check if it is eligible for collapse;
  * if not, we check the left and right halves of the PTE page table we are
  * examining at a lower order.
  *
- * For each of these, we determine how many PTE entries are occupied in the
- * range of PTE entries we propose to collapse, then we compare this to a
- * threshold number of PTE entries which would need to be occupied for a
- * collapse to be permitted at that order (accounting for max_ptes_none).
+ * For each of these, we count the eligible PTEs in the range we propose to
+ * collapse, then we compare this to the number of eligible PTEs the range
+ * would need for a collapse to be permitted at that order (accounting for
+ * max_ptes_none).
  *
  * If a collapse is permitted, we attempt to collapse the PTE range into a
  * mTHP.
  */
-static enum scan_result mthp_collapse(struct mm_struct *mm,
-		unsigned long address, int referenced, int unmapped,
-		struct collapse_control *cc, unsigned long enabled_orders)
+static enum scan_result mthp_collapse(struct mm_struct *mm, unsigned long address,
+		struct collapse_control *cc)
 {
-	unsigned int nr_occupied_ptes, nr_ptes, max_ptes_none;
+	unsigned int nr_eligible_ptes, nr_ptes, max_ptes_none;
 	enum scan_result last_result = SCAN_FAIL;
 	int collapsed = 0;
 	bool alloc_failed = false;
@@ -1510,27 +1489,26 @@ static enum scan_result mthp_collapse(struct mm_struct *mm,
 	while (offset < HPAGE_PMD_NR) {
 		nr_ptes = 1UL << order;
 
-		if (!test_bit(order, &enabled_orders))
+		if (!test_bit(order, &cc->scan_orders))
 			goto next_order;
 
 		max_ptes_none = collapse_max_ptes_none(cc, NULL, order);
-		nr_occupied_ptes = bitmap_weight_from(cc->mthp_present_ptes, offset,
+		nr_eligible_ptes = bitmap_weight_from(cc->eligible_ptes, offset,
 						      offset + nr_ptes);
 
 		/*
-		 * Swap PTEs accepted during the scan are counted in @unmapped,
-		 * not in the present-PTE bitmap. Account them for the PMD-order
-		 * candidate.
+		 * Swap PTEs accepted during the scan are counted in
+		 * cc->scan_unmapped, not in cc->eligible_ptes. Account them for
+		 * the PMD-order candidate.
 		 */
 		if (is_pmd_order(order))
-			nr_occupied_ptes += unmapped;
+			nr_eligible_ptes += cc->scan_unmapped;
 
-		if (nr_occupied_ptes >= nr_ptes - max_ptes_none) {
+		if (nr_eligible_ptes >= nr_ptes - max_ptes_none) {
 			enum scan_result ret;
 
 			collapse_address = address + offset * PAGE_SIZE;
-			ret = collapse_huge_page(mm, collapse_address, referenced,
-						 unmapped, cc, order);
+			ret = collapse_huge_page(mm, collapse_address, cc, order);
 
 			switch (ret) {
 			/* Cases where we continue to next collapse candidate */
@@ -1571,8 +1549,8 @@ static enum scan_result mthp_collapse(struct mm_struct *mm,
 		 * any smaller order enabled. When at the smallest order
 		 * we must always move to the next offset.
 		 */
-		if (order > KHUGEPAGED_MIN_MTHP_ORDER &&
-			(enabled_orders & GENMASK(order - 1, 0))) {
+		if (order > COLLAPSE_MIN_MTHP_ORDER &&
+		    (cc->scan_orders & GENMASK(order - 1, 0))) {
 			order--;
 			continue;
 		}
@@ -1597,14 +1575,14 @@ static enum scan_result mthp_collapse(struct mm_struct *mm,
 	return last_result;
 }
 
-static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
-		struct vm_area_struct *vma, unsigned long start_addr,
-		bool *lock_dropped, struct collapse_control *cc)
+static enum scan_result collapse_scan_anon_pmd(struct vm_area_struct *vma,
+		unsigned long start_addr, struct collapse_control *cc,
+		unsigned long enabled_orders)
 {
 	const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, HPAGE_PMD_ORDER);
 	const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER);
 	unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER);
-	enum tva_type tva_flags = cc->is_khugepaged ? TVA_KHUGEPAGED : TVA_FORCED_COLLAPSE;
+	struct mm_struct *mm = vma->vm_mm;
 	pmd_t *pmd;
 	pte_t *pte, *_pte, pteval;
 	int i;
@@ -1612,8 +1590,8 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 	enum scan_result result = SCAN_FAIL;
 	struct page *page = NULL;
 	struct folio *folio = NULL;
+	unsigned long failed_pfn = -1;
 	unsigned long addr;
-	unsigned long enabled_orders;
 	spinlock_t *ptl;
 	int node = NUMA_NO_NODE, unmapped = 0;
 
@@ -1625,9 +1603,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		goto out;
 	}
 
-	collapse_control_init_scan(cc);
-
-	enabled_orders = collapse_possible_orders(vma, vma->vm_flags, tva_flags);
+	collapse_scan_reset(cc);
 
 	/*
 	 * If PMD is the only enabled order, enforce max_ptes_none, otherwise
@@ -1635,7 +1611,7 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 	 * is then checked again in mthp_collapse() for each attempted order.
 	 */
 	if (enabled_orders != BIT(HPAGE_PMD_ORDER))
-		max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
+		max_ptes_none = COLLAPSE_MAX_PTES_LIMIT;
 
 	pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl);
 	if (!pte) {
@@ -1703,14 +1679,16 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		 * If the vma has the VM_DROPPABLE flag, the collapse will
 		 * preserve the lazyfree property without needing to skip.
 		 */
-		if (cc->is_khugepaged && !(vma->vm_flags & VM_DROPPABLE) &&
+		if (cc->policy.anon_skip_lazyfree && !(vma->vm_flags & VM_DROPPABLE) &&
 		    folio_test_lazyfree(folio) && !pte_dirty(pteval)) {
 			result = SCAN_PAGE_LAZYFREE;
+			failed_pfn = folio_pfn(folio);
 			goto out_unmap;
 		}
 
 		if (!folio_test_anon(folio)) {
 			result = SCAN_PAGE_ANON;
+			failed_pfn = folio_pfn(folio);
 			goto out_unmap;
 		}
 
@@ -1721,14 +1699,15 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		if (folio_maybe_mapped_shared(folio)) {
 			if (++shared > max_ptes_shared) {
 				result = SCAN_EXCEED_SHARED_PTE;
+				failed_pfn = folio_pfn(folio);
 				count_collapse_event(HPAGE_PMD_ORDER, THP_SCAN_EXCEED_SHARED_PTE,
 						     MTHP_STAT_COLLAPSE_EXCEED_SHARED);
 				goto out_unmap;
 			}
 		}
 
-		/* Set bit for occupied pages */
-		__set_bit(i, cc->mthp_present_ptes);
+		/* The scan accepted this PTE as a collapse source */
+		__set_bit(i, cc->eligible_ptes);
 		/*
 		 * Record which node the original page is from and save this
 		 * information to cc->node_load[].
@@ -1738,15 +1717,18 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		node = folio_nid(folio);
 		if (collapse_scan_abort(node, cc)) {
 			result = SCAN_SCAN_ABORT;
+			failed_pfn = folio_pfn(folio);
 			goto out_unmap;
 		}
 		cc->node_load[node]++;
 		if (!folio_test_lru(folio)) {
 			result = SCAN_PAGE_LRU;
+			failed_pfn = folio_pfn(folio);
 			goto out_unmap;
 		}
 		if (folio_test_locked(folio)) {
 			result = SCAN_PAGE_LOCK;
+			failed_pfn = folio_pfn(folio);
 			goto out_unmap;
 		}
 
@@ -1759,16 +1741,17 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		 */
 		if (folio_expected_ref_count(folio) != folio_ref_count(folio)) {
 			result = SCAN_PAGE_COUNT;
+			failed_pfn = folio_pfn(folio);
 			goto out_unmap;
 		}
 
-		if (cc->is_khugepaged &&
+		if (cc->policy.anon_require_referenced &&
 		    folio_pte_referenced(folio, vma, addr, pteval))
 			referenced++;
 	}
-	if (cc->is_khugepaged &&
-		   (!referenced ||
-		    (unmapped && referenced < HPAGE_PMD_NR / 2))) {
+	if (cc->policy.anon_require_referenced &&
+	    (!referenced ||
+	     (unmapped && referenced < HPAGE_PMD_NR / 2))) {
 		result = SCAN_LACK_REFERENCED_PAGE;
 	} else {
 		result = SCAN_SUCCEED;
@@ -1776,15 +1759,12 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 out_unmap:
 	pte_unmap_unlock(pte, ptl);
 	if (result == SCAN_SUCCEED) {
-		/* collapse_huge_page() expects the lock to be dropped before calling */
-		mmap_read_unlock(mm);
-		result = mthp_collapse(mm, start_addr, referenced,
-				       unmapped, cc, enabled_orders);
-		/* mmap_lock was released above, set lock_dropped */
-		*lock_dropped = true;
+		cc->scan_orders = enabled_orders;
+		cc->scan_referenced = referenced;
+		cc->scan_unmapped = unmapped;
 	}
 out:
-	trace_mm_khugepaged_scan_pmd(mm, folio, referenced,
+	trace_mm_khugepaged_scan_pmd(mm, failed_pfn, referenced,
 				     none_or_zero, result, unmapped);
 	return result;
 }
@@ -2099,6 +2079,9 @@ static bool file_backed_vma_is_retractable(struct vm_area_struct *vma)
 	 * Check vma->anon_vma to exclude MAP_PRIVATE mappings that
 	 * got written to. These VMAs are likely not worth removing
 	 * page tables from, as PMD-mapping is likely to be split later.
+	 *
+	 * Can't use vma_has_anon_rmap() here as the VMA may be stabilised
+	 * by the file rmap lock.
 	 */
 	if (READ_ONCE(vma->anon_vma))
 		return false;
@@ -2185,9 +2168,9 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
 
 		/*
 		 * Huge page lock is still held, so normally the page table must
-		 * remain empty; and we have already skipped anon_vma and
-		 * userfaultfd_wp() vmas.  But since the mmap_lock is not held,
-		 * it is still possible for a racing userfaultfd_ioctl() or
+		 * remain empty; and we have already skipped vmas with an anon
+		 * rmap and userfaultfd_wp() vmas.  But since the mmap_lock is not
+		 * held, it is still possible for a racing userfaultfd_ioctl() or
 		 * madvise() to have inserted ptes or markers.  Now that we hold
 		 * ptlock, repeating the retractable checks protects us from
 		 * races against the prior checks.
@@ -2245,6 +2228,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 	struct address_space *mapping = file->f_mapping;
 	struct page *dst;
 	struct folio *folio, *tmp, *new_folio;
+	unsigned long new_pfn = -1;
 	pgoff_t index = 0, end = start + HPAGE_PMD_NR;
 	LIST_HEAD(pagelist);
 	XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER);
@@ -2264,6 +2248,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 	result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER);
 	if (result != SCAN_SUCCEED)
 		goto out;
+	new_pfn = folio_pfn(new_folio);
 
 	mapping_set_update(&xas, mapping);
 
@@ -2618,11 +2603,11 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 	xas_unlock_irq(&xas);
 
 	/*
-	 * Remove pte page tables, so we can re-fault the page as huge.
-	 * If MADV_COLLAPSE, adjust result to call try_collapse_pte_mapped_thp().
+	 * Remove pte page tables, so we can re-fault the page as huge.  A
+	 * caller that wants the PMD mapped now is told to go and do that.
 	 */
 	retract_page_tables(mapping, start);
-	if (cc && !cc->is_khugepaged)
+	if (cc->policy.file_install_pmd)
 		result = SCAN_PTE_MAPPED_HUGEPAGE;
 	folio_unlock(new_folio);
 
@@ -2667,7 +2652,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 	folio_put(new_folio);
 out:
 	VM_BUG_ON(!list_empty(&pagelist));
-	trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result);
+	trace_mm_khugepaged_collapse_file(mm, new_pfn, index, addr, is_shmem, file, HPAGE_PMD_NR, result);
 	return result;
 }
 
@@ -2683,10 +2668,11 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 	int present, swap;
 	int node = NUMA_NO_NODE;
 	enum scan_result result = SCAN_SUCCEED;
+	unsigned long failed_pfn = -1;
 
 	present = 0;
 	swap = 0;
-	collapse_control_init_scan(cc);
+	collapse_scan_reset(cc);
 	rcu_read_lock();
 	xas_for_each(&xas, folio, start + HPAGE_PMD_NR - 1) {
 		if (xas_retry(&xas, folio))
@@ -2715,6 +2701,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 
 		if (is_pmd_order(folio_order(folio))) {
 			result = SCAN_PTE_MAPPED_HUGEPAGE;
+			failed_pfn = folio_pfn(folio);
 			/*
 			 * PMD-sized THP implies that we can only try
 			 * retracting the PTE table.
@@ -2726,6 +2713,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 		node = folio_nid(folio);
 		if (collapse_scan_abort(node, cc)) {
 			result = SCAN_SCAN_ABORT;
+			failed_pfn = folio_pfn(folio);
 			folio_put(folio);
 			break;
 		}
@@ -2733,12 +2721,14 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 
 		if (!folio_test_lru(folio)) {
 			result = SCAN_PAGE_LRU;
+			failed_pfn = folio_pfn(folio);
 			folio_put(folio);
 			break;
 		}
 
 		if (folio_expected_ref_count(folio) + 1 != folio_ref_count(folio)) {
 			result = SCAN_PAGE_COUNT;
+			failed_pfn = folio_pfn(folio);
 			folio_put(folio);
 			break;
 		}
@@ -2764,53 +2754,116 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 	else
 		cc->progress += HPAGE_PMD_NR;
 
-	if (result == SCAN_SUCCEED) {
-		if (present < HPAGE_PMD_NR - max_ptes_none) {
-			result = SCAN_EXCEED_NONE_PTE;
-			count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
-		} else {
-			result = collapse_file(mm, addr, file, start, cc);
-		}
+	if (result == SCAN_SUCCEED && present < HPAGE_PMD_NR - max_ptes_none) {
+		result = SCAN_EXCEED_NONE_PTE;
+		count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
 	}
 
-	trace_mm_khugepaged_scan_file(mm, folio, file, present, swap, result);
+	trace_mm_khugepaged_scan_file(mm, failed_pfn, file, present, swap,
+				      result);
 	return result;
 }
 
-/*
- * Try to collapse a single PMD starting at a PMD aligned addr, and return
- * the results.
+/**
+ * collapse_control_init - set up a control before its first scan
+ * @cc: the control the caller carries across its scans
+ *
+ * cc->policy is the caller's to fill.
  */
-static enum scan_result collapse_single_pmd(unsigned long addr,
-		struct vm_area_struct *vma, bool *lock_dropped,
-		struct collapse_control *cc)
+void collapse_control_init(struct collapse_control *cc)
+{
+	cc->progress = 0;
+	cc->scan_file = NULL;
+}
+
+/**
+ * collapse_scan_pmd - scan one PTE table for a collapse candidate
+ * @vma: the VMA the table belongs to
+ * @addr: start of the table, PMD aligned
+ * @cc: the caller's control
+ * @orders: the orders the caller allows for @vma
+ *
+ * Called with mmap_lock held for reading and returns with it still held.
+ * Almost every table it is offered has nothing to collapse, so a caller walks
+ * a whole VMA under the one lock it took to get there.
+ *
+ * Return: SCAN_SUCCEED when there is something to collapse;
+ * SCAN_PTE_MAPPED_HUGEPAGE when the page cache already holds the PMD folio and
+ * only the PTE table is left to retract.  Both are work for collapse_run_pmd(),
+ * which is handed what the scan returned.  Anything else is why there is
+ * nothing to do.
+ */
+enum scan_result collapse_scan_pmd(struct vm_area_struct *vma,
+		unsigned long addr, struct collapse_control *cc,
+		unsigned long orders)
 {
-	struct mm_struct *mm = vma->vm_mm;
-	bool triggered_wb = false;
 	enum scan_result result;
-	struct file *file;
 	pgoff_t pgoff;
 
-	mmap_assert_locked(mm);
-
-	if (vma_is_anonymous(vma)) {
-		result = collapse_scan_pmd(mm, vma, addr, lock_dropped, cc);
-		goto end;
+	mmap_assert_locked(vma->vm_mm);
+	/* Whatever the last scan found has to have been run by now */
+	if (WARN_ON_ONCE(cc->scan_file)) {
+		fput(cc->scan_file);
+		cc->scan_file = NULL;
 	}
 
-	file = get_file(vma->vm_file);
-	pgoff = linear_page_index(vma, addr);
+	if (vma_is_anonymous(vma))
+		return collapse_scan_anon_pmd(vma, addr, cc, orders);
 
-	mmap_read_unlock(mm);
-	*lock_dropped = true;
-retry:
-	result = collapse_scan_file(mm, addr, file, pgoff, cc);
+	pgoff = linear_page_index(vma, addr);
+	result = collapse_scan_file(vma->vm_mm, addr, vma->vm_file, pgoff, cc);
+	/*
+	 * SCAN_PTE_MAPPED_HUGEPAGE is work too: the page cache already holds
+	 * the PMD folio, and retracting the PTE table is the run's job.
+	 */
+	if (result != SCAN_SUCCEED && result != SCAN_PTE_MAPPED_HUGEPAGE)
+		return result;
 
 	/*
-	 * For MADV_COLLAPSE, when encountering dirty pages, try to writeback,
-	 * then retry the collapse one time.
+	 * A file collapse works on the page cache and never sees a VMA, so take
+	 * what it needs from this one while it is still here.
 	 */
-	if (!cc->is_khugepaged && result == SCAN_PAGE_DIRTY_OR_WRITEBACK &&
+	cc->scan_file = get_file(vma->vm_file);
+	cc->scan_pgoff = pgoff;
+	return result;
+}
+
+/**
+ * collapse_run_pmd - collapse the table a scan found work in
+ * @mm: the mm
+ * @addr: start of the table, as given to the scan
+ * @result: what the scan returned
+ * @cc: the control the scan ran with
+ *
+ * Called without mmap_lock and returns without it, taking what it needs in
+ * between: what it does -- allocate, isolate, copy, flush -- is slow enough
+ * that a writer would wait behind it.  The caller gives the lock up first,
+ * and with it the VMA and anything derived under it.  The run revalidates for
+ * itself rather than trusting what the scan saw.
+ *
+ * Return: what the collapse made of the table.
+ */
+enum scan_result collapse_run_pmd(struct mm_struct *mm, unsigned long addr,
+		enum scan_result result, struct collapse_control *cc)
+{
+	struct file *file = cc->scan_file;
+	bool triggered_wb = false;
+	pgoff_t pgoff;
+
+	if (!file)
+		return mthp_collapse(mm, addr, cc);
+
+	cc->scan_file = NULL;
+	pgoff = cc->scan_pgoff;
+
+	/* The scan found the PMD folio in place: nothing to collapse */
+	if (result == SCAN_PTE_MAPPED_HUGEPAGE)
+		goto put;
+retry:
+	result = collapse_file(mm, addr, file, pgoff, cc);
+
+	/* Dirty pages are worth a writeback and one more try, if asked for */
+	if (cc->policy.file_writeback_dirty && result == SCAN_PAGE_DIRTY_OR_WRITEBACK &&
 	    !triggered_wb && mapping_can_writeback(file->f_mapping)) {
 		const loff_t lstart = (loff_t)pgoff << PAGE_SHIFT;
 		const loff_t lend = lstart + HPAGE_PMD_SIZE - 1;
@@ -2819,22 +2872,24 @@ static enum scan_result collapse_single_pmd(unsigned long addr,
 		triggered_wb = true;
 		goto retry;
 	}
+put:
 	fput(file);
 
+	/*
+	 * A PMD folio is in the page cache, whether the collapse just put it
+	 * there or found it: retract the PTE table, and map the PMD if asked.
+	 */
 	if (result == SCAN_PTE_MAPPED_HUGEPAGE) {
 		mmap_read_lock(mm);
 		if (collapse_test_exit_or_disable(mm))
 			result = SCAN_ANY_PROCESS;
 		else
 			result = try_collapse_pte_mapped_thp(mm, addr,
-							     !cc->is_khugepaged);
+							cc->policy.file_install_pmd);
 		if (result == SCAN_PMD_MAPPED)
 			result = SCAN_SUCCEED;
 		mmap_read_unlock(mm);
 	}
-end:
-	if (cc->is_khugepaged && result == SCAN_SUCCEED)
-		++khugepaged_pages_collapsed;
 	return result;
 }
 
@@ -2877,14 +2932,17 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
 
 	vma_iter_init(&vmi, mm, khugepaged_scan.address);
 	for_each_vma(vmi, vma) {
-		unsigned long hstart, hend;
+		unsigned long hstart, hend, orders;
 
 		cond_resched();
 		if (unlikely(collapse_test_exit_or_disable(mm))) {
 			cc->progress++;
 			break;
 		}
-		if (!collapse_possible(vma, vma->vm_flags, TVA_KHUGEPAGED)) {
+		/* One mask for the whole VMA */
+		orders = collapse_possible_orders(vma, vma->vm_flags,
+						  TVA_KHUGEPAGED);
+		if (!orders) {
 			cc->progress++;
 			continue;
 		}
@@ -2899,7 +2957,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
 		VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
 
 		while (khugepaged_scan.address < hend) {
-			bool lock_dropped = false;
+			unsigned long addr;
 
 			cond_resched();
 			if (unlikely(collapse_test_exit_or_disable(mm)))
@@ -2909,21 +2967,30 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
 				  khugepaged_scan.address + HPAGE_PMD_SIZE >
 				  hend);
 
-			*result = collapse_single_pmd(khugepaged_scan.address,
-						      vma, &lock_dropped, cc);
+			addr = khugepaged_scan.address;
 			/* move to next address */
 			khugepaged_scan.address += HPAGE_PMD_SIZE;
-			if (lock_dropped)
-				/*
-				 * We released mmap_lock so break loop.  Note
-				 * that we drop mmap_lock before all hugepage
-				 * allocations, so if allocation fails, we are
-				 * guaranteed to break here and report the
-				 * correct result back to caller.
-				 */
-				goto breakouterloop_mmap_lock;
-			if (cc->progress >= progress_max)
-				goto breakouterloop;
+
+			*result = collapse_scan_pmd(vma, addr, cc, orders);
+			/* Nothing to do here, and the lock is still ours */
+			if (*result != SCAN_SUCCEED &&
+			    *result != SCAN_PTE_MAPPED_HUGEPAGE) {
+				if (cc->progress >= progress_max)
+					goto breakouterloop;
+				continue;
+			}
+
+			/*
+			 * A collapse takes its own locks and is slow enough
+			 * that a writer should not wait behind it, so give the
+			 * lock up.  That ends this walk: vma and the mm are
+			 * whatever the collapse leaves them.
+			 */
+			mmap_read_unlock(mm);
+			*result = collapse_run_pmd(mm, addr, *result, cc);
+			if (*result == SCAN_SUCCEED)
+				khugepaged_pages_collapsed++;
+			goto breakouterloop_mmap_lock;
 		}
 	}
 breakouterloop:
@@ -2977,7 +3044,9 @@ static void khugepaged_do_scan(struct collapse_control *cc)
 
 	lru_add_drain_all();
 
-	cc->progress = 0;
+	collapse_control_init(cc);
+	collapse_policy_khugepaged(&cc->policy);
+
 	while (true) {
 		cond_resched();
 
@@ -3146,126 +3215,3 @@ bool current_is_khugepaged(void)
 {
 	return kthread_func(current) == khugepaged;
 }
-
-static int madvise_collapse_errno(enum scan_result r)
-{
-	/*
-	 * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide
-	 * actionable feedback to caller, so they may take an appropriate
-	 * fallback measure depending on the nature of the failure.
-	 */
-	switch (r) {
-	case SCAN_ALLOC_HUGE_PAGE_FAIL:
-		return -ENOMEM;
-	case SCAN_CGROUP_CHARGE_FAIL:
-	case SCAN_EXCEED_NONE_PTE:
-		return -EBUSY;
-	/* Resource temporary unavailable - trying again might succeed */
-	case SCAN_PAGE_COUNT:
-	case SCAN_PAGE_LOCK:
-	case SCAN_PAGE_LRU:
-	case SCAN_DEL_PAGE_LRU:
-	case SCAN_PAGE_FILLED:
-	case SCAN_PAGE_HAS_PRIVATE:
-	case SCAN_PAGE_DIRTY_OR_WRITEBACK:
-		return -EAGAIN;
-	/*
-	 * Other: Trying again likely not to succeed / error intrinsic to
-	 * specified memory range. khugepaged likely won't be able to collapse
-	 * either.
-	 */
-	default:
-		return -EINVAL;
-	}
-}
-
-int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
-		     unsigned long end, bool *lock_dropped)
-{
-	struct collapse_control *cc;
-	struct mm_struct *mm = vma->vm_mm;
-	unsigned long hstart, hend, addr;
-	enum scan_result last_fail = SCAN_FAIL;
-	int thps = 0;
-	bool mmap_unlocked = false;
-
-	BUG_ON(vma->vm_start > start);
-	BUG_ON(vma->vm_end < end);
-
-	if (!collapse_possible(vma, vma->vm_flags, TVA_FORCED_COLLAPSE))
-		return -EINVAL;
-
-	hstart = ALIGN(start, HPAGE_PMD_SIZE);
-	hend = ALIGN_DOWN(end, HPAGE_PMD_SIZE);
-
-	if (hstart >= hend)
-		return 0;
-
-	cc = kmalloc_obj(*cc);
-	if (!cc)
-		return -ENOMEM;
-	cc->is_khugepaged = false;
-	cc->progress = 0;
-
-	mmgrab(mm);
-	lru_add_drain_all();
-
-	for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
-		enum scan_result result = SCAN_FAIL;
-
-		if (mmap_unlocked) {
-			cond_resched();
-			mmap_read_lock(mm);
-			mmap_unlocked = false;
-			*lock_dropped = true;
-			result = hugepage_vma_revalidate(mm, addr, false, &vma,
-							 cc, HPAGE_PMD_ORDER);
-			if (result != SCAN_SUCCEED) {
-				last_fail = result;
-				goto out_nolock;
-			}
-
-			hend = min(hend, vma->vm_end & HPAGE_PMD_MASK);
-		}
-
-		result = collapse_single_pmd(addr, vma, &mmap_unlocked, cc);
-
-		switch (result) {
-		case SCAN_SUCCEED:
-		case SCAN_PMD_MAPPED:
-			++thps;
-			break;
-		/* Whitelisted set of results where continuing OK */
-		case SCAN_NO_PTE_TABLE:
-		case SCAN_PTE_NON_PRESENT:
-		case SCAN_PTE_UFFD:
-		case SCAN_LACK_REFERENCED_PAGE:
-		case SCAN_PAGE_NULL:
-		case SCAN_PAGE_COUNT:
-		case SCAN_PAGE_LOCK:
-		case SCAN_PAGE_COMPOUND:
-		case SCAN_PAGE_LRU:
-		case SCAN_DEL_PAGE_LRU:
-			last_fail = result;
-			break;
-		default:
-			last_fail = result;
-			/* Other error, exit */
-			goto out_maybelock;
-		}
-	}
-
-out_maybelock:
-	/* Caller expects us to hold mmap_lock on return */
-	if (mmap_unlocked) {
-		*lock_dropped = true;
-		mmap_read_lock(mm);
-	}
-out_nolock:
-	mmap_assert_locked(mm);
-	mmdrop(mm);
-	kfree(cc);
-
-	return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0
-			: madvise_collapse_errno(last_fail);
-}
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 8fa409a4f9fb26..20c0eb41c8730b 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -238,7 +238,7 @@ static struct task_struct *scan_thread;
 static unsigned long jiffies_min_age;
 /* consecutive scans an object must stay unreferenced before reporting */
 static unsigned int min_unref_scans =
-	IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 2 : 1;
+	IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_VERBOSE) ? 3 : 1;
 module_param(min_unref_scans, uint, 0644);
 static unsigned long jiffies_last_scan;
 /* delay between automatic memory scannings */
@@ -1855,6 +1855,52 @@ static void dedup_flush(struct xarray *dedup)
 	}
 }
 
+/*
+ * Scan the struct pages of a zone, skipping memory holes, pages that belong to
+ * another zone and pages that are not in use. Returns 1 if the scan should be
+ * stopped.
+ */
+static int scan_zone_pages(struct zone *zone)
+{
+	const unsigned int max_batch = MAX_SCAN_SIZE / sizeof(struct page);
+	unsigned long start_pfn = zone->zone_start_pfn;
+	unsigned long end_pfn = zone_end_pfn(zone);
+	struct page *first = NULL, *last = NULL;
+	unsigned int batch = 0;
+	unsigned long pfn;
+
+	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
+		struct page *page = pfn_to_online_page(pfn);
+
+		if (!(pfn & 63))
+			cond_resched_tasks_rcu_qs();
+
+		/* only scan in-use pages belonging to this zone */
+		if (page && (page_zone(page) != zone ||
+			     page_count(page) == 0))
+			page = NULL;
+
+		if (page && first && page == last + 1 &&
+		    batch < max_batch) {
+			last = page;
+			batch++;
+			continue;
+		}
+
+		if (first && scan_block(first, last + 1, NULL))
+			return 1;
+
+		first = page;
+		last = page;
+		batch = page ? 1 : 0;
+	}
+
+	if (first && scan_block(first, last + 1, NULL))
+		return 1;
+
+	return 0;
+}
+
 /*
  * Scan data sections and all the referenced memory blocks allocated via the
  * kernel's standard allocators. This function must be called with the
@@ -1928,29 +1974,7 @@ static int __kmemleak_scan(bool full)
 	 */
 	get_online_mems();
 	for_each_populated_zone(zone) {
-		unsigned long start_pfn = zone->zone_start_pfn;
-		unsigned long end_pfn = zone_end_pfn(zone);
-		unsigned long pfn;
-
-		for (pfn = start_pfn; pfn < end_pfn; pfn++) {
-			struct page *page = pfn_to_online_page(pfn);
-
-			if (!(pfn & 63))
-				cond_resched_tasks_rcu_qs();
-
-			if (!page)
-				continue;
-
-			/* only scan pages belonging to this zone */
-			if (page_zone(page) != zone)
-				continue;
-			/* only scan if page is in use */
-			if (page_count(page) == 0)
-				continue;
-			stop = scan_block(page, page + 1, NULL);
-			if (stop)
-				break;
-		}
+		stop = scan_zone_pages(zone);
 		if (stop)
 			break;
 	}
diff --git a/mm/ksm.c b/mm/ksm.c
index 49d48d1e099801..f33523a84a1a13 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -348,7 +348,7 @@ static enum ksm_advisor_type ksm_advisor;
  * Only called through the sysfs control interface:
  */
 
-/* At least scan this many pages per batch. */
+/* Initial number of pages to scan per batch. */
 static unsigned long ksm_advisor_min_pages_to_scan = 500;
 
 static void set_advisor_defaults(void)
@@ -747,9 +747,7 @@ static bool ksm_compatible(const struct file *file, vma_flags_t vma_flags)
 	if (vma_flags_test_any(&vma_flags, VMA_SHARED_BIT, VMA_MAYSHARE_BIT,
 			       VMA_HUGETLB_BIT))
 		return false;
-	if (vma_flags_test_single_mask(&vma_flags, VMA_DROPPABLE))
-		return false;
-	if (vma_flags_test_any_mask(&vma_flags, VMA_SPECIAL_FLAGS))
+	if (!vma_flags_is_persistent(&vma_flags))
 		return false;
 	if (file_is_dax(file))
 		return false;
@@ -777,7 +775,7 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,
 	if (ksm_test_exit(mm))
 		return NULL;
 	vma = vma_lookup(mm, addr);
-	if (!vma || !(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
+	if (!vma || !(vma->vm_flags & VM_MERGEABLE) || !vma_has_anon_rmap(vma))
 		return NULL;
 	return vma;
 }
@@ -795,7 +793,7 @@ static void break_cow(struct ksm_rmap_item *rmap_item)
 
 	/*
 	 * It is not an accident that whenever we want to break COW
-	 * to undo, we also need to drop a reference to the anon_vma.
+	 * to undo, we also need to drop a reference to the anon rmap.
 	 */
 	put_anon_vma(rmap_item->anon_vma);
 	/*
@@ -1115,7 +1113,8 @@ static inline void folio_set_stable_node(struct folio *folio,
 					 struct ksm_stable_node *stable_node)
 {
 	VM_WARN_ON_FOLIO(folio_test_anon(folio) && PageAnonExclusive(&folio->page), folio);
-	folio->mapping = (void *)((unsigned long)stable_node | FOLIO_MAPPING_KSM);
+	WRITE_ONCE(folio->mapping,
+		   (void *)((unsigned long)stable_node | FOLIO_MAPPING_KSM));
 }
 
 #ifdef CONFIG_SYSFS
@@ -1240,7 +1239,7 @@ static int unmerge_and_remove_all_rmap_items(void)
 			goto mm_exiting;
 
 		for_each_vma(vmi, vma) {
-			if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
+			if (!(vma->vm_flags & VM_MERGEABLE) || !vma_has_anon_rmap(vma))
 				continue;
 			err = break_ksm(vma, vma->vm_start, vma->vm_end, false);
 			if (err)
@@ -1414,7 +1413,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
 		goto out;
 	/*
 	 * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at()
-	 * without holding anon_vma lock for write.  So when looking for a
+	 * without holding the anon rmap lock for write.  So when looking for a
 	 * genuine pmde (in which to find pte), test present and !THP together.
 	 */
 	pmde = pmdp_get_lockless(pmd);
@@ -1618,7 +1617,7 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item,
 
 	/*
 	 * We can consider the VMA only while still holding the mmap lock,
-	 * so lock, so reference the anon_vma and calculate the linear
+	 * so reference the anon rmap and calculate the linear
 	 * page index early, before stable_tree_append(). If anything goes
 	 * wrong that prevents the rmap_item from being added to the
 	 * stable_tree, break_cow() will clean it up.
@@ -2690,7 +2689,7 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page)
 			continue;
 		if (ksm_scan.address < vma->vm_start)
 			ksm_scan.address = vma->vm_start;
-		if (!vma->anon_vma)
+		if (!vma_has_anon_rmap(vma))
 			ksm_scan.address = vma->vm_end;
 
 		while (ksm_scan.address < vma->vm_end) {
@@ -2881,7 +2880,7 @@ static int __ksm_del_vma(struct vm_area_struct *vma)
 	if (!(vma->vm_flags & VM_MERGEABLE))
 		return 0;
 
-	if (vma->anon_vma) {
+	if (vma_has_anon_rmap(vma)) {
 		err = break_ksm(vma, vma->vm_start, vma->vm_end, true);
 		if (err)
 			return err;
@@ -3033,7 +3032,7 @@ int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
 		if (!(*vm_flags & VM_MERGEABLE))
 			return 0;		/* just ignore the advice */
 
-		if (vma->anon_vma) {
+		if (vma_has_anon_rmap(vma)) {
 			err = break_ksm(vma, start, end, true);
 			if (err)
 				return err;
@@ -3316,7 +3315,7 @@ void folio_migrate_ksm(struct folio *newfolio, struct folio *folio)
 	stable_node = folio_stable_node(folio);
 	if (stable_node) {
 		VM_BUG_ON_FOLIO(stable_node->kpfn != folio_pfn(folio), folio);
-		stable_node->kpfn = folio_pfn(newfolio);
+		WRITE_ONCE(stable_node->kpfn, folio_pfn(newfolio));
 		/*
 		 * newfolio->mapping was set in advance; now we need smp_wmb()
 		 * to make sure that the new stable_node->kpfn is visible
diff --git a/mm/list_lru.c b/mm/list_lru.c
index 36662d02ff9631..d1d256822ccc73 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -428,7 +428,7 @@ unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
 		xa_for_each(&lru->xa, index, mlru) {
 			rcu_read_lock();
 			memcg = mem_cgroup_from_private_id(index);
-			if (!mem_cgroup_tryget(memcg)) {
+			if (!memcg || !mem_cgroup_tryget(memcg)) {
 				rcu_read_unlock();
 				continue;
 			}
@@ -587,7 +587,7 @@ static int __memcg_list_lru_alloc(struct mem_cgroup *memcg,
 	 */
 	do {
 		/*
-		 * Keep finding the farest parent that wasn't populated
+		 * Keep finding the farthest parent that wasn't populated
 		 * until found memcg itself.
 		 */
 		pos = memcg;
@@ -666,12 +666,17 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, struct shrinker *shr
 	int i;
 
 #ifdef CONFIG_MEMCG
-	if (shrinker)
+	/*
+	 * If the shrinker fell back to being non-memcg-aware (e.g. with
+	 * cgroup.memory=nokmem), its id was never assigned and holds a
+	 * stale 0. Don't let set_shrinker_bit() act on it.
+	 */
+	if (shrinker && (shrinker->flags & SHRINKER_MEMCG_AWARE))
 		lru->shrinker_id = shrinker->id;
 	else
 		lru->shrinker_id = -1;
 
-	if (mem_cgroup_kmem_disabled())
+	if (mem_cgroup_disabled() || mem_cgroup_kmem_disabled())
 		memcg_aware = false;
 #endif
 
diff --git a/mm/madvise.c b/mm/madvise.c
index eeee82cf2b3f4b..d83ce6abf8c323 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -38,6 +38,7 @@
 
 #include "internal.h"
 #include "swap.h"
+#include "collapse.h"
 
 #define __MADV_SET_ANON_VMA_NAME (-1)
 
@@ -297,11 +298,12 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior)
 	loff_t offset;
 
 #ifdef CONFIG_SWAP
-	if (!file) {
+	if (vma_is_cow_mapping(vma) && vma->anon_vma) {
 		walk_page_range_vma(vma, start, end, &swapin_walk_ops, vma);
 		lru_add_drain(); /* Push any new pages onto the LRU now */
-		return 0;
 	}
+	if (!file)
+		return 0;
 
 	if (shmem_mapping(file->f_mapping)) {
 		shmem_swapin_range(vma, start, end, file->f_mapping);
@@ -393,16 +395,18 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 			return 0;
 
 		orig_pmd = *pmd;
-		if (is_huge_zero_pmd(orig_pmd))
-			goto huge_unlock;
-
 		if (unlikely(!pmd_present(orig_pmd))) {
 			VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) &&
 					!pmd_is_device_private_entry(orig_pmd));
 			goto huge_unlock;
 		}
 
-		folio = pmd_folio(orig_pmd);
+		folio = vm_normal_folio_pmd(vma, addr, orig_pmd);
+		if (!folio)
+			goto huge_unlock;
+
+		if (folio_is_zone_device(folio))
+			goto huge_unlock;
 
 		/* Do not interfere with other mappings of this folio */
 		if (folio_maybe_mapped_shared(folio))
@@ -414,9 +418,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 		if (next - addr != HPAGE_PMD_SIZE) {
 			int err;
 
+			if (!folio_trylock(folio))
+				goto huge_unlock;
 			folio_get(folio);
 			spin_unlock(ptl);
-			folio_lock(folio);
 			err = split_folio(folio);
 			folio_unlock(folio);
 			folio_put(folio);
@@ -459,7 +464,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 restart:
 	start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
 	if (!start_pte)
-		return 0;
+		goto out;
 	flush_tlb_batched_pending(mm);
 	lazy_mmu_mode_enable();
 	for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) {
@@ -563,6 +568,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 			folio_deactivate(folio);
 	}
 
+out:
 	if (start_pte) {
 		lazy_mmu_mode_disable();
 		pte_unmap_unlock(start_pte, ptl);
@@ -877,7 +883,7 @@ bool madvise_dontneed_free_valid_vma(struct madvise_behavior *madv_behavior)
 	int behavior = madv_behavior->behavior;
 	struct madvise_behavior_range *range = &madv_behavior->range;
 
-	if (!is_vm_hugetlb_page(vma)) {
+	if (!vma_is_hugetlb(vma)) {
 		unsigned int forbidden = VM_PFNMAP;
 
 		if (behavior != MADV_DONTNEED_LOCKED)
@@ -902,6 +908,172 @@ bool madvise_dontneed_free_valid_vma(struct madvise_behavior *madv_behavior)
 	return true;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+
+/* MADV_COLLAPSE was asked for explicitly, so it is not held to those */
+static void collapse_policy_madvise(struct collapse_policy *p)
+{
+	p->pmd.max_ptes_none = HPAGE_PMD_NR;
+	p->pmd.max_ptes_swap = HPAGE_PMD_NR;
+	p->pmd.max_ptes_shared = HPAGE_PMD_NR;
+	/* Never read: MADV_COLLAPSE collapses to PMD order only */
+	p->sub_pmd = p->pmd;
+
+	p->anon_skip_lazyfree = false;
+	p->anon_require_referenced = false;
+	p->file_install_pmd = true;
+	p->file_writeback_dirty = true;
+	p->gfp = GFP_TRANSHUGE;
+	p->tva_type = TVA_FORCED_COLLAPSE;
+}
+
+static int madvise_collapse_errno(enum scan_result r)
+{
+	/*
+	 * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide
+	 * actionable feedback to caller, so they may take an appropriate
+	 * fallback measure depending on the nature of the failure.
+	 */
+	switch (r) {
+	case SCAN_ALLOC_HUGE_PAGE_FAIL:
+		return -ENOMEM;
+	case SCAN_CGROUP_CHARGE_FAIL:
+	case SCAN_EXCEED_NONE_PTE:
+		return -EBUSY;
+	/* Resource temporary unavailable - trying again might succeed */
+	case SCAN_PAGE_COUNT:
+	case SCAN_PAGE_LOCK:
+	case SCAN_PAGE_LRU:
+	case SCAN_DEL_PAGE_LRU:
+	case SCAN_PAGE_FILLED:
+	case SCAN_PAGE_HAS_PRIVATE:
+	case SCAN_PAGE_DIRTY_OR_WRITEBACK:
+		return -EAGAIN;
+	/*
+	 * Other: Trying again likely not to succeed / error intrinsic to
+	 * specified memory range. khugepaged likely won't be able to collapse
+	 * either.
+	 */
+	default:
+		return -EINVAL;
+	}
+}
+
+static int madvise_collapse(struct madvise_behavior *madv_behavior)
+{
+	struct madvise_behavior_range *range = &madv_behavior->range;
+	struct vm_area_struct *vma = madv_behavior->vma;
+	struct mm_struct *mm = madv_behavior->mm;
+	struct collapse_control *cc;
+	unsigned long hstart, hend, addr, orders;
+	enum scan_result last_fail = SCAN_FAIL;
+	int thps = 0;
+
+	BUG_ON(vma->vm_start > range->start);
+	BUG_ON(vma->vm_end < range->end);
+
+	orders = collapse_possible_orders(vma, vma->vm_flags,
+					  TVA_FORCED_COLLAPSE);
+	if (!orders)
+		return -EINVAL;
+
+	hstart = ALIGN(range->start, HPAGE_PMD_SIZE);
+	hend = ALIGN_DOWN(range->end, HPAGE_PMD_SIZE);
+
+	if (hstart >= hend)
+		return 0;
+
+	cc = kmalloc_obj(*cc);
+	if (!cc)
+		return -ENOMEM;
+	collapse_control_init(cc);
+	collapse_policy_madvise(&cc->policy);
+
+	lru_add_drain_all();
+
+	for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
+		struct vm_area_struct *found;
+		enum scan_result result;
+
+		/*
+		 * A collapse gives the lock up, so the VMA has to be found
+		 * again after one: it can shrink while nothing is held.  A scan
+		 * that finds nothing to collapse leaves the lock alone, so a
+		 * range that is already collapsed walks on without relocking.
+		 */
+		if (!vma) {
+			cond_resched();
+			mmap_read_lock(mm);
+			result = collapse_vma_revalidate(mm, addr, false, &found,
+							 cc, HPAGE_PMD_ORDER);
+			if (result != SCAN_SUCCEED) {
+				last_fail = result;
+				goto out_locked;
+			}
+			vma = found;
+			hend = min(hend, vma->vm_end & HPAGE_PMD_MASK);
+			orders = collapse_possible_orders(vma, vma->vm_flags,
+							  TVA_FORCED_COLLAPSE);
+		}
+
+		result = collapse_scan_pmd(vma, addr, cc, orders);
+		/* Nothing to do here, and the lock is still ours */
+		if (result != SCAN_SUCCEED && result != SCAN_PTE_MAPPED_HUGEPAGE)
+			goto tally;
+
+		/* The collapse takes its own locks, so give this up */
+		mmap_read_unlock(mm);
+		mark_mmap_lock_dropped(madv_behavior);
+		vma = NULL;
+
+		result = collapse_run_pmd(mm, addr, result, cc);
+tally:
+		switch (result) {
+		case SCAN_SUCCEED:
+		case SCAN_PMD_MAPPED:
+			++thps;
+			break;
+		/* Whitelisted set of results where continuing OK */
+		case SCAN_NO_PTE_TABLE:
+		case SCAN_PTE_NON_PRESENT:
+		case SCAN_PTE_UFFD:
+		case SCAN_LACK_REFERENCED_PAGE:
+		case SCAN_PAGE_NULL:
+		case SCAN_PAGE_COUNT:
+		case SCAN_PAGE_LOCK:
+		case SCAN_PAGE_COMPOUND:
+		case SCAN_PAGE_LRU:
+		case SCAN_DEL_PAGE_LRU:
+			last_fail = result;
+			break;
+		default:
+			last_fail = result;
+			/* Other error, exit */
+			goto out;
+		}
+	}
+
+out:
+	/* Caller expects us to hold mmap_lock on return */
+	if (!vma)
+		mmap_read_lock(mm);
+out_locked:
+	mmap_assert_locked(mm);
+	kfree(cc);
+
+	return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0
+			: madvise_collapse_errno(last_fail);
+}
+
+#else	/* CONFIG_TRANSPARENT_HUGEPAGE */
+
+static int madvise_collapse(struct madvise_behavior *madv_behavior)
+{
+	return -EINVAL;
+}
+
+#endif	/* CONFIG_TRANSPARENT_HUGEPAGE */
+
 static long madvise_dontneed_free(struct madvise_behavior *madv_behavior)
 {
 	struct mm_struct *mm = madv_behavior->mm;
@@ -1052,19 +1224,25 @@ static long madvise_remove(struct madvise_behavior *madv_behavior)
 	return error;
 }
 
-static bool is_valid_guard_vma(struct vm_area_struct *vma, bool allow_locked)
+static bool is_valid_guard_vma(const struct vm_area_struct *vma,
+			       bool allow_locked)
 {
-	vm_flags_t disallowed = VM_SPECIAL | VM_HUGETLB;
-
 	/*
-	 * A user could lock after setting a guard range but that's fine, as
+	 * A user could lock after setting a guard range but that's fine as
 	 * they'd not be able to fault in. The issue arises when we try to zap
 	 * existing locked VMAs. We don't want to do that.
 	 */
-	if (!allow_locked)
-		disallowed |= VM_LOCKED;
+	if (!allow_locked && vma_test(vma, VMA_LOCKED_BIT))
+		return false;
+	/*
+	 * Guard regions require a VMA whose page tables are managed solely by
+	 * the core, which is also what merging requires, so disallow any flags
+	 * that would prevent a merge.
+	 */
+	if (!vma_can_merge(vma))
+		return false;
 
-	return !(vma->vm_flags & disallowed);
+	return true;
 }
 
 static bool is_guard_pte_marker(pte_t ptent)
@@ -1147,13 +1325,13 @@ static long madvise_guard_install(struct madvise_behavior *madv_behavior)
 
 	/*
 	 * If anonymous and we are establishing page tables the VMA ought to
-	 * have an anon_vma associated with it.
+	 * have an anon rmap associated with it.
 	 *
 	 * We will hold an mmap read lock if this is necessary, this is checked
 	 * as part of the VMA lock logic.
 	 */
 	if (vma_is_anonymous(vma)) {
-		VM_WARN_ON_ONCE(!vma->anon_vma &&
+		VM_WARN_ON_ONCE(!vma_has_anon_rmap(vma) &&
 				madv_behavior->lock_mode != MADVISE_MMAP_READ_LOCK);
 
 		err = anon_vma_prepare(vma);
@@ -1293,7 +1471,7 @@ static bool is_discard(int behavior)
  * We are restricted from madvise()'ing mseal()'d VMAs only in very particular
  * circumstances - discarding of data from read-only anonymous SEALED mappings.
  *
- * This is because users cannot trivally discard data from these VMAs, and may
+ * This is because users cannot trivially discard data from these VMAs, and may
  * only do so via an appropriate madvise() call.
  */
 static bool can_madvise_modify(struct madvise_behavior *madv_behavior)
@@ -1369,8 +1547,7 @@ static int madvise_vma_behavior(struct madvise_behavior *madv_behavior)
 	case MADV_DONTNEED_LOCKED:
 		return madvise_dontneed_free(madv_behavior);
 	case MADV_COLLAPSE:
-		return madvise_collapse(vma, range->start, range->end,
-			&madv_behavior->lock_dropped);
+		return madvise_collapse(madv_behavior);
 	case MADV_GUARD_INSTALL:
 		return madvise_guard_install(madv_behavior);
 	case MADV_GUARD_REMOVE:
@@ -1391,7 +1568,7 @@ static int madvise_vma_behavior(struct madvise_behavior *madv_behavior)
 		new_flags |= VM_DONTCOPY;
 		break;
 	case MADV_DOFORK:
-		if (new_flags & VM_SPECIAL)
+		if (!vma_can_merge(vma))
 			return -EINVAL;
 		new_flags &= ~VM_DONTCOPY;
 		break;
@@ -1410,8 +1587,8 @@ static int madvise_vma_behavior(struct madvise_behavior *madv_behavior)
 		new_flags |= VM_DONTDUMP;
 		break;
 	case MADV_DODUMP:
-		if ((!is_vm_hugetlb_page(vma) && (new_flags & VM_SPECIAL)) ||
-		    (new_flags & VM_DROPPABLE))
+		/* Non-persistent memory cannot be dumped. */
+		if (!vma_is_persistent(vma))
 			return -EINVAL;
 		new_flags &= ~VM_DONTDUMP;
 		break;
@@ -1612,11 +1789,11 @@ static bool is_vma_lock_sufficient(struct vm_area_struct *vma,
 	 * anon_vma_prepare() explicitly requires an mmap lock for
 	 * serialisation, so we cannot use a VMA lock in this case.
 	 *
-	 * Note we might race with anon_vma being set, however this makes this
-	 * check overly paranoid which is safe.
+	 * Note we might race with the anon rmap being assigned, however this
+	 * makes this check overly paranoid which is safe.
 	 */
 	if (vma_is_anonymous(vma) &&
-	    prepares_anon_vma(madv_behavior->behavior) && !vma->anon_vma)
+	    prepares_anon_vma(madv_behavior->behavior) && !vma_has_anon_rmap(vma))
 		return false;
 
 	return true;
diff --git a/mm/memblock.c b/mm/memblock.c
index 021db49eb7fc65..cbb172fbf06a9a 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -113,11 +113,11 @@ unsigned long min_low_pfn;
 unsigned long max_pfn;
 unsigned long long max_possible_pfn;
 
-#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH
-/* When set to true, only allocate from MEMBLOCK_KHO_SCRATCH ranges */
-static bool kho_scratch_only;
+#ifdef CONFIG_KEXEC_HANDOVER
+/* When set to true, only allocate from MEMBLOCK_KHO_NOPRSRV ranges */
+static bool kho_noprsrv_only;
 #else
-#define kho_scratch_only false
+#define kho_noprsrv_only false
 #endif
 
 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock;
@@ -179,9 +179,9 @@ bool __init_memblock memblock_has_mirror(void)
 
 static enum memblock_flags __init_memblock choose_memblock_flags(void)
 {
-	/* skip non-scratch memory for kho early boot allocations */
-	if (kho_scratch_only)
-		return MEMBLOCK_KHO_SCRATCH;
+	/* only use KHO_NOPRSRV memory for kho early boot allocations */
+	if (kho_noprsrv_only)
+		return MEMBLOCK_KHO_NOPRSRV;
 
 	return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
 }
@@ -614,7 +614,7 @@ static int __init_memblock memblock_add_range(struct memblock_type *type,
 	bool insert = false;
 	phys_addr_t obase = base;
 	phys_addr_t end = base + memblock_cap_size(base, &size);
-	int idx, nr_new, start_rgn = -1, end_rgn;
+	int idx, nr_new, start_rgn = -1, end_rgn = 0;
 	struct memblock_region *rgn;
 
 	if (!size)
@@ -1175,33 +1175,33 @@ int __init_memblock memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t si
 }
 
 /**
- * memblock_mark_kho_scratch - Mark a memory region as MEMBLOCK_KHO_SCRATCH.
+ * memblock_mark_kho_noprsrv - Mark a memory region as MEMBLOCK_KHO_NOPRSRV.
  * @base: the base phys addr of the region
  * @size: the size of the region
  *
- * Only memory regions marked with %MEMBLOCK_KHO_SCRATCH will be considered
+ * Only memory regions marked with %MEMBLOCK_KHO_NOPRSRV will be considered
  * for allocations during early boot with kexec handover.
  *
  * Return: 0 on success, -errno on failure.
  */
-__init int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size)
+__init int memblock_mark_kho_noprsrv(phys_addr_t base, phys_addr_t size)
 {
 	return memblock_setclr_flag(&memblock.memory, base, size, 1,
-				    MEMBLOCK_KHO_SCRATCH);
+				    MEMBLOCK_KHO_NOPRSRV);
 }
 
 /**
- * memblock_clear_kho_scratch - Clear MEMBLOCK_KHO_SCRATCH flag for a
+ * memblock_clear_kho_noprsrv - Clear MEMBLOCK_KHO_NOPRSRV flag for a
  * specified region.
  * @base: the base phys addr of the region
  * @size: the size of the region
  *
  * Return: 0 on success, -errno on failure.
  */
-__init int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size)
+__init int memblock_clear_kho_noprsrv(phys_addr_t base, phys_addr_t size)
 {
 	return memblock_setclr_flag(&memblock.memory, base, size, 0,
-				    MEMBLOCK_KHO_SCRATCH);
+				    MEMBLOCK_KHO_NOPRSRV);
 }
 
 static bool should_skip_region(struct memblock_type *type,
@@ -1237,9 +1237,9 @@ static bool should_skip_region(struct memblock_type *type,
 
 	/*
 	 * In early alloc during kexec handover, we can only consider
-	 * MEMBLOCK_KHO_SCRATCH regions for the allocations
+	 * MEMBLOCK_KHO_NOPRSRV regions for the allocations
 	 */
-	if ((flags & MEMBLOCK_KHO_SCRATCH) && !memblock_is_kho_scratch(m))
+	if ((flags & MEMBLOCK_KHO_NOPRSRV) && !memblock_is_kho_noprsrv(m))
 		return true;
 
 	return false;
@@ -1794,7 +1794,7 @@ void * __init memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid)
 	if (numa_valid_node(nid) && !exact_nid) {
 		nid = NUMA_NO_NODE;
 		/*
-		 * If a previous candidate overlapped with KHO scratch, it would
+		 * If a previous candidate overlapped with KHO bootmem, it would
 		 * update start or end. Now that the search is opening to all
 		 * nodes, reset them.
 		 */
@@ -1810,12 +1810,12 @@ void * __init memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid)
 found:
 	/*
 	 * HugeTLB pages can be preserved with KHO and no preserved memory can
-	 * be in scratch. So retry if found address overlaps with scratch.
+	 * be in bootmem. So retry if found address overlaps with bootmem.
 	 *
-	 * Scratch areas are normally not very large, so this shouldn't take too
+	 * Bootmem areas are normally not very large, so this shouldn't take too
 	 * many retries.
 	 */
-	if (kho_scratch_overlap(addr, size)) {
+	if (kho_bootmem_overlap(addr, size)) {
 		if (memblock_bottom_up())
 			start = addr + size;
 		else
@@ -2465,8 +2465,6 @@ static unsigned long __init free_low_memory_core_early(void)
 	return count;
 }
 
-static int reset_managed_pages_done __initdata;
-
 static void __init reset_node_managed_pages(pg_data_t *pgdat)
 {
 	struct zone *z;
@@ -2475,17 +2473,12 @@ static void __init reset_node_managed_pages(pg_data_t *pgdat)
 		atomic_long_set(&z->managed_pages, 0);
 }
 
-void __init reset_all_zones_managed_pages(void)
+static void __init reset_all_zones_managed_pages(void)
 {
 	struct pglist_data *pgdat;
 
-	if (reset_managed_pages_done)
-		return;
-
 	for_each_online_pgdat(pgdat)
 		reset_node_managed_pages(pgdat);
-
-	reset_managed_pages_done = 1;
 }
 
 /**
@@ -2498,7 +2491,7 @@ void __init memblock_free_all(void)
 	free_unused_memmap();
 	reset_all_zones_managed_pages();
 
-	memblock_clear_kho_scratch_only();
+	memblock_clear_kho_noprsrv_only();
 	pages = free_low_memory_core_early();
 	totalram_pages_add(pages);
 }
@@ -2596,15 +2589,15 @@ int reserve_mem_release_by_name(const char *name)
 	return 1;
 }
 
-#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH
-__init void memblock_set_kho_scratch_only(void)
+#ifdef CONFIG_KEXEC_HANDOVER
+__init void memblock_set_kho_noprsrv_only(void)
 {
-	kho_scratch_only = true;
+	kho_noprsrv_only = true;
 }
 
-__init void memblock_clear_kho_scratch_only(void)
+__init void memblock_clear_kho_noprsrv_only(void)
 {
-	kho_scratch_only = false;
+	kho_noprsrv_only = false;
 }
 #endif
 
@@ -2700,15 +2693,10 @@ static int __init prepare_kho_fdt(void)
 
 static int __init reserve_mem_init(void)
 {
-	int err;
-
 	if (!kho_is_enabled() || !reserved_mem_count)
 		return 0;
 
-	err = prepare_kho_fdt();
-	if (err)
-		return err;
-	return err;
+	return prepare_kho_fdt();
 }
 late_initcall(reserve_mem_init);
 
@@ -2828,12 +2816,17 @@ static int __init reserve_mem(char *p)
 	if (*p != ':')
 		goto err_param;
 
+	if (!IS_ALIGNED(size, PAGE_SIZE) || !IS_ALIGNED(align, PAGE_SIZE)) {
+		pr_err("reserve_mem: size and align must be multiples of the page size\n");
+		return -EINVAL;
+	}
+
 	/*
 	 * memblock_phys_alloc() doesn't like a zero size align,
 	 * but it is OK for this command to have it.
 	 */
-	if (align < SMP_CACHE_BYTES)
-		align = SMP_CACHE_BYTES;
+	if (!align)
+		align = PAGE_SIZE;
 
 	name = p + 1;
 	len = strlen(name);
@@ -2860,7 +2853,7 @@ static int __init reserve_mem(char *p)
 	if (reserve_mem_kho_revive(name, size, align))
 		return 1;
 
-	/* TODO: Allocation must be outside of scratch region */
+	/* TODO: Allocation must be outside of KHO_NOPRSRV region */
 	start = memblock_phys_alloc(size, align);
 	if (!start) {
 		pr_err("reserve_mem: memblock allocation failed\n");
@@ -2885,7 +2878,8 @@ static const char * const flagname[] = {
 	[ilog2(MEMBLOCK_DRIVER_MANAGED)] = "DRV_MNG",
 	[ilog2(MEMBLOCK_RSRV_NOINIT)] = "RSV_NIT",
 	[ilog2(MEMBLOCK_RSRV_KERN)] = "RSV_KERN",
-	[ilog2(MEMBLOCK_KHO_SCRATCH)] = "KHO_SCRATCH",
+	[ilog2(MEMBLOCK_KHO_NOPRSRV)] = "KHO_NOPRSRV",
+	[ilog2(MEMBLOCK_RSRV_HUGETLB)] = "RSV_HUGETLB",
 };
 
 static int memblock_debug_show(struct seq_file *m, void *private)
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 05ef55cae4dc61..b7f28688850715 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -17,30 +17,6 @@
 #include "swap_table.h"
 #include "memcontrol-v1.h"
 
-/*
- * Cgroups above their limits are maintained in a RB-Tree, independent of
- * their hierarchy representation
- */
-
-struct mem_cgroup_tree_per_node {
-	struct rb_root rb_root;
-	struct rb_node *rb_rightmost;
-	spinlock_t lock;
-};
-
-struct mem_cgroup_tree {
-	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
-};
-
-static struct mem_cgroup_tree soft_limit_tree __read_mostly;
-
-/*
- * Maximum loops in mem_cgroup_soft_reclaim(), used for soft
- * limit reclaim to prevent infinite loops, if they ever occur.
- */
-#define	MEM_CGROUP_MAX_RECLAIM_LOOPS		100
-#define	MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS	2
-
 /* for OOM */
 struct mem_cgroup_eventfd_list {
 	struct list_head list;
@@ -106,301 +82,6 @@ static struct lockdep_map memcg_oom_lock_dep_map = {
 
 DEFINE_SPINLOCK(memcg_oom_lock);
 
-static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
-					 struct mem_cgroup_tree_per_node *mctz,
-					 unsigned long new_usage_in_excess)
-{
-	struct rb_node **p = &mctz->rb_root.rb_node;
-	struct rb_node *parent = NULL;
-	struct mem_cgroup_per_node *mz_node;
-	bool rightmost = true;
-
-	if (mz->on_tree)
-		return;
-
-	mz->usage_in_excess = new_usage_in_excess;
-	if (!mz->usage_in_excess)
-		return;
-	while (*p) {
-		parent = *p;
-		mz_node = rb_entry(parent, struct mem_cgroup_per_node,
-					tree_node);
-		if (mz->usage_in_excess < mz_node->usage_in_excess) {
-			p = &(*p)->rb_left;
-			rightmost = false;
-		} else {
-			p = &(*p)->rb_right;
-		}
-	}
-
-	if (rightmost)
-		mctz->rb_rightmost = &mz->tree_node;
-
-	rb_link_node(&mz->tree_node, parent, p);
-	rb_insert_color(&mz->tree_node, &mctz->rb_root);
-	mz->on_tree = true;
-}
-
-static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
-					 struct mem_cgroup_tree_per_node *mctz)
-{
-	if (!mz->on_tree)
-		return;
-
-	if (&mz->tree_node == mctz->rb_rightmost)
-		mctz->rb_rightmost = rb_prev(&mz->tree_node);
-
-	rb_erase(&mz->tree_node, &mctz->rb_root);
-	mz->on_tree = false;
-}
-
-static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
-				       struct mem_cgroup_tree_per_node *mctz)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mctz->lock, flags);
-	__mem_cgroup_remove_exceeded(mz, mctz);
-	spin_unlock_irqrestore(&mctz->lock, flags);
-}
-
-static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
-{
-	unsigned long nr_pages = page_counter_read(&memcg->memory);
-	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
-	unsigned long excess = 0;
-
-	if (nr_pages > soft_limit)
-		excess = nr_pages - soft_limit;
-
-	return excess;
-}
-
-static void memcg1_update_tree(struct mem_cgroup *memcg, int nid)
-{
-	unsigned long excess;
-	struct mem_cgroup_per_node *mz;
-	struct mem_cgroup_tree_per_node *mctz;
-
-	if (lru_gen_enabled()) {
-		if (soft_limit_excess(memcg))
-			lru_gen_soft_reclaim(memcg, nid);
-		return;
-	}
-
-	mctz = soft_limit_tree.rb_tree_per_node[nid];
-	if (!mctz)
-		return;
-	/*
-	 * Necessary to update all ancestors when hierarchy is used.
-	 * because their event counter is not touched.
-	 */
-	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
-		mz = memcg->nodeinfo[nid];
-		excess = soft_limit_excess(memcg);
-		/*
-		 * We have to update the tree if mz is on RB-tree or
-		 * mem is over its softlimit.
-		 */
-		if (excess || mz->on_tree) {
-			unsigned long flags;
-
-			spin_lock_irqsave(&mctz->lock, flags);
-			/* if on-tree, remove it */
-			if (mz->on_tree)
-				__mem_cgroup_remove_exceeded(mz, mctz);
-			/*
-			 * Insert again. mz->usage_in_excess will be updated.
-			 * If excess is 0, no tree ops.
-			 */
-			__mem_cgroup_insert_exceeded(mz, mctz, excess);
-			spin_unlock_irqrestore(&mctz->lock, flags);
-		}
-	}
-}
-
-void memcg1_remove_from_trees(struct mem_cgroup *memcg)
-{
-	struct mem_cgroup_tree_per_node *mctz;
-	struct mem_cgroup_per_node *mz;
-	int nid;
-
-	for_each_node(nid) {
-		mz = memcg->nodeinfo[nid];
-		mctz = soft_limit_tree.rb_tree_per_node[nid];
-		if (mctz)
-			mem_cgroup_remove_exceeded(mz, mctz);
-	}
-}
-
-static struct mem_cgroup_per_node *
-__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
-{
-	struct mem_cgroup_per_node *mz;
-
-retry:
-	mz = NULL;
-	if (!mctz->rb_rightmost)
-		goto done;		/* Nothing to reclaim from */
-
-	mz = rb_entry(mctz->rb_rightmost,
-		      struct mem_cgroup_per_node, tree_node);
-	/*
-	 * Remove the node now but someone else can add it back,
-	 * we will to add it back at the end of reclaim to its correct
-	 * position in the tree.
-	 */
-	__mem_cgroup_remove_exceeded(mz, mctz);
-	if (!soft_limit_excess(mz->memcg) ||
-	    !css_tryget(&mz->memcg->css))
-		goto retry;
-done:
-	return mz;
-}
-
-static struct mem_cgroup_per_node *
-mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
-{
-	struct mem_cgroup_per_node *mz;
-
-	spin_lock_irq(&mctz->lock);
-	mz = __mem_cgroup_largest_soft_limit_node(mctz);
-	spin_unlock_irq(&mctz->lock);
-	return mz;
-}
-
-static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
-				   pg_data_t *pgdat,
-				   gfp_t gfp_mask,
-				   unsigned long *total_scanned)
-{
-	struct mem_cgroup *victim = NULL;
-	int total = 0;
-	int loop = 0;
-	unsigned long excess;
-	unsigned long nr_scanned;
-	struct mem_cgroup_reclaim_cookie reclaim = {
-		.pgdat = pgdat,
-	};
-
-	excess = soft_limit_excess(root_memcg);
-
-	while (1) {
-		victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
-		if (!victim) {
-			loop++;
-			if (loop >= 2) {
-				/*
-				 * If we have not been able to reclaim
-				 * anything, it might because there are
-				 * no reclaimable pages under this hierarchy
-				 */
-				if (!total)
-					break;
-				/*
-				 * We want to do more targeted reclaim.
-				 * excess >> 2 is not to excessive so as to
-				 * reclaim too much, nor too less that we keep
-				 * coming back to reclaim from this cgroup
-				 */
-				if (total >= (excess >> 2) ||
-					(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
-					break;
-			}
-			continue;
-		}
-		total += mem_cgroup_shrink_node(victim, gfp_mask, false,
-					pgdat, &nr_scanned);
-		*total_scanned += nr_scanned;
-		if (!soft_limit_excess(root_memcg))
-			break;
-	}
-	mem_cgroup_iter_break(root_memcg, victim);
-	return total;
-}
-
-unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order,
-					    gfp_t gfp_mask,
-					    unsigned long *total_scanned)
-{
-	unsigned long nr_reclaimed = 0;
-	struct mem_cgroup_per_node *mz, *next_mz = NULL;
-	unsigned long reclaimed;
-	int loop = 0;
-	struct mem_cgroup_tree_per_node *mctz;
-	unsigned long excess;
-
-	if (lru_gen_enabled())
-		return 0;
-
-	if (order > 0)
-		return 0;
-
-	mctz = soft_limit_tree.rb_tree_per_node[pgdat->node_id];
-
-	/*
-	 * Do not even bother to check the largest node if the root
-	 * is empty. Do it lockless to prevent lock bouncing. Races
-	 * are acceptable as soft limit is best effort anyway.
-	 */
-	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
-		return 0;
-
-	/*
-	 * This loop can run a while, specially if mem_cgroup's continuously
-	 * keep exceeding their soft limit and putting the system under
-	 * pressure
-	 */
-	do {
-		if (next_mz)
-			mz = next_mz;
-		else
-			mz = mem_cgroup_largest_soft_limit_node(mctz);
-		if (!mz)
-			break;
-
-		reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
-						    gfp_mask, total_scanned);
-		nr_reclaimed += reclaimed;
-		spin_lock_irq(&mctz->lock);
-
-		/*
-		 * If we failed to reclaim anything from this memory cgroup
-		 * it is time to move on to the next cgroup
-		 */
-		next_mz = NULL;
-		if (!reclaimed)
-			next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
-
-		excess = soft_limit_excess(mz->memcg);
-		/*
-		 * One school of thought says that we should not add
-		 * back the node to the tree if reclaim returns 0.
-		 * But our reclaim could return 0, simply because due
-		 * to priority we are exposing a smaller subset of
-		 * memory to reclaim from. Consider this as a longer
-		 * term TODO.
-		 */
-		/* If excess == 0, no tree ops */
-		__mem_cgroup_insert_exceeded(mz, mctz, excess);
-		spin_unlock_irq(&mctz->lock);
-		css_put(&mz->memcg->css);
-		loop++;
-		/*
-		 * Could not reclaim anything and there are no more
-		 * mem cgroups to try or we seem to be looping without
-		 * reclaiming anything.
-		 */
-		if (!nr_reclaimed &&
-			(next_mz == NULL ||
-			loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
-			break;
-	} while (!nr_reclaimed);
-	if (next_mz)
-		css_put(&next_mz->memcg->css);
-	return nr_reclaimed;
-}
-
 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
 				struct cftype *cft)
 {
@@ -511,7 +192,7 @@ static void mem_cgroup_threshold(struct mem_cgroup *memcg)
 	}
 }
 
-/* Cgroup1: threshold notifications & softlimit tree updates */
+/* Cgroup1: threshold notifications */
 
 /*
  * Per memcg event counter is incremented at every pagein/pageout. With THP,
@@ -519,15 +200,9 @@ static void mem_cgroup_threshold(struct mem_cgroup *memcg)
  * to trigger some periodic events. This is straightforward and better
  * than using jiffies etc. to handle periodic memcg event.
  */
-enum mem_cgroup_events_target {
-	MEM_CGROUP_TARGET_THRESH,
-	MEM_CGROUP_TARGET_SOFTLIMIT,
-	MEM_CGROUP_NTARGETS,
-};
-
 struct memcg1_events_percpu {
 	unsigned long nr_page_events;
-	unsigned long targets[MEM_CGROUP_NTARGETS];
+	unsigned long threshold_target;
 };
 
 static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages)
@@ -544,53 +219,29 @@ static void memcg1_charge_statistics(struct mem_cgroup *memcg, int nr_pages)
 }
 
 #define THRESHOLDS_EVENTS_TARGET 128
-#define SOFTLIMIT_EVENTS_TARGET 1024
 
-static bool memcg1_event_ratelimit(struct mem_cgroup *memcg,
-				enum mem_cgroup_events_target target)
+static bool memcg1_event_ratelimit(struct mem_cgroup *memcg)
 {
 	unsigned long val, next;
 
 	val = __this_cpu_read(memcg->events_percpu->nr_page_events);
-	next = __this_cpu_read(memcg->events_percpu->targets[target]);
+	next = __this_cpu_read(memcg->events_percpu->threshold_target);
 	/* from time_after() in jiffies.h */
 	if ((long)(next - val) < 0) {
-		switch (target) {
-		case MEM_CGROUP_TARGET_THRESH:
-			next = val + THRESHOLDS_EVENTS_TARGET;
-			break;
-		case MEM_CGROUP_TARGET_SOFTLIMIT:
-			next = val + SOFTLIMIT_EVENTS_TARGET;
-			break;
-		default:
-			break;
-		}
-		__this_cpu_write(memcg->events_percpu->targets[target], next);
+		__this_cpu_write(memcg->events_percpu->threshold_target,
+				 val + THRESHOLDS_EVENTS_TARGET);
 		return true;
 	}
 	return false;
 }
 
-/*
- * Check events in order.
- *
- */
-static void memcg1_check_events(struct mem_cgroup *memcg, int nid)
+static void memcg1_check_events(struct mem_cgroup *memcg)
 {
 	if (IS_ENABLED(CONFIG_PREEMPT_RT))
 		return;
 
-	/* threshold event is triggered in finer grain than soft limit */
-	if (unlikely(memcg1_event_ratelimit(memcg,
-						MEM_CGROUP_TARGET_THRESH))) {
-		bool do_softlimit;
-
-		do_softlimit = memcg1_event_ratelimit(memcg,
-						MEM_CGROUP_TARGET_SOFTLIMIT);
+	if (unlikely(memcg1_event_ratelimit(memcg)))
 		mem_cgroup_threshold(memcg);
-		if (unlikely(do_softlimit))
-			memcg1_update_tree(memcg, nid);
-	}
 }
 
 void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg)
@@ -599,7 +250,7 @@ void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg)
 
 	local_irq_save(flags);
 	memcg1_charge_statistics(memcg, folio_nr_pages(folio));
-	memcg1_check_events(memcg, folio_nid(folio));
+	memcg1_check_events(memcg);
 	local_irq_restore(flags);
 }
 
@@ -617,9 +268,10 @@ void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg)
  */
 void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci)
 {
-	struct mem_cgroup *memcg, *swap_memcg;
+	struct mem_cgroup *memcg;
 	struct obj_cgroup *objcg;
 	unsigned int nr_entries;
+	unsigned short private_id;
 
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
@@ -642,25 +294,23 @@ void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci)
 	/*
 	 * In case the memcg owning these pages has been offlined and doesn't
 	 * have an ID allocated to it anymore, charge the closest online
-	 * ancestor for the swap instead and transfer the memory+swap charge.
+	 * ancestor for the swap instead and cancel the memory+swap charge
+	 * if the ID refers to the root memcg.
 	 */
 	nr_entries = folio_nr_pages(folio);
-	swap_memcg = mem_cgroup_private_id_get_online(memcg, nr_entries);
-	mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
+	private_id = mem_cgroup_private_id_get(memcg, nr_entries);
+	mod_memcg_state(memcg, MEMCG_SWAP, nr_entries);
 
-	__swap_cgroup_set(ci, swp_cluster_offset(folio->swap), nr_entries,
-			  mem_cgroup_private_id(swap_memcg));
+	__swap_cgroup_set(ci, swp_cluster_offset(folio->swap), nr_entries, private_id);
 
 	folio_unqueue_deferred_split(folio);
 	folio->memcg_data = 0;
 
-	if (!obj_cgroup_is_root(objcg))
+	if (!obj_cgroup_is_root(objcg)) {
 		page_counter_uncharge(&memcg->memory, nr_entries);
 
-	if (memcg != swap_memcg) {
-		if (!mem_cgroup_is_root(swap_memcg))
-			page_counter_charge(&swap_memcg->memsw, nr_entries);
-		page_counter_uncharge(&memcg->memsw, nr_entries);
+		if (mem_cgroup_private_id_is_root(private_id))
+			page_counter_uncharge(&memcg->memsw, nr_entries);
 	}
 
 	/*
@@ -672,7 +322,7 @@ void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci)
 	VM_WARN_ON_IRQS_ENABLED();
 	memcg1_charge_statistics(memcg, -folio_nr_pages(folio));
 	preempt_enable_nested();
-	memcg1_check_events(memcg, folio_nid(folio));
+	memcg1_check_events(memcg);
 
 	rcu_read_unlock();
 	obj_cgroup_put(objcg);
@@ -726,14 +376,14 @@ void memcg1_swapin(struct folio *folio)
 #endif
 
 void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
-			   unsigned long nr_memory, int nid)
+			   unsigned long nr_memory)
 {
 	unsigned long flags;
 
 	local_irq_save(flags);
 	count_memcg_events(memcg, PGPGOUT, pgpgout);
 	__this_cpu_add(memcg->events_percpu->nr_page_events, nr_memory);
-	memcg1_check_events(memcg, nid);
+	memcg1_check_events(memcg);
 	local_irq_restore(flags);
 }
 
@@ -2566,22 +2216,3 @@ void memcg1_free_events(struct mem_cgroup *memcg)
 {
 	free_percpu(memcg->events_percpu);
 }
-
-static int __init memcg1_init(void)
-{
-	int node;
-
-	for_each_node(node) {
-		struct mem_cgroup_tree_per_node *rtpn;
-
-		rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, node);
-
-		rtpn->rb_root = RB_ROOT;
-		rtpn->rb_rightmost = NULL;
-		spin_lock_init(&rtpn->lock);
-		soft_limit_tree.rb_tree_per_node[node] = rtpn;
-	}
-
-	return 0;
-}
-subsys_initcall(memcg1_init);
diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h
index 1e394269c613d8..8201396571842a 100644
--- a/mm/memcontrol-v1.h
+++ b/mm/memcontrol-v1.h
@@ -22,8 +22,15 @@ void drain_all_stock(struct mem_cgroup *root_memcg);
 
 int memory_stat_show(struct seq_file *m, void *v);
 
-struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg,
-						    unsigned int n);
+static inline bool mem_cgroup_private_id_is_root(unsigned short id)
+{
+	return id == mem_cgroup_private_id(root_mem_cgroup);
+}
+
+unsigned short mem_cgroup_private_id_get(struct mem_cgroup *memcg, unsigned int n);
+
+void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg,
+				       struct mem_cgroup *parent, int idx);
 
 /* Cgroup v1-specific declarations */
 #ifdef CONFIG_MEMCG_V1
@@ -34,19 +41,13 @@ static inline bool do_memsw_account(void)
 	return !cgroup_subsys_on_dfl(memory_cgrp_subsys);
 }
 
-unsigned long memcg_events_local(struct mem_cgroup *memcg, int event);
-unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx);
-unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item);
+unsigned long memcg_events_local(const struct mem_cgroup *memcg, int event);
+unsigned long memcg_page_state_local(const struct mem_cgroup *memcg, int idx);
+unsigned long memcg_page_state_local_output(const struct mem_cgroup *memcg, int item);
 bool memcg1_alloc_events(struct mem_cgroup *memcg);
 void memcg1_free_events(struct mem_cgroup *memcg);
 
 void memcg1_memcg_init(struct mem_cgroup *memcg);
-void memcg1_remove_from_trees(struct mem_cgroup *memcg);
-
-static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg)
-{
-	WRITE_ONCE(memcg->soft_limit, PAGE_COUNTER_MAX);
-}
 
 struct cgroup_taskset;
 void memcg1_css_offline(struct mem_cgroup *memcg);
@@ -65,7 +66,7 @@ void memcg1_oom_recover(struct mem_cgroup *memcg);
 
 void memcg1_commit_charge(struct folio *folio, struct mem_cgroup *memcg);
 void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
-			   unsigned long nr_memory, int nid);
+			   unsigned long nr_memory);
 
 void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s);
 void reparent_memcg1_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent);
@@ -73,8 +74,6 @@ void reparent_memcg1_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgr
 
 void reparent_memcg_state_local(struct mem_cgroup *memcg,
 				struct mem_cgroup *parent, int idx);
-void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg,
-				       struct mem_cgroup *parent, int idx);
 
 void memcg1_account_kmem(struct mem_cgroup *memcg, int nr_pages);
 static inline bool memcg1_tcpmem_active(struct mem_cgroup *memcg)
@@ -98,8 +97,6 @@ static inline bool memcg1_alloc_events(struct mem_cgroup *memcg) { return true;
 static inline void memcg1_free_events(struct mem_cgroup *memcg) {}
 
 static inline void memcg1_memcg_init(struct mem_cgroup *memcg) {}
-static inline void memcg1_remove_from_trees(struct mem_cgroup *memcg) {}
-static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) {}
 static inline void memcg1_css_offline(struct mem_cgroup *memcg) {}
 
 static inline bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked)
@@ -115,7 +112,7 @@ static inline void memcg1_commit_charge(struct folio *folio,
 
 static inline void memcg1_uncharge_batch(struct mem_cgroup *memcg,
 					 unsigned long pgpgout,
-					 unsigned long nr_memory, int nid) {}
+					 unsigned long nr_memory) {}
 
 static inline void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) {}
 
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 856a7d07586ccc..aad0498a7bd658 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -62,6 +62,7 @@
 #include <linux/seq_buf.h>
 #include <linux/sched/isolation.h>
 #include <linux/kmemleak.h>
+#include <linux/irq_work.h>
 #include "internal.h"
 #include "swap.h"
 #include "swap_table.h"
@@ -232,14 +233,29 @@ static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memc
 	return objcg;
 }
 
-#ifdef CONFIG_MEMCG_V1
 static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force);
 
-static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
+/*
+ * Reparent the non-hierarchical lruvec stats that count_shadow_nodes() reads
+ * to approximate the shadow node budget.  They are not exposed to userspace
+ * on cgroup v2, but they must follow the reparented folios; otherwise the
+ * ancestor would only receive the negative deltas when the folios are freed
+ * without ever having received the positive base, and its local stats would
+ * permanently underflow.
+ */
+static void reparent_v2_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
 {
-	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
-		return;
+	int i;
 
+	for (i = 0; i < NR_LRU_LISTS; i++)
+		reparent_memcg_lruvec_state_local(memcg, parent, NR_LRU_BASE + i);
+
+	reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_RECLAIMABLE_B);
+	reparent_memcg_lruvec_state_local(memcg, parent, NR_SLAB_UNRECLAIMABLE_B);
+}
+
+static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
+{
 	/*
 	 * Reparent stats exposed non-hierarchically. Flush @memcg's stats first
 	 * to read its stats accurately , and conservatively flush @parent's
@@ -248,17 +264,18 @@ static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgr
 	 */
 	__mem_cgroup_flush_stats(memcg, true);
 
-	/* The following counts are all non-hierarchical and need to be reparented. */
-	reparent_memcg1_state_local(memcg, parent);
-	reparent_memcg1_lruvec_state_local(memcg, parent);
+	if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
+		reparent_v2_lruvec_state_local(memcg, parent);
+	} else {
+#ifdef CONFIG_MEMCG_V1
+		/* The following counts are all non-hierarchical and need to be reparented. */
+		reparent_memcg1_state_local(memcg, parent);
+		reparent_memcg1_lruvec_state_local(memcg, parent);
+#endif
+	}
 
 	__mem_cgroup_flush_stats(parent, true);
 }
-#else
-static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
-{
-}
-#endif
 
 static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid)
 {
@@ -331,7 +348,7 @@ EXPORT_SYMBOL(memcg_bpf_enabled_key);
  * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
  * is returned.
  */
-struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio)
+struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio)
 {
 	struct mem_cgroup *memcg;
 
@@ -356,7 +373,7 @@ struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio)
  * after page_cgroup_ino() returns, so it only should be used by callers that
  * do not care (such as procfs interfaces).
  */
-ino_t page_cgroup_ino(struct page *page)
+ino_t page_cgroup_ino(const struct page *page)
 {
 	struct mem_cgroup *memcg;
 	unsigned long ino = 0;
@@ -489,9 +506,9 @@ struct lruvec_stats {
 	long state_pending[NR_MEMCG_NODE_STAT_ITEMS];
 };
 
-unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx)
+unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx)
 {
-	struct mem_cgroup_per_node *pn;
+	const struct mem_cgroup_per_node *pn;
 	long x;
 	int i;
 
@@ -502,7 +519,7 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx)
 	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
 		return 0;
 
-	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
+	pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec);
 	x = READ_ONCE(pn->lruvec_stats->state[i]);
 #ifdef CONFIG_SMP
 	if (x < 0)
@@ -530,10 +547,10 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx)
  * monotonically-incremented event counters are stored in
  * enum node_stat_item.
  */
-unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
+unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec,
 					  enum node_stat_item idx)
 {
-	struct mem_cgroup_per_node *pn;
+	const struct mem_cgroup_per_node *pn;
 	int i;
 
 	if (mem_cgroup_disabled())
@@ -543,14 +560,14 @@ unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec,
 	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
 		return 0;
 
-	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
+	pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec);
 	return (unsigned long)READ_ONCE(pn->lruvec_stats->state[i]);
 }
 
-unsigned long lruvec_page_state_local(struct lruvec *lruvec,
+unsigned long lruvec_page_state_local(const struct lruvec *lruvec,
 				      enum node_stat_item idx)
 {
-	struct mem_cgroup_per_node *pn;
+	const struct mem_cgroup_per_node *pn;
 	long x;
 	int i;
 
@@ -561,7 +578,7 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec,
 	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
 		return 0;
 
-	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
+	pn = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec);
 	x = READ_ONCE(pn->lruvec_stats->state_local[i]);
 #ifdef CONFIG_SMP
 	if (x < 0)
@@ -570,7 +587,6 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec,
 	return x;
 }
 
-#ifdef CONFIG_MEMCG_V1
 static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn,
 				     enum node_stat_item idx, long val);
 
@@ -592,7 +608,6 @@ void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg,
 		__mod_memcg_lruvec_state(parent_pn, idx, value);
 	}
 }
-#endif
 
 /* Subset of vm_event_item to report for memcg event stats */
 static const unsigned int memcg_vm_event_stat[] = {
@@ -675,9 +690,11 @@ struct memcg_vmstats {
 	long			state[MEMCG_VMSTAT_SIZE];
 	unsigned long		events[NR_MEMCG_EVENTS];
 
+#ifdef CONFIG_MEMCG_V1
 	/* Non-hierarchical (CPU aggregated) page state & events */
 	long			state_local[MEMCG_VMSTAT_SIZE];
 	unsigned long		events_local[NR_MEMCG_EVENTS];
+#endif
 
 	/* Pending child counts during tree propagation */
 	long			state_pending[MEMCG_VMSTAT_SIZE];
@@ -687,6 +704,31 @@ struct memcg_vmstats {
 	atomic_long_t		stats_updates;
 };
 
+/*
+ * The non-hierarchical memcg-wide counters are read back only by the legacy
+ * memory.stat and memory.numa_stat, and by reparenting on offline, all of which
+ * are v1-only. If the kernel is built without CONFIG_MEMCG_V1, or if the boot
+ * param cgroup_no_v1= has blocked the memory controller from v1 hierarchies,
+ * then nothing reads them and writers can skip the updates.
+ */
+static long *memcg_state_local_array(struct mem_cgroup *memcg)
+{
+#ifdef CONFIG_MEMCG_V1
+	if (!cgroup1_ssid_disabled(memory_cgrp_id))
+		return memcg->vmstats->state_local;
+#endif
+	return NULL;
+}
+
+static unsigned long *memcg_events_local_array(struct mem_cgroup *memcg)
+{
+#ifdef CONFIG_MEMCG_V1
+	if (!cgroup1_ssid_disabled(memory_cgrp_id))
+		return memcg->vmstats->events_local;
+#endif
+	return NULL;
+}
+
 /*
  * memcg and lruvec stats flushing
  *
@@ -699,7 +741,7 @@ struct memcg_vmstats {
  *
  * 2) Flush the stats synchronously on reader side only when there are more than
  *    (MEMCG_CHARGE_BATCH * nr_cpus) update events. Though this optimization
- *    will let stats be out of sync by atmost (MEMCG_CHARGE_BATCH * nr_cpus) but
+ *    will let stats be out of sync by at most (MEMCG_CHARGE_BATCH * nr_cpus) but
  *    only for 2 seconds due to (1).
  */
 static void flush_memcg_stats_dwork(struct work_struct *w);
@@ -757,7 +799,7 @@ static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force)
 		return;
 
 	if (mem_cgroup_is_root(memcg))
-		WRITE_ONCE(flush_last_time, jiffies_64);
+		WRITE_ONCE(flush_last_time, get_jiffies_64());
 
 	css_rstat_flush(&memcg->css);
 }
@@ -785,7 +827,7 @@ void mem_cgroup_flush_stats(struct mem_cgroup *memcg)
 void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg)
 {
 	/* Only flush if the periodic flusher is one full cycle late */
-	if (time_after64(jiffies_64, READ_ONCE(flush_last_time) + 2*FLUSH_TIME))
+	if (time_after64(get_jiffies_64(), READ_ONCE(flush_last_time) + 2 * FLUSH_TIME))
 		mem_cgroup_flush_stats(memcg);
 }
 
@@ -799,7 +841,7 @@ static void flush_memcg_stats_dwork(struct work_struct *w)
 	queue_delayed_work(system_dfl_wq, &stats_flush_dwork, FLUSH_TIME);
 }
 
-unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx)
+unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx)
 {
 	long x;
 	int i = memcg_stats_index(idx);
@@ -845,16 +887,21 @@ static long memcg_state_val_in_pages(int idx, long val)
 	return val < 0 ? -res : res;
 }
 
-#ifdef CONFIG_MEMCG_V1
 /*
- * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race with
- * reparenting of non-hierarchical state_locals.
+ * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race
+ * with reparenting of non-hierarchical state_locals.  Offlining a
+ * memcg is rare, so do the redirection for all cgroup hierarchies.
  */
-static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg,
-							   bool *rcu_locked)
+static inline struct mem_cgroup *
+get_non_dying_memcg_start(struct mem_cgroup *memcg, bool *rcu_locked)
 {
-	/* Rebinding can cause this value to be changed at runtime */
-	if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
+	/*
+	 * Fast path: the caller holds a reference to @memcg, so reading
+	 * its CSS_DYING flag without the RCU lock is safe.  The RCU lock
+	 * is only needed to walk up to a non-dying ancestor, which
+	 * happens only while a memcg is actually being offlined.
+	 */
+	if (!memcg_is_dying(memcg)) {
 		*rcu_locked = false;
 		return memcg;
 	}
@@ -875,17 +922,6 @@ static inline void get_non_dying_memcg_end(bool rcu_locked)
 
 	rcu_read_unlock();
 }
-#else
-static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg,
-							   bool *rcu_locked)
-{
-	return memcg;
-}
-
-static inline void get_non_dying_memcg_end(bool rcu_locked)
-{
-}
-#endif
 
 static void __mod_memcg_state(struct mem_cgroup *memcg,
 			      enum memcg_stat_item idx, long val)
@@ -928,7 +964,7 @@ void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx,
 
 #ifdef CONFIG_MEMCG_V1
 /* idx can be of type enum memcg_stat_item or node_stat_item. */
-unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)
+unsigned long memcg_page_state_local(const struct mem_cgroup *memcg, int idx)
 {
 	long x;
 	int i = memcg_stats_index(idx);
@@ -979,9 +1015,8 @@ static void __mod_memcg_lruvec_state(struct mem_cgroup_per_node *pn,
 	put_cpu();
 }
 
-static void mod_memcg_lruvec_state(struct lruvec *lruvec,
-				     enum node_stat_item idx,
-				     int val)
+void mod_memcg_lruvec_state(struct lruvec *lruvec,
+			    enum node_stat_item idx, int val)
 {
 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
 	struct mem_cgroup_per_node *pn;
@@ -1091,7 +1126,7 @@ void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
 	put_cpu();
 }
 
-unsigned long memcg_events(struct mem_cgroup *memcg, int event)
+unsigned long memcg_events(const struct mem_cgroup *memcg, int event)
 {
 	int i = memcg_events_index(event);
 
@@ -1110,7 +1145,7 @@ bool memcg_vm_event_item_valid(enum vm_event_item idx)
 }
 
 #ifdef CONFIG_MEMCG_V1
-unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
+unsigned long memcg_events_local(const struct mem_cgroup *memcg, int event)
 {
 	int i = memcg_events_index(event);
 
@@ -1220,7 +1255,7 @@ struct mem_cgroup *get_mem_cgroup_from_current(void)
  *
  * See folio_memcg() for folio->objcg/memcg binding rules.
  */
-struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio)
+struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio)
 {
 	struct mem_cgroup *memcg;
 
@@ -1441,7 +1476,7 @@ void mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
  *
  * Return: The lruvec this folio is on with its lock held and rcu read lock held.
  */
-struct lruvec *folio_lruvec_lock(struct folio *folio)
+struct lruvec *folio_lruvec_lock(const struct folio *folio)
 {
 	struct lruvec *lruvec;
 
@@ -1469,7 +1504,7 @@ struct lruvec *folio_lruvec_lock(struct folio *folio)
  * Return: The lruvec this folio is on with its lock held and interrupts
  * disabled and rcu read lock held.
  */
-struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
+struct lruvec *folio_lruvec_lock_irq(const struct folio *folio)
 {
 	struct lruvec *lruvec;
 
@@ -1498,7 +1533,7 @@ struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
  * Return: The lruvec this folio is on with its lock held and interrupts
  * disabled and rcu read lock held.
  */
-struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
+struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio,
 		unsigned long *flags)
 {
 	struct lruvec *lruvec;
@@ -1529,28 +1564,12 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
 				int zid, long nr_pages)
 {
 	struct mem_cgroup_per_node *mz;
-	unsigned long *lru_size;
-	long size;
 
 	if (mem_cgroup_disabled())
 		return;
 
 	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
-	lru_size = &mz->lru_zone_size[zid][lru];
-
-	if (nr_pages < 0)
-		*lru_size += nr_pages;
-
-	size = *lru_size;
-	if (WARN_ONCE(size < 0,
-		"%s(%p, %d, %ld): lru_size %ld\n",
-		__func__, lruvec, lru, nr_pages, size)) {
-		VM_BUG_ON(1);
-		*lru_size = 0;
-	}
-
-	if (nr_pages > 0)
-		*lru_size += nr_pages;
+	mz->lru_zone_size[zid][lru] += nr_pages;
 }
 
 /**
@@ -1709,14 +1728,14 @@ static int memcg_page_state_output_unit(int item)
 	}
 }
 
-unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item)
+unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item)
 {
 	return memcg_page_state(memcg, item) *
 		memcg_page_state_output_unit(item);
 }
 
 #ifdef CONFIG_MEMCG_V1
-unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item)
+unsigned long memcg_page_state_local_output(const struct mem_cgroup *memcg, int item)
 {
 	return memcg_page_state_local(memcg, item) *
 		memcg_page_state_output_unit(item);
@@ -1812,7 +1831,8 @@ static void memory_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
  * enabled
  */
-void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
+void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg,
+				  struct task_struct *p)
 {
 	rcu_read_lock();
 
@@ -1877,7 +1897,7 @@ void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
 /*
  * Return the memory (and swap, if configured) limit for a memcg.
  */
-unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
+unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg)
 {
 	unsigned long max = READ_ONCE(memcg->memory.max);
 
@@ -2048,6 +2068,15 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
  * nr_pages in a single cacheline. This may change in future.
  */
 #define NR_MEMCG_STOCK 7
+
+/*
+ * Watermarks for a charge stock slot, in the spirit of pcp->high and
+ * pcp->batch: MEMCG_STOCK_HIGH is the high watermark at which a slot is
+ * trimmed, and it is trimmed down to MEMCG_STOCK_LOW rather than emptied.
+ */
+#define MEMCG_STOCK_LOW		(MEMCG_CHARGE_BATCH / 2)
+#define MEMCG_STOCK_HIGH	(MEMCG_CHARGE_BATCH)
+
 #define FLUSHING_CACHED_CHARGE	0
 struct memcg_stock_pcp {
 	local_trylock_t lock;
@@ -2228,17 +2257,18 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 {
 	struct memcg_stock_pcp *stock;
 	struct mem_cgroup *cached;
-	uint8_t stock_pages;
+	unsigned int stock_pages;
 	bool success = false;
 	int empty_slot = -1;
 	int i;
 
 	/*
-	 * For now limit MEMCG_CHARGE_BATCH to 127 and less. In future if we
-	 * decide to increase it more than 127 then we will need more careful
-	 * handling of nr_pages[] in struct memcg_stock_pcp.
+	 * nr_pages[] is a uint8_t and a slot's count is capped at
+	 * MEMCG_STOCK_HIGH. Raising MEMCG_CHARGE_BATCH beyond 127 would need
+	 * more careful handling of nr_pages[] in struct memcg_stock_pcp.
 	 */
 	BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S8_MAX);
+	BUILD_BUG_ON(MEMCG_STOCK_HIGH > U8_MAX);
 
 	VM_WARN_ON_ONCE(mem_cgroup_is_root(memcg));
 
@@ -2259,9 +2289,12 @@ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 			empty_slot = i;
 		if (memcg == READ_ONCE(stock->cached[i])) {
 			stock_pages = READ_ONCE(stock->nr_pages[i]) + nr_pages;
+			if (stock_pages > MEMCG_STOCK_HIGH) {
+				memcg_uncharge(memcg,
+					       stock_pages - MEMCG_STOCK_LOW);
+				stock_pages = MEMCG_STOCK_LOW;
+			}
 			WRITE_ONCE(stock->nr_pages[i], stock_pages);
-			if (stock_pages > MEMCG_CHARGE_BATCH)
-				drain_stock(stock, i);
 			success = true;
 			break;
 		}
@@ -2306,7 +2339,7 @@ static bool is_memcg_drain_needed(struct memcg_stock_pcp *stock,
 	return flush;
 }
 
-static void schedule_drain_work(int cpu, struct work_struct *work)
+static bool schedule_drain_work(int cpu, struct work_struct *work)
 {
 	/*
 	 * Protect housekeeping cpumask read and work enqueue together
@@ -2315,8 +2348,11 @@ static void schedule_drain_work(int cpu, struct work_struct *work)
 	 * pending work on newly isolated CPUs.
 	 */
 	guard(rcu)();
-	if (!cpu_is_isolated(cpu))
-		queue_work_on(cpu, memcg_wq, work);
+	if (cpu_is_isolated(cpu))
+		return false;
+
+	queue_work_on(cpu, memcg_wq, work);
+	return true;
 }
 
 /*
@@ -2348,8 +2384,9 @@ void drain_all_stock(struct mem_cgroup *root_memcg)
 				      &memcg_st->flags)) {
 			if (cpu == curcpu)
 				drain_local_memcg_stock(&memcg_st->work);
-			else
-				schedule_drain_work(cpu, &memcg_st->work);
+			else if (!schedule_drain_work(cpu, &memcg_st->work))
+				clear_bit(FLUSHING_CACHED_CHARGE,
+					  &memcg_st->flags);
 		}
 
 		if (!test_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags) &&
@@ -2358,8 +2395,9 @@ void drain_all_stock(struct mem_cgroup *root_memcg)
 				      &obj_st->flags)) {
 			if (cpu == curcpu)
 				drain_local_obj_stock(&obj_st->work);
-			else
-				schedule_drain_work(cpu, &obj_st->work);
+			else if (!schedule_drain_work(cpu, &obj_st->work))
+				clear_bit(FLUSHING_CACHED_CHARGE,
+					  &obj_st->flags);
 		}
 	}
 	migrate_enable();
@@ -2368,9 +2406,21 @@ void drain_all_stock(struct mem_cgroup *root_memcg)
 
 static int memcg_hotplug_cpu_dead(unsigned int cpu)
 {
+	struct memcg_stock_pcp *memcg_st = &per_cpu(memcg_stock, cpu);
+	struct obj_stock_pcp *obj_st = &per_cpu(obj_stock, cpu);
+
 	/* no need for the local lock */
-	drain_obj_stock(&per_cpu(obj_stock, cpu));
-	drain_stock_fully(&per_cpu(memcg_stock, cpu));
+	drain_obj_stock(obj_st);
+	drain_stock_fully(memcg_st);
+
+	/*
+	 * A drain work queued before the CPU went away is executed by an
+	 * unbound worker on some other CPU and clears that CPU's flag, so
+	 * clear the flags here to make these stocks drainable again once
+	 * the CPU comes back online.
+	 */
+	clear_bit(FLUSHING_CACHED_CHARGE, &memcg_st->flags);
+	clear_bit(FLUSHING_CACHED_CHARGE, &obj_st->flags);
 
 	return 0;
 }
@@ -2410,6 +2460,11 @@ static void high_work_func(struct work_struct *work)
 	reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
 }
 
+static void high_irq_work_func(struct irq_work *work)
+{
+	schedule_work(&container_of(work, struct mem_cgroup, high_irq_work)->high_work);
+}
+
 /*
  * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
  * enough to still cause a significant slowdown in most cases, while still
@@ -2515,8 +2570,7 @@ static u64 swap_find_max_overage(struct mem_cgroup *memcg)
  * Get the number of jiffies that we should penalise a mischievous cgroup which
  * is exceeding its memory.high by checking both it and its ancestors.
  */
-static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
-					  unsigned int nr_pages,
+static unsigned long calculate_high_delay(unsigned int nr_pages,
 					  u64 max_overage)
 {
 	unsigned long penalty_jiffies;
@@ -2594,10 +2648,10 @@ void __mem_cgroup_handle_over_high(gfp_t gfp_mask)
 	 * memory.high is breached and reclaim is unable to keep up. Throttle
 	 * allocators proactively to slow down excessive growth.
 	 */
-	penalty_jiffies = calculate_high_delay(memcg, nr_pages,
+	penalty_jiffies = calculate_high_delay(nr_pages,
 					       mem_find_max_overage(memcg));
 
-	penalty_jiffies += calculate_high_delay(memcg, nr_pages,
+	penalty_jiffies += calculate_high_delay(nr_pages,
 						swap_find_max_overage(memcg));
 
 	/*
@@ -2656,10 +2710,11 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
 	bool raised_max_event = false;
 	unsigned long pflags;
 	bool allow_spinning = gfpflags_allow_spinning(gfp_mask);
+	int ret = 0;
 
 retry:
 	if (consume_stock(memcg, nr_pages))
-		return 0;
+		return ret;
 
 	if (!allow_spinning)
 		/* Avoid the refill and flush of the older stock */
@@ -2770,16 +2825,11 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
 	 * put the burden of reclaim on regular allocation requests
 	 * and let these go through as privileged allocations.
 	 */
-	if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH)))
-		return -ENOMEM;
+	if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH))) {
+		ret = -ENOMEM;
+		goto out;
+	}
 force:
-	/*
-	 * If the allocation has to be enforced, don't forget to raise
-	 * a MEMCG_MAX event.
-	 */
-	if (!raised_max_event)
-		__memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);
-
 	/*
 	 * The allocation either can't fail or will lead to more memory
 	 * being freed very soon.  Allow memory usage go over the limit
@@ -2789,7 +2839,15 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
 	if (do_memsw_account())
 		page_counter_charge(&memcg->memsw, nr_pages);
 
-	return 0;
+out:
+	/*
+	 * Don't forget to raise a MEMCG_MAX event for forced or rejected
+	 * requests.
+	 */
+	if (!raised_max_event)
+		__memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning);
+
+	return ret;
 
 done_restock:
 	if (batch > nr_pages)
@@ -2815,7 +2873,10 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
 		/* Don't bother a random interrupted task */
 		if (!in_task()) {
 			if (mem_high) {
-				schedule_work(&memcg->high_work);
+				if (allow_spinning)
+					schedule_work(&memcg->high_work);
+				else
+					irq_work_queue(&memcg->high_irq_work);
 				break;
 			}
 			continue;
@@ -2848,7 +2909,7 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
 	    !(current->flags & PF_MEMALLOC) &&
 	    gfpflags_allow_blocking(gfp_mask))
 		__mem_cgroup_handle_over_high(gfp_mask);
-	return 0;
+	return ret;
 }
 
 static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
@@ -3093,7 +3154,7 @@ __always_inline struct obj_cgroup *current_obj_cgroup(void)
 	return rcu_dereference_check(root_mem_cgroup->nodeinfo[nid]->objcg, 1);
 }
 
-struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)
+struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio)
 {
 	struct obj_cgroup *objcg;
 
@@ -3756,7 +3817,7 @@ static void memcg_online_kmem(struct mem_cgroup *memcg)
 
 	static_branch_enable(&memcg_kmem_online_key);
 
-	memcg->kmemcg_id = memcg->id.id;
+	memcg->kmemcg_id = memcg->private_id;
 }
 
 static void memcg_offline_kmem(struct mem_cgroup *memcg)
@@ -3946,7 +4007,7 @@ void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
 {
 	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
 	unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
-	u64 now = jiffies_64;
+	u64 now = get_jiffies_64();
 	int i;
 
 	for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
@@ -4013,27 +4074,59 @@ static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
 #define MEM_CGROUP_ID_MAX	((1UL << MEM_CGROUP_ID_SHIFT) - 1)
 static DEFINE_XARRAY_ALLOC1(mem_cgroup_private_ids);
 
+static inline struct obj_cgroup *obj_cgroup_from_private_id(unsigned short id)
+{
+	lockdep_assert_once(rcu_read_lock_held());
+
+	return xa_load(&mem_cgroup_private_ids, id);
+}
+
 static void mem_cgroup_private_id_remove(struct mem_cgroup *memcg)
 {
-	if (memcg->id.id > 0) {
-		xa_erase(&mem_cgroup_private_ids, memcg->id.id);
-		memcg->id.id = 0;
+	if (memcg->private_id > 0) {
+		xa_erase(&mem_cgroup_private_ids, memcg->private_id);
+		memcg->private_id = 0;
 	}
 }
 
-static inline void mem_cgroup_private_id_put(struct mem_cgroup *memcg, unsigned int n)
+static void __mem_cgroup_private_id_put(struct obj_cgroup *objcg,
+		unsigned short id, unsigned int n)
 {
-	if (refcount_sub_and_test(n, &memcg->id.ref)) {
-		mem_cgroup_private_id_remove(memcg);
+	struct obj_cgroup *objcg_free;
 
-		/* Memcg ID pins CSS */
-		css_put(&memcg->css);
+	if (refcount_sub_and_test(n, &objcg->private_id_ref)) {
+		objcg_free = xa_erase(&mem_cgroup_private_ids, id);
+		VM_WARN_ON(objcg_free != objcg);
+
+		/* Memcg ID pins the objcg */
+		obj_cgroup_put(objcg);
 	}
 }
 
-struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, unsigned int n)
+static inline void mem_cgroup_private_id_put(unsigned short id, unsigned int n)
+{
+	struct obj_cgroup *objcg;
+
+	lockdep_assert_in_rcu_read_lock();
+
+	objcg = obj_cgroup_from_private_id(id);
+	__mem_cgroup_private_id_put(objcg, id, n);
+}
+
+static void mem_cgroup_private_id_kill(struct mem_cgroup *memcg)
+{
+	__mem_cgroup_private_id_put(memcg->private_id_objcg, memcg->private_id, 1);
+}
+
+unsigned short mem_cgroup_private_id_get(struct mem_cgroup *memcg, unsigned int n)
 {
-	while (!refcount_add_not_zero(n, &memcg->id.ref)) {
+	struct obj_cgroup *objcg;
+
+	lockdep_assert_once(rcu_read_lock_held());
+
+	objcg = memcg->private_id_objcg;
+
+	while (!refcount_add_not_zero(n, &objcg->private_id_ref)) {
 		/*
 		 * The root cgroup cannot be destroyed, so it's refcount must
 		 * always be >= 1.
@@ -4043,8 +4136,10 @@ struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, un
 			break;
 		}
 		memcg = parent_mem_cgroup(memcg);
+		objcg = memcg->private_id_objcg;
 	}
-	return memcg;
+
+	return mem_cgroup_private_id(memcg);
 }
 
 /**
@@ -4052,11 +4147,25 @@ struct mem_cgroup *mem_cgroup_private_id_get_online(struct mem_cgroup *memcg, un
  * @id: the memcg id to look up
  *
  * Caller must hold rcu_read_lock().
+ *
+ * @return: the memcg, or NULL if the memcg referred to is already dead.
  */
 struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id)
 {
+	struct obj_cgroup *objcg;
+	struct mem_cgroup *memcg;
+
 	WARN_ON_ONCE(!rcu_read_lock_held());
-	return xa_load(&mem_cgroup_private_ids, id);
+
+	objcg = obj_cgroup_from_private_id(id);
+	if (!objcg)
+		return NULL;
+
+	memcg = obj_cgroup_memcg(objcg);
+	if (mem_cgroup_private_id(memcg) != id)
+		return NULL;
+
+	return memcg;
 }
 
 struct mem_cgroup *mem_cgroup_get_from_id(u64 id)
@@ -4157,7 +4266,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
 	if (!memcg)
 		return ERR_PTR(-ENOMEM);
 
-	error = xa_alloc(&mem_cgroup_private_ids, &memcg->id.id, NULL,
+	error = xa_alloc(&mem_cgroup_private_ids, &memcg->private_id, NULL,
 			 XA_LIMIT(1, MEM_CGROUP_ID_MAX), GFP_KERNEL);
 	if (error)
 		goto fail;
@@ -4190,6 +4299,7 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
 		goto fail;
 
 	INIT_WORK(&memcg->high_work, high_work_func);
+	init_irq_work(&memcg->high_irq_work, high_irq_work_func);
 	vmpressure_init(&memcg->vmpressure);
 	INIT_LIST_HEAD(&memcg->memory_peaks);
 	INIT_LIST_HEAD(&memcg->swap_peaks);
@@ -4228,7 +4338,6 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 		return ERR_CAST(memcg);
 
 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
-	memcg1_soft_limit_reset(memcg);
 #ifdef CONFIG_ZSWAP
 	memcg->zswap_max = PAGE_COUNTER_MAX;
 	WRITE_ONCE(memcg->zswap_writeback, true);
@@ -4303,9 +4412,10 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
 				   FLUSH_TIME);
 	lru_gen_online_memcg(memcg);
 
-	/* Online state pins memcg ID, memcg ID pins CSS */
-	refcount_set(&memcg->id.ref, 1);
-	css_get(css);
+	/* CSS pins memcg ID, memcg ID pins obj cgroup */
+	memcg->private_id_objcg = objcg;
+	refcount_set(&memcg->private_id_objcg->private_id_ref, 1);
+	obj_cgroup_get(memcg->private_id_objcg);
 
 	/*
 	 * Ensure mem_cgroup_from_private_id() works once we're fully online.
@@ -4317,7 +4427,7 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
 	 * publish it here at the end of onlining. This matches the
 	 * regular ID destruction during offlining.
 	 */
-	xa_store(&mem_cgroup_private_ids, memcg->id.id, memcg, GFP_KERNEL);
+	xa_store(&mem_cgroup_private_ids, memcg->private_id, memcg->private_id_objcg, GFP_KERNEL);
 
 	return 0;
 free_objcg:
@@ -4369,7 +4479,7 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
 
 	drain_all_stock(memcg);
 
-	mem_cgroup_private_id_put(memcg, 1);
+	mem_cgroup_private_id_kill(memcg);
 }
 
 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
@@ -4399,8 +4509,8 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
 		static_branch_dec(&memcg_bpf_enabled_key);
 
 	vmpressure_cleanup(&memcg->vmpressure);
+	irq_work_sync(&memcg->high_irq_work);
 	cancel_work_sync(&memcg->high_work);
-	memcg1_remove_from_trees(memcg);
 	free_shrinker_info(memcg);
 	mem_cgroup_free(memcg);
 }
@@ -4436,7 +4546,6 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
 	page_counter_set_min(&memcg->memory, 0);
 	page_counter_set_low(&memcg->memory, 0);
 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
-	memcg1_soft_limit_reset(memcg);
 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
 	memcg_wb_domain_size_changed(memcg);
 }
@@ -4444,7 +4553,10 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
 struct aggregate_control {
 	/* pointer to the aggregated (CPU and subtree aggregated) counters */
 	long *aggregate;
-	/* pointer to the non-hierarchichal (CPU aggregated) counters */
+	/*
+	 * pointer to the non-hierarchical (CPU aggregated) counters or NULL to
+	 * skip updating them (see memcg_state_local_array())
+	 */
 	long *local;
 	/* pointer to the pending child counters during tree propagation */
 	long *pending;
@@ -4483,7 +4595,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac)
 		}
 
 		/* Aggregate counts on this level and propagate upwards */
-		if (delta_cpu)
+		if (delta_cpu && ac->local)
 			ac->local[i] += delta_cpu;
 
 		if (delta) {
@@ -4497,6 +4609,7 @@ static void mem_cgroup_stat_aggregate(struct aggregate_control *ac)
 #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
 static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
 {
+	long *state_local = memcg_state_local_array(memcg);
 	int nid;
 
 	if (atomic_read(&memcg->kmem_stat)) {
@@ -4504,7 +4617,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
 		int index = memcg_stats_index(MEMCG_KMEM);
 
 		memcg->vmstats->state[index] += kmem;
-		memcg->vmstats->state_local[index] += kmem;
+		if (state_local)
+			state_local[index] += kmem;
 		if (parent)
 			parent->vmstats->state_pending[index] += kmem;
 	}
@@ -4526,7 +4640,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
 			if (plstats)
 				plstats->state_pending[index] += slab;
 			memcg->vmstats->state[index] += slab;
-			memcg->vmstats->state_local[index] += slab;
+			if (state_local)
+				state_local[index] += slab;
 			if (parent)
 				parent->vmstats->state_pending[index] += slab;
 		}
@@ -4539,7 +4654,8 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent)
 			if (plstats)
 				plstats->state_pending[index] += slab;
 			memcg->vmstats->state[index] += slab;
-			memcg->vmstats->state_local[index] += slab;
+			if (state_local)
+				state_local[index] += slab;
 			if (parent)
 				parent->vmstats->state_pending[index] += slab;
 		}
@@ -4564,7 +4680,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
 
 	ac = (struct aggregate_control) {
 		.aggregate = memcg->vmstats->state,
-		.local = memcg->vmstats->state_local,
+		.local = memcg_state_local_array(memcg),
 		.pending = memcg->vmstats->state_pending,
 		.ppending = parent ? parent->vmstats->state_pending : NULL,
 		.cstat = statc->state,
@@ -4575,7 +4691,7 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
 
 	ac = (struct aggregate_control) {
 		.aggregate = memcg->vmstats->events,
-		.local = memcg->vmstats->events_local,
+		.local = memcg_events_local_array(memcg),
 		.pending = memcg->vmstats->events_pending,
 		.ppending = parent ? parent->vmstats->events_pending : NULL,
 		.cstat = statc->events,
@@ -4719,6 +4835,7 @@ static int memory_peak_show(struct seq_file *sf, void *v)
 {
 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
 
+	guard(spinlock)(&memcg->peaks_lock);
 	return peak_show(sf, v, &memcg->memory);
 }
 
@@ -4748,7 +4865,7 @@ static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes,
 			  loff_t off, struct page_counter *pc,
 			  struct list_head *watchers)
 {
-	unsigned long usage;
+	unsigned long usage, old_watermark;
 	struct cgroup_of_peak *peer_ctx;
 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
 	struct cgroup_of_peak *ofp = of_peak(of);
@@ -4756,11 +4873,12 @@ static ssize_t peak_write(struct kernfs_open_file *of, char *buf, size_t nbytes,
 	spin_lock(&memcg->peaks_lock);
 
 	usage = page_counter_read(pc);
+	old_watermark = READ_ONCE(pc->local_watermark);
 	WRITE_ONCE(pc->local_watermark, usage);
 
 	list_for_each_entry(peer_ctx, watchers, list)
-		if (usage > peer_ctx->value)
-			WRITE_ONCE(peer_ctx->value, usage);
+		if (peer_ctx != ofp && old_watermark > peer_ctx->value)
+			WRITE_ONCE(peer_ctx->value, old_watermark);
 
 	/* initial write, register watcher */
 	if (ofp->value == OFP_PEAK_UNSET)
@@ -5198,6 +5316,51 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
 	page_counter_calculate_protection(&root->memory, &memcg->memory, recursive_protection);
 }
 
+#ifdef CONFIG_LRU_GEN
+/**
+ * mem_cgroup_calculate_protection_path - compute protection along a path
+ * @root: the top ancestor of the sub-tree being checked (NULL for root_mem_cgroup)
+ * @memcg: the target memory cgroup
+ *
+ * Walk the ancestor path from @root down to @memcg and compute the effective
+ * protection at each level.  This is safe for isolated queries because it
+ * ensures parents are computed before children.
+ */
+void mem_cgroup_calculate_protection_path(struct mem_cgroup *root,
+					  struct mem_cgroup *memcg)
+{
+	bool recursive_protection =
+		cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT;
+	struct cgroup *cg;
+	int root_level, i;
+
+	if (mem_cgroup_disabled())
+		return;
+
+	if (!root)
+		root = root_mem_cgroup;
+
+	if (memcg == root)
+		return;
+
+	root_level = root->css.cgroup->level;
+	cg = memcg->css.cgroup;
+
+	rcu_read_lock();
+	for (i = root_level + 1; i <= cg->level; i++) {
+		struct mem_cgroup *cur;
+
+		cur = mem_cgroup_from_css(cgroup_css(cg->ancestors[i],
+						     &memory_cgrp_subsys));
+		if (cur)
+			page_counter_calculate_protection(&root->memory,
+							  &cur->memory,
+							  recursive_protection);
+	}
+	rcu_read_unlock();
+}
+#endif /* CONFIG_LRU_GEN */
+
 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
 			gfp_t gfp)
 {
@@ -5233,6 +5396,7 @@ int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp)
 /**
  * mem_cgroup_charge_hugetlb - charge the memcg for a hugetlb folio
  * @folio: folio being charged
+ * @mm: mm context of the allocation target
  * @gfp: reclaim mode
  *
  * This function is called when allocating a huge page folio, after the page has
@@ -5242,9 +5406,10 @@ int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp)
  * Returns ENOMEM if the memcg is already full.
  * Returns 0 if either the charge was successful, or if we skip the charging.
  */
-int mem_cgroup_charge_hugetlb(struct folio *folio, gfp_t gfp)
+int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm,
+			      gfp_t gfp)
 {
-	struct mem_cgroup *memcg = get_mem_cgroup_from_current();
+	struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm);
 	int ret = 0;
 
 	/*
@@ -5302,7 +5467,6 @@ struct uncharge_gather {
 	unsigned long nr_memory;
 	unsigned long pgpgout;
 	unsigned long nr_kmem;
-	int nid;
 };
 
 static inline void uncharge_gather_clear(struct uncharge_gather *ug)
@@ -5325,7 +5489,7 @@ static void uncharge_batch(const struct uncharge_gather *ug)
 		memcg1_oom_recover(memcg);
 	}
 
-	memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory, ug->nid);
+	memcg1_uncharge_batch(memcg, ug->pgpgout, ug->nr_memory);
 	rcu_read_unlock();
 
 	/* drop reference from uncharge_folio */
@@ -5354,7 +5518,6 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
 			uncharge_gather_clear(ug);
 		}
 		ug->objcg = objcg;
-		ug->nid = folio_nid(folio);
 
 		/* pairs with obj_cgroup_put in uncharge_batch */
 		obj_cgroup_get(objcg);
@@ -5684,6 +5847,108 @@ __setup("cgroup.memory=", cgroup_memory);
  * basically everything that doesn't depend on a specific mem_cgroup structure
  * should be initialized from here.
  */
+/*
+ * Fields are grouped by access pattern. Putting a field in the wrong group
+ * breaks the build here.
+ */
+static void __init memcg_struct_check(void)
+{
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      socket_pressure);
+#if BITS_PER_LONG < 64
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      socket_pressure_seqlock);
+#endif
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      memory_events);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      memory_events_local);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      vmpressure);
+#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      kmem_stat);
+#endif
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      high_irq_work);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_write_hot,
+				      high_work);
+
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      memory_peaks);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      swap_peaks);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      peaks_lock);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      events_file);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      events_local_file);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      swap_events_file);
+#ifdef CONFIG_CGROUP_WRITEBACK
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      cgwb_list);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      cgwb_domain);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      cgwb_frn);
+#endif
+#ifdef CONFIG_LRU_GEN_WALKS_MMU
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_cold,
+				      mm_list);
+#endif
+
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      vmstats_percpu);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      vmstats);
+#ifdef CONFIG_ZSWAP
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      zswap_max);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      zswap_writeback);
+#endif
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      private_id_objcg);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      private_id);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      kmemcg_id);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup, memcg_read_mostly,
+				      oom_group);
+
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_read_mostly, memcg);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_read_mostly, lruvec_stats_percpu);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_read_mostly, lruvec_stats);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_read_mostly, shrinker_info);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_read_mostly, objcg);
+
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_lruvec, lruvec);
+
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_write_hot, lru_zone_size);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_write_hot, iter);
+#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_write_hot, slab_reclaimable);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_write_hot, slab_unreclaimable);
+#endif
+
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_cold, orig_objcg);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct mem_cgroup_per_node,
+				      memcg_pn_cold, objcg_list);
+}
+
 int __init mem_cgroup_init(void)
 {
 	unsigned int memcg_size;
@@ -5697,6 +5962,8 @@ int __init mem_cgroup_init(void)
 	 */
 	BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
 
+	memcg_struct_check();
+
 	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
 				  memcg_hotplug_cpu_dead);
 
@@ -5736,6 +6003,7 @@ int __mem_cgroup_try_charge_swap(struct folio *folio)
 	struct page_counter *counter;
 	struct mem_cgroup *memcg;
 	struct obj_cgroup *objcg;
+	unsigned short private_id;
 
 	if (do_memsw_account())
 		return 0;
@@ -5745,30 +6013,27 @@ int __mem_cgroup_try_charge_swap(struct folio *folio)
 	if (!objcg)
 		return 0;
 
-	rcu_read_lock();
-	memcg = obj_cgroup_memcg(objcg);
-	if (!folio_test_swapcache(folio)) {
-		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
-		rcu_read_unlock();
-		return 0;
-	}
+	scoped_guard(rcu) {
+		memcg = obj_cgroup_memcg(objcg);
+		if (!folio_test_swapcache(folio)) {
+			memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
+			return 0;
+		}
 
-	memcg = mem_cgroup_private_id_get_online(memcg, nr_pages);
-	/* memcg is pined by memcg ID. */
-	rcu_read_unlock();
+		private_id = mem_cgroup_private_id_get(memcg, nr_pages);
 
-	if (!mem_cgroup_is_root(memcg) &&
-	    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
-		memcg_memory_event(memcg, MEMCG_SWAP_MAX);
-		memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
-		mem_cgroup_private_id_put(memcg, nr_pages);
-		return -ENOMEM;
+		if (!mem_cgroup_private_id_is_root(private_id) &&
+		    !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
+			memcg_memory_event(memcg, MEMCG_SWAP_MAX);
+			memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
+			mem_cgroup_private_id_put(private_id, nr_pages);
+			return -ENOMEM;
+		}
+		mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
 	}
-	mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
 
 	ci = swap_cluster_get_and_lock(folio);
-	__swap_cgroup_set(ci, swp_cluster_offset(folio->swap), nr_pages,
-			  mem_cgroup_private_id(memcg));
+	__swap_cgroup_set(ci, swp_cluster_offset(folio->swap), nr_pages, private_id);
 	swap_cluster_unlock(ci);
 
 	return 0;
@@ -5781,37 +6046,59 @@ int __mem_cgroup_try_charge_swap(struct folio *folio)
  */
 void __mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages)
 {
+	struct obj_cgroup *objcg;
 	struct mem_cgroup *memcg;
 
 	rcu_read_lock();
-	memcg = mem_cgroup_from_private_id(id);
+	objcg = obj_cgroup_from_private_id(id);
+	memcg = obj_cgroup_memcg(objcg);
 	if (memcg) {
-		if (!mem_cgroup_is_root(memcg)) {
+		if (!mem_cgroup_private_id_is_root(id)) {
 			if (do_memsw_account())
 				page_counter_uncharge(&memcg->memsw, nr_pages);
 			else
 				page_counter_uncharge(&memcg->swap, nr_pages);
 		}
 		mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
-		mem_cgroup_private_id_put(memcg, nr_pages);
+		__mem_cgroup_private_id_put(objcg, id, nr_pages);
 	}
 	rcu_read_unlock();
 }
 
-long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
+long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg)
 {
 	long nr_swap_pages = get_nr_swap_pages();
 
-	if (mem_cgroup_disabled() || do_memsw_account())
-		return nr_swap_pages;
-	for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg))
-		nr_swap_pages = min_t(long, nr_swap_pages,
-				      READ_ONCE(memcg->swap.max) -
-				      page_counter_read(&memcg->swap));
+	if (!mem_cgroup_disabled() && !do_memsw_account())
+		nr_swap_pages = min(nr_swap_pages, page_counter_margin(&memcg->swap));
+
 	return nr_swap_pages;
 }
 
-bool mem_cgroup_swap_full(struct folio *folio)
+/**
+ * mem_cgroup_get_folio_swap_margin - get a folio's memcg swap margin
+ * @folio: folio whose memcg margin is queried
+ *
+ * Return: Remaining chargeable pages in the folio's memcg hierarchy.
+ */
+long mem_cgroup_get_folio_swap_margin(const struct folio *folio)
+{
+	struct mem_cgroup *memcg;
+	long margin;
+
+	if (mem_cgroup_disabled() || do_memsw_account() ||
+	    !folio_memcg_charged(folio))
+		return PAGE_COUNTER_MAX;
+
+	rcu_read_lock();
+	memcg = folio_memcg(folio);
+	margin = page_counter_margin(&memcg->swap);
+	rcu_read_unlock();
+
+	return margin;
+}
+
+bool mem_cgroup_swap_full(const struct folio *folio)
 {
 	struct mem_cgroup *memcg;
 	bool ret = false;
@@ -5864,6 +6151,7 @@ static int swap_peak_show(struct seq_file *sf, void *v)
 {
 	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
 
+	guard(spinlock)(&memcg->peaks_lock);
 	return peak_show(sf, v, &memcg->swap);
 }
 
@@ -6078,7 +6366,7 @@ void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)
 	rcu_read_unlock();
 }
 
-bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg)
+bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg)
 {
 	/* if zswap is disabled, do not block pages going to the swapping device */
 	if (!zswap_is_enabled())
diff --git a/mm/memfd.c b/mm/memfd.c
index c708d92533f4ab..0f6fff004f5e4c 100644
--- a/mm/memfd.c
+++ b/mm/memfd.c
@@ -82,22 +82,31 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx)
 		struct hstate *h = hstate_file(memfd);
 		int err = -ENOMEM;
 		long nr_resv;
+		u32 hash;
 
 		gfp_mask = htlb_alloc_mask(h);
 		gfp_mask &= ~(__GFP_HIGHMEM | __GFP_MOVABLE);
 		idx >>= huge_page_order(h);
 
+		/*
+		 * Serialize hugepage allocation and instantiation to prevent
+		 * races with concurrent allocations, as required by all other
+		 * callers of hugetlb_add_to_page_cache().
+		 */
+		hash = hugetlb_fault_mutex_hash(memfd->f_mapping, idx);
+		mutex_lock(&hugetlb_fault_mutex_table[hash]);
+
 		nr_resv = hugetlb_reserve_pages(inode, idx, idx + 1, NULL, EMPTY_VMA_FLAGS);
-		if (nr_resv < 0)
-			return ERR_PTR(nr_resv);
+		if (nr_resv < 0) {
+			err = nr_resv;
+			goto out_unlock;
+		}
 
 		folio = alloc_hugetlb_folio_reserve(h,
 						    numa_node_id(),
 						    NULL,
 						    gfp_mask);
 		if (folio) {
-			u32 hash;
-
 			/*
 			 * Zero the folio to prevent information leaks to userspace.
 			 * Use folio_zero_user() which is optimized for huge/gigantic
@@ -112,20 +121,9 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx)
 			 */
 			__folio_mark_uptodate(folio);
 
-			/*
-			 * Serialize hugepage allocation and instantiation to prevent
-			 * races with concurrent allocations, as required by all other
-			 * callers of hugetlb_add_to_page_cache().
-			 */
-			hash = hugetlb_fault_mutex_hash(memfd->f_mapping, idx);
-			mutex_lock(&hugetlb_fault_mutex_table[hash]);
-
 			err = hugetlb_add_to_page_cache(folio,
 							memfd->f_mapping,
 							idx);
-
-			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
-
 			if (err) {
 				folio_put(folio);
 				goto err_unresv;
@@ -133,11 +131,14 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx)
 
 			hugetlb_set_folio_subpool(folio, subpool_inode(inode));
 			folio_unlock(folio);
+			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 			return folio;
 		}
 err_unresv:
 		if (nr_resv > 0)
 			hugetlb_unreserve_pages(inode, idx, idx + 1, 0);
+out_unlock:
+		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 		return ERR_PTR(err);
 	}
 #endif
diff --git a/mm/memfd_luo.c b/mm/memfd_luo.c
index 59de210bee5f9e..6b68220275ebeb 100644
--- a/mm/memfd_luo.c
+++ b/mm/memfd_luo.c
@@ -121,8 +121,7 @@ static int memfd_luo_preserve_folios(struct file *file,
 	/*
 	 * Pin the folios so they don't move around behind our back. This also
 	 * ensures none of the folios are in CMA -- which ensures they don't
-	 * fall in KHO scratch memory. It also moves swapped out folios back to
-	 * memory.
+	 * fall in KHO bootmem. It also moves swapped out folios back to memory.
 	 *
 	 * A side effect of doing this is that it allocates a folio for all
 	 * indices in the file. This might waste memory on sparse memfds. If
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index a8b03e2920ba8a..b705f9f97f420f 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -96,6 +96,46 @@ void num_poisoned_pages_sub(unsigned long pfn, long i)
 		memblk_nr_poison_sub(pfn, i);
 }
 
+/*
+ * Return the first or the last hardware-poisoned online page in [start,
+ * start + size), or PHYS_ADDR_MAX if the range is clean.
+ */
+static phys_addr_t range_hwpoison(phys_addr_t start, unsigned long size,
+				  bool first)
+{
+	phys_addr_t poison = PHYS_ADDR_MAX;
+	unsigned long pfn, end_pfn;
+
+	if (!size || !atomic_long_read(&num_poisoned_pages))
+		return poison;
+
+	end_pfn = PHYS_PFN(start + size - 1);
+	for (pfn = PHYS_PFN(start); pfn <= end_pfn; pfn++) {
+		struct page *page = pfn_to_online_page(pfn);
+
+		if (page && is_page_hwpoison(page)) {
+			if (first)
+				return PFN_PHYS(pfn);
+
+			poison = PFN_PHYS(pfn);
+		}
+
+		cond_resched();
+	}
+
+	return poison;
+}
+
+phys_addr_t range_first_hwpoison(phys_addr_t start, unsigned long size)
+{
+	return range_hwpoison(start, size, true);
+}
+
+phys_addr_t range_last_hwpoison(phys_addr_t start, unsigned long size)
+{
+	return range_hwpoison(start, size, false);
+}
+
 /**
  * MF_ATTR_RO - Create sysfs entry for each memory failure statistics.
  * @_name: name of the file in the per NUMA sysfs directory.
@@ -847,24 +887,25 @@ static int kill_accessing_process(struct task_struct *p, unsigned long pfn,
 }
 
 /*
- * MF_IGNORED - The m-f() handler marks the page as PG_hwpoisoned'ed.
+ * MF_IGNORED - The m-f() handler marks the page as PG_hwpoison'ed.
  * But it could not do more to isolate the page from being accessed again,
  * nor does it kill the process. This is extremely rare and one of the
  * potential causes is that the page state has been changed due to
  * underlying race condition. This is the most severe outcomes.
  *
- * MF_FAILED - The m-f() handler marks the page as PG_hwpoisoned'ed.
+ * MF_FAILED - The m-f() handler marks the page as PG_hwpoison'ed.
  * It should have killed the process, but it can't isolate the page,
  * due to conditions such as extra pin, unmap failure, etc. Accessing
  * the page again may trigger another MCE and the process will be killed
  * by the m-f() handler immediately.
  *
- * MF_DELAYED - The m-f() handler marks the page as PG_hwpoisoned'ed.
- * The page is unmapped, and is removed from the LRU or file mapping.
- * An attempt to access the page again will trigger page fault and the
- * PF handler will kill the process.
+ * MF_DELAYED - The m-f() handler marks the page as PG_hwpoison'ed.
+ * It means the page was unmapped and partially isolated (e.g. removed from
+ * file mapping or the LRU) but full cleanup is deferred (e.g. the metadata
+ * for the memory, as in struct page/folio, is still referenced). Any
+ * further access to the page will result in the process being killed.
  *
- * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoisoned'ed.
+ * MF_RECOVERED - The m-f() handler marks the page as PG_hwpoison'ed.
  * The page has been completely isolated, that is, unmapped, taken out of
  * the buddy system, or hole-punched out of the file mapping.
  */
@@ -938,10 +979,12 @@ static int truncate_error_folio(struct folio *folio, unsigned long pfn,
 	if (mapping->a_ops->error_remove_folio) {
 		int err = mapping->a_ops->error_remove_folio(mapping, folio);
 
-		if (err != 0)
+		if (err == MF_DELAYED)
+			ret = err;
+		else if (err != 0)
 			pr_info("%#lx: Failed to punch page: %d\n", pfn, err);
 		else if (!filemap_release_folio(folio, GFP_NOIO))
-			pr_info("%#lx: failed to release buffers\n", pfn);
+			pr_info("%#lx: Failed to release buffers\n", pfn);
 		else
 			ret = MF_RECOVERED;
 	} else {
@@ -1036,18 +1079,14 @@ static int me_pagecache_clean(struct page_state *ps, struct page *p)
 		goto out;
 	}
 
-	/*
-	 * The shmem page is kept in page cache instead of truncating
-	 * so is expected to have an extra refcount after error-handling.
-	 */
-	extra_pins = shmem_mapping(mapping);
-
 	/*
 	 * Truncation is a bit tricky. Enable it per file system for now.
 	 *
 	 * Open: to take i_rwsem or not for this? Right now we don't.
 	 */
 	ret = truncate_error_folio(folio, page_to_pfn(p), mapping);
+
+	extra_pins = ret == MF_DELAYED;
 	if (has_extra_refcount(ps, p, extra_pins))
 		ret = MF_FAILED;
 
@@ -2561,7 +2600,7 @@ int memory_failure(unsigned long pfn, int flags)
 	/*
 	 * We're only intended to deal with the non-Compound page here.
 	 * The page cannot become compound pages again as folio has been
-	 * splited and extra refcnt is held.
+	 * split and extra refcnt is held.
 	 */
 	WARN_ON(folio_test_large(folio));
 
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index 54851d8a195b03..25e121851b5862 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -370,7 +370,7 @@ int next_demotion_node(int node, const nodemask_t *allowed_mask)
 	 * closest demotion target.
 	 */
 	nodes_complement(mask, *allowed_mask);
-	return find_next_best_node(node, &mask);
+	return find_next_best_node_in(node, &mask, &node_states[N_MEMORY]);
 }
 
 static void disable_all_demotion_targets(void)
@@ -450,7 +450,7 @@ static void establish_demotion_targets(void)
 		memtier = list_next_entry(memtier, list);
 		tier_nodes = get_memtier_nodemask(memtier);
 		/*
-		 * find_next_best_node, use 'used' nodemask as a skip list.
+		 * find_next_best_node_in, use 'used' nodemask as a skip list.
 		 * Add all memory nodes except the selected memory tier
 		 * nodelist to skip list so that we find the best node from the
 		 * memtier nodelist.
@@ -463,7 +463,8 @@ static void establish_demotion_targets(void)
 		 * in the preferred mask when allocating pages during demotion.
 		 */
 		do {
-			target = find_next_best_node(node, &tier_nodes);
+			target = find_next_best_node_in(node, &tier_nodes,
+							&node_states[N_MEMORY]);
 			if (target == NUMA_NO_NODE)
 				break;
 
diff --git a/mm/memory.c b/mm/memory.c
index 8b0c2c735d3de7..0d9db2aec58682 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -492,34 +492,10 @@ static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
 			add_mm_counter(mm, i, rss[i]);
 }
 
-static bool is_bad_page_map_ratelimited(void)
-{
-	static unsigned long resume;
-	static unsigned long nr_shown;
-	static unsigned long nr_unshown;
+/* Allow a burst of 60 bad page map reports per minute. */
+static DEFINE_RATELIMIT_STATE(bad_page_map_ratelimit, 60 * HZ, 60);
 
-	/*
-	 * Allow a burst of 60 reports, then keep quiet for that minute;
-	 * or allow a steady drip of one report per second.
-	 */
-	if (nr_shown == 60) {
-		if (time_before(jiffies, resume)) {
-			nr_unshown++;
-			return true;
-		}
-		if (nr_unshown) {
-			pr_alert("BUG: Bad page map: %lu messages suppressed\n",
-				 nr_unshown);
-			nr_unshown = 0;
-		}
-		nr_shown = 0;
-	}
-	if (nr_shown++ == 0)
-		resume = jiffies + 60 * HZ;
-	return false;
-}
-
-static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size)
+void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size)
 {
 	if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) {
 		snprintf(buf, buf_size, "overflow");
@@ -546,19 +522,6 @@ static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry
 	}
 }
 
-#define ptval_to_str(buf, val)								\
-	do {										\
-		auto __val = (val);							\
-											\
-		ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val));	\
-	} while (0)
-
-#if defined(__SIZEOF_INT128__)
-#define PTVAL_STR_MAX	(32 + 1) /* Max 128-bit value in hex + NUL */
-#else
-#define PTVAL_STR_MAX	(16 + 1) /* Max 64-bit value in hex + NUL */
-#endif
-
 static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr)
 {
 	char pgd_str[PTVAL_STR_MAX];
@@ -633,7 +596,7 @@ static void print_bad_page_map(struct vm_area_struct *vma,
 	char entry_str[PTVAL_STR_MAX];
 	pgoff_t index, anon_index;
 
-	if (is_bad_page_map_ratelimited())
+	if (!__ratelimit(&bad_page_map_ratelimit))
 		return;
 
 	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
@@ -1328,8 +1291,8 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
 	 * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
 	 * error handling here, assume that exclusive mmap_lock on dst and src
 	 * protects anon from unexpected THP transitions; with shmem and file
-	 * protected by mmap_lock-less collapse skipping areas with anon_vma
-	 * (whereas vma_needs_copy() skips areas without anon_vma).  A rework
+	 * protected by mmap_lock-less collapse skipping areas with an anon rmap
+	 * (whereas vma_needs_copy() skips areas without one).  A rework
 	 * can remove such assumptions later, but this is good enough for now.
 	 */
 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
@@ -1570,10 +1533,10 @@ vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
 	if (dst_vma->vm_flags & VM_COPY_ON_FORK)
 		return true;
 	/*
-	 * The presence of an anon_vma indicates an anonymous VMA has page
-	 * tables which naturally cannot be reconstituted on page fault.
+	 * The presence of an anon rmap indicates the VMA may map anonymous
+	 * folios which naturally cannot be reconstituted on page fault.
 	 */
-	if (src_vma->anon_vma)
+	if (vma_has_anon_rmap(src_vma))
 		return true;
 
 	/*
@@ -1601,7 +1564,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
 	if (!vma_needs_copy(dst_vma, src_vma))
 		return 0;
 
-	if (is_vm_hugetlb_page(src_vma))
+	if (vma_is_hugetlb(src_vma))
 		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
 
 	/*
@@ -2215,7 +2178,7 @@ static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	if (vma->vm_file && !reaping)
 		uprobe_munmap(vma, start, end);
 
-	if (unlikely(is_vm_hugetlb_page(vma))) {
+	if (unlikely(vma_is_hugetlb(vma))) {
 		zap_flags_t zap_flags = details ? details->zap_flags : 0;
 
 		VM_WARN_ON_ONCE(reaping);
@@ -2350,7 +2313,7 @@ void zap_vma_range_batched(struct mmu_gather *tlb,
 	 */
 	__zap_vma_range(tlb, vma, address, end, details);
 	mmu_notifier_invalidate_range_end(&range);
-	if (is_vm_hugetlb_page(vma)) {
+	if (vma_is_hugetlb(vma)) {
 		/*
 		 * flush tlb and free resources before hugetlb_zap_end(), to
 		 * avoid concurrent page faults' allocation failure.
@@ -2380,19 +2343,19 @@ void zap_vma_range(struct vm_area_struct *vma, unsigned long address,
 }
 
 /**
- * zap_special_vma_range - zap all page table entries in a special vma range
+ * zap_special_vma_range - zap all page table entries in a kernel-owned VMA
  * @vma: the vma covering the range to zap
  * @address: starting address of the range to zap
  * @size: number of bytes to zap
  *
  * This function does nothing when the provided address range is not fully
- * contained in @vma, or when the @vma is not VM_PFNMAP or VM_MIXEDMAP.
+ * contained in @vma, or when @vma is not kernel-owned.
  */
 void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address,
 		unsigned long size)
 {
 	if (!range_in_vma(vma, address, address + size) ||
-	   !(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)))
+	   !vma_is_kernel_owned(vma))
 		return;
 
 	zap_vma_range(vma, address, size);
@@ -2454,11 +2417,11 @@ static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma)
 	 * be problematic as soon as the zeropage gets replaced by a different
 	 * page due to vma->vm_ops->pfn_mkwrite, because what's mapped would
 	 * now differ to what GUP looked up. FSDAX is incompatible to
-	 * FOLL_LONGTERM and VM_IO is incompatible to GUP completely (see
-	 * check_vma_flags).
+	 * FOLL_LONGTERM and memory-mapped I/O is incompatible to GUP completely
+	 * (see vma_can_gup()).
 	 */
 	return vma->vm_ops && vma->vm_ops->pfn_mkwrite &&
-	       (vma_is_fsdax(vma) || vma->vm_flags & VM_IO);
+	       (vma_is_fsdax(vma) || vma_test(vma, VMA_IO_BIT));
 }
 
 static int validate_page_before_insert(struct vm_area_struct *vma,
@@ -2565,20 +2528,22 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
 	unsigned long curr_page_idx = 0;
 	unsigned long remaining_pages_total = *num;
 	unsigned long pages_to_write_in_pmd;
-	int ret;
+	int err = 0;
 more:
-	ret = -EFAULT;
 	pmd = walk_to_pmd(mm, addr);
-	if (!pmd)
+	if (!pmd) {
+		err = -ENOMEM;
 		goto out;
+	}
 
 	pages_to_write_in_pmd = min_t(unsigned long,
 		remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
 
 	/* Allocate the PTE if necessary; takes PMD lock once only. */
-	ret = -ENOMEM;
-	if (pte_alloc(mm, pmd))
+	if (pte_alloc(mm, pmd)) {
+		err = -ENOMEM;
 		goto out;
+	}
 
 	while (pages_to_write_in_pmd) {
 		int pte_idx = 0;
@@ -2586,15 +2551,14 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
 
 		start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
 		if (!start_pte) {
-			ret = -EFAULT;
+			err = -EFAULT;
 			goto out;
 		}
 		for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
-			int err = insert_page_in_batch_locked(vma, pte,
-				addr, pages[curr_page_idx], prot);
+			err = insert_page_in_batch_locked(vma, pte, addr,
+							  pages[curr_page_idx], prot);
 			if (unlikely(err)) {
 				pte_unmap_unlock(start_pte, pte_lock);
-				ret = err;
 				remaining_pages_total -= pte_idx;
 				goto out;
 			}
@@ -2607,10 +2571,9 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
 	}
 	if (remaining_pages_total)
 		goto more;
-	ret = 0;
 out:
 	*num = remaining_pages_total;
-	return ret;
+	return err;
 }
 
 /**
@@ -2646,17 +2609,23 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
 }
 EXPORT_SYMBOL(vm_insert_pages);
 
+static void __map_kernel_pages_prepare(struct vm_area_desc *desc)
+{
+	if (vma_desc_test(desc, VMA_MIXEDMAP_BIT))
+		return;
+
+	VM_WARN_ON_ONCE(mmap_read_trylock(desc->mm));
+	VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT));
+	vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT);
+}
+
 int map_kernel_pages_prepare(struct vm_area_desc *desc)
 {
 	const struct mmap_action *action = &desc->action;
 	const unsigned long addr = action->map_kernel.start;
 	unsigned long nr_pages, end;
 
-	if (!vma_desc_test(desc, VMA_MIXEDMAP_BIT)) {
-		VM_WARN_ON_ONCE(mmap_read_trylock(desc->mm));
-		VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_PFNMAP_BIT));
-		vma_desc_set_flags(desc, VMA_MIXEDMAP_BIT);
-	}
+	__map_kernel_pages_prepare(desc);
 
 	nr_pages = action->map_kernel.nr_pages;
 	end = addr + PAGE_SIZE * nr_pages;
@@ -2665,7 +2634,6 @@ int map_kernel_pages_prepare(struct vm_area_desc *desc)
 
 	return 0;
 }
-EXPORT_SYMBOL(map_kernel_pages_prepare);
 
 int map_kernel_pages_complete(struct vm_area_struct *vma,
 			      struct mmap_action *action)
@@ -2677,7 +2645,98 @@ int map_kernel_pages_complete(struct vm_area_struct *vma,
 			    action->map_kernel.pages,
 			    &nr_pages, vma->vm_page_prot);
 }
-EXPORT_SYMBOL(map_kernel_pages_complete);
+
+int map_discontig_kernel_pages_prepare(struct vm_area_desc *desc)
+{
+	const struct mmap_action *action = &desc->action;
+	const struct discontig_kernel_page_ops *ops =
+		action->map_kernel_discontig.ops;
+
+	/* At minimum need to be able to get pages. */
+	if (WARN_ON_ONCE(!ops || !ops->get))
+		return -EINVAL;
+
+	__map_kernel_pages_prepare(desc);
+	return 0;
+}
+
+static int apply_discontig_action(struct vm_area_struct *vma,
+				  struct discontig_kernel_page_state *state)
+{
+	unsigned long nr_pages = state->__nr_pages;
+	unsigned long addr = state->addr;
+	unsigned long i;
+
+	if (state->action == DISCONTIG_KERNEL_PAGE_MAP_PAGE)
+		return insert_page(vma, addr, state->__page,
+				   vma->vm_page_prot, /*mkwrite=*/false);
+	if (state->action == DISCONTIG_KERNEL_PAGE_MAP_PAGE_RANGE)
+		return insert_pages(vma, addr, state->__page_arr,
+				    &nr_pages, vma->vm_page_prot);
+
+	/* Compound folio - have to iterate through each page. */
+	for (i = 0; i < nr_pages; i++, addr += PAGE_SIZE) {
+		struct page *page = folio_page(state->__folio, i);
+		int err;
+
+		err = insert_page(vma, addr, page, vma->vm_page_prot,
+				  /*mkwrite=*/false);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
+int map_discontig_kernel_pages_complete(struct vm_area_struct *vma,
+					struct mmap_action *action)
+{
+	const struct discontig_kernel_page_ops *ops =
+		action->map_kernel_discontig.ops;
+	struct discontig_kernel_page_state state = {
+		.start = vma->vm_start,
+		.end = vma->vm_end,
+		.addr = vma->vm_start,
+		.pgoff = vma->vm_pgoff,
+		.nr_pages_mapped = 0,
+		.nr_pages_remain = vma_pages(vma),
+		.vm_private_data = vma->vm_private_data,
+		.private = action->map_kernel_discontig.init_private,
+	};
+	int err = 0;
+
+	if (ops->init)
+		err = ops->init(vma->vm_private_data, &state.private);
+	if (err)
+		return err;
+
+	do {
+		unsigned long end, pgoff_end;
+		unsigned long nr_pages;
+
+		/* Default to abort. */
+		state.action = DISCONTIG_KERNEL_PAGE_ABORT;
+		err = ops->get(&state);
+		if (err || state.action == DISCONTIG_KERNEL_PAGE_ABORT)
+			return err;
+		nr_pages = state.__nr_pages;
+
+		if (!nr_pages || nr_pages > state.nr_pages_remain)
+			return -EINVAL;
+		end = state.addr + PAGE_SIZE * nr_pages;
+		pgoff_end = state.pgoff + nr_pages;
+
+		err = apply_discontig_action(vma, &state);
+		if (err)
+			return err;
+
+		state.addr = end;
+		state.pgoff = pgoff_end;
+		state.nr_pages_mapped += nr_pages;
+		state.nr_pages_remain -= nr_pages;
+	} while (state.addr < vma->vm_end);
+
+	return 0;
+}
 
 /**
  * vm_insert_page - insert single page into user vma
@@ -2897,6 +2956,9 @@ vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
 	BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma));
 	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
 
+	if (unlikely(is_zero_pfn(pfn)))
+		return VM_FAULT_SIGBUS;
+
 	if (addr < vma->vm_start || addr >= vma->vm_end)
 		return VM_FAULT_SIGBUS;
 
@@ -3938,10 +4000,10 @@ static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf)
  *
  * When preparing to insert an anonymous page into a VMA from a
  * fault handler, call this function rather than anon_vma_prepare().
- * If this vma does not already have an associated anon_vma and we are
+ * If this vma does not already have an anon rmap and we are
  * only protected by the per-VMA lock, the caller must retry with the
  * mmap_lock held.  __anon_vma_prepare() will look at adjacent VMAs to
- * determine if this VMA can share its anon_vma, and that's not safe to
+ * determine if this VMA can share its anon rmap, and that's not safe to
  * do with only the per-VMA lock held for this VMA.
  *
  * Return: 0 if fault handling can proceed.  Any other value should be
@@ -3952,7 +4014,7 @@ vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf)
 	struct vm_area_struct *vma = vmf->vma;
 	vm_fault_t ret = 0;
 
-	if (likely(vma->anon_vma))
+	if (likely(vma_has_anon_rmap(vma)))
 		return 0;
 	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
 		if (!mmap_read_trylock(vma->vm_mm))
@@ -4926,18 +4988,21 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 				goto unlock;
 
 			/*
-			 * Get a page reference while we know the page can't be
-			 * freed.
+			 * Get a folio reference while we know the folio can't
+			 * be freed.
 			 */
-			if (trylock_page(vmf->page)) {
+			folio = page_folio(vmf->page);
+			if (folio_trylock(folio)) {
 				struct dev_pagemap *pgmap;
 
-				get_page(vmf->page);
+				folio_get(folio);
 				pte_unmap_unlock(vmf->pte, vmf->ptl);
 				pgmap = page_pgmap(vmf->page);
 				ret = pgmap->ops->migrate_to_ram(vmf);
-				unlock_page(vmf->page);
-				put_page(vmf->page);
+				/* migrate_to_ram() might have split the folio. */
+				folio = page_folio(vmf->page);
+				folio_unlock(folio);
+				folio_put(folio);
 			} else {
 				pte_unmap(vmf->pte);
 				softleaf_entry_wait_on_locked(entry, vmf->ptl);
@@ -4953,10 +5018,13 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 		goto out;
 	}
 
-	/* Prevent swapoff from happening to us. */
+	/* Prevent swapoff from happening to us, and reject a bad entry. */
 	si = get_swap_device(entry);
-	if (unlikely(!si))
+	if (IS_ERR_OR_NULL(si)) {
+		if (IS_ERR(si))
+			ret = VM_FAULT_SIGBUS;
 		goto out;
+	}
 
 	folio = swap_cache_get_folio(entry);
 	if (folio)
@@ -5097,7 +5165,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	 * have changed, so sub pages might got charged to the wrong cgroup,
 	 * or even should be shmem. So we have to free it and fallback.
 	 * Nothing should have touched it, both anon and shmem checks if a
-	 * large folio is fully appliable before use.
+	 * large folio is fully applicable before use.
 	 *
 	 * This will be removed once we unify folio allocation in the swap cache
 	 * layer, where allocation of a folio stabilizes the swap entries.
@@ -5265,7 +5333,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	if (vmf->pte)
 		pte_unmap_unlock(vmf->pte, vmf->ptl);
 out:
-	if (si)
+	if (!IS_ERR_OR_NULL(si))
 		put_swap_device(si);
 	return ret;
 out_nomap:
@@ -6817,7 +6885,6 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
 				 !vma_is_cow_mapping(vma)))
 			return VM_FAULT_SIGSEGV;
 	}
-#ifdef CONFIG_PER_VMA_LOCK
 	/*
 	 * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
 	 * the assumption that lock is dropped on VM_FAULT_RETRY.
@@ -6826,7 +6893,6 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
 			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
 			(FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
 		return VM_FAULT_SIGSEGV;
-#endif
 
 	return 0;
 }
@@ -6870,7 +6936,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
 
 	lru_gen_enter_fault(vma);
 
-	if (unlikely(is_vm_hugetlb_page(vma)))
+	if (unlikely(vma_is_hugetlb(vma)))
 		ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
 	else
 		ret = __handle_mm_fault(vma, address, flags);
@@ -7053,7 +7119,8 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args)
 	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
 		goto out;
 
-	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
+	/* Only mappings GUP cannot handle are followed here. */
+	if (vma_can_gup(vma))
 		goto out;
 retry:
 	pgdp = pgd_offset(mm, address);
@@ -7164,6 +7231,12 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
 	bool writable;
 	struct follow_pfnmap_args args = { .vma = vma, .address = addr };
 
+	/*
+	 * Limit access to one page at a time, as that's what follow_pfnmap_start()
+	 * guarantees; expect the caller to retry to read larger ranges.
+	 */
+	len = min_t(int, len, PAGE_SIZE - offset);
+
 retry:
 	if (follow_pfnmap_start(&args))
 		return -EINVAL;
@@ -7175,7 +7248,7 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
 	if ((write & FOLL_WRITE) && !writable)
 		return -EINVAL;
 
-	maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
+	maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot);
 	if (!maddr)
 		return -ENOMEM;
 
@@ -7183,7 +7256,7 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
 		goto out_unmap;
 
 	if ((pgprot_val(prot) != pgprot_val(args.pgprot)) ||
-	    (phys_addr != (args.pfn << PAGE_SHIFT)) ||
+	    (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) ||
 	    (writable != args.writable)) {
 		follow_pfnmap_end(&args);
 		iounmap(maddr);
@@ -7247,8 +7320,9 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
 			}
 
 			/*
-			 * Check if this is a VM_IO | VM_PFNMAP VMA, which
-			 * we can access using slightly different code.
+			 * GUP failed, perhaps because this is a mapping it
+			 * cannot handle (see vma_can_gup()) - such mappings may
+			 * provide access via vm_ops->access() instead.
 			 */
 			bytes = 0;
 #ifdef CONFIG_HAVE_IOREMAP_PROT
@@ -7326,13 +7400,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr,
 }
 EXPORT_SYMBOL_GPL(access_process_vm);
 
-#ifdef CONFIG_BPF_SYSCALL
 /*
  * Copy a string from another process's address space as given in mm.
  * If there is any error return -EFAULT.
  */
-static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
-				void *buf, int len, unsigned int gup_flags)
+int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr,
+		void *buf, int len, unsigned int gup_flags)
 {
 	void *old_buf = buf;
 	int err = 0;
@@ -7408,44 +7481,6 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
 	return buf - old_buf;
 }
 
-/**
- * copy_remote_vm_str - copy a string from another process's address space.
- * @tsk:	the task of the target address space
- * @addr:	start address to read from
- * @buf:	destination buffer
- * @len:	number of bytes to copy
- * @gup_flags:	flags modifying lookup behaviour
- *
- * The caller must hold a reference on @mm.
- *
- * Return: number of bytes copied from @addr (source) to @buf (destination);
- * not including the trailing NUL. Always guaranteed to leave NUL-terminated
- * buffer. On any error, return -EFAULT.
- */
-int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
-		       void *buf, int len, unsigned int gup_flags)
-{
-	struct mm_struct *mm;
-	int ret;
-
-	if (unlikely(len == 0))
-		return 0;
-
-	mm = get_task_mm(tsk);
-	if (!mm) {
-		*(char *)buf = '\0';
-		return -EFAULT;
-	}
-
-	ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags);
-
-	mmput(mm);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(copy_remote_vm_str);
-#endif /* CONFIG_BPF_SYSCALL */
-
 /*
  * Print the name of a VMA.
  */
@@ -7734,12 +7769,12 @@ void ptlock_free(struct ptdesc *ptdesc)
 
 void vma_pgtable_walk_begin(struct vm_area_struct *vma)
 {
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		hugetlb_vma_lock_read(vma);
 }
 
 void vma_pgtable_walk_end(struct vm_area_struct *vma)
 {
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		hugetlb_vma_unlock_read(vma);
 }
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 226ab9cb078ad1..3076f68925106f 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -43,6 +43,7 @@
 #include "mm_init.h"
 #include "page_alloc.h"
 #include "shuffle.h"
+#include "sparse.h"
 
 enum {
 	MEMMAP_ON_MEMORY_DISABLE = 0,
@@ -344,6 +345,8 @@ struct page *pfn_to_online_page(unsigned long pfn)
 	struct dev_pagemap *pgmap;
 	struct mem_section *ms;
 
+	BUILD_BUG_ON(IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID));
+
 	if (nr >= NR_MEM_SECTIONS)
 		return NULL;
 
@@ -351,13 +354,6 @@ struct page *pfn_to_online_page(unsigned long pfn)
 	if (!online_section(ms))
 		return NULL;
 
-	/*
-	 * Save some code text when online_section() +
-	 * pfn_section_valid() are sufficient.
-	 */
-	if (IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) && !pfn_valid(pfn))
-		return NULL;
-
 	if (!pfn_section_valid(ms, pfn))
 		return NULL;
 
@@ -425,49 +421,39 @@ int __add_pages(int nid, unsigned long pfn, unsigned long nr_pages,
 	return err;
 }
 
-/* find the smallest valid pfn in the range [start_pfn, end_pfn) */
-static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,
-				     unsigned long start_pfn,
-				     unsigned long end_pfn)
+static bool subsection_overlaps_zone(unsigned long pfn, struct zone *zone)
 {
-	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) {
-		if (unlikely(!pfn_to_online_page(start_pfn)))
-			continue;
+	const unsigned long start_pfn = ALIGN_DOWN(pfn, PAGES_PER_SUBSECTION);
+	const unsigned long end_pfn = start_pfn + PAGES_PER_SUBSECTION - 1;
 
-		if (unlikely(pfn_to_nid(start_pfn) != nid))
-			continue;
+	/* All pages in a subsection are either online or offline. */
+	if (unlikely(!pfn_to_online_page(start_pfn)))
+		return false;
 
-		if (zone != page_zone(pfn_to_page(start_pfn)))
-			continue;
+	/* Checking start+end is sufficient. */
+	return zone == page_zone(pfn_to_page(start_pfn)) ||
+	       zone == page_zone(pfn_to_page(end_pfn));
+}
 
-		return start_pfn;
+/* find the smallest valid pfn in the range [start_pfn, end_pfn) */
+static unsigned long find_smallest_section_pfn(struct zone *zone,
+		unsigned long start_pfn, unsigned long end_pfn)
+{
+	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) {
+		if (subsection_overlaps_zone(start_pfn, zone))
+			return start_pfn;
 	}
-
 	return 0;
 }
 
 /* find the biggest valid pfn in the range [start_pfn, end_pfn). */
-static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,
-				    unsigned long start_pfn,
-				    unsigned long end_pfn)
+static unsigned long find_biggest_section_pfn(struct zone *zone,
+		unsigned long start_pfn, unsigned long end_pfn)
 {
-	unsigned long pfn;
-
-	/* pfn is the end pfn of a memory section. */
-	pfn = end_pfn - 1;
-	for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) {
-		if (unlikely(!pfn_to_online_page(pfn)))
-			continue;
-
-		if (unlikely(pfn_to_nid(pfn) != nid))
-			continue;
-
-		if (zone != page_zone(pfn_to_page(pfn)))
-			continue;
-
-		return pfn;
+	for (; end_pfn > start_pfn; end_pfn -= PAGES_PER_SUBSECTION) {
+		if (subsection_overlaps_zone(end_pfn - 1, zone))
+			return end_pfn - 1;
 	}
-
 	return 0;
 }
 
@@ -475,7 +461,6 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
 			     unsigned long end_pfn)
 {
 	unsigned long pfn;
-	int nid = zone_to_nid(zone);
 
 	if (zone->zone_start_pfn == start_pfn) {
 		/*
@@ -484,7 +469,7 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
 		 * In this case, we find second smallest valid mem_section
 		 * for shrinking zone.
 		 */
-		pfn = find_smallest_section_pfn(nid, zone, end_pfn,
+		pfn = find_smallest_section_pfn(zone, end_pfn,
 						zone_end_pfn(zone));
 		if (pfn) {
 			zone->spanned_pages = zone_end_pfn(zone) - pfn;
@@ -500,7 +485,7 @@ static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
 		 * In this case, we find second biggest valid mem_section for
 		 * shrinking zone.
 		 */
-		pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn,
+		pfn = find_biggest_section_pfn(zone, zone->zone_start_pfn,
 					       start_pfn);
 		if (pfn)
 			zone->spanned_pages = pfn - zone->zone_start_pfn + 1;
@@ -554,24 +539,20 @@ void remove_pfn_range_from_zone(struct zone *zone,
 		/* Select all remaining pages up to the next section boundary */
 		cur_nr_pages =
 			min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn);
-		page_init_poison(pfn_to_page(pfn),
-				 sizeof(struct page) * cur_nr_pages);
+		if (!section_vmemmap_optimizable(__pfn_to_section(pfn)))
+			page_init_poison(pfn_to_page(pfn),
+					 sizeof(struct page) * cur_nr_pages);
 	}
 
 	/*
 	 * Zone shrinking code cannot properly deal with ZONE_DEVICE. So
-	 * we will not try to shrink the zones - which is okay as
-	 * set_zone_contiguous() cannot deal with ZONE_DEVICE either way.
+	 * we will not try to shrink it.
 	 */
 	if (zone_is_zone_device(zone))
 		return;
 
-	clear_zone_contiguous(zone);
-
 	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
 	update_pgdat_span(pgdat);
-
-	set_zone_contiguous(zone);
 }
 
 /**
@@ -749,8 +730,6 @@ void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
 	struct pglist_data *pgdat = zone->zone_pgdat;
 	int nid = pgdat->node_id;
 
-	clear_zone_contiguous(zone);
-
 	if (zone_is_empty(zone))
 		init_currently_empty_zone(zone, start_pfn, nr_pages);
 	resize_zone_range(zone, start_pfn, nr_pages);
@@ -778,8 +757,6 @@ void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
 	memmap_init_range(nr_pages, nid, zone_idx(zone), start_pfn, 0,
 			 MEMINIT_HOTPLUG, altmap, migratetype,
 			 isolate_pageblock);
-
-	set_zone_contiguous(zone);
 }
 
 struct auto_movable_stats {
@@ -1075,6 +1052,7 @@ void adjust_present_page_count(struct page *page, struct memory_group *group,
 	if (early_section(__pfn_to_section(page_to_pfn(page))))
 		zone->present_early_pages += nr_pages;
 	zone->present_pages += nr_pages;
+	zone->pages_with_online_memmap += nr_pages;
 	zone->zone_pgdat->node_present_pages += nr_pages;
 
 	if (group && movable)
@@ -1785,6 +1763,37 @@ bool mhp_range_allowed(u64 start, u64 size, bool need_mapping)
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
+
+static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg)
+{
+	int nid = *(int *)arg;
+
+	/*
+	 * If a memory block belongs to multiple nodes, the stored nid is not
+	 * reliable. However, such blocks are always online (e.g., cannot get
+	 * offlined) and, therefore, are still spanned by the node.
+	 */
+	return mem->nid == nid ? -EEXIST : 0;
+}
+
+/* Caller must hold the memory hotplug lock for this check. */
+static bool node_is_memoryless(int nid)
+{
+	/*
+	 * A node still spanning pages (especially ZONE_DEVICE) is not
+	 * memoryless.  A node spans memory after move_pfn_range_to_zone(),
+	 * e.g. once a memory block has been onlined.
+	 */
+	if (node_spanned_pages(nid))
+		return false;
+	/*
+	 * Offline memory blocks may not be spanned by the node yet, but they
+	 * link to it in sysfs and can be onlined later, so the node is not
+	 * memoryless while any remain.
+	 */
+	return !for_each_memory_block(&nid, check_no_memblock_for_node_cb);
+}
+
 /*
  * Scan pfn range [start,end) to find movable/migratable pages (LRU and
  * hugetlb folio, movable_ops pages). Will skip over most unmovable
@@ -2214,18 +2223,6 @@ static int check_cpu_on_node(int nid)
 	return 0;
 }
 
-static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg)
-{
-	int nid = *(int *)arg;
-
-	/*
-	 * If a memory block belongs to multiple nodes, the stored nid is not
-	 * reliable. However, such blocks are always online (e.g., cannot get
-	 * offlined) and, therefore, are still spanned by the node.
-	 */
-	return mem->nid == nid ? -EEXIST : 0;
-}
-
 /**
  * try_offline_node
  * @nid: the node ID
@@ -2237,23 +2234,7 @@ static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg)
  */
 void try_offline_node(int nid)
 {
-	int rc;
-
-	/*
-	 * If the node still spans pages (especially ZONE_DEVICE), don't
-	 * offline it. A node spans memory after move_pfn_range_to_zone(),
-	 * e.g., after the memory block was onlined.
-	 */
-	if (node_spanned_pages(nid))
-		return;
-
-	/*
-	 * Especially offline memory blocks might not be spanned by the
-	 * node. They will get spanned by the node once they get onlined.
-	 * However, they link to the node in sysfs and can get onlined later.
-	 */
-	rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb);
-	if (rc)
+	if (!node_is_memoryless(nid))
 		return;
 
 	if (check_cpu_on_node(nid))
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 79053ece02cd48..40744658483b27 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -112,6 +112,7 @@
 #include <linux/printk.h>
 #include <linux/leafops.h>
 #include <linux/gcd.h>
+#include <linux/srcu.h>
 
 #include <asm/tlbflush.h>
 #include <asm/tlb.h>
@@ -157,6 +158,7 @@ static const int weightiness = 32;
  */
 struct weighted_interleave_state {
 	bool mode_auto;
+	struct rcu_head rcu;
 	u8 iw_table[];
 };
 static struct weighted_interleave_state __rcu *wi_state;
@@ -168,6 +170,24 @@ static unsigned int *node_bw_table;
  */
 static DEFINE_MUTEX(wi_state_lock);
 
+/* Readers that sleep while walking iw_table hold this instead */
+DEFINE_STATIC_SRCU_FAST(wi_srcu);
+
+static void wi_state_free_rcu(struct rcu_head *head)
+{
+	struct weighted_interleave_state *state =
+		container_of(head, struct weighted_interleave_state, rcu);
+
+	kfree_rcu(state, rcu);
+}
+
+/* Retire through both flavors: sleeping readers use SRCU, the rest RCU */
+static void wi_state_retire(struct weighted_interleave_state *state)
+{
+	if (state)
+		call_srcu(&wi_srcu, &state->rcu, wi_state_free_rcu);
+}
+
 static u8 get_il_weight(int node)
 {
 	struct weighted_interleave_state *state;
@@ -266,10 +286,7 @@ int mempolicy_set_node_perf(unsigned int node, struct access_coordinate *coords)
 	rcu_assign_pointer(wi_state, new_wi_state);
 
 	mutex_unlock(&wi_state_lock);
-	if (old_wi_state) {
-		synchronize_rcu();
-		kfree(old_wi_state);
-	}
+	wi_state_retire(old_wi_state);
 out:
 	kfree(old_bw);
 	return 0;
@@ -650,7 +667,8 @@ static inline bool queue_folio_required(struct folio *folio,
 	return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
 }
 
-static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
+static void queue_folios_pmd(pmd_t *pmd, unsigned long addr,
+		struct mm_walk *walk)
 {
 	struct folio *folio;
 	struct queue_pages *qp = walk->private;
@@ -661,11 +679,9 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
 			qp->nr_failed++;
 		return;
 	}
-	folio = pmd_folio(pmdval);
-	if (is_huge_zero_folio(folio)) {
-		walk->action = ACTION_CONTINUE;
+	folio = vm_normal_folio_pmd(walk->vma, addr, pmdval);
+	if (!folio || folio_is_zone_device(folio))
 		return;
-	}
 	if (!queue_folio_required(folio, qp))
 		return;
 	if (!(qp->flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
@@ -698,7 +714,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
 
 	ptl = pmd_trans_huge_lock(pmd, vma);
 	if (ptl) {
-		queue_folios_pmd(pmd, walk);
+		queue_folios_pmd(pmd, addr, walk);
 		spin_unlock(ptl);
 		goto out;
 	}
@@ -1992,7 +2008,8 @@ SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
 
 bool vma_migratable(struct vm_area_struct *vma)
 {
-	if (vma->vm_flags & (VM_IO | VM_PFNMAP))
+	/* Pages which GUP cannot obtain cannot be migrated either. */
+	if (!vma_can_gup(vma))
 		return false;
 
 	/*
@@ -2002,7 +2019,7 @@ bool vma_migratable(struct vm_area_struct *vma)
 	if (vma_is_dax(vma))
 		return false;
 
-	if (is_vm_hugetlb_page(vma) &&
+	if (vma_is_hugetlb(vma) &&
 		!hugepage_migration_supported(hstate_vma(vma)))
 		return false;
 
@@ -2180,34 +2197,15 @@ unsigned int mempolicy_slab_node(void)
 	}
 }
 
-static unsigned int read_once_policy_nodemask(struct mempolicy *pol,
-					      nodemask_t *mask)
-{
-	/*
-	 * barrier stabilizes the nodemask locally so that it can be iterated
-	 * over safely without concern for changes. Allocators validate node
-	 * selection does not violate mems_allowed, so this is safe.
-	 */
-	barrier();
-	memcpy(mask, &pol->nodes, sizeof(nodemask_t));
-	barrier();
-	return nodes_weight(*mask);
-}
-
 static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx)
 {
 	struct weighted_interleave_state *state;
-	nodemask_t nodemask;
-	unsigned int target, nr_nodes;
+	unsigned int target, nnodes = 0;
 	u8 *table = NULL;
 	unsigned int weight_total = 0;
 	u8 weight;
 	int nid = 0;
 
-	nr_nodes = read_once_policy_nodemask(pol, &nodemask);
-	if (!nr_nodes)
-		return numa_node_id();
-
 	rcu_read_lock();
 
 	state = rcu_dereference(wi_state);
@@ -2215,22 +2213,45 @@ static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx)
 	if (state)
 		table = state->iw_table;
 
-	/* calculate the total weight */
-	for_each_node_mask(nid, nodemask)
+	/* calculate the total weight and the node count */
+	for_each_node_mask(nid, pol->nodes) {
 		weight_total += table ? table[nid] : 1;
+		nnodes++;
+	}
+
+	/* the mask is empty */
+	if (!weight_total) {
+		rcu_read_unlock();
+		return numa_node_id();
+	}
 
 	/* Calculate the node offset based on totals */
 	target = ilx % weight_total;
-	nid = first_node(nodemask);
-	while (target) {
+	nid = first_node(pol->nodes);
+
+	/*
+	 * The target was calculated in a separate loop, and a concurrent
+	 * rebind can change the contents of pol->nodes as we calculate.
+	 * Access is safe, in the worst case we suddenly perceive an empty
+	 * nodemask and return numa_node_id() below - otherwise we may
+	 * simply cause a skew in allocations.
+	 *
+	 * Clamp this loop to a single pass (nnodes) to keep the walk
+	 * bounded by node count.
+	 */
+	while (target && nnodes-- && nid < MAX_NUMNODES) {
 		/* detect system default usage */
 		weight = table ? table[nid] : 1;
 		if (target < weight)
 			break;
 		target -= weight;
-		nid = next_node_in(nid, nodemask);
+		nid = next_node_in(nid, pol->nodes);
 	}
 	rcu_read_unlock();
+
+	/* the mask emptied under the walk */
+	if (nid >= MAX_NUMNODES)
+		return numa_node_id();
 	return nid;
 }
 
@@ -2241,18 +2262,23 @@ static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx)
  */
 static unsigned int interleave_nid(struct mempolicy *pol, pgoff_t ilx)
 {
-	nodemask_t nodemask;
 	unsigned int target, nnodes;
 	int i;
 	int nid;
 
-	nnodes = read_once_policy_nodemask(pol, &nodemask);
+	nnodes = nodes_weight(pol->nodes);
 	if (!nnodes)
 		return numa_node_id();
 	target = ilx % nnodes;
-	nid = first_node(nodemask);
-	for (i = 0; i < target; i++)
-		nid = next_node(nid, nodemask);
+	nid = first_node(pol->nodes);
+
+	/* A concurrent cpuset rebind may cause us to see an empty nodemask */
+	for (i = 0; i < target && nid < MAX_NUMNODES; i++)
+		nid = next_node_in(nid, pol->nodes);
+
+	/* the mask emptied under the walk */
+	if (nid >= MAX_NUMNODES)
+		return numa_node_id();
 	return nid;
 }
 
@@ -2592,6 +2618,7 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp,
 		struct mempolicy *pol, unsigned long nr_pages,
 		struct page **page_array)
 {
+	unsigned int cpuset_mems_cookie;
 	int nodes;
 	unsigned long nr_pages_per_node;
 	int delta;
@@ -2599,7 +2626,16 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp,
 	unsigned long nr_allocated;
 	unsigned long total_allocated = 0;
 
-	nodes = nodes_weight(pol->nodes);
+	/* count the nodes, retry if a rebind happened during the read */
+	do {
+		cpuset_mems_cookie = read_mems_allowed_begin();
+		nodes = nodes_weight(pol->nodes);
+	} while (read_mems_allowed_retry(cpuset_mems_cookie));
+
+	/* if the nodemask has become invalid, we cannot do anything */
+	if (!nodes)
+		return 0;
+
 	nr_pages_per_node = nr_pages / nodes;
 	delta = nr_pages - nodes * nr_pages_per_node;
 
@@ -2634,10 +2670,10 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
 	unsigned long nr_allocated = 0;
 	unsigned long rounds;
 	unsigned long node_pages, delta;
-	u8 *weights, weight;
+	struct srcu_ctr __percpu *scp;
+	u8 *table, weight;
 	unsigned int weight_total = 0;
 	unsigned long rem_pages = nr_pages;
-	nodemask_t nodes;
 	int nnodes, node;
 	int resume_node = MAX_NUMNODES - 1;
 	u8 resume_weight = 0;
@@ -2647,10 +2683,10 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
 	if (!nr_pages)
 		return 0;
 
-	/* read the nodes onto the stack, retry if done during rebind */
+	/* count the nodes, retry if a rebind happened during the read */
 	do {
 		cpuset_mems_cookie = read_mems_allowed_begin();
-		nnodes = read_once_policy_nodemask(pol, &nodes);
+		nnodes = nodes_weight(pol->nodes);
 	} while (read_mems_allowed_retry(cpuset_mems_cookie));
 
 	/* if the nodemask has become invalid, we cannot do anything */
@@ -2660,7 +2696,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
 	/* Continue allocating from most recent node and adjust the nr_pages */
 	node = me->il_prev;
 	weight = me->il_weight;
-	if (weight && node_isset(node, nodes)) {
+	if (weight && node_isset(node, pol->nodes)) {
 		node_pages = min(rem_pages, weight);
 		nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
 						  page_array);
@@ -2678,25 +2714,18 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
 	me->il_weight = 0;
 	prev_node = node;
 
-	/* create a local copy of node weights to operate on outside rcu */
-	weights = kmalloc(nr_node_ids, gfp & GFP_RECLAIM_MASK);
-	if (!weights)
-		return total_allocated;
-
-	rcu_read_lock();
-	state = rcu_dereference(wi_state);
-	if (state) {
-		memcpy(weights, state->iw_table, nr_node_ids * sizeof(u8));
-		rcu_read_unlock();
-	} else {
-		rcu_read_unlock();
-		for (i = 0; i < nr_node_ids; i++)
-			weights[i] = 1;
-	}
+	/* The page allocator may sleep, pin the weight table with SRCU */
+	scp = srcu_read_lock_fast(&wi_srcu);
+	state = srcu_dereference(wi_state, &wi_srcu);
+	table = state ? state->iw_table : NULL;
 
 	/* calculate total, detect system default usage */
-	for_each_node_mask(node, nodes)
-		weight_total += weights[node];
+	for_each_node_mask(node, pol->nodes)
+		weight_total += table ? table[node] : 1;
+
+	/* the mask emptied since it was counted */
+	if (!weight_total)
+		goto out;
 
 	/*
 	 * Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls.
@@ -2707,11 +2736,15 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
 	 */
 	rounds = rem_pages / weight_total;
 	delta = rem_pages % weight_total;
-	resume_node = next_node_in(prev_node, nodes);
-	resume_weight = weights[resume_node];
+	resume_node = next_node_in(prev_node, pol->nodes);
+	if (resume_node >= MAX_NUMNODES)
+		goto out;
+	resume_weight = table ? table[resume_node] : 1;
 	for (i = 0; i < nnodes; i++) {
-		node = next_node_in(prev_node, nodes);
-		weight = weights[node];
+		node = next_node_in(prev_node, pol->nodes);
+		if (node >= MAX_NUMNODES)
+			break;
+		weight = table ? table[node] : 1;
 		node_pages = weight * rounds;
 		/* If a delta exists, add this node's portion of the delta */
 		if (delta > weight) {
@@ -2727,6 +2760,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
 		/* node_pages can be 0 if an allocation fails and rounds == 0 */
 		if (!node_pages)
 			break;
+		/* a rebind can invalidate the counts: never overrun page_array */
+		node_pages = min(node_pages, nr_pages - total_allocated);
 		nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
 						  page_array);
 		page_array += nr_allocated;
@@ -2737,7 +2772,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
 	}
 	me->il_prev = resume_node;
 	me->il_weight = resume_weight;
-	kfree(weights);
+out:
+	srcu_read_unlock_fast(&wi_srcu, scp);
 	return total_allocated;
 }
 
@@ -3663,10 +3699,7 @@ static ssize_t node_store(struct kobject *kobj, struct kobj_attribute *attr,
 
 	rcu_assign_pointer(wi_state, new_wi_state);
 	mutex_unlock(&wi_state_lock);
-	if (old_wi_state) {
-		synchronize_rcu();
-		kfree(old_wi_state);
-	}
+	wi_state_retire(old_wi_state);
 	return count;
 }
 
@@ -3732,10 +3765,7 @@ static ssize_t weighted_interleave_auto_store(struct kobject *kobj,
 update_wi_state:
 	rcu_assign_pointer(wi_state, new_wi_state);
 	mutex_unlock(&wi_state_lock);
-	if (old_wi_state) {
-		synchronize_rcu();
-		kfree(old_wi_state);
-	}
+	wi_state_retire(old_wi_state);
 	return count;
 }
 
@@ -3779,10 +3809,7 @@ static void wi_state_free(void)
 	rcu_assign_pointer(wi_state, NULL);
 	mutex_unlock(&wi_state_lock);
 
-	if (old_wi_state) {
-		synchronize_rcu();
-		kfree(old_wi_state);
-	}
+	wi_state_retire(old_wi_state);
 }
 
 static struct kobj_attribute wi_auto_attr = {
diff --git a/mm/migrate.c b/mm/migrate.c
index 15b45832bcfa7e..7bdcdb57652f8f 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -776,8 +776,9 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
 {
 	int cpupid;
 
-	if (folio_test_referenced(folio))
-		folio_set_referenced(newfolio);
+	/* Copy the reference state, including PG_referenced */
+	folio_migrate_lru_refs(newfolio, folio);
+
 	if (folio_test_uptodate(folio))
 		folio_mark_uptodate(newfolio);
 	if (folio_test_clear_active(folio)) {
@@ -807,7 +808,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
 	if (folio_test_idle(folio))
 		folio_set_idle(newfolio);
 
-	folio_migrate_refs(newfolio, folio);
 	/*
 	 * Copy NUMA information to the new page, to prevent over-eager
 	 * future migrations of this same page.
@@ -835,7 +835,6 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
 	 */
 	if (folio_test_swapcache(folio))
 		folio_clear_swapcache(folio);
-	folio_clear_private(folio);
 
 	/* page->private contains hugetlb specific flags */
 	if (!folio_test_hugetlb(folio))
@@ -1175,7 +1174,7 @@ static void migrate_folio_undo_src(struct folio *src, int was_mapped,
 {
 	if (was_mapped)
 		remove_migration_ptes(src, src, 0);
-	/* Drop an anon_vma reference if we took one */
+	/* Drop an anon rmap reference if we took one */
 	if (anon_vma)
 		put_anon_vma(anon_vma);
 	if (locked)
@@ -1282,15 +1281,15 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 
 	/*
 	 * By try_to_migrate(), src->mapcount goes down to 0 here. In this case,
-	 * we cannot notice that anon_vma is freed while we migrate a page.
-	 * This get_anon_vma() delays freeing anon_vma pointer until the end
+	 * we cannot notice that the anon rmap is freed while we migrate a page.
+	 * This get_anon_vma() delays freeing the anon rmap until the end
 	 * of migration. File cache pages are no problem because of page_lock()
 	 * File Caches may use write_page() or lock_page() in migration, then,
 	 * just care Anon page here.
 	 *
 	 * Only folio_get_anon_vma() understands the subtleties of
-	 * getting a hold on an anon_vma from outside one of its mms.
-	 * But if we cannot get anon_vma, then we won't need it anyway,
+	 * getting a hold on an anon rmap from outside one of its mms.
+	 * But if we cannot get the anon rmap, then we won't need it anyway,
 	 * because that implies that the anon page is no longer mapped
 	 * (and cannot be remapped so long as we hold the page lock).
 	 */
@@ -1327,7 +1326,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 	 * free the metadata, so the page can be freed.
 	 */
 	if (!src->mapping) {
-		if (folio_test_private(src)) {
+		if (folio_has_attached_private(src)) {
 			try_to_free_buffers(src);
 			goto out;
 		}
@@ -1433,7 +1432,7 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
 	 * and will be freed.
 	 */
 	list_del(&src->lru);
-	/* Drop an anon_vma reference if we took one */
+	/* Drop an anon rmap reference if we took one */
 	if (anon_vma)
 		put_anon_vma(anon_vma);
 	folio_unlock(src);
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 009bfa8b212d5b..b74c0ae4276826 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -739,19 +739,21 @@ static void migrate_vma_unmap(struct migrate_vma *migrate)
  */
 int migrate_vma_setup(struct migrate_vma *args)
 {
+	const struct vm_area_struct *vma = args->vma;
 	long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
 
 	args->start &= PAGE_MASK;
 	args->end &= PAGE_MASK;
-	if (!args->vma || is_vm_hugetlb_page(args->vma) ||
-	    (args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
+	if (!vma)
+		return -EINVAL;
+	if (vma_is_kernel_owned(vma) || vma_is_fixed_mapping(vma) ||
+	    vma_is_dax(vma))
 		return -EINVAL;
 	if (nr_pages <= 0)
 		return -EINVAL;
-	if (args->start < args->vma->vm_start ||
-	    args->start >= args->vma->vm_end)
+	if (args->start < vma->vm_start || args->start >= vma->vm_end)
 		return -EINVAL;
-	if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
+	if (args->end <= vma->vm_start || args->end > vma->vm_end)
 		return -EINVAL;
 	if (!args->src || !args->dst)
 		return -EINVAL;
@@ -1398,9 +1400,9 @@ static unsigned long migrate_device_pfn_lock(unsigned long pfn)
  * @start: starting pfn in the range to migrate.
  * @npages: number of pages to migrate.
  *
- * migrate_vma_setup() is similar in concept to migrate_vma_setup() except that
- * instead of looking up pages based on virtual address mappings a range of
- * device pfns that should be migrated to system memory is used instead.
+ * migrate_device_range() is similar in concept to migrate_vma_setup(), except
+ * that instead of looking up pages based on virtual address mappings, a range
+ * of device pfns that should be migrated to system memory is used.
  *
  * This is useful when a driver needs to free device memory but doesn't know the
  * virtual mappings of every page that may be in device memory. For example this
diff --git a/mm/mincore.c b/mm/mincore.c
index ff4ac828176837..0fe50f8a7e6291 100644
--- a/mm/mincore.c
+++ b/mm/mincore.c
@@ -71,7 +71,7 @@ static unsigned char mincore_swap(swp_entry_t entry, bool shmem)
 	 */
 	if (shmem) {
 		si = get_swap_device(entry);
-		if (!si)
+		if (IS_ERR_OR_NULL(si))
 			return 0;
 	}
 	folio = swap_cache_get_folio(entry);
@@ -235,24 +235,22 @@ static const struct mm_walk_ops mincore_walk_ops = {
 	.pmd_entry		= mincore_pte_range,
 	.pte_hole		= mincore_unmapped_range,
 	.hugetlb_entry		= mincore_hugetlb,
-	.walk_lock		= PGWALK_RDLOCK,
+	.walk_lock		= PGWALK_VMA_RDLOCK_VERIFY,
 };
 
 /*
  * Do a chunk of "sys_mincore()". We've already checked
- * all the arguments, we hold the mmap semaphore: we should
+ * all the arguments, we hold the VMA read lock: we should
  * just return the amount of info we're asked for.
  */
-static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec)
+static long do_mincore(struct vm_area_struct *vma, unsigned long addr,
+		unsigned long pages, unsigned char *vec)
 {
-	struct vm_area_struct *vma;
-	unsigned long end;
+	unsigned long end = min(vma->vm_end, addr + (pages << PAGE_SHIFT));
 	int err;
 
-	vma = vma_lookup(current->mm, addr);
-	if (!vma)
-		return -ENOMEM;
-	end = min(vma->vm_end, addr + (pages << PAGE_SHIFT));
+	vma_assert_locked(vma);
+
 	if (!can_do_mincore(vma)) {
 		unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE);
 		memset(vec, 1, pages);
@@ -319,13 +317,20 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len,
 
 	retval = 0;
 	while (pages) {
+		struct vm_area_struct *vma;
+
+		vma = vma_start_read_unlocked(current->mm, start);
+		if (!vma) {
+			retval = -ENOMEM;
+			break;
+		}
+
 		/*
 		 * Do at most PAGE_SIZE entries per iteration, due to
 		 * the temporary buffer size.
 		 */
-		mmap_read_lock(current->mm);
-		retval = do_mincore(start, min(pages, PAGE_SIZE), tmp);
-		mmap_read_unlock(current->mm);
+		retval = do_mincore(vma, start, min(pages, PAGE_SIZE), tmp);
+		vma_end_read(vma);
 
 		if (retval <= 0)
 			break;
diff --git a/mm/mlock.c b/mm/mlock.c
index 39215a3eab1fbf..4235a1518fc9e4 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -316,22 +316,10 @@ static inline unsigned int folio_mlock_step(struct folio *folio,
 	return folio_pte_batch(folio, pte, ptent, count);
 }
 
-static inline bool allow_mlock_munlock(struct folio *folio,
+static inline bool allow_mlock(struct folio *folio,
 		struct vm_area_struct *vma, unsigned long start,
 		unsigned long end, unsigned int step)
 {
-	/*
-	 * For unlock, allow munlock large folio which is partially
-	 * mapped to VMA. As it's possible that large folio is
-	 * mlocked and VMA is split later.
-	 *
-	 * During memory pressure, such kind of large folio can
-	 * be split. And the pages are not in VM_LOCKed VMA
-	 * can be reclaimed.
-	 */
-	if (!vma_test(vma, VMA_LOCKED_BIT))
-		return true;
-
 	/* folio_within_range() cannot take KSM, but any small folio is OK */
 	if (!folio_test_large(folio))
 		return true;
@@ -352,6 +340,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
 
 {
 	struct vm_area_struct *vma = walk->vma;
+	const bool lock = walk->private;
 	spinlock_t *ptl;
 	pte_t *start_pte, *pte;
 	pte_t ptent;
@@ -368,7 +357,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
 		folio = pmd_folio(*pmd);
 		if (folio_is_zone_device(folio))
 			goto out;
-		if (vma_test(vma, VMA_LOCKED_BIT))
+		if (lock)
 			mlock_folio(folio);
 		else
 			munlock_folio(folio);
@@ -390,10 +379,10 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
 			continue;
 
 		step = folio_mlock_step(folio, pte, addr, end);
-		if (!allow_mlock_munlock(folio, vma, start, end, step))
+		if (lock && !allow_mlock(folio, vma, start, end, step))
 			goto next_entry;
 
-		if (vma_test(vma, VMA_LOCKED_BIT))
+		if (lock)
 			mlock_folio(folio);
 		else
 			munlock_folio(folio);
@@ -428,31 +417,29 @@ static void mlock_vma_pages_range(struct vm_area_struct *vma,
 		.pmd_entry = mlock_pte_range,
 		.walk_lock = PGWALK_WRLOCK_VERIFY,
 	};
+	const bool lock = vma_flags_test(new_vma_flags, VMA_LOCKED_BIT);
+	vma_flags_t walk_flags = *new_vma_flags;
 
 	/*
-	 * There is a slight chance that concurrent page migration,
-	 * or page reclaim finding a page of this now-VMA_LOCKED_BIT vma,
-	 * will call mlock_vma_folio() and raise page's mlock_count:
-	 * double counting, leaving the page unevictable indefinitely.
-	 * Communicate this danger to mlock_vma_folio() with VMA_IO_BIT,
-	 * which is a VMA_SPECIAL_FLAGS flag not allowed on VMA_LOCKED_BIT vmas.
-	 * mmap_lock is held in write mode here, so this weird
-	 * combination should not be visible to other mmap_lock users;
-	 * but WRITE_ONCE so rmap walkers must see VMA_IO_BIT if VMA_LOCKED_BIT.
+	 * LOCKONFAULT without LOCKED never otherwise occurs: it marks a walk in
+	 * progress so that rmap-side callers, which test VMA_LOCKED_BIT, do not
+	 * count folios, while try_to_unmap_one(), which tests VMA_LOCKED_MASK,
+	 * still refuses to unmap them.
 	 */
-	if (vma_flags_test(new_vma_flags, VMA_LOCKED_BIT))
-		vma_flags_set(new_vma_flags, VMA_IO_BIT);
+	if (lock) {
+		vma_flags_clear(&walk_flags, VMA_LOCKED_BIT);
+		vma_flags_set(&walk_flags, VMA_LOCKONFAULT_BIT);
+	}
+
 	vma_start_write(vma);
-	vma_flags_reset_once(vma, new_vma_flags);
+	vma_flags_reset_once(vma, &walk_flags);
 
 	lru_add_drain();
-	walk_page_range_vma(vma, start, end, &mlock_walk_ops, NULL);
+	walk_page_range_vma(vma, start, end, &mlock_walk_ops, (void *)lock);
 	lru_add_drain();
 
-	if (vma_flags_test(new_vma_flags, VMA_IO_BIT)) {
-		vma_flags_clear(new_vma_flags, VMA_IO_BIT);
+	if (lock)
 		vma_flags_reset_once(vma, new_vma_flags);
-	}
 }
 
 /*
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 1533aebafb6885..96db0c3bc90f26 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -29,6 +29,7 @@
 #include <linux/cma.h>
 #include <linux/crash_dump.h>
 #include <linux/execmem.h>
+#include <linux/sizes.h>
 #include <linux/vmstat.h>
 #include <linux/kexec_handover.h>
 #include <linux/hugetlb.h>
@@ -677,21 +678,33 @@ static inline void fixup_hashdist(void)
 static inline void fixup_hashdist(void) {}
 #endif /* CONFIG_NUMA */
 
-#if defined(CONFIG_ZONE_DEVICE) || defined(CONFIG_DEFERRED_STRUCT_PAGE_INIT)
 static __meminit void pageblock_migratetype_init_range(unsigned long pfn,
-		unsigned long nr_pages, int migratetype, bool atomic)
+		unsigned long nr_pages, int migratetype, bool isolate, bool atomic)
 {
 	const unsigned long end = pfn + nr_pages;
 
 	for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) {
-		enum migratetype mt = kho_scratch_migratetype(pfn, migratetype);
+		enum migratetype mt = kho_bootmem_migratetype(pfn, migratetype);
 
-		init_pageblock_migratetype(pfn_to_page(pfn), mt, false);
-		if (!atomic && IS_ALIGNED(pfn, PAGES_PER_SECTION))
+		init_pageblock_migratetype(pfn_to_page(pfn), mt, isolate);
+		if (!atomic && IS_ALIGNED(pfn, PFN_DOWN(SZ_1G)))
 			cond_resched();
 	}
 }
-#endif
+
+struct zone __meminit *pfn_to_zone(unsigned long pfn, int nid)
+{
+	pg_data_t *pgdat = NODE_DATA(nid);
+
+	for (enum zone_type zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
+		struct zone *zone = &pgdat->node_zones[zone_type];
+
+		if (zone_spans_pfn(zone, pfn))
+			return zone;
+	}
+
+	return NULL;
+}
 
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
@@ -751,23 +764,17 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
 
 static void __meminit __init_deferred_page(unsigned long pfn, int nid)
 {
-	pg_data_t *pgdat = NODE_DATA(nid);
-	int zid;
+	struct zone *zone;
 
 	if (early_page_initialised(pfn, nid))
 		return;
 
-	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
-		struct zone *zone = &pgdat->node_zones[zid];
-
-		if (zone_spans_pfn(zone, pfn))
-			break;
-	}
-	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
-
+	zone = pfn_to_zone(pfn, nid);
+	__init_single_page(pfn_to_page(pfn), pfn,
+			   zone ? zone_idx(zone) : MAX_NR_ZONES, nid);
 	if (pageblock_aligned(pfn)) {
 		enum migratetype mt =
-			kho_scratch_migratetype(pfn, MIGRATE_MOVABLE);
+			kho_bootmem_migratetype(pfn, MIGRATE_MOVABLE);
 		init_pageblock_migratetype(pfn_to_page(pfn), mt, false);
 	}
 }
@@ -816,23 +823,41 @@ void __meminit init_deferred_page(unsigned long pfn, int nid)
  *   the zone boundary; the pages in such holes will be prepended to the
  *   zone/node above the hole except for the trailing pages in the last
  *   section that will be appended to the zone/node below.
+ *
+ * The function counts pages that should be added to
+ * zone->pages_with_online_memmap. With CONFIG_SPARSEMEM_VMEMMAP there could
+ * be offline subsections even though the entire memory map is valid and all
+ * the early sections are online. Count only pages in online subsections
+ * (see pfn_to_online_page()).
  */
-static void __init init_unavailable_range(unsigned long spfn,
-					  unsigned long epfn,
-					  int zone, int node)
+static unsigned long __init init_unavailable_range(unsigned long spfn,
+						   unsigned long epfn,
+						   int zone, int node)
 {
+	unsigned long next_chunk_pfn __maybe_unused = spfn;
 	unsigned long pfn;
-	u64 pgcnt = 0;
+	u64 online_pgcnt = 0, pgcnt = 0;
+	bool is_online = true;
 
 	for_each_valid_pfn(pfn, spfn, epfn) {
 		__init_single_page(pfn_to_page(pfn), pfn, zone, node);
 		__SetPageReserved(pfn_to_page(pfn));
 		pgcnt++;
+
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+		if (pfn >= next_chunk_pfn) {
+			is_online = pfn_section_valid(__pfn_to_section(pfn), pfn);
+			next_chunk_pfn = min(SUBSECTION_ALIGN_UP(pfn + 1), epfn);
+		}
+#endif
+		if (is_online)
+			online_pgcnt++;
 	}
 
 	if (pgcnt)
 		pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n",
 			node, zone_names[zone], pgcnt);
+	return online_pgcnt;
 }
 
 /*
@@ -886,6 +911,17 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 			}
 		}
 
+		/*
+		 * Vmemmap-optimizable PFNs are backed by shared tail struct pages,
+		 * which have already been initialized during vmemmap population.
+		 */
+		if (vmemmap_optimizable_pfn(pfn)) {
+			const unsigned int order = pfn_to_section_compound_order(pfn);
+
+			pfn = min(ALIGN(pfn, 1UL << order), end_pfn);
+			continue;
+		}
+
 		page = pfn_to_page(pfn);
 		__init_single_page(page, pfn, zone, nid);
 		if (context == MEMINIT_HOTPLUG) {
@@ -897,19 +933,13 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 				__SetPageOffline(page);
 		}
 
-		/*
-		 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
-		 * such that unmovable allocations won't be scattered all
-		 * over the place during system boot.
-		 */
-		if (pageblock_aligned(pfn)) {
-			enum migratetype mt = kho_scratch_migratetype(pfn, migratetype);
-
-			init_pageblock_migratetype(page, mt, isolate_pageblock);
+		if (pageblock_aligned(pfn))
 			cond_resched();
-		}
 		pfn++;
 	}
+
+	pageblock_migratetype_init_range(start_pfn, pfn - start_pfn, migratetype,
+					 isolate_pageblock, /* atomic */ false);
 }
 
 static void __init memmap_init_zone_range(struct zone *zone,
@@ -930,9 +960,21 @@ static void __init memmap_init_zone_range(struct zone *zone,
 	memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
 			  zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE,
 			  false);
+	zone->pages_with_online_memmap += end_pfn - start_pfn;
+
+	if (*hole_pfn < start_pfn) {
+		unsigned long hole_start_pfn = *hole_pfn;
+		unsigned long pgcnt;
 
-	if (*hole_pfn < start_pfn)
-		init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
+		if (hole_start_pfn < zone_start_pfn) {
+			init_unavailable_range(hole_start_pfn, zone_start_pfn,
+					       zone_id, nid);
+			hole_start_pfn = zone_start_pfn;
+		}
+		pgcnt = init_unavailable_range(hole_start_pfn, start_pfn,
+					       zone_id, nid);
+		zone->pages_with_online_memmap += pgcnt;
+	}
 
 	*hole_pfn = end_pfn;
 }
@@ -1000,13 +1042,9 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 	page->zone_device_data = NULL;
 
 	/*
-	 * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released
-	 * directly to the driver page allocator which will set the page count
-	 * to 1 when allocating the page.
-	 *
-	 * MEMORY_TYPE_GENERIC and MEMORY_TYPE_FS_DAX pages automatically have
-	 * their refcount reset to one whenever they are freed (ie. after
-	 * their refcount drops to 0).
+	 * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free
+	 * path. The remaining ZONE_DEVICE types start from 0 here and raise
+	 * the count again when the allocator or driver hands the page out.
 	 */
 	switch (pgmap->type) {
 	case MEMORY_DEVICE_FS_DAX:
@@ -1021,6 +1059,17 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 	}
 }
 
+static void zone_device_page_init_from_template(struct page *page,
+		unsigned long pfn, struct page *template)
+{
+	set_page_section_from_pfn(template, pfn);
+#ifdef WANT_PAGE_VIRTUAL
+	if (!is_highmem_idx(ZONE_DEVICE))
+		set_page_address(template, __va(pfn << PAGE_SHIFT));
+#endif
+	memcpy_nontemporal(page, template, sizeof(*page));
+}
+
 /*
  * With compound page geometry and when struct pages are stored in ram most
  * tail pages are reused. Consequently, the amount of unique struct pages to
@@ -1030,19 +1079,14 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
  * of an altmap. See vmemmap_populate_compound_pages().
  */
 static inline unsigned long compound_nr_pages(unsigned long pfn,
-					      struct vmem_altmap *altmap,
 					      struct dev_pagemap *pgmap)
 {
-	/*
-	 * If DAX memory is hot-plugged into an unoccupied subsection
-	 * of an early section, the unoptimized boot memmap is reused.
-	 * See section_activate().
-	 */
-	if (early_section(__pfn_to_section(pfn)) ||
-	    !vmemmap_can_optimize(altmap, pgmap))
+	const struct mem_section *ms = __pfn_to_section(pfn);
+
+	if (!section_vmemmap_optimizable(ms))
 		return pgmap_vmemmap_nr(pgmap);
 
-	return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page));
+	return VMEMMAP_OPTIMIZATION_PAGES * (PAGE_SIZE / sizeof(struct page));
 }
 
 static void __ref memmap_init_compound(struct page *head,
@@ -1053,6 +1097,8 @@ static void __ref memmap_init_compound(struct page *head,
 {
 	unsigned long pfn, end_pfn = head_pfn + nr_pages;
 	unsigned int order = pgmap->vmemmap_shift;
+	struct page template;
+	struct page *page;
 
 	/*
 	 * We have to initialize the pages, including setting up page links.
@@ -1061,13 +1107,23 @@ static void __ref memmap_init_compound(struct page *head,
 	 * the pages in the same go.
 	 */
 	__SetPageHead(head);
-	for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) {
-		struct page *page = pfn_to_page(pfn);
 
-		__init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
-		prep_compound_tail(page, head, order);
-		set_page_count(page, 0);
-	}
+	/*
+	 * All tails of the same compound page share the state established by
+	 * prep_compound_tail(). Reuse one tail template for the whole range and
+	 * refresh only the PFN-dependent fields in that template before each copy.
+	 */
+	pfn = head_pfn + 1;
+	page = pfn_to_page(pfn);
+	__init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
+	prep_compound_tail(page, head, order);
+	set_page_count(page, 0);
+	memcpy(&template, page, sizeof(*page));
+
+	/* Initialize the remaining tail pages from template. */
+	for (pfn = head_pfn + 2; pfn < end_pfn; pfn++)
+		zone_device_page_init_from_template(pfn_to_page(pfn), pfn,
+						    &template);
 	prep_compound_head(head, order);
 }
 
@@ -1083,6 +1139,8 @@ void __ref memmap_init_zone_device(struct zone *zone,
 	unsigned long zone_idx = zone_idx(zone);
 	unsigned long start = jiffies;
 	int nid = pgdat->node_id;
+	struct page template;
+	struct page *page;
 
 	if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))
 		return;
@@ -1097,10 +1155,28 @@ void __ref memmap_init_zone_device(struct zone *zone,
 		nr_pages = end_pfn - start_pfn;
 	}
 
-	for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {
-		struct page *page = pfn_to_page(pfn);
+	if (!nr_pages)
+		return;
+
+	/*
+	 * Seed the reusable head-page template from the first real struct
+	 * page. The normal page-init and refcount helpers must operate on
+	 * a real memmap entry rather than a stack object.
+	 */
+	pfn = start_pfn;
+	page = pfn_to_page(pfn);
+	__init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
+	memcpy(&template, page, sizeof(*page));
+	if (pfns_per_compound != 1)
+		memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
+				     compound_nr_pages(pfn, pgmap));
+	pfn += pfns_per_compound;
+
+	/* Initialize the remaining head pages from template. */
+	for (; pfn < end_pfn; pfn += pfns_per_compound) {
+		page = pfn_to_page(pfn);
 
-		__init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
+		zone_device_page_init_from_template(page, pfn, &template);
 
 		if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
 			cond_resched();
@@ -1109,10 +1185,11 @@ void __ref memmap_init_zone_device(struct zone *zone,
 			continue;
 
 		memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
-				     compound_nr_pages(pfn, altmap, pgmap));
+				     compound_nr_pages(pfn, pgmap));
 	}
 
-	pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE, false);
+	pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE,
+					 /* isolate */ false, /* atomic */ false);
 
 	pr_debug("%s initialised %lu pages in %ums\n", __func__,
 		nr_pages, jiffies_to_msecs(jiffies - start));
@@ -1166,6 +1243,10 @@ static unsigned long __init __absent_pages_in_range(int nid,
 	unsigned long start_pfn, end_pfn;
 	int i;
 
+	/* range is empty, nothing to do */
+	if (!nr_absent)
+		return 0;
+
 	for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
 		start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
 		end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
@@ -1187,22 +1268,9 @@ unsigned long __init absent_pages_in_range(unsigned long start_pfn,
 	return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
 }
 
-/* Return the number of page frames in holes in a zone on a node */
-static unsigned long __init zone_absent_pages_in_node(int nid,
-					unsigned long zone_type,
-					unsigned long zone_start_pfn,
-					unsigned long zone_end_pfn)
-{
-	/* zone is empty, we don't have any absent pages */
-	if (zone_start_pfn == zone_end_pfn)
-		return 0;
-
-	return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
-}
-
 /*
  * Return the number of pages a zone spans in a node, including holes
- * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
+ * present_pages = zone_spanned_pages_in_node() - __absent_pages_in_range()
  */
 static unsigned long __init zone_spanned_pages_in_node(int nid,
 					unsigned long zone_type,
@@ -1292,9 +1360,8 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat,
 						     node_end_pfn,
 						     &zone_start_pfn,
 						     &zone_end_pfn);
-		absent = zone_absent_pages_in_node(pgdat->node_id, i,
-						   zone_start_pfn,
-						   zone_end_pfn);
+		absent = __absent_pages_in_range(pgdat->node_id, zone_start_pfn,
+						 zone_end_pfn);
 
 		real_size = spanned - absent;
 
@@ -1921,7 +1988,8 @@ static void __init deferred_free_pages(unsigned long pfn,
 	if (!nr_pages)
 		return;
 
-	pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE, true);
+	pageblock_migratetype_init_range(pfn, nr_pages, MIGRATE_MOVABLE,
+					 /* isolate */ false, /* atomic */ true);
 
 	page = pfn_to_page(pfn);
 
@@ -2188,28 +2256,6 @@ void __init init_cma_pageblock(struct page *page)
 }
 #endif
 
-void set_zone_contiguous(struct zone *zone)
-{
-	unsigned long block_start_pfn = zone->zone_start_pfn;
-	unsigned long block_end_pfn;
-
-	block_end_pfn = pageblock_end_pfn(block_start_pfn);
-	for (; block_start_pfn < zone_end_pfn(zone);
-			block_start_pfn = block_end_pfn,
-			 block_end_pfn += pageblock_nr_pages) {
-
-		block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
-
-		if (!__pageblock_pfn_to_page(block_start_pfn,
-					     block_end_pfn, zone))
-			return;
-		cond_resched();
-	}
-
-	/* We confirm that there is no hole */
-	zone->contiguous = true;
-}
-
 /*
  * Check if a PFN range intersects multiple zones on one or more
  * NUMA nodes. Specify the @nid argument if it is known that this
@@ -2628,6 +2674,7 @@ void __init mm_core_init_early(void)
 {
 	kho_memory_init_early();
 
+	sparse_sections_init();
 	free_area_init();
 
 	hugetlb_cma_reserve();
diff --git a/mm/mm_init.h b/mm/mm_init.h
index 39f75df9be1c26..393006c1c6efef 100644
--- a/mm/mm_init.h
+++ b/mm/mm_init.h
@@ -20,15 +20,9 @@ struct vmem_altmap;
 /* perform sanity checks on struct pages being allocated or freed */
 DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
 
-void set_zone_contiguous(struct zone *zone);
 bool pfn_range_intersects_zones(int nid, unsigned long start_pfn,
 			   unsigned long nr_pages);
 
-static inline void clear_zone_contiguous(struct zone *zone)
-{
-	zone->contiguous = false;
-}
-
 void memblock_free_pages(unsigned long pfn, unsigned int order);
 
 void *memmap_alloc(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr,
@@ -39,6 +33,7 @@ void memmap_init_range(unsigned long size, int nid, unsigned long zone,
 		enum meminit_context context,
 		struct vmem_altmap *altmap, int migratetype,
 		bool isolate_pageblock);
+struct zone *pfn_to_zone(unsigned long pfn, int nid);
 
 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */
diff --git a/mm/mmap.c b/mm/mmap.c
index 4bf26b0f1e6e36..148ba01c03739f 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -547,7 +547,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 		}
 		case MAP_PRIVATE:
 			/*
-			 * Set pgoff according to addr for anon_vma.
+			 * Set pgoff according to addr for the anon rmap.
 			 */
 			pgoff = addr >> PAGE_SHIFT;
 			break;
@@ -1774,8 +1774,8 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 		if (vma_test(tmp, VMA_WIPEONFORK_BIT)) {
 			/*
 			 * VMA_WIPEONFORK_BIT gets a clean slate in the child.
-			 * Don't prepare anon_vma until fault since we don't
-			 * copy page for current vma.
+			 * Don't prepare the anon rmap until fault since we
+			 * don't copy pages for the current vma.
 			 */
 			tmp->anon_vma = NULL;
 		} else if (anon_vma_fork(tmp, mpnt))
@@ -1786,7 +1786,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 		/*
 		 * Copy/update hugetlb private vma information.
 		 */
-		if (is_vm_hugetlb_page(tmp))
+		if (vma_is_hugetlb(tmp))
 			hugetlb_dup_vma_private(tmp);
 
 		/*
diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c
index 898c2ef1e95803..2f94ee0fdee2a2 100644
--- a/mm/mmap_lock.c
+++ b/mm/mmap_lock.c
@@ -43,9 +43,6 @@ void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write)
 EXPORT_SYMBOL(__mmap_lock_do_trace_released);
 #endif /* CONFIG_TRACING */
 
-#ifdef CONFIG_MMU
-#ifdef CONFIG_PER_VMA_LOCK
-
 /* State shared across __vma_[start, end]_exclude_readers. */
 struct vma_exclude_readers_state {
 	/* Input parameters. */
@@ -299,6 +296,8 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
 	MA_STATE(mas, &mm->mm_mt, address, address);
 	struct vm_area_struct *vma;
 
+	if (!IS_ENABLED(CONFIG_MMU))
+		return NULL;
 retry:
 	rcu_read_lock();
 	vma = mas_walk(&mas);
@@ -341,6 +340,41 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm,
 	return NULL;
 }
 
+/**
+ * vma_start_read_unlocked() - Find the VMA covering 'address' and read-lock it.
+ * @mm: the mm_struct of the address space to search
+ * @address: address that the vma should contain
+ *
+ * The fast path does not take mmap_lock. Waits for writers to finish if the
+ * VMA is being modified by taking mmap_lock.
+ * Use when mmap_lock is not held, otherwise use vma_start_read_locked().
+ * Nothing prevents VMAs being unmapped/mapped before or after the VMA is
+ * looked up, if a stronger guarantee is required, take an mmap_lock.
+ *
+ * Return: If a VMA exists which spans @address, return that VMA, read-locked.
+ * If no VMA is mapped there or, very unlikely, a reference count overflow
+ * occurred, return NULL.
+ */
+struct vm_area_struct *vma_start_read_unlocked(struct mm_struct *mm,
+					       unsigned long address)
+{
+	struct vm_area_struct *vma;
+
+	/* Fast path: return stable VMA covering 'address': */
+	vma = lock_vma_under_rcu(mm, address);
+	if (vma)
+		return vma;
+
+	/* Slow path: preclude VMA writers by temporarily getting mmap read lock. */
+	mmap_read_lock(mm);
+	vma = vma_lookup(mm, address);
+	if (vma && !vma_start_read_locked(vma))
+		vma = NULL;
+	mmap_read_unlock(mm);
+
+	return vma;
+}
+
 static struct vm_area_struct *lock_next_vma_under_mmap_lock(struct mm_struct *mm,
 							    struct vma_iterator *vmi,
 							    unsigned long from_addr)
@@ -431,7 +465,6 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm,
 
 	return vma;
 }
-#endif /* CONFIG_PER_VMA_LOCK */
 
 #ifdef CONFIG_LOCK_MM_AND_FIND_VMA
 #include <linux/extable.h>
@@ -548,23 +581,3 @@ struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
 	return NULL;
 }
 #endif /* CONFIG_LOCK_MM_AND_FIND_VMA */
-
-#else /* CONFIG_MMU */
-
-/*
- * At least xtensa ends up having protection faults even with no
- * MMU.. No stack expansion, at least.
- */
-struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
-			unsigned long addr, struct pt_regs *regs)
-{
-	struct vm_area_struct *vma;
-
-	mmap_read_lock(mm);
-	vma = vma_lookup(mm, addr);
-	if (!vma)
-		mmap_read_unlock(mm);
-	return vma;
-}
-
-#endif /* CONFIG_MMU */
diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 3985d856de7f9b..9f353f0e2ef4d2 100644
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -218,8 +218,6 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_
 
 #endif /* MMU_GATHER_NO_GATHER */
 
-#ifdef CONFIG_MMU_GATHER_TABLE_FREE
-
 static void __tlb_remove_table_free(struct mmu_table_batch *batch)
 {
 	int i;
@@ -230,10 +228,8 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch)
 	free_page((unsigned long)batch);
 }
 
-#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
-
 /*
- * Semi RCU freeing of the page directories.
+ * RCU freeing of the page directories.
  *
  * This is needed by some architectures to implement software pagetable walkers.
  *
@@ -259,13 +255,13 @@ static void __tlb_remove_table_free(struct mmu_table_batch *batch)
  * means.
  *
  * What we do is batch the freed directory pages (tables) and RCU free them.
- * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
- * holds off grace periods.
+ * Disabling IRQs or preemption holds off RCU grace periods, so this protects
+ * both rcu_read_lock() and IRQ-disabling walkers.
  *
  * However, in order to batch these pages we need to allocate storage, this
  * allocation is deep inside the MM code and can thus easily fail on memory
- * pressure. To guarantee progress we fall back to single table freeing, see
- * the implementation of tlb_remove_table_one().
+ * pressure. To guarantee progress we fall back to single table freeing, which
+ * is also RCU-deferred - see the implementation of tlb_remove_table_one().
  *
  */
 
@@ -315,15 +311,6 @@ void tlb_remove_table_sync_rcu(void)
 	synchronize_rcu();
 }
 
-#else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
-
-static void tlb_remove_table_free(struct mmu_table_batch *batch)
-{
-	__tlb_remove_table_free(batch);
-}
-
-#endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
-
 /*
  * If we want tlb_remove_table() to imply TLB invalidates.
  */
@@ -403,13 +390,6 @@ static inline void tlb_table_init(struct mmu_gather *tlb)
 	tlb->batch = NULL;
 }
 
-#else /* !CONFIG_MMU_GATHER_TABLE_FREE */
-
-static inline void tlb_table_flush(struct mmu_gather *tlb) { }
-static inline void tlb_table_init(struct mmu_gather *tlb) { }
-
-#endif /* CONFIG_MMU_GATHER_TABLE_FREE */
-
 static void tlb_flush_mmu_free(struct mmu_gather *tlb)
 {
 	tlb_table_flush(tlb);
@@ -500,7 +480,7 @@ void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
 {
 	tlb_gather_mmu(tlb, vma->vm_mm);
 	tlb_update_vma_flags(tlb, vma);
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		/* All entries have the same size. */
 		tlb_change_page_size(tlb, huge_page_size(hstate_vma(vma)));
 }
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 2888ee638d872a..e59c69cb5a2388 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -717,7 +717,7 @@ long change_protection(struct mmu_gather *tlb,
 	    (cp_flags & MM_CP_UFFD_RWP))
 		newprot = PAGE_NONE;
 
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		pages = hugetlb_change_protection(vma, start, end, newprot,
 						  cp_flags);
 	else
@@ -783,8 +783,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 	 * uncommon case, so doesn't need to be very optimized.
 	 */
 	if (arch_has_pfn_modify_check() &&
-	    vma_flags_test_any(&old_vma_flags, VMA_PFNMAP_BIT,
-			       VMA_MIXEDMAP_BIT) &&
+	    vma_flags_is_kernel_owned(&old_vma_flags) &&
 	    !vma_flags_test_any_mask(&new_vma_flags, VMA_ACCESS_FLAGS)) {
 		pgprot_t new_pgprot = vm_get_page_prot(newflags);
 
@@ -797,8 +796,8 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 	/*
 	 * If we make a private mapping writable we increase our commit;
 	 * but (without finer accounting) cannot reduce our commit if we
-	 * make it unwritable again except in the anonymous case where no
-	 * anon_vma has yet to be assigned.
+	 * make it unwritable again except in the anonymous case where the
+	 * VMA's anon rmap has yet to be assigned.
 	 *
 	 * hugetlb mapping were accounted for even if read-only so there is
 	 * no need to account for them here.
@@ -817,7 +816,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb,
 			vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT);
 		}
 	} else if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) &&
-		   vma_is_anonymous(vma) && !vma->anon_vma) {
+		   vma_is_anonymous(vma) && !vma_has_anon_rmap(vma)) {
 		vma_flags_clear(&new_vma_flags, VMA_ACCOUNT_BIT);
 	}
 
diff --git a/mm/mremap.c b/mm/mremap.c
index 7c368440fafe24..1b8d06195bf37f 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -144,13 +144,13 @@ static void take_rmap_locks(struct vm_area_struct *vma)
 {
 	if (vma->vm_file)
 		i_mmap_lock_write(vma->vm_file->f_mapping);
-	if (vma->anon_vma)
+	if (vma_has_anon_rmap(vma))
 		anon_vma_lock_write(vma->anon_vma);
 }
 
 static void drop_rmap_locks(struct vm_area_struct *vma)
 {
-	if (vma->anon_vma)
+	if (vma_has_anon_rmap(vma))
 		anon_vma_unlock_write(vma->anon_vma);
 	if (vma->vm_file)
 		i_mmap_unlock_write(vma->vm_file->f_mapping);
@@ -214,7 +214,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	int err = 0;
 
 	/*
-	 * When need_rmap_locks is true, we take the i_mmap_rwsem and anon_vma
+	 * When need_rmap_locks is true, we take the i_mmap_rwsem and anon rmap
 	 * locks to ensure that rmap will always observe either the old or the
 	 * new ptes. This is the easiest way to avoid races with
 	 * truncate_pagecache(), page migration, etc...
@@ -812,7 +812,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc)
 	if (!pmc->len_in)
 		return 0;
 
-	if (is_vm_hugetlb_page(pmc->old))
+	if (vma_is_hugetlb(pmc->old))
 		return move_hugetlb_page_tables(pmc->old, pmc->new, pmc->old_addr,
 						pmc->new_addr, pmc->len_in);
 
@@ -1037,6 +1037,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm,
 }
 
 static bool __check_map_count_against_split(struct mm_struct *mm,
+					    bool is_dontunmap,
 					    bool before_unmaps)
 {
 	const int sys_map_count = get_sysctl_max_map_count();
@@ -1088,26 +1089,38 @@ static bool __check_map_count_against_split(struct mm_struct *mm,
 	 * Therefore we must check to ensure we have headroom of 2 additional
 	 * VMAs.
 	 */
-	return map_count + 2 <= sys_map_count;
+	map_count += 2;
+
+	/*
+	 * If MREMAP_DONTUNMAP is set and a partial operation is performed,
+	 * the VMA is split ahead of time and the -1 observed above doesn't
+	 * apply.
+	 */
+	if (is_dontunmap)
+		map_count++;
+
+	return map_count <= sys_map_count;
 }
 
 /* Do we violate the map count limit if we split VMAs when moving the VMA? */
-static bool check_map_count_against_split(void)
+static bool check_map_count_against_split(struct vma_remap_struct *vrm)
 {
 	return __check_map_count_against_split(current->mm,
+					       vrm->flags & MREMAP_DONTUNMAP,
 					       /*before_unmaps=*/false);
 }
 
 /* Do we violate the map count limit if we split VMAs prior to early unmaps? */
-static bool check_map_count_against_split_early(void)
+static bool check_map_count_against_split_early(struct vma_remap_struct *vrm)
 {
 	return __check_map_count_against_split(current->mm,
+					       vrm->flags & MREMAP_DONTUNMAP,
 					       /*before_unmaps=*/true);
 }
 
 /*
- * Perform checks before attempting to write a VMA prior to it being
- * moved.
+ * Perform checks and preparation before attempting to write a VMA prior to it
+ * being moved.
  */
 static unsigned long prep_move_vma(struct vma_remap_struct *vrm)
 {
@@ -1116,19 +1129,17 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm)
 	unsigned long old_addr = vrm->addr;
 	unsigned long old_len = vrm->old_len;
 	vm_flags_t dummy = vma->vm_flags;
+	const bool split_before = vma->vm_start != old_addr;
+	const bool split_after = vma->vm_end != old_addr + old_len;
 
-	/*
-	 * We'd prefer to avoid failure later on in do_munmap: we copy a VMA,
-	 * which may not merge, then (if MREMAP_DONTUNMAP is not set) unmap the
-	 * source, which may split, causing a net increase of 2 mappings.
-	 */
-	if (!check_map_count_against_split())
+	/* Avoid failure later on. */
+	if (!check_map_count_against_split(vrm))
 		return -ENOMEM;
 
 	if (vma->vm_ops && vma->vm_ops->may_split) {
-		if (vma->vm_start != old_addr)
+		if (split_before)
 			err = vma->vm_ops->may_split(vma, old_addr);
-		if (!err && vma->vm_end != old_addr + old_len)
+		if (!err && split_after)
 			err = vma->vm_ops->may_split(vma, old_addr + old_len);
 		if (err)
 			return err;
@@ -1146,6 +1157,21 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm)
 	if (err)
 		return err;
 
+	/*
+	 * To account mlock()'d pages correctly in the MREMAP_DONTUNMAP
+	 * case perform any split ahead of time.
+	 */
+	if (vrm->flags & MREMAP_DONTUNMAP) {
+		VMA_ITERATOR(vmi, vma->vm_mm, old_addr);
+
+		if (split_before)
+			err = split_vma(&vmi, vma, old_addr, 1);
+		if (!err && split_after)
+			err = split_vma(&vmi, vma, old_addr + old_len, 0);
+		vrm->vmi_needs_invalidate = true;
+		return err;
+	}
+
 	return 0;
 }
 
@@ -1275,7 +1301,8 @@ static int copy_vma_and_data(struct vma_remap_struct *vrm,
 	PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len);
 
 	new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff,
-			   new_anon_pgoff, &pmc.need_rmap_locks);
+			   new_anon_pgoff, &pmc.need_rmap_locks,
+			   vrm->flags & MREMAP_DONTUNMAP);
 	if (!new_vma) {
 		vrm_uncharge(vrm);
 		*new_vma_ptr = NULL;
@@ -1329,32 +1356,28 @@ static int copy_vma_and_data(struct vma_remap_struct *vrm,
 static void dontunmap_complete(struct vma_remap_struct *vrm,
 			       struct vm_area_struct *new_vma)
 {
-	unsigned long start = vrm->addr;
-	unsigned long end = vrm->addr + vrm->old_len;
 	struct vm_area_struct *vma = vrm->vma;
-	unsigned long old_start = vma->vm_start;
-	unsigned long old_end = vma->vm_end;
+	const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT;
+
+	/* Self-merge is disallowed. */
+	VM_WARN_ON_ONCE(new_vma == vma);
 
 	/* We always clear VMA_LOCKED[ONFAULT]_BIT on the old VMA. */
 	vma_clear_flags_mask(vma, VMA_LOCKED_MASK);
 
 	/*
-	 * anon_vma links of the old vma is no longer needed after its page
-	 * table has been moved.
+	 * The anon rmap links of the old vma are no longer needed after its
+	 * page table has been moved.
 	 */
-	if (new_vma != vma && start == old_start && end == old_end) {
-		const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT;
-
-		unlink_anon_vmas(vma);
-		/*
-		 * The VMA is now unfaulted and it is an invariant that
-		 * unfaulted anonymous VMAs have page offset equal to
-		 * vma->vm_start >> PAGE_SHIFT.
-		 */
-		vma_set_anon_pgoff(vma, pgoff_unfaulted);
-		if (vma_is_anonymous(vma) && !vma->vm_file)
-			vma_set_pgoff(vma, pgoff_unfaulted);
-	}
+	unlink_anon_vmas(vma);
+	/*
+	 * The VMA is now unfaulted and it is an invariant that
+	 * unfaulted anonymous VMAs have page offset equal to
+	 * vma->vm_start >> PAGE_SHIFT.
+	 */
+	vma_set_anon_pgoff(vma, pgoff_unfaulted);
+	if (vma_is_anonymous(vma) && !vma->vm_file)
+		vma_set_pgoff(vma, pgoff_unfaulted);
 }
 
 static unsigned long move_vma(struct vma_remap_struct *vrm)
@@ -1735,7 +1758,7 @@ static bool vma_multi_allowed(struct vm_area_struct *vma)
 	/* Known good. */
 	if (vma_is_shmem(vma))
 		return true;
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		return true;
 	if (file->f_op->get_unmapped_area == thp_get_unmapped_area)
 		return true;
@@ -1758,7 +1781,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm)
 		return -EPERM;
 
 	/* Align to hugetlb page size, if required. */
-	if (is_vm_hugetlb_page(vma) && !align_hugetlb(vrm))
+	if (vma_is_hugetlb(vma) && !align_hugetlb(vrm))
 		return -EINVAL;
 
 	vrm_set_delta(vrm);
@@ -1788,8 +1811,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm)
 		return -EINVAL;
 	}
 
-	if ((vrm->flags & MREMAP_DONTUNMAP) &&
-	    vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT))
+	if ((vrm->flags & MREMAP_DONTUNMAP) && vma_is_fixed_mapping(vma))
 		return -EINVAL;
 
 	/*
@@ -1827,7 +1849,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm)
 	if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
 		return -EINVAL;
 
-	if (vma_test_any(vma, VMA_DONTEXPAND_BIT, VMA_PFNMAP_BIT))
+	if (vma_is_fixed_mapping(vma))
 		return -EFAULT;
 
 	if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), vrm->delta))
@@ -2002,7 +2024,7 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm)
 		return -EINTR;
 	vrm->mmap_locked = true;
 
-	if (!check_map_count_against_split_early()) {
+	if (!check_map_count_against_split_early(vrm)) {
 		mmap_write_unlock(mm);
 		return -ENOMEM;
 	}
diff --git a/mm/nommu.c b/mm/nommu.c
index 498e01ee40b056..91ab65afda7055 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -1674,6 +1674,9 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
 	struct vm_area_struct *vma;
 	int write = gup_flags & FOLL_WRITE;
 
+	if (addr + len < addr)
+		return 0;
+
 	if (mmap_read_lock_killable(mm))
 		return 0;
 
@@ -1727,9 +1730,6 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in
 {
 	struct mm_struct *mm;
 
-	if (addr + len < addr)
-		return 0;
-
 	mm = get_task_mm(tsk);
 	if (!mm)
 		return 0;
@@ -1741,13 +1741,12 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, in
 }
 EXPORT_SYMBOL_GPL(access_process_vm);
 
-#ifdef CONFIG_BPF_SYSCALL
 /*
  * Copy a string from another process's address space as given in mm.
  * If there is any error return -EFAULT.
  */
-static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
-				void *buf, int len)
+int __copy_remote_mm_str(struct mm_struct *mm, unsigned long addr,
+		void *buf, int len, unsigned int gup_flags)
 {
 	unsigned long addr_end;
 	struct vm_area_struct *vma;
@@ -1782,44 +1781,6 @@ static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
 	return ret;
 }
 
-/**
- * copy_remote_vm_str - copy a string from another process's address space.
- * @tsk:	the task of the target address space
- * @addr:	start address to read from
- * @buf:	destination buffer
- * @len:	number of bytes to copy
- * @gup_flags:	flags modifying lookup behaviour (unused)
- *
- * The caller must hold a reference on @mm.
- *
- * Return: number of bytes copied from @addr (source) to @buf (destination);
- * not including the trailing NUL. Always guaranteed to leave NUL-terminated
- * buffer. On any error, return -EFAULT.
- */
-int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
-		       void *buf, int len, unsigned int gup_flags)
-{
-	struct mm_struct *mm;
-	int ret;
-
-	if (unlikely(len == 0))
-		return 0;
-
-	mm = get_task_mm(tsk);
-	if (!mm) {
-		*(char *)buf = '\0';
-		return -EFAULT;
-	}
-
-	ret = __copy_remote_vm_str(mm, addr, buf, len);
-
-	mmput(mm);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(copy_remote_vm_str);
-#endif /* CONFIG_BPF_SYSCALL */
-
 /**
  * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
  * @inode: The inode to check
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 5f372f6e26fa32..5d48bd862c27b2 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -230,7 +230,7 @@ long oom_badness(struct task_struct *p, unsigned long totalpages)
 	task_unlock(p);
 
 	/* Normalize to oom_score_adj units */
-	adj *= totalpages / 1000;
+	adj *= totalpages / OOM_SCORE_ADJ_MAX;
 	points += adj;
 
 	return points;
@@ -267,18 +267,11 @@ static enum oom_constraint constrained_alloc(struct oom_control *oc)
 
 	if (!oc->zonelist)
 		return CONSTRAINT_NONE;
-	/*
-	 * Reach here only when __GFP_NOFAIL is used. So, we should avoid
-	 * to kill current.We have to random task kill in this case.
-	 * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now.
-	 */
-	if (oc->gfp_mask & __GFP_THISNODE)
-		return CONSTRAINT_NONE;
 
 	/*
-	 * This is not a __GFP_THISNODE allocation, so a truncated nodemask in
-	 * the page allocator means a mempolicy is in effect.  Cpuset policy
-	 * is enforced in get_page_from_freelist().
+	 * A truncated nodemask in the page allocator means a mempolicy
+	 * is in effect.  Cpuset policy is enforced in
+	 * get_page_from_freelist().
 	 */
 	if (oc->nodemask &&
 	    !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) {
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index eeab25d6ce3647..499a35473e4f31 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -2705,7 +2705,7 @@ bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio)
 	if (folio_test_set_dirty(folio))
 		return false;
 
-	__folio_mark_dirty(folio, mapping, !folio_test_private(folio));
+	__folio_mark_dirty(folio, mapping, !folio_has_attached_private(folio));
 
 	if (mapping->host) {
 		/* !PageAnon && !swapper_space */
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 12fac9084c483d..f13439d10b16da 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -37,6 +37,7 @@
 #include <linux/vmstat.h>
 #include <linux/fault-inject.h>
 #include <linux/compaction.h>
+#include <linux/crash_dump.h>
 #include <trace/events/kmem.h>
 #include <trace/events/oom.h>
 #include <linux/prefetch.h>
@@ -613,31 +614,13 @@ static inline bool __maybe_unused bad_range(struct zone *zone, struct page *page
 }
 #endif
 
+/* Allow a burst of 60 reports per minute */
+static DEFINE_RATELIMIT_STATE(bad_page_ratelimit, 60 * HZ, 60);
+
 static void bad_page(struct page *page, const char *reason)
 {
-	static unsigned long resume;
-	static unsigned long nr_shown;
-	static unsigned long nr_unshown;
-
-	/*
-	 * Allow a burst of 60 reports, then keep quiet for that minute;
-	 * or allow a steady drip of one report per second.
-	 */
-	if (nr_shown == 60) {
-		if (time_before(jiffies, resume)) {
-			nr_unshown++;
-			goto out;
-		}
-		if (nr_unshown) {
-			pr_alert(
-			      "BUG: Bad page state: %lu messages suppressed\n",
-				nr_unshown);
-			nr_unshown = 0;
-		}
-		nr_shown = 0;
-	}
-	if (nr_shown++ == 0)
-		resume = jiffies + 60 * HZ;
+	if (!__ratelimit(&bad_page_ratelimit))
+		goto out;
 
 	pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
 		current->comm, page_to_pfn(page));
@@ -2178,10 +2161,17 @@ static inline bool boost_watermark(struct zone *zone)
 
 	if (!watermark_boost_factor)
 		return false;
+
+	/*
+	 * A kdump capture kernel exits before a boost can pay off, while
+	 * the raised watermark can exceed the memory left for the dump.
+	 */
+	if (is_kdump_kernel())
+		return false;
+
 	/*
 	 * Don't bother in zones that are unlikely to produce results.
-	 * On small machines, including kdump capture kernels running
-	 * in a small area, boosting the watermark can cause an out of
+	 * On small machines, boosting the watermark can cause an out of
 	 * memory situation immediately.
 	 */
 	if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
@@ -4127,6 +4117,31 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
 	return page;
 }
 
+/*
+ * If fallbacks are not permitted (defrag_mode), we either need to
+ * reclaim space in a block of matching type, or clear out an entire
+ * block to allow __rmqueue_claim() to convert.
+ *
+ * Reclaim by itself is primarily freeing space in movable blocks,
+ * since that's where the LRU pages live. So this works for movable
+ * requests, but not for others.
+ *
+ * For those, promote the order of reclaim and compaction to help make
+ * blocks, instead of spinning in reclaim alone unproductively. Retry
+ * decisions based on the outcome of that work - reclaim progress and
+ * compaction results - must account for the promotion as well, see
+ * should_reclaim_retry() and should_compact_retry().
+ */
+static inline unsigned int nofrag_promote_order(unsigned int order,
+						unsigned int alloc_flags,
+						const struct alloc_context *ac)
+{
+	if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE)
+		return max(order, pageblock_order);
+
+	return order;
+}
+
 /*
  * Maximum number of compaction retries with a progress before OOM
  * killer is consider as the only way to move forward.
@@ -4149,22 +4164,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 		.order = order,
 		.page = NULL,
 	};
-	int compact_order = order;
-
-	/*
-	 * If fallbacks are not permitted (defrag_mode), we either
-	 * need to reclaim space in a block of matching type, or clear
-	 * out an entire block to allow __rmqueue_claim() to convert.
-	 *
-	 * Reclaim by itself is primarily freeing space in movable
-	 * blocks, since that's where the LRU pages live. So this
-	 * works for movable requests, but not for others.
-	 *
-	 * For those, promote the order to help make blocks, instead
-	 * of spinning in reclaim alone unproductively.
-	 */
-	if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE)
-		compact_order = max(order, pageblock_order);
+	unsigned int compact_order = nofrag_promote_order(order, alloc_flags, ac);
 
 	if (!compact_order)
 		return NULL;
@@ -4256,8 +4256,11 @@ should_compact_retry(gfp_t gfp_mask, struct alloc_context *ac, int order,
 	bool ret = false;
 	int retries = *compaction_retries;
 	enum compact_priority priority = *compact_priority;
+	unsigned int compact_order;
 
-	if (!order)
+	/* Check the compaction result at the order compaction ran at */
+	compact_order = nofrag_promote_order(order, alloc_flags, ac);
+	if (!compact_order)
 		return false;
 
 	if (fatal_signal_pending(current))
@@ -4266,10 +4269,14 @@ should_compact_retry(gfp_t gfp_mask, struct alloc_context *ac, int order,
 	/*
 	 * Compaction was skipped due to a lack of free order-0
 	 * migration targets. Continue if reclaim can help.
+	 *
+	 * Promoted requests have exhausted their reclaim retries at
+	 * this point, and they can fall back instead.
 	 */
 	if (compact_result == COMPACT_SKIPPED) {
-		ret = compaction_zonelist_suitable(ac, order, alloc_flags,
-						   gfp_mask);
+		if (compact_order == order)
+			ret = compaction_zonelist_suitable(ac, order, alloc_flags,
+							   gfp_mask);
 		goto out;
 	}
 
@@ -4287,7 +4294,7 @@ should_compact_retry(gfp_t gfp_mask, struct alloc_context *ac, int order,
 		 * need much more detailed feedback from compaction to
 		 * make a better decision.
 		 */
-		if (order > PAGE_ALLOC_COSTLY_ORDER)
+		if (compact_order > PAGE_ALLOC_COSTLY_ORDER)
 			max_retries /= 4;
 
 		if (++(*compaction_retries) <= max_retries) {
@@ -4299,7 +4306,7 @@ should_compact_retry(gfp_t gfp_mask, struct alloc_context *ac, int order,
 	/*
 	 * Compaction failed. Retry with increasing priority.
 	 */
-	min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
+	min_priority = (compact_order > PAGE_ALLOC_COSTLY_ORDER) ?
 			MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
 
 	if (*compact_priority > min_priority) {
@@ -4468,11 +4475,7 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
 	struct page *page = NULL;
 	unsigned long pflags;
 	bool drained = false;
-	int reclaim_order = order;
-
-	/* Match the slowpath compaction promotion in __alloc_pages_direct_compact */
-	if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE)
-		reclaim_order = max(order, pageblock_order);
+	unsigned int reclaim_order = nofrag_promote_order(order, alloc_flags, ac);
 
 	psi_memstall_enter(&pflags);
 	*did_some_progress = __perform_reclaim(gfp_mask, reclaim_order, ac);
@@ -4648,9 +4651,17 @@ should_reclaim_retry(gfp_t gfp_mask, unsigned order,
 	/*
 	 * Costly allocations might have made a progress but this doesn't mean
 	 * their order will become available due to high fragmentation so
-	 * always increment the no progress counter for them
+	 * always increment the no progress counter for them.
+	 *
+	 * The same goes for requests whose reclaim is promoted to make whole
+	 * blocks. At that order, reclaim also reports progress when it backs
+	 * off for compaction without freeing anything.
+	 *
+	 * The watermark check below stays at the request order: it asks
+	 * whether the request itself could succeed after reclaim.
 	 */
-	if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
+	if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER &&
+	    nofrag_promote_order(order, alloc_flags, ac) == order)
 		*no_progress_loops = 0;
 	else
 		(*no_progress_loops)++;
@@ -4784,10 +4795,10 @@ static inline struct page *
 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 						struct alloc_context *ac)
 {
-	bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
-	bool can_compact = can_direct_reclaim && gfp_compaction_allowed(gfp_mask);
-	bool nofail = gfp_mask & __GFP_NOFAIL;
 	const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
+	bool can_direct_reclaim;
+	bool can_compact;
+	bool nofail;
 	struct page *page = NULL;
 	unsigned int alloc_flags;
 	unsigned long did_some_progress;
@@ -4802,9 +4813,21 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 	bool can_retry_reserves = true;
 	unsigned long alloc_start_time = jiffies;
 
+	/*
+	 * Costly __GFP_NORETRY callers have a cheap fallback, so don't stall
+	 * them in reclaim or compaction. __GFP_THISNODE callers are exempt.
+	 */
+	if (costly_order && (gfp_mask & __GFP_NORETRY) &&
+	    !(gfp_mask & __GFP_THISNODE))
+		gfp_mask &= ~__GFP_DIRECT_RECLAIM;
+
+	can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
+	can_compact = can_direct_reclaim && gfp_compaction_allowed(gfp_mask);
+	nofail = gfp_mask & __GFP_NOFAIL;
+
 	if (unlikely(nofail)) {
 		/*
-		 * Also we don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM,
+		 * We don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM,
 		 * otherwise, we may result in lockup.
 		 */
 		WARN_ON_ONCE(!can_direct_reclaim);
@@ -5012,9 +5035,15 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 				 &compaction_retries))
 		goto retry;
 
-	/* Reclaim/compaction failed to prevent the fallback */
+	/*
+	 * Reclaim/compaction failed to prevent the fallback. The retry
+	 * budget was spent on making blocks, not on the request itself;
+	 * give the fallback a fresh one before considering OOM.
+	 */
 	if (defrag_mode && (alloc_flags & ALLOC_NOFRAGMENT)) {
 		alloc_flags &= ~ALLOC_NOFRAGMENT;
+		no_progress_loops = 0;
+		compaction_retries = 0;
 		goto retry;
 	}
 
@@ -5207,6 +5236,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 
 	/* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
 	gfp &= gfp_allowed_mask;
+	gfp = current_gfp_context(gfp);
 	if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &gfp, &alloc_flags))
 		goto out;
 
@@ -5279,6 +5309,8 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 		nr_account++;
 
 		prep_new_page(page, 0, gfp, ALLOC_DEFAULT);
+		trace_mm_page_alloc(page, 0, gfp, ac.migratetype);
+		kmsan_alloc_page(page, 0, gfp & ~__GFP_RECLAIM);
 		set_page_refcounted(page);
 		page_array[nr_populated++] = page;
 	}
@@ -5655,7 +5687,7 @@ EXPORT_SYMBOL(alloc_pages_exact_noprof);
  *
  * Return: pointer to the allocated area or %NULL in case of error.
  */
-void * __meminit alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask)
+void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask)
 {
 	unsigned int order = get_order(size);
 	struct page *p;
@@ -5797,9 +5829,10 @@ static int numa_zonelist_order_handler(const struct ctl_table *table, int write,
 static int node_load[MAX_NUMNODES];
 
 /**
- * find_next_best_node - find the next node that should appear in a given node's fallback list
+ * find_next_best_node_in - find the next node that should appear in a given node's fallback list
  * @node: node whose fallback list we're appending
  * @used_node_mask: nodemask_t of already used nodes
+ * @candidates: nodemask_t of nodes eligible for selection
  *
  * We use a number of factors to determine which is the next node that should
  * appear on a given node's fallback list.  The node should not have appeared
@@ -5811,7 +5844,8 @@ static int node_load[MAX_NUMNODES];
  *
  * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
  */
-int find_next_best_node(int node, nodemask_t *used_node_mask)
+int find_next_best_node_in(int node, nodemask_t *used_node_mask,
+			   const nodemask_t *candidates)
 {
 	int n, val;
 	int min_val = INT_MAX;
@@ -5821,12 +5855,12 @@ int find_next_best_node(int node, nodemask_t *used_node_mask)
 	 * Use the local node if we haven't already, but for memoryless local
 	 * node, we should skip it and fall back to other nodes.
 	 */
-	if (!node_isset(node, *used_node_mask) && node_state(node, N_MEMORY)) {
+	if (!node_isset(node, *used_node_mask) && node_isset(node, *candidates)) {
 		node_set(node, *used_node_mask);
 		return node;
 	}
 
-	for_each_node_state(n, N_MEMORY) {
+	for_each_node_mask(n, *candidates) {
 
 		/* Don't want a node to appear more than once */
 		if (node_isset(n, *used_node_mask))
@@ -5859,31 +5893,6 @@ int find_next_best_node(int node, nodemask_t *used_node_mask)
 }
 
 
-/*
- * Build zonelists ordered by node and zones within node.
- * This results in maximum locality--normal zone overflows into local
- * DMA zone, if any--but risks exhausting DMA zone.
- */
-static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
-		unsigned nr_nodes)
-{
-	struct zoneref *zonerefs;
-	int i;
-
-	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
-
-	for (i = 0; i < nr_nodes; i++) {
-		int nr_zones;
-
-		pg_data_t *node = NODE_DATA(node_order[i]);
-
-		nr_zones = build_zonerefs_node(node, zonerefs);
-		zonerefs += nr_zones;
-	}
-	zonerefs->zone = NULL;
-	zonerefs->zone_idx = 0;
-}
-
 /*
  * Build __GFP_THISNODE zonelists
  */
@@ -5899,19 +5908,24 @@ static void build_thisnode_zonelists(pg_data_t *pgdat)
 	zonerefs->zone_idx = 0;
 }
 
-static void build_zonelists(pg_data_t *pgdat)
+/*
+ * Build one zonelist ordered by node and zones within node. This results in
+ * maximum locality--normal zone overflows into local DMA zone, if any--but
+ * risks exhausting DMA zone.
+ */
+static void build_node_zonelist(pg_data_t *pgdat, const nodemask_t *candidates,
+				int zlidx)
 {
-	static int node_order[MAX_NUMNODES];
-	int node, nr_nodes = 0;
+	struct zoneref *zonerefs = pgdat->node_zonelists[zlidx]._zonerefs;
 	nodemask_t used_mask = NODE_MASK_NONE;
-	int local_node, prev_node;
+	int local_node = pgdat->node_id;
+	int prev_node = local_node;
+	int node;
 
-	/* NUMA-aware ordering of nodes */
-	local_node = pgdat->node_id;
-	prev_node = local_node;
+	pr_info("Fallback order for Node %d: ", local_node);
 
-	memset(node_order, 0, sizeof(node_order));
-	while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
+	while ((node = find_next_best_node_in(local_node, &used_mask,
+					      candidates)) >= 0) {
 		/*
 		 * We don't want to pressure a particular node.
 		 * So adding penalty to the first node in same
@@ -5921,18 +5935,22 @@ static void build_zonelists(pg_data_t *pgdat)
 		    node_distance(local_node, prev_node))
 			node_load[node] += 1;
 
-		node_order[nr_nodes++] = node;
+		zonerefs += build_zonerefs_node(NODE_DATA(node), zonerefs);
+		pr_cont("%d ", node);
 		prev_node = node;
 	}
 
-	build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
-	build_thisnode_zonelists(pgdat);
-	pr_info("Fallback order for Node %d: ", local_node);
-	for (node = 0; node < nr_nodes; node++)
-		pr_cont("%d ", node_order[node]);
+	zonerefs->zone = NULL;
+	zonerefs->zone_idx = 0;
 	pr_cont("\n");
 }
 
+static void build_zonelists(pg_data_t *pgdat)
+{
+	build_node_zonelist(pgdat, &node_states[N_MEMORY], ZONELIST_FALLBACK);
+	build_thisnode_zonelists(pgdat);
+}
+
 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
 /*
  * Return node id of node used for "local" allocations.
@@ -7978,7 +7996,7 @@ static bool cond_accept_memory(struct zone *zone, unsigned int order,
 	/*
 	 * Watermarks have not been initialized yet.
 	 *
-	 * Accepting one MAX_ORDER page to ensure progress.
+	 * Accepting one MAX_PAGE_ORDER page to ensure progress.
 	 */
 	if (!wmark)
 		return try_to_accept_memory_one(zone);
diff --git a/mm/page_alloc.h b/mm/page_alloc.h
index b9259deddb59d7..d4182f7cb7ddf6 100644
--- a/mm/page_alloc.h
+++ b/mm/page_alloc.h
@@ -214,7 +214,7 @@ extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
 static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
 				unsigned long end_pfn, struct zone *zone)
 {
-	if (zone->contiguous)
+	if (zone_is_contiguous(zone))
 		return pfn_to_page(start_pfn);
 
 	return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
diff --git a/mm/page_counter.c b/mm/page_counter.c
index 661e0f2a5127a5..9167ffd1380c82 100644
--- a/mm/page_counter.c
+++ b/mm/page_counter.c
@@ -8,6 +8,7 @@
 #include <linux/page_counter.h>
 #include <linux/atomic.h>
 #include <linux/kernel.h>
+#include <linux/math64.h>
 #include <linux/string.h>
 #include <linux/sched.h>
 #include <linux/bug.h>
@@ -46,6 +47,26 @@ static void propagate_protected_usage(struct page_counter *c,
 	}
 }
 
+/**
+ * page_counter_margin - remaining usable space within hierarchical limits
+ * @counter: counter
+ *
+ * Return: The minimum value of max minus usage across @counter and all of
+ * its ancestors. The value may be negative during a concurrent charge.
+ */
+long page_counter_margin(const struct page_counter *counter)
+{
+	long margin = PAGE_COUNTER_MAX;
+
+	do {
+		long m = READ_ONCE(counter->max) - page_counter_read(counter);
+
+		margin = min(margin, m);
+	} while ((counter = counter->parent));
+
+	return margin;
+}
+
 /**
  * page_counter_cancel - take pages out of the local counter
  * @counter: counter
@@ -356,7 +377,8 @@ static unsigned long effective_protection(unsigned long usage,
 	 * otherwise get a smaller chunk than what they claimed.
 	 */
 	if (siblings_protected > parent_effective)
-		return protected * parent_effective / siblings_protected;
+		return mul_u64_u64_div_u64(protected, parent_effective,
+					   siblings_protected);
 
 	/*
 	 * Ok, utilized protection of all children is within what the
@@ -397,13 +419,20 @@ static unsigned long effective_protection(unsigned long usage,
 	if (parent_effective > siblings_protected &&
 	    parent_usage > siblings_protected &&
 	    usage > protected) {
-		unsigned long unclaimed;
+		unsigned long parent_unclaimed, parent_unprotected, unprotected;
 
-		unclaimed = parent_effective - siblings_protected;
-		unclaimed *= usage - protected;
-		unclaimed /= parent_usage - siblings_protected;
+		parent_unclaimed = parent_effective - siblings_protected;
+		parent_unprotected = parent_usage - siblings_protected;
+
+		/*
+		 * The usages aren't read atomically, so a child can transiently
+		 * appear to use more than its parent, making the ratio exceed 1
+		 * and the quotient overflow 64 bits (#DE on x86).  Cap it.
+		 */
+		unprotected = min(usage - protected, parent_unprotected);
 
-		ep += unclaimed;
+		ep += mul_u64_u64_div_u64(parent_unclaimed, unprotected,
+					  parent_unprotected);
 	}
 
 	return ep;
diff --git a/mm/page_io.c b/mm/page_io.c
index 88962571cb931e..c6824fcd483e0a 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -135,7 +135,7 @@ static bool is_folio_zero_filled(struct folio *folio)
 	for (i = 0; i < folio_nr_pages(folio); i++) {
 		data = kmap_local_folio(folio, i * PAGE_SIZE);
 		/*
-		 * Check last word first, incase the page is zero-filled at
+		 * Check last word first, in case the page is zero-filled at
 		 * the start and has non-zero data at the end, which is common
 		 * in real-world workloads.
 		 */
@@ -160,7 +160,7 @@ static void swap_zeromap_folio_set(struct folio *folio)
 	struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio);
 	int nr_pages = folio_nr_pages(folio);
 	struct swap_cluster_info *ci;
-	swp_entry_t entry;
+	swp_entry_t entry = folio->swap;
 	unsigned int i;
 
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
@@ -168,8 +168,8 @@ static void swap_zeromap_folio_set(struct folio *folio)
 
 	ci = swap_cluster_get_and_lock(folio);
 	for (i = 0; i < folio_nr_pages(folio); i++) {
-		entry = page_swap_entry(folio_page(folio, i));
 		__swap_table_set_zero(ci, swp_cluster_offset(entry));
+		entry.val++;
 	}
 	swap_cluster_unlock(ci);
 
@@ -183,7 +183,7 @@ static void swap_zeromap_folio_set(struct folio *folio)
 static void swap_zeromap_folio_clear(struct folio *folio)
 {
 	struct swap_cluster_info *ci;
-	swp_entry_t entry;
+	swp_entry_t entry = folio->swap;
 	unsigned int i;
 
 	VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
@@ -191,8 +191,8 @@ static void swap_zeromap_folio_clear(struct folio *folio)
 
 	ci = swap_cluster_get_and_lock(folio);
 	for (i = 0; i < folio_nr_pages(folio); i++) {
-		entry = page_swap_entry(folio_page(folio, i));
 		__swap_table_clear_zero(ci, swp_cluster_offset(entry));
+		entry.val++;
 	}
 	swap_cluster_unlock(ci);
 }
@@ -209,7 +209,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
 		goto out_unlock;
 
 	/*
-	 * Arch code may have to preserve more data than just the page
+	 * Arch code may have to preserve more data than just the folio
 	 * contents, e.g. memory tags.
 	 */
 	ret = arch_prepare_to_swap(folio);
@@ -220,7 +220,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
 
 	/*
 	 * Use the swap table zero mark to avoid doing IO for zero-filled
-	 * pages. The zero mark is protected by the cluster lock, which is
+	 * folios. The zero mark is protected by the cluster lock, which is
 	 * acquired internally by swap_zeromap_folio_set/clear.
 	 */
 	if (is_folio_zero_filled(folio)) {
@@ -248,7 +248,7 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
 	}
 	rcu_read_unlock();
 
-	__swap_writepage(ctx, folio);
+	__swap_writeout(ctx, folio);
 	return 0;
 out_unlock:
 	folio_unlock(folio);
@@ -256,12 +256,13 @@ int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
 }
 
 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
-static struct cgroup_subsys_state *folio_memcg_blkg_css(struct folio *folio)
+static struct cgroup_subsys_state *folio_memcg_blkg_css(const struct folio *folio)
 {
 	return cgroup_e_css(folio_memcg(folio)->css.cgroup, &io_cgrp_subsys);
 }
 
-static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
+static bool folio_blkg_can_merge(const struct folio *folio,
+				 const struct folio *prev_folio)
 {
 	bool can_merge = true;
 
@@ -277,7 +278,8 @@ static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
 	return can_merge;
 }
 
-static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
+static void bio_associate_blkg_from_folio(struct bio *bio,
+					  const struct folio *folio)
 {
 	struct cgroup_subsys_state *css;
 
@@ -294,11 +296,15 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
 		css_put(css);
 }
 #else
-static bool folio_blkg_can_merge(struct folio *folio, struct folio *prev_folio)
+static bool folio_blkg_can_merge(const struct folio *folio,
+				 const struct folio *prev_folio)
 {
 	return true;
 }
-#define bio_associate_blkg_from_page(bio, folio)		do { } while (0)
+static void bio_associate_blkg_from_folio(struct bio *bio,
+					  const struct folio *folio)
+{
+}
 #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
 
 static mempool_t *sio_pool;
@@ -367,7 +373,7 @@ static void swap_add_folio(struct swap_io_ctx *ctx, struct folio *folio, int rw)
 	}
 }
 
-void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio)
+void __swap_writeout(struct swap_io_ctx *ctx, struct folio *folio)
 {
 	VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
 
@@ -497,13 +503,13 @@ static void swap_write_end(struct swap_iocb *sio, bool failed)
 	int p;
 
 	for (p = 0; p < sio->nr_bvecs; p++) {
-		struct page *page = sio->bvecs[p].bv_page;
+		struct folio *folio = bvec_folio(&sio->bvecs[p]);
 
 		if (failed) {
-			set_page_dirty(page);
-			ClearPageReclaim(page);
+			folio_mark_dirty(folio);
+			folio_clear_reclaim(folio);
 		}
-		end_page_writeback(page);
+		folio_end_writeback(folio);
 	}
 	mempool_free(sio, sio_pool);
 }
@@ -514,16 +520,16 @@ static void swap_fs_write_complete(struct kiocb *iocb, long ret)
 	bool failed = ret != sio->len;
 
 	if (failed) {
-		struct page *page = sio->bvecs[0].bv_page;
+		struct folio *folio = bvec_folio(&sio->bvecs[0]);
 
 		/*
 		 * In the case of swap-over-nfs, this can be a temporary failure
 		 * if the system has limited memory for allocating transmit
-		 * buffers.  Mark the page dirty and avoid
+		 * buffers.  Mark the folio dirty and avoid
 		 * folio_rotate_reclaimable but rate-limit the messages.
 		 */
 		pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
-				   ret, swap_dev_pos(page_swap_entry(page)));
+				   ret, swap_dev_pos(folio->swap));
 	}
 
 	swap_write_end(sio, failed);
@@ -596,12 +602,21 @@ static void swap_bdev_submit_write(struct swap_io_ctx *ctx)
 			REQ_OP_WRITE | REQ_SWAP);
 	bio->bi_iter.bi_size = sio->len;
 	bio->bi_iter.bi_sector = swap_folio_sector(bio_first_folio_all(bio));
-	bio_associate_blkg_from_page(bio, bio_first_folio_all(bio));
+	bio_associate_blkg_from_folio(bio, bio_first_folio_all(bio));
 
 	if (ctx->sis->flags & SWP_SYNCHRONOUS_IO) {
 		submit_bio_wait(bio);
 		end_swap_bio_write(bio);
 	} else {
+		int p;
+
+		for (p = 0; p < sio->nr_bvecs; p++) {
+			if (folio_test_dropbehind(bvec_folio(&sio->bvecs[p]))) {
+				bio_set_flag(bio, BIO_COMPLETE_IN_TASK);
+				break;
+			}
+		}
+
 		bio->bi_end_io = end_swap_bio_write;
 		submit_bio(bio);
 	}
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index e5dfc7bf494460..5aaad037384e44 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -388,13 +388,19 @@ static int isolate_single_pageblock(unsigned long boundary_pfn,
 		}
 
 		if (PageBuddy(page)) {
-			int order = buddy_order(page);
+			unsigned int order = buddy_order_unsafe(page);
 
-			/* pageblock_isolate_and_move_free_pages() handled this */
-			VM_WARN_ON_ONCE(pfn + (1 << order) > boundary_pfn);
-
-			pfn += 1UL << order;
-			continue;
+			/* buddy_order_unsafe() is racy. Validate the order before shifting. */
+			if (order <= MAX_PAGE_ORDER &&
+				/*
+				 * pageblock_isolate_and_move_free_pages() splits
+				 * cross-boundary PageBuddy, verify it.
+				 */
+			    pfn + (1UL << order) <= boundary_pfn) {
+				pfn += 1UL << order;
+				continue;
+			}
+			goto failed;
 		}
 
 		/*
@@ -413,10 +419,28 @@ static int isolate_single_pageblock(unsigned long boundary_pfn,
 		if (PageCompound(page)) {
 			struct page *head = compound_head(page);
 			unsigned long head_pfn = page_to_pfn(head);
-			unsigned long nr_pages = compound_nr(head);
+			unsigned int order = compound_order(head);
+			unsigned long nr_pages;
+
+			/* compound_order() is racy. Cap it at MAX_FOLIO_ORDER. */
+			if (order > MAX_FOLIO_ORDER)
+				goto failed;
+
+			nr_pages = 1UL << order;
+
+			/*
+			 * compound_head() is also racy, so the derived head_pfn
+			 * needs additional checks to make sure it is valid.
+			 * Otherwise, just fail the check. pfn comes from
+			 * __first_valid_page() as a legitimate PFN, so use it to
+			 * check head_pfn.
+			 */
+			if (head_pfn > pfn || !IS_ALIGNED(head_pfn, nr_pages) ||
+			    pfn - head_pfn >= nr_pages)
+				goto failed;
 
 			if (head_pfn + nr_pages <= boundary_pfn ||
-			    PageHuge(page)) {
+			    PageHuge(head)) {
 				pfn = head_pfn + nr_pages;
 				continue;
 			}
diff --git a/mm/page_owner.c b/mm/page_owner.c
index fbbda7ba914ba5..cfc31c92d76570 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -307,8 +307,8 @@ static inline void __update_page_owner_free_handle(struct page *page,
 			page_owner->free_handle = handle;
 		}
 		page_owner->free_ts_nsec = free_ts_nsec;
-		page_owner->free_pid = current->pid;
-		page_owner->free_tgid = current->tgid;
+		page_owner->free_pid = pid;
+		page_owner->free_tgid = tgid;
 	}
 	rcu_read_unlock();
 }
@@ -575,7 +575,7 @@ static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
 	}
 
 	objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK);
-	memcg = objcg ? obj_cgroup_memcg(objcg) : NULL;
+	memcg = obj_cgroup_memcg(objcg);
 	if (!memcg)
 		goto out_unlock;
 
diff --git a/mm/page_table_check.c b/mm/page_table_check.c
index 6ffc536359cd06..143a918c0bde22 100644
--- a/mm/page_table_check.c
+++ b/mm/page_table_check.c
@@ -67,7 +67,7 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
 	struct page *page;
 	bool anon;
 
-	if (!pfn_valid(pfn))
+	if (!pfn_valid(pfn) || is_zero_pfn(pfn) || is_huge_zero_pfn(pfn))
 		return;
 
 	page = pfn_to_page(pfn);
@@ -102,7 +102,7 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
 	struct page *page;
 	bool anon;
 
-	if (!pfn_valid(pfn))
+	if (!pfn_valid(pfn) || is_zero_pfn(pfn) || is_huge_zero_pfn(pfn))
 		return;
 
 	page = pfn_to_page(pfn);
@@ -278,7 +278,7 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm,
 	if (&init_mm == mm)
 		return;
 
-	if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
+	if (!pmd_none(pmd) && !pmd_bad(pmd) && !pmd_leaf(pmd)) {
 		pte_t *ptep = pte_offset_map(&pmd, addr);
 		unsigned long i;
 
diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c
index 4e964545e5e85a..8408aee7571b56 100644
--- a/mm/page_vma_mapped.c
+++ b/mm/page_vma_mapped.c
@@ -109,7 +109,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr)
 	unsigned long pfn;
 	pte_t ptent;
 
-	if (is_vm_hugetlb_page(pvmw->vma))
+	if (vma_is_hugetlb(pvmw->vma))
 		ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address,
 				      pvmw->pte);
 	else
@@ -142,6 +142,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr)
 	return true;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 /* Returns true if the two ranges overlap.  Careful to not overflow. */
 static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
 {
@@ -151,6 +152,12 @@ static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
 		return false;
 	return true;
 }
+#else
+static bool check_pmd(unsigned long pfn, struct page_vma_mapped_walk *pvmw)
+{
+	return false;
+}
+#endif
 
 static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size)
 {
@@ -199,7 +206,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
 	if (pvmw->pmd && !pvmw->pte)
 		return not_found(pvmw);
 
-	if (unlikely(is_vm_hugetlb_page(vma))) {
+	if (unlikely(vma_is_hugetlb(vma))) {
 		struct hstate *hstate = hstate_vma(vma);
 		unsigned long size = huge_page_size(hstate);
 		/* The only possible mapping was handled on last iteration */
diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index cc07fcf50e87b3..e6493bbe6919e5 100644
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -408,7 +408,7 @@ static int __walk_page_range(unsigned long start, unsigned long end,
 	int err = 0;
 	struct vm_area_struct *vma = walk->vma;
 	const struct mm_walk_ops *ops = walk->ops;
-	bool is_hugetlb = is_vm_hugetlb_page(vma);
+	bool is_hugetlb = vma_is_hugetlb(vma);
 
 	/* We do not support hugetlb PTE installation. */
 	if (ops->install_pte && is_hugetlb)
@@ -444,7 +444,6 @@ static inline void process_mm_walk_lock(struct mm_struct *mm,
 static inline void process_vma_walk_lock(struct vm_area_struct *vma,
 					 enum page_walk_lock walk_lock)
 {
-#ifdef CONFIG_PER_VMA_LOCK
 	switch (walk_lock) {
 	case PGWALK_WRLOCK:
 		vma_start_write(vma);
@@ -459,7 +458,6 @@ static inline void process_vma_walk_lock(struct vm_area_struct *vma,
 		/* PGWALK_RDLOCK is handled by process_mm_walk_lock */
 		break;
 	}
-#endif
 }
 
 /*
diff --git a/mm/percpu.c b/mm/percpu.c
index a802d72c116fbb..3eff382e565ad5 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -100,7 +100,7 @@
 
 /*
  * The slots are sorted by the size of the biggest continuous free area.
- * 1-31 bytes share the same slot.
+ * [PCPU_MIN_ALLOC_SIZE..15] bytes share the same slot.
  */
 #define PCPU_SLOT_BASE_SHIFT		5
 /* chunks in slots below this are subject to being sidelined on failed alloc */
@@ -758,7 +758,6 @@ static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan)
 		chunk_md->contig_hint = 0;
 	}
 
-	bits = 0;
 	pcpu_for_each_md_free_region(chunk, bit_off, bits)
 		pcpu_block_update(chunk_md, bit_off, bit_off + bits);
 }
@@ -1122,7 +1121,6 @@ static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits,
 		return -1;
 
 	bit_off = pcpu_next_hint(chunk_md, alloc_bits);
-	bits = 0;
 	pcpu_for_each_fit_region(chunk, alloc_bits, align, bit_off, bits) {
 		if (!pop_only || pcpu_is_populated(chunk, bit_off, bits,
 						   &next_off))
@@ -2819,7 +2817,7 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info(
 	static int group_cnt[NR_CPUS] __initdata;
 	static struct cpumask mask __initdata;
 	const size_t static_size = __per_cpu_end - __per_cpu_start;
-	int nr_groups = 1, nr_units = 0;
+	int nr_groups, nr_units = 0;
 	size_t size_sum, min_unit_size, alloc_size;
 	int upa, max_upa, best_upa;	/* units_per_alloc */
 	int last_allocs, group, unit;
@@ -2830,7 +2828,6 @@ static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info(
 	/* this function may be called multiple times */
 	memset(group_map, 0, sizeof(group_map));
 	memset(group_cnt, 0, sizeof(group_cnt));
-	cpumask_clear(&mask);
 
 	/* calculate size_sum and ensure dyn_size is enough for early alloc */
 	size_sum = PFN_ALIGN(static_size + reserved_size +
@@ -3181,8 +3178,6 @@ void __init __weak pcpu_populate_pte(unsigned long addr)
 		new = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE);
 		pmd_populate_kernel(&init_mm, pmd, new);
 	}
-
-	return;
 }
 
 /**
diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c
index b91b1a98029c7f..6e83ce3801b8af 100644
--- a/mm/pgtable-generic.c
+++ b/mm/pgtable-generic.c
@@ -26,14 +26,20 @@
 
 void pgd_clear_bad(pgd_t *pgd)
 {
-	pgd_ERROR(*pgd);
+	char str[PTVAL_STR_MAX];
+
+	ptval_to_str(str, pgd_val(*pgd));
+	pr_err("bad pgd %s.\n", str);
 	pgd_clear(pgd);
 }
 
 #ifndef __PAGETABLE_P4D_FOLDED
 void p4d_clear_bad(p4d_t *p4d)
 {
-	p4d_ERROR(*p4d);
+	char str[PTVAL_STR_MAX];
+
+	ptval_to_str(str, p4d_val(*p4d));
+	pr_err("bad p4d %s.\n", str);
 	p4d_clear(p4d);
 }
 #endif
@@ -41,7 +47,10 @@ void p4d_clear_bad(p4d_t *p4d)
 #ifndef __PAGETABLE_PUD_FOLDED
 void pud_clear_bad(pud_t *pud)
 {
-	pud_ERROR(*pud);
+	char str[PTVAL_STR_MAX];
+
+	ptval_to_str(str, pud_val(*pud));
+	pr_err("bad pud %s.\n", str);
 	pud_clear(pud);
 }
 #endif
@@ -53,7 +62,10 @@ void pud_clear_bad(pud_t *pud)
  */
 void pmd_clear_bad(pmd_t *pmd)
 {
-	pmd_ERROR(*pmd);
+	char str[PTVAL_STR_MAX];
+
+	ptval_to_str(str, pmd_val(*pmd));
+	pr_err("bad pmd %s.\n", str);
 	pmd_clear(pmd);
 }
 
@@ -337,7 +349,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
  * pte_offset_map_lock(mm, pmd, addr, ptlp) is usually called with the pmd
  * pointer for addr, reached by walking down the mm's pgd, p4d, pud for addr:
  * either while holding mmap_lock or vma lock for read or for write; or in
- * truncate or rmap context, while holding file's i_mmap_lock or anon_vma lock
+ * truncate or rmap context, while holding file's i_mmap_lock or anon rmap lock
  * for read (or for write). In a few cases, it may be used with pmd pointing to
  * a pmd_t already copied to or constructed on the stack.
  *
@@ -385,10 +397,17 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
  * Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will
  * be read-only/read-write protected.
  *
- * Note that free_pgtables(), used after unmapping detached vmas, or when
- * exiting the whole mm, does not take page table lock before freeing a page
- * table, and may not use RCU at all: "outsiders" like khugepaged should avoid
- * pte_offset_map() and co once the vma is detached from mm or mm_users is zero.
+ * Note that free_pgtables(), used after unmapping detached vmas or when exiting
+ * the whole mm, does not take a page table lock before freeing a page table.
+ *
+ * As page table freeing itself is RCU-safe, page table readers can safely run
+ * concurrently with page table teardown.
+ *
+ * However, writers CANNOT as, without a lock being held, nothing prevents
+ * concurrent teardown.
+ *
+ * Also note that the PGD itself is freed at mmdrop() time, not under RCU - so
+ * the walker must keep the mm alive either by pinning the mm or the VMA.
  */
 pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
 			   unsigned long addr, spinlock_t **ptlp)
@@ -438,12 +457,30 @@ static void kernel_pgtable_work_func(struct work_struct *work)
 		__pagetable_free(pt);
 }
 
+static void schedule_kernel_pgtable_free(void)
+{
+	schedule_work(&kernel_pgtable_work.work);
+}
+
 void pagetable_free_kernel(struct ptdesc *pt)
 {
 	spin_lock(&kernel_pgtable_work.lock);
 	list_add(&pt->pt_list, &kernel_pgtable_work.list);
 	spin_unlock(&kernel_pgtable_work.lock);
 
-	schedule_work(&kernel_pgtable_work.work);
+	/*
+	 * The workqueue may not exist yet while the system is booting.
+	 * kernel_pgtable_drain_early() schedules the work once it does.
+	 */
+	if (system_state != SYSTEM_BOOTING)
+		schedule_kernel_pgtable_free();
+}
+
+static int __init kernel_pgtable_drain_early(void)
+{
+	/* Free the kernel page tables queued while booting. */
+	schedule_kernel_pgtable_free();
+	return 0;
 }
+core_initcall(kernel_pgtable_drain_early);
 #endif
diff --git a/mm/rmap.c b/mm/rmap.c
index d1819fd6993800..fbd66a2823b7b3 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -208,8 +208,12 @@ int __anon_vma_prepare(struct vm_area_struct *vma)
 	anon_vma_lock_write(anon_vma);
 	/* page_table_lock to protect against threads */
 	spin_lock(&mm->page_table_lock);
-	if (likely(!vma->anon_vma)) {
-		vma->anon_vma = anon_vma;
+	if (likely(!vma_has_anon_rmap(vma))) {
+		/*
+		 * Make anon_vma fields visible before anon_vma is published.
+		 * Paired with an address dependency in reusable_anon_vma().
+		 */
+		smp_store_release(&vma->anon_vma, anon_vma);
 		anon_vma_chain_assign(vma, avc, anon_vma);
 		anon_rmap_tree_insert(avc, anon_vma);
 		anon_vma->num_active_vmas++;
@@ -242,36 +246,34 @@ static void check_anon_vma_clone(struct vm_area_struct *dst,
 	VM_WARN_ON_ONCE(operation != VMA_OP_FORK && dst->vm_mm != src->vm_mm);
 
 	/* If we have anything to do src->anon_vma must be provided. */
-	VM_WARN_ON_ONCE(!src->anon_vma && !list_empty(&src->anon_vma_chain));
-	VM_WARN_ON_ONCE(!src->anon_vma && dst->anon_vma);
+	VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && !list_empty(&src->anon_vma_chain));
+	VM_WARN_ON_ONCE(!vma_has_anon_rmap(src) && vma_has_anon_rmap(dst));
 	/* We are establishing a new anon_vma_chain. */
 	VM_WARN_ON_ONCE(!list_empty(&dst->anon_vma_chain));
 	/*
 	 * On fork, dst->anon_vma is set NULL (temporarily). Otherwise, anon_vma
 	 * must be the same across dst and src.
 	 */
-	VM_WARN_ON_ONCE(dst->anon_vma && dst->anon_vma != src->anon_vma);
+	VM_WARN_ON_ONCE(vma_has_anon_rmap(dst) && dst->anon_vma != src->anon_vma);
 	/*
 	 * Essentially equivalent to above - if not a no-op, we should expect
 	 * dst->anon_vma to be set for everything except a fork.
 	 */
-	VM_WARN_ON_ONCE(operation != VMA_OP_FORK && src->anon_vma &&
-			!dst->anon_vma);
+	VM_WARN_ON_ONCE(operation != VMA_OP_FORK && vma_has_anon_rmap(src) &&
+			!vma_has_anon_rmap(dst));
 	/* For the anon_vma to be compatible, it can only be singular. */
 	VM_WARN_ON_ONCE(operation == VMA_OP_MERGE_UNFAULTED &&
 			!list_is_singular(&src->anon_vma_chain));
-#ifdef CONFIG_PER_VMA_LOCK
 	/* Only merging an unfaulted VMA leaves the destination attached. */
 	VM_WARN_ON_ONCE(operation != VMA_OP_MERGE_UNFAULTED &&
 			vma_is_attached(dst));
-#endif
 }
 
 static void maybe_reuse_anon_vma(struct vm_area_struct *dst,
 		struct anon_vma *anon_vma)
 {
 	/* If already populated, nothing to do.*/
-	if (dst->anon_vma)
+	if (vma_has_anon_rmap(dst))
 		return;
 
 	/*
@@ -325,7 +327,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,
 
 	check_anon_vma_clone(dst, src, operation);
 
-	if (!active_anon_vma)
+	if (!vma_has_anon_rmap(src))
 		return 0;
 
 	/*
@@ -382,7 +384,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
 	int rc;
 
 	/* Don't bother if the parent process has no anon_vma here. */
-	if (!pvma->anon_vma)
+	if (!vma_has_anon_rmap(pvma))
 		return 0;
 
 	/* Drop inherited anon_vma, we'll reuse existing or allocate new. */
@@ -403,7 +405,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
 	 */
 	rc = anon_vma_clone(vma, pvma, VMA_OP_FORK);
 	/* An error arose or an existing anon_vma was reused, all done then. */
-	if (rc || vma->anon_vma) {
+	if (rc || vma_has_anon_rmap(vma)) {
 		put_anon_vma(anon_vma);
 		anon_vma_chain_free(avc);
 		return rc;
@@ -862,7 +864,7 @@ unsigned long page_address_in_vma(const struct folio *folio,
 		 * Note: swapoff's unuse_vma() is more efficient with this
 		 * check, and needs it to match anon_vma when KSM is active.
 		 */
-		if (!vma->anon_vma || !anon_vma ||
+		if (!vma_has_anon_rmap(vma) || !anon_vma ||
 		    vma->anon_vma->root != anon_vma->root)
 			return -EFAULT;
 		/* KSM folios don't reach here because of the !anon_vma check */
@@ -2145,7 +2147,7 @@ static bool ttu_anon_swapbacked_folio(struct vm_area_struct *vma,
 {
 	const bool anon_exclusive = folio_test_anon(folio) &&
 				    PageAnonExclusive(page);
-	swp_entry_t entry = page_swap_entry(page);
+	swp_entry_t entry = folio_page_swap_entry(folio, page);
 	struct mm_struct *mm = vma->vm_mm;
 
 	if (folio_dup_swap(folio, page) < 0)
@@ -2237,9 +2239,11 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 
 		/*
 		 * If the folio is in an mlock()d vma, we must not swap it out.
+		 * VMA_LOCKONFAULT_BIT alone marks an mlock walk in progress, see
+		 * mlock_vma_pages_range().
 		 */
 		if (!(flags & TTU_IGNORE_MLOCK) &&
-		    (vma->vm_flags & VM_LOCKED)) {
+		    vma_test_any_mask(vma, VMA_LOCKED_MASK)) {
 			ptes++;
 
 			/*
@@ -2299,11 +2303,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 		VM_BUG_ON_FOLIO(!pvmw.pte, folio);
 
 		address = pvmw.address;
-		if (folio_test_hugetlb(folio)) {
-			pteval = huge_ptep_get(mm, address, pvmw.pte);
-		} else {
-			pteval = ptep_get(pvmw.pte);
-		}
+		pteval = ptep_get(pvmw.pte);
+
 		if (likely(pte_present(pteval))) {
 			pfn = pte_pfn(pteval);
 		} else {
diff --git a/mm/secretmem.c b/mm/secretmem.c
index 384f5cfc457f9e..b3f79ad56790a4 100644
--- a/mm/secretmem.c
+++ b/mm/secretmem.c
@@ -39,7 +39,7 @@
 
 static bool secretmem_enable __ro_after_init = 1;
 module_param_named(enable, secretmem_enable, bool, 0400);
-MODULE_PARM_DESC(secretmem_enable,
+MODULE_PARM_DESC(enable,
 		 "Enable secretmem and memfd_secret(2) system call");
 
 static atomic_t secretmem_users;
@@ -139,7 +139,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf)
 			goto out;
 		}
 
-		err = set_direct_map_invalid_noflush(folio_page(folio, 0));
+		err = set_direct_map_invalid_noflush(folio_page(folio, 0), 1);
 		if (err) {
 			secretmem_unaccount_folio(state, folio);
 			folio_put(folio);
@@ -156,7 +156,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf)
 			 * already happened when we marked the page invalid
 			 * which guarantees that this call won't fail
 			 */
-			set_direct_map_default_noflush(folio_page(folio, 0));
+			set_direct_map_default_noflush(folio_page(folio, 0), 1);
 			folio_put(folio);
 			if (err == -EEXIST)
 				goto retry;
@@ -228,7 +228,7 @@ static int secretmem_migrate_folio(struct address_space *mapping,
 
 static void secretmem_free_folio(struct folio *folio)
 {
-	set_direct_map_default_noflush(folio_page(folio, 0));
+	set_direct_map_default_noflush(folio_page(folio, 0), 1);
 	folio_zero_segment(folio, 0, folio_size(folio));
 }
 
@@ -238,7 +238,7 @@ const struct address_space_operations secretmem_aops = {
 	.migrate_folio	= secretmem_migrate_folio,
 };
 
-static int secretmem_setattr(struct mnt_idmap *idmap,
+static int secretmem_setattr(const struct mnt_idmap *idmap,
 			     struct dentry *dentry, struct iattr *iattr)
 {
 	struct inode *inode = d_inode(dentry);
diff --git a/mm/shmem.c b/mm/shmem.c
index 897fa2b61346f6..94dc2981fbbaaa 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -725,51 +725,258 @@ static const char *shmem_format_huge(int huge)
 }
 #endif
 
-static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
-		struct shrink_control *sc, unsigned long nr_to_free)
+static bool is_shmem_unused_huge_isolated(struct shmem_inode_info *info)
 {
-	LIST_HEAD(list), *pos, *next;
-	struct inode *inode;
+
+	return info->shrinklist_nid == -1;
+}
+
+static void set_shmem_unused_huge_isolated(struct shmem_inode_info *info)
+{
+	info->shrinklist_nid = -1;
+}
+
+static struct obj_cgroup *shmem_get_and_clear_objcg(struct shmem_inode_info *info)
+{
+	struct obj_cgroup *objcg = info->shrinklist_objcg;
+
+	info->shrinklist_objcg = NULL;
+
+	return objcg;
+}
+
+#ifdef CONFIG_MEMCG
+static struct obj_cgroup *
+shmem_unused_huge_alloc_lru(struct shmem_sb_info *sbinfo, struct folio *folio,
+			    gfp_t gfp)
+{
+	int ret;
+
+	ret = folio_memcg_list_lru_alloc(folio, &sbinfo->shrinklist, gfp);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return get_obj_cgroup_from_folio(folio);
+}
+#else
+static struct obj_cgroup *
+shmem_unused_huge_alloc_lru(struct shmem_sb_info *sbinfo, struct folio *folio,
+			    gfp_t gfp)
+{
+	return NULL;
+}
+#endif
+
+static void shmem_unused_huge_lru_add(struct shmem_sb_info *sbinfo,
+				      struct list_head *item, int nid,
+				      struct obj_cgroup *objcg)
+{
+	struct mem_cgroup *memcg;
+
+	rcu_read_lock();
+	memcg = obj_cgroup_memcg(objcg);
+	list_lru_add(&sbinfo->shrinklist, item, nid, memcg);
+	rcu_read_unlock();
+}
+
+static void shmem_unused_huge_lru_del(struct shmem_sb_info *sbinfo,
+				      struct list_head *item, int nid,
+				      struct obj_cgroup *objcg)
+{
+	struct mem_cgroup *memcg;
+
+	rcu_read_lock();
+	memcg = obj_cgroup_memcg(objcg);
+	list_lru_del(&sbinfo->shrinklist, item, nid, memcg);
+	rcu_read_unlock();
+}
+
+static void shmem_unused_huge_add(struct inode *inode, struct folio *folio,
+				  gfp_t gfp)
+{
+	struct shmem_inode_info *info = SHMEM_I(inode);
+	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+	int nid = folio_nid(folio);
+	struct obj_cgroup *objcg = NULL, *old_objcg = NULL;
+
+	objcg = shmem_unused_huge_alloc_lru(sbinfo, folio, gfp);
+	if (IS_ERR(objcg))
+		return;
+
+	spin_lock(&info->lock);
+	if (!list_empty(&info->shrinklist)) {
+		/* isolated on scan list, let shrink handle it */
+		if (is_shmem_unused_huge_isolated(info))
+			goto unlock;
+
+		if (info->shrinklist_nid == nid &&
+		    info->shrinklist_objcg == objcg)
+			goto unlock;
+
+		shmem_unused_huge_lru_del(sbinfo, &info->shrinklist,
+					  info->shrinklist_nid,
+					  info->shrinklist_objcg);
+		old_objcg = shmem_get_and_clear_objcg(info);
+	}
+
+	info->shrinklist_objcg = objcg;
+	info->shrinklist_nid = nid;
+	shmem_unused_huge_lru_add(sbinfo, &info->shrinklist, nid, objcg);
+	objcg = NULL;
+unlock:
+	spin_unlock(&info->lock);
+	obj_cgroup_put(old_objcg);
+	obj_cgroup_put(objcg);
+}
+
+static void shmem_unused_huge_del(struct inode *inode)
+{
+	struct shmem_inode_info *info = SHMEM_I(inode);
+	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+	struct obj_cgroup *objcg = NULL;
+
+	spin_lock(&info->lock);
+	if (!list_empty(&info->shrinklist)) {
+		shmem_unused_huge_lru_del(sbinfo, &info->shrinklist,
+					  info->shrinklist_nid,
+					  info->shrinklist_objcg);
+		objcg = shmem_get_and_clear_objcg(info);
+	}
+	spin_unlock(&info->lock);
+
+	obj_cgroup_put(objcg);
+}
+
+struct shmem_unused_huge_scan {
+	struct list_head list;
+	struct shrink_control *sc;
+};
+
+static enum lru_status shmem_unused_huge_isolate(struct list_head *item,
+						 struct list_lru_one *lru,
+						 void *arg)
+{
+	struct shmem_unused_huge_scan *scan = arg;
 	struct shmem_inode_info *info;
-	struct folio *folio;
-	unsigned long batch = sc ? sc->nr_to_scan : 128;
-	unsigned long split = 0, freed = 0;
+	struct inode *inode;
+	struct obj_cgroup *objcg = NULL;
 
-	if (list_empty(&sbinfo->shrinklist))
-		return SHRINK_STOP;
+	info = list_entry(item, struct shmem_inode_info, shrinklist);
 
-	spin_lock(&sbinfo->shrinklist_lock);
-	list_for_each_safe(pos, next, &sbinfo->shrinklist) {
-		info = list_entry(pos, struct shmem_inode_info, shrinklist);
+	/*
+	 * Use trylock to avoid ABBA deadlock: add/del path takes info->lock
+	 * before the list_lru bucket lock, while here the order is reversed.
+	 */
+	if (!spin_trylock(&info->lock))
+		return LRU_SKIP;
 
-		/* pin the inode */
-		inode = igrab(&info->vfs_inode);
+	/* pin the inode */
+	inode = igrab(&info->vfs_inode);
+	/* inode is about to be evicted */
+	if (!inode) {
+		list_lru_isolate(lru, item);
+		objcg = shmem_get_and_clear_objcg(info);
+		spin_unlock(&info->lock);
+		obj_cgroup_put(objcg);
+		return LRU_REMOVED;
+	}
 
-		/* inode is about to be evicted */
-		if (!inode) {
-			list_del_init(&info->shrinklist);
-			goto next;
-		}
+	list_lru_isolate(lru, item);
+	objcg = shmem_get_and_clear_objcg(info);
+	set_shmem_unused_huge_isolated(info);
+	list_add_tail(&info->shrinklist, &scan->list);
+	spin_unlock(&info->lock);
+	obj_cgroup_put(objcg);
 
-		list_move(&info->shrinklist, &list);
-next:
-		sbinfo->shrinklist_len--;
-		if (!--batch)
-			break;
+	return LRU_REMOVED;
+}
+
+static bool is_shmem_unused_huge_match(struct folio *folio,
+				       struct shrink_control *sc)
+{
+	struct mem_cgroup *memcg = NULL;
+	bool match;
+
+	/* shmem quota reclaim has no NUMA node or memcg restriction */
+	if (!sc)
+		return true;
+
+	if (folio_nid(folio) != sc->nid)
+		return false;
+
+	/*
+	 * Only non-root memcg reclaim needs to match the folio charge against
+	 * sc->memcg. Skip the folio memcg check for global shrinker reclaim and
+	 * root memcg reclaim.
+	 */
+	if (!sc->memcg || mem_cgroup_is_root(sc->memcg))
+		return true;
+
+	memcg = get_mem_cgroup_from_folio(folio);
+	match = memcg == sc->memcg;
+	mem_cgroup_put(memcg);
+
+	return match;
+}
+
+static void shmem_unused_huge_drop(struct inode *inode)
+{
+	struct shmem_inode_info *info = SHMEM_I(inode);
+
+	spin_lock(&info->lock);
+	list_del_init(&info->shrinklist);
+	spin_unlock(&info->lock);
+}
+
+static void shmem_unused_huge_requeue(struct inode *inode, struct folio *folio)
+{
+	struct shmem_inode_info *info = SHMEM_I(inode);
+	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+	struct obj_cgroup *objcg;
+	int nid = folio_nid(folio);
+
+	objcg = shmem_unused_huge_alloc_lru(sbinfo, folio, GFP_NOWAIT);
+	if (IS_ERR(objcg)) {
+		shmem_unused_huge_drop(inode);
+		return;
 	}
-	spin_unlock(&sbinfo->shrinklist_lock);
 
-	list_for_each_safe(pos, next, &list) {
-		pgoff_t next, end;
+	spin_lock(&info->lock);
+	/* Requeue the inode to shrinklist */
+	list_del_init(&info->shrinklist);
+	shmem_unused_huge_lru_add(sbinfo, &info->shrinklist, nid, objcg);
+	info->shrinklist_objcg = objcg;
+	info->shrinklist_nid = nid;
+	spin_unlock(&info->lock);
+}
+
+static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
+		struct shrink_control *sc, unsigned long nr_to_free)
+{
+	struct shmem_unused_huge_scan scan;
+	struct inode *inode;
+	struct shmem_inode_info *info;
+	struct folio *folio;
+	struct list_head *pos, *next;
+	unsigned long split = 0, freed = 0;
+
+	INIT_LIST_HEAD(&scan.list);
+	scan.sc = sc;
+	if (sc)
+		list_lru_shrink_walk(&sbinfo->shrinklist, sc,
+				     shmem_unused_huge_isolate, &scan);
+	else
+		list_lru_walk(&sbinfo->shrinklist, shmem_unused_huge_isolate,
+			      &scan, 128);
+
+	list_for_each_safe(pos, next, &scan.list) {
+		pgoff_t folio_end, end;
 		loff_t i_size;
 		int ret;
 
 		info = list_entry(pos, struct shmem_inode_info, shrinklist);
 		inode = &info->vfs_inode;
 
-		if (nr_to_free && freed >= nr_to_free)
-			goto move_back;
-
 		i_size = i_size_read(inode);
 		folio = filemap_get_entry(inode->i_mapping, i_size / PAGE_SIZE);
 		if (!folio || xa_is_value(folio))
@@ -782,13 +989,19 @@ static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 		}
 
 		/* Check if there is anything to gain from splitting */
-		next = folio_next_index(folio);
+		folio_end = folio_next_index(folio);
 		end = shmem_fallocend(inode, DIV_ROUND_UP(i_size, PAGE_SIZE));
-		if (end <= folio->index || end >= next) {
+		if (end <= folio->index || end >= folio_end) {
 			folio_put(folio);
 			goto drop;
 		}
 
+		if (!is_shmem_unused_huge_match(folio, scan.sc))
+			goto move_back;
+
+		if (nr_to_free && freed >= nr_to_free)
+			goto move_back;
+
 		/*
 		 * Move the inode on the list back to shrinklist if we failed
 		 * to lock the page at this time.
@@ -796,35 +1009,30 @@ static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 		 * Waiting for the lock may lead to deadlock in the
 		 * reclaim path.
 		 */
-		if (!folio_trylock(folio)) {
-			folio_put(folio);
+		if (!folio_trylock(folio))
+			goto move_back;
+
+		if (!is_shmem_unused_huge_match(folio, scan.sc)) {
+			folio_unlock(folio);
 			goto move_back;
 		}
 
 		ret = split_folio(folio);
 		folio_unlock(folio);
-		folio_put(folio);
 
 		/* If split failed move the inode on the list back to shrinklist */
 		if (ret)
 			goto move_back;
 
-		freed += next - end;
+		freed += folio_end - end;
 		split++;
+		folio_put(folio);
 drop:
-		list_del_init(&info->shrinklist);
+		shmem_unused_huge_drop(inode);
 		goto put;
 move_back:
-		/*
-		 * Make sure the inode is either on the global list or deleted
-		 * from any local list before iput() since it could be deleted
-		 * in another thread once we put the inode (then the local list
-		 * is corrupted).
-		 */
-		spin_lock(&sbinfo->shrinklist_lock);
-		list_move(&info->shrinklist, &sbinfo->shrinklist);
-		sbinfo->shrinklist_len++;
-		spin_unlock(&sbinfo->shrinklist_lock);
+		shmem_unused_huge_requeue(inode, folio);
+		folio_put(folio);
 put:
 		iput(inode);
 	}
@@ -837,7 +1045,7 @@ static long shmem_unused_huge_scan(struct super_block *sb,
 {
 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 
-	if (!READ_ONCE(sbinfo->shrinklist_len))
+	if (!list_lru_shrink_count(&sbinfo->shrinklist, sc))
 		return SHRINK_STOP;
 
 	return shmem_unused_huge_shrink(sbinfo, sc, 0);
@@ -848,21 +1056,21 @@ static long shmem_unused_huge_count(struct super_block *sb,
 {
 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 
-	/*
-	 * The per-superblock shrinklist is filesystem-global and does not
-	 * honour sc->memcg, so it is only meaningful on the global (kswapd or
-	 * root direct reclaim) shrink path. Skip the per-memcg iterations of
-	 * shrink_slab_memcg() to avoid queueing duplicate global work.
-	 */
-	if (!mem_cgroup_shrink_is_root(sc))
-		return 0;
-
-	return READ_ONCE(sbinfo->shrinklist_len);
+	return list_lru_shrink_count(&sbinfo->shrinklist, sc);
 }
 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
 
 #define shmem_huge SHMEM_HUGE_DENY
 
+static void shmem_unused_huge_add(struct inode *inode, struct folio *folio,
+				  gfp_t gfp)
+{
+}
+
+static void shmem_unused_huge_del(struct inode *inode)
+{
+}
+
 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
 		struct shrink_control *sc, unsigned long nr_to_free)
 {
@@ -976,6 +1184,7 @@ static long shmem_free_swap(struct address_space *mapping,
 			    pgoff_t index, pgoff_t end, void *radswap)
 {
 	XA_STATE(xas, &mapping->i_pages, index);
+	const softleaf_t swp = radix_to_swp_entry(radswap);
 	unsigned int nr_pages = 0;
 	pgoff_t base;
 	void *entry;
@@ -992,8 +1201,9 @@ static long shmem_free_swap(struct address_space *mapping,
 	}
 	xas_unlock_irq(&xas);
 
-	if (nr_pages)
-		swap_put_entries_direct(radix_to_swp_entry(radswap), nr_pages);
+	/* A swapin-error marker holds no swap slot, so just drop it. */
+	if (nr_pages && softleaf_is_swap(swp))
+		swap_put_entries_direct(swp, nr_pages);
 
 	return nr_pages;
 }
@@ -1159,7 +1369,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
 		}
 		folio_batch_remove_exceptionals(&fbatch);
 		folio_batch_release(&fbatch);
-		cond_resched();
+		cond_resched_tasks_rcu_qs();
 	}
 
 	/*
@@ -1176,11 +1386,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
 	if (folio) {
 		same_folio = lend < folio_next_pos(folio);
 		folio_mark_dirty(folio);
-		if (!truncate_inode_partial_folio(folio, lstart, lend)) {
-			start = folio_next_index(folio);
-			if (same_folio)
-				end = folio->index;
-		}
+		truncate_inode_partial_folio(folio, lstart, lend, &start,
+					     same_folio ? &end : NULL);
 		folio_unlock(folio);
 		folio_put(folio);
 		folio = NULL;
@@ -1190,8 +1397,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
 		folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
 	if (folio) {
 		folio_mark_dirty(folio);
-		if (!truncate_inode_partial_folio(folio, lstart, lend))
-			end = folio->index;
+		truncate_inode_partial_folio(folio, lstart, lend, NULL, &end);
 		folio_unlock(folio);
 		folio_put(folio);
 	}
@@ -1200,7 +1406,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
 
 	index = start;
 	while (index < end) {
-		cond_resched();
+		cond_resched_tasks_rcu_qs();
 
 		if (!find_get_entries(mapping, &index, end - 1, &fbatch,
 				indices)) {
@@ -1259,7 +1465,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend,
 
 				if (!folio_test_large(folio)) {
 					truncate_inode_folio(mapping, folio);
-				} else if (truncate_inode_partial_folio(folio, lstart, lend)) {
+				} else if (truncate_inode_partial_folio(folio,
+						lstart, lend, NULL, NULL)) {
 					/*
 					 * If we split a page, reset the loop so
 					 * that we pick up the new sub pages.
@@ -1292,7 +1499,7 @@ void shmem_truncate_range(struct inode *inode, loff_t lstart, uoff_t lend)
 }
 EXPORT_SYMBOL_GPL(shmem_truncate_range);
 
-static int shmem_getattr(struct mnt_idmap *idmap,
+static int shmem_getattr(const struct mnt_idmap *idmap,
 			 const struct path *path, struct kstat *stat,
 			 u32 request_mask, unsigned int query_flags)
 {
@@ -1326,7 +1533,7 @@ static int shmem_getattr(struct mnt_idmap *idmap,
 	return 0;
 }
 
-static int shmem_setattr(struct mnt_idmap *idmap,
+static int shmem_setattr(const struct mnt_idmap *idmap,
 			 struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_inode(dentry);
@@ -1418,14 +1625,7 @@ static void shmem_evict_inode(struct inode *inode)
 		inode->i_size = 0;
 		mapping_set_exiting(inode->i_mapping);
 		shmem_truncate_range(inode, 0, (loff_t)-1);
-		if (!list_empty(&info->shrinklist)) {
-			spin_lock(&sbinfo->shrinklist_lock);
-			if (!list_empty(&info->shrinklist)) {
-				list_del_init(&info->shrinklist);
-				sbinfo->shrinklist_len--;
-			}
-			spin_unlock(&sbinfo->shrinklist_lock);
-		}
+		shmem_unused_huge_del(inode);
 		while (!list_empty(&info->swaplist)) {
 			/* Wait while shmem_unuse() is scanning this inode... */
 			wait_var_event(&info->stop_eviction,
@@ -1612,7 +1812,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
 	struct shmem_inode_info *info = SHMEM_I(inode);
 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
 	pgoff_t index;
-	int nr_pages;
+	int nr_pages, ret;
 	bool split = false;
 
 	if ((info->flags & SHMEM_F_LOCKED) || sbinfo->noswap)
@@ -1693,7 +1893,8 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
 		folio_mark_uptodate(folio);
 	}
 
-	if (!folio_alloc_swap(folio)) {
+	ret = folio_alloc_swap(folio);
+	if (!ret) {
 		bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages);
 		int error;
 
@@ -1746,7 +1947,7 @@ int shmem_writeout(struct swap_io_ctx *ctx, struct folio *folio,
 		swap_cache_del_folio(folio);
 		goto redirty;
 	}
-	if (nr_pages > 1)
+	if (nr_pages > 1 && ret == -E2BIG)
 		goto try_split;
 redirty:
 	folio_mark_dirty(folio);
@@ -1847,8 +2048,11 @@ unsigned long shmem_allowable_huge_orders(struct inode *inode,
 
 	global_orders = shmem_huge_global_enabled(inode, index, write_end,
 						  shmem_huge_force, vma, vm_flags);
-	/* Tmpfs huge pages allocation */
-	if (!vma || !vma_is_anon_shmem(vma))
+	/*
+	 * Tmpfs huge pages allocation or forced collapse ignores
+	 * sysfs configs.
+	 */
+	if (!vma || !vma_is_anon_shmem(vma) || shmem_huge_force)
 		return global_orders;
 
 	/*
@@ -2277,7 +2481,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
 
 	si = get_swap_device(index_entry);
 	order = shmem_confirm_swap(mapping, index, index_entry);
-	if (unlikely(!si)) {
+	if (IS_ERR_OR_NULL(si)) {
 		if (order < 0)
 			return -EEXIST;
 		else
@@ -2536,27 +2740,6 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
 
 alloced:
 	alloced = true;
-	if (folio_test_large(folio) &&
-	    DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
-					folio_next_index(folio)) {
-		struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
-		struct shmem_inode_info *info = SHMEM_I(inode);
-		/*
-		 * Part of the large folio is beyond i_size: subject
-		 * to shrink under memory pressure.
-		 */
-		spin_lock(&sbinfo->shrinklist_lock);
-		/*
-		 * _careful to defend against unlocked access to
-		 * ->shrink_list in shmem_unused_huge_shrink()
-		 */
-		if (list_empty_careful(&info->shrinklist)) {
-			list_add_tail(&info->shrinklist,
-				      &sbinfo->shrinklist);
-			sbinfo->shrinklist_len++;
-		}
-		spin_unlock(&sbinfo->shrinklist_lock);
-	}
 
 	if (sgp == SGP_WRITE)
 		folio_set_referenced(folio);
@@ -2586,6 +2769,20 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
 		error = -EINVAL;
 		goto unlock;
 	}
+
+	/*
+	 * Queue the inode on the shrink list only after all checks that might
+	 * remove the folio have passed. Otherwise the inode could be left on
+	 * the shrinker list with a stale folio.
+	 */
+	if (alloced && folio_test_large(folio) &&
+	    DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < folio_next_index(folio)) {
+		/*
+		 * Part of the large folio is beyond i_size: subject
+		 * to shrink under memory pressure.
+		 */
+		shmem_unused_huge_add(inode, folio, gfp);
+	}
 out:
 	*foliop = folio;
 	return 0;
@@ -2786,7 +2983,7 @@ unsigned long shmem_get_unmapped_area(struct file *file,
 			sb = file_inode(file)->i_sb;
 		} else {
 			/*
-			 * Called directly from mm/mmap.c, or drivers/char/mem.c
+			 * Called directly from mm/mmap.c, or mm/char-mem.c
 			 * for "/dev/zero", to create a shared anonymous object.
 			 */
 			if (IS_ERR(shm_mnt))
@@ -3022,7 +3219,7 @@ static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode)
 	return &SHMEM_I(inode)->dir_offsets;
 }
 
-static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
+static struct inode *__shmem_get_inode(const struct mnt_idmap *idmap,
 				       struct super_block *sb,
 				       struct inode *dir, umode_t mode,
 				       dev_t dev, vma_flags_t flags)
@@ -3062,6 +3259,10 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
 	if (info->fsflags)
 		shmem_set_inode_flags(inode, info->fsflags, NULL);
 	INIT_LIST_HEAD(&info->shrinklist);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	info->shrinklist_objcg = NULL;
+	info->shrinklist_nid = -1;
+#endif
 	INIT_LIST_HEAD(&info->swaplist);
 	cache_no_acl(inode);
 	if (sbinfo->noswap)
@@ -3102,7 +3303,7 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
 }
 
 #ifdef CONFIG_TMPFS_QUOTA
-static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
+static struct inode *shmem_get_inode(const struct mnt_idmap *idmap,
 				     struct super_block *sb, struct inode *dir,
 				     umode_t mode, dev_t dev, vma_flags_t flags)
 {
@@ -3130,7 +3331,7 @@ static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
 	return ERR_PTR(err);
 }
 #else
-static struct inode *shmem_get_inode(struct mnt_idmap *idmap,
+static struct inode *shmem_get_inode(const struct mnt_idmap *idmap,
 				     struct super_block *sb, struct inode *dir,
 				     umode_t mode, dev_t dev, vma_flags_t flags)
 {
@@ -3818,7 +4019,7 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
  * File creation. Allocate an inode, and we're done..
  */
 static int
-shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
+shmem_mknod(const struct mnt_idmap *idmap, struct inode *dir,
 	    struct dentry *dentry, umode_t mode, dev_t dev)
 {
 	struct inode *inode;
@@ -3857,7 +4058,7 @@ shmem_mknod(struct mnt_idmap *idmap, struct inode *dir,
 }
 
 static int
-shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
+shmem_tmpfile(const struct mnt_idmap *idmap, struct inode *dir,
 	      struct file *file, umode_t mode)
 {
 	struct inode *inode;
@@ -3885,7 +4086,7 @@ shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
 	return error;
 }
 
-static struct dentry *shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *shmem_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	int error;
@@ -3897,7 +4098,7 @@ static struct dentry *shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	return NULL;
 }
 
-static int shmem_create(struct mnt_idmap *idmap, struct inode *dir,
+static int shmem_create(const struct mnt_idmap *idmap, struct inode *dir,
 			struct dentry *dentry, umode_t mode)
 {
 	return shmem_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
@@ -3970,7 +4171,7 @@ static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
 	return shmem_unlink(dir, dentry);
 }
 
-static int shmem_whiteout(struct mnt_idmap *idmap,
+static int shmem_whiteout(const struct mnt_idmap *idmap,
 			  struct inode *old_dir, struct dentry *old_dentry)
 {
 	struct dentry *whiteout;
@@ -3991,7 +4192,7 @@ static int shmem_whiteout(struct mnt_idmap *idmap,
  * it exists so that the VFS layer correctly free's it when it
  * gets overwritten.
  */
-static int shmem_rename2(struct mnt_idmap *idmap,
+static int shmem_rename2(const struct mnt_idmap *idmap,
 			 struct inode *old_dir, struct dentry *old_dentry,
 			 struct inode *new_dir, struct dentry *new_dentry,
 			 unsigned int flags)
@@ -4047,7 +4248,7 @@ static int shmem_rename2(struct mnt_idmap *idmap,
 	return 0;
 }
 
-static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir,
+static int shmem_symlink(const struct mnt_idmap *idmap, struct inode *dir,
 			 struct dentry *dentry, const char *symname)
 {
 	int error;
@@ -4159,7 +4360,7 @@ static int shmem_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 	return 0;
 }
 
-static int shmem_fileattr_set(struct mnt_idmap *idmap,
+static int shmem_fileattr_set(const struct mnt_idmap *idmap,
 			      struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
@@ -4264,7 +4465,7 @@ static int shmem_xattr_handler_get(const struct xattr_handler *handler,
 }
 
 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
-				   struct mnt_idmap *idmap,
+				   const struct mnt_idmap *idmap,
 				   struct dentry *unused, struct inode *inode,
 				   const char *name, const void *value,
 				   size_t size, int flags)
@@ -4527,6 +4728,7 @@ static int shmem_parse_opt_casefold(struct fs_context *fc, struct fs_parameter *
 	pr_info("tmpfs: Using encoding : utf8-%u.%u.%u\n",
 		unicode_major(version), unicode_minor(version), unicode_rev(version));
 
+	utf8_unload(ctx->encoding);
 	ctx->encoding = encoding;
 
 	return 0;
@@ -4936,6 +5138,9 @@ static void shmem_put_super(struct super_block *sb)
 #endif
 	free_percpu(sbinfo->ino_batch);
 	percpu_counter_destroy(&sbinfo->used_blocks);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	list_lru_destroy(&sbinfo->shrinklist);
+#endif
 	mpol_put(sbinfo->mpol);
 #ifdef CONFIG_TMPFS_XATTR
 	simple_xattr_cache_cleanup(&sbinfo->xa_cache);
@@ -4995,6 +5200,7 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
 
 	if (ctx->encoding) {
 		sb->s_encoding = ctx->encoding;
+		ctx->encoding = NULL;
 		set_default_d_op(sb, &shmem_ci_dentry_ops);
 		if (ctx->strict_encoding)
 			sb->s_encoding_flags = SB_ENC_STRICT_MODE_FL;
@@ -5029,8 +5235,11 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
 	raw_spin_lock_init(&sbinfo->stat_lock);
 	if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
 		goto failed;
-	spin_lock_init(&sbinfo->shrinklist_lock);
-	INIT_LIST_HEAD(&sbinfo->shrinklist);
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+	if (list_lru_init_memcg(&sbinfo->shrinklist, sb->s_shrink))
+		goto failed;
+#endif
 
 	sb->s_maxbytes = MAX_LFS_FILESIZE;
 	sb->s_blocksize = PAGE_SIZE;
@@ -5092,6 +5301,9 @@ static void shmem_free_fc(struct fs_context *fc)
 	struct shmem_options *ctx = fc->fs_private;
 
 	if (ctx) {
+#if IS_ENABLED(CONFIG_UNICODE)
+		utf8_unload(ctx->encoding);
+#endif
 		mpol_put(ctx->mpol);
 		kfree(ctx);
 	}
@@ -5155,7 +5367,7 @@ static void __init shmem_destroy_inodecache(void)
 static int shmem_error_remove_folio(struct address_space *mapping,
 				   struct folio *folio)
 {
-	return 0;
+	return MF_DELAYED;
 }
 
 static const struct address_space_operations shmem_aops = {
@@ -5792,7 +6004,7 @@ static inline void shmem_unacct_size(unsigned long flags, loff_t size)
 {
 }
 
-static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap,
+static inline struct inode *shmem_get_inode(const struct mnt_idmap *idmap,
 				struct super_block *sb, struct inode *dir,
 				umode_t mode, dev_t dev, vma_flags_t flags)
 {
diff --git a/mm/shmem_quota.c b/mm/shmem_quota.c
index d0b92d6da50f52..a3fa4bd93e753a 100644
--- a/mm/shmem_quota.c
+++ b/mm/shmem_quota.c
@@ -30,6 +30,7 @@
 #include <linux/slab.h>
 #include <linux/rbtree.h>
 #include <linux/shmem_fs.h>
+#include <linux/magic.h>
 
 #include <linux/quotaops.h>
 #include <linux/quota.h>
@@ -54,6 +55,10 @@ struct quota_id {
 
 static int shmem_check_quota_file(struct super_block *sb, int type)
 {
+	/* Verify enabling happens on tmpfs superblock */
+	if (sb->s_magic != TMPFS_MAGIC)
+		return 0;
+
 	/* There is no real quota file, nothing to do */
 	return 1;
 }
diff --git a/mm/slab.h b/mm/slab.h
index 8fd6835e42352b..e94ac63d10dc03 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -149,21 +149,6 @@ static_assert(sizeof(struct slab) <= sizeof(struct page));
 static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(struct freelist_counters)));
 #endif
 
-/**
- * slab_folio - The folio allocated for a slab
- * @s: The slab.
- *
- * Slabs are allocated as folios that contain the individual objects and are
- * using some fields in the first struct page of the folio - those fields are
- * now accessed by struct slab. It is occasionally necessary to convert back to
- * a folio in order to communicate with the rest of the mm.  Please use this
- * helper function instead of casting yourself, as the implementation may change
- * in the future.
- */
-#define slab_folio(s)		(_Generic((s),				\
-	const struct slab *:	(const struct folio *)s,		\
-	struct slab *:		(struct folio *)s))
-
 /**
  * page_slab - Converts from struct page to its slab.
  * @page: A page which may or may not belong to a slab.
@@ -184,15 +169,16 @@ static inline struct slab *page_slab(const struct page *page)
  * slab_page - The first struct page allocated for a slab
  * @s: The slab.
  *
- * A convenience wrapper for converting slab to the first struct page of the
- * underlying folio, to communicate with code not yet converted to folio or
- * struct slab.
+ * A convenience wrapper for returning the first page allocated to this slab.
+ * This is for the benefit of code not yet converted to struct slab.
  */
-#define slab_page(s) folio_page(slab_folio(s), 0)
+#define slab_page(s) 		(_Generic((s),				\
+	const struct slab *:	(const struct page *)s,			\
+	struct slab *:		(struct page *)s))
 
 static inline void *slab_address(const struct slab *slab)
 {
-	return folio_address(slab_folio(slab));
+	return page_address(slab_page(slab));
 }
 
 static inline int slab_nid(const struct slab *slab)
@@ -212,7 +198,7 @@ static inline struct slab *virt_to_slab(const void *addr)
 
 static inline int slab_order(const struct slab *slab)
 {
-	return folio_order(slab_folio(slab));
+	return compound_order(slab_page(slab));
 }
 
 static inline size_t slab_size(const struct slab *slab)
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 7223a7596dabbd..28bf035a58f326 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -102,7 +102,7 @@ static bool kmem_cache_is_duplicate_name(const char *name)
 
 static int kmem_cache_sanity_check(const char *name, unsigned int size)
 {
-	if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
+	if (!name || in_interrupt() || !size || size > KMALLOC_MAX_SIZE) {
 		pr_err("kmem_cache_create(%s) integrity check failed\n", name);
 		return -EINVAL;
 	}
diff --git a/mm/slub.c b/mm/slub.c
index 54ec1250335711..0315a42691766b 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -360,33 +360,64 @@ enum add_mode {
 	ADD_TO_TAIL,
 };
 
+#define STAT_ITEMS							\
+	/* Allocation from percpu sheaves */				\
+	ITEM_EXP(ALLOC_FASTPATH, alloc_fastpath)			\
+	/* Allocation from partial or new slab */			\
+	ITEM_EXP(ALLOC_SLOWPATH, alloc_slowpath)			\
+	/* Free to rcu_free sheaf */					\
+	ITEM_EXP(FREE_RCU_SHEAF, free_rcu_sheaf)			\
+	/* Failed to free to a rcu_free sheaf */			\
+	ITEM_EXP(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail)		\
+	/* Free to percpu sheaves */					\
+	ITEM_EXP(FREE_FASTPATH, free_fastpath)				\
+	/* Free to a slab */						\
+	ITEM_EXP(FREE_SLOWPATH, free_slowpath)				\
+	/* Freeing moves slab to partial list */			\
+	ITEM_EXP(FREE_ADD_PARTIAL, free_add_partial)			\
+	/* Freeing removes last object */				\
+	ITEM_EXP(FREE_REMOVE_PARTIAL, free_remove_partial)		\
+	/* New slab acquired from page allocator */			\
+	ITEM_EXP(ALLOC_SLAB, alloc_slab)				\
+	/* Requested node different from cpu sheaf */			\
+	ITEM_EXP(ALLOC_NODE_MISMATCH, alloc_node_mismatch)		\
+	/* Slab freed to the page allocator */				\
+	ITEM_EXP(FREE_SLAB, free_slab)					\
+	/* Number of times fallback was necessary */			\
+	ITEM_EXP(ORDER_FALLBACK, order_fallback)			\
+	/* Failures of slab freelist update */				\
+	ITEM_EXP(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail)		\
+	/* Objects flushed from a sheaf */				\
+	ITEM_EXP(SHEAF_FLUSH, sheaf_flush)				\
+	/* Objects refilled to a sheaf */				\
+	ITEM_EXP(SHEAF_REFILL, sheaf_refill)				\
+	/* Allocation of an empty sheaf including oversized ones */	\
+	ITEM_EXP(SHEAF_ALLOC, sheaf_alloc)				\
+	/* Freeing of an empty sheaf including oversized ones */	\
+	ITEM_EXP(SHEAF_FREE, sheaf_free)				\
+	/* Got full sheaf from barn */					\
+	ITEM_EXP(BARN_GET, barn_get)					\
+	/* Failed to get full sheaf from barn */			\
+	ITEM_EXP(BARN_GET_FAIL, barn_get_fail)				\
+	/* Put full sheaf to barn */					\
+	ITEM_EXP(BARN_PUT, barn_put)					\
+	/* Failed to put full sheaf to barn */				\
+	ITEM_EXP(BARN_PUT_FAIL, barn_put_fail)				\
+	/* Sheaf prefill grabbed the spare sheaf */			\
+	ITEM_EXP(SHEAF_PREFILL_FAST, sheaf_prefill_fast)		\
+	/* Sheaf prefill found no spare sheaf */			\
+	ITEM_EXP(SHEAF_PREFILL_SLOW, sheaf_prefill_slow)		\
+	/* Allocation of oversize sheaf for prefill */			\
+	ITEM_EXP(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize)	\
+	/* Sheaf return reattached spare sheaf */			\
+	ITEM_EXP(SHEAF_RETURN_FAST, sheaf_return_fast)			\
+	/* Sheaf return could not reattach spare */			\
+	ITEM_EXP(SHEAF_RETURN_SLOW, sheaf_return_slow)
+
 enum stat_item {
-	ALLOC_FASTPATH,		/* Allocation from percpu sheaves */
-	ALLOC_SLOWPATH,		/* Allocation from partial or new slab */
-	FREE_RCU_SHEAF,		/* Free to rcu_free sheaf */
-	FREE_RCU_SHEAF_FAIL,	/* Failed to free to a rcu_free sheaf */
-	FREE_FASTPATH,		/* Free to percpu sheaves */
-	FREE_SLOWPATH,		/* Free to a slab */
-	FREE_ADD_PARTIAL,	/* Freeing moves slab to partial list */
-	FREE_REMOVE_PARTIAL,	/* Freeing removes last object */
-	ALLOC_SLAB,		/* New slab acquired from page allocator */
-	ALLOC_NODE_MISMATCH,	/* Requested node different from cpu sheaf */
-	FREE_SLAB,		/* Slab freed to the page allocator */
-	ORDER_FALLBACK,		/* Number of times fallback was necessary */
-	CMPXCHG_DOUBLE_FAIL,	/* Failures of slab freelist update */
-	SHEAF_FLUSH,		/* Objects flushed from a sheaf */
-	SHEAF_REFILL,		/* Objects refilled to a sheaf */
-	SHEAF_ALLOC,		/* Allocation of an empty sheaf including oversized ones */
-	SHEAF_FREE,		/* Freeing of an empty sheaf including oversized ones */
-	BARN_GET,		/* Got full sheaf from barn */
-	BARN_GET_FAIL,		/* Failed to get full sheaf from barn */
-	BARN_PUT,		/* Put full sheaf to barn */
-	BARN_PUT_FAIL,		/* Failed to put full sheaf to barn */
-	SHEAF_PREFILL_FAST,	/* Sheaf prefill grabbed the spare sheaf */
-	SHEAF_PREFILL_SLOW,	/* Sheaf prefill found no spare sheaf */
-	SHEAF_PREFILL_OVERSIZE,	/* Allocation of oversize sheaf for prefill */
-	SHEAF_RETURN_FAST,	/* Sheaf return reattached spare sheaf */
-	SHEAF_RETURN_SLOW,	/* Sheaf return could not reattach spare */
+#define ITEM_EXP(name, unused) name,
+	STAT_ITEMS
+#undef ITEM_EXP
 	NR_SLUB_STAT_ITEMS
 };
 
@@ -422,6 +453,7 @@ struct node_barn {
 	spinlock_t lock;
 	struct list_head sheaves_full;
 	struct list_head sheaves_empty;
+	struct slab_sheaf *sheaf_partial;
 	unsigned int nr_full;
 	unsigned int nr_empty;
 };
@@ -3298,6 +3330,7 @@ static void barn_init(struct node_barn *barn)
 	spin_lock_init(&barn->lock);
 	INIT_LIST_HEAD(&barn->sheaves_full);
 	INIT_LIST_HEAD(&barn->sheaves_empty);
+	barn->sheaf_partial = NULL;
 	barn->nr_full = 0;
 	barn->nr_empty = 0;
 }
@@ -3315,6 +3348,10 @@ static void barn_shrink(struct kmem_cache *s, struct node_barn *barn)
 	barn->nr_full = 0;
 	list_splice_init(&barn->sheaves_empty, &empty_list);
 	barn->nr_empty = 0;
+	if (barn->sheaf_partial) {
+		list_add(&barn->sheaf_partial->barn_list, &full_list);
+		barn->sheaf_partial = NULL;
+	}
 
 	spin_unlock_irqrestore(&barn->lock, flags);
 
@@ -5073,12 +5110,87 @@ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int nod
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
 
+/*
+ * Refill @sheaf from the barn: from its partial sheaf first, then from its full
+ * sheaves. A sheaf that objects were copied from stays in the barn as the
+ * partial sheaf if it still holds objects, or goes on the empty list if it is
+ * empty.
+ *
+ * Returns true if the sheaf is now full, at s->sheaf_capacity.
+ * Returns false if the sheaf is still not full.
+ */
+static bool refill_sheaf_from_barn(struct kmem_cache *s,
+				   struct slab_sheaf *sheaf)
+{
+	struct node_barn *barn = get_barn(s);
+	struct slab_sheaf *src;
+	unsigned int to_move;
+	unsigned long flags;
+
+	if (!barn)
+		return false;
+
+	if (!data_race(barn->nr_full) && !data_race(barn->sheaf_partial))
+		return false;
+
+	spin_lock_irqsave(&barn->lock, flags);
+
+	/*
+	 * The partial sheaf can hold fewer objects than @sheaf needs to
+	 * reach capacity, and a sheaf on the full list is not necessarily
+	 * full (see the comment in rcu_free_sheaf()), so keep taking from
+	 * the barn until @sheaf is full or nothing is left.
+	 */
+	while (sheaf->size < s->sheaf_capacity) {
+		src = barn->sheaf_partial;
+		barn->sheaf_partial = NULL;
+		if (!src) {
+			if (!barn->nr_full)
+				break;
+			src = list_first_entry(&barn->sheaves_full,
+					       struct slab_sheaf, barn_list);
+			list_del(&src->barn_list);
+			barn->nr_full--;
+		}
+
+		to_move = min(s->sheaf_capacity - sheaf->size, src->size);
+		src->size -= to_move;
+		memcpy(&sheaf->objects[sheaf->size], &src->objects[src->size],
+		       to_move * sizeof(void *));
+		sheaf->size += to_move;
+
+		if (src->size) {
+			barn->sheaf_partial = src;
+		} else {
+			/*
+			 * No empty-limit check: the sheaf put on the empty list
+			 * was already in the barn, so the barn holds no more
+			 * sheaves than before. barn_replace_empty_sheaf() skips
+			 * the check for the same reason.
+			 */
+			list_add(&src->barn_list, &barn->sheaves_empty);
+			barn->nr_empty++;
+		}
+	}
+
+	spin_unlock_irqrestore(&barn->lock, flags);
+
+	if (sheaf->size < s->sheaf_capacity)
+		return false;
+
+	stat(s, BARN_GET);
+	return true;
+}
+
 static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
 				      struct slab_sheaf *sheaf, gfp_t gfp)
 {
 	gfp_t gfp_nomemalloc;
 	int ret;
 
+	if (refill_sheaf_from_barn(s, sheaf))
+		return 0;
+
 	gfp_nomemalloc = gfp | __GFP_NOMEMALLOC;
 	if (gfp_pfmemalloc_allowed(gfp))
 		gfp_nomemalloc |= __GFP_NOWARN;
@@ -5342,7 +5454,12 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
 	struct page *page;
 	void *ptr = NULL;
-	unsigned int order = get_order(size);
+	unsigned int order;
+
+	if (WARN_ON_ONCE_GFP(size > KMALLOC_MAX_SIZE, flags))
+		return NULL;
+
+	order = get_order(size);
 
 	if (unlikely(flags & GFP_SLAB_BUG_MASK))
 		flags = kmalloc_fix_flags(flags);
@@ -5751,76 +5868,80 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
 		new.inuse -= cnt;
 
 		/*
-		 * Might need to be taken off (due to becoming empty) or added
-		 * to (due to not being full anymore) the partial list.
-		 * Unless it's frozen.
+		 * partial->partial: if the slab was on the node partial list,
+		 * it stays there, and if it was off, it stays off, so we need
+		 * no list handling and no list_lock.
+		 *
+		 * Note that "continue;" in a do-while goes on to evaluate the
+		 * condition below, so we do perform the freelist update.
 		 */
-		if (!new.inuse || was_full) {
-
-			n = get_node(s, slab_nid(slab));
-			/*
-			 * Speculatively acquire the list_lock.
-			 * If the cmpxchg does not succeed then we may
-			 * drop the list_lock without any processing.
-			 *
-			 * Otherwise the list_lock will synchronize with
-			 * other processors updating the list of slabs.
-			 */
-			spin_lock_irqsave(&n->list_lock, flags);
+		if (!was_full && new.inuse)
+			continue;
 
-			on_node_partial = slab_test_node_partial(slab);
-		}
+		/*
+		 * The slab might need to be taken off (due to becoming empty)
+		 * or added to (due to not being full anymore) the partial
+		 * list.
+		 *
+		 * Speculatively acquire list_lock prior to cmpxchg(), as
+		 * performing cmpxchg() before lock acquisition races with
+		 * concurrent partial list operations.
+		 *
+		 * If the cmpxchg does not succeed then we will retry.
+		 */
+		n = get_node(s, slab_nid(slab));
+		spin_lock_irqsave(&n->list_lock, flags);
 
 	} while (!slab_update_freelist(s, slab, &old, &new, "__slab_free"));
 
-	if (likely(!n)) {
+	/* partial->partial: we didn't take the list_lock. */
+	if (likely(!n))
+		return;
+
+	on_node_partial = slab_test_node_partial(slab);
+
+	if (!was_full && !on_node_partial) {
 		/*
-		 * We didn't take the list_lock because the slab was already on
-		 * the partial list and will remain there.
+		 * partial->empty, offlist: a bulk refill has taken the slab
+		 * off the partial list and will put it back, so its list
+		 * handling is not ours to do.
 		 */
+		spin_unlock_irqrestore(&n->list_lock, flags);
 		return;
 	}
 
 	/*
-	 * This slab was partially empty but not on the per-node partial list,
-	 * in which case we shouldn't manipulate its list, just return.
+	 * full/partial->empty, exceed: we have enough partial slabs already.
 	 */
-	if (!was_full && !on_node_partial) {
+	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
+		/* partial->empty, onlist, exceed */
+		if (likely(!was_full)) {
+			remove_partial(n, slab);
+			stat(s, FREE_REMOVE_PARTIAL);
+		}
+		/* else, full->empty, exceed: it is on no list to remove from */
+
 		spin_unlock_irqrestore(&n->list_lock, flags);
+		stat(s, FREE_SLAB);
+		discard_slab(s, slab);
 		return;
 	}
 
 	/*
-	 * If slab became empty, should we add/keep it on the partial list or we
-	 * have enough?
+	 * At this point, only three cases remain:
+	 *   full->partial
+	 *   full->empty, not exceed
+	 *   partial->empty, onlist, not exceed
 	 */
-	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
-		goto slab_empty;
 
-	/*
-	 * Objects left in the slab. If it was not on the partial list before
-	 * then add it.
-	 */
+	/* full->partial; full->empty, not exceed */
 	if (unlikely(was_full)) {
 		add_partial(n, slab, ADD_TO_TAIL);
 		stat(s, FREE_ADD_PARTIAL);
 	}
-	spin_unlock_irqrestore(&n->list_lock, flags);
-	return;
-
-slab_empty:
-	/*
-	 * The slab could have a single object and thus go from full to empty in
-	 * a single free, but more likely it was on the partial list. Remove it.
-	 */
-	if (likely(!was_full)) {
-		remove_partial(n, slab);
-		stat(s, FREE_REMOVE_PARTIAL);
-	}
+	/* else, partial->empty, onlist, not exceed: it stays on partial list */
 
 	spin_unlock_irqrestore(&n->list_lock, flags);
-	stat(s, FREE_SLAB);
-	discard_slab(s, slab);
 }
 
 /*
@@ -5969,7 +6090,8 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
 	if (!allow_spin)
 		return NULL;
 
-	empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT);
+	/* Don't wake up kswapd, it will cause deadlock under pi_lock */
+	empty = alloc_empty_sheaf(s, __GFP_NOWARN, SLAB_ALLOC_DEFAULT);
 	if (empty)
 		goto got_empty;
 
@@ -6128,7 +6250,6 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
 		struct slab_sheaf *empty;
 		struct node_barn *barn;
 		unsigned int alloc_flags = to_alloc_flags(free_flags);
-		gfp_t gfp = allow_spin ? GFP_NOWAIT : __GFP_NOWARN;
 
 		/* Bootstrap or debug cache, fall back */
 		if (unlikely(!cache_has_sheaves(s))) {
@@ -6157,7 +6278,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
 
 		local_unlock(&s->cpu_sheaves->lock);
 
-		empty = alloc_empty_sheaf(s, gfp, alloc_flags);
+		/* Don't wake up kswapd, it will cause deadlock under pi_lock */
+		empty = alloc_empty_sheaf(s, __GFP_NOWARN, alloc_flags);
 
 		if (!empty)
 			goto fail;
@@ -9514,34 +9636,12 @@ static ssize_t text##_store(struct kmem_cache *s,		\
 	clear_stat(s, si);					\
 	return length;						\
 }								\
-SLAB_ATTR(text);						\
-
-STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
-STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
-STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf);
-STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail);
-STAT_ATTR(FREE_FASTPATH, free_fastpath);
-STAT_ATTR(FREE_SLOWPATH, free_slowpath);
-STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
-STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
-STAT_ATTR(ALLOC_SLAB, alloc_slab);
-STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
-STAT_ATTR(FREE_SLAB, free_slab);
-STAT_ATTR(ORDER_FALLBACK, order_fallback);
-STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
-STAT_ATTR(SHEAF_FLUSH, sheaf_flush);
-STAT_ATTR(SHEAF_REFILL, sheaf_refill);
-STAT_ATTR(SHEAF_ALLOC, sheaf_alloc);
-STAT_ATTR(SHEAF_FREE, sheaf_free);
-STAT_ATTR(BARN_GET, barn_get);
-STAT_ATTR(BARN_GET_FAIL, barn_get_fail);
-STAT_ATTR(BARN_PUT, barn_put);
-STAT_ATTR(BARN_PUT_FAIL, barn_put_fail);
-STAT_ATTR(SHEAF_PREFILL_FAST, sheaf_prefill_fast);
-STAT_ATTR(SHEAF_PREFILL_SLOW, sheaf_prefill_slow);
-STAT_ATTR(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize);
-STAT_ATTR(SHEAF_RETURN_FAST, sheaf_return_fast);
-STAT_ATTR(SHEAF_RETURN_SLOW, sheaf_return_slow);
+SLAB_ATTR(text);
+
+#define ITEM_EXP(name, text) STAT_ATTR(name, text)
+STAT_ITEMS
+#undef ITEM_EXP
+
 #endif	/* CONFIG_SLUB_STATS */
 
 #ifdef CONFIG_KFENCE
@@ -9604,32 +9704,9 @@ static const struct attribute *const slab_attrs[] = {
 	&remote_node_defrag_ratio_attr.attr,
 #endif
 #ifdef CONFIG_SLUB_STATS
-	&alloc_fastpath_attr.attr,
-	&alloc_slowpath_attr.attr,
-	&free_rcu_sheaf_attr.attr,
-	&free_rcu_sheaf_fail_attr.attr,
-	&free_fastpath_attr.attr,
-	&free_slowpath_attr.attr,
-	&free_add_partial_attr.attr,
-	&free_remove_partial_attr.attr,
-	&alloc_slab_attr.attr,
-	&alloc_node_mismatch_attr.attr,
-	&free_slab_attr.attr,
-	&order_fallback_attr.attr,
-	&cmpxchg_double_fail_attr.attr,
-	&sheaf_flush_attr.attr,
-	&sheaf_refill_attr.attr,
-	&sheaf_alloc_attr.attr,
-	&sheaf_free_attr.attr,
-	&barn_get_attr.attr,
-	&barn_get_fail_attr.attr,
-	&barn_put_attr.attr,
-	&barn_put_fail_attr.attr,
-	&sheaf_prefill_fast_attr.attr,
-	&sheaf_prefill_slow_attr.attr,
-	&sheaf_prefill_oversize_attr.attr,
-	&sheaf_return_fast_attr.attr,
-	&sheaf_return_slow_attr.attr,
+#define ITEM_EXP(unused, text) &text##_attr.attr,
+	STAT_ITEMS
+#undef ITEM_EXP
 #endif
 #ifdef CONFIG_FAILSLAB
 	&failslab_attr.attr,
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 5a2469fb1838c7..111efc80246579 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -32,39 +32,23 @@
 #include <asm/dma.h>
 #include <asm/tlbflush.h>
 
-#include "hugetlb_vmemmap.h"
-
 /*
  * Flags for vmemmap_populate_range and friends.
  */
-/* Get a ref on the head page struct page, for ZONE_DEVICE compound pages */
-#define VMEMMAP_POPULATE_PAGEREF	0x0001
+/* Vmemmap population for ZONE_DEVICE compound pages */
+#define VMEMMAP_POPULATE_DAX		0x0001
 
 #include "internal.h"
 #include "mm_init.h"
 #include "sparse.h"
 
-/*
- * Allocate a block of memory to be used to back the virtual memory map
- * or to back the page tables that are used to create the mapping.
- * Uses the main allocators if they are available, else bootmem.
- */
-
-static void * __ref __earlyonly_bootmem_alloc(int node,
-				unsigned long size,
-				unsigned long align,
-				unsigned long goal)
-{
-	return memmap_alloc(size, align, goal, node, false);
-}
-
-void * __meminit vmemmap_alloc_block(unsigned long size, int node)
+void __ref *vmemmap_alloc_block(unsigned long size, int node)
 {
 	/* If the main allocator is up use that, fallback to bootmem. */
 	if (slab_is_available()) {
 		gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
 		int order = get_order(size);
-		static bool warned __meminitdata;
+		static bool warned;
 		struct page *page;
 
 		page = alloc_pages_node(node, gfp_mask, order);
@@ -78,8 +62,7 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node)
 		}
 		return NULL;
 	} else
-		return __earlyonly_bootmem_alloc(node, size, size,
-				__pa(MAX_DMA_ADDRESS));
+		return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), node, false);
 }
 
 static void * __meminit altmap_alloc_block_buf(unsigned long size,
@@ -148,17 +131,146 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
 			start, end - 1);
 }
 
+int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages)
+{
+	const struct mem_section *ms = __pfn_to_section(pfn);
+	const int order = section_compound_order(ms);
+	const unsigned long pages_per_compound = 1UL << order;
+
+	VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION));
+	VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION);
+
+	if (!section_vmemmap_optimizable(ms))
+		return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
+
+	if (order < PFN_SECTION_SHIFT) {
+		VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
+		return VMEMMAP_OPTIMIZATION_PAGES * nr_pages / pages_per_compound;
+	}
+
+	VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
+
+	if (IS_ALIGNED(pfn, pages_per_compound))
+		return VMEMMAP_OPTIMIZATION_PAGES;
+
+	return 0;
+}
+
+static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
+{
+	void *p = vmemmap_alloc_block(size, node);
+
+	if (!p)
+		return NULL;
+	memset(p, 0, size);
+
+	return p;
+}
+
+#ifdef CONFIG_VMEMMAP_OPTIMIZATION
+#define VMEMMAP_OPTIMIZATION_NR_ORDERS	(MAX_FOLIO_ORDER - VMEMMAP_OPTIMIZATION_MIN_ORDER + 1)
+
+static __ref struct page **vmemmap_tails_alloc(struct zone *zone)
+{
+	struct page **pages;
+	const size_t size = array_size(VMEMMAP_OPTIMIZATION_NR_ORDERS, sizeof(*pages));
+
+	pages = slab_is_available() ? kzalloc_objs(*pages, VMEMMAP_OPTIMIZATION_NR_ORDERS) :
+		memblock_alloc(size, __alignof__(*pages));
+	if (!pages)
+		return NULL;
+
+	if (cmpxchg(&zone->vmemmap_tails, NULL, pages) != NULL) {
+		if (slab_is_available())
+			kfree(pages);
+		else
+			memblock_free(pages, size);
+		pages = READ_ONCE(zone->vmemmap_tails);
+	}
+
+	return pages;
+}
+
+struct page __ref *vmemmap_shared_tail_page(unsigned int order, struct zone *zone)
+{
+	void *addr;
+	struct page *page, **pages;
+	const unsigned int idx = order - VMEMMAP_OPTIMIZATION_MIN_ORDER;
+
+	if (WARN_ON_ONCE(idx >= VMEMMAP_OPTIMIZATION_NR_ORDERS))
+		return NULL;
+
+	pages = READ_ONCE(zone->vmemmap_tails) ? : vmemmap_tails_alloc(zone);
+	if (!pages)
+		return NULL;
+
+	page = READ_ONCE(pages[idx]);
+	if (likely(page))
+		return page;
+
+	addr = vmemmap_alloc_block(PAGE_SIZE, zone_to_nid(zone));
+	if (!addr)
+		return NULL;
+
+	for (int i = 0; i < PAGE_SIZE / sizeof(struct page); i++) {
+		page = (struct page *)addr + i;
+		mm_zero_struct_page(page);
+		atomic_set(&page->_mapcount, -1);
+		set_page_node(page, zone_to_nid(zone));
+		set_page_zone(page, zone_idx(zone));
+		prep_compound_tail(page, NULL, order);
+		if (zone_is_zone_device(zone))
+			__SetPageReserved(page);
+	}
+
+	page = virt_to_page(addr);
+	if (cmpxchg(&pages[idx], NULL, page) != NULL) {
+		if (slab_is_available())
+			__free_page(page);
+		else
+			memblock_free(addr, PAGE_SIZE);
+		page = READ_ONCE(pages[idx]);
+	}
+
+	return page;
+}
+#endif
+
+static __meminit void *vmemmap_alloc_pte(unsigned long pfn, int node,
+		struct vmem_altmap *altmap, unsigned long flags)
+{
+	struct zone *zone;
+	struct page *page;
+	const unsigned int order = pfn_to_section_compound_order(pfn);
+
+	/*
+	 * Device DAX still relies on vmemmap_populate_compound_pages() for
+	 * head/first-tail allocation and tail-page reuse.
+	 */
+	if (!vmemmap_optimizable_pfn(pfn) || flags & VMEMMAP_POPULATE_DAX)
+		return vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
+
+	zone = pfn_to_zone(pfn, node);
+	page = vmemmap_shared_tail_page(order, zone);
+	if (!page)
+		return NULL;
+
+	return page_address(page);
+}
+
 static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
 				       struct vmem_altmap *altmap,
 				       unsigned long ptpfn, unsigned long flags)
 {
 	pte_t *pte = pte_offset_kernel(pmd, addr);
+	unsigned long pfn = page_to_pfn((struct page *)addr);
+
 	if (pte_none(ptep_get(pte))) {
 		pte_t entry;
-		void *p;
 
 		if (ptpfn == (unsigned long)-1) {
-			p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
+			void *p = vmemmap_alloc_pte(pfn, node, altmap, flags);
+
 			if (!p)
 				return NULL;
 			ptpfn = PHYS_PFN(__pa(p));
@@ -166,30 +278,24 @@ static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, in
 			/*
 			 * When a PTE/PMD entry is freed from the init_mm
 			 * there's a free_pages() call to this page allocated
-			 * above. Thus this get_page() is paired with the
+			 * above. Thus this try_get_page() is paired with the
 			 * put_page_testzero() on the freeing path.
 			 * This can only called by certain ZONE_DEVICE path,
 			 * and through vmemmap_populate_compound_pages() when
 			 * slab is available.
+			 *
+			 * Use try_get_page() to prevent the shared page refcount
+			 * from overflowing.
 			 */
-			if (flags & VMEMMAP_POPULATE_PAGEREF)
-				get_page(pfn_to_page(ptpfn));
+			if ((flags & VMEMMAP_POPULATE_DAX) &&
+			    !try_get_page(pfn_to_page(ptpfn)))
+				return NULL;
 		}
 		entry = pfn_pte(ptpfn, PAGE_KERNEL);
 		set_pte_at(&init_mm, addr, pte, entry);
-	}
-	return pte;
-}
-
-static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
-{
-	void *p = vmemmap_alloc_block(size, node);
-
-	if (!p)
+	} else if (WARN_ON_ONCE(vmemmap_optimizable_pfn(pfn)))
 		return NULL;
-	memset(p, 0, size);
-
-	return p;
+	return pte;
 }
 
 static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
@@ -322,69 +428,6 @@ void vmemmap_wrprotect_hvo(unsigned long addr, unsigned long end,
 	}
 }
 
-#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
-static __meminit struct page *vmemmap_get_tail(unsigned int order, struct zone *zone)
-{
-	struct page *p, *tail;
-	unsigned int idx;
-	int node = zone_to_nid(zone);
-
-	if (WARN_ON_ONCE(order < VMEMMAP_TAIL_MIN_ORDER))
-		return NULL;
-	if (WARN_ON_ONCE(order > MAX_FOLIO_ORDER))
-		return NULL;
-
-	idx = order - VMEMMAP_TAIL_MIN_ORDER;
-	tail = zone->vmemmap_tails[idx];
-	if (tail)
-		return tail;
-
-	/*
-	 * Only allocate the page, but do not initialize it.
-	 *
-	 * Any initialization done here will be overwritten by memmap_init().
-	 *
-	 * hugetlb_bootmem_struct_page_init() will take care of initialization
-	 * after memmap_init().
-	 */
-
-	p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
-	if (!p)
-		return NULL;
-
-	tail = virt_to_page(p);
-	zone->vmemmap_tails[idx] = tail;
-
-	return tail;
-}
-
-int __meminit vmemmap_populate_hvo(unsigned long addr, unsigned long end,
-				       unsigned int order, struct zone *zone,
-				       unsigned long headsize)
-{
-	unsigned long maddr;
-	struct page *tail;
-	pte_t *pte;
-	int node = zone_to_nid(zone);
-
-	tail = vmemmap_get_tail(order, zone);
-	if (!tail)
-		return -ENOMEM;
-
-	for (maddr = addr; maddr < addr + headsize; maddr += PAGE_SIZE) {
-		pte = vmemmap_populate_address(maddr, node, NULL, -1, 0);
-		if (!pte)
-			return -ENOMEM;
-	}
-
-	/*
-	 * Reuse the last page struct page mapped above for the rest.
-	 */
-	return vmemmap_populate_range(maddr, end, node, NULL,
-				      page_to_pfn(tail), 0);
-}
-#endif
-
 void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
 				      unsigned long addr, unsigned long next)
 {
@@ -412,6 +455,9 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
 	pmd_t *pmd;
 
 	for (addr = start; addr < end; addr = next) {
+		unsigned long pfn = page_to_pfn((struct page *)addr);
+		const struct mem_section *ms = __pfn_to_section(pfn);
+
 		next = pmd_addr_end(addr, end);
 
 		pgd = vmemmap_pgd_populate(addr, node);
@@ -427,7 +473,7 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
 			return -ENOMEM;
 
 		pmd = pmd_offset(pud, addr);
-		if (pmd_none(pmdp_get(pmd))) {
+		if (pmd_none(pmdp_get(pmd)) && !section_vmemmap_optimizable(ms)) {
 			void *p;
 
 			p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
@@ -445,8 +491,11 @@ int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
 				 */
 				return -ENOMEM;
 			}
-		} else if (vmemmap_check_pmd(pmd, node, addr, next))
+		} else if (vmemmap_check_pmd(pmd, node, addr, next)) {
+			if (WARN_ON_ONCE(section_vmemmap_optimizable(ms)))
+				return -EOPNOTSUPP;
 			continue;
+		}
 		if (vmemmap_populate_basepages(addr, next, node, altmap))
 			return -ENOMEM;
 	}
@@ -474,23 +523,6 @@ static bool __meminit reuse_compound_section(unsigned long start_pfn,
 	return !IS_ALIGNED(offset, nr_pages) && nr_pages > PAGES_PER_SUBSECTION;
 }
 
-static pte_t * __meminit compound_section_tail_page(unsigned long addr)
-{
-	pte_t *pte;
-
-	addr -= PAGE_SIZE;
-
-	/*
-	 * Assuming sections are populated sequentially, the previous section's
-	 * page data can be reused.
-	 */
-	pte = pte_offset_kernel(pmd_off_k(addr), addr);
-	if (!pte)
-		return NULL;
-
-	return pte;
-}
-
 static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 						     unsigned long start,
 						     unsigned long end, int node,
@@ -499,44 +531,34 @@ static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
 	unsigned long size, addr;
 	pte_t *pte;
 	int rc;
+	unsigned long flags = VMEMMAP_POPULATE_DAX;
+	struct page *page;
+	unsigned int order = pfn_to_section_compound_order(start_pfn);
 
-	if (reuse_compound_section(start_pfn, pgmap)) {
-		pte = compound_section_tail_page(start);
-		if (!pte)
-			return -ENOMEM;
+	page = vmemmap_shared_tail_page(order, device_zone(node));
+	if (!page)
+		return -ENOMEM;
 
-		/*
-		 * Reuse the page that was populated in the prior iteration
-		 * with just tail struct pages.
-		 */
+	if (reuse_compound_section(start_pfn, pgmap))
 		return vmemmap_populate_range(start, end, node, NULL,
-					      pte_pfn(ptep_get(pte)),
-					      VMEMMAP_POPULATE_PAGEREF);
-	}
+					      page_to_pfn(page), flags);
 
-	size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page));
+	size = min(end - start, (1UL << order) * sizeof(struct page));
 	for (addr = start; addr < end; addr += size) {
 		unsigned long next, last = addr + size;
 
 		/* Populate the head page vmemmap page */
-		pte = vmemmap_populate_address(addr, node, NULL, -1, 0);
-		if (!pte)
-			return -ENOMEM;
-
-		/* Populate the tail pages vmemmap page */
-		next = addr + PAGE_SIZE;
-		pte = vmemmap_populate_address(next, node, NULL, -1, 0);
+		pte = vmemmap_populate_address(addr, node, NULL, -1, flags);
 		if (!pte)
 			return -ENOMEM;
 
 		/*
-		 * Reuse the previous page for the rest of tail pages
+		 * Reuse the shared page for the rest of tail pages
 		 * See layout diagram in Documentation/mm/vmemmap_dedup.rst
 		 */
-		next += PAGE_SIZE;
+		next = addr + PAGE_SIZE;
 		rc = vmemmap_populate_range(next, last, node, NULL,
-					    pte_pfn(ptep_get(pte)),
-					    VMEMMAP_POPULATE_PAGEREF);
+					    page_to_pfn(page), flags);
 		if (rc)
 			return -ENOMEM;
 	}
@@ -558,7 +580,7 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
 		!IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
 		return NULL;
 
-	if (vmemmap_can_optimize(altmap, pgmap))
+	if (pgmap && section_vmemmap_optimizable(__pfn_to_section(pfn)))
 		r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap);
 	else
 		r = vmemmap_populate(start, end, nid, altmap);
@@ -571,20 +593,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
 	return pfn_to_page(pfn);
 }
 
-#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
-/*
- * This is called just before initializing sections for a NUMA node.
- * Any special initialization that needs to be done before the
- * generic initialization can be done from here. Sections that
- * are initialized in hooks called from here will be skipped by
- * the generic initialization.
- */
-void __init sparse_vmemmap_init_nid_early(int nid)
-{
-	hugetlb_vmemmap_init_early(nid);
-}
-#endif
-
 static void subsection_mask_set(unsigned long *map, unsigned long pfn,
 		unsigned long nr_pages)
 {
@@ -654,31 +662,6 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
 	}
 }
 
-static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
-		struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
-{
-	const unsigned int order = pgmap ? pgmap->vmemmap_shift : 0;
-	const unsigned long pages_per_compound = 1UL << order;
-
-	VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION));
-	VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION);
-
-	if (!vmemmap_can_optimize(altmap, pgmap))
-		return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
-
-	if (order < PFN_SECTION_SHIFT) {
-		VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
-		return VMEMMAP_RESERVE_NR * nr_pages / pages_per_compound;
-	}
-
-	VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
-
-	if (IS_ALIGNED(pfn, pages_per_compound))
-		return VMEMMAP_RESERVE_NR;
-
-	return 0;
-}
-
 static struct page * __meminit populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
 		struct dev_pagemap *pgmap)
@@ -686,7 +669,7 @@ static struct page * __meminit populate_section_memmap(unsigned long pfn,
 	struct page *page = __populate_section_memmap(pfn, nr_pages, nid, altmap,
 						      pgmap);
 
-	memmap_pages_add(section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap));
+	memmap_pages_add(section_nr_vmemmap_pages(pfn, nr_pages));
 
 	return page;
 }
@@ -697,7 +680,7 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
 	unsigned long start = (unsigned long) pfn_to_page(pfn);
 	unsigned long end = start + nr_pages * sizeof(struct page);
 
-	memmap_pages_add(-section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap));
+	memmap_pages_add(-section_nr_vmemmap_pages(pfn, nr_pages));
 	vmemmap_free(start, end, altmap);
 }
 
@@ -707,8 +690,7 @@ static void free_map_bootmem(struct page *memmap)
 	unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
 	unsigned long pfn = page_to_pfn(memmap);
 
-	memmap_boot_pages_add(-section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION,
-							NULL, NULL));
+	memmap_boot_pages_add(-section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION));
 	vmemmap_free(start, end, NULL);
 }
 
@@ -816,8 +798,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
 	else if (memmap)
 		free_map_bootmem(memmap);
 
-	if (empty)
+	if (empty) {
 		ms->section_mem_map = (unsigned long)NULL;
+		section_set_compound_order(ms, 0);
+	}
 }
 
 static struct page * __meminit section_activate(int nid, unsigned long pfn,
@@ -827,8 +811,13 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn,
 	struct mem_section *ms = __pfn_to_section(pfn);
 	struct mem_section_usage *usage = NULL;
 	struct page *memmap;
+	unsigned int order;
 	int rc;
 
+	order = vmemmap_can_optimize(altmap, pgmap) ? pgmap->vmemmap_shift : 0;
+	if (nr_pages < PAGES_PER_SECTION && section_compound_order(ms))
+		return ERR_PTR(-EOPNOTSUPP);
+
 	if (!ms->usage) {
 		usage = kzalloc(mem_section_usage_size(), GFP_KERNEL);
 		if (!usage)
@@ -854,6 +843,7 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn,
 	if (nr_pages < PAGES_PER_SECTION && early_section(ms))
 		return pfn_to_page(pfn);
 
+	section_set_compound_order_range(pfn, nr_pages, order);
 	memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
 	if (!memmap) {
 		section_deactivate(pfn, nr_pages, altmap, pgmap);
@@ -899,19 +889,20 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn,
 	if (IS_ERR(memmap))
 		return PTR_ERR(memmap);
 
+	ms = __nr_to_section(section_nr);
 	/*
 	 * Poison uninitialized struct pages in order to catch invalid flags
 	 * combinations.
 	 */
-	page_init_poison(memmap, sizeof(struct page) * nr_pages);
+	if (!section_vmemmap_optimizable(ms))
+		page_init_poison(memmap, sizeof(struct page) * nr_pages);
 
-	ms = __nr_to_section(section_nr);
-	__section_mark_present(ms, section_nr);
+	__highest_used_section_nr = max(section_nr, __highest_used_section_nr);
 
 	/* Align memmap to section boundary in the subsection case */
 	if (section_nr_to_pfn(section_nr) != start_pfn)
 		memmap = pfn_to_page(section_nr_to_pfn(section_nr));
-	sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0);
+	sparse_init_one_section(ms, section_nr, memmap, ms->usage);
 
 	return 0;
 }
diff --git a/mm/sparse.c b/mm/sparse.c
index 7c15406e77f5d2..702904b4170063 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -103,11 +103,22 @@ int __meminit sparse_index_init(unsigned long section_nr, int nid)
 
 	return 0;
 }
+
+static void __init sparse_extreme_init(void)
+{
+	const unsigned long size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
+
+	mem_section = memblock_alloc_or_panic(size, INTERNODE_CACHE_BYTES);
+}
 #else /* !SPARSEMEM_EXTREME */
 int __meminit sparse_index_init(unsigned long section_nr, int nid)
 {
 	return 0;
 }
+
+static void __init sparse_extreme_init(void)
+{
+}
 #endif
 
 /*
@@ -153,83 +164,50 @@ static void __init mminit_validate_memmodel_limits(unsigned long *start_pfn,
 }
 
 /*
- * There are a number of times that we loop over NR_MEM_SECTIONS,
- * looking for section_present() on each.  But, when we have very
- * large physical address spaces, NR_MEM_SECTIONS can also be
- * very large which makes the loops quite long.
- *
- * Keeping track of this gives us an easy way to break out of
- * those loops early.
+ * Looping over all possible memory sections is expensive, especially if
+ * NR_MEM_SECTIONS is large but only a fraction is actually used. Keep track of
+ * the highest section number we ever used.
  */
-unsigned long __highest_present_section_nr;
+unsigned long __highest_used_section_nr;
 
-static inline unsigned long first_present_section_nr(void)
+static inline unsigned long first_early_section_nr(void)
 {
-	return next_present_section_nr(-1);
+	return next_early_section_nr(-1);
 }
 
-/* Record a memory area against a node. */
-static void __init memory_present(int nid, unsigned long start, unsigned long end)
-{
-	unsigned long pfn;
-
-	start &= PAGE_SECTION_MASK;
-	mminit_validate_memmodel_limits(&start, &end);
-	for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
-		unsigned long section_nr = pfn_to_section_nr(pfn);
-		struct mem_section *ms;
-
-		sparse_index_init(section_nr, nid);
-		set_section_nid(section_nr, nid);
-
-		ms = __nr_to_section(section_nr);
-		if (!ms->section_mem_map) {
-			ms->section_mem_map = sparse_encode_early_nid(nid) |
-							SECTION_IS_ONLINE;
-			__section_mark_present(ms, section_nr);
-		}
-	}
-}
-
-/*
- * Mark all memblocks as present using memory_present().
- * This is a convenience function that is useful to mark all of the systems
- * memory as present during initialization.
- */
-static void __init memblocks_present(void)
+void __init sparse_sections_init(void)
 {
-	unsigned long start, end;
+	const unsigned long flags = SECTION_IS_EARLY | SECTION_IS_ONLINE;
+	unsigned long pfn, start_pfn, end_pfn, section_nr;
 	int i, nid;
 
-#ifdef CONFIG_SPARSEMEM_EXTREME
-	unsigned long size, align;
+	sparse_extreme_init();
 
-	size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
-	align = 1 << (INTERNODE_CACHE_SHIFT);
-	mem_section = memblock_alloc_or_panic(size, align);
-#endif
+	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
+		start_pfn &= PAGE_SECTION_MASK;
+		mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
 
-	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)
-		memory_present(nid, start, end);
-}
+		for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
+			struct mem_section *ms;
 
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-unsigned long __init section_map_size(void)
-{
-	return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
-}
+			section_nr = pfn_to_section_nr(pfn);
+			sparse_index_init(section_nr, nid);
+			ms = __nr_to_section(section_nr);
+			if (ms->section_mem_map)
+				continue;
 
-#else
-unsigned long __init section_map_size(void)
-{
-	return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
+			set_section_nid(section_nr, nid);
+			ms->section_mem_map = sparse_encode_early_nid(nid) | flags;
+		}
+	}
+	__highest_used_section_nr = section_nr;
 }
-
+#ifndef CONFIG_SPARSEMEM_VMEMMAP
 struct page __init *__populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
 		struct dev_pagemap *pgmap)
 {
-	unsigned long size = section_map_size();
+	const unsigned long size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
 
 	return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false);
 }
@@ -239,78 +217,58 @@ void __weak __meminit vmemmap_populate_print_last(void)
 {
 }
 
-static void *sparse_usagebuf __initdata;
-static void *sparse_usagebuf_end __initdata;
-
-/*
- * Helper function that is used for generic section initialization, and
- * can also be used by any hooks added above.
- */
-void __init sparse_init_early_section(int nid, struct page *map,
-				      unsigned long pnum, unsigned long flags)
+static void __init sparse_metadata_init_nid(int nid,
+		unsigned long start_section_nr, unsigned long end_section_nr,
+		unsigned long nr_sections)
 {
-	BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end);
-	sparse_init_one_section(__nr_to_section(pnum), pnum, map,
-			sparse_usagebuf, SECTION_IS_EARLY | flags);
-	sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size();
-}
+	struct mem_section_usage *usage;
+	unsigned long section_nr;
 
-static int __init sparse_usage_init(int nid, unsigned long map_count)
-{
-	unsigned long size;
+	usage = memblock_alloc_node(nr_sections * mem_section_usage_size(),
+				    SMP_CACHE_BYTES, nid);
+	if (!usage)
+		panic("Failed to allocate usemap for node %d\n", nid);
 
-	size = mem_section_usage_size() * map_count;
-	sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid);
-	if (!sparse_usagebuf) {
-		sparse_usagebuf_end = NULL;
-		return -ENOMEM;
-	}
+	for_each_early_section_nr(start_section_nr, section_nr) {
+		const unsigned long pfn = section_nr_to_pfn(section_nr);
+		struct page *mem_map;
 
-	sparse_usagebuf_end = sparse_usagebuf + size;
-	return 0;
-}
+		if (section_nr >= end_section_nr)
+			break;
 
-static void __init sparse_usage_fini(void)
-{
-	sparse_usagebuf = sparse_usagebuf_end = NULL;
+		mem_map = __populate_section_memmap(pfn, PAGES_PER_SECTION, nid,
+						    NULL, NULL);
+		if (!mem_map)
+			panic("Failed to allocate memmap for section %lu\n",
+			      section_nr);
+		memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION));
+		sparse_init_one_section(__nr_to_section(section_nr), section_nr,
+					mem_map, usage);
+		usage = (void *)usage + mem_section_usage_size();
+	}
 }
 
-/*
- * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
- * And number of present sections in this node is map_count.
- */
-static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
-				   unsigned long pnum_end,
-				   unsigned long map_count)
+static void __init sparse_metadata_init(void)
 {
-	unsigned long pnum;
-
-	if (sparse_usage_init(nid, map_count))
-		panic("Failed to allocate usemap for node %d\n", nid);
-
-	sparse_vmemmap_init_nid_early(nid);
+	unsigned long start_section_nr = first_early_section_nr();
+	int nid_begin = sparse_early_nid(__nr_to_section(start_section_nr));
+	unsigned long section_nr, nr_sections = 1;
 
-	for_each_present_section_nr(pnum_begin, pnum) {
-		struct mem_section *ms;
-		unsigned long pfn = section_nr_to_pfn(pnum);
+	for_each_early_section_nr(start_section_nr + 1, section_nr) {
+		const int nid = sparse_early_nid(__nr_to_section(section_nr));
 
-		if (pnum >= pnum_end)
-			break;
-
-		ms = __nr_to_section(pnum);
-		if (!preinited_vmemmap_section(ms)) {
-			struct page *map;
-
-			map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
-							nid, NULL, NULL);
-			if (!map)
-				panic("Failed to allocate memmap for section %lu\n", pnum);
-			memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
-							   PAGE_SIZE));
-			sparse_init_early_section(nid, map, pnum, 0);
+		if (nid == nid_begin) {
+			nr_sections++;
+			continue;
 		}
+		sparse_metadata_init_nid(nid_begin, start_section_nr,
+					 section_nr, nr_sections);
+		nid_begin = nid;
+		start_section_nr = section_nr;
+		nr_sections = 1;
 	}
-	sparse_usage_fini();
+	sparse_metadata_init_nid(nid_begin, start_section_nr, section_nr,
+				 nr_sections);
 }
 
 /*
@@ -319,36 +277,12 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
  */
 void __init sparse_init(void)
 {
-	unsigned long pnum_end, pnum_begin, map_count = 1;
-	int nid_begin;
-
-	/* see include/linux/mmzone.h 'struct mem_section' definition */
-	BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
-	memblocks_present();
-
 	if (compound_info_has_mask()) {
 		VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0),
 				    MAX_FOLIO_VMEMMAP_ALIGN));
 	}
 
-	pnum_begin = first_present_section_nr();
-	nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
-
-	for_each_present_section_nr(pnum_begin + 1, pnum_end) {
-		int nid = sparse_early_nid(__nr_to_section(pnum_end));
-
-		if (nid == nid_begin) {
-			map_count++;
-			continue;
-		}
-		/* Init node with sections in range [pnum_begin, pnum_end) */
-		sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
-		nid_begin = nid;
-		pnum_begin = pnum_end;
-		map_count = 1;
-	}
-	/* cover the last node */
-	sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
+	sparse_metadata_init();
 	sparse_init_subsection_map();
 	vmemmap_populate_print_last();
 }
diff --git a/mm/sparse.h b/mm/sparse.h
index 3b744667a7e624..fdbc4cacf91d17 100644
--- a/mm/sparse.h
+++ b/mm/sparse.h
@@ -9,52 +9,51 @@
 #define __MM_SPARSE_H
 
 #include <linux/mmzone.h>
+#include <linux/vmemmap-optimization.h>
 
 /*
  * mm/sparse.c
  */
 #ifdef CONFIG_SPARSEMEM
 void sparse_init(void);
+void sparse_sections_init(void);
 int sparse_index_init(unsigned long section_nr, int nid);
 
 static inline void sparse_init_one_section(struct mem_section *ms,
-		unsigned long pnum, struct page *mem_map,
-		struct mem_section_usage *usage, unsigned long flags)
+		unsigned long section_nr, struct page *mem_map,
+		struct mem_section_usage *usage)
 {
 	unsigned long coded_mem_map;
 
 	BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT);
+	VM_WARN_ON_ONCE(section_nr > __highest_used_section_nr);
 
 	/*
 	 * We encode the start PFN of the section into the mem_map such that
 	 * page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it
 	 * from the page pointer to obtain the PFN.
 	 */
-	coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum));
+	coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(section_nr));
 	VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK);
 
 	ms->section_mem_map &= ~SECTION_MAP_MASK;
-	ms->section_mem_map |= coded_mem_map;
-	ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP;
+	ms->section_mem_map |= coded_mem_map | SECTION_HAS_MEM_MAP;
 	ms->usage = usage;
 }
 
-static inline void __section_mark_present(struct mem_section *ms,
-		unsigned long section_nr)
-{
-	if (section_nr > __highest_present_section_nr)
-		__highest_present_section_nr = section_nr;
-
-	ms->section_mem_map |= SECTION_MARKED_PRESENT;
-}
-
 static inline size_t mem_section_usage_size(void)
 {
 	return struct_size_t(struct mem_section_usage, pageblock_flags,
 			     BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS));
 }
+
+static inline bool section_vmemmap_optimizable(const struct mem_section *ms)
+{
+	return vmemmap_optimizable_order(section_compound_order(ms));
+}
 #else
 static inline void sparse_init(void) {}
+static inline void sparse_sections_init(void) {}
 #endif /* CONFIG_SPARSEMEM */
 
 /*
@@ -62,8 +61,13 @@ static inline void sparse_init(void) {}
  */
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 void sparse_init_subsection_map(void);
+int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages);
 #else
 static inline void sparse_init_subsection_map(void) {}
+static inline int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages)
+{
+	return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
+}
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
 
 #endif /* __MM_SPARSE_H */
diff --git a/mm/swap.h b/mm/swap.h
index 90a551a88df63b..d5bf21f517dcea 100644
--- a/mm/swap.h
+++ b/mm/swap.h
@@ -82,7 +82,7 @@ enum swap_cluster_flags {
 
 extern int vm_swappiness;
 
-static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
+static inline int mem_cgroup_swappiness(const struct mem_cgroup *memcg)
 {
 #ifdef CONFIG_MEMCG_V1
 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
@@ -258,7 +258,7 @@ void swap_read_folio(struct swap_io_ctx *ctx, struct folio *folio);
 void swap_read_submit(struct swap_io_ctx *ctx);
 void swap_write_submit(struct swap_io_ctx *ctx);
 int swap_writeout(struct swap_io_ctx *ctx, struct folio *folio);
-void __swap_writepage(struct swap_io_ctx *ctx, struct folio *folio);
+void __swap_writeout(struct swap_io_ctx *ctx, struct folio *folio);
 
 /* linux/mm/swap_state.c */
 extern struct address_space swap_space __read_mostly;
@@ -312,19 +312,19 @@ bool swap_cache_has_folio(swp_entry_t entry);
 struct folio *swap_cache_get_folio(swp_entry_t entry);
 void *swap_cache_get_shadow(swp_entry_t entry);
 void swap_cache_del_folio(struct folio *folio);
-struct folio *swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask,
-				     unsigned long orders, struct vm_fault *vmf,
-				     struct mempolicy *mpol, pgoff_t ilx);
+struct folio *__swap_cache_alloc_folio(swp_entry_t target_entry, gfp_t gfp_mask,
+				       unsigned long orders, struct vm_fault *vmf,
+				       struct mempolicy *mpol, pgoff_t ilx);
 /* Below helpers require the caller to lock and pass in the swap cluster. */
 void __swap_cache_add_folio(struct swap_cluster_info *ci,
 			    struct folio *folio, swp_entry_t entry);
 void __swap_cache_del_folio(struct swap_cluster_info *ci,
-			    struct folio *folio, swp_entry_t entry, void *shadow);
+			    struct folio *folio, swp_entry_t entry, void *shadow,
+			    bool swapout);
 void __swap_cache_replace_folio(struct swap_cluster_info *ci,
 				struct folio *old, struct folio *new);
 
 void show_swap_cache_info(void);
-void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
 struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
 		gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr);
 struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
@@ -453,7 +453,8 @@ static inline void swap_cache_del_folio(struct folio *folio)
 }
 
 static inline void __swap_cache_del_folio(struct swap_cluster_info *ci,
-		struct folio *folio, swp_entry_t entry, void *shadow)
+		struct folio *folio, swp_entry_t entry, void *shadow,
+		bool swapout)
 {
 }
 
diff --git a/mm/swap_state.c b/mm/swap_state.c
index f3961fdd857dc6..2475ba29126dca 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -306,21 +306,28 @@ static void __swap_cache_do_del_folio(struct swap_cluster_info *ci,
  * @folio: The folio.
  * @entry: The first swap entry that the folio corresponds to.
  * @shadow: shadow value to be filled in the swap cache.
+ * @swapout: whether this folio is being reclaimed after swapout.
  *
  * Removes a folio from the swap cache and fills a shadow in place.
  * This won't put the folio's refcount. The caller has to do that.
  *
  * Context: Caller must ensure the folio is locked and in the swap cache
  * using the index of @entry, and lock the cluster that holds the entries.
+ * If @swapout is set, the folio should be in reclaim path and IRQs
+ * should be disabled.
  */
 void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio,
-			    swp_entry_t entry, void *shadow)
+			    swp_entry_t entry, void *shadow, bool swapout)
 {
 	unsigned long nr_pages = folio_nr_pages(folio);
 
-	__swap_cache_do_del_folio(ci, folio, entry, shadow);
 	node_stat_mod_folio(folio, NR_FILE_PAGES, -nr_pages);
 	lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr_pages);
+
+	if (swapout)
+		__memcg1_swapout(folio, ci);
+
+	__swap_cache_do_del_folio(ci, folio, entry, shadow);
 }
 
 /**
@@ -339,7 +346,7 @@ void swap_cache_del_folio(struct folio *folio)
 	swp_entry_t entry = folio->swap;
 
 	ci = swap_cluster_lock(__swap_entry_to_info(entry), swp_offset(entry));
-	__swap_cache_del_folio(ci, folio, entry, NULL);
+	__swap_cache_del_folio(ci, folio, entry, NULL, false);
 	swap_cluster_unlock(ci);
 
 	folio_ref_sub(folio, folio_nr_pages(folio));
@@ -389,8 +396,9 @@ void __swap_cache_replace_folio(struct swap_cluster_info *ci,
 	    folio_order(old) != folio_order(new)) {
 		ci_off = swp_cluster_offset(old->swap);
 		ci_end = ci_off + folio_nr_pages(old);
-		while (ci_off++ < ci_end)
+		do {
 			WARN_ON_ONCE(swp_tb_to_folio(__swap_table_get(ci, ci_off)) != old);
+		} while (++ci_off < ci_end);
 	}
 }
 
@@ -489,13 +497,11 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
 	node_stat_mod_folio(folio, NR_FILE_PAGES, nr_pages);
 	lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr_pages);
 
-	/* Caller will initiate read into locked new_folio */
-	folio_add_lru(folio);
 	return folio;
 }
 
 /**
- * swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache.
+ * __swap_cache_alloc_folio - Allocate folio for swapped out slot in swap cache.
  * @targ_entry: swap entry indicating the target slot
  * @gfp: memory allocation flags
  * @orders: allocation orders, must be non zero
@@ -507,13 +513,17 @@ static struct folio *__swap_cache_alloc(struct swap_cluster_info *ci,
  * doing IO (e.g. swap in or zswap writeback). The swap slot indicated by
  * @targ_entry must have a non-zero swap count (swapped out).
  *
+ * The returned folio is locked and is NOT on the LRU. The caller must either
+ * add it to the LRU with folio_add_lru() so page reclaim can find it, or free
+ * it directly once done; a folio left off the LRU is unreclaimable and leaks.
+ *
  * Context: Caller must protect the swap device with reference count or locks.
  * Return: Returns the folio if allocation succeeded and folio is in the swap
  * cache. Returns error code if failed due to race, OOM or invalid arguments.
  */
-struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
-				     unsigned long orders, struct vm_fault *vmf,
-				     struct mempolicy *mpol, pgoff_t ilx)
+struct folio *__swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
+				       unsigned long orders, struct vm_fault *vmf,
+				       struct mempolicy *mpol, pgoff_t ilx)
 {
 	int order, err;
 	struct folio *ret;
@@ -541,6 +551,48 @@ struct folio *swap_cache_alloc_folio(swp_entry_t targ_entry, gfp_t gfp,
 	return ret;
 }
 
+/**
+ * swap_writeback_dropbehind_folio - drop a dropbehind swap cache folio
+ * @folio: the off-LRU folio whose writeback has completed
+ *
+ * Context: task context, with the reference taken by folio_end_writeback()
+ * donated to us.
+ */
+void swap_writeback_dropbehind_folio(struct folio *folio)
+{
+	struct mem_cgroup *memcg;
+
+	folio_lock(folio);
+
+	/* The folio was allocated off the LRU and nothing re-adds it here. */
+	VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
+
+	rcu_read_lock();
+	memcg = folio_memcg(folio);
+	if (!mem_cgroup_tryget(memcg))
+		memcg = NULL;
+	rcu_read_unlock();
+
+	/*
+	 * Gate remove_mapping_set_shadow() on folio_test_swapcache(): a racing
+	 * swapin may have freed the swap slot (folio_free_swap()) and dropped the
+	 * folio from the cache, and it must not run on a non-swapcache folio (it
+	 * would trip __remove_mapping()'s mapping == folio_mapping() check).
+	 */
+	if (!folio_test_swapcache(folio) || folio_test_writeback(folio) ||
+	    !remove_mapping_set_shadow(swap_address_space(folio->swap), folio,
+				       memcg)) {
+		/* Raced: the folio is now owned by the swapin; put it back. */
+		folio_clear_dropbehind(folio);
+		folio_add_lru(folio);
+	}
+
+	mem_cgroup_put(memcg);
+
+	folio_unlock(folio);
+	folio_put(folio);
+}
+
 /*
  * If we are the only user, then try to free up the swap cache.
  *
@@ -649,12 +701,13 @@ static struct folio *swap_cache_read_folio(struct swap_io_ctx *ctx,
 		folio = swap_cache_get_folio(entry);
 		if (folio)
 			return folio;
-		folio = swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx);
+		folio = __swap_cache_alloc_folio(entry, gfp, BIT(0), NULL, mpol, ilx);
 	} while (PTR_ERR(folio) == -EEXIST);
 
 	if (IS_ERR_OR_NULL(folio))
 		return NULL;
 
+	folio_add_lru(folio);
 	swap_read_folio(ctx, folio);
 	if (readahead) {
 		folio_set_readahead(folio);
@@ -690,12 +743,13 @@ struct folio *swapin_sync(swp_entry_t entry, gfp_t gfp, unsigned long orders,
 		folio = swap_cache_get_folio(entry);
 		if (folio)
 			return folio;
-		folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx);
+		folio = __swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx);
 	} while (PTR_ERR(folio) == -EEXIST);
 
 	if (IS_ERR(folio))
 		return folio;
 
+	folio_add_lru(folio);
 	swap_read_folio(&ctx, folio);
 	swap_read_submit(&ctx);
 	return folio;
@@ -716,7 +770,7 @@ struct folio *read_swap_cache_async(struct swap_io_ctx *ctx, swp_entry_t entry,
 	struct folio *folio;
 
 	si = get_swap_device(entry);
-	if (!si)
+	if (IS_ERR_OR_NULL(si))
 		return NULL;
 
 	mpol = get_vma_policy(vma, addr, 0, &ilx);
@@ -952,7 +1006,7 @@ static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
 		 */
 		if (swp_type(entry) != swp_type(targ_entry)) {
 			si = get_swap_device(entry);
-			if (!si)
+			if (IS_ERR_OR_NULL(si))
 				continue;
 		}
 		folio = swap_cache_read_folio(&ctx, entry, gfp_mask, mpol, ilx,
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 601979b97f95b2..6187c02ec5ec73 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -370,8 +370,6 @@ static void discard_swap_cluster(struct swap_info_struct *si,
 	}
 }
 
-#define LATENCY_LIMIT		256
-
 static inline bool cluster_is_empty(struct swap_cluster_info *info)
 {
 	return info->count == 0;
@@ -418,6 +416,17 @@ static void swap_cluster_free_table_folio_rcu_cb(struct rcu_head *head)
 	folio_put(folio);
 }
 
+static void swap_cluster_free_count_table(struct swap_table *table)
+{
+	if (!SWP_TABLE_USE_PAGE) {
+		kmem_cache_free(swap_table_cachep, table);
+		return;
+	}
+
+	call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
+		 swap_cluster_free_table_folio_rcu_cb);
+}
+
 static void swap_cluster_free_table(struct swap_cluster_info *ci)
 {
 	struct swap_table *table;
@@ -437,13 +446,7 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci)
 		return;
 
 	rcu_assign_pointer(ci->table, NULL);
-	if (!SWP_TABLE_USE_PAGE) {
-		kmem_cache_free(swap_table_cachep, table);
-		return;
-	}
-
-	call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
-		 swap_cluster_free_table_folio_rcu_cb);
+	swap_cluster_free_count_table(table);
 }
 
 static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
@@ -466,14 +469,12 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
 	if (!table)
 		return -ENOMEM;
 
-	rcu_assign_pointer(ci->table, table);
-
 #ifdef CONFIG_MEMCG
 	if (!mem_cgroup_disabled()) {
 		VM_WARN_ON_ONCE(ci->memcg_table);
 		ci->memcg_table = kzalloc_obj(*ci->memcg_table, gfp);
 		if (!ci->memcg_table) {
-			swap_cluster_free_table(ci);
+			swap_cluster_free_count_table(table);
 			return -ENOMEM;
 		}
 	}
@@ -484,9 +485,16 @@ static int swap_cluster_alloc_table(struct swap_cluster_info *ci, gfp_t gfp)
 	ci->zero_bitmap = bitmap_zalloc(SWAPFILE_CLUSTER, gfp);
 	if (!ci->zero_bitmap) {
 		swap_cluster_free_table(ci);
+		swap_cluster_free_count_table(table);
 		return -ENOMEM;
 	}
 #endif
+
+	/*
+	 * Make tables visible to cluster_is_usable() after everything is
+	 * ready.
+	 */
+	rcu_assign_pointer(ci->table, table);
 	return 0;
 }
 
@@ -1318,10 +1326,8 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
 {
 	unsigned long end = offset + nr_entries - 1;
 	void (*swap_slot_free_notify)(struct block_device *, unsigned long);
-	unsigned int i;
 
-	for (i = 0; i < nr_entries; i++)
-		zswap_invalidate(swp_entry(si->type, offset + i));
+	zswap_invalidate(si->type, offset, nr_entries);
 
 	if (si->flags & SWP_BLKDEV)
 		swap_slot_free_notify =
@@ -1504,7 +1510,7 @@ int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp)
 	unsigned long offset = swp_offset(entry);
 
 	si = get_swap_device(entry);
-	if (!si)
+	if (IS_ERR_OR_NULL(si))
 		return 0;
 
 	ci = __swap_offset_to_cluster(si, offset);
@@ -1517,20 +1523,17 @@ int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp)
 static void swap_extend_table_try_free(struct swap_cluster_info *ci)
 {
 	unsigned long i;
-	bool can_free = true;
 
 	if (!ci->extend_table)
 		return;
 
 	for (i = 0; i < SWAPFILE_CLUSTER; i++) {
 		if (ci->extend_table[i])
-			can_free = false;
+			return;
 	}
 
-	if (can_free) {
-		kfree(ci->extend_table);
-		ci->extend_table = NULL;
-	}
+	kfree(ci->extend_table);
+	ci->extend_table = NULL;
 }
 
 /* Decrease the swap count of one slot, without freeing it */
@@ -1722,7 +1725,6 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si,
 failed:
 	while (ci_off-- > ci_start)
 		__swap_cluster_put_entry(ci, ci_off);
-	swap_extend_table_try_free(ci);
 	swap_cluster_unlock(ci);
 	return err;
 }
@@ -1735,7 +1737,9 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si,
  * swap cache.
  *
  * Context: Caller needs to hold the folio lock.
- * Return: Whether the folio was added to the swap cache.
+ * Return: %0 on success, %-E2BIG if splitting the folio might allow swapout,
+ * %-ENOSPC if no global swap space is available, or %-ENOMEM if splitting
+ * would not help.
  */
 int folio_alloc_swap(struct folio *folio)
 {
@@ -1747,11 +1751,11 @@ int folio_alloc_swap(struct folio *folio)
 
 	if (order) {
 		/*
-		 * Reject large allocation when THP_SWAP is disabled,
-		 * the caller should split the folio and try again.
+		 * Reject large allocation when THP_SWAP is disabled. Check below
+		 * whether splitting and retrying can make progress.
 		 */
 		if (!IS_ENABLED(CONFIG_THP_SWAP))
-			return -EAGAIN;
+			goto failed;
 
 		/*
 		 * Allocation size should never exceed cluster size
@@ -1759,7 +1763,7 @@ int folio_alloc_swap(struct folio *folio)
 		 */
 		if (size > SWAPFILE_CLUSTER) {
 			VM_WARN_ON_ONCE(1);
-			return -EINVAL;
+			goto failed;
 		}
 	}
 
@@ -1775,13 +1779,23 @@ int folio_alloc_swap(struct folio *folio)
 	}
 
 	/* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */
-	if (unlikely(mem_cgroup_try_charge_swap(folio)))
+	if (unlikely(mem_cgroup_try_charge_swap(folio))) {
 		swap_cache_del_folio(folio);
+		goto failed;
+	}
 
 	if (unlikely(!folio_test_swapcache(folio)))
-		return -ENOMEM;
+		goto failed;
 
 	return 0;
+
+failed:
+	if (get_nr_swap_pages() <= 0)
+		return -ENOSPC;
+	if (mem_cgroup_get_folio_swap_margin(folio) <= 0)
+		return -ENOMEM;
+
+	return order ? -E2BIG : -ENOMEM;
 }
 
 /**
@@ -1810,7 +1824,7 @@ int folio_dup_swap(struct folio *folio, struct page *page)
 	VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio);
 
 	if (page) {
-		entry.val += folio_page_idx(folio, page);
+		entry = folio_page_swap_entry(folio, page);
 		nr_pages = 1;
 	}
 
@@ -1837,7 +1851,7 @@ void folio_put_swap(struct folio *folio, struct page *page)
 	VM_WARN_ON_FOLIO(!folio_test_swapcache(folio), folio);
 
 	if (page) {
-		entry.val += folio_page_idx(folio, page);
+		entry = folio_page_swap_entry(folio, page);
 		nr_pages = 1;
 	}
 
@@ -1859,7 +1873,10 @@ void folio_put_swap(struct folio *folio, struct page *page)
  * Check whether swap entry is valid in the swap device.  If so,
  * return pointer to swap_info_struct, and keep the swap entry valid
  * via preventing the swap device from being swapoff, until
- * put_swap_device() is called.  Otherwise return NULL.
+ * put_swap_device() is called.  Return NULL for an empty entry or a
+ * device that is going away, and ERR_PTR(-EIO) if the entry's type
+ * names no swap device or its offset is past the end of one. These EIOs
+ * are preceded by pr_err().
  *
  * Notice that swapoff or swapoff+swapon can still happen before the
  * percpu_ref_tryget_live() in get_swap_device() or after the
@@ -1870,7 +1887,7 @@ void folio_put_swap(struct folio *folio, struct page *page)
  *   CPU1				CPU2
  *   do_swap_page()
  *     ...				swapoff+swapon
- *     swap_cache_alloc_folio()
+ *     __swap_cache_alloc_folio()
  *       // check swap_map
  *     // verify PTE not changed
  *
@@ -1900,12 +1917,14 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry)
 	return si;
 bad_nofile:
 	pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val);
+	return ERR_PTR(-EIO);
+
 out:
 	return NULL;
 put_out:
 	pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
 	percpu_ref_put(&si->users);
-	return NULL;
+	return ERR_PTR(-EIO);
 }
 
 /*
@@ -2001,7 +2020,7 @@ int swp_swapcount(swp_entry_t entry)
 	int count;
 
 	si = get_swap_device(entry);
-	if (!si)
+	if (IS_ERR_OR_NULL(si))
 		return 0;
 
 	ci = swap_cluster_lock(si, swp_offset(entry));
@@ -2127,7 +2146,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr)
 	struct swap_info_struct *si;
 
 	si = get_swap_device(entry);
-	if (WARN_ON_ONCE(!si))
+	if (WARN_ON_ONCE(IS_ERR_OR_NULL(si)))
 		return;
 	if (WARN_ON_ONCE(end_offset > si->max))
 		goto out;
@@ -2688,7 +2707,7 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type)
 	if (check_stable_address_space(mm))
 		goto unlock;
 	for_each_vma(vmi, vma) {
-		if (vma->anon_vma && !is_vm_hugetlb_page(vma)) {
+		if (vma_has_anon_rmap(vma) && !vma_is_hugetlb(vma)) {
 			ret = unuse_vma(vma, type);
 			if (ret)
 				break;
@@ -2709,7 +2728,9 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type)
 static unsigned int find_next_to_unuse(struct swap_info_struct *si,
 					unsigned int prev)
 {
-	unsigned int i;
+	struct swap_cluster_info *ci;
+	unsigned long i, end;
+	unsigned int ci_off;
 	unsigned long swp_tb;
 
 	/*
@@ -2718,19 +2739,36 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
 	 * hits are okay, and sys_swapoff() has already prevented new
 	 * allocations from this area (while holding swap_lock).
 	 */
-	for (i = prev + 1; i < si->max; i++) {
-		swp_tb = swap_table_get(__swap_offset_to_cluster(si, i),
-					i % SWAPFILE_CLUSTER);
-		if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb))
-			break;
-		if ((i % LATENCY_LIMIT) == 0)
+	i = prev + 1;
+	while (i < si->max) {
+		ci = __swap_offset_to_cluster(si, i);
+		end = min_t(unsigned long,
+			    ALIGN_DOWN(i, SWAPFILE_CLUSTER) + SWAPFILE_CLUSTER,
+			    si->max);
+
+		/*
+		 * An empty cluster has no slot in use, so skip it whole.
+		 * A slot is uncounted only after its folio left the swap
+		 * cache, so there is nothing here for try_to_unuse() to act on.
+		 * Count only drops here, so a READ_ONCE() without ci->lock is
+		 * enough, unlike in every other cluster_is_empty() caller.
+		 */
+		if (!READ_ONCE(ci->count)) {
+			i = end;
 			cond_resched();
-	}
+			continue;
+		}
 
-	if (i == si->max)
-		i = 0;
+		ci_off = i % SWAPFILE_CLUSTER;
+		for (; i < end; ci_off++, i++) {
+			swp_tb = swap_table_get(ci, ci_off);
+			if (!swp_tb_is_null(swp_tb) && !swp_tb_is_bad(swp_tb))
+				return i;
+		}
+		cond_resched();
+	}
 
-	return i;
+	return 0;
 }
 
 static int try_to_unuse(unsigned int type)
@@ -2928,7 +2966,7 @@ EXPORT_SYMBOL_GPL(add_swap_extent);
 /*
  * A `swap extent' is a simple thing which maps a contiguous range of pages
  * onto a contiguous range of disk blocks.  A rbtree of swap extents is
- * built at swapon time and is then used at swap_writepage/swap_read_folio
+ * built at swapon time and is then used at swap_writeout/swap_read_folio
  * time for locating where on disk a page belongs.
  *
  * If the swapfile is an S_ISBLK block device, a single extent is installed.
@@ -3721,11 +3759,6 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
 
 	maxpages = si->max;
 
-	/* Set up the swap cluster info */
-	error = setup_swap_clusters_info(si, swap_header, maxpages);
-	if (error)
-		goto bad_swap_unlock_inode;
-
 	if (si->bdev && bdev_stable_writes(si->bdev))
 		si->flags |= SWP_STABLE_WRITES;
 
@@ -3739,6 +3772,14 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
 		inced_nr_rotate_swap = true;
 	}
 
+	/*
+	 * Set up the swap cluster info after SWP_ flags handling as
+	 * setup_swap_clusters_info() checks SWP_SOLIDSTATE.
+	 */
+	error = setup_swap_clusters_info(si, swap_header, maxpages);
+	if (error)
+		goto bad_swap_unlock_inode;
+
 	if ((swap_flags & SWAP_FLAG_DISCARD) &&
 	    si->bdev && bdev_max_discard_sectors(si->bdev)) {
 		/*
diff --git a/mm/truncate.c b/mm/truncate.c
index b58ba940be4740..5b1b13cf8ff0a3 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -206,35 +206,47 @@ static int folio_split_or_unmap(struct folio *folio, struct page *split_at,
 /*
  * Handle partial folios.  The folio may be entirely within the
  * range if a split has raced with us.  If not, we zero the part of the
- * folio that's within the [start, end] range, and then split the folio if
+ * folio that's within the [lstart, lend] range, and then split the folio if
  * it's large.  split_page_range() will discard pages which now lie beyond
  * i_size, and we rely on the caller to discard pages which lie within a
  * newly created hole.
  *
- * Returns false if splitting failed so the caller can avoid
- * discarding the entire folio which is stubbornly unsplit.
+ * When @pstart and/or @pend are non-NULL they receive the indexes of the
+ * folio range fully covered by [lstart, lend] after any split (or none),
+ * aligned inwards to min_order, i.e. the range of folios wholly within
+ * [lstart, lend] and so safe to discard.
+ *
+ * Return %true if at least one split succeeded, %false otherwise.
  */
-bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
+bool truncate_inode_partial_folio(struct folio *folio, loff_t lstart,
+				  loff_t lend, pgoff_t *pstart, pgoff_t *pend)
 {
 	loff_t pos = folio_pos(folio);
 	size_t size = folio_size(folio);
 	unsigned int offset, length;
 	struct page *split_at, *split_at2;
+	unsigned long min_nrbytes;
 	unsigned int min_order;
 
-	if (pos < start)
-		offset = start - pos;
+	if (pos < lstart)
+		offset = lstart - pos;
 	else
 		offset = 0;
-	if (pos + size <= (u64)end)
+	if (pos + size <= (u64)lend)
 		length = size - offset;
 	else
-		length = end + 1 - pos - offset;
+		length = lend + 1 - pos - offset;
+
+	if (pstart)
+		*pstart = offset ? folio_next_index(folio) : folio->index;
+	if (pend)
+		*pend = (pos + size > (u64)lend) ? folio->index :
+						   folio_next_index(folio);
 
 	folio_wait_writeback(folio);
 	if (length == size) {
 		truncate_inode_folio(folio->mapping, folio);
-		return true;
+		return false;
 	}
 
 	/*
@@ -247,10 +259,12 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
 
 	if (folio_needs_release(folio))
 		folio_invalidate(folio, offset, length);
-	if (!folio_test_large(folio))
-		return true;
 
 	min_order = mapping_min_folio_order(folio->mapping);
+	min_nrbytes = mapping_min_folio_nrbytes(folio->mapping);
+	if (folio_order(folio) == min_order)
+		return false;
+
 	split_at = folio_page(folio, PAGE_ALIGN_DOWN(offset) / PAGE_SIZE);
 	if (!folio_split_or_unmap(folio, split_at, min_order)) {
 		/*
@@ -259,38 +273,62 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
 		 * for shmem truncate
 		 */
 		struct folio *folio2;
+		pgoff_t end, aligned_end = round_down(pos + offset + length,
+						min_nrbytes) >> PAGE_SHIFT;
 
-		if (offset + length == size)
-			goto no_split;
+		if (pstart)
+			*pstart = round_up(pos + offset,
+					   min_nrbytes) >> PAGE_SHIFT;
 
-		split_at2 = folio_page(folio,
-				PAGE_ALIGN_DOWN(offset + length) / PAGE_SIZE);
-		folio2 = page_folio(split_at2);
-
-		if (!folio_try_get(folio2))
-			goto no_split;
-
-		if (!folio_test_large(folio2))
+		end = aligned_end;
+		if (offset + length == size)
 			goto out;
 
-		if (!folio_trylock(folio2))
+		/*
+		 * After the first split at the start edge, the folio at the
+		 * end edge may be freed and reused concurrently.
+		 * __filemap_get_folio() looks up the straddler at aligned_end
+		 * and returns it locked and ref'd with the mapping
+		 * validated.
+		 */
+		folio2 = __filemap_get_folio(folio->mapping, aligned_end,
+					     FGP_LOCK | FGP_NOWAIT, 0);
+		if (IS_ERR(folio2)) {
+			/*
+			 * No sub-folio straddles the boundary when
+			 * aligned_end is empty, so discarding up to it is
+			 * safe.  Otherwise the straddler is locked by
+			 * someone else and we cannot obtain a reliable end
+			 * position, so we fall back to folio->index, which
+			 * is safe but leaves the sub-folios split off at
+			 * the offset edge in the page cache.
+			 */
+			if (PTR_ERR(folio2) != -ENOENT)
+				end = folio->index;
 			goto out;
+		}
+
+		/* Already at the minimum order, nothing to split */
+		if (folio_order(folio2) == min_order)
+			goto out_put;
+
+		split_at2 = folio_page(folio2, (aligned_end - folio2->index));
 
-		/* make sure folio2 is large and does not change its mapping */
-		if (folio_test_large(folio2) &&
-		    folio2->mapping == folio->mapping)
-			folio_split_or_unmap(folio2, split_at2, min_order);
+		/* Split failed, keep the straddler intact */
+		if (folio_split_or_unmap(folio2, split_at2, min_order))
+			end = folio2->index;
 
+out_put:
 		folio_unlock(folio2);
-out:
 		folio_put(folio2);
-no_split:
+out:
+		if (pend)
+			*pend = end;
 		return true;
 	}
-	if (folio_test_dirty(folio))
-		return false;
-	truncate_inode_folio(folio->mapping, folio);
-	return true;
+	if (!folio_test_dirty(folio))
+		truncate_inode_folio(folio->mapping, folio);
+	return false;
 }
 
 /*
@@ -345,9 +383,11 @@ long mapping_evict_folio(struct address_space *mapping, struct folio *folio)
  * @lstart: offset from which to truncate
  * @lend: offset to which to truncate (inclusive)
  *
- * Truncate the page cache, removing the pages that are between
- * specified offsets (and zeroing out partial pages
- * if lstart or lend + 1 is not page aligned).
+ * Truncate the page cache, removing the folios that are between specified
+ * offsets (and zeroing out partial folios if lstart or lend + 1 is not
+ * folio aligned).  For mappings with a non-zero minimum folio order, the
+ * boundaries are aligned inwards to 1 << min_order so the edge sub-folio
+ * straddling the range is kept.
  *
  * Truncate takes two passes - the first pass is nonblocking.  It will not
  * block on page locks and it will not block on writeback.  The second pass
@@ -366,6 +406,7 @@ long mapping_evict_folio(struct address_space *mapping, struct folio *folio)
 void truncate_inode_pages_range(struct address_space *mapping,
 				loff_t lstart, uoff_t lend)
 {
+	pgoff_t		min_nrpages = mapping_min_folio_nrpages(mapping);
 	pgoff_t		start;		/* inclusive */
 	pgoff_t		end;		/* exclusive */
 	struct folio_batch fbatch;
@@ -379,9 +420,8 @@ void truncate_inode_pages_range(struct address_space *mapping,
 		return;
 
 	/*
-	 * 'start' and 'end' always covers the range of pages to be fully
-	 * truncated. Partial pages are covered with 'partial_start' at the
-	 * start of the range and 'partial_end' at the end of the range.
+	 * 'start' and 'end' always covers the range of folios to be fully
+	 * truncated, with both boundaries aligned inwards to 1 << min_order.
 	 * Note that 'end' is exclusive while 'lend' is inclusive.
 	 */
 	start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
@@ -395,6 +435,10 @@ void truncate_inode_pages_range(struct address_space *mapping,
 	else
 		end = (lend + 1) >> PAGE_SHIFT;
 
+	start = round_up(start, min_nrpages);
+	if (end != (pgoff_t)-1)
+		end = round_down(end, min_nrpages);
+
 	folio_batch_init(&fbatch);
 	index = start;
 	while (index < end && find_lock_entries(mapping, &index, end - 1,
@@ -413,11 +457,8 @@ void truncate_inode_pages_range(struct address_space *mapping,
 	folio = __filemap_get_folio(mapping, lstart >> PAGE_SHIFT, FGP_LOCK, 0);
 	if (!IS_ERR(folio)) {
 		same_folio = lend < folio_next_pos(folio);
-		if (!truncate_inode_partial_folio(folio, lstart, lend)) {
-			start = folio_next_index(folio);
-			if (same_folio)
-				end = folio->index;
-		}
+		truncate_inode_partial_folio(folio, lstart, lend, &start,
+					     same_folio ? &end : NULL);
 		folio_unlock(folio);
 		folio_put(folio);
 		folio = NULL;
@@ -427,8 +468,8 @@ void truncate_inode_pages_range(struct address_space *mapping,
 		folio = __filemap_get_folio(mapping, lend >> PAGE_SHIFT,
 						FGP_LOCK, 0);
 		if (!IS_ERR(folio)) {
-			if (!truncate_inode_partial_folio(folio, lstart, lend))
-				end = folio->index;
+			truncate_inode_partial_folio(folio, lstart, lend,
+						     NULL, &end);
 			folio_unlock(folio);
 			folio_put(folio);
 		}
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 74f04c323c50fb..81967c63fc1785 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -122,7 +122,6 @@ struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm,
 	return vma;
 }
 
-#ifdef CONFIG_PER_VMA_LOCK
 /*
  * uffd_lock_vma() - Lookup and lock vma corresponding to @address.
  * @mm: mm to search vma in.
@@ -130,8 +129,10 @@ struct vm_area_struct *find_vma_and_prepare_anon(struct mm_struct *mm,
  *
  * Should be called without holding mmap_lock.
  *
- * Return: A locked vma containing @address, -ENOENT if no vma is found, or
- * -ENOMEM if anon_vma couldn't be allocated.
+ * Return: A locked vma containing @address, -ENOENT if no vma is found,
+ * -ENOMEM if the anon rmap couldn't be allocated, or -EAGAIN if vma refcount
+ * overflow happened due to high number of readers and the caller should
+ * retry later.
  */
 static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm,
 				       unsigned long address)
@@ -141,10 +142,10 @@ static struct vm_area_struct *uffd_lock_vma(struct mm_struct *mm,
 	vma = lock_vma_under_rcu(mm, address);
 	if (vma) {
 		/*
-		 * We know we're going to need to use anon_vma, so check
-		 * that early.
+		 * We know we're going to need an anon rmap, so check that
+		 * early.
 		 */
-		if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma))
+		if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma_has_anon_rmap(vma)))
 			vma_end_read(vma);
 		else
 			return vma;
@@ -182,34 +183,6 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma)
 	vma_end_read(vma);
 }
 
-#else
-
-static struct vm_area_struct *uffd_mfill_lock(struct mm_struct *dst_mm,
-					      unsigned long dst_start,
-					      unsigned long len)
-{
-	struct vm_area_struct *dst_vma;
-
-	mmap_read_lock(dst_mm);
-	dst_vma = find_vma_and_prepare_anon(dst_mm, dst_start);
-	if (IS_ERR(dst_vma))
-		goto out_unlock;
-
-	if (validate_dst_vma(dst_vma, dst_start + len))
-		return dst_vma;
-
-	dst_vma = ERR_PTR(-ENOENT);
-out_unlock:
-	mmap_read_unlock(dst_mm);
-	return dst_vma;
-}
-
-static void uffd_mfill_unlock(struct vm_area_struct *vma)
-{
-	mmap_read_unlock(vma->vm_mm);
-}
-#endif
-
 static void mfill_put_vma(struct mfill_state *state)
 {
 	if (!state->vma)
@@ -264,7 +237,7 @@ static int mfill_get_vma(struct mfill_state *state)
 	if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP))
 		goto out_unlock;
 
-	if (is_vm_hugetlb_page(dst_vma))
+	if (vma_is_hugetlb(dst_vma))
 		return 0;
 
 	ops = vma_uffd_ops(dst_vma);
@@ -831,7 +804,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb(
 		}
 
 		err = -ENOENT;
-		if (!is_vm_hugetlb_page(dst_vma))
+		if (!vma_is_hugetlb(dst_vma))
 			goto out_unlock_vma;
 
 		err = -EINVAL;
@@ -994,7 +967,7 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,
 	/*
 	 * If this is a HUGETLB vma, pass off to appropriate routine
 	 */
-	if (is_vm_hugetlb_page(state.vma))
+	if (vma_is_hugetlb(state.vma))
 		return  mfill_atomic_hugetlb(ctx, state.vma, dst_start,
 					     src_start, len, flags);
 
@@ -1141,7 +1114,7 @@ static int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
 			break;
 		}
 
-		if (is_vm_hugetlb_page(dst_vma)) {
+		if (vma_is_hugetlb(dst_vma)) {
 			err = -EINVAL;
 			page_mask = vma_kernel_pagesize(dst_vma) - 1;
 			if ((start & page_mask) || (len & page_mask))
@@ -1199,7 +1172,7 @@ int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
 		if (!userfaultfd_rwp(dst_vma))
 			return -ENOENT;
 
-		if (is_vm_hugetlb_page(dst_vma)) {
+		if (vma_is_hugetlb(dst_vma)) {
 			unsigned long page_mask;
 
 			page_mask = vma_kernel_pagesize(dst_vma) - 1;
@@ -1449,6 +1422,7 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma,
 	orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte);
 	if (pgtable_supports_soft_dirty())
 		orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte);
+	orig_src_pte = pte_swp_clear_uffd(orig_src_pte);
 	/* Re-arm RWP on the moved swap entry if dst_vma is RWP-registered. */
 	if (userfaultfd_rwp(dst_vma))
 		orig_src_pte = pte_swp_mkuffd(orig_src_pte);
@@ -1700,14 +1674,14 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd
 		}
 
 		si = get_swap_device(entry);
-		if (unlikely(!si)) {
+		if (IS_ERR_OR_NULL(si)) {
 			ret = -EAGAIN;
 			goto out;
 		}
 		/*
 		 * Verify the existence of the swapcache. If present, the folio's
 		 * index and mapping must be updated even when the PTE is a swap
-		 * entry. The anon_vma lock is not taken during this process since
+		 * entry. The anon rmap lock is not taken during this process since
 		 * the folio has already been unmapped, and the swap entry is
 		 * exclusive, preventing rmap walks.
 		 *
@@ -1757,7 +1731,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd
 	if (dst_pte)
 		pte_unmap(dst_pte);
 	mmu_notifier_invalidate_range_end(&range);
-	if (si)
+	if (!IS_ERR_OR_NULL(si))
 		put_swap_device(si);
 
 	return ret;
@@ -1781,10 +1755,18 @@ static inline bool move_splits_huge_pmd(unsigned long dst_addr,
 }
 #endif
 
-static inline bool vma_move_compatible(struct vm_area_struct *vma)
+static inline bool vma_move_compatible(const struct vm_area_struct *vma)
 {
-	return !(vma->vm_flags & (VM_PFNMAP | VM_IO |  VM_HUGETLB |
-				  VM_MIXEDMAP | VM_SHADOW_STACK));
+	/* uffd is generally incompatible with kernel-owned mappings. */
+	if (vma_is_kernel_owned(vma))
+		return false;
+	/* The shadow stack should not be written to by userspace. */
+	if (vma_test_single_mask(vma, VMA_SHADOW_STACK))
+		return false;
+	/* hugetlb mappings cannot be safely moved. */
+	if (vma_is_hugetlb(vma))
+		return false;
+	return true;
 }
 
 static int validate_move_areas(struct userfaultfd_ctx *ctx,
@@ -1850,7 +1832,6 @@ int find_vmas_mm_locked(struct mm_struct *mm,
 	return 0;
 }
 
-#ifdef CONFIG_PER_VMA_LOCK
 static int uffd_move_lock(struct mm_struct *mm,
 			  unsigned long dst_start,
 			  unsigned long src_start,
@@ -1925,31 +1906,6 @@ static void uffd_move_unlock(struct vm_area_struct *dst_vma,
 		vma_end_read(dst_vma);
 }
 
-#else
-
-static int uffd_move_lock(struct mm_struct *mm,
-			  unsigned long dst_start,
-			  unsigned long src_start,
-			  struct vm_area_struct **dst_vmap,
-			  struct vm_area_struct **src_vmap)
-{
-	int err;
-
-	mmap_read_lock(mm);
-	err = find_vmas_mm_locked(mm, dst_start, src_start, dst_vmap, src_vmap);
-	if (err)
-		mmap_read_unlock(mm);
-	return err;
-}
-
-static void uffd_move_unlock(struct vm_area_struct *dst_vma,
-			     struct vm_area_struct *src_vma)
-{
-	mmap_assert_locked(src_vma->vm_mm);
-	mmap_read_unlock(dst_vma->vm_mm);
-}
-#endif
-
 /**
  * move_pages - move arbitrary anonymous pages of an existing vma
  * @ctx: pointer to the userfaultfd context
@@ -1962,7 +1918,7 @@ static void uffd_move_unlock(struct vm_area_struct *dst_vma,
  *
  * move_pages() remaps arbitrary anonymous pages atomically in zero
  * copy. It only works on non shared anonymous pages because those can
- * be relocated without generating non linear anon_vmas in the rmap
+ * be relocated without generating non linear anon rmaps in the rmap
  * code.
  *
  * It provides a zero copy mechanism to handle userspace page faults.
@@ -2199,10 +2155,11 @@ static bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags,
 {
 	const struct vm_uffd_ops *ops = vma_uffd_ops(vma);
 
-	if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK))
+	/* Non-persistent memory is inherently not controllable by userspace. */
+	if (!vma_is_persistent(vma))
 		return false;
-
-	if (!is_vm_hugetlb_page(vma) && (vma->vm_flags & VM_SPECIAL))
+	/* The shadow stack should not be written to by userspace. */
+	if (vma_test_single_mask(vma, VMA_SHADOW_STACK))
 		return false;
 
 	vm_flags &= __VM_UFFD_FLAGS;
@@ -2372,7 +2329,7 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx,
 		 */
 		userfaultfd_set_ctx(vma, ctx, vm_flags);
 
-		if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
+		if (vma_is_hugetlb(vma) && uffd_disable_huge_pmd_share(vma))
 			hugetlb_unshare_all_pmds(vma);
 
 skip:
@@ -2948,7 +2905,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
 	 * (sleepable) vma lock can modify the current task state, that
 	 * must be before explicitly calling set_current_state().
 	 */
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		hugetlb_vma_lock_read(vma);
 
 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
@@ -2965,7 +2922,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
 	set_current_state(blocking_state);
 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
 
-	if (is_vm_hugetlb_page(vma)) {
+	if (vma_is_hugetlb(vma)) {
 		must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason);
 		hugetlb_vma_unlock_read(vma);
 	} else {
@@ -3797,7 +3754,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 	 * If the first vma contains huge pages, make sure start address
 	 * is aligned to huge page size.
 	 */
-	if (is_vm_hugetlb_page(vma)) {
+	if (vma_is_hugetlb(vma)) {
 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
 
 		if (start & (vma_hpagesize - 1))
@@ -3848,7 +3805,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 		 * If this vma contains ending address, and huge pages
 		 * check alignment.
 		 */
-		if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
+		if (vma_is_hugetlb(cur) && end <= cur->vm_end &&
 		    end > cur->vm_start) {
 			unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
 
@@ -3884,7 +3841,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 		/*
 		 * Note vmas containing huge pages
 		 */
-		if (is_vm_hugetlb_page(cur))
+		if (vma_is_hugetlb(cur))
 			basic_ioctls = true;
 
 		found = true;
@@ -3970,7 +3927,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
 	 * If the first vma contains huge pages, make sure start address
 	 * is aligned to huge page size.
 	 */
-	if (is_vm_hugetlb_page(vma)) {
+	if (vma_is_hugetlb(vma)) {
 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
 
 		if (start & (vma_hpagesize - 1))
@@ -4811,12 +4768,12 @@ static int new_userfaultfd(int flags)
 		   anon_inode_create_getfile("[userfaultfd]", &userfaultfd_fops, ctx,
 					     O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS),
 					     NULL));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
 	/* prevent the mm struct to be freed */
 	mmgrab(ctx->mm);
-	fd_prepare_file(fdf)->f_mode |= FMODE_NOWAIT;
+	fdf->file->f_mode |= FMODE_NOWAIT;
 	retain_and_null_ptr(ctx);
 	return fd_publish(fdf);
 }
diff --git a/mm/util.c b/mm/util.c
index bf0513d1d3d086..20346509d1225e 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -1061,6 +1061,66 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen)
 	return res;
 }
 
+/**
+ * copy_remote_mm_str - copy a string from a remote address space.
+ * @mm:         the remote address space
+ * @addr:       start address to read from
+ * @buf:        destination buffer
+ * @len:        number of bytes to copy
+ * @gup_flags:  flags modifying lookup behaviour
+ *
+ * The caller must hold a reference on @mm.
+ *
+ * Return: number of bytes copied from @addr (source) to @buf (destination),
+ * not including the trailing NUL. If @len is zero, return 0 without accessing
+ * @buf. Otherwise, @buf is always NUL-terminated. On any error, return
+ * -EFAULT.
+ */
+int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr,
+		void *buf, int len, unsigned int gup_flags)
+{
+	if (unlikely(len == 0))
+		return 0;
+
+	return __copy_remote_mm_str(mm, addr, buf, len, gup_flags);
+}
+
+/**
+ * copy_remote_vm_str - copy a string from another process's address space.
+ * @tsk:	the task of the target address space
+ * @addr:	start address to read from
+ * @buf:	destination buffer
+ * @len:	number of bytes to copy
+ * @gup_flags:	flags modifying lookup behaviour
+ *
+ * Return: number of bytes copied from @addr (source) to @buf (destination),
+ * not including the trailing NUL. If @len is zero, return 0 without accessing
+ * @buf. Otherwise, @buf is always NUL-terminated. On any error, return
+ * -EFAULT.
+ */
+int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
+		void *buf, int len, unsigned int gup_flags)
+{
+	struct mm_struct *mm;
+	int ret;
+
+	if (unlikely(len == 0))
+		return 0;
+
+	mm = get_task_mm(tsk);
+	if (!mm) {
+		*(char *)buf = '\0';
+		return -EFAULT;
+	}
+
+	ret = __copy_remote_mm_str(mm, addr, buf, len, gup_flags);
+
+	mmput(mm);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(copy_remote_vm_str);
+
 int __weak memcmp_pages(struct page *page1, struct page *page2)
 {
 	char *addr1, *addr2;
@@ -1224,23 +1284,35 @@ EXPORT_SYMBOL(compat_set_desc_from_vma);
 int __compat_vma_mmap(struct vm_area_desc *desc,
 		      struct vm_area_struct *vma)
 {
+	struct vm_area_desc prev_desc;
 	int err;
 
+	/* Derive state prior to mmap_prepare hook. */
+	compat_set_desc_from_vma(&prev_desc, desc->file, vma);
 	/* Perform any preparatory tasks for mmap action. */
 	err = mmap_action_prepare(desc);
 	if (err)
-		return err;
+		goto err_put;
+	/* Check the caller did nothing crazy. */
+	err = mmap_prepare_validate(&prev_desc, desc);
+	if (err)
+		goto err_put;
 	/* Update the VMA from the descriptor. */
 	compat_set_vma_from_desc(vma, desc);
 	/* Complete any specified mmap actions. */
 	return mmap_action_complete(vma, &desc->action, /*is_compat=*/true);
+
+err_put:
+	if (desc->vm_file != vma->vm_file)
+		fput(desc->vm_file);
+	return err;
 }
 EXPORT_SYMBOL(__compat_vma_mmap);
 
 /**
  * compat_vma_mmap() - Apply the file's .mmap_prepare() hook to an
  * existing VMA and execute any requested actions.
- * @file: The file which possesss an f_op->mmap_prepare() hook.
+ * @file: The file which possesses an f_op->mmap_prepare() hook.
  * @vma: The VMA to apply the .mmap_prepare() hook to.
  *
  * Ordinarily, .mmap_prepare() is invoked directly upon mmap(). However, certain
@@ -1455,8 +1527,10 @@ int mmap_action_prepare(struct vm_area_desc *desc)
 		return io_remap_pfn_range_prepare(desc);
 	case MMAP_SIMPLE_IO_REMAP:
 		return simple_ioremap_prepare(desc);
-	case MMAP_MAP_KERNEL_PAGES:
+	case MMAP_KERNEL_PAGES:
 		return map_kernel_pages_prepare(desc);
+	case MMAP_DISCONTIG_KERNEL_PAGES:
+		return map_discontig_kernel_pages_prepare(desc);
 	}
 
 	WARN_ON_ONCE(1);
@@ -1486,9 +1560,12 @@ int mmap_action_complete(struct vm_area_struct *vma,
 	case MMAP_REMAP_PFN:
 		err = remap_pfn_range_complete(vma, action);
 		break;
-	case MMAP_MAP_KERNEL_PAGES:
+	case MMAP_KERNEL_PAGES:
 		err = map_kernel_pages_complete(vma, action);
 		break;
+	case MMAP_DISCONTIG_KERNEL_PAGES:
+		err = map_discontig_kernel_pages_complete(vma, action);
+		break;
 	case MMAP_IO_REMAP_PFN:
 	case MMAP_SIMPLE_IO_REMAP:
 		/* Should have been delegated. */
@@ -1509,7 +1586,8 @@ int mmap_action_prepare(struct vm_area_desc *desc)
 	case MMAP_REMAP_PFN:
 	case MMAP_IO_REMAP_PFN:
 	case MMAP_SIMPLE_IO_REMAP:
-	case MMAP_MAP_KERNEL_PAGES:
+	case MMAP_KERNEL_PAGES:
+	case MMAP_DISCONTIG_KERNEL_PAGES:
 		WARN_ON_ONCE(1); /* nommu cannot handle these. */
 		break;
 	}
@@ -1530,7 +1608,8 @@ int mmap_action_complete(struct vm_area_struct *vma,
 	case MMAP_REMAP_PFN:
 	case MMAP_IO_REMAP_PFN:
 	case MMAP_SIMPLE_IO_REMAP:
-	case MMAP_MAP_KERNEL_PAGES:
+	case MMAP_KERNEL_PAGES:
+	case MMAP_DISCONTIG_KERNEL_PAGES:
 		WARN_ON_ONCE(1); /* nommu cannot handle this. */
 
 		err = -EINVAL;
diff --git a/mm/vma.c b/mm/vma.c
index f29abb30956bb9..cfe31dbf0e03c6 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -24,7 +24,8 @@ struct mmap_state {
 		vm_flags_t vm_flags;
 		vma_flags_t vma_flags;
 	};
-	struct file *file;
+	struct file *file;	/* mmap()-specified file. */
+	struct file *vm_file;	/* May be updated by mmap_prepare. */
 	pgprot_t page_prot;
 
 	/* User-defined fields, perhaps updated by .mmap_prepare(). */
@@ -43,8 +44,6 @@ struct mmap_state {
 
 	/* Determine if we can check KSM flags early in mmap() logic. */
 	bool check_ksm_early :1;
-	/* If .mmap_prepare changed the file, we don't need to pin. */
-	bool file_doesnt_need_get :1;
 };
 
 #define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, anon_pgoff_, vma_flags_, file_) \
@@ -58,6 +57,7 @@ struct mmap_state {
 		.pglen = PHYS_PFN(len_),				\
 		.vma_flags = vma_flags_,				\
 		.file = file_,						\
+		.vm_file = file_,					\
 		.page_prot = vma_flags_to_page_prot(vma_flags_),	\
 	}
 
@@ -70,7 +70,7 @@ struct mmap_state {
 		.vma_flags = (map_)->vma_flags,				\
 		.pgoff = (map_)->pgoff,					\
 		.anon_pgoff = (map_)->anon_pgoff,			\
-		.file = (map_)->file,					\
+		.file = (map_)->vm_file,				\
 		.prev = (map_)->prev,					\
 		.middle = vma_,						\
 		.next = (vma_) ? NULL : (map_)->next,			\
@@ -97,10 +97,11 @@ static bool vma_is_fork_child(struct vm_area_struct *vma)
 {
 	/*
 	 * The list_is_singular() test is to avoid merging VMA cloned from
-	 * parents. This can improve scalability caused by the anon_vma root
+	 * parents. This can improve scalability caused by the anon rmap root
 	 * lock.
 	 */
-	return vma && vma->anon_vma && !list_is_singular(&vma->anon_vma_chain);
+	return vma && vma_has_anon_rmap(vma) &&
+		!list_is_singular(&vma->anon_vma_chain);
 }
 
 static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
@@ -134,13 +135,13 @@ static bool is_mergeable_anon_vma(struct vma_merge_struct *vmg, bool merge_next)
 
 	/*
 	 * We _can_ have !src, vmg->anon_vma via copy_vma(). In this instance we
-	 * will remove the existing VMA's anon_vma's so there's no scalability
+	 * will remove the existing VMA's anon rmap so there's no scalability
 	 * concerns.
 	 */
 	VM_WARN_ON(src && src_anon != src->anon_vma);
 
 	/* Case 1 - we will dup_anon_vma() from src into tgt. */
-	if (!tgt_anon && src_anon) {
+	if (!vma_has_anon_rmap(tgt) && src_anon) {
 		struct vm_area_struct *copied_from = vmg->copied_from;
 
 		if (vma_is_fork_child(src))
@@ -150,10 +151,10 @@ static bool is_mergeable_anon_vma(struct vma_merge_struct *vmg, bool merge_next)
 
 		return true;
 	}
-	/* Case 2 - we will simply use tgt's anon_vma. */
-	if (tgt_anon && !src_anon)
+	/* Case 2 - we will simply use tgt's anon rmap. */
+	if (vma_has_anon_rmap(tgt) && !src_anon)
 		return !vma_is_fork_child(tgt);
-	/* Case 3 - the anon_vma's are already shared. */
+	/* Case 3 - src and tgt already share an anon rmap. */
 	return src_anon == tgt_anon;
 }
 
@@ -190,18 +191,25 @@ static void init_multi_vma_prep(struct vma_prepare *vp,
 		adjust = NULL;
 
 	vp->adj_next = adjust;
-	if (!vp->anon_vma && adjust)
+	if (!vma_has_anon_rmap(vma) && adjust)
 		vp->anon_vma = adjust->anon_vma;
 
-	VM_WARN_ON(vp->anon_vma && adjust && adjust->anon_vma &&
+	VM_WARN_ON(vma_has_anon_rmap(vma) && adjust && vma_has_anon_rmap(adjust) &&
 		   vp->anon_vma != adjust->anon_vma);
 
 	vp->file = vma->vm_file;
 	if (vp->file)
 		vp->mapping = vma->vm_file->f_mapping;
 
-	if (vmg && vmg->skip_vma_uprobe)
+	if (!vmg)
+		return;
+
+	if (vmg->skip_vma_uprobe)
 		vp->skip_vma_uprobe = true;
+	if (vma_start_pgoff(vma) == vmg_start_pgoff(vmg))
+		vp->pgoff_unchanged = true;
+	if (vma_start_anon_pgoff(vma) == vmg_start_anon_pgoff(vmg))
+		vp->anon_pgoff_unchanged = true;
 }
 
 /*
@@ -227,8 +235,8 @@ static bool needs_adjacent_anon_pgoff(const struct vma_merge_struct *vmg)
  * Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff)
  * in front of (at a lower virtual address and file offset than) the vma.
  *
- * We cannot merge two vmas if they have differently assigned (non-NULL)
- * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
+ * We cannot merge two vmas if they have differently assigned anon rmaps,
+ * nor if the same anon rmap is assigned but offsets incompatible.
  *
  * We don't check here for the merged mmap wrapping around the end of pagecache
  * indices (16TB on ia32) because do_mmap() does not permit mmap's which
@@ -254,8 +262,8 @@ static bool can_vma_merge_before(struct vma_merge_struct *vmg)
  * Return true if we can merge this (vma_flags,anon_vma,file,vm_pgoff)
  * beyond (at a higher virtual address and file offset than) the vma.
  *
- * We cannot merge two vmas if they have differently assigned (non-NULL)
- * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
+ * We cannot merge two vmas if they have differently assigned anon rmaps,
+ * nor if the same anon rmap is assigned but offsets incompatible.
  *
  * We assume that vma is not removed as part of the merge.
  */
@@ -299,18 +307,18 @@ static void __remove_shared_vm_struct(struct vm_area_struct *vma,
 }
 
 /*
- * vma has some anon_vma assigned, and is already inserted on that
- * anon_vma's interval trees.
+ * vma has an anon rmap assigned, and is already inserted on its interval
+ * trees.
  *
  * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
- * vma must be removed from the anon_vma's interval trees using
+ * vma must be removed from the anon rmap's interval trees using
  * anon_rmap_tree_pre_update_vma().
  *
  * After the update, the vma will be reinserted using
  * anon_rmap_tree_post_update_vma().
  *
  * The entire update must be protected by exclusive mmap_lock and by
- * the root anon_vma's mutex.
+ * the anon rmap root lock.
  */
 static void
 anon_rmap_tree_pre_update_vma(struct vm_area_struct *vma)
@@ -330,6 +338,15 @@ anon_rmap_tree_post_update_vma(struct vm_area_struct *vma)
 		anon_rmap_tree_insert(avc, avc->anon_vma);
 }
 
+static void
+anon_rmap_tree_update_vma_inplace(struct vm_area_struct *vma)
+{
+	struct anon_vma_chain *avc;
+
+	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
+		anon_rmap_tree_update_inplace(avc);
+}
+
 /*
  * vma_prepare() - Helper function for handling locking VMAs prior to altering
  * @vp: The initialized vma_prepare struct
@@ -358,14 +375,16 @@ static void vma_prepare(struct vma_prepare *vp)
 
 	if (vp->anon_vma) {
 		anon_vma_lock_write(vp->anon_vma);
-		anon_rmap_tree_pre_update_vma(vp->vma);
+		if (!vp->anon_pgoff_unchanged)
+			anon_rmap_tree_pre_update_vma(vp->vma);
 		if (vp->adj_next)
 			anon_rmap_tree_pre_update_vma(vp->adj_next);
 	}
 
 	if (vp->file) {
 		flush_dcache_mmap_lock(vp->mapping);
-		mapping_rmap_tree_remove(vp->vma, vp->mapping);
+		if (!vp->pgoff_unchanged)
+			mapping_rmap_tree_remove(vp->vma, vp->mapping);
 		if (vp->adj_next)
 			mapping_rmap_tree_remove(vp->adj_next, vp->mapping);
 	}
@@ -386,7 +405,11 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
 	if (vp->file) {
 		if (vp->adj_next)
 			mapping_rmap_tree_insert(vp->adj_next, vp->mapping);
-		mapping_rmap_tree_insert(vp->vma, vp->mapping);
+		/* Need only propagate the change inplace. */
+		if (vp->pgoff_unchanged)
+			mapping_rmap_tree_update_inplace(vp->vma);
+		else
+			mapping_rmap_tree_insert(vp->vma, vp->mapping);
 		flush_dcache_mmap_unlock(vp->mapping);
 	}
 
@@ -405,7 +428,11 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
 	}
 
 	if (vp->anon_vma) {
-		anon_rmap_tree_post_update_vma(vp->vma);
+		/* Need only propagate the change inplace. */
+		if (vp->anon_pgoff_unchanged)
+			anon_rmap_tree_update_vma_inplace(vp->vma);
+		else
+			anon_rmap_tree_post_update_vma(vp->vma);
 		if (vp->adj_next)
 			anon_rmap_tree_post_update_vma(vp->adj_next);
 		anon_vma_unlock_write(vp->anon_vma);
@@ -430,7 +457,7 @@ static void vma_complete(struct vma_prepare *vp, struct vma_iterator *vmi,
 				      vp->remove->vm_end);
 			fput(vp->file);
 		}
-		if (vp->remove->anon_vma)
+		if (vma_has_anon_rmap(vp->remove))
 			unlink_anon_vmas(vp->remove);
 		mm->map_count--;
 		mpol_put(vma_policy(vp->remove));
@@ -478,7 +505,7 @@ static bool can_vma_merge_left(struct vma_merge_struct *vmg)
  * account the end position of the proposed range.
  *
  * In addition, if we can merge with the left VMA, ensure that left and right
- * anon_vma's are also compatible.
+ * anon rmaps are also compatible.
  */
 static bool can_vma_merge_right(struct vma_merge_struct *vmg,
 				bool can_merge_left)
@@ -494,13 +521,13 @@ static bool can_vma_merge_right(struct vma_merge_struct *vmg,
 
 	/*
 	 * If we can merge with prev (left) and next (right), indicating that
-	 * each VMA's anon_vma is compatible with the proposed anon_vma, this
+	 * each VMA's anon rmap is compatible with the proposed anon rmap, this
 	 * does not mean prev and next are compatible with EACH OTHER.
 	 *
 	 * We therefore check this in addition to mergeability to either side.
 	 */
 	prev = vmg->prev;
-	return !prev->anon_vma || !next->anon_vma ||
+	return !vma_has_anon_rmap(prev) || !vma_has_anon_rmap(next) ||
 		prev->anon_vma == next->anon_vma;
 }
 
@@ -592,6 +619,8 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 
 	init_vma_prep(&vp, vma);
 	vp.insert = new;
+	vp.pgoff_unchanged = !new_below;
+	vp.anon_pgoff_unchanged = !new_below;
 	vma_prepare(&vp);
 
 	/*
@@ -599,7 +628,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	 * boundary.
 	 */
 	vma_adjust_trans_huge(vma, vma->vm_start, addr, NULL);
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		hugetlb_split(vma, addr);
 
 	if (new_below) {
@@ -634,7 +663,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
  * Split a vma into two pieces at address 'addr', a new vma is allocated
  * either for the first part or the tail.
  */
-static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
+int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 		     unsigned long addr, int new_below)
 {
 	if (vma->vm_mm->map_count >= get_sysctl_max_map_count())
@@ -644,8 +673,8 @@ static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
 }
 
 /*
- * dup_anon_vma() - Helper function to duplicate anon_vma on VMA merge in the
- * instance that the destination VMA has no anon_vma but the source does.
+ * dup_anon_vma() - Helper function to duplicate the anon rmap on VMA merge in
+ * the instance that the destination VMA has no anon rmap but the source does.
  *
  * @dst: The destination VMA
  * @src: The source VMA
@@ -658,19 +687,19 @@ static int dup_anon_vma(struct vm_area_struct *dst,
 {
 	/*
 	 * There are three cases to consider for correctly propagating
-	 * anon_vma's on merge.
+	 * anon rmaps on merge.
 	 *
-	 * The first is trivial - neither VMA has anon_vma, we need not do
+	 * The first is trivial - neither VMA has an anon rmap, we need not do
 	 * anything.
 	 *
-	 * The second where both have anon_vma is also a no-op, as they must
+	 * The second where both have an anon rmap is also a no-op, as they must
 	 * then be the same, so there is simply nothing to copy.
 	 *
-	 * Here we cover the third - if the destination VMA has no anon_vma,
+	 * Here we cover the third - if the destination VMA has no anon rmap,
 	 * that is it is unfaulted, we need to ensure that the newly merged
-	 * range is referenced by the anon_vma's of the source.
+	 * range is referenced by the anon rmap of the source.
 	 */
-	if (src->anon_vma && !dst->anon_vma) {
+	if (vma_has_anon_rmap(src) && !vma_has_anon_rmap(dst)) {
 		int ret;
 
 		vma_assert_write_locked(dst);
@@ -720,7 +749,7 @@ void validate_mm(struct mm_struct *mm)
 		}
 
 #ifdef CONFIG_DEBUG_VM_RB
-		if (anon_vma) {
+		if (vma_has_anon_rmap(vma)) {
 			anon_vma_lock_read(anon_vma);
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
 				anon_rmap_tree_verify(avc);
@@ -924,13 +953,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
 
 	vmg->state = VMA_MERGE_NOMERGE;
 
+	if (!vma_flags_can_merge(&vmg->vma_flags))
+		return NULL;
 	/*
-	 * If a special mapping or if the range being modified is neither at the
-	 * furthermost left or right side of the VMA, then we have no chance of
-	 * merging and should abort.
+	 * If the range being modified is neither at the furthermost left or
+	 * right side of the VMA, then we have no chance of merging and should
+	 * abort.
 	 */
-	if (vma_flags_test_any_mask(&vmg->vma_flags, VMA_SPECIAL_FLAGS) ||
-	    (!left_side && !right_side))
+	if (!left_side && !right_side)
 		return NULL;
 
 	if (left_side)
@@ -1015,11 +1045,11 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
 		vmg->anon_pgoff = vma_start_anon_pgoff(prev);
 
 		/*
-		 * We already ensured anon_vma compatibility above, so now it's
-		 * simply a case of, if prev has no anon_vma object, which of
-		 * next or middle contains the anon_vma we must duplicate.
+		 * We already ensured anon rmap compatibility above, so now it's
+		 * simply a case of, if prev has no anon rmap, which of next or
+		 * middle contains the anon rmap we must duplicate.
 		 */
-		err = dup_anon_vma(prev, next->anon_vma ? next : middle,
+		err = dup_anon_vma(prev, vma_has_anon_rmap(next) ? next : middle,
 				   &anon_dup);
 	} else if (merge_left) {
 		/*
@@ -1083,7 +1113,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
 		unlink_anon_vmas(anon_dup);
 
 	/*
-	 * This means we have failed to clone anon_vma's correctly, but no
+	 * This means we have failed to clone the anon rmap correctly, but no
 	 * actual changes to VMAs have occurred, so no harm no foul - if the
 	 * user doesn't want this reported and instead just wants to give up on
 	 * the merge, allow it.
@@ -1152,9 +1182,11 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg)
 
 	vmg->state = VMA_MERGE_NOMERGE;
 
-	/* Special VMAs are unmergeable, also if no prev/next. */
-	if (vma_flags_test_any_mask(&vmg->vma_flags, VMA_SPECIAL_FLAGS) ||
-	    (!prev && !next))
+	if (!vma_flags_can_merge(&vmg->vma_flags))
+		return NULL;
+
+	/* VMAs with no prev/next are unmergeable. */
+	if (!prev && !next)
 		return NULL;
 
 	can_merge_left = can_vma_merge_left(vmg);
@@ -1278,7 +1310,7 @@ int vma_expand(struct vma_merge_struct *vmg)
 
 	/*
 	 * If we are removing the next VMA or copying from a VMA
-	 * (e.g. mremap()'ing), we must propagate anon_vma state.
+	 * (e.g. mremap()'ing), we must propagate anon rmap state.
 	 *
 	 * Note that, by convention, callers ignore OOM for this case, so
 	 * we don't need to account for vmg->give_up_on_mm here.
@@ -1342,6 +1374,8 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
 	vma_start_write(vma);
 
 	init_vma_prep(&vp, vma);
+	vp.pgoff_unchanged = true;
+	vp.anon_pgoff_unchanged = true;
 	vma_prepare(&vp);
 	vma_adjust_trans_huge(vma, vma->vm_start, end, NULL);
 
@@ -1943,7 +1977,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
  */
 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 	unsigned long addr, unsigned long len, pgoff_t pgoff,
-	pgoff_t anon_pgoff, bool *need_rmap_locks)
+	pgoff_t anon_pgoff, bool *need_rmap_locks, bool keep_source)
 {
 	struct vm_area_struct *vma = *vmap;
 	unsigned long old_vma_start = vma->vm_start;
@@ -1957,7 +1991,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 	 * If a vma has not yet been faulted, update its anonymous pgoff to
 	 * match the new location to increase its chance of merging.
 	 */
-	if (!vma->anon_vma) {
+	if (!vma_has_anon_rmap(vma)) {
 		anon_pgoff = addr >> PAGE_SHIFT;
 
 		if (vma_is_anonymous(vma)) {
@@ -1981,6 +2015,21 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 	vmg.pgoff = pgoff;
 	vmg.anon_pgoff = anon_pgoff;
 	vmg.next = vma_iter_next_rewind(&vmi, NULL);
+
+	/*
+	 * If the original VMA is kept (MREMAP_DONTUNMAP), the source and
+	 * destination VMA must be treated distinctly.
+	 *
+	 * A merge violates this, so in this case disallow a self-merge.
+	 */
+	if (can_self_merge && keep_source) {
+		if (vmg.prev == vma)
+			vmg.prev = NULL;
+		if (vmg.next == vma)
+			vmg.next = NULL;
+		can_self_merge = false;
+	}
+
 	new_vma = vma_merge_copied_range(&vmg);
 
 	if (new_vma) {
@@ -2040,16 +2089,16 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 
 /*
  * Rough compatibility check to quickly see if it's even worth looking
- * at sharing an anon_vma.
+ * at sharing an anon rmap.
  *
  * They need to have the same vm_file, and the flags can only differ
  * in things that mprotect may change.
  *
- * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
+ * NOTE! The fact that we share an anon rmap doesn't _have_ to mean that
  * we can merge the two vma's. For example, we refuse to merge a vma if
  * there is a vm_ops->close() function, because that indicates that the
  * driver is doing some kind of reference counting. But that doesn't
- * really matter for the anon_vma sharing case.
+ * really matter for the anon rmap sharing case.
  */
 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
 {
@@ -2082,21 +2131,28 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *
 }
 
 /*
- * Do some basic sanity checking to see if we can re-use the anon_vma
+ * Do some basic sanity checking to see if we can re-use the anon rmap
  * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
  * the same as 'old', the other will be the new one that is trying
- * to share the anon_vma.
+ * to share the anon rmap.
  *
  * NOTE! This runs with mmap_lock held for reading, so it is possible that
- * the anon_vma of 'old' is concurrently in the process of being set up
+ * the anon rmap of 'old' is concurrently in the process of being set up
  * by another page fault trying to merge _that_. But that's ok: if it
  * is being set up, that automatically means that it will be a singleton
  * acceptable for merging, so we can do all of this optimistically. But
  * we do that READ_ONCE() to make sure that we never re-load the pointer.
  *
+ * The READ_ONCE() establishes an address dependency between anon_vma and
+ * any access to its fields, which pairs with the assignment to
+ * vma->anon_vma performed with release semantics in __anon_vma_prepare().
+ *
+ * This is especially important as anon_vma's are SLAB_TYPESAFE_BY_RCU so
+ * accessing an uninitialised anon_vma's fields may result in a UAF.
+ *
  * IOW: that the "list_is_singular()" test on the anon_vma_chain only
- * matters for the 'stable anon_vma' case (ie the thing we want to avoid
- * is to return an anon_vma that is "complex" due to having gone through
+ * matters for the 'stable anon rmap' case (ie the thing we want to avoid
+ * is to return an anon rmap that is "complex" due to having gone through
  * a fork).
  *
  * We also make sure that the two vma's are compatible (adjacent,
@@ -2108,6 +2164,7 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old,
 					  struct vm_area_struct *b)
 {
 	if (anon_vma_compatible(a, b)) {
+		/* Paired with a memory barrier in __anon_vma_prepare(). */
 		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
 
 		if (anon_vma && list_is_singular(&old->anon_vma_chain))
@@ -2118,10 +2175,10 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old,
 
 /*
  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
- * neighbouring vmas for a suitable anon_vma, before it goes off
- * to allocate a new anon_vma.  It checks because a repetitive
+ * neighbouring vmas for a suitable anon rmap, before it goes off
+ * to allocate a new anon rmap.  It checks because a repetitive
  * sequence of mprotects and faults may otherwise lead to distinct
- * anon_vmas being allocated, preventing vma merge in subsequent
+ * anon rmaps being allocated, preventing vma merge in subsequent
  * mprotect.
  */
 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
@@ -2147,13 +2204,13 @@ struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
 
 	/*
 	 * We might reach here with anon_vma == NULL if we can't find
-	 * any reusable anon_vma.
+	 * any reusable anon rmap.
 	 * There's no absolute need to look only at touching neighbours:
-	 * we could search further afield for "compatible" anon_vmas.
+	 * we could search further afield for "compatible" anon rmaps.
 	 * But it would probably just be a waste of time searching,
-	 * or lead to too many vmas hanging off the same anon_vma.
+	 * or lead to too many vmas hanging off the same anon rmap.
 	 * We're trying to allow mprotect remerging later on,
-	 * not trying to minimize memory used for anon_vmas.
+	 * not trying to minimize memory used for anon rmaps.
 	 */
 	return anon_vma;
 }
@@ -2225,7 +2282,7 @@ bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
 	 * Do we need to track softdirty? hugetlb does not support softdirty
 	 * tracking yet.
 	 */
-	if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
+	if (vma_soft_dirty_enabled(vma) && !vma_is_hugetlb(vma))
 		return true;
 
 	/* Do we need write faults for uffd-wp tracking? */
@@ -2249,7 +2306,7 @@ static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
 		/*
 		 * We can safely modify head.next after taking the
 		 * anon_vma->root->rwsem. If some other vma in this mm shares
-		 * the same anon_vma we won't take it again.
+		 * the same anon rmap we won't take it again.
 		 *
 		 * No need of atomic instructions here, head.next
 		 * can't change from under us thanks to the
@@ -2291,14 +2348,14 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
  * mmap_lock in write mode is required in order to block all operations
  * that could modify pagetables and free pages without need of
  * altering the vma layout. It's also needed in write mode to avoid new
- * anon_vmas to be associated with existing vmas.
+ * anon rmaps being associated with existing vmas.
  *
  * A single task can't take more than one mm_take_all_locks() in a row
  * or it would deadlock.
  *
  * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
  * mapping->flags avoid to take the same lock twice, if more than one
- * vma in this mm is backed by the same anon_vma or address_space.
+ * vma in this mm is backed by the same anon rmap or address_space.
  *
  * We take locks in following order, accordingly to comment at beginning
  * of mm/rmap.c:
@@ -2344,7 +2401,7 @@ int mm_take_all_locks(struct mm_struct *mm)
 		if (signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
-				is_vm_hugetlb_page(vma))
+				vma_is_hugetlb(vma))
 			vm_lock_mapping(mm, vma->vm_file->f_mapping);
 	}
 
@@ -2353,7 +2410,7 @@ int mm_take_all_locks(struct mm_struct *mm)
 		if (signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
-				!is_vm_hugetlb_page(vma))
+				!vma_is_hugetlb(vma))
 			vm_lock_mapping(mm, vma->vm_file->f_mapping);
 	}
 
@@ -2361,7 +2418,7 @@ int mm_take_all_locks(struct mm_struct *mm)
 	for_each_vma(vmi, vma) {
 		if (signal_pending(current))
 			goto out_unlock;
-		if (vma->anon_vma)
+		if (vma_has_anon_rmap(vma))
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
 				vm_lock_anon_vma(mm, avc->anon_vma);
 	}
@@ -2423,7 +2480,7 @@ void mm_drop_all_locks(struct mm_struct *mm)
 	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
 
 	for_each_vma(vmi, vma) {
-		if (vma->anon_vma)
+		if (vma_has_anon_rmap(vma))
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
 				vm_unlock_anon_vma(avc->anon_vma);
 		if (vma->vm_file && vma->vm_file->f_mapping)
@@ -2439,7 +2496,7 @@ void mm_drop_all_locks(struct mm_struct *mm)
  */
 static bool accountable_mapping(struct mmap_state *map)
 {
-	const struct file *file = map->file;
+	const struct file *file = map->vm_file;
 
 	/*
 	 * hugetlb has its own accounting separate from the core VM
@@ -2488,7 +2545,7 @@ static void vms_abort_munmap_vmas(struct vma_munmap_struct *vms,
 
 static void update_ksm_flags(struct mmap_state *map)
 {
-	map->vma_flags = ksm_vma_flags(map->mm, map->file, map->vma_flags);
+	map->vma_flags = ksm_vma_flags(map->mm, map->vm_file, map->vma_flags);
 }
 
 static void set_desc_from_map(struct vm_area_desc *desc,
@@ -2498,7 +2555,7 @@ static void set_desc_from_map(struct vm_area_desc *desc,
 	desc->end = map->end;
 
 	desc->pgoff = map->pgoff;
-	desc->vm_file = map->file;
+	desc->vm_file = map->vm_file;
 	desc->vma_flags = map->vma_flags;
 	desc->page_prot = map->page_prot;
 }
@@ -2578,6 +2635,10 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc,
 	return 0;
 }
 
+static bool map_same_file(struct mmap_state *map)
+{
+	return map->vm_file == map->file;
+}
 
 static int __mmap_new_file_vma(struct mmap_state *map,
 			       struct vm_area_struct *vma)
@@ -2585,42 +2646,65 @@ static int __mmap_new_file_vma(struct mmap_state *map,
 	struct vma_iterator *vmi = map->vmi;
 	int error;
 
-	vma->vm_file = map->file;
-	if (!map->file_doesnt_need_get)
-		get_file(map->file);
+	vma->vm_file = map->vm_file;
+	if (map_same_file(map))
+		get_file(map->vm_file);
 
-	if (!map->file->f_op->mmap)
+	if (!map->vm_file->f_op->mmap)
 		return 0;
 
+	/*
+	 * Driver-specified flags may make the lock flags invalid, so clear
+	 * VMA_LOCKED_MASK and reinstate it afterwards if appropriate.
+	 */
+	vma_clear_flags_mask(vma, VMA_LOCKED_MASK);
 	error = mmap_file(vma->vm_file, vma);
+	map->vm_file = vma->vm_file;
+
 	if (error) {
 		UNMAP_STATE(unmap, vmi, vma, vma->vm_start, vma->vm_end,
 			    map->prev, map->next);
-		fput(vma->vm_file);
-		vma->vm_file = NULL;
+		if (map_same_file(map))
+			fput(map->vm_file);
 
+		vma->vm_file = NULL;
 		vma_iter_set(vmi, vma->vm_end);
 		/* Undo any partial mapping done by a device driver. */
 		unmap_region(&unmap);
 		return error;
 	}
 
-	/* Drivers cannot alter the address of the VMA. */
-	WARN_ON_ONCE(map->addr != vma->vm_start);
-	/*
-	 * Drivers should not permit writability when previously it was
-	 * disallowed.
-	 */
-	VM_WARN_ON_ONCE(!vma_flags_same_pair(&map->vma_flags, &vma->flags) &&
-			!vma_flags_test(&map->vma_flags, VMA_MAYWRITE_BIT) &&
-			vma_test(vma, VMA_MAYWRITE_BIT));
+	/* If VMA flags still valid for locked mask, reinstate. */
+	if (vma_supports_mlock(vma)) {
+		const vma_flags_t mask =
+			vma_flags_and_mask(&map->vma_flags,
+					   VMA_LOCKED_MASK);
+
+		vma_set_flags_mask(vma, mask);
+	}
 
-	map->file = vma->vm_file;
 	map->vma_flags = vma->flags;
 
 	return 0;
 }
 
+static void map_set_anon(struct mmap_state *map)
+{
+	map->vm_file = NULL;
+	map->vm_ops = NULL;
+	map->pgoff = map->addr >> PAGE_SHIFT;
+}
+
+static bool map_is_private(const struct mmap_state *map)
+{
+	return !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT);
+}
+
+static bool map_is_anon(const struct mmap_state *map)
+{
+	return map_is_private(map) && !map->vm_file;
+}
+
 /*
  * __mmap_new_vma() - Allocate a new VMA for the region, as merging was not
  * possible.
@@ -2634,8 +2718,7 @@ static int __mmap_new_file_vma(struct mmap_state *map,
 static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap,
 	struct mmap_action *action)
 {
-	const bool is_anon = !map->file &&
-		!vma_flags_test(&map->vma_flags, VMA_SHARED_BIT);
+	const bool is_anon = map_is_anon(map);
 	struct vma_iterator *vmi = map->vmi;
 	int error = 0;
 	struct vm_area_struct *vma;
@@ -2664,7 +2747,7 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap,
 	}
 
 	/* Invoke callbacks. */
-	if (map->file)
+	if (map->vm_file)
 		error = __mmap_new_file_vma(map, vma);
 	else if (!is_anon)
 		error = shmem_zero_setup(vma);
@@ -2677,11 +2760,6 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap,
 		vma->flags = map->vma_flags;
 	}
 
-#ifdef CONFIG_SPARC64
-	/* TODO: Fix SPARC ADI! */
-	WARN_ON_ONCE(!arch_validate_flags(map->vm_flags));
-#endif
-
 	/* Lock the VMA since it is modified after insertion into VMA tree */
 	vma_start_write(vma);
 	vma_iter_store_new(vmi, vma);
@@ -2744,6 +2822,100 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma)
 	vma_set_page_prot(vma);
 }
 
+/* Check to ensure that the VMA flags of a newly mapped VMA are sane. */
+static int mmap_validate_vma_flags(const vma_flags_t *flags)
+{
+#ifdef CONFIG_SPARC64
+	const vm_flags_t legacy_flags = vma_flags_to_legacy(*flags);
+
+	/* TODO: Fix SPARC ADI! */
+	if (WARN_ON_ONCE(!arch_validate_flags(legacy_flags)))
+		return -EINVAL;
+#endif
+
+	if (!vma_flags_is_kernel_owned(flags)) {
+		/* Only kernel-owned mappings may set VMA_IO_BIT. */
+		if (WARN_ON_ONCE(vma_flags_test(flags, VMA_IO_BIT)))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+/* Check to ensure a driver hasn't done something crazy. */
+static int mmap_validate(unsigned long prev_start, unsigned long prev_end,
+			 unsigned long curr_start, unsigned long curr_end,
+			 const vma_flags_t *prev_flags,
+			 const vma_flags_t *curr_flags)
+{
+	bool was_maywrite, is_maywrite;
+
+	/* Drivers cannot alter the range of the VMA. */
+	if (WARN_ON_ONCE(prev_start != curr_start || prev_end != curr_end))
+		return -EINVAL;
+
+	was_maywrite = vma_flags_test(prev_flags, VMA_MAYWRITE_BIT);
+	is_maywrite = vma_flags_test(curr_flags, VMA_MAYWRITE_BIT);
+
+	/* A driver may not make a previously unwritable mapping writable. */
+	if (WARN_ON_ONCE(!was_maywrite && is_maywrite))
+		return -EINVAL;
+
+	/* Only kernel-owned mappings may clear VMA_MAYWRITE_BIT. */
+	if (!vma_flags_is_kernel_owned(curr_flags) &&
+	    WARN_ON_ONCE(was_maywrite && !is_maywrite))
+		return -EINVAL;
+
+	return mmap_validate_vma_flags(curr_flags);
+}
+
+/**
+ * mmap_prepare_validate() - Ensure the driver hasn't violated invariants in its
+ * f_op->mmap_prepare hook.
+ * @prev_desc: The VMA descriptor prior to the mmap_prepare hook being called.
+ * @desc: The VMA descriptor after the mmap_prepare hook has been called.
+ *
+ * Returns: 0 on success, otherwise an error.
+ */
+int mmap_prepare_validate(const struct vm_area_desc *prev_desc,
+			  const struct vm_area_desc *desc)
+{
+	/*
+	 * It is not valid to execute mmap actions for VMAs which can be merged,
+	 * as any such merge would leave portions of the mapping incorrectly
+	 * unmapped.
+	 */
+	if (vma_flags_can_merge(&desc->vma_flags) &&
+	    WARN_ON_ONCE(desc->action.type != MMAP_NOTHING))
+		return -EINVAL;
+
+	return mmap_validate(prev_desc->start, prev_desc->end,
+			     desc->start, desc->end,
+			     &prev_desc->vma_flags, &desc->vma_flags);
+}
+
+/**
+ * mmap_hook_validate() - Ensure the driver hasn't violated invariants in
+ * its f_op->mmap hook.
+ * @prev_start: The start of the mapping prior to the mmap hook.
+ * @prev_end: The end of the mapping prior to the mmap hook.
+ * @prev_flags: The VMA flags set for the VMA prior to the mmap hook.
+ * @vma: The VMA after the hook has been applied.
+ *
+ * Returns: 0 on success, otherwise an error.
+ */
+int mmap_hook_validate(unsigned long prev_start, unsigned long prev_end,
+		       const vma_flags_t *prev_flags,
+		       const struct vm_area_struct *vma)
+{
+	const unsigned long start = vma->vm_start;
+	const unsigned long end = vma->vm_end;
+	const vma_flags_t *flags = &vma->flags;
+
+	return mmap_validate(prev_start, prev_end, start, end, prev_flags,
+			     flags);
+}
+
 static int call_action_prepare(struct mmap_state *map,
 			       struct vm_area_desc *desc)
 {
@@ -2770,39 +2942,54 @@ static int call_action_prepare(struct mmap_state *map,
 static int call_mmap_prepare(struct mmap_state *map,
 		struct vm_area_desc *desc)
 {
+	const struct vm_area_desc prev_desc = *desc;
 	int err;
 
 	/* Invoke the hook. */
-	err = vfs_mmap_prepare(map->file, desc);
+	err = vfs_mmap_prepare(map->vm_file, desc);
 	if (err)
 		return err;
 
+	/* Update first so file refcount tracked correctly. */
+	if (desc->vm_file != map->vm_file)
+		map->vm_file = desc->vm_file;
+
+	/* It's invalid for mmap_prepare hooks to clear vm_ops. */
+	if (!desc->vm_ops)
+		return -EINVAL;
+
 	err = call_action_prepare(map, desc);
 	if (err)
 		return err;
 
+	/* Check the caller did nothing crazy. */
+	err = mmap_prepare_validate(&prev_desc, desc);
+	if (err)
+		return err;
+
 	/* Update fields permitted to be changed. */
 	map->pgoff = desc->pgoff;
-	if (desc->vm_file != map->file) {
-		map->file_doesnt_need_get = true;
-		map->file = desc->vm_file;
-	}
 	map->vma_flags = desc->vma_flags;
 	map->page_prot = desc->page_prot;
 	/* User-defined fields. */
 	map->vm_ops = desc->vm_ops;
 	map->vm_private_data = desc->private_data;
 
+	/*
+	 * MAP_PRIVATE-/dev/zero mappings are an ancient way of getting
+	 * anonymous mappings. Rather than allowing these mappings to be odd
+	 * outliers, simply make them truly anonymous.
+	 */
+	if (map_is_private(map) && file_is_dev_zero(map->vm_file))
+		map_set_anon(map);
+
 	return 0;
 }
 
 static void set_vma_user_defined_fields(struct vm_area_struct *vma,
 		struct mmap_state *map)
 {
-	if (map->vm_ops)
-		vma->vm_ops = map->vm_ops;
-	else	/* Only /dev/zero should do this. */
-		vma_set_anonymous(vma);
+	vma->vm_ops = map->vm_ops;
 	vma->vm_private_data = map->vm_private_data;
 }
 
@@ -2812,7 +2999,7 @@ static void set_vma_user_defined_fields(struct vm_area_struct *vma,
  */
 static bool can_set_ksm_flags_early(struct mmap_state *map)
 {
-	struct file *file = map->file;
+	struct file *file = map->vm_file;
 
 	/* Anonymous mappings have no driver which can change them. */
 	if (!file)
@@ -2835,13 +3022,27 @@ static bool can_set_ksm_flags_early(struct mmap_state *map)
 	return false;
 }
 
+static void put_map(struct mmap_state *map)
+{
+	/*
+	 * An error occurred or the VMA was merged.
+	 *
+	 * If the file was changed by the driver (which is required to increment
+	 * the replacement file's reference count), drop its reference count.
+	 *
+	 * On error, the caller always drops the original file regardless.
+	 */
+	if (map->vm_file && !map_same_file(map))
+		fput(map->vm_file);
+}
+
 static unsigned long __mmap_region(struct file *file, unsigned long addr,
 		unsigned long len, vma_flags_t vma_flags,
 		unsigned long pgoff, struct list_head *uf)
 {
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma = NULL;
-	bool have_mmap_prepare = file && file->f_op->mmap_prepare;
+	const bool have_mmap_prepare = file && file->f_op->mmap_prepare;
 	VMA_ITERATOR(vmi, mm, addr);
 	const pgoff_t anon_pgoff = addr >> PAGE_SHIFT;
 	MMAP_STATE(map, mm, &vmi, addr, len, pgoff, anon_pgoff, vma_flags, file);
@@ -2884,12 +3085,15 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
 		allocated_new = true;
 	}
 
-	if (have_mmap_prepare)
+	if (have_mmap_prepare && !map_is_anon(&map) && allocated_new)
 		set_vma_user_defined_fields(vma, &map);
 
 	__mmap_complete(&map, vma);
 
-	if (have_mmap_prepare && allocated_new) {
+	if (!allocated_new) {
+		/* Merged, so need to drop refcount. */
+		put_map(&map);
+	} else if (have_mmap_prepare) {
 		error = mmap_action_complete(vma, &desc.action,
 					     /*is_compat=*/false);
 		if (error)
@@ -2903,13 +3107,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
 	if (map.charged)
 		vm_unacct_memory(map.charged);
 abort_munmap:
-	/*
-	 * This indicates that .mmap_prepare has set a new file, differing from
-	 * desc->vm_file. But since we're aborting the operation, only the
-	 * original file will be cleaned up. Ensure we clean up both.
-	 */
-	if (map.file_doesnt_need_get)
-		fput(map.file);
+	put_map(&map);
 	vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
 	return error;
 }
@@ -3251,7 +3449,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
 	if (next && vma_is_accessible(next)) {
 		if (!vma_test(next, VMA_GROWSUP_BIT))
 			return -ENOMEM;
-		/* Check that both stack segments have the same anon_vma? */
+		/* Check that both stack segments have the same anon rmap? */
 	}
 
 	if (next)
@@ -3261,7 +3459,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
 	if (vma_iter_prealloc(&vmi, vma))
 		return -ENOMEM;
 
-	/* We must make sure the anon_vma is allocated. */
+	/* We must make sure the anon rmap is allocated. */
 	if (unlikely(anon_vma_prepare(vma))) {
 		vma_iter_free(&vmi);
 		return -ENOMEM;
@@ -3324,7 +3522,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address)
 
 	/* Enforce stack_guard_gap */
 	prev = vma_prev(&vmi);
-	/* Check that both stack segments have the same anon_vma? */
+	/* Check that both stack segments have the same anon rmap? */
 	if (prev) {
 		if (!vma_test(prev, VMA_GROWSDOWN_BIT) &&
 		    vma_is_accessible(prev) &&
@@ -3339,7 +3537,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address)
 	if (vma_iter_prealloc(&vmi, vma))
 		return -ENOMEM;
 
-	/* We must make sure the anon_vma is allocated. */
+	/* We must make sure the anon rmap is allocated. */
 	if (unlikely(anon_vma_prepare(vma))) {
 		vma_iter_free(&vmi);
 		return -ENOMEM;
@@ -3404,17 +3602,22 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock)
 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
 {
 	unsigned long charged = vma_pages(vma);
+	int err;
 
 	if (find_vma_intersection(mm, vma->vm_start, vma->vm_end))
 		return -ENOMEM;
 
+	err = mmap_validate_vma_flags(&vma->flags);
+	if (err)
+		return err;
+
 	if (vma_test(vma, VMA_ACCOUNT_BIT) &&
 	     security_vm_enough_memory_mm(mm, charged))
 		return -ENOMEM;
 
 	/*
 	 * The vm_pgoff of a purely anonymous vma should be irrelevant
-	 * until its first write fault, when page's anon_vma and index
+	 * until its first write fault, when page's anon rmap and index
 	 * are set.  But now set the vm_pgoff it will almost certainly
 	 * end up with (unless mremap moves it elsewhere before that
 	 * first wfault), so /proc/pid/maps tells a consistent story.
@@ -3425,7 +3628,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
 	 * Similarly in do_mmap and in do_brk_flags.
 	 */
 	if (vma_is_anonymous(vma)) {
-		WARN_ON_ONCE(vma->anon_vma);
+		WARN_ON_ONCE(vma_has_anon_rmap(vma));
 		vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT);
 	}
 	vma_set_anon_pgoff(vma, vma->vm_start >> PAGE_SHIFT);
diff --git a/mm/vma.h b/mm/vma.h
index 024fabe63560a0..336b4ced82c918 100644
--- a/mm/vma.h
+++ b/mm/vma.h
@@ -28,6 +28,8 @@ struct vma_prepare {
 	struct vm_area_struct *remove2;
 
 	bool skip_vma_uprobe :1;
+	bool pgoff_unchanged :1;
+	bool anon_pgoff_unchanged :1;
 };
 
 struct unlink_vma_file_batch {
@@ -255,6 +257,33 @@ static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg)
 	return vmg_start_pgoff(vmg) + vmg_pages(vmg);
 }
 
+/**
+ * vma_has_anon_rmap() - does @vma possess an anonymous reverse mapping?
+ * @vma: The VMA to be checked.
+ *
+ * If the VMA is attached, a VMA or mmap lock must be held.
+ *
+ * This state is only possible for CoW mappings, see the comment for
+ * vma_flags_is_cow_mapping() for details.
+ *
+ * Importantly, a VMA which possesses an anonymous rmap may map anonymous
+ * folios.
+ *
+ * This function will not result in a false positive.
+ *
+ * However, if only a read lock is held, it may give a false negative, in which
+ * case it should be re-checked with mm->page_table_lock held.
+ *
+ * Returns: true if @vma has an anonymous reverse mapping, otherwise false.
+ */
+static inline bool vma_has_anon_rmap(const struct vm_area_struct *vma)
+{
+	if (vma_is_attached(vma))
+		vma_assert_stabilised(vma);
+	/* KCSAN gets confused about the optimistic check. Silence it. */
+	return data_race(vma->anon_vma);
+}
+
 static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
 {
 	/* nommu doesn't set a virtual pgoff for anon VMAs. */
@@ -267,11 +296,8 @@ static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff)
 	 */
 	if (!vma_is_anonymous(vma))
 		return;
-	/* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */
-	if (vma->vm_file)
-		return;
 	/* If faulted in, could have been remapped. */
-	if (vma->anon_vma)
+	if (vma_has_anon_rmap(vma))
 		return;
 	/* OK this is really an anon VMA - expect virtual page offset. */
 	VM_WARN_ON_ONCE(pgoff != vma->vm_start >> PAGE_SHIFT);
@@ -397,8 +423,10 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma,
 
 	/* Mutable fields. Populated with initial state. */
 	vma_set_pgoff(vma, desc->pgoff);
-	if (desc->vm_file != vma->vm_file)
-		vma_set_file(vma, desc->vm_file);
+	if (desc->vm_file != vma->vm_file) {
+		fput(vma->vm_file);
+		vma->vm_file = desc->vm_file;
+	}
 	vma->flags = desc->vma_flags;
 	vma->vm_page_prot = desc->page_prot;
 
@@ -535,7 +563,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
 
 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
 	unsigned long addr, unsigned long len, pgoff_t pgoff,
-	pgoff_t anon_pgoff, bool *need_rmap_locks);
+	pgoff_t anon_pgoff, bool *need_rmap_locks, bool keep_source);
 
 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
 
@@ -556,6 +584,11 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma,
 unsigned long unmapped_area(struct vm_unmapped_area_info *info);
 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
 
+#ifdef CONFIG_MMU
+int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
+	      unsigned long addr, int new_below);
+#endif
+
 static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
 {
 	/*
@@ -783,14 +816,19 @@ struct vm_area_struct *vm_area_alloc(struct mm_struct *mm);
 struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig);
 void vm_area_free(struct vm_area_struct *vma);
 
-/* vma_exec.c */
 #ifdef CONFIG_MMU
+int mmap_prepare_validate(const struct vm_area_desc *prev_desc,
+			  const struct vm_area_desc *desc);
+
+int mmap_hook_validate(unsigned long prev_start, unsigned long prev_end,
+		       const vma_flags_t *prev_flags,
+		       const struct vm_area_struct *vma);
+
+/* vma_exec.c */
 int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap,
 			  unsigned long *top_mem_p);
 int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift);
-#endif
 
-#ifdef CONFIG_MMU
 /*
  * Denies creating a writable executable mapping or gaining executable permissions.
  *
@@ -839,6 +877,20 @@ static inline bool map_deny_write_exec(const vma_flags_t *old,
 
 	return false;
 }
+#else
+static inline int mmap_prepare_validate(const struct vm_area_desc *prev_desc,
+					const struct vm_area_desc *desc)
+{
+	return 0;
+}
+
+static inline int mmap_hook_validate(unsigned long prev_start,
+				     unsigned long prev_end,
+				     const vma_flags_t *prev_flags,
+				     const struct vm_area_struct *vma)
+{
+	return 0;
+}
 #endif
 
 struct vm_area_struct *__install_special_mapping(struct mm_struct *mm,
diff --git a/mm/vma_internal.h b/mm/vma_internal.h
index 4d300e7bbaf4c2..4f73f0a4db796b 100644
--- a/mm/vma_internal.h
+++ b/mm/vma_internal.h
@@ -18,7 +18,6 @@
 #include <linux/fs.h>
 #include <linux/huge_mm.h>
 #include <linux/hugetlb.h>
-#include <linux/hugetlb_inline.h>
 #include <linux/kernel.h>
 #include <linux/ksm.h>
 #include <linux/khugepaged.h>
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index bea9f76ed7e742..a9fce7efe4f496 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -972,6 +972,7 @@ static struct vmap_node {
 	struct list_head purge_list;
 	struct work_struct purge_work;
 	unsigned long nr_purged;
+	bool work_queued;
 } single;
 
 /*
@@ -1089,7 +1090,14 @@ RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb,
 static void reclaim_and_purge_vmap_areas(void);
 static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
 static void drain_vmap_area_work(struct work_struct *work);
-static DECLARE_WORK(drain_vmap_work, drain_vmap_area_work);
+/*
+ * Keep the work item, whose pending bit is updated by freeing CPUs,
+ * away from vmap metadata read by allocation and free paths.
+ */
+static __cacheline_aligned_in_smp
+DECLARE_WORK(drain_vmap_work, drain_vmap_area_work);
+static struct workqueue_struct *drain_vmap_helpers_wq;
+static struct workqueue_struct *drain_vmap_wq;
 
 static __cacheline_aligned_in_smp atomic_long_t vmap_lazy_nr;
 
@@ -2351,6 +2359,16 @@ static void purge_vmap_node(struct work_struct *work)
 	reclaim_list_global(&local_list);
 }
 
+static bool
+schedule_drain_vmap_work(struct workqueue_struct *wq,
+		struct work_struct *work)
+{
+	if (wq)
+		return queue_work(wq, work);
+
+	return false;
+}
+
 /*
  * Purges all lazily-freed vmap areas.
  */
@@ -2358,19 +2376,12 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
 		bool full_pool_decay)
 {
 	unsigned long nr_purged_areas = 0;
+	unsigned int nr_purge_nodes = 0;
 	unsigned int nr_purge_helpers;
-	static cpumask_t purge_nodes;
-	unsigned int nr_purge_nodes;
 	struct vmap_node *vn;
-	int i;
 
 	lockdep_assert_held(&vmap_purge_lock);
 
-	/*
-	 * Use cpumask to mark which node has to be processed.
-	 */
-	purge_nodes = CPU_MASK_NONE;
-
 	for_each_vmap_node(vn) {
 		INIT_LIST_HEAD(&vn->purge_list);
 		vn->skip_populate = full_pool_decay;
@@ -2390,10 +2401,9 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
 		end = max(end, list_last_entry(&vn->purge_list,
 			struct vmap_area, list)->va_end);
 
-		cpumask_set_cpu(node_to_id(vn), &purge_nodes);
+		nr_purge_nodes++;
 	}
 
-	nr_purge_nodes = cpumask_weight(&purge_nodes);
 	if (nr_purge_nodes > 0) {
 		flush_tlb_kernel_range(start, end);
 
@@ -2401,29 +2411,31 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end,
 		nr_purge_helpers = atomic_long_read(&vmap_lazy_nr) / lazy_max_pages();
 		nr_purge_helpers = clamp(nr_purge_helpers, 1U, nr_purge_nodes) - 1;
 
-		for_each_cpu(i, &purge_nodes) {
-			vn = &vmap_nodes[i];
+		for_each_vmap_node(vn) {
+			vn->work_queued = false;
+
+			if (list_empty(&vn->purge_list))
+				continue;
 
 			if (nr_purge_helpers > 0) {
 				INIT_WORK(&vn->purge_work, purge_vmap_node);
+				vn->work_queued = schedule_drain_vmap_work(
+					READ_ONCE(drain_vmap_helpers_wq), &vn->purge_work);
 
-				if (cpumask_test_cpu(i, cpu_online_mask))
-					schedule_work_on(i, &vn->purge_work);
-				else
-					schedule_work(&vn->purge_work);
-
-				nr_purge_helpers--;
-			} else {
-				vn->purge_work.func = NULL;
-				purge_vmap_node(&vn->purge_work);
-				nr_purged_areas += vn->nr_purged;
+				if (vn->work_queued) {
+					nr_purge_helpers--;
+					continue;
+				}
 			}
-		}
 
-		for_each_cpu(i, &purge_nodes) {
-			vn = &vmap_nodes[i];
+			/* Sync path. Process locally. */
+			purge_vmap_node(&vn->purge_work);
+			nr_purged_areas += vn->nr_purged;
+		}
 
-			if (vn->purge_work.func) {
+		/* Wait for completion if queued any. */
+		for_each_vmap_node(vn) {
+			if (vn->work_queued) {
 				flush_work(&vn->purge_work);
 				nr_purged_areas += vn->nr_purged;
 			}
@@ -2487,7 +2499,8 @@ static void free_vmap_area_noflush(struct vmap_area *va)
 
 	/* After this point, we may free va at any time */
 	if (unlikely(nr_lazy > nr_lazy_max))
-		schedule_work(&drain_vmap_work);
+		schedule_drain_vmap_work(READ_ONCE(drain_vmap_wq),
+			&drain_vmap_work);
 }
 
 /*
@@ -2503,67 +2516,94 @@ static void free_unmap_vmap_area(struct vmap_area *va)
 	free_vmap_area_noflush(va);
 }
 
-struct vmap_area *find_vmap_area(unsigned long addr)
+static inline int next_vmap_node_id(int i)
+{
+	return (i + nr_vmap_nodes - 1) % nr_vmap_nodes;
+}
+
+enum vmap_lock_mode {
+	VMAP_LOCK,
+	VMAP_TRYLOCK,
+};
+
+/*
+ * Search for a vmap_area at @addr across all vmap nodes.  An
+ * addr_to_node_id(addr) converts an address to a node index where
+ * a VA is located. If VA spans several zones and passed addr is not
+ * the same as va->va_start, what is not common, we may need to scan
+ * extra nodes. See an example:
+ *
+ *      <----va---->
+ * -|-----|-----|-----|-----|-
+ *     1     2     0     1
+ *
+ * VA resides in node 1 whereas it spans 1, 2 an 0. If passed addr
+ * is within 2 or 0 nodes we should do extra work.
+ *
+ * Returns the VA with @locked_vn->busy.lock held; the caller must
+ * release it. If @mode is VMAP_TRYLOCK, nodes that cannot be locked
+ * are skipped.
+ */
+static struct vmap_area *
+find_vmap_area_lock(unsigned long addr, struct vmap_node **locked_vn,
+		enum vmap_lock_mode mode)
 {
 	struct vmap_node *vn;
 	struct vmap_area *va;
 	int i, j;
 
+	*locked_vn = NULL;
+
 	if (unlikely(!vmap_initialized))
 		return NULL;
 
-	/*
-	 * An addr_to_node_id(addr) converts an address to a node index
-	 * where a VA is located. If VA spans several zones and passed
-	 * addr is not the same as va->va_start, what is not common, we
-	 * may need to scan extra nodes. See an example:
-	 *
-	 *      <----va---->
-	 * -|-----|-----|-----|-----|-
-	 *     1     2     0     1
-	 *
-	 * VA resides in node 1 whereas it spans 1, 2 an 0. If passed
-	 * addr is within 2 or 0 nodes we should do extra work.
-	 */
 	i = j = addr_to_node_id(addr);
 	do {
 		vn = &vmap_nodes[i];
 
-		spin_lock(&vn->busy.lock);
-		va = __find_vmap_area(addr, &vn->busy.root);
-		spin_unlock(&vn->busy.lock);
+		if (mode == VMAP_LOCK) {
+			spin_lock(&vn->busy.lock);
+		} else {
+			if (!spin_trylock(&vn->busy.lock))
+				continue;
+		}
 
-		if (va)
+		va = __find_vmap_area(addr, &vn->busy.root);
+		if (va) {
+			*locked_vn = vn;
 			return va;
-	} while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j);
+		}
+
+		spin_unlock(&vn->busy.lock);
+	} while ((i = next_vmap_node_id(i)) != j);
 
 	return NULL;
 }
 
-static struct vmap_area *find_unlink_vmap_area(unsigned long addr)
+struct vmap_area *find_vmap_area(unsigned long addr)
 {
 	struct vmap_node *vn;
 	struct vmap_area *va;
-	int i, j;
 
-	/*
-	 * Check the comment in the find_vmap_area() about the loop.
-	 */
-	i = j = addr_to_node_id(addr);
-	do {
-		vn = &vmap_nodes[i];
-
-		spin_lock(&vn->busy.lock);
-		va = __find_vmap_area(addr, &vn->busy.root);
-		if (va)
-			unlink_va(va, &vn->busy.root);
+	va = find_vmap_area_lock(addr, &vn, VMAP_LOCK);
+	if (va)
 		spin_unlock(&vn->busy.lock);
 
-		if (va)
-			return va;
-	} while ((i = (i + nr_vmap_nodes - 1) % nr_vmap_nodes) != j);
+	return va;
+}
 
-	return NULL;
+static struct vmap_area *find_unlink_vmap_area(unsigned long addr)
+{
+	struct vmap_node *vn;
+	struct vmap_area *va;
+
+	va = find_vmap_area_lock(addr, &vn, VMAP_LOCK);
+	if (va) {
+		unlink_va(va, &vn->busy.root);
+		spin_unlock(&vn->busy.lock);
+	}
+
+	return va;
 }
 
 /*** Per cpu kva allocator ***/
@@ -3124,7 +3164,7 @@ EXPORT_SYMBOL(vm_map_ram);
 
 static struct vm_struct *vmlist __initdata;
 
-static inline unsigned int vm_area_page_order(struct vm_struct *vm)
+static inline unsigned int vm_area_page_order(const struct vm_struct *vm)
 {
 #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
 	return vm->page_order;
@@ -3133,7 +3173,7 @@ static inline unsigned int vm_area_page_order(struct vm_struct *vm)
 #endif
 }
 
-unsigned int get_vm_area_page_order(struct vm_struct *vm)
+unsigned int get_vm_area_page_order(const struct vm_struct *vm)
 {
 	return vm_area_page_order(vm);
 }
@@ -3358,14 +3398,18 @@ struct vm_struct *remove_vm_area(const void *addr)
 }
 
 static inline void set_area_direct_map(const struct vm_struct *area,
-				       int (*set_direct_map)(struct page *page))
+				       int (*set_direct_map)(struct page *page,
+							     unsigned int nr))
 {
-	unsigned long i;
+	unsigned int nr = (1U << vm_area_page_order(area));
 
-	/* HUGE_VMALLOC passes small pages to set_direct_map */
-	for (i = 0; i < area->nr_pages; i++)
-		if (page_address(area->pages[i]))
-			set_direct_map(area->pages[i]);
+	for (unsigned long i = 0; i < area->nr_pages; i += nr) {
+		if (page_address(area->pages[i])) {
+			int err = set_direct_map(area->pages[i], nr);
+
+			WARN_ON_ONCE(err);
+		}
+	}
 }
 
 /*
@@ -3681,7 +3725,7 @@ vm_area_alloc_pages(gfp_t gfp, int nid,
 	/*
 	 * Initially, attempt to have the page allocator give us large order
 	 * pages. Do not attempt allocating smaller than order chunks since
-	 * __vmap_pages_range() expects physically contigous pages of exactly
+	 * __vmap_pages_range() expects physically contiguous pages of exactly
 	 * order long chunks.
 	 */
 	while (large_order > order && nr_remaining) {
@@ -3872,7 +3916,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 	unsigned long size = get_vm_area_size(area);
 	unsigned long array_size;
 	unsigned long nr_small_pages = size >> PAGE_SHIFT;
-	unsigned int page_order;
+	unsigned int page_order = page_shift - PAGE_SHIFT;
 	unsigned int flags;
 	int ret;
 
@@ -3900,9 +3944,6 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 		goto fail;
 	}
 
-	set_vm_area_page_order(area, page_shift - PAGE_SHIFT);
-	page_order = vm_area_page_order(area);
-
 	/*
 	 * High-order nofail allocations are really expensive and
 	 * potentially dangerous (pre-mature OOM, disruptive reclaim
@@ -3957,6 +3998,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 		goto fail;
 	}
 
+	set_vm_area_page_order(area, page_order);
 	return area->addr;
 
 fail:
@@ -4019,6 +4061,12 @@ static gfp_t vmalloc_fix_flags(gfp_t flags)
  * %__GFP_SKIP_KASAN can be used to skip unpoisoning of mapped pages
  * (when prot=%PAGE_KERNEL).
  *
+ * %VM_ALLOW_HUGE_VMAP allocates huge pages when possible and falls back to
+ * base pages if huge page allocation fails.
+ *
+ * %VM_REQUIRE_HUGE_VMAP implies %VM_ALLOW_HUGE_VMAP and fails instead of
+ * silently falling back to base pages.
+ *
  * Can not be called from interrupt nor NMI contexts.
  * Return: the address of the area or %NULL on failure
  */
@@ -4044,6 +4092,10 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 		return NULL;
 	}
 
+	/* VM_REQUIRE_HUGE_VMAP implies VM_ALLOW_HUGE_VMAP */
+	if (vm_flags & VM_REQUIRE_HUGE_VMAP)
+		vm_flags |= VM_ALLOW_HUGE_VMAP;
+
 	if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) {
 		/*
 		 * Try huge pages. Only try for PAGE_KERNEL allocations,
@@ -4060,6 +4112,9 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 		align = max(original_align, 1UL << shift);
 	}
 
+	if ((vm_flags & VM_REQUIRE_HUGE_VMAP) && shift == PAGE_SHIFT)
+		return NULL;
+
 again:
 	area = __get_vm_area_node(size, align, shift, VM_ALLOC |
 				  VM_UNINITIALIZED | vm_flags, start, end, node,
@@ -4134,7 +4189,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 	return area->addr;
 
 fail:
-	if (shift > PAGE_SHIFT) {
+	if (shift > PAGE_SHIFT && !(vm_flags & VM_REQUIRE_HUGE_VMAP)) {
 		shift = PAGE_SHIFT;
 		align = original_align;
 		goto again;
@@ -5107,9 +5162,14 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 
 		ret = va_clip(&free_vmap_area_root,
 			&free_vmap_area_list, va, start, size);
-		if (WARN_ON_ONCE(unlikely(ret)))
-			/* It is a BUG(), but trigger recovery instead. */
+		if (unlikely(ret)) {
+			/*
+			 * -ENOMEM from the GFP_NOWAIT fallback is expected.
+			 * Anything else is a BUG(), but trigger recovery instead.
+			 */
+			WARN_ON_ONCE(ret != -ENOMEM);
 			goto recovery;
+		}
 
 		/* Allocated area. */
 		va = vas[area];
@@ -5248,15 +5308,12 @@ bool vmalloc_dump_obj(void *object)
 	unsigned long addr;
 	unsigned long nr_pages;
 
-	addr = PAGE_ALIGN((unsigned long) object);
-	vn = addr_to_node(addr);
+	addr = PAGE_ALIGN_DOWN((unsigned long) object);
 
-	if (!spin_trylock(&vn->busy.lock))
-		return false;
-
-	va = __find_vmap_area(addr, &vn->busy.root);
+	va = find_vmap_area_lock(addr, &vn, VMAP_TRYLOCK);
 	if (!va || !va->vm) {
-		spin_unlock(&vn->busy.lock);
+		if (va)
+			spin_unlock(&vn->busy.lock);
 		return false;
 	}
 
@@ -5308,7 +5365,7 @@ static void show_purge_info(struct seq_file *m)
 	for_each_vmap_node(vn) {
 		spin_lock(&vn->lazy.lock);
 		list_for_each_entry(va, &vn->lazy.head, list) {
-			seq_printf(m, "0x%pK-0x%pK %7ld unpurged vm_area\n",
+			seq_printf(m, "0x%p-0x%p %7ld unpurged vm_area\n",
 				(void *)va->va_start, (void *)va->va_end,
 				va_size(va));
 		}
@@ -5316,7 +5373,7 @@ static void show_purge_info(struct seq_file *m)
 	}
 }
 
-static int vmalloc_info_show(struct seq_file *m, void *p)
+static void show_busy_info(struct seq_file *m)
 {
 	struct vmap_node *vn;
 	struct vmap_area *va;
@@ -5331,7 +5388,7 @@ static int vmalloc_info_show(struct seq_file *m, void *p)
 		list_for_each_entry(va, &vn->busy.head, list) {
 			if (!va->vm) {
 				if (va->flags & VMAP_RAM)
-					seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n",
+					seq_printf(m, "0x%p-0x%p %7ld vm_map_ram\n",
 						(void *)va->va_start, (void *)va->va_end,
 						va_size(va));
 
@@ -5345,7 +5402,7 @@ static int vmalloc_info_show(struct seq_file *m, void *p)
 			/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
 			smp_rmb();
 
-			seq_printf(m, "0x%pK-0x%pK %7ld",
+			seq_printf(m, "0x%p-0x%p %7ld",
 				v->addr, v->addr + v->size, v->size);
 
 			if (v->caller)
@@ -5386,12 +5443,18 @@ static int vmalloc_info_show(struct seq_file *m, void *p)
 		spin_unlock(&vn->busy.lock);
 	}
 
+	if (IS_ENABLED(CONFIG_NUMA))
+		kfree(counters);
+}
+
+static int vmalloc_info_show(struct seq_file *m, void *p)
+{
+	show_busy_info(m);
+
 	/*
 	 * As a final step, dump "unpurged" areas.
 	 */
 	show_purge_info(m);
-	if (IS_ENABLED(CONFIG_NUMA))
-		kfree(counters);
 	return 0;
 }
 
@@ -5404,11 +5467,23 @@ module_init(proc_vmalloc_init);
 
 #endif
 
+static void __init vmap_insert_free_area(unsigned long start, unsigned long end)
+{
+	struct vmap_area *free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
+
+	if (!WARN_ON_ONCE(!free)) {
+		free->va_start = start;
+		free->va_end = end;
+		insert_vmap_area_augment(free, NULL,
+					 &free_vmap_area_root,
+					 &free_vmap_area_list);
+	}
+}
+
 static void __init vmap_init_free_space(void)
 {
 	unsigned long vmap_start = 1;
 	const unsigned long vmap_end = ULONG_MAX;
-	struct vmap_area *free;
 	struct vm_struct *busy;
 
 	/*
@@ -5418,32 +5493,15 @@ static void __init vmap_init_free_space(void)
 	 *  |<--------------------------------->|
 	 */
 	for (busy = vmlist; busy; busy = busy->next) {
-		if ((unsigned long) busy->addr - vmap_start > 0) {
-			free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
-			if (!WARN_ON_ONCE(!free)) {
-				free->va_start = vmap_start;
-				free->va_end = (unsigned long) busy->addr;
-
-				insert_vmap_area_augment(free, NULL,
-					&free_vmap_area_root,
-						&free_vmap_area_list);
-			}
-		}
+		if ((unsigned long) busy->addr - vmap_start > 0)
+			vmap_insert_free_area(vmap_start,
+					      (unsigned long) busy->addr);
 
 		vmap_start = (unsigned long) busy->addr + busy->size;
 	}
 
-	if (vmap_end - vmap_start > 0) {
-		free = kmem_cache_zalloc(vmap_area_cachep, GFP_NOWAIT);
-		if (!WARN_ON_ONCE(!free)) {
-			free->va_start = vmap_start;
-			free->va_end = vmap_end;
-
-			insert_vmap_area_augment(free, NULL,
-				&free_vmap_area_root,
-					&free_vmap_area_list);
-		}
-	}
+	if (vmap_end - vmap_start > 0)
+		vmap_insert_free_area(vmap_start, vmap_end);
 }
 
 static void vmap_init_nodes(void)
@@ -5546,10 +5604,11 @@ void __init vmalloc_init(void)
 		vbq = &per_cpu(vmap_block_queue, i);
 		spin_lock_init(&vbq->lock);
 		INIT_LIST_HEAD(&vbq->free);
+		xa_init(&vbq->vmap_blocks);
+
 		p = &per_cpu(vfree_deferred, i);
 		init_llist_head(&p->list);
 		INIT_WORK(&p->wq, delayed_vfree_work);
-		xa_init(&vbq->vmap_blocks);
 	}
 
 	/*
@@ -5587,3 +5646,20 @@ void __init vmalloc_init(void)
 	vmap_node_shrinker->scan_objects = vmap_node_shrink_scan;
 	shrinker_register(vmap_node_shrinker);
 }
+
+static int __init vmalloc_init_workqueue(void)
+{
+	struct workqueue_struct *drain_wq, *helpers_wq;
+	unsigned int flags = WQ_UNBOUND | WQ_MEM_RECLAIM;
+
+	drain_wq = alloc_workqueue("vmap_drain", flags, 0);
+	WARN_ON_ONCE(drain_wq == NULL);
+	WRITE_ONCE(drain_vmap_wq, drain_wq);
+
+	helpers_wq = alloc_workqueue("vmap_drain_helpers", flags, 0);
+	WARN_ON_ONCE(helpers_wq == NULL);
+	WRITE_ONCE(drain_vmap_helpers_wq, helpers_wq);
+
+	return 0;
+}
+early_initcall(vmalloc_init_workqueue);
diff --git a/mm/vmalloc.h b/mm/vmalloc.h
index 8866ddcff6681b..211869f365095f 100644
--- a/mm/vmalloc.h
+++ b/mm/vmalloc.h
@@ -12,7 +12,7 @@ void __init vmalloc_init(void);
 int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
 		pgprot_t prot, struct page **pages,
 		unsigned int page_shift, gfp_t gfp_mask);
-unsigned int get_vm_area_page_order(struct vm_struct *vm);
+unsigned int get_vm_area_page_order(const struct vm_struct *vm);
 #else
 static inline void vmalloc_init(void) {}
 
diff --git a/mm/vmpressure.c b/mm/vmpressure.c
index 9629240d77adc7..3de99fef392894 100644
--- a/mm/vmpressure.c
+++ b/mm/vmpressure.c
@@ -30,9 +30,6 @@
  *
  * As the vmscan reclaimer logic works with chunks which are multiple of
  * SWAP_CLUSTER_MAX, it makes sense to use it for the window size as well.
- *
- * TODO: Make the window size depend on machine size, as we do for vmstat
- * thresholds. Currently we set it to 512 pages (2MB for 4KB pages).
  */
 const unsigned long vmpressure_win = SWAP_CLUSTER_MAX * 16;
 
diff --git a/mm/vmscan.c b/mm/vmscan.c
index f11491ee9ed5c1..91295070ca336c 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -182,17 +182,37 @@ struct scan_control {
 };
 
 #ifdef ARCH_HAS_PREFETCHW
-#define prefetchw_prev_lru_folio(_folio, _base, _field)			\
-	do {								\
-		if ((_folio)->lru.prev != _base) {			\
-			struct folio *prev;				\
-									\
-			prev = lru_to_folio(&(_folio->lru));		\
-			prefetchw(&prev->_field);			\
-		}							\
-	} while (0)
+static inline void prefetchw_prev_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+	if (folio->lru.prev != base) {
+		struct folio *prev;
+
+		prev = lru_to_folio(&folio->lru);
+		prefetchw(&prev->flags);
+	}
+}
+
+static inline void prefetchw_next_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+	if (folio->lru.next != base) {
+		struct folio *next;
+
+		next = list_entry(folio->lru.next, struct folio, lru);
+		prefetchw(&next->flags);
+	}
+}
 #else
-#define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0)
+static inline void prefetchw_prev_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+}
+
+static inline void prefetchw_next_lru_folio(struct folio *folio,
+		struct list_head *base)
+{
+}
 #endif
 
 /*
@@ -342,20 +362,72 @@ static bool can_demote(int nid, struct scan_control *sc,
 	return !nodes_empty(allowed_mask);
 }
 
+#ifdef CONFIG_SWAP
+static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg,
+		int nid, struct scan_control *sc)
+{
+	pg_data_t *pgdat = NODE_DATA(nid);
+	unsigned long anon_pages, swapcache;
+
+	/*
+	 * A GFP_NOIO reclaimer can only reclaim anon that is already in the
+	 * swapcache (adding anon to the swapcache needs IO). When swapcache is
+	 * far below the anon LRU, scanning anon reclaims nothing and only burns
+	 * CPU. The 1/64 threshold keeps this to the case where anon is
+	 * effectively unreclaimable.
+	 */
+	if (!sc || (sc->gfp_mask & __GFP_IO))
+		return false;
+
+	/*
+	 * FIXME: MGLRU doesn't fully respect can_reclaim_anon_pages() for the
+	 * scanning type, so only apply this to the traditional LRU for now.
+	 */
+	if (lru_gen_enabled())
+		return false;
+
+	if (memcg) {
+		struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
+
+		anon_pages = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
+			     lruvec_page_state(lruvec, NR_ACTIVE_ANON);
+		swapcache = lruvec_page_state(lruvec, NR_SWAPCACHE);
+	} else {
+		anon_pages = node_page_state(pgdat, NR_INACTIVE_ANON) +
+			     node_page_state(pgdat, NR_ACTIVE_ANON);
+		swapcache = node_page_state(pgdat, NR_SWAPCACHE);
+	}
+
+	return swapcache < (anon_pages >> 6);
+}
+#else
+static inline bool reclaimable_anon_is_low(struct mem_cgroup *memcg,
+		int nid, struct scan_control *sc)
+{
+	return true;
+}
+#endif /* CONFIG_SWAP */
+
 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
 					  int nid,
 					  struct scan_control *sc)
 {
 	if (memcg == NULL) {
 		/*
-		 * For non-memcg reclaim, is there
-		 * space in any swap device?
+		 * For non-memcg reclaim, is there space in any swap device?
+		 * And under GFP_NOIO, is there enough swapcached anon to make
+		 * scanning anon worthwhile?
 		 */
-		if (get_nr_swap_pages() > 0)
+		if (get_nr_swap_pages() > 0 &&
+		    !reclaimable_anon_is_low(memcg, nid, sc))
 			return true;
 	} else {
-		/* Is the memcg below its swap limit? */
-		if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
+		/*
+		 * Is the memcg below its swap limit, and under GFP_NOIO does
+		 * it have enough swapcached anon to make scanning worthwhile?
+		 */
+		if (mem_cgroup_get_nr_swap_pages(memcg) > 0 &&
+		    !reclaimable_anon_is_low(memcg, nid, sc))
 			return true;
 	}
 
@@ -735,8 +807,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
 
 		if (reclaimed && !mapping_exiting(mapping))
 			shadow = workingset_eviction(folio, target_memcg);
-		__memcg1_swapout(folio, ci);
-		__swap_cache_del_folio(ci, folio, swap, shadow);
+		__swap_cache_del_folio(ci, folio, swap, shadow, true);
 		swap_cluster_unlock_irq(ci);
 	} else {
 		void (*free_folio)(struct folio *);
@@ -783,6 +854,22 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
 	return 0;
 }
 
+static long __remove_mapping_unfreeze(struct address_space *mapping,
+				      struct folio *folio, bool reclaimed,
+				      struct mem_cgroup *target_memcg)
+{
+	if (__remove_mapping(mapping, folio, reclaimed, target_memcg)) {
+		/*
+		 * Unfreezing the refcount with 1 effectively
+		 * drops the pagecache ref for us without requiring another
+		 * atomic operation.
+		 */
+		folio_ref_unfreeze(folio, 1);
+		return folio_nr_pages(folio);
+	}
+	return 0;
+}
+
 /**
  * remove_mapping() - Attempt to remove a folio from its mapping.
  * @mapping: The address space.
@@ -797,16 +884,29 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
  */
 long remove_mapping(struct address_space *mapping, struct folio *folio)
 {
-	if (__remove_mapping(mapping, folio, false, NULL)) {
-		/*
-		 * Unfreezing the refcount with 1 effectively
-		 * drops the pagecache ref for us without requiring another
-		 * atomic operation.
-		 */
-		folio_ref_unfreeze(folio, 1);
-		return folio_nr_pages(folio);
-	}
-	return 0;
+	return __remove_mapping_unfreeze(mapping, folio, false, NULL);
+}
+
+/**
+ * remove_mapping_set_shadow() - Remove a folio and record an eviction shadow.
+ * @mapping: The address space.
+ * @folio: The folio to remove.
+ * @target_memcg: The memcg to charge the eviction shadow to; the caller must
+ *                keep it alive across the call.
+ *
+ * Like remove_mapping(), but stores a workingset eviction shadow the way page
+ * reclaim does, so that a later refault can be detected and the folio
+ * re-activated.
+ * Return: The number of pages removed from the mapping.  0 if the folio
+ * could not be removed.
+ * Context: The caller should have a single refcount on the folio and
+ * hold its lock.
+ */
+long remove_mapping_set_shadow(struct address_space *mapping,
+			       struct folio *folio,
+			       struct mem_cgroup *target_memcg)
+{
+	return __remove_mapping_unfreeze(mapping, folio, true, target_memcg);
 }
 
 /**
@@ -843,19 +943,22 @@ static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
 	if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) {
 		/* Activate file-backed executable folios after first usage. */
 		if (is_exec_file_folio(folio, vma_flags)) {
-			set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
+			folio_set_workingset(folio);
+			folio_set_lru_refs(folio, 0);
 			return true;
 		}
 
-		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
+		folio_set_lru_refs(folio, 1);
 		return false;
 	}
 
 	/* Promote on second access */
-	if (folio_lru_refs(folio) > 1)
-		set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset));
-	else
+	if (folio_lru_refs(folio) > 1) {
+		folio_set_workingset(folio);
+		folio_set_lru_refs(folio, 0);
+	} else {
 		folio_mark_accessed(folio);
+	}
 	return true;
 }
 #else
@@ -955,7 +1058,7 @@ static void folio_check_dirty_writeback(struct folio *folio,
 	*writeback = folio_test_writeback(folio);
 
 	/* Verify dirty/writeback state if the filesystem supports it */
-	if (!folio_test_private(folio))
+	if (!folio_has_attached_private(folio))
 		return;
 
 	mapping = folio_mapping(folio);
@@ -1259,6 +1362,8 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 		 */
 		if (folio_test_anon(folio) && folio_test_swapbacked(folio) &&
 				!folio_test_swapcache(folio)) {
+			int ret;
+
 			if (!(sc->gfp_mask & __GFP_IO))
 				goto keep_locked;
 			if (folio_maybe_dma_pinned(folio))
@@ -1277,11 +1382,14 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
 				    split_folio_to_list(folio, folio_list))
 					goto activate_locked;
 			}
-			if (folio_alloc_swap(folio)) {
+			ret = folio_alloc_swap(folio);
+			if (ret) {
 				int __maybe_unused order = folio_order(folio);
 
 				if (!folio_test_large(folio))
 					goto activate_locked_split;
+				if (ret != -E2BIG)
+					goto activate_locked;
 				/* Fallback to swap normal pages */
 				if (split_folio_to_list(folio, folio_list))
 					goto activate_locked;
@@ -1695,7 +1803,7 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
 		struct folio *folio;
 
 		folio = lru_to_folio(src);
-		prefetchw_prev_lru_folio(folio, src, flags);
+		prefetchw_prev_lru_folio(folio, src);
 
 		nr_pages = folio_nr_pages(folio);
 		total_scan += nr_pages;
@@ -1863,7 +1971,7 @@ static bool too_many_isolated(struct pglist_data *pgdat, int file,
  *
  * Note: The caller must not hold any lruvec lock.
  */
-static unsigned int move_folios_to_lru(struct list_head *list)
+static unsigned int move_folios_to_lru(struct list_head *list, bool do_rotate)
 {
 	int nr_pages, nr_moved = 0;
 	struct lruvec *lruvec = NULL;
@@ -1910,7 +2018,19 @@ static unsigned int move_folios_to_lru(struct list_head *list)
 			continue;
 		}
 
-		lruvec_add_folio(lruvec, folio);
+		/*
+		 * Put clean, unreferenced and unpinned folios that may have
+		 * missed folio_rotate_reclaimable() at the tail to avoid
+		 * cold/hot inversion.
+		 */
+		if (do_rotate && !folio_test_active(folio) && !folio_mapped(folio) &&
+		    !folio_test_dirty(folio) && !folio_test_writeback(folio) &&
+		    !folio_test_referenced(folio) &&
+		    folio_ref_count(folio) == folio_expected_ref_count(folio))
+			lruvec_add_folio_tail(lruvec, folio);
+		else
+			lruvec_add_folio(lruvec, folio);
+
 		nr_pages = folio_nr_pages(folio);
 		nr_moved += nr_pages;
 		if (folio_test_active(folio))
@@ -2027,7 +2147,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
 	nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false,
 					 lruvec_memcg(lruvec));
 
-	move_folios_to_lru(&folio_list);
+	move_folios_to_lru(&folio_list, true);
 
 	mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
 					stat.nr_demoted);
@@ -2138,8 +2258,8 @@ static void shrink_active_list(unsigned long nr_to_scan,
 	/*
 	 * Move folios back to the lru list.
 	 */
-	nr_activate = move_folios_to_lru(&l_active);
-	nr_deactivate = move_folios_to_lru(&l_inactive);
+	nr_activate = move_folios_to_lru(&l_active, false);
+	nr_deactivate = move_folios_to_lru(&l_inactive, false);
 
 	count_vm_events(PGDEACTIVATE, nr_deactivate);
 	count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
@@ -3195,8 +3315,11 @@ struct ctrl_pos {
 	int gain;
 };
 
-static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
-			  struct ctrl_pos *pos)
+/*
+ * __noipa works around gcc-16 warning for uninitialized use of pos->refaulted
+ */
+static void __noipa read_ctrl_pos(struct lruvec *lruvec, int type, int tier_min,
+				  int tier_max, int gain, struct ctrl_pos *pos)
 {
 	int i;
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
@@ -3205,7 +3328,7 @@ static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
 	pos->gain = gain;
 	pos->refaulted = pos->total = 0;
 
-	for (i = tier % MAX_NR_TIERS; i <= min(tier, MAX_NR_TIERS - 1); i++) {
+	for (i = tier_min; i <= tier_max; i++) {
 		pos->refaulted += lrugen->avg_refaulted[type][i] +
 				  atomic_long_read(&lrugen->refaulted[hist][type][i]);
 		pos->total += lrugen->avg_total[type][i] +
@@ -3266,11 +3389,11 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
  ******************************************************************************/
 
 /* promote pages accessed through page tables */
-static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma_flags)
+static int folio_update_gen(struct folio *folio, int new_gen, int *type,
+			    const vma_flags_t *vma_flags)
 {
-	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
-
-	VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);
+	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+	int old_gen;
 
 	/*
 	 * See the comment on LRU_REFS_FLAGS, and activate file-backed
@@ -3279,53 +3402,71 @@ static int folio_update_gen(struct folio *folio, int gen, const vma_flags_t *vma
 	 */
 	if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
 	    !is_exec_file_folio(folio, vma_flags)) {
-		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced));
+		folio_set_lru_refs(folio, 1);
 		return -1;
 	}
 
 	do {
+		old_gen = lru_get_gen_flags(old_flags);
+		new_flags = old_flags;
+
 		/* lru_gen_del_folio() has isolated this page? */
-		if (!(old_flags & LRU_GEN_MASK))
-			return -1;
+		if (old_gen < 0)
+			break;
 
-		new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
-		new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset);
-	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+		lru_set_gen_flags(&new_flags, new_gen);
+		lru_set_refs_flags(&new_flags, 0);
+		new_flags |= BIT(PG_workingset);
+	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
 
-	return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+	*type = folio_flags_is_file_lru(&old_flags);
+	return old_gen;
 }
 
-/* protect pages accessed multiple times through file descriptors */
-static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
+static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increased)
 {
-	int type = folio_is_file_lru(folio);
-	struct lru_gen_folio *lrugen = &lruvec->lrugen;
-	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
-	unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f);
-
-	VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio);
+	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
+	int new_gen;
 
 	do {
-		new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
+		new_gen = lru_get_gen_flags(old_flags);
+
 		/* folio_update_gen() has promoted this page? */
-		if (new_gen >= 0 && new_gen != old_gen)
+		if (new_gen >= 0 && new_gen != old_gen) {
+			if (increased)
+				*increased = false;
 			return new_gen;
+		}
 
+		new_flags = old_flags;
 		new_gen = (old_gen + 1) % MAX_NR_GENS;
 
-		new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
-		new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
-	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
+		lru_set_gen_flags(&new_flags, new_gen);
+		lru_set_refs_flags(&new_flags, 0);
+	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
 
-	lru_gen_update_size(lruvec, folio, old_gen, new_gen);
+	if (increased)
+		*increased = true;
+	return new_gen;
+}
 
+/* protect pages accessed multiple times through file descriptors */
+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
+{
+	int type = folio_is_file_lru(folio);
+	struct lru_gen_folio *lrugen = &lruvec->lrugen;
+	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+	bool gen_increased;
+
+	new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
+	if (gen_increased)
+		lru_gen_update_size(lruvec, folio, old_gen, new_gen);
 	return new_gen;
 }
 
 static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio,
-			      int old_gen, int new_gen)
+			      int old_gen, int new_gen, int type)
 {
-	int type = folio_is_file_lru(folio);
 	int zone = folio_zonenum(folio);
 	int delta = folio_nr_pages(folio);
 
@@ -3374,13 +3515,14 @@ static int should_skip_vma(unsigned long start, unsigned long end, struct mm_wal
 	if (!vma_is_accessible(vma))
 		return true;
 
-	if (is_vm_hugetlb_page(vma))
+	if (vma_is_hugetlb(vma))
 		return true;
 
 	if (!vma_has_recency(vma))
 		return true;
 
-	if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL))
+	if (vma_test(vma, VMA_LOCKED_BIT) || vma_is_kernel_owned(vma) ||
+	    vma_is_fixed_mapping(vma))
 		return true;
 
 	if (vma == get_gate_vma(vma->vm_mm))
@@ -3512,22 +3654,24 @@ static bool suitable_to_scan(int total, int young)
 }
 
 static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struct *vma,
-		struct folio *folio, int new_gen, bool dirty)
+		struct lruvec *lruvec, struct folio *folio, bool dirty)
 {
-	int old_gen;
+	int new_gen, old_gen, type;
 
 	if (!folio)
 		return;
 
+	new_gen = lru_gen_from_seq(READ_ONCE(lruvec->lrugen.max_seq));
+
 	if (dirty && !folio_test_dirty(folio) &&
 	    !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
 	      !folio_test_swapcache(folio)))
 		folio_mark_dirty(folio);
 
 	if (walk) {
-		old_gen = folio_update_gen(folio, new_gen, &vma->flags);
+		old_gen = folio_update_gen(folio, new_gen, &type, &vma->flags);
 		if (old_gen >= 0 && old_gen != new_gen)
-			update_batch_size(walk, folio, old_gen, new_gen);
+			update_batch_size(walk, folio, old_gen, new_gen, type);
 	} else if (lru_gen_set_refs(folio, &vma->flags)) {
 		old_gen = folio_lru_gen(folio);
 		if (old_gen >= 0 && old_gen != new_gen)
@@ -3549,8 +3693,6 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
 	struct lru_gen_mm_walk *walk = args->private;
 	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
 	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
-	DEFINE_MAX_SEQ(walk->lruvec);
-	int gen = lru_gen_from_seq(max_seq);
 	unsigned int nr;
 	pmd_t pmdval;
 
@@ -3601,7 +3743,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
 			continue;
 
 		if (last != folio) {
-			walk_update_folio(walk, args->vma, last, gen, dirty);
+			walk_update_folio(walk, args->vma, walk->lruvec, last, dirty);
 
 			last = folio;
 			dirty = false;
@@ -3614,7 +3756,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
 		walk->mm_stats[MM_LEAF_YOUNG] += nr;
 	}
 
-	walk_update_folio(walk, args->vma, last, gen, dirty);
+	walk_update_folio(walk, args->vma, walk->lruvec, last, dirty);
 	last = NULL;
 
 	if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end))
@@ -3637,8 +3779,6 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 	struct lru_gen_mm_walk *walk = args->private;
 	struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
 	struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
-	DEFINE_MAX_SEQ(walk->lruvec);
-	int gen = lru_gen_from_seq(max_seq);
 
 	VM_WARN_ON_ONCE(pud_leaf(*pud));
 
@@ -3692,7 +3832,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 			goto next;
 
 		if (last != folio) {
-			walk_update_folio(walk, vma, last, gen, dirty);
+			walk_update_folio(walk, vma, walk->lruvec, last, dirty);
 
 			last = folio;
 			dirty = false;
@@ -3706,7 +3846,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
 		i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1;
 	} while (i <= MIN_LRU_BATCH);
 
-	walk_update_folio(walk, vma, last, gen, dirty);
+	walk_update_folio(walk, vma, walk->lruvec, last, dirty);
 
 	lazy_mmu_mode_disable();
 	spin_unlock(ptl);
@@ -3897,6 +4037,19 @@ static void clear_mm_walk(void)
 		kfree(walk);
 }
 
+static inline void flush_lru_batch(struct list_head *head, struct list_head **batch_end,
+				   struct list_head *dst)
+{
+	LIST_HEAD(movable);
+
+	if (!*batch_end)
+		return;
+
+	list_cut_position(&movable, head, *batch_end);
+	list_splice_tail_init(&movable, dst);
+	*batch_end = NULL;
+}
+
 static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 {
 	int zone;
@@ -3904,6 +4057,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 	int hist = lru_hist_from_seq(lrugen->min_seq[type]);
 	int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]);
+	int target_gen = (old_gen + 1) % MAX_NR_GENS;
 
 	/* For file type, skip the check if swappiness is anon only */
 	if (type && (swappiness == SWAPPINESS_ANON_ONLY))
@@ -3913,35 +4067,63 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 	if (!type && !swappiness)
 		goto done;
 
+	VM_WARN_ON_ONCE(get_nr_gens(lruvec, type) != MAX_NR_GENS);
+	VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) !=
+			lru_gen_is_active(lruvec, target_gen));
 	/* prevent cold/hot inversion if the type is evictable */
 	for (zone = 0; zone < MAX_NR_ZONES; zone++) {
+		struct list_head *target_list = &lrugen->folios[target_gen][type][zone];
 		struct list_head *head = &lrugen->folios[old_gen][type][zone];
+		struct list_head *pos = head->next;
+		struct list_head *batch_end = NULL;
+		long delta = 0;
 
-		while (!list_empty(head)) {
-			struct folio *folio = lru_to_folio(head);
+		while (pos != head) {
+			struct folio *folio = list_entry(pos, struct folio, lru);
+			long nr_pages = folio_nr_pages(folio);
 			int refs = folio_lru_refs(folio);
 			bool workingset = folio_test_workingset(folio);
+			bool gen_increased;
 
 			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
 
-			new_gen = folio_inc_gen(lruvec, folio);
-			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
+			prefetchw_next_lru_folio(folio, head);
+			pos = pos->next;
+			new_gen = __folio_inc_gen(folio, old_gen, &gen_increased);
+			/*
+			 * If gen_increased is false, this is a promotion. Put folios
+			 * at the head of the promoted gen. Otherwise, put them at
+			 * the tail of the second-oldest gen.
+			 */
+			if (gen_increased) {
+				delta += nr_pages;
+				batch_end = &folio->lru;
 
-			/* don't count the workingset being lazily promoted */
-			if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
-				int tier = lru_tier_from_refs(refs, workingset);
-				int delta = folio_nr_pages(folio);
+				/* don't count the workingset being lazily promoted */
+				if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
+					int tier = lru_tier_from_refs(refs, workingset);
 
-				WRITE_ONCE(lrugen->protected[hist][type][tier],
-					   lrugen->protected[hist][type][tier] + delta);
+					WRITE_ONCE(lrugen->protected[hist][type][tier],
+						   lrugen->protected[hist][type][tier] + nr_pages);
+				}
+			} else {
+				flush_lru_batch(head, &batch_end, target_list);
+				list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]);
 			}
-
 			if (!--remaining)
-				return false;
+				break;
 		}
+		flush_lru_batch(head, &batch_end, target_list);
+
+		WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
+			   lrugen->nr_pages[old_gen][type][zone] - delta);
+		WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone],
+			   lrugen->nr_pages[target_gen][type][zone] + delta);
+		if (!remaining)
+			return false;
 	}
 done:
 	reset_ctrl_pos(lruvec, type, true);
@@ -4270,8 +4452,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
 	struct pglist_data *pgdat = folio_pgdat(folio);
 	struct lruvec *lruvec;
 	struct lru_gen_mm_state *mm_state;
-	unsigned long max_seq;
-	int gen;
 
 	lockdep_assert_held(pvmw->ptl);
 	VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio);
@@ -4282,8 +4462,8 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
 	if (spin_is_contended(pvmw->ptl))
 		return true;
 
-	/* exclude special VMAs containing anon pages from COW */
-	if (vma->vm_flags & VM_SPECIAL)
+	/* exclude kernel-owned and fixed VMAs containing anon pages from COW */
+	if (vma_is_kernel_owned(vma) || vma_is_fixed_mapping(vma))
 		return true;
 
 	/* avoid taking the LRU lock under the PTL when possible */
@@ -4308,8 +4488,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
 
 	memcg = get_mem_cgroup_from_folio(folio);
 	lruvec = mem_cgroup_lruvec(memcg, pgdat);
-	max_seq = READ_ONCE((lruvec)->lrugen.max_seq);
-	gen = lru_gen_from_seq(max_seq);
 	mm_state = get_mm_state(lruvec);
 
 	lazy_mmu_mode_enable();
@@ -4341,7 +4519,7 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
 			continue;
 
 		if (last != folio) {
-			walk_update_folio(walk, vma, last, gen, dirty);
+			walk_update_folio(walk, vma, lruvec, last, dirty);
 
 			last = folio;
 			dirty = false;
@@ -4353,13 +4531,14 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
 		young += nr;
 	}
 
-	walk_update_folio(walk, vma, last, gen, dirty);
+	walk_update_folio(walk, vma, lruvec, last, dirty);
 
 	lazy_mmu_mode_disable();
 
 	/* feedback from rmap walkers to page table walkers */
 	if (mm_state && suitable_to_scan(i, young))
-		update_bloom_filter(mm_state, max_seq, pvmw->pmd);
+		update_bloom_filter(mm_state, READ_ONCE(lruvec->lrugen.max_seq),
+				    pvmw->pmd);
 
 	mem_cgroup_put(memcg);
 
@@ -4373,7 +4552,6 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr)
 /* see the comment on MEMCG_NR_GENS */
 enum {
 	MEMCG_LRU_NOP,
-	MEMCG_LRU_HEAD,
 	MEMCG_LRU_TAIL,
 	MEMCG_LRU_OLD,
 	MEMCG_LRU_YOUNG,
@@ -4395,9 +4573,7 @@ static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
 	new = old = lruvec->lrugen.gen;
 
 	/* see the comment on MEMCG_NR_GENS */
-	if (op == MEMCG_LRU_HEAD)
-		seg = MEMCG_LRU_HEAD;
-	else if (op == MEMCG_LRU_TAIL)
+	if (op == MEMCG_LRU_TAIL)
 		seg = MEMCG_LRU_TAIL;
 	else if (op == MEMCG_LRU_OLD)
 		new = get_memcg_gen(pgdat->memcg_lru.seq);
@@ -4411,7 +4587,7 @@ static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
 
 	hlist_nulls_del_rcu(&lruvec->lrugen.list);
 
-	if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD)
+	if (op == MEMCG_LRU_OLD)
 		hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
 	else
 		hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
@@ -4489,15 +4665,6 @@ void lru_gen_release_memcg(struct mem_cgroup *memcg)
 	}
 }
 
-void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid)
-{
-	struct lruvec *lruvec = get_lruvec(memcg, nid);
-
-	/* see the comment on MEMCG_NR_GENS */
-	if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD)
-		lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
-}
-
 bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid)
 {
 	struct lruvec *lruvec = get_lruvec(memcg, nid);
@@ -4716,7 +4883,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
 
 	/* see the comment on LRU_REFS_FLAGS */
 	if (!folio_test_referenced(folio))
-		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0);
+		folio_set_lru_refs(folio, 0);
 
 	success = lru_gen_del_folio(lruvec, folio, true);
 	VM_WARN_ON_ONCE_FOLIO(!success, folio);
@@ -4802,16 +4969,16 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 static int get_tier_idx(struct lruvec *lruvec, int type)
 {
 	int tier;
-	struct ctrl_pos sp, pv = {};
+	struct ctrl_pos sp, pv;
 
 	/*
 	 * To leave a margin for fluctuations, use a larger gain factor (2:3).
 	 * This value is chosen because any other tier would have at least twice
 	 * as many refaults as the first tier.
 	 */
-	read_ctrl_pos(lruvec, type, 0, 2, &sp);
-	for (tier = 1; tier < MAX_NR_TIERS; tier++) {
-		read_ctrl_pos(lruvec, type, tier, 3, &pv);
+	read_ctrl_pos(lruvec, type, LRU_TIER_MIN, LRU_TIER_MIN, 2, &sp);
+	for (tier = LRU_TIER_MIN + 1; tier <= LRU_TIER_MAX; tier++) {
+		read_ctrl_pos(lruvec, type, tier, tier, 3, &pv);
 		if (!positive_ctrl_err(&sp, &pv))
 			break;
 	}
@@ -4821,7 +4988,7 @@ static int get_tier_idx(struct lruvec *lruvec, int type)
 
 static int get_type_to_scan(struct lruvec *lruvec, int swappiness)
 {
-	struct ctrl_pos sp, pv = {};
+	struct ctrl_pos sp, pv;
 
 	if (swappiness <= MIN_SWAPPINESS + 1)
 		return LRU_GEN_FILE;
@@ -4832,41 +4999,59 @@ static int get_type_to_scan(struct lruvec *lruvec, int swappiness)
 	 * Compare the sum of all tiers of anon with that of file to determine
 	 * which type to scan.
 	 */
-	read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp);
-	read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv);
+	read_ctrl_pos(lruvec, LRU_GEN_ANON, LRU_TIER_MIN, LRU_TIER_MAX,
+		      swappiness, &sp);
+	read_ctrl_pos(lruvec, LRU_GEN_FILE, LRU_TIER_MIN, LRU_TIER_MAX,
+		      MAX_SWAPPINESS - swappiness, &pv);
 
 	return positive_ctrl_err(&sp, &pv);
 }
 
+static inline bool is_single_type_reclaim(int swappiness)
+{
+	return swappiness == MIN_SWAPPINESS ||
+	       swappiness == SWAPPINESS_ANON_ONLY;
+}
+
 static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 			  struct scan_control *sc, int swappiness,
 			  struct list_head *list, int *isolated,
 			  int *isolate_type, int *isolate_scanned)
 {
-	int i;
-	int total_scanned = 0;
+	bool type_fallback_allowed = !is_single_type_reclaim(swappiness);
 	int type = get_type_to_scan(lruvec, swappiness);
+	int total_scanned = 0, scanned, tier;
+	bool tried = false;
 
-	for_each_evictable_type(i, swappiness) {
-		int scanned;
-		int tier = get_tier_idx(lruvec, type);
+retry:
+	tier = get_tier_idx(lruvec, type);
+	scanned = scan_folios(nr_to_scan, lruvec, sc,
+			      type, tier, list, isolated);
 
-		scanned = scan_folios(nr_to_scan, lruvec, sc,
-				      type, tier, list, isolated);
+	total_scanned += scanned;
+	if (*isolated) {
+		*isolate_type = type;
+		*isolate_scanned = scanned;
+		return total_scanned;
+	}
 
-		total_scanned += scanned;
-		if (*isolated) {
-			*isolate_type = type;
-			*isolate_scanned = scanned;
-			break;
-		}
-		/*
-		 * If scanned > 0 and isolated == 0, avoid falling back to the
-		 * other type, as this type remains sufficient. Falling back
-		 * too readily can disrupt the positive_ctrl_err() bias.
-		 */
-		if (!scanned)
-			type = !type;
+	/*
+	 * We are running out of the current reclaim type. Fall back to
+	 * the other type if allowed.
+	 */
+	if (!scanned && !tried && type_fallback_allowed) {
+		type = !type;
+		tried = true;
+		type_fallback_allowed = false;
+		goto retry;
+	}
+	/*
+	 * We scanned some folios but failed to isolate any due to promotions,
+	 * protections, or races. Retry once to avoid a larger loop.
+	 */
+	if (scanned && !tried) {
+		tried = true;
+		goto retry;
 	}
 
 	return total_scanned;
@@ -4931,12 +5116,18 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 			continue;
 		}
 
-		/* don't add rejected folios to the oldest generation */
+		/*
+		 * See the comments on LRU_REFS_FLAGS.
+		 *
+		 * The rejected folios are never added to the oldest generation,
+		 * so this effectively promotes them by at least one generation.
+		 */
+		folio_set_lru_refs(folio, 0);
 		if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
-			set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active));
+			folio_set_active(folio);
 	}
 
-	move_folios_to_lru(&list);
+	move_folios_to_lru(&list, false);
 
 	walk = current->reclaim_state->mm_walk;
 	if (walk && walk->batched) {
@@ -5102,7 +5293,13 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
 
-	/* lru_gen_age_node() called mem_cgroup_calculate_protection() */
+	/*
+	 * For kswapd, mem_cgroup_calculate_protection() has already
+	 * been called during the top-down cgroup traversal.
+	 */
+	if (!current_is_kswapd())
+		mem_cgroup_calculate_protection_path(NULL, memcg);
+
 	if (mem_cgroup_below_min(NULL, memcg))
 		return MEMCG_LRU_YOUNG;
 
@@ -5307,7 +5504,17 @@ static bool fill_evictable(struct lruvec *lruvec)
 
 		while (!list_empty(head)) {
 			bool success;
-			struct folio *folio = lru_to_folio(head);
+			struct folio *folio;
+
+			/*
+			 * lru_gen_add_folio() uses list_add_tail() rather
+			 * than list_add() when reclaiming is true.  Match
+			 * its ordering to avoid cold/hot inversion.
+			 */
+			if (active)
+				folio = lru_to_folio(head);
+			else
+				folio = list_first_entry(head, struct folio, lru);
 
 			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio);
@@ -5315,7 +5522,11 @@ static bool fill_evictable(struct lruvec *lruvec)
 			VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio);
 
 			lruvec_del_folio(lruvec, folio);
-			success = lru_gen_add_folio(lruvec, folio, false);
+			/*
+			 * Borrow reclaiming=true to place inactive folios in
+			 * the older gens.
+			 */
+			success = lru_gen_add_folio(lruvec, folio, !active);
 			VM_WARN_ON_ONCE(!success);
 
 			if (!--remaining)
@@ -5670,6 +5881,9 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq,
 {
 	DEFINE_MAX_SEQ(lruvec);
 
+	if (seq < max_seq)
+		return 0;
+
 	if (seq > max_seq)
 		return -EINVAL;
 
@@ -6425,8 +6639,6 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
 {
 	struct zoneref *z;
 	struct zone *zone;
-	unsigned long nr_soft_reclaimed;
-	unsigned long nr_soft_scanned;
 	gfp_t orig_mask;
 	pg_data_t *last_pgdat = NULL;
 	pg_data_t *first_pgdat = NULL;
@@ -6468,35 +6680,17 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
 				sc->compaction_ready = true;
 				continue;
 			}
-
-			/*
-			 * Shrink each node in the zonelist once. If the
-			 * zonelist is ordered by zone (not the default) then a
-			 * node may be shrunk multiple times but in that case
-			 * the user prefers lower zones being preserved.
-			 */
-			if (zone->zone_pgdat == last_pgdat)
-				continue;
-
-			/*
-			 * This steals pages from memory cgroups over softlimit
-			 * and returns the number of reclaimed pages and
-			 * scanned pages. This works for global memory pressure
-			 * and balancing, not for a memcg's limit.
-			 */
-			nr_soft_scanned = 0;
-			nr_soft_reclaimed = memcg1_soft_limit_reclaim(zone->zone_pgdat,
-								      sc->order, sc->gfp_mask,
-								      &nr_soft_scanned);
-			sc->nr_reclaimed += nr_soft_reclaimed;
-			sc->nr_scanned += nr_soft_scanned;
-			/* need some check for avoid more shrink_zone() */
 		}
 
 		if (!first_pgdat)
 			first_pgdat = zone->zone_pgdat;
 
-		/* See comment about same check for global reclaim above */
+		/*
+		 * Shrink each node in the zonelist once. If the zonelist is
+		 * ordered by zone (not the default) then a node may be shrunk
+		 * multiple times but in that case the user prefers lower zones
+		 * being preserved.
+		 */
 		if (zone->zone_pgdat == last_pgdat)
 			continue;
 		last_pgdat = zone->zone_pgdat;
@@ -6811,47 +7005,6 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 
 #ifdef CONFIG_MEMCG
 
-/* Only used by soft limit reclaim. Do not reuse for anything else. */
-unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
-						gfp_t gfp_mask, bool noswap,
-						pg_data_t *pgdat,
-						unsigned long *nr_scanned)
-{
-	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
-	struct scan_control sc = {
-		.nr_to_reclaim = SWAP_CLUSTER_MAX,
-		.target_mem_cgroup = memcg,
-		.may_writepage = 1,
-		.may_unmap = 1,
-		.reclaim_idx = MAX_NR_ZONES - 1,
-		.may_swap = !noswap,
-	};
-
-	WARN_ON_ONCE(!current->reclaim_state);
-
-	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
-			(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
-
-	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.gfp_mask,
-						      sc.order,
-						      memcg);
-
-	/*
-	 * NOTE: Although we can get the priority field, using it
-	 * here is not a good idea, since it limits the pages we can scan.
-	 * if we don't reclaim here, the shrink_node from balance_pgdat
-	 * will pick up pages from other mem cgroup's as well. We hack
-	 * the priority and make it zero.
-	 */
-	shrink_lruvec(lruvec, &sc);
-
-	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed, memcg);
-
-	*nr_scanned = sc.nr_scanned;
-
-	return sc.nr_reclaimed;
-}
-
 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
 					   unsigned long nr_pages,
 					   gfp_t gfp_mask,
@@ -7157,8 +7310,6 @@ clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
 {
 	int i;
-	unsigned long nr_soft_reclaimed;
-	unsigned long nr_soft_scanned;
 	unsigned long pflags;
 	unsigned long nr_boost_reclaim;
 	unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
@@ -7264,12 +7415,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
 		 */
 		kswapd_age_node(pgdat, &sc);
 
-		/* Call soft limit reclaim before calling shrink_node. */
 		sc.nr_scanned = 0;
-		nr_soft_scanned = 0;
-		nr_soft_reclaimed = memcg1_soft_limit_reclaim(pgdat, sc.order,
-							      sc.gfp_mask, &nr_soft_scanned);
-		sc.nr_reclaimed += nr_soft_reclaimed;
 
 		/*
 		 * There should be no need to raise the scanning priority if
@@ -7508,7 +7654,7 @@ static int kswapd(void *p)
 	 * us from recursively trying to free more memory as we're
 	 * trying to free the first piece of memory in the first place).
 	 */
-	tsk->flags |= PF_MEMALLOC | PF_KSWAPD;
+	tsk->flags |= PF_MEMALLOC;
 	set_freezable();
 
 	WRITE_ONCE(pgdat->kswapd_order, 0);
@@ -7558,11 +7704,17 @@ static int kswapd(void *p)
 			goto kswapd_try_sleep;
 	}
 
-	tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD);
+	tsk->flags &= ~PF_MEMALLOC;
 
 	return 0;
 }
 
+bool current_is_kswapd(void)
+{
+	return kthread_func(current) == kswapd;
+}
+EXPORT_SYMBOL_GPL(current_is_kswapd);
+
 /*
  * A zone is low on free memory or too fragmented for high-order memory.  If
  * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
@@ -7851,16 +8003,9 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat,
 	noreclaim_flag = memalloc_noreclaim_save();
 	set_task_reclaim_state(p, &sc->reclaim_state);
 
-	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages ||
-	    node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) {
-		/*
-		 * Free memory by calling shrink node with increasing
-		 * priorities until we have enough memory freed.
-		 */
-		do {
-			shrink_node(pgdat, sc);
-		} while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0);
-	}
+	do {
+		shrink_node(pgdat, sc);
+	} while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0);
 
 	set_task_reclaim_state(p, NULL);
 	memalloc_noreclaim_restore(noreclaim_flag);
diff --git a/mm/vmstat.c b/mm/vmstat.c
index cb57714539fb5e..a3e809c57f295a 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1837,6 +1837,11 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
 		struct per_cpu_zonestat __maybe_unused *pzstats;
 
 		pcp = per_cpu_ptr(zone->per_cpu_pageset, i);
+		/*
+		 * Access to the per-cpu pageset fields is lockless as they
+		 * are used only for printing purposes. Use data_race to
+		 * avoid KCSAN warning.
+		 */
 		seq_printf(m,
 			   "\n    cpu: %i"
 			   "\n              count:    %i"
@@ -1845,15 +1850,15 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
 			   "\n              high_min: %i"
 			   "\n              high_max: %i",
 			   i,
-			   pcp->count,
-			   pcp->high,
-			   pcp->batch,
-			   pcp->high_min,
-			   pcp->high_max);
+			   data_race(pcp->count),
+			   data_race(pcp->high),
+			   data_race(pcp->batch),
+			   data_race(pcp->high_min),
+			   data_race(pcp->high_max));
 #ifdef CONFIG_SMP
 		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);
 		seq_printf(m, "\n  vm stats threshold: %d",
-				pzstats->stat_threshold);
+				data_race(pzstats->stat_threshold));
 #endif
 	}
 	seq_printf(m,
diff --git a/mm/workingset.c b/mm/workingset.c
index 7ac2b88c80ae56..1504f91cdca5f6 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -470,7 +470,7 @@ bool workingset_test_recent(void *shadow, bool file, bool *workingset,
 	 * configurations instead.
 	 */
 	eviction_memcg = mem_cgroup_from_private_id(memcgid);
-	if (!mem_cgroup_tryget(eviction_memcg))
+	if (eviction_memcg && !mem_cgroup_tryget(eviction_memcg))
 		eviction_memcg = NULL;
 	rcu_read_unlock();
 
@@ -688,10 +688,9 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker,
 
 		mem_cgroup_flush_stats_ratelimited(sc->memcg);
 		lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
-
 		for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
-			pages += lruvec_lru_size(lruvec, i, MAX_NR_ZONES - 1);
-
+			pages += lruvec_page_state_local(lruvec,
+							 NR_LRU_BASE + i);
 		pages += lruvec_page_state_local(
 			lruvec, NR_SLAB_RECLAIMABLE_B) >> PAGE_SHIFT;
 		pages += lruvec_page_state_local(
diff --git a/mm/zpdesc.h b/mm/zpdesc.h
index b8258dc78548d1..4fd81c2e80769f 100644
--- a/mm/zpdesc.h
+++ b/mm/zpdesc.h
@@ -26,8 +26,8 @@
  * with memcg_data.
  *
  * Page flags used:
- * * PG_private identifies the first component page.
  * * PG_locked is used by page migration code.
+ * The first component page has zpdesc->zspage->first_zpdesc == zpdesc
  */
 struct zpdesc {
 	unsigned long flags;
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 825022a7a328fe..7ef80e0da62676 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -290,11 +290,6 @@ struct zs_pool {
 	atomic_t compaction_in_progress;
 };
 
-static inline void zpdesc_set_first(struct zpdesc *zpdesc)
-{
-	SetPagePrivate(zpdesc_page(zpdesc));
-}
-
 static inline void zpdesc_inc_zone_page_state(struct zpdesc *zpdesc)
 {
 	inc_zone_page_state(zpdesc_page(zpdesc), NR_ZSPAGES);
@@ -476,11 +471,6 @@ static void record_obj(unsigned long handle, unsigned long obj)
 	WRITE_ONCE(*(unsigned long *)handle, obj);
 }
 
-static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc)
-{
-	return PagePrivate(zpdesc_page(zpdesc));
-}
-
 /* Protected by class->lock */
 static inline int get_zspage_inuse(struct zspage *zspage)
 {
@@ -496,7 +486,8 @@ static struct zpdesc *get_first_zpdesc(struct zspage *zspage)
 {
 	struct zpdesc *first_zpdesc = zspage->first_zpdesc;
 
-	VM_BUG_ON_PAGE(!is_first_zpdesc(first_zpdesc), zpdesc_page(first_zpdesc));
+	/* the first zpdesc must point back to this zspage */
+	VM_WARN_ON_ONCE_PAGE(first_zpdesc->zspage != zspage, zpdesc_page(first_zpdesc));
 	return first_zpdesc;
 }
 
@@ -838,7 +829,8 @@ static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj,
 	struct zspage *zspage = get_zspage(zpdesc);
 
 	if (unlikely(ZsHugePage(zspage))) {
-		VM_BUG_ON_PAGE(!is_first_zpdesc(zpdesc), zpdesc_page(zpdesc));
+		/* only first zpdesc holds the handle */
+		VM_WARN_ON_ONCE_PAGE(zspage->first_zpdesc != zpdesc, zpdesc_page(zpdesc));
 		handle = zpdesc->handle;
 	} else
 		handle = *(unsigned long *)obj;
@@ -853,9 +845,6 @@ static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj,
 
 static void reset_zpdesc(struct zpdesc *zpdesc)
 {
-	struct page *page = zpdesc_page(zpdesc);
-
-	ClearPagePrivate(page);
 	zpdesc->zspage = NULL;
 	zpdesc->next = NULL;
 	/* PageZsmalloc is sticky until the page is freed to the buddy. */
@@ -1006,8 +995,8 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage,
 	 * 1. all pages are linked together using zpdesc->next
 	 * 2. each sub-page point to zspage using zpdesc->zspage
 	 *
-	 * we set PG_private to identify the first zpdesc (i.e. no other zpdesc
-	 * has this flag set).
+	 * The first zpdesc has its zspage->first_zpdesc set to itself, no
+	 * other zpdesc has this set.
 	 */
 	for (i = 0; i < nr_zpdescs; i++) {
 		zpdesc = zpdescs[i];
@@ -1015,7 +1004,6 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage,
 		zpdesc->next = NULL;
 		if (i == 0) {
 			zspage->first_zpdesc = zpdesc;
-			zpdesc_set_first(zpdesc);
 			if (unlikely(class->objs_per_zspage == 1 &&
 					class->pages_per_zspage == 1))
 				SetZsHugePage(zspage);
@@ -1134,83 +1122,6 @@ unsigned long zs_get_total_pages(struct zs_pool *pool)
 }
 EXPORT_SYMBOL_GPL(zs_get_total_pages);
 
-void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle,
-			size_t mem_len, void *local_copy)
-{
-	struct zspage *zspage;
-	struct zpdesc *zpdesc;
-	unsigned long obj, off;
-	unsigned int obj_idx;
-	struct size_class *class;
-	void *addr;
-
-	/* Guarantee we can get zspage from handle safely */
-	read_lock(&pool->lock);
-	obj = handle_to_obj(handle);
-	obj_to_location(obj, &zpdesc, &obj_idx);
-	zspage = get_zspage(zpdesc);
-
-	/* Make sure migration doesn't move any pages in this zspage */
-	zspage_read_lock(zspage);
-	read_unlock(&pool->lock);
-
-	class = zspage_class(pool, zspage);
-	off = offset_in_page(class->size * obj_idx);
-
-	if (!ZsHugePage(zspage))
-		off += ZS_HANDLE_SIZE;
-
-	if (off + mem_len <= PAGE_SIZE) {
-		/* this object is contained entirely within a page */
-		addr = kmap_local_zpdesc(zpdesc);
-		addr += off;
-	} else {
-		size_t sizes[2];
-
-		/* this object spans two pages */
-		sizes[0] = PAGE_SIZE - off;
-		sizes[1] = mem_len - sizes[0];
-		addr = local_copy;
-
-		memcpy_from_page(addr, zpdesc_page(zpdesc),
-				 off, sizes[0]);
-		zpdesc = get_next_zpdesc(zpdesc);
-		memcpy_from_page(addr + sizes[0],
-				 zpdesc_page(zpdesc),
-				 0, sizes[1]);
-	}
-
-	return addr;
-}
-EXPORT_SYMBOL_GPL(zs_obj_read_begin);
-
-void zs_obj_read_end(struct zs_pool *pool, unsigned long handle,
-		     size_t mem_len, void *handle_mem)
-{
-	struct zspage *zspage;
-	struct zpdesc *zpdesc;
-	unsigned long obj, off;
-	unsigned int obj_idx;
-	struct size_class *class;
-
-	obj = handle_to_obj(handle);
-	obj_to_location(obj, &zpdesc, &obj_idx);
-	zspage = get_zspage(zpdesc);
-	class = zspage_class(pool, zspage);
-	off = offset_in_page(class->size * obj_idx);
-
-	if (!ZsHugePage(zspage))
-		off += ZS_HANDLE_SIZE;
-
-	if (off + mem_len <= PAGE_SIZE) {
-		handle_mem -= off;
-		kunmap_local(handle_mem);
-	}
-
-	zspage_read_unlock(zspage);
-}
-EXPORT_SYMBOL_GPL(zs_obj_read_end);
-
 void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle,
 			  struct scatterlist *sg, size_t mem_len)
 {
diff --git a/mm/zswap.c b/mm/zswap.c
index 37f34e406c8e3b..ae19e301fced7e 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -34,6 +34,7 @@
 #include <linux/writeback.h>
 #include <linux/pagemap.h>
 #include <linux/workqueue.h>
+#include <linux/xarray.h>
 #include <linux/list_lru.h>
 #include <linux/zsmalloc.h>
 
@@ -154,12 +155,23 @@ struct zswap_pool {
 	struct zs_pool *zs_pool;
 	struct crypto_acomp_ctx __percpu *acomp_ctx;
 	struct percpu_ref ref;
-	struct list_head list;
-	struct work_struct release_work;
+	struct rcu_work release_rwork;
 	struct hlist_node node;
+	u8 idx;
 	char tfm_name[CRYPTO_MAX_ALG_NAME];
 };
 
+/*
+ * Live pools keyed by id (1..ZSWAP_MAX_POOL_ID).  XA_FLAGS_ALLOC1 keeps id 0
+ * reserved so it is never handed to a live pool.  XA_FLAGS_LOCK_BH makes the
+ * xa_lock softirq-safe: it is taken from __zswap_pool_empty(), which runs from
+ * a percpu_ref release callback in softirq context.
+ */
+#define ZSWAP_FIRST_POOL_ID 1
+#define ZSWAP_MAX_POOL_ID U8_MAX
+static DEFINE_XARRAY_FLAGS(zswap_pools, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH);
+static struct zswap_pool __rcu *zswap_current_pool;
+
 /* Global LRU lists shared by all zswap pools. */
 static struct list_lru zswap_list_lru;
 
@@ -182,7 +194,7 @@ static struct shrinker *zswap_shrinker;
  *              writeback logic. The entry is only reclaimed by the writeback
  *              logic if referenced is unset. See comments in the shrinker
  *              section for context.
- * pool - the zswap_pool the entry's data is in
+ * pool_idx - id of the zswap_pool that the entry's data is in.
  * handle - zsmalloc allocation handle that stores the compressed page data
  * objcg - the obj_cgroup that the compressed memory is charged to
  * lru - handle to the pool's lru used to evict pages.
@@ -191,19 +203,25 @@ struct zswap_entry {
 	swp_entry_t swpentry;
 	unsigned int length;
 	bool referenced;
-	struct zswap_pool *pool;
+	u8 pool_idx;
 	unsigned long handle;
 	struct obj_cgroup *objcg;
 	struct list_head lru;
 };
 
+/*
+ * No RCU section is needed around the returned pointer: a stored entry pins
+ * its pool via percpu_ref (taken in zswap_store_page()), so the id cannot be
+ * reused under us.  Callers WARN and handle a NULL from a corrupt pool_idx.
+ */
+static struct zswap_pool *zswap_entry_pool(struct zswap_entry *entry)
+{
+	return xa_load(&zswap_pools, entry->pool_idx);
+}
+
 static struct xarray *zswap_trees[MAX_SWAPFILES];
 static unsigned int nr_zswap_trees[MAX_SWAPFILES];
 
-/* RCU-protected iteration */
-static LIST_HEAD(zswap_pools);
-/* protects zswap_pools list modification */
-static DEFINE_SPINLOCK(zswap_pools_lock);
 /* pool counter to provide unique names to zsmalloc */
 static atomic_t zswap_pools_count = ATOMIC_INIT(0);
 
@@ -228,10 +246,15 @@ static bool zswap_has_pool;
 /* One swap address space for each 64M swap space */
 #define ZSWAP_ADDRESS_SPACE_SHIFT 14
 #define ZSWAP_ADDRESS_SPACE_PAGES (1 << ZSWAP_ADDRESS_SPACE_SHIFT)
+
+static inline struct xarray *zswap_tree(int type, pgoff_t offset)
+{
+	return &zswap_trees[type][offset >> ZSWAP_ADDRESS_SPACE_SHIFT];
+}
+
 static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
 {
-	return &zswap_trees[swp_type(swp)][swp_offset(swp)
-		>> ZSWAP_ADDRESS_SPACE_SHIFT];
+	return zswap_tree(swp_type(swp), swp_offset(swp));
 }
 
 #define zswap_pool_debug(msg, p)			\
@@ -275,6 +298,7 @@ static struct zswap_pool *zswap_pool_create(char *compressor)
 	struct zswap_pool *pool;
 	char name[38]; /* 'zswap' + 32 char (max) num + \0 */
 	int ret, cpu;
+	u32 id;
 
 	if (!zswap_has_pool && !strcmp(compressor, ZSWAP_PARAM_UNSET))
 		return NULL;
@@ -320,12 +344,31 @@ static struct zswap_pool *zswap_pool_create(char *compressor)
 			      PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
 	if (ret)
 		goto ref_fail;
-	INIT_LIST_HEAD(&pool->list);
+
+	/*
+	 * Publish only after the pool is fully built, so lockless walkers
+	 * never see a half-initialized pool.  The _bh variant pairs with the
+	 * softirq-context xa_lock taken in __zswap_pool_empty().
+	 */
+	ret = xa_alloc_bh(&zswap_pools, &id, pool,
+			  XA_LIMIT(ZSWAP_FIRST_POOL_ID, ZSWAP_MAX_POOL_ID),
+			  GFP_KERNEL);
+	if (ret) {
+		if (ret == -EBUSY)
+			pr_err("cannot allocate pool id (max %d live pools)\n",
+			       ZSWAP_MAX_POOL_ID - ZSWAP_FIRST_POOL_ID + 1);
+		goto xa_fail;
+	}
+	pool->idx = id;
 
 	zswap_pool_debug("created", pool);
 
+	static_branch_enable(&zswap_ever_enabled);
+
 	return pool;
 
+xa_fail:
+	percpu_ref_exit(&pool->ref);
 ref_fail:
 	cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
 
@@ -379,37 +422,29 @@ static void zswap_pool_destroy(struct zswap_pool *pool)
 
 static void __zswap_pool_release(struct work_struct *work)
 {
-	struct zswap_pool *pool = container_of(work, typeof(*pool),
-						release_work);
-
-	synchronize_rcu();
+	struct zswap_pool *pool = container_of(to_rcu_work(work),
+					       typeof(*pool), release_rwork);
 
 	/* nobody should have been able to get a ref... */
 	WARN_ON(!percpu_ref_is_zero(&pool->ref));
 	percpu_ref_exit(&pool->ref);
 
-	/* pool is now off zswap_pools list and has no references. */
+	/* The pool is no longer in zswap_pools and has no references. */
 	zswap_pool_destroy(pool);
 }
 
-static struct zswap_pool *zswap_pool_current(void);
-
 static void __zswap_pool_empty(struct percpu_ref *ref)
 {
 	struct zswap_pool *pool;
 
 	pool = container_of(ref, typeof(*pool), ref);
 
-	spin_lock_bh(&zswap_pools_lock);
-
-	WARN_ON(pool == zswap_pool_current());
-
-	list_del_rcu(&pool->list);
+	WARN_ON(pool == rcu_access_pointer(zswap_current_pool));
 
-	INIT_WORK(&pool->release_work, __zswap_pool_release);
-	schedule_work(&pool->release_work);
+	xa_erase_bh(&zswap_pools, pool->idx);
 
-	spin_unlock_bh(&zswap_pools_lock);
+	INIT_RCU_WORK(&pool->release_rwork, __zswap_pool_release);
+	queue_rcu_work(system_percpu_wq, &pool->release_rwork);
 }
 
 static int __must_check zswap_pool_tryget(struct zswap_pool *pool)
@@ -435,20 +470,13 @@ static struct zswap_pool *__zswap_pool_current(void)
 {
 	struct zswap_pool *pool;
 
-	pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
+	pool = rcu_dereference(zswap_current_pool);
 	WARN_ONCE(!pool && zswap_has_pool,
 		  "%s: no page storage pool!\n", __func__);
 
 	return pool;
 }
 
-static struct zswap_pool *zswap_pool_current(void)
-{
-	assert_spin_locked(&zswap_pools_lock);
-
-	return __zswap_pool_current();
-}
-
 static struct zswap_pool *zswap_pool_current_get(void)
 {
 	struct zswap_pool *pool;
@@ -464,23 +492,28 @@ static struct zswap_pool *zswap_pool_current_get(void)
 	return pool;
 }
 
-/* type and compressor must be null-terminated */
+/* compressor must be null-terminated */
 static struct zswap_pool *zswap_pool_find_get(char *compressor)
 {
 	struct zswap_pool *pool;
+	unsigned long id;
 
-	assert_spin_locked(&zswap_pools_lock);
-
-	list_for_each_entry_rcu(pool, &zswap_pools, list) {
+	/*
+	 * __zswap_pool_empty() can erase from zswap_pools in softirq while we
+	 * walk.  rcu_read_lock() keeps the walk consistent and each pool alive
+	 * across tryget().  xa_for_each()'s own RCU does not span the loop body.
+	 */
+	rcu_read_lock();
+	xa_for_each(&zswap_pools, id, pool) {
 		if (strcmp(pool->tfm_name, compressor))
 			continue;
 		/* if we can't get it, it's about to be destroyed */
-		if (!zswap_pool_tryget(pool))
-			continue;
-		return pool;
+		if (zswap_pool_tryget(pool))
+			break;
 	}
+	rcu_read_unlock();
 
-	return NULL;
+	return pool;
 }
 
 static unsigned long zswap_max_pages(void)
@@ -497,9 +530,14 @@ unsigned long zswap_total_pages(void)
 {
 	struct zswap_pool *pool;
 	unsigned long total = 0;
+	unsigned long id;
 
+	/*
+	 * rcu_read_lock() keeps each pool alive across zs_get_total_pages().
+	 * xa_for_each()'s own RCU does not span the loop body.
+	 */
 	rcu_read_lock();
-	list_for_each_entry_rcu(pool, &zswap_pools, list)
+	xa_for_each(&zswap_pools, id, pool)
 		total += zs_get_total_pages(pool->zs_pool);
 	rcu_read_unlock();
 
@@ -556,20 +594,13 @@ static int zswap_compressor_param_set(const char *val, const struct kernel_param
 		return -ENOENT;
 	}
 
-	spin_lock_bh(&zswap_pools_lock);
-
 	pool = zswap_pool_find_get(s);
-	if (pool) {
+	if (!pool) {
+		pool = zswap_pool_create(s);
+	} else {
 		zswap_pool_debug("using existing", pool);
-		WARN_ON(pool == zswap_pool_current());
-		list_del_rcu(&pool->list);
-	}
-
-	spin_unlock_bh(&zswap_pools_lock);
+		WARN_ON(pool == rcu_access_pointer(zswap_current_pool));
 
-	if (!pool)
-		pool = zswap_pool_create(s);
-	else {
 		/*
 		 * Restore the initial ref dropped by percpu_ref_kill()
 		 * when the pool was decommissioned and switch it again
@@ -586,24 +617,18 @@ static int zswap_compressor_param_set(const char *val, const struct kernel_param
 	else
 		ret = -EINVAL;
 
-	spin_lock_bh(&zswap_pools_lock);
-
+	/*
+	 * Compressor switches are serialized by the kernel param lock, so this
+	 * is the only writer of zswap_current_pool: no xa_lock needed.
+	 */
 	if (!ret) {
-		put_pool = zswap_pool_current();
-		list_add_rcu(&pool->list, &zswap_pools);
+		put_pool = rcu_access_pointer(zswap_current_pool);
+		rcu_assign_pointer(zswap_current_pool, pool);
 		zswap_has_pool = true;
 	} else if (pool) {
-		/*
-		 * Add the possibly pre-existing pool to the end of the pools
-		 * list; if it's new (and empty) then it'll be removed and
-		 * destroyed by the put after we drop the lock
-		 */
-		list_add_tail_rcu(&pool->list, &zswap_pools);
 		put_pool = pool;
 	}
 
-	spin_unlock_bh(&zswap_pools_lock);
-
 	/*
 	 * Drop the ref from either the old current pool,
 	 * or the new pool we failed to add
@@ -647,19 +672,6 @@ static int zswap_enabled_param_set(const char *val,
 * lru functions
 **********************************/
 
-/* should be called under RCU */
-#ifdef CONFIG_MEMCG
-static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
-{
-	return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL;
-}
-#else
-static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
-{
-	return NULL;
-}
-#endif
-
 static inline int entry_to_nid(struct zswap_entry *entry)
 {
 	return page_to_nid(virt_to_page(entry));
@@ -682,7 +694,7 @@ static void zswap_lru_add(struct zswap_entry *entry)
 	 * Similar reasoning holds for list_lru_del().
 	 */
 	rcu_read_lock();
-	memcg = mem_cgroup_from_entry(entry);
+	memcg = obj_cgroup_memcg(entry->objcg);
 	/* will always succeed */
 	list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg);
 	rcu_read_unlock();
@@ -694,7 +706,7 @@ static void zswap_lru_del(struct zswap_entry *entry)
 	struct mem_cgroup *memcg;
 
 	rcu_read_lock();
-	memcg = mem_cgroup_from_entry(entry);
+	memcg = obj_cgroup_memcg(entry->objcg);
 	/* will always succeed */
 	list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg);
 	rcu_read_unlock();
@@ -764,9 +776,13 @@ static void zswap_entry_cache_free(struct zswap_entry *entry)
  */
 static void zswap_entry_free(struct zswap_entry *entry)
 {
+	struct zswap_pool *pool = zswap_entry_pool(entry);
+
 	zswap_lru_del(entry);
-	zs_free(entry->pool->zs_pool, entry->handle);
-	zswap_pool_put(entry->pool);
+	if (!WARN_ON_ONCE(!pool)) {
+		zs_free(pool->zs_pool, entry->handle);
+		zswap_pool_put(pool);
+	}
 	if (entry->objcg) {
 		obj_cgroup_uncharge_zswap(entry->objcg, entry->length);
 		obj_cgroup_put(entry->objcg);
@@ -837,8 +853,8 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
 	return ret;
 }
 
-static bool zswap_compress(struct page *page, struct zswap_entry *entry,
-			   struct zswap_pool *pool)
+static bool zswap_compress(struct folio *folio, long index,
+			   struct zswap_entry *entry, struct zswap_pool *pool)
 {
 	struct crypto_acomp_ctx *acomp_ctx;
 	struct scatterlist input, output;
@@ -854,7 +870,7 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry,
 
 	dst = acomp_ctx->buffer;
 	sg_init_table(&input, 1);
-	sg_set_page(&input, page, PAGE_SIZE, 0);
+	sg_set_folio(&input, folio, PAGE_SIZE, index * PAGE_SIZE);
 
 	sg_init_one(&output, dst, PAGE_SIZE);
 	acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
@@ -883,8 +899,7 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry,
 	 */
 	if (comp_ret || !dlen || dlen >= PAGE_SIZE) {
 		rcu_read_lock();
-		if (!mem_cgroup_zswap_writeback_enabled(
-					folio_memcg(page_folio(page)))) {
+		if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio))) {
 			rcu_read_unlock();
 			comp_ret = comp_ret ? comp_ret : -EINVAL;
 			goto unlock;
@@ -892,12 +907,12 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry,
 		rcu_read_unlock();
 		comp_ret = 0;
 		dlen = PAGE_SIZE;
-		dst = kmap_local_page(page);
+		dst = kmap_local_folio(folio, index * PAGE_SIZE);
 		mapped = true;
 	}
 
 	gfp = GFP_NOWAIT | __GFP_NORETRY | __GFP_HIGHMEM | __GFP_MOVABLE;
-	handle = zs_malloc(pool->zs_pool, dlen, gfp, page_to_nid(page));
+	handle = zs_malloc(pool->zs_pool, dlen, gfp, folio_nid(folio));
 	if (IS_ERR_VALUE(handle)) {
 		alloc_ret = PTR_ERR((void *)handle);
 		goto unlock;
@@ -923,12 +938,15 @@ static bool zswap_compress(struct page *page, struct zswap_entry *entry,
 
 static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio)
 {
-	struct zswap_pool *pool = entry->pool;
+	struct zswap_pool *pool = zswap_entry_pool(entry);
 	struct scatterlist input[2]; /* zsmalloc returns an SG list 1-2 entries */
 	struct scatterlist output;
 	struct crypto_acomp_ctx *acomp_ctx;
 	int ret = 0, dlen;
 
+	if (WARN_ON_ONCE(!pool))
+		return false;
+
 	acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
 	mutex_lock(&acomp_ctx->mutex);
 	zs_obj_read_sg_begin(pool->zs_pool, entry->handle, input, entry->length);
@@ -964,7 +982,7 @@ static bool zswap_decompress(struct zswap_entry *entry, struct folio *folio)
 	pr_alert_ratelimited("Decompression error from zswap (%d:%lu %s %u->%d)\n",
 						swp_type(entry->swpentry),
 						swp_offset(entry->swpentry),
-						entry->pool->tfm_name,
+						pool->tfm_name,
 						entry->length, dlen);
 	return false;
 }
@@ -997,12 +1015,12 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 
 	/* try to allocate swap cache folio */
 	si = get_swap_device(swpentry);
-	if (!si)
-		return -EEXIST;
+	if (IS_ERR_OR_NULL(si))
+		return -ENOENT;
 
 	mpol = get_task_policy(current);
-	folio = swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol,
-				       NO_INTERLEAVE_INDEX);
+	folio = __swap_cache_alloc_folio(swpentry, GFP_KERNEL, BIT(0), NULL, mpol,
+					 NO_INTERLEAVE_INDEX);
 	put_swap_device(si);
 
 	/*
@@ -1015,6 +1033,8 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	if (IS_ERR(folio))
 		return PTR_ERR(folio);
 
+	folio_clear_active(folio);
+
 	/*
 	 * folio is locked, and the swapcache is now secured against
 	 * concurrent swapping to and from the slot, and concurrent
@@ -1027,12 +1047,12 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	tree = swap_zswap_tree(swpentry);
 	if (entry != xa_load(tree, offset)) {
 		ret = -ENOMEM;
-		goto out;
+		goto err;
 	}
 
 	if (!zswap_decompress(entry, folio)) {
 		ret = -EIO;
-		goto out;
+		goto err;
 	}
 
 	xa_erase(tree, offset);
@@ -1046,18 +1066,30 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	/* folio is up to date */
 	folio_mark_uptodate(folio);
 
-	/* move it to the tail of the inactive list after end_writeback */
-	folio_set_reclaim(folio);
+	folio_set_dropbehind(folio);
+
+	/*
+	 * Drop our reference before starting writeback so the swap cache holds
+	 * the only one: the drop in folio_end_writeback() needs that for
+	 * remove_mapping_set_shadow() to succeed, otherwise the folio is
+	 * handed back to reclaim instead.
+	 *
+	 * Nothing can free the folio in the meantime: we hold the folio lock
+	 * until writeback starts, PG_writeback then blocks swap cache removal,
+	 * and folio_end_writeback() takes its own reference before clearing
+	 * PG_writeback and donates it to the drop.
+	 */
+	folio_put(folio);
 
 	/* start writeback */
-	__swap_writepage(&ctx, folio);
+	__swap_writeout(&ctx, folio);
 	swap_write_submit(&ctx);
 
-out:
-	if (ret) {
-		swap_cache_del_folio(folio);
-		folio_unlock(folio);
-	}
+	return 0;
+
+err:
+	swap_cache_del_folio(folio);
+	folio_unlock(folio);
 	folio_put(folio);
 	return ret;
 }
@@ -1405,23 +1437,30 @@ static void shrink_worker(struct work_struct *w)
 * main API
 **********************************/
 
-static bool zswap_store_page(struct page *page,
+static bool zswap_store_page(struct folio *folio, long index,
 			     struct obj_cgroup *objcg,
 			     struct zswap_pool *pool)
 {
-	swp_entry_t page_swpentry = page_swap_entry(page);
+	swp_entry_t page_swpentry = folio_swap_entry(folio, index);
 	struct zswap_entry *entry, *old;
 
 	/* allocate entry */
-	entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
+	entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio));
 	if (!entry) {
 		zswap_reject_kmemcache_fail++;
 		return false;
 	}
 
-	if (!zswap_compress(page, entry, pool))
+	if (!zswap_compress(folio, index, entry, pool))
 		goto compress_failed;
 
+	/*
+	 * Set pool_idx before the xa_store() below publishes the entry, or a
+	 * concurrent reader could resolve a stale pool_idx left by slab reuse
+	 * to an unrelated live pool.
+	 */
+	entry->pool_idx = pool->idx;
+
 	old = xa_store(swap_zswap_tree(page_swpentry),
 		       swp_offset(page_swpentry),
 		       entry, GFP_KERNEL);
@@ -1467,7 +1506,6 @@ static bool zswap_store_page(struct page *page,
 	 *    The publishing order matters to prevent writeback from seeing
 	 *    an incoherent entry.
 	 */
-	entry->pool = pool;
 	entry->swpentry = page_swpentry;
 	entry->objcg = objcg;
 	entry->referenced = true;
@@ -1528,9 +1566,7 @@ bool zswap_store(struct folio *folio)
 	}
 
 	for (index = 0; index < nr_pages; ++index) {
-		struct page *page = folio_page(folio, index);
-
-		if (!zswap_store_page(page, objcg, pool))
+		if (!zswap_store_page(folio, index, objcg, pool))
 			goto put_pool;
 	}
 
@@ -1554,23 +1590,38 @@ bool zswap_store(struct folio *folio)
 	 * offsets corresponding to each page of the folio. Otherwise,
 	 * writeback could overwrite the new data in the swapfile.
 	 */
-	if (!ret) {
-		unsigned type = swp_type(swp);
-		pgoff_t offset = swp_offset(swp);
-		struct zswap_entry *entry;
-		struct xarray *tree;
-
-		for (index = 0; index < nr_pages; ++index) {
-			tree = swap_zswap_tree(swp_entry(type, offset + index));
-			entry = xa_erase(tree, offset + index);
-			if (entry)
-				zswap_entry_free(entry);
-		}
-	}
+	if (!ret)
+		zswap_invalidate(swp_type(swp), swp_offset(swp), nr_pages);
 
 	return ret;
 }
 
+/**
+ * zswap_is_present() - is any slot in [entry, entry + nr) in zswap?
+ * @entry: base swap entry of the range
+ * @nr: number of contiguous slots to check
+ *
+ * Context: The caller must keep the range pinned, otherwise the answer can
+ * change under it.
+ * Return: true if at least one slot in the range is in zswap.
+ */
+static bool zswap_is_present(swp_entry_t entry, unsigned int nr)
+{
+	pgoff_t offset = swp_offset(entry);
+	struct xarray *tree = swap_zswap_tree(entry);
+	unsigned long index = offset;
+
+	/*
+	 * A pinned range is at most SWAPFILE_CLUSTER slots and is aligned to
+	 * its own size, so one tree covers all of it and a single lookup is
+	 * enough. Scanning only part of the range would report a false
+	 * "absent" and let the caller read a stale copy from the device.
+	 */
+	BUILD_BUG_ON(SWAPFILE_CLUSTER > ZSWAP_ADDRESS_SPACE_PAGES);
+
+	return xa_find(tree, &index, offset + nr - 1, XA_PRESENT);
+}
+
 /**
  * zswap_load() - load a folio from zswap
  * @folio: folio to load
@@ -1578,15 +1629,12 @@ bool zswap_store(struct folio *folio)
  * Return: 0 on success, with the folio unlocked and marked up-to-date, or one
  * of the following error codes:
  *
- *  -EIO: if the swapped out content was in zswap, but could not be loaded
- *  into the page due to a decompression failure. The folio is unlocked, but
- *  NOT marked up-to-date, so that an IO error is emitted (e.g. do_swap_page()
- *  will SIGBUS).
- *
- *  -EINVAL: if the swapped out content was in zswap, but the page belongs
- *  to a large folio, which is not supported by zswap. The folio is unlocked,
- *  but NOT marked up-to-date, so that an IO error is emitted (e.g.
- *  do_swap_page() will SIGBUS).
+ *  -EIO: if the swapped out content was in zswap but could not be handed
+ *  back, either because decompression failed or because a slot in a
+ *  large-folio range is still in zswap and zswap cannot reconstruct a large
+ *  folio from per-page entries. The folio is unlocked, but NOT marked
+ *  up-to-date, so that an IO error is emitted (e.g. do_swap_page() will
+ *  SIGBUS).
  *
  *  -ENOENT: if the swapped out content was not in zswap. The folio remains
  *  locked on return.
@@ -1605,13 +1653,18 @@ int zswap_load(struct folio *folio)
 		return -ENOENT;
 
 	/*
-	 * Large folios should not be swapped in while zswap is being used, as
-	 * they are not properly handled. Zswap does not properly load large
-	 * folios, and a large folio may only be partially in zswap.
+	 * A large folio can legitimately reach zswap_load() with its whole
+	 * range on the backing device, so scan the range rather than rejecting
+	 * it outright. The caller has pinned every slot, so zswap cannot start
+	 * a store or a writeback into the range while we look.
 	 */
-	if (WARN_ON_ONCE(folio_test_large(folio))) {
-		folio_unlock(folio);
-		return -EINVAL;
+	if (folio_test_large(folio)) {
+		if (WARN_ON_ONCE(zswap_is_present(swp,
+						  folio_nr_pages(folio)))) {
+			folio_unlock(folio);
+			return -EIO;
+		}
+		return -ENOENT;
 	}
 
 	entry = xa_load(tree, offset);
@@ -1644,18 +1697,25 @@ int zswap_load(struct folio *folio)
 	return 0;
 }
 
-void zswap_invalidate(swp_entry_t swp)
+void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries)
 {
-	pgoff_t offset = swp_offset(swp);
-	struct xarray *tree = swap_zswap_tree(swp);
 	struct zswap_entry *entry;
+	struct xarray *tree;
+	unsigned long i;
 
-	if (xa_empty(tree))
+	if (zswap_never_enabled())
 		return;
 
-	entry = xa_erase(tree, offset);
-	if (entry)
-		zswap_entry_free(entry);
+	for (i = 0; i < nr_entries; i++) {
+		tree = zswap_tree(type, offset + i);
+
+		if (xa_empty(tree))
+			continue;
+
+		entry = xa_erase(tree, offset + i);
+		if (entry)
+			zswap_entry_free(entry);
+	}
 }
 
 int zswap_swapon(int type, unsigned long nr_pages)
@@ -1803,9 +1863,8 @@ static int zswap_setup(void)
 	pool = __zswap_pool_create_fallback();
 	if (pool) {
 		pr_info("loaded using pool %s\n", pool->tfm_name);
-		list_add(&pool->list, &zswap_pools);
+		rcu_assign_pointer(zswap_current_pool, pool);
 		zswap_has_pool = true;
-		static_branch_enable(&zswap_ever_enabled);
 	} else {
 		pr_err("pool creation failed\n");
 		zswap_enabled = false;
diff --git a/net/6lowpan/debugfs.c b/net/6lowpan/debugfs.c
index 600b9563bfc534..d959b132c06704 100644
--- a/net/6lowpan/debugfs.c
+++ b/net/6lowpan/debugfs.c
@@ -21,10 +21,7 @@ static int lowpan_ctx_flag_active_set(void *data, u64 val)
 	if (val != 0 && val != 1)
 		return -EINVAL;
 
-	if (val)
-		set_bit(LOWPAN_IPHC_CTX_FLAG_ACTIVE, &ctx->flags);
-	else
-		clear_bit(LOWPAN_IPHC_CTX_FLAG_ACTIVE, &ctx->flags);
+	assign_bit(LOWPAN_IPHC_CTX_FLAG_ACTIVE, &ctx->flags, val);
 
 	return 0;
 }
@@ -46,10 +43,7 @@ static int lowpan_ctx_flag_c_set(void *data, u64 val)
 	if (val != 0 && val != 1)
 		return -EINVAL;
 
-	if (val)
-		set_bit(LOWPAN_IPHC_CTX_FLAG_COMPRESSION, &ctx->flags);
-	else
-		clear_bit(LOWPAN_IPHC_CTX_FLAG_COMPRESSION, &ctx->flags);
+	assign_bit(LOWPAN_IPHC_CTX_FLAG_COMPRESSION, &ctx->flags, val);
 
 	return 0;
 }
diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c
index 2d2efb877975d5..ec9a7cc8afc743 100644
--- a/net/8021q/vlan.c
+++ b/net/8021q/vlan.c
@@ -49,7 +49,7 @@ const char vlan_version[] = DRV_VERSION;
 static int vlan_group_prealloc_vid(struct vlan_group *vg,
 				   __be16 vlan_proto, u16 vlan_id)
 {
-	struct net_device **array;
+	struct net_device __rcu **array;
 	unsigned int vidx;
 	unsigned int size;
 	int pidx;
@@ -70,10 +70,8 @@ static int vlan_group_prealloc_vid(struct vlan_group *vg,
 	if (array == NULL)
 		return -ENOBUFS;
 
-	/* paired with smp_rmb() in __vlan_group_get_device() */
-	smp_wmb();
-
-	vg->vlan_devices_arrays[pidx][vidx] = array;
+	/* paired with smp_load_acquire() in __vlan_group_get_device() */
+	smp_store_release(&vg->vlan_devices_arrays[pidx][vidx], array);
 	return 0;
 }
 
diff --git a/net/8021q/vlan.h b/net/8021q/vlan.h
index c41caaf94095f3..4930e6e8610913 100644
--- a/net/8021q/vlan.h
+++ b/net/8021q/vlan.h
@@ -22,8 +22,8 @@ enum vlan_protos {
 struct vlan_group {
 	unsigned int		nr_vlan_devs;
 	struct hlist_node	hlist;	/* linked list */
-	struct net_device **vlan_devices_arrays[VLAN_PROTO_NUM]
-					       [VLAN_GROUP_ARRAY_SPLIT_PARTS];
+	struct net_device __rcu **vlan_devices_arrays[VLAN_PROTO_NUM]
+						     [VLAN_GROUP_ARRAY_SPLIT_PARTS];
 };
 
 struct vlan_info {
@@ -54,15 +54,13 @@ static inline struct net_device *__vlan_group_get_device(struct vlan_group *vg,
 							 unsigned int pidx,
 							 u16 vlan_id)
 {
-	struct net_device **array;
+	struct net_device __rcu **array;
 
-	array = vg->vlan_devices_arrays[pidx]
-				       [vlan_id / VLAN_GROUP_ARRAY_PART_LEN];
-
-	/* paired with smp_wmb() in vlan_group_prealloc_vid() */
-	smp_rmb();
+	/* Pairs with smp_store_release() in vlan_group_prealloc_vid() */
+	array = smp_load_acquire(&vg->vlan_devices_arrays[pidx]
+				 [vlan_id / VLAN_GROUP_ARRAY_PART_LEN]);
 
-	return array ? array[vlan_id % VLAN_GROUP_ARRAY_PART_LEN] : NULL;
+	return array ? rcu_dereference_raw(array[vlan_id % VLAN_GROUP_ARRAY_PART_LEN]) : NULL;
 }
 
 static inline struct net_device *vlan_group_get_device(struct vlan_group *vg,
@@ -82,13 +80,13 @@ static inline void vlan_group_set_device(struct vlan_group *vg,
 					 struct net_device *dev)
 {
 	int pidx = vlan_proto_idx(vlan_proto);
-	struct net_device **array;
+	struct net_device __rcu **array;
 
 	if (!vg || pidx < 0)
 		return;
 	array = vg->vlan_devices_arrays[pidx]
 				       [vlan_id / VLAN_GROUP_ARRAY_PART_LEN];
-	array[vlan_id % VLAN_GROUP_ARRAY_PART_LEN] = dev;
+	rcu_assign_pointer(array[vlan_id % VLAN_GROUP_ARRAY_PART_LEN], dev);
 }
 
 /* Must be invoked with rcu_read_lock or with RTNL. */
@@ -160,7 +158,7 @@ static inline u32 vlan_get_ingress_priority(struct net_device *dev,
 {
 	struct vlan_dev_priv *vip = vlan_dev_priv(dev);
 
-	return vip->ingress_priority_map[(vlan_tci >> VLAN_PRIO_SHIFT) & 0x7];
+	return READ_ONCE(vip->ingress_priority_map[(vlan_tci >> VLAN_PRIO_SHIFT) & 0x7]);
 }
 
 #ifdef CONFIG_VLAN_8021Q_GVRP
diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c
index d23965e76c167b..1fc6ebd3313218 100644
--- a/net/8021q/vlan_core.c
+++ b/net/8021q/vlan_core.c
@@ -38,7 +38,7 @@ bool vlan_do_receive(struct sk_buff **skbp)
 			skb->pkt_type = PACKET_HOST;
 	}
 
-	if (!(vlan_dev_priv(vlan_dev)->flags & VLAN_FLAG_REORDER_HDR) &&
+	if (!(READ_ONCE(vlan_dev_priv(vlan_dev)->flags) & VLAN_FLAG_REORDER_HDR) &&
 	    !netif_is_macvlan_port(vlan_dev) &&
 	    !netif_is_bridge_port(vlan_dev)) {
 		unsigned int offset = skb->data - skb_mac_header(skb);
diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
index 2859cbac3f266b..cb8f3cdbf1f732 100644
--- a/net/8021q/vlan_dev.c
+++ b/net/8021q/vlan_dev.c
@@ -54,7 +54,12 @@ static int vlan_dev_hard_header(struct sk_buff *skb, struct net_device *dev,
 	u16 vlan_tci = 0;
 	int rc;
 
-	if (!(vlan->flags & VLAN_FLAG_REORDER_HDR)) {
+	if (!(READ_ONCE(vlan->flags) & VLAN_FLAG_REORDER_HDR)) {
+		unsigned int hlen = READ_ONCE(dev->hard_header_len) +
+				    READ_ONCE(dev->needed_headroom);
+
+		if (skb_cow_head(skb, hlen) < 0)
+			return -ENOMEM;
 		vhdr = skb_push(skb, VLAN_HLEN);
 
 		vlan_tci = vlan->vlan_id;
@@ -110,7 +115,7 @@ static netdev_tx_t vlan_dev_hard_start_xmit(struct sk_buff *skb,
 	 * NOTE: THIS ASSUMES DIX ETHERNET, SPECIFICALLY NOT SUPPORTING
 	 * OTHER THINGS LIKE FDDI/TokenRing/802.3 SNAPs...
 	 */
-	if (vlan->flags & VLAN_FLAG_REORDER_HDR ||
+	if (READ_ONCE(vlan->flags) & VLAN_FLAG_REORDER_HDR ||
 	    veth->h_vlan_proto != vlan->vlan_proto) {
 		u16 vlan_tci;
 		vlan_tci = vlan->vlan_id;
@@ -159,13 +164,16 @@ void vlan_dev_set_ingress_priority(const struct net_device *dev,
 				   u32 skb_prio, u16 vlan_prio)
 {
 	struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
+	u32 *map = &vlan->ingress_priority_map[vlan_prio & 0x7];
 
-	if (vlan->ingress_priority_map[vlan_prio & 0x7] && !skb_prio)
-		vlan->nr_ingress_mappings--;
-	else if (!vlan->ingress_priority_map[vlan_prio & 0x7] && skb_prio)
-		vlan->nr_ingress_mappings++;
+	if (*map && !skb_prio)
+		WRITE_ONCE(vlan->nr_ingress_mappings,
+			   vlan->nr_ingress_mappings - 1);
+	else if (!*map && skb_prio)
+		WRITE_ONCE(vlan->nr_ingress_mappings,
+			   vlan->nr_ingress_mappings + 1);
 
-	vlan->ingress_priority_map[vlan_prio & 0x7] = skb_prio;
+	WRITE_ONCE(*map, skb_prio);
 }
 
 int vlan_dev_set_egress_priority(const struct net_device *dev,
@@ -185,7 +193,8 @@ int vlan_dev_set_egress_priority(const struct net_device *dev,
 		if (mp->priority == skb_prio) {
 			if (!vlan_qos) {
 				rcu_assign_pointer(*mpp, rtnl_dereference(mp->next));
-				vlan->nr_egress_mappings--;
+				WRITE_ONCE(vlan->nr_egress_mappings,
+					   vlan->nr_egress_mappings - 1);
 				kfree_rcu(mp, rcu);
 			} else {
 				WRITE_ONCE(mp->vlan_qos, vlan_qos);
@@ -209,7 +218,8 @@ int vlan_dev_set_egress_priority(const struct net_device *dev,
 	RCU_INIT_POINTER(np->next, rtnl_dereference(vlan->egress_priority_map[bucket]));
 	rcu_assign_pointer(vlan->egress_priority_map[bucket], np);
 	if (vlan_qos)
-		vlan->nr_egress_mappings++;
+		WRITE_ONCE(vlan->nr_egress_mappings,
+			   vlan->nr_egress_mappings + 1);
 	return 0;
 }
 
@@ -220,23 +230,25 @@ int vlan_dev_change_flags(const struct net_device *dev, u32 flags, u32 mask)
 {
 	struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
 	u32 old_flags = vlan->flags;
+	u32 new_flags;
 
 	if (mask & ~(VLAN_FLAG_REORDER_HDR | VLAN_FLAG_GVRP |
 		     VLAN_FLAG_LOOSE_BINDING | VLAN_FLAG_MVRP |
 		     VLAN_FLAG_BRIDGE_BINDING))
 		return -EINVAL;
 
-	vlan->flags = (old_flags & ~mask) | (flags & mask);
+	new_flags = (old_flags & ~mask) | (flags & mask);
+	WRITE_ONCE(vlan->flags, new_flags);
 
-	if (netif_running(dev) && (vlan->flags ^ old_flags) & VLAN_FLAG_GVRP) {
-		if (vlan->flags & VLAN_FLAG_GVRP)
+	if (netif_running(dev) && (new_flags ^ old_flags) & VLAN_FLAG_GVRP) {
+		if (new_flags & VLAN_FLAG_GVRP)
 			vlan_gvrp_request_join(dev);
 		else
 			vlan_gvrp_request_leave(dev);
 	}
 
-	if (netif_running(dev) && (vlan->flags ^ old_flags) & VLAN_FLAG_MVRP) {
-		if (vlan->flags & VLAN_FLAG_MVRP)
+	if (netif_running(dev) && (new_flags ^ old_flags) & VLAN_FLAG_MVRP) {
+		if (new_flags & VLAN_FLAG_MVRP)
 			vlan_mvrp_request_join(dev);
 		else
 			vlan_mvrp_request_leave(dev);
@@ -599,7 +611,7 @@ void vlan_dev_free_egress_priority(const struct net_device *dev)
 			pm = next;
 		}
 	}
-	vlan->nr_egress_mappings = 0;
+	WRITE_ONCE(vlan->nr_egress_mappings, 0);
 }
 
 static void vlan_dev_uninit(struct net_device *dev)
diff --git a/net/8021q/vlan_netlink.c b/net/8021q/vlan_netlink.c
index 368d53ca7d8709..8c0c33cff4539f 100644
--- a/net/8021q/vlan_netlink.c
+++ b/net/8021q/vlan_netlink.c
@@ -209,71 +209,83 @@ static inline size_t vlan_qos_map_size(unsigned int n)
 
 static size_t vlan_get_size(const struct net_device *dev)
 {
-	struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
+	const struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
 
 	return nla_total_size(2) +	/* IFLA_VLAN_PROTOCOL */
 	       nla_total_size(2) +	/* IFLA_VLAN_ID */
 	       nla_total_size(sizeof(struct ifla_vlan_flags)) + /* IFLA_VLAN_FLAGS */
-	       vlan_qos_map_size(vlan->nr_ingress_mappings) +
-	       vlan_qos_map_size(vlan->nr_egress_mappings);
+	       vlan_qos_map_size(READ_ONCE(vlan->nr_ingress_mappings)) +
+	       vlan_qos_map_size(READ_ONCE(vlan->nr_egress_mappings));
 }
 
 static int vlan_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
-	struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
-	struct vlan_priority_tci_mapping *pm;
-	struct ifla_vlan_flags f;
+	const struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
+	const struct vlan_priority_tci_mapping *pm;
 	struct ifla_vlan_qos_mapping m;
+	struct ifla_vlan_flags f;
 	struct nlattr *nest;
 	unsigned int i;
+	u32 flags;
 
 	if (nla_put_be16(skb, IFLA_VLAN_PROTOCOL, vlan->vlan_proto) ||
 	    nla_put_u16(skb, IFLA_VLAN_ID, vlan->vlan_id))
 		goto nla_put_failure;
-	if (vlan->flags) {
-		f.flags = vlan->flags;
+
+	flags = READ_ONCE(vlan->flags);
+	if (flags) {
+		f.flags = flags;
 		f.mask  = ~0;
 		if (nla_put(skb, IFLA_VLAN_FLAGS, sizeof(f), &f))
 			goto nla_put_failure;
 	}
-	if (vlan->nr_ingress_mappings) {
+
+	rcu_read_lock();
+
+	if (READ_ONCE(vlan->nr_ingress_mappings)) {
 		nest = nla_nest_start_noflag(skb, IFLA_VLAN_INGRESS_QOS);
-		if (nest == NULL)
-			goto nla_put_failure;
+		if (!nest)
+			goto nla_put_failure_unlock;
 
 		for (i = 0; i < ARRAY_SIZE(vlan->ingress_priority_map); i++) {
-			if (!vlan->ingress_priority_map[i])
+			u32 skb_prio = READ_ONCE(vlan->ingress_priority_map[i]);
+
+			if (!skb_prio)
 				continue;
 
 			m.from = i;
-			m.to   = vlan->ingress_priority_map[i];
+			m.to   = skb_prio;
 			if (nla_put(skb, IFLA_VLAN_QOS_MAPPING,
 				    sizeof(m), &m))
-				goto nla_put_failure;
+				goto nla_put_failure_unlock;
 		}
 		nla_nest_end(skb, nest);
 	}
 
-	if (vlan->nr_egress_mappings) {
+	if (READ_ONCE(vlan->nr_egress_mappings)) {
 		nest = nla_nest_start_noflag(skb, IFLA_VLAN_EGRESS_QOS);
-		if (nest == NULL)
-			goto nla_put_failure;
+		if (!nest)
+			goto nla_put_failure_unlock;
 
 		for (i = 0; i < ARRAY_SIZE(vlan->egress_priority_map); i++) {
-			for (pm = rcu_dereference_rtnl(vlan->egress_priority_map[i]); pm;
-			     pm = rcu_dereference_rtnl(pm->next)) {
+			for (pm = rcu_dereference(vlan->egress_priority_map[i]); pm;
+			     pm = rcu_dereference(pm->next)) {
 				u16 vlan_qos = READ_ONCE(pm->vlan_qos);
+
 				m.from = pm->priority;
 				m.to   = (vlan_qos >> 13) & 0x7;
 				if (nla_put(skb, IFLA_VLAN_QOS_MAPPING,
 					    sizeof(m), &m))
-					goto nla_put_failure;
+					goto nla_put_failure_unlock;
 			}
 		}
 		nla_nest_end(skb, nest);
 	}
+	rcu_read_unlock();
 	return 0;
 
+nla_put_failure_unlock:
+	rcu_read_unlock();
 nla_put_failure:
 	return -EMSGSIZE;
 }
diff --git a/net/8021q/vlanproc.c b/net/8021q/vlanproc.c
index 0e424e0895b7e8..5dd27438db9b39 100644
--- a/net/8021q/vlanproc.c
+++ b/net/8021q/vlanproc.c
@@ -240,7 +240,7 @@ static int vlandev_seq_show(struct seq_file *seq, void *offset)
 	seq_printf(seq,
 		   "%s  VID: %d	 REORDER_HDR: %i  dev->priv_flags: %x\n",
 		   vlandev->name, vlan->vlan_id,
-		   (int)(vlan->flags & 1), (u32)vlandev->priv_flags);
+		   (int)(READ_ONCE(vlan->flags) & 1), (u32)vlandev->priv_flags);
 
 	seq_printf(seq, fmt64, "total frames received", stats->rx_packets);
 	seq_printf(seq, fmt64, "total bytes received", stats->rx_bytes);
@@ -252,14 +252,14 @@ static int vlandev_seq_show(struct seq_file *seq, void *offset)
 	/* now show all PRIORITY mappings relating to this VLAN */
 	seq_printf(seq, "\nINGRESS priority mappings: "
 			"0:%u  1:%u  2:%u  3:%u  4:%u  5:%u  6:%u 7:%u\n",
-		   vlan->ingress_priority_map[0],
-		   vlan->ingress_priority_map[1],
-		   vlan->ingress_priority_map[2],
-		   vlan->ingress_priority_map[3],
-		   vlan->ingress_priority_map[4],
-		   vlan->ingress_priority_map[5],
-		   vlan->ingress_priority_map[6],
-		   vlan->ingress_priority_map[7]);
+		   READ_ONCE(vlan->ingress_priority_map[0]),
+		   READ_ONCE(vlan->ingress_priority_map[1]),
+		   READ_ONCE(vlan->ingress_priority_map[2]),
+		   READ_ONCE(vlan->ingress_priority_map[3]),
+		   READ_ONCE(vlan->ingress_priority_map[4]),
+		   READ_ONCE(vlan->ingress_priority_map[5]),
+		   READ_ONCE(vlan->ingress_priority_map[6]),
+		   READ_ONCE(vlan->ingress_priority_map[7]));
 
 	seq_printf(seq, " EGRESS priority mappings: ");
 	rcu_read_lock();
diff --git a/net/Kconfig b/net/Kconfig
index e384773935515d..76ab44aa439a6f 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -68,7 +68,6 @@ config SKB_EXTENSIONS
 
 config NET_DEVMEM
 	def_bool y
-	select GENERIC_ALLOCATOR
 	depends on DMA_SHARED_BUFFER
 	depends on PAGE_POOL
 
diff --git a/net/atm/proc.c b/net/atm/proc.c
index 8f20b49b9c02ab..ca0fc6b90998cf 100644
--- a/net/atm/proc.c
+++ b/net/atm/proc.c
@@ -159,7 +159,7 @@ static void vcc_info(struct seq_file *seq, struct atm_vcc *vcc)
 {
 	struct sock *sk = sk_atm(vcc);
 
-	seq_printf(seq, "%pK ", vcc);
+	seq_puts(seq, "      0 ");
 	if (!vcc->dev)
 		seq_printf(seq, "Unassigned    ");
 	else
@@ -228,9 +228,8 @@ static const struct seq_operations pvc_seq_ops = {
 static int vcc_seq_show(struct seq_file *seq, void *v)
 {
 	if (v == SEQ_START_TOKEN) {
-		seq_printf(seq, sizeof(void *) == 4 ? "%-8s%s" : "%-16s%s",
-			"Address ", "Itf VPI VCI   Fam Flags Reply "
-			"Send buffer     Recv buffer      [refcnt]\n");
+		seq_puts(seq, "Address Itf VPI VCI   Fam Flags Reply "
+			 "Send buffer     Recv buffer     [refcnt]\n");
 	} else {
 		struct vcc_state *state = seq->private;
 		struct atm_vcc *vcc = atm_sk(state->sk);
diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c
index 53fbdbbe8f4f70..89b499f7a37dd7 100644
--- a/net/batman-adv/bat_iv_ogm.c
+++ b/net/batman-adv/bat_iv_ogm.c
@@ -2422,27 +2422,28 @@ static int batadv_iv_ogm_neigh_cmp(struct batadv_neigh_node *neigh1,
 }
 
 /**
- * batadv_iv_ogm_neigh_is_sob() - check if neigh1 is similarly good or better
- *  than neigh2 from the metric prospective
- * @neigh1: the first neighbor object of the comparison
- * @if_outgoing1: outgoing interface for the first neighbor
- * @neigh2: the second neighbor object of the comparison
- * @if_outgoing2: outgoing interface for the second neighbor
+ * batadv_iv_ogm_neigh_is_sob() - check if @candidate is similarly good or
+ *  better than @reference from the metric prospective
+ * @candidate: the first neighbor object of the comparison
+ * @if_outgoing_cand: outgoing interface for the @candidate neighbor
+ * @reference: the second neighbor object of the comparison
+ * @if_outgoing_ref: outgoing interface for the @reference neighbor
  *
- * Return: true if the metric via neigh1 is equally good or better than
- * the metric via neigh2, false otherwise.
+ * Return: true if the metric via @candidate is equally good or better than
+ * the metric via @reference, false otherwise.
  */
 static bool
-batadv_iv_ogm_neigh_is_sob(struct batadv_neigh_node *neigh1,
-			   struct batadv_hard_iface *if_outgoing1,
-			   struct batadv_neigh_node *neigh2,
-			   struct batadv_hard_iface *if_outgoing2)
+batadv_iv_ogm_neigh_is_sob(struct batadv_neigh_node *candidate,
+			   struct batadv_hard_iface *if_outgoing_cand,
+			   struct batadv_neigh_node *reference,
+			   struct batadv_hard_iface *if_outgoing_ref)
 {
 	bool ret;
 	int diff;
 
-	ret = batadv_iv_ogm_neigh_diff(neigh1, if_outgoing1, neigh2,
-				       if_outgoing2, &diff);
+	ret = batadv_iv_ogm_neigh_diff(candidate, if_outgoing_cand,
+				       reference, if_outgoing_ref,
+				       &diff);
 	if (!ret)
 		return false;
 
diff --git a/net/batman-adv/bat_v.c b/net/batman-adv/bat_v.c
index 0c27447cf68895..596889b3522d05 100644
--- a/net/batman-adv/bat_v.c
+++ b/net/batman-adv/bat_v.c
@@ -514,40 +514,40 @@ static int batadv_v_neigh_cmp(struct batadv_neigh_node *neigh1,
 /**
  * batadv_v_neigh_is_sob() - check whether two B.A.T.M.A.N. V neighbours have
  *  a similar or better throughput
- * @neigh1: first neighbour to compare
- * @if_outgoing1: outgoing interface to use for @neigh1
- * @neigh2: second neighbour to compare
- * @if_outgoing2: outgoing interface to use for @neigh2
+ * @candidate: the first neighbor object of the comparison
+ * @if_outgoing_cand: outgoing interface for the @candidate neighbor
+ * @reference: the second neighbor object of the comparison
+ * @if_outgoing_ref: outgoing interface for the @reference neighbor
  *
- * Return: true if the throughput of @neigh2 is at least 3/4 of the
- *  @neigh1 throughput
+ * Return: true if the throughput of @candidate is more than 3/4 of the
+ *  @reference throughput
  */
-static bool batadv_v_neigh_is_sob(struct batadv_neigh_node *neigh1,
-				  struct batadv_hard_iface *if_outgoing1,
-				  struct batadv_neigh_node *neigh2,
-				  struct batadv_hard_iface *if_outgoing2)
+static bool batadv_v_neigh_is_sob(struct batadv_neigh_node *candidate,
+				  struct batadv_hard_iface *if_outgoing_cand,
+				  struct batadv_neigh_node *reference,
+				  struct batadv_hard_iface *if_outgoing_ref)
 {
-	struct batadv_neigh_ifinfo *ifinfo1;
-	struct batadv_neigh_ifinfo *ifinfo2;
+	struct batadv_neigh_ifinfo *ifinfo_cand;
+	struct batadv_neigh_ifinfo *ifinfo_ref;
 	bool ret = false;
 	u32 threshold;
 
-	ifinfo1 = batadv_neigh_ifinfo_get(neigh1, if_outgoing1);
-	if (!ifinfo1)
+	ifinfo_cand = batadv_neigh_ifinfo_get(candidate, if_outgoing_cand);
+	if (!ifinfo_cand)
 		goto err_ifinfo1;
 
-	ifinfo2 = batadv_neigh_ifinfo_get(neigh2, if_outgoing2);
-	if (!ifinfo2)
+	ifinfo_ref = batadv_neigh_ifinfo_get(reference, if_outgoing_ref);
+	if (!ifinfo_ref)
 		goto err_ifinfo2;
 
-	threshold = ifinfo1->bat_v.throughput / 4;
-	threshold = ifinfo1->bat_v.throughput - threshold;
+	threshold = ifinfo_ref->bat_v.throughput / 4;
+	threshold = ifinfo_ref->bat_v.throughput - threshold;
 
-	ret = ifinfo2->bat_v.throughput > threshold;
+	ret = ifinfo_cand->bat_v.throughput > threshold;
 
-	batadv_neigh_ifinfo_put(ifinfo2);
+	batadv_neigh_ifinfo_put(ifinfo_ref);
 err_ifinfo2:
-	batadv_neigh_ifinfo_put(ifinfo1);
+	batadv_neigh_ifinfo_put(ifinfo_cand);
 err_ifinfo1:
 	return ret;
 }
diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c
index f635da4b8ca650..ad6ab4a50658fc 100644
--- a/net/batman-adv/bridge_loop_avoidance.c
+++ b/net/batman-adv/bridge_loop_avoidance.c
@@ -663,7 +663,7 @@ static void batadv_bla_send_request(struct batadv_bla_backbone_gw *backbone_gw)
 static void batadv_bla_send_announce(struct batadv_priv *bat_priv,
 				     struct batadv_bla_backbone_gw *backbone_gw)
 {
-	u8 mac[ETH_ALEN];
+	u8 mac[ETH_ALEN] __aligned(2);
 	__be16 crc;
 
 	memcpy(mac, batadv_announce_mac, 4);
@@ -1582,7 +1582,9 @@ static struct lock_class_key batadv_backbone_hash_lock_class_key;
  */
 int batadv_bla_init(struct batadv_priv *bat_priv)
 {
-	u8 claim_dest[ETH_ALEN] = {0xff, 0x43, 0x05, 0x00, 0x00, 0x00};
+	static const u8 claim_dest[ETH_ALEN] __aligned(2) = {
+		0xff, 0x43, 0x05, 0x00, 0x00, 0x00
+	};
 	struct batadv_hard_iface *primary_if;
 	unsigned long entrytime;
 	u16 crc;
diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c
index 011cfdc47fab4c..a4be5b2a87d7a5 100644
--- a/net/batman-adv/distributed-arp-table.c
+++ b/net/batman-adv/distributed-arp-table.c
@@ -548,7 +548,7 @@ static void batadv_dbg_arp(struct batadv_priv *bat_priv, struct sk_buff *skb,
 			break;
 		default:
 			batadv_dbg(BATADV_DBG_DAT, bat_priv, "* type: Unknown (%u)!\n",
-				   unicast_4addr_packet->u.packet_type);
+				   unicast_4addr_packet->subtype);
 		}
 		break;
 	case BATADV_BCAST:
@@ -940,7 +940,7 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid,
 			    struct netlink_callback *cb,
 			    struct batadv_dat_entry *dat_entry)
 {
-	u8 mac[ETH_ALEN];
+	u8 mac[ETH_ALEN] __aligned(2);
 	u64 u64_mac;
 	int msecs;
 	void *hdr;
@@ -1236,10 +1236,10 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv,
 {
 	struct net_device *mesh_iface = bat_priv->mesh_iface;
 	struct batadv_dat_entry *dat_entry = NULL;
+	u8 mac[ETH_ALEN] __aligned(2);
 	struct sk_buff *skb_new;
 	unsigned short vid;
 	bool ret = false;
-	u8 mac[ETH_ALEN];
 	int hdr_size = 0;
 	__be32 ip_dst;
 	__be32 ip_src;
@@ -1344,10 +1344,10 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv,
 					   struct sk_buff *skb, int hdr_size)
 {
 	struct batadv_dat_entry *dat_entry = NULL;
+	u8 mac[ETH_ALEN] __aligned(2);
 	struct sk_buff *skb_new;
 	unsigned short vid;
 	bool ret = false;
-	u8 mac[ETH_ALEN];
 	__be32 ip_src;
 	__be32 ip_dst;
 	u64 u64_mac;
@@ -1474,9 +1474,9 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv,
 					 struct sk_buff *skb, int hdr_size)
 {
 	struct batadv_dat_entry *dat_entry = NULL;
+	u8 mac[ETH_ALEN] __aligned(2);
 	bool dropped = false;
 	unsigned short vid;
-	u8 mac[ETH_ALEN];
 	__be32 ip_src;
 	__be32 ip_dst;
 	u64 u64_mac;
diff --git a/net/batman-adv/hash.h b/net/batman-adv/hash.h
index e67afe01304cec..45a432cc15789e 100644
--- a/net/batman-adv/hash.h
+++ b/net/batman-adv/hash.h
@@ -137,7 +137,7 @@ static inline int batadv_hash_add(struct batadv_hashtable *hash,
  * data could be the structure you use with just the key filled, we just need
  * the key for comparing.
  *
- * Return: returns pointer to data on success, so you can remove the used
+ * Return: returns pointer to hlist_node on success, so you can remove the used
  * structure yourself, or NULL on error
  */
 static inline void *batadv_hash_remove(struct batadv_hashtable *hash,
diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c
index e202088cf1fd4b..63af21954cf903 100644
--- a/net/batman-adv/mesh-interface.c
+++ b/net/batman-adv/mesh-interface.c
@@ -141,9 +141,9 @@ static struct net_device_stats *batadv_interface_stats(struct net_device *dev)
 static int batadv_interface_set_mac_addr(struct net_device *dev, void *p)
 {
 	struct batadv_priv *bat_priv = netdev_priv(dev);
+	u8 old_addr[ETH_ALEN] __aligned(2);
 	struct batadv_meshif_vlan *vlan;
 	struct sockaddr *addr = p;
-	u8 old_addr[ETH_ALEN];
 
 	if (!is_valid_ether_addr(addr->sa_data))
 		return -EADDRNOTAVAIL;
@@ -213,13 +213,18 @@ static void batadv_interface_set_rx_mode(struct net_device *dev)
 static netdev_tx_t batadv_interface_tx(struct sk_buff *skb,
 				       struct net_device *mesh_iface)
 {
-	static const u8 ectp_addr[ETH_ALEN] = {0xCF, 0x00, 0x00, 0x00, 0x00, 0x00};
-	static const u8 stp_addr[ETH_ALEN] = {0x01, 0x80, 0xC2, 0x00, 0x00, 0x00};
 	struct batadv_priv *bat_priv = netdev_priv(mesh_iface);
 	enum batadv_dhcp_recipient dhcp_rcp = BATADV_DHCP_NO;
 	enum batadv_forw_mode forw_mode = BATADV_FORW_BCAST;
+	static const u8 ectp_addr[ETH_ALEN] __aligned(2) = {
+		0xCF, 0x00, 0x00, 0x00, 0x00, 0x00
+	};
+	static const u8 stp_addr[ETH_ALEN] __aligned(2) = {
+		0x01, 0x80, 0xC2, 0x00, 0x00, 0x00
+	};
 	struct batadv_hard_iface *primary_if = NULL;
 	struct batadv_bcast_packet *bcast_packet;
+	u8 chaddr[ETH_ALEN] __aligned(2);
 	int network_offset = ETH_HLEN;
 	unsigned int header_len = 0;
 	unsigned long brd_delay = 0;
@@ -229,7 +234,6 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb,
 	struct ethhdr *ethhdr;
 	bool do_bcast = false;
 	u8 *dst_hint = NULL;
-	u8 chaddr[ETH_ALEN];
 	unsigned short vid;
 	bool client_added;
 	__be16 proto;
diff --git a/net/batman-adv/multicast.c b/net/batman-adv/multicast.c
index 82fd527b61c02c..22dd28d5e1abcc 100644
--- a/net/batman-adv/multicast.c
+++ b/net/batman-adv/multicast.c
@@ -368,9 +368,9 @@ batadv_mcast_mla_meshif_get_ipv4(struct net_device *dev,
 				 struct hlist_head *mcast_list,
 				 struct batadv_mcast_mla_flags *flags)
 {
+	u8 mcast_addr[ETH_ALEN] __aligned(2);
 	struct batadv_hw_addr *new;
 	struct in_device *in_dev;
-	u8 mcast_addr[ETH_ALEN];
 	struct ip_mc_list *pmc;
 	int ret = 0;
 
@@ -435,9 +435,9 @@ batadv_mcast_mla_meshif_get_ipv6(struct net_device *dev,
 				 struct hlist_head *mcast_list,
 				 struct batadv_mcast_mla_flags *flags)
 {
+	u8 mcast_addr[ETH_ALEN] __aligned(2);
 	struct batadv_hw_addr *new;
 	struct inet6_dev *in6_dev;
-	u8 mcast_addr[ETH_ALEN];
 	struct ifmcaddr6 *pmc6;
 	int ret = 0;
 
@@ -587,10 +587,10 @@ static int batadv_mcast_mla_bridge_get(struct net_device *dev,
 				       struct batadv_mcast_mla_flags *flags)
 {
 	struct list_head bridge_mcast_list = LIST_HEAD_INIT(bridge_mcast_list);
+	u8 mcast_addr[ETH_ALEN] __aligned(2);
 	u8 tvlv_flags = flags->tvlv_flags;
 	struct br_ip_list *br_ip_entry;
 	struct batadv_hw_addr *new;
-	u8 mcast_addr[ETH_ALEN];
 	struct br_ip_list *tmp;
 	int ret;
 
diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c
index 68f79853032ed2..12ea557cb80334 100644
--- a/net/batman-adv/translation-table.c
+++ b/net/batman-adv/translation-table.c
@@ -13,6 +13,7 @@
 #include <linux/build_bug.h>
 #include <linux/byteorder/generic.h>
 #include <linux/cache.h>
+#include <linux/cleanup.h>
 #include <linux/compiler.h>
 #include <linux/container_of.h>
 #include <linux/crc32.h>
@@ -40,6 +41,7 @@
 #include <linux/string.h>
 #include <linux/workqueue.h>
 #include <net/genetlink.h>
+#include <net/net_namespace.h>
 #include <net/netlink.h>
 #include <uapi/linux/batadv_packet.h>
 #include <uapi/linux/batman_adv.h>
@@ -95,6 +97,26 @@ static bool batadv_compare_tt(const struct hlist_node *node, const void *data2)
 	return (tt1->vid == tt2->vid) && batadv_compare_eth(data1, data2);
 }
 
+/**
+ * batadv_compare_tt_entry() - check if a hash node is a specific TT entry
+ * @node: the list element pointer of the TT entry stored in the bucket
+ * @data2: pointer to the tt_common_entry which is looked for
+ *
+ * Unlike batadv_compare_tt(), this only matches the very object which is
+ * passed as @data2 and not just any entry for the same TT client. It is meant
+ * for batadv_hash_remove() callers which must not unlink an entry they did not
+ * look up themselves.
+ *
+ * Return: true if @node belongs to @data2, false otherwise
+ */
+static bool batadv_compare_tt_entry(const struct hlist_node *node,
+				    const void *data2)
+{
+	const struct batadv_tt_common_entry *tt = data2;
+
+	return node == &tt->hash_entry;
+}
+
 /**
  * batadv_choose_tt() - return the index of the tt entry in the hash table
  * @data: pointer to the tt_common_entry object to map
@@ -312,6 +334,10 @@ static void batadv_tt_local_size_mod(struct batadv_priv *bat_priv,
  *  given vid
  * @bat_priv: the bat priv with all the mesh interface information
  * @vid: the VLAN identifier
+ *
+ * It must only be called when removing the NEW flag of a
+ * batadv_tt_local_entry while it is still part of the bat_priv->tt.local_hash.
+ * It must therefore be checked under the specific list_locks[i].
  */
 static void batadv_tt_local_size_inc(struct batadv_priv *bat_priv,
 				     unsigned short vid)
@@ -324,6 +350,11 @@ static void batadv_tt_local_size_inc(struct batadv_priv *bat_priv,
  *  given vid
  * @bat_priv: the bat priv with all the mesh interface information
  * @vid: the VLAN identifier
+ *
+ * It must only be called after a batadv_tt_local_entry without NEW flag
+ * was removed from bat_priv->tt.local_hash (under the specific
+ * list_locks[i]) or when it is ensured that it is not accessible
+ * in this list.
  */
 static void batadv_tt_local_size_dec(struct batadv_priv *bat_priv,
 				     unsigned short vid)
@@ -413,6 +444,22 @@ batadv_tt_orig_list_entry_put(struct batadv_tt_orig_list_entry *orig_entry)
 	kref_put(&orig_entry->refcount, batadv_tt_orig_list_entry_release);
 }
 
+/**
+ * batadv_tt_flags_get() - get a snapshot of the flags of a TT entry
+ * @common: tt local & tt global common data
+ *
+ * Return: the flags of the TT entry as observed under its flags_lock.
+ */
+static u16 batadv_tt_flags_get(struct batadv_tt_common_entry *common)
+{
+	u16 flags;
+
+	scoped_guard(spinlock_bh, &common->flags_lock)
+		flags = common->flags;
+
+	return flags;
+}
+
 /**
  * batadv_tt_local_event() - store a local TT event (ADD/DEL)
  * @bat_priv: the bat priv with all the mesh interface information
@@ -425,23 +472,25 @@ static void batadv_tt_local_event(struct batadv_priv *bat_priv,
 {
 	struct batadv_tt_common_entry *common = &tt_local_entry->common;
 	struct batadv_tt_change_node *tt_change_node;
-	u8 flags = common->flags | event_flags;
 	struct batadv_tt_change_node *entry;
 	struct batadv_tt_change_node *safe;
 	bool del_op_requested;
 	bool del_op_entry;
 	size_t changes;
+	u8 flags;
 
 	tt_change_node = kmem_cache_alloc(batadv_tt_change_cache, GFP_ATOMIC);
 	if (!tt_change_node)
 		return;
 
-	tt_change_node->change.flags = flags;
 	memset(tt_change_node->change.reserved, 0,
 	       sizeof(tt_change_node->change.reserved));
 	ether_addr_copy(tt_change_node->change.addr, common->addr);
 	tt_change_node->change.vid = htons(common->vid);
 
+	flags = batadv_tt_flags_get(common) | event_flags;
+
+	tt_change_node->change.flags = flags;
 	del_op_requested = flags & BATADV_TT_CLIENT_DEL;
 
 	/* check for ADD+DEL, DEL+ADD, ADD+ADD or DEL+DEL events */
@@ -575,7 +624,6 @@ static void batadv_tt_global_free(struct batadv_priv *bat_priv,
 				  struct batadv_tt_global_entry *tt_global,
 				  const char *message)
 {
-	struct batadv_tt_global_entry *tt_removed_entry;
 	struct hlist_node *tt_removed_node;
 
 	batadv_dbg(BATADV_DBG_TT, bat_priv,
@@ -583,97 +631,168 @@ static void batadv_tt_global_free(struct batadv_priv *bat_priv,
 		   tt_global->common.addr,
 		   batadv_print_vid(tt_global->common.vid), message);
 
+	/* remove exactly this object when still present in hash */
 	tt_removed_node = batadv_hash_remove(bat_priv->tt.global_hash,
-					     batadv_compare_tt,
+					     batadv_compare_tt_entry,
 					     batadv_choose_tt,
 					     &tt_global->common);
 	if (!tt_removed_node)
 		return;
 
 	/* drop reference of remove hash entry */
-	tt_removed_entry = hlist_entry(tt_removed_node,
-				       struct batadv_tt_global_entry,
-				       common.hash_entry);
-	batadv_tt_global_entry_put(tt_removed_entry);
+	batadv_tt_global_entry_put(tt_global);
 }
 
 /**
- * batadv_tt_local_add() - add a new client to the local table or update an
- *  existing client
- * @mesh_iface: netdev struct of the mesh interface
- * @addr: the mac address of the client to add
- * @vid: VLAN identifier
- * @ifindex: index of the interface where the client is connected to (useful to
- *  identify wireless clients)
- * @mark: the value contained in the skb->mark field of the received packet (if
- *  any)
- *
- * Return: true if the client was successfully added, false otherwise.
+ * batadv_tt_local_add_roam() - handle roamed clients during batadv_tt_local_add()
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @tt_global: the global TT entry
+ * @roamed_back: whether @tt_global roamed back
  */
-bool batadv_tt_local_add(struct net_device *mesh_iface, const u8 *addr,
-			 unsigned short vid, int ifindex, u32 mark)
+static void batadv_tt_local_add_roam(struct batadv_priv *bat_priv,
+				     struct batadv_tt_global_entry *tt_global,
+				     bool roamed_back)
 {
-	struct batadv_priv *bat_priv = netdev_priv(mesh_iface);
-	struct batadv_tt_global_entry *tt_global = NULL;
 	struct batadv_tt_orig_list_entry *orig_entry;
-	struct batadv_tt_local_entry *tt_local;
-	struct net *net = dev_net(mesh_iface);
-	struct net_device *in_dev = NULL;
-	struct batadv_meshif_vlan *vlan;
-	bool roamed_back = false;
-	bool iif_is_wifi = false;
 	struct hlist_head *head;
-	int packet_size_max;
-	bool ret = false;
-	u8 remote_flags;
-	int hash_added;
-	int table_size;
-	u32 match_mark;
 
-	if (ifindex != BATADV_NULL_IFINDEX)
-		in_dev = dev_get_by_index(net, ifindex);
+	if (!tt_global)
+		return;
 
-	if (in_dev) {
-		u32 wifi_flags = batadv_netdev_get_wifi_flags(in_dev);
+	/* Check whether it is a roaming, but don't do anything if the roaming
+	 * process has already been handled
+	 */
+	scoped_guard(spinlock_bh, &tt_global->common.flags_lock) {
+		if (tt_global->common.flags & BATADV_TT_CLIENT_ROAM)
+			return;
 
-		iif_is_wifi = batadv_is_wifi(wifi_flags);
+		if (!roamed_back) {
+			/* The global entry has to be marked as ROAMING and has to be
+			 * kept for consistency purpose.
+			 *
+			 * batadv_tt_global_to_purge() evaluates roam_at as soon as it
+			 * observes BATADV_TT_CLIENT_ROAM, so the timeout has to be
+			 * stamped before the flag is published. Otherwise the entry can
+			 * be deleted right away as "Roaming timeout".
+			 */
+			tt_global->roam_at = jiffies;
+			tt_global->common.flags |= BATADV_TT_CLIENT_ROAM;
+		}
 	}
 
-	tt_local = batadv_tt_local_hash_find(bat_priv, addr, vid);
+	/* These node are probably going to update their tt table */
+	head = &tt_global->orig_list;
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(orig_entry, head, list) {
+		batadv_send_roam_adv(bat_priv, tt_global->common.addr,
+				     tt_global->common.vid,
+				     orig_entry->orig_node);
+	}
+	rcu_read_unlock();
 
-	if (!is_multicast_ether_addr(addr))
-		tt_global = batadv_tt_global_hash_find(bat_priv, addr, vid);
+	if (roamed_back)
+		batadv_tt_global_free(bat_priv, tt_global, "Roaming canceled");
+}
 
-	if (tt_local) {
-		tt_local->last_seen = jiffies;
-		if (tt_local->common.flags & BATADV_TT_CLIENT_PENDING) {
+/**
+ * batadv_tt_iif_is_wifi() - check whether a client is connected via wifi
+ * @net: namespace to search the incoming interface in
+ * @ifindex: index of the interface where the client is connected to
+ *
+ * Return: true if @ifindex refers to a wifi interface, false otherwise (which
+ * includes the case of an unknown or missing incoming interface).
+ */
+static bool batadv_tt_iif_is_wifi(struct net *net, int ifindex)
+{
+	struct net_device *in_dev;
+	u32 wifi_flags;
+
+	if (ifindex == BATADV_NULL_IFINDEX)
+		return false;
+
+	in_dev = dev_get_by_index(net, ifindex);
+	if (!in_dev)
+		return false;
+
+	wifi_flags = batadv_netdev_get_wifi_flags(in_dev);
+	dev_put(in_dev);
+
+	return batadv_is_wifi(wifi_flags);
+}
+
+/**
+ * batadv_tt_local_add_existing() - refresh an already known local TT entry
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @tt_local: the local TT entry which was found in the local table
+ * @roamed_back: set to true when the client returned to its original location
+ *
+ * Return: true when the client has to be announced to the mesh again, false
+ * otherwise.
+ */
+static bool batadv_tt_local_add_existing(struct batadv_priv *bat_priv,
+					 struct batadv_tt_local_entry *tt_local,
+					 bool *roamed_back)
+{
+	struct batadv_tt_common_entry *common = &tt_local->common;
+
+	tt_local->last_seen = jiffies;
+
+	scoped_guard(spinlock_bh, &common->flags_lock) {
+		if (common->flags & BATADV_TT_CLIENT_PENDING) {
 			batadv_dbg(BATADV_DBG_TT, bat_priv,
 				   "Re-adding pending client %pM (vid: %d)\n",
-				   addr, batadv_print_vid(vid));
+				   common->addr, batadv_print_vid(common->vid));
 			/* whatever the reason why the PENDING flag was set,
 			 * this is a client which was enqueued to be removed in
 			 * this orig_interval. Since it popped up again, the
 			 * flag can be reset like it was never enqueued
 			 */
-			tt_local->common.flags &= ~BATADV_TT_CLIENT_PENDING;
-			goto add_event;
+			common->flags &= ~BATADV_TT_CLIENT_PENDING;
+
+			return true;
 		}
 
-		if (tt_local->common.flags & BATADV_TT_CLIENT_ROAM) {
+		if (common->flags & BATADV_TT_CLIENT_ROAM) {
 			batadv_dbg(BATADV_DBG_TT, bat_priv,
 				   "Roaming client %pM (vid: %d) came back to its original location\n",
-				   addr, batadv_print_vid(vid));
+				   common->addr, batadv_print_vid(common->vid));
 			/* the ROAM flag is set because this client roamed away
 			 * and the node got a roaming_advertisement message. Now
 			 * that the client popped up again at its original
 			 * location such flag can be unset
 			 */
-			tt_local->common.flags &= ~BATADV_TT_CLIENT_ROAM;
-			roamed_back = true;
+			common->flags &= ~BATADV_TT_CLIENT_ROAM;
+			*roamed_back = true;
 		}
-		goto check_roaming;
 	}
 
+	return false;
+}
+
+/**
+ * batadv_tt_local_create() - allocate and initialize a local TT entry
+ * @mesh_iface: netdev struct of the mesh interface
+ * @addr: the mac address of the client to add
+ * @vid: VLAN identifier
+ * @iif_is_wifi: whether the client is connected via a wifi interface
+ *
+ * The returned entry is not yet part of bat_priv->tt.local_hash. It is marked
+ * as BATADV_TT_CLIENT_NEW to avoid sending it in a full table response going
+ * out before the next ttvn increment (consistency check).
+ *
+ * Return: the new entry with an initialized reference counter on success, NULL
+ * otherwise.
+ */
+static struct batadv_tt_local_entry *
+batadv_tt_local_create(struct net_device *mesh_iface, const u8 *addr,
+		       unsigned short vid, bool iif_is_wifi)
+{
+	struct batadv_priv *bat_priv = netdev_priv(mesh_iface);
+	struct batadv_tt_local_entry *tt_local;
+	struct batadv_meshif_vlan *vlan;
+	int packet_size_max;
+	int table_size;
+
 	/* Ignore the client if we cannot send it in a full table response. */
 	table_size = batadv_tt_local_table_transmit_size(bat_priv);
 	table_size += batadv_tt_len(1);
@@ -682,12 +801,12 @@ bool batadv_tt_local_add(struct net_device *mesh_iface, const u8 *addr,
 		net_ratelimited_function(batadv_info, mesh_iface,
 					 "Local translation table size (%i) exceeds maximum packet size (%i); Ignoring new local tt entry: %pM\n",
 					 table_size, packet_size_max, addr);
-		goto out;
+		return NULL;
 	}
 
 	tt_local = kmem_cache_alloc(batadv_tl_cache, GFP_ATOMIC);
 	if (!tt_local)
-		goto out;
+		return NULL;
 
 	/* increase the refcounter of the related vlan */
 	vlan = batadv_meshif_vlan_get(bat_priv, vid);
@@ -696,8 +815,7 @@ bool batadv_tt_local_add(struct net_device *mesh_iface, const u8 *addr,
 					 "adding TT local entry %pM to non-existent VLAN %d\n",
 					 addr, batadv_print_vid(vid));
 		kmem_cache_free(batadv_tl_cache, tt_local);
-		tt_local = NULL;
-		goto out;
+		return NULL;
 	}
 
 	batadv_dbg(BATADV_DBG_TT, bat_priv,
@@ -706,96 +824,146 @@ bool batadv_tt_local_add(struct net_device *mesh_iface, const u8 *addr,
 		   (u8)atomic_read(&bat_priv->tt.vn));
 
 	ether_addr_copy(tt_local->common.addr, addr);
-	/* The local entry has to be marked as NEW to avoid to send it in
-	 * a full table response going out before the next ttvn increment
-	 * (consistency check)
-	 */
-	tt_local->common.flags = BATADV_TT_CLIENT_NEW;
 	tt_local->common.vid = vid;
-	if (iif_is_wifi)
-		tt_local->common.flags |= BATADV_TT_CLIENT_WIFI;
 	kref_init(&tt_local->common.refcount);
 	tt_local->last_seen = jiffies;
 	tt_local->common.added_at = tt_local->last_seen;
 	tt_local->vlan = vlan;
+	spin_lock_init(&tt_local->common.flags_lock);
 
-	/* the batman interface mac and multicast addresses should never be
-	 * purged
-	 */
-	if (batadv_compare_eth(addr, mesh_iface->dev_addr) ||
-	    is_multicast_ether_addr(addr))
-		tt_local->common.flags |= BATADV_TT_CLIENT_NOPURGE;
-
-	kref_get(&tt_local->common.refcount);
-	hash_added = batadv_hash_add(bat_priv->tt.local_hash, batadv_compare_tt,
-				     batadv_choose_tt, &tt_local->common,
-				     &tt_local->common.hash_entry);
-
-	if (unlikely(hash_added != 0)) {
-		/* remove the reference for the hash */
-		batadv_tt_local_entry_put(tt_local);
-		goto out;
+	scoped_guard(spinlock_bh, &tt_local->common.flags_lock) {
+		tt_local->common.flags = BATADV_TT_CLIENT_NEW;
+		if (iif_is_wifi)
+			tt_local->common.flags |= BATADV_TT_CLIENT_WIFI;
+
+		/* the batman interface mac and multicast addresses should never
+		 * be purged
+		 */
+		if (batadv_compare_eth(addr, mesh_iface->dev_addr) ||
+		    is_multicast_ether_addr(addr))
+			tt_local->common.flags |= BATADV_TT_CLIENT_NOPURGE;
 	}
 
-add_event:
-	batadv_tt_local_event(bat_priv, tt_local, BATADV_NO_FLAGS);
+	return tt_local;
+}
 
-check_roaming:
-	/* Check whether it is a roaming, but don't do anything if the roaming
-	 * process has already been handled
-	 */
-	if (tt_global && !(tt_global->common.flags & BATADV_TT_CLIENT_ROAM)) {
-		/* These node are probably going to update their tt table */
-		head = &tt_global->orig_list;
-		rcu_read_lock();
-		hlist_for_each_entry_rcu(orig_entry, head, list) {
-			batadv_send_roam_adv(bat_priv, tt_global->common.addr,
-					     tt_global->common.vid,
-					     orig_entry->orig_node);
-		}
-		rcu_read_unlock();
-		if (roamed_back) {
-			batadv_tt_global_free(bat_priv, tt_global,
-					      "Roaming canceled");
-		} else {
-			/* The global entry has to be marked as ROAMING and
-			 * has to be kept for consistency purpose
-			 */
-			tt_global->common.flags |= BATADV_TT_CLIENT_ROAM;
-			tt_global->roam_at = jiffies;
-		}
+/**
+ * batadv_tt_local_update_flags() - update the dynamic flags of a local entry
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @tt_local: the local TT entry to update
+ * @iif_is_wifi: whether the client is connected via a wifi interface
+ * @mark: the value contained in the skb->mark field of the received packet (if
+ *  any)
+ *
+ * Return: true if a flag announced to the other nodes was modified, false
+ * otherwise.
+ */
+static bool
+batadv_tt_local_update_flags(struct batadv_priv *bat_priv,
+			     struct batadv_tt_local_entry *tt_local,
+			     bool iif_is_wifi, u32 mark)
+{
+	struct batadv_tt_common_entry *common = &tt_local->common;
+	u8 remote_flags;
+	u32 match_mark;
+	bool modified;
+
+	scoped_guard(spinlock_bh, &common->flags_lock) {
+		/* store the current remote flags before altering them. This
+		 * helps understanding is flags are changing or not
+		 */
+		remote_flags = common->flags & BATADV_TT_REMOTE_MASK;
+
+		if (iif_is_wifi)
+			common->flags |= BATADV_TT_CLIENT_WIFI;
+		else
+			common->flags &= ~BATADV_TT_CLIENT_WIFI;
+
+		/* check the mark in the skb: if it's equal to the configured
+		 * isolation_mark, it means the packet is coming from an
+		 * isolated non-mesh client
+		 */
+		match_mark = (mark & bat_priv->isolation_mark_mask);
+		if (bat_priv->isolation_mark_mask &&
+		    match_mark == bat_priv->isolation_mark)
+			common->flags |= BATADV_TT_CLIENT_ISOLA;
+		else
+			common->flags &= ~BATADV_TT_CLIENT_ISOLA;
+
+		modified = remote_flags ^ (common->flags & BATADV_TT_REMOTE_MASK);
 	}
 
-	/* store the current remote flags before altering them. This helps
-	 * understanding is flags are changing or not
-	 */
-	remote_flags = tt_local->common.flags & BATADV_TT_REMOTE_MASK;
+	return modified;
+}
 
-	if (iif_is_wifi)
-		tt_local->common.flags |= BATADV_TT_CLIENT_WIFI;
-	else
-		tt_local->common.flags &= ~BATADV_TT_CLIENT_WIFI;
+/**
+ * batadv_tt_local_add() - add a new client to the local table or update an
+ *  existing client
+ * @mesh_iface: netdev struct of the mesh interface
+ * @addr: the mac address of the client to add
+ * @vid: VLAN identifier
+ * @ifindex: index of the interface where the client is connected to (useful to
+ *  identify wireless clients)
+ * @mark: the value contained in the skb->mark field of the received packet (if
+ *  any)
+ *
+ * Return: true if the client was successfully added, false otherwise.
+ */
+bool batadv_tt_local_add(struct net_device *mesh_iface, const u8 *addr,
+			 unsigned short vid, int ifindex, u32 mark)
+{
+	struct batadv_priv *bat_priv = netdev_priv(mesh_iface);
+	struct batadv_tt_global_entry *tt_global = NULL;
+	struct batadv_tt_local_entry *tt_local;
+	bool roamed_back = false;
+	bool added = false;
+	bool iif_is_wifi;
+	bool ret = false;
+	int hash_added;
 
-	/* check the mark in the skb: if it's equal to the configured
-	 * isolation_mark, it means the packet is coming from an isolated
-	 * non-mesh client
-	 */
-	match_mark = (mark & bat_priv->isolation_mark_mask);
-	if (bat_priv->isolation_mark_mask &&
-	    match_mark == bat_priv->isolation_mark)
-		tt_local->common.flags |= BATADV_TT_CLIENT_ISOLA;
-	else
-		tt_local->common.flags &= ~BATADV_TT_CLIENT_ISOLA;
+	iif_is_wifi = batadv_tt_iif_is_wifi(dev_net(mesh_iface), ifindex);
+
+	tt_local = batadv_tt_local_hash_find(bat_priv, addr, vid);
+
+	if (!is_multicast_ether_addr(addr))
+		tt_global = batadv_tt_global_hash_find(bat_priv, addr, vid);
+
+	if (tt_local) {
+		added = batadv_tt_local_add_existing(bat_priv, tt_local,
+						     &roamed_back);
+	} else {
+		tt_local = batadv_tt_local_create(mesh_iface, addr, vid, iif_is_wifi);
+		if (!tt_local)
+			goto out;
+
+		kref_get(&tt_local->common.refcount);
+		hash_added = batadv_hash_add(bat_priv->tt.local_hash, batadv_compare_tt,
+					     batadv_choose_tt, &tt_local->common,
+					     &tt_local->common.hash_entry);
+
+		if (unlikely(hash_added != 0)) {
+			/* remove the reference for the hash */
+			batadv_tt_local_entry_put(tt_local);
+			goto out;
+		}
+
+		added = true;
+	}
+
+	/* announce the (re-)added client to the mesh */
+	if (added)
+		batadv_tt_local_event(bat_priv, tt_local, BATADV_NO_FLAGS);
+
+	batadv_tt_local_add_roam(bat_priv, tt_global, roamed_back);
 
 	/* if any "dynamic" flag has been modified, resend an ADD event for this
 	 * entry so that all the nodes can get the new flags
 	 */
-	if (remote_flags ^ (tt_local->common.flags & BATADV_TT_REMOTE_MASK))
+	if (batadv_tt_local_update_flags(bat_priv, tt_local, iif_is_wifi, mark))
 		batadv_tt_local_event(bat_priv, tt_local, BATADV_NO_FLAGS);
 
 	ret = true;
 out:
-	dev_put(in_dev);
 	batadv_tt_local_entry_put(tt_local);
 	batadv_tt_global_entry_put(tt_global);
 	return ret;
@@ -1115,6 +1283,7 @@ batadv_tt_local_dump_entry(struct sk_buff *msg, u32 portid,
 	struct batadv_meshif_vlan *vlan;
 	unsigned int last_seen_msecs;
 	void *hdr;
+	u16 flags;
 	u32 crc;
 
 	local = container_of(common, struct batadv_tt_local_entry, common);
@@ -1136,13 +1305,15 @@ batadv_tt_local_dump_entry(struct sk_buff *msg, u32 portid,
 
 	genl_dump_check_consistent(cb, hdr);
 
+	flags = batadv_tt_flags_get(common);
+
 	if (nla_put(msg, BATADV_ATTR_TT_ADDRESS, ETH_ALEN, common->addr) ||
 	    nla_put_u32(msg, BATADV_ATTR_TT_CRC32, crc) ||
 	    nla_put_u16(msg, BATADV_ATTR_TT_VID, common->vid) ||
-	    nla_put_u32(msg, BATADV_ATTR_TT_FLAGS, common->flags))
+	    nla_put_u32(msg, BATADV_ATTR_TT_FLAGS, flags))
 		goto nla_put_failure;
 
-	if (!(common->flags & BATADV_TT_CLIENT_NOPURGE) &&
+	if (!(flags & BATADV_TT_CLIENT_NOPURGE) &&
 	    nla_put_u32(msg, BATADV_ATTR_LAST_SEEN_MSECS, last_seen_msecs))
 		goto nla_put_failure;
 
@@ -1249,35 +1420,96 @@ int batadv_tt_local_dump(struct sk_buff *msg, struct netlink_callback *cb)
 }
 
 /**
- * batadv_tt_local_set_pending() - mark a local TT entry as pending removal
+ * batadv_tt_local_set_pending_event() - trigger events for TT pending removal
  * @bat_priv: the bat priv with all the mesh interface information
- * @tt_local_entry: local TT entry to mark
+ * @tt_local_entry: local TT entry which was marked as BATADV_TT_CLIENT_PENDING
  * @flags: TT change flags to announce together with the pending removal
  * @message: debug message describing the reason for the change
  *
- * Schedule the TT change announcement and set BATADV_TT_CLIENT_PENDING on the
- * entry. The entry is kept in the local table until the next TTVN increment
- * so that a consistency-check response can still be answered.
+ * Schedule the TT change announcement for the entry. The caller must already
+ * have added BATADV_TT_CLIENT_PENDING to the @tt_local_entry
  */
 static void
-batadv_tt_local_set_pending(struct batadv_priv *bat_priv,
-			    struct batadv_tt_local_entry *tt_local_entry,
-			    u16 flags, const char *message)
+batadv_tt_local_set_pending_event(struct batadv_priv *bat_priv,
+				  struct batadv_tt_local_entry *tt_local_entry,
+				  u16 flags, const char *message)
 {
 	batadv_tt_local_event(bat_priv, tt_local_entry, flags);
 
-	/* The local client has to be marked as "pending to be removed" but has
-	 * to be kept in the table in order to send it in a full table
-	 * response issued before the net ttvn increment (consistency check)
-	 */
-	tt_local_entry->common.flags |= BATADV_TT_CLIENT_PENDING;
-
 	batadv_dbg(BATADV_DBG_TT, bat_priv,
 		   "Local tt entry (%pM, vid: %d) pending to be removed: %s\n",
 		   tt_local_entry->common.addr,
 		   batadv_print_vid(tt_local_entry->common.vid), message);
 }
 
+/**
+ * batadv_tt_local_mark_removed() - mark a local entry as removed
+ * @tt_local_entry: local TT entry to mark
+ * @roaming: true if the deletion is due to a roaming event
+ * @curr_flags: pointer to store the flags of the entry before it was marked
+ *
+ * Return: true if the entry has to be kept in the local table until the next
+ * ttvn increment, false if it can be purged immediately.
+ */
+static bool
+batadv_tt_local_mark_removed(struct batadv_tt_local_entry *tt_local_entry,
+			     bool roaming, u16 *curr_flags)
+{
+	struct batadv_tt_common_entry *common = &tt_local_entry->common;
+	bool pending = false;
+
+	scoped_guard(spinlock_bh, &common->flags_lock) {
+		*curr_flags = common->flags;
+
+		/* mark the local client as ROAMed */
+		if (roaming)
+			common->flags |= BATADV_TT_CLIENT_ROAM;
+
+		if (!(common->flags & BATADV_TT_CLIENT_NEW)) {
+			common->flags |= BATADV_TT_CLIENT_PENDING;
+			pending = true;
+		}
+	}
+
+	return pending;
+}
+
+/**
+ * batadv_tt_local_remove_now() - purge a local entry which was never announced
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @tt_local_entry: local TT entry to purge
+ *
+ * A client which was added right after the last ttvn increment was never sent
+ * to the other nodes. It can therefore be dropped from the local table without
+ * waiting for the next ttvn increment.
+ */
+static void
+batadv_tt_local_remove_now(struct batadv_priv *bat_priv,
+			   struct batadv_tt_local_entry *tt_local_entry)
+{
+	struct batadv_tt_common_entry *common = &tt_local_entry->common;
+	struct hlist_node *tt_removed_node;
+
+	batadv_tt_local_event(bat_priv, tt_local_entry, BATADV_TT_CLIENT_DEL);
+
+	/* remove exactly this object when still present in hash */
+	tt_removed_node = batadv_hash_remove(bat_priv->tt.local_hash,
+					     batadv_compare_tt_entry,
+					     batadv_choose_tt, common);
+	if (!tt_removed_node)
+		return;
+
+	/* batadv_tt_local_transition_new() may have committed the entry and
+	 * thus counted it in the local table size since the
+	 * BATADV_TT_CLIENT_NEW check in batadv_tt_local_mark_removed().
+	 */
+	if (!(batadv_tt_flags_get(common) & BATADV_TT_CLIENT_NEW))
+		batadv_tt_local_size_dec(bat_priv, common->vid);
+
+	/* drop reference of remove hash entry */
+	batadv_tt_local_entry_put(tt_local_entry);
+}
+
 /**
  * batadv_tt_local_remove() - logically remove an entry from the local table
  * @bat_priv: the bat priv with all the mesh interface information
@@ -1292,53 +1524,31 @@ u16 batadv_tt_local_remove(struct batadv_priv *bat_priv, const u8 *addr,
 			   unsigned short vid, const char *message,
 			   bool roaming)
 {
-	struct batadv_tt_local_entry *tt_removed_entry;
 	struct batadv_tt_local_entry *tt_local_entry;
-	struct hlist_node *tt_removed_node;
-	u16 curr_flags = BATADV_NO_FLAGS;
+	u16 curr_flags;
 	u16 flags;
 
 	tt_local_entry = batadv_tt_local_hash_find(bat_priv, addr, vid);
 	if (!tt_local_entry)
-		goto out;
-
-	curr_flags = tt_local_entry->common.flags;
+		return BATADV_NO_FLAGS;
 
 	flags = BATADV_TT_CLIENT_DEL;
 	/* if this global entry addition is due to a roaming, the node has to
 	 * mark the local entry as "roamed" in order to correctly reroute
 	 * packets later
 	 */
-	if (roaming) {
+	if (roaming)
 		flags |= BATADV_TT_CLIENT_ROAM;
-		/* mark the local client as ROAMed */
-		tt_local_entry->common.flags |= BATADV_TT_CLIENT_ROAM;
-	}
-
-	if (!(tt_local_entry->common.flags & BATADV_TT_CLIENT_NEW)) {
-		batadv_tt_local_set_pending(bat_priv, tt_local_entry, flags,
-					    message);
-		goto out;
-	}
-	/* if this client has been added right now, it is possible to
-	 * immediately purge it
-	 */
-	batadv_tt_local_event(bat_priv, tt_local_entry, BATADV_TT_CLIENT_DEL);
 
-	tt_removed_node = batadv_hash_remove(bat_priv->tt.local_hash,
-					     batadv_compare_tt,
-					     batadv_choose_tt,
-					     &tt_local_entry->common);
-	if (!tt_removed_node)
-		goto out;
-
-	/* drop reference of remove hash entry */
-	tt_removed_entry = hlist_entry(tt_removed_node,
-				       struct batadv_tt_local_entry,
-				       common.hash_entry);
-	batadv_tt_local_entry_put(tt_removed_entry);
+	if (batadv_tt_local_mark_removed(tt_local_entry, roaming, &curr_flags))
+		batadv_tt_local_set_pending_event(bat_priv, tt_local_entry,
+						  flags, message);
+	else
+		/* if this client has been added right now, it is possible to
+		 * immediately purge it
+		 */
+		batadv_tt_local_remove_now(bat_priv, tt_local_entry);
 
-out:
 	batadv_tt_local_entry_put(tt_local_entry);
 
 	return curr_flags;
@@ -1361,21 +1571,37 @@ static void batadv_tt_local_purge_list(struct batadv_priv *bat_priv,
 
 	hlist_for_each_entry_safe(tt_common_entry, node_tmp, head,
 				  hash_entry) {
+		bool cont = false;
+
 		tt_local_entry = container_of(tt_common_entry,
 					      struct batadv_tt_local_entry,
 					      common);
-		if (tt_local_entry->common.flags & BATADV_TT_CLIENT_NOPURGE)
-			continue;
 
-		/* entry already marked for deletion */
-		if (tt_local_entry->common.flags & BATADV_TT_CLIENT_PENDING)
-			continue;
+		scoped_guard(spinlock_bh, &tt_local_entry->common.flags_lock) {
+			if (tt_local_entry->common.flags & BATADV_TT_CLIENT_NOPURGE) {
+				cont = true;
+				break;
+			}
+
+			/* entry already marked for deletion */
+			if (tt_local_entry->common.flags & BATADV_TT_CLIENT_PENDING) {
+				cont = true;
+				break;
+			}
+
+			if (!batadv_has_timed_out(tt_local_entry->last_seen, timeout)) {
+				cont = true;
+				break;
+			}
+
+			tt_local_entry->common.flags |= BATADV_TT_CLIENT_PENDING;
+		}
 
-		if (!batadv_has_timed_out(tt_local_entry->last_seen, timeout))
+		if (cont)
 			continue;
 
-		batadv_tt_local_set_pending(bat_priv, tt_local_entry,
-					    BATADV_TT_CLIENT_DEL, "timed out");
+		batadv_tt_local_set_pending_event(bat_priv, tt_local_entry,
+						  BATADV_TT_CLIENT_DEL, "timed out");
 	}
 }
 
@@ -1581,8 +1807,10 @@ batadv_tt_global_sync_flags(struct batadv_tt_global_entry *tt_global)
 		flags |= orig_entry->flags;
 	rcu_read_unlock();
 
-	flags |= tt_global->common.flags & (~BATADV_TT_SYNC_MASK);
-	tt_global->common.flags = flags;
+	scoped_guard(spinlock_bh, &tt_global->common.flags_lock) {
+		flags |= tt_global->common.flags & (~BATADV_TT_SYNC_MASK);
+		tt_global->common.flags = flags;
+	}
 }
 
 /**
@@ -1636,6 +1864,147 @@ batadv_tt_global_orig_entry_add(struct batadv_tt_global_entry *tt_global,
 	spin_unlock_bh(&tt_global->list_lock);
 }
 
+/**
+ * batadv_tt_global_purge_local() - drop the local entry of an announced client
+ * @bat_priv: the bat priv with all the mesh interface information
+ * @tt_global_entry: the global TT entry of the announced client
+ * @flags: TT flags announced for this non-mesh client
+ *
+ * A client which is announced by another originator is no longer a local
+ * client. Remove it from the local table and take over the WIFI flag it was
+ * tracked with.
+ */
+static void
+batadv_tt_global_purge_local(struct batadv_priv *bat_priv,
+			     struct batadv_tt_global_entry *tt_global_entry,
+			     u16 flags)
+{
+	struct batadv_tt_common_entry *common = &tt_global_entry->common;
+	u16 local_flags;
+
+	/* Do not remove multicast addresses from the local hash on
+	 * global additions
+	 */
+	if (is_multicast_ether_addr(common->addr))
+		return;
+
+	/* remove address from local hash if present */
+	local_flags = batadv_tt_local_remove(bat_priv, common->addr, common->vid,
+					     "global tt received",
+					     flags & BATADV_TT_CLIENT_ROAM);
+
+	scoped_guard(spinlock_bh, &common->flags_lock) {
+		common->flags |= local_flags & BATADV_TT_CLIENT_WIFI;
+
+		if (!(flags & BATADV_TT_CLIENT_ROAM))
+			/* this is a normal global add. Therefore the client is
+			 * not in a roaming state anymore.
+			 */
+			common->flags &= ~BATADV_TT_CLIENT_ROAM;
+	}
+}
+
+/**
+ * batadv_tt_global_merge_flags() - merge announced flags into a global TT entry
+ * @tt_global_entry: the global TT entry to update
+ * @flags: TT flags announced for this non-mesh client
+ *
+ * Return: true if the originator list of @tt_global_entry has to be purged
+ * before the announced originator is added, false otherwise.
+ */
+static bool
+batadv_tt_global_merge_flags(struct batadv_tt_global_entry *tt_global_entry,
+			     u16 flags)
+{
+	struct batadv_tt_common_entry *common = &tt_global_entry->common;
+	bool delete = false;
+
+	scoped_guard(spinlock_bh, &common->flags_lock) {
+		/* if the client was temporary added before receiving the first
+		 * OGM announcing it, we have to clear the TEMP flag. Also,
+		 * remove the previous temporary orig node and re-add it
+		 * if required. If the orig entry changed, the new one which
+		 * is a non-temporary entry is preferred.
+		 */
+		if (common->flags & BATADV_TT_CLIENT_TEMP) {
+			delete = true;
+			common->flags &= ~BATADV_TT_CLIENT_TEMP;
+		}
+
+		/* the change can carry possible "attribute" flags like the
+		 * TT_CLIENT_TEMP, therefore they have to be copied in the
+		 * client entry
+		 */
+		if (!is_multicast_ether_addr(common->addr))
+			common->flags |= flags & (~BATADV_TT_SYNC_MASK);
+
+		/* If there is the BATADV_TT_CLIENT_ROAM flag set, there is only
+		 * one originator left in the list and we previously received a
+		 * delete + roaming change for this originator.
+		 *
+		 * We should first delete the old originator before adding the
+		 * new one.
+		 */
+		if (common->flags & BATADV_TT_CLIENT_ROAM) {
+			delete = true;
+			tt_global_entry->roam_at = 0;
+			common->flags &= ~BATADV_TT_CLIENT_ROAM;
+		}
+	}
+
+	return delete;
+}
+
+/**
+ * batadv_tt_global_create() - allocate and initialize a global TT entry
+ * @tt_addr: the mac address of the non-mesh client
+ * @vid: VLAN identifier
+ * @flags: TT flags that have to be set for this non-mesh client
+ *
+ * The returned entry is not yet part of bat_priv->tt.global_hash and has an
+ * empty originator list.
+ *
+ * Return: the new entry with an initialized reference counter on success, NULL
+ * otherwise.
+ */
+static struct batadv_tt_global_entry *
+batadv_tt_global_create(const unsigned char *tt_addr, unsigned short vid,
+			u16 flags)
+{
+	struct batadv_tt_global_entry *tt_global_entry;
+	struct batadv_tt_common_entry *common;
+
+	tt_global_entry = kmem_cache_zalloc(batadv_tg_cache, GFP_ATOMIC);
+	if (!tt_global_entry)
+		return NULL;
+
+	common = &tt_global_entry->common;
+	ether_addr_copy(common->addr, tt_addr);
+	common->vid = vid;
+	spin_lock_init(&common->flags_lock);
+
+	if (!is_multicast_ether_addr(common->addr)) {
+		scoped_guard(spinlock_bh, &common->flags_lock)
+			common->flags = flags & (~BATADV_TT_SYNC_MASK);
+	}
+
+	tt_global_entry->roam_at = 0;
+	/* node must store current time in case of roaming. This is
+	 * needed to purge this entry out on timeout (if nobody claims
+	 * it)
+	 */
+	if (flags & BATADV_TT_CLIENT_ROAM)
+		tt_global_entry->roam_at = jiffies;
+	kref_init(&common->refcount);
+	common->added_at = jiffies;
+
+	INIT_HLIST_HEAD(&tt_global_entry->orig_list);
+	atomic_set(&tt_global_entry->orig_list_count, 0);
+	spin_lock_init(&tt_global_entry->list_lock);
+
+	return tt_global_entry;
+}
+
 /**
  * batadv_tt_global_add() - add a new TT global entry or update an existing one
  * @bat_priv: the bat priv with all the mesh interface information
@@ -1664,8 +2033,10 @@ static bool batadv_tt_global_add(struct batadv_priv *bat_priv,
 	struct batadv_tt_local_entry *tt_local_entry;
 	struct batadv_tt_common_entry *common;
 	bool ret = false;
+	u16 global_flags;
 	u16 local_flags;
 	int hash_added;
+	bool delete;
 
 	/* ignore global entries from backbone nodes */
 	if (batadv_bla_is_backbone_gw_orig(bat_priv, orig_node->orig, vid))
@@ -1678,36 +2049,18 @@ static bool batadv_tt_global_add(struct batadv_priv *bat_priv,
 	 * for a roaming advertisement instead of manually messing up the global
 	 * table
 	 */
-	if ((flags & BATADV_TT_CLIENT_TEMP) && tt_local_entry &&
-	    !(tt_local_entry->common.flags & BATADV_TT_CLIENT_NEW))
-		goto out;
+	if ((flags & BATADV_TT_CLIENT_TEMP) && tt_local_entry) {
+		local_flags = batadv_tt_flags_get(&tt_local_entry->common);
+		if (!(local_flags & BATADV_TT_CLIENT_NEW))
+			goto out;
+	}
 
 	if (!tt_global_entry) {
-		tt_global_entry = kmem_cache_zalloc(batadv_tg_cache,
-						    GFP_ATOMIC);
+		tt_global_entry = batadv_tt_global_create(tt_addr, vid, flags);
 		if (!tt_global_entry)
 			goto out;
 
 		common = &tt_global_entry->common;
-		ether_addr_copy(common->addr, tt_addr);
-		common->vid = vid;
-
-		if (!is_multicast_ether_addr(common->addr))
-			common->flags = flags & (~BATADV_TT_SYNC_MASK);
-
-		tt_global_entry->roam_at = 0;
-		/* node must store current time in case of roaming. This is
-		 * needed to purge this entry out on timeout (if nobody claims
-		 * it)
-		 */
-		if (flags & BATADV_TT_CLIENT_ROAM)
-			tt_global_entry->roam_at = jiffies;
-		kref_init(&common->refcount);
-		common->added_at = jiffies;
-
-		INIT_HLIST_HEAD(&tt_global_entry->orig_list);
-		atomic_set(&tt_global_entry->orig_list_count, 0);
-		spin_lock_init(&tt_global_entry->list_lock);
 
 		kref_get(&common->refcount);
 		hash_added = batadv_hash_add(bat_priv->tt.global_hash,
@@ -1733,47 +2086,23 @@ static bool batadv_tt_global_add(struct batadv_priv *bat_priv,
 		 *    originator list and add the new one orig_entry
 		 */
 		if (flags & BATADV_TT_CLIENT_TEMP) {
-			if (!(common->flags & BATADV_TT_CLIENT_TEMP))
+			global_flags = batadv_tt_flags_get(common);
+			if (!(global_flags & BATADV_TT_CLIENT_TEMP))
 				goto out;
+
 			if (batadv_tt_global_entry_has_orig(tt_global_entry,
 							    orig_node, NULL))
 				goto out_remove;
-			batadv_tt_global_del_orig_list(tt_global_entry);
-			goto add_orig_entry;
-		}
 
-		/* if the client was temporary added before receiving the first
-		 * OGM announcing it, we have to clear the TEMP flag. Also,
-		 * remove the previous temporary orig node and re-add it
-		 * if required. If the orig entry changed, the new one which
-		 * is a non-temporary entry is preferred.
-		 */
-		if (common->flags & BATADV_TT_CLIENT_TEMP) {
-			batadv_tt_global_del_orig_list(tt_global_entry);
-			common->flags &= ~BATADV_TT_CLIENT_TEMP;
+			delete = true;
+		} else {
+			delete = batadv_tt_global_merge_flags(tt_global_entry, flags);
 		}
 
-		/* the change can carry possible "attribute" flags like the
-		 * TT_CLIENT_TEMP, therefore they have to be copied in the
-		 * client entry
-		 */
-		if (!is_multicast_ether_addr(common->addr))
-			common->flags |= flags & (~BATADV_TT_SYNC_MASK);
-
-		/* If there is the BATADV_TT_CLIENT_ROAM flag set, there is only
-		 * one originator left in the list and we previously received a
-		 * delete + roaming change for this originator.
-		 *
-		 * We should first delete the old originator before adding the
-		 * new one.
-		 */
-		if (common->flags & BATADV_TT_CLIENT_ROAM) {
+		if (delete)
 			batadv_tt_global_del_orig_list(tt_global_entry);
-			common->flags &= ~BATADV_TT_CLIENT_ROAM;
-			tt_global_entry->roam_at = 0;
-		}
 	}
-add_orig_entry:
+
 	/* add the new orig_entry (if needed) or update it */
 	batadv_tt_global_orig_entry_add(tt_global_entry, orig_node, ttvn,
 					flags & BATADV_TT_SYNC_MASK);
@@ -1785,23 +2114,7 @@ static bool batadv_tt_global_add(struct batadv_priv *bat_priv,
 	ret = true;
 
 out_remove:
-	/* Do not remove multicast addresses from the local hash on
-	 * global additions
-	 */
-	if (is_multicast_ether_addr(tt_addr))
-		goto out;
-
-	/* remove address from local hash if present */
-	local_flags = batadv_tt_local_remove(bat_priv, tt_addr, vid,
-					     "global tt received",
-					     flags & BATADV_TT_CLIENT_ROAM);
-	tt_global_entry->common.flags |= local_flags & BATADV_TT_CLIENT_WIFI;
-
-	if (!(flags & BATADV_TT_CLIENT_ROAM))
-		/* this is a normal global add. Therefore the client is not in a
-		 * roaming state anymore.
-		 */
-		tt_global_entry->common.flags &= ~BATADV_TT_CLIENT_ROAM;
+	batadv_tt_global_purge_local(bat_priv, tt_global_entry, flags);
 
 out:
 	batadv_tt_global_entry_put(tt_global_entry);
@@ -1871,9 +2184,9 @@ batadv_tt_global_dump_subentry(struct sk_buff *msg, u32 portid, u32 seq,
 			       struct batadv_tt_orig_list_entry *orig,
 			       bool best)
 {
-	u16 flags = (common->flags & (~BATADV_TT_SYNC_MASK)) | orig->flags;
 	struct batadv_orig_node_vlan *vlan;
 	u8 last_ttvn;
+	u16 flags;
 	void *hdr;
 	u32 crc;
 
@@ -1892,6 +2205,7 @@ batadv_tt_global_dump_subentry(struct sk_buff *msg, u32 portid, u32 seq,
 	if (!hdr)
 		return -ENOBUFS;
 
+	flags = (batadv_tt_flags_get(common) & (~BATADV_TT_SYNC_MASK)) | orig->flags;
 	last_ttvn = READ_ONCE(orig->orig_node->last_ttvn);
 
 	if (nla_put(msg, BATADV_ATTR_TT_ADDRESS, ETH_ALEN, common->addr) ||
@@ -2179,9 +2493,12 @@ batadv_tt_global_del_roaming(struct batadv_priv *bat_priv,
 	rcu_read_unlock();
 
 	if (last_entry) {
-		/* its the last one, mark for roaming. */
-		tt_global_entry->common.flags |= BATADV_TT_CLIENT_ROAM;
-		tt_global_entry->roam_at = jiffies;
+		scoped_guard(spinlock_bh, &tt_global_entry->common.flags_lock) {
+			/* its the last one, mark for roaming. */
+			tt_global_entry->roam_at = jiffies;
+
+			tt_global_entry->common.flags |= BATADV_TT_CLIENT_ROAM;
+		}
 	} else {
 		/* there is another entry, we can simply delete this
 		 * one and can still use the other one.
@@ -2330,16 +2647,18 @@ static bool batadv_tt_global_to_purge(struct batadv_tt_global_entry *tt_global,
 	unsigned long temp_timeout = BATADV_TT_CLIENT_TEMP_TIMEOUT;
 	bool purge = false;
 
-	if ((tt_global->common.flags & BATADV_TT_CLIENT_ROAM) &&
-	    batadv_has_timed_out(tt_global->roam_at, roam_timeout)) {
-		purge = true;
-		*msg = "Roaming timeout\n";
-	}
+	scoped_guard(spinlock_bh, &tt_global->common.flags_lock) {
+		if ((tt_global->common.flags & BATADV_TT_CLIENT_ROAM) &&
+		    batadv_has_timed_out(tt_global->roam_at, roam_timeout)) {
+			purge = true;
+			*msg = "Roaming timeout\n";
+		}
 
-	if ((tt_global->common.flags & BATADV_TT_CLIENT_TEMP) &&
-	    batadv_has_timed_out(tt_global->common.added_at, temp_timeout)) {
-		purge = true;
-		*msg = "Temporary client timeout\n";
+		if ((tt_global->common.flags & BATADV_TT_CLIENT_TEMP) &&
+		    batadv_has_timed_out(tt_global->common.added_at, temp_timeout)) {
+			purge = true;
+			*msg = "Temporary client timeout\n";
+		}
 	}
 
 	return purge;
@@ -2437,8 +2756,8 @@ static void batadv_tt_global_table_free(struct batadv_priv *bat_priv)
 /**
  * _batadv_is_ap_isolated() - check whether two clients are AP-isolated from
  *  each other
- * @tt_local_entry: local TT entry of the sending client
- * @tt_global_entry: global TT entry of the destination client
+ * @tt_local_entry: local TT entry of one local client
+ * @tt_global_entry: global TT entry of the remote client
  *
  * Return: true if traffic between the two clients should be dropped because
  *  either both are WiFi clients or both carry the ISOLATION flag; false
@@ -2448,13 +2767,19 @@ static bool
 _batadv_is_ap_isolated(struct batadv_tt_local_entry *tt_local_entry,
 		       struct batadv_tt_global_entry *tt_global_entry)
 {
-	if (tt_local_entry->common.flags & BATADV_TT_CLIENT_WIFI &&
-	    tt_global_entry->common.flags & BATADV_TT_CLIENT_WIFI)
+	u16 global_flags;
+	u16 local_flags;
+
+	global_flags = batadv_tt_flags_get(&tt_global_entry->common);
+	local_flags = batadv_tt_flags_get(&tt_local_entry->common);
+
+	if (local_flags & BATADV_TT_CLIENT_WIFI &&
+	    global_flags & BATADV_TT_CLIENT_WIFI)
 		return true;
 
 	/* check if the two clients are marked as isolated */
-	if (tt_local_entry->common.flags & BATADV_TT_CLIENT_ISOLA &&
-	    tt_global_entry->common.flags & BATADV_TT_CLIENT_ISOLA)
+	if (local_flags & BATADV_TT_CLIENT_ISOLA &&
+	    global_flags & BATADV_TT_CLIENT_ISOLA)
 		return true;
 
 	return false;
@@ -2483,11 +2808,15 @@ struct batadv_orig_node *batadv_transtable_search(struct batadv_priv *bat_priv,
 	struct batadv_tt_local_entry *tt_local_entry = NULL;
 	struct batadv_tt_orig_list_entry *best_entry;
 	struct batadv_orig_node *orig_node = NULL;
+	u16 flags;
 
 	if (src && batadv_vlan_ap_isola_get(bat_priv, vid)) {
 		tt_local_entry = batadv_tt_local_hash_find(bat_priv, src, vid);
-		if (!tt_local_entry ||
-		    (tt_local_entry->common.flags & BATADV_TT_CLIENT_PENDING))
+		if (!tt_local_entry)
+			goto out;
+
+		flags = batadv_tt_flags_get(&tt_local_entry->common);
+		if (flags & BATADV_TT_CLIENT_PENDING)
 			goto out;
 	}
 
@@ -2562,6 +2891,8 @@ static u32 batadv_tt_global_crc(struct batadv_priv *bat_priv,
 
 		rcu_read_lock();
 		hlist_for_each_entry_rcu(tt_common, head, hash_entry) {
+			u16 tt_flags;
+
 			tt_global = container_of(tt_common,
 						 struct batadv_tt_global_entry,
 						 common);
@@ -2571,18 +2902,21 @@ static u32 batadv_tt_global_crc(struct batadv_priv *bat_priv,
 			if (tt_common->vid != vid)
 				continue;
 
+			tt_flags = batadv_tt_flags_get(tt_common);
+
 			/* Roaming clients are in the global table for
 			 * consistency only. They don't have to be
 			 * taken into account while computing the
 			 * global crc
 			 */
-			if (tt_common->flags & BATADV_TT_CLIENT_ROAM)
+			if (tt_flags & BATADV_TT_CLIENT_ROAM)
 				continue;
+
 			/* Temporary clients have not been announced yet, so
 			 * they have to be skipped while computing the global
 			 * crc
 			 */
-			if (tt_common->flags & BATADV_TT_CLIENT_TEMP)
+			if (tt_flags & BATADV_TT_CLIENT_TEMP)
 				continue;
 
 			/* find out if this global entry is announced by this
@@ -2642,18 +2976,24 @@ static u32 batadv_tt_local_crc(struct batadv_priv *bat_priv,
 
 		rcu_read_lock();
 		hlist_for_each_entry_rcu(tt_common, head, hash_entry) {
+			u16 tt_flags;
+
 			/* compute the CRC only for entries belonging to the
 			 * VLAN identified by vid
 			 */
 			if (tt_common->vid != vid)
 				continue;
 
+			tt_flags = batadv_tt_flags_get(tt_common);
+
 			/* not yet committed clients have not to be taken into
 			 * account while computing the CRC
 			 */
-			if (tt_common->flags & BATADV_TT_CLIENT_NEW)
+			if (tt_flags & BATADV_TT_CLIENT_NEW)
 				continue;
 
+			flags = tt_flags & BATADV_TT_SYNC_MASK;
+
 			/* use network order to read the VID: this ensures that
 			 * every node reads the bytes in the same order.
 			 */
@@ -2663,7 +3003,6 @@ static u32 batadv_tt_local_crc(struct batadv_priv *bat_priv,
 			/* compute the CRC on flags that have to be kept in sync
 			 * among nodes
 			 */
-			flags = tt_common->flags & BATADV_TT_SYNC_MASK;
 			crc_tmp = crc32c(crc_tmp, &flags, sizeof(flags));
 
 			crc ^= crc32c(crc_tmp, tt_common->addr, ETH_ALEN);
@@ -2821,17 +3160,19 @@ batadv_tt_req_node_new(struct batadv_priv *bat_priv,
  *
  * Return: true if the entry is a valid, false otherwise.
  */
-static bool batadv_tt_local_valid(const void *entry_ptr,
+static bool batadv_tt_local_valid(void *entry_ptr,
 				  const void *data_ptr,
 				  u8 *flags)
 {
-	const struct batadv_tt_common_entry *tt_common_entry = entry_ptr;
+	struct batadv_tt_common_entry *tt_common_entry = entry_ptr;
+	u16 tt_flags;
 
-	if (tt_common_entry->flags & BATADV_TT_CLIENT_NEW)
+	tt_flags = batadv_tt_flags_get(tt_common_entry);
+	if (tt_flags & BATADV_TT_CLIENT_NEW)
 		return false;
 
 	if (flags)
-		*flags = tt_common_entry->flags;
+		*flags = tt_flags;
 
 	return true;
 }
@@ -2848,16 +3189,18 @@ static bool batadv_tt_local_valid(const void *entry_ptr,
  *
  * Return: true if the entry is a valid, false otherwise.
  */
-static bool batadv_tt_global_valid(const void *entry_ptr,
+static bool batadv_tt_global_valid(void *entry_ptr,
 				   const void *data_ptr,
 				   u8 *flags)
 {
-	const struct batadv_tt_common_entry *tt_common_entry = entry_ptr;
-	const struct batadv_tt_global_entry *tt_global_entry;
+	struct batadv_tt_common_entry *tt_common_entry = entry_ptr;
 	const struct batadv_orig_node *orig_node = data_ptr;
+	struct batadv_tt_global_entry *tt_global_entry;
+	u16 tt_flags;
 
-	if (tt_common_entry->flags & BATADV_TT_CLIENT_ROAM ||
-	    tt_common_entry->flags & BATADV_TT_CLIENT_TEMP)
+	tt_flags = batadv_tt_flags_get(tt_common_entry);
+	if (tt_flags & BATADV_TT_CLIENT_ROAM ||
+	    tt_flags & BATADV_TT_CLIENT_TEMP)
 		return false;
 
 	tt_global_entry = container_of(tt_common_entry,
@@ -2886,7 +3229,7 @@ static bool batadv_tt_global_valid(const void *entry_ptr,
 static u16 batadv_tt_tvlv_generate(struct batadv_priv *bat_priv,
 				   struct batadv_hashtable *hash,
 				   void *tvlv_buff, u16 tt_len,
-				   bool (*valid_cb)(const void *,
+				   bool (*valid_cb)(void *,
 						    const void *,
 						    u8 *flags),
 				   void *cb_data)
@@ -3511,19 +3854,21 @@ bool batadv_is_my_client(struct batadv_priv *bat_priv, const u8 *addr,
 			 unsigned short vid)
 {
 	struct batadv_tt_local_entry *tt_local_entry;
-	bool ret = false;
+	u16 tt_flags;
+	bool ret;
 
 	tt_local_entry = batadv_tt_local_hash_find(bat_priv, addr, vid);
 	if (!tt_local_entry)
-		goto out;
+		return false;
+
+	tt_flags = batadv_tt_flags_get(&tt_local_entry->common);
+
 	/* Check if the client has been logically deleted (but is kept for
 	 * consistency purpose)
 	 */
-	if ((tt_local_entry->common.flags & BATADV_TT_CLIENT_PENDING) ||
-	    (tt_local_entry->common.flags & BATADV_TT_CLIENT_ROAM))
-		goto out;
-	ret = true;
-out:
+	ret = !((tt_flags & BATADV_TT_CLIENT_PENDING) ||
+		(tt_flags & BATADV_TT_CLIENT_ROAM));
+
 	batadv_tt_local_entry_put(tt_local_entry);
 	return ret;
 }
@@ -3782,16 +4127,13 @@ void batadv_tt_free(struct batadv_priv *bat_priv)
 }
 
 /**
- * batadv_tt_local_set_flags() - set or unset the specified flags on the local
- *  table and possibly count them in the TT size
+ * batadv_tt_local_transition_new() - unset the NEW flag on the local
+ *  table and count them in the TT size
  * @bat_priv: the bat priv with all the mesh interface information
- * @flags: the flag to switch
- * @enable: whether to set or unset the flag
- * @count: whether to increase the TT size by the number of changed entries
  */
-static void batadv_tt_local_set_flags(struct batadv_priv *bat_priv, u16 flags,
-				      bool enable, bool count)
+static void batadv_tt_local_transition_new(struct batadv_priv *bat_priv)
 {
+	spinlock_t *list_lock; /* protects write access to the hash lists */
 	struct batadv_hashtable *hash = bat_priv->tt.local_hash;
 	struct batadv_tt_common_entry *tt_common_entry;
 	struct hlist_head *head;
@@ -3802,27 +4144,28 @@ static void batadv_tt_local_set_flags(struct batadv_priv *bat_priv, u16 flags,
 
 	for (i = 0; i < hash->size; i++) {
 		head = &hash->table[i];
+		list_lock = &hash->list_locks[i];
 
-		rcu_read_lock();
-		hlist_for_each_entry_rcu(tt_common_entry,
-					 head, hash_entry) {
-			if (enable) {
-				if ((tt_common_entry->flags & flags) == flags)
-					continue;
-				tt_common_entry->flags |= flags;
-			} else {
-				if (!(tt_common_entry->flags & flags))
-					continue;
-				tt_common_entry->flags &= ~flags;
+		spin_lock_bh(list_lock);
+		hlist_for_each_entry(tt_common_entry, head, hash_entry) {
+			bool cont = false;
+
+			scoped_guard(spinlock_bh, &tt_common_entry->flags_lock) {
+				if (!(tt_common_entry->flags & BATADV_TT_CLIENT_NEW)) {
+					cont = true;
+					break;
+				}
+
+				tt_common_entry->flags &= ~BATADV_TT_CLIENT_NEW;
 			}
 
-			if (!count)
+			if (cont)
 				continue;
 
 			batadv_tt_local_size_inc(bat_priv,
 						 tt_common_entry->vid);
 		}
-		rcu_read_unlock();
+		spin_unlock_bh(list_lock);
 	}
 }
 
@@ -3854,16 +4197,31 @@ static void batadv_tt_local_purge_pending_clients(struct batadv_priv *bat_priv)
 		spin_lock_bh(list_lock);
 		hlist_for_each_entry_safe(tt_common, node_tmp, head,
 					  hash_entry) {
-			if (!(tt_common->flags & BATADV_TT_CLIENT_PENDING))
-				continue;
+			bool cont = false;
 
-			batadv_dbg(BATADV_DBG_TT, bat_priv,
-				   "Deleting local tt entry (%pM, vid: %d): pending\n",
-				   tt_common->addr,
-				   batadv_print_vid(tt_common->vid));
+			scoped_guard(spinlock_bh, &tt_common->flags_lock) {
+				if (!(tt_common->flags & BATADV_TT_CLIENT_PENDING)) {
+					cont = true;
+					break;
+				}
+
+				batadv_dbg(BATADV_DBG_TT, bat_priv,
+					   "Deleting local tt entry (%pM, vid: %d): pending\n",
+					   tt_common->addr,
+					   batadv_print_vid(tt_common->vid));
+
+				hlist_del_rcu(&tt_common->hash_entry);
+
+				/* An entry which still carries BATADV_TT_CLIENT_NEW was
+				 * never counted and must not be uncounted here.
+				 */
+				if (!(tt_common->flags & BATADV_TT_CLIENT_NEW))
+					batadv_tt_local_size_dec(bat_priv, tt_common->vid);
+			}
+
+			if (cont)
+				continue;
 
-			batadv_tt_local_size_dec(bat_priv, tt_common->vid);
-			hlist_del_rcu(&tt_common->hash_entry);
 			tt_local = container_of(tt_common,
 						struct batadv_tt_local_entry,
 						common);
@@ -3891,7 +4249,7 @@ static void batadv_tt_local_commit_changes_nolock(struct batadv_priv *bat_priv)
 		return;
 	}
 
-	batadv_tt_local_set_flags(bat_priv, BATADV_TT_CLIENT_NEW, false, true);
+	batadv_tt_local_transition_new(bat_priv);
 
 	batadv_tt_local_purge_pending_clients(bat_priv);
 	batadv_tt_local_update_crc(bat_priv);
@@ -4061,15 +4419,16 @@ bool batadv_tt_global_client_is_roaming(struct batadv_priv *bat_priv,
 					u8 *addr, unsigned short vid)
 {
 	struct batadv_tt_global_entry *tt_global_entry;
-	bool ret = false;
+	bool ret;
 
 	tt_global_entry = batadv_tt_global_hash_find(bat_priv, addr, vid);
 	if (!tt_global_entry)
-		goto out;
+		return false;
+
+	ret = batadv_tt_flags_get(&tt_global_entry->common) & BATADV_TT_CLIENT_ROAM;
 
-	ret = tt_global_entry->common.flags & BATADV_TT_CLIENT_ROAM;
 	batadv_tt_global_entry_put(tt_global_entry);
-out:
+
 	return ret;
 }
 
@@ -4087,15 +4446,16 @@ bool batadv_tt_local_client_is_roaming(struct batadv_priv *bat_priv,
 				       u8 *addr, unsigned short vid)
 {
 	struct batadv_tt_local_entry *tt_local_entry;
-	bool ret = false;
+	bool ret;
 
 	tt_local_entry = batadv_tt_local_hash_find(bat_priv, addr, vid);
 	if (!tt_local_entry)
-		goto out;
+		return false;
+
+	ret = batadv_tt_flags_get(&tt_local_entry->common) & BATADV_TT_CLIENT_ROAM;
 
-	ret = tt_local_entry->common.flags & BATADV_TT_CLIENT_ROAM;
 	batadv_tt_local_entry_put(tt_local_entry);
-out:
+
 	return ret;
 }
 
@@ -4404,7 +4764,7 @@ bool batadv_tt_global_is_isolated(struct batadv_priv *bat_priv,
 	if (!tt)
 		return false;
 
-	ret = tt->common.flags & BATADV_TT_CLIENT_ISOLA;
+	ret = batadv_tt_flags_get(&tt->common) & BATADV_TT_CLIENT_ISOLA;
 
 	batadv_tt_global_entry_put(tt);
 
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index 9bdc5a3e799e2a..67872927cfb50f 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -410,7 +410,7 @@ struct batadv_orig_bat_iv {
  */
 struct batadv_orig_node {
 	/** @orig: originator ethernet address */
-	u8 orig[ETH_ALEN];
+	u8 orig[ETH_ALEN] __aligned(2);
 
 	/** @ifinfo_list: list for routers per outgoing interface */
 	struct hlist_head ifinfo_list;
@@ -634,12 +634,12 @@ struct batadv_hardif_neigh_node {
 	struct hlist_node list;
 
 	/** @addr: the MAC address of the neighboring interface */
-	u8 addr[ETH_ALEN];
+	u8 addr[ETH_ALEN] __aligned(2);
 
 	/**
 	 * @orig: the address of the originator this neighbor node belongs to
 	 */
-	u8 orig[ETH_ALEN];
+	u8 orig[ETH_ALEN] __aligned(2);
 
 	/** @if_incoming: pointer to incoming hard-interface */
 	struct batadv_hard_iface *if_incoming;
@@ -677,7 +677,7 @@ struct batadv_neigh_node {
 #endif
 
 	/** @addr: the MAC address of the neighboring interface */
-	u8 addr[ETH_ALEN];
+	u8 addr[ETH_ALEN] __aligned(2);
 
 	/** @ifinfo_list: list for routing metrics per outgoing interface */
 	struct hlist_head ifinfo_list;
@@ -777,7 +777,7 @@ struct batadv_neigh_ifinfo {
  */
 struct batadv_bcast_duplist_entry {
 	/** @orig: mac address of orig node originating the broadcast */
-	u8 orig[ETH_ALEN];
+	u8 orig[ETH_ALEN] __aligned(2);
 
 	/** @crc: crc32 checksum of broadcast payload */
 	u32 crc;
@@ -1085,7 +1085,7 @@ struct batadv_priv_bla {
 	struct batadv_hashtable *backbone_hash;
 
 	/** @loopdetect_addr: MAC address used for own loopdetection frames */
-	u8 loopdetect_addr[ETH_ALEN];
+	u8 loopdetect_addr[ETH_ALEN] __aligned(2);
 
 	/**
 	 * @loopdetect_lasttime: time when the loopdetection frames were sent
@@ -1352,7 +1352,7 @@ struct batadv_tp_vars_common {
 	struct batadv_priv *bat_priv;
 
 	/** @other_end: mac address of remote */
-	u8 other_end[ETH_ALEN];
+	u8 other_end[ETH_ALEN] __aligned(2);
 
 	/** @session: TP session identifier */
 	u8 session[2];
@@ -1765,7 +1765,7 @@ struct batadv_bla_backbone_gw {
 	 * @orig: originator address of backbone node (mac address of primary
 	 *  iface)
 	 */
-	u8 orig[ETH_ALEN];
+	u8 orig[ETH_ALEN] __aligned(2);
 
 	/** @vid: vlan id this gateway was detected on */
 	unsigned short vid;
@@ -1810,7 +1810,7 @@ struct batadv_bla_backbone_gw {
  */
 struct batadv_bla_claim {
 	/** @addr: mac address of claimed non-mesh client */
-	u8 addr[ETH_ALEN];
+	u8 addr[ETH_ALEN] __aligned(2);
 
 	/** @vid: vlan id this client was detected on */
 	unsigned short vid;
@@ -1840,7 +1840,7 @@ struct batadv_bla_claim {
  */
 struct batadv_tt_common_entry {
 	/** @addr: mac address of non-mesh client */
-	u8 addr[ETH_ALEN];
+	u8 addr[ETH_ALEN] __aligned(2);
 
 	/** @vid: VLAN identifier */
 	unsigned short vid;
@@ -1851,12 +1851,21 @@ struct batadv_tt_common_entry {
 	 */
 	struct hlist_node hash_entry;
 
-	/** @flags: various state handling flags (see batadv_tt_client_flags) */
+	/**
+	 * @flags: various state handling flags (see batadv_tt_client_flags),
+	 * protected by @flags_lock
+	 */
 	u16 flags;
 
 	/** @added_at: timestamp used for purging stale tt common entries */
 	unsigned long added_at;
 
+	/**
+	 * @flags_lock: protect modifications of @flags and
+	 *  @batadv_tt_global_entry.roam_at
+	 */
+	spinlock_t flags_lock;
+
 	/** @refcount: number of contexts the object is used */
 	struct kref refcount;
 
@@ -1942,7 +1951,7 @@ struct batadv_tt_req_node {
 	/**
 	 * @addr: mac address of the originator this request was sent to
 	 */
-	u8 addr[ETH_ALEN];
+	u8 addr[ETH_ALEN] __aligned(2);
 
 	/** @issued_at: timestamp used for purging stale tt requests */
 	unsigned long issued_at;
@@ -1959,7 +1968,7 @@ struct batadv_tt_req_node {
  */
 struct batadv_tt_roam_node {
 	/** @addr: mac address of the client in the roaming phase */
-	u8 addr[ETH_ALEN];
+	u8 addr[ETH_ALEN] __aligned(2);
 
 	/** @vid: VLAN identifier */
 	u16 vid;
@@ -2087,14 +2096,14 @@ struct batadv_algo_neigh_ops {
 		   struct batadv_hard_iface *if_outgoing2);
 
 	/**
-	 * @is_similar_or_better: check if neigh1 is equally similar or better
-	 *  than neigh2 for their respective outgoing interface from the metric
+	 * @is_similar_or_better: check if @candidate is equally similar or better
+	 *  than @reference for their respective outgoing interface from the metric
 	 *  prospective
 	 */
-	bool (*is_similar_or_better)(struct batadv_neigh_node *neigh1,
-				     struct batadv_hard_iface *if_outgoing1,
-				     struct batadv_neigh_node *neigh2,
-				     struct batadv_hard_iface *if_outgoing2);
+	bool (*is_similar_or_better)(struct batadv_neigh_node *candidate,
+				     struct batadv_hard_iface *if_outgoing_cand,
+				     struct batadv_neigh_node *reference,
+				     struct batadv_hard_iface *if_outgoing_ref);
 
 	/** @dump: dump neighbors to a netlink socket (optional) */
 	void (*dump)(struct sk_buff *msg, struct netlink_callback *cb,
@@ -2204,7 +2213,7 @@ struct batadv_hw_addr {
 	struct hlist_node list;
 
 	/** @addr: the MAC address of this list entry */
-	unsigned char addr[ETH_ALEN];
+	unsigned char addr[ETH_ALEN] __aligned(2);
 };
 
 /**
diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c
index 30f4afa18bc8f2..836add41f5d16f 100644
--- a/net/bluetooth/6lowpan.c
+++ b/net/bluetooth/6lowpan.c
@@ -722,6 +722,8 @@ static int setup_netdev(struct l2cap_chan *chan, struct lowpan_btle_dev **dev)
 }
 
 static inline void chan_ready_cb(struct l2cap_chan *chan)
+	__must_hold(&chan->lock)
+	__must_hold(&chan->conn->lock)
 {
 	struct lowpan_btle_dev *dev;
 	bool new_netdev = false;
@@ -912,18 +914,27 @@ static int bt_6lowpan_connect(bdaddr_t *addr, u8 dst_type)
 static int bt_6lowpan_disconnect(struct l2cap_conn *conn, u8 dst_type)
 {
 	struct lowpan_peer *peer;
+	struct l2cap_chan *chan;
 
 	BT_DBG("conn %p dst type %u", conn, dst_type);
 
+	spin_lock(&devices_lock);
+
 	peer = lookup_peer(conn);
-	if (!peer)
+	if (!peer) {
+		spin_unlock(&devices_lock);
 		return -ENOENT;
+	}
+
+	chan = peer->chan;
+	l2cap_chan_hold(chan);
+
+	spin_unlock(&devices_lock);
 
-	BT_DBG("peer %p chan %p", peer, peer->chan);
+	BT_DBG("peer %p chan %p", peer, chan);
 
-	l2cap_chan_lock(peer->chan);
-	l2cap_chan_close(peer->chan, ENOENT);
-	l2cap_chan_unlock(peer->chan);
+	l2cap_chan_close_unlocked(chan, ENOENT);
+	l2cap_chan_put(chan);
 
 	return 0;
 }
@@ -1025,9 +1036,9 @@ static void disconnect_all_peers(void)
 	struct lowpan_peer *peer;
 	int nchans;
 
-	/* l2cap_chan_close() cannot be called from RCU, and lock ordering
-	 * chan->lock > devices_lock prevents taking write side lock, so copy
-	 * then close.
+	/* l2cap_chan_close_unlocked() cannot be called from RCU, and lock
+	 * ordering chan->lock > devices_lock prevents taking write side lock,
+	 * so copy then close.
 	 */
 
 	rcu_read_lock();
@@ -1062,9 +1073,7 @@ static void disconnect_all_peers(void)
 		spin_unlock(&devices_lock);
 
 		for (i = 0; i < nchans; ++i) {
-			l2cap_chan_lock(chans[i]);
-			l2cap_chan_close(chans[i], ENOENT);
-			l2cap_chan_unlock(chans[i]);
+			l2cap_chan_close_unlocked(chans[i], ENOENT);
 			l2cap_chan_put(chans[i]);
 		}
 	} while (nchans);
@@ -1082,9 +1091,7 @@ static void do_enable_set(bool flag)
 
 	mutex_lock(&set_lock);
 	if (listen_chan) {
-		l2cap_chan_lock(listen_chan);
-		l2cap_chan_close(listen_chan, 0);
-		l2cap_chan_unlock(listen_chan);
+		l2cap_chan_close_unlocked(listen_chan, 0);
 		l2cap_chan_put(listen_chan);
 	}
 
@@ -1132,9 +1139,7 @@ static ssize_t lowpan_control_write(struct file *fp,
 
 		mutex_lock(&set_lock);
 		if (listen_chan) {
-			l2cap_chan_lock(listen_chan);
-			l2cap_chan_close(listen_chan, 0);
-			l2cap_chan_unlock(listen_chan);
+			l2cap_chan_close_unlocked(listen_chan, 0);
 			l2cap_chan_put(listen_chan);
 			listen_chan = NULL;
 		}
@@ -1303,10 +1308,9 @@ static void __exit bt_6lowpan_exit(void)
 	debugfs_remove(lowpan_control_debugfs);
 
 	if (listen_chan) {
-		l2cap_chan_lock(listen_chan);
-		l2cap_chan_close(listen_chan, 0);
-		l2cap_chan_unlock(listen_chan);
+		l2cap_chan_close_unlocked(listen_chan, 0);
 		l2cap_chan_put(listen_chan);
+		listen_chan = NULL;
 	}
 
 	disconnect_devices();
diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile
index ff466ea97436a0..b78ad98864d4f6 100644
--- a/net/bluetooth/Makefile
+++ b/net/bluetooth/Makefile
@@ -14,7 +14,7 @@ bluetooth_6lowpan-y := 6lowpan.o
 bluetooth-y := af_bluetooth.o hci_core.o hci_conn.o hci_event.o mgmt.o \
 	hci_sock.o hci_sysfs.o l2cap_core.o l2cap_sock.o smp.o lib.o \
 	ecdh_helper.o mgmt_util.o mgmt_config.o hci_codec.o eir.o hci_sync.o \
-	hci_drv.o
+	hci_drv.o hci_h4.o
 
 bluetooth-$(CONFIG_DEV_COREDUMP) += coredump.o
 
diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c
index f7d88c33e23e4f..06854f46ba34d7 100644
--- a/net/bluetooth/bnep/core.c
+++ b/net/bluetooth/bnep/core.c
@@ -270,9 +270,14 @@ static int bnep_rx_extension(struct bnep_session *s, struct sk_buff *skb)
 
 		BT_DBG("type 0x%x len %u", h->type, h->len);
 
+		if (skb->len < h->len) {
+			err = -EILSEQ;
+			break;
+		}
+
 		switch (h->type & BNEP_TYPE_MASK) {
 		case BNEP_EXT_CONTROL:
-			bnep_rx_control(s, skb->data, skb->len);
+			bnep_rx_control(s, skb->data, h->len);
 			break;
 
 		default:
@@ -373,6 +378,11 @@ static int bnep_rx_frame(struct bnep_session *s, struct sk_buff *skb)
 			goto badframe;
 	}
 
+	if ((type & BNEP_TYPE_MASK) == BNEP_CONTROL) {
+		kfree_skb(skb);
+		return 0;
+	}
+
 	/* Strip 802.1p header */
 	if (ntohs(s->eh.h_proto) == ETH_P_8021Q) {
 		if (!skb_pull(skb, 4))
@@ -451,6 +461,11 @@ static int bnep_tx_frame(struct bnep_session *s, struct sk_buff *skb)
 		goto send;
 	}
 
+	if (skb->len < ETH_HLEN) {
+		kfree_skb(skb);
+		return 0;
+	}
+
 	iv[il++] = (struct kvec) { &type, 1 };
 	len++;
 
@@ -480,10 +495,7 @@ static int bnep_tx_frame(struct bnep_session *s, struct sk_buff *skb)
 	iv[il++] = (struct kvec) { skb->data, skb->len };
 	len += skb->len;
 
-	/* FIXME: linearize skb */
-	{
-		len = kernel_sendmsg(sock, &s->msg, iv, il, len);
-	}
+	len = kernel_sendmsg(sock, &s->msg, iv, il, len);
 	kfree_skb(skb);
 
 	if (len > 0) {
@@ -524,9 +536,12 @@ static int bnep_session(void *arg)
 			break;
 
 		/* TX */
-		while ((skb = skb_dequeue(&sk->sk_write_queue)))
-			if (bnep_tx_frame(s, skb))
+		while ((skb = skb_dequeue(&sk->sk_write_queue))) {
+			if (skb_linearize(skb))
+				kfree_skb(skb);
+			else if (bnep_tx_frame(s, skb))
 				break;
+		}
 		netif_wake_queue(dev);
 
 		/*
@@ -670,7 +685,7 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock)
 		goto failed;
 	}
 
-	strcpy(req->device, dev->name);
+	strscpy(req->device, dev->name, sizeof(req->device));
 	up_write(&bnep_session_sem);
 	return 0;
 
@@ -712,7 +727,7 @@ static void __bnep_copy_ci(struct bnep_conninfo *ci, struct bnep_session *s)
 
 	memset(ci, 0, sizeof(*ci));
 	memcpy(ci->dst, s->eh.h_source, ETH_ALEN);
-	strcpy(ci->device, s->dev->name);
+	strscpy(ci->device, s->dev->name, sizeof(ci->device));
 	ci->flags = s->flags & valid_flags;
 	ci->state = s->state;
 	ci->role  = s->role;
diff --git a/net/bluetooth/bnep/netdev.c b/net/bluetooth/bnep/netdev.c
index ee1e39a3daffb8..b451ef457741f2 100644
--- a/net/bluetooth/bnep/netdev.c
+++ b/net/bluetooth/bnep/netdev.c
@@ -166,6 +166,12 @@ static netdev_tx_t bnep_net_xmit(struct sk_buff *skb,
 
 	BT_DBG("skb %p, dev %p", skb, dev);
 
+	if (!pskb_may_pull(skb, ETH_HLEN)) {
+		dev->stats.tx_dropped++;
+		kfree_skb(skb);
+		return NETDEV_TX_OK;
+	}
+
 #ifdef CONFIG_BT_BNEP_MC_FILTER
 	if (bnep_net_mc_filter(skb, s)) {
 		kfree_skb(skb);
@@ -218,7 +224,7 @@ void bnep_net_setup(struct net_device *dev)
 	dev->addr_len = ETH_ALEN;
 
 	ether_setup(dev);
-	dev->min_mtu = 0;
+	dev->min_mtu = ETH_MIN_MTU;
 	dev->max_mtu = ETH_MAX_MTU;
 	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
 	dev->netdev_ops = &bnep_netdev_ops;
diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c
index fa72cf8aaa7a0a..cf44452e0766e9 100644
--- a/net/bluetooth/hci_conn.c
+++ b/net/bluetooth/hci_conn.c
@@ -302,11 +302,13 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data)
 	cp.tx_bandwidth   = cpu_to_le32(0x00001f40);
 	cp.rx_bandwidth   = cpu_to_le32(0x00001f40);
 
+	hci_dev_lock(hdev);
+
 	switch (conn->codec.id) {
 	case BT_CODEC_MSBC:
 		if (!find_next_esco_param(conn, esco_param_msbc,
 					  ARRAY_SIZE(esco_param_msbc)))
-			return -EINVAL;
+			goto unlock;
 
 		param = &esco_param_msbc[conn->attempt - 1];
 		cp.tx_coding_format.id = 0x05;
@@ -332,7 +334,7 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data)
 	case BT_CODEC_TRANSPARENT:
 		if (!find_next_esco_param(conn, esco_param_msbc,
 					  ARRAY_SIZE(esco_param_msbc)))
-			return -EINVAL;
+			goto unlock;
 
 		param = &esco_param_msbc[conn->attempt - 1];
 		cp.tx_coding_format.id = 0x03;
@@ -359,11 +361,11 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data)
 		if (conn->parent && lmp_esco_capable(conn->parent)) {
 			if (!find_next_esco_param(conn, esco_param_cvsd,
 						  ARRAY_SIZE(esco_param_cvsd)))
-				return -EINVAL;
+				goto unlock;
 			param = &esco_param_cvsd[conn->attempt - 1];
 		} else {
 			if (conn->attempt > ARRAY_SIZE(sco_param_cvsd))
-				return -EINVAL;
+				goto unlock;
 			param = &sco_param_cvsd[conn->attempt - 1];
 		}
 		cp.tx_coding_format.id = 2;
@@ -386,9 +388,11 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data)
 		cp.out_transport_unit_size = 16;
 		break;
 	default:
-		return -EINVAL;
+		goto unlock;
 	}
 
+	hci_dev_unlock(hdev);
+
 	cp.retrans_effort = param->retrans_effort;
 	cp.pkt_type = __cpu_to_le16(param->pkt_type);
 	cp.max_latency = __cpu_to_le16(param->max_latency);
@@ -397,6 +401,10 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data)
 		return -EIO;
 
 	return 0;
+
+unlock:
+	hci_dev_unlock(hdev);
+	return -EINVAL;
 }
 
 static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle)
@@ -2079,6 +2087,8 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst,
 		cis->conn_timeout = timeout;
 	}
 
+	hci_conn_hold(cis);
+
 	if (cis->state == BT_CONNECTED)
 		return cis;
 
@@ -2120,7 +2130,6 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst,
 		return ERR_PTR(-EINVAL);
 	}
 
-	hci_conn_hold(cis);
 	cis->state = BT_BOUND;
 
 	return cis;
@@ -2374,10 +2383,13 @@ struct hci_conn *hci_bind_bis(struct hci_dev *hdev, bdaddr_t *dst, __u8 sid,
 	parent = hci_conn_hash_lookup_big(hdev,
 					  conn->iso_qos.bcast.big);
 	if (parent && parent != conn) {
+		hci_conn_hold(parent);
 		link = hci_conn_link(parent, conn);
 		hci_conn_drop(conn);
-		if (!link)
+		if (!link) {
+			hci_conn_drop(parent);
 			return ERR_PTR(-ENOLINK);
+		}
 	}
 
 	return conn;
@@ -2497,6 +2509,12 @@ struct hci_conn *hci_connect_cis(struct hci_dev *hdev, bdaddr_t *dst,
 		return cis;
 	}
 
+	/* The existing link already owns the hold on its parent. */
+	if (cis->link) {
+		hci_conn_drop(le);
+		return cis;
+	}
+
 	link = hci_conn_link(le, cis);
 	hci_conn_drop(cis);
 	if (!link) {
diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c
index d183efaf906321..05d1ca25f44f30 100644
--- a/net/bluetooth/hci_core.c
+++ b/net/bluetooth/hci_core.c
@@ -1489,8 +1489,10 @@ int hci_remove_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr,
 
 	BT_DBG("%s removing %pMR (%u)", hdev->name, bdaddr, bdaddr_type);
 
+	mutex_lock(&hdev->remote_oob_lock);
 	list_del(&data->list);
 	kfree(data);
+	mutex_unlock(&hdev->remote_oob_lock);
 
 	return 0;
 }
@@ -1499,10 +1501,12 @@ void hci_remote_oob_data_clear(struct hci_dev *hdev)
 {
 	struct oob_data *data, *n;
 
+	mutex_lock(&hdev->remote_oob_lock);
 	list_for_each_entry_safe(data, n, &hdev->remote_oob_data, list) {
 		list_del(&data->list);
 		kfree(data);
 	}
+	mutex_unlock(&hdev->remote_oob_lock);
 }
 
 int hci_add_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr,
@@ -1511,11 +1515,14 @@ int hci_add_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr,
 {
 	struct oob_data *data;
 
+	mutex_lock(&hdev->remote_oob_lock);
 	data = hci_find_remote_oob_data(hdev, bdaddr, bdaddr_type);
 	if (!data) {
 		data = kmalloc_obj(*data);
-		if (!data)
+		if (!data) {
+			mutex_unlock(&hdev->remote_oob_lock);
 			return -ENOMEM;
+		}
 
 		bacpy(&data->bdaddr, bdaddr);
 		data->bdaddr_type = bdaddr_type;
@@ -1548,6 +1555,8 @@ int hci_add_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr,
 
 	BT_DBG("%s for %pMR", hdev->name, bdaddr);
 
+	mutex_unlock(&hdev->remote_oob_lock);
+
 	return 0;
 }
 
@@ -2485,6 +2494,7 @@ struct hci_dev *hci_alloc_dev_priv(int sizeof_priv)
 	mutex_init(&hdev->lock);
 	mutex_init(&hdev->req_lock);
 	mutex_init(&hdev->mgmt_pending_lock);
+	mutex_init(&hdev->remote_oob_lock);
 
 	ida_init(&hdev->unset_handle_ida);
 
@@ -2557,8 +2567,10 @@ int hci_register_dev(struct hci_dev *hdev)
 		return id;
 
 	error = dev_set_name(&hdev->dev, "hci%u", id);
-	if (error)
+	if (error) {
+		ida_free(&hci_index_ida, id);
 		return error;
+	}
 
 	hdev->name = dev_name(&hdev->dev);
 	hdev->id = id;
@@ -2917,6 +2929,8 @@ int hci_recv_frame(struct hci_dev *hdev, struct sk_buff *skb)
 		break;
 	case HCI_ISODATA_PKT:
 		break;
+	case HCI_VENDOR_PKT:
+		break;
 	case HCI_DRV_PKT:
 		break;
 	default:
@@ -3053,6 +3067,41 @@ static int hci_send_conn_frame(struct hci_dev *hdev, struct hci_conn *conn,
 	return hci_send_frame(hdev, skb);
 }
 
+/**
+ * hci_send_vendor_frame - Send an HCI_VENDOR_PKT frame to the HCI driver
+ * @hdev: The HCI device
+ * @iter: iov_iter carrying the frame
+ *
+ * Return: 0 on success, or a negative errno on failure.
+ */
+int hci_send_vendor_frame(struct hci_dev *hdev, struct iov_iter *iter)
+{
+	struct sk_buff *skb;
+	unsigned int len;
+
+	if (WARN_ON(!iov_iter_is_kvec(iter)))
+		return -EINVAL;
+
+	/* Vendor frames are opaque, the caller guarantees the size. */
+	len = (unsigned int)iov_iter_count(iter);
+	if (!len)
+		return -EINVAL;
+
+	skb = bt_skb_alloc(len, GFP_KERNEL);
+	if (!skb)
+		return -ENOMEM;
+
+	if (!copy_from_iter_full(skb_put(skb, len), len, iter)) {
+		kfree_skb(skb);
+		return -EFAULT;
+	}
+
+	hci_skb_pkt_type(skb) = HCI_VENDOR_PKT;
+
+	return hci_send_frame(hdev, skb);
+}
+EXPORT_SYMBOL(hci_send_vendor_frame);
+
 /* Send HCI command */
 int hci_send_cmd(struct hci_dev *hdev, __u16 opcode, __u32 plen,
 		 const void *param)
@@ -3325,6 +3374,8 @@ static void hci_queue_iso(struct hci_conn *conn, struct sk_buff_head *queue,
 
 		skb_shinfo(skb)->frag_list = NULL;
 
+		spin_lock_bh(&queue->lock);
+
 		__skb_queue_tail(queue, skb);
 
 		do {
@@ -3339,6 +3390,8 @@ static void hci_queue_iso(struct hci_conn *conn, struct sk_buff_head *queue,
 
 			__skb_queue_tail(queue, skb);
 		} while (list);
+
+		spin_unlock_bh(&queue->lock);
 	}
 
 	bt_dev_dbg(hdev, "hcon %p queued %d", conn, skb_queue_len(queue));
@@ -3402,9 +3455,6 @@ static struct hci_conn *hci_low_sent(struct hci_dev *hdev, __u8 type,
 	struct hci_conn *conn = NULL, *c;
 	unsigned int num = 0, min = ~0;
 
-	/* We don't have to lock device here. Connections are always
-	 * added and removed with TX task disabled. */
-
 	rcu_read_lock();
 
 	list_for_each_entry_rcu(c, &h->list, list) {
@@ -3609,13 +3659,15 @@ static void __check_timeout(struct hci_dev *hdev, unsigned int cnt, u8 type)
 }
 
 /* Schedule SCO */
-static void hci_sched_sco(struct hci_dev *hdev, __u8 type)
+static void __hci_sched_sco(struct hci_dev *hdev, __u8 type)
 {
 	struct hci_conn *conn;
 	struct sk_buff *skb;
 	int quote, *cnt;
 	unsigned int pkts = hdev->sco_pkts;
 
+	lockdep_assert_held(&hdev->lock);
+
 	bt_dev_dbg(hdev, "type %u", type);
 
 	if (!hci_conn_num(hdev, type) || !pkts)
@@ -3650,6 +3702,13 @@ static void hci_sched_sco(struct hci_dev *hdev, __u8 type)
 		queue_work(hdev->workqueue, &hdev->tx_work);
 }
 
+static void hci_sched_sco(struct hci_dev *hdev, __u8 type)
+{
+	hci_dev_lock(hdev);
+	__hci_sched_sco(hdev, type);
+	hci_dev_unlock(hdev);
+}
+
 static void hci_sched_acl_pkt(struct hci_dev *hdev)
 {
 	unsigned int cnt = hdev->acl_cnt;
@@ -3659,6 +3718,8 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev)
 
 	__check_timeout(hdev, cnt, ACL_LINK);
 
+	hci_dev_lock(hdev);
+
 	while (hdev->acl_cnt &&
 	       (chan = hci_chan_sent(hdev, ACL_LINK, &quote))) {
 		u32 priority = (skb_peek(&chan->data_q))->priority;
@@ -3683,13 +3744,15 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev)
 			chan->conn->sent++;
 
 			/* Send pending SCO packets right away */
-			hci_sched_sco(hdev, SCO_LINK);
-			hci_sched_sco(hdev, ESCO_LINK);
+			__hci_sched_sco(hdev, SCO_LINK);
+			__hci_sched_sco(hdev, ESCO_LINK);
 		}
 	}
 
 	if (cnt != hdev->acl_cnt)
 		hci_prio_recalculate(hdev, ACL_LINK);
+
+	hci_dev_unlock(hdev);
 }
 
 static void hci_sched_acl(struct hci_dev *hdev)
@@ -3718,6 +3781,8 @@ static void hci_sched_le(struct hci_dev *hdev)
 
 	__check_timeout(hdev, *cnt, LE_LINK);
 
+	hci_dev_lock(hdev);
+
 	tmp = *cnt;
 	while (*cnt && (chan = hci_chan_sent(hdev, LE_LINK, &quote))) {
 		u32 priority = (skb_peek(&chan->data_q))->priority;
@@ -3739,13 +3804,15 @@ static void hci_sched_le(struct hci_dev *hdev)
 			chan->conn->sent++;
 
 			/* Send pending SCO packets right away */
-			hci_sched_sco(hdev, SCO_LINK);
-			hci_sched_sco(hdev, ESCO_LINK);
+			__hci_sched_sco(hdev, SCO_LINK);
+			__hci_sched_sco(hdev, ESCO_LINK);
 		}
 	}
 
 	if (*cnt != tmp)
 		hci_prio_recalculate(hdev, LE_LINK);
+
+	hci_dev_unlock(hdev);
 }
 
 /* Schedule iso */
@@ -3764,6 +3831,8 @@ static void hci_sched_iso(struct hci_dev *hdev, __u8 type)
 
 	__check_timeout(hdev, *cnt, type);
 
+	hci_dev_lock(hdev);
+
 	while (*cnt && (conn = hci_low_sent(hdev, type, &quote))) {
 		while (quote-- && (skb = skb_dequeue(&conn->data_q))) {
 			BT_DBG("skb %p len %d", skb, skb->len);
@@ -3777,6 +3846,8 @@ static void hci_sched_iso(struct hci_dev *hdev, __u8 type)
 			(*cnt)--;
 		}
 	}
+
+	hci_dev_unlock(hdev);
 }
 
 static void hci_tx_work(struct work_struct *work)
@@ -4068,6 +4139,14 @@ static void hci_rx_work(struct work_struct *work)
 			hci_isodata_packet(hdev, skb);
 			break;
 
+		case HCI_VENDOR_PKT:
+			BT_DBG("%s Vendor packet", hdev->name);
+			if (hdev->recv_vendor_pkt)
+				hdev->recv_vendor_pkt(hdev, skb);
+			else
+				kfree_skb(skb);
+			break;
+
 		default:
 			kfree_skb(skb);
 			break;
@@ -4087,7 +4166,7 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb)
 	if (!hdev->sent_cmd) {
 		skb_queue_head(&hdev->cmd_q, skb);
 		queue_work(hdev->workqueue, &hdev->cmd_work);
-		return -EINVAL;
+		return -ENOMEM;
 	}
 
 	if (hci_skb_opcode(skb) != HCI_OP_NOP) {
diff --git a/net/bluetooth/hci_debugfs.c b/net/bluetooth/hci_debugfs.c
index aadffaaff20e13..2559fb6324d63d 100644
--- a/net/bluetooth/hci_debugfs.c
+++ b/net/bluetooth/hci_debugfs.c
@@ -364,7 +364,7 @@ static int inquiry_cache_show(struct seq_file *f, void *p)
 
 	list_for_each_entry(e, &cache->all, all) {
 		struct inquiry_data *data = &e->data;
-		seq_printf(f, "%pMR %d %d %d 0x%.2x%.2x%.2x 0x%.4x %d %d %u\n",
+		seq_printf(f, "%pMR %d %d %d 0x%.2x%.2x%.2x 0x%.4x %d %d %lu\n",
 			   &data->bdaddr,
 			   data->pscan_rep_mode, data->pscan_period_mode,
 			   data->pscan_mode, data->dev_class[2],
diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index 2f5e21ff975294..5ac2357e92c942 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -404,10 +404,7 @@ static u8 hci_cc_write_auth_enable(struct hci_dev *hdev, void *data,
 	if (!rp->status) {
 		__u8 param = *((__u8 *) sent);
 
-		if (param == AUTH_ENABLED)
-			set_bit(HCI_AUTH, &hdev->flags);
-		else
-			clear_bit(HCI_AUTH, &hdev->flags);
+		assign_bit(HCI_AUTH, &hdev->flags, param == AUTH_ENABLED);
 	}
 
 	if (hci_dev_test_flag(hdev, HCI_MGMT))
@@ -436,10 +433,7 @@ static u8 hci_cc_write_encrypt_mode(struct hci_dev *hdev, void *data,
 
 	param = *((__u8 *) sent);
 
-	if (param)
-		set_bit(HCI_ENCRYPT, &hdev->flags);
-	else
-		clear_bit(HCI_ENCRYPT, &hdev->flags);
+	assign_bit(HCI_ENCRYPT, &hdev->flags, param);
 
 	return rp->status;
 }
@@ -466,15 +460,9 @@ static u8 hci_cc_write_scan_enable(struct hci_dev *hdev, void *data,
 		goto done;
 	}
 
-	if (param & SCAN_INQUIRY)
-		set_bit(HCI_ISCAN, &hdev->flags);
-	else
-		clear_bit(HCI_ISCAN, &hdev->flags);
+	assign_bit(HCI_ISCAN, &hdev->flags, param & SCAN_INQUIRY);
 
-	if (param & SCAN_PAGE)
-		set_bit(HCI_PSCAN, &hdev->flags);
-	else
-		clear_bit(HCI_PSCAN, &hdev->flags);
+	assign_bit(HCI_PSCAN, &hdev->flags, param & SCAN_PAGE);
 
 done:
 	hci_dev_unlock(hdev);
@@ -4099,6 +4087,9 @@ static u8 hci_cc_le_set_per_adv_enable(struct hci_dev *hdev, void *data,
 #define HCI_CC_STATUS(_op, _func) \
 	HCI_CC(_op, _func, sizeof(struct hci_ev_status))
 
+#define HCI_MAX_CC_PLEN \
+	(HCI_MAX_EVENT_PLEN - sizeof(struct hci_ev_cmd_complete))
+
 static const struct hci_cc {
 	u16  op;
 	u8 (*func)(struct hci_dev *hdev, void *data, struct sk_buff *skb);
@@ -4192,7 +4183,7 @@ static const struct hci_cc {
 	       sizeof(struct hci_rp_le_read_local_features)),
 	HCI_CC_VL(HCI_OP_LE_READ_CONN_INTERVAL, hci_cc_le_read_conn_interval,
 		  sizeof(struct hci_rp_le_read_conn_interval),
-		  HCI_MAX_EVENT_SIZE),
+		  HCI_MAX_CC_PLEN),
 	HCI_CC(HCI_OP_LE_READ_ADV_TX_POWER, hci_cc_le_read_adv_tx_power,
 	       sizeof(struct hci_rp_le_read_adv_tx_power)),
 	HCI_CC(HCI_OP_USER_CONFIRM_REPLY, hci_cc_user_confirm_reply,
@@ -4264,7 +4255,7 @@ static const struct hci_cc {
 	HCI_CC(HCI_OP_LE_READ_BUFFER_SIZE_V2, hci_cc_le_read_buffer_size_v2,
 	       sizeof(struct hci_rp_le_read_buffer_size_v2)),
 	HCI_CC_VL(HCI_OP_LE_SET_CIG_PARAMS, hci_cc_le_set_cig_params,
-		  sizeof(struct hci_rp_le_set_cig_params), HCI_MAX_EVENT_SIZE),
+		  sizeof(struct hci_rp_le_set_cig_params), HCI_MAX_CC_PLEN),
 	HCI_CC(HCI_OP_LE_SETUP_ISO_PATH, hci_cc_le_setup_iso_path,
 	       sizeof(struct hci_rp_le_setup_iso_path)),
 	HCI_CC(HCI_OP_LE_READ_ALL_LOCAL_FEATURES,
@@ -4611,10 +4602,8 @@ static void hci_mode_change_evt(struct hci_dev *hdev, void *data,
 
 		if (!test_and_clear_bit(HCI_CONN_MODE_CHANGE_PEND,
 					&conn->flags)) {
-			if (conn->mode == HCI_CM_ACTIVE)
-				set_bit(HCI_CONN_POWER_SAVE, &conn->flags);
-			else
-				clear_bit(HCI_CONN_POWER_SAVE, &conn->flags);
+			assign_bit(HCI_CONN_POWER_SAVE, &conn->flags,
+				   conn->mode == HCI_CM_ACTIVE);
 		}
 
 		if (test_and_clear_bit(HCI_CONN_SCO_SETUP_PEND, &conn->flags))
@@ -4814,10 +4803,7 @@ static void hci_link_key_notify_evt(struct hci_dev *hdev, void *data,
 		goto unlock;
 	}
 
-	if (persistent)
-		clear_bit(HCI_CONN_FLUSH_KEY, &conn->flags);
-	else
-		set_bit(HCI_CONN_FLUSH_KEY, &conn->flags);
+	assign_bit(HCI_CONN_FLUSH_KEY, &conn->flags, !persistent);
 
 unlock:
 	hci_dev_unlock(hdev);
@@ -6311,6 +6297,15 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 	if (irk) {
 		bdaddr = &irk->bdaddr;
 		bdaddr_type = irk->addr_type;
+	} else {
+		/* The peer is on air with its identity address, so whatever
+		 * RPA is cached for it has been abandoned. Drop it, or
+		 * hci_connect_le() would swap it back in and dial an address
+		 * the peer no longer answers.
+		 */
+		irk = hci_find_irk_by_addr(hdev, bdaddr, bdaddr_type);
+		if (irk)
+			bacpy(&irk->rpa, BDADDR_ANY);
 	}
 
 	bdaddr_type = ev_bdaddr_type(hdev, bdaddr_type, &bdaddr_resolved);
@@ -7465,6 +7460,9 @@ static void hci_le_conn_rate_change_evt(struct hci_dev *hdev, void *data,
 #define HCI_LE_EV_STATUS(_op, _func) \
 	HCI_LE_EV(_op, _func, sizeof(struct hci_ev_status))
 
+#define HCI_MAX_LE_SUBEVT_PLEN \
+	(HCI_MAX_EVENT_PLEN - sizeof(struct hci_ev_le_meta))
+
 /* Entries in this table shall have their position according to the subevent
  * opcode they handle so the use of the macros above is recommend since it does
  * attempt to initialize at its proper index using Designated Initializers that
@@ -7481,7 +7479,7 @@ static const struct hci_le_ev {
 	/* [0x02 = HCI_EV_LE_ADVERTISING_REPORT] */
 	HCI_LE_EV_VL(HCI_EV_LE_ADVERTISING_REPORT, hci_le_adv_report_evt,
 		     sizeof(struct hci_ev_le_advertising_report),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x03 = HCI_EV_LE_CONN_UPDATE_COMPLETE] */
 	HCI_LE_EV(HCI_EV_LE_CONN_UPDATE_COMPLETE,
 		  hci_le_conn_update_complete_evt,
@@ -7504,14 +7502,14 @@ static const struct hci_le_ev {
 	/* [0x0b = HCI_EV_LE_DIRECT_ADV_REPORT] */
 	HCI_LE_EV_VL(HCI_EV_LE_DIRECT_ADV_REPORT, hci_le_direct_adv_report_evt,
 		     sizeof(struct hci_ev_le_direct_adv_report),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x0c = HCI_EV_LE_PHY_UPDATE_COMPLETE] */
 	HCI_LE_EV(HCI_EV_LE_PHY_UPDATE_COMPLETE, hci_le_phy_update_evt,
 		  sizeof(struct hci_ev_le_phy_update_complete)),
 	/* [0x0d = HCI_EV_LE_EXT_ADV_REPORT] */
 	HCI_LE_EV_VL(HCI_EV_LE_EXT_ADV_REPORT, hci_le_ext_adv_report_evt,
 		     sizeof(struct hci_ev_le_ext_adv_report),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x0e = HCI_EV_LE_PA_SYNC_ESTABLISHED] */
 	HCI_LE_EV(HCI_EV_LE_PA_SYNC_ESTABLISHED,
 		  hci_le_pa_sync_established_evt,
@@ -7520,7 +7518,7 @@ static const struct hci_le_ev {
 	HCI_LE_EV_VL(HCI_EV_LE_PER_ADV_REPORT,
 				 hci_le_per_adv_report_evt,
 				 sizeof(struct hci_ev_le_per_adv_report),
-				 HCI_MAX_EVENT_SIZE),
+				 HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x10 = HCI_EV_LE_PA_SYNC_LOST] */
 	HCI_LE_EV(HCI_EV_LE_PA_SYNC_LOST, hci_le_pa_sync_lost_evt,
 		  sizeof(struct hci_ev_le_pa_sync_lost)),
@@ -7541,28 +7539,28 @@ static const struct hci_le_ev {
 	HCI_LE_EV_VL(HCI_EVT_LE_CREATE_BIG_COMPLETE,
 		     hci_le_create_big_complete_evt,
 		     sizeof(struct hci_evt_le_create_big_complete),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x1d = HCI_EV_LE_BIG_SYNC_ESTABLISHED] */
 	HCI_LE_EV_VL(HCI_EVT_LE_BIG_SYNC_ESTABLISHED,
 		     hci_le_big_sync_established_evt,
 		     sizeof(struct hci_evt_le_big_sync_established),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x1e = HCI_EVT_LE_BIG_SYNC_LOST] */
 	HCI_LE_EV_VL(HCI_EVT_LE_BIG_SYNC_LOST,
 		     hci_le_big_sync_lost_evt,
 		     sizeof(struct hci_evt_le_big_sync_lost),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x22 = HCI_EVT_LE_BIG_INFO_ADV_REPORT] */
 	HCI_LE_EV_VL(HCI_EVT_LE_BIG_INFO_ADV_REPORT,
 		     hci_le_big_info_adv_report_evt,
 		     sizeof(struct hci_evt_le_big_info_adv_report),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x2b = HCI_EVT_LE_ALL_REMOTE_FEATURES_COMPLETE] */
 	HCI_LE_EV_VL(HCI_EVT_LE_ALL_REMOTE_FEATURES_COMPLETE,
 		     hci_le_read_all_remote_features_evt,
 		     sizeof(struct
 			    hci_evt_le_read_all_remote_features_complete),
-		     HCI_MAX_EVENT_SIZE),
+		     HCI_MAX_LE_SUBEVT_PLEN),
 	/* [0x37 = HCI_EVT_LE_CONN_RATE_CHANGE] */
 	HCI_LE_EV(HCI_EVT_LE_CONN_RATE_CHANGE, hci_le_conn_rate_change_evt,
 		  sizeof(struct hci_evt_le_conn_rate_change)),
diff --git a/net/bluetooth/hci_h4.c b/net/bluetooth/hci_h4.c
new file mode 100644
index 00000000000000..86f809018062f5
--- /dev/null
+++ b/net/bluetooth/hci_h4.c
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *  Bluetooth HCI H:4 packet reassembly
+ *
+ *  Copyright (C) 2000-2001  Qualcomm Incorporated
+ *  Copyright (C) 2002-2003  Maxim Krasnyansky <maxk@qualcomm.com>
+ *  Copyright (C) 2004-2005  Marcel Holtmann <marcel@holtmann.org>
+ */
+
+#include <linux/export.h>
+#include <linux/skbuff.h>
+#include <linux/unaligned.h>
+
+#include <net/bluetooth/bluetooth.h>
+#include <net/bluetooth/hci_core.h>
+#include <net/bluetooth/hci_h4.h>
+
+/* h4_recv_skb - Reassemble H:4 framed packets
+ * @hdev: HCI device the packets are received on
+ * @alignment: optional packet alignment, NULL or 0 means no alignment
+ * @padding: optional padding state carried over between calls
+ * @skb: partially received packet from a previous call, may be NULL or an
+ *	 ERR_PTR returned by a previous call
+ * @buffer: buffer holding the received data
+ * @count: number of bytes in @buffer
+ * @pkts: table of supported packet types
+ * @pkts_count: number of entries in @pkts
+ *
+ * Returns the partially received packet to be passed to the next call, or an
+ * ERR_PTR on error. The returned value can be fed back into this function as
+ * is, but must be checked with IS_ERR() before being freed.
+ */
+struct sk_buff *h4_recv_skb(struct hci_dev *hdev, u8 *alignment, u8 *padding,
+			    struct sk_buff *skb, const unsigned char *buffer,
+			    int count, const struct h4_recv_pkt *pkts,
+			    int pkts_count)
+{
+	u8 align = alignment && *alignment ? *alignment : 1;
+
+	/* Check for error from previous call */
+	if (IS_ERR(skb))
+		skb = NULL;
+
+	while (count) {
+		int i, len;
+
+		/* remove padding bytes from buffer */
+		if (padding) {
+			for (; (*padding) && count > 0; (*padding)--) {
+				count--;
+				buffer++;
+			}
+		}
+
+		if (!count)
+			break;
+
+		if (!skb) {
+			for (i = 0; i < pkts_count; i++) {
+				if (buffer[0] != pkts[i].type)
+					continue;
+
+				skb = bt_skb_alloc(pkts[i].maxlen,
+						   GFP_ATOMIC);
+				if (!skb)
+					return ERR_PTR(-ENOMEM);
+
+				hci_skb_pkt_type(skb) = pkts[i].type;
+				hci_skb_expect(skb) = pkts[i].hlen;
+				break;
+			}
+
+			/* Check for invalid packet type */
+			if (!skb)
+				return ERR_PTR(-EILSEQ);
+
+			count -= 1;
+			buffer += 1;
+		}
+
+		len = min_t(uint, hci_skb_expect(skb) - skb->len, count);
+		skb_put_data(skb, buffer, len);
+
+		count -= len;
+		buffer += len;
+
+		/* Check for partial packet */
+		if (skb->len < hci_skb_expect(skb))
+			continue;
+
+		for (i = 0; i < pkts_count; i++) {
+			if (hci_skb_pkt_type(skb) == pkts[i].type)
+				break;
+		}
+
+		if (i >= pkts_count) {
+			kfree_skb(skb);
+			return ERR_PTR(-EILSEQ);
+		}
+
+		if (skb->len == pkts[i].hlen) {
+			u16 dlen;
+
+			switch (pkts[i].lsize) {
+			case 0:
+				/* No variable data length */
+				dlen = 0;
+				break;
+			case 1:
+				/* Single octet variable length */
+				dlen = skb->data[pkts[i].loff];
+				hci_skb_expect(skb) += dlen;
+
+				if (skb_tailroom(skb) < dlen) {
+					kfree_skb(skb);
+					return ERR_PTR(-EMSGSIZE);
+				}
+				break;
+			case 2:
+				/* Double octet variable length */
+				dlen = get_unaligned_le16(skb->data +
+							  pkts[i].loff);
+				hci_skb_expect(skb) += dlen;
+
+				if (skb_tailroom(skb) < dlen) {
+					kfree_skb(skb);
+					return ERR_PTR(-EMSGSIZE);
+				}
+				break;
+			default:
+				/* Unsupported variable length */
+				kfree_skb(skb);
+				return ERR_PTR(-EILSEQ);
+			}
+
+			if (!dlen) {
+				if (padding) {
+					*padding = (skb->len + 1) % align;
+					*padding = (align - *padding) % align;
+				}
+
+				/* No more data, complete frame */
+				pkts[i].recv(hdev, skb);
+				skb = NULL;
+			}
+		} else {
+			if (padding) {
+				*padding = (skb->len + 1) % align;
+				*padding = (align - *padding) % align;
+			}
+
+			/* Complete frame */
+			pkts[i].recv(hdev, skb);
+			skb = NULL;
+		}
+	}
+
+	return skb;
+}
+EXPORT_SYMBOL_GPL(h4_recv_skb);
diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c
index 070ca388f9ace8..9235b3b343eed4 100644
--- a/net/bluetooth/hci_sock.c
+++ b/net/bluetooth/hci_sock.c
@@ -164,6 +164,7 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb)
 {
 	struct hci_filter *flt;
 	int flt_type, flt_event;
+	u8 event;
 
 	/* Apply filter */
 	flt = &hci_pi(sk)->filter;
@@ -177,7 +178,11 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb)
 	if (hci_skb_pkt_type(skb) != HCI_EVENT_PKT)
 		return false;
 
-	flt_event = (*(__u8 *)skb->data & HCI_FLT_EVENT_BITS);
+	if (skb->len < 1)
+		return true;
+
+	event = *(__u8 *)skb->data;
+	flt_event = event & HCI_FLT_EVENT_BITS;
 
 	if (!hci_test_bit(flt_event, &flt->event_mask))
 		return true;
@@ -186,11 +191,17 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb)
 	if (!flt->opcode)
 		return false;
 
-	if (flt_event == HCI_EV_CMD_COMPLETE &&
+	if (event == HCI_EV_CMD_COMPLETE && skb->len < 5)
+		return true;
+
+	if (event == HCI_EV_CMD_COMPLETE &&
 	    flt->opcode != get_unaligned((__le16 *)(skb->data + 3)))
 		return true;
 
-	if (flt_event == HCI_EV_CMD_STATUS &&
+	if (event == HCI_EV_CMD_STATUS && skb->len < 6)
+		return true;
+
+	if (event == HCI_EV_CMD_STATUS &&
 	    flt->opcode != get_unaligned((__le16 *)(skb->data + 4)))
 		return true;
 
@@ -233,6 +244,7 @@ void hci_send_to_sock(struct hci_dev *hdev, struct sk_buff *skb)
 			    hci_skb_pkt_type(skb) != HCI_ACLDATA_PKT &&
 			    hci_skb_pkt_type(skb) != HCI_SCODATA_PKT &&
 			    hci_skb_pkt_type(skb) != HCI_ISODATA_PKT &&
+			    hci_skb_pkt_type(skb) != HCI_VENDOR_PKT &&
 			    hci_skb_pkt_type(skb) != HCI_DRV_PKT)
 				continue;
 		} else {
@@ -390,6 +402,12 @@ void hci_send_to_monitor(struct hci_dev *hdev, struct sk_buff *skb)
 		else
 			opcode = cpu_to_le16(HCI_MON_ISO_TX_PKT);
 		break;
+	case HCI_VENDOR_PKT:
+		if (bt_cb(skb)->incoming)
+			opcode = cpu_to_le16(HCI_MON_VENDOR_RX_PKT);
+		else
+			opcode = cpu_to_le16(HCI_MON_VENDOR_TX_PKT);
+		break;
 	case HCI_DRV_PKT:
 		if (bt_cb(skb)->incoming)
 			opcode = cpu_to_le16(HCI_MON_DRV_RX_PKT);
@@ -1868,6 +1886,7 @@ static int hci_sock_sendmsg(struct socket *sock, struct msghdr *msg,
 		    hci_skb_pkt_type(skb) != HCI_ACLDATA_PKT &&
 		    hci_skb_pkt_type(skb) != HCI_SCODATA_PKT &&
 		    hci_skb_pkt_type(skb) != HCI_ISODATA_PKT &&
+		    hci_skb_pkt_type(skb) != HCI_VENDOR_PKT &&
 		    hci_skb_pkt_type(skb) != HCI_DRV_PKT) {
 			err = -EINVAL;
 			goto drop;
@@ -1881,7 +1900,8 @@ static int hci_sock_sendmsg(struct socket *sock, struct msghdr *msg,
 		u16 ocf = hci_opcode_ocf(opcode);
 
 		if (((ogf > HCI_SFLT_MAX_OGF) ||
-		     !hci_test_bit(ocf & HCI_FLT_OCF_BITS,
+		     (ocf > HCI_FLT_OCF_BITS) ||
+		     !hci_test_bit(ocf,
 				   &hci_sec_filter.ocf_mask[ogf])) &&
 		    !capable(CAP_NET_RAW)) {
 			err = -EPERM;
diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c
index 74e2b04c84b260..1fe11d3ef1aaa8 100644
--- a/net/bluetooth/hci_sync.c
+++ b/net/bluetooth/hci_sync.c
@@ -312,6 +312,10 @@ static void hci_cmd_sync_work(struct work_struct *work)
 	while (1) {
 		struct hci_cmd_sync_work_entry *entry;
 
+		/* Leave the backlog to hci_cmd_sync_clear() */
+		if (hci_dev_test_flag(hdev, HCI_UNREGISTER))
+			break;
+
 		mutex_lock(&hdev->cmd_sync_work_lock);
 		entry = list_first_entry_or_null(&hdev->cmd_sync_work_list,
 						 struct hci_cmd_sync_work_entry,
@@ -658,6 +662,8 @@ void hci_cmd_sync_clear(struct hci_dev *hdev)
 {
 	struct hci_cmd_sync_work_entry *entry, *tmp;
 
+	/* cmd_work is disabled, the pending request can only time out */
+	hci_cmd_sync_cancel_sync(hdev, ENODEV);
 	cancel_work_sync(&hdev->cmd_sync_work);
 	cancel_work_sync(&hdev->reenable_adv_work);
 
@@ -5376,6 +5382,30 @@ static int hci_dev_init_sync(struct hci_dev *hdev)
 	return ret;
 }
 
+static void hci_dev_drop_last_cmd_req_and_close(struct hci_dev *hdev)
+{
+	/* Drop last sent command */
+	if (hdev->sent_cmd) {
+		cancel_delayed_work_sync(&hdev->cmd_timer);
+		kfree_skb(hdev->sent_cmd);
+		hdev->sent_cmd = NULL;
+	}
+
+	/* Drop last request */
+	if (hdev->req_skb) {
+		kfree_skb(hdev->req_skb);
+		hdev->req_skb = NULL;
+		hci_dev_clear_flag(hdev, HCI_CMD_PENDING);
+	}
+
+	clear_bit(HCI_RUNNING, &hdev->flags);
+	hci_sock_dev_event(hdev, HCI_DEV_CLOSE);
+
+	/* After this point our queues are empty and no tasks are scheduled. */
+	hdev->close(hdev);
+	hdev->flags &= BIT(HCI_RAW);
+}
+
 int hci_dev_open_sync(struct hci_dev *hdev)
 {
 	int ret;
@@ -5462,23 +5492,7 @@ int hci_dev_open_sync(struct hci_dev *hdev)
 		if (hdev->flush)
 			hdev->flush(hdev);
 
-		if (hdev->sent_cmd) {
-			cancel_delayed_work_sync(&hdev->cmd_timer);
-			kfree_skb(hdev->sent_cmd);
-			hdev->sent_cmd = NULL;
-		}
-
-		if (hdev->req_skb) {
-			kfree_skb(hdev->req_skb);
-			hdev->req_skb = NULL;
-			hci_dev_clear_flag(hdev, HCI_CMD_PENDING);
-		}
-
-		clear_bit(HCI_RUNNING, &hdev->flags);
-		hci_sock_dev_event(hdev, HCI_DEV_CLOSE);
-
-		hdev->close(hdev);
-		hdev->flags &= BIT(HCI_RAW);
+		hci_dev_drop_last_cmd_req_and_close(hdev);
 	}
 
 done:
@@ -5645,28 +5659,10 @@ int hci_dev_close_sync(struct hci_dev *hdev)
 	skb_queue_purge(&hdev->cmd_q);
 	skb_queue_purge(&hdev->raw_q);
 
-	/* Drop last sent command */
-	if (hdev->sent_cmd) {
-		cancel_delayed_work_sync(&hdev->cmd_timer);
-		kfree_skb(hdev->sent_cmd);
-		hdev->sent_cmd = NULL;
-	}
-
-	/* Drop last request */
-	if (hdev->req_skb) {
-		kfree_skb(hdev->req_skb);
-		hdev->req_skb = NULL;
-		hci_dev_clear_flag(hdev, HCI_CMD_PENDING);
-	}
-
-	clear_bit(HCI_RUNNING, &hdev->flags);
-	hci_sock_dev_event(hdev, HCI_DEV_CLOSE);
-
-	/* After this point our queues are empty and no tasks are scheduled. */
-	hdev->close(hdev);
+	/* Drop last sent command, last request and close */
+	hci_dev_drop_last_cmd_req_and_close(hdev);
 
 	/* Clear flags */
-	hdev->flags &= BIT(HCI_RAW);
 	hci_dev_clear_volatile_flags(hdev);
 	hci_dev_clear_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE);
 
@@ -5764,6 +5760,7 @@ int hci_stop_discovery_sync(struct hci_dev *hdev)
 {
 	struct discovery_state *d = &hdev->discovery;
 	struct inquiry_entry *e;
+	bdaddr_t addr;
 	int err;
 
 	bt_dev_dbg(hdev, "state %u", hdev->discovery.state);
@@ -5799,15 +5796,21 @@ int hci_stop_discovery_sync(struct hci_dev *hdev)
 		return 0;
 
 	if (d->state == DISCOVERY_RESOLVING || d->state == DISCOVERY_STOPPING) {
+		hci_dev_lock(hdev);
 		e = hci_inquiry_cache_lookup_resolve(hdev, BDADDR_ANY,
 						     NAME_PENDING);
-		if (!e)
+		if (!e) {
+			hci_dev_unlock(hdev);
 			return 0;
+		}
+
+		bacpy(&addr, &e->data.bdaddr);
+		hci_dev_unlock(hdev);
 
 		/* Ignore cancel errors since it should interfere with stopping
 		 * of the discovery.
 		 */
-		hci_remote_name_cancel_sync(hdev, &e->data.bdaddr);
+		hci_remote_name_cancel_sync(hdev, &addr);
 	}
 
 	return 0;
@@ -7236,6 +7239,7 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data)
 	bacpy(&cp.bdaddr, &conn->dst);
 	cp.pscan_rep_mode = 0x02;
 
+	hci_dev_lock(hdev);
 	ie = hci_inquiry_cache_lookup(hdev, &conn->dst);
 	if (ie) {
 		if (inquiry_entry_age(ie) <= INQUIRY_ENTRY_AGE_MAX) {
@@ -7247,6 +7251,7 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data)
 
 		memcpy(conn->dev_class, ie->data.dev_class, 3);
 	}
+	hci_dev_unlock(hdev);
 
 	cp.pkt_type = cpu_to_le16(conn->pkt_type);
 	if (lmp_rswitch_capable(hdev) && !(hdev->link_mode & HCI_LM_MASTER))
diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c
index eb99653f33f919..e52bd59ba1b3e3 100644
--- a/net/bluetooth/iso.c
+++ b/net/bluetooth/iso.c
@@ -496,6 +496,7 @@ static int iso_connect_cis(struct sock *sk)
 	struct hci_dev  *hdev;
 	bdaddr_t src, dst;
 	u8 src_type;
+	bool already_attached;
 	int err;
 
 	lock_sock(sk);
@@ -568,8 +569,14 @@ static int iso_connect_cis(struct sock *sk)
 		goto unlock;
 	}
 
+	iso_conn_lock(conn);
+	already_attached = iso_pi(sk)->conn == conn && conn->sk == sk;
+	iso_conn_unlock(conn);
+
 	err = iso_chan_add(conn, sk, NULL);
 	iso_conn_put(conn);
+	if (already_attached || err == -EBUSY)
+		hci_conn_drop(hcon);
 	if (err)
 		goto unlock;
 
@@ -1935,10 +1942,7 @@ static int iso_sock_setsockopt(struct socket *sock, int level, int optname,
 		if (err)
 			break;
 
-		if (opt)
-			set_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
-		else
-			clear_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
+		assign_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags, opt);
 		break;
 
 	case BT_PKT_STATUS:
@@ -1946,10 +1950,7 @@ static int iso_sock_setsockopt(struct socket *sock, int level, int optname,
 		if (err)
 			break;
 
-		if (opt)
-			set_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags);
-		else
-			clear_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags);
+		assign_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags, opt);
 		break;
 
 	case BT_PKT_SEQNUM:
@@ -1957,10 +1958,7 @@ static int iso_sock_setsockopt(struct socket *sock, int level, int optname,
 		if (err)
 			break;
 
-		if (opt)
-			set_bit(BT_SK_PKT_SEQNUM, &bt_sk(sk)->flags);
-		else
-			clear_bit(BT_SK_PKT_SEQNUM, &bt_sk(sk)->flags);
+		assign_bit(BT_SK_PKT_SEQNUM, &bt_sk(sk)->flags, opt);
 		break;
 
 	case BT_ISO_QOS:
diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
index 644e31160d55bc..49a9988049089b 100644
--- a/net/bluetooth/l2cap_core.c
+++ b/net/bluetooth/l2cap_core.c
@@ -44,8 +44,8 @@ bool enable_ecred = IS_ENABLED(CONFIG_BT_LE_L2CAP_ECRED);
 
 static u32 l2cap_feat_mask = L2CAP_FEAT_FIXED_CHAN | L2CAP_FEAT_UCD;
 
-static LIST_HEAD(chan_list);
 static DEFINE_RWLOCK(chan_list_lock);
+static __guarded_by(&chan_list_lock) LIST_HEAD(chan_list);
 
 static struct sk_buff *l2cap_build_cmd(struct l2cap_conn *conn,
 				       u8 code, u8 ident, u16 dlen, void *data);
@@ -59,6 +59,8 @@ static void l2cap_tx(struct l2cap_chan *chan, struct l2cap_ctrl *control,
 static void l2cap_retrans_timeout(struct work_struct *work);
 static void l2cap_monitor_timeout(struct work_struct *work);
 static void l2cap_ack_timeout(struct work_struct *work);
+static void __l2cap_chan_close(struct l2cap_chan *chan, int reason)
+	__must_hold(&chan->lock) __must_hold(&chan->conn->lock);
 
 static inline u8 bdaddr_type(u8 link_type, u8 bdaddr_type)
 {
@@ -86,6 +88,7 @@ static inline u8 bdaddr_dst_type(struct hci_conn *hcon)
 
 static struct l2cap_chan *__l2cap_get_chan_by_dcid(struct l2cap_conn *conn,
 						   u16 cid)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *c;
 
@@ -98,6 +101,7 @@ static struct l2cap_chan *__l2cap_get_chan_by_dcid(struct l2cap_conn *conn,
 
 static struct l2cap_chan *__l2cap_get_chan_by_scid(struct l2cap_conn *conn,
 						   u16 cid)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *c;
 
@@ -109,45 +113,40 @@ static struct l2cap_chan *__l2cap_get_chan_by_scid(struct l2cap_conn *conn,
 }
 
 /* Find channel with given SCID.
- * Returns a reference locked channel.
+ * Returns a reference.
  */
 static struct l2cap_chan *l2cap_get_chan_by_scid(struct l2cap_conn *conn,
 						 u16 cid)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *c;
 
 	c = __l2cap_get_chan_by_scid(conn, cid);
-	if (c) {
-		/* Only lock if chan reference is not 0 */
+	if (c)
 		c = l2cap_chan_hold_unless_zero(c);
-		if (c)
-			l2cap_chan_lock(c);
-	}
 
 	return c;
 }
 
 /* Find channel with given DCID.
- * Returns a reference locked channel.
+ * Returns a reference.
  */
 static struct l2cap_chan *l2cap_get_chan_by_dcid(struct l2cap_conn *conn,
 						 u16 cid)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *c;
 
 	c = __l2cap_get_chan_by_dcid(conn, cid);
-	if (c) {
-		/* Only lock if chan reference is not 0 */
+	if (c)
 		c = l2cap_chan_hold_unless_zero(c);
-		if (c)
-			l2cap_chan_lock(c);
-	}
 
 	return c;
 }
 
 static struct l2cap_chan *__l2cap_get_chan_by_ident(struct l2cap_conn *conn,
 						    u8 ident)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *c;
 
@@ -160,6 +159,7 @@ static struct l2cap_chan *__l2cap_get_chan_by_ident(struct l2cap_conn *conn,
 
 static struct l2cap_chan *__l2cap_global_chan_by_addr(__le16 psm, bdaddr_t *src,
 						      u8 src_type)
+	__must_hold_shared(&chan_list_lock)
 {
 	struct l2cap_chan *c;
 
@@ -237,6 +237,7 @@ int l2cap_add_scid(struct l2cap_chan *chan,  __u16 scid)
 }
 
 static u16 l2cap_alloc_cid(struct l2cap_conn *conn)
+	__must_hold(&conn->lock)
 {
 	u16 cid, dyn_end;
 
@@ -430,7 +431,7 @@ static void l2cap_chan_timeout(struct work_struct *work)
 	else
 		reason = ETIMEDOUT;
 
-	l2cap_chan_close(chan, reason);
+	__l2cap_chan_close(chan, reason);
 
 	chan->ops->close(chan);
 
@@ -622,6 +623,10 @@ void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan)
 	BT_DBG("conn %p, psm 0x%2.2x, dcid 0x%4.4x", conn,
 	       __le16_to_cpu(chan->psm), chan->dcid);
 
+	/* Caller must ensure l2cap_chan is linked to l2cap_conn only once */
+	if (WARN_ON_ONCE(chan->conn || test_bit(FLAG_DEL, &chan->flags)))
+		return;
+
 	conn->disc_reason = HCI_ERROR_REMOTE_USER_TERM;
 
 	chan->conn = l2cap_conn_get(conn);
@@ -673,12 +678,16 @@ void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan)
 void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan)
 {
 	mutex_lock(&conn->lock);
+	l2cap_chan_lock(chan);
 	__l2cap_chan_add(conn, chan);
+	l2cap_chan_unlock(chan);
 	mutex_unlock(&conn->lock);
 }
 
 void l2cap_chan_del(struct l2cap_chan *chan, int err)
 {
+	lockdep_assert(!chan->conn || lockdep_is_held(&chan->conn->lock));
+
 	__clear_chan_timer(chan);
 
 	BT_DBG("chan %p, err %d, state %s", chan, err,
@@ -733,6 +742,7 @@ EXPORT_SYMBOL_GPL(l2cap_chan_del);
 
 static void __l2cap_chan_list_id(struct l2cap_conn *conn, u16 id,
 				 l2cap_chan_func_t func, void *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *chan, *l;
 
@@ -744,6 +754,7 @@ static void __l2cap_chan_list_id(struct l2cap_conn *conn, u16 id,
 
 static void __l2cap_chan_list(struct l2cap_conn *conn, l2cap_chan_func_t func,
 			      void *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *chan;
 
@@ -808,6 +819,8 @@ static void l2cap_chan_le_connect_reject(struct l2cap_chan *chan)
 }
 
 static void l2cap_chan_ecred_connect_reject(struct l2cap_chan *chan)
+	__must_hold(&chan->lock)
+	__must_hold(&chan->conn->lock)
 {
 	l2cap_state_change(chan, BT_DISCONN);
 
@@ -835,7 +848,7 @@ static void l2cap_chan_connect_reject(struct l2cap_chan *chan)
 	l2cap_send_cmd(conn, chan->ident, L2CAP_CONN_RSP, sizeof(rsp), &rsp);
 }
 
-void l2cap_chan_close(struct l2cap_chan *chan, int reason)
+static void __l2cap_chan_close(struct l2cap_chan *chan, int reason)
 {
 	struct l2cap_conn *conn = chan->conn;
 
@@ -884,7 +897,54 @@ void l2cap_chan_close(struct l2cap_chan *chan, int reason)
 		break;
 	}
 }
-EXPORT_SYMBOL(l2cap_chan_close);
+
+/* Take chan->lock. If chan->conn is non-NULL, take new reference on it, take
+ * chan->conn->lock, and return true. Otherwise return false.
+ */
+bool l2cap_chan_lock_conn(struct l2cap_chan *chan)
+	__context_unsafe(/* conditional locking */)
+{
+	/* Handle conn->lock > chan->lock ordering + race on chan->conn */
+	for (;;) {
+		struct l2cap_conn *conn;
+
+		l2cap_chan_lock(chan);
+		conn = chan->conn;
+		if (conn)
+			l2cap_conn_get(conn);
+		l2cap_chan_unlock(chan);
+
+		if (conn)
+			mutex_lock(&conn->lock);
+
+		l2cap_chan_lock(chan);
+
+		if (chan->conn != conn) {
+			l2cap_chan_unlock(chan);
+			if (conn) {
+				mutex_unlock(&conn->lock);
+				l2cap_conn_put(conn);
+			}
+			schedule();
+			continue;
+		}
+
+		return chan->conn;
+	}
+}
+
+void l2cap_chan_close_unlocked(struct l2cap_chan *chan, int reason)
+{
+	bool have_conn;
+
+	have_conn = l2cap_chan_lock_conn(chan);
+
+	/* Context analysis: consider chan->conn->lock held also if conn NULL */
+	context_unsafe(__l2cap_chan_close(chan, reason));
+
+	l2cap_chan_unlock_conn(chan, have_conn);
+}
+EXPORT_SYMBOL(l2cap_chan_close_unlocked);
 
 static inline u8 l2cap_get_auth_type(struct l2cap_chan *chan)
 {
@@ -1283,6 +1343,8 @@ void l2cap_send_conn_req(struct l2cap_chan *chan)
 }
 
 static void l2cap_chan_ready(struct l2cap_chan *chan)
+	__must_hold(&chan->lock)
+	__must_hold(&chan->conn->lock)
 {
 	/* The channel may have already been flagged as connected in
 	 * case of receiving data before the L2CAP info req/rsp
@@ -1384,6 +1446,7 @@ static void l2cap_ecred_defer_connect(struct l2cap_chan *chan, void *data)
 }
 
 static void l2cap_ecred_connect(struct l2cap_chan *chan)
+	__must_hold(&chan->conn->lock)
 {
 	struct l2cap_conn *conn = chan->conn;
 	struct l2cap_ecred_conn_data data;
@@ -1417,6 +1480,8 @@ static void l2cap_ecred_connect(struct l2cap_chan *chan)
 }
 
 static void l2cap_le_start(struct l2cap_chan *chan)
+	__must_hold(&chan->lock)
+	__must_hold(&chan->conn->lock)
 {
 	struct l2cap_conn *conn = chan->conn;
 
@@ -1437,6 +1502,8 @@ static void l2cap_le_start(struct l2cap_chan *chan)
 }
 
 static void l2cap_start_connection(struct l2cap_chan *chan)
+	__must_hold(&chan->lock)
+	__must_hold(&chan->conn->lock)
 {
 	if (chan->conn->hcon->type == LE_LINK) {
 		l2cap_le_start(chan);
@@ -1486,6 +1553,8 @@ static bool l2cap_check_enc_key_size(struct hci_conn *hcon,
 }
 
 static void l2cap_do_start(struct l2cap_chan *chan)
+	__must_hold(&chan->lock)
+	__must_hold(&chan->conn->lock)
 {
 	struct l2cap_conn *conn = chan->conn;
 
@@ -1552,6 +1621,7 @@ static void l2cap_send_disconn_req(struct l2cap_chan *chan, int err)
 
 /* ---- L2CAP connections ---- */
 static void l2cap_conn_start(struct l2cap_conn *conn)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *chan, *tmp;
 
@@ -1560,6 +1630,8 @@ static void l2cap_conn_start(struct l2cap_conn *conn)
 	list_for_each_entry_safe(chan, tmp, &conn->chan_l, list) {
 		l2cap_chan_lock(chan);
 
+		lockdep_assert_held(&chan->conn->lock);
+
 		if (chan->chan_type != L2CAP_CHAN_CONN_ORIENTED) {
 			l2cap_chan_ready(chan);
 			l2cap_chan_unlock(chan);
@@ -1576,7 +1648,7 @@ static void l2cap_conn_start(struct l2cap_conn *conn)
 			if (!l2cap_mode_supported(chan->mode, conn->feat_mask)
 			    && test_bit(CONF_STATE2_DEVICE,
 					&chan->conf_state)) {
-				l2cap_chan_close(chan, ECONNRESET);
+				__l2cap_chan_close(chan, ECONNRESET);
 				l2cap_chan_unlock(chan);
 				continue;
 			}
@@ -1584,7 +1656,7 @@ static void l2cap_conn_start(struct l2cap_conn *conn)
 			if (l2cap_check_enc_key_size(conn->hcon, chan))
 				l2cap_start_connection(chan);
 			else
-				l2cap_chan_close(chan, ECONNREFUSED);
+				__l2cap_chan_close(chan, ECONNREFUSED);
 
 		} else if (chan->state == BT_CONNECT2) {
 			struct l2cap_conn_rsp rsp;
@@ -1676,6 +1748,8 @@ static void l2cap_conn_ready(struct l2cap_conn *conn)
 
 		l2cap_chan_lock(chan);
 
+		lockdep_assert_held(&chan->conn->lock);
+
 		if (hcon->type == LE_LINK) {
 			l2cap_le_start(chan);
 		} else if (chan->chan_type != L2CAP_CHAN_CONN_ORIENTED) {
@@ -1698,6 +1772,7 @@ static void l2cap_conn_ready(struct l2cap_conn *conn)
 
 /* Notify sockets that we cannot guaranty reliability anymore */
 static void l2cap_conn_unreliable(struct l2cap_conn *conn, int err)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *chan;
 
@@ -1830,6 +1905,8 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err)
 		l2cap_chan_hold(chan);
 		l2cap_chan_lock(chan);
 
+		lockdep_assert_held(&chan->conn->lock);
+
 		l2cap_chan_del(chan, err);
 
 		chan->ops->close(chan);
@@ -2995,6 +3072,7 @@ static void l2cap_pass_to_tx_fbit(struct l2cap_chan *chan,
 
 /* Copy frame to all raw sockets on that connection */
 static void l2cap_raw_recv(struct l2cap_conn *conn, struct sk_buff *skb)
+	__must_hold(&conn->lock)
 {
 	struct sk_buff *nskb;
 	struct l2cap_chan *chan;
@@ -3888,6 +3966,7 @@ static void l2cap_ecred_list_defer(struct l2cap_chan *chan, void *data)
 }
 
 struct l2cap_ecred_rsp_data {
+	struct l2cap_chan *locked_chan;
 	struct {
 		struct l2cap_ecred_conn_rsp_hdr rsp;
 		__le16 scid[L2CAP_ECRED_MAX_CID];
@@ -3895,11 +3974,42 @@ struct l2cap_ecred_rsp_data {
 	int count;
 };
 
+/* Lock @chan if it is not @locked_chan, and has same or lower nesting level.
+ *
+ * They must have the same chan->conn, and conn->lock must be held.
+ *
+ * Caller must ensure @chan has lock nesting level <= that of @locked_chan, as
+ * nested locking of l2cap_chan of different levels is allowed also without
+ * holding conn->lock.
+ *
+ * See l2cap.h for the global l2cap_chan locking rules.
+ */
+static bool l2cap_chan_try_sibling_lock(struct l2cap_chan *chan,
+					struct l2cap_chan *locked_chan)
+	__must_hold(&locked_chan->lock)
+	__must_hold(&locked_chan->conn->lock)
+	__cond_acquires(true, &chan->lock)
+{
+	if (chan == locked_chan)
+		return false;
+
+	if (WARN_ON_ONCE(locked_chan->conn != chan->conn))
+		return false;
+
+	if (WARN_ON_ONCE(atomic_read(&locked_chan->nesting)
+			 < atomic_read(&chan->nesting)))
+		return false;
+
+	mutex_lock_nest_lock(&chan->lock, &locked_chan->conn->lock);
+	return true;
+}
+
 static void l2cap_ecred_rsp_defer(struct l2cap_chan *chan, void *data)
 {
 	struct l2cap_ecred_rsp_data *rsp = data;
 	struct l2cap_ecred_conn_rsp *rsp_flex =
 		container_of(&rsp->pdu.rsp, struct l2cap_ecred_conn_rsp, hdr);
+	bool locked;
 
 	if (chan->mode != L2CAP_MODE_EXT_FLOWCTL)
 		return;
@@ -3911,6 +4021,22 @@ static void l2cap_ecred_rsp_defer(struct l2cap_chan *chan, void *data)
 	    !test_and_clear_bit(FLAG_DEFER_SETUP, &chan->flags))
 		return;
 
+	lockdep_assert_held(&rsp->locked_chan->lock);
+	lockdep_assert_held(&rsp->locked_chan->conn->lock);
+
+	l2cap_chan_hold(chan);
+
+	locked = l2cap_chan_try_sibling_lock(chan, rsp->locked_chan);
+
+	/* Cannot occur: PARENT channels do not appear in chan_l, and SMP
+	 * channels never have FLAG_DEFER_SETUP.
+	 */
+	if (context_unsafe(!locked && chan != rsp->locked_chan))
+		goto done;
+
+	lockdep_assert_held(&chan->lock);
+	lockdep_assert_held(&chan->conn->lock);
+
 	/* Reset ident so only one response is sent */
 	chan->ident = 0;
 
@@ -3923,6 +4049,12 @@ static void l2cap_ecred_rsp_defer(struct l2cap_chan *chan, void *data)
 		rsp_flex->dcid[rsp->count++] = cpu_to_le16(chan->scid);
 	else
 		l2cap_chan_del(chan, ECONNRESET);
+
+done:
+	if (locked)
+		l2cap_chan_unlock(chan);
+
+	l2cap_chan_put(chan);
 }
 
 void __l2cap_ecred_conn_rsp_defer(struct l2cap_chan *chan)
@@ -3934,11 +4066,15 @@ void __l2cap_ecred_conn_rsp_defer(struct l2cap_chan *chan)
 
 	if (!id)
 		return;
+	if (!test_bit(FLAG_DEFER_SETUP, &chan->flags))
+		return;
 
 	BT_DBG("chan %p id %d", chan, id);
 
 	memset(&data, 0, sizeof(data));
 
+	data.locked_chan = chan;
+
 	data.pdu.rsp.mtu     = cpu_to_le16(chan->imtu);
 	data.pdu.rsp.mps     = cpu_to_le16(chan->mps);
 	data.pdu.rsp.credits = cpu_to_le16(chan->rx_credits);
@@ -4048,6 +4184,7 @@ static void l2cap_conf_rfc_get(struct l2cap_chan *chan, void *rsp, int len)
 static inline int l2cap_command_rej(struct l2cap_conn *conn,
 				    struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				    u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_cmd_rej_unk *rej = (struct l2cap_cmd_rej_unk *) data;
 
@@ -4080,6 +4217,8 @@ static inline int l2cap_command_rej(struct l2cap_conn *conn,
  */
 static struct l2cap_chan *l2cap_new_connection(struct l2cap_conn *conn,
 					       struct l2cap_chan *pchan)
+	__must_hold(&conn->lock)
+	__must_hold(&pchan->lock)
 {
 	struct l2cap_chan *chan;
 
@@ -4087,23 +4226,31 @@ static struct l2cap_chan *l2cap_new_connection(struct l2cap_conn *conn,
 	if (!chan)
 		return NULL;
 
+	l2cap_chan_lock(chan);
+
 	l2cap_chan_set_defaults(chan, pchan);
 	chan->ops = pchan->ops;
 
 	__l2cap_chan_add(conn, chan);
 
+	lockdep_assert_held(&chan->conn->lock);
+
 	if (pchan->ops->new_connection &&
 	    pchan->ops->new_connection(pchan, chan) < 0) {
 		l2cap_chan_del(chan, 0);
+		l2cap_chan_unlock(chan);
 		l2cap_chan_put(chan);
 		return NULL;
 	}
 
+	l2cap_chan_unlock(chan);
+
 	return chan;
 }
 
 static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd,
 			  u8 *data, u8 rsp_code)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_conn_req *req = (struct l2cap_conn_req *) data;
 	struct l2cap_conn_rsp rsp;
@@ -4120,7 +4267,13 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd,
 					 &conn->hcon->dst, ACL_LINK);
 	if (!pchan) {
 		result = L2CAP_CR_BAD_PSM;
-		goto response;
+
+		rsp.scid   = cpu_to_le16(scid);
+		rsp.dcid   = cpu_to_le16(dcid);
+		rsp.result = cpu_to_le16(result);
+		rsp.status = cpu_to_le16(status);
+		l2cap_send_cmd(conn, cmd->ident, rsp_code, sizeof(rsp), &rsp);
+		return;
 	}
 
 	l2cap_chan_lock(pchan);
@@ -4202,9 +4355,6 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd,
 	rsp.status = cpu_to_le16(status);
 	l2cap_send_cmd(conn, cmd->ident, rsp_code, sizeof(rsp), &rsp);
 
-	if (!pchan)
-		return;
-
 	if (result == L2CAP_CR_PEND && status == L2CAP_CS_NO_INFO) {
 		struct l2cap_info_req info;
 		info.type = cpu_to_le16(L2CAP_IT_FEAT_MASK);
@@ -4233,6 +4383,7 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd,
 
 static int l2cap_connect_req(struct l2cap_conn *conn,
 			     struct l2cap_cmd_hdr *cmd, u16 cmd_len, u8 *data)
+	__must_hold(&conn->lock)
 {
 	if (cmd_len < sizeof(struct l2cap_conn_req))
 		return -EPROTO;
@@ -4244,6 +4395,7 @@ static int l2cap_connect_req(struct l2cap_conn *conn,
 static int l2cap_connect_create_rsp(struct l2cap_conn *conn,
 				    struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				    u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_conn_rsp *rsp = (struct l2cap_conn_rsp *) data;
 	u16 scid, dcid, result, status;
@@ -4284,6 +4436,8 @@ static int l2cap_connect_create_rsp(struct l2cap_conn *conn,
 
 	l2cap_chan_lock(chan);
 
+	lockdep_assert_held(&chan->conn->lock);
+
 	switch (result) {
 	case L2CAP_CR_SUCCESS:
 		if (__l2cap_get_chan_by_dcid(conn, dcid)) {
@@ -4361,6 +4515,7 @@ static void cmd_reject_invalid_cid(struct l2cap_conn *conn, u8 ident,
 static inline int l2cap_config_req(struct l2cap_conn *conn,
 				   struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				   u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_conf_req *req = (struct l2cap_conf_req *) data;
 	u16 dcid, flags;
@@ -4382,6 +4537,10 @@ static inline int l2cap_config_req(struct l2cap_conn *conn,
 		return 0;
 	}
 
+	l2cap_chan_lock(chan);
+
+	lockdep_assert_held(&chan->conn->lock);
+
 	if (chan->state != BT_CONFIG && chan->state != BT_CONNECT2 &&
 	    chan->state != BT_CONNECTED) {
 		cmd_reject_invalid_cid(conn, cmd->ident, chan->scid,
@@ -4472,6 +4631,7 @@ static inline int l2cap_config_req(struct l2cap_conn *conn,
 static inline int l2cap_config_rsp(struct l2cap_conn *conn,
 				   struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				   u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_conf_rsp *rsp = (struct l2cap_conf_rsp *)data;
 	u16 scid, flags, result;
@@ -4493,6 +4653,10 @@ static inline int l2cap_config_rsp(struct l2cap_conn *conn,
 	if (!chan)
 		return 0;
 
+	l2cap_chan_lock(chan);
+
+	lockdep_assert_held(&chan->conn->lock);
+
 	switch (result) {
 	case L2CAP_CONF_SUCCESS:
 		l2cap_conf_rfc_get(chan, rsp->data, len);
@@ -4579,6 +4743,7 @@ static inline int l2cap_config_rsp(struct l2cap_conn *conn,
 static inline int l2cap_disconnect_req(struct l2cap_conn *conn,
 				       struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				       u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_disconn_req *req = (struct l2cap_disconn_req *) data;
 	struct l2cap_disconn_rsp rsp;
@@ -4599,6 +4764,10 @@ static inline int l2cap_disconnect_req(struct l2cap_conn *conn,
 		return 0;
 	}
 
+	l2cap_chan_lock(chan);
+
+	lockdep_assert_held(&chan->conn->lock);
+
 	rsp.dcid = cpu_to_le16(chan->scid);
 	rsp.scid = cpu_to_le16(chan->dcid);
 	l2cap_send_cmd(conn, cmd->ident, L2CAP_DISCONN_RSP, sizeof(rsp), &rsp);
@@ -4618,6 +4787,7 @@ static inline int l2cap_disconnect_req(struct l2cap_conn *conn,
 static inline int l2cap_disconnect_rsp(struct l2cap_conn *conn,
 				       struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				       u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_disconn_rsp *rsp = (struct l2cap_disconn_rsp *) data;
 	u16 dcid, scid;
@@ -4636,6 +4806,10 @@ static inline int l2cap_disconnect_rsp(struct l2cap_conn *conn,
 		return 0;
 	}
 
+	l2cap_chan_lock(chan);
+
+	lockdep_assert_held(&chan->conn->lock);
+
 	if (chan->state != BT_DISCONN) {
 		l2cap_chan_unlock(chan);
 		l2cap_chan_put(chan);
@@ -4703,6 +4877,7 @@ static inline int l2cap_information_req(struct l2cap_conn *conn,
 static inline int l2cap_information_rsp(struct l2cap_conn *conn,
 					struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 					u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_info_rsp *rsp = (struct l2cap_info_rsp *) data;
 	u16 type, result;
@@ -4810,6 +4985,7 @@ static inline int l2cap_conn_param_update_req(struct l2cap_conn *conn,
 static int l2cap_le_connect_rsp(struct l2cap_conn *conn,
 				struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_le_conn_rsp *rsp = (struct l2cap_le_conn_rsp *) data;
 	struct hci_conn *hcon = conn->hcon;
@@ -4846,6 +5022,8 @@ static int l2cap_le_connect_rsp(struct l2cap_conn *conn,
 
 	l2cap_chan_lock(chan);
 
+	lockdep_assert_held(&chan->conn->lock);
+
 	switch (result) {
 	case L2CAP_CR_LE_SUCCESS:
 		if (__l2cap_get_chan_by_dcid(conn, dcid)) {
@@ -4916,6 +5094,7 @@ static void l2cap_put_ident(struct l2cap_conn *conn, u8 code, u8 id)
 static inline int l2cap_bredr_sig_cmd(struct l2cap_conn *conn,
 				      struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				      u8 *data)
+	__must_hold(&conn->lock)
 {
 	int err = 0;
 
@@ -4977,6 +5156,7 @@ static inline int l2cap_bredr_sig_cmd(struct l2cap_conn *conn,
 static int l2cap_le_connect_req(struct l2cap_conn *conn,
 				struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_le_conn_req *req = (struct l2cap_le_conn_req *) data;
 	struct l2cap_le_conn_rsp rsp;
@@ -5060,6 +5240,10 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn,
 		goto response_unlock;
 	}
 
+	l2cap_chan_lock(chan);
+
+	lockdep_assert_held(&chan->conn->lock);
+
 	bacpy(&chan->src, &conn->hcon->src);
 	bacpy(&chan->dst, &conn->hcon->dst);
 	chan->src_type = bdaddr_src_type(conn->hcon);
@@ -5093,6 +5277,8 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn,
 		result = L2CAP_CR_LE_SUCCESS;
 	}
 
+	l2cap_chan_unlock(chan);
+
 response_unlock:
 	l2cap_chan_unlock(pchan);
 	l2cap_chan_put(pchan);
@@ -5121,6 +5307,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn,
 static inline int l2cap_le_credits(struct l2cap_conn *conn,
 				   struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				   u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_le_credits *pkt;
 	struct l2cap_chan *chan;
@@ -5139,6 +5326,8 @@ static inline int l2cap_le_credits(struct l2cap_conn *conn,
 	if (!chan)
 		return -EBADSLT;
 
+	l2cap_chan_lock(chan);
+
 	max_credits = LE_FLOWCTL_MAX_CREDITS - chan->tx_credits;
 	if (credits > max_credits) {
 		BT_ERR("LE credits overflow");
@@ -5168,6 +5357,7 @@ static inline int l2cap_le_credits(struct l2cap_conn *conn,
 static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn,
 				       struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				       u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_ecred_conn_req *req = (void *) data;
 	DEFINE_RAW_FLEX(struct l2cap_ecred_conn_rsp, pdu, dcid, L2CAP_ECRED_MAX_CID);
@@ -5283,6 +5473,10 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn,
 			continue;
 		}
 
+		l2cap_chan_lock(chan);
+
+		lockdep_assert_held(&chan->conn->lock);
+
 		bacpy(&chan->src, &conn->hcon->src);
 		bacpy(&chan->dst, &conn->hcon->dst);
 		chan->src_type = bdaddr_src_type(conn->hcon);
@@ -5315,6 +5509,8 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn,
 		} else {
 			l2cap_chan_ready(chan);
 		}
+
+		l2cap_chan_unlock(chan);
 	}
 
 unlock:
@@ -5336,6 +5532,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn,
 static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
 				       struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				       u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_ecred_conn_rsp *rsp = (void *) data;
 	struct hci_conn *hcon = conn->hcon;
@@ -5366,12 +5563,16 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
 		    chan->state == BT_CONNECTED)
 			continue;
 
+		l2cap_chan_hold(chan);
 		l2cap_chan_lock(chan);
 
+		lockdep_assert_held(&chan->conn->lock);
+
 		/* Check that there is a dcid for each pending channel */
 		if (cmd_len < sizeof(dcid)) {
 			l2cap_chan_del(chan, ECONNREFUSED);
 			l2cap_chan_unlock(chan);
+			l2cap_chan_put(chan);
 			continue;
 		}
 
@@ -5410,6 +5611,8 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
 				__set_chan_timer(orig, 0);
 				l2cap_chan_unlock(orig);
 			}
+
+			l2cap_chan_put(chan);
 			continue;
 		}
 
@@ -5455,6 +5658,7 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
 		}
 
 		l2cap_chan_unlock(chan);
+		l2cap_chan_put(chan);
 	}
 
 	return err;
@@ -5463,6 +5667,7 @@ static inline int l2cap_ecred_conn_rsp(struct l2cap_conn *conn,
 static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn,
 					 struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 					 u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_ecred_reconf_req *req = (void *) data;
 	struct l2cap_ecred_reconf_rsp rsp;
@@ -5561,6 +5766,7 @@ static inline int l2cap_ecred_reconf_req(struct l2cap_conn *conn,
 static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn,
 					 struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 					 u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *chan, *tmp;
 	struct l2cap_ecred_reconf_rsp *rsp = (void *)data;
@@ -5589,6 +5795,8 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn,
 			continue;
 		l2cap_chan_lock(chan);
 
+		lockdep_assert_held(&chan->conn->lock);
+
 		l2cap_chan_del(chan, ECONNRESET);
 
 		l2cap_chan_unlock(chan);
@@ -5601,6 +5809,7 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn,
 static inline int l2cap_le_command_rej(struct l2cap_conn *conn,
 				       struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				       u8 *data)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_cmd_rej_unk *rej = (struct l2cap_cmd_rej_unk *) data;
 	struct l2cap_chan *chan;
@@ -5617,6 +5826,7 @@ static inline int l2cap_le_command_rej(struct l2cap_conn *conn,
 		goto done;
 
 	l2cap_chan_lock(chan);
+	lockdep_assert_held(&chan->conn->lock);
 	l2cap_chan_del(chan, ECONNREFUSED);
 	l2cap_chan_unlock(chan);
 	l2cap_chan_put(chan);
@@ -5628,6 +5838,7 @@ static inline int l2cap_le_command_rej(struct l2cap_conn *conn,
 static inline int l2cap_le_sig_cmd(struct l2cap_conn *conn,
 				   struct l2cap_cmd_hdr *cmd, u16 cmd_len,
 				   u8 *data)
+	__must_hold(&conn->lock)
 {
 	int err = 0;
 
@@ -5692,6 +5903,7 @@ static inline int l2cap_le_sig_cmd(struct l2cap_conn *conn,
 
 static inline void l2cap_le_sig_channel(struct l2cap_conn *conn,
 					struct sk_buff *skb)
+	__must_hold(&conn->lock)
 {
 	struct hci_conn *hcon = conn->hcon;
 	struct l2cap_cmd_hdr *cmd;
@@ -5750,6 +5962,7 @@ static inline void l2cap_sig_send_mtu_rej(struct l2cap_conn *conn, u8 ident)
 
 static inline void l2cap_sig_channel(struct l2cap_conn *conn,
 				     struct sk_buff *skb)
+	__must_hold(&conn->lock)
 {
 	struct hci_conn *hcon = conn->hcon;
 	struct l2cap_cmd_hdr *cmd;
@@ -6702,9 +6915,17 @@ static int l2cap_stream_rx(struct l2cap_chan *chan, struct l2cap_ctrl *control,
 static int l2cap_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb)
 {
 	struct l2cap_ctrl *control = &bt_cb(skb)->l2cap;
-	u16 len;
+	u16 len, min_len;
 	u8 event;
 
+	min_len = test_bit(FLAG_EXT_CTRL, &chan->flags) ?
+		  L2CAP_EXT_CTRL_SIZE : L2CAP_ENH_CTRL_SIZE;
+	if (chan->fcs == L2CAP_FCS_CRC16)
+		min_len += L2CAP_FCS_SIZE;
+
+	if (skb->len < min_len)
+		goto drop;
+
 	__unpack_control(chan, skb);
 
 	len = skb->len;
@@ -6987,6 +7208,7 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb)
 
 static void l2cap_data_channel(struct l2cap_conn *conn, u16 cid,
 			       struct sk_buff *skb)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_chan *chan;
 
@@ -6998,6 +7220,10 @@ static void l2cap_data_channel(struct l2cap_conn *conn, u16 cid,
 		return;
 	}
 
+	l2cap_chan_lock(chan);
+
+	lockdep_assert_held(&chan->conn->lock);
+
 	BT_DBG("chan %p, len %d", chan, skb->len);
 
 	/* If we receive data on a fixed channel before the info req/rsp
@@ -7093,6 +7319,7 @@ static void l2cap_conless_channel(struct l2cap_conn *conn, __le16 psm,
 }
 
 static void l2cap_recv_frame(struct l2cap_conn *conn, struct sk_buff *skb)
+	__must_hold(&conn->lock)
 {
 	struct l2cap_hdr *lh = (void *) skb->data;
 	struct hci_conn *hcon = conn->hcon;
@@ -7202,9 +7429,9 @@ static struct l2cap_conn *l2cap_conn_add(struct hci_conn *hcon)
 	     hci_dev_test_flag(hcon->hdev, HCI_FORCE_BREDR_SMP)))
 		conn->local_fixed_chan |= L2CAP_FC_SMP_BREDR;
 
-	mutex_init(&conn->lock);
-
-	INIT_LIST_HEAD(&conn->chan_l);
+	scoped_guard(mutex_init, &conn->lock) {
+		INIT_LIST_HEAD(&conn->chan_l);
+	}
 	INIT_LIST_HEAD(&conn->users);
 
 	INIT_DELAYED_WORK(&conn->info_timer, l2cap_info_timeout);
@@ -7408,7 +7635,8 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid,
 		}
 	}
 
-	if (cid && __l2cap_get_chan_by_dcid(conn, cid)) {
+	if ((cid && __l2cap_get_chan_by_dcid(conn, cid)) || chan->conn ||
+	    test_bit(FLAG_DEL, &chan->flags)) {
 		hci_conn_drop(hcon);
 		err = -EBUSY;
 		goto chan_unlock;
@@ -7420,6 +7648,8 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid,
 
 	__l2cap_chan_add(conn, chan);
 
+	lockdep_assert_held(&chan->conn->lock);
+
 	/* l2cap_chan_add takes its own ref so we can drop this one */
 	hci_conn_drop(hcon);
 
@@ -7583,6 +7813,8 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status)
 	 * we left off, because the list lock would prevent calling the
 	 * potentially sleeping l2cap_chan_lock() function.
 	 */
+	mutex_lock(&conn->lock);
+
 	pchan = l2cap_global_fixed_chan(NULL, hcon);
 	while (pchan) {
 		struct l2cap_chan *chan, *next;
@@ -7607,6 +7839,8 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status)
 		pchan = next;
 	}
 
+	mutex_unlock(&conn->lock);
+
 	l2cap_conn_ready(conn);
 }
 
@@ -7638,6 +7872,8 @@ static void l2cap_disconn_cfm(struct hci_conn *hcon, u8 reason)
 }
 
 static inline void l2cap_check_encryption(struct l2cap_chan *chan, u8 encrypt)
+	__must_hold(&chan->lock)
+	__must_hold(&chan->conn->lock)
 {
 	if (chan->chan_type != L2CAP_CHAN_CONN_ORIENTED)
 		return;
@@ -7647,7 +7883,7 @@ static inline void l2cap_check_encryption(struct l2cap_chan *chan, u8 encrypt)
 			__set_chan_timer(chan, L2CAP_ENC_TIMEOUT);
 		} else if (chan->sec_level == BT_SECURITY_HIGH ||
 			   chan->sec_level == BT_SECURITY_FIPS)
-			l2cap_chan_close(chan, ECONNREFUSED);
+			__l2cap_chan_close(chan, ECONNREFUSED);
 	} else {
 		if (chan->sec_level == BT_SECURITY_MEDIUM)
 			__clear_chan_timer(chan);
@@ -7670,6 +7906,8 @@ static void l2cap_security_cfm(struct hci_conn *hcon, u8 status, u8 encrypt)
 	list_for_each_entry(chan, &conn->chan_l, list) {
 		l2cap_chan_lock(chan);
 
+		lockdep_assert_held(&chan->conn->lock);
+
 		BT_DBG("chan %p scid 0x%4.4x state %s", chan, chan->scid,
 		       state_to_string(chan->state));
 
diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
index 1194c37e466f18..c92f31b997ddb0 100644
--- a/net/bluetooth/l2cap_sock.c
+++ b/net/bluetooth/l2cap_sock.c
@@ -109,6 +109,7 @@ static int l2cap_sock_bind(struct socket *sock, struct sockaddr_unsized *addr, i
 			return -EINVAL;
 	}
 
+	l2cap_chan_lock(chan);
 	lock_sock(sk);
 
 	if (sk->sk_state != BT_OPEN) {
@@ -174,6 +175,7 @@ static int l2cap_sock_bind(struct socket *sock, struct sockaddr_unsized *addr, i
 
 done:
 	release_sock(sk);
+	l2cap_chan_unlock(chan);
 	return err;
 }
 
@@ -865,15 +867,11 @@ static int l2cap_sock_setsockopt_old(struct socket *sock, int optname,
 		if (opt & L2CAP_LM_SECURE)
 			chan->sec_level = BT_SECURITY_HIGH;
 
-		if (opt & L2CAP_LM_MASTER)
-			set_bit(FLAG_ROLE_SWITCH, &chan->flags);
-		else
-			clear_bit(FLAG_ROLE_SWITCH, &chan->flags);
+		assign_bit(FLAG_ROLE_SWITCH, &chan->flags,
+			   opt & L2CAP_LM_MASTER);
 
-		if (opt & L2CAP_LM_RELIABLE)
-			set_bit(FLAG_FORCE_RELIABLE, &chan->flags);
-		else
-			clear_bit(FLAG_FORCE_RELIABLE, &chan->flags);
+		assign_bit(FLAG_FORCE_RELIABLE, &chan->flags,
+			   opt & L2CAP_LM_RELIABLE);
 		break;
 
 	default:
@@ -1040,10 +1038,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname,
 			}
 		}
 
-		if (opt)
-			set_bit(FLAG_FLUSHABLE, &chan->flags);
-		else
-			clear_bit(FLAG_FLUSHABLE, &chan->flags);
+		assign_bit(FLAG_FLUSHABLE, &chan->flags, opt);
 		break;
 
 	case BT_POWER:
@@ -1059,10 +1054,7 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname,
 		if (err)
 			break;
 
-		if (pwr.force_active)
-			set_bit(FLAG_FORCE_ACTIVE, &chan->flags);
-		else
-			clear_bit(FLAG_FORCE_ACTIVE, &chan->flags);
+		assign_bit(FLAG_FORCE_ACTIVE, &chan->flags, pwr.force_active);
 		break;
 
 	case BT_CHANNEL_POLICY:
@@ -1243,40 +1235,68 @@ static void l2cap_publish_rx_avail(struct l2cap_chan *chan)
 		l2cap_chan_rx_avail(chan, -1);
 }
 
-static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg,
-			      size_t len, int flags)
+static int l2cap_sock_defer(struct sock *sk)
 {
-	struct sock *sk = sock->sk;
-	struct l2cap_pinfo *pi = l2cap_pi(sk);
-	int err;
+	struct l2cap_chan *chan = l2cap_pi(sk)->chan;
+	bool have_conn;
+	int err = 0;
 
-	if (unlikely(flags & MSG_ERRQUEUE))
-		return sock_recv_errqueue(sk, msg, len, SOL_BLUETOOTH,
-					  BT_SCM_ERROR);
+	/* Fast path check */
+	lock_sock(sk);
+	if (sk->sk_state != BT_CONNECT2) {
+		release_sock(sk);
+		return 0;
+	}
+	release_sock(sk);
 
+	have_conn = l2cap_chan_lock_conn(chan);
 	lock_sock(sk);
 
 	if (sk->sk_state == BT_CONNECT2 && test_bit(BT_SK_DEFER_SETUP,
 						    &bt_sk(sk)->flags)) {
-		if (pi->chan->mode == L2CAP_MODE_EXT_FLOWCTL) {
+		err = 1;
+
+		if (!have_conn) {
+			release_sock(sk);
+			err = -ENOTCONN;
+		} else if (chan->mode == L2CAP_MODE_EXT_FLOWCTL) {
 			sk->sk_state = BT_CONNECTED;
-			pi->chan->state = BT_CONNECTED;
-			__l2cap_ecred_conn_rsp_defer(pi->chan);
-		} else if (bdaddr_type_is_le(pi->chan->src_type)) {
+			chan->state = BT_CONNECTED;
+			release_sock(sk);
+			__l2cap_ecred_conn_rsp_defer(chan);
+		} else if (bdaddr_type_is_le(chan->src_type)) {
 			sk->sk_state = BT_CONNECTED;
-			pi->chan->state = BT_CONNECTED;
-			__l2cap_le_connect_rsp_defer(pi->chan);
+			chan->state = BT_CONNECTED;
+			release_sock(sk);
+			__l2cap_le_connect_rsp_defer(chan);
 		} else {
 			sk->sk_state = BT_CONFIG;
-			pi->chan->state = BT_CONFIG;
-			__l2cap_connect_rsp_defer(pi->chan);
+			chan->state = BT_CONFIG;
+			release_sock(sk);
+			__l2cap_connect_rsp_defer(chan);
 		}
-
-		err = 0;
-		goto done;
+	} else {
+		release_sock(sk);
 	}
 
-	release_sock(sk);
+	l2cap_chan_unlock_conn(chan, have_conn);
+	return err;
+}
+
+static int l2cap_sock_recvmsg(struct socket *sock, struct msghdr *msg,
+			      size_t len, int flags)
+{
+	struct sock *sk = sock->sk;
+	struct l2cap_pinfo *pi = l2cap_pi(sk);
+	int err;
+
+	if (unlikely(flags & MSG_ERRQUEUE))
+		return sock_recv_errqueue(sk, msg, len, SOL_BLUETOOTH,
+					  BT_SCM_ERROR);
+
+	err = l2cap_sock_defer(sk);
+	if (err)
+		return err < 0 ? err : 0;
 
 	if (sock->type == SOCK_STREAM)
 		err = bt_sock_stream_recvmsg(sock, msg, len, flags);
@@ -1406,7 +1426,6 @@ static int l2cap_sock_shutdown(struct socket *sock, int how)
 {
 	struct sock *sk = sock->sk;
 	struct l2cap_chan *chan;
-	struct l2cap_conn *conn;
 	int err = 0;
 
 	BT_DBG("sock %p, sk %p, how %d", sock, sk, how);
@@ -1463,23 +1482,7 @@ static int l2cap_sock_shutdown(struct socket *sock, int how)
 	sk->sk_shutdown |= SEND_SHUTDOWN;
 	release_sock(sk);
 
-	l2cap_chan_lock(chan);
-	/* prevent conn structure from being freed */
-	conn = l2cap_conn_hold_unless_zero(chan->conn);
-	l2cap_chan_unlock(chan);
-
-	if (conn)
-		/* mutex lock must be taken before l2cap_chan_lock() */
-		mutex_lock(&conn->lock);
-
-	l2cap_chan_lock(chan);
-	l2cap_chan_close(chan, 0);
-	l2cap_chan_unlock(chan);
-
-	if (conn) {
-		mutex_unlock(&conn->lock);
-		l2cap_conn_put(conn);
-	}
+	l2cap_chan_close_unlocked(chan, 0);
 
 	lock_sock(sk);
 
@@ -1784,6 +1787,7 @@ static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state,
 static struct sk_buff *l2cap_sock_alloc_skb_cb(struct l2cap_chan *chan,
 					       unsigned long hdr_len,
 					       unsigned long len, int nb)
+	__must_hold(&chan->lock)
 {
 	struct sock *sk = chan->data;
 	struct sk_buff *skb;
diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c
index ac4864e56ec727..9b4688015abf60 100644
--- a/net/bluetooth/mgmt.c
+++ b/net/bluetooth/mgmt.c
@@ -301,6 +301,8 @@ static u8 mgmt_errno_status(int err)
 		return MGMT_STATUS_ALREADY_CONNECTED;
 	case -ENOTCONN:
 		return MGMT_STATUS_DISCONNECTED;
+	case -ECANCELED:
+		return MGMT_STATUS_CANCELLED;
 	}
 
 	return MGMT_STATUS_FAILED;
@@ -496,13 +498,9 @@ static int read_unconf_index_list(struct sock *sk, struct hci_dev *hdev,
 
 	read_lock(&hci_dev_list_lock);
 
-	count = 0;
-	list_for_each_entry(d, &hci_dev_list, list) {
-		if (hci_dev_test_flag(d, HCI_UNCONFIGURED))
-			count++;
-	}
+	count = list_count_nodes(&hci_dev_list);
 
-	rp_len = sizeof(*rp) + (2 * count);
+	rp_len = sizeof(*rp) + (sizeof(__le16) * count);
 	rp = kmalloc(rp_len, GFP_ATOMIC);
 	if (!rp) {
 		read_unlock(&hci_dev_list_lock);
@@ -1499,7 +1497,7 @@ static void cmd_complete_rsp(struct mgmt_pending_cmd *cmd, void *data)
 		return;
 	}
 
-	cmd_status_rsp(cmd, data);
+	cmd_status_rsp(cmd, &match->mgmt_status);
 }
 
 static int generic_cmd_complete(struct mgmt_pending_cmd *cmd, u8 status)
@@ -2316,6 +2314,8 @@ static void mesh_send_start_complete(struct hci_dev *hdev, void *data, int err)
 		hci_dev_clear_flag(hdev, HCI_MESH_SENDING);
 		/* Send Complete Error Code for handle */
 		mesh_send_complete(hdev, mesh_tx, false);
+		if (err != -ECANCELED)
+			mesh_next(hdev, NULL, 0);
 		return;
 	}
 
@@ -2425,19 +2425,28 @@ static int send_cancel(struct hci_dev *hdev, void *data)
 		do {
 			mesh_tx = mgmt_mesh_next(hdev, cmd->sk);
 
-			if (mesh_tx)
-				mesh_send_complete(hdev, mesh_tx, false);
+			if (mesh_tx) {
+				if (!hci_cmd_sync_dequeue(hdev, mesh_send_sync,
+							  mesh_tx, NULL))
+					mesh_send_complete(hdev, mesh_tx, false);
+			}
 		} while (mesh_tx);
 	} else {
 		mesh_tx = mgmt_mesh_find(hdev, cancel->handle);
 
-		if (mesh_tx && mesh_tx->sk == cmd->sk)
-			mesh_send_complete(hdev, mesh_tx, false);
+		if (mesh_tx && mesh_tx->sk == cmd->sk) {
+			if (!hci_cmd_sync_dequeue(hdev, mesh_send_sync,
+						  mesh_tx, NULL))
+				mesh_send_complete(hdev, mesh_tx, false);
+		}
 	}
 
 	mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL,
 			  0, NULL, 0);
 
+	if (!hci_dev_test_flag(hdev, HCI_MESH_SENDING))
+		mesh_next(hdev, NULL, 0);
+
 	return 0;
 }
 
@@ -5675,8 +5684,18 @@ static void mgmt_remove_adv_monitor_complete(struct hci_dev *hdev,
 	struct mgmt_pending_cmd *cmd = data;
 	struct mgmt_cp_remove_adv_monitor *cp;
 
-	if (status == -ECANCELED)
+	/* Reply to a cancelled command so bluetoothd's serialised mgmt queue
+	 * is not blocked.
+	 */
+	if (status == -ECANCELED) {
+		cp = cmd->param;
+		rp.monitor_handle = cp->monitor_handle;
+
+		mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode,
+				  mgmt_status(status), &rp, sizeof(rp));
+		mgmt_pending_free(cmd);
 		return;
+	}
 
 	hci_dev_lock(hdev);
 
diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c
index f7463f09228329..8a4205347c4ac9 100644
--- a/net/bluetooth/rfcomm/core.c
+++ b/net/bluetooth/rfcomm/core.c
@@ -62,10 +62,9 @@ static void rfcomm_make_uih(struct sk_buff *skb, u8 addr);
 
 static void rfcomm_process_connect(struct rfcomm_session *s);
 
-static struct rfcomm_session *rfcomm_session_create(bdaddr_t *src,
-							bdaddr_t *dst,
-							u8 sec_level,
-							int *err);
+static struct socket *rfcomm_session_connect(bdaddr_t *src, bdaddr_t *dst,
+					     u8 sec_level, int *err);
+static struct rfcomm_session *rfcomm_session_add(struct socket *sock, int state);
 static struct rfcomm_session *rfcomm_session_get(bdaddr_t *src, bdaddr_t *dst);
 static struct rfcomm_session *rfcomm_session_del(struct rfcomm_session *s);
 
@@ -365,28 +364,17 @@ static int rfcomm_check_channel(u8 channel)
 	return channel < 1 || channel > 30;
 }
 
-static int __rfcomm_dlc_open(struct rfcomm_dlc *d, bdaddr_t *src, bdaddr_t *dst, u8 channel)
+static int __rfcomm_dlc_open(struct rfcomm_dlc *d, struct rfcomm_session *s,
+			     u8 channel)
 {
-	struct rfcomm_session *s;
-	int err = 0;
 	u8 dlci;
 
-	BT_DBG("dlc %p state %ld %pMR -> %pMR channel %d",
-	       d, d->state, src, dst, channel);
-
-	if (rfcomm_check_channel(channel))
-		return -EINVAL;
+	BT_DBG("dlc %p state %ld session %p channel %d",
+	       d, d->state, s, channel);
 
 	if (d->state != BT_OPEN && d->state != BT_CLOSED)
 		return 0;
 
-	s = rfcomm_session_get(src, dst);
-	if (!s) {
-		s = rfcomm_session_create(src, dst, d->sec_level, &err);
-		if (!s)
-			return err;
-	}
-
 	dlci = __dlci(__session_dir(s), channel);
 
 	/* Check if DLCI already exists */
@@ -421,14 +409,84 @@ static int __rfcomm_dlc_open(struct rfcomm_dlc *d, bdaddr_t *src, bdaddr_t *dst,
 
 int rfcomm_dlc_open(struct rfcomm_dlc *d, bdaddr_t *src, bdaddr_t *dst, u8 channel)
 {
-	int r;
+	struct rfcomm_session *s;
+	struct socket *sock;
+	int err;
+
+	BT_DBG("dlc %p state %ld %pMR -> %pMR channel %d",
+	       d, d->state, src, dst, channel);
+
+	if (rfcomm_check_channel(channel))
+		return -EINVAL;
 
 	rfcomm_lock();
 
-	r = __rfcomm_dlc_open(d, src, dst, channel);
+	clear_bit(RFCOMM_CLOSED, &d->flags);
 
+	/* Do not page the remote device for a DLC that cannot be opened
+	 * anyway.  __rfcomm_dlc_open() looks at the state again once the
+	 * lock has been re-acquired below.
+	 */
+	if (d->state != BT_OPEN && d->state != BT_CLOSED) {
+		rfcomm_unlock();
+		return 0;
+	}
+
+	s = rfcomm_session_get(src, dst);
+	if (s) {
+		err = __rfcomm_dlc_open(d, s, channel);
+		rfcomm_unlock();
+		return err;
+	}
 	rfcomm_unlock();
-	return r;
+
+	/* There is no session for this pair yet.  kernel_connect() ends up in
+	 * l2cap_chan_connect(), which takes hdev->lock, and the HCI event
+	 * path takes rfcomm_mutex while holding hdev->lock, so the socket has
+	 * to be connected with rfcomm_mutex released.
+	 */
+	sock = rfcomm_session_connect(src, dst, d->sec_level, &err);
+	if (!sock)
+		return err;
+
+	rfcomm_lock();
+
+	/* The DLC may have been closed while the socket was connecting.  It
+	 * was not on a session, so rfcomm_dlc_close() could only mark it;
+	 * attaching it now would leave it with no owner.
+	 */
+	if (test_bit(RFCOMM_CLOSED, &d->flags)) {
+		rfcomm_unlock();
+		sock_release(sock);
+		return -ECONNRESET;
+	}
+
+	/* Another opener may have added a session for the same pair in the
+	 * meantime; that one is used and this socket is dropped.
+	 */
+	s = rfcomm_session_get(src, dst);
+	if (!s) {
+		s = rfcomm_session_add(sock, BT_BOUND);
+		if (s) {
+			s->initiator = 1;
+			sock = NULL;
+		}
+	}
+
+	err = s ? __rfcomm_dlc_open(d, s, channel) : -ENOMEM;
+
+	rfcomm_unlock();
+
+	if (sock)
+		sock_release(sock);
+
+	/* Over an existing ACL link the connection can complete before the
+	 * session reaches the list, and that wakeup is then lost, so let
+	 * krfcommd look at the socket state now.
+	 */
+	rfcomm_schedule();
+
+	return err;
 }
 
 static void __rfcomm_dlc_disconn(struct rfcomm_dlc *d)
@@ -508,8 +566,14 @@ int rfcomm_dlc_close(struct rfcomm_dlc *d, int err)
 	rfcomm_lock();
 
 	s = d->session;
-	if (!s)
+	if (!s) {
+		/* Not on a session yet: rfcomm_dlc_open() may be connecting a
+		 * socket for it with rfcomm_mutex released.  Leave a mark so
+		 * that it does not attach the DLC once it holds the lock again.
+		 */
+		set_bit(RFCOMM_CLOSED, &d->flags);
 		goto no_session;
+	}
 
 	/* after waiting on the mutex check the session still exists
 	 * then check the dlc still exists
@@ -757,12 +821,12 @@ static struct rfcomm_session *rfcomm_session_close(struct rfcomm_session *s,
 	return rfcomm_session_del(s);
 }
 
-static struct rfcomm_session *rfcomm_session_create(bdaddr_t *src,
-							bdaddr_t *dst,
-							u8 sec_level,
-							int *err)
+/* Creates the L2CAP socket a new session will run on and starts connecting
+ * it.  Must be called with rfcomm_mutex released.
+ */
+static struct socket *rfcomm_session_connect(bdaddr_t *src, bdaddr_t *dst,
+					     u8 sec_level, int *err)
 {
-	struct rfcomm_session *s = NULL;
 	struct sockaddr_l2 addr;
 	struct socket *sock;
 	struct sock *sk;
@@ -792,24 +856,16 @@ static struct rfcomm_session *rfcomm_session_create(bdaddr_t *src,
 		l2cap_pi(sk)->chan->mode = L2CAP_MODE_ERTM;
 	release_sock(sk);
 
-	s = rfcomm_session_add(sock, BT_BOUND);
-	if (!s) {
-		*err = -ENOMEM;
-		goto failed;
-	}
-
-	s->initiator = 1;
-
 	bacpy(&addr.l2_bdaddr, dst);
 	addr.l2_family = AF_BLUETOOTH;
 	addr.l2_psm    = cpu_to_le16(L2CAP_PSM_RFCOMM);
 	addr.l2_cid    = 0;
 	addr.l2_bdaddr_type = BDADDR_BREDR;
 	*err = kernel_connect(sock, (struct sockaddr_unsized *)&addr, sizeof(addr), O_NONBLOCK);
-	if (*err == 0 || *err == -EINPROGRESS)
-		return s;
+	if (*err && *err != -EINPROGRESS)
+		goto failed;
 
-	return rfcomm_session_del(s);
+	return sock;
 
 failed:
 	sock_release(sock);
@@ -1684,10 +1740,8 @@ static int rfcomm_recv_msc(struct rfcomm_session *s, int cr, struct sk_buff *skb
 		return 0;
 
 	if (cr) {
-		if (msc->v24_sig & RFCOMM_V24_FC && !d->cfc)
-			set_bit(RFCOMM_TX_THROTTLED, &d->flags);
-		else
-			clear_bit(RFCOMM_TX_THROTTLED, &d->flags);
+		assign_bit(RFCOMM_TX_THROTTLED, &d->flags,
+			   msc->v24_sig & RFCOMM_V24_FC && !d->cfc);
 
 		rfcomm_dlc_lock(d);
 
@@ -1817,7 +1871,8 @@ static struct rfcomm_session *rfcomm_recv_frame(struct rfcomm_session *s,
 		return s;
 	}
 
-	if (skb->len < sizeof(*hdr) + 1) {
+	if (skb->len < sizeof(*hdr) + 1 ||
+	    (!__test_ea(hdr->len) && skb->len < sizeof(*hdr) + 2)) {
 		kfree_skb(skb);
 		return s;
 	}
diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c
index e2486bc11cbce0..3fb8b547494822 100644
--- a/net/bluetooth/rfcomm/sock.c
+++ b/net/bluetooth/rfcomm/sock.c
@@ -48,8 +48,11 @@ static void rfcomm_sock_kill(struct sock *sk);
 static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb)
 {
 	struct sock *sk = d->owner;
-	if (!sk)
+
+	if (!sk) {
+		kfree_skb(skb);
 		return;
+	}
 
 	atomic_add(skb->len, &sk->sk_rmem_alloc);
 	skb_queue_tail(&sk->sk_receive_queue, skb);
@@ -721,10 +724,7 @@ static int rfcomm_sock_setsockopt(struct socket *sock, int level, int optname,
 		if (err)
 			break;
 
-		if (opt)
-			set_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
-		else
-			clear_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
+		assign_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags, opt);
 
 		break;
 
@@ -786,8 +786,10 @@ static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname,
 		break;
 
 	case RFCOMM_CONNINFO:
-		if (sk->sk_state != BT_CONNECTED &&
-					!rfcomm_pi(sk)->dlc->defer_setup) {
+		if ((sk->sk_state != BT_CONNECTED &&
+		     !(sk->sk_state == BT_CONNECT2 &&
+		       rfcomm_pi(sk)->dlc->defer_setup)) ||
+		    !rfcomm_pi(sk)->dlc->session) {
 			err = -ENOTCONN;
 			break;
 		}
diff --git a/net/bluetooth/rfcomm/tty.c b/net/bluetooth/rfcomm/tty.c
index b2c1060394e601..dc3cdf614defe2 100644
--- a/net/bluetooth/rfcomm/tty.c
+++ b/net/bluetooth/rfcomm/tty.c
@@ -462,7 +462,7 @@ static int __rfcomm_release_dev(void __user *arg)
 	/* Shut down TTY synchronously before freeing rfcomm_dev */
 	tty_port_tty_vhangup(&dev->port);
 
-	if (!test_bit(RFCOMM_TTY_OWNED, &dev->status))
+	if (!test_and_set_bit(RFCOMM_TTY_OWNED, &dev->status))
 		tty_port_put(&dev->port);
 
 	tty_port_put(&dev->port);
@@ -724,10 +724,9 @@ static int rfcomm_tty_install(struct tty_driver *driver, struct tty_struct *tty)
 	 * when the last process closes the tty. The behaviour is expected by
 	 * userspace.
 	 */
-	if (test_bit(RFCOMM_RELEASE_ONHUP, &dev->flags)) {
-		set_bit(RFCOMM_TTY_OWNED, &dev->status);
+	if (test_bit(RFCOMM_RELEASE_ONHUP, &dev->flags) &&
+	    !test_and_set_bit(RFCOMM_TTY_OWNED, &dev->status))
 		tty_port_put(&dev->port);
-	}
 
 	return 0;
 }
diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c
index 3d4362a09df4cd..8be48ca9f4c5b6 100644
--- a/net/bluetooth/sco.c
+++ b/net/bluetooth/sco.c
@@ -84,12 +84,14 @@ static void sco_conn_free(struct kref *ref)
 	if (conn->sk)
 		sco_pi(conn->sk)->conn = NULL;
 
-	if (conn->hcon) {
-		conn->hcon->sco_data = NULL;
-		hci_conn_drop(conn->hcon);
-	}
+	/* hcon->sco_data is cleared and the association's reference on the
+	 * sco_conn is dropped in sco_conn_del() under hdev->lock, and the
+	 * hci_conn is now owned by the socket (held in __sco_chan_add() and
+	 * dropped in sco_chan_del()/sco_sock_destruct()), so there is nothing
+	 * left to release towards hcon here.
+	 */
 
-	/* Ensure no more work items will run since hci_conn has been dropped */
+	/* Ensure no more work items will run before the connection is freed */
 	disable_delayed_work_sync(&conn->timeout_work);
 
 	kfree(conn);
@@ -188,25 +190,19 @@ static void sco_sock_clear_timer(struct sock *sk)
 }
 
 /* ---- SCO connections ---- */
-/* Consumes a reference on @hcon, which the returned sco_conn owns until it is
- * freed. On failure (NULL return) the reference is left for the caller to drop.
+/* Returns a new reference the caller must drop with sco_conn_put(). The
+ * hcon->sco_data association holds its own reference on the sco_conn for the
+ * connection's lifetime; it is dropped in sco_conn_del() under hdev->lock.
+ * @hcon is not consumed: the hci_conn reference is taken and owned by the
+ * socket in __sco_chan_add().
  */
 static struct sco_conn *sco_conn_add(struct hci_conn *hcon)
 {
 	struct sco_conn *conn = hcon->sco_data;
 
 	conn = sco_conn_hold_unless_zero(conn);
-	if (conn) {
-		if (!conn->hcon) {
-			sco_conn_lock(conn);
-			conn->hcon = hcon;
-			sco_conn_unlock(conn);
-		} else {
-			/* conn already owns a reference on hcon */
-			hci_conn_drop(hcon);
-		}
+	if (conn)
 		return conn;
-	}
 
 	conn = kzalloc_obj(struct sco_conn);
 	if (!conn)
@@ -227,7 +223,10 @@ static struct sco_conn *sco_conn_add(struct hci_conn *hcon)
 
 	BT_DBG("hcon %p conn %p", hcon, conn);
 
-	return conn;
+	/* kref_init() above set the association reference owned by
+	 * hcon->sco_data; hand the caller its own reference.
+	 */
+	return sco_conn_hold(conn);
 }
 
 /* Delete channel.
@@ -242,6 +241,19 @@ static void sco_chan_del(struct sock *sk, int err)
 	BT_DBG("sk %p, conn %p, err %d", sk, conn, err);
 
 	if (conn) {
+		struct hci_conn *hcon;
+
+		sco_conn_lock(conn);
+		hcon = conn->hcon;
+		sco_conn_unlock(conn);
+
+		/* Drop the socket's hci_conn reference BEFORE clearing
+		 * conn->sk, so sco_conn_del() on another CPU cannot free
+		 * the hci_conn while we still hold a pointer to it.
+		 */
+		if (hcon)
+			hci_conn_drop(hcon);
+
 		sco_conn_lock(conn);
 		conn->sk = NULL;
 		sco_conn_unlock(conn);
@@ -266,6 +278,13 @@ static void sco_conn_del(struct hci_conn *hcon, int err)
 
 	BT_DBG("hcon %p conn %p, err %d", hcon, conn, err);
 
+	/* Detach from the hci_conn and drop the association's reference.
+	 * The caller holds hdev->lock, which serialises this against the
+	 * read of hcon->sco_data in sco_recv_scodata().
+	 */
+	hcon->sco_data = NULL;
+	sco_conn_put(conn);
+
 	sco_conn_lock(conn);
 	sk = sco_sock_hold(conn);
 	sco_conn_unlock(conn);
@@ -290,6 +309,11 @@ static void __sco_chan_add(struct sco_conn *conn, struct sock *sk,
 	sco_pi(sk)->conn = sco_conn_hold(conn);
 	conn->sk = sk;
 
+	/* The socket owns an hci_conn reference for as long as it stays
+	 * attached; it is dropped in sco_chan_del()/sco_sock_destruct().
+	 */
+	hci_conn_hold(conn->hcon);
+
 	if (parent)
 		bt_accept_enqueue(parent, sk, true);
 }
@@ -371,6 +395,7 @@ static int sco_connect(struct sock *sk)
 	if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) {
 		release_sock(sk);
 		sco_conn_put(conn);
+		hci_conn_drop(hcon);
 		err = -EBADFD;
 		goto unlock;
 	}
@@ -379,9 +404,13 @@ static int sco_connect(struct sock *sk)
 	sco_conn_put(conn);
 	if (err) {
 		release_sock(sk);
+		hci_conn_drop(hcon);
 		goto unlock;
 	}
 
+	/* __sco_chan_add() took its own hci_conn reference; drop ours. */
+	hci_conn_drop(hcon);
+
 	/* Update source addr of the socket */
 	bacpy(&sco_pi(sk)->src, &hcon->src);
 
@@ -495,9 +524,25 @@ static struct sock *sco_get_sock_listen(bdaddr_t *src)
 
 static void sco_sock_destruct(struct sock *sk)
 {
+	struct sco_conn *conn = sco_pi(sk)->conn;
+
 	BT_DBG("sk %p", sk);
 
-	sco_conn_put(sco_pi(sk)->conn);
+	/* If the channel was not already torn down via sco_chan_del(), drop
+	 * the socket's own references here.
+	 */
+	if (conn) {
+		struct hci_conn *hcon;
+
+		sco_conn_lock(conn);
+		hcon = conn->hcon;
+		sco_conn_unlock(conn);
+
+		if (hcon)
+			hci_conn_drop(hcon);
+		sco_pi(sk)->conn = NULL;
+		sco_conn_put(conn);
+	}
 
 	skb_queue_purge(&sk->sk_receive_queue);
 	skb_queue_purge(&sk->sk_write_queue);
@@ -992,10 +1037,7 @@ static int sco_sock_setsockopt(struct socket *sock, int level, int optname,
 		if (err)
 			break;
 
-		if (opt)
-			set_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
-		else
-			clear_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
+		assign_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags, opt);
 		break;
 
 	case BT_VOICE:
@@ -1035,10 +1077,7 @@ static int sco_sock_setsockopt(struct socket *sock, int level, int optname,
 		if (err)
 			break;
 
-		if (opt)
-			set_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags);
-		else
-			clear_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags);
+		assign_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags, opt);
 		break;
 
 	case BT_CODEC:
@@ -1511,12 +1550,10 @@ static void sco_connect_cfm(struct hci_conn *hcon, __u8 status)
 	if (!status) {
 		struct sco_conn *conn;
 
-		conn = sco_conn_add(hci_conn_hold(hcon));
+		conn = sco_conn_add(hcon);
 		if (conn) {
 			sco_conn_ready(conn);
 			sco_conn_put(conn);
-		} else {
-			hci_conn_drop(hcon);
 		}
 	} else
 		sco_conn_del(hcon, bt_to_errno(status));
diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c
index 6091c47cb00284..5c3bd42783860d 100644
--- a/net/bluetooth/smp.c
+++ b/net/bluetooth/smp.c
@@ -366,7 +366,7 @@ static int smp_e(const u8 *k, u8 *r)
 	err = aes_prepareenckey(&aes, tmp, 16);
 	if (err) {
 		BT_ERR("cipher setkey failed: %d", err);
-		return err;
+		goto out_clear;
 	}
 
 	/* Most significant octet of plaintextData corresponds to data[0] */
@@ -379,6 +379,8 @@ static int smp_e(const u8 *k, u8 *r)
 
 	SMP_DBG("r %16phN", r);
 
+out_clear:
+	memzero_explicit(tmp, sizeof(tmp));
 	memzero_explicit(&aes, sizeof(aes));
 	return err;
 }
@@ -662,6 +664,7 @@ static void build_pairing_cmd(struct l2cap_conn *conn,
 		else
 			bdaddr_type = BDADDR_LE_RANDOM;
 
+		mutex_lock(&hdev->remote_oob_lock);
 		oob_data = hci_find_remote_oob_data(hdev, &hcon->dst,
 						    bdaddr_type);
 		if (oob_data && oob_data->present) {
@@ -672,6 +675,7 @@ static void build_pairing_cmd(struct l2cap_conn *conn,
 			SMP_DBG("OOB Remote Confirmation: %16phN", smp->pcnf);
 			SMP_DBG("OOB Remote Random: %16phN", smp->rr);
 		}
+		mutex_unlock(&hdev->remote_oob_lock);
 
 	} else {
 		authreq &= ~SMP_AUTH_SC;
@@ -2269,6 +2273,23 @@ static u8 smp_cmd_security_req(struct l2cap_conn *conn, struct sk_buff *skb)
 
 	bt_dev_dbg(hdev, "conn %p", conn);
 
+	/* SMP over BR/EDR only covers cross-transport key derivation; the
+	 * Security Request procedure has no BR/EDR counterpart. Reject it
+	 * here, otherwise smp_ltk_encrypt() finds the peer's LE LTK
+	 * (ADDR_LE_DEV_PUBLIC and BDADDR_BREDR are both 0) and issues
+	 * HCI_OP_LE_START_ENC on the ACL handle, which the controller
+	 * rejects and hci_cs_le_start_enc() turns into a disconnect. Reply
+	 * without smp_failure(): this is not an authentication failure, and
+	 * MGMT_EV_AUTH_FAILED would make bluetoothd drop the device.
+	 */
+	if (hcon->type != LE_LINK) {
+		u8 reason = SMP_CMD_NOTSUPP;
+
+		smp_send_cmd(conn, SMP_CMD_PAIRING_FAIL, sizeof(reason),
+			     &reason);
+		return 0;
+	}
+
 	if (skb->len < sizeof(*rp))
 		return SMP_INVALID_PARAMS;
 
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 5d51f6cb7d1545..513354e928cb58 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -205,8 +205,8 @@ static void xdp_test_run_teardown(struct xdp_test_data *xdp)
 {
 	xdp_unreg_mem_model(&xdp->mem);
 	page_pool_destroy(xdp->pp);
-	kfree(xdp->frames);
-	kfree(xdp->skbs);
+	kvfree(xdp->frames);
+	kvfree(xdp->skbs);
 }
 
 static bool frame_was_changed(const struct xdp_page_head *head)
diff --git a/net/bridge/br.c b/net/bridge/br.c
index a5e5b2db110ec7..09a120b3b76cb1 100644
--- a/net/bridge/br.c
+++ b/net/bridge/br.c
@@ -396,10 +396,7 @@ void br_opt_toggle(struct net_bridge *br, enum net_bridge_opts opt, bool on)
 	if (cur == on)
 		return;
 
-	if (on)
-		set_bit(opt, &br->options);
-	else
-		clear_bit(opt, &br->options);
+	assign_bit(opt, &br->options, on);
 }
 
 static void __net_exit br_net_exit_rtnl(struct net *net,
diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c
index 87a40e2f1c502d..da15f4d7c1aef5 100644
--- a/net/bridge/br_arp_nd_proxy.c
+++ b/net/bridge/br_arp_nd_proxy.c
@@ -40,6 +40,23 @@ void br_recalculate_neigh_suppress_enabled(struct net_bridge *br)
 }
 
 #if IS_ENABLED(CONFIG_INET)
+static bool
+br_is_neigh_suppress_enabled_vid(const struct net_bridge_port *p, u16 vid)
+{
+	const struct net_bridge_vlan *v = NULL;
+
+	if (p && vid && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) {
+		struct net_bridge_vlan_group *vg;
+
+		vg = nbp_vlan_group_rcu(p);
+		v = br_vlan_find(vg, vid);
+		if (!v)
+			return false;
+	}
+
+	return br_is_neigh_suppress_enabled(p, v);
+}
+
 static void br_arp_send(struct net_bridge *br, struct net_bridge_port *p,
 			struct net_device *dev, __be32 dest_ip, __be32 src_ip,
 			const unsigned char *dest_hw,
@@ -159,7 +176,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br,
 		return;
 
 	if (br_opt_get(br, BROPT_NEIGH_SUPPRESS_ENABLED)) {
-		if (br_is_neigh_suppress_enabled(p, vid))
+		if (br_is_neigh_suppress_enabled_vid(p, vid))
 			return;
 		if (is_unicast_ether_addr(eth_hdr(skb)->h_dest) &&
 		    parp->ar_op == htons(ARPOP_REQUEST))
@@ -193,7 +210,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br,
 		return;
 	}
 
-	n = neigh_lookup(&arp_tbl, &tip, vlandev);
+	n = neigh_lookup(arp_table(dev_net(vlandev)), &tip, vlandev);
 	if (n) {
 		struct net_bridge_fdb_entry *f;
 		u8 ha[ETH_ALEN] __aligned(2);
@@ -211,7 +228,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br,
 
 			if ((p && test_bit(BR_PROXYARP_BIT, &p->flags)) ||
 			    (dst && test_bit(BR_PROXYARP_WIFI_BIT, &dst->flags)) ||
-			    br_is_neigh_suppress_enabled(dst, vid)) {
+			    br_is_neigh_suppress_enabled_vid(dst, vid)) {
 				if (!vid)
 					br_arp_send(br, p, skb->dev, sip, tip,
 						    sha, ha, sha, 0, 0);
@@ -424,7 +441,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 	BR_INPUT_SKB_CB(skb)->proxyarp_replied = 0;
 	BR_INPUT_SKB_CB(skb)->grat_arp = 0;
 
-	if (br_is_neigh_suppress_enabled(p, vid))
+	if (br_is_neigh_suppress_enabled_vid(p, vid))
 		return;
 
 	if (is_unicast_ether_addr(eth_hdr(skb)->h_dest) &&
@@ -470,7 +487,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 		return;
 	}
 
-	n = neigh_lookup(&nd_tbl, &msg->target, vlandev);
+	n = neigh_lookup(nd_table(dev_net(vlandev)), &msg->target, vlandev);
 	if (n) {
 		struct net_bridge_fdb_entry *f;
 		u8 ha[ETH_ALEN] __aligned(2);
@@ -486,7 +503,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 			const struct net_bridge_port *dst = READ_ONCE(f->dst);
 			bool replied = false;
 
-			if (br_is_neigh_suppress_enabled(dst, vid)) {
+			if (br_is_neigh_suppress_enabled_vid(dst, vid)) {
 				if (vid != 0)
 					br_nd_send(br, p, skb, n, ha,
 						   skb->vlan_proto,
@@ -509,33 +526,25 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 }
 #endif
 
-bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, u16 vid)
+bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p,
+				  const struct net_bridge_vlan *v)
 {
 	if (!p)
 		return false;
 
-	if (vid && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) {
-		struct net_bridge_vlan_group *vg = nbp_vlan_group_rcu(p);
-		struct net_bridge_vlan *v;
+	if (v && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags))
+		return !!(READ_ONCE(v->priv_flags) &
+			  BR_VLFLAG_NEIGH_SUPPRESS_ENABLED);
 
-		v = br_vlan_find(vg, vid);
-		if (!v)
-			return false;
-		return !!(v->priv_flags & BR_VLFLAG_NEIGH_SUPPRESS_ENABLED);
-	}
 	return test_bit(BR_NEIGH_SUPPRESS_BIT, &p->flags);
 }
 
-bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, u16 vid)
+bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p,
+				      const struct net_bridge_vlan *v)
 {
-	if (vid && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags)) {
-		struct net_bridge_vlan_group *vg = nbp_vlan_group_rcu(p);
-		struct net_bridge_vlan *v;
+	if (v && test_bit(BR_NEIGH_VLAN_SUPPRESS_BIT, &p->flags))
+		return !!(READ_ONCE(v->priv_flags) &
+			  BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED);
 
-		v = br_vlan_find(vg, vid);
-		if (!v)
-			return false;
-		return !!(v->priv_flags & BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED);
-	}
 	return test_bit(BR_NEIGH_FORWARD_GRAT_BIT, &p->flags);
 }
diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c
index e01c44a90d84b5..ce9ea9ac3d0ade 100644
--- a/net/bridge/br_device.c
+++ b/net/bridge/br_device.c
@@ -89,10 +89,10 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	dest = eth_hdr(skb)->h_dest;
 	if (is_broadcast_ether_addr(dest)) {
-		br_flood(br, skb, BR_PKT_BROADCAST, false, true, vid);
+		br_flood(br, vlan, skb, BR_PKT_BROADCAST, false, true);
 	} else if (is_multicast_ether_addr(dest)) {
 		if (unlikely(netpoll_tx_running(dev))) {
-			br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid);
+			br_flood(br, vlan, skb, BR_PKT_MULTICAST, false, true);
 			goto out;
 		}
 		if (br_multicast_rcv(&brmctx, &pmctx_null, vlan, skb, vid)) {
@@ -105,11 +105,11 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev)
 		    br_multicast_querier_exists(brmctx, eth_hdr(skb), mdst))
 			br_multicast_flood(mdst, skb, brmctx, false, true);
 		else
-			br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid);
+			br_flood(br, vlan, skb, BR_PKT_MULTICAST, false, true);
 	} else if ((dst = br_fdb_find_rcu(br, dest, vid)) != NULL) {
 		br_forward(READ_ONCE(dst->dst), skb, false, true);
 	} else {
-		br_flood(br, skb, BR_PKT_UNICAST, false, true, vid);
+		br_flood(br, vlan, skb, BR_PKT_UNICAST, false, true);
 	}
 out:
 	rcu_read_unlock();
diff --git a/net/bridge/br_forward.c b/net/bridge/br_forward.c
index 46c762ca5177e5..b5eece1ff9e8c8 100644
--- a/net/bridge/br_forward.c
+++ b/net/bridge/br_forward.c
@@ -7,7 +7,6 @@
  *	Lennert Buytenhek		<buytenh@gnu.org>
  */
 
-#include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
@@ -17,16 +16,31 @@
 #include <linux/netfilter_bridge.h>
 #include "br_private.h"
 
+struct br_fwd_dst {
+	const struct net_bridge_port *port;
+	struct net_bridge_vlan *vlan;
+};
+
+static bool should_deliver_vlan(const struct br_fwd_dst *fwd,
+				const struct sk_buff *skb)
+{
+	if (fwd->vlan)
+		return br_vlan_state_allowed(br_vlan_get_state(fwd->vlan),
+					     false);
+
+	return br_allowed_egress(nbp_vlan_group_rcu(fwd->port), skb);
+}
+
 /* Don't forward packets to originating port or forwarding disabled */
-static inline int should_deliver(const struct net_bridge_port *p,
-				 const struct sk_buff *skb)
+static __always_inline bool should_deliver(const struct br_fwd_dst *fwd,
+					   const struct sk_buff *skb)
 {
-	struct net_bridge_vlan_group *vg;
+	const struct net_bridge_port *p = fwd->port;
 
-	vg = nbp_vlan_group_rcu(p);
 	return (test_bit(BR_HAIRPIN_MODE_BIT, &p->flags) || skb->dev != p->dev) &&
 		(br_mst_is_enabled(p) || p->state == BR_STATE_FORWARDING) &&
-		br_allowed_egress(vg, skb) && nbp_switchdev_allowed_egress(p, skb) &&
+		should_deliver_vlan(fwd, skb) &&
+		nbp_switchdev_allowed_egress(p, skb) &&
 		!br_skb_isolated(p, skb);
 }
 
@@ -70,9 +84,10 @@ int br_forward_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
 }
 EXPORT_SYMBOL_GPL(br_forward_finish);
 
-static void __br_forward(const struct net_bridge_port *to,
+static void __br_forward(const struct br_fwd_dst *fwd,
 			 struct sk_buff *skb, bool local_orig)
 {
+	const struct net_bridge_port *to = fwd->port;
 	struct net_bridge_vlan_group *vg;
 	struct net_device *indev;
 	struct net *net;
@@ -83,8 +98,8 @@ static void __br_forward(const struct net_bridge_port *to,
 	 */
 	nbp_switchdev_frame_mark_tx_fwd_offload(to, skb);
 
-	vg = nbp_vlan_group_rcu(to);
-	skb = br_handle_vlan(to->br, to, vg, skb);
+	vg = fwd->vlan ? NULL : nbp_vlan_group_rcu(to);
+	skb = br_handle_vlan(to->br, to, vg, fwd->vlan, skb);
 	if (!skb)
 		return;
 
@@ -117,7 +132,7 @@ static void __br_forward(const struct net_bridge_port *to,
 		br_forward_finish);
 }
 
-static int deliver_clone(const struct net_bridge_port *prev,
+static int deliver_clone(const struct br_fwd_dst *fwd,
 			 struct sk_buff *skb, bool local_orig)
 {
 	struct net_device *dev = BR_INPUT_SKB_CB(skb)->brdev;
@@ -128,7 +143,7 @@ static int deliver_clone(const struct net_bridge_port *prev,
 		return -ENOMEM;
 	}
 
-	__br_forward(prev, skb, local_orig);
+	__br_forward(fwd, skb, local_orig);
 	return 0;
 }
 
@@ -144,6 +159,8 @@ static int deliver_clone(const struct net_bridge_port *prev,
 void br_forward(const struct net_bridge_port *to,
 		struct sk_buff *skb, bool local_rcv, bool local_orig)
 {
+	struct br_fwd_dst fwd;
+
 	if (unlikely(!to))
 		goto out;
 
@@ -159,11 +176,13 @@ void br_forward(const struct net_bridge_port *to,
 		to = backup_port;
 	}
 
-	if (should_deliver(to, skb)) {
+	fwd.port = to;
+	fwd.vlan = NULL;
+	if (should_deliver(&fwd, skb)) {
 		if (local_rcv)
-			deliver_clone(to, skb, local_orig);
+			deliver_clone(&fwd, skb, local_orig);
 		else
-			__br_forward(to, skb, local_orig);
+			__br_forward(&fwd, skb, local_orig);
 		return;
 	}
 
@@ -173,95 +192,162 @@ void br_forward(const struct net_bridge_port *to,
 }
 EXPORT_SYMBOL_GPL(br_forward);
 
-static struct net_bridge_port *maybe_deliver(
-	struct net_bridge_port *prev, struct net_bridge_port *p,
-	struct sk_buff *skb, bool local_orig)
+static int maybe_deliver(struct br_fwd_dst *prev, const struct br_fwd_dst *fwd,
+			 struct sk_buff *skb, bool local_orig)
 {
+	const struct net_bridge_port *p = fwd->port;
 	u8 igmp_type = br_multicast_igmp_type(skb);
 	int err;
 
-	if (!should_deliver(p, skb))
-		return prev;
+	if (!should_deliver(fwd, skb))
+		return 0;
 
 	nbp_switchdev_frame_mark_tx_fwd_to_hwdom(p, skb);
 
-	if (!prev)
+	if (!prev->port)
 		goto out;
 
 	err = deliver_clone(prev, skb, local_orig);
 	if (err)
-		return ERR_PTR(err);
+		return err;
 out:
 	br_multicast_count(p->br, p, skb, igmp_type, BR_MCAST_DIR_TX);
+	*prev = *fwd;
 
-	return p;
+	return 0;
 }
 
-/* called under rcu_read_lock */
-void br_flood(struct net_bridge *br, struct sk_buff *skb,
-	      enum br_pkt_type pkt_type, bool local_rcv, bool local_orig,
-	      u16 vid)
+static void br_flood_finish(const struct br_fwd_dst *fwd, int err,
+			    struct sk_buff *skb,
+			    bool local_rcv, bool local_orig)
 {
 	enum skb_drop_reason reason = SKB_DROP_REASON_NO_TX_TARGET;
-	struct net_bridge_port *prev = NULL;
-	struct net_bridge_port *p;
 
-	br_tc_skb_miss_set(skb, pkt_type != BR_PKT_BROADCAST);
+	if (err || !fwd->port) {
+		if (err)
+			reason = err == -ENOMEM ? SKB_DROP_REASON_NOMEM :
+						  SKB_DROP_REASON_NOT_SPECIFIED;
+
+		if (!local_rcv)
+			kfree_skb_reason(skb, reason);
+		return;
+	}
 
-	list_for_each_entry_rcu(p, &br->port_list, list) {
-		/* Do not flood unicast traffic to ports that turn it off, nor
-		 * other traffic if flood off, except for traffic we originate
+	if (local_rcv)
+		deliver_clone(fwd, skb, local_orig);
+	else
+		__br_forward(fwd, skb, local_orig);
+}
+
+static int br_flood_port(struct br_fwd_dst *prev,
+			 const struct br_fwd_dst *fwd, struct sk_buff *skb,
+			 enum br_pkt_type pkt_type, bool local_orig)
+{
+	const struct net_bridge_port *p = fwd->port;
+
+	/* Do not flood unicast traffic to ports that turn it off, nor
+	 * other traffic if flood off, except for traffic we originate
+	 */
+	switch (pkt_type) {
+	case BR_PKT_UNICAST:
+		if (!test_bit(BR_FLOOD_BIT, &p->flags))
+			return 0;
+		break;
+	case BR_PKT_MULTICAST:
+		if (!test_bit(BR_MCAST_FLOOD_BIT, &p->flags) &&
+		    skb->dev != p->br->dev)
+			return 0;
+		break;
+	case BR_PKT_BROADCAST:
+		if (!test_bit(BR_BCAST_FLOOD_BIT, &p->flags) &&
+		    skb->dev != p->br->dev)
+			return 0;
+		break;
+	}
+
+	/* Do not flood to ports that enable proxy ARP */
+	if (test_bit(BR_PROXYARP_BIT, &p->flags))
+		return 0;
+	if (BR_INPUT_SKB_CB(skb)->proxyarp_replied) {
+		if (test_bit(BR_PROXYARP_WIFI_BIT, &p->flags))
+			return 0;
+		/* For gratuitous ARPs/NAs, check neigh_forward_grat.
+		 * For regular ARPs/NDs, check only neigh_suppress.
 		 */
-		switch (pkt_type) {
-		case BR_PKT_UNICAST:
-			if (!test_bit(BR_FLOOD_BIT, &p->flags))
-				continue;
-			break;
-		case BR_PKT_MULTICAST:
-			if (!test_bit(BR_MCAST_FLOOD_BIT, &p->flags) && skb->dev != br->dev)
-				continue;
-			break;
-		case BR_PKT_BROADCAST:
-			if (!test_bit(BR_BCAST_FLOOD_BIT, &p->flags) && skb->dev != br->dev)
-				continue;
-			break;
-		}
+		if (br_is_neigh_suppress_enabled(p, fwd->vlan) &&
+		    (!BR_INPUT_SKB_CB(skb)->grat_arp ||
+		     !br_is_neigh_forward_grat_enabled(p, fwd->vlan)))
+			return 0;
+	}
 
-		/* Do not flood to ports that enable proxy ARP */
-		if (test_bit(BR_PROXYARP_BIT, &p->flags))
-			continue;
-		if (BR_INPUT_SKB_CB(skb)->proxyarp_replied) {
-			if (test_bit(BR_PROXYARP_WIFI_BIT, &p->flags))
-				continue;
-			/* For gratuitous ARPs/NAs, check neigh_forward_grat.
-			 * For regular ARPs/NDs, check only neigh_suppress.
-			 */
-			if (br_is_neigh_suppress_enabled(p, vid) &&
-			    (!BR_INPUT_SKB_CB(skb)->grat_arp ||
-			     !br_is_neigh_forward_grat_enabled(p, vid)))
-				continue;
-		}
+	return maybe_deliver(prev, fwd, skb, local_orig);
+}
 
-		prev = maybe_deliver(prev, p, skb, local_orig);
-		if (IS_ERR(prev)) {
-			reason = PTR_ERR(prev) == -ENOMEM ? SKB_DROP_REASON_NOMEM :
-				 SKB_DROP_REASON_NOT_SPECIFIED;
-			goto out;
+static int br_flood_vlan(struct br_fwd_dst *prev,
+			 struct net_bridge_vlan *v, struct sk_buff *skb,
+			 enum br_pkt_type pkt_type, bool local_orig)
+{
+	struct net_bridge_vlan_port_array *array;
+	struct net_bridge_vlan *masterv, *pv;
+	struct br_fwd_dst dst;
+	int err;
+
+	masterv = br_vlan_is_master(v) ? v : v->brvlan;
+	array = rcu_dereference(masterv->port_array);
+	if (array) {
+		unsigned int i;
+
+		for (i = 0; i < array->count; i++) {
+			pv = array->vlans[i];
+			dst.port = pv->port;
+			dst.vlan = pv;
+			err = br_flood_port(prev, &dst, skb, pkt_type,
+					    local_orig);
+			if (err)
+				return err;
+		}
+	} else {
+		list_for_each_entry_rcu(pv, &masterv->port_vlist, port_vlist) {
+			dst.port = pv->port;
+			dst.vlan = pv;
+			err = br_flood_port(prev, &dst, skb, pkt_type,
+					    local_orig);
+			if (err)
+				return err;
 		}
 	}
 
-	if (!prev)
-		goto out;
+	return 0;
+}
 
-	if (local_rcv)
-		deliver_clone(prev, skb, local_orig);
-	else
-		__br_forward(prev, skb, local_orig);
-	return;
+/* called under rcu_read_lock */
+void br_flood(struct net_bridge *br, struct net_bridge_vlan *v,
+	      struct sk_buff *skb, enum br_pkt_type pkt_type,
+	      bool local_rcv, bool local_orig)
+{
+	struct br_fwd_dst prev = {};
+	int err = 0;
 
-out:
-	if (!local_rcv)
-		kfree_skb_reason(skb, reason);
+	br_tc_skb_miss_set(skb, pkt_type != BR_PKT_BROADCAST);
+
+	if (v) {
+		err = br_flood_vlan(&prev, v, skb, pkt_type, local_orig);
+	} else {
+		struct net_bridge_port *p;
+
+		list_for_each_entry_rcu(p, &br->port_list, list) {
+			struct br_fwd_dst fwd = {
+				.port = p,
+			};
+
+			err = br_flood_port(&prev, &fwd, skb, pkt_type,
+					    local_orig);
+			if (err)
+				break;
+		}
+	}
+
+	br_flood_finish(&prev, err, skb, local_rcv, local_orig);
 }
 
 #ifdef CONFIG_BRIDGE_IGMP_SNOOPING
@@ -270,9 +356,12 @@ static void maybe_deliver_addr(struct net_bridge_port *p, struct sk_buff *skb,
 {
 	struct net_device *dev = BR_INPUT_SKB_CB(skb)->brdev;
 	const unsigned char *src = eth_hdr(skb)->h_source;
+	struct br_fwd_dst fwd = {
+		.port = p,
+	};
 	struct sk_buff *nskb;
 
-	if (!should_deliver(p, skb))
+	if (!should_deliver(&fwd, skb))
 		return;
 
 	/* Even with hairpin, no soliloquies - prevent breaking IPv6 DAD */
@@ -292,7 +381,7 @@ static void maybe_deliver_addr(struct net_bridge_port *p, struct sk_buff *skb,
 	if (!is_broadcast_ether_addr(addr))
 		memcpy(eth_hdr(skb)->h_dest, addr, ETH_ALEN);
 
-	__br_forward(p, skb, local_orig);
+	__br_forward(&fwd, skb, local_orig);
 }
 
 /* called with rcu_read_lock */
@@ -301,11 +390,11 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst,
 			struct net_bridge_mcast *brmctx,
 			bool local_rcv, bool local_orig)
 {
-	enum skb_drop_reason reason = SKB_DROP_REASON_NO_TX_TARGET;
-	struct net_bridge_port *prev = NULL;
 	struct net_bridge_port_group *p;
 	bool allow_mode_include = true;
+	struct br_fwd_dst prev = {};
 	struct hlist_node *rp;
+	int err = 0;
 
 	rp = br_multicast_get_first_rport_node(brmctx, skb);
 
@@ -321,6 +410,7 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst,
 
 	while (p || rp) {
 		struct net_bridge_port *port, *lport, *rport;
+		struct br_fwd_dst fwd = {};
 
 		lport = p ? p->key.port : NULL;
 		rport = br_multicast_rport_from_node_skb(rp, skb);
@@ -342,12 +432,11 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst,
 			port = rport;
 		}
 
-		prev = maybe_deliver(prev, port, skb, local_orig);
-		if (IS_ERR(prev)) {
-			reason = PTR_ERR(prev) == -ENOMEM ? SKB_DROP_REASON_NOMEM :
-				 SKB_DROP_REASON_NOT_SPECIFIED;
-			goto out;
-		}
+		fwd.port = port;
+		err = maybe_deliver(&prev, &fwd, skb, local_orig);
+		if (err)
+			break;
+
 delivered:
 		if ((unsigned long)lport >= (unsigned long)port)
 			p = rcu_dereference(p->next);
@@ -355,17 +444,6 @@ void br_multicast_flood(struct net_bridge_mdb_entry *mdst,
 			rp = rcu_dereference(hlist_next_rcu(rp));
 	}
 
-	if (!prev)
-		goto out;
-
-	if (local_rcv)
-		deliver_clone(prev, skb, local_orig);
-	else
-		__br_forward(prev, skb, local_orig);
-	return;
-
-out:
-	if (!local_rcv)
-		kfree_skb_reason(skb, reason);
+	br_flood_finish(&prev, err, skb, local_rcv, local_orig);
 }
 #endif
diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c
index 8bed72baf16150..4357d78524a687 100644
--- a/net/bridge/br_input.c
+++ b/net/bridge/br_input.c
@@ -58,7 +58,7 @@ static int br_pass_frame_up(struct sk_buff *skb, bool promisc)
 
 	indev = skb->dev;
 	skb->dev = brdev;
-	skb = br_handle_vlan(br, NULL, vg, skb);
+	skb = br_handle_vlan(br, NULL, vg, NULL, skb);
 	if (!skb)
 		return NET_RX_DROP;
 	/* update the multicast stats if the packet is IGMP/MLD */
@@ -226,7 +226,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb
 		br_forward(READ_ONCE(dst->dst), skb, local_rcv, false);
 	} else {
 		if (!mcast_hit)
-			br_flood(br, skb, pkt_type, local_rcv, false, vid);
+			br_flood(br, vlan, skb, pkt_type, local_rcv, false);
 		else
 			br_multicast_flood(mdst, skb, brmctx, local_rcv, false);
 	}
diff --git a/net/bridge/br_mdb.c b/net/bridge/br_mdb.c
index e0c7020b12f5fc..a01bd280c722b6 100644
--- a/net/bridge/br_mdb.c
+++ b/net/bridge/br_mdb.c
@@ -1523,6 +1523,8 @@ static void br_mdb_flush_pgs(struct net_bridge *br,
 		}
 
 		br_multicast_del_pg(mp, p, pp);
+		/* br_multicast_del_pg() can remove other groups from this list. */
+		pp = &mp->ports;
 	}
 }
 
diff --git a/net/bridge/br_mst.c b/net/bridge/br_mst.c
index 1654efd3045b0a..252d5c927f7167 100644
--- a/net/bridge/br_mst.c
+++ b/net/bridge/br_mst.c
@@ -70,7 +70,7 @@ int br_mst_get_state(const struct net_device *dev, u16 msti, u8 *state)
 
 	list_for_each_entry(v, &vg->vlan_list, vlist) {
 		if (v->brvlan->msti == msti) {
-			*state = v->state;
+			*state = br_vlan_get_state(v);
 			return 0;
 		}
 	}
@@ -79,16 +79,11 @@ int br_mst_get_state(const struct net_device *dev, u16 msti, u8 *state)
 }
 EXPORT_SYMBOL_GPL(br_mst_get_state);
 
-static void br_mst_vlan_set_state(struct net_bridge_vlan_group *vg,
-				  struct net_bridge_vlan *v,
-				  u8 state)
+static void br_mst_vlan_set_state(struct net_bridge_vlan *v, u8 state)
 {
 	if (br_vlan_get_state(v) == state)
 		return;
 
-	if (v->vid == vg->pvid)
-		br_vlan_set_pvid_state(vg, state);
-
 	br_vlan_set_state(v, state);
 }
 
@@ -126,10 +121,10 @@ int br_mst_set_state(struct net_bridge_port *p, u16 msti, u8 state,
 		goto out_rcu_unlock;
 
 	list_for_each_entry_rcu(v, &vg->vlan_list, vlist) {
-		if (v->brvlan->msti != msti)
+		if (READ_ONCE(v->brvlan->msti) != msti)
 			continue;
 
-		br_mst_vlan_set_state(vg, v, state);
+		br_mst_vlan_set_state(v, state);
 	}
 
 out_rcu_unlock:
@@ -149,13 +144,13 @@ static void br_mst_vlan_sync_state(struct net_bridge_vlan *pv, u16 msti)
 		 * it.
 		 */
 		if (v != pv && v->brvlan->msti == msti) {
-			br_mst_vlan_set_state(vg, pv, v->state);
+			br_mst_vlan_set_state(pv, br_vlan_get_state(v));
 			return;
 		}
 	}
 
 	/* Otherwise, start out in a new MSTI with all ports disabled. */
-	return br_mst_vlan_set_state(vg, pv, BR_STATE_DISABLED);
+	return br_mst_vlan_set_state(pv, BR_STATE_DISABLED);
 }
 
 int br_mst_vlan_set_msti(struct net_bridge_vlan *mv, u16 msti)
@@ -180,7 +175,7 @@ int br_mst_vlan_set_msti(struct net_bridge_vlan *mv, u16 msti)
 	if (err && err != -EOPNOTSUPP)
 		return err;
 
-	mv->msti = msti;
+	WRITE_ONCE(mv->msti, msti);
 
 	list_for_each_entry(p, &mv->br->port_list, list) {
 		vg = nbp_vlan_group(p);
@@ -253,7 +248,9 @@ size_t br_mst_info_size(const struct net_bridge_vlan_group *vg)
 	sz = nla_total_size(0);
 
 	list_for_each_entry_rcu(v, &vg->vlan_list, vlist) {
-		if (test_bit(v->brvlan->msti, seen))
+		u16 msti = READ_ONCE(v->brvlan->msti);
+
+		if (test_bit(msti, seen))
 			continue;
 
 		/* IFLA_BRIDGE_MST_ENTRY */
@@ -263,7 +260,7 @@ size_t br_mst_info_size(const struct net_bridge_vlan_group *vg)
 			/* IFLA_BRIDGE_MST_ENTRY_STATE */
 			nla_total_size(sizeof(u8));
 
-		__set_bit(v->brvlan->msti, seen);
+		__set_bit(msti, seen);
 	}
 
 	return sz;
@@ -284,7 +281,8 @@ int br_mst_fill_info(struct sk_buff *skb,
 		nest = nla_nest_start_noflag(skb, IFLA_BRIDGE_MST_ENTRY);
 		if (!nest ||
 		    nla_put_u16(skb, IFLA_BRIDGE_MST_ENTRY_MSTI, v->brvlan->msti) ||
-		    nla_put_u8(skb, IFLA_BRIDGE_MST_ENTRY_STATE, v->state)) {
+		    nla_put_u8(skb, IFLA_BRIDGE_MST_ENTRY_STATE,
+			       br_vlan_get_state(v))) {
 			err = -EMSGSIZE;
 			break;
 		}
diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c
index 2f9bb30e1a1f69..ca7273c5a7f0d4 100644
--- a/net/bridge/br_multicast.c
+++ b/net/bridge/br_multicast.c
@@ -923,7 +923,7 @@ static void __br_multicast_query_handle_vlan(struct net_bridge_mcast *brmctx,
 	else if (br_multicast_ctx_is_vlan(brmctx))
 		vlan = brmctx->vlan;
 
-	if (vlan && !(vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED)) {
+	if (vlan && !(READ_ONCE(vlan->flags) & BRIDGE_VLAN_INFO_UNTAGGED)) {
 		u16 vlan_proto;
 
 		if (br_vlan_get_proto(brmctx->br->dev, &vlan_proto) != 0)
@@ -2184,7 +2184,7 @@ static void br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx)
 
 	spin_lock_bh(&br->multicast_lock);
 	if (br_multicast_port_ctx_is_vlan(pmctx) &&
-	    !(pmctx->vlan->priv_flags & BR_VLFLAG_MCAST_ENABLED)) {
+	    !(READ_ONCE(pmctx->vlan->priv_flags) & BR_VLFLAG_MCAST_ENABLED)) {
 		spin_unlock_bh(&br->multicast_lock);
 		return;
 	}
@@ -2221,7 +2221,7 @@ static void br_multicast_disable_port_ctx(struct net_bridge_mcast_port *pmctx)
 
 	spin_lock_bh(&br->multicast_lock);
 	if (br_multicast_port_ctx_is_vlan(pmctx) &&
-	    !(pmctx->vlan->priv_flags & BR_VLFLAG_MCAST_ENABLED)) {
+	    !(READ_ONCE(pmctx->vlan->priv_flags) & BR_VLFLAG_MCAST_ENABLED)) {
 		spin_unlock_bh(&br->multicast_lock);
 		return;
 	}
@@ -4095,7 +4095,8 @@ int br_multicast_rcv(struct net_bridge_mcast **brmctx,
 			*pmctx = &vlan->port_mcast_ctx;
 		}
 
-		if (!(masterv->priv_flags & BR_VLFLAG_GLOBAL_MCAST_ENABLED))
+		if (!(READ_ONCE(masterv->priv_flags) &
+		      BR_VLFLAG_GLOBAL_MCAST_ENABLED))
 			return 0;
 	}
 
@@ -4395,7 +4396,8 @@ void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on)
 			return;
 
 		spin_lock_bh(&br->multicast_lock);
-		vlan->priv_flags ^= BR_VLFLAG_MCAST_ENABLED;
+		WRITE_ONCE(vlan->priv_flags, vlan->priv_flags ^
+					    BR_VLFLAG_MCAST_ENABLED);
 		spin_unlock_bh(&br->multicast_lock);
 
 		if (on)
@@ -4411,7 +4413,8 @@ void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on)
 
 		br = vlan->port->br;
 		spin_lock_bh(&br->multicast_lock);
-		vlan->priv_flags ^= BR_VLFLAG_MCAST_ENABLED;
+		WRITE_ONCE(vlan->priv_flags, vlan->priv_flags ^
+					    BR_VLFLAG_MCAST_ENABLED);
 		if (on)
 			__br_multicast_enable_port_ctx(&vlan->port_mcast_ctx);
 		else
@@ -4489,7 +4492,8 @@ bool br_multicast_toggle_global_vlan(struct net_bridge_vlan *vlan, bool on)
 	if (on == !!(vlan->priv_flags & BR_VLFLAG_GLOBAL_MCAST_ENABLED))
 		return false;
 
-	vlan->priv_flags ^= BR_VLFLAG_GLOBAL_MCAST_ENABLED;
+	WRITE_ONCE(vlan->priv_flags, vlan->priv_flags ^
+				    BR_VLFLAG_GLOBAL_MCAST_ENABLED);
 	br_multicast_toggle_vlan(vlan, on);
 
 	return true;
diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c
index b2cd4e39326d06..ae76df0de05a0f 100644
--- a/net/bridge/br_netlink.c
+++ b/net/bridge/br_netlink.c
@@ -41,7 +41,7 @@ static int __get_num_vlan_infos(struct net_bridge_vlan_group *vg,
 		if (v->vid == pvid)
 			flags |= BRIDGE_VLAN_INFO_PVID;
 
-		if (v->flags & BRIDGE_VLAN_INFO_UNTAGGED)
+		if (READ_ONCE(v->flags) & BRIDGE_VLAN_INFO_UNTAGGED)
 			flags |= BRIDGE_VLAN_INFO_UNTAGGED;
 
 		if (vid_range_start == 0) {
@@ -81,7 +81,7 @@ static int br_get_num_vlan_infos(struct net_bridge_vlan_group *vg,
 		return 0;
 
 	if (filter_mask & RTEXT_FILTER_BRVLAN)
-		return vg->num_vlans;
+		return READ_ONCE(vg->num_vlans);
 
 	rcu_read_lock();
 	num_vlans = __get_num_vlan_infos(vg, filter_mask);
@@ -385,7 +385,7 @@ static int br_fill_ifvlaninfo_compressed(struct sk_buff *skb,
 		if (v->vid == pvid)
 			flags |= BRIDGE_VLAN_INFO_PVID;
 
-		if (v->flags & BRIDGE_VLAN_INFO_UNTAGGED)
+		if (READ_ONCE(v->flags) & BRIDGE_VLAN_INFO_UNTAGGED)
 			flags |= BRIDGE_VLAN_INFO_UNTAGGED;
 
 		if (vid_range_start == 0) {
@@ -437,7 +437,7 @@ static int br_fill_ifvlaninfo(struct sk_buff *skb,
 		if (v->vid == pvid)
 			vinfo.flags |= BRIDGE_VLAN_INFO_PVID;
 
-		if (v->flags & BRIDGE_VLAN_INFO_UNTAGGED)
+		if (READ_ONCE(v->flags) & BRIDGE_VLAN_INFO_UNTAGGED)
 			vinfo.flags |= BRIDGE_VLAN_INFO_UNTAGGED;
 
 		if (nla_put(skb, IFLA_BRIDGE_VLAN_INFO,
@@ -531,7 +531,7 @@ static int br_fill_ifinfo(struct sk_buff *skb,
 		else
 			vg = br_vlan_group_rcu(br);
 
-		if (!vg || !vg->num_vlans) {
+		if (!vg || !READ_ONCE(vg->num_vlans)) {
 			rcu_read_unlock();
 			goto done;
 		}
diff --git a/net/bridge/br_netlink_tunnel.c b/net/bridge/br_netlink_tunnel.c
index e7eceab5b515d4..e64054628ac9e3 100644
--- a/net/bridge/br_netlink_tunnel.c
+++ b/net/bridge/br_netlink_tunnel.c
@@ -29,8 +29,10 @@ static size_t __get_vlan_tinfo_size(void)
 bool vlan_tunid_inrange(const struct net_bridge_vlan *v_curr,
 			const struct net_bridge_vlan *v_last)
 {
-	__be32 tunid_curr = tunnel_id_to_key32(v_curr->tinfo.tunnel_id);
-	__be32 tunid_last = tunnel_id_to_key32(v_last->tinfo.tunnel_id);
+	__be32 tunid_curr, tunid_last;
+
+	tunid_curr = tunnel_id_to_key32(READ_ONCE(v_curr->tinfo.tunnel_id));
+	tunid_last = tunnel_id_to_key32(READ_ONCE(v_last->tinfo.tunnel_id));
 
 	return (be32_to_cpu(tunid_curr) - be32_to_cpu(tunid_last)) == 1;
 }
@@ -43,7 +45,7 @@ static int __get_num_vlan_tunnel_infos(struct net_bridge_vlan_group *vg)
 	/* Count number of vlan infos */
 	list_for_each_entry_rcu(v, &vg->vlan_list, vlist) {
 		/* only a context, bridge vlan not activated */
-		if (!br_vlan_should_use(v) || !v->tinfo.tunnel_id)
+		if (!br_vlan_should_use(v) || !READ_ONCE(v->tinfo.tunnel_id))
 			continue;
 
 		if (!vtbegin) {
@@ -124,19 +126,19 @@ static int br_fill_vlan_tinfo_range(struct sk_buff *skb,
 	if (vtend && (vtend->vid - vtbegin->vid) > 0) {
 		/* add range to skb */
 		err = br_fill_vlan_tinfo(skb, vtbegin->vid,
-					 vtbegin->tinfo.tunnel_id,
+					 READ_ONCE(vtbegin->tinfo.tunnel_id),
 					 BRIDGE_VLAN_INFO_RANGE_BEGIN);
 		if (err)
 			return err;
 
 		err = br_fill_vlan_tinfo(skb, vtend->vid,
-					 vtend->tinfo.tunnel_id,
+					 READ_ONCE(vtend->tinfo.tunnel_id),
 					 BRIDGE_VLAN_INFO_RANGE_END);
 		if (err)
 			return err;
 	} else {
 		err = br_fill_vlan_tinfo(skb, vtbegin->vid,
-					 vtbegin->tinfo.tunnel_id,
+					 READ_ONCE(vtbegin->tinfo.tunnel_id),
 					 0);
 		if (err)
 			return err;
diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h
index b01997ea95089e..67117fb3dc880f 100644
--- a/net/bridge/br_private.h
+++ b/net/bridge/br_private.h
@@ -190,6 +190,17 @@ enum {
 	BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED = BIT(6),
 };
 
+/* start publishing arrays when there're > BR_VLAN_PORT_ARRAY_THRESHOLD
+ * port-VLANs
+ */
+#define BR_VLAN_PORT_ARRAY_THRESHOLD 8
+
+struct net_bridge_vlan_port_array {
+	struct rcu_head		rcu;
+	unsigned int		count;
+	struct net_bridge_vlan	*vlans[];
+};
+
 /**
  * struct net_bridge_vlan - per-vlan entry
  *
@@ -210,6 +221,8 @@ enum {
  * @port_mcast_ctx: if MASTER flag unset, this is the per-port/vlan multicast
  *                  context
  * @msti: if MASTER flag set, this holds the VLANs MST instance
+ * @port_array: if MASTER flag set, this is the port-VLAN array
+ * @port_vlist: if MASTER flag set, this is the port-VLAN list
  * @vlist: sorted list of VLAN entries
  * @rcu: used for entry destruction
  *
@@ -244,6 +257,8 @@ struct net_bridge_vlan {
 
 	u16				msti;
 
+	struct net_bridge_vlan_port_array __rcu *port_array;
+	struct list_head		port_vlist;
 	struct list_head		vlist;
 
 	struct rcu_head			rcu;
@@ -256,8 +271,7 @@ struct net_bridge_vlan {
  * @tunnel_hash: Hash table to map from tunnel key ID (e.g. VXLAN VNI) to VLAN
  * @vlan_list: sorted VLAN entry list
  * @num_vlans: number of total VLAN entries
- * @pvid: PVID VLAN id
- * @pvid_state: PVID's STP state (e.g. forwarding, learning, blocking)
+ * @pvid: RCU-protected PVID VLAN entry
  *
  * IMPORTANT: Be careful when checking if there're VLAN entries using list
  *            primitives because the bridge can have entries in its list which
@@ -269,9 +283,8 @@ struct net_bridge_vlan_group {
 	struct rhashtable		vlan_hash;
 	struct rhashtable		tunnel_hash;
 	struct list_head		vlan_list;
+	struct net_bridge_vlan		__rcu *pvid;
 	u16				num_vlans;
-	u16				pvid;
-	u8				pvid_state;
 };
 
 /* bridge fdb flags */
@@ -666,13 +679,13 @@ static inline int br_is_root_bridge(const struct net_bridge *br)
 /* check if a VLAN entry is global */
 static inline bool br_vlan_is_master(const struct net_bridge_vlan *v)
 {
-	return v->flags & BRIDGE_VLAN_INFO_MASTER;
+	return READ_ONCE(v->flags) & BRIDGE_VLAN_INFO_MASTER;
 }
 
 /* check if a VLAN entry is used by the bridge */
 static inline bool br_vlan_is_brentry(const struct net_bridge_vlan *v)
 {
-	return v->flags & BRIDGE_VLAN_INFO_BRENTRY;
+	return READ_ONCE(v->flags) & BRIDGE_VLAN_INFO_BRENTRY;
 }
 
 /* check if we should use the vlan entry, returns false if it's only context */
@@ -688,6 +701,15 @@ static inline bool br_vlan_should_use(const struct net_bridge_vlan *v)
 	return true;
 }
 
+/* The vlan state can be changed with only rcu held by the mst code so
+ * annotate the lock-free read. br_vlan_set_state() is kept further down
+ * because it needs br_multicast_update_vlan_mcast_ctx().
+ */
+static inline u8 br_vlan_get_state(const struct net_bridge_vlan *v)
+{
+	return READ_ONCE(v->state);
+}
+
 static inline bool nbp_state_should_learn(const struct net_bridge_port *p)
 {
 	return p->state == BR_STATE_LEARNING || p->state == BR_STATE_FORWARDING;
@@ -903,9 +925,9 @@ int br_dev_queue_push_xmit(struct net *net, struct sock *sk, struct sk_buff *skb
 void br_forward(const struct net_bridge_port *to, struct sk_buff *skb,
 		bool local_rcv, bool local_orig);
 int br_forward_finish(struct net *net, struct sock *sk, struct sk_buff *skb);
-void br_flood(struct net_bridge *br, struct sk_buff *skb,
-	      enum br_pkt_type pkt_type, bool local_rcv, bool local_orig,
-	      u16 vid);
+void br_flood(struct net_bridge *br, struct net_bridge_vlan *v,
+	      struct sk_buff *skb, enum br_pkt_type pkt_type,
+	      bool local_rcv, bool local_orig);
 
 /* return true if both source port and dest port are isolated */
 static inline bool br_skb_isolated(const struct net_bridge_port *to,
@@ -1264,21 +1286,24 @@ br_multicast_ctx_vlan_global_disabled(const struct net_bridge_mcast *brmctx)
 {
 	return br_multicast_ctx_is_vlan(brmctx) &&
 	       (!br_opt_get(brmctx->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED) ||
-		!(brmctx->vlan->priv_flags & BR_VLFLAG_GLOBAL_MCAST_ENABLED));
+		!(READ_ONCE(brmctx->vlan->priv_flags) &
+		  BR_VLFLAG_GLOBAL_MCAST_ENABLED));
 }
 
 static inline bool
 br_multicast_ctx_vlan_disabled(const struct net_bridge_mcast *brmctx)
 {
 	return br_multicast_ctx_is_vlan(brmctx) &&
-	       !(brmctx->vlan->priv_flags & BR_VLFLAG_MCAST_ENABLED);
+	       !(READ_ONCE(brmctx->vlan->priv_flags) &
+		 BR_VLFLAG_MCAST_ENABLED);
 }
 
 static inline bool
 br_multicast_port_ctx_vlan_disabled(const struct net_bridge_mcast_port *pmctx)
 {
 	return br_multicast_port_ctx_is_vlan(pmctx) &&
-	       !(pmctx->vlan->priv_flags & BR_VLFLAG_MCAST_ENABLED);
+	       !(READ_ONCE(pmctx->vlan->priv_flags) &
+		 BR_VLFLAG_MCAST_ENABLED);
 }
 
 static inline bool
@@ -1287,7 +1312,7 @@ br_multicast_port_ctx_state_disabled(const struct net_bridge_mcast_port *pmctx)
 	return pmctx->port->state == BR_STATE_DISABLED ||
 	       (br_multicast_port_ctx_is_vlan(pmctx) &&
 		(br_multicast_port_ctx_vlan_disabled(pmctx) ||
-		 pmctx->vlan->state == BR_STATE_DISABLED));
+		 br_vlan_get_state(pmctx->vlan) == BR_STATE_DISABLED));
 }
 
 static inline bool
@@ -1296,7 +1321,7 @@ br_multicast_port_ctx_state_stopped(const struct net_bridge_mcast_port *pmctx)
 	return br_multicast_port_ctx_state_disabled(pmctx) ||
 	       pmctx->port->state == BR_STATE_BLOCKING ||
 	       (br_multicast_port_ctx_is_vlan(pmctx) &&
-		pmctx->vlan->state == BR_STATE_BLOCKING);
+		br_vlan_get_state(pmctx->vlan) == BR_STATE_BLOCKING);
 }
 
 static inline bool
@@ -1581,6 +1606,7 @@ bool br_should_learn(struct net_bridge_port *p, struct sk_buff *skb, u16 *vid);
 struct sk_buff *br_handle_vlan(struct net_bridge *br,
 			       const struct net_bridge_port *port,
 			       struct net_bridge_vlan_group *vg,
+			       struct net_bridge_vlan *vlan,
 			       struct sk_buff *skb);
 int br_vlan_add(struct net_bridge *br, u16 vid, u16 flags,
 		bool *changed, struct netlink_ext_ack *extack);
@@ -1675,11 +1701,16 @@ static inline int br_vlan_get_tag(const struct sk_buff *skb, u16 *vid)
 
 static inline u16 br_get_pvid(const struct net_bridge_vlan_group *vg)
 {
+	struct net_bridge_vlan *pvid;
+
 	if (!vg)
 		return 0;
 
-	smp_rmb();
-	return vg->pvid;
+	pvid = rcu_dereference_rtnl(vg->pvid);
+	if (!pvid)
+		return 0;
+
+	return pvid->vid;
 }
 
 static inline u16 br_vlan_flags(const struct net_bridge_vlan *v, u16 pvid)
@@ -1713,6 +1744,7 @@ static inline bool br_should_learn(struct net_bridge_port *p,
 static inline struct sk_buff *br_handle_vlan(struct net_bridge *br,
 					     const struct net_bridge_port *port,
 					     struct net_bridge_vlan_group *vg,
+					     struct net_bridge_vlan *vlan,
 					     struct sk_buff *skb)
 {
 	return skb;
@@ -1905,31 +1937,12 @@ bool br_vlan_global_opts_can_enter_range(const struct net_bridge_vlan *v_curr,
 bool br_vlan_global_opts_fill(struct sk_buff *skb, u16 vid, u16 vid_range,
 			      const struct net_bridge_vlan *v_opts);
 
-/* vlan state manipulation helpers using *_ONCE to annotate lock-free access,
- * while br_vlan_set_state() may access data protected by multicast_lock.
- */
-static inline u8 br_vlan_get_state(const struct net_bridge_vlan *v)
-{
-	return READ_ONCE(v->state);
-}
-
 static inline void br_vlan_set_state(struct net_bridge_vlan *v, u8 state)
 {
 	WRITE_ONCE(v->state, state);
 	br_multicast_update_vlan_mcast_ctx(v, state);
 }
 
-static inline u8 br_vlan_get_pvid_state(const struct net_bridge_vlan_group *vg)
-{
-	return READ_ONCE(vg->pvid_state);
-}
-
-static inline void br_vlan_set_pvid_state(struct net_bridge_vlan_group *vg,
-					  u8 state)
-{
-	WRITE_ONCE(vg->pvid_state, state);
-}
-
 /* learn_allow is true at ingress and false at egress */
 static inline bool br_vlan_state_allowed(u8 state, bool learn_allow)
 {
@@ -1942,6 +1955,11 @@ static inline bool br_vlan_state_allowed(u8 state, bool learn_allow)
 		return false;
 	}
 }
+#else
+static inline bool br_vlan_state_allowed(u8 state, bool learn_allow)
+{
+	return false;
+}
 #endif
 
 /* br_mst.c */
@@ -2372,6 +2390,8 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br,
 void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 		       u16 vid, struct net_bridge_port *p, struct nd_msg *msg);
 struct nd_msg *br_is_nd_neigh_msg(struct sk_buff *skb);
-bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, u16 vid);
-bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, u16 vid);
+bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p,
+				  const struct net_bridge_vlan *v);
+bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p,
+				      const struct net_bridge_vlan *v);
 #endif
diff --git a/net/bridge/br_stp_bpdu.c b/net/bridge/br_stp_bpdu.c
index 74ec42ba1e7d08..21d092f5acbb81 100644
--- a/net/bridge/br_stp_bpdu.c
+++ b/net/bridge/br_stp_bpdu.c
@@ -52,7 +52,10 @@ static void br_send_bpdu(struct net_bridge_port *p,
 			    LLC_SAP_BSPAN, LLC_PDU_CMD);
 	llc_pdu_init_as_ui_cmd(skb);
 
-	llc_mac_hdr_init(skb, p->dev->dev_addr, p->br->group_addr);
+	if (llc_mac_hdr_init(skb, p->dev->dev_addr, p->br->group_addr)) {
+		kfree_skb(skb);
+		return;
+	}
 
 	skb_reset_mac_header(skb);
 
diff --git a/net/bridge/br_stp_if.c b/net/bridge/br_stp_if.c
index a7e5422eb5d140..89bc161a4b4740 100644
--- a/net/bridge/br_stp_if.c
+++ b/net/bridge/br_stp_if.c
@@ -8,7 +8,7 @@
  */
 
 #include <linux/kernel.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/etherdevice.h>
 #include <linux/rtnetlink.h>
 #include <net/switchdev.h>
diff --git a/net/bridge/br_sysfs_if.c b/net/bridge/br_sysfs_if.c
index bde731c01e10ad..b9bcafa7f034c9 100644
--- a/net/bridge/br_sysfs_if.c
+++ b/net/bridge/br_sysfs_if.c
@@ -64,10 +64,7 @@ static int store_flag(struct net_bridge_port *p, unsigned long v,
 	int err;
 
 	oflags = flags;
-	if (v)
-		__set_bit(bitnr, &flags);
-	else
-		__clear_bit(bitnr, &flags);
+	__assign_bit(bitnr, &flags, v);
 
 	if (flags == oflags)
 		return 0;
@@ -77,10 +74,7 @@ static int store_flag(struct net_bridge_port *p, unsigned long v,
 		netdev_err(p->dev, "%s\n", extack._msg);
 		return err;
 	}
-	if (v)
-		set_bit(bitnr, &p->flags);
-	else
-		clear_bit(bitnr, &p->flags);
+	assign_bit(bitnr, &p->flags, v);
 	br_port_flags_change(p, BIT(bitnr));
 	return 0;
 }
diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c
index 92b3cb621a264b..ce5aa15c454024 100644
--- a/net/bridge/br_vlan.c
+++ b/net/bridge/br_vlan.c
@@ -35,23 +35,21 @@ static struct net_bridge_vlan *br_vlan_lookup(struct rhashtable *tbl, u16 vid)
 }
 
 static void __vlan_add_pvid(struct net_bridge_vlan_group *vg,
-			    const struct net_bridge_vlan *v)
+			    struct net_bridge_vlan *v)
 {
-	if (vg->pvid == v->vid)
+	if (rcu_access_pointer(vg->pvid) == v)
 		return;
 
-	smp_wmb();
-	br_vlan_set_pvid_state(vg, v->state);
-	vg->pvid = v->vid;
+	RCU_INIT_POINTER(vg->pvid, v);
 }
 
-static void __vlan_delete_pvid(struct net_bridge_vlan_group *vg, u16 vid)
+static void __vlan_delete_pvid(struct net_bridge_vlan_group *vg,
+			       struct net_bridge_vlan *v)
 {
-	if (vg->pvid != vid)
+	if (rcu_access_pointer(vg->pvid) != v)
 		return;
 
-	smp_wmb();
-	vg->pvid = 0;
+	RCU_INIT_POINTER(vg->pvid, NULL);
 }
 
 /* Update the BRIDGE_VLAN_INFO_PVID and BRIDGE_VLAN_INFO_UNTAGGED flags of @v.
@@ -62,6 +60,8 @@ static bool __vlan_flags_update(struct net_bridge_vlan *v, u16 flags,
 				bool commit)
 {
 	struct net_bridge_vlan_group *vg;
+	struct net_bridge_vlan *pvid;
+	u16 vlan_flags;
 	bool change;
 
 	if (br_vlan_is_master(v))
@@ -70,8 +70,10 @@ static bool __vlan_flags_update(struct net_bridge_vlan *v, u16 flags,
 		vg = nbp_vlan_group(v->port);
 
 	/* check if anything would be changed on commit */
-	change = !!(flags & BRIDGE_VLAN_INFO_PVID) == !!(vg->pvid != v->vid) ||
-		 ((flags ^ v->flags) & BRIDGE_VLAN_INFO_UNTAGGED);
+	vlan_flags = v->flags;
+	pvid = rtnl_dereference(vg->pvid);
+	change = !!(flags & BRIDGE_VLAN_INFO_PVID) == !!(pvid != v) ||
+		 ((flags ^ vlan_flags) & BRIDGE_VLAN_INFO_UNTAGGED);
 
 	if (!commit)
 		goto out;
@@ -79,12 +81,13 @@ static bool __vlan_flags_update(struct net_bridge_vlan *v, u16 flags,
 	if (flags & BRIDGE_VLAN_INFO_PVID)
 		__vlan_add_pvid(vg, v);
 	else
-		__vlan_delete_pvid(vg, v->vid);
+		__vlan_delete_pvid(vg, v);
 
 	if (flags & BRIDGE_VLAN_INFO_UNTAGGED)
-		v->flags |= BRIDGE_VLAN_INFO_UNTAGGED;
+		vlan_flags |= BRIDGE_VLAN_INFO_UNTAGGED;
 	else
-		v->flags &= ~BRIDGE_VLAN_INFO_UNTAGGED;
+		vlan_flags &= ~BRIDGE_VLAN_INFO_UNTAGGED;
+	WRITE_ONCE(v->flags, vlan_flags);
 
 out:
 	return change;
@@ -127,10 +130,12 @@ static void __vlan_add_list(struct net_bridge_vlan *v)
 	struct list_head *headp, *hpos;
 	struct net_bridge_vlan *vent;
 
-	if (br_vlan_is_master(v))
+	if (br_vlan_is_master(v)) {
 		vg = br_vlan_group(v->br);
-	else
+	} else {
 		vg = nbp_vlan_group(v->port);
+		list_add_rcu(&v->port_vlist, &v->brvlan->port_vlist);
+	}
 
 	headp = &vg->vlan_list;
 	list_for_each_prev(hpos, headp) {
@@ -144,6 +149,10 @@ static void __vlan_add_list(struct net_bridge_vlan *v)
 static void __vlan_del_list(struct net_bridge_vlan *v)
 {
 	list_del_rcu(&v->vlist);
+	if (br_vlan_is_master(v))
+		WARN_ON(!list_empty(&v->port_vlist));
+	else
+		list_del_rcu(&v->port_vlist);
 }
 
 static int __vlan_vid_del(struct net_device *dev, struct net_bridge *br,
@@ -249,6 +258,36 @@ static void br_vlan_init_state(struct net_bridge_vlan *v)
 	v->msti = 0;
 }
 
+static unsigned int br_vlan_num_ports(const struct net_bridge_vlan *masterv)
+{
+	return refcount_read(&masterv->refcnt) - br_vlan_is_brentry(masterv);
+}
+
+static void br_vlan_rebuild_port_array(struct net_bridge_vlan *masterv,
+				       unsigned int count)
+{
+	struct net_bridge_vlan_port_array *array = NULL, *old;
+	unsigned int i = 0;
+
+	WARN_ON(!br_vlan_is_master(masterv));
+
+	if (count > BR_VLAN_PORT_ARRAY_THRESHOLD)
+		array = kvmalloc(struct_size(array, vlans, count), GFP_KERNEL);
+
+	if (array) {
+		struct net_bridge_vlan *pv;
+
+		array->count = count;
+		list_for_each_entry(pv, &masterv->port_vlist, port_vlist)
+			array->vlans[i++] = pv;
+	}
+
+	old = rtnl_dereference(masterv->port_array);
+	rcu_assign_pointer(masterv->port_array, array);
+	if (old)
+		kvfree_rcu(old, rcu);
+}
+
 /* This is the shared VLAN add function which works for both ports and bridge
  * devices. There are four possible calls to this function in terms of the
  * vlan entry type:
@@ -332,6 +371,7 @@ static int __vlan_add(struct net_bridge_vlan *v, u16 flags,
 		}
 		br_multicast_ctx_init(br, v, &v->br_mcast_ctx);
 		v->priv_flags |= BR_VLFLAG_GLOBAL_MCAST_ENABLED;
+		INIT_LIST_HEAD(&v->port_vlist);
 	}
 
 	/* Add the dev mac and count the vlan only if it's usable */
@@ -343,7 +383,7 @@ static int __vlan_add(struct net_bridge_vlan *v, u16 flags,
 				goto out_filt;
 			}
 		}
-		vg->num_vlans++;
+		WRITE_ONCE(vg->num_vlans, vg->num_vlans + 1);
 	}
 
 	/* set the state before publishing */
@@ -358,15 +398,17 @@ static int __vlan_add(struct net_bridge_vlan *v, u16 flags,
 	__vlan_flags_commit(v, flags);
 	br_multicast_toggle_one_vlan(v, true);
 
-	if (p)
+	if (p) {
+		br_vlan_rebuild_port_array(masterv, br_vlan_num_ports(masterv));
 		nbp_vlan_set_vlan_dev_state(p, v->vid);
+	}
 out:
 	return err;
 
 out_fdb_insert:
 	if (br_vlan_should_use(v)) {
 		br_fdb_find_delete_local(br, p, dev->dev_addr, v->vid);
-		vg->num_vlans--;
+		WRITE_ONCE(vg->num_vlans, vg->num_vlans - 1);
 	}
 
 out_filt:
@@ -402,7 +444,7 @@ static void __vlan_del(struct net_bridge_vlan *v)
 		masterv = v->brvlan;
 	}
 
-	__vlan_delete_pvid(vg, v->vid);
+	__vlan_delete_pvid(vg, v);
 	if (p) {
 		err = __vlan_vid_del(p->dev, p->br, v);
 		if (err)
@@ -419,8 +461,8 @@ static void __vlan_del(struct net_bridge_vlan *v)
 	}
 
 	if (br_vlan_should_use(v)) {
-		v->flags &= ~BRIDGE_VLAN_INFO_BRENTRY;
-		vg->num_vlans--;
+		WRITE_ONCE(v->flags, v->flags & ~BRIDGE_VLAN_INFO_BRENTRY);
+		WRITE_ONCE(vg->num_vlans, vg->num_vlans - 1);
 	}
 
 	if (masterv != v) {
@@ -428,6 +470,9 @@ static void __vlan_del(struct net_bridge_vlan *v)
 		rhashtable_remove_fast(&vg->vlan_hash, &v->vnode,
 				       br_vlan_rht_params);
 		__vlan_del_list(v);
+		/* -1 because br_vlan_put_master() is called later */
+		br_vlan_rebuild_port_array(masterv,
+					   br_vlan_num_ports(masterv) - 1);
 		nbp_vlan_set_vlan_dev_state(p, v->vid);
 		br_multicast_toggle_one_vlan(v, false);
 		br_multicast_port_ctx_deinit(&v->port_mcast_ctx);
@@ -452,7 +497,7 @@ static void __vlan_flush(const struct net_bridge *br,
 	struct net_bridge_vlan *vlan, *tmp;
 	u16 v_start = 0, v_end = 0;
 
-	__vlan_delete_pvid(vg, vg->pvid);
+	__vlan_delete_pvid(vg, rtnl_dereference(vg->pvid));
 	list_for_each_entry_safe(vlan, tmp, &vg->vlan_list, vlist) {
 		/* take care of disjoint ranges */
 		if (!v_start) {
@@ -475,10 +520,10 @@ static void __vlan_flush(const struct net_bridge *br,
 struct sk_buff *br_handle_vlan(struct net_bridge *br,
 			       const struct net_bridge_port *p,
 			       struct net_bridge_vlan_group *vg,
+			       struct net_bridge_vlan *v,
 			       struct sk_buff *skb)
 {
 	struct pcpu_sw_netstats *stats;
-	struct net_bridge_vlan *v;
 	u16 vid;
 
 	/* If this packet was not filtered at input, let it pass */
@@ -489,19 +534,22 @@ struct sk_buff *br_handle_vlan(struct net_bridge *br,
 	 * a valid vlan id.  If the vlan id has untagged flag set,
 	 * send untagged; otherwise, send tagged.
 	 */
-	br_vlan_get_tag(skb, &vid);
-	v = br_vlan_find(vg, vid);
-	/* Vlan entry must be configured at this point.  The
-	 * only exception is the bridge is set in promisc mode and the
-	 * packet is destined for the bridge device.  In this case
-	 * pass the packet as is.
-	 */
-	if (!v || !br_vlan_should_use(v)) {
-		if ((br->dev->flags & IFF_PROMISC) && skb->dev == br->dev) {
-			goto out;
-		} else {
-			kfree_skb(skb);
-			return NULL;
+	if (!v) {
+		br_vlan_get_tag(skb, &vid);
+		v = br_vlan_find(vg, vid);
+		/* Vlan entry must be configured at this point.  The
+		 * only exception is the bridge is set in promisc mode and the
+		 * packet is destined for the bridge device.  In this case
+		 * pass the packet as is.
+		 */
+		if (!v || !br_vlan_should_use(v)) {
+			if ((br->dev->flags & IFF_PROMISC) &&
+			    skb->dev == br->dev) {
+				goto out;
+			} else {
+				kfree_skb(skb);
+				return NULL;
+			}
 		}
 	}
 	if (br_opt_get(br, BROPT_VLAN_STATS_ENABLED)) {
@@ -519,7 +567,7 @@ struct sk_buff *br_handle_vlan(struct net_bridge *br,
 	 * hardware on each egress port as appropriate. So only strip the VLAN
 	 * header if forwarding offload is not being used.
 	 */
-	if (v->flags & BRIDGE_VLAN_INFO_UNTAGGED &&
+	if (READ_ONCE(v->flags) & BRIDGE_VLAN_INFO_UNTAGGED &&
 	    !br_switchdev_frame_uses_tx_fwd_offload(skb))
 		__vlan_hwaccel_clear_tag(skb);
 
@@ -578,42 +626,32 @@ static bool __allowed_ingress(const struct net_bridge *br,
 	}
 
 	if (!*vid) {
-		u16 pvid = br_get_pvid(vg);
-
+		v = vg ? rcu_dereference(vg->pvid) : NULL;
 		/* Frame had a tag with VID 0 or did not have a tag.
 		 * See if pvid is set on this port.  That tells us which
 		 * vlan untagged or priority-tagged traffic belongs to.
 		 */
-		if (!pvid)
+		if (!v)
 			goto drop;
 
 		/* PVID is set on this port.  Any untagged or priority-tagged
 		 * ingress frame is considered to belong to this vlan.
 		 */
-		*vid = pvid;
+		*vid = v->vid;
 		if (likely(!tagged))
 			/* Untagged Frame. */
-			__vlan_hwaccel_put_tag(skb, br->vlan_proto, pvid);
+			__vlan_hwaccel_put_tag(skb, br->vlan_proto, v->vid);
 		else
 			/* Priority-tagged Frame.
 			 * At this point, we know that skb->vlan_tci VID
 			 * field was 0.
 			 * We update only VID field and preserve PCP field.
 			 */
-			skb->vlan_tci |= pvid;
-
-		/* if snooping and stats are disabled we can avoid the lookup */
-		if (!br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED) &&
-		    !br_opt_get(br, BROPT_VLAN_STATS_ENABLED)) {
-			if (*state == BR_STATE_FORWARDING) {
-				*state = br_vlan_get_pvid_state(vg);
-				if (!br_vlan_state_allowed(*state, true))
-					goto drop;
-			}
-			return true;
-		}
+			skb->vlan_tci |= v->vid;
+	} else {
+		v = br_vlan_find(vg, *vid);
 	}
-	v = br_vlan_find(vg, *vid);
+
 	if (!v || !br_vlan_should_use(v))
 		goto drop;
 
@@ -689,18 +727,17 @@ bool br_should_learn(struct net_bridge_port *p, struct sk_buff *skb, u16 *vid)
 		return true;
 
 	vg = nbp_vlan_group_rcu(p);
-	if (!vg || !vg->num_vlans)
+	if (!vg || !READ_ONCE(vg->num_vlans))
 		return false;
 
 	if (!br_vlan_get_tag(skb, vid) && skb->vlan_proto != br->vlan_proto)
 		*vid = 0;
 
 	if (!*vid) {
-		*vid = br_get_pvid(vg);
-		if (!*vid ||
-		    !br_vlan_state_allowed(br_vlan_get_pvid_state(vg), true))
+		v = rcu_dereference(vg->pvid);
+		if (!v || !br_vlan_state_allowed(br_vlan_get_state(v), true))
 			return false;
-
+		*vid = v->vid;
 		return true;
 	}
 
@@ -750,8 +787,8 @@ static int br_vlan_add_existing(struct net_bridge *br,
 		}
 
 		refcount_inc(&vlan->refcnt);
-		vlan->flags |= BRIDGE_VLAN_INFO_BRENTRY;
-		vg->num_vlans++;
+		WRITE_ONCE(vlan->flags, vlan->flags | BRIDGE_VLAN_INFO_BRENTRY);
+		WRITE_ONCE(vg->num_vlans, vg->num_vlans + 1);
 		*changed = true;
 		br_multicast_toggle_one_vlan(vlan, true);
 	}
@@ -1060,17 +1097,10 @@ int br_vlan_set_stats_per_port(struct net_bridge *br, unsigned long val)
 
 static bool vlan_default_pvid(struct net_bridge_vlan_group *vg, u16 vid)
 {
-	struct net_bridge_vlan *v;
-
-	if (vid != vg->pvid)
-		return false;
-
-	v = br_vlan_lookup(&vg->vlan_hash, vid);
-	if (v && br_vlan_should_use(v) &&
-	    (v->flags & BRIDGE_VLAN_INFO_UNTAGGED))
-		return true;
+	struct net_bridge_vlan *pvid = rtnl_dereference(vg->pvid);
 
-	return false;
+	return pvid && pvid->vid == vid && br_vlan_should_use(pvid) &&
+	       (pvid->flags & BRIDGE_VLAN_INFO_UNTAGGED);
 }
 
 static void br_vlan_disable_default_pvid(struct net_bridge *br)
@@ -1488,12 +1518,12 @@ int br_vlan_fill_forward_path_mode(struct net_bridge *br,
 	if (!v || !br_vlan_should_use(v))
 		return -EINVAL;
 
-	if (!(v->flags & BRIDGE_VLAN_INFO_UNTAGGED))
+	if (!(READ_ONCE(v->flags) & BRIDGE_VLAN_INFO_UNTAGGED))
 		return 0;
 
 	if (path->bridge.vlan_mode == DEV_PATH_BR_VLAN_TAG)
 		path->bridge.vlan_mode = DEV_PATH_BR_VLAN_KEEP;
-	else if (v->priv_flags & BR_VLFLAG_TAGGING_BY_SWITCHDEV)
+	else if (READ_ONCE(v->priv_flags) & BR_VLFLAG_TAGGING_BY_SWITCHDEV)
 		path->bridge.vlan_mode = DEV_PATH_BR_VLAN_UNTAG_HW;
 	else
 		path->bridge.vlan_mode = DEV_PATH_BR_VLAN_UNTAG;
@@ -1523,7 +1553,7 @@ int br_vlan_get_info(const struct net_device *dev, u16 vid,
 
 	p_vinfo->vid = vid;
 	p_vinfo->flags = v->flags;
-	if (vid == br_get_pvid(vg))
+	if (v == rcu_access_pointer(vg->pvid))
 		p_vinfo->flags |= BRIDGE_VLAN_INFO_PVID;
 	return 0;
 }
@@ -1549,8 +1579,8 @@ int br_vlan_get_info_rcu(const struct net_device *dev, u16 vid,
 		return -ENOENT;
 
 	p_vinfo->vid = vid;
-	p_vinfo->flags = v->flags;
-	if (vid == br_get_pvid(vg))
+	p_vinfo->flags = READ_ONCE(v->flags);
+	if (v == rcu_access_pointer(vg->pvid))
 		p_vinfo->flags |= BRIDGE_VLAN_INFO_PVID;
 	return 0;
 }
@@ -1955,7 +1985,7 @@ void br_vlan_notify(const struct net_bridge *br,
 			goto out_kfree;
 
 		flags = v->flags;
-		if (br_get_pvid(vg) == v->vid)
+		if (v == rcu_access_pointer(vg->pvid))
 			flags |= BRIDGE_VLAN_INFO_PVID;
 		break;
 	case RTM_DELVLAN:
diff --git a/net/bridge/br_vlan_options.c b/net/bridge/br_vlan_options.c
index cb0f556ff40dbe..e4b7a147e42b21 100644
--- a/net/bridge/br_vlan_options.c
+++ b/net/bridge/br_vlan_options.c
@@ -43,7 +43,7 @@ bool br_vlan_opts_eq_range(const struct net_bridge_vlan *v_curr,
 	u8 range_mc_rtr = br_vlan_multicast_router(range_end);
 	u8 curr_mc_rtr = br_vlan_multicast_router(v_curr);
 
-	if (v_curr->state != range_end->state)
+	if (br_vlan_get_state(v_curr) != br_vlan_get_state(range_end))
 		return false;
 
 	if (!__vlan_tun_can_enter_range(v_curr, range_end))
@@ -110,8 +110,7 @@ size_t br_vlan_opts_nl_size(void)
 	       + 0;
 }
 
-static int br_vlan_modify_state(struct net_bridge_vlan_group *vg,
-				struct net_bridge_vlan *v,
+static int br_vlan_modify_state(struct net_bridge_vlan *v,
 				u8 state,
 				bool *changed,
 				struct netlink_ext_ack *extack)
@@ -143,9 +142,6 @@ static int br_vlan_modify_state(struct net_bridge_vlan_group *vg,
 	if (v->state == state)
 		return 0;
 
-	if (v->vid == br_get_pvid(vg))
-		br_vlan_set_pvid_state(vg, state);
-
 	br_vlan_set_state(v, state);
 	*changed = true;
 
@@ -216,7 +212,6 @@ static int br_vlan_modify_tunnel(const struct net_bridge_port *p,
 
 static int br_vlan_process_one_opts(const struct net_bridge *br,
 				    const struct net_bridge_port *p,
-				    struct net_bridge_vlan_group *vg,
 				    struct net_bridge_vlan *v,
 				    struct nlattr **tb,
 				    bool *changed,
@@ -228,7 +223,7 @@ static int br_vlan_process_one_opts(const struct net_bridge *br,
 	if (tb[BRIDGE_VLANDB_ENTRY_STATE]) {
 		u8 state = nla_get_u8(tb[BRIDGE_VLANDB_ENTRY_STATE]);
 
-		err = br_vlan_modify_state(vg, v, state, changed, extack);
+		err = br_vlan_modify_state(v, state, changed, extack);
 		if (err)
 			return err;
 	}
@@ -276,7 +271,8 @@ static int br_vlan_process_one_opts(const struct net_bridge *br,
 		}
 
 		if (val != enabled) {
-			v->priv_flags ^= BR_VLFLAG_NEIGH_SUPPRESS_ENABLED;
+			WRITE_ONCE(v->priv_flags, v->priv_flags ^
+				   BR_VLFLAG_NEIGH_SUPPRESS_ENABLED);
 			*changed = true;
 		}
 	}
@@ -292,7 +288,8 @@ static int br_vlan_process_one_opts(const struct net_bridge *br,
 		}
 
 		if (val != enabled) {
-			v->priv_flags ^= BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED;
+			WRITE_ONCE(v->priv_flags, v->priv_flags ^
+				   BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED);
 			*changed = true;
 		}
 	}
@@ -337,8 +334,7 @@ int br_vlan_process_options(const struct net_bridge *br,
 			break;
 		}
 
-		err = br_vlan_process_one_opts(br, p, vg, v, tb, &changed,
-					       extack);
+		err = br_vlan_process_one_opts(br, p, v, tb, &changed, extack);
 		if (err)
 			break;
 
diff --git a/net/bridge/br_vlan_tunnel.c b/net/bridge/br_vlan_tunnel.c
index 257cae9f156983..04751f15d70843 100644
--- a/net/bridge/br_vlan_tunnel.c
+++ b/net/bridge/br_vlan_tunnel.c
@@ -22,7 +22,7 @@ static inline int br_vlan_tunid_cmp(struct rhashtable_compare_arg *arg,
 	const struct net_bridge_vlan *vle = ptr;
 	__be64 tunid = *(__be64 *)arg->key;
 
-	return vle->tinfo.tunnel_id != tunid;
+	return READ_ONCE(vle->tinfo.tunnel_id) != tunid;
 }
 
 static const struct rhashtable_params br_vlan_tunnel_rht_params = {
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index f36ce5ae75680a..621c5de2cfc668 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -7,7 +7,6 @@
 #include <linux/highmem.h>
 #include <linux/mm.h>
 #include <linux/overflow.h>
-#include <linux/pagemap.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #ifdef CONFIG_BLOCK
diff --git a/net/ceph/pagevec.c b/net/ceph/pagevec.c
index a6aa5b3b7a1ef7..12798cf743a6e7 100644
--- a/net/ceph/pagevec.c
+++ b/net/ceph/pagevec.c
@@ -6,6 +6,7 @@
 #include <linux/slab.h>
 #include <linux/file.h>
 #include <linux/namei.h>
+#include <linux/pagemap.h>
 #include <linux/writeback.h>
 
 #include <linux/ceph/libceph.h>
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 173b5d97bd409a..b9b50ce8ea7cad 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -49,7 +49,6 @@
 #include <linux/highmem.h>
 #include <linux/spinlock.h>
 #include <linux/slab.h>
-#include <linux/pagemap.h>
 #include <linux/iov_iter.h>
 #include <linux/indirect_call_wrapper.h>
 #include <linux/crc32.h>
diff --git a/net/core/dev.c b/net/core/dev.c
index c67900354fa64b..8582fed58fd151 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -568,6 +568,41 @@ static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 }
 #endif
 
+#ifdef CONFIG_PROVE_LOCKING
+static int netdev_lock_cmp_fn(const struct lockdep_map *a,
+			      const struct lockdep_map *b)
+{
+	if (a == b)
+		return 0;
+
+	/* @a and @b must be of same class - both virtual or physical.
+	 * cmp_fn won't be called for devices of different classes.
+	 *
+	 * For the same class only allow nesting under the protection
+	 * of rtnl_lock. Note that we can't use lockdep_rtnl_is_held()
+	 * here, it always answers UNKNOWN from within lockdep.
+	 */
+	return rtnl_is_locked() ? -1 : 1;
+}
+
+/* A virtual device can be locked before the physical device it leases
+ * queues from, see netdev_nl_queue_create_doit(). Keep the two kinds
+ * in separate classes so the dependency graph enforces the order;
+ * netdev_lock_cmp_fn() then only has to rule on same-class nesting.
+ */
+void netdev_set_instance_lock_class(struct net_device *dev)
+{
+	static struct lock_class_key netdev_virt_instance_lock_key;
+
+	if (dev->dev.parent)
+		return;
+
+	lockdep_set_class(&dev->lock, &netdev_virt_instance_lock_key);
+	lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL);
+}
+EXPORT_SYMBOL_GPL(netdev_set_instance_lock_class);
+#endif
+
 /*******************************************************************************
  *
  *		Protocol management and registration routines
@@ -2901,7 +2936,7 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
 		dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
 
 		tc = netdev_txq_to_tc(dev, index);
-		if (tc < 0)
+		if (tc < 0 || tc >= num_tc)
 			return -EINVAL;
 	}
 
@@ -3818,20 +3853,36 @@ static netdev_features_t dflt_features_check(struct sk_buff *skb,
 	return vlan_features_check(skb, features);
 }
 
-static bool skb_gso_has_extension_hdr(const struct sk_buff *skb)
+static bool __skb_has_ipv6_ext_hdr(const struct sk_buff *skb, int nhoff)
 {
-	if (!skb->encapsulation)
-		return ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 ||
-			 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 &&
-			  vlan_get_protocol(skb) == htons(ETH_P_IPV6))) &&
-			skb_transport_header_was_set(skb) &&
-			skb_network_header_len(skb) != sizeof(struct ipv6hdr));
-	else
-		return (!skb_inner_network_header_was_set(skb) ||
-			((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 ||
-			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 &&
-			   inner_ip_hdr(skb)->version == 6)) &&
-			 skb_inner_network_header_len(skb) != sizeof(struct ipv6hdr)));
+	const struct ipv6hdr *ip6h;
+	struct ipv6hdr _ip6h;
+
+	ip6h = skb_header_pointer(skb, nhoff, sizeof(_ip6h), &_ip6h);
+	return ip6h && ip6h->version == 6 && ipv6_ext_hdr(ip6h->nexthdr);
+}
+
+static bool skb_has_ipv6_extension_hdr(const struct sk_buff *skb)
+{
+	if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
+		if (__skb_has_ipv6_ext_hdr(skb, skb_network_offset(skb)))
+			return true;
+
+		/* Catch tunnels without skb->encapsulation (e.g., virtio). */
+		if (!skb->encapsulation &&
+		    skb_transport_header_was_set(skb) &&
+		    skb_network_header_len(skb) != sizeof(struct ipv6hdr))
+			return true;
+	}
+
+	/* Tunnels without an inner network header, such as SCTP-in-UDP or
+	 * PSP, have no inner IP header and thus no inner extension header.
+	 */
+	if (skb->encapsulation && skb_inner_network_header_was_set(skb) &&
+	    __skb_has_ipv6_ext_hdr(skb, skb_inner_network_offset(skb)))
+		return true;
+
+	return false;
 }
 
 static netdev_features_t gso_features_check(const struct sk_buff *skb,
@@ -3886,7 +3937,7 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb,
 	 * so neither does TSO that depends on it.
 	 */
 	if (features & NETIF_F_IPV6_CSUM &&
-	    skb_gso_has_extension_hdr(skb))
+	    skb_has_ipv6_extension_hdr(skb))
 		features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4);
 
 	return features;
@@ -3988,8 +4039,7 @@ int skb_csum_hwoffload_help(struct sk_buff *skb,
 		return 0;
 
 	if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
-		if (vlan_get_protocol(skb) == htons(ETH_P_IPV6) &&
-		    skb_network_header_len(skb) != sizeof(struct ipv6hdr))
+		if (skb_has_ipv6_extension_hdr(skb))
 			goto sw_checksum;
 
 		switch (skb->csum_offset) {
@@ -4404,9 +4454,14 @@ EXPORT_SYMBOL(dev_loopback_xmit);
 static struct netdev_queue *
 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
 {
-	int qm = skb_get_queue_mapping(skb);
+	int queue = skb_get_queue_mapping(skb);
+	int capped;
 
-	return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm));
+	capped = netdev_cap_txqueue(dev, queue);
+	if (unlikely(capped != queue))
+		skb_set_queue_mapping(skb, capped);
+
+	return netdev_get_tx_queue(dev, capped);
 }
 
 #ifndef CONFIG_PREEMPT_RT
@@ -4415,9 +4470,13 @@ static bool netdev_xmit_txqueue_skipped(void)
 	return __this_cpu_read(softnet_data.xmit.skip_txqueue);
 }
 
-void netdev_xmit_skip_txqueue(bool skip)
+bool netdev_xmit_skip_txqueue(bool skip)
 {
+	bool prev = netdev_xmit_txqueue_skipped();
+
 	__this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
+
+	return prev;
 }
 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
 
@@ -4427,9 +4486,13 @@ static bool netdev_xmit_txqueue_skipped(void)
 	return current->net_xmit.skip_txqueue;
 }
 
-void netdev_xmit_skip_txqueue(bool skip)
+bool netdev_xmit_skip_txqueue(bool skip)
 {
+	bool prev = netdev_xmit_txqueue_skipped();
+
 	current->net_xmit.skip_txqueue = skip;
+
+	return prev;
 }
 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
 #endif
@@ -4848,21 +4911,25 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
 	tcx_set_ingress(skb, false);
 #ifdef CONFIG_NET_EGRESS
 	if (static_branch_unlikely(&egress_needed_key)) {
+		bool skip_txq;
+
 		if (nf_hook_egress_active()) {
 			skb = nf_hook_egress(skb, &rc, dev);
 			if (!skb)
 				goto out;
 		}
 
-		netdev_xmit_skip_txqueue(false);
+		skip_txq = netdev_xmit_skip_txqueue(false);
 
 		nf_skip_egress(skb, true);
 		skb = sch_handle_egress(skb, &rc, dev);
-		if (!skb)
+		if (!skb) {
+			netdev_xmit_skip_txqueue(skip_txq);
 			goto out;
+		}
 		nf_skip_egress(skb, false);
 
-		if (netdev_xmit_txqueue_skipped())
+		if (netdev_xmit_skip_txqueue(skip_txq))
 			txq = netdev_tx_queue_mapping(dev, skb);
 	}
 #endif
@@ -5376,7 +5443,8 @@ void kick_defer_list_purge(unsigned int cpu)
 		backlog_unlock_irq_restore(sd, flags);
 
 	} else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
-		smp_call_function_single_async(cpu, &sd->defer_csd);
+		if (smp_call_function_single_async(cpu, &sd->defer_csd))
+			WRITE_ONCE(sd->defer_ipi_scheduled, 0);
 	}
 }
 
@@ -6900,25 +6968,35 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
 }
 EXPORT_SYMBOL(napi_complete_done);
 
-static void skb_defer_free_flush(void)
+static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget)
 {
 	struct llist_node *free_list;
 	struct sk_buff *skb, *next;
+
+	if (llist_empty(&sdn->defer_list))
+		return;
+	atomic_long_set(&sdn->defer_count, 0);
+	free_list = llist_del_all(&sdn->defer_list);
+
+	llist_for_each_entry_safe(skb, next, free_list, ll_node) {
+		prefetch(next);
+		napi_consume_skb(skb, budget);
+	}
+}
+
+void skb_defer_node_flush(struct skb_defer_node *sdn)
+{
+	__skb_defer_free_flush(sdn, 0);
+}
+
+static void skb_defer_free_flush(void)
+{
 	struct skb_defer_node *sdn;
 	int node;
 
 	for_each_node(node) {
 		sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node;
-
-		if (llist_empty(&sdn->defer_list))
-			continue;
-		atomic_long_set(&sdn->defer_count, 0);
-		free_list = llist_del_all(&sdn->defer_list);
-
-		llist_for_each_entry_safe(skb, next, free_list, ll_node) {
-			prefetch(next);
-			napi_consume_skb(skb, 1);
-		}
+		__skb_defer_free_flush(sdn, 1);
 	}
 }
 
@@ -12172,6 +12250,8 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
 #endif
 
 	mutex_init(&dev->lock);
+	/* see also netdev_set_instance_lock_class() */
+	lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL);
 	netif_rx_mode_init(dev);
 
 	dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
@@ -12897,6 +12977,7 @@ static int dev_cpu_dead(unsigned int oldcpu)
 	struct sk_buff **list_skb;
 	struct sk_buff *skb;
 	unsigned int cpu;
+	int node;
 	struct softnet_data *sd, *oldsd, *remsd = NULL;
 
 	local_irq_disable();
@@ -12957,6 +13038,17 @@ static int dev_cpu_dead(unsigned int oldcpu)
 		rps_input_queue_head_incr(oldsd);
 	}
 
+	for_each_node(node)
+		skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
+						 oldcpu) + node);
+	node = cpu_to_node(oldcpu);
+	if (node_possible(node) &&
+	    !cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) {
+		for_each_possible_cpu(cpu)
+			skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
+							 cpu) + node);
+	}
+
 	return 0;
 }
 
diff --git a/net/core/dev.h b/net/core/dev.h
index b757faead4d1a3..51921623fc0a41 100644
--- a/net/core/dev.h
+++ b/net/core/dev.h
@@ -95,6 +95,8 @@ extern int		netdev_unregister_timeout_secs;
 extern int		weight_p;
 extern int		dev_weight_rx_bias;
 extern int		dev_weight_tx_bias;
+extern u8		netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
+extern bool		netdev_rss_key_initialized;
 
 extern struct rw_semaphore dev_addr_sem;
 
@@ -399,6 +401,8 @@ static inline void napi_assert_will_not_race(const struct napi_struct *napi)
 	WARN_ON(READ_ONCE(napi->list_owner) != -1);
 }
 
+struct skb_defer_node;
+void skb_defer_node_flush(struct skb_defer_node *sdn);
 void kick_defer_list_purge(unsigned int cpu);
 
 int dev_set_hwtstamp_phylib(struct net_device *dev,
diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c
index a320e264eaaf09..f6029f60c1dc96 100644
--- a/net/core/dev_ioctl.c
+++ b/net/core/dev_ioctl.c
@@ -267,7 +267,8 @@ int dev_get_hwtstamp_phylib(struct net_device *dev,
 		    hwprov->phydev)
 			return phy_hwtstamp_get(hwprov->phydev, cfg);
 
-		if (hwprov->source == HWTSTAMP_SOURCE_NETDEV)
+		if (hwprov->source == HWTSTAMP_SOURCE_NETDEV &&
+		    dev->netdev_ops->ndo_hwtstamp_get)
 			return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg);
 
 		return -EOPNOTSUPP;
@@ -276,19 +277,18 @@ int dev_get_hwtstamp_phylib(struct net_device *dev,
 	if (phy_is_default_hwtstamp(dev->phydev))
 		return phy_hwtstamp_get(dev->phydev, cfg);
 
+	if (!dev->netdev_ops->ndo_hwtstamp_get)
+		return -EOPNOTSUPP;
+
 	return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg);
 }
 
 static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr)
 {
-	const struct net_device_ops *ops = dev->netdev_ops;
 	struct kernel_hwtstamp_config kernel_cfg = {};
 	struct hwtstamp_config cfg;
 	int err;
 
-	if (!ops->ndo_hwtstamp_get)
-		return -EOPNOTSUPP;
-
 	if (!netif_device_present(dev))
 		return -ENODEV;
 
@@ -359,12 +359,18 @@ int dev_set_hwtstamp_phylib(struct net_device *dev,
 	cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV;
 
 	if (phy_ts && dev->see_all_hwtstamp_requests) {
+		if (!ops->ndo_hwtstamp_get)
+			return -EOPNOTSUPP;
+
 		err = ops->ndo_hwtstamp_get(dev, &old_cfg);
 		if (err)
 			return err;
 	}
 
 	if (!phy_ts || dev->see_all_hwtstamp_requests) {
+		if (!ops->ndo_hwtstamp_set)
+			return -EOPNOTSUPP;
+
 		err = ops->ndo_hwtstamp_set(dev, cfg, extack);
 		if (err) {
 			if (extack->_msg)
@@ -390,7 +396,6 @@ int dev_set_hwtstamp_phylib(struct net_device *dev,
 
 static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr)
 {
-	const struct net_device_ops *ops = dev->netdev_ops;
 	struct kernel_hwtstamp_config kernel_cfg = {};
 	struct netlink_ext_ack extack = {};
 	struct hwtstamp_config cfg;
@@ -413,9 +418,6 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr)
 		return err;
 	}
 
-	if (!ops->ndo_hwtstamp_set)
-		return -EOPNOTSUPP;
-
 	if (!netif_device_present(dev))
 		return -ENODEV;
 
@@ -441,15 +443,11 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr)
 int generic_hwtstamp_get_lower(struct net_device *dev,
 			       struct kernel_hwtstamp_config *kernel_cfg)
 {
-	const struct net_device_ops *ops = dev->netdev_ops;
 	int err;
 
 	if (!netif_device_present(dev))
 		return -ENODEV;
 
-	if (!ops->ndo_hwtstamp_get)
-		return -EOPNOTSUPP;
-
 	netdev_lock_ops(dev);
 	err = dev_get_hwtstamp_phylib(dev, kernel_cfg);
 	netdev_unlock_ops(dev);
@@ -462,15 +460,11 @@ int generic_hwtstamp_set_lower(struct net_device *dev,
 			       struct kernel_hwtstamp_config *kernel_cfg,
 			       struct netlink_ext_ack *extack)
 {
-	const struct net_device_ops *ops = dev->netdev_ops;
 	int err;
 
 	if (!netif_device_present(dev))
 		return -ENODEV;
 
-	if (!ops->ndo_hwtstamp_set)
-		return -EOPNOTSUPP;
-
 	netdev_lock_ops(dev);
 	err = dev_set_hwtstamp_phylib(dev, kernel_cfg, extack);
 	netdev_unlock_ops(dev);
diff --git a/net/core/devmem.c b/net/core/devmem.c
index f4d60654ce7fd2..71c83730d73d6e 100644
--- a/net/core/devmem.c
+++ b/net/core/devmem.c
@@ -8,7 +8,6 @@
  */
 
 #include <linux/dma-buf.h>
-#include <linux/genalloc.h>
 #include <linux/mm.h>
 #include <linux/netdevice.h>
 #include <linux/types.h>
@@ -18,11 +17,9 @@
 #include <net/page_pool/memory_provider.h>
 #include <net/sock.h>
 #include <net/tcp.h>
-#include <trace/events/page_pool.h>
 
 #include "devmem.h"
 #include "mp_dmabuf_devmem.h"
-#include "page_pool_priv.h"
 
 /* Device memory support */
 
@@ -30,23 +27,27 @@ static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
 
 static const struct memory_provider_ops dmabuf_devmem_ops;
 
-static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
-					       struct gen_pool_chunk *chunk,
-					       void *not_used)
+static struct net_iov *net_iov_free_pop(struct net_iov_area *area,
+					u32 *freelist, u32 *free_count)
 {
-	struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
+	u32 idx;
 
-	kvfree(owner->area.niovs);
-	kfree(owner);
+	if (unlikely(!*free_count))
+		return NULL;
+
+	idx = freelist[--(*free_count)];
+	return &area->niovs[idx];
 }
 
-static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
+static void net_iov_free_push(struct net_iov_area *area,
+			      u32 *freelist, u32 *free_count,
+			      struct net_iov *niov)
 {
-	struct dmabuf_genpool_chunk_owner *owner;
+	if (WARN_ON_ONCE(net_iov_owner(niov) != area ||
+			 *free_count >= area->num_niovs))
+		return;
 
-	owner = net_devmem_iov_to_chunk_owner(niov);
-	return owner->base_dma_addr +
-	       ((dma_addr_t)net_iov_idx(niov) << owner->binding->niov_shift);
+	freelist[(*free_count)++] = net_iov_idx(niov);
 }
 
 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref)
@@ -62,65 +63,52 @@ void __net_devmem_dmabuf_binding_free(struct work_struct *wq)
 {
 	struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
 
-	size_t size, avail;
-
-	gen_pool_for_each_chunk(binding->chunk_pool,
-				net_devmem_dmabuf_free_chunk_owner, NULL);
-
-	size = gen_pool_size(binding->chunk_pool);
-	avail = gen_pool_avail(binding->chunk_pool);
-
-	if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
-		  size, avail))
-		gen_pool_destroy(binding->chunk_pool);
+	if (binding->freelist)
+		WARN(binding->free_count != binding->area.num_niovs,
+		     "destroying dmabuf binding with outstanding net_iovs: total=%zu, free=%u",
+		     binding->area.num_niovs, binding->free_count);
 
+	kvfree(binding->area.niovs);
 	dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
 					  binding->direction);
 	dma_buf_detach(binding->dmabuf, binding->attachment);
 	dma_buf_put(binding->dmabuf);
 	xa_destroy(&binding->bound_rxqs);
 	percpu_ref_exit(&binding->ref);
+	kvfree(binding->freelist);
 	kvfree(binding->tx_vec);
 	kfree(binding);
 }
 
-struct net_iov *
-net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
+static unsigned int
+net_devmem_alloc_dmabuf_bulk(struct net_devmem_dmabuf_binding *binding,
+			     netmem_ref *netmems, unsigned int count)
 {
-	struct dmabuf_genpool_chunk_owner *owner;
-	unsigned long dma_addr;
 	struct net_iov *niov;
-	ssize_t offset;
-	ssize_t index;
+	unsigned int i;
 
-	dma_addr = gen_pool_alloc_owner(binding->chunk_pool,
-					1UL << binding->niov_shift,
-					(void **)&owner);
-	if (!dma_addr)
-		return NULL;
+	spin_lock_bh(&binding->freelist_lock);
 
-	offset = dma_addr - owner->base_dma_addr;
-	index = offset >> binding->niov_shift;
-	niov = &owner->area.niovs[index];
+	count = min(count, binding->free_count);
+	for (i = 0; i < count; i++) {
+		niov = net_iov_free_pop(&binding->area, binding->freelist,
+					&binding->free_count);
+		netmems[i] = net_iov_to_netmem(niov);
+	}
 
-	niov->desc.pp_magic = 0;
-	niov->desc.pp = NULL;
-	atomic_long_set(&niov->desc.pp_ref_count, 0);
+	spin_unlock_bh(&binding->freelist_lock);
 
-	return niov;
+	return count;
 }
 
 void net_devmem_free_dmabuf(struct net_iov *niov)
 {
 	struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
-	unsigned long dma_addr = net_devmem_get_dma_addr(niov);
-	size_t niov_size = 1UL << binding->niov_shift;
 
-	if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
-				       niov_size)))
-		return;
-
-	gen_pool_free(binding->chunk_pool, dma_addr, niov_size);
+	spin_lock_bh(&binding->freelist_lock);
+	net_iov_free_push(&binding->area, binding->freelist,
+			  &binding->free_count, niov);
+	spin_unlock_bh(&binding->freelist_lock);
 }
 
 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
@@ -198,8 +186,9 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 	static u32 id_alloc_next;
 	struct scatterlist *sg;
 	struct dma_buf *dmabuf;
-	unsigned int sg_idx, i;
-	unsigned long virtual;
+	unsigned int sg_idx;
+	size_t niov_idx;
+	size_t i;
 	int err;
 
 	if (!dma_dev) {
@@ -249,33 +238,52 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 		goto err_detach;
 	}
 
+	if (!IS_ALIGNED(dmabuf->size, niov_size)) {
+		err = -EINVAL;
+		NL_SET_ERR_MSG_FMT(extack,
+				   "dmabuf size %zu not aligned to niov size %zu",
+				   dmabuf->size, niov_size);
+		goto err_unmap;
+	}
+
+	if ((dmabuf->size >> niov_shift) > U32_MAX) {
+		err = -E2BIG;
+		NL_SET_ERR_MSG(extack, "dmabuf contains too many net_iovs");
+		goto err_unmap;
+	}
+
+	binding->area.base_virtual = 0;
+	binding->area.num_niovs = dmabuf->size >> niov_shift;
 	if (direction == DMA_TO_DEVICE) {
-		if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) {
-			err = -EINVAL;
-			NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE");
-			goto err_unmap;
-		}
 		binding->tx_vec = kvmalloc_objs(struct net_iov *,
-						dmabuf->size / PAGE_SIZE);
+						binding->area.num_niovs);
 		if (!binding->tx_vec) {
 			err = -ENOMEM;
 			goto err_unmap;
 		}
+	} else {
+		spin_lock_init(&binding->freelist_lock);
+		binding->freelist = kvmalloc_array(binding->area.num_niovs,
+						   sizeof(binding->freelist[0]),
+						   GFP_KERNEL);
+		if (!binding->freelist) {
+			err = -ENOMEM;
+			goto err_unmap;
+		}
 	}
-
-	binding->chunk_pool = gen_pool_create(niov_shift,
-					      dev_to_node(&dev->dev));
-	if (!binding->chunk_pool) {
+	binding->area.niovs = kvzalloc_objs(*binding->area.niovs,
+					    binding->area.num_niovs);
+	if (!binding->area.niovs) {
 		err = -ENOMEM;
-		goto err_tx_vec;
+		goto err_free_freelist;
 	}
 
-	virtual = 0;
+	niov_idx = 0;
 	for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
 		dma_addr_t dma_addr = sg_dma_address(sg);
-		struct dmabuf_genpool_chunk_owner *owner;
 		size_t len = sg_dma_len(sg);
 		struct net_iov *niov;
+		size_t nr_niovs;
 
 		if (!IS_ALIGNED(dma_addr, niov_size) ||
 		    !IS_ALIGNED(len, niov_size)) {
@@ -283,64 +291,43 @@ net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 			NL_SET_ERR_MSG_FMT(extack,
 					   "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu",
 					   &dma_addr, len, niov_size);
-			goto err_free_chunks;
-		}
-
-		owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
-				     dev_to_node(&dev->dev));
-		if (!owner) {
-			err = -ENOMEM;
-			goto err_free_chunks;
+			goto err_free_niovs;
 		}
 
-		owner->area.base_virtual = virtual;
-		owner->base_dma_addr = dma_addr;
-		owner->area.num_niovs = len >> niov_shift;
-		owner->binding = binding;
-
-		err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
-					 dma_addr, len, dev_to_node(&dev->dev),
-					 owner);
-		if (err) {
-			kfree(owner);
-			err = -EINVAL;
-			goto err_free_chunks;
-		}
-
-		owner->area.niovs = kvmalloc_objs(*owner->area.niovs,
-						  owner->area.num_niovs);
-		if (!owner->area.niovs) {
-			err = -ENOMEM;
-			goto err_free_chunks;
-		}
-
-		for (i = 0; i < owner->area.num_niovs; i++) {
-			niov = &owner->area.niovs[i];
-			net_iov_init(niov, &owner->area, NET_IOV_DMABUF);
-			page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
-						      net_devmem_get_dma_addr(niov));
+		nr_niovs = len >> niov_shift;
+		for (i = 0; i < nr_niovs; i++, niov_idx++) {
+			niov = &binding->area.niovs[niov_idx];
+			net_iov_init(niov, &binding->area, NET_IOV_DMABUF);
+			if (net_mp_niov_set_dma_addr(niov, dma_addr)) {
+				err = -EFAULT;
+				NL_SET_ERR_MSG(extack,
+					       "dmabuf DMA address cannot be represented");
+				goto err_free_niovs;
+			}
 			if (direction == DMA_TO_DEVICE)
-				binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
+				binding->tx_vec[niov_idx] = niov;
+			else
+				net_iov_free_push(&binding->area,
+						  binding->freelist,
+						  &binding->free_count, niov);
+			dma_addr += niov_size;
 		}
-
-		virtual += len;
 	}
 
 	err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
 			      binding, xa_limit_32b, &id_alloc_next,
 			      GFP_KERNEL);
 	if (err < 0)
-		goto err_free_chunks;
+		goto err_free_niovs;
 
 	list_add(&binding->list, &priv->bindings);
 
 	return binding;
 
-err_free_chunks:
-	gen_pool_for_each_chunk(binding->chunk_pool,
-				net_devmem_dmabuf_free_chunk_owner, NULL);
-	gen_pool_destroy(binding->chunk_pool);
-err_tx_vec:
+err_free_niovs:
+	kvfree(binding->area.niovs);
+err_free_freelist:
+	kvfree(binding->freelist);
 	kvfree(binding->tx_vec);
 err_unmap:
 	dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
@@ -471,20 +458,25 @@ int mp_dmabuf_devmem_init(struct page_pool *pool)
 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp)
 {
 	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
-	struct net_iov *niov;
-	netmem_ref netmem;
+	netmem_ref *netmems = pool->alloc.cache;
+	unsigned int allocated, i;
 
-	niov = net_devmem_alloc_dmabuf(binding);
-	if (!niov)
+	if (WARN_ON_ONCE(pool->alloc.count))
 		return 0;
 
-	netmem = net_iov_to_netmem(niov);
+	allocated = net_devmem_alloc_dmabuf_bulk(binding, netmems,
+						 PP_ALLOC_CACHE_REFILL);
+	if (unlikely(!allocated))
+		return 0;
 
-	page_pool_set_pp_info(pool, netmem);
+	for (i = 0; i < allocated; i++)
+		net_mp_niov_set_page_pool(pool,
+					  netmem_to_net_iov(netmems[i]));
 
-	pool->pages_state_hold_cnt++;
-	trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
-	return netmem;
+	/* Return the last one, the rest stay in the page_pool cache. */
+	allocated--;
+	pool->alloc.count = allocated;
+	return netmems[allocated];
 }
 
 void mp_dmabuf_devmem_destroy(struct page_pool *pool)
@@ -504,8 +496,7 @@ bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem)
 	if (WARN_ON_ONCE(refcount != 1))
 		return false;
 
-	page_pool_clear_pp_info(netmem);
-
+	net_mp_niov_clear_page_pool(netmem_to_net_iov(netmem));
 	net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
 
 	/* We don't want the page pool put_page()ing our net_iovs. */
diff --git a/net/core/devmem.h b/net/core/devmem.h
index 4a293a7d1149c8..28d6fa7bcb5795 100644
--- a/net/core/devmem.h
+++ b/net/core/devmem.h
@@ -16,6 +16,7 @@
 struct netlink_ext_ack;
 
 struct net_devmem_dmabuf_binding {
+	struct net_iov_area area;
 	struct dma_buf *dmabuf;
 	struct dma_buf_attachment *attachment;
 	struct sg_table *sgt;
@@ -26,7 +27,6 @@ struct net_devmem_dmabuf_binding {
 	 * dereferenced.
 	 */
 	void *vdev;
-	struct gen_pool *chunk_pool;
 	/* Protect dev */
 	struct mutex lock;
 
@@ -57,6 +57,10 @@ struct net_devmem_dmabuf_binding {
 	/* rxq's this binding is active on. */
 	struct xarray bound_rxqs;
 
+	spinlock_t freelist_lock ____cacheline_aligned_in_smp;
+	u32 free_count;
+	u32 *freelist;
+
 	/* ID of this binding. Globally unique to all bindings currently
 	 * active.
 	 */
@@ -77,19 +81,6 @@ struct net_devmem_dmabuf_binding {
 };
 
 #if defined(CONFIG_NET_DEVMEM)
-/* Owner of the dma-buf chunks inserted into the gen pool. Each scatterlist
- * entry from the dmabuf is inserted into the genpool as a chunk, and needs
- * this owner struct to keep track of some metadata necessary to create
- * allocations from this chunk.
- */
-struct dmabuf_genpool_chunk_owner {
-	struct net_iov_area area;
-	struct net_devmem_dmabuf_binding *binding;
-
-	/* dma_addr of the start of the chunk.  */
-	dma_addr_t base_dma_addr;
-};
-
 void __net_devmem_dmabuf_binding_free(struct work_struct *wq);
 struct net_devmem_dmabuf_binding *
 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
@@ -104,18 +95,11 @@ int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
 				    struct net_devmem_dmabuf_binding *binding,
 				    struct netlink_ext_ack *extack);
 
-static inline struct dmabuf_genpool_chunk_owner *
-net_devmem_iov_to_chunk_owner(const struct net_iov *niov)
-{
-	struct net_iov_area *owner = net_iov_owner(niov);
-
-	return container_of(owner, struct dmabuf_genpool_chunk_owner, area);
-}
-
 static inline struct net_devmem_dmabuf_binding *
 net_devmem_iov_binding(const struct net_iov *niov)
 {
-	return net_devmem_iov_to_chunk_owner(niov)->binding;
+	return container_of(net_iov_owner(niov),
+			    struct net_devmem_dmabuf_binding, area);
 }
 
 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov)
@@ -125,11 +109,11 @@ static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov)
 
 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov)
 {
-	struct dmabuf_genpool_chunk_owner *co =
-		net_devmem_iov_to_chunk_owner(niov);
+	struct net_devmem_dmabuf_binding *binding =
+		net_devmem_iov_binding(niov);
 
 	return net_iov_owner(niov)->base_virtual +
-	       ((unsigned long)net_iov_idx(niov) << co->binding->niov_shift);
+	       ((unsigned long)net_iov_idx(niov) << binding->niov_shift);
 }
 
 static inline bool
@@ -147,8 +131,6 @@ net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding)
 void net_devmem_get_net_iov(struct net_iov *niov);
 void net_devmem_put_net_iov(struct net_iov *niov);
 
-struct net_iov *
-net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding);
 void net_devmem_free_dmabuf(struct net_iov *ppiov);
 
 
@@ -205,12 +187,6 @@ net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
 	return -EOPNOTSUPP;
 }
 
-static inline struct net_iov *
-net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
-{
-	return NULL;
-}
-
 static inline void net_devmem_free_dmabuf(struct net_iov *ppiov)
 {
 }
diff --git a/net/core/filter.c b/net/core/filter.c
index 61940e75355233..ba536be2915fb9 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3961,12 +3961,6 @@ static u32 __bpf_skb_min_len(const struct sk_buff *skb)
 		if (offset > 0)
 			min_len = offset;
 	}
-	if (skb->ip_summed == CHECKSUM_PARTIAL) {
-		offset = skb_checksum_start_offset(skb) +
-			 skb->csum_offset + sizeof(__sum16);
-		if (offset > 0)
-			min_len = offset;
-	}
 	return min_len;
 }
 
@@ -3983,6 +3977,11 @@ static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len)
 
 static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len)
 {
+	if (skb->ip_summed == CHECKSUM_PARTIAL &&
+	    new_len < skb_checksum_start_offset(skb) + skb->csum_offset +
+		      sizeof(__sum16))
+		skb->ip_summed = CHECKSUM_NONE;
+
 	return __skb_trim_rcsum(skb, new_len);
 }
 
@@ -8053,17 +8052,14 @@ static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend,
 	return ERR_PTR(-ENOMSG);
 }
 
-BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
-	   void *, search_res, u32, len, u64, flags)
+int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock,
+				void *search_res, u32 len, u64 flags)
 {
 	bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN;
 	const u8 *op, *opend, *magic, *search = search_res;
 	u8 search_kind, search_len, copy_len, magic_len;
 	int ret;
 
-	if (!is_locked_tcp_sock_ops(bpf_sock))
-		return -EOPNOTSUPP;
-
 	/* 2 byte is the minimal option len except TCPOPT_NOP and
 	 * TCPOPT_EOL which are useless for the bpf prog to learn
 	 * and this helper disallow loading them also.
@@ -8124,6 +8120,15 @@ BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
 	return ret;
 }
 
+BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
+	   void *, search_res, u32, len, u64, flags)
+{
+	if (!is_locked_tcp_sock_ops(bpf_sock))
+		return -EOPNOTSUPP;
+
+	return __bpf_sock_ops_load_hdr_opt(bpf_sock, search_res, len, flags);
+}
+
 static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = {
 	.func		= bpf_sock_ops_load_hdr_opt,
 	.gpl_only	= false,
@@ -8134,17 +8139,14 @@ static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = {
 	.arg4_type	= ARG_ANYTHING,
 };
 
-BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
-	   const void *, from, u32, len, u64, flags)
+int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock,
+				 const void *from, u32 len, u64 flags)
 {
 	u8 new_kind, new_kind_len, magic_len = 0, *opend;
 	const u8 *op, *new_op, *magic = NULL;
 	struct sk_buff *skb;
 	bool eol;
 
-	if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB)
-		return -EPERM;
-
 	if (len < 2 || flags)
 		return -EINVAL;
 
@@ -8202,6 +8204,15 @@ BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
 	return 0;
 }
 
+BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
+	   const void *, from, u32, len, u64, flags)
+{
+	if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB)
+		return -EPERM;
+
+	return __bpf_sock_ops_store_hdr_opt(bpf_sock, from, len, flags);
+}
+
 static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = {
 	.func		= bpf_sock_ops_store_hdr_opt,
 	.gpl_only	= false,
@@ -9045,6 +9056,8 @@ static const struct bpf_func_proto *
 lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
 	switch (func_id) {
+	case BPF_FUNC_skb_pull_data:
+		return NULL;
 #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
 	case BPF_FUNC_lwt_seg6_store_bytes:
 		return &bpf_lwt_seg6_store_bytes_proto;
@@ -10565,11 +10578,12 @@ u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
 				       target_size));
 		*insn++ = BPF_JMP_IMM(BPF_JNE, si->dst_reg, NO_QUEUE_MAPPING,
 				      1);
-		*insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
+		*insn++ = BPF_MOV32_IMM(si->dst_reg, -1);
 #else
-		*insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
-		*target_size = 2;
+		*insn++ = BPF_MOV32_IMM(si->dst_reg, -1);
 #endif
+		*target_size = sizeof_field(struct bpf_sock, rx_queue_mapping);
+
 		break;
 	}
 
@@ -11105,18 +11119,7 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type,
 		break;
 
 	case offsetof(struct bpf_sock_ops, rtt_min):
-		BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
-			     sizeof(struct minmax));
-		BUILD_BUG_ON(sizeof(struct minmax) <
-			     sizeof(struct minmax_sample));
-
-		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
-						struct bpf_sock_ops_kern, sk),
-				      si->dst_reg, si->src_reg,
-				      offsetof(struct bpf_sock_ops_kern, sk));
-		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
-				      offsetof(struct tcp_sock, rtt_min) +
-				      sizeof_field(struct minmax_sample, t));
+		SOCK_OPS_GET_FIELD(rtt_min, rtt_min.s[0].v, struct tcp_sock);
 		break;
 
 	case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags):
@@ -12887,6 +12890,7 @@ static int __init bpf_kfunc_init(void)
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_NETFILTER, &bpf_kfunc_set_skb);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_kfunc_set_skb);
+	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_kfunc_set_skb);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb_meta);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb_meta);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp);
@@ -12912,8 +12916,9 @@ __bpf_kfunc_start_defs();
  * @sock: Pointer to socket to be destroyed
  *
  * Return:
- * On error, may return EPROTONOSUPPORT, EINVAL.
- * EPROTONOSUPPORT if protocol specific destroy handler is not supported.
+ * On error, may return EOPNOTSUPP, or whatever the protocol specific
+ * destroy handler returns.
+ * EOPNOTSUPP if protocol specific destroy handler is not supported.
  * 0 otherwise
  */
 __bpf_kfunc int bpf_sock_destroy(struct sock_common *sock)
@@ -12925,8 +12930,12 @@ __bpf_kfunc int bpf_sock_destroy(struct sock_common *sock)
 	 * Supporting protocols will need to acquire sock lock in the BPF context
 	 * prior to invoking this kfunc.
 	 */
-	if (!sk->sk_prot->diag_destroy || (sk->sk_protocol != IPPROTO_TCP &&
-					   sk->sk_protocol != IPPROTO_UDP))
+	if (!sk->sk_prot->diag_destroy)
+		return -EOPNOTSUPP;
+
+	if (sk_fullsock(sk) &&
+	    sk->sk_protocol != IPPROTO_TCP &&
+	    sk->sk_protocol != IPPROTO_UDP)
 		return -EOPNOTSUPP;
 
 	return sk->sk_prot->diag_destroy(sk, ECONNABORTED);
diff --git a/net/core/gro_cells.c b/net/core/gro_cells.c
index d8c0a286712019..1d64207bf960f4 100644
--- a/net/core/gro_cells.c
+++ b/net/core/gro_cells.c
@@ -14,11 +14,13 @@ struct gro_cell {
 int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb)
 {
 	struct net_device *dev = skb->dev;
+	enum skb_drop_reason reason;
 	bool have_bh_lock = false;
 	struct gro_cell *cell;
 	int res;
 
 	rcu_read_lock();
+	reason = SKB_DROP_REASON_DEV_READY;
 	if (unlikely(!(dev->flags & IFF_UP)))
 		goto drop;
 
@@ -34,9 +36,10 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb)
 	cell = this_cpu_ptr(gcells->cells);
 
 	if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(net_hotdata.max_backlog)) {
+		reason = SKB_DROP_REASON_CPU_BACKLOG;
 drop:
 		dev_core_stats_rx_dropped_inc(dev);
-		kfree_skb(skb);
+		kfree_skb_reason(skb, reason);
 		res = NET_RX_DROP;
 		goto unlock;
 	}
diff --git a/net/core/gso.c b/net/core/gso.c
index bcd156372f4df0..e96ef635006487 100644
--- a/net/core/gso.c
+++ b/net/core/gso.c
@@ -117,6 +117,7 @@ struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
 
 	SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
 	SKB_GSO_CB(skb)->encap_level = 0;
+	SKB_GSO_CB(skb)->recursion_counter = 0;
 
 	skb_reset_mac_header(skb);
 	skb_reset_mac_len(skb);
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 7448320f7ad52c..6ed8eb07514698 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -55,6 +55,23 @@ static void neigh_notify(struct neighbour *n, int type, int flags, u32 pid);
 static void __neigh_notify(struct neighbour *n, int type, int flags, u32 pid);
 static void pneigh_ifdown(struct neigh_table *tbl, struct net_device *dev,
 			  bool skip_perm);
+static void neigh_table_free(struct neigh_table *tbl);
+
+static void neigh_table_get(struct neigh_table *tbl)
+{
+	refcount_inc(&tbl->entries);
+}
+
+static void neigh_table_put(struct neigh_table *tbl)
+{
+	if (refcount_dec_and_test(&tbl->entries))
+		neigh_table_free(tbl);
+}
+
+static int neigh_table_entries(struct neigh_table *tbl)
+{
+	return refcount_read(&tbl->entries) - 1;
+}
 
 #ifdef CONFIG_PROC_FS
 static const struct seq_operations neigh_stat_seq_ops;
@@ -131,10 +148,11 @@ unsigned long neigh_rand_reach_time(unsigned long base)
 {
 	return base ? get_random_u32_below(base) + (base >> 1) : 0;
 }
-EXPORT_SYMBOL(neigh_rand_reach_time);
 
 static void neigh_mark_dead(struct neighbour *n)
 {
+	lockdep_assert_held(&n->tbl->lock);
+
 	n->dead = 1;
 	if (!list_empty(&n->gc_list)) {
 		list_del_init(&n->gc_list);
@@ -148,11 +166,6 @@ static void neigh_update_gc_list(struct neighbour *n)
 {
 	bool on_gc_list, exempt_from_gc;
 
-	spin_lock_bh(&n->tbl->lock);
-	write_lock(&n->lock);
-	if (n->dead)
-		goto out;
-
 	/* remove from the gc list if new state is permanent or if neighbor is
 	 * externally learned / validated; otherwise entry should be on the gc
 	 * list
@@ -169,20 +182,12 @@ static void neigh_update_gc_list(struct neighbour *n)
 		list_add_tail(&n->gc_list, &n->tbl->gc_list);
 		atomic_inc(&n->tbl->gc_entries);
 	}
-out:
-	write_unlock(&n->lock);
-	spin_unlock_bh(&n->tbl->lock);
 }
 
 static void neigh_update_managed_list(struct neighbour *n)
 {
 	bool on_managed_list, add_to_managed;
 
-	spin_lock_bh(&n->tbl->lock);
-	write_lock(&n->lock);
-	if (n->dead)
-		goto out;
-
 	add_to_managed = n->flags & NTF_MANAGED;
 	on_managed_list = !list_empty(&n->managed_list);
 
@@ -190,9 +195,6 @@ static void neigh_update_managed_list(struct neighbour *n)
 		list_del_init(&n->managed_list);
 	else if (add_to_managed && !on_managed_list)
 		list_add_tail(&n->managed_list, &n->tbl->managed_list);
-out:
-	write_unlock(&n->lock);
-	spin_unlock_bh(&n->tbl->lock);
 }
 
 static void neigh_update_flags(struct neighbour *neigh, u32 flags, int *notify,
@@ -349,8 +351,7 @@ static void neigh_parms_qlen_dec(struct net_device *dev, int family)
 	rcu_read_unlock();
 }
 
-static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net,
-			       int family)
+static void pneigh_queue_purge(struct sk_buff_head *list, int family)
 {
 	struct sk_buff_head tmp;
 	unsigned long flags;
@@ -361,13 +362,11 @@ static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net,
 	skb = skb_peek(list);
 	while (skb != NULL) {
 		struct sk_buff *skb_next = skb_peek_next(skb, list);
-		struct net_device *dev = skb->dev;
 
-		if (net == NULL || net_eq(dev_net(dev), net)) {
-			neigh_parms_qlen_dec(dev, family);
-			__skb_unlink(skb, list);
-			__skb_queue_tail(&tmp, skb);
-		}
+		neigh_parms_qlen_dec(skb->dev, family);
+		__skb_unlink(skb, list);
+		__skb_queue_tail(&tmp, skb);
+
 		skb = skb_next;
 	}
 	spin_unlock_irqrestore(&list->lock, flags);
@@ -471,8 +470,7 @@ static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev,
 	spin_unlock_bh(&tbl->lock);
 
 	pneigh_ifdown(tbl, dev, skip_perm);
-	pneigh_queue_purge(&tbl->proxy_queue, dev ? dev_net(dev) : NULL,
-			   tbl->family);
+	pneigh_queue_purge(&tbl->proxy_queue, tbl->family);
 	if (skb_queue_empty_lockless(&tbl->proxy_queue))
 		timer_delete_sync(&tbl->proxy_timer);
 	return 0;
@@ -537,7 +535,7 @@ static struct neighbour *neigh_alloc(struct neigh_table *tbl,
 	INIT_LIST_HEAD(&n->gc_list);
 	INIT_LIST_HEAD(&n->managed_list);
 
-	atomic_inc(&tbl->entries);
+	neigh_table_get(tbl);
 out:
 	return n;
 
@@ -687,7 +685,7 @@ ___neigh_create(struct neigh_table *tbl, const void *pkey,
 	nht = rcu_dereference_protected(tbl->nht,
 					lockdep_is_held(&tbl->lock));
 
-	if (atomic_read(&tbl->entries) > (1 << nht->hash_shift))
+	if (neigh_table_entries(tbl) > (1 << nht->hash_shift))
 		nht = neigh_hash_grow(tbl, nht->hash_shift + 1);
 
 	hash_val = tbl->hash(n->primary_key, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
@@ -752,8 +750,7 @@ static u32 pneigh_hash(const void *pkey, unsigned int key_len)
 }
 
 struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl,
-				   struct net *net, const void *pkey,
-				   struct net_device *dev)
+				   const void *pkey, struct net_device *dev)
 {
 	struct pneigh_entry *n;
 	unsigned int key_len;
@@ -766,7 +763,6 @@ struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl,
 
 	while (n) {
 		if (!memcmp(n->key, pkey, key_len) &&
-		    net_eq(pneigh_net(n), net) &&
 		    (n->dev == dev || !n->dev))
 			return n;
 
@@ -776,7 +772,7 @@ struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl,
 	return NULL;
 }
 
-int pneigh_create(struct neigh_table *tbl, struct net *net,
+int pneigh_create(struct neigh_table *tbl,
 		  const void *pkey, struct net_device *dev,
 		  u32 flags, u8 protocol, bool permanent)
 {
@@ -787,7 +783,7 @@ int pneigh_create(struct neigh_table *tbl, struct net *net,
 
 	mutex_lock(&tbl->phash_lock);
 
-	n = pneigh_lookup(tbl, net, pkey, dev);
+	n = pneigh_lookup(tbl, pkey, dev);
 	if (n)
 		goto update;
 
@@ -798,7 +794,6 @@ int pneigh_create(struct neigh_table *tbl, struct net *net,
 		goto out;
 	}
 
-	write_pnet(&n->net, net);
 	memcpy(n->key, pkey, key_len);
 	n->dev = dev;
 	netdev_hold(dev, &n->dev_tracker, GFP_KERNEL);
@@ -831,7 +826,7 @@ static void pneigh_destroy(struct rcu_head *rcu)
 	kfree(n);
 }
 
-int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey,
+int pneigh_delete(struct neigh_table *tbl, const void *pkey,
 		  struct net_device *dev)
 {
 	struct pneigh_entry *n, __rcu **np;
@@ -846,8 +841,7 @@ int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey,
 	for (np = &tbl->phash_buckets[hash_val];
 	     (n = rcu_dereference_protected(*np, 1)) != NULL;
 	     np = &n->next) {
-		if (!memcmp(n->key, pkey, key_len) && n->dev == dev &&
-		    net_eq(pneigh_net(n), net)) {
+		if (!memcmp(n->key, pkey, key_len) && n->dev == dev) {
 			rcu_assign_pointer(*np, n->next);
 
 			mutex_unlock(&tbl->phash_lock);
@@ -939,7 +933,7 @@ void neigh_destroy(struct neighbour *neigh)
 
 	neigh_dbg(2, "neigh %p is destroyed\n", neigh);
 
-	atomic_dec(&neigh->tbl->entries);
+	neigh_table_put(neigh->tbl);
 	kfree_rcu(neigh, rcu);
 }
 EXPORT_SYMBOL(neigh_destroy);
@@ -994,7 +988,7 @@ static void neigh_periodic_work(struct work_struct *work)
 			neigh_set_reach_time(p);
 	}
 
-	if (atomic_read(&tbl->entries) < READ_ONCE(tbl->gc_thresh1))
+	if (neigh_table_entries(tbl) < READ_ONCE(tbl->gc_thresh1))
 		goto out;
 
 	for (i = 0 ; i < (1 << nht->hash_shift); i++) {
@@ -1523,10 +1517,23 @@ static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
 
 	write_unlock_bh(&neigh->lock);
 
-	if (((new ^ old) & NUD_PERMANENT) || gc_update)
-		neigh_update_gc_list(neigh);
-	if (managed_update)
-		neigh_update_managed_list(neigh);
+	gc_update |= !!((new ^ old) & NUD_PERMANENT);
+	if (gc_update || managed_update) {
+		spin_lock_bh(&neigh->tbl->lock);
+
+		if (!neigh->dead) {
+			write_lock(&neigh->lock);
+
+			if (gc_update)
+				neigh_update_gc_list(neigh);
+			if (managed_update)
+				neigh_update_managed_list(neigh);
+
+			write_unlock(&neigh->lock);
+		}
+
+		spin_unlock_bh(&neigh->tbl->lock);
+	}
 
 	if (notify)
 		call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);
@@ -1540,7 +1547,6 @@ int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,
 {
 	return __neigh_update(neigh, lladdr, new, flags, nlmsg_pid, NULL);
 }
-EXPORT_SYMBOL(neigh_update);
 
 /* Update the neigh to listen temporarily for probe responses, even if it is
  * in a NUD_FAILED state. The caller has to hold neigh->lock for writing.
@@ -1558,7 +1564,6 @@ void __neigh_set_probe_once(struct neighbour *neigh)
 			jiffies + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
 				      HZ/100));
 }
-EXPORT_SYMBOL(__neigh_set_probe_once);
 
 struct neighbour *neigh_event_ns(struct neigh_table *tbl,
 				 u8 *lladdr, void *saddr,
@@ -1571,7 +1576,6 @@ struct neighbour *neigh_event_ns(struct neigh_table *tbl,
 			     NEIGH_UPDATE_F_OVERRIDE, 0);
 	return neigh;
 }
-EXPORT_SYMBOL(neigh_event_ns);
 
 /* called with read_lock_bh(&n->lock); */
 static void neigh_hh_init(struct neighbour *n)
@@ -1624,7 +1628,6 @@ int neigh_resolve_output(struct neighbour *neigh, struct sk_buff *skb)
 	kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_HH_FILLFAIL);
 	goto out;
 }
-EXPORT_SYMBOL(neigh_resolve_output);
 
 /* As fast as possible without hh cache */
 
@@ -1741,16 +1744,15 @@ void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p,
 	mod_timer(&tbl->proxy_timer, sched_next);
 	spin_unlock(&tbl->proxy_queue.lock);
 }
-EXPORT_SYMBOL(pneigh_enqueue);
 
 static inline struct neigh_parms *lookup_neigh_parms(struct neigh_table *tbl,
-						      struct net *net, int ifindex)
+						     int ifindex)
 {
 	struct neigh_parms *p;
 
 	list_for_each_entry(p, &tbl->parms_list, list) {
-		if ((p->dev && p->dev->ifindex == ifindex && net_eq(neigh_parms_net(p), net)) ||
-		    (!p->dev && !ifindex && net_eq(net, &init_net)))
+		if ((p->dev && p->dev->ifindex == ifindex) ||
+		    (!p->dev && !ifindex))
 			return p;
 	}
 
@@ -1789,7 +1791,6 @@ struct neigh_parms *neigh_parms_alloc(struct net_device *dev,
 	}
 	return p;
 }
-EXPORT_SYMBOL(neigh_parms_alloc);
 
 static void neigh_rcu_free_parms(struct rcu_head *head)
 {
@@ -1812,113 +1813,171 @@ void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms)
 	netdev_put(parms->dev, &parms->dev_tracker);
 	call_rcu(&parms->rcu_head, neigh_rcu_free_parms);
 }
-EXPORT_SYMBOL(neigh_parms_release);
 
 static struct lock_class_key neigh_table_proxy_queue_class;
 
-static struct neigh_table __rcu *neigh_tables[NEIGH_NR_TABLES] __read_mostly;
-
-void neigh_table_init(int index, struct neigh_table *tbl)
+static int neigh_table_init(struct net *net, struct neigh_table *tbl)
 {
 	unsigned long now = jiffies;
 	unsigned long phsize;
 
-	INIT_LIST_HEAD(&tbl->parms_list);
-	INIT_LIST_HEAD(&tbl->gc_list);
-	INIT_LIST_HEAD(&tbl->managed_list);
+	RCU_INIT_POINTER(tbl->nht, neigh_hash_alloc(3));
+	if (!tbl->nht)
+		goto err_hash;
 
-	list_add(&tbl->parms.list, &tbl->parms_list);
-	write_pnet(&tbl->parms.net, &init_net);
-	refcount_set(&tbl->parms.refcnt, 1);
-	neigh_set_reach_time(&tbl->parms);
-	tbl->parms.qlen = 0;
+	phsize = (PNEIGH_HASHMASK + 1) * sizeof(struct pneigh_entry *);
+	tbl->phash_buckets = kzalloc(phsize, GFP_KERNEL);
+	if (!tbl->phash_buckets)
+		goto err_phash;
+
+	tbl->entry_size = ALIGN(offsetof(struct neighbour, primary_key) +
+				tbl->key_len, NEIGH_PRIV_ALIGN);
 
 	tbl->stats = alloc_percpu(struct neigh_statistics);
 	if (!tbl->stats)
-		panic("cannot create neighbour cache statistics");
+		goto err_stats;
 
 #ifdef CONFIG_PROC_FS
-	if (!proc_create_seq_data(tbl->id, 0, init_net.proc_net_stat,
-			      &neigh_stat_seq_ops, tbl))
-		panic("cannot create neighbour proc dir entry");
+	if (!proc_create_net_data(tbl->id, 0, net->proc_net_stat,
+				  &neigh_stat_seq_ops,
+				  sizeof(struct seq_net_private), tbl))
+		goto err_proc;
 #endif
 
-	RCU_INIT_POINTER(tbl->nht, neigh_hash_alloc(3));
-
-	phsize = (PNEIGH_HASHMASK + 1) * sizeof(struct pneigh_entry *);
-	tbl->phash_buckets = kzalloc(phsize, GFP_KERNEL);
-
-	if (!tbl->nht || !tbl->phash_buckets)
-		panic("cannot allocate neighbour cache hashes");
-
-	if (!tbl->entry_size)
-		tbl->entry_size = ALIGN(offsetof(struct neighbour, primary_key) +
-					tbl->key_len, NEIGH_PRIV_ALIGN);
-	else
-		WARN_ON(tbl->entry_size % NEIGH_PRIV_ALIGN);
+	tbl->parms.tbl = tbl;
+	tbl->parms.qlen = 0;
+	INIT_LIST_HEAD(&tbl->parms_list);
+	list_add(&tbl->parms.list, &tbl->parms_list);
+	write_pnet(&tbl->parms.net, net);
+	refcount_set(&tbl->parms.refcnt, 1);
+	neigh_set_reach_time(&tbl->parms);
+	tbl->last_flush = now;
+	tbl->last_rand	= now + tbl->parms.reachable_time * 20;
 
+	refcount_set(&tbl->entries, 1);
 	spin_lock_init(&tbl->lock);
 	mutex_init(&tbl->phash_lock);
+	skb_queue_head_init_class(&tbl->proxy_queue,
+				  &neigh_table_proxy_queue_class);
+	timer_setup(&tbl->proxy_timer, neigh_proxy_process, 0);
 
+	INIT_LIST_HEAD(&tbl->gc_list);
 	INIT_DEFERRABLE_WORK(&tbl->gc_work, neigh_periodic_work);
 	queue_delayed_work(system_power_efficient_wq, &tbl->gc_work,
-			tbl->parms.reachable_time);
+			   tbl->parms.reachable_time);
+
+	INIT_LIST_HEAD(&tbl->managed_list);
 	INIT_DEFERRABLE_WORK(&tbl->managed_work, neigh_managed_work);
 	queue_delayed_work(system_power_efficient_wq, &tbl->managed_work, 0);
 
-	timer_setup(&tbl->proxy_timer, neigh_proxy_process, 0);
-	skb_queue_head_init_class(&tbl->proxy_queue,
-			&neigh_table_proxy_queue_class);
-
-	tbl->last_flush = now;
-	tbl->last_rand	= now + tbl->parms.reachable_time * 20;
+	return 0;
 
-	rcu_assign_pointer(neigh_tables[index], tbl);
+#ifdef CONFIG_PROC_FS
+err_proc:
+	free_percpu(tbl->stats);
+#endif
+err_stats:
+	kfree(tbl->phash_buckets);
+err_phash:
+	neigh_hash_free_rcu(&rcu_dereference_protected(tbl->nht, 1)->rcu);
+err_hash:
+	return -ENOMEM;
 }
 
-/*
- * Only called from ndisc_cleanup(), which means this is dead code
- * because we no longer can unload IPv6 module.
- */
-int neigh_table_clear(int index, struct neigh_table *tbl)
+static void neigh_table_free(struct neigh_table *tbl)
 {
-	RCU_INIT_POINTER(neigh_tables[index], NULL);
-	synchronize_rcu();
+	struct neigh_hash_table *nht;
+
+	free_percpu(tbl->stats);
+	tbl->stats = NULL;
+
+	kfree(tbl->phash_buckets);
+	tbl->phash_buckets = NULL;
 
-	/* It is not clean... Fix it to unload IPv6 module safely */
+	nht = rcu_dereference_protected(tbl->nht, 1);
+	tbl->nht = NULL;
+	neigh_hash_free_rcu(&nht->rcu);
+
+	kfree(tbl);
+}
+
+static void neigh_table_clear(struct net *net, struct neigh_table *tbl)
+{
 	cancel_delayed_work_sync(&tbl->managed_work);
 	cancel_delayed_work_sync(&tbl->gc_work);
-	timer_delete_sync(&tbl->proxy_timer);
-	pneigh_queue_purge(&tbl->proxy_queue, NULL, tbl->family);
+	timer_shutdown_sync(&tbl->proxy_timer);
+
 	neigh_ifdown(tbl, NULL);
-	if (atomic_read(&tbl->entries))
-		pr_crit("neighbour leakage\n");
+	remove_proc_entry(tbl->id, net->proc_net_stat);
 
-	call_rcu(&rcu_dereference_protected(tbl->nht, 1)->rcu,
-		 neigh_hash_free_rcu);
-	tbl->nht = NULL;
+	neigh_table_put(tbl);
+}
 
-	kfree(tbl->phash_buckets);
-	tbl->phash_buckets = NULL;
+int sysctl_neigh_inherit_init_net __read_mostly = 1;
 
-	remove_proc_entry(tbl->id, init_net.proc_net_stat);
+static void neigh_table_inherit(struct net *net, struct neigh_table *tbl,
+				int index)
+{
+	const struct neigh_table *init_tbl = init_net.neigh_tables[index];
+	bool inherit = READ_ONCE(sysctl_neigh_inherit_init_net);
+	int i;
 
-	free_percpu(tbl->stats);
-	tbl->stats = NULL;
+	if (net_eq(net, &init_net) || !inherit)
+		return;
+
+	tbl->gc_interval = READ_ONCE(init_tbl->gc_interval);
+	tbl->gc_thresh1 = READ_ONCE(init_tbl->gc_thresh1);
+	tbl->gc_thresh2 = READ_ONCE(init_tbl->gc_thresh2);
+	tbl->gc_thresh3 = READ_ONCE(init_tbl->gc_thresh3);
+
+	for (i = 0; i < NEIGH_VAR_DATA_MAX; i++)
+		tbl->parms.data[i] = READ_ONCE(init_tbl->parms.data[i]);
+}
+
+int neigh_table_register(struct net *net, struct neigh_table *tbl, int index)
+{
+	int err;
+
+	tbl = kmemdup(tbl, sizeof(*tbl), GFP_KERNEL);
+	if (!tbl) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	neigh_table_inherit(net, tbl, index);
+
+	err = neigh_table_init(net, tbl);
+	if (err)
+		goto free_table;
+
+	net->neigh_tables[index] = tbl;
 
 	return 0;
+
+free_table:
+	kfree(tbl);
+err:
+	return err;
+}
+
+void neigh_table_unregister(struct net *net, int index)
+{
+	struct neigh_table *tbl = net->neigh_tables[index];
+
+	net->neigh_tables[index] = NULL;
+	neigh_table_clear(net, tbl);
 }
 
-static struct neigh_table *neigh_find_table(int family)
+static struct neigh_table *neigh_find_table(struct net *net, int family)
 {
 	struct neigh_table *tbl = NULL;
 
 	switch (family) {
 	case AF_INET:
-		tbl = rcu_dereference_rtnl(neigh_tables[NEIGH_ARP_TABLE]);
+		tbl = net->neigh_tables[NEIGH_ARP_TABLE];
 		break;
 	case AF_INET6:
-		tbl = rcu_dereference_rtnl(neigh_tables[NEIGH_ND_TABLE]);
+		tbl = net->neigh_tables[NEIGH_ND_TABLE];
 		break;
 	}
 
@@ -1972,7 +2031,7 @@ static int neigh_delete(struct sk_buff *skb, struct nlmsghdr *nlh,
 		}
 	}
 
-	tbl = neigh_find_table(ndm->ndm_family);
+	tbl = neigh_find_table(net, ndm->ndm_family);
 	if (tbl == NULL)
 		return -EAFNOSUPPORT;
 
@@ -1982,7 +2041,7 @@ static int neigh_delete(struct sk_buff *skb, struct nlmsghdr *nlh,
 	}
 
 	if (ndm->ndm_flags & NTF_PROXY) {
-		err = pneigh_delete(tbl, net, nla_data(dst_attr), dev);
+		err = pneigh_delete(tbl, nla_data(dst_attr), dev);
 		goto out;
 	}
 
@@ -2058,7 +2117,7 @@ static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh,
 		}
 	}
 
-	tbl = neigh_find_table(ndm->ndm_family);
+	tbl = neigh_find_table(net, ndm->ndm_family);
 	if (tbl == NULL)
 		return -EAFNOSUPPORT;
 
@@ -2078,7 +2137,7 @@ static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh,
 			goto out;
 		}
 
-		err = pneigh_create(tbl, net, dst, dev, ndm_flags, protocol,
+		err = pneigh_create(tbl, dst, dev, ndm_flags, protocol,
 				    !!(ndm->ndm_state & NUD_PERMANENT));
 		goto out;
 	}
@@ -2267,7 +2326,7 @@ static int neightbl_fill_info(struct sk_buff *skb, struct neigh_table *tbl,
 		struct ndt_config ndc = {
 			.ndtc_key_len		= tbl->key_len,
 			.ndtc_entry_size	= tbl->entry_size,
-			.ndtc_entries		= atomic_read(&tbl->entries),
+			.ndtc_entries		= neigh_table_entries(tbl),
 			.ndtc_last_flush	= jiffies_to_msecs(flush_delta),
 			.ndtc_last_rand		= jiffies_to_msecs(rand_delta),
 			.ndtc_proxy_qlen	= READ_ONCE(tbl->proxy_queue.qlen),
@@ -2408,10 +2467,8 @@ static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	ndtmsg = nlmsg_data(nlh);
 
-	rcu_read_lock();
-
 	for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
-		tbl = rcu_dereference(neigh_tables[tidx]);
+		tbl = net->neigh_tables[tidx];
 		if (!tbl)
 			continue;
 
@@ -2425,7 +2482,6 @@ static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
 	}
 
 	if (!found) {
-		rcu_read_unlock();
 		err = -ENOENT;
 		goto errout;
 	}
@@ -2450,8 +2506,8 @@ static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
 		if (tbp[NDTPA_IFINDEX])
 			ifindex = nla_get_u32(tbp[NDTPA_IFINDEX]);
 
-		p = lookup_neigh_parms(tbl, net, ifindex);
-		if (p == NULL) {
+		p = lookup_neigh_parms(tbl, ifindex);
+		if (!p) {
 			err = -ENOENT;
 			goto errout_tbl_lock;
 		}
@@ -2532,12 +2588,6 @@ static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
 		}
 	}
 
-	err = -ENOENT;
-	if ((tb[NDTA_THRESH1] || tb[NDTA_THRESH2] ||
-	     tb[NDTA_THRESH3] || tb[NDTA_GC_INTERVAL]) &&
-	    !net_eq(net, &init_net))
-		goto errout_tbl_lock;
-
 	if (tb[NDTA_THRESH1])
 		WRITE_ONCE(tbl->gc_thresh1, nla_get_u32(tb[NDTA_THRESH1]));
 
@@ -2554,7 +2604,6 @@ static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
 
 errout_tbl_lock:
 	spin_unlock_bh(&tbl->lock);
-	rcu_read_unlock();
 errout:
 	return err;
 }
@@ -2607,7 +2656,7 @@ static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
 	for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
 		struct neigh_parms *p;
 
-		tbl = rcu_dereference(neigh_tables[tidx]);
+		tbl = net->neigh_tables[tidx];
 		if (!tbl)
 			continue;
 
@@ -2623,9 +2672,6 @@ static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
 		default_skip = 1;
 
 		list_for_each_entry_rcu(p, &tbl->parms_list, list) {
-			if (!net_eq(neigh_parms_net(p), net))
-				continue;
-
 			if (!p->dev || p->dev == blackhole_netdev)
 				continue;
 
@@ -2809,12 +2855,11 @@ static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
 			    struct netlink_callback *cb,
 			    struct neigh_dump_filter *filter)
 {
-	struct net *net = sock_net(skb->sk);
-	struct neighbour *n;
-	int err = 0, h, s_h = cb->args[1];
 	int idx, s_idx = idx = cb->args[2];
-	struct neigh_hash_table *nht;
+	int err = 0, h, s_h = cb->args[1];
 	unsigned int flags = NLM_F_MULTI;
+	struct neigh_hash_table *nht;
+	struct neighbour *n;
 
 	if (filter->dev_idx || filter->master_idx)
 		flags |= NLM_F_DUMP_FILTERED;
@@ -2826,7 +2871,7 @@ static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
 			s_idx = 0;
 		idx = 0;
 		neigh_for_each_in_bucket_rcu(n, &nht->hash_heads[h]) {
-			if (idx < s_idx || !net_eq(dev_net(n->dev), net))
+			if (idx < s_idx)
 				goto next;
 			if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
 			    neigh_master_filtered(n->dev, filter->master_idx))
@@ -2850,11 +2895,10 @@ static int pneigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
 			     struct netlink_callback *cb,
 			     struct neigh_dump_filter *filter)
 {
-	struct pneigh_entry *n;
-	struct net *net = sock_net(skb->sk);
-	int err = 0, h, s_h = cb->args[3];
 	int idx, s_idx = idx = cb->args[4];
+	int err = 0, h, s_h = cb->args[3];
 	unsigned int flags = NLM_F_MULTI;
+	struct pneigh_entry *n;
 
 	if (filter->dev_idx || filter->master_idx)
 		flags |= NLM_F_DUMP_FILTERED;
@@ -2865,7 +2909,7 @@ static int pneigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
 		for (n = rcu_dereference(tbl->phash_buckets[h]), idx = 0;
 		     n;
 		     n = rcu_dereference(n->next)) {
-			if (idx < s_idx || pneigh_net(n) != net)
+			if (idx < s_idx)
 				goto next;
 			if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
 			    neigh_master_filtered(n->dev, filter->master_idx))
@@ -2951,6 +2995,7 @@ static int neigh_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	const struct nlmsghdr *nlh = cb->nlh;
 	struct neigh_dump_filter filter = {};
+	struct net *net = sock_net(skb->sk);
 	struct neigh_table *tbl;
 	int t, family, s_t;
 	int proxy = 0;
@@ -2974,8 +3019,7 @@ static int neigh_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
 
 	rcu_read_lock();
 	for (t = 0; t < NEIGH_NR_TABLES; t++) {
-		tbl = rcu_dereference(neigh_tables[t]);
-
+		tbl = net->neigh_tables[t];
 		if (!tbl)
 			continue;
 		if (t < s_t || (family && tbl->family != family))
@@ -3097,7 +3141,7 @@ static int neigh_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,
 
 	rcu_read_lock();
 
-	tbl = neigh_find_table(ndm->ndm_family);
+	tbl = neigh_find_table(net, ndm->ndm_family);
 	if (!tbl) {
 		NL_SET_ERR_MSG(extack, "Unsupported family in header for neighbor get request");
 		err = -EAFNOSUPPORT;
@@ -3124,7 +3168,7 @@ static int neigh_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,
 	if (ndm->ndm_flags & NTF_PROXY) {
 		struct pneigh_entry *pn;
 
-		pn = pneigh_lookup(tbl, net, dst, dev);
+		pn = pneigh_lookup(tbl, dst, dev);
 		if (!pn) {
 			NL_SET_ERR_MSG(extack, "Proxy neighbour entry not found");
 			err = -ENOENT;
@@ -3177,37 +3221,6 @@ void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void
 }
 EXPORT_SYMBOL(neigh_for_each);
 
-/* The tbl->lock must be held as a writer and BH disabled. */
-void __neigh_for_each_release(struct neigh_table *tbl,
-			      int (*cb)(struct neighbour *))
-{
-	struct neigh_hash_table *nht;
-	int chain;
-
-	nht = rcu_dereference_protected(tbl->nht,
-					lockdep_is_held(&tbl->lock));
-	for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
-		struct hlist_node *tmp;
-		struct neighbour *n;
-
-		neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[chain]) {
-			int release;
-
-			write_lock(&n->lock);
-			release = cb(n);
-			if (release) {
-				hlist_del_rcu(&n->hash);
-				hlist_del_rcu(&n->dev_list);
-				neigh_mark_dead(n);
-			}
-			write_unlock(&n->lock);
-			if (release)
-				neigh_cleanup_and_release(n);
-		}
-	}
-}
-EXPORT_SYMBOL(__neigh_for_each_release);
-
 int neigh_xmit(int index, struct net_device *dev,
 	       const void *addr, struct sk_buff *skb)
 {
@@ -3216,9 +3229,11 @@ int neigh_xmit(int index, struct net_device *dev,
 	if (likely(index < NEIGH_NR_TABLES)) {
 		struct neigh_table *tbl;
 		struct neighbour *neigh;
+		struct net *net;
 
 		rcu_read_lock();
-		tbl = rcu_dereference(neigh_tables[index]);
+		net = dev_net_rcu(dev);
+		tbl = net->neigh_tables[index];
 		if (!tbl) {
 			rcu_read_unlock();
 			goto out_kfree_skb;
@@ -3261,10 +3276,6 @@ static struct neighbour *neigh_get_valid(struct seq_file *seq,
 					 loff_t *pos)
 {
 	struct neigh_seq_state *state = seq->private;
-	struct net *net = seq_file_net(seq);
-
-	if (!net_eq(dev_net(n->dev), net))
-		return NULL;
 
 	if (state->neigh_sub_iter) {
 		loff_t fakep = 0;
@@ -3353,7 +3364,6 @@ static struct neighbour *neigh_get_idx(struct seq_file *seq, loff_t *pos)
 static struct pneigh_entry *pneigh_get_first(struct seq_file *seq)
 {
 	struct neigh_seq_state *state = seq->private;
-	struct net *net = seq_file_net(seq);
 	struct neigh_table *tbl = state->tbl;
 	struct pneigh_entry *pn = NULL;
 	int bucket;
@@ -3361,9 +3371,6 @@ static struct pneigh_entry *pneigh_get_first(struct seq_file *seq)
 	state->flags |= NEIGH_SEQ_IS_PNEIGH;
 	for (bucket = 0; bucket <= PNEIGH_HASHMASK; bucket++) {
 		pn = rcu_dereference(tbl->phash_buckets[bucket]);
-
-		while (pn && !net_eq(pneigh_net(pn), net))
-			pn = rcu_dereference(pn->next);
 		if (pn)
 			break;
 	}
@@ -3377,21 +3384,15 @@ static struct pneigh_entry *pneigh_get_next(struct seq_file *seq,
 					    loff_t *pos)
 {
 	struct neigh_seq_state *state = seq->private;
-	struct net *net = seq_file_net(seq);
 	struct neigh_table *tbl = state->tbl;
 
-	do {
-		pn = rcu_dereference(pn->next);
-	} while (pn && !net_eq(pneigh_net(pn), net));
+	pn = rcu_dereference(pn->next);
 
 	while (!pn) {
 		if (++state->bucket > PNEIGH_HASHMASK)
 			break;
 
 		pn = rcu_dereference(tbl->phash_buckets[state->bucket]);
-
-		while (pn && !net_eq(pneigh_net(pn), net))
-			pn = rcu_dereference(pn->next);
 		if (pn)
 			break;
 	}
@@ -3446,7 +3447,6 @@ void *neigh_seq_start(struct seq_file *seq, loff_t *pos, struct neigh_table *tbl
 
 	return *pos ? neigh_get_idx_any(seq, pos) : SEQ_START_TOKEN;
 }
-EXPORT_SYMBOL(neigh_seq_start);
 
 void *neigh_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 {
@@ -3473,7 +3473,6 @@ void *neigh_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 	++(*pos);
 	return rc;
 }
-EXPORT_SYMBOL(neigh_seq_next);
 
 void neigh_seq_stop(struct seq_file *seq, void *v)
 	__releases(tbl->lock)
@@ -3485,7 +3484,6 @@ void neigh_seq_stop(struct seq_file *seq, void *v)
 	spin_unlock_bh(&tbl->lock);
 	rcu_read_unlock();
 }
-EXPORT_SYMBOL(neigh_seq_stop);
 
 /* statistics via seq_file */
 
@@ -3539,7 +3537,7 @@ static int neigh_stat_seq_show(struct seq_file *seq, void *v)
 	seq_printf(seq, "%08x %08lx %08lx %08lx   %08lx %08lx %08lx   "
 			"%08lx         %08lx         %08lx         "
 			"%08lx       %08lx            %08lx\n",
-		   atomic_read(&tbl->entries),
+		   neigh_table_entries(tbl),
 
 		   st->allocs,
 		   st->destroys,
@@ -3609,7 +3607,6 @@ void neigh_app_ns(struct neighbour *n)
 {
 	neigh_notify(n, RTM_GETNEIGH, NLM_F_REQUEST, 0);
 }
-EXPORT_SYMBOL(neigh_app_ns);
 
 #ifdef CONFIG_SYSCTL
 static int unres_qlen_max = INT_MAX / SKB_TRUESIZE(ETH_FRAME_LEN);
@@ -3706,7 +3703,6 @@ int neigh_proc_dointvec(const struct ctl_table *ctl, int write, void *buffer,
 	neigh_proc_update(ctl, write);
 	return ret;
 }
-EXPORT_SYMBOL(neigh_proc_dointvec);
 
 int neigh_proc_dointvec_jiffies(const struct ctl_table *ctl, int write, void *buffer,
 				size_t *lenp, loff_t *ppos)
@@ -3716,7 +3712,6 @@ int neigh_proc_dointvec_jiffies(const struct ctl_table *ctl, int write, void *bu
 	neigh_proc_update(ctl, write);
 	return ret;
 }
-EXPORT_SYMBOL(neigh_proc_dointvec_jiffies);
 
 static int neigh_proc_dointvec_userhz_jiffies(const struct ctl_table *ctl, int write,
 					      void *buffer, size_t *lenp,
@@ -3736,7 +3731,6 @@ int neigh_proc_dointvec_ms_jiffies(const struct ctl_table *ctl, int write,
 	neigh_proc_update(ctl, write);
 	return ret;
 }
-EXPORT_SYMBOL(neigh_proc_dointvec_ms_jiffies);
 
 static int neigh_proc_dointvec_unres_qlen(const struct ctl_table *ctl, int write,
 					  void *buffer, size_t *lenp,
@@ -3943,7 +3937,6 @@ int neigh_sysctl_register(struct net_device *dev, struct neigh_parms *p,
 err:
 	return -ENOBUFS;
 }
-EXPORT_SYMBOL(neigh_sysctl_register);
 
 void neigh_sysctl_unregister(struct neigh_parms *p)
 {
@@ -3954,7 +3947,6 @@ void neigh_sysctl_unregister(struct neigh_parms *p)
 		kfree(t);
 	}
 }
-EXPORT_SYMBOL(neigh_sysctl_unregister);
 
 #endif	/* CONFIG_SYSCTL */
 
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index 352173df757850..fa790e4425704e 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -118,6 +118,49 @@ static int sysfs_rtnl_lock(struct kobject *kobj, struct attribute *attr,
 	return ret;
 }
 
+static int sysfs_get_link_ksettings(struct device *dev,
+				    struct device_attribute *attr,
+				    struct ethtool_link_ksettings *cmd)
+{
+	struct net_device *netdev = to_net_dev(dev);
+	bool need_rtnl;
+	int ret;
+
+	/*
+	 * The check is also done in netif_get_link_ksettings; this helps
+	 * returning early without hitting the locking section below.
+	 */
+	if (!netdev->ethtool_ops->get_link_ksettings)
+		return -EINVAL;
+
+	need_rtnl = !netdev_need_ops_lock(netdev) ||
+		    (netdev->ethtool_ops->op_needs_rtnl &
+		     ETHTOOL_OP_NEEDS_RTNL_LINKSETTINGS);
+	if (need_rtnl) {
+		ret = sysfs_rtnl_lock(&dev->kobj, &attr->attr, netdev);
+		if (ret)
+			return ret;
+	}
+	netdev_lock_ops(netdev);
+
+	if (!dev_isalive(netdev)) {
+		ret = -ENODEV;
+		goto unlock;
+	}
+
+	ret = -EINVAL;
+	if (netif_running(netdev)) {
+		if (!netif_get_link_ksettings(netdev, cmd))
+			ret = 0;
+	}
+
+unlock:
+	netdev_unlock_ops(netdev);
+	if (need_rtnl)
+		rtnl_unlock();
+	return ret;
+}
+
 /* use same locking rules as GIF* ioctl's */
 static ssize_t netdev_show(const struct device *dev,
 			   struct device_attribute *attr, char *buf,
@@ -332,70 +375,41 @@ static DEVICE_ATTR_RW(carrier);
 static ssize_t speed_show(struct device *dev,
 			  struct device_attribute *attr, char *buf)
 {
-	struct net_device *netdev = to_net_dev(dev);
-	int ret = -EINVAL;
-
-	/* The check is also done in __ethtool_get_link_ksettings; this helps
-	 * returning early without hitting the locking section below.
-	 */
-	if (!netdev->ethtool_ops->get_link_ksettings)
-		return ret;
+	struct ethtool_link_ksettings cmd;
+	int ret;
 
-	ret = sysfs_rtnl_lock(&dev->kobj, &attr->attr, netdev);
+	ret = sysfs_get_link_ksettings(dev, attr, &cmd);
 	if (ret)
 		return ret;
 
-	ret = -EINVAL;
-	if (netif_running(netdev)) {
-		struct ethtool_link_ksettings cmd;
-
-		if (!__ethtool_get_link_ksettings(netdev, &cmd))
-			ret = sysfs_emit(buf, fmt_dec, cmd.base.speed);
-	}
-	rtnl_unlock();
-	return ret;
+	return sysfs_emit(buf, fmt_dec, cmd.base.speed);
 }
 static DEVICE_ATTR_RO(speed);
 
 static ssize_t duplex_show(struct device *dev,
 			   struct device_attribute *attr, char *buf)
 {
-	struct net_device *netdev = to_net_dev(dev);
-	int ret = -EINVAL;
-
-	/* The check is also done in __ethtool_get_link_ksettings; this helps
-	 * returning early without hitting the locking section below.
-	 */
-	if (!netdev->ethtool_ops->get_link_ksettings)
-		return ret;
+	struct ethtool_link_ksettings cmd;
+	const char *duplex;
+	int ret;
 
-	ret = sysfs_rtnl_lock(&dev->kobj, &attr->attr, netdev);
+	ret = sysfs_get_link_ksettings(dev, attr, &cmd);
 	if (ret)
 		return ret;
 
-	ret = -EINVAL;
-	if (netif_running(netdev)) {
-		struct ethtool_link_ksettings cmd;
-
-		if (!__ethtool_get_link_ksettings(netdev, &cmd)) {
-			const char *duplex;
-
-			switch (cmd.base.duplex) {
-			case DUPLEX_HALF:
-				duplex = "half";
-				break;
-			case DUPLEX_FULL:
-				duplex = "full";
-				break;
-			default:
-				duplex = "unknown";
-				break;
-			}
-			ret = sysfs_emit(buf, "%s\n", duplex);
-		}
+	switch (cmd.base.duplex) {
+	case DUPLEX_HALF:
+		duplex = "half";
+		break;
+	case DUPLEX_FULL:
+		duplex = "full";
+		break;
+	default:
+		duplex = "unknown";
+		break;
 	}
-	rtnl_unlock();
-	return ret;
+
+	return sysfs_emit(buf, "%s\n", duplex);
 }
 static DEVICE_ATTR_RO(duplex);
 
diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c
index cb18db681640f0..690c27f63d2600 100644
--- a/net/core/netdev-genl.c
+++ b/net/core/netdev-genl.c
@@ -312,11 +312,14 @@ int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 			err = -ENODEV;
 		}
 	} else {
+		unsigned long start_ifindex = ctx->ifindex;
+
 		for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
+			if (ctx->ifindex != start_ifindex)
+				ctx->napi_id = 0;
 			err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
 			if (err < 0)
 				break;
-			ctx->napi_id = 0;
 		}
 	}
 
@@ -434,7 +437,7 @@ netdev_nl_queue_fill_lease(struct sk_buff *rsp, struct net_device *netdev,
 nla_put_failure_unlock:
 	rcu_read_unlock();
 nla_put_failure:
-	return -ENOMEM;
+	return -EMSGSIZE;
 }
 
 static int
@@ -636,13 +639,17 @@ int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 			err = -ENODEV;
 		}
 	} else {
+		unsigned long start_ifindex = ctx->ifindex;
+
 		for_each_netdev_lock_ops_compat_scoped(net, netdev,
 						       ctx->ifindex) {
+			if (ctx->ifindex != start_ifindex) {
+				ctx->rxq_idx = 0;
+				ctx->txq_idx = 0;
+			}
 			err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
 			if (err < 0)
 				break;
-			ctx->rxq_idx = 0;
-			ctx->txq_idx = 0;
 		}
 	}
 
@@ -768,7 +775,7 @@ netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
 	const struct netdev_stat_ops *ops = netdev->stat_ops;
 	int i, err;
 
-	if (!(netdev->flags & IFF_UP))
+	if (!netdev->up)
 		return 0;
 
 	i = ctx->rxq_idx;
@@ -788,8 +795,6 @@ netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
 		ctx->txq_idx = ++i;
 	}
 
-	ctx->rxq_idx = 0;
-	ctx->txq_idx = 0;
 	return 0;
 }
 
@@ -904,6 +909,7 @@ int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
 	struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
 	const struct genl_info *info = genl_info_dump(cb);
 	struct net *net = sock_net(skb->sk);
+	unsigned long start_ifindex;
 	struct net_device *netdev;
 	unsigned int ifindex;
 	unsigned int scope;
@@ -936,7 +942,13 @@ int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
 		return err;
 	}
 
+	start_ifindex = ctx->ifindex;
+
 	for_each_netdev_lock_ops_compat_scoped(net, netdev, ctx->ifindex) {
+		if (ctx->ifindex != start_ifindex) {
+			ctx->rxq_idx = 0;
+			ctx->txq_idx = 0;
+		}
 		err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
 						    info, ctx);
 		if (err < 0)
@@ -1168,6 +1180,12 @@ netdev_find_netmem_tx_dev(struct net_device *dev)
 	return NULL;
 }
 
+/* Note: NETDEV_CMD_BIND_TX is intentionally unprivileged (no
+ * GENL_ADMIN_PERM / GENL_UNS_ADMIN_PERM). Unlike bind-rx, which configures
+ * shared NIC RX queues, bind-tx only DMA-maps the caller's dmabuf so they can
+ * transmit from it on their own sockets without affecting other traffic or
+ * device state.
+ */
 int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct net_devmem_dmabuf_binding *binding;
diff --git a/net/core/netdev_work.c b/net/core/netdev_work.c
index e721a06d58df1f..c99967907e185a 100644
--- a/net/core/netdev_work.c
+++ b/net/core/netdev_work.c
@@ -3,7 +3,6 @@
 #include <linux/export.h>
 #include <linux/list.h>
 #include <linux/netdevice.h>
-#include <linux/rtnetlink.h>
 #include <linux/spinlock.h>
 #include <linux/workqueue.h>
 #include <net/netdev_lock.h>
@@ -129,8 +128,6 @@ static void netdev_work_run(struct net_device *dev, unsigned long events,
 
 static void netdev_work_proc(struct work_struct *work)
 {
-	rtnl_lock();
-
 	while (true) {
 		unsigned long events = 0, core = 0;
 		netdevice_tracker tracker;
@@ -143,16 +140,10 @@ static void netdev_work_proc(struct work_struct *work)
 		}
 		dev = list_first_entry(&netdev_work_list, struct net_device,
 				       work_node);
-		/* Take a temporary reference so @dev can't be freed while we
-		 * drop the lock to grab its ops lock; the work reference is
-		 * only released once we claim the work below.
-		 * The re-locking dance is to ensure that ops lock is enough
-		 * to ensure canceling work is not racy with dequeue.
-		 */
 		netdev_hold(dev, &tracker, GFP_ATOMIC);
 		spin_unlock_bh(&netdev_work_lock);
 
-		netdev_lock_ops(dev);
+		netdev_lock_ops_compat(dev);
 		spin_lock_bh(&netdev_work_lock);
 		if (!list_empty(&dev->work_node)) {
 			list_del_init(&dev->work_node);
@@ -169,10 +160,8 @@ static void netdev_work_proc(struct work_struct *work)
 		spin_unlock_bh(&netdev_work_lock);
 
 		netdev_work_run(dev, events, core);
-		netdev_unlock_ops(dev);
+		netdev_unlock_ops_compat(dev);
 
 		netdev_put(dev, &tracker);
 	}
-
-	rtnl_unlock();
 }
diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 08d7f35cf60894..d7c88c0b67e532 100644
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -951,8 +951,8 @@ static void page_pool_recycle_ring_bulk(struct page_pool *pool,
 		}
 	}
 
-	page_pool_producer_unlock(pool, in_softirq);
 	recycle_stat_add(pool, ring, i);
+	page_pool_producer_unlock(pool, in_softirq);
 
 	/* Hopefully all pages were returned into ptr_ring */
 	if (likely(i == bulk_len))
diff --git a/net/core/page_pool_priv.h b/net/core/page_pool_priv.h
index 2fb06d5f6d5599..430b97cd88dad5 100644
--- a/net/core/page_pool_priv.h
+++ b/net/core/page_pool_priv.h
@@ -18,18 +18,8 @@ void page_pool_unlist(struct page_pool *pool);
 static inline bool
 page_pool_set_dma_addr_netmem(netmem_ref netmem, dma_addr_t addr)
 {
-	if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) {
-		netmem_set_dma_addr(netmem, addr >> PAGE_SHIFT);
-
-		/* We assume page alignment to shave off bottom bits,
-		 * if this "compression" doesn't work we need to drop.
-		 */
-		return addr != (dma_addr_t)netmem_get_dma_addr(netmem)
-				       << PAGE_SHIFT;
-	}
-
-	netmem_set_dma_addr(netmem, addr);
-	return false;
+	netmem_set_dma_addr(netmem, netmem_dma_addr_encode(addr));
+	return !netmem_dma_addr_fits(addr);
 }
 
 static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c
index ef4261c0e8eaf4..f30f3779624cb6 100644
--- a/net/core/page_pool_user.c
+++ b/net/core/page_pool_user.c
@@ -84,6 +84,7 @@ netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
 	struct page_pool_dump_cb *state = (void *)cb->ctx;
 	const struct genl_info *info = genl_info_dump(cb);
 	struct net *net = sock_net(skb->sk);
+	unsigned long start_ifindex;
 	struct net_device *netdev;
 	struct page_pool *pool;
 	int err = 0;
@@ -91,6 +92,8 @@ netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
 	if (ifindex_attr)
 		state->ifindex = nla_get_u32(ifindex_attr);
 
+	start_ifindex = state->ifindex;
+
 	rtnl_lock();
 	mutex_lock(&page_pools_lock);
 	for_each_netdev_dump(net, netdev, state->ifindex) {
@@ -99,6 +102,9 @@ netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
 		    netdev->ifindex != nla_get_u32(ifindex_attr))
 			break;
 
+		if (state->ifindex != start_ifindex)
+			state->pp_id = 0;
+
 		hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
 			if (state->pp_id && state->pp_id < pool->user.id)
 				continue;
@@ -108,8 +114,6 @@ netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
 			if (err)
 				goto out;
 		}
-
-		state->pp_id = 0;
 	}
 out:
 	mutex_unlock(&page_pools_lock);
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index 7f81aed466723b..d63eeadddfccd4 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -2024,11 +2024,11 @@ static const struct proc_ops pktgen_thread_proc_ops = {
 };
 
 /* Think find or remove for NN */
-static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn,
-					      const char *ifname, int remove)
+static bool __pktgen_NN_threads(const struct pktgen_net *pn,
+				const char *ifname, int remove)
 {
 	struct pktgen_thread *t;
-	struct pktgen_dev *pkt_dev = NULL;
+	struct pktgen_dev *pkt_dev;
 	bool exact = (remove == FIND);
 
 	list_for_each_entry(t, &pn->pktgen_threads, th_list) {
@@ -2038,10 +2038,10 @@ static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn,
 				pkt_dev->removal_mark = 1;
 				t->control |= T_REMDEV;
 			}
-			break;
+			return true;
 		}
 	}
-	return pkt_dev;
+	return false;
 }
 
 /*
@@ -2049,7 +2049,6 @@ static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn,
  */
 static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname)
 {
-	struct pktgen_dev *pkt_dev = NULL;
 	const int max_tries = 10, msec_per_try = 125;
 	int i = 0;
 
@@ -2058,8 +2057,7 @@ static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname)
 
 	while (1) {
 
-		pkt_dev = __pktgen_NN_threads(pn, ifname, REMOVE);
-		if (pkt_dev == NULL)
+		if (!__pktgen_NN_threads(pn, ifname, REMOVE))
 			break;	/* success */
 
 		mutex_unlock(&pktgen_thread_lock);
@@ -2346,11 +2344,11 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
 			set_current_state(TASK_INTERRUPTIBLE);
 			hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS);
 
-			if (likely(t.task))
+			if (likely(hrtimer_sleeper_task_get(&t)))
 				schedule();
 
 			hrtimer_cancel(&t.timer);
-		} while (t.task && pkt_dev->running && !signal_pending(current));
+		} while (hrtimer_sleeper_task_get(&t) && pkt_dev->running && !signal_pending(current));
 		__set_current_state(TASK_RUNNING);
 		end_time = ktime_get();
 	}
@@ -3835,8 +3833,7 @@ static int pktgen_add_device(struct pktgen_thread *t, const char *ifname)
 
 	/* We don't allow a device to be on several threads */
 
-	pkt_dev = __pktgen_NN_threads(t->net, ifname, FIND);
-	if (pkt_dev) {
+	if (__pktgen_NN_threads(t->net, ifname, FIND)) {
 		pr_err("ERROR: interface already used\n");
 		return -EBUSY;
 	}
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index be9d1625bac310..e3444fd240615a 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -1396,6 +1396,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev,
 	       + rtnl_devlink_port_size(dev)
 	       + rtnl_dpll_pin_size()
 	       + nla_total_size(8)  /* IFLA_MAX_PACING_OFFLOAD_HORIZON */
+	       + nla_total_size(4)  /* IFLA_PACING_OFFLOAD */
 	       + nla_total_size(2)  /* IFLA_HEADROOM */
 	       + nla_total_size(2)  /* IFLA_TAILROOM */
 	       + rtnl_dev_parent_size(dev)
@@ -2176,6 +2177,8 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
 			READ_ONCE(dev->tso_max_segs)) ||
 	    nla_put_uint(skb, IFLA_MAX_PACING_OFFLOAD_HORIZON,
 			 READ_ONCE(dev->max_pacing_offload_horizon)) ||
+	    nla_put_u32(skb, IFLA_PACING_OFFLOAD,
+			dev->pacing_offload) ||
 #ifdef CONFIG_RPS
 	    nla_put_u32(skb, IFLA_NUM_RX_QUEUES,
 			READ_ONCE(dev->num_rx_queues)) ||
@@ -2349,9 +2352,11 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = {
 	[IFLA_ALLMULTI]		= { .type = NLA_REJECT },
 	[IFLA_GSO_IPV4_MAX_SIZE]	= NLA_POLICY_MIN(NLA_U32, MAX_TCP_HEADER + 1),
 	[IFLA_GRO_IPV4_MAX_SIZE]	= { .type = NLA_U32 },
+	[IFLA_MAX_PACING_OFFLOAD_HORIZON] = { .type = NLA_REJECT },
 	[IFLA_NETNS_IMMUTABLE]	= { .type = NLA_REJECT },
 	[IFLA_HEADROOM]		= { .type = NLA_REJECT },
 	[IFLA_TAILROOM]		= { .type = NLA_REJECT },
+	[IFLA_PACING_OFFLOAD]	= NLA_POLICY_MAX(NLA_U32, 1),
 };
 
 static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = {
@@ -2825,6 +2830,14 @@ static int validate_linkmsg(struct net_device *dev, struct nlattr *tb[],
 		return -EINVAL;
 	}
 
+	if (tb[IFLA_PACING_OFFLOAD]) {
+		if (nla_get_u32(tb[IFLA_PACING_OFFLOAD]) &&
+		    !dev->max_pacing_offload_horizon) {
+			NL_SET_ERR_MSG(extack, "pacing offload not supported by device");
+			return -EOPNOTSUPP;
+		}
+	}
+
 	if (tb[IFLA_AF_SPEC]) {
 		struct nlattr *af;
 		int rem, err;
@@ -3342,6 +3355,15 @@ static int do_setlink(const struct sk_buff *skb, struct net_device *dev,
 		}
 	}
 
+	if (tb[IFLA_PACING_OFFLOAD]) {
+		bool val = nla_get_u32(tb[IFLA_PACING_OFFLOAD]);
+
+		if (dev->pacing_offload != val) {
+			dev->pacing_offload = val;
+			status |= DO_SETLINK_MODIFIED;
+		}
+	}
+
 	if (tb[IFLA_OPERSTATE])
 		set_operstate(dev, nla_get_u8(tb[IFLA_OPERSTATE]));
 
diff --git a/net/core/scm.c b/net/core/scm.c
index f0d44ecdb11fca..15c330784a6992 100644
--- a/net/core/scm.c
+++ b/net/core/scm.c
@@ -364,14 +364,14 @@ int scm_recv_one_fd(struct file *f, int __user *ufd, unsigned int flags,
 		return notrunc ? put_user(error, ufd) : error;
 
 	FD_PREPARE(fdf, flags, get_file(f));
-	if (fdf.err)
-		return fdf.err;
+	if (fdf->fd < 0)
+		return fdf->fd;
 
-	error = put_user(fd_prepare_fd(fdf), ufd);
+	error = put_user(fdf->fd, ufd);
 	if (error)
 		return error;
 
-	__receive_sock(fd_prepare_file(fdf));
+	__receive_sock(fdf->file);
 	return fd_publish(fdf);
 }
 
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 609f2c7f4a47ad..5c4024a03e1055 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -5977,7 +5977,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
 			err = skb_maybe_pull_tail(skb,
 						  off +
 						  sizeof(struct ipv6_opt_hdr),
-						  MAX_IPV6_HDR_LEN);
+						  off +
+						  sizeof(struct ipv6_opt_hdr));
 			if (err < 0)
 				goto out;
 
@@ -5992,7 +5993,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
 			err = skb_maybe_pull_tail(skb,
 						  off +
 						  sizeof(struct ip_auth_hdr),
-						  MAX_IPV6_HDR_LEN);
+						  off +
+						  sizeof(struct ip_auth_hdr));
 			if (err < 0)
 				goto out;
 
@@ -6007,7 +6009,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
 			err = skb_maybe_pull_tail(skb,
 						  off +
 						  sizeof(struct frag_hdr),
-						  MAX_IPV6_HDR_LEN);
+						  off +
+						  sizeof(struct frag_hdr));
 			if (err < 0)
 				goto out;
 
@@ -7271,16 +7274,23 @@ static void skb_ext_put_sp(struct sec_path *sp)
 {
 	unsigned int i;
 
+	if (!sp->len)
+		return;
+
 	for (i = 0; i < sp->len; i++)
 		xfrm_state_put(sp->xvec[i]);
+	sp->len = 0;
 }
 #endif
 
 #ifdef CONFIG_MCTP_FLOWS
 static void skb_ext_put_mctp(struct mctp_flow *flow)
 {
-	if (flow->key)
-		mctp_key_unref(flow->key);
+	if (!flow->key)
+		return;
+
+	mctp_key_unref(flow->key);
+	flow->key = NULL;
 }
 #endif
 
@@ -7292,15 +7302,20 @@ void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
 	if (skb->active_extensions == 0) {
 		skb->extensions = NULL;
 		__skb_ext_put(ext);
-#ifdef CONFIG_XFRM
-	} else if (id == SKB_EXT_SEC_PATH &&
-		   refcount_read(&ext->refcnt) == 1) {
-		struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
+		return;
+	}
 
-		skb_ext_put_sp(sp);
-		sp->len = 0;
+	if (refcount_read(&ext->refcnt) > 1)
+		return;
+
+#ifdef CONFIG_XFRM
+	if (id == SKB_EXT_SEC_PATH)
+		skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
+#endif
+#ifdef CONFIG_MCTP_FLOWS
+	if (id == SKB_EXT_MCTP)
+		skb_ext_put_mctp(skb_ext_get_ptr(ext, SKB_EXT_MCTP));
 #endif
-	}
 }
 EXPORT_SYMBOL(__skb_ext_del);
 
@@ -7357,8 +7372,8 @@ void skb_attempt_defer_free(struct sk_buff *skb)
 	struct skb_defer_node *sdn;
 	unsigned long defer_count;
 	unsigned int defer_max;
+	int cpu, my_cpu;
 	bool kick;
-	int cpu;
 
 	if (static_branch_unlikely(&skb_defer_disable_key))
 		goto nodefer;
@@ -7368,7 +7383,8 @@ void skb_attempt_defer_free(struct sk_buff *skb)
 		goto nodefer;
 
 	cpu = skb->alloc_cpu;
-	if (cpu == raw_smp_processor_id() ||
+	my_cpu = raw_smp_processor_id();
+	if (cpu == my_cpu ||
 	    WARN_ON_ONCE(cpu >= nr_cpu_ids) ||
 	    !cpu_online(cpu)) {
 nodefer:	kfree_skb_napi_cache(skb);
@@ -7379,7 +7395,7 @@ nodefer:	kfree_skb_napi_cache(skb);
 	DEBUG_NET_WARN_ON_ONCE(skb->destructor);
 	DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb));
 
-	sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + numa_node_id();
+	sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu);
 
 	defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
 	defer_count = atomic_long_inc_return(&sdn->defer_count);
@@ -7389,6 +7405,11 @@ nodefer:	kfree_skb_napi_cache(skb);
 
 	llist_add(&skb->ll_node, &sdn->defer_list);
 
+	if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) {
+		skb_defer_node_flush(sdn);
+		return;
+	}
+
 	/* Send an IPI every time queue reaches half capacity. */
 	kick = (defer_count - 1) == (defer_max >> 1);
 
@@ -7400,7 +7421,8 @@ nodefer:	kfree_skb_napi_cache(skb);
 }
 
 static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
-				 size_t offset, size_t len)
+				 size_t offset, size_t len,
+				 unsigned int csum_offset)
 {
 	const char *kaddr;
 	__wsum csum;
@@ -7408,7 +7430,7 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
 	kaddr = kmap_local_page(page);
 	csum = csum_partial(kaddr + offset, len, 0);
 	kunmap_local(kaddr);
-	skb->csum = csum_block_add(skb->csum, csum, skb->len);
+	skb->csum = csum_block_add(skb->csum, csum, csum_offset);
 }
 
 /**
@@ -7468,7 +7490,8 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
 			}
 
 			if (skb->ip_summed == CHECKSUM_NONE)
-				skb_splice_csum_page(skb, page, off, part);
+				skb_splice_csum_page(skb, page, off, part,
+						     skb->len + spliced);
 
 			off = 0;
 			spliced += part;
diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index 2521b643fa05d4..df385a5a961e83 100644
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -1000,6 +1000,10 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
 	int err = 0;
 	u32 len, off;
 
+	if (verdict == __SK_REDIRECT && skb_bpf_ingress(skb) &&
+	    skb_bpf_redirect_fetch(skb) == psock->sk)
+		verdict = __SK_PASS;
+
 	switch (verdict) {
 	case __SK_PASS:
 		err = -EIO;
diff --git a/net/core/sock.c b/net/core/sock.c
index d23333bb4f3faf..972c72c5bcc68a 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -771,7 +771,7 @@ bool sk_mc_loop(const struct sock *sk)
 		return false;
 	if (!sk)
 		return true;
-	/* IPV6_ADDRFORM can change sk->sk_family under us. */
+
 	switch (READ_ONCE(sk->sk_family)) {
 	case AF_INET:
 		return inet_test_bit(MC_LOOP, sk);
@@ -1034,7 +1034,7 @@ static int sock_reserve_memory(struct sock *sk, int bytes)
 	bool charged;
 	int pages;
 
-	if (!mem_cgroup_sk_enabled(sk) || !sk_has_account(sk))
+	if (!mem_cgroup_sk_enabled(sk) || !sk_is_tcp(sk))
 		return -EOPNOTSUPP;
 
 	if (!bytes)
@@ -1661,7 +1661,7 @@ int sk_setsockopt(struct sock *sk, int level, int optname,
 	{
 		int delta;
 
-		if (val < 0) {
+		if (val < 0 || val > SZ_1G) {
 			ret = -EINVAL;
 			break;
 		}
@@ -4024,7 +4024,6 @@ int sock_common_getsockopt(struct socket *sock, int level, int optname,
 {
 	struct sock *sk = sock->sk;
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen);
 }
 EXPORT_SYMBOL(sock_common_getsockopt);
@@ -4046,7 +4045,6 @@ int sock_common_setsockopt(struct socket *sock, int level, int optname,
 {
 	struct sock *sk = sock->sk;
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen);
 }
 EXPORT_SYMBOL(sock_common_setsockopt);
diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index ca49bc7f8687ce..38df842843289a 100644
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -41,6 +41,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
 	struct bpf_stab *stab;
 
 	if (attr->max_entries == 0 ||
+	    attr->max_entries > INT_MAX ||
 	    attr->key_size    != 4 ||
 	    (attr->value_size != sizeof(u32) &&
 	     attr->value_size != sizeof(u64)) ||
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index eb35da3556f4aa..473c162736d692 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -21,6 +21,7 @@
 
 #include <net/ip.h>
 #include <net/sock.h>
+#include <net/neighbour.h>
 #include <net/net_ratelimit.h>
 #include <net/busy_poll.h>
 #include <net/pkt_sched.h>
@@ -336,10 +337,15 @@ static int proc_do_rss_key(const struct ctl_table *table, int write,
 {
 	char buf[NETDEV_RSS_KEY_LEN * 3];
 	struct ctl_table fake_table;
+	bool initialized;
 	char *pos = buf;
 
+	initialized = READ_ONCE(netdev_rss_key_initialized);
+	/* Pair with smp_wmb() in netdev_rss_key_fill(). */
+	smp_rmb();
+
 	for (int i = 0; i < NETDEV_RSS_KEY_LEN; i++) {
-		pos = hex_byte_pack(pos, netdev_rss_key[i]);
+		pos = hex_byte_pack(pos, initialized ? netdev_rss_key[i] : 0);
 		*pos++ = ':';
 	}
 	*(--pos) = 0;
@@ -676,6 +682,15 @@ static struct ctl_table net_core_table[] = {
 		.proc_handler	= proc_do_skb_defer_max,
 		.extra1		= SYSCTL_ZERO,
 	},
+	{
+		.procname	= "neigh_inherit_init_net",
+		.data		= &sysctl_neigh_inherit_init_net,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
 };
 
 static const struct ctl_table netns_core_table[] = {
diff --git a/net/devlink/dev.c b/net/devlink/dev.c
index 55959b0ff5ab4d..b85bb02a9e8072 100644
--- a/net/devlink/dev.c
+++ b/net/devlink/dev.c
@@ -1169,7 +1169,7 @@ int devlink_nl_flash_update_doit(struct sk_buff *skb, struct genl_info *info)
 
 	nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
 	file_name = nla_data(nla_file_name);
-	ret = request_firmware(&params.fw, file_name, devlink->dev);
+	ret = request_firmware_direct(&params.fw, file_name, devlink->dev);
 	if (ret) {
 		NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name,
 				    "failed to locate the requested firmware file");
@@ -1245,7 +1245,7 @@ int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
 		goto out_unlock;
 	}
 
-	ret = request_firmware(&params.fw, file_name, devlink->dev);
+	ret = request_firmware_direct(&params.fw, file_name, devlink->dev);
 	if (ret)
 		goto out_unlock;
 
diff --git a/net/devlink/netlink_gen.c b/net/devlink/netlink_gen.c
index dec00133178d12..9e1b4c081a7b3c 100644
--- a/net/devlink/netlink_gen.c
+++ b/net/devlink/netlink_gen.c
@@ -52,11 +52,11 @@ const struct nla_policy devlink_dl_parent_dev_nl_policy[DEVLINK_ATTR_INDEX + 1]
 	[DEVLINK_ATTR_INDEX] = NLA_POLICY_FULL_RANGE(NLA_UINT, &devlink_attr_index_range),
 };
 
-const struct nla_policy devlink_dl_port_function_nl_policy[DEVLINK_PORT_FN_ATTR_CAPS + 1] = {
+const struct nla_policy devlink_dl_port_function_set_nl_policy[DEVLINK_PORT_FN_ATTR_MAX_IO_EQS + 1] = {
 	[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY, },
 	[DEVLINK_PORT_FN_ATTR_STATE] = NLA_POLICY_MAX(NLA_U8, 1),
-	[DEVLINK_PORT_FN_ATTR_OPSTATE] = NLA_POLICY_MAX(NLA_U8, 1),
 	[DEVLINK_PORT_FN_ATTR_CAPS] = NLA_POLICY_BITFIELD32(15),
+	[DEVLINK_PORT_FN_ATTR_MAX_IO_EQS] = { .type = NLA_U32, },
 };
 
 const struct nla_policy devlink_dl_rate_tc_bws_nl_policy[DEVLINK_RATE_TC_ATTR_BW + 1] = {
@@ -97,7 +97,7 @@ static const struct nla_policy devlink_port_set_nl_policy[DEVLINK_ATTR_INDEX + 1
 	[DEVLINK_ATTR_INDEX] = NLA_POLICY_FULL_RANGE(NLA_UINT, &devlink_attr_index_range),
 	[DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32, },
 	[DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_MAX(NLA_U16, 3),
-	[DEVLINK_ATTR_PORT_FUNCTION] = NLA_POLICY_NESTED(devlink_dl_port_function_nl_policy),
+	[DEVLINK_ATTR_PORT_FUNCTION] = NLA_POLICY_NESTED(devlink_dl_port_function_set_nl_policy),
 };
 
 /* DEVLINK_CMD_PORT_NEW - do */
@@ -334,7 +334,7 @@ static const struct nla_policy devlink_reload_nl_policy[DEVLINK_ATTR_INDEX + 1]
 	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_INDEX] = NLA_POLICY_FULL_RANGE(NLA_UINT, &devlink_attr_index_range),
 	[DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, 1, 2),
-	[DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(6),
+	[DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(3),
 	[DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32, },
 	[DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32, },
 	[DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32, },
@@ -618,6 +618,7 @@ static const struct nla_policy devlink_rate_set_nl_policy[DEVLINK_ATTR_PARENT_DE
 	[DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_INDEX] = NLA_POLICY_FULL_RANGE(NLA_UINT, &devlink_attr_index_range),
+	[DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32, },
 	[DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING, },
 	[DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64, },
 	[DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64, },
diff --git a/net/devlink/netlink_gen.h b/net/devlink/netlink_gen.h
index a70e0e4769aa55..99ccacc693b76f 100644
--- a/net/devlink/netlink_gen.h
+++ b/net/devlink/netlink_gen.h
@@ -14,7 +14,7 @@
 
 /* Common nested types */
 extern const struct nla_policy devlink_dl_parent_dev_nl_policy[DEVLINK_ATTR_INDEX + 1];
-extern const struct nla_policy devlink_dl_port_function_nl_policy[DEVLINK_PORT_FN_ATTR_CAPS + 1];
+extern const struct nla_policy devlink_dl_port_function_set_nl_policy[DEVLINK_PORT_FN_ATTR_MAX_IO_EQS + 1];
 extern const struct nla_policy devlink_dl_rate_tc_bws_nl_policy[DEVLINK_RATE_TC_ATTR_BW + 1];
 extern const struct nla_policy devlink_dl_selftest_id_nl_policy[DEVLINK_ATTR_SELFTEST_ID_FLASH + 1];
 
diff --git a/net/devlink/port.c b/net/devlink/port.c
index 1528f2d148dfc6..803429d9a0089c 100644
--- a/net/devlink/port.c
+++ b/net/devlink/port.c
@@ -6,19 +6,6 @@
 
 #include "devl_internal.h"
 
-#define DEVLINK_PORT_FN_CAPS_VALID_MASK \
-	(_BITUL(__DEVLINK_PORT_FN_ATTR_CAPS_MAX) - 1)
-
-static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
-	[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
-	[DEVLINK_PORT_FN_ATTR_STATE] =
-		NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
-				 DEVLINK_PORT_FN_STATE_ACTIVE),
-	[DEVLINK_PORT_FN_ATTR_CAPS] =
-		NLA_POLICY_BITFIELD32(DEVLINK_PORT_FN_CAPS_VALID_MASK),
-	[DEVLINK_PORT_FN_ATTR_MAX_IO_EQS] = { .type = NLA_U32 },
-};
-
 #define ASSERT_DEVLINK_PORT_REGISTERED(devlink_port)				\
 	WARN_ON_ONCE(!(devlink_port)->registered)
 #define ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port)			\
@@ -782,11 +769,11 @@ static int devlink_port_function_set(struct devlink_port *port,
 				     const struct nlattr *attr,
 				     struct netlink_ext_ack *extack)
 {
-	struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
+	struct nlattr *tb[ARRAY_SIZE(devlink_dl_port_function_set_nl_policy)];
 	int err;
 
-	err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
-			       devlink_function_nl_policy, extack);
+	err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, attr,
+			       devlink_dl_port_function_set_nl_policy, extack);
 	if (err < 0) {
 		NL_SET_ERR_MSG(extack, "Fail to parse port function attributes");
 		return err;
diff --git a/net/dsa/Kconfig b/net/dsa/Kconfig
index 23b4b74004ed5e..4f44bf3ede238d 100644
--- a/net/dsa/Kconfig
+++ b/net/dsa/Kconfig
@@ -125,6 +125,12 @@ config NET_DSA_TAG_MXL_GSW1XX
 	  Say Y or M if you want to enable support for tagging frames for
 	  MaxLinear GSW1xx switches.
 
+config NET_DSA_TAG_KS8995
+	tristate "Tag driver for Micrel KS8995 switch"
+	help
+	  Say Y if you want to enable support for tagging frames for the
+	  Micrel KS8995 switch.
+
 config NET_DSA_TAG_KSZ
 	tristate "Tag driver for Microchip 8795/937x/9477/9893 families of switches"
 	help
diff --git a/net/dsa/Makefile b/net/dsa/Makefile
index d15bcf5c68f02a..1f9cc30e99886f 100644
--- a/net/dsa/Makefile
+++ b/net/dsa/Makefile
@@ -25,6 +25,7 @@ obj-$(CONFIG_NET_DSA_TAG_BRCM_COMMON) += tag_brcm.o
 obj-$(CONFIG_NET_DSA_TAG_DSA_COMMON) += tag_dsa.o
 obj-$(CONFIG_NET_DSA_TAG_GSWIP) += tag_gswip.o
 obj-$(CONFIG_NET_DSA_TAG_HELLCREEK) += tag_hellcreek.o
+obj-$(CONFIG_NET_DSA_TAG_KS8995) += tag_ks8995.o
 obj-$(CONFIG_NET_DSA_TAG_KSZ) += tag_ksz.o
 obj-$(CONFIG_NET_DSA_TAG_LAN9303) += tag_lan9303.o
 obj-$(CONFIG_NET_DSA_TAG_MT7628) += tag_mt7628.o
diff --git a/net/dsa/tag_ks8995.c b/net/dsa/tag_ks8995.c
new file mode 100644
index 00000000000000..f011bf01845efe
--- /dev/null
+++ b/net/dsa/tag_ks8995.c
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 Linus Walleij <linusw@kernel.org>
+ */
+#include <linux/bitfield.h>
+#include <linux/etherdevice.h>
+#include <linux/log2.h>
+#include <linux/list.h>
+#include <linux/net.h>
+#include <linux/slab.h>
+
+#include "tag.h"
+
+/* The Micrel KS8995XA / Microchip KSZ8995XA Special Tag Packet ID (STPID)
+ * pushes its tag in a modified VLAN (802.1Q) tag.
+ * -----------------------------------------------------------
+ * | MAC DA | MAC SA | 2 bytes tag | 2 bytes TCI | EtherType |
+ * -----------------------------------------------------------
+ * The tag is: 0x8100 |= BIT(port), ports 0,1,2,3
+ */
+
+#define KS8995_NAME "ks8995"
+
+#define KS8995_TAG_LEN		VLAN_HLEN
+/* Reserve room for the switch tag and a hardware-accelerated VLAN tag. */
+#define KS8995_NEEDED_HEADROOM	(KS8995_TAG_LEN + VLAN_HLEN)
+
+#define KS8995M_STPID_STD	GENMASK(15, 4)
+#define KS8995M_STPID_PORTMASK	GENMASK(3, 0)
+#define KS8995M_STPID(portmask)	htons(ETH_P_8021Q | FIELD_PREP(KS8995M_STPID_PORTMASK, portmask))
+
+static struct sk_buff *ks8995_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct vlan_ethhdr *hdr;
+	u16 portmask;
+
+	/* VLAN insertion requires a complete, linear Ethernet header. */
+	if (unlikely(!pskb_may_pull(skb, ETH_HLEN))) {
+		kfree_skb(skb);
+		return NULL;
+	}
+
+	/* Prepare the special KS8995 tags */
+	portmask = dsa_xmit_port_mask(skb, dev);
+
+	/* The switch expects the special tag at offset 12. Move any hardware
+	 * accelerated VLAN tag into the payload so the conduit cannot insert
+	 * it outside the special tag.
+	 */
+	if (unlikely(skb_vlan_tag_present(skb))) {
+		skb = __vlan_hwaccel_push_inside(skb);
+		if (!skb)
+			return NULL;
+	}
+
+	/* Always add a distinct outer tag. The user port removes this field on
+	 * egress, so reusing an existing 802.1Q tag would consume that VLAN tag.
+	 */
+	skb = vlan_insert_tag(skb, KS8995M_STPID(portmask), 0);
+	/* vlan_insert_tag() drops the skb on failure */
+	if (!skb)
+		return NULL;
+	hdr = skb_vlan_eth_hdr(skb);
+	netdev_dbg(dev, "%s: inserted VLAN TAG %04x TCI %04x\n",
+		   __func__, ntohs(hdr->h_vlan_proto),
+		   ntohs(hdr->h_vlan_TCI));
+
+	return skb;
+}
+
+static struct sk_buff *ks8995_rcv(struct sk_buff *skb, struct net_device *dev)
+{
+	int portmask;
+	int port;
+	u16 etype;
+
+	/* The special tag may be in the packet or VLAN metadata. In either
+	 * case, its TPID must retain the source-port bits.
+	 *
+	 * Read the in-band TPID directly because skb->protocol contains
+	 * ETH_P_XDSA.
+	 */
+	if (skb_vlan_tag_present(skb))
+		etype = ntohs(skb->vlan_proto);
+	else
+		etype = ntohs(*(__be16 *)dsa_etype_header_pos_rx(skb));
+	if ((etype & KS8995M_STPID_STD) != ETH_P_8021Q) {
+		netdev_dbg(dev, "%s: dropped ethertype 0x%04x\n",
+			   __func__, etype);
+		kfree_skb(skb);
+		return NULL;
+	}
+
+	portmask = FIELD_GET(KS8995M_STPID_PORTMASK, etype);
+	if (unlikely(!is_power_of_2(portmask))) {
+		netdev_dbg(dev, "%s: dropped invalid port mask 0x%04x\n",
+			   __func__, portmask);
+		kfree_skb(skb);
+		return NULL;
+	}
+	port = ilog2(portmask);
+
+	netdev_dbg(dev, "%s: received ethertype %04x\n",
+		   __func__, etype);
+
+	/* Move an in-band special tag into VLAN metadata. If already
+	 * offloaded, its TPID was validated above.
+	 */
+	if (!skb_vlan_tag_present(skb)) {
+		skb = skb_vlan_untag(skb);
+		if (!skb) {
+			/* skb_vlan_untag drops the skb on failure */
+			if (net_ratelimit())
+				netdev_err(dev, "%s: unable to untag skb\n", __func__);
+			return NULL;
+		}
+	}
+
+	netdev_dbg(dev, "%s: etype %04x portmask %04x (%d)\n",
+		   __func__, etype, portmask, port);
+	skb->dev = dsa_conduit_find_user(dev, 0, port);
+	if (!skb->dev) {
+		kfree_skb(skb);
+		return NULL;
+	}
+
+	/* The special tag's TCI is now in the hardware-accelerated VLAN
+	 * metadata. The switch preserves the TCI of tagged
+	 * frames, but inserts the ingress port's default tag for untagged frames.
+	 * Since the driver programs the default tag to 0, a zero TCI identifies
+	 * an originally untagged frame. Preserve every non-zero TCI as an
+	 * 802.1Q tag, including VID 0 frames with PCP or DEI set.
+	 */
+	if (!skb->vlan_tci) {
+		netdev_dbg(dev, "%s: clear VLAN tag from frame\n", __func__);
+		__vlan_hwaccel_clear_tag(skb);
+	} else {
+		skb->vlan_proto = htons(ETH_P_8021Q);
+		netdev_dbg(dev, "%s: vlan_tci = 0x%04x VLAN frame\n",
+			   __func__, skb->vlan_tci);
+	}
+
+	dsa_default_offload_fwd_mark(skb);
+
+	return skb;
+}
+
+static void ks8995_flow_dissect(const struct sk_buff *skb, __be16 *proto,
+				int *offset)
+{
+	const __be16 *encap_proto;
+	__be16 buffer;
+
+	*proto = 0;
+	*offset = 0;
+
+	/* Extra TX headroom does not increase the in-band RX tag length. */
+	encap_proto = skb_header_pointer(skb, KS8995_TAG_LEN - sizeof(buffer),
+					 sizeof(buffer), &buffer);
+	if (!encap_proto)
+		return;
+
+	*proto = *encap_proto;
+	*offset = KS8995_TAG_LEN;
+}
+
+static const struct dsa_device_ops ks8995_netdev_ops = {
+	.name = KS8995_NAME,
+	.proto	= DSA_TAG_PROTO_KS8995,
+	.xmit = ks8995_xmit,
+	.rcv = ks8995_rcv,
+	.flow_dissect = ks8995_flow_dissect,
+	.needed_headroom = KS8995_NEEDED_HEADROOM,
+};
+
+MODULE_DESCRIPTION("DSA tag driver for Micrel KS8995 family of switches");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_DSA_TAG_DRIVER(DSA_TAG_PROTO_KS8995, KS8995_NAME);
+
+module_dsa_tag_driver(ks8995_netdev_ops);
diff --git a/net/dsa/tag_ksz.c b/net/dsa/tag_ksz.c
index b4d70ba930dcbe..b0340a7be8c9a6 100644
--- a/net/dsa/tag_ksz.c
+++ b/net/dsa/tag_ksz.c
@@ -41,10 +41,7 @@ static void ksz_hwtstamp_set_state(struct dsa_switch *ds, bool on)
 {
 	struct ksz_tagger_private *priv = ksz_tagger_private(ds);
 
-	if (on)
-		set_bit(KSZ_HWTS_EN, &priv->state);
-	else
-		clear_bit(KSZ_HWTS_EN, &priv->state);
+	assign_bit(KSZ_HWTS_EN, &priv->state, on);
 }
 
 static void ksz_disconnect(struct dsa_switch *ds)
diff --git a/net/ethtool/common.c b/net/ethtool/common.c
index 23db40618fed14..b4e766e60d38ae 100644
--- a/net/ethtool/common.c
+++ b/net/ethtool/common.c
@@ -2,6 +2,9 @@
 
 #include <linux/ethtool_netlink.h>
 #include <linux/net_tstamp.h>
+#include <linux/once.h>
+#include <linux/random.h>
+#include <linux/unaligned.h>
 #include <linux/phy.h>
 #include <linux/rtnetlink.h>
 #include <linux/ptp_clock_kernel.h>
@@ -1400,3 +1403,174 @@ enum ethtool_link_medium ethtool_str_to_medium(const char *str)
 	return ETHTOOL_LINK_MEDIUM_NONE;
 }
 EXPORT_SYMBOL_GPL(ethtool_str_to_medium);
+
+u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
+bool netdev_rss_key_initialized __read_mostly;
+
+/* Toeplitz is linear over GF(2): the hash is the XOR of the 32-bit key
+ * windows selected by the set bits of the input, and hardware indexes the
+ * indirection table with the low order bits of the hash. Only the tail of
+ * each window therefore matters for queue selection:
+ *
+ *	v(i) = key bits [i + 32 - q .. i + 31]
+ *
+ * for input bit @i, with q = log2(number of RX queues). Consecutive input
+ * bits give windows overlapping in q - 1 positions, so the matrix formed by
+ * the windows of the q lowest bits of a header field is a Toeplitz matrix
+ * built from 2 * q - 1 key bits, not from q * q independent ones. A random
+ * Toeplitz matrix over GF(2) is singular with probability 1/2, whatever its
+ * size, and when it is singular the flows of a burst differing only in the
+ * low order bits of that field (consecutive ephemeral ports, typically)
+ * cannot reach all the RX queues no matter how many of them are configured.
+ *
+ * Keep the key random, but constrain the few bits that decide this. The
+ * fixup is applied at every 16-bit aligned position of the key, rather than
+ * at the offsets of the one hash input layout the software happens to know
+ * about: hardware is free to hash whatever it wants, but the fields it picks
+ * are 16 bits wide at the smallest and are not expected to straddle that
+ * grid, so an encapsulated or offloaded layout is covered like the usual
+ * 2-tuple and 4-tuple ones. Each position constrains 2 * q - 1 bits, so
+ * NETDEV_RSS_KEY_QMAX is both enough for 256 queues and the largest value
+ * keeping the ranges of two adjacent positions disjoint.
+ */
+#define NETDEV_RSS_KEY_QMAX	8
+#define NETDEV_RSS_KEY_SPAN	(2 * NETDEV_RSS_KEY_QMAX - 1)
+
+static bool netdev_rss_key_bit(const u8 *key, unsigned int bit)
+{
+	return key[bit / BITS_PER_BYTE] & (0x80 >> (bit % BITS_PER_BYTE));
+}
+
+static void netdev_rss_key_assign_bit(u8 *key, unsigned int bit, bool value)
+{
+	u8 mask = 0x80 >> (bit % BITS_PER_BYTE);
+
+	if (value)
+		key[bit / BITS_PER_BYTE] |= mask;
+	else
+		key[bit / BITS_PER_BYTE] &= ~mask;
+}
+
+/* Writing d[t] for key bit (@lsb + 31 - t), the matrices of all the q values
+ * up to NETDEV_RSS_KEY_QMAX are non singular if and only if
+ *
+ *	d[2 * i] = 1 ^ d[i] ^ d[i + 1] ^ ... ^ d[2 * i - 1]
+ *
+ * The odd positions stay free, so this is a one pass fixup rather than a
+ * search. It is also a bijection onto the set of the values having the
+ * property, so the key stays uniformly distributed over that set. It costs
+ * NETDEV_RSS_KEY_QMAX bits of entropy per position.
+ */
+static void netdev_rss_key_fixup_field(u8 *key, unsigned int lsb)
+{
+	bool d[NETDEV_RSS_KEY_SPAN];
+	unsigned int i, t;
+
+	for (t = 0; t < NETDEV_RSS_KEY_SPAN; t++)
+		d[t] = netdev_rss_key_bit(key, lsb + 31 - t);
+
+	for (i = 0; 2 * i < NETDEV_RSS_KEY_SPAN; i++) {
+		bool value = true;
+
+		for (t = i; t < 2 * i; t++)
+			value ^= d[t];
+
+		d[2 * i] = value;
+	}
+
+	for (t = 0; t < NETDEV_RSS_KEY_SPAN; t++)
+		netdev_rss_key_assign_bit(key, lsb + 31 - t, d[t]);
+}
+
+/* The 32 key bits starting at @bit, which is what input bit @bit contributes
+ * to the hash. @bit + 32 must fit in the key.
+ */
+static u32 netdev_rss_key_window(const u8 *key, unsigned int bit)
+{
+	unsigned int byte = bit / BITS_PER_BYTE;
+	unsigned int shift = bit % BITS_PER_BYTE;
+	u32 window = get_unaligned_be32(key + byte);
+
+	if (shift)
+		window = (window << shift) |
+			 (key[byte + 4] >> (BITS_PER_BYTE - shift));
+
+	return window;
+}
+
+/* Two input bits contributing the same window are indistinguishable to the
+ * hash, since flipping both of them leaves it unchanged. For a uniformly
+ * random key that is a 2 ** -32 event per pair of positions, but the fixup
+ * makes it likelier: it derives 8 of every 16 bits from the 8 others, so a
+ * 16-bit aligned word only takes 2 ** 8 values and two windows a whole
+ * number of words apart collide with probability 2 ** -16 instead. Half a
+ * word apart is less affected but still well clear of the random odds, so
+ * cover every distance that is a multiple of 8. What is left after that is
+ * below what a plain random key gives.
+ *
+ * Two windows at a distance that is a multiple of 8 are two windows at the
+ * same offset modulo 8. Caching a whole class would put 253 u32 on the
+ * stack, so cache one class modulo 16 and stream the class 8 bits above it
+ * against it.
+ */
+static bool netdev_rss_key_aliases(const u8 *key, unsigned int bits)
+{
+	u32 windows[NETDEV_RSS_KEY_LEN * BITS_PER_BYTE / 16];
+	unsigned int i, j, n, r;
+
+	for (r = 0; r < 16; r++) {
+		n = 0;
+		for (i = r; i + 32 <= bits; i += 16)
+			windows[n++] = netdev_rss_key_window(key, i);
+
+		for (i = 0; i < n; i++)
+			for (j = i + 1; j < n; j++)
+				if (windows[i] == windows[j])
+					return true;
+
+		if (r >= 8)
+			continue;
+
+		for (i = r + 8; i + 32 <= bits; i += 16) {
+			u32 window = netdev_rss_key_window(key, i);
+
+			for (j = 0; j < n; j++)
+				if (windows[j] == window)
+					return true;
+		}
+	}
+
+	return false;
+}
+
+static void netdev_rss_key_init(u8 *key, size_t len)
+{
+	unsigned int lsb, bits = len * BITS_PER_BYTE;
+
+	/* Four keys out of five come out of the fixup free of aliases, so
+	 * drawing another one is both simpler and cheaper than repairing.
+	 */
+	do {
+		get_random_bytes(key, len);
+
+		/* A field ending at bit @lsb uses key bits [.. , @lsb + 31],
+		 * so stop as soon as a 32-bit window no longer fits in the
+		 * key.
+		 */
+		for (lsb = 15; lsb + 32 <= bits; lsb += 16)
+			netdev_rss_key_fixup_field(key, lsb);
+	} while (netdev_rss_key_aliases(key, bits));
+
+	/* Pair with smp_rmb() in proc_do_rss_key(). */
+	smp_wmb();
+	WRITE_ONCE(netdev_rss_key_initialized, true);
+}
+
+void netdev_rss_key_fill(void *buffer, size_t len)
+{
+	if (WARN_ON_ONCE(len > sizeof(netdev_rss_key)))
+		len = sizeof(netdev_rss_key);
+	DO_ONCE(netdev_rss_key_init, netdev_rss_key, sizeof(netdev_rss_key));
+	memcpy(buffer, netdev_rss_key, len);
+}
+EXPORT_SYMBOL(netdev_rss_key_fill);
diff --git a/net/ethtool/common.h b/net/ethtool/common.h
index 4e5356e26f400a..ae32e7fdb563c1 100644
--- a/net/ethtool/common.h
+++ b/net/ethtool/common.h
@@ -163,6 +163,8 @@ ethtool_ioctl_needs_rtnl(const struct net_device *dev, u32 ethcmd)
 		return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_RSS;
 	case ETHTOOL_GLINK:
 		return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_GLINK;
+	case ETHTOOL_TEST:
+		return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_TEST;
 	}
 	return false;
 }
diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c
index 4b0bc503f93078..2b449d08ddcf30 100644
--- a/net/ethtool/ioctl.c
+++ b/net/ethtool/ioctl.c
@@ -1301,15 +1301,6 @@ static int ethtool_copy_validate_indir(u32 *indir, void __user *useraddr,
 	return 0;
 }
 
-u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
-
-void netdev_rss_key_fill(void *buffer, size_t len)
-{
-	BUG_ON(len > sizeof(netdev_rss_key));
-	net_get_random_once(netdev_rss_key, sizeof(netdev_rss_key));
-	memcpy(buffer, netdev_rss_key, len);
-}
-EXPORT_SYMBOL(netdev_rss_key_fill);
 
 static noinline_for_stack int ethtool_get_rxfh_indir(struct net_device *dev,
 						     void __user *useraddr)
diff --git a/net/ethtool/ts.h b/net/ethtool/ts.h
index d901a879a67100..ab980530835352 100644
--- a/net/ethtool/ts.h
+++ b/net/ethtool/ts.h
@@ -5,9 +5,15 @@
 
 #include "netlink.h"
 
+/* phc_index is an int and -1 means no PHC, so keep the request non-negative */
+static const struct netlink_range_validation ethnl_ts_prov_index_range = {
+	.max = INT_MAX,
+};
+
 static const struct nla_policy
 ethnl_ts_hwtst_prov_policy[ETHTOOL_A_TS_HWTSTAMP_PROVIDER_MAX + 1] = {
-	[ETHTOOL_A_TS_HWTSTAMP_PROVIDER_INDEX] = { .type = NLA_U32 },
+	[ETHTOOL_A_TS_HWTSTAMP_PROVIDER_INDEX] =
+		NLA_POLICY_FULL_RANGE(NLA_U32, &ethnl_ts_prov_index_range),
 	[ETHTOOL_A_TS_HWTSTAMP_PROVIDER_QUALIFIER] =
 		NLA_POLICY_MAX(NLA_U32, HWTSTAMP_PROVIDER_QUALIFIER_CNT - 1)
 };
diff --git a/net/ethtool/tsconfig.c b/net/ethtool/tsconfig.c
index 6be3aa5d4bc1ed..2db0e7ba8b9f56 100644
--- a/net/ethtool/tsconfig.c
+++ b/net/ethtool/tsconfig.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 #include <linux/net_tstamp.h>
+#include <linux/phy.h>
 #include <linux/ptp_clock_kernel.h>
 #include <net/netdev_lock.h>
 
@@ -42,7 +43,9 @@ static int tsconfig_prepare_data(const struct ethnl_req_info *req_base,
 	struct kernel_hwtstamp_config cfg = {};
 	int ret;
 
-	if (!dev->netdev_ops->ndo_hwtstamp_get)
+	if (!dev->netdev_ops->ndo_hwtstamp_get &&
+	    !phy_is_default_hwtstamp(dev->phydev) &&
+	    !netdev_ops_lock_dereference(dev->hwprov, dev))
 		return -EOPNOTSUPP;
 
 	ret = ethnl_ops_begin(dev);
@@ -248,17 +251,6 @@ static int tsconfig_send_reply(struct net_device *dev, struct genl_info *info)
 	return ret;
 }
 
-static int ethnl_set_tsconfig_validate(struct ethnl_req_info *req_base,
-				       struct genl_info *info)
-{
-	const struct net_device_ops *ops = req_base->dev->netdev_ops;
-
-	if (!ops->ndo_hwtstamp_set || !ops->ndo_hwtstamp_get)
-		return -EOPNOTSUPP;
-
-	return 1;
-}
-
 static struct hwtstamp_provider *
 tsconfig_set_hwprov_from_desc(struct net_device *dev,
 			      struct genl_info *info,
@@ -272,7 +264,7 @@ tsconfig_set_hwprov_from_desc(struct net_device *dev,
 	int ret;
 
 	ret = ethtool_net_get_ts_info_by_phc(dev, &ts_info, hwprov_desc);
-	if (!ret) {
+	if (!ret && dev->netdev_ops->ndo_hwtstamp_set) {
 		/* Found */
 		source = HWTSTAMP_SOURCE_NETDEV;
 	} else {
@@ -313,6 +305,11 @@ static int ethnl_set_tsconfig(struct ethnl_req_info *req_base,
 	if (!netif_device_present(dev))
 		return -ENODEV;
 
+	if (!dev->netdev_ops->ndo_hwtstamp_set &&
+	    !phy_is_default_hwtstamp(dev->phydev) &&
+	    !netdev_ops_lock_dereference(dev->hwprov, dev))
+		return -EOPNOTSUPP;
+
 	if (tb[ETHTOOL_A_TSCONFIG_HWTSTAMP_PROVIDER]) {
 		struct hwtstamp_provider_desc __hwprov_desc = {.index = -1};
 		struct hwtstamp_provider *__hwprov;
@@ -459,6 +456,5 @@ const struct ethnl_request_ops ethnl_tsconfig_request_ops = {
 	.reply_size		= tsconfig_reply_size,
 	.fill_reply		= tsconfig_fill_reply,
 
-	.set_validate		= ethnl_set_tsconfig_validate,
 	.set			= ethnl_set_tsconfig,
 };
diff --git a/net/handshake/genl.c b/net/handshake/genl.c
index feac1ad063ee72..26e3efec64ca74 100644
--- a/net/handshake/genl.c
+++ b/net/handshake/genl.c
@@ -14,14 +14,14 @@
 
 /* HANDSHAKE_CMD_ACCEPT - do */
 static const struct nla_policy handshake_accept_nl_policy[HANDSHAKE_A_ACCEPT_HANDLER_CLASS + 1] = {
-	[HANDSHAKE_A_ACCEPT_HANDLER_CLASS] = NLA_POLICY_MAX(NLA_U32, 2),
+	[HANDSHAKE_A_ACCEPT_HANDLER_CLASS] = NLA_POLICY_MAX(NLA_U32, 1),
 };
 
 /* HANDSHAKE_CMD_DONE - do */
 static const struct nla_policy handshake_done_nl_policy[HANDSHAKE_A_DONE_REMOTE_AUTH + 1] = {
 	[HANDSHAKE_A_DONE_STATUS] = NLA_POLICY_MAX(NLA_U32, MAX_ERRNO),
 	[HANDSHAKE_A_DONE_SOCKFD] = { .type = NLA_S32, },
-	[HANDSHAKE_A_DONE_REMOTE_AUTH] = { .type = NLA_U32, },
+	[HANDSHAKE_A_DONE_REMOTE_AUTH] = { .type = NLA_S32, },
 };
 
 /* Ops table for handshake */
diff --git a/net/handshake/handshake-test.c b/net/handshake/handshake-test.c
index 3dd507470d5fea..4b99acd9f9c7e7 100644
--- a/net/handshake/handshake-test.c
+++ b/net/handshake/handshake-test.c
@@ -42,7 +42,7 @@ static struct handshake_proto handshake_req_alloc_proto_2 = {
 };
 
 static struct handshake_proto handshake_req_alloc_proto_3 = {
-	.hp_handler_class	= HANDSHAKE_HANDLER_CLASS_MAX,
+	.hp_handler_class	= __HANDSHAKE_HANDLER_CLASS_MAX,
 };
 
 static struct handshake_proto handshake_req_alloc_proto_4 = {
diff --git a/net/handshake/netlink.c b/net/handshake/netlink.c
index 3fd4fef9bab1a4..73b8314d9010ce 100644
--- a/net/handshake/netlink.c
+++ b/net/handshake/netlink.c
@@ -107,17 +107,17 @@ int handshake_nl_accept_doit(struct sk_buff *skb, struct genl_info *info)
 	req = handshake_req_next(hn, class);
 	if (req) {
 		FD_PREPARE(fdf, O_CLOEXEC, req->hr_file);
-		if (fdf.err) {
+		if (fdf->fd < 0) {
 			fput(req->hr_file); /* drop ref from handshake_req_next() */
-			err = fdf.err;
+			err = fdf->fd;
 			goto out_complete;
 		}
 
-		err = req->hr_proto->hp_accept(req, info, fd_prepare_fd(fdf));
+		err = req->hr_proto->hp_accept(req, info, fdf->fd);
 		if (err)
 			goto out_complete; /* Automatic cleanup handles fput */
 
-		trace_handshake_cmd_accept(net, req, req->hr_sk, fd_prepare_fd(fdf));
+		trace_handshake_cmd_accept(net, req, req->hr_sk, fdf->fd);
 		fd_publish(fdf);
 		return 0;
 	}
diff --git a/net/handshake/request.c b/net/handshake/request.c
index cd30d54d0501d3..ff739ee81cb82c 100644
--- a/net/handshake/request.c
+++ b/net/handshake/request.c
@@ -115,7 +115,7 @@ struct handshake_req *handshake_req_alloc(const struct handshake_proto *proto,
 		return NULL;
 	if (proto->hp_handler_class <= HANDSHAKE_HANDLER_CLASS_NONE)
 		return NULL;
-	if (proto->hp_handler_class >= HANDSHAKE_HANDLER_CLASS_MAX)
+	if (proto->hp_handler_class > HANDSHAKE_HANDLER_CLASS_MAX)
 		return NULL;
 	if (!proto->hp_accept || !proto->hp_done)
 		return NULL;
diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c
index d14de44e14b7a9..fc512204600cd8 100644
--- a/net/hsr/hsr_device.c
+++ b/net/hsr/hsr_device.c
@@ -333,7 +333,7 @@ static void send_hsr_supervision_frame(struct hsr_port *port,
 		hsr->sup_sequence_nr++;
 	} else {
 		hsr_stag->sequence_nr = htons(hsr->sequence_nr);
-		hsr->sequence_nr++;
+		WRITE_ONCE(hsr->sequence_nr, hsr->sequence_nr + 1);
 	}
 
 	hsr_stag->tlv.HSR_TLV_type = type;
diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c
index 7734a521a96c23..44f6d9c83180a6 100644
--- a/net/hsr/hsr_forward.c
+++ b/net/hsr/hsr_forward.c
@@ -645,7 +645,7 @@ static void handle_std_frame(struct sk_buff *skb,
 		/* Sequence nr for the master/interlink node */
 		lockdep_assert_held(&hsr->seqnr_lock);
 		frame->sequence_nr = hsr->sequence_nr;
-		hsr->sequence_nr++;
+		WRITE_ONCE(hsr->sequence_nr, hsr->sequence_nr + 1);
 	}
 }
 
diff --git a/net/hsr/hsr_netlink.c b/net/hsr/hsr_netlink.c
index 88940e8014b24f..fc7820eae88413 100644
--- a/net/hsr/hsr_netlink.c
+++ b/net/hsr/hsr_netlink.c
@@ -147,44 +147,61 @@ static void hsr_dellink(struct net_device *dev, struct list_head *head)
 	unregister_netdevice_queue(dev, head);
 }
 
+/* RCU variant of hsr_port_get_hsr() */
+static struct hsr_port *hsr_port_get_hsr_rcu(const struct hsr_priv *hsr,
+					     enum hsr_port_type pt)
+{
+	struct hsr_port *port;
+
+	hsr_for_each_port(hsr, port)
+		if (port->type == pt)
+			return port;
+	return NULL;
+}
+
 static int hsr_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
-	struct hsr_priv *hsr = netdev_priv(dev);
+	const struct hsr_priv *hsr = netdev_priv(dev);
+	const struct hsr_port *port;
 	u8 proto = HSR_PROTOCOL_HSR;
-	struct hsr_port *port;
 
-	port = hsr_port_get_hsr(hsr, HSR_PT_SLAVE_A);
+	rcu_read_lock();
+
+	port = hsr_port_get_hsr_rcu(hsr, HSR_PT_SLAVE_A);
 	if (port) {
 		if (nla_put_u32(skb, IFLA_HSR_SLAVE1, port->dev->ifindex))
 			goto nla_put_failure;
 	}
 
-	port = hsr_port_get_hsr(hsr, HSR_PT_SLAVE_B);
+	port = hsr_port_get_hsr_rcu(hsr, HSR_PT_SLAVE_B);
 	if (port) {
 		if (nla_put_u32(skb, IFLA_HSR_SLAVE2, port->dev->ifindex))
 			goto nla_put_failure;
 	}
 
-	port = hsr_port_get_hsr(hsr, HSR_PT_INTERLINK);
+	port = hsr_port_get_hsr_rcu(hsr, HSR_PT_INTERLINK);
 	if (port) {
 		if (nla_put_u32(skb, IFLA_HSR_INTERLINK, port->dev->ifindex))
 			goto nla_put_failure;
 	}
 
+	rcu_read_unlock();
+
 	if (nla_put(skb, IFLA_HSR_SUPERVISION_ADDR, ETH_ALEN,
 		    hsr->sup_multicast_addr) ||
-	    nla_put_u16(skb, IFLA_HSR_SEQ_NR, hsr->sequence_nr))
-		goto nla_put_failure;
+	    nla_put_u16(skb, IFLA_HSR_SEQ_NR, READ_ONCE(hsr->sequence_nr)))
+		return -EMSGSIZE;
 	if (hsr->prot_version == PRP_V1)
 		proto = HSR_PROTOCOL_PRP;
 	else if (nla_put_u8(skb, IFLA_HSR_VERSION, hsr->prot_version))
-		goto nla_put_failure;
+		return -EMSGSIZE;
 	if (nla_put_u8(skb, IFLA_HSR_PROTOCOL, proto))
-		goto nla_put_failure;
+		return -EMSGSIZE;
 
 	return 0;
 
 nla_put_failure:
+	rcu_read_unlock();
 	return -EMSGSIZE;
 }
 
diff --git a/net/ieee802154/6lowpan/tx.c b/net/ieee802154/6lowpan/tx.c
index 4df76ff50699ed..4f511476b992c7 100644
--- a/net/ieee802154/6lowpan/tx.c
+++ b/net/ieee802154/6lowpan/tx.c
@@ -58,8 +58,9 @@ int lowpan_header_create(struct sk_buff *skb, struct net_device *ldev,
 		info->daddr.mode = IEEE802154_ADDR_SHORT;
 	} else {
 		__le16 short_addr = cpu_to_le16(IEEE802154_ADDR_SHORT_UNSPEC);
+		struct neigh_table *tbl = nd_table(dev_net(ldev));
 
-		n = neigh_lookup(&nd_tbl, &hdr->daddr, ldev);
+		n = neigh_lookup(tbl, &hdr->daddr, ldev);
 		if (n) {
 			llneigh = lowpan_802154_neigh(neighbour_priv(n));
 			read_lock_bh(&n->lock);
diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c
index 5a36e87893f647..be8ca1f053a2b6 100644
--- a/net/ieee802154/socket.c
+++ b/net/ieee802154/socket.c
@@ -139,8 +139,9 @@ static int ieee802154_dev_ioctl(struct sock *sk, struct ifreq __user *arg,
 	if (!dev)
 		return -ENODEV;
 
-	if (dev->type == ARPHRD_IEEE802154 && dev->netdev_ops->ndo_do_ioctl)
-		ret = dev->netdev_ops->ndo_do_ioctl(dev, &ifr, cmd);
+	if (dev->type == ARPHRD_IEEE802154 && dev->ieee802154_ptr &&
+	    dev->ieee802154_ptr->ops)
+		ret = dev->ieee802154_ptr->ops->ioctl(dev, &ifr, cmd);
 
 	if (!ret && put_user_ifreq(&ifr, arg))
 		ret = -EFAULT;
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index 06e21c26b76f2e..afbac63d1cb437 100644
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -70,6 +70,7 @@ obj-$(CONFIG_TCP_AO) += tcp_ao.o
 
 ifeq ($(CONFIG_BPF_JIT),y)
 obj-$(CONFIG_BPF_SYSCALL) += bpf_tcp_ca.o
+obj-$(CONFIG_CGROUP_BPF) += bpf_tcp_ops.o
 endif
 
 ifdef CONFIG_GCOV_PROFILE_NETFILTER
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 32d006c1a8eeda..e94154c0ac89c9 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -227,6 +227,7 @@ int __inet_listen_sk(struct sock *sk, int backlog)
 			return err;
 
 		tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL);
+		bpf_tcp_ops_call(listen, sk);
 	}
 	return 0;
 }
@@ -582,7 +583,6 @@ int inet_dgram_connect(struct socket *sock, struct sockaddr_unsized *uaddr,
 	if (addr_len < sizeof(uaddr->sa_family))
 		return -EINVAL;
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	prot = READ_ONCE(sk->sk_prot);
 
 	if (uaddr->sa_family == AF_UNSPEC)
@@ -789,7 +789,6 @@ int inet_accept(struct socket *sock, struct socket *newsock,
 {
 	struct sock *sk1 = sock->sk, *sk2;
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	arg->err = -EINVAL;
 	sk2 = READ_ONCE(sk1->sk_prot)->accept(sk1, arg);
 	if (!sk2)
@@ -875,7 +874,6 @@ void inet_splice_eof(struct socket *sock)
 	if (unlikely(inet_send_prepare(sk)))
 		return;
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	prot = READ_ONCE(sk->sk_prot);
 	if (prot->splice_eof)
 		prot->splice_eof(sock);
@@ -1375,6 +1373,9 @@ struct sk_buff *inet_gso_segment(struct sk_buff *skb,
 	int id;
 
 	skb_reset_network_header(skb);
+	if (unlikely(gso_recursion_inc_test(skb,
+					   IP_TUNNEL_RECURSION_LIMIT)))
+		goto out;
 	nhoff = skb_network_header(skb) - skb_mac_header(skb);
 	if (unlikely(!pskb_may_pull(skb, sizeof(*iph))))
 		goto out;
diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c
index d409f606aec0e1..95f3359d0f9292 100644
--- a/net/ipv4/arp.c
+++ b/net/ipv4/arp.c
@@ -149,7 +149,7 @@ static const struct neigh_ops arp_direct_ops = {
 	.connected_output =	neigh_direct_output,
 };
 
-struct neigh_table arp_tbl = {
+static struct neigh_table arp_tbl = {
 	.family		= AF_INET,
 	.key_len	= 4,
 	.protocol	= cpu_to_be16(ETH_P_IP),
@@ -182,7 +182,6 @@ struct neigh_table arp_tbl = {
 	.gc_thresh2	= 512,
 	.gc_thresh3	= 1024,
 };
-EXPORT_SYMBOL(arp_tbl);
 
 int arp_mc_map(__be32 addr, u8 *haddr, struct net_device *dev, int dir)
 {
@@ -222,14 +221,18 @@ static bool arp_key_eq(const struct neighbour *neigh, const void *pkey)
 
 static int arp_constructor(struct neighbour *neigh)
 {
-	__be32 addr;
 	struct net_device *dev = neigh->dev;
-	struct in_device *in_dev;
-	struct neigh_parms *parms;
+	struct net *net = dev_net(dev);
 	u32 inaddr_any = INADDR_ANY;
+	struct neigh_parms *parms;
+	struct in_device *in_dev;
+	struct neigh_table *tbl;
+	__be32 addr;
+
+	tbl = arp_table(net);
 
 	if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))
-		memcpy(neigh->primary_key, &inaddr_any, arp_tbl.key_len);
+		memcpy(neigh->primary_key, &inaddr_any, tbl->key_len);
 
 	addr = *(__be32 *)neigh->primary_key;
 	rcu_read_lock();
@@ -239,7 +242,7 @@ static int arp_constructor(struct neighbour *neigh)
 		return -EINVAL;
 	}
 
-	neigh->type = inet_addr_type_dev_table(dev_net(dev), dev, addr);
+	neigh->type = inet_addr_type_dev_table(net, dev, addr);
 
 	parms = in_dev->arp_parms;
 	__neigh_parms_put(neigh->parms);
@@ -701,24 +704,24 @@ static bool arp_is_garp(struct net *net, struct net_device *dev,
 
 static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb)
 {
+	struct neigh_table *tbl = arp_table(net);
+	struct dst_entry *reply_dst = NULL;
 	struct net_device *dev = skb->dev;
-	struct in_device *in_dev = __in_dev_get_rcu(dev);
-	struct arphdr *arp;
+	unsigned char *sha, *tha = NULL;
+	struct in_device *in_dev;
+	u16 dev_type = dev->type;
 	unsigned char *arp_ptr;
+	bool is_garp = false;
+	struct neighbour *n;
+	struct arphdr *arp;
 	struct rtable *rt;
-	unsigned char *sha;
-	unsigned char *tha = NULL;
 	__be32 sip, tip;
-	u16 dev_type = dev->type;
 	int addr_type;
-	struct neighbour *n;
-	struct dst_entry *reply_dst = NULL;
-	bool is_garp = false;
 
 	/* arp_rcv below verifies the ARP header and verifies the device
 	 * is ARP'able.
 	 */
-
+	in_dev = __in_dev_get_rcu(dev);
 	if (!in_dev)
 		goto out_free_skb;
 
@@ -850,7 +853,7 @@ static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb)
 			if (!dont_send && IN_DEV_ARPFILTER(in_dev))
 				dont_send = arp_filter(sip, tip, dev);
 			if (!dont_send) {
-				n = neigh_event_ns(&arp_tbl, sha, &sip, dev);
+				n = neigh_event_ns(tbl, sha, &sip, dev);
 				if (n) {
 					arp_send_dst(ARPOP_REPLY, ETH_P_ARP,
 						     sip, dev, tip, sha,
@@ -865,8 +868,8 @@ static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb)
 			    (arp_fwd_proxy(in_dev, dev, rt) ||
 			     arp_fwd_pvlan(in_dev, dev, rt, sip, tip) ||
 			     (rt->dst.dev != dev &&
-			      pneigh_lookup(&arp_tbl, net, &tip, dev)))) {
-				n = neigh_event_ns(&arp_tbl, sha, &sip, dev);
+			      pneigh_lookup(tbl, &tip, dev)))) {
+				n = neigh_event_ns(tbl, sha, &sip, dev);
 				if (n)
 					neigh_release(n);
 
@@ -878,7 +881,7 @@ static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb)
 						     dev->dev_addr, sha,
 						     reply_dst);
 				} else {
-					pneigh_enqueue(&arp_tbl,
+					pneigh_enqueue(tbl,
 						       in_dev->arp_parms, skb);
 					goto out_free_dst;
 				}
@@ -889,7 +892,7 @@ static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb)
 
 	/* Update our ARP tables */
 
-	n = __neigh_lookup(&arp_tbl, &sip, dev, 0);
+	n = __neigh_lookup(tbl, &sip, dev, 0);
 
 	addr_type = -1;
 	if (n || arp_accept(in_dev, sip)) {
@@ -910,7 +913,7 @@ static int arp_process(struct net *net, struct sock *sk, struct sk_buff *skb)
 			/* postpone calculation to as late as possible */
 			inet_addr_type_dev_table(net, dev, sip) ==
 				RTN_UNICAST)))))
-			n = __neigh_lookup(&arp_tbl, &sip, dev, 1);
+			n = __neigh_lookup(tbl, &sip, dev, 1);
 	}
 
 	if (n) {
@@ -1077,9 +1080,10 @@ static int arp_req_set_proxy(struct net *net, struct net_device *dev, int on)
 }
 
 static int arp_req_set_public(struct net *net, struct arpreq *r,
-		struct net_device *dev)
+			      struct net_device *dev)
 {
 	__be32 mask = ((struct sockaddr_in *)&r->arp_netmask)->sin_addr.s_addr;
+	struct neigh_table *tbl = arp_table(net);
 
 	if (!dev && (r->arp_flags & ATF_COM)) {
 		dev = dev_getbyhwaddr(net, r->arp_ha.sa_family,
@@ -1090,7 +1094,7 @@ static int arp_req_set_public(struct net *net, struct arpreq *r,
 	if (mask) {
 		__be32 ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;
 
-		return pneigh_create(&arp_tbl, net, &ip, dev, 0, 0, false);
+		return pneigh_create(tbl, &ip, dev, 0, 0, false);
 	}
 
 	return arp_req_set_proxy(net, dev, 1);
@@ -1098,6 +1102,7 @@ static int arp_req_set_public(struct net *net, struct arpreq *r,
 
 static int arp_req_set(struct net *net, struct arpreq *r)
 {
+	struct neigh_table *tbl = arp_table(net);
 	struct neighbour *neigh;
 	struct net_device *dev;
 	__be32 ip;
@@ -1133,7 +1138,7 @@ static int arp_req_set(struct net *net, struct arpreq *r)
 
 	ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;
 
-	neigh = __neigh_lookup_errno(&arp_tbl, &ip, dev);
+	neigh = __neigh_lookup_errno(tbl, &ip, dev);
 	err = PTR_ERR(neigh);
 	if (!IS_ERR(neigh)) {
 		unsigned int state = NUD_STALE;
@@ -1169,6 +1174,7 @@ static unsigned int arp_state_to_flags(struct neighbour *neigh)
 static int arp_req_get(struct net *net, struct arpreq *r)
 {
 	__be32 ip = ((struct sockaddr_in *) &r->arp_pa)->sin_addr.s_addr;
+	struct neigh_table *tbl = arp_table(net);
 	struct neighbour *neigh;
 	struct net_device *dev;
 
@@ -1179,7 +1185,7 @@ static int arp_req_get(struct net *net, struct arpreq *r)
 	if (IS_ERR(dev))
 		return PTR_ERR(dev);
 
-	neigh = neigh_lookup(&arp_tbl, &ip, dev);
+	neigh = neigh_lookup(tbl, &ip, dev);
 	if (!neigh)
 		return -ENXIO;
 
@@ -1204,10 +1210,11 @@ static int arp_req_get(struct net *net, struct arpreq *r)
 
 int arp_invalidate(struct net_device *dev, __be32 ip, bool force)
 {
-	struct neighbour *neigh = neigh_lookup(&arp_tbl, &ip, dev);
+	struct neigh_table *tbl = arp_table(dev_net(dev));
+	struct neighbour *neigh;
 	int err = -ENXIO;
-	struct neigh_table *tbl = &arp_tbl;
 
+	neigh = neigh_lookup(tbl, &ip, dev);
 	if (neigh) {
 		if ((READ_ONCE(neigh->nud_state) & NUD_VALID) && !force) {
 			neigh_release(neigh);
@@ -1228,14 +1235,15 @@ int arp_invalidate(struct net_device *dev, __be32 ip, bool force)
 }
 
 static int arp_req_delete_public(struct net *net, struct arpreq *r,
-		struct net_device *dev)
+				 struct net_device *dev)
 {
 	__be32 mask = ((struct sockaddr_in *)&r->arp_netmask)->sin_addr.s_addr;
+	struct neigh_table *tbl = arp_table(net);
 
 	if (mask) {
 		__be32 ip = ((struct sockaddr_in *)&r->arp_pa)->sin_addr.s_addr;
 
-		return pneigh_delete(&arp_tbl, net, &ip, dev);
+		return pneigh_delete(tbl, &ip, dev);
 	}
 
 	return arp_req_set_proxy(net, dev, 0);
@@ -1278,6 +1286,7 @@ int arp_ioctl(struct net *net, unsigned int cmd, void __user *arg)
 		err = copy_from_user(&r, arg, sizeof(struct arpreq));
 		if (err)
 			return -EFAULT;
+		r.arp_dev[IFNAMSIZ - 1] = '\0';
 		break;
 	default:
 		return -EINVAL;
@@ -1325,18 +1334,22 @@ static int arp_netdev_event(struct notifier_block *this, unsigned long event,
 {
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 	struct netdev_notifier_change_info *change_info;
+	struct net *net = dev_net(dev);
 	struct in_device *in_dev;
+	struct neigh_table *tbl;
 	bool evict_nocarrier;
 
+	tbl = arp_table(net);
+
 	switch (event) {
 	case NETDEV_CHANGEADDR:
-		neigh_changeaddr(&arp_tbl, dev);
-		rt_cache_flush(dev_net(dev));
+		neigh_changeaddr(tbl, dev);
+		rt_cache_flush(net);
 		break;
 	case NETDEV_CHANGE:
 		change_info = ptr;
 		if (change_info->flags_changed & IFF_NOARP)
-			neigh_changeaddr(&arp_tbl, dev);
+			neigh_changeaddr(tbl, dev);
 
 		in_dev = __in_dev_get_rtnl(dev);
 		if (!in_dev)
@@ -1345,7 +1358,7 @@ static int arp_netdev_event(struct notifier_block *this, unsigned long event,
 			evict_nocarrier = IN_DEV_ARP_EVICT_NOCARRIER(in_dev);
 
 		if (evict_nocarrier && !netif_carrier_ok(dev))
-			neigh_carrier_down(&arp_tbl, dev);
+			neigh_carrier_down(tbl, dev);
 		break;
 	default:
 		break;
@@ -1364,7 +1377,9 @@ static struct notifier_block arp_netdev_notifier = {
  */
 void arp_ifdown(struct net_device *dev)
 {
-	neigh_ifdown(&arp_tbl, dev);
+	struct neigh_table *tbl = arp_table(dev_net(dev));
+
+	neigh_ifdown(tbl, dev);
 }
 
 
@@ -1479,10 +1494,12 @@ static int arp_seq_show(struct seq_file *seq, void *v)
 
 static void *arp_seq_start(struct seq_file *seq, loff_t *pos)
 {
+	struct neigh_table *tbl = arp_table(seq_file_net(seq));
+
 	/* Don't want to confuse "arp -a" w/ magic entries,
 	 * so we tell the generic iterator to skip NUD_NOARP.
 	 */
-	return neigh_seq_start(seq, pos, &arp_tbl, NEIGH_SEQ_SKIP_NOARP);
+	return neigh_seq_start(seq, pos, tbl, NEIGH_SEQ_SKIP_NOARP);
 }
 
 static const struct seq_operations arp_seq_ops = {
@@ -1495,15 +1512,47 @@ static const struct seq_operations arp_seq_ops = {
 
 static int __net_init arp_net_init(struct net *net)
 {
+	int err;
+
+	err = neigh_table_register(net, &arp_tbl, NEIGH_ARP_TABLE);
+	if (err)
+		goto err;
+
+#ifdef CONFIG_PROC_FS
+#ifdef CONFIG_SYSCTL
+	err = neigh_sysctl_register(NULL, &arp_table(net)->parms, NULL);
+	if (err)
+		goto err_sysctl;
+#endif
+
 	if (!proc_create_net("arp", 0444, net->proc_net, &arp_seq_ops,
-			sizeof(struct neigh_seq_state)))
-		return -ENOMEM;
+			     sizeof(struct neigh_seq_state))) {
+		err = -ENOMEM;
+		goto err_proc_create;
+	}
+#endif
+
 	return 0;
+
+#ifdef CONFIG_PROC_FS
+err_proc_create:
+#ifdef CONFIG_SYSCTL
+	neigh_sysctl_unregister(&arp_table(net)->parms);
+err_sysctl:
+#endif
+	neigh_table_unregister(net, NEIGH_ARP_TABLE);
+#endif
+err:
+	return err;
 }
 
 static void __net_exit arp_net_exit(struct net *net)
 {
 	remove_proc_entry("arp", net->proc_net);
+#ifdef CONFIG_SYSCTL
+	neigh_sysctl_unregister(&arp_table(net)->parms);
+#endif
+	neigh_table_unregister(net, NEIGH_ARP_TABLE);
 }
 
 static struct pernet_operations arp_net_ops = {
@@ -1513,12 +1562,9 @@ static struct pernet_operations arp_net_ops = {
 
 void __init arp_init(void)
 {
-	neigh_table_init(NEIGH_ARP_TABLE, &arp_tbl);
+	if (register_pernet_subsys(&arp_net_ops))
+		panic("Cannot allocate arp table\n");
 
 	dev_add_pack(&arp_packet_type);
-	register_pernet_subsys(&arp_net_ops);
-#ifdef CONFIG_SYSCTL
-	neigh_sysctl_register(NULL, &arp_tbl.parms, NULL);
-#endif
 	register_netdevice_notifier(&arp_netdev_notifier);
 }
diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c
index 791e15063237c9..2a71dc81562f46 100644
--- a/net/ipv4/bpf_tcp_ca.c
+++ b/net/ipv4/bpf_tcp_ca.c
@@ -194,6 +194,7 @@ BTF_ID_FLAGS(func, tcp_reno_cong_avoid)
 BTF_ID_FLAGS(func, tcp_reno_undo_cwnd)
 BTF_ID_FLAGS(func, tcp_slow_start)
 BTF_ID_FLAGS(func, tcp_cong_avoid_ai)
+BTF_ID_FLAGS(func, tcp_tso_autosize)
 BTF_KFUNCS_END(bpf_tcp_ca_check_kfunc_ids)
 
 static const struct btf_kfunc_id_set bpf_tcp_ca_kfunc_set = {
@@ -284,7 +285,7 @@ static void bpf_tcp_ca_pkts_acked(struct sock *sk, const struct ack_sample *samp
 {
 }
 
-static u32 bpf_tcp_ca_min_tso_segs(struct sock *sk)
+static u32 bpf_tcp_ca_tso_segs(struct sock *sk, u32 mss_now)
 {
 	return 0;
 }
@@ -320,7 +321,7 @@ static struct tcp_congestion_ops __bpf_ops_tcp_congestion_ops = {
 	.cwnd_event_tx_start = bpf_tcp_ca_cwnd_event_tx_start,
 	.in_ack_event = bpf_tcp_ca_in_ack_event,
 	.pkts_acked = bpf_tcp_ca_pkts_acked,
-	.min_tso_segs = bpf_tcp_ca_min_tso_segs,
+	.tso_segs = bpf_tcp_ca_tso_segs,
 	.cong_control = bpf_tcp_ca_cong_control,
 	.undo_cwnd = bpf_tcp_ca_undo_cwnd,
 	.sndbuf_expand = bpf_tcp_ca_sndbuf_expand,
@@ -339,6 +340,22 @@ static struct bpf_struct_ops bpf_tcp_congestion_ops = {
 	.validate = bpf_tcp_ca_validate,
 	.name = "tcp_congestion_ops",
 	.cfi_stubs = &__bpf_ops_tcp_congestion_ops,
+	/* The struct_ops's function may switch to another struct_ops.
+	 *
+	 * For example, bpf_tcp_cc_x->init() may switch to
+	 * another tcp_cc_y by calling
+	 * setsockopt(TCP_CONGESTION, "tcp_cc_y").
+	 * During the switch,  bpf_struct_ops_put(tcp_cc_x) is called
+	 * and its refcount may reach 0 which then free its
+	 * trampoline image while tcp_cc_x is still running.
+	 *
+	 * A vanilla rcu gp is to wait for all bpf-tcp-cc prog
+	 * to finish. bpf-tcp-cc prog is non sleepable.
+	 * A rcu_tasks gp is to wait for the last few insn
+	 * in the tramopline image to finish before releasing
+	 * the trampoline image.
+	 */
+	.free_after_tasks_rcu_gp = true,
 	.owner = THIS_MODULE,
 };
 
diff --git a/net/ipv4/bpf_tcp_ops.c b/net/ipv4/bpf_tcp_ops.c
new file mode 100644
index 00000000000000..681fed64299994
--- /dev/null
+++ b/net/ipv4/bpf_tcp_ops.c
@@ -0,0 +1,326 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bpf.h>
+#include <linux/btf_ids.h>
+#include <linux/bpf_verifier.h>
+#include <linux/filter.h>
+#include <net/bpf_sk_storage.h>
+#include <net/tcp.h>
+
+static int timeout_init_stub(struct sock *sk, struct request_sock *req__nullable)
+{
+	struct bpf_tramp_run_ctx *ctx =
+		container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx);
+
+	return ctx->retval;
+}
+
+static int rwnd_init_stub(struct sock *sk, struct request_sock *req__nullable)
+{
+	struct bpf_tramp_run_ctx *ctx =
+		container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx);
+
+	return ctx->retval;
+}
+
+static void active_established_stub(struct sock *sk, struct sk_buff *skb__nullable)
+{
+}
+
+static void passive_established_stub(struct sock *sk, struct sk_buff *skb)
+{
+}
+
+static void rto_stub(struct sock *sk)
+{
+}
+
+static void rtt_stub(struct sock *sk, long mrtt, u32 srtt)
+{
+}
+
+static void set_state_stub(struct sock *sk, int state)
+{
+}
+
+static void retrans_stub(struct sock *sk, struct sk_buff *skb, int err)
+{
+}
+
+static void connect_stub(struct sock *sk)
+{
+}
+
+static void listen_stub(struct sock *sk)
+{
+}
+
+static void parse_hdr_stub(struct sock *sk, struct sk_buff *skb)
+{
+}
+
+static void hdr_opt_len_stub(struct sock *sk, struct sk_buff *skb__nullable,
+			     struct request_sock *req__nullable,
+			     struct sk_buff *syn_skb__nullable,
+			     enum tcp_synack_type synack_type,
+			     unsigned int *remaining)
+{
+}
+
+static void write_hdr_opt_stub(struct sock *sk, struct sk_buff *skb,
+			       struct request_sock *req__nullable,
+			       struct sk_buff *syn_skb__nullable,
+			       enum tcp_synack_type synack_type,
+			       u32 opt_off)
+{
+}
+
+static struct bpf_tcp_ops __bpf_tcp_ops = {
+	.timeout_init = timeout_init_stub,
+	.rwnd_init = rwnd_init_stub,
+	.active_established = active_established_stub,
+	.passive_established = passive_established_stub,
+	.rto = rto_stub,
+	.rtt = rtt_stub,
+	.set_state = set_state_stub,
+	.retrans = retrans_stub,
+	.connect = connect_stub,
+	.listen = listen_stub,
+	.parse_hdr = parse_hdr_stub,
+	.hdr_opt_len = hdr_opt_len_stub,
+	.write_hdr_opt = write_hdr_opt_stub,
+};
+
+BPF_CALL_4(bpf_tcp_ops_store_hdr_opt, void *, ctx, const void *, from,
+	   u32, len, u64, flags)
+{
+	u64 *args = ctx;
+	struct sk_buff *skb = (void *)(unsigned long)args[1];
+	struct bpf_sock_ops_kern sock_ops = {};
+	u32 opt_off = args[5];
+	u8 *op, *opend;
+
+	/*
+	 * bpf_tcp_ops does not keep track of the end of the written TCP header
+	 * options, so search for it every time the helper is called. The free
+	 * space is NOP-filled, so a TCPOPT_NOP ends the search rather than being
+	 * skipped as in a normal option walk in sockops.
+	 */
+	op = skb->data + opt_off;
+	opend = skb->data + tcp_hdrlen(skb);
+	while (op < opend && *op != TCPOPT_NOP) {
+		if (*op == TCPOPT_EOL || op + 1 >= opend || op[1] < 2)
+			break;
+		op += op[1];
+	}
+
+	sock_ops.skb = skb;
+	sock_ops.skb_data_end = op;
+	sock_ops.remaining_opt_len = opend - op;
+
+	return __bpf_sock_ops_store_hdr_opt(&sock_ops, from, len, flags);
+}
+
+static const struct bpf_func_proto bpf_tcp_ops_store_hdr_opt_proto = {
+	.func		= bpf_tcp_ops_store_hdr_opt,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
+	.arg3_type	= ARG_MEM_SIZE,
+	.arg4_type	= ARG_ANYTHING,
+};
+
+BPF_CALL_4(bpf_tcp_ops_load_hdr_opt, void *, ctx, void *, search_res,
+	   u32, len, u64, flags)
+{
+	u64 *args = ctx;
+	struct sk_buff *skb = (void *)(unsigned long)args[1];
+	struct bpf_sock_ops_kern sock_ops = {};
+
+	/*
+	 * No flags supported. In particular BPF_LOAD_HDR_OPT_TCP_SYN, which
+	 * loads from the saved SYN, is not available because bpf_tcp_ops has no
+	 * carrier to track the SYN source across the hooks.
+	 */
+	if (flags)
+		return -EINVAL;
+
+	sock_ops.skb = skb;
+	sock_ops.skb_data_end = skb->data + tcp_hdrlen(skb);
+
+	return __bpf_sock_ops_load_hdr_opt(&sock_ops, search_res, len, flags);
+}
+
+static const struct bpf_func_proto bpf_tcp_ops_load_hdr_opt_proto = {
+	.func		= bpf_tcp_ops_load_hdr_opt,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_PTR_TO_MEM | MEM_WRITE,
+	.arg3_type	= ARG_MEM_SIZE,
+	.arg4_type	= ARG_ANYTHING,
+};
+
+BPF_CALL_3(bpf_tcp_ops_reserve_hdr_opt, void *, ctx, u32, len, u64, flags)
+{
+	u64 *args = ctx;
+	unsigned int *remaining = (void *)(unsigned long)args[5];
+
+	if (flags || len < 2)
+		return -EINVAL;
+
+	if (len > *remaining)
+		return -ENOSPC;
+
+	*remaining -= len;
+	return 0;
+}
+
+static const struct bpf_func_proto bpf_tcp_ops_reserve_hdr_opt_proto = {
+	.func		= bpf_tcp_ops_reserve_hdr_opt,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+	.arg2_type	= ARG_ANYTHING,
+	.arg3_type	= ARG_ANYTHING,
+};
+
+BPF_CALL_0(bpf_tcp_ops_get_retval)
+{
+	struct bpf_tramp_run_ctx *ctx =
+		container_of(current->bpf_ctx, struct bpf_tramp_run_ctx, run_ctx);
+
+	/* bpf_get_retval() is only exposed to timeout_init/rwnd_init, which
+	 * always run via bpf_tcp_ops_call_int(). Its run_ctx carries the int
+	 * return value chained across the bpf_tcp_ops attached to the cgroup
+	 * and is this program's saved_run_ctx.
+	 */
+	if (WARN_ON_ONCE(!ctx->saved_run_ctx))
+		return 0;
+
+	return container_of(ctx->saved_run_ctx, struct bpf_tramp_run_ctx,
+			    run_ctx)->retval;
+}
+
+const struct bpf_func_proto bpf_tcp_ops_get_retval_proto = {
+	.func		= bpf_tcp_ops_get_retval,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+};
+
+static const struct bpf_func_proto *
+get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+	u32 moff = prog->aux->attach_st_ops_member_off;
+
+	switch (func_id) {
+	case BPF_FUNC_sk_storage_get:
+		return &bpf_sk_storage_get_proto;
+	case BPF_FUNC_sk_storage_delete:
+		return &bpf_sk_storage_delete_proto;
+	case BPF_FUNC_setsockopt:
+		/* The sk may be an unlocked listener (synack path) or NULL
+		 * fullsock; disable for members that can run unlocked.
+		 */
+		if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) ||
+		    moff == offsetof(struct bpf_tcp_ops, timeout_init) ||
+		    moff == offsetof(struct bpf_tcp_ops, hdr_opt_len) ||
+		    moff == offsetof(struct bpf_tcp_ops, write_hdr_opt))
+			return NULL;
+		return &bpf_sk_setsockopt_proto;
+	case BPF_FUNC_getsockopt:
+		if (moff == offsetof(struct bpf_tcp_ops, rwnd_init) ||
+		    moff == offsetof(struct bpf_tcp_ops, timeout_init) ||
+		    moff == offsetof(struct bpf_tcp_ops, hdr_opt_len) ||
+		    moff == offsetof(struct bpf_tcp_ops, write_hdr_opt))
+			return NULL;
+		return &bpf_sk_getsockopt_proto;
+	case BPF_FUNC_get_retval:
+		if (moff == offsetof(struct bpf_tcp_ops, timeout_init) ||
+		    moff == offsetof(struct bpf_tcp_ops, rwnd_init))
+			return &bpf_tcp_ops_get_retval_proto;
+		return NULL;
+	case BPF_FUNC_reserve_hdr_opt:
+		if (moff == offsetof(struct bpf_tcp_ops, hdr_opt_len))
+			return &bpf_tcp_ops_reserve_hdr_opt_proto;
+		return NULL;
+	case BPF_FUNC_load_hdr_opt:
+		if (moff == offsetof(struct bpf_tcp_ops, parse_hdr) ||
+		    moff == offsetof(struct bpf_tcp_ops, write_hdr_opt))
+			return &bpf_tcp_ops_load_hdr_opt_proto;
+		return NULL;
+	case BPF_FUNC_store_hdr_opt:
+		if (moff == offsetof(struct bpf_tcp_ops, write_hdr_opt))
+			return &bpf_tcp_ops_store_hdr_opt_proto;
+		return NULL;
+	default:
+		return bpf_base_func_proto(func_id, prog);
+	}
+}
+
+static bool is_valid_access(int off, int size, enum bpf_access_type type,
+			    const struct bpf_prog *prog, struct bpf_insn_access_aux *info)
+{
+	if (!bpf_tracing_btf_ctx_access(off, size, type, prog, info))
+		return false;
+
+	if (base_type(info->reg_type) == PTR_TO_BTF_ID &&
+	    !bpf_type_has_unsafe_modifiers(info->reg_type) &&
+	    info->btf_id == btf_sock_ids[BTF_SOCK_TYPE_SOCK])
+		/* promote it to tcp_sock */
+		info->btf_id = btf_sock_ids[BTF_SOCK_TYPE_TCP];
+
+	return true;
+}
+
+static int bpf_tcp_ops_init_member(const struct btf_type *t,
+				   const struct btf_member *member,
+				   void *kdata, const void *udata)
+{
+	return 0;
+}
+
+static int bpf_tcp_ops_check_member(const struct btf_type *t,
+				    const struct btf_member *member,
+				    const struct bpf_prog *prog)
+{
+	if (prog->sleepable)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int bpf_tcp_ops_init(struct btf *btf)
+{
+	return 0;
+}
+
+static int bpf_tcp_ops_validate(void *kdata)
+{
+	return 0;
+}
+
+static const struct bpf_verifier_ops bpf_tcp_ops_verifier = {
+	.get_func_proto		= get_func_proto,
+	.is_valid_access	= is_valid_access,
+};
+
+static struct bpf_struct_ops bpf_tcp_ops = {
+	.verifier_ops = &bpf_tcp_ops_verifier,
+	.init_member = bpf_tcp_ops_init_member,
+	.check_member = bpf_tcp_ops_check_member,
+	.init = bpf_tcp_ops_init,
+	.validate = bpf_tcp_ops_validate,
+	.name = "bpf_tcp_ops",
+	.cgroup_atype = CGROUP_TCP_SOCK_OPS,
+	.cfi_stubs = &__bpf_tcp_ops,
+	.owner = THIS_MODULE,
+};
+
+static int __init __bpf_tcp_ops_init(void)
+{
+	return register_bpf_struct_ops(&bpf_tcp_ops, bpf_tcp_ops);
+}
+late_initcall(__bpf_tcp_ops_init);
diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c
index a90be57c63be15..e84d3cf9224741 100644
--- a/net/ipv4/devinet.c
+++ b/net/ipv4/devinet.c
@@ -265,7 +265,9 @@ EXPORT_SYMBOL(in_dev_finish_destroy);
 
 static struct in_device *inetdev_init(struct net_device *dev)
 {
+	struct net *net = dev_net(dev);
 	struct in_device *in_dev;
+	struct neigh_table *tbl;
 	int err = -ENOMEM;
 
 	ASSERT_RTNL();
@@ -273,11 +275,12 @@ static struct in_device *inetdev_init(struct net_device *dev)
 	in_dev = kzalloc_obj(*in_dev);
 	if (!in_dev)
 		goto out;
-	memcpy(&in_dev->cnf, dev_net(dev)->ipv4.devconf_dflt,
-			sizeof(in_dev->cnf));
+
+	tbl = arp_table(net);
+	memcpy(&in_dev->cnf, net->ipv4.devconf_dflt, sizeof(in_dev->cnf));
 	in_dev->cnf.sysctl = NULL;
 	in_dev->dev = dev;
-	in_dev->arp_parms = neigh_parms_alloc(dev, &arp_tbl);
+	in_dev->arp_parms = neigh_parms_alloc(dev, tbl);
 	if (!in_dev->arp_parms)
 		goto out_kfree;
 	if (IPV4_DEVCONF(in_dev->cnf, FORWARDING))
@@ -291,7 +294,7 @@ static struct in_device *inetdev_init(struct net_device *dev)
 		err = devinet_sysctl_register(in_dev);
 		if (err) {
 			in_dev->dead = 1;
-			neigh_parms_release(&arp_tbl, in_dev->arp_parms);
+			neigh_parms_release(tbl, in_dev->arp_parms);
 			in_dev_put(in_dev);
 			in_dev = NULL;
 			goto out;
@@ -313,13 +316,12 @@ static struct in_device *inetdev_init(struct net_device *dev)
 
 static void inetdev_destroy(struct in_device *in_dev)
 {
-	struct net_device *dev;
+	struct net_device *dev = in_dev->dev;
+	struct net *net = dev_net(dev);
 	struct in_ifaddr *ifa;
 
 	ASSERT_RTNL();
 
-	dev = in_dev->dev;
-
 	in_dev->dead = 1;
 
 	RCU_INIT_POINTER(dev->ip_ptr, NULL);
@@ -332,7 +334,7 @@ static void inetdev_destroy(struct in_device *in_dev)
 	}
 
 	devinet_sysctl_unregister(in_dev);
-	neigh_parms_release(&arp_tbl, in_dev->arp_parms);
+	neigh_parms_release(arp_table(net), in_dev->arp_parms);
 	arp_ifdown(dev);
 
 	in_dev_put(in_dev);
@@ -2117,9 +2119,10 @@ static int inet_validate_link_af(const struct net_device *dev,
 		return err;
 
 	if (tb[IFLA_INET_CONF]) {
-		err = nla_parse_nested(nested_tb, IPV4_DEVCONF_MAX,
-				       tb[IFLA_INET_CONF], inet_devconf_policy,
-				       extack);
+		err = nla_parse(nested_tb, IPV4_DEVCONF_MAX,
+				nla_data(tb[IFLA_INET_CONF]),
+				nla_len(tb[IFLA_INET_CONF]),
+				inet_devconf_policy, extack);
 
 		if (err < 0)
 			return err;
diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c
index 7a362f2e2c2bd0..e3bcc25229b0d0 100644
--- a/net/ipv4/fib_semantics.c
+++ b/net/ipv4/fib_semantics.c
@@ -610,13 +610,14 @@ static int fib_detect_death(struct fib_info *fi, int order,
 			    int dflt)
 {
 	const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
-	struct neighbour *n;
+	struct net *net = fi->fib_net;
 	int state = NUD_NONE;
+	struct neighbour *n;
 
 	if (likely(nhc->nhc_gw_family == AF_INET))
-		n = neigh_lookup(&arp_tbl, &nhc->nhc_gw.ipv4, nhc->nhc_dev);
+		n = neigh_lookup(arp_table(net), &nhc->nhc_gw.ipv4, nhc->nhc_dev);
 	else if (IS_ENABLED(CONFIG_IPV6) && nhc->nhc_gw_family == AF_INET6)
-		n = neigh_lookup(&nd_tbl, &nhc->nhc_gw.ipv6, nhc->nhc_dev);
+		n = neigh_lookup(nd_table(net), &nhc->nhc_gw.ipv6, nhc->nhc_dev);
 	else
 		n = NULL;
 
@@ -2176,6 +2177,15 @@ static bool fib_good_nh(const struct fib_nh *nh)
 	return !!(state & NUD_VALID);
 }
 
+static __be32 fib_nh_saddr(struct net *net, const struct fib_info *fi,
+			   struct fib_nh *nh, int genid)
+{
+	if (READ_ONCE(nh->nh_saddr_genid) == genid)
+		return READ_ONCE(nh->nh_saddr);
+
+	return fib_info_update_nhc_saddr(net, &nh->nh_common, fi->fib_scope);
+}
+
 void fib_select_multipath(struct fib_result *res, int hash,
 			  const struct flowi4 *fl4)
 {
@@ -2184,6 +2194,7 @@ void fib_select_multipath(struct fib_result *res, int hash,
 	bool use_neigh;
 	int score = -1;
 	__be32 saddr;
+	int genid;
 
 	if (unlikely(res->fi->nh)) {
 		nexthop_path_fib_result(res, hash);
@@ -2192,6 +2203,7 @@ void fib_select_multipath(struct fib_result *res, int hash,
 
 	use_neigh = READ_ONCE(net->ipv4.sysctl_fib_multipath_use_neigh);
 	saddr = fl4 ? fl4->saddr : 0;
+	genid = saddr ? atomic_read(&net->ipv4.dev_addr_genid) : 0;
 
 	change_nexthops(fi) {
 		int nh_upper_bound, nh_score = 0;
@@ -2204,7 +2216,7 @@ void fib_select_multipath(struct fib_result *res, int hash,
 		    (use_neigh && !fib_good_nh(nexthop_nh)))
 			continue;
 
-		if (saddr && nexthop_nh->nh_saddr == saddr)
+		if (saddr && fib_nh_saddr(net, fi, nexthop_nh, genid) == saddr)
 			nh_score += 2;
 		if (hash <= nh_upper_bound)
 			nh_score++;
diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c
index 5e867f1b5c1d39..076fdca44f5180 100644
--- a/net/ipv4/fou_core.c
+++ b/net/ipv4/fou_core.c
@@ -600,6 +600,10 @@ static int fou_create(struct net *net, struct fou_cfg *cfg,
 	/* Initial for fou type */
 	switch (cfg->type) {
 	case FOU_ENCAP_DIRECT:
+		if (!cfg->protocol) {
+			err = -EINVAL;
+			goto error;
+		}
 		tunnel_cfg.encap_rcv = fou_udp_recv;
 		tunnel_cfg.gro_receive = fou_gro_receive;
 		tunnel_cfg.gro_complete = fou_gro_complete;
diff --git a/net/ipv4/fou_nl.c b/net/ipv4/fou_nl.c
index 309d5ba983d04e..ccd134ba352d27 100644
--- a/net/ipv4/fou_nl.c
+++ b/net/ipv4/fou_nl.c
@@ -16,11 +16,11 @@ const struct nla_policy fou_nl_policy[FOU_ATTR_IFINDEX + 1] = {
 	[FOU_ATTR_PORT] = { .type = NLA_BE16, },
 	[FOU_ATTR_AF] = { .type = NLA_U8, },
 	[FOU_ATTR_IPPROTO] = NLA_POLICY_MIN(NLA_U8, 1),
-	[FOU_ATTR_TYPE] = { .type = NLA_U8, },
+	[FOU_ATTR_TYPE] = NLA_POLICY_MAX(NLA_U8, 2),
 	[FOU_ATTR_REMCSUM_NOPARTIAL] = { .type = NLA_FLAG, },
-	[FOU_ATTR_LOCAL_V4] = { .type = NLA_U32, },
+	[FOU_ATTR_LOCAL_V4] = { .type = NLA_BE32, },
 	[FOU_ATTR_LOCAL_V6] = NLA_POLICY_EXACT_LEN(16),
-	[FOU_ATTR_PEER_V4] = { .type = NLA_U32, },
+	[FOU_ATTR_PEER_V4] = { .type = NLA_BE32, },
 	[FOU_ATTR_PEER_V6] = NLA_POLICY_EXACT_LEN(16),
 	[FOU_ATTR_PEER_PORT] = { .type = NLA_BE16, },
 	[FOU_ATTR_IFINDEX] = { .type = NLA_S32, },
diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c
index d56355aca79776..144fca158adcb0 100644
--- a/net/ipv4/igmp.c
+++ b/net/ipv4/igmp.c
@@ -2709,8 +2709,8 @@ int ip_mc_msfilter(struct sock *sk, struct ip_msfilter *msf, int ifindex)
 		err = ip_mc_leave_group(sk, &imr);
 	return err;
 }
-int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
-		 sockptr_t optval, sockptr_t optlen)
+
+int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf, sockopt_t *opt)
 {
 	int err, len, count, copycount, msf_size;
 	struct ip_mreqn	imr;
@@ -2755,14 +2755,19 @@ int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
 	len = flex_array_size(psl, sl_addr, copycount);
 	msf->imsf_numsrc = count;
 	msf_size = IP_MSFILTER_SIZE(copycount);
-	if (copy_to_sockptr(optlen, &msf_size, sizeof(int)) ||
-	    copy_to_sockptr(optval, msf, IP_MSFILTER_SIZE(0))) {
+
+	/* The source list is sized by the imsf_numsrc the caller left in
+	 * optval, not by optlen, which only has to cover the fixed part.
+	 */
+	err = sockopt_expand_out(opt, msf_size);
+	if (err)
+		return err;
+
+	opt->optlen = msf_size;
+	if (copy_to_iter(msf, IP_MSFILTER_SIZE(0), &opt->iter_out) !=
+	    IP_MSFILTER_SIZE(0))
 		return -EFAULT;
-	}
-	if (len &&
-	    copy_to_sockptr_offset(optval,
-				   offsetof(struct ip_msfilter, imsf_slist_flex),
-				   psl->sl_addr, len))
+	if (len && copy_to_iter(psl->sl_addr, len, &opt->iter_out) != len)
 		return -EFAULT;
 	return 0;
 done:
diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index 6257459bcee247..c688c454422a3d 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -1253,6 +1253,7 @@ struct sock *inet_csk_clone_lock(const struct sock *sk,
 	memset(&newicsk->icsk_accept_queue, 0,
 	       sizeof(newicsk->icsk_accept_queue));
 
+	newicsk->unhashed_state = 0;
 	inet_sk_set_state(newsk, TCP_SYN_RECV);
 
 	inet_clone_ulp(req, newsk, priority);
@@ -1520,7 +1521,8 @@ void inet_csk_listen_stop(struct sock *sk)
 		local_bh_enable();
 		sock_put(child);
 
-		cond_resched();
+		if (!has_current_bpf_ctx())
+			cond_resched();
 	}
 	if (queue->fastopenq.rskq_rst_head) {
 		/* Free all the reqs queued in rskq_rst_head. */
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index ba0faa9ae2bb3a..5abcb0debb2702 100644
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -803,6 +803,11 @@ int inet_hash(struct sock *sk)
 	inet_init_ehash_secret();
 
 	WARN_ON(!sk_unhashed(sk));
+
+	if (unlikely(inet_csk(sk)->unhashed_state &&
+		     inet_csk(sk)->unhashed_state != TCP_LISTEN))
+		return -EINVAL;
+
 	ilb2 = inet_lhash2_bucket_sk(hashinfo, sk);
 
 	spin_lock(&ilb2->lock);
@@ -832,6 +837,9 @@ void inet_unhash(struct sock *sk)
 		return;
 
 	sock_rps_delete_flow(sk);
+
+	inet_csk(sk)->unhashed_state = sk->sk_state;
+
 	if (sk->sk_state == TCP_LISTEN) {
 		struct inet_listen_hashbucket *ilb2;
 
@@ -1058,6 +1066,9 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
 	int ret, i, low, high;
 	bool local_ports;
 
+	if (unlikely(inet_csk(sk)->unhashed_state == TCP_LISTEN))
+		return -EINVAL;
+
 	if (port) {
 		local_bh_disable();
 		ret = check_established(death_row, sk, port, NULL, false,
diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c
index d4c781a0667fe5..a1d86026aefb0f 100644
--- a/net/ipv4/inet_timewait_sock.c
+++ b/net/ipv4/inet_timewait_sock.c
@@ -105,10 +105,12 @@ void inet_twsk_hashdance_schedule(struct inet_timewait_sock *tw,
 				  struct inet_hashinfo *hashinfo,
 				  int timeo)
 {
-	const struct inet_sock *inet = inet_sk(sk);
-	const struct inet_connection_sock *icsk = inet_csk(sk);
 	spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
+	struct inet_connection_sock *icsk = inet_csk(sk);
 	struct inet_bind_hashbucket *bhead, *bhead2;
+	const struct inet_sock *inet = inet_sk(sk);
+
+	icsk->unhashed_state = sk->sk_state;
 
 	/* Put TW into bind hash. Original socket stays there too.
 	 * Note, that any socket with inet->num != 0 MUST be bound in
diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c
index 8b65f12583eb6f..b242561d37e732 100644
--- a/net/ipv4/ip_forward.c
+++ b/net/ipv4/ip_forward.c
@@ -174,7 +174,7 @@ int ip_forward(struct sk_buff *skb)
 	/* Tell the sender its packet died... */
 	__IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS);
 	icmp_send(skb, ICMP_TIME_EXCEEDED, ICMP_EXC_TTL, 0);
-	SKB_DR_SET(reason, IP_INHDR);
+	SKB_DR_SET(reason, IP_TTL_EXCEEDED);
 drop:
 	kfree_skb_reason(skb, reason);
 	return NET_RX_DROP;
diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c
index 82309efd417e0f..51fcd603939c5b 100644
--- a/net/ipv4/ip_gre.c
+++ b/net/ipv4/ip_gre.c
@@ -1084,7 +1084,7 @@ static int __net_init ipgre_init_net(struct net *net)
 static void __net_exit ipgre_exit_rtnl(struct net *net,
 				       struct list_head *dev_to_kill)
 {
-	ip_tunnel_delete_net(net, ipgre_net_id, &ipgre_link_ops, dev_to_kill);
+	ip_tunnel_delete_net(net, ipgre_net_id, dev_to_kill);
 }
 
 static struct pernet_operations ipgre_net_ops = {
@@ -1464,6 +1464,12 @@ static int ipgre_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (!rtnl_dev_link_net_capable(dev, t->net))
 		return -EPERM;
 
+	if (data && data[IFLA_GRE_COLLECT_METADATA] && !t->collect_md) {
+		NL_SET_ERR_MSG(extack,
+			       "Enabling collect_md on an existing device is not supported");
+		return -EOPNOTSUPP;
+	}
+
 	err = ipgre_newlink_encap_setup(dev, data);
 	if (err)
 		return err;
@@ -1496,6 +1502,12 @@ static int erspan_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (!rtnl_dev_link_net_capable(dev, t->net))
 		return -EPERM;
 
+	if (data && data[IFLA_GRE_COLLECT_METADATA] && !t->collect_md) {
+		NL_SET_ERR_MSG(extack,
+			       "Enabling collect_md on an existing device is not supported");
+		return -EOPNOTSUPP;
+	}
+
 	err = ipgre_newlink_encap_setup(dev, data);
 	if (err)
 		return err;
@@ -1728,7 +1740,7 @@ static int __net_init ipgre_tap_init_net(struct net *net)
 static void __net_exit ipgre_tap_exit_rtnl(struct net *net,
 					   struct list_head *dev_to_kill)
 {
-	ip_tunnel_delete_net(net, gre_tap_net_id, &ipgre_tap_ops, dev_to_kill);
+	ip_tunnel_delete_net(net, gre_tap_net_id, dev_to_kill);
 }
 
 static struct pernet_operations ipgre_tap_net_ops = {
@@ -1747,7 +1759,7 @@ static int __net_init erspan_init_net(struct net *net)
 static void __net_exit erspan_exit_rtnl(struct net *net,
 					struct list_head *dev_to_kill)
 {
-	ip_tunnel_delete_net(net, erspan_net_id, &erspan_link_ops, dev_to_kill);
+	ip_tunnel_delete_net(net, erspan_net_id, dev_to_kill);
 }
 
 static struct pernet_operations erspan_net_ops = {
diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c
index a55ef327ec932c..e06c1f48ecad6e 100644
--- a/net/ipv4/ip_sockglue.c
+++ b/net/ipv4/ip_sockglue.c
@@ -1706,6 +1706,8 @@ int do_ip_getsockopt(struct sock *sk, int level, int optname,
 	case IP_MSFILTER:
 	{
 		struct ip_msfilter msf;
+		struct kvec kvec;
+		sockopt_t opt;
 
 		if (len < IP_MSFILTER_SIZE(0)) {
 			err = -EINVAL;
@@ -1715,7 +1717,13 @@ int do_ip_getsockopt(struct sock *sk, int level, int optname,
 			err = -EFAULT;
 			goto out;
 		}
-		err = ip_mc_msfget(sk, &msf, optval, optlen);
+		err = sockptr_to_sockopt(&opt, optval, optlen, &kvec);
+		if (err)
+			goto out;
+
+		err = ip_mc_msfget(sk, &msf, &opt);
+		if (!err && copy_to_sockptr(optlen, &opt.optlen, sizeof(int)))
+			err = -EFAULT;
 		goto out;
 	}
 	case MCAST_MSFILTER:
diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c
index e6bcf01411d0bc..0875474a578a4a 100644
--- a/net/ipv4/ip_tunnel.c
+++ b/net/ipv4/ip_tunnel.c
@@ -205,6 +205,11 @@ static void ip_tunnel_del(struct ip_tunnel_net *itn, struct ip_tunnel *t)
 	hlist_del_init_rcu(&t->hash_node);
 }
 
+static bool ip_tunnel_unregistering(struct ip_tunnel *t)
+{
+	return hlist_unhashed(&t->hash_node);
+}
+
 static struct ip_tunnel *ip_tunnel_find(struct ip_tunnel_net *itn,
 					struct ip_tunnel_parm_kern *parms,
 					int type)
@@ -219,7 +224,8 @@ static struct ip_tunnel *ip_tunnel_find(struct ip_tunnel_net *itn,
 
 	ip_tunnel_flags_copy(flags, parms->i_flags);
 
-	hlist_for_each_entry_rcu(t, head, hash_node, lockdep_rtnl_is_held()) {
+	hlist_for_each_entry_rcu(t, head, hash_node,
+				 lockdep_is_held(&itn->tunnels_lock)) {
 		if (local == t->parms.iph.saddr &&
 		    remote == t->parms.iph.daddr &&
 		    link == READ_ONCE(t->parms.link) &&
@@ -491,19 +497,17 @@ int ip_tunnel_encap_setup(struct ip_tunnel *t,
 {
 	int hlen;
 
-	memset(&t->encap, 0, sizeof(t->encap));
-
 	hlen = ip_encap_hlen(ipencap);
 	if (hlen < 0)
 		return hlen;
 
-	t->encap.type = ipencap->type;
-	t->encap.sport = ipencap->sport;
-	t->encap.dport = ipencap->dport;
-	t->encap.flags = ipencap->flags;
+	WRITE_ONCE(t->encap.type, ipencap->type);
+	WRITE_ONCE(t->encap.sport, ipencap->sport);
+	WRITE_ONCE(t->encap.dport, ipencap->dport);
+	WRITE_ONCE(t->encap.flags, ipencap->flags);
 
-	t->encap_hlen = hlen;
-	t->hlen = t->encap_hlen + t->tun_hlen;
+	WRITE_ONCE(t->encap_hlen, hlen);
+	WRITE_ONCE(t->hlen, hlen + t->tun_hlen);
 
 	return 0;
 }
@@ -894,13 +898,37 @@ static void ip_tunnel_update(struct ip_tunnel_net *itn,
 	netdev_state_change(dev);
 }
 
+static void __ip_tunnel_dellink(struct net *net, struct net_device *dev,
+				struct list_head *head)
+{
+	struct ip_tunnel *tunnel = netdev_priv(dev);
+	struct ip_tunnel_net *itn;
+
+	itn = net_generic(tunnel->net, tunnel->ip_tnl_net_id);
+	ip_tunnel_del(itn, tunnel);
+	unregister_netdevice_queue_net(net, dev, head);
+}
+
 int ip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p,
 		  int cmd)
 {
-	int err = 0;
 	struct ip_tunnel *t = netdev_priv(dev);
+	struct net *orig_net = dev_net(dev);
+	struct ip_tunnel_net *itn;
+	LIST_HEAD(dev_kill_list);
 	struct net *net = t->net;
-	struct ip_tunnel_net *itn = net_generic(net, t->ip_tnl_net_id);
+	int err = 0;
+
+	DEBUG_NET_WARN_ON_ONCE(netdev_need_ops_lock(dev));
+
+	itn = net_generic(net, t->ip_tnl_net_id);
+
+	mutex_lock(&itn->tunnels_lock);
+
+	if (!check_net(net)) {
+		err = -EBUSY;
+		goto done;
+	}
 
 	switch (cmd) {
 	case SIOCGETTUNNEL:
@@ -961,7 +989,7 @@ int ip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p,
 			}
 		}
 
-		if (t) {
+		if (t && !ip_tunnel_unregistering(t)) {
 			err = 0;
 			ip_tunnel_update(itn, t, dev, p, true, 0);
 		} else {
@@ -984,7 +1012,10 @@ int ip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p,
 				goto done;
 			dev = t->dev;
 		}
-		unregister_netdevice(dev);
+
+		if (!ip_tunnel_unregistering(t))
+			__ip_tunnel_dellink(orig_net, dev, &dev_kill_list);
+
 		err = 0;
 		break;
 
@@ -993,6 +1024,10 @@ int ip_tunnel_ctl(struct net_device *dev, struct ip_tunnel_parm_kern *p,
 	}
 
 done:
+	mutex_unlock(&itn->tunnels_lock);
+
+	unregister_netdevice_many(&dev_kill_list);
+
 	return err;
 }
 EXPORT_SYMBOL_GPL(ip_tunnel_ctl);
@@ -1087,8 +1122,10 @@ void ip_tunnel_dellink(struct net_device *dev, struct list_head *head)
 	itn = net_generic(tunnel->net, tunnel->ip_tnl_net_id);
 
 	if (itn->fb_tunnel_dev != dev) {
-		ip_tunnel_del(itn, netdev_priv(dev));
-		unregister_netdevice_queue(dev, head);
+		mutex_lock(&itn->tunnels_lock);
+		if (!ip_tunnel_unregistering(tunnel))
+			__ip_tunnel_dellink(dev_net(dev), dev, head);
+		mutex_unlock(&itn->tunnels_lock);
 	}
 }
 EXPORT_SYMBOL_GPL(ip_tunnel_dellink);
@@ -1120,6 +1157,8 @@ int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id,
 	for (i = 0; i < IP_TNL_HASH_SIZE; i++)
 		INIT_HLIST_HEAD(&itn->tunnels[i]);
 
+	mutex_init(&itn->tunnels_lock);
+
 	if (!ops || !net_has_fallback_tunnels(net)) {
 		struct ip_tunnel_net *it_init_net;
 
@@ -1151,31 +1190,27 @@ int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id,
 EXPORT_SYMBOL_GPL(ip_tunnel_init_net);
 
 void ip_tunnel_delete_net(struct net *net, unsigned int id,
-			  struct rtnl_link_ops *ops,
 			  struct list_head *head)
 {
 	struct ip_tunnel_net *itn = net_generic(net, id);
-	struct net_device *dev, *aux;
 	int h;
 
 	ASSERT_RTNL_NET(net);
 
-	for_each_netdev_safe(net, dev, aux)
-		if (dev->rtnl_link_ops == ops)
-			unregister_netdevice_queue(dev, head);
+	mutex_lock(&itn->tunnels_lock);
+
+	WRITE_ONCE(itn->fb_tunnel_dev, NULL);
 
 	for (h = 0; h < IP_TNL_HASH_SIZE; h++) {
-		struct ip_tunnel *t;
-		struct hlist_node *n;
 		struct hlist_head *thead = &itn->tunnels[h];
+		struct hlist_node *n;
+		struct ip_tunnel *t;
 
 		hlist_for_each_entry_safe(t, n, thead, hash_node)
-			/* If dev is in the same netns, it has already
-			 * been added to the list by the previous loop.
-			 */
-			if (!net_eq(dev_net(t->dev), net))
-				unregister_netdevice_queue(t->dev, head);
+			__ip_tunnel_dellink(net, t->dev, head);
 	}
+
+	mutex_unlock(&itn->tunnels_lock);
 }
 EXPORT_SYMBOL_GPL(ip_tunnel_delete_net);
 
@@ -1183,21 +1218,25 @@ int ip_tunnel_newlink(struct net *net, struct net_device *dev,
 		      struct nlattr *tb[], struct ip_tunnel_parm_kern *p,
 		      __u32 fwmark)
 {
-	struct ip_tunnel *nt;
 	struct ip_tunnel_net *itn;
+	struct ip_tunnel *nt;
+	int err = 0;
 	int mtu;
-	int err;
 
 	nt = netdev_priv(dev);
 	itn = net_generic(net, nt->ip_tnl_net_id);
 
+	mutex_lock(&itn->tunnels_lock);
+
 	if (nt->collect_md) {
 		if (rtnl_dereference(itn->collect_md_tun))
-			return -EEXIST;
+			err = -EEXIST;
 	} else {
 		if (ip_tunnel_find(itn, p, dev->type))
-			return -EEXIST;
+			err = -EEXIST;
 	}
+	if (err)
+		goto out;
 
 	nt->net = net;
 	nt->parms = *p;
@@ -1224,31 +1263,41 @@ int ip_tunnel_newlink(struct net *net, struct net_device *dev,
 		goto err_dev_set_mtu;
 
 	ip_tunnel_add(itn, nt);
-	return 0;
+out:
+	mutex_unlock(&itn->tunnels_lock);
+
+	return err;
 
 err_dev_set_mtu:
 	unregister_netdevice(dev);
 err_register_netdevice:
-	return err;
+	goto out;
 }
 EXPORT_SYMBOL_GPL(ip_tunnel_newlink);
 
 int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[],
 			 struct ip_tunnel_parm_kern *p, __u32 fwmark)
 {
-	struct ip_tunnel *t;
 	struct ip_tunnel *tunnel = netdev_priv(dev);
 	struct net *net = tunnel->net;
-	struct ip_tunnel_net *itn = net_generic(net, tunnel->ip_tnl_net_id);
+	struct ip_tunnel_net *itn;
+	struct ip_tunnel *t;
+	int err = 0;
+
+	itn = net_generic(net, tunnel->ip_tnl_net_id);
 
 	if (dev == itn->fb_tunnel_dev)
 		return -EINVAL;
 
+	mutex_lock(&itn->tunnels_lock);
+
 	t = ip_tunnel_find(itn, p, dev->type);
 
 	if (t) {
-		if (t->dev != dev)
-			return -EEXIST;
+		if (t->dev != dev) {
+			err = -EEXIST;
+			goto out;
+		}
 	} else {
 		t = tunnel;
 
@@ -1261,13 +1310,23 @@ int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[],
 				nflags = IFF_POINTOPOINT;
 
 			if ((dev->flags ^ nflags) &
-			    (IFF_POINTOPOINT | IFF_BROADCAST))
-				return -EINVAL;
+			    (IFF_POINTOPOINT | IFF_BROADCAST)) {
+				err = -EINVAL;
+				goto out;
+			}
 		}
 	}
 
+	if (ip_tunnel_unregistering(t)) {
+		err = -ENODEV;
+		goto out;
+	}
+
 	ip_tunnel_update(itn, t, dev, p, !tb[IFLA_MTU], fwmark);
-	return 0;
+out:
+	mutex_unlock(&itn->tunnels_lock);
+
+	return err;
 }
 EXPORT_SYMBOL_GPL(ip_tunnel_changelink);
 
@@ -1305,13 +1364,6 @@ EXPORT_SYMBOL_GPL(__ip_tunnel_init);
 void ip_tunnel_uninit(struct net_device *dev)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
-	struct net *net = tunnel->net;
-	struct ip_tunnel_net *itn;
-
-	itn = net_generic(net, tunnel->ip_tnl_net_id);
-	ip_tunnel_del(itn, netdev_priv(dev));
-	if (itn->fb_tunnel_dev == dev)
-		WRITE_ONCE(itn->fb_tunnel_dev, NULL);
 
 	dst_cache_reset(&tunnel->dst_cache);
 }
diff --git a/net/ipv4/ip_vti.c b/net/ipv4/ip_vti.c
index 3b80929994a007..c4f14d42df2485 100644
--- a/net/ipv4/ip_vti.c
+++ b/net/ipv4/ip_vti.c
@@ -526,7 +526,7 @@ static int __net_init vti_init_net(struct net *net)
 static void __net_exit vti_exit_rtnl(struct net *net,
 				     struct list_head *dev_to_kill)
 {
-	ip_tunnel_delete_net(net, vti_net_id, &vti_link_ops, dev_to_kill);
+	ip_tunnel_delete_net(net, vti_net_id, dev_to_kill);
 }
 
 static struct pernet_operations vti_net_ops = {
diff --git a/net/ipv4/ipconfig.c b/net/ipv4/ipconfig.c
index 155db067eaecbe..1b8585404a41db 100644
--- a/net/ipv4/ipconfig.c
+++ b/net/ipv4/ipconfig.c
@@ -676,6 +676,24 @@ static const u8 ic_bootp_cookie[4] = { 99, 130, 83, 99 };
 
 #ifdef IPCONFIG_DHCP
 
+static bool __init
+ic_dhcp_add_option(u8 **options, const u8 *end, u8 type, const void *value,
+		   int len)
+{
+	u8 *e = *options;
+
+	/* leave room for the option header and the END marker */
+	if (len > U8_MAX || end - e < len + 3)
+		return false;
+
+	*e++ = type;
+	*e++ = len;
+	memcpy(e, value, len);
+	*options = e + len;
+
+	return true;
+}
+
 static void __init
 ic_dhcp_init_options(u8 *options, struct ic_device *d)
 {
@@ -691,6 +709,7 @@ ic_dhcp_init_options(u8 *options, struct ic_device *d)
 		42,	/* NTP servers */
 	};
 	u8 mt = (ic_servaddr == NONE) ? DHCPDISCOVER : DHCPREQUEST;
+	u8 *end = options + sizeof(((struct bootp_pkt *)0)->exten);
 	u8 *e = options;
 	int len;
 
@@ -721,31 +740,19 @@ ic_dhcp_init_options(u8 *options, struct ic_device *d)
 	e += sizeof(ic_req_params);
 
 	if (ic_host_name_set) {
-		*e++ = 12;	/* host-name */
 		len = strlen(utsname()->nodename);
-		*e++ = len;
-		memcpy(e, utsname()->nodename, len);
-		e += len;
+		ic_dhcp_add_option(&e, end, 12, utsname()->nodename, len);
 	}
 	if (*vendor_class_identifier) {
-		pr_info("DHCP: sending class identifier \"%s\"\n",
-			vendor_class_identifier);
-		*e++ = 60;	/* Class-identifier */
 		len = strlen(vendor_class_identifier);
-		*e++ = len;
-		memcpy(e, vendor_class_identifier, len);
-		e += len;
+		if (ic_dhcp_add_option(&e, end, 60, vendor_class_identifier, len))
+			pr_info("DHCP: sending class identifier \"%s\"\n",
+				vendor_class_identifier);
 	}
 	len = strlen(dhcp_client_identifier + 1);
-	/* the minimum length of identifier is 2, include 1 byte type,
-	 * and can not be larger than the length of options
-	 */
-	if (len >= 1 && len < 312 - (e - options) - 1) {
-		*e++ = 61;
-		*e++ = len + 1;
-		memcpy(e, dhcp_client_identifier, len + 1);
-		e += len + 1;
-	}
+	/* the minimum length of identifier is 2, include 1 byte type */
+	if (len >= 1)
+		ic_dhcp_add_option(&e, end, 61, dhcp_client_identifier, len + 1);
 
 	*e++ = 255;	/* End of the list */
 }
diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c
index f684baf8e58f78..2ffe64e736e08b 100644
--- a/net/ipv4/ipip.c
+++ b/net/ipv4/ipip.c
@@ -645,7 +645,7 @@ static int __net_init ipip_init_net(struct net *net)
 static void __net_exit ipip_exit_rtnl(struct net *net,
 				      struct list_head *dev_to_kill)
 {
-	ip_tunnel_delete_net(net, ipip_net_id, &ipip_link_ops, dev_to_kill);
+	ip_tunnel_delete_net(net, ipip_net_id, dev_to_kill);
 }
 
 static struct pernet_operations ipip_net_ops = {
diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
index b9c544d48c4521..e4c51ca473019c 100644
--- a/net/ipv4/ipmr.c
+++ b/net/ipv4/ipmr.c
@@ -479,6 +479,7 @@ static struct net_device *ipmr_new_tunnel(struct net *net, struct vifctl *v)
 {
 	struct net_device *tunnel_dev, *new_dev;
 	struct ip_tunnel_parm_kern p = { };
+	LIST_HEAD(dev_kill_list);
 	int err;
 
 	tunnel_dev = __dev_get_by_name(net, "tunl0");
@@ -520,7 +521,8 @@ static struct net_device *ipmr_new_tunnel(struct net *net, struct vifctl *v)
 	return new_dev;
 
 out_unregister:
-	unregister_netdevice(new_dev);
+	new_dev->rtnl_link_ops->dellink(new_dev, &dev_kill_list);
+	unregister_netdevice_many(&dev_kill_list);
 out:
 	return ERR_PTR(-ENOBUFS);
 }
@@ -733,8 +735,12 @@ static int vif_delete(struct mr_table *mrt, int vifi, int notify,
 		ip_rt_multicast_event(in_dev);
 	}
 
-	if (v->flags & (VIFF_TUNNEL | VIFF_REGISTER) && !notify)
-		unregister_netdevice_queue(dev, head);
+	if (!notify) {
+		if (v->flags & VIFF_TUNNEL)
+			dev->rtnl_link_ops->dellink(dev, head);
+		else if (v->flags & VIFF_REGISTER)
+			unregister_netdevice_queue(dev, head);
+	}
 
 	netdev_put(dev, &v->dev_tracker);
 	return 0;
@@ -3178,8 +3184,9 @@ static int ipmr_vif_seq_show(struct seq_file *seq, void *v)
 		seq_printf(seq,
 			   "%2td %-10s %8ld %7ld  %8ld %7ld %05X %08X %08X\n",
 			   vif - mrt->vif_table,
-			   name, vif->bytes_in, vif->pkt_in,
-			   vif->bytes_out, vif->pkt_out,
+			   name,
+			   READ_ONCE(vif->bytes_in), READ_ONCE(vif->pkt_in),
+			   READ_ONCE(vif->bytes_out), READ_ONCE(vif->pkt_out),
 			   vif->flags, vif->local, vif->remote);
 	}
 	return 0;
diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c
index db307fa49f3f6b..cf747e1e17a363 100644
--- a/net/ipv4/netfilter/arp_tables.c
+++ b/net/ipv4/netfilter/arp_tables.c
@@ -58,7 +58,7 @@ static inline int arp_devaddr_compare(const struct arpt_devaddr_info *ap,
 /*
  * Unfortunately, _b and _mask are not aligned to an int (or long int)
  * Some arches dont care, unrolling the loop is a win on them.
- * For other arches, we only have a 16bit alignement.
+ * For other arches, we only have a 16bit alignment.
  */
 static unsigned long ifname_compare(const char *_a, const char *_b, const char *_mask)
 {
diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c
index d36f1e273fde43..223a0108b74cc0 100644
--- a/net/ipv4/ping.c
+++ b/net/ipv4/ping.c
@@ -1095,15 +1095,14 @@ static void ping_v4_format_sock(struct sock *sp, struct seq_file *f,
 	__u16 srcp = ntohs(inet->inet_sport);
 
 	seq_printf(f, "%5d: %08X:%04X %08X:%04X"
-		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d %pK %u",
+		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d 0 %u",
 		bucket, src, srcp, dest, destp, sp->sk_state,
 		sk_wmem_alloc_get(sp),
 		sk_rmem_alloc_get(sp),
 		0, 0L, 0,
 		from_kuid_munged(seq_user_ns(f), sk_uid(sp)),
 		0, sock_i_ino(sp),
-		refcount_read(&sp->sk_refcnt), sp,
-		sk_drops_read(sp));
+		refcount_read(&sp->sk_refcnt), sk_drops_read(sp));
 }
 
 static int ping_v4_seq_show(struct seq_file *seq, void *v)
diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
index d756e5e31b90db..9d01006d8189e1 100644
--- a/net/ipv4/raw.c
+++ b/net/ipv4/raw.c
@@ -1054,14 +1054,14 @@ static void raw_sock_seq_show(struct seq_file *seq, struct sock *sp, int i)
 	      srcp  = inet->inet_num;
 
 	seq_printf(seq, "%4d: %08X:%04X %08X:%04X"
-		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d %pK %u\n",
+		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d 0 %u\n",
 		i, src, srcp, dest, destp, sp->sk_state,
 		sk_wmem_alloc_get(sp),
 		sk_rmem_alloc_get(sp),
 		0, 0L, 0,
 		from_kuid_munged(seq_user_ns(seq), sk_uid(sp)),
 		0, sock_i_ino(sp),
-		refcount_read(&sp->sk_refcnt), sp, sk_drops_read(sp));
+		refcount_read(&sp->sk_refcnt), sk_drops_read(sp));
 }
 
 static int raw_seq_show(struct seq_file *seq, void *v)
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 37674d76f90f04..d7da2f1acbb59d 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -817,7 +817,7 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
 
 	n = __ipv4_neigh_lookup(rt->dst.dev, (__force u32)new_gw);
 	if (!n)
-		n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
+		n = neigh_create(arp_table(net), &new_gw, rt->dst.dev);
 	if (!IS_ERR(n)) {
 		if (!(READ_ONCE(n->nud_state) & NUD_VALID)) {
 			neigh_event_send(n, NULL);
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 562752352afe4d..9b80cfa0dd7d91 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -578,8 +578,13 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 	 * blocking on fresh not-connected or disconnected socket. --ANK
 	 */
 	shutdown = READ_ONCE(sk->sk_shutdown);
-	if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
+	if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) {
 		mask |= EPOLLHUP;
+		/* Coupled with smp_wmb() in tcp_done_with_error() to ensure
+		 * sk->sk_err is visible if socket closure was observed.
+		 */
+		smp_rmb();
+	}
 	if (shutdown & RCV_SHUTDOWN)
 		mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
 
@@ -626,8 +631,6 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 		 */
 		mask |= EPOLLOUT | EPOLLWRNORM;
 	}
-	/* This barrier is coupled with smp_wmb() in tcp_done_with_error() */
-	smp_rmb();
 	if (READ_ONCE(sk->sk_err) ||
 	    !skb_queue_empty_lockless(&sk->sk_error_queue))
 		mask |= EPOLLERR;
@@ -2167,27 +2170,18 @@ static void tcp_zc_finalize_rx_tstamp(struct sock *sk,
 }
 
 static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm,
-					   unsigned long address,
-					   bool *mmap_locked)
+					   unsigned long address)
 {
-	struct vm_area_struct *vma = lock_vma_under_rcu(mm, address);
+	struct vm_area_struct *vma = vma_start_read_unlocked(mm, address);
 
-	if (vma) {
-		if (vma->vm_ops != &tcp_vm_ops) {
-			vma_end_read(vma);
-			return NULL;
-		}
-		*mmap_locked = false;
-		return vma;
-	}
+	if (!vma)
+		return NULL;
 
-	mmap_read_lock(mm);
-	vma = vma_lookup(mm, address);
-	if (!vma || vma->vm_ops != &tcp_vm_ops) {
-		mmap_read_unlock(mm);
+	if (vma->vm_ops != &tcp_vm_ops) {
+		vma_end_read(vma);
 		return NULL;
 	}
-	*mmap_locked = true;
+
 	return vma;
 }
 
@@ -2208,7 +2202,6 @@ static int tcp_zerocopy_receive(struct sock *sk,
 	u32 seq = tp->copied_seq;
 	u32 total_bytes_to_map;
 	int inq = tcp_inq(sk);
-	bool mmap_locked;
 	int ret;
 
 	zc->copybuf_len = 0;
@@ -2233,7 +2226,7 @@ static int tcp_zerocopy_receive(struct sock *sk,
 		return 0;
 	}
 
-	vma = find_tcp_vma(current->mm, address, &mmap_locked);
+	vma = find_tcp_vma(current->mm, address);
 	if (!vma)
 		return -EINVAL;
 
@@ -2315,10 +2308,7 @@ static int tcp_zerocopy_receive(struct sock *sk,
 						   zc, total_bytes_to_map);
 	}
 out:
-	if (mmap_locked)
-		mmap_read_unlock(current->mm);
-	else
-		vma_end_read(vma);
+	vma_end_read(vma);
 	/* Try to copy straggler data. */
 	if (!ret)
 		copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss);
@@ -2996,6 +2986,7 @@ void tcp_set_state(struct sock *sk, int state)
 
 	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
 		tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
+	bpf_tcp_ops_call(set_state, sk, state);
 
 	switch (state) {
 	case TCP_ESTABLISHED:
diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c
index 82378a2bfd1e19..ecf11be46f3820 100644
--- a/net/ipv4/tcp_bbr.c
+++ b/net/ipv4/tcp_bbr.c
@@ -297,11 +297,18 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain)
 }
 
 /* override sysctl_tcp_min_tso_segs */
-__bpf_kfunc static u32 bbr_min_tso_segs(struct sock *sk)
+static u32 bbr_min_tso_segs(struct sock *sk)
 {
 	return READ_ONCE(sk->sk_pacing_rate) < (bbr_min_tso_rate >> 3) ? 1 : 2;
 }
 
+__bpf_kfunc static u32 bbr_tso_segs(struct sock *sk, u32 mss_now)
+{
+	if (unlikely(!mss_now))
+		return bbr_min_tso_segs(sk);
+	return tcp_tso_autosize(sk, mss_now, bbr_min_tso_segs(sk));
+}
+
 static u32 bbr_tso_segs_goal(struct sock *sk)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
@@ -1151,7 +1158,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = {
 	.undo_cwnd	= bbr_undo_cwnd,
 	.cwnd_event_tx_start	= bbr_cwnd_event_tx_start,
 	.ssthresh	= bbr_ssthresh,
-	.min_tso_segs	= bbr_min_tso_segs,
+	.tso_segs	= bbr_tso_segs,
 	.get_info	= bbr_get_info,
 	.set_state	= bbr_set_state,
 };
@@ -1163,7 +1170,7 @@ BTF_ID_FLAGS(func, bbr_sndbuf_expand)
 BTF_ID_FLAGS(func, bbr_undo_cwnd)
 BTF_ID_FLAGS(func, bbr_cwnd_event_tx_start)
 BTF_ID_FLAGS(func, bbr_ssthresh)
-BTF_ID_FLAGS(func, bbr_min_tso_segs)
+BTF_ID_FLAGS(func, bbr_tso_segs)
 BTF_ID_FLAGS(func, bbr_set_state)
 BTF_KFUNCS_END(tcp_bbr_check_kfunc_ids)
 
diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index 2e234d155b5e61..d5fcf3ce4861a2 100644
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -108,7 +108,6 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
 		off  = sge->offset;
 		page = sg_page(sge);
 
-		tcp_rate_check_app_limited(sk);
 retry:
 		msghdr.msg_flags = flags | MSG_SPLICE_PAGES;
 		has_tx_ulp = tls_sw_has_ctx_tx(sk);
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 92bc60716f33d8..cfa4edcd0fd10c 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -208,6 +208,18 @@ static void bpf_skops_established(struct sock *sk, int bpf_op,
 }
 #endif
 
+static void bpf_tcp_ops_parse_hdr(struct sock *sk, struct sk_buff *skb)
+{
+	switch (sk->sk_state) {
+	case TCP_SYN_RECV:
+	case TCP_SYN_SENT:
+	case TCP_LISTEN:
+		return;
+	}
+
+	bpf_tcp_ops_call(parse_hdr, sk, skb);
+}
+
 static __cold void tcp_gro_dev_warn(const struct sock *sk, const struct sk_buff *skb,
 				    unsigned int len)
 {
@@ -4465,6 +4477,12 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
 	return 1;
 
 old_ack:
+	/* An old ACK can carry new data. Update TS.Recent before SACK
+	 * processing can trigger a retransmission.
+	 */
+	if (flag & FLAG_UPDATE_TS_RECENT)
+		tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq);
+
 	/* If data was SACKed, tag it and see if we should send more data.
 	 * If data was DSACKed, see if we can undo a cwnd reduction.
 	 */
@@ -6461,6 +6479,7 @@ static bool tcp_validate_incoming(struct sock *sk, struct sk_buff *skb,
 
 pass:
 	bpf_skops_parse_hdr(sk, skb);
+	bpf_tcp_ops_parse_hdr(sk, skb);
 
 	return true;
 
@@ -6725,6 +6744,10 @@ void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb)
 	tp->snd_cwnd_stamp = tcp_jiffies32;
 
 	bpf_skops_established(sk, bpf_op, skb);
+	if (bpf_op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB)
+		bpf_tcp_ops_call(active_established, sk, skb);
+	else
+		bpf_tcp_ops_call(passive_established, sk, skb);
 	/* Initialize congestion control unless BPF initialized it already: */
 	if (!icsk->icsk_ca_initialized)
 		tcp_init_congestion_control(sk);
@@ -7233,19 +7256,14 @@ tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb)
 				  FLAG_UPDATE_TS_RECENT |
 				  FLAG_NO_CHALLENGE_ACK);
 
-	if ((int)reason <= 0) {
-		if (sk->sk_state == TCP_SYN_RECV) {
+	/* accept old ack (reason == 0) during closing */
+	if ((int)reason < 0) {
+		reason = -reason;
+		if (sk->sk_state == TCP_SYN_RECV)
 			/* send one RST */
-			if (!reason)
-				return SKB_DROP_REASON_TCP_OLD_ACK;
-			return -reason;
-		}
-		/* accept old ack during closing */
-		if ((int)reason < 0) {
-			tcp_send_challenge_ack(sk, false);
-			reason = -reason;
-			goto discard;
-		}
+			return reason;
+		tcp_send_challenge_ack(sk, false);
+		goto discard;
 	}
 	SKB_DR_SET(reason, NOT_SPECIFIED);
 	switch (sk->sk_state) {
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 04dbb2babbcdc1..c107b7e44e5b50 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -2748,7 +2748,7 @@ static void get_openreq4(const struct request_sock *req,
 	long delta = req->rsk_timer.expires - jiffies;
 
 	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
-		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
+		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d 0",
 		i,
 		ireq->ir_loc_addr,
 		ireq->ir_num,
@@ -2763,8 +2763,7 @@ static void get_openreq4(const struct request_sock *req,
 				 sk_uid(req->rsk_listener)),
 		0,  /* non standard timer */
 		0, /* open_requests have no inode */
-		0,
-		req);
+		0);
 }
 
 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
@@ -2811,7 +2810,7 @@ static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
 				      READ_ONCE(tp->copied_seq), 0);
 
 	seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
-			"%08X %5u %8d %llu %d %pK %lu %lu %u %u %d",
+			"%08X %5u %8d %llu %d 0 %lu %lu %u %u %d",
 		i, src, srcp, dest, destp, state,
 		READ_ONCE(tp->write_seq) - tp->snd_una,
 		rx_queue,
@@ -2821,7 +2820,7 @@ static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
 		from_kuid_munged(seq_user_ns(f), sk_uid(sk)),
 		READ_ONCE(icsk->icsk_probes_out),
 		sock_i_ino(sk),
-		refcount_read(&sk->sk_refcnt), sk,
+		refcount_read(&sk->sk_refcnt),
 		jiffies_to_clock_t(icsk->icsk_rto),
 		jiffies_to_clock_t(icsk->icsk_ack.ato),
 		(icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
@@ -2844,10 +2843,10 @@ static void get_timewait4_sock(const struct inet_timewait_sock *tw,
 	srcp  = ntohs(tw->tw_sport);
 
 	seq_printf(f, "%4d: %08X:%04X %08X:%04X"
-		" %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
+		" %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0",
 		i, src, srcp, dest, destp, READ_ONCE(tw->tw_substate), 0, 0,
 		3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
-		refcount_read(&tw->tw_refcnt), tw);
+		refcount_read(&tw->tw_refcnt));
 }
 
 #define TMPSZ 150
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index d960e3de7d50d0..b284f6fa603aee 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -536,43 +536,53 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
 				    enum tcp_synack_type synack_type,
 				    struct tcp_out_options *opts)
 {
-	u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len;
-	struct bpf_sock_ops_kern sock_ops;
-	int err;
+	u8 first_opt_off, nr_written = 0, max_opt_len = opts->bpf_opt_len;
 
 	if (likely(!max_opt_len))
 		return;
 
-	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
+	first_opt_off = tcp_hdrlen(skb) - max_opt_len;
 
-	sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
+	if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
+				   BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) {
+		struct bpf_sock_ops_kern sock_ops;
+		int err;
 
-	if (req) {
-		sock_ops.sk = (struct sock *)req;
-		sock_ops.syn_skb = syn_skb;
-	} else {
-		sock_owned_by_me(sk);
+		memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
 
-		sock_ops.is_fullsock = 1;
-		sock_ops.is_locked_tcp_sock = 1;
-		sock_ops.sk = sk;
-	}
+		sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
 
-	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
-	sock_ops.remaining_opt_len = max_opt_len;
-	first_opt_off = tcp_hdrlen(skb) - max_opt_len;
-	bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
+		if (req) {
+			sock_ops.sk = (struct sock *)req;
+			sock_ops.syn_skb = syn_skb;
+		} else {
+			sock_owned_by_me(sk);
 
-	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
+			sock_ops.is_fullsock = 1;
+			sock_ops.is_locked_tcp_sock = 1;
+			sock_ops.sk = sk;
+		}
 
-	if (err)
-		nr_written = 0;
-	else
-		nr_written = max_opt_len - sock_ops.remaining_opt_len;
+		sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
+		sock_ops.remaining_opt_len = max_opt_len;
+		bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
+
+		err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
+		if (!err)
+			nr_written = max_opt_len - sock_ops.remaining_opt_len;
+	}
 
 	if (nr_written < max_opt_len)
 		memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP,
 		       max_opt_len - nr_written);
+
+	/*
+	 * bpf_tcp_ops portion is NOP-filled (everything past the sockops
+	 * writer's bytes). The writer finds the append point by scanning from
+	 * first_opt_off + nr_written to the first NOP.
+	 */
+	bpf_tcp_ops_call(write_hdr_opt, sk, skb, req, syn_skb, synack_type,
+			 first_opt_off + nr_written);
 }
 #else
 static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
@@ -594,6 +604,32 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
 }
 #endif
 
+static u32 bpf_tcp_ops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
+				   struct request_sock *req,
+				   struct sk_buff *syn_skb,
+				   enum tcp_synack_type synack_type,
+				   struct tcp_out_options *opts,
+				   u32 remaining)
+{
+	unsigned int remaining_out = remaining, reserved;
+
+	if (!remaining)
+		return 0;
+
+	/* bpf_tcp_ops_reserve_hdr_opt() reserves space via remaining_out */
+	bpf_tcp_ops_call(hdr_opt_len, sk, skb, req, syn_skb, synack_type, &remaining_out);
+
+	reserved = remaining - remaining_out;
+	if (!reserved)
+		return remaining;
+
+	/* round up to 4 bytes */
+	reserved = (reserved + 3) & ~3;
+
+	opts->bpf_opt_len += reserved;
+	return remaining - reserved;
+}
+
 static __be32 *process_tcp_ao_options(struct tcp_sock *tp,
 				      const struct tcp_request_sock *tcprsk,
 				      struct tcp_out_options *opts,
@@ -1053,6 +1089,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
 
 	remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
 					  remaining);
+	remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
+					    remaining);
 
 	return MAX_TCP_OPTION_SPACE - remaining;
 }
@@ -1141,6 +1179,8 @@ static unsigned int tcp_synack_options(const struct sock *sk,
 
 	remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
 					  synack_type, opts, remaining);
+	remaining = bpf_tcp_ops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
+					    synack_type, opts, remaining);
 
 	return MAX_TCP_OPTION_SPACE - remaining;
 }
@@ -1157,6 +1197,7 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
 	unsigned int eff_sacks;
 
 	opts->options = 0;
+	opts->bpf_opt_len = 0;
 
 	/* Better than switch (key.type) as it has static branches */
 	if (tcp_key_is_md5(key)) {
@@ -1244,6 +1285,15 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
 		size = MAX_TCP_OPTION_SPACE - remaining;
 	}
 
+	if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) {
+		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
+
+		remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
+						    remaining);
+
+		size = MAX_TCP_OPTION_SPACE - remaining;
+	}
+
 	return size;
 }
 
@@ -2252,13 +2302,21 @@ static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
  * in bigger TSO bursts. We we cut the RTT-based allowance in half
  * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance
  * is below 1500 bytes after 6 * ~500 usec = 3ms.
+ *
+ * The min_tso_segs is floored to 1 to avoid surprising conversion. Also,
+ * BPF callers may pass mss_now == 0. In that case the function returns the
+ * sanitized min_tso_segs value and skips autosizing.
  */
-static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
-			    int min_tso_segs)
+__bpf_kfunc u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
+				 int min_tso_segs)
 {
+	u32 min_tso = max(min_tso_segs, 1);
 	unsigned long bytes;
 	u32 r;
 
+	if (unlikely(!mss_now))
+		return min_tso;
+
 	bytes = READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift);
 
 	r = tcp_min_rtt(tcp_sk(sk)) >> READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_rtt_log);
@@ -2267,8 +2325,9 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
 
 	bytes = min_t(unsigned long, bytes, sk->sk_gso_max_size);
 
-	return max_t(u32, bytes / mss_now, min_tso_segs);
+	return max_t(u32, bytes / mss_now, min_tso);
 }
+EXPORT_SYMBOL_GPL(tcp_tso_autosize);
 
 /* Return the number of segments we want in the skb we are transmitting.
  * See if congestion control module wants to decide; otherwise, autosize.
@@ -2276,14 +2335,13 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
 static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
 {
 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
-	u32 min_tso, tso_segs;
+	u32 tso_segs;
 
-	min_tso = ca_ops->min_tso_segs ?
-			ca_ops->min_tso_segs(sk) :
-			READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs);
-
-	tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);
-	return min_t(u32, tso_segs, sk->sk_gso_max_segs);
+	tso_segs = ca_ops->tso_segs ?
+			ca_ops->tso_segs(sk, mss_now) :
+			tcp_tso_autosize(sk, mss_now,
+					 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs));
+	return clamp_t(u32, tso_segs, 1, sk->sk_gso_max_segs);
 }
 
 /* Returns the portion of skb which can be sent right away */
@@ -3678,6 +3736,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
 	if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))
 		tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB,
 				  TCP_SKB_CB(skb)->seq, segs, err);
+	bpf_tcp_ops_call(retrans, sk, skb, err);
 
 	if (unlikely(err) && err != -EBUSY)
 		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs);
@@ -3886,6 +3945,7 @@ void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason)
  */
 int tcp_send_synack(struct sock *sk)
 {
+	struct tcp_sock *tp = tcp_sk(sk);
 	struct sk_buff *skb;
 
 	skb = tcp_rtx_queue_head(sk);
@@ -3903,6 +3963,8 @@ int tcp_send_synack(struct sock *sk)
 			if (!nskb)
 				return -ENOMEM;
 			INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor);
+			if (skb == tp->retransmit_skb_hint)
+				tp->retransmit_skb_hint = nskb;
 			tcp_highest_sack_replace(sk, skb, nskb);
 			tcp_rtx_queue_unlink_and_free(skb, sk);
 			__skb_header_release(nskb);
@@ -4302,6 +4364,7 @@ int tcp_connect(struct sock *sk)
 	int err;
 
 	tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL);
+	bpf_tcp_ops_call(connect, sk);
 
 #if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO)
 	/* Has to be checked late, after setting daddr/saddr/ops.
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c
index e56eae4bc341e9..3d49adc5176632 100644
--- a/net/ipv4/tcp_timer.c
+++ b/net/ipv4/tcp_timer.c
@@ -290,6 +290,7 @@ static int tcp_write_timeout(struct sock *sk)
 		tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RTO_CB,
 				  icsk->icsk_retransmits,
 				  icsk->icsk_rto, (int)expired);
+	bpf_tcp_ops_call(rto, sk);
 
 	if (expired) {
 		/* Has it gone just too far? */
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index bb8cfc62cb0043..303d0102519ac1 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -139,25 +139,26 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num,
 	kuid_t uid = sk_uid(sk);
 	struct sock *sk2;
 
+	/* With @bitmap we are scanning for a free port: every port in use
+	 * is marked, whatever reuse options the sockets have. Without it
+	 * we are checking a specific port and honour the reuse options.
+	 */
 	sk_for_each(sk2, &hslot->head) {
 		if (net_eq(sock_net(sk2), net) &&
 		    sk2 != sk &&
 		    (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
-		    (!sk2->sk_reuse || !sk->sk_reuse) &&
+		    (bitmap || !sk2->sk_reuse || !sk->sk_reuse) &&
 		    (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
 		     sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
 		    inet_rcv_saddr_equal(sk, sk2, true)) {
-			if (sk2->sk_reuseport && sk->sk_reuseport &&
-			    !rcu_access_pointer(sk->sk_reuseport_cb) &&
-			    uid_eq(uid, sk_uid(sk2))) {
-				if (!bitmap)
+			if (!bitmap) {
+				if (sk2->sk_reuseport && sk->sk_reuseport &&
+				    !rcu_access_pointer(sk->sk_reuseport_cb) &&
+				    uid_eq(uid, sk_uid(sk2)))
 					return 0;
-			} else {
-				if (!bitmap)
-					return 1;
-				__set_bit(udp_sk(sk2)->udp_port_hash >> log,
-					  bitmap);
+				return 1;
 			}
+			__set_bit(udp_sk(sk2)->udp_port_hash >> log, bitmap);
 		}
 	}
 	return 0;
@@ -617,14 +618,23 @@ void udp_lib_hash4(struct sock *sk, u16 hash)
 	struct net *net = sock_net(sk);
 	struct udp_table *udptable;
 
-	/* Connected udp socket can re-connect to another remote address, which
-	 * will be handled by rehash. Thus no need to redo hash4 here.
+	udptable = net->ipv4.udp_table;
+	hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
+
+	/* A connected socket can re-connect to another address. rehash()
+	 * relocates it, but only runs when the local address changes, so a
+	 * socket bound to a specific address would stay filed under the
+	 * previous peer's hash. Move it here.
 	 */
-	if (udp_hashed4(sk))
+	if (udp_hashed4(sk)) {
+		if (udp_sk(sk)->udp_lrpa_hash != hash) {
+			spin_lock_bh(&hslot->lock);
+			udp_rehash4(udptable, sk, hash);
+			spin_unlock_bh(&hslot->lock);
+		}
 		return;
+	}
 
-	udptable = net->ipv4.udp_table;
-	hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
 	hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
 	hslot4 = udp_hashslot4(udptable, hash);
 	udp_sk(sk)->udp_lrpa_hash = hash;
@@ -2197,9 +2207,31 @@ int __udp_disconnect(struct sock *sk, int flags)
 }
 EXPORT_SYMBOL(__udp_disconnect);
 
+/* __udp_disconnect() takes a socket out of the 4-tuple hash table only via
+ * ->rehash() or ->unhash(), and neither runs for a socket bound to a
+ * specific address and port. Remove it here, before its peer is cleared.
+ */
+static void udp_unhash4_on_disconnect(struct sock *sk)
+{
+	struct net *net = sock_net(sk);
+	struct udp_table *udptable;
+	struct udp_hslot *hslot;
+
+	if (!udp_hashed4(sk))
+		return;
+
+	udptable = net->ipv4.udp_table;
+	hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
+
+	spin_lock_bh(&hslot->lock);
+	udp_unhash4(udptable, sk);
+	spin_unlock_bh(&hslot->lock);
+}
+
 int udp_disconnect(struct sock *sk, int flags)
 {
 	lock_sock(sk);
+	udp_unhash4_on_disconnect(sk);
 	__udp_disconnect(sk, flags);
 	release_sock(sk);
 	return 0;
@@ -3131,6 +3163,7 @@ int udp_abort(struct sock *sk, int err)
 
 	sk->sk_err = err;
 	sk_error_report(sk);
+	udp_unhash4_on_disconnect(sk);
 	__udp_disconnect(sk, 0);
 
 out:
@@ -3294,15 +3327,14 @@ static void udp4_format_sock(struct sock *sp, struct seq_file *f,
 	__u16 srcp	  = ntohs(inet->inet_sport);
 
 	seq_printf(f, "%5d: %08X:%04X %08X:%04X"
-		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d %pK %u",
+		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d 0 %u",
 		bucket, src, srcp, dest, destp, sp->sk_state,
 		sk_wmem_alloc_get(sp),
 		udp_rqueue_get(sp),
 		0, 0L, 0,
 		from_kuid_munged(seq_user_ns(f), sk_uid(sp)),
 		0, sock_i_ino(sp),
-		refcount_read(&sp->sk_refcnt), sp,
-		sk_drops_read(sp));
+		refcount_read(&sp->sk_refcnt), sk_drops_read(sp));
 }
 
 static int udp4_seq_show(struct seq_file *seq, void *v)
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index 9d89be7e0544be..7bd0e8820abb96 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -376,6 +376,8 @@ static int snmp6_alloc_dev(struct inet6_dev *idev)
 
 static struct inet6_dev *ipv6_add_dev(struct net_device *dev)
 {
+	struct net *net = dev_net(dev);
+	struct neigh_table *tbl;
 	struct inet6_dev *ndev;
 	int err = -ENOMEM;
 
@@ -393,14 +395,15 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev)
 	ndev->dev = dev;
 	INIT_LIST_HEAD(&ndev->addr_list);
 	timer_setup(&ndev->rs_timer, addrconf_rs_timer, 0);
-	memcpy(&ndev->cnf, dev_net(dev)->ipv6.devconf_dflt, sizeof(ndev->cnf));
+	memcpy(&ndev->cnf, net->ipv6.devconf_dflt, sizeof(ndev->cnf));
 
 	if (ndev->cnf.stable_secret.initialized)
 		ndev->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_STABLE_PRIVACY;
 
+	tbl = nd_table(net);
 	ndev->cnf.mtu6 = dev->mtu;
 	ndev->ra_mtu = 0;
-	ndev->nd_parms = neigh_parms_alloc(dev, &nd_tbl);
+	ndev->nd_parms = neigh_parms_alloc(dev, tbl);
 	if (!ndev->nd_parms) {
 		kfree(ndev);
 		return ERR_PTR(err);
@@ -413,7 +416,7 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev)
 	if (snmp6_alloc_dev(ndev) < 0) {
 		netdev_dbg(dev, "%s: cannot allocate memory for statistics\n",
 			   __func__);
-		neigh_parms_release(&nd_tbl, ndev->nd_parms);
+		neigh_parms_release(tbl, ndev->nd_parms);
 		netdev_put(dev, &ndev->dev_tracker);
 		kfree(ndev);
 		return ERR_PTR(err);
@@ -481,7 +484,7 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev)
 	return ndev;
 
 err_release:
-	neigh_parms_release(&nd_tbl, ndev->nd_parms);
+	neigh_parms_release(tbl, ndev->nd_parms);
 	ndev->dead = 1;
 	in6_dev_finish_destroy(ndev);
 	return ERR_PTR(err);
@@ -4039,9 +4042,11 @@ static int addrconf_ifdown(struct net_device *dev, bool unregister)
 
 	/* Last: Shot the device (if unregistered) */
 	if (unregister) {
+		struct neigh_table *tbl = nd_table(net);
+
 		addrconf_sysctl_unregister(idev);
-		neigh_parms_release(&nd_tbl, idev->nd_parms);
-		neigh_ifdown(&nd_tbl, dev);
+		neigh_parms_release(tbl, idev->nd_parms);
+		neigh_ifdown(tbl, dev);
 		in6_dev_put(idev);
 	}
 	return 0;
@@ -4874,7 +4879,7 @@ static int modify_prefix_route(struct net *net, struct inet6_ifaddr *ifp,
 			fib6_clean_expires(f6i);
 			fib6_may_remove_gc_list(net, f6i);
 		} else {
-			fib6_set_expires(f6i, expires);
+			fib6_set_expires(f6i, jiffies + clock_t_to_jiffies(expires));
 			fib6_add_gc_list(f6i);
 		}
 
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index 282912a1199992..f0efdc13baf4e0 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -429,7 +429,6 @@ int inet6_bind_sk(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len)
 	const struct proto *prot;
 	int err = 0;
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	prot = READ_ONCE(sk->sk_prot);
 	/* If the socket has its own bind function then use it. */
 	if (prot->bind)
@@ -567,7 +566,6 @@ int inet6_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
 	case SIOCSIFDSTADDR:
 		return addrconf_set_dstaddr(net, argp);
 	default:
-		/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 		prot = READ_ONCE(sk->sk_prot);
 		if (!prot->ioctl)
 			return -ENOIOCTLCMD;
@@ -636,7 +634,6 @@ int inet6_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 	if (unlikely(inet_send_prepare(sk)))
 		return -EAGAIN;
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	prot = READ_ONCE(sk->sk_prot);
 	return INDIRECT_CALL_2(prot->sendmsg, tcp_sendmsg, udpv6_sendmsg,
 			       sk, msg, size);
@@ -651,7 +648,6 @@ int inet6_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
 	if (likely(!(flags & MSG_ERRQUEUE)))
 		sock_rps_record_flow(sk);
 
-	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
 	prot = READ_ONCE(sk->sk_prot);
 	return INDIRECT_CALL_2(prot->recvmsg, tcp_recvmsg, udpv6_recvmsg,
 			       sk, msg, size, flags);
diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c
index c96f7e0d0a4815..ec9aba2bc6e4e2 100644
--- a/net/ipv6/ah6.c
+++ b/net/ipv6/ah6.c
@@ -169,7 +169,7 @@ static bool zero_out_mutable_opts(struct ipv6_opt_hdr *opthdr)
 /**
  *	ipv6_rearrange_destopt - rearrange IPv6 destination options header
  *	@iph: IPv6 header
- *	@destopt: destionation options header
+ *	@destopt: destination options header
  */
 static void ipv6_rearrange_destopt(struct ipv6hdr *iph, struct ipv6_opt_hdr *destopt)
 {
diff --git a/net/ipv6/datagram.c b/net/ipv6/datagram.c
index 38d7b484528174..191c9733ff9fa1 100644
--- a/net/ipv6/datagram.c
+++ b/net/ipv6/datagram.c
@@ -1102,7 +1102,7 @@ void __ip6_dgram_sock_seq_show(struct seq_file *seq, struct sock *sp,
 	src   = &sp->sk_v6_rcv_saddr;
 	seq_printf(seq,
 		   "%5d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
-		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d %pK %u\n",
+		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d 0 %u\n",
 		   bucket,
 		   src->s6_addr32[0], src->s6_addr32[1],
 		   src->s6_addr32[2], src->s6_addr32[3], srcp,
@@ -1115,6 +1115,5 @@ void __ip6_dgram_sock_seq_show(struct seq_file *seq, struct sock *sp,
 		   from_kuid_munged(seq_user_ns(seq), sk_uid(sp)),
 		   0,
 		   sock_i_ino(sp),
-		   refcount_read(&sp->sk_refcnt), sp,
-		   sk_drops_read(sp));
+		   refcount_read(&sp->sk_refcnt), sk_drops_read(sp));
 }
diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c
index 09a4552f7f08aa..55391e2e561294 100644
--- a/net/ipv6/exthdrs.c
+++ b/net/ipv6/exthdrs.c
@@ -464,7 +464,7 @@ static int ipv6_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev)
 			__IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
 			icmpv6_send(skb, ICMPV6_TIME_EXCEED,
 				    ICMPV6_EXC_HOPLIMIT, 0);
-			kfree_skb(skb);
+			kfree_skb_reason(skb, SKB_DROP_REASON_IP_TTL_EXCEEDED);
 			return -1;
 		}
 		ipv6_hdr(skb)->hop_limit--;
@@ -623,7 +623,7 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev)
 			__IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
 			icmpv6_send(skb, ICMPV6_TIME_EXCEED,
 				    ICMPV6_EXC_HOPLIMIT, 0);
-			kfree_skb(skb);
+			kfree_skb_reason(skb, SKB_DROP_REASON_IP_TTL_EXCEEDED);
 			return -1;
 		}
 		ipv6_hdr(skb)->hop_limit--;
@@ -815,7 +815,7 @@ static int ipv6_rthdr_rcv(struct sk_buff *skb)
 			__IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
 			icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT,
 				    0);
-			kfree_skb(skb);
+			kfree_skb_reason(skb, SKB_DROP_REASON_IP_TTL_EXCEEDED);
 			return -1;
 		}
 		ipv6_hdr(skb)->hop_limit--;
diff --git a/net/ipv6/exthdrs_core.c b/net/ipv6/exthdrs_core.c
index 9d06d487e8b103..4a9748338cf400 100644
--- a/net/ipv6/exthdrs_core.c
+++ b/net/ipv6/exthdrs_core.c
@@ -278,6 +278,9 @@ int ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset,
 			hdrlen = ipv6_optlen(hp);
 
 		if (!found) {
+			if (skb->len - start < hdrlen)
+				return -EBADMSG;
+
 			nexthdr = hp->nexthdr;
 			start += hdrlen;
 		}
diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
index 9ea75703b38d4a..0dc6ebee6e5686 100644
--- a/net/ipv6/ip6_fib.c
+++ b/net/ipv6/ip6_fib.c
@@ -1019,12 +1019,6 @@ static int fib6_nh_drop_pcpu_from(struct fib6_nh *nh, void *_arg)
 
 static void fib6_drop_pcpu_from(struct fib6_info *f6i)
 {
-	/* Make sure rt6_make_pcpu_route() wont add other percpu routes
-	 * while we are cleaning them here.
-	 */
-	f6i->fib6_destroying = 1;
-	mb(); /* paired with the cmpxchg() in rt6_make_pcpu_route() */
-
 	if (f6i->nh) {
 		rcu_read_lock();
 		nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_drop_pcpu_from, f6i);
@@ -1042,9 +1036,15 @@ static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,
 {
 	struct fib6_table *table = rt->fib6_table;
 
-	/* Flush all cached dst in exception table */
-	rt6_flush_exceptions(rt);
+	/* Make sure rt6_make_pcpu_route() / rt6_insert_exception()
+	 * will not add other percpu / exception routes after cleaning
+	 * them up in __fib6_drop_pcpu_from() / rt6_flush_exceptions().
+	 */
+	rt->fib6_destroying = 1;
+	mb(); /* paired with the cmpxchg() in rt6_make_pcpu_route() */
+
 	fib6_drop_pcpu_from(rt);
+	rt6_flush_exceptions(rt);
 
 	if (rt->nh) {
 		spin_lock(&rt->nh->lock);
diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c
index 8ebda0b6a78b22..258239a7c53bbc 100644
--- a/net/ipv6/ip6_gre.c
+++ b/net/ipv6/ip6_gre.c
@@ -64,7 +64,8 @@ MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");
 
 static unsigned int ip6gre_net_id __read_mostly;
 struct ip6gre_net {
-	struct ip6_tnl __rcu *tunnels[4][IP6_GRE_HASH_SIZE];
+	struct hlist_head tunnels[4][IP6_GRE_HASH_SIZE];
+	struct mutex tunnels_lock;
 
 	struct ip6_tnl __rcu *collect_md_tun;
 	struct ip6_tnl __rcu *collect_md_tun_erspan;
@@ -79,6 +80,8 @@ static void ip6gre_tunnel_setup(struct net_device *dev);
 static void ip6gre_tunnel_link(struct ip6gre_net *ign, struct ip6_tnl *t);
 static void ip6gre_tnl_link_config(struct ip6_tnl *t, int set_mtu);
 static void ip6erspan_tnl_link_config(struct ip6_tnl *t, int set_mtu);
+static void __ip6gre_dellink(struct net *net, struct net_device *dev,
+			     struct list_head *head);
 
 /* Tunnel hash table */
 
@@ -141,20 +144,24 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev,
 		const struct in6_addr *remote, const struct in6_addr *local,
 		__be32 key, __be16 gre_proto)
 {
-	struct net *net = dev_net(dev);
-	int link = dev->ifindex;
-	unsigned int h0 = HASH_ADDR(remote);
-	unsigned int h1 = HASH_KEY(key);
-	struct ip6_tnl *t, *cand = NULL;
-	struct ip6gre_net *ign = net_generic(net, ip6gre_net_id);
 	int dev_type = (gre_proto == htons(ETH_P_TEB) ||
 			gre_proto == htons(ETH_P_ERSPAN) ||
 			gre_proto == htons(ETH_P_ERSPAN2)) ?
 		       ARPHRD_ETHER : ARPHRD_IP6GRE;
+	unsigned int h0 = HASH_ADDR(remote);
+	unsigned int h1 = HASH_KEY(key);
+	struct ip6_tnl *t, *cand = NULL;
+	struct net *net = dev_net(dev);
 	struct net_device *ndev;
+	struct hlist_head *head;
+	int link = dev->ifindex;
+	struct ip6gre_net *ign;
 	int cand_score = 4;
 
-	for_each_ip_tunnel_rcu(t, ign->tunnels_r_l[h0 ^ h1]) {
+	ign = net_generic(net, ip6gre_net_id);
+
+	head = &ign->tunnels_r_l[h0 ^ h1];
+	hlist_for_each_entry_rcu(t, head, hash_node) {
 		if (!ipv6_addr_equal(local, &t->parms.laddr) ||
 		    !ipv6_addr_equal(remote, &t->parms.raddr) ||
 		    key != t->parms.i_key ||
@@ -165,7 +172,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev,
 			return cand;
 	}
 
-	for_each_ip_tunnel_rcu(t, ign->tunnels_r[h0 ^ h1]) {
+	head = &ign->tunnels_r[h0 ^ h1];
+	hlist_for_each_entry_rcu(t, head, hash_node) {
 		if (!ipv6_addr_equal(remote, &t->parms.raddr) ||
 		    key != t->parms.i_key ||
 		    !(t->dev->flags & IFF_UP))
@@ -175,7 +183,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev,
 			return cand;
 	}
 
-	for_each_ip_tunnel_rcu(t, ign->tunnels_l[h1]) {
+	head = &ign->tunnels_l[h1];
+	hlist_for_each_entry_rcu(t, head, hash_node) {
 		if ((!ipv6_addr_equal(local, &t->parms.laddr) &&
 			  (!ipv6_addr_equal(local, &t->parms.raddr) ||
 				 !ipv6_addr_is_multicast(local))) ||
@@ -187,7 +196,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev,
 			return cand;
 	}
 
-	for_each_ip_tunnel_rcu(t, ign->tunnels_wc[h1]) {
+	head = &ign->tunnels_wc[h1];
+	hlist_for_each_entry_rcu(t, head, hash_node) {
 		if (t->parms.i_key != key ||
 		    !(t->dev->flags & IFF_UP))
 			continue;
@@ -215,8 +225,8 @@ static struct ip6_tnl *ip6gre_tunnel_lookup(struct net_device *dev,
 	return NULL;
 }
 
-static struct ip6_tnl __rcu **__ip6gre_bucket(struct ip6gre_net *ign,
-		const struct __ip6_tnl_parm *p)
+static struct hlist_head *__ip6gre_bucket(struct ip6gre_net *ign,
+					  const struct __ip6_tnl_parm *p)
 {
 	const struct in6_addr *remote = &p->raddr;
 	const struct in6_addr *local = &p->laddr;
@@ -258,56 +268,51 @@ static void ip6erspan_tunnel_unlink_md(struct ip6gre_net *ign,
 		rcu_assign_pointer(ign->collect_md_tun_erspan, NULL);
 }
 
-static inline struct ip6_tnl __rcu **ip6gre_bucket(struct ip6gre_net *ign,
-		const struct ip6_tnl *t)
+static inline struct hlist_head *ip6gre_bucket(struct ip6gre_net *ign,
+					       const struct ip6_tnl *t)
 {
 	return __ip6gre_bucket(ign, &t->parms);
 }
 
 static void ip6gre_tunnel_link(struct ip6gre_net *ign, struct ip6_tnl *t)
 {
-	struct ip6_tnl __rcu **tp = ip6gre_bucket(ign, t);
+	struct hlist_head *head = ip6gre_bucket(ign, t);
 
-	rcu_assign_pointer(t->next, rtnl_dereference(*tp));
-	rcu_assign_pointer(*tp, t);
+	hlist_add_head_rcu(&t->hash_node, head);
 }
 
 static void ip6gre_tunnel_unlink(struct ip6gre_net *ign, struct ip6_tnl *t)
 {
-	struct ip6_tnl __rcu **tp;
-	struct ip6_tnl *iter;
+	hlist_del_init_rcu(&t->hash_node);
+}
 
-	for (tp = ip6gre_bucket(ign, t);
-	     (iter = rtnl_dereference(*tp)) != NULL;
-	     tp = &iter->next) {
-		if (t == iter) {
-			rcu_assign_pointer(*tp, t->next);
-			break;
-		}
-	}
+static bool ip6gre_tunnel_unregistering(struct ip6_tnl *t)
+{
+	return hlist_unhashed(&t->hash_node);
 }
 
 static struct ip6_tnl *ip6gre_tunnel_find(struct net *net,
 					   const struct __ip6_tnl_parm *parms,
 					   int type)
 {
+	struct ip6gre_net *ign = net_generic(net, ip6gre_net_id);
 	const struct in6_addr *remote = &parms->raddr;
 	const struct in6_addr *local = &parms->laddr;
 	__be32 key = parms->i_key;
+	struct hlist_head *head;
 	int link = parms->link;
 	struct ip6_tnl *t;
-	struct ip6_tnl __rcu **tp;
-	struct ip6gre_net *ign = net_generic(net, ip6gre_net_id);
 
-	for (tp = __ip6gre_bucket(ign, parms);
-	     (t = rtnl_dereference(*tp)) != NULL;
-	     tp = &t->next)
+	head = __ip6gre_bucket(ign, parms);
+
+	hlist_for_each_entry(t, head, hash_node) {
 		if (ipv6_addr_equal(local, &t->parms.laddr) &&
 		    ipv6_addr_equal(remote, &t->parms.raddr) &&
 		    key == t->parms.i_key &&
 		    link == t->parms.link &&
 		    type == t->dev->type)
 			break;
+	}
 
 	return t;
 }
@@ -362,10 +367,7 @@ static struct ip6_tnl *ip6gre_tunnel_locate(struct net *net,
 static void ip6erspan_tunnel_uninit(struct net_device *dev)
 {
 	struct ip6_tnl *t = netdev_priv(dev);
-	struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id);
 
-	ip6erspan_tunnel_unlink_md(ign, t);
-	ip6gre_tunnel_unlink(ign, t);
 	dst_cache_reset(&t->dst_cache);
 	netdev_put(dev, &t->dev_tracker);
 }
@@ -373,12 +375,7 @@ static void ip6erspan_tunnel_uninit(struct net_device *dev)
 static void ip6gre_tunnel_uninit(struct net_device *dev)
 {
 	struct ip6_tnl *t = netdev_priv(dev);
-	struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id);
 
-	ip6gre_tunnel_unlink_md(ign, t);
-	ip6gre_tunnel_unlink(ign, t);
-	if (ign->fb_tunnel_dev == dev)
-		WRITE_ONCE(ign->fb_tunnel_dev, NULL);
 	dst_cache_reset(&t->dst_cache);
 	netdev_put(dev, &t->dev_tracker);
 }
@@ -1256,14 +1253,25 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev,
 					struct ifreq *ifr, void __user *data,
 					int cmd)
 {
-	int err = 0;
-	struct ip6_tnl_parm2 p;
-	struct __ip6_tnl_parm p1;
 	struct ip6_tnl *t = netdev_priv(dev);
+	struct net *orig_net = dev_net(dev);
+	struct __ip6_tnl_parm p1 = {};
+	LIST_HEAD(dev_kill_list);
 	struct net *net = t->net;
-	struct ip6gre_net *ign = net_generic(net, ip6gre_net_id);
+	struct ip6_tnl_parm2 p;
+	struct ip6gre_net *ign;
+	int err = 0;
+
+	DEBUG_NET_WARN_ON_ONCE(netdev_need_ops_lock(dev));
 
-	memset(&p1, 0, sizeof(p1));
+	ign = net_generic(net, ip6gre_net_id);
+
+	mutex_lock(&ign->tunnels_lock);
+
+	if (!check_net(net)) {
+		err = -EBUSY;
+		goto done;
+	}
 
 	switch (cmd) {
 	case SIOCGETTUNNEL:
@@ -1306,7 +1314,7 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev,
 		t = ip6gre_tunnel_locate(net, &p1, cmd == SIOCADDTUNNEL);
 
 		if (dev != ign->fb_tunnel_dev && cmd == SIOCCHGTUNNEL) {
-			if (t) {
+			if (t && !ip6gre_tunnel_unregistering(t)) {
 				if (t->dev != dev) {
 					err = -EEXIST;
 					break;
@@ -1314,23 +1322,26 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev,
 			} else {
 				t = netdev_priv(dev);
 
-				ip6gre_tunnel_unlink(ign, t);
-				synchronize_net();
-				ip6gre_tnl_change(t, &p1, 1);
-				ip6gre_tunnel_link(ign, t);
-				netdev_state_change(dev);
+				if (!ip6gre_tunnel_unregistering(t)) {
+					ip6gre_tunnel_unlink(ign, t);
+					synchronize_net();
+					ip6gre_tnl_change(t, &p1, 1);
+					ip6gre_tunnel_link(ign, t);
+					netdev_state_change(dev);
+				}
 			}
 		}
 
-		if (t) {
+		if (t && !ip6gre_tunnel_unregistering(t)) {
 			err = 0;
 
 			memset(&p, 0, sizeof(p));
 			ip6gre_tnl_parm_to_user(&p, &t->parms);
 			if (copy_to_user(data, &p, sizeof(p)))
 				err = -EFAULT;
-		} else
+		} else {
 			err = (cmd == SIOCADDTUNNEL ? -ENOBUFS : -ENOENT);
+		}
 		break;
 
 	case SIOCDELTUNNEL:
@@ -1352,7 +1363,9 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev,
 				goto done;
 			dev = t->dev;
 		}
-		unregister_netdevice(dev);
+
+		if (!ip6gre_tunnel_unregistering(t))
+			__ip6gre_dellink(orig_net, dev, &dev_kill_list);
 		err = 0;
 		break;
 
@@ -1361,6 +1374,9 @@ static int ip6gre_tunnel_siocdevprivate(struct net_device *dev,
 	}
 
 done:
+	mutex_unlock(&ign->tunnels_lock);
+
+	unregister_netdevice_many(&dev_kill_list);
 	return err;
 }
 
@@ -1551,44 +1567,47 @@ static struct inet6_protocol ip6gre_protocol __read_mostly = {
 	.flags       = INET6_PROTO_FINAL,
 };
 
-static void __net_exit ip6gre_exit_rtnl_net(struct net *net, struct list_head *head)
+static void __net_exit ip6gre_exit_rtnl_net(struct net *net,
+					    struct list_head *dev_kill_list)
 {
 	struct ip6gre_net *ign = net_generic(net, ip6gre_net_id);
-	struct net_device *dev, *aux;
 	int prio;
 
-	for_each_netdev_safe(net, dev, aux)
-		if (dev->rtnl_link_ops == &ip6gre_link_ops ||
-		    dev->rtnl_link_ops == &ip6gre_tap_ops ||
-		    dev->rtnl_link_ops == &ip6erspan_tap_ops)
-			unregister_netdevice_queue(dev, head);
+	WRITE_ONCE(ign->fb_tunnel_dev, NULL);
+
+	mutex_lock(&ign->tunnels_lock);
 
 	for (prio = 0; prio < 4; prio++) {
 		int h;
+
 		for (h = 0; h < IP6_GRE_HASH_SIZE; h++) {
+			struct hlist_head *head = &ign->tunnels[prio][h];
+			struct hlist_node *tmp;
 			struct ip6_tnl *t;
 
-			t = rtnl_net_dereference(net, ign->tunnels[prio][h]);
-
-			while (t) {
-				/* If dev is in the same netns, it has already
-				 * been added to the list by the previous loop.
-				 */
-				if (!net_eq(dev_net(t->dev), net))
-					unregister_netdevice_queue(t->dev, head);
-
-				t = rtnl_net_dereference(net, t->next);
-			}
+			hlist_for_each_entry_safe(t, tmp, head, hash_node)
+				__ip6gre_dellink(net, t->dev, dev_kill_list);
 		}
 	}
+
+	mutex_unlock(&ign->tunnels_lock);
 }
 
 static int __net_init ip6gre_init_net(struct net *net)
 {
 	struct ip6gre_net *ign = net_generic(net, ip6gre_net_id);
 	struct net_device *ndev;
+	struct ip6_tnl *t;
+	int prio, h;
 	int err;
 
+	for (prio = 0; prio < 4; prio++) {
+		for (h = 0; h < IP6_GRE_HASH_SIZE; h++)
+			INIT_HLIST_HEAD(&ign->tunnels[prio][h]);
+	}
+
+	mutex_init(&ign->tunnels_lock);
+
 	if (!net_has_fallback_tunnels(net))
 		return 0;
 	ndev = alloc_netdev(sizeof(struct ip6_tnl), "ip6gre0",
@@ -1607,12 +1626,13 @@ static int __net_init ip6gre_init_net(struct net *net)
 	ip6gre_fb_tunnel_init(ign->fb_tunnel_dev);
 	ign->fb_tunnel_dev->rtnl_link_ops = &ip6gre_link_ops;
 
+	t = netdev_priv(ign->fb_tunnel_dev);
+	hlist_add_head_rcu(&t->hash_node, &ign->tunnels_wc[0]);
+
 	err = register_netdev(ign->fb_tunnel_dev);
 	if (err)
 		goto err_reg_dev;
 
-	rcu_assign_pointer(ign->tunnels_wc[0],
-			   netdev_priv(ign->fb_tunnel_dev));
 	return 0;
 
 err_reg_dev:
@@ -1984,18 +2004,22 @@ static int ip6gre_newlink(struct net_device *dev,
 	struct nlattr **data = params->data;
 	struct nlattr **tb = params->tb;
 	struct ip6gre_net *ign;
-	int err;
+	int err = 0;
 
 	ip6gre_netlink_parms(data, &nt->parms);
 	ign = net_generic(net, ip6gre_net_id);
 
+	mutex_lock(&ign->tunnels_lock);
+
 	if (nt->parms.collect_md) {
 		if (rtnl_dereference(ign->collect_md_tun))
-			return -EEXIST;
+			err = -EEXIST;
 	} else {
 		if (ip6gre_tunnel_find(net, &nt->parms, dev->type))
-			return -EEXIST;
+			err = -EEXIST;
 	}
+	if (err)
+		goto unlock;
 
 	err = ip6gre_newlink_common(net, dev, tb, data, extack);
 	if (!err) {
@@ -2003,6 +2027,10 @@ static int ip6gre_newlink(struct net_device *dev,
 		ip6gre_tunnel_link_md(ign, nt);
 		ip6gre_tunnel_link(net_generic(net, ip6gre_net_id), nt);
 	}
+
+unlock:
+	mutex_unlock(&ign->tunnels_lock);
+
 	return err;
 }
 
@@ -2037,6 +2065,9 @@ ip6gre_changelink_common(struct net_device *dev, struct nlattr *tb[],
 		t = nt;
 	}
 
+	if (ip6gre_tunnel_unregistering(t))
+		return ERR_PTR(-ENODEV);
+
 	return t;
 }
 
@@ -2045,31 +2076,65 @@ static int ip6gre_changelink(struct net_device *dev, struct nlattr *tb[],
 			     struct netlink_ext_ack *extack)
 {
 	struct ip6_tnl *t = netdev_priv(dev);
-	struct ip6gre_net *ign = net_generic(t->net, ip6gre_net_id);
 	struct __ip6_tnl_parm p;
+	struct ip6gre_net *ign;
+	int err = 0;
+
+	ign = net_generic(t->net, ip6gre_net_id);
 
 	if (!rtnl_dev_link_net_capable(dev, t->net))
 		return -EPERM;
 
+	mutex_lock(&ign->tunnels_lock);
+
 	t = ip6gre_changelink_common(dev, tb, data, &p, extack);
-	if (IS_ERR(t))
-		return PTR_ERR(t);
+	if (IS_ERR(t)) {
+		err = PTR_ERR(t);
+		goto unlock;
+	}
 
 	ip6gre_tunnel_unlink_md(ign, t);
 	ip6gre_tunnel_unlink(ign, t);
 	ip6gre_tnl_change(t, &p, !tb[IFLA_MTU]);
 	ip6gre_tunnel_link_md(ign, t);
 	ip6gre_tunnel_link(ign, t);
-	return 0;
+unlock:
+	mutex_unlock(&ign->tunnels_lock);
+
+	return err;
+}
+
+static void __ip6gre_dellink(struct net *net, struct net_device *dev,
+			     struct list_head *head)
+{
+	struct ip6_tnl *t = netdev_priv(dev);
+	struct ip6gre_net *ign;
+
+	ign = net_generic(t->net, ip6gre_net_id);
+
+	if (dev->rtnl_link_ops == &ip6erspan_tap_ops)
+		ip6erspan_tunnel_unlink_md(ign, t);
+	else
+		ip6gre_tunnel_unlink_md(ign, t);
+
+	ip6gre_tunnel_unlink(ign, t);
+	unregister_netdevice_queue_net(net, dev, head);
 }
 
 static void ip6gre_dellink(struct net_device *dev, struct list_head *head)
 {
-	struct net *net = dev_net(dev);
-	struct ip6gre_net *ign = net_generic(net, ip6gre_net_id);
+	struct ip6_tnl *t = netdev_priv(dev);
+	struct ip6gre_net *ign;
+
+	ign = net_generic(t->net, ip6gre_net_id);
 
-	if (dev != ign->fb_tunnel_dev)
-		unregister_netdevice_queue(dev, head);
+	mutex_lock(&ign->tunnels_lock);
+
+	if (dev != ign->fb_tunnel_dev &&
+	    !ip6gre_tunnel_unregistering(t))
+		__ip6gre_dellink(dev_net(dev), dev, head);
+
+	mutex_unlock(&ign->tunnels_lock);
 }
 
 static size_t ip6gre_get_size(const struct net_device *dev)
@@ -2225,19 +2290,23 @@ static int ip6erspan_newlink(struct net_device *dev,
 	struct nlattr **data = params->data;
 	struct nlattr **tb = params->tb;
 	struct ip6gre_net *ign;
-	int err;
+	int err = 0;
 
 	ip6gre_netlink_parms(data, &nt->parms);
 	ip6erspan_set_version(data, &nt->parms);
 	ign = net_generic(net, ip6gre_net_id);
 
+	mutex_lock(&ign->tunnels_lock);
+
 	if (nt->parms.collect_md) {
 		if (rtnl_dereference(ign->collect_md_tun_erspan))
-			return -EEXIST;
+			err = -EEXIST;
 	} else {
 		if (ip6gre_tunnel_find(net, &nt->parms, dev->type))
-			return -EEXIST;
+			err = -EEXIST;
 	}
+	if (err)
+		goto unlock;
 
 	err = ip6gre_newlink_common(net, dev, tb, data, extack);
 	if (!err) {
@@ -2245,6 +2314,10 @@ static int ip6erspan_newlink(struct net_device *dev,
 		ip6erspan_tunnel_link_md(ign, nt);
 		ip6gre_tunnel_link(net_generic(net, ip6gre_net_id), nt);
 	}
+
+unlock:
+	mutex_unlock(&ign->tunnels_lock);
+
 	return err;
 }
 
@@ -2269,22 +2342,31 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[],
 	struct ip6_tnl *t = netdev_priv(dev);
 	struct __ip6_tnl_parm p;
 	struct ip6gre_net *ign;
+	int err = 0;
 
 	if (!rtnl_dev_link_net_capable(dev, t->net))
 		return -EPERM;
 
 	ign = net_generic(t->net, ip6gre_net_id);
+
+	mutex_lock(&ign->tunnels_lock);
+
 	t = ip6gre_changelink_common(dev, tb, data, &p, extack);
-	if (IS_ERR(t))
-		return PTR_ERR(t);
+	if (IS_ERR(t)) {
+		err = PTR_ERR(t);
+		goto unlock;
+	}
 
 	ip6erspan_set_version(data, &p);
-	ip6gre_tunnel_unlink_md(ign, t);
+	ip6erspan_tunnel_unlink_md(ign, t);
 	ip6gre_tunnel_unlink(ign, t);
 	ip6erspan_tnl_change(t, &p, !tb[IFLA_MTU]);
 	ip6erspan_tunnel_link_md(ign, t);
 	ip6gre_tunnel_link(ign, t);
-	return 0;
+unlock:
+	mutex_unlock(&ign->tunnels_lock);
+
+	return err;
 }
 
 static struct rtnl_link_ops ip6gre_link_ops __read_mostly = {
@@ -2311,6 +2393,7 @@ static struct rtnl_link_ops ip6gre_tap_ops __read_mostly = {
 	.validate	= ip6gre_tap_validate,
 	.newlink	= ip6gre_newlink,
 	.changelink	= ip6gre_changelink,
+	.dellink	= ip6gre_dellink,
 	.get_size	= ip6gre_get_size,
 	.fill_info	= ip6gre_fill_info,
 	.get_link_net	= ip6_tnl_get_link_net,
@@ -2325,6 +2408,7 @@ static struct rtnl_link_ops ip6erspan_tap_ops __read_mostly = {
 	.validate	= ip6erspan_tap_validate,
 	.newlink	= ip6erspan_newlink,
 	.changelink	= ip6erspan_changelink,
+	.dellink	= ip6gre_dellink,
 	.get_size	= ip6gre_get_size,
 	.fill_info	= ip6gre_fill_info,
 	.get_link_net	= ip6_tnl_get_link_net,
diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
index 78f50c93c536cc..700f1861d8eb7d 100644
--- a/net/ipv6/ip6_offload.c
+++ b/net/ipv6/ip6_offload.c
@@ -17,6 +17,7 @@
 #include <net/udp.h>
 #include <net/gro.h>
 #include <net/gso.h>
+#include <net/ip_tunnels.h>
 
 #include "ip6_offload.h"
 #include "tcpv6_offload.c"
@@ -104,6 +105,9 @@ static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
 	bool gso_partial;
 
 	skb_reset_network_header(skb);
+	if (unlikely(gso_recursion_inc_test(skb,
+					   IP_TUNNEL_RECURSION_LIMIT)))
+		goto out;
 	nhoff = skb_network_header(skb) - skb_mac_header(skb);
 	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
 		goto out;
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 5509650589915c..10146a57b69e3c 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -127,7 +127,7 @@ static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *
 
 	if (IS_ERR_OR_NULL(neigh)) {
 		if (unlikely(!neigh))
-			neigh = __neigh_create(&nd_tbl, nexthop, dev, false);
+			neigh = __neigh_create(nd_table(net), nexthop, dev, false);
 		if (IS_ERR(neigh)) {
 			IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTNOROUTES);
 			kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_CREATEFAIL);
@@ -577,13 +577,13 @@ int ip6_forward(struct sk_buff *skb)
 		icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
 		__IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
 
-		kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR);
+		kfree_skb_reason(skb, SKB_DROP_REASON_IP_TTL_EXCEEDED);
 		return -ETIMEDOUT;
 	}
 
 	/* XXX: idev->cnf.proxy_ndp? */
 	if (READ_ONCE(net->ipv6.devconf_all->proxy_ndp) &&
-	    pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev)) {
+	    pneigh_lookup(nd_table(net), &hdr->daddr, skb->dev)) {
 		int proxied = ip6_forward_proxy_check(skb);
 
 		hdr = ipv6_hdr(skb);
diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c
index ab94b3a4ba9c3c..c790278391519c 100644
--- a/net/ipv6/ip6_vti.c
+++ b/net/ipv6/ip6_vti.c
@@ -437,7 +437,7 @@ static bool vti6_state_check(const struct xfrm_state *x,
  * vti6_xmit - send a packet
  *   @skb: the outgoing socket buffer
  *   @dev: the outgoing tunnel device
- *   @fl: the flow informations for the xfrm_lookup
+ *   @fl: the flow information for the xfrm_lookup
  **/
 static int
 vti6_xmit(struct sk_buff *skb, struct net_device *dev, struct flowi *fl)
diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c
index 9d8116b5edb173..36f117ad367081 100644
--- a/net/ipv6/ip6mr.c
+++ b/net/ipv6/ip6mr.c
@@ -485,8 +485,9 @@ static int ip6mr_vif_seq_show(struct seq_file *seq, void *v)
 		seq_printf(seq,
 			   "%2td %-10s %8ld %7ld  %8ld %7ld %05X\n",
 			   vif - mrt->vif_table,
-			   name, vif->bytes_in, vif->pkt_in,
-			   vif->bytes_out, vif->pkt_out,
+			   name,
+			   READ_ONCE(vif->bytes_in), READ_ONCE(vif->pkt_in),
+			   READ_ONCE(vif->bytes_out), READ_ONCE(vif->pkt_out),
 			   vif->flags);
 	}
 	return 0;
diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c
index b4c977434c2e0a..4b3536571c9804 100644
--- a/net/ipv6/ipv6_sockglue.c
+++ b/net/ipv6/ipv6_sockglue.c
@@ -398,6 +398,21 @@ int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
 
 	/* Handle options that can be set without locking the socket. */
 	switch (optname) {
+	case IPV6_FREEBIND:
+		if (optlen < sizeof(int))
+			return -EINVAL;
+		/* Shared with IP_FREEBIND. */
+		inet_assign_bit(FREEBIND, sk, valbool);
+		return 0;
+	case IPV6_TRANSPARENT:
+		if (valbool && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW) &&
+		    !sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN))
+			return -EPERM;
+		if (optlen < sizeof(int))
+			return -EINVAL;
+		/* Shared with IP_TRANSPARENT. */
+		inet_assign_bit(TRANSPARENT, sk, valbool);
+		return 0;
 	case IPV6_UNICAST_HOPS:
 		if (optlen < sizeof(int))
 			return -EINVAL;
@@ -547,86 +562,7 @@ int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
 
 	sockopt_lock_sock(sk);
 
-	/* Another thread has converted the socket into IPv4 with
-	 * IPV6_ADDRFORM concurrently.
-	 */
-	if (unlikely(sk->sk_family != AF_INET6))
-		goto unlock;
-
 	switch (optname) {
-
-	case IPV6_ADDRFORM:
-		if (optlen < sizeof(int))
-			goto e_inval;
-		if (val == PF_INET) {
-			if (sk->sk_type == SOCK_RAW)
-				break;
-
-			if (sk->sk_protocol == IPPROTO_UDP) {
-				if (udp_sk(sk)->pending == AF_INET6) {
-					retv = -EBUSY;
-					break;
-				}
-			} else if (sk->sk_protocol == IPPROTO_TCP) {
-				if (sk->sk_prot != &tcpv6_prot) {
-					retv = -EBUSY;
-					break;
-				}
-			} else {
-				break;
-			}
-
-			if (sk->sk_state != TCP_ESTABLISHED) {
-				retv = -ENOTCONN;
-				break;
-			}
-
-			if (ipv6_only_sock(sk) ||
-			    !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) {
-				retv = -EADDRNOTAVAIL;
-				break;
-			}
-
-			__ipv6_sock_mc_close(sk);
-			__ipv6_sock_ac_close(sk);
-
-			if (sk->sk_protocol == IPPROTO_TCP) {
-				struct inet_connection_sock *icsk = inet_csk(sk);
-
-				sock_prot_inuse_add(net, sk->sk_prot, -1);
-				sock_prot_inuse_add(net, &tcp_prot, 1);
-
-				/* Paired with READ_ONCE(sk->sk_prot) in inet6_stream_ops */
-				WRITE_ONCE(sk->sk_prot, &tcp_prot);
-				/* Paired with READ_ONCE() in tcp_(get|set)sockopt() */
-				WRITE_ONCE(icsk->icsk_af_ops, &ipv4_specific);
-				WRITE_ONCE(sk->sk_socket->ops, &inet_stream_ops);
-				WRITE_ONCE(sk->sk_family, PF_INET);
-				tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
-			} else {
-				sock_prot_inuse_add(net, sk->sk_prot, -1);
-				sock_prot_inuse_add(net, &udp_prot, 1);
-
-				/* Paired with READ_ONCE(sk->sk_prot) in inet6_dgram_ops */
-				WRITE_ONCE(sk->sk_prot, &udp_prot);
-				WRITE_ONCE(sk->sk_socket->ops, &inet_dgram_ops);
-				WRITE_ONCE(sk->sk_family, PF_INET);
-			}
-
-			/* Disable all options not to allocate memory anymore,
-			 * but there is still a race.  See the lockless path
-			 * in udpv6_sendmsg() and ipv6_local_rxpmtu().
-			 */
-			np->rxopt.all = 0;
-
-			inet6_cleanup_sock(sk);
-
-			module_put(THIS_MODULE);
-			retv = 0;
-			break;
-		}
-		goto e_inval;
-
 	case IPV6_V6ONLY:
 		if (optlen < sizeof(int) ||
 		    inet_sk(sk)->inet_num)
@@ -745,27 +681,6 @@ int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
 		retv = 0;
 		break;
 
-	case IPV6_TRANSPARENT:
-		if (valbool && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW) &&
-		    !sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN)) {
-			retv = -EPERM;
-			break;
-		}
-		if (optlen < sizeof(int))
-			goto e_inval;
-		/* we don't have a separate transparent bit for IPV6 we use the one in the IPv4 socket */
-		inet_assign_bit(TRANSPARENT, sk, valbool);
-		retv = 0;
-		break;
-
-	case IPV6_FREEBIND:
-		if (optlen < sizeof(int))
-			goto e_inval;
-		/* we also don't have a separate freebind bit for IPV6 */
-		inet_assign_bit(FREEBIND, sk, valbool);
-		retv = 0;
-		break;
-
 	case IPV6_RECVORIGDSTADDR:
 		if (optlen < sizeof(int))
 			goto e_inval;
@@ -1088,14 +1003,6 @@ int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	switch (optname) {
-	case IPV6_ADDRFORM:
-		if (sk->sk_protocol != IPPROTO_UDP &&
-		    sk->sk_protocol != IPPROTO_TCP)
-			return -ENOPROTOOPT;
-		if (sk->sk_state != TCP_ESTABLISHED)
-			return -ENOTCONN;
-		val = sk->sk_family;
-		break;
 	case MCAST_MSFILTER:
 		if (in_compat_syscall())
 			return compat_ipv6_get_msfilter(sk, optval, optlen, len);
diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c
index 75515fd99383b6..12d85d7f82342d 100644
--- a/net/ipv6/ndisc.c
+++ b/net/ipv6/ndisc.c
@@ -767,10 +767,11 @@ static void ndisc_solicit(struct neighbour *neigh, struct sk_buff *skb)
 static int pndisc_is_router(const void *pkey,
 			    struct net_device *dev)
 {
+	struct net *net = dev_net(dev);
 	struct pneigh_entry *n;
 	int ret = -1;
 
-	n = pneigh_lookup(&nd_tbl, dev_net(dev), pkey, dev);
+	n = pneigh_lookup(nd_table(net), pkey, dev);
 	if (n)
 		ret = !!(READ_ONCE(n->flags) & NTF_ROUTER);
 
@@ -788,19 +789,21 @@ void ndisc_update(const struct net_device *dev, struct neighbour *neigh,
 
 static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb)
 {
+	u32 ndoptlen = skb_tail_pointer(skb) - (skb_transport_header(skb) +
+				    offsetof(struct nd_msg, opt));
 	struct nd_msg *msg = (struct nd_msg *)skb_transport_header(skb);
 	const struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
 	const struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
-	u8 *lladdr = NULL;
-	u32 ndoptlen = skb_tail_pointer(skb) - (skb_transport_header(skb) +
-				    offsetof(struct nd_msg, opt));
-	struct ndisc_options ndopts;
 	struct net_device *dev = skb->dev;
-	struct inet6_ifaddr *ifp;
+	struct net *net = dev_net(dev);
+	int dad = ipv6_addr_any(saddr);
 	struct inet6_dev *idev = NULL;
+	struct ndisc_options ndopts;
+	struct inet6_ifaddr *ifp;
+	struct neigh_table *tbl;
 	struct neighbour *neigh;
-	int dad = ipv6_addr_any(saddr);
 	int is_router = -1;
+	u8 *lladdr = NULL;
 	SKB_DR(reason);
 	u64 nonce = 0;
 	bool inc;
@@ -845,9 +848,10 @@ static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb)
 	if (ndopts.nd_opts_nonce && ndopts.nd_opts_nonce->nd_opt_len == 1)
 		memcpy(&nonce, (u8 *)(ndopts.nd_opts_nonce + 1), 6);
 
-	inc = ipv6_addr_is_multicast(daddr);
+	tbl = nd_table(net);
 
-	ifp = ipv6_get_ifaddr(dev_net(dev), &msg->target, dev, 1);
+	inc = ipv6_addr_is_multicast(daddr);
+	ifp = ipv6_get_ifaddr(net, &msg->target, dev, 1);
 	if (ifp) {
 have_ifp:
 		if (ifp->flags & (IFA_F_TENTATIVE|IFA_F_OPTIMISTIC)) {
@@ -880,8 +884,6 @@ static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb)
 
 		idev = ifp->idev;
 	} else {
-		struct net *net = dev_net(dev);
-
 		/* perhaps an address on the master device */
 		if (netif_is_l3_slave(dev)) {
 			struct net_device *mdev;
@@ -918,7 +920,7 @@ static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb)
 				 */
 				struct sk_buff *n = skb_clone(skb, GFP_ATOMIC);
 				if (n)
-					pneigh_enqueue(&nd_tbl, idev->nd_parms, n);
+					pneigh_enqueue(tbl, idev->nd_parms, n);
 				goto out;
 			}
 		} else {
@@ -937,15 +939,15 @@ static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb)
 	}
 
 	if (inc)
-		NEIGH_CACHE_STAT_INC(&nd_tbl, rcv_probes_mcast);
+		NEIGH_CACHE_STAT_INC(tbl, rcv_probes_mcast);
 	else
-		NEIGH_CACHE_STAT_INC(&nd_tbl, rcv_probes_ucast);
+		NEIGH_CACHE_STAT_INC(tbl, rcv_probes_ucast);
 
 	/*
 	 *	update / create cache entry
 	 *	for the source address
 	 */
-	neigh = __neigh_lookup(&nd_tbl, saddr, dev,
+	neigh = __neigh_lookup(tbl, saddr, dev,
 			       !inc || lladdr || !dev->addr_len);
 	if (neigh)
 		ndisc_update(dev, neigh, lladdr, NUD_STALE,
@@ -971,13 +973,13 @@ static enum skb_drop_reason ndisc_recv_ns(struct sk_buff *skb)
 static int accept_untracked_na(struct inet6_dev *idev, struct in6_addr *saddr)
 {
 	switch (READ_ONCE(idev->cnf.accept_untracked_na)) {
-	case 0: /* Don't accept untracked na (absent in neighbor cache) */
+	case 0: /* Don't accept untracked NA (absent or FAILED) */
 		return 0;
-	case 1: /* Create new entries from na if currently untracked */
+	case 1: /* Create new or update FAILED entries from NA */
 		return 1;
-	case 2: /* Create new entries from untracked na only if saddr is in the
+	case 2: /* Create new or update FAILED entries only if saddr is in the
 		 * same subnet as an address configured on the interface that
-		 * received the na
+		 * received the NA
 		 */
 		return !!ipv6_chk_prefix(saddr, idev->dev);
 	default:
@@ -987,17 +989,19 @@ static int accept_untracked_na(struct inet6_dev *idev, struct in6_addr *saddr)
 
 static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb)
 {
-	struct nd_msg *msg = (struct nd_msg *)skb_transport_header(skb);
-	struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
-	const struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
-	u8 *lladdr = NULL;
 	u32 ndoptlen = skb_tail_pointer(skb) - (skb_transport_header(skb) +
 				    offsetof(struct nd_msg, opt));
-	struct ndisc_options ndopts;
+	struct nd_msg *msg = (struct nd_msg *)skb_transport_header(skb);
+	const struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
+	struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
 	struct net_device *dev = skb->dev;
-	struct inet6_dev *idev = __in6_dev_get(dev);
+	struct net *net = dev_net(dev);
+	struct ndisc_options ndopts;
 	struct inet6_ifaddr *ifp;
+	struct neigh_table *tbl;
 	struct neighbour *neigh;
+	struct inet6_dev *idev;
+	u8 *lladdr = NULL;
 	SKB_DR(reason);
 	u8 new_state;
 
@@ -1020,6 +1024,7 @@ static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb)
 	 * and thus should not be accepted.
 	 * drop_unsolicited_na takes precedence over accept_untracked_na
 	 */
+	idev = __in6_dev_get(dev);
 	if (!msg->icmph.icmp6_solicited && idev &&
 	    READ_ONCE(idev->cnf.drop_unsolicited_na))
 		return reason;
@@ -1034,7 +1039,7 @@ static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb)
 			return reason;
 		}
 	}
-	ifp = ipv6_get_ifaddr(dev_net(dev), &msg->target, dev, 1);
+	ifp = ipv6_get_ifaddr(net, &msg->target, dev, 1);
 	if (ifp) {
 		if (skb->pkt_type != PACKET_LOOPBACK
 		    && (ifp->flags & IFA_F_TENTATIVE)) {
@@ -1058,37 +1063,42 @@ static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb)
 		return reason;
 	}
 
-	neigh = neigh_lookup(&nd_tbl, &msg->target, dev);
+	tbl = nd_table(net);
+	neigh = neigh_lookup(tbl, &msg->target, dev);
 
 	/* RFC 9131 updates original Neighbour Discovery RFC 4861.
-	 * NAs with Target LL Address option without a corresponding
-	 * entry in the neighbour cache can now create a STALE neighbour
-	 * cache entry on routers.
+	 * NAs with Target LL Address option can now create a STALE neighbor
+	 * cache entry on routers if the NA does not have a corresponding entry
+	 * in the neighbour cache or has a corresponding FAILED entry.
 	 *
-	 *   entry accept  fwding  solicited        behaviour
-	 * ------- ------  ------  ---------    ----------------------
-	 * present      X       X         0     Set state to STALE
-	 * present      X       X         1     Set state to REACHABLE
-	 *  absent      0       X         X     Do nothing
-	 *  absent      1       0         X     Do nothing
-	 *  absent      1       1         X     Add a new STALE entry
+	 *       entry accept  fwding  solicited        behaviour
+	 * ----------- ------  ------  ---------    ----------------------
+	 *  non-FAILED      X       X         0     Set state to STALE
+	 *  non-FAILED      X       X         1     Set state to REACHABLE
+	 *      FAILED      0       X         X     Do nothing
+	 *      FAILED      1       0         X     Do nothing
+	 *      FAILED      1       1         X     Set state to STALE
+	 *      absent      0       X         X     Do nothing
+	 *      absent      1       0         X     Do nothing
+	 *      absent      1       1         X     Add a new STALE entry
 	 *
 	 * Note that we don't do a (daddr == all-routers-mcast) check.
 	 */
 	new_state = msg->icmph.icmp6_solicited ? NUD_REACHABLE : NUD_STALE;
-	if (!neigh && lladdr && idev && READ_ONCE(idev->cnf.forwarding)) {
-		if (accept_untracked_na(idev, saddr)) {
-			neigh = neigh_create(&nd_tbl, &msg->target, dev);
-			new_state = NUD_STALE;
+	if (!neigh || (READ_ONCE(neigh->nud_state) & NUD_FAILED)) {
+		if (!lladdr || !idev || !READ_ONCE(idev->cnf.forwarding) ||
+		    !accept_untracked_na(idev, saddr)) {
+			if (neigh)
+				neigh_release(neigh);
+			return reason;
 		}
+		if (!neigh)
+			neigh = neigh_create(tbl, &msg->target, dev);
+		new_state = NUD_STALE;
 	}
 
 	if (neigh && !IS_ERR(neigh)) {
 		u8 old_flags = neigh->flags;
-		struct net *net = dev_net(dev);
-
-		if (READ_ONCE(neigh->nud_state) & NUD_FAILED)
-			goto out;
 
 		/*
 		 * Don't update the neighbor cache entry on a proxy NA from
@@ -1098,7 +1108,7 @@ static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb)
 		if (lladdr && !memcmp(lladdr, dev->dev_addr, dev->addr_len) &&
 		    READ_ONCE(net->ipv6.devconf_all->forwarding) &&
 		    READ_ONCE(net->ipv6.devconf_all->proxy_ndp) &&
-		    pneigh_lookup(&nd_tbl, net, &msg->target, dev)) {
+		    pneigh_lookup(tbl, &msg->target, dev)) {
 			/* XXX: idev->cnf.proxy_ndp */
 			goto out;
 		}
@@ -1127,18 +1137,20 @@ static enum skb_drop_reason ndisc_recv_na(struct sk_buff *skb)
 static enum skb_drop_reason ndisc_recv_rs(struct sk_buff *skb)
 {
 	struct rs_msg *rs_msg = (struct rs_msg *)skb_transport_header(skb);
+	const struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
 	unsigned long ndoptlen = skb->len - sizeof(*rs_msg);
+	struct net_device *dev = skb->dev;
+	struct ndisc_options ndopts;
+	struct neigh_table *tbl;
 	struct neighbour *neigh;
 	struct inet6_dev *idev;
-	const struct in6_addr *saddr = &ipv6_hdr(skb)->saddr;
-	struct ndisc_options ndopts;
 	u8 *lladdr = NULL;
 	SKB_DR(reason);
 
 	if (skb->len < sizeof(*rs_msg))
 		return SKB_DROP_REASON_PKT_TOO_SMALL;
 
-	idev = __in6_dev_get(skb->dev);
+	idev = __in6_dev_get(dev);
 	if (!idev) {
 		net_err_ratelimited("RS: can't find in6 device\n");
 		return reason;
@@ -1156,19 +1168,19 @@ static enum skb_drop_reason ndisc_recv_rs(struct sk_buff *skb)
 		goto out;
 
 	/* Parse ND options */
-	if (!ndisc_parse_options(skb->dev, rs_msg->opt, ndoptlen, &ndopts))
+	if (!ndisc_parse_options(dev, rs_msg->opt, ndoptlen, &ndopts))
 		return SKB_DROP_REASON_IPV6_NDISC_BAD_OPTIONS;
 
 	if (ndopts.nd_opts_src_lladdr) {
-		lladdr = ndisc_opt_addr_data(ndopts.nd_opts_src_lladdr,
-					     skb->dev);
+		lladdr = ndisc_opt_addr_data(ndopts.nd_opts_src_lladdr, dev);
 		if (!lladdr)
 			goto out;
 	}
 
-	neigh = __neigh_lookup(&nd_tbl, saddr, skb->dev, 1);
+	tbl = nd_table(dev_net(dev));
+	neigh = __neigh_lookup(tbl, saddr, dev, 1);
 	if (neigh) {
-		ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
+		ndisc_update(dev, neigh, lladdr, NUD_STALE,
 			     NEIGH_UPDATE_F_WEAK_OVERRIDE|
 			     NEIGH_UPDATE_F_OVERRIDE|
 			     NEIGH_UPDATE_F_OVERRIDE_ISROUTER,
@@ -1232,6 +1244,7 @@ static void ndisc_ra_useropt(struct sk_buff *ra, struct nd_opt_hdr *opt)
 static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb)
 {
 	struct ra_msg *ra_msg = (struct ra_msg *)skb_transport_header(skb);
+	struct net *net = dev_net(skb->dev);
 	bool send_ifinfo_notify = false;
 	struct neighbour *neigh = NULL;
 	struct ndisc_options ndopts;
@@ -1241,7 +1254,6 @@ static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb)
 	u32 defrtr_usr_metric;
 	unsigned int pref = 0;
 	__u32 old_if_flags;
-	struct net *net;
 	SKB_DR(reason);
 	int lifetime;
 	int optlen;
@@ -1330,7 +1342,6 @@ static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb)
 	/* Do not accept RA with source-addr found on local machine unless
 	 * accept_ra_from_local is set to true.
 	 */
-	net = dev_net(in6_dev->dev);
 	if (!READ_ONCE(in6_dev->cnf.accept_ra_from_local) &&
 	    ipv6_chk_addr(net, &ipv6_hdr(skb)->saddr, in6_dev->dev, 0)) {
 		net_dbg_ratelimited("RA from local address detected on dev: %s: default router ignored\n",
@@ -1468,7 +1479,7 @@ static enum skb_drop_reason ndisc_router_discovery(struct sk_buff *skb)
 	 */
 
 	if (!neigh)
-		neigh = __neigh_lookup(&nd_tbl, &ipv6_hdr(skb)->saddr,
+		neigh = __neigh_lookup(nd_table(net), &ipv6_hdr(skb)->saddr,
 				       skb->dev, 1);
 	if (neigh) {
 		u8 *lladdr = NULL;
@@ -1861,12 +1872,15 @@ static int ndisc_netdev_event(struct notifier_block *this, unsigned long event,
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 	struct netdev_notifier_change_info *change_info;
 	struct net *net = dev_net(dev);
+	struct neigh_table *tbl;
 	struct inet6_dev *idev;
 	bool evict_nocarrier;
 
+	tbl = nd_table(net);
+
 	switch (event) {
 	case NETDEV_CHANGEADDR:
-		neigh_changeaddr(&nd_tbl, dev);
+		neigh_changeaddr(tbl, dev);
 		fib6_run_gc(0, net, false);
 		fallthrough;
 	case NETDEV_UP:
@@ -1890,12 +1904,12 @@ static int ndisc_netdev_event(struct notifier_block *this, unsigned long event,
 
 		change_info = ptr;
 		if (change_info->flags_changed & IFF_NOARP)
-			neigh_changeaddr(&nd_tbl, dev);
+			neigh_changeaddr(tbl, dev);
 		if (evict_nocarrier && !netif_carrier_ok(dev))
-			neigh_carrier_down(&nd_tbl, dev);
+			neigh_carrier_down(tbl, dev);
 		break;
 	case NETDEV_DOWN:
-		neigh_ifdown(&nd_tbl, dev);
+		neigh_ifdown(tbl, dev);
 		fib6_run_gc(0, net, false);
 		break;
 	case NETDEV_NOTIFY_PEERS:
@@ -1974,12 +1988,23 @@ static int __net_init ndisc_net_init(struct net *net)
 	struct sock *sk;
 	int err;
 
+	err = neigh_table_register(net, &nd_tbl, NEIGH_ND_TABLE);
+	if (err)
+		goto err;
+
+#ifdef CONFIG_SYSCTL
+	err = neigh_sysctl_register(NULL, &nd_table(net)->parms,
+				    ndisc_ifinfo_sysctl_change);
+	if (err)
+		goto err_sysctl;
+#endif
+
 	err = inet_ctl_sock_create(&sk, PF_INET6,
 				   SOCK_RAW, IPPROTO_ICMPV6, net);
 	if (err < 0) {
 		net_err_ratelimited("NDISC: Failed to initialize the control socket (err %d)\n",
 				    err);
-		return err;
+		goto err_sock_create;
 	}
 
 	net->ipv6.ndisc_sk = sk;
@@ -1990,11 +2015,24 @@ static int __net_init ndisc_net_init(struct net *net)
 	inet6_clear_bit(MC6_LOOP, sk);
 
 	return 0;
+
+err_sock_create:
+#ifdef CONFIG_SYSCTL
+	neigh_sysctl_unregister(&nd_table(net)->parms);
+err_sysctl:
+#endif
+	neigh_table_unregister(net, NEIGH_ND_TABLE);
+err:
+	return err;
 }
 
 static void __net_exit ndisc_net_exit(struct net *net)
 {
 	inet_ctl_sock_destroy(net->ipv6.ndisc_sk);
+#ifdef CONFIG_SYSCTL
+	neigh_sysctl_unregister(&nd_table(net)->parms);
+#endif
+	neigh_table_unregister(net, NEIGH_ND_TABLE);
 }
 
 static struct pernet_operations ndisc_net_ops = {
@@ -2004,30 +2042,7 @@ static struct pernet_operations ndisc_net_ops = {
 
 int __init ndisc_init(void)
 {
-	int err;
-
-	err = register_pernet_subsys(&ndisc_net_ops);
-	if (err)
-		return err;
-	/*
-	 * Initialize the neighbour table
-	 */
-	neigh_table_init(NEIGH_ND_TABLE, &nd_tbl);
-
-#ifdef CONFIG_SYSCTL
-	err = neigh_sysctl_register(NULL, &nd_tbl.parms,
-				    ndisc_ifinfo_sysctl_change);
-	if (err)
-		goto out_unregister_pernet;
-out:
-#endif
-	return err;
-
-#ifdef CONFIG_SYSCTL
-out_unregister_pernet:
-	unregister_pernet_subsys(&ndisc_net_ops);
-	goto out;
-#endif
+	return register_pernet_subsys(&ndisc_net_ops);
 }
 
 int __init ndisc_late_init(void)
@@ -2042,9 +2057,5 @@ void ndisc_late_cleanup(void)
 
 void ndisc_cleanup(void)
 {
-#ifdef CONFIG_SYSCTL
-	neigh_sysctl_unregister(&nd_tbl.parms);
-#endif
-	neigh_table_clear(NEIGH_ND_TABLE, &nd_tbl);
 	unregister_pernet_subsys(&ndisc_net_ops);
 }
diff --git a/net/ipv6/netfilter/ip6t_rpfilter.c b/net/ipv6/netfilter/ip6t_rpfilter.c
index 67c87a88cde4f8..b5def30c3127e5 100644
--- a/net/ipv6/netfilter/ip6t_rpfilter.c
+++ b/net/ipv6/netfilter/ip6t_rpfilter.c
@@ -61,7 +61,7 @@ static bool rpfilter_lookup_reverse6(struct net *net, const struct sk_buff *skb,
 		fl6.flowi6_oif = dev->ifindex;
 
 	rt = (void *)ip6_route_lookup(net, &fl6, skb, lookup_flags);
-	if (rt->dst.error)
+	if (rt->dst.error || !rt->rt6i_idev)
 		goto out;
 
 	if (rt->rt6i_flags & (RTF_REJECT|RTF_ANYCAST))
diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c
index 8051425213ddae..9880faf3cc7dd7 100644
--- a/net/ipv6/netfilter/ip6t_rt.c
+++ b/net/ipv6/netfilter/ip6t_rt.c
@@ -96,7 +96,8 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par)
 			unsigned int i = 0;
 
 			for (temp = 0;
-			     temp < (unsigned int)((hdrlen - 8) / 16);
+			     temp < (unsigned int)((hdrlen - 8) / 16) &&
+			     i < rtinfo->addrnr;
 			     temp++) {
 				ap = skb_header_pointer(skb,
 							ptr
@@ -112,8 +113,6 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par)
 
 				if (ipv6_addr_equal(ap, &rtinfo->addrs[i]))
 					i++;
-				if (i == rtinfo->addrnr)
-					break;
 			}
 			if (i == rtinfo->addrnr)
 				return ret;
@@ -162,6 +161,12 @@ static int rt_mt6_check(const struct xt_mtchk_param *par)
 		pr_info_ratelimited("too many addresses specified\n");
 		return -EINVAL;
 	}
+
+	if ((rtinfo->flags & IP6T_RT_FST_MASK) && !rtinfo->addrnr) {
+		pr_info_ratelimited("address list match requested but addrnr is 0\n");
+		return -EINVAL;
+	}
+
 	if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) &&
 	    (!(rtinfo->flags & IP6T_RT_TYP) ||
 	     (rtinfo->rt_type != 0) ||
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 08bd68f1b5bb42..a76869ff87cd67 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -139,6 +139,7 @@ void rt6_uncached_list_add(struct rt6_info *rt)
 {
 	struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list);
 
+	/* Set once and never cleared: non-NULL marks an uncached route. */
 	rt->dst.rt_uncached_list = ul;
 
 	spin_lock_bh(&ul->lock);
@@ -217,7 +218,7 @@ struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw,
 	if (n)
 		return n;
 
-	n = neigh_create(&nd_tbl, daddr, dev);
+	n = neigh_create(nd_table(dev_net(dev)), daddr, dev);
 	return IS_ERR(n) ? NULL : n;
 }
 
@@ -1662,13 +1663,6 @@ static unsigned int fib6_mtu(const struct fib6_result *res)
 	return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
 }
 
-#define FIB6_EXCEPTION_BUCKET_FLUSHED  0x1UL
-
-/* used when the flushed bit is not relevant, only access to the bucket
- * (ie., all bucket users except rt6_insert_exception);
- *
- * called under rcu lock; sometimes called with rt6_exception_lock held
- */
 static
 struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,
 						       spinlock_t *lock)
@@ -1681,40 +1675,9 @@ struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,
 	else
 		bucket = rcu_dereference(nh->rt6i_exception_bucket);
 
-	/* remove bucket flushed bit if set */
-	if (bucket) {
-		unsigned long p = (unsigned long)bucket;
-
-		p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED;
-		bucket = (struct rt6_exception_bucket *)p;
-	}
-
 	return bucket;
 }
 
-static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket)
-{
-	unsigned long p = (unsigned long)bucket;
-
-	return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED);
-}
-
-/* called with rt6_exception_lock held */
-static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,
-					      spinlock_t *lock)
-{
-	struct rt6_exception_bucket *bucket;
-	unsigned long p;
-
-	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
-					   lockdep_is_held(lock));
-
-	p = (unsigned long)bucket;
-	p |= FIB6_EXCEPTION_BUCKET_FLUSHED;
-	bucket = (struct rt6_exception_bucket *)p;
-	rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
-}
-
 static int rt6_insert_exception(struct rt6_info *nrt,
 				const struct fib6_result *res)
 {
@@ -1729,6 +1692,11 @@ static int rt6_insert_exception(struct rt6_info *nrt,
 
 	spin_lock_bh(&rt6_exception_lock);
 
+	if (f6i->fib6_destroying) {
+		err = -ENOENT;
+		goto out;
+	}
+
 	bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
 					  lockdep_is_held(&rt6_exception_lock));
 	if (!bucket) {
@@ -1739,9 +1707,6 @@ static int rt6_insert_exception(struct rt6_info *nrt,
 			goto out;
 		}
 		rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
-	} else if (fib6_nh_excptn_bucket_flushed(bucket)) {
-		err = -EINVAL;
-		goto out;
 	}
 
 #ifdef CONFIG_IPV6_SUBTREES
@@ -1812,10 +1777,6 @@ static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)
 	if (!bucket)
 		goto out;
 
-	/* Prevent rt6_insert_exception() to recreate the bucket list */
-	if (!from)
-		fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);
-
 	for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
 		hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) {
 			if (!from ||
@@ -2721,8 +2682,8 @@ struct dst_entry *ip6_route_output_flags(struct net *net,
 	rcu_read_lock();
 	dst = ip6_route_output_flags_noref(net, sk, fl6, flags);
 	rt6 = dst_rt6_info(dst);
-	/* For dst cached in uncached_list, refcnt is already taken. */
-	if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) {
+	/* For an uncached dst, refcnt is already taken. */
+	if (!rt6->dst.rt_uncached_list && !dst_hold_safe(dst)) {
 		dst = &net->ipv6.ip6_null_entry->dst;
 		dst_hold(dst);
 	}
@@ -2831,7 +2792,7 @@ INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst,
 	from = rcu_dereference(rt->from);
 
 	if (from && (rt->rt6i_flags & RTF_PCPU ||
-	    unlikely(!list_empty(&rt->dst.rt_uncached))))
+	    unlikely(rt->dst.rt_uncached_list)))
 		dst_ret = rt6_dst_from_check(rt, from, cookie);
 	else
 		dst_ret = rt6_check(rt, from, cookie);
@@ -4244,6 +4205,7 @@ static int ip6_route_del(struct fib6_config *cfg,
 static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 {
 	struct netevent_redirect netevent;
+	struct net_device *dev = skb->dev;
 	struct rt6_info *rt, *nrt = NULL;
 	struct fib6_result res = {};
 	struct ndisc_options ndopts;
@@ -4277,7 +4239,7 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
 		return;
 	}
 
-	in6_dev = __in6_dev_get(skb->dev);
+	in6_dev = __in6_dev_get(dev);
 	if (!in6_dev)
 		return;
 	if (READ_ONCE(in6_dev->cnf.forwarding) ||
@@ -4289,15 +4251,14 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
 	 *	first-hop router for the specified ICMP Destination Address.
 	 */
 
-	if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) {
+	if (!ndisc_parse_options(dev, msg->opt, optlen, &ndopts)) {
 		net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
 		return;
 	}
 
 	lladdr = NULL;
 	if (ndopts.nd_opts_tgt_lladdr) {
-		lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr,
-					     skb->dev);
+		lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr, dev);
 		if (!lladdr) {
 			net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n");
 			return;
@@ -4316,7 +4277,7 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
 	 */
 	dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr);
 
-	neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1);
+	neigh = __neigh_lookup(nd_table(dev_net(dev)), &msg->target, dev, 1);
 	if (!neigh)
 		return;
 
@@ -4324,7 +4285,7 @@ static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_bu
 	 *	We have finally decided to accept it.
 	 */
 
-	ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
+	ndisc_update(dev, neigh, lladdr, NUD_STALE,
 		     NEIGH_UPDATE_F_WEAK_OVERRIDE|
 		     NEIGH_UPDATE_F_OVERRIDE|
 		     (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
@@ -5045,9 +5006,11 @@ void rt6_sync_down_dev(struct net_device *dev, unsigned long event)
 
 void rt6_disable_ip(struct net_device *dev, unsigned long event)
 {
+	struct net *net = dev_net(dev);
+
 	rt6_sync_down_dev(dev, event);
 	rt6_uncached_list_flush_dev(dev);
-	neigh_ifdown(&nd_tbl, dev);
+	neigh_ifdown(nd_table(net), dev);
 }
 
 struct rt6_mtu_change_arg {
diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c
index 62a7eb77920265..8c2b156c227a73 100644
--- a/net/ipv6/seg6.c
+++ b/net/ipv6/seg6.c
@@ -138,8 +138,8 @@ void seg6_icmp_srh(struct sk_buff *skb, struct inet6_skb_parm *opt)
 static struct genl_family seg6_genl_family;
 
 static const struct nla_policy seg6_genl_policy[SEG6_ATTR_MAX + 1] = {
-	[SEG6_ATTR_DST]				= { .type = NLA_BINARY,
-		.len = sizeof(struct in6_addr) },
+	[SEG6_ATTR_DST]				=
+		NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)),
 	[SEG6_ATTR_DSTLEN]			= { .type = NLA_S32, },
 	[SEG6_ATTR_HMACKEYID]		= { .type = NLA_U32, },
 	[SEG6_ATTR_SECRET]			= { .type = NLA_BINARY, },
diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c
index d1070aec7b72b9..53f7b245257598 100644
--- a/net/ipv6/seg6_local.c
+++ b/net/ipv6/seg6_local.c
@@ -278,13 +278,43 @@ static bool decap_and_validate(struct sk_buff *skb, int proto)
 	return true;
 }
 
-static void advance_nextseg(struct ipv6_sr_hdr *srh, struct in6_addr *daddr)
+/* advance the SRH to the next segment and set the IPv6 DA accordingly.
+ * skb pointers may change: after this call, the caller must evaluate again
+ * any pointer into the packet data.
+ *
+ * This function returns:
+ *  - the SRH on success;
+ *  - NULL when the skb cannot be made writable. In this case, the function
+ *    sets the drop reason to SKB_DROP_REASON_NOMEM.
+ */
+static struct ipv6_sr_hdr *advance_nextseg(struct sk_buff *skb,
+					   struct ipv6_sr_hdr *srh,
+					   enum skb_drop_reason *reason)
 {
 	struct in6_addr *addr;
+	int srhoff;
+	int wlen;
+
+	srhoff = (unsigned char *)srh - skb->data;
+	/* we write only the Segment Left field and the IPv6 DA. The segment
+	 * list is only read, and seg6_get_srh() already pulled it into the
+	 * linear area.
+	 */
+	wlen = srhoff + sizeof(*srh);
+
+	if (unlikely(skb_ensure_writable(skb, wlen))) {
+		*reason = SKB_DROP_REASON_NOMEM;
+		return NULL;
+	}
+
+	/* skb_ensure_writable() may change skb pointers; evaluate srh again */
+	srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
 
 	srh->segments_left--;
 	addr = srh->segments + srh->segments_left;
-	*daddr = *addr;
+	ipv6_hdr(skb)->daddr = *addr;
+
+	return srh;
 }
 
 static int
@@ -382,12 +412,26 @@ static bool seg6_next_csid_is_arg_zero(const struct in6_addr *addr,
 	return true;
 }
 
-/* assume that DA.Argument length > 0 */
-static void seg6_next_csid_advance_arg(struct in6_addr *addr,
-				       const struct seg6_flavors_info *finfo)
+/* assume that DA.Argument length > 0.
+ * skb pointers may change: after this call, the caller must evaluate again
+ * any pointer into the packet data.
+ *
+ * This function returns:
+ *  - SKB_NOT_DROPPED_YET on success;
+ *  - SKB_DROP_REASON_NOMEM when the skb cannot be made writable.
+ */
+static enum skb_drop_reason
+seg6_next_csid_advance_arg(struct sk_buff *skb,
+			   const struct seg6_flavors_info *finfo)
 {
 	__u8 fnc_octects = seg6_flv_lcnode_func_octects(finfo);
 	__u8 blk_octects = seg6_flv_lcblock_octects(finfo);
+	struct in6_addr *addr;
+
+	if (unlikely(skb_ensure_writable(skb, sizeof(struct ipv6hdr))))
+		return SKB_DROP_REASON_NOMEM;
+
+	addr = &ipv6_hdr(skb)->daddr;
 
 	/* advance DA.Argument */
 	memmove(&addr->s6_addr[blk_octects],
@@ -395,6 +439,8 @@ static void seg6_next_csid_advance_arg(struct in6_addr *addr,
 		16 - blk_octects - fnc_octects);
 
 	memset(&addr->s6_addr[16 - fnc_octects], 0x00, fnc_octects);
+
+	return SKB_NOT_DROPPED_YET;
 }
 
 static int input_action_end_finish(struct sk_buff *skb,
@@ -408,31 +454,42 @@ static int input_action_end_finish(struct sk_buff *skb,
 static int input_action_end_core(struct sk_buff *skb,
 				 struct seg6_local_lwt *slwt)
 {
+	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
 	struct ipv6_sr_hdr *srh;
+	int err = -EINVAL;
 
 	srh = get_and_validate_srh(skb);
 	if (!srh)
 		goto drop;
 
-	advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
+	srh = advance_nextseg(skb, srh, &reason);
+	if (!srh) {
+		err = -ENOMEM;
+		goto drop;
+	}
 
 	return input_action_end_finish(skb, slwt);
 
 drop:
-	kfree_skb(skb);
-	return -EINVAL;
+	kfree_skb_reason(skb, reason);
+	return err;
 }
 
 static int end_next_csid_core(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 {
 	const struct seg6_flavors_info *finfo = &slwt->flv_info;
 	struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
+	enum skb_drop_reason reason;
 
 	if (seg6_next_csid_is_arg_zero(daddr, finfo))
 		return input_action_end_core(skb, slwt);
 
 	/* update DA */
-	seg6_next_csid_advance_arg(daddr, finfo);
+	reason = seg6_next_csid_advance_arg(skb, finfo);
+	if (reason) {
+		kfree_skb_reason(skb, reason);
+		return -ENOMEM;
+	}
 
 	return input_action_end_finish(skb, slwt);
 }
@@ -448,19 +505,25 @@ static int input_action_end_x_finish(struct sk_buff *skb,
 static int input_action_end_x_core(struct sk_buff *skb,
 				   struct seg6_local_lwt *slwt)
 {
+	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
 	struct ipv6_sr_hdr *srh;
+	int err = -EINVAL;
 
 	srh = get_and_validate_srh(skb);
 	if (!srh)
 		goto drop;
 
-	advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
+	srh = advance_nextseg(skb, srh, &reason);
+	if (!srh) {
+		err = -ENOMEM;
+		goto drop;
+	}
 
 	return input_action_end_x_finish(skb, slwt);
 
 drop:
-	kfree_skb(skb);
-	return -EINVAL;
+	kfree_skb_reason(skb, reason);
+	return err;
 }
 
 static int end_x_next_csid_core(struct sk_buff *skb,
@@ -468,12 +531,17 @@ static int end_x_next_csid_core(struct sk_buff *skb,
 {
 	const struct seg6_flavors_info *finfo = &slwt->flv_info;
 	struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
+	enum skb_drop_reason reason;
 
 	if (seg6_next_csid_is_arg_zero(daddr, finfo))
 		return input_action_end_x_core(skb, slwt);
 
 	/* update DA */
-	seg6_next_csid_advance_arg(daddr, finfo);
+	reason = seg6_next_csid_advance_arg(skb, finfo);
+	if (reason) {
+		kfree_skb_reason(skb, reason);
+		return -ENOMEM;
+	}
 
 	return input_action_end_x_finish(skb, slwt);
 }
@@ -760,10 +828,12 @@ static bool seg6_pop_srh(struct sk_buff *skb, int srhoff)
  */
 static int end_flv8986_core(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 {
+	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
 	const struct seg6_flavors_info *finfo = &slwt->flv_info;
 	enum seg6_local_flv_action action;
 	enum seg6_local_pktinfo pinfo;
 	struct ipv6_sr_hdr *srh;
+	int err = -EINVAL;
 	__u32 flvmask;
 	int srhoff;
 
@@ -787,10 +857,18 @@ static int end_flv8986_core(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 	switch (action) {
 	case SEG6_LOCAL_FLV_ACT_END:
 		/* process the packet as the "standard" End behavior */
-		advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
+		srh = advance_nextseg(skb, srh, &reason);
+		if (!srh) {
+			err = -ENOMEM;
+			goto drop;
+		}
 		break;
 	case SEG6_LOCAL_FLV_ACT_PSP:
-		advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
+		srh = advance_nextseg(skb, srh, &reason);
+		if (!srh) {
+			err = -ENOMEM;
+			goto drop;
+		}
 
 		if (unlikely(!seg6_pop_srh(skb, srhoff)))
 			goto drop;
@@ -807,8 +885,8 @@ static int end_flv8986_core(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 	return input_action_end_finish(skb, slwt);
 
 drop:
-	kfree_skb(skb);
-	return -EINVAL;
+	kfree_skb_reason(skb, reason);
+	return err;
 }
 
 /* regular endpoint function */
@@ -847,21 +925,27 @@ static int input_action_end_x(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 
 static int input_action_end_t(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 {
+	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
 	struct ipv6_sr_hdr *srh;
+	int err = -EINVAL;
 
 	srh = get_and_validate_srh(skb);
 	if (!srh)
 		goto drop;
 
-	advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
+	srh = advance_nextseg(skb, srh, &reason);
+	if (!srh) {
+		err = -ENOMEM;
+		goto drop;
+	}
 
 	seg6_lookup_nexthop(skb, NULL, slwt->table);
 
 	return dst_input(skb);
 
 drop:
-	kfree_skb(skb);
-	return -EINVAL;
+	kfree_skb_reason(skb, reason);
+	return err;
 }
 
 /* decapsulate and forward inner L2 frame on specified interface */
@@ -1375,6 +1459,7 @@ static int input_action_end_b6(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 static int input_action_end_b6_encap(struct sk_buff *skb,
 				     struct seg6_local_lwt *slwt)
 {
+	enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
 	struct ipv6_sr_hdr *srh;
 	int err = -EINVAL;
 
@@ -1382,7 +1467,11 @@ static int input_action_end_b6_encap(struct sk_buff *skb,
 	if (!srh)
 		goto drop;
 
-	advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
+	srh = advance_nextseg(skb, srh, &reason);
+	if (!srh) {
+		err = -ENOMEM;
+		goto drop;
+	}
 
 	skb_reset_inner_headers(skb);
 	skb->encapsulation = 1;
@@ -1398,7 +1487,7 @@ static int input_action_end_b6_encap(struct sk_buff *skb,
 	return dst_input(skb);
 
 drop:
-	kfree_skb(skb);
+	kfree_skb_reason(skb, reason);
 	return err;
 }
 
@@ -1434,6 +1523,7 @@ static int input_action_end_bpf(struct sk_buff *skb,
 				struct seg6_local_lwt *slwt)
 {
 	struct seg6_bpf_srh_state *srh_state;
+	enum skb_drop_reason reason;
 	struct ipv6_sr_hdr *srh;
 	int ret;
 
@@ -1442,7 +1532,11 @@ static int input_action_end_bpf(struct sk_buff *skb,
 		kfree_skb(skb);
 		return -EINVAL;
 	}
-	advance_nextseg(srh, &ipv6_hdr(skb)->daddr);
+	srh = advance_nextseg(skb, srh, &reason);
+	if (!srh) {
+		kfree_skb_reason(skb, reason);
+		return -ENOMEM;
+	}
 
 	/* The access to the per-CPU buffer srh_state is protected by running
 	 * always in softirq context (with disabled BH). On PREEMPT_RT the
diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
index 19b7fa8d1a2a07..a7ec800216ade1 100644
--- a/net/ipv6/sit.c
+++ b/net/ipv6/sit.c
@@ -108,24 +108,33 @@ static struct ip_tunnel *ipip6_tunnel_lookup(struct net *net,
 	int ifindex = dev ? dev->ifindex : 0;
 
 	for_each_ip_tunnel_rcu(t, sitn->tunnels_r_l[h0 ^ h1]) {
-		if (local == t->parms.iph.saddr &&
-		    remote == t->parms.iph.daddr &&
-		    (!dev || !t->parms.link || ifindex == t->parms.link ||
-		     sifindex == t->parms.link) &&
+		const struct ip_tunnel_parm_kern *parms;
+
+		parms = rcu_dereference(t->sit_parms);
+		if (local == parms->iph.saddr &&
+		    remote == parms->iph.daddr &&
+		    (!dev || !parms->link || ifindex == parms->link ||
+		     sifindex == parms->link) &&
 		    (t->dev->flags & IFF_UP))
 			return t;
 	}
 	for_each_ip_tunnel_rcu(t, sitn->tunnels_r[h0]) {
-		if (remote == t->parms.iph.daddr &&
-		    (!dev || !t->parms.link || ifindex == t->parms.link ||
-		     sifindex == t->parms.link) &&
+		const struct ip_tunnel_parm_kern *parms;
+
+		parms = rcu_dereference(t->sit_parms);
+		if (remote == parms->iph.daddr &&
+		    (!dev || !parms->link || ifindex == parms->link ||
+		     sifindex == parms->link) &&
 		    (t->dev->flags & IFF_UP))
 			return t;
 	}
 	for_each_ip_tunnel_rcu(t, sitn->tunnels_l[h1]) {
-		if (local == t->parms.iph.saddr &&
-		    (!dev || !t->parms.link || ifindex == t->parms.link ||
-		     sifindex == t->parms.link) &&
+		const struct ip_tunnel_parm_kern *parms;
+
+		parms = rcu_dereference(t->sit_parms);
+		if (local == parms->iph.saddr &&
+		    (!dev || !parms->link || ifindex == parms->link ||
+		     sifindex == parms->link) &&
 		    (t->dev->flags & IFF_UP))
 			return t;
 	}
@@ -157,7 +166,7 @@ __ipip6_bucket(struct sit_net *sitn, struct ip_tunnel_parm_kern *parms)
 static inline struct ip_tunnel __rcu **ipip6_bucket(struct sit_net *sitn,
 		struct ip_tunnel *t)
 {
-	return __ipip6_bucket(sitn, &t->parms);
+	return __ipip6_bucket(sitn, rtnl_dereference(t->sit_parms));
 }
 
 static void ipip6_tunnel_unlink(struct sit_net *sitn, struct ip_tunnel *t)
@@ -183,33 +192,66 @@ static void ipip6_tunnel_link(struct sit_net *sitn, struct ip_tunnel *t)
 	rcu_assign_pointer(*tp, t);
 }
 
-static void ipip6_tunnel_clone_6rd(struct net_device *dev, struct sit_net *sitn)
+static int ipip6_tunnel_clone_6rd(struct net_device *dev, struct sit_net *sitn)
 {
 #ifdef CONFIG_IPV6_SIT_6RD
 	struct ip_tunnel *t = netdev_priv(dev);
+	struct ip_tunnel_6rd_parm *new_6rd, *old_6rd;
+
+	new_6rd = kmalloc_obj(*new_6rd);
+	if (!new_6rd)
+		return -ENOMEM;
 
 	if (dev == sitn->fb_tunnel_dev || !sitn->fb_tunnel_dev) {
-		ipv6_addr_set(&t->ip6rd.prefix, htonl(0x20020000), 0, 0, 0);
-		t->ip6rd.relay_prefix = 0;
-		t->ip6rd.prefixlen = 16;
-		t->ip6rd.relay_prefixlen = 0;
+		ipv6_addr_set(&new_6rd->prefix, htonl(0x20020000), 0, 0, 0);
+		new_6rd->relay_prefix = 0;
+		new_6rd->prefixlen = 16;
+		new_6rd->relay_prefixlen = 0;
 	} else {
 		struct ip_tunnel *t0 = netdev_priv(sitn->fb_tunnel_dev);
-		memcpy(&t->ip6rd, &t0->ip6rd, sizeof(t->ip6rd));
+		struct ip_tunnel_6rd_parm *t0_6rd;
+
+		t0_6rd = rtnl_dereference(t0->ip6rd);
+		if (t0_6rd) {
+			*new_6rd = *t0_6rd;
+		} else {
+			ipv6_addr_set(&new_6rd->prefix, htonl(0x20020000), 0, 0, 0);
+			new_6rd->relay_prefix = 0;
+			new_6rd->prefixlen = 16;
+			new_6rd->relay_prefixlen = 0;
+		}
+	}
+
+	old_6rd = rcu_dereference_protected(t->ip6rd,
+					    lockdep_rtnl_is_held() ||
+					    dev->reg_state == NETREG_UNINITIALIZED);
+	rcu_assign_pointer(t->ip6rd, new_6rd);
+	if (old_6rd) {
+		dst_cache_reset(&t->dst_cache);
+		netdev_state_change(t->dev);
+		kfree_rcu(old_6rd, rcu);
 	}
 #endif
+	return 0;
 }
 
 static int ipip6_tunnel_create(struct net_device *dev)
 {
 	struct ip_tunnel *t = netdev_priv(dev);
 	struct sit_net *sitn = net_generic(t->net, sit_net_id);
+	struct ip_tunnel_parm_kern *parms;
 	int err;
 
-	__dev_addr_set(dev, &t->parms.iph.saddr, 4);
-	memcpy(dev->broadcast, &t->parms.iph.daddr, 4);
+	err = ipip6_tunnel_clone_6rd(dev, sitn);
+	if (err < 0)
+		goto out;
+
+	parms = rtnl_dereference(t->sit_parms);
+	t->parms = *parms;
+	__dev_addr_set(dev, &parms->iph.saddr, 4);
+	memcpy(dev->broadcast, &parms->iph.daddr, 4);
 
-	if (test_bit(IP_TUNNEL_SIT_ISATAP_BIT, t->parms.i_flags))
+	if (test_bit(IP_TUNNEL_SIT_ISATAP_BIT, parms->i_flags))
 		dev->priv_flags |= IFF_ISATAP;
 
 	dev->rtnl_link_ops = &sit_link_ops;
@@ -218,8 +260,6 @@ static int ipip6_tunnel_create(struct net_device *dev)
 	if (err < 0)
 		goto out;
 
-	ipip6_tunnel_clone_6rd(dev, sitn);
-
 	ipip6_tunnel_link(sitn, t);
 	return 0;
 
@@ -235,6 +275,7 @@ static struct ip_tunnel *ipip6_tunnel_locate(struct net *net,
 	__be32 local = parms->iph.saddr;
 	struct ip_tunnel *t, *nt;
 	struct ip_tunnel __rcu **tp;
+	struct ip_tunnel_parm_kern *nt_parms;
 	struct net_device *dev;
 	char name[IFNAMSIZ];
 	struct sit_net *sitn = net_generic(net, sit_net_id);
@@ -242,9 +283,12 @@ static struct ip_tunnel *ipip6_tunnel_locate(struct net *net,
 	for (tp = __ipip6_bucket(sitn, parms);
 	    (t = rtnl_dereference(*tp)) != NULL;
 	     tp = &t->next) {
-		if (local == t->parms.iph.saddr &&
-		    remote == t->parms.iph.daddr &&
-		    parms->link == t->parms.link) {
+		const struct ip_tunnel_parm_kern *tparms;
+
+		tparms = rtnl_dereference(t->sit_parms);
+		if (local == tparms->iph.saddr &&
+		    remote == tparms->iph.daddr &&
+		    parms->link == tparms->link) {
 			if (create)
 				return NULL;
 			else
@@ -271,7 +315,11 @@ static struct ip_tunnel *ipip6_tunnel_locate(struct net *net,
 	nt = netdev_priv(dev);
 
 	nt->net = net;
-	nt->parms = *parms;
+	nt_parms = kmalloc_obj(*nt_parms);
+	if (!nt_parms)
+		goto failed_free;
+	*nt_parms = *parms;
+	rcu_assign_pointer(nt->sit_parms, nt_parms);
 	if (ipip6_tunnel_create(dev) < 0)
 		goto failed_free;
 
@@ -281,6 +329,7 @@ static struct ip_tunnel *ipip6_tunnel_locate(struct net *net,
 	return nt;
 
 failed_free:
+	ipip6_dev_free(dev);
 	free_netdev(dev);
 failed:
 	return NULL;
@@ -395,7 +444,7 @@ ipip6_tunnel_add_prl(struct ip_tunnel *t, struct ip_tunnel_prl *a, int chg)
 		goto out;
 	}
 
-	p = kzalloc_obj(struct ip_tunnel_prl_entry);
+	p = kzalloc_obj(struct ip_tunnel_prl_entry, GFP_KERNEL_ACCOUNT);
 	if (!p) {
 		err = -ENOBUFS;
 		goto out;
@@ -436,7 +485,7 @@ ipip6_tunnel_del_prl(struct ip_tunnel *t, struct ip_tunnel_prl *a)
 		     (x = rtnl_dereference(*p)) != NULL;
 		     p = &x->next) {
 			if (x->addr == a->addr) {
-				*p = x->next;
+				rcu_assign_pointer(*p, rtnl_dereference(x->next));
 				kfree_rcu(x, rcu_head);
 				t->prl_count--;
 				goto out;
@@ -447,8 +496,8 @@ ipip6_tunnel_del_prl(struct ip_tunnel *t, struct ip_tunnel_prl *a)
 		x = rtnl_dereference(t->prl);
 		if (x) {
 			t->prl_count = 0;
+			RCU_INIT_POINTER(t->prl, NULL);
 			call_rcu(&x->rcu_head, prl_list_destroy_rcu);
-			t->prl = NULL;
 		}
 	}
 out:
@@ -566,41 +615,45 @@ static int ipip6_err(struct sk_buff *skb, u32 info)
 	err = -ENOENT;
 
 	sifindex = netif_is_l3_master(skb->dev) ? IPCB(skb)->iif : 0;
+	rcu_read_lock();
 	t = ipip6_tunnel_lookup(dev_net(skb->dev), skb->dev,
 				iph->daddr, iph->saddr, sifindex);
-	if (!t)
-		goto out;
+	if (t) {
+		const struct ip_tunnel_parm_kern *parms;
 
-	if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED) {
-		ipv4_update_pmtu(skb, dev_net(skb->dev), info,
-				 t->parms.link, iph->protocol);
-		err = 0;
-		goto out;
-	}
-	if (type == ICMP_REDIRECT) {
-		ipv4_redirect(skb, dev_net(skb->dev), t->parms.link,
-			      iph->protocol);
-		err = 0;
-		goto out;
-	}
+		parms = rcu_dereference(t->sit_parms);
+		if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED) {
+			ipv4_update_pmtu(skb, dev_net(skb->dev), info,
+					 parms->link, iph->protocol);
+			err = 0;
+			goto out;
+		}
+		if (type == ICMP_REDIRECT) {
+			ipv4_redirect(skb, dev_net(skb->dev), parms->link,
+				      iph->protocol);
+			err = 0;
+			goto out;
+		}
 
-	err = 0;
-	if (__in6_dev_get(skb->dev) &&
-	    !ip6_err_gen_icmpv6_unreach(skb, iph->ihl * 4, type, data_len))
-		goto out;
+		err = 0;
+		if (__in6_dev_get(skb->dev) &&
+		    !ip6_err_gen_icmpv6_unreach(skb, iph->ihl * 4, type, data_len))
+			goto out;
 
-	if (t->parms.iph.daddr == 0)
-		goto out;
+		if (parms->iph.daddr == 0)
+			goto out;
 
-	if (t->parms.iph.ttl == 0 && type == ICMP_TIME_EXCEEDED)
-		goto out;
+		if (parms->iph.ttl == 0 && type == ICMP_TIME_EXCEEDED)
+			goto out;
 
-	if (time_before(jiffies, READ_ONCE(t->err_time) + IPTUNNEL_ERR_TIMEO))
-		WRITE_ONCE(t->err_count, READ_ONCE(t->err_count) + 1);
-	else
-		WRITE_ONCE(t->err_count, 1);
-	WRITE_ONCE(t->err_time, jiffies);
+		if (time_before(jiffies, READ_ONCE(t->err_time) + IPTUNNEL_ERR_TIMEO))
+			WRITE_ONCE(t->err_count, READ_ONCE(t->err_count) + 1);
+		else
+			WRITE_ONCE(t->err_count, 1);
+		WRITE_ONCE(t->err_time, jiffies);
+	}
 out:
+	rcu_read_unlock();
 	return err;
 }
 
@@ -631,8 +684,13 @@ static bool only_dnatted(const struct ip_tunnel *tunnel,
 	int prefix_len;
 
 #ifdef CONFIG_IPV6_SIT_6RD
-	prefix_len = tunnel->ip6rd.prefixlen + 32
-		- tunnel->ip6rd.relay_prefixlen;
+	const struct ip_tunnel_6rd_parm *ip6rd;
+
+	ip6rd = rcu_dereference(tunnel->ip6rd);
+	if (ip6rd)
+		prefix_len = ip6rd->prefixlen + 32 - ip6rd->relay_prefixlen;
+	else
+		prefix_len = 48;
 #else
 	prefix_len = 48;
 #endif
@@ -688,8 +746,11 @@ static int ipip6_rcv(struct sk_buff *skb)
 	tunnel = ipip6_tunnel_lookup(dev_net(skb->dev), skb->dev,
 				     iph->saddr, iph->daddr, sifindex);
 	if (tunnel) {
-		if (tunnel->parms.iph.protocol != IPPROTO_IPV6 &&
-		    tunnel->parms.iph.protocol != 0)
+		const struct ip_tunnel_parm_kern *parms;
+
+		parms = rcu_dereference(tunnel->sit_parms);
+		if (parms->iph.protocol != IPPROTO_IPV6 &&
+		    parms->iph.protocol != 0)
 			goto out;
 
 		skb->mac_header = skb->network_header;
@@ -762,10 +823,12 @@ static int sit_tunnel_rcv(struct sk_buff *skb, u8 ipproto)
 	tunnel = ipip6_tunnel_lookup(dev_net(skb->dev), skb->dev,
 				     iph->saddr, iph->daddr, sifindex);
 	if (tunnel) {
+		const struct ip_tunnel_parm_kern *parms;
 		const struct tnl_ptk_info *tpi;
 
-		if (tunnel->parms.iph.protocol != ipproto &&
-		    tunnel->parms.iph.protocol != 0)
+		parms = rcu_dereference(tunnel->sit_parms);
+		if (parms->iph.protocol != ipproto &&
+		    parms->iph.protocol != 0)
 			goto drop;
 
 		if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
@@ -810,25 +873,28 @@ static bool check_6rd(struct ip_tunnel *tunnel, const struct in6_addr *v6dst,
 		      __be32 *v4dst)
 {
 #ifdef CONFIG_IPV6_SIT_6RD
-	if (ipv6_prefix_equal(v6dst, &tunnel->ip6rd.prefix,
-			      tunnel->ip6rd.prefixlen)) {
+	const struct ip_tunnel_6rd_parm *ip6rd;
+
+	ip6rd = rcu_dereference(tunnel->ip6rd);
+	if (ip6rd && ipv6_prefix_equal(v6dst, &ip6rd->prefix,
+				       ip6rd->prefixlen)) {
 		unsigned int pbw0, pbi0;
 		int pbi1;
 		u32 d;
 
-		pbw0 = tunnel->ip6rd.prefixlen >> 5;
-		pbi0 = tunnel->ip6rd.prefixlen & 0x1f;
+		pbw0 = ip6rd->prefixlen >> 5;
+		pbi0 = ip6rd->prefixlen & 0x1f;
 
-		d = tunnel->ip6rd.relay_prefixlen < 32 ?
+		d = ip6rd->relay_prefixlen < 32 ?
 			(ntohl(v6dst->s6_addr32[pbw0]) << pbi0) >>
-		    tunnel->ip6rd.relay_prefixlen : 0;
+		    ip6rd->relay_prefixlen : 0;
 
-		pbi1 = pbi0 - tunnel->ip6rd.relay_prefixlen;
+		pbi1 = pbi0 - ip6rd->relay_prefixlen;
 		if (pbi1 > 0)
 			d |= ntohl(v6dst->s6_addr32[pbw0 + 1]) >>
 			     (32 - pbi1);
 
-		*v4dst = tunnel->ip6rd.relay_prefix | htonl(d);
+		*v4dst = ip6rd->relay_prefix | htonl(d);
 		return true;
 	}
 #else
@@ -901,20 +967,27 @@ static netdev_tx_t ipip6_tunnel_xmit(struct sk_buff *skb,
 				     struct net_device *dev)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
-	const struct iphdr  *tiph = &tunnel->parms.iph;
+	const struct ip_tunnel_parm_kern *parms;
+	const struct iphdr  *tiph;
 	const struct ipv6hdr *iph6 = ipv6_hdr(skb);
-	u8     tos = tunnel->parms.iph.tos;
-	__be16 df = tiph->frag_off;
+	u8     tos;
+	__be16 df;
 	struct rtable *rt;		/* Route to the other host */
 	struct net_device *tdev;	/* Device to other host */
 	unsigned int max_headroom;	/* The extra header space needed */
-	__be32 dst = tiph->daddr;
+	__be32 dst;
 	int err_count, mtu;
 	struct flowi4 fl4;
 	u8 ttl;
 	u8 protocol = IPPROTO_IPV6;
 	int t_hlen = tunnel->hlen + sizeof(struct iphdr);
 
+	parms = rcu_dereference(tunnel->sit_parms);
+	tiph = &parms->iph;
+	tos = parms->iph.tos;
+	df = tiph->frag_off;
+	dst = tiph->daddr;
+
 	if (tos == 1)
 		tos = ipv6_get_dsfield(iph6);
 
@@ -929,7 +1002,7 @@ static netdev_tx_t ipip6_tunnel_xmit(struct sk_buff *skb,
 	if (!dst && !ipip6_tunnel_dst_find(skb, &dst, false))
 		goto tx_error;
 
-	flowi4_init_output(&fl4, tunnel->parms.link, tunnel->fwmark,
+	flowi4_init_output(&fl4, parms->link, READ_ONCE(tunnel->fwmark),
 			   tos & INET_DSCP_MASK, RT_SCOPE_UNIVERSE,
 			   IPPROTO_IPV6, 0, dst, tiph->saddr, 0, 0,
 			   sock_net_uid(tunnel->net, NULL));
@@ -977,7 +1050,7 @@ static netdev_tx_t ipip6_tunnel_xmit(struct sk_buff *skb,
 			df = 0;
 		}
 
-		if (tunnel->parms.iph.daddr)
+		if (parms->iph.daddr)
 			skb_dst_update_pmtu_no_confirm(skb, mtu);
 
 		if (skb->len > mtu && !skb_is_gso(skb)) {
@@ -1046,13 +1119,17 @@ static netdev_tx_t sit_tunnel_xmit__(struct sk_buff *skb,
 				     struct net_device *dev, u8 ipproto)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
-	const struct iphdr  *tiph = &tunnel->parms.iph;
+	const struct ip_tunnel_parm_kern *parms;
+	const struct iphdr  *tiph;
 
 	if (iptunnel_handle_offloads(skb, SKB_GSO_IPXIP4))
 		goto tx_error;
 
 	skb_set_inner_ipproto(skb, ipproto);
 
+	parms = rcu_dereference(tunnel->sit_parms);
+	tiph = &parms->iph;
+
 	ip_tunnel_xmit(skb, dev, tiph, ipproto);
 	return NETDEV_TX_OK;
 tx_error:
@@ -1067,6 +1144,7 @@ static netdev_tx_t sit_tunnel_xmit(struct sk_buff *skb,
 	if (!pskb_inet_may_pull(skb))
 		goto tx_err;
 
+	rcu_read_lock();
 	switch (skb->protocol) {
 	case htons(ETH_P_IP):
 		sit_tunnel_xmit__(skb, dev, IPPROTO_IPIP);
@@ -1080,8 +1158,10 @@ static netdev_tx_t sit_tunnel_xmit(struct sk_buff *skb,
 		break;
 #endif
 	default:
+		rcu_read_unlock();
 		goto tx_err;
 	}
+	rcu_read_unlock();
 
 	return NETDEV_TX_OK;
 
@@ -1089,19 +1169,20 @@ static netdev_tx_t sit_tunnel_xmit(struct sk_buff *skb,
 	DEV_STATS_INC(dev, tx_errors);
 	kfree_skb(skb);
 	return NETDEV_TX_OK;
-
 }
 
 static void ipip6_tunnel_bind_dev(struct net_device *dev)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
 	int t_hlen = tunnel->hlen + sizeof(struct iphdr);
+	const struct ip_tunnel_parm_kern *parms;
 	struct net_device *tdev = NULL;
 	int hlen = LL_MAX_HEADER;
 	const struct iphdr *iph;
 	struct flowi4 fl4;
 
-	iph = &tunnel->parms.iph;
+	parms = rtnl_dereference(tunnel->sit_parms);
+	iph = &parms->iph;
 
 	if (iph->daddr) {
 		struct rtable *rt = ip_route_output_ports(tunnel->net, &fl4,
@@ -1110,7 +1191,7 @@ static void ipip6_tunnel_bind_dev(struct net_device *dev)
 							  0, 0,
 							  IPPROTO_IPV6,
 							  iph->tos & INET_DSCP_MASK,
-							  tunnel->parms.link);
+							  parms->link);
 
 		if (!IS_ERR(rt)) {
 			tdev = rt->dst.dev;
@@ -1119,8 +1200,8 @@ static void ipip6_tunnel_bind_dev(struct net_device *dev)
 		dev->flags |= IFF_POINTOPOINT;
 	}
 
-	if (!tdev && tunnel->parms.link)
-		tdev = __dev_get_by_index(tunnel->net, tunnel->parms.link);
+	if (!tdev && parms->link)
+		tdev = __dev_get_by_index(tunnel->net, parms->link);
 
 	if (tdev && !netif_is_l3_master(tdev)) {
 		int mtu;
@@ -1134,36 +1215,57 @@ static void ipip6_tunnel_bind_dev(struct net_device *dev)
 	dev->needed_headroom = ip_tunnel_limit_headroom(t_hlen + hlen);
 }
 
-static void ipip6_tunnel_update(struct ip_tunnel *t,
-				struct ip_tunnel_parm_kern *p,
-				__u32 fwmark)
+static int ipip6_tunnel_update(struct ip_tunnel *t,
+			       struct ip_tunnel_parm_kern *p,
+			       __u32 fwmark)
 {
 	struct net *net = t->net;
 	struct sit_net *sitn = net_generic(net, sit_net_id);
-
-	ipip6_tunnel_unlink(sitn, t);
-	synchronize_net();
-	t->parms.iph.saddr = p->iph.saddr;
-	t->parms.iph.daddr = p->iph.daddr;
-	__dev_addr_set(t->dev, &p->iph.saddr, 4);
-	memcpy(t->dev->broadcast, &p->iph.daddr, 4);
-	ipip6_tunnel_link(sitn, t);
-	t->parms.iph.ttl = p->iph.ttl;
-	t->parms.iph.tos = p->iph.tos;
-	t->parms.iph.frag_off = p->iph.frag_off;
-	if (t->parms.link != p->link || t->fwmark != fwmark) {
-		t->parms.link = p->link;
-		t->fwmark = fwmark;
+	struct ip_tunnel_parm_kern *new_p, *old_p;
+	bool move;
+
+	old_p = rtnl_dereference(t->sit_parms);
+	new_p = kmalloc_obj(*new_p);
+	if (!new_p)
+		return -ENOMEM;
+	*new_p = *old_p;
+	new_p->iph.saddr = p->iph.saddr;
+	new_p->iph.daddr = p->iph.daddr;
+	new_p->iph.ttl = p->iph.ttl;
+	new_p->iph.tos = p->iph.tos;
+	new_p->iph.frag_off = p->iph.frag_off;
+	new_p->link = p->link;
+	move = old_p->iph.saddr != p->iph.saddr ||
+	       old_p->iph.daddr != p->iph.daddr;
+
+	if (move)
+		ipip6_tunnel_unlink(sitn, t);
+
+	t->parms.iph = new_p->iph;
+	WRITE_ONCE(t->parms.link, new_p->link);
+	rcu_assign_pointer(t->sit_parms, new_p);
+
+	if (move) {
+		synchronize_net();
+		__dev_addr_set(t->dev, &p->iph.saddr, 4);
+		memcpy(t->dev->broadcast, &p->iph.daddr, 4);
+		ipip6_tunnel_link(sitn, t);
+	}
+	if (old_p->link != p->link || t->fwmark != fwmark) {
+		WRITE_ONCE(t->fwmark, fwmark);
 		ipip6_tunnel_bind_dev(t->dev);
 	}
 	dst_cache_reset(&t->dst_cache);
 	netdev_state_change(t->dev);
+	kfree_rcu(old_p, rcu);
+	return 0;
 }
 
 #ifdef CONFIG_IPV6_SIT_6RD
 static int ipip6_tunnel_update_6rd(struct ip_tunnel *t,
 				   struct ip_tunnel_6rd *ip6rd)
 {
+	struct ip_tunnel_6rd_parm *new_6rd, *old_6rd;
 	struct in6_addr prefix;
 	__be32 relay_prefix;
 
@@ -1183,10 +1285,20 @@ static int ipip6_tunnel_update_6rd(struct ip_tunnel *t,
 	if (relay_prefix != ip6rd->relay_prefix)
 		return -EINVAL;
 
-	t->ip6rd.prefix = prefix;
-	t->ip6rd.relay_prefix = relay_prefix;
-	t->ip6rd.prefixlen = ip6rd->prefixlen;
-	t->ip6rd.relay_prefixlen = ip6rd->relay_prefixlen;
+	new_6rd = kmalloc_obj(*new_6rd);
+	if (!new_6rd)
+		return -ENOMEM;
+
+	new_6rd->prefix = prefix;
+	new_6rd->relay_prefix = relay_prefix;
+	new_6rd->prefixlen = ip6rd->prefixlen;
+	new_6rd->relay_prefixlen = ip6rd->relay_prefixlen;
+
+	old_6rd = rtnl_dereference(t->ip6rd);
+	rcu_assign_pointer(t->ip6rd, new_6rd);
+	if (old_6rd)
+		kfree_rcu(old_6rd, rcu);
+
 	dst_cache_reset(&t->dst_cache);
 	netdev_state_change(t->dev);
 	return 0;
@@ -1195,6 +1307,7 @@ static int ipip6_tunnel_update_6rd(struct ip_tunnel *t,
 static int
 ipip6_tunnel_get6rd(struct net_device *dev, struct ip_tunnel_parm __user *data)
 {
+	const struct ip_tunnel_6rd_parm *ip6rd_parm;
 	struct ip_tunnel *t = netdev_priv(dev);
 	struct ip_tunnel_parm_kern p;
 	struct ip_tunnel_6rd ip6rd;
@@ -1207,10 +1320,15 @@ ipip6_tunnel_get6rd(struct net_device *dev, struct ip_tunnel_parm __user *data)
 	if (!t)
 		t = netdev_priv(dev);
 
-	ip6rd.prefix = t->ip6rd.prefix;
-	ip6rd.relay_prefix = t->ip6rd.relay_prefix;
-	ip6rd.prefixlen = t->ip6rd.prefixlen;
-	ip6rd.relay_prefixlen = t->ip6rd.relay_prefixlen;
+	ip6rd_parm = rtnl_dereference(t->ip6rd);
+	if (ip6rd_parm) {
+		ip6rd.prefix = ip6rd_parm->prefix;
+		ip6rd.relay_prefix = ip6rd_parm->relay_prefix;
+		ip6rd.prefixlen = ip6rd_parm->prefixlen;
+		ip6rd.relay_prefixlen = ip6rd_parm->relay_prefixlen;
+	} else {
+		memset(&ip6rd, 0, sizeof(ip6rd));
+	}
 	if (copy_to_user(data, &ip6rd, sizeof(ip6rd)))
 		return -EFAULT;
 	return 0;
@@ -1222,20 +1340,16 @@ ipip6_tunnel_6rdctl(struct net_device *dev, struct ip_tunnel_6rd __user *data,
 {
 	struct ip_tunnel *t = netdev_priv(dev);
 	struct ip_tunnel_6rd ip6rd;
-	int err;
 
 	if (!ns_capable(t->net->user_ns, CAP_NET_ADMIN))
 		return -EPERM;
 	if (copy_from_user(&ip6rd, data, sizeof(ip6rd)))
 		return -EFAULT;
 
-	if (cmd != SIOCDEL6RD) {
-		err = ipip6_tunnel_update_6rd(t, &ip6rd);
-		if (err < 0)
-			return err;
-	} else
-		ipip6_tunnel_clone_6rd(dev, dev_to_sit_net(dev));
-	return 0;
+	if (cmd != SIOCDEL6RD)
+		return ipip6_tunnel_update_6rd(t, &ip6rd);
+
+	return ipip6_tunnel_clone_6rd(dev, dev_to_sit_net(dev));
 }
 
 #endif /* CONFIG_IPV6_SIT_6RD */
@@ -1271,12 +1385,14 @@ static int
 ipip6_tunnel_get(struct net_device *dev, struct ip_tunnel_parm_kern *p)
 {
 	struct ip_tunnel *t = netdev_priv(dev);
+	const struct ip_tunnel_parm_kern *parms;
 
 	if (dev == dev_to_sit_net(dev)->fb_tunnel_dev)
 		t = ipip6_tunnel_locate(t->net, p, 0);
 	if (!t)
 		t = netdev_priv(dev);
-	memcpy(p, &t->parms, sizeof(*p));
+	parms = rtnl_dereference(t->sit_parms);
+	memcpy(p, parms, sizeof(*p));
 	return 0;
 }
 
@@ -1321,7 +1437,9 @@ ipip6_tunnel_change(struct net_device *dev, struct ip_tunnel_parm_kern *p)
 			t = netdev_priv(dev);
 		}
 
-		ipip6_tunnel_update(t, p, t->fwmark);
+		err = ipip6_tunnel_update(t, p, t->fwmark);
+		if (err)
+			return err;
 	}
 
 	return 0;
@@ -1394,20 +1512,47 @@ ipip6_tunnel_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
 	}
 }
 
+static int ipip6_get_iflink(const struct net_device *dev)
+{
+	struct ip_tunnel *tunnel = netdev_priv(dev);
+	const struct ip_tunnel_parm_kern *parms;
+	int link;
+
+	rcu_read_lock();
+	parms = rcu_dereference(tunnel->sit_parms);
+	link = parms ? parms->link : 0;
+	rcu_read_unlock();
+
+	return link;
+}
+
 static const struct net_device_ops ipip6_netdev_ops = {
 	.ndo_init	= ipip6_tunnel_init,
 	.ndo_uninit	= ipip6_tunnel_uninit,
 	.ndo_start_xmit	= sit_tunnel_xmit,
 	.ndo_siocdevprivate = ipip6_tunnel_siocdevprivate,
-	.ndo_get_iflink = ip_tunnel_get_iflink,
+	.ndo_get_iflink = ipip6_get_iflink,
 	.ndo_tunnel_ctl = ipip6_tunnel_ctl,
 };
 
 static void ipip6_dev_free(struct net_device *dev)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
+	struct ip_tunnel_parm_kern *parms;
+#ifdef CONFIG_IPV6_SIT_6RD
+	struct ip_tunnel_6rd_parm *ip6rd;
 
-	dst_cache_destroy(&tunnel->dst_cache);
+	ip6rd = rcu_dereference_protected(tunnel->ip6rd, 1);
+	RCU_INIT_POINTER(tunnel->ip6rd, NULL);
+	kfree(ip6rd);
+#endif
+	parms = rcu_dereference_protected(tunnel->sit_parms, 1);
+	RCU_INIT_POINTER(tunnel->sit_parms, NULL);
+	kfree(parms);
+	if (tunnel->dst_cache.cache) {
+		dst_cache_destroy(&tunnel->dst_cache);
+		tunnel->dst_cache.cache = NULL;
+	}
 }
 
 #define SIT_FEATURES (NETIF_F_SG	   | \
@@ -1443,10 +1588,12 @@ static void ipip6_tunnel_setup(struct net_device *dev)
 static int ipip6_tunnel_init(struct net_device *dev)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
+	struct ip_tunnel_parm_kern *parms;
 	int err;
 
 	tunnel->dev = dev;
-	strscpy(tunnel->parms.name, dev->name);
+	parms = rtnl_dereference(tunnel->sit_parms);
+	strscpy(parms->name, dev->name);
 
 	ipip6_tunnel_bind_dev(dev);
 
@@ -1462,15 +1609,8 @@ static int ipip6_tunnel_init(struct net_device *dev)
 static void __net_init ipip6_fb_tunnel_init(struct net_device *dev)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
-	struct iphdr *iph = &tunnel->parms.iph;
 	struct net *net = dev_net(dev);
 	struct sit_net *sitn = net_generic(net, sit_net_id);
-
-	iph->version		= 4;
-	iph->protocol		= IPPROTO_IPV6;
-	iph->ihl		= 5;
-	iph->ttl		= 64;
-
 	rcu_assign_pointer(sitn->tunnels_wc[0], tunnel);
 }
 
@@ -1557,6 +1697,8 @@ static int ipip6_newlink(struct net_device *dev,
 #ifdef CONFIG_IPV6_SIT_6RD
 	struct ip_tunnel_6rd ip6rd;
 #endif
+	struct ip_tunnel_parm_kern *nt_parms;
+	struct ip_tunnel_parm_kern p;
 	struct net *net;
 	int err;
 
@@ -1570,14 +1712,22 @@ static int ipip6_newlink(struct net_device *dev,
 			return err;
 	}
 
-	ipip6_netlink_parms(data, &nt->parms, &nt->fwmark);
+	ipip6_netlink_parms(data, &p, &nt->fwmark);
 
-	if (ipip6_tunnel_locate(net, &nt->parms, 0))
+	if (ipip6_tunnel_locate(net, &p, 0))
 		return -EEXIST;
 
+	nt_parms = kmalloc_obj(*nt_parms);
+	if (!nt_parms)
+		return -ENOMEM;
+	*nt_parms = p;
+	rcu_assign_pointer(nt->sit_parms, nt_parms);
+
 	err = ipip6_tunnel_create(dev);
-	if (err < 0)
+	if (err < 0) {
+		ipip6_dev_free(dev);
 		return err;
+	}
 
 	if (tb[IFLA_MTU]) {
 		u32 mtu = nla_get_u32(tb[IFLA_MTU]);
@@ -1639,7 +1789,9 @@ static int ipip6_changelink(struct net_device *dev, struct nlattr *tb[],
 	} else
 		t = netdev_priv(dev);
 
-	ipip6_tunnel_update(t, &p, fwmark);
+	err = ipip6_tunnel_update(t, &p, fwmark);
+	if (err)
+		return err;
 
 #ifdef CONFIG_IPV6_SIT_6RD
 	if (ipip6_netlink_6rd_parms(data, &ip6rd))
@@ -1694,7 +1846,18 @@ static size_t ipip6_get_size(const struct net_device *dev)
 static int ipip6_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
 	struct ip_tunnel *tunnel = netdev_priv(dev);
-	struct ip_tunnel_parm_kern *parm = &tunnel->parms;
+	const struct ip_tunnel_parm_kern *parm;
+#ifdef CONFIG_IPV6_SIT_6RD
+	const struct ip_tunnel_6rd_parm *ip6rd;
+#endif
+
+	rcu_read_lock();
+	parm = rcu_dereference(tunnel->sit_parms);
+	/* If the device is being dismantled, there is nothing to report. */
+	if (!parm) {
+		rcu_read_unlock();
+		return 0;
+	}
 
 	if (nla_put_u32(skb, IFLA_IPTUN_LINK, parm->link) ||
 	    nla_put_in_addr(skb, IFLA_IPTUN_LOCAL, parm->iph.saddr) ||
@@ -1706,34 +1869,38 @@ static int ipip6_fill_info(struct sk_buff *skb, const struct net_device *dev)
 	    nla_put_u8(skb, IFLA_IPTUN_PROTO, parm->iph.protocol) ||
 	    nla_put_be16(skb, IFLA_IPTUN_FLAGS,
 			 ip_tunnel_flags_to_be16(parm->i_flags)) ||
-	    nla_put_u32(skb, IFLA_IPTUN_FWMARK, tunnel->fwmark))
+	    nla_put_u32(skb, IFLA_IPTUN_FWMARK, READ_ONCE(tunnel->fwmark)))
 		goto nla_put_failure;
 
 #ifdef CONFIG_IPV6_SIT_6RD
-	if (nla_put_in6_addr(skb, IFLA_IPTUN_6RD_PREFIX,
-			     &tunnel->ip6rd.prefix) ||
-	    nla_put_in_addr(skb, IFLA_IPTUN_6RD_RELAY_PREFIX,
-			    tunnel->ip6rd.relay_prefix) ||
-	    nla_put_u16(skb, IFLA_IPTUN_6RD_PREFIXLEN,
-			tunnel->ip6rd.prefixlen) ||
-	    nla_put_u16(skb, IFLA_IPTUN_6RD_RELAY_PREFIXLEN,
-			tunnel->ip6rd.relay_prefixlen))
+	ip6rd = rcu_dereference(tunnel->ip6rd);
+	if (ip6rd &&
+	    (nla_put_in6_addr(skb, IFLA_IPTUN_6RD_PREFIX,
+			      &ip6rd->prefix) ||
+	     nla_put_in_addr(skb, IFLA_IPTUN_6RD_RELAY_PREFIX,
+			     ip6rd->relay_prefix) ||
+	     nla_put_u16(skb, IFLA_IPTUN_6RD_PREFIXLEN,
+			 ip6rd->prefixlen) ||
+	     nla_put_u16(skb, IFLA_IPTUN_6RD_RELAY_PREFIXLEN,
+			 ip6rd->relay_prefixlen)))
 		goto nla_put_failure;
 #endif
 
 	if (nla_put_u16(skb, IFLA_IPTUN_ENCAP_TYPE,
-			tunnel->encap.type) ||
+			READ_ONCE(tunnel->encap.type)) ||
 	    nla_put_be16(skb, IFLA_IPTUN_ENCAP_SPORT,
-			tunnel->encap.sport) ||
+			 READ_ONCE(tunnel->encap.sport)) ||
 	    nla_put_be16(skb, IFLA_IPTUN_ENCAP_DPORT,
-			tunnel->encap.dport) ||
+			 READ_ONCE(tunnel->encap.dport)) ||
 	    nla_put_u16(skb, IFLA_IPTUN_ENCAP_FLAGS,
-			tunnel->encap.flags))
+			READ_ONCE(tunnel->encap.flags)))
 		goto nla_put_failure;
 
+	rcu_read_unlock();
 	return 0;
 
 nla_put_failure:
+	rcu_read_unlock();
 	return -EMSGSIZE;
 }
 
@@ -1835,6 +2002,7 @@ static void __net_exit sit_exit_rtnl_net(struct net *net, struct list_head *head
 static int __net_init sit_init_net(struct net *net)
 {
 	struct sit_net *sitn = net_generic(net, sit_net_id);
+	struct ip_tunnel_parm_kern *nt_parms;
 	struct ip_tunnel *t;
 	int err;
 
@@ -1862,18 +2030,32 @@ static int __net_init sit_init_net(struct net *net)
 
 	t = netdev_priv(sitn->fb_tunnel_dev);
 	t->net = net;
+	nt_parms = kzalloc_obj(*nt_parms);
+	if (!nt_parms) {
+		err = -ENOMEM;
+		goto err_reg_dev;
+	}
+	nt_parms->iph.version	= 4;
+	nt_parms->iph.protocol	= IPPROTO_IPV6;
+	nt_parms->iph.ihl	= 5;
+	nt_parms->iph.ttl	= 64;
+	strscpy(nt_parms->name, sitn->fb_tunnel_dev->name);
+	t->parms = *nt_parms;
+	rcu_assign_pointer(t->sit_parms, nt_parms);
+
+	err = ipip6_tunnel_clone_6rd(sitn->fb_tunnel_dev, sitn);
+	if (err < 0)
+		goto err_reg_dev;
 
 	err = register_netdev(sitn->fb_tunnel_dev);
 	if (err)
 		goto err_reg_dev;
 
-	ipip6_tunnel_clone_6rd(sitn->fb_tunnel_dev, sitn);
 	ipip6_fb_tunnel_init(sitn->fb_tunnel_dev);
-
-	strscpy(t->parms.name, sitn->fb_tunnel_dev->name);
 	return 0;
 
 err_reg_dev:
+	ipip6_dev_free(sitn->fb_tunnel_dev);
 	free_netdev(sitn->fb_tunnel_dev);
 err_alloc_dev:
 	return err;
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 7fa4ed2fd4f166..afb4d8e9fa67b1 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -2104,7 +2104,7 @@ static void get_openreq6(struct seq_file *seq,
 
 	seq_printf(seq,
 		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
-		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d %pK\n",
+		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %d %d 0\n",
 		   i,
 		   src->s6_addr32[0], src->s6_addr32[1],
 		   src->s6_addr32[2], src->s6_addr32[3],
@@ -2121,7 +2121,7 @@ static void get_openreq6(struct seq_file *seq,
 				    sk_uid(req->rsk_listener)),
 		   0,  /* non standard timer */
 		   0, /* open_requests have no inode */
-		   0, req);
+		   0);
 }
 
 static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
@@ -2172,7 +2172,7 @@ static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
 
 	seq_printf(seq,
 		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
-		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d %pK %lu %lu %u %u %d\n",
+		   "%02X %08X:%08X %02X:%08lX %08X %5u %8d %llu %d 0 %lu %lu %u %u %d\n",
 		   i,
 		   src->s6_addr32[0], src->s6_addr32[1],
 		   src->s6_addr32[2], src->s6_addr32[3], srcp,
@@ -2187,7 +2187,7 @@ static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i)
 		   from_kuid_munged(seq_user_ns(seq), sk_uid(sp)),
 		   READ_ONCE(icsk->icsk_probes_out),
 		   sock_i_ino(sp),
-		   refcount_read(&sp->sk_refcnt), sp,
+		   refcount_read(&sp->sk_refcnt),
 		   jiffies_to_clock_t(icsk->icsk_rto),
 		   jiffies_to_clock_t(icsk->icsk_ack.ato),
 		   (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sp),
@@ -2212,7 +2212,7 @@ static void get_timewait6_sock(struct seq_file *seq,
 
 	seq_printf(seq,
 		   "%4d: %08X%08X%08X%08X:%04X %08X%08X%08X%08X:%04X "
-		   "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK\n",
+		   "%02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0\n",
 		   i,
 		   src->s6_addr32[0], src->s6_addr32[1],
 		   src->s6_addr32[2], src->s6_addr32[3], srcp,
@@ -2220,7 +2220,7 @@ static void get_timewait6_sock(struct seq_file *seq,
 		   dest->s6_addr32[2], dest->s6_addr32[3], destp,
 		   READ_ONCE(tw->tw_substate), 0, 0,
 		   3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
-		   refcount_read(&tw->tw_refcnt), tw);
+		   refcount_read(&tw->tw_refcnt));
 }
 
 static int tcp6_seq_show(struct seq_file *seq, void *v)
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 93478d1ad5769c..59f9872c355211 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -751,13 +751,7 @@ static int udpv6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 			harderr = 1;
 	}
 	if (type == NDISC_REDIRECT) {
-		if (tunnel) {
-			ip6_redirect(skb, sock_net(sk), inet6_iif(skb),
-				     READ_ONCE(sk->sk_mark),
-				     sk_uid(sk));
-		} else {
-			ip6_sk_redirect(skb, sk);
-		}
+		ip6_sk_redirect(skb, sk);
 		goto out;
 	}
 
diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c
index db261ecd19af10..5ba875798aa414 100644
--- a/net/iucv/af_iucv.c
+++ b/net/iucv/af_iucv.c
@@ -90,7 +90,7 @@ do {									\
 static struct sock *iucv_accept_dequeue(struct sock *parent,
 					struct socket *newsock);
 static void iucv_sock_kill(struct sock *sk);
-static void iucv_sock_close(struct sock *sk);
+static void __iucv_sock_close(struct sock *sk);
 
 static void afiucv_hs_callback_txnotify(struct sock *sk, enum iucv_tx_notify);
 
@@ -302,7 +302,9 @@ static void iucv_sock_cleanup_listen(struct sock *parent)
 
 	/* Close non-accepted connections */
 	while ((sk = iucv_accept_dequeue(parent, NULL))) {
-		iucv_sock_close(sk);
+		lock_sock(sk);
+		__iucv_sock_close(sk);
+		release_sock(sk);
 		iucv_sock_kill(sk);
 	}
 
@@ -394,15 +396,16 @@ static int iucv_send_ctrl(struct sock *sk, u8 flags)
 	return err;
 }
 
-/* Close an IUCV socket */
-static void iucv_sock_close(struct sock *sk)
+/**
+ * __iucv_sock_close() - close socket, caller holds lock_sock(sk)
+ * @sk: network socket
+ */
+static void __iucv_sock_close(struct sock *sk)
 {
 	struct iucv_sock *iucv = iucv_sk(sk);
 	unsigned long timeo;
 	int err = 0;
 
-	lock_sock(sk);
-
 	switch (sk->sk_state) {
 	case IUCV_LISTEN:
 		iucv_sock_cleanup_listen(sk);
@@ -454,7 +457,12 @@ static void iucv_sock_close(struct sock *sk)
 
 	/* mark socket for deletion by iucv_sock_kill() */
 	sock_set_flag(sk, SOCK_ZAPPED);
+}
 
+static void iucv_sock_close(struct sock *sk)
+{
+	lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
+	__iucv_sock_close(sk);
 	release_sock(sk);
 }
 
diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c
index 71af69d442f211..962ee2c4acd2fb 100644
--- a/net/kcm/kcmsock.c
+++ b/net/kcm/kcmsock.c
@@ -1580,10 +1580,10 @@ static int kcm_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
 		struct kcm_clone info;
 
 		FD_PREPARE(fdf, 0, kcm_clone(sock));
-		if (fdf.err)
-			return fdf.err;
+		if (fdf->fd < 0)
+			return fdf->fd;
 
-		info.fd = fd_prepare_fd(fdf);
+		info.fd = fdf->fd;
 		if (copy_to_user((void __user *)arg, &info, sizeof(info)))
 			return -EFAULT;
 
diff --git a/net/key/af_key.c b/net/key/af_key.c
index 1d8965d7f4f3c7..d215b33b7492c2 100644
--- a/net/key/af_key.c
+++ b/net/key/af_key.c
@@ -3803,10 +3803,9 @@ static int pfkey_seq_show(struct seq_file *f, void *v)
 	struct sock *s = sk_entry(v);
 
 	if (v == SEQ_START_TOKEN)
-		seq_printf(f ,"sk       RefCnt Rmem   Wmem   User   Inode\n");
+		seq_puts(f, "sk RefCnt Rmem   Wmem   User   Inode\n");
 	else
-		seq_printf(f, "%pK %-6d %-6u %-6u %-6u %-6llu\n",
-			       s,
+		seq_printf(f, "0  %-6d %-6u %-6u %-6u %-6llu\n",
 			       refcount_read(&s->sk_refcnt),
 			       sk_rmem_alloc_get(s),
 			       sk_wmem_alloc_get(s),
diff --git a/net/llc/llc_c_ac.c b/net/llc/llc_c_ac.c
index 724ecd741d4cf4..1aa7fe28acddb2 100644
--- a/net/llc/llc_c_ac.c
+++ b/net/llc/llc_c_ac.c
@@ -437,7 +437,7 @@ int llc_conn_ac_resend_i_xxx_x_set_0_or_send_rr(struct sock *sk,
 		if (likely(!rc))
 			llc_conn_send_pdu(sk, nskb);
 		else
-			kfree_skb(skb);
+			kfree_skb(nskb);
 	}
 	if (rc) {
 		nr = LLC_I_GET_NR(pdu);
diff --git a/net/llc/llc_conn.c b/net/llc/llc_conn.c
index 260460d50f54c4..4b6fe9a473cb39 100644
--- a/net/llc/llc_conn.c
+++ b/net/llc/llc_conn.c
@@ -193,7 +193,7 @@ void llc_conn_rtn_pdu(struct sock *sk, struct sk_buff *skb)
 }
 
 /**
- *	llc_conn_resend_i_pdu_as_cmd - resend all all unacknowledged I PDUs
+ *	llc_conn_resend_i_pdu_as_cmd - resend all unacknowledged I PDUs
  *	@sk: active connection
  *	@nr: NR
  *	@first_p_bit: p_bit value of first pdu
diff --git a/net/llc/llc_s_ac.c b/net/llc/llc_s_ac.c
index 98deee56037351..831998211b52ee 100644
--- a/net/llc/llc_s_ac.c
+++ b/net/llc/llc_s_ac.c
@@ -121,6 +121,8 @@ int llc_sap_action_send_xid_r(struct llc_sap *sap, struct sk_buff *skb)
 	rc = llc_mac_hdr_init(nskb, mac_sa, mac_da);
 	if (likely(!rc))
 		rc = dev_queue_xmit(nskb);
+	else
+		kfree_skb(nskb);
 out:
 	return rc;
 }
@@ -170,6 +172,8 @@ int llc_sap_action_send_test_r(struct llc_sap *sap, struct sk_buff *skb)
 	rc = llc_mac_hdr_init(nskb, mac_sa, mac_da);
 	if (likely(!rc))
 		rc = dev_queue_xmit(nskb);
+	else
+		kfree_skb(nskb);
 out:
 	return rc;
 }
diff --git a/net/llc/llc_sap.c b/net/llc/llc_sap.c
index 1bd446a21092f8..3904a1b4ba8445 100644
--- a/net/llc/llc_sap.c
+++ b/net/llc/llc_sap.c
@@ -19,12 +19,12 @@
 #include <linux/llc.h>
 #include <linux/slab.h>
 
-static int llc_mac_header_len(unsigned short devtype)
+static int llc_mac_header_len(struct net_device *dev)
 {
-	switch (devtype) {
+	switch (dev->type) {
 	case ARPHRD_ETHER:
 	case ARPHRD_LOOPBACK:
-		return sizeof(struct ethhdr);
+		return LL_RESERVED_SPACE(dev);
 	}
 	return 0;
 }
@@ -45,7 +45,7 @@ struct sk_buff *llc_alloc_frame(struct sock *sk, struct net_device *dev,
 	int hlen = type == LLC_PDU_TYPE_U ? 3 : 4;
 	struct sk_buff *skb;
 
-	hlen += llc_mac_header_len(dev->type);
+	hlen += llc_mac_header_len(dev);
 	skb = alloc_skb(hlen + data_size, GFP_ATOMIC);
 
 	if (skb) {
diff --git a/net/mac80211/agg-rx.c b/net/mac80211/agg-rx.c
index 9629e00069a1f3..01d6703767c638 100644
--- a/net/mac80211/agg-rx.c
+++ b/net/mac80211/agg-rx.c
@@ -102,7 +102,7 @@ void __ieee80211_stop_rx_ba_session(struct sta_info *sta, u16 tid,
 	if (!tid_rx)
 		return;
 
-	timer_delete_sync(&tid_rx->session_timer);
+	timer_shutdown_sync(&tid_rx->session_timer);
 
 	/* make sure ieee80211_sta_reorder_release() doesn't re-arm the timer */
 	spin_lock_bh(&tid_rx->reorder_lock);
diff --git a/net/mac80211/agg-tx.c b/net/mac80211/agg-tx.c
index 0832213430f431..f3ecdd8601a042 100644
--- a/net/mac80211/agg-tx.c
+++ b/net/mac80211/agg-tx.c
@@ -98,7 +98,7 @@ static void ieee80211_send_addba_request(struct sta_info *sta, u16 tid,
 	if (sta->sta.deflink.he_cap.has_he)
 		ieee80211_add_addbaext(skb, 0, agg_size);
 
-	ieee80211_tx_skb_tid(sdata, skb, tid, -1);
+	ieee80211_tx_skb_tid(sdata, skb, NULL, tid, -1);
 }
 
 void ieee80211_send_bar(struct ieee80211_vif *vif, u8 *ra, u16 tid, u16 ssn)
@@ -127,7 +127,7 @@ void ieee80211_send_bar(struct ieee80211_vif *vif, u8 *ra, u16 tid, u16 ssn)
 
 	IEEE80211_SKB_CB(skb)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT |
 					IEEE80211_TX_CTL_REQ_TX_STATUS;
-	ieee80211_tx_skb_tid(sdata, skb, tid, -1);
+	ieee80211_tx_skb_tid(sdata, skb, NULL, tid, -1);
 }
 EXPORT_SYMBOL(ieee80211_send_bar);
 
@@ -212,10 +212,7 @@ ieee80211_agg_start_txq(struct sta_info *sta, int tid, bool enable)
 
 	txqi = to_txq_info(txq);
 
-	if (enable)
-		set_bit(IEEE80211_TXQ_AMPDU, &txqi->flags);
-	else
-		clear_bit(IEEE80211_TXQ_AMPDU, &txqi->flags);
+	assign_bit(IEEE80211_TXQ_AMPDU, &txqi->flags, enable);
 
 	clear_bit(IEEE80211_TXQ_STOP, &txqi->flags);
 	local_bh_disable();
diff --git a/net/mac80211/ap.c b/net/mac80211/ap.c
index e7ac99c5a22ebe..fe255891c35f09 100644
--- a/net/mac80211/ap.c
+++ b/net/mac80211/ap.c
@@ -82,9 +82,6 @@ ieee80211_rx_eml_op_mode_notif(struct ieee80211_sub_if_data *sdata,
 	if (!ift_ext_capa)
 		return;
 
-	if (!status->link_valid)
-		return;
-
 	sta = sta_info_get_bss(sdata, mgmt->sa);
 	if (!sta)
 		return;
diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
index a1753335eb9d02..bd1a857c812d45 100644
--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -615,12 +615,15 @@ static int ieee80211_set_tx(struct ieee80211_sub_if_data *sdata,
 }
 
 static int ieee80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			     int link_id, u8 key_idx, bool pairwise,
+			     int link_id, u8 key_idx,
+			     enum nl80211_key_type type,
 			     const u8 *mac_addr, struct key_params *params)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev);
 	struct ieee80211_link_data *link =
 		ieee80211_link_or_deflink(sdata, link_id, false);
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
+	bool cigtk = type == NL80211_KEYTYPE_CIGTK;
 	struct ieee80211_local *local = sdata->local;
 	struct sta_info *sta = NULL;
 	struct ieee80211_key *key;
@@ -664,6 +667,9 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 		key->conf.link_id = -1;
 	} else {
 		key->conf.link_id = link->link_id;
+
+		if (cigtk)
+			key->conf.flags |= IEEE80211_KEY_FLAG_CIP;
 	}
 
 	if (params->mode == NL80211_KEY_NO_TX)
@@ -741,10 +747,13 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 
 static struct ieee80211_key *
 ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, int link_id,
-		     u8 key_idx, bool pairwise, const u8 *mac_addr)
+		     u8 key_idx, enum nl80211_key_type type,
+		     const u8 *mac_addr)
 {
 	struct ieee80211_local *local __maybe_unused = sdata->local;
 	struct ieee80211_link_data *link = &sdata->deflink;
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
+	bool cigtk = type == NL80211_KEYTYPE_CIGTK;
 	struct ieee80211_key *key;
 
 	if (link_id >= 0) {
@@ -774,7 +783,11 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, int link_id,
 			return wiphy_dereference(local->hw.wiphy,
 						 sta->ptk[key_idx]);
 
-		if (!pairwise &&
+		if (cigtk && key_idx < NUM_CTRL_KEYS)
+			return wiphy_dereference(local->hw.wiphy,
+						 link_sta->cigtk[key_idx]);
+
+		if (!pairwise && !cigtk &&
 		    key_idx < NUM_DEFAULT_KEYS +
 			      NUM_DEFAULT_MGMT_KEYS +
 			      NUM_DEFAULT_BEACON_KEYS)
@@ -787,6 +800,9 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, int link_id,
 	if (pairwise && key_idx < NUM_DEFAULT_KEYS)
 		return wiphy_dereference(local->hw.wiphy, sdata->keys[key_idx]);
 
+	if (cigtk)
+		return wiphy_dereference(local->hw.wiphy, link->cigtk[key_idx]);
+
 	key = wiphy_dereference(local->hw.wiphy, link->gtk[key_idx]);
 	if (key)
 		return key;
@@ -799,8 +815,8 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, int link_id,
 }
 
 static int ieee80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			     int link_id, u8 key_idx, bool pairwise,
-			     const u8 *mac_addr)
+			     int link_id, u8 key_idx,
+			     enum nl80211_key_type type, const u8 *mac_addr)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev);
 	struct ieee80211_local *local = sdata->local;
@@ -808,7 +824,7 @@ static int ieee80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 
 	lockdep_assert_wiphy(local->hw.wiphy);
 
-	key = ieee80211_lookup_key(sdata, link_id, key_idx, pairwise, mac_addr);
+	key = ieee80211_lookup_key(sdata, link_id, key_idx, type, mac_addr);
 	if (!key)
 		return -ENOENT;
 
@@ -818,7 +834,8 @@ static int ieee80211_del_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 }
 
 static int ieee80211_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
-			     int link_id, u8 key_idx, bool pairwise,
+			     int link_id, u8 key_idx,
+			     enum nl80211_key_type type,
 			     const u8 *mac_addr, void *cookie,
 			     void (*callback)(void *cookie,
 					      struct key_params *params))
@@ -837,7 +854,7 @@ static int ieee80211_get_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 
 	rcu_read_lock();
 
-	key = ieee80211_lookup_key(sdata, link_id, key_idx, pairwise, mac_addr);
+	key = ieee80211_lookup_key(sdata, link_id, key_idx, type, mac_addr);
 	if (!key)
 		goto out;
 
@@ -2296,6 +2313,8 @@ static int sta_link_apply_parameters(struct ieee80211_local *local,
 	case STA_LINK_MODE_NEW:
 		if (!params->link_mac)
 			return -EINVAL;
+		if (sta->sta.cip && !params->cip_cap_set)
+			return -EINVAL;
 		break;
 	case STA_LINK_MODE_LINK_MODIFY:
 		break;
@@ -2418,6 +2437,9 @@ static int sta_link_apply_parameters(struct ieee80211_local *local,
 		ieee80211_s1g_cap_to_sta_s1g_cap(sdata, params->s1g_capa,
 						 link_sta);
 
+	if (params->cip_cap_set)
+		link_sta->pub->cip_cap = params->cip_cap;
+
 	switch (sdata->vif.type) {
 	case NL80211_IFTYPE_NAN:
 	case NL80211_IFTYPE_NAN_DATA:
@@ -2580,6 +2602,9 @@ static int sta_apply_parameters(struct ieee80211_local *local,
 	if (params->eml_cap_present)
 		sta->sta.eml_cap = params->eml_cap;
 
+	if (params->sta_flags_set & BIT(NL80211_STA_FLAG_CIP))
+		sta->sta.cip = true;
+
 	ret = sta_link_apply_parameters(local, sta, STA_LINK_MODE_STA_MODIFY,
 					&params->link_sta_params);
 	if (ret)
@@ -5987,6 +6012,18 @@ ieee80211_set_peer_nan_sched(struct wiphy *wiphy,
 	return ieee80211_nan_set_peer_sched(sdata, sched);
 }
 
+static int
+ieee80211_set_nan_non_evac_channels(struct wiphy *wiphy,
+				    struct wireless_dev *wdev,
+				    struct cfg80211_nan_non_evac_channels *channels)
+{
+	struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev);
+
+	lockdep_assert_wiphy(wiphy);
+
+	return ieee80211_nan_set_non_evac_channels(sdata, channels);
+}
+
 const struct cfg80211_ops mac80211_config_ops = {
 	.add_virtual_intf = ieee80211_add_iface,
 	.del_virtual_intf = ieee80211_del_iface,
@@ -6105,4 +6142,5 @@ const struct cfg80211_ops mac80211_config_ops = {
 	.set_epcs = ieee80211_set_epcs,
 	.nan_set_local_sched = ieee80211_set_local_nan_sched,
 	.nan_set_peer_sched = ieee80211_set_peer_nan_sched,
+	.nan_set_non_evac_channels = ieee80211_set_nan_non_evac_channels,
 };
diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c
index 75bb204ad7432c..9cba795077673c 100644
--- a/net/mac80211/chan.c
+++ b/net/mac80211/chan.c
@@ -231,6 +231,13 @@ int ieee80211_chanctx_refcount(struct ieee80211_local *local,
 	return num;
 }
 
+static bool
+ieee80211_chanctx_has_replace_partner(struct ieee80211_chanctx *ctx)
+{
+	return ctx->replace_state == IEEE80211_CHANCTX_WILL_BE_REPLACED &&
+	       ctx->replace_ctx;
+}
+
 static int ieee80211_num_chanctx(struct ieee80211_local *local, int radio_idx)
 {
 	struct ieee80211_chanctx *ctx;
@@ -1405,6 +1412,7 @@ void ieee80211_link_unreserve_chanctx(struct ieee80211_link_data *link)
 {
 	struct ieee80211_sub_if_data *sdata = link->sdata;
 	struct ieee80211_chanctx *ctx = link->reserved_chanctx;
+	struct ieee80211_chanctx *old_ctx = NULL;
 
 	lockdep_assert_wiphy(sdata->local->hw.wiphy);
 
@@ -1418,6 +1426,7 @@ void ieee80211_link_unreserve_chanctx(struct ieee80211_link_data *link)
 			if (WARN_ON(!ctx->replace_ctx))
 				return;
 
+			old_ctx = ctx->replace_ctx;
 			WARN_ON(ctx->replace_ctx->replace_state !=
 			        IEEE80211_CHANCTX_WILL_BE_REPLACED);
 			WARN_ON(ctx->replace_ctx->replace_ctx != ctx);
@@ -1428,6 +1437,11 @@ void ieee80211_link_unreserve_chanctx(struct ieee80211_link_data *link)
 
 			list_del_rcu(&ctx->list);
 			kfree_rcu(ctx, rcu_head);
+
+			if (ieee80211_chanctx_refcount(sdata->local,
+						       old_ctx) == 0)
+				ieee80211_free_chanctx(sdata->local, old_ctx,
+						       false);
 		} else {
 			ieee80211_free_chanctx(sdata->local, ctx, false);
 		}
@@ -1707,7 +1721,8 @@ ieee80211_link_use_reserved_reassign(struct ieee80211_link_data *link)
 
 	ieee80211_check_fast_xmit_iface(sdata);
 
-	if (ieee80211_chanctx_refcount(local, old_ctx) == 0)
+	if (ieee80211_chanctx_refcount(local, old_ctx) == 0 &&
+	    !ieee80211_chanctx_has_replace_partner(old_ctx))
 		ieee80211_free_chanctx(local, old_ctx, false);
 
 	ieee80211_recalc_chanctx_min_def(local, new_ctx);
@@ -1772,7 +1787,8 @@ ieee80211_link_use_reserved_assign(struct ieee80211_link_data *link)
 }
 
 static bool
-ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link)
+ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link,
+					struct ieee80211_chanctx *ctx)
 {
 	struct ieee80211_sub_if_data *sdata = link->sdata;
 	struct ieee80211_chanctx *old_ctx, *new_ctx;
@@ -1782,6 +1798,9 @@ ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link)
 	new_ctx = link->reserved_chanctx;
 	old_ctx = ieee80211_link_get_chanctx(link);
 
+	if (new_ctx != ctx)
+		return false;
+
 	if (!old_ctx)
 		return false;
 
@@ -1794,6 +1813,12 @@ ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link)
 	if (new_ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
 		return false;
 
+	if (new_ctx->replace_ctx != old_ctx)
+		return false;
+
+	if (old_ctx->replace_ctx != new_ctx)
+		return false;
+
 	return true;
 }
 
@@ -1823,7 +1848,8 @@ static int ieee80211_chsw_switch_vifs(struct ieee80211_local *local,
 		}
 
 		for_each_chanctx_user_reserved(local, ctx, &iter) {
-			if (!ieee80211_link_has_in_place_reservation(iter.link))
+			if (!ieee80211_link_has_in_place_reservation(iter.link,
+								     ctx))
 				continue;
 
 			old_ctx = ieee80211_link_get_chanctx(iter.link);
@@ -1925,7 +1951,9 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local)
 
 		for_each_chanctx_user_assigned(local, ctx->replace_ctx, &iter) {
 			n_assigned++;
-			if (iter.link && iter.link->reserved_chanctx) {
+			if (iter.link &&
+			    ieee80211_link_has_in_place_reservation(iter.link,
+								    ctx)) {
 				n_reserved++;
 				if (iter.link->reserved_ready)
 					n_ready++;
@@ -1950,7 +1978,8 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local)
 use_reserved:
 		ctx->conf.radar_enabled = false;
 		for_each_chanctx_user_reserved(local, ctx, &iter) {
-			if (ieee80211_link_has_in_place_reservation(iter.link) &&
+			if (ieee80211_link_has_in_place_reservation(iter.link,
+								    ctx) &&
 			    !iter.link->reserved_ready)
 				return -EAGAIN;
 
@@ -1991,7 +2020,8 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local)
 		}
 
 		for_each_chanctx_user_reserved(local, ctx, &iter) {
-			if (!ieee80211_link_has_in_place_reservation(iter.link))
+			if (!ieee80211_link_has_in_place_reservation(iter.link,
+								     ctx))
 				continue;
 
 			ieee80211_chan_bw_change(local,
@@ -2040,7 +2070,7 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local)
 			struct ieee80211_bss_conf *link_conf = link->conf;
 			u64 changed = 0;
 
-			if (!ieee80211_link_has_in_place_reservation(link))
+			if (!ieee80211_link_has_in_place_reservation(link, ctx))
 				continue;
 
 			rcu_assign_pointer(link_conf->chanctx_conf,
@@ -2091,7 +2121,8 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local)
 		for_each_chanctx_user_reserved(local, ctx, &iter) {
 			struct ieee80211_link_data *link = iter.link;
 
-			if (WARN_ON(ieee80211_link_has_in_place_reservation(link)))
+			if (WARN_ON(ieee80211_link_has_in_place_reservation(link,
+									    ctx)))
 				continue;
 
 			if (!link->reserved_ready)
@@ -2176,7 +2207,8 @@ void __ieee80211_link_release_channel(struct ieee80211_link_data *link,
 	}
 
 	ieee80211_assign_link_chanctx(link, NULL, false);
-	if (ieee80211_chanctx_refcount(local, ctx) == 0)
+	if (ieee80211_chanctx_refcount(local, ctx) == 0 &&
+	    !ieee80211_chanctx_has_replace_partner(ctx))
 		ieee80211_free_chanctx(local, ctx, skip_idle_recalc);
 
 	link->radar_required = false;
diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c
index e38631d7cfb423..1f914b48e3bbe1 100644
--- a/net/mac80211/debugfs.c
+++ b/net/mac80211/debugfs.c
@@ -96,8 +96,8 @@ static ssize_t aqm_read(struct file *file,
 			"RW fq_quantum %u\n",
 			fq->flows_cnt,
 			fq->backlog,
-			fq->overmemory,
 			fq->overlimit,
+			fq->overmemory,
 			fq->collisions,
 			fq->memory_usage,
 			fq->memory_limit,
diff --git a/net/mac80211/debugfs_key.c b/net/mac80211/debugfs_key.c
index 117f58af5ff9d3..3a4f8e9e54dc16 100644
--- a/net/mac80211/debugfs_key.c
+++ b/net/mac80211/debugfs_key.c
@@ -4,7 +4,7 @@
  * Copyright (c) 2006	Jiri Benc <jbenc@suse.cz>
  * Copyright 2007	Johannes Berg <johannes@sipsolutions.net>
  * Copyright (C) 2015	Intel Deutschland GmbH
- * Copyright (C) 2021-2023   Intel Corporation
+ * Copyright (C) 2021-2023, 2026 Intel Corporation
  */
 
 #include <linux/kobject.h>
@@ -42,9 +42,10 @@ static const struct debugfs_short_fops key_ ##name## _ops = {		\
 		 KEY_READ_##format(name)				\
 		 KEY_OPS(name)
 
-#define KEY_CONF_READ(name, format_string)				\
-	KEY_READ(conf_##name, conf.name, format_string)
-#define KEY_CONF_READ_D(name) KEY_CONF_READ(name, "%d\n")
+#define KEY_CONF_READ(name, val, format_string)				\
+	KEY_READ(conf_##name, conf.val, format_string)
+#define KEY_CONF_READ_D(name, val) KEY_CONF_READ(name, val, "%d\n")
+#define KEY_CONF_READ_X(name, val) KEY_CONF_READ(name, val, "0x%x\n")
 
 #define KEY_CONF_OPS(name)						\
 static const struct debugfs_short_fops key_ ##name## _ops = {		\
@@ -53,13 +54,18 @@ static const struct debugfs_short_fops key_ ##name## _ops = {		\
 }
 
 #define KEY_CONF_FILE(name, format)					\
-		 KEY_CONF_READ_##format(name)				\
+		 KEY_CONF_READ_##format(name, name)			\
+		 KEY_CONF_OPS(name)
+#define KEY_CONF_FILE_NAMED(name, val, format)				\
+		 KEY_CONF_READ_##format(name, val)			\
 		 KEY_CONF_OPS(name)
 
 KEY_CONF_FILE(keylen, D);
 KEY_CONF_FILE(keyidx, D);
 KEY_CONF_FILE(hw_key_idx, D);
+KEY_CONF_FILE_NAMED(conf_flags, flags, X);
 KEY_FILE(flags, X);
+KEY_CONF_FILE(link_id, D);
 KEY_READ(ifindex, sdata->name, "%s\n");
 KEY_OPS(ifindex);
 
@@ -356,6 +362,8 @@ void ieee80211_debugfs_key_add(struct ieee80211_key *key)
 	DEBUGFS_ADD(mic_failures);
 	DEBUGFS_ADD(key);
 	DEBUGFS_ADD(ifindex);
+	DEBUGFS_ADD(conf_flags);
+	DEBUGFS_ADD(link_id);
 };
 
 void ieee80211_debugfs_key_remove(struct ieee80211_key *key)
diff --git a/net/mac80211/debugfs_netdev.c b/net/mac80211/debugfs_netdev.c
index 6aba2249367080..81d18ab8354353 100644
--- a/net/mac80211/debugfs_netdev.c
+++ b/net/mac80211/debugfs_netdev.c
@@ -1033,7 +1033,13 @@ void ieee80211_debugfs_remove_netdev(struct ieee80211_sub_if_data *sdata)
 
 void ieee80211_debugfs_rename_netdev(struct ieee80211_sub_if_data *sdata)
 {
-	debugfs_change_name(sdata->vif.debugfs_dir, "netdev:%s", sdata->name);
+	struct dentry *dir;
+
+	wiphy_lock(sdata->local->hw.wiphy);
+	dir = sdata->vif.debugfs_dir;
+	if (dir)
+		debugfs_change_name(dir, "netdev:%s", sdata->name);
+	wiphy_unlock(sdata->local->hw.wiphy);
 }
 
 void ieee80211_debugfs_recreate_netdev(struct ieee80211_sub_if_data *sdata,
diff --git a/net/mac80211/drop.h b/net/mac80211/drop.h
index f06a8aa905c53c..f433ee17cce514 100644
--- a/net/mac80211/drop.h
+++ b/net/mac80211/drop.h
@@ -108,6 +108,7 @@ typedef unsigned int __bitwise ieee80211_rx_result;
 	R(RX_DROP_U_UNHANDLED_PREQ)		\
 	R(RX_DROP_U_UNHANDLED_MGMT_STYPE)	\
 	R(RX_DROP_U_NO_LINK)			\
+	R(RX_DROP_U_DEFRAG_OVERFLOW)		\
 /* this line for the trailing \ - add before this */
 
 /* having two enums allows for checking ieee80211_rx_result use with sparse */
diff --git a/net/mac80211/fils_aead.c b/net/mac80211/fils_aead.c
index 293590976489a0..9c4f00d61981d2 100644
--- a/net/mac80211/fils_aead.c
+++ b/net/mac80211/fils_aead.c
@@ -89,6 +89,14 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len,
 	if (res)
 		return res;
 
+	/* With an empty plaintext there is no ciphertext to produce and
+	 * the S2V result is the complete output.
+	 */
+	if (!plain_len) {
+		memcpy(out, v, AES_BLOCK_SIZE);
+		return 0;
+	}
+
 	/* Use a temporary buffer of the plaintext to handle need for
 	 * overwriting this during AES-CTR.
 	 */
diff --git a/net/mac80211/ht.c b/net/mac80211/ht.c
index e1e1b7f82f3fea..c5c2113fa4f79a 100644
--- a/net/mac80211/ht.c
+++ b/net/mac80211/ht.c
@@ -545,7 +545,7 @@ int ieee80211_send_smps_action(struct ieee80211_sub_if_data *sdata,
 	info->status_data = IEEE80211_STATUS_TYPE_SMPS |
 			    u16_encode_bits(status_link_id << 2 | smps,
 					    IEEE80211_STATUS_SUBDATA_MASK);
-	ieee80211_tx_skb_tid(sdata, skb, 7, link_id);
+	ieee80211_tx_skb_tid(sdata, skb, NULL, 7, link_id);
 
 	return 0;
 }
diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
index d05f594673990a..1430527d216cc9 100644
--- a/net/mac80211/ieee80211_i.h
+++ b/net/mac80211/ieee80211_i.h
@@ -191,6 +191,15 @@ struct ieee80211_tx_data {
 	unsigned int flags;
 };
 
+static inline bool ieee80211_is_tx_data(struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+
+	return info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP ||
+	       ieee80211_is_data(hdr->frame_control);
+}
+
 /**
  * enum ieee80211_packet_rx_flags - packet RX flags
  * @IEEE80211_RX_AMSDU: a-MSDU packet
@@ -477,6 +486,7 @@ struct ieee80211_mgd_assoc_data {
 	bool comeback; /* whether the AP has requested association comeback */
 	bool s1g;
 	bool spp_amsdu;
+	bool cip;
 
 	s8 assoc_link_id;
 
@@ -1104,6 +1114,7 @@ struct ieee80211_link_data {
 	struct ieee80211_key __rcu *gtk[NUM_DEFAULT_KEYS +
 					NUM_DEFAULT_MGMT_KEYS +
 					NUM_DEFAULT_BEACON_KEYS];
+	struct ieee80211_key __rcu *cigtk[NUM_CTRL_KEYS];
 	struct ieee80211_key __rcu *default_multicast_key;
 	struct ieee80211_key __rcu *default_mgmt_key;
 	struct ieee80211_key __rcu *default_beacon_key;
@@ -1459,6 +1470,7 @@ struct ieee80211_local {
 	spinlock_t active_txq_lock[IEEE80211_NUM_ACS];
 	struct list_head active_txqs[IEEE80211_NUM_ACS];
 	u16 schedule_round[IEEE80211_NUM_ACS];
+	bool schedule_open[IEEE80211_NUM_ACS];
 
 	/* serializes ieee80211_handle_wake_tx_queue */
 	spinlock_t handle_wake_tx_queue_lock;
@@ -1868,6 +1880,7 @@ struct ieee802_11_elems {
 	const struct ieee80211_ttlm_elem *ttlm[IEEE80211_TTLM_MAX_CNT];
 	const struct ieee80211_uhr_cap *uhr_cap;
 	const struct ieee80211_uhr_operation *uhr_operation;
+	const struct ieee80211_cip_cap *cip_cap;
 
 	/* not the order in the psd values is per element, not per chandef */
 	struct ieee80211_parsed_tpe tpe;
@@ -2078,6 +2091,8 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata,
 				  struct cfg80211_nan_local_sched *sched);
 int ieee80211_nan_set_peer_sched(struct ieee80211_sub_if_data *sdata,
 				 struct cfg80211_nan_peer_sched *sched);
+int ieee80211_nan_set_non_evac_channels(struct ieee80211_sub_if_data *sdata,
+					struct cfg80211_nan_non_evac_channels *channels);
 void ieee80211_nan_free_peer_sched(struct ieee80211_nan_peer_sched *sched);
 void ieee80211_nan_update_ndi_carrier(struct ieee80211_sub_if_data *ndi_sdata);
 struct ieee80211_nan_channel *
@@ -2233,8 +2248,7 @@ netdev_tx_t ieee80211_subif_start_xmit_8023(struct sk_buff *skb,
 void __ieee80211_subif_start_xmit(struct sk_buff *skb,
 				  struct net_device *dev,
 				  u32 info_flags,
-				  u32 ctrl_flags,
-				  u64 cookie);
+				  u32 ctrl_flags);
 struct sk_buff *
 ieee80211_build_data_template(struct ieee80211_sub_if_data *sdata,
 			      struct sk_buff *skb, u32 info_flags);
@@ -2388,7 +2402,6 @@ ieee80211_he_op_ie_to_bss_conf(struct ieee80211_vif *vif,
 			const struct ieee80211_he_operation *he_op_ie_elem);
 
 /* S1G */
-void ieee80211_s1g_sta_rate_init(struct sta_info *sta);
 bool ieee80211_s1g_is_twt_setup(struct sk_buff *skb);
 void ieee80211_s1g_rx_twt_action(struct ieee80211_sub_if_data *sdata,
 				 struct sk_buff *skb);
@@ -2463,7 +2476,8 @@ void ieee80211_xmit(struct ieee80211_sub_if_data *sdata,
 		    struct sta_info *sta, struct sk_buff *skb);
 
 void __ieee80211_tx_skb_tid_band(struct ieee80211_sub_if_data *sdata,
-				 struct sk_buff *skb, int tid, int link_id,
+				 struct sk_buff *skb, struct sta_info *sta,
+				 int tid, int link_id,
 				 enum nl80211_band band);
 
 static inline bool ieee80211_require_encrypted_assoc(__le16 fc,
@@ -2472,29 +2486,26 @@ static inline bool ieee80211_require_encrypted_assoc(__le16 fc,
 	return sta && sta->sta.epp_peer && ieee80211_is_assoc(fc);
 }
 
-/* sta_out needs to be checked for ERR_PTR() before using */
-int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata,
-			    struct sk_buff *skb,
-			    struct sta_info **sta_out);
-
 static inline void
 ieee80211_tx_skb_tid_band(struct ieee80211_sub_if_data *sdata,
 			  struct sk_buff *skb, int tid,
 			  enum nl80211_band band)
 {
 	rcu_read_lock();
-	__ieee80211_tx_skb_tid_band(sdata, skb, tid, -1, band);
+	__ieee80211_tx_skb_tid_band(sdata, skb, ERR_PTR(-ENOENT),
+				    tid, -1, band);
 	rcu_read_unlock();
 }
 
 void ieee80211_tx_skb_tid(struct ieee80211_sub_if_data *sdata,
-			  struct sk_buff *skb, int tid, int link_id);
+			  struct sk_buff *skb, struct sta_info *sta,
+			  int tid, int link_id);
 
 static inline void ieee80211_tx_skb(struct ieee80211_sub_if_data *sdata,
 				    struct sk_buff *skb)
 {
 	/* Send all internal mgmt frames on VO. Accordingly set TID to 7. */
-	ieee80211_tx_skb_tid(sdata, skb, 7, -1);
+	ieee80211_tx_skb_tid(sdata, skb, NULL, 7, -1);
 }
 
 /**
@@ -2772,6 +2783,8 @@ int ieee80211_put_uhr_cap(struct sk_buff *skb,
 			  const struct ieee80211_supported_band *sband);
 void ieee80211_put_reg_conn(struct ieee80211_sub_if_data *sdata,
 			    struct sk_buff *skb);
+int ieee80211_put_cip_cap(struct sk_buff *skb,
+			  struct ieee80211_sub_if_data *sdata);
 
 /* channel management */
 bool ieee80211_chandef_ht_oper(const struct ieee80211_ht_operation *ht_oper,
diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c
index 889c32fd8de19f..172bb53788e2d2 100644
--- a/net/mac80211/iface.c
+++ b/net/mac80211/iface.c
@@ -632,8 +632,9 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do
 				hw_reconf_flags |= IEEE80211_CONF_CHANGE_MONITOR;
 			}
 
-			ieee80211_adjust_monitor_flags(sdata, -1);
 		}
+
+		ieee80211_adjust_monitor_flags(sdata, -1);
 		break;
 	case NL80211_IFTYPE_NAN:
 		/* Check if any open NAN_DATA interfaces */
@@ -993,13 +994,13 @@ static u16 ieee80211_monitor_select_queue(struct net_device *dev,
 	/* reset flags and info before parsing radiotap header */
 	memset(info, 0, sizeof(*info));
 
-	if (!ieee80211_parse_tx_radiotap(skb, dev, NULL))
+	if (!ieee80211_parse_tx_radiotap(skb, dev, NULL, false))
 		return 0; /* doesn't matter, frame will be dropped */
 
 	len_rthdr = ieee80211_get_radiotap_len(skb->data);
 	hdr = (struct ieee80211_hdr *)(skb->data + len_rthdr);
-	if (skb->len < len_rthdr + 2 ||
-	    skb->len < len_rthdr + ieee80211_hdrlen(hdr->frame_control))
+	if (skb_headlen(skb) < len_rthdr + 2 ||
+	    skb_headlen(skb) < len_rthdr + ieee80211_hdrlen(hdr->frame_control))
 		return 0; /* doesn't matter, frame will be dropped */
 
 	return ieee80211_select_queue_80211(sdata, skb, hdr);
@@ -1706,11 +1707,8 @@ static void ieee80211_iface_process_skb(struct ieee80211_local *local,
 				break;
 
 			status = IEEE80211_SKB_RXCB(skb);
-			if (!status->link_valid)
-				link_sta = &sta->deflink;
-			else
-				link_sta = rcu_dereference_protected(sta->link[status->link_id],
-							lockdep_is_held(&local->hw.wiphy->mtx));
+			link_sta = wiphy_dereference(local->hw.wiphy,
+						     sta->link[status->link_id]);
 			if (link_sta)
 				ieee80211_ht_handle_chanwidth_notif(local, sdata, sta,
 								    link_sta, chanwidth,
diff --git a/net/mac80211/key.c b/net/mac80211/key.c
index f45e792abede57..4c9788ec0b74f8 100644
--- a/net/mac80211/key.c
+++ b/net/mac80211/key.c
@@ -205,6 +205,10 @@ static int ieee80211_key_enable_hw_accel(struct ieee80211_key *key)
 			  sta ? sta->sta.addr : bcast_addr, ret);
 
  out_unsupported:
+	/* Control Integrity Protocol can only be done in hardware */
+	if (key->conf.flags & IEEE80211_KEY_FLAG_CIP)
+		return -EINVAL;
+
 	switch (key->conf.cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
 	case WLAN_CIPHER_SUITE_WEP104:
@@ -440,11 +444,13 @@ void ieee80211_set_default_beacon_key(struct ieee80211_link_data *link,
 static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
 				 struct ieee80211_link_data *link,
 				 struct sta_info *sta,
-				 bool pairwise,
+				 enum ieee80211_key_flags flags,
 				 struct ieee80211_key *old,
 				 struct ieee80211_key *new)
 {
 	struct link_sta_info *link_sta = sta ? &sta->deflink : NULL;
+	bool pairwise = flags & IEEE80211_KEY_FLAG_PAIRWISE;
+	bool cip = flags & IEEE80211_KEY_FLAG_CIP;
 	int link_id;
 	int idx;
 	int ret = 0;
@@ -535,6 +541,8 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
 			if (new &&
 			    !(new->conf.flags & IEEE80211_KEY_FLAG_NO_AUTO_TX))
 				_ieee80211_set_tx_key(new, true);
+		} else if (cip) {
+			rcu_assign_pointer(link_sta->cigtk[idx], new);
 		} else {
 			rcu_assign_pointer(link_sta->gtk[idx], new);
 		}
@@ -569,6 +577,8 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
 
 		if (is_wep || pairwise)
 			rcu_assign_pointer(sdata->keys[idx], new);
+		else if (cip)
+			rcu_assign_pointer(link->cigtk[idx], new);
 		else
 			rcu_assign_pointer(link->gtk[idx], new);
 
@@ -883,6 +893,10 @@ int ieee80211_key_link(struct ieee80211_key *key,
 			ret = -EOPNOTSUPP;
 			goto out;
 		}
+
+		/* Set CIP flag if enabled for the station */
+		if (sta->sta.cip)
+			key->conf.flags |= IEEE80211_KEY_FLAG_CIP;
 	} else if (sta) {
 		struct link_sta_info *link_sta = &sta->deflink;
 		int link_id = key->conf.link_id;
@@ -896,15 +910,25 @@ int ieee80211_key_link(struct ieee80211_key *key,
 			}
 		}
 
-		old_key = wiphy_dereference(sdata->local->hw.wiphy,
-					    link_sta->gtk[idx]);
-	} else {
-		if (idx < NUM_DEFAULT_KEYS)
+		if (key->conf.flags & IEEE80211_KEY_FLAG_CIP)
 			old_key = wiphy_dereference(sdata->local->hw.wiphy,
-						    sdata->keys[idx]);
-		if (!old_key)
+						    link_sta->cigtk[idx]);
+		else
 			old_key = wiphy_dereference(sdata->local->hw.wiphy,
-						    link->gtk[idx]);
+						    link_sta->gtk[idx]);
+	} else {
+		if (key->conf.flags & IEEE80211_KEY_FLAG_CIP) {
+			old_key = wiphy_dereference(sdata->local->hw.wiphy,
+						    link->cigtk[idx]);
+		} else {
+			if (idx < NUM_DEFAULT_KEYS)
+				old_key = wiphy_dereference(sdata->local->hw.wiphy,
+							    sdata->keys[idx]);
+
+			if (!old_key)
+				old_key = wiphy_dereference(sdata->local->hw.wiphy,
+							    link->gtk[idx]);
+		}
 	}
 
 	/* Non-pairwise keys must also not switch the cipher on rekey */
@@ -938,9 +962,17 @@ int ieee80211_key_link(struct ieee80211_key *key,
 	if (sta && sta->sta.spp_amsdu)
 		key->conf.flags |= IEEE80211_KEY_FLAG_SPP_AMSDU;
 
+	/* A CIP related key must be GCMP-256 (really GMAC-256) */
+	if ((key->conf.flags & IEEE80211_KEY_FLAG_CIP) &&
+	    key->conf.cipher != WLAN_CIPHER_SUITE_GCMP_256) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	increment_tailroom_need_count(sdata);
 
-	ret = ieee80211_key_replace(sdata, link, sta, pairwise, old_key, key);
+	ret = ieee80211_key_replace(sdata, link, sta, key->conf.flags,
+				    old_key, key);
 
 	if (!ret) {
 		ieee80211_debugfs_key_add(key);
@@ -966,8 +998,7 @@ void ieee80211_key_free(struct ieee80211_key *key, bool delay_tailroom)
 	 */
 	if (key->sdata)
 		ieee80211_key_replace(key->sdata, NULL, key->sta,
-				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				      key, NULL);
+				      key->conf.flags, key, NULL);
 	ieee80211_key_destroy(key, delay_tailroom);
 }
 
@@ -1098,8 +1129,7 @@ static void ieee80211_free_keys_iface(struct ieee80211_sub_if_data *sdata,
 
 	list_for_each_entry_safe(key, tmp, &sdata->key_list, list) {
 		ieee80211_key_replace(key->sdata, NULL, key->sta,
-				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				      key, NULL);
+				      key->conf.flags, key, NULL);
 		list_add_tail(&key->list, keys);
 	}
 
@@ -1119,8 +1149,7 @@ void ieee80211_remove_link_keys(struct ieee80211_link_data *link,
 		if (key->conf.link_id != link->link_id)
 			continue;
 		ieee80211_key_replace(key->sdata, link, key->sta,
-				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				      key, NULL);
+				      key->conf.flags, key, NULL);
 		list_add_tail(&key->list, keys);
 	}
 }
@@ -1196,8 +1225,17 @@ void ieee80211_free_sta_keys(struct ieee80211_local *local,
 		if (!key)
 			continue;
 		ieee80211_key_replace(key->sdata, NULL, key->sta,
-				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				      key, NULL);
+				      key->conf.flags, key, NULL);
+		__ieee80211_key_destroy(key, key->sdata->vif.type ==
+					NL80211_IFTYPE_STATION);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sta->deflink.cigtk); i++) {
+		key = wiphy_dereference(local->hw.wiphy, sta->deflink.cigtk[i]);
+		if (!key)
+			continue;
+		ieee80211_key_replace(key->sdata, NULL, key->sta,
+				      key->conf.flags, key, NULL);
 		__ieee80211_key_destroy(key, key->sdata->vif.type ==
 					NL80211_IFTYPE_STATION);
 	}
@@ -1207,8 +1245,7 @@ void ieee80211_free_sta_keys(struct ieee80211_local *local,
 		if (!key)
 			continue;
 		ieee80211_key_replace(key->sdata, NULL, key->sta,
-				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				      key, NULL);
+				      key->conf.flags, key, NULL);
 		__ieee80211_key_destroy(key, key->sdata->vif.type ==
 					NL80211_IFTYPE_STATION);
 	}
@@ -1366,7 +1403,7 @@ EXPORT_SYMBOL_GPL(ieee80211_set_key_rx_seq);
 struct ieee80211_key_conf *
 ieee80211_gtk_rekey_add(struct ieee80211_vif *vif,
 			u8 idx, u8 *key_data, u8 key_len,
-			int link_id)
+			int link_id, bool cigtk)
 {
 	struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
 	struct ieee80211_local *local = sdata->local;
@@ -1390,10 +1427,20 @@ ieee80211_gtk_rekey_add(struct ieee80211_vif *vif,
 		    NUM_DEFAULT_BEACON_KEYS))
 		return ERR_PTR(-EINVAL);
 
-	prev_key = wiphy_dereference(local->hw.wiphy,
-				     link_data->gtk[idx]);
+	if (WARN_ON(cigtk && idx >= NUM_CTRL_KEYS))
+		return ERR_PTR(-EINVAL);
+
+	if (cigtk)
+		prev_key = wiphy_dereference(local->hw.wiphy,
+					     link_data->cigtk[idx]);
+	else
+		prev_key = wiphy_dereference(local->hw.wiphy,
+					     link_data->gtk[idx]);
 	if (!prev_key) {
-		if (idx < NUM_DEFAULT_KEYS) {
+		if (cigtk) {
+			prev_key = wiphy_dereference(local->hw.wiphy,
+						     link_data->cigtk[idx ^ 1]);
+		} else if (idx < NUM_DEFAULT_KEYS) {
 			for (int i = 0; i < NUM_DEFAULT_KEYS; i++) {
 				if (i == idx)
 					continue;
@@ -1424,6 +1471,9 @@ ieee80211_gtk_rekey_add(struct ieee80211_vif *vif,
 	if (sdata->u.mgd.mfp != IEEE80211_MFP_DISABLED)
 		key->conf.flags |= IEEE80211_KEY_FLAG_RX_MGMT;
 
+	if (prev_key->conf.flags & IEEE80211_KEY_FLAG_CIP)
+		key->conf.flags |= IEEE80211_KEY_FLAG_CIP;
+
 	key->conf.link_id = link_data->link_id;
 
 	err = ieee80211_key_link(key, link_data, NULL);
diff --git a/net/mac80211/key.h b/net/mac80211/key.h
index 826e4e9387c55f..c85992493afd1c 100644
--- a/net/mac80211/key.h
+++ b/net/mac80211/key.h
@@ -2,7 +2,7 @@
 /*
  * Copyright 2002-2004, Instant802 Networks, Inc.
  * Copyright 2005, Devicescape Software, Inc.
- * Copyright (C) 2019, 2022-2023 Intel Corporation
+ * Copyright (C) 2019, 2022-2023, 2026 Intel Corporation
  */
 
 #ifndef IEEE80211_KEY_H
@@ -19,6 +19,7 @@
 #define NUM_DEFAULT_KEYS 4
 #define NUM_DEFAULT_MGMT_KEYS 2
 #define NUM_DEFAULT_BEACON_KEYS 2
+#define NUM_CTRL_KEYS 2
 #define INVALID_PTK_KEYIDX 2 /* Keyidx always pointing to a NULL key for PTK */
 
 struct ieee80211_local;
diff --git a/net/mac80211/main.c b/net/mac80211/main.c
index 6408e8464338dc..7816395681fa4a 100644
--- a/net/mac80211/main.c
+++ b/net/mac80211/main.c
@@ -1322,12 +1322,13 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
 					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G;
 
 			/* currently no support for HE client where HT has 40 MHz but not HT */
-			if (iftd->he_cap.has_he &&
-			    iftd->types_mask & (BIT(NL80211_IFTYPE_STATION) |
-						BIT(NL80211_IFTYPE_P2P_CLIENT)) &&
-			    sband->ht_cap.ht_supported &&
-			    sband->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 &&
-			    !(iftd->he_cap.he_cap_elem.phy_cap_info[0] & he_40_mhz_cap))
+			if (WARN_ON(iftd->he_cap.has_he &&
+				    iftd->types_mask & (BIT(NL80211_IFTYPE_STATION) |
+							BIT(NL80211_IFTYPE_P2P_CLIENT)) &&
+				    sband->ht_cap.ht_supported &&
+				    sband->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 &&
+				    !(iftd->he_cap.he_cap_elem.phy_cap_info[0] &
+				      he_40_mhz_cap)))
 				return -EINVAL;
 
 			/* no support for per-band vendor elems with MLO */
diff --git a/net/mac80211/mesh_pathtbl.c b/net/mac80211/mesh_pathtbl.c
index 03171cf008557f..7b2f71186dfbd6 100644
--- a/net/mac80211/mesh_pathtbl.c
+++ b/net/mac80211/mesh_pathtbl.c
@@ -247,9 +247,10 @@ static struct mesh_path *mpath_lookup(struct mesh_table *tbl, const u8 *dst,
 
 	mpath = rhashtable_lookup(&tbl->rhead, dst, mesh_rht_params);
 
-	if (mpath && mpath_expired(mpath)) {
+	if (mpath) {
 		spin_lock_bh(&mpath->state_lock);
-		mpath->flags &= ~MESH_PATH_ACTIVE;
+		if (mpath_expired(mpath))
+			mpath->flags &= ~MESH_PATH_ACTIVE;
 		spin_unlock_bh(&mpath->state_lock);
 	}
 	return mpath;
@@ -290,11 +291,11 @@ __mesh_path_lookup_by_idx(struct mesh_table *tbl, int idx)
 	if (!mpath)
 		return NULL;
 
-	if (mpath_expired(mpath)) {
-		spin_lock_bh(&mpath->state_lock);
+	spin_lock_bh(&mpath->state_lock);
+	if (mpath_expired(mpath))
 		mpath->flags &= ~MESH_PATH_ACTIVE;
-		spin_unlock_bh(&mpath->state_lock);
-	}
+	spin_unlock_bh(&mpath->state_lock);
+
 	return mpath;
 }
 
@@ -577,6 +578,12 @@ void mesh_fast_tx_cache(struct ieee80211_sub_if_data *sdata,
 		goto unlock_sta;
 
 	spin_lock(&cache->walk_lock);
+	if ((READ_ONCE(mpath->flags) & MESH_PATH_DELETED) ||
+	    (mppath && (READ_ONCE(mppath->flags) & MESH_PATH_DELETED))) {
+		kfree(entry);
+		goto unlock_cache;
+	}
+
 	prev = rhashtable_lookup_get_insert_fast(&cache->rht,
 						 &entry->rhash,
 						 fast_tx_rht_params);
@@ -673,7 +680,7 @@ void mesh_fast_tx_flush_addr(struct ieee80211_sub_if_data *sdata,
  * @sdata: local subif
  * @dst: destination address of the path (ETH_ALEN length)
  *
- * Returns: 0 on success
+ * Returns: the new or the existing path on success, an ERR_PTR() on failure
  *
  * State: the initial state of the new path is set to 0
  */
@@ -694,8 +701,10 @@ struct mesh_path *mesh_path_add(struct ieee80211_sub_if_data *sdata,
 		return ERR_PTR(-ENOSPC);
 
 	new_mpath = mesh_path_new(sdata, dst, GFP_ATOMIC);
-	if (!new_mpath)
+	if (!new_mpath) {
+		atomic_dec(&sdata->u.mesh.mpaths);
 		return ERR_PTR(-ENOMEM);
+	}
 
 	tbl = &sdata->u.mesh.mesh_paths;
 	spin_lock_bh(&tbl->walk_lock);
@@ -708,6 +717,7 @@ struct mesh_path *mesh_path_add(struct ieee80211_sub_if_data *sdata,
 
 	if (mpath) {
 		kfree(new_mpath);
+		atomic_dec(&sdata->u.mesh.mpaths);
 
 		if (IS_ERR(mpath))
 			return mpath;
@@ -733,10 +743,15 @@ int mpp_path_add(struct ieee80211_sub_if_data *sdata,
 	if (is_multicast_ether_addr(dst))
 		return -EOPNOTSUPP;
 
+	if (!atomic_add_unless(&sdata->u.mesh.mpaths, 1, MESH_MAX_MPATHS))
+		return -ENOSPC;
+
 	new_mpath = mesh_path_new(sdata, dst, GFP_ATOMIC);
 
-	if (!new_mpath)
+	if (!new_mpath) {
+		atomic_dec(&sdata->u.mesh.mpaths);
 		return -ENOMEM;
+	}
 
 	memcpy(new_mpath->mpp, mpp, ETH_ALEN);
 	tbl = &sdata->u.mesh.mpp_paths;
@@ -749,10 +764,12 @@ int mpp_path_add(struct ieee80211_sub_if_data *sdata,
 		hlist_add_head_rcu(&new_mpath->walk_list, &tbl->walk_head);
 	spin_unlock_bh(&tbl->walk_lock);
 
-	if (ret)
+	if (ret) {
 		kfree(new_mpath);
-	else
+		atomic_dec(&sdata->u.mesh.mpaths);
+	} else {
 		mesh_fast_tx_flush_addr(sdata, dst);
+	}
 
 	sdata->u.mesh.mpp_paths_generation++;
 	return ret;
@@ -798,7 +815,8 @@ static void mesh_path_free_rcu(struct mesh_table *tbl,
 	struct ieee80211_sub_if_data *sdata = mpath->sdata;
 
 	spin_lock_bh(&mpath->state_lock);
-	mpath->flags |= MESH_PATH_RESOLVING | MESH_PATH_DELETED;
+	WRITE_ONCE(mpath->flags,
+		   mpath->flags | MESH_PATH_RESOLVING | MESH_PATH_DELETED);
 	mesh_gate_del(tbl, mpath);
 	spin_unlock_bh(&mpath->state_lock);
 	timer_shutdown_sync(&mpath->timer);
@@ -812,6 +830,9 @@ static void __mesh_path_del(struct mesh_table *tbl, struct mesh_path *mpath)
 {
 	hlist_del_rcu(&mpath->walk_list);
 	rhashtable_remove_fast(&tbl->rhead, &mpath->rhash, mesh_rht_params);
+	spin_lock_bh(&mpath->state_lock);
+	WRITE_ONCE(mpath->flags, mpath->flags | MESH_PATH_DELETED);
+	spin_unlock_bh(&mpath->state_lock);
 	if (tbl == &mpath->sdata->u.mesh.mpp_paths)
 		mesh_fast_tx_flush_addr(mpath->sdata, mpath->dst);
 	else
diff --git a/net/mac80211/mesh_plink.c b/net/mac80211/mesh_plink.c
index 268857467f2928..6ce3ae3fb39bd7 100644
--- a/net/mac80211/mesh_plink.c
+++ b/net/mac80211/mesh_plink.c
@@ -399,11 +399,13 @@ u64 mesh_plink_deactivate(struct sta_info *sta)
 {
 	struct ieee80211_sub_if_data *sdata = sta->sdata;
 	u64 changed;
+	bool send_close;
 
 	spin_lock_bh(&sta->mesh->plink_lock);
+	send_close = sta->mesh->plink_state != NL80211_PLINK_LISTEN;
 	changed = __mesh_plink_deactivate(sta);
 
-	if (!sdata->u.mesh.user_mpm) {
+	if (!sdata->u.mesh.user_mpm && send_close) {
 		sta->mesh->reason = WLAN_REASON_MESH_PEER_CANCELED;
 		mesh_plink_frame_tx(sdata, sta, WLAN_SP_MESH_PEERING_CLOSE,
 				    sta->sta.addr, sta->mesh->llid,
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index f51167f0fc46d3..5fee0e9ca7b4b9 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -2337,6 +2337,10 @@ ieee80211_add_link_elems(struct ieee80211_sub_if_data *sdata,
 		ieee80211_put_eht_cap(skb, sdata, sband,
 				      &assoc_data->link[link_id].conn);
 
+	/* Insert CIP only on the assoc link (it will be inherited) */
+	if (link_id == assoc_data->assoc_link_id && assoc_data->cip)
+		ieee80211_put_cip_cap(skb, sdata);
+
 	if (assoc_data->link[link_id].conn.mode >= IEEE80211_CONN_MODE_UHR)
 		ieee80211_put_uhr_cap(skb, sdata, sband);
 
@@ -2613,7 +2617,8 @@ static int ieee80211_send_assoc(struct ieee80211_sub_if_data *sdata)
 	       assoc_data->ie_len + /* extra IEs */
 	       (assoc_data->fils_kek_len ? 16 /* AES-SIV */ : 0) +
 	       9 /* WMM */ +
-	       4 /* regulatory connectivity, if 6 GHz is supported */;
+	       4 /* regulatory connectivity, if 6 GHz is supported */ +
+	       (assoc_data->cip ? 4 /* CIP capabilities */ : 0);
 
 	for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) {
 		struct cfg80211_bss *cbss = assoc_data->link[link_id].bss;
@@ -5877,6 +5882,7 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 	bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ;
 	bool is_s1g = cbss->channel->band == NL80211_BAND_S1GHZ;
 	const struct cfg80211_bss_ies *bss_ies = NULL;
+	struct ieee802_11_elems *bss_elems = NULL;
 	struct ieee80211_supported_band *sband;
 	struct ieee802_11_elems *elems;
 	u16 capab_info;
@@ -6007,7 +6013,6 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 	     (is_5ghz && link->u.mgd.conn.mode >= IEEE80211_CONN_MODE_VHT &&
 	      (!elems->vht_cap_elem || !elems->vht_operation)))) {
 		const struct cfg80211_bss_ies *ies;
-		struct ieee802_11_elems *bss_elems;
 
 		rcu_read_lock();
 		ies = rcu_dereference(cbss->ies);
@@ -6069,7 +6074,6 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 					   "AP bug: VHT operation missing from AssocResp\n");
 			}
 		}
-		kfree(bss_elems);
 	}
 
 	/*
@@ -6274,6 +6278,17 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 		/* TODO: OPEN: what happens if BSS color disable is set? */
 	}
 
+	if (assoc_data->cip) {
+		if (elems->cip_cap) {
+			link_sta->pub->cip_cap = elems->cip_cap->v;
+		} else {
+			sdata_info(sdata,
+				   "CIP Capabilities not included in association response\n");
+			ret = false;
+			goto out;
+		}
+	}
+
 	if (cbss->transmitted_bss) {
 		bss_conf->nontransmitted = true;
 		ether_addr_copy(bss_conf->transmitter_bssid,
@@ -6342,12 +6357,12 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 	ret = true;
 out:
 	kfree(elems);
+	kfree(bss_elems);
 	kfree(bss_ies);
 	return ret;
 }
 
 static int ieee80211_mgd_setup_link_sta(struct ieee80211_link_data *link,
-					struct sta_info *sta,
 					struct link_sta_info *link_sta,
 					struct cfg80211_bss *cbss)
 {
@@ -6362,11 +6377,9 @@ static int ieee80211_mgd_setup_link_sta(struct ieee80211_link_data *link,
 	memcpy(link_sta->addr, cbss->bssid, ETH_ALEN);
 	memcpy(link_sta->pub->addr, cbss->bssid, ETH_ALEN);
 
-	/* TODO: S1G Basic Rate Set is expressed elsewhere */
-	if (cbss->channel->band == NL80211_BAND_S1GHZ) {
-		ieee80211_s1g_sta_rate_init(sta);
+	/* S1G does not use basic rates */
+	if (cbss->channel->band == NL80211_BAND_S1GHZ)
 		return 0;
-	}
 
 	sband = local->hw.wiphy->bands[cbss->channel->band];
 
@@ -6516,7 +6529,7 @@ ieee80211_determine_our_sta_mode(struct ieee80211_sub_if_data *sdata,
 	struct ieee80211_sta_ht_cap sta_ht_cap = sband->ht_cap;
 	bool is_5ghz = sband->band == NL80211_BAND_5GHZ;
 	bool is_6ghz = sband->band == NL80211_BAND_6GHZ;
-	const struct ieee80211_sta_he_cap *he_cap;
+	const struct ieee80211_sta_he_cap *he_cap = NULL;
 	const struct ieee80211_sta_eht_cap *eht_cap;
 	const struct ieee80211_sta_uhr_cap *uhr_cap;
 	struct ieee80211_sta_vht_cap vht_cap;
@@ -6580,7 +6593,15 @@ ieee80211_determine_our_sta_mode(struct ieee80211_sub_if_data *sdata,
 		goto out;
 	}
 
-	if (vht_cap.vht_supported && is_5ghz) {
+	if (req && req->flags & ASSOC_REQ_DISABLE_HE && !is_6ghz)
+		mlme_link_id_dbg(sdata, link_id,
+				 "HE disabled by flag, limiting to HT/VHT\n");
+	else
+		he_cap = ieee80211_get_he_iftype_cap_vif(sband, &sdata->vif);
+
+	if (vht_cap.vht_supported && is_5ghz && he_cap) {
+		/* nothing - since HE we can be 20 MHz-only non-AP STA */
+	} else if (vht_cap.vht_supported && is_5ghz) {
 		bool have_80mhz = false;
 		unsigned int i;
 
@@ -6626,17 +6647,11 @@ ieee80211_determine_our_sta_mode(struct ieee80211_sub_if_data *sdata,
 				 "no VHT 160 MHz capability on 5 GHz, limiting to 80 MHz");
 	}
 
-	if (req && req->flags & ASSOC_REQ_DISABLE_HE) {
-		mlme_link_id_dbg(sdata, link_id,
-				 "HE disabled by flag, limiting to HT/VHT\n");
-		goto out;
-	}
-
-	he_cap = ieee80211_get_he_iftype_cap_vif(sband, &sdata->vif);
 	if (!he_cap) {
 		WARN_ON(is_6ghz);
-		mlme_link_id_dbg(sdata, link_id,
-				 "no HE support, limiting to HT/VHT\n");
+		if (!req || !(req->flags & ASSOC_REQ_DISABLE_HE))
+			mlme_link_id_dbg(sdata, link_id,
+					 "no HE support, limiting to HT/VHT\n");
 		goto out;
 	}
 
@@ -7000,6 +7015,7 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata,
 		goto out_err;
 
 	sta->sta.spp_amsdu = assoc_data->spp_amsdu;
+	sta->sta.cip = assoc_data->cip;
 
 	if (ieee80211_vif_is_mld(&sdata->vif)) {
 		if (!elems->ml_basic)
@@ -7107,7 +7123,7 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata,
 			}
 		}
 
-		err = ieee80211_mgd_setup_link_sta(link, sta, link_sta,
+		err = ieee80211_mgd_setup_link_sta(link, link_sta,
 						   assoc_data->link[link_id].bss);
 		if (err)
 			goto out_err;
@@ -9094,6 +9110,57 @@ void ieee80211_mgd_conn_tx_status(struct ieee80211_sub_if_data *sdata,
 	wiphy_work_queue(local->hw.wiphy, &sdata->work);
 }
 
+static void
+ieee80211_assoc_timeout_teardown(struct ieee80211_sub_if_data *sdata)
+{
+	struct ieee80211_mgd_assoc_data *assoc_data = sdata->u.mgd.assoc_data;
+	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_event event = {
+		.type = MLME_EVENT,
+		.u.mlme.data = ASSOC_EVENT,
+		.u.mlme.status = MLME_TIMEOUT,
+	};
+	struct sta_info *sta;
+
+	lockdep_assert_wiphy(local->hw.wiphy);
+
+	/*
+	 * With an EPP station, the AP is already maintaining a state for the
+	 * station. Send a deauthentication frame, so that the AP clears its
+	 * state for this station (to allow additional connection attempts).
+	 * Note that this needs to be done before the station is removed
+	 * locally as the deauthentication frame needs to be sent encrypted.
+	 */
+	sta = sta_info_get_bss(sdata, assoc_data->ap_addr);
+	if (sta && sta->sta.epp_peer &&
+	    wiphy_dereference(local->hw.wiphy, sta->ptk[sta->ptk_idx])) {
+		u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];
+		struct ieee80211_prep_tx_info info = {
+			.subtype = IEEE80211_STYPE_DEAUTH,
+			.link_id = assoc_data->assoc_link_id,
+		};
+
+		drv_mgd_prepare_tx(local, sdata, &info);
+
+		ieee80211_send_deauth_disassoc(sdata, assoc_data->ap_addr,
+					       assoc_data->ap_addr,
+					       IEEE80211_STYPE_DEAUTH,
+					       WLAN_REASON_DEAUTH_LEAVING,
+					       true, frame_buf);
+
+		/* make sure the deauth is out before the station is removed */
+		ieee80211_flush_queues(local, sdata, false);
+
+		drv_mgd_complete_tx(local, sdata, &info);
+
+		cfg80211_tx_mlme_mgmt(sdata->dev, frame_buf, sizeof(frame_buf),
+				      false);
+	}
+
+	ieee80211_destroy_assoc_data(sdata, ASSOC_TIMEOUT, NULL);
+	drv_event_callback(local, sdata, &event);
+}
+
 void ieee80211_sta_work(struct ieee80211_sub_if_data *sdata)
 {
 	struct ieee80211_local *local = sdata->local;
@@ -9176,17 +9243,8 @@ void ieee80211_sta_work(struct ieee80211_sub_if_data *sdata)
 	    time_after(jiffies, ifmgd->assoc_data->timeout)) {
 		if ((ifmgd->assoc_data->need_beacon &&
 		     !sdata->deflink.u.mgd.have_beacon) ||
-		    ieee80211_do_assoc(sdata)) {
-			struct ieee80211_event event = {
-				.type = MLME_EVENT,
-				.u.mlme.data = ASSOC_EVENT,
-				.u.mlme.status = MLME_TIMEOUT,
-			};
-
-			ieee80211_destroy_assoc_data(sdata, ASSOC_TIMEOUT,
-						     NULL);
-			drv_event_callback(sdata->local, sdata, &event);
-		}
+		    ieee80211_do_assoc(sdata))
+			ieee80211_assoc_timeout_teardown(sdata);
 	} else if (ifmgd->assoc_data && ifmgd->assoc_data->timeout_started)
 		run_again(sdata, ifmgd->assoc_data->timeout);
 
@@ -9699,8 +9757,7 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
 			goto out_err;
 		}
 
-		err = ieee80211_mgd_setup_link_sta(link, new_sta,
-						   link_sta, cbss);
+		err = ieee80211_mgd_setup_link_sta(link, link_sta, cbss);
 		if (err) {
 			rcu_read_unlock();
 			sta_info_free(local, new_sta);
@@ -10538,6 +10595,7 @@ int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata,
 	}
 
 	assoc_data->spp_amsdu = req->flags & ASSOC_REQ_SPP_AMSDU;
+	assoc_data->cip = req->flags & ASSOC_REQ_CIP;
 
 	if (ifmgd->auth_data && !ifmgd->auth_data->done) {
 		err = -EBUSY;
@@ -11090,7 +11148,7 @@ ieee80211_process_ml_reconf_resp(struct ieee80211_sub_if_data *sdata,
 			goto disconnect;
 		}
 
-		if (ieee80211_mgd_setup_link_sta(link, sta, link_sta,
+		if (ieee80211_mgd_setup_link_sta(link, link_sta,
 						 add_links_data->link[link_id].bss))
 			goto disconnect;
 
@@ -11831,12 +11889,10 @@ void ieee80211_sta_rx_queued_frame(struct ieee80211_sub_if_data *sdata,
 	rx_status = (struct ieee80211_rx_status *) skb->cb;
 	fc = le16_to_cpu(mgmt->frame_control);
 
-	if (rx_status->link_valid) {
-		link = sdata_dereference(sdata->link[rx_status->link_id],
-					 sdata);
-		if (!link)
-			return;
-	}
+	link = sdata_dereference(sdata->link[rx_status->link_id],
+				 sdata);
+	if (!link)
+		return;
 
 	switch (fc & IEEE80211_FCTL_STYPE) {
 	case IEEE80211_STYPE_BEACON:
diff --git a/net/mac80211/nan.c b/net/mac80211/nan.c
index c7769da9617b55..dc14b5d676fcb6 100644
--- a/net/mac80211/nan.c
+++ b/net/mac80211/nan.c
@@ -248,6 +248,35 @@ ieee80211_nan_find_free_channel(struct ieee80211_nan_sched_cfg *sched_cfg)
 	return NULL;
 }
 
+int
+ieee80211_nan_set_non_evac_channels(struct ieee80211_sub_if_data *sdata,
+				    struct cfg80211_nan_non_evac_channels *channels)
+{
+	struct ieee80211_nan_sched_cfg *sched_cfg = &sdata->vif.cfg.nan_sched;
+
+	/*
+	 * cfg80211 already validated that all provided channels belong to the
+	 * current schedule, so just clear all non-evacuable flags and set the
+	 * ones in the list.
+	 */
+	for (int j = 0; j < ARRAY_SIZE(sched_cfg->channels); j++)
+		sched_cfg->channels[j].no_evacuate = false;
+
+	for (int i = 0; i < channels->n_channels; i++) {
+		for (int j = 0; j < ARRAY_SIZE(sched_cfg->channels); j++) {
+			if (!sched_cfg->channels[j].chanreq.oper.chan)
+				continue;
+			if (cfg80211_chandef_identical(&sched_cfg->channels[j].chanreq.oper,
+						       &channels->chandefs[i])) {
+				sched_cfg->channels[j].no_evacuate = true;
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
+
 int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata,
 				  struct cfg80211_nan_local_sched *sched)
 {
@@ -785,6 +814,10 @@ ieee80211_nan_find_evac_chan(struct ieee80211_local *local,
 
 		usable_channels++;
 
+		/* No-evacuate channel is usable but cannot be evacuated */
+		if (chan->no_evacuate)
+			continue;
+
 		chan_ctx = container_of(chan->chanctx_conf,
 					struct ieee80211_chanctx, conf);
 
diff --git a/net/mac80211/offchannel.c b/net/mac80211/offchannel.c
index e30767853c361b..c46cb81fab22b7 100644
--- a/net/mac80211/offchannel.c
+++ b/net/mac80211/offchannel.c
@@ -861,8 +861,10 @@ int ieee80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 			need_offchan = true;
 
 		rcu_read_lock();
-		sta = sta_info_get_bss(sdata, mgmt->da);
-		mlo_sta = sta && sta->sta.mlo;
+		if (!params->no_sta) {
+			sta = sta_info_get_bss(sdata, mgmt->da);
+			mlo_sta = sta && sta->sta.mlo;
+		}
 
 		if (!ieee80211_is_action(mgmt->frame_control) ||
 		    mgmt->u.action.category == WLAN_CATEGORY_PUBLIC ||
@@ -891,7 +893,8 @@ int ieee80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 		     local->ops->remain_on_channel &&
 		     memcmp(sdata->vif.cfg.ap_addr, mgmt->bssid, ETH_ALEN))) {
 			need_offchan = true;
-		} else if (sdata->u.mgd.associated &&
+		} else if (!params->no_sta &&
+			   sdata->u.mgd.associated &&
 			   ether_addr_equal(sdata->vif.cfg.ap_addr, mgmt->da)) {
 			sta = sta_info_get_bss(sdata, mgmt->da);
 			mlo_sta = sta && sta->sta.mlo;
@@ -1029,7 +1032,9 @@ int ieee80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	}
 
 	if (!need_offchan) {
-		ieee80211_tx_skb_tid(sdata, skb, 7, link_id);
+		ieee80211_tx_skb_tid(sdata, skb,
+				     params->no_sta ? ERR_PTR(-ENOENT) : sta,
+				     7, link_id);
 		ret = 0;
 		goto out_unlock;
 	}
@@ -1070,10 +1075,14 @@ void ieee80211_roc_purge(struct ieee80211_local *local,
 			 struct ieee80211_sub_if_data *sdata)
 {
 	struct ieee80211_roc_work *roc, *tmp;
+	bool start_next_roc = false;
 	bool work_to_do = false;
 
 	lockdep_assert_wiphy(local->hw.wiphy);
 
+	if (local->ops->remain_on_channel)
+		wiphy_work_flush(local->hw.wiphy, &local->hw_roc_start);
+
 	list_for_each_entry_safe(roc, tmp, &local->roc_list, list) {
 		if (sdata && roc->sdata != sdata)
 			continue;
@@ -1083,6 +1092,7 @@ void ieee80211_roc_purge(struct ieee80211_local *local,
 				/* can race, so ignore return value */
 				drv_cancel_remain_on_channel(local, roc->sdata);
 				ieee80211_roc_notify_destroy(roc);
+				start_next_roc = true;
 			} else {
 				roc->abort = true;
 				work_to_do = true;
@@ -1093,4 +1103,9 @@ void ieee80211_roc_purge(struct ieee80211_local *local,
 	}
 	if (work_to_do)
 		__ieee80211_roc_work(local);
+	else if (start_next_roc) {
+		wiphy_work_cancel(local->hw.wiphy, &local->hw_roc_done);
+		if (!list_empty(&local->roc_list))
+			ieee80211_start_next_roc(local);
+	}
 }
diff --git a/net/mac80211/parse.c b/net/mac80211/parse.c
index cb2be167cde795..3b66ca7d7a5c21 100644
--- a/net/mac80211/parse.c
+++ b/net/mac80211/parse.c
@@ -205,6 +205,10 @@ ieee80211_parse_extension_element(u32 *crc,
 			elems->ttlm_num++;
 		}
 		break;
+	case WLAN_EID_EXT_CIP_CAPA:
+		if (len >= sizeof(*elems->cip_cap))
+			elems->cip_cap = data;
+		break;
 	case WLAN_EID_EXT_UHR_OPER:
 		if (params->mode < IEEE80211_CONN_MODE_UHR)
 			break;
diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c
index e910f03af77756..f75257a1906434 100644
--- a/net/mac80211/rate.c
+++ b/net/mac80211/rate.c
@@ -57,7 +57,6 @@ void rate_control_rate_init(struct link_sta_info *link_sta)
 
 	/* TODO: check for minstrel_s1g ? */
 	if (sband->band == NL80211_BAND_S1GHZ) {
-		ieee80211_s1g_sta_rate_init(sta);
 		rcu_read_unlock();
 		return;
 	}
diff --git a/net/mac80211/rc80211_minstrel_ht.c b/net/mac80211/rc80211_minstrel_ht.c
index b73ef3adfcc576..45e7a7ea769008 100644
--- a/net/mac80211/rc80211_minstrel_ht.c
+++ b/net/mac80211/rc80211_minstrel_ht.c
@@ -7,6 +7,7 @@
 #include <linux/types.h>
 #include <linux/skbuff.h>
 #include <linux/debugfs.h>
+#include <linux/limits.h>
 #include <linux/random.h>
 #include <linux/moduleparam.h>
 #include <linux/ieee80211.h>
@@ -1193,6 +1194,54 @@ minstrel_ht_update_stats(struct minstrel_priv *mp, struct minstrel_ht_sta *mi)
 	mi->sample_time = jiffies;
 }
 
+/*
+ * Check whether an HT/VHT rate from a TX status entry maps to an entry
+ * in the minstrel_ht rate tables.  Values that cannot be represented
+ * there must not be used for indexing mi->groups[] and the MCS groups.
+ */
+static bool
+minstrel_ht_txstat_rate_valid(struct ieee80211_tx_rate *rate)
+{
+	unsigned int bw;
+
+	if (!(rate->flags & (IEEE80211_TX_RC_MCS | IEEE80211_TX_RC_VHT_MCS)))
+		return true;
+
+	if (rate->flags & IEEE80211_TX_RC_MCS) {
+		/* minstrel_ht supports up to MINSTREL_MAX_STREAMS streams */
+		return rate->idx < MINSTREL_MAX_STREAMS * 8;
+	}
+
+	/* minstrel_ht has no VHT groups for 160 MHz and wider */
+	bw = !!(rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH) +
+	     2 * !!(rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH);
+	if ((rate->flags & IEEE80211_TX_RC_160_MHZ_WIDTH) || bw > BW_80)
+		return false;
+
+	return ieee80211_rate_get_vht_nss(rate) <= MINSTREL_MAX_STREAMS &&
+	       ieee80211_rate_get_vht_mcs(rate) < MCS_GROUP_RATES;
+}
+
+static bool
+minstrel_ht_ri_txstat_rate_valid(struct rate_info *rate)
+{
+	if (!(rate->flags & (RATE_INFO_FLAGS_MCS | RATE_INFO_FLAGS_VHT_MCS)))
+		return true;
+
+	if (rate->flags & RATE_INFO_FLAGS_MCS) {
+		/* minstrel_ht supports up to MINSTREL_MAX_STREAMS streams */
+		return rate->mcs < MINSTREL_MAX_STREAMS * 8;
+	}
+
+	/* minstrel_ht has VHT groups only for 20/40/80 MHz */
+	if (rate->bw != RATE_INFO_BW_20 && rate->bw != RATE_INFO_BW_40 &&
+	    rate->bw != RATE_INFO_BW_80)
+		return false;
+
+	return rate->nss <= MINSTREL_MAX_STREAMS &&
+	       rate->mcs < MCS_GROUP_RATES;
+}
+
 static bool
 minstrel_ht_txstat_valid(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
 			 struct ieee80211_tx_rate *rate)
@@ -1205,9 +1254,8 @@ minstrel_ht_txstat_valid(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
 	if (!rate->count)
 		return false;
 
-	if (rate->flags & IEEE80211_TX_RC_MCS ||
-	    rate->flags & IEEE80211_TX_RC_VHT_MCS)
-		return true;
+	if (rate->flags & (IEEE80211_TX_RC_MCS | IEEE80211_TX_RC_VHT_MCS))
+		return minstrel_ht_txstat_rate_valid(rate);
 
 	for (i = 0; i < ARRAY_SIZE(mp->cck_rates); i++)
 		if (rate->idx == mp->cck_rates[i])
@@ -1235,9 +1283,9 @@ minstrel_ht_ri_txstat_valid(struct minstrel_priv *mp,
 	if (!rate_status->try_count)
 		return false;
 
-	if (rate_status->rate_idx.flags & RATE_INFO_FLAGS_MCS ||
-	    rate_status->rate_idx.flags & RATE_INFO_FLAGS_VHT_MCS)
-		return true;
+	if (rate_status->rate_idx.flags &
+	    (RATE_INFO_FLAGS_MCS | RATE_INFO_FLAGS_VHT_MCS))
+		return minstrel_ht_ri_txstat_rate_valid(&rate_status->rate_idx);
 
 	for (i = 0; i < ARRAY_SIZE(mp->cck_rates); i++) {
 		if (rate_status->rate_idx.legacy ==
@@ -1947,14 +1995,39 @@ minstrel_ht_alloc(struct ieee80211_hw *hw)
 }
 
 #ifdef CONFIG_MAC80211_DEBUGFS
+static int minstrel_ht_fixed_rate_idx_get(void *data, u64 *val)
+{
+	*val = *(u32 *)data;
+	return 0;
+}
+
+static int minstrel_ht_fixed_rate_idx_set(void *data, u64 val)
+{
+	u32 idx = val;
+
+	/* U32_MAX is the default and keeps fixed rate processing disabled */
+	if (val != U32_MAX &&
+	    (val > U16_MAX ||
+	     MI_RATE_GROUP(idx) >= ARRAY_SIZE(minstrel_mcs_groups) ||
+	     MI_RATE_IDX(idx) >= MCS_GROUP_RATES))
+		return -EINVAL;
+
+	*(u32 *)data = idx;
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(minstrel_ht_fixed_rate_idx_fops,
+			 minstrel_ht_fixed_rate_idx_get,
+			 minstrel_ht_fixed_rate_idx_set, "%llu\n");
+
 static void minstrel_ht_add_debugfs(struct ieee80211_hw *hw, void *priv,
 				    struct dentry *debugfsdir)
 {
 	struct minstrel_priv *mp = priv;
 
 	mp->fixed_rate_idx = (u32) -1;
-	debugfs_create_u32("fixed_rate_idx", S_IRUGO | S_IWUGO, debugfsdir,
-			   &mp->fixed_rate_idx);
+	debugfs_create_file("fixed_rate_idx", S_IRUGO | S_IWUGO, debugfsdir,
+			    &mp->fixed_rate_idx, &minstrel_ht_fixed_rate_idx_fops);
 }
 #endif
 
diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
index 5e26be8e27d8f0..f5893f9c07c679 100644
--- a/net/mac80211/rx.c
+++ b/net/mac80211/rx.c
@@ -222,43 +222,21 @@ ieee80211_rx_radiotap_hdrlen(struct ieee80211_local *local,
 }
 
 static void __ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata,
-					   int link_id,
-					   struct sta_info *sta,
+					   struct link_sta_info *link_sta,
 					   struct sk_buff *skb)
 {
-	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
-
-	if (link_id >= 0) {
-		status->link_valid = 1;
-		status->link_id = link_id;
-	} else {
-		status->link_valid = 0;
-	}
-
 	skb_queue_tail(&sdata->skb_queue, skb);
 	wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
-	if (sta) {
-		struct link_sta_info *link_sta_info;
-
-		if (link_id >= 0) {
-			link_sta_info = rcu_dereference(sta->link[link_id]);
-			if (!link_sta_info)
-				return;
-		} else {
-			link_sta_info = &sta->deflink;
-		}
-
-		link_sta_info->rx_stats.packets++;
-	}
+	if (link_sta)
+		link_sta->rx_stats.packets++;
 }
 
 static void ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata,
-					 int link_id,
-					 struct sta_info *sta,
+					 struct link_sta_info *link_sta,
 					 struct sk_buff *skb)
 {
 	skb->protocol = 0;
-	__ieee80211_queue_skb_to_iface(sdata, link_id, sta, skb);
+	__ieee80211_queue_skb_to_iface(sdata, link_sta, skb);
 }
 
 static void ieee80211_handle_mu_mimo_mon(struct ieee80211_sub_if_data *sdata,
@@ -301,7 +279,7 @@ static void ieee80211_handle_mu_mimo_mon(struct ieee80211_sub_if_data *sdata,
 	if (!skb)
 		return;
 
-	ieee80211_queue_skb_to_iface(sdata, -1, NULL, skb);
+	ieee80211_queue_skb_to_iface(sdata, NULL, skb);
 }
 
 /*
@@ -452,6 +430,8 @@ ieee80211_add_rx_radiotap_header(struct ieee80211_local *local,
 	if (status->band == NL80211_BAND_5GHZ ||
 	    status->band == NL80211_BAND_6GHZ)
 		channel_flags |= IEEE80211_CHAN_OFDM | IEEE80211_CHAN_5GHZ;
+	else if (status->band == NL80211_BAND_S1GHZ)
+		channel_flags |= IEEE80211_CHAN_OFDM | IEEE80211_CHAN_900MHZ;
 	else if (status->encoding != RX_ENC_LEGACY)
 		channel_flags |= IEEE80211_CHAN_DYN | IEEE80211_CHAN_2GHZ;
 	else if (rate && rate->flags & IEEE80211_RATE_ERP_G)
@@ -1498,7 +1478,7 @@ static void ieee80211_rx_reorder_ampdu(struct ieee80211_rx_data *rx,
 	/* if this mpdu is fragmented - terminate rx aggregation session */
 	sc = le16_to_cpu(hdr->seq_ctrl);
 	if (sc & IEEE80211_SCTL_FRAG) {
-		ieee80211_queue_skb_to_iface(rx->sdata, rx->link_id, NULL, skb);
+		ieee80211_queue_skb_to_iface(rx->sdata, NULL, skb);
 		return;
 	}
 
@@ -1712,10 +1692,7 @@ static void sta_ps_start(struct sta_info *sta)
 			list_del_init(&txqi->schedule_order);
 		spin_unlock(&local->active_txq_lock[txq->ac]);
 
-		if (txq_has_queue(txq))
-			set_bit(tid, &sta->txq_buffered_tids);
-		else
-			clear_bit(tid, &sta->txq_buffered_tids);
+		assign_bit(tid, &sta->txq_buffered_tids, txq_has_queue(txq));
 	}
 
 	sta_info_recalc_tim(sta);
@@ -1744,7 +1721,7 @@ static void sta_ps_end(struct sta_info *sta)
 	ieee80211_sta_ps_deliver_wakeup(sta);
 }
 
-int ieee80211_sta_ps_transition(struct ieee80211_sta *pubsta, bool start)
+void ieee80211_sta_ps_transition(struct ieee80211_sta *pubsta, bool start)
 {
 	struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
 	bool in_ps;
@@ -1753,15 +1730,13 @@ int ieee80211_sta_ps_transition(struct ieee80211_sta *pubsta, bool start)
 
 	/* Don't let the same PS state be set twice */
 	in_ps = test_sta_flag(sta, WLAN_STA_PS_STA);
-	if ((start && in_ps) || (!start && !in_ps))
-		return -EINVAL;
+	if (start == in_ps)
+		return;
 
 	if (start)
 		sta_ps_start(sta);
 	else
 		sta_ps_end(sta);
-
-	return 0;
 }
 EXPORT_SYMBOL(ieee80211_sta_ps_transition);
 
@@ -2499,6 +2474,12 @@ ieee80211_rx_h_defragment(struct ieee80211_rx_data *rx)
 	}
 
 	skb_pull(rx->skb, ieee80211_hdrlen(fc));
+	if (unlikely((u32)entry->extra_len + rx->skb->len > U16_MAX)) {
+		I802_DEBUG_INC(rx->local->rx_handlers_drop_defrag);
+		__skb_queue_purge(&entry->skb_list);
+		return RX_DROP_U_DEFRAG_OVERFLOW;
+	}
+
 	__skb_queue_tail(&entry->skb_list, rx->skb);
 	entry->last_frag = frag;
 	entry->extra_len += rx->skb->len;
@@ -2524,7 +2505,7 @@ ieee80211_rx_h_defragment(struct ieee80211_rx_data *rx)
 
  out:
 	ieee80211_led_rx(rx->local);
-	if (rx->sta)
+	if (rx->link_sta)
 		rx->link_sta->rx_stats.packets++;
 	return RX_CONTINUE;
 }
@@ -3335,8 +3316,8 @@ ieee80211_rx_h_data(struct ieee80211_rx_data *rx)
 		    (tf->action_code == WLAN_TDLS_CHANNEL_SWITCH_REQUEST ||
 		     tf->action_code == WLAN_TDLS_CHANNEL_SWITCH_RESPONSE)) {
 			rx->skb->protocol = cpu_to_be16(ETH_P_TDLS);
-			__ieee80211_queue_skb_to_iface(sdata, rx->link_id,
-						       rx->sta, rx->skb);
+			__ieee80211_queue_skb_to_iface(sdata, rx->link_sta,
+						       rx->skb);
 			return RX_QUEUED;
 		}
 	}
@@ -3991,7 +3972,7 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx)
 	return RX_QUEUED;
 
  queue:
-	ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb);
+	ieee80211_queue_skb_to_iface(sdata, rx->link_sta, rx->skb);
 	return RX_QUEUED;
 }
 
@@ -4005,6 +3986,7 @@ ieee80211_rx_h_userspace_mgmt(struct ieee80211_rx_data *rx)
 		.len = rx->skb->len,
 		.link_id = rx->link_id,
 		.have_link_id = rx->link_id >= 0,
+		.no_sta = !rx->sta,
 	};
 
 	/* skip known-bad action frames and return them in the next handler */
@@ -4127,7 +4109,7 @@ ieee80211_rx_h_action_return(struct ieee80211_rx_data *rx)
 					local->hw.offchannel_tx_hw_queue;
 		}
 
-		__ieee80211_tx_skb_tid_band(rx->sdata, nskb, 7, -1,
+		__ieee80211_tx_skb_tid_band(rx->sdata, nskb, rx->sta, 7, -1,
 					    status->band);
 	}
 
@@ -4147,7 +4129,7 @@ ieee80211_rx_h_ext(struct ieee80211_rx_data *rx)
 		return RX_DROP_U_UNEXPECTED_EXT_FRAME;
 
 	/* for now only beacons are ext, so queue them */
-	ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb);
+	ieee80211_queue_skb_to_iface(sdata, rx->link_sta, rx->skb);
 
 	return RX_QUEUED;
 }
@@ -4204,7 +4186,7 @@ ieee80211_rx_h_mgmt(struct ieee80211_rx_data *rx)
 		return RX_DROP_U_UNHANDLED_MGMT_STYPE;
 	}
 
-	ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb);
+	ieee80211_queue_skb_to_iface(sdata, rx->link_sta, rx->skb);
 
 	return RX_QUEUED;
 }
@@ -4248,12 +4230,28 @@ static void ieee80211_rx_handlers(struct ieee80211_rx_data *rx,
 	spin_lock_bh(&rx->local->rx_path_lock);
 
 	while ((skb = __skb_dequeue(frames))) {
+		struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
+
 		/*
-		 * all the other fields are valid across frames
-		 * that belong to an aMPDU since they are on the
-		 * same TID from the same station
+		 * Most fields are valid across frames. However,
+		 * in the case of reordering frames may have arrived
+		 * on different links, so adjust the link_id, link
+		 * and link_sta accordingly.
 		 */
 		rx->skb = skb;
+		if (ieee80211_vif_is_mld(&rx->sdata->vif) &&
+		    unlikely(rx->link_id != status->link_id)) {
+			if (rx->sta) {
+				rx->link_sta =
+					rcu_dereference(rx->sta->link[rx->link_id]);
+
+				/* Link got disabled, just use deflink */
+				if (!rx->link_sta)
+					rx->link_sta = &rx->sta->deflink;
+			}
+			rx->link_id = status->link_id;
+			rx->link = rcu_dereference(rx->sdata->link[status->link_id]);
+		}
 
 		if (WARN_ON_ONCE(!rx->link)) {
 			res = RX_DROP_U_NO_LINK;
@@ -4323,7 +4321,34 @@ static void ieee80211_invoke_rx_handlers(struct ieee80211_rx_data *rx)
 static bool
 ieee80211_rx_is_valid_sta_link_id(struct ieee80211_sta *sta, u8 link_id)
 {
-	return !!(sta->valid_links & BIT(link_id));
+	if (sta->mlo)
+		return sta->valid_links & BIT(link_id);
+
+	return sta->deflink.link_id == link_id;
+}
+
+static bool ieee80211_rx_data_set_link_sta(struct ieee80211_rx_data *rx,
+					   struct link_sta_info *link_sta)
+{
+	struct sta_info *sta;
+
+	if (WARN_ON_ONCE(!link_sta) ||
+	    !ieee80211_rx_is_valid_sta_link_id(&link_sta->sta->sta,
+					       link_sta->link_id))
+		return false;
+
+	sta = link_sta->sta;
+
+	rx->sta = sta;
+	rx->local = sta->sdata->local;
+	rx->link = rcu_dereference(sta->sdata->link[link_sta->link_id]);
+	if (!rx->link)
+		return false;
+
+	rx->link_id = rx->link->link_id;
+	rx->link_sta = link_sta;
+
+	return true;
 }
 
 static bool ieee80211_rx_data_set_link(struct ieee80211_rx_data *rx,
@@ -4360,11 +4385,13 @@ static bool ieee80211_rx_data_set_sta(struct ieee80211_rx_data *rx,
 
 	if (link_id < 0) {
 		if (ieee80211_vif_is_mld(&rx->sdata->vif) &&
-		    sta && !sta->sta.valid_links)
+		    sta && !sta->sta.valid_links) {
 			rx->link =
 				rcu_dereference(rx->sdata->link[sta->deflink.link_id]);
-		else
+			rx->link_sta = &sta->deflink;
+		} else {
 			rx->link = &rx->sdata->deflink;
+		}
 	} else if (!ieee80211_rx_data_set_link(rx, link_id)) {
 		return false;
 	}
@@ -4387,7 +4414,7 @@ void ieee80211_release_reorder_timeout(struct sta_info *sta, int tid)
 	struct tid_ampdu_rx *tid_agg_rx;
 	int link_id = -1;
 
-	/* FIXME: statistics won't be right with this */
+	/* NOTE: the correct link STA for the frame is set later */
 	if (sta->sta.valid_links)
 		link_id = ffs(sta->sta.valid_links) - 1;
 
@@ -4914,20 +4941,14 @@ static void ieee80211_rx_8023(struct ieee80211_rx_data *rx,
 {
 	struct ieee80211_sta_rx_stats *stats;
 	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);
-	struct sta_info *sta = rx->sta;
-	struct link_sta_info *link_sta;
+	struct link_sta_info *link_sta = rx->link_sta;
 	struct sk_buff *skb = rx->skb;
 	void *sa = skb->data + ETH_ALEN;
 	void *da = skb->data;
 
-	if (rx->link_id >= 0) {
-		link_sta = rcu_dereference(sta->link[rx->link_id]);
-		if (WARN_ON_ONCE(!link_sta)) {
-			dev_kfree_skb(rx->skb);
-			return;
-		}
-	} else {
-		link_sta = &sta->deflink;
+	if (WARN_ON_ONCE(!link_sta)) {
+		dev_kfree_skb(rx->skb);
+		return;
 	}
 
 	stats = &link_sta->rx_stats;
@@ -5154,6 +5175,9 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
 		goto drop;
 	}
 
+	status = IEEE80211_SKB_RXCB(skb);
+	status->link_id = rx->link_id < 0 ? 0 : rx->link_id;
+
 	ieee80211_rx_8023(rx, fast_rx, orig_len);
 
 	return true;
@@ -5178,6 +5202,7 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx,
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	struct link_sta_info *link_sta = rx->link_sta;
 	struct ieee80211_link_data *link = rx->link;
+	struct ieee80211_rx_status *status;
 
 	rx->skb = skb;
 
@@ -5221,6 +5246,9 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx,
 		hdr = (struct ieee80211_hdr *)rx->skb->data;
 	}
 
+	status = IEEE80211_SKB_RXCB(rx->skb);
+	status->link_id = rx->link_id < 0 ? 0 : rx->link_id;
+
 	if (ieee80211_is_s1g_beacon(hdr->frame_control)) {
 		ieee80211_invoke_rx_handlers(rx);
 		return true;
@@ -5256,22 +5284,19 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx,
 }
 
 static void __ieee80211_rx_handle_8023(struct ieee80211_hw *hw,
-				       struct ieee80211_sta *pubsta,
+				       struct ieee80211_link_sta *link_pubsta,
 				       struct sk_buff *skb,
 				       struct list_head *list)
 {
 	struct ieee80211_local *local = hw_to_local(hw);
-	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_fast_rx *fast_rx;
 	struct ieee80211_rx_data rx;
 	struct sta_info *sta;
-	int link_id = -1;
 
 	memset(&rx, 0, sizeof(rx));
 	rx.skb = skb;
 	rx.local = local;
 	rx.list = list;
-	rx.link_id = -1;
 
 	I802_DEBUG_INC(local->dot11ReceivedFragmentCount);
 
@@ -5279,23 +5304,15 @@ static void __ieee80211_rx_handle_8023(struct ieee80211_hw *hw,
 	if (skb->len < sizeof(struct ethhdr))
 		goto drop;
 
-	if (!pubsta)
+	if (!link_pubsta)
 		goto drop;
 
-	if (status->link_valid)
-		link_id = status->link_id;
-
-	/*
-	 * TODO: Should the frame be dropped if the right link_id is not
-	 * available? Or may be it is fine in the current form to proceed with
-	 * the frame processing because with frame being in 802.3 format,
-	 * link_id is used only for stats purpose and updating the stats on
-	 * the deflink is fine?
-	 */
-	sta = container_of(pubsta, struct sta_info, sta);
-	if (!ieee80211_rx_data_set_sta(&rx, sta, link_id))
+	sta = container_of(link_pubsta->sta, struct sta_info, sta);
+	if (!ieee80211_rx_data_set_sta(&rx, sta, link_pubsta->link_id))
 		goto drop;
 
+	rx.link_id = link_pubsta->link_id;
+
 	fast_rx = rcu_dereference(rx.sta->fast_rx);
 	if (!fast_rx)
 		goto drop;
@@ -5307,59 +5324,19 @@ static void __ieee80211_rx_handle_8023(struct ieee80211_hw *hw,
 	dev_kfree_skb(skb);
 }
 
-static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx,
-				       struct sk_buff *skb, bool consume)
+static bool ieee80211_rx_valid_freq(int freq, struct ieee80211_link_data *link)
 {
-	struct link_sta_info *link_sta;
-	struct ieee80211_hdr *hdr = (void *)skb->data;
-	struct sta_info *sta;
-	int link_id = -1;
-
-	if (ieee80211_is_s1g_beacon(hdr->frame_control)) {
-		if (!ieee80211_rx_data_set_sta(rx, NULL, -1))
-			return false;
-
-		return ieee80211_prepare_and_rx_handle(rx, skb, consume);
-	}
-
-	/*
-	 * Look up link station first, in case there's a
-	 * chance that they might have a link address that
-	 * is identical to the MLD address, that way we'll
-	 * have the link information if needed.
-	 */
-	link_sta = link_sta_info_get_bss(rx->sdata, hdr->addr2);
-	if (link_sta) {
-		sta = link_sta->sta;
-		link_id = link_sta->link_id;
-	} else {
-		struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
+	struct ieee80211_chanctx_conf *conf;
 
-		sta = sta_info_get_bss(rx->sdata, hdr->addr2);
-		if (status->link_valid) {
-			link_id = status->link_id;
-		} else if (ieee80211_vif_is_mld(&rx->sdata->vif) &&
-			   status->freq) {
-			struct ieee80211_link_data *link;
-			struct ieee80211_chanctx_conf *conf;
-
-			for_each_link_data_rcu(rx->sdata, link) {
-				conf = rcu_dereference(link->conf->chanctx_conf);
-				if (!conf || !conf->def.chan)
-					continue;
-
-				if (status->freq == conf->def.chan->center_freq) {
-					link_id = link->link_id;
-					break;
-				}
-			}
-		}
-	}
+	if (!freq || link->sdata->vif.type == NL80211_IFTYPE_NAN ||
+	    link->sdata->vif.type == NL80211_IFTYPE_NAN_DATA)
+		return true;
 
-	if (!ieee80211_rx_data_set_sta(rx, sta, link_id))
+	conf = rcu_dereference(link->conf->chanctx_conf);
+	if (!conf || !conf->def.chan)
 		return false;
 
-	return ieee80211_prepare_and_rx_handle(rx, skb, consume);
+	return freq == conf->def.chan->center_freq;
 }
 
 /*
@@ -5367,18 +5344,20 @@ static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx,
  * be called with rcu_read_lock protection.
  */
 static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
-					 struct ieee80211_sta *pubsta,
+					 struct ieee80211_link_sta *link_pubsta,
 					 struct sk_buff *skb,
 					 struct list_head *list)
 {
 	struct ieee80211_local *local = hw_to_local(hw);
-	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_sub_if_data *sdata;
+	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_hdr *hdr;
+	struct link_sta_info *link_sta;
+	struct sta_info *sta;
 	__le16 fc;
 	struct ieee80211_rx_data rx;
-	struct ieee80211_sub_if_data *prev;
 	struct rhlist_head *tmp;
+	bool rx_data_pending;
 	int err = 0;
 
 	fc = ((struct ieee80211_hdr *)skb->data)->frame_control;
@@ -5386,7 +5365,6 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 	rx.skb = skb;
 	rx.local = local;
 	rx.list = list;
-	rx.link_id = -1;
 
 	if (ieee80211_is_data(fc) || ieee80211_is_mgmt(fc))
 		I802_DEBUG_INC(local->dot11ReceivedFragmentCount);
@@ -5426,97 +5404,102 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 		     ieee80211_is_s1g_beacon(hdr->frame_control)))
 		ieee80211_scan_rx(local, skb);
 
+	/*
+	 * RX of a data frame should only happen to existing stations.
+	 * It is therefore more efficient to use the one provided by the driver
+	 * or search based on the station's address.
+	 *
+	 * Note we will fall through and handle a spurious data frame in the
+	 * same way as a management frame.
+	 */
 	if (ieee80211_is_data(fc)) {
-		struct sta_info *sta, *prev_sta;
-		int link_id = -1;
-
-		if (status->link_valid)
-			link_id = status->link_id;
-
-		if (pubsta) {
-			sta = container_of(pubsta, struct sta_info, sta);
-			if (!ieee80211_rx_data_set_sta(&rx, sta, link_id))
-				goto out;
-
-			/*
-			 * In MLO connection, fetch the link_id using addr2
-			 * when the driver does not pass link_id in status.
-			 * When the address translation is already performed by
-			 * driver/hw, the valid link_id must be passed in
-			 * status.
-			 */
-
-			if (!status->link_valid && pubsta->mlo) {
-				struct link_sta_info *link_sta;
-
-				link_sta = link_sta_info_get_bss(rx.sdata,
-								 hdr->addr2);
-				if (!link_sta)
-					goto out;
-
-				if (!ieee80211_rx_data_set_link(&rx,
-								link_sta->link_id))
-					goto out;
-			}
-
-			if (ieee80211_prepare_and_rx_handle(&rx, skb, true))
+		if (link_pubsta) {
+			sta = container_of(link_pubsta->sta, struct sta_info,
+					   sta);
+			link_sta = rcu_dereference(sta->link[link_pubsta->link_id]);
+
+			rx.sdata = sta->sdata;
+			if (ieee80211_rx_data_set_link_sta(&rx, link_sta) &&
+			    ieee80211_prepare_and_rx_handle(&rx, skb, true))
 				return;
+
 			goto out;
 		}
 
-		prev_sta = NULL;
+		rx_data_pending = false;
 
+		/* Search for stations on non-MLD interfaces */
 		for_each_sta_info(local, hdr->addr2, sta, tmp) {
-			if (!prev_sta) {
-				prev_sta = sta;
-				continue;
-			}
+			struct ieee80211_link_data *link;
 
-			rx.sdata = prev_sta->sdata;
-			if (!status->link_valid && prev_sta->sta.mlo) {
-				struct link_sta_info *link_sta;
+			if (ieee80211_vif_is_mld(&sta->sdata->vif))
+				continue;
 
-				link_sta = link_sta_info_get_bss(rx.sdata,
-								 hdr->addr2);
-				if (!link_sta)
-					continue;
+			link = &sta->sdata->deflink;
+			if (!ieee80211_rx_valid_freq(status->freq, link))
+				continue;
 
-				link_id = link_sta->link_id;
+			if (rx_data_pending) {
+				ieee80211_prepare_and_rx_handle(&rx, skb,
+								false);
+				rx_data_pending = false;
 			}
 
-			if (!ieee80211_rx_data_set_sta(&rx, prev_sta, link_id))
-				goto out;
-
-			ieee80211_prepare_and_rx_handle(&rx, skb, false);
+			rx.sdata = sta->sdata;
+			if (!ieee80211_rx_data_set_link_sta(&rx, &sta->deflink))
+				continue;
 
-			prev_sta = sta;
+			rx_data_pending = true;
 		}
 
-		if (prev_sta) {
-			rx.sdata = prev_sta->sdata;
-			if (!status->link_valid && prev_sta->sta.mlo) {
-				struct link_sta_info *link_sta;
+		/* And search stations on MLD interfaces */
+		for_each_link_sta_info(local, hdr->addr2, link_sta, tmp) {
+			struct ieee80211_link_data *link;
 
-				link_sta = link_sta_info_get_bss(rx.sdata,
-								 hdr->addr2);
-				if (!link_sta)
-					goto out;
+			sta = link_sta->sta;
+			sdata =	sta->sdata;
+			link = rcu_dereference(sdata->link[link_sta->link_id]);
+
+			if (!ieee80211_rx_valid_freq(status->freq, link))
+				continue;
 
-				link_id = link_sta->link_id;
+			if (rx_data_pending) {
+				ieee80211_prepare_and_rx_handle(&rx, skb,
+								false);
+				rx_data_pending = false;
 			}
 
-			if (!ieee80211_rx_data_set_sta(&rx, prev_sta, link_id))
-				goto out;
+			rx.sdata = sta->sdata;
+			if (!ieee80211_rx_data_set_link_sta(&rx, link_sta))
+				continue;
+
+			rx_data_pending = true;
+		}
 
+		if (rx_data_pending) {
 			if (ieee80211_prepare_and_rx_handle(&rx, skb, true))
 				return;
+
 			goto out;
 		}
+
+		/* fall through, e.g. for spurious frame notification */
 	}
 
-	prev = NULL;
+	/*
+	 * This is a management frame (or a data frame without a station) and
+	 * it will be delivered independent of whether a station exists,
+	 * so iterate the interfaces.
+	 */
+	rx_data_pending = false;
+
+	/* We expect the driver to provide frequency information */
+	if (WARN_ON_ONCE(!local->emulate_chanctx && !status->freq))
+		goto out;
 
 	list_for_each_entry_rcu(sdata, &local->interfaces, list) {
+		struct ieee80211_link_data *link = NULL;
+
 		if (!ieee80211_sdata_running(sdata))
 			continue;
 
@@ -5524,30 +5507,85 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 		    sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
 			continue;
 
+		link_sta = NULL;
+		link = NULL;
+
+		/* Try to resolve the station */
+		if (!ieee80211_vif_is_mld(&sdata->vif)) {
+			sta = sta_info_get_bss(sdata, hdr->addr2);
+
+			if (sta) {
+				link_sta = &sta->deflink;
+				link = &sdata->deflink;
+			}
+		} else {
+			link_sta = link_sta_info_get_bss(sdata, hdr->addr2);
+
+			if (link_sta)
+				link = rcu_dereference(sdata->link[link_sta->link_id]);
+		}
+
 		/*
-		 * frame is destined for this interface, but if it's
-		 * not also for the previous one we handle that after
-		 * the loop to avoid copying the SKB once too much
+		 * If we found a STA and it has a valid link information, then
+		 * RX using the station.
 		 */
+		if (link_sta && link &&
+		    ieee80211_rx_valid_freq(status->freq, link)) {
+			if (rx_data_pending) {
+				ieee80211_prepare_and_rx_handle(&rx, skb, false);
+				rx_data_pending = false;
+			}
+
+			/* No valid_links check as we need to RX beacons */
+
+			rx.sdata = sdata;
+			if (ieee80211_rx_data_set_link_sta(&rx, link_sta))
+				rx_data_pending = true;
 
-		if (!prev) {
-			prev = sdata;
 			continue;
 		}
 
-		rx.sdata = prev;
-		ieee80211_rx_for_interface(&rx, skb, false);
+		/* No station, try to resolve the link and RX */
+		if (ieee80211_vif_is_mld(&sdata->vif)) {
+			struct ieee80211_chanctx_conf *conf;
+			bool found = false;
 
-		prev = sdata;
-	}
+			for_each_link_data_rcu(sdata, link) {
+				conf = rcu_dereference(link->conf->chanctx_conf);
+				if (!conf || !conf->def.chan)
+					continue;
 
-	if (prev) {
-		rx.sdata = prev;
+				if (status->freq == conf->def.chan->center_freq) {
+					found = true;
+					break;
+				}
+			}
 
-		if (ieee80211_rx_for_interface(&rx, skb, true))
-			return;
+			if (!found)
+				link = &sdata->deflink;
+		} else {
+			link = &sdata->deflink;
+		}
+
+		if (rx_data_pending) {
+			ieee80211_prepare_and_rx_handle(&rx, skb, false);
+			rx_data_pending = false;
+		}
+
+		rx.sdata = sdata;
+		rx.local = sdata->local;
+		rx.link = link;
+		rx.link_id = link->link_id;
+		rx.sta = NULL;
+		rx.link_sta = NULL;
+
+		rx_data_pending = true;
 	}
 
+	if (rx_data_pending &&
+	    ieee80211_prepare_and_rx_handle(&rx, skb, true))
+		return;
+
  out:
 	dev_kfree_skb(skb);
 }
@@ -5556,7 +5594,8 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
  * This is the receive path handler. It is called by a low level driver when an
  * 802.11 MPDU is received from the hardware.
  */
-void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
+void ieee80211_rx_list(struct ieee80211_hw *hw,
+		       struct ieee80211_link_sta *link_pubsta,
 		       struct sk_buff *skb, struct list_head *list)
 {
 	struct ieee80211_local *local = hw_to_local(hw);
@@ -5694,9 +5733,6 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
 		}
 	}
 
-	if (WARN_ON_ONCE(status->link_id >= IEEE80211_LINK_UNSPECIFIED))
-		goto drop;
-
 	status->rx_flags = 0;
 
 	kcov_remote_start_common(skb_get_kcov_handle(skb));
@@ -5715,9 +5751,10 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
 			ieee80211_tpt_led_trig_rx(local, skb->len);
 
 		if (status->flag & RX_FLAG_8023)
-			__ieee80211_rx_handle_8023(hw, pubsta, skb, list);
+			__ieee80211_rx_handle_8023(hw, link_pubsta, skb, list);
 		else
-			__ieee80211_rx_handle_packet(hw, pubsta, skb, list);
+			__ieee80211_rx_handle_packet(hw, link_pubsta, skb,
+						     list);
 	}
 
 	kcov_remote_stop();
@@ -5727,7 +5764,8 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
 }
 EXPORT_SYMBOL(ieee80211_rx_list);
 
-void ieee80211_rx_napi(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
+void ieee80211_rx_napi(struct ieee80211_hw *hw,
+		       struct ieee80211_link_sta *link_pubsta,
 		       struct sk_buff *skb, struct napi_struct *napi)
 {
 	struct sk_buff *tmp;
@@ -5740,7 +5778,7 @@ void ieee80211_rx_napi(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
 	 * receive processing
 	 */
 	rcu_read_lock();
-	ieee80211_rx_list(hw, pubsta, skb, &list);
+	ieee80211_rx_list(hw, link_pubsta, skb, &list);
 	rcu_read_unlock();
 
 	if (!napi) {
diff --git a/net/mac80211/s1g.c b/net/mac80211/s1g.c
index bd7baf48183677..4b70afa6e6d78c 100644
--- a/net/mac80211/s1g.c
+++ b/net/mac80211/s1g.c
@@ -9,14 +9,6 @@
 #include "ieee80211_i.h"
 #include "driver-ops.h"
 
-void ieee80211_s1g_sta_rate_init(struct sta_info *sta)
-{
-	/* avoid indicating legacy bitrates for S1G STAs */
-	sta->deflink.tx_stats.last_rate.flags |= IEEE80211_TX_RC_S1G_MCS;
-	sta->deflink.rx_stats.last_rate =
-			STA_STATS_FIELD(TYPE, STA_STATS_RATE_TYPE_S1G);
-}
-
 bool ieee80211_s1g_is_twt_setup(struct sk_buff *skb)
 {
 	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c
index 8e950ef6d1ead9..a4bca412e57775 100644
--- a/net/mac80211/scan.c
+++ b/net/mac80211/scan.c
@@ -9,7 +9,7 @@
  * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
  * Copyright 2013-2015  Intel Mobile Communications GmbH
  * Copyright 2016-2017  Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 
 #include <linux/if_arp.h>
@@ -204,12 +204,10 @@ ieee80211_bss_info_update(struct ieee80211_local *local,
 		 * an indication on which of the links the frame was received
 		 */
 		if (ieee80211_vif_is_mld(&scan_sdata->vif)) {
-			if (rx_status->link_valid) {
-				s8 link_id = rx_status->link_id;
+			s8 link_id = rx_status->link_id;
 
-				link_conf =
-					rcu_dereference(scan_sdata->vif.link_conf[link_id]);
-			}
+			link_conf =
+				rcu_dereference(scan_sdata->vif.link_conf[link_id]);
 		} else {
 			link_conf = &scan_sdata->vif.bss_conf;
 		}
diff --git a/net/mac80211/spectmgmt.c b/net/mac80211/spectmgmt.c
index 880f4625775dce..f5e4970869b6d4 100644
--- a/net/mac80211/spectmgmt.c
+++ b/net/mac80211/spectmgmt.c
@@ -111,8 +111,8 @@ validate_chandef_by_ht_vht_oper(struct ieee80211_sub_if_data *sdata,
 
 	switch (chan_width) {
 	case NL80211_CHAN_WIDTH_320:
-		WARN_ON(1);
-		break;
+		chandef->chan = NULL;
+		return;
 	case NL80211_CHAN_WIDTH_160:
 		vht_oper.chan_width = IEEE80211_VHT_CHANWIDTH_80MHZ;
 		vht_oper.center_freq_seg1_idx = vht_oper.center_freq_seg0_idx;
diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c
index fdf00cbf49d8b4..e81cd155197b92 100644
--- a/net/mac80211/sta_info.c
+++ b/net/mac80211/sta_info.c
@@ -137,6 +137,8 @@ static void __cleanup_single_sta(struct sta_info *sta)
 	struct ieee80211_local *local = sdata->local;
 	struct ps_data *ps;
 
+	cancel_work_sync(&sta->drv_deliver_wk);
+
 	if (test_sta_flag(sta, WLAN_STA_PS_STA) ||
 	    test_sta_flag(sta, WLAN_STA_PS_DRIVER) ||
 	    test_sta_flag(sta, WLAN_STA_PS_DELIVER)) {
@@ -166,8 +168,6 @@ static void __cleanup_single_sta(struct sta_info *sta)
 	if (ieee80211_vif_is_mesh(&sdata->vif))
 		mesh_sta_cleanup(sta);
 
-	cancel_work_sync(&sta->drv_deliver_wk);
-
 	/*
 	 * Destroy aggregation state here. It would be nice to wait for the
 	 * driver to finish aggregation stop and then clean up, but for now
@@ -1582,6 +1582,8 @@ static void __sta_info_destroy_part2(struct sta_info *sta, bool recalc)
 
 	sta->dead = true;
 
+	cancel_work_sync(&sta->drv_deliver_wk);
+
 	local->num_sta--;
 	local->sta_generation++;
 
diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h
index ee0d32877c5b47..f409f6a3afa1f1 100644
--- a/net/mac80211/sta_info.h
+++ b/net/mac80211/sta_info.h
@@ -485,6 +485,7 @@ struct ieee80211_fragment_cache {
  * @link_hash_node: hash node for rhashtable
  * @sta: Points to the STA info
  * @gtk: group keys negotiated with this station, if any
+ * @cigtk: control integrity group keys negotiated with this station, if any
  * @tx_stats: TX statistics
  * @tx_stats.packets: # of packets transmitted
  * @tx_stats.bytes: # of bytes in all packets transmitted
@@ -535,6 +536,7 @@ struct link_sta_info {
 	struct ieee80211_key __rcu *gtk[NUM_DEFAULT_KEYS +
 					NUM_DEFAULT_MGMT_KEYS +
 					NUM_DEFAULT_BEACON_KEYS];
+	struct ieee80211_key __rcu *cigtk[NUM_CTRL_KEYS];
 	struct ieee80211_sta_rx_stats __percpu *pcpu_rx_stats;
 
 	/* Updated from RX path only, no locking requirements */
diff --git a/net/mac80211/status.c b/net/mac80211/status.c
index 3d811f652603ef..414979e2a1d930 100644
--- a/net/mac80211/status.c
+++ b/net/mac80211/status.c
@@ -1164,6 +1164,73 @@ void ieee80211_tx_status_skb(struct ieee80211_hw *hw, struct sk_buff *skb)
 }
 EXPORT_SYMBOL(ieee80211_tx_status_skb);
 
+/*
+ * Check whether the HT/VHT rate information in a TX status entry is
+ * valid for its encoding.  This is not specific to any rate control
+ * algorithm; all status consumers rely on the values being sane.
+ */
+static bool ieee80211_tx_status_rate_info_valid(const struct rate_info *rate)
+{
+	if (rate->flags & RATE_INFO_FLAGS_MCS)
+		return rate->mcs < 32;
+
+	if (rate->flags & RATE_INFO_FLAGS_VHT_MCS)
+		return rate->nss >= 1 && rate->nss <= 8 && rate->mcs <= 11;
+
+	return true;
+}
+
+static bool
+ieee80211_tx_status_tx_rate_valid(const struct ieee80211_tx_rate *rate)
+{
+	if (rate->flags & IEEE80211_TX_RC_MCS)
+		return rate->idx < 32;
+
+	if (rate->flags & IEEE80211_TX_RC_VHT_MCS)
+		return ieee80211_rate_get_vht_mcs(rate) <= 11;
+
+	return true;
+}
+
+/*
+ * Drop TX status rate entries that don't describe a valid rate.  The
+ * status information is used by rate control and by other mac80211
+ * code, and a malformed entry must not be able to corrupt state beyond
+ * the driver that reported it.
+ */
+static void
+ieee80211_tx_status_drop_invalid_rates(struct ieee80211_tx_status *status)
+{
+	int i;
+
+	for (i = 0; i < status->n_rates; i++) {
+		struct ieee80211_rate_status *rs = &status->rates[i];
+
+		if (ieee80211_tx_status_rate_info_valid(&rs->rate_idx))
+			continue;
+
+		rs->try_count = 0;
+		memset(&rs->rate_idx, 0, sizeof(rs->rate_idx));
+	}
+
+	if (!status->info)
+		return;
+
+	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
+		struct ieee80211_tx_rate *rate;
+
+		rate = &status->info->status.rates[i];
+		if (rate->idx < 0)
+			break;
+
+		if (ieee80211_tx_status_tx_rate_valid(rate))
+			continue;
+
+		rate->idx = -1;
+		rate->count = 0;
+	}
+}
+
 void ieee80211_tx_status_ext(struct ieee80211_hw *hw,
 			     struct ieee80211_tx_status *status)
 {
@@ -1176,6 +1243,8 @@ void ieee80211_tx_status_ext(struct ieee80211_hw *hw,
 	bool acked, noack_success, ack_signal_valid;
 	u16 tx_time_est;
 
+	ieee80211_tx_status_drop_invalid_rates(status);
+
 	if (pubsta) {
 		sta = container_of(pubsta, struct sta_info, sta);
 
@@ -1300,6 +1369,7 @@ void ieee80211_tx_rate_update(struct ieee80211_hw *hw,
 		.sta = pubsta,
 	};
 
+	ieee80211_tx_status_drop_invalid_rates(&status);
 	rate_control_tx_status(local, &status);
 
 	if (ieee80211_hw_check(&local->hw, HAS_RATE_CONTROL))
diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c
index f663d28d9209cf..68bd64328937d6 100644
--- a/net/mac80211/tdls.c
+++ b/net/mac80211/tdls.c
@@ -1066,7 +1066,7 @@ ieee80211_tdls_prep_mgmt_packet(struct wiphy *wiphy, struct net_device *dev,
 	}
 
 	if (action_code == WLAN_PUB_ACTION_TDLS_DISCOVER_RES) {
-		ieee80211_tx_skb_tid(sdata, skb, 7, link_id);
+		ieee80211_tx_skb_tid(sdata, skb, sta, 7, link_id);
 		return 0;
 	}
 
@@ -1121,7 +1121,7 @@ ieee80211_tdls_prep_mgmt_packet(struct wiphy *wiphy, struct net_device *dev,
 	/* disable bottom halves when entering the Tx path */
 	local_bh_disable();
 	__ieee80211_subif_start_xmit(skb, dev, flags,
-				     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC, 0);
+				     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC);
 	local_bh_enable();
 
 	return ret;
diff --git a/net/mac80211/tests/chan-mode.c b/net/mac80211/tests/chan-mode.c
index ab7d38ef6a3a14..85dc87215d2366 100644
--- a/net/mac80211/tests/chan-mode.c
+++ b/net/mac80211/tests/chan-mode.c
@@ -219,10 +219,8 @@ static void test_determine_chan_mode(struct kunit *test)
 	/* To force EHT downgrade to HE on punctured 80 MHz downgraded to 40 MHz */
 	set_bit(IEEE80211_HW_DISALLOW_PUNCTURING, t_sdata->local.hw.flags);
 
-	if (params->strict)
-		set_bit(IEEE80211_HW_STRICT, t_sdata->local.hw.flags);
-	else
-		clear_bit(IEEE80211_HW_STRICT, t_sdata->local.hw.flags);
+	assign_bit(IEEE80211_HW_STRICT, t_sdata->local.hw.flags,
+		   params->strict);
 
 	t_sdata->sdata->u.mgd.ht_capa_mask = params->ht_capa_mask;
 	t_sdata->sdata->u.mgd.vht_capa = params->vht_capa;
diff --git a/net/mac80211/tests/util.c b/net/mac80211/tests/util.c
index 0b01264af577ea..89ff4f389de389 100644
--- a/net/mac80211/tests/util.c
+++ b/net/mac80211/tests/util.c
@@ -2,7 +2,7 @@
 /*
  * Utilities for mac80211 unit testing
  *
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024, 2026 Intel Corporation
  */
 #include <linux/ieee80211.h>
 #include <net/mac80211.h>
@@ -226,10 +226,6 @@ int t_sdata_init(struct kunit_resource *resource, void *ctx)
 		sband = t_sdata->wiphy->bands[band];
 		sband->band = band;
 
-		sband->bitrates =
-			kmemdup(bitrates, sizeof(bitrates), GFP_KERNEL);
-		sband->n_bitrates = ARRAY_SIZE(bitrates);
-
 		/* Initialize channels, feel free to add more channels/bands */
 		switch (band) {
 		case NL80211_BAND_2GHZ:
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index 814399989b5e48..1953ec35c783ec 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -269,7 +269,6 @@ ieee80211_tx_h_dynamic_ps(struct ieee80211_tx_data *tx)
 static ieee80211_tx_result debug_noinline
 ieee80211_tx_h_check_assoc(struct ieee80211_tx_data *tx)
 {
-
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx->skb->data;
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb);
 	bool assoc = false;
@@ -277,20 +276,6 @@ ieee80211_tx_h_check_assoc(struct ieee80211_tx_data *tx)
 	if (unlikely(info->flags & IEEE80211_TX_CTL_INJECTED))
 		return TX_CONTINUE;
 
-	if (unlikely(test_bit(SCAN_SW_SCANNING, &tx->local->scanning)) &&
-	    test_bit(SDATA_STATE_OFFCHANNEL, &tx->sdata->state) &&
-	    !ieee80211_is_probe_req(hdr->frame_control) &&
-	    !ieee80211_is_any_nullfunc(hdr->frame_control))
-		/*
-		 * When software scanning only nullfunc frames (to notify
-		 * the sleep state to the AP) and probe requests (for the
-		 * active scan) are allowed, all other frames should not be
-		 * sent and we should not get here, but if we do
-		 * nonetheless, drop them to avoid sending them
-		 * off-channel. See __ieee80211_start_scan() for more.
-		 */
-		return TX_DROP;
-
 	if (tx->sdata->vif.type == NL80211_IFTYPE_OCB)
 		return TX_CONTINUE;
 
@@ -589,6 +574,43 @@ ieee80211_select_link_key(struct ieee80211_tx_data *tx)
 	return NULL;
 }
 
+/*
+ * An 802.3 frame is always a data frame, so there are no 802.11 addresses to
+ * look at: with a station the pairwise key is all that can apply, without one
+ * the frame is group addressed. On an MLD the driver has to pick the group key
+ * itself, as the same frame goes out on several links that each have their own.
+ */
+static ieee80211_tx_result
+ieee80211_select_key_8023(struct ieee80211_tx_data *tx)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb);
+
+	if (unlikely(info->flags & IEEE80211_TX_INTFL_DONT_ENCRYPT))
+		return TX_CONTINUE;
+
+	if (tx->sta) {
+		tx->key = rcu_dereference(tx->sta->ptk[tx->sta->ptk_idx]);
+		if (!tx->key)
+			tx->key = rcu_dereference(tx->sdata->default_unicast_key);
+
+		if (!tx->key && test_sta_flag(tx->sta, WLAN_STA_USES_ENCRYPTION))
+			return TX_DROP;
+	} else if (!ieee80211_vif_is_mld(&tx->sdata->vif)) {
+		tx->key = rcu_dereference(tx->sdata->deflink.default_multicast_key);
+	}
+
+	if (!tx->key)
+		return TX_CONTINUE;
+
+	if (unlikely(tx->key->flags & KEY_FLAG_TAINTED))
+		return TX_DROP;
+
+	if (tx->key->flags & KEY_FLAG_UPLOADED_TO_HARDWARE)
+		info->control.hw_key = &tx->key->conf;
+
+	return TX_CONTINUE;
+}
+
 static ieee80211_tx_result debug_noinline
 ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx)
 {
@@ -596,6 +618,9 @@ ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx)
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb);
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx->skb->data;
 
+	if (info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP)
+		return ieee80211_select_key_8023(tx);
+
 	if (unlikely(info->flags & IEEE80211_TX_INTFL_DONT_ENCRYPT)) {
 		tx->key = NULL;
 		return TX_CONTINUE;
@@ -612,12 +637,6 @@ ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx)
 	else
 		tx->key = NULL;
 
-	if (info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) {
-		if (tx->key && tx->key->flags & KEY_FLAG_UPLOADED_TO_HARDWARE)
-			info->control.hw_key = &tx->key->conf;
-		return TX_CONTINUE;
-	}
-
 	if (tx->key) {
 		bool skip_hw = false;
 
@@ -807,6 +826,33 @@ ieee80211_tx_h_rate_ctrl(struct ieee80211_tx_data *tx)
 	return TX_CONTINUE;
 }
 
+static ieee80211_tx_result debug_noinline
+ieee80211_tx_h_check_offchannel(struct ieee80211_tx_data *tx)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx->skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb);
+
+	if (unlikely(info->flags & IEEE80211_TX_CTL_INJECTED))
+		return TX_CONTINUE;
+
+	if (unlikely(test_bit(SCAN_SW_SCANNING, &tx->local->scanning)) &&
+	    test_bit(SDATA_STATE_OFFCHANNEL, &tx->sdata->state) &&
+	    !ieee80211_is_probe_req(hdr->frame_control) &&
+	    !ieee80211_is_any_nullfunc(hdr->frame_control)) {
+		/*
+		 * When software scanning only nullfunc frames (to notify
+		 * the sleep state to the AP) and probe requests (for the
+		 * active scan) are allowed, all other frames should not be
+		 * sent and we should not get here, but if we do
+		 * nonetheless, drop them to avoid sending them
+		 * off-channel. See __ieee80211_start_scan() for more.
+		 */
+		return TX_DROP;
+	}
+
+	return TX_CONTINUE;
+}
+
 static __le16 ieee80211_tx_next_seq(struct sta_info *sta, int tid)
 {
 	u16 *seq = &sta->tid_seq[tid];
@@ -1873,6 +1919,7 @@ static int invoke_tx_handlers_late(struct ieee80211_tx_data *tx)
 		goto txh_done;
 	}
 
+	CALL_TXH(ieee80211_tx_h_check_offchannel);
 	CALL_TXH(ieee80211_tx_h_michael_mic_add);
 	CALL_TXH(ieee80211_tx_h_sequence);
 	CALL_TXH(ieee80211_tx_h_fragment);
@@ -2091,7 +2138,7 @@ static bool ieee80211_validate_radiotap_len(struct sk_buff *skb)
 		(struct ieee80211_radiotap_header *)skb->data;
 
 	/* check for not even having the fixed radiotap header part */
-	if (unlikely(skb->len < sizeof(struct ieee80211_radiotap_header)))
+	if (unlikely(skb_headlen(skb) < sizeof(struct ieee80211_radiotap_header)))
 		return false; /* too short to be possibly valid */
 
 	/* is it a header version we can trust to find length from? */
@@ -2099,7 +2146,7 @@ static bool ieee80211_validate_radiotap_len(struct sk_buff *skb)
 		return false; /* only version 0 is supported */
 
 	/* does the skb contain enough to deliver on the alleged length? */
-	if (unlikely(skb->len < ieee80211_get_radiotap_len(skb->data)))
+	if (unlikely(skb_headlen(skb) < ieee80211_get_radiotap_len(skb->data)))
 		return false; /* skb too short for claimed rt header extent */
 
 	return true;
@@ -2127,7 +2174,8 @@ static bool ieee80211_rate_bw_usable(u16 rate_flags,
 
 bool ieee80211_parse_tx_radiotap(struct sk_buff *skb,
 				 struct net_device *dev,
-				 const struct cfg80211_chan_def *chandef)
+				 const struct cfg80211_chan_def *chandef,
+				 bool trim_fcs)
 {
 	struct ieee80211_local *local = wdev_priv(dev->ieee80211_ptr);
 	struct ieee80211_radiotap_iterator iterator;
@@ -2184,7 +2232,8 @@ bool ieee80211_parse_tx_radiotap(struct sk_buff *skb,
 				if (skb->len < (iterator._max_length + FCS_LEN))
 					return false;
 
-				skb_trim(skb, skb->len - FCS_LEN);
+				if (trim_fcs)
+					skb_trim(skb, skb->len - FCS_LEN);
 			}
 			if (*iterator.this_arg & IEEE80211_RADIOTAP_F_WEP)
 				info->flags &= ~IEEE80211_TX_INTFL_DONT_ENCRYPT;
@@ -2388,13 +2437,13 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb,
 	skb_set_network_header(skb, len_rthdr);
 	skb_set_transport_header(skb, len_rthdr);
 
-	if (skb->len < len_rthdr + 2)
+	if (skb_headlen(skb) < len_rthdr + 2)
 		goto fail;
 
 	hdr = (struct ieee80211_hdr *)(skb->data + len_rthdr);
 	hdrlen = ieee80211_hdrlen(hdr->frame_control);
 
-	if (skb->len < len_rthdr + hdrlen)
+	if (skb_headlen(skb) < len_rthdr + hdrlen)
 		goto fail;
 
 	/*
@@ -2402,7 +2451,7 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb,
 	 * carrying a rfc1042 header
 	 */
 	if (ieee80211_is_data(hdr->frame_control) &&
-	    skb->len >= len_rthdr + hdrlen + sizeof(rfc1042_header) + 2) {
+	    skb_headlen(skb) >= len_rthdr + hdrlen + sizeof(rfc1042_header) + 2) {
 		u8 *payload = (u8 *)hdr + hdrlen;
 
 		if (ether_addr_equal(payload, rfc1042_header))
@@ -2503,7 +2552,7 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb,
 	 * selected chandef above to accurately set injection rates and
 	 * retransmissions.
 	 */
-	if (!ieee80211_parse_tx_radiotap(skb, dev, chandef))
+	if (!ieee80211_parse_tx_radiotap(skb, dev, chandef, true))
 		goto fail_rcu;
 
 	/* remove the injection radiotap header */
@@ -2530,9 +2579,16 @@ static inline bool ieee80211_is_tdls_setup(struct sk_buff *skb)
 	       skb->data[14] == WLAN_TDLS_SNAP_RFTYPE;
 }
 
-int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata,
-			    struct sk_buff *skb,
-			    struct sta_info **sta_out)
+/*
+ * Returns an error if the frame should be dropped, otherwise
+ * the *sta_out pointer is filled:
+ *  - valid sta pointer: frame goes to that station
+ *  - NULL: frame will be unicast to a yet unknown station
+ *  - ERR_PTR(-ENOENT): frame will be group addressed
+ */
+static int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata,
+				   struct sk_buff *skb,
+				   struct sta_info **sta_out, bool bss)
 {
 	struct sta_info *sta;
 
@@ -2553,10 +2609,17 @@ int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata,
 			*sta_out = ERR_PTR(-ENOENT);
 			return 0;
 		}
-		sta = sta_info_get_bss(sdata, skb->data);
+		if (bss)
+			sta = sta_info_get_bss(sdata, skb->data);
+		else
+			sta = sta_info_get(sdata, skb->data);
 		break;
 #ifdef CONFIG_MAC80211_MESH
 	case NL80211_IFTYPE_MESH_POINT:
+		if (is_multicast_ether_addr(skb->data)) {
+			*sta_out = ERR_PTR(-ENOENT);
+			return 0;
+		}
 		/* determined much later */
 		*sta_out = NULL;
 		return 0;
@@ -2599,17 +2662,15 @@ int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata,
 		return -EINVAL;
 	}
 
-	*sta_out = sta ?: ERR_PTR(-ENOENT);
+	*sta_out = sta;
 	return 0;
 }
 
-static u16 ieee80211_store_ack_skb(struct ieee80211_local *local,
-				   struct sk_buff *skb,
-				   u32 *info_flags,
-				   u64 cookie)
+static void ieee80211_store_ack_skb(struct ieee80211_local *local,
+				    struct sk_buff *skb, u64 cookie)
 {
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	struct sk_buff *ack_skb;
-	u16 info_id = 0;
 
 	if (skb->sk)
 		ack_skb = skb_clone_sk(skb);
@@ -2626,58 +2687,41 @@ static u16 ieee80211_store_ack_skb(struct ieee80211_local *local,
 		spin_unlock_irqrestore(&local->ack_status_lock, flags);
 
 		if (id >= 0) {
-			info_id = id;
-			*info_flags |= IEEE80211_TX_CTL_REQ_TX_STATUS;
+			info->status_data = id;
+			info->status_data_idr = 1;
+			info->flags |= IEEE80211_TX_CTL_REQ_TX_STATUS;
 			if (cookie)
 				IEEE80211_SKB_CB(ack_skb)->ack.cookie = cookie;
 		} else {
 			kfree_skb(ack_skb);
 		}
 	}
-
-	return info_id;
-}
-
-static void ieee80211_remove_ack_skb(struct ieee80211_local *local, u16 info_id)
-{
-	struct sk_buff *ack_skb;
-	unsigned long flags;
-
-	spin_lock_irqsave(&local->ack_status_lock, flags);
-	ack_skb = idr_remove(&local->ack_status_frames, info_id);
-	spin_unlock_irqrestore(&local->ack_status_lock, flags);
-
-	kfree_skb(ack_skb);
 }
 
 /**
  * ieee80211_build_hdr - build 802.11 header in the given frame
  * @sdata: virtual interface to build the header for
  * @skb: the skb to build the header in
- * @info_flags: skb flags to set
  * @sta: the station pointer
- * @ctrl_flags: info control flags to set
- * @cookie: cookie pointer to fill (if not %NULL)
  *
  * This function takes the skb with 802.3 header and reformats the header to
  * the appropriate IEEE 802.11 header based on which interface the packet is
  * being transmitted on.
  *
- * Note that this function also takes care of the TX status request and
- * potential unsharing of the SKB - this needs to be interleaved with the
- * header building.
+ * Note that this function also takes care of the TX status request. The skb
+ * must not be shared, and its TX info must already be initialised - this
+ * function only fills in what it determines itself.
  *
  * The function requires the read-side RCU lock held
  *
  * Returns: the (possibly reallocated) skb or an ERR_PTR() code
  */
 static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
-					   struct sk_buff *skb, u32 info_flags,
-					   struct sta_info *sta, u32 ctrl_flags,
-					   u64 cookie)
+					   struct sk_buff *skb,
+					   struct sta_info *sta)
 {
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_local *local = sdata->local;
-	struct ieee80211_tx_info *info;
 	int head_need;
 	u16 ethertype, hdrlen,  meshhdrlen = 0;
 	__le16 fc;
@@ -2689,18 +2733,18 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 	bool wme_sta = false, authorized = false;
 	bool tdls_peer;
 	bool multicast;
-	u16 info_id = 0;
 	struct ieee80211_chanctx_conf *chanctx_conf = NULL;
 	enum nl80211_band band;
 	int ret;
-	u8 link_id = u32_get_bits(ctrl_flags, IEEE80211_TX_CTRL_MLO_LINK);
+	u8 link_id = u32_get_bits(info->control.flags,
+				  IEEE80211_TX_CTRL_MLO_LINK);
 
 	if (IS_ERR(sta))
 		sta = NULL;
 
 #ifdef CONFIG_MAC80211_DEBUGFS
 	if (local->force_tx_status)
-		info_flags |= IEEE80211_TX_CTL_REQ_TX_STATUS;
+		info->flags |= IEEE80211_TX_CTL_REQ_TX_STATUS;
 #endif
 
 	/* convert Ethernet header to proper 802.11 header (based on
@@ -2841,7 +2885,7 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 		/* For injected frames, fill RA right away as nexthop lookup
 		 * will be skipped.
 		 */
-		if ((ctrl_flags & IEEE80211_TX_CTRL_SKIP_MPATH_LOOKUP) &&
+		if ((info->control.flags & IEEE80211_TX_CTRL_SKIP_MPATH_LOOKUP) &&
 		    is_zero_ether_addr(hdr.addr1))
 			memcpy(hdr.addr1, skb->data, ETH_ALEN);
 		break;
@@ -2970,34 +3014,6 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 		goto free;
 	}
 
-	if (unlikely(!multicast &&
-		     (sk_requests_wifi_status(skb->sk) ||
-		      ctrl_flags & IEEE80211_TX_CTL_REQ_TX_STATUS)))
-		info_id = ieee80211_store_ack_skb(local, skb, &info_flags,
-						  cookie);
-
-	/*
-	 * If the skb is shared we need to obtain our own copy.
-	 */
-	skb = skb_share_check(skb, GFP_ATOMIC);
-	if (unlikely(!skb)) {
-		/* skb_share_check() already freed the skb */
-		if (info_id)
-			ieee80211_remove_ack_skb(local, info_id);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	/* set this up so failure paths can clean up ack skb */
-	info = IEEE80211_SKB_CB(skb);
-	memset(info, 0, sizeof(*info));
-
-	info->flags = info_flags;
-	if (info_id) {
-		info->status_data = info_id;
-		info->status_data_idr = 1;
-	}
-	info->band = band;
-
 	hdr.frame_control = fc;
 	hdr.duration_id = 0;
 	hdr.seq_ctrl = 0;
@@ -3037,7 +3053,7 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 		head_need = max_t(int, 0, head_need);
 		if (ieee80211_skb_resize(sdata, skb, head_need, ENCRYPT_DATA)) {
 			ret = -ENOMEM;
-			goto free_txskb;
+			goto free;
 		}
 	}
 
@@ -3064,41 +3080,14 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 
 	skb_reset_mac_header(skb);
 
-	if (likely(!cookie)) {
-		ctrl_flags |= u32_encode_bits(link_id,
-					      IEEE80211_TX_CTRL_MLO_LINK);
-	} else {
-		unsigned int pre_conf_link_id;
-
-		/*
-		 * ctrl_flags already have been set by
-		 * ieee80211_tx_control_port(), here
-		 * we just sanity check that
-		 */
-
-		pre_conf_link_id = u32_get_bits(ctrl_flags,
-						IEEE80211_TX_CTRL_MLO_LINK);
-
-		if (pre_conf_link_id != link_id &&
-		    link_id != IEEE80211_LINK_UNSPECIFIED) {
-#ifdef CONFIG_MAC80211_VERBOSE_DEBUG
-			net_info_ratelimited("%s: dropped frame to %pM with bad link ID request (%d vs. %d)\n",
-					     sdata->name, hdr.addr1,
-					     pre_conf_link_id, link_id);
-#endif
-			ret = -EINVAL;
-			goto free_txskb;
-		}
-	}
-
-	info->control.flags = ctrl_flags;
+	info->band = band;
 
+	info->control.flags =
+		u32_replace_bits(info->control.flags, link_id,
+				 IEEE80211_TX_CTRL_MLO_LINK);
 	return skb;
- free_txskb:
-	ieee80211_free_txskb(&local->hw, skb);
-	return ERR_PTR(ret);
  free:
-	kfree_skb(skb);
+	ieee80211_free_txskb(&local->hw, skb);
 	return ERR_PTR(ret);
 }
 
@@ -3905,6 +3894,38 @@ static bool ieee80211_xmit_fast(struct ieee80211_sub_if_data *sdata,
 	return true;
 }
 
+static bool ieee80211_drop_unauth_port(struct ieee80211_tx_data *tx)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb);
+	struct ieee80211_hdr *hdr;
+
+	if (likely(test_sta_flag(tx->sta, WLAN_STA_AUTHORIZED)))
+		return false;
+
+	if (info->flags & IEEE80211_TX_CTL_INJECTED)
+		return false;
+
+	if (ieee80211_vif_is_mesh(&tx->sdata->vif) ||
+	    tx->sdata->vif.type == NL80211_IFTYPE_OCB)
+		return false;
+
+	/* no need to check forwarding here */
+	if (info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP)
+		return !(info->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO);
+
+	hdr = (struct ieee80211_hdr *)tx->skb->data;
+
+	if (!ieee80211_is_data_present(hdr->frame_control) ||
+	    is_multicast_ether_addr(hdr->addr1))
+		return false;
+
+	if ((info->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO) &&
+	    ieee80211_is_our_addr(tx->sdata, hdr->addr2, NULL))
+		return false;
+
+	return true;
+}
+
 struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
 				     struct ieee80211_txq *txq)
 {
@@ -3971,20 +3992,8 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
 
 	if (txq->sta) {
 		tx.sta = container_of(txq->sta, struct sta_info, sta);
-		/*
-		 * Drop unicast frames to unauthorised stations unless they are
-		 * injected frames or EAPOL frames from the local station.
-		 */
-		if (unlikely(!(info->flags & IEEE80211_TX_CTL_INJECTED) &&
-			     ieee80211_is_data_present(hdr->frame_control) &&
-			     !ieee80211_vif_is_mesh(&tx.sdata->vif) &&
-			     tx.sdata->vif.type != NL80211_IFTYPE_OCB &&
-			     !is_multicast_ether_addr(hdr->addr1) &&
-			     !test_sta_flag(tx.sta, WLAN_STA_AUTHORIZED) &&
-			     (!(info->control.flags &
-				IEEE80211_TX_CTRL_PORT_CTRL_PROTO) ||
-			      !ieee80211_is_our_addr(tx.sdata, hdr->addr2,
-						     NULL)))) {
+
+		if (unlikely(ieee80211_drop_unauth_port(&tx))) {
 			I802_DEBUG_INC(local->tx_handlers_drop_unauth_port);
 			ieee80211_free_txskb(&local->hw, skb);
 			goto begin;
@@ -4156,7 +4165,7 @@ struct ieee80211_txq *ieee80211_next_txq(struct ieee80211_hw *hw, u8 ac)
 
 	spin_lock_bh(&local->active_txq_lock[ac]);
 
-	if (!local->schedule_round[ac])
+	if (!local->schedule_open[ac])
 		goto out;
 
  begin:
@@ -4381,12 +4390,12 @@ void ieee80211_txq_schedule_start(struct ieee80211_hw *hw, u8 ac)
 
 	spin_lock_bh(&local->active_txq_lock[ac]);
 
-	if (ieee80211_txq_schedule_airtime_check(local, ac)) {
+	local->schedule_open[ac] =
+		ieee80211_txq_schedule_airtime_check(local, ac);
+	if (local->schedule_open[ac]) {
 		local->schedule_round[ac]++;
 		if (!local->schedule_round[ac])
 			local->schedule_round[ac]++;
-	} else {
-		local->schedule_round[ac] = 0;
 	}
 
 	spin_unlock_bh(&local->active_txq_lock[ac]);
@@ -4396,13 +4405,14 @@ EXPORT_SYMBOL(ieee80211_txq_schedule_start);
 void __ieee80211_subif_start_xmit(struct sk_buff *skb,
 				  struct net_device *dev,
 				  u32 info_flags,
-				  u32 ctrl_flags,
-				  u64 cookie)
+				  u32 ctrl_flags)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
 	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_tx_info *info;
 	struct sta_info *sta;
 	struct sk_buff *next;
+	bool group_addressed;
 	int len = skb->len;
 
 	if (unlikely(!ieee80211_sdata_running(sdata) || skb->len < ETH_HLEN)) {
@@ -4419,9 +4429,11 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb,
 	    ieee80211_mesh_xmit_fast(sdata, skb, ctrl_flags))
 		goto out;
 
-	if (ieee80211_lookup_ra_sta(sdata, skb, &sta))
+	if (ieee80211_lookup_ra_sta(sdata, skb, &sta,
+				    skb->protocol == sdata->control_port_protocol))
 		goto out_free;
 
+	group_addressed = IS_ERR(sta);
 	if (IS_ERR(sta))
 		sta = NULL;
 
@@ -4454,8 +4466,22 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb,
 		if (skb->protocol == sdata->control_port_protocol)
 			ctrl_flags |= IEEE80211_TX_CTRL_SKIP_MPATH_LOOKUP;
 
-		skb = ieee80211_build_hdr(sdata, skb, info_flags,
-					  sta, ctrl_flags, cookie);
+		skb = skb_share_check(skb, GFP_ATOMIC);
+		if (unlikely(!skb)) {
+			kfree_skb_list(next);
+			goto out;
+		}
+
+		info = IEEE80211_SKB_CB(skb);
+		memset(info, 0, sizeof(*info));
+		info->flags = info_flags;
+		info->control.flags = ctrl_flags;
+
+		if (unlikely(!group_addressed &&
+			     sk_requests_wifi_status(skb->sk)))
+			ieee80211_store_ack_skb(local, skb, 0);
+
+		skb = ieee80211_build_hdr(sdata, skb, sta);
 		if (IS_ERR(skb)) {
 			kfree_skb_list(next);
 			goto out;
@@ -4592,7 +4618,7 @@ static void ieee80211_mlo_multicast_tx_one(struct ieee80211_sub_if_data *sdata,
 		return;
 
 	ctrl_flags |= u32_encode_bits(link_id, IEEE80211_TX_CTRL_MLO_LINK);
-	__ieee80211_subif_start_xmit(out, sdata->dev, 0, ctrl_flags, 0);
+	__ieee80211_subif_start_xmit(out, sdata->dev, 0, ctrl_flags);
 }
 
 static void ieee80211_mlo_multicast_tx(struct net_device *dev,
@@ -4607,7 +4633,7 @@ static void ieee80211_mlo_multicast_tx(struct net_device *dev,
 		ctrl_flags |= u32_encode_bits(__ffs(links),
 					      IEEE80211_TX_CTRL_MLO_LINK);
 
-		__ieee80211_subif_start_xmit(skb, sdata->dev, 0, ctrl_flags, 0);
+		__ieee80211_subif_start_xmit(skb, sdata->dev, 0, ctrl_flags);
 		return;
 	}
 
@@ -4648,8 +4674,7 @@ netdev_tx_t ieee80211_subif_start_xmit(struct sk_buff *skb,
 		ieee80211_convert_to_unicast(skb, dev, &queue);
 		while ((skb = __skb_dequeue(&queue)))
 			__ieee80211_subif_start_xmit(skb, dev, 0,
-						     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC,
-						     0);
+						     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC);
 	} else if (ieee80211_vif_is_mld(&sdata->vif) &&
 		   ((sdata->vif.type == NL80211_IFTYPE_AP &&
 		     !ieee80211_hw_check(&sdata->local->hw, MLO_MCAST_MULTI_LINK_TX)) ||
@@ -4659,8 +4684,7 @@ netdev_tx_t ieee80211_subif_start_xmit(struct sk_buff *skb,
 	} else {
 normal:
 		__ieee80211_subif_start_xmit(skb, dev, 0,
-					     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC,
-					     0);
+					     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC);
 	}
 
 	return NETDEV_TX_OK;
@@ -4731,6 +4755,7 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata,
 {
 	struct ieee80211_tx_info *info;
 	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_chanctx_conf *chanctx_conf;
 	struct tid_ampdu_tx *tid_tx = NULL;
 	struct sk_buff *seg, *next;
 	unsigned int skbs = 0, len = 0;
@@ -4740,10 +4765,6 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata,
 	queue = ieee80211_select_queue(sdata, sta, skb);
 	skb_set_queue_mapping(skb, queue);
 
-	if (unlikely(test_bit(SCAN_SW_SCANNING, &local->scanning)) &&
-	    test_bit(SDATA_STATE_OFFCHANNEL, &sdata->state))
-		goto out_free;
-
 	skb = skb_share_check(skb, GFP_ATOMIC);
 	if (unlikely(!skb))
 		return;
@@ -4758,8 +4779,7 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata,
 		if (!test_bit(HT_AGG_STATE_OPERATIONAL, &tid_tx->state)) {
 			/* fall back to non-offload slow path */
 			__ieee80211_subif_start_xmit(skb, dev, 0,
-						     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC,
-						     0);
+						     IEEE80211_TX_CTRL_MLO_LINK_UNSPEC);
 			return;
 		}
 
@@ -4780,6 +4800,16 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata,
 		sdata = container_of(sdata->bss,
 				     struct ieee80211_sub_if_data, u.ap);
 
+	/* MLD transmissions must not rely on the band */
+	if (!ieee80211_vif_is_mld(&sdata->vif)) {
+		chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
+		if (unlikely(!chanctx_conf)) {
+			kfree_skb_list(skb);
+			return;
+		}
+		info->band = chanctx_conf->def.chan->band;
+	}
+
 	info->flags |= IEEE80211_TX_CTL_HW_80211_ENCAP;
 	info->control.vif = &sdata->vif;
 
@@ -4793,12 +4823,8 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata,
 			memcpy(IEEE80211_SKB_CB(seg), info, sizeof(*info));
 	}
 
-	if (unlikely(sk_requests_wifi_status(skb->sk))) {
-		info->status_data = ieee80211_store_ack_skb(local, skb,
-							    &info->flags, 0);
-		if (info->status_data)
-			info->status_data_idr = 1;
-	}
+	if (unlikely(sk_requests_wifi_status(skb->sk)))
+		ieee80211_store_ack_skb(local, skb, 0);
 
 	dev_sw_netstats_tx_add(dev, skbs, len);
 
@@ -4810,11 +4836,6 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata,
 	ieee80211_tpt_led_trig_tx(local, len);
 
 	ieee80211_tx_8023(sdata, skb, sta, false);
-
-	return;
-
-out_free:
-	kfree_skb(skb);
 }
 
 static bool ieee80211_check_mcast_offload(struct ieee80211_sub_if_data *sdata,
@@ -4848,7 +4869,10 @@ static void __ieee80211_subif_start_xmit_8023(struct sk_buff *skb,
 
 	rcu_read_lock();
 
-	if (ieee80211_lookup_ra_sta(sdata, skb, &sta)) {
+	if (unlikely(sdata->control_port_protocol == ehdr->h_proto))
+		goto skip_offload;
+
+	if (ieee80211_lookup_ra_sta(sdata, skb, &sta, false)) {
 		kfree_skb(skb);
 		goto out;
 	}
@@ -4869,8 +4893,7 @@ static void __ieee80211_subif_start_xmit_8023(struct sk_buff *skb,
 		link = &sdata->deflink;
 		key = rcu_dereference(link->default_multicast_key);
 	} else if (unlikely(IS_ERR_OR_NULL(sta) || !sta->uploaded ||
-		   !test_sta_flag(sta, WLAN_STA_AUTHORIZED) ||
-	    sdata->control_port_protocol == ehdr->h_proto)) {
+		   !test_sta_flag(sta, WLAN_STA_AUTHORIZED))) {
 		goto skip_offload;
 	} else {
 		key = rcu_dereference(sta->ptk[sta->ptk_idx]);
@@ -4927,18 +4950,23 @@ ieee80211_build_data_template(struct ieee80211_sub_if_data *sdata,
 		.local = sdata->local,
 		.sdata = sdata,
 	};
+	struct ieee80211_tx_info *info;
 	struct sta_info *sta;
 
 	rcu_read_lock();
 
-	if (ieee80211_lookup_ra_sta(sdata, skb, &sta)) {
+	if (ieee80211_lookup_ra_sta(sdata, skb, &sta, false)) {
 		kfree_skb(skb);
 		skb = ERR_PTR(-EINVAL);
 		goto out;
 	}
 
-	skb = ieee80211_build_hdr(sdata, skb, info_flags, sta,
-				  IEEE80211_TX_CTRL_MLO_LINK_UNSPEC, 0);
+	info = IEEE80211_SKB_CB(skb);
+	memset(info, 0, sizeof(*info));
+	info->flags = info_flags;
+	info->control.flags = IEEE80211_TX_CTRL_MLO_LINK_UNSPEC;
+
+	skb = ieee80211_build_hdr(sdata, skb, sta);
 	if (IS_ERR(skb))
 		goto out;
 
@@ -5002,7 +5030,7 @@ static bool ieee80211_tx_pending_skb(struct ieee80211_local *local,
 		}
 		result = ieee80211_tx(sdata, NULL, skb, true);
 	} else if (info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) {
-		if (ieee80211_lookup_ra_sta(sdata, skb, &sta)) {
+		if (ieee80211_lookup_ra_sta(sdata, skb, &sta, true)) {
 			dev_kfree_skb(skb);
 			return true;
 		}
@@ -6467,7 +6495,8 @@ void ieee80211_unreserve_tid(struct ieee80211_sta *pubsta, u8 tid)
 EXPORT_SYMBOL(ieee80211_unreserve_tid);
 
 void __ieee80211_tx_skb_tid_band(struct ieee80211_sub_if_data *sdata,
-				 struct sk_buff *skb, int tid, int link_id,
+				 struct sk_buff *skb, struct sta_info *sta,
+				 int tid, int link_id,
 				 enum nl80211_band band)
 {
 	const struct ieee80211_hdr *hdr = (void *)skb->data;
@@ -6519,12 +6548,13 @@ void __ieee80211_tx_skb_tid_band(struct ieee80211_sub_if_data *sdata,
 	 */
 	local_bh_disable();
 	IEEE80211_SKB_CB(skb)->band = band;
-	ieee80211_xmit(sdata, NULL, skb);
+	ieee80211_xmit(sdata, sta, skb);
 	local_bh_enable();
 }
 
 void ieee80211_tx_skb_tid(struct ieee80211_sub_if_data *sdata,
-			  struct sk_buff *skb, int tid, int link_id)
+			  struct sk_buff *skb, struct sta_info *sta,
+			  int tid, int link_id)
 {
 	struct ieee80211_chanctx_conf *chanctx_conf;
 	enum nl80211_band band;
@@ -6539,7 +6569,7 @@ void ieee80211_tx_skb_tid(struct ieee80211_sub_if_data *sdata,
 			rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
 		if (WARN_ON(!chanctx_conf)) {
 			rcu_read_unlock();
-			kfree_skb(skb);
+			ieee80211_free_txskb(&sdata->local->hw, skb);
 			return;
 		}
 		band = chanctx_conf->def.chan->band;
@@ -6550,7 +6580,7 @@ void ieee80211_tx_skb_tid(struct ieee80211_sub_if_data *sdata,
 		band = 0;
 	}
 
-	__ieee80211_tx_skb_tid_band(sdata, skb, tid, link_id, band);
+	__ieee80211_tx_skb_tid_band(sdata, skb, sta, tid, link_id, band);
 	rcu_read_unlock();
 }
 
@@ -6561,6 +6591,8 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev,
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
 	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_tx_info *info;
+	bool group_addressed;
 	struct sta_info *sta;
 	struct sk_buff *skb;
 	struct ethhdr *ehdr;
@@ -6585,9 +6617,6 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev,
 	if (unencrypted)
 		flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT;
 
-	if (cookie)
-		ctrl_flags |= IEEE80211_TX_CTL_REQ_TX_STATUS;
-
 	flags |= IEEE80211_TX_INTFL_NL80211_FRAME_TX;
 
 	skb = dev_alloc_skb(local->hw.extra_tx_headroom +
@@ -6604,15 +6633,11 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev,
 
 	/* we may override the SA for MLO STA later */
 	if (link_id < 0) {
-		ctrl_flags |= u32_encode_bits(IEEE80211_LINK_UNSPECIFIED,
-					      IEEE80211_TX_CTRL_MLO_LINK);
+		link_id = IEEE80211_LINK_UNSPECIFIED;
 		memcpy(ehdr->h_source, sdata->vif.addr, ETH_ALEN);
 	} else {
 		struct ieee80211_bss_conf *link_conf;
 
-		ctrl_flags |= u32_encode_bits(link_id,
-					      IEEE80211_TX_CTRL_MLO_LINK);
-
 		rcu_read_lock();
 		link_conf = rcu_dereference(sdata->vif.link_conf[link_id]);
 		if (!link_conf) {
@@ -6631,39 +6656,59 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev,
 	skb_reset_network_header(skb);
 	skb_reset_mac_header(skb);
 
-	if (local->hw.queues < IEEE80211_NUM_ACS)
-		goto start_xmit;
-
 	/* update QoS header to prioritize control port frames if possible,
 	 * prioritization also happens for control port frames send over
 	 * AF_PACKET
 	 */
 	rcu_read_lock();
-	err = ieee80211_lookup_ra_sta(sdata, skb, &sta);
+	err = ieee80211_lookup_ra_sta(sdata, skb, &sta, true);
 	if (err) {
 		dev_kfree_skb(skb);
 		rcu_read_unlock();
 		return err;
 	}
 
-	if (!IS_ERR(sta)) {
-		u16 queue = ieee80211_select_queue(sdata, sta, skb);
+	group_addressed = IS_ERR(sta);
+	if (IS_ERR(sta))
+		sta = NULL;
 
-		skb_set_queue_mapping(skb, queue);
+	skb_set_queue_mapping(skb, ieee80211_select_queue(sdata, sta, skb));
 
-		/*
-		 * for MLO STA, the SA should be the AP MLD address, but
-		 * the link ID has been selected already
-		 */
-		if (sta && sta->sta.mlo)
-			memcpy(ehdr->h_source, sdata->vif.addr, ETH_ALEN);
+	/*
+	 * for MLO STA, the SA should be the AP MLD address, but
+	 * the link ID has been selected already
+	 */
+	if (sta && sta->sta.mlo)
+		memcpy(ehdr->h_source, sdata->vif.addr, ETH_ALEN);
+
+	info = IEEE80211_SKB_CB(skb);
+	memset(info, 0, sizeof(*info));
+	info->flags = flags;
+	ctrl_flags |= u32_encode_bits(link_id, IEEE80211_TX_CTRL_MLO_LINK);
+	info->control.flags = ctrl_flags;
+
+	if (!group_addressed)
+		ieee80211_store_ack_skb(local, skb, cookie);
+
+	skb = ieee80211_build_hdr(sdata, skb, sta);
+	if (IS_ERR(skb)) {
+		rcu_read_unlock();
+		return PTR_ERR(skb);
+	}
+
+	if (link_id != u32_get_bits(info->control.flags,
+				    IEEE80211_TX_CTRL_MLO_LINK)) {
+		ieee80211_free_txskb(&local->hw, skb);
+		rcu_read_unlock();
+		return -EINVAL;
 	}
-	rcu_read_unlock();
 
-start_xmit:
 	local_bh_disable();
-	__ieee80211_subif_start_xmit(skb, skb->dev, flags, ctrl_flags, cookie);
+	dev_sw_netstats_tx_add(dev, 1, skb->len);
+	ieee80211_tpt_led_trig_tx(local, skb->len);
+	ieee80211_xmit(sdata, sta, skb);
 	local_bh_enable();
+	rcu_read_unlock();
 
 	return 0;
 }
@@ -6691,8 +6736,7 @@ int ieee80211_probe_mesh_link(struct wiphy *wiphy, struct net_device *dev,
 
 	local_bh_disable();
 	__ieee80211_subif_start_xmit(skb, skb->dev, 0,
-				     IEEE80211_TX_CTRL_SKIP_MPATH_LOOKUP,
-				     0);
+				     IEEE80211_TX_CTRL_SKIP_MPATH_LOOKUP);
 	local_bh_enable();
 
 	return 0;
diff --git a/net/mac80211/util.c b/net/mac80211/util.c
index a96078a6bfa299..9e1749d2e427c9 100644
--- a/net/mac80211/util.c
+++ b/net/mac80211/util.c
@@ -47,7 +47,7 @@ struct ieee80211_hw *wiphy_to_ieee80211_hw(struct wiphy *wiphy)
 EXPORT_SYMBOL(wiphy_to_ieee80211_hw);
 
 const struct ieee80211_conn_settings ieee80211_conn_settings_unlimited = {
-	.mode = IEEE80211_CONN_MODE_EHT,
+	.mode = IEEE80211_CONN_MODE_HIGHEST,
 	.bw_limit = IEEE80211_CONN_BW_LIMIT_320,
 };
 
@@ -1283,6 +1283,28 @@ static int ieee80211_put_preq_ies_band(struct sk_buff *skb,
 	if (band == NL80211_BAND_S1GHZ)
 		return ieee80211_put_s1g_cap(skb, &sband->s1g_cap);
 
+	/*
+	 * For EPP minimal content probes, restrict supported rates:
+	 * - 2 GHz: only 1, 2, 5.5, 6, 11, 12, 24 Mb/s
+	 * - 5/6 GHz: only 6, 12, 24 Mb/s
+	 */
+	if (flags & IEEE80211_PROBE_FLAG_MIN_CONTENT) {
+		for (i = 0; i < sband->n_bitrates; i++) {
+			u16 rate = sband->bitrates[i].bitrate;
+
+			if (band == NL80211_BAND_2GHZ) {
+				if (rate != 10 && rate != 20 &&
+				    rate != 55 && rate != 60 &&
+				    rate != 110 && rate != 120 &&
+				    rate != 240)
+					rate_mask &= ~BIT(i);
+			} else {
+				if (rate != 60 && rate != 120 && rate != 240)
+					rate_mask &= ~BIT(i);
+			}
+		}
+	}
+
 	err = ieee80211_put_srates_elem(skb, sband, 0,
 					~rate_mask, WLAN_EID_SUPP_RATES);
 	if (err)
@@ -4680,6 +4702,25 @@ int ieee80211_put_uhr_cap(struct sk_buff *skb,
 	return 0;
 }
 
+int ieee80211_put_cip_cap(struct sk_buff *skb,
+			  struct ieee80211_sub_if_data *sdata)
+{
+	const struct wiphy_iftype_ext_capab *ift_ext_capa =
+		cfg80211_get_iftype_ext_capa(sdata->local->hw.wiphy,
+					     ieee80211_vif_type_p2p(&sdata->vif));
+	u8 cip_capabilities = ift_ext_capa ? ift_ext_capa->cip_capabilities : 0;
+
+	if (skb_tailroom(skb) < 4)
+		return -ENOBUFS;
+
+	skb_put_u8(skb, WLAN_EID_EXTENSION);
+	skb_put_u8(skb, 2);
+	skb_put_u8(skb, WLAN_EID_EXT_CIP_CAPA);
+	skb_put_u8(skb, cip_capabilities);
+
+	return 0;
+}
+
 const char *ieee80211_conn_mode_str(enum ieee80211_conn_mode mode)
 {
 	static const char * const modes[] = {
diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c
index 31353795fa240a..f437b8daa00d0c 100644
--- a/net/mac802154/iface.c
+++ b/net/mac802154/iface.c
@@ -406,8 +406,9 @@ static int ieee802154_header_create(struct sk_buff *skb,
 	return hlen;
 }
 
-static const struct wpan_dev_header_ops ieee802154_header_ops = {
-	.create		= ieee802154_header_create,
+static const struct wpan_dev_ops ieee802154_ops = {
+	.header_create	= ieee802154_header_create,
+	.ioctl		= mac802154_wpan_ioctl,
 };
 
 /* This header create functionality assumes a 8 byte array for
@@ -497,7 +498,6 @@ static const struct net_device_ops mac802154_wpan_ops = {
 	.ndo_open		= mac802154_wpan_open,
 	.ndo_stop		= mac802154_slave_close,
 	.ndo_start_xmit		= ieee802154_subif_start_xmit,
-	.ndo_do_ioctl		= mac802154_wpan_ioctl,
 	.ndo_set_mac_address	= mac802154_wpan_mac_addr,
 };
 
@@ -521,7 +521,7 @@ static void ieee802154_if_setup(struct net_device *dev)
 
 	/* Let hard_header_len set to IEEE802154_MIN_HEADER_LEN. AF_PACKET
 	 * will not send frames without any payload, but ack frames
-	 * has no payload, so substract one that we can send a 3 bytes
+	 * has no payload, so subtract one that we can send a 3 bytes
 	 * frame. The xmit callback assumes at least a hard header where two
 	 * bytes fc and sequence field are set.
 	 */
@@ -583,7 +583,7 @@ ieee802154_setup_sdata(struct ieee802154_sub_if_data *sdata,
 		sdata->dev->netdev_ops = &mac802154_wpan_ops;
 		sdata->dev->ml_priv = &mac802154_mlme_wpan;
 		sdata->iface_default_filtering = IEEE802154_FILTERING_4_FRAME_FIELDS;
-		wpan_dev->header_ops = &ieee802154_header_ops;
+		wpan_dev->ops = &ieee802154_ops;
 
 		mutex_init(&sdata->sec_mtx);
 
diff --git a/net/mctp/device.c b/net/mctp/device.c
index 822120e860c82a..9f76ff092559ab 100644
--- a/net/mctp/device.c
+++ b/net/mctp/device.c
@@ -344,13 +344,13 @@ static struct mctp_dev *mctp_add_dev(struct net_device *dev)
 
 	mdev->net = mctp_default_net(dev_net(dev));
 
-	/* associate to net_device */
 	refcount_set(&mdev->refs, 1);
-	rcu_assign_pointer(dev->mctp_ptr, mdev);
-
 	dev_hold(dev);
 	mdev->dev = dev;
 
+	/* Associate to net_device, publish the mdev. */
+	rcu_assign_pointer(dev->mctp_ptr, mdev);
+
 	return mdev;
 }
 
diff --git a/net/mctp/route.c b/net/mctp/route.c
index b19c63a5691a87..e7c95eeacb486f 100644
--- a/net/mctp/route.c
+++ b/net/mctp/route.c
@@ -825,7 +825,7 @@ static struct mctp_sk_key *mctp_lookup_prealloc_tag(struct mctp_sock *msk,
 
 	spin_lock_irqsave(&mns->keys_lock, flags);
 
-	hlist_for_each_entry(tmp, &mns->keys, hlist) {
+	hlist_for_each_entry(tmp, &msk->keys, sklist) {
 		if (tmp->net != netid)
 			continue;
 
diff --git a/net/mptcp/options.c b/net/mptcp/options.c
index ce0de02f5a3a15..c2f4d7bd080cb7 100644
--- a/net/mptcp/options.c
+++ b/net/mptcp/options.c
@@ -374,10 +374,10 @@ static void mptcp_parse_option(const struct sk_buff *skb,
 			break;
 
 		ptr += 2;
-		mp_opt->rcvr_key = get_unaligned_be64(ptr);
+		mp_opt->fc_recv_key = get_unaligned_be64(ptr);
 		ptr += 8;
 		mp_opt->suboptions |= OPTION_MPTCP_FASTCLOSE;
-		pr_debug("MP_FASTCLOSE: recv_key=%llu\n", mp_opt->rcvr_key);
+		pr_debug("MP_FASTCLOSE: recv_key=%llu\n", mp_opt->fc_recv_key);
 		break;
 
 	case MPTCPOPT_RST:
@@ -1258,7 +1258,7 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
 
 	if (unlikely(mp_opt.suboptions != OPTION_MPTCP_DSS)) {
 		if ((mp_opt.suboptions & OPTION_MPTCP_FASTCLOSE) &&
-		    READ_ONCE(msk->local_key) == mp_opt.rcvr_key) {
+		    READ_ONCE(msk->local_key) == mp_opt.fc_recv_key) {
 			WRITE_ONCE(msk->rcv_fastclose, true);
 			mptcp_schedule_work((struct sock *)msk);
 			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFASTCLOSERX);
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index 0384d6a023f9d8..29b03275a59ee9 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -145,13 +145,13 @@ static inline bool before64(__u64 seq1, __u64 seq2)
 #define after64(seq2, seq1)	before64(seq1, seq2)
 
 struct mptcp_options_received {
-	u64	sndr_key;
-	u64	rcvr_key;
-	u64	data_ack;
-	u64	data_seq;
-	u32	subflow_seq;
-	u16	data_len;
-	__sum16	csum;
+	struct { /* DSS, also used by MP_CAPABLE with data */
+		u64	data_ack;
+		u64	data_seq;
+		u32	subflow_seq;
+		u16	data_len;
+		__sum16	csum;
+	};
 	struct_group(status,
 		u16 suboptions;
 		u16 use_map:1,
@@ -167,15 +167,29 @@ struct mptcp_options_received {
 		    deny_join_id0:1,
 		    __unused:2;
 	);
-	u8	join_id;
-	u32	token;
-	u32	nonce;
-	u64	thmac;
-	u8	hmac[MPTCPOPT_HMAC_LEN];
-	struct mptcp_addr_info addr;
 	struct mptcp_rm_list rm_list;
-	u64	ahmac;
-	u64	fail_seq;
+	/* Options below are mutually exclusive, see mptcp_parse_option() */
+	union {
+		struct { /* MP_CAPABLE */
+			u64	sndr_key;
+			u64	rcvr_key;
+		};
+		struct { /* MP_JOIN */
+			u32	nonce;
+			u8	join_id;
+			union {
+				u32	token;			 /* SYN */
+				u64	thmac;			 /* SYN + ACK */
+				u8	hmac[MPTCPOPT_HMAC_LEN]; /* ACK */
+			};
+		};
+		struct { /* ADD_ADDR */
+			struct mptcp_addr_info addr;
+			u64	ahmac;
+		};
+		u64	fail_seq;	/* MP_FAIL */
+		u64	fc_recv_key;	/* MP_FASTCLOSE */
+	};
 };
 
 static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field)
@@ -593,7 +607,6 @@ struct mptcp_subflow_context {
 	bool	fully_established;  /* path validated */
 	u32	lent_mem_frag;
 	u32	remote_nonce;
-	u64	thmac;
 	u32	local_nonce;
 	u32	remote_token;
 	union {
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index f0a6725d2c3762..14aa82647c3d16 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -408,20 +408,21 @@ static struct dst_entry *subflow_v6_route_req(const struct sock *sk,
 #endif
 
 /* validate received truncated hmac and create hmac for third ACK */
-static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
+static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow,
+				u64 thmac)
 {
 	u8 hmac[SHA256_DIGEST_SIZE];
-	u64 thmac;
+	u64 expected_thmac;
 
 	subflow_generate_hmac(subflow->remote_key, subflow->local_key,
 			      subflow->remote_nonce, subflow->local_nonce,
 			      hmac);
 
-	thmac = get_unaligned_be64(hmac);
-	pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
-		 subflow, subflow->token, thmac, subflow->thmac);
+	expected_thmac = get_unaligned_be64(hmac);
+	pr_debug("subflow=%p, token=%u, expected_thmac=%llu, thmac=%llu\n",
+		 subflow, subflow->token, expected_thmac, thmac);
 
-	return thmac == subflow->thmac;
+	return expected_thmac == thmac;
 }
 
 void mptcp_subflow_reset(struct sock *ssk)
@@ -575,14 +576,13 @@ static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
 		}
 
 		subflow->backup = mp_opt.backup;
-		subflow->thmac = mp_opt.thmac;
 		subflow->remote_nonce = mp_opt.nonce;
 		WRITE_ONCE(subflow->remote_id, mp_opt.join_id);
 		pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u backup=%d\n",
-			 subflow, subflow->thmac, subflow->remote_nonce,
+			 subflow, mp_opt.thmac, subflow->remote_nonce,
 			 subflow->backup);
 
-		if (!subflow_thmac_valid(subflow)) {
+		if (!subflow_thmac_valid(subflow, mp_opt.thmac)) {
 			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKMAC);
 			subflow->reset_reason = MPTCP_RST_EMPTCP;
 			goto do_reset;
diff --git a/net/ncsi/ncsi-rsp.c b/net/ncsi/ncsi-rsp.c
index fbd84bc8026a39..9c91872a2921f3 100644
--- a/net/ncsi/ncsi-rsp.c
+++ b/net/ncsi/ncsi-rsp.c
@@ -884,10 +884,7 @@ static int ncsi_rsp_handler_gp(struct ncsi_request *nr)
 	spin_lock_irqsave(&nc->lock, flags);
 	bitmap = &ncmf->bitmap;
 	for (i = 0; i < rsp->mac_cnt; i++, pdata += 6) {
-		if (!(enable & (0x1 << i)))
-			clear_bit(i, bitmap);
-		else
-			set_bit(i, bitmap);
+		assign_bit(i, bitmap, enable & (0x1 << i));
 
 		memcpy(&ncmf->addrs[i * ETH_ALEN], pdata, ETH_ALEN);
 	}
@@ -899,10 +896,7 @@ static int ncsi_rsp_handler_gp(struct ncsi_request *nr)
 	bitmap = &ncvf->bitmap;
 	spin_lock_irqsave(&nc->lock, flags);
 	for (i = 0; i < rsp->vlan_cnt; i++, pdata += 2) {
-		if (!(enable & (0x1 << i)))
-			clear_bit(i, bitmap);
-		else
-			set_bit(i, bitmap);
+		assign_bit(i, bitmap, enable & (0x1 << i));
 
 		ncvf->vids[i] = ntohs(*(__be16 *)pdata);
 	}
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index 675a1034b3400a..940dea2663e955 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -386,6 +386,15 @@ static void nf_static_key_dec(const struct nf_hook_ops *reg, int pf)
 #endif
 }
 
+static void bump_hook_base_seq(struct net *net)
+{
+	unsigned int base_seq = READ_ONCE(net->nf.hook_base_seq);
+
+	while (++base_seq == 0)
+		;
+	smp_store_release(&net->nf.hook_base_seq, base_seq);
+}
+
 static int __nf_register_net_hook(struct net *net, int pf,
 				  const struct nf_hook_ops *reg)
 {
@@ -430,6 +439,7 @@ static int __nf_register_net_hook(struct net *net, int pf,
 	if (!IS_ERR(new_hooks)) {
 		hooks_validate(new_hooks);
 		rcu_assign_pointer(*pp, new_hooks);
+		bump_hook_base_seq(net);
 	}
 
 	mutex_unlock(&nf_hook_mutex);
@@ -483,6 +493,7 @@ static void __nf_unregister_net_hook(struct net *net, int pf,
 {
 	struct nf_hook_entries __rcu **pp;
 	struct nf_hook_entries *p;
+	bool found;
 
 	pp = nf_hook_entry_head(net, pf, reg->hooknum, reg->dev);
 	if (!pp)
@@ -496,7 +507,8 @@ static void __nf_unregister_net_hook(struct net *net, int pf,
 		return;
 	}
 
-	if (nf_remove_net_hook(p, reg)) {
+	found = nf_remove_net_hook(p, reg);
+	if (found) {
 #ifdef CONFIG_NETFILTER_INGRESS
 		if (nf_ingress_hook(reg, pf))
 			net_dec_ingress_queue();
@@ -511,6 +523,8 @@ static void __nf_unregister_net_hook(struct net *net, int pf,
 	}
 
 	p = __nf_hook_entries_try_shrink(p, pp);
+	if (found)
+		bump_hook_base_seq(net);
 	mutex_unlock(&nf_hook_mutex);
 	if (!p)
 		return;
@@ -784,6 +798,8 @@ static int __net_init netfilter_net_init(struct net *net)
 		return -ENOMEM;
 	}
 #endif
+	net->nf.hook_base_seq = 1;
+	net->nf.nat_hook_base_seq = 1;
 
 	return 0;
 }
diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c
index 0a86a170ba906c..4aeb767088cfad 100644
--- a/net/netfilter/ipset/ip_set_core.c
+++ b/net/netfilter/ipset/ip_set_core.c
@@ -1426,7 +1426,7 @@ static int ip_set_swap(struct sk_buff *skb, const struct nfnl_info *info,
 		return -IPSET_ERR_EXIST_SETNAME2;
 
 	/* Features must not change.
-	 * Not an artifical restriction anymore, as we must prevent
+	 * Not an artificial restriction anymore, as we must prevent
 	 * possible loops created by swapping in setlist type of sets.
 	 */
 	if (!(from->type->features == to->type->features &&
diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c
index ba0957798bad06..fd503f0efb5783 100644
--- a/net/netfilter/ipvs/ip_vs_core.c
+++ b/net/netfilter/ipvs/ip_vs_core.c
@@ -1960,6 +1960,12 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related,
 		/* Ensure the IP header is present in headroom */
 		if (!pskb_may_pull(skb, hlen_orig))
 			goto ignore_tunnel;
+		skb_set_transport_header(skb, hlen_orig);
+		/* Before now we may used ihl from skb frag, revalidate it after
+		 * copying it into skb head to prevent out-of-bounds access
+		 */
+		if (ip_hdr(skb)->ihl * 4 != hlen_orig)
+			goto ignore_tunnel;
 		IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n",
 			&ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
 			type, code, ntohl(info));
diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c
index 5383aeafb0aed4..4dacd5af65db9e 100644
--- a/net/netfilter/ipvs/ip_vs_sync.c
+++ b/net/netfilter/ipvs/ip_vs_sync.c
@@ -1344,7 +1344,7 @@ static void set_mcast_pmtudisc(struct sock *sk, int val)
 }
 
 /*
- *      Specifiy default interface for outgoing multicasts
+ *      Specify default interface for outgoing multicasts
  */
 static int set_mcast_if(struct sock *sk, struct net_device *dev)
 {
diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c
index 4292c3d9addde9..eca7a8d330ce6e 100644
--- a/net/netfilter/nf_conncount.c
+++ b/net/netfilter/nf_conncount.c
@@ -692,7 +692,7 @@ struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen
 	    keylen == 0)
 		return ERR_PTR(-EINVAL);
 
-	data = kvzalloc_obj(*data);
+	data = kvzalloc_obj(*data, GFP_KERNEL_ACCOUNT);
 	if (!data)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
index d0d9e5ea84a095..f9b8927327ba96 100644
--- a/net/netfilter/nf_conntrack_core.c
+++ b/net/netfilter/nf_conntrack_core.c
@@ -1169,7 +1169,6 @@ nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h,
 		return ret;
 
 drop:
-	NF_CT_STAT_INC(net, drop);
 	NF_CT_STAT_INC(net, insert_failed);
 	return NF_DROP;
 }
@@ -1259,7 +1258,6 @@ __nf_conntrack_confirm(struct sk_buff *skb)
 		if (chainlen++ > max_chainlen) {
 chaintoolong:
 			NF_CT_STAT_INC(net, chaintoolong);
-			NF_CT_STAT_INC(net, insert_failed);
 			ret = NF_DROP;
 			goto dying;
 		}
@@ -2067,9 +2065,10 @@ nf_conntrack_in(struct sk_buff *skb, const struct nf_hook_state *state)
 		if (ret == -NF_REPEAT)
 			goto repeat;
 
-		NF_CT_STAT_INC_ATOMIC(state->net, invalid);
 		if (ret == NF_DROP)
 			NF_CT_STAT_INC_ATOMIC(state->net, drop);
+		else
+			NF_CT_STAT_INC_ATOMIC(state->net, invalid);
 
 		ret = -ret;
 		goto out;
diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c
index 579ada063b1bb0..4b6abe36028e39 100644
--- a/net/netfilter/nf_conntrack_netlink.c
+++ b/net/netfilter/nf_conntrack_netlink.c
@@ -911,6 +911,7 @@ struct ctnetlink_filter {
 static const struct nla_policy cta_filter_nla_policy[CTA_FILTER_MAX + 1] = {
 	[CTA_FILTER_ORIG_FLAGS]		= NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL),
 	[CTA_FILTER_REPLY_FLAGS]	= NLA_POLICY_MASK(NLA_U32, CTA_FILTER_F_ALL),
+	[CTA_FILTER_ZONE]		= { .type = NLA_FLAG },
 };
 
 static int ctnetlink_parse_filter(const struct nlattr *attr,
@@ -930,6 +931,9 @@ static int ctnetlink_parse_filter(const struct nlattr *attr,
 	if (tb[CTA_FILTER_REPLY_FLAGS])
 		filter->reply_flags = nla_get_u32(tb[CTA_FILTER_REPLY_FLAGS]);
 
+	if (tb[CTA_FILTER_ZONE])
+		filter->zone_filter = true;
+
 	return 0;
 }
 
@@ -1006,6 +1010,7 @@ ctnetlink_alloc_filter(const struct nlattr * const cda[], u8 family)
 	if (err)
 		goto err_filter;
 
+	/* CTA_ZONE is allowed without CTA_FILTER_ZONE. */
 	if (cda[CTA_ZONE]) {
 		err = ctnetlink_parse_zone(cda[CTA_ZONE], &filter->zone);
 		if (err < 0)
@@ -1020,6 +1025,12 @@ ctnetlink_alloc_filter(const struct nlattr * const cda[], u8 family)
 	if (err < 0)
 		goto err_filter;
 
+	/* CTA_FILTER_ZONE cannot be set without CTA_ZONE. */
+	if (filter->zone_filter && !cda[CTA_ZONE]) {
+		err = -EINVAL;
+		goto err_filter;
+	}
+
 	if (filter->orig_flags) {
 		if (!cda[CTA_TUPLE_ORIG]) {
 			err = -EINVAL;
@@ -1098,13 +1109,13 @@ static int ctnetlink_filter_match_tuple(struct nf_conntrack_tuple *filter_tuple,
 		break;
 	case NFPROTO_IPV6:
 		if ((flags & CTA_FILTER_FLAG(CTA_IP_SRC)) &&
-		    !ipv6_addr_cmp(&filter_tuple->src.u3.in6,
-				   &ct_tuple->src.u3.in6))
+		    !ipv6_addr_equal(&filter_tuple->src.u3.in6,
+				     &ct_tuple->src.u3.in6))
 			return 0;
 
 		if ((flags & CTA_FILTER_FLAG(CTA_IP_DST)) &&
-		    !ipv6_addr_cmp(&filter_tuple->dst.u3.in6,
-				   &ct_tuple->dst.u3.in6))
+		    !ipv6_addr_equal(&filter_tuple->dst.u3.in6,
+				     &ct_tuple->dst.u3.in6))
 			return 0;
 		break;
 	}
@@ -3392,7 +3403,8 @@ static bool expect_iter_name(struct nf_conntrack_expect *exp, void *data)
 	struct nf_conntrack_helper *helper;
 	const char *name = data;
 
-	helper = rcu_dereference(exp->helper);
+	helper = rcu_dereference_protected(exp->helper,
+					   lockdep_is_held(&nf_conntrack_expect_lock));
 	if (!helper)
 		return false;
 
diff --git a/net/netfilter/nf_conntrack_ovs.c b/net/netfilter/nf_conntrack_ovs.c
index b4085af3ad1ce0..fe525b324af41c 100644
--- a/net/netfilter/nf_conntrack_ovs.c
+++ b/net/netfilter/nf_conntrack_ovs.c
@@ -93,7 +93,7 @@ int nf_ct_add_helper(struct nf_conn *ct, const char *name, u8 family,
 	if (!helper)
 		return -EINVAL;
 
-	help = nf_ct_helper_ext_add(ct, GFP_KERNEL);
+	help = nf_ct_helper_ext_add(ct, GFP_KERNEL_ACCOUNT);
 	if (!help) {
 		nf_conntrack_helper_put(helper);
 		return -ENOMEM;
diff --git a/net/netfilter/nf_conntrack_seqadj.c b/net/netfilter/nf_conntrack_seqadj.c
index 220216a4edc513..b7b166a8ad58cf 100644
--- a/net/netfilter/nf_conntrack_seqadj.c
+++ b/net/netfilter/nf_conntrack_seqadj.c
@@ -10,20 +10,19 @@
 int nf_ct_seqadj_init(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 		      s32 off)
 {
+	struct nf_conn_seqadj *seqadj = nfct_seqadj(ct);
 	enum ip_conntrack_dir dir = CTINFO2DIR(ctinfo);
-	struct nf_conn_seqadj *seqadj;
 	struct nf_ct_seqadj *this_way;
 
 	if (off == 0)
 		return 0;
 
-	spin_lock_bh(&ct->lock);
-	seqadj = nfct_seqadj(ct);
-	if (!seqadj) {
-		spin_unlock_bh(&ct->lock);
+	if (unlikely(!seqadj))
 		return 0;
-	}
+
 	set_bit(IPS_SEQ_ADJUST_BIT, &ct->status);
+
+	spin_lock_bh(&ct->lock);
 	this_way = &seqadj->seq[dir];
 	this_way->offset_before	 = off;
 	this_way->offset_after	 = off;
@@ -32,6 +31,23 @@ int nf_ct_seqadj_init(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 }
 EXPORT_SYMBOL_GPL(nf_ct_seqadj_init);
 
+void nf_ct_seqadj_reset(struct nf_conn *ct, enum ip_conntrack_info ctinfo)
+{
+	struct nf_conn_seqadj *seqadj = nfct_seqadj(ct);
+	enum ip_conntrack_dir dir = CTINFO2DIR(ctinfo);
+	struct nf_ct_seqadj *this_way;
+
+	if (unlikely(!seqadj))
+		return;
+
+	spin_lock_bh(&ct->lock);
+	this_way = &seqadj->seq[dir];
+	this_way->offset_before	 = 0;
+	this_way->offset_after	 = 0;
+	spin_unlock_bh(&ct->lock);
+}
+EXPORT_SYMBOL_GPL(nf_ct_seqadj_reset);
+
 int nf_ct_seqadj_set(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 		     __be32 seq, s32 off)
 {
diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c
index f4f2d82192d54e..d88a10230862e5 100644
--- a/net/netfilter/nf_conntrack_standalone.c
+++ b/net/netfilter/nf_conntrack_standalone.c
@@ -1009,7 +1009,7 @@ static int nf_conntrack_standalone_init_sysctl(struct net *net)
 	BUILD_BUG_ON(ARRAY_SIZE(nf_ct_sysctl_table) != NF_SYSCTL_CT_LAST_SYSCTL);
 
 	table = kmemdup(nf_ct_sysctl_table, sizeof(nf_ct_sysctl_table),
-			GFP_KERNEL);
+			GFP_KERNEL_ACCOUNT);
 	if (!table)
 		return -ENOMEM;
 
diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c
index 934c6151f558b1..bd8f9cf394ff1b 100644
--- a/net/netfilter/nf_flow_table_core.c
+++ b/net/netfilter/nf_flow_table_core.c
@@ -737,14 +737,9 @@ static void nf_flow_table_do_cleanup(struct nf_flowtable *flow_table,
 {
 	struct net_device *dev = data;
 
-	if (!dev) {
-		flow_offload_teardown(flow);
-		return;
-	}
-
-	if (net_eq(nf_ct_net(flow->ct), dev_net(dev)) &&
-	    (flow->tuplehash[0].tuple.iifidx == dev->ifindex ||
-	     flow->tuplehash[1].tuple.iifidx == dev->ifindex))
+	if (!dev ||
+	    flow->tuplehash[0].tuple.iifidx == dev->ifindex ||
+	    flow->tuplehash[1].tuple.iifidx == dev->ifindex)
 		flow_offload_teardown(flow);
 }
 
@@ -761,8 +756,10 @@ void nf_flow_table_cleanup(struct net_device *dev)
 	struct nf_flowtable *flowtable;
 
 	mutex_lock(&flowtable_lock);
-	list_for_each_entry(flowtable, &flowtables, list)
-		nf_flow_table_gc_cleanup(flowtable, dev);
+	list_for_each_entry(flowtable, &flowtables, list) {
+		if (net_eq(read_pnet(&flowtable->net), dev_net(dev)))
+			nf_flow_table_gc_cleanup(flowtable, dev);
+	}
 	mutex_unlock(&flowtable_lock);
 }
 EXPORT_SYMBOL_GPL(nf_flow_table_cleanup);
diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c
index 801a3dd9ceea3d..6757fd89c1f1a6 100644
--- a/net/netfilter/nf_flow_table_offload.c
+++ b/net/netfilter/nf_flow_table_offload.c
@@ -995,7 +995,6 @@ static void flow_offload_work_del(struct flow_offload_work *offload)
 	flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL);
 	if (test_bit(NF_FLOW_HW_BIDIRECTIONAL, &offload->flow->flags))
 		flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY);
-	set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags);
 }
 
 static void flow_offload_tuple_stats(struct flow_offload_work *offload,
@@ -1059,6 +1058,12 @@ static void flow_offload_work_handler(struct work_struct *work)
 	}
 
 	clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags);
+	if (offload->cmd == FLOW_CLS_DESTROY) {
+		/* Publish after the worker's last flow access. */
+		smp_mb__before_atomic();
+		set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags);
+	}
+
 	kfree(offload);
 }
 
diff --git a/net/netfilter/nf_hooks_lwtunnel.c b/net/netfilter/nf_hooks_lwtunnel.c
index 4e1eef1ba0f155..40c6fce14a63f7 100644
--- a/net/netfilter/nf_hooks_lwtunnel.c
+++ b/net/netfilter/nf_hooks_lwtunnel.c
@@ -73,7 +73,7 @@ static int __net_init nf_lwtunnel_net_init(struct net *net)
 	if (!net_eq(net, &init_net)) {
 		table = kmemdup(nf_lwtunnel_sysctl_table,
 				sizeof(nf_lwtunnel_sysctl_table),
-				GFP_KERNEL);
+				GFP_KERNEL_ACCOUNT);
 		if (!table)
 			goto err_alloc;
 	}
diff --git a/net/netfilter/nf_log.c b/net/netfilter/nf_log.c
index f4d80654dfe6ab..f1d3649e3fc621 100644
--- a/net/netfilter/nf_log.c
+++ b/net/netfilter/nf_log.c
@@ -493,7 +493,7 @@ static int netfilter_log_sysctl_init(struct net *net)
 	if (!net_eq(net, &init_net)) {
 		table = kmemdup(nf_log_sysctl_table,
 				 sizeof(nf_log_sysctl_table),
-				 GFP_KERNEL);
+				 GFP_KERNEL_ACCOUNT);
 		if (!table)
 			goto err_alloc;
 	} else {
diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c
index a4858c2b2d65a4..5ddd5fc95b24a1 100644
--- a/net/netfilter/nf_nat_core.c
+++ b/net/netfilter/nf_nat_core.c
@@ -1160,6 +1160,15 @@ nfnetlink_parse_nat_setup(struct nf_conn *ct,
 }
 #endif
 
+static void bump_nat_hook_base_seq(struct net *net)
+{
+	unsigned int base_seq = READ_ONCE(net->nf.nat_hook_base_seq);
+
+	while (++base_seq == 0)
+		;
+	smp_store_release(&net->nf.nat_hook_base_seq, base_seq);
+}
+
 static struct nf_ct_helper_expectfn follow_master_nat = {
 	.name		= "nat-follow-master",
 	.expectfn	= nf_nat_follow_master,
@@ -1204,14 +1213,15 @@ int nf_nat_register_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops,
 	if (!nat_proto_net->nat_hook_ops) {
 		WARN_ON(nat_proto_net->users != 0);
 
-		nat_ops = kmemdup_array(orig_nat_ops, ops_count, sizeof(*orig_nat_ops), GFP_KERNEL);
+		nat_ops = kmemdup_array(orig_nat_ops, ops_count, sizeof(*orig_nat_ops),
+					GFP_KERNEL_ACCOUNT);
 		if (!nat_ops) {
 			mutex_unlock(&nf_nat_proto_mutex);
 			return -ENOMEM;
 		}
 
 		for (i = 0; i < ops_count; i++) {
-			priv = kzalloc_obj(*priv);
+			priv = kzalloc_obj(*priv, GFP_KERNEL_ACCOUNT);
 			if (priv) {
 				nat_ops[i].priv = priv;
 				continue;
@@ -1244,6 +1254,7 @@ int nf_nat_register_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops,
 		nat_proto_net->nat_hook_ops = nat_ops;
 
 	nat_proto_net->users++;
+	bump_nat_hook_base_seq(net);
 
 	mutex_unlock(&nf_nat_proto_mutex);
 
@@ -1298,6 +1309,7 @@ void nf_nat_unregister_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops,
 		goto unlock;
 	priv = nat_ops[hooknum].priv;
 	nf_hook_entries_delete_raw(&priv->entries, ops);
+	bump_nat_hook_base_seq(net);
 
 	if (nat_proto_net->users == 0) {
 		nf_unregister_net_hooks(net, nat_ops, ops_count);
diff --git a/net/netfilter/nf_synproxy_core.c b/net/netfilter/nf_synproxy_core.c
index acd360515972f7..401ce5a13f3f4b 100644
--- a/net/netfilter/nf_synproxy_core.c
+++ b/net/netfilter/nf_synproxy_core.c
@@ -336,7 +336,7 @@ static int __net_init synproxy_net_init(struct net *net)
 	struct nf_conn *ct;
 	int err = -ENOMEM;
 
-	ct = nf_ct_tmpl_alloc(net, &nf_ct_zone_dflt, GFP_KERNEL);
+	ct = nf_ct_tmpl_alloc(net, &nf_ct_zone_dflt, GFP_KERNEL_ACCOUNT);
 	if (!ct)
 		goto err1;
 
@@ -686,7 +686,7 @@ ipv4_synproxy_hook(void *priv, struct sk_buff *skb,
 		 * adjustments, they will get initialized once the connection is
 		 * reestablished.
 		 */
-		nf_ct_seqadj_init(ct, ctinfo, 0);
+		nf_ct_seqadj_reset(ct, ctinfo);
 		synproxy->tsoff = 0;
 		this_cpu_inc(snet->stats->conn_reopened);
 		fallthrough;
@@ -1116,7 +1116,7 @@ ipv6_synproxy_hook(void *priv, struct sk_buff *skb,
 		 * adjustments, they will get initialized once the connection is
 		 * reestablished.
 		 */
-		nf_ct_seqadj_init(ct, ctinfo, 0);
+		nf_ct_seqadj_reset(ct, ctinfo);
 		synproxy->tsoff = 0;
 		this_cpu_inc(snet->stats->conn_reopened);
 		fallthrough;
diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
index c0b754a2d45b0a..b59628e6240c1b 100644
--- a/net/netfilter/nf_tables_api.c
+++ b/net/netfilter/nf_tables_api.c
@@ -6995,11 +6995,14 @@ static int nft_setelem_catchall_insert(const struct net *net,
 {
 	struct nft_set_elem_catchall *catchall;
 	u8 genmask = nft_genmask_next(net);
+	u64 tstamp = nft_net_tstamp(net);
 	struct nft_set_ext *ext;
 
 	list_for_each_entry(catchall, &set->catchall_list, list) {
 		ext = nft_set_elem_ext(set, catchall->elem);
-		if (nft_set_elem_active(ext, genmask)) {
+		if (nft_set_elem_active(ext, genmask) &&
+		    !__nft_set_elem_expired(ext, tstamp) &&
+		    !nft_set_elem_is_dead(ext)) {
 			*priv = catchall->elem;
 			return -EEXIST;
 		}
@@ -7092,11 +7095,14 @@ static int nft_setelem_catchall_deactivate(const struct net *net,
 					   struct nft_set_elem *elem)
 {
 	struct nft_set_elem_catchall *catchall;
+	u64 tstamp = nft_net_tstamp(net);
 	struct nft_set_ext *ext;
 
 	list_for_each_entry(catchall, &set->catchall_list, list) {
 		ext = nft_set_elem_ext(set, catchall->elem);
-		if (!nft_is_active_next(net, ext))
+		if (!nft_is_active_next(net, ext) ||
+		    __nft_set_elem_expired(ext, tstamp) ||
+		    nft_set_elem_is_dead(ext))
 			continue;
 
 		kfree(elem->priv);
diff --git a/net/netfilter/nfnetlink_acct.c b/net/netfilter/nfnetlink_acct.c
index 8ff1e0ad5cb0d4..1a366fbd813323 100644
--- a/net/netfilter/nfnetlink_acct.c
+++ b/net/netfilter/nfnetlink_acct.c
@@ -112,7 +112,7 @@ static int nfnl_acct_new(struct sk_buff *skb, const struct nfnl_info *info,
 		size += sizeof(u64);
 	}
 
-	nfacct = kzalloc(sizeof(struct nf_acct) + size, GFP_KERNEL);
+	nfacct = kzalloc(sizeof(struct nf_acct) + size, GFP_KERNEL_ACCOUNT);
 	if (nfacct == NULL)
 		return -ENOMEM;
 
diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c
index 5b6a09f46903f8..a09b2fdbf9b4e4 100644
--- a/net/netfilter/nfnetlink_cthelper.c
+++ b/net/netfilter/nfnetlink_cthelper.c
@@ -335,7 +335,8 @@ static int nfnl_cthelper_update_policy_all(struct nlattr *tb[],
 	struct nf_conntrack_expect_policy *policy;
 	int i, ret = 0;
 
-	new_policy = kmalloc_objs(*new_policy, helper->expect_class_max + 1);
+	new_policy = kmalloc_objs(*new_policy, helper->expect_class_max + 1,
+				  GFP_KERNEL_ACCOUNT);
 	if (!new_policy)
 		return -ENOMEM;
 
diff --git a/net/netfilter/nfnetlink_cttimeout.c b/net/netfilter/nfnetlink_cttimeout.c
index e751f2b9c51d41..c667a29d92f1db 100644
--- a/net/netfilter/nfnetlink_cttimeout.c
+++ b/net/netfilter/nfnetlink_cttimeout.c
@@ -71,7 +71,8 @@ ctnl_timeout_parse_policy(void *timeout,
 	struct nlattr **tb;
 	int ret = 0;
 
-	tb = kzalloc_objs(*tb, l4proto->ctnl_timeout.nlattr_max + 1);
+	tb = kzalloc_objs(*tb, l4proto->ctnl_timeout.nlattr_max + 1,
+			  GFP_KERNEL_ACCOUNT);
 
 	if (!tb)
 		return -ENOMEM;
@@ -150,14 +151,15 @@ static int cttimeout_new_timeout(struct sk_buff *skb,
 		goto err_proto_put;
 	}
 
-	timeout = kzalloc_obj(*timeout);
+	timeout = kzalloc_obj(*timeout, GFP_KERNEL_ACCOUNT);
 	if (timeout == NULL) {
 		ret = -ENOMEM;
 		goto err_proto_put;
 	}
 
 	timeout->timeout = kzalloc(sizeof(*timeout->timeout) +
-				   l4proto->ctnl_timeout.obj_size, GFP_KERNEL);
+				   l4proto->ctnl_timeout.obj_size,
+				   GFP_KERNEL_ACCOUNT);
 	if (!timeout->timeout) {
 		ret = -ENOMEM;
 		goto err;
diff --git a/net/netfilter/nfnetlink_hook.c b/net/netfilter/nfnetlink_hook.c
index 95005e9a606688..b05fd79397c5b2 100644
--- a/net/netfilter/nfnetlink_hook.c
+++ b/net/netfilter/nfnetlink_hook.c
@@ -54,7 +54,6 @@ static int nf_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
 
 struct nfnl_dump_hook_data {
 	char devname[IFNAMSIZ];
-	unsigned long headv;
 	u8 hook;
 };
 
@@ -338,27 +337,47 @@ nfnl_hook_entries_head(u8 pf, unsigned int hook, struct net *net, const char *de
 }
 
 static int nfnl_hook_dump_nat(struct sk_buff *nlskb,
-			      const struct nfnl_dump_hook_data *ctx,
-			      const struct nf_hook_ops *ops,
-			      int family, unsigned int seq)
+			      struct netlink_callback *cb,
+			      const struct nf_hook_ops *ops, int family)
 {
 	struct nf_nat_lookup_hook_priv *priv = ops->priv;
-	struct nf_hook_entries *e = rcu_dereference(priv->entries);
+	struct nfnl_dump_hook_data *ctx = cb->data;
+	struct net *net = sock_net(nlskb->sk);
 	struct nf_hook_ops **nat_ops;
-	int i, err;
+	unsigned int i = cb->args[1];
+	struct nf_hook_entries *e;
+	unsigned int base_seq;
+	int err = 0;
 
+	base_seq = smp_load_acquire(&net->nf.nat_hook_base_seq);
+
+	e = rcu_dereference(priv->entries);
 	if (!e)
-		return 0;
+		goto out;
 
 	nat_ops = nf_hook_entries_get_hook_ops(e);
 
-	for (i = 0; i < e->num_hook_entries; i++) {
-		err = nfnl_hook_dump_one(nlskb, ctx, nat_ops[i],
-					 ops->priority, family, seq);
+	for (; i < e->num_hook_entries; i++) {
+		err = nfnl_hook_dump_one(nlskb, ctx,
+					 READ_ONCE(nat_ops[i]),
+					 ops->priority, family,
+					 cb->nlh->nlmsg_seq);
 		if (err)
-			return err;
+			break;
+
 	}
-	return 0;
+out:
+	if (!err)
+		i = 0;
+	cb->args[1] = i;
+
+	if (cb->args[2] && base_seq != cb->args[2]) {
+		cb->seq++;
+		err = -EINTR;
+	}
+	cb->args[2] = base_seq;
+
+	return err;
 }
 
 static int nfnl_hook_dump(struct sk_buff *nlskb,
@@ -373,35 +392,31 @@ static int nfnl_hook_dump(struct sk_buff *nlskb,
 	unsigned int i = cb->args[0];
 
 	rcu_read_lock();
+	cb->seq = smp_load_acquire(&net->nf.hook_base_seq);
 
 	e = nfnl_hook_entries_head(family, ctx->hook, net, ctx->devname);
-	if (!e)
+	if (!e || IS_ERR(e))
 		goto done;
 
-	if (IS_ERR(e)) {
-		cb->seq++;
-		goto done;
-	}
-
-	if ((unsigned long)e != ctx->headv || i >= e->num_hook_entries)
-		cb->seq++;
-
 	ops = nf_hook_entries_get_hook_ops(e);
 
 	for (; i < e->num_hook_entries; i++) {
-		if (ops[i]->hook_ops_type == NF_HOOK_OP_NAT)
-			err = nfnl_hook_dump_nat(nlskb, ctx, ops[i], family,
-						 cb->nlh->nlmsg_seq);
-		else
-			err = nfnl_hook_dump_one(nlskb, ctx, ops[i],
-						 ops[i]->priority, family,
+		const struct nf_hook_ops *cur = READ_ONCE(ops[i]);
+
+		if (cur->hook_ops_type == NF_HOOK_OP_NAT) {
+			err = nfnl_hook_dump_nat(nlskb, cb, cur, family);
+		} else {
+			err = nfnl_hook_dump_one(nlskb, ctx, cur,
+						 cur->priority, family,
 						 cb->nlh->nlmsg_seq);
+		}
 		if (err)
 			break;
 	}
 
 done:
-	nl_dump_check_consistent(cb, nlmsg_hdr(nlskb));
+	if (nlskb->len > 0)
+		nl_dump_check_consistent(cb, nlmsg_hdr(nlskb));
 	rcu_read_unlock();
 	cb->args[0] = i;
 	return nlskb->len;
@@ -442,10 +457,7 @@ static int nfnl_hook_dump_start(struct netlink_callback *cb)
 		return -ENOMEM;
 
 	strscpy(ctx->devname, name, sizeof(ctx->devname));
-	ctx->headv = (unsigned long)head;
 	ctx->hook = hooknum;
-
-	cb->seq = 1;
 	cb->data = ctx;
 
 	return 0;
diff --git a/net/netfilter/nfnetlink_osf.c b/net/netfilter/nfnetlink_osf.c
index 92002079f8eab6..c8cd6af95db475 100644
--- a/net/netfilter/nfnetlink_osf.c
+++ b/net/netfilter/nfnetlink_osf.c
@@ -36,7 +36,6 @@ static inline int nf_osf_ttl(const struct sk_buff *skb,
 	switch (ttl_check) {
 	case NF_OSF_TTL_TRUE:
 		return ip->ttl == f_ttl;
-		break;
 	case NF_OSF_TTL_NOCHECK:
 		return 1;
 	case NF_OSF_TTL_LESS:
@@ -325,7 +324,7 @@ static int nfnl_osf_add_callback(struct sk_buff *skb,
 	    !memchr(f->version, 0, MAXGENRELEN))
 		return -EINVAL;
 
-	kf = kmalloc_obj(struct nf_osf_finger);
+	kf = kmalloc_obj(struct nf_osf_finger, GFP_KERNEL_ACCOUNT);
 	if (!kf)
 		return -ENOMEM;
 
diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c
index c727668b0c5be7..a3bc0028005109 100644
--- a/net/netfilter/nfnetlink_queue.c
+++ b/net/netfilter/nfnetlink_queue.c
@@ -1593,6 +1593,7 @@ nfqnl_rcv_nl_event(struct notifier_block *this,
 	if (event == NETLINK_URELEASE && n->protocol == NETLINK_NETFILTER) {
 		int i;
 
+		nfnl_lock(NFNL_SUBSYS_QUEUE);
 		/* destroy all instances for this portid */
 		spin_lock(&q->instances_lock);
 		for (i = 0; i < INSTANCE_BUCKETS; i++) {
@@ -1606,6 +1607,7 @@ nfqnl_rcv_nl_event(struct notifier_block *this,
 			}
 		}
 		spin_unlock(&q->instances_lock);
+		nfnl_unlock(NFNL_SUBSYS_QUEUE);
 	}
 	return NOTIFY_DONE;
 }
@@ -1925,9 +1927,9 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info,
 
 	/* Lookup queue under RCU. After peer_portid check (or for new queue
 	 * in BIND case), the queue is owned by the socket sending this message.
-	 * A socket cannot simultaneously send a message and close, so while
-	 * processing this CONFIG message, nfqnl_rcv_nl_event() (triggered by
-	 * socket close) cannot destroy this queue. Safe to use without RCU.
+	 * nfqnl_rcv_nl_event() will block on the nfnl subsys mutex that is
+	 * held by the caller, so the queue cannot be destroyed in parallel,
+	 * even after we drop the RCU read lock.
 	 */
 	rcu_read_lock();
 	queue = instance_lookup(q, queue_num);
diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c
index 9dbf127df9c81e..3c4c2faa73984d 100644
--- a/net/netfilter/nft_ct.c
+++ b/net/netfilter/nft_ct.c
@@ -848,7 +848,8 @@ nft_ct_timeout_parse_policy(void *timeouts,
 	struct nlattr **tb;
 	int ret = 0;
 
-	tb = kzalloc_objs(*tb, l4proto->ctnl_timeout.nlattr_max + 1);
+	tb = kzalloc_objs(*tb, l4proto->ctnl_timeout.nlattr_max + 1,
+			  GFP_KERNEL_ACCOUNT);
 
 	if (!tb)
 		return -ENOMEM;
@@ -934,7 +935,7 @@ static int nft_ct_timeout_obj_init(const struct nft_ctx *ctx,
 	}
 
 	timeout = kzalloc(sizeof(struct nf_ct_timeout) +
-			  l4proto->ctnl_timeout.obj_size, GFP_KERNEL);
+			  l4proto->ctnl_timeout.obj_size, GFP_KERNEL_ACCOUNT);
 	if (timeout == NULL) {
 		ret = -ENOMEM;
 		goto err_proto_put;
diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c
index 978bb0c0110682..a40f9d952184c3 100644
--- a/net/netfilter/nft_set_pipapo.c
+++ b/net/netfilter/nft_set_pipapo.c
@@ -1441,7 +1441,7 @@ static struct nft_pipapo_match *pipapo_clone(struct nft_pipapo_match *old)
 	new->field_count = old->field_count;
 	new->bsize_max = old->bsize_max;
 
-	new->scratch = alloc_percpu(*new->scratch);
+	new->scratch = alloc_percpu_gfp(*new->scratch, GFP_KERNEL_ACCOUNT);
 	if (!new->scratch)
 		goto out_scratch;
 
@@ -2298,14 +2298,14 @@ static int nft_pipapo_init(const struct nft_set *set,
 	if (field_count > NFT_PIPAPO_MAX_FIELDS)
 		return -EINVAL;
 
-	m = kmalloc_flex(*m, f, field_count);
+	m = kmalloc_flex(*m, f, field_count, GFP_KERNEL_ACCOUNT);
 	if (!m)
 		return -ENOMEM;
 
 	m->field_count = field_count;
 	m->bsize_max = 0;
 
-	m->scratch = alloc_percpu(struct nft_pipapo_scratch *);
+	m->scratch = alloc_percpu_gfp(struct nft_pipapo_scratch *, GFP_KERNEL_ACCOUNT);
 	if (!m->scratch) {
 		err = -ENOMEM;
 		goto out_scratch;
diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c
index 9ed288c9d16880..554a96a000f403 100644
--- a/net/netfilter/nft_synproxy.c
+++ b/net/netfilter/nft_synproxy.c
@@ -118,7 +118,8 @@ static void nft_synproxy_do_eval(const struct nft_synproxy *priv,
 		return;
 	}
 
-	if (nf_ip_checksum(skb, nft_hook(pkt), thoff, IPPROTO_TCP)) {
+	if (nf_checksum(skb, nft_hook(pkt), thoff, IPPROTO_TCP,
+			nft_pf(pkt))) {
 		regs->verdict.code = NF_DROP;
 		return;
 	}
diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c
index e78660dfdf4b64..205bc6dd981259 100644
--- a/net/netfilter/xt_CT.c
+++ b/net/netfilter/xt_CT.c
@@ -91,7 +91,7 @@ xt_ct_set_helper(struct nf_conn *ct, const char *helper_name,
 		return -ENOENT;
 	}
 
-	help = nf_ct_helper_ext_add(ct, GFP_KERNEL);
+	help = nf_ct_helper_ext_add(ct, GFP_KERNEL_ACCOUNT);
 	if (help == NULL) {
 		nf_conntrack_helper_put(helper);
 		return -ENOMEM;
@@ -182,7 +182,7 @@ static int xt_ct_tg_check(const struct xt_tgchk_param *par,
 	if (info->flags & XT_CT_ZONE_MARK)
 		zone.flags |= NF_CT_FLAG_MARK;
 
-	ct = nf_ct_tmpl_alloc(par->net, &zone, GFP_KERNEL);
+	ct = nf_ct_tmpl_alloc(par->net, &zone, GFP_KERNEL_ACCOUNT);
 	if (!ct) {
 		ret = -ENOMEM;
 		goto err2;
@@ -190,7 +190,7 @@ static int xt_ct_tg_check(const struct xt_tgchk_param *par,
 
 	if ((info->ct_events || info->exp_events) &&
 	    !nf_ct_ecache_ext_add(ct, info->ct_events, info->exp_events,
-				  GFP_KERNEL)) {
+				  GFP_KERNEL_ACCOUNT)) {
 		ret = -EINVAL;
 		goto err3;
 	}
diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c
index fe7d8d19629b40..71b78b5da698f7 100644
--- a/net/netfilter/xt_IDLETIMER.c
+++ b/net/netfilter/xt_IDLETIMER.c
@@ -147,7 +147,7 @@ static int idletimer_tg_create(struct idletimer_tg_info *info)
 {
 	int ret;
 
-	info->timer = kzalloc_obj(*info->timer);
+	info->timer = kzalloc_obj(*info->timer, GFP_KERNEL_ACCOUNT);
 	if (!info->timer) {
 		ret = -ENOMEM;
 		goto out;
@@ -158,7 +158,7 @@ static int idletimer_tg_create(struct idletimer_tg_info *info)
 		goto out_free_timer;
 
 	sysfs_attr_init(&info->timer->attr.attr);
-	info->timer->attr.attr.name = kstrdup(info->label, GFP_KERNEL);
+	info->timer->attr.attr.name = kstrdup(info->label, GFP_KERNEL_ACCOUNT);
 	if (!info->timer->attr.attr.name) {
 		ret = -ENOMEM;
 		goto out_free_timer;
@@ -196,7 +196,7 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
 {
 	int ret;
 
-	info->timer = kmalloc_obj(*info->timer);
+	info->timer = kmalloc_obj(*info->timer, GFP_KERNEL_ACCOUNT);
 	if (!info->timer) {
 		ret = -ENOMEM;
 		goto out;
@@ -207,7 +207,7 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
 		goto out_free_timer;
 
 	sysfs_attr_init(&info->timer->attr.attr);
-	info->timer->attr.attr.name = kstrdup(info->label, GFP_KERNEL);
+	info->timer->attr.attr.name = kstrdup(info->label, GFP_KERNEL_ACCOUNT);
 	if (!info->timer->attr.attr.name) {
 		ret = -ENOMEM;
 		goto out_free_timer;
diff --git a/net/netfilter/xt_LED.c b/net/netfilter/xt_LED.c
index caaaf4d2c58427..3cbb8d61d41756 100644
--- a/net/netfilter/xt_LED.c
+++ b/net/netfilter/xt_LED.c
@@ -111,11 +111,12 @@ static int led_tg_check(const struct xt_tgchk_param *par)
 	}
 
 	err = -ENOMEM;
-	ledinternal = kzalloc_obj(struct xt_led_info_internal);
+	ledinternal = kzalloc_obj(struct xt_led_info_internal,
+				  GFP_KERNEL_ACCOUNT);
 	if (!ledinternal)
 		goto exit_mutex_only;
 
-	ledinternal->trigger_id = kstrdup(ledinfo->id, GFP_KERNEL);
+	ledinternal->trigger_id = kstrdup(ledinfo->id, GFP_KERNEL_ACCOUNT);
 	if (!ledinternal->trigger_id)
 		goto exit_internal_alloc;
 
diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c
index 91270d467ffdf8..2f6b512b71e80b 100644
--- a/net/netfilter/xt_RATEEST.c
+++ b/net/netfilter/xt_RATEEST.c
@@ -139,7 +139,7 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par)
 	}
 
 	ret = -ENOMEM;
-	est = kzalloc_obj(*est);
+	est = kzalloc_obj(*est, GFP_KERNEL_ACCOUNT);
 	if (!est)
 		goto err1;
 
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5d34ceb893eda0..48b4104f085986 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -106,7 +106,7 @@ static int tee_tg_check(const struct xt_tgchk_param *par)
 		if (info->oif[sizeof(info->oif)-1] != '\0')
 			return -EINVAL;
 
-		priv = kzalloc_obj(*priv);
+		priv = kzalloc_obj(*priv, GFP_KERNEL_ACCOUNT);
 		if (priv == NULL)
 			return -ENOMEM;
 
diff --git a/net/netfilter/xt_hashlimit.c b/net/netfilter/xt_hashlimit.c
index 9af0fa895f7302..57ac455bc33178 100644
--- a/net/netfilter/xt_hashlimit.c
+++ b/net/netfilter/xt_hashlimit.c
@@ -294,7 +294,7 @@ static int htable_create(struct net *net, struct hashlimit_cfg3 *cfg,
 		if (size < 16)
 			size = 16;
 	}
-	hinfo = kvmalloc_flex(*hinfo, hash, size);
+	hinfo = kvmalloc_flex(*hinfo, hash, size, GFP_KERNEL_ACCOUNT);
 	if (hinfo == NULL)
 		return -ENOMEM;
 	*out_hinfo = hinfo;
@@ -319,7 +319,7 @@ static int htable_create(struct net *net, struct hashlimit_cfg3 *cfg,
 	hinfo->count = 0;
 	hinfo->family = family;
 	hinfo->rnd_initialized = false;
-	hinfo->name = kstrdup(name, GFP_KERNEL);
+	hinfo->name = kstrdup(name, GFP_KERNEL_ACCOUNT);
 	if (!hinfo->name) {
 		kvfree(hinfo);
 		return -ENOMEM;
diff --git a/net/netfilter/xt_limit.c b/net/netfilter/xt_limit.c
index 87d74da14c0b76..ad48b6879b1303 100644
--- a/net/netfilter/xt_limit.c
+++ b/net/netfilter/xt_limit.c
@@ -115,7 +115,7 @@ static int limit_mt_check(const struct xt_mtchk_param *par)
 		return -ERANGE;
 	}
 
-	priv = kmalloc_obj(*priv);
+	priv = kmalloc_obj(*priv, GFP_KERNEL_ACCOUNT);
 	if (priv == NULL)
 		return -ENOMEM;
 
diff --git a/net/netfilter/xt_quota.c b/net/netfilter/xt_quota.c
index b05c5c8dac78ba..e21ec152d16a73 100644
--- a/net/netfilter/xt_quota.c
+++ b/net/netfilter/xt_quota.c
@@ -50,7 +50,7 @@ static int quota_mt_check(const struct xt_mtchk_param *par)
 	if (q->flags & ~XT_QUOTA_MASK)
 		return -EINVAL;
 
-	q->master = kmalloc_obj(*q->master);
+	q->master = kmalloc_obj(*q->master, GFP_KERNEL_ACCOUNT);
 	if (q->master == NULL)
 		return -ENOMEM;
 
diff --git a/net/netfilter/xt_recent.c b/net/netfilter/xt_recent.c
index d34831ce3adfbb..55f1f2d89952d0 100644
--- a/net/netfilter/xt_recent.c
+++ b/net/netfilter/xt_recent.c
@@ -391,7 +391,8 @@ static int recent_mt_check(const struct xt_mtchk_param *par,
 		goto out;
 	}
 
-	t = kvzalloc_flex(*t, iphash, ip_list_hash_size);
+	t = kvzalloc_flex(*t, iphash, ip_list_hash_size,
+			  GFP_KERNEL_ACCOUNT);
 	if (t == NULL) {
 		ret = -ENOMEM;
 		goto out;
diff --git a/net/netfilter/xt_statistic.c b/net/netfilter/xt_statistic.c
index 334e09771abf9f..1ffc59d90bb0a8 100644
--- a/net/netfilter/xt_statistic.c
+++ b/net/netfilter/xt_statistic.c
@@ -58,7 +58,7 @@ static int statistic_mt_check(const struct xt_mtchk_param *par)
 	    info->flags & ~XT_STATISTIC_MASK)
 		return -EINVAL;
 
-	info->master = kzalloc_obj(*info->master);
+	info->master = kzalloc_obj(*info->master, GFP_KERNEL_ACCOUNT);
 	if (info->master == NULL)
 		return -ENOMEM;
 	atomic_set(&info->master->count, info->u.nth.count);
diff --git a/net/netfilter/xt_string.c b/net/netfilter/xt_string.c
index 8ce25bc9b2775e..52c1deb02cdc65 100644
--- a/net/netfilter/xt_string.c
+++ b/net/netfilter/xt_string.c
@@ -54,7 +54,7 @@ static int string_mt_check(const struct xt_mtchk_param *par)
 	if (conf->u.v1.flags & XT_STRING_FLAG_IGNORECASE)
 		flags |= TS_IGNORECASE;
 	ts_conf = textsearch_prepare(conf->algo, conf->pattern, conf->patlen,
-				     GFP_KERNEL, flags);
+				     GFP_KERNEL_ACCOUNT, flags);
 	if (IS_ERR(ts_conf))
 		return PTR_ERR(ts_conf);
 
diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index 9fdf964224ab48..6d2530e33540dc 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -293,8 +293,6 @@ static int __netlink_deliver_tap_skb(struct sk_buff *skb,
 	if (!net_eq(dev_net(dev), sock_net(sk)))
 		return 0;
 
-	dev_hold(dev);
-
 	if (is_vmalloc_addr(skb->head))
 		nskb = netlink_to_full_skb(skb, GFP_ATOMIC);
 	else
@@ -310,7 +308,6 @@ static int __netlink_deliver_tap_skb(struct sk_buff *skb,
 			ret = net_xmit_errno(ret);
 	}
 
-	dev_put(dev);
 	return ret;
 }
 
@@ -2719,7 +2716,7 @@ static int netlink_native_seq_show(struct seq_file *seq, void *v)
 {
 	if (v == SEQ_START_TOKEN) {
 		seq_puts(seq,
-			 "sk               Eth Pid        Groups   "
+			 "sk Eth Pid        Groups   "
 			 "Rmem     Wmem     Dump  Locks    Drops    Inode\n");
 	} else {
 		struct sock *s = v;
@@ -2732,8 +2729,7 @@ static int netlink_native_seq_show(struct seq_file *seq, void *v)
 		 */
 		groups = READ_ONCE(nlk->groups);
 
-		seq_printf(seq, "%pK %-3d %-10u %08x %-8d %-8d %-5d %-8d %-8u %-8llu\n",
-			   s,
+		seq_printf(seq, "0  %-3d %-10u %08x %-8d %-8d %-5d %-8d %-8u %-8llu\n",
 			   s->sk_protocol,
 			   nlk->portid,
 			   groups ? (u32)groups[0] : 0,
diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c
index 41d37442f18681..5cc1037d4917e0 100644
--- a/net/netlink/genetlink.c
+++ b/net/netlink/genetlink.c
@@ -1656,7 +1656,7 @@ static void *ctrl_dumppolicy_prep(struct sk_buff *skb,
 }
 
 static int ctrl_dumppolicy_put_op(struct sk_buff *skb,
-				  struct netlink_callback *cb,
+				  struct netlink_callback *cb, u32 cmd,
 				  struct genl_split_ops *doit,
 				  struct genl_split_ops *dumpit)
 {
@@ -1677,7 +1677,7 @@ static int ctrl_dumppolicy_put_op(struct sk_buff *skb,
 	if (!nest_pol)
 		goto err;
 
-	nest_op = nla_nest_start(skb, doit->cmd);
+	nest_op = nla_nest_start(skb, cmd);
 	if (!nest_op)
 		goto err;
 
@@ -1721,7 +1721,8 @@ static int ctrl_dumppolicy(struct sk_buff *skb, struct netlink_callback *cb)
 						      &doit, &dumpit)))
 				return -ENOENT;
 
-			if (ctrl_dumppolicy_put_op(skb, cb, &doit, &dumpit))
+			if (ctrl_dumppolicy_put_op(skb, cb, ctx->op,
+						   &doit, &dumpit))
 				return skb->len;
 
 			/* done with the per-op policy index list */
@@ -1730,6 +1731,7 @@ static int ctrl_dumppolicy(struct sk_buff *skb, struct netlink_callback *cb)
 
 		while (ctx->dump_map) {
 			if (ctrl_dumppolicy_put_op(skb, cb,
+						   ctx->op_iter->cmd,
 						   &ctx->op_iter->doit,
 						   &ctx->op_iter->dumpit))
 				return skb->len;
diff --git a/net/netlink/policy.c b/net/netlink/policy.c
index 08b006c48f06e3..574d44b7d5182f 100644
--- a/net/netlink/policy.c
+++ b/net/netlink/policy.c
@@ -234,6 +234,11 @@ int netlink_policy_dump_attr_size_estimate(const struct nla_policy *pt)
 		/* maximum is common, u64 min/max with padding */
 		return common +
 		       2 * (nla_attr_size(0) + nla_attr_size(sizeof(u64)));
+	case NLA_BE16:
+	case NLA_BE32:
+		/* same as the unsigned types, plus the byte order */
+		return common + nla_attr_size(sizeof(u32)) +
+		       2 * (nla_attr_size(0) + nla_attr_size(sizeof(u64)));
 	case NLA_BITFIELD32:
 		return common + nla_attr_size(sizeof(u32));
 	case NLA_STRING:
@@ -289,21 +294,28 @@ __netlink_policy_dump_write_attr(struct netlink_policy_dump_state *state,
 	case NLA_U16:
 	case NLA_U32:
 	case NLA_U64:
+	case NLA_BE16:
+	case NLA_BE32:
 	case NLA_UINT:
 	case NLA_MSECS: {
 		struct netlink_range_validation range;
 
 		if (pt->type == NLA_U8)
 			type = NL_ATTR_TYPE_U8;
-		else if (pt->type == NLA_U16)
+		else if (pt->type == NLA_U16 || pt->type == NLA_BE16)
 			type = NL_ATTR_TYPE_U16;
-		else if (pt->type == NLA_U32)
+		else if (pt->type == NLA_U32 || pt->type == NLA_BE32)
 			type = NL_ATTR_TYPE_U32;
 		else if (pt->type == NLA_U64)
 			type = NL_ATTR_TYPE_U64;
 		else
 			type = NL_ATTR_TYPE_UINT;
 
+		if ((pt->type == NLA_BE16 || pt->type == NLA_BE32) &&
+		    nla_put_u32(skb, NL_POLICY_TYPE_ATTR_BYTE_ORDER,
+				NL_POLICY_BYTE_ORDER_BIG_ENDIAN))
+			goto nla_put_failure;
+
 		if (pt->validation_type == NLA_VALIDATE_MASK) {
 			if (nla_put_u64_64bit(skb, NL_POLICY_TYPE_ATTR_MASK,
 					      pt->mask,
diff --git a/net/nfc/core.c b/net/nfc/core.c
index a92a6566e6a0d4..f521669293f05f 100644
--- a/net/nfc/core.c
+++ b/net/nfc/core.c
@@ -279,10 +279,10 @@ static struct nfc_target *nfc_find_target(struct nfc_dev *dev, u32 target_idx)
 
 int nfc_dep_link_up(struct nfc_dev *dev, int target_index, u8 comm_mode)
 {
-	int rc = 0;
-	u8 *gb;
-	size_t gb_len;
 	struct nfc_target *target;
+	u8 gb[NFC_MAX_GT_LEN];
+	size_t gb_len = 0;
+	int rc = 0;
 
 	pr_debug("dev_name=%s comm %d\n", dev_name(&dev->dev), comm_mode);
 
@@ -301,7 +301,7 @@ int nfc_dep_link_up(struct nfc_dev *dev, int target_index, u8 comm_mode)
 		goto error;
 	}
 
-	gb = nfc_llcp_general_bytes(dev, &gb_len);
+	nfc_get_local_general_bytes(dev, gb, sizeof(gb), &gb_len);
 	if (gb_len > NFC_MAX_GT_LEN) {
 		rc = -EINVAL;
 		goto error;
@@ -644,11 +644,10 @@ int nfc_set_remote_general_bytes(struct nfc_dev *dev, const u8 *gb, u8 gb_len)
 }
 EXPORT_SYMBOL(nfc_set_remote_general_bytes);
 
-u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, size_t *gb_len)
+u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, u8 *out_gb,
+				size_t gb_max_len, size_t *gb_len)
 {
-	pr_debug("dev_name=%s\n", dev_name(&dev->dev));
-
-	return nfc_llcp_general_bytes(dev, gb_len);
+	return nfc_llcp_general_bytes(dev, out_gb, gb_max_len, gb_len);
 }
 EXPORT_SYMBOL(nfc_get_local_general_bytes);
 
diff --git a/net/nfc/digital_dep.c b/net/nfc/digital_dep.c
index 3982fa084737f7..968547c306a5d8 100644
--- a/net/nfc/digital_dep.c
+++ b/net/nfc/digital_dep.c
@@ -1490,14 +1490,14 @@ static int digital_tg_send_atr_res(struct nfc_digital_dev *ddev,
 				   struct digital_atr_req *atr_req)
 {
 	struct digital_atr_res *atr_res;
+	u8 gb[NFC_MAX_GT_LEN];
 	struct sk_buff *skb;
-	u8 *gb, payload_bits;
+	u8 payload_bits;
 	size_t gb_len;
 	int rc;
 
-	gb = nfc_get_local_general_bytes(ddev->nfc_dev, &gb_len);
-	if (!gb)
-		gb_len = 0;
+	nfc_get_local_general_bytes(ddev->nfc_dev, gb, sizeof(gb),
+				    &gb_len);
 
 	skb = digital_skb_alloc(ddev, sizeof(struct digital_atr_res) + gb_len);
 	if (!skb)
diff --git a/net/nfc/llcp.h b/net/nfc/llcp.h
index d8345ed57c9542..23ae7a0112d37d 100644
--- a/net/nfc/llcp.h
+++ b/net/nfc/llcp.h
@@ -91,6 +91,7 @@ struct nfc_llcp_local {
 	struct hlist_head pending_sdreqs;
 	struct timer_list sdreq_timer;
 	struct work_struct sdreq_timeout_work;
+	struct work_struct release_work;
 	u8 sdreq_next_tid;
 
 	/* sockets array */
diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c
index ca89fe967d6a27..80a00938c86965 100644
--- a/net/nfc/llcp_commands.c
+++ b/net/nfc/llcp_commands.c
@@ -135,7 +135,7 @@ struct nfc_llcp_sdp_tlv *nfc_llcp_build_sdreq_tlv(u8 tid, const char *uri,
 {
 	struct nfc_llcp_sdp_tlv *sdreq;
 
-	pr_debug("uri: %s, len: %zu\n", uri, uri_len);
+	pr_debug("uri: %.*s, len: %zu\n", (int)uri_len, uri, uri_len);
 
 	/* sdreq->tlv_len is u8, takes uri_len, + 3 for header, + 1 for NULL */
 	if (WARN_ON_ONCE(uri_len > U8_MAX - 4))
diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c
index cac1b5487064d0..74bf817007cf27 100644
--- a/net/nfc/llcp_core.c
+++ b/net/nfc/llcp_core.c
@@ -20,6 +20,8 @@ static LIST_HEAD(llcp_devices);
 /* Protects llcp_devices list */
 static DEFINE_SPINLOCK(llcp_devices_lock);
 
+static struct workqueue_struct *llcp_wq;
+
 static void nfc_llcp_rx_skb(struct nfc_llcp_local *local, struct sk_buff *skb);
 
 void nfc_llcp_sock_link(struct llcp_sock_list *l, struct sock *sk)
@@ -63,21 +65,33 @@ static void nfc_llcp_socket_purge(struct nfc_llcp_sock *sock)
 	}
 }
 
+static struct sock *nfc_llcp_sock_list_pop(struct llcp_sock_list *l)
+{
+	struct sock *sk;
+
+	write_lock(&l->lock);
+	sk = sk_head(&l->head);
+	if (sk) {
+		sock_hold(sk);
+		sk_del_node_init(sk);
+	}
+	write_unlock(&l->lock);
+
+	return sk;
+}
+
 static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device,
 				    int err)
 {
 	struct sock *sk;
-	struct hlist_node *tmp;
 	struct nfc_llcp_sock *llcp_sock;
 
 	skb_queue_purge(&local->tx_queue);
 
-	write_lock(&local->sockets.lock);
-
-	sk_for_each_safe(sk, tmp, &local->sockets.head) {
+	while ((sk = nfc_llcp_sock_list_pop(&local->sockets))) {
 		llcp_sock = nfc_llcp_sock(sk);
 
-		bh_lock_sock(sk);
+		lock_sock(sk);
 
 		nfc_llcp_socket_purge(llcp_sock);
 
@@ -91,17 +105,27 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device,
 			list_for_each_entry_safe(lsk, n,
 						 &llcp_sock->accept_queue,
 						 accept_queue) {
-				accept_sk = &lsk->sk;
-				bh_lock_sock(accept_sk);
+				bool put_creation = false;
 
-				nfc_llcp_accept_unlink(accept_sk);
-
-				if (err)
-					accept_sk->sk_err = err;
-				accept_sk->sk_state = LLCP_CLOSED;
-				accept_sk->sk_state_change(sk);
+				accept_sk = &lsk->sk;
+				lock_sock_nested(accept_sk,
+						 SINGLE_DEPTH_NESTING);
+
+				if (nfc_llcp_sock(accept_sk)->parent == sk) {
+					nfc_llcp_accept_unlink(accept_sk);
+					nfc_llcp_sock_unlink(&local->sockets, accept_sk);
+
+					if (err)
+						accept_sk->sk_err = err;
+					accept_sk->sk_state = LLCP_CLOSED;
+					accept_sk->sk_state_change(accept_sk);
+					sock_orphan(accept_sk);
+					put_creation = true;
+				}
 
-				bh_unlock_sock(accept_sk);
+				release_sock(accept_sk);
+				if (put_creation)
+					sock_put(accept_sk); /* creation ref */
 			}
 		}
 
@@ -110,23 +134,18 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device,
 		sk->sk_state = LLCP_CLOSED;
 		sk->sk_state_change(sk);
 
-		bh_unlock_sock(sk);
-
-		sk_del_node_init(sk);
+		release_sock(sk);
+		sock_put(sk);
 	}
 
-	write_unlock(&local->sockets.lock);
-
 	/* If we still have a device, we keep the RAW sockets alive */
 	if (device == true)
 		return;
 
-	write_lock(&local->raw_sockets.lock);
-
-	sk_for_each_safe(sk, tmp, &local->raw_sockets.head) {
+	while ((sk = nfc_llcp_sock_list_pop(&local->raw_sockets))) {
 		llcp_sock = nfc_llcp_sock(sk);
 
-		bh_lock_sock(sk);
+		lock_sock(sk);
 
 		nfc_llcp_socket_purge(llcp_sock);
 
@@ -135,26 +154,20 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device,
 		sk->sk_state = LLCP_CLOSED;
 		sk->sk_state_change(sk);
 
-		bh_unlock_sock(sk);
-
-		sk_del_node_init(sk);
+		release_sock(sk);
+		sock_put(sk);
 	}
-
-	write_unlock(&local->raw_sockets.lock);
 }
 
 static struct nfc_llcp_local *nfc_llcp_local_get(struct nfc_llcp_local *local)
 {
-	/* Since using nfc_llcp_local may result in usage of nfc_dev, whenever
-	 * we hold a reference to local, we also need to hold a reference to
-	 * the device to avoid UAF.
-	 */
-	if (!nfc_get_device(local->dev->idx))
+	if (!local)
 		return NULL;
 
-	kref_get(&local->ref);
+	if (kref_get_unless_zero(&local->ref))
+		return local;
 
-	return local;
+	return NULL;
 }
 
 static void local_cleanup(struct nfc_llcp_local *local)
@@ -172,30 +185,34 @@ static void local_cleanup(struct nfc_llcp_local *local)
 	nfc_llcp_free_sdp_tlv_list(&local->pending_sdreqs);
 }
 
-static void local_release(struct kref *ref)
+static void local_release_work(struct work_struct *work)
 {
 	struct nfc_llcp_local *local;
+	struct nfc_dev *dev;
 
-	local = container_of(ref, struct nfc_llcp_local, ref);
+	local = container_of(work, struct nfc_llcp_local, release_work);
+	dev = local->dev;
 
 	local_cleanup(local);
 	kfree(local);
+	nfc_put_device(dev);
 }
 
-int nfc_llcp_local_put(struct nfc_llcp_local *local)
+static void local_release(struct kref *ref)
 {
-	struct nfc_dev *dev;
-	int ret;
+	struct nfc_llcp_local *local;
 
-	if (local == NULL)
-		return 0;
+	local = container_of(ref, struct nfc_llcp_local, ref);
 
-	dev = local->dev;
+	queue_work(llcp_wq, &local->release_work);
+}
 
-	ret = kref_put(&local->ref, local_release);
-	nfc_put_device(dev);
+int nfc_llcp_local_put(struct nfc_llcp_local *local)
+{
+	if (!local)
+		return 0;
 
-	return ret;
+	return kref_put(&local->ref, local_release);
 }
 
 static struct nfc_llcp_sock *nfc_llcp_sock_get(struct nfc_llcp_local *local,
@@ -341,7 +358,7 @@ static int nfc_llcp_wks_sap(const char *service_name, size_t service_name_len)
 {
 	int sap, num_wks;
 
-	pr_debug("%s\n", service_name);
+	pr_debug("%.*s\n", (int)service_name_len, service_name);
 
 	if (service_name == NULL)
 		return -EINVAL;
@@ -352,7 +369,8 @@ static int nfc_llcp_wks_sap(const char *service_name, size_t service_name_len)
 		if (wks[sap] == NULL)
 			continue;
 
-		if (strncmp(wks[sap], service_name, service_name_len) == 0)
+		if (strlen(wks[sap]) == service_name_len &&
+		    !strncmp(wks[sap], service_name, service_name_len))
 			return sap;
 	}
 
@@ -635,23 +653,32 @@ static int nfc_llcp_build_gb(struct nfc_llcp_local *local)
 	return ret;
 }
 
-u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, size_t *general_bytes_len)
+u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, u8 *out_gb, size_t gb_max_len,
+			   size_t *general_bytes_len)
 {
 	struct nfc_llcp_local *local;
 
+	if (!out_gb || !general_bytes_len)
+		return NULL;
+
 	local = nfc_llcp_find_local(dev);
-	if (local == NULL) {
+	if (!local) {
 		*general_bytes_len = 0;
 		return NULL;
 	}
 
 	nfc_llcp_build_gb(local);
 
-	*general_bytes_len = local->gb_len;
+	if (local->gb_len) {
+		*general_bytes_len = min_t(size_t, local->gb_len, gb_max_len);
+		memcpy(out_gb, local->gb, *general_bytes_len);
+	} else {
+		*general_bytes_len = 0;
+	}
 
 	nfc_llcp_local_put(local);
 
-	return local->gb;
+	return out_gb;
 }
 
 int nfc_llcp_set_remote_gb(struct nfc_dev *dev, const u8 *gb, u8 gb_len)
@@ -1074,6 +1101,9 @@ static void nfc_llcp_recv_hdlc(struct nfc_llcp_local *local,
 	struct sock *sk;
 	u8 dsap, ssap, ptype, ns, nr;
 
+	if (!pskb_may_pull(skb, LLCP_HEADER_SIZE + LLCP_SEQUENCE_SIZE))
+		return;
+
 	ptype = nfc_llcp_ptype(skb);
 	dsap = nfc_llcp_dsap(skb);
 	ssap = nfc_llcp_ssap(skb);
@@ -1251,6 +1281,7 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local,
 	struct nfc_llcp_sock *llcp_sock;
 	struct sock *sk;
 	u8 dsap, ssap, reason;
+	bool connecting = false;
 
 	dsap = nfc_llcp_dsap(skb);
 	ssap = nfc_llcp_ssap(skb);
@@ -1262,6 +1293,7 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local,
 	case LLCP_DM_NOBOUND:
 	case LLCP_DM_REJ:
 		llcp_sock = nfc_llcp_connecting_sock_get(local, dsap);
+		connecting = true;
 		break;
 
 	default:
@@ -1276,10 +1308,33 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local,
 
 	sk = &llcp_sock->sk;
 
+	lock_sock(sk);
+
+	/* Check if socket was destroyed whilst waiting for the lock */
+	if (!sk_hashed(sk)) {
+		release_sock(sk);
+		nfc_llcp_sock_put(llcp_sock);
+		return;
+	}
+
+	/*
+	 * For DM(NOBOUND)/DM(REJ) the socket is still linked on the
+	 * connecting_sockets list.  Unlink it here, under the socket lock,
+	 * before moving it to LLCP_CLOSED: llcp_sock_release() selects the
+	 * list to unlink from by sk_state, so leaving a connecting socket
+	 * in the CLOSED state would make it unlink from the wrong list and
+	 * corrupt the connecting_sockets list / desync the socket refcount.
+	 * This mirrors nfc_llcp_recv_cc().
+	 */
+	if (connecting)
+		nfc_llcp_sock_unlink(&local->connecting_sockets, sk);
+
 	sk->sk_err = ENXIO;
 	sk->sk_state = LLCP_CLOSED;
 	sk->sk_state_change(sk);
 
+	release_sock(sk);
+
 	nfc_llcp_sock_put(llcp_sock);
 }
 
@@ -1680,6 +1735,7 @@ int nfc_llcp_register_device(struct nfc_dev *ndev)
 	INIT_WORK(&local->rx_work, nfc_llcp_rx_work);
 
 	INIT_WORK(&local->timeout_work, nfc_llcp_timeout_work);
+	INIT_WORK(&local->release_work, local_release_work);
 
 	rwlock_init(&local->sockets.lock);
 	rwlock_init(&local->connecting_sockets.lock);
@@ -1723,10 +1779,23 @@ void nfc_llcp_unregister_device(struct nfc_dev *dev)
 
 int __init nfc_llcp_init(void)
 {
-	return nfc_llcp_sock_init();
+	int ret;
+
+	llcp_wq = alloc_workqueue("nfc_llcp_wq", WQ_UNBOUND, 0);
+	if (!llcp_wq)
+		return -ENOMEM;
+
+	ret = nfc_llcp_sock_init();
+	if (ret) {
+		destroy_workqueue(llcp_wq);
+		return ret;
+	}
+
+	return 0;
 }
 
 void nfc_llcp_exit(void)
 {
 	nfc_llcp_sock_exit();
+	destroy_workqueue(llcp_wq);
 }
diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c
index 5558d8a4d48b36..1e5ee4bcde684f 100644
--- a/net/nfc/llcp_sock.c
+++ b/net/nfc/llcp_sock.c
@@ -392,11 +392,12 @@ void nfc_llcp_accept_unlink(struct sock *sk)
 
 	pr_debug("state %d\n", sk->sk_state);
 
-	list_del_init(&llcp_sock->accept_queue);
-	sk_acceptq_removed(llcp_sock->parent);
-	llcp_sock->parent = NULL;
-
-	sock_put(sk);
+	if (llcp_sock->parent) {
+		list_del_init(&llcp_sock->accept_queue);
+		sk_acceptq_removed(llcp_sock->parent);
+		llcp_sock->parent = NULL;
+		sock_put(sk);
+	}
 }
 
 void nfc_llcp_accept_enqueue(struct sock *parent, struct sock *sk)
@@ -423,12 +424,20 @@ struct sock *nfc_llcp_accept_dequeue(struct sock *parent,
 
 	list_for_each_entry_safe(lsk, n, &llcp_parent->accept_queue,
 				 accept_queue) {
+		struct nfc_llcp_local *local;
+
 		sk = &lsk->sk;
-		lock_sock(sk);
+		lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 
 		if (sk->sk_state == LLCP_CLOSED) {
-			release_sock(sk);
+			local = nfc_llcp_sock(sk)->local;
+
 			nfc_llcp_accept_unlink(sk);
+			if (local)
+				nfc_llcp_sock_unlink(&local->sockets, sk);
+			sock_orphan(sk);
+			release_sock(sk);
+			sock_put(sk);
 			continue;
 		}
 
@@ -464,7 +473,7 @@ static int llcp_sock_accept(struct socket *sock, struct socket *newsock,
 
 	pr_debug("parent %p\n", sk);
 
-	lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
+	lock_sock(sk);
 
 	if (sk->sk_state != LLCP_LISTEN) {
 		ret = -EBADFD;
@@ -490,7 +499,12 @@ static int llcp_sock_accept(struct socket *sock, struct socket *newsock,
 
 		release_sock(sk);
 		timeo = schedule_timeout(timeo);
-		lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
+		lock_sock(sk);
+
+		if (sk->sk_state != LLCP_LISTEN) {
+			ret = -EBADFD;
+			break;
+		}
 	}
 	__set_current_state(TASK_RUNNING);
 	remove_wait_queue(sk_sleep(sk), &wait);
@@ -629,13 +643,24 @@ static int llcp_sock_release(struct socket *sock)
 
 		list_for_each_entry_safe(lsk, n, &llcp_sock->accept_queue,
 					 accept_queue) {
+			bool put_creation = false;
+
 			accept_sk = &lsk->sk;
-			lock_sock(accept_sk);
+			lock_sock_nested(accept_sk, SINGLE_DEPTH_NESTING);
+
+			if (nfc_llcp_sock(accept_sk)->parent == sk) {
+				nfc_llcp_send_disconnect(lsk);
+				nfc_llcp_accept_unlink(accept_sk);
+				nfc_llcp_sock_unlink(&local->sockets, accept_sk);
 
-			nfc_llcp_send_disconnect(lsk);
-			nfc_llcp_accept_unlink(accept_sk);
+				accept_sk->sk_state = LLCP_CLOSED;
+				sock_orphan(accept_sk);
+				put_creation = true;
+			}
 
 			release_sock(accept_sk);
+			if (put_creation)
+				sock_put(accept_sk); /* creation ref */
 		}
 	}
 
@@ -734,12 +759,16 @@ static int llcp_sock_connect(struct socket *sock, struct sockaddr_unsized *_addr
 	llcp_sock->service_name_len = min_t(unsigned int,
 					    addr->service_name_len,
 					    NFC_LLCP_MAX_SERVICE_NAME);
-	llcp_sock->service_name = kmemdup(addr->service_name,
-					  llcp_sock->service_name_len,
-					  GFP_KERNEL);
-	if (!llcp_sock->service_name) {
-		ret = -ENOMEM;
-		goto sock_llcp_release;
+	if (llcp_sock->service_name_len == 0) {
+		llcp_sock->service_name = NULL;
+	} else {
+		llcp_sock->service_name = kmemdup(addr->service_name,
+						  llcp_sock->service_name_len,
+						  GFP_KERNEL);
+		if (!llcp_sock->service_name) {
+			ret = -ENOMEM;
+			goto sock_llcp_release;
+		}
 	}
 
 	nfc_llcp_sock_link(&local->connecting_sockets, sk);
diff --git a/net/nfc/nci/core.c b/net/nfc/nci/core.c
index 5f46c4b5720f6c..73e3a96470ace7 100644
--- a/net/nfc/nci/core.c
+++ b/net/nfc/nci/core.c
@@ -780,15 +780,15 @@ static int nci_set_local_general_bytes(struct nfc_dev *nfc_dev)
 {
 	struct nci_dev *ndev = nfc_get_drvdata(nfc_dev);
 	struct nci_set_config_param param;
+	u8 gb[NFC_MAX_GT_LEN];
 	int rc;
 
-	param.val = nfc_get_local_general_bytes(nfc_dev, &param.len);
-	if ((param.val == NULL) || (param.len == 0))
+	nfc_get_local_general_bytes(nfc_dev, gb, sizeof(gb),
+				    &param.len);
+	if (param.len == 0)
 		return 0;
 
-	if (param.len > NFC_MAX_GT_LEN)
-		return -EINVAL;
-
+	param.val = gb;
 	param.id = NCI_PN_ATR_REQ_GEN_BYTES;
 
 	rc = nci_request(ndev, nci_set_config_req, &param,
diff --git a/net/nfc/netlink.c b/net/nfc/netlink.c
index 0c58824cb150dd..224bdfa2dd0dc6 100644
--- a/net/nfc/netlink.c
+++ b/net/nfc/netlink.c
@@ -1181,7 +1181,7 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info)
 
 		if (rc != 0) {
 			rc = -EINVAL;
-			goto put_local;
+			goto free_list;
 		}
 
 		if (!sdp_attrs[NFC_SDP_ATTR_URI])
@@ -1200,7 +1200,7 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info)
 		sdreq = nfc_llcp_build_sdreq_tlv(tid, uri, uri_len);
 		if (sdreq == NULL) {
 			rc = -ENOMEM;
-			goto put_local;
+			goto free_list;
 		}
 
 		tlvs_len += sdreq->tlv_len;
@@ -1215,6 +1215,9 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info)
 
 	rc = nfc_llcp_send_snl_sdreq(local, &sdreq_list, tlvs_len);
 
+free_list:
+	nfc_llcp_free_sdp_tlv_list(&sdreq_list);
+
 put_local:
 	nfc_llcp_local_put(local);
 
diff --git a/net/nfc/nfc.h b/net/nfc/nfc.h
index 0b1e6466f4fbf5..82c5dfdad10e24 100644
--- a/net/nfc/nfc.h
+++ b/net/nfc/nfc.h
@@ -49,7 +49,8 @@ void nfc_llcp_mac_is_up(struct nfc_dev *dev, u32 target_idx,
 int nfc_llcp_register_device(struct nfc_dev *dev);
 void nfc_llcp_unregister_device(struct nfc_dev *dev);
 int nfc_llcp_set_remote_gb(struct nfc_dev *dev, const u8 *gb, u8 gb_len);
-u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, size_t *general_bytes_len);
+u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, u8 *out_gb, size_t gb_max_len,
+			   size_t *general_bytes_len);
 int nfc_llcp_data_received(struct nfc_dev *dev, struct sk_buff *skb);
 struct nfc_llcp_local *nfc_llcp_find_local(struct nfc_dev *dev);
 int nfc_llcp_local_put(struct nfc_llcp_local *local);
diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c
index 0f433688e17b98..d3326edcabf76f 100644
--- a/net/openvswitch/conntrack.c
+++ b/net/openvswitch/conntrack.c
@@ -734,6 +734,18 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
 	enum ip_conntrack_info ctinfo;
 	struct nf_conn *ct;
 
+	/* If the ct entry is not confirmed and shared with some other skb,
+	 * e.g., a cloned one, we can't just modify it with the commit as we
+	 * must not modify the extension set.  Reset.
+	 */
+	if (cached && info->commit) {
+		ct = nf_ct_get(skb, &ctinfo);
+		if (ct && !nf_ct_is_confirmed(ct) && nf_ct_shared(ct)) {
+			nf_reset_ct(skb);
+			cached = false;
+		}
+	}
+
 	if (!cached) {
 		struct nf_hook_state state = {
 			.hook = NF_INET_PRE_ROUTING,
@@ -766,8 +778,6 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
 
 	ct = nf_ct_get(skb, &ctinfo);
 	if (ct) {
-		bool add_helper = false;
-
 		/* Packets starting a new connection must be NATted before the
 		 * helper, so that the helper knows about the NAT.  We enforce
 		 * this by delaying both NAT and helper calls for unconfirmed
@@ -799,7 +809,6 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
 							    GFP_ATOMIC);
 			if (err)
 				return err;
-			add_helper = true;
 
 			/* helper installed, add seqadj if NAT is required */
 			if (info->nat && !nfct_seqadj(ct)) {
@@ -808,14 +817,14 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
 			}
 		}
 
-		/* Call the helper only if:
-		 * - nf_conntrack_in() was executed above ("!cached") or a
-		 *   helper was just attached ("add_helper") for a confirmed
-		 *   connection, or
-		 * - When committing an unconfirmed connection.
+		/* Call the helper only if nf_conntrack_in() was executed
+		 * above ("!cached").
+		 *
+		 * For unconfirmed connections it will be called later during
+		 * commit as we need to have all the other extensions allocated
+		 * before the call.
 		 */
-		if ((nf_ct_is_confirmed(ct) ? !cached || add_helper :
-					      info->commit)) {
+		if (nf_ct_is_confirmed(ct) && !cached) {
 			int err = nf_ct_helper(skb, ct, ctinfo, info->family);
 
 			err = verdict_to_errno(err);
@@ -1019,6 +1028,14 @@ static int ovs_ct_commit(struct net *net, struct sw_flow_key *key,
 			return err;
 
 		nf_conn_act_ct_ext_add(skb, ct, ctinfo);
+
+		/* Call the helpers now.  We couldn't do this before as
+		 * all the extensions must be allocated before the call.
+		 */
+		err = nf_ct_helper(skb, ct, ctinfo, info->family);
+		err = verdict_to_errno(err);
+		if (err)
+			return err;
 	} else if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
 		   labels_nonzero(&info->labels.mask)) {
 		err = ovs_ct_set_labels(ct, key, &info->labels.value,
diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c
index 631a03136fa142..21870341432552 100644
--- a/net/openvswitch/datapath.c
+++ b/net/openvswitch/datapath.c
@@ -1921,6 +1921,10 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
 
 	ovs_unlock();
 
+	/* Start periodic mask rebalancing if it wasn't already. */
+	schedule_delayed_work(&ovs_net->masks_rebalance,
+			      DP_MASKS_REBALANCE_INTERVAL);
+
 	ovs_notify(&dp_datapath_genl_family, reply, info);
 	return 0;
 
@@ -2598,16 +2602,20 @@ static void ovs_dp_masks_rebalance(struct work_struct *work)
 	struct ovs_net *ovs_net = container_of(work, struct ovs_net,
 					       masks_rebalance.work);
 	struct datapath *dp;
+	bool rearm;
 
 	ovs_lock();
 
 	list_for_each_entry(dp, &ovs_net->dps, list_node)
 		ovs_flow_masks_rebalance(&dp->table);
 
+	rearm = !list_empty(&ovs_net->dps);
+
 	ovs_unlock();
 
-	schedule_delayed_work(&ovs_net->masks_rebalance,
-			      msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
+	if (rearm)
+		schedule_delayed_work(&ovs_net->masks_rebalance,
+				      DP_MASKS_REBALANCE_INTERVAL);
 }
 
 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
@@ -2713,8 +2721,6 @@ static int __net_init ovs_init_net(struct net *net)
 	if (err)
 		return err;
 
-	schedule_delayed_work(&ovs_net->masks_rebalance,
-			      msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
 	return 0;
 }
 
diff --git a/net/openvswitch/datapath.h b/net/openvswitch/datapath.h
index b2c2b8da12d40b..7a7afaeceee5ee 100644
--- a/net/openvswitch/datapath.h
+++ b/net/openvswitch/datapath.h
@@ -23,7 +23,7 @@
 
 #define DP_MAX_PORTS                USHRT_MAX
 #define DP_VPORT_HASH_BUCKETS       1024
-#define DP_MASKS_REBALANCE_INTERVAL 4000
+#define DP_MASKS_REBALANCE_INTERVAL msecs_to_jiffies(4000)
 
 /**
  * struct dp_stats_percpu - per-cpu packet processing statistics for a given
diff --git a/net/openvswitch/vport-internal_dev.c b/net/openvswitch/vport-internal_dev.c
index 125d310871e93e..77a16d5ebd5434 100644
--- a/net/openvswitch/vport-internal_dev.c
+++ b/net/openvswitch/vport-internal_dev.c
@@ -117,6 +117,8 @@ static void do_setup(struct net_device *netdev)
 	netdev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX;
 	netdev->hw_features = netdev->features;
 
+	netif_set_tso_max_size(netdev, GSO_MAX_SIZE);
+
 	eth_hw_addr_random(netdev);
 }
 
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 50cae32ae26922..e9a7ff36118a99 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -617,7 +617,7 @@ static int prb_calc_retire_blk_tmo(struct packet_sock *po,
 		return DEFAULT_PRB_RETIRE_TOV;
 
 	div = ecmd.base.speed / 1000;
-	mbits = (blk_size_in_bytes * 8) / (1024 * 1024);
+	mbits = (u64)blk_size_in_bytes * 8 / (1024 * 1024);
 
 	if (div)
 		mbits /= div;
@@ -2530,26 +2530,6 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
 	goto drop_n_restore;
 }
 
-static void tpacket_destruct_skb(struct sk_buff *skb)
-{
-	struct packet_sock *po = pkt_sk(skb->sk);
-
-	if (likely(po->tx_ring.pg_vec)) {
-		void *ph;
-		__u32 ts;
-
-		ph = skb_zcopy_get_nouarg(skb);
-
-		ts = __packet_set_timestamp(po, ph, skb);
-		__packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
-
-		packet_dec_pending(&po->tx_ring);
-		complete(&po->skb_completion);
-	}
-
-	sock_wfree(skb);
-}
-
 static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
 {
 	if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
@@ -2589,27 +2569,56 @@ static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
 	return 0;
 }
 
+struct tpacket_uarg {
+	struct ubuf_info	ubuf;
+	struct packet_sock	*po;
+	void			*ph;
+};
+
+static void tpacket_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
+				  bool success)
+{
+	struct tpacket_uarg *tu = container_of(uarg, struct tpacket_uarg, ubuf);
+	struct packet_sock *po = tu->po;
+	void *ph = tu->ph;
+	__u32 ts;
+
+	DEBUG_NET_WARN_ON_ONCE(!skb);
+
+	if (!refcount_dec_and_test(&uarg->refcnt))
+		return;
+
+	ts = __packet_set_timestamp(po, ph, skb);
+	__packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);
+
+	packet_dec_pending(&po->tx_ring);
+	complete(&po->skb_completion);
+
+	kfree(tu);
+	sk_free(&po->sk);
+}
+
+static const struct ubuf_info_ops tpacket_ubuf_ops = {
+	.complete = tpacket_ubuf_complete,
+};
+
 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
-		void *frame, struct net_device *dev, void *data, int tp_len,
+		struct net_device *dev, void *data, int tp_len,
 		__be16 proto, unsigned char *addr, int hlen, int copylen,
 		int hard_header_len,
 		const struct sockcm_cookie *sockc)
 {
-	union tpacket_uhdr ph;
 	int to_write, offset, len, nr_frags, len_max;
 	struct socket *sock = po->sk.sk_socket;
 	struct page *page;
 	int err;
 
-	ph.raw = frame;
-
 	skb->protocol = proto;
 	skb->dev = dev;
 	skb->priority = sockc->priority;
 	skb->mark = sockc->mark;
 	skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, po->sk.sk_clockid);
 	skb_setup_tx_timestamp(skb, sockc);
-	skb_zcopy_set_nouarg(skb, ph.raw);
 
 	skb_reserve(skb, hlen);
 	skb_reset_network_header(skb);
@@ -2749,6 +2758,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
 	struct virtio_net_hdr vnet_hdr;
 	bool has_vnet_hdr = false;
 	struct sockcm_cookie sockc;
+	struct tpacket_uarg *uarg;
 	__be16 proto;
 	int err, reserve = 0;
 	void *ph;
@@ -2864,6 +2874,9 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
 						    vnet_hdr.hdr_len);
 			has_vnet_hdr = true;
 		}
+		/* Prevent the byte count from wrapping into a negative errno */
+		if (unlikely(tp_len > INT_MAX - len_sum))
+			break;
 		copylen = max_t(int, copylen, hard_header_len);
 		skb = sock_alloc_send_skb(&po->sk,
 				hlen + tlen + sizeof(struct sockaddr_ll) +
@@ -2876,7 +2889,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
 				err = len_sum;
 			goto out_status;
 		}
-		tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
+		tp_len = tpacket_fill_skb(po, skb, dev, data, tp_len, proto,
 					  addr, hlen, copylen, hard_header_len,
 					  &sockc);
 		if (likely(tp_len >= 0) &&
@@ -2908,7 +2921,24 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
 			virtio_net_hdr_set_proto(skb, &vnet_hdr);
 		}
 
-		skb->destructor = tpacket_destruct_skb;
+		uarg = kmalloc(sizeof(*uarg), GFP_KERNEL);
+		if (unlikely(!uarg)) {
+			if (likely(len_sum > 0))
+				err = len_sum;
+			else
+				err = -ENOMEM;
+			goto out_status;
+		}
+		uarg->po = po;
+		uarg->ph = ph;
+		uarg->ubuf.ops = &tpacket_ubuf_ops;
+		uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
+		refcount_set(&uarg->ubuf.refcnt, 1);
+
+		/* Hold a sk_wmem_alloc reference until completion */
+		refcount_inc(&po->sk.sk_wmem_alloc);
+		skb_zcopy_init(skb, &uarg->ubuf);
+
 		__packet_set_status(po, ph, TP_STATUS_SENDING);
 		packet_inc_pending(&po->tx_ring);
 
@@ -4486,21 +4516,20 @@ static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring
 	vec->len = block_nr;
 	pg_vec = vec->pg_vec;
 
+	if (tx_ring) {
+		vec->deferred = kzalloc_obj(*vec->deferred,
+					    GFP_KERNEL | __GFP_NOWARN);
+		if (!vec->deferred)
+			goto out_free_pgvec;
+		vec->deferred->vec = vec;
+		INIT_DELAYED_WORK(&vec->deferred->work,
+				  packet_free_pg_vec_work);
+	}
+
 	for (i = 0; i < block_nr; i++) {
 		pg_vec[i].buffer = alloc_one_pg_vec_page(order);
 		if (unlikely(!pg_vec[i].buffer))
 			goto out_free_pgvec;
-
-		if (tx_ring && !vec->deferred &&
-		    is_vmalloc_addr(pg_vec[i].buffer)) {
-			vec->deferred = kzalloc_obj(*vec->deferred,
-						    GFP_KERNEL | __GFP_NOWARN);
-			if (!vec->deferred)
-				goto out_free_pgvec;
-			vec->deferred->vec = vec;
-			INIT_DELAYED_WORK(&vec->deferred->work,
-					  packet_free_pg_vec_work);
-		}
 	}
 
 out:
@@ -4828,16 +4857,13 @@ static void packet_seq_stop(struct seq_file *seq, void *v)
 static int packet_seq_show(struct seq_file *seq, void *v)
 {
 	if (v == SEQ_START_TOKEN)
-		seq_printf(seq,
-			   "%*sRefCnt Type Proto  Iface R Rmem   User   Inode\n",
-			   IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk");
+		seq_puts(seq, "sk RefCnt Type Proto  Iface R Rmem   User   Inode\n");
 	else {
 		struct sock *s = sk_entry(v);
 		const struct packet_sock *po = pkt_sk(s);
 
 		seq_printf(seq,
-			   "%pK %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6llu\n",
-			   s,
+			   "0  %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6llu\n",
 			   refcount_read(&s->sk_refcnt),
 			   s->sk_type,
 			   ntohs(READ_ONCE(po->num)),
diff --git a/net/phonet/socket.c b/net/phonet/socket.c
index 631a99cdbd006b..ad12b746d4fcad 100644
--- a/net/phonet/socket.c
+++ b/net/phonet/socket.c
@@ -586,14 +586,13 @@ static int pn_sock_seq_show(struct seq_file *seq, void *v)
 		struct pn_sock *pn = pn_sk(sk);
 
 		seq_printf(seq, "%2d %04X:%04X:%02X %02X %08X:%08X %5d %llu "
-			"%d %pK %u",
+			"%d 0 %u",
 			sk->sk_protocol, pn->sobject, pn->dobject,
 			pn->resource, sk->sk_state,
 			sk_wmem_alloc_get(sk), sk_rmem_alloc_get(sk),
 			from_kuid_munged(seq_user_ns(seq), sk_uid(sk)),
 			sock_i_ino(sk),
-			refcount_read(&sk->sk_refcnt), sk,
-			sk_drops_read(sk));
+			refcount_read(&sk->sk_refcnt), sk_drops_read(sk));
 	}
 	seq_pad(seq, '\n');
 	return 0;
diff --git a/net/psp/psp.h b/net/psp/psp.h
index 86eeba823ced1f..b123c242790507 100644
--- a/net/psp/psp.h
+++ b/net/psp/psp.h
@@ -19,6 +19,7 @@ bool psp_has_assoc_dev_in_ns(struct psp_dev *psd, struct net *net);
 int psp_attach_netdev_notifier(void);
 
 void psp_nl_notify_dev(struct psp_dev *psd, u32 cmd);
+void psp_nl_notify_disassoc(struct psp_dev *psd, struct net *net);
 
 struct psp_assoc *psp_assoc_create(struct psp_dev *psd);
 struct psp_dev *psp_dev_get_for_sock(struct sock *sk);
@@ -53,4 +54,16 @@ static inline bool psp_dev_is_registered(struct psp_dev *psd)
 	return !!psd->ops;
 }
 
+static inline bool psp_dev_has_sadb(struct psp_dev *psd)
+{
+	lockdep_assert_held(&psd->lock);
+	return !!psd->ops->tx_key_del;
+}
+
+static inline bool psp_assoc_needs_tx_key_del(struct psp_assoc *pas)
+{
+	lockdep_assert_held(&pas->psd->lock);
+	return psp_dev_has_sadb(pas->psd) && pas->tx.spi;
+}
+
 #endif /* __PSP_PSP_H */
diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c
index c9c1a8826b7f92..273b010d2355c9 100644
--- a/net/psp/psp_main.c
+++ b/net/psp/psp_main.c
@@ -68,9 +68,9 @@ psp_dev_create(struct net_device *netdev,
 		    !psd_ops->set_config ||
 		    !psd_ops->key_rotate ||
 		    !psd_ops->rx_spi_alloc ||
-		    !psd_ops->tx_key_add ||
-		    !psd_ops->tx_key_del ||
-		    !psd_ops->get_stats))
+		    !psd_ops->get_stats ||
+		    (!psd_ops->tx_key_add != !psd_ops->tx_key_del) ||
+		    (psd_caps->assoc_drv_spc && !psd_ops->tx_key_add)))
 		return ERR_PTR(-EINVAL);
 
 	psd = kzalloc_obj(*psd);
@@ -147,8 +147,11 @@ void psp_dev_unregister(struct psp_dev *psd)
 
 	list_splice_init(&psd->active_assocs, &psd->prev_assocs);
 	list_splice_init(&psd->prev_assocs, &psd->stale_assocs);
-	list_for_each_entry_safe(pas, next, &psd->stale_assocs, assocs_list)
-		psp_dev_tx_key_del(psd, pas);
+	list_for_each_entry_safe(pas, next, &psd->stale_assocs, assocs_list) {
+		if (psp_assoc_needs_tx_key_del(pas))
+			psp_dev_tx_key_del(psd, pas);
+		list_del(&pas->assocs_list);
+	}
 
 	list_for_each_entry_safe(entry, entry_tmp, &psd->assoc_dev_list,
 				 dev_list) {
@@ -405,7 +408,7 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv)
 }
 EXPORT_SYMBOL(psp_dev_rcv);
 
-static void psp_dev_disassoc_one(struct psp_dev *psd, struct net_device *dev)
+static bool psp_dev_disassoc_one(struct psp_dev *psd, struct net_device *dev)
 {
 	struct psp_assoc_dev *entry;
 
@@ -416,9 +419,11 @@ static void psp_dev_disassoc_one(struct psp_dev *psd, struct net_device *dev)
 			rcu_assign_pointer(entry->assoc_dev->psp_dev, NULL);
 			netdev_put(entry->assoc_dev, &entry->dev_tracker);
 			kfree(entry);
-			return;
+			return true;
 		}
 	}
+
+	return false;
 }
 
 static int psp_netdev_event(struct notifier_block *nb, unsigned long event,
@@ -435,9 +440,13 @@ static int psp_netdev_event(struct notifier_block *nb, unsigned long event,
 	if (psd && psp_dev_tryget(psd)) {
 		rcu_read_unlock();
 		mutex_lock(&psd->lock);
-		if (psp_dev_is_registered(psd))
-			psp_nl_notify_dev(psd, PSP_CMD_DEV_CHANGE_NTF);
-		psp_dev_disassoc_one(psd, dev);
+		/* Nothing to report if the device was never on the list,
+		 * dev-assoc may have failed after publishing dev->psp_dev,
+		 * and this is also the main netdevice's path.
+		 */
+		if (psp_dev_disassoc_one(psd, dev) &&
+		    psp_dev_is_registered(psd))
+			psp_nl_notify_disassoc(psd, dev_net(dev));
 		mutex_unlock(&psd->lock);
 		psp_dev_put(psd);
 	} else {
diff --git a/net/psp/psp_nl.c b/net/psp/psp_nl.c
index f91665748dde0c..cdfc2d72fb390a 100644
--- a/net/psp/psp_nl.c
+++ b/net/psp/psp_nl.c
@@ -294,13 +294,16 @@ psp_nl_dev_fill(struct psp_dev *psd, struct sk_buff *rsp,
 		return -EMSGSIZE;
 
 	if (nla_put_u32(rsp, PSP_A_DEV_ID, psd->id) ||
-	    nla_put_u32(rsp, PSP_A_DEV_IFINDEX, psd->main_netdev->ifindex) ||
 	    nla_put_u32(rsp, PSP_A_DEV_PSP_VERSIONS_CAP, psd->caps->versions) ||
 	    nla_put_u32(rsp, PSP_A_DEV_PSP_VERSIONS_ENA, psd->config.versions))
 		goto err_cancel_msg;
 
 	if (cur_net == dev_net(psd->main_netdev)) {
-		/* Primary device - dump assoc list */
+		/* Primary device - report the netdev, dump assoc list. */
+		if (nla_put_u32(rsp, PSP_A_DEV_IFINDEX,
+				psd->main_netdev->ifindex))
+			goto err_cancel_msg;
+
 		err = psp_nl_fill_assoc_dev_list(psd, rsp, cur_net, NULL);
 		if (err)
 			goto err_cancel_msg;
@@ -353,6 +356,40 @@ void psp_nl_notify_dev(struct psp_dev *psd, u32 cmd)
 				psp_nl_build_dev_ntf, &cmd);
 }
 
+/**
+ * psp_nl_notify_disassoc() - notify about a device losing an association
+ * @psd: PSP device (must be locked)
+ * @net: netns of the netdevice which got disassociated
+ *
+ * Must be called once @psd no longer has the association, so that the
+ * notifications carry the state after the change.
+ */
+void psp_nl_notify_disassoc(struct psp_dev *psd, struct net *net)
+{
+	struct sk_buff *ntf;
+	bool still_visible;
+	u32 cmd;
+
+	lockdep_assert_held(&psd->lock);
+
+	psp_nl_notify_dev(psd, PSP_CMD_DEV_CHANGE_NTF);
+
+	/* psp_nl_notify_dev() reaches the main netdevice's netns and every
+	 * netns which still has an associated device. If @net is neither,
+	 * the device is gone from @net and we should send a delete ntf.
+	 */
+	still_visible = !psp_dev_check_access(psd, net, false);
+	if (still_visible || !maybe_get_net(net))
+		return;
+
+	cmd = PSP_CMD_DEV_DEL_NTF;
+	ntf = psp_nl_build_dev_ntf(psd, net, &cmd);
+	if (ntf)
+		genlmsg_multicast_netns(&psp_nl_family, net, ntf, 0,
+					PSP_NLGRP_MGMT, GFP_KERNEL);
+	put_net(net);
+}
+
 int psp_nl_dev_get_doit(struct sk_buff *req, struct genl_info *info)
 {
 	struct psp_dev *psd = info->user_ptr[0];
@@ -617,13 +654,6 @@ int psp_nl_dev_disassoc_doit(struct sk_buff *skb, struct genl_info *info)
 		return -ENOMEM;
 	}
 
-	put_net(net);
-
-	/* Notify before removal so listeners in the disassociated namespace
-	 * still receive the notification.
-	 */
-	psp_nl_notify_dev(psd, PSP_CMD_DEV_CHANGE_NTF);
-
 	/* Remove from the association list */
 	list_del(&found->dev_list);
 	psd->assoc_dev_cnt--;
@@ -631,6 +661,9 @@ int psp_nl_dev_disassoc_doit(struct sk_buff *skb, struct genl_info *info)
 	netdev_put(found->assoc_dev, &found->dev_tracker);
 	kfree(found);
 
+	psp_nl_notify_disassoc(psd, net);
+	put_net(net);
+
 	return psp_nl_reply_send(rsp, info);
 }
 
diff --git a/net/psp/psp_sock.c b/net/psp/psp_sock.c
index e9b53eedf8dba0..a9cfeebe4ba1b9 100644
--- a/net/psp/psp_sock.c
+++ b/net/psp/psp_sock.c
@@ -78,16 +78,33 @@ static struct psp_assoc *psp_assoc_dummy(struct psp_assoc *pas)
 }
 
 static int psp_dev_tx_key_add(struct psp_dev *psd, struct psp_assoc *pas,
+			      struct psp_key_parsed *key,
 			      struct netlink_ext_ack *extack)
 {
-	return psd->ops->tx_key_add(psd, pas, extack);
+	struct psp_assoc *dummy;
+	int err;
+
+	/* Pass a fake association to drivers to make sure they don't
+	 * try to store pointers to it. For re-keying we'll need to
+	 * re-allocate the assoc structures.
+	 */
+	dummy = psp_assoc_dummy(pas);
+	if (!dummy)
+		return -ENOMEM;
+
+	memcpy(&dummy->tx, key, sizeof(*key));
+	err = psd->ops->tx_key_add(psd, dummy, extack);
+	if (!err)
+		memcpy(pas->drv_data, dummy->drv_data,
+		       psd->caps->assoc_drv_spc);
+
+	kfree(dummy);
+	return err;
 }
 
 void psp_dev_tx_key_del(struct psp_dev *psd, struct psp_assoc *pas)
 {
-	if (pas->tx.spi)
-		psd->ops->tx_key_del(psd, pas);
-	list_del(&pas->assocs_list);
+	psd->ops->tx_key_del(psd, pas);
 }
 
 static void psp_assoc_free(struct work_struct *work)
@@ -96,8 +113,11 @@ static void psp_assoc_free(struct work_struct *work)
 	struct psp_dev *psd = pas->psd;
 
 	mutex_lock(&psd->lock);
-	if (psp_dev_is_registered(psd))
-		psp_dev_tx_key_del(psd, pas);
+	if (psp_dev_is_registered(psd)) {
+		if (psp_assoc_needs_tx_key_del(pas))
+			psp_dev_tx_key_del(psd, pas);
+		list_del(&pas->assocs_list);
+	}
 	mutex_unlock(&psd->lock);
 	psp_dev_put(psd);
 	kfree(pas);
@@ -159,6 +179,22 @@ int psp_sock_assoc_set_rx(struct sock *sk, struct psp_assoc *pas,
 	return err;
 }
 
+static int psp_assoc_set_tx(struct psp_dev *psd, struct psp_assoc *pas,
+			    struct psp_key_parsed *key,
+			    struct netlink_ext_ack *extack)
+{
+	int err;
+
+	if (psp_dev_has_sadb(psd)) {
+		err = psp_dev_tx_key_add(psd, pas, key, extack);
+		if (err)
+			return err;
+	}
+
+	memcpy(&pas->tx, key, sizeof(*key));
+	return 0;
+}
+
 static int psp_sock_recv_queue_check(struct sock *sk, struct psp_assoc *pas)
 {
 	struct psp_skb_ext *pse;
@@ -178,12 +214,40 @@ static int psp_sock_recv_queue_check(struct sock *sk, struct psp_assoc *pas)
 	return 0;
 }
 
+static int
+psp_sock_set_tx_key(struct sock *sk, struct psp_dev *psd, struct psp_assoc *pas,
+		    struct psp_key_parsed *key, struct netlink_ext_ack *extack)
+{
+	struct inet_connection_sock *icsk;
+	int err;
+
+	err = psp_sock_recv_queue_check(sk, pas);
+	if (err) {
+		NL_SET_ERR_MSG(extack,
+			       "Socket has incompatible segments already in the recv queue");
+		return err;
+	}
+
+	err = psp_assoc_set_tx(psd, pas, key, extack);
+	if (err)
+		return err;
+
+	WRITE_ONCE(sk->sk_validate_xmit_skb, psp_validate_xmit);
+	tcp_write_collapse_fence(sk);
+	pas->upgrade_seq = tcp_sk(sk)->rcv_nxt;
+
+	icsk = inet_csk(sk);
+	icsk->icsk_ext_hdr_len += psp_sk_overhead(sk);
+	icsk->icsk_sync_mss(sk, icsk->icsk_pmtu_cookie);
+
+	return err;
+}
+
 int psp_sock_assoc_set_tx(struct sock *sk, struct psp_dev *psd,
 			  u32 version, struct psp_key_parsed *key,
 			  struct netlink_ext_ack *extack)
 {
-	struct inet_connection_sock *icsk;
-	struct psp_assoc *pas, *dummy;
+	struct psp_assoc *pas;
 	int err;
 
 	lock_sock(sk);
@@ -211,40 +275,7 @@ int psp_sock_assoc_set_tx(struct sock *sk, struct psp_dev *psd,
 		goto exit_unlock;
 	}
 
-	err = psp_sock_recv_queue_check(sk, pas);
-	if (err) {
-		NL_SET_ERR_MSG(extack, "Socket has incompatible segments already in the recv queue");
-		goto exit_unlock;
-	}
-
-	/* Pass a fake association to drivers to make sure they don't
-	 * try to store pointers to it. For re-keying we'll need to
-	 * re-allocate the assoc structures.
-	 */
-	dummy = psp_assoc_dummy(pas);
-	if (!dummy) {
-		err = -ENOMEM;
-		goto exit_unlock;
-	}
-
-	memcpy(&dummy->tx, key, sizeof(*key));
-	err = psp_dev_tx_key_add(psd, dummy, extack);
-	if (err)
-		goto exit_free_dummy;
-
-	memcpy(pas->drv_data, dummy->drv_data, psd->caps->assoc_drv_spc);
-	memcpy(&pas->tx, key, sizeof(*key));
-
-	WRITE_ONCE(sk->sk_validate_xmit_skb, psp_validate_xmit);
-	tcp_write_collapse_fence(sk);
-	pas->upgrade_seq = tcp_sk(sk)->rcv_nxt;
-
-	icsk = inet_csk(sk);
-	icsk->icsk_ext_hdr_len += psp_sk_overhead(sk);
-	icsk->icsk_sync_mss(sk, icsk->icsk_pmtu_cookie);
-
-exit_free_dummy:
-	kfree(dummy);
+	err = psp_sock_set_tx_key(sk, psd, pas, key, extack);
 exit_unlock:
 	release_sock(sk);
 	return err;
diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c
index 78347c937af76b..e7b3647424b8f1 100644
--- a/net/qrtr/af_qrtr.c
+++ b/net/qrtr/af_qrtr.c
@@ -623,6 +623,19 @@ static void qrtr_hello_work(struct work_struct *work)
 	qrtr_port_put(ctrl);
 }
 
+/* Trigger the HELLO handshake after the remote has been reset, eg on resume */
+void qrtr_endpoint_hello(struct qrtr_endpoint *ep)
+{
+	struct qrtr_node *node = ep->node;
+
+	mutex_lock(&node->ep_lock);
+	node->hello_sent = false;
+	mutex_unlock(&node->ep_lock);
+
+	schedule_delayed_work(&node->say_hello, 0);
+}
+EXPORT_SYMBOL_GPL(qrtr_endpoint_hello);
+
 /**
  * qrtr_endpoint_register() - register a new endpoint
  * @ep: endpoint to register
diff --git a/net/qrtr/mhi.c b/net/qrtr/mhi.c
index 3990da1a65dc4f..e9a4bb92ce76ff 100644
--- a/net/qrtr/mhi.c
+++ b/net/qrtr/mhi.c
@@ -183,6 +183,7 @@ static int __maybe_unused qcom_mhi_qrtr_pm_suspend_late(struct device *dev)
 static int __maybe_unused qcom_mhi_qrtr_pm_resume_early(struct device *dev)
 {
 	struct mhi_device *mhi_dev = container_of(dev, struct mhi_device, dev);
+	struct qrtr_mhi_dev *qdev = dev_get_drvdata(dev);
 	enum mhi_state state;
 	int rc;
 
@@ -201,7 +202,13 @@ static int __maybe_unused qcom_mhi_qrtr_pm_resume_early(struct device *dev)
 		return rc;
 	}
 
-	return qcom_mhi_qrtr_queue_dl_buffers(mhi_dev);
+	rc = qcom_mhi_qrtr_queue_dl_buffers(mhi_dev);
+	if (rc)
+		return rc;
+
+	qrtr_endpoint_hello(&qdev->ep);
+
+	return 0;
 }
 
 static const struct dev_pm_ops qcom_mhi_qrtr_pm_ops = {
diff --git a/net/qrtr/qrtr.h b/net/qrtr/qrtr.h
index 3f2d28696062a5..de2de69a619936 100644
--- a/net/qrtr/qrtr.h
+++ b/net/qrtr/qrtr.h
@@ -27,6 +27,8 @@ int qrtr_endpoint_register(struct qrtr_endpoint *ep, unsigned int nid);
 
 void qrtr_endpoint_unregister(struct qrtr_endpoint *ep);
 
+void qrtr_endpoint_hello(struct qrtr_endpoint *ep);
+
 int qrtr_endpoint_post(struct qrtr_endpoint *ep, const void *data, size_t len);
 
 int qrtr_ns_init(void);
diff --git a/net/rds/connection.c b/net/rds/connection.c
index b6c4beb50eaf0e..c752a8623cfca0 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -276,6 +276,12 @@ static struct rds_connection *__rds_conn_create(struct net *net,
 
 	conn->c_trans = trans;
 
+	/* The transport may just have been swapped for loopback; size the
+	 * set of paths - which is also what rds_conn_destroy() tears down
+	 * again - by the transport the connection actually uses.
+	 */
+	npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
+
 	init_waitqueue_head(&conn->c_hs_waitq);
 	for (i = 0; i < npaths; i++) {
 		__rds_conn_path_init(conn, &conn->c_path[i],
diff --git a/net/rds/ib.h b/net/rds/ib.h
index 5ff346a1e8baa6..1901226368c982 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -159,8 +159,8 @@ struct rds_ib_connection {
 	atomic_t		i_fastreg_inuse_count;
 
 	/* interrupt handling */
-	struct tasklet_struct	i_send_tasklet;
-	struct tasklet_struct	i_recv_tasklet;
+	struct work_struct	i_send_work;
+	struct work_struct	i_recv_work;
 
 	/* tx */
 	struct rds_ib_work_ring	i_send_ring;
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 6e3110a04ae6bd..3ebe13d00953aa 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -253,7 +253,7 @@ static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
 
 	rds_ib_stats_inc(s_ib_evt_handler_call);
 
-	tasklet_schedule(&ic->i_recv_tasklet);
+	queue_work(system_bh_wq, &ic->i_recv_work);
 }
 
 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
@@ -279,9 +279,9 @@ static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
 	}
 }
 
-static void rds_ib_tasklet_fn_send(unsigned long data)
+static void rds_ib_send_worker(struct work_struct *work)
 {
-	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
+	struct rds_ib_connection *ic = from_work(ic, work, i_send_work);
 	struct rds_connection *conn = ic->conn;
 
 	rds_ib_stats_inc(s_ib_tasklet_call);
@@ -319,9 +319,9 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
 	}
 }
 
-static void rds_ib_tasklet_fn_recv(unsigned long data)
+static void rds_ib_recv_worker(struct work_struct *work)
 {
-	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
+	struct rds_ib_connection *ic = from_work(ic, work, i_recv_work);
 	struct rds_connection *conn = ic->conn;
 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
 	struct rds_ib_ack_state state;
@@ -381,7 +381,7 @@ static void rds_ib_cq_comp_handler_send(struct ib_cq *cq, void *context)
 
 	rds_ib_stats_inc(s_ib_evt_handler_call);
 
-	tasklet_schedule(&ic->i_send_tasklet);
+	queue_work(system_bh_wq, &ic->i_send_work);
 }
 
 static inline int ibdev_get_unused_vector(struct rds_ib_device *rds_ibdev)
@@ -1099,12 +1099,12 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
 		while (!wait_event_timeout(rds_ib_ring_empty_wait,
 					   rds_ib_conn_path_shutdown_check_wait(cp) == 0,
 					   msecs_to_jiffies(1000))) {
-			tasklet_schedule(&ic->i_send_tasklet);
-			tasklet_schedule(&ic->i_recv_tasklet);
+			queue_work(system_bh_wq, &ic->i_send_work);
+			queue_work(system_bh_wq, &ic->i_recv_work);
 		}
 
-		tasklet_kill(&ic->i_send_tasklet);
-		tasklet_kill(&ic->i_recv_tasklet);
+		disable_work_sync(&ic->i_send_work);
+		disable_work_sync(&ic->i_recv_work);
 
 		atomic_set(&ic->i_cq_quiesce, 1);
 
@@ -1123,6 +1123,9 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
 			ib_destroy_cq(ic->i_recv_cq);
 		}
 
+		enable_work(&ic->i_send_work);
+		enable_work(&ic->i_recv_work);
+
 		if (ic->rds_ibdev) {
 			/* then free the resources that ib callbacks use */
 			if (ic->i_send_hdrs) {
@@ -1234,10 +1237,8 @@ int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp)
 	}
 
 	INIT_LIST_HEAD(&ic->ib_node);
-	tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,
-		     (unsigned long)ic);
-	tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,
-		     (unsigned long)ic);
+	INIT_WORK(&ic->i_send_work, rds_ib_send_worker);
+	INIT_WORK(&ic->i_recv_work, rds_ib_recv_worker);
 	mutex_init(&ic->i_recv_mutex);
 #ifndef KERNEL_HAS_ATOMIC64
 	spin_lock_init(&ic->i_ack_lock);
diff --git a/net/rds/ib_frmr.c b/net/rds/ib_frmr.c
index bd861191157b54..8397aa4a17ac87 100644
--- a/net/rds/ib_frmr.c
+++ b/net/rds/ib_frmr.c
@@ -204,19 +204,16 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev,
 	 */
 	rds_ib_teardown_mr(ibmr);
 
-	ibmr->sg = sg;
-	ibmr->sg_len = sg_len;
-	ibmr->sg_dma_len = 0;
 	frmr->sg_byte_len = 0;
-	WARN_ON(ibmr->sg_dma_len);
-	ibmr->sg_dma_len = ib_dma_map_sg(dev, ibmr->sg, ibmr->sg_len,
+	ibmr->sg_dma_len = ib_dma_map_sg(dev, sg, sg_len,
 					 DMA_BIDIRECTIONAL);
 	if (unlikely(!ibmr->sg_dma_len)) {
 		pr_warn("RDS/IB: %s failed!\n", __func__);
 		return -EBUSY;
 	}
 
-	frmr->sg_byte_len = 0;
+	ibmr->sg = sg;
+	ibmr->sg_len = sg_len;
 	frmr->dma_npages = 0;
 	len = 0;
 
@@ -264,6 +261,8 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev,
 	ib_dma_unmap_sg(rds_ibdev->dev, ibmr->sg, ibmr->sg_len,
 			DMA_BIDIRECTIONAL);
 	ibmr->sg_dma_len = 0;
+	ibmr->sg = NULL;
+	ibmr->sg_len = 0;
 	return ret;
 }
 
diff --git a/net/rds/rdma.c b/net/rds/rdma.c
index 078090d292fa31..81cfbb2b2e87b8 100644
--- a/net/rds/rdma.c
+++ b/net/rds/rdma.c
@@ -30,7 +30,6 @@
  * SOFTWARE.
  *
  */
-#include <linux/pagemap.h>
 #include <linux/slab.h>
 #include <linux/rbtree.h>
 #include <linux/dma-mapping.h> /* for DMA_*_DEVICE */
diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index 3f653721c45feb..6e48b4bc2d75d9 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -475,6 +475,15 @@ static size_t tcf_action_full_attrs_size(size_t sz)
 		+ sz;
 }
 
+/* tca_get_fill() may append TCA_ROOT_EXT_WARN_MSG from extack->_msg */
+static size_t tcf_action_warn_attr_size(const struct netlink_ext_ack *extack)
+{
+	if (unlikely(extack && extack->_msg))
+		return nla_total_size(strlen(extack->_msg) + 1);
+
+	return 0;
+}
+
 static size_t tcf_action_fill_size(const struct tc_action *act)
 {
 	size_t sz = tcf_action_shared_attrs_size(act);
@@ -758,7 +767,7 @@ static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index)
 
 	mutex_lock(&idrinfo->lock);
 	p = idr_find(&idrinfo->action_idr, index);
-	if (!p) {
+	if (IS_ERR_OR_NULL(p)) {
 		mutex_unlock(&idrinfo->lock);
 		return -ENOENT;
 	}
@@ -1980,7 +1989,8 @@ static struct sk_buff *tcf_del_notify_msg(struct net *net, struct nlmsghdr *n,
 {
 	struct sk_buff *skb;
 
-	skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL);
+	skb = alloc_skb(max(attr_size + tcf_action_warn_attr_size(extack),
+			    NLMSG_GOODSIZE), GFP_KERNEL);
 	if (!skb)
 		return ERR_PTR(-ENOBUFS);
 
@@ -2078,7 +2088,8 @@ static struct sk_buff *tcf_add_notify_msg(struct net *net, struct nlmsghdr *n,
 {
 	struct sk_buff *skb;
 
-	skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL);
+	skb = alloc_skb(max(attr_size + tcf_action_warn_attr_size(extack),
+			    NLMSG_GOODSIZE), GFP_KERNEL);
 	if (!skb)
 		return ERR_PTR(-ENOBUFS);
 
diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c
index c634af5edfcdcb..0b6ac6be60d248 100644
--- a/net/sched/act_connmark.c
+++ b/net/sched/act_connmark.c
@@ -236,12 +236,18 @@ static void tcf_connmark_cleanup(struct tc_action *a)
 		kfree_rcu(parms, rcu);
 }
 
+static size_t tcf_connmark_get_fill_size(const struct tc_action *act)
+{
+	return nla_total_size(sizeof(struct tc_connmark)); /* TCA_CONNMARK_PARMS */
+}
+
 static struct tc_action_ops act_connmark_ops = {
 	.kind		=	"connmark",
 	.id		=	TCA_ID_CONNMARK,
 	.owner		=	THIS_MODULE,
 	.act		=	tcf_connmark_act,
 	.dump		=	tcf_connmark_dump,
+	.get_fill_size	=	tcf_connmark_get_fill_size,
 	.init		=	tcf_connmark_init,
 	.cleanup	=	tcf_connmark_cleanup,
 	.size		=	sizeof(struct tcf_connmark_info),
diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c
index 9080cb386c169b..e26c0d5b2176bc 100644
--- a/net/sched/act_ct.c
+++ b/net/sched/act_ct.c
@@ -349,10 +349,10 @@ static int tcf_ct_flow_table_get(struct net *net, struct tcf_ct_params *params)
 	ct_ft->nf_ft.type = &flowtable_ct;
 	ct_ft->nf_ft.flags |= NF_FLOWTABLE_HW_OFFLOAD |
 			      NF_FLOWTABLE_COUNTER;
+	write_pnet(&ct_ft->nf_ft.net, net);
 	err = nf_flow_table_init(&ct_ft->nf_ft);
 	if (err)
 		goto err_init;
-	write_pnet(&ct_ft->nf_ft.net, net);
 
 	__module_get(THIS_MODULE);
 out_unlock:
@@ -432,11 +432,10 @@ static void tcf_ct_flow_table_add(struct tcf_ct_flow_table *ct_ft,
 	if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status))
 		return;
 
+	/* NULL if ct is dying (raced flush) or the atomic alloc failed. */
 	entry = flow_offload_alloc(ct);
-	if (!entry) {
-		WARN_ON_ONCE(1);
+	if (!entry)
 		goto err_alloc;
-	}
 
 	if (tcp) {
 		ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
@@ -980,14 +979,13 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
 				 struct tcf_result *res)
 {
 	struct net *net = dev_net(skb->dev);
+	bool cached, commit, clear, nat;
 	enum ip_conntrack_info ctinfo;
 	struct tcf_ct *c = to_ct(a);
 	struct nf_conn *tmpl = NULL;
 	struct nf_hook_state state;
-	bool cached, commit, clear;
 	int nh_ofs, err, retval;
 	struct tcf_ct_params *p;
-	bool add_helper = false;
 	bool skb_is_ours = false;
 	bool skip_add = false;
 	bool defrag = false;
@@ -999,6 +997,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
 	retval = p->action;
 	commit = p->ct_action & TCA_CT_ACT_COMMIT;
 	clear = p->ct_action & TCA_CT_ACT_CLEAR;
+	nat = p->ct_action & TCA_CT_ACT_NAT;
 	tmpl = p->tmpl;
 
 	tcf_lastuse_update(&c->tcf_tm);
@@ -1047,6 +1046,19 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
 	 * different zone.
 	 */
 	cached = tcf_ct_skb_nfct_cached(net, skb, p);
+
+	/* If the ct entry is not confirmed and shared with some other skb,
+	 * e.g., a cloned one, we can't just modify it with a commit or nat
+	 * as we must not modify the extension set.  Reset.
+	 */
+	if (cached && (commit || nat)) {
+		ct = nf_ct_get(skb, &ctinfo);
+		if (ct && !nf_ct_is_confirmed(ct) && nf_ct_shared(ct)) {
+			nf_reset_ct(skb);
+			cached = false;
+		}
+	}
+
 	if (!cached) {
 		if (tcf_ct_flow_table_lookup(p, skb, family)) {
 			skip_add = true;
@@ -1083,26 +1095,32 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
 		err = __nf_ct_try_assign_helper(ct, p->tmpl, GFP_ATOMIC);
 		if (err)
 			goto drop;
-		add_helper = true;
-		if (p->ct_action & TCA_CT_ACT_NAT && !nfct_seqadj(ct)) {
+
+		if (nat && !nfct_seqadj(ct)) {
 			if (!nfct_seqadj_ext_add(ct))
 				goto drop;
 		}
 	}
 
-	if (nf_ct_is_confirmed(ct) ? ((!cached && !skip_add) || add_helper) : commit) {
-		err = nf_ct_helper(skb, ct, ctinfo, family);
-		if (err != NF_ACCEPT)
-			goto nf_error;
-	}
-
 	if (commit) {
 		tcf_ct_act_set_mark(ct, p->mark, p->mark_mask);
 		tcf_ct_act_set_labels(ct, p->labels, p->labels_mask);
 
 		if (!nf_ct_is_confirmed(ct))
 			nf_conn_act_ct_ext_add(skb, ct, ctinfo);
+	}
+
+	/* Run helpers for the connection if nf_conntrack_in() was executed
+	 * or if we're about to commit.  This has to be done after all the
+	 * extensions are already added.
+	 */
+	if (nf_ct_is_confirmed(ct) ? (!cached && !skip_add) : commit) {
+		err = nf_ct_helper(skb, ct, ctinfo, family);
+		if (err != NF_ACCEPT)
+			goto nf_error;
+	}
 
+	if (commit) {
 		/* This will take care of sending queued events
 		 * even if the connection is already confirmed.
 		 */
diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c
index 5d228a4022040a..6d6d45e03c079c 100644
--- a/net/sched/act_gate.c
+++ b/net/sched/act_gate.c
@@ -681,7 +681,35 @@ static void tcf_gate_stats_update(struct tc_action *a, u64 bytes, u64 packets,
 
 static size_t tcf_gate_get_fill_size(const struct tc_action *act)
 {
-	return nla_total_size(sizeof(struct tc_gate));
+	struct tcf_gate *gact = to_gate(act);
+	const struct tcf_gate_params *p;
+	struct tcfg_gate_entry *entry;
+	size_t size = nla_total_size(sizeof(struct tc_gate)) /* TCA_GATE_PARMS */
+		+ 3 * nla_total_size_64bit(sizeof(u64)) /* TCA_GATE_BASE_TIME
+							 * TCA_GATE_CYCLE_TIME
+							 * TCA_GATE_CYCLE_TIME_EXT
+							 */
+		+ nla_total_size(sizeof(s32)) /* TCA_GATE_CLOCKID */
+		+ nla_total_size(sizeof(u32)) /* TCA_GATE_FLAGS */
+		+ nla_total_size(sizeof(s32)) /* TCA_GATE_PRIORITY */
+		+ nla_total_size(0); /* TCA_GATE_ENTRY_LIST */
+	/* TCA_GATE_TM is budgeted by tcf_action_shared_attrs_size() */
+
+	rcu_read_lock();
+	p = rcu_dereference(gact->param);
+	if (p) {
+		list_for_each_entry_rcu(entry, &p->entries, list)
+			/* TCA_GATE_ONE_ENTRY nest and its attributes */
+			size += nla_total_size(0)
+				+ nla_total_size(sizeof(u32)) /* TCA_GATE_ENTRY_INDEX */
+				+ nla_total_size(0) /* TCA_GATE_ENTRY_GATE */
+				+ nla_total_size(sizeof(u32)) /* TCA_GATE_ENTRY_INTERVAL */
+				+ nla_total_size(sizeof(s32)) /* TCA_GATE_ENTRY_MAX_OCTETS */
+				+ nla_total_size(sizeof(s32)); /* TCA_GATE_ENTRY_IPV */
+	}
+	rcu_read_unlock();
+
+	return size;
 }
 
 static void tcf_gate_entry_destructor(void *priv)
diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c
index 9cea71fc1db3d4..2afd68983ece40 100644
--- a/net/sched/act_ife.c
+++ b/net/sched/act_ife.c
@@ -737,6 +737,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
 		u8 *curr_data;
 		u16 mtype;
 		u16 dlen;
+		int ret;
 
 		curr_data = ife_tlv_meta_decode(tlv_data, ifehdr_end, &mtype,
 						&dlen, NULL);
@@ -745,13 +746,19 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
 			return TC_ACT_SHOT;
 		}
 
-		if (find_decode_metaid(skb, p, mtype, dlen, curr_data)) {
-			/* abuse overlimits to count when we receive metadata
-			 * but dont have an ops for it
+		ret = find_decode_metaid(skb, p, mtype, dlen, curr_data);
+		if (ret < 0) {
+			/* abuse overlimits to count metadata we cannot
+			 * decode: no ops for it, or the decoder rejected it
 			 */
-			pr_info_ratelimited("Unknown metaid %d dlen %d\n",
-					    mtype, dlen);
 			qstats_cpu_overlimit_inc(ife->common.cpu_qstats);
+
+			if (ret == -ENOENT)
+				pr_info_ratelimited("Unknown metaid %d dlen %d\n",
+						    mtype, dlen);
+			else
+				pr_info_ratelimited("Failed to decode metaid %d dlen %d err %d\n",
+						    mtype, dlen, ret);
 		}
 	}
 
diff --git a/net/sched/act_meta_mark.c b/net/sched/act_meta_mark.c
index ea0573cb8b2d6c..e2f61b22bf0f87 100644
--- a/net/sched/act_meta_mark.c
+++ b/net/sched/act_meta_mark.c
@@ -10,6 +10,7 @@
 #include <linux/string.h>
 #include <linux/errno.h>
 #include <linux/skbuff.h>
+#include <linux/unaligned.h>
 #include <linux/rtnetlink.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -28,9 +29,10 @@ static int skbmark_encode(struct sk_buff *skb, void *skbdata,
 
 static int skbmark_decode(struct sk_buff *skb, void *data, u16 len)
 {
-	u32 ifemark = *(u32 *)data;
+	if (len != sizeof(u32))
+		return -EINVAL;
 
-	skb->mark = ntohl(ifemark);
+	skb->mark = get_unaligned_be32(data);
 	return 0;
 }
 
diff --git a/net/sched/act_meta_skbprio.c b/net/sched/act_meta_skbprio.c
index 2df3133ce5adc9..5cdb57931eab4b 100644
--- a/net/sched/act_meta_skbprio.c
+++ b/net/sched/act_meta_skbprio.c
@@ -10,6 +10,7 @@
 #include <linux/string.h>
 #include <linux/errno.h>
 #include <linux/skbuff.h>
+#include <linux/unaligned.h>
 #include <linux/rtnetlink.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -33,9 +34,10 @@ static int skbprio_encode(struct sk_buff *skb, void *skbdata,
 
 static int skbprio_decode(struct sk_buff *skb, void *data, u16 len)
 {
-	u32 ifeprio = *(u32 *)data;
+	if (len != sizeof(u32))
+		return -EINVAL;
 
-	skb->priority = ntohl(ifeprio);
+	skb->priority = get_unaligned_be32(data);
 	return 0;
 }
 
diff --git a/net/sched/act_meta_skbtcindex.c b/net/sched/act_meta_skbtcindex.c
index 44547caead4690..8803710c09058d 100644
--- a/net/sched/act_meta_skbtcindex.c
+++ b/net/sched/act_meta_skbtcindex.c
@@ -10,6 +10,7 @@
 #include <linux/string.h>
 #include <linux/errno.h>
 #include <linux/skbuff.h>
+#include <linux/unaligned.h>
 #include <linux/rtnetlink.h>
 #include <linux/module.h>
 #include <linux/init.h>
@@ -28,9 +29,10 @@ static int skbtcindex_encode(struct sk_buff *skb, void *skbdata,
 
 static int skbtcindex_decode(struct sk_buff *skb, void *data, u16 len)
 {
-	u16 ifetc_index = *(u16 *)data;
+	if (len != sizeof(u16))
+		return -EINVAL;
 
-	skb->tc_index = ntohs(ifetc_index);
+	skb->tc_index = get_unaligned_be16(data);
 	return 0;
 }
 
diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c
index 553342c55cf7c6..1c98e4d81ebe63 100644
--- a/net/sched/act_mirred.c
+++ b/net/sched/act_mirred.c
@@ -607,7 +607,8 @@ tcf_mirred_get_dev(const struct tc_action *a,
 
 static size_t tcf_mirred_get_fill_size(const struct tc_action *act)
 {
-	return nla_total_size(sizeof(struct tc_mirred));
+	return nla_total_size(sizeof(struct tc_mirred)) /* TCA_MIRRED_PARMS */
+		+ nla_total_size(sizeof(u32)); /* TCA_MIRRED_BLOCKID */
 }
 
 static void tcf_offload_mirred_get_dev(struct flow_action_entry *entry,
diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c
index 4ea8b2e08c3a4d..aa56521b717985 100644
--- a/net/sched/act_mpls.c
+++ b/net/sched/act_mpls.c
@@ -438,12 +438,23 @@ static int tcf_mpls_offload_act_setup(struct tc_action *act, void *entry_data,
 	return 0;
 }
 
+static size_t tcf_mpls_get_fill_size(const struct tc_action *act)
+{
+	return nla_total_size(sizeof(struct tc_mpls)) /* TCA_MPLS_PARMS */
+		+ nla_total_size(sizeof(u32)) /* TCA_MPLS_LABEL */
+		+ nla_total_size(sizeof(u8)) /* TCA_MPLS_TC */
+		+ nla_total_size(sizeof(u8)) /* TCA_MPLS_TTL */
+		+ nla_total_size(sizeof(u8)) /* TCA_MPLS_BOS */
+		+ nla_total_size(sizeof(u16)); /* TCA_MPLS_PROTO */
+}
+
 static struct tc_action_ops act_mpls_ops = {
 	.kind		=	"mpls",
 	.id		=	TCA_ID_MPLS,
 	.owner		=	THIS_MODULE,
 	.act		=	tcf_mpls_act,
 	.dump		=	tcf_mpls_dump,
+	.get_fill_size	=	tcf_mpls_get_fill_size,
 	.init		=	tcf_mpls_init,
 	.cleanup	=	tcf_mpls_cleanup,
 	.offload_act_setup =	tcf_mpls_offload_act_setup,
diff --git a/net/sched/act_nat.c b/net/sched/act_nat.c
index abb332dee836d9..a201979ac7cf75 100644
--- a/net/sched/act_nat.c
+++ b/net/sched/act_nat.c
@@ -313,12 +313,18 @@ static void tcf_nat_cleanup(struct tc_action *a)
 		kfree_rcu(parms, rcu);
 }
 
+static size_t tcf_nat_get_fill_size(const struct tc_action *act)
+{
+	return nla_total_size(sizeof(struct tc_nat)); /* TCA_NAT_PARMS */
+}
+
 static struct tc_action_ops act_nat_ops = {
 	.kind		=	"nat",
 	.id		=	TCA_ID_NAT,
 	.owner		=	THIS_MODULE,
 	.act		=	tcf_nat_act,
 	.dump		=	tcf_nat_dump,
+	.get_fill_size	=	tcf_nat_get_fill_size,
 	.init		=	tcf_nat_init,
 	.cleanup	=	tcf_nat_cleanup,
 	.size		=	sizeof(struct tcf_nat),
diff --git a/net/sched/act_simple.c b/net/sched/act_simple.c
index 8e69a919b4fee0..80e5084b4dbd78 100644
--- a/net/sched/act_simple.c
+++ b/net/sched/act_simple.c
@@ -79,6 +79,12 @@ static int reset_policy(struct tc_action *a, const struct nlattr *defdata,
 	return 0;
 }
 
+static size_t tcf_simp_get_fill_size(const struct tc_action *act)
+{
+	return nla_total_size(sizeof(struct tc_defact)) /* TCA_DEF_PARMS */
+		+ nla_total_size(SIMP_MAX_DATA); /* TCA_DEF_DATA */
+}
+
 static const struct nla_policy simple_policy[TCA_DEF_MAX + 1] = {
 	[TCA_DEF_PARMS]	= { .len = sizeof(struct tc_defact) },
 	[TCA_DEF_DATA]	= { .type = NLA_STRING, .len = SIMP_MAX_DATA },
@@ -204,6 +210,7 @@ static struct tc_action_ops act_simp_ops = {
 	.owner		=	THIS_MODULE,
 	.act		=	tcf_simp_act,
 	.dump		=	tcf_simp_dump,
+	.get_fill_size	=	tcf_simp_get_fill_size,
 	.cleanup	=	tcf_simp_release,
 	.init		=	tcf_simp_init,
 	.size		=	sizeof(struct tcf_defact),
diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c
index bfec6b66841031..26258b353cdeec 100644
--- a/net/sched/act_skbedit.c
+++ b/net/sched/act_skbedit.c
@@ -72,9 +72,8 @@ TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb,
 	}
 	if (params->flags & SKBEDIT_F_QUEUE_MAPPING &&
 	    skb->dev->real_num_tx_queues > params->queue_mapping) {
-#ifdef CONFIG_NET_EGRESS
-		netdev_xmit_skip_txqueue(true);
-#endif
+		if (skb_at_tc_egress(skb))
+			netdev_xmit_skip_txqueue(true);
 		skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb));
 	}
 	if (params->flags & SKBEDIT_F_MARK) {
diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c
index 7579cf1e0ff37c..09e9dcf5825317 100644
--- a/net/sched/act_skbmod.c
+++ b/net/sched/act_skbmod.c
@@ -279,12 +279,21 @@ static int tcf_skbmod_dump(struct sk_buff *skb, struct tc_action *a,
 	return -1;
 }
 
+static size_t tcf_skbmod_get_fill_size(const struct tc_action *act)
+{
+	return nla_total_size(sizeof(struct tc_skbmod)) /* TCA_SKBMOD_PARMS */
+		+ nla_total_size(ETH_ALEN) /* TCA_SKBMOD_DMAC */
+		+ nla_total_size(ETH_ALEN) /* TCA_SKBMOD_SMAC */
+		+ nla_total_size(sizeof(u16)); /* TCA_SKBMOD_ETYPE */
+}
+
 static struct tc_action_ops act_skbmod_ops = {
 	.kind		=	"skbmod",
 	.id		=	TCA_ACT_SKBMOD,
 	.owner		=	THIS_MODULE,
 	.act		=	tcf_skbmod_act,
 	.dump		=	tcf_skbmod_dump,
+	.get_fill_size	=	tcf_skbmod_get_fill_size,
 	.init		=	tcf_skbmod_init,
 	.cleanup	=	tcf_skbmod_cleanup,
 	.size		=	sizeof(struct tcf_skbmod),
diff --git a/net/sched/bpf_qdisc.c b/net/sched/bpf_qdisc.c
index 098ca02aed89c2..5691c13781a8b0 100644
--- a/net/sched/bpf_qdisc.c
+++ b/net/sched/bpf_qdisc.c
@@ -280,7 +280,6 @@ BTF_KFUNCS_START(qdisc_kfunc_ids)
 BTF_ID_FLAGS(func, bpf_skb_get_hash)
 BTF_ID_FLAGS(func, bpf_kfree_skb, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_qdisc_skb_drop, KF_RELEASE)
-BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
 BTF_ID_FLAGS(func, bpf_qdisc_watchdog_schedule)
 BTF_ID_FLAGS(func, bpf_qdisc_init_prologue)
 BTF_ID_FLAGS(func, bpf_qdisc_reset_destroy_epilogue)
@@ -290,7 +289,6 @@ BTF_KFUNCS_END(qdisc_kfunc_ids)
 BTF_SET_START(qdisc_common_kfunc_set)
 BTF_ID(func, bpf_skb_get_hash)
 BTF_ID(func, bpf_kfree_skb)
-BTF_ID(func, bpf_dynptr_from_skb)
 BTF_SET_END(qdisc_common_kfunc_set)
 
 BTF_SET_START(qdisc_enqueue_kfunc_set)
diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index c47d2ee13641d4..a9f54988561f2f 100644
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -2463,7 +2463,20 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
 	}
 
 errout:
-	if (err && tp_state == TP_CREATED)
+	if (err && !IS_ERR_OR_NULL(tp) &&
+	    (tp_state == TP_CREATED || tp->ops->delete_empty))
+		/*
+		 * The request is dropping its reference to tp. If it was
+		 * the last user (the idr is empty now), reclaim the proto.
+		 * A tp this request created is reclaimed unconditionally:
+		 * it is the only owner, so marking it for deletion is
+		 * safe. Otherwise only classifiers with a delete_empty
+		 * callback are reclaimed -- the callback admits an empty
+		 * proto only, so a live shared proto is never unlinked.
+		 * Classifiers without one (deleting is set
+		 * unconditionally) are rtnl-serialized, so the raced
+		 * window this guard closes cannot arise for them.
+		 */
 		tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL);
 errout_tp:
 	if (chain) {
diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c
index 1cefea571efd7d..0c4beff18d6839 100644
--- a/net/sched/cls_flower.c
+++ b/net/sched/cls_flower.c
@@ -1358,17 +1358,29 @@ static int fl_set_vxlan_opt(const struct nlattr *nla, struct fl_flow_key *key,
 
 static int fl_set_erspan_opt(const struct nlattr *nla, struct fl_flow_key *key,
 			     int depth, int option_len,
-			     struct netlink_ext_ack *extack)
+			     struct netlink_ext_ack *extack, u8 *key_ver)
 {
 	struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1];
 	struct erspan_metadata *md;
 	int err;
 
 	md = (struct erspan_metadata *)&key->enc_opts.data[key->enc_opts.len];
-	md->version = 1;
 
-	if (!depth)
+	if (!depth) {
+		/* No mask attribute supplied: exact-match every field the
+		 * key set, keyed off the key's own version so the v2 dir/hwid
+		 * bytes are masked without touching the overlapping
+		 * timestamp/sgt bytes of the union (see 292207809486).
+		 */
+		md->version = *key_ver;
+		if (md->version == 1) {
+			memset(&md->u.index, 0xff, sizeof(md->u.index));
+		} else if (md->version == 2) {
+			md->u.md2.dir = 1;
+			set_hwid(&md->u.md2, 0xff);
+		}
 		return sizeof(*md);
+	}
 
 	if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_ERSPAN) {
 		NL_SET_ERR_MSG(extack, "Non-erspan option type for mask");
@@ -1387,6 +1399,22 @@ static int fl_set_erspan_opt(const struct nlattr *nla, struct fl_flow_key *key,
 
 	if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER])
 		md->version = nla_get_u8(tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]);
+	else
+		md->version = 1;
+
+	/* The key and mask must describe the same union layout: the
+	 * per-version default below only exact-matches the fields of its own
+	 * version, so a mask version that differs from the key's would
+	 * silently wildcard the key's matched fields.  A mask without a
+	 * version keeps the historical v1 default.
+	 */
+	if (option_len && md->version != *key_ver) {
+		NL_SET_ERR_MSG(extack, "Tunnel key erspan option ver mismatch");
+		return -EINVAL;
+	}
+
+	if (!option_len)
+		*key_ver = md->version;
 
 	if (md->version == 1) {
 		if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) {
@@ -1509,6 +1537,7 @@ static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
 {
 	const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL;
 	int err, option_len, key_depth, msk_depth = 0;
+	u8 key_ver = 1;
 
 	err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS],
 					     TCA_FLOWER_KEY_ENC_OPTS_MAX,
@@ -1607,7 +1636,7 @@ static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
 			key->enc_opts.dst_opt_type = IP_TUNNEL_ERSPAN_OPT_BIT;
 			option_len = fl_set_erspan_opt(nla_opt_key, key,
 						       key_depth, option_len,
-						       extack);
+						       extack, &key_ver);
 			if (option_len < 0)
 				return option_len;
 
@@ -1618,7 +1647,7 @@ static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
 			mask->enc_opts.dst_opt_type = IP_TUNNEL_ERSPAN_OPT_BIT;
 			option_len = fl_set_erspan_opt(nla_opt_msk, mask,
 						       msk_depth, option_len,
-						       extack);
+						       extack, &key_ver);
 			if (option_len < 0)
 				return option_len;
 
diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c
index a3e65c8cf29ef3..76ce2d124079d5 100644
--- a/net/sched/cls_u32.c
+++ b/net/sched/cls_u32.c
@@ -1003,8 +1003,16 @@ static int u32_change(struct net *net, struct sk_buff *in_skb,
 				return -ENOMEM;
 			}
 		} else {
-			err = idr_alloc_u32(&tp_c->handle_idr, ht, &handle,
-					    handle, GFP_KERNEL);
+			/* The IDR is keyed on the mapped id, and that is
+			 * what the destroy paths remove. Ask for it here,
+			 * so a manual handle colliding with the
+			 * auto-allocated id space is rejected (-ENOSPC)
+			 * instead of aliasing a future auto id.
+			 */
+			u32 id = handle2id(handle);
+
+			err = idr_alloc_u32(&tp_c->handle_idr, ht, &id, id,
+					    GFP_KERNEL);
 			if (err) {
 				kfree(ht);
 				return err;
diff --git a/net/sched/em_text.c b/net/sched/em_text.c
index 343f1aebeec2ae..4132f8c3c5fc95 100644
--- a/net/sched/em_text.c
+++ b/net/sched/em_text.c
@@ -113,7 +113,7 @@ static void em_text_destroy(struct tcf_ematch *m)
 static int em_text_dump(struct sk_buff *skb, struct tcf_ematch *m)
 {
 	struct text_match *tm = EM_TEXT_PRIV(m);
-	struct tcf_em_text conf;
+	struct tcf_em_text conf = {};
 
 	strscpy(conf.algo, tm->config->ops->name);
 	conf.from_offset = tm->from_offset;
diff --git a/net/sched/ematch.c b/net/sched/ematch.c
index 5c1235e6076ae8..a8e06cf053c9a3 100644
--- a/net/sched/ematch.c
+++ b/net/sched/ematch.c
@@ -470,9 +470,12 @@ int tcf_em_tree_dump(struct sk_buff *skb, struct tcf_ematch_tree *tree, int tlv)
 				goto nla_put_failure;
 		} else if (tcf_em_is_container(em) || tcf_em_is_simple(em)) {
 			u32 u = em->data;
-			nla_put_nohdr(skb, sizeof(u), &u);
-		} else if (em->datalen > 0)
-			nla_put_nohdr(skb, em->datalen, (void *) em->data);
+			if (nla_put_nohdr(skb, sizeof(u), &u))
+				goto nla_put_failure;
+		} else if (em->datalen > 0) {
+			if (nla_put_nohdr(skb, em->datalen, (void *)em->data))
+				goto nla_put_failure;
+		}
 
 		tail = skb_tail_pointer(skb);
 		match_start->nla_len = tail - (u8 *)match_start;
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 463ededcdcfe0c..f36c41224bccf6 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -21,6 +21,7 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/kmod.h>
+#include <linux/bitmap.h>
 #include <linux/list.h>
 #include <linux/hrtimer.h>
 #include <linux/slab.h>
@@ -770,22 +771,75 @@ void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
 }
 EXPORT_SYMBOL(qdisc_class_hash_remove);
 
-/* Allocate an unique handle from space managed by kernel
- * Possible range is [8000-FFFF]:0000 (0x8000 values)
+#define QDISC_AUTO_HANDLE_BASE	0x8000	/* first auto-handle major */
+#define QDISC_AUTO_HANDLE_END	0xFFFE	/* last auto-handle major */
+#define QDISC_AUTO_HANDLE_COUNT	(QDISC_AUTO_HANDLE_END - \
+				 QDISC_AUTO_HANDLE_BASE + 1)
+
+/* Allocate a unique handle from space managed by kernel
+ * Possible range is [8000-FFFE]:0000 (0x7FFF values); 0xFFFF is the
+ * TC_H_ROOT/ingress major.
  */
-static u32 qdisc_alloc_handle(struct net_device *dev)
+static int qdisc_alloc_handle(struct net_device *dev, u32 *handlep)
 {
-	int i = 0x8000;
 	static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
+	unsigned long start, bit;
+	unsigned long *bitmap;
+	struct Qdisc *q;
+	u32 handle, maj;
+	int b;
+
+	start = ((autohandle >> 16) - QDISC_AUTO_HANDLE_BASE + 1) %
+		QDISC_AUTO_HANDLE_COUNT;
+
+	/* The cursor moves past every handle it hands out, so the next
+	 * handle is normally free: try it with one hashed lookup before
+	 * walking the whole hash.
+	 */
+	handle = (start + QDISC_AUTO_HANDLE_BASE) << 16;
+	if (!qdisc_lookup(dev, handle)) {
+		autohandle = handle;
+		*handlep = handle;
+		return 0;
+	}
 
-	do {
-		autohandle += TC_H_MAKE(0x10000U, 0);
-		if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
-			autohandle = TC_H_MAKE(0x80000000U, 0);
-		if (!qdisc_lookup(dev, autohandle))
-			return autohandle;
-		cond_resched();
-	} while	(--i > 0);
+	bitmap = bitmap_zalloc(QDISC_AUTO_HANDLE_COUNT, GFP_KERNEL);
+	if (!bitmap)
+		return -ENOMEM;
+
+	/* Mark the occupied auto-handle majors: the hash holds all
+	 * qdiscs except root and ingress ones, and the root can carry
+	 * an auto-range handle.
+	 */
+	q = rtnl_dereference(dev->qdisc);
+	maj = TC_H_MAJ(q->handle) >> 16;
+	if (!(q->flags & TCQ_F_BUILTIN) && !TC_H_MIN(q->handle) &&
+	    maj >= QDISC_AUTO_HANDLE_BASE && maj <= QDISC_AUTO_HANDLE_END)
+		set_bit(maj - QDISC_AUTO_HANDLE_BASE, bitmap);
+
+	hash_for_each(dev->qdisc_hash, b, q, hash) {
+		maj = TC_H_MAJ(q->handle) >> 16;
+		if (maj >= QDISC_AUTO_HANDLE_BASE &&
+		    maj <= QDISC_AUTO_HANDLE_END && !TC_H_MIN(q->handle))
+			set_bit(maj - QDISC_AUTO_HANDLE_BASE, bitmap);
+	}
+
+	bit = find_next_zero_bit(bitmap, QDISC_AUTO_HANDLE_COUNT, start);
+	if (bit >= QDISC_AUTO_HANDLE_COUNT) {
+		bit = find_first_zero_bit(bitmap, start);
+		if (bit >= start)
+			bit = QDISC_AUTO_HANDLE_COUNT;
+	}
+
+	if (bit >= QDISC_AUTO_HANDLE_COUNT) {
+		bitmap_free(bitmap);
+		return -ENOSPC;
+	}
+
+	handle = (bit + QDISC_AUTO_HANDLE_BASE) << 16;
+	autohandle = handle;
+	bitmap_free(bitmap);
+	*handlep = handle;
 
 	return 0;
 }
@@ -1308,10 +1362,10 @@ static struct Qdisc *qdisc_create(struct net_device *dev,
 		handle = TC_H_MAKE(TC_H_INGRESS, 0);
 	} else {
 		if (handle == 0) {
-			handle = qdisc_alloc_handle(dev);
-			if (handle == 0) {
-				NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded");
-				err = -ENOSPC;
+			err = qdisc_alloc_handle(dev, &handle);
+			if (err) {
+				NL_SET_ERR_MSG(extack,
+					       "Failed to allocate a qdisc handle");
 				goto err_out3;
 			}
 		}
diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c
index 6aa5829d696124..a1ba2e355f2d18 100644
--- a/net/sched/sch_codel.c
+++ b/net/sched/sch_codel.c
@@ -205,7 +205,7 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt,
 	codel_params_init(&q->params);
 	codel_vars_init(&q->vars);
 	codel_stats_init(&q->stats);
-	q->params.mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20);
+	q->params.mtu = min_t(u32, psched_mtu(qdisc_dev(sch)), 1 << 20);
 
 	if (opt) {
 		int err = codel_change(sch, opt, extack);
diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c
index 35f940b2205df1..a282812c192eed 100644
--- a/net/sched/sch_fq.c
+++ b/net/sched/sch_fq.c
@@ -159,6 +159,9 @@ struct fq_sched_data {
 	u64		stat_allocation_errors;
 };
 
+/* EDT timestamps to clear beyond now. */
+static const int fq_offload_slack_ns = 400;
+
 /* return the i-th 2-bit value ("crumb") */
 static u8 fq_prio2band(const u8 *prio2band, unsigned int prio)
 {
@@ -301,6 +304,26 @@ static void fq_gc(struct fq_sched_data *q,
 	q->stat_gc_flows += fcnt;
 }
 
+static u64 fq_offload_horizon(const struct Qdisc *sch,
+			      const struct fq_sched_data *q)
+{
+	const struct net_device *dev;
+	u64 offload_horizon;
+
+	offload_horizon = READ_ONCE(q->offload_horizon);
+	if (!offload_horizon)
+		return 0;
+
+	dev = qdisc_dev(sch);
+	if (!dev->pacing_offload)
+		return 0;
+
+	if (offload_horizon > READ_ONCE(dev->max_pacing_offload_horizon))
+		return 0;
+
+	return offload_horizon;
+}
+
 /* Fast path can be used if :
  * 1) Packet tstamp is in the past, or within the pacing offload horizon.
  * 2) FQ qlen == 0   OR
@@ -312,12 +335,12 @@ static void fq_gc(struct fq_sched_data *q,
  * FQ can not use generic TCQ_F_CAN_BYPASS infrastructure.
  */
 static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb,
-			      u64 now)
+			      u64 now, u64 offload_horizon)
 {
 	const struct fq_sched_data *q = qdisc_priv(sch);
 	const struct sock *sk;
 
-	if (fq_skb_cb(skb)->time_to_send > now + q->offload_horizon)
+	if (fq_skb_cb(skb)->time_to_send > now + offload_horizon)
 		return false;
 
 	if (sch->q.qlen != 0) {
@@ -338,7 +361,7 @@ static bool fq_fastpath_check(const struct Qdisc *sch, struct sk_buff *skb,
 		/* Ordering invariants fall apart if some delayed flows
 		 * are ready but we haven't serviced them, yet.
 		 */
-		if (q->time_next_delayed_flow <= now + q->offload_horizon)
+		if (q->time_next_delayed_flow <= now + offload_horizon)
 			return false;
 	}
 
@@ -357,6 +380,7 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb,
 				   u64 now)
 {
 	struct fq_sched_data *q = qdisc_priv(sch);
+	u64 offload_horizon = fq_offload_horizon(sch, q);
 	struct rb_node **p, *parent;
 	struct sock *sk = skb->sk;
 	struct rb_root *root;
@@ -393,12 +417,17 @@ static struct fq_flow *fq_classify(struct Qdisc *sch, struct sk_buff *skb,
 		sk = (struct sock *)((hash << 1) | 1UL);
 	}
 
-	if (fq_fastpath_check(sch, skb, now)) {
+	if (fq_fastpath_check(sch, skb, now, offload_horizon)) {
 		q->internal.stat_fastpath_packets++;
 		if (skb->sk == sk && q->rate_enable &&
 		    READ_ONCE(sk->sk_pacing_status) != SK_PACING_FQ)
 			smp_store_release(&sk->sk_pacing_status,
 					  SK_PACING_FQ);
+
+		if (offload_horizon &&
+		    fq_skb_cb(skb)->time_to_send <= now + fq_offload_slack_ns)
+			skb_set_delivery_time(skb, 0, SKB_CLOCK_REALTIME);
+
 		return &q->internal;
 	}
 
@@ -661,12 +690,13 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	return NET_XMIT_SUCCESS;
 }
 
-static void fq_check_throttled(struct fq_sched_data *q, u64 now)
+static void fq_check_throttled(struct fq_sched_data *q, u64 now,
+			       u64 offload_horizon)
 {
 	unsigned long sample;
 	struct rb_node *p;
 
-	if (q->time_next_delayed_flow > now + q->offload_horizon)
+	if (q->time_next_delayed_flow > now + offload_horizon)
 		return;
 
 	/* Update unthrottle latency EWMA.
@@ -677,7 +707,7 @@ static void fq_check_throttled(struct fq_sched_data *q, u64 now)
 		q->unthrottle_latency_ns -= q->unthrottle_latency_ns >> 3;
 		q->unthrottle_latency_ns += sample >> 3;
 	}
-	now += q->offload_horizon;
+	now += offload_horizon;
 
 	q->time_next_delayed_flow = ~0ULL;
 	while ((p = rb_first(&q->delayed)) != NULL) {
@@ -705,8 +735,10 @@ static struct fq_flow_head *fq_pband_head_select(struct fq_perband_flows *pband)
 static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 {
 	struct fq_sched_data *q = qdisc_priv(sch);
+	u64 offload_horizon = fq_offload_horizon(sch, q);
 	struct fq_perband_flows *pband;
 	struct fq_flow_head *head;
+	u64 time_next_packet = 0;
 	struct sk_buff *skb;
 	struct fq_flow *f;
 	unsigned long rate;
@@ -721,11 +753,11 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 	if (skb) {
 		q->internal.qlen--;
 		fq_dequeue_skb(sch, &q->internal, skb);
-		goto out;
+		return skb;
 	}
 
 	now = ktime_get_ns();
-	fq_check_throttled(q, now);
+	fq_check_throttled(q, now, offload_horizon);
 	retry = 0;
 	pband = &q->band_flows[q->band_nr];
 begin:
@@ -758,10 +790,10 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 
 	skb = fq_peek(f);
 	if (skb) {
-		u64 time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send,
-					     f->time_next_packet);
+		time_next_packet = max_t(u64, fq_skb_cb(skb)->time_to_send,
+					 f->time_next_packet);
 
-		if (now + q->offload_horizon < time_next_packet) {
+		if (now + offload_horizon < time_next_packet) {
 			head->first = f->next;
 			f->time_next_packet = time_next_packet;
 			fq_flow_set_throttled(q, f);
@@ -836,7 +868,12 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch)
 		}
 		f->time_next_packet = now + len;
 	}
+
 out:
+	if (offload_horizon &&
+	    time_next_packet && time_next_packet <= now + fq_offload_slack_ns)
+		skb_set_delivery_time(skb, 0, SKB_CLOCK_REALTIME);
+
 	return skb;
 }
 
@@ -1179,11 +1216,15 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt,
 		u64 offload_horizon = (u64)NSEC_PER_USEC *
 				      nla_get_u32(tb[TCA_FQ_OFFLOAD_HORIZON]);
 
-		if (offload_horizon <= qdisc_dev(sch)->max_pacing_offload_horizon) {
-			WRITE_ONCE(q->offload_horizon, offload_horizon);
-		} else {
+		if (offload_horizon && !qdisc_dev(sch)->pacing_offload) {
+			NL_SET_ERR_MSG_MOD(extack, "device pacing offload is disabled");
+			err = -EINVAL;
+		} else if (offload_horizon >
+			   qdisc_dev(sch)->max_pacing_offload_horizon) {
 			NL_SET_ERR_MSG_MOD(extack, "invalid offload_horizon");
 			err = -EINVAL;
+		} else {
+			WRITE_ONCE(q->offload_horizon, offload_horizon);
 		}
 	}
 	if (!err) {
diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c
index 969b2510b0b86e..e6c87a32950f9b 100644
--- a/net/sched/sch_fq_codel.c
+++ b/net/sched/sch_fq_codel.c
@@ -517,8 +517,8 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt,
 	q->flows_cnt = 1024;
 	q->memory_limit = 32 << 20; /* 32 MBytes */
 	q->drop_batch_size = 64;
-	mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, FQ_CODEL_QUANTUM_MAX);
-	q->quantum = mtu;
+	mtu = min_t(u32, psched_mtu(qdisc_dev(sch)), FQ_CODEL_QUANTUM_MAX);
+	q->quantum = clamp_t(u32, mtu, 256, FQ_CODEL_QUANTUM_MAX);
 	INIT_LIST_HEAD(&q->new_flows);
 	INIT_LIST_HEAD(&q->old_flows);
 	codel_params_init(&q->cparams);
diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index e87f5021a19958..284490fd6ca91e 100644
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -386,6 +386,15 @@ cftree_update(struct hfsc_class *cl)
 #define	SM_MASK		((1ULL << SM_SHIFT) - 1)
 #define	ISM_MASK	((1ULL << ISM_SHIFT) - 1)
 
+/*
+ * Cap on the non-descending hops a classify walk may take before its
+ * filter chain is treated as misconfigured. A flowid binding that was
+ * legal at bind time can become lateral once hfsc_adjust_levels()
+ * raises a class level; a few such hops are legitimate, an unbounded
+ * run means the chain cycles.
+ */
+#define	HFSC_CLASSIFY_MAX_DRIFT	8
+
 static inline u64
 seg_x2y(u64 x, u64 sm)
 {
@@ -1133,6 +1142,7 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
 	struct hfsc_class *head, *cl;
 	struct tcf_result res;
 	struct tcf_proto *tcf;
+	unsigned int drift;
 	int result;
 
 	if (TC_H_MAJ(skb->priority ^ sch->handle) == 0 &&
@@ -1142,6 +1152,7 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
 
 	*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
 	head = &q->root;
+	drift = HFSC_CLASSIFY_MAX_DRIFT;
 	tcf = rcu_dereference_bh(q->root.filter_list);
 	while (tcf && (result = tcf_classify_qdisc(skb, tcf, &res, false)) >= 0) {
 #ifdef CONFIG_NET_CLS_ACT
@@ -1167,6 +1178,17 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
 		if (cl->level == 0)
 			return cl; /* hit leaf class */
 
+		/*
+		 * flowid binds skip the level check above (res.class is set
+		 * at bind time and levels drift after), so a walk can follow
+		 * lateral hops without descending; a bounded number of them
+		 * is legal, more means the chain cycles.
+		 */
+		if (cl->level >= head->level && drift-- == 0) {
+			pr_warn_ratelimited("hfsc: classify hop budget exhausted, dropping packet\n");
+			return NULL;
+		}
+
 		/* apply inner filter chain */
 		tcf = rcu_dereference_bh(cl->filter_list);
 		head = cl;
diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c
index 9e52afc2d9808c..409ce50cc0dbc7 100644
--- a/net/sched/sch_teql.c
+++ b/net/sched/sch_teql.c
@@ -265,14 +265,11 @@ __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res,
 	}
 
 	if (neigh_event_send(n, skb_res) == 0) {
-		int err;
 		char haddr[MAX_ADDR_LEN];
 
 		neigh_ha_snapshot(haddr, n, dev);
-		err = dev_hard_header(skb, dev, ntohs(skb_protocol(skb, false)),
-				      haddr, NULL, skb->len);
-
-		if (err < 0)
+		if (dev_hard_header(skb, dev, ntohs(skb_protocol(skb, false)),
+				    haddr, NULL, skb->len) < 0)
 			err = -EINVAL;
 	} else {
 		err = (skb_res == NULL) ? -EAGAIN : 1;
diff --git a/net/sctp/associola.c b/net/sctp/associola.c
index c0512c827d0f56..4521be3bd85aa5 100644
--- a/net/sctp/associola.c
+++ b/net/sctp/associola.c
@@ -1289,18 +1289,19 @@ void sctp_assoc_update_retran_path(struct sctp_association *asoc)
 		/* Manually skip the head element. */
 		if (&trans->transports == &asoc->peer.transport_addr_list)
 			continue;
-		if (trans->state == SCTP_UNCONFIRMED)
-			continue;
-		trans_next = sctp_trans_elect_best(trans, trans_next);
-		/* Active is good enough for immediate return. */
-		if (trans_next->state == SCTP_ACTIVE)
-			break;
+		if (trans->state != SCTP_UNCONFIRMED) {
+			trans_next = sctp_trans_elect_best(trans, trans_next);
+			/* Active is good enough for immediate return. */
+			if (trans_next->state == SCTP_ACTIVE)
+				break;
+		}
 		/* We've reached the end, time to update path. */
 		if (trans == asoc->peer.retran_path)
 			break;
 	}
 
-	asoc->peer.retran_path = trans_next;
+	if (trans_next)
+		asoc->peer.retran_path = trans_next;
 
 	pr_debug("%s: association:%p updated new path to addr:%pISpc\n",
 		 __func__, asoc, &asoc->peer.retran_path->ipaddr.sa);
diff --git a/net/sctp/input.c b/net/sctp/input.c
index 864741fae4187e..9494cfa51106c1 100644
--- a/net/sctp/input.c
+++ b/net/sctp/input.c
@@ -436,9 +436,10 @@ void sctp_icmp_proto_unreachable(struct sock *sk,
 		if (timer_pending(&t->proto_unreach_timer))
 			return;
 		else {
-			if (!mod_timer(&t->proto_unreach_timer,
-						jiffies + (HZ/20)))
-				sctp_transport_hold(t);
+			sctp_transport_hold(t);
+			if (mod_timer(&t->proto_unreach_timer,
+				      jiffies + (HZ / 20)))
+				sctp_transport_put(t);
 		}
 	} else {
 		struct net *net = sock_net(sk);
diff --git a/net/sctp/proc.c b/net/sctp/proc.c
index 43433d7e2acd70..7ea123b90aa59d 100644
--- a/net/sctp/proc.c
+++ b/net/sctp/proc.c
@@ -174,7 +174,7 @@ static int sctp_eps_seq_show(struct seq_file *seq, void *v)
 		sk = ep->base.sk;
 		if (!net_eq(sock_net(sk), seq_file_net(seq)))
 			continue;
-		seq_printf(seq, "%8pK %8pK %-3d %-3d %-4d %-5d %5u %5llu ", ep, sk,
+		seq_printf(seq, "%8d %8d %-3d %-3d %-4d %-5d %5u %5llu ", 0, 0,
 			   sctp_sk(sk)->type, sk->sk_state, hash,
 			   ep->base.bind_addr.port,
 			   from_kuid_munged(seq_user_ns(seq), sk_uid(sk)),
@@ -260,9 +260,9 @@ static int sctp_assocs_seq_show(struct seq_file *seq, void *v)
 	sk = epb->sk;
 
 	seq_printf(seq,
-		   "%8pK %8pK %-3d %-3d %-2d %-4d "
+		   "%8d %8d %-3d %-3d %-2d %-4d "
 		   "%4d %8d %8d %7u %5llu %-5d %5d ",
-		   assoc, sk, sctp_sk(sk)->type, sk->sk_state,
+		   0, 0, sctp_sk(sk)->type, sk->sk_state,
 		   assoc->state, 0,
 		   assoc->assoc_id,
 		   assoc->sndbuf_used,
diff --git a/net/sctp/sm_sideeffect.c b/net/sctp/sm_sideeffect.c
index 0d99b7e8c082f0..35f540fb15fc14 100644
--- a/net/sctp/sm_sideeffect.c
+++ b/net/sctp/sm_sideeffect.c
@@ -244,8 +244,9 @@ void sctp_generate_t3_rtx_event(struct timer_list *t)
 		pr_debug("%s: sock is busy\n", __func__);
 
 		/* Try again later.  */
-		if (!mod_timer(&transport->T3_rtx_timer, jiffies + (HZ/20)))
-			sctp_transport_hold(transport);
+		sctp_transport_hold(transport);
+		if (mod_timer(&transport->T3_rtx_timer, jiffies + (HZ / 20)))
+			sctp_transport_put(transport);
 		goto out_unlock;
 	}
 
@@ -280,8 +281,9 @@ static void sctp_generate_timeout_event(struct sctp_association *asoc,
 			 timeout_type);
 
 		/* Try again later.  */
-		if (!mod_timer(&asoc->timers[timeout_type], jiffies + (HZ/20)))
-			sctp_association_hold(asoc);
+		sctp_association_hold(asoc);
+		if (mod_timer(&asoc->timers[timeout_type], jiffies + (HZ / 20)))
+			sctp_association_put(asoc);
 		goto out_unlock;
 	}
 
@@ -378,8 +380,9 @@ void sctp_generate_heartbeat_event(struct timer_list *t)
 		pr_debug("%s: sock is busy\n", __func__);
 
 		/* Try again later.  */
-		if (!mod_timer(&transport->hb_timer, jiffies + (HZ/20)))
-			sctp_transport_hold(transport);
+		sctp_transport_hold(transport);
+		if (mod_timer(&transport->hb_timer, jiffies + (HZ / 20)))
+			sctp_transport_put(transport);
 		goto out_unlock;
 	}
 
@@ -388,8 +391,9 @@ void sctp_generate_heartbeat_event(struct timer_list *t)
 	timeout = sctp_transport_timeout(transport);
 	if (elapsed < timeout) {
 		elapsed = timeout - elapsed;
-		if (!mod_timer(&transport->hb_timer, jiffies + elapsed))
-			sctp_transport_hold(transport);
+		sctp_transport_hold(transport);
+		if (mod_timer(&transport->hb_timer, jiffies + elapsed))
+			sctp_transport_put(transport);
 		goto out_unlock;
 	}
 
@@ -422,9 +426,10 @@ void sctp_generate_proto_unreach_event(struct timer_list *t)
 		pr_debug("%s: sock is busy\n", __func__);
 
 		/* Try again later.  */
-		if (!mod_timer(&transport->proto_unreach_timer,
-				jiffies + (HZ/20)))
-			sctp_transport_hold(transport);
+		sctp_transport_hold(transport);
+		if (mod_timer(&transport->proto_unreach_timer,
+			      jiffies + (HZ / 20)))
+			sctp_transport_put(transport);
 		goto out_unlock;
 	}
 
@@ -458,8 +463,9 @@ void sctp_generate_reconf_event(struct timer_list *t)
 		pr_debug("%s: sock is busy\n", __func__);
 
 		/* Try again later.  */
-		if (!mod_timer(&transport->reconf_timer, jiffies + (HZ / 20)))
-			sctp_transport_hold(transport);
+		sctp_transport_hold(transport);
+		if (mod_timer(&transport->reconf_timer, jiffies + (HZ / 20)))
+			sctp_transport_put(transport);
 		goto out_unlock;
 	}
 
@@ -495,8 +501,9 @@ void sctp_generate_probe_event(struct timer_list *t)
 		pr_debug("%s: sock is busy\n", __func__);
 
 		/* Try again later.  */
-		if (!mod_timer(&transport->probe_timer, jiffies + (HZ / 20)))
-			sctp_transport_hold(transport);
+		sctp_transport_hold(transport);
+		if (mod_timer(&transport->probe_timer, jiffies + (HZ / 20)))
+			sctp_transport_put(transport);
 		goto out_unlock;
 	}
 
diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c
index c701cff6ea67cc..8dc65498763cc1 100644
--- a/net/sctp/sm_statefuns.c
+++ b/net/sctp/sm_statefuns.c
@@ -2654,6 +2654,8 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale(
 
 	sctp_add_cmd_sf(commands, SCTP_CMD_REPLY, SCTP_CHUNK(reply));
 
+	sctp_add_cmd_sf(commands, SCTP_CMD_DISCARD_PACKET, SCTP_NULL());
+
 	return SCTP_DISPOSITION_CONSUME;
 
 nomem:
diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index 04aedd957543a5..9974149659c2f2 100644
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -1849,6 +1849,7 @@ void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport)
 	struct smc_link_group *lgr, *n;
 	int i;
 
+	spin_lock_bh(&smc_lgr_list.lock);
 	list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) {
 		if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id,
 			    SMC_MAX_PNETID_LEN))
@@ -1863,6 +1864,7 @@ void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport)
 				smcr_link_down_cond_sched(lnk);
 		}
 	}
+	spin_unlock_bh(&smc_lgr_list.lock);
 }
 
 static void smc_link_down_work(struct work_struct *work)
diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c
index 9bb495707445eb..daaa8a72da90fe 100644
--- a/net/smc/smc_ib.c
+++ b/net/smc/smc_ib.c
@@ -333,6 +333,7 @@ static bool smc_ib_check_link_gid(u8 gid[SMC_GID_SIZE], bool smcrv2,
 static void smc_ib_gid_check(struct smc_ib_device *smcibdev, u8 ibport)
 {
 	struct smc_link_group *lgr;
+	bool stale_gid = false;
 	int i;
 
 	spin_lock_bh(&smc_lgr_list.lock);
@@ -348,11 +349,16 @@ static void smc_ib_gid_check(struct smc_ib_device *smcibdev, u8 ibport)
 				continue;
 			if (!smc_ib_check_link_gid(lgr->lnk[i].gid,
 						   lgr->smc_version == SMC_V2,
-						   smcibdev, ibport))
-				smcr_port_err(smcibdev, ibport);
+						   smcibdev, ibport)) {
+				stale_gid = true;
+				goto out;
+			}
 		}
 	}
+out:
 	spin_unlock_bh(&smc_lgr_list.lock);
+	if (stale_gid)
+		smcr_port_err(smcibdev, ibport);
 }
 
 static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
diff --git a/net/socket.c b/net/socket.c
index c05d86e63abf7d..ffb911ad8814a6 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -422,7 +422,7 @@ static const struct xattr_handler sockfs_xattr_handler = {
 };
 
 static int sockfs_security_xattr_set(const struct xattr_handler *handler,
-				     struct mnt_idmap *idmap,
+				     const struct mnt_idmap *idmap,
 				     struct dentry *dentry, struct inode *inode,
 				     const char *suffix, const void *value,
 				     size_t size, int flags)
@@ -447,7 +447,7 @@ static int sockfs_user_xattr_get(const struct xattr_handler *handler,
 }
 
 static int sockfs_user_xattr_set(const struct xattr_handler *handler,
-				 struct mnt_idmap *idmap,
+				 const struct mnt_idmap *idmap,
 				 struct dentry *dentry, struct inode *inode,
 				 const char *suffix, const void *value,
 				 size_t size, int flags)
@@ -670,7 +670,7 @@ static ssize_t sockfs_listxattr(struct dentry *dentry, char *buffer,
 	return used;
 }
 
-static int sockfs_setattr(struct mnt_idmap *idmap,
+static int sockfs_setattr(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, struct iattr *iattr)
 {
 	int err = simple_setattr(&nop_mnt_idmap, dentry, iattr);
@@ -800,7 +800,10 @@ static inline int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg)
 	int ret = INDIRECT_CALL_INET(READ_ONCE(sock->ops)->sendmsg, inet6_sendmsg,
 				     inet_sendmsg, sock, msg,
 				     msg_data_left(msg));
-	BUG_ON(ret == -EIOCBQUEUED);
+	if (unlikely(ret == -EIOCBQUEUED)) {
+		DEBUG_NET_WARN_ON_ONCE(1);
+		ret = -EIO;
+	}
 
 	if (trace_sock_send_length_enabled())
 		call_trace_sock_send_length(sock->sk, ret, 0);
@@ -1851,10 +1854,7 @@ int __sys_socketpair(int family, int type, int protocol, int __user *usockvec)
 	if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK))
 		flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
 
-	/*
-	 * reserve descriptors and make sure we won't fail
-	 * to return them to userland.
-	 */
+	/* Reserve both descriptors before creating the sockets. */
 	fd1 = get_unused_fd_flags(flags);
 	if (unlikely(fd1 < 0))
 		return fd1;
@@ -1865,14 +1865,6 @@ int __sys_socketpair(int family, int type, int protocol, int __user *usockvec)
 		return fd2;
 	}
 
-	err = put_user(fd1, &usockvec[0]);
-	if (err)
-		goto out;
-
-	err = put_user(fd2, &usockvec[1]);
-	if (err)
-		goto out;
-
 	/*
 	 * Obtain the first socket and check if the underlying protocol
 	 * supports the socketpair call.
@@ -1916,6 +1908,16 @@ int __sys_socketpair(int family, int type, int protocol, int __user *usockvec)
 		goto out;
 	}
 
+	/* Publish the descriptors now that it shouldn't fail. */
+	err = put_user(fd1, &usockvec[0]);
+	if (!err)
+		err = put_user(fd2, &usockvec[1]);
+	if (err) {
+		fput(newfile2);
+		fput(newfile1);
+		goto out;
+	}
+
 	audit_fd_pair(fd1, fd2);
 
 	fd_install(fd1, newfile1);
@@ -2437,8 +2439,8 @@ INDIRECT_CALLABLE_DECLARE(bool tcp_bpf_bypass_getsockopt(int level,
  * It is important to remember that both iov points to the same data, but,
  * .iter_in is read-only and .iter_out is write-only by the protocol callbacks
  */
-static int sockptr_to_sockopt(sockopt_t *opt, sockptr_t optval,
-			      sockptr_t optlen, struct kvec *kvec)
+int sockptr_to_sockopt(sockopt_t *opt, sockptr_t optval,
+		       sockptr_t optlen, struct kvec *kvec)
 {
 	int koptlen;
 
diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
index 8ddc65e894dab8..eb4a32382a1b4a 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+++ b/net/sunrpc/auth_gss/auth_gss.c
@@ -16,7 +16,6 @@
 #include <linux/types.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
-#include <linux/pagemap.h>
 #include <linux/sunrpc/clnt.h>
 #include <linux/sunrpc/auth.h>
 #include <linux/sunrpc/auth_gss.h>
diff --git a/net/sunrpc/auth_gss/gss_krb5_crypto.c b/net/sunrpc/auth_gss/gss_krb5_crypto.c
index 09a7c97e87da7f..5140d98ede1fc5 100644
--- a/net/sunrpc/auth_gss/gss_krb5_crypto.c
+++ b/net/sunrpc/auth_gss/gss_krb5_crypto.c
@@ -39,7 +39,6 @@
 #include <linux/mm.h>
 #include <linux/scatterlist.h>
 #include <linux/highmem.h>
-#include <linux/pagemap.h>
 #include <linux/sunrpc/gss_krb5.h>
 #include <linux/sunrpc/xdr.h>
 
diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c
index d3f61c4b5a1375..789901e0f83a00 100644
--- a/net/sunrpc/auth_gss/gss_krb5_wrap.c
+++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c
@@ -31,7 +31,6 @@
 #include <linux/types.h>
 #include <linux/jiffies.h>
 #include <linux/sunrpc/gss_krb5.h>
-#include <linux/pagemap.h>
 
 #include "gss_krb5_internal.h"
 
diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index 967e9d53080d79..44f4b093193df9 100644
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -41,7 +41,6 @@
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/module.h>
-#include <linux/pagemap.h>
 #include <linux/user_namespace.h>
 
 #include <linux/sunrpc/auth_gss.h>
diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index 1bc04109d213d7..825e47833e0954 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -27,7 +27,6 @@
 #include <linux/net.h>
 #include <linux/workqueue.h>
 #include <linux/mutex.h>
-#include <linux/pagemap.h>
 #include <asm/ioctls.h>
 #include <linux/sunrpc/types.h>
 #include <linux/sunrpc/cache.h>
diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c
index 9d349cfbc48312..b4054fad2e0331 100644
--- a/net/sunrpc/rpc_pipe.c
+++ b/net/sunrpc/rpc_pipe.c
@@ -12,7 +12,6 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/string.h>
-#include <linux/pagemap.h>
 #include <linux/mount.h>
 #include <linux/fs_context.h>
 #include <linux/namei.h>
diff --git a/net/sunrpc/socklib.c b/net/sunrpc/socklib.c
index d8d8842c7de5cd..83ec61c4536571 100644
--- a/net/sunrpc/socklib.c
+++ b/net/sunrpc/socklib.c
@@ -12,7 +12,6 @@
 #include <linux/gfp.h>
 #include <linux/skbuff.h>
 #include <linux/types.h>
-#include <linux/pagemap.h>
 #include <linux/udp.h>
 #include <linux/sunrpc/msg_prot.h>
 #include <linux/sunrpc/sched.h>
diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index 40040af588fb23..77e28dcc4d2ab3 100644
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -64,8 +64,10 @@ static LIST_HEAD(svc_xprt_class_list);
  *		- Can be set or cleared at any time.
  *		- After a set, svc_xprt_enqueue must be called to enqueue
  *		  the transport for processing.
- *		- After a clear, the transport must be read/accepted.
- *		  If this succeeds, it must be set again.
+ *		- After clearing XPT_CONN, the transport must be
+ *		  accepted. If this succeeds, the bit must be set again.
+ *		- xpo_recvfrom decides when XPT_DATA is cleared; see
+ *		  svc_xprt_received.
  *	XPT_CLOSE:
  *		- Can set at any time. It is never cleared.
  *      XPT_DEAD:
@@ -218,8 +220,9 @@ EXPORT_SYMBOL_GPL(svc_xprt_init);
  * The caller must hold the XPT_BUSY bit and must
  * not thereafter touch transport data.
  *
- * Note: XPT_DATA only gets cleared when a read-attempt finds no (or
- * insufficient) data.
+ * Note: xpo_recvfrom decides when to clear XPT_DATA. A transport may
+ * leave the bit set until a read attempt finds no (or insufficient)
+ * data, or clear it as soon as it consumes the last queued receive.
  */
 void svc_xprt_received(struct svc_xprt *xprt)
 {
@@ -475,11 +478,11 @@ static bool svc_xprt_ready(struct svc_xprt *xprt)
 
 	/*
 	 * If another cpu has recently updated xpt_flags,
-	 * sk_sock->flags, xpt_reserved, or xpt_nr_rqsts, we need to
-	 * know about it; otherwise it's possible that both that cpu and
-	 * this one could call svc_xprt_enqueue() without either
-	 * svc_xprt_enqueue() recognizing that the conditions below
-	 * are satisfied, and we could stall indefinitely:
+	 * sk_sock->flags, xpt_reserved (UDP only), or xpt_nr_rqsts,
+	 * we need to know about it; otherwise it's possible that both
+	 * that cpu and this one could call svc_xprt_enqueue() without
+	 * either svc_xprt_enqueue() recognizing that the conditions
+	 * below are satisfied, and we could stall indefinitely:
 	 */
 	smp_rmb();
 	xpt_flags = READ_ONCE(xprt->xpt_flags);
@@ -551,6 +554,10 @@ static struct svc_xprt *svc_xprt_dequeue(struct svc_pool *pool)
  * to make sure the reply fits.  This function reduces that reserved
  * space to be the amount of space used already, plus @space.
  *
+ * The transport's reservation is tracked only on classes that set
+ * SVC_XPRT_FLAG_WSPACE_RESERVE.  On the others, only @rqstp's
+ * reservation is updated.
+ *
  */
 void svc_reserve(struct svc_rqst *rqstp, int space)
 {
@@ -559,10 +566,12 @@ void svc_reserve(struct svc_rqst *rqstp, int space)
 	space += rqstp->rq_res.head[0].iov_len;
 
 	if (xprt && space < rqstp->rq_reserved) {
-		atomic_sub((rqstp->rq_reserved - space),
-			   &xprt->xpt_reserved);
+		if (xprt->xpt_class->xcl_flags & SVC_XPRT_FLAG_WSPACE_RESERVE) {
+			atomic_sub((rqstp->rq_reserved - space),
+				   &xprt->xpt_reserved);
+			svc_xprt_resource_released(xprt);
+		}
 		rqstp->rq_reserved = space;
-		svc_xprt_resource_released(xprt);
 	}
 }
 EXPORT_SYMBOL_GPL(svc_reserve);
@@ -869,7 +878,8 @@ static void svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt)
 		else
 			len = xprt->xpt_ops->xpo_recvfrom(rqstp);
 		rqstp->rq_reserved = serv->sv_max_mesg;
-		atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);
+		if (xprt->xpt_class->xcl_flags & SVC_XPRT_FLAG_WSPACE_RESERVE)
+			atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);
 		if (len <= 0)
 			goto out;
 
diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c
index 31a1bc60a5f6d3..a68fe44d1f6259 100644
--- a/net/sunrpc/svcauth_unix.c
+++ b/net/sunrpc/svcauth_unix.c
@@ -540,6 +540,13 @@ static int unix_gid_parse(struct cache_detail *cd,
 	if (ugp) {
 		struct cache_head *ch;
 		ug.h.flags = 0;
+		/*
+		 * mountd sends at least the user's primary group on
+		 * success, so an empty list can only mean the lookup
+		 * failed. Keep the credential's own groups instead.
+		 */
+		if (gids == 0)
+			set_bit(CACHE_NEGATIVE, &ug.h.flags);
 		ug.h.expiry_time = expiry;
 		ch = sunrpc_cache_update(cd,
 					 &ug.h, &ugp->h,
@@ -730,6 +737,8 @@ static int sunrpc_nl_parse_one_unix_gid(struct cache_detail *cd,
 			   boot.tv_sec;
 
 	if (tb[SUNRPC_A_UNIX_GID_NEGATIVE]) {
+		/* failed lookup: keep the credential's own groups */
+		set_bit(CACHE_NEGATIVE, &ug.h.flags);
 		ug.gi = groups_alloc(0);
 		if (!ug.gi)
 			return -ENOMEM;
diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c
index 50e5e7f5b762de..e5459d504b6a91 100644
--- a/net/sunrpc/svcsock.c
+++ b/net/sunrpc/svcsock.c
@@ -8,15 +8,6 @@
  * evenly when servicing a single client. May need to modify the
  * svc_xprt_enqueue procedure...
  *
- * TCP support is largely untested and may be a little slow. The problem
- * is that we currently do two separate recvfrom's, one for the 4-byte
- * record length, and the second for the actual record. This could possibly
- * be improved by always reading a minimum size of around 100 bytes and
- * tucking any superfluous bytes away in a temporary store. Still, that
- * leaves write requests out in the rain. An alternative may be to peek at
- * the first skb in the queue, and if it matches the next TCP sequence
- * number, to extract the record marker. Yuck.
- *
  * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
  */
 
@@ -238,136 +229,108 @@ static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining)
 	return len;
 }
 
-static int
-svc_tcp_sock_process_cmsg(struct socket *sock, struct msghdr *msg,
-			  struct cmsghdr *cmsg, int ret)
-{
-	u8 content_type = tls_get_record_type(sock->sk, cmsg);
-	u8 level, description;
-
-	switch (content_type) {
-	case 0:
-		break;
-	case TLS_RECORD_TYPE_DATA:
-		/* TLS sets EOR at the end of each application data
-		 * record, even though there might be more frames
-		 * waiting to be decrypted.
-		 */
-		msg->msg_flags &= ~MSG_EOR;
-		break;
-	case TLS_RECORD_TYPE_ALERT:
-		tls_alert_recv(sock->sk, msg, &level, &description);
-		ret = (level == TLS_ALERT_LEVEL_FATAL) ?
-			-ENOTCONN : -EAGAIN;
-		break;
-	default:
-		/* discard this record type */
-		ret = -EAGAIN;
-	}
-	return ret;
-}
-
-static int
-svc_tcp_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags)
+/*
+ * The ->read_sock data path invokes neither security_socket_recvmsg()
+ * nor the sock:sock_recv_length tracepoint. Dispatch ->recvmsg
+ * directly so the whole receive path behaves one way.
+ */
+static int svc_tcp_recv_cmsg(struct socket *sock, int flags,
+			     struct kvec *payload, u8 *type,
+			     unsigned int *msg_flags)
 {
+	const struct proto_ops *ops = READ_ONCE(sock->ops);
 	union {
 		struct cmsghdr	cmsg;
 		u8		buf[CMSG_SPACE(sizeof(u8))];
-	} u;
-	u8 alert[2];
-	struct kvec alert_kvec = {
-		.iov_base = alert,
-		.iov_len = sizeof(alert),
-	};
+	} u = {};
 	struct msghdr msg = {
-		.msg_flags = *msg_flags,
-		.msg_control = &u,
-		.msg_controllen = sizeof(u),
+		.msg_control	= &u,
+		.msg_controllen	= sizeof(u),
 	};
 	int ret;
 
-	iov_iter_kvec(&msg.msg_iter, ITER_DEST, &alert_kvec, 1,
-		      alert_kvec.iov_len);
-	ret = sock_recvmsg(sock, &msg, MSG_DONTWAIT);
-	if (ret > 0 &&
-	    tls_get_record_type(sock->sk, &u.cmsg) == TLS_RECORD_TYPE_ALERT) {
-		iov_iter_revert(&msg.msg_iter, ret);
-		ret = svc_tcp_sock_process_cmsg(sock, &msg, &u.cmsg, -EAGAIN);
-	}
+	iov_iter_kvec(&msg.msg_iter, ITER_DEST, payload, 1, payload->iov_len);
+	ret = ops->recvmsg(sock, &msg, msg_data_left(&msg), flags);
+	if (ret < 0)
+		return ret;
+	*msg_flags = msg.msg_flags;
+	*type = tls_get_record_type(sock->sk, &u.cmsg);
+	if (!*type && ret)
+		return -EBADMSG;
 	return ret;
 }
 
-static int
-svc_tcp_sock_recvmsg(struct svc_sock *svsk, struct msghdr *msg)
+static int svc_tcp_recv_ctrl_record(struct svc_sock *svsk)
 {
-	int ret;
+	u8 alert[2], type, level, description;
 	struct socket *sock = svsk->sk_sock;
-
-	ret = sock_recvmsg(sock, msg, MSG_DONTWAIT);
-	if (msg->msg_flags & MSG_CTRUNC) {
-		msg->msg_flags &= ~(MSG_CTRUNC | MSG_EOR);
-		if (ret == 0 || ret == -EIO)
-			ret = svc_tcp_sock_recv_cmsg(sock, &msg->msg_flags);
-	}
-	return ret;
-}
-
-#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
-static void svc_flush_bvec(const struct bio_vec *bvec, size_t size, size_t seek)
-{
-	struct bvec_iter bi = {
-		.bi_size	= size + seek,
+	struct kvec recv_kvec = {
+		.iov_base	= alert,
+		.iov_len	= sizeof(alert),
 	};
-	struct bio_vec bv;
-
-	bvec_iter_advance(bvec, &bi, seek & PAGE_MASK);
-	for_each_bvec(bv, bvec, bi, bi)
-		flush_dcache_page(bv.bv_page);
-}
-#else
-static inline void svc_flush_bvec(const struct bio_vec *bvec, size_t size,
-				  size_t seek)
-{
-}
-#endif
-
-/*
- * Read from @rqstp's transport socket. The incoming message fills whole
- * pages in @rqstp's rq_pages array until the last page of the message
- * has been received into a partial page.
- */
-static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen,
-				size_t seek)
-{
-	struct svc_sock *svsk =
-		container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
-	struct bio_vec *bvec = rqstp->rq_bvec;
-	struct msghdr msg = { NULL };
-	unsigned int i;
-	ssize_t len;
-	size_t t;
-
-	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
+	unsigned int msg_flags;
+	struct msghdr msg = {};
+	int ret;
 
-	for (i = 0, t = 0; t < buflen; i++, t += PAGE_SIZE)
-		bvec_set_page(&bvec[i], rqstp->rq_pages[i], PAGE_SIZE, 0);
+	if (!test_bit(XPT_TLS_SESSION, &svsk->sk_xprt.xpt_flags))
+		return 0;
 
-	iov_iter_bvec(&msg.msg_iter, ITER_DEST, bvec, i, buflen);
-	if (seek) {
-		iov_iter_advance(&msg.msg_iter, seek);
-		buflen -= seek;
+	/* A data record can become ready between ->read_sock returning
+	 * and this probe. A plain receive would take two octets of it
+	 * as RPC payload, so peek.
+	 */
+	ret = svc_tcp_recv_cmsg(sock, MSG_DONTWAIT | MSG_PEEK,
+				&recv_kvec, &type, &msg_flags);
+	if (ret == -EAGAIN || (!ret && !type))
+		return 0;
+	if (ret < 0)
+		return ret;
+	if (type == TLS_RECORD_TYPE_DATA) {
+		/* The peek parks the decrypted record on ctx->rx_list,
+		 * where it draws no further data_ready. Re-arm or the
+		 * RPC hangs until the client times out.
+		 */
+		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
+		return 0;
 	}
-	len = svc_tcp_sock_recvmsg(svsk, &msg);
-	if (len > 0)
-		svc_flush_bvec(bvec, len, seek);
+	if (type != TLS_RECORD_TYPE_ALERT)
+		return -EPROTO;
+
+	ret = svc_tcp_recv_cmsg(sock, MSG_DONTWAIT, &recv_kvec, &type,
+				&msg_flags);
+	/* The peek found a record at the head, so an -EAGAIN here is
+	 * spurious. Propagating it strands the record with no later
+	 * announcement, so return -EBADMSG, which closes the transport.
+	 */
+	if (ret == -EAGAIN)
+		return -EBADMSG;
+	if (ret < 0)
+		return ret;
+
+	/* An Alert record carries exactly one two-octet message (RFC
+	 * 8446 Section 5.1). recv_kvec caps the receive at two, so a
+	 * longer record produces the same count. MSG_EOR appears only
+	 * once kTLS has drained the whole record.
+	 */
+	if (ret != sizeof(alert) || !(msg_flags & MSG_EOR))
+		return -EBADMSG;
+
+	iov_iter_kvec(&msg.msg_iter, ITER_DEST, &recv_kvec, 1,
+		      recv_kvec.iov_len);
+	tls_alert_recv(sock->sk, &msg, &level, &description);
 
-	/* If we read a full record, then assume there may be more
-	 * data to read (stream based sockets only!)
+	/* RFC 8446 Section 6: every alert but a closure alert is
+	 * an error alert. kTLS raises no data_ready for records it
+	 * already holds, so re-arm for what sits behind the alert.
 	 */
-	if (len == buflen)
+	switch (description) {
+	case TLS_ALERT_DESC_CLOSE_NOTIFY:
+	case TLS_ALERT_DESC_USER_CANCELED:
 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
-
-	return len;
+		return -EAGAIN;
+	default:
+		return -ENOTCONN;
+	}
 }
 
 /*
@@ -837,6 +800,7 @@ static struct svc_xprt_class svc_udp_class = {
 	.xcl_ops = &svc_udp_ops,
 	.xcl_max_payload = RPCSVC_MAXPAYLOAD_UDP,
 	.xcl_ident = XPRT_TRANSPORT_UDP,
+	.xcl_flags = SVC_XPRT_FLAG_WSPACE_RESERVE,
 };
 
 static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv)
@@ -992,14 +956,14 @@ static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt)
 	return NULL;
 }
 
-static size_t svc_tcp_restore_pages(struct svc_sock *svsk,
-				    struct svc_rqst *rqstp)
+static void svc_tcp_restore_pages(struct svc_sock *svsk,
+				  struct svc_rqst *rqstp)
 {
 	size_t len = svsk->sk_datalen;
 	unsigned int i, npages;
 
 	if (!len)
-		return 0;
+		return;
 	npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 	for (i = 0; i < npages; i++) {
 		if (rqstp->rq_pages[i] != NULL)
@@ -1009,7 +973,6 @@ static size_t svc_tcp_restore_pages(struct svc_sock *svsk,
 		svsk->sk_pages[i] = NULL;
 	}
 	rqstp->rq_arg.head[0].iov_base = page_address(rqstp->rq_pages[0]);
-	return len;
 }
 
 static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)
@@ -1048,50 +1011,6 @@ static void svc_tcp_clear_pages(struct svc_sock *svsk)
 	svsk->sk_datalen = 0;
 }
 
-/*
- * Receive fragment record header into sk_marker.
- */
-static ssize_t svc_tcp_read_marker(struct svc_sock *svsk,
-				   struct svc_rqst *rqstp)
-{
-	ssize_t want, len;
-
-	/* If we haven't gotten the record length yet,
-	 * get the next four bytes.
-	 */
-	if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) {
-		struct msghdr	msg = { NULL };
-		struct kvec	iov;
-
-		want = sizeof(rpc_fraghdr) - svsk->sk_tcplen;
-		iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen;
-		iov.iov_len  = want;
-		iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, want);
-		len = svc_tcp_sock_recvmsg(svsk, &msg);
-		if (len < 0)
-			return len;
-		svsk->sk_tcplen += len;
-		if (len < want) {
-			/* call again to read the remaining bytes */
-			goto err_short;
-		}
-		trace_svcsock_marker(&svsk->sk_xprt, svsk->sk_marker);
-		if (svc_sock_reclen(svsk) + svsk->sk_datalen >
-		    svsk->sk_xprt.xpt_server->sv_max_mesg)
-			goto err_too_large;
-	}
-	return svc_sock_reclen(svsk);
-
-err_too_large:
-	net_notice_ratelimited("svc: %s oversized RPC fragment (%u octets) from %pISpc\n",
-			       svsk->sk_xprt.xpt_server->sv_name,
-			       svc_sock_reclen(svsk),
-			       (struct sockaddr *)&svsk->sk_xprt.xpt_remote);
-	svc_xprt_deferred_close(&svsk->sk_xprt);
-err_short:
-	return -EAGAIN;
-}
-
 static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp)
 {
 	struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt;
@@ -1129,20 +1048,162 @@ static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp)
 
 static void svc_tcp_fragment_received(struct svc_sock *svsk)
 {
-	/* If we have more data, signal svc_xprt_enqueue() to try again */
 	svsk->sk_tcplen = 0;
 	svsk->sk_marker = xdr_zero;
 }
 
+/*
+ * A non-final fragment carries four octets of marker and may carry
+ * no payload at all. sk_datalen advances only by the payload, so a
+ * run of tiny fragments exhausts ->read_sock's byte budget before
+ * the sv_max_mesg check trips, and a run of empty ones never trips
+ * it. Cap the fragments per socket-lock hold. The cap leaves the
+ * record incomplete, and svc_tcp_recvfrom() resumes it on the next
+ * call.
+ */
+#define SVC_TCP_MAX_FRAGS		256
+
+struct svc_tcp_recv_ctx {
+	struct svc_rqst		*rqstp;
+	unsigned int		frags;
+	bool			complete;
+};
+
+/*
+ * Nothing reads the message body before the message is complete, and
+ * partial receives refill the same pages. Flush once here, after the
+ * socket lock is released, rather than once per copy in the actor.
+ */
+static void svc_tcp_flush_pages(struct svc_sock *svsk,
+				struct svc_rqst *rqstp)
+{
+	unsigned int pg, pages = DIV_ROUND_UP(svsk->sk_datalen, PAGE_SIZE);
+
+	for (pg = 0; pg < pages; pg++)
+		flush_dcache_page(rqstp->rq_pages[pg]);
+}
+
+/*
+ * Mapping the message's unfilled remainder would re-map untouched
+ * pages on every call, at a cost that grows with the message rather
+ * than with the octets copied.
+ */
+static void svc_tcp_recv_iter_init(struct svc_rqst *rqstp,
+				   struct iov_iter *iter, size_t body_off,
+				   size_t len)
+{
+	unsigned int first = body_off >> PAGE_SHIFT;
+	size_t seek = offset_in_page(body_off);
+	unsigned int i, pages = DIV_ROUND_UP(seek + len, PAGE_SIZE);
+
+	for (i = 0; i < pages; i++)
+		bvec_set_page(&rqstp->rq_bvec[i], rqstp->rq_pages[first + i],
+			      PAGE_SIZE, 0);
+
+	iov_iter_bvec(iter, ITER_DEST, rqstp->rq_bvec, pages, seek + len);
+	iov_iter_advance(iter, seek);
+}
+
+/*
+ * ->read_sock actor, called under the socket lock. sk_datalen is both
+ * the count of body octets received so far and their write offset into
+ * rq_pages.
+ */
+static int svc_tcp_recv_actor(read_descriptor_t *desc, struct sk_buff *skb,
+			      unsigned int offset, size_t len)
+{
+	struct svc_tcp_recv_ctx *ctx = desc->arg.data;
+	struct svc_rqst *rqstp = ctx->rqstp;
+	struct svc_sock *svsk =
+		container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
+	size_t reclen, received, want, take, n;
+	size_t consumed = 0;
+
+	if (!desc->count)
+		return 0;
+
+	len = min(len, desc->count);
+
+	if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) {
+		want = sizeof(rpc_fraghdr) - svsk->sk_tcplen;
+		n = min(want, len);
+
+		if (skb_copy_bits(skb, offset,
+				  (char *)&svsk->sk_marker + svsk->sk_tcplen,
+				  n))
+			goto fault;
+		svsk->sk_tcplen += n;
+		offset += n;
+		len -= n;
+		consumed += n;
+		desc->count -= n;
+
+		if (svsk->sk_tcplen < sizeof(rpc_fraghdr))
+			return consumed;
+
+		trace_svcsock_marker(&svsk->sk_xprt, svsk->sk_marker);
+		if (svc_sock_reclen(svsk) + svsk->sk_datalen >
+		    svsk->sk_xprt.xpt_server->sv_max_mesg) {
+			net_notice_ratelimited("svc: %s oversized RPC fragment (%u octets) from %pISpc\n",
+					       svsk->sk_xprt.xpt_server->sv_name,
+					       svc_sock_reclen(svsk),
+					       (struct sockaddr *)&svsk->sk_xprt.xpt_remote);
+			desc->error = -EMSGSIZE;
+			desc->count = 0;
+			return consumed;
+		}
+	}
+
+	reclen = svc_sock_reclen(svsk);
+	received = svsk->sk_tcplen - sizeof(rpc_fraghdr);
+	want = reclen - received;
+	take = min(want, len);
+
+	if (take) {
+		struct iov_iter iter;
+
+		svc_tcp_recv_iter_init(rqstp, &iter, svsk->sk_datalen, take);
+		if (skb_copy_datagram_iter(skb, offset, &iter, take))
+			goto fault;
+		svsk->sk_datalen += take;
+		svsk->sk_tcplen += take;
+		consumed += take;
+		desc->count -= take;
+	}
+
+	if (take == want) {
+		if (svc_sock_final_rec(svsk)) {
+			ctx->complete = true;
+			desc->count = 0;
+		} else {
+			svc_tcp_fragment_received(svsk);
+			if (++ctx->frags >= SVC_TCP_MAX_FRAGS)
+				desc->count = 0;
+		}
+	}
+
+	return consumed;
+
+fault:
+	desc->error = -EFAULT;
+	desc->count = 0;
+	return consumed;
+}
+
+static bool svc_tcp_at_urg_mark(struct sock *sk)
+{
+	const struct tcp_sock *tp = tcp_sk(sk);
+
+	return tp->urg_data && tp->urg_seq == tp->copied_seq;
+}
+
 /**
  * svc_tcp_recvfrom - Receive data from a TCP socket
  * @rqstp: request structure into which to receive an RPC Call
  *
  * Called in a loop when XPT_DATA has been set.
  *
- * Read the 4-byte stream record marker, then use the record length
- * in that marker to set up exactly the resources needed to receive
- * the next RPC message into @rqstp.
+ * Context: Process context. Takes and releases the socket lock.
  *
  * Returns:
  *   On success, the number of bytes in a received RPC Call, or
@@ -1157,29 +1218,69 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp)
 	struct svc_sock	*svsk =
 		container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt);
 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;
-	size_t want, base;
+	struct svc_tcp_recv_ctx ctx = {
+		.rqstp		= rqstp,
+	};
+	read_descriptor_t desc = {
+		.arg.data	= &ctx,
+		.count		= serv->sv_max_mesg + sizeof(rpc_fraghdr),
+	};
+	struct socket *sock = svsk->sk_sock;
 	ssize_t len;
 	__be32 *p;
 	__be32 calldir;
 
 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
-	len = svc_tcp_read_marker(svsk, rqstp);
-	if (len < 0)
-		goto error;
+	svc_tcp_restore_pages(svsk, rqstp);
 
-	base = svc_tcp_restore_pages(svsk, rqstp);
-	want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr));
-	len = svc_tcp_read_msg(rqstp, base + want, base);
-	if (len >= 0) {
-		trace_svcsock_tcp_recv(&svsk->sk_xprt, len);
-		svsk->sk_tcplen += len;
-		svsk->sk_datalen += len;
+	lock_sock(sock->sk);
+	len = sock->ops->read_sock(sock->sk, &desc, svc_tcp_recv_actor);
+	/* ->read_sock stops at urgent data and consumes none of it.
+	 * Only recvmsg() clears the condition, and this path calls
+	 * none, so every later read stops at the same octet. An RPC
+	 * stream carries no urgent data, so close the connection.
+	 *
+	 * The read that first reaches the mark consumes the octets
+	 * ahead of it, so the stop does not show up as a zero len. A
+	 * record completed ahead of the mark is returned first. The
+	 * XPT_DATA set below brings the next call back here with
+	 * nothing left to consume.
+	 */
+	if (!ctx.complete && svc_tcp_at_urg_mark(sock->sk))
+		desc.error = -EPROTO;
+	release_sock(sock->sk);
+
+	/* ->read_sock returns the octets consumed before an actor
+	 * failure, so a positive len can accompany desc.error.
+	 */
+	if (desc.error < 0) {
+		len = desc.error;
+		goto err_nuts;
 	}
-	if (len != want || !svc_sock_final_rec(svsk))
+	if (len >= 0)
+		trace_svcsock_tcp_recv(&svsk->sk_xprt, len);
+
+	if (!ctx.complete) {
+		if (!desc.count) {
+			set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
+			goto err_incomplete;
+		}
+		/* A zero return leaves no record at the head to classify.
+		 * -EINVAL means a control record sits there. Screen the
+		 * other errors out first, because a probe calls
+		 * sock_error(), whose xchg clears sk->sk_err as it reads.
+		 */
+		if (len <= 0 && len != -EINVAL)
+			goto err_incomplete;
+
+		len = svc_tcp_recv_ctrl_record(svsk);
 		goto err_incomplete;
+	}
 	if (svsk->sk_datalen < 8)
 		goto err_nuts;
 
+	svc_tcp_flush_pages(svsk, rqstp);
+
 	rqstp->rq_arg.len = svsk->sk_datalen;
 	rqstp->rq_arg.page_base = 0;
 	if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
@@ -1195,6 +1296,12 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp)
 	else
 		clear_bit(RQ_LOCAL, &rqstp->rq_flags);
 
+	/* Completing one message stops ->read_sock with whatever
+	 * follows still queued, and no path from here re-arms XPT_DATA.
+	 * The queued message would wait for unrelated traffic.
+	 */
+	set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);
+
 	p = (__be32 *)rqstp->rq_arg.head[0].iov_base;
 	calldir = p[1];
 	if (calldir)
@@ -1219,19 +1326,21 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp)
 	svc_tcp_save_pages(svsk, rqstp);
 	if (len < 0 && len != -EAGAIN)
 		goto err_delete;
-	if (len == want)
-		svc_tcp_fragment_received(svsk);
-	else
+	if (svsk->sk_tcplen >= sizeof(rpc_fraghdr))
 		trace_svcsock_tcp_recv_short(&svsk->sk_xprt,
 				svc_sock_reclen(svsk),
 				svsk->sk_tcplen - sizeof(rpc_fraghdr));
+	else
+		trace_svcsock_tcp_recv_eagain(&svsk->sk_xprt, 0);
 	goto err_noclose;
 error:
-	if (len != -EAGAIN)
-		goto err_delete;
 	trace_svcsock_tcp_recv_eagain(&svsk->sk_xprt, 0);
 	goto err_noclose;
 err_nuts:
+	/* svc_tcp_save_pages() has not run, so svsk->sk_pages[] is
+	 * empty. A non-zero sk_datalen makes the teardown-time
+	 * svc_tcp_clear_pages() walk empty slots and WARN.
+	 */
 	svsk->sk_datalen = 0;
 err_delete:
 	trace_svcsock_tcp_recv_err(&svsk->sk_xprt, len);
@@ -1541,6 +1650,9 @@ int svc_addsock(struct svc_serv *serv, struct net *net, const int fd,
 	err = -EISCONN;
 	if (so->state > SS_UNCONNECTED)
 		goto out;
+	err = -EBUSY;
+	if (so->sk->sk_user_data)
+		goto out;
 	err = -ENOENT;
 	if (!try_module_get(THIS_MODULE))
 		goto out;
diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c
index c42fd338c6076c..2454bedb290244 100644
--- a/net/sunrpc/xdr.c
+++ b/net/sunrpc/xdr.c
@@ -12,7 +12,6 @@
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
-#include <linux/pagemap.h>
 #include <linux/errno.h>
 #include <linux/sunrpc/xdr.h>
 #include <linux/sunrpc/msg_prot.h>
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index fdfed1be97da9f..d029bcb7a5c0a0 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -925,8 +925,9 @@ static noinline void svc_rdma_read_complete(struct svc_rqst *rqstp,
  *	%-ENOTCONN if posting failed (connection is lost),
  *	%-EIO if rdma_rw initialization failed (DMA mapping, etc).
  *
- * Called in a loop when XPT_DATA is set. XPT_DATA is cleared only
- * when there are no remaining ctxt's to process.
+ * Called in a loop when XPT_DATA is set. XPT_DATA is cleared as
+ * soon as both receive queues are empty, so a consumed ctxt does
+ * not leave a stale bit behind.
  *
  * The next ctxt is removed from the "receive" lists.
  *
@@ -960,6 +961,15 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
 	ctxt = svc_rdma_next_recv_ctxt(&rdma_xprt->sc_read_complete_q);
 	if (ctxt) {
 		list_del(&ctxt->rc_list);
+		/* Producers add to these queues and set XPT_DATA under
+		 * this lock, so the clear cannot race one. The clear can
+		 * drop the XPT_DATA that svc_rdma_send_ctxt_put() sets
+		 * for sc_send_release_list. svc_xprt_release() drains
+		 * that list before this thread looks for more work.
+		 */
+		if (list_empty(&rdma_xprt->sc_read_complete_q) &&
+		    list_empty(&rdma_xprt->sc_rq_dto_q))
+			clear_bit(XPT_DATA, &xprt->xpt_flags);
 		spin_unlock(&rdma_xprt->sc_rq_dto_lock);
 		svc_xprt_received(xprt);
 		svc_rdma_read_complete(rqstp, ctxt);
@@ -968,8 +978,8 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp)
 	ctxt = svc_rdma_next_recv_ctxt(&rdma_xprt->sc_rq_dto_q);
 	if (ctxt)
 		list_del(&ctxt->rc_list);
-	else
-		/* No new incoming requests, terminate the loop */
+	/* sc_read_complete_q was empty above, under this same lock. */
+	if (list_empty(&rdma_xprt->sc_rq_dto_q))
 		clear_bit(XPT_DATA, &xprt->xpt_flags);
 	spin_unlock(&rdma_xprt->sc_rq_dto_lock);
 
diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
index 927269598ac24b..f949601b214471 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
@@ -471,7 +471,6 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 	newxprt->sc_max_requests = svcrdma_max_requests;
 	newxprt->sc_max_bc_requests = svcrdma_max_bc_requests;
 	newxprt->sc_recv_batch = RPCRDMA_MAX_RECV_BATCH;
-	newxprt->sc_fc_credits = cpu_to_be32(newxprt->sc_max_requests);
 
 	/* Qualify the transport's resource defaults with the
 	 * capabilities of this particular device.
@@ -492,6 +491,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 		newxprt->sc_max_bc_requests = 2;
 	}
 
+	newxprt->sc_fc_credits = cpu_to_be32(newxprt->sc_max_requests);
+
 	/* Estimate the needed number of rdma_rw contexts. The maximum
 	 * Read and Write chunks have one segment each. Each request
 	 * can involve one Read chunk and either a Write chunk or Reply
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 7f60723fa64d84..385fbddfc602e5 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -24,7 +24,6 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/capability.h>
-#include <linux/pagemap.h>
 #include <linux/errno.h>
 #include <linux/socket.h>
 #include <linux/in.h>
@@ -356,35 +355,6 @@ xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp)
 	return want;
 }
 
-static int
-xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg,
-		     unsigned int *msg_flags, struct cmsghdr *cmsg, int ret)
-{
-	u8 content_type = tls_get_record_type(sock->sk, cmsg);
-	u8 level, description;
-
-	switch (content_type) {
-	case 0:
-		break;
-	case TLS_RECORD_TYPE_DATA:
-		/* TLS sets EOR at the end of each application data
-		 * record, even though there might be more frames
-		 * waiting to be decrypted.
-		 */
-		*msg_flags &= ~MSG_EOR;
-		break;
-	case TLS_RECORD_TYPE_ALERT:
-		tls_alert_recv(sock->sk, msg, &level, &description);
-		ret = (level == TLS_ALERT_LEVEL_FATAL) ?
-			-EACCES : -EAGAIN;
-		break;
-	default:
-		/* discard this record type */
-		ret = -EAGAIN;
-	}
-	return ret;
-}
-
 static int
 xs_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags, int flags)
 {
@@ -402,16 +372,42 @@ xs_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags, int flags)
 		.msg_control = &u,
 		.msg_controllen = sizeof(u),
 	};
+	u8 level, description;
 	int ret;
 
 	iov_iter_kvec(&msg.msg_iter, ITER_DEST, &alert_kvec, 1,
 		      alert_kvec.iov_len);
 	ret = sock_recvmsg(sock, &msg, flags);
-	if (ret > 0) {
-		if (tls_get_record_type(sock->sk, &u.cmsg) == TLS_RECORD_TYPE_ALERT)
-			iov_iter_revert(&msg.msg_iter, ret);
-		ret = xs_sock_process_cmsg(sock, &msg, msg_flags, &u.cmsg,
-					   -EAGAIN);
+	/* put_cmsg() shrinks msg_controllen, so a short one means
+	 * kTLS filled in u.cmsg.
+	 */
+	if (ret >= 0 && msg.msg_controllen < sizeof(u)) {
+		if (tls_get_record_type(sock->sk, &u.cmsg) !=
+		    TLS_RECORD_TYPE_ALERT)
+			return -EAGAIN;
+		/* RFC 8446 Section 5.1: a record with an Alert type carries
+		 * exactly one message, and an alert is two octets.
+		 * tls_alert_recv() reads both without checking the length.
+		 * alert_kvec caps the count at two, so a longer record
+		 * fills it as well. kTLS sets MSG_EOR only once the
+		 * record has been drained.
+		 */
+		if (ret != sizeof(alert) || !(msg.msg_flags & MSG_EOR))
+			return -EACCES;
+		iov_iter_revert(&msg.msg_iter, ret);
+		tls_alert_recv(sock->sk, &msg, &level, &description);
+		/* RFC 8446 Section 6: every alert but a closure alert is
+		 * an error alert, whatever the legacy AlertLevel octet
+		 * says.
+		 */
+		switch (description) {
+		case TLS_ALERT_DESC_CLOSE_NOTIFY:
+		case TLS_ALERT_DESC_USER_CANCELED:
+			ret = -EAGAIN;
+			break;
+		default:
+			ret = -EACCES;
+		}
 	}
 	return ret;
 }
@@ -425,6 +421,10 @@ xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek)
 	ret = sock_recvmsg(sock, msg, flags);
 	/* Handle TLS inband control message lazily */
 	if (msg->msg_flags & MSG_CTRUNC) {
+		/* TLS sets EOR at the end of each application data
+		 * record, even though there might be more frames
+		 * waiting to be decrypted.
+		 */
 		msg->msg_flags &= ~(MSG_CTRUNC | MSG_EOR);
 		if (ret == 0 || ret == -EIO)
 			ret = xs_sock_recv_cmsg(sock, &msg->msg_flags, flags);
diff --git a/net/tipc/bearer.c b/net/tipc/bearer.c
index 05dcd2f9e887a6..65fa18928fe376 100644
--- a/net/tipc/bearer.c
+++ b/net/tipc/bearer.c
@@ -195,9 +195,10 @@ struct tipc_bearer *tipc_bearer_find(struct net *net, const char *name)
 /*     tipc_bearer_get_name - get the bearer name from its id.
  *     @net: network namespace
  *     @name: a pointer to the buffer where the name will be stored.
+ *     @len: size of the destination buffer
  *     @bearer_id: the id to get the name from.
  */
-int tipc_bearer_get_name(struct net *net, char *name, u32 bearer_id)
+int tipc_bearer_get_name(struct net *net, char *name, size_t len, u32 bearer_id)
 {
 	struct tipc_net *tn = tipc_net(net);
 	struct tipc_bearer *b;
@@ -209,7 +210,8 @@ int tipc_bearer_get_name(struct net *net, char *name, u32 bearer_id)
 	if (!b)
 		return -EINVAL;
 
-	strcpy(name, b->name);
+	if (strscpy(name, b->name, len) < 0)
+		return -E2BIG;
 	return 0;
 }
 
diff --git a/net/tipc/bearer.h b/net/tipc/bearer.h
index 41eac1ee0c0912..9ccc9ffa925a27 100644
--- a/net/tipc/bearer.h
+++ b/net/tipc/bearer.h
@@ -226,7 +226,8 @@ int tipc_l2_send_msg(struct net *net, struct sk_buff *buf,
 void tipc_bearer_add_dest(struct net *net, u32 bearer_id, u32 dest);
 void tipc_bearer_remove_dest(struct net *net, u32 bearer_id, u32 dest);
 struct tipc_bearer *tipc_bearer_find(struct net *net, const char *name);
-int tipc_bearer_get_name(struct net *net, char *name, u32 bearer_id);
+int tipc_bearer_get_name(struct net *net, char *name,
+			 size_t len, u32 bearer_id);
 struct tipc_media *tipc_media_find(const char *name);
 int tipc_bearer_setup(void);
 void tipc_bearer_cleanup(void);
diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c
index 16f1ed1f6b1b51..4409bdb70fe981 100644
--- a/net/tipc/crypto.c
+++ b/net/tipc/crypto.c
@@ -144,7 +144,7 @@ struct tipc_tfm {
  * @rcu: struct rcu_head
  * @key: the aead key
  * @gen: the key's generation
- * @seqno: the key seqno (cluster scope)
+ * @seqno: the per-key TX nonce counter
  * @refcnt: the key reference counter
  */
 struct tipc_aead {
@@ -190,7 +190,6 @@ struct tipc_crypto_stats {
  * @rekeying_intv: rekeying interval (in minutes)
  * @stats: the crypto statistics
  * @name: the crypto name
- * @sndnxt: the per-peer sndnxt (TX)
  * @timer1: general timer 1 (jiffies)
  * @timer2: general timer 2 (jiffies)
  * @working: the crypto is working or not
@@ -219,7 +218,6 @@ struct tipc_crypto {
 	struct tipc_crypto_stats __percpu *stats;
 	char name[48];
 
-	atomic64_t sndnxt ____cacheline_aligned;
 	unsigned long timer1;
 	unsigned long timer2;
 	union {
@@ -1051,14 +1049,11 @@ static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
 	WARN_ON(skb_headroom(skb) < ehsz);
 	ehdr = (struct tipc_ehdr *)skb_push(skb, ehsz);
 
-	/* Obtain a seqno first:
-	 * Use the key seqno (= cluster wise) if dest is unknown or we're in
-	 * cluster key mode, otherwise it's better for a per-peer seqno!
+	/*
+	 * Keep the nonce unique for the lifetime of the TX key,
+	 * including key state changes and peer reconnection.
 	 */
-	if (!__rx || aead->mode == CLUSTER_KEY)
-		seqno = atomic64_inc_return(&aead->seqno);
-	else
-		seqno = atomic64_inc_return(&__rx->sndnxt);
+	seqno = atomic64_inc_return(&aead->seqno);
 
 	/* Revoke the key if seqno is wrapped around */
 	if (unlikely(!seqno))
@@ -1237,7 +1232,6 @@ void tipc_crypto_key_flush(struct tipc_crypto *c)
 	tipc_crypto_key_set_state(c, 0, 0, 0);
 	for (k = KEY_MIN; k <= KEY_MAX; k++)
 		tipc_crypto_key_detach(c->aead[k], &c->lock);
-	atomic64_set(&c->sndnxt, 0);
 	spin_unlock_bh(&c->lock);
 }
 
@@ -1384,8 +1378,6 @@ static struct tipc_aead *tipc_crypto_key_pick_tx(struct tipc_crypto *tx,
  * It also considers if peer has no key, then we need to make own master key
  * (if any) taking over i.e. starting grace period and also trigger key
  * distributing process.
- *
- * The "per-peer" sndnxt is also reset when the peer key has switched.
  */
 static void tipc_crypto_key_synch(struct tipc_crypto *rx, struct sk_buff *skb)
 {
@@ -1436,7 +1428,6 @@ static void tipc_crypto_key_synch(struct tipc_crypto *rx, struct sk_buff *skb)
 		if (cur)
 			tipc_aead_users_dec(tx->aead[cur], 0);
 
-		atomic64_set(&rx->sndnxt, 0);
 		/* Mark the point TX key users changed */
 		tx->timer1 = jiffies;
 
@@ -1501,7 +1492,6 @@ int tipc_crypto_start(struct tipc_crypto **crypto, struct net *net,
 	tipc_crypto_key_set_state(c, 0, 0, 0);
 	atomic_set(&c->key_distr, 0);
 	atomic_set(&c->peer_rx_active, 0);
-	atomic64_set(&c->sndnxt, 0);
 	c->timer1 = jiffies;
 	c->timer2 = jiffies;
 	c->rekeying_intv = TIPC_REKEYING_INTV_DEF;
diff --git a/net/tipc/group.c b/net/tipc/group.c
index 14e6732624e28e..74f6d3dac0784d 100644
--- a/net/tipc/group.c
+++ b/net/tipc/group.c
@@ -797,10 +797,10 @@ void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup,
 		tipc_group_open(m, usr_wakeup);
 		return;
 	case GRP_ACK_MSG:
-		if (!m)
+		if (!m || !grp->bc_ackers)
 			return;
 		acked = msg_grp_bc_acked(hdr);
-		if (less_eq(acked, m->bc_acked))
+		if (acked != grp->bc_snd_nxt || m->bc_acked == acked)
 			return;
 		m->bc_acked = acked;
 		if (--grp->bc_ackers)
diff --git a/net/tipc/monitor.c b/net/tipc/monitor.c
index a94b9b36a70082..0164a034b57e9b 100644
--- a/net/tipc/monitor.c
+++ b/net/tipc/monitor.c
@@ -632,9 +632,10 @@ static void mon_timeout(struct timer_list *t)
 {
 	struct tipc_monitor *mon = timer_container_of(mon, t, timer);
 	struct tipc_peer *self;
-	int best_member_cnt = dom_size(mon->peer_cnt) - 1;
+	int best_member_cnt;
 
 	write_lock_bh(&mon->lock);
+	best_member_cnt = dom_size(mon->peer_cnt) - 1;
 	self = mon->self;
 	if (self && (best_member_cnt != self->applied)) {
 		mon_update_local_domain(mon);
@@ -833,7 +834,8 @@ int __tipc_nl_add_monitor(struct net *net, struct tipc_nl_msg *msg,
 	void *hdr;
 	int ret;
 
-	ret = tipc_bearer_get_name(net, bearer_name, bearer_id);
+	ret = tipc_bearer_get_name(net, bearer_name, sizeof(bearer_name),
+				   bearer_id);
 	if (ret || !mon)
 		return 0;
 
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index fbb274287aa5f2..8bdb78718a8d44 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -185,8 +185,6 @@ int tls_push_sg(struct sock *sk,
 
 	ctx->splicing_pages = true;
 	while (1) {
-		/* is sending application-limited? */
-		tcp_rate_check_app_limited(sk);
 		p = sg_page(sg);
 retry:
 		bvec_set_page(&bvec, p, size, offset);
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index 42cffeafc8c144..21a8c7247acf55 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -1361,7 +1361,7 @@ static int unix_bind_bsd(struct sock *sk, struct sockaddr_un *sunaddr,
 	struct unix_sock *u = unix_sk(sk);
 	unsigned int new_hash, old_hash;
 	struct net *net = sock_net(sk);
-	struct mnt_idmap *idmap;
+	const struct mnt_idmap *idmap;
 	struct unix_address *addr;
 	struct dentry *dentry;
 	struct path parent;
@@ -3566,8 +3566,7 @@ static int unix_seq_show(struct seq_file *seq, void *v)
 		struct unix_sock *u = unix_sk(s);
 		unix_state_lock(s);
 
-		seq_printf(seq, "%pK: %08X %08X %08X %04X %02X %5llu",
-			s,
+		seq_printf(seq, "%08u: %08X %08X %08X %04X %02X %5llu", 0,
 			refcount_read(&s->sk_refcnt),
 			0,
 			s->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0,
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index f840498b58afbc..44cee74519555e 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -2543,6 +2543,35 @@ static int __vsock_seqpacket_recvmsg(struct sock *sk, struct msghdr *msg,
 	return err;
 }
 
+/* Bytes a following receive can consume, 1 if it would only see EOF, or -1
+ * if the transport cannot tell.
+ *
+ * Called under the socket lock after a nonnegative stream receive, so a NULL
+ * transport implies SOCK_DONE.
+ */
+static int vsock_stream_inq_hint(struct sock *sk)
+{
+	struct vsock_sock *vsk = vsock_sk(sk);
+	s64 data;
+
+	if ((sk->sk_shutdown & RCV_SHUTDOWN) || !vsk->transport ||
+	    (sock_flag(sk, SOCK_DONE) && sk->sk_state != TCP_ESTABLISHED))
+		return 1;
+
+	data = vsock_stream_has_data(vsk);
+	if (data < 0)
+		return -1;
+	if (data > 0)
+		return min_t(s64, data, INT_MAX);
+
+	/* Empty but finished: keep the caller reading so it sees EOF. */
+	if (sock_flag(sk, SOCK_DONE) ||
+	    (READ_ONCE(vsk->peer_shutdown) & SEND_SHUTDOWN))
+		return 1;
+
+	return 0;
+}
+
 int
 __vsock_connectible_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
 			    int flags)
@@ -2606,6 +2635,12 @@ __vsock_connectible_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
 		err = __vsock_seqpacket_recvmsg(sk, msg, len, flags);
 
 out:
+	/* Seqpacket has_data counts messages, while io_uring treats msg_inq as
+	 * a byte length when sizing retries, so only streams report a hint.
+	 */
+	if (msg->msg_get_inq && err >= 0 && sk->sk_type == SOCK_STREAM)
+		msg->msg_inq = vsock_stream_inq_hint(sk);
+
 	release_sock(sk);
 	return err;
 }
@@ -2889,6 +2924,9 @@ static int vsock_net_child_mode_string(const struct ctl_table *table, int write,
 
 	net = container_of(table->data, struct net, vsock.child_ns_mode);
 
+	if (!*lenp)
+		return 0;
+
 	ret = __vsock_net_mode_string(table, write, buffer, lenp, ppos,
 				      vsock_net_child_mode(net), &new_mode);
 	if (ret)
diff --git a/net/vmw_vsock/af_vsock_tap.c b/net/vmw_vsock/af_vsock_tap.c
index 30ee7e4fe25115..7c5cdceabb2a5f 100644
--- a/net/vmw_vsock/af_vsock_tap.c
+++ b/net/vmw_vsock/af_vsock_tap.c
@@ -64,14 +64,10 @@ static int __vsock_deliver_tap_skb(struct sk_buff *skb,
 	struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
 
 	if (nskb) {
-		dev_hold(dev);
-
 		nskb->dev = dev;
 		ret = dev_queue_xmit(nskb);
 		if (unlikely(ret > 0))
 			ret = net_xmit_errno(ret);
-
-		dev_put(dev);
 	}
 
 	return ret;
diff --git a/net/wireless/ap.c b/net/wireless/ap.c
index 9a9a870806f500..fa5441987a903b 100644
--- a/net/wireless/ap.c
+++ b/net/wireless/ap.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Parts of this file are
- * Copyright (C) 2022-2023 Intel Corporation
+ * Copyright (C) 2022-2023, 2026 Intel Corporation
  */
 #include <linux/ieee80211.h>
 #include <linux/export.h>
@@ -33,9 +33,8 @@ static int ___cfg80211_stop_ap(struct cfg80211_registered_device *rdev,
 	err = rdev_stop_ap(rdev, dev, link_id);
 	if (!err) {
 		wdev->conn_owner_nlportid = 0;
-		wdev->links[link_id].ap.beacon_interval = 0;
-		memset(&wdev->links[link_id].ap.chandef, 0,
-		       sizeof(wdev->links[link_id].ap.chandef));
+		memset(&wdev->links[link_id].ap, 0,
+		       sizeof(wdev->links[link_id].ap));
 		wdev->u.ap.ssid_len = 0;
 		rdev_set_qos_map(rdev, dev, NULL);
 		if (notify)
diff --git a/net/wireless/chan.c b/net/wireless/chan.c
index 7f6af1790736cd..c743b6fb7e309e 100644
--- a/net/wireless/chan.c
+++ b/net/wireless/chan.c
@@ -568,8 +568,8 @@ int cfg80211_chandef_add_npca(struct wiphy *wiphy,
 			      const struct ieee80211_uhr_npca_info *npca)
 {
 	struct cfg80211_chan_def new_chandef = *chandef;
-	u32 width, npca_freq;
-	u8 offs;
+	u32 npca_freq;
+	u8 chan;
 
 	if (chandef->npca_chan || chandef->npca_punctured)
 		return -EINVAL;
@@ -589,11 +589,10 @@ int cfg80211_chandef_add_npca(struct wiphy *wiphy,
 		return -EINVAL;
 	}
 
-	offs = le32_get_bits(npca->params,
-			     IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN_OFFS);
+	chan = le32_get_bits(npca->params,
+			     IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN);
 
-	width = cfg80211_chandef_get_width(chandef);
-	npca_freq = chandef->center_freq1 - width / 2 + 10 + 20 * offs;
+	npca_freq = ieee80211_channel_to_frequency(chan, chandef->chan->band);
 	new_chandef.npca_chan = ieee80211_get_channel(wiphy, npca_freq);
 	if (!new_chandef.npca_chan)
 		return -EINVAL;
@@ -1811,13 +1810,44 @@ bool cfg80211_reg_check_beaconing(struct wiphy *wiphy,
 }
 EXPORT_SYMBOL(cfg80211_reg_check_beaconing);
 
+static bool cfg80211_can_set_monitor_channel(struct cfg80211_registered_device *rdev,
+					     struct cfg80211_chan_def *chandef)
+{
+	struct wireless_dev *wdev;
+	int radio_idx;
+
+	lockdep_assert_held(&rdev->wiphy.mtx);
+
+	if (rdev->num_running_monitor_ifaces < 1)
+		return false;
+
+	if (cfg80211_has_monitors_only(rdev))
+		return true;
+
+	radio_idx = cfg80211_get_radio_idx_by_chan(&rdev->wiphy, chandef->chan);
+	/* No defined radio covers this channel. Fall back on global behavior */
+	if (radio_idx < 0)
+		return false;
+
+	list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
+		if (wdev->iftype == NL80211_IFTYPE_MONITOR)
+			continue;
+		if (!wdev->netdev)
+			continue;
+		if (rdev_get_radio_mask(rdev, wdev->netdev) & BIT(radio_idx))
+			return false;
+	}
+
+	return true;
+}
+
 int cfg80211_set_monitor_channel(struct cfg80211_registered_device *rdev,
 				 struct net_device *dev,
 				 struct cfg80211_chan_def *chandef)
 {
 	if (!rdev->ops->set_monitor_channel)
 		return -EOPNOTSUPP;
-	if (!cfg80211_has_monitors_only(rdev))
+	if (!cfg80211_can_set_monitor_channel(rdev, chandef))
 		return -EBUSY;
 
 	return rdev_set_monitor_channel(rdev, dev, chandef);
diff --git a/net/wireless/core.c b/net/wireless/core.c
index cde3ca85494de4..dc1e0522340a25 100644
--- a/net/wireless/core.c
+++ b/net/wireless/core.c
@@ -312,6 +312,8 @@ int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev,
 				    struct wireless_dev *wdev,
 				    struct cfg80211_nan_local_sched *sched)
 {
+	struct cfg80211_chan_def *chandefs = NULL, *old_chandefs;
+	u8 old_n_channels;
 	int ret;
 
 	lockdep_assert_held(&rdev->wiphy.mtx);
@@ -322,30 +324,79 @@ int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev,
 	if (wdev->u.nan.sched_update_pending)
 		return -EBUSY;
 
+	/*
+	 * Pre-allocate chandefs, so we don't ruin the current
+	 * schedule on allocation failure.
+	 */
+	if (sched->n_channels) {
+		chandefs = kcalloc(sched->n_channels, sizeof(*chandefs),
+				   GFP_KERNEL);
+		if (!chandefs)
+			return -ENOMEM;
+
+		for (int i = 0; i < sched->n_channels; i++)
+			chandefs[i] = sched->nan_channels[i].chandef;
+	}
+
+	/*
+	 * Swap in the new chandefs before calling the driver and set the
+	 * deferred flag, so if the driver completes the update synchronously
+	 * the new schedule will be already in place.
+	 */
+	old_chandefs = wdev->u.nan.chandefs;
+	old_n_channels = wdev->u.nan.n_channels;
+	wdev->u.nan.chandefs = chandefs;
+	wdev->u.nan.n_channels = sched->n_channels;
+	wdev->u.nan.sched_update_pending = sched->deferred;
+
 	ret = rdev_nan_set_local_sched(rdev, wdev, sched);
-	if (ret)
+	if (ret) {
+		/* Restore the previous schedule on failure */
+		wdev->u.nan.sched_update_pending = false;
+		wdev->u.nan.chandefs = old_chandefs;
+		wdev->u.nan.n_channels = old_n_channels;
+		kfree(chandefs);
 		return ret;
+	}
 
-	wdev->u.nan.sched_update_pending = sched->deferred;
+	kfree(old_chandefs);
 
-	kfree(wdev->u.nan.chandefs);
-	wdev->u.nan.chandefs = NULL;
-	wdev->u.nan.n_channels = 0;
+	return 0;
+}
 
-	if (!sched->n_channels)
-		return 0;
+int cfg80211_nan_set_non_evac_channels(struct cfg80211_registered_device *rdev,
+				       struct wireless_dev *wdev,
+				       struct cfg80211_nan_non_evac_channels *channels)
+{
+	lockdep_assert_held(&rdev->wiphy.mtx);
+
+	if (wdev->iftype != NL80211_IFTYPE_NAN || !wdev_running(wdev))
+		return -EINVAL;
+
+	/*
+	 * Don't allow updating the non-evacuable channels while a deferred
+	 * schedule update is pending, as the set of channels may still change.
+	 */
+	if (wdev->u.nan.sched_update_pending)
+		return -EBUSY;
 
-	wdev->u.nan.chandefs = kzalloc_objs(*wdev->u.nan.chandefs,
-					    sched->n_channels);
-	if (!wdev->u.nan.chandefs)
-		return -ENOMEM;
+	/* All provided channels must belong to the current local schedule. */
+	for (int i = 0; i < channels->n_channels; i++) {
+		bool found = false;
 
-	for (int i = 0; i < sched->n_channels; i++)
-		wdev->u.nan.chandefs[i] = sched->nan_channels[i].chandef;
+		for (int j = 0; j < wdev->u.nan.n_channels; j++) {
+			if (cfg80211_chandef_identical(&wdev->u.nan.chandefs[j],
+						       &channels->chandefs[i])) {
+				found = true;
+				break;
+			}
+		}
 
-	wdev->u.nan.n_channels = sched->n_channels;
+		if (!found)
+			return -ENOENT;
+	}
 
-	return 0;
+	return rdev_nan_set_non_evac_channels(rdev, wdev, channels);
 }
 
 void cfg80211_stop_pd(struct cfg80211_registered_device *rdev,
diff --git a/net/wireless/core.h b/net/wireless/core.h
index 6138d207caf41a..58445c09026a26 100644
--- a/net/wireless/core.h
+++ b/net/wireless/core.h
@@ -453,13 +453,16 @@ void cfg80211_sme_abandon_assoc(struct wireless_dev *wdev);
 
 /* internal helpers */
 bool cfg80211_supported_cipher_suite(struct wiphy *wiphy, u32 cipher);
+bool cfg80211_cigtk_supported(struct wireless_dev *wdev,
+			      struct genl_info *info);
 bool cfg80211_valid_key_idx(struct wireless_dev *wdev,
-			    int key_idx, bool pairwise,
+			    int key_idx, enum nl80211_key_type type,
 			    const u8 *mac_addr);
 int cfg80211_validate_key_settings(struct cfg80211_registered_device *rdev,
 				   struct wireless_dev *wdev,
 				   struct key_params *params, int key_idx,
-				   bool pairwise, const u8 *mac_addr);
+				   enum nl80211_key_type type,
+				   const u8 *mac_addr);
 void __cfg80211_scan_done(struct wiphy *wiphy, struct wiphy_work *wk);
 void ___cfg80211_scan_done(struct cfg80211_registered_device *rdev,
 			   bool send_message);
@@ -575,6 +578,10 @@ int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev,
 				    struct wireless_dev *wdev,
 				    struct cfg80211_nan_local_sched *sched);
 
+int cfg80211_nan_set_non_evac_channels(struct cfg80211_registered_device *rdev,
+				       struct wireless_dev *wdev,
+				       struct cfg80211_nan_non_evac_channels *channels);
+
 struct cfg80211_internal_bss *
 cfg80211_bss_update(struct cfg80211_registered_device *rdev,
 		    struct cfg80211_internal_bss *tmp,
diff --git a/net/wireless/ibss.c b/net/wireless/ibss.c
index 7f6779d326b813..81dff0bace2866 100644
--- a/net/wireless/ibss.c
+++ b/net/wireless/ibss.c
@@ -176,7 +176,8 @@ void cfg80211_clear_ibss(struct net_device *dev, bool nowext)
 	 */
 	if (rdev->ops->del_key)
 		for (i = 0; i < 6; i++)
-			rdev_del_key(rdev, wdev, -1, i, false, NULL);
+			rdev_del_key(rdev, wdev, -1, i, NL80211_KEYTYPE_GROUP,
+				     NULL);
 
 	if (wdev->u.ibss.current_bss) {
 		cfg80211_unhold_bss(wdev->u.ibss.current_bss);
diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c
index 9fafd7673d853b..a182404557e006 100644
--- a/net/wireless/nl80211.c
+++ b/net/wireless/nl80211.c
@@ -569,6 +569,7 @@ static const struct nla_policy nl80211_txattr_policy[NL80211_TXRATE_MAX + 1] = {
 	[NL80211_TXRATE_EHT_LTF] = NLA_POLICY_RANGE(NLA_U8,
 						   NL80211_RATE_INFO_EHT_1XLTF,
 						   NL80211_RATE_INFO_EHT_8XLTF),
+	[NL80211_TXRATE_6GHZ_NON_HT_DUP] = { .type = NLA_FLAG },
 
 };
 
@@ -876,7 +877,7 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = {
 	[NL80211_ATTR_HT_CAPABILITY_MASK] = {
 		.len = NL80211_HT_CAPABILITY_LEN
 	},
-	[NL80211_ATTR_NOACK_MAP] = { .type = NLA_U16 },
+	[NL80211_ATTR_TID_BITMAP] = { .type = NLA_U16 },
 	[NL80211_ATTR_INACTIVITY_TIMEOUT] = { .type = NLA_U16 },
 	[NL80211_ATTR_BG_SCAN_PERIOD] = { .type = NLA_U16 },
 	[NL80211_ATTR_WDEV] = { .type = NLA_U64 },
@@ -1096,6 +1097,9 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = {
 	[NL80211_ATTR_NPCA_PUNCT_BITMAP] =
 		NLA_POLICY_FULL_RANGE(NLA_U32, &nl80211_punct_bitmap_range),
 	[NL80211_ATTR_STA_DUMP_LINK_STATS] = { .type = NLA_FLAG },
+	[NL80211_ATTR_FRAME_NO_STA] = { .type = NLA_FLAG },
+	[NL80211_ATTR_ASSOC_CIP] = { .type = NLA_FLAG },
+	[NL80211_ATTR_CIP_CAPABILITIES] = { .type = NLA_U8 },
 };
 
 /* policy for the key attributes */
@@ -1761,6 +1765,11 @@ static int nl80211_parse_key(struct genl_info *info, struct key_parse *k)
 				GENL_SET_ERR_MSG(info, "def key idx not 0-3");
 				return -EINVAL;
 			}
+		} else if (k->type == NL80211_KEYTYPE_CIGTK) {
+			if (k->idx < 0 || k->idx > 1) {
+				GENL_SET_ERR_MSG(info, "CIGTK idx not 0-1");
+				return -EINVAL;
+			}
 		} else {
 			if (k->idx < 0 || k->idx > 7) {
 				GENL_SET_ERR_MSG(info, "key idx not 0-7");
@@ -1825,7 +1834,9 @@ nl80211_parse_connkeys(struct cfg80211_registered_device *rdev,
 		} else if (parse.defmgmt)
 			goto error;
 		err = cfg80211_validate_key_settings(rdev, wdev, &parse.p,
-						     parse.idx, false, NULL);
+						     parse.idx,
+						     NL80211_KEYTYPE_GROUP,
+						     NULL);
 		if (err)
 			goto error;
 		if (parse.p.cipher != WLAN_CIPHER_SUITE_WEP40 &&
@@ -3615,6 +3626,11 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev,
 						NL80211_ATTR_EXT_MLD_CAPA_AND_OPS,
 						capab->ext_mld_capa_and_ops))
 					goto nla_put_failure;
+				if (capab->cip_supported &&
+				    nla_put_u8(msg,
+					       NL80211_ATTR_CIP_CAPABILITIES,
+					       capab->cip_capabilities))
+					goto nla_put_failure;
 
 				nla_nest_end(msg, nested_ext_capab);
 				if (state->split)
@@ -4140,12 +4156,17 @@ static int __nl80211_set_channel(struct cfg80211_registered_device *rdev,
 	switch (iftype) {
 	case NL80211_IFTYPE_AP:
 	case NL80211_IFTYPE_P2P_GO:
-		if (!cfg80211_reg_can_beacon_relax(&rdev->wiphy, &chandef,
-						   iftype))
-			return -EINVAL;
 		if (wdev->links[link_id].ap.beacon_interval) {
 			struct ieee80211_channel *cur_chan;
-
+			struct cfg80211_beaconing_check_config config = {
+				.iftype = iftype,
+				.reg_power = wdev->links[link_id].ap.reg_power,
+				.relax = true,
+			};
+
+			if (!cfg80211_reg_check_beaconing(&rdev->wiphy,
+							  &chandef, &config))
+				return -EINVAL;
 			if (!dev || !rdev->ops->set_ap_chanwidth ||
 			    !(rdev->wiphy.features &
 			      NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE))
@@ -4563,12 +4584,13 @@ static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info)
 			rdev->wiphy.retry_long = retry_long;
 		if (changed & WIPHY_PARAM_FRAG_THRESHOLD)
 			rdev->wiphy.frag_threshold = frag_threshold;
-		if ((changed & WIPHY_PARAM_RTS_THRESHOLD) &&
-		    old_radio_rts_threshold) {
+		if (changed & WIPHY_PARAM_RTS_THRESHOLD) {
 			rdev->wiphy.rts_threshold = rts_threshold;
-			for (i = 0 ; i < rdev->wiphy.n_radio; i++)
-				rdev->wiphy.radio_cfg[i].rts_threshold =
-					rdev->wiphy.rts_threshold;
+			if (old_radio_rts_threshold) {
+				for (i = 0; i < rdev->wiphy.n_radio; i++)
+					rdev->wiphy.radio_cfg[i].rts_threshold =
+						rdev->wiphy.rts_threshold;
+			}
 		}
 		if (changed & WIPHY_PARAM_COVERAGE_CLASS)
 			rdev->wiphy.coverage_class = coverage_class;
@@ -5280,13 +5302,13 @@ static int nl80211_set_noack_map(struct sk_buff *skb, struct genl_info *info)
 	struct net_device *dev = info->user_ptr[1];
 	u16 noack_map;
 
-	if (!info->attrs[NL80211_ATTR_NOACK_MAP])
+	if (!info->attrs[NL80211_ATTR_TID_BITMAP])
 		return -EINVAL;
 
 	if (!rdev->ops->set_noack_map)
 		return -EOPNOTSUPP;
 
-	noack_map = nla_get_u16(info->attrs[NL80211_ATTR_NOACK_MAP]);
+	noack_map = nla_get_u16(info->attrs[NL80211_ATTR_TID_BITMAP]);
 
 	return rdev_set_noack_map(rdev, dev, noack_map);
 }
@@ -5371,6 +5393,7 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info)
 	struct wireless_dev *wdev = info->user_ptr[1];
 	u8 key_idx = 0;
 	const u8 *mac_addr = NULL;
+	enum nl80211_key_type type;
 	bool pairwise;
 	struct get_key_cookie cookie = {
 		.error = 0,
@@ -5403,19 +5426,24 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info)
 		mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);
 
 	pairwise = !!mac_addr;
+	type = pairwise ? NL80211_KEYTYPE_PAIRWISE : NL80211_KEYTYPE_GROUP;
 	if (info->attrs[NL80211_ATTR_KEY_TYPE]) {
-		u32 kt = nla_get_u32(info->attrs[NL80211_ATTR_KEY_TYPE]);
+		type = nla_get_u32(info->attrs[NL80211_ATTR_KEY_TYPE]);
 
-		if (kt != NL80211_KEYTYPE_GROUP &&
-		    kt != NL80211_KEYTYPE_PAIRWISE)
+		if (type != NL80211_KEYTYPE_GROUP &&
+		    type != NL80211_KEYTYPE_PAIRWISE &&
+		    type != NL80211_KEYTYPE_CIGTK) {
+			GENL_SET_ERR_MSG(info, "key type not pairwise, group or CIGTK");
 			return -EINVAL;
-		pairwise = kt == NL80211_KEYTYPE_PAIRWISE;
+		}
+
+		pairwise = type == NL80211_KEYTYPE_PAIRWISE;
 	}
 
 	if (!rdev->ops->get_key)
 		return -EOPNOTSUPP;
 
-	if (!cfg80211_valid_key_idx(wdev, key_idx, pairwise, mac_addr))
+	if (!cfg80211_valid_key_idx(wdev, key_idx, type, mac_addr))
 		return -ENOENT;
 
 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
@@ -5444,7 +5472,7 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info)
 	if (err)
 		goto free_msg;
 
-	err = rdev_get_key(rdev, wdev, link_id, key_idx, pairwise, mac_addr,
+	err = rdev_get_key(rdev, wdev, link_id, key_idx, type, mac_addr,
 			   &cookie, get_key_callback);
 
 	if (err)
@@ -5603,8 +5631,9 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info)
 
 	/* for now */
 	if (key.type != NL80211_KEYTYPE_PAIRWISE &&
-	    key.type != NL80211_KEYTYPE_GROUP) {
-		GENL_SET_ERR_MSG(info, "key type not pairwise or group");
+	    key.type != NL80211_KEYTYPE_GROUP &&
+	    key.type != NL80211_KEYTYPE_CIGTK) {
+		GENL_SET_ERR_MSG(info, "key type not pairwise, group or CIGTK");
 		return -EINVAL;
 	}
 
@@ -5616,8 +5645,7 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info)
 		return -EOPNOTSUPP;
 
 	if (cfg80211_validate_key_settings(rdev, wdev, &key.p, key.idx,
-					   key.type == NL80211_KEYTYPE_PAIRWISE,
-					   mac_addr)) {
+					   key.type, mac_addr)) {
 		GENL_SET_ERR_MSG(info, "key setting validation failed");
 		return -EINVAL;
 	}
@@ -5631,8 +5659,7 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info)
 				key.type == NL80211_KEYTYPE_PAIRWISE);
 
 	if (!err) {
-		err = rdev_add_key(rdev, wdev, link_id, key.idx,
-				   key.type == NL80211_KEYTYPE_PAIRWISE,
+		err = rdev_add_key(rdev, wdev, link_id, key.idx, key.type,
 				    mac_addr, &key.p);
 		if (err)
 			GENL_SET_ERR_MSG(info, "key addition failed");
@@ -5666,12 +5693,13 @@ static int nl80211_del_key(struct sk_buff *skb, struct genl_info *info)
 
 	/* for now */
 	if (key.type != NL80211_KEYTYPE_PAIRWISE &&
-	    key.type != NL80211_KEYTYPE_GROUP)
+	    key.type != NL80211_KEYTYPE_GROUP &&
+	    key.type != NL80211_KEYTYPE_CIGTK) {
+		GENL_SET_ERR_MSG(info, "key type not pairwise, group or CIGTK");
 		return -EINVAL;
+	}
 
-	if (!cfg80211_valid_key_idx(wdev, key.idx,
-				    key.type == NL80211_KEYTYPE_PAIRWISE,
-				    mac_addr))
+	if (!cfg80211_valid_key_idx(wdev, key.idx, key.type, mac_addr))
 		return -EINVAL;
 
 	if (!rdev->ops->del_key)
@@ -5685,8 +5713,7 @@ static int nl80211_del_key(struct sk_buff *skb, struct genl_info *info)
 
 	if (!err)
 		err = rdev_del_key(rdev, wdev, link_id, key.idx,
-				   key.type == NL80211_KEYTYPE_PAIRWISE,
-				   mac_addr);
+				   key.type, mac_addr);
 
 #ifdef CONFIG_CFG80211_WEXT
 	if (!err) {
@@ -6303,6 +6330,14 @@ static int nl80211_parse_tx_bitrate_mask(struct genl_info *info,
 			mask->control[band].eht_ltf =
 				nla_get_u8(tb[NL80211_TXRATE_EHT_LTF]);
 
+		if (tb[NL80211_TXRATE_6GHZ_NON_HT_DUP]) {
+			if (band != NL80211_BAND_6GHZ ||
+			    (wdev->iftype != NL80211_IFTYPE_AP &&
+			     wdev->iftype != NL80211_IFTYPE_P2P_GO))
+				return -EINVAL;
+			mask->control[band].nonht_dup_6ghz = true;
+		}
+
 		if (mask->control[band].legacy == 0) {
 			/* don't allow empty legacy rates if HT, VHT, HE or EHT
 			 * are not even supported.
@@ -6344,6 +6379,7 @@ static int validate_beacon_tx_rate(struct cfg80211_registered_device *rdev,
 {
 	u32 count_ht, count_vht, count_he, count_eht, i;
 	u32 rate = beacon_rate->control[band].legacy;
+	bool nonht_dup = beacon_rate->control[band].nonht_dup_6ghz;
 
 	/* Allow only one rate */
 	if (hweight32(rate) > 1)
@@ -6405,6 +6441,9 @@ static int validate_beacon_tx_rate(struct cfg80211_registered_device *rdev,
 	    (!rate && !count_ht && !count_vht && !count_he && !count_eht))
 		return -EINVAL;
 
+	if (nonht_dup && !rate)
+		return -EINVAL;
+
 	if (rate &&
 	    !wiphy_ext_feature_isset(&rdev->wiphy,
 				     NL80211_EXT_FEATURE_BEACON_RATE_LEGACY))
@@ -7364,6 +7403,7 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
 					     params->beacon.tail_len, 0);
 	if (!cfg80211_reg_check_beaconing(&rdev->wiphy, &params->chandef,
 					  &beacon_check)) {
+		GENL_SET_ERR_MSG(info, "Channel rejected by regulatory");
 		err = -EINVAL;
 		goto out;
 	}
@@ -7478,6 +7518,7 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
 	err = rdev_start_ap(rdev, dev, params);
 	if (!err) {
 		wdev->links[link_id].ap.beacon_interval = params->beacon_interval;
+		wdev->links[link_id].ap.reg_power = beacon_check.reg_power;
 		wdev->links[link_id].ap.chandef = params->chandef;
 		wdev->u.ap.ssid_len = params->ssid_len;
 		memcpy(wdev->u.ap.ssid, params->ssid,
@@ -7570,6 +7611,8 @@ static int nl80211_set_beacon(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	err = rdev_change_beacon(rdev, dev, params);
+	if (!err)
+		wdev->links[link_id].ap.reg_power = beacon_check.reg_power;
 
 out:
 	kfree(params->beacon.mbssid_ies);
@@ -8805,7 +8848,7 @@ int cfg80211_check_station_change(struct wiphy *wiphy,
 		return -EINVAL;
 
 	/* When you run into this, adjust the code below for the new flag */
-	BUILD_BUG_ON(NL80211_STA_FLAG_MAX != 8);
+	BUILD_BUG_ON(NL80211_STA_FLAG_MAX != 9);
 
 	switch (statype) {
 	case CFG80211_STA_MESH_PEER_KERNEL:
@@ -8899,7 +8942,8 @@ int cfg80211_check_station_change(struct wiphy *wiphy,
 				  BIT(NL80211_STA_FLAG_SHORT_PREAMBLE) |
 				  BIT(NL80211_STA_FLAG_WME) |
 				  BIT(NL80211_STA_FLAG_MFP) |
-				  BIT(NL80211_STA_FLAG_SPP_AMSDU)))
+				  BIT(NL80211_STA_FLAG_SPP_AMSDU) |
+				  BIT(NL80211_STA_FLAG_CIP)))
 			return -EINVAL;
 
 		/* but authenticated/associated only if driver handles it */
@@ -9284,6 +9328,22 @@ static int nl80211_set_station(struct sk_buff *skb, struct genl_info *info)
 			nla_get_u16(info->attrs[NL80211_ATTR_EML_CAPABILITY]);
 	}
 
+	if (params.sta_flags_mask & BIT(NL80211_STA_FLAG_CIP) &&
+	    params.sta_flags_set & BIT(NL80211_STA_FLAG_CIP)) {
+		if (!cfg80211_cigtk_supported(wdev, info))
+			return -EINVAL;
+
+		/* CIP capabilities are required if CIP is being enabled */
+		if (info->attrs[NL80211_ATTR_CIP_CAPABILITIES]) {
+			params.link_sta_params.cip_cap_set = true;
+			params.link_sta_params.cip_cap =
+				nla_get_u8(info->attrs[NL80211_ATTR_CIP_CAPABILITIES]);
+		} else {
+			GENL_SET_ERR_MSG(info, "No CIP capabilities provided");
+			return -EINVAL;
+		}
+	}
+
 	if (info->attrs[NL80211_ATTR_AIRTIME_WEIGHT])
 		params.airtime_weight =
 			nla_get_u16(info->attrs[NL80211_ATTR_AIRTIME_WEIGHT]);
@@ -9493,6 +9553,22 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info)
 			nla_get_u16(info->attrs[NL80211_ATTR_EML_CAPABILITY]);
 	}
 
+	if (params.sta_flags_mask & BIT(NL80211_STA_FLAG_CIP) &&
+	    params.sta_flags_set & BIT(NL80211_STA_FLAG_CIP)) {
+		if (!cfg80211_cigtk_supported(wdev, info))
+			return -EINVAL;
+
+		/* CIP capabilities are required if CIP is enabled */
+		if (info->attrs[NL80211_ATTR_CIP_CAPABILITIES]) {
+			params.link_sta_params.cip_cap_set = true;
+			params.link_sta_params.cip_cap =
+				nla_get_u8(info->attrs[NL80211_ATTR_CIP_CAPABILITIES]);
+		} else {
+			GENL_SET_ERR_MSG(info, "No CIP capabilities provided");
+			return -EINVAL;
+		}
+	}
+
 	if (info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY])
 		params.link_sta_params.he_6ghz_capa =
 			nla_data(info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY]);
@@ -9570,7 +9646,7 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info)
 		return -EINVAL;
 
 	/* When you run into this, adjust the code below for the new flag */
-	BUILD_BUG_ON(NL80211_STA_FLAG_MAX != 8);
+	BUILD_BUG_ON(NL80211_STA_FLAG_MAX != 9);
 
 	switch (wdev->iftype) {
 	case NL80211_IFTYPE_AP:
@@ -9603,6 +9679,10 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info)
 		    params.sta_flags_mask & BIT(NL80211_STA_FLAG_SPP_AMSDU))
 			return -EINVAL;
 
+		if (params.sta_flags_mask & BIT(NL80211_STA_FLAG_CIP) &&
+		    !cfg80211_cigtk_supported(wdev, info))
+			return -EINVAL;
+
 		/* Older userspace, or userspace wanting to be compatible with
 		 * !NL80211_FEATURE_FULL_AP_CLIENT_STATE, will not set the auth
 		 * and assoc flags in the mask, but assumes the station will be
@@ -12244,6 +12324,7 @@ static int nl80211_parse_counter_offsets(struct cfg80211_registered_device *rdev
 static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 {
 	struct cfg80211_registered_device *rdev = info->user_ptr[0];
+	struct cfg80211_beaconing_check_config beacon_check = {};
 	unsigned int link_id = nl80211_link_id(info->attrs);
 	struct net_device *dev = info->user_ptr[1];
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
@@ -12368,8 +12449,13 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 		goto free;
 
 skip_beacons:
-	if (!cfg80211_reg_can_beacon_relax(&rdev->wiphy, &params.chandef,
-					   wdev->iftype)) {
+	beacon_check.iftype = wdev->iftype;
+	beacon_check.relax = true;
+	beacon_check.reg_power =
+		cfg80211_get_6ghz_power_type(params.beacon_after.tail,
+					     params.beacon_after.tail_len, 0);
+	if (!cfg80211_reg_check_beaconing(&rdev->wiphy, &params.chandef,
+					  &beacon_check)) {
 		err = -EINVAL;
 		goto free;
 	}
@@ -12405,6 +12491,10 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 	params.link_id = link_id;
 	err = rdev_channel_switch(rdev, dev, &params);
 
+	if (!err && (wdev->iftype == NL80211_IFTYPE_AP ||
+		     wdev->iftype == NL80211_IFTYPE_P2P_GO))
+		wdev->links[link_id].ap.csa_reg_power = beacon_check.reg_power;
+
 free:
 	kfree(params.beacon_after.mbssid_ies);
 	kfree(params.beacon_csa.mbssid_ies);
@@ -13325,6 +13415,13 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info)
 		req.flags |= ASSOC_REQ_SPP_AMSDU;
 	}
 
+	if (nla_get_flag(info->attrs[NL80211_ATTR_ASSOC_CIP])) {
+		if (!cfg80211_cigtk_supported(dev->ieee80211_ptr, info))
+			return -EINVAL;
+
+		req.flags |= ASSOC_REQ_CIP;
+	}
+
 	req.link_id = nl80211_link_id_or_invalid(info->attrs);
 
 	if (info->attrs[NL80211_ATTR_MLO_LINKS]) {
@@ -14878,6 +14975,9 @@ static int nl80211_tx_mgmt(struct sk_buff *skb, struct genl_info *info)
 	    !(wdev->valid_links & BIT(params.link_id)))
 		return -EINVAL;
 
+	params.no_sta =
+		nla_get_flag(info->attrs[NL80211_ATTR_FRAME_NO_STA]);
+
 	params.buf = nla_data(info->attrs[NL80211_ATTR_FRAME]);
 	params.len = nla_len(info->attrs[NL80211_ATTR_FRAME]);
 
@@ -17467,8 +17567,7 @@ nl80211_parse_nan_peer_map(struct genl_info *info, struct nlattr *map_attr,
 					  map->schedule, n_channels);
 }
 
-static int nl80211_nan_validate_map_pair(struct wiphy *wiphy,
-					 struct genl_info *info,
+static int nl80211_nan_validate_map_pair(struct genl_info *info,
 					 const struct cfg80211_nan_peer_map *map1,
 					 const struct cfg80211_nan_peer_map *map2,
 					 struct cfg80211_nan_channel *nan_channels)
@@ -17503,24 +17602,6 @@ static int nl80211_nan_validate_map_pair(struct wiphy *wiphy,
 		}
 	}
 
-	/*
-	 * Check for conflicting time slots between maps.
-	 * Only check for single-radio devices (n_radio <= 1) which cannot
-	 * operate on multiple channels simultaneously.
-	 */
-	if (wiphy->n_radio > 1)
-		return 0;
-
-	for (int i = 0; i < ARRAY_SIZE(map1->schedule); i++) {
-		if (map1->schedule[i] != NL80211_NAN_SCHED_NOT_AVAIL_SLOT &&
-		    map2->schedule[i] != NL80211_NAN_SCHED_NOT_AVAIL_SLOT) {
-			NL_SET_ERR_MSG_FMT(info->extack,
-					   "Maps %u and %u both schedule slot %d",
-					   map1->map_id, map2->map_id, i);
-			return -EINVAL;
-		}
-	}
-
 	return 0;
 }
 
@@ -17637,7 +17718,7 @@ static int nl80211_nan_set_peer_sched(struct sk_buff *skb,
 
 			/* Validate against previous maps */
 			for (int j = 0; j < n_maps; j++) {
-				ret = nl80211_nan_validate_map_pair(&rdev->wiphy, info,
+				ret = nl80211_nan_validate_map_pair(info,
 								    &sched.maps[j],
 								    &sched.maps[n_maps],
 								    nan_channels);
@@ -17765,6 +17846,69 @@ static int nl80211_nan_set_local_sched(struct sk_buff *skb,
 	return cfg80211_nan_set_local_schedule(rdev, wdev, sched);
 }
 
+static int
+nl80211_parse_non_evac_channel(struct cfg80211_registered_device *rdev,
+			       struct nlattr *channel, struct genl_info *info,
+			       struct cfg80211_chan_def *chandef)
+{
+	struct nlattr **channel_parsed __free(kfree) =
+		kcalloc(NL80211_ATTR_MAX + 1, sizeof(*channel_parsed),
+			GFP_KERNEL);
+	int ret;
+
+	if (!channel_parsed)
+		return -ENOMEM;
+
+	ret = nla_parse_nested(channel_parsed, NL80211_ATTR_MAX, channel, NULL,
+			       info->extack);
+	if (ret)
+		return ret;
+
+	return nl80211_parse_chandef(rdev, info->extack, channel_parsed,
+				     chandef, false);
+}
+
+static int nl80211_nan_set_non_evac_channels(struct sk_buff *skb,
+					     struct genl_info *info)
+{
+	struct cfg80211_registered_device *rdev = info->user_ptr[0];
+	struct wireless_dev *wdev = info->user_ptr[1];
+	int rem, i = 0, n_channels = 0;
+	struct nlattr *channel;
+
+	if (wdev->iftype != NL80211_IFTYPE_NAN)
+		return -EOPNOTSUPP;
+
+	if (!wdev_running(wdev))
+		return -ENOTCONN;
+
+	/* Count how many channel attributes we got */
+	nlmsg_for_each_attr_type(channel, NL80211_ATTR_NAN_CHANNEL,
+				 info->nlhdr, GENL_HDRLEN, rem)
+		n_channels++;
+
+	struct cfg80211_nan_non_evac_channels *channels __free(kfree) =
+		kzalloc(struct_size(channels, chandefs, n_channels),
+			GFP_KERNEL);
+	if (!channels)
+		return -ENOMEM;
+
+	channels->n_channels = n_channels;
+
+	nlmsg_for_each_attr_type(channel, NL80211_ATTR_NAN_CHANNEL,
+				 info->nlhdr, GENL_HDRLEN, rem) {
+		int ret;
+
+		ret = nl80211_parse_non_evac_channel(rdev, channel, info,
+						     &channels->chandefs[i]);
+		if (ret)
+			return ret;
+		i++;
+	}
+
+	return cfg80211_nan_set_non_evac_channels(rdev, wdev, channels);
+}
+
 static int nl80211_get_protocol_features(struct sk_buff *skb,
 					 struct genl_info *info)
 {
@@ -18449,6 +18593,7 @@ static int nl80211_set_multicast_to_unicast(struct sk_buff *skb,
 static int nl80211_set_pmk(struct sk_buff *skb, struct genl_info *info)
 {
 	struct cfg80211_registered_device *rdev = info->user_ptr[0];
+	struct wiphy *wiphy = &rdev->wiphy;
 	struct net_device *dev = info->user_ptr[1];
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
 	struct cfg80211_pmk_conf pmk_conf = {};
@@ -18457,7 +18602,9 @@ static int nl80211_set_pmk(struct sk_buff *skb, struct genl_info *info)
 	    wdev->iftype != NL80211_IFTYPE_P2P_CLIENT)
 		return -EOPNOTSUPP;
 
-	if (!wiphy_ext_feature_isset(&rdev->wiphy,
+	if (!wiphy_ext_feature_isset(wiphy,
+				     NL80211_EXT_FEATURE_FAST_ROAM_OFFLOAD) &&
+	    !wiphy_ext_feature_isset(wiphy,
 				     NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X))
 		return -EOPNOTSUPP;
 
@@ -18487,6 +18634,7 @@ static int nl80211_set_pmk(struct sk_buff *skb, struct genl_info *info)
 static int nl80211_del_pmk(struct sk_buff *skb, struct genl_info *info)
 {
 	struct cfg80211_registered_device *rdev = info->user_ptr[0];
+	struct wiphy *wiphy = &rdev->wiphy;
 	struct net_device *dev = info->user_ptr[1];
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
 	const u8 *aa;
@@ -18495,7 +18643,9 @@ static int nl80211_del_pmk(struct sk_buff *skb, struct genl_info *info)
 	    wdev->iftype != NL80211_IFTYPE_P2P_CLIENT)
 		return -EOPNOTSUPP;
 
-	if (!wiphy_ext_feature_isset(&rdev->wiphy,
+	if (!wiphy_ext_feature_isset(wiphy,
+				     NL80211_EXT_FEATURE_FAST_ROAM_OFFLOAD) &&
+	    !wiphy_ext_feature_isset(wiphy,
 				     NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X))
 		return -EOPNOTSUPP;
 
@@ -18917,6 +19067,7 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info)
 	struct cfg80211_color_change_settings params = {};
 	struct net_device *dev = info->user_ptr[1];
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
+	enum ieee80211_ap_reg_power reg_power;
 	struct nlattr **tb;
 	u16 offset;
 	int err;
@@ -19018,6 +19169,15 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 	}
 
+	reg_power =
+		cfg80211_get_6ghz_power_type(params.beacon_color_change.tail,
+					     params.beacon_color_change.tail_len, 0);
+	if (wdev->links[params.link_id].ap.reg_power != reg_power) {
+		GENL_SET_ERR_MSG(info, "6 GHz power type change not allowed");
+		err = -EINVAL;
+		goto out;
+	}
+
 	err = rdev_color_change(rdev, dev, &params);
 
 out:
@@ -19193,6 +19353,23 @@ nl80211_add_mod_link_station(struct sk_buff *skb, struct genl_info *info,
 		params.he_6ghz_capa =
 			nla_data(info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY]);
 
+	if (info->attrs[NL80211_ATTR_CIP_CAPABILITIES]) {
+		struct wireless_dev *wdev = dev->ieee80211_ptr;
+
+		if (!cfg80211_cigtk_supported(wdev, info))
+			return -EINVAL;
+
+		params.cip_cap_set = true;
+		params.cip_cap =
+			nla_get_u8(info->attrs[NL80211_ATTR_CIP_CAPABILITIES]);
+
+		/* Only permit CIP capabilities when adding link stations */
+		if (!add) {
+			GENL_SET_ERR_MSG(info, "Cannot modify CIP capabilities");
+			return -EINVAL;
+		}
+	}
+
 	if (info->attrs[NL80211_ATTR_OPMODE_NOTIF]) {
 		params.opmode_notif_used = true;
 		params.opmode_notif =
@@ -20621,6 +20798,12 @@ static const struct genl_small_ops nl80211_small_ops[] = {
 		.flags = GENL_ADMIN_PERM,
 		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV_UP),
 	},
+	{
+		.cmd = NL80211_CMD_NAN_SET_NON_EVAC_CHANNELS,
+		.doit = nl80211_nan_set_non_evac_channels,
+		.flags = GENL_ADMIN_PERM,
+		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV_UP),
+	},
 };
 
 static struct genl_family nl80211_fam __ro_after_init = {
@@ -21538,7 +21721,6 @@ void nl80211_mlo_reconf_add_done(struct net_device *dev,
 
 	nl80211_send_mlme_event(rdev, dev, &event, GFP_KERNEL);
 }
-EXPORT_SYMBOL(nl80211_mlo_reconf_add_done);
 
 void nl80211_send_ibss_bssid(struct cfg80211_registered_device *rdev,
 			     struct net_device *netdev, const u8 *bssid,
@@ -22039,7 +22221,9 @@ int nl80211_send_mgmt(struct cfg80211_registered_device *rdev,
 	    (info->ack_tstamp && nla_put_u64_64bit(msg,
 						   NL80211_ATTR_TX_HW_TIMESTAMP,
 						   info->ack_tstamp,
-						   NL80211_ATTR_PAD)))
+						   NL80211_ATTR_PAD)) ||
+	    (info->no_sta &&
+	     nla_put_flag(msg, NL80211_ATTR_FRAME_NO_STA)))
 		goto nla_put_failure;
 
 	genlmsg_end(msg, hdr);
@@ -22574,6 +22758,8 @@ void cfg80211_ch_switch_notify(struct net_device *dev,
 	case NL80211_IFTYPE_AP:
 	case NL80211_IFTYPE_P2P_GO:
 		wdev->links[link_id].ap.chandef = *chandef;
+		wdev->links[link_id].ap.reg_power =
+			wdev->links[link_id].ap.csa_reg_power;
 		break;
 	case NL80211_IFTYPE_ADHOC:
 		wdev->u.ibss.chandef = *chandef;
diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h
index adcfd0278da32b..1f1311c20e9468 100644
--- a/net/wireless/rdev-ops.h
+++ b/net/wireless/rdev-ops.h
@@ -78,42 +78,43 @@ rdev_change_virtual_intf(struct cfg80211_registered_device *rdev,
 
 static inline int rdev_add_key(struct cfg80211_registered_device *rdev,
 			       struct wireless_dev *wdev, int link_id,
-			       u8 key_index, bool pairwise, const u8 *mac_addr,
-			       struct key_params *params)
+			       u8 key_index, enum nl80211_key_type type,
+			       const u8 *mac_addr, struct key_params *params)
 {
 	int ret;
-	trace_rdev_add_key(&rdev->wiphy, wdev, link_id, key_index, pairwise,
+	trace_rdev_add_key(&rdev->wiphy, wdev, link_id, key_index, type,
 			   mac_addr, params->mode);
 	ret = rdev->ops->add_key(&rdev->wiphy, wdev, link_id, key_index,
-				  pairwise, mac_addr, params);
+				  type, mac_addr, params);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
 
 static inline int
 rdev_get_key(struct cfg80211_registered_device *rdev, struct wireless_dev *wdev,
-	     int link_id, u8 key_index, bool pairwise, const u8 *mac_addr,
-	     void *cookie,
+	     int link_id, u8 key_index, enum nl80211_key_type type,
+	     const u8 *mac_addr, void *cookie,
 	     void (*callback)(void *cookie, struct key_params*))
 {
 	int ret;
-	trace_rdev_get_key(&rdev->wiphy, wdev, link_id, key_index, pairwise,
+	trace_rdev_get_key(&rdev->wiphy, wdev, link_id, key_index, type,
 			   mac_addr);
 	ret = rdev->ops->get_key(&rdev->wiphy, wdev, link_id, key_index,
-				  pairwise, mac_addr, cookie, callback);
+				 type, mac_addr, cookie, callback);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
 
 static inline int rdev_del_key(struct cfg80211_registered_device *rdev,
 			       struct wireless_dev *wdev, int link_id,
-			       u8 key_index, bool pairwise, const u8 *mac_addr)
+			       u8 key_index, enum nl80211_key_type type,
+			       const u8 *mac_addr)
 {
 	int ret;
-	trace_rdev_del_key(&rdev->wiphy, wdev, link_id, key_index, pairwise,
+	trace_rdev_del_key(&rdev->wiphy, wdev, link_id, key_index, type,
 			   mac_addr);
 	ret = rdev->ops->del_key(&rdev->wiphy, wdev, link_id, key_index,
-				  pairwise, mac_addr);
+				  type, mac_addr);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
@@ -1096,6 +1097,23 @@ rdev_nan_set_peer_sched(struct cfg80211_registered_device *rdev,
 	return ret;
 }
 
+static inline int
+rdev_nan_set_non_evac_channels(struct cfg80211_registered_device *rdev,
+			       struct wireless_dev *wdev,
+			       struct cfg80211_nan_non_evac_channels *channels)
+{
+	int ret;
+
+	trace_rdev_nan_set_non_evac_channels(&rdev->wiphy, wdev, channels);
+	if (rdev->ops->nan_set_non_evac_channels)
+		ret = rdev->ops->nan_set_non_evac_channels(&rdev->wiphy, wdev,
+							   channels);
+	else
+		ret = -EOPNOTSUPP;
+	trace_rdev_return_int(&rdev->wiphy, ret);
+	return ret;
+}
+
 static inline int rdev_start_pd(struct cfg80211_registered_device *rdev,
 				struct wireless_dev *wdev)
 {
diff --git a/net/wireless/reg.c b/net/wireless/reg.c
index 11665e0a7efc8a..b7ef479a74f013 100644
--- a/net/wireless/reg.c
+++ b/net/wireless/reg.c
@@ -820,13 +820,13 @@ static bool valid_regdb(const u8 *data, unsigned int size)
 	country = &hdr->country[0];
 	while ((u8 *)(country + 1) <= data + size) {
 		if (!country->coll_ptr)
-			break;
+			return true;
 		if (!valid_country(data, size, country))
 			return false;
 		country++;
 	}
 
-	return true;
+	return false;
 }
 
 static void set_wmm_rule(const struct fwdb_header *db,
@@ -2361,6 +2361,10 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev)
 	}
 
 	for (link = 0; link < ARRAY_SIZE(wdev->links); link++) {
+		struct cfg80211_beaconing_check_config config = {
+			.iftype = iftype,
+			.relax = true,
+		};
 		struct ieee80211_channel *chan;
 
 		if (!wdev->valid_links && link > 0)
@@ -2373,6 +2377,7 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev)
 			if (!wdev->links[link].ap.beacon_interval)
 				continue;
 			chandef = wdev->links[link].ap.chandef;
+			config.reg_power = wdev->links[link].ap.reg_power;
 			break;
 		case NL80211_IFTYPE_MESH_POINT:
 			if (!wdev->u.mesh.beacon_interval)
@@ -2426,8 +2431,8 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev)
 		case NL80211_IFTYPE_P2P_GO:
 		case NL80211_IFTYPE_ADHOC:
 		case NL80211_IFTYPE_MESH_POINT:
-			ret = cfg80211_reg_can_beacon_relax(wiphy, &chandef,
-							    iftype);
+			ret = cfg80211_reg_check_beaconing(wiphy, &chandef,
+							   &config);
 			if (!ret)
 				return ret;
 			break;
diff --git a/net/wireless/scan.c b/net/wireless/scan.c
index caa9c6495f20cc..d35c826c4b1a70 100644
--- a/net/wireless/scan.c
+++ b/net/wireless/scan.c
@@ -1992,20 +1992,22 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev,
 			return false;
 		}
 
-		old = rcu_access_pointer(known->pub.beacon_ies);
+		if (!new->pub.hidden_beacon_bss) {
+			old = rcu_access_pointer(known->pub.beacon_ies);
 
-		rcu_assign_pointer(known->pub.beacon_ies, new->pub.beacon_ies);
+			rcu_assign_pointer(known->pub.beacon_ies, new->pub.beacon_ies);
 
-		/* Override IEs if they were from a beacon before */
-		if (old == rcu_access_pointer(known->pub.ies))
-			rcu_assign_pointer(known->pub.ies, new->pub.beacon_ies);
+			/* Override IEs if they were from a beacon before */
+			if (old == rcu_access_pointer(known->pub.ies))
+				rcu_assign_pointer(known->pub.ies, new->pub.beacon_ies);
 
-		cfg80211_update_hidden_bsses(known,
-					     rcu_access_pointer(new->pub.beacon_ies),
-					     old);
+			cfg80211_update_hidden_bsses(known,
+						     rcu_access_pointer(new->pub.beacon_ies),
+						     old);
 
-		if (old)
-			kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head);
+			if (old)
+				kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head);
+		}
 	}
 
 	known->pub.beacon_interval = new->pub.beacon_interval;
diff --git a/net/wireless/sme.c b/net/wireless/sme.c
index 2a719b5c487e39..458f72d8351ab2 100644
--- a/net/wireless/sme.c
+++ b/net/wireless/sme.c
@@ -1387,7 +1387,15 @@ void __cfg80211_disconnected(struct net_device *dev, const u8 *ie,
 			    NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT))
 			max_key_idx = 7;
 		for (i = 0; i <= max_key_idx; i++)
-			rdev_del_key(rdev, wdev, -1, i, false, NULL);
+			rdev_del_key(rdev, wdev, -1, i, NL80211_KEYTYPE_GROUP,
+				     NULL);
+
+		if (cfg80211_cigtk_supported(wdev, NULL)) {
+			rdev_del_key(rdev, wdev, -1, 0, NL80211_KEYTYPE_CIGTK,
+				     NULL);
+			rdev_del_key(rdev, wdev, -1, 1, NL80211_KEYTYPE_CIGTK,
+				     NULL);
+		}
 	}
 
 	rdev_set_qos_map(rdev, dev, NULL);
diff --git a/net/wireless/sysfs.c b/net/wireless/sysfs.c
index 3ec25d3f004e3a..89ca3f080b7c31 100644
--- a/net/wireless/sysfs.c
+++ b/net/wireless/sysfs.c
@@ -5,7 +5,7 @@
  *
  * Copyright 2005-2006	Jiri Benc <jbenc@suse.cz>
  * Copyright 2006	Johannes Berg <johannes@sipsolutions.net>
- * Copyright (C) 2020-2021, 2023-2024 Intel Corporation
+ * Copyright (C) 2020-2021, 2023-2024, 2026 Intel Corporation
  */
 
 #include <linux/device.h>
@@ -52,16 +52,16 @@ static ssize_t addresses_show(struct device *dev,
 			      char *buf)
 {
 	struct wiphy *wiphy = &dev_to_rdev(dev)->wiphy;
-	char *start = buf;
-	int i;
+	int i, len = 0;
 
 	if (!wiphy->addresses)
-		return sprintf(buf, "%pM\n", wiphy->perm_addr);
+		return sysfs_emit(buf, "%pM\n", wiphy->perm_addr);
 
 	for (i = 0; i < wiphy->n_addresses; i++)
-		buf += sprintf(buf, "%pM\n", wiphy->addresses[i].addr);
+		len += sysfs_emit_at(buf, len, "%pM\n",
+				     wiphy->addresses[i].addr);
 
-	return buf - start;
+	return len;
 }
 static DEVICE_ATTR_RO(addresses);
 
diff --git a/net/wireless/trace.h b/net/wireless/trace.h
index 8c2a91b85c3960..83007a824010f3 100644
--- a/net/wireless/trace.h
+++ b/net/wireless/trace.h
@@ -557,15 +557,15 @@ TRACE_EVENT(rdev_change_virtual_intf,
 
 DECLARE_EVENT_CLASS(key_handle,
 	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
-		 u8 key_index, bool pairwise, const u8 *mac_addr),
-	TP_ARGS(wiphy, wdev, link_id, key_index, pairwise, mac_addr),
+		 u8 key_index, enum nl80211_key_type type, const u8 *mac_addr),
+	TP_ARGS(wiphy, wdev, link_id, key_index, type, mac_addr),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		WDEV_ENTRY
 		MAC_ENTRY(mac_addr)
 		__field(int, link_id)
 		__field(u8, key_index)
-		__field(bool, pairwise)
+		__field(u8, type)
 	),
 	TP_fast_assign(
 		WIPHY_ASSIGN;
@@ -573,38 +573,40 @@ DECLARE_EVENT_CLASS(key_handle,
 		MAC_ASSIGN(mac_addr, mac_addr);
 		__entry->link_id = link_id;
 		__entry->key_index = key_index;
-		__entry->pairwise = pairwise;
+		__entry->type = type;
 	),
 	TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT ", link_id: %d, "
-		  "key_index: %u, pairwise: %s, mac addr: %pM",
+		  "key_index: %u, pairwise: %s, type: %d, mac addr: %pM",
 		  WIPHY_PR_ARG, WDEV_PR_ARG, __entry->link_id,
-		  __entry->key_index, BOOL_TO_STR(__entry->pairwise),
-		  __entry->mac_addr)
+		  __entry->key_index,
+		  BOOL_TO_STR(__entry->type == NL80211_KEYTYPE_PAIRWISE),
+		  __entry->type, __entry->mac_addr)
 );
 
 DEFINE_EVENT(key_handle, rdev_get_key,
 	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
-		 u8 key_index, bool pairwise, const u8 *mac_addr),
-	TP_ARGS(wiphy, wdev, link_id, key_index, pairwise, mac_addr)
+		 u8 key_index, enum nl80211_key_type type, const u8 *mac_addr),
+	TP_ARGS(wiphy, wdev, link_id, key_index, type, mac_addr)
 );
 
 DEFINE_EVENT(key_handle, rdev_del_key,
 	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
-		 u8 key_index, bool pairwise, const u8 *mac_addr),
-	TP_ARGS(wiphy, wdev, link_id, key_index, pairwise, mac_addr)
+		 u8 key_index, enum nl80211_key_type type, const u8 *mac_addr),
+	TP_ARGS(wiphy, wdev, link_id, key_index, type, mac_addr)
 );
 
 TRACE_EVENT(rdev_add_key,
 	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id,
-		 u8 key_index, bool pairwise, const u8 *mac_addr, u8 mode),
-	TP_ARGS(wiphy, wdev, link_id, key_index, pairwise, mac_addr, mode),
+		 u8 key_index, enum nl80211_key_type type, const u8 *mac_addr,
+		 u8 mode),
+	TP_ARGS(wiphy, wdev, link_id, key_index, type, mac_addr, mode),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		WDEV_ENTRY
 		MAC_ENTRY(mac_addr)
 		__field(int, link_id)
 		__field(u8, key_index)
-		__field(bool, pairwise)
+		__field(u8, type)
 		__field(u8, mode)
 	),
 	TP_fast_assign(
@@ -613,15 +615,16 @@ TRACE_EVENT(rdev_add_key,
 		MAC_ASSIGN(mac_addr, mac_addr);
 		__entry->link_id = link_id;
 		__entry->key_index = key_index;
-		__entry->pairwise = pairwise;
+		__entry->type = type;
 		__entry->mode = mode;
 	),
 	TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT ", link_id: %d, "
-		  "key_index: %u, mode: %u, pairwise: %s, "
+		  "key_index: %u, mode: %u, pairwise: %s, type: %d, "
 		  "mac addr: %pM",
 		  WIPHY_PR_ARG, WDEV_PR_ARG, __entry->link_id,
 		  __entry->key_index, __entry->mode,
-		  BOOL_TO_STR(__entry->pairwise), __entry->mac_addr)
+		  BOOL_TO_STR(__entry->type == NL80211_KEYTYPE_PAIRWISE),
+		  __entry->type, __entry->mac_addr)
 );
 
 TRACE_EVENT(rdev_set_default_key,
@@ -2483,6 +2486,31 @@ TRACE_EVENT(rdev_nan_set_peer_sched,
 	)
 );
 
+TRACE_EVENT(rdev_nan_set_non_evac_channels,
+	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev,
+		 struct cfg80211_nan_non_evac_channels *channels),
+	TP_ARGS(wiphy, wdev, channels),
+	TP_STRUCT__entry(
+		WIPHY_ENTRY
+		WDEV_ENTRY
+		__field(u8, n_channels)
+		__dynamic_array(u32, freqs, channels->n_channels)
+	),
+	TP_fast_assign(
+		u32 *freqs = __get_dynamic_array(freqs);
+
+		WIPHY_ASSIGN;
+		WDEV_ASSIGN;
+		__entry->n_channels = channels->n_channels;
+		for (int i = 0; i < channels->n_channels; i++)
+			freqs[i] = channels->chandefs[i].chan->center_freq;
+	),
+	TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT ", n_channels: %u, freqs: %s",
+		  WIPHY_PR_ARG, WDEV_PR_ARG, __entry->n_channels,
+		  __print_array(__get_dynamic_array(freqs),
+				__entry->n_channels, sizeof(u32)))
+);
+
 TRACE_EVENT(rdev_set_mac_acl,
 	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev,
 		 struct cfg80211_acl_data *params),
diff --git a/net/wireless/util.c b/net/wireless/util.c
index f2464d2ce0d5aa..9d5456defc90b1 100644
--- a/net/wireless/util.c
+++ b/net/wireless/util.c
@@ -258,8 +258,24 @@ static bool cfg80211_igtk_cipher_supported(struct wiphy *wiphy)
 	return false;
 }
 
+bool cfg80211_cigtk_supported(struct wireless_dev *wdev,
+			      struct genl_info *info)
+{
+	const struct wiphy_iftype_ext_capab *ext_capab =
+		cfg80211_get_iftype_ext_capa(wdev->wiphy, wdev->iftype);
+
+	if (ext_capab && ext_capab->cip_supported)
+		return true;
+
+	if (info)
+		GENL_SET_ERR_MSG(info,
+				 "Control Integrity Protocol not supported");
+
+	return false;
+}
+
 bool cfg80211_valid_key_idx(struct wireless_dev *wdev,
-			    int key_idx, bool pairwise,
+			    int key_idx, enum nl80211_key_type type,
 			    const u8 *mac_addr)
 {
 	if (WARN_ON(!wdev))
@@ -272,13 +288,24 @@ bool cfg80211_valid_key_idx(struct wireless_dev *wdev,
 	 * Can't differentiate ciphers here so allow 0..3.
 	 * Pairwise keys must be for a station (MAC address given).
 	 */
-	if (pairwise) {
+	if (type == NL80211_KEYTYPE_PAIRWISE) {
 		if (!mac_addr)
 			return false;
 
 		return key_idx < 4;
 	}
 
+	/* Check the wdev/iftype supports CIGTK */
+	if (type == NL80211_KEYTYPE_CIGTK) {
+		if (!cfg80211_cigtk_supported(wdev, NULL))
+			return false;
+
+		if (key_idx >= 2)
+			return false;
+
+		/* fallthrough for mac_addr checks */
+	}
+
 	/*
 	 * For group keys, mac_addr==NULL means setting a group key
 	 * for TX, which is only supported on some interface types,
@@ -343,9 +370,12 @@ bool cfg80211_valid_key_idx(struct wireless_dev *wdev,
 int cfg80211_validate_key_settings(struct cfg80211_registered_device *rdev,
 				   struct wireless_dev *wdev,
 				   struct key_params *params, int key_idx,
-				   bool pairwise, const u8 *mac_addr)
+				   enum nl80211_key_type type,
+				   const u8 *mac_addr)
 {
-	if (!cfg80211_valid_key_idx(wdev, key_idx, pairwise, mac_addr))
+	bool pairwise = type == NL80211_KEYTYPE_PAIRWISE;
+
+	if (!cfg80211_valid_key_idx(wdev, key_idx, type, mac_addr))
 		return -EINVAL;
 
 	switch (params->cipher) {
@@ -1190,7 +1220,7 @@ void cfg80211_upload_connect_keys(struct wireless_dev *wdev)
 	for (i = 0; i < 4; i++) {
 		if (!wdev->connect_keys->params[i].cipher)
 			continue;
-		if (rdev_add_key(rdev, wdev, -1, i, false, NULL,
+		if (rdev_add_key(rdev, wdev, -1, i, NL80211_KEYTYPE_GROUP, NULL,
 				 &wdev->connect_keys->params[i])) {
 			netdev_err(dev, "failed to set key %d\n", i);
 			continue;
diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c
index d45bc08c0de497..ec2389f6b8b189 100644
--- a/net/wireless/wext-compat.c
+++ b/net/wireless/wext-compat.c
@@ -401,6 +401,8 @@ static int cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
 				   int idx, struct key_params *params)
 {
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
+	enum nl80211_key_type key_type =
+		pairwise ? NL80211_KEYTYPE_PAIRWISE : NL80211_KEYTYPE_GROUP;
 	int err, i;
 	bool rejoin = false;
 
@@ -454,11 +456,11 @@ static int cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
 				rejoin = true;
 			}
 
-			if (!cfg80211_valid_key_idx(wdev, idx, pairwise, addr))
+			if (!cfg80211_valid_key_idx(wdev, idx, key_type, addr))
 				err = -ENOENT;
 			else
-				err = rdev_del_key(rdev, wdev, -1, idx, pairwise,
-						   addr);
+				err = rdev_del_key(rdev, wdev, -1, idx,
+						   key_type, addr);
 		}
 		wdev->wext.connect.privacy = false;
 		/*
diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c
index 033e7d059f58c6..f45c9ec7d2c683 100644
--- a/net/x25/af_x25.c
+++ b/net/x25/af_x25.c
@@ -439,10 +439,7 @@ static int x25_setsockopt(struct socket *sock, int level, int optname,
 	if (copy_from_sockptr(&opt, optval, sizeof(int)))
 		goto out;
 
-	if (opt)
-		set_bit(X25_Q_BIT_FLAG, &x25_sk(sk)->flags);
-	else
-		clear_bit(X25_Q_BIT_FLAG, &x25_sk(sk)->flags);
+	assign_bit(X25_Q_BIT_FLAG, &x25_sk(sk)->flags, opt);
 	rc = 0;
 out:
 	return rc;
diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c
index 3bff346308d0f1..bf00d6463c1915 100644
--- a/net/xdp/xskmap.c
+++ b/net/xdp/xskmap.c
@@ -124,7 +124,7 @@ static int xsk_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 	struct bpf_insn *insn = insn_buf;
 
 	*insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
-	*insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5);
+	*insn++ = BPF_JMP32_IMM(BPF_JGE, ret, map->max_entries, 5);
 	*insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(sizeof(struct xsk_sock *)));
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, mp, offsetof(struct xsk_map, xsk_map));
 	*insn++ = BPF_ALU64_REG(BPF_ADD, ret, mp);
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 513c9f2283347c..f6f40ba713d5a8 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -2157,6 +2157,7 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
 	struct xfrm_pol_inexact_bin *bin;
 	struct xfrm_policy *pol, *ret;
 	struct hlist_head *chain;
+	unsigned int inexact_sequence;
 	unsigned int sequence;
 	int err;
 
@@ -2191,12 +2192,18 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
 		goto skip_inexact;
 
 	bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
-	if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr,
-							 daddr))
+	if (!bin)
+		goto skip_inexact;
+
+	inexact_sequence = read_seqcount_begin(&bin->count);
+	if (!xfrm_policy_find_inexact_candidates(&cand, bin, saddr, daddr))
 		goto skip_inexact;
 
 	pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,
 					  family, if_id);
+	if (read_seqcount_retry(&bin->count, inexact_sequence))
+		goto retry;
+
 	if (pol) {
 		ret = pol;
 		if (IS_ERR(pol))
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
index 4b31aa7f432f59..cd0317418f1491 100644
--- a/rust/bindings/bindings_helper.h
+++ b/rust/bindings/bindings_helper.h
@@ -53,6 +53,7 @@
 #include <linux/debugfs.h>
 #include <linux/device/faux.h>
 #include <linux/dma-direction.h>
+#include <linux/dma-fence.h>
 #include <linux/dma-mapping.h>
 #include <linux/dma-resv.h>
 #include <linux/errname.h>
@@ -106,8 +107,8 @@
 #include <../../drivers/base/base.h>
 
 #if defined(CONFIG_DRM_PANIC_SCREEN_QR_CODE)
-// Used by `#[export]` in `drivers/gpu/drm/drm_panic_qr.rs`.
-#include <drm/drm_panic.h>
+// Used by `#[export]` in `drivers/gpu/drm/drm_panic_helper_qr.rs`.
+#include <drm/drm_panic_helper.h>
 #endif
 
 /* `bindgen` gets confused at certain things. */
diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs
index 439ab88a5da13f..98313fc310242b 100644
--- a/rust/bindings/lib.rs
+++ b/rust/bindings/lib.rs
@@ -83,3 +83,8 @@ pub const compat_ptr_ioctl: Option<
         None
     }
 };
+
+// `bindgen` keeps the first definition of a macro and ignores a later `#undef`.
+// The generated `HZ` is therefore `__USER_HZ` from
+// `include/uapi/asm-generic/param.h`, not `CONFIG_HZ`.
+pub const HZ: u32 = CONFIG_HZ;
diff --git a/rust/helpers/dma.c b/rust/helpers/dma.c
index 9fbeb507b08c6c..ff8f24dae9df4e 100644
--- a/rust/helpers/dma.c
+++ b/rust/helpers/dma.c
@@ -49,3 +49,8 @@ __rust_helper void rust_helper_dma_set_max_seg_size(struct device *dev,
 {
 	dma_set_max_seg_size(dev, size);
 }
+
+__rust_helper unsigned int rust_helper_dma_get_max_seg_size(struct device *dev)
+{
+	return dma_get_max_seg_size(dev);
+}
diff --git a/rust/helpers/dma_fence.c b/rust/helpers/dma_fence.c
new file mode 100644
index 00000000000000..549f6b6a717163
--- /dev/null
+++ b/rust/helpers/dma_fence.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/dma-fence.h>
+
+__rust_helper void rust_helper_dma_fence_get(struct dma_fence *f)
+{
+	dma_fence_get(f);
+}
+
+__rust_helper void rust_helper_dma_fence_put(struct dma_fence *f)
+{
+	dma_fence_put(f);
+}
+
+__rust_helper bool rust_helper_dma_fence_begin_signalling(void)
+{
+	return dma_fence_begin_signalling();
+}
+
+__rust_helper void rust_helper_dma_fence_end_signalling(bool cookie)
+{
+	dma_fence_end_signalling(cookie);
+}
+
+__rust_helper bool rust_helper_dma_fence_is_signaled(struct dma_fence *f)
+{
+	return dma_fence_is_signaled(f);
+}
+
+__rust_helper bool rust_helper_dma_fence_test_signaled_flag(struct dma_fence *f)
+{
+	return dma_fence_test_signaled_flag(f);
+}
+
+__rust_helper void rust_helper_dma_fence_lock_irqsave(struct dma_fence *f, unsigned long *flags)
+{
+	dma_fence_lock_irqsave(f, *flags);
+}
+
+__rust_helper void rust_helper_dma_fence_unlock_irqrestore(struct dma_fence *f,
+							   unsigned long *flags)
+{
+	dma_fence_unlock_irqrestore(f, *flags);
+}
+
+__rust_helper void rust_helper_dma_fence_set_error(struct dma_fence *f, int error)
+{
+	dma_fence_set_error(f, error);
+}
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index 440fb7638e3cb1..ab3e80cf1d705d 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -58,6 +58,7 @@
 #include "device.c"
 #include "dma.c"
 #ifdef CONFIG_DMA_SHARED_BUFFER
+#include "dma_fence.c"
 #include "dma-resv.c"
 #endif
 #include "drm.c"
@@ -73,6 +74,7 @@
 #include "kunit.c"
 #include "list.c"
 #include "maple_tree.c"
+#include "math.c"
 #include "mm.c"
 #include "mutex.c"
 #include "net/genetlink.c"
diff --git a/rust/helpers/math.c b/rust/helpers/math.c
new file mode 100644
index 00000000000000..e2ee29bcce0fae
--- /dev/null
+++ b/rust/helpers/math.c
@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/math64.h>
+
+__rust_helper u64 rust_helper_mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d)
+{
+	return mul_u64_add_u64_div_u64(a, b, c, d);
+}
diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c
index a714cc2bfb7ab2..3686e405160d48 100644
--- a/rust/helpers/pci.c
+++ b/rust/helpers/pci.c
@@ -19,6 +19,12 @@ __rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev,
 	return pci_resource_len(pdev, bar);
 }
 
+__rust_helper unsigned long rust_helper_pci_resource_flags(const struct pci_dev *pdev,
+							   int bar)
+{
+	return pci_resource_flags(pdev, bar);
+}
+
 __rust_helper bool rust_helper_dev_is_pci(const struct device *dev)
 {
 	return dev_is_pci(dev);
diff --git a/rust/kernel/Kconfig.test b/rust/kernel/Kconfig.test
index e6a5c7a795f0fc..0087749995d2b8 100644
--- a/rust/kernel/Kconfig.test
+++ b/rust/kernel/Kconfig.test
@@ -83,4 +83,14 @@ config RUST_BITFIELD_KUNIT_TEST
 
 	  If unsure, say N.
 
+config RUST_TIME_KUNIT_TEST
+	bool "KUnit tests for the Rust time API" if !KUNIT_ALL_TESTS
+	default KUNIT_ALL_TESTS
+	help
+	  This option enables KUnit tests for the Rust time API.
+	  These are only for development and testing, not for regular
+	  kernel use cases.
+
+	  If unsure, say N.
+
 endif
diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs
index 21067bde686078..68d80f3fa6c865 100644
--- a/rust/kernel/alloc.rs
+++ b/rust/kernel/alloc.rs
@@ -130,6 +130,12 @@ impl NumaNode {
         }
         Ok(Self(node))
     }
+
+    /// Returns the raw NUMA node identifier as an `i32`.
+    #[inline]
+    pub fn id(&self) -> i32 {
+        self.0
+    }
 }
 
 /// Specify necessary constant to pass the information to Allocator that the caller doesn't care
diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs
index c63d6acdbb6f90..2c929eb1c3d65f 100644
--- a/rust/kernel/alloc/kbox.rs
+++ b/rust/kernel/alloc/kbox.rs
@@ -345,7 +345,7 @@ where
     /// }
     ///
     /// // Allocate a boxed slice of 10 `Example`s.
-    /// let s = KBox::pin_slice(
+    /// let s = KBox::pin_slice::<_, _, Error, _>(
     ///     | _i | Example::new(),
     ///     10,
     ///     GFP_KERNEL
@@ -355,15 +355,16 @@ where
     /// assert_eq!(s[3].d.lock().a, 20);
     /// # Ok::<(), Error>(())
     /// ```
-    pub fn pin_slice<Func, Item, E>(
+    pub fn pin_slice<Func, Item, E, E2>(
         mut init: Func,
         len: usize,
         flags: Flags,
     ) -> Result<Pin<Box<[T], A>>, E>
     where
         Func: FnMut(usize) -> Item,
-        Item: PinInit<T, E>,
+        Item: PinInit<T, E2>,
         E: From<AllocError>,
+        E: From<E2>,
     {
         let mut buffer = super::Vec::<T, A>::with_capacity(len, flags)?;
         for i in 0..len {
diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs
index 60dfbec8f3302b..9d3de109b3813d 100644
--- a/rust/kernel/auxiliary.rs
+++ b/rust/kernel/auxiliary.rs
@@ -137,10 +137,18 @@ pub struct DeviceId(bindings::auxiliary_device_id);
 
 impl DeviceId {
     /// Create a new [`DeviceId`] from name.
+    ///
+    /// This is only intended to be called in const context, when constructing a
+    /// device ID table, where exceeding `AUXILIARY_NAME_SIZE` is a compile time error.
     pub const fn new(modname: &'static CStr, name: &'static CStr) -> Self {
         let name = name.to_bytes_with_nul();
         let modname = modname.to_bytes_with_nul();
 
+        assert!(
+            modname.len().saturating_add(name.len()) <= bindings::AUXILIARY_NAME_SIZE as usize,
+            "auxiliary device ID is too long"
+        );
+
         let mut id: bindings::auxiliary_device_id = pin_init::zeroed();
         let mut i = 0;
         while i < modname.len() {
@@ -305,6 +313,10 @@ impl Device<device::Bound> {
     /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned
     /// reference to the registration data.
     ///
+    /// The outer reference carries the `&self` lifetime while the inner type carries the HRTB
+    /// lifetime `'a`, implying `'a` outlives `&self`. This allows the closure to coerce covariant
+    /// sub-fields (e.g. `&'a T` to the caller's lifetime) and return them directly in `R`.
+    ///
     /// For covariant types that implement [`trait@CovariantForLt`], prefer
     /// [`registration_data`](Self::registration_data) which returns a direct reference.
     ///
@@ -314,13 +326,14 @@ impl Device<device::Bound> {
     /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was
     /// registered by a C driver.
     #[inline]
-    pub fn registration_data_with<F: ForLt + 'static, R>(
-        &self,
-        f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R,
+    pub fn registration_data_with<'this, F: ForLt + 'static, R>(
+        &'this self,
+        f: impl for<'a> FnOnce(Pin<&'this F::Of<'a>>) -> R,
     ) -> Result<R> {
-        // SAFETY: The HRTB closure prevents the caller from smuggling in references with a
-        // concrete short lifetime, making the round-trip from `'static` sound regardless of
-        // variance.
+        // SAFETY: The HRTB on the inner type prevents the caller from exploiting a specific
+        // choice of `'a`. Covariant sub-fields can be safely coerced to `'this`, while
+        // invariant fields cannot be coerced and thus cannot escape with an incorrect
+        // lifetime.
         let pinned = unsafe { self.registration_data_pinned::<F>()? };
 
         Ok(f(pinned))
diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs
index a0d089423f21c9..15c78790e151fb 100644
--- a/rust/kernel/bitfield.rs
+++ b/rust/kernel/bitfield.rs
@@ -346,6 +346,15 @@ macro_rules! bitfield {
                 Self::from_raw(val)
             }
         }
+
+        // SAFETY: `$name` is transparent over `$storage` and `$storage` has no interior mutability.
+        unsafe impl $crate::mem::AsRepr for $name {
+            // Normalize `$storage` to the canonical repr type in case it is signed.
+            type Repr = <$storage as $crate::mem::AsRepr>::Repr;
+        }
+
+        // SAFETY: `$name` is transparent over `$storage`.
+        unsafe impl $crate::mem::AsReprMut for $name {}
     };
 
     // Definitions requiring knowledge of individual fields: private and public field accessors,
diff --git a/rust/kernel/block/mq/gen_disk.rs b/rust/kernel/block/mq/gen_disk.rs
index fc97dd8739746a..0fe640f3aec63c 100644
--- a/rust/kernel/block/mq/gen_disk.rs
+++ b/rust/kernel/block/mq/gen_disk.rs
@@ -7,14 +7,26 @@
 
 use crate::{
     bindings,
-    block::mq::{Operations, TagSet},
-    error::{self, from_err_ptr, Result},
-    fmt::{self, Write},
+    block::mq::{
+        Operations,
+        TagSet, //
+    },
+    error::{
+        from_err_ptr,
+        to_result, //
+    },
+    fmt::{
+        self,
+        Write, //
+    },
     prelude::*,
     static_lock_class,
     str::NullTerminatedFormatter,
     sync::Arc,
-    types::{ForeignOwnable, ScopeGuard},
+    types::{
+        ForeignOwnable,
+        ScopeGuard, //
+    }, //
 };
 
 /// A builder for [`GenDisk`].
@@ -54,7 +66,7 @@ impl GenDiskBuilder {
     /// and that it is a power of two.
     pub fn validate_block_size(size: u32) -> Result {
         if !(512..=bindings::PAGE_SIZE as u32).contains(&size) || !size.is_power_of_two() {
-            Err(error::code::EINVAL)
+            Err(EINVAL)
         } else {
             Ok(())
         }
@@ -63,7 +75,7 @@ impl GenDiskBuilder {
     /// Set the logical block size of the device to be built.
     ///
     /// This method will check that block size is a power of two and between 512
-    /// and 4096. If not, an error is returned and the block size is not set.
+    /// and `PAGE_SIZE`. If not, an error is returned and the block size is not set.
     ///
     /// This is the smallest unit the storage device can address. It is
     /// typically 4096 bytes.
@@ -76,7 +88,7 @@ impl GenDiskBuilder {
     /// Set the physical block size of the device to be built.
     ///
     /// This method will check that block size is a power of two and between 512
-    /// and 4096. If not, an error is returned and the block size is not set.
+    /// and `PAGE_SIZE`. If not, an error is returned and the block size is not set.
     ///
     /// This is the smallest unit a physical storage device can write
     /// atomically. It is usually the same as the logical block size but may be
@@ -125,30 +137,9 @@ impl GenDiskBuilder {
             )
         })?;
 
-        const TABLE: bindings::block_device_operations = bindings::block_device_operations {
-            submit_bio: None,
-            open: None,
-            release: None,
-            ioctl: None,
-            compat_ioctl: None,
-            check_events: None,
-            unlock_native_capacity: None,
-            getgeo: None,
-            set_read_only: None,
-            swap_slot_free_notify: None,
-            report_zones: None,
-            devnode: None,
-            alternative_gpt_sector: None,
-            get_unique_id: None,
-            // TODO: Set to `THIS_MODULE`.
-            owner: core::ptr::null_mut(),
-            pr_ops: core::ptr::null_mut(),
-            free_disk: None,
-            poll_bio: None,
-        };
-
-        // SAFETY: `gendisk` is a valid pointer as we initialized it above
-        unsafe { (*gendisk).fops = &TABLE };
+        // SAFETY: `gendisk` is a valid pointer. We have exclusive access,
+        // since the disk is not added to the VFS yet.
+        unsafe { (*gendisk).fops = &GenDisk::<T>::VTABLE };
 
         let cleanup_failure = ScopeGuard::new_with_data((gendisk, data), |(gendisk, data)| {
             // SAFETY: `gendisk` came from `__blk_mq_alloc_disk()` above and
@@ -177,7 +168,7 @@ impl GenDiskBuilder {
         // operation, so we will not race.
         unsafe { bindings::set_capacity(gendisk, self.capacity_sectors) };
 
-        crate::error::to_result(
+        to_result(
             // SAFETY: `gendisk` points to a valid and initialized instance of
             // `struct gendisk`.
             unsafe {
@@ -211,9 +202,22 @@ pub struct GenDisk<T: Operations> {
     gendisk: *mut bindings::gendisk,
 }
 
+impl<T: Operations> GenDisk<T> {
+    const VTABLE: bindings::block_device_operations = bindings::block_device_operations {
+        owner: crate::module::this_module::<T::OwnerModule>().as_ptr(),
+        ..pin_init::zeroed()
+    };
+}
+
 // SAFETY: `GenDisk` is an owned pointer to a `struct gendisk` and an `Arc` to a
-// `TagSet` It is safe to send this to other threads as long as T is Send.
-unsafe impl<T: Operations + Send> Send for GenDisk<T> {}
+// `TagSet`. It is safe to send this to other threads as long as these two are `Send`.
+unsafe impl<T> Send for GenDisk<T>
+where
+    T: Operations,
+    T::QueueData: Send,
+    Arc<TagSet<T>>: Send,
+{
+}
 
 impl<T: Operations> Drop for GenDisk<T> {
     fn drop(&mut self) {
diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs
index 861903e18fbfa2..08111d8368eb87 100644
--- a/rust/kernel/block/mq/operations.rs
+++ b/rust/kernel/block/mq/operations.rs
@@ -6,11 +6,17 @@
 
 use crate::{
     bindings,
-    block::mq::{request::RequestDataWrapper, Request},
-    error::{from_result, Result},
+    block::mq::{
+        request::RequestDataWrapper,
+        Request, //
+    },
+    error::from_result,
     prelude::*,
-    sync::{aref::ARef, Refcount},
-    types::ForeignOwnable,
+    sync::{
+        aref::ARef,
+        Refcount, //
+    },
+    types::ForeignOwnable, //
 };
 use core::marker::PhantomData;
 
@@ -30,7 +36,7 @@ type ForeignBorrowed<'a, T> = <T as ForeignOwnable>::Borrowed<'a>;
 pub trait Operations: Sized {
     /// Data associated with the `struct request_queue` that is allocated for
     /// the `GenDisk` associated with this `Operations` implementation.
-    type QueueData: ForeignOwnable;
+    type QueueData: ForeignOwnable + Sync;
 
     /// Called by the kernel to queue a request with the driver. If `is_last` is
     /// `false`, the driver is allowed to defer committing the request.
diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs
index ce3e30c81cb5ed..6115f9aec22859 100644
--- a/rust/kernel/block/mq/request.rs
+++ b/rust/kernel/block/mq/request.rs
@@ -7,15 +7,21 @@
 use crate::{
     bindings,
     block::mq::Operations,
-    error::Result,
+    prelude::*,
     sync::{
-        aref::{ARef, AlwaysRefCounted},
+        aref::{
+            ARef,
+            AlwaysRefCounted, //
+        },
         atomic::Relaxed,
-        Refcount,
+        Refcount, //
     },
-    types::Opaque,
+    types::Opaque, //
+};
+use core::{
+    marker::PhantomData,
+    ptr::NonNull, //
 };
-use core::{marker::PhantomData, ptr::NonNull};
 
 /// A wrapper around a blk-mq [`struct request`]. This represents an IO request.
 ///
diff --git a/rust/kernel/block/mq/tag_set.rs b/rust/kernel/block/mq/tag_set.rs
index dae9df408a8625..df3f90bfbb817d 100644
--- a/rust/kernel/block/mq/tag_set.rs
+++ b/rust/kernel/block/mq/tag_set.rs
@@ -4,17 +4,21 @@
 //!
 //! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
 
-use core::pin::Pin;
-
 use crate::{
     bindings,
-    block::mq::{operations::OperationsVTable, request::RequestDataWrapper, Operations},
-    error::{self, Result},
-    prelude::try_pin_init,
-    types::Opaque,
+    block::mq::{
+        operations::OperationsVTable,
+        request::RequestDataWrapper,
+        Operations, //
+    },
+    error::to_result,
+    prelude::*,
+    types::Opaque, //
+};
+use core::{
+    convert::TryInto,
+    marker::PhantomData, //
 };
-use core::{convert::TryInto, marker::PhantomData};
-use pin_init::{pin_data, pinned_drop, PinInit};
 
 /// A wrapper for the C `struct blk_mq_tag_set`.
 ///
@@ -33,11 +37,7 @@ pub struct TagSet<T: Operations> {
 
 impl<T: Operations> TagSet<T> {
     /// Try to create a new tag set
-    pub fn new(
-        nr_hw_queues: u32,
-        num_tags: u32,
-        num_maps: u32,
-    ) -> impl PinInit<Self, error::Error> {
+    pub fn new(nr_hw_queues: u32, num_tags: u32, num_maps: u32) -> impl PinInit<Self, Error> {
         let tag_set: bindings::blk_mq_tag_set = pin_init::zeroed();
         let tag_set: Result<_> = core::mem::size_of::<RequestDataWrapper>()
             .try_into()
@@ -63,7 +63,7 @@ impl<T: Operations> TagSet<T> {
                 // SAFETY: we do not move out of `tag_set`.
                 let tag_set: &mut Opaque<_> = unsafe { Pin::get_unchecked_mut(tag_set) };
                 // SAFETY: `tag_set` is a reference to an initialized `blk_mq_tag_set`.
-                error::to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())})
+                to_result( unsafe { bindings::blk_mq_alloc_tag_set(tag_set.get())})
             }),
             _p: PhantomData,
         })
diff --git a/rust/kernel/clk.rs b/rust/kernel/clk.rs
index 7abbd0767d8cff..f04f5c4a03d640 100644
--- a/rust/kernel/clk.rs
+++ b/rust/kernel/clk.rs
@@ -35,16 +35,31 @@ impl Hertz {
     const GHZ_TO_HZ: c_ulong = 1_000_000_000;
 
     /// Create a new instance from kilohertz (kHz)
+    ///
+    /// # Panics
+    ///
+    /// Panics if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled and `khz` is greater
+    /// than `c_ulong::MAX / 1_000`.
     pub const fn from_khz(khz: c_ulong) -> Self {
         Self(khz * Self::KHZ_TO_HZ)
     }
 
     /// Create a new instance from megahertz (MHz)
+    ///
+    /// # Panics
+    ///
+    /// Panics if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled and `mhz` is greater
+    /// than `c_ulong::MAX / 1_000_000`.
     pub const fn from_mhz(mhz: c_ulong) -> Self {
         Self(mhz * Self::MHZ_TO_HZ)
     }
 
     /// Create a new instance from gigahertz (GHz)
+    ///
+    /// # Panics
+    ///
+    /// Panics if `CONFIG_RUST_OVERFLOW_CHECKS` is enabled and `ghz` is greater
+    /// than `c_ulong::MAX / 1_000_000_000`.
     pub const fn from_ghz(ghz: c_ulong) -> Self {
         Self(ghz * Self::GHZ_TO_HZ)
     }
diff --git a/rust/kernel/configfs.rs b/rust/kernel/configfs.rs
index cd082b83e9e745..04cea2a1894a6b 100644
--- a/rust/kernel/configfs.rs
+++ b/rust/kernel/configfs.rs
@@ -135,8 +135,9 @@ pub struct Subsystem<Data> {
 // SAFETY: We do not provide any operations on `Subsystem`.
 unsafe impl<Data> Sync for Subsystem<Data> {}
 
-// SAFETY: Ownership of `Subsystem` can safely be transferred to other threads.
-unsafe impl<Data> Send for Subsystem<Data> {}
+// SAFETY: Ownership of `Subsystem` can safely be transferred to other threads
+// if its data can be transferred as well.
+unsafe impl<Data: Send> Send for Subsystem<Data> {}
 
 impl<Data> Subsystem<Data> {
     /// Create an initializer for a [`Subsystem`].
@@ -249,6 +250,13 @@ pub struct Group<Data> {
     data: Data,
 }
 
+// SAFETY: We do not provide any operations on `Group`.
+unsafe impl<Data> Sync for Group<Data> {}
+
+// SAFETY: Ownership of `Group` can safely be transferred to other threads if
+// its data can be transferred as well.
+unsafe impl<Data: Send> Send for Group<Data> {}
+
 impl<Data> Group<Data> {
     /// Create an initializer for a new group.
     ///
@@ -297,26 +305,36 @@ unsafe impl<Data> HasGroup<Data> for Group<Data> {
 ///
 /// `this` must be a valid pointer.
 ///
-/// If `this` does not represent the root group of a configfs subsystem,
-/// `this` must be a pointer to a `bindings::config_group` embedded in a
-/// `Group<Parent>`.
+/// If `this` is the `su_group` field of a `bindings::configfs_subsystem`, that
+/// `configfs_subsystem` must be embedded in a `Subsystem<Parent>`.
 ///
-/// Otherwise, `this` must be a pointer to a `bindings::config_group` that
-/// is embedded in a `bindings::configfs_subsystem` that is embedded in a
-/// `Subsystem<Parent>`.
+/// Otherwise, `this` must be a pointer to a `bindings::config_group` embedded
+/// in a `Group<Parent>`.
 unsafe fn get_group_data<'a, Parent>(this: *mut bindings::config_group) -> &'a Parent {
     // SAFETY: `this` is a valid pointer.
-    let is_root = unsafe { (*this).cg_subsys.is_null() };
-
-    if !is_root {
-        // SAFETY: By C API contact,`this` was returned from a call to
-        // `make_group`. The pointer is known to be embedded within a
-        // `Group<Parent>`.
-        unsafe { &(*Group::<Parent>::container_of(this)).data }
-    } else {
-        // SAFETY: By C API contract, `this` is a pointer to the
-        // `bindings::config_group` field within a `Subsystem<Parent>`.
+    let subsys = unsafe { (*this).cg_subsys };
+    // `link_group()` in `fs/configfs/dir.c` assigns `cg_subsys` for every
+    // `config_group` attached anywhere in a registered subsystem, including
+    // the subsystem's own `su_group` (which gets a pointer to itself). The
+    // only `config_group` with a NULL `cg_subsys` is the configfs root, and
+    // userspace cannot trigger callbacks on it. The group is therefore the
+    // subsystem's `su_group` iff it equals `&cg_subsys->su_group`.
+    //
+    // SAFETY: For every `config_group` the configfs core dispatches a
+    // callback on, `cg_subsys` was set by `link_group()` at registration time
+    // and points to a valid `configfs_subsystem` that outlives the callback.
+    let is_root = !subsys.is_null()
+        && core::ptr::eq(this.cast_const(), unsafe { &raw const (*subsys).su_group });
+
+    if is_root {
+        // SAFETY: By the above, `this` is the `su_group` field of a
+        // `configfs_subsystem` that, by function safety requirements, is
+        // embedded in a `Subsystem<Parent>`.
         unsafe { &(*Subsystem::container_of(this)).data }
+    } else {
+        // SAFETY: By function safety requirements, `this` is a
+        // `config_group` embedded in a `Group<Parent>`.
+        unsafe { &(*Group::<Parent>::container_of(this)).data }
     }
 }
 
@@ -325,7 +343,9 @@ struct GroupOperationsVTable<Parent, Child>(PhantomData<(Parent, Child)>);
 impl<Parent, Child> GroupOperationsVTable<Parent, Child>
 where
     Parent: GroupOperations<Child = Child>,
-    Child: 'static,
+    // We transfer `Arc<Group<Data>>` across a thread boundary in `make_group`
+    // and `drop_item`.
+    Arc<Group<Child>>: Send,
 {
     /// # Safety
     ///
@@ -344,8 +364,9 @@ where
         this: *mut bindings::config_group,
         name: *const kernel::ffi::c_char,
     ) -> *mut bindings::config_group {
-        // SAFETY: By function safety requirements of this function, this call
-        // is safe.
+        // SAFETY: By function safety requirements, `this` points to a configfs
+        // group containing `Parent`. The `GroupOperations` bound guarantees
+        // that `Parent: Sync`, so it is safe to share it with this thread.
         let parent_data = unsafe { get_group_data(this) };
 
         let group_init = match Parent::make_group(
@@ -390,8 +411,9 @@ where
         this: *mut bindings::config_group,
         item: *mut bindings::config_item,
     ) {
-        // SAFETY: By function safety requirements of this function, this call
-        // is safe.
+        // SAFETY: By function safety requirements, `this` points to a configfs
+        // group containing `Parent`. The `GroupOperations` bound guarantees
+        // that `Parent: Sync`, so it is safe to share it with this thread.
         let parent_data = unsafe { get_group_data(this) };
 
         // SAFETY: By function safety requirements, `item` is embedded in a
@@ -403,7 +425,9 @@ where
 
         if Parent::HAS_DROP_ITEM {
             // SAFETY: We called `into_raw` to produce `r_child_group_ptr` in
-            // `make_group`.
+            // `make_group`. This function may be executing on a different
+            // thread than `into_raw`. As `Arc<Group<Child>>: Send` this
+            // ownership transfer is safe.
             let arc: Arc<Group<Child>> = unsafe { Arc::from_raw(r_child_group_ptr.cast_mut()) };
 
             Parent::drop_item(parent_data, arc.as_arc_borrow());
@@ -434,11 +458,13 @@ struct ItemOperationsVTable<Container, Data>(PhantomData<(Container, Data)>);
 impl<Data> ItemOperationsVTable<Group<Data>, Data>
 where
     Data: 'static,
+    // We transfer `Arc<Group<Data>>` across a thread boundary in `release`.
+    Arc<Group<Data>>: Send,
 {
     /// # Safety
     ///
-    /// `this` must be a pointer to a `bindings::config_group` embedded in a
-    /// `Group<Parent>`.
+    /// `this` must be a pointer to the `cg_item` field of a `bindings::config_group` embedded in a
+    /// valid `Group<Data>`.
     ///
     /// This function will destroy the pointee of `this`. The pointee of `this`
     /// must not be accessed after the function returns.
@@ -450,8 +476,9 @@ where
         // embedded within a `Group<Data>`.
         let r_group_ptr = unsafe { Group::<Data>::container_of(c_group_ptr) };
 
-        // SAFETY: We called `into_raw` on `r_group_ptr` in
-        // `make_group`.
+        // SAFETY: We called `into_raw` on `r_group_ptr` in `make_group`. This
+        // function may be running on a different thread than the thread that
+        // called `into_raw`. As `Arc<Group<Data>>: Send`, this is safe.
         let pin_self: Arc<Group<Data>> = unsafe { Arc::from_raw(r_group_ptr.cast_mut()) };
         drop(pin_self);
     }
@@ -483,12 +510,12 @@ impl<Data> ItemOperationsVTable<Subsystem<Data>, Data> {
 ///
 /// Implement this trait on structs that embed a [`Subsystem`] or a [`Group`].
 #[vtable]
-pub trait GroupOperations {
+pub trait GroupOperations: Sync {
     /// The child data object type.
     ///
     /// This group will create subgroups (subdirectories) backed by this kind of
     /// object.
-    type Child: 'static;
+    type Child: 'static + Send;
 
     /// Creates a new subgroup.
     ///
@@ -555,8 +582,10 @@ where
             // `config_group`.
             unsafe { container_of!(item, bindings::config_group, cg_item) };
 
-        // SAFETY: The function safety requirements for this function satisfy
-        // the conditions for this call.
+        // SAFETY: By function safety requirements, `c_group` points to a
+        // configfs group containing `Data`. The `AttributeOperations` bound
+        // guarantees that `Data: Sync`, so it is safe to share it with this
+        // thread.
         let data: &Data = unsafe { get_group_data(c_group) };
 
         // SAFETY: By function safety requirements, `page` is writable for `PAGE_SIZE`.
@@ -589,8 +618,10 @@ where
         // `config_group`.
             unsafe { container_of!(item, bindings::config_group, cg_item) };
 
-        // SAFETY: The function safety requirements for this function satisfy
-        // the conditions for this call.
+        // SAFETY: By function safety requirements, `c_group` points to a
+        // configfs group containing `Data`. The `AttributeOperations` bound
+        // guarantees that `Data: Sync`, so it is safe to share it with this
+        // thread.
         let data: &Data = unsafe { get_group_data(c_group) };
 
         let ret = O::store(
@@ -643,7 +674,7 @@ where
 pub trait AttributeOperations<const ID: u64 = 0> {
     /// The type of the object that contains the field that is backing the
     /// attribute for this operation.
-    type Data;
+    type Data: Sync;
 
     /// Renders the value of an attribute.
     ///
@@ -749,15 +780,18 @@ macro_rules! impl_item_type {
             ) -> Self
             where
                 Data: GroupOperations<Child = Child>,
-                Child: 'static,
+                Arc<Group<Child>>: Send,
+                Arc<Group<Data>>: Send,
             {
                 Self {
                     item_type: Opaque::new(bindings::config_item_type {
                         ct_owner: owner.as_ptr(),
-                        ct_group_ops: GroupOperationsVTable::<Data, Child>::vtable_ptr().cast_mut(),
-                        ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(),
-                        ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(),
-                        ct_bin_attrs: core::ptr::null_mut(),
+                        ct_group_ops: GroupOperationsVTable::<Data, Child>::vtable_ptr(),
+                        ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr(),
+                        __bindgen_anon_1: bindings::config_item_type__bindgen_ty_1 {
+                            ct_attrs_const: core::ptr::from_ref(attributes).cast(),
+                        },
+                        ct_bin_attrs: core::ptr::null(),
                     }),
                     _p: PhantomData,
                 }
@@ -767,14 +801,19 @@ macro_rules! impl_item_type {
             pub const fn new<const N: usize>(
                 owner: &'static ThisModule,
                 attributes: &'static AttributeList<N, Data>,
-            ) -> Self {
+            ) -> Self
+            where
+                Arc<Group<Data>>: Send,
+            {
                 Self {
                     item_type: Opaque::new(bindings::config_item_type {
                         ct_owner: owner.as_ptr(),
-                        ct_group_ops: core::ptr::null_mut(),
-                        ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr().cast_mut(),
-                        ct_attrs: core::ptr::from_ref(attributes).cast_mut().cast(),
-                        ct_bin_attrs: core::ptr::null_mut(),
+                        ct_group_ops: core::ptr::null(),
+                        ct_item_ops: ItemOperationsVTable::<$tpe, Data>::vtable_ptr(),
+                        __bindgen_anon_1: bindings::config_item_type__bindgen_ty_1 {
+                            ct_attrs_const: core::ptr::from_ref(attributes).cast(),
+                        },
+                        ct_bin_attrs: core::ptr::null(),
                     }),
                     _p: PhantomData,
                 }
diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs
index affa2b9490efc9..4b992ea9a0f01d 100644
--- a/rust/kernel/cpufreq.rs
+++ b/rust/kernel/cpufreq.rs
@@ -14,7 +14,13 @@ use crate::{
     cpumask,
     device::{Bound, Device},
     devres,
-    error::{code::*, from_err_ptr, from_result, to_result, Result, VTABLE_DEFAULT_ERROR},
+    error::{
+        code::*,
+        from_result,
+        to_result,
+        Result,
+        VTABLE_DEFAULT_ERROR, //
+    },
     ffi::{c_char, c_ulong},
     prelude::*,
     types::ForeignOwnable,
@@ -29,7 +35,10 @@ use core::{
     marker::PhantomData,
     ops::{Deref, DerefMut},
     pin::Pin,
-    ptr,
+    ptr::{
+        self,
+        NonNull, //
+    },
 };
 
 use macros::vtable;
@@ -687,12 +696,13 @@ struct PolicyCpu<'a>(&'a mut Policy);
 impl<'a> PolicyCpu<'a> {
     fn from_cpu(cpu: CpuId) -> Result<Self> {
         // SAFETY: It is safe to call `cpufreq_cpu_get` for any valid CPU.
-        let ptr = from_err_ptr(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) })?;
+        let ptr =
+            NonNull::new(unsafe { bindings::cpufreq_cpu_get(u32::from(cpu)) }).ok_or(ENODEV)?;
 
         Ok(Self(
             // SAFETY: The `ptr` is guaranteed to be valid and remains valid for the lifetime of
             // the returned reference.
-            unsafe { Policy::from_raw_mut(ptr) },
+            unsafe { Policy::from_raw_mut(ptr.as_ptr()) },
         ))
     }
 }
diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs
index d7b8014a647469..2beb55d444ca1a 100644
--- a/rust/kernel/debugfs.rs
+++ b/rust/kernel/debugfs.rs
@@ -538,7 +538,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
         }
     }
 
-    fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &'static FileOps<T>) {
+    fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &FileOps<T>) {
         #[cfg(CONFIG_DEBUG_FS)]
         core::mem::forget(Entry::file(name, &self.entry, data, vtable));
     }
@@ -550,7 +550,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
     /// This function does not produce an owning handle to the file. The created
     /// file is removed when the [`Scope`] that this directory belongs
     /// to is dropped.
-    pub fn read_only_file<T: Writer + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) {
+    pub fn read_only_file<T: Writer + Send + Sync>(&self, name: &CStr, data: &'data T) {
         self.create_file(name, data, &T::FILE_OPS)
     }
 
@@ -560,11 +560,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
     ///
     /// This function does not produce an owning handle to the file. The created file is removed
     /// when the [`Scope`] that this directory belongs to is dropped.
-    pub fn read_binary_file<T: BinaryWriter + Send + Sync + 'static>(
-        &self,
-        name: &CStr,
-        data: &'data T,
-    ) {
+    pub fn read_binary_file<T: BinaryWriter + Send + Sync>(&self, name: &CStr, data: &'data T) {
         self.create_file(name, data, &T::FILE_OPS)
     }
 
@@ -596,11 +592,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
     /// This function does not produce an owning handle to the file. The created
     /// file is removed when the [`Scope`] that this directory belongs
     /// to is dropped.
-    pub fn read_write_file<T: Writer + Reader + Send + Sync + 'static>(
-        &self,
-        name: &CStr,
-        data: &'data T,
-    ) {
+    pub fn read_write_file<T: Writer + Reader + Send + Sync>(&self, name: &CStr, data: &'data T) {
         let vtable = &<T as ReadWriteFile<_>>::FILE_OPS;
         self.create_file(name, data, vtable)
     }
@@ -612,7 +604,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
     ///
     /// This function does not produce an owning handle to the file. The created file is removed
     /// when the [`Scope`] that this directory belongs to is dropped.
-    pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync + 'static>(
+    pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync>(
         &self,
         name: &CStr,
         data: &'data T,
@@ -655,7 +647,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
     /// This function does not produce an owning handle to the file. The created
     /// file is removed when the [`Scope`] that this directory belongs
     /// to is dropped.
-    pub fn write_only_file<T: Reader + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) {
+    pub fn write_only_file<T: Reader + Send + Sync>(&self, name: &CStr, data: &'data T) {
         let vtable = &<T as WriteFile<_>>::FILE_OPS;
         self.create_file(name, data, vtable)
     }
@@ -666,11 +658,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
     ///
     /// This function does not produce an owning handle to the file. The created file is removed
     /// when the [`Scope`] that this directory belongs to is dropped.
-    pub fn write_binary_file<T: BinaryReader + Send + Sync + 'static>(
-        &self,
-        name: &CStr,
-        data: &'data T,
-    ) {
+    pub fn write_binary_file<T: BinaryReader + Send + Sync>(&self, name: &CStr, data: &'data T) {
         self.create_file(name, data, &T::FILE_OPS)
     }
 
diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs
index 46aad64896ecb3..88a870d8c295ff 100644
--- a/rust/kernel/debugfs/entry.rs
+++ b/rust/kernel/debugfs/entry.rs
@@ -74,7 +74,7 @@ impl Entry<'static> {
                 parent.as_ptr(),
                 core::ptr::from_ref(data) as *mut c_void,
                 core::ptr::null(),
-                &**file_ops,
+                file_ops.fops(),
             )
         };
 
@@ -127,7 +127,7 @@ impl<'a> Entry<'a> {
                 parent.as_ptr(),
                 core::ptr::from_ref(data) as *mut c_void,
                 core::ptr::null(),
-                &**file_ops,
+                file_ops.fops(),
             )
         };
 
diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs
index f15908f71c4a2e..47acc4851d4746 100644
--- a/rust/kernel/debugfs/file_ops.rs
+++ b/rust/kernel/debugfs/file_ops.rs
@@ -20,9 +20,6 @@ use crate::{
 
 use core::marker::PhantomData;
 
-#[cfg(CONFIG_DEBUG_FS)]
-use core::ops::Deref;
-
 /// # Invariant
 ///
 /// `FileOps<T>` will always contain an `operations` which is safe to use for a file backed
@@ -30,7 +27,7 @@ use core::ops::Deref;
 /// into a reference.
 pub(super) struct FileOps<T> {
     #[cfg(CONFIG_DEBUG_FS)]
-    operations: bindings::file_operations,
+    operations: &'static bindings::file_operations,
     #[cfg(CONFIG_DEBUG_FS)]
     mode: u16,
     _phantom: PhantomData<T>,
@@ -41,7 +38,7 @@ impl<T> FileOps<T> {
     ///
     /// The caller asserts that the provided `operations` is safe to use for a file whose
     /// inode has a pointer to `T` in its private data that is safe to convert into a reference.
-    const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self {
+    const unsafe fn new(operations: &'static bindings::file_operations, mode: u16) -> Self {
         Self {
             #[cfg(CONFIG_DEBUG_FS)]
             operations,
@@ -65,11 +62,11 @@ impl<T: Adapter> FileOps<T> {
 }
 
 #[cfg(CONFIG_DEBUG_FS)]
-impl<T> Deref for FileOps<T> {
-    type Target = bindings::file_operations;
-
-    fn deref(&self) -> &Self::Target {
-        &self.operations
+impl<T> FileOps<T> {
+    /// Returns a `'static` reference to the inner `file_operations`.
+    #[inline]
+    pub(crate) fn fops(&self) -> &'static bindings::file_operations {
+        self.operations
     }
 }
 
@@ -130,7 +127,7 @@ pub(crate) trait ReadFile<T> {
 
 impl<T: Writer + Sync> ReadFile<T> for T {
     const FILE_OPS: FileOps<T> = {
-        let operations = bindings::file_operations {
+        let operations = &bindings::file_operations {
             read: Some(bindings::seq_read),
             llseek: Some(bindings::seq_lseek),
             release: Some(bindings::single_release),
@@ -181,7 +178,7 @@ pub(crate) trait ReadWriteFile<T> {
 
 impl<T: Writer + Reader + Sync> ReadWriteFile<T> for T {
     const FILE_OPS: FileOps<T> = {
-        let operations = bindings::file_operations {
+        let operations = &bindings::file_operations {
             open: Some(writer_open::<T>),
             read: Some(bindings::seq_read),
             write: Some(write::<T>),
@@ -238,7 +235,7 @@ pub(crate) trait WriteFile<T> {
 
 impl<T: Reader + Sync> WriteFile<T> for T {
     const FILE_OPS: FileOps<T> = {
-        let operations = bindings::file_operations {
+        let operations = &bindings::file_operations {
             open: Some(write_only_open),
             write: Some(write_only_write::<T>),
             llseek: Some(bindings::noop_llseek),
@@ -290,7 +287,7 @@ pub(crate) trait BinaryReadFile<T> {
 
 impl<T: BinaryWriter + Sync> BinaryReadFile<T> for T {
     const FILE_OPS: FileOps<T> = {
-        let operations = bindings::file_operations {
+        let operations = &bindings::file_operations {
             read: Some(blob_read::<T>),
             llseek: Some(bindings::default_llseek),
             open: Some(bindings::simple_open),
@@ -344,7 +341,7 @@ pub(crate) trait BinaryWriteFile<T> {
 
 impl<T: BinaryReader + Sync> BinaryWriteFile<T> for T {
     const FILE_OPS: FileOps<T> = {
-        let operations = bindings::file_operations {
+        let operations = &bindings::file_operations {
             write: Some(blob_write::<T>),
             llseek: Some(bindings::default_llseek),
             open: Some(bindings::simple_open),
@@ -368,7 +365,7 @@ pub(crate) trait BinaryReadWriteFile<T> {
 
 impl<T: BinaryWriter + BinaryReader + Sync> BinaryReadWriteFile<T> for T {
     const FILE_OPS: FileOps<T> = {
-        let operations = bindings::file_operations {
+        let operations = &bindings::file_operations {
             read: Some(blob_read::<T>),
             write: Some(blob_write::<T>),
             llseek: Some(bindings::default_llseek),
diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs
index c81fca5b498639..f0b9cb84e58ea6 100644
--- a/rust/kernel/device_id.rs
+++ b/rust/kernel/device_id.rs
@@ -146,8 +146,7 @@ impl<T: RawDeviceId, const N: usize> IdArray<T, (), N> {
     /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead.
     pub const fn new_without_index(ids: [T; N]) -> Self {
         // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them.
-        let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { core::mem::transmute_copy(&ids) };
-        core::mem::forget(ids);
+        let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { crate::mem::transmute(ids) };
 
         Self {
             ids: raw_ids,
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 2ce09f8e90c693..4ce914b7d1dad9 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -24,7 +24,6 @@ use crate::{
     },
     prelude::*,
     ptr::KnownSize,
-    sync::aref::ARef,
     transmute::{
         AsBytes,
         FromBytes, //
@@ -223,7 +222,7 @@ impl DmaMask {
 ///
 /// # fn test(dev: &Device<Bound>) -> Result {
 /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
-/// let c: Coherent<[u64]> =
+/// let c: Coherent<'_, [u64]> =
 ///     Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?;
 /// # Ok::<(), Error>(()) }
 /// ```
@@ -390,9 +389,9 @@ impl From<DataDirection> for bindings::dma_data_direction {
 /// };
 ///
 /// # fn test(dev: &Device<Bound>) -> Result {
-/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
+/// let mut dmem: CoherentBox<'_, u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
 /// *dmem = 42;
-/// let dmem: Coherent<u64> = dmem.into();
+/// let dmem: Coherent<'_, u64> = dmem.into();
 /// # Ok::<(), Error>(()) }
 /// ```
 ///
@@ -410,18 +409,18 @@ impl From<DataDirection> for bindings::dma_data_direction {
 /// };
 ///
 /// # fn test(dev: &Device<Bound>) -> Result {
-/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
+/// let mut dmem: CoherentBox<'_, [u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
 /// dmem.fill(42);
-/// let dmem: Coherent<[u64]> = dmem.into();
+/// let dmem: Coherent<'_, [u64]> = dmem.into();
 /// # Ok::<(), Error>(()) }
 /// ```
-pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>);
+pub struct CoherentBox<'a, T: KnownSize + ?Sized>(Coherent<'a, T>);
 
-impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
+impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, [T]> {
     /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`].
     #[inline]
     pub fn zeroed_slice_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         count: usize,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
@@ -432,7 +431,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
     /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`.
     #[inline]
     pub fn zeroed_slice(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         count: usize,
         gfp_flags: kernel::alloc::Flags,
     ) -> Result<Self> {
@@ -480,14 +479,14 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
     ///
     /// # fn test(dev: &Device<Bound>) -> Result {
     /// let data = [0u8, 1u8, 2u8, 3u8];
-    /// let c: CoherentBox<[u8]> =
+    /// let c: CoherentBox<'_, [u8]> =
     ///     CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
     ///
     /// assert_eq!(c.deref(), &data);
     /// # Ok::<(), Error>(()) }
     /// ```
     pub fn from_slice_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         data: &[T],
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
@@ -512,7 +511,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
     /// `dma_attrs` is 0 by default.
     #[inline]
     pub fn from_slice(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         data: &[T],
         gfp_flags: kernel::alloc::Flags,
     ) -> Result<Self>
@@ -523,11 +522,11 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
     }
 }
 
-impl<T: AsBytes + FromBytes> CoherentBox<T> {
+impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, T> {
     /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element.
     #[inline]
     pub fn zeroed_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
     ) -> Result<Self> {
@@ -536,12 +535,12 @@ impl<T: AsBytes + FromBytes> CoherentBox<T> {
 
     /// Same as [`CoherentBox::zeroed_slice`], but for a single element.
     #[inline]
-    pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
+    pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
         Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
     }
 }
 
-impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> {
+impl<T: KnownSize + ?Sized> Deref for CoherentBox<'_, T> {
     type Target = T;
 
     #[inline]
@@ -554,7 +553,7 @@ impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> {
     }
 }
 
-impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> {
+impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<'_, T> {
     #[inline]
     fn deref_mut(&mut self) -> &mut Self::Target {
         // SAFETY:
@@ -565,9 +564,9 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> {
     }
 }
 
-impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coherent<T> {
+impl<'a, T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<'a, T>> for Coherent<'a, T> {
     #[inline]
-    fn from(value: CoherentBox<T>) -> Self {
+    fn from(value: CoherentBox<'a, T>) -> Self {
         value.0
     }
 }
@@ -588,26 +587,20 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coher
 ///   to an allocated region of coherent memory and `dma_addr` is the DMA address base of the
 ///   region.
 /// - The size in bytes of the allocation is equal to size information via pointer.
-// TODO
 //
-// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
-// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
-// that device resources can never survive device unbind.
-//
-// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
-// allocation from surviving device unbind; it would require RCU read side critical sections to
-// access the memory, which may require subsequent unnecessary copies.
-//
-// Hence, find a way to revoke the device resources of a `Coherent`, but not the
-// entire `Coherent` including the allocated memory itself.
-pub struct Coherent<T: KnownSize + ?Sized> {
-    dev: ARef<device::Device>,
+// The lifetime parameter ties DMA allocations to the device's bound scope, ensuring they are freed
+// before the device is unbound under normal circumstances. However, if a `Coherent` is leaked (e.g.
+// via `mem::forget`), device resources such as IOMMU mappings will not be released.  Making all
+// constructors `unsafe` to prevent this is considered too restrictive for the common case; this
+// soundness hole is accepted for now.
+pub struct Coherent<'a, T: KnownSize + ?Sized> {
+    dev: &'a device::Device<Bound>,
     dma_addr: DmaAddress,
     cpu_addr: NonNull<T>,
     dma_attrs: Attrs,
 }
 
-impl<T: KnownSize + ?Sized> Coherent<T> {
+impl<T: KnownSize + ?Sized> Coherent<'_, T> {
     /// Returns the size in bytes of this allocation.
     #[inline]
     pub fn size(&self) -> usize {
@@ -663,10 +656,10 @@ impl<T: KnownSize + ?Sized> Coherent<T> {
     }
 }
 
-impl<T: AsBytes + FromBytes> Coherent<T> {
+impl<'a, T: AsBytes + FromBytes> Coherent<'a, T> {
     /// Allocates a region of `T` of coherent memory.
     fn alloc_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
     ) -> Result<Self> {
@@ -692,9 +685,9 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
         // INVARIANT:
         // - We just successfully allocated a coherent region which is adequately sized for `T`,
         //   hence the cpu address is valid.
-        // - We also hold a refcounted reference to the device.
+        // - `dev` is a valid reference to a bound device that outlives this allocation.
         Ok(Self {
-            dev: dev.into(),
+            dev,
             dma_addr,
             cpu_addr,
             dma_attrs,
@@ -716,13 +709,13 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     /// };
     ///
     /// # fn test(dev: &Device<Bound>) -> Result {
-    /// let c: Coherent<[u64; 4]> =
+    /// let c: Coherent<'_, [u64; 4]> =
     ///     Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
     /// # Ok::<(), Error>(()) }
     /// ```
     #[inline]
     pub fn zeroed_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
     ) -> Result<Self> {
@@ -732,14 +725,14 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the
     /// `dma_attrs` is 0 by default.
     #[inline]
-    pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
+    pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
         Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
     }
 
     /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is
     /// initialized with `init`.
     pub fn init_with_attrs<E>(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
         init: impl Init<T, E>,
@@ -764,7 +757,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     /// with `init`.
     #[inline]
     pub fn init<E>(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         gfp_flags: kernel::alloc::Flags,
         init: impl Init<T, E>,
     ) -> Result<Self>
@@ -776,11 +769,11 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
 
     /// Allocates a region of `[T; len]` of coherent memory.
     fn alloc_slice_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         len: usize,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
-    ) -> Result<Coherent<[T]>> {
+    ) -> Result<Coherent<'a, [T]>> {
         const {
             assert!(
                 core::mem::size_of::<T>() > 0,
@@ -809,9 +802,9 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
         // INVARIANT:
         // - We just successfully allocated a coherent region which is adequately sized for
         //   `[T; len]`, hence the cpu address is valid.
-        // - We also hold a refcounted reference to the device.
+        // - `dev` is a valid reference to a bound device that outlives this allocation.
         Ok(Coherent {
-            dev: dev.into(),
+            dev,
             dma_addr,
             cpu_addr,
             dma_attrs,
@@ -836,17 +829,17 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     /// };
     ///
     /// # fn test(dev: &Device<Bound>) -> Result {
-    /// let c: Coherent<[u64]> =
+    /// let c: Coherent<'_, [u64]> =
     ///     Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
     /// # Ok::<(), Error>(()) }
     /// ```
     #[inline]
     pub fn zeroed_slice_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         len: usize,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
-    ) -> Result<Coherent<[T]>> {
+    ) -> Result<Coherent<'a, [T]>> {
         Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs)
     }
 
@@ -854,10 +847,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     /// `dma_attrs` is 0 by default.
     #[inline]
     pub fn zeroed_slice(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         len: usize,
         gfp_flags: kernel::alloc::Flags,
-    ) -> Result<Coherent<[T]>> {
+    ) -> Result<Coherent<'a, [T]>> {
         Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0))
     }
 
@@ -876,18 +869,18 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     /// # fn test(dev: &Device<Bound>) -> Result {
     /// let data = [0u8, 1u8, 2u8, 3u8];
     /// // `c` has the same content as `data`.
-    /// let c: Coherent<[u8]> =
+    /// let c: Coherent<'_, [u8]> =
     ///     Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
     ///
     /// # Ok::<(), Error>(()) }
     /// ```
     #[inline]
     pub fn from_slice_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         data: &[T],
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
-    ) -> Result<Coherent<[T]>>
+    ) -> Result<Coherent<'a, [T]>>
     where
         T: Copy,
     {
@@ -898,10 +891,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     /// `dma_attrs` is 0 by default.
     #[inline]
     pub fn from_slice(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         data: &[T],
         gfp_flags: kernel::alloc::Flags,
-    ) -> Result<Coherent<[T]>>
+    ) -> Result<Coherent<'a, [T]>>
     where
         T: Copy,
     {
@@ -909,7 +902,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
     }
 }
 
-impl<T> Coherent<[T]> {
+impl<T> Coherent<'_, [T]> {
     /// Returns the number of elements `T` in this allocation.
     ///
     /// Note that this is not the size of the allocation in bytes, which is provided by
@@ -922,10 +915,10 @@ impl<T> Coherent<[T]> {
 }
 
 /// Note that the device configured to do DMA must be halted before this object is dropped.
-impl<T: KnownSize + ?Sized> Drop for Coherent<T> {
+impl<T: KnownSize + ?Sized> Drop for Coherent<'_, T> {
     fn drop(&mut self) {
         let size = T::size(self.cpu_addr.as_ptr());
-        // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
+        // SAFETY: Device pointer is guaranteed as valid by the lifetime of this `Coherent`.
         // The cpu address, and the dma address are valid due to the type invariants on
         // `Coherent`.
         unsafe {
@@ -942,15 +935,15 @@ impl<T: KnownSize + ?Sized> Drop for Coherent<T> {
 
 // SAFETY: It is safe to send a `Coherent` to another thread if `T`
 // can be sent to another thread.
-unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<T> {}
+unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<'_, T> {}
 
 // SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all
 // methods that access the buffer contents (`field_read`, `field_write`, `as_slice`,
 // `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur.
 // The safe methods only return metadata or raw pointers whose use requires `unsafe`.
-unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<T> {}
+unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<'_, T> {}
 
-impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> {
+impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<'_, T> {
     fn write_to_slice(
         &self,
         writer: &mut UserSliceWriter,
@@ -996,15 +989,15 @@ impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> {
 /// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`.
 /// - `dma_attrs` contains the attributes used for the allocation, always including
 ///   `DMA_ATTR_NO_KERNEL_MAPPING`.
-pub struct CoherentHandle {
-    dev: ARef<device::Device>,
+pub struct CoherentHandle<'a> {
+    dev: &'a device::Device<Bound>,
     dma_addr: DmaAddress,
     cpu_handle: NonNull<c_void>,
     size: usize,
     dma_attrs: Attrs,
 }
 
-impl CoherentHandle {
+impl<'a> CoherentHandle<'a> {
     /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
     ///
     /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is
@@ -1012,7 +1005,7 @@ impl CoherentHandle {
     ///
     /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails.
     pub fn alloc_with_attrs(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         size: usize,
         gfp_flags: kernel::alloc::Flags,
         dma_attrs: Attrs,
@@ -1038,9 +1031,9 @@ impl CoherentHandle {
 
         // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call
         // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address,
-        // and we hold a refcounted reference to the device.
+        // and `dev` is a valid reference to a bound device that outlives this allocation.
         Ok(Self {
-            dev: dev.into(),
+            dev,
             dma_addr,
             cpu_handle,
             size,
@@ -1051,7 +1044,7 @@ impl CoherentHandle {
     /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
     #[inline]
     pub fn alloc(
-        dev: &device::Device<Bound>,
+        dev: &'a device::Device<Bound>,
         size: usize,
         gfp_flags: kernel::alloc::Flags,
     ) -> Result<Self> {
@@ -1073,7 +1066,7 @@ impl CoherentHandle {
     }
 }
 
-impl Drop for CoherentHandle {
+impl Drop for CoherentHandle<'_> {
     fn drop(&mut self) {
         // SAFETY: All values are valid by the type invariants on `CoherentHandle`.
         // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged.
@@ -1091,12 +1084,12 @@ impl Drop for CoherentHandle {
 
 // SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle,
 // and a size. None of these are tied to a specific thread.
-unsafe impl Send for CoherentHandle {}
+unsafe impl Send for CoherentHandle<'_> {}
 
 // SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only
 // operations on `&CoherentHandle` are reading the DMA address and size, both of which are
 // plain `Copy` values.
-unsafe impl Sync for CoherentHandle {}
+unsafe impl Sync for CoherentHandle<'_> {}
 
 /// View type for `Coherent`.
 ///
@@ -1236,7 +1229,7 @@ impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> {
     }
 }
 
-impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> {
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<'_, T> {
     type Backend = CoherentIoBackend;
     type Target = T;
 
diff --git a/rust/kernel/dma_buf/dma_fence.rs b/rust/kernel/dma_buf/dma_fence.rs
new file mode 100644
index 00000000000000..18a43e1bb442f6
--- /dev/null
+++ b/rust/kernel/dma_buf/dma_fence.rs
@@ -0,0 +1,1022 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025-2026 Red Hat Inc.
+ * Author: Philipp Stanner <pstanner@redhat.com>
+ */
+
+//! DMA Fence support.
+//!
+//! Reference: <https://docs.kernel.org/driver-api/dma-buf.html#c.dma_fence>
+//!
+//! header: [`include/linux/dma-fence.h`](srctree/include/linux/dma-fence.h)
+
+use crate::{
+    alloc::AllocError,
+    bindings,
+    container_of,
+    error::to_result,
+    prelude::*,
+    types::ForeignOwnable,
+    types::Opaque, //
+};
+
+use core::{
+    marker::PhantomData,
+    mem::ManuallyDrop,
+    ops::Deref,
+    ptr,
+    ptr::{
+        drop_in_place,
+        NonNull, //
+    }, //
+};
+
+use kernel::{
+    str::CString,
+    sync::{
+        aref::{
+            ARef,
+            AlwaysRefCounted, //
+        },
+        atomic::{
+            Atomic,
+            Relaxed, //
+        },
+        rcu::rcu_barrier, //
+    }, //
+};
+
+/// VTable for dma_fence backend_ops callbacks.
+//
+// Mandatory dma_fence backend_ops are implemented implicitly through
+// [`FenceContext`]. Additional ones shall get implemented on this trait.
+pub trait FenceContextOps {
+    /// The generic payload data for [`DriverFence`]s created on this fctx.
+    type FenceDataType: Send + Sync;
+}
+
+/// A dma-fence context. A fence context takes care of associating related fences
+/// with each other, providing each with raising sequence numbers and a common
+/// identifier.
+#[pin_data(PinnedDrop)]
+pub struct FenceContext<T: FenceContextOps + Send + Sync> {
+    /// The fence context number.
+    nr: u64,
+    /// The sequence number for the next fence created.
+    seqno: Atomic<u64>,
+    // The name parameters can be accessed by the dma_fence backend_ops. UAF
+    // errors are prevented by the `call_rcu()` in `drop_driver_fence_data()`.
+    /// The name of the driver this FenceContext's fences belong to.
+    driver_name: CString,
+    /// The name of the timeline this FenceContext's fences belong to.
+    timeline_name: CString,
+    /// The number of all unsignaled fences on this context.
+    // Used to prevent bugs due to forgotten fences.
+    //
+    // The lifetime on `DriverFence`s should typically prevent this from
+    // happening.
+    //
+    // However, we cannot fully guarantee in Rust that `DriverFence`s will not
+    // be forgotten, e.g., through `core::mem::forget()`. This could circumvent
+    // the lifetime which intends to enforce that all fences disappear before
+    // their context.
+    nr_of_unsignaled_fences: Atomic<usize>,
+    /// The user's data.
+    #[pin]
+    data: T,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> FenceContext<T> {
+    // This can later be extended as a vtable in case other parties need support
+    // for the more "exotic" callbacks.
+    const OPS: bindings::dma_fence_ops = bindings::dma_fence_ops {
+        get_driver_name: Some(Self::get_driver_name),
+        get_timeline_name: Some(Self::get_timeline_name),
+        enable_signaling: None,
+        signaled: None,
+        // Deprecated.
+        wait: None,
+        // Must never be implemented for these abstractions.
+        release: None,
+        set_deadline: None,
+    };
+
+    /// Create a new `FenceContext`.
+    pub fn new<E>(
+        initial_seqno: u64,
+        driver_name: &CStr,
+        timeline_name: &CStr,
+        data: impl PinInit<T, E>,
+    ) -> impl PinInit<Self, Error>
+    where
+        Error: From<E>,
+    {
+        let driver_name = CString::try_from(driver_name);
+        let timeline_name = CString::try_from(timeline_name);
+        try_pin_init!(Self {
+            // SAFETY: `dma_fence_context_alloc()` merely works on a global
+            // atomic. Parameter `1` is the number of contexts we want to
+            // allocate.
+            nr: unsafe { bindings::dma_fence_context_alloc(1) },
+            seqno: Atomic::new(initial_seqno),
+            driver_name: driver_name?,
+            timeline_name: timeline_name?,
+            nr_of_unsignaled_fences: Atomic::new(0),
+            data <- data,
+        })
+    }
+
+    fn next_seqno(&self) -> u64 {
+        self.seqno.fetch_add(1, Relaxed)
+    }
+
+    /// Allocate the memory for a [`DriverFence`] and already store `data` inside.
+    ///
+    /// This is needed because many times, creation of a [`DriverFence`] must not
+    /// fail, and allocating might deadlock in some situations.
+    ///
+    /// The `data` you pass here must not perform any operations that are illegal
+    /// in atomic context in its [`Drop`] implementation.
+    pub fn new_fence_allocation(
+        &self,
+        data: T::FenceDataType,
+    ) -> Result<DriverFenceAllocation<'_, T>> {
+        let fence_data = DriverFenceData {
+            rcu_head: Default::default(),
+            // `inner` remains uninitialized until a `DriverFence` takes over.
+            inner: Fence {
+                inner: Opaque::uninit(),
+            },
+            fctx: self,
+            data,
+        };
+
+        // In order to support the C dma_fence callbacks, it is necessary for
+        // a `Fence` and a `DriverFence` to live in the same allocation,
+        // because the C backend passes a dma_fence, from which the driver most
+        // likely wants to be able to access its `data` in `DriverFence`.
+        //
+        // Hence, we need the manage the memory manually. It will be freed by the
+        // C backend automatically once the refcount within `Fence` drops to 0.
+        let data = KBox::new(fence_data, GFP_KERNEL | __GFP_ZERO)?;
+
+        Ok(DriverFenceAllocation {
+            data,
+            ops: &Self::OPS,
+        })
+    }
+
+    extern "C" fn get_driver_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+        // SAFETY: The C backend only invokes this callback with `ptr` pointing
+        // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+        // this module always reside within `Fence` which always resides in a
+        // `DriverFenceData`, thus satisfying the function's safety
+        // requirements.
+        let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+        fctx.driver_name.as_char_ptr()
+    }
+
+    extern "C" fn get_timeline_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+        // SAFETY: The C backend only invokes this callback with `ptr` pointing
+        // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+        // this module always reside within `Fence` which always resides in a
+        // `DriverFenceData`, thus satisfying the function's safety
+        // requirements.
+        let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+        fctx.timeline_name.as_char_ptr()
+    }
+
+    /// Create a [`FenceContext`] from an associated [`bindings::dma_fence`].
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must be a valid pointer to a [`bindings::dma_fence`] which resides
+    /// within a [`Fence`], which in turn resides in a [`DriverFenceData`].
+    unsafe fn from_raw_fence(ptr: *mut bindings::dma_fence) -> &'a Self {
+        let opaque_fence = Opaque::cast_from(ptr);
+
+        // SAFETY: Safe due to the function's overall safety requirements.
+        let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+        // CAST: `DriverFenceData` is `repr(C)` and a `Fence` is its first member.
+        let fence_data_ptr: *const DriverFenceData<'a, T> = fence_ptr.cast();
+
+        // SAFETY: Safe because of the comments directly above.
+        let fence_data = unsafe { &*fence_data_ptr };
+
+        fence_data.fctx
+    }
+}
+
+#[pinned_drop]
+impl<T: FenceContextOps + Send + Sync> PinnedDrop for FenceContext<T> {
+    fn drop(self: Pin<&mut Self>) {
+        // Fence ops callbacks can be called on unsignaled fences. Since these
+        // callbacks can access the fence context and its data, it needs to be
+        // guaranteed that a context only drops after all associated
+        // `DriverFence`s have been dropped. This is unlikely to occur, but
+        // would result in silent UAF. Throw a panic to prevent that.
+        //
+        // TODO:
+        // It would be better if the fence context signals all forgotten fences
+        // itself. To do so, it would keep a list of unsignaled fences. That
+        // list's members would have to be pre-allocated (see
+        // `FenceCallback::new_fence_allocation()`).
+        if self.nr_of_unsignaled_fences.load(Relaxed) != 0 {
+            panic!("Forgotten fences in FenceContext.");
+        }
+
+        // Ensure that the driver cannot unload while there are still dma_fence
+        // callbacks running. At the same time, the RCU barrier addresses the
+        // problem inherited by the C backend, in which backend ops callbacks
+        // might be accessing the fence while it is being signaled (or shortly
+        // after). This could cause UAF access on the fence context's
+        // `fctx.driver_name` and `fctx.timeline_name`.
+        //
+        // Wait for the RCU callbacks in `DriverFence::drop`.
+        rcu_barrier();
+    }
+}
+
+/// Error type for fence callback registration.
+///
+/// Generic over `T` so that `AlreadySignaled` can return the callback to the
+/// caller, allowing it to reclaim any resources owned by the callback (e.g.,
+/// a fence handle that needs to be signaled).
+#[derive(Debug)]
+pub enum CallbackError<T> {
+    /// The fence was already signaled. The callback is returned so the caller
+    /// can extract owned resources without losing them.
+    AlreadySignaled(T),
+    /// Some other error occurred during registration.
+    Other(Error),
+}
+
+impl<T> From<CallbackError<T>> for Error {
+    #[inline]
+    fn from(err: CallbackError<T>) -> Self {
+        match err {
+            CallbackError::AlreadySignaled(_) => ENOENT,
+            CallbackError::Other(e) => e,
+        }
+    }
+}
+
+impl<T> From<AllocError> for CallbackError<T> {
+    #[inline]
+    fn from(e: AllocError) -> Self {
+        CallbackError::Other(Error::from(e))
+    }
+}
+
+/// Trait for callbacks that can be registered on fences.
+///
+/// When the fence signals, the callback will be invoked.
+///
+/// # Example
+///
+/// ```rust
+/// use kernel::dma_buf::FenceCallback;
+///
+/// struct MyCallback {
+///     // Your callback state here
+/// }
+///
+/// impl FenceCallback for MyCallback {
+///     fn on_signal(&mut self) {
+///         pr_info!("Fence signaled!\n");
+///         // Handle fence completion
+///     }
+/// }
+/// ```
+pub trait FenceCallback: Send + 'static {
+    /// Called when the fence is signaled.
+    ///
+    /// This is called from the fence signaling path, which may be in interrupt
+    /// context or with locks held, which is why `self` is only borrowed, so that
+    /// it cannot drop. Implementations must not sleep or perform
+    /// long-running operations.
+    ///
+    /// An implementation likely wants to inform itself (e.g., through a work item)
+    /// within this callback that the associated [`FenceCallbackRegistration`]
+    /// can now be dropped.
+    fn on_signal(&mut self);
+}
+
+/// A callback registration on a fence.
+///
+/// When this object is dropped, the callback is automatically removed if it
+/// hasn't been called yet.
+#[pin_data(PinnedDrop)]
+pub struct FenceCallbackRegistration<T: FenceCallback + 'static> {
+    #[pin]
+    callback_foreign: Opaque<bindings::dma_fence_cb>,
+    callback: ManuallyDrop<T>,
+    fence: ARef<Fence>,
+}
+
+impl<T: FenceCallback> FenceCallbackRegistration<T> {
+    /// Create a [`PinInit`] closure for registering a callback on a fence.
+    ///
+    /// The actual attempt at registering the callback will take place once you
+    /// call an allocator's `pin_init()` function.
+    ///
+    /// On success the callback is pinned in place and will fire when the fence
+    /// signals. On `AlreadySignaled` the callback is returned to the caller so
+    /// that owned resources can be reclaimed.
+    pub fn new<'a>(fence: &'a Fence, callback: T) -> impl PinInit<Self, CallbackError<T>> + 'a
+    where
+        T: 'a,
+    {
+        try_pin_init!(Self {
+            // We need to fully initialize the fence because after
+            // `dma_fence_add_callback()` ran, the callback might immediately
+            // get invoked.
+            callback: ManuallyDrop::new(callback),
+            fence: ARef::from(fence),
+            callback_foreign <- Opaque::try_ffi_init(|ptr| {
+                // SAFETY: `fence.inner.get()` is a valid, initialized `struct
+                // dma_fence`. `ptr` points to the `struct dma_fence_cb` field
+                // within the pinned allocation, so it remains valid until
+                // `dma_fence_remove_callback()` in `PinnedDrop` or until the
+                // callback fires.
+                let ret = unsafe {
+                    to_result(bindings::dma_fence_add_callback(
+                        fence.inner.get(),
+                        ptr,
+                        Some(Self::dma_fence_callback),
+                    ))
+                };
+                match ret {
+                    Ok(()) => Ok(()),
+                    Err(e) => {
+                        // SAFETY: We could not register the callback. Thus,
+                        // C will not use it. So we can just take it back
+                        // and pass it to the user again.
+                        let cb_back = unsafe { ManuallyDrop::take(callback) };
+                        if e == ENOENT {
+                            Err(CallbackError::AlreadySignaled(cb_back))
+                        } else {
+                            Err(CallbackError::Other(e))
+                        }
+                    },
+                }
+            }),
+        }? CallbackError<T>)
+    }
+
+    /// Raw dma fence callback that is called by the C code.
+    ///
+    /// # Safety
+    ///
+    /// This is only called by the dma_fence subsystem with valid pointers.
+    unsafe extern "C" fn dma_fence_callback(
+        _fence: *mut bindings::dma_fence,
+        callback_foreign: *mut bindings::dma_fence_cb,
+    ) {
+        let ptr = Opaque::cast_from(callback_foreign).cast_mut();
+
+        // SAFETY: All callbacks we can receive here have been created in such a way that they are
+        // embedded into a `FenceCallbackRegistration`.
+        let reg: *mut Self = unsafe { container_of!(ptr, Self, callback_foreign) };
+
+        // SAFETY: `reg` is a valid `Self` pointer.
+        //
+        // The backend ensures synchronisation so whoever holds the registration object cannot drop
+        // it while this code is running. See `FenceCallbackRegistration::drop`.
+        unsafe { (*reg).callback.on_signal() };
+    }
+
+    /// Returns a reference to the fence this callback is registered on.
+    #[inline]
+    pub fn fence(&self) -> &Fence {
+        &self.fence
+    }
+}
+
+#[pinned_drop]
+impl<T: FenceCallback> PinnedDrop for FenceCallbackRegistration<T> {
+    fn drop(self: Pin<&mut Self>) {
+        // Always call `dma_fence_remove_callback()`, even if the callback
+        // already ran. This is necessary for synchronization:
+        // `dma_fence_remove_callback()` acquires `fence->lock`, which ensures
+        // that any in-flight `dma_fence_signal()` (which calls our callback
+        // while holding the same lock) has completed before we free the struct.
+        //
+        // Without this, Drop can race with a concurrent signal:
+        //   CPU0 (signal, lock held): take() -> on_signal(fence_ref) (in progress)
+        //   CPU1 (drop): skips lock -> frees struct
+        //   CPU0: accesses fence_ref -> use-after-free
+        //
+        // When the callback has already fired, the signal path detached the
+        // list node via `INIT_LIST_HEAD()`, so dma_fence_remove_callback just
+        // sees an empty node and returns false — the lock acquisition is the
+        // only thing that matters.
+        //
+        // SAFETY: The fence pointer is valid and the cb was initialized by
+        // `dma_fence_add_callback()` during construction.
+        unsafe {
+            bindings::dma_fence_remove_callback(self.fence.as_raw(), self.callback_foreign.get())
+        };
+
+        // SAFETY: This is literally the drop implementation, so no one has
+        // dropped this so far; so we can do it now.
+        unsafe { ManuallyDrop::<T>::drop(self.project().callback) };
+    }
+}
+
+// SAFETY: FenceCallbackRegistration can be sent between threads.
+unsafe impl<T: FenceCallback> Send for FenceCallbackRegistration<T> {}
+
+// SAFETY: &FenceCallbackRegistration can be shared between threads if &T can.
+unsafe impl<T: FenceCallback> Sync for FenceCallbackRegistration<T> where T: Sync {}
+
+/// The receiving counterpart of a [`DriverFence`].
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on the [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+#[repr(transparent)]
+pub struct Fence {
+    /// The actual dma_fence passed to C.
+    inner: Opaque<bindings::dma_fence>,
+}
+
+/// Guard helper for locking within this module.
+///
+/// Its only purpose for now is to avoid a number of unsafe lock-unlock cycles.
+/// It is never used outside of this module.
+// TODO: This should be made more canonical, probably by basing it on a
+// SpinLockIrqGuard once available.
+struct FenceGuard<'a> {
+    inner: &'a Fence,
+    flags: usize,
+}
+
+impl<'a> Deref for FenceGuard<'a> {
+    type Target = &'a Fence;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl Drop for FenceGuard<'_> {
+    fn drop(&mut self) {
+        // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+        // merely a pointer to an integer, which lives as long as this function.
+        // When a `FenceGuard` exists, the lock has been taken by definition.
+        unsafe { bindings::dma_fence_unlock_irqrestore(self.as_raw(), &raw mut self.flags) };
+    }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Send for Fence {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Sync for Fence {}
+
+impl Fence {
+    /// Check whether the fence was signaled at the moment of the function call.
+    ///
+    /// Note that this can return `true` for a [`Fence`] whose [`DriverFence`]
+    /// has not yet been dropped. The reason is that the fence ops callbacks can
+    /// cause the fence to get signaled by the C backend.
+    #[inline]
+    pub fn is_signaled(&self) -> bool {
+        // We should not use `dma_fence_is_signaled_locked()` here, because
+        // according to the C backend's recommendations, that function is
+        // problematic and we should avoid calling that function with a lock
+        // held.
+
+        // SAFETY: Inner `fence` is valid because `self` is valid.
+        let ret = unsafe { bindings::dma_fence_is_signaled(self.as_raw()) };
+
+        // To be as robust as possible for the future we guarantee that an API
+        // caller can 100% rely on the signaling being completed (i.e., all
+        // fence callbacks ran), so we have to take the lock.
+        //
+        // The reason is that the C dma_fence backend currently does not
+        // carefully synchronize the `dma_fence_is_signaled()` function with the
+        // proper spinlock. This can lead to the function returning `true` while
+        // fence callbacks are still being executed. This can be mitigated by
+        // guarding the entire function with the spinlock.
+        //
+        // The fundamental reason is that the C backend currently does guard
+        // setting of the fence's signaled-bit with the fence's spinlock, but
+        // reading is done locklessly.
+        //
+        // See commit c8a5d5ea3ba6a.
+        let _ = self.lock();
+
+        ret
+    }
+
+    /// Lock the fence. A helper only to be used internally in this module.
+    fn lock(&self) -> FenceGuard<'_> {
+        let mut guard = FenceGuard {
+            inner: self,
+            flags: 0,
+        };
+
+        // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+        // merely a pointer to an integer, whose lifetime is tied to the guard
+        // object.
+        unsafe { bindings::dma_fence_lock_irqsave(self.as_raw(), &raw mut guard.flags) };
+
+        guard
+    }
+
+    /// Get the fence's sequence number.
+    #[inline]
+    pub fn seqno(&self) -> u64 {
+        // SAFETY: Valid because `self` is valid.
+        unsafe { (*self.as_raw()).seqno }
+    }
+
+    fn as_raw(&self) -> *mut bindings::dma_fence {
+        self.inner.get()
+    }
+
+    /// Create a [`Fence`] from a raw C [`bindings::dma_fence`].
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must point to an initialized fence that is embedded into a [`Fence`].
+    #[inline]
+    pub unsafe fn from_raw<'a>(ptr: *mut bindings::dma_fence) -> &'a Self {
+        // SAFETY: Safe as per the function's overall safety requirements.
+        unsafe { &*ptr.cast() }
+    }
+}
+
+// SAFETY: These implement the C backends refcounting methods which are proven
+// to work correctly.
+unsafe impl AlwaysRefCounted for Fence {
+    fn inc_ref(&self) {
+        // SAFETY: `self.as_raw()` is a pointer to a valid `struct dma_fence`.
+        unsafe { bindings::dma_fence_get(self.as_raw()) }
+    }
+
+    unsafe fn dec_ref(ptr: NonNull<Self>) {
+        // SAFETY: `ptr` is never a NULL pointer; and when `dec_ref()` is called
+        // the fence is by definition still valid.
+        let fence = unsafe { (*ptr.as_ptr()).inner.get() };
+
+        // SAFETY: `fence` was created validly above. When `dec_ref()` is called,
+        // there is by definition still a reference alive that can be put.
+        unsafe { bindings::dma_fence_put(fence) }
+    }
+}
+
+// Necessary to guarantee that `inner` always comes first and can be freed by C.
+// Also useful for using casts instead of container_of().
+#[repr(C)]
+#[pin_data]
+struct DriverFenceData<'a, T: Send + Sync + FenceContextOps> {
+    #[pin]
+    /// The inner fence.
+    // Must always be the first member so that unsafe casting works; but also
+    // necessary so that the C backend can free the allocation (coming from our
+    // Rust code) with kfree_rcu().
+    inner: Fence,
+    /// Callback head for dropping this in a deferred manner through RCU.
+    rcu_head: bindings::callback_head,
+    /// Reference to access the FenceContext.
+    fctx: &'a FenceContext<T>,
+    /// The API user's data. It is essential that the data only performs
+    /// operations legal in atomic context in its [`Drop`] implementation.
+    #[pin]
+    data: T::FenceDataType,
+}
+
+/// A synchronization primitive mainly for GPU drivers.
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can be shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on a [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+///
+/// It is crucial that a [`DriverFence`] always correctly represents the state
+/// of the associated job on the hardware. Especially, it is strictly necessary
+/// that the owner ensures that all [`DriverFence`]s eventually get signaled.
+/// As a last resort, a [`DriverFence`] will signal itself if it drops
+/// unsignaled and print a warning.
+///
+/// This design intends to implement the [`bindings::dma_fence_ops`] in such a
+/// way that the driver-data necessary to implement the callback's functionality
+/// resides in the [`FenceContext`]. Thus, a [`DriverFence`] contains a
+/// reference to the context, which can be accessed in the callbacks. The
+/// implementation, therefore, ensures that a [`DriverFence`] cannot outlive its
+/// [`FenceContext`]. Unfortunately, this can be circumvented under certain
+/// circumstances in Rust (e.g., usage of [`core::mem::forget`]).
+///
+/// In the unlikely case of such violations, a panic is thrown.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::{
+///     dma_buf::{
+///         DriverFence,
+///         FenceContext,
+///         FenceContextOps,
+///         FenceCallback,
+///         FenceCallbackRegistration,
+///     },
+///     str::CString,
+///     sync::aref::ARef, //
+/// };
+/// use core::fmt::Display;
+///
+/// struct CallbackData { }
+///
+/// impl FenceCallback for CallbackData {
+///     fn on_signal(&mut self) {
+///         pr_info!("DmaFence callback executed.\n");
+///     }
+/// }
+///
+/// #[pin_data]
+/// struct FenceContextData {}
+///
+/// impl FenceContextData {
+///     fn new() -> impl PinInit<Self> {
+///         pin_init!(Self {})
+///     }
+/// }
+///
+/// impl FenceContextOps for FenceContextData {
+///     type FenceDataType = FenceData;
+/// }
+///
+/// let fctx_data = FenceContextData::new();
+///
+///
+/// let mut fctx = KBox::pin_init(
+///     FenceContext::new(0, c"dummy_driver", c"dummy_timeline", fctx_data),
+///     GFP_KERNEL
+/// )?;
+///
+/// struct FenceData {
+///     data: CString,
+/// }
+///
+/// let fence_data = FenceData { data: c"dummy_data".try_into()? };
+///
+/// let fence_alloc = fctx.new_fence_allocation(fence_data)?;
+/// let mut fence = fence_alloc.new_fence();
+///
+/// let cb_data = CallbackData { };
+/// let waiting_fence = ARef::from(fence.as_fence());
+/// let cb_reg = FenceCallbackRegistration::new(&waiting_fence, cb_data);
+/// let cb_reg = KBox::pin_init(cb_reg, GFP_KERNEL)?;
+///
+/// // TODO signalling guards
+/// assert_eq!(waiting_fence.is_signaled(), false);
+/// fence.signal(Ok(()));
+/// assert_eq!(waiting_fence.is_signaled(), true);
+///
+/// Ok::<(), Error>(())
+/// ```
+pub struct DriverFence<'a, T: Send + Sync + FenceContextOps> {
+    /// The actual content of the fence. Lives in a [`NonNull`] so that its
+    /// memory can be managed independently. Valid until both the [`DriverFence`]
+    /// and all associated [`Fence`]s have disappeared.
+    data: NonNull<DriverFenceData<'a, T>>,
+}
+
+/// A pre-prepared DMA fence, carrying the user's data and the memory it and the
+/// fence reside in. Only useful for creating a [`DriverFence`]. Splitting
+/// allocation and full initialization is necessary because fences cannot be
+/// allocated dynamically in some circumstances (deadlock).
+pub struct DriverFenceAllocation<'a, T: Send + Sync + FenceContextOps> {
+    /// The memory for the actual content of the fence.
+    /// Handed over to a [`DriverFence`], or deallocated once the
+    /// [`DriverFenceAllocation`] drops.
+    data: KBox<DriverFenceData<'a, T>>,
+    /// Reference for the ops for the associated [`FenceContext`]
+    ops: &'static bindings::dma_fence_ops,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFenceAllocation<'a, T> {
+    /// Create a new [`DriverFence`], the signalable counterpart of a [`Fence`].
+    ///
+    /// This increments the sequence number in the associated [`FenceContext`].
+    pub fn new_fence(self) -> DriverFence<'a, T> {
+        // We feed the C dma_fence backend a NULL for the spinlock so that it
+        // uses per-fence locks automatically.
+        let null_ptr: *mut bindings::spinlock = ptr::null_mut();
+        let seqno = self.data.fctx.next_seqno();
+        let fence_ptr = self.as_raw();
+        // SAFETY: `fence_ptr` has been created directly above. It will live
+        // at least as long as `Self`. The same applies to `&Self::OPS`.
+        unsafe {
+            bindings::dma_fence_init(fence_ptr, self.ops, null_ptr, self.data.fctx.nr, seqno)
+        };
+
+        self.data.fctx.nr_of_unsignaled_fences.fetch_add(1, Relaxed);
+
+        // A `DriverFenceAllocation`'s purpose is to carry allocated memory, so
+        // that `DriverFence`s can always be created without allocating. In this
+        // method, ownership over that memory is transferred to the new
+        // `DriverFence` and managed through refcounting. The C dma_fence
+        // backend will ultimately free the memory once the refcount reaches 0.
+        let ptr = KBox::into_raw(self.data);
+        // SAFETY: `ptr` was just created validly directly above.
+        let ptr = unsafe { NonNull::new_unchecked(ptr) };
+
+        DriverFence { data: ptr }
+    }
+
+    fn as_raw(&self) -> *mut bindings::dma_fence {
+        self.data.inner.inner.get()
+    }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFence<'a, T> {
+    fn as_raw(&self) -> *mut bindings::dma_fence {
+        // SAFETY: Valid because `self` is valid.
+        let fence_data = unsafe { &*self.data.as_ptr() };
+
+        fence_data.inner.inner.get()
+    }
+
+    /// Create a [`DriverFence`] from a raw pointer to a [`bindings::dma_fence`].
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must be a valid pointer to a `dma_fence` that was obtained through
+    /// a [`DriverFence`] with matching generic data for both fence and associated
+    /// [`FenceContext`].
+    unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Self {
+        let opaque_fence = Opaque::cast_from(ptr);
+
+        // SAFETY: Safe due to the function's overall safety requirements.
+        let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+        // DriverFenceData is `repr(C)` and a Fence is its first member.
+        let fence_data_ptr = fence_ptr as *mut DriverFenceData<'a, T>;
+
+        // SAFETY: `fence_data_ptr` was created validly above.
+        let data = unsafe { NonNull::new_unchecked(fence_data_ptr) };
+
+        Self { data }
+    }
+
+    /// Return the underlying [`Fence`].
+    #[inline]
+    pub fn as_fence(&self) -> &Fence {
+        // SAFETY: `self` is by definition still valid, and it cannot drop until
+        // this new reference is gone.
+        unsafe { Fence::from_raw(self.as_raw()) }
+    }
+
+    /// Signal the fence. This will invoke all registered callbacks.
+    pub fn signal(self, res: Result) {
+        let fence = self.as_fence().lock();
+
+        // SAFETY: `fence` is valid because `self` is valid. The lock must be
+        // held, which we acquired directly above.
+        if !unsafe { bindings::dma_fence_test_signaled_flag(fence.as_raw()) } {
+            if let Err(err) = res {
+                // SAFETY: `fence` is valid because `self` is valid. The fence
+                // must not have been signaled yet, which we check directly above.
+                unsafe { bindings::dma_fence_set_error(fence.as_raw(), err.to_errno()) };
+            }
+            // SAFETY: `fence` is valid because `self` is valid. The lock must
+            // be held, which we acquired above.
+            unsafe { bindings::dma_fence_signal_locked(fence.as_raw()) };
+        }
+
+        // SAFETY: `self.data` is valid because `self` is valid.
+        let fctx = unsafe { self.data.as_ref().fctx };
+        let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+    }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Send for DriverFence<'a, T> {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Sync for DriverFence<'a, T> {}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFence<'a, T> {
+    type Target = T::FenceDataType;
+
+    fn deref(&self) -> &Self::Target {
+        // SAFETY: Thanks to refcounting, `data` is always valid as long as `self` is.
+        let data = unsafe { &*self.data.as_ptr() };
+
+        &data.data
+    }
+}
+
+/// A borrow wrapper for [`DriverFence`]. Implements [`Deref`].
+pub struct DriverFenceBorrow<'a, T: Send + Sync + FenceContextOps> {
+    driver_fence: ManuallyDrop<DriverFence<'a, T>>,
+    _lifetime: PhantomData<&'a T>,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFenceBorrow<'a, T> {
+    type Target = DriverFence<'a, T>;
+
+    fn deref(&self) -> &Self::Target {
+        self.driver_fence.deref()
+    }
+}
+
+// SAFETY: The Rust dma_fence abstractions are already designed around the inner
+// C `dma_fence`, which can serve safely as the identification point when being
+// owned by C. Moreover, safety is ensured by not dropping `DriverFence` and by
+// only allowing operations without side effects on the Borrowed type.
+unsafe impl<T: Send + Sync + FenceContextOps> ForeignOwnable for DriverFence<'_, T> {
+    type Borrowed<'a>
+        = DriverFenceBorrow<'a, T>
+    where
+        Self: 'a;
+    type BorrowedMut<'a>
+        = DriverFenceBorrow<'a, T>
+    where
+        Self: 'a;
+
+    const FOREIGN_ALIGN: usize = core::mem::align_of::<bindings::dma_fence>();
+
+    fn into_foreign(self) -> *mut c_void {
+        let fence = self;
+
+        let ptr = fence.as_raw();
+
+        // DriverFence must not drop.
+        let _ = ManuallyDrop::new(fence);
+
+        ptr.cast()
+    }
+
+    unsafe fn from_foreign(ptr: *mut c_void) -> Self {
+        // SAFETY: Safe because the trait implementation only invokes this with
+        // a valid `ptr`, associated to a `DriverFence` with matching generic data.
+        unsafe { Self::from_raw(ptr.cast()) }
+    }
+
+    unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a>
+    where
+        Self: 'a,
+    {
+        // SAFETY: The trait implementation ensures that `ptr` always resides
+        // within a [`Fence`] within a [`DriverFenceData`].
+        let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+        let driver_fence = ManuallyDrop::new(driver_fence);
+
+        DriverFenceBorrow {
+            driver_fence,
+            _lifetime: PhantomData,
+        }
+    }
+
+    unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a>
+    // FIXME: The bound below and the one above in `borrow` should actually be
+    // unnecessary since the compiler should be able to completely derive all
+    // necessary information automatically. There is currently a compiler bug
+    // preventing that, though:
+    //
+    // https://github.com/rust-lang/rust/issues/155430.
+    //
+    // (Help to) fix the compiler bug and remove the bounds afterwards.
+    where
+        Self: 'a,
+    {
+        // SAFETY: The trait implementation ensures that `ptr` always resides
+        // within a [`Fence`] within a [`DriverFenceData`].
+        let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+        let driver_fence = ManuallyDrop::new(driver_fence);
+
+        DriverFenceBorrow {
+            driver_fence,
+            _lifetime: PhantomData,
+        }
+    }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Drop for DriverFence<'a, T> {
+    fn drop(&mut self) {
+        let guard = self.as_fence().lock();
+
+        // Use dma_fence_test_signaled_flag() instead of
+        // dma_fence_is_signaled_locked() because the C backend wants to get rid
+        // of the latter.
+
+        // SAFETY: `guard` is valid until the `call_rcu()` below.
+        let signaled: bool = unsafe { bindings::dma_fence_test_signaled_flag(guard.as_raw()) };
+        if !signaled {
+            pr_err!("DriverFence drops unsignaled. Danger of memory corruption!\n");
+            // SAFETY: `guard` is valid until the `call_rcu()` below. The fence
+            // must not have been signaled yet, which we check directly above.
+            unsafe { bindings::dma_fence_set_error(guard.as_raw(), ECANCELED.to_errno()) };
+            // SAFETY: `guard` is valid until the `call_rcu()` below. The lock
+            // must be held, which we acquired above.
+            unsafe { bindings::dma_fence_signal_locked(guard.as_raw()) };
+
+            // SAFETY: `self.data` is valid because `self` is valid.
+            let fctx = unsafe { self.data.as_ref().fctx };
+            let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+        }
+        drop(guard);
+
+        // `DriverFenceData` could be accessed through some dma_fence
+        // callbacks right now. Access is being revoked in principle above by
+        // signaling the fence, but since the C backend does not guarantee
+        // perfect full synchronization, we have to wait for one grace period to
+        // ensure that all accessors of `DriverFenceData` (through the
+        // dma_fence_ops accessible through a `Fence`) are gone.
+
+        if !core::mem::needs_drop::<T::FenceDataType>() {
+            // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is
+            // valid and initialized. It is valid until the refcount drops to 0,
+            // which can earliest happen once we drop the `DriverFence`'s
+            // reference here.
+            unsafe { bindings::dma_fence_put(self.as_raw()) };
+            return;
+        }
+
+        // SAFETY: Valid because `self` is valid.
+        let rcu_head_ptr = unsafe { &raw mut (*self.data.as_ptr()).rcu_head };
+
+        // SAFETY: `call_rcu()` is always safe to be called. `rcu_head_ptr` was
+        // created validly above. The module must perform a `synchronize_rcu()`
+        // or `rcu_barrier()` call to guard against module unload.
+        unsafe { bindings::call_rcu(rcu_head_ptr, Some(drop_driver_fence_data::<T>)) };
+    }
+}
+
+// TODO:
+// The entire call_rcu() mechanism in the drop above and the code below would be
+// unnecessary if C's dma_fence_signal() could be reworked in a way that after it
+// ran, the caller knows that no fence_ops callbacks can be running anymore.
+// In other words, if the dma_fence backend would use its spinlock for full
+// synchronization.
+//
+// Then we could move the drop_in_place() and dma_fence_put() upwards into the
+// drop() implementation and call it a day.
+
+/// Finally really drop this `DriverFence<T>`
+///
+/// # Safety
+///
+/// `head` references the `rcu_head` field of an `DriverFenceData<T>`. All
+/// accessors to that `DriverFenceData<T>` must be gone by now. This must be
+/// ensured by signalling the associated `DriverFence<T>` and then waiting
+/// for a grace period until calling this function here.
+unsafe extern "C" fn drop_driver_fence_data<T: Send + Sync + FenceContextOps>(
+    head: *mut bindings::callback_head,
+) {
+    // SAFETY: Caller provides a pointer to the `rcu_head` field of a `DriverFenceData<C>`.
+    let fence_data = unsafe { container_of!(head, DriverFenceData<'_, T>, rcu_head) };
+
+    // SAFETY: `fence_data` was created validly above. All the fence's data will
+    // only drop below, but the raw pointer to the raw C `dma_fence` remains
+    // valid because the reference count is only decremented at the end of the
+    // function.
+    let fence = unsafe { (*fence_data).inner.inner.get() };
+
+    // SAFETY: `fence_data` was created validly above. The user has already
+    // dropped the only conventional accessor to the user data, the `DriverFence`,
+    // one grace period ago. All accessors are gone now.
+    unsafe { drop_in_place(&raw mut (*fence_data).data) };
+
+    // The inner `Fence` explicitly does not get dropped because there may be
+    // many more users / consumers, each holding their own reference.
+
+    // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is valid
+    // and initialized. It is valid until the refcount drops to 0, which can
+    // earliest happen once we drop the `DriverFence`'s reference here.
+    unsafe { bindings::dma_fence_put(fence) };
+
+    // The actual memory the data associated with a `DriverFence` lives in
+    // gets freed by the C dma_fence backend once the fence's refcount reaches 0.
+}
diff --git a/rust/kernel/dma_buf/mod.rs b/rust/kernel/dma_buf/mod.rs
new file mode 100644
index 00000000000000..4764a828642e94
--- /dev/null
+++ b/rust/kernel/dma_buf/mod.rs
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DMA-buf subsystem abstractions.
+
+pub mod dma_fence;
+
+pub use self::dma_fence::{
+    DriverFence,
+    Fence,
+    FenceCallback,
+    FenceCallbackRegistration,
+    FenceContext,
+    FenceContextOps, //
+};
diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs
index e1ebad77ebe2a5..85657a39e1464b 100644
--- a/rust/kernel/drm/gem/mod.rs
+++ b/rust/kernel/drm/gem/mod.rs
@@ -266,18 +266,7 @@ impl<T: DriverObject, Ctx: DeviceContext> Object<T, Ctx> {
         free: Some(Self::free_callback),
         open: Some(open_callback::<T>),
         close: Some(close_callback::<T>),
-        print_info: None,
-        export: None,
-        pin: None,
-        unpin: None,
-        get_sg_table: None,
-        vmap: None,
-        vunmap: None,
-        mmap: None,
-        status: None,
-        vm_ops: core::ptr::null_mut(),
-        evict: None,
-        rss: None,
+        ..pin_init::zeroed()
     };
 
     /// Returns the `Device` that owns this GEM object.
diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs
index a687d46d170d70..24cb5343e0db9b 100644
--- a/rust/kernel/drm/gem/shmem.rs
+++ b/rust/kernel/drm/gem/shmem.rs
@@ -132,19 +132,16 @@ impl<T: DriverObject> Object<T> {
         open: Some(super::open_callback::<T>),
         close: Some(super::close_callback::<T>),
         print_info: Some(bindings::drm_gem_shmem_object_print_info),
-        export: None,
         pin: Some(bindings::drm_gem_shmem_object_pin),
         unpin: Some(bindings::drm_gem_shmem_object_unpin),
         get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table),
         vmap: Some(bindings::drm_gem_shmem_object_vmap),
         vunmap: Some(bindings::drm_gem_shmem_object_vunmap),
         mmap: Some(bindings::drm_gem_shmem_object_mmap),
-        status: None,
-        rss: None,
         #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")]
         // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side.
         vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops },
-        evict: None,
+        ..pin_init::zeroed()
     };
 
     /// Return a raw pointer to the embedded drm_gem_shmem_object.
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index 5ce9fd129068c7..de8ef8e2aec413 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -11,6 +11,10 @@ use core::{
 
 use crate::{
     bindings,
+    mem::{
+        AsRepr,
+        AsReprMut, //
+    },
     prelude::*,
     ptr::{
         Alignment,
@@ -226,6 +230,17 @@ fn io_view<'a, IO: Io<'a>, U>(
     Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
 }
 
+/// Returns the primitive view of a I/O view.
+#[inline]
+fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>(
+    this: IO,
+) -> <IO::Backend as IoBackend>::View<'a, T::Repr> {
+    let view = this.as_view();
+
+    // SAFETY: `AsRepr` guarantees layout compatibility.
+    unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::<T::Repr>()) }
+}
+
 /// I/O backends.
 ///
 /// This is an abstract representation to be implemented by arbitrary I/O
@@ -353,15 +368,12 @@ pub trait IoCopyable: IoBackend {
 ///
 /// - The valid `Base` to operate on. For most registers, this should be [`Region`].
 /// - The offset to access (returned by [`IoLoc::offset`]),
-/// - The width of the access (determined by [`IoLoc::IoType`]),
-/// - The type `T` in which the raw data is returned or provided.
+/// - The type `T` in which the data is returned or provided.
 ///
-/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type
-/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`).
+/// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T:
+/// AsRepr` bound and the `<T as AsRepr>::Repr` type would be used to perform I/O and converted to
+/// `T` instead.
 pub trait IoLoc<Base: ?Sized, T> {
-    /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset).
-    type IoType: Into<T> + From<T>;
-
     /// Consumes `self` and returns the offset of this location.
     fn offset(self) -> usize;
 }
@@ -372,8 +384,6 @@ macro_rules! impl_usize_ioloc {
     ($($ty:ty),*) => {
         $(
             impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize {
-                type IoType = $ty;
-
                 #[inline(always)]
                 fn offset(self) -> usize {
                     self
@@ -437,6 +447,45 @@ pub trait Io<'a>: IoBase<'a> {
         self.len() == 0
     }
 
+    /// Convert into a different typed I/O view.
+    ///
+    /// The target type must be known (statically) to be of the same or smaller size to current
+    /// type, and the current view must be properly aligned for the target type.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use kernel::io::{
+    ///     io_project,
+    ///     Mmio,
+    ///     Io,
+    ///     Region,
+    /// };
+    /// #[derive(FromBytes, IntoBytes)]
+    /// #[repr(C)]
+    /// struct MyStruct { field: u32, }
+    ///
+    /// # fn test(mmio: &Mmio<'_, Region<0x1000>>) {
+    /// // let mmio: Mmio<'_, Region<0x1000>>;
+    /// let whole: Mmio<'_, MyStruct> = mmio.cast();
+    /// # }
+    /// ```
+    #[inline]
+    fn cast<U>(self) -> <Self::Backend as IoBackend>::View<'a, U>
+    where
+        Self::Target: FromBytes + IntoBytes,
+        U: FromBytes + IntoBytes,
+    {
+        let view = self.as_view();
+        let ptr = Self::Backend::as_ptr(view);
+
+        const_assert!(size_of::<U>() <= Self::Target::MIN_SIZE);
+        const_assert!(align_of::<U>() <= Self::Target::MIN_ALIGN.as_usize());
+
+        // SAFETY: We have checked bounds and alignment, so this is a valid projection.
+        unsafe { Self::Backend::project_view(view, ptr.cast()) }
+    }
+
     /// Try to convert into a different typed I/O view.
     ///
     /// A runtime check is performed to ensure that the target type is of same or smaller size to
@@ -498,10 +547,10 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline]
     fn read_val(self) -> Self::Target
     where
-        Self::Backend: IoCapable<Self::Target>,
-        Self::Target: Sized,
+        Self::Target: AsReprMut,
+        Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
     {
-        Self::Backend::io_read(self.as_view())
+        Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self)))
     }
 
     /// Write a value to I/O.
@@ -520,10 +569,10 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline]
     fn write_val(self, value: Self::Target)
     where
-        Self::Backend: IoCapable<Self::Target>,
-        Self::Target: Sized,
+        Self::Target: AsRepr,
+        Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
     {
-        Self::Backend::io_write(self.as_view(), value)
+        Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value))
     }
 
     /// Copy-read from I/O memory.
@@ -645,7 +694,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_read8(self, offset: usize) -> Result<u8>
     where
-        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        usize: IoLoc<Self::Target, u8>,
         Self::Backend: IoCapable<u8>,
     {
         self.try_read(offset)
@@ -655,7 +704,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_read16(self, offset: usize) -> Result<u16>
     where
-        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        usize: IoLoc<Self::Target, u16>,
         Self::Backend: IoCapable<u16>,
     {
         self.try_read(offset)
@@ -665,7 +714,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_read32(self, offset: usize) -> Result<u32>
     where
-        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        usize: IoLoc<Self::Target, u32>,
         Self::Backend: IoCapable<u32>,
     {
         self.try_read(offset)
@@ -675,7 +724,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_read64(self, offset: usize) -> Result<u64>
     where
-        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        usize: IoLoc<Self::Target, u64>,
         Self::Backend: IoCapable<u64>,
     {
         self.try_read(offset)
@@ -685,7 +734,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_write8(self, value: u8, offset: usize) -> Result
     where
-        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        usize: IoLoc<Self::Target, u8>,
         Self::Backend: IoCapable<u8>,
     {
         self.try_write(offset, value)
@@ -695,7 +744,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_write16(self, value: u16, offset: usize) -> Result
     where
-        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        usize: IoLoc<Self::Target, u16>,
         Self::Backend: IoCapable<u16>,
     {
         self.try_write(offset, value)
@@ -705,7 +754,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_write32(self, value: u32, offset: usize) -> Result
     where
-        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        usize: IoLoc<Self::Target, u32>,
         Self::Backend: IoCapable<u32>,
     {
         self.try_write(offset, value)
@@ -715,7 +764,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_write64(self, value: u64, offset: usize) -> Result
     where
-        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        usize: IoLoc<Self::Target, u64>,
         Self::Backend: IoCapable<u64>,
     {
         self.try_write(offset, value)
@@ -727,7 +776,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn read8(self, offset: usize) -> u8
     where
-        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        usize: IoLoc<Self::Target, u8>,
         Self::Backend: IoCapable<u8>,
     {
         self.read(offset)
@@ -739,7 +788,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn read16(self, offset: usize) -> u16
     where
-        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        usize: IoLoc<Self::Target, u16>,
         Self::Backend: IoCapable<u16>,
     {
         self.read(offset)
@@ -751,7 +800,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn read32(self, offset: usize) -> u32
     where
-        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        usize: IoLoc<Self::Target, u32>,
         Self::Backend: IoCapable<u32>,
     {
         self.read(offset)
@@ -763,7 +812,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn read64(self, offset: usize) -> u64
     where
-        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        usize: IoLoc<Self::Target, u64>,
         Self::Backend: IoCapable<u64>,
     {
         self.read(offset)
@@ -775,7 +824,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn write8(self, value: u8, offset: usize)
     where
-        usize: IoLoc<Self::Target, u8, IoType = u8>,
+        usize: IoLoc<Self::Target, u8>,
         Self::Backend: IoCapable<u8>,
     {
         self.write(offset, value)
@@ -787,7 +836,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn write16(self, value: u16, offset: usize)
     where
-        usize: IoLoc<Self::Target, u16, IoType = u16>,
+        usize: IoLoc<Self::Target, u16>,
         Self::Backend: IoCapable<u16>,
     {
         self.write(offset, value)
@@ -799,7 +848,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn write32(self, value: u32, offset: usize)
     where
-        usize: IoLoc<Self::Target, u32, IoType = u32>,
+        usize: IoLoc<Self::Target, u32>,
         Self::Backend: IoCapable<u32>,
     {
         self.write(offset, value)
@@ -811,7 +860,7 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn write64(self, value: u64, offset: usize)
     where
-        usize: IoLoc<Self::Target, u64, IoType = u64>,
+        usize: IoLoc<Self::Target, u64>,
         Self::Backend: IoCapable<u64>,
     {
         self.write(offset, value)
@@ -843,11 +892,11 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_read<T, L>(self, location: L) -> Result<T>
     where
+        T: AsReprMut,
         L: IoLoc<Self::Target, T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
     {
-        let view = io_view::<Self, L::IoType>(self, location.offset())?;
-        Ok(Self::Backend::io_read(view).into())
+        Ok(io_read!(self, try: location))
     }
 
     /// Generic fallible write with runtime bounds check.
@@ -876,12 +925,11 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_write<T, L>(self, location: L, value: T) -> Result
     where
+        T: AsRepr,
         L: IoLoc<Self::Target, T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
     {
-        let view = io_view::<Self, L::IoType>(self, location.offset())?;
-        let io_value = value.into();
-        Self::Backend::io_write(view, io_value);
+        io_write!(self, try: location, value);
         Ok(())
     }
 
@@ -900,6 +948,8 @@ pub trait Io<'a>: IoBase<'a> {
     /// };
     ///
     /// register! {
+    ///     base: Region;
+    ///
     ///     VERSION(u32) @ 0x100 {
     ///         15:8 major;
     ///         7:0  minor;
@@ -920,9 +970,10 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_write_reg<T, L, V>(self, value: V) -> Result
     where
+        T: AsRepr,
         L: IoLoc<Self::Target, T>,
         V: LocatedRegister<Self::Target, Location = L, Value = T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
     {
         let (location, value) = value.into_io_op();
 
@@ -954,16 +1005,13 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn try_update<T, L, F>(self, location: L, f: F) -> Result
     where
+        T: AsReprMut,
         L: IoLoc<Self::Target, T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
         F: FnOnce(T) -> T,
     {
-        let view = io_view::<Self, L::IoType>(self, location.offset())?;
-
-        let value: T = Self::Backend::io_read(view).into();
-        let io_value = f(value).into();
-        Self::Backend::io_write(view, io_value);
-
+        let view = io_project!(self, try: location);
+        view.write_val(f(view.read_val()));
         Ok(())
     }
 
@@ -991,11 +1039,11 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn read<T, L>(self, location: L) -> T
     where
+        T: AsReprMut,
         L: IoLoc<Self::Target, T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
     {
-        let view = io_view_assert::<Self, L::IoType>(self, location.offset());
-        Self::Backend::io_read(view).into()
+        io_read!(self, build: location)
     }
 
     /// Generic infallible write with compile-time bounds check.
@@ -1022,12 +1070,11 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn write<T, L>(self, location: L, value: T)
     where
+        T: AsRepr,
         L: IoLoc<Self::Target, T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
     {
-        let view = io_view_assert::<Self, L::IoType>(self, location.offset());
-        let io_value = value.into();
-        Self::Backend::io_write(view, io_value);
+        io_write!(self, build: location, value);
     }
 
     /// Generic infallible write of a fully-located register value.
@@ -1045,6 +1092,8 @@ pub trait Io<'a>: IoBase<'a> {
     /// };
     ///
     /// register! {
+    ///     base: Region<0x1000>;
+    ///
     ///     VERSION(u32) @ 0x100 {
     ///         15:8 major;
     ///         7:0  minor;
@@ -1064,9 +1113,10 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn write_reg<T, L, V>(self, value: V)
     where
+        T: AsRepr,
         L: IoLoc<Self::Target, T>,
         V: LocatedRegister<Self::Target, Location = L, Value = T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
     {
         let (location, value) = value.into_io_op();
 
@@ -1098,14 +1148,13 @@ pub trait Io<'a>: IoBase<'a> {
     #[inline(always)]
     fn update<T, L, F>(self, location: L, f: F)
     where
+        T: AsReprMut,
         L: IoLoc<Self::Target, T>,
-        Self::Backend: IoCapable<L::IoType>,
+        Self::Backend: IoCapable<<T as AsRepr>::Repr>,
         F: FnOnce(T) -> T,
     {
-        let view = io_view_assert::<Self, L::IoType>(self, location.offset());
-        let value: T = Self::Backend::io_read(view).into();
-        let io_value = f(value).into();
-        Self::Backend::io_write(view, io_value);
+        let view = io_project!(self, build: location);
+        view.write_val(f(view.read_val()));
     }
 }
 
@@ -1649,6 +1698,25 @@ where
         // SAFETY: Per safety requirement.
         unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) }
     }
+
+    #[inline(always)]
+    pub fn try_project_loc<U, L>(
+        self,
+        location: L,
+    ) -> Result<<T::Backend as IoBackend>::View<'a, U>>
+    where
+        L: IoLoc<T::Target, U>,
+    {
+        io_view::<_, U>(self.0, location.offset())
+    }
+
+    #[inline(always)]
+    pub fn project_loc<U, L>(self, location: L) -> <T::Backend as IoBackend>::View<'a, U>
+    where
+        L: IoLoc<T::Target, U>,
+    {
+        io_view_assert::<_, U>(self.0, location.offset())
+    }
 }
 
 /// Project an I/O type to a subview of it.
@@ -1656,26 +1724,54 @@ where
 /// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that
 /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
 ///
+/// `io_project!` can also project to a subview of registers defined with [`register!`] macro.
+/// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and
+/// `io_project!(io, build: REGISTER)` for infallible projection.
+///
 /// # Examples
 ///
 /// ```
 /// use kernel::io::{
 ///     io_project,
+///     register,
 ///     Mmio,
 /// };
 /// #[repr(C)]
 /// struct MyStruct { field: u32, }
 ///
+/// register! {
+///     base: MyStruct;
+///     FIELD(u32) @ 0 {
+///         31:0 val;
+///     }
+/// }
+///
 /// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result {
 /// // let mmio: Mmio<[MyStruct]>;
 /// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field);
 /// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]);
 /// let nested: Mmio<'_, u32> = io_project!(whole, .field);
+/// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD);
 /// # Ok::<(), Error>(()) }
 /// ```
 #[macro_export]
 #[doc(hidden)]
 macro_rules! io_project {
+    // Register projection
+    ($io:expr, try: $ioloc:expr) => {{
+        #[allow(unused)]
+        use $crate::io::IoBase as _;
+        let view = $crate::io::ProjectHelper($io.as_view());
+        view.try_project_loc($ioloc)?
+    }};
+    ($io:expr, build: $ioloc:expr) => {{
+        #[allow(unused)]
+        use $crate::io::IoBase as _;
+        let view = $crate::io::ProjectHelper($io.as_view());
+        view.project_loc($ioloc)
+    }};
+
+    // Field or index projection
     ($io:expr, $($proj:tt)*) => {{
         #[allow(unused)]
         use $crate::io::IoBase as _;
@@ -1746,6 +1842,12 @@ macro_rules! io_write {
     (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
         $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*])
     };
+    (@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => {
+        $crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*])
+    };
+    (@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => {
+        $crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*])
+    };
     ($io:expr, $($rest:tt)*) => {
         $crate::io_write!(@parse [$io] [] [$($rest)*])
     };
diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs
index 03dfd2ff48c7a8..b6513fa0f412c8 100644
--- a/rust/kernel/io/register.rs
+++ b/rust/kernel/io/register.rs
@@ -8,14 +8,19 @@
 //!
 //! Note: most of the items in this module are public so they can be referenced by the macro, but
 //! most are not to be used directly by users. Outside of the `register!` macro itself, the only
-//! items you might want to import from this module are [`WithBase`] and [`Array`].
+//! item you might want to import from this module is [`Array`].
 //!
 //! # Simple example
 //!
 //! ```no_run
-//! use kernel::io::register;
+//! use kernel::io::{
+//!     register,
+//!     Region,
+//! };
 //!
 //! register! {
+//!     base: Region<0x1000>;
+//!
 //!     /// Basic information about the chip.
 //!     pub BOOT_0(u32) @ 0x00000100 {
 //!         /// Vendor ID.
@@ -55,11 +60,14 @@
 //!         register,
 //!         Io,
 //!         IoLoc,
+//!         Region,
 //!     },
 //!     num::Bounded,
 //! };
-//! # use kernel::io::{Mmio, Region};
+//! # use kernel::io::Mmio;
 //! # register! {
+//! #     base: Region<0x1000>;
+//! #
 //! #     pub BOOT_0(u32) @ 0x00000100 {
 //! #         15:8 vendor_id;
 //! #         7:4 major_revision;
@@ -113,149 +121,50 @@ use crate::{
     io::IoLoc, //
 };
 
-use super::Region;
-
-/// Trait implemented by all registers.
-pub trait Register: Sized {
-    /// Backing primitive type of the register.
-    type Storage: Into<Self> + From<Self>;
-
-    /// Start offset of the register.
-    ///
-    /// The interpretation of this offset depends on the type of the register.
-    const OFFSET: usize;
-}
-
-/// Trait implemented by registers with a fixed offset.
-pub trait FixedRegister: Register {}
-
 /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when
-/// passing a [`FixedRegister`] value.
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for ()
+/// passing a [`FixedIoLoc`] value.
+impl<Base: ?Sized, T> IoLoc<Base, T> for ()
 where
-    T: FixedRegister,
+    T: FixedIoLoc<Base>,
 {
-    type IoType = T::Storage;
-
     #[inline(always)]
     fn offset(self) -> usize {
-        T::OFFSET
+        T::LOCATION.offset()
     }
 }
 
-/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used
-/// as an [`IoLoc`].
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T
-where
-    T: FixedRegister,
-{
-    type IoType = T::Storage;
-
-    #[inline(always)]
-    fn offset(self) -> usize {
-        T::OFFSET
-    }
-}
-
-/// Location of a fixed register.
-pub struct FixedRegisterLoc<T: FixedRegister>(PhantomData<T>);
-
-impl<T: FixedRegister> FixedRegisterLoc<T> {
-    /// Returns the location of `T`.
-    #[inline(always)]
-    // We do not implement `Default` so we can be const.
-    #[expect(clippy::new_without_default)]
-    pub const fn new() -> Self {
-        Self(PhantomData)
-    }
-}
-
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T>
-where
-    T: FixedRegister,
-{
-    type IoType = T::Storage;
-
-    #[inline(always)]
-    fn offset(self) -> usize {
-        T::OFFSET
-    }
-}
-
-/// Trait providing a base address to be added to the offset of a relative register to obtain
-/// its actual offset.
-///
-/// The `T` generic argument is used to distinguish which base to use, in case a type provides
-/// several bases. It is given to the `register!` macro to restrict the use of the register to
-/// implementors of this particular variant.
-pub trait RegisterBase<T> {
-    /// Base address to which register offsets are added.
-    const BASE: usize;
-}
-
-/// Trait implemented by all registers that are relative to a base.
-pub trait WithBase {
-    /// Family of bases applicable to this register.
-    type BaseFamily;
+// Provides a `IoLoc` impl that for a fixed offset.
+#[doc(hidden)]
+pub struct OffsetLoc<Base: ?Sized, T>(usize, PhantomData<(T, Base)>);
 
-    /// Returns the absolute location of this type when using `B` as its base.
-    #[inline(always)]
-    fn of<B: RegisterBase<Self::BaseFamily>>() -> RelativeRegisterLoc<Self, B>
-    where
-        Self: Register,
-    {
-        RelativeRegisterLoc::new()
-    }
-}
-
-/// Trait implemented by relative registers.
-pub trait RelativeRegister: Register + WithBase {}
-
-/// Location of a relative register.
-///
-/// This can either be an immediately accessible regular [`RelativeRegister`], or a
-/// [`RelativeRegisterArray`] that needs one additional resolution through
-/// [`RelativeRegisterLoc::at`].
-pub struct RelativeRegisterLoc<T: WithBase, B: ?Sized>(PhantomData<T>, PhantomData<B>);
-
-impl<T, B> RelativeRegisterLoc<T, B>
-where
-    T: Register + WithBase,
-    B: RegisterBase<T::BaseFamily> + ?Sized,
-{
-    /// Returns the location of a relative register or register array.
-    #[inline(always)]
-    // We do not implement `Default` so we can be const.
-    #[expect(clippy::new_without_default)]
-    pub const fn new() -> Self {
-        Self(PhantomData, PhantomData)
+impl<Base: ?Sized, T> OffsetLoc<Base, T> {
+    #[inline]
+    pub const fn new(offset: usize) -> Self {
+        Self(offset, PhantomData)
     }
 
-    // Returns the absolute offset of the relative register using base `B`.
-    //
-    // This is implemented as a private const method so it can be reused by the [`IoLoc`]
-    // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`].
     #[inline]
-    const fn offset(self) -> usize {
-        B::BASE + T::OFFSET
+    pub const fn const_offset(self) -> usize {
+        self.0
     }
 }
 
-impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B>
-where
-    T: RelativeRegister,
-    B: RegisterBase<T::BaseFamily> + ?Sized,
-{
-    type IoType = T::Storage;
-
+impl<Base: ?Sized, T> IoLoc<Base, T> for OffsetLoc<Base, T> {
     #[inline(always)]
     fn offset(self) -> usize {
-        RelativeRegisterLoc::offset(self)
+        self.0
     }
 }
 
 /// Trait implemented by arrays of registers.
-pub trait RegisterArray: Register {
+pub trait RegisterArray: Sized {
+    /// Base type for this register.
+    type Base: ?Sized;
+
+    /// Start offset of the register.
+    ///
+    /// The interpretation of this offset depends on the type of the register.
+    const OFFSET: usize;
     /// Number of elements in the registers array.
     const SIZE: usize;
     /// Number of bytes between the start of elements in the registers array.
@@ -285,12 +194,10 @@ impl<T: RegisterArray> RegisterArrayLoc<T> {
     }
 }
 
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T>
+impl<Base: ?Sized, T> IoLoc<Base, T> for RegisterArrayLoc<T>
 where
-    T: RegisterArray,
+    T: RegisterArray<Base = Base>,
 {
-    type IoType = T::Storage;
-
     #[inline(always)]
     fn offset(self) -> usize {
         T::OFFSET + self.0 * T::STRIDE
@@ -318,71 +225,15 @@ pub trait Array {
     }
 }
 
-/// Trait implemented by arrays of relative registers.
-pub trait RelativeRegisterArray: RegisterArray + WithBase {}
-
-/// Location of a relative array register.
-pub struct RelativeRegisterArrayLoc<
-    T: RelativeRegisterArray,
-    B: RegisterBase<T::BaseFamily> + ?Sized,
->(RelativeRegisterLoc<T, B>, usize);
-
-impl<T, B> RelativeRegisterArrayLoc<T, B>
-where
-    T: RelativeRegisterArray,
-    B: RegisterBase<T::BaseFamily> + ?Sized,
-{
-    /// Returns the location of register `T` from the base `B` at index `idx`, with build-time
-    /// validation.
-    #[inline(always)]
-    pub fn new(idx: usize) -> Self {
-        build_assert!(idx < T::SIZE);
-
-        Self(RelativeRegisterLoc::new(), idx)
-    }
-
-    /// Attempts to return the location of register `T` from the base `B` at index `idx`, with
-    /// runtime validation.
-    #[inline(always)]
-    pub fn try_new(idx: usize) -> Option<Self> {
-        if idx < T::SIZE {
-            Some(Self(RelativeRegisterLoc::new(), idx))
-        } else {
-            None
-        }
-    }
-}
-
-/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`].
-impl<T, B> RelativeRegisterLoc<T, B>
-where
-    T: RelativeRegisterArray,
-    B: RegisterBase<T::BaseFamily> + ?Sized,
-{
-    /// Returns the location of the register at position `idx`, with build-time validation.
-    #[inline(always)]
-    pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc<T, B> {
-        RelativeRegisterArrayLoc::new(idx)
-    }
-
-    /// Returns the location of the register at position `idx`, with runtime validation.
-    #[inline(always)]
-    pub fn try_at(self, idx: usize) -> Option<RelativeRegisterArrayLoc<T, B>> {
-        RelativeRegisterArrayLoc::try_new(idx)
-    }
-}
-
-impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B>
-where
-    T: RelativeRegisterArray,
-    B: RegisterBase<T::BaseFamily> + ?Sized,
-{
-    type IoType = T::Storage;
+/// Trait implemented by types that indicate there is a fixed I/O location for this given type.
+///
+/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
+pub trait FixedIoLoc<Base: ?Sized>: Sized {
+    /// Type of [`FixedIoLoc::LOCATION`].
+    type Location: IoLoc<Base, Self>;
 
-    #[inline(always)]
-    fn offset(self) -> usize {
-        self.0.offset() + self.1 * T::STRIDE
-    }
+    /// Location of this type within given base.
+    const LOCATION: Self::Location;
 }
 
 /// Trait implemented by items that contain both a register value and the absolute I/O location at
@@ -390,8 +241,8 @@ where
 ///
 /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
 pub trait LocatedRegister<Base: ?Sized> {
-    /// Register value to write.
-    type Value: Register;
+    /// Value to write.
+    type Value;
     /// Full location information at which to write the value.
     type Location: IoLoc<Base, Self::Value>;
 
@@ -400,27 +251,38 @@ pub trait LocatedRegister<Base: ?Sized> {
     fn into_io_op(self) -> (Self::Location, Self::Value);
 }
 
-impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T
+impl<Base: ?Sized, T> LocatedRegister<Base> for T
 where
-    T: FixedRegister,
+    T: FixedIoLoc<Base>,
 {
-    type Location = FixedRegisterLoc<Self::Value>;
+    type Location = T::Location;
     type Value = T;
 
     #[inline(always)]
-    fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
-        (FixedRegisterLoc::new(), self)
+    fn into_io_op(self) -> (T::Location, T) {
+        (T::LOCATION, self)
     }
 }
 
+/// Helper function for register element alias implementation.
+///
+/// This is used to enforce base matching and provide bounds checking.
+#[doc(hidden)]
+#[inline(always)] // for const eval only
+pub const fn element_alias_offset<Base: ?Sized, Alias: RegisterArray<Base = Base>>(
+    idx: usize,
+) -> usize {
+    assert!(idx < Alias::SIZE);
+    Alias::OFFSET + idx * Alias::STRIDE
+}
+
 /// Defines a dedicated type for a register, including getter and setter methods for its fields and
 /// methods to read and write it from an [`Io`](kernel::io::Io) region.
 ///
 /// This documentation focuses on how to declare registers. See the [module-level
 /// documentation](mod@kernel::io::register) for examples of how to access them.
 ///
-/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of
-/// registers, and relative arrays of registers.
+/// Registers can either be fixed offset registers or arrays of registers.
 ///
 /// ## Fixed offset registers
 ///
@@ -444,11 +306,14 @@ where
 ///     io::{
 ///         register,
 ///         Io,
+///         Region,
 ///     },
 /// };
-/// # use kernel::io::{Mmio, Region};
+/// # use kernel::io::Mmio;
 ///
 /// register! {
+///     base: Region<0x1000>;
+///
 ///     FIXED_REG(u32) @ 0x100 {
 ///         15:8 high_byte;
 ///         7:0  low_byte;
@@ -479,9 +344,14 @@ where
 /// the context:
 ///
 /// ```no_run
-/// use kernel::io::register;
+/// use kernel::io::{
+///     register,
+///     Region,
+/// };
 ///
 /// register! {
+///     base: Region<0x1000>;
+///
 ///     /// Scratch register.
 ///     pub SCRATCH(u32) @ 0x00000200 {
 ///         31:0 value;
@@ -497,113 +367,45 @@ where
 /// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing
 /// its own `completed` field.
 ///
-/// ## Relative registers
-///
-/// Relative registers can be instantiated several times at a relative offset of a group of bases.
-/// For instance, imagine the following I/O space:
+/// If you do not wish to have a bitfield defined, you can also create a register using an existing
+/// type.
 ///
-/// ```text
-///           +-----------------------------+
-///           |             ...             |
-///           |                             |
-///  0x100--->+------------CPU0-------------+
-///           |                             |
-///  0x110--->+-----------------------------+
-///           |           CPU_CTL           |
-///           +-----------------------------+
-///           |             ...             |
-///           |                             |
-///           |                             |
-///  0x200--->+------------CPU1-------------+
-///           |                             |
-///  0x210--->+-----------------------------+
-///           |           CPU_CTL           |
-///           +-----------------------------+
-///           |             ...             |
-///           +-----------------------------+
-/// ```
-///
-/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O
-/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define
-/// them twice and would prefer a way to select which one to use from a single definition.
-///
-/// This can be done using the `Base + Offset` syntax when specifying the register's address:
-///
-/// ```ignore
+/// ```no_run
+/// # use kernel::io::*;
 /// register! {
-///     pub RELATIVE_REG(u32) @ Base + 0x80 {
-///         ...
-///     }
+///     base: Region<0x1000>;
+///
+///     /// UART RX register.
+///     pub UART_RX: u8 @ 0x100;
 /// }
 /// ```
 ///
-/// This creates a register with an offset of `0x80` from a given base.
+/// In case there is a fixed register associated with a specific type in the base, you can apply
+/// `#[unique]` attribute which enables `write_reg` shorthand. This is automatically applied to
+/// bitfields instantiated via the `register!` macro.
 ///
-/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the
-/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for
-/// this register needs to implement `RegisterBase<Base>`.
-///
-/// The location of relative registers can be built using the [`WithBase::of`] method to specify
-/// its base. All relative registers implement [`WithBase`].
-///
-/// Here is the above layout translated into code:
+/// This should only be used when types meaningfully represent a register. For example, in the
+/// previous `UART_RX` example, even if only a single register is defined with `u8` type, it is a
+/// bad idea to annotate it with `#[unique]`.
 ///
 /// ```no_run
-/// use kernel::{
-///     io::{
-///         register,
-///         register::{
-///             RegisterBase,
-///             WithBase,
-///         },
-///         Io,
-///     },
-/// };
-/// # use kernel::io::{Mmio, Region};
-///
-/// // Type used to identify the base.
-/// pub struct CpuCtlBase;
-///
-/// // ZST describing `CPU0`.
-/// struct Cpu0;
-/// impl RegisterBase<CpuCtlBase> for Cpu0 {
-///     const BASE: usize = 0x100;
-/// }
-///
-/// // ZST describing `CPU1`.
-/// struct Cpu1;
-/// impl RegisterBase<CpuCtlBase> for Cpu1 {
-///     const BASE: usize = 0x200;
-/// }
+/// # use kernel::{bitfield, io::*};
 ///
-/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase<CpuCtlBase>`.
-/// register! {
-///     /// CPU core control.
-///     pub CPU_CTL(u32) @ CpuCtlBase + 0x10 {
-///         0:0 start;
+/// bitfield! {
+///     pub struct Reset(u32) {
+///         0:0 reset;
 ///     }
 /// }
 ///
-/// # fn test(io: Mmio<'_, Region<0x1000>>) {
-/// // Read the status of `Cpu0`.
-/// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>());
-///
-/// // Stop `Cpu0`.
-/// io.write(WithBase::of::<Cpu0>(), CPU_CTL::zeroed());
-/// # }
-///
-/// // Aliases can also be defined for relative register.
 /// register! {
-///     /// Alias to CPU core control.
-///     pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL {
-///         /// Start the aliased CPU core.
-///         1:1 alias_start;
-///     }
+///     base: Region<0x1000>;
+///
+///     pub RESET: #[unique] Reset @ 0x100;
 /// }
 ///
-/// # fn test2(io: Mmio<'_, Region<0x1000>>) {
-/// // Start the aliased `CPU0`, leaving its other fields untouched.
-/// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true));
+/// # fn test(mmio: Mmio<'_, Region<0x1000>>) {
+/// // let mmio: Mmio<'_, Region<0x1000>>;
+/// mmio.write_reg(Reset::zeroed().with_const_reset::<1>());
 /// # }
 /// ```
 ///
@@ -636,15 +438,18 @@ where
 ///         register,
 ///         register::Array,
 ///         Io,
+///         Region,
 ///     },
 /// };
-/// # use kernel::io::{Mmio, Region};
+/// # use kernel::io::Mmio;
 /// # fn get_scratch_idx() -> usize {
 /// #   0x15
 /// # }
 ///
 /// // Array of 64 consecutive registers with the same layout starting at offset `0x80`.
 /// register! {
+///     base: Region<0x1000>;
+///
 ///     /// Scratch registers.
 ///     pub SCRATCH(u32)[64] @ 0x00000080 {
 ///         31:0 value;
@@ -670,6 +475,8 @@ where
 /// // Alias to a specific register in an array.
 /// // Here `SCRATCH[8]` is used to convey the firmware exit code.
 /// register! {
+///     base: Region<0x1000>;
+///
 ///     /// Firmware exit status code.
 ///     pub FIRMWARE_STATUS(u32) => SCRATCH[8] {
 ///         7:0 status;
@@ -682,6 +489,8 @@ where
 /// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the
 /// // registers of the two declarations below are interleaved.
 /// register! {
+///     base: Region<0x1000>;
+///
 ///     /// Scratch registers bank 0.
 ///     pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 {
 ///         31:0 value;
@@ -696,332 +505,88 @@ where
 /// # }
 /// ```
 ///
-/// ## Relative arrays of registers
+/// ## Relative registers
 ///
-/// Combining the two features described in the sections above, arrays of registers accessible from
-/// a base can also be defined:
+/// There are cases where a register region is subdivided into small subregions, and you may wish to
+/// have your register definition be relative to these subregions. This may be needed, for example,
+/// if these subregions are instantiated several times, or you just want it for encapsulation
+/// purpose.
 ///
-/// ```ignore
-/// register! {
-///     pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 {
-///         ...
-///     }
-/// }
+/// For instance, imagine the following I/O space:
+///
+/// ```text
+///           +-----------------------------+
+///           |             ...             |
+///           |                             |
+///  0x100--->+------------CPU0-------------+
+///           |                             |
+///  0x110--->+-----------------------------+
+///           |           CPU_CTL           |
+///           +-----------------------------+
+///           |             ...             |
+///           |                             |
+///           |                             |
+///  0x200--->+------------CPU1-------------+
+///           |                             |
+///  0x210--->+-----------------------------+
+///           |           CPU_CTL           |
+///           +-----------------------------+
+///           |             ...             |
+///           +-----------------------------+
 /// ```
 ///
-/// Like relative registers, they implement the [`WithBase`] trait. However the return value of
-/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the
-/// [`at`](RelativeRegisterLoc::at) method.
+/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O
+/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define
+/// them twice and would prefer a way to select which one to use from a single definition.
+///
+/// This can be done by defining a new type for the subregion, and then defining registers that use
+/// the new type as the base:
 ///
 /// ```no_run
 /// use kernel::{
 ///     io::{
+///         io_project,
 ///         register,
-///         register::{
-///             RegisterBase,
-///             WithBase,
-///         },
 ///         Io,
+///         Region,
 ///     },
 /// };
-/// # use kernel::io::{Mmio, Region};
-/// # fn get_scratch_idx() -> usize {
-/// #   0x15
-/// # }
+/// # use kernel::io::Mmio;
 ///
-/// // Type used as parameter of `RegisterBase` to specify the base.
-/// pub struct CpuCtlBase;
-///
-/// // ZST describing `CPU0`.
-/// struct Cpu0;
-/// impl RegisterBase<CpuCtlBase> for Cpu0 {
-///     const BASE: usize = 0x100;
-/// }
-///
-/// // ZST describing `CPU1`.
-/// struct Cpu1;
-/// impl RegisterBase<CpuCtlBase> for Cpu1 {
-///     const BASE: usize = 0x200;
-/// }
+/// // Subregion type. Make sure it has adequate size and alignment.
+/// #[repr(align(4))]
+/// #[derive(FromBytes, IntoBytes)]
+/// pub struct CpuCtl([u8; 0x100]);
 ///
-/// // 64 per-cpu scratch registers, arranged as a contiguous array.
 /// register! {
-///     /// Per-CPU scratch registers.
-///     pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 {
-///         31:0 value;
-///     }
-/// }
-///
-/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
-/// // Read scratch register 0 of CPU0.
-/// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0));
+///     base: Region<0x1000>;
 ///
-/// // Write the retrieved value into scratch register 15 of CPU1.
-/// io.write(WithBase::of::<Cpu1>().at(15), scratch);
-///
-/// // This won't build.
-/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::<Cpu0>().at(128)).value();
-///
-/// // Runtime-obtained array index.
-/// let scratch_idx = get_scratch_idx();
-/// // Access on a runtime index returns an error if it is out-of-bounds.
-/// let cpu0_scratch = io.read(
-///     CPU_SCRATCH::of::<Cpu0>().try_at(scratch_idx).ok_or(EINVAL)?
-/// ).value();
-/// # Ok(())
-/// # }
-///
-/// // Alias to `SCRATCH[8]` used to convey the firmware exit code.
-/// register! {
-///     /// Per-CPU firmware exit status code.
-///     pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] {
-///         7:0 status;
-///     }
+///     // Subregions can just be defined like normal registers.
+///     CPU0: CpuCtl @ 0x100;
+///     CPU1: CpuCtl @ 0x200;
 /// }
 ///
-/// // Non-contiguous relative register arrays can be defined by adding a stride parameter.
-/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the
-/// // registers of the two declarations below are interleaved.
+/// // Then you can define new registers on the subregion.
 /// register! {
-///     /// Scratch registers bank 0.
-///     pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 {
-///         31:0 value;
-///     }
+///     base: CpuCtl;
 ///
-///     /// Scratch registers bank 1.
-///     pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 {
-///         31:0 value;
+///     /// CPU core control.
+///     pub CPU_CTL(u32) @ 0x10 {
+///         0:0 start;
 ///     }
 /// }
 ///
-/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
-/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status();
-/// # Ok(())
+/// # fn test(io: Mmio<'_, Region<0x1000>>) {
+/// // Read the status of `Cpu0`.
+/// let cpu0_started = io_project!(io, build: CPU0).read(CPU_CTL);
+///
+/// // Stop `Cpu0`.
+/// io_project!(io, build: CPU0).write_reg(CPU_CTL::zeroed());
 /// # }
 /// ```
 #[macro_export]
 macro_rules! register {
-    // Entry point for the macro, allowing multiple registers to be defined in one call.
-    // It matches all possible register declaration patterns to dispatch them to corresponding
-    // `@reg` rule that defines a single register.
-    //
-    // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O
-    // projections.
-    (
-        $(
-            $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
-                $([ $size:expr $(, stride = $stride:expr)? ])?
-                $(@ $($base:ident +)? $offset:literal)?
-                $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )?
-            { $($fields:tt)* }
-        )*
-    ) => {
-        $(
-        $crate::register!(
-            @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])?
-                $(@ $($base +)? $offset)?
-                $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )?
-            { $($fields)* }
-        );
-        )*
-    };
-
-    // All the rules below are private helpers.
-
-    // Creates a register at a fixed offset of the MMIO space.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal
-            { $($fields:tt)* }
-    ) => {
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(@io_base $name($storage) @ $offset);
-        $crate::register!(@io_fixed $(#[$attr])* $vis $name);
-    };
-
-    // Creates an alias register of fixed offset register `alias` with its own fields.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path
-            { $($fields:tt)* }
-    ) => {
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(
-            @io_base $name($storage) @
-            <$alias as $crate::io::register::Register>::OFFSET
-        );
-        $crate::register!(@io_fixed $(#[$attr])* $vis $name);
-    };
-
-    // Creates a register at a relative offset from a base address provider.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal
-            { $($fields:tt)* }
-    ) => {
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(@io_base $name($storage) @ $offset);
-        $crate::register!(@io_relative $name @ $base);
-    };
-
-    // Creates an alias register of relative offset register `alias` with its own fields.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident
-            { $($fields:tt)* }
-    ) => {
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(
-            @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET
-        );
-        $crate::register!(@io_relative $name @ $base);
-    };
-
-    // Creates an array of registers at a fixed offset of the MMIO space.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
-            [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* }
-    ) => {
-        $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride);
-
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(@io_base $name($storage) @ $offset);
-        $crate::register!(@io_array $name [ $size, stride = $stride ]);
-    };
-
-    // Shortcut for contiguous array of registers (stride == size of element).
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal
-            { $($fields:tt)* }
-    ) => {
-        $crate::register!(
-            @reg $(#[$attr])* $vis $name($storage)
-                [ $size, stride = ::core::mem::size_of::<$storage>() ]
-                @ $offset { $($fields)* }
-        );
-    };
-
-    // Creates an alias of register `idx` of array of registers `alias` with its own fields.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ]
-            { $($fields:tt)* }
-    ) => {
-        $crate::build_assert::static_assert!(
-            $idx < <$alias as $crate::io::register::RegisterArray>::SIZE
-        );
-
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(
-            @io_base $name($storage) @
-            <$alias as $crate::io::register::Register>::OFFSET
-                + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE
-        );
-        $crate::register!(@io_fixed $(#[$attr])* $vis $name);
-    };
-
-    // Creates an array of registers at a relative offset from a base address provider.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
-            [ $size:expr, stride = $stride:expr ]
-            @ $base:ident + $offset:literal { $($fields:tt)* }
-    ) => {
-        $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride);
-
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(@io_base $name($storage) @ $offset);
-        $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base);
-    };
-
-    // Shortcut for contiguous array of relative registers (stride == size of element).
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ]
-            @ $base:ident + $offset:literal { $($fields:tt)* }
-    ) => {
-        $crate::register!(
-            @reg $(#[$attr])* $vis $name($storage)
-                [ $size, stride = ::core::mem::size_of::<$storage>() ]
-                @ $base + $offset { $($fields)* }
-        );
-    };
-
-    // Creates an alias of register `idx` of relative array of registers `alias` with its own
-    // fields.
-    (
-        @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
-            => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* }
-    ) => {
-        $crate::build_assert::static_assert!(
-            $idx < <$alias as $crate::io::register::RegisterArray>::SIZE
-        );
-
-        $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
-        $crate::register!(
-            @io_base $name($storage) @
-                <$alias as $crate::io::register::Register>::OFFSET +
-                $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE
-        );
-        $crate::register!(@io_relative $name @ $base);
-    };
-
-    // Generates the bitfield for the register.
-    //
-    // `#[allow(non_camel_case_types)]` is added since register names typically use
-    // `SCREAMING_CASE`.
-    (
-        @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* }
-    ) => {
-        $crate::bitfield!(
-            #[allow(non_camel_case_types)]
-            $(#[$attr])* $vis struct $name($storage) { $($fields)* }
-        );
-    };
-
-    // Implementations shared by all registers types.
-    (@io_base $name:ident($storage:ty) @ $offset:expr) => {
-        impl $crate::io::register::Register for $name {
-            type Storage = $storage;
-
-            const OFFSET: usize = $offset;
-        }
-    };
-
-    // Implementations of fixed registers.
-    (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => {
-        impl $crate::io::register::FixedRegister for $name {}
-
-        $(#[$attr])*
-        $vis const $name: $crate::io::register::FixedRegisterLoc<$name> =
-            $crate::io::register::FixedRegisterLoc::<$name>::new();
-    };
-
-    // Implementations of relative registers.
-    (@io_relative $name:ident @ $base:ident) => {
-        impl $crate::io::register::WithBase for $name {
-            type BaseFamily = $base;
-        }
-
-        impl $crate::io::register::RelativeRegister for $name {}
-    };
-
-    // Implementations of register arrays.
-    (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => {
-        impl $crate::io::register::Array for $name {}
-
-        impl $crate::io::register::RegisterArray for $name {
-            const SIZE: usize = $size;
-            const STRIDE: usize = $stride;
-        }
-    };
-
-    // Implementations of relative array registers.
-    (
-        @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident
-    ) => {
-        impl $crate::io::register::WithBase for $name {
-            type BaseFamily = $base;
-        }
-
-        impl $crate::io::register::RegisterArray for $name {
-            const SIZE: usize = $size;
-            const STRIDE: usize = $stride;
-        }
-
-        impl $crate::io::register::RelativeRegisterArray for $name {}
+    ($($tt:tt)*) => {
+        $crate::macros::register!($($tt)*);
     };
 }
diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs
index 17b0c174cfc559..0d3b34f8333468 100644
--- a/rust/kernel/io/resource.rs
+++ b/rust/kernel/io/resource.rs
@@ -226,10 +226,18 @@ impl Flags {
     /// Resource represents a memory region that must be ioremaped using `ioremap_np`.
     pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED);
 
+    /// Memory region uses a 64-bit address (consumes two consecutive PCI resource slots).
+    pub const IORESOURCE_MEM_64: Flags = Flags::new(bindings::IORESOURCE_MEM_64);
+
     // Always inline to optimize out error path of `build_assert`.
     #[inline(always)]
     const fn new(value: u32) -> Self {
         build_assert!(value as u64 <= c_ulong::MAX as u64);
         Flags(value as c_ulong)
     }
+
+    /// Wrap a raw `c_ulong` value returned by a C API into [`Flags`].
+    pub(crate) const fn from_raw(value: c_ulong) -> Self {
+        Flags(value)
+    }
 }
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 4d5c96ddc49c79..f9ef36217bb5d9 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -67,6 +67,8 @@ pub mod device;
 pub mod device_id;
 pub mod devres;
 pub mod dma;
+#[cfg(CONFIG_DMA_SHARED_BUFFER)]
+pub mod dma_buf;
 pub mod driver;
 #[cfg(CONFIG_DRM = "y")]
 pub mod drm;
@@ -98,6 +100,7 @@ pub mod jump_label;
 pub mod kunit;
 pub mod list;
 pub mod maple_tree;
+pub mod mem;
 pub mod miscdevice;
 pub mod mm;
 pub mod module;
diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs
index 265d6396a78a17..7abe41228cb702 100644
--- a/rust/kernel/maple_tree.rs
+++ b/rust/kernel/maple_tree.rs
@@ -16,7 +16,11 @@ use kernel::{
     alloc::Flags,
     error::to_result,
     prelude::*,
-    types::{ForeignOwnable, Opaque},
+    types::{
+        ForeignOwnable,
+        NotThreadSafe,
+        Opaque, //
+    },
 };
 
 /// A maple tree optimized for storing non-overlapping ranges.
@@ -240,7 +244,10 @@ impl<T: ForeignOwnable> MapleTree<T> {
         unsafe { bindings::spin_lock(self.ma_lock()) };
 
         // INVARIANT: We just took the spinlock.
-        MapleGuard(self)
+        MapleGuard {
+            tree: self,
+            _not_send: NotThreadSafe,
+        }
     }
 
     #[inline]
@@ -302,19 +309,30 @@ impl<T: ForeignOwnable> PinnedDrop for MapleTree<T> {
     }
 }
 
+// SAFETY: `MapleTree<T>` is `Send` if `T` is `Send` because `MapleTree` owns its elements.
+unsafe impl<T: ForeignOwnable + Send> Send for MapleTree<T> {}
+
+// SAFETY: `&MapleTree<T>` allows inserting and erasing entries from any thread, so `T: Send` is
+// required, and shared borrows of entries require `T: Sync`.
+unsafe impl<T: ForeignOwnable + Send + Sync> Sync for MapleTree<T> {}
+
 /// A reference to a [`MapleTree`] that owns the inner lock.
 ///
 /// # Invariants
 ///
 /// This guard owns the inner spinlock.
 #[must_use = "if unused, the lock will be immediately unlocked"]
-pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree<T>);
+pub struct MapleGuard<'tree, T: ForeignOwnable> {
+    tree: &'tree MapleTree<T>,
+    // A held spinlock must be released on the same CPU that acquired it.
+    _not_send: NotThreadSafe,
+}
 
 impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> {
     #[inline]
     fn drop(&mut self) {
         // SAFETY: By the type invariants, we hold this spinlock.
-        unsafe { bindings::spin_unlock(self.0.ma_lock()) };
+        unsafe { bindings::spin_unlock(self.tree.ma_lock()) };
     }
 }
 
@@ -323,7 +341,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> {
     pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> {
         // SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s
         // read/write permissions to the maple tree.
-        unsafe { MaState::new_raw(self.0, first, end) }
+        unsafe { MaState::new_raw(self.tree, first, end) }
     }
 
     /// Load the value at the given index.
@@ -375,7 +393,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> {
     #[inline]
     pub fn load(&mut self, index: usize) -> Option<T::BorrowedMut<'_>> {
         // SAFETY: `self.tree` contains a valid maple tree.
-        let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) };
+        let ret = unsafe { bindings::mtree_load(self.tree.tree.get(), index) };
         if ret.is_null() {
             return None;
         }
diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs
new file mode 100644
index 00000000000000..f2d4cdf87d00b3
--- /dev/null
+++ b/rust/kernel/mem.rs
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Basic utilities for dealing with memory, values, and types.
+
+use crate::prelude::*;
+
+/// Transmute between two types.
+///
+/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this
+/// cannot be proven by the compiler.
+///
+/// This is equivalent to Rust's `transmute_unchecked` intrinsics.
+///
+/// # Safety
+///
+/// All safety requirements of [`core::mem::transmute`] apply, plus that the size `Src` and `Dst`
+/// must match.
+///
+/// # Examples
+///
+/// This can be used when types are known to have the same size, but only at runtime.
+///
+/// ```no_run
+/// # use core::any::TypeId;
+/// fn to_u32<T: 'static>(v: T) -> Option<u32> {
+///     if TypeId::of::<T>() != TypeId::of::<u32>() {
+///         return None;
+///     }
+///
+///     // `core::mem::transmute` won't work here.
+///     // SAFETY: We've checked that `T` is `u32`!
+///     Some(unsafe { kernel::mem::transmute_unchecked(v) })
+/// }
+///
+/// to_u32(1u32);
+/// ```
+#[inline(always)]
+pub const unsafe fn transmute_unchecked<Src, Dst>(val: Src) -> Dst {
+    // SAFETY: This is identical to `transmute` except that we bypassed the size check; which is
+    // true per safety requirement.
+    unsafe { core::mem::transmute_copy(&core::mem::ManuallyDrop::new(val)) }
+}
+
+/// Version of `transmute` that performs size check at monomorphization-time.
+///
+/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this
+/// cannot be proven by the compiler during type checking and can be proven during monomorphization.
+///
+/// The signature is equivalent to Rust standard library's unstable `transmute_neo` and that of
+/// [RFC 3844](https://github.com/rust-lang/rfcs/pull/3844).
+///
+/// # Safety
+///
+/// Same as [`core::mem::transmute`].
+///
+/// # Examples
+///
+/// This is typically used in generic code where it's known that type will have the same size, but
+/// the compiler cannot prove it generically.
+///
+/// ```no_run
+/// trait IsU32 {}
+/// impl IsU32 for u32 {}
+///
+/// fn to_u32<T: IsU32>(v: T) -> u32 {
+///     // `core::mem::transmute` won't work here.
+///     // SAFETY: We know that `v` is u32!
+///     unsafe { kernel::mem::transmute(v) }
+/// }
+///
+/// to_u32(1u32);
+/// ```
+#[inline(always)]
+pub const unsafe fn transmute<Src, Dst>(val: Src) -> Dst {
+    const_assert!(size_of::<Src>() == size_of::<Dst>());
+
+    // SAFETY: Size is checked above. Other safety requirements follow those of the function.
+    unsafe { transmute_unchecked(val) }
+}
+
+/// Safely transmutes a value of one type to a value of another type of the same size.
+///
+/// The sizes are checked during monomorphization.
+///
+/// This can be considered as generic version of [`zerocopy::transmute!`] macro that defers the size
+/// check and thus can be used in more cases.
+///
+/// # Examples
+///
+/// ```no_run
+/// fn to_u32<T: FromBytes + IntoBytes>(v: T) -> u32 {
+///     // `zerocopy::transmute!` won't work here.
+///     kernel::mem::safe_transmute(v)
+/// }
+///
+/// to_u32(1i32);
+/// ```
+#[inline(always)]
+pub const fn safe_transmute<Src: IntoBytes, Dst: FromBytes>(val: Src) -> Dst {
+    // SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds.
+    unsafe { transmute(val) }
+}
+
+/// Type that is layout-compatible with a primitive representation.
+///
+/// # Safety
+///
+/// - [`Self`] must have the same size and alignment as [`Self::Repr`].
+/// - [`Self`] must be [transmutable] to [`Self::Repr`].
+/// - Neither [`Self`] nor [`Self::Repr`] contains interior mutability.
+///
+/// The above basically says that `&Self` can be transmuted to `&Self::Repr`.
+///
+/// [transmutable]: core::mem::transmute
+pub unsafe trait AsRepr: Sized {
+    /// Primitive representation of this type.
+    type Repr;
+
+    /// Convert from [`&Self`](Self) to [`&Self::Repr`](AsRepr::Repr).
+    #[inline(always)]
+    fn as_repr(this: &Self) -> &Self::Repr {
+        // SAFETY: Per safety requirement of the trait.
+        unsafe { core::mem::transmute(this) }
+    }
+
+    /// Convert from [`Self`] to [`Self::Repr`].
+    #[inline(always)]
+    fn into_repr(this: Self) -> Self::Repr {
+        // SAFETY: Per safety requirement of the trait.
+        unsafe { transmute(this) }
+    }
+
+    /// Convert from [`Self::Repr`] to [`Self`].
+    ///
+    /// # Safety
+    ///
+    /// `repr` must be a valid bit pattern of [`Self`] and satisfy type-specific invariants of it.
+    ///
+    /// Alternatively, if `repr` is previously obtained using [`Self::into_repr`], and each
+    /// `from_repr_unchecked` should correspond to a unique `into_repr` call, then it is safe to
+    /// call as well (this means that we're undoing a `into_repr` call getting the exact bytes
+    /// back).
+    ///
+    /// No guarantee is made if the result of a `into_repr` is passed to multiple
+    /// `from_repr_unchecked` (i.e. copies are made), to allow for cases where `Repr` is a pointer
+    /// and the user of the API wants ownership transfer. Users that want the ability to call
+    /// `from_repr_unchecked` after copying can require `Copy` bound explicitly.
+    #[inline(always)]
+    unsafe fn from_repr_unchecked(repr: Self::Repr) -> Self {
+        // SAFETY: Per safety requirement, `repr` is valid repr of `Self`, or it is previously from
+        // `into_repr`, in which case we're undoing the transmute so it is also safe.
+        unsafe { transmute(repr) }
+    }
+}
+
+/// Type that is bi-directionally transmutable with a primitive representation.
+///
+/// # Safety
+///
+/// - [`Self`] must be [transmutable] from [`Self::Repr`].
+///
+/// [transmutable]: core::mem::transmute
+/// [`Self::Repr`]: AsRepr::Repr
+pub unsafe trait AsReprMut: AsRepr {
+    /// Convert from `&mut Self` to [`&mut Self::Repr`](AsRepr::Repr).
+    #[inline(always)]
+    fn as_repr_mut(this: &mut Self) -> &mut Self::Repr {
+        // SAFETY: Per safety requirement of the trait.
+        unsafe { core::mem::transmute(this) }
+    }
+
+    /// Convert from [`Self::Repr`](AsRepr::Repr) to `Self`.
+    #[inline(always)]
+    fn from_repr(repr: Self::Repr) -> Self {
+        // SAFETY: Per safety requirement of the trait.
+        unsafe { transmute(repr) }
+    }
+}
+
+// SAFETY: `bool` has the same size and alignment as `u8`, and Rust guarantees that `bool` has
+// only two valid bit patterns: 0 (`false`) and 1 (`true`). Thus `bool` can be transmuted to `u8`.
+// Neither types contain interior mutability.
+unsafe impl AsRepr for bool {
+    type Repr = u8;
+}
+
+// SAFETY: `*mut T` has the same size and alignment with `*const c_void`, and thus `*mut T` is
+// transmutable to `*const c_void`. Neither types contain interior mutability.
+unsafe impl<T> AsRepr for *mut T {
+    type Repr = *const c_void;
+}
+
+// SAFETY: `*mut T` is transmutable from `*const c_void`.
+unsafe impl<T> AsReprMut for *mut T {}
+
+// SAFETY: `*const T` has the same size and alignment with `*const c_void`, and is transmutable to
+// `*const c_void`. Neither types contain interior mutability.
+unsafe impl<T> AsRepr for *const T {
+    type Repr = *const c_void;
+}
+
+// SAFETY: `*const T` is transmutable from `*const c_void`.
+unsafe impl<T> AsReprMut for *const T {}
+
+macro_rules! int_impl {
+    ($($unsigned:ident $signed:ident ,)*) => {$(
+        // SAFETY: `$unsigned` has the same size and alignment with itself, and is transmutable to
+        // itself. It does not contain interior mutability.
+        unsafe impl AsRepr for $unsigned {
+            type Repr = $unsigned;
+        }
+
+        // SAFETY: `$unsigned` is transmutable from itself.
+        unsafe impl AsReprMut for $unsigned {}
+
+        // SAFETY: `$signed` has the same size and alignment with `$unsigned`, and is transmutable
+        // to it Neither types contain interior mutability.
+        unsafe impl AsRepr for $signed {
+            type Repr = $unsigned;
+        }
+
+        // SAFETY: `$signed` is transmutable from `$unsigned`.
+        unsafe impl AsReprMut for $signed {}
+    )*};
+}
+
+int_impl! {
+    u8 i8,
+    u16 i16,
+    u32 i32,
+    u64 i64,
+    // `usize` is not normalized to particular integer for portability.
+    usize isize,
+}
diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs
index 4764d7b68f2a7f..58bc1793fdaf5e 100644
--- a/rust/kernel/mm.rs
+++ b/rust/kernel/mm.rs
@@ -170,30 +170,48 @@ impl MmWithUser {
     ///
     /// This is an optimistic trylock operation, so it may fail if there is contention. In that
     /// case, you should fall back to taking the mmap read lock.
-    ///
-    /// When per-vma locks are disabled, this always returns `None`.
     #[inline]
     pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option<VmaReadGuard<'_>> {
-        #[cfg(CONFIG_PER_VMA_LOCK)]
-        {
-            // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where
-            // `mm_users` is non-zero.
-            let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) };
-            if !vma.is_null() {
-                return Some(VmaReadGuard {
-                    // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a
-                    // valid vma. The vma is stable for as long as the vma read lock is held.
-                    vma: unsafe { VmaRef::from_raw(vma) },
-                    _nts: NotThreadSafe,
-                });
-            }
+        // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where
+        // `mm_users` is non-zero.
+        let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) };
+        if vma.is_null() {
+            return None;
         }
+        Some(VmaReadGuard {
+            // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a
+            // valid vma. The vma is stable for as long as the vma read lock is held.
+            vma: unsafe { VmaRef::from_raw(vma) },
+            _nts: NotThreadSafe,
+        })
+    }
 
-        // Silence warnings about unused variables.
-        #[cfg(not(CONFIG_PER_VMA_LOCK))]
-        let _ = vma_addr;
-
-        None
+    /// Find the VMA covering 'address' and read-lock it.
+    ///
+    /// The fast path does not take mmap_lock. Waits for writers to finish if the
+    /// VMA is being modified by taking mmap_lock.
+    /// Use when mmap_lock is not held, otherwise use vma_start_read_locked().
+    /// Nothing prevents VMAs being unmapped/mapped before or after the VMA is
+    /// looked up, if a stronger guarantee is required, take an mmap_lock.
+    ///
+    /// Return: If a VMA exists which spans @address, return that VMA, read-locked.
+    /// If no VMA is mapped there or, very unlikely, a reference count overflow
+    /// occurred, return NULL.
+    #[inline]
+    pub fn vma_start_read_unlocked(&self, vma_addr: usize) -> Option<VmaReadGuard<'_>> {
+        // SAFETY: We may invoke `vma_start_read_unlocked` because we know this `mm` has non-zero
+        // `mm_users`.
+        let vma = unsafe { bindings::vma_start_read_unlocked(self.as_raw(), vma_addr) };
+        if vma.is_null() {
+            return None;
+        }
+        // INVARIANT: We just acquired the VMA read lock.
+        Some(VmaReadGuard {
+            // SAFETY: If `vma_start_read_unlocked` returns a non-null ptr, then it points at a
+            // valid vma. The vma is stable for as long as the vma read lock is held.
+            vma: unsafe { VmaRef::from_raw(vma) },
+            _nts: NotThreadSafe,
+        })
     }
 
     /// Lock the mmap read lock.
diff --git a/rust/kernel/net/netlink.rs b/rust/kernel/net/netlink.rs
index 22ef3dde36fa1a..36f2e39c3ab7b0 100644
--- a/rust/kernel/net/netlink.rs
+++ b/rust/kernel/net/netlink.rs
@@ -11,8 +11,8 @@
 use kernel::{
     alloc::{self, AllocError},
     error::to_result,
+    num::casts::u16_as_usize,
     prelude::*,
-    transmute::AsBytes,
     types::Opaque,
     ThisModule,
 };
@@ -22,6 +22,11 @@ use core::{
     ptr::NonNull, //
 };
 
+use zerocopy::{
+    Immutable,
+    IntoBytes, //
+};
+
 /// The default netlink message size.
 pub const GENLMSG_DEFAULT_SIZE: usize = bindings::GENLMSG_DEFAULT_SIZE;
 
@@ -84,11 +89,19 @@ impl GenlMsg {
     #[inline]
     fn put<T>(&mut self, attrtype: c_int, value: &T) -> Result
     where
-        T: ?Sized + AsBytes,
+        T: ?Sized + IntoBytes + Immutable,
     {
+        // `nla_len` is a 16-bit field that encodes the total attribute length
+        // (header + payload). Subtracting the header size from `u16::MAX` gives
+        // the largest payload that still fits within that field.
+        const MAX_PAYLOAD_LEN: usize = u16_as_usize(u16::MAX) - size_of::<bindings::nlattr>();
+
         let skb = self.skb.skb.as_ptr();
         let len = size_of_val(value);
         let ptr = core::ptr::from_ref(value).cast::<c_void>();
+        if len > MAX_PAYLOAD_LEN {
+            return Err(EMSGSIZE);
+        }
         // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and the provided value is
         // readable and initialized for its `size_of` bytes.
         to_result(unsafe { bindings::nla_put(skb, attrtype, len as c_int, ptr) })
diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs
index 3ec897709e890c..19a219847c17f5 100644
--- a/rust/kernel/pci.rs
+++ b/rust/kernel/pci.rs
@@ -17,6 +17,7 @@ use crate::{
         from_result,
         to_result, //
     },
+    io::resource,
     prelude::*,
     str::CStr,
     types::Opaque,
@@ -439,6 +440,19 @@ impl Device {
         Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) })
     }
 
+    /// Returns the resource flags (`IORESOURCE_*`) of the given PCI BAR.
+    pub fn resource_flags(&self, bar: u32) -> Result<resource::Flags> {
+        if !Bar::index_is_valid(bar) {
+            return Err(EINVAL);
+        }
+
+        // SAFETY:
+        // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`,
+        // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`.
+        let raw = unsafe { bindings::pci_resource_flags(self.as_raw(), bar.try_into()?) };
+        Ok(resource::Flags::from_raw(raw))
+    }
+
     /// Returns the PCI class as a `Class` struct.
     #[inline]
     pub fn pci_class(&self) -> Class {
diff --git a/rust/kernel/scatterlist.rs b/rust/kernel/scatterlist.rs
index b83c468b5c6331..44eeafed05e391 100644
--- a/rust/kernel/scatterlist.rs
+++ b/rust/kernel/scatterlist.rs
@@ -350,15 +350,23 @@ where
             page_vec.push(page.as_ptr(), flags)?;
         }
 
+        // Cap segments at both the DMA mapping-path limit and the device's declared
+        // max segment size.
+        //
         // `dma_max_mapping_size` returns `size_t`, but `sg_alloc_table_from_pages_segment()` takes
         // an `unsigned int`.
         //
         // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`.
-        let max_segment = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } {
+        let max_mapping_size = match unsafe { bindings::dma_max_mapping_size(dev.as_raw()) } {
             0 => u32::MAX,
-            max_segment => u32::try_from(max_segment).unwrap_or(u32::MAX),
+            max_mapping_size => u32::try_from(max_mapping_size).unwrap_or(u32::MAX),
         };
 
+        // SAFETY: `dev.as_raw()` is a valid pointer to a `struct device`.
+        let max_seg_size = unsafe { bindings::dma_get_max_seg_size(dev.as_raw()) };
+
+        let max_segment = max_mapping_size.min(max_seg_size);
+
         Ok(try_pin_init!(&this in Self {
             // SAFETY:
             // - `page_vec` is a `KVec` of valid `struct page *` obtained from `pages`.
diff --git a/rust/kernel/security.rs b/rust/kernel/security.rs
index 9d271695265fb4..4dc3eba6ce84d8 100644
--- a/rust/kernel/security.rs
+++ b/rust/kernel/security.rs
@@ -62,8 +62,7 @@ impl SecurityCtx {
     /// Get the security context given its id.
     #[inline]
     pub fn from_secid(secid: u32) -> Result<Self> {
-        // SAFETY: `struct lsm_context` can be initialized to all zeros.
-        let mut ctx: bindings::lsm_context = unsafe { core::mem::zeroed() };
+        let mut ctx: bindings::lsm_context = pin_init::zeroed();
 
         // SAFETY: Just a C FFI call. The pointer is valid for writes.
         to_result(unsafe { bindings::security_secid_to_secctx(secid, &mut ctx) })?;
diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs
index 9cd009d57e352d..6d27898add42c2 100644
--- a/rust/kernel/sync/atomic.rs
+++ b/rust/kernel/sync/atomic.rs
@@ -140,7 +140,7 @@ pub unsafe trait AtomicAdd<Rhs = Self>: AtomicType {
 const fn into_repr<T: AtomicType>(v: T) -> T::Repr {
     // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to
     // `T::Repr`, therefore the transmute operation is sound.
-    unsafe { core::mem::transmute_copy(&v) }
+    unsafe { crate::mem::transmute(v) }
 }
 
 /// # Safety
@@ -149,7 +149,7 @@ const fn into_repr<T: AtomicType>(v: T) -> T::Repr {
 #[inline(always)]
 const unsafe fn from_repr<T: AtomicType>(r: T::Repr) -> T {
     // SAFETY: Per the safety requirement of the function, the transmute operation is sound.
-    unsafe { core::mem::transmute_copy(&r) }
+    unsafe { crate::mem::transmute(r) }
 }
 
 impl<T: AtomicType> Atomic<T> {
diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs
index 6c0a5e8090d00a..5ae377e70dd880 100644
--- a/rust/kernel/time.rs
+++ b/rust/kernel/time.rs
@@ -39,6 +39,10 @@ pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64;
 /// The number of nanoseconds per second.
 pub const NSEC_PER_SEC: i64 = bindings::NSEC_PER_SEC as i64;
 
+/// The C side `MAX_JIFFY_OFFSET`, i.e. `((LONG_MAX >> 1) - 1)`. It is the upper
+/// bound the kernel uses for a jiffies span, not a wait-forever value.
+const MAX_JIFFY_OFFSET: isize = (isize::MAX >> 1) - 1;
+
 /// The time unit of Linux kernel. One jiffy equals (1/HZ) second.
 pub type Jiffies = crate::ffi::c_ulong;
 
@@ -53,9 +57,9 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies {
     unsafe { bindings::__msecs_to_jiffies(msecs) }
 }
 
-/// Trait for clock sources.
+/// Trait for kernel clock identifiers.
 ///
-/// Selection of the clock source depends on the use case. In some cases the usage of a
+/// Selection of the clock depends on the use case. In some cases the usage of a
 /// particular clock is mandatory, e.g. in network protocols, filesystems. In other
 /// cases the user of the clock has to decide which clock is best suited for the
 /// purpose. In most scenarios clock [`Monotonic`] is the best choice as it
@@ -66,13 +70,13 @@ pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies {
 /// Implementers must ensure that `ktime_get()` returns a value in the inclusive range
 /// `0..=KTIME_MAX` (i.e., greater than or equal to 0 and less than or equal to
 /// `KTIME_MAX`, where `KTIME_MAX` equals `i64::MAX`).
-pub unsafe trait ClockSource {
-    /// The kernel clock ID associated with this clock source.
+pub unsafe trait ClockId {
+    /// The kernel clock ID associated with this clock.
     ///
     /// This constant corresponds to the C side `clockid_t` value.
     const ID: bindings::clockid_t;
 
-    /// Get the current time from the clock source.
+    /// Get the current time from the clock.
     ///
     /// The function must return a value in the range `0..=KTIME_MAX`.
     fn ktime_get() -> bindings::ktime_t;
@@ -93,7 +97,7 @@ pub struct Monotonic;
 
 // SAFETY: The kernel's `ktime_get()` is guaranteed to return a value
 // in `0..=KTIME_MAX`.
-unsafe impl ClockSource for Monotonic {
+unsafe impl ClockId for Monotonic {
     const ID: bindings::clockid_t = bindings::CLOCK_MONOTONIC as bindings::clockid_t;
 
     fn ktime_get() -> bindings::ktime_t {
@@ -120,7 +124,7 @@ pub struct RealTime;
 
 // SAFETY: The kernel's `ktime_get_real()` is guaranteed to return a value
 // in `0..=KTIME_MAX`.
-unsafe impl ClockSource for RealTime {
+unsafe impl ClockId for RealTime {
     const ID: bindings::clockid_t = bindings::CLOCK_REALTIME as bindings::clockid_t;
 
     fn ktime_get() -> bindings::ktime_t {
@@ -140,7 +144,7 @@ pub struct BootTime;
 
 // SAFETY: The kernel's `ktime_get_boottime()` is guaranteed to return a value
 // in `0..=KTIME_MAX`.
-unsafe impl ClockSource for BootTime {
+unsafe impl ClockId for BootTime {
     const ID: bindings::clockid_t = bindings::CLOCK_BOOTTIME as bindings::clockid_t;
 
     fn ktime_get() -> bindings::ktime_t {
@@ -164,7 +168,7 @@ pub struct Tai;
 
 // SAFETY: The kernel's `ktime_get_clocktai()` is guaranteed to return a value
 // in `0..=KTIME_MAX`.
-unsafe impl ClockSource for Tai {
+unsafe impl ClockId for Tai {
     const ID: bindings::clockid_t = bindings::CLOCK_TAI as bindings::clockid_t;
 
     fn ktime_get() -> bindings::ktime_t {
@@ -180,24 +184,24 @@ unsafe impl ClockSource for Tai {
 /// The `inner` value is in the range from 0 to `KTIME_MAX`.
 #[repr(transparent)]
 #[derive(PartialEq, PartialOrd, Eq, Ord)]
-pub struct Instant<C: ClockSource> {
+pub struct Instant<C: ClockId> {
     inner: bindings::ktime_t,
     _c: PhantomData<C>,
 }
 
-impl<C: ClockSource> Clone for Instant<C> {
+impl<C: ClockId> Clone for Instant<C> {
     fn clone(&self) -> Self {
         *self
     }
 }
 
-impl<C: ClockSource> Copy for Instant<C> {}
+impl<C: ClockId> Copy for Instant<C> {}
 
-impl<C: ClockSource> Instant<C> {
+impl<C: ClockId> Instant<C> {
     /// Get the current time from the clock source.
     #[inline]
     pub fn now() -> Self {
-        // INVARIANT: The `ClockSource::ktime_get()` function returns a value in the range
+        // INVARIANT: The `ClockId::ktime_get()` function returns a value in the range
         // from 0 to `KTIME_MAX`.
         Self {
             inner: C::ktime_get(),
@@ -239,7 +243,7 @@ impl<C: ClockSource> Instant<C> {
     }
 }
 
-impl<C: ClockSource> ops::Sub for Instant<C> {
+impl<C: ClockId> ops::Sub for Instant<C> {
     type Output = Delta;
 
     // By the type invariant, it never overflows.
@@ -251,7 +255,7 @@ impl<C: ClockSource> ops::Sub for Instant<C> {
     }
 }
 
-impl<T: ClockSource> ops::Add<Delta> for Instant<T> {
+impl<T: ClockId> ops::Add<Delta> for Instant<T> {
     type Output = Self;
 
     #[inline]
@@ -271,7 +275,7 @@ impl<T: ClockSource> ops::Add<Delta> for Instant<T> {
     }
 }
 
-impl<T: ClockSource> ops::Sub<Delta> for Instant<T> {
+impl<T: ClockId> ops::Sub<Delta> for Instant<T> {
     type Output = Self;
 
     #[inline]
@@ -554,6 +558,62 @@ impl Delta {
         }
     }
 
+    /// Convert this span to a [`Delta<Jiffy>`] suitable for use as a timeout.
+    ///
+    /// Unless the result saturates, the value is rounded up to the next whole
+    /// jiffy, so the resulting timeout is never shorter than `self`.
+    ///
+    /// A negative span saturates at zero jiffies, i.e. an immediate timeout.
+    ///
+    /// A span that does not fit saturates at the kernel's [`MAX_JIFFY_OFFSET`],
+    /// the upper bound for a jiffies span. That is a finite timeout, so a
+    /// saturated result can be shorter than the requested span. It is derived
+    /// from `long`, so only 32 bit can reach it, at about 12 days with `HZ=1000`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use kernel::time::Delta;
+    ///
+    /// // A negative span is an immediate timeout.
+    /// assert_eq!(Delta::from_millis(-1).to_jiffies_timeout().as_jiffies(), 0);
+    ///
+    /// // A span shorter than a jiffy still waits, i.e. the timeout is never
+    /// // shorter than the span.
+    /// assert!(Delta::from_nanos(1).to_jiffies_timeout().as_jiffies() >= 1);
+    /// ```
+    ///
+    /// [`MAX_JIFFY_OFFSET`]: srctree/include/linux/jiffies.h
+    #[inline]
+    pub fn to_jiffies_timeout(self) -> Delta<Jiffy> {
+        const HZ: u64 = bindings::HZ as u64;
+
+        // The quotient `(nsecs * HZ + NSEC_PER_SEC - 1) / NSEC_PER_SEC` has to fit in
+        // `u64`; `nsecs * HZ` does not. With `HZ <= NSEC_PER_SEC` the numerator is at
+        // most `(nsecs + 1) * NSEC_PER_SEC - 1`, so the quotient is at most `nsecs`.
+        crate::static_assert!(HZ <= NSEC_PER_SEC as u64);
+
+        // CAST: `i64::max()` makes the value non-negative, so the cast keeps it.
+        let nsecs = i64::max(self.as_nanos(), 0) as u64;
+
+        // SAFETY: `mul_u64_add_u64_div_u64()` must not be called with a zero divisor,
+        // and its result must fit in `u64`. `NSEC_PER_SEC` is a non-zero constant, and
+        // the assertion above bounds the quotient by `nsecs`.
+        let jiffies = unsafe {
+            bindings::mul_u64_add_u64_div_u64(
+                nsecs,
+                HZ,
+                (NSEC_PER_SEC - 1) as u64,
+                NSEC_PER_SEC as u64,
+            )
+        };
+
+        // CAST: `jiffies` is clamped to `MAX_JIFFY_OFFSET`, which is `<= isize::MAX`.
+        let jiffies = u64::min(jiffies, MAX_JIFFY_OFFSET as u64) as isize;
+
+        Delta::<Jiffy>::from_jiffies(jiffies)
+    }
+
     /// Return `self % dividend` where `dividend` is in nanoseconds.
     ///
     /// The kernel doesn't have any emulation for `s64 % s64` on 32 bit platforms, so this is
@@ -580,3 +640,48 @@ impl Delta {
         }
     }
 }
+
+#[cfg(CONFIG_RUST_TIME_KUNIT_TEST)]
+#[macros::kunit_tests(rust_kernel_time)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn to_jiffies_timeout_converts() {
+        const HZ: isize = bindings::HZ as isize;
+
+        // One second is exactly `HZ` jiffies, and the round-up must not add one.
+        assert_eq!(Delta::from_secs(1).to_jiffies_timeout().as_jiffies(), HZ);
+
+        // One nanosecond more has to round up to the next whole jiffy.
+        assert_eq!(
+            Delta::from_nanos(NSEC_PER_SEC + 1)
+                .to_jiffies_timeout()
+                .as_jiffies(),
+            HZ + 1
+        );
+    }
+
+    #[test]
+    fn to_jiffies_timeout_saturates() {
+        // The result never exceeds `MAX_JIFFY_OFFSET`. On 32 bit with `HZ=1000` this
+        // span is 2147483647 jiffies, so the clamp is what keeps it in range; on 64
+        // bit it fits and the check holds for every possible return value.
+        let clamped = Delta::from_millis(i64::from(i32::MAX)).to_jiffies_timeout();
+        assert!(clamped.as_jiffies() <= MAX_JIFFY_OFFSET);
+
+        // `MAX_JIFFY_OFFSET` is derived from `long`, so only 32 bit can reach it. On
+        // 64 bit `i64::MAX` nanoseconds is about 292 years, which is 9223372036855
+        // jiffies with `HZ=1000`, far below the limit.
+        #[cfg(not(CONFIG_64BIT))]
+        {
+            // An overlong span is clamped to `MAX_JIFFY_OFFSET`.
+            let overlong = Delta::from_nanos(i64::MAX).to_jiffies_timeout();
+            assert_eq!(overlong.as_jiffies(), MAX_JIFFY_OFFSET);
+        }
+
+        // A negative span is an immediate timeout, however long it is.
+        let negative = Delta::from_nanos(i64::MIN).to_jiffies_timeout();
+        assert_eq!(negative.as_jiffies(), 0);
+    }
+}
diff --git a/rust/kernel/time/delay.rs b/rust/kernel/time/delay.rs
index b5b1b42797a0fe..4debe23731bf55 100644
--- a/rust/kernel/time/delay.rs
+++ b/rust/kernel/time/delay.rs
@@ -8,7 +8,10 @@
 //! C header: [`include/linux/delay.h`](srctree/include/linux/delay.h).
 
 use super::Delta;
-use crate::prelude::*;
+use crate::{
+    pr_warn_once,
+    prelude::*, //
+};
 
 /// Sleeps for a given duration at least.
 ///
@@ -18,8 +21,7 @@ use crate::prelude::*;
 /// `delta` must be within `[0, i32::MAX]` microseconds;
 /// otherwise, it is erroneous behavior. That is, it is considered a bug
 /// to call this function with an out-of-range value, in which case the function
-/// will sleep for at least the maximum value in the range and may warn
-/// in the future.
+/// will sleep for at least the maximum value in the range and warns once.
 ///
 /// The behavior above differs from the C side [`fsleep()`] for which out-of-range
 /// values mean "infinite timeout" instead.
@@ -32,10 +34,13 @@ pub fn fsleep(delta: Delta) {
     // overflow inside fsleep, which could lead to unintentional infinite sleep.
     const MAX_DELTA: Delta = Delta::from_micros(i32::MAX as i64);
 
+    debug_assert!(delta.as_nanos() >= 0);
+    debug_assert!(delta <= MAX_DELTA);
+
     let delta = if (Delta::ZERO..=MAX_DELTA).contains(&delta) {
         delta
     } else {
-        // TODO: Add WARN_ONCE() when it's supported.
+        pr_warn_once!("attempted to fsleep() with out of range delta\n");
         MAX_DELTA
     };
 
@@ -54,7 +59,8 @@ pub fn fsleep(delta: Delta) {
 ///
 /// `delta` must be within `[0, MAX_UDELAY_MS]` in milliseconds;
 /// otherwise, it is erroneous behavior. That is, it is considered a bug to
-/// call this function with an out-of-range value.
+/// call this function with an out-of-range value, in which case the function
+/// will delay for at least the maximum value in the range and warns once.
 ///
 /// The behavior above differs from the C side [`udelay()`] for which out-of-range
 /// values could lead to an overflow and unexpected behavior.
@@ -69,6 +75,7 @@ pub fn udelay(delta: Delta) {
     let delta = if (Delta::ZERO..=MAX_UDELAY_DELTA).contains(&delta) {
         delta
     } else {
+        pr_warn_once!("attempted to udelay() with out of range delta\n");
         MAX_UDELAY_DELTA
     };
 
diff --git a/rust/kernel/time/hrtimer.rs b/rust/kernel/time/hrtimer.rs
index 2d7f1131a8131d..c0180b8ca5d31f 100644
--- a/rust/kernel/time/hrtimer.rs
+++ b/rust/kernel/time/hrtimer.rs
@@ -402,15 +402,26 @@
 //! ```
 //!
 //! [`Arc`]: kernel::sync::Arc
-
-use super::{ClockSource, Delta, Instant};
+// Implementation details
+//
+// The reasoning for adopting a handle based approach:
+// - If we explicitly drop the target of a timer callback in the timer callback, we
+//   may get a dangling reference.
+// - If the callback owns the last reference to the target, target may be dropped
+//   in non-sleepable context when the callback is finished.
+// - When dropping an object that is the target of an armed timer, we may drop
+//   fields accessed by the timer callback before we cancel the timer (drop order).
+//
+// By using a handle, we can make the handle own the callback target and avoid these problems.
+
+use super::{ClockId, Delta, Instant};
 use crate::{prelude::*, types::Opaque};
 use core::{marker::PhantomData, ptr::NonNull};
 use pin_init::PinInit;
 
 /// A type-alias to refer to the [`Instant<C>`] for a given `T` from [`HrTimer<T>`].
 ///
-/// Where `C` is the [`ClockSource`] of the [`HrTimer`].
+/// Where `C` is the [`ClockId`] of the [`HrTimer`].
 pub type HrTimerInstant<T> = Instant<<<T as HasHrTimer<T>>::TimerMode as HrTimerMode>::Clock>;
 
 /// A timer backed by a C `struct hrtimer`.
@@ -855,7 +866,7 @@ pub trait HrTimerExpires {
     fn as_nanos(&self) -> i64;
 }
 
-impl<C: ClockSource> HrTimerExpires for Instant<C> {
+impl<C: ClockId> HrTimerExpires for Instant<C> {
     #[inline]
     fn as_nanos(&self) -> i64 {
         Instant::<C>::as_nanos(self)
@@ -870,22 +881,22 @@ impl HrTimerExpires for Delta {
 }
 
 mod private {
-    use crate::time::ClockSource;
+    use crate::time::ClockId;
 
     pub trait Sealed {}
 
-    impl<C: ClockSource> Sealed for super::AbsoluteMode<C> {}
-    impl<C: ClockSource> Sealed for super::RelativeMode<C> {}
-    impl<C: ClockSource> Sealed for super::AbsolutePinnedMode<C> {}
-    impl<C: ClockSource> Sealed for super::RelativePinnedMode<C> {}
-    impl<C: ClockSource> Sealed for super::AbsoluteSoftMode<C> {}
-    impl<C: ClockSource> Sealed for super::RelativeSoftMode<C> {}
-    impl<C: ClockSource> Sealed for super::AbsolutePinnedSoftMode<C> {}
-    impl<C: ClockSource> Sealed for super::RelativePinnedSoftMode<C> {}
-    impl<C: ClockSource> Sealed for super::AbsoluteHardMode<C> {}
-    impl<C: ClockSource> Sealed for super::RelativeHardMode<C> {}
-    impl<C: ClockSource> Sealed for super::AbsolutePinnedHardMode<C> {}
-    impl<C: ClockSource> Sealed for super::RelativePinnedHardMode<C> {}
+    impl<C: ClockId> Sealed for super::AbsoluteMode<C> {}
+    impl<C: ClockId> Sealed for super::RelativeMode<C> {}
+    impl<C: ClockId> Sealed for super::AbsolutePinnedMode<C> {}
+    impl<C: ClockId> Sealed for super::RelativePinnedMode<C> {}
+    impl<C: ClockId> Sealed for super::AbsoluteSoftMode<C> {}
+    impl<C: ClockId> Sealed for super::RelativeSoftMode<C> {}
+    impl<C: ClockId> Sealed for super::AbsolutePinnedSoftMode<C> {}
+    impl<C: ClockId> Sealed for super::RelativePinnedSoftMode<C> {}
+    impl<C: ClockId> Sealed for super::AbsoluteHardMode<C> {}
+    impl<C: ClockId> Sealed for super::RelativeHardMode<C> {}
+    impl<C: ClockId> Sealed for super::AbsolutePinnedHardMode<C> {}
+    impl<C: ClockId> Sealed for super::RelativePinnedHardMode<C> {}
 }
 
 /// Operational mode of [`HrTimer`].
@@ -894,16 +905,16 @@ pub trait HrTimerMode: private::Sealed {
     const C_MODE: bindings::hrtimer_mode;
 
     /// Type representing the clock source.
-    type Clock: ClockSource;
+    type Clock: ClockId;
 
     /// Type representing the expiration specification (absolute or relative time).
     type Expires: HrTimerExpires;
 }
 
 /// Timer that expires at a fixed point in time.
-pub struct AbsoluteMode<C: ClockSource>(PhantomData<C>);
+pub struct AbsoluteMode<C: ClockId>(PhantomData<C>);
 
-impl<C: ClockSource> HrTimerMode for AbsoluteMode<C> {
+impl<C: ClockId> HrTimerMode for AbsoluteMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS;
 
     type Clock = C;
@@ -911,9 +922,9 @@ impl<C: ClockSource> HrTimerMode for AbsoluteMode<C> {
 }
 
 /// Timer that expires after a delay from now.
-pub struct RelativeMode<C: ClockSource>(PhantomData<C>);
+pub struct RelativeMode<C: ClockId>(PhantomData<C>);
 
-impl<C: ClockSource> HrTimerMode for RelativeMode<C> {
+impl<C: ClockId> HrTimerMode for RelativeMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL;
 
     type Clock = C;
@@ -921,8 +932,8 @@ impl<C: ClockSource> HrTimerMode for RelativeMode<C> {
 }
 
 /// Timer with absolute expiration time, pinned to its current CPU.
-pub struct AbsolutePinnedMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for AbsolutePinnedMode<C> {
+pub struct AbsolutePinnedMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for AbsolutePinnedMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED;
 
     type Clock = C;
@@ -930,8 +941,8 @@ impl<C: ClockSource> HrTimerMode for AbsolutePinnedMode<C> {
 }
 
 /// Timer with relative expiration time, pinned to its current CPU.
-pub struct RelativePinnedMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for RelativePinnedMode<C> {
+pub struct RelativePinnedMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for RelativePinnedMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED;
 
     type Clock = C;
@@ -939,8 +950,8 @@ impl<C: ClockSource> HrTimerMode for RelativePinnedMode<C> {
 }
 
 /// Timer with absolute expiration, handled in soft irq context.
-pub struct AbsoluteSoftMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for AbsoluteSoftMode<C> {
+pub struct AbsoluteSoftMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for AbsoluteSoftMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_SOFT;
 
     type Clock = C;
@@ -948,8 +959,8 @@ impl<C: ClockSource> HrTimerMode for AbsoluteSoftMode<C> {
 }
 
 /// Timer with relative expiration, handled in soft irq context.
-pub struct RelativeSoftMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for RelativeSoftMode<C> {
+pub struct RelativeSoftMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for RelativeSoftMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_SOFT;
 
     type Clock = C;
@@ -957,8 +968,8 @@ impl<C: ClockSource> HrTimerMode for RelativeSoftMode<C> {
 }
 
 /// Timer with absolute expiration, pinned to CPU and handled in soft irq context.
-pub struct AbsolutePinnedSoftMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for AbsolutePinnedSoftMode<C> {
+pub struct AbsolutePinnedSoftMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for AbsolutePinnedSoftMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_SOFT;
 
     type Clock = C;
@@ -966,8 +977,8 @@ impl<C: ClockSource> HrTimerMode for AbsolutePinnedSoftMode<C> {
 }
 
 /// Timer with absolute expiration, pinned to CPU and handled in soft irq context.
-pub struct RelativePinnedSoftMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for RelativePinnedSoftMode<C> {
+pub struct RelativePinnedSoftMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for RelativePinnedSoftMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_SOFT;
 
     type Clock = C;
@@ -975,8 +986,8 @@ impl<C: ClockSource> HrTimerMode for RelativePinnedSoftMode<C> {
 }
 
 /// Timer with absolute expiration, handled in hard irq context.
-pub struct AbsoluteHardMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for AbsoluteHardMode<C> {
+pub struct AbsoluteHardMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for AbsoluteHardMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_HARD;
 
     type Clock = C;
@@ -984,8 +995,8 @@ impl<C: ClockSource> HrTimerMode for AbsoluteHardMode<C> {
 }
 
 /// Timer with relative expiration, handled in hard irq context.
-pub struct RelativeHardMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for RelativeHardMode<C> {
+pub struct RelativeHardMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for RelativeHardMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_HARD;
 
     type Clock = C;
@@ -993,8 +1004,8 @@ impl<C: ClockSource> HrTimerMode for RelativeHardMode<C> {
 }
 
 /// Timer with absolute expiration, pinned to CPU and handled in hard irq context.
-pub struct AbsolutePinnedHardMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for AbsolutePinnedHardMode<C> {
+pub struct AbsolutePinnedHardMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for AbsolutePinnedHardMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_ABS_PINNED_HARD;
 
     type Clock = C;
@@ -1002,8 +1013,8 @@ impl<C: ClockSource> HrTimerMode for AbsolutePinnedHardMode<C> {
 }
 
 /// Timer with relative expiration, pinned to CPU and handled in hard irq context.
-pub struct RelativePinnedHardMode<C: ClockSource>(PhantomData<C>);
-impl<C: ClockSource> HrTimerMode for RelativePinnedHardMode<C> {
+pub struct RelativePinnedHardMode<C: ClockId>(PhantomData<C>);
+impl<C: ClockId> HrTimerMode for RelativePinnedHardMode<C> {
     const C_MODE: bindings::hrtimer_mode = bindings::hrtimer_mode_HRTIMER_MODE_REL_PINNED_HARD;
 
     type Clock = C;
diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs
index 5f6c4d7a1a516d..f1aecda7a90ced 100644
--- a/rust/kernel/uaccess.rs
+++ b/rust/kernel/uaccess.rs
@@ -520,14 +520,14 @@ impl UserSliceWriter {
     ///
     /// fn copy_dma_to_user(
     ///     mut writer: UserSliceWriter,
-    ///     alloc: &Coherent<[u8]>,
+    ///     alloc: &Coherent<'_, [u8]>,
     /// ) -> Result {
     ///     writer.write_dma(alloc, 0, 256)
     /// }
     /// ```
     pub fn write_dma<T: KnownSize + AsBytes + ?Sized>(
         &mut self,
-        alloc: &Coherent<T>,
+        alloc: &Coherent<'_, T>,
         offset: usize,
         count: usize,
     ) -> Result {
@@ -558,6 +558,7 @@ impl UserSliceWriter {
     /// truncates the write to the boundaries of `self` and `data`.
     ///
     /// On success, returns the number of bytes written.
+    #[inline]
     pub fn write_slice_partial(&mut self, data: &[u8], offset: usize) -> Result<usize> {
         let end = offset.saturating_add(self.len()).min(data.len());
 
@@ -624,6 +625,19 @@ impl UserSliceWriter {
         self.length -= len;
         Ok(())
     }
+
+    /// Writes as much of the provided value as fits in the remaining buffer.
+    ///
+    /// Copies `min(size_of::<T>(), self.len())` bytes to userspace. Returns the number of bytes
+    /// actually written. This is useful for versioned structs where an older userspace may provide
+    /// a smaller buffer than the current kernel struct.
+    ///
+    /// Fails with [`EFAULT`] if the write happens on a bad address. This call may modify the
+    /// associated userspace slice even if it returns an error.
+    #[inline]
+    pub fn write_truncated<T: AsBytes>(&mut self, value: &T) -> Result<usize> {
+        self.write_slice_partial(value.as_bytes(), 0)
+    }
 }
 
 /// Reads a nul-terminated string into `dst` and returns the length.
diff --git a/rust/macros/io/mod.rs b/rust/macros/io/mod.rs
new file mode 100644
index 00000000000000..87f7742f461928
--- /dev/null
+++ b/rust/macros/io/mod.rs
@@ -0,0 +1,3 @@
+// SPDX-License-Identifier: GPL-2.0
+
+pub(crate) mod register;
diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs
new file mode 100644
index 00000000000000..420c3ad052b823
--- /dev/null
+++ b/rust/macros/io/register.rs
@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Documentation and usage example of the macro can be found at `rust/kernel/io/register.rs`.
+
+use proc_macro2::{
+    Group,
+    Literal,
+    Span,
+    TokenStream, //
+};
+use quote::{
+    quote,
+    quote_spanned, //
+};
+use syn::{
+    bracketed,
+    parenthesized,
+    parse::Parse,
+    parse_quote,
+    spanned::Spanned,
+    token,
+    Attribute,
+    Error,
+    Expr,
+    Ident,
+    Path,
+    Result,
+    Token,
+    Type,
+    Visibility, //
+};
+
+mod kw {
+    syn::custom_keyword!(base);
+    syn::custom_keyword!(stride);
+}
+
+/// Definition of a register array.
+///
+/// Specify a size, and optionally a stride. Syntax is of form `[EXPR $(, stride = EXPR)?]`.
+struct RegArrayDef {
+    size: Expr,
+    stride: Option<Expr>,
+}
+
+/// Offset of a register.
+///
+/// Can be either of form
+/// * `@ offset` for fixed offset
+/// * `=> alias` for alias of register `alias`.
+/// * `=> alias[idx]` for alias of register array `alias`'s `idx`-th element.
+enum RegOffset {
+    /// Register is located at fixed address.
+    Fixed { offset: Literal },
+    /// Register is an alias of a fixed register.
+    Alias { alias: Path },
+    /// Register is an alias of an element of a register array.
+    ElementAlias { alias: Path, idx: Expr },
+}
+
+/// Definition of a single register.
+struct Reg {
+    attrs: Vec<Attribute>,
+    vis: Visibility,
+    name: Ident,
+    unique: bool,
+    ty: Type,
+    array: Option<RegArrayDef>,
+    offset: RegOffset,
+    bitfield: Option<(Type, Group)>,
+}
+
+impl Parse for Reg {
+    fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> {
+        let attrs = input.call(Attribute::parse_outer)?;
+        let vis = input.parse()?;
+        let name = input.parse()?;
+
+        let lh = input.lookahead1();
+        let (unique, ty, bitfield_storage) = if lh.peek(Token![:]) {
+            let _: Token![:] = input.parse()?;
+
+            let mut attrs = input.call(Attribute::parse_outer)?;
+            let unique = attrs
+                .extract_if(.., |attr| attr.path().is_ident("unique"))
+                .count()
+                != 0;
+            if !attrs.is_empty() {
+                Err(Error::new_spanned(&attrs[0], "unexpected attributes"))?
+            }
+
+            (unique, input.parse()?, None)
+        } else if lh.peek(token::Paren) {
+            let content;
+            parenthesized!(content in input);
+            let bitfield_storage = Some(content.parse()?);
+
+            // For bitfields, bitfield macro will generate a type with the same name as `name`.
+            (true, parse_quote!(#name), bitfield_storage)
+        } else {
+            Err(lh.error())?
+        };
+
+        let array = if input.peek(token::Bracket) {
+            let content;
+            bracketed!(content in input);
+            let size = content.parse()?;
+            let stride = if content.peek(Token![,]) {
+                let _: Token![,] = content.parse()?;
+                let _: kw::stride = content.parse()?;
+                let _: Token![=] = content.parse()?;
+                Some(content.parse()?)
+            } else {
+                None
+            };
+            Some(RegArrayDef { size, stride })
+        } else {
+            None
+        };
+
+        // Parse offset and the base it's relative to.
+        let lh = input.lookahead1();
+        let offset = if lh.peek(Token![@]) {
+            let _: Token![@] = input.parse()?;
+
+            RegOffset::Fixed {
+                offset: input.parse()?,
+            }
+        } else if lh.peek(Token![=>]) {
+            let _: Token![=>] = input.parse()?;
+            let alias: Path = input.parse()?;
+
+            if input.peek(token::Bracket) {
+                let content;
+                bracketed!(content in input);
+                RegOffset::ElementAlias {
+                    alias,
+                    idx: content.parse()?,
+                }
+            } else {
+                RegOffset::Alias { alias }
+            }
+        } else {
+            Err(lh.error())?
+        };
+
+        let bitfield = if let Some(storage) = bitfield_storage {
+            let lh = input.lookahead1();
+            let args = if lh.peek(token::Brace) {
+                input.parse()?
+            } else {
+                Err(lh.error())?
+            };
+            Some((storage, args))
+        } else {
+            let _: Token![;] = input.parse()?;
+            None
+        };
+
+        Ok(Self {
+            attrs,
+            vis,
+            name,
+            unique,
+            ty,
+            array,
+            offset,
+            bitfield,
+        })
+    }
+}
+
+pub(crate) struct RegDef {
+    base: Type,
+    regs: Vec<Reg>,
+}
+
+impl Parse for RegDef {
+    fn parse(input: syn::parse::ParseStream<'_>) -> Result<Self> {
+        let _: kw::base = input.parse().map_err(|e| {
+            Error::new(
+                e.span(),
+                "a base type needs to be specified for `register!` invocation with `base: ty;`",
+            )
+        })?;
+
+        let _: Token![:] = input.parse()?;
+        let base = input.parse()?;
+        let _: Token![;] = input.parse()?;
+
+        let mut regs = Vec::new();
+        while !input.is_empty() {
+            regs.push(input.parse()?);
+        }
+        Ok(RegDef { base, regs })
+    }
+}
+
+pub(crate) fn register(def: RegDef) -> Result<TokenStream> {
+    let mut outputs = TokenStream::new();
+
+    let base = &def.base;
+    for reg in def.regs {
+        let Reg {
+            attrs,
+            vis,
+            name,
+            unique,
+            ty,
+            array,
+            offset,
+            bitfield,
+        } = reg;
+
+        // Use register name's span for generated code, so error messages (if any) can point to it
+        // instead of the entire register allocation.
+        let span = name.span().resolved_at(Span::mixed_site());
+
+        let offset = match offset {
+            RegOffset::Fixed { offset } => quote!(#offset),
+            RegOffset::Alias { alias } => {
+                quote_spanned!(alias.span().resolved_at(span) =>
+                    ::kernel::io::register::OffsetLoc::<#base, _>::const_offset(#alias)
+                )
+            }
+            RegOffset::ElementAlias { alias, idx } => {
+                quote_spanned!(alias.span().resolved_at(span) =>
+                    ::kernel::io::register::element_alias_offset::<#base, #alias>(#idx)
+                )
+            }
+        };
+
+        if let Some((storage, args)) = &bitfield {
+            outputs.extend(quote_spanned!(span =>
+                ::kernel::bitfield!(
+                    // `#[allow(non_camel_case_types)]` is added since register names typically use
+                    // `SCREAMING_CASE`.
+                    #[allow(non_camel_case_types)]
+                    #(#attrs)* #vis struct #name(#storage) #args
+                );
+            ));
+        }
+
+        match array {
+            None => {
+                if unique {
+                    outputs.extend(quote!(
+                        impl ::kernel::io::register::FixedIoLoc<#base> for #ty {
+                            type Location = ::kernel::io::register::OffsetLoc<#base, #ty>;
+                            const LOCATION: Self::Location = #name;
+                        }
+                    ))
+                }
+
+                outputs.extend(quote_spanned!(span =>
+                    #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> =
+                        ::kernel::io::register::OffsetLoc::new(#offset);
+                ));
+            }
+
+            Some(def) => {
+                if bitfield.is_none() {
+                    Err(Error::new_spanned(
+                        &ty,
+                        "defining without bitfield is not yet supported for this type of register",
+                    ))?
+                }
+
+                let size = &def.size;
+                let stride = if let Some(stride) = &def.stride {
+                    outputs.extend(quote_spanned!(stride.span().resolved_at(span) =>
+                        ::kernel::build_assert::static_assert!(
+                            ::core::mem::size_of::<#ty>() <= #stride
+                        );
+                    ));
+                    quote!(#stride)
+                } else {
+                    quote_spanned!(span => ::core::mem::size_of::<#ty>())
+                };
+
+                outputs.extend(quote_spanned!(span =>
+                    impl ::kernel::io::register::Array for #name {}
+
+                    impl ::kernel::io::register::RegisterArray for #name {
+                        type Base = #base;
+                        const OFFSET: usize = #offset;
+                        const SIZE: usize = #size;
+                        const STRIDE: usize = #stride;
+                    }
+                ));
+            }
+        };
+    }
+
+    Ok(outputs)
+}
diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs
index 24f96feaeb34a1..9b76efe1476f63 100644
--- a/rust/macros/lib.rs
+++ b/rust/macros/lib.rs
@@ -19,6 +19,7 @@ mod export;
 mod fmt;
 mod for_lt;
 mod helpers;
+mod io;
 mod kunit;
 mod module;
 mod paste;
@@ -481,6 +482,14 @@ pub fn paste(input: TokenStream) -> TokenStream {
         .into()
 }
 
+#[doc(hidden)] // Documented in `kernel` crate.
+#[proc_macro]
+pub fn register(input: TokenStream) -> TokenStream {
+    io::register::register(parse_macro_input!(input))
+        .unwrap_or_else(|e| e.into_compile_error())
+        .into()
+}
+
 /// Registers a KUnit test suite and its test cases using a user-space like syntax.
 ///
 /// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module
diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs
index c7d9b3e624fc8a..e88e520e326b66 100644
--- a/rust/pin-init/internal/src/diagnostics.rs
+++ b/rust/pin-init/internal/src/diagnostics.rs
@@ -1,44 +1,149 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
+use std::cell::RefCell;
 use std::fmt::Display;
+use std::marker::PhantomData;
 
-use proc_macro2::TokenStream;
-use quote::quote_spanned;
+use proc_macro2::{Span, TokenStream};
+use quote::{quote, quote_spanned};
 use syn::{spanned::Spanned, Error};
 
-pub(crate) struct DiagCtxt(TokenStream);
+pub(crate) struct DiagCtxt(PhantomData<*mut ()>);
 pub(crate) struct ErrorGuaranteed(());
 
+struct DiagCtxtData {
+    diag: TokenStream,
+}
+
+thread_local! {
+    static DIAGNOSTICS: RefCell<Option<DiagCtxtData>> = const { RefCell::new(None) };
+}
+
+// Allows `syn::Error` to be emitted into the current diagnostic context with just `?`.
+impl From<syn::Error> for ErrorGuaranteed {
+    fn from(error: syn::Error) -> Self {
+        DIAGNOSTICS.with_borrow_mut(|data| {
+            data.as_mut()
+                .unwrap()
+                .diag
+                .extend(error.into_compile_error());
+        });
+        Self(())
+    }
+}
+
 impl DiagCtxt {
-    pub(crate) fn error(&mut self, span: impl Spanned, msg: impl Display) -> ErrorGuaranteed {
-        let error = Error::new(span.span(), msg);
-        self.0.extend(error.into_compile_error());
-        ErrorGuaranteed(())
+    pub(crate) fn error(&self, span: impl Spanned, msg: impl Display) -> ErrorGuaranteed {
+        Error::new(span.span(), msg).into()
     }
 
-    pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) {
+    pub(crate) fn warn(&self, span: impl Spanned, msg: impl Display) {
         // Have the message start on a new line for visual clarity.
         let msg = format!("\n{}", msg);
-        self.0.extend(quote_spanned!(span.span() =>
-            // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable.
-            const _: () = {
-                #[deprecated = #msg]
-                const fn warn() {}
-                warn();
-            };
-        ));
+        DIAGNOSTICS.with_borrow_mut(|data| {
+            data.as_mut()
+                .unwrap()
+                .diag
+                .extend(quote_spanned!(span.span() =>
+                    // Approximate using deprecated warning while `proc_macro_diagnostic` is
+                    // unstable.
+                    const _: () = {
+                        #[deprecated = #msg]
+                        const fn warn() {}
+                        warn();
+                    };
+                ))
+        });
     }
 
-    pub(crate) fn with(
-        fun: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>,
+    /// Execute the provided function with the current diagnostic context.
+    pub(crate) fn current<R>(f: impl FnOnce(&DiagCtxt) -> R) -> R {
+        DIAGNOSTICS.with_borrow(|data| {
+            assert!(data.is_some(), "No active `DiagCtxt`");
+        });
+
+        f(&DiagCtxt(PhantomData))
+    }
+
+    fn with(
+        f: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>,
+        merge_diag: impl FnOnce(TokenStream, TokenStream) -> TokenStream,
+        convert_diag: impl FnOnce(TokenStream) -> TokenStream,
     ) -> TokenStream {
-        let mut dcx = Self(TokenStream::new());
-        match fun(&mut dcx) {
-            Ok(mut stream) => {
-                stream.extend(dcx.0);
-                stream
+        DIAGNOSTICS.with_borrow_mut(|data| {
+            assert!(data.is_none(), "`DiagCtxt` cannot be nested");
+            *data = Some(DiagCtxtData {
+                diag: TokenStream::new(),
+            });
+        });
+
+        let mut dcx = DiagCtxt(PhantomData);
+        let result = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(&mut dcx))) {
+            Ok(result) => result,
+            Err(payload) => {
+                // Robustness against panicking in macros.
+                //
+                // Ensure that any error messages are still emitted when this happens.
+                let message = if let Some(&s) = payload.downcast_ref::<&'static str>() {
+                    s
+                } else if let Some(s) = payload.downcast_ref::<String>() {
+                    s.as_str()
+                } else {
+                    "Box<dyn Any>"
+                };
+
+                Err(dcx.error(
+                    Span::mixed_site(),
+                    format!("proc macro panicked: {message}"),
+                ))
             }
-            Err(ErrorGuaranteed(())) => dcx.0,
+        };
+
+        let data = DIAGNOSTICS.with_borrow_mut(|data| data.take().unwrap());
+
+        match result {
+            Ok(stream) => {
+                if data.diag.is_empty() {
+                    stream
+                } else {
+                    merge_diag(stream, data.diag)
+                }
+            }
+            Err(ErrorGuaranteed(())) => convert_diag(data.diag),
         }
     }
+
+    pub(crate) fn for_item(
+        f: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>,
+    ) -> TokenStream {
+        Self::with(
+            f,
+            |mut out, diag| {
+                out.extend(diag);
+                out
+            },
+            std::convert::identity,
+        )
+    }
+
+    pub(crate) fn for_expr(
+        f: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>,
+    ) -> TokenStream {
+        Self::with(
+            f,
+            |out, diag| {
+                // Diagnostics that we generate are always items.
+                // So for expressions create a block to place diagnostics in item position.
+                quote!({
+                    #diag
+                    #out
+                })
+            },
+            |diag| {
+                quote!({
+                    #diag
+                })
+            },
+        )
+    }
 }
diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs
index fd0b5ea4a0a30c..c5b5a0ce5c4e41 100644
--- a/rust/pin-init/internal/src/init.rs
+++ b/rust/pin-init/internal/src/init.rs
@@ -1,26 +1,86 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 use proc_macro2::{Span, TokenStream};
-use quote::{format_ident, quote};
+use quote::{format_ident, quote, quote_spanned, ToTokens, TokenStreamExt};
 use syn::{
-    braced,
+    braced, parenthesized,
     parse::{End, Parse},
     parse_quote,
-    punctuated::Punctuated,
+    punctuated::{Pair, Punctuated},
     spanned::Spanned,
-    token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, Path, Token, Type,
+    token, Attribute, Block, Expr, ExprCall, ExprPath, Ident, Index, LitInt, Member, Path, Token,
+    Type,
 };
 
-use crate::diagnostics::{DiagCtxt, ErrorGuaranteed};
+use crate::{
+    diagnostics::{DiagCtxt, ErrorGuaranteed},
+    util::*,
+};
 
-pub(crate) struct Initializer {
+pub(crate) struct Initializer<Kind = InitExprKind> {
     attrs: Vec<InitializerAttribute>,
     this: Option<This>,
+    kind: Kind,
+    error: Option<(Token![?], Type)>,
+}
+
+pub(crate) struct InitExprStruct {
     path: Path,
     brace_token: token::Brace,
     fields: Punctuated<InitializerField, Token![,]>,
     rest: Option<(Token![..], Expr)>,
-    error: Option<(Token![?], Type)>,
+}
+
+pub(crate) struct InitExprTuple {
+    path: Path,
+    paren_token: token::Paren,
+    fields: Punctuated<InitTupleField, Token![,]>,
+}
+
+pub(crate) enum InitExprKind {
+    Struct(InitExprStruct),
+    Tuple(InitExprTuple),
+}
+
+struct InitTupleField {
+    attrs: Vec<Attribute>,
+    value: Expr,
+}
+
+impl InitExprTuple {
+    fn normalize(self) -> InitExprStruct {
+        let InitExprTuple {
+            path,
+            paren_token,
+            fields,
+        } = self;
+        InitExprStruct {
+            path,
+            brace_token: token::Brace {
+                span: paren_token.span,
+            },
+            fields: fields
+                .into_pairs()
+                .enumerate()
+                .map(|(index, pair)| {
+                    let (field, comma) = pair.into_tuple();
+                    let span = field.value.span();
+                    let field = InitializerField {
+                        attrs: field.attrs,
+                        kind: InitializerKind::Value {
+                            member: Member::Unnamed(Index {
+                                index: index.try_into().unwrap(),
+                                span,
+                            }),
+                            value: Some((Token![:](span), field.value)),
+                        },
+                    };
+                    Pair::new(field, comma)
+                })
+                .collect(),
+            rest: None,
+        }
+    }
 }
 
 struct This {
@@ -36,11 +96,11 @@ struct InitializerField {
 
 enum InitializerKind {
     Value {
-        ident: Ident,
+        member: Member,
         value: Option<(Token![:], Expr)>,
     },
     Init {
-        ident: Ident,
+        member: Member,
         _left_arrow_token: Token![<-],
         value: Expr,
     },
@@ -52,9 +112,9 @@ enum InitializerKind {
 }
 
 impl InitializerKind {
-    fn ident(&self) -> Option<&Ident> {
+    fn member(&self) -> Option<&Member> {
         match self {
-            Self::Value { ident, .. } | Self::Init { ident, .. } => Some(ident),
+            Self::Value { member, .. } | Self::Init { member, .. } => Some(member),
             Self::Code { .. } => None,
         }
     }
@@ -68,16 +128,101 @@ struct DefaultErrorAttribute {
     ty: Box<Type>,
 }
 
-pub(crate) fn expand(
+pub(crate) fn expand_with_cfg(
+    initializer: Initializer,
+    default_error: Option<&'static str>,
+    pinned: bool,
+    dcx: &mut DiagCtxt,
+) -> Result<TokenStream, ErrorGuaranteed> {
+    let initializer = match initializer.kind {
+        InitExprKind::Tuple(expr) => {
+            let mut initializer = Initializer {
+                attrs: initializer.attrs,
+                this: initializer.this,
+                kind: expr,
+                error: initializer.error,
+            };
+
+            // Removing a tuple field renumbers every field after it, which cannot be expressed with
+            // a `cfg` attribute on the initializer of a single field. Therefore, resolve tuple
+            // field cfgs before continuing. Struct expression syntax uses explicit numbers, so
+            // there is no need to pre-expand them and we only need to emit their cfgs on generated
+            // code.
+            for (field_idx, field) in initializer.kind.fields.iter_mut().enumerate() {
+                let cfg = field.attrs.extract_cfg_attrs();
+
+                if cfg.is_empty() {
+                    continue;
+                }
+
+                let true_initializer = initializer.to_token_stream();
+                initializer.kind.fields = initializer
+                    .kind
+                    .fields
+                    .into_pairs()
+                    .enumerate()
+                    .filter(|&(index, _)| index != field_idx)
+                    .map(|(_, pair)| pair)
+                    .collect();
+
+                let false_initializer = &initializer;
+
+                let macro_name = if pinned {
+                    quote!(::pin_init::pin_init)
+                } else {
+                    quote!(::pin_init::init)
+                };
+
+                // Resolve one field at a time until we've got no more tuple field cfgs.
+                //
+                // This is linear time because macro invocations with false cfg will not be
+                // expanded.
+                return Ok(quote! {
+                    {
+                        // Use `{}` delimiter here so semicolon is not required, otherwise the
+                        // expression becomes unit type.
+                        #[cfg(all(#(#cfg,)*))]
+                        #macro_name! { #true_initializer }
+
+                        #[cfg(not(all(#(#cfg,)*)))]
+                        #macro_name! { #false_initializer }
+                    }
+                });
+            }
+
+            // No cfgs left, we can normalize the initializer to the struct kind.
+            Initializer {
+                attrs: initializer.attrs,
+                this: initializer.this,
+                kind: initializer.kind.normalize(),
+                error: initializer.error,
+            }
+        }
+
+        InitExprKind::Struct(expr) => Initializer {
+            attrs: initializer.attrs,
+            this: initializer.this,
+            kind: expr,
+            error: initializer.error,
+        },
+    };
+
+    expand(initializer, default_error, pinned, dcx)
+}
+
+fn expand(
     Initializer {
         attrs,
         this,
-        path,
-        brace_token,
-        fields,
-        rest,
+        kind:
+            InitExprStruct {
+                path,
+                brace_token,
+                fields,
+                rest,
+            },
         error,
-    }: Initializer,
+    }: Initializer<InitExprStruct>,
     default_error: Option<&'static str>,
     pinned: bool,
     dcx: &mut DiagCtxt,
@@ -96,30 +241,31 @@ pub(crate) fn expand(
             } else if let Some(default_error) = default_error {
                 syn::parse_str(default_error).unwrap()
             } else {
-                dcx.error(brace_token.span.close(), "expected `? <type>` after `}`");
+                dcx.error(
+                    brace_token.span.close(),
+                    "expected `? <type>` after initializer",
+                );
                 parse_quote!(::core::convert::Infallible)
             }
         },
         |(_, err)| Box::new(err),
     );
-    let slot = format_ident!("slot");
-    let (has_data_trait, get_data, init_from_closure) = if pinned {
+    let (get_pin_data, init_from_closure) = if pinned {
         (
-            format_ident!("HasPinData"),
-            format_ident!("__pin_data"),
+            Some(
+                quote_spanned! { path.span().resolved_at(Span::mixed_site()) =>
+                    let data = ::pin_init::__internal::HasPinData::__pin_data(data);
+                },
+            ),
             format_ident!("pin_init_from_closure"),
         )
     } else {
-        (
-            format_ident!("HasInitData"),
-            format_ident!("__init_data"),
-            format_ident!("init_from_closure"),
-        )
+        (None, format_ident!("init_from_closure"))
     };
     let init_kind = get_init_kind(rest, dcx);
     let zeroable_check = match init_kind {
         InitKind::Normal => quote!(),
-        InitKind::Zeroing => quote! {
+        InitKind::Zeroing => quote_spanned! { Span::mixed_site() =>
             // The user specified `..Zeroable::zeroed()` at the end of the list of fields.
             // Therefore we check if the struct implements `Zeroable` and then zero the memory.
             // This allows us to also remove the check that all fields are present (since we
@@ -128,34 +274,35 @@ pub(crate) fn expand(
             where T: ::pin_init::Zeroable
             {}
             // Ensure that the struct is indeed `Zeroable`.
-            assert_zeroable(#slot);
+            assert_zeroable(slot);
             // SAFETY: The type implements `Zeroable` by the check above.
-            unsafe { ::core::ptr::write_bytes(#slot, 0, 1) };
+            unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
         },
     };
     let this = match this {
         None => quote!(),
-        Some(This { ident, .. }) => quote! {
+        Some(This { ident, .. }) => quote_spanned! { Span::mixed_site() =>
             // Create the `this` so it can be referenced by the user inside of the
             // expressions creating the individual fields.
             let #ident = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };
         },
     };
     // `mixed_site` ensures that the data is not accessible to the user-controlled code.
-    let data = Ident::new("__data", Span::mixed_site());
-    let init_fields = init_fields(&fields, pinned, &data, &slot);
+    let init_fields = init_fields(&fields, pinned);
     let field_check = make_field_check(&fields, init_kind, &path);
-    Ok(quote! {{
+    Ok(quote_spanned! { Span::mixed_site() => {
         // Get the data about fields from the supplied type.
-        // SAFETY: TODO
-        let #data = unsafe {
-            use ::pin_init::__internal::#has_data_trait;
+        let data = {
+            use ::pin_init::__internal::HasInitData;
             // Can't use `<#path as #has_data_trait>::#get_data`, since the user is able to omit
             // generics (which need to be present with that syntax).
-            #path::#get_data()
+            #path::__init_data()
         };
-        // Ensure that `#data` really is of type `#data` and help with type inference:
-        let init = #data.__make_closure::<_, #error>(
+
+        #get_pin_data
+
+        // Ensure that `data` really is of type `data` and help with type inference:
+        let init = data.__make_closure::<_, #error>(
             move |slot| {
                 #zeroable_check
                 #this
@@ -213,12 +360,7 @@ fn get_init_kind(rest: Option<(Token![..], Expr)>, dcx: &mut DiagCtxt) -> InitKi
 }
 
 /// Generate the code that initializes the fields of the struct using the initializers in `field`.
-fn init_fields(
-    fields: &Punctuated<InitializerField, Token![,]>,
-    pinned: bool,
-    data: &Ident,
-    slot: &Ident,
-) -> TokenStream {
+fn init_fields(fields: &Punctuated<InitializerField, Token![,]>, pinned: bool) -> TokenStream {
     let mut guards = vec![];
     let mut guard_attrs = vec![];
     let mut res = TokenStream::new();
@@ -229,9 +371,9 @@ fn init_fields(
             cfgs
         };
 
-        let ident = match kind {
-            InitializerKind::Value { ident, .. } => ident,
-            InitializerKind::Init { ident, .. } => ident,
+        let member = match kind {
+            InitializerKind::Value { member, .. } => member,
+            InitializerKind::Init { member, .. } => member,
             InitializerKind::Code { block, .. } => {
                 let stmt = &block.stmts;
                 res.extend(quote! {
@@ -243,27 +385,29 @@ fn init_fields(
                 continue;
             }
         };
+        let ident = member.as_ident();
+        let span = Span::mixed_site().located_at(ident.span());
 
         let slot = if pinned {
-            quote! {
+            quote_spanned! { span =>
                 // SAFETY:
                 // - `slot` is valid and properly aligned.
-                // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
-                // - `make_field_check` prevents `#ident` from being used twice, therefore
-                //   `(*slot).#ident` is exclusively accessed and has not been initialized.
-                (unsafe { #data.#ident(#slot) })
+                // - `make_field_check` checks that `&raw mut (*slot).#member` is properly aligned.
+                // - `make_field_check` prevents `#member` from being used twice, therefore
+                //   `(*slot).#member` is exclusively accessed and has not been initialized.
+                (unsafe { data.#ident(slot) })
             }
         } else {
-            quote! {
+            quote_spanned! { span =>
                 // For `init!()` macro, everything is unpinned.
                 // SAFETY:
-                // - `&raw mut (*slot).#ident` is valid.
-                // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
-                // - `make_field_check` prevents `#ident` from being used twice, therefore
-                //   `(*slot).#ident` is exclusively accessed and has not been initialized.
+                // - `&raw mut (*slot).#member` is valid.
+                // - `make_field_check` checks that `&raw mut (*slot).#member` is properly aligned.
+                // - `make_field_check` prevents `#member` from being used twice, therefore
+                //   `(*slot).#member` is exclusively accessed and has not been initialized.
                 (unsafe {
                     ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new(
-                        &raw mut (*#slot).#ident
+                        &raw mut (*slot).#member
                     )
                 })
             }
@@ -271,22 +415,22 @@ fn init_fields(
 
         // `mixed_site` ensures that the guard is not accessible to the user-controlled code.
         let guard = format_ident!("__{ident}_guard", span = Span::mixed_site());
+        let full_span = kind.span();
 
         let init = match kind {
-            InitializerKind::Value { ident, value } => {
+            InitializerKind::Value { value, .. } => {
                 let value = value
                     .as_ref()
                     .map(|(_, value)| quote!(#value))
-                    .unwrap_or_else(|| quote!(#ident));
+                    .unwrap_or_else(|| quote!(#member));
 
-                quote! {
+                quote_spanned! { full_span =>
                     #(#attrs)*
                     let mut #guard = #slot.write(#value);
-
                 }
             }
             InitializerKind::Init { value, .. } => {
-                quote! {
+                quote_spanned! { full_span =>
                     #(#attrs)*
                     let mut #guard = #slot.init(#value)?;
                 }
@@ -294,14 +438,23 @@ fn init_fields(
             InitializerKind::Code { .. } => unreachable!(),
         };
 
+        // A tuple field has no name that could be bound here (the `_0` identifiers are considered
+        // implementation detail and not user-facing).
+        let binding = match member {
+            Member::Named(ident) => quote_spanned! { span =>
+                #(#cfgs)*
+                // Allow `non_snake_case` since the same warning is going to be reported for the
+                // struct field.
+                #[allow(unused_variables, non_snake_case)]
+                let #ident = #guard.let_binding();
+            },
+            Member::Unnamed(_) => quote!(),
+        };
+
         res.extend(quote! {
             #init
 
-            #(#cfgs)*
-            // Allow `non_snake_case` since the same warning is going to be reported for the struct
-            // field.
-            #[allow(unused_variables, non_snake_case)]
-            let #ident = #guard.let_binding();
+            #binding
         });
 
         guards.push(guard);
@@ -326,16 +479,16 @@ fn make_field_check(
 ) -> TokenStream {
     let field_attrs: Vec<_> = fields
         .iter()
-        .filter_map(|f| f.kind.ident().map(|_| &f.attrs))
+        .filter_map(|f| f.kind.member().map(|_| &f.attrs))
         .collect();
-    let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect();
+    let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.member()).collect();
     let zeroing_trailer = match init_kind {
         InitKind::Normal => None,
         InitKind::Zeroing => Some(quote! {
             ..::core::mem::zeroed()
         }),
     };
-    quote! {
+    quote_spanned! { Span::mixed_site() =>
         #[allow(unreachable_code)]
         // We use unreachable code to perform field checks. They're still checked by the compiler.
         // SAFETY: this code is never executed.
@@ -364,11 +517,8 @@ fn make_field_check(
     }
 }
 
-impl Parse for Initializer {
-    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
-        let attrs = input.call(Attribute::parse_outer)?;
-        let this = input.peek(Token![&]).then(|| input.parse()).transpose()?;
-        let path = input.parse()?;
+impl InitExprStruct {
+    fn parse_with_path(path: Path, input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
         let content;
         let brace_token = braced!(content in input);
         let mut fields = Punctuated::new();
@@ -376,7 +526,8 @@ impl Parse for Initializer {
             let lh = content.lookahead1();
             if lh.peek(End) || lh.peek(Token![..]) {
                 break;
-            } else if lh.peek(Ident) || lh.peek(Token![_]) || lh.peek(Token![#]) {
+            } else if lh.peek(Ident) || lh.peek(LitInt) || lh.peek(Token![_]) || lh.peek(Token![#])
+            {
                 fields.push_value(content.parse()?);
                 let lh = content.lookahead1();
                 if lh.peek(End) {
@@ -394,6 +545,62 @@ impl Parse for Initializer {
             .peek(Token![..])
             .then(|| Ok::<_, syn::Error>((content.parse()?, content.parse()?)))
             .transpose()?;
+        Ok(Self {
+            path,
+            brace_token,
+            fields,
+            rest,
+        })
+    }
+}
+
+impl InitExprTuple {
+    fn parse_with_path(path: Path, input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let content;
+        let paren_token = parenthesized!(content in input);
+        let mut fields = Punctuated::new();
+        while !content.is_empty() {
+            let attrs = content.call(Attribute::parse_outer)?;
+
+            if let Some(left_arrow_token) = content.parse::<Option<Token![<-]>>()? {
+                DiagCtxt::current(|dcx| {
+                    dcx.error(
+                        left_arrow_token,
+                        "`<-` is not supported in tuple constructor syntax; name the fields by \
+                        index instead, e.g. `Type { 0 <- initializer, 1: value }`",
+                    )
+                });
+            }
+
+            fields.push_value(InitTupleField {
+                attrs,
+                value: content.parse()?,
+            });
+            if content.is_empty() {
+                break;
+            }
+            fields.push_punct(content.parse()?);
+        }
+        Ok(InitExprTuple {
+            path,
+            paren_token,
+            fields,
+        })
+    }
+}
+
+impl Parse for Initializer {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let attrs = input.call(Attribute::parse_outer)?;
+        let this = input.peek(Token![&]).then(|| input.parse()).transpose()?;
+        let path = input.parse()?;
+        let kind = if input.peek(token::Brace) {
+            InitExprKind::Struct(InitExprStruct::parse_with_path(path, input)?)
+        } else if input.peek(token::Paren) {
+            InitExprKind::Tuple(InitExprTuple::parse_with_path(path, input)?)
+        } else {
+            return Err(input.error("expected curly braces or parentheses"));
+        };
         let error = input
             .peek(Token![?])
             .then(|| Ok::<_, syn::Error>((input.parse()?, input.parse()?)))
@@ -412,10 +619,7 @@ impl Parse for Initializer {
         Ok(Self {
             attrs,
             this,
-            path,
-            brace_token,
-            fields,
-            rest,
+            kind,
             error,
         })
     }
@@ -450,33 +654,167 @@ impl Parse for InitializerField {
 impl Parse for InitializerKind {
     fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
         let lh = input.lookahead1();
-        if lh.peek(Token![_]) {
-            Ok(Self::Code {
+        let member = if lh.peek(Token![_]) {
+            return Ok(Self::Code {
                 _underscore_token: input.parse()?,
                 _colon_token: input.parse()?,
                 block: input.parse()?,
+            });
+        } else if lh.peek(Ident) || lh.peek(LitInt) {
+            input.parse::<Member>()?
+        } else {
+            return Err(lh.error());
+        };
+
+        let lh = input.lookahead1();
+        if lh.peek(Token![<-]) {
+            Ok(Self::Init {
+                member,
+                _left_arrow_token: input.parse()?,
+                value: input.parse()?,
+            })
+        } else if lh.peek(Token![:]) {
+            Ok(Self::Value {
+                member,
+                value: Some((input.parse()?, input.parse()?)),
+            })
+        } else if matches!(member, Member::Named(_)) && (lh.peek(Token![,]) || lh.peek(End)) {
+            // Short-hand syntax, available for named fields only.
+            Ok(Self::Value {
+                member,
+                value: None,
             })
-        } else if lh.peek(Ident) {
-            let ident = input.parse()?;
-            let lh = input.lookahead1();
-            if lh.peek(Token![<-]) {
-                Ok(Self::Init {
-                    ident,
-                    _left_arrow_token: input.parse()?,
-                    value: input.parse()?,
-                })
-            } else if lh.peek(Token![:]) {
-                Ok(Self::Value {
-                    ident,
-                    value: Some((input.parse()?, input.parse()?)),
-                })
-            } else if lh.peek(Token![,]) || lh.peek(End) {
-                Ok(Self::Value { ident, value: None })
-            } else {
-                Err(lh.error())
-            }
         } else {
             Err(lh.error())
         }
     }
 }
+
+impl<Kind: ToTokens> ToTokens for Initializer<Kind> {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        let Self {
+            attrs,
+            this,
+            kind,
+            error,
+        } = self;
+        tokens.append_all(attrs);
+        this.to_tokens(tokens);
+        kind.to_tokens(tokens);
+        if let Some((question, ty)) = error {
+            question.to_tokens(tokens);
+            ty.to_tokens(tokens);
+        }
+    }
+}
+
+impl ToTokens for InitExprKind {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        match self {
+            Self::Struct(init) => init.to_tokens(tokens),
+            Self::Tuple(init) => init.to_tokens(tokens),
+        }
+    }
+}
+
+impl ToTokens for InitExprStruct {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        let Self {
+            path,
+            brace_token,
+            fields,
+            rest,
+        } = self;
+        path.to_tokens(tokens);
+        brace_token.surround(tokens, |tokens| {
+            fields.to_tokens(tokens);
+            if let Some((dotdot, expr)) = rest {
+                dotdot.to_tokens(tokens);
+                expr.to_tokens(tokens);
+            }
+        });
+    }
+}
+
+impl ToTokens for InitExprTuple {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        let Self {
+            path,
+            paren_token,
+            fields,
+        } = self;
+        path.to_tokens(tokens);
+        paren_token.surround(tokens, |tokens| fields.to_tokens(tokens));
+    }
+}
+
+impl ToTokens for InitTupleField {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        let Self { attrs, value } = self;
+        tokens.append_all(attrs);
+        value.to_tokens(tokens);
+    }
+}
+
+impl ToTokens for InitializerAttribute {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        match self {
+            Self::DefaultError(DefaultErrorAttribute { ty }) => {
+                quote!(#[default_error(#ty)]).to_tokens(tokens);
+            }
+        }
+    }
+}
+
+impl ToTokens for This {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        let Self {
+            _and_token,
+            ident,
+            _in_token,
+        } = self;
+        _and_token.to_tokens(tokens);
+        ident.to_tokens(tokens);
+        _in_token.to_tokens(tokens);
+    }
+}
+
+impl ToTokens for InitializerField {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        let Self { attrs, kind } = self;
+        tokens.append_all(attrs);
+        kind.to_tokens(tokens);
+    }
+}
+
+impl ToTokens for InitializerKind {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        match self {
+            Self::Value { member, value } => {
+                member.to_tokens(tokens);
+                if let Some((colon, expr)) = value {
+                    colon.to_tokens(tokens);
+                    expr.to_tokens(tokens);
+                }
+            }
+            Self::Init {
+                member,
+                _left_arrow_token,
+                value,
+            } => {
+                member.to_tokens(tokens);
+                _left_arrow_token.to_tokens(tokens);
+                value.to_tokens(tokens);
+            }
+            Self::Code {
+                _underscore_token,
+                _colon_token,
+                block,
+            } => {
+                _underscore_token.to_tokens(tokens);
+                _colon_token.to_tokens(tokens);
+                block.to_tokens(tokens);
+            }
+        }
+    }
+}
diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs
index 60d5093f3128e7..07b3d33d228248 100644
--- a/rust/pin-init/internal/src/lib.rs
+++ b/rust/pin-init/internal/src/lib.rs
@@ -10,7 +10,6 @@
 #![allow(missing_docs)]
 
 use proc_macro::TokenStream;
-use syn::parse_macro_input;
 
 use crate::diagnostics::DiagCtxt;
 
@@ -18,42 +17,51 @@ mod diagnostics;
 mod init;
 mod pin_data;
 mod pinned_drop;
+mod util;
 mod zeroable;
 
 #[proc_macro_attribute]
 pub fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream {
-    let args = parse_macro_input!(args);
-    let input = parse_macro_input!(input);
-    DiagCtxt::with(|dcx| pin_data::pin_data(args, input, dcx)).into()
+    DiagCtxt::for_item(|dcx| pin_data::pin_data(syn::parse(args)?, syn::parse(input)?, dcx)).into()
 }
 
 #[proc_macro_attribute]
 pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream {
-    let args = parse_macro_input!(args);
-    let input = parse_macro_input!(input);
-    DiagCtxt::with(|dcx| pinned_drop::pinned_drop(args, input, dcx)).into()
+    DiagCtxt::for_item(|dcx| pinned_drop::pinned_drop(syn::parse(args)?, syn::parse(input)?, dcx))
+        .into()
 }
 
 #[proc_macro_derive(Zeroable)]
 pub fn derive_zeroable(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input);
-    DiagCtxt::with(|dcx| zeroable::derive(input, dcx)).into()
+    DiagCtxt::for_item(|dcx| zeroable::derive(syn::parse(input)?, dcx)).into()
 }
 
 #[proc_macro_derive(MaybeZeroable)]
 pub fn maybe_derive_zeroable(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input);
-    DiagCtxt::with(|dcx| zeroable::maybe_derive(input, dcx)).into()
+    DiagCtxt::for_item(|dcx| zeroable::maybe_derive(syn::parse(input)?, dcx)).into()
 }
 #[proc_macro]
 pub fn init(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input);
-    DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), false, dcx))
-        .into()
+    DiagCtxt::for_expr(|dcx| {
+        init::expand_with_cfg(
+            syn::parse(input)?,
+            Some("::core::convert::Infallible"),
+            false,
+            dcx,
+        )
+    })
+    .into()
 }
 
 #[proc_macro]
 pub fn pin_init(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input);
-    DiagCtxt::with(|dcx| init::expand(input, Some("::core::convert::Infallible"), true, dcx)).into()
+    DiagCtxt::for_expr(|dcx| {
+        init::expand_with_cfg(
+            syn::parse(input)?,
+            Some("::core::convert::Infallible"),
+            true,
+            dcx,
+        )
+    })
+    .into()
 }
diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs
index ff194d27565e82..982a38ceea9335 100644
--- a/rust/pin-init/internal/src/pin_data.rs
+++ b/rust/pin-init/internal/src/pin_data.rs
@@ -5,12 +5,16 @@ use quote::{format_ident, quote, ToTokens};
 use syn::{
     parse::{End, Nothing, Parse},
     parse_quote, parse_quote_spanned,
+    punctuated::Punctuated,
     spanned::Spanned,
     visit_mut::VisitMut,
-    Field, Fields, Generics, Ident, Item, PathSegment, Type, TypePath, Visibility, WhereClause,
+    Field, Fields, Generics, Ident, Index, Item, Member, PathSegment, Type, TypePath, Visibility,
 };
 
-use crate::diagnostics::{DiagCtxt, ErrorGuaranteed};
+use crate::{
+    diagnostics::{DiagCtxt, ErrorGuaranteed},
+    util::*,
+};
 
 pub(crate) mod kw {
     syn::custom_keyword!(PinnedDrop);
@@ -46,6 +50,7 @@ impl ToTokens for Args {
 
 struct FieldInfo<'a> {
     field: &'a Field,
+    member: Member,
     pinned: bool,
 }
 
@@ -81,21 +86,11 @@ pub(crate) fn pin_data(
     //
     // We need to perform this after parsing so we can reliably detect field cfgs.
     for (field_idx, field) in struct_.fields.iter_mut().enumerate() {
-        let cfg: Vec<_> = field
-            .attrs
-            .iter()
-            .filter(|a| a.path().is_ident("cfg"))
-            .map(|a| {
-                a.parse_args::<TokenStream>()
-                    .expect("parse as token stream cannot fail")
-            })
-            .collect();
-
+        let cfg = field.attrs.extract_cfg_attrs();
         if cfg.is_empty() {
             continue;
         }
 
-        field.attrs.retain(|a| !a.path().is_ident("cfg"));
         let cfg_true_struct = quote!(#struct_);
 
         let punctuated = match &mut struct_.fields {
@@ -136,10 +131,12 @@ pub(crate) fn pin_data(
     replacer.visit_generics_mut(&mut struct_.generics);
     replacer.visit_fields_mut(&mut struct_.fields);
 
+    let is_tuple_struct = matches!(struct_.fields, Fields::Unnamed(_));
     let fields: Vec<FieldInfo<'_>> = struct_
         .fields
         .iter_mut()
-        .map(|field| {
+        .enumerate()
+        .map(|(index, field)| {
             let len = field.attrs.len();
             field.attrs.retain(|a| !a.path().is_ident("pin"));
             let pinned_count = len - field.attrs.len();
@@ -151,23 +148,30 @@ pub(crate) fn pin_data(
                 !field.attrs.iter().any(|a| a.path().is_ident("cfg")),
                 "cfgs should be all resolved at this point"
             );
+            let member = match &field.ident {
+                Some(ident) => Member::Named(ident.clone()),
+                None => Member::Unnamed(Index {
+                    index: index as u32,
+                    span: field.span(),
+                }),
+            };
 
             FieldInfo {
                 field: &*field,
+                member,
                 pinned: pinned_count != 0,
             }
         })
         .collect();
 
     for field in &fields {
-        let ident = field.field.ident.as_ref().unwrap();
-
         if !field.pinned && is_phantom_pinned(&field.field.ty) {
             dcx.warn(
                 field.field,
                 format!(
-                    "The field `{ident}` of type `PhantomPinned` only has an effect \
+                    "The field {} of type `PhantomPinned` only has an effect \
                     if it has the `#[pin]` attribute",
+                    field.member.display_name(),
                 ),
             );
         }
@@ -175,8 +179,13 @@ pub(crate) fn pin_data(
 
     let unpin_impl = generate_unpin_impl(&struct_.ident, &struct_.generics, &fields);
     let drop_impl = generate_drop_impl(&struct_.ident, &struct_.generics, args);
-    let projections =
-        generate_projections(&struct_.vis, &struct_.ident, &struct_.generics, &fields);
+    let projections = generate_projections(
+        &struct_.vis,
+        &struct_.ident,
+        &struct_.generics,
+        is_tuple_struct,
+        &fields,
+    );
     let the_pin_data =
         generate_the_pin_data(&struct_.vis, &struct_.ident, &struct_.generics, &fields);
 
@@ -222,23 +231,13 @@ fn generate_unpin_impl(
     generics: &Generics,
     fields: &[FieldInfo<'_>],
 ) -> TokenStream {
-    let (_, ty_generics, _) = generics.split_for_impl();
-    let mut generics_with_pin_lt = generics.clone();
-    generics_with_pin_lt.params.insert(0, parse_quote!('__pin));
-    generics_with_pin_lt.make_where_clause();
-    let (
-        impl_generics_with_pin_lt,
-        ty_generics_with_pin_lt,
-        Some(WhereClause {
-            where_token,
-            predicates,
-        }),
-    ) = generics_with_pin_lt.split_for_impl()
-    else {
-        unreachable!()
-    };
+    let (impl_generics, ty_generics, whr) = generics.split_for_impl();
+    let predicates = whr
+        .map(|x| &x.predicates)
+        .unwrap_or(const { &Punctuated::new() });
+
     let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| {
-        let ident = f.field.ident.as_ref().unwrap();
+        let ident = f.member.as_ident();
         let ty = &f.field.ty;
         quote!(
             #ident: #ty
@@ -251,19 +250,18 @@ fn generate_unpin_impl(
             dead_code, // The fields below are never used.
             non_snake_case // The warning will be emitted on the struct definition.
         )]
-        struct __Unpin #generics_with_pin_lt
-        #where_token
-            #predicates
+        struct __Unpin #generics #whr
         {
-            __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>,
             __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>,
             #(#pinned_fields),*
         }
 
         #[doc(hidden)]
-        impl #impl_generics_with_pin_lt ::core::marker::Unpin for #ident #ty_generics
-        #where_token
-            __Unpin #ty_generics_with_pin_lt: ::core::marker::Unpin,
+        impl #impl_generics ::core::marker::Unpin for #ident #ty_generics
+        where
+            // the `for<'__dummy>` HRTB makes this not error without the `trivial_bounds`
+            // feature <https://github.com/rust-lang/rust/issues/48214#issuecomment-2557829956>.
+            for<'__dummy> __Unpin #ty_generics: ::core::marker::Unpin,
             #predicates
         {}
     }
@@ -320,6 +318,7 @@ fn generate_projections(
     vis: &Visibility,
     ident: &Ident,
     generics: &Generics,
+    is_tuple_struct: bool,
     fields: &[FieldInfo<'_>],
 ) -> TokenStream {
     let (impl_generics, ty_generics, _) = generics.split_for_impl();
@@ -332,28 +331,32 @@ fn generate_projections(
     let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields
         .iter()
         .map(|field| {
-            let Field { vis, ident, ty, .. } = &field.field;
+            let Field { vis, ty, .. } = &field.field;
+            let member = &field.member;
+            // The projection of a tuple struct is a tuple struct itself, so its fields are
+            // positional and must not be named.
+            let name = (!is_tuple_struct).then(|| {
+                let ident = field.member.as_ident();
+                quote!(#ident:)
+            });
 
-            let ident = ident
-                .as_ref()
-                .expect("only structs with named fields are supported");
             if field.pinned {
                 (
                     quote!(
-                        #vis #ident: ::core::pin::Pin<&'__pin mut #ty>,
+                        #vis #name ::core::pin::Pin<&'__pin mut #ty>,
                     ),
                     quote!(
                         // SAFETY: this field is structurally pinned.
-                        #ident: unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#ident) },
+                        #name unsafe { ::core::pin::Pin::new_unchecked(&mut #this.#member) },
                     ),
                 )
             } else {
                 (
                     quote!(
-                        #vis #ident: &'__pin mut #ty,
+                        #vis #name &'__pin mut #ty,
                     ),
                     quote!(
-                        #ident: &mut #this.#ident,
+                        #name &mut #this.#member,
                     ),
                 )
             }
@@ -362,24 +365,52 @@ fn generate_projections(
     let structurally_pinned_fields_docs = fields
         .iter()
         .filter(|f| f.pinned)
-        .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap()));
+        .map(|f| format!(" - {}", f.member.display_name()));
     let not_structurally_pinned_fields_docs = fields
         .iter()
         .filter(|f| !f.pinned)
-        .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap()));
+        .map(|f| format!(" - {}", f.member.display_name()));
     let docs = format!(" Pin-projections of [`{ident}`]");
+    let (projection_def, projection_init) = if is_tuple_struct {
+        (
+            quote! {
+                #vis struct #projection #generics_with_pin_lt (
+                    #(#fields_decl)*
+                    ::core::marker::PhantomData<&'__pin mut ()>,
+                ) #whr;
+            },
+            quote! {
+                #projection(
+                    #(#fields_proj)*
+                    ::core::marker::PhantomData,
+                )
+            },
+        )
+    } else {
+        (
+            quote! {
+                #vis struct #projection #generics_with_pin_lt
+                    #whr
+                {
+                    #(#fields_decl)*
+                    ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>,
+                }
+            },
+            quote! {
+                #projection {
+                    #(#fields_proj)*
+                    ___pin_phantom_data: ::core::marker::PhantomData,
+                }
+            },
+        )
+    };
     quote! {
         #[doc = #docs]
         // Allow `non_snake_case` since the same warning will be emitted on
         // the struct definition.
         #[allow(dead_code, non_snake_case)]
         #[doc(hidden)]
-        #vis struct #projection #generics_with_pin_lt
-            #whr
-        {
-            #(#fields_decl)*
-            ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>,
-        }
+        #projection_def
 
         impl #impl_generics #ident #ty_generics
             #whr
@@ -397,10 +428,7 @@ fn generate_projections(
             ) -> #projection #ty_generics_with_pin_lt {
                 // SAFETY: we only give access to `&mut` for fields not structurally pinned.
                 let #this = unsafe { ::core::pin::Pin::get_unchecked_mut(self) };
-                #projection {
-                    #(#fields_proj)*
-                    ___pin_phantom_data: ::core::marker::PhantomData,
-                }
+                #projection_init
             }
         }
     }
@@ -421,11 +449,9 @@ fn generate_the_pin_data(
     let field_accessors = fields
         .iter()
         .map(|f| {
-            let Field { vis, ident, ty, .. } = f.field;
-
-            let field_name = ident
-                .as_ref()
-                .expect("only structs with named fields are supported");
+            let Field { vis, ty, .. } = f.field;
+            let field_name = f.member.as_ident();
+            let member = &f.member;
             let pin_marker = if f.pinned {
                 quote!(Pinned)
             } else {
@@ -450,7 +476,7 @@ fn generate_the_pin_data(
                     // - If `#pin_marker` is `Pinned`, the corresponding field is structurally
                     //   pinned.
                     // - Other safety requirements follows the safety requirement.
-                    unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) }
+                    unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#member) }
                 }
             }
         })
@@ -501,7 +527,7 @@ fn generate_the_pin_data(
             type PinData = __ThePinData #ty_generics;
 
             #[inline]
-            unsafe fn __pin_data() -> Self::PinData {
+            fn __pin_data(_: ::pin_init::__internal::InitData<Self>) -> Self::PinData {
                 __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() }
             }
         }
diff --git a/rust/pin-init/internal/src/util.rs b/rust/pin-init/internal/src/util.rs
new file mode 100644
index 00000000000000..0b5c28b2109152
--- /dev/null
+++ b/rust/pin-init/internal/src/util.rs
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use proc_macro2::{Ident, TokenStream};
+use quote::format_ident;
+use syn::{Attribute, Index, Member};
+
+pub(crate) trait AttrListExt {
+    fn extract_cfg_attrs(&mut self) -> Vec<TokenStream>;
+}
+
+impl AttrListExt for Vec<Attribute> {
+    fn extract_cfg_attrs(&mut self) -> Vec<TokenStream> {
+        let cfg: Vec<_> = self
+            .iter()
+            .filter(|a| a.path().is_ident("cfg"))
+            .map(|a| {
+                a.parse_args::<TokenStream>()
+                    .expect("parse as token stream cannot fail")
+            })
+            .collect();
+
+        if !cfg.is_empty() {
+            self.retain(|a| !a.path().is_ident("cfg"));
+        }
+
+        cfg
+    }
+}
+
+pub(crate) trait MemberExt {
+    /// Returns an identifier for the member.
+    ///
+    /// Tuple fields have no name of their own, so they are named `_0`, `_1`, ... instead.
+    fn as_ident(&self) -> Ident;
+
+    /// Obtain a display name for the member in diagnostics.
+    fn display_name(&self) -> String;
+}
+
+impl MemberExt for Member {
+    fn as_ident(&self) -> Ident {
+        match self {
+            Member::Named(ident) => ident.clone(),
+            Member::Unnamed(Index { index, span }) => format_ident!("_{index}", span = *span),
+        }
+    }
+
+    fn display_name(&self) -> String {
+        match self {
+            Member::Named(ident) => format!("`{ident}`"),
+            Member::Unnamed(Index { index, .. }) => format!("index `{index}`"),
+        }
+    }
+}
diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs
index 8e9fd18b993f99..cba53b8c3c9495 100644
--- a/rust/pin-init/src/__internal.rs
+++ b/rust/pin-init/src/__internal.rs
@@ -42,22 +42,6 @@ impl<T: ?Sized> PhantomInvariant<T> {
     }
 }
 
-/// Zero-sized type used to mark a lifetime as invariant.
-///
-/// This is a polyfill for the [unstable type] in the standard library of the same name.
-///
-/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html
-#[repr(transparent)]
-#[derive(Clone, Copy, Default)]
-pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>);
-
-impl PhantomInvariantLifetime<'_> {
-    #[inline(always)]
-    pub const fn new() -> Self {
-        Self(PhantomInvariant::new())
-    }
-}
-
 /// Token type to signify successful initialization.
 ///
 /// Can only be constructed via the unsafe [`Self::new`] function. The initializer macros use this
@@ -81,39 +65,43 @@ impl InitOk {
 ///
 /// # Safety
 ///
-/// Only the `init` module is allowed to use this trait.
+/// `pin-init` relies on the correctness of the helper functions defined on `PinData`.
+/// Thus, only the `#[pin_data]` can implement this trait.
+#[diagnostic::on_unimplemented(
+    message = "`{Self}` cannot be used with `pin_init!` macro",
+    note = "did you forget to add `#[pin_data]` attribute to the struct?"
+)]
 pub unsafe trait HasPinData {
     type PinData;
 
-    #[expect(clippy::missing_safety_doc)]
-    unsafe fn __pin_data() -> Self::PinData;
+    fn __pin_data(_: InitData<Self>) -> Self::PinData;
 }
 
-/// This trait is automatically implemented for every type. It aims to provide the same type
-/// inference help as `HasPinData`.
+/// This trait is automatically implemented for every type.
 ///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait HasInitData {
-    type InitData;
-
-    #[expect(clippy::missing_safety_doc)]
-    unsafe fn __init_data() -> Self::InitData;
+/// It aims to provide type inference help; `PATH::__init_data()` would be able to retrieve an
+/// instance of `InitData<PATH<Generics>>` without having to mention the generics explicitly.
+pub trait HasInitData {
+    #[inline]
+    fn __init_data() -> InitData<Self> {
+        InitData(PhantomInvariant::new())
+    }
 }
 
-pub struct AllData<T: ?Sized>(PhantomInvariant<T>);
+impl<T: ?Sized> HasInitData for T {}
+
+pub struct InitData<T: ?Sized>(PhantomInvariant<T>);
 
-impl<T: ?Sized> Clone for AllData<T> {
+impl<T: ?Sized> Clone for InitData<T> {
     #[inline]
     fn clone(&self) -> Self {
         *self
     }
 }
 
-impl<T: ?Sized> Copy for AllData<T> {}
+impl<T: ?Sized> Copy for InitData<T> {}
 
-impl<T: ?Sized> AllData<T> {
+impl<T: ?Sized> InitData<T> {
     /// Type inference helper function.
     #[inline(always)]
     pub fn __make_closure<F, E>(self, f: F) -> F
@@ -124,16 +112,6 @@ impl<T: ?Sized> AllData<T> {
     }
 }
 
-// SAFETY: TODO.
-unsafe impl<T: ?Sized> HasInitData for T {
-    type InitData = AllData<T>;
-
-    #[inline]
-    unsafe fn __init_data() -> Self::InitData {
-        AllData(PhantomInvariant::new())
-    }
-}
-
 /// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive.
 ///
 /// # Invariants
diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs
index f1463be9479d67..d1ed0561bb27f6 100644
--- a/rust/pin-init/src/lib.rs
+++ b/rust/pin-init/src/lib.rs
@@ -304,6 +304,9 @@ pub use alloc::InPlaceInit;
 /// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`,
 /// then `#[pin]` directs the type of initializer that is required.
 ///
+/// Tuple structs are supported as well. Their fields have no names, so the generated projection
+/// is a tuple struct too and its fields are accessed by index.
+///
 /// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this
 /// macro, and change your `Drop` implementation to `PinnedDrop` annotated with
 /// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care.
@@ -327,6 +330,26 @@ pub use alloc::InPlaceInit;
 /// }
 /// ```
 ///
+/// The same as a tuple struct, projected by index:
+///
+/// ```
+/// # #![feature(allocator_api)]
+/// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
+/// use core::pin::Pin;
+/// use pin_init::pin_data;
+///
+/// enum Command {
+///     /* ... */
+/// }
+///
+/// #[pin_data]
+/// struct DriverData(#[pin] CMutex<Vec<Command>>, Box<[u8; 1024 * 1024]>);
+///
+/// fn queue(data: Pin<&mut DriverData>) -> Pin<&mut CMutex<Vec<Command>>> {
+///     data.project().0
+/// }
+/// ```
+///
 /// ```
 /// # #![feature(allocator_api)]
 /// # #[path = "../examples/mutex.rs"] mod mutex; use mutex::*;
@@ -569,7 +592,7 @@ macro_rules! stack_try_pin_init {
     };
 }
 
-/// Construct an in-place, fallible pinned initializer for `struct`s.
+/// Construct an in-place, fallible pinned initializer for structs, including tuple structs.
 ///
 /// The error type defaults to [`Infallible`]; if you need a different one, write `? Error` at the
 /// end, after the struct initializer.
@@ -603,6 +626,42 @@ macro_rules! stack_try_pin_init {
 /// # Box::pin_init(demo()).unwrap();
 /// ```
 ///
+/// The fields of a tuple struct are addressed by their index:
+///
+/// ```rust
+/// # use pin_init::*;
+/// # use core::pin::Pin;
+/// #[pin_data]
+/// struct Pair(usize, Bar);
+///
+/// #[pin_data]
+/// struct Bar {
+///     x: u32,
+/// }
+///
+/// # fn demo() -> impl PinInit<Pair> {
+/// let initializer = pin_init!(Pair {
+///     0: 42,
+///     1 <- Bar { x: 64 },
+/// });
+/// # initializer }
+/// # Box::pin_init(demo()).unwrap();
+/// ```
+///
+/// A tuple struct whose fields are all set to a value can also be written like a call to its
+/// constructor:
+///
+/// ```rust
+/// # use pin_init::*;
+/// #[pin_data]
+/// struct Pair(usize, usize);
+///
+/// # fn demo() -> impl PinInit<Pair> {
+/// let initializer = pin_init!(Pair(42, 64));
+/// # initializer }
+/// # Box::pin_init(demo()).unwrap();
+/// ```
+///
 /// Arbitrary Rust expressions can be used to set the value of a variable.
 ///
 /// The fields are initialized in the order that they appear in the initializer. So it is possible
@@ -721,9 +780,13 @@ macro_rules! stack_try_pin_init {
 ///
 /// # Syntax
 ///
-/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
-/// the following modifications is expected:
+/// As already mentioned in the examples above, inside of `pin_init!` a struct initializer with the
+/// following modifications is expected:
 /// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
+/// - Tuple struct fields are named by their index, as in `0: value` or `0 <- initializer`. They
+///   are not exposed by a `let` binding, since they have no name to bind.
+/// - A tuple struct can also be initialized with constructor syntax, as in `Type(value, value)`.
+///   Since its arguments are not named, they cannot use `<-`; write them out by index instead.
 /// - You can use `_: { /* run any user-code here */ },` anywhere where you can place fields in
 ///   order to run arbitrary code.
 /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
@@ -762,7 +825,7 @@ macro_rules! stack_try_pin_init {
 /// [`NonNull<Self>`]: core::ptr::NonNull
 pub use pin_init_internal::pin_init;
 
-/// Construct an in-place, fallible initializer for `struct`s.
+/// Construct an in-place, fallible initializer for structs, including tuple structs.
 ///
 /// This macro defaults the error to [`Infallible`]; if you need a different one, write `? Error`
 /// at the end, after the struct initializer.
@@ -857,7 +920,8 @@ macro_rules! assert_pinned {
     ($ty:ty, $field:ident, $field_ty:ty, inline) => {
         // SAFETY: This code is unreachable.
         let _ = move |ptr: *mut $ty| unsafe {
-            let data = <$ty as $crate::__internal::HasPinData>::__pin_data();
+            let data = <$ty as $crate::__internal::HasInitData>::__init_data();
+            let data = $crate::__internal::HasPinData::__pin_data(data);
             _ = data
                 .$field(ptr)
                 .init($crate::__internal::AlwaysFail::<$field_ty>::new());
@@ -902,6 +966,9 @@ macro_rules! assert_pinned {
 #[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")]
 #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
 #[must_use = "An initializer must be used in order to create its value."]
+#[diagnostic::on_unimplemented(
+    message = "`{Self}` cannot be used to initialize `{T}` with error `{E}`"
+)]
 pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
     /// Alias of [`PinInit::__init`].
     ///
@@ -1036,6 +1103,11 @@ where
 #[cfg_attr(not(kernel), doc = "[`Arc<T>`]: alloc::alloc::sync::Arc")]
 #[cfg_attr(not(kernel), doc = "[`Box<T>`]: alloc::alloc::boxed::Box")]
 #[must_use = "An initializer must be used in order to create its value."]
+#[diagnostic::on_unimplemented(
+    message = "`{Self}` cannot be used to movably initialize `{T}` with error `{E}`",
+    note = "if your type implements `PinInit` but not `Init`, \
+            you might be forgetting a `#[pin]` annotation on fields"
+)]
 pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
     /// First initializes the value using `self` then calls the function `f` with the initialized
     /// value.
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
index 95a4fa1f1e4474..c28e65046986d3 100644
--- a/samples/bpf/Makefile
+++ b/samples/bpf/Makefile
@@ -342,6 +342,7 @@ $(obj)/xdp_router_ipv4.bpf.o: $(obj)/xdp_sample.bpf.o
 $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/xdp_sample_shared.h
 	@echo "  CLANG-BPF " $@
 	$(Q)$(CLANG) -g -O2 --target=bpf -D__TARGET_ARCH_$(SRCARCH) \
+		-fms-extensions -Wno-microsoft-anon-tag \
 		-Wno-compare-distinct-pointer-types -I$(srctree)/include \
 		-I$(srctree)/samples/bpf -I$(srctree)/tools/include \
 		-I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \
diff --git a/samples/bpf/hash_func01.h b/samples/bpf/hash_func01.h
deleted file mode 100644
index 38255812e3769b..00000000000000
--- a/samples/bpf/hash_func01.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/* SPDX-License-Identifier: LGPL-2.1
- *
- * Based on Paul Hsieh's (LGPG 2.1) hash function
- * From: http://www.azillionmonkeys.com/qed/hash.html
- */
-
-#define get16bits(d) (*((const __u16 *) (d)))
-
-static __always_inline
-__u32 SuperFastHash (const char *data, int len, __u32 initval) {
-	__u32 hash = initval;
-	__u32 tmp;
-	int rem;
-
-	if (len <= 0 || data == NULL) return 0;
-
-	rem = len & 3;
-	len >>= 2;
-
-	/* Main loop */
-#pragma clang loop unroll(full)
-	for (;len > 0; len--) {
-		hash  += get16bits (data);
-		tmp    = (get16bits (data+2) << 11) ^ hash;
-		hash   = (hash << 16) ^ tmp;
-		data  += 2*sizeof (__u16);
-		hash  += hash >> 11;
-	}
-
-	/* Handle end cases */
-	switch (rem) {
-        case 3: hash += get16bits (data);
-                hash ^= hash << 16;
-                hash ^= ((signed char)data[sizeof (__u16)]) << 18;
-                hash += hash >> 11;
-                break;
-        case 2: hash += get16bits (data);
-                hash ^= hash << 11;
-                hash += hash >> 17;
-                break;
-        case 1: hash += (signed char)*data;
-                hash ^= hash << 10;
-                hash += hash >> 1;
-	}
-
-	/* Force "avalanching" of final 127 bits */
-	hash ^= hash << 3;
-	hash += hash >> 5;
-	hash ^= hash << 4;
-	hash += hash >> 17;
-	hash ^= hash << 25;
-	hash += hash >> 6;
-
-	return hash;
-}
diff --git a/samples/configfs/configfs_sample.c b/samples/configfs/configfs_sample.c
index c1b108ec4ea02e..6e3a8e0061aba7 100644
--- a/samples/configfs/configfs_sample.c
+++ b/samples/configfs/configfs_sample.c
@@ -84,7 +84,7 @@ CONFIGFS_ATTR_RO(childless_, showme);
 CONFIGFS_ATTR(childless_, storeme);
 CONFIGFS_ATTR_RO(childless_, description);
 
-static struct configfs_attribute *childless_attrs[] = {
+static const struct configfs_attribute *const childless_attrs[] = {
 	&childless_attr_showme,
 	&childless_attr_storeme,
 	&childless_attr_description,
@@ -92,7 +92,7 @@ static struct configfs_attribute *childless_attrs[] = {
 };
 
 static const struct config_item_type childless_type = {
-	.ct_attrs	= childless_attrs,
+	.ct_attrs_const	= childless_attrs,
 	.ct_owner	= THIS_MODULE,
 };
 
@@ -148,7 +148,7 @@ static ssize_t simple_child_storeme_store(struct config_item *item,
 
 CONFIGFS_ATTR(simple_child_, storeme);
 
-static struct configfs_attribute *simple_child_attrs[] = {
+static const struct configfs_attribute *const simple_child_attrs[] = {
 	&simple_child_attr_storeme,
 	NULL,
 };
@@ -164,7 +164,7 @@ static const struct configfs_item_operations simple_child_item_ops = {
 
 static const struct config_item_type simple_child_type = {
 	.ct_item_ops	= &simple_child_item_ops,
-	.ct_attrs	= simple_child_attrs,
+	.ct_attrs_const	= simple_child_attrs,
 	.ct_owner	= THIS_MODULE,
 };
 
@@ -205,7 +205,7 @@ static ssize_t simple_children_description_show(struct config_item *item,
 
 CONFIGFS_ATTR_RO(simple_children_, description);
 
-static struct configfs_attribute *simple_children_attrs[] = {
+static const struct configfs_attribute *const simple_children_attrs[] = {
 	&simple_children_attr_description,
 	NULL,
 };
@@ -230,7 +230,7 @@ static const struct configfs_group_operations simple_children_group_ops = {
 static const struct config_item_type simple_children_type = {
 	.ct_item_ops	= &simple_children_item_ops,
 	.ct_group_ops	= &simple_children_group_ops,
-	.ct_attrs	= simple_children_attrs,
+	.ct_attrs_const	= simple_children_attrs,
 	.ct_owner	= THIS_MODULE,
 };
 
@@ -283,7 +283,7 @@ static ssize_t group_children_description_show(struct config_item *item,
 
 CONFIGFS_ATTR_RO(group_children_, description);
 
-static struct configfs_attribute *group_children_attrs[] = {
+static const struct configfs_attribute *const group_children_attrs[] = {
 	&group_children_attr_description,
 	NULL,
 };
@@ -298,7 +298,7 @@ static const struct configfs_group_operations group_children_group_ops = {
 
 static const struct config_item_type group_children_type = {
 	.ct_group_ops	= &group_children_group_ops,
-	.ct_attrs	= group_children_attrs,
+	.ct_attrs_const	= group_children_attrs,
 	.ct_owner	= THIS_MODULE,
 };
 
@@ -313,6 +313,126 @@ static struct configfs_subsystem group_children_subsys = {
 
 /* ----------------------------------------------------------------- */
 
+/*
+ * 04-symlink-children
+ *
+ * This example has children that are valid sources for symlink(2).  A
+ * child accepts a link to any other config_item and reports how many
+ * links it currently holds, so ->allow_link() and ->drop_link() are
+ * observable from userspace.
+ */
+
+struct symlink_child {
+	struct config_item item;
+	int nlinks;
+};
+
+static inline struct symlink_child *to_symlink_child(struct config_item *item)
+{
+	return container_of(item, struct symlink_child, item);
+}
+
+static ssize_t symlink_child_nlinks_show(struct config_item *item, char *page)
+{
+	return sprintf(page, "%d\n", to_symlink_child(item)->nlinks);
+}
+
+CONFIGFS_ATTR_RO(symlink_child_, nlinks);
+
+static struct configfs_attribute *symlink_child_attrs[] = {
+	&symlink_child_attr_nlinks,
+	NULL,
+};
+
+/*
+ * The VFS holds the source item's directory locked across symlink(2) and
+ * unlink(2), so ->nlinks needs no lock of its own.
+ */
+static int symlink_child_allow_link(struct config_item *src,
+		struct config_item *target)
+{
+	to_symlink_child(src)->nlinks++;
+
+	return 0;
+}
+
+static void symlink_child_drop_link(struct config_item *src,
+		struct config_item *target)
+{
+	to_symlink_child(src)->nlinks--;
+}
+
+static void symlink_child_release(struct config_item *item)
+{
+	kfree(to_symlink_child(item));
+}
+
+static const struct configfs_item_operations symlink_child_item_ops = {
+	.release	= symlink_child_release,
+	.allow_link	= symlink_child_allow_link,
+	.drop_link	= symlink_child_drop_link,
+};
+
+static const struct config_item_type symlink_child_type = {
+	.ct_item_ops	= &symlink_child_item_ops,
+	.ct_attrs	= symlink_child_attrs,
+	.ct_owner	= THIS_MODULE,
+};
+
+static struct config_item *symlink_children_make_item(
+		struct config_group *group, const char *name)
+{
+	struct symlink_child *symlink_child;
+
+	symlink_child = kzalloc_obj(*symlink_child, GFP_KERNEL);
+	if (!symlink_child)
+		return ERR_PTR(-ENOMEM);
+
+	config_item_init_type_name(&symlink_child->item, name,
+				   &symlink_child_type);
+
+	return &symlink_child->item;
+}
+
+static ssize_t symlink_children_description_show(struct config_item *item,
+		char *page)
+{
+	return sprintf(page,
+"[04-symlink-children]\n"
+"\n"
+"This subsystem allows the creation of child config_items that\n"
+"symlink(2) can point at other config_items from.  Each child\n"
+"reports the number of links it holds.\n");
+}
+
+CONFIGFS_ATTR_RO(symlink_children_, description);
+
+static struct configfs_attribute *symlink_children_attrs[] = {
+	&symlink_children_attr_description,
+	NULL,
+};
+
+static const struct configfs_group_operations symlink_children_group_ops = {
+	.make_item	= symlink_children_make_item,
+};
+
+static const struct config_item_type symlink_children_type = {
+	.ct_group_ops	= &symlink_children_group_ops,
+	.ct_attrs	= symlink_children_attrs,
+	.ct_owner	= THIS_MODULE,
+};
+
+static struct configfs_subsystem symlink_children_subsys = {
+	.su_group = {
+		.cg_item = {
+			.ci_namebuf = "04-symlink-children",
+			.ci_type = &symlink_children_type,
+		},
+	},
+};
+
+/* ----------------------------------------------------------------- */
+
 /*
  * We're now done with our subsystem definitions.
  * For convenience in this module, here's a list of them all.  It
@@ -324,6 +444,7 @@ static struct configfs_subsystem *example_subsys[] = {
 	&childless_subsys.subsys,
 	&simple_children_subsys,
 	&group_children_subsys,
+	&symlink_children_subsys,
 	NULL,
 };
 
diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c
index d1123ebbfab906..27dc88bdf7a0ef 100644
--- a/samples/damon/mtier.c
+++ b/samples/damon/mtier.c
@@ -52,6 +52,11 @@ module_param(detect_node_addresses, bool, 0600);
 
 static struct damon_ctx *ctxs[2];
 
+/*
+ * Use phys_addr_t instead of damon_addr_range (unsigned long) for physical
+ * addresses.  On 32-bit systems with more than 4GB memory, phys_addr_t will
+ * be 64-bit while unsigned long is 32-bit.
+ */
 struct region_range {
 	phys_addr_t start;
 	phys_addr_t end;
@@ -156,9 +161,6 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote)
 	if (!scheme)
 		goto free_out;
 	damon_set_schemes(ctx, &scheme, 1);
-	/* zero target value causes division by zero in damos_quota_store() */
-	if (!node0_mem_used_bp || !node0_mem_free_bp)
-		goto free_out;
 	quota_goal = damos_new_quota_goal(
 			promote ? DAMOS_QUOTA_NODE_MEM_USED_BP :
 			DAMOS_QUOTA_NODE_MEM_FREE_BP,
diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c
index 842099bd622861..83ddf12811d57f 100644
--- a/samples/damon/prcl.c
+++ b/samples/damon/prcl.c
@@ -32,7 +32,6 @@ module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
 MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_PRCL");
 
 static struct damon_ctx *ctx;
-static struct pid *target_pidp;
 
 static int damon_sample_prcl_repeat_call_fn(void *data)
 {
@@ -79,12 +78,10 @@ static int damon_sample_prcl_start(void)
 		return -ENOMEM;
 	}
 	damon_add_target(ctx, target);
-	target_pidp = find_get_pid(target_pid);
-	if (!target_pidp) {
+	if (damon_set_target_pid(target, target_pid)) {
 		damon_destroy_ctx(ctx);
 		return -EINVAL;
 	}
-	target->pid = target_pidp;
 
 	scheme = damon_new_scheme(
 			&(struct damos_access_pattern) {
diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c
index 37fd5da2015885..53944aea8428ea 100644
--- a/samples/damon/wsse.c
+++ b/samples/damon/wsse.c
@@ -33,7 +33,6 @@ module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
 MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_WSSE");
 
 static struct damon_ctx *ctx;
-static struct pid *target_pidp;
 
 static int damon_sample_wsse_repeat_call_fn(void *data)
 {
@@ -79,12 +78,10 @@ static int damon_sample_wsse_start(void)
 		return -ENOMEM;
 	}
 	damon_add_target(ctx, target);
-	target_pidp = find_get_pid(target_pid);
-	if (!target_pidp) {
+	if (damon_set_target_pid(target, target_pid)) {
 		damon_destroy_ctx(ctx);
 		return -EINVAL;
 	}
-	target->pid = target_pidp;
 
 	err = damon_start(&ctx, 1, true);
 	if (err) {
diff --git a/samples/rpmsg/rpmsg_client_sample.c b/samples/rpmsg/rpmsg_client_sample.c
index ae508166228391..8c629065c64cf3 100644
--- a/samples/rpmsg/rpmsg_client_sample.c
+++ b/samples/rpmsg/rpmsg_client_sample.c
@@ -52,6 +52,7 @@ static int rpmsg_sample_probe(struct rpmsg_device *rpdev)
 {
 	int ret;
 	struct instance_data *idata;
+	ssize_t mtu, msg_len;
 
 	dev_info(&rpdev->dev, "new channel: 0x%x -> 0x%x!\n",
 					rpdev->src, rpdev->dst);
@@ -62,8 +63,25 @@ static int rpmsg_sample_probe(struct rpmsg_device *rpdev)
 
 	dev_set_drvdata(&rpdev->dev, idata);
 
+	mtu = rpmsg_get_mtu(rpdev->ept);
+	if (mtu < 0) {
+		dev_warn(&rpdev->dev, "invalid rpmsg MTU size = %zd\n", mtu);
+		return mtu;
+	}
+
+	dev_info(&rpdev->dev, "rpmsg MTU size = %zd\n", mtu);
+
+	msg_len = strlen(MSG);
+	/* make sure our message fits in a single rpmsg buffer */
+	if (msg_len > mtu) {
+		dev_err(&rpdev->dev,
+			"message size %zd exceeds rpmsg MTU size %zd\n",
+			msg_len, mtu);
+		return -EMSGSIZE;
+	}
+
 	/* send a message to our remote processor */
-	ret = rpmsg_send(rpdev->ept, MSG, strlen(MSG));
+	ret = rpmsg_send(rpdev->ept, MSG, msg_len);
 	if (ret) {
 		dev_err(&rpdev->dev, "rpmsg_send failed: %d\n", ret);
 		return ret;
diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs
index bd60034ded23a0..ffb693544673ff 100644
--- a/samples/rust/rust_dma.rs
+++ b/samples/rust/rust_dma.rs
@@ -5,7 +5,10 @@
 //! To make this driver probe, QEMU must be run with `-device pci-testdev`.
 
 use kernel::{
-    device::Core,
+    device::{
+        Bound,
+        Core, //
+    },
     dma::{
         Coherent,
         DataDirection,
@@ -23,14 +26,15 @@ use kernel::{
     scatterlist::{
         Owned,
         SGTable, //
-    },
-    sync::aref::ARef, //
+    }, //
 };
 
+struct DmaSampleDriver;
+
 #[pin_data(PinnedDrop)]
-struct DmaSampleDriver {
-    pdev: ARef<pci::Device>,
-    ca: Coherent<[MyStruct]>,
+struct DmaSampleData<'bound> {
+    pdev: &'bound pci::Device<Bound>,
+    ca: Coherent<'bound, [MyStruct]>,
     #[pin]
     sgt: SGTable<Owned<VVec<u8>>>,
 }
@@ -67,13 +71,13 @@ kernel::pci_device_table!(
 
 impl pci::Driver for DmaSampleDriver {
     type IdInfo = ();
-    type Data<'bound> = Self;
+    type Data<'bound> = DmaSampleData<'bound>;
     const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE;
 
     fn probe<'bound>(
         pdev: &'bound pci::Device<Core<'_>>,
         _info: Option<&'bound Self::IdInfo>,
-    ) -> impl PinInit<Self, Error> + 'bound {
+    ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound {
         pin_init::pin_init_scope(move || {
             dev_info!(pdev, "Probe DMA test driver.\n");
 
@@ -82,7 +86,7 @@ impl pci::Driver for DmaSampleDriver {
             // SAFETY: There are no concurrent calls to DMA allocation and mapping primitives.
             unsafe { pdev.dma_set_mask_and_coherent(mask)? };
 
-            let ca: Coherent<[MyStruct]> =
+            let ca: Coherent<'_, [MyStruct]> =
                 Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?;
 
             for (i, value) in TEST_VALUES.into_iter().enumerate() {
@@ -94,8 +98,8 @@ impl pci::Driver for DmaSampleDriver {
 
             let sgt = SGTable::new(pdev.as_ref(), pages, DataDirection::ToDevice, GFP_KERNEL);
 
-            Ok(try_pin_init!(Self {
-                pdev: pdev.into(),
+            Ok(try_pin_init!(Self::Data {
+                pdev,
                 ca,
                 sgt <- sgt,
             }))
@@ -103,7 +107,7 @@ impl pci::Driver for DmaSampleDriver {
     }
 }
 
-impl DmaSampleDriver {
+impl DmaSampleData<'_> {
     fn check_dma(&self) {
         for (i, value) in TEST_VALUES.into_iter().enumerate() {
             let val0 = io_read!(self.ca, [panic: i].h);
@@ -116,7 +120,7 @@ impl DmaSampleDriver {
 }
 
 #[pinned_drop]
-impl PinnedDrop for DmaSampleDriver {
+impl PinnedDrop for DmaSampleData<'_> {
     fn drop(self: Pin<&mut Self>) {
         dev_info!(self.pdev, "Unload DMA test driver.\n");
 
diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs
index 2282191e629236..13b035a957568f 100644
--- a/samples/rust/rust_driver_pci.rs
+++ b/samples/rust/rust_driver_pci.rs
@@ -23,6 +23,8 @@ mod regs {
     use super::*;
 
     register! {
+        base: kernel::io::Region<END>;
+
         pub(super) TEST(u8) @ 0x0 {
             7:0 index => TestIndex;
         }
@@ -102,6 +104,8 @@ impl SampleDriverData<'_> {
 
         // Some PCI configuration space registers.
         register! {
+            base: pci::Normal;
+
             VENDOR_ID(u16) @ 0x0 {
                 15:0 vendor_id;
             }
diff --git a/scripts/.gitignore b/scripts/.gitignore
index 4215c2208f7e41..70045654391074 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 /asn1_compiler
+/gen_init_cpio
 /gen_packed_field_checks
 /generate_rust_target
 /insert-sys-cert
diff --git a/scripts/Kconfig.include b/scripts/Kconfig.include
index fc10671c297cf3..622d21e6953f5a 100644
--- a/scripts/Kconfig.include
+++ b/scripts/Kconfig.include
@@ -23,13 +23,33 @@ success = $(if-success,$(1),y,n)
 # Return n if <command> exits with 0, y otherwise
 failure = $(if-success,$(1),n,y)
 
+# $(cc-probe,<flag>)
+# The command $(cc-option) and $(cc-option-str) test <flag> with. Not meant
+# to be called directly; use one of those two instead.
+cc-probe = trap "rm -rf .tmp_$$" EXIT; mkdir .tmp_$$; $(CC) -Werror $(CLANG_FLAGS) $(1) -c -x c /dev/null -o .tmp_$$/tmp.o
+
 # $(cc-option,<flag>)
 # Return y if the compiler supports <flag>, n otherwise
-cc-option = $(success,trap "rm -rf .tmp_$$" EXIT; mkdir .tmp_$$; $(CC) -Werror $(CLANG_FLAGS) $(1) -c -x c /dev/null -o .tmp_$$/tmp.o)
+cc-option = $(success,$(cc-probe,$(1)))
+
+# $(cc-option-str,<flag>[,<fallback>])
+# Return <flag> if the compiler supports it, <fallback> otherwise.
+# <fallback> defaults to an empty string and is returned untested.
+cc-option-str = $(if-success,$(cc-probe,$(1)),$(1),$(2))
+
+# $(ld-probe,<flag>)
+# The command $(ld-option) and $(ld-option-str) test <flag> with. Not meant
+# to be called directly; use one of those two instead.
+ld-probe = $(LD) -v $(1)
 
 # $(ld-option,<flag>)
 # Return y if the linker supports <flag>, n otherwise
-ld-option = $(success,$(LD) -v $(1))
+ld-option = $(success,$(ld-probe,$(1)))
+
+# $(ld-option-str,<flag>[,<fallback>])
+# Return <flag> if the linker supports it, <fallback> otherwise.
+# <fallback> defaults to an empty string and is returned untested.
+ld-option-str = $(if-success,$(ld-probe,$(1)),$(1),$(2))
 
 # $(as-instr,<instr>)
 # Return y if the assembler supports <instr>, n otherwise
@@ -61,9 +81,8 @@ ld-version := $(shell,set -- $(ld-info) && echo $2)
 # machine bit flags
 #  $(m32-flag): -m32 if the compiler supports it, or an empty string otherwise.
 #  $(m64-flag): -m64 if the compiler supports it, or an empty string otherwise.
-cc-option-bit = $(if-success,$(CC) -Werror $(1) -E -x c /dev/null -o /dev/null,$(1))
-m32-flag := $(cc-option-bit,-m32)
-m64-flag := $(cc-option-bit,-m64)
+m32-flag := $(cc-option-str,-m32)
+m64-flag := $(cc-option-str,-m64)
 
 # Test whether the compiler can link userspace applications
 cc_can_link_user = $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(1))
diff --git a/scripts/Kconfig.toolchain b/scripts/Kconfig.toolchain
new file mode 100644
index 00000000000000..d708a175fe4866
--- /dev/null
+++ b/scripts/Kconfig.toolchain
@@ -0,0 +1,272 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config CC_VERSION_TEXT
+	string
+	default "$(CC_VERSION_TEXT)"
+	help
+	  This is used in unclear ways:
+
+	  - Re-run Kconfig when the compiler is updated
+	    The 'default' property references the environment variable,
+	    CC_VERSION_TEXT so it is recorded in include/config/auto.conf.cmd.
+	    When the compiler is updated, Kconfig will be invoked.
+
+	  - Ensure full rebuild when the compiler is updated
+	    include/linux/compiler-version.h contains this option in the comment
+	    line so fixdep adds include/config/CC_VERSION_TEXT into the
+	    auto-generated dependency. When the compiler is updated, syncconfig
+	    will touch it and then every file will be rebuilt.
+
+config CC_IS_GCC
+	def_bool $(success,test "$(cc-name)" = GCC)
+
+config GCC_VERSION
+	int
+	default $(cc-version) if CC_IS_GCC
+	default 0
+
+config CC_IS_CLANG
+	def_bool $(success,test "$(cc-name)" = Clang)
+
+config CLANG_VERSION
+	int
+	default $(cc-version) if CC_IS_CLANG
+	default 0
+
+config AS_IS_GNU
+	def_bool $(success,test "$(as-name)" = GNU)
+
+config AS_IS_LLVM
+	def_bool $(success,test "$(as-name)" = LLVM)
+
+config AS_VERSION
+	int
+	# Use clang version if this is the integrated assembler
+	default CLANG_VERSION if AS_IS_LLVM
+	default $(as-version)
+
+config LD_IS_BFD
+	def_bool $(success,test "$(ld-name)" = BFD)
+
+config LD_VERSION
+	int
+	default $(ld-version) if LD_IS_BFD
+	default 0
+
+config LD_IS_LLD
+	def_bool $(success,test "$(ld-name)" = LLD)
+
+config LLD_VERSION
+	int
+	default $(ld-version) if LD_IS_LLD
+	default 0
+
+config RUSTC_VERSION
+	int
+	default $(rustc-version)
+	help
+	  It does not depend on `RUST` since that one may need to use the version
+	  in a `depends on`.
+
+config RUST_IS_AVAILABLE
+	def_bool $(success,$(srctree)/scripts/rust_is_available.sh)
+	help
+	  This shows whether a suitable Rust toolchain is available (found).
+
+	  Please see Documentation/rust/quick-start.rst for instructions on how
+	  to satisfy the build requirements of Rust support.
+
+	  In particular, the Makefile target 'rustavailable' is useful to check
+	  why the Rust toolchain is not being detected.
+
+config RUSTC_LLVM_VERSION
+	int
+	default $(rustc-llvm-version)
+
+config RUSTC_LLVM_MAJOR_VERSION
+	int
+	default $(shell,expr $(rustc-llvm-version) / 10000)
+
+config RUSTC_CLANG_LLVM_COMPATIBLE
+	bool
+	default y if CC_IS_CLANG && RUSTC_LLVM_MAJOR_VERSION = $(shell,expr $(cc-version) / 10000)
+	help
+	  This indicates whether Rust and Clang use LLVM of the same major
+	  version.
+
+	  Operations involving handling LLVM IR or bitcode (e.g. cross-language
+	  LTO) require the same LLVM major version to work properly. For best
+	  compatibility it is recommended that the exact same LLVM is used.
+
+config ARCH_HAS_CC_CAN_LINK
+	bool
+
+config CC_CAN_LINK
+	bool
+	default ARCH_CC_CAN_LINK if ARCH_HAS_CC_CAN_LINK
+	default $(cc_can_link_user,$(m64-flag)) if 64BIT
+	default $(cc_can_link_user,$(m32-flag))
+
+# Fixed in GCC 14, 13.3, 12.4 and 11.5
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=113921
+config GCC_ASM_GOTO_OUTPUT_BROKEN
+	bool
+	depends on CC_IS_GCC
+	default y if GCC_VERSION < 110500
+	default y if GCC_VERSION >= 120000 && GCC_VERSION < 120400
+	default y if GCC_VERSION >= 130000 && GCC_VERSION < 130300
+
+config CC_HAS_ASM_GOTO_OUTPUT
+	def_bool y
+	depends on !GCC_ASM_GOTO_OUTPUT_BROKEN
+	depends on $(success,echo 'int foo(int x) { asm goto ("": "=r"(x) ::: bar); return x; bar: return 0; }' | $(CC) -x c - -c -o /dev/null)
+
+config CC_HAS_ASM_GOTO_TIED_OUTPUT
+	depends on CC_HAS_ASM_GOTO_OUTPUT
+	# Detect buggy gcc and clang, fixed in gcc-11 clang-14.
+	def_bool $(success,echo 'int foo(int *x) { asm goto (".long (%l[bar]) - .": "+m"(*x) ::: bar); return *x; bar: return 0; }' | $CC -x c - -c -o /dev/null)
+
+config TOOLS_SUPPORT_RELR
+	def_bool $(success,env "CC=$(CC)" "LD=$(LD)" "NM=$(NM)" "OBJCOPY=$(OBJCOPY)" $(srctree)/scripts/tools-support-relr.sh)
+
+config CC_HAS_ASM_INLINE
+	def_bool $(success,echo 'void foo(void) { asm inline (""); }' | $(CC) -x c - -c -o /dev/null)
+
+config CC_HAS_ASSUME
+	bool
+	# clang needs to be at least 19.1.0 since the meaning of the assume
+	# attribute changed:
+	# https://github.com/llvm/llvm-project/commit/c44fa3e8a9a44c2e9a575768a3c185354b9f6c17
+	default y if CC_IS_CLANG && CLANG_VERSION >= 190100
+	# supported since gcc 13.1.0
+	# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106654
+	default y if CC_IS_GCC && GCC_VERSION >= 130100
+
+config CC_HAS_NO_PROFILE_FN_ATTR
+	def_bool $(success,echo '__attribute__((no_profile_instrument_function)) int x();' | $(CC) -x c - -c -o /dev/null -Werror)
+
+config CC_OPT_ZERO_INIT_PADDING_BITS
+	def_string "$(cc-option-str,-fzero-init-padding-bits=all)"
+
+config CC_OPT_NO_ADDRSIG
+	def_string "$(cc-option-str,-fno-addrsig)"
+
+config CC_OPT_DIAGNOSTICS_SHOW_CONTEXT
+	def_string "$(cc-option-str,-fdiagnostics-show-context=2)"
+
+config CC_OPT_DIAGNOSTICS_SHOW_INLINING_CHAIN
+	def_string "$(cc-option-str,-fdiagnostics-show-inlining-chain)"
+
+config CC_OPT_STRICT_FLEX_ARRAYS
+	def_string "$(cc-option-str,-fstrict-flex-arrays=3)"
+
+# gcc-10 renamed --param=allow-store-data-races=0 to -fno-allow-store-data-races
+config CC_OPT_ALLOW_STORE_DATA_RACES_PARAM
+	def_string "$(cc-option-str,--param=allow-store-data-races=0)" if CC_IS_GCC
+
+config CC_OPT_NO_ALLOW_STORE_DATA_RACES
+	def_string "$(cc-option-str,-fno-allow-store-data-races)" if CC_IS_GCC
+
+config CC_HAS_MNOP_MCOUNT
+	def_bool $(cc-option,-mnop-mcount)
+
+# s390-linux-gnu-gcc did not support -mfentry until gcc-9.
+config CC_HAS_MFENTRY
+	def_bool $(cc-option,-mfentry)
+
+config LD_OPT_FORCE_GROUP_ALLOCATION
+	def_string "$(ld-option-str,--force-group-allocation)"
+
+config LD_OPT_NO_WARN_RWX_SEGMENTS
+	def_string "$(ld-option-str,--no-warn-rwx-segments)"
+
+config CC_OPT_WNO_ADDRESS_OF_PACKED_MEMBER
+	def_string "-Wno-address-of-packed-member" if $(cc-option,-Waddress-of-packed-member)
+
+config CC_OPT_WNO_FORMAT_OVERFLOW_NON_KPRINTF
+	def_string "-Wno-format-overflow-non-kprintf" if CC_IS_CLANG && $(cc-option,-Wformat-overflow-non-kprintf)
+
+config CC_OPT_WNO_FORMAT_TRUNCATION_NON_KPRINTF
+	def_string "-Wno-format-truncation-non-kprintf" if CC_IS_CLANG && $(cc-option,-Wformat-truncation-non-kprintf)
+
+config CC_OPT_WNO_DEFAULT_CONST_INIT_UNSAFE
+	def_string "-Wno-default-const-init-unsafe" if CC_IS_CLANG && $(cc-option,-Wdefault-const-init-unsafe)
+
+config CC_OPT_WNO_DANGLING_POINTER
+	def_string "-Wno-dangling-pointer" if $(cc-option,-Wdangling-pointer)
+
+config CC_OPT_WVLA_LARGER_THAN
+	def_string "$(cc-option-str,-Wvla-larger-than=1)"
+
+config CC_HAS_WSTRINGOP_OVERFLOW
+	def_bool $(cc-option,-Wstringop-overflow)
+
+config CC_OPT_WNO_UNTERMINATED_STRING_INITIALIZATION
+	def_string "-Wno-unterminated-string-initialization" if $(cc-option,-Wunterminated-string-initialization)
+
+config CC_OPT_WERROR_DESIGNATED_INIT
+	def_string "$(cc-option-str,-Werror=designated-init)"
+
+config CC_OPT_WENUM_CONVERSION
+	def_string "$(cc-option-str,-Wenum-conversion)"
+
+config CC_OPT_WNO_PACKED_NOT_ALIGNED
+	def_string "-Wno-packed-not-aligned" if $(cc-option,-Wpacked-not-aligned)
+
+config CC_OPT_WNO_FORMAT_OVERFLOW
+	def_string "-Wno-format-overflow" if $(cc-option,-Wformat-overflow)
+
+config CC_OPT_WNO_STRINGOP_TRUNCATION
+	def_string "-Wno-stringop-truncation" if $(cc-option,-Wstringop-truncation)
+
+config CC_HAS_COUNTED_BY
+	bool
+	# clang needs to be at least 20.1.0 to avoid potential crashes
+	# when building structures that contain __counted_by
+	# https://github.com/ClangBuiltLinux/linux/issues/2114
+	# https://github.com/llvm/llvm-project/commit/160fb1121cdf703c3ef5e61fb26c5659eb581489
+	default y if CC_IS_CLANG && CLANG_VERSION >= 200100
+	# supported since gcc 15.1.0
+	# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108896
+	default y if CC_IS_GCC && GCC_VERSION >= 150100
+
+config CC_HAS_COUNTED_BY_PTR
+	bool
+	# supported since clang 22
+	default y if CC_IS_CLANG && CLANG_VERSION >= 220100
+	# supported since gcc 16.0.0
+	default y if CC_IS_GCC && GCC_VERSION >= 160000
+
+config CC_HAS_BROKEN_COUNTED_BY_REF
+	bool
+	# https://github.com/llvm/llvm-project/issues/182575
+	default y if CC_IS_CLANG && CLANG_VERSION < 220100
+
+config CC_HAS_ALLOC_TOKEN
+	def_bool $(cc-option,-falloc-token-max=123)
+
+config CC_HAS_MULTIDIMENSIONAL_NONSTRING
+	def_bool $(success,echo 'char tag[][4] __attribute__((__nonstring__)) = { };' | $(CC) $(CLANG_FLAGS) -x c - -c -o /dev/null -Werror)
+
+config LD_CAN_USE_KEEP_IN_OVERLAY
+	# ld.lld prior to 21.0.0 did not support KEEP within an overlay description
+	# https://github.com/llvm/llvm-project/pull/130661
+	def_bool LD_IS_BFD || LLD_VERSION >= 210000
+
+config RUSTC_HAS_SPAN_FILE
+	def_bool RUSTC_VERSION >= 108800
+
+config RUSTC_HAS_UNNECESSARY_TRANSMUTES
+	def_bool RUSTC_VERSION >= 108800
+
+config RUSTC_HAS_FILE_WITH_NUL
+	def_bool RUSTC_VERSION >= 108900
+
+config RUSTC_HAS_FILE_AS_C_STR
+	def_bool RUSTC_VERSION >= 109100
+
+config RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS
+	def_bool RUSTC_VERSION >= 109800
+
+config PAHOLE_VERSION
+	int
+	default "$(PAHOLE_VERSION)"
diff --git a/scripts/Makefile b/scripts/Makefile
index 3434a82a119f09..d7fbb7ca098b18 100644
--- a/scripts/Makefile
+++ b/scripts/Makefile
@@ -3,7 +3,7 @@
 # scripts contains sources for various helper programs used throughout
 # the kernel for the build process.
 
-hostprogs-always-$(CONFIG_KALLSYMS)			+= kallsyms
+hostprogs-always-y				+= kallsyms
 hostprogs-always-$(BUILD_C_RECORDMCOUNT)		+= recordmcount
 hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT)		+= sorttable
 hostprogs-always-$(CONFIG_ASN1)				+= asn1_compiler
@@ -13,6 +13,7 @@ hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS)		+= rustdoc_test_builder
 hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS)		+= rustdoc_test_gen
 hostprogs-always-$(CONFIG_TRACEPOINTS)			+= tracepoint-update
 
+kallsyms-objs := kallsyms.o kallsyms-sysmap.o elf-parse.o
 sorttable-objs := sorttable.o elf-parse.o
 tracepoint-update-objs := tracepoint-update.o elf-parse.o
 
@@ -30,6 +31,8 @@ rustdoc_test_builder-rust := y
 rustdoc_test_gen-rust := y
 
 HOSTCFLAGS_tracepoint-update.o = -I$(srctree)/tools/include
+HOSTCFLAGS_kallsyms.o = -I$(srctree)/tools/include
+HOSTCFLAGS_kallsyms-sysmap.o = -I$(srctree)/tools/include
 HOSTCFLAGS_elf-parse.o = -I$(srctree)/tools/include
 HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include
 HOSTLDLIBS_sorttable = -lpthread
@@ -53,6 +56,9 @@ ifdef CONFIG_BUILDTIME_MCOUNT_SORT
 HOSTCFLAGS_sorttable.o += -DMCOUNT_SORT_ENABLED
 endif
 
+hostprogs += gen_init_cpio
+always-$(CONFIG_BLK_DEV_INITRD) += gen_init_cpio
+
 # The following programs are only built on demand
 hostprogs += unifdef gen_packed_field_checks
 
diff --git a/scripts/Makefile.gcc-plugins b/scripts/Makefile.gcc-plugins
index b0e1423b09c213..9b98995805712f 100644
--- a/scripts/Makefile.gcc-plugins
+++ b/scripts/Makefile.gcc-plugins
@@ -8,6 +8,10 @@ ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
 endif
 export DISABLE_LATENT_ENTROPY_PLUGIN
 
+# Disable recording GCC options in debug info when building external modules to
+# avoid leaking absolute host build paths
+gcc-plugin-cflags-$(if $(KBUILD_EXTMOD),y) += -gno-record-gcc-switches
+
 # All the plugin CFLAGS are collected here in case a build target needs to
 # filter them out of the KBUILD_CFLAGS.
 GCC_PLUGINS_CFLAGS := $(strip $(addprefix -fplugin=$(objtree)/scripts/gcc-plugins/, $(gcc-plugin-y)) $(gcc-plugin-cflags-y)) -DGCC_PLUGINS
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 0a4fdd8bd975de..19e96b30a122f7 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -333,11 +333,11 @@ quiet_cmd_ar = AR      $@
 quiet_cmd_objcopy = OBJCOPY $@
 cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@
 
-# Gzip
+# Gzip, for the kernel image
 # ---------------------------------------------------------------------------
 
 quiet_cmd_gzip = GZIP    $@
-      cmd_gzip = cat $(real-prereqs) | $(KGZIP) -n -f -9 > $@
+      cmd_gzip = cat $(real-prereqs) | $(KPGZIP) -n -f -9 > $@
 
 # Bzip2
 # ---------------------------------------------------------------------------
diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index 6d36786ba31cef..948ad5b679d110 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -198,7 +198,7 @@ tar%-pkg: linux-$(KERNELRELEASE)-$(ARCH).tar.% FORCE
 	$(Q)$(MAKE) -f $(srctree)/Makefile INSTALL_MOD_PATH=$@/$(INSTALL_MOD_PATH) modules_install
 
 quiet_cmd_cpio = CPIO    $@
-      cmd_cpio = $(CONFIG_SHELL) $(srctree)/usr/gen_initramfs.sh -o $@ $<
+      cmd_cpio = $(CONFIG_SHELL) $(srctree)/scripts/gen_initramfs.sh -o $@ $<
 
 modules-$(KERNELRELEASE)-$(ARCH).cpio: .tmp_modules_cpio
 	$(call cmd,cpio)
diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux
index fcae1e432d9add..95b523fe4f9eeb 100644
--- a/scripts/Makefile.vmlinux
+++ b/scripts/Makefile.vmlinux
@@ -89,11 +89,8 @@ remove-section-$(CONFIG_ARCH_VMLINUX_NEEDS_RELOCS) += '.rel.*'
 
 remove-symbols := -w --strip-unneeded-symbol='__mod_device_table__*'
 
-# To avoid warnings: "empty loadable segment detected at ..." from GNU objcopy,
-# it is necessary to remove the PT_LOAD flag from the segment.
 quiet_cmd_strip_relocs = OBJCOPY $@
-      cmd_strip_relocs = $(OBJCOPY) $(patsubst %,--set-section-flags %=noload,$(remove-section-y)) $< $@; \
-                         $(OBJCOPY) $(addprefix --remove-section=,$(remove-section-y)) $(remove-symbols) $@
+      cmd_strip_relocs = $(OBJCOPY) $(addprefix --remove-section=,$(remove-section-y)) $(remove-symbols) $< $@
 
 targets += vmlinux
 vmlinux: vmlinux.unstripped FORCE
@@ -102,21 +99,8 @@ vmlinux: vmlinux.unstripped FORCE
 # modules.builtin.modinfo
 # ---------------------------------------------------------------------------
 
-# .modinfo in vmlinux.unstripped is aligned to 8 bytes for compatibility with
-# tools that expect vmlinux to have sufficiently aligned sections but the
-# additional bytes used for padding .modinfo to satisfy this requirement break
-# certain versions of kmod with
-#
-#   depmod: ERROR: kmod_builtin_iter_next: unexpected string without modname prefix
-#
-# Strip the trailing padding bytes after extracting .modinfo to comply with
-# what kmod expects to parse.
 quiet_cmd_modules_builtin_modinfo = GEN     $@
-      cmd_modules_builtin_modinfo = $(cmd_objcopy); \
-                                    sed -i 's/\x00\+$$/\x00/g' $@; \
-                                    chmod -x $@
-
-OBJCOPYFLAGS_modules.builtin.modinfo := -j .modinfo -O binary
+      cmd_modules_builtin_modinfo = $(OBJCOPY) -O binary -j .modinfo --dump-section .modinfo=$@ $< /dev/null
 
 targets += modules.builtin.modinfo
 modules.builtin.modinfo: vmlinux.unstripped FORCE
diff --git a/scripts/Makefile.warn b/scripts/Makefile.warn
index 35af7d6c6d1832..556fa249ca071a 100644
--- a/scripts/Makefile.warn
+++ b/scripts/Makefile.warn
@@ -17,7 +17,7 @@ KBUILD_CFLAGS += -Werror=strict-prototypes
 KBUILD_CFLAGS += -Wno-format-security
 KBUILD_CFLAGS += -Wno-trigraphs
 KBUILD_CFLAGS += -Wno-frame-address
-KBUILD_CFLAGS += $(call cc-option, -Wno-address-of-packed-member)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_ADDRESS_OF_PACKED_MEMBER)
 KBUILD_CFLAGS += -Wmissing-declarations
 KBUILD_CFLAGS += -Wmissing-prototypes
 
@@ -30,8 +30,8 @@ KBUILD_CFLAGS-$(CONFIG_CC_NO_ARRAY_BOUNDS) += -Wno-array-bounds
 ifdef CONFIG_CC_IS_CLANG
 # Clang checks for overflow/truncation with '%p', while GCC does not:
 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111219
-KBUILD_CFLAGS += $(call cc-option, -Wno-format-overflow-non-kprintf)
-KBUILD_CFLAGS += $(call cc-option, -Wno-format-truncation-non-kprintf)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_FORMAT_OVERFLOW_NON_KPRINTF)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_FORMAT_TRUNCATION_NON_KPRINTF)
 
 # Clang may emit a warning when a const variable, such as the dummy variables
 # in typecheck(), or const member of an aggregate type are not initialized,
@@ -43,7 +43,7 @@ KBUILD_CFLAGS += $(call cc-option, -Wno-format-truncation-non-kprintf)
 # disabled with this same switch, there should not be too much coverage lost
 # because -Wuninitialized will still flag when an uninitialized const variable
 # is used.
-KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_DEFAULT_CONST_INIT_UNSAFE)
 else
 
 # gcc inanely warns about local variables called 'main'
@@ -54,7 +54,7 @@ endif
 KBUILD_CFLAGS += -Wno-type-limits
 
 # These result in bogus false positives
-KBUILD_CFLAGS += $(call cc-option, -Wno-dangling-pointer)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_DANGLING_POINTER)
 
 # Stack Variable Length Arrays (VLAs) must not be used in the kernel.
 # Function array parameters should, however, be usable, but -Wvla will
@@ -62,7 +62,7 @@ KBUILD_CFLAGS += $(call cc-option, -Wno-dangling-pointer)
 # types, so depend on GCC for now to keep stack VLAs out of the tree.
 # https://github.com/llvm/llvm-project/issues/57098
 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98217
-KBUILD_CFLAGS += $(call cc-option,-Wvla-larger-than=1)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WVLA_LARGER_THAN)
 
 # disable pointer signed / unsigned warnings in gcc 4.0
 KBUILD_CFLAGS += -Wno-pointer-sign
@@ -73,11 +73,11 @@ KBUILD_CFLAGS += -Wno-pointer-sign
 KBUILD_CFLAGS += -Wcast-function-type
 
 # Currently, disable -Wstringop-overflow for GCC 11, globally.
-KBUILD_CFLAGS-$(CONFIG_CC_NO_STRINGOP_OVERFLOW) += $(call cc-option, -Wno-stringop-overflow)
-KBUILD_CFLAGS-$(CONFIG_CC_STRINGOP_OVERFLOW) += $(call cc-option, -Wstringop-overflow)
+KBUILD_CFLAGS-$(CONFIG_CC_NO_STRINGOP_OVERFLOW) += $(if $(CONFIG_CC_HAS_WSTRINGOP_OVERFLOW),-Wno-stringop-overflow)
+KBUILD_CFLAGS-$(CONFIG_CC_STRINGOP_OVERFLOW) += $(if $(CONFIG_CC_HAS_WSTRINGOP_OVERFLOW),-Wstringop-overflow)
 
 # Currently, disable -Wunterminated-string-initialization as broken
-KBUILD_CFLAGS += $(call cc-option, -Wno-unterminated-string-initialization)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_UNTERMINATED_STRING_INITIALIZATION)
 
 # The allocators already balk at large sizes, so silence the compiler
 # warnings for bounds checks involving those possible values. While
@@ -100,10 +100,10 @@ KBUILD_CFLAGS += -Werror=date-time
 KBUILD_CFLAGS += -Werror=incompatible-pointer-types
 
 # Require designated initializers for all marked structures
-KBUILD_CFLAGS += $(call cc-option,-Werror=designated-init)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WERROR_DESIGNATED_INIT)
 
 # Warn if there is an enum types mismatch
-KBUILD_CFLAGS += $(call cc-option,-Wenum-conversion)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WENUM_CONVERSION)
 
 KBUILD_CFLAGS += -Wunused
 
@@ -125,12 +125,12 @@ else
 # Suppress them by using -Wno... except for W=1.
 KBUILD_CFLAGS += -Wno-unused-but-set-variable
 KBUILD_CFLAGS += -Wno-unused-const-variable
-KBUILD_CFLAGS += $(call cc-option, -Wno-packed-not-aligned)
-KBUILD_CFLAGS += $(call cc-option, -Wno-format-overflow)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_PACKED_NOT_ALIGNED)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_FORMAT_OVERFLOW)
 ifdef CONFIG_CC_IS_GCC
 KBUILD_CFLAGS += -Wno-format-truncation
 endif
-KBUILD_CFLAGS += $(call cc-option, -Wno-stringop-truncation)
+KBUILD_CFLAGS += $(CONFIG_CC_OPT_WNO_STRINGOP_TRUNCATION)
 
 KBUILD_CFLAGS += -Wno-override-init # alias for -Wno-initializer-overrides in clang
 
diff --git a/scripts/check-function-names.sh b/scripts/check-function-names.sh
index 08071133e5a512..94883e690627da 100755
--- a/scripts/check-function-names.sh
+++ b/scripts/check-function-names.sh
@@ -13,7 +13,8 @@ if [ ! -f "$objfile" ]; then
 	exit 1
 fi
 
-bad_symbols=$(${NM:-nm} "$objfile" | awk '$2 ~ /^[TtWw]$/ {print $3}' | grep -E '^(startup|exit|split|unlikely|hot|unknown)(\.|$)')
+bad_symbols=$(${NM:-nm} -p "$objfile" | awk '$2 ~ /^[TtWw]$/ {print $3}' |
+	      grep -E '^(startup|exit|split|unlikely|hot|unknown)(\.|$)')
 
 if [ -n "$bad_symbols" ]; then
 	echo "$bad_symbols" | while read -r sym; do
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 8a7787d228a63d..598e3ed743bd9e 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -63,6 +63,7 @@
 my $max_line_length = 100;
 my $ignore_perl_version = 0;
 my $spdx_cxx_comments = 0;
+my $userspace;
 my $minimum_perl_version = 5.10.0;
 my $min_conf_desc_length = 4;
 my $spelling_file = "$D/spelling.txt";
@@ -143,6 +144,7 @@ sub help {
                              (required by old toolchains), allow also C++
                              comments (//).
                              NOTE: it should *not* be used for Linux mainline.
+  --userspace                Force rules specific for userspace.
   --codespell                Use the codespell dictionary for spelling/typos
                              (default:$codespellfile)
   --codespellfile            Use this codespell dictionary
@@ -358,6 +360,7 @@ sub load_docs {
 	'codespell!'	=> \$codespell,
 	'codespellfile=s'	=> \$user_codespellfile,
 	'typedefsfile=s'	=> \$typedefsfile,
+	'userspace!'	=> \$userspace,
 	'color=s'	=> \$color,
 	'no-color'	=> \$color,	#keep old behaviors of -nocolor
 	'nocolor'	=> \$color,	#keep old behaviors of -nocolor
@@ -1206,6 +1209,8 @@ sub seed_camelcase_includes {
 		my $git_last_include_commit = `${git_command} log --no-merges --pretty=format:"%h%n" -1 -- include`;
 		chomp $git_last_include_commit;
 		$camelcase_cache = ".checkpatch-camelcase.git.$git_last_include_commit";
+	} elsif (not defined $root) {
+		return;
 	} else {
 		my $last_mod_date = 0;
 		$files = `find $root/include -name "*.h"`;
@@ -2665,7 +2670,12 @@ sub exclude_global_initialisers {
 
 sub is_userspace {
     my ($realfile) = @_;
-    return ($realfile =~ m@^tools/@ || $realfile =~ m@^scripts/@);
+
+    return $userspace if (defined $userspace);
+
+    return ($realfile =~ m@^tools/@ ||
+		$realfile =~ m@^scripts/@ ||
+		$realfile =~ m@^arch/.*/tools/@);
 }
 
 sub process {
@@ -5777,7 +5787,21 @@ sub process {
 			my ($s, $c) = ($stat, $cond);
 			my $fixed_assign_in_if = 0;
 
+			# ACQUIRE_ERR() and its wrappers, e.g. PM_RUNTIME_ACQUIRE_ERR()
+			# and IIO_DEV_ACQUIRE_FAILED(), are meant to be evaluated in an
+			# if condition, with the error assigned in the condition:
+			#	if ((rc = ACQUIRE_ERR(name, &lock)))
+			# Allow that only when every assignment in the condition assigns
+			# the result of such a call, so that a mixed condition keeps
+			# getting flagged:
+			#	if ((rc = regular_function()) || (ret = ACQUIRE_ERR(name, &lock)))
+			my $assign_in_if = 0;
 			if ($c =~ /\bif\s*\(.*[^<>!=]=[^=].*/s) {
+				my $has_assignment = $c =~ /\b$Lval\s*=\s*[^,)&|=]+/;
+				my $has_other_assignment = $c =~ /\b$Lval\s*=\s*(?!\s*\w*ACQUIRE_(?:ERR|FAILED)\s*\()[^,)&|=]+/;
+				$assign_in_if = !$has_assignment || $has_other_assignment;
+			}
+			if ($assign_in_if) {
 				if (ERROR("ASSIGN_IN_IF",
 					  "do not use assignment in if condition\n" . $herecurr) &&
 				    $fix && $perl_version_ok) {
@@ -5946,6 +5970,8 @@ sub process {
 #Ignore SI style variants like nS, mV and dB
 #(ie: max_uV, regulator_min_uA_show, RANGE_mA_VALUE)
 			    $var !~ /^(?:[a-z0-9_]*|[A-Z0-9_]*)?_?[a-z][A-Z](?:_[a-z0-9_]+|_[A-Z0-9_]+)?$/ &&
+#Ignore <inttypes.h> format macros (e.g. PRIu64, SCNu64)
+			    (is_userspace($realfile) ? $var !~ /^(?:PRI|SCN)[dioux][A-Z0-9]+$/ : 1) &&
 #Ignore some three character SI units explicitly, like MiB and KHz
 			    $var !~ /^(?:[a-z_]*?)_?(?:[KMGT]iB|[KMGT]?Hz)(?:_[a-z_]+)?$/) {
 				while ($var =~ m{\b($Ident)}g) {
@@ -6684,7 +6710,8 @@ sub process {
 		}
 
 # prefer usleep_range over udelay
-		if ($line =~ /\budelay\s*\(\s*(\d+)\s*\)/) {
+		if (!is_userspace($realfile) &&
+			$line =~ /\budelay\s*\(\s*(\d+)\s*\)/) {
 			my $delay = $1;
 			# ignore udelay's < 10, however
 			if (! ($delay < 10) ) {
@@ -6698,7 +6725,8 @@ sub process {
 		}
 
 # warn about unexpectedly long msleep's
-		if ($line =~ /\bmsleep\s*\((\d+)\);/) {
+		if (!is_userspace($realfile) &&
+			$line =~ /\bmsleep\s*\((\d+)\);/) {
 			if ($1 < 20) {
 				WARN("MSLEEP",
 				     "msleep < 20ms can sleep for up to 20ms; see function description of msleep().\n" . $herecurr);
@@ -6920,7 +6948,7 @@ sub process {
 
 # check for c99 types like uint8_t used outside of uapi/ and tools/
 		if ($realfile !~ m@\binclude/uapi/@ &&
-		    $realfile !~ m@\btools/@ &&
+		    !is_userspace($realfile) &&
 		    $line =~ /\b($Declare)\s*$Ident\s*[=;,\[]/) {
 			my $type = $1;
 			if ($type =~ /\b($typeC99Typedefs)\b/) {
@@ -7170,6 +7198,7 @@ sub process {
 # check usleep_range arguments
 		if ($perl_version_ok &&
 		    defined $stat &&
+		    !is_userspace($realfile) &&
 		    $stat =~ /^\+(?:.*?)\busleep_range\s*\(\s*($FuncArg)\s*,\s*($FuncArg)\s*\)/) {
 			my $min = $1;
 			my $max = $7;
@@ -7413,6 +7442,7 @@ sub process {
 
 # check for #defines like: 1 << <digit> that could be BIT(digit), it is not exported to uapi
 		if ($realfile !~ m@^include/uapi/@ &&
+		    !is_userspace($realfile) &&
 		    $line =~ /#\s*define\s+\w+\s+\(?\s*1\s*([ulUL]*)\s*\<\<\s*(?:\d+|$Ident)\s*\)?/) {
 			my $ull = "";
 			$ull = "_ULL" if (defined($1) && $1 =~ /ll/i);
diff --git a/scripts/checkstack.pl b/scripts/checkstack.pl
index 14ce31f732ee8a..f8011e789a9b45 100755
--- a/scripts/checkstack.pl
+++ b/scripts/checkstack.pl
@@ -66,8 +66,10 @@
 		#c0105234:       81 ec ac 05 00 00       sub    $0x5ac,%esp
 		# or
 		#    2f60:    48 81 ec e8 05 00 00       sub    $0x5e8,%rsp
-		$re = qr/^.*[as][du][db]    \$(0x$x{1,8}),\%(e|r)sp$/o;
-		$dre = qr/^.*[as][du][db]    (%.*),\%(e|r)sp$/o;
+		# or
+		#       0: 48 81 ec e8 05 00 00          subq   $0x5e8, %rsp
+		$re = qr/^.*[as][du][db][ql]?\s+\$(0x$x{1,8}),\s*\%(e|r)sp/o;
+		$dre = qr/^.*[as][du][db][ql]?\s+(%.*),\s*\%(e|r)sp/o;
 	} elsif ($arch eq 'm68k') {
 		#    2b6c:       4e56 fb70       linkw %fp,#-1168
 		#  1df770:       defc ffe4       addaw #-28,%sp
diff --git a/scripts/coccinelle/api/kmalloc_objs.cocci b/scripts/coccinelle/api/kmalloc_objs.cocci
index e9a415b7b6f45a..0a98ddf03eca48 100644
--- a/scripts/coccinelle/api/kmalloc_objs.cocci
+++ b/scripts/coccinelle/api/kmalloc_objs.cocci
@@ -24,27 +24,57 @@ def alloc_array(name):
 		print(f"Unknown transform for {name}", file=sys.stderr)
 	return func
 
-// This excludes anything that is assigning to or from integral types or
-// string literals. Everything else gets the sizeof() extracted for the
-// kmalloc_obj() type/var argument. sizeof(void *) is also excluded because
-// it will need case-by-case double-checking to make sure the right type is
+// Allocations sized by a byte-sized type (BYTE_TYPES) or a string literal,
+// and allocations assigned to a pointer to a byte-sized type, are byte
+// buffers and are left alone. sizeof(void *) is also excluded because it
+// will need case-by-case double-checking to make sure the right type is
 // being assigned.
+//
+// Allocations sized by a multi-byte integral type (MULTIBYTE_TYPES) are
+// converted when they are assigned to a pointer to that same type.
+// Otherwise they are left alone: the target may be a pointer to an array
+// of that type, such as "s16 (*pairs)[2]", for which the converted
+// allocation would have the wrong pointer type. For the same reason,
+// arrays of pointers to integral types sized as sizeof(char *) and the
+// like are left alone. Allocations of other types assigned to a pointer to
+// a multi-byte integral type are left alone too.
+//
+// Everything else gets the sizeof() extracted for the kmalloc_obj()
+// type/var argument.
+//
+// The first matching alternative below wins, so the exclusions must come
+// before the more general conversions.
 @direct depends on patch && !(file in "tools") && !(file in "samples")@
 typedef u8, u16, u32, u64;
 typedef __u8, __u16, __u32, __u64;
 typedef uint8_t, uint16_t, uint32_t, uint64_t;
+typedef s8, s16, s32, s64;
+typedef __s8, __s16, __s32, __s64;
+typedef int8_t, int16_t, int32_t, int64_t;
 typedef uchar, ushort, uint, ulong;
 typedef __le16, __le32, __le64;
 typedef __be16, __be32, __be64;
 typedef wchar_t;
-type INTEGRAL = {u8,__u8,uint8_t,char,unsigned char,uchar,wchar_t,
-		 u16,__u16,uint16_t,unsigned short,ushort,
-		 u32,__u32,uint32_t,unsigned int,uint,
-		 u64,__u64,uint64_t,unsigned long,ulong,
-		 __le16,__le32,__le64,__be16,__be32,__be64};
+type BYTE_TYPES = {char,signed char,unsigned char,uchar,
+		   u8,__u8,uint8_t,s8,__s8,int8_t};
+type MULTIBYTE_TYPES = {short,short int,signed short,signed short int,
+			unsigned short,unsigned short int,ushort,
+			int,signed,signed int,unsigned,unsigned int,uint,
+			long,long int,signed long,signed long int,
+			unsigned long,unsigned long int,ulong,
+			long long,long long int,
+			signed long long,signed long long int,
+			unsigned long long,unsigned long long int,
+			u16,__u16,uint16_t,s16,__s16,int16_t,
+			u32,__u32,uint32_t,s32,__s32,int32_t,
+			u64,__u64,uint64_t,s64,__s64,int64_t,
+			__le16,__le32,__le64,__be16,__be32,__be64,
+			wchar_t};
 char [] STRING;
-INTEGRAL *BYTES;
-INTEGRAL **BYTES_PTRS;
+BYTE_TYPES *BYTES;
+MULTIBYTE_TYPES *MULTIBYTES;
+const MULTIBYTE_TYPES *CONST_MULTIBYTES;
+MULTIBYTE_TYPES MULTIBYTE;
 type TYPE;
 expression VAR;
 expression GFP;
@@ -59,66 +89,133 @@ fresh identifier ALLOC_OBJS = script:python(ALLOC_ARRAY) { alloc_array(ALLOC_ARR
 @@
 
 (
+// Convert a single object sized by its target: p = kmalloc(sizeof(*p), gfp)
 -	VAR = ALLOC((sizeof(*VAR)), GFP)
 +	VAR = ALLOC_OBJ(*VAR, GFP)
 |
-	ALLOC((\(sizeof(STRING)\|sizeof(INTEGRAL)\|sizeof(INTEGRAL *)\)), GFP)
+// Exclude byte buffers and integral pointers: kmalloc(sizeof(u8), gfp),
+//   kmalloc(sizeof("str"), gfp), kmalloc(sizeof(char *), gfp)
+	ALLOC((\(sizeof(STRING)\|sizeof(BYTE_TYPES)\|
+		sizeof(BYTE_TYPES *)\|sizeof(MULTIBYTE_TYPES *)\)), GFP)
 |
-	BYTES = ALLOC((sizeof(E)), GFP)
+// Exclude anything assigned to a byte pointer:
+//   u8 *buf = kmalloc(sizeof(*hdr), gfp)
+	BYTES = ALLOC((\(sizeof(E)\|sizeof(TYPE)\)), GFP)
 |
-	BYTES = ALLOC((sizeof(TYPE)), GFP)
+// Convert a multi-byte type to a pointer to it:
+//   u32 *p = kmalloc(sizeof(u32), gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) =
+-		ALLOC((sizeof(MULTIBYTE_TYPES)), GFP)
++		ALLOC_OBJ(MULTIBYTE_TYPES, GFP)
 |
-	BYTES_PTRS = ALLOC((sizeof(E)), GFP)
+// Same by expression: u32 *p = kmalloc(sizeof(p[0]), gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) =
+-		ALLOC((sizeof(MULTIBYTE)), GFP)
++		ALLOC_OBJ(MULTIBYTE, GFP)
 |
-	BYTES_PTRS = ALLOC((sizeof(TYPE)), GFP)
+// Exclude other multi-byte sizes, e.g. to pointers to arrays: s16 (*p)[2] = ...
+	ALLOC((\(sizeof(MULTIBYTE_TYPES)\|sizeof(MULTIBYTE)\)), GFP)
 |
+// Exclude anything else assigned to a multi-byte pointer:
+//   u32 *p = kmalloc(sizeof(*hdr), gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) = ALLOC((\(sizeof(E)\|sizeof(TYPE)\)), GFP)
+|
+// Exclude void pointers, to be checked by hand: kmalloc(sizeof(void *), gfp)
 	ALLOC((sizeof(void *)), GFP)
 |
+// Convert any other expression: p = kmalloc(sizeof(s->item), gfp)
 -	ALLOC((sizeof(E)), GFP)
 +	ALLOC_OBJ(E, GFP)
 |
+// Convert any other type: p = kmalloc(sizeof(struct item), gfp)
 -	ALLOC((sizeof(TYPE)), GFP)
 +	ALLOC_OBJ(TYPE, GFP)
 |
-	ALLOC_ARRAY(COUNT, (\(sizeof(STRING)\|sizeof(INTEGRAL)\|sizeof(INTEGRAL *)\)), GFP)
-|
-	BYTES = ALLOC_ARRAY(COUNT, (sizeof(E)), GFP)
-|
-	BYTES = ALLOC_ARRAY(COUNT, (sizeof(TYPE)), GFP)
-|
-	BYTES_PTRS = ALLOC_ARRAY(COUNT, (sizeof(E)), GFP)
+// The same, for arrays allocated as (count, size):
+// Exclude byte buffers and integral pointers: kcalloc(n, sizeof(u8), gfp),
+//   kcalloc(n, sizeof(char *), gfp)
+	ALLOC_ARRAY(COUNT, (\(sizeof(STRING)\|sizeof(BYTE_TYPES)\|
+			    sizeof(BYTE_TYPES *)\|sizeof(MULTIBYTE_TYPES *)\)), GFP)
 |
-	BYTES_PTRS = ALLOC_ARRAY(COUNT, (sizeof(TYPE)), GFP)
+// Exclude arrays assigned to a byte pointer:
+//   u8 *buf = kcalloc(n, sizeof(*hdr), gfp)
+	BYTES = ALLOC_ARRAY(COUNT, (\(sizeof(E)\|sizeof(TYPE)\)), GFP)
 |
-	ALLOC_ARRAY((\(sizeof(STRING)\|sizeof(INTEGRAL)\|sizeof(INTEGRAL *)\)), COUNT, GFP)
+// Convert a multi-byte array to a pointer to it:
+//   u32 *p = kcalloc(n, sizeof(u32), gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) =
+-		ALLOC_ARRAY(COUNT, (sizeof(MULTIBYTE_TYPES)), GFP)
++		ALLOC_OBJS(MULTIBYTE_TYPES, COUNT, GFP)
 |
-	BYTES = ALLOC_ARRAY((sizeof(E)), COUNT, GFP)
+// Same by expression: u32 *p = kcalloc(n, sizeof(*p), gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) =
+-		ALLOC_ARRAY(COUNT, (sizeof(MULTIBYTE)), GFP)
++		ALLOC_OBJS(MULTIBYTE, COUNT, GFP)
 |
-	BYTES = ALLOC_ARRAY((sizeof(TYPE)), COUNT, GFP)
+// Exclude other multi-byte arrays:
+//   s16 (*pairs)[2] = kcalloc(n, sizeof(s16), gfp)
+	ALLOC_ARRAY(COUNT, (\(sizeof(MULTIBYTE_TYPES)\|sizeof(MULTIBYTE)\)), GFP)
 |
-	BYTES_PTRS = ALLOC_ARRAY((sizeof(E)), COUNT, GFP)
-|
-	BYTES_PTRS = ALLOC_ARRAY((sizeof(TYPE)), COUNT, GFP)
+// Exclude other arrays assigned to a multi-byte pointer:
+//   u32 *p = kcalloc(n, sizeof(*hdr), gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) = ALLOC_ARRAY(COUNT, (\(sizeof(E)\|sizeof(TYPE)\)), GFP)
 |
+// Exclude arrays of void pointers: kcalloc(n, sizeof(void *), gfp)
 	ALLOC_ARRAY(COUNT, (sizeof(void *)), GFP)
 |
-	ALLOC_ARRAY((sizeof(void *)), COUNT, GFP)
-|
+// Convert any other expression: p = kcalloc(n, sizeof(*p), gfp)
 -	ALLOC_ARRAY(COUNT, (sizeof(E)), GFP)
 +	ALLOC_OBJS(E, COUNT, GFP)
 |
+// Convert any other type: p = kcalloc(n, sizeof(struct item), gfp)
 -	ALLOC_ARRAY(COUNT, (sizeof(TYPE)), GFP)
 +	ALLOC_OBJS(TYPE, COUNT, GFP)
 |
+// The same, for arrays allocated as (size, count):
+// Exclude byte buffers and integral pointers: kcalloc(sizeof(u8), n, gfp),
+//   kcalloc(sizeof(char *), n, gfp)
+	ALLOC_ARRAY((\(sizeof(STRING)\|sizeof(BYTE_TYPES)\|
+		      sizeof(BYTE_TYPES *)\|sizeof(MULTIBYTE_TYPES *)\)), COUNT, GFP)
+|
+// Exclude arrays assigned to a byte pointer:
+//   u8 *buf = kcalloc(sizeof(*hdr), n, gfp)
+	BYTES = ALLOC_ARRAY((\(sizeof(E)\|sizeof(TYPE)\)), COUNT, GFP)
+|
+// Convert a multi-byte array to a pointer to it:
+//   u32 *p = kcalloc(sizeof(u32), n, gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) =
+-		ALLOC_ARRAY((sizeof(MULTIBYTE_TYPES)), COUNT, GFP)
++		ALLOC_OBJS(MULTIBYTE_TYPES, COUNT, GFP)
+|
+// Same by expression: u32 *p = kcalloc(sizeof(*p), n, gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) =
+-		ALLOC_ARRAY((sizeof(MULTIBYTE)), COUNT, GFP)
++		ALLOC_OBJS(MULTIBYTE, COUNT, GFP)
+|
+// Exclude other multi-byte arrays:
+//   s16 (*pairs)[2] = kcalloc(sizeof(s16), n, gfp)
+	ALLOC_ARRAY((\(sizeof(MULTIBYTE_TYPES)\|sizeof(MULTIBYTE)\)), COUNT, GFP)
+|
+// Exclude other arrays assigned to a multi-byte pointer:
+//   u32 *p = kcalloc(sizeof(*hdr), n, gfp)
+	\(MULTIBYTES\|CONST_MULTIBYTES\) = ALLOC_ARRAY((\(sizeof(E)\|sizeof(TYPE)\)), COUNT, GFP)
+|
+// Exclude arrays of void pointers: kcalloc(sizeof(void *), n, gfp)
+	ALLOC_ARRAY((sizeof(void *)), COUNT, GFP)
+|
+// Convert any other expression: p = kcalloc(sizeof(*p), n, gfp)
 -	ALLOC_ARRAY((sizeof(E)), COUNT, GFP)
 +	ALLOC_OBJS(E, COUNT, GFP)
 |
+// Convert any other type: p = kcalloc(sizeof(struct item), n, gfp)
 -	ALLOC_ARRAY((sizeof(TYPE)), COUNT, GFP)
 +	ALLOC_OBJS(TYPE, COUNT, GFP)
 |
+// Convert flexible array structures: p = kmalloc(struct_size(p, data, n), gfp)
 -	ALLOC(struct_size(VAR, FLEX, COUNT), GFP)
 +	ALLOC_FLEX(*VAR, FLEX, COUNT, GFP)
 |
+// Same by type: kmalloc(struct_size_t(struct item, data, n), gfp)
 -	ALLOC(struct_size_t(TYPE, FLEX, COUNT), GFP)
 +	ALLOC_FLEX(TYPE, FLEX, COUNT, GFP)
 )
diff --git a/scripts/coccinelle/hid/ff_race.cocci b/scripts/coccinelle/hid/ff_race.cocci
index 479f5d1e31848e..e861de00cb10d0 100644
--- a/scripts/coccinelle/hid/ff_race.cocci
+++ b/scripts/coccinelle/hid/ff_race.cocci
@@ -7,18 +7,19 @@
 
 virtual report
 
-@r@
+@r exists@
 identifier probe_fn;
-expression hdev, flags;
+identifier hdev;
+expression flags;
 position p1, p2;
 @@
 
 probe_fn(struct hid_device *hdev, ...) {
-  <...
+  ... when any
   hid_hw_start@p1(hdev, flags)
   ...
-  \(input_ff_create\|input_ff_create_memless\)@p2(...)
-  ...>
+  \(input_ff_create@p2\|input_ff_create_memless@p2\)(...)
+  ... when any
 }
 
 @script:python depends on report@
diff --git a/scripts/context-analysis-suppression.txt b/scripts/context-analysis-suppression.txt
index 1c51b6153f0854..d4476d9ed10a38 100644
--- a/scripts/context-analysis-suppression.txt
+++ b/scripts/context-analysis-suppression.txt
@@ -32,3 +32,4 @@ src:*include/linux/seqlock*.h=emit
 src:*include/linux/spinlock*.h=emit
 src:*include/linux/srcu*.h=emit
 src:*include/linux/ww_mutex.h=emit
+src:*include/net/bluetooth/*=emit
diff --git a/scripts/dtc/dt-check-style b/scripts/dtc/dt-check-style
index 96deffc0d8a7c3..6bbd12b308a168 100755
--- a/scripts/dtc/dt-check-style
+++ b/scripts/dtc/dt-check-style
@@ -5,12 +5,14 @@
 # .dts/.dtsi/.dtso source files. Enforces rules from
 # Documentation/devicetree/bindings/dts-coding-style.rst.
 #
-# Two modes:
+# Three modes:
 #   --mode=relaxed (default)
 #     Only rules that produce zero warnings on the current tree.
 #     Suitable for dt_binding_check.
 #   --mode=strict
-#     All rules. Required for new submissions.
+#     Most of the rules. Required for new submissions.
+#   --mode=stricter
+#     All rules, including ones having false positives.
 #
 # Two input types (auto-detected by file extension):
 #   *.yaml             -- DT binding; check each example block
@@ -42,7 +44,6 @@ class LineType(Enum):
     NODE_OPEN = auto()       # something { (with optional label/name/addr)
     NODE_CLOSE = auto()      # };
     PROPERTY = auto()        # name = value; or name;
-    CONTINUATION = auto()    # continuation of a multi-line property
 
 
 re_cpp_directive = re.compile(
@@ -77,24 +78,30 @@ def is_preprocessor(stripped):
 
 
 class DtsLine:
-    __slots__ = ('lineno', 'raw', 'linetype', 'indent_str', 'stripped',
-                 'prop_name', 'continuations',
-                 'node_name', 'node_addr', 'label', 'ref_name', 'depth',
+    __slots__ = ('lineno', 'raw', 'code', 'linetype', 'indent_str', 'stripped', 'is_root',
+                 'prop_name', 'continuation', 'continuations',
+                 'node_name', 'node_addr', 'label', 'ref_name', 'parent', 'depth',
                  'closures')
 
-    def __init__(self, lineno, raw, linetype, depth, indent_str, stripped):
+    def __init__(self, lineno, raw, linetype, depth, indent_str, stripped,
+                 continuation = False, is_root = False):
         self.lineno = lineno      # 1-based within the block
-        self.raw = raw
+        self.raw = raw            # Entire raw line
         self.linetype = linetype
         self.indent_str = indent_str  # leading whitespace as-is
         self.depth = depth
-        self.stripped = stripped
+        self.stripped = stripped      # Code without indentation
+        # Only the code, skipping trailing comments and space between code and trailing comment
+        self.code = _strip_strings_and_comments(stripped).rstrip()
+        self.is_root = is_root
         self.prop_name = None
+        self.continuation = continuation    # Continuation of a multi-line property
         self.continuations = []
         self.node_name = None
         self.node_addr = None
         self.label = None
         self.ref_name = None
+        self.parent = None        # DtsLine of parent node
         self.closures = 1         # count of '}' on a NODE_CLOSE line
 
 
@@ -169,7 +176,7 @@ def classify_lines(text):
             continue
 
         if not stripped:
-            dl = DtsLine(i, raw, LineType.BLANK, depth, '', '')
+            dl = DtsLine(i, raw, LineType.BLANK, depth, '', '', continuation=not prev_complete)
             out.append(dl)
             continue
 
@@ -178,7 +185,7 @@ def classify_lines(text):
                      else LineType.COMMENT_BODY)
             if ltype == LineType.COMMENT_END:
                 in_block_comment = False
-            dl = DtsLine(i, raw, ltype, depth, indent_str, stripped)
+            dl = DtsLine(i, raw, ltype, depth, indent_str, stripped, continuation=not prev_complete)
             out.append(dl)
             continue
 
@@ -201,18 +208,10 @@ def classify_lines(text):
         # structural classification entirely.
         if not code:
             ltype = LineType.COMMENT_START if opens_block else LineType.COMMENT
-            dl = DtsLine(i, raw, ltype, depth, indent_str, stripped)
+            dl = DtsLine(i, raw, ltype, depth, indent_str, stripped, continuation=not prev_complete)
             out.append(dl)
             continue
 
-        if not prev_complete:
-            dl = DtsLine(i, raw, LineType.CONTINUATION, depth, indent_str, code)
-            out.append(dl)
-            prev_complete = (code.endswith(';') or
-                             code.endswith('{') or
-                             code.endswith('};'))
-            continue
-
         # NODE_CLOSE: the canonical form is "}" or "};" alone. A line
         # that is nothing but closures (e.g. "}; };") is still treated
         # as NODE_CLOSE for depth tracking, but the multi-closure case
@@ -228,7 +227,10 @@ def classify_lines(text):
             continue
 
         if code.endswith('{'):
-            dl = DtsLine(i, raw, LineType.NODE_OPEN, depth, indent_str, code)
+            is_root = False
+            if re.search(r'&{/}\s*{', code) or re.search(r'^/\s*\{$', code):
+                is_root = True
+            dl = DtsLine(i, raw, LineType.NODE_OPEN, depth, indent_str, code, is_root=is_root)
             parse_node_header(dl)
             out.append(dl)
             depth += 1
@@ -236,7 +238,8 @@ def classify_lines(text):
             continue
 
         # Property (or first line of a multi-line property).
-        dl = DtsLine(i, raw, LineType.PROPERTY, depth, indent_str, code)
+        dl = DtsLine(i, raw, LineType.PROPERTY, depth, indent_str, code,
+                     continuation=not prev_complete)
         parse_property_name(dl)
         out.append(dl)
         prev_complete = code.endswith(';')
@@ -245,7 +248,7 @@ def classify_lines(text):
     last_prop = None
     grouped = []
     for dl in out:
-        if dl.linetype == LineType.CONTINUATION and last_prop is not None:
+        if dl.continuation and last_prop is not None:
             last_prop.continuations.append(dl)
             continue
         if dl.linetype == LineType.PROPERTY:
@@ -272,7 +275,7 @@ def parse_node_header(dl):
 
 
 def parse_property_name(dl):
-    m = re.match(r'^([a-zA-Z0-9#][a-zA-Z0-9,._+#-]*)\s*[=;]', dl.stripped)
+    m = re.match(r'^([a-zA-Z0-9#][a-zA-Z0-9,._+?#-]*)\s*[=;]', dl.stripped)
     if m:
         dl.prop_name = m.group(1)
 
@@ -309,13 +312,13 @@ def collect_labels_and_refs(text):
 
 class Ctx:
     """Context passed to each rule check. Carries the parsed lines,
-    raw text, mode and kind."""
+    raw text, mode and file_type."""
 
-    def __init__(self, lines, text, mode, kind):
+    def __init__(self, lines, text, mode, file_type):
         self.lines = lines
         self.text = text
-        self.mode = mode               # 'relaxed' or 'strict'
-        if kind in DTS_FAMILY:
+        self.mode = mode               # 'relaxed', 'strict' or 'stricter'
+        if file_type in DTS_FAMILY:
             self.file_type = 'dts'
         else:
             self.file_type = 'yaml'
@@ -327,7 +330,7 @@ class Rule:
     def __init__(self, name, mode, description, check,
                  applies_to=('yaml', 'dts', 'dtsi', 'dtso')):
         self.name = name
-        self.mode = mode               # 'relaxed' or 'strict'
+        self.mode = mode    # 'relaxed', 'strict' or 'stricter'
         self.description = description
         self.check = check
         self.applies_to = applies_to   # input types this rule covers
@@ -335,52 +338,8 @@ class Rule:
 
 # --- individual rule check functions --------------------------------------
 
-def check_trailing_whitespace(ctx):
-    for dl in ctx.lines:
-        if dl.raw != dl.raw.rstrip():
-            yield (dl.lineno, 'trailing whitespace')
-
 
-def check_tab_in_yaml_example(ctx):
-    """Reject literal tabs in DTS lines when input is YAML.
-
-    For YAML examples, indent and content must use spaces. Tabs inside
-    a #define value are tolerated (those are CPP macros, not DTS).
-    For .dts files, this rule does not apply -- tabs are required.
-    """
-    if ctx.file_type != 'yaml':
-        return
-    for dl in ctx.lines:
-        if dl.linetype == LineType.PREPROCESSOR:
-            continue
-        if dl.linetype == LineType.BLANK:
-            continue
-        if '\t' in dl.raw:
-            yield (dl.lineno, 'tab character not allowed in DTS example')
-
-
-def check_mixed_indent_chars(ctx):
-    """Indent must be all-tabs, except for aligning indentation (comments
-    or continued lines)."""
-    for dl in ctx.lines:
-        if not dl.indent_str:
-            continue
-        if dl.linetype == LineType.PREPROCESSOR:
-            continue
-        if re.search(r' \t', dl.indent_str):
-            yield (dl.lineno, 'mixed tabs and spaces in indent')
-        if dl.indent_str.count(' ') > 7:
-            yield (dl.lineno, 'too many space characters in indent (more than 7)')
-        for cont in dl.continuations:
-            if not cont.indent_str:
-                continue
-            if cont.linetype == LineType.PREPROCESSOR:
-                continue
-            if re.search(r' \t', cont.indent_str):
-                yield (cont.lineno, 'mixed tabs and spaces in indent')
-
-
-def detect_indent_unit(ctx):
+def _detect_indent_unit(ctx):
     """Find the indent unit used at depth 1 in this block.
 
     Returns tuple of string (one of: '  ' (2 spaces), '    ' (4 spaces),
@@ -406,64 +365,70 @@ def detect_indent_unit(ctx):
     return (None, None)
 
 
-def check_indent_unit_relaxed(ctx):
-    """YAML examples: 2 or 4 spaces. Never tabs or other widths."""
-    (unit, lineno) = detect_indent_unit(ctx)
-    if unit is None:
-        return
-    if unit not in ('  ', '    '):
-        yield (lineno, 'indent unit must be 2 or 4 spaces, got %r' % unit)
-
+def _display_col(text):
+    """Visual column width of text, with tabs expanded to the next
+    8-column stop, matching how printf and most editors render a
+    line and the kernel-wide line length convention."""
+    col = 0
+    for ch in text:
+        if ch == '\t':
+            col = (col // 8 + 1) * 8
+        else:
+            col += 1
+    return col
 
-def check_indent_unit_dts(ctx):
-    """DTS files: 1 tab per level. Always required."""
-    (unit, lineno) = detect_indent_unit(ctx)
-    if unit is None:
-        return
-    if unit != '\t':
-        yield (lineno, 'indent unit must be 1 tab in DTS, got %r' % unit)
 
+def _natural_sort_key(s):
+    """Split a string into a tuple of (kind, value) pairs that compares
+    numeric runs as ints, so 'foo10' sorts after 'foo2'."""
+    parts = []
+    for part in re.split(r'(\d+)', s):
+        if part.isdigit():
+            parts.append((0, int(part)))
+        else:
+            parts.append((1, part))
+    return tuple(parts)
 
-def check_indent_unit_strict(ctx):
-    """YAML: must be exactly 4 spaces. DTS: 1 tab (same as relaxed)."""
-    (unit, lineno) = detect_indent_unit(ctx)
-    if unit is None:
-        return
-    if ctx.file_type == 'yaml':
-        if unit != '    ':
-            yield (lineno, 'indent unit must be 4 spaces in strict mode, '
-                   'got %r' % unit)
 
+def _strip_strings_and_comments(text):
+    """Remove string literals and /* */ + // comments from a single
+    line, replacing them with empty strings. Used so syntactic checks
+    (whitespace, hex case, etc.) don't false-positive on contents of
+    quoted strings or comments. An unclosed /* on the line is treated
+    as a comment running to end of line."""
+    text = re.sub(r'"(?:[^"\\]|\\.)*"', '""', text)
+    text = re.sub(r'/\*.*?\*/', '', text)
+    text = re.sub(r'/\*.*$', '', text)
+    text = re.sub(r'//.*$', '', text)
+    return text
 
-def check_indent_consistent(ctx):
-    """All indented lines must be a multiple of the detected unit."""
-    (unit, lineno) = detect_indent_unit(ctx)
-    if unit is None:
-        return
-    if ctx.file_type == 'yaml':
-        if unit not in ('  ', '    '):
-            return  # let check_indent_unit_* report this
-    else:
-        if unit != '\t':
-            return
 
-    for dl in ctx.lines:
-        if dl.linetype in (LineType.BLANK, LineType.PREPROCESSOR):
-            continue
-        if dl.linetype == LineType.CONTINUATION:
-            continue   # continuations align to <, not to indent unit
-        if dl.linetype in (LineType.COMMENT_BODY, LineType.COMMENT_END):
+def _walk_bodies(lines):
+    """Yield lists of immediate-child NODE_OPEN lines for each node body
+    in the input. Skips ref-nodes (&label) since those don't have an
+    intrinsic ordering."""
+    # Array of stacked nodes (parent/child)
+    body_stack = [[]]
+    # Current stack of nodes, purely to track parent relationship for each node
+    node_stack = []
+    parent_dl = None
+    for dl in lines:
+        if dl.linetype == LineType.NODE_OPEN:
+            dl.parent = parent_dl
+            node_stack.append(dl)
+            body_stack[-1].append(dl)
+            body_stack.append([])
+            parent_dl = dl
             continue
-        if not dl.indent_str:
+        if dl.linetype == LineType.NODE_CLOSE:
+            if len(body_stack) <= 1:
+                # Unbalanced; ignore to avoid crashing on malformed input
+                continue
+            parent_dl = node_stack.pop().parent
+            yield body_stack.pop()
             continue
-        # The indent must be 'unit' repeated dl.depth times, exactly.
-        # NODE_CLOSE lines have depth equal to the post-decrement value,
-        # which matches the indent expected.
-        expected = unit * dl.depth
-        if dl.indent_str != expected:
-            yield (dl.lineno,
-                   'indent mismatch (expected depth %d * %r)' %
-                   (dl.depth, unit))
+    while body_stack:
+        yield body_stack.pop()
 
 
 def check_blank_lines(ctx):
@@ -475,6 +440,12 @@ def check_blank_lines(ctx):
         if lines[i].linetype == LineType.BLANK and \
                 lines[i - 1].linetype == LineType.BLANK:
             yield (lines[i].lineno, 'consecutive blank lines')
+        prev_is_blank = lines[i].linetype == LineType.BLANK
+        for cont in lines[i].continuations:
+            if cont.linetype == LineType.BLANK and prev_is_blank:
+                yield (cont.lineno, 'consecutive blank lines')
+            prev_is_blank = cont.linetype == LineType.BLANK
+
     # Blank right after { or right before }
     for i, dl in enumerate(lines):
         if dl.linetype != LineType.BLANK:
@@ -487,46 +458,20 @@ def check_blank_lines(ctx):
             yield (dl.lineno, 'blank line at end of node body')
 
 
-def _walk_bodies(lines):
-    """Yield lists of immediate-child NODE_OPEN lines for each node body
-    in the input. Skips ref-nodes (&label) since those don't have an
-    intrinsic ordering."""
-    body_stack = [[]]
-    for dl in lines:
-        if dl.linetype == LineType.NODE_OPEN:
-            body_stack[-1].append(dl)
-            body_stack.append([])
-            continue
-        if dl.linetype == LineType.NODE_CLOSE:
-            if len(body_stack) <= 1:
-                # Unbalanced; ignore to avoid crashing on malformed input
-                continue
-            yield body_stack.pop()
-            continue
-    while body_stack:
-        yield body_stack.pop()
-
-
-def _natural_sort_key(s):
-    """Split a string into a tuple of (kind, value) pairs that compares
-    numeric runs as ints, so 'foo10' sorts after 'foo2'."""
-    parts = []
-    for part in re.split(r'(\d+)', s):
-        if part.isdigit():
-            parts.append((0, int(part)))
-        else:
-            parts.append((1, part))
-    return tuple(parts)
-
-
 def check_child_address_order(ctx):
     """Addressed siblings (foo@N) must appear in ascending address
-    order within their parent node body."""
+    order within their parent node body.
+    Exception: Top-level in DTS follows name order, regardless of unit address
+    in memory@N and soc@N nodes
+    """
     for children in _walk_bodies(ctx.lines):
         addressed = []
         for c in children:
             if c.node_addr is None:
                 continue
+            if c.parent and c.parent.is_root:
+                # Top-level does not use unit address sorting usually
+                continue
             try:
                 parts = tuple(int(p, 16) for p in c.node_addr.split(','))
             except ValueError:
@@ -544,59 +489,267 @@ def check_child_name_order(ctx):
     """Unaddressed siblings must appear in natural-sort order by node
     name within their parent node body. Addressed children are scoped
     by check_child_address_order; reference nodes (&label { ... }) and
-    the root node are skipped."""
+    the root node are skipped.
+    However root node has children with and without unit address, and
+    sorting should be only by name."""
     for children in _walk_bodies(ctx.lines):
         unaddressed = []
         for c in children:
             if c.node_addr is not None:
-                continue
+                # Skip nodes with unit address, except when sorting top-level
+                if not c.parent or not c.parent.is_root:
+                    continue
             if c.node_name in (None, '/'):
                 continue
-            if c.ref_name is not None:
+            if c.ref_name is not None:
+                continue
+            unaddressed.append((_natural_sort_key(c.node_name), c))
+        for i in range(1, len(unaddressed)):
+            if unaddressed[i][0] < unaddressed[i - 1][0]:
+                dl = unaddressed[i][1]
+                yield (dl.lineno,
+                       'child node %r out of name order' % dl.node_name)
+
+
+def check_continuation_alignment(ctx):
+    """A multi-line property's continuation lines must align their
+    first non-whitespace character to the display column of:
+    1. the first '<' or '"' after the '=' in the leading line, if continuation is with '<' or '"'
+    2. the first value, if the continuation is still the same phandle.
+    Display columns are used so tab-indented .dts files (where a continuation
+    aligns with tabs plus spaces) are compared correctly."""
+    for dl in ctx.lines:
+        if dl.linetype != LineType.PROPERTY:
+            continue
+        if not dl.continuations:
+            continue
+        eq = dl.raw.find('=')
+        if eq < 0:
+            continue
+        # First '<' or '"' after '=', but ignore comments and strip trailing
+        # whitespace (e.g. remaining after removing the comment)
+        rest = _strip_strings_and_comments(dl.raw[eq + 1:]).rstrip()
+        m = re.search(r'\s*([<"])', rest)
+        if not m:
+            continue
+        dl_value_complete = rest.endswith('",') or rest.endswith('>,')
+        target_col = _display_col(_strip_strings_and_comments(dl.raw[:eq + 1 + m.start(1)]))
+        for cont in dl.continuations:
+            if cont.linetype == LineType.BLANK:
+                continue
+            target_offset = 0
+            err_msg_explanation = 'to < or "'
+            if not dl_value_complete:
+                target_offset = 1
+                err_msg_explanation = 'to the value under <'
+            if _display_col(cont.indent_str) != target_col + target_offset:
+                yield (cont.lineno,
+                       'continuation should align to column %d '
+                       '(%s)' % (target_col + target_offset + 1, err_msg_explanation))
+            # Align to the value within <> or "" of continuation (so the previous line)
+            if len(cont.code):
+                dl_value_complete = cont.code.endswith('",') or cont.code.endswith('>,')
+
+
+def _check_hex_case(dl):
+    if dl.linetype in (LineType.BLANK, LineType.COMMENT,
+                        LineType.COMMENT_START, LineType.COMMENT_BODY,
+                        LineType.COMMENT_END, LineType.PREPROCESSOR):
+        return
+    for m in re.finditer(r'\b0[xX][0-9a-fA-F]+\b', dl.code):
+        lit = m.group(0)
+        if any(c.isupper() for c in lit[2:]) or lit[1] == 'X':
+            yield (dl.lineno,
+                    'hex literal %r must be lowercase' % lit)
+
+
+def check_hex_case(ctx):
+    """Hex literals (0xN) must use lowercase digits and prefix."""
+    for dl in ctx.lines:
+        yield from _check_hex_case(dl)
+        for cont in dl.continuations:
+            yield from _check_hex_case(cont)
+
+
+def check_indent_consistent(ctx):
+    """All indented lines must be a multiple of the detected unit."""
+    (unit, lineno) = _detect_indent_unit(ctx)
+    if unit is None:
+        return
+    if ctx.file_type == 'yaml':
+        if unit not in ('  ', '    '):
+            return  # let check_indent_unit_* report this
+    else:
+        if unit != '\t':
+            return
+
+    for dl in ctx.lines:
+        if dl.continuation:
+            continue   # continuations align to <, not to indent unit
+        if dl.linetype in (LineType.BLANK, LineType.PREPROCESSOR):
+            continue
+        if dl.linetype in (LineType.COMMENT_BODY, LineType.COMMENT_END):
+            continue
+        if not dl.indent_str:
+            continue
+        # The indent must be 'unit' repeated dl.depth times, exactly.
+        # NODE_CLOSE lines have depth equal to the post-decrement value,
+        # which matches the indent expected.
+        expected = unit * dl.depth
+        if dl.indent_str != expected:
+            yield (dl.lineno,
+                   'indent mismatch (expected depth %d * %r)' %
+                   (dl.depth, unit))
+
+
+def check_indent_unit_dts(ctx):
+    """DTS files: 1 tab per level. Always required."""
+    (unit, lineno) = _detect_indent_unit(ctx)
+    if unit is None:
+        return
+    if unit != '\t':
+        yield (lineno, 'indent unit must be 1 tab in DTS, got %r' % unit)
+
+
+def check_indent_unit_relaxed(ctx):
+    """YAML examples: 2 or 4 spaces. Never tabs or other widths."""
+    (unit, lineno) = _detect_indent_unit(ctx)
+    if unit is None:
+        return
+    if unit not in ('  ', '    '):
+        yield (lineno, 'indent unit must be 2 or 4 spaces, got %r' % unit)
+
+
+def check_indent_unit_strict(ctx):
+    """YAML: must be exactly 4 spaces. DTS: 1 tab (same as relaxed)."""
+    (unit, lineno) = _detect_indent_unit(ctx)
+    if unit is None:
+        return
+    if ctx.file_type == 'yaml':
+        if unit != '    ':
+            yield (lineno, 'indent unit must be 4 spaces in strict mode, '
+                   'got %r' % unit)
+
+
+def _check_line_length(dl):
+    if dl.linetype == LineType.BLANK:
+        return
+    cols = _display_col(dl.raw)
+    if cols > 80:
+        yield (dl.lineno,
+                'line exceeds 80 columns (%d)' % cols)
+
+
+def check_line_length(ctx):
+    """Lines must not exceed 80 columns; tabs count as 8 (see
+    _display_col)."""
+    for dl in ctx.lines:
+        yield from _check_line_length(dl)
+        for cont in dl.continuations:
+            yield from _check_line_length(cont)
+
+
+def _check_mixed_indent_chars(dl):
+    if not dl.indent_str:
+        return
+    if dl.linetype == LineType.PREPROCESSOR:
+        return
+    if re.search(r' \t', dl.indent_str):
+        yield (dl.lineno, 'mixed tabs and spaces in indent')
+    if dl.indent_str.count(' ') > 7:
+        yield (dl.lineno, 'too many space characters in indent (more than 7)')
+
+
+def check_mixed_indent_chars(ctx):
+    """Indent must be all-tabs, except for aligning indentation (comments
+    or continued lines)."""
+    for dl in ctx.lines:
+        yield from _check_mixed_indent_chars(dl)
+        for cont in dl.continuations:
+            yield from _check_mixed_indent_chars(cont)
+
+
+def check_node_close_alone(ctx):
+    """The closing '};' of a node must be on its own line. The
+    classifier accepts a canonical "}" or "};" as NODE_CLOSE; a line
+    that is all closures (e.g. "}; };") is still NODE_CLOSE for depth
+    tracking but is flagged here via dl.closures. Any other line that
+    still contains '};' (in code, not in strings or comments) is
+    mixing a node close with something else."""
+    for dl in ctx.lines:
+        if dl.linetype == LineType.NODE_CLOSE:
+            if dl.closures > 1:
+                yield (dl.lineno,
+                       'closing brace must be on its own line')
+            continue
+        if dl.linetype in (LineType.BLANK, LineType.COMMENT,
+                           LineType.COMMENT_START, LineType.COMMENT_BODY,
+                           LineType.COMMENT_END, LineType.PREPROCESSOR):
+            continue
+        if '};' in dl.code:
+            yield (dl.lineno,
+                   'closing brace must be on its own line')
+
+
+def check_node_name(ctx):
+    """Only recommended characters are used in node names."""
+    for dl in ctx.lines:
+        if not dl.node_name in (None, '/'):
+            if not re.match(r'[0-9a-z][0-9a-z-]*(?<!-)$', dl.node_name):
+                yield (dl.lineno, f'node name "{dl.node_name}" is using discouraged style')
+
+
+def check_property_name(ctx):
+    """Only recommended characters are used in property names."""
+    for dl in ctx.lines:
+        if dl.prop_name:
+            exceptions = ['cpu_off', 'cpu_on', 'cpu_suspend',
+                          'device_type', 'dr_mode',
+                          'mmc-hs200-1_2v', 'mmc-hs200-1_8v', 'mmc-hs400-1_2v', 'mmc-hs400-1_8v',
+                          'opp-avg-kBps', 'opp-peak-kBps', 'phy_type']
+
+            if re.match(r'\#([0-9a-z][0-9a-z-]*(?<!-),)?[a-z-]+-cells$', dl.prop_name):
+                continue
+            if dl.prop_name in exceptions:
                 continue
-            unaddressed.append((_natural_sort_key(c.node_name), c))
-        for i in range(1, len(unaddressed)):
-            if unaddressed[i][0] < unaddressed[i - 1][0]:
-                dl = unaddressed[i][1]
-                yield (dl.lineno,
-                       'child node %r out of name order' % dl.node_name)
+            if not re.match(r'([0-9a-z][0-9a-z-]*(?<!-),)?[0-9a-z][0-9a-z-]*(?<!-)$', dl.prop_name):
+                yield (dl.lineno, f'property name "{dl.prop_name}" is using discouraged style')
 
 
-def _property_bucket(name):
-    """Return the canonical bucket index for a property:
-       0 device_type
+def _property_bucket(name, is_root=False):
+    """Return the canonical bucket (group) index for a property:
+       0 device_type ('model' in the root node)
        1 compatible
-       2 reg / reg-names
+       2 reg, reg-names
        3 ranges
        4 standard properties (no vendor comma in #-stripped name)
        5 vendor-specific properties
        6 status
-    Plus a sub-key inside the bucket for fixed slots (device_type, compatible,
-    reg, reg-names, ranges, status). 'standard' and 'vendor' return None for
-    the sub-key, signalling that the within-bucket key is computed by
-    the pairing rules."""
+    Properties sharing a bucket compare equal: their relative order
+    within the group is not checked, only the pairing rules below
+    apply."""
     stripped = name.lstrip('#')
-    if name == 'device_type':
-        return (0, 0)
+    if is_root:
+        if name == 'model':
+            return 0
+    elif name == 'device_type':
+        return 0
     if name == 'compatible':
-        return (1, 0)
-    if name == 'reg':
-        return (2, 0)
-    if name == 'reg-names':
-        return (2, 1)
+        return 1
+    if name in ('reg', 'reg-names'):
+        return 2
     if name == 'ranges':
-        return (3, 0)
+        return 3
     if name == 'status':
-        return (6, 0)
-    return (5 if ',' in stripped else 4, None)
+        return 6
+    return 5 if ',' in stripped else 4
 
 
 # Declarative pairing rules: each is a callable
 #   (name, all_names) -> anchor_name_or_None
-# If a rule returns an anchor, the property sorts immediately after the
+# If a rule returns an anchor, the property must be placed after the
 # anchor. Rules are tried in order; the first match wins. If none
-# matches, the within-bucket key falls back to natural sort by the
-# #-stripped name.
+# matches, the property can be placed anywhere within its bucket.
 
 def _pair_pinctrl_names(name, all_names):
     """pinctrl-names follows the highest pinctrl-N in the same node."""
@@ -611,8 +764,7 @@ def _pair_pinctrl_names(name, all_names):
 def _pair_x_names(name, all_names):
     """Generic <x>-names follows its owning property. The owner is
     usually plural (clocks/clock-names, dmas/dma-names,
-    resets/reset-names) but occasionally singular (reg/reg-names is
-    handled by the fixed slot above; this rule catches anything else)."""
+    resets/reset-names) but occasionally singular (reg/reg-names)."""
     if not name.endswith('-names'):
         return None
     base = name[:-len('-names')]
@@ -627,30 +779,27 @@ def _pair_x_names(name, all_names):
 PAIRING_RULES = (_pair_pinctrl_names, _pair_x_names)
 
 
-def _property_sort_key(name, all_names):
-    """Sort key for a property among its node-body siblings.
-
-    Format: (bucket, within_key, tiebreak). 'within_key' for
-    standard/vendor buckets follows pairing rules: a property paired
-    with anchor X sorts as if it were X with a higher tiebreak."""
-    bucket, fixed_sub = _property_bucket(name)
-    if fixed_sub is not None:
-        return (bucket, (), fixed_sub)
-
+def _property_anchor(name, all_names):
+    """Return the property this one must be placed after, or None."""
     for rule in PAIRING_RULES:
         anchor = rule(name, all_names)
         if anchor is not None:
-            return (bucket, _natural_sort_key(anchor.lstrip('#')), 1)
-
-    return (bucket, _natural_sort_key(name.lstrip('#')), 0)
+            return anchor
+    return None
 
 
 def check_property_order(ctx):
-    """Properties within a node body must appear in canonical order:
-    compatible, reg(/reg-names), ranges, then the standard group, then
-    the vendor-specific group, then status. Inside the standard and
-    vendor groups, pairing rules apply (e.g. <x>-names follows <x>);
-    everything else falls back to natural sort by the #-stripped name."""
+    """Properties within a node body must be grouped in canonical
+    order: device_type ('model' for the root node), compatible,
+    reg(/reg-names), ranges, then the standard group, then the
+    vendor-specific group, then status.
+
+    The order of properties within one group is free, e.g. it does not
+    matter whether '#address-cells' comes before or after 'clocks'.
+    Paired properties still keep their relative order: <x>-names
+    follows <x> (clocks/clock-names, reg/reg-names, ...), pinctrl-names
+    follows the pinctrl-N states and #size-cells follows
+    #address-cells."""
     lines = ctx.lines
     for i, dl in enumerate(lines):
         if dl.linetype != LineType.NODE_OPEN:
@@ -668,29 +817,62 @@ def check_property_order(ctx):
         if len(props) < 2:
             continue
         all_names = [p.prop_name for p in props]
-        keyed = [(p, _property_sort_key(p.prop_name, all_names))
-                 for p in props]
-        for k in range(1, len(keyed)):
-            if keyed[k][1] < keyed[k - 1][1]:
-                p = keyed[k][0]
-                prev = keyed[k - 1][0]
-                yield (p.lineno,
-                       'property %r out of canonical order '
-                       '(should sort before %r)' %
-                       (p.prop_name, prev.prop_name))
+        buckets = [_property_bucket(p.prop_name, dl.is_root) for p in props]
+        findings = []
+        # Group (bucket) order
+        for k in range(1, len(props)):
+            if buckets[k] < buckets[k - 1]:
+                findings.append((props[k].lineno,
+                                 'property %r out of canonical order '
+                                 '(should sort before %r)' %
+                                 (props[k].prop_name,
+                                  props[k - 1].prop_name)))
+        # Pairing within a group: the anchor must come first
+        seen = set()
+        for p in props:
+            anchor = _property_anchor(p.prop_name, all_names)
+            if anchor is not None and anchor not in seen:
+                findings.append((p.lineno,
+                                 'property %r must be placed after %r' %
+                                 (p.prop_name, anchor)))
+            seen.add(p.prop_name)
+        yield from sorted(findings)
+
+
+def _check_redundant_whitespace(dl):
+    if dl.linetype in (LineType.COMMENT, LineType.COMMENT_BODY,
+                        LineType.COMMENT_END, LineType.COMMENT_START,
+                        LineType.PREPROCESSOR):
+        return
+    if re.search(r'(\s\s+|\t)\{', dl.code):
+        yield (dl.lineno, 'extra whitespace before {')
+    if re.search(r':(\s\s+|\t)', dl.code):
+        yield (dl.lineno, 'extra whitespace after :')
+    if re.search(r'\s+;', dl.code):
+        yield (dl.lineno, 'extra whitespace before ;')
 
 
-def _strip_strings_and_comments(text):
-    """Remove string literals and /* */ + // comments from a single
-    line, replacing them with empty strings. Used so syntactic checks
-    (whitespace, hex case, etc.) don't false-positive on contents of
-    quoted strings or comments. An unclosed /* on the line is treated
-    as a comment running to end of line."""
-    text = re.sub(r'"(?:[^"\\]|\\.)*"', '""', text)
-    text = re.sub(r'/\*.*?\*/', '', text)
-    text = re.sub(r'/\*.*$', '', text)
-    text = re.sub(r'//.*$', '', text)
-    return text
+def check_redundant_whitespace(ctx):
+    """No whitespace between brackets or other code elements.
+    See also check_value_whitespace() for more checks."""
+    for dl in ctx.lines:
+        yield from _check_redundant_whitespace(dl)
+        for cont in dl.continuations:
+            yield from _check_redundant_whitespace(cont)
+
+
+def check_redundant_whitespace_strict(ctx):
+    """No whitespace between brackets or other code elements.
+    See also check_value_whitespace() for more checks."""
+    for dl in ctx.lines:
+        if dl.linetype in (LineType.COMMENT, LineType.COMMENT_BODY,
+                           LineType.COMMENT_END, LineType.COMMENT_START,
+                           LineType.PREPROCESSOR):
+            continue
+        if re.search(r'(\s\s+|\t)=', dl.code):
+            yield (dl.lineno, 'extra whitespace before =')
+        if re.search(r'=(\s\s+|\t)', dl.code):
+            yield (dl.lineno, 'extra whitespace after =')
 
 
 def check_required_blank_lines(ctx):
@@ -710,6 +892,8 @@ def check_required_blank_lines(ctx):
             if d.linetype == LineType.NODE_CLOSE and \
                     d.depth == body_depth - 1 and depth_inside == 0:
                 break
+            if d.continuation:
+                continue
             # Track depth inside nested children so we only look at
             # immediate-body items.
             if d.linetype == LineType.NODE_OPEN and \
@@ -729,8 +913,6 @@ def check_required_blank_lines(ctx):
                               LineType.COMMENT_BODY, LineType.COMMENT_END,
                               LineType.PREPROCESSOR):
                 continue
-            if d.linetype == LineType.CONTINUATION:
-                continue
 
             needs_blank = False
             if d.linetype == LineType.NODE_OPEN:
@@ -752,19 +934,58 @@ def check_required_blank_lines(ctx):
             between_blanks = 0
 
 
-def check_hex_case(ctx):
-    """Hex literals (0xN) must use lowercase digits and prefix."""
+def check_tab_in_yaml_example(ctx):
+    """Reject literal tabs in DTS lines when input is YAML.
+
+    For YAML examples, indent and content must use spaces. Tabs inside
+    a #define value are tolerated (those are CPP macros, not DTS).
+    For .dts files, this rule does not apply -- tabs are required.
+    """
+    if ctx.file_type != 'yaml':
+        return
     for dl in ctx.lines:
-        if dl.linetype in (LineType.BLANK, LineType.COMMENT,
-                           LineType.COMMENT_START, LineType.COMMENT_BODY,
-                           LineType.COMMENT_END, LineType.PREPROCESSOR):
+        if dl.linetype == LineType.PREPROCESSOR:
             continue
-        text = _strip_strings_and_comments(dl.raw)
-        for m in re.finditer(r'\b0[xX][0-9a-fA-F]+\b', text):
-            lit = m.group(0)
-            if any(c.isupper() for c in lit[2:]) or lit[1] == 'X':
-                yield (dl.lineno,
-                       'hex literal %r must be lowercase' % lit)
+        if dl.linetype == LineType.BLANK:
+            continue
+        if '\t' in dl.raw:
+            yield (dl.lineno, 'tab character not allowed in DTS example')
+        for cont in dl.continuations:
+            if '\t' in cont.raw:
+                yield (cont.lineno, 'tab character not allowed in DTS example')
+
+
+def _check_trailing_whitespace(dl):
+    if dl.raw != dl.raw.rstrip():
+        yield (dl.lineno, 'trailing whitespace')
+
+
+def check_trailing_whitespace(ctx):
+    for dl in ctx.lines:
+        yield from _check_trailing_whitespace(dl)
+        for cont in dl.continuations:
+            yield from _check_trailing_whitespace(cont)
+
+
+def check_unclosed_block_comment(ctx):
+    """Every /* must have a matching */ in the same block. Catches both
+    a comment opened on its own line (COMMENT_START) and a tail comment
+    opened on a PROPERTY or other code line (where in_block_comment is
+    set by _split_code so the next line becomes COMMENT_BODY without a
+    preceding COMMENT_START)."""
+    open_lineno = None
+    for dl in ctx.lines:
+        if dl.linetype == LineType.COMMENT_START:
+            open_lineno = dl.lineno
+        elif dl.linetype == LineType.COMMENT_END:
+            open_lineno = None
+        elif dl.linetype == LineType.COMMENT_BODY and open_lineno is None:
+            # Block was opened by a /* tail on a code line; report at
+            # the first orphan body line since the originating line is
+            # already classified as something else.
+            open_lineno = dl.lineno
+    if open_lineno is not None:
+        yield (open_lineno, 'unclosed /* block comment')
 
 
 def check_unit_address_format(ctx):
@@ -798,6 +1019,17 @@ def check_unit_address_format(ctx):
                 break
 
 
+def check_unused_labels(ctx):
+    """Labels defined but never referenced are clutter."""
+    defined, referenced = collect_labels_and_refs(ctx.text)
+    for label in sorted(defined - referenced):
+        # Find the line where this label is defined for line-number
+        # reporting.
+        m = re.search(r'(?m)^.*\b' + re.escape(label) + r'\s*:', ctx.text)
+        lineno = ctx.text[:m.start()].count('\n') + 1 if m else 1
+        yield (lineno, 'label %r defined but never &-referenced' % label)
+
+
 def check_value_whitespace(ctx):
     """A <...> cell list must have no whitespace directly after '<'
     or directly before '>'. Continuation lines are joined onto the
@@ -808,9 +1040,9 @@ def check_value_whitespace(ctx):
     for dl in ctx.lines:
         if dl.linetype != LineType.PROPERTY:
             continue
-        segs = [_strip_strings_and_comments(dl.raw).strip()]
+        segs = [dl.code.strip()]
         for cont in dl.continuations:
-            segs.append(_strip_strings_and_comments(cont.stripped).strip())
+            segs.append(cont.code.strip())
         text = ''
         for s in segs:
             if not s:
@@ -826,131 +1058,13 @@ def check_value_whitespace(ctx):
                 break
 
 
-def check_node_close_alone(ctx):
-    """The closing '};' of a node must be on its own line. The
-    classifier accepts a canonical "}" or "};" as NODE_CLOSE; a line
-    that is all closures (e.g. "}; };") is still NODE_CLOSE for depth
-    tracking but is flagged here via dl.closures. Any other line that
-    still contains '};' (in code, not in strings or comments) is
-    mixing a node close with something else."""
-    for dl in ctx.lines:
-        if dl.linetype == LineType.NODE_CLOSE:
-            if dl.closures > 1:
-                yield (dl.lineno,
-                       'closing brace must be on its own line')
-            continue
-        if dl.linetype in (LineType.BLANK, LineType.COMMENT,
-                           LineType.COMMENT_START, LineType.COMMENT_BODY,
-                           LineType.COMMENT_END, LineType.PREPROCESSOR):
-            continue
-        text = _strip_strings_and_comments(dl.raw)
-        if '};' in text:
-            yield (dl.lineno,
-                   'closing brace must be on its own line')
-
-
-def _display_col(text):
-    """Visual column width of text, with tabs expanded to the next
-    8-column stop, matching how printf and most editors render a
-    line and the kernel-wide line length convention."""
-    col = 0
-    for ch in text:
-        if ch == '\t':
-            col = (col // 8 + 1) * 8
-        else:
-            col += 1
-    return col
-
-
-def check_line_length(ctx):
-    """Lines must not exceed 80 columns; tabs count as 8 (see
-    _display_col)."""
-    for dl in ctx.lines:
-        if dl.linetype == LineType.BLANK:
-            continue
-        cols = _display_col(dl.raw)
-        if cols > 80:
-            yield (dl.lineno,
-                   'line exceeds 80 columns (%d)' % cols)
-
-
-def check_continuation_alignment(ctx):
-    """A multi-line property's continuation lines must align their
-    first non-whitespace character to the display column of the first
-    '<' or '"' after the '=' in the leading line. Display columns are
-    used so tab-indented .dts files (where a continuation aligns with
-    tabs plus spaces) are compared correctly."""
-    for dl in ctx.lines:
-        if dl.linetype != LineType.PROPERTY:
-            continue
-        if not dl.continuations:
-            continue
-        eq = dl.raw.find('=')
-        if eq < 0:
-            continue
-        # First '<' or '"' after '='
-        rest = dl.raw[eq + 1:]
-        m = re.search(r'[<"]', rest)
-        if not m:
-            continue
-        target_col = _display_col(dl.raw[:eq + 1 + m.start()])
-        for cont in dl.continuations:
-            if _display_col(cont.indent_str) != target_col:
-                yield (cont.lineno,
-                       'continuation should align to column %d '
-                       '(under "<" or \\")' % (target_col + 1))
-
-
-def check_unclosed_block_comment(ctx):
-    """Every /* must have a matching */ in the same block. Catches both
-    a comment opened on its own line (COMMENT_START) and a tail comment
-    opened on a PROPERTY or other code line (where in_block_comment is
-    set by _split_code so the next line becomes COMMENT_BODY without a
-    preceding COMMENT_START)."""
-    open_lineno = None
-    for dl in ctx.lines:
-        if dl.linetype == LineType.COMMENT_START:
-            open_lineno = dl.lineno
-        elif dl.linetype == LineType.COMMENT_END:
-            open_lineno = None
-        elif dl.linetype == LineType.COMMENT_BODY and open_lineno is None:
-            # Block was opened by a /* tail on a code line; report at
-            # the first orphan body line since the originating line is
-            # already classified as something else.
-            open_lineno = dl.lineno
-    if open_lineno is not None:
-        yield (open_lineno, 'unclosed /* block comment')
-
-
-def check_unused_labels(ctx):
-    """Labels defined but never referenced are clutter."""
-    defined, referenced = collect_labels_and_refs(ctx.text)
-    for label in sorted(defined - referenced):
-        # Find the line where this label is defined for line-number
-        # reporting.
-        m = re.search(r'(?m)^.*\b' + re.escape(label) + r'\s*:', ctx.text)
-        lineno = ctx.text[:m.start()].count('\n') + 1 if m else 1
-        yield (lineno, 'label %r defined but never &-referenced' % label)
-
-
 # --- registry --------------------------------------------------------------
 
 RULES = [
     # 'relaxed' is the default; rules in this group must produce zero
     # output on a clean kernel tree (post the small prep-cleanup
     # commit at the head of this series).
-    Rule('trailing-whitespace', 'relaxed',
-         'no trailing whitespace on any line',
-         check_trailing_whitespace),
-    Rule('tab-in-yaml', 'relaxed',
-         'YAML (also DTS examples) may not contain tab characters',
-         check_tab_in_yaml_example, applies_to=('yaml',)),
-    Rule('mixed-indent-chars', 'relaxed',
-         'indent must not mix tabs and spaces',
-         check_mixed_indent_chars, applies_to=('dts', 'dtsi', 'dtso')),
-    Rule('unclosed-block-comment', 'relaxed',
-         'every /* block comment must close with */',
-         check_unclosed_block_comment),
+    # Rules are sorted here by group (relaxed, strict, stricter) and name
 
     # DTS files always use tabs; this is not negotiable per kernel
     # coding style (.dts files are real source). Relaxed mode.
@@ -958,19 +1072,26 @@ RULES = [
          'DTS files: 1 tab per nesting level',
          check_indent_unit_dts,
          applies_to=('dts', 'dtsi', 'dtso')),
+    Rule('mixed-indent-chars', 'relaxed',
+         'indent must not mix tabs and spaces',
+         check_mixed_indent_chars, applies_to=('dts', 'dtsi', 'dtso')),
+    # See also check_redundant_whitespace_strict() and check_value_whitespace()
+    Rule('redundant-whitespace', 'relaxed',
+         'no redundant whitespace within code',
+         check_redundant_whitespace),
+    Rule('tab-in-yaml', 'relaxed',
+         'YAML (also DTS examples) may not contain tab characters',
+         check_tab_in_yaml_example, applies_to=('yaml',)),
+    Rule('trailing-whitespace', 'relaxed',
+         'no trailing whitespace on any line',
+         check_trailing_whitespace),
+    Rule('unclosed-block-comment', 'relaxed',
+         'every /* block comment must close with */',
+         check_unclosed_block_comment),
 
     # 'strict' rules are opt-in (e.g. for new submissions via
     # checkpatch.pl in a follow-up series). They flag many existing
     # files and can be promoted to relaxed once those are cleaned up.
-    Rule('indent-unit', 'strict',
-         'YAML: 2 or 4 spaces per level',
-         check_indent_unit_relaxed, applies_to=('yaml',)),
-    Rule('indent-unit-strict', 'strict',
-         'YAML: must be 4 spaces per level',
-         check_indent_unit_strict, applies_to=('yaml',)),
-    Rule('indent-consistent', 'strict',
-         'every line indented at depth * unit',
-         check_indent_consistent),
     Rule('blank-lines', 'strict',
          'no consecutive blanks; no blanks at node body edges',
          check_blank_lines),
@@ -980,45 +1101,72 @@ RULES = [
     Rule('child-name-order', 'strict',
          'unaddressed siblings must be in natural-sort name order',
          check_child_name_order),
+    Rule('continuation-alignment', 'strict',
+         'multi-line property continuations align under <, " or the value under <',
+         check_continuation_alignment),
+    Rule('hex-case', 'strict',
+         'hex literals must be lowercase',
+         check_hex_case),
+    Rule('indent-consistent', 'strict',
+         'every line indented at depth * unit',
+         check_indent_consistent),
+    Rule('indent-unit', 'strict',
+         'YAML: 2 or 4 spaces per level',
+         check_indent_unit_relaxed, applies_to=('yaml',)),
+    Rule('indent-unit-strict', 'strict',
+         'YAML: must be 4 spaces per level',
+         check_indent_unit_strict, applies_to=('yaml',)),
+    Rule('line-length', 'strict',
+         'lines must not exceed 80 columns',
+         check_line_length, applies_to=('yaml',)),
+    Rule('line-length-dts', 'stricter',
+         'lines must not exceed 80 columns',
+         check_line_length, applies_to=('dts', 'dtsi', 'dtso')),
+    Rule('node-close-alone', 'strict',
+         'closing brace must be on its own line',
+         check_node_close_alone),
+    Rule('node-name', 'strict',
+         'node names use only recommended characters (see DTS Coding Style)',
+         check_node_name),
+    Rule('property-name', 'strict',
+         'property names use only recommended characters (see DTS Coding Style)',
+         check_property_name),
     Rule('property-order', 'strict',
-         'canonical bucket + pairing + natural-sort order of properties',
+         'canonical group + pairing order of properties (order within '
+         'a group is not checked)',
          check_property_order),
+    # See also check_redundant_whitespace() and check_value_whitespace()
+    Rule('redundant-whitespace-strict', 'strict',
+         'no redundant whitespace within code',
+         check_redundant_whitespace_strict),
     Rule('required-blank-lines', 'strict',
          'blank line before child nodes and before "status"',
          check_required_blank_lines),
-    Rule('hex-case', 'strict',
-         'hex literals must be lowercase',
-         check_hex_case),
     Rule('unit-address-format', 'strict',
          'unit addresses must be lowercase hex without leading zeros',
          check_unit_address_format),
-    Rule('value-whitespace', 'strict',
-         'no whitespace directly inside <...> brackets',
-         check_value_whitespace),
-    Rule('node-close-alone', 'strict',
-         'closing brace must be on its own line',
-         check_node_close_alone),
-    Rule('line-length', 'strict',
-         'lines must not exceed 80 columns',
-         check_line_length),
-    Rule('continuation-alignment', 'strict',
-         'multi-line property continuations align under "<" or "\\""',
-         check_continuation_alignment),
     Rule('unused-labels', 'strict',
          'every label must be &-referenced in the same example/file '
          '(skipped for .dtsi/.dtso since labels there are exported)',
-         check_unused_labels, applies_to=('yaml', 'dts')),
+         check_unused_labels, applies_to=('yaml',)),
+    Rule('unused-labels-dts', 'stricter',
+         'every label must be &-referenced in the same example/file '
+         '(skipped for .dtsi/.dtso since labels there are exported)',
+         check_unused_labels, applies_to=('dts',)),
+    Rule('value-whitespace', 'strict',
+         'no whitespace directly inside <...> brackets',
+         check_value_whitespace),
 ]
 
 
-def select_rules(mode, input_kind):
+def select_rules(mode, file_type):
     """Return rules that apply to the given mode and input type."""
-    rank = {'relaxed': 0, 'strict': 1}
+    rank = {'relaxed': 0, 'strict': 1, 'stricter': 2}
     out = []
     for r in RULES:
         if rank[r.mode] > rank[mode]:
             continue
-        if input_kind not in r.applies_to:
+        if file_type not in r.applies_to:
             continue
         out.append(r)
     return out
@@ -1028,12 +1176,12 @@ def select_rules(mode, input_kind):
 # Block runner
 # ---------------------------------------------------------------------------
 
-def check_block(text, mode, input_type):
+def check_block(text, mode, file_type):
     """Run all selected rules on a single block of DTS text. Returns a
     list of (lineno, rule_name, message) tuples."""
     lines = classify_lines(text)
-    ctx = Ctx(lines, text, mode, input_type)
-    rules = select_rules(mode, input_type)
+    ctx = Ctx(lines, text, mode, file_type)
+    rules = select_rules(mode, file_type)
     findings = []
     for r in rules:
         for lineno, msg in r.check(ctx):
@@ -1090,7 +1238,7 @@ def iter_dts_file(filepath):
 # Top-level processing
 # ---------------------------------------------------------------------------
 
-def input_kind(filepath):
+def get_file_type(filepath):
     p = filepath.lower()
     if p.endswith('.yaml') or p.endswith('.yml'):
         return 'yaml'
@@ -1110,17 +1258,17 @@ DTS_FAMILY = ('dts', 'dtsi', 'dtso')
 def collect_findings(filepath, mode):
     """Return a (lines, count) pair for filepath. lines is a list of
     formatted output strings; count is the number of findings."""
-    kind = input_kind(filepath)
-    if kind == 'yaml':
+    file_type = get_file_type(filepath)
+    if file_type == 'yaml':
         iterator = iter_yaml_examples(filepath)
-    elif kind in DTS_FAMILY:
+    elif file_type in DTS_FAMILY:
         iterator = iter_dts_file(filepath)
     else:
         return (['%s: unknown file type, skipping' % filepath], 0)
 
     out = []
     for text, base, idx in iterator:
-        for lineno, rule, msg in check_block(text, mode, kind):
+        for lineno, rule, msg in check_block(text, mode, file_type):
             abs_line = base + lineno - 1
             ex_tag = '' if idx is None else ' example %d' % idx
             out.append('%s:%d:%s [%s] %s' %
@@ -1141,7 +1289,7 @@ def main():
         description='Check DTS coding style on YAML examples and '
         '.dts/.dtsi/.dtso files.',
         fromfile_prefix_chars='@')
-    ap.add_argument('--mode', choices=('relaxed', 'strict'),
+    ap.add_argument('--mode', choices=('relaxed', 'strict', 'stricter'),
                     default='relaxed',
                     help='which rule set to apply (default: relaxed)')
     ap.add_argument('-j', '--jobs', type=int, default=0,
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-blank-lines.dts b/scripts/dtc/dt-style-selftest/bad/dts-blank-lines.dts
new file mode 100644
index 00000000000000..f2e99fb7bb29a9
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-blank-lines.dts
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+/ {
+	compatible = "example,test-board";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	interrupt-controller@10000 {
+		compatible = "example,intc";
+
+		reg = <0x10000 0x1000>;
+
+		clocks = <1 2 3>,
+
+
+			 /* Comment is not a blank line */
+			 <4 5 6>,
+
+			 /* Comment is not a blank line */
+			 <7 8 9>;
+
+
+		interrupts = <1>;
+
+		/* Comment is not a blank line */
+		qcom,calibration-variant = "foo";
+	};
+
+
+	interrupt-controller@20000 {
+
+		compatible = "example,intc";
+		reg = <0x20000 0x1000>;
+
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-child-name-order.dtso b/scripts/dtc/dt-style-selftest/bad/dts-child-name-order.dtso
new file mode 100644
index 00000000000000..74b49be69e9869
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-child-name-order.dtso
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	pmu {
+		compatible = "example,pmu";
+
+		/* Include labels to be sure they do not affect sorting */
+		foo: foo {
+			label = "foo";
+		};
+
+		label_bar: bar {
+			label = "bar";
+		};
+	};
+
+	memory@a0000000 {
+		device_type = "memory";
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	pmu-2 {
+		compatible = "example,pmu";
+
+		/* Just reference labels to avoid strict warnings */
+		example,foo = <&foo>, <&label_bar>;
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-cont-align.dts b/scripts/dtc/dt-style-selftest/bad/dts-cont-align.dts
new file mode 100644
index 00000000000000..91a74887c774dd
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-cont-align.dts
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+/ {
+	compatible = "example,test-board";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	interrupt-controller@10000 {
+		compatible = "example,intc";
+		reg = <0x10000 0x1000>;
+		clocks = <1 2 3>, /* comments with " < , should not matter */
+						     <4 5 6>,
+				/* but comments should be placed properly */
+				<7 8 9>;
+		interrupts = <1 2 3>, /* comments with " < , should not ... */
+				<4 5 6>,
+				<7 8 9>;
+		pinmux = <0x01
+			 0x02>,
+			<0x03
+			 0x04>;
+		power-domain-names = "foo",
+				"bar", /* comments with " < , should not */
+							"baz";
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dts b/scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dts
new file mode 100644
index 00000000000000..74c95639832458
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	memory@a0000000 {
+		device_type = "memory";
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	pmu {
+		compatible = "example,pmu";
+	};
+
+	soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		serial@20000 {
+			compatible = "example,serial";
+			reg = <0x20000 0x1000>;
+		};
+
+		interrupt-controller@10000 {
+			compatible = "example,intc";
+			reg = <0x10000 0x1000>;
+			interrupts = <1 2 3>;
+		};
+
+		serial@30000 {
+			compatible = "example,serial";
+			reg = <0x30000 0x1000>;
+		};
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dtso b/scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dtso
new file mode 100644
index 00000000000000..052c02935a45a5
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-digit-node-order.dtso
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	memory@a0000000 {
+		device_type = "memory";
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	pmu {
+		compatible = "example,pmu";
+	};
+
+	soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		serial@20000 {
+			compatible = "example,serial";
+			reg = <0x20000 0x1000>;
+		};
+
+		interrupt-controller@10000 {
+			compatible = "example,intc";
+			reg = <0x10000 0x1000>;
+			interrupts = <1 2 3>;
+		};
+
+		serial@30000 {
+			compatible = "example,serial";
+			reg = <0x30000 0x1000>;
+		};
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-extend-node-child-name-order.dtso b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-child-name-order.dtso
new file mode 100644
index 00000000000000..d9cf670f19f662
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-child-name-order.dtso
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	pmu {
+		compatible = "example,pmu";
+
+		/* Just reference labels to avoid strict warnings */
+		example,foo = <&foo>, <&label_bar>;
+	};
+};
+
+&pmu {
+	/* Include labels to be sure they do not affect sorting */
+	foo: foo {
+		label = "foo";
+	};
+
+	label_bar: bar {
+		label = "bar";
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-extend-node-digit-node-order.dtso b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-digit-node-order.dtso
new file mode 100644
index 00000000000000..4b2cf60e5a9257
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-extend-node-digit-node-order.dtso
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&soc {
+	serial@20000 {
+		compatible = "example,serial";
+		reg = <0x20000 0x1000>;
+	};
+
+	interrupt-controller@10000 {
+		compatible = "example,intc";
+		reg = <0x10000 0x1000>;
+		interrupts = <1 2 3>;
+	};
+
+	serial@30000 {
+		compatible = "example,serial";
+		reg = <0x30000 0x1000>;
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-line-length.dts b/scripts/dtc/dt-style-selftest/bad/dts-line-length.dts
new file mode 100644
index 00000000000000..adf40e3c95f72a
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-line-length.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/*
+ * Test fixture: Line length in DTS
+ */
+
+/dts-v1/;
+
+/ {
+	soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		foo@1000 {
+			compatible = "example,test-line-length-this-is-a-very-long-name-indeed-yeah",
+				     "example,test-line-length-this-is-a-very-long-name-indeed-yeah-second";
+			reg = <0x1000 0x100>;
+		};
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-node-name.dts b/scripts/dtc/dt-style-selftest/bad/dts-node-name.dts
new file mode 100644
index 00000000000000..b897cbd74dd6cc
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-node-name.dts
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	foo_bar {
+		compatible = "example,test-node-name";
+	};
+
+	foo,bar {
+		compatible = "example,test-node-name";
+	};
+
+	foo.bar {
+		compatible = "example,test-node-name";
+	};
+
+	foo+bar {
+		compatible = "example,test-node-name";
+	};
+
+	foo-bar- {
+		compatible = "example,test-node-name";
+	};
+
+	soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		foo_bar@1000 {
+			compatible = "example,test-node-name";
+			reg = <0x1000 0x100>;
+		};
+
+		foo,bar@2000 {
+			compatible = "example,test-node-name";
+			reg = <0x2000 0x100>;
+		};
+
+		foo.bar@3000 {
+			compatible = "example,test-node-name";
+			reg = <0x3000 0x100>;
+		};
+
+		foo+bar@4000 {
+			compatible = "example,test-node-name";
+			reg = <0x4000 0x100>;
+		};
+
+		foo-bar-@5000 {
+			compatible = "example,test-node-name";
+			reg = <0x5000 0x100>;
+		};
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-property-name.dts b/scripts/dtc/dt-style-selftest/bad/dts-property-name.dts
new file mode 100644
index 00000000000000..110e33a524dd1d
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-property-name.dts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	foo {
+		compatible = "example,test-property-name";
+
+		property#foo;
+		property+foo;
+		property-foo;
+		property-foo-;
+		property.foo;
+		property?foo;
+		property_foo;
+
+		vendor,property#foo;
+		vendor,property+foo;
+		vendor,property,foo;
+		vendor,property-foo;
+		vendor,property-foo-;
+		vendor,property.foo;
+		vendor,property?foo;
+		vendor,property_foo;
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-property-order.dts b/scripts/dtc/dt-style-selftest/bad/dts-property-order.dts
index f31abb6ceae404..eb36daa4fe723c 100644
--- a/scripts/dtc/dt-style-selftest/bad/dts-property-order.dts
+++ b/scripts/dtc/dt-style-selftest/bad/dts-property-order.dts
@@ -6,6 +6,11 @@
 /dts-v1/;
 
 / {
+	compatible = "example,test-board", "example,test-soc";
+	model = "DT style selftest";
+	qcom,board-id = <8 0>;
+	chassis-type = "handset";
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -38,6 +43,19 @@
 				     <4 5 6>,
 				     <7 8 9>;
 			compatible = "example,intc";
+			qcom,calibration-variant = "foo";
+			clocks = <6>;
+			qcom,opp-fuse-level = <1>;
 		};
 	};
 };
+
+/  {
+	compatible = "example,test-board", "example,test-soc";
+	model = "DT style selftest";
+};
+
+/{
+	compatible = "example,test-board", "example,test-soc";
+	model = "DT style selftest";
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-property-order.dtso b/scripts/dtc/dt-style-selftest/bad/dts-property-order.dtso
new file mode 100644
index 00000000000000..b2e564f2af2b95
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-property-order.dtso
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/*
+ * Test fixture: Incorrect property order
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	compatible = "example,test-board", "example,test-soc";
+	model = "DT style selftest";
+	qcom,board-id = <8 0>;
+	chassis-type = "handset";
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			reg = <0x0 0x0>;
+			compatible = "arm,cortex-a57";
+			device_type = "cpu";
+			enable-method = "psci";
+		};
+	};
+
+	pmu {
+		compatible = "example,pmu";
+
+		status = "disabled";
+		dma-coherent;
+	};
+
+	soc@0 {
+		ranges = <0 0 0 0xc0000000>;
+		compatible = "simple-bus";
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		interrupt-controller@10000 {
+			reg = <0x10000 0x1000>;
+			interrupts = <1 2 3>,
+				     <4 5 6>,
+				     <7 8 9>;
+			compatible = "example,intc";
+			qcom,calibration-variant = "foo";
+			clocks = <6>;
+			qcom,opp-fuse-level = <1>;
+		};
+	};
+};
+
+&{/}  {
+	compatible = "example,test-board", "example,test-soc";
+	model = "DT style selftest";
+};
+
+&{/}{
+	compatible = "example,test-board", "example,test-soc";
+	model = "DT style selftest";
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws-strict.dts b/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws-strict.dts
new file mode 100644
index 00000000000000..201e3940ba1c4d
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws-strict.dts
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+#define SOME_MACRO(foo)					\
+	(foo) ? <1> : <2> ;
+
+/  {
+	compatible = "example,test-board";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	/* comments  { are okay  =  though ; */
+	soc:  soc@0  { /* comments  { are okay  =  though ; */
+		compatible = "simple-bus";  /* comments  { are okay  =  though ; */
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells	= <1>;
+		#size-cells =	<1>;
+	} ;
+}; /* comments  { are okay  =  though ; */
+
+&soc	{
+	serial:	serial@20000  { /* comments  { are okay  =  though ; */
+		compatible  = "example,serial";
+		reg =  <0x20000 0x1000>;
+	}	;
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dts b/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dts
new file mode 100644
index 00000000000000..f6480ddc508ac1
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+#define SOME_MACRO(foo)					\
+	(foo) ? <1> : <2> ;
+
+/  {
+	compatible = "example,test-board";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	/* comments  { are okay  =  though ; */
+	soc:  soc@0  { /* comments  { are okay  =  though ; */
+		compatible = "simple-bus";  /* comments  { are okay  =  though ; */
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+	} ;
+}; /* comments  { are okay  =  though ; */
+
+&soc	{
+	serial:	serial@20000  { /* comments  { are okay  =  though ; */
+		compatible = "example,serial";
+		reg = <0x20000 0x1000>,
+		      <0x30000 0x1000> ;
+	}	; /* comments  { are okay  =  though ; */
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dtso b/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dtso
new file mode 100644
index 00000000000000..01d94e14071a83
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-redundant-ws.dtso
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/}  {
+	compatible = "example,test-board";
+	#address-cells = <1>;
+	#size-cells = <1>;
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-trailing-ws.dts b/scripts/dtc/dt-style-selftest/bad/dts-trailing-ws.dts
new file mode 100644
index 00000000000000..73c24525ce187e
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-trailing-ws.dts
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+
+/ {
+	compatible = "example,test-board";  
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	interrupt-controller@10000 {
+		compatible = "example,intc";
+		reg = <0x10000 0x1000>;
+		clocks = <1 2 3>,
+			 <4 5 6>,  
+			 <7 8 9>;
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/dts-unused-label.dts b/scripts/dtc/dt-style-selftest/bad/dts-unused-label.dts
new file mode 100644
index 00000000000000..90802ae107e1bf
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/dts-unused-label.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/*
+ * Test fixture: Unused label in DTS
+ */
+
+/dts-v1/;
+
+/ {
+	soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		dev: device@1000 {
+			compatible = "example,test-unused-label";
+			reg = <0x1000 0x100>;
+		};
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-blank-lines.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-blank-lines.yaml
new file mode 100644
index 00000000000000..a58cd534a0033e
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-blank-lines.yaml
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/test-bad-blank-lines.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Test fixture with too many blank lines
+
+maintainers:
+  - Test User <test@example.com>
+
+properties:
+  compatible:
+    const: example,test-blank-lines
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    foo@1000 {
+        compatible = "example,test-blank-lines";
+
+        reg = <0x1000 0x100>;
+
+        clocks = <1 2 3>,
+
+
+                 /* Comment is not a blank line */
+                 <4 5 6>,
+
+                 /* Comment is not a blank line */
+                 <7 8 9>;
+
+
+        interrupts = <1>;
+
+        /* Comment is not a blank line */
+        qcom,calibration-variant = "foo";
+    };
+
+
+    foo@2000 {
+
+        compatible = "example,test-blank-lines";
+
+        reg = <0x2000 0x100>;
+
+    };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-child-addr-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-child-addr-order.yaml
index 3df56e69a1ff83..7c3b731e059ab3 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-child-addr-order.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-child-addr-order.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with addressed children out of order
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-child-name-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-child-name-order.yaml
index 35d85e5573c2bd..f3483033859aa6 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-child-name-order.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-child-name-order.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with unaddressed children out of name order
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-cont-align.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-cont-align.yaml
index 92778540b05605..0189b654a5b0d0 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-cont-align.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-cont-align.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with mis-aligned multi-line property
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
@@ -25,6 +25,9 @@ examples:
   - |
     foo@1000 {
         compatible = "example,test-cont-align";
-        reg = <0x1000 0x100>,
-            <0x2000 0x100>;
+        reg = <0x1000 0x100>, /* comments with " < , should not matter */
+            <0x2000 0x100>, /* comments with " < , should not matter */
+            /* but comments should be placed properly */
+              <0x3000
+              0x100>;
     };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-digit-node-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-digit-node-order.yaml
index 44a9d25e5ba0cc..068542a985760b 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-digit-node-order.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-digit-node-order.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with digit-leading nodes out of address order
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-hex-case.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-hex-case.yaml
index b26d1bf58de9cf..b0b8683b883a11 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-hex-case.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-hex-case.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with uppercase hex literals
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
@@ -25,5 +25,8 @@ examples:
   - |
     foo@1000 {
         compatible = "example,test-hex-case";
-        reg = <0xABCD 0x100>;
+        reg = <0xABCD 0x100>,
+              <0x2BCD 0x100>,
+              <0x3BCD
+               0x100>;
     };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-indent-strict.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-indent-strict.yaml
index bee4cf118d73fb..155060a798873f 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-indent-strict.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-indent-strict.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture using 2-space indent (rejected by strict mode)
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-label-in-string.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-label-in-string.yaml
index ba512869b702ef..4a69b503af4b04 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-label-in-string.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-label-in-string.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture where a label is only "referenced" inside a string
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-line-length.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-line-length.yaml
index 64427bf1c385d1..6b1209ee4f2612 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-line-length.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-line-length.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture exceeding 80 columns
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
@@ -24,6 +24,7 @@ additionalProperties: false
 examples:
   - |
     foo@1000 {
-        compatible = "example,test-line-length-this-is-a-very-long-name-indeed-yeah";
+        compatible = "example,test-line-length-this-is-a-very-long-name-indeed-yeah",
+                     "example,test-line-length-this-is-a-very-long-name-indeed-yeah-second";
         reg = <0x1000 0x100>;
     };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-mixed-indent.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-mixed-indent.yaml
index 5401d1a423a1d7..4f76188b5e23c5 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-mixed-indent.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-mixed-indent.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture mixing tabs and spaces in indent
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
@@ -26,4 +26,6 @@ examples:
     device@1000 {
     	   compatible = "example,test-mixed";
         reg = <0x1000 0x100>;
+        clocks = <1>,
+    	           <2>;
     };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-multi-close.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-multi-close.yaml
index 4d9fa27b50a277..afc202a698eb3d 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-multi-close.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-multi-close.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with two closing braces on one line
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-node-close.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-node-close.yaml
index e107659fd9e8c4..9e714f7e7d4796 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-node-close.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-node-close.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with closing brace not on its own line
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-node-name.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-node-name.yaml
new file mode 100644
index 00000000000000..03373cae9a66d9
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-node-name.yaml
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/test-node-name.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Test fixture with incorrect node names
+
+maintainers:
+  - Rob Herring <robh@kernel.org>
+
+properties:
+  compatible:
+    const: example,test-node-name
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+
+additionalProperties: false
+
+examples:
+  - |
+    foo_bar@1000 {
+        compatible = "example,test-node-name";
+        reg = <0x1000 0x100>;
+    };
+
+    foo,bar@2000 {
+        compatible = "example,test-node-name";
+        reg = <0x2000 0x100>;
+    };
+
+    foo.bar@3000 {
+        compatible = "example,test-node-name";
+        reg = <0x3000 0x100>;
+    };
+
+    foo+bar@4000 {
+        compatible = "example,test-node-name";
+        reg = <0x4000 0x100>;
+    };
+
+    foo-bar-@5000 {
+        compatible = "example,test-node-name";
+        reg = <0x5000 0x100>;
+    };
+
+    // TODO handle also "-foo-bar" which is against spec but accepted by DTC
+
+    foo_bar {
+        compatible = "example,test-node-name";
+    };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-prop-order-device-type.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order-device-type.yaml
index e2c69e9ff4523f..c71fb2c0a3e528 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-prop-order-device-type.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order-device-type.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with device_type
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-prop-order.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order.yaml
index 75582a3d2f6e1d..4d8a50d0e35a75 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-prop-order.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-prop-order.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with reg before compatible
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
@@ -26,4 +26,9 @@ examples:
     device@1000 {
         reg = <0x1000 0x100>;
         compatible = "example,test-prop-order";
+
+        interrupts = <1>;
+        qcom,calibration-variant = "foo";
+        clocks = <6>;
+        qcom,opp-fuse-level = <1>;
     };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-prop-pairing.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-prop-pairing.yaml
index 767ab21c39f341..51abc540af11de 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-prop-pairing.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-prop-pairing.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture exercising <x>-names and pinctrl-names pairing
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-property-name.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-property-name.yaml
new file mode 100644
index 00000000000000..08e0d1eb152a48
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-property-name.yaml
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/test-property-name.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Test fixture with incorrect property names
+
+maintainers:
+  - Rob Herring <robh@kernel.org>
+
+properties:
+  compatible:
+    const: example,test-property-name
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    foo@1000 {
+        compatible = "example,test-property-name";
+        reg = <0x1000 0x100>;
+
+        property#foo;
+        property+foo;
+        property-foo;
+        property-foo-;
+        property.foo;
+        property?foo;
+        property_foo;
+
+        vendor,property#foo;
+        vendor,property+foo;
+        vendor,property,foo;
+        vendor,property-foo;
+        vendor,property-foo-;
+        vendor,property.foo;
+        vendor,property?foo;
+        vendor,property_foo;
+    };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws-strict.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws-strict.yaml
new file mode 100644
index 00000000000000..739c44fd7217dc
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws-strict.yaml
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/yaml-redundant-ws-strict.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Test fixture with redundant whitespace
+
+maintainers:
+  - Rob Herring <robh@kernel.org>
+
+properties:
+  compatible:
+    const: example,test-redundant
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    device@1000  {
+        compatible  = "example,test-trailing";
+        reg =  <0x1000 0x100>;
+        clocks = <&clk 0>; /* comments  { are okay  =  though ; */
+        clock-names = "bus";  /* comments  { are okay  =  though ; */
+    } ;
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws.yaml
new file mode 100644
index 00000000000000..b2f51969b0ff69
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-redundant-ws.yaml
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/yaml-redundant-ws.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Test fixture with redundant whitespace
+
+maintainers:
+  - Rob Herring <robh@kernel.org>
+
+properties:
+  compatible:
+    const: example,test-redundant
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    device@1000  { /* comments  { are okay  =  though ; */
+        compatible = "example,test-trailing";
+        reg = <0x1000 0x100>,
+              <0x3000 0x100> ;
+    } ;
+
+    device@2000 { /* comments  { are okay  =  though ; */
+        compatible = "example,test-trailing";
+        reg = <0x2000 0x100>;
+    }; /* comments  { are okay  =  though ; */
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-required-blank.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-required-blank.yaml
index 8bb53240cffaeb..0036d5e0c4eade 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-required-blank.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-required-blank.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture missing required blank lines
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml
index 487d07ff8cb62c..937ecdd923b0a6 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-tab.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with a tab in a DTS line
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-trailing-comment.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-comment.yaml
index 2368ada8106f1b..695d046e84548e 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-trailing-comment.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-comment.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with properties out of order behind trailing comments
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-trailing-ws.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-ws.yaml
index 5c4b4bd833c570..17eeed3411c66d 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-trailing-ws.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-trailing-ws.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with trailing whitespace
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
@@ -25,5 +25,8 @@ examples:
   - |
     device@1000 {
         compatible = "example,test-trailing";  
-        reg = <0x1000 0x100>;
+        reg = <0x1000 0x100>,
+              <0x2000 0x100>, 
+              <0x3000 
+               0x100>;
     };
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unclosed-comment.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unclosed-comment.yaml
index 63c1c08712a579..191ed775384b8f 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-unclosed-comment.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-unclosed-comment.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with an unclosed /* block comment
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr-prefix.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr-prefix.yaml
index 9b3fe508c5fdd2..62590948d2de06 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr-prefix.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr-prefix.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with 0x-prefixed unit address
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr.yaml
index 93705cd4541092..5bb19fba5c192d 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-unit-addr.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with malformed unit address
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-unused-label.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-unused-label.yaml
index 28d7176cbf08e9..7f862ae5a1756f 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-unused-label.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-unused-label.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with an unused label
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-value-ws-multiline.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws-multiline.yaml
index 504bf0931c2735..1d7bc8142fd707 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-value-ws-multiline.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws-multiline.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with extra whitespace in a multi-line cell array
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/bad/yaml-value-ws.yaml b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws.yaml
index 342ab9f399f128..a5082a15815519 100644
--- a/scripts/dtc/dt-style-selftest/bad/yaml-value-ws.yaml
+++ b/scripts/dtc/dt-style-selftest/bad/yaml-value-ws.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture with extra whitespace inside <...>
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-blank-lines.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-blank-lines.dts.txt
new file mode 100644
index 00000000000000..97617bc42a17c6
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-blank-lines.dts.txt
@@ -0,0 +1,6 @@
+# mode=strict
+bad/dts-blank-lines.dts:16: [blank-lines] consecutive blank lines
+bad/dts-blank-lines.dts:23: [blank-lines] consecutive blank lines
+bad/dts-blank-lines.dts:30: [blank-lines] consecutive blank lines
+bad/dts-blank-lines.dts:32: [blank-lines] blank line at start of node body
+bad/dts-blank-lines.dts:35: [blank-lines] blank line at end of node body
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dts.txt
index e2eea0862102d1..312a45ed913a54 100644
--- a/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dts.txt
+++ b/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dts.txt
@@ -1,2 +1,3 @@
 # mode=strict
 bad/dts-child-name-order.dts:16: [child-name-order] child node 'bar' out of name order
+bad/dts-child-name-order.dts:21: [child-name-order] child node 'memory' out of name order
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dtso.txt b/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dtso.txt
new file mode 100644
index 00000000000000..e44ceb24ece815
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-child-name-order.dtso.txt
@@ -0,0 +1,3 @@
+# mode=strict
+bad/dts-child-name-order.dtso:17: [child-name-order] child node 'bar' out of name order
+bad/dts-child-name-order.dtso:22: [child-name-order] child node 'memory' out of name order
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-cont-align.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-cont-align.dts.txt
new file mode 100644
index 00000000000000..fd7f389d6ccee8
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-cont-align.dts.txt
@@ -0,0 +1,11 @@
+# mode=strict
+bad/dts-cont-align.dts:13: [continuation-alignment] continuation should align to column 26 (to < or ")
+bad/dts-cont-align.dts:14: [continuation-alignment] continuation should align to column 26 (to < or ")
+bad/dts-cont-align.dts:15: [continuation-alignment] continuation should align to column 26 (to < or ")
+bad/dts-cont-align.dts:17: [continuation-alignment] continuation should align to column 30 (to < or ")
+bad/dts-cont-align.dts:18: [continuation-alignment] continuation should align to column 30 (to < or ")
+bad/dts-cont-align.dts:20: [continuation-alignment] continuation should align to column 27 (to the value under <)
+bad/dts-cont-align.dts:21: [continuation-alignment] continuation should align to column 26 (to < or ")
+bad/dts-cont-align.dts:22: [continuation-alignment] continuation should align to column 27 (to the value under <)
+bad/dts-cont-align.dts:24: [continuation-alignment] continuation should align to column 38 (to < or ")
+bad/dts-cont-align.dts:25: [continuation-alignment] continuation should align to column 38 (to < or ")
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dts.txt
new file mode 100644
index 00000000000000..1f41acdea0b02e
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dts.txt
@@ -0,0 +1,2 @@
+# mode=strict
+bad/dts-digit-node-order.dts:29: [child-address-order] child node @10000 out of address order
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dtso.txt b/scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dtso.txt
new file mode 100644
index 00000000000000..21db32f6e6390c
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-digit-node-order.dtso.txt
@@ -0,0 +1,2 @@
+# mode=strict
+bad/dts-digit-node-order.dtso:30: [child-address-order] child node @10000 out of address order
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-extend-node-child-name-order.dtso.txt b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-child-name-order.dtso.txt
new file mode 100644
index 00000000000000..d7941a89113562
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-child-name-order.dtso.txt
@@ -0,0 +1,2 @@
+# mode=strict
+bad/dts-extend-node-child-name-order.dtso:23: [child-name-order] child node 'bar' out of name order
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-extend-node-digit-node-order.dtso.txt b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-digit-node-order.dtso.txt
new file mode 100644
index 00000000000000..408796d5bb0312
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-extend-node-digit-node-order.dtso.txt
@@ -0,0 +1,2 @@
+# mode=strict
+bad/dts-extend-node-digit-node-order.dtso:24: [child-address-order] child node @10000 out of address order
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-line-length.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-line-length.dts.txt
new file mode 100644
index 00000000000000..9cdb7550b56c7d
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-line-length.dts.txt
@@ -0,0 +1,3 @@
+# mode=stricter
+bad/dts-line-length.dts:17: [line-length-dts] line exceeds 80 columns (101)
+bad/dts-line-length.dts:18: [line-length-dts] line exceeds 80 columns (108)
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-mixed-indent.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-mixed-indent.dts.txt
index 93146cfb51c78d..1706390f56648e 100644
--- a/scripts/dtc/dt-style-selftest/expected/dts-mixed-indent.dts.txt
+++ b/scripts/dtc/dt-style-selftest/expected/dts-mixed-indent.dts.txt
@@ -6,4 +6,5 @@ bad/dts-mixed-indent.dts:12: [mixed-indent-chars] mixed tabs and spaces in inden
 bad/dts-mixed-indent.dts:13: [indent-consistent] indent mismatch (expected depth 2 * '\t')
 bad/dts-mixed-indent.dts:13: [mixed-indent-chars] mixed tabs and spaces in indent
 bad/dts-mixed-indent.dts:16: [mixed-indent-chars] mixed tabs and spaces in indent
+bad/dts-mixed-indent.dts:18: [mixed-indent-chars] too many space characters in indent (more than 7)
 bad/dts-mixed-indent.dts:19: [indent-consistent] indent mismatch (expected depth 2 * '\t')
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-node-name.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-node-name.dts.txt
new file mode 100644
index 00000000000000..bd8d7e21e243a5
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-node-name.dts.txt
@@ -0,0 +1,13 @@
+# mode=strict
+bad/dts-node-name.dts:8: [node-name] node name "foo_bar" is using discouraged style
+bad/dts-node-name.dts:12: [child-name-order] child node 'foo,bar' out of name order
+bad/dts-node-name.dts:12: [node-name] node name "foo,bar" is using discouraged style
+bad/dts-node-name.dts:16: [node-name] node name "foo.bar" is using discouraged style
+bad/dts-node-name.dts:20: [child-name-order] child node 'foo+bar' out of name order
+bad/dts-node-name.dts:20: [node-name] node name "foo+bar" is using discouraged style
+bad/dts-node-name.dts:24: [node-name] node name "foo-bar-" is using discouraged style
+bad/dts-node-name.dts:35: [node-name] node name "foo_bar" is using discouraged style
+bad/dts-node-name.dts:40: [node-name] node name "foo,bar" is using discouraged style
+bad/dts-node-name.dts:45: [node-name] node name "foo.bar" is using discouraged style
+bad/dts-node-name.dts:50: [node-name] node name "foo+bar" is using discouraged style
+bad/dts-node-name.dts:55: [node-name] node name "foo-bar-" is using discouraged style
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-property-name.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-property-name.dts.txt
new file mode 100644
index 00000000000000..f447743383dada
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-property-name.dts.txt
@@ -0,0 +1,14 @@
+# mode=strict
+bad/dts-property-name.dts:11: [property-name] property name "property#foo" is using discouraged style
+bad/dts-property-name.dts:12: [property-name] property name "property+foo" is using discouraged style
+bad/dts-property-name.dts:14: [property-name] property name "property-foo-" is using discouraged style
+bad/dts-property-name.dts:15: [property-name] property name "property.foo" is using discouraged style
+bad/dts-property-name.dts:16: [property-name] property name "property?foo" is using discouraged style
+bad/dts-property-name.dts:17: [property-name] property name "property_foo" is using discouraged style
+bad/dts-property-name.dts:19: [property-name] property name "vendor,property#foo" is using discouraged style
+bad/dts-property-name.dts:20: [property-name] property name "vendor,property+foo" is using discouraged style
+bad/dts-property-name.dts:21: [property-name] property name "vendor,property,foo" is using discouraged style
+bad/dts-property-name.dts:23: [property-name] property name "vendor,property-foo-" is using discouraged style
+bad/dts-property-name.dts:24: [property-name] property name "vendor,property.foo" is using discouraged style
+bad/dts-property-name.dts:25: [property-name] property name "vendor,property?foo" is using discouraged style
+bad/dts-property-name.dts:26: [property-name] property name "vendor,property_foo" is using discouraged style
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-property-order.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-property-order.dts.txt
index 4bc21328625f95..155b072f0ac2f6 100644
--- a/scripts/dtc/dt-style-selftest/expected/dts-property-order.dts.txt
+++ b/scripts/dtc/dt-style-selftest/expected/dts-property-order.dts.txt
@@ -1,6 +1,12 @@
 # mode=strict
-bad/dts-property-order.dts:15: [property-order] property 'compatible' out of canonical order (should sort before 'reg')
-bad/dts-property-order.dts:16: [property-order] property 'device_type' out of canonical order (should sort before 'compatible')
-bad/dts-property-order.dts:25: [property-order] property 'dma-coherent' out of canonical order (should sort before 'status')
-bad/dts-property-order.dts:30: [property-order] property 'compatible' out of canonical order (should sort before 'ranges')
-bad/dts-property-order.dts:40: [property-order] property 'compatible' out of canonical order (should sort before 'interrupts')
+bad/dts-property-order.dts:10: [property-order] property 'model' out of canonical order (should sort before 'compatible')
+bad/dts-property-order.dts:12: [property-order] property 'chassis-type' out of canonical order (should sort before 'qcom,board-id')
+bad/dts-property-order.dts:20: [property-order] property 'compatible' out of canonical order (should sort before 'reg')
+bad/dts-property-order.dts:21: [property-order] property 'device_type' out of canonical order (should sort before 'compatible')
+bad/dts-property-order.dts:30: [property-order] property 'dma-coherent' out of canonical order (should sort before 'status')
+bad/dts-property-order.dts:35: [property-order] property 'compatible' out of canonical order (should sort before 'ranges')
+bad/dts-property-order.dts:45: [property-order] property 'compatible' out of canonical order (should sort before 'interrupts')
+bad/dts-property-order.dts:47: [property-order] property 'clocks' out of canonical order (should sort before 'qcom,calibration-variant')
+bad/dts-property-order.dts:53: [redundant-whitespace] extra whitespace before {
+bad/dts-property-order.dts:55: [property-order] property 'model' out of canonical order (should sort before 'compatible')
+bad/dts-property-order.dts:60: [property-order] property 'model' out of canonical order (should sort before 'compatible')
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-property-order.dtso.txt b/scripts/dtc/dt-style-selftest/expected/dts-property-order.dtso.txt
new file mode 100644
index 00000000000000..48c020d21b64c1
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-property-order.dtso.txt
@@ -0,0 +1,12 @@
+# mode=strict
+bad/dts-property-order.dtso:11: [property-order] property 'model' out of canonical order (should sort before 'compatible')
+bad/dts-property-order.dtso:13: [property-order] property 'chassis-type' out of canonical order (should sort before 'qcom,board-id')
+bad/dts-property-order.dtso:21: [property-order] property 'compatible' out of canonical order (should sort before 'reg')
+bad/dts-property-order.dtso:22: [property-order] property 'device_type' out of canonical order (should sort before 'compatible')
+bad/dts-property-order.dtso:31: [property-order] property 'dma-coherent' out of canonical order (should sort before 'status')
+bad/dts-property-order.dtso:36: [property-order] property 'compatible' out of canonical order (should sort before 'ranges')
+bad/dts-property-order.dtso:46: [property-order] property 'compatible' out of canonical order (should sort before 'interrupts')
+bad/dts-property-order.dtso:48: [property-order] property 'clocks' out of canonical order (should sort before 'qcom,calibration-variant')
+bad/dts-property-order.dtso:54: [redundant-whitespace] extra whitespace before {
+bad/dts-property-order.dtso:56: [property-order] property 'model' out of canonical order (should sort before 'compatible')
+bad/dts-property-order.dtso:61: [property-order] property 'model' out of canonical order (should sort before 'compatible')
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws-strict.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws-strict.dts.txt
new file mode 100644
index 00000000000000..ac0d57bdecdf23
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws-strict.dts.txt
@@ -0,0 +1,13 @@
+# mode=strict
+bad/dts-redundant-ws-strict.dts:7: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws-strict.dts:13: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws-strict.dts:13: [redundant-whitespace] extra whitespace after :
+bad/dts-redundant-ws-strict.dts:17: [redundant-whitespace-strict] extra whitespace before =
+bad/dts-redundant-ws-strict.dts:18: [redundant-whitespace-strict] extra whitespace after =
+bad/dts-redundant-ws-strict.dts:19: [redundant-whitespace] extra whitespace before ;
+bad/dts-redundant-ws-strict.dts:22: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws-strict.dts:23: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws-strict.dts:23: [redundant-whitespace] extra whitespace after :
+bad/dts-redundant-ws-strict.dts:24: [redundant-whitespace-strict] extra whitespace before =
+bad/dts-redundant-ws-strict.dts:25: [redundant-whitespace-strict] extra whitespace after =
+bad/dts-redundant-ws-strict.dts:26: [redundant-whitespace] extra whitespace before ;
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dts.txt
new file mode 100644
index 00000000000000..3b2a989045edb5
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dts.txt
@@ -0,0 +1,10 @@
+# mode=relaxed
+bad/dts-redundant-ws.dts:7: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws.dts:13: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws.dts:13: [redundant-whitespace] extra whitespace after :
+bad/dts-redundant-ws.dts:19: [redundant-whitespace] extra whitespace before ;
+bad/dts-redundant-ws.dts:22: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws.dts:23: [redundant-whitespace] extra whitespace before {
+bad/dts-redundant-ws.dts:23: [redundant-whitespace] extra whitespace after :
+bad/dts-redundant-ws.dts:26: [redundant-whitespace] extra whitespace before ;
+bad/dts-redundant-ws.dts:27: [redundant-whitespace] extra whitespace before ;
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dtso.txt b/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dtso.txt
new file mode 100644
index 00000000000000..5f9a6306170907
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-redundant-ws.dtso.txt
@@ -0,0 +1,2 @@
+# mode=strict
+bad/dts-redundant-ws.dtso:5: [redundant-whitespace] extra whitespace before {
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-trailing-ws.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-trailing-ws.dts.txt
new file mode 100644
index 00000000000000..a15a7d637c30f1
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-trailing-ws.dts.txt
@@ -0,0 +1,3 @@
+# mode=relaxed
+bad/dts-trailing-ws.dts:5: [trailing-whitespace] trailing whitespace
+bad/dts-trailing-ws.dts:13: [trailing-whitespace] trailing whitespace
diff --git a/scripts/dtc/dt-style-selftest/expected/dts-unused-label.dts.txt b/scripts/dtc/dt-style-selftest/expected/dts-unused-label.dts.txt
new file mode 100644
index 00000000000000..4cdcaba3ba2f4d
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/dts-unused-label.dts.txt
@@ -0,0 +1,2 @@
+# mode=stricter
+bad/dts-unused-label.dts:16: [unused-labels-dts] label 'dev' defined but never &-referenced
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-blank-lines.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-blank-lines.yaml.txt
new file mode 100644
index 00000000000000..8a8db8d35b1ef6
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-blank-lines.yaml.txt
@@ -0,0 +1,6 @@
+# mode=strict
+bad/yaml-blank-lines.yaml:33: example 0 [blank-lines] consecutive blank lines
+bad/yaml-blank-lines.yaml:40: example 0 [blank-lines] consecutive blank lines
+bad/yaml-blank-lines.yaml:47: example 0 [blank-lines] consecutive blank lines
+bad/yaml-blank-lines.yaml:49: example 0 [blank-lines] blank line at start of node body
+bad/yaml-blank-lines.yaml:53: example 0 [blank-lines] blank line at end of node body
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-cont-align.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-cont-align.yaml.txt
index b5576dd0f6b152..9e98c28867aad8 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-cont-align.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-cont-align.yaml.txt
@@ -1,2 +1,4 @@
 # mode=strict
-bad/yaml-cont-align.yaml:29: example 0 [continuation-alignment] continuation should align to column 11 (under "<" or \")
+bad/yaml-cont-align.yaml:29: example 0 [continuation-alignment] continuation should align to column 11 (to < or ")
+bad/yaml-cont-align.yaml:30: example 0 [continuation-alignment] continuation should align to column 11 (to < or ")
+bad/yaml-cont-align.yaml:32: example 0 [continuation-alignment] continuation should align to column 12 (to the value under <)
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-hex-case.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-hex-case.yaml.txt
index 6600f7cd1ba59b..f42490256939d4 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-hex-case.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-hex-case.yaml.txt
@@ -1,2 +1,4 @@
 # mode=strict
 bad/yaml-hex-case.yaml:28: example 0 [hex-case] hex literal '0xABCD' must be lowercase
+bad/yaml-hex-case.yaml:29: example 0 [hex-case] hex literal '0x2BCD' must be lowercase
+bad/yaml-hex-case.yaml:30: example 0 [hex-case] hex literal '0x3BCD' must be lowercase
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-line-length.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-line-length.yaml.txt
index 89b36360caa42c..f21b823c6136bd 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-line-length.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-line-length.yaml.txt
@@ -1,2 +1,3 @@
 # mode=strict
 bad/yaml-line-length.yaml:27: example 0 [line-length] line exceeds 80 columns (81)
+bad/yaml-line-length.yaml:28: example 0 [line-length] line exceeds 80 columns (88)
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-mixed-indent.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-mixed-indent.yaml.txt
index bc3fc3cf00ccdd..442f21f90e83b3 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-mixed-indent.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-mixed-indent.yaml.txt
@@ -1,2 +1,3 @@
 # mode=relaxed
 bad/yaml-mixed-indent.yaml:27: example 0 [tab-in-yaml] tab character not allowed in DTS example
+bad/yaml-mixed-indent.yaml:30: example 0 [tab-in-yaml] tab character not allowed in DTS example
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-node-name.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-node-name.yaml.txt
new file mode 100644
index 00000000000000..d9cf749e35c219
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-node-name.yaml.txt
@@ -0,0 +1,7 @@
+# mode=strict
+bad/yaml-node-name.yaml:25: example 0 [node-name] node name "foo_bar" is using discouraged style
+bad/yaml-node-name.yaml:30: example 0 [node-name] node name "foo,bar" is using discouraged style
+bad/yaml-node-name.yaml:35: example 0 [node-name] node name "foo.bar" is using discouraged style
+bad/yaml-node-name.yaml:40: example 0 [node-name] node name "foo+bar" is using discouraged style
+bad/yaml-node-name.yaml:45: example 0 [node-name] node name "foo-bar-" is using discouraged style
+bad/yaml-node-name.yaml:52: example 0 [node-name] node name "foo_bar" is using discouraged style
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-prop-order.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-prop-order.yaml.txt
index 578df7209170c2..bb0dd5db513545 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-prop-order.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-prop-order.yaml.txt
@@ -1,2 +1,3 @@
 # mode=strict
 bad/yaml-prop-order.yaml:28: example 0 [property-order] property 'compatible' out of canonical order (should sort before 'reg')
+bad/yaml-prop-order.yaml:32: example 0 [property-order] property 'clocks' out of canonical order (should sort before 'qcom,calibration-variant')
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-prop-pairing.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-prop-pairing.yaml.txt
index e6e21349a939b0..ee666559424ad4 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-prop-pairing.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-prop-pairing.yaml.txt
@@ -1,3 +1,3 @@
 # mode=strict
-bad/yaml-prop-pairing.yaml:30: example 0 [property-order] property 'clocks' out of canonical order (should sort before 'clock-names')
-bad/yaml-prop-pairing.yaml:32: example 0 [property-order] property 'pinctrl-0' out of canonical order (should sort before 'pinctrl-names')
+bad/yaml-prop-pairing.yaml:29: example 0 [property-order] property 'clock-names' must be placed after 'clocks'
+bad/yaml-prop-pairing.yaml:31: example 0 [property-order] property 'pinctrl-names' must be placed after 'pinctrl-0'
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-property-name.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-property-name.yaml.txt
new file mode 100644
index 00000000000000..6aa3a004532b67
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-property-name.yaml.txt
@@ -0,0 +1,14 @@
+# mode=strict
+bad/yaml-property-name.yaml:30: example 0 [property-name] property name "property#foo" is using discouraged style
+bad/yaml-property-name.yaml:31: example 0 [property-name] property name "property+foo" is using discouraged style
+bad/yaml-property-name.yaml:33: example 0 [property-name] property name "property-foo-" is using discouraged style
+bad/yaml-property-name.yaml:34: example 0 [property-name] property name "property.foo" is using discouraged style
+bad/yaml-property-name.yaml:35: example 0 [property-name] property name "property?foo" is using discouraged style
+bad/yaml-property-name.yaml:36: example 0 [property-name] property name "property_foo" is using discouraged style
+bad/yaml-property-name.yaml:38: example 0 [property-name] property name "vendor,property#foo" is using discouraged style
+bad/yaml-property-name.yaml:39: example 0 [property-name] property name "vendor,property+foo" is using discouraged style
+bad/yaml-property-name.yaml:40: example 0 [property-name] property name "vendor,property,foo" is using discouraged style
+bad/yaml-property-name.yaml:42: example 0 [property-name] property name "vendor,property-foo-" is using discouraged style
+bad/yaml-property-name.yaml:43: example 0 [property-name] property name "vendor,property.foo" is using discouraged style
+bad/yaml-property-name.yaml:44: example 0 [property-name] property name "vendor,property?foo" is using discouraged style
+bad/yaml-property-name.yaml:45: example 0 [property-name] property name "vendor,property_foo" is using discouraged style
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws-strict.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws-strict.yaml.txt
new file mode 100644
index 00000000000000..fbf323832c91d7
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws-strict.yaml.txt
@@ -0,0 +1,5 @@
+# mode=strict
+bad/yaml-redundant-ws-strict.yaml:26: example 0 [redundant-whitespace] extra whitespace before {
+bad/yaml-redundant-ws-strict.yaml:27: example 0 [redundant-whitespace-strict] extra whitespace before =
+bad/yaml-redundant-ws-strict.yaml:28: example 0 [redundant-whitespace-strict] extra whitespace after =
+bad/yaml-redundant-ws-strict.yaml:31: example 0 [redundant-whitespace] extra whitespace before ;
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws.yaml.txt
new file mode 100644
index 00000000000000..a162967e433642
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-redundant-ws.yaml.txt
@@ -0,0 +1,4 @@
+# mode=relaxed
+bad/yaml-redundant-ws.yaml:26: example 0 [redundant-whitespace] extra whitespace before {
+bad/yaml-redundant-ws.yaml:29: example 0 [redundant-whitespace] extra whitespace before ;
+bad/yaml-redundant-ws.yaml:30: example 0 [redundant-whitespace] extra whitespace before ;
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-trailing-ws.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-trailing-ws.yaml.txt
index cfdbc8476c7396..4f2ec9572bc698 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-trailing-ws.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-trailing-ws.yaml.txt
@@ -1,2 +1,4 @@
 # mode=relaxed
 bad/yaml-trailing-ws.yaml:27: example 0 [trailing-whitespace] trailing whitespace
+bad/yaml-trailing-ws.yaml:29: example 0 [trailing-whitespace] trailing whitespace
+bad/yaml-trailing-ws.yaml:30: example 0 [trailing-whitespace] trailing whitespace
diff --git a/scripts/dtc/dt-style-selftest/expected/yaml-value-ws-multiline.yaml.txt b/scripts/dtc/dt-style-selftest/expected/yaml-value-ws-multiline.yaml.txt
index 3df55b1762d06a..d25b5b425e3f73 100644
--- a/scripts/dtc/dt-style-selftest/expected/yaml-value-ws-multiline.yaml.txt
+++ b/scripts/dtc/dt-style-selftest/expected/yaml-value-ws-multiline.yaml.txt
@@ -1,2 +1,3 @@
 # mode=strict
 bad/yaml-value-ws-multiline.yaml:25: example 0 [value-whitespace] extra whitespace inside <...>
+bad/yaml-value-ws-multiline.yaml:26: example 0 [continuation-alignment] continuation should align to column 12 (to the value under <)
diff --git a/scripts/dtc/dt-style-selftest/good/dts-child-name-order.dtso b/scripts/dtc/dt-style-selftest/good/dts-child-name-order.dtso
new file mode 100644
index 00000000000000..1fd75a6285ce3c
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/good/dts-child-name-order.dtso
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	memory@a0000000 {
+		device_type = "memory";
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	pmu {
+		compatible = "example,pmu";
+
+		/* Include labels to be sure they do not affect sorting */
+		label_bar: bar {
+			label = "bar";
+		};
+
+		foo: foo {
+			label = "foo";
+		};
+	};
+
+	pmu-2 {
+		compatible = "example,pmu";
+
+		/* Just reference labels to avoid strict warnings */
+		example,foo = <&foo>, <&label_bar>;
+	};
+};
+
+&pmu {
+	/* Include labels to be sure they do not affect sorting */
+	label_bar_2: bar-2 {
+		label = "bar";
+	};
+
+	foo_2: foo-2 {
+		label = "foo";
+	};
+}
diff --git a/scripts/dtc/dt-style-selftest/good/dts-cont-align.dts b/scripts/dtc/dt-style-selftest/good/dts-cont-align.dts
index 36fb4eefcd83b9..e6450a1d8209e8 100644
--- a/scripts/dtc/dt-style-selftest/good/dts-cont-align.dts
+++ b/scripts/dtc/dt-style-selftest/good/dts-cont-align.dts
@@ -15,12 +15,16 @@
 	interrupt-controller@10000 {
 		compatible = "example,intc";
 		reg = <0x10000 0x1000>;
-		interrupts = <1 2 3>,
+		interrupts = <1 2 3>, /* comments with " < , should not ... */
 			     <4 5 6>,
+			     /* but comments should be placed properly */
 			     <7 8 9>;
-		pinmux = <
-			 0x01
-			 0x02
-			 >;
+		pinmux = <0x01
+			  0x02>,
+			 <0x03 /* comments with " < , should not matter */
+			  0x04>;
+		power-domain-names = "foo", /* comments with " < , should not */
+				     "bar",
+				     "baz";
 	};
 };
diff --git a/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dts b/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dts
index cdf3f91ebe013d..2b21dde7f3c8eb 100644
--- a/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dts
+++ b/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dts
@@ -5,13 +5,10 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
-	/* TODO: uncomment when child-address-order is fixed for top-level */
-	/*
 	memory@a0000000 {
 		device_type = "memory";
 		reg = <0x0 0xa0000000 0x0 0x0>;
 	};
-	*/
 
 	pmu {
 		compatible = "example,pmu";
diff --git a/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dtso b/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dtso
new file mode 100644
index 00000000000000..efafa1cccc15bd
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/good/dts-digit-node-order.dtso
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	memory@a0000000 {
+		device_type = "memory";
+		reg = <0x0 0xa0000000 0x0 0x0>;
+	};
+
+	pmu {
+		compatible = "example,pmu";
+	};
+
+	soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		interrupt-controller@10000 {
+			compatible = "example,intc";
+			reg = <0x10000 0x1000>;
+			interrupts = <1 2 3>;
+		};
+
+		serial@20000 {
+			compatible = "example,serial";
+			reg = <0x20000 0x1000>;
+		};
+
+		serial@30000 {
+			compatible = "example,serial";
+			reg = <0x30000 0x1000>;
+		};
+	};
+};
+
+&soc {
+	interrupt-controller@110000 {
+		compatible = "example,intc";
+		reg = <0x110000 0x1000>;
+		interrupts = <1 2 3>;
+	};
+
+	serial@120000 {
+		compatible = "example,serial";
+		reg = <0x120000 0x1000>;
+	};
+
+	serial@130000 {
+		compatible = "example,serial";
+		reg = <0x130000 0x1000>;
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/good/dts-extend-node-child-name-order.dtso b/scripts/dtc/dt-style-selftest/good/dts-extend-node-child-name-order.dtso
new file mode 100644
index 00000000000000..b00c2a19517928
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/good/dts-extend-node-child-name-order.dtso
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	pmu {
+		compatible = "example,pmu";
+
+		/* Just reference labels to avoid strict warnings */
+		example,foo = <&foo>, <&label_bar>;
+	};
+};
+
+&pmu {
+	/* Include labels to be sure they do not affect sorting */
+	label_bar: bar {
+		label = "bar";
+	};
+
+	foo: foo {
+		label = "foo";
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/good/dts-extend-node-digit-node-order.dtso b/scripts/dtc/dt-style-selftest/good/dts-extend-node-digit-node-order.dtso
new file mode 100644
index 00000000000000..1ce18afb80de89
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/good/dts-extend-node-digit-node-order.dtso
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	soc: soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+};
+
+&soc {
+	interrupt-controller@10000 {
+		compatible = "example,intc";
+		reg = <0x10000 0x1000>;
+		interrupts = <1 2 3>;
+	};
+
+	serial@20000 {
+		compatible = "example,serial";
+		reg = <0x20000 0x1000>;
+	};
+
+	serial@30000 {
+		compatible = "example,serial";
+		reg = <0x30000 0x1000>;
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/good/dts-property-order.dts b/scripts/dtc/dt-style-selftest/good/dts-property-order.dts
index 0e183e3459cdf9..288fcfb9888f81 100644
--- a/scripts/dtc/dt-style-selftest/good/dts-property-order.dts
+++ b/scripts/dtc/dt-style-selftest/good/dts-property-order.dts
@@ -6,6 +6,11 @@
 /dts-v1/;
 
 / {
+	model = "DT style selftest";
+	compatible = "example,test-board", "example,test-soc";
+	chassis-type = "handset";
+	qcom,board-id = <8 0>;
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -35,7 +40,10 @@
 		interrupt-controller@10000 {
 			compatible = "example,intc";
 			reg = <0x10000 0x1000>;
+			clocks = <6>;
 			interrupts = <1 2 3>;
+			qcom,calibration-variant = "foo";
+			qcom,opp-fuse-level = <1>;
 		};
 	};
 };
diff --git a/scripts/dtc/dt-style-selftest/good/dts-property-order.dtso b/scripts/dtc/dt-style-selftest/good/dts-property-order.dtso
new file mode 100644
index 00000000000000..60c68b3f908a1d
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/good/dts-property-order.dtso
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+/*
+ * Test fixture: Incorrect property order
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	model = "DT style selftest";
+	compatible = "example,test-board", "example,test-soc";
+	chassis-type = "handset";
+	qcom,board-id = <8 0>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a57";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+		};
+	};
+
+	pmu {
+		compatible = "example,pmu";
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	soc@0 {
+		compatible = "simple-bus";
+		ranges = <0 0 0 0xc0000000>;
+
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		interrupt-controller@10000 {
+			compatible = "example,intc";
+			reg = <0x10000 0x1000>;
+			clocks = <6>;
+			interrupts = <1 2 3>;
+			qcom,calibration-variant = "foo";
+			qcom,opp-fuse-level = <1>;
+		};
+	};
+};
diff --git a/scripts/dtc/dt-style-selftest/good/yaml-4space.yaml b/scripts/dtc/dt-style-selftest/good/yaml-4space.yaml
index 1502f803c24c14..b91b8a8a95127f 100644
--- a/scripts/dtc/dt-style-selftest/good/yaml-4space.yaml
+++ b/scripts/dtc/dt-style-selftest/good/yaml-4space.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Test fixture for dt-check-style
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/good/yaml-cont-align.yaml b/scripts/dtc/dt-style-selftest/good/yaml-cont-align.yaml
new file mode 100644
index 00000000000000..4a5b5ad43ee823
--- /dev/null
+++ b/scripts/dtc/dt-style-selftest/good/yaml-cont-align.yaml
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/test-good-cont-align.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Test fixture with aligned multi-line property
+
+maintainers:
+  - Rob Herring <robh@kernel.org>
+
+properties:
+  compatible:
+    const: example,test-cont-align
+  reg:
+    maxItems: 2
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    foo@1000 {
+        compatible = "example,test-cont-align";
+        reg = <0x1000 0x100>,
+              <0x2000 0x100>,
+              /* but comments should be placed properly */
+              <0x3000
+               0x100>;
+    };
diff --git a/scripts/dtc/dt-style-selftest/good/yaml-tricky-parsing.yaml b/scripts/dtc/dt-style-selftest/good/yaml-tricky-parsing.yaml
index a836d5f36b9341..a95d07f883effe 100644
--- a/scripts/dtc/dt-style-selftest/good/yaml-tricky-parsing.yaml
+++ b/scripts/dtc/dt-style-selftest/good/yaml-tricky-parsing.yaml
@@ -17,7 +17,7 @@ description: |
       not leave the parser in block-comment state.
 
 maintainers:
-  - Test User <test@example.com>
+  - Rob Herring <robh@kernel.org>
 
 properties:
   compatible:
diff --git a/scripts/dtc/dt-style-selftest/run.sh b/scripts/dtc/dt-style-selftest/run.sh
index 8117dd9be90a66..5691301d6a4acf 100755
--- a/scripts/dtc/dt-style-selftest/run.sh
+++ b/scripts/dtc/dt-style-selftest/run.sh
@@ -26,7 +26,7 @@ run() {
 # good/ -- must exit 0 and produce no output in both modes
 for f in "$here"/good/*; do
     [ -e "$f" ] || continue
-    for mode in relaxed strict; do
+    for mode in relaxed strict stricter; do
         out=$(run "$f" "$mode")
         rc=$?
         if [ -n "$out" ] || [ "$rc" -ne 0 ]; then
diff --git a/scripts/elf-parse.c b/scripts/elf-parse.c
index 99869ff91a8c7f..ce1c2a670244eb 100644
--- a/scripts/elf-parse.c
+++ b/scripts/elf-parse.c
@@ -17,15 +17,21 @@ struct elf_funcs elf_parser;
  * Get the whole file as a programming convenience in order to avoid
  * malloc+lseek+read+free of many pieces.  If successful, then mmap
  * avoids copying unused pieces; else just read the whole file.
- * Open for both read and write.
+ * Open for both read and write if writable is true, otherwise open
+ * read-only.
  */
-static void *map_file(char const *fname, size_t *size)
+static void *map_file(char const *fname, size_t *size, bool writable)
 {
-	int fd;
+	int fd, prot = PROT_READ, flags = MAP_PRIVATE;
 	struct stat sb;
 	void *addr = NULL;
 
-	fd = open(fname, O_RDWR);
+	if (writable) {
+		prot |= PROT_WRITE;
+		flags = MAP_SHARED;
+	}
+
+	fd = open(fname, writable ? O_RDWR : O_RDONLY);
 	if (fd < 0) {
 		perror(fname);
 		return NULL;
@@ -39,7 +45,7 @@ static void *map_file(char const *fname, size_t *size)
 		goto out;
 	}
 
-	addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+	addr = mmap(0, sb.st_size, prot, flags, fd, 0);
 	if (addr == MAP_FAILED) {
 		fprintf(stderr, "Could not mmap file: %s\n", fname);
 		goto out;
@@ -57,6 +63,12 @@ static int elf_parse(const char *fname, void *addr, uint32_t types)
 	Elf_Ehdr *ehdr = addr;
 	uint16_t type;
 
+	if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 ||
+	    ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) {
+		fprintf(stderr, "unrecognized ELF file %s\n", fname);
+		return -1;
+	}
+
 	switch (ehdr->e32.e_ident[EI_DATA]) {
 	case ELFDATA2LSB:
 		elf_parser.r	= rle;
@@ -78,12 +90,6 @@ static int elf_parse(const char *fname, void *addr, uint32_t types)
 		return -1;
 	}
 
-	if (memcmp(ELFMAG, ehdr->e32.e_ident, SELFMAG) != 0 ||
-	    ehdr->e32.e_ident[EI_VERSION] != EV_CURRENT) {
-		fprintf(stderr, "unrecognized ELF file %s\n", fname);
-		return -1;
-	}
-
 	type = elf_parser.r2(&ehdr->e32.e_type);
 	if (!((1 << type) & types)) {
 		fprintf(stderr, "Invalid ELF type file %s\n", fname);
@@ -103,7 +109,9 @@ static int elf_parse(const char *fname, void *addr, uint32_t types)
 		elf_parser.shdr_name		= shdr32_name;
 		elf_parser.shdr_type		= shdr32_type;
 		elf_parser.shdr_entsize		= shdr32_entsize;
+		elf_parser.shdr_flags		= shdr32_flags;
 		elf_parser.sym_type		= sym32_type;
+		elf_parser.sym_bind		= sym32_bind;
 		elf_parser.sym_name		= sym32_name;
 		elf_parser.sym_value		= sym32_value;
 		elf_parser.sym_shndx		= sym32_shndx;
@@ -133,7 +141,9 @@ static int elf_parse(const char *fname, void *addr, uint32_t types)
 		elf_parser.shdr_name		= shdr64_name;
 		elf_parser.shdr_type		= shdr64_type;
 		elf_parser.shdr_entsize		= shdr64_entsize;
+		elf_parser.shdr_flags		= shdr64_flags;
 		elf_parser.sym_type		= sym64_type;
+		elf_parser.sym_bind		= sym64_bind;
 		elf_parser.sym_name		= sym64_name;
 		elf_parser.sym_value		= sym64_value;
 		elf_parser.sym_shndx		= sym64_shndx;
@@ -174,12 +184,13 @@ int elf_map_long_size(void *addr)
 	return ehdr->e32.e_ident[EI_CLASS] == ELFCLASS32 ? 4 : 8;
 }
 
-void *elf_map(char const *fname, size_t *size, uint32_t types)
+static void *__elf_map(char const *fname, size_t *size, uint32_t types,
+		       bool writable)
 {
 	void *addr;
 	int ret;
 
-	addr = map_file(fname, size);
+	addr = map_file(fname, size, writable);
 	if (!addr)
 		return NULL;
 
@@ -192,7 +203,73 @@ void *elf_map(char const *fname, size_t *size, uint32_t types)
 	return addr;
 }
 
+void *elf_map(char const *fname, size_t *size, uint32_t types)
+{
+	return __elf_map(fname, size, types, true);
+}
+
+/* For tools that only read the file. */
+void *elf_map_ro(char const *fname, size_t *size, uint32_t types)
+{
+	return __elf_map(fname, size, types, false);
+}
+
 void elf_unmap(void *addr, size_t size)
 {
 	munmap(addr, size);
 }
+
+/*
+ * Open an ELF file and map it read-only, locating its section headers, section
+ * names and symbol table which are populated in the elf out parameter.
+ *
+ * Returns 0 on success, otherwise -1.
+ */
+int elf_open_ro(char const *fname, uint32_t types, struct elf_file *elf)
+{
+	Elf_Ehdr *ehdr;
+	Elf_Shdr *first;
+	unsigned int shstrndx, i;
+
+	memset(elf, 0, sizeof(*elf));
+	elf->base = elf_map_ro(fname, &elf->size, types);
+	if (!elf->base)
+		return -1;
+
+	ehdr = elf->base;
+	elf->shdrs = (const char *)elf->base + ehdr_shoff(ehdr);
+	elf->shentsize = ehdr_shentsize(ehdr);
+	first = elf_section(elf, 0);
+
+	/* A count or index too large for the header lives in section 0. */
+	elf->shnum = ehdr_shnum(ehdr);
+	if (elf->shnum == SHN_UNDEF)
+		elf->shnum = shdr_size(first);
+	shstrndx = ehdr_shstrndx(ehdr);
+	if (shstrndx == SHN_XINDEX)
+		shstrndx = shdr_link(first);
+	elf->shstrtab = (const char *)elf->base +
+			shdr_offset(elf_section(elf, shstrndx));
+
+	for (i = 0; i < elf->shnum && !elf->symtab; i++)
+		if (shdr_type(elf_section(elf, i)) == SHT_SYMTAB)
+			elf->symtab = elf_section(elf, i);
+
+	if (!elf->symtab) {
+		fprintf(stderr, "%s: no symbol table\n", fname);
+		elf_close(elf);
+		return -1;
+	}
+
+	elf->strtab = (const char *)elf->base +
+		      shdr_offset(elf_section(elf, shdr_link(elf->symtab)));
+	elf->nr_syms = shdr_size(elf->symtab) / shdr_entsize(elf->symtab);
+
+	return 0;
+}
+
+void elf_close(struct elf_file *elf)
+{
+	elf_unmap(elf->base, elf->size);
+	elf->base = NULL;
+}
diff --git a/scripts/elf-parse.h b/scripts/elf-parse.h
index f4411e03069dc5..c5ad754da25403 100644
--- a/scripts/elf-parse.h
+++ b/scripts/elf-parse.h
@@ -37,10 +37,12 @@ struct elf_funcs {
 	uint64_t (*shdr_offset)(Elf_Shdr *shdr);
 	uint64_t (*shdr_size)(Elf_Shdr *shdr);
 	uint64_t (*shdr_entsize)(Elf_Shdr *shdr);
+	uint64_t (*shdr_flags)(Elf_Shdr *shdr);
 	uint32_t (*shdr_link)(Elf_Shdr *shdr);
 	uint32_t (*shdr_name)(Elf_Shdr *shdr);
 	uint32_t (*shdr_type)(Elf_Shdr *shdr);
 	uint8_t (*sym_type)(Elf_Sym *sym);
+	uint8_t (*sym_bind)(Elf_Sym *sym);
 	uint32_t (*sym_name)(Elf_Sym *sym);
 	uint64_t (*sym_value)(Elf_Sym *sym);
 	uint16_t (*sym_shndx)(Elf_Sym *sym);
@@ -143,6 +145,7 @@ SHDR_ADDR(addr)
 SHDR_ADDR(offset)
 SHDR_ADDR(size)
 SHDR_ADDR(entsize)
+SHDR_ADDR(flags)
 
 SHDR_WORD(link)
 SHDR_WORD(name)
@@ -211,6 +214,21 @@ static inline uint8_t sym_type(Elf_Sym *sym)
 	return elf_parser.sym_type(sym);
 }
 
+static inline uint8_t sym64_bind(Elf_Sym *sym)
+{
+	return ELF64_ST_BIND(sym->e64.st_info);
+}
+
+static inline uint8_t sym32_bind(Elf_Sym *sym)
+{
+	return ELF32_ST_BIND(sym->e32.st_info);
+}
+
+static inline uint8_t sym_bind(Elf_Sym *sym)
+{
+	return elf_parser.sym_bind(sym);
+}
+
 SYM_ADDR(value)
 SYM_WORD(name)
 SYM_HALF(shndx)
@@ -298,8 +316,44 @@ static inline void w8le(uint64_t val, uint64_t *x)
 }
 
 void *elf_map(char const *fname, size_t *size, uint32_t types);
+void *elf_map_ro(char const *fname, size_t *size, uint32_t types);
 void elf_unmap(void *addr, size_t size);
 int elf_map_machine(void *addr);
 int elf_map_long_size(void *addr);
 
+/* A mapped file with its section headers, section names and symbol table. */
+struct elf_file {
+	void *base;
+	size_t size;
+	const char *shdrs;
+	unsigned int shnum, shentsize;
+	const char *shstrtab;
+	Elf_Shdr *symtab;
+	const char *strtab;
+	size_t nr_syms;
+};
+
+int elf_open_ro(char const *fname, uint32_t types, struct elf_file *elf);
+void elf_close(struct elf_file *elf);
+
+static inline Elf_Shdr *elf_section(const struct elf_file *elf,
+				    unsigned int index)
+{
+	return (Elf_Shdr *)(elf->shdrs + (size_t)index * elf->shentsize);
+}
+
+static inline const char *elf_section_name(const struct elf_file *elf,
+					   Elf_Shdr *shdr)
+{
+	return elf->shstrtab + shdr_name(shdr);
+}
+
+static inline Elf_Sym *elf_symbol(const struct elf_file *elf, size_t index)
+{
+	const char *base = elf->base;
+
+	return (Elf_Sym *)(base + shdr_offset(elf->symtab) +
+			   index * shdr_entsize(elf->symtab));
+}
+
 #endif /* _SCRIPTS_ELF_PARSE_H */
diff --git a/scripts/faddr2line b/scripts/faddr2line
index 622875396bcfcb..3aca197d513f05 100755
--- a/scripts/faddr2line
+++ b/scripts/faddr2line
@@ -243,7 +243,7 @@ __faddr2line() {
 			local cur_sym_elf_size=${fields[2]}
 			local cur_sym_name=${fields[7]:-}
 
-			# is_mapping_symbol(cur_sym_name)
+			# is_ignored_kernel_symbol(cur_sym_name)
 			if [[ ${cur_sym_name} =~ ^(\.L|L0|\$) ]]; then
 				continue
 			fi
diff --git a/scripts/gcc-plugins/randomize_layout_plugin.c b/scripts/gcc-plugins/randomize_layout_plugin.c
index ff65a4f87f240a..1e66f45fe29bc7 100644
--- a/scripts/gcc-plugins/randomize_layout_plugin.c
+++ b/scripts/gcc-plugins/randomize_layout_plugin.c
@@ -22,7 +22,7 @@
 #define ORIG_TYPE_NAME(node) \
 	(TYPE_NAME(TYPE_MAIN_VARIANT(node)) != NULL_TREE ? ((const unsigned char *)IDENTIFIER_POINTER(TYPE_NAME(TYPE_MAIN_VARIANT(node)))) : (const unsigned char *)"anonymous")
 
-#define INFORM(loc, msg, ...)	inform(loc, "randstruct: " msg, ##__VA_ARGS__)
+#define INFORM(loc, msg, ...)	warning_at(loc, 0, "randstruct: " msg, ##__VA_ARGS__)
 #define MISMATCH(loc, how, ...)	INFORM(loc, "casting between randomized structure pointer types (" how "): %qT and %qT\n", __VA_ARGS__)
 
 __visible int plugin_is_GPL_compatible;
@@ -698,6 +698,63 @@ static void handle_local_var_initializers(void)
 	}
 }
 
+/*
+ * Does @container reach a field of type @member_type by a chain of
+ * by-value members? That is the relationship container_of() expresses --
+ * its @member argument may be a dotted path, e.g.
+ * container_of(inode, struct ceph_inode_info, netfs.inode) -- so a cast
+ * from @member_type * to @container * is legitimate rather than a
+ * layout-confusing one.
+ *
+ * container_of() used to leave a "void *__mptr" temporary behind, and this
+ * pass recognised such casts by that name. Commit f9e7a7564834
+ * ("container_of: remove local __mptr variable") removed it to stop nested
+ * container_of() shadowing itself, and the cast now folds to a bare SSA
+ * copy when the member sits at offset 0, leaving nothing syntactic to key
+ * on. Match the type relationship instead.
+ *
+ * The depth bound keeps this cheap; container_of() paths are one or two
+ * members deep in practice.
+ */
+#define CONTAINER_OF_MAX_DEPTH 4
+
+static bool is_container_of_cast(const_tree container, const_tree member_type,
+				 int depth)
+{
+	const_tree field;
+
+	if (container == NULL_TREE || depth > CONTAINER_OF_MAX_DEPTH)
+		return false;
+
+	if (TREE_CODE(container) != RECORD_TYPE &&
+	    TREE_CODE(container) != UNION_TYPE)
+		return false;
+
+	for (field = TYPE_FIELDS(container); field; field = DECL_CHAIN(field)) {
+		const_tree field_type;
+
+		if (TREE_CODE(field) != FIELD_DECL)
+			continue;
+
+		/*
+		 * Only a member at offset 0 can reach here: for any other
+		 * offset container_of()'s subtraction survives folding, the
+		 * cast's rhs stays void *, and the caller skipped it above.
+		 */
+		if (!integer_zerop(byte_position(field)))
+			continue;
+
+		field_type = TYPE_MAIN_VARIANT(TREE_TYPE(field));
+		if (field_type == member_type)
+			return true;
+
+		if (is_container_of_cast(field_type, member_type, depth + 1))
+			return true;
+	}
+
+	return false;
+}
+
 /*
  * iterate over all statements to find "bad" casts:
  * those where the address of the start of a structure is cast
@@ -799,10 +856,8 @@ static unsigned int find_bad_casts_execute(void)
 #endif
 				MISMATCH(gimple_location(stmt), "op0", ptr_lhs_type, op0_type);
 			} else {
-				const_tree ssa_name_var = SSA_NAME_VAR(rhs1);
 				/* skip bogus type casts introduced by container_of */
-				if (ssa_name_var != NULL_TREE && DECL_NAME(ssa_name_var) && 
-				    !strcmp((const char *)DECL_NAME_POINTER(ssa_name_var), "__mptr"))
+				if (is_container_of_cast(ptr_lhs_type, ptr_rhs_type, 0))
 					continue;
 #ifndef __DEBUG_PLUGIN
 				if (lookup_attribute("randomize_performed", TYPE_ATTRIBUTES(ptr_rhs_type)))
diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py
index dffadccbb01d24..7398105ff3f677 100644
--- a/scripts/gdb/linux/mm.py
+++ b/scripts/gdb/linux/mm.py
@@ -56,7 +56,7 @@ def __init__(self):
 
         self.MAX_PHYSMEM_BITS = 46
         self.SECTION_SIZE_BITS = 27
-        self.MAX_ORDER = 10
+        self.MAX_PAGE_ORDER = 10
 
         self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS
         self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT
@@ -70,14 +70,9 @@ def __init__(self):
             self.SECTIONS_PER_ROOT = 1
 
         self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT)
-        self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1
 
-        try:
-            self.SECTION_HAS_MEM_MAP = 1 << int(gdb.parse_and_eval('SECTION_HAS_MEM_MAP_BIT'))
-            self.SECTION_IS_EARLY = 1 << int(gdb.parse_and_eval('SECTION_IS_EARLY_BIT'))
-        except:
-            self.SECTION_HAS_MEM_MAP = 1 << 0
-            self.SECTION_IS_EARLY = 1 << 3
+        self.SECTION_HAS_MEM_MAP = 1 << int(gdb.parse_and_eval('SECTION_HAS_MEM_MAP_BIT'))
+        self.SECTION_IS_EARLY = 1 << int(gdb.parse_and_eval('SECTION_IS_EARLY_BIT'))
 
         self.SUBSECTION_SHIFT = 21
         self.PAGES_PER_SUBSECTION = 1 << (self.SUBSECTION_SHIFT - self.PAGE_SHIFT)
@@ -100,7 +95,7 @@ def SECTION_NR_TO_ROOT(self, sec):
     def __nr_to_section(self, nr):
         root = self.SECTION_NR_TO_ROOT(nr)
         mem_section = gdb.parse_and_eval("mem_section")
-        return mem_section[root][nr & self.SECTION_ROOT_MASK]
+        return mem_section[root][nr % self.SECTIONS_PER_ROOT]
 
     def pfn_to_section_nr(self, pfn):
         return pfn >> self.PFN_SECTION_SHIFT
@@ -233,11 +228,11 @@ def __init__(self):
         self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS
 
         if str(constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER).isdigit():
-            self.MAX_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER
+            self.MAX_PAGE_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER
         else:
-            self.MAX_ORDER = 10
+            self.MAX_PAGE_ORDER = 10
 
-        self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_ORDER)
+        self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_PAGE_ORDER)
         self.PFN_SECTION_SHIFT = self.SECTION_SIZE_BITS - self.PAGE_SHIFT
         self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT
         self.PAGES_PER_SECTION = 1 << self.PFN_SECTION_SHIFT
@@ -249,7 +244,6 @@ def __init__(self):
             self.SECTIONS_PER_ROOT = 1
 
         self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT)
-        self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1
         self.SUBSECTION_SHIFT = 21
         self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT
         self.PFN_SUBSECTION_SHIFT = self.SUBSECTION_SHIFT - self.PAGE_SHIFT
@@ -304,7 +298,7 @@ def SECTION_NR_TO_ROOT(self, sec):
     def __nr_to_section(self, nr):
         root = self.SECTION_NR_TO_ROOT(nr)
         mem_section = gdb.parse_and_eval("mem_section")
-        return mem_section[root][nr & self.SECTION_ROOT_MASK]
+        return mem_section[root][nr % self.SECTIONS_PER_ROOT]
 
     def pfn_to_section_nr(self, pfn):
         return pfn >> self.PFN_SECTION_SHIFT
diff --git a/usr/gen_init_cpio.c b/scripts/gen_init_cpio.c
similarity index 100%
rename from usr/gen_init_cpio.c
rename to scripts/gen_init_cpio.c
diff --git a/usr/gen_initramfs.sh b/scripts/gen_initramfs.sh
similarity index 97%
rename from usr/gen_initramfs.sh
rename to scripts/gen_initramfs.sh
index 7eba2fddf0ef2e..9c6168a9338d44 100755
--- a/usr/gen_initramfs.sh
+++ b/scripts/gen_initramfs.sh
@@ -6,7 +6,7 @@
 #
 # Generate a cpio packed initramfs. It uses gen_init_cpio to generate
 # the cpio archive.
-# This script assumes that gen_init_cpio is located in usr/ directory
+# This script assumes that gen_init_cpio is located in scripts/ directory
 
 # error out on errors
 set -e
@@ -247,4 +247,4 @@ done
 
 # If output_file is set we will generate cpio archive
 # we are careful to delete tmp files
-usr/gen_init_cpio $output $timestamp $cpio_list
+scripts/gen_init_cpio $output $timestamp $cpio_list
diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index dc1219736f77a9..621895bae128f0 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -250,6 +250,7 @@ def append_sysroot_crate(
         "macros",
         srctree / "rust" / "macros" / "lib.rs",
         [std, proc_macro, proc_macro2, quote, syn],
+        cfg=generated_cfg,
     )
 
     zerocopy_derive = append_proc_macro_crate(
diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs
index 3bf296581a88ec..7687b0dd5474e5 100644
--- a/scripts/generate_rust_target.rs
+++ b/scripts/generate_rust_target.rs
@@ -224,6 +224,11 @@ fn main() {
             features += ",+harden-sls-ijmp";
             features += ",+harden-sls-ret";
         }
+        if cfg.has("X86_NATIVE_CPU") {
+            // Prevent the backend from generating APX instructions. The kernel is not yet prepared
+            // for general in-kernel EGPR use.
+            features += ",-apxf";
+        }
         ts.push("features", features);
         ts.push("llvm-target", "x86_64-linux-gnu");
         ts.push("supported-sanitizers", ["kcfi", "kernel-address"]);
diff --git a/scripts/jobserver-exec b/scripts/jobserver-exec
index 21b319e6c9a5f5..89119acb1b9e22 100755
--- a/scripts/jobserver-exec
+++ b/scripts/jobserver-exec
@@ -4,7 +4,8 @@
 """
 Determines how many parallel tasks "make" is expecting, as it is
 not exposed via any special variables, reserves them all, runs a subprocess
-with PARALLELISM environment variable set, and releases the jobs back again.
+with PARALLELISM environment variable set and any %PARALLELISM% in its
+arguments replaced by the count, and releases the jobs back again.
 
 See:
     https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver
diff --git a/scripts/kallsyms-sysmap.c b/scripts/kallsyms-sysmap.c
new file mode 100644
index 00000000000000..a482968eba5112
--- /dev/null
+++ b/scripts/kallsyms-sysmap.c
@@ -0,0 +1,248 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Obtain symbols from vmlinux for usage by kallsyms. Replaces mksysmap.
+ *
+ * To retain compatibility, it provides the same output as nm, only faster.
+ */
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <xalloc.h>
+
+#include "elf-parse.h"
+#include "kallsyms.h"
+
+/*
+ * What mksysmap dropped from System.map, by name: symbols that are not needed
+ * there, or not suitable for kallsyms.
+ */
+static const char *const sysmap_omit_prefixes[] = {
+	/* local symbols for ARM, MIPS, etc. */
+	"$",
+	/* local labels, .LBB, .Ltmpxxx, .L__unnamed_xx, .LASANPC, etc. */
+	".L",
+	/* arm64 EFI stub namespace */
+	"__efistub_",
+	/* arm64 local symbols in PIE namespace */
+	"__pi_$",
+	"__pi_.L",
+	/* arm64 local symbols in non-VHE KVM namespace */
+	"__kvm_nvhe_$",
+	"__kvm_nvhe_.L",
+	/* CFI type identifiers */
+	"__kcfi_typeid_",
+	"__kvm_nvhe___kcfi_typeid_",
+	"__pi___kcfi_typeid_",
+	/* CRC from modversions */
+	"__crc_",
+	/* EXPORT_SYMBOL (symbol name) */
+	"__kstrtab_",
+	/* EXPORT_SYMBOL (namespace) */
+	"__kstrtabns_",
+	/* MODULE_DEVICE_TABLE (symbol name) */
+	"__mod_device_table__",
+};
+
+static const char *const sysmap_omit_suffixes[] = {
+	/* arm */
+	"_from_arm",
+	"_from_thumb",
+	"_veneer",
+};
+
+static const char *const sysmap_omit_names[] = {
+	/* for LoongArch? */
+	"L0",
+	/* ppc */
+	"_SDA_BASE_",
+	"_SDA2_BASE_",
+};
+
+/* Symbols that contain the pattern anywhere are dropped. */
+static const char *const sysmap_omit_patterns[] = {
+	/* ppc stub */
+	".long_branch.",
+	".plt_branch.",
+};
+
+/* __<alnum>*Thunk_: the linker's range extension thunks on arm. */
+static bool is_range_thunk(const char *name)
+{
+	const char *p;
+
+	if (!string_starts_with(name, "__"))
+		return false;
+	for (p = name + 2; isalnum((unsigned char)*p); p++)
+		;
+	return p - name >= 7 && *p == '_' && strncmp(p - 5, "Thunk", 5) == 0;
+}
+
+/* __UNIQUE_ID_modinfo_<n>: the MODULE_INFO() strings of built-in code. */
+static bool is_modinfo_id(const char *name)
+{
+	static const char prefix[] = "__UNIQUE_ID_modinfo_";
+	const char *p;
+
+	if (!string_starts_with(name, prefix))
+		return false;
+	for (p = name + strlen(prefix); isdigit((unsigned char)*p); p++)
+		;
+	return !*p;
+}
+
+static bool sysmap_omits(const char *name, char type)
+{
+	size_t i;
+
+	/* Absolute, undefined and debugging symbols. */
+	if (type == 'a' || type == 'N' || type == 'U' || type == 'w')
+		return true;
+
+	for (i = 0; i < ARRAY_SIZE(sysmap_omit_prefixes); i++)
+		if (string_starts_with(name, sysmap_omit_prefixes[i]))
+			return true;
+	for (i = 0; i < ARRAY_SIZE(sysmap_omit_suffixes); i++)
+		if (string_ends_with(name, sysmap_omit_suffixes[i]))
+			return true;
+	for (i = 0; i < ARRAY_SIZE(sysmap_omit_names); i++)
+		if (strcmp(name, sysmap_omit_names[i]) == 0)
+			return true;
+	for (i = 0; i < ARRAY_SIZE(sysmap_omit_patterns); i++)
+		if (strstr(name, sysmap_omit_patterns[i]))
+			return true;
+
+	return is_range_thunk(name) || is_modinfo_id(name);
+}
+
+/* nm's letter for a symbol defined in a section, as BFD classifies it. */
+static char section_symbol_type(Elf_Shdr *shdr, const char *secname)
+{
+	static const char *const debug_prefixes[] = {
+		".debug", ".zdebug", ".gnu.debuglto_.debug_",
+		".gnu.linkonce.wi.", ".line", ".stab",
+	};
+	const uint64_t flags = shdr_flags(shdr);
+	size_t i;
+
+	if (flags & SHF_EXECINSTR)
+		return 't';
+	if (flags & SHF_ALLOC) {
+		if (shdr_type(shdr) == SHT_NOBITS)
+			return 'b';
+		return flags & SHF_WRITE ? 'd' : 'r';
+	}
+	for (i = 0; i < ARRAY_SIZE(debug_prefixes); i++)
+		if (string_starts_with(secname, debug_prefixes[i]))
+			return 'N';
+	if (shdr_type(shdr) != SHT_NOBITS && !(flags & SHF_WRITE))
+		return 'n';
+	return '?';
+}
+
+/* The letter nm prints for a symbol, or 0 for one it leaves out. */
+static char elf_symbol_type(const struct elf_file *elf, Elf_Sym *sym)
+{
+	unsigned int bind = sym_bind(sym), type = sym_type(sym);
+	unsigned int shndx = sym_shndx(sym);
+	Elf_Shdr *shdr;
+	char c;
+
+	if (type == STT_SECTION || type == STT_FILE)
+		return 0;
+	if (shndx == SHN_COMMON)
+		return 'C';
+	if (shndx == SHN_UNDEF) {
+		if (bind == STB_WEAK)
+			return type == STT_OBJECT ? 'v' : 'w';
+		return 'U';
+	}
+	if (type == STT_GNU_IFUNC)
+		return 'i';
+	if (bind == STB_WEAK)
+		return type == STT_OBJECT ? 'V' : 'W';
+	if (bind == STB_GNU_UNIQUE)
+		return 'u';
+	if (bind != STB_GLOBAL && bind != STB_LOCAL)
+		return '?';
+
+	if (shndx == SHN_ABS) {
+		c = 'a';
+	} else if (shndx < elf->shnum) {
+		shdr = elf_section(elf, shndx);
+		c = section_symbol_type(shdr, elf_section_name(elf, shdr));
+	} else {
+		return '?';
+	}
+
+	return bind == STB_GLOBAL ? toupper(c) : c;
+}
+
+/* nm -n order: by address, then by name. */
+static int compare_symbols(const void *a, const void *b)
+{
+	const struct sysmap_symbol *sa = a, *sb = b;
+
+	if (sa->addr != sb->addr)
+		return sa->addr < sb->addr ? -1 : 1;
+	return strcmp(sa->name, sb->name);
+}
+
+/* The symbols "nm -n | mksysmap" would list, in that order. */
+static struct sysmap_symbol *elf_read_symbols(const struct elf_file *elf,
+					   size_t *nr_kept)
+{
+	struct sysmap_symbol *syms = xmalloc(elf->nr_syms * sizeof(*syms));
+	size_t i, n = 0;
+
+	for (i = 1; i < elf->nr_syms; i++) {
+		Elf_Sym *sym = elf_symbol(elf, i);
+		const char *name = elf->strtab + sym_name(sym);
+		char type = elf_symbol_type(elf, sym);
+
+		if (!type || sysmap_omits(name, type))
+			continue;
+
+		syms[n].addr = sym_value(sym);
+		syms[n].name = name;
+		syms[n].type = type;
+		n++;
+	}
+
+	qsort(syms, n, sizeof(*syms), compare_symbols);
+	*nr_kept = n;
+	return syms;
+}
+
+struct sysmap *sysmap_read(const char *path)
+{
+	struct sysmap *map = xcalloc(1, sizeof(*map));
+
+	/* The names point into the mapping; keep it until the map is freed. */
+	if (elf_open_ro(path, (1 << ET_EXEC) | (1 << ET_DYN), &map->elf))
+		exit(EXIT_FAILURE);
+	map->syms = elf_read_symbols(&map->elf, &map->nr_syms);
+	map->addr_width = elf_map_long_size(map->elf.base) * 2;
+
+	return map;
+}
+
+void sysmap_write(const struct sysmap *map, FILE *out)
+{
+	size_t i;
+
+	for (i = 0; i < map->nr_syms; i++) {
+		const struct sysmap_symbol *s = &map->syms[i];
+
+		fprintf(out, "%0*llx %c %s\n", map->addr_width, s->addr, s->type,
+			s->name);
+	}
+}
+
+void sysmap_free(struct sysmap *map)
+{
+	free(map->syms);
+	elf_close(&map->elf);
+	free(map);
+}
diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c
index 494852ade6d87a..bd9e9ce20e8b70 100644
--- a/scripts/kallsyms.c
+++ b/scripts/kallsyms.c
@@ -5,7 +5,15 @@
  * This software may be used and distributed according to the terms
  * of the GNU General Public License, incorporated herein by reference.
  *
- * Usage: kallsyms [--all-symbols] in.map > out.S
+ * Usage: kallsyms [--all-symbols] [--pc-relative] [--sysmap=out.map] in out.bin > out.S
+ *        kallsyms --sysmap=out.map in
+ *
+ *      in is vmlinux; an empty file stands for the first link, which has no
+ *  symbols yet, and gives an empty table. --sysmap also writes the symbols
+ *  in System.map format.
+ *
+ *      The byte tables go to out.bin and are pulled into out.S with .incbin;
+ *  wider tables stay assembler source for endianness and relocations.
  *
  *      Table compression uses all the unused char codes on the symbols and
  *  maps these to the most used substrings (tokens). For instance, it might
@@ -18,7 +26,6 @@
  *
  */
 
-#include <errno.h>
 #include <getopt.h>
 #include <stdbool.h>
 #include <stdio.h>
@@ -26,10 +33,11 @@
 #include <string.h>
 #include <ctype.h>
 #include <limits.h>
-
+#include <sys/stat.h>
 #include <xalloc.h>
 
-#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
+#include "../kernel/kallsyms_internal.h"
+#include "kallsyms.h"
 
 #define KSYM_NAME_LEN		512
 
@@ -58,20 +66,60 @@ static unsigned int table_size, table_cnt;
 static int all_symbols;
 static int pc_relative;
 
-static int token_profit[0x10000];
+/* A dynamic array of symbols, encoded by symbol index. */
+struct sym_arr {
+	unsigned int *sym_indexes;
+	unsigned int cnt, cap;
+};
+
+/* Every pair of bytes is a token. */
+#define NR_TOKENS	0x10000
+
+static int token_profit[NR_TOKENS];
+static struct sym_arr token_syms[NR_TOKENS];
 
 /* the table that holds the result of the compression */
 static unsigned char best_table[256][2];
 static unsigned char best_table_len[256];
 
+static unsigned int sym_arr_last(const struct sym_arr *arr)
+{
+	return arr->cnt ? arr->sym_indexes[arr->cnt - 1] : UINT_MAX;
+}
+
+static void sym_arr_maybe_expand(struct sym_arr *arr)
+{
+	if (arr->cap > arr->cnt)
+		return;
+
+	arr->cap = arr->cap ? arr->cap * 2 : 16;
+	arr->sym_indexes = xrealloc(arr->sym_indexes,
+				    arr->cap * sizeof(*arr->sym_indexes));
+}
+
+static void sym_arr_add(struct sym_arr *arr, unsigned int sym_idx)
+{
+	sym_arr_maybe_expand(arr);
+	arr->sym_indexes[arr->cnt++] = sym_idx;
+}
+
+static void sym_arr_free(struct sym_arr *arr)
+{
+	free(arr->sym_indexes);
+	arr->sym_indexes = NULL;
+	arr->cnt = 0;
+	arr->cap = 0;
+}
 
 static void usage(void)
 {
-	fprintf(stderr, "Usage: kallsyms [--all-symbols] in.map > out.S\n");
+	fprintf(stderr, "Usage: kallsyms [--all-symbols] [--pc-relative] [--sysmap=out.map]\n"
+			"                in out.bin > out.S\n"
+			"       kallsyms --sysmap=out.map vmlinux\n");
 	exit(1);
 }
 
-static char *sym_name(const struct sym_entry *s)
+static char *sym_entry_name(const struct sym_entry *s)
 {
 	return (char *)s->sym + 1;
 }
@@ -112,37 +160,12 @@ static void check_symbol_range(const char *sym, unsigned long long addr,
 	}
 }
 
-static struct sym_entry *read_symbol(FILE *in, char **buf, size_t *buf_len)
+static struct sym_entry *add_symbol(unsigned long long addr, char type,
+				    const char *name)
 {
-	char *name, type, *p;
-	unsigned long long addr;
-	size_t len;
-	ssize_t readlen;
+	size_t len = strlen(name);
 	struct sym_entry *sym;
 
-	errno = 0;
-	readlen = getline(buf, buf_len, in);
-	if (readlen < 0) {
-		if (errno) {
-			perror("read_symbol");
-			exit(EXIT_FAILURE);
-		}
-		return NULL;
-	}
-
-	if ((*buf)[readlen - 1] == '\n')
-		(*buf)[readlen - 1] = 0;
-
-	addr = strtoull(*buf, &p, 16);
-
-	if (*buf == p || *p++ != ' ' || !isascii((type = *p++)) || *p++ != ' ') {
-		fprintf(stderr, "line format error\n");
-		exit(EXIT_FAILURE);
-	}
-
-	name = p;
-	len = strlen(name);
-
 	if (len >= KSYM_NAME_LEN) {
 		fprintf(stderr, "Symbol %s too long for kallsyms (%zu >= %d).\n"
 				"Please increase KSYM_NAME_LEN both in kernel and kallsyms.c\n",
@@ -167,7 +190,7 @@ static struct sym_entry *read_symbol(FILE *in, char **buf, size_t *buf_len)
 	sym->addr = addr;
 	sym->len = len;
 	sym->sym[0] = type;
-	strcpy(sym_name(sym), name);
+	strcpy(sym_entry_name(sym), name);
 
 	return sym;
 }
@@ -188,14 +211,9 @@ static int symbol_in_range(const struct sym_entry *s,
 	return 0;
 }
 
-static bool string_starts_with(const char *s, const char *prefix)
-{
-	return strncmp(s, prefix, strlen(prefix)) == 0;
-}
-
 static int symbol_valid(const struct sym_entry *s)
 {
-	const char *name = sym_name(s);
+	const char *name = sym_entry_name(s);
 
 	/* if --all-symbols is not specified, then symbols outside the text
 	 * and inittext sections are discarded */
@@ -245,36 +263,55 @@ static void shrink_table(void)
 	table_cnt = pos;
 }
 
-static void read_map(const char *in)
+static void add_table_entry(struct sym_entry *sym)
 {
-	FILE *fp;
-	struct sym_entry *sym;
-	char *buf = NULL;
-	size_t buflen = 0;
+	sym->seq = table_cnt;
 
-	fp = fopen(in, "r");
-	if (!fp) {
-		perror(in);
-		exit(1);
+	if (table_cnt >= table_size) {
+		table_size = table_size ? table_size * 2 : 10000;
+		table = xrealloc(table, sizeof(*table) * table_size);
 	}
 
-	while (!feof(fp)) {
-		sym = read_symbol(fp, &buf, &buflen);
-		if (!sym)
-			continue;
+	table[table_cnt++] = sym;
+}
 
-		sym->seq = table_cnt;
+static bool file_is_empty(const char *path)
+{
+	struct stat st;
 
-		if (table_cnt >= table_size) {
-			table_size += 10000;
-			table = xrealloc(table, sizeof(*table) * table_size);
-		}
+	if (stat(path, &st)) {
+		perror(path);
+		exit(EXIT_FAILURE);
+	}
+
+	return st.st_size == 0;
+}
 
-		table[table_cnt++] = sym;
+/*
+ * Read the symbols from vmlinux, writing System.map if asked to. The first
+ * link has no symbols yet: an empty file gives an empty table.
+ */
+static void read_elf(const char *path, FILE *sysmap_out)
+{
+	struct sysmap *map;
+	size_t i;
+
+	if (file_is_empty(path))
+		return;
+
+	map = sysmap_read(path);
+	if (sysmap_out)
+		sysmap_write(map, sysmap_out);
+
+	for (i = 0; i < map->nr_syms; i++) {
+		const struct sysmap_symbol *s = &map->syms[i];
+		struct sym_entry *sym = add_symbol(s->addr, s->type, s->name);
+
+		if (sym)
+			add_table_entry(sym);
 	}
 
-	free(buf);
-	fclose(fp);
+	sysmap_free(map);
 }
 
 static void output_label(const char *label)
@@ -284,6 +321,26 @@ static void output_label(const char *label)
 	printf("%s:\n", label);
 }
 
+static long bin_pos(FILE *file)
+{
+	const long pos = ftell(file);
+
+	if (pos < 0) {
+		perror("kallsyms: ftell");
+		exit(EXIT_FAILURE);
+	}
+
+	return pos;
+}
+
+static void write_incbin(const char *filename, long start, long end)
+{
+	if (start >= end)
+		return;
+
+	printf("\t.incbin \"%s\", %ld, %ld\n", filename, start, end - start);
+}
+
 /* uncompress a compressed symbol. When this function is called, the best table
  * might still be compressed itself, so the function needs to be recursive */
 static int expand_symbol(const unsigned char *data, int len, char *result)
@@ -317,7 +374,7 @@ static int compare_names(const void *a, const void *b)
 	const struct sym_entry *sa = *(const struct sym_entry **)a;
 	const struct sym_entry *sb = *(const struct sym_entry **)b;
 
-	ret = strcmp(sym_name(sa), sym_name(sb));
+	ret = strcmp(sym_entry_name(sa), sym_entry_name(sb));
 	if (!ret) {
 		if (sa->addr > sb->addr)
 			return 1;
@@ -336,11 +393,12 @@ static void sort_symbols_by_name(void)
 	qsort(table, table_cnt, sizeof(table[0]), compare_names);
 }
 
-static void write_src(void)
+static void write_src(FILE *out_bin_file, const char *out_bin_name)
 {
-	unsigned int i, k, off;
+	unsigned int i, off;
 	unsigned int best_idx[256];
 	unsigned int *markers, markers_cnt;
+	long bin_start;
 	char buf[KSYM_NAME_LEN];
 
 	printf("\t.section .rodata, \"a\"\n");
@@ -349,16 +407,19 @@ static void write_src(void)
 	printf("\t.long\t%u\n", table_cnt);
 	printf("\n");
 
-	/* table of offset markers, that give the offset in the compressed stream
-	 * every 256 symbols */
-	markers_cnt = (table_cnt + 255) / 256;
+	/*
+	 * Table of offset markers, giving the offset in the compressed stream
+	 * every (1 << KALLSYMS_MARKER_SHIFT) symbols.
+	 */
+	markers_cnt = (table_cnt + KALLSYMS_MARKER_MASK) >> KALLSYMS_MARKER_SHIFT;
 	markers = xmalloc(sizeof(*markers) * markers_cnt);
 
 	output_label("kallsyms_names");
+	bin_start = bin_pos(out_bin_file);
 	off = 0;
 	for (i = 0; i < table_cnt; i++) {
-		if ((i & 0xFF) == 0)
-			markers[i >> 8] = off;
+		if ((i & KALLSYMS_MARKER_MASK) == 0)
+			markers[i >> KALLSYMS_MARKER_SHIFT] = off;
 		table[i]->seq = i;
 
 		/* There cannot be any symbol of length zero. */
@@ -378,26 +439,24 @@ static void write_src(void)
 		/* Encode length with ULEB128. */
 		if (table[i]->len <= 0x7F) {
 			/* Most symbols use a single byte for the length. */
-			printf("\t.byte 0x%02x", table[i]->len);
+			fputc(table[i]->len, out_bin_file);
 			off += table[i]->len + 1;
 		} else {
 			/* "Big" symbols use two bytes. */
-			printf("\t.byte 0x%02x, 0x%02x",
-				(table[i]->len & 0x7F) | 0x80,
-				(table[i]->len >> 7) & 0x7F);
+			fputc((table[i]->len & 0x7F) | 0x80, out_bin_file);
+			fputc((table[i]->len >> 7) & 0x7F, out_bin_file);
 			off += table[i]->len + 2;
 		}
-		for (k = 0; k < table[i]->len; k++)
-			printf(", 0x%02x", table[i]->sym[k]);
+		fwrite(table[i]->sym, 1, table[i]->len, out_bin_file);
 
 		/*
 		 * Now that we wrote out the compressed symbol name, restore the
-		 * original name and print it in the comment.
+		 * original name for the comments below.
 		 */
 		expand_symbol(table[i]->sym, table[i]->len, buf);
 		strcpy((char *)table[i]->sym, buf);
-		printf("\t/* %s */\n", table[i]->sym);
 	}
+	write_incbin(out_bin_name, bin_start, bin_pos(out_bin_file));
 	printf(".size kallsyms_names, . - kallsyms_names\n");
 	printf("\n");
 
@@ -410,13 +469,15 @@ static void write_src(void)
 	free(markers);
 
 	output_label("kallsyms_token_table");
+	bin_start = bin_pos(out_bin_file);
 	off = 0;
 	for (i = 0; i < 256; i++) {
 		best_idx[i] = off;
 		expand_symbol(best_table[i], best_table_len[i], buf);
-		printf("\t.asciz\t\"%s\"\n", buf);
+		fwrite(buf, 1, strlen(buf) + 1, out_bin_file);
 		off += strlen(buf) + 1;
 	}
+	write_incbin(out_bin_name, bin_start, bin_pos(out_bin_file));
 	printf(".size kallsyms_token_table, . - kallsyms_token_table\n");
 	printf("\n");
 
@@ -449,15 +510,25 @@ static void write_src(void)
 
 	sort_symbols_by_name();
 	output_label("kallsyms_seqs_of_names");
-	for (i = 0; i < table_cnt; i++)
-		printf("\t.byte 0x%02x, 0x%02x, 0x%02x\t/* %s */\n",
-			(unsigned char)(table[i]->seq >> 16),
-			(unsigned char)(table[i]->seq >> 8),
-			(unsigned char)(table[i]->seq >> 0),
-		       table[i]->sym);
+	bin_start = bin_pos(out_bin_file);
+	for (i = 0; i < table_cnt; i++) {
+		fputc(table[i]->seq >> 16, out_bin_file);
+		fputc(table[i]->seq >> 8, out_bin_file);
+		fputc(table[i]->seq >> 0, out_bin_file);
+	}
+	write_incbin(out_bin_name, bin_start, bin_pos(out_bin_file));
 	printf("\n");
 }
 
+static unsigned int token_index(unsigned char first, unsigned char second)
+{
+	return first + (second << 8);
+}
+
+static unsigned int sym_token_index(const unsigned char *symbol, int first_idx)
+{
+	return token_index(symbol[first_idx], symbol[first_idx + 1]);
+}
 
 /* table lookup compression functions */
 
@@ -467,7 +538,7 @@ static void learn_symbol(const unsigned char *symbol, int len)
 	int i;
 
 	for (i = 0; i < len - 1; i++)
-		token_profit[ symbol[i] + (symbol[i + 1] << 8) ]++;
+		token_profit[sym_token_index(symbol, i)]++;
 }
 
 /* decrease the count for all the possible tokens in a symbol */
@@ -476,16 +547,76 @@ static void forget_symbol(const unsigned char *symbol, int len)
 	int i;
 
 	for (i = 0; i < len - 1; i++)
-		token_profit[ symbol[i] + (symbol[i + 1] << 8) ]--;
+		token_profit[sym_token_index(symbol, i)]--;
+}
+
+static void token_add_symbol(unsigned int token_idx, unsigned int sym_idx)
+{
+	struct sym_arr *arr = &token_syms[token_idx];
+
+	/* Symbol indexes kept in sorted order, check for duplicate. */
+	if (sym_arr_last(arr) == sym_idx)
+		return;
+
+	sym_arr_add(arr, sym_idx);
+}
+
+static void symbol_index_all_tokens(const unsigned char *symbol, int len,
+				    unsigned int sym_idx)
+{
+	int i;
+
+	for (i = 0; i < len - 1; i++) {
+		const unsigned int token_idx = sym_token_index(symbol, i);
+
+		token_add_symbol(token_idx, sym_idx);
+	}
+}
+
+/*
+ * The symbol just got compressed. The only parts of the symbol that changed
+ * meaningfully are those containing the newly assigned compressed char, so
+ * index those.
+ */
+static void symbol_index_new_tokens(const unsigned char *symbol, int len,
+				    unsigned int sym_idx, int compressed_chr)
+{
+	int i;
+
+	for (i = 0; i < len - 1; i++) {
+		const unsigned int token_idx = sym_token_index(symbol, i);
+
+		if (symbol[i] == compressed_chr ||
+		    symbol[i + 1] == compressed_chr)
+			token_add_symbol(token_idx, sym_idx);
+	}
 }
 
-/* do the initial token count */
 static void build_initial_token_table(void)
 {
 	unsigned int i;
 
 	for (i = 0; i < table_cnt; i++)
 		learn_symbol(table[i]->sym, table[i]->len);
+
+	/*
+	 * The initial occurrence counts tell us exactly how much memory should
+	 * be reserved for each token's symbol array.
+	 */
+	for (i = 0; i < ARRAY_SIZE(token_syms); i++) {
+		const int nr_syms = token_profit[i];
+
+		if (!nr_syms)
+			continue;
+
+		token_syms[i].cap = nr_syms;
+		token_syms[i].sym_indexes =
+			xmalloc(nr_syms * sizeof(unsigned int));
+	}
+
+	/* For every symbol, index every token -> symbol it is present in. */
+	for (i = 0; i < table_cnt; i++)
+		symbol_index_all_tokens(table[i]->sym, table[i]->len, i);
 }
 
 static unsigned char *find_token(unsigned char *str, int len,
@@ -502,27 +633,31 @@ static unsigned char *find_token(unsigned char *str, int len,
 
 /* replace a given token in all the valid symbols. Use the sampled symbols
  * to update the counts */
-static void compress_symbols(const unsigned char *str, int idx)
+static void compress_symbols(const unsigned char *str, int compressed_chr)
 {
-	unsigned int i, len, size;
+	const unsigned int token_idx = sym_token_index(str, 0);
+	struct sym_arr *arr = &token_syms[token_idx];
+	unsigned int sym_idx, j, len, size;
 	unsigned char *p1, *p2;
 
-	for (i = 0; i < table_cnt; i++) {
+	/* Iterate through all symbols this token is found in and compress. */
+	for (j = 0; j < arr->cnt; j++) {
+		sym_idx = arr->sym_indexes[j];
 
-		len = table[i]->len;
-		p1 = table[i]->sym;
+		len = table[sym_idx]->len;
+		p1 = table[sym_idx]->sym;
 
 		/* find the token on the symbol */
 		p2 = find_token(p1, len, str);
 		if (!p2) continue;
 
 		/* decrease the counts for this symbol's tokens */
-		forget_symbol(table[i]->sym, len);
+		forget_symbol(table[sym_idx]->sym, len);
 
 		size = len;
 
 		do {
-			*p2 = idx;
+			*p2 = compressed_chr;
 			p2++;
 			size -= (p2 - p1);
 			memmove(p2, p2 + 1, size);
@@ -536,11 +671,15 @@ static void compress_symbols(const unsigned char *str, int idx)
 
 		} while (p2);
 
-		table[i]->len = len;
+		table[sym_idx]->len = len;
 
 		/* increase the counts for this symbol's new tokens */
-		learn_symbol(table[i]->sym, len);
+		learn_symbol(table[sym_idx]->sym, len);
+		symbol_index_new_tokens(table[sym_idx]->sym, len, sym_idx,
+					compressed_chr);
 	}
+
+	sym_arr_free(arr); /* No symbol contains this token any more. */
 }
 
 /* search the token with the maximum profit */
@@ -551,7 +690,7 @@ static int find_best_token(void)
 	bestprofit=-10000;
 	best = 0;
 
-	for (i = 0; i < 0x10000; i++) {
+	for (i = 0; i < ARRAY_SIZE(token_profit); i++) {
 		if (token_profit[i] > bestprofit) {
 			best = i;
 			bestprofit = token_profit[i];
@@ -615,7 +754,7 @@ static void optimize_token_table(void)
 /* guess for "linker script provide" symbol */
 static int may_be_linker_script_provide_symbol(const struct sym_entry *se)
 {
-	const char *symbol = sym_name(se);
+	const char *symbol = sym_entry_name(se);
 	int len = se->len - 1;
 
 	if (len < 8)
@@ -672,8 +811,8 @@ static int compare_symbols(const void *a, const void *b)
 		return wa - wb;
 
 	/* sort by the number of prefix underscores */
-	wa = strspn(sym_name(sa), "_");
-	wb = strspn(sym_name(sb), "_");
+	wa = strspn(sym_entry_name(sa), "_");
+	wb = strspn(sym_entry_name(sb), "_");
 	if (wa != wb)
 		return wa - wb;
 
@@ -688,10 +827,14 @@ static void sort_symbols(void)
 
 int main(int argc, char **argv)
 {
+	const char *in, *sysmap = NULL, *out_bin_name;
+	FILE *sysmap_out = NULL, *out_bin_file;
+
 	while (1) {
 		static const struct option long_options[] = {
 			{"all-symbols",     no_argument, &all_symbols,     1},
 			{"pc-relative",     no_argument, &pc_relative,     1},
+			{"sysmap",    required_argument, NULL,           's'},
 			{},
 		};
 
@@ -699,18 +842,55 @@ int main(int argc, char **argv)
 
 		if (c == -1)
 			break;
-		if (c != 0)
+		if (c == 's')
+			sysmap = optarg;
+		else if (c != 0)
 			usage();
 	}
 
-	if (optind >= argc)
+	if (optind + 2 != argc && !(sysmap && optind + 1 == argc))
 		usage();
 
-	read_map(argv[optind]);
+	in = argv[optind];
+	if (sysmap) {
+		sysmap_out = fopen(sysmap, "w");
+		if (!sysmap_out) {
+			perror(sysmap);
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	if (optind + 1 == argc) {
+		read_elf(in, sysmap_out);
+		if (ferror(sysmap_out) || fclose(sysmap_out)) {
+			perror(sysmap);
+			exit(EXIT_FAILURE);
+		}
+		return 0;
+	}
+
+	out_bin_name = argv[optind + 1];
+	out_bin_file = fopen(out_bin_name, "w");
+	if (!out_bin_file) {
+		perror(out_bin_name);
+		exit(EXIT_FAILURE);
+	}
+
+	read_elf(in, sysmap_out);
+	if (sysmap_out && (ferror(sysmap_out) || fclose(sysmap_out))) {
+		perror(sysmap);
+		exit(EXIT_FAILURE);
+	}
 	shrink_table();
 	sort_symbols();
 	optimize_token_table();
-	write_src();
+	write_src(out_bin_file, out_bin_name);
+
+	/* Write errors are latched in the stream, so one check at the end covers them all. */
+	if (ferror(out_bin_file) || fclose(out_bin_file)) {
+		perror(out_bin_name);
+		exit(EXIT_FAILURE);
+	}
 
 	return 0;
 }
diff --git a/scripts/kallsyms.h b/scripts/kallsyms.h
new file mode 100644
index 00000000000000..4b2801ba628593
--- /dev/null
+++ b/scripts/kallsyms.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef KALLSYMS_H
+#define KALLSYMS_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <array_size.h>
+#include "elf-parse.h"
+
+static inline bool string_starts_with(const char *s, const char *prefix)
+{
+	return strncmp(s, prefix, strlen(prefix)) == 0;
+}
+
+static inline bool string_ends_with(const char *s, const char *suffix)
+{
+	size_t len = strlen(s), suffix_len = strlen(suffix);
+
+	return len >= suffix_len && strcmp(s + len - suffix_len, suffix) == 0;
+}
+
+/* A symbol as nm lists it. */
+struct sysmap_symbol {
+	unsigned long long addr;
+	const char *name;
+	char type;
+};
+
+/* The symbols of an ELF file that System.map lists, in its order. */
+struct sysmap {
+	struct sysmap_symbol *syms;
+	size_t nr_syms;
+	int addr_width;		/* hex digits of an address */
+	struct elf_file elf;	/* the mapping the names point into */
+};
+
+struct sysmap *sysmap_read(const char *path);
+void sysmap_write(const struct sysmap *map, FILE *out);
+void sysmap_free(struct sysmap *map);
+
+#endif /* KALLSYMS_H */
diff --git a/scripts/kconfig/conf.c b/scripts/kconfig/conf.c
index fe8ba09b003910..ca6d9d735922fb 100644
--- a/scripts/kconfig/conf.c
+++ b/scripts/kconfig/conf.c
@@ -186,12 +186,23 @@ static void conf_set_all_new_symbols(enum conf_def_mode mode)
 
 	if (mode == def_random) {
 		int n, p[3];
+		bool warned = false;
 		char *env = getenv("KCONFIG_PROBABILITY");
 
 		n = 0;
 		while (env && *env) {
 			char *endp;
-			int tmp = strtol(env, &endp, 10);
+			long tmp = strtol(env, &endp, 10);
+
+			if (!isdigit((unsigned char)*env) ||
+			    (*endp && *endp != ':') ||
+			    (*endp == ':' && (!endp[1] || n == 2))) {
+				if (!warned) {
+					fprintf(stderr,
+						"warning: KCONFIG_PROBABILITY has malformed format\n");
+					warned = true;
+				}
+			}
 
 			if (tmp >= 0 && tmp <= 100) {
 				p[n++] = tmp;
@@ -227,6 +238,9 @@ static void conf_set_all_new_symbols(enum conf_def_mode mode)
 			perror("KCONFIG_PROBABILITY");
 			exit(1);
 		}
+
+		if (warned && getenv("KCONFIG_WERROR"))
+			exit(1);
 	}
 
 	menu_for_each_entry(menu) {
diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c
index 4234a51d16fd18..2227d89d6328f8 100644
--- a/scripts/kconfig/confdata.c
+++ b/scripts/kconfig/confdata.c
@@ -354,6 +354,12 @@ static int conf_set_sym_val(struct symbol *sym, int def, int def_flags, char *p)
 	case S_INT:
 	case S_HEX:
 		if (sym_string_valid(sym, p)) {
+			if (def != S_DEF_AUTO &&
+			    !sym_string_check_bounds(sym, p))
+				/* hex uses 64-bit unsigned integer */
+				conf_warning("value '%s' for %s is outside the 64-bit %s integer bounds",
+					     p, sym->name,
+					     sym->type == S_INT ? "signed" : "unsigned");
 			sym->def[def].val = xstrdup(p);
 			sym->flags |= def_flags;
 		} else {
diff --git a/scripts/kconfig/kconfig-sym-check.pl b/scripts/kconfig/kconfig-sym-check.pl
index daa5285fdefc85..c8dd07f8b27c58 100755
--- a/scripts/kconfig/kconfig-sym-check.pl
+++ b/scripts/kconfig/kconfig-sym-check.pl
@@ -90,7 +90,7 @@ sub indent_depth {
 			next;
 		}
 
-		if (/^\s*(default|def_bool|def_tristate|select|depends\s+on|imply|visible\s+if|range|if|bool|tristate|int|hex|string|prompt)\s+(.+)\s*$/) {
+		if (/^\s*(default|def_bool|def_tristate|def_string|def_int|def_hex|select|depends\s+on|imply|visible\s+if|range|if|bool|tristate|int|hex|string|prompt)\s+(.+)\s*$/) {
 			my $s = $2;
 			$s =~ s/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'//g;
 			$s =~ s/#.*//;
diff --git a/scripts/kconfig/lexer.l b/scripts/kconfig/lexer.l
index a6155422b4a688..1fa521199d4a30 100644
--- a/scripts/kconfig/lexer.l
+++ b/scripts/kconfig/lexer.l
@@ -105,6 +105,9 @@ n	[A-Za-z0-9_-]
 "comment"		return T_COMMENT;
 "config"		return T_CONFIG;
 "def_bool"		return T_DEF_BOOL;
+"def_hex"		return T_DEF_HEX;
+"def_int"		return T_DEF_INT;
+"def_string"		return T_DEF_STRING;
 "def_tristate"		return T_DEF_TRISTATE;
 "default"		return T_DEFAULT;
 "depends"		return T_DEPENDS;
diff --git a/scripts/kconfig/lkc.h b/scripts/kconfig/lkc.h
index 7e6f6ca299cf1a..bbc99f75b416da 100644
--- a/scripts/kconfig/lkc.h
+++ b/scripts/kconfig/lkc.h
@@ -89,7 +89,7 @@ struct property *menu_add_prompt(enum prop_type type, const char *prompt,
 				 struct expr *dep);
 void menu_add_expr(enum prop_type type, struct expr *expr, struct expr *dep);
 void menu_add_symbol(enum prop_type type, struct symbol *sym, struct expr *dep);
-void menu_finalize(void);
+int menu_finalize(void);
 void menu_set_type(int type);
 
 extern struct menu rootmenu;
diff --git a/scripts/kconfig/lkc_proto.h b/scripts/kconfig/lkc_proto.h
index 8914b4e8f2a811..8b436c87ba4ae3 100644
--- a/scripts/kconfig/lkc_proto.h
+++ b/scripts/kconfig/lkc_proto.h
@@ -31,6 +31,7 @@ bool sym_set_tristate_value(struct symbol *sym,tristate tri);
 void choice_set_value(struct menu *choice, struct symbol *sym);
 tristate sym_toggle_tristate_value(struct symbol *sym);
 bool sym_string_valid(struct symbol *sym, const char *newval);
+bool sym_string_check_bounds(struct symbol *sym, const char *str);
 bool sym_string_within_range(struct symbol *sym, const char *str);
 bool sym_set_string_value(struct symbol *sym, const char *newval);
 bool sym_is_changeable(const struct symbol *sym);
diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c
index 9c079e92a9ed62..bda3e26502161f 100644
--- a/scripts/kconfig/menu.c
+++ b/scripts/kconfig/menu.c
@@ -234,17 +234,42 @@ void menu_add_symbol(enum prop_type type, struct symbol *sym, struct expr *dep)
 	menu_add_prop(type, expr_alloc_symbol(sym), dep);
 }
 
-static int menu_validate_number(struct symbol *sym, struct symbol *sym2)
-{
-	return sym2->type == S_INT || sym2->type == S_HEX ||
-	       (sym2->type == S_UNKNOWN && sym_string_valid(sym, sym2->name));
+/* Validate the sym2 value for numeric sym. */
+static int menu_validate_number(struct symbol *sym, struct symbol *sym2,
+				const struct property *prop)
+{
+	if (sym->type != S_INT && sym->type != S_HEX)
+		return 0;
+
+	if (sym2->type == sym->type)
+		return 0;
+
+	if (sym2->type != S_UNKNOWN ||
+		!sym_string_valid(sym, sym2->name)) {
+		fprintf(stderr, "%s:%d: error: '%s' is an invalid value for '%s'\n",
+			prop->filename, prop->lineno, sym2->name,
+			sym_type_name(sym->type));
+		return 1;
+	}
+
+	if (!sym_string_check_bounds(sym, sym2->name)) {
+		fprintf(stderr,
+			"%s:%d: error: %s constant '%s' is outside the 64-bit %s bounds\n",
+			prop->filename, prop->lineno, sym_type_name(sym->type),
+			sym2->name, sym->type == S_INT ? "signed" : "unsigned");
+
+		return 1;
+	}
+
+	return 0;
 }
 
-static void sym_check_prop(struct symbol *sym)
+static int sym_check_prop(struct symbol *sym)
 {
 	struct property *prop;
 	struct symbol *sym2;
 	char *use;
+	int errors = 0;
 
 	for (prop = sym->prop; prop; prop = prop->next) {
 		switch (prop->type) {
@@ -258,10 +283,7 @@ static void sym_check_prop(struct symbol *sym)
 				break;
 			sym2 = prop_get_symbol(prop);
 			if (sym->type == S_HEX || sym->type == S_INT) {
-				if (!menu_validate_number(sym, sym2))
-					prop_warn(prop,
-					    "'%s': number is invalid",
-					    sym->name);
+				errors += menu_validate_number(sym, sym2, prop);
 			}
 			if (sym_is_choice(sym)) {
 				struct menu *choice = sym_get_choice_menu(sym2);
@@ -287,27 +309,33 @@ static void sym_check_prop(struct symbol *sym)
 				    "'%s' has wrong type. '%s' only "
 				    "accept arguments of bool and "
 				    "tristate type", sym2->name, use);
+			if (sym_is_choice_value(sym2))
+				prop_warn(prop,
+					  "config symbol '%s' uses %s for '%s', but '%s' is a choice value",
+					  sym->name, use, sym2->name, sym2->name);
 			break;
 		case P_RANGE:
 			if (sym->type != S_INT && sym->type != S_HEX)
 				prop_warn(prop, "range is only allowed "
 						"for int or hex symbols");
-			if (!menu_validate_number(sym, prop->expr->left.sym) ||
-			    !menu_validate_number(sym, prop->expr->right.sym))
-				prop_warn(prop, "range is invalid");
+			errors += menu_validate_number(sym, prop->expr->left.sym, prop);
+			errors += menu_validate_number(sym, prop->expr->right.sym, prop);
 			break;
 		default:
 			;
 		}
 	}
+
+	return errors;
 }
 
-static void _menu_finalize(struct menu *parent, bool inside_choice)
+static int _menu_finalize(struct menu *parent, bool inside_choice)
 {
 	struct menu *menu, *last_menu;
 	struct symbol *sym;
 	struct property *prop;
 	struct expr *basedep, *dep, *dep2;
+	int errors = 0;
 
 	sym = parent->sym;
 	if (parent->list) {
@@ -393,7 +421,7 @@ static void _menu_finalize(struct menu *parent, bool inside_choice)
 		 * moving on
 		 */
 		for (menu = parent->list; menu; menu = menu->next)
-			_menu_finalize(menu, sym && sym_is_choice(sym));
+			errors += _menu_finalize(menu, sym && sym_is_choice(sym));
 	} else if (!inside_choice && sym) {
 		/*
 		 * Automatic submenu creation. If sym is a symbol and A, B, C,
@@ -461,7 +489,7 @@ static void _menu_finalize(struct menu *parent, bool inside_choice)
 			}
 			/* Superset, put in submenu */
 		next:
-			_menu_finalize(menu, false);
+			errors += _menu_finalize(menu, false);
 			menu->parent = parent;
 			last_menu = menu;
 		}
@@ -519,14 +547,16 @@ static void _menu_finalize(struct menu *parent, bool inside_choice)
 			menu_warn(parent, "config symbol defined without type");
 
 		/* Check properties connected to this symbol */
-		sym_check_prop(sym);
+		errors += sym_check_prop(sym);
 		sym->flags |= SYMBOL_WARNED;
 	}
+
+	return errors;
 }
 
-void menu_finalize(void)
+int menu_finalize(void)
 {
-	_menu_finalize(&rootmenu, false);
+	return _menu_finalize(&rootmenu, false);
 }
 
 bool menu_has_prompt(const struct menu *menu)
diff --git a/scripts/kconfig/parser.y b/scripts/kconfig/parser.y
index 5fb6f07b6ad292..ecb18c213d83b0 100644
--- a/scripts/kconfig/parser.y
+++ b/scripts/kconfig/parser.y
@@ -53,6 +53,9 @@ struct menu *current_menu, *current_entry, *current_choice;
 %token T_CONFIG
 %token T_DEFAULT
 %token T_DEF_BOOL
+%token T_DEF_HEX
+%token T_DEF_INT
+%token T_DEF_STRING
 %token T_DEF_TRISTATE
 %token T_DEPENDS
 %token T_ENDCHOICE
@@ -309,6 +312,9 @@ type:
 default:
 	  T_DEFAULT		{ $$ = S_UNKNOWN; }
 	| T_DEF_BOOL		{ $$ = S_BOOLEAN; }
+	| T_DEF_HEX		{ $$ = S_HEX; }
+	| T_DEF_INT		{ $$ = S_INT; }
+	| T_DEF_STRING		{ $$ = S_STRING; }
 	| T_DEF_TRISTATE	{ $$ = S_TRISTATE; }
 
 /* if entry */
@@ -587,7 +593,7 @@ void conf_parse(const char *name)
 		menu_add_prompt(P_MENU, "Main menu", NULL);
 	}
 
-	menu_finalize();
+	yynerrs += menu_finalize();
 
 	menu_for_each_entry(menu) {
 		struct menu *child;
diff --git a/scripts/kconfig/symbol.c b/scripts/kconfig/symbol.c
index 7e81b3676ee944..dcb4b45e674b80 100644
--- a/scripts/kconfig/symbol.c
+++ b/scripts/kconfig/symbol.c
@@ -5,6 +5,7 @@
 
 #include <sys/types.h>
 #include <ctype.h>
+#include <errno.h>
 #include <stdlib.h>
 #include <string.h>
 #include <regex.h>
@@ -141,8 +142,15 @@ struct property *sym_get_range_prop(struct symbol *sym)
 	return NULL;
 }
 
-static long long sym_get_range_val(struct symbol *sym, int base)
+union range_value {
+	long long s;
+	unsigned long long u;
+};
+
+static union range_value sym_get_range_val(struct symbol *sym, int base)
 {
+	union range_value val;
+
 	sym_calc_value(sym);
 	switch (sym->type) {
 	case S_INT:
@@ -154,15 +162,25 @@ static long long sym_get_range_val(struct symbol *sym, int base)
 	default:
 		break;
 	}
-	return strtoll(sym->curr.val, NULL, base);
+	if (base == 10)
+		val.s = strtoll(sym->curr.val, NULL, base);
+	else
+		/* HEX */
+		val.u = strtoull(sym->curr.val, NULL, base);
+	return val;
 }
 
-static void sym_validate_range(struct symbol *sym)
+/*
+ * Return the nearest range bound for an out-of-range default value.
+ * Return NULL if the value is valid or the symbol has no active range.
+ */
+static struct symbol *sym_get_near_range_bound(struct symbol *sym,
+					  const char *value)
 {
 	struct property *prop;
 	struct symbol *range_sym;
 	int base;
-	long long val, val2;
+	union range_value val, val2;
 
 	switch (sym->type) {
 	case S_INT:
@@ -172,21 +190,37 @@ static void sym_validate_range(struct symbol *sym)
 		base = 16;
 		break;
 	default:
-		return;
+		return NULL;
 	}
 	prop = sym_get_range_prop(sym);
 	if (!prop)
-		return;
-	val = strtoll(sym->curr.val, NULL, base);
+		return NULL;
+	val = sym_get_range_val(sym, base);
 	range_sym = prop->expr->left.sym;
 	val2 = sym_get_range_val(range_sym, base);
-	if (val >= val2) {
+
+	if (base == 10 && val.s >= val2.s) {
+		range_sym = prop->expr->right.sym;
+		val2 = sym_get_range_val(range_sym, base);
+		if (val.s <= val2.s)
+			return NULL;
+	} else if (base == 16 && val.u >= val2.u) {
 		range_sym = prop->expr->right.sym;
 		val2 = sym_get_range_val(range_sym, base);
-		if (val <= val2)
-			return;
+		if (val.u <= val2.u)
+			return NULL;
 	}
-	sym->curr.val = range_sym->curr.val;
+
+	return range_sym;
+}
+
+static void sym_validate_range(struct symbol *sym)
+{
+	struct symbol *range_sym;
+
+	range_sym = sym_get_near_range_bound(sym, sym->curr.val);
+	if (range_sym)
+		sym->curr.val = range_sym->curr.val;
 }
 
 static void sym_set_changed(struct symbol *sym)
@@ -711,10 +745,26 @@ bool sym_string_valid(struct symbol *sym, const char *str)
 	}
 }
 
+bool sym_string_check_bounds(struct symbol *sym, const char *str)
+{
+	errno = 0;
+
+	if (sym->type == S_INT)
+		strtoll(str, NULL, 10);
+	else if (sym->type == S_HEX)
+		strtoull(str, NULL, 16);
+	else
+		/* string */
+		return true;
+
+	return errno != ERANGE;
+}
+
 bool sym_string_within_range(struct symbol *sym, const char *str)
 {
 	struct property *prop;
 	long long val;
+	unsigned long long uval;
 
 	switch (sym->type) {
 	case S_STRING:
@@ -722,21 +772,25 @@ bool sym_string_within_range(struct symbol *sym, const char *str)
 	case S_INT:
 		if (!sym_string_valid(sym, str))
 			return false;
+		if (!sym_string_check_bounds(sym, str))
+			return false;
 		prop = sym_get_range_prop(sym);
 		if (!prop)
 			return true;
 		val = strtoll(str, NULL, 10);
-		return val >= sym_get_range_val(prop->expr->left.sym, 10) &&
-		       val <= sym_get_range_val(prop->expr->right.sym, 10);
+		return val >= sym_get_range_val(prop->expr->left.sym, 10).s &&
+		       val <= sym_get_range_val(prop->expr->right.sym, 10).s;
 	case S_HEX:
 		if (!sym_string_valid(sym, str))
 			return false;
+		if (!sym_string_check_bounds(sym, str))
+			return false;
 		prop = sym_get_range_prop(sym);
 		if (!prop)
 			return true;
-		val = strtoll(str, NULL, 16);
-		return val >= sym_get_range_val(prop->expr->left.sym, 16) &&
-		       val <= sym_get_range_val(prop->expr->right.sym, 16);
+		uval = strtoull(str, NULL, 16);
+		return uval >= sym_get_range_val(prop->expr->left.sym, 16).u &&
+		       uval <= sym_get_range_val(prop->expr->right.sym, 16).u;
 	case S_BOOLEAN:
 	case S_TRISTATE:
 		switch (str[0]) {
@@ -812,7 +866,7 @@ bool sym_set_string_value(struct symbol *sym, const char *newval)
 const char *sym_get_string_default(struct symbol *sym)
 {
 	struct property *prop;
-	struct symbol *ds;
+	struct symbol *ds, *range_sym;
 	const char *str = "";
 	tristate val;
 
@@ -830,11 +884,6 @@ const char *sym_get_string_default(struct symbol *sym)
 			val = EXPR_AND(expr_calc_value(prop->expr), prop->visible.tri);
 			break;
 		default:
-			/*
-			 * The following fails to handle the situation
-			 * where a default value is further limited by
-			 * the valid range.
-			 */
 			ds = prop_get_symbol(prop);
 			if (ds != NULL) {
 				sym_calc_value(ds);
@@ -878,6 +927,11 @@ const char *sym_get_string_default(struct symbol *sym)
 	default:
 		break;
 	}
+
+	range_sym = sym_get_near_range_bound(sym, str);
+	if (range_sym)
+		str = range_sym->curr.val;
+
 	return str;
 }
 
@@ -1196,6 +1250,7 @@ static struct symbol *sym_check_expr_deps(const struct expr *e)
 	case E_GTH:
 	case E_LEQ:
 	case E_LTH:
+	case E_RANGE:
 	case E_UNEQUAL:
 		sym = sym_check_deps(e->left.sym);
 		if (sym)
@@ -1243,7 +1298,9 @@ static struct symbol *sym_check_sym_deps(struct symbol *sym)
 		sym2 = sym_check_expr_deps(prop->visible.expr);
 		if (sym2)
 			break;
-		if (prop->type != P_DEFAULT || sym_is_choice(sym))
+		if (sym_is_choice(sym))
+			continue;
+		if (prop->type != P_DEFAULT && prop->type != P_RANGE)
 			continue;
 		stack.expr = &prop->expr;
 		sym2 = sym_check_expr_deps(prop->expr);
diff --git a/scripts/kconfig/tests/conftest.py b/scripts/kconfig/tests/conftest.py
index 66f95e4ed58c19..d22c080a4e4065 100644
--- a/scripts/kconfig/tests/conftest.py
+++ b/scripts/kconfig/tests/conftest.py
@@ -57,9 +57,12 @@ def _run_conf(self, mode, dot_config=None, out_file='.config',
         # Override 'srctree' environment to make the test as the top directory
         extra_env['srctree'] = self._test_dir
 
-        # Clear KCONFIG_DEFCONFIG_LIST to keep unit tests from being affected
-        # by the user's environment.
-        extra_env['KCONFIG_DEFCONFIG_LIST'] = ''
+        # Clear all KCONFIG_* environment variables
+        env = {
+            varname: os.environ[varname]
+            for varname in os.environ.keys()
+            if not varname.startswith("KCONFIG_")
+        }
 
         # Run Kconfig in a temporary directory.
         # This directory is automatically removed when done.
@@ -75,7 +78,7 @@ def _run_conf(self, mode, dot_config=None, out_file='.config',
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   cwd=temp_dir,
-                                  env=dict(os.environ, **extra_env))
+                                  env=dict(env, **extra_env))
 
             # If input key sequence is given, feed it to stdin.
             if in_keys:
@@ -156,13 +159,13 @@ def oldconfig(self, dot_config=None, in_keys=None):
         return self._run_conf('--oldconfig', dot_config=dot_config,
                               interactive=True, in_keys=in_keys)
 
-    def olddefconfig(self, dot_config=None):
+    def olddefconfig(self, dot_config=None, **kw):
         """Run olddefconfig.
 
         dot_config: .config file to use for configuration base (optional)
         returncode: exit status of the Kconfig executable
         """
-        return self._run_conf('--olddefconfig', dot_config=dot_config)
+        return self._run_conf('--olddefconfig', dot_config=dot_config, **kw)
 
     def defconfig(self, defconfig):
         """Run defconfig.
@@ -226,13 +229,15 @@ def randconfig(self, all_config=None, seed=None):
 
         return self._allconfig('rand', all_config, extra_env=extra_env)
 
-    def savedefconfig(self, dot_config):
+    def savedefconfig(self, dot_config, out_file='defconfig', **kw):
         """Run savedefconfig.
 
         dot_config: .config file for input
+        out_file: defconfig file for output
         returncode: exit status of the Kconfig executable
         """
-        return self._run_conf('--savedefconfig', out_file='defconfig')
+        return self._run_conf('--savedefconfig={}'.format(out_file),
+                              dot_config=dot_config, out_file=out_file, **kw)
 
     def listnewconfig(self, dot_config=None):
         """Run listnewconfig.
@@ -312,6 +317,12 @@ def stderr_matches(self, expected):
         return self._matches('stderr', expected)
 
 
+@pytest.fixture(autouse=True)
+def clear_werror(monkeypatch):
+    # extra_env can set strict mode, but cannot remove an inherited flag.
+    monkeypatch.delenv('KCONFIG_WERROR', raising=False)
+
+
 @pytest.fixture(scope="module")
 def conf(request):
     """Create a Conf instance and provide it to test functions."""
diff --git a/scripts/kconfig/tests/def_type/Kconfig b/scripts/kconfig/tests/def_type/Kconfig
new file mode 100644
index 00000000000000..fbee37a631790f
--- /dev/null
+++ b/scripts/kconfig/tests/def_type/Kconfig
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0
+# The def_<type> shorthands: a type definition plus a default value.
+
+config MODULES
+	bool "Enable loadable module support"
+	modules
+	default y
+
+config GUARD
+	bool "Guard symbol"
+
+config DEF_BOOL
+	def_bool GUARD
+
+config DEF_TRISTATE
+	def_tristate m if GUARD
+
+config DEF_STRING
+	def_string "guarded" if GUARD
+	default "fallback"
+
+config DEF_INT
+	def_int 64 if GUARD
+	default 32
+
+config DEF_HEX
+	def_hex 0xdead if GUARD
+	default 0x0
diff --git a/scripts/kconfig/tests/def_type/__init__.py b/scripts/kconfig/tests/def_type/__init__.py
new file mode 100644
index 00000000000000..1ebaf5da07c821
--- /dev/null
+++ b/scripts/kconfig/tests/def_type/__init__.py
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0
+"""
+Set a symbol's type and its default value in one line.
+
+"def_bool", "def_tristate", "def_string", "def_int" and "def_hex" are
+shorthand for a type definition plus a "default" property.  Check that
+each one sets the type, and that an "if" on the shorthand does not
+disturb the usual default cascade: the shorthand is only the first arm
+of the list, so a later "default" still applies when its condition is
+not met.
+"""
+
+def test(conf):
+    assert conf.olddefconfig(dot_config='guard_y.config') == 0
+    assert conf.config_matches('expected_guard_y')
+
+    assert conf.olddefconfig(dot_config='guard_n.config') == 0
+    assert conf.config_matches('expected_guard_n')
diff --git a/scripts/kconfig/tests/def_type/expected_guard_n b/scripts/kconfig/tests/def_type/expected_guard_n
new file mode 100644
index 00000000000000..14719720a8b931
--- /dev/null
+++ b/scripts/kconfig/tests/def_type/expected_guard_n
@@ -0,0 +1,9 @@
+#
+# Automatically generated file; DO NOT EDIT.
+# Main menu
+#
+CONFIG_MODULES=y
+# CONFIG_GUARD is not set
+CONFIG_DEF_STRING="fallback"
+CONFIG_DEF_INT=32
+CONFIG_DEF_HEX=0x0
diff --git a/scripts/kconfig/tests/def_type/expected_guard_y b/scripts/kconfig/tests/def_type/expected_guard_y
new file mode 100644
index 00000000000000..b2844072d0b8f4
--- /dev/null
+++ b/scripts/kconfig/tests/def_type/expected_guard_y
@@ -0,0 +1,11 @@
+#
+# Automatically generated file; DO NOT EDIT.
+# Main menu
+#
+CONFIG_MODULES=y
+CONFIG_GUARD=y
+CONFIG_DEF_BOOL=y
+CONFIG_DEF_TRISTATE=m
+CONFIG_DEF_STRING="guarded"
+CONFIG_DEF_INT=64
+CONFIG_DEF_HEX=0xdead
diff --git a/scripts/kconfig/tests/def_type/guard_n.config b/scripts/kconfig/tests/def_type/guard_n.config
new file mode 100644
index 00000000000000..ed9ad6c1a2d4ec
--- /dev/null
+++ b/scripts/kconfig/tests/def_type/guard_n.config
@@ -0,0 +1 @@
+# CONFIG_GUARD is not set
diff --git a/scripts/kconfig/tests/def_type/guard_y.config b/scripts/kconfig/tests/def_type/guard_y.config
new file mode 100644
index 00000000000000..afe35b0845425f
--- /dev/null
+++ b/scripts/kconfig/tests/def_type/guard_y.config
@@ -0,0 +1 @@
+CONFIG_GUARD=y
diff --git a/scripts/kconfig/tests/err_num_bounds/Kconfig b/scripts/kconfig/tests/err_num_bounds/Kconfig
new file mode 100644
index 00000000000000..c439366c03b676
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_bounds/Kconfig
@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: GPL-2.0
+# Test bounds checks for 'int' and 'hex' constants
+
+config INT_SOURCE
+	int
+
+config HEX_SOURCE
+	hex
+
+config BOOL_SOURCE
+	bool
+
+# Valid values at the limits of the type
+
+config INT_MIN
+	int
+	default -9223372036854775808
+
+config INT_MAX
+	int
+	default 9223372036854775807
+
+config HEX_MIN
+	hex
+	default 0x0
+
+config HEX_MAX
+	hex
+	default 0xffffffffffffffff
+
+config INT_RANGE_LIMITS
+	int
+	range -9223372036854775808 9223372036854775807
+
+config HEX_RANGE_LIMITS
+	hex
+	range 0 0xffffffffffffffff
+
+config INT_FROM_INT
+	int
+	default INT_SOURCE
+
+config HEX_FROM_HEX
+	hex
+	default HEX_SOURCE
+
+# Constants outside the bounds
+
+config INT_DEFAULT_TOO_HIGH
+	int
+	default 10000000000000000000
+
+config INT_DEFAULT_TOO_LOW
+	int
+	default -9223372036854775809
+
+config INT_RANGE_TOO_HIGH
+	int
+	range 0 10000000000000000000
+
+config INT_RANGE_TOO_LOW
+	int
+	range -10000000000000000000 0
+
+config INT_RANGE_BOTH_OUTSIDE
+	int
+	range -9223372036854775809 10000000000000000000
+
+config HEX_DEFAULT_TOO_HIGH
+	hex
+	default 0x10000000000000000
+
+config HEX_RANGE_TOO_HIGH
+	hex
+	range 0 0x10000000000000000
+
+config HEX_RANGE_BOTH_TOO_HIGH
+	hex
+	range 0x10000000000000000 0x20000000000000000
diff --git a/scripts/kconfig/tests/err_num_bounds/__init__.py b/scripts/kconfig/tests/err_num_bounds/__init__.py
new file mode 100644
index 00000000000000..72ac6aa244918b
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_bounds/__init__.py
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0
+"""
+Detect constants outside the 'int' and 'hex' bounds.
+
+An int constant must fit in a signed 64-bit integer, and a hex constant must
+fit in an unsigned 64-bit integer.
+"""
+
+
+def test(conf):
+    assert conf.olddefconfig() == 1
+    assert conf.stderr_matches('expected_stderr')
diff --git a/scripts/kconfig/tests/err_num_bounds/expected_stderr b/scripts/kconfig/tests/err_num_bounds/expected_stderr
new file mode 100644
index 00000000000000..5f89cf4afff8ac
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_bounds/expected_stderr
@@ -0,0 +1,10 @@
+Kconfig:51: error: integer constant '10000000000000000000' is outside the 64-bit signed bounds
+Kconfig:55: error: integer constant '-9223372036854775809' is outside the 64-bit signed bounds
+Kconfig:59: error: integer constant '10000000000000000000' is outside the 64-bit signed bounds
+Kconfig:63: error: integer constant '-10000000000000000000' is outside the 64-bit signed bounds
+Kconfig:67: error: integer constant '-9223372036854775809' is outside the 64-bit signed bounds
+Kconfig:67: error: integer constant '10000000000000000000' is outside the 64-bit signed bounds
+Kconfig:71: error: hex constant '0x10000000000000000' is outside the 64-bit unsigned bounds
+Kconfig:75: error: hex constant '0x10000000000000000' is outside the 64-bit unsigned bounds
+Kconfig:79: error: hex constant '0x10000000000000000' is outside the 64-bit unsigned bounds
+Kconfig:79: error: hex constant '0x20000000000000000' is outside the 64-bit unsigned bounds
diff --git a/scripts/kconfig/tests/err_num_mismatch/Kconfig b/scripts/kconfig/tests/err_num_mismatch/Kconfig
new file mode 100644
index 00000000000000..8406f3bb6419ff
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_mismatch/Kconfig
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: GPL-2.0
+# Test 'int' and 'hex' symbols that reference each other
+
+config INT_SOURCE
+	int
+
+config HEX_SOURCE
+	hex
+
+# 'hex' reference from 'int'
+
+config INT_DEFAULT_HEX
+	int
+	default HEX_SOURCE
+
+config INT_RANGE_HEX
+	int
+	range HEX_SOURCE 1
+
+# A hex symbol must not reference an int symbol
+
+config HEX_DEFAULT_INT
+	hex
+	default INT_SOURCE
+
+config HEX_RANGE_INT
+	hex
+	range INT_SOURCE 0x1
+
+# Referencing the same type is valid
+
+config INT_FROM_INT
+	int
+	default INT_SOURCE
+
+config HEX_FROM_HEX
+	hex
+	range 0 HEX_SOURCE
diff --git a/scripts/kconfig/tests/err_num_mismatch/__init__.py b/scripts/kconfig/tests/err_num_mismatch/__init__.py
new file mode 100644
index 00000000000000..275f2a6e9a5bf2
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_mismatch/__init__.py
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+"""
+Reject direct references ('default' or 'range') between int and hex options.
+"""
+
+
+def test(conf):
+    assert conf.olddefconfig() == 1
+    assert conf.stderr_matches('expected_stderr')
diff --git a/scripts/kconfig/tests/err_num_mismatch/expected_stderr b/scripts/kconfig/tests/err_num_mismatch/expected_stderr
new file mode 100644
index 00000000000000..587c34467ae632
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_mismatch/expected_stderr
@@ -0,0 +1,4 @@
+Kconfig:14: error: 'HEX_SOURCE' is an invalid value for 'integer'
+Kconfig:18: error: 'HEX_SOURCE' is an invalid value for 'integer'
+Kconfig:24: error: 'INT_SOURCE' is an invalid value for 'hex'
+Kconfig:28: error: 'INT_SOURCE' is an invalid value for 'hex'
diff --git a/scripts/kconfig/tests/err_num_non_numeric_ref/Kconfig b/scripts/kconfig/tests/err_num_non_numeric_ref/Kconfig
new file mode 100644
index 00000000000000..0ca7a4a460f6cf
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_non_numeric_ref/Kconfig
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: GPL-2.0
+# Test non-numeric symbol references from numeric symbols
+
+config BOOL_SOURCE
+	bool
+
+config TRISTATE_SOURCE
+	tristate
+
+config STRING_SOURCE
+	string
+
+# Invalid int defaults
+
+config INT_DEFAULT_BOOL
+	int
+	default BOOL_SOURCE
+
+config INT_DEFAULT_TRISTATE
+	int
+	default TRISTATE_SOURCE
+
+config INT_DEFAULT_STRING
+	int
+	default STRING_SOURCE
+
+# Invalid hex defaults
+
+config HEX_DEFAULT_BOOL
+	hex
+	default BOOL_SOURCE
+
+config HEX_DEFAULT_TRISTATE
+	hex
+	default TRISTATE_SOURCE
+
+config HEX_DEFAULT_STRING
+	hex
+	default STRING_SOURCE
+
+# Invalid int ranges
+
+config INT_RANGE_BOOL
+	int
+	range BOOL_SOURCE 1
+
+config INT_RANGE_TRISTATE
+	int
+	range TRISTATE_SOURCE 1
+
+config INT_RANGE_STRING
+	int
+	range STRING_SOURCE 1
+
+config INT_RANGE_MULTIPLE
+	int
+	range BOOL_SOURCE TRISTATE_SOURCE
+
+# Invalid hex ranges
+
+config HEX_RANGE_BOOL
+	hex
+	range BOOL_SOURCE 0x1
+
+config HEX_RANGE_TRISTATE
+	hex
+	range TRISTATE_SOURCE 0x1
+
+config HEX_RANGE_STRING
+	hex
+	range STRING_SOURCE 0x1
+
+config HEX_RANGE_MULTIPLE
+	hex
+	range BOOL_SOURCE TRISTATE_SOURCE
diff --git a/scripts/kconfig/tests/err_num_non_numeric_ref/__init__.py b/scripts/kconfig/tests/err_num_non_numeric_ref/__init__.py
new file mode 100644
index 00000000000000..9632907abead8b
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_non_numeric_ref/__init__.py
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+"""
+Reject nonnumeric symbol references from int and hex properties.
+"""
+
+
+def test(conf):
+    assert conf.olddefconfig() == 1
+    assert conf.stderr_matches('expected_stderr')
diff --git a/scripts/kconfig/tests/err_num_non_numeric_ref/expected_stderr b/scripts/kconfig/tests/err_num_non_numeric_ref/expected_stderr
new file mode 100644
index 00000000000000..005f855ecbddf4
--- /dev/null
+++ b/scripts/kconfig/tests/err_num_non_numeric_ref/expected_stderr
@@ -0,0 +1,16 @@
+Kconfig:17: error: 'BOOL_SOURCE' is an invalid value for 'integer'
+Kconfig:21: error: 'TRISTATE_SOURCE' is an invalid value for 'integer'
+Kconfig:25: error: 'STRING_SOURCE' is an invalid value for 'integer'
+Kconfig:31: error: 'BOOL_SOURCE' is an invalid value for 'hex'
+Kconfig:35: error: 'TRISTATE_SOURCE' is an invalid value for 'hex'
+Kconfig:39: error: 'STRING_SOURCE' is an invalid value for 'hex'
+Kconfig:45: error: 'BOOL_SOURCE' is an invalid value for 'integer'
+Kconfig:49: error: 'TRISTATE_SOURCE' is an invalid value for 'integer'
+Kconfig:53: error: 'STRING_SOURCE' is an invalid value for 'integer'
+Kconfig:57: error: 'BOOL_SOURCE' is an invalid value for 'integer'
+Kconfig:57: error: 'TRISTATE_SOURCE' is an invalid value for 'integer'
+Kconfig:63: error: 'BOOL_SOURCE' is an invalid value for 'hex'
+Kconfig:67: error: 'TRISTATE_SOURCE' is an invalid value for 'hex'
+Kconfig:71: error: 'STRING_SOURCE' is an invalid value for 'hex'
+Kconfig:75: error: 'BOOL_SOURCE' is an invalid value for 'hex'
+Kconfig:75: error: 'TRISTATE_SOURCE' is an invalid value for 'hex'
diff --git a/scripts/kconfig/tests/err_recursive_dep/Kconfig b/scripts/kconfig/tests/err_recursive_dep/Kconfig
index ebdb3ffd87170a..bd1ff92bcab247 100644
--- a/scripts/kconfig/tests/err_recursive_dep/Kconfig
+++ b/scripts/kconfig/tests/err_recursive_dep/Kconfig
@@ -61,3 +61,27 @@ config G
 	bool "G"
 
 endmenu
+
+# depends on and range
+
+config H1
+	int
+	depends on H2 < H3
+
+config H2
+	int
+	range H3 H1
+
+config H3
+	int
+	range H1 H2
+
+# range
+
+config I1
+	int
+	range I2 I2
+
+config I2
+	int
+	range I1 I1
diff --git a/scripts/kconfig/tests/err_recursive_dep/expected_stderr b/scripts/kconfig/tests/err_recursive_dep/expected_stderr
index fc2e860af082cb..08f8430876782b 100644
--- a/scripts/kconfig/tests/err_recursive_dep/expected_stderr
+++ b/scripts/kconfig/tests/err_recursive_dep/expected_stderr
@@ -36,3 +36,16 @@ error: recursive dependency detected!
 	symbol G depends on G
 For a resolution refer to Documentation/kbuild/kconfig-language.rst
 subsection "Kconfig recursive dependency limitations"
+
+error: recursive dependency detected!
+	symbol H1 depends on H2
+	symbol H2 range value contains H3
+	symbol H3 range value contains H1
+For a resolution refer to Documentation/kbuild/kconfig-language.rst
+subsection "Kconfig recursive dependency limitations"
+
+error: recursive dependency detected!
+	symbol I1 range value contains I2
+	symbol I2 range value contains I1
+For a resolution refer to Documentation/kbuild/kconfig-language.rst
+subsection "Kconfig recursive dependency limitations"
diff --git a/scripts/kconfig/tests/randconfig_probability/Kconfig b/scripts/kconfig/tests/randconfig_probability/Kconfig
new file mode 100644
index 00000000000000..84f4e5fcc51e4b
--- /dev/null
+++ b/scripts/kconfig/tests/randconfig_probability/Kconfig
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+config MODULES
+	bool
+	default y
+	modules
+
+config BOOL
+	bool "Bool"
+
+config TRI
+	tristate "Tristate"
diff --git a/scripts/kconfig/tests/randconfig_probability/__init__.py b/scripts/kconfig/tests/randconfig_probability/__init__.py
new file mode 100644
index 00000000000000..e6e5dda10ad929
--- /dev/null
+++ b/scripts/kconfig/tests/randconfig_probability/__init__.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Validate KCONFIG_PROBABILITY without changing the supported distributions."""
+
+
+def test_malformed_warns(conf):
+    probabilities = [
+        'invalid', ' ', '50%', '50 ', '0x32', '10 20', '10:20x',
+        '10:20:invalid', '10:20:30x',
+        ':50', '50:', '10::20', '10:20:', '10:20:30:', '10:20:30:40',
+        '-0', '+0', '+', '-', '+100:0', ' \t100:0', '0: \t100:0',
+    ]
+    for probability in probabilities:
+        assert conf._run_conf('--randconfig', extra_env={
+            'KCONFIG_PROBABILITY': probability,
+            'KCONFIG_SEED': '0',
+        }) == 0, repr(probability)
+        assert ('warning: KCONFIG_PROBABILITY has malformed format' in
+                conf.stderr), repr(probability)
+        assert conf.stderr.count('warning:') == 1, repr(probability)
+        assert conf.config is not None, repr(probability)
+
+
+def test_malformed_preserves_config(conf):
+    probabilities = [('50%', '50:0:0'), ('-0', '0')]
+    for seed in range(20):
+        for probability, equivalent in probabilities:
+            context = 'probability={!r}, equivalent={!r}, seed={}'.format(
+                probability, equivalent, seed)
+            assert conf._run_conf('--randconfig', extra_env={
+                'KCONFIG_PROBABILITY': probability,
+                'KCONFIG_SEED': hex(seed),
+            }) == 0, context
+            assert ('warning: KCONFIG_PROBABILITY has malformed format' in
+                    conf.stderr), context
+            assert conf.stderr.count('warning:') == 1, context
+            malformed = conf.config
+
+            assert conf._run_conf('--randconfig', extra_env={
+                'KCONFIG_PROBABILITY': equivalent,
+                'KCONFIG_SEED': hex(seed),
+            }) == 0, context
+            assert 'warning:' not in conf.stderr, context
+            assert conf.config == malformed, context
+
+
+def test_werror(conf):
+    probabilities = [
+        ('', 0), ('50', 0), ('50%', 1), ('-0', 1), ('10:20:30:40', 1),
+    ]
+    for probability, status in probabilities:
+        for werror in ['', '0', '1']:
+            context = 'probability={!r}, werror={!r}'.format(
+                probability, werror)
+            assert conf._run_conf('--randconfig', extra_env={
+                'KCONFIG_PROBABILITY': probability,
+                'KCONFIG_SEED': '0',
+                'KCONFIG_WERROR': werror,
+            }) == status, context
+            if status:
+                assert ('warning: KCONFIG_PROBABILITY has malformed format' in
+                        conf.stderr), context
+                assert conf.stderr.count('warning:') == 1, context
+            else:
+                assert 'warning:' not in conf.stderr, context
+
+
+def test_out_of_range(conf):
+    probabilities = [
+        '-1', '101', '+101', '0:101', '0:0:101', '60:41', '0:60:41',
+        '4294967296', '-4294967296',
+        '999999999999999999999999', '-999999999999999999999999',
+    ]
+    for probability in probabilities:
+        assert conf._run_conf('--randconfig', extra_env={
+            'KCONFIG_PROBABILITY': probability,
+            'KCONFIG_SEED': '0',
+        }) == 1, repr(probability)
+        assert 'KCONFIG_PROBABILITY:' in conf.stderr, repr(probability)
+
+
+def test_valid(conf):
+    probabilities = [
+        ('0', 'n', 'n'),
+        ('0:0', 'n', 'n'),
+        ('100:0', 'y', 'y'),
+        ('0:100', 'y', 'm'),
+        ('100:0:0', 'y', 'n'),
+        ('0:100:0', 'n', 'y'),
+        ('0:0:100', 'n', 'm'),
+        ('000:000:100', 'n', 'm'),
+    ]
+    for probability, boolean, tristate in probabilities:
+        assert conf._run_conf('--randconfig', extra_env={
+            'KCONFIG_PROBABILITY': probability,
+            'KCONFIG_SEED': '0',
+        }) == 0, repr(probability)
+        assert 'warning:' not in conf.stderr, repr(probability)
+        for symbol, value in [('BOOL', boolean), ('TRI', tristate)]:
+            if value == 'n':
+                expected = '# CONFIG_{} is not set'.format(symbol)
+            else:
+                expected = 'CONFIG_{}={}'.format(symbol, value)
+            assert expected in conf.config.splitlines(), repr(probability)
+
+
+def test_single_probability_matches_tristate_split(conf):
+    for seed in range(20):
+        context = 'seed={}'.format(seed)
+        assert conf._run_conf('--randconfig', extra_env={
+            'KCONFIG_PROBABILITY': '50',
+            'KCONFIG_SEED': hex(seed),
+        }) == 0, context
+        assert 'warning:' not in conf.stderr, context
+        single = conf.config
+
+        # 50% boolean y; 25% tristate y, 25% m, and 50% n.
+        assert conf._run_conf('--randconfig', extra_env={
+            'KCONFIG_PROBABILITY': '50:25:25',
+            'KCONFIG_SEED': hex(seed),
+        }) == 0, context
+        assert 'warning:' not in conf.stderr, context
+        assert conf.config == single, context
+
+
+def test_empty(conf):
+    assert conf._run_conf('--randconfig', extra_env={
+        'KCONFIG_PROBABILITY': '',
+        'KCONFIG_SEED': '0',
+    }) == 0
+    assert 'warning:' not in conf.stderr
diff --git a/scripts/kconfig/tests/savedefconfig_range/Kconfig b/scripts/kconfig/tests/savedefconfig_range/Kconfig
new file mode 100644
index 00000000000000..fd59d9082ed5ba
--- /dev/null
+++ b/scripts/kconfig/tests/savedefconfig_range/Kconfig
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: GPL-2.0
+
+# Static default and range values
+
+config INT_DEFAULT_ABOVE_RANGE
+	int
+	range 1 1
+	default 16
+
+config INT_DEFAULT_BELOW_RANGE
+	int
+	range 4 8
+	default 2
+
+# Default and range values determined by other options
+
+config RANGE_UPPER_BOUND
+	int
+	default 3
+
+config INT_DYNAMIC_RANGE
+	int
+	range 0 RANGE_UPPER_BOUND
+	default 4
+
+# Hex
+
+config HEX_DEFAULT_ABOVE_RANGE
+	hex
+	range 0x10 0x20
+	default 0x40
+
+# Implicit default value of 0
+
+config INT_IMPLICIT_DEFAULT_ZERO
+	int
+	range 1 4
+
+# Conditional range
+
+config USE_FIRST_RANGE
+	bool
+	default y
+
+config INT_CONDITIONAL_RANGE
+	int
+	range 1 2 if USE_FIRST_RANGE
+	range 3 4 if !USE_FIRST_RANGE
+	default 3
+
+# Conditional default
+
+config USE_DEFAULT
+	bool
+	default y
+
+config INT_CONDITIONAL_DEFAULT
+	int
+	range 1 2
+	default 3 if USE_DEFAULT
diff --git a/scripts/kconfig/tests/savedefconfig_range/__init__.py b/scripts/kconfig/tests/savedefconfig_range/__init__.py
new file mode 100644
index 00000000000000..961454be732c2f
--- /dev/null
+++ b/scripts/kconfig/tests/savedefconfig_range/__init__.py
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+"""Test savedefconfig with numerical defaults outside of ranges."""
+
+
+def test(conf):
+    assert conf._run_conf('--savedefconfig=defconfig', dot_config='config',
+                          out_file='defconfig') == 0
+    assert conf.config_matches('expected_defconfig')
diff --git a/scripts/kconfig/tests/savedefconfig_range/config b/scripts/kconfig/tests/savedefconfig_range/config
new file mode 100644
index 00000000000000..d939cfe5fd3494
--- /dev/null
+++ b/scripts/kconfig/tests/savedefconfig_range/config
@@ -0,0 +1,7 @@
+CONFIG_INT_DEFAULT_ABOVE_RANGE=1
+CONFIG_INT_DEFAULT_BELOW_RANGE=4
+CONFIG_INT_DYNAMIC_RANGE=3
+CONFIG_HEX_DEFAULT_ABOVE_RANGE=0x20
+CONFIG_INT_IMPLICIT_DEFAULT_ZERO=1
+CONFIG_INT_CONDITIONAL_RANGE=2
+CONFIG_INT_CONDITIONAL_DEFAULT=2
diff --git a/tools/perf/util/help-unknown-cmd.h b/scripts/kconfig/tests/savedefconfig_range/expected_defconfig
similarity index 100%
rename from tools/perf/util/help-unknown-cmd.h
rename to scripts/kconfig/tests/savedefconfig_range/expected_defconfig
diff --git a/scripts/kconfig/tests/warn_changed_input/Kconfig b/scripts/kconfig/tests/warn_changed_input/Kconfig
index 69845e2f3fb34b..df4183a08420f9 100644
--- a/scripts/kconfig/tests/warn_changed_input/Kconfig
+++ b/scripts/kconfig/tests/warn_changed_input/Kconfig
@@ -26,6 +26,11 @@ config NUM
 	  Kconfig resolves it to the constrained in-range value.
 	  The warning should report that adjustment.
 
+config HEX_RANGE
+	hex
+	range 0 0xfffffffffffffffe
+	default 0xffffffffffffffff
+
 config DUP
 	bool "DUP"
 	depends on DEP
diff --git a/scripts/kconfig/tests/warn_changed_input/__init__.py b/scripts/kconfig/tests/warn_changed_input/__init__.py
index 4c3bca6af846e0..17b602dd280e88 100644
--- a/scripts/kconfig/tests/warn_changed_input/__init__.py
+++ b/scripts/kconfig/tests/warn_changed_input/__init__.py
@@ -8,26 +8,21 @@
 
 
 def test(conf):
+    warn_changed_input = {
+        "KCONFIG_WARN_CHANGED_INPUT": "1",
+    }
+
     assert conf.olddefconfig('config') == 0
     assert 'user-provided values changed by Kconfig' not in conf.stderr
 
-    assert conf._run_conf('--olddefconfig', dot_config='config',
-                          extra_env={
-                              'KCONFIG_WARN_CHANGED_INPUT': '1',
-                          }) == 0
+    assert conf.olddefconfig('config', extra_env=warn_changed_input) == 0
     assert conf.stderr_contains('expected_stderr')
     assert conf.config_matches('expected_config')
 
-    assert conf._run_conf('--olddefconfig', dot_config='config',
-                          extra_env={
-                              'KCONFIG_WARN_CHANGED_INPUT': '1',
-                          }, silent=True) == 0
+    assert conf.olddefconfig('config', extra_env=warn_changed_input,
+                             silent=True) == 0
     assert conf.stderr_contains('expected_stderr')
 
-    assert conf._run_conf('--savedefconfig=defconfig', dot_config='config',
-                          out_file='defconfig',
-                          extra_env={
-                              'KCONFIG_WARN_CHANGED_INPUT': '1',
-                          }) == 0
+    assert conf.savedefconfig('config', extra_env=warn_changed_input) == 0
     assert conf.stderr_contains('expected_stderr')
     assert conf.config_matches('expected_defconfig')
diff --git a/scripts/kconfig/tests/warn_changed_input/expected_config b/scripts/kconfig/tests/warn_changed_input/expected_config
index fe8bbec66c5358..6a8d98a69b7e77 100644
--- a/scripts/kconfig/tests/warn_changed_input/expected_config
+++ b/scripts/kconfig/tests/warn_changed_input/expected_config
@@ -4,3 +4,4 @@
 #
 # CONFIG_DEP is not set
 CONFIG_NUM=20
+CONFIG_HEX_RANGE=0xfffffffffffffffe
diff --git a/scripts/kconfig/tests/warn_num_bounds/Kconfig b/scripts/kconfig/tests/warn_num_bounds/Kconfig
new file mode 100644
index 00000000000000..a810225a58a397
--- /dev/null
+++ b/scripts/kconfig/tests/warn_num_bounds/Kconfig
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: GPL-2.0
+
+mainmenu "Numeric bounds test"
+
+config INT_TOO_LOW
+	int "Integer below its type bounds"
+
+config INT_TOO_HIGH
+	int "Integer above its type bounds"
+
+config HEX_TOO_HIGH
+	hex "Hex value above its type bounds"
+
+config INT_MIN
+	int "Minimum valid integer"
+
+config INT_MAX
+	int "Maximum valid integer"
+
+config HEX_MIN
+	hex "Minimum valid hex value"
+
+config HEX_MAX
+	hex "Maximum valid hex value"
diff --git a/scripts/kconfig/tests/warn_num_bounds/__init__.py b/scripts/kconfig/tests/warn_num_bounds/__init__.py
new file mode 100644
index 00000000000000..db75182582178b
--- /dev/null
+++ b/scripts/kconfig/tests/warn_num_bounds/__init__.py
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: GPL-2.0
+"""Test user values outside the numeric type bounds."""
+
+
+def test(conf):
+    in_keys = (
+        '-9223372036854775809\n'
+        '-1\n'
+        '9223372036854775808\n'
+        '1\n'
+        '0x10000000000000000\n'
+        '0x1\n'
+        '-9223372036854775808\n'
+        '9223372036854775807\n'
+        '0x0\n'
+        '0xffffffffffffffff\n'
+    )
+
+    assert conf.oldaskconfig(in_keys=in_keys) == 0
+    assert conf.stderr_matches('expected_frontend_stderr')
+    assert conf.config_matches('expected_frontend_config')
+
+    assert conf.olddefconfig('config') == 0
+    assert conf.stderr_matches('expected_config_stderr')
+    assert conf.config_matches('expected_config')
diff --git a/scripts/kconfig/tests/warn_num_bounds/config b/scripts/kconfig/tests/warn_num_bounds/config
new file mode 100644
index 00000000000000..74bf263f99bf23
--- /dev/null
+++ b/scripts/kconfig/tests/warn_num_bounds/config
@@ -0,0 +1,7 @@
+CONFIG_INT_TOO_LOW=-9223372036854775809
+CONFIG_INT_TOO_HIGH=9223372036854775808
+CONFIG_HEX_TOO_HIGH=0x10000000000000000
+CONFIG_INT_MIN=-9223372036854775808
+CONFIG_INT_MAX=9223372036854775807
+CONFIG_HEX_MIN=0x0
+CONFIG_HEX_MAX=0xffffffffffffffff
diff --git a/scripts/kconfig/tests/warn_num_bounds/expected_config b/scripts/kconfig/tests/warn_num_bounds/expected_config
new file mode 100644
index 00000000000000..8b1aef612bc6b2
--- /dev/null
+++ b/scripts/kconfig/tests/warn_num_bounds/expected_config
@@ -0,0 +1,11 @@
+#
+# Automatically generated file; DO NOT EDIT.
+# Numeric bounds test
+#
+CONFIG_INT_TOO_LOW=-9223372036854775809
+CONFIG_INT_TOO_HIGH=9223372036854775808
+CONFIG_HEX_TOO_HIGH=0x10000000000000000
+CONFIG_INT_MIN=-9223372036854775808
+CONFIG_INT_MAX=9223372036854775807
+CONFIG_HEX_MIN=0x0
+CONFIG_HEX_MAX=0xffffffffffffffff
diff --git a/scripts/kconfig/tests/warn_num_bounds/expected_config_stderr b/scripts/kconfig/tests/warn_num_bounds/expected_config_stderr
new file mode 100644
index 00000000000000..be2ed7fbf64851
--- /dev/null
+++ b/scripts/kconfig/tests/warn_num_bounds/expected_config_stderr
@@ -0,0 +1,3 @@
+.config:1:warning: value '-9223372036854775809' for INT_TOO_LOW is outside the 64-bit signed integer bounds
+.config:2:warning: value '9223372036854775808' for INT_TOO_HIGH is outside the 64-bit signed integer bounds
+.config:3:warning: value '0x10000000000000000' for HEX_TOO_HIGH is outside the 64-bit unsigned integer bounds
diff --git a/scripts/kconfig/tests/warn_num_bounds/expected_frontend_config b/scripts/kconfig/tests/warn_num_bounds/expected_frontend_config
new file mode 100644
index 00000000000000..a7e842517026ef
--- /dev/null
+++ b/scripts/kconfig/tests/warn_num_bounds/expected_frontend_config
@@ -0,0 +1,11 @@
+#
+# Automatically generated file; DO NOT EDIT.
+# Numeric bounds test
+#
+CONFIG_INT_TOO_LOW=-1
+CONFIG_INT_TOO_HIGH=1
+CONFIG_HEX_TOO_HIGH=0x1
+CONFIG_INT_MIN=-9223372036854775808
+CONFIG_INT_MAX=9223372036854775807
+CONFIG_HEX_MIN=0x0
+CONFIG_HEX_MAX=0xffffffffffffffff
diff --git a/scripts/kconfig/tests/warn_num_bounds/expected_frontend_stderr b/scripts/kconfig/tests/warn_num_bounds/expected_frontend_stderr
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
index ab0b8125c8cbc6..c520741f85c2d4 100755
--- a/scripts/link-vmlinux.sh
+++ b/scripts/link-vmlinux.sh
@@ -96,6 +96,11 @@ vmlinux_link()
 		ldflags="${ldflags} ${wl}--strip-debug"
 	fi
 
+	# Only the final link actually requires the relocations.
+	if [ "${output}" = "${VMLINUX}" ] && is_enabled CONFIG_ARCH_VMLINUX_NEEDS_RELOCS; then
+		ldflags="${ldflags} ${wl}--emit-relocs"
+	fi
+
 	if [ -n "${generate_map}" ];  then
 		ldflags="${ldflags} ${wl}-Map=vmlinux.map"
 	fi
@@ -118,11 +123,15 @@ kallsymso_changed()
 	! cmp -s "${kallsymso_prev}.sym" "${kallsymso}.sym"
 }
 
-# Create ${2}.o file with all symbols from the ${1} object file
+# Create ${2}.o with the kallsyms tables for ${1} (the vmlinux, or an empty
+# listing for the first pass); list the symbols used in ${3} if given.
 kallsyms()
 {
 	local kallsymopt;
 
+	if [ -n "${3:-}" ]; then
+		kallsymopt="--sysmap=${3}"
+	fi
 	if is_enabled CONFIG_KALLSYMS_ALL; then
 		kallsymopt="${kallsymopt} --all-symbols"
 	fi
@@ -132,7 +141,7 @@ kallsyms()
 	fi
 
 	info KSYMS "${2}.S"
-	scripts/kallsyms ${kallsymopt} "${1}" > "${2}.S"
+	scripts/kallsyms ${kallsymopt} "${1}" "${2}.bin" > "${2}.S"
 
 	info AS "${2}.o"
 	${CC} ${NOSTDINC_FLAGS} ${LINUXINCLUDE} ${KBUILD_CPPFLAGS} \
@@ -145,23 +154,20 @@ kallsyms()
 # Perform kallsyms for the given temporary vmlinux.
 sysmap_and_kallsyms()
 {
-	mksysmap "${1}" "${1}.syms"
-	kallsyms "${1}.syms" "${1}.kallsyms"
-
+	kallsyms "${1}" "${1}.kallsyms" "${1}.syms"
 	kallsyms_sysmap=${1}.syms
 }
 
 # Create map file with all symbols from ${1}
-# See mksymap for additional details
 mksysmap()
 {
-	info NM ${2}
-	${NM} -n "${1}" | sed -f "${srctree}/scripts/mksysmap" > "${2}"
+	info SYSMAP ${2}
+	scripts/kallsyms --sysmap="${2}" "${1}"
 }
 
 sorttable()
 {
-	${NM} -S ${1} > .tmp_vmlinux.nm-sort
+	${NM} -p -S ${1} > .tmp_vmlinux.nm-sort
 	${objtree}/scripts/sorttable -s .tmp_vmlinux.nm-sort ${1}
 }
 
diff --git a/scripts/mkcompile_h b/scripts/mkcompile_h
index 2596f78e52ef95..bf0d5669d9bcd5 100755
--- a/scripts/mkcompile_h
+++ b/scripts/mkcompile_h
@@ -11,7 +11,7 @@ else
 	LINUX_COMPILE_BY=$KBUILD_BUILD_USER
 fi
 if test -z "$KBUILD_BUILD_HOST"; then
-	LINUX_COMPILE_HOST=`uname -n`
+	LINUX_COMPILE_HOST=$(uname -n)
 else
 	LINUX_COMPILE_HOST=$KBUILD_BUILD_HOST
 fi
diff --git a/scripts/mksysmap b/scripts/mksysmap
deleted file mode 100755
index 856b26ba2ac024..00000000000000
--- a/scripts/mksysmap
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/bin/sed -f
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# sed script to filter out symbols that are not needed for System.map,
-# or not suitable for kallsyms. The input should be 'nm -n <file>'.
-#
-# System.map is used by module-init tools and some debugging
-# tools to retrieve the actual addresses of symbols in the kernel.
-#
-# readprofile starts reading symbols when _stext is found, and
-# continue until it finds a symbol which is not either of 'T', 't',
-# 'W' or 'w'.
-#
-# ---------------------------------------------------------------------------
-# Ignored symbol types
-#
-
-# a: local absolute symbols
-# N: debugging symbols
-# U: undefined global symbols
-# w: local weak symbols
-/ [aNUw] /d
-
-# ---------------------------------------------------------------------------
-# Ignored prefixes
-#  (do not forget a space before each pattern)
-
-# local symbols for ARM, MIPS, etc.
-/ \$/d
-
-# local labels, .LBB, .Ltmpxxx, .L__unnamed_xx, .LASANPC, etc.
-/ \.L/d
-
-# arm64 EFI stub namespace
-/ __efistub_/d
-
-# arm64 local symbols in PIE namespace
-/ __pi_\$/d
-/ __pi_\.L/d
-
-# arm64 local symbols in non-VHE KVM namespace
-/ __kvm_nvhe_\$/d
-/ __kvm_nvhe_\.L/d
-
-# lld arm/aarch64/mips thunks
-/ __[[:alnum:]]*Thunk_/d
-
-# CFI type identifiers
-/ __kcfi_typeid_/d
-/ __kvm_nvhe___kcfi_typeid_/d
-/ __pi___kcfi_typeid_/d
-
-# CRC from modversions
-/ __crc_/d
-
-# EXPORT_SYMBOL (symbol name)
-/ __kstrtab_/d
-
-# EXPORT_SYMBOL (namespace)
-/ __kstrtabns_/d
-
-# MODULE_DEVICE_TABLE (symbol name)
-/ __mod_device_table__/d
-
-# ---------------------------------------------------------------------------
-# Ignored suffixes
-#  (do not forget '$' after each pattern)
-
-# arm
-/_from_arm$/d
-/_from_thumb$/d
-/_veneer$/d
-
-# ---------------------------------------------------------------------------
-# Ignored symbols (exact match)
-#  (do not forget a space before and '$' after each pattern)
-
-# for LoongArch?
-/ L0$/d
-
-# ppc
-/ _SDA_BASE_$/d
-/ _SDA2_BASE_$/d
-
-# MODULE_INFO()
-/ __UNIQUE_ID_modinfo_[0-9]*$/d
-
-# ---------------------------------------------------------------------------
-# Ignored patterns
-#  (symbols that contain the pattern are ignored)
-
-# ppc stub
-/\.long_branch\./d
-/\.plt_branch\./d
diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c
index b4178c42d08f50..92596bfa6aec15 100644
--- a/scripts/mod/devicetable-offsets.c
+++ b/scripts/mod/devicetable-offsets.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #define COMPILE_OFFSETS
+#include <linux/device-id/scmi.h>
 #include <linux/kbuild.h>
 #include <linux/mod_devicetable.h>
 
@@ -144,6 +145,10 @@ int main(void)
 	DEVID(rpmsg_device_id);
 	DEVID_FIELD(rpmsg_device_id, name);
 
+	DEVID(scmi_device_id);
+	DEVID_FIELD(scmi_device_id, protocol_id);
+	DEVID_FIELD(scmi_device_id, name);
+
 	DEVID(i2c_device_id);
 	DEVID_FIELD(i2c_device_id, name);
 
@@ -254,6 +259,9 @@ int main(void)
 	DEVID(auxiliary_device_id);
 	DEVID_FIELD(auxiliary_device_id, name);
 
+	DEVID(arm_smccc_device_id);
+	DEVID_FIELD(arm_smccc_device_id, func_id);
+
 	DEVID(ssam_device_id);
 	DEVID_FIELD(ssam_device_id, match_flags);
 	DEVID_FIELD(ssam_device_id, domain);
diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
index 8d36c74dec2d55..61a831c18721c3 100644
--- a/scripts/mod/file2alias.c
+++ b/scripts/mod/file2alias.c
@@ -121,6 +121,7 @@ typedef struct {
 /* Big exception to the "don't include kernel headers into userspace, which
  * even potentially has different endianness and word sizes, since
  * we handle those differences explicitly below */
+#include "../../include/linux/device-id/scmi.h"
 #include "../../include/linux/mod_devicetable.h"
 
 struct devtable {
@@ -852,6 +853,16 @@ static void do_rpmsg_entry(struct module *mod, void *symval)
 	module_alias_printf(mod, false, RPMSG_DEVICE_MODALIAS_FMT, *name);
 }
 
+/* Looks like: scmi:NN:S */
+static void do_scmi_entry(struct module *mod, void *symval)
+{
+	DEF_FIELD(symval, scmi_device_id, protocol_id);
+	DEF_FIELD_ADDR(symval, scmi_device_id, name);
+
+	module_alias_printf(mod, false, SCMI_MODULE_PREFIX "%02x:%s",
+			    protocol_id, *name);
+}
+
 /* Looks like: i2c:S */
 static void do_i2c_entry(struct module *mod, void *symval)
 {
@@ -1349,6 +1360,13 @@ static void do_auxiliary_entry(struct module *mod, void *symval)
 	module_alias_printf(mod, false, AUXILIARY_MODULE_PREFIX "%s", *name);
 }
 
+static void do_arm_smccc_entry(struct module *mod, void *symval)
+{
+	DEF_FIELD(symval, arm_smccc_device_id, func_id);
+
+	module_alias_printf(mod, false, ARM_SMCCC_MODULE_PREFIX "f%08X", func_id);
+}
+
 /*
  * Looks like: ssam:dNcNtNiNfN
  *
@@ -1491,6 +1509,7 @@ static const struct devtable devtable[] = {
 	{"virtio", SIZE_virtio_device_id, do_virtio_entry},
 	{"vmbus", SIZE_hv_vmbus_device_id, do_vmbus_entry},
 	{"rpmsg", SIZE_rpmsg_device_id, do_rpmsg_entry},
+	{"scmi", SIZE_scmi_device_id, do_scmi_entry},
 	{"i2c", SIZE_i2c_device_id, do_i2c_entry},
 	{"i3c", SIZE_i3c_device_id, do_i3c_entry},
 	{"slim", SIZE_slim_device_id, do_slim_entry},
@@ -1519,6 +1538,7 @@ static const struct devtable devtable[] = {
 	{"mhi", SIZE_mhi_device_id, do_mhi_entry},
 	{"mhi_ep", SIZE_mhi_device_id, do_mhi_ep_entry},
 	{"auxiliary", SIZE_auxiliary_device_id, do_auxiliary_entry},
+	{"arm_smccc", SIZE_arm_smccc_device_id, do_arm_smccc_entry},
 	{"ssam", SIZE_ssam_device_id, do_ssam_entry},
 	{"dfl", SIZE_dfl_device_id, do_dfl_entry},
 	{"ishtp", SIZE_ishtp_device_id, do_ishtp_entry},
diff --git a/scripts/mod/modpost.h b/scripts/mod/modpost.h
index d5f6d82837d5b4..666beca685e48f 100644
--- a/scripts/mod/modpost.h
+++ b/scripts/mod/modpost.h
@@ -198,7 +198,7 @@ static inline bool is_valid_name(struct elf_info *elf, Elf_Sym *sym)
 
 	if (!name || !strlen(name))
 		return false;
-	return !is_mapping_symbol(name);
+	return !is_ignored_kernel_symbol(name);
 }
 
 /* symsearch.c */
diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files
index 6e39fa8540df2f..e46337e4bfb8f8 100755
--- a/scripts/remove-stale-files
+++ b/scripts/remove-stale-files
@@ -26,3 +26,5 @@ rm -f scripts/selinux/genheaders/genheaders
 rm -f *.spec
 
 rm -f lib/test_fortify.log
+
+rm -f usr/gen_init_cpio
diff --git a/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py b/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py
index d7776f072e0342..93787a1cd1566d 100644
--- a/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py
+++ b/scripts/sbom/tests/cmd_graph/test_savedcmd_parser.py
@@ -338,7 +338,7 @@ def test_xen_hypercalls(self):
 
     # gen_initramfs.sh command tests
     def test_gen_initramfs(self):
-        cmd = "sh ../usr/gen_initramfs.sh -o usr/initramfs_data.cpio -l usr/.initramfs_data.cpio.d    ../usr/default_cpio_list"
+        cmd = "sh ../scripts/gen_initramfs.sh -o usr/initramfs_data.cpio -l usr/.initramfs_data.cpio.d    ../usr/default_cpio_list"
         expected = "../usr/default_cpio_list"
         self._assert_parsing(cmd, expected)
 
diff --git a/scripts/setlocalversion b/scripts/setlocalversion
index 28169d7e143b69..f264c2460ad779 100755
--- a/scripts/setlocalversion
+++ b/scripts/setlocalversion
@@ -33,7 +33,7 @@ if test $# -gt 0 -o ! -d "$srctree"; then
 fi
 
 try_tag() {
-	tag="$1"
+	local tag="$1"
 
 	# Is $tag an annotated tag?
 	if [ "$(git cat-file -t "$tag" 2> /dev/null)" != tag ]; then
@@ -59,7 +59,6 @@ scm_version()
 {
 	local short=false
 	local no_dirty=false
-	local tag
 
 	while [ $# -gt 0 ];
 	do
diff --git a/scripts/sorttable.c b/scripts/sorttable.c
index d7b50581c7329f..88e49a5c425127 100644
--- a/scripts/sorttable.c
+++ b/scripts/sorttable.c
@@ -322,10 +322,23 @@ static int parse_symbols(const char *fname)
 		return -1;
 	}
 
-	while (fscanf(fp, "%16s %16s %c %*s\n", addr_str, size_str, &type) == 3) {
+	while (fscanf(fp, "%16s %16s %c%*[^\n]", addr_str, size_str, &type) == 3) {
 		uint64_t addr;
 		uint64_t size;
 
+		/*
+		 * nm outputs size-less entries with a missing 2nd value, which
+		 * means fscanf() just misread its fields.
+		 *
+		 * These don't matter as a call site cannot be in an empty
+		 * function, so just skip them.
+		 *
+		 * nm pads address and size to the same width, so if their
+		 * widths differ, this is a misread.
+		 */
+		if (strlen(size_str) != strlen(addr_str))
+			continue;
+
 		/* Only care about functions */
 		if (type != 't' && type != 'T' && type != 'W')
 			continue;
diff --git a/scripts/spelling.txt b/scripts/spelling.txt
index 3372873cd7bb7d..9ec2529bc58472 100644
--- a/scripts/spelling.txt
+++ b/scripts/spelling.txt
@@ -872,8 +872,8 @@ infromation||information
 ingore||ignore
 inheritence||inheritance
 inital||initial
-initalised||initialized
-initalise||initialize
+initalised||initialised
+initalise||initialise
 initalized||initialized
 initalize||initialize
 initation||initiation
diff --git a/scripts/ver_linux b/scripts/ver_linux
index 00bdaf30d590d5..e06d7c650710c7 100755
--- a/scripts/ver_linux
+++ b/scripts/ver_linux
@@ -57,6 +57,7 @@ BEGIN {
 	printversion("pahole", version("pahole --version"))
 	printversion("Pcmcia-cs", version("cardmgr -V"))
 	printversion("pcmciautils", version("pccardctl -V"))
+	printversion("pigz", version("pigz --version"))
 	printversion("PPP", version("pppd --version"))
 	printversion("procps", version("ps --version"))
 	printversion("Python", version("python3 -V"))
diff --git a/security/apparmor/af_unix.c b/security/apparmor/af_unix.c
index b908e744818c9b..de80561348fc44 100644
--- a/security/apparmor/af_unix.c
+++ b/security/apparmor/af_unix.c
@@ -715,7 +715,7 @@ static void update_peer_ctx(struct sock *sk, struct aa_sk_ctx *ctx,
  * boundaries. Otherwise cached info off file is sufficient
  */
 int aa_unix_file_perm(const struct cred *subj_cred, struct aa_label *label,
-		      const char *op, u32 request, struct file *file)
+		      const char *op, u32 request, const struct file *file)
 {
 	struct socket *sock = (struct socket *) file->private_data;
 	struct sockaddr_un *addr, *peer_addr;
diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
index 5b42140e12e872..d1386722537d0f 100644
--- a/security/apparmor/apparmorfs.c
+++ b/security/apparmor/apparmorfs.c
@@ -2073,7 +2073,7 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent)
 	return error;
 }
 
-static struct dentry *ns_mkdir_op(struct mnt_idmap *idmap, struct inode *dir,
+static struct dentry *ns_mkdir_op(const struct mnt_idmap *idmap, struct inode *dir,
 				  struct dentry *dentry, umode_t mode)
 {
 	struct aa_ns *ns, *parent;
diff --git a/security/apparmor/file.c b/security/apparmor/file.c
index 3e74b613db325c..305fdb341ddbb5 100644
--- a/security/apparmor/file.c
+++ b/security/apparmor/file.c
@@ -489,7 +489,7 @@ static void update_file_ctx(struct aa_file_ctx *fctx, struct aa_label *label,
 
 static int __file_path_perm(const char *op, const struct cred *subj_cred,
 			    struct aa_label *label,
-			    struct aa_label *flabel, struct file *file,
+			    struct aa_label *flabel, const struct file *file,
 			    u32 request, u32 denied, bool in_atomic)
 {
 	struct aa_profile *profile;
@@ -550,7 +550,7 @@ static int __file_path_perm(const char *op, const struct cred *subj_cred,
 
 static int __file_sock_perm(const char *op, const struct cred *subj_cred,
 			    struct aa_label *label,
-			    struct aa_label *flabel, struct file *file,
+			    struct aa_label *flabel, const struct file *file,
 			    u32 request, u32 denied)
 {
 	int error;
@@ -579,7 +579,7 @@ static bool __file_is_delegated(struct aa_label *obj_label)
 	return unconfined(obj_label);
 }
 
-static bool __is_unix_file(struct file *file)
+static bool __is_unix_file(const struct file *file)
 {
 	struct socket *sock = (struct socket *) file->private_data;
 
@@ -595,7 +595,7 @@ static bool __is_unix_file(struct file *file)
 	return false;
 }
 
-static bool __unix_needs_revalidation(struct file *file, struct aa_label *label,
+static bool __unix_needs_revalidation(const struct file *file, struct aa_label *label,
 				      u32 request)
 {
 	struct socket *sock = (struct socket *) file->private_data;
@@ -624,7 +624,7 @@ static bool __unix_needs_revalidation(struct file *file, struct aa_label *label,
  * Returns: %0 if access allowed else error
  */
 int aa_file_perm(const char *op, const struct cred *subj_cred,
-		 struct aa_label *label, struct file *file,
+		 struct aa_label *label, const struct file *file,
 		 u32 request, bool in_atomic)
 {
 	struct aa_file_ctx *fctx;
diff --git a/security/apparmor/include/af_unix.h b/security/apparmor/include/af_unix.h
index 4a62e600d82b07..62e3269273eb71 100644
--- a/security/apparmor/include/af_unix.h
+++ b/security/apparmor/include/af_unix.h
@@ -50,6 +50,6 @@ int aa_unix_msg_perm(const char *op, u32 request, struct socket *sock,
 int aa_unix_opt_perm(const char *op, u32 request, struct socket *sock, int level,
 		     int optname);
 int aa_unix_file_perm(const struct cred *subj_cred, struct aa_label *label,
-		      const char *op, u32 request, struct file *file);
+		      const char *op, u32 request, const struct file *file);
 
 #endif /* __AA_AF_UNIX_H */
diff --git a/security/apparmor/include/file.h b/security/apparmor/include/file.h
index 1614c07fc53eea..d69a0611a981eb 100644
--- a/security/apparmor/include/file.h
+++ b/security/apparmor/include/file.h
@@ -29,7 +29,7 @@ struct path;
 				 AA_MAY_CHMOD | AA_MAY_CHOWN | AA_MAY_LOCK | \
 				 AA_EXEC_MMAP | AA_MAY_LINK)
 
-static inline struct aa_file_ctx *file_ctx(struct file *file)
+static inline struct aa_file_ctx *file_ctx(const struct file *file)
 {
 	return file->f_security + apparmor_blob_sizes.lbs_file;
 }
@@ -97,7 +97,7 @@ int aa_path_link(const struct cred *subj_cred, struct aa_label *label,
 		 struct dentry *new_dentry);
 
 int aa_file_perm(const char *op, const struct cred *subj_cred,
-		 struct aa_label *label, struct file *file,
+		 struct aa_label *label, const struct file *file,
 		 u32 request, bool in_atomic);
 
 void aa_inherit_files(const struct cred *cred, struct files_struct *files);
diff --git a/security/apparmor/include/net.h b/security/apparmor/include/net.h
index 375341929cb651..e387b50920b3b3 100644
--- a/security/apparmor/include/net.h
+++ b/security/apparmor/include/net.h
@@ -112,7 +112,7 @@ int aa_label_sk_perm(const struct cred *subj_cred, struct aa_label *label,
 		     const char *op, u32 request, const struct sock *sk);
 int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label,
 		      const char *op, u32 request,
-		      struct file *file);
+		      const struct file *file);
 
 int apparmor_secmark_check(struct aa_label *label, char *op, u32 request,
 			   u32 secid, const struct sock *sk);
diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c
index d502ad0ac26f5c..9d572cf8486b2c 100644
--- a/security/apparmor/lsm.c
+++ b/security/apparmor/lsm.c
@@ -397,7 +397,7 @@ static int apparmor_path_rename(const struct path *old_dir, struct dentry *old_d
 
 	label = begin_current_label_crit_section(&needput);
 	if (!unconfined(label)) {
-		struct mnt_idmap *idmap = mnt_idmap(old_dir->mnt);
+		const struct mnt_idmap *idmap = mnt_idmap(old_dir->mnt);
 		vfsuid_t vfsuid;
 		struct path old_path = { .mnt = old_dir->mnt,
 					 .dentry = old_dentry };
@@ -485,7 +485,7 @@ static int apparmor_file_open(struct file *file)
 
 	label = aa_get_newest_cred_label_condref(file->f_cred, &needput);
 	if (!unconfined(label)) {
-		struct mnt_idmap *idmap = file_mnt_idmap(file);
+		const struct mnt_idmap *idmap = file_mnt_idmap(file);
 		struct inode *inode = file_inode(file);
 		vfsuid_t vfsuid;
 		struct path_cond cond = {
@@ -525,7 +525,7 @@ static void apparmor_file_free_security(struct file *file)
 		aa_put_label(rcu_access_pointer(ctx->label));
 }
 
-static int common_file_perm(const char *op, struct file *file, u32 mask)
+static int common_file_perm(const char *op, const struct file *file, u32 mask)
 {
 	struct aa_label *label;
 	bool needput;
@@ -543,7 +543,7 @@ static int apparmor_file_receive(struct file *file)
 	return common_file_perm(OP_FRECEIVE, file, aa_map_file_to_perms(file));
 }
 
-static int apparmor_file_permission(struct file *file, int mask)
+static int apparmor_file_permission(const struct file *file, int mask)
 {
 	return common_file_perm(OP_FPERM, file, mask);
 }
diff --git a/security/apparmor/net.c b/security/apparmor/net.c
index a333e6aff9262d..6048e3c5533d71 100644
--- a/security/apparmor/net.c
+++ b/security/apparmor/net.c
@@ -325,7 +325,7 @@ int aa_sk_perm(const char *op, u32 request, const struct sock *sk)
 
 
 int aa_sock_file_perm(const struct cred *subj_cred, struct aa_label *label,
-		      const char *op, u32 request, struct file *file)
+		      const char *op, u32 request, const struct file *file)
 {
 	struct socket *sock = (struct socket *) file->private_data;
 
diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c
index 7b98f5d1e2be83..8f8c3de3035f01 100644
--- a/security/bpf/hooks.c
+++ b/security/bpf/hooks.c
@@ -33,6 +33,7 @@ static int __init bpf_lsm_init(void)
 
 struct lsm_blob_sizes bpf_lsm_blob_sizes __ro_after_init = {
 	.lbs_inode = sizeof(struct bpf_storage_blob),
+	.lbs_xattr_count = BPF_LSM_INODE_INIT_XATTRS,
 };
 
 DEFINE_LSM(bpf) = {
diff --git a/security/commoncap.c b/security/commoncap.c
index 3399535808fec7..565c893c6d4b4a 100644
--- a/security/commoncap.c
+++ b/security/commoncap.c
@@ -10,13 +10,10 @@
 #include <linux/file.h>
 #include <linux/mm.h>
 #include <linux/mman.h>
-#include <linux/pagemap.h>
-#include <linux/swap.h>
 #include <linux/skbuff.h>
 #include <linux/netlink.h>
 #include <linux/ptrace.h>
 #include <linux/xattr.h>
-#include <linux/hugetlb.h>
 #include <linux/mount.h>
 #include <linux/sched.h>
 #include <linux/prctl.h>
@@ -348,7 +345,7 @@ int cap_inode_need_killpriv(struct dentry *dentry)
  *
  * Return: 0 if successful, -ve on error.
  */
-int cap_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry)
+int cap_inode_killpriv(const struct mnt_idmap *idmap, struct dentry *dentry)
 {
 	int error;
 
@@ -417,7 +414,7 @@ static bool is_v3header(int size, const struct vfs_cap_data *cap)
  * by the integrity subsystem, which really wants the unconverted values -
  * so that's good.
  */
-int cap_inode_getsecurity(struct mnt_idmap *idmap,
+int cap_inode_getsecurity(const struct mnt_idmap *idmap,
 			  struct inode *inode, const char *name, void **buffer,
 			  bool alloc)
 {
@@ -566,7 +563,7 @@ static bool validheader(size_t size, const struct vfs_cap_data *cap)
  *
  * Return: On success, return the new size; on error, return < 0.
  */
-int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry,
+int cap_convert_nscap(const struct mnt_idmap *idmap, struct dentry *dentry,
 		      const void **ivalue, size_t size)
 {
 	struct vfs_ns_cap_data *nscap;
@@ -672,7 +669,7 @@ static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps,
  * permissions. On non-idmapped mounts or if permission checking is to be
  * performed on the raw inode simply pass @nop_mnt_idmap.
  */
-int get_vfs_caps_from_disk(struct mnt_idmap *idmap,
+int get_vfs_caps_from_disk(const struct mnt_idmap *idmap,
 			   const struct dentry *dentry,
 			   struct cpu_vfs_cap_data *cpu_caps)
 {
@@ -1063,7 +1060,7 @@ int cap_inode_setxattr(struct dentry *dentry, const char *name,
  * This is used to make sure security xattrs don't get removed by those who
  * aren't privileged to remove them.
  */
-int cap_inode_removexattr(struct mnt_idmap *idmap,
+int cap_inode_removexattr(const struct mnt_idmap *idmap,
 			  struct dentry *dentry, const char *name)
 {
 	struct user_namespace *user_ns = dentry->d_sb->s_user_ns;
@@ -1169,11 +1166,13 @@ int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags)
 		break;
 
 	case LSM_SETID_FS:
-		/* juggle the capabilities to follow FSUID changes, unless
-		 * otherwise suppressed
+		/* Juggle the capabilities to follow FSUID changes, unless
+		 * otherwise suppressed.
 		 *
-		 * FIXME - is fsuser used for all CAP_FS_MASK capabilities?
-		 *          if not, we might be a bit too harsh here.
+		 * CAP_FS_SET is exactly the set of capabilities that an
+		 * fsuid of 0 historically granted (see CAP_FS_MASK in
+		 * <linux/capability.h>), so we drop that set when fsuid
+		 * leaves 0.
 		 */
 		if (!issecure(SECURE_NO_SETUID_FIXUP)) {
 			kuid_t root_uid = make_kuid(old->user_ns, 0);
diff --git a/security/inode.c b/security/inode.c
index 08040236767418..346172b8f81471 100644
--- a/security/inode.c
+++ b/security/inode.c
@@ -15,7 +15,6 @@
 #include <linux/fs.h>
 #include <linux/fs_context.h>
 #include <linux/mount.h>
-#include <linux/pagemap.h>
 #include <linux/init.h>
 #include <linux/namei.h>
 #include <linux/security.h>
diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c
index b59e3f121b8a89..ac8328812e6ef9 100644
--- a/security/integrity/evm/evm_main.c
+++ b/security/integrity/evm/evm_main.c
@@ -481,7 +481,7 @@ static enum integrity_status evm_verify_current_integrity(struct dentry *dentry)
  *
  * Returns 1 if passed xattr value differs from current value, 0 otherwise.
  */
-static int evm_xattr_change(struct mnt_idmap *idmap,
+static int evm_xattr_change(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, const char *xattr_name,
 			    const void *xattr_value, size_t xattr_value_len)
 {
@@ -517,7 +517,7 @@ static int evm_xattr_change(struct mnt_idmap *idmap,
  * For posix xattr acls only, permit security.evm, even if it currently
  * doesn't exist, to be updated unless the EVM signature is immutable.
  */
-static int evm_protect_xattr(struct mnt_idmap *idmap,
+static int evm_protect_xattr(const struct mnt_idmap *idmap,
 			     struct dentry *dentry, const char *xattr_name,
 			     const void *xattr_value, size_t xattr_value_len)
 {
@@ -563,7 +563,7 @@ static int evm_protect_xattr(struct mnt_idmap *idmap,
 				    dentry->d_inode, dentry->d_name.name,
 				    "update_metadata",
 				    integrity_status_msg[evm_status],
-				    -EPERM, 0);
+				    -EPERM, 0, -EPERM);
 	}
 out:
 	/* Exception if the HMAC is not going to be calculated. */
@@ -588,7 +588,7 @@ static int evm_protect_xattr(struct mnt_idmap *idmap,
 		integrity_audit_msg(AUDIT_INTEGRITY_METADATA, d_backing_inode(dentry),
 				    dentry->d_name.name, "appraise_metadata",
 				    integrity_status_msg[evm_status],
-				    -EPERM, 0);
+				    -EPERM, 0, -EPERM);
 	return evm_status == INTEGRITY_PASS ? 0 : -EPERM;
 }
 
@@ -607,7 +607,7 @@ static int evm_protect_xattr(struct mnt_idmap *idmap,
  * userspace from writing HMAC value.  Writing 'security.evm' requires
  * requires CAP_SYS_ADMIN privileges.
  */
-static int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int evm_inode_setxattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			      const char *xattr_name, const void *xattr_value,
 			      size_t xattr_value_len, int flags)
 {
@@ -639,7 +639,7 @@ static int evm_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
  * Removing 'security.evm' requires CAP_SYS_ADMIN privileges and that
  * the current value is valid.
  */
-static int evm_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int evm_inode_removexattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 				 const char *xattr_name)
 {
 	/* Policy permits modification of the protected xattrs even though
@@ -652,7 +652,7 @@ static int evm_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
 }
 
 #ifdef CONFIG_FS_POSIX_ACL
-static int evm_inode_set_acl_change(struct mnt_idmap *idmap,
+static int evm_inode_set_acl_change(const struct mnt_idmap *idmap,
 				    struct dentry *dentry, const char *name,
 				    struct posix_acl *kacl)
 {
@@ -671,7 +671,7 @@ static int evm_inode_set_acl_change(struct mnt_idmap *idmap,
 	return 0;
 }
 #else
-static inline int evm_inode_set_acl_change(struct mnt_idmap *idmap,
+static inline int evm_inode_set_acl_change(const struct mnt_idmap *idmap,
 					   struct dentry *dentry,
 					   const char *name,
 					   struct posix_acl *kacl)
@@ -693,7 +693,7 @@ static inline int evm_inode_set_acl_change(struct mnt_idmap *idmap,
  *
  * Return: zero on success, -EPERM on failure.
  */
-static int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+static int evm_inode_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			     const char *acl_name, struct posix_acl *kacl)
 {
 	enum integrity_status evm_status;
@@ -729,7 +729,7 @@ static int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 		integrity_audit_msg(AUDIT_INTEGRITY_METADATA, d_backing_inode(dentry),
 				    dentry->d_name.name, "appraise_metadata",
 				    integrity_status_msg[evm_status],
-				    -EPERM, 0);
+				    -EPERM, 0, -EPERM);
 	return -EPERM;
 }
 
@@ -745,7 +745,7 @@ static int evm_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
  *
  * Return: zero on success, -EPERM on failure.
  */
-static int evm_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+static int evm_inode_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 				const char *acl_name)
 {
 	return evm_inode_set_acl(idmap, dentry, acl_name, NULL);
@@ -926,14 +926,14 @@ static void evm_inode_post_removexattr(struct dentry *dentry,
  * Update the 'security.evm' xattr with the EVM HMAC re-calculated after
  * removing posix acls.
  */
-static inline void evm_inode_post_remove_acl(struct mnt_idmap *idmap,
+static inline void evm_inode_post_remove_acl(const struct mnt_idmap *idmap,
 					     struct dentry *dentry,
 					     const char *acl_name)
 {
 	evm_inode_post_removexattr(dentry, acl_name);
 }
 
-static int evm_attr_change(struct mnt_idmap *idmap,
+static int evm_attr_change(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, struct iattr *attr)
 {
 	struct inode *inode = d_backing_inode(dentry);
@@ -956,7 +956,7 @@ static int evm_attr_change(struct mnt_idmap *idmap,
  * Permit update of file attributes when files have a valid EVM signature,
  * except in the case of them having an immutable portable signature.
  */
-static int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int evm_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			     struct iattr *attr)
 {
 	unsigned int ia_valid = attr->ia_valid;
@@ -992,7 +992,8 @@ static int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 
 	integrity_audit_msg(AUDIT_INTEGRITY_METADATA, d_backing_inode(dentry),
 			    dentry->d_name.name, "appraise_metadata",
-			    integrity_status_msg[evm_status], -EPERM, 0);
+			    integrity_status_msg[evm_status], -EPERM, 0,
+			    -EPERM);
 	return -EPERM;
 }
 
@@ -1008,7 +1009,7 @@ static int evm_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
  * This function is called from notify_change(), which expects the caller
  * to lock the inode's i_mutex.
  */
-static void evm_inode_post_setattr(struct mnt_idmap *idmap,
+static void evm_inode_post_setattr(const struct mnt_idmap *idmap,
 				   struct dentry *dentry, int ia_valid)
 {
 	if (!evm_revalidate_status(NULL))
@@ -1140,7 +1141,7 @@ static void evm_file_release(struct file *file)
 		iint->flags &= ~EVM_NEW_FILE;
 }
 
-static void evm_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry)
+static void evm_post_path_mknod(const struct mnt_idmap *idmap, struct dentry *dentry)
 {
 	struct inode *inode = d_backing_inode(dentry);
 	struct evm_iint_cache *iint = evm_iint_inode(inode);
diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h
index 10214f73ca1ed9..b502854f28ee71 100644
--- a/security/integrity/ima/ima.h
+++ b/security/integrity/ima/ima.h
@@ -423,7 +423,7 @@ static inline void ima_process_queued_keys(void) {}
 #endif /* CONFIG_IMA_QUEUE_EARLY_BOOT_KEYS */
 
 /* LIM API function definitions */
-int ima_get_action(struct mnt_idmap *idmap, struct inode *inode,
+int ima_get_action(const struct mnt_idmap *idmap, struct inode *inode,
 		   const struct cred *cred, struct lsm_prop *prop, int mask,
 		   enum ima_hooks func, int *pcr,
 		   struct ima_template_desc **template_desc,
@@ -437,7 +437,7 @@ void ima_store_measurement(struct ima_iint_cache *iint, struct file *file,
 			   struct evm_ima_xattr_data *xattr_value,
 			   int xattr_len, const struct modsig *modsig, int pcr,
 			   struct ima_template_desc *template_desc);
-int process_buffer_measurement(struct mnt_idmap *idmap,
+int process_buffer_measurement(const struct mnt_idmap *idmap,
 			       struct inode *inode, const void *buf, int size,
 			       const char *eventname, enum ima_hooks func,
 			       int pcr, const char *func_data,
@@ -454,7 +454,7 @@ void ima_free_template_entry(struct ima_template_entry *entry);
 const char *ima_d_path(const struct path *path, char **pathbuf, char *filename);
 
 /* IMA policy related functions */
-int ima_match_policy(struct mnt_idmap *idmap, struct inode *inode,
+int ima_match_policy(const struct mnt_idmap *idmap, struct inode *inode,
 		     const struct cred *cred, struct lsm_prop *prop,
 		     enum ima_hooks func, int mask, int flags, int *pcr,
 		     struct ima_template_desc **template_desc,
@@ -489,7 +489,7 @@ int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint,
 			     struct evm_ima_xattr_data *xattr_value,
 			     int xattr_len, const struct modsig *modsig,
 			     bool bprm_is_check);
-int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode,
+int ima_must_appraise(const struct mnt_idmap *idmap, struct inode *inode,
 		      int mask, enum ima_hooks func);
 void ima_update_xattr(struct ima_iint_cache *iint, struct file *file);
 enum integrity_status ima_get_cache_status(struct ima_iint_cache *iint,
@@ -519,7 +519,7 @@ static inline int ima_appraise_measurement(enum ima_hooks func,
 	return INTEGRITY_UNKNOWN;
 }
 
-static inline int ima_must_appraise(struct mnt_idmap *idmap,
+static inline int ima_must_appraise(const struct mnt_idmap *idmap,
 				    struct inode *inode, int mask,
 				    enum ima_hooks func)
 {
diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c
index 122d127e108dcc..45701f04ba62fc 100644
--- a/security/integrity/ima/ima_api.c
+++ b/security/integrity/ima/ima_api.c
@@ -116,7 +116,7 @@ int ima_store_template(struct ima_template_entry *entry,
 		if (result < 0) {
 			integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode,
 					    template_name, op,
-					    audit_cause, result, 0);
+					    audit_cause, result, 0, result);
 			return result;
 		}
 	}
@@ -159,7 +159,7 @@ void ima_add_violation(struct file *file, const unsigned char *filename,
 		ima_free_template_entry(entry);
 err_out:
 	integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
-			    op, cause, result, 0);
+			    op, cause, result, 0, result);
 }
 
 /**
@@ -188,7 +188,7 @@ void ima_add_violation(struct file *file, const unsigned char *filename,
  * Returns IMA_MEASURE, IMA_APPRAISE mask.
  *
  */
-int ima_get_action(struct mnt_idmap *idmap, struct inode *inode,
+int ima_get_action(const struct mnt_idmap *idmap, struct inode *inode,
 		   const struct cred *cred, struct lsm_prop *prop, int mask,
 		   enum ima_hooks func, int *pcr,
 		   struct ima_template_desc **template_desc,
@@ -330,7 +330,7 @@ int ima_collect_measurement(struct ima_iint_cache *iint, struct file *file,
 
 		integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode,
 				    filename.name.name, "collect_data",
-				    audit_cause, result, 0);
+				    audit_cause, result, 0, result);
 
 		release_dentry_name_snapshot(&filename);
 	}
@@ -383,7 +383,7 @@ void ima_store_measurement(struct ima_iint_cache *iint, struct file *file,
 	result = ima_alloc_init_template(&event_data, &entry, template_desc);
 	if (result < 0) {
 		integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
-				    op, audit_cause, result, 0);
+				    op, audit_cause, result, 0, result);
 		return;
 	}
 
diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c
index b280488e15fcb0..134174d2e3f0ec 100644
--- a/security/integrity/ima/ima_appraise.c
+++ b/security/integrity/ima/ima_appraise.c
@@ -71,7 +71,7 @@ bool is_ima_appraise_enabled(void)
  *
  * Return 1 to appraise or hash
  */
-int ima_must_appraise(struct mnt_idmap *idmap, struct inode *inode,
+int ima_must_appraise(const struct mnt_idmap *idmap, struct inode *inode,
 		      int mask, enum ima_hooks func)
 {
 	struct lsm_prop prop;
@@ -564,7 +564,7 @@ int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint,
 		status = INTEGRITY_FAIL;
 		cause = "unverifiable-signature";
 		integrity_audit_msg(audit_msgno, inode, filename,
-				    op, cause, rc, 0);
+				    op, cause, rc, 0, rc < 0 ? rc : 0);
 	} else if (status != INTEGRITY_PASS) {
 		/* Fix mode, but don't replace file signatures. */
 		if ((ima_appraise & IMA_APPRAISE_FIX) && !try_modsig &&
@@ -589,7 +589,7 @@ int ima_appraise_measurement(enum ima_hooks func, struct ima_iint_cache *iint,
 		}
 
 		integrity_audit_msg(audit_msgno, inode, filename,
-				    op, cause, rc, 0);
+				    op, cause, rc, 0, rc < 0 ? rc : 0);
 	} else {
 		ima_cache_flags(iint, func);
 	}
@@ -634,7 +634,7 @@ void ima_update_xattr(struct ima_iint_cache *iint, struct file *file)
  * This function is called from notify_change(), which expects the caller
  * to lock the inode's i_mutex.
  */
-static void ima_inode_post_setattr(struct mnt_idmap *idmap,
+static void ima_inode_post_setattr(const struct mnt_idmap *idmap,
 				   struct dentry *dentry, int ia_valid)
 {
 	struct inode *inode = d_backing_inode(dentry);
@@ -752,14 +752,14 @@ static int validate_hash_algo(struct dentry *dentry,
 		path = NULL;
 
 	integrity_audit_msg(AUDIT_INTEGRITY_DATA, d_inode(dentry), path,
-			    "set_data", errmsg, -EACCES, 0);
+			    "set_data", errmsg, -EACCES, 0, -EACCES);
 
 	kfree(pathbuf);
 
 	return -EACCES;
 }
 
-static int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int ima_inode_setxattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			      const char *xattr_name, const void *xattr_value,
 			      size_t xattr_value_len, int flags)
 {
@@ -792,7 +792,7 @@ static int ima_inode_setxattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	return result;
 }
 
-static int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+static int ima_inode_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 			     const char *acl_name, struct posix_acl *kacl)
 {
 	if (evm_revalidate_status(acl_name))
@@ -801,7 +801,7 @@ static int ima_inode_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
 	return 0;
 }
 
-static int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int ima_inode_removexattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 				 const char *xattr_name)
 {
 	int result, digsig = -1;
@@ -817,7 +817,7 @@ static int ima_inode_removexattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	return result;
 }
 
-static int ima_inode_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry,
+static int ima_inode_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry,
 				const char *acl_name)
 {
 	return ima_inode_set_acl(idmap, dentry, acl_name, NULL);
diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c
index 2a0bca55431619..3b8174a91a3451 100644
--- a/security/integrity/ima/ima_fs.c
+++ b/security/integrity/ima/ima_fs.c
@@ -592,11 +592,11 @@ static ssize_t ima_write_policy(struct file *file, const char __user *buf,
 	if (data[0] == '/') {
 		result = ima_read_policy(data);
 	} else if (ima_appraise & IMA_APPRAISE_POLICY) {
+		result = -EACCES;
 		pr_err("signed policy file (specified as an absolute pathname) required\n");
 		integrity_audit_msg(AUDIT_INTEGRITY_STATUS, NULL, NULL,
 				    "policy_update", "signed policy required",
-				    1, 0);
-		result = -EACCES;
+				    result, 0, result);
 	} else {
 		ima_measure_raw_policy(data, datalen);
 		result = ima_parse_add_rule(data);
@@ -745,7 +745,8 @@ static int ima_release_policy(struct inode *inode, struct file *file)
 
 	pr_info("policy update %s\n", cause);
 	integrity_audit_msg(AUDIT_INTEGRITY_STATUS, NULL, NULL,
-			    "policy_update", cause, !valid_policy, 0);
+			    "policy_update", cause, !valid_policy, 0,
+			    !valid_policy ? -EINVAL : 0);
 
 	if (!valid_policy) {
 		ima_delete_rules();
diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c
index d53f4d89a53e56..7a76e268626748 100644
--- a/security/integrity/ima/ima_init.c
+++ b/security/integrity/ima/ima_init.c
@@ -103,7 +103,7 @@ static int __init ima_add_boot_aggregate(void)
 	return 0;
 err_out:
 	integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, filename, op,
-			    audit_cause, result, 0);
+			    audit_cause, result, 0, result);
 	return result;
 }
 
diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c
index ab1e53b3210d3a..fed80da3ee096e 100644
--- a/security/integrity/ima/ima_main.c
+++ b/security/integrity/ima/ima_main.c
@@ -119,7 +119,7 @@ static int mmap_violation_check(enum ima_hooks func, struct file *file,
 			*pathname = ima_d_path(&file->f_path, pathbuf,
 					       filename);
 		integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode, *pathname,
-				    "mmap_file", "mmapped_writers", rc, 0);
+				    "mmap_file", "mmapped_writers", rc, 0, rc);
 	}
 	return rc;
 }
@@ -466,7 +466,7 @@ static int process_measurement(struct file *file, const struct cred *cred,
 
 		integrity_audit_msg(AUDIT_INTEGRITY_DATA, file_inode(file),
 				    pathname, "collect_data",
-				    "denied-hash-algorithm", rc, 0);
+				    "denied-hash-algorithm", rc, 0, rc);
 	}
 out_locked:
 	if ((mask & MAY_WRITE) && test_bit(IMA_DIGSIG, &iint->atomic_flags) &&
@@ -579,7 +579,8 @@ static int ima_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot,
 	file = vma->vm_file;
 	pathname = ima_d_path(&file->f_path, &pathbuf, filename);
 	integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode, pathname,
-			    "collect_data", "failed-mprotect", result, 0);
+			    "collect_data", "failed-mprotect", result, 0,
+			    result);
 	if (pathbuf)
 		__putname(pathbuf);
 
@@ -846,7 +847,7 @@ EXPORT_SYMBOL_GPL(ima_inode_hash);
  * Skip calling process_measurement(), but indicate which newly, created
  * tmpfiles are in policy.
  */
-static void ima_post_create_tmpfile(struct mnt_idmap *idmap,
+static void ima_post_create_tmpfile(const struct mnt_idmap *idmap,
 				    struct inode *inode)
 
 {
@@ -879,7 +880,7 @@ static void ima_post_create_tmpfile(struct mnt_idmap *idmap,
  * Mark files created via the mknodat syscall as new, so that the
  * file data can be written later.
  */
-static void ima_post_path_mknod(struct mnt_idmap *idmap, struct dentry *dentry)
+static void ima_post_path_mknod(const struct mnt_idmap *idmap, struct dentry *dentry)
 {
 	struct ima_iint_cache *iint;
 	struct inode *inode = dentry->d_inode;
@@ -1095,7 +1096,7 @@ static int ima_post_load_data(char *buf, loff_t size,
  * has been written to the passed location but not added to a measurement entry,
  * a negative value otherwise.
  */
-int process_buffer_measurement(struct mnt_idmap *idmap,
+int process_buffer_measurement(const struct mnt_idmap *idmap,
 			       struct inode *inode, const void *buf, int size,
 			       const char *eventname, enum ima_hooks func,
 			       int pcr, const char *func_data,
@@ -1195,9 +1196,9 @@ int process_buffer_measurement(struct mnt_idmap *idmap,
 
 out:
 	if (ret < 0)
-		integrity_audit_message(AUDIT_INTEGRITY_PCR, NULL, eventname,
-					func_measure_str(func),
-					audit_cause, ret, 0, ret);
+		integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, eventname,
+				    func_measure_str(func),
+				    audit_cause, ret, 0, ret);
 
 	return ret;
 }
diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c
index 68d9a5e6c232ea..2dc1d93aa768e1 100644
--- a/security/integrity/ima/ima_policy.c
+++ b/security/integrity/ima/ima_policy.c
@@ -580,7 +580,7 @@ static bool ima_match_rule_data(struct ima_rule_entry *rule,
  * Returns true on rule match, false on failure.
  */
 static bool ima_match_rules(struct ima_rule_entry *rule,
-			    struct mnt_idmap *idmap,
+			    const struct mnt_idmap *idmap,
 			    struct inode *inode, const struct cred *cred,
 			    struct lsm_prop *prop, enum ima_hooks func, int mask,
 			    const char *func_data)
@@ -762,7 +762,7 @@ static int get_subaction(struct ima_rule_entry *rule, enum ima_hooks func)
  * list when walking it.  Reads are many orders of magnitude more numerous
  * than writes so ima_match_policy() is classical RCU candidate.
  */
-int ima_match_policy(struct mnt_idmap *idmap, struct inode *inode,
+int ima_match_policy(const struct mnt_idmap *idmap, struct inode *inode,
 		     const struct cred *cred, struct lsm_prop *prop,
 		     enum ima_hooks func, int mask, int flags, int *pcr,
 		     struct ima_template_desc **template_desc,
@@ -1992,7 +1992,8 @@ ssize_t ima_parse_add_rule(char *rule)
 	entry = kzalloc_obj(*entry);
 	if (!entry) {
 		integrity_audit_msg(AUDIT_INTEGRITY_STATUS, NULL,
-				    NULL, op, "-ENOMEM", -ENOMEM, audit_info);
+				    NULL, op, "-ENOMEM", -ENOMEM, audit_info,
+				    -ENOMEM);
 		return -ENOMEM;
 	}
 
@@ -2003,7 +2004,7 @@ ssize_t ima_parse_add_rule(char *rule)
 		ima_free_rule(entry);
 		integrity_audit_msg(AUDIT_INTEGRITY_STATUS, NULL,
 				    NULL, op, "invalid-policy", result,
-				    audit_info);
+				    audit_info, result);
 		return result;
 	}
 
@@ -2421,7 +2422,7 @@ void ima_measure_loaded_policy(void)
 	rule = kmalloc(rule_len, GFP_KERNEL);
 	if (!rule) {
 		integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, event_name,
-				    op, "ENOMEM", result, 0);
+				    op, "ENOMEM", result, 0, result);
 		return;
 	}
 
@@ -2440,7 +2441,7 @@ void ima_measure_loaded_policy(void)
 			result = -E2BIG;
 			integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL,
 					    event_name, op, "rule_length",
-					    result, 0);
+					    result, 0, result);
 			rcu_read_unlock();
 			goto free_rule;
 		}
@@ -2454,7 +2455,7 @@ void ima_measure_loaded_policy(void)
 	file.buf = kmalloc(file_len, GFP_KERNEL);
 	if (!file.buf) {
 		integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL, event_name,
-				    op, "ENOMEM", result, 0);
+				    op, "ENOMEM", result, 0, result);
 		goto free_rule;
 	}
 
diff --git a/security/integrity/ima/ima_queue.c b/security/integrity/ima/ima_queue.c
index 0f1b7e4113c4f0..67782c004cdf71 100644
--- a/security/integrity/ima/ima_queue.c
+++ b/security/integrity/ima/ima_queue.c
@@ -291,7 +291,7 @@ int ima_add_template_entry(struct ima_template_entry *entry, int violation,
 out:
 	mutex_unlock(&ima_extend_list_mutex);
 	integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
-			    op, audit_cause, result, audit_info);
+			    op, audit_cause, result, audit_info, result);
 	return result;
 }
 
diff --git a/security/integrity/ima/ima_queue_keys.c b/security/integrity/ima/ima_queue_keys.c
index b5ed33cbb272fd..8adc8863ad5a54 100644
--- a/security/integrity/ima/ima_queue_keys.c
+++ b/security/integrity/ima/ima_queue_keys.c
@@ -90,10 +90,10 @@ static struct ima_key_entry *ima_alloc_key_entry(struct key *keyring,
 
 out:
 	if (rc) {
-		integrity_audit_message(AUDIT_INTEGRITY_PCR, NULL,
-					keyring->description,
-					func_measure_str(KEY_CHECK),
-					audit_cause, rc, 0, rc);
+		integrity_audit_msg(AUDIT_INTEGRITY_PCR, NULL,
+				    keyring->description,
+				    func_measure_str(KEY_CHECK),
+				    audit_cause, rc, 0, rc);
 		ima_free_key_entry(entry);
 		entry = NULL;
 	}
diff --git a/security/integrity/ima/ima_template_lib.c b/security/integrity/ima/ima_template_lib.c
index 8a89236f926c10..9bd80ca1159a2f 100644
--- a/security/integrity/ima/ima_template_lib.c
+++ b/security/integrity/ima/ima_template_lib.c
@@ -392,7 +392,7 @@ int ima_eventdigest_init(struct ima_event_data *event_data,
 	if (result) {
 		integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode,
 				    event_data->filename, "collect_data",
-				    "failed", result, 0);
+				    "failed", result, 0, result);
 		return result;
 	}
 	cur_digest = hash_hdr->digest;
diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h
index 0c581c03c5da72..52c5ec881cca46 100644
--- a/security/integrity/integrity.h
+++ b/security/integrity/integrity.h
@@ -218,12 +218,8 @@ static inline void evm_load_x509(void)
 /* declarations */
 void integrity_audit_msg(int audit_msgno, struct inode *inode,
 			 const unsigned char *fname, const char *op,
-			 const char *cause, int result, int info);
-
-void integrity_audit_message(int audit_msgno, struct inode *inode,
-			     const unsigned char *fname, const char *op,
-			     const char *cause, int result, int info,
-			     int errno);
+			 const char *cause, int result, int info,
+			 int errno);
 
 static inline struct audit_buffer *
 integrity_audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, int type)
@@ -235,15 +231,7 @@ integrity_audit_log_start(struct audit_context *ctx, gfp_t gfp_mask, int type)
 static inline void integrity_audit_msg(int audit_msgno, struct inode *inode,
 				       const unsigned char *fname,
 				       const char *op, const char *cause,
-				       int result, int info)
-{
-}
-
-static inline void integrity_audit_message(int audit_msgno,
-					   struct inode *inode,
-					   const unsigned char *fname,
-					   const char *op, const char *cause,
-					   int result, int info, int errno)
+				       int result, int info, int errno)
 {
 }
 
diff --git a/security/integrity/integrity_audit.c b/security/integrity/integrity_audit.c
index d8d9e5ff1cd22b..22106aaeb0df21 100644
--- a/security/integrity/integrity_audit.c
+++ b/security/integrity/integrity_audit.c
@@ -27,16 +27,8 @@ __setup("integrity_audit=", integrity_audit_setup);
 
 void integrity_audit_msg(int audit_msgno, struct inode *inode,
 			 const unsigned char *fname, const char *op,
-			 const char *cause, int result, int audit_info)
-{
-	integrity_audit_message(audit_msgno, inode, fname, op, cause,
-				result, audit_info, 0);
-}
-
-void integrity_audit_message(int audit_msgno, struct inode *inode,
-			     const unsigned char *fname, const char *op,
-			     const char *cause, int result, int audit_info,
-			     int errno)
+			 const char *cause, int result, int audit_info,
+			 int errno)
 {
 	struct audit_buffer *ab;
 	char name[TASK_COMM_LEN];
diff --git a/security/ipe/eval.c b/security/ipe/eval.c
index 21439c5be33648..4a7be96c84f030 100644
--- a/security/ipe/eval.c
+++ b/security/ipe/eval.c
@@ -134,10 +134,14 @@ static bool evaluate_boot_verified(const struct ipe_eval_ctx *const ctx)
 static bool evaluate_dmv_roothash(const struct ipe_eval_ctx *const ctx,
 				  struct ipe_prop *p)
 {
-	return !!ctx->ipe_bdev &&
-	       !!ctx->ipe_bdev->root_hash &&
-	       ipe_digest_eval(p->value,
-			       ctx->ipe_bdev->root_hash);
+	const struct digest_info *root_hash;
+
+	if (!ctx->ipe_bdev)
+		return false;
+
+	root_hash = rcu_dereference(ctx->ipe_bdev->root_hash);
+
+	return root_hash && ipe_digest_eval(p->value, root_hash);
 }
 #else
 static bool evaluate_dmv_roothash(const struct ipe_eval_ctx *const ctx,
diff --git a/security/ipe/eval.h b/security/ipe/eval.h
index fef65a36468cb8..b26d1147d0708b 100644
--- a/security/ipe/eval.h
+++ b/security/ipe/eval.h
@@ -27,7 +27,7 @@ struct ipe_bdev {
 #ifdef CONFIG_IPE_PROP_DM_VERITY_SIGNATURE
 	bool dm_verity_signed;
 #endif /* CONFIG_IPE_PROP_DM_VERITY_SIGNATURE */
-	struct digest_info *root_hash;
+	struct digest_info __rcu *root_hash;
 };
 #endif /* CONFIG_IPE_PROP_DM_VERITY */
 
diff --git a/security/ipe/fs.c b/security/ipe/fs.c
index 076c111c85c869..847a76afb93d7d 100644
--- a/security/ipe/fs.c
+++ b/security/ipe/fs.c
@@ -159,18 +159,16 @@ static ssize_t new_policy(struct file *f, const char __user *data,
 	}
 
 	rc = ipe_new_policyfs_node(p);
-	if (rc)
-		goto out;
 
 out:
 	kfree(copy);
 	if (rc < 0) {
 		ipe_free_policy(p);
 		ipe_audit_policy_load(ERR_PTR(rc));
-	} else {
-		ipe_audit_policy_load(p);
+		return rc;
 	}
-	return (rc < 0) ? rc : len;
+
+	return len;
 }
 
 static const struct file_operations np_fops = {
diff --git a/security/ipe/hooks.c b/security/ipe/hooks.c
index 0ae54a880405a9..d878b52ceec847 100644
--- a/security/ipe/hooks.c
+++ b/security/ipe/hooks.c
@@ -9,6 +9,7 @@
 #include <linux/binfmts.h>
 #include <linux/mman.h>
 #include <linux/blk_types.h>
+#include <linux/rcupdate.h>
 
 #include "ipe.h"
 #include "hooks.h"
@@ -232,7 +233,20 @@ void ipe_bdev_free_security(struct block_device *bdev)
 {
 	struct ipe_bdev *blob = ipe_bdev(bdev);
 
-	ipe_digest_free(blob->root_hash);
+	ipe_digest_free(rcu_access_pointer(blob->root_hash));
+}
+
+static void ipe_set_dmverity_roothash(struct ipe_bdev *blob,
+				      struct digest_info *info)
+{
+	struct digest_info *old;
+
+	/* Protected by device-mapper's md->suspend_lock */
+	old = rcu_replace_pointer(blob->root_hash, info, true);
+	if (old) {
+		synchronize_rcu();
+		ipe_digest_free(old);
+	}
 }
 
 #ifdef CONFIG_IPE_PROP_DM_VERITY_SIGNATURE
@@ -280,8 +294,7 @@ int ipe_bdev_setintegrity(struct block_device *bdev, enum lsm_integrity_type typ
 		return -EINVAL;
 
 	if (!value) {
-		ipe_digest_free(blob->root_hash);
-		blob->root_hash = NULL;
+		ipe_set_dmverity_roothash(blob, NULL);
 
 		return 0;
 	}
@@ -301,8 +314,7 @@ int ipe_bdev_setintegrity(struct block_device *bdev, enum lsm_integrity_type typ
 
 	info->digest_len = digest->digest_len;
 
-	ipe_digest_free(blob->root_hash);
-	blob->root_hash = info;
+	ipe_set_dmverity_roothash(blob, info);
 
 	return 0;
 err:
diff --git a/security/ipe/policy_fs.c b/security/ipe/policy_fs.c
index 9d92d8a14b13dd..a7aeb57483c6a7 100644
--- a/security/ipe/policy_fs.c
+++ b/security/ipe/policy_fs.c
@@ -481,6 +481,9 @@ int ipe_new_policyfs_node(struct ipe_policy *p)
 	inode_lock(root);
 	p->policyfs = policyfs;
 	root->i_private = p;
+	/* Only audit signed policies from userspace */
+	if (p->pkcs7)
+		ipe_audit_policy_load(p);
 	inode_unlock(root);
 
 	return 0;
diff --git a/security/keys/persistent.c b/security/keys/persistent.c
index 97af230aa4b22b..bdab9953932613 100644
--- a/security/keys/persistent.c
+++ b/security/keys/persistent.c
@@ -33,25 +33,31 @@ static int key_create_persistent_register(struct user_namespace *ns)
 }
 
 /*
- * Create the persistent keyring for the specified user.
+ * Get or create the persistent keyring for the specified user.
  *
  * Called with the namespace's sem locked for writing.
+ *
+ * Returns a boolean indicating whether the keyring already existed,
+ * or a negative error. On success, *persistent_ref holds a reference
+ * to the keyring.
  */
-static key_ref_t key_create_persistent(struct user_namespace *ns, kuid_t uid,
-				       struct keyring_index_key *index_key)
+static int key_get_or_create_persistent(struct user_namespace *ns, kuid_t uid,
+					struct keyring_index_key *index_key,
+					key_ref_t *persistent_ref)
 {
 	struct key *persistent;
-	key_ref_t reg_ref, persistent_ref;
+	key_ref_t reg_ref;
 
 	if (!ns->persistent_keyring_register) {
-		long err = key_create_persistent_register(ns);
+		int err = key_create_persistent_register(ns);
+
 		if (err < 0)
-			return ERR_PTR(err);
+			return err;
 	} else {
 		reg_ref = make_key_ref(ns->persistent_keyring_register, true);
-		persistent_ref = find_key_to_update(reg_ref, index_key);
-		if (persistent_ref)
-			return persistent_ref;
+		*persistent_ref = find_key_to_update(reg_ref, index_key);
+		if (*persistent_ref)
+			return 1;
 	}
 
 	persistent = keyring_alloc(index_key->description,
@@ -61,9 +67,10 @@ static key_ref_t key_create_persistent(struct user_namespace *ns, kuid_t uid,
 				   KEY_ALLOC_NOT_IN_QUOTA, NULL,
 				   ns->persistent_keyring_register);
 	if (IS_ERR(persistent))
-		return ERR_CAST(persistent);
+		return PTR_ERR(persistent);
 
-	return make_key_ref(persistent, true);
+	*persistent_ref = make_key_ref(persistent, true);
+	return 0;
 }
 
 /*
@@ -78,6 +85,7 @@ static long key_get_persistent(struct user_namespace *ns, kuid_t uid,
 	key_ref_t reg_ref, persistent_ref;
 	char buf[32];
 	long ret;
+	bool created = false;
 
 	/* Look in the register if it exists */
 	memset(&index_key, 0, sizeof(index_key));
@@ -100,23 +108,24 @@ static long key_get_persistent(struct user_namespace *ns, kuid_t uid,
 	 * also need to create the register.
 	 */
 	down_write(&ns->keyring_sem);
-	persistent_ref = key_create_persistent(ns, uid, &index_key);
+	ret = key_get_or_create_persistent(ns, uid, &index_key,
+					   &persistent_ref);
 	up_write(&ns->keyring_sem);
-	if (!IS_ERR(persistent_ref))
-		goto found;
-
-	return PTR_ERR(persistent_ref);
+	if (ret < 0)
+		return ret;
+	created = ret == 0;
 
 found:
+	persistent = key_ref_to_ptr(persistent_ref);
 	ret = key_task_permission(persistent_ref, current_cred(), KEY_NEED_LINK);
-	if (ret == 0) {
-		persistent = key_ref_to_ptr(persistent_ref);
+	if (ret == 0)
 		ret = key_link(key_ref_to_ptr(dest_ref), persistent);
-		if (ret == 0) {
-			key_set_timeout(persistent, persistent_keyring_expiry);
-			ret = persistent->serial;
-		}
-	}
+
+	if (ret == 0 || created)
+		key_set_timeout(persistent, persistent_keyring_expiry);
+
+	if (ret == 0)
+		ret = persistent->serial;
 
 	key_ref_put(persistent_ref);
 	return ret;
diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c
index a63c46bb2d1480..44358388e3959d 100644
--- a/security/keys/process_keys.c
+++ b/security/keys/process_keys.c
@@ -22,6 +22,9 @@
 /* Session keyring create vs join semaphore */
 static DEFINE_MUTEX(key_session_mutex);
 
+/* BPF keyring reachable through KEY_SPEC_BPF_KEYRING */
+static struct key *bpf_keyring __ro_after_init;
+
 /* The root user's tracking struct */
 struct key_user root_key_user = {
 	.usage		= REFCOUNT_INIT(3),
@@ -590,6 +593,20 @@ bool lookup_user_key_possessed(const struct key *key,
 	return key == match_data->raw_data;
 }
 
+/**
+ * key_register_bpf_keyring - Publish the BPF keyring for KEY_SPEC_BPF_KEYRING
+ * @keyring: The keyring to publish
+ *
+ * Make @keyring reachable by userspace through the KEY_SPEC_BPF_KEYRING
+ * special key ID, so that provisioning it does not require scraping its
+ * serial out of /proc/keys first. Called once, from an initcall, and never
+ * undone.
+ */
+void key_register_bpf_keyring(struct key *keyring)
+{
+	bpf_keyring = keyring;
+}
+
 /*
  * Look up a key ID given us by userspace with a given permissions mask to get
  * the key it refers to.
@@ -741,6 +758,14 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags,
 		key_ref = make_key_ref(key, 1);
 		break;
 
+	case KEY_SPEC_BPF_KEYRING:
+		key = bpf_keyring;
+		if (!key)
+			goto error;
+		__key_get(key);
+		key_ref = make_key_ref(key, 0);
+		break;
+
 	default:
 		key_ref = ERR_PTR(-EINVAL);
 		if (id < 1)
diff --git a/security/keys/request_key.c b/security/keys/request_key.c
index fa2bb9f2f53812..e6ba2d0543990d 100644
--- a/security/keys/request_key.c
+++ b/security/keys/request_key.c
@@ -9,7 +9,7 @@
 
 #include <linux/export.h>
 #include <linux/sched.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/err.h>
 #include <linux/keyctl.h>
 #include <linux/slab.h>
diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c
index bf0bf7f3697052..4ad277cdd23345 100644
--- a/security/keys/trusted-keys/trusted_tpm1.c
+++ b/security/keys/trusted-keys/trusted_tpm1.c
@@ -101,7 +101,7 @@ static inline void dump_tpm_buf(unsigned char *buf)
 static int TSS_rawhmac(unsigned char *digest, const unsigned char *key,
 		       unsigned int keylen, ...)
 {
-	struct hmac_sha1_ctx hmac_ctx;
+	struct hmac_sha1_ctx hmac_ctx __cleanup(hmac_sha1_zeroize_ctx);
 	va_list argp;
 	unsigned int dlen;
 	unsigned char *data;
@@ -923,8 +923,10 @@ static int trusted_tpm_unseal(struct trusted_key_payload *p, char *datablob)
 		ret = tpm2_unseal_trusted(chip, p, options);
 	else
 		ret = key_unseal(p, options);
-	if (ret < 0)
+	if (ret < 0) {
 		pr_info("key_unseal failed (%d)\n", ret);
+		goto out;
+	}
 
 	if (options->pcrlock) {
 		ret = pcrlock(options->pcrlock);
diff --git a/security/keys/trusted-keys/trusted_tpm2.c b/security/keys/trusted-keys/trusted_tpm2.c
index 01f18bb370477e..c176ceba7859f5 100644
--- a/security/keys/trusted-keys/trusted_tpm2.c
+++ b/security/keys/trusted-keys/trusted_tpm2.c
@@ -115,7 +115,7 @@ static int tpm2_key_decode(struct trusted_key_payload *payload,
 	if (ctx.priv_len + ctx.pub_len > MAX_BLOB_SIZE)
 		return -EINVAL;
 
-	blob = kmalloc(ctx.priv_len + ctx.pub_len + 4, GFP_KERNEL);
+	blob = kmalloc(ctx.priv_len + ctx.pub_len, GFP_KERNEL);
 	if (!blob)
 		return -ENOMEM;
 
@@ -413,6 +413,8 @@ static int tpm2_load_cmd(struct tpm_chip *chip,
 	public_len = get_unaligned_be16(blob + 2 + private_len);
 	if (private_len + 2 + public_len + 2 > blob_len)
 		return -E2BIG;
+	if (public_len < 8)
+		return -E2BIG;
 
 	pub = blob + 2 + private_len + 2;
 	/* key attributes are always at offset 4 */
diff --git a/security/landlock/access.h b/security/landlock/access.h
index bbbb41f41147cf..f843835851d014 100644
--- a/security/landlock/access.h
+++ b/security/landlock/access.h
@@ -61,6 +61,10 @@ union access_masks_all {
 static_assert(sizeof(typeof_member(union access_masks_all, masks)) ==
 	      sizeof(typeof_member(union access_masks_all, all)));
 
+#define _LANDLOCK_LAYER_MASK_PADDING                              \
+	(BITS_PER_TYPE(access_mask_t) - LANDLOCK_NUM_ACCESS_MAX - \
+	 IS_ENABLED(CONFIG_SECURITY_LANDLOCK_LOG))
+
 /**
  * struct layer_mask - The access rights and rule flags for a layer.
  *
@@ -81,6 +85,10 @@ struct layer_mask {
 	 */
 	access_mask_t quiet : 1;
 #endif /* CONFIG_SECURITY_LANDLOCK_LOG */
+	/**
+	 * @__pad: Padding for the compiler's bitfield initialization.
+	 */
+	access_mask_t __pad : _LANDLOCK_LAYER_MASK_PADDING;
 } __packed __aligned(sizeof(access_mask_t));
 
 /*
diff --git a/security/landlock/domain.h b/security/landlock/domain.h
index 5ce2f91488d5d0..caa3d19d2c4321 100644
--- a/security/landlock/domain.h
+++ b/security/landlock/domain.h
@@ -243,7 +243,8 @@ struct landlock_domain {
 			 * overlapping access rights.  These layers are set once
 			 * and never changed for the lifetime of the domain.
 			 */
-			struct access_masks handled_masks[];
+			struct access_masks
+				handled_masks[] __counted_by(num_layers);
 		};
 	};
 };
diff --git a/security/landlock/fs.c b/security/landlock/fs.c
index 330a1871bf94a6..cab43892ec2f0d 100644
--- a/security/landlock/fs.c
+++ b/security/landlock/fs.c
@@ -356,14 +356,14 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset,
 	 * held for BTF consistency (enforced by lockdep_assert_held in
 	 * TP_fast_assign).
 	 */
-	if (!err && trace_landlock_add_rule_fs_enabled()) {
+	if (!err && trace_landlock_add_rule_path_beneath_enabled()) {
 		char *buffer __free(__putname) = __getname();
 		const char *pathname =
 			buffer ? resolve_path_for_trace(path, buffer) :
 				 "<no_mem>";
 
-		trace_landlock_add_rule_fs(ruleset, access_rights, path,
-					   pathname);
+		trace_landlock_add_rule_path_beneath(
+			ruleset, flags, access_rights, path, pathname);
 	}
 	mutex_unlock(&ruleset->lock);
 
@@ -423,8 +423,8 @@ static bool unmask_layers_fs(const struct landlock_domain *const domain,
 
 	ret = landlock_unmask_layers(domain, id, masks, &rule);
 	if (rule)
-		trace_landlock_check_rule_fs(domain, rule, access_request,
-					     dentry);
+		trace_landlock_check_rule_inode(domain, rule, access_request,
+						dentry);
 	return ret;
 }
 
diff --git a/security/landlock/log.h b/security/landlock/log.h
index e0a6e44f3ddd32..faa30e26e42a38 100644
--- a/security/landlock/log.h
+++ b/security/landlock/log.h
@@ -15,6 +15,8 @@
 
 struct landlock_cred_security;
 struct landlock_hierarchy;
+struct sockaddr;
+struct task_struct;
 
 enum landlock_request_type {
 	LANDLOCK_REQUEST_PTRACE = 1,
@@ -25,6 +27,31 @@ enum landlock_request_type {
 	LANDLOCK_REQUEST_SCOPE_SIGNAL,
 };
 
+struct landlock_blockers {
+	access_mask_t access;
+	enum landlock_request_type type;
+};
+
+#ifdef CONFIG_TRACEPOINTS
+
+struct landlock_net_trace {
+	const struct sockaddr *address;
+	int addrlen;
+	u16 socket_family;
+};
+
+struct landlock_ptrace_trace {
+	u64 tracee_domain_id;
+	const struct task_struct *tracer;
+};
+
+struct landlock_signal_trace {
+	u64 target_domain_id;
+	int signal;
+};
+
+#endif /* CONFIG_TRACEPOINTS */
+
 /*
  * We should be careful to only use a variable of this type for
  * landlock_log_denial().  This way, the compiler can remove it entirely if
@@ -52,13 +79,24 @@ struct landlock_request {
 	deny_masks_t deny_masks;
 	optional_access_t quiet_optional_accesses;
 
-	/*
-	 * Other-party domain ID for a relational (scope/ptrace) denial, or 0 if
-	 * that party is unsandboxed.  An ID, not a pointer: the other task can
-	 * replace its credential and free the domain it referenced.  Trace path
-	 * only; audit ignores it.
-	 */
-	u64 other_domain_id;
+#ifdef CONFIG_TRACEPOINTS
+	union {
+		/*
+		 * Other-party domain ID for an abstract UNIX socket scope
+		 * denial, or 0 if that party is unsandboxed.  Store an ID, not
+		 * a pointer: the other task can replace its credential and free
+		 * the domain it referenced.
+		 */
+		u64 other_domain_id;
+
+		/* Synchronous context for a network denial. */
+		const struct landlock_net_trace *trace_net;
+		/* Synchronous context for a ptrace denial. */
+		const struct landlock_ptrace_trace *trace_ptrace;
+		/* Synchronous context for a signal denial. */
+		const struct landlock_signal_trace *trace_signal;
+	};
+#endif /* CONFIG_TRACEPOINTS */
 };
 
 #ifdef CONFIG_SECURITY_LANDLOCK_LOG
diff --git a/security/landlock/net.c b/security/landlock/net.c
index 8f2aaac54b3347..6fe0dbde3b787d 100644
--- a/security/landlock/net.c
+++ b/security/landlock/net.c
@@ -47,7 +47,8 @@ int landlock_append_net_rule(struct landlock_ruleset *const ruleset,
 	 * TP_fast_assign).
 	 */
 	if (!err)
-		trace_landlock_add_rule_net(ruleset, access_rights, port);
+		trace_landlock_add_rule_net_port(ruleset, flags, access_rights,
+						 port);
 	mutex_unlock(&ruleset->lock);
 
 	return err;
@@ -63,7 +64,7 @@ static bool unmask_layers_net(const struct landlock_domain *const domain,
 
 	ret = landlock_unmask_layers(domain, id, masks, &rule);
 	if (rule)
-		trace_landlock_check_rule_net(
+		trace_landlock_check_rule_net_port(
 			domain, rule, access_request,
 			ntohs((__force __be16)id.key.data));
 	return ret;
@@ -92,7 +93,7 @@ static int current_check_access_socket(struct socket *const sock,
 		return 0;
 
 	/* Checks for minimal header length to safely read sa_family. */
-	if (addrlen < offsetofend(typeof(*address), sa_family))
+	if (addrlen < (int)offsetofend(typeof(*address), sa_family))
 		return -EINVAL;
 
 	/*
@@ -144,6 +145,17 @@ static int current_check_access_socket(struct socket *const sock,
 						.audit.u.net = &audit_net,
 						.access = access_request,
 						.layer_masks = &layer_masks,
+#ifdef CONFIG_TRACEPOINTS
+						.trace_net =
+							&(struct landlock_net_trace){
+								.address =
+									address,
+								.addrlen =
+									addrlen,
+								.socket_family =
+									sock_family,
+							},
+#endif /* CONFIG_TRACEPOINTS */
 					});
 				return -EACCES;
 			}
@@ -275,14 +287,22 @@ static int current_check_access_socket(struct socket *const sock,
 
 	audit_net.family = address->sa_family;
 	audit_net.sk = sock->sk;
-	landlock_log_denial(subject,
-			    &(struct landlock_request){
-				    .type = LANDLOCK_REQUEST_NET_ACCESS,
-				    .audit.type = LSM_AUDIT_DATA_NET,
-				    .audit.u.net = &audit_net,
-				    .access = access_request,
-				    .layer_masks = &layer_masks,
-			    });
+	landlock_log_denial(
+		subject, &(struct landlock_request){
+				 .type = LANDLOCK_REQUEST_NET_ACCESS,
+				 .audit.type = LSM_AUDIT_DATA_NET,
+				 .audit.u.net = &audit_net,
+				 .access = access_request,
+				 .layer_masks = &layer_masks,
+#ifdef CONFIG_TRACEPOINTS
+				 .trace_net =
+					 &(struct landlock_net_trace){
+						 .address = address,
+						 .addrlen = addrlen,
+						 .socket_family = sock_family,
+					 },
+#endif /* CONFIG_TRACEPOINTS */
+			 });
 	return -EACCES;
 }
 
diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h
index b536fa0425b7a5..cf77f1806a9505 100644
--- a/security/landlock/ruleset.h
+++ b/security/landlock/ruleset.h
@@ -171,9 +171,10 @@ struct landlock_ruleset {
 	 * @version: Counter incremented on each successful
 	 * landlock_add_rule(2), including when it only extends an existing
 	 * rule's access rights.  Used by tracepoints to correlate a domain with
-	 * the exact ruleset state it was created from.  Protected by @lock.
+	 * the exact successful rule history it was created from.  Protected by
+	 * @lock.
 	 */
-	u32 version;
+	u64 version;
 	/**
 	 * @id: Unique identifier for this ruleset, used for tracing.
 	 */
diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c
index 1d02d57f4c48c4..400a2f928de313 100644
--- a/security/landlock/syscalls.c
+++ b/security/landlock/syscalls.c
@@ -204,7 +204,7 @@ const int landlock_abi_version = 11;
  *   scoped;
  * - %E2BIG: @attr or @size inconsistencies;
  * - %EFAULT: @attr or @size inconsistencies;
- * - %ENOMSG: empty &landlock_ruleset_attr.handled_access_fs.
+ * - %ENOMSG: empty handled_access_fs, handled_access_net, and scoped fields.
  *
  * .. kernel-doc:: include/uapi/linux/landlock.h
  *     :identifiers: landlock_create_ruleset_flags
diff --git a/security/landlock/task.c b/security/landlock/task.c
index 4491ce31ae04ac..d9eae86fc5529c 100644
--- a/security/landlock/task.c
+++ b/security/landlock/task.c
@@ -88,7 +88,9 @@ static int hook_ptrace_access_check(struct task_struct *const child,
 				    const unsigned int mode)
 {
 	const struct landlock_cred_security *parent_subject;
+#ifdef CONFIG_TRACEPOINTS
 	u64 tracee_domain_id = 0;
+#endif /* CONFIG_TRACEPOINTS */
 	int err;
 
 	/* Quick return for non-landlocked tasks. */
@@ -100,10 +102,10 @@ static int hook_ptrace_access_check(struct task_struct *const child,
 		const struct landlock_domain *const child_dom =
 			landlock_get_task_domain(child);
 		err = domain_ptrace(parent_subject->domain, child_dom);
-#ifdef CONFIG_SECURITY_LANDLOCK_LOG
+#ifdef CONFIG_TRACEPOINTS
 		if (child_dom)
 			tracee_domain_id = child_dom->hierarchy->id;
-#endif /* CONFIG_SECURITY_LANDLOCK_LOG */
+#endif /* CONFIG_TRACEPOINTS */
 	}
 
 	if (!err)
@@ -121,7 +123,12 @@ static int hook_ptrace_access_check(struct task_struct *const child,
 				.u.tsk = child,
 			},
 			.layer_plus_one = parent_subject->domain->num_layers,
-			.other_domain_id = tracee_domain_id,
+#ifdef CONFIG_TRACEPOINTS
+			.trace_ptrace = &(struct landlock_ptrace_trace) {
+				.tracee_domain_id = tracee_domain_id,
+				.tracer = current,
+			},
+#endif /* CONFIG_TRACEPOINTS */
 		});
 
 	return err;
@@ -142,7 +149,6 @@ static int hook_ptrace_traceme(struct task_struct *const parent)
 {
 	const struct landlock_cred_security *parent_subject;
 	const struct landlock_domain *child_dom;
-	u64 tracee_domain_id = 0;
 	int err;
 
 	child_dom = landlock_get_current_domain();
@@ -154,12 +160,6 @@ static int hook_ptrace_traceme(struct task_struct *const parent)
 	if (!err)
 		return 0;
 
-#ifdef CONFIG_SECURITY_LANDLOCK_LOG
-	/* The tracee is the current task; its domain is stable here. */
-	if (child_dom)
-		tracee_domain_id = child_dom->hierarchy->id;
-#endif /* CONFIG_SECURITY_LANDLOCK_LOG */
-
 	/*
 	 * For the ptrace_traceme case, we log the domain which is the cause of
 	 * the denial, which means the parent domain instead of the current
@@ -174,7 +174,13 @@ static int hook_ptrace_traceme(struct task_struct *const parent)
 			.u.tsk = current,
 		},
 		.layer_plus_one = parent_subject->domain->num_layers,
-		.other_domain_id = tracee_domain_id,
+#ifdef CONFIG_TRACEPOINTS
+		.trace_ptrace = &(struct landlock_ptrace_trace) {
+			/* The current task's domain is stable here. */
+			.tracee_domain_id = child_dom ? child_dom->hierarchy->id : 0,
+			.tracer = parent,
+		},
+#endif /* CONFIG_TRACEPOINTS */
 	});
 	return err;
 }
@@ -250,8 +256,7 @@ static bool domain_is_scoped(const struct landlock_domain *const client,
 }
 
 static bool sock_is_scoped(struct sock *const other,
-			   const struct landlock_domain *const domain,
-			   u64 *const peer_domain_id)
+			   const struct landlock_domain *const domain)
 {
 	const struct landlock_domain *dom_other;
 
@@ -269,13 +274,23 @@ static bool sock_is_scoped(struct sock *const other,
 		return false;
 
 	dom_other = landlock_cred(other->sk_socket->file->f_cred)->domain;
-#ifdef CONFIG_SECURITY_LANDLOCK_LOG
-	*peer_domain_id = dom_other ? dom_other->hierarchy->id : 0;
-#endif /* CONFIG_SECURITY_LANDLOCK_LOG */
 	return domain_is_scoped(domain, dom_other,
 				LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET);
 }
 
+#ifdef CONFIG_TRACEPOINTS
+
+static u64 get_socket_domain_id(const struct sock *const other)
+{
+	const struct landlock_domain *domain;
+
+	lockdep_assert_held(&unix_sk(other)->lock);
+	domain = landlock_cred(other->sk_socket->file->f_cred)->domain;
+	return domain ? domain->hierarchy->id : 0;
+}
+
+#endif /* CONFIG_TRACEPOINTS */
+
 static bool is_abstract_socket(struct sock *const sock)
 {
 	struct unix_address *addr = unix_sk(sock)->addr;
@@ -299,7 +314,6 @@ static int hook_unix_stream_connect(struct sock *const sock,
 				    struct sock *const newsk)
 {
 	size_t handle_layer;
-	u64 peer_domain_id = 0;
 	const struct landlock_cred_security *const subject =
 		landlock_get_applicable_subject(current_cred(), unix_scope,
 						&handle_layer);
@@ -311,7 +325,7 @@ static int hook_unix_stream_connect(struct sock *const sock,
 	if (!is_abstract_socket(other))
 		return 0;
 
-	if (!sock_is_scoped(other, subject->domain, &peer_domain_id))
+	if (!sock_is_scoped(other, subject->domain))
 		return 0;
 
 	landlock_log_denial(subject, &(struct landlock_request) {
@@ -323,7 +337,9 @@ static int hook_unix_stream_connect(struct sock *const sock,
 			},
 		},
 		.layer_plus_one = handle_layer + 1,
-		.other_domain_id = peer_domain_id,
+#ifdef CONFIG_TRACEPOINTS
+		.other_domain_id = get_socket_domain_id(other),
+#endif /* CONFIG_TRACEPOINTS */
 	});
 	return -EPERM;
 }
@@ -332,7 +348,6 @@ static int hook_unix_may_send(struct socket *const sock,
 			      struct socket *const other)
 {
 	size_t handle_layer;
-	u64 peer_domain_id = 0;
 	const struct landlock_cred_security *const subject =
 		landlock_get_applicable_subject(current_cred(), unix_scope,
 						&handle_layer);
@@ -350,7 +365,7 @@ static int hook_unix_may_send(struct socket *const sock,
 	if (!is_abstract_socket(other->sk))
 		return 0;
 
-	if (!sock_is_scoped(other->sk, subject->domain, &peer_domain_id))
+	if (!sock_is_scoped(other->sk, subject->domain))
 		return 0;
 
 	landlock_log_denial(subject, &(struct landlock_request) {
@@ -362,7 +377,9 @@ static int hook_unix_may_send(struct socket *const sock,
 			},
 		},
 		.layer_plus_one = handle_layer + 1,
-		.other_domain_id = peer_domain_id,
+#ifdef CONFIG_TRACEPOINTS
+		.other_domain_id = get_socket_domain_id(other->sk),
+#endif /* CONFIG_TRACEPOINTS */
 	});
 	return -EPERM;
 }
@@ -377,7 +394,9 @@ static int hook_task_kill(struct task_struct *const p,
 {
 	bool is_scoped;
 	size_t handle_layer;
+#ifdef CONFIG_TRACEPOINTS
 	u64 target_domain_id = 0;
+#endif /* CONFIG_TRACEPOINTS */
 	const struct landlock_cred_security *subject;
 
 	if (!cred) {
@@ -409,10 +428,10 @@ static int hook_task_kill(struct task_struct *const p,
 
 		is_scoped = domain_is_scoped(subject->domain, other,
 					     signal_scope.scope);
-#ifdef CONFIG_SECURITY_LANDLOCK_LOG
+#ifdef CONFIG_TRACEPOINTS
 		if (other)
 			target_domain_id = other->hierarchy->id;
-#endif /* CONFIG_SECURITY_LANDLOCK_LOG */
+#endif /* CONFIG_TRACEPOINTS */
 	}
 
 	if (!is_scoped)
@@ -425,7 +444,12 @@ static int hook_task_kill(struct task_struct *const p,
 			.u.tsk = p,
 		},
 		.layer_plus_one = handle_layer + 1,
-		.other_domain_id = target_domain_id,
+#ifdef CONFIG_TRACEPOINTS
+		.trace_signal = &(struct landlock_signal_trace) {
+			.target_domain_id = target_domain_id,
+			.signal = sig,
+		},
+#endif /* CONFIG_TRACEPOINTS */
 	});
 	return -EPERM;
 }
@@ -435,7 +459,9 @@ static int hook_file_send_sigiotask(struct task_struct *tsk,
 {
 	const struct landlock_cred_security *subject;
 	bool is_scoped = false;
+#ifdef CONFIG_TRACEPOINTS
 	u64 target_domain_id = 0;
+#endif /* CONFIG_TRACEPOINTS */
 
 	/* Lock already held by send_sigio() and send_sigurg(). */
 	lockdep_assert_held(&fown->lock);
@@ -468,10 +494,10 @@ static int hook_file_send_sigiotask(struct task_struct *tsk,
 
 		is_scoped = domain_is_scoped(subject->domain, other,
 					     signal_scope.scope);
-#ifdef CONFIG_SECURITY_LANDLOCK_LOG
+#ifdef CONFIG_TRACEPOINTS
 		if (other)
 			target_domain_id = other->hierarchy->id;
-#endif /* CONFIG_SECURITY_LANDLOCK_LOG */
+#endif /* CONFIG_TRACEPOINTS */
 	}
 
 	if (!is_scoped)
@@ -486,7 +512,12 @@ static int hook_file_send_sigiotask(struct task_struct *tsk,
 #ifdef CONFIG_SECURITY_LANDLOCK_LOG
 		.layer_plus_one = landlock_file(fown->file)->fown_layer + 1,
 #endif /* CONFIG_SECURITY_LANDLOCK_LOG */
-		.other_domain_id = target_domain_id,
+#ifdef CONFIG_TRACEPOINTS
+		.trace_signal = &(struct landlock_signal_trace) {
+			.target_domain_id = target_domain_id,
+			.signal = signum ? signum : SIGIO,
+		},
+#endif /* CONFIG_TRACEPOINTS */
 	});
 	return -EPERM;
 }
diff --git a/security/landlock/trace.c b/security/landlock/trace.c
index 8c21e5de6f0de2..225dbf37bab095 100644
--- a/security/landlock/trace.c
+++ b/security/landlock/trace.c
@@ -12,6 +12,7 @@
 #include <linux/err.h>
 #include <linux/fs.h>
 #include <linux/lsm_audit.h>
+#include <linux/socket.h>
 #include <net/sock.h>
 
 #include "access.h"
@@ -61,8 +62,8 @@ void landlock_trace_free_domain(const struct landlock_hierarchy *const hierarchy
  *
  * @request: Detail of the user space request.
  * @youngest_denied: The youngest hierarchy node that denied the access.
- * @missing: The set of denied access rights.
- * @same_exec: Whether the current task is the same executable that called
+ * @missing: The final missing access subset, when applicable.
+ * @same_exec: Whether the policy subject is the same executable that called
  *             landlock_restrict_self() for the denying domain, as computed
  *             by landlock_log_denial().
  * @logged: Whether the domain's policy selects this denial for logging, as
@@ -83,6 +84,10 @@ void landlock_trace_denial(
 	case LANDLOCK_REQUEST_FS_ACCESS:
 	case LANDLOCK_REQUEST_FS_CHANGE_TOPOLOGY:
 		if (trace_landlock_deny_access_fs_enabled()) {
+			const struct landlock_blockers blockers = {
+				.access = missing,
+				.type = request->type,
+			};
 			char *buf __free(__putname) = __getname();
 			struct path dentry_path;
 			const char *pathname;
@@ -147,29 +152,67 @@ void landlock_trace_denial(
 
 			trace_landlock_deny_access_fs(youngest_denied,
 						      same_exec, logged,
-						      missing, path, pathname);
+						      &blockers, path,
+						      pathname);
 		}
 		break;
 	case LANDLOCK_REQUEST_NET_ACCESS:
-		if (trace_landlock_deny_access_net_enabled())
+		if (trace_landlock_deny_access_net_enabled()) {
+			const struct landlock_net_trace *const trace_net =
+				request->trace_net;
+			const struct landlock_blockers blockers = {
+				.access = missing,
+				.type = request->type,
+			};
+			struct sockaddr_storage address = {};
+
+			if (WARN_ON_ONCE(!trace_net || !trace_net->address))
+				return;
+
+			if (WARN_ON_ONCE(
+				    trace_net->addrlen <
+					    (int)offsetofend(struct sockaddr,
+							     sa_family) ||
+				    trace_net->addrlen > (int)sizeof(address)))
+				return;
+
+			memcpy(&address, trace_net->address,
+			       trace_net->addrlen);
 			trace_landlock_deny_access_net(
-				youngest_denied, same_exec, logged, missing,
+				youngest_denied, same_exec, logged, &blockers,
 				request->audit.u.net->sk,
-				ntohs(request->audit.u.net->sport),
-				ntohs(request->audit.u.net->dport));
+				trace_net->socket_family, &address,
+				trace_net->addrlen);
+		}
 		break;
 	case LANDLOCK_REQUEST_PTRACE:
-		if (trace_landlock_deny_ptrace_enabled())
-			trace_landlock_deny_ptrace(youngest_denied, same_exec,
-						   logged,
-						   request->other_domain_id,
-						   request->audit.u.tsk);
+		if (trace_landlock_deny_ptrace_enabled()) {
+			const struct landlock_ptrace_trace *const trace_ptrace =
+				request->trace_ptrace;
+
+			if (WARN_ON_ONCE(!trace_ptrace ||
+					 !trace_ptrace->tracer))
+				return;
+
+			trace_landlock_deny_ptrace(
+				youngest_denied, same_exec, logged,
+				trace_ptrace->tracee_domain_id,
+				request->audit.u.tsk, trace_ptrace->tracer);
+		}
 		break;
 	case LANDLOCK_REQUEST_SCOPE_SIGNAL:
-		if (trace_landlock_deny_scope_signal_enabled())
+		if (trace_landlock_deny_scope_signal_enabled()) {
+			const struct landlock_signal_trace *const trace_signal =
+				request->trace_signal;
+
+			if (WARN_ON_ONCE(!trace_signal))
+				return;
+
 			trace_landlock_deny_scope_signal(
 				youngest_denied, same_exec, logged,
-				request->other_domain_id, request->audit.u.tsk);
+				trace_signal->target_domain_id,
+				request->audit.u.tsk, trace_signal->signal);
+		}
 		break;
 	case LANDLOCK_REQUEST_SCOPE_ABSTRACT_UNIX_SOCKET:
 		if (trace_landlock_deny_scope_abstract_unix_socket_enabled())
diff --git a/security/lsm_audit.c b/security/lsm_audit.c
index 737f5a263a8f79..955b2e7b2c8e4e 100644
--- a/security/lsm_audit.c
+++ b/security/lsm_audit.c
@@ -182,7 +182,7 @@ void audit_log_lsm_data(struct audit_buffer *ab,
 	 * start making this union too large!  See struct lsm_network_audit
 	 * as an example of how to deal with large data.
 	 */
-	BUILD_BUG_ON(sizeof(a->u) > sizeof(void *)*2);
+	BUILD_BUG_ON(sizeof(a->u) > (sizeof(u64) * 2));
 
 	switch (a->type) {
 	case LSM_AUDIT_DATA_NONE:
@@ -231,7 +231,7 @@ void audit_log_lsm_data(struct audit_buffer *ab,
 			audit_log_format(ab, " ino=%llu", inode->i_ino);
 		}
 
-		audit_log_format(ab, " ioctlcmd=0x%hx", a->u.op->cmd);
+		audit_log_format(ab, " ioctlcmd=0x%x", a->u.op->cmd);
 		break;
 	}
 	case LSM_AUDIT_DATA_DENTRY: {
@@ -403,6 +403,10 @@ void audit_log_lsm_data(struct audit_buffer *ab,
 	case LSM_AUDIT_DATA_NLMSGTYPE:
 		audit_log_format(ab, " nl-msgtype=%hu", a->u.nlmsg_type);
 		break;
+	case LSM_AUDIT_DATA_NS:
+		audit_log_format(ab, " namespace_type=0x%x namespace_id=%llu",
+				 a->u.ns.ns_type, a->u.ns.ns_id);
+		break;
 	} /* switch (a->type) */
 }
 
diff --git a/security/lsm_init.c b/security/lsm_init.c
index a1ad641811de06..8d5ecea99c46b8 100644
--- a/security/lsm_init.c
+++ b/security/lsm_init.c
@@ -290,7 +290,6 @@ static void __init lsm_prepare(struct lsm_info *lsm)
 		return;
 
 	/* Register the LSM blob sizes. */
-	blobs = lsm->blobs;
 	lsm_blob_size_update(&blobs->lbs_cred, &blob_sizes.lbs_cred);
 	lsm_blob_size_update(&blobs->lbs_file, &blob_sizes.lbs_file);
 	lsm_blob_size_update(&blobs->lbs_backing_file,
@@ -303,6 +302,7 @@ static void __init lsm_prepare(struct lsm_info *lsm)
 	lsm_blob_size_update(&blobs->lbs_ipc, &blob_sizes.lbs_ipc);
 	lsm_blob_size_update(&blobs->lbs_key, &blob_sizes.lbs_key);
 	lsm_blob_size_update(&blobs->lbs_msg_msg, &blob_sizes.lbs_msg_msg);
+	lsm_blob_size_update(&blobs->lbs_ns, &blob_sizes.lbs_ns);
 	lsm_blob_size_update(&blobs->lbs_perf_event,
 			     &blob_sizes.lbs_perf_event);
 	lsm_blob_size_update(&blobs->lbs_sock, &blob_sizes.lbs_sock);
@@ -431,8 +431,7 @@ int __init security_init(void)
 	}
 
 	if (lsm_order_cmdline) {
-		if (lsm_order_legacy)
-			lsm_order_legacy = NULL;
+		lsm_order_legacy = NULL;
 		lsm_order_parse(lsm_order_cmdline, "cmdline");
 	} else
 		lsm_order_parse(lsm_order_builtin, "builtin");
@@ -450,6 +449,7 @@ int __init security_init(void)
 		lsm_pr("blob(ipc) size %d\n", blob_sizes.lbs_ipc);
 		lsm_pr("blob(key) size %d\n", blob_sizes.lbs_key);
 		lsm_pr("blob(msg_msg)_size %d\n", blob_sizes.lbs_msg_msg);
+		lsm_pr("blob(ns) size %d\n", blob_sizes.lbs_ns);
 		lsm_pr("blob(sock) size %d\n", blob_sizes.lbs_sock);
 		lsm_pr("blob(superblock) size %d\n", blob_sizes.lbs_superblock);
 		lsm_pr("blob(perf_event) size %d\n", blob_sizes.lbs_perf_event);
@@ -476,8 +476,7 @@ int __init security_init(void)
 						    blob_sizes.lbs_inode, 0,
 						    SLAB_PANIC, NULL);
 
-	if (lsm_cred_alloc((struct cred *)unrcu_pointer(current->cred),
-			   GFP_KERNEL))
+	if (lsm_cred_alloc((struct cred *)current->cred, GFP_KERNEL))
 		panic("early LSM cred alloc failed\n");
 	if (lsm_task_alloc(current))
 		panic("early LSM task alloc failed\n");
diff --git a/security/security.c b/security/security.c
index 2ee276ab15c5bc..da8d48cd0a2a47 100644
--- a/security/security.c
+++ b/security/security.c
@@ -26,6 +26,7 @@
 #include <linux/string.h>
 #include <linux/xattr.h>
 #include <linux/msg.h>
+#include <linux/ns_common.h>
 #include <linux/overflow.h>
 #include <linux/perf_event.h>
 #include <linux/fs.h>
@@ -381,6 +382,19 @@ static int lsm_superblock_alloc(struct super_block *sb)
 			      GFP_KERNEL);
 }
 
+/**
+ * lsm_ns_alloc - allocate a composite namespace blob
+ * @ns: the namespace that needs a blob
+ *
+ * Allocate the namespace blob for all the modules
+ *
+ * Returns 0, or -ENOMEM if memory can't be allocated.
+ */
+static int lsm_ns_alloc(struct ns_common *ns)
+{
+	return lsm_blob_alloc(&ns->ns_security, blob_sizes.lbs_ns, GFP_KERNEL);
+}
+
 /**
  * lsm_fill_user_ctx - Fill a user space lsm_ctx structure
  * @uctx: a userspace LSM context to be filled
@@ -595,6 +609,31 @@ int security_ptrace_traceme(struct task_struct *parent)
 	return call_int_hook(ptrace_traceme, parent);
 }
 
+/**
+ * security_mem_foll_force() - Check if FOLL_FORCE is allowed
+ * @subject: credentials using which /proc/$pid/mem was opened
+ * @opened_by_owner: whether checks on open() were bypassed because the opener
+ *                   has the same MM as the target
+ *
+ * Check if FOLL_FORCE is allowed for accessing process memory through
+ * /proc/$pid/mem. opened_by_owner signals whether the opener's MM was the same
+ * as the target MM, meaning the security_ptrace_access_check() hook was
+ * bypassed on open().
+ * (Current current->mm does not matter for this; for example, if write() is
+ * called on an FD that was received from another process which obtained it with
+ * open("/proc/self/mem"), @opened_by_owner is still true.)
+ *
+ * Note that this hook is only designed to be useful in the opened_by_owner
+ * case, where the subject credentials effectively also describe the object.
+ *
+ * Return: Returns 0 if permission is granted.
+ */
+int security_mem_foll_force(const struct cred *subject,
+					    bool opened_by_owner)
+{
+	return call_int_hook(mem_foll_force, subject, opened_by_owner);
+}
+
 /**
  * security_capget() - Get the capability sets for a process
  * @target: target process
@@ -1425,7 +1464,7 @@ EXPORT_SYMBOL(security_path_mknod);
  *
  * Update inode security field after a regular file has been created.
  */
-void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry)
+void security_path_post_mknod(const struct mnt_idmap *idmap, struct dentry *dentry)
 {
 	if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
 		return;
@@ -1614,6 +1653,7 @@ int security_path_chroot(const struct path *path)
 
 /**
  * security_inode_create() - Check if creating a file is allowed
+ * @idmap: idmap of the mount
  * @dir: the parent directory
  * @dentry: the file being created
  * @mode: requested file mode
@@ -1622,12 +1662,12 @@ int security_path_chroot(const struct path *path)
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_create(struct inode *dir, struct dentry *dentry,
-			  umode_t mode)
+int security_inode_create(const struct mnt_idmap *idmap, struct inode *dir,
+			  struct dentry *dentry, umode_t mode)
 {
 	if (unlikely(IS_PRIVATE(dir)))
 		return 0;
-	return call_int_hook(inode_create, dir, dentry, mode);
+	return call_int_hook(inode_create, idmap, dir, dentry, mode);
 }
 EXPORT_SYMBOL_GPL(security_inode_create);
 
@@ -1638,7 +1678,7 @@ EXPORT_SYMBOL_GPL(security_inode_create);
  *
  * Update inode security data after a tmpfile has been created.
  */
-void security_inode_post_create_tmpfile(struct mnt_idmap *idmap,
+void security_inode_post_create_tmpfile(const struct mnt_idmap *idmap,
 					struct inode *inode)
 {
 	if (unlikely(IS_PRIVATE(inode)))
@@ -1648,6 +1688,7 @@ void security_inode_post_create_tmpfile(struct mnt_idmap *idmap,
 
 /**
  * security_inode_link() - Check if creating a hard link is allowed
+ * @idmap: idmap of the mount
  * @old_dentry: existing file
  * @dir: new parent directory
  * @new_dentry: new link
@@ -1656,12 +1697,12 @@ void security_inode_post_create_tmpfile(struct mnt_idmap *idmap,
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_link(struct dentry *old_dentry, struct inode *dir,
-			struct dentry *new_dentry)
+int security_inode_link(const struct mnt_idmap *idmap, struct dentry *old_dentry,
+			struct inode *dir, struct dentry *new_dentry)
 {
 	if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry))))
 		return 0;
-	return call_int_hook(inode_link, old_dentry, dir, new_dentry);
+	return call_int_hook(inode_link, idmap, old_dentry, dir, new_dentry);
 }
 
 /**
@@ -1682,6 +1723,7 @@ int security_inode_unlink(struct inode *dir, struct dentry *dentry)
 
 /**
  * security_inode_symlink() - Check if creating a symbolic link is allowed
+ * @idmap: idmap of the mount
  * @dir: parent directory
  * @dentry: symbolic link
  * @old_name: existing filename
@@ -1690,16 +1732,17 @@ int security_inode_unlink(struct inode *dir, struct dentry *dentry)
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_symlink(struct inode *dir, struct dentry *dentry,
-			   const char *old_name)
+int security_inode_symlink(const struct mnt_idmap *idmap, struct inode *dir,
+			   struct dentry *dentry, const char *old_name)
 {
 	if (unlikely(IS_PRIVATE(dir)))
 		return 0;
-	return call_int_hook(inode_symlink, dir, dentry, old_name);
+	return call_int_hook(inode_symlink, idmap, dir, dentry, old_name);
 }
 
 /**
  * security_inode_mkdir() - Check if creating a new directory is allowed
+ * @idmap: idmap of the mount
  * @dir: parent directory
  * @dentry: new directory
  * @mode: new directory mode
@@ -1709,11 +1752,12 @@ int security_inode_symlink(struct inode *dir, struct dentry *dentry,
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
+int security_inode_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
+			 struct dentry *dentry, umode_t mode)
 {
 	if (unlikely(IS_PRIVATE(dir)))
 		return 0;
-	return call_int_hook(inode_mkdir, dir, dentry, mode);
+	return call_int_hook(inode_mkdir, idmap, dir, dentry, mode);
 }
 EXPORT_SYMBOL_GPL(security_inode_mkdir);
 
@@ -1735,6 +1779,7 @@ int security_inode_rmdir(struct inode *dir, struct dentry *dentry)
 
 /**
  * security_inode_mknod() - Check if creating a special file is allowed
+ * @idmap: idmap of the mount
  * @dir: parent directory
  * @dentry: new file
  * @mode: new file mode
@@ -1747,12 +1792,12 @@ int security_inode_rmdir(struct inode *dir, struct dentry *dentry)
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_mknod(struct inode *dir, struct dentry *dentry,
-			 umode_t mode, dev_t dev)
+int security_inode_mknod(const struct mnt_idmap *idmap, struct inode *dir,
+			 struct dentry *dentry, umode_t mode, dev_t dev)
 {
 	if (unlikely(IS_PRIVATE(dir)))
 		return 0;
-	return call_int_hook(inode_mknod, dir, dentry, mode, dev);
+	return call_int_hook(inode_mknod, idmap, dir, dentry, mode, dev);
 }
 
 /**
@@ -1823,6 +1868,7 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode,
 
 /**
  * security_inode_permission() - Check if accessing an inode is allowed
+ * @idmap: idmap of the mount
  * @inode: inode
  * @mask: access mask
  *
@@ -1835,11 +1881,12 @@ int security_inode_follow_link(struct dentry *dentry, struct inode *inode,
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_permission(struct inode *inode, int mask)
+int security_inode_permission(const struct mnt_idmap *idmap, struct inode *inode,
+			      int mask)
 {
 	if (unlikely(IS_PRIVATE(inode)))
 		return 0;
-	return call_int_hook(inode_permission, inode, mask);
+	return call_int_hook(inode_permission, idmap, inode, mask);
 }
 
 /**
@@ -1855,7 +1902,7 @@ int security_inode_permission(struct inode *inode, int mask)
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_setattr(struct mnt_idmap *idmap,
+int security_inode_setattr(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, struct iattr *attr)
 {
 	if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
@@ -1872,7 +1919,7 @@ EXPORT_SYMBOL_GPL(security_inode_setattr);
  *
  * Update inode security field after successful setting file attributes.
  */
-void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+void security_inode_post_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 				 int ia_valid)
 {
 	if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
@@ -1921,7 +1968,7 @@ int security_inode_getattr(const struct path *path)
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_setxattr(struct mnt_idmap *idmap,
+int security_inode_setxattr(const struct mnt_idmap *idmap,
 			    struct dentry *dentry, const char *name,
 			    const void *value, size_t size, int flags)
 {
@@ -1953,7 +2000,7 @@ int security_inode_setxattr(struct mnt_idmap *idmap,
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_set_acl(struct mnt_idmap *idmap,
+int security_inode_set_acl(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, const char *acl_name,
 			   struct posix_acl *kacl)
 {
@@ -1990,7 +2037,7 @@ void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name,
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_get_acl(struct mnt_idmap *idmap,
+int security_inode_get_acl(const struct mnt_idmap *idmap,
 			   struct dentry *dentry, const char *acl_name)
 {
 	if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
@@ -2009,7 +2056,7 @@ int security_inode_get_acl(struct mnt_idmap *idmap,
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_remove_acl(struct mnt_idmap *idmap,
+int security_inode_remove_acl(const struct mnt_idmap *idmap,
 			      struct dentry *dentry, const char *acl_name)
 {
 	if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
@@ -2026,7 +2073,7 @@ int security_inode_remove_acl(struct mnt_idmap *idmap,
  * Update inode security data after successfully removing posix acls on
  * @dentry in @idmap. The posix acls are identified by @acl_name.
  */
-void security_inode_post_remove_acl(struct mnt_idmap *idmap,
+void security_inode_post_remove_acl(const struct mnt_idmap *idmap,
 				    struct dentry *dentry, const char *acl_name)
 {
 	if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
@@ -2108,7 +2155,7 @@ int security_inode_listxattr(struct dentry *dentry)
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_inode_removexattr(struct mnt_idmap *idmap,
+int security_inode_removexattr(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, const char *name)
 {
 	int rc;
@@ -2197,7 +2244,7 @@ int security_inode_need_killpriv(struct dentry *dentry)
  * Return: Return 0 on success.  If error is returned, then the operation
  *         causing setuid bit removal is failed.
  */
-int security_inode_killpriv(struct mnt_idmap *idmap,
+int security_inode_killpriv(const struct mnt_idmap *idmap,
 			    struct dentry *dentry)
 {
 	return call_int_hook(inode_killpriv, idmap, dentry);
@@ -2219,7 +2266,7 @@ int security_inode_killpriv(struct mnt_idmap *idmap,
  *
  * Return: Returns size of buffer on success.
  */
-int security_inode_getsecurity(struct mnt_idmap *idmap,
+int security_inode_getsecurity(const struct mnt_idmap *idmap,
 			       struct inode *inode, const char *name,
 			       void **buffer, bool alloc)
 {
@@ -2387,7 +2434,7 @@ int security_kernfs_init_security(struct kernfs_node *kn_dir,
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_file_permission(struct file *file, int mask)
+int security_file_permission(const struct file *file, int mask)
 {
 	return call_int_hook(file_permission, file, mask);
 }
@@ -2486,9 +2533,8 @@ void security_backing_file_free(struct file *backing_file)
 {
 	void *blob = backing_file_security(backing_file);
 
-	call_void_hook(backing_file_free, backing_file);
-
 	if (blob) {
+		call_void_hook(backing_file_free, backing_file);
 		backing_file_set_security(backing_file, NULL);
 		kmem_cache_free(lsm_backing_file_cache, blob);
 	}
@@ -3357,6 +3403,64 @@ int security_create_user_ns(const struct cred *cred)
 	return call_int_hook(userns_create, cred);
 }
 
+/**
+ * security_namespace_init() - Initialize LSM security data for a namespace
+ * @ns: the namespace being initialized
+ *
+ * Initialize the LSM security blob attached to the namespace. The namespace type
+ * is available via ns->ns_type, and the owning user namespace (if any)
+ * via ns->ops->owner(ns).
+ *
+ * Return: Returns 0 if successful, otherwise < 0 error code.
+ */
+int security_namespace_init(struct ns_common *ns)
+{
+	int rc;
+
+	rc = lsm_ns_alloc(ns);
+	if (unlikely(rc))
+		return rc;
+
+	rc = call_int_hook(namespace_init, ns);
+	if (unlikely(rc))
+		security_namespace_free(ns);
+
+	return rc;
+}
+
+/**
+ * security_namespace_free() - Release LSM security data from a namespace
+ * @ns: the namespace being freed
+ *
+ * Release security data attached to the namespace. Called before the
+ * namespace structure is freed.
+ */
+void security_namespace_free(struct ns_common *ns)
+{
+	if (!ns->ns_security)
+		return;
+
+	call_void_hook(namespace_free, ns);
+
+	kfree(ns->ns_security);
+	ns->ns_security = NULL;
+}
+
+/**
+ * security_namespace_install() - Check permission to install a namespace
+ * @nsset: the target nsset being configured
+ * @ns: the namespace being installed
+ *
+ * Check permission before allowing a namespace to be installed into the
+ * process's set of namespaces via setns(2).
+ *
+ * Return: Returns 0 if permission is granted, otherwise < 0 error code.
+ */
+int security_namespace_install(const struct nsset *nsset, struct ns_common *ns)
+{
+	return call_int_hook(namespace_install, nsset, ns);
+}
+
 /**
  * security_ipc_permission() - Check if sysv ipc access is allowed
  * @ipcp: ipc permission structure
diff --git a/security/selinux/avc.c b/security/selinux/avc.c
index a9401d6c2e5f1c..2a9d4888384b2c 100644
--- a/security/selinux/avc.c
+++ b/security/selinux/avc.c
@@ -190,8 +190,8 @@ avc_xperms_decision_lookup(u8 driver, u8 base_perm,
 }
 
 static inline unsigned int
-avc_xperms_has_perm(struct extended_perms_decision *xpd,
-					u8 perm, u8 which)
+avc_xperms_has_perm(const struct extended_perms_decision *xpd,
+		    u8 perm, u8 which)
 {
 	unsigned int rc = 0;
 
@@ -247,7 +247,7 @@ static void avc_xperms_free(struct avc_xperms_node *xp_node)
 }
 
 static void avc_copy_xperms_decision(struct extended_perms_decision *dest,
-					struct extended_perms_decision *src)
+				     const struct extended_perms_decision *src)
 {
 	dest->base_perm = src->base_perm;
 	dest->driver = src->driver;
@@ -269,7 +269,7 @@ static void avc_copy_xperms_decision(struct extended_perms_decision *dest,
  */
 static inline void avc_quick_copy_xperms_decision(u8 perm,
 			struct extended_perms_decision *dest,
-			struct extended_perms_decision *src)
+			const struct extended_perms_decision *src)
 {
 	/*
 	 * compute index of the u32 of the 256 bits (8 u32s) that contain this
@@ -323,7 +323,7 @@ static struct avc_xperms_decision_node
 }
 
 static int avc_add_xperms_decision(struct avc_node *node,
-			struct extended_perms_decision *src)
+				   const struct extended_perms_decision *src)
 {
 	struct avc_xperms_decision_node *dest_xpd;
 
@@ -348,7 +348,7 @@ static struct avc_xperms_node *avc_xperms_alloc(void)
 }
 
 static int avc_xperms_populate(struct avc_node *node,
-				struct avc_xperms_node *src)
+			       const struct avc_xperms_node *src)
 {
 	struct avc_xperms_node *dest;
 	struct avc_xperms_decision_node *dest_xpd;
@@ -381,14 +381,17 @@ static int avc_xperms_populate(struct avc_node *node,
 }
 
 static inline u32 avc_xperms_audit_required(u32 requested,
-					struct av_decision *avd,
-					struct extended_perms_decision *xpd,
-					u8 perm,
-					int result,
-					u32 *deniedp)
+				const struct av_decision *avd,
+				const struct extended_perms_decision *xpd,
+				u8 perm,
+				int result,
+				u32 *deniedp)
 {
 	u32 denied, audited;
 
+	if (avd->flags & AVD_FLAGS_NEVERAUDIT)
+		return 0;
+
 	denied = requested & ~avd->allowed;
 	if (unlikely(denied)) {
 		audited = denied & avd->auditdeny;
@@ -411,8 +414,8 @@ static inline u32 avc_xperms_audit_required(u32 requested,
 }
 
 static inline int avc_xperms_audit(u32 ssid, u32 tsid, u16 tclass,
-				   u32 requested, struct av_decision *avd,
-				   struct extended_perms_decision *xpd,
+				   u32 requested, const struct av_decision *avd,
+				   const struct extended_perms_decision *xpd,
 				   u8 perm, int result,
 				   struct common_audit_data *ad)
 {
@@ -509,7 +512,8 @@ static struct avc_node *avc_alloc_node(void)
 	return node;
 }
 
-static void avc_node_populate(struct avc_node *node, u32 ssid, u32 tsid, u16 tclass, struct av_decision *avd)
+static void avc_node_populate(struct avc_node *node, u32 ssid, u32 tsid,
+			      u16 tclass, const struct av_decision *avd)
 {
 	node->ae.ssid = ssid;
 	node->ae.tsid = tsid;
@@ -578,7 +582,7 @@ static int avc_latest_notif_update(u32 seqno, int is_insert)
 		}
 	} else {
 		if (seqno > selinux_avc.avc_cache.latest_notif)
-			selinux_avc.avc_cache.latest_notif = seqno;
+			WRITE_ONCE(selinux_avc.avc_cache.latest_notif, seqno);
 	}
 	spin_unlock_irqrestore(&notif_lock, flag);
 
@@ -603,7 +607,8 @@ static int avc_latest_notif_update(u32 seqno, int is_insert)
  * the access vectors into a cache entry.
  */
 static void avc_insert(u32 ssid, u32 tsid, u16 tclass,
-		       struct av_decision *avd, struct avc_xperms_node *xp_node)
+		       const struct av_decision *avd,
+		       const struct avc_xperms_node *xp_node)
 {
 	struct avc_node *pos, *node = NULL;
 	u32 hvalue;
@@ -828,7 +833,7 @@ int __init avc_add_callback(int (*callback)(u32 event), u32 events)
  */
 static int avc_update_node(u32 event, u32 perms, u8 driver, u8 base_perm,
 			   u8 xperm, u32 ssid, u32 tsid, u16 tclass, u32 seqno,
-			   struct extended_perms_decision *xpd, u32 flags)
+			   const struct extended_perms_decision *xpd, u32 flags)
 {
 	u32 hvalue;
 	int rc = 0;
@@ -1149,8 +1154,11 @@ inline int avc_has_perm_noaudit(u32 ssid, u32 tsid,
 	u32 denied;
 	struct avc_node *node;
 
-	if (WARN_ON(!requested))
+	if (WARN_ON(!requested)) {
+		/* Provide a deny-all, audit-all decision to the caller. */
+		*avd = (struct av_decision){ .auditdeny = 0xffffffff };
 		return -EACCES;
+	}
 
 	rcu_read_lock();
 	node = avc_lookup(ssid, tsid, tclass);
@@ -1203,5 +1211,6 @@ int avc_has_perm(u32 ssid, u32 tsid, u16 tclass,
 
 u32 avc_policy_seqno(void)
 {
-	return selinux_avc.avc_cache.latest_notif;
+	/* Writers hold notif_lock; readers only sample the sequence number. */
+	return READ_ONCE(selinux_avc.avc_cache.latest_notif);
 }
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index e5e17f100aae68..e8d1e988c1d6b6 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -35,7 +35,6 @@
 #include <linux/mm.h>
 #include <linux/mman.h>
 #include <linux/slab.h>
-#include <linux/pagemap.h>
 #include <linux/proc_fs.h>
 #include <linux/swap.h>
 #include <linux/spinlock.h>
@@ -1674,26 +1673,32 @@ static int cred_has_capability(const struct cred *cred,
 	return rc;
 }
 
-/* Check whether a task has a particular permission to an inode.
-   The 'adp' parameter is optional and allows other audit
-   data to be passed (e.g. the dentry). */
-static int inode_has_perm(const struct cred *cred,
-			  struct inode *inode,
-			  u32 perms,
-			  struct common_audit_data *adp)
+/*
+ * Check whether a SID has a particular permission to an inode.  The 'adp'
+ * parameter is optional and allows other audit data to be passed (e.g. the
+ * dentry).
+ */
+static int inode_sid_has_perm(u32 sid, struct inode *inode, u32 perms,
+			      struct common_audit_data *adp)
 {
 	struct inode_security_struct *isec;
-	u32 sid;
 
 	if (unlikely(IS_PRIVATE(inode)))
 		return 0;
 
-	sid = cred_sid(cred);
 	isec = selinux_inode(inode);
 
 	return avc_has_perm(sid, isec->sid, isec->sclass, perms, adp);
 }
 
+static int inode_has_perm(const struct cred *cred,
+			  struct inode *inode,
+			  u32 perms,
+			  struct common_audit_data *adp)
+{
+	return inode_sid_has_perm(cred_sid(cred), inode, perms, adp);
+}
+
 /* Same as inode_has_perm, but pass explicit audit data containing
    the dentry to help the auditing code to more easily generate the
    pathname if needed. */
@@ -2159,6 +2164,36 @@ static int selinux_ptrace_traceme(struct task_struct *parent)
 			    SECCLASS_PROCESS, PROCESS__PTRACE, NULL);
 }
 
+/**
+ * selinux_mem_foll_force() - Determine whether /proc/$pid/mem can use FOLL_FORCE
+ * @subject: credentials using which /proc/$pid/mem was opened
+ * @opened_by_owner: whether checks on open() were bypassed because the opener
+ *                   has the same MM as the target
+ *
+ * Decide whether it should be possible to read non-readable VMAs and write
+ * non-writable VMAs via /proc/self/mem.
+ * The @opened_by_owner case only applies to systems configured with
+ * PROC_MEM_FORCE_ALWAYS, and only happens on accesses that are not visible to
+ * selinux_ptrace_access_check() because of the introspection exceptions in
+ * may_access_mm() and __ptrace_may_access().
+ *
+ * This allows a process to overwrite read-only code in its own address space.
+ *
+ * Creating an audit record on denial doesn't make sense here, since we can't
+ * tell whether FOLL_FORCE matters for the accessed VMAs.
+ */
+static int selinux_mem_foll_force(const struct cred *subject, bool opened_by_owner)
+{
+	struct av_decision avd;
+	u32 sid;
+
+	if (!opened_by_owner)
+		return 0;
+	sid = cred_sid(subject);
+
+	return avc_has_perm_noaudit(sid, sid, SECCLASS_PROCESS, PROCESS__PTRACE, 0, &avd);
+}
+
 static int selinux_capget(const struct task_struct *target, kernel_cap_t *effective,
 			  kernel_cap_t *inheritable, kernel_cap_t *permitted)
 {
@@ -3074,12 +3109,14 @@ static int selinux_inode_init_security_anon(struct inode *inode,
 			    &ad);
 }
 
-static int selinux_inode_create(struct inode *dir, struct dentry *dentry, umode_t mode)
+static int selinux_inode_create(const struct mnt_idmap *idmap, struct inode *dir,
+				struct dentry *dentry, umode_t mode)
 {
 	return may_create(dir, dentry, SECCLASS_FILE);
 }
 
-static int selinux_inode_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry)
+static int selinux_inode_link(const struct mnt_idmap *idmap, struct dentry *old_dentry,
+			      struct inode *dir, struct dentry *new_dentry)
 {
 	return may_link(dir, old_dentry, MAY_LINK);
 }
@@ -3089,12 +3126,14 @@ static int selinux_inode_unlink(struct inode *dir, struct dentry *dentry)
 	return may_link(dir, dentry, MAY_UNLINK);
 }
 
-static int selinux_inode_symlink(struct inode *dir, struct dentry *dentry, const char *name)
+static int selinux_inode_symlink(const struct mnt_idmap *idmap, struct inode *dir,
+				 struct dentry *dentry, const char *name)
 {
 	return may_create(dir, dentry, SECCLASS_LNK_FILE);
 }
 
-static int selinux_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mask)
+static int selinux_inode_mkdir(const struct mnt_idmap *idmap, struct inode *dir,
+			       struct dentry *dentry, umode_t mask)
 {
 	return may_create(dir, dentry, SECCLASS_DIR);
 }
@@ -3104,7 +3143,8 @@ static int selinux_inode_rmdir(struct inode *dir, struct dentry *dentry)
 	return may_link(dir, dentry, MAY_RMDIR);
 }
 
-static int selinux_inode_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
+static int selinux_inode_mknod(const struct mnt_idmap *idmap, struct inode *dir,
+			       struct dentry *dentry, umode_t mode, dev_t dev)
 {
 	return may_create(dir, dentry, inode_mode_to_security_class(mode));
 }
@@ -3236,13 +3276,15 @@ static inline void task_avdcache_update(struct task_security_struct *tsec,
 
 /**
  * selinux_inode_permission - Check if the current task can access an inode
+ * @idmap: idmap of the mount
  * @inode: the inode that is being accessed
  * @requested: the accesses being requested
  *
  * Check if the current task is allowed to access @inode according to
  * @requested.  Returns 0 if allowed, negative values otherwise.
  */
-static int selinux_inode_permission(struct inode *inode, int requested)
+static int selinux_inode_permission(const struct mnt_idmap *idmap,
+				    struct inode *inode, int requested)
 {
 	int mask;
 	u32 perms;
@@ -3297,7 +3339,7 @@ static int selinux_inode_permission(struct inode *inode, int requested)
 	return rc;
 }
 
-static int selinux_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int selinux_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 				 struct iattr *iattr)
 {
 	const struct cred *cred = current_cred();
@@ -3367,7 +3409,7 @@ static int selinux_inode_xattr_skipcap(const char *name)
 	return !strcmp(name, XATTR_NAME_SELINUX);
 }
 
-static int selinux_inode_setxattr(struct mnt_idmap *idmap,
+static int selinux_inode_setxattr(const struct mnt_idmap *idmap,
 				  struct dentry *dentry, const char *name,
 				  const void *value, size_t size, int flags)
 {
@@ -3453,20 +3495,20 @@ static int selinux_inode_setxattr(struct mnt_idmap *idmap,
 			    &ad);
 }
 
-static int selinux_inode_set_acl(struct mnt_idmap *idmap,
+static int selinux_inode_set_acl(const struct mnt_idmap *idmap,
 				 struct dentry *dentry, const char *acl_name,
 				 struct posix_acl *kacl)
 {
 	return dentry_has_perm(current_cred(), dentry, FILE__SETATTR);
 }
 
-static int selinux_inode_get_acl(struct mnt_idmap *idmap,
+static int selinux_inode_get_acl(const struct mnt_idmap *idmap,
 				 struct dentry *dentry, const char *acl_name)
 {
 	return dentry_has_perm(current_cred(), dentry, FILE__GETATTR);
 }
 
-static int selinux_inode_remove_acl(struct mnt_idmap *idmap,
+static int selinux_inode_remove_acl(const struct mnt_idmap *idmap,
 				    struct dentry *dentry, const char *acl_name)
 {
 	return dentry_has_perm(current_cred(), dentry, FILE__SETATTR);
@@ -3526,7 +3568,7 @@ static int selinux_inode_listxattr(struct dentry *dentry)
 	return dentry_has_perm(cred, dentry, FILE__GETATTR);
 }
 
-static int selinux_inode_removexattr(struct mnt_idmap *idmap,
+static int selinux_inode_removexattr(const struct mnt_idmap *idmap,
 				     struct dentry *dentry, const char *name)
 {
 	/* if not a selinux xattr, only check the ordinary setattr perm */
@@ -3606,7 +3648,7 @@ static int selinux_path_notify(const struct path *path, u64 mask,
  *
  * Permission check is handled by selinux_inode_getxattr hook.
  */
-static int selinux_inode_getsecurity(struct mnt_idmap *idmap,
+static int selinux_inode_getsecurity(const struct mnt_idmap *idmap,
 				     struct inode *inode, const char *name,
 				     void **buffer, bool alloc)
 {
@@ -3799,7 +3841,7 @@ static int selinux_kernfs_init_security(struct kernfs_node *kn_dir,
 
 /* file security operations */
 
-static int selinux_revalidate_file_permission(struct file *file, int mask)
+static int selinux_revalidate_file_permission(const struct file *file, int mask)
 {
 	const struct cred *cred = current_cred();
 	struct inode *inode = file_inode(file);
@@ -3812,7 +3854,7 @@ static int selinux_revalidate_file_permission(struct file *file, int mask)
 			     file_mask_to_av(inode->i_mode, mask));
 }
 
-static int selinux_file_permission(struct file *file, int mask)
+static int selinux_file_permission(const struct file *file, int mask)
 {
 	struct inode *inode = file_inode(file);
 	struct file_security_struct *fsec = selinux_file(file);
@@ -3843,23 +3885,80 @@ static int selinux_file_alloc_security(struct file *file)
 	return 0;
 }
 
+static inline u32 selinux_file_user_sid(const struct file *file)
+{
+	if (unlikely(file->f_mode & FMODE_BACKING))
+		return selinux_backing_file(file)->uf_sid;
+	return selinux_file(file)->sid;
+}
+
 static int selinux_backing_file_alloc(struct file *backing_file,
 				      const struct file *user_file)
 {
 	struct backing_file_security_struct *bfsec;
+	const struct backing_file_security_struct *ubfsec;
+	struct backing_file_security_layer *layer;
+	u32 i;
 
 	bfsec = selinux_backing_file(backing_file);
-	bfsec->uf_sid = selinux_file(user_file)->sid;
+	bfsec->uf_sid = selinux_file_user_sid(user_file);
+	if (!(user_file->f_mode & FMODE_BACKING))
+		return 0;
+
+	ubfsec = selinux_backing_file(user_file);
+	/* a wrapped count would make kmalloc_array() return ZERO_SIZE_PTR */
+	if (unlikely(ubfsec->layer_count == U32_MAX))
+		return -EOVERFLOW;
+
+	/*
+	 * The final VMA only retains the lowest backing file, so record the
+	 * whole chain here rather than in the mmap hook, where concurrent
+	 * mappings would have to be serialized.  Size it dynamically: erofs
+	 * inode sharing adds a backing file without bumping s_stack_depth.
+	 */
+	bfsec->layers = kmalloc_array(ubfsec->layer_count + 1,
+				      sizeof(*bfsec->layers), GFP_KERNEL);
+	if (!bfsec->layers)
+		return -ENOMEM;
+
+	for (i = 0; i < ubfsec->layer_count; i++) {
+		layer = &bfsec->layers[i];
+		*layer = ubfsec->layers[i];
+		path_get(&layer->path);
+	}
+
+	/* f_path, not file_user_path(): this layer, not the top-level file */
+	layer = &bfsec->layers[i];
+	layer->path = user_file->f_path;
+	layer->mounter_sid = cred_sid(user_file->f_cred);
+	layer->fd_sid = selinux_file(user_file)->sid;
+	path_get(&layer->path);
+	bfsec->layer_count = ubfsec->layer_count + 1;
 
 	return 0;
 }
 
+static void selinux_backing_file_free(struct file *backing_file)
+{
+	struct backing_file_security_struct *bfsec;
+
+	/* security_backing_file_free() may be called twice after an error */
+	if (!backing_file_security(backing_file))
+		return;
+
+	bfsec = selinux_backing_file(backing_file);
+	while (bfsec->layer_count)
+		path_put(&bfsec->layers[--bfsec->layer_count].path);
+	kfree(bfsec->layers);
+	bfsec->layers = NULL;
+}
+
 /*
  * Check whether a task has the ioctl permission and cmd
  * operation to an inode.
  */
 static int ioctl_has_perm(const struct cred *cred, struct file *file,
-		u32 requested, u16 cmd)
+		u32 requested, unsigned int cmd)
 {
 	struct common_audit_data ad;
 	struct file_security_struct *fsec = selinux_file(file);
@@ -3930,14 +4029,14 @@ static int selinux_file_ioctl(struct file *file, unsigned int cmd,
 	case FIOCLEX:
 	case FIONCLEX:
 		if (!selinux_policycap_ioctl_skip_cloexec())
-			error = ioctl_has_perm(cred, file, FILE__IOCTL, (u16) cmd);
+			error = ioctl_has_perm(cred, file, FILE__IOCTL, cmd);
 		break;
 
 	/* default case assumes that the command will go
 	 * to the file's ioctl() function.
 	 */
 	default:
-		error = ioctl_has_perm(cred, file, FILE__IOCTL, (u16) cmd);
+		error = ioctl_has_perm(cred, file, FILE__IOCTL, cmd);
 	}
 	return error;
 }
@@ -3971,6 +4070,53 @@ static int selinux_file_ioctl_compat(struct file *file, unsigned int cmd,
 
 static int default_noexec __ro_after_init;
 
+static u32 file_map_prot_to_av(unsigned long prot, bool shared)
+{
+	u32 av = FILE__READ;
+
+	if (shared && (prot & PROT_WRITE))
+		av |= FILE__WRITE;
+	if (prot & PROT_EXEC)
+		av |= FILE__EXECUTE;
+
+	return av;
+}
+
+static int backing_mounters_has_perm(const struct file *file, u32 av)
+{
+	const struct backing_file_security_struct *bfsec;
+	const struct backing_file_security_layer *layer;
+	struct common_audit_data ad;
+	struct inode *inode;
+	u32 i;
+	int rc;
+
+	if (WARN_ON_ONCE(!(file->f_mode & FMODE_BACKING)))
+		return -EIO;
+
+	bfsec = selinux_backing_file(file);
+	for (i = 0; i < bfsec->layer_count; i++) {
+		layer = &bfsec->layers[i];
+		inode = d_inode(layer->path.dentry);
+
+		ad.type = LSM_AUDIT_DATA_PATH;
+		ad.u.path = layer->path;
+
+		if (layer->mounter_sid != layer->fd_sid) {
+			rc = avc_has_perm(layer->mounter_sid, layer->fd_sid,
+					  SECCLASS_FD, FD__USE, &ad);
+			if (rc)
+				return rc;
+		}
+
+		rc = inode_sid_has_perm(layer->mounter_sid, inode, av, &ad);
+		if (rc)
+			return rc;
+	}
+
+	return 0;
+}
+
 static int __file_map_prot_check(const struct file *file, unsigned long prot,
 				 bool shared, bool mounter_check,
 				 bool bf_user_file)
@@ -4004,14 +4150,10 @@ static int __file_map_prot_check(const struct file *file, unsigned long prot,
 	if (file) {
 		const struct cred *cred = mounter_check ?
 				file->f_cred : current_cred();
-		/* "read" always possible, "write" only if shared */
-		u32 av = FILE__READ;
-		if (shared && prot_write)
-			av |= FILE__WRITE;
-		if (prot_exec)
-			av |= FILE__EXECUTE;
 
-		return __file_has_perm(cred, file, av, bf_user_file);
+		return __file_has_perm(cred, file,
+				       file_map_prot_to_av(prot, shared),
+				       bf_user_file);
 	}
 
 	return 0;
@@ -4106,6 +4248,7 @@ static int selinux_file_mprotect(struct vm_area_struct *vma,
 	int rc;
 	const struct cred *cred = current_cred();
 	u32 sid = cred_sid(cred);
+	u32 av;
 	const struct file *file = vma->vm_file;
 	bool backing_file;
 	bool shared = vma->vm_flags & VM_SHARED;
@@ -4149,6 +4292,10 @@ static int selinux_file_mprotect(struct vm_area_struct *vma,
 			if (rc)
 				return rc;
 			if (backing_file) {
+				rc = backing_mounters_has_perm(file,
+							       FILE__EXECMOD);
+				if (rc)
+					return rc;
 				rc = file_has_perm(file->f_cred, file,
 						   FILE__EXECMOD);
 				if (rc)
@@ -4161,6 +4308,10 @@ static int selinux_file_mprotect(struct vm_area_struct *vma,
 	if (rc)
 		return rc;
 	if (backing_file) {
+		av = file_map_prot_to_av(prot, shared);
+		rc = backing_mounters_has_perm(file, av);
+		if (rc)
+			return rc;
 		rc = file_map_prot_check(file, prot, shared, true);
 		if (rc)
 			return rc;
@@ -7549,6 +7700,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = {
 
 	LSM_HOOK_INIT(ptrace_access_check, selinux_ptrace_access_check),
 	LSM_HOOK_INIT(ptrace_traceme, selinux_ptrace_traceme),
+	LSM_HOOK_INIT(mem_foll_force, selinux_mem_foll_force),
 	LSM_HOOK_INIT(capget, selinux_capget),
 	LSM_HOOK_INIT(capset, selinux_capset),
 	LSM_HOOK_INIT(capable, selinux_capable),
@@ -7619,6 +7771,7 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(file_permission, selinux_file_permission),
 	LSM_HOOK_INIT(file_alloc_security, selinux_file_alloc_security),
 	LSM_HOOK_INIT(backing_file_alloc, selinux_backing_file_alloc),
+	LSM_HOOK_INIT(backing_file_free, selinux_backing_file_free),
 	LSM_HOOK_INIT(file_ioctl, selinux_file_ioctl),
 	LSM_HOOK_INIT(file_ioctl_compat, selinux_file_ioctl_compat),
 	LSM_HOOK_INIT(mmap_file, selinux_mmap_file),
diff --git a/security/selinux/include/avc.h b/security/selinux/include/avc.h
index 01b5167fee1a89..87027a934a118d 100644
--- a/security/selinux/include/avc.h
+++ b/security/selinux/include/avc.h
@@ -61,8 +61,9 @@ struct selinux_audit_data {
 
 void __init avc_init(void);
 
-static inline u32 avc_audit_required(u32 requested, struct av_decision *avd,
-				     int result, u32 auditdeny, u32 *deniedp)
+static inline u32 avc_audit_required(u32 requested,
+				     const struct av_decision *avd, int result,
+				     u32 auditdeny, u32 *deniedp)
 {
 	u32 denied, audited;
 
@@ -121,7 +122,7 @@ int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, u32 requested, u32 audited,
  * before calling the auditing code.
  */
 static inline int avc_audit(u32 ssid, u32 tsid, u16 tclass, u32 requested,
-			    struct av_decision *avd, int result,
+			    const struct av_decision *avd, int result,
 			    struct common_audit_data *a)
 {
 	u32 audited, denied;
diff --git a/security/selinux/include/objsec.h b/security/selinux/include/objsec.h
index 3c0a16ec978b01..2f21568251ffeb 100644
--- a/security/selinux/include/objsec.h
+++ b/security/selinux/include/objsec.h
@@ -86,8 +86,16 @@ struct file_security_struct {
 	u32 pseqno; /* Policy seqno at the time of file open */
 };
 
+struct backing_file_security_layer {
+	struct path path; /* this layer's real path */
+	u32 mounter_sid; /* SID of the mounter that opened it */
+	u32 fd_sid; /* SID of its open file description */
+};
+
 struct backing_file_security_struct {
-	u32 uf_sid; /* associated user file fsec->sid */
+	u32 uf_sid; /* top-level user file fsec->sid */
+	u32 layer_count; /* number of intermediate backing files */
+	struct backing_file_security_layer *layers;
 };
 
 struct superblock_security_struct {
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index c7d91476971cb5..7efe097de208d0 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -13,7 +13,6 @@
  */
 
 #include <linux/kernel.h>
-#include <linux/pagemap.h>
 #include <linux/slab.h>
 #include <linux/vmalloc.h>
 #include <linux/fs.h>
@@ -75,8 +74,8 @@ struct selinux_fs_info {
 	char **bool_pending_names;
 	int *bool_pending_values;
 	struct dentry *class_dir;
-	unsigned long last_class_ino;
-	unsigned long last_ino;
+	u64 last_class_ino;
+	u64 last_ino;
 	struct super_block *sb;
 };
 
@@ -247,11 +246,6 @@ static int sel_mmap_handle_status(struct file *filp,
 	/* only allows one page from the head */
 	if (vma->vm_pgoff > 0 || size != PAGE_SIZE)
 		return -EIO;
-	/* disallow writable mapping */
-	if (vma->vm_flags & VM_WRITE)
-		return -EPERM;
-	/* disallow mprotect() turns it into writable */
-	vm_flags_clear(vma, VM_MAYWRITE);
 
 	return remap_pfn_range(vma, vma->vm_start,
 			       page_to_pfn(status),
@@ -304,15 +298,14 @@ static int sel_make_bools(struct selinux_policy *newpolicy, struct dentry *bool_
 			  int **bool_pending_values);
 static int sel_make_classes(struct selinux_policy *newpolicy,
 			    struct dentry *class_dir,
-			    unsigned long *last_class_ino);
+			    u64 *last_class_ino);
 
 /* declaration for sel_make_class_dirs */
 static struct dentry *sel_make_dir(struct dentry *dir, const char *name,
-			unsigned long *ino);
+				   u64 *ino);
 
 /* declaration for sel_make_policy_nodes */
-static struct dentry *sel_make_swapover_dir(struct super_block *sb,
-						unsigned long *ino);
+static struct dentry *sel_make_swapover_dir(struct super_block *sb, u64 *ino);
 
 static ssize_t sel_read_mls(struct file *filp, char __user *buf,
 				size_t count, loff_t *ppos)
@@ -340,6 +333,9 @@ static int sel_open_policy(struct inode *inode, struct file *filp)
 	struct policy_load_memory *plm = NULL;
 	int rc;
 
+	if (filp->f_mode & FMODE_WRITE)
+		return -EACCES;
+
 	rc = avc_has_perm(current_sid(), SECINITSID_SECURITY,
 			  SECCLASS_SECURITY, SECURITY__READ_POLICY, NULL);
 	if (rc)
@@ -424,14 +420,6 @@ static const struct vm_operations_struct sel_mmap_policy_ops = {
 
 static int sel_mmap_policy(struct file *filp, struct vm_area_struct *vma)
 {
-	if (vma->vm_flags & VM_SHARED) {
-		/* do not allow mprotect to make mapping writable */
-		vm_flags_clear(vma, VM_MAYWRITE);
-
-		if (vma->vm_flags & VM_WRITE)
-			return -EACCES;
-	}
-
 	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP);
 	vma->vm_ops = &sel_mmap_policy_ops;
 
@@ -467,7 +455,7 @@ static int sel_make_policy_nodes(struct selinux_fs_info *fsi,
 	unsigned int bool_num = 0;
 	char **bool_names = NULL;
 	int *bool_values = NULL;
-	unsigned long tmp_ino = fsi->last_ino; /* Don't increment last_ino in this function */
+	u64 tmp_ino = fsi->last_ino; /* Don't increment last_ino in this function */
 
 	tmp_parent = sel_make_swapover_dir(fsi->sb, &tmp_ino);
 	if (IS_ERR(tmp_parent))
@@ -1568,30 +1556,32 @@ static int sel_make_initcon_files(struct dentry *dir)
 	return err;
 }
 
-static inline unsigned long sel_class_to_ino(u16 class)
+static inline u64 sel_class_to_ino(u16 class)
 {
 	return (class * (SEL_VEC_MAX + 1)) | SEL_CLASS_INO_OFFSET;
 }
 
-static inline u16 sel_ino_to_class(unsigned long ino)
+static inline u16 sel_ino_to_class(u64 ino)
 {
-	return (ino & SEL_INO_MASK) / (SEL_VEC_MAX + 1);
+	u32 ino_masked = ino & SEL_INO_MASK;
+	return ino_masked / (SEL_VEC_MAX + 1);
 }
 
-static inline unsigned long sel_perm_to_ino(u16 class, u32 perm)
+static inline u64 sel_perm_to_ino(u16 class, u32 perm)
 {
 	return (class * (SEL_VEC_MAX + 1) + perm) | SEL_CLASS_INO_OFFSET;
 }
 
-static inline u32 sel_ino_to_perm(unsigned long ino)
+static inline u32 sel_ino_to_perm(u64 ino)
 {
-	return (ino & SEL_INO_MASK) % (SEL_VEC_MAX + 1);
+	u32 ino_masked = ino & SEL_INO_MASK;
+	return ino_masked % (SEL_VEC_MAX + 1);
 }
 
 static ssize_t sel_read_class(struct file *file, char __user *buf,
 				size_t count, loff_t *ppos)
 {
-	unsigned long ino = file_inode(file)->i_ino;
+	u64 ino = file_inode(file)->i_ino;
 	char res[TMPBUFLEN];
 	ssize_t len = scnprintf(res, sizeof(res), "%d", sel_ino_to_class(ino));
 	return simple_read_from_buffer(buf, count, ppos, res, len);
@@ -1605,7 +1595,7 @@ static const struct file_operations sel_class_ops = {
 static ssize_t sel_read_perm(struct file *file, char __user *buf,
 				size_t count, loff_t *ppos)
 {
-	unsigned long ino = file_inode(file)->i_ino;
+	u64 ino = file_inode(file)->i_ino;
 	char res[TMPBUFLEN];
 	ssize_t len = scnprintf(res, sizeof(res), "%d", sel_ino_to_perm(ino));
 	return simple_read_from_buffer(buf, count, ppos, res, len);
@@ -1622,7 +1612,7 @@ static ssize_t sel_read_policycap(struct file *file, char __user *buf,
 	int value;
 	char tmpbuf[TMPBUFLEN];
 	ssize_t length;
-	unsigned long i_ino = file_inode(file)->i_ino;
+	u64 i_ino = file_inode(file)->i_ino;
 
 	value = security_policycap_supported(i_ino & SEL_INO_MASK);
 	length = scnprintf(tmpbuf, TMPBUFLEN, "%d", value);
@@ -1698,7 +1688,7 @@ static int sel_make_class_dir_entries(struct selinux_policy *newpolicy,
 
 static int sel_make_classes(struct selinux_policy *newpolicy,
 			    struct dentry *class_dir,
-			    unsigned long *last_class_ino)
+			    u64 *last_class_ino)
 {
 	u32 i, nclasses;
 	int rc;
@@ -1763,7 +1753,7 @@ static int sel_make_policycap(struct dentry *dir)
 }
 
 static struct dentry *sel_make_dir(struct dentry *dir, const char *name,
-			unsigned long *ino)
+				   u64 *ino)
 {
 	struct inode *inode;
 
@@ -1782,7 +1772,7 @@ static struct dentry *sel_make_dir(struct dentry *dir, const char *name,
 	return sel_attach(dir, name, inode);
 }
 
-static int reject_all(struct mnt_idmap *idmap, struct inode *inode, int mask)
+static int reject_all(const struct mnt_idmap *idmap, struct inode *inode, int mask)
 {
 	return -EPERM;	// no access for anyone, root or no root.
 }
@@ -1792,8 +1782,7 @@ static const struct inode_operations swapover_dir_inode_operations = {
 	.permission	= reject_all,
 };
 
-static struct dentry *sel_make_swapover_dir(struct super_block *sb,
-						unsigned long *ino)
+static struct dentry *sel_make_swapover_dir(struct super_block *sb, u64 *ino)
 {
 	struct dentry *dentry;
 	struct inode *inode;
@@ -1820,6 +1809,17 @@ static struct dentry *sel_make_swapover_dir(struct super_block *sb,
 
 #define NULL_FILE_NAME "null"
 
+static void sel_mark_immutable(struct dentry *root, const char *name)
+{
+	struct qstr q = QSTR(name);
+	struct dentry *dentry = try_lookup_noperm(&q, root);
+
+	if (!IS_ERR_OR_NULL(dentry)) {
+		d_inode(dentry)->i_flags |= S_IMMUTABLE;
+		dput(dentry);
+	}
+}
+
 static int sel_fill_super(struct super_block *sb, struct fs_context *fc)
 {
 	struct selinux_fs_info *fsi;
@@ -1859,6 +1859,9 @@ static int sel_fill_super(struct super_block *sb, struct fs_context *fc)
 	if (ret)
 		goto err;
 
+	sel_mark_immutable(sb->s_root, "status");
+	sel_mark_immutable(sb->s_root, "policy");
+
 	fsi = sb->s_fs_info;
 	fsi->bool_dir = sel_make_dir(sb->s_root, BOOL_DIR_NAME, &fsi->last_ino);
 	if (IS_ERR(fsi->bool_dir)) {
diff --git a/security/selinux/ss/mls.c b/security/selinux/ss/mls.c
index 40c62600650e8e..b48f8256dd4cf9 100644
--- a/security/selinux/ss/mls.c
+++ b/security/selinux/ss/mls.c
@@ -400,44 +400,6 @@ int mls_range_set(struct context *context, struct mls_range *range)
 	return rc;
 }
 
-int mls_setup_user_range(struct policydb *p, struct context *fromcon,
-			 struct user_datum *user, struct context *usercon)
-{
-	if (p->mls_enabled) {
-		struct mls_level *fromcon_sen = &(fromcon->range.level[0]);
-		struct mls_level *fromcon_clr = &(fromcon->range.level[1]);
-		struct mls_level *user_low = &(user->range.level[0]);
-		struct mls_level *user_clr = &(user->range.level[1]);
-		struct mls_level *user_def = &(user->dfltlevel);
-		struct mls_level *usercon_sen = &(usercon->range.level[0]);
-		struct mls_level *usercon_clr = &(usercon->range.level[1]);
-
-		/* Honor the user's default level if we can */
-		if (mls_level_between(user_def, fromcon_sen, fromcon_clr))
-			*usercon_sen = *user_def;
-		else if (mls_level_between(fromcon_sen, user_def, user_clr))
-			*usercon_sen = *fromcon_sen;
-		else if (mls_level_between(fromcon_clr, user_low, user_def))
-			*usercon_sen = *user_low;
-		else
-			return -EINVAL;
-
-		/* Lower the clearance of available contexts
-		   if the clearance of "fromcon" is lower than
-		   that of the user's default clearance (but
-		   only if the "fromcon" clearance dominates
-		   the user's computed sensitivity level) */
-		if (mls_level_dom(user_clr, fromcon_clr))
-			*usercon_clr = *fromcon_clr;
-		else if (mls_level_dom(fromcon_clr, user_clr))
-			*usercon_clr = *user_clr;
-		else
-			return -EINVAL;
-	}
-
-	return 0;
-}
-
 /*
  * Convert the MLS fields in the security context
  * structure `oldc' from the values specified in the
diff --git a/security/selinux/ss/mls.h b/security/selinux/ss/mls.h
index 93cde1b22992ca..c107b5182fb7b3 100644
--- a/security/selinux/ss/mls.h
+++ b/security/selinux/ss/mls.h
@@ -46,9 +46,6 @@ int mls_compute_sid(struct policydb *p, struct context *scontext,
 		    struct context *tcontext, u16 tclass, u32 specified,
 		    struct context *newcontext, bool sock);
 
-int mls_setup_user_range(struct policydb *p, struct context *fromcon,
-			 struct user_datum *user, struct context *usercon);
-
 #ifdef CONFIG_NETLABEL
 void mls_export_netlbl_lvl(struct policydb *p, struct context *context,
 			   struct netlbl_lsm_secattr *secattr);
diff --git a/security/selinux/ss/mls_types.h b/security/selinux/ss/mls_types.h
index 51df2ebd12115e..d36a03f104c231 100644
--- a/security/selinux/ss/mls_types.h
+++ b/security/selinux/ss/mls_types.h
@@ -42,9 +42,6 @@ static inline int mls_level_dom(const struct mls_level *l1,
 #define mls_level_incomp(l1, l2) \
 	(!mls_level_dom((l1), (l2)) && !mls_level_dom((l2), (l1)))
 
-#define mls_level_between(l1, l2, l3) \
-	(mls_level_dom((l1), (l2)) && mls_level_dom((l3), (l1)))
-
 #define mls_range_contains(r1, r2)                        \
 	(mls_level_dom(&(r2).level[0], &(r1).level[0]) && \
 	 mls_level_dom(&(r1).level[1], &(r2).level[1]))
diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c
index b75c977c6673db..13d88e47db3c9f 100644
--- a/security/selinux/ss/policydb.c
+++ b/security/selinux/ss/policydb.c
@@ -560,6 +560,9 @@ static int common_index(void *key, void *datum, void *datap)
 	if (!comdatum->value || comdatum->value > p->p_commons.nprim)
 		return -EINVAL;
 
+	if (p->sym_val_to_name[SYM_COMMONS][comdatum->value - 1])
+		return -EINVAL;
+
 	p->sym_val_to_name[SYM_COMMONS][comdatum->value - 1] = key;
 
 	return 0;
@@ -575,6 +578,9 @@ static int class_index(void *key, void *datum, void *datap)
 	if (!cladatum->value || cladatum->value > p->p_classes.nprim)
 		return -EINVAL;
 
+	if (p->class_val_to_struct[cladatum->value - 1])
+		return -EINVAL;
+
 	p->sym_val_to_name[SYM_CLASSES][cladatum->value - 1] = key;
 	p->class_val_to_struct[cladatum->value - 1] = cladatum;
 	return 0;
@@ -591,6 +597,9 @@ static int role_index(void *key, void *datum, void *datap)
 	    role->bounds > p->p_roles.nprim)
 		return -EINVAL;
 
+	if (p->role_val_to_struct[role->value - 1])
+		return -EINVAL;
+
 	p->sym_val_to_name[SYM_ROLES][role->value - 1] = key;
 	p->role_val_to_struct[role->value - 1] = role;
 	return 0;
@@ -613,6 +622,8 @@ static int type_index(void *key, void *datum, void *datap)
 	}
 
 	if (typdatum->primary) {
+		if (p->type_val_to_struct[typdatum->value - 1])
+			return -EINVAL;
 		p->sym_val_to_name[SYM_TYPES][typdatum->value - 1] = key;
 		p->type_val_to_struct[typdatum->value - 1] = typdatum;
 	}
@@ -631,6 +642,9 @@ static int user_index(void *key, void *datum, void *datap)
 	    usrdatum->bounds > p->p_users.nprim)
 		return -EINVAL;
 
+	if (p->user_val_to_struct[usrdatum->value - 1])
+		return -EINVAL;
+
 	p->sym_val_to_name[SYM_USERS][usrdatum->value - 1] = key;
 	p->user_val_to_struct[usrdatum->value - 1] = usrdatum;
 	return 0;
@@ -647,8 +661,11 @@ static int sens_index(void *key, void *datum, void *datap)
 	if (!levdatum->level.sens || levdatum->level.sens > p->p_levels.nprim)
 		return -EINVAL;
 
-	if (!levdatum->isalias)
+	if (!levdatum->isalias) {
+		if (p->sym_val_to_name[SYM_LEVELS][levdatum->level.sens - 1])
+			return -EINVAL;
 		p->sym_val_to_name[SYM_LEVELS][levdatum->level.sens - 1] = key;
+	}
 
 	return 0;
 }
@@ -664,8 +681,11 @@ static int cat_index(void *key, void *datum, void *datap)
 	if (!catdatum->value || catdatum->value > p->p_cats.nprim)
 		return -EINVAL;
 
-	if (!catdatum->isalias)
+	if (!catdatum->isalias) {
+		if (p->sym_val_to_name[SYM_CATS][catdatum->value - 1])
+			return -EINVAL;
 		p->sym_val_to_name[SYM_CATS][catdatum->value - 1] = key;
+	}
 
 	return 0;
 }
@@ -1037,13 +1057,13 @@ bool policydb_context_isvalid(const struct policydb *p, const struct context *c)
 	const struct role_datum *role;
 	const struct user_datum *usrdatum;
 
-	if (!c->role || c->role > p->p_roles.nprim)
+	if (!policydb_role_isvalid(p, c->role))
 		return false;
 
-	if (!c->user || c->user > p->p_users.nprim)
+	if (!policydb_user_isvalid(p, c->user))
 		return false;
 
-	if (!c->type || c->type > p->p_types.nprim)
+	if (!policydb_simpletype_isvalid(p, c->type))
 		return false;
 
 	if (c->role != OBJECT_R_VAL) {
@@ -1051,7 +1071,7 @@ bool policydb_context_isvalid(const struct policydb *p, const struct context *c)
 		 * Role must be authorized for the type.
 		 */
 		role = p->role_val_to_struct[c->role - 1];
-		if (!role || !ebitmap_get_bit(&role->types, c->type - 1))
+		if (!ebitmap_get_bit(&role->types, c->type - 1))
 			/* role may not be associated with type */
 			return false;
 
@@ -1059,9 +1079,6 @@ bool policydb_context_isvalid(const struct policydb *p, const struct context *c)
 		 * User must be authorized for the role.
 		 */
 		usrdatum = p->user_val_to_struct[c->user - 1];
-		if (!usrdatum)
-			return false;
-
 		if (!ebitmap_get_bit(&usrdatum->roles, c->role - 1))
 			/* user may not be associated with role */
 			return false;
@@ -2350,6 +2367,10 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp
 		if (rc)
 			goto out;
 
+		rc = -EINVAL;
+		if (ebitmap_get_highest_set_bit(&datum->stypes) >= p->p_types.nprim)
+			goto out;
+
 		rc = next_entry(buf, fp, sizeof(u32));
 		if (rc)
 			goto out;
@@ -2847,7 +2868,8 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 		goto bad;
 	}
 
-	if ((le32_to_cpu(buf[1]) & POLICYDB_CONFIG_MLS)) {
+	val = le32_to_cpu(buf[1]);
+	if (val & POLICYDB_CONFIG_MLS) {
 		p->mls_enabled = 1;
 
 		rc = -EINVAL;
@@ -2858,8 +2880,13 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 			goto bad;
 		}
 	}
-	p->reject_unknown = !!(le32_to_cpu(buf[1]) & REJECT_UNKNOWN);
-	p->allow_unknown = !!(le32_to_cpu(buf[1]) & ALLOW_UNKNOWN);
+	rc = -EINVAL;
+	if ((val & (REJECT_UNKNOWN | ALLOW_UNKNOWN)) == (REJECT_UNKNOWN | ALLOW_UNKNOWN)) {
+		pr_err("SELinux:  policydb configuration both rejects and allows unknown classes and permissions\n");
+		goto bad;
+	}
+	p->reject_unknown = !!(val & REJECT_UNKNOWN);
+	p->allow_unknown = !!(val & ALLOW_UNKNOWN);
 
 	if (p->policyvers >= POLICYDB_VERSION_POLCAP) {
 		rc = ebitmap_read(&p->policycaps, fp);
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index 8e88ac65fd7fcf..3524649f402da9 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -16,7 +16,6 @@
  */
 
 #include <linux/xattr.h>
-#include <linux/pagemap.h>
 #include <linux/mount.h>
 #include <linux/stat.h>
 #include <linux/kd.h>
@@ -1089,14 +1088,15 @@ static int smack_inode_init_security(struct inode *inode, struct inode *dir,
 
 /**
  * smack_inode_link - Smack check on link
+ * @idmap: idmap of the mount
  * @old_dentry: the existing object
  * @dir: unused
  * @new_dentry: the new object
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_link(struct dentry *old_dentry, struct inode *dir,
-			    struct dentry *new_dentry)
+static int smack_inode_link(const struct mnt_idmap *idmap, struct dentry *old_dentry,
+			    struct inode *dir, struct dentry *new_dentry)
 {
 	struct smack_known *isp;
 	struct smk_audit_info ad;
@@ -1226,6 +1226,7 @@ static int smack_inode_rename(struct inode *old_inode,
 
 /**
  * smack_inode_permission - Smack version of permission()
+ * @idmap: idmap of the mount
  * @inode: the inode in question
  * @mask: the access requested
  *
@@ -1233,7 +1234,8 @@ static int smack_inode_rename(struct inode *old_inode,
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_permission(struct inode *inode, int mask)
+static int smack_inode_permission(const struct mnt_idmap *idmap, struct inode *inode,
+				  int mask)
 {
 	struct superblock_smack *sbsp = smack_superblock(inode->i_sb);
 	struct smk_audit_info ad;
@@ -1270,7 +1272,7 @@ static int smack_inode_permission(struct inode *inode, int mask)
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+static int smack_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
 			       struct iattr *iattr)
 {
 	struct smk_audit_info ad;
@@ -1348,7 +1350,7 @@ static int smack_inode_xattr_skipcap(const char *name)
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_setxattr(struct mnt_idmap *idmap,
+static int smack_inode_setxattr(const struct mnt_idmap *idmap,
 				struct dentry *dentry, const char *name,
 				const void *value, size_t size, int flags)
 {
@@ -1482,7 +1484,7 @@ static int smack_inode_getxattr(struct dentry *dentry, const char *name)
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_removexattr(struct mnt_idmap *idmap,
+static int smack_inode_removexattr(const struct mnt_idmap *idmap,
 				   struct dentry *dentry, const char *name)
 {
 	struct inode_smack *isp;
@@ -1543,7 +1545,7 @@ static int smack_inode_removexattr(struct mnt_idmap *idmap,
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_set_acl(struct mnt_idmap *idmap,
+static int smack_inode_set_acl(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, const char *acl_name,
 			       struct posix_acl *kacl)
 {
@@ -1566,7 +1568,7 @@ static int smack_inode_set_acl(struct mnt_idmap *idmap,
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_get_acl(struct mnt_idmap *idmap,
+static int smack_inode_get_acl(const struct mnt_idmap *idmap,
 			       struct dentry *dentry, const char *acl_name)
 {
 	struct smk_audit_info ad;
@@ -1588,7 +1590,7 @@ static int smack_inode_get_acl(struct mnt_idmap *idmap,
  *
  * Returns 0 if access is permitted, an error code otherwise
  */
-static int smack_inode_remove_acl(struct mnt_idmap *idmap,
+static int smack_inode_remove_acl(const struct mnt_idmap *idmap,
 				  struct dentry *dentry, const char *acl_name)
 {
 	struct smk_audit_info ad;
@@ -1612,7 +1614,7 @@ static int smack_inode_remove_acl(struct mnt_idmap *idmap,
  *
  * Returns the size of the attribute or an error code
  */
-static int smack_inode_getsecurity(struct mnt_idmap *idmap,
+static int smack_inode_getsecurity(const struct mnt_idmap *idmap,
 				   struct inode *inode, const char *name,
 				   void **buffer, bool alloc)
 {
diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h
index d098cf8aae6109..d2603400091387 100644
--- a/security/tomoyo/common.h
+++ b/security/tomoyo/common.h
@@ -16,7 +16,7 @@
 #include <linux/string.h>
 #include <linux/mm.h>
 #include <linux/file.h>
-#include <linux/kmod.h>
+#include <linux/umh.h>
 #include <linux/fs.h>
 #include <linux/sched.h>
 #include <linux/namei.h>
diff --git a/sound/core/control.c b/sound/core/control.c
index 4199342d4ffe09..535ceba294acf8 100644
--- a/sound/core/control.c
+++ b/sound/core/control.c
@@ -1802,6 +1802,8 @@ static int snd_ctl_elem_add(struct snd_ctl_file *file,
 	alloc_size = compute_user_elem_size(private_size, count);
 
 	guard(rwsem_write)(&card->controls_rwsem);
+	if (card->shutdown)
+		return -ENODEV;
 	if (check_user_elem_overflow(card, alloc_size))
 		return -ENOMEM;
 
diff --git a/sound/core/init.c b/sound/core/init.c
index 1bcb6a2e75507a..a51b71812fd9b3 100644
--- a/sound/core/init.c
+++ b/sound/core/init.c
@@ -1174,12 +1174,12 @@ EXPORT_SYMBOL(snd_card_file_remove);
 int snd_power_ref_and_wait(struct snd_card *card)
 {
 	snd_power_ref(card);
-	if (snd_power_get_state(card) == SNDRV_CTL_POWER_D0)
-		return 0;
-	wait_event_cmd(card->power_sleep,
-		       card->shutdown ||
-		       snd_power_get_state(card) == SNDRV_CTL_POWER_D0,
-		       snd_power_unref(card), snd_power_ref(card));
+	if (snd_power_get_state(card) != SNDRV_CTL_POWER_D0) {
+		wait_event_cmd(card->power_sleep,
+			       card->shutdown ||
+			       snd_power_get_state(card) == SNDRV_CTL_POWER_D0,
+			       snd_power_unref(card), snd_power_ref(card));
+	}
 	if (card->shutdown) {
 		snd_power_unref(card);
 		return  -ENODEV;
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index 6d32c12fb79bdf..af801f0e320039 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -2267,7 +2267,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream,
 static int snd_pcm_drop(struct snd_pcm_substream *substream)
 {
 	struct snd_pcm_runtime *runtime;
-	int result = 0;
+	int result;
 	
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
@@ -2282,7 +2282,7 @@ static int snd_pcm_drop(struct snd_pcm_substream *substream)
 	if (runtime->state == SNDRV_PCM_STATE_PAUSED)
 		snd_pcm_pause(substream, false);
 
-	snd_pcm_stop(substream, SNDRV_PCM_STATE_SETUP);
+	result = snd_pcm_stop(substream, SNDRV_PCM_STATE_SETUP);
 	/* runtime->control->appl_ptr = runtime->status->hw_ptr; */
 
 	return result;
@@ -2930,7 +2930,7 @@ static int snd_pcm_open(struct file *file, struct snd_pcm *pcm, int stream)
 	mutex_lock(&pcm->open_mutex);
 	while (1) {
 		err = snd_pcm_open_file(file, pcm, stream);
-		if (err >= 0)
+		if (!err)
 			break;
 		if (err == -EAGAIN) {
 			if (file->f_flags & O_NONBLOCK) {
@@ -3760,39 +3760,27 @@ static __poll_t snd_pcm_poll(struct file *file, poll_table *wait)
 /*
  * mmap status record
  */
-static vm_fault_t snd_pcm_mmap_status_fault(struct vm_fault *vmf)
+static int snd_pcm_mmap_status(struct snd_pcm_substream *substream, struct file *file,
+			       struct vm_area_struct *vma)
 {
-	struct snd_pcm_substream *substream = vmf->vma->vm_private_data;
+	const unsigned long size = vma->vm_end - vma->vm_start;
 	struct snd_pcm_runtime *runtime;
-	
-	if (substream == NULL)
-		return VM_FAULT_SIGBUS;
-	runtime = substream->runtime;
-	vmf->page = virt_to_page(runtime->status);
-	get_page(vmf->page);
-	return 0;
-}
+	struct page *page;
 
-static const struct vm_operations_struct snd_pcm_vm_ops_status =
-{
-	.fault =	snd_pcm_mmap_status_fault,
-};
+	BUILD_BUG_ON(sizeof(struct snd_pcm_mmap_status) > PAGE_SIZE);
 
-static int snd_pcm_mmap_status(struct snd_pcm_substream *substream, struct file *file,
-			       struct vm_area_struct *area)
-{
-	long size;
-	if (!(area->vm_flags & VM_READ))
+	if (!(vma->vm_flags & VM_READ))
 		return -EINVAL;
-	size = area->vm_end - area->vm_start;
-	if (size != PAGE_ALIGN(sizeof(struct snd_pcm_mmap_status)))
+	if (size != PAGE_SIZE)
 		return -EINVAL;
-	area->vm_ops = &snd_pcm_vm_ops_status;
-	area->vm_private_data = substream;
-	vm_flags_mod(area, VM_DONTEXPAND | VM_DONTDUMP,
+
+	vm_flags_mod(vma, VM_DONTEXPAND | VM_DONTDUMP,
 		     VM_WRITE | VM_MAYWRITE);
+	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
 
-	return 0;
+	runtime = substream->runtime;
+	page = virt_to_page(runtime->status);
+	return vm_insert_page(vma, vma->vm_start, page);
 }
 
 /*
diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c
index 2617bb5b4faf50..c43cedb5c12158 100644
--- a/sound/core/rawmidi.c
+++ b/sound/core/rawmidi.c
@@ -36,6 +36,13 @@ module_param_array(amidi_map, int, NULL, 0444);
 MODULE_PARM_DESC(amidi_map, "Raw MIDI device number assigned to 2nd OSS device.");
 #endif /* CONFIG_SND_OSSEMUL */
 
+/* upper bound for draining the output buffer */
+#define SNDRV_RAWMIDI_DRAIN_TIMEOUT	(10 * HZ)
+/* interval at which drain progress is re-checked */
+#define SNDRV_RAWMIDI_DRAIN_POLL	(HZ / 5)
+/* polls without progress before the drain is considered stalled */
+#define SNDRV_RAWMIDI_DRAIN_STALLS	5
+
 static int snd_rawmidi_dev_free(struct snd_device *device);
 static int snd_rawmidi_dev_register(struct snd_device *device);
 static int snd_rawmidi_dev_disconnect(struct snd_device *device);
@@ -208,14 +215,14 @@ static int snd_rawmidi_runtime_free(struct snd_rawmidi_substream *substream)
 
 static inline void snd_rawmidi_output_trigger(struct snd_rawmidi_substream *substream, int up)
 {
-	if (!substream->opened)
+	if (!substream->opened || substream->disconnected)
 		return;
 	substream->ops->trigger(substream, up);
 }
 
 static void snd_rawmidi_input_trigger(struct snd_rawmidi_substream *substream, int up)
 {
-	if (!substream->opened)
+	if (!substream->opened || substream->disconnected)
 		return;
 	substream->ops->trigger(substream, up);
 	if (!up)
@@ -246,31 +253,74 @@ int snd_rawmidi_drop_output(struct snd_rawmidi_substream *substream)
 }
 EXPORT_SYMBOL(snd_rawmidi_drop_output);
 
+static bool output_drained(struct snd_rawmidi_runtime *runtime)
+{
+	return runtime->avail >= runtime->buffer_size;
+}
+
 int snd_rawmidi_drain_output(struct snd_rawmidi_substream *substream)
 {
-	int err = 0;
-	long timeout;
 	struct snd_rawmidi_runtime *runtime;
+	size_t avail, prev_avail;
+	unsigned int stalls = 0;
+	unsigned long deadline;
+	long timeout, wait;
+	bool done;
+	int err = 0;
 
 	scoped_guard(spinlock_irq, &substream->lock) {
 		runtime = substream->runtime;
-		if (!substream->opened || !runtime || !runtime->buffer)
+		if (!substream->opened || !runtime || !runtime->buffer ||
+		    substream->disconnected)
 			return -EINVAL;
 		snd_rawmidi_buffer_ref(runtime);
 		runtime->drain = 1;
+		prev_avail = runtime->avail;
+	}
+
+	/*
+	 * Wait for the device to consume the buffer.  Rather than always
+	 * sleeping for the whole timeout, sample the free space and stop
+	 * early once it has not increased for a second.  A substream that
+	 * is still making progress is given as long as it needs, within the
+	 * same overall limit as before.
+	 */
+	deadline = jiffies + SNDRV_RAWMIDI_DRAIN_TIMEOUT;
+	for (;;) {
+		/* signed difference, so this is safe across a jiffies wrap */
+		wait = (long)(deadline - jiffies);
+		if (wait <= 0) {
+			timeout = 0;
+			break;
+		}
+		wait = min_t(long, SNDRV_RAWMIDI_DRAIN_POLL, wait);
+		timeout = wait_event_interruptible_timeout(runtime->sleep,
+							   output_drained(runtime), wait);
+		scoped_guard(spinlock_irq, &substream->lock) {
+			avail = runtime->avail;
+			done = output_drained(runtime);
+		}
+		if (done || signal_pending(current))
+			break;
+		if (avail == prev_avail) {
+			if (++stalls >= SNDRV_RAWMIDI_DRAIN_STALLS) {
+				timeout = 0;
+				break;
+			}
+		} else {
+			stalls = 0;
+			prev_avail = avail;
+		}
 	}
 
-	timeout = wait_event_interruptible_timeout(runtime->sleep,
-				(runtime->avail >= runtime->buffer_size),
-				10*HZ);
-
 	scoped_guard(spinlock_irq, &substream->lock) {
 		if (signal_pending(current))
 			err = -ERESTARTSYS;
 		if (runtime->avail < runtime->buffer_size && !timeout) {
-			rmidi_warn(substream->rmidi,
-				   "rawmidi drain error (avail = %li, buffer_size = %li)\n",
-				   (long)runtime->avail, (long)runtime->buffer_size);
+			dev_warn_ratelimited(substream->rmidi->dev,
+					     "rawmidi drain error (avail = %li, buffer_size = %li)\n",
+					     (long)runtime->avail,
+					     (long)runtime->buffer_size);
 			err = -EIO;
 		}
 		runtime->drain = 0;
@@ -537,7 +587,7 @@ static void close_substream(struct snd_rawmidi *rmidi,
 	if (--substream->use_count)
 		return;
 
-	if (cleanup) {
+	if (cleanup && !substream->disconnected) {
 		if (substream->stream == SNDRV_RAWMIDI_STREAM_INPUT)
 			snd_rawmidi_input_trigger(substream, 0);
 		else {
@@ -1151,7 +1201,7 @@ int snd_rawmidi_receive(struct snd_rawmidi_substream *substream,
 	struct snd_rawmidi_runtime *runtime;
 
 	guard(spinlock_irqsave)(&substream->lock);
-	if (!substream->opened)
+	if (!substream->opened || substream->disconnected)
 		return -EBADFD;
 	runtime = substream->runtime;
 	if (!runtime || !runtime->buffer) {
@@ -2063,8 +2113,13 @@ static int snd_rawmidi_dev_disconnect(struct snd_device *device)
 		struct snd_rawmidi_substream *s;
 
 		list_for_each_entry(s, &rmidi->streams[dir].substreams, list) {
-			if (s->runtime)
+			scoped_guard(spinlock_irq, &s->lock)
+				s->disconnected = true;
+			if (s->runtime) {
+				s->ops->trigger(s, 0);
+				cancel_work_sync(&s->runtime->event_work);
 				wake_up(&s->runtime->sleep);
+			}
 		}
 	}
 
diff --git a/sound/core/seq/Kconfig b/sound/core/seq/Kconfig
index e4f58cb985d47c..0d606159103e71 100644
--- a/sound/core/seq/Kconfig
+++ b/sound/core/seq/Kconfig
@@ -27,14 +27,17 @@ config SND_SEQ_DUMMY
 	  will be called snd-seq-dummy.
 
 config SND_SEQUENCER_OSS
-	tristate "OSS Sequencer API"
+	tristate "OSS Sequencer API (deprecated)"
 	depends on SND_OSSEMUL
 	select SND_SEQ_MIDI_EVENT
 	help
 	  Say Y here to enable OSS sequencer emulation (both
 	  /dev/sequencer and /dev/music interfaces).
 
-	  Many programs still use the OSS API, so say Y.
+	  The OSS sequencer API is almost nowhere used, hence this option will
+	  be dropped soon.
+	  Use user-space wrapper (alsa-oss) or CUSE daemon (oss-seq-emu)
+	  instead of the kernel-side emulation.
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called snd-seq-oss.
diff --git a/sound/core/seq/oss/seq_oss.c b/sound/core/seq/oss/seq_oss.c
index 2835576040ed4f..64a7e62029ccb8 100644
--- a/sound/core/seq/oss/seq_oss.c
+++ b/sound/core/seq/oss/seq_oss.c
@@ -67,6 +67,9 @@ static int __init alsa_seq_oss_init(void)
 {
 	int rc;
 
+	pr_warn("seq-oss: kernel OSS sequencer emulation is deprecated\n");
+	pr_warn("seq-oss: Use user-space solution (alsa-oss or oss-seq-emu) instead.\n");
+
 	rc = register_device();
 	if (rc < 0)
 		goto error;
diff --git a/sound/core/seq/oss/seq_oss_init.c b/sound/core/seq/oss/seq_oss_init.c
index 1aece46c8b064c..6586e07431c35f 100644
--- a/sound/core/seq/oss/seq_oss_init.c
+++ b/sound/core/seq/oss/seq_oss_init.c
@@ -376,7 +376,7 @@ delete_seq_queue(int queue)
 
 
 /*
- * free device informations - private_free callback of port
+ * free device information - private_free callback of port
  */
 static void
 free_devinfo(void *private)
diff --git a/sound/core/seq/seq_compat.c b/sound/core/seq/seq_compat.c
index 22679dca9aae32..80110501da6ff5 100644
--- a/sound/core/seq/seq_compat.c
+++ b/sound/core/seq/seq_compat.c
@@ -44,7 +44,9 @@ static int snd_seq_call_port_info_ioctl(struct snd_seq_client *client, unsigned
 		return -EFAULT;
 	data->kernel = NULL;
 
-	err = snd_seq_kernel_client_ctl(client->number, cmd, data);
+	scoped_guard(mutex, &client->ioctl_mutex) {
+		err = snd_seq_kernel_client_ctl(client->number, cmd, data);
+	}
 	if (err < 0)
 		return err;
 
diff --git a/sound/core/timer.c b/sound/core/timer.c
index 679b2643567090..69a59b25fe0a80 100644
--- a/sound/core/timer.c
+++ b/sound/core/timer.c
@@ -2176,6 +2176,7 @@ static const struct file_operations snd_utimer_fops = {
 	.llseek		=	noop_llseek,
 	.release	=	snd_utimer_release,
 	.unlocked_ioctl	=	snd_utimer_ioctl,
+	.compat_ioctl	=	snd_utimer_ioctl,
 };
 
 static int snd_utimer_start(struct snd_timer *t)
diff --git a/sound/core/timer_compat.c b/sound/core/timer_compat.c
index 25ee81c1668b75..5bab00466de7f9 100644
--- a/sound/core/timer_compat.c
+++ b/sound/core/timer_compat.c
@@ -99,6 +99,7 @@ static long __snd_timer_user_ioctl_compat(struct file *file, unsigned int cmd,
 	case SNDRV_TIMER_IOCTL_PAUSE:
 	case SNDRV_TIMER_IOCTL_PAUSE_OLD:
 	case SNDRV_TIMER_IOCTL_NEXT_DEVICE:
+	case SNDRV_TIMER_IOCTL_CREATE:
 		return __snd_timer_user_ioctl(file, cmd, (unsigned long)argp, true);
 	case SNDRV_TIMER_IOCTL_GPARAMS32:
 		return snd_timer_user_gparams_compat(file, argp);
diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c
index 3f6bfee2998634..6b0a06e5c87638 100644
--- a/sound/drivers/dummy.c
+++ b/sound/drivers/dummy.c
@@ -1063,11 +1063,13 @@ static int snd_dummy_probe(struct platform_device *devptr)
 			break;
 		}
 	}
+
+	if (pcm_substreams[dev] < 1)
+		pcm_substreams[dev] = 1;
+	if (pcm_substreams[dev] > MAX_PCM_SUBSTREAMS)
+		pcm_substreams[dev] = MAX_PCM_SUBSTREAMS;
+
 	for (idx = 0; idx < MAX_PCM_DEVICES && idx < pcm_devs[dev]; idx++) {
-		if (pcm_substreams[dev] < 1)
-			pcm_substreams[dev] = 1;
-		if (pcm_substreams[dev] > MAX_PCM_SUBSTREAMS)
-			pcm_substreams[dev] = MAX_PCM_SUBSTREAMS;
 		err = snd_card_dummy_pcm(dummy, idx, pcm_substreams[dev]);
 		if (err < 0)
 			return err;
diff --git a/sound/drivers/mpu401/mpu401_uart.c b/sound/drivers/mpu401/mpu401_uart.c
index 6de03d72a90874..fcf2b686f9f5e2 100644
--- a/sound/drivers/mpu401/mpu401_uart.c
+++ b/sound/drivers/mpu401/mpu401_uart.c
@@ -155,20 +155,20 @@ static void snd_mpu401_uart_timer(struct timer_list *t)
 
 	scoped_guard(spinlock_irqsave, &mpu->timer_lock) {
 		/*mpu->mode |= MPU401_MODE_TIMER;*/
-		mod_timer(&mpu->timer,  1 + jiffies);
+		if (mpu->timer_invoked)
+			mod_timer(&mpu->timer,  1 + jiffies);
 	}
 	if (mpu->rmidi)
 		_snd_mpu401_uart_interrupt(mpu);
 }
 
 /*
- * initialize the timer callback if not programmed yet
+ * arm the timer if it is not already active
  */
 static void snd_mpu401_uart_add_timer (struct snd_mpu401 *mpu, int input)
 {
 	guard(spinlock_irqsave)(&mpu->timer_lock);
 	if (mpu->timer_invoked == 0) {
-		timer_setup(&mpu->timer, snd_mpu401_uart_timer, 0);
 		mod_timer(&mpu->timer, 1 + jiffies);
 	} 
 	mpu->timer_invoked |= input ? MPU401_MODE_INPUT_TIMER :
@@ -474,6 +474,7 @@ static void snd_mpu401_uart_free(struct snd_rawmidi *rmidi)
 	struct snd_mpu401 *mpu = rmidi->private_data;
 	if (mpu->irq >= 0)
 		free_irq(mpu->irq, (void *) mpu);
+	timer_shutdown_sync(&mpu->timer);
 	release_and_free_resource(mpu->res);
 	kfree(mpu);
 }
@@ -528,6 +529,7 @@ int snd_mpu401_uart_new(struct snd_card *card, int device,
 	spin_lock_init(&mpu->input_lock);
 	spin_lock_init(&mpu->output_lock);
 	spin_lock_init(&mpu->timer_lock);
+	timer_setup(&mpu->timer, snd_mpu401_uart_timer, 0);
 	mpu->hardware = hardware;
 	mpu->irq = -1;
 	mpu->rmidi = rmidi;
diff --git a/sound/drivers/serial-generic.c b/sound/drivers/serial-generic.c
index 766206c6ca75a8..8ba5e35af4a154 100644
--- a/sound/drivers/serial-generic.c
+++ b/sound/drivers/serial-generic.c
@@ -183,10 +183,7 @@ static void snd_serial_generic_input_trigger(struct snd_rawmidi_substream *subst
 {
 	struct snd_serial_generic *drvdata = substream->rmidi->card->private_data;
 
-	if (up)
-		set_bit(SERIAL_MODE_INPUT_TRIGGERED, &drvdata->filemode);
-	else
-		clear_bit(SERIAL_MODE_INPUT_TRIGGERED, &drvdata->filemode);
+	assign_bit(SERIAL_MODE_INPUT_TRIGGERED, &drvdata->filemode, up);
 }
 
 static int snd_serial_generic_output_open(struct snd_rawmidi_substream *substream)
@@ -230,10 +227,7 @@ static void snd_serial_generic_output_trigger(struct snd_rawmidi_substream *subs
 {
 	struct snd_serial_generic *drvdata = substream->rmidi->card->private_data;
 
-	if (up)
-		set_bit(SERIAL_MODE_OUTPUT_TRIGGERED, &drvdata->filemode);
-	else
-		clear_bit(SERIAL_MODE_OUTPUT_TRIGGERED, &drvdata->filemode);
+	assign_bit(SERIAL_MODE_OUTPUT_TRIGGERED, &drvdata->filemode, up);
 
 	if (up)
 		snd_serial_generic_tx_wakeup(drvdata);
diff --git a/sound/firewire/bebob/bebob_hwdep.c b/sound/firewire/bebob/bebob_hwdep.c
index 216d1fceb6e737..1c523525fad2ea 100644
--- a/sound/firewire/bebob/bebob_hwdep.c
+++ b/sound/firewire/bebob/bebob_hwdep.c
@@ -8,7 +8,7 @@
 /*
  * This codes give three functionality.
  *
- * 1.get firewire node infomation
+ * 1.get firewire node information
  * 2.get notification about starting/stopping stream
  * 3.lock/unlock stream
  */
diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c
index 9bfdbf5c903259..91cbd728372f1c 100644
--- a/sound/hda/codecs/conexant.c
+++ b/sound/hda/codecs/conexant.c
@@ -312,6 +312,7 @@ enum {
 	CXT_FIXUP_HP_A_U,
 	CXT_FIXUP_ACER_SWIFT_HP,
 	CXT_FIXUP_HUAWEI_MATEBOOK_HP,
+	CXT_FIXUP_MATEBOOK14_SN6140,
 };
 
 /* for hda_fixup_thinkpad_acpi() */
@@ -817,6 +818,67 @@ static void cxt_fixup_hp_a_u(struct hda_codec *codec,
 		cxt_setup_gpio_unmute(codec, 0x2);
 }
 
+/* Huawei MateBook 14 (SN6140): the speaker output follows the headphone
+ * pin's connection select in hardware.  Both pins must be routed to the
+ * shared DAC 0x10, which carries the playback stream; the generic
+ * parser mutes that DAC when no headphone is plugged, silencing the
+ * speaker.  Suppress the auto-mute, bind the speaker pin to the shared
+ * DAC, remove its mute switch and keep it unmuted; the auto-mute hook
+ * re-applies the routing after every output update.
+ */
+static void cxt_mb14_route_unmute(struct hda_codec *codec)
+{
+	/* route both pins to the shared DAC 0x10, enable the speaker
+	 * EAPD and unmute the DAC at a moderate gain; the vmaster
+	 * volume takes over once userspace starts
+	 */
+	snd_hda_codec_write(codec, 0x16, 0, AC_VERB_SET_CONNECT_SEL, 0);
+	snd_hda_codec_write(codec, 0x17, 0, AC_VERB_SET_CONNECT_SEL, 0);
+	snd_hda_codec_write(codec, 0x17, 0, AC_VERB_SET_EAPD_BTLENABLE, 0x02);
+	snd_hda_codec_amp_stereo(codec, 0x10, HDA_OUTPUT, 0,
+				 HDA_AMP_VOLMASK | HDA_AMP_MUTE, 0x40);
+}
+
+static void cxt_mb14_automute(struct hda_codec *codec)
+{
+	snd_hda_gen_update_outputs(codec);
+	cxt_mb14_route_unmute(codec);
+}
+
+static void cxt_fixup_matebook14_sn6140(struct hda_codec *codec,
+					const struct hda_fixup *fix, int action)
+{
+	struct conexant_spec *spec = codec->spec;
+
+	if (action == HDA_FIXUP_ACT_PRE_PROBE) {
+		static const hda_nid_t dac_0x10[] = { 0x10 };
+
+		spec->gen.suppress_auto_mute = 1;
+		spec->gen.automute_hook = cxt_mb14_automute;
+		/* Bind the speaker pin to the shared DAC 0x10.
+		 */
+		snd_hda_override_conn_list(codec, 0x17, 1, dac_0x10);
+		return;
+	}
+	if (action == HDA_FIXUP_ACT_PROBE) {
+		unsigned int caps;
+
+		/* Remove the mute switch on the shared DAC so that
+		 * userspace (WirePlumber) cannot mute it when the
+		 * headphone route becomes unavailable (e.g. after
+		 * resume).  Must be done here: the codec regmap is not
+		 * available at pre-probe time.
+		 */
+		caps = query_amp_caps(codec, 0x10, HDA_OUTPUT);
+		snd_hda_override_amp_caps(codec, 0x10, HDA_OUTPUT,
+					  caps & ~AC_AMPCAP_MUTE);
+		return;
+	}
+	if (action != HDA_FIXUP_ACT_INIT)
+		return;
+	cxt_mb14_route_unmute(codec);
+}
+
 /* ThinkPad X200 & co with cxt5051 */
 static const struct hda_pintbl cxt_pincfg_lenovo_x200[] = {
 	{ 0x16, 0x042140ff }, /* HP (seq# overridden) */
@@ -1062,6 +1124,10 @@ static const struct hda_fixup cxt_fixups[] = {
 			{ }
 		},
 	},
+	[CXT_FIXUP_MATEBOOK14_SN6140] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = cxt_fixup_matebook14_sn6140,
+	},
 };
 
 static const struct hda_quirk cxt5045_fixups[] = {
@@ -1164,6 +1230,7 @@ static const struct hda_quirk cxt5066_fixups[] = {
 	SND_PCI_QUIRK(0x17aa, 0x397b, "Lenovo S205", CXT_FIXUP_STEREO_DMIC),
 	SND_PCI_QUIRK_VENDOR(0x17aa, "Thinkpad/Ideapad", CXT_FIXUP_LENOVO_XPAD_ACPI),
 	SND_PCI_QUIRK(0x19e5, 0x3289, "Huawei Matebook", CXT_FIXUP_HUAWEI_MATEBOOK_HP),
+	SND_PCI_QUIRK(0x19e5, 0x329e, "Huawei MateBook 14", CXT_FIXUP_MATEBOOK14_SN6140),
 	SND_PCI_QUIRK(0x1c06, 0x2011, "Lemote A1004", CXT_PINCFG_LEMOTE_A1004),
 	SND_PCI_QUIRK(0x1c06, 0x2012, "Lemote A1205", CXT_PINCFG_LEMOTE_A1205),
 	SND_PCI_QUIRK(0x1d05, 0x3012, "MECHREVO Wujie 15X Pro", CXT_FIXUP_HEADSET_MIC),
diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c
index 1e7a05c8773311..a8aac0b36c73f9 100644
--- a/sound/hda/codecs/hdmi/hdmi.c
+++ b/sound/hda/codecs/hdmi/hdmi.c
@@ -1671,7 +1671,7 @@ int snd_hda_hdmi_generic_pcm_prepare(struct hda_pcm_stream *hinfo,
 	 * After S3, the audio driver restores pin:cvt selections
 	 * but this can happen before gfx is ready and such selection
 	 * is overlooked by HW. Thus multiple pins can share a same
-	 * default convertor and mute control will affect each other,
+	 * default converter and mute control will affect each other,
 	 * which can cause a resumed audio playback become silent
 	 * after S3.
 	 */
diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c
index 5127a111c59cd7..67b9f41720abec 100644
--- a/sound/hda/codecs/realtek/alc269.c
+++ b/sound/hda/codecs/realtek/alc269.c
@@ -1840,6 +1840,48 @@ struct alc298_samsung_amp_desc {
 	unsigned short init_seq[2][2];
 };
 
+/*
+ * The MSI GE66 Raider 11UE drives its speakers through a Realtek ALC1316 smart
+ * amplifier, reached over the ALC298 coefficient tunnel. Left unconfigured the
+ * amplifier's silence detection mutes the speaker path after about 0.2 s of
+ * quiet and takes about 0.4 s to reopen, so the start of every sound that
+ * follows a pause is lost. Headphones do not pass through the amplifier and are
+ * unaffected.
+ *
+ * Tunnel layout for ALC286/ALC298: coefficient 0x22 selects the amplifier, 0x23
+ * carries the 16-bit register, 0x24 and 0x25 the value's high and low halves,
+ * and 0x26 commits. Bit 4 of the commit word is a self-clearing go bit.
+ */
+static void alc298_fixup_alc1316_amp(struct hda_codec *codec,
+				     const struct hda_fixup *fix, int action)
+{
+	static const unsigned short init_seq[][2] = {
+		{ 0x0000, 0x01 }, { 0xc001, 0x14 }, { 0xc003, 0x00 }, { 0xc004, 0x11 },
+		{ 0xc005, 0x00 }, { 0xc006, 0x00 }, { 0xc007, 0x11 }, { 0xc008, 0x11 },
+		{ 0xc009, 0x00 }, { 0xc730, 0x06 }, { 0xc612, 0x16 }, { 0xc00f, 0xf7 },
+		{ 0xc00a, 0xd8 }, { 0xc00b, 0xb7 }, { 0xc60b, 0xff }, { 0xc60c, 0x37 },
+		{ 0xc605, 0xf0 }, { 0xc020, 0x00 }, { 0xc022, 0xd6 }, { 0xc023, 0x00 },
+		{ 0xc025, 0xd6 }, { 0xc602, 0x07 }, { 0xc603, 0x07 }, { 0xd101, 0x00 },
+		{ 0xc093, 0x80 }, { 0xc090, 0x87 }, { 0xc091, 0x11 }, { 0xc614, 0x20 },
+		{ 0xc615, 0x0a }, { 0xc616, 0x02 }, { 0xc617, 0x00 }, { 0xc050, 0x80 },
+		{ 0xc051, 0x7f }, { 0xc052, 0x00 }, { 0xc053, 0x70 }, { 0xc054, 0x00 },
+		{ 0xc0a2, 0x01 }, { 0xc09d, 0x80 }, { 0xc09c, 0x71 }
+	};
+	int i;
+
+	if (action != HDA_FIXUP_ACT_INIT)
+		return;
+
+	alc_write_coef_idx(codec, 0x22, 0x0010);
+
+	for (i = 0; i < ARRAY_SIZE(init_seq); i++) {
+		alc_write_coef_idx(codec, 0x23, init_seq[i][0]);
+		alc_write_coef_idx(codec, 0x24, 0x0000);
+		alc_write_coef_idx(codec, 0x25, init_seq[i][1]);
+		alc_write_coef_idx(codec, 0x26, 0xb031);
+	}
+}
+
 static void alc298_fixup_samsung_amp(struct hda_codec *codec,
 				     const struct hda_fixup *fix, int action)
 {
@@ -2721,6 +2763,17 @@ static void alc294_fixup_bass_speaker_15(struct hda_codec *codec,
 	}
 }
 
+/* route Speaker (0x1b) to DAC 0x02, the only DAC reachable from Bass Speaker (0x14) */
+static void alc256_fixup_honor_dra_xx_share_dac(struct hda_codec *codec,
+						const struct hda_fixup *fix, int action)
+{
+	if (action == HDA_FIXUP_ACT_PRE_PROBE) {
+		static const hda_nid_t conn[] = { 0x02 };
+
+		snd_hda_override_conn_list(codec, 0x1b, ARRAY_SIZE(conn), conn);
+	}
+}
+
 /* Hook to update amp GPIO4 for automute */
 static void alc280_hp_gpio4_automute_hook(struct hda_codec *codec,
 					  struct hda_jack_callback *jack)
@@ -3433,6 +3486,28 @@ static void alc287_fixup_legion_16ithg6_speakers(struct hda_codec *cdc, const st
 	comp_generic_fixup(cdc, action, "i2c", "CLSA0101", "-%s:00-cs35l41-hda.%d", 2);
 }
 
+static void alc287_fixup_yoga_slim7_carbon_speakers(struct hda_codec *cdc,
+						    const struct hda_fixup *fix, int action)
+{
+	/*
+	 * The bass speakers are driven by two CS35L41 amps fed over I2S. The codec only
+	 * clocks the I2S bus while pin 0x17 is enabled, but the BIOS marks it unconnected.
+	 * Keep the pin on DAC 0x02 with the other speakers: DAC 0x06 gets powered down
+	 * with stereo streams, which stops the clock and the amps fail to power up.
+	 */
+	static const struct hda_pintbl pincfgs[] = {
+		{ 0x17, 0x90170121 },
+		{ }
+	};
+	static const hda_nid_t conn[] = { 0x02 };
+
+	if (action == HDA_FIXUP_ACT_PRE_PROBE) {
+		snd_hda_apply_pincfgs(cdc, pincfgs);
+		snd_hda_override_conn_list(cdc, 0x17, ARRAY_SIZE(conn), conn);
+	}
+	comp_generic_fixup(cdc, action, "i2c", "CLSA0102", "-%s:00-cs35l41-hda.%d", 2);
+}
+
 static void alc285_fixup_asus_ga403u(struct hda_codec *cdc, const struct hda_fixup *fix, int action)
 {
 	/*
@@ -3537,6 +3612,27 @@ static void alc233_fixup_no_audio_jack(struct hda_codec *codec,
 	alc_process_coef_fw(codec, alc233_fixup_no_audio_jack_coefs);
 }
 
+static const struct coef_fw alc256_asus_speaker_coefs[] = {
+	WRITE_COEF(0x10, 0x7f20), WRITE_COEF(0x16, 0x0c50), WRITE_COEF(0x35, 0x8d6a),
+	WRITE_COEF(0x37, 0xfe06), WRITE_COEF(0x57, 0x7f7f),
+	{}
+};
+
+static void alc256_fixup_asus_speaker_coefs(struct hda_codec *codec,
+					    const struct hda_fixup *fix,
+					    int action)
+{
+	/*
+	 * The internal speakers of the ASUS ExpertBook PM3606CHA stay silent
+	 * with the default COEF values. These are the values set by the
+	 * Windows driver. They are applied on every init so that they are
+	 * restored after resume, too.
+	 */
+	if (action != HDA_FIXUP_ACT_INIT)
+		return;
+	alc_process_coef_fw(codec, alc256_asus_speaker_coefs);
+}
+
 static void alc256_fixup_mic_no_presence_and_resume(struct hda_codec *codec,
 						    const struct hda_fixup *fix,
 						    int action)
@@ -4025,6 +4121,7 @@ enum {
 	ALC269_FIXUP_SONY_VAIO,
 	ALC275_FIXUP_SONY_VAIO_GPIO2,
 	ALC269_FIXUP_DELL_M101Z,
+	ALC256_FIXUP_ASUS_PM3406CHA,
 	ALC269_FIXUP_SKU_IGNORE,
 	ALC269_FIXUP_ASUS_G73JW,
 	ALC269_FIXUP_ASUS_N7601ZM_PINS,
@@ -4124,6 +4221,7 @@ enum {
 	ALC285_FIXUP_HP_SPECTRE_X360_EB1,
 	ALC285_FIXUP_HP_SPECTRE_X360_DF1,
 	ALC285_FIXUP_HP_ENVY_X360,
+	ALC285_FIXUP_HP_ENVY_13AQ,
 	ALC288_FIXUP_DELL_HEADSET_MODE,
 	ALC288_FIXUP_DELL1_MIC_NO_PRESENCE,
 	ALC288_FIXUP_DELL_XPS_13,
@@ -4199,6 +4297,7 @@ enum {
 	ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE,
 	ALC225_FIXUP_WYSE_AUTO_MUTE,
 	ALC225_FIXUP_WYSE_DISABLE_MIC_VREF,
+	ALC225_FIXUP_HP_AIO_SPK,
 	ALC286_FIXUP_ACER_AIO_HEADSET_MIC,
 	ALC256_FIXUP_ASUS_HEADSET_MIC,
 	ALC256_FIXUP_ASUS_MIC_NO_PRESENCE,
@@ -4238,6 +4337,7 @@ enum {
 	ALC236_FIXUP_HP_15_FD0XXX,
 	ALC236_FIXUP_LENOVO_INV_DMIC,
 	ALC298_FIXUP_SAMSUNG_AMP,
+	ALC298_FIXUP_MSI_GE66_ALC1316_AMP,
 	ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS,
 	ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS,
 	ALC298_FIXUP_LG_GRAM_STYLE_14,
@@ -4314,6 +4414,7 @@ enum {
 	ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE,
 	ALC295_FIXUP_FRAMEWORK_LAPTOP_LIMIT_INT_MIC_BOOST,
 	ALC287_FIXUP_LEGION_16ITHG6,
+	ALC287_FIXUP_YOGA_SLIM7_CARBON_SPEAKERS,
 	ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK,
 	ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN,
 	ALC287_FIXUP_YOGA9_14IMH9_BASS_SPK_PIN,
@@ -4374,6 +4475,8 @@ enum {
 	ALC245_FIXUP_CS35L41_I2C_2_MUTE_LED,
 	ALC236_FIXUP_HP_DMIC,
 	ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO,
+	ALC256_FIXUP_HONOR_DRA_XX_SPEAKERS,
+	ALC256_FIXUP_HONOR_DRA_XX_SHARE_DAC,
 	ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX,
 	ALC287_FIXUP_ACER_MICMUTE_LED,
 	ALC236_FIXUP_DELL_HP_POP_NOISE,
@@ -4384,6 +4487,9 @@ enum {
 	ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN_HEADSET,
 	ALC285_LENOVO_DAC_RENAME,
 	ALC287_FIXUP_YOGA9_SPEAKER2_TO_DAC1,
+	ALC256_FIXUP_IPASON_SMARTBOOK_S1,
+	ALC256_FIXUP_ASUS_SPEAKER_COEFS,
+	ALC245_FIXUP_MINISFORUM_V3_MIC_NO_PRESENCE,
 };
 
 /* A special fixup for Lenovo C940 and Yoga Duet 7;
@@ -4481,6 +4587,14 @@ static const struct hda_fixup alc269_fixups[] = {
 			{}
 		}
 	},
+	[ALC256_FIXUP_ASUS_PM3406CHA] = {
+		.type = HDA_FIXUP_VERBS,
+		.v.verbs = (const struct hda_verb[]) {
+			{ 0x20, AC_VERB_SET_COEF_INDEX, 0x10 },
+			{ 0x20, AC_VERB_SET_PROC_COEF, 0x7f20 },
+			{}
+		}
+	},
 	[ALC269_FIXUP_SKU_IGNORE] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc_fixup_sku_ignore,
@@ -5719,6 +5833,14 @@ static const struct hda_fixup alc269_fixups[] = {
 		.chained = true,
 		.chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC
 	},
+	[ALC225_FIXUP_HP_AIO_SPK] = {
+		.type = HDA_FIXUP_VERBS,
+		.v.verbs = (const struct hda_verb[]) {
+			{ 0x20, AC_VERB_SET_COEF_INDEX, 0x10 },
+			{ 0x20, AC_VERB_SET_PROC_COEF, 0x0220 },
+			{ }
+		},
+	},
 	[ALC286_FIXUP_ACER_AIO_HEADSET_MIC] = {
 		.type = HDA_FIXUP_VERBS,
 		.v.verbs = (const struct hda_verb[]) {
@@ -6035,6 +6157,10 @@ static const struct hda_fixup alc269_fixups[] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc295_fixup_hp_pavilion_mute_led_1b,
 	},
+	[ALC298_FIXUP_MSI_GE66_ALC1316_AMP] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc298_fixup_alc1316_amp,
+	},
 	[ALC298_FIXUP_SAMSUNG_AMP] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc298_fixup_samsung_amp,
@@ -6400,6 +6526,15 @@ static const struct hda_fixup alc269_fixups[] = {
 		.chained = true,
 		.chain_id = ALC285_FIXUP_HP_GPIO_AMP_INIT,
 	},
+	[ALC285_FIXUP_HP_ENVY_13AQ] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x14, 0x90170150 }, /* front speakers */
+			{ }
+		},
+		.chained = true,
+		.chain_id = ALC285_FIXUP_HP_GPIO_AMP_INIT,
+	},
 	[ALC287_FIXUP_IDEAPAD_BASS_SPK_AMP] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc285_fixup_ideapad_s740_coef,
@@ -6684,6 +6819,10 @@ static const struct hda_fixup alc269_fixups[] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc287_fixup_legion_16ithg6_speakers,
 	},
+	[ALC287_FIXUP_YOGA_SLIM7_CARBON_SPEAKERS] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc287_fixup_yoga_slim7_carbon_speakers,
+	},
 	[ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK] = {
 		.type = HDA_FIXUP_VERBS,
 		.v.verbs = (const struct hda_verb[]) {
@@ -7072,6 +7211,19 @@ static const struct hda_fixup alc269_fixups[] = {
 			{ }
 		}
 	},
+	[ALC256_FIXUP_HONOR_DRA_XX_SPEAKERS] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x14, 0x90170111 }, /* bass speakers */
+			{ }
+		},
+		.chained = true,
+		.chain_id = ALC256_FIXUP_HONOR_DRA_XX_SHARE_DAC
+	},
+	[ALC256_FIXUP_HONOR_DRA_XX_SHARE_DAC] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc256_fixup_honor_dra_xx_share_dac,
+	},
 	[ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = cs35l41_fixup_i2c_two,
@@ -7087,6 +7239,8 @@ static const struct hda_fixup alc269_fixups[] = {
 	[ALC236_FIXUP_DELL_HP_POP_NOISE] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc285_fixup_invalidate_dacs,
+		.chained = true,
+		.chain_id = ALC255_FIXUP_DELL1_MIC_NO_PRESENCE
 	},
 	[ALC274_FIXUP_HP_89E9_GPIO] = {
 		.type = HDA_FIXUP_FUNC,
@@ -7136,6 +7290,28 @@ static const struct hda_fixup alc269_fixups[] = {
 		.chained = true,
 		.chain_id = ALC287_FIXUP_TXNW2781_I2C,
 	},
+	[ALC256_FIXUP_IPASON_SMARTBOOK_S1] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x1b, 0x90170110 },	/* the real internal speaker */
+			{ }
+		},
+	},
+	[ALC256_FIXUP_ASUS_SPEAKER_COEFS] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc256_fixup_asus_speaker_coefs,
+		.chained = true,
+		.chain_id = ALC256_FIXUP_ASUS_MIC_NO_PRESENCE,
+	},
+	[ALC245_FIXUP_MINISFORUM_V3_MIC_NO_PRESENCE] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x19, 0x04a19150 }, /* headset mic jack, no presence detect */
+			{ }
+		},
+		.chained = true,
+		.chain_id = ALC245_FIXUP_BASS_HP_DAC
+	},
 };
 
 static const struct hda_quirk alc269_fixup_tbl[] = {
@@ -7183,6 +7359,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1025, 0x142b, "Acer Swift SF314-42", ALC255_FIXUP_ACER_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1025, 0x1430, "Acer TravelMate B311R-31", ALC256_FIXUP_ACER_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1025, 0x1466, "Acer Aspire A515-56", ALC255_FIXUP_ACER_HEADPHONE_AND_MIC),
+	SND_PCI_QUIRK(0x1025, 0x151e, "Acer Nitro AN515-45", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1025, 0x1534, "Acer Predator PH315-54", ALC255_FIXUP_ACER_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1025, 0x1539, "Acer Nitro 5 AN515-57", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC287_FIXUP_ACER_MICMUTE_LED),
@@ -7196,6 +7373,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1025, 0x1679, "Acer Nitro 16 AN16-41", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1025, 0x169a, "Acer Swift SFG16", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED),
 	SND_PCI_QUIRK(0x1025, 0x171e, "Acer Nitro ANV15-51", ALC245_FIXUP_ACER_MICMUTE_LED),
+	SND_PCI_QUIRK(0x1025, 0x172c, "Acer Nitro ANV16-41", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1025, 0x1731, "Acer Predator PHN16-72", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1025, 0x173a, "Acer Swift SFG14-73", ALC245_FIXUP_ACER_MICMUTE_LED),
 	SND_PCI_QUIRK(0x1025, 0x1758, "Acer Nitro ANV15-41", ALC245_FIXUP_ACER_MICMUTE_LED),
@@ -7303,6 +7481,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1028, 0x164b, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x103c, 0x1586, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC2),
 	SND_PCI_QUIRK(0x103c, 0x18e6, "HP", ALC269_FIXUP_HP_GPIO_LED),
+	SND_PCI_QUIRK(0x103c, 0x2164, "HP Pavilion 15 Notebook PC", ALC269_FIXUP_LIMIT_INT_MIC_BOOST_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x218b, "HP", ALC269_FIXUP_LIMIT_INT_MIC_BOOST_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x21f9, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1),
 	SND_PCI_QUIRK(0x103c, 0x2210, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1),
@@ -7385,9 +7564,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x854a, "HP EliteBook 830 G6", ALC285_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x854d, "HP EliteBook 840 G6", ALC285_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x856a, "HP Pavilion 15-cs1xxx", ALC295_FIXUP_HP_PAVILION_MUTE_LED_1B),
+	HDA_CODEC_QUIRK(0x103c, 0x85ba, "HP Pavilion AiO 24", ALC225_FIXUP_HP_AIO_SPK),
 	SND_PCI_QUIRK(0x103c, 0x85c6, "HP Pavilion x360 Convertible 14-dy1xxx", ALC295_FIXUP_HP_MUTE_LED_COEFBIT11),
 	SND_PCI_QUIRK(0x103c, 0x85de, "HP Envy x360 13-ar0xxx", ALC285_FIXUP_HP_ENVY_X360),
 	SND_PCI_QUIRK(0x103c, 0x85f0, "HP Laptop 15-dw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
+	SND_PCI_QUIRK(0x103c, 0x85f1, "HP Laptop 15-dw1xxx", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO),
 	SND_PCI_QUIRK(0x103c, 0x85f3, "HP 250 G8 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
 	SND_PCI_QUIRK(0x103c, 0x8603, "HP Omen 17-cb0xxx", ALC285_FIXUP_HP_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x860c, "HP ZBook 17 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT),
@@ -7395,6 +7576,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x861f, "HP Elite Dragonfly G1", ALC285_FIXUP_HP_GPIO_AMP_INIT),
 	SND_PCI_QUIRK(0x103c, 0x864f, "HP Laptop 15-dy0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
 	SND_PCI_QUIRK(0x103c, 0x869d, "HP", ALC236_FIXUP_HP_MUTE_LED),
+	SND_PCI_QUIRK(0x103c, 0x86ad, "HP ENVY 13-aq1xxx", ALC285_FIXUP_HP_ENVY_13AQ),
 	SND_PCI_QUIRK(0x103c, 0x86c1, "HP Laptop 15-da3001TU", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
 	SND_PCI_QUIRK(0x103c, 0x86c7, "HP Envy AiO 32", ALC274_FIXUP_HP_ENVY_GPIO),
 	SND_PCI_QUIRK(0x103c, 0x86c8, "HP Laptop 14s-dr1xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
@@ -7423,6 +7605,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8786, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x8787, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x8788, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED),
+	SND_PCI_QUIRK(0x103c, 0x87b2, "HP Pavilion Gaming Laptop 15-ec1xxx", ALC285_FIXUP_HP_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x87b7, "HP Laptop 14-fq0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
 	SND_PCI_QUIRK(0x103c, 0x87c8, "HP", ALC287_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x87cb, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED),
@@ -7581,6 +7764,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8bb4, "HP Slim OMEN", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8bb6, "HP Laptop 15-fd0039nt", ALC236_FIXUP_HP_15_FD0XXX),
 	SND_PCI_QUIRK(0x103c, 0x8bbe, "HP Victus 16-r0xxx (MB 8BBE)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
+	SND_PCI_QUIRK(0x103c, 0x8bc7, "HP Pavilion 15-eh3174nw", ALC287_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8bc8, "HP Victus 15-fa1xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8bcd, "HP Omen 16-xd0xxx", ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8bd4, "HP Victus 16-s0xxx (MB 8BD4)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
@@ -7603,6 +7787,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8c17, "HP Spectre x360 2-in-1 Laptop 16-aa0xxx", ALC245_FIXUP_HP_SPECTRE_X360_16_AA0XXX),
 	SND_PCI_QUIRK(0x103c, 0x8c21, "HP Pavilion Plus Laptop 14-ey0XXX", ALC245_FIXUP_HP_X360_MUTE_LEDS),
 	SND_PCI_QUIRK(0x103c, 0x8c2d, "HP Victus 15-fa1xxx (MB 8C2D)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
+	SND_PCI_QUIRK(0x103c, 0x8c2f, "HP Victus 15-fb2xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8c30, "HP Victus 15-fb1xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8c3f, "HP Victus 15-fa1xxx (MB 8C3F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8c46, "HP EliteBook 830 G11", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
@@ -7712,6 +7897,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8e37, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8e3a, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8e3b, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2),
+	SND_PCI_QUIRK(0x103c, 0x8e43, "HP Laptop 15-fd2xxx", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO),
 	SND_PCI_QUIRK(0x103c, 0x8e60, "HP OmniBook 7 Laptop 16-bh0xxx", ALC245_FIXUP_CS35L41_I2C_2_MUTE_LED),
 	SND_PCI_QUIRK(0x103c, 0x8e61, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8e62, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2),
@@ -7814,6 +8000,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1043, 0x1652, "ASUS ROG Zephyrus Do 15 SE", ALC289_FIXUP_ASUS_ZEPHYRUS_DUAL_SPK),
 	SND_PCI_QUIRK(0x1043, 0x1662, "ASUS GV301QH", ALC294_FIXUP_ASUS_DUAL_SPK),
 	SND_PCI_QUIRK(0x1043, 0x1663, "ASUS GU603ZI/ZJ/ZQ/ZU/ZV", ALC285_FIXUP_ASUS_HEADSET_MIC),
+	SND_PCI_QUIRK(0x1043, 0x1664, "ASUS ExpertBook PM3406CHA", ALC256_FIXUP_ASUS_PM3406CHA),
 	SND_PCI_QUIRK(0x1043, 0x1683, "ASUS UM3402YAR", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x1043, 0x16a3, "ASUS UX3402VA", ALC245_FIXUP_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1043, 0x16b2, "ASUS GU603", ALC289_FIXUP_ASUS_GA401),
@@ -7898,6 +8085,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1043, 0x31e1, "ASUS B5605CCA", ALC294_FIXUP_ASUS_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1043, 0x31f1, "ASUS B3605CCA", ALC294_FIXUP_ASUS_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1043, 0x3391, "ASUS PM3606CKA", ALC287_FIXUP_CS35L41_I2C_2),
+	SND_PCI_QUIRK(0x1043, 0x3501, "ASUS PM3606CHA", ALC256_FIXUP_ASUS_SPEAKER_COEFS),
 	SND_PCI_QUIRK(0x1043, 0x3601, "ASUS PM5406CGA", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x1043, 0x3611, "ASUS PM5606CGA", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x1043, 0x3701, "ASUS P5406CCA", ALC245_FIXUP_CS35L41_SPI_2),
@@ -7916,7 +8104,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1043, 0x3ef0, "ASUS Strix G635LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C),
 	SND_PCI_QUIRK(0x1043, 0x3f00, "ASUS Strix G815LH_LM_LP", ALC287_FIXUP_TXNW2781_I2C),
 	SND_PCI_QUIRK(0x1043, 0x3f10, "ASUS Strix G835LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TXNW2781_I2C_ASUS),
 	SND_PCI_QUIRK(0x1043, 0x3f30, "ASUS Strix G815LR_LW", ALC287_FIXUP_TXNW2781_I2C),
 	SND_PCI_QUIRK(0x1043, 0x3fd0, "ASUS B3605CVA", ALC245_FIXUP_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1043, 0x3ff0, "ASUS B5405CVA", ALC245_FIXUP_CS35L41_SPI_2),
@@ -7939,6 +8127,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x10cf, 0x1757, "Lifebook E752", ALC269_FIXUP_LIFEBOOK_HP_PIN),
 	SND_PCI_QUIRK(0x10cf, 0x1845, "Lifebook U904", ALC269_FIXUP_LIFEBOOK_EXTMIC),
 	SND_PCI_QUIRK(0x10ec, 0x10f2, "Intel Reference board", ALC700_FIXUP_INTEL_REFERENCE),
+	SND_PCI_QUIRK(0x10ec, 0x111e, "Higole F9B", ALC290_FIXUP_MONO_SPEAKERS),
 	SND_PCI_QUIRK(0x10ec, 0x118c, "Medion EE4254 MD62100", ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE),
 	SND_PCI_QUIRK(0x10ec, 0x119e, "Positivo SU C1400", ALC269_FIXUP_ASPIRE_HEADSET_MIC),
 	SND_PCI_QUIRK(0x10ec, 0x11bc, "VAIO VJFE-IL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
@@ -7978,6 +8167,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x144d, 0xc1cc, "Samsung Galaxy Book3 Ultra (NT960XFH)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS),
 	SND_PCI_QUIRK(0x1458, 0x900e, "Gigabyte G5 KF5 (2023)", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC),
+	SND_PCI_QUIRK(0x1462, 0x12fb, "MSI GE66 Raider 11UE",
+		      ALC298_FIXUP_MSI_GE66_ALC1316_AMP),
 	SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x152d, 0x1082, "Quanta NL3", ALC269_FIXUP_LIFEBOOK),
@@ -8164,7 +8355,6 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x17aa, 0x3834, "Lenovo IdeaPad Slim 9i 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
 	SND_PCI_QUIRK(0x17aa, 0x383d, "Legion Y9000X 2019", ALC285_FIXUP_LEGION_Y9000X_SPEAKERS),
 	SND_PCI_QUIRK(0x17aa, 0x3843, "Lenovo Yoga 9i / Yoga Book 9i", ALC287_FIXUP_LENOVO_YOGA_BOOK_9I),
-	SND_PCI_QUIRK(0x17aa, 0x3846, "Lenovo Yoga Pro 9 16IAH10", ALC287_FIXUP_YOGA9_SPEAKER2_TO_DAC1),
 	/* Yoga Pro 7 14IMH9 shares PCI SSID 17aa:3847 with Legion 7 16ACHG6;
 	 * use codec SSID to distinguish them
 	 */
@@ -8178,6 +8368,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x17aa, 0x3852, "Lenovo Yoga 7 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
 	SND_PCI_QUIRK(0x17aa, 0x3853, "Lenovo Yoga 7 15ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS),
 	SND_PCI_QUIRK(0x17aa, 0x3855, "Legion 7 16ITHG6", ALC287_FIXUP_LEGION_16ITHG6),
+	HDA_CODEC_QUIRK(0x17aa, 0x3856, "Lenovo Yoga Slim 7 Carbon 14ACN6", ALC287_FIXUP_YOGA_SLIM7_CARBON_SPEAKERS),
 	SND_PCI_QUIRK(0x17aa, 0x3862, "Lenovo IdeaPad Slim 3 15ABR8", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
 	SND_PCI_QUIRK(0x17aa, 0x3865, "Lenovo 13X", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x17aa, 0x3866, "Lenovo 13X", ALC287_FIXUP_CS35L41_I2C_2),
@@ -8189,6 +8380,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	HDA_CODEC_QUIRK(0x17aa, 0x38a8, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */
 	SND_PCI_QUIRK(0x17aa, 0x386f, "Legion Pro 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x17aa, 0x3870, "Lenovo Yoga 7 14ARB7", ALC287_FIXUP_YOGA7_14ARB7_I2C),
+	SND_PCI_QUIRK(0x17aa, 0x3871, "Lenovo ThinkBook 16p Gen 3 ARH", ALC287_FIXUP_YOGA7_14ARB7_I2C),
 	SND_PCI_QUIRK(0x17aa, 0x3874, "Legion 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x17aa, 0x3877, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x17aa, 0x3878, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2),
@@ -8239,6 +8431,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x17aa, 0x38d5, "Yoga S990-16 Pro IMH YC Quad", ALC287_FIXUP_TAS2781_I2C),
 	SND_PCI_QUIRK(0x17aa, 0x38d6, "Yoga S990-16 Pro IMH VECO Quad", ALC287_FIXUP_TAS2781_I2C),
 	SND_PCI_QUIRK(0x17aa, 0x38d7, "Lenovo Yoga 9 14IMH9", ALC287_FIXUP_YOGA9_14IMH9_BASS_SPK_PIN),
+	SND_PCI_QUIRK(0x17aa, 0x38dc, "Lenovo ThinkBook G8", ALC285_FIXUP_SPEAKER2_TO_DAC1),
 	SND_PCI_QUIRK(0x17aa, 0x38df, "Yoga Y990 Intel YC Dual", ALC287_FIXUP_TAS2781_I2C),
 	SND_PCI_QUIRK(0x17aa, 0x38e0, "Yoga Y990 Intel VECO Dual", ALC287_FIXUP_TAS2781_I2C),
 	SND_PCI_QUIRK(0x17aa, 0x38f8, "Yoga Book 9i", ALC287_FIXUP_TAS2781_I2C),
@@ -8258,7 +8451,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x17aa, 0x3912, "Lenovo Xiaoxin 14 GT", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN),
 	SND_PCI_QUIRK(0x17aa, 0x3913, "Lenovo 145", ALC236_FIXUP_LENOVO_INV_DMIC),
 	SND_PCI_QUIRK(0x17aa, 0x391f, "Yoga S990-16 pro Quad YC Quad", ALC287_FIXUP_TXNW2781_I2C),
-	SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_YOGA9_SPEAKER2_TO_DAC1),
 	SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD),
 	SND_PCI_QUIRK(0x17aa, 0x392b, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD),
 	HDA_CODEC_QUIRK(0x17aa, 0x3936, "Legion R9000P ADR10H", ALC287_FIXUP_LENOVO_LEGION_AW88399),
@@ -8345,15 +8538,18 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1e50, 0x7007, "Positivo DN50E", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
 	SND_PCI_QUIRK(0x1e50, 0x7036, "Acer Gadget E10 ETBook", ALC233_FIXUP_WUJIE_SPEAKERS),
 	SND_PCI_QUIRK(0x1e50, 0x7038, "Positivo DN140", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
+	SND_PCI_QUIRK(0x1ee7, 0x204e, "HONOR DRA-XX M1020", ALC256_FIXUP_HONOR_DRA_XX_SPEAKERS),
 	SND_PCI_QUIRK(0x1ee7, 0x2078, "HONOR BRB-X M1010", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1ee7, 0x2081, "HONOR MRB-XXX M1020", ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO),
 	SND_PCI_QUIRK(0x1f4c, 0xb020, "Minisforum AI X1 Pro",
 		      ALC245_FIXUP_MINISFORUM_JACK_DETECT),
-	SND_PCI_QUIRK(0x1f4c, 0xe001, "Minisforum V3 (SE)", ALC245_FIXUP_BASS_HP_DAC),
+	SND_PCI_QUIRK(0x1f4c, 0xe001, "Minisforum V3 (SE)",
+		      ALC245_FIXUP_MINISFORUM_V3_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1f66, 0x0105, "Ayaneo Portable Game Player", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
 	SND_PCI_QUIRK(0x2039, 0x0001, "Inspur S14-G1", ALC295_FIXUP_CHROME_BOOK),
 	SND_PCI_QUIRK(0x2145, 0x0001, "Star Labs StarFighter", ALC233_FIXUP_STARLABS_STARFIGHTER),
+	SND_PCI_QUIRK(0x2782, 0x0206, "IPASON SmartBook S1", ALC256_FIXUP_IPASON_SMARTBOOK_S1),
 	SND_PCI_QUIRK(0x2782, 0x0214, "VAIO VJFE-CL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
 	SND_PCI_QUIRK(0x2782, 0x0228, "Infinix ZERO BOOK 13", ALC269VB_FIXUP_INFINIX_ZERO_BOOK_13),
 	SND_PCI_QUIRK(0x2782, 0x0232, "CHUWI CoreBook XPro", ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO),
@@ -8560,6 +8756,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = {
 	{.id = ALC298_FIXUP_HUAWEI_MBX_STEREO, .name = "huawei-mbx-stereo"},
 	{.id = ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE, .name = "alc256-medion-headset"},
 	{.id = ALC298_FIXUP_SAMSUNG_AMP, .name = "alc298-samsung-amp"},
+	{.id = ALC298_FIXUP_MSI_GE66_ALC1316_AMP, .name = "alc298-alc1316-amp"},
 	{.id = ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS, .name = "alc298-samsung-amp-v2-2-amps"},
 	{.id = ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS, .name = "alc298-samsung-amp-v2-4-amps"},
 	{.id = ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, .name = "alc256-samsung-headphone"},
diff --git a/sound/hda/codecs/realtek/alc882.c b/sound/hda/codecs/realtek/alc882.c
index fd466b6985f05e..800c3d6f5134bf 100644
--- a/sound/hda/codecs/realtek/alc882.c
+++ b/sound/hda/codecs/realtek/alc882.c
@@ -372,7 +372,7 @@ static const struct hda_fixup alc882_fixups[] = {
 	[ALC883_FIXUP_ACER_EAPD] = {
 		.type = HDA_FIXUP_VERBS,
 		.v.verbs = (const struct hda_verb[]) {
-			/* eanable EAPD on Acer laptops */
+			/* enable EAPD on Acer laptops */
 			{ 0x20, AC_VERB_SET_COEF_INDEX, 0x07 },
 			{ 0x20, AC_VERB_SET_PROC_COEF, 0x3050 },
 			{ }
diff --git a/sound/hda/codecs/side-codecs/aw88399_hda.c b/sound/hda/codecs/side-codecs/aw88399_hda.c
index 11cef49230242d..439ad45557a4d2 100644
--- a/sound/hda/codecs/side-codecs/aw88399_hda.c
+++ b/sound/hda/codecs/side-codecs/aw88399_hda.c
@@ -209,8 +209,7 @@ static int aw88399_hda_acpi_probe(struct aw88399_hda *aw88399)
 		return -ENODEV;
 	}
 
-	struct device *physdev __free(put_device) =
-		get_device(acpi_get_first_physical_node(adev));
+	struct device *physdev __free(put_device) = acpi_bus_get_primary_device(adev);
 	acpi_dev_put(adev);
 	if (!physdev)
 		return -ENODEV;
diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c b/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c
index fdf406e92fca4d..59b82238f2b8b5 100644
--- a/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c
+++ b/sound/hda/codecs/side-codecs/cs35l41_hda_i2c.c
@@ -23,6 +23,8 @@ static int cs35l41_hda_i2c_probe(struct i2c_client *clt)
 		device_name = "CLSA0100";
 	else if (strstr(dev_name(&clt->dev), "CLSA0101"))
 		device_name = "CLSA0101";
+	else if (strstr(dev_name(&clt->dev), "CLSA0102"))
+		device_name = "CLSA0102";
 	else if (strstr(dev_name(&clt->dev), "CSC3551"))
 		device_name = "CSC3551";
 	else
@@ -45,6 +47,7 @@ static const struct i2c_device_id cs35l41_hda_i2c_id[] = {
 static const struct acpi_device_id cs35l41_acpi_hda_match[] = {
 	{"CLSA0100", 0 },
 	{"CLSA0101", 0 },
+	{"CLSA0102", 0 },
 	{"CSC3551", 0 },
 	{}
 };
diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda_property.c b/sound/hda/codecs/side-codecs/cs35l41_hda_property.c
index 416d7bf3e289c0..c4f2bbed07480a 100644
--- a/sound/hda/codecs/side-codecs/cs35l41_hda_property.c
+++ b/sound/hda/codecs/side-codecs/cs35l41_hda_property.c
@@ -439,6 +439,33 @@ static int lenovo_legion_no_acpi(struct cs35l41_hda *cs35l41, struct device *phy
 	return 0;
 }
 
+/*
+ * Device CLSA0102 (Lenovo Yoga Slim 7 Carbon 14ACN6) has the same ACPI layout as CLSA010(0/1):
+ * no _DSD, both amps in one node, reset on GPIO index 0. It has no speaker ID GPIO, the
+ * interrupt GPIO sits at index 2. The speakers use an external VSPK supply switched by GPIO1
+ * (the Windows driver sets ExternalVspkControl and keeps the boost converter off).
+ */
+static int lenovo_yoga_slim7_carbon_no_acpi(struct cs35l41_hda *cs35l41, struct device *physdev,
+					    int id, const char *hid)
+{
+	struct cs35l41_hw_cfg *hw_cfg = &cs35l41->hw_cfg;
+
+	/* check I2C address to assign the index */
+	cs35l41->index = id == 0x40 ? 0 : 1;
+	cs35l41->channel_index = 0;
+	cs35l41->reset_gpio = gpiod_get_index(physdev, NULL, 0, GPIOD_OUT_HIGH);
+	cs35l41->speaker_id = -ENOENT;
+	hw_cfg->spk_pos = cs35l41->index;
+	hw_cfg->bst_type = CS35L41_EXT_BOOST;
+	hw_cfg->gpio1.func = CS35l41_VSPK_SWITCH;
+	hw_cfg->gpio1.valid = true;
+	hw_cfg->gpio2.func = CS35L41_INTERRUPT;
+	hw_cfg->gpio2.valid = true;
+	hw_cfg->valid = true;
+
+	return 0;
+}
+
 static int missing_speaker_id_gpio2(struct cs35l41_hda *cs35l41, struct device *physdev, int id,
 				    const char *hid)
 {
@@ -463,6 +490,7 @@ struct cs35l41_prop_model {
 static const struct cs35l41_prop_model cs35l41_prop_model_table[] = {
 	{ "CLSA0100", NULL, lenovo_legion_no_acpi },
 	{ "CLSA0101", NULL, lenovo_legion_no_acpi },
+	{ "CLSA0102", NULL, lenovo_yoga_slim7_carbon_no_acpi },
 	{ "CSC3551", "10251826", generic_dsd_config },
 	{ "CSC3551", "1025182C", generic_dsd_config },
 	{ "CSC3551", "10251844", generic_dsd_config },
diff --git a/sound/hda/codecs/sigmatel.c b/sound/hda/codecs/sigmatel.c
index ee3bd21adc366a..9a8ba43667f770 100644
--- a/sound/hda/codecs/sigmatel.c
+++ b/sound/hda/codecs/sigmatel.c
@@ -361,7 +361,7 @@ static unsigned int stac_vref_led_power_filter(struct hda_codec *codec,
 	return snd_hda_gen_path_power_filter(codec, nid, power_state);
 }
 
-/* update mute-LED accoring to the master switch */
+/* update mute-LED according to the master switch */
 static void stac_update_led_status(struct hda_codec *codec, bool muted)
 {
 	struct sigmatel_spec *spec = codec->spec;
@@ -1710,7 +1710,7 @@ static const struct hda_pintbl ref92hd73xx_pin_configs[] = {
 	{ 0x11, 0x01014030 },
 	// CD in
 	{ 0x12, 0x02319040 },
-	// Digial Mic ins
+	// Digital Mic ins
 	{ 0x13, 0x90a000f0 },
 	{ 0x14, 0x90a000f0 },
 	// Digital outs
@@ -3817,7 +3817,7 @@ static const struct hda_pintbl dell_3st_pin_configs[] = {
 static void stac927x_fixup_ref_no_jd(struct hda_codec *codec,
 				     const struct hda_fixup *fix, int action)
 {
-	/* no jack detecion for ref-no-jd model */
+	/* no jack detection for ref-no-jd model */
 	if (action == HDA_FIXUP_ACT_PRE_PROBE)
 		codec->no_jack_detect = 1;
 }
diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c
index 5d9b53bd64ea0a..c61fd79c48f3fa 100644
--- a/sound/hda/common/codec.c
+++ b/sound/hda/common/codec.c
@@ -3774,7 +3774,7 @@ int snd_hda_multi_out_analog_open(struct hda_codec *codec,
 EXPORT_SYMBOL_GPL(snd_hda_multi_out_analog_open);
 
 /**
- * snd_hda_multi_out_analog_prepare - Preapre the analog outputs.
+ * snd_hda_multi_out_analog_prepare - Prepare the analog outputs.
  * @codec: the HDA codec
  * @mout: hda_multi_out object
  * @stream_tag: stream tag to assign
diff --git a/sound/hda/controllers/acpi.c b/sound/hda/controllers/acpi.c
index 505cc97e0ee9a7..04b261dde22eaf 100644
--- a/sound/hda/controllers/acpi.c
+++ b/sound/hda/controllers/acpi.c
@@ -266,6 +266,38 @@ static void hda_acpi_shutdown(struct platform_device *pdev)
 		azx_stop_chip(chip);
 }
 
+static int hda_acpi_runtime_suspend(struct device *dev)
+{
+	struct snd_card *card = dev_get_drvdata(dev);
+	struct azx *chip = card->private_data;
+
+	if (chip && chip->running) {
+		/* enable controller wake up event */
+		azx_writew(chip, WAKEEN, azx_readw(chip, WAKEEN) |
+			   STATESTS_INT_MASK);
+
+		azx_stop_chip(chip);
+		azx_enter_link_reset(chip);
+	}
+
+	return 0;
+}
+
+static int hda_acpi_runtime_resume(struct device *dev)
+{
+	struct snd_card *card = dev_get_drvdata(dev);
+	struct azx *chip = card->private_data;
+
+	if (chip && chip->running) {
+		azx_init_chip(chip, 1);
+		/* disable controller wake up event */
+		azx_writew(chip, WAKEEN, azx_readw(chip, WAKEEN) &
+			   ~STATESTS_INT_MASK);
+	}
+
+	return 0;
+}
+
 static int hda_acpi_suspend(struct device *dev)
 {
 	struct snd_card *card = dev_get_drvdata(dev);
@@ -274,6 +306,12 @@ static int hda_acpi_suspend(struct device *dev)
 	rc = pm_runtime_force_suspend(dev);
 	if (rc < 0)
 		return rc;
+	/*
+	 * Runtime PM is never enabled for this device, so the force_suspend
+	 * above does not reach the runtime callback; stop the controller
+	 * explicitly.
+	 */
+	hda_acpi_runtime_suspend(dev);
 	snd_power_change_state(card, SNDRV_CTL_POWER_D3hot);
 
 	return 0;
@@ -287,6 +325,8 @@ static int hda_acpi_resume(struct device *dev)
 	rc = pm_runtime_force_resume(dev);
 	if (rc < 0)
 		return rc;
+	/* See hda_acpi_suspend(): re-init the controller explicitly. */
+	hda_acpi_runtime_resume(dev);
 	snd_power_change_state(card, SNDRV_CTL_POWER_D0);
 
 	return 0;
@@ -294,6 +334,7 @@ static int hda_acpi_resume(struct device *dev)
 
 static const struct dev_pm_ops hda_acpi_pm = {
 	SYSTEM_SLEEP_PM_OPS(hda_acpi_suspend, hda_acpi_resume)
+	RUNTIME_PM_OPS(hda_acpi_runtime_suspend, hda_acpi_runtime_resume, NULL)
 };
 
 static const struct hda_data nvidia_hda_data = {
diff --git a/sound/hda/controllers/tegra.c b/sound/hda/controllers/tegra.c
index 31c14c4bbe683b..674cc0f79c3c88 100644
--- a/sound/hda/controllers/tegra.c
+++ b/sound/hda/controllers/tegra.c
@@ -374,7 +374,8 @@ static int hda_tegra_first_init(struct azx *chip, struct platform_device *pdev)
 	 * in powers of 2, next available ratio is 16 which can be
 	 * used as a limiting factor here.
 	 */
-	if (of_device_is_compatible(np, "nvidia,tegra30-hda"))
+	if (of_device_is_compatible(np, "nvidia,tegra30-hda") ||
+	    of_device_is_compatible(np, "nvidia,tegra194-hda"))
 		chip->bus.core.sdo_limit = 16;
 
 	/* codec detection */
diff --git a/sound/hda/core/bus.c b/sound/hda/core/bus.c
index 81498f1e413e2a..20fe1c4a2977e1 100644
--- a/sound/hda/core/bus.c
+++ b/sound/hda/core/bus.c
@@ -21,7 +21,7 @@ static const struct hdac_bus_ops default_ops = {
 };
 
 /**
- * snd_hdac_bus_init - initialize a HD-audio bas bus
+ * snd_hdac_bus_init - initialize a HD-audio bus device
  * @bus: the pointer to bus object
  * @dev: device pointer
  * @ops: bus verb operators
diff --git a/sound/hda/core/component.c b/sound/hda/core/component.c
index 04755903880e6b..83b31186b02765 100644
--- a/sound/hda/core/component.c
+++ b/sound/hda/core/component.c
@@ -70,10 +70,7 @@ void snd_hdac_display_power(struct hdac_bus *bus, unsigned int idx, bool enable)
 	dev_dbg(bus->dev, "display power %s\n", str_enable_disable(enable));
 
 	guard(mutex)(&bus->lock);
-	if (enable)
-		set_bit(idx, &bus->display_power_status);
-	else
-		clear_bit(idx, &bus->display_power_status);
+	assign_bit(idx, &bus->display_power_status, enable);
 
 	if (!acomp || !acomp->ops)
 		return;
diff --git a/sound/hda/core/controller.c b/sound/hda/core/controller.c
index 78855ac357c607..e8d40c24ed00ba 100644
--- a/sound/hda/core/controller.c
+++ b/sound/hda/core/controller.c
@@ -781,9 +781,6 @@ EXPORT_SYMBOL_GPL(snd_hdac_bus_free_stream_pages);
  */
 void snd_hdac_bus_link_power(struct hdac_device *codec, bool enable)
 {
-	if (enable)
-		set_bit(codec->addr, &codec->bus->codec_powered);
-	else
-		clear_bit(codec->addr, &codec->bus->codec_powered);
+	assign_bit(codec->addr, &codec->bus->codec_powered, enable);
 }
 EXPORT_SYMBOL_GPL(snd_hdac_bus_link_power);
diff --git a/sound/hda/core/intel-nhlt.c b/sound/hda/core/intel-nhlt.c
index 6d72a871bda0b5..af4d469dbd764b 100644
--- a/sound/hda/core/intel-nhlt.c
+++ b/sound/hda/core/intel-nhlt.c
@@ -351,7 +351,7 @@ int intel_nhlt_ssp_device_type(struct device *dev, struct nhlt_acpi_table *nhlt,
 	int i;
 
 	if (!nhlt) {
-		dev_err(dev, "%s: NHLT table is missing (query for SSP%d)\n",
+		dev_dbg(dev, "%s: NHLT table is missing (query for SSP%d)\n",
 			__func__, virtual_bus_id);
 		return -EINVAL;
 	}
@@ -369,7 +369,7 @@ int intel_nhlt_ssp_device_type(struct device *dev, struct nhlt_acpi_table *nhlt,
 		epnt = (struct nhlt_endpoint *)((u8 *)epnt + epnt->length);
 	}
 
-	dev_err(dev, "%s: No match for SSP%d in NHLT table\n", __func__,
+	dev_dbg(dev, "%s: No match for SSP%d in NHLT table\n", __func__,
 		virtual_bus_id);
 
 	dev_dbg(dev, "Available endpoints:\n");
diff --git a/sound/i2c/other/ak4113.c b/sound/i2c/other/ak4113.c
index 6ef5fffda8d71c..9405d985a87ddb 100644
--- a/sound/i2c/other/ak4113.c
+++ b/sound/i2c/other/ak4113.c
@@ -130,7 +130,7 @@ void snd_ak4113_reinit(struct ak4113 *chip)
 	scoped_guard(mutex, &chip->reinit_mutex) {
 		ak4113_init_regs(chip);
 	}
-	/* bring up statistics / event queing */
+	/* bring up statistics / event queuing */
 	if (atomic_dec_and_test(&chip->wq_processing))
 		schedule_delayed_work(&chip->work, HZ / 10);
 }
diff --git a/sound/i2c/other/ak4114.c b/sound/i2c/other/ak4114.c
index 71efb29b6c7c0e..4c3dbbf8c3161a 100644
--- a/sound/i2c/other/ak4114.c
+++ b/sound/i2c/other/ak4114.c
@@ -135,7 +135,7 @@ void snd_ak4114_reinit(struct ak4114 *chip)
 	scoped_guard(mutex, &chip->reinit_mutex) {
 		ak4114_init_regs(chip);
 	}
-	/* bring up statistics / event queing */
+	/* bring up statistics / event queuing */
 	if (atomic_dec_and_test(&chip->wq_processing))
 		schedule_delayed_work(&chip->work, HZ / 10);
 }
diff --git a/sound/i2c/other/ak4xxx-adda.c b/sound/i2c/other/ak4xxx-adda.c
index 9dd36b82a6ac47..e7410f0f866089 100644
--- a/sound/i2c/other/ak4xxx-adda.c
+++ b/sound/i2c/other/ak4xxx-adda.c
@@ -147,7 +147,7 @@ EXPORT_SYMBOL(snd_akm4xxx_reset);
  * Volume conversion table for non-linear volumes
  * from -63.5dB (mute) to 0dB step 0.5dB
  *
- * Used for AK4524/AK4620 input/ouput attenuation, AK4528, and
+ * Used for AK4524/AK4620 input/output attenuation, AK4528, and
  * AK5365 input attenuation
  */
 static const unsigned char vol_cvt_datt[128] = {
diff --git a/sound/isa/opti9xx/opti92x-ad1848.c b/sound/isa/opti9xx/opti92x-ad1848.c
index abaa3ed3ab5c2f..8a6db40ff89384 100644
--- a/sound/isa/opti9xx/opti92x-ad1848.c
+++ b/sound/isa/opti9xx/opti92x-ad1848.c
@@ -378,7 +378,7 @@ static int snd_opti9xx_configure(struct snd_opti9xx *chip,
 		 */
 		snd_opti9xx_write_mask(chip, OPTi9XX_MC_REG(21), 0x82, 0xff);
 		/* 
-		 * This bit sets OPTI931 to automaticaly select FM
+		 * This bit sets OPTI931 to automatically select FM
 		 * or digital input signal.
 		 */
 		snd_opti9xx_write_mask(chip, OPTi9XX_MC_REG(26), 0x01, 0x01);
diff --git a/sound/mips/hal2.c b/sound/mips/hal2.c
index 2beb6a0dc3b7af..84f265335cf9c9 100644
--- a/sound/mips/hal2.c
+++ b/sound/mips/hal2.c
@@ -375,7 +375,7 @@ static void hal2_setup_dac(struct snd_hal2 *hal2)
 	hal2_i_clearbit16(hal2, H2I_DMA_PORT_EN, H2I_DMA_PORT_EN_CODECTX);
 	/* Setup the HAL2 for playback */
 	hal2_set_dac_rate(hal2);
-	/* Set endianess */
+	/* Set endianness */
 	hal2_i_clearbit16(hal2, H2I_DMA_END, H2I_DMA_END_CODECTX);
 	/* Set DMA bus */
 	hal2_i_setbit16(hal2, H2I_DMA_DRV, (1 << pbus->pbusnr));
@@ -400,7 +400,7 @@ static void hal2_setup_adc(struct snd_hal2 *hal2)
 	hal2_i_clearbit16(hal2, H2I_DMA_PORT_EN, H2I_DMA_PORT_EN_CODECR);
 	/* Setup the HAL2 for record */
 	hal2_set_adc_rate(hal2);
-	/* Set endianess */
+	/* Set endianness */
 	hal2_i_clearbit16(hal2, H2I_DMA_END, H2I_DMA_END_CODECR);
 	/* Set DMA bus */
 	hal2_i_setbit16(hal2, H2I_DMA_DRV, (1 << pbus->pbusnr));
diff --git a/sound/mips/sgio2audio.c b/sound/mips/sgio2audio.c
index 497d84cff12f30..35648b0b97b908 100644
--- a/sound/mips/sgio2audio.c
+++ b/sound/mips/sgio2audio.c
@@ -742,7 +742,8 @@ static struct {
 
 /* ALSA driver */
 
-static int snd_sgio2audio_free(struct snd_sgio2audio *chip)
+static int snd_sgio2audio_free(struct snd_sgio2audio *chip,
+			       unsigned int irq_count)
 {
 	int i;
 
@@ -752,7 +753,7 @@ static int snd_sgio2audio_free(struct snd_sgio2audio *chip)
 	writeq(0, &mace->perif.audio.control);
 
 	/* release IRQ's */
-	for (i = 0; i < ARRAY_SIZE(snd_sgio2_isr_table); i++)
+	for (i = 0; i < irq_count; i++)
 		free_irq(snd_sgio2_isr_table[i].irq,
 			 &chip->channel[snd_sgio2_isr_table[i].idx]);
 
@@ -768,7 +769,7 @@ static int snd_sgio2audio_dev_free(struct snd_device *device)
 {
 	struct snd_sgio2audio *chip = device->device_data;
 
-	return snd_sgio2audio_free(chip);
+	return snd_sgio2audio_free(chip, ARRAY_SIZE(snd_sgio2_isr_table));
 }
 
 static const struct snd_device_ops ops = {
@@ -819,7 +820,7 @@ static int snd_sgio2audio_create(struct snd_card *card,
 				0,
 				snd_sgio2_isr_table[i].desc,
 				&chip->channel[snd_sgio2_isr_table[i].idx])) {
-			snd_sgio2audio_free(chip);
+			snd_sgio2audio_free(chip, i);
 			printk(KERN_ERR "sgio2audio: cannot allocate irq %d\n",
 			       snd_sgio2_isr_table[i].irq);
 			return -EBUSY;
@@ -843,13 +844,13 @@ static int snd_sgio2audio_create(struct snd_card *card,
 	/* initialize the AD1843 codec */
 	err = ad1843_init(&chip->ad1843);
 	if (err < 0) {
-		snd_sgio2audio_free(chip);
+		snd_sgio2audio_free(chip, ARRAY_SIZE(snd_sgio2_isr_table));
 		return err;
 	}
 
 	err = snd_device_new(card, SNDRV_DEV_LOWLEVEL, chip, &ops);
 	if (err < 0) {
-		snd_sgio2audio_free(chip);
+		snd_sgio2audio_free(chip, ARRAY_SIZE(snd_sgio2_isr_table));
 		return err;
 	}
 	*rchip = chip;
diff --git a/sound/mips/snd-n64.c b/sound/mips/snd-n64.c
index f17e63f2ff5a67..e2cf9df15d485c 100644
--- a/sound/mips/snd-n64.c
+++ b/sound/mips/snd-n64.c
@@ -298,7 +298,7 @@ static int __init n64audio_probe(struct platform_device *pdev)
 	priv->card = card;
 
 	priv->ring_base = dma_alloc_coherent(card->dev, 32 * 1024, &priv->ring_base_dma,
-					     GFP_DMA|GFP_KERNEL);
+					     GFP_KERNEL);
 	if (!priv->ring_base) {
 		err = -ENOMEM;
 		goto fail_card;
diff --git a/sound/oss/dmasound/dmasound_q40.c b/sound/oss/dmasound/dmasound_q40.c
index e25a78dd1bf2af..339311e5d80702 100644
--- a/sound/oss/dmasound/dmasound_q40.c
+++ b/sound/oss/dmasound/dmasound_q40.c
@@ -49,9 +49,8 @@ static int Q40SetFormat(int format);
 static int Q40SetVolume(int volume);
 static void Q40PlayNextFrame(int index);
 static void Q40Play(void);
-static irqreturn_t Q40StereoInterrupt(int irq, void *dummy);
-static irqreturn_t Q40MonoInterrupt(int irq, void *dummy);
-static void Q40Interrupt(void);
+static irqreturn_t q40_audio_interrupt(int irq, void *dummy);
+static void q40_frame_done(void);
 
 
 /*** Mid level stuff *********************************************************/
@@ -372,8 +371,8 @@ static void Q40Free(void *ptr, unsigned int size)
 static int __init Q40IrqInit(void)
 {
 	/* Register interrupt handler. */
-	if (request_irq(Q40_IRQ_SAMPLE, Q40StereoInterrupt, 0,
-		    "DMA sound", Q40Interrupt))
+	if (request_irq(Q40_IRQ_SAMPLE, q40_audio_interrupt, 0,
+			"DMA sound", q40_frame_done))
 		return 0;
 
 	return(1);
@@ -384,7 +383,7 @@ static int __init Q40IrqInit(void)
 static void Q40IrqCleanUp(void)
 {
         master_outb(0,SAMPLE_ENABLE_REG);
-	free_irq(Q40_IRQ_SAMPLE, Q40Interrupt);
+	free_irq(Q40_IRQ_SAMPLE, q40_frame_done);
 }
 #endif /* MODULE */
 
@@ -403,7 +402,6 @@ static void Q40PlayNextFrame(int index)
 	u_char *start;
 	u_long size;
 	u_char speed;
-	int error;
 
 	/* used by Q40Play() if all doubts whether there really is something
 	 * to be played are already wiped out.
@@ -420,15 +418,6 @@ static void Q40PlayNextFrame(int index)
 	speed=(dmasound.hard.speed==10000 ? 0 : 1);
 
 	master_outb( 0,SAMPLE_ENABLE_REG);
-	free_irq(Q40_IRQ_SAMPLE, Q40Interrupt);
-	if (dmasound.soft.stereo)
-		error = request_irq(Q40_IRQ_SAMPLE, Q40StereoInterrupt, 0,
-				    "Q40 sound", Q40Interrupt);
-	  else
-		error = request_irq(Q40_IRQ_SAMPLE, Q40MonoInterrupt, 0,
-				    "Q40 sound", Q40Interrupt);
-	if (error && printk_ratelimit())
-		pr_err("Couldn't register sound interrupt\n");
 
 	master_outb( speed, SAMPLE_RATE_REG);
 	master_outb( 1,SAMPLE_CLEAR_REG);
@@ -456,31 +445,35 @@ static void Q40Play(void)
 	spin_unlock_irqrestore(&dmasound.lock, flags);
 }
 
-static irqreturn_t Q40StereoInterrupt(int irq, void *dummy)
-{
-	spin_lock(&dmasound.lock);
-        if (q40_sc>1){
-            *DAC_LEFT=*q40_pp++;
-	    *DAC_RIGHT=*q40_pp++;
-	    q40_sc -=2;
-	    master_outb(1,SAMPLE_CLEAR_REG);
-	}else Q40Interrupt();
-	spin_unlock(&dmasound.lock);
-	return IRQ_HANDLED;
-}
-static irqreturn_t Q40MonoInterrupt(int irq, void *dummy)
+static irqreturn_t q40_audio_interrupt(int irq, void *dummy)
 {
+	bool frame_done = false;
+
 	spin_lock(&dmasound.lock);
-        if (q40_sc>0){
-            *DAC_LEFT=*q40_pp;
-	    *DAC_RIGHT=*q40_pp++;
-	    q40_sc --;
-	    master_outb(1,SAMPLE_CLEAR_REG);
-	}else Q40Interrupt();
+	if (dmasound.hard.stereo) {
+		if (q40_sc > 1) {
+			*DAC_LEFT = *q40_pp++;
+			*DAC_RIGHT = *q40_pp++;
+			q40_sc -= 2;
+			master_outb(1, SAMPLE_CLEAR_REG);
+		} else {
+			frame_done = true;
+		}
+	} else if (q40_sc > 0) {
+		*DAC_LEFT = *q40_pp;
+		*DAC_RIGHT = *q40_pp++;
+		q40_sc--;
+		master_outb(1, SAMPLE_CLEAR_REG);
+	} else {
+		frame_done = true;
+	}
 	spin_unlock(&dmasound.lock);
+	if (frame_done)
+		q40_frame_done();
 	return IRQ_HANDLED;
 }
-static void Q40Interrupt(void)
+
+static void q40_frame_done(void)
 {
 	if (!write_sq.active) {
 	          /* playing was interrupted and sq_reset() has already cleared
diff --git a/sound/pci/ac97/ac97_codec.c b/sound/pci/ac97/ac97_codec.c
index 0bb65be021d973..207ce083593faa 100644
--- a/sound/pci/ac97/ac97_codec.c
+++ b/sound/pci/ac97/ac97_codec.c
@@ -2960,7 +2960,7 @@ int snd_ac97_tune_hardware(struct snd_ac97 *ac97,
 {
 	int result;
 
-	/* quirk overriden? */
+	/* quirk overridden? */
 	if (override && strcmp(override, "-1") && strcmp(override, "default")) {
 		result = apply_quirk_str(ac97, override);
 		if (result < 0)
diff --git a/sound/pci/ac97/ac97_patch.c b/sound/pci/ac97/ac97_patch.c
index 64cc39dd20083c..8577a2e0510c48 100644
--- a/sound/pci/ac97/ac97_patch.c
+++ b/sound/pci/ac97/ac97_patch.c
@@ -430,7 +430,7 @@ static int patch_yamaha_ymf753(struct snd_ac97 * ac97)
 	/* Patch for Yamaha YMF753, Copyright (c) by David Shust, dshust@shustring.com.
 	   This chip has nonstandard and extended behaviour with regard to its S/PDIF output.
 	   The AC'97 spec states that the S/PDIF signal is to be output at pin 48.
-	   The YMF753 will ouput the S/PDIF signal to pin 43, 47 (EAPD), or 48.
+	   The YMF753 will output the S/PDIF signal to pin 43, 47 (EAPD), or 48.
 	   By default, no output pin is selected, and the S/PDIF signal is not output.
 	   There is also a bit to mute S/PDIF output in a vendor-specific register.
 	*/
diff --git a/sound/pci/au88x0/au88x0.h b/sound/pci/au88x0/au88x0.h
index 6cbb2bc4a0483a..f97a0303da3e8f 100644
--- a/sound/pci/au88x0/au88x0.h
+++ b/sound/pci/au88x0/au88x0.h
@@ -28,7 +28,7 @@
 
 /* Vortex MPU401 defines. */
 #define	MIDI_CLOCK_DIV		0x61
-/* Standart MPU401 defines. */
+/* Standard MPU401 defines. */
 #define	MPU401_RESET		0xff
 #define	MPU401_ENTER_UART	0x3f
 #define	MPU401_ACK		0xfe
diff --git a/sound/pci/au88x0/au88x0_mpu401.c b/sound/pci/au88x0/au88x0_mpu401.c
index 164f6b7039ab56..dff4bf80ab9653 100644
--- a/sound/pci/au88x0/au88x0_mpu401.c
+++ b/sound/pci/au88x0/au88x0_mpu401.c
@@ -22,7 +22,7 @@
 
 /* Vortex MPU401 defines. */
 #define MIDI_CLOCK_DIV      0x61
-/* Standart MPU401 defines. */
+/* Standard MPU401 defines. */
 #define MPU401_RESET		0xff
 #define MPU401_ENTER_UART	0x3f
 #define MPU401_ACK		    0xfe
diff --git a/sound/pci/aw2/aw2-saa7146.c b/sound/pci/aw2/aw2-saa7146.c
index c84f1a45194f17..6857a0ec29248a 100644
--- a/sound/pci/aw2/aw2-saa7146.c
+++ b/sound/pci/aw2/aw2-saa7146.c
@@ -147,7 +147,7 @@ void snd_aw2_saa7146_pcm_init_playback(struct snd_aw2_saa7146 *chip,
 	unsigned long dw_page, dw_limit;
 
 	/* Configure DMA for substream
-	   Configuration informations: ALSA has allocated continuous memory
+	   Configuration information: ALSA has allocated continuous memory
 	   pages. So we don't need to use MMU of saa7146.
 	 */
 
@@ -204,7 +204,7 @@ void snd_aw2_saa7146_pcm_init_capture(struct snd_aw2_saa7146 *chip,
 	unsigned long dw_page, dw_limit;
 
 	/* Configure DMA for substream
-	   Configuration informations: ALSA has allocated continuous memory
+	   Configuration information: ALSA has allocated continuous memory
 	   pages. So we don't need to use MMU of saa7146.
 	 */
 
diff --git a/sound/pci/cs4281.c b/sound/pci/cs4281.c
index f51f4bb6376646..27ceda6278c5e9 100644
--- a/sound/pci/cs4281.c
+++ b/sound/pci/cs4281.c
@@ -324,7 +324,7 @@ MODULE_PARM_DESC(dual_codec, "Secondary Codec ID (0 = disabled).");
 #define BA0_SSCR_XLPSRC		(1<<8)	/* External SRC Loopback Mode */
 #define BA0_SSCR_LPSRC		(1<<7)	/* SRC Loopback Mode */
 #define BA0_SSCR_CDTX		(1<<5)	/* CD Transfer Data */
-#define BA0_SSCR_HVC		(1<<3)	/* Harware Volume Control Enable */
+#define BA0_SSCR_HVC		(1<<3)	/* Hardware Volume Control Enable */
 
 #define BA0_FMLVC		0x0754	/* FM Synthesis Left Volume Control */
 #define BA0_FMRVC		0x0758	/* FM Synthesis Right Volume Control */
diff --git a/sound/pci/cs46xx/cs46xx_lib.c b/sound/pci/cs46xx/cs46xx_lib.c
index f6db08b75649c7..797a5f09cade15 100644
--- a/sound/pci/cs46xx/cs46xx_lib.c
+++ b/sound/pci/cs46xx/cs46xx_lib.c
@@ -315,7 +315,7 @@ int snd_cs46xx_download(struct snd_cs46xx *chip,
 	dst = chip->region.idx[bank+1].remap_addr + offset;
 	len /= sizeof(u32);
 
-	/* writel already converts 32-bit value to right endianess */
+	/* writel already converts 32-bit value to right endianness */
 	while (len-- > 0) {
 		writel(*src++, dst);
 		dst += sizeof(u32);
@@ -474,7 +474,7 @@ int snd_cs46xx_clear_BA1(struct snd_cs46xx *chip,
 	dst = chip->region.idx[bank+1].remap_addr + offset;
 	len /= sizeof(u32);
 
-	/* writel already converts 32-bit value to right endianess */
+	/* writel already converts 32-bit value to right endianness */
 	while (len-- > 0) {
 		writel(0, dst);
 		dst += sizeof(u32);
@@ -3039,7 +3039,7 @@ static int snd_cs46xx_chip_init(struct snd_cs46xx *chip)
 #endif
 
 	/*
-	 *  Assert the vaid frame signal so that we can start sending commands
+	 *  Assert the valid frame signal so that we can start sending commands
 	 *  to the AC97 codec.
 	 */
 	snd_cs46xx_pokeBA0(chip, BA0_ACCTL, ACCTL_VFRM | ACCTL_ESYN | ACCTL_RSTN);
diff --git a/sound/pci/cs46xx/dsp_spos_scb_lib.c b/sound/pci/cs46xx/dsp_spos_scb_lib.c
index fd19365026b4a6..71eee5cb478f40 100644
--- a/sound/pci/cs46xx/dsp_spos_scb_lib.c
+++ b/sound/pci/cs46xx/dsp_spos_scb_lib.c
@@ -160,15 +160,15 @@ void cs46xx_dsp_remove_scb (struct snd_cs46xx *chip, struct dsp_scb_descriptor *
 {
 	struct dsp_spos_instance * ins = chip->dsp_spos_instance;
 
-	/* check integrety */
+	/* check integrity */
 	if (snd_BUG_ON(scb->index < 0 ||
 		       scb->index >= ins->nscb ||
 		       (ins->scbs + scb->index) != scb))
 		return;
 
 #if 0
-	/* can't remove a SCB with childs before 
-	   removing childs first  */
+	/* can't remove a SCB with children before
+	   removing children first  */
 	if (snd_BUG_ON(scb->sub_list_ptr != ins->the_null_scb ||
 		       scb->next_scb_ptr != ins->the_null_scb))
 		goto _end;
@@ -1656,7 +1656,7 @@ int  cs46xx_dsp_disable_spdif_out (struct snd_cs46xx *chip)
 		return -EBUSY;
 	}
 
-	/* check integrety */
+	/* check integrity */
 	if (snd_BUG_ON(!ins->asynch_tx_scb))
 		return -EINVAL;
 	if (snd_BUG_ON(!ins->spdif_pcm_input_scb))
diff --git a/sound/pci/echoaudio/echoaudio.c b/sound/pci/echoaudio/echoaudio.c
index 214d68657707d3..8b4637231debe0 100644
--- a/sound/pci/echoaudio/echoaudio.c
+++ b/sound/pci/echoaudio/echoaudio.c
@@ -270,7 +270,7 @@ static int pcm_open(struct snd_pcm_substream *substream,
 		return -ENOMEM;
 	pipe->index = -1;		/* Not configured yet */
 
-	/* Set up hw capabilities and contraints */
+	/* Set up hw capabilities and constraints */
 	memcpy(&pipe->hw, &pcm_hardware_skel, sizeof(struct snd_pcm_hardware));
 	dev_dbg(chip->card->dev, "max_channels=%d\n", max_channels);
 	pipe->constr.list = channels_list;
@@ -1720,7 +1720,7 @@ static const struct snd_kcontrol_new snd_echo_vumeters = {
 
 
 
-/*** Channels info - it exports informations about the number of channels ***/
+/*** Channels info - it exports information about the number of channels ***/
 static int snd_echo_channels_info_info(struct snd_kcontrol *kcontrol,
 				       struct snd_ctl_elem_info *uinfo)
 {
diff --git a/sound/pci/echoaudio/echoaudio.h b/sound/pci/echoaudio/echoaudio.h
index 7a5744bf3155a0..ac4507b7cc7856 100644
--- a/sound/pci/echoaudio/echoaudio.h
+++ b/sound/pci/echoaudio/echoaudio.h
@@ -394,8 +394,8 @@ struct echoaudio {
 
 	u16 digital_modes;		/* Bitmask of supported modes
 					 * (see ECHOCAPS_HAS_DIGITAL_MODE_*) */
-	u16 input_clock_types;		/* Suppoted input clock types */
-	u16 output_clock_types;		/* Suppoted output clock types -
+	u16 input_clock_types;		/* Supported input clock types */
+	u16 output_clock_types;		/* Supported output clock types -
 					 * Layla20 only */
 	u16 device_id, subdevice_id;
 	u16 *dsp_code;			/* Current DSP code loaded,
diff --git a/sound/pci/lx6464es/lx_core.c b/sound/pci/lx6464es/lx_core.c
index 9909f106787161..ae265dd193331d 100644
--- a/sound/pci/lx6464es/lx_core.c
+++ b/sound/pci/lx6464es/lx_core.c
@@ -364,7 +364,7 @@ int lx_dsp_get_mac(struct lx6464es *chip)
 	macmsb = lx_dsp_reg_read(chip, eReg_ADMACESMSB) & 0x00FFFFFF;
 	maclsb = lx_dsp_reg_read(chip, eReg_ADMACESLSB) & 0x00FFFFFF;
 
-	/* todo: endianess handling */
+	/* todo: endianness handling */
 	chip->mac_address[5] = ((u8 *)(&maclsb))[0];
 	chip->mac_address[4] = ((u8 *)(&maclsb))[1];
 	chip->mac_address[3] = ((u8 *)(&maclsb))[2];
diff --git a/sound/pci/lx6464es/lx_defs.h b/sound/pci/lx6464es/lx_defs.h
index eca5367ba561c7..89e433d3f7a2ff 100644
--- a/sound/pci/lx6464es/lx_defs.h
+++ b/sound/pci/lx6464es/lx_defs.h
@@ -315,7 +315,7 @@ enum stream_flags {
 #define ED_XILINX_ERROR                 (ED_RT | 0x07)
 #define ED_COBRANET_ITF_NOT_RESPONDING  (ED_RT | 0x08)
 
-/* Complete BOARD error code for the invaid standard object class */
+/* Complete BOARD error code for the invalid standard object class */
 #define EB_ISO          (ERROR_VALUE | E_SOURCE_BOARD | \
 			 E_CLASS_INVALID_STD_OBJECT)
 #define EB_INVALID_EFFECT               (EB_ISO | 0x00)
diff --git a/sound/pci/oxygen/oxygen_pcm.c b/sound/pci/oxygen/oxygen_pcm.c
index b716356010b898..75efbf59a67524 100644
--- a/sound/pci/oxygen/oxygen_pcm.c
+++ b/sound/pci/oxygen/oxygen_pcm.c
@@ -171,16 +171,14 @@ static int oxygen_open(struct snd_pcm_substream *substream,
 	snd_pcm_set_sync(substream);
 	chip->streams[channel] = substream;
 
-	guard(mutex)(&chip->mutex);
-	chip->pcm_active |= 1 << channel;
-	if (channel == PCM_SPDIF) {
-		chip->spdif_pcm_bits = chip->spdif_bits;
-		chip->controls[CONTROL_SPDIF_PCM]->vd[0].access &=
-			~SNDRV_CTL_ELEM_ACCESS_INACTIVE;
-		snd_ctl_notify(chip->card, SNDRV_CTL_EVENT_MASK_VALUE |
-			       SNDRV_CTL_EVENT_MASK_INFO,
-			       &chip->controls[CONTROL_SPDIF_PCM]->id);
+	scoped_guard(mutex, &chip->mutex) {
+		chip->pcm_active |= 1 << channel;
+		if (channel == PCM_SPDIF)
+			chip->spdif_pcm_bits = chip->spdif_bits;
 	}
+	if (channel == PCM_SPDIF)
+		snd_ctl_activate_id(chip->card,
+				    &chip->controls[CONTROL_SPDIF_PCM]->id, 1);
 
 	return 0;
 }
@@ -220,19 +218,16 @@ static int oxygen_close(struct snd_pcm_substream *substream)
 	struct oxygen *chip = snd_pcm_substream_chip(substream);
 	unsigned int channel = oxygen_substream_channel(substream);
 
-	guard(mutex)(&chip->mutex);
-	chip->pcm_active &= ~(1 << channel);
-	if (channel == PCM_SPDIF) {
-		chip->controls[CONTROL_SPDIF_PCM]->vd[0].access |=
-			SNDRV_CTL_ELEM_ACCESS_INACTIVE;
-		snd_ctl_notify(chip->card, SNDRV_CTL_EVENT_MASK_VALUE |
-			       SNDRV_CTL_EVENT_MASK_INFO,
-			       &chip->controls[CONTROL_SPDIF_PCM]->id);
-	}
-	if (channel == PCM_SPDIF || channel == PCM_MULTICH)
-		oxygen_update_spdif_source(chip);
+	scoped_guard(mutex, &chip->mutex) {
+		chip->pcm_active &= ~(1 << channel);
+		if (channel == PCM_SPDIF || channel == PCM_MULTICH)
+			oxygen_update_spdif_source(chip);
 
-	chip->streams[channel] = NULL;
+		chip->streams[channel] = NULL;
+	}
+	if (channel == PCM_SPDIF)
+		snd_ctl_activate_id(chip->card,
+				    &chip->controls[CONTROL_SPDIF_PCM]->id, 0);
 	return 0;
 }
 
diff --git a/sound/pci/pcxhr/pcxhr_core.c b/sound/pci/pcxhr/pcxhr_core.c
index b3b9ab4f303ee5..f855d3bfad4b63 100644
--- a/sound/pci/pcxhr/pcxhr_core.c
+++ b/sound/pci/pcxhr/pcxhr_core.c
@@ -802,7 +802,7 @@ static int pcxhr_prepair_pipe_start(struct pcxhr_mgr *mgr,
 					   err);
 				return err;
 			}
-			/* if the pipe couldn't be prepaired for start,
+			/* if the pipe couldn't be prepared for start,
 			 * retry it later
 			 */
 			if (rmh.stat[0] == 0)
@@ -913,7 +913,7 @@ int pcxhr_set_pipe_state(struct pcxhr_mgr *mgr, int playback_mask,
 			if (err)
 				return err;
 			if (state == 0)
-				break;	/* success, all pipes prepaired */
+				break;	/* success, all pipes prepared */
 			mdelay(1);	/* wait 1 millisecond and retry */
 		}
 	} else {
diff --git a/sound/pci/riptide/riptide.c b/sound/pci/riptide/riptide.c
index 99c00e46ce23dc..697fece744e7f5 100644
--- a/sound/pci/riptide/riptide.c
+++ b/sound/pci/riptide/riptide.c
@@ -264,7 +264,7 @@ MODULE_PARM_DESC(opl3_port, "OPL3 port # for Riptide driver.");
 #define SEND_SI2S(p,b)             sendcmd(p,PARM,SI2S,WORD2(b),RET(0))
 
 #define EOB_STATUS         0x80000000	/* status flags : block boundary */
-#define EOS_STATUS         0x40000000	/*              : stoppped */
+#define EOS_STATUS         0x40000000	/*              : stopped */
 #define EOC_STATUS         0x20000000	/*              : stream end */
 #define ERR_STATUS         0x10000000
 #define EMPTY_STATUS       0x08000000
@@ -453,7 +453,7 @@ struct snd_riptide {
 	int in_suspend;
 };
 
-struct sgd {			/* scatter gather desriptor */
+struct sgd {			/* scatter gather descriptor */
 	__le32 dwNextLink;
 	__le32 dwSegPtrPhys;
 	__le32 dwSegLen;
diff --git a/sound/pci/rme32.c b/sound/pci/rme32.c
index 454a30a2c07e17..1d5390621908e9 100644
--- a/sound/pci/rme32.c
+++ b/sound/pci/rme32.c
@@ -8,7 +8,7 @@
  *      Thanks to :        Anders Torger <torger@ludd.luth.se>,
  *                         Henk Hesselink <henk@anda.nl>
  *                         for writing the digi96-driver 
- *                         and RME for all informations.
+ *                         and RME for all information.
  * 
  * ****************************************************************************
  * 
diff --git a/sound/pci/rme9652/hdsp.c b/sound/pci/rme9652/hdsp.c
index 31cc2d91c8d2f8..0ade7305773183 100644
--- a/sound/pci/rme9652/hdsp.c
+++ b/sound/pci/rme9652/hdsp.c
@@ -3664,7 +3664,7 @@ snd_hdsp_proc_read(struct snd_info_entry *entry, struct snd_info_buffer *buffer)
 
 	snd_iprintf(buffer, "\n");
 
-	/* Informations about H9632 specific controls */
+	/* Information about H9632 specific controls */
 	if (hdsp->io_type == H9632) {
 		char *tmp;
 
diff --git a/sound/pci/trident/trident_memory.c b/sound/pci/trident/trident_memory.c
index 81f6348191dc44..4ed0e7d683228c 100644
--- a/sound/pci/trident/trident_memory.c
+++ b/sound/pci/trident/trident_memory.c
@@ -27,7 +27,7 @@
 #if PAGE_SIZE == 4096
 /* page size == SNDRV_TRIDENT_PAGE_SIZE */
 #define ALIGN_PAGE_SIZE		PAGE_SIZE	/* minimum page size for allocation */
-#define MAX_ALIGN_PAGES		SNDRV_TRIDENT_MAX_PAGES	/* maxmium aligned pages */
+#define MAX_ALIGN_PAGES		SNDRV_TRIDENT_MAX_PAGES	/* maximum aligned pages */
 /* fill TLB entrie(s) corresponding to page with ptr */
 #define set_tlb_bus(trident,page,addr) __set_tlb_bus(trident,page,addr)
 /* fill TLB entrie(s) corresponding to page with silence pointer */
diff --git a/sound/pci/via82xx.c b/sound/pci/via82xx.c
index 24ee1302f27d33..58816dea49963e 100644
--- a/sound/pci/via82xx.c
+++ b/sound/pci/via82xx.c
@@ -1242,11 +1242,7 @@ static int snd_via8233_playback_open(struct snd_pcm_substream *substream)
 				VIA_DXS_MAX_VOLUME - (dxs_init_volume & 31);
 		chip->playback_volume[stream][1] =
 				VIA_DXS_MAX_VOLUME - (dxs_init_volume & 31);
-		chip->dxs_controls[stream]->vd[0].access &=
-			~SNDRV_CTL_ELEM_ACCESS_INACTIVE;
-		snd_ctl_notify(chip->card, SNDRV_CTL_EVENT_MASK_VALUE |
-			       SNDRV_CTL_EVENT_MASK_INFO,
-			       &chip->dxs_controls[stream]->id);
+		snd_ctl_activate_id(chip->card, &chip->dxs_controls[stream]->id, 1);
 	}
 	return 0;
 }
@@ -1333,10 +1329,7 @@ static int snd_via8233_playback_close(struct snd_pcm_substream *substream)
 
 	stream = viadev->reg_offset / 0x10;
 	if (chip->dxs_controls[stream]) {
-		chip->dxs_controls[stream]->vd[0].access |=
-			SNDRV_CTL_ELEM_ACCESS_INACTIVE;
-		snd_ctl_notify(chip->card, SNDRV_CTL_EVENT_MASK_INFO,
-			       &chip->dxs_controls[stream]->id);
+		snd_ctl_activate_id(chip->card, &chip->dxs_controls[stream]->id, 0);
 	}
 	return snd_via82xx_pcm_close(substream);
 }
diff --git a/sound/pci/vx222/vx222_ops.c b/sound/pci/vx222/vx222_ops.c
index b6459dbdb1b050..40f3716c478f49 100644
--- a/sound/pci/vx222/vx222_ops.c
+++ b/sound/pci/vx222/vx222_ops.c
@@ -431,7 +431,7 @@ static int vx2_load_dsp(struct vx_core *vx, int index, const struct firmware *ds
 /*
  * vx_test_and_ack - test and acknowledge interrupt
  *
- * called from irq hander, too
+ * called from irq handler, too
  *
  * spinlock held!
  */
diff --git a/sound/pcmcia/vx/vxp_ops.c b/sound/pcmcia/vx/vxp_ops.c
index 4211e7239138c4..e52b61125629d2 100644
--- a/sound/pcmcia/vx/vxp_ops.c
+++ b/sound/pcmcia/vx/vxp_ops.c
@@ -265,7 +265,7 @@ static int vxp_load_dsp(struct vx_core *vx, int index, const struct firmware *fw
 /*
  * vx_test_and_ack - test and acknowledge interrupt
  *
- * called from irq hander, too
+ * called from irq handler, too
  *
  * spinlock held!
  */
diff --git a/sound/ppc/tumbler.c b/sound/ppc/tumbler.c
index 139c7b25927a97..7ba52e7a24d7d8 100644
--- a/sound/ppc/tumbler.c
+++ b/sound/ppc/tumbler.c
@@ -1134,7 +1134,7 @@ static long tumbler_find_device(const char *device, const char *platform,
 		gp->active_val = 0x4;
 		gp->inactive_val = 0x5;
 		/* Here are some crude hacks to extract the GPIO polarity and
-		 * open collector informations out of the do-platform script
+		 * open collector information out of the do-platform script
 		 * as we don't yet have an interpreter for these things
 		 */
 		if (platform)
diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c
index 2d50be8190c10e..b3b73096ac74a6 100644
--- a/sound/soc/amd/acp-config.c
+++ b/sound/soc/amd/acp-config.c
@@ -51,6 +51,13 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "83V9"),
 		},
 	},
+	{
+		/* Lenovo Yoga Pro 7 14ASP9 (Strix Point, ACP 7.0) */
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "83HN"),
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC"),
@@ -63,6 +70,13 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook 18 M1807GA"),
 		},
 	},
+	{
+		/* ASUS Zenbook S14 UM5406GA (Strix Point, ACP 7.0) */
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Zenbook S14 UM5406GA"),
+		},
+	},
 	{
 		/* HP OmniBook X Flip 14-kc0xxx (Strix Point, ACP 7.2) */
 		.matches = {
diff --git a/sound/soc/amd/acp-es8336.c b/sound/soc/amd/acp-es8336.c
index 9f3f11256788e2..0cb0ee76191d03 100644
--- a/sound/soc/amd/acp-es8336.c
+++ b/sound/soc/amd/acp-es8336.c
@@ -30,7 +30,6 @@
 
 static unsigned long acp2x_machine_id;
 static struct snd_soc_jack st_jack;
-static struct device *codec_dev;
 static struct gpio_desc *gpio_pa;
 
 static int sof_es8316_speaker_power_event(struct snd_soc_dapm_widget *w,
@@ -191,6 +190,7 @@ static const struct acpi_gpio_mapping acpi_es8336_gpios[] = {
 
 static int st_es8336_late_probe(struct snd_soc_card *card)
 {
+	struct device *codec_dev __free(put_device) = NULL;
 	struct acpi_device *adev;
 	int ret;
 
@@ -198,7 +198,7 @@ static int st_es8336_late_probe(struct snd_soc_card *card)
 	if (!adev)
 		return -ENODEV;
 
-	codec_dev = acpi_get_first_physical_node(adev);
+	codec_dev = acpi_bus_get_primary_device(adev);
 	acpi_dev_put(adev);
 	if (!codec_dev) {
 		dev_err(card->dev, "can not find codec dev\n");
@@ -213,7 +213,6 @@ static int st_es8336_late_probe(struct snd_soc_card *card)
 	if (IS_ERR(gpio_pa)) {
 		ret = dev_err_probe(card->dev, PTR_ERR(gpio_pa),
 				    "could not get pa-enable GPIO\n");
-		put_device(codec_dev);
 		return ret;
 	}
 	return 0;
diff --git a/sound/soc/amd/acp/Kconfig b/sound/soc/amd/acp/Kconfig
index a04c301df4d616..b7b7b4f4a55ca3 100644
--- a/sound/soc/amd/acp/Kconfig
+++ b/sound/soc/amd/acp/Kconfig
@@ -167,7 +167,7 @@ config SND_SOC_AMD_SOF_SDW_MACH
 
 config SND_SOC_AMD_LEGACY_SDW_MACH
 	tristate "AMD Legacy(No DSP) Soundwire Machine Driver Support"
-	depends on X86 && PCI && ACPI
+	depends on X86 && PCI && ACPI && EFI
 	depends on SOUNDWIRE
 	select SND_SOC_AMD_SDW_MACH_COMMON
 	select SND_SOC_SDW_UTILS
@@ -181,6 +181,7 @@ config SND_SOC_AMD_LEGACY_SDW_MACH
 	select SND_SOC_RT715_SDW
 	select SND_SOC_RT715_SDCA_SDW
 	select SND_SOC_RT722_SDCA_SDW
+	select SND_SOC_TAS2783_SDW if SND_SOC_SDCA && EFI
 	help
 	  This option enables Legacy(No DSP) sound card support for SoundWire
 	  enabled AMD platforms along with ACP PDM controller.
diff --git a/sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c b/sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c
index 3a640e65231420..eb0e9a6af6ef10 100644
--- a/sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c
+++ b/sound/soc/amd/acp/acp3x-es83xx/acp3x-es83xx.c
@@ -10,6 +10,7 @@
 #include <sound/soc-dapm.h>
 #include <sound/jack.h>
 #include <sound/soc-acpi.h>
+#include <acpi/acpi_bus.h>
 #include <linux/clk.h>
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
@@ -41,6 +42,11 @@ struct acp3x_es83xx_private {
 	struct snd_soc_dapm_route mic_map[2];
 };
 
+static void acp3x_es83xx_put_codec_device(void *data)
+{
+	put_device(data);
+}
+
 static const unsigned int channels[] = {
 	DUAL_CHANNEL,
 };
@@ -428,7 +434,7 @@ static int acp3x_es83xx_probe(struct snd_soc_card *card)
 			return -ENXIO;
 		}
 
-		codec_dev = acpi_get_first_physical_node(adev);
+		codec_dev = acpi_bus_get_primary_device(adev);
 		acpi_dev_put(adev);
 		if (!codec_dev) {
 			dev_warn(dev, "Error cannot find codec device, will defer probe\n");
@@ -441,6 +447,12 @@ static int acp3x_es83xx_probe(struct snd_soc_card *card)
 			return -ENOMEM;
 		}
 
+		ret = devm_add_action_or_reset(dev,
+					       acp3x_es83xx_put_codec_device,
+					       codec_dev);
+		if (ret)
+			return ret;
+
 		priv->codec_dev = codec_dev;
 		priv->quirk = (unsigned long)dmi_id->driver_data;
 		acp_drvdata->mach_priv = priv;
diff --git a/sound/soc/amd/acp/amd-acp70-acpi-match.c b/sound/soc/amd/acp/amd-acp70-acpi-match.c
index 815e088b437d7f..2a9da6de7bbcae 100644
--- a/sound/soc/amd/acp/amd-acp70-acpi-match.c
+++ b/sound/soc/amd/acp/amd-acp70-acpi-match.c
@@ -748,7 +748,27 @@ static const struct snd_soc_acpi_link_adr acp70_rt721_only[] = {
 	{}
 };
 
+static const struct snd_soc_acpi_link_adr acp70_tas2783a_l0_rt712_vb_l1[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(tas2783_2_adr),
+		.adr_d = tas2783_2_adr,
+	},
+	{
+		.mask = BIT(1),
+		.num_adr = ARRAY_SIZE(rt712_vb_1_group1_adr),
+		.adr_d = rt712_vb_1_group1_adr,
+	},
+	{}
+};
+
 struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sdw_machines[] = {
+	{
+		.link_mask = BIT(0) | BIT(1),
+		.links = acp70_tas2783a_l0_rt712_vb_l1,
+		.machine_check = snd_soc_acpi_amd_sdca_is_device_rt712_vb,
+		.drv_name = "amd_sdw",
+	},
 	{
 		.link_mask = BIT(0),
 		.links = acp70_tas2783_2,
diff --git a/sound/soc/amd/acp/soc_amd_sdw_common.h b/sound/soc/amd/acp/soc_amd_sdw_common.h
index 3930cc46fa5871..17e4e97fb3d1d6 100644
--- a/sound/soc/amd/acp/soc_amd_sdw_common.h
+++ b/sound/soc/amd/acp/soc_amd_sdw_common.h
@@ -23,6 +23,19 @@
 #define ACP71_PCI_REV			0x71
 #define ACP72_PCI_REV			0x72
 
+/**
+ * struct amd_pdm_pdata - platform data passed via mach->pdata to machine driver
+ * @pdm_sel: active PDM controller (ACP7X_PDM_DMIC0 or ACP7X_PDM_DMIC1),
+ *           non-zero when a PDM controller was identified via ACPI _DSD
+ *
+ * Carries the PDM controller selection for ACP7.B/7.F platforms, derived
+ * from the acp-audio-ep-port ACPI _DSD property and passed via mach->pdata
+ * to the machine driver.
+ */
+struct amd_pdm_pdata {
+	unsigned int pdm_sel;
+};
+
 #define SOC_JACK_JDSRC(quirk)		((quirk) & GENMASK(3, 0))
 #define ASOC_SDW_FOUR_SPK		BIT(4)
 #define ASOC_SDW_ACP_DMIC		BIT(5)
diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c
index 385fede6d77f3d..e798bf6a9cd065 100644
--- a/sound/soc/amd/yc/acp6x-mach.c
+++ b/sound/soc/amd/yc/acp6x-mach.c
@@ -248,6 +248,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "21J6"),
 		}
 	},
+	{
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "21JN"),
+		}
+	},
 	{
 		.driver_data = &acp6x_card,
 		.matches = {
@@ -381,6 +388,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "83Q3"),
 		}
 	},
+	{
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "83UU"),
+		}
+	},
 	{
 		.driver_data = &acp6x_card,
 		.matches = {
diff --git a/sound/soc/atmel/atmel-pcm-dma.c b/sound/soc/atmel/atmel-pcm-dma.c
index 7306e04da513b1..e4b73ae3df4f4c 100644
--- a/sound/soc/atmel/atmel-pcm-dma.c
+++ b/sound/soc/atmel/atmel-pcm-dma.c
@@ -47,7 +47,7 @@ static const struct snd_pcm_hardware atmel_pcm_dma_hardware = {
  * atmel_pcm_dma_irq: SSC interrupt handler for DMAENGINE enabled SSC
  *
  * We use DMAENGINE to send/receive data to/from SSC so this ISR is only to
- * check if any overrun occured.
+ * check if any overrun occurred.
  */
 static void atmel_pcm_dma_irq(u32 ssc_sr,
 	struct snd_pcm_substream *substream)
diff --git a/sound/soc/bcm/cygnus-ssp.h b/sound/soc/bcm/cygnus-ssp.h
index 4925e03c3c3045..7848e15efbe148 100644
--- a/sound/soc/bcm/cygnus-ssp.h
+++ b/sound/soc/bcm/cygnus-ssp.h
@@ -27,7 +27,7 @@ struct ringbuf_regs {
 	unsigned wraddr;
 	unsigned baseaddr;
 	unsigned endaddr;
-	unsigned fmark;   /* freemark for play, fullmark for caputure */
+	unsigned fmark;   /* freemark for play, fullmark for capture */
 	unsigned period_bytes;
 	unsigned buf_size;
 };
diff --git a/sound/soc/codecs/88pm860x-codec.c b/sound/soc/codecs/88pm860x-codec.c
index b4f5b24cde4562..7596227fbccfc9 100644
--- a/sound/soc/codecs/88pm860x-codec.c
+++ b/sound/soc/codecs/88pm860x-codec.c
@@ -1136,11 +1136,15 @@ static int pm860x_set_bias_level(struct snd_soc_component *component,
 	return 0;
 }
 
+static const u64 pm860x_selectable_formats = SND_SOC_POSSIBLE_DAIFMT_I2S;
+
 static const struct snd_soc_dai_ops pm860x_pcm_dai_ops = {
 	.mute_stream	= pm860x_mute_stream,
 	.hw_params	= pm860x_pcm_hw_params,
 	.set_fmt	= pm860x_pcm_set_dai_fmt,
 	.set_sysclk	= pm860x_set_dai_sysclk,
+	.auto_selectable_formats	= &pm860x_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
@@ -1149,6 +1153,8 @@ static const struct snd_soc_dai_ops pm860x_i2s_dai_ops = {
 	.hw_params	= pm860x_i2s_hw_params,
 	.set_fmt	= pm860x_i2s_set_dai_fmt,
 	.set_sysclk	= pm860x_set_dai_sysclk,
+	.auto_selectable_formats	= &pm860x_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index d88593c2bac8b9..ba59dabeeccde5 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -123,6 +123,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_ES8328_I2C
 	imply SND_SOC_ES8375
 	imply SND_SOC_ES8389
+	imply SND_SOC_ES9039Q2M
 	imply SND_SOC_ES9356
 	imply SND_SOC_ES7134
 	imply SND_SOC_ES7241
@@ -173,6 +174,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_MT6660
 	imply SND_SOC_NAU8315
 	imply SND_SOC_NAU8325
+	imply SND_SOC_NAU8360
 	imply SND_SOC_NAU8540
 	imply SND_SOC_NAU8810
 	imply SND_SOC_NAU8821
@@ -255,6 +257,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_SIMPLE_MUX
 	imply SND_SOC_SMA1303
 	imply SND_SOC_SMA1307
+	imply SND_SOC_SN624X_SDCA_SDW
 	imply SND_SOC_SPDIF
 	imply SND_SOC_SRC4XXX_I2C
 	imply SND_SOC_SSM2305
@@ -1315,6 +1318,19 @@ config SND_SOC_ES8389
 	tristate "Everest Semi ES8389 CODEC"
 	depends on I2C
 
+config SND_SOC_ES9039Q2M
+	tristate "ESS Technology ES9039Q2M CODEC"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Enable support for the ESS Technology ES9039Q2M, a 32-bit
+	  two-channel audio DAC with an asynchronous sample rate converter,
+	  supporting PCM, DSD, DoP and S/PDIF input. This driver covers the
+	  I2C control interface, which the MODE pin selects.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called snd-soc-es9039q2m.
+
 config SND_SOC_ES9356
 	tristate "Everest Semi ES9356 CODEC SDW"
 	depends on SND_SOC_SDCA
@@ -1989,9 +2005,7 @@ config SND_SOC_RT715_SDCA_SDW
 config SND_SOC_RT766_SDCA_SDW
 	tristate "Realtek RT766 SDCA Codec - SDW"
 	depends on SOUNDWIRE
-	depends on SND_SOC_SDCA
-	select SND_SOC_SDCA_HID
-	select SND_SOC_SDCA_IRQ
+	depends on SND_SOC_SDCA_CLASS
 	select REGMAP_SOUNDWIRE
 	select REGMAP_SOUNDWIRE_MBQ
 
@@ -2099,6 +2113,18 @@ config SND_SOC_SMA1307
 	  If you are using a system with an SMA1307 amplifier connected
 	  via I2C, enable this option.
 
+config SND_SOC_SN624X_SDCA_SDW
+	tristate "Senary SN624x SDCA Codec - SDW"
+	depends on SND_SOC_SDCA
+	depends on SOUNDWIRE
+	select REGMAP_SOUNDWIRE
+	select REGMAP_SOUNDWIRE_MBQ
+	help
+	  Enable support for Senary SN624x SDCA SoundWire codecs.
+	  These multi-function devices provide headset jack, speaker
+	  amplifier, and digital microphone endpoints on the SoundWire
+	  bus.
+
 config SND_SOC_SPDIF
 	tristate "S/PDIF CODEC"
 
@@ -2526,6 +2552,17 @@ config SND_SOC_WCD939X_SDW
 	  The WCD9390/9395 is a audio codec IC Integrated in
 	  Qualcomm SoCs like SM8650.
 
+config SND_SOC_WCD9378_SDCA
+	tristate "Qualcomm Tambora (WCD9378) SDCA codec"
+	depends on SOUNDWIRE
+	depends on SND_SOC_SDCA_CLASS
+	help
+	  This enables support for the Qualcomm Tambora (WCD9378) headset
+	  codec when driven via the SDCA class driver on ARM platforms
+	  without ACPI/DisCo tables. It provides the static SDCA topology
+	  and SoundWire data port properties transcribed from the factory
+	  ACPI tables.
+
 config SND_SOC_WM0010
 	tristate
 	depends on SPI_MASTER
@@ -2903,6 +2940,16 @@ config SND_SOC_NAU8825
 	tristate
 	depends on I2C
 
+config SND_SOC_NAU8360
+	tristate "Nuvoton Technology Corporation NAU83G60 Stereo Smart Amplifier"
+	depends on I2C
+	select FW_LOADER
+	help
+	  The NAU83G60 is a stereo 30W+30W smart amplifier with integrated
+	  low-latency Advanced Audio DSP running the KCS (Klippel Controlled
+	  Sound) speaker control algorithm. KCS provides mechanical and
+	  thermal speaker protection with voltage and current sensing.
+
 config SND_SOC_NTPFW
 	tristate
 
diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile
index 3d122ace75adee..3826f9e4fce8a2 100644
--- a/sound/soc/codecs/Makefile
+++ b/sound/soc/codecs/Makefile
@@ -140,6 +140,7 @@ snd-soc-es8328-i2c-y := es8328-i2c.o
 snd-soc-es8328-spi-y := es8328-spi.o
 snd-soc-es8375-y := es8375.o
 snd-soc-es8389-y := es8389.o
+snd-soc-es9039q2m-y := es9039q2m.o
 snd-soc-es9356-y := es9356.o
 snd-soc-framer-y := framer-codec.o
 snd-soc-fs-amp-lib-y := fs-amp-lib.o
@@ -198,6 +199,7 @@ snd-soc-mt6359-accdet-y := mt6359-accdet.o
 snd-soc-mt6660-y := mt6660.o
 snd-soc-nau8315-y := nau8315.o
 snd-soc-nau8325-y := nau8325.o
+snd-soc-nau8360-y := nau8360.o nau8360-dsp.o
 snd-soc-nau8540-y := nau8540.o
 snd-soc-nau8810-y := nau8810.o
 snd-soc-nau8821-y := nau8821.o
@@ -301,6 +303,7 @@ snd-soc-sigmadsp-regmap-y := sigmadsp-regmap.o
 snd-soc-si476x-y := si476x.o
 snd-soc-sma1303-y := sma1303.o
 snd-soc-sma1307-y := sma1307.o
+snd-soc-sn624x-sdca-y := sn624x-sdca-sdw.o
 snd-soc-spdif-tx-y := spdif_transmitter.o
 snd-soc-spdif-rx-y := spdif_receiver.o
 snd-soc-src4xxx-y := src4xxx.o
@@ -366,6 +369,7 @@ snd-soc-wcd938x-y := wcd938x.o
 snd-soc-wcd938x-sdw-y := wcd938x-sdw.o
 snd-soc-wcd939x-y := wcd939x.o
 snd-soc-wcd939x-sdw-y := wcd939x-sdw.o
+snd-soc-wcd9378-y := wcd9378-sdw.o wcd9378-sdca.o
 snd-soc-wm-adsp-y := wm_adsp.o
 snd-soc-wm-adsp-test-y := wm_adsp_fw_find_test.o
 snd-soc-wm0010-y := wm0010.o
@@ -584,6 +588,7 @@ obj-$(CONFIG_SND_SOC_ES8328_I2C)+= snd-soc-es8328-i2c.o
 obj-$(CONFIG_SND_SOC_ES8328_SPI)+= snd-soc-es8328-spi.o
 obj-$(CONFIG_SND_SOC_ES8375)    += snd-soc-es8375.o
 obj-$(CONFIG_SND_SOC_ES8389)    += snd-soc-es8389.o
+obj-$(CONFIG_SND_SOC_ES9039Q2M)	+= snd-soc-es9039q2m.o
 obj-$(CONFIG_SND_SOC_ES9356)    += snd-soc-es9356.o
 obj-$(CONFIG_SND_SOC_FRAMER)	+= snd-soc-framer.o
 obj-$(CONFIG_SND_SOC_FS_AMP_LIB)+= snd-soc-fs-amp-lib.o
@@ -637,6 +642,7 @@ obj-$(CONFIG_SND_SOC_MT6359_ACCDET) += mt6359-accdet.o
 obj-$(CONFIG_SND_SOC_MT6660)	+= snd-soc-mt6660.o
 obj-$(CONFIG_SND_SOC_NAU8315)   += snd-soc-nau8315.o
 obj-$(CONFIG_SND_SOC_NAU8325)   += snd-soc-nau8325.o
+obj-$(CONFIG_SND_SOC_NAU8360)   += snd-soc-nau8360.o
 obj-$(CONFIG_SND_SOC_NAU8540)   += snd-soc-nau8540.o
 obj-$(CONFIG_SND_SOC_NAU8810)   += snd-soc-nau8810.o
 obj-$(CONFIG_SND_SOC_NAU8821)   += snd-soc-nau8821.o
@@ -742,6 +748,7 @@ obj-$(CONFIG_SND_SOC_SIGMADSP_REGMAP)	+= snd-soc-sigmadsp-regmap.o
 obj-$(CONFIG_SND_SOC_SI476X)	+= snd-soc-si476x.o
 obj-$(CONFIG_SND_SOC_SMA1303)	+= snd-soc-sma1303.o
 obj-$(CONFIG_SND_SOC_SMA1307)	+= snd-soc-sma1307.o
+obj-$(CONFIG_SND_SOC_SN624X_SDCA_SDW)    += snd-soc-sn624x-sdca.o
 obj-$(CONFIG_SND_SOC_SPDIF)	+= snd-soc-spdif-rx.o snd-soc-spdif-tx.o
 obj-$(CONFIG_SND_SOC_SRC4XXX)	+= snd-soc-src4xxx.o
 obj-$(CONFIG_SND_SOC_SRC4XXX_I2C)	+= snd-soc-src4xxx-i2c.o
@@ -819,6 +826,7 @@ ifdef CONFIG_SND_SOC_WCD939X_SDW
 # avoid link failure by forcing sdw code built-in when needed
 obj-$(CONFIG_SND_SOC_WCD939X) += snd-soc-wcd939x-sdw.o
 endif
+obj-$(CONFIG_SND_SOC_WCD9378_SDCA) += snd-soc-wcd9378.o
 obj-$(CONFIG_SND_SOC_WM0010)	+= snd-soc-wm0010.o
 obj-$(CONFIG_SND_SOC_WM1250_EV1) += snd-soc-wm1250-ev1.o
 obj-$(CONFIG_SND_SOC_WM2000)	+= snd-soc-wm2000.o
diff --git a/sound/soc/codecs/ab8500-codec.c b/sound/soc/codecs/ab8500-codec.c
index e1a0e35836e6c8..3e3ff9bfe00d1b 100644
--- a/sound/soc/codecs/ab8500-codec.c
+++ b/sound/soc/codecs/ab8500-codec.c
@@ -299,7 +299,6 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = {
 	SND_SOC_DAPM_REGULATOR_SUPPLY("V-AUD", 0, 0),
 	SND_SOC_DAPM_REGULATOR_SUPPLY("V-AMIC1", 0, 0),
 	SND_SOC_DAPM_REGULATOR_SUPPLY("V-AMIC2", 0, 0),
-	SND_SOC_DAPM_REGULATOR_SUPPLY("V-DMIC", 0, 0),
 
 	/* Power */
 	SND_SOC_DAPM_SUPPLY("Audio Power",
@@ -324,7 +323,6 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = {
 	SND_SOC_DAPM_AIF_OUT("AD_OUT1", NULL, 0, SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_AIF_OUT("AD_OUT2", NULL, 0, SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_AIF_OUT("AD_OUT3", NULL, 0, SND_SOC_NOPM, 0, 0),
-	SND_SOC_DAPM_AIF_OUT("AD_OUT4", NULL, 0, SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_AIF_OUT("AD_OUT57", NULL, 0, SND_SOC_NOPM, 0, 0),
 	SND_SOC_DAPM_AIF_OUT("AD_OUT68", NULL, 0, SND_SOC_NOPM, 0, 0),
 
@@ -584,36 +582,6 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = {
 	SND_SOC_DAPM_SUPPLY("AD5768 Enable", AB8500_ADPATHENA,
 			    AB8500_ADPATHENA_ENAD5768, 0, NULL, 0),
 
-	/* Digital Microphone path */
-
-	SND_SOC_DAPM_INPUT("DMic 1"),
-	SND_SOC_DAPM_INPUT("DMic 2"),
-	SND_SOC_DAPM_INPUT("DMic 3"),
-	SND_SOC_DAPM_INPUT("DMic 4"),
-	SND_SOC_DAPM_INPUT("DMic 5"),
-	SND_SOC_DAPM_INPUT("DMic 6"),
-
-	SND_SOC_DAPM_MIXER("DMIC1",
-			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC1, 0,
-			NULL, 0),
-	SND_SOC_DAPM_MIXER("DMIC2",
-			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC2, 0,
-			NULL, 0),
-	SND_SOC_DAPM_MIXER("DMIC3",
-			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC3, 0,
-			NULL, 0),
-	SND_SOC_DAPM_MIXER("DMIC4",
-			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC4, 0,
-			NULL, 0),
-	SND_SOC_DAPM_MIXER("DMIC5",
-			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC5, 0,
-			NULL, 0),
-	SND_SOC_DAPM_MIXER("DMIC6",
-			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC6, 0,
-			NULL, 0),
-	SND_SOC_DAPM_MIXER("AD4 Channel Volume",
-			SND_SOC_NOPM, 0, 0,
-			NULL, 0),
 	/* Acoustical Noise Cancellation path */
 
 	SND_SOC_DAPM_MUX("ANC Source",
@@ -648,6 +616,40 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = {
 			NULL, 0),
 };
 
+static const struct snd_soc_dapm_widget ab8500_dmic_dapm_widgets[] = {
+	SND_SOC_DAPM_REGULATOR_SUPPLY("V-DMIC", 0, 0),
+	SND_SOC_DAPM_AIF_OUT("AD_OUT4", NULL, 0, SND_SOC_NOPM, 0, 0),
+
+	SND_SOC_DAPM_INPUT("DMic 1"),
+	SND_SOC_DAPM_INPUT("DMic 2"),
+	SND_SOC_DAPM_INPUT("DMic 3"),
+	SND_SOC_DAPM_INPUT("DMic 4"),
+	SND_SOC_DAPM_INPUT("DMic 5"),
+	SND_SOC_DAPM_INPUT("DMic 6"),
+
+	SND_SOC_DAPM_MIXER("DMIC1",
+			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC1, 0,
+			NULL, 0),
+	SND_SOC_DAPM_MIXER("DMIC2",
+			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC2, 0,
+			NULL, 0),
+	SND_SOC_DAPM_MIXER("DMIC3",
+			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC3, 0,
+			NULL, 0),
+	SND_SOC_DAPM_MIXER("DMIC4",
+			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC4, 0,
+			NULL, 0),
+	SND_SOC_DAPM_MIXER("DMIC5",
+			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC5, 0,
+			NULL, 0),
+	SND_SOC_DAPM_MIXER("DMIC6",
+			AB8500_DIGMICCONF, AB8500_DIGMICCONF_ENDMIC6, 0,
+			NULL, 0),
+	SND_SOC_DAPM_MIXER("AD4 Channel Volume",
+			SND_SOC_NOPM, 0, 0,
+			NULL, 0),
+};
+
 /*
  * DAPM-routes
  */
@@ -853,35 +855,6 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = {
 	{"AD_OUT68", NULL, "Main Supply"},
 	{"AD_OUT68", NULL, "AD6 Channel Volume"},
 
-	/* Digital Microphone path */
-
-	{"DMic 1", NULL, "V-DMIC"},
-	{"DMic 2", NULL, "V-DMIC"},
-	{"DMic 3", NULL, "V-DMIC"},
-	{"DMic 4", NULL, "V-DMIC"},
-	{"DMic 5", NULL, "V-DMIC"},
-	{"DMic 6", NULL, "V-DMIC"},
-
-	{"DMIC1", NULL, "DMic 1"},
-	{"DMIC2", NULL, "DMic 2"},
-	{"DMIC3", NULL, "DMic 3"},
-	{"DMIC4", NULL, "DMic 4"},
-	{"DMIC5", NULL, "DMic 5"},
-	{"DMIC6", NULL, "DMic 6"},
-
-	{"AD1 Source Select", "DMic 1", "DMIC1"},
-	{"AD2 Source Select", "DMic 2", "DMIC2"},
-	{"AD3 Source Select", "DMic 3", "DMIC3"},
-	{"AD5 Source Select", "DMic 5", "DMIC5"},
-	{"AD6 Source Select", "DMic 6", "DMIC6"},
-
-	{"AD4 Channel Volume", NULL, "DMIC4"},
-	{"AD4 Channel Volume", NULL, "AD34 Enable"},
-
-	{"ab8500_0c", NULL, "AD_OUT4"},
-	{"AD_OUT4", NULL, "Main Supply"},
-	{"AD_OUT4", NULL, "AD4 Channel Volume"},
-
 	/* LineIn Bypass path */
 
 	{"LINL to HSL Volume", NULL, "LINL Enable"},
@@ -911,7 +884,6 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = {
 	{"Sidetone Left Source", "Headset Left", "DA_IN1"},
 	{"Sidetone Right Source", "LineIn Right", "AD2 Channel Volume"},
 	{"Sidetone Right Source", "Mic 1", "AD3 Channel Volume"},
-	{"Sidetone Right Source", "DMic 4", "AD4 Channel Volume"},
 	{"Sidetone Right Source", "Headset Right", "DA_IN2"},
 
 	{"STFIR1 Control", NULL, "Sidetone Left Source"},
@@ -924,6 +896,37 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = {
 	{"DA2 Enable", NULL, "STFIR2 Volume"},
 };
 
+static const struct snd_soc_dapm_route ab8500_dmic_dapm_routes[] = {
+	{"DMic 1", NULL, "V-DMIC"},
+	{"DMic 2", NULL, "V-DMIC"},
+	{"DMic 3", NULL, "V-DMIC"},
+	{"DMic 4", NULL, "V-DMIC"},
+	{"DMic 5", NULL, "V-DMIC"},
+	{"DMic 6", NULL, "V-DMIC"},
+
+	{"DMIC1", NULL, "DMic 1"},
+	{"DMIC2", NULL, "DMic 2"},
+	{"DMIC3", NULL, "DMic 3"},
+	{"DMIC4", NULL, "DMic 4"},
+	{"DMIC5", NULL, "DMic 5"},
+	{"DMIC6", NULL, "DMic 6"},
+
+	{"AD1 Source Select", "DMic 1", "DMIC1"},
+	{"AD2 Source Select", "DMic 2", "DMIC2"},
+	{"AD3 Source Select", "DMic 3", "DMIC3"},
+	{"AD5 Source Select", "DMic 5", "DMIC5"},
+	{"AD6 Source Select", "DMic 6", "DMIC6"},
+
+	{"AD4 Channel Volume", NULL, "DMIC4"},
+	{"AD4 Channel Volume", NULL, "AD34 Enable"},
+
+	{"ab8500_0c", NULL, "AD_OUT4"},
+	{"AD_OUT4", NULL, "Main Supply"},
+	{"AD_OUT4", NULL, "AD4 Channel Volume"},
+
+	{"Sidetone Right Source", "DMic 4", "AD4 Channel Volume"},
+};
+
 static const struct snd_soc_dapm_route ab8500_dapm_routes_mic1a_vamicx[] = {
 	{"MIC1A V-AMICx Enable", NULL, "V-AMIC1"},
 	{"MIC1A V-AMICx Enable", NULL, "V-AMIC2"},
@@ -1950,9 +1953,22 @@ static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai,
 	return 0;
 }
 
+static const u64 ab8500_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_GATED	|
+	SND_SOC_POSSIBLE_DAIFMT_CONT	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops ab8500_codec_ops = {
 	.set_fmt = ab8500_codec_set_dai_fmt,
 	.set_tdm_slot = ab8500_codec_set_dai_tdm_slot,
+	.auto_selectable_formats = &ab8500_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver ab8500_codec_dai[] = {
@@ -2043,13 +2059,30 @@ static int ab8500_codec_probe(struct snd_soc_component *component)
 {
 	struct device *dev = component->dev;
 	struct device_node *np = dev->of_node;
+	struct ab8500 *ab8500 = dev_get_drvdata(dev->parent);
 	struct ab8500_codec_platform_data codec_pdata;
+	struct snd_soc_dapm_context *dapm;
 	int status;
 
 	dev_dbg(dev, "%s: Enter.\n", __func__);
 
 	ab8500_codec_of_probe(dev, np, &codec_pdata);
 
+	if (is_ab8500(ab8500)) {
+		dapm = snd_soc_component_to_dapm(component);
+		status = snd_soc_dapm_new_controls(dapm,
+						   ab8500_dmic_dapm_widgets,
+						   ARRAY_SIZE(ab8500_dmic_dapm_widgets));
+		if (status)
+			return status;
+
+		status = snd_soc_dapm_add_routes(dapm,
+						 ab8500_dmic_dapm_routes,
+						 ARRAY_SIZE(ab8500_dmic_dapm_routes));
+		if (status)
+			return status;
+	}
+
 	status = ab8500_audio_init_audioblock(component);
 	if (status < 0) {
 		dev_err(dev, "%s: failed to init audio-block (%d)!\n",
diff --git a/sound/soc/codecs/adau1372.c b/sound/soc/codecs/adau1372.c
index cc174ec3a1f825..0830d7d2dd7c59 100644
--- a/sound/soc/codecs/adau1372.c
+++ b/sound/soc/codecs/adau1372.c
@@ -881,12 +881,24 @@ static const struct snd_soc_component_driver adau1372_driver = {
 	.endianness = 1,
 };
 
+static const u64 adau1372_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops adau1372_dai_ops = {
 	.set_fmt = adau1372_set_dai_fmt,
 	.set_tdm_slot = adau1372_set_tdm_slot,
 	.set_tristate = adau1372_set_tristate,
 	.hw_params = adau1372_hw_params,
 	.startup = adau1372_startup,
+	.auto_selectable_formats = &adau1372_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 #define ADAU1372_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | \
diff --git a/sound/soc/codecs/adau1373.c b/sound/soc/codecs/adau1373.c
index 5dd961b233ce95..aa7d3a4b9f5885 100644
--- a/sound/soc/codecs/adau1373.c
+++ b/sound/soc/codecs/adau1373.c
@@ -1196,10 +1196,22 @@ static int adau1373_set_dai_sysclk(struct snd_soc_dai *dai,
 	return 0;
 }
 
+static const u64 adau1373_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops adau1373_dai_ops = {
 	.hw_params	= adau1373_hw_params,
 	.set_sysclk	= adau1373_set_dai_sysclk,
 	.set_fmt	= adau1373_set_dai_fmt,
+	.auto_selectable_formats	= &adau1373_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 #define ADAU1373_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE | \
diff --git a/sound/soc/codecs/adau1701.c b/sound/soc/codecs/adau1701.c
index 329ab01b62c002..4afbf46ffaa397 100644
--- a/sound/soc/codecs/adau1701.c
+++ b/sound/soc/codecs/adau1701.c
@@ -4,7 +4,7 @@
  *
  * Copyright 2011 Analog Devices Inc.
  * Author: Lars-Peter Clausen <lars@metafoo.de>
- *	based on an inital version by Cliff Cai <cliff.cai@analog.com>
+ *	based on an initial version by Cliff Cai <cliff.cai@analog.com>
  */
 
 #include <linux/module.h>
@@ -627,11 +627,22 @@ static int adau1701_startup(struct snd_pcm_substream *substream,
 #define ADAU1701_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
 	SNDRV_PCM_FMTBIT_S24_LE)
 
+static const u64 adau1701_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops adau1701_dai_ops = {
 	.set_fmt	= adau1701_set_dai_fmt,
 	.hw_params	= adau1701_hw_params,
 	.mute_stream	= adau1701_mute_stream,
 	.startup	= adau1701_startup,
+	.auto_selectable_formats	= &adau1701_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
@@ -688,7 +699,7 @@ static int adau1701_probe(struct snd_soc_component *component)
 	 */
 	adau1701->pll_clkdiv = ADAU1707_CLKDIV_UNSET;
 
-	/* initalize with pre-configured pll mode settings */
+	/* initialize with pre-configured pll mode settings */
 	ret = adau1701_reset(component, adau1701->pll_clkdiv, 0);
 	if (ret < 0)
 		goto exit_regulators_disable;
diff --git a/sound/soc/codecs/adau17x1.c b/sound/soc/codecs/adau17x1.c
index 26d7eb437ad183..207189cfd3a6cc 100644
--- a/sound/soc/codecs/adau17x1.c
+++ b/sound/soc/codecs/adau17x1.c
@@ -776,6 +776,17 @@ static int adau17x1_startup(struct snd_pcm_substream *substream,
 	return 0;
 }
 
+static const u64 adau17x1_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 const struct snd_soc_dai_ops adau17x1_dai_ops = {
 	.hw_params	= adau17x1_hw_params,
 	.set_sysclk	= adau17x1_set_dai_sysclk,
@@ -783,6 +794,8 @@ const struct snd_soc_dai_ops adau17x1_dai_ops = {
 	.set_pll	= adau17x1_set_dai_pll,
 	.set_tdm_slot	= adau17x1_set_dai_tdm_slot,
 	.startup	= adau17x1_startup,
+	.auto_selectable_formats	= &adau17x1_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 EXPORT_SYMBOL_GPL(adau17x1_dai_ops);
 
@@ -989,9 +1002,7 @@ int adau17x1_resume(struct snd_soc_component *component)
 	if (adau->switch_mode)
 		adau->switch_mode(component->dev);
 
-	regcache_sync(adau->regmap);
-
-	return 0;
+	return regcache_sync(adau->regmap);
 }
 EXPORT_SYMBOL_GPL(adau17x1_resume);
 
diff --git a/sound/soc/codecs/adau1977.c b/sound/soc/codecs/adau1977.c
index b42f35b1a60367..b9d98539342167 100644
--- a/sound/soc/codecs/adau1977.c
+++ b/sound/soc/codecs/adau1977.c
@@ -741,6 +741,17 @@ static int adau1977_set_tristate(struct snd_soc_dai *dai, int tristate)
 		ADAU1977_SAI_OVERTEMP_DRV_HIZ, val);
 }
 
+static const u64 adau1977_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops adau1977_dai_ops = {
 	.startup	= adau1977_startup,
 	.hw_params	= adau1977_hw_params,
@@ -748,6 +759,8 @@ static const struct snd_soc_dai_ops adau1977_dai_ops = {
 	.set_fmt	= adau1977_set_dai_fmt,
 	.set_tdm_slot	= adau1977_set_tdm_slot,
 	.set_tristate	= adau1977_set_tristate,
+	.auto_selectable_formats	= &adau1977_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver adau1977_dai = {
diff --git a/sound/soc/codecs/adau7118.c b/sound/soc/codecs/adau7118.c
index 5ddea63a6612c9..9749299bc7f2a7 100644
--- a/sound/soc/codecs/adau7118.c
+++ b/sound/soc/codecs/adau7118.c
@@ -418,12 +418,24 @@ static int adau7118_component_probe(struct snd_soc_component *component)
 	return ret;
 }
 
+static const u64 adau7118_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops adau7118_ops = {
 	.hw_params = adau7118_hw_params,
 	.set_channel_map = adau7118_set_channel_map,
 	.set_fmt = adau7118_set_fmt,
 	.set_tdm_slot = adau7118_set_tdm_slot,
 	.set_tristate = adau7118_set_tristate,
+	.auto_selectable_formats = &adau7118_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver adau7118_dai = {
diff --git a/sound/soc/codecs/ak4104.c b/sound/soc/codecs/ak4104.c
index 6ea7cb78cd445c..5ebc3afccccf63 100644
--- a/sound/soc/codecs/ak4104.c
+++ b/sound/soc/codecs/ak4104.c
@@ -147,9 +147,16 @@ static int ak4104_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
+static const u64 ak4101_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops ak4101_dai_ops = {
 	.hw_params = ak4104_hw_params,
 	.set_fmt = ak4104_set_dai_fmt,
+	.auto_selectable_formats = &ak4101_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver ak4104_dai = {
diff --git a/sound/soc/codecs/ak4118.c b/sound/soc/codecs/ak4118.c
index 03dadba0b038fb..2b96b82d4e40ab 100644
--- a/sound/soc/codecs/ak4118.c
+++ b/sound/soc/codecs/ak4118.c
@@ -236,9 +236,16 @@ static int ak4118_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
+/* NOTE: Common formats only are auto detected */
+static const u64 ak4118_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops ak4118_dai_ops = {
 	.hw_params = ak4118_hw_params,
 	.set_fmt   = ak4118_set_dai_fmt,
+	.auto_selectable_formats	= &ak4118_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver ak4118_dai = {
diff --git a/sound/soc/codecs/ak4458.c b/sound/soc/codecs/ak4458.c
index f81cd8cebdd898..a74be99e877ddb 100644
--- a/sound/soc/codecs/ak4458.c
+++ b/sound/soc/codecs/ak4458.c
@@ -591,12 +591,21 @@ static int ak4458_startup(struct snd_pcm_substream *substream,
 					  &ak4458_rate_constraints);
 }
 
+static const u64 ak4458_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_PDM;
+
 static const struct snd_soc_dai_ops ak4458_dai_ops = {
 	.startup        = ak4458_startup,
 	.hw_params	= ak4458_hw_params,
 	.set_fmt	= ak4458_set_dai_fmt,
 	.mute_stream	= ak4458_set_dai_mute,
 	.set_tdm_slot	= ak4458_set_tdm_slot,
+	.auto_selectable_formats	= &ak4458_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/ak4535.c b/sound/soc/codecs/ak4535.c
index 37555674cd157e..6cabad4c9f74c4 100644
--- a/sound/soc/codecs/ak4535.c
+++ b/sound/soc/codecs/ak4535.c
@@ -346,11 +346,17 @@ static int ak4535_set_bias_level(struct snd_soc_component *component,
 		SNDRV_PCM_RATE_16000 | SNDRV_PCM_RATE_22050 |\
 		SNDRV_PCM_RATE_44100 | SNDRV_PCM_RATE_48000)
 
+static const u64 ak4535_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops ak4535_dai_ops = {
 	.hw_params	= ak4535_hw_params,
 	.set_fmt	= ak4535_set_dai_fmt,
 	.mute_stream	= ak4535_mute,
 	.set_sysclk	= ak4535_set_dai_sysclk,
+	.auto_selectable_formats	= &ak4535_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/ak4619.c b/sound/soc/codecs/ak4619.c
index d9c9f6b200284a..bae8e8b6a43016 100644
--- a/sound/soc/codecs/ak4619.c
+++ b/sound/soc/codecs/ak4619.c
@@ -152,6 +152,7 @@
 struct ak4619_priv {
 	struct regmap *regmap;
 	struct snd_pcm_hw_constraint_list constraint;
+	struct gpio_desc *pdn_gpio;
 	int deemph_en;
 	unsigned int playback_rate;
 	unsigned int sysclk;
@@ -799,7 +800,26 @@ static const struct snd_soc_dai_ops ak4619_dai_ops = {
 	.num_auto_selectable_formats	= ARRAY_SIZE(ak4619_dai_formats),
 };
 
+static int ak4619_suspend(struct snd_soc_component *component)
+{
+	struct regmap *regmap = dev_get_regmap(component->dev, NULL);
+
+	regcache_cache_only(regmap, true);
+	regcache_mark_dirty(regmap);
+	return 0;
+}
+
+static int ak4619_resume(struct snd_soc_component *component)
+{
+	struct regmap *regmap = dev_get_regmap(component->dev, NULL);
+
+	regcache_cache_only(regmap, false);
+	return regcache_sync(regmap);
+}
+
 static const struct snd_soc_component_driver soc_component_dev_ak4619 = {
+	.suspend		= ak4619_suspend,
+	.resume			= ak4619_resume,
 	.set_bias_level		= ak4619_set_bias_level,
 	.controls		= ak4619_snd_controls,
 	.num_controls		= ARRAY_SIZE(ak4619_snd_controls),
@@ -875,10 +895,20 @@ static int ak4619_i2c_probe(struct i2c_client *i2c)
 
 	i2c_set_clientdata(i2c, ak4619);
 
+	ak4619->pdn_gpio = devm_gpiod_get_optional(dev, "powerdown",
+		GPIOD_OUT_LOW);
+	if (IS_ERR(ak4619->pdn_gpio))
+		return dev_err_probe(dev, PTR_ERR(ak4619->pdn_gpio),
+			"powerdown GPIO request failed\n");
+	if (ak4619->pdn_gpio)
+		msleep(10);
+
 	ak4619->regmap = devm_regmap_init_i2c(i2c, &ak4619_regmap_cfg);
 	if (IS_ERR(ak4619->regmap)) {
 		ret = PTR_ERR(ak4619->regmap);
 		dev_err(dev, "regmap_init() failed: %d\n", ret);
+		if (ak4619->pdn_gpio)
+			gpiod_set_value_cansleep(ak4619->pdn_gpio, 1);
 		return ret;
 	}
 
@@ -887,6 +917,8 @@ static int ak4619_i2c_probe(struct i2c_client *i2c)
 	if (ret < 0) {
 		dev_err(dev, "Failed to register ak4619 component: %d\n",
 			ret);
+		if (ak4619->pdn_gpio)
+			gpiod_set_value_cansleep(ak4619->pdn_gpio, 1);
 		return ret;
 	}
 
diff --git a/sound/soc/codecs/ak4642.c b/sound/soc/codecs/ak4642.c
index 08ec2035b2705e..a0de45660b001a 100644
--- a/sound/soc/codecs/ak4642.c
+++ b/sound/soc/codecs/ak4642.c
@@ -493,12 +493,18 @@ static int ak4642_set_bias_level(struct snd_soc_component *component,
 	return 0;
 }
 
+static const u64 ak4642_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops ak4642_dai_ops = {
 	.startup	= ak4642_dai_startup,
 	.shutdown	= ak4642_dai_shutdown,
 	.set_sysclk	= ak4642_dai_set_sysclk,
 	.set_fmt	= ak4642_dai_set_fmt,
 	.hw_params	= ak4642_dai_hw_params,
+	.auto_selectable_formats	= &ak4642_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver ak4642_dai = {
@@ -533,8 +539,7 @@ static int ak4642_resume(struct snd_soc_component *component)
 	struct regmap *regmap = dev_get_regmap(component->dev, NULL);
 
 	regcache_cache_only(regmap, false);
-	regcache_sync(regmap);
-	return 0;
+	return regcache_sync(regmap);
 }
 static int ak4642_probe(struct snd_soc_component *component)
 {
@@ -607,14 +612,13 @@ static struct clk *ak4642_of_parse_mcko(struct device *dev)
 	struct device_node *np = dev->of_node;
 	struct clk *clk;
 	const char *clk_name = np->name;
-	const char *parent_clk_name = NULL;
+	const char *parent_clk_name;
 	u32 rate;
 
 	if (of_property_read_u32(np, "clock-frequency", &rate))
 		return NULL;
 
-	if (of_property_read_bool(np, "clocks"))
-		parent_clk_name = of_clk_get_parent_name(np, 0);
+	parent_clk_name = of_clk_get_parent_name(np, 0);
 
 	of_property_read_string(np, "clock-output-names", &clk_name);
 
diff --git a/sound/soc/codecs/ak4671.c b/sound/soc/codecs/ak4671.c
index aac54e9d54ee7c..1d804f03a21d8d 100644
--- a/sound/soc/codecs/ak4671.c
+++ b/sound/soc/codecs/ak4671.c
@@ -582,10 +582,17 @@ static int ak4671_set_bias_level(struct snd_soc_component *component,
 
 #define AK4671_FORMATS		SNDRV_PCM_FMTBIT_S16_LE
 
+static const u64 ak4671_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A;
+
 static const struct snd_soc_dai_ops ak4671_dai_ops = {
 	.hw_params	= ak4671_hw_params,
 	.set_sysclk	= ak4671_set_dai_sysclk,
 	.set_fmt	= ak4671_set_dai_fmt,
+	.auto_selectable_formats	= &ak4671_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver ak4671_dai = {
diff --git a/sound/soc/codecs/ak5386.c b/sound/soc/codecs/ak5386.c
index 6525d50b7ab207..933ca90b7e2878 100644
--- a/sound/soc/codecs/ak5386.c
+++ b/sound/soc/codecs/ak5386.c
@@ -128,10 +128,16 @@ static int ak5386_hw_free(struct snd_pcm_substream *substream,
 	return 0;
 }
 
+static const u64 ak5386_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops ak5386_dai_ops = {
 	.set_fmt	= ak5386_set_dai_fmt,
 	.hw_params	= ak5386_hw_params,
 	.hw_free	= ak5386_hw_free,
+	.auto_selectable_formats	= &ak5386_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver ak5386_dai = {
diff --git a/sound/soc/codecs/ak5558.c b/sound/soc/codecs/ak5558.c
index 691db6565c20dc..0c9cdf9a53bb49 100644
--- a/sound/soc/codecs/ak5558.c
+++ b/sound/soc/codecs/ak5558.c
@@ -285,12 +285,19 @@ static int ak5558_startup(struct snd_pcm_substream *substream,
 					  &ak5558_rate_constraints);
 }
 
+static const u64 ak5558_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B;
+
 static const struct snd_soc_dai_ops ak5558_dai_ops = {
 	.startup        = ak5558_startup,
 	.hw_params	= ak5558_hw_params,
 
 	.set_fmt	= ak5558_set_dai_fmt,
 	.set_tdm_slot   = ak5558_set_tdm_slot,
+	.auto_selectable_formats	= &ak5558_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver ak5558_dai = {
diff --git a/sound/soc/codecs/alc5623.c b/sound/soc/codecs/alc5623.c
index f5a666c4936150..96bb1d59277292 100644
--- a/sound/soc/codecs/alc5623.c
+++ b/sound/soc/codecs/alc5623.c
@@ -822,12 +822,23 @@ static int alc5623_set_bias_level(struct snd_soc_component *component,
 			| SNDRV_PCM_FMTBIT_S24_LE \
 			| SNDRV_PCM_FMTBIT_S32_LE)
 
+static const u64 alc5623_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF;
+
 static const struct snd_soc_dai_ops alc5623_dai_ops = {
 		.hw_params = alc5623_pcm_hw_params,
 		.mute_stream = alc5623_mute,
 		.set_fmt = alc5623_set_dai_fmt,
 		.set_sysclk = alc5623_set_dai_sysclk,
 		.set_pll = alc5623_set_dai_pll,
+		.auto_selectable_formats = &alc5623_selectable_formats,
+		.num_auto_selectable_formats = 1,
 		.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/alc5632.c b/sound/soc/codecs/alc5632.c
index 2fd3dd7ef8b340..a5e95f3fc2474e 100644
--- a/sound/soc/codecs/alc5632.c
+++ b/sound/soc/codecs/alc5632.c
@@ -1003,12 +1003,25 @@ static int alc5632_set_bias_level(struct snd_soc_component *component,
 			| SNDRV_PCM_FMTBIT_S24_LE \
 			| SNDRV_PCM_FMTBIT_S32_LE)
 
+
+static const u64 alc5632_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops alc5632_dai_ops = {
 		.hw_params = alc5632_pcm_hw_params,
 		.mute_stream = alc5632_mute,
 		.set_fmt = alc5632_set_dai_fmt,
 		.set_sysclk = alc5632_set_dai_sysclk,
 		.set_pll = alc5632_set_dai_pll,
+		.auto_selectable_formats = &alc5632_selectable_formats,
+		.num_auto_selectable_formats = 1,
 		.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/arizona-jack.c b/sound/soc/codecs/arizona-jack.c
index 819d080b118893..6587d46a186705 100644
--- a/sound/soc/codecs/arizona-jack.c
+++ b/sound/soc/codecs/arizona-jack.c
@@ -699,6 +699,7 @@ static void arizona_start_hpdet_acc_id(struct arizona_priv *info)
 	snd_soc_jack_report(info->jack, SND_JACK_HEADPHONE,
 			    SND_JACK_LINEOUT | SND_JACK_HEADPHONE);
 
+	pm_runtime_put_autosuspend(arizona->dev);
 	info->hpdet_active = false;
 }
 
diff --git a/sound/soc/codecs/arizona.c b/sound/soc/codecs/arizona.c
index 4411165c37345a..7c296c5b24f726 100644
--- a/sound/soc/codecs/arizona.c
+++ b/sound/soc/codecs/arizona.c
@@ -2016,6 +2016,27 @@ static int arizona_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 	return 0;
 }
 
+static const u64 arizona_selectable_formats[] = {
+	/* 1st priority */
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF,
+	/*
+	 * 2nd priority
+	 *
+	 * _DSP_B / _LEFT_J can't be clock consumer (= arizona_set_fmt()).
+	 */
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF,
+};
+
 const struct snd_soc_dai_ops arizona_dai_ops = {
 	.startup = arizona_startup,
 	.set_fmt = arizona_set_fmt,
@@ -2023,6 +2044,8 @@ const struct snd_soc_dai_ops arizona_dai_ops = {
 	.hw_params = arizona_hw_params,
 	.set_sysclk = arizona_dai_set_sysclk,
 	.set_tristate = arizona_set_tristate,
+	.auto_selectable_formats = arizona_selectable_formats,
+	.num_auto_selectable_formats = ARRAY_SIZE(arizona_selectable_formats),
 };
 EXPORT_SYMBOL_GPL(arizona_dai_ops);
 
diff --git a/sound/soc/codecs/cpcap.c b/sound/soc/codecs/cpcap.c
index fc41e1547bdae7..dda68165b8173c 100644
--- a/sound/soc/codecs/cpcap.c
+++ b/sound/soc/codecs/cpcap.c
@@ -1244,11 +1244,24 @@ static int cpcap_hifi_set_mute(struct snd_soc_dai *dai, int mute, int direction)
 	return regmap_update_bits(cpcap->regmap, reg, mask, val);
 }
 
+static const u64 cpcap_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cpcap_dai_hifi_ops = {
 	.hw_params	= cpcap_hifi_hw_params,
 	.set_sysclk	= cpcap_hifi_set_dai_sysclk,
 	.set_fmt	= cpcap_hifi_set_dai_fmt,
 	.mute_stream	= cpcap_hifi_set_mute,
+	.auto_selectable_formats	= &cpcap_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
@@ -1406,6 +1419,8 @@ static const struct snd_soc_dai_ops cpcap_dai_voice_ops = {
 	.set_sysclk	= cpcap_voice_set_dai_sysclk,
 	.set_fmt	= cpcap_voice_set_dai_fmt,
 	.mute_stream	= cpcap_voice_set_mute,
+	.auto_selectable_formats	= &cpcap_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
@@ -1607,21 +1622,12 @@ static int cpcap_soc_probe(struct snd_soc_component *component)
 {
 	struct platform_device *pdev = to_platform_device(component->dev);
 	struct snd_soc_card *card = component->card;
-	struct cpcap_audio *cpcap;
+	struct cpcap_audio *cpcap = dev_get_drvdata(component->dev);
 	int err;
 
-	cpcap = devm_kzalloc(component->dev, sizeof(*cpcap), GFP_KERNEL);
-	if (!cpcap)
-		return -ENOMEM;
-
 	snd_soc_component_set_drvdata(component, cpcap);
 	cpcap->component = component;
 
-	cpcap->vaudio = devm_regulator_get(component->dev, "VAUDIO");
-	if (IS_ERR(cpcap->vaudio))
-		return dev_err_probe(component->dev, PTR_ERR(cpcap->vaudio),
-				     "Cannot get VAUDIO regulator\n");
-
 	err = snd_soc_card_jack_new(card, "Headphones",
 				    SND_JACK_HEADSET | SND_JACK_BTN_0,
 				    &cpcap->jack);
@@ -1645,13 +1651,13 @@ static int cpcap_soc_probe(struct snd_soc_component *component)
 	if (cpcap->hsirq < 0)
 		return cpcap->hsirq;
 
-	err = devm_request_threaded_irq(component->dev, cpcap->hsirq, NULL,
-					cpcap_hs_irq_thread,
-					IRQF_TRIGGER_RISING |
-					IRQF_TRIGGER_FALLING |
-					IRQF_ONESHOT,
-					"cpcap-codec-hs",
-					component);
+	err = request_threaded_irq(cpcap->hsirq, NULL,
+				   cpcap_hs_irq_thread,
+				   IRQF_TRIGGER_RISING |
+				   IRQF_TRIGGER_FALLING |
+				   IRQF_ONESHOT,
+				   "cpcap-codec-hs",
+				   component);
 	if (err) {
 		dev_warn(component->dev, "no HS irq%i: %i\n",
 			 cpcap->hsirq, err);
@@ -1659,25 +1665,27 @@ static int cpcap_soc_probe(struct snd_soc_component *component)
 	}
 
 	cpcap->mb2irq = platform_get_irq_byname(pdev, "mb2");
-	if (cpcap->mb2irq < 0)
-		return cpcap->mb2irq;
-
-	err = devm_request_threaded_irq(component->dev, cpcap->mb2irq, NULL,
-					cpcap_mb2_irq_thread,
-					IRQF_TRIGGER_RISING |
-					IRQF_TRIGGER_FALLING |
-					IRQF_ONESHOT,
-					"cpcap-codec-mb2",
-					component);
+	if (cpcap->mb2irq < 0) {
+		err = cpcap->mb2irq;
+		goto err_free_hsirq;
+	}
+
+	err = request_threaded_irq(cpcap->mb2irq, NULL,
+				   cpcap_mb2_irq_thread,
+				   IRQF_TRIGGER_RISING |
+				   IRQF_TRIGGER_FALLING |
+				   IRQF_ONESHOT,
+				   "cpcap-codec-mb2",
+				   component);
 	if (err) {
 		dev_warn(component->dev, "no MB2 irq%i: %i\n",
 			 cpcap->mb2irq, err);
-		return err;
+		goto err_free_hsirq;
 	}
 
 	err = cpcap_audio_reset(component, false);
 	if (err)
-		return err;
+		goto err_free_mb2irq;
 
 	cpcap_hs_irq_thread(cpcap->hsirq, component);
 
@@ -1685,6 +1693,13 @@ static int cpcap_soc_probe(struct snd_soc_component *component)
 	enable_irq_wake(cpcap->mb2irq);
 
 	return 0;
+
+err_free_mb2irq:
+	free_irq(cpcap->mb2irq, component);
+err_free_hsirq:
+	free_irq(cpcap->hsirq, component);
+
+	return err;
 }
 
 static void cpcap_soc_remove(struct snd_soc_component *component)
@@ -1693,6 +1708,9 @@ static void cpcap_soc_remove(struct snd_soc_component *component)
 
 	disable_irq_wake(cpcap->hsirq);
 	disable_irq_wake(cpcap->mb2irq);
+
+	free_irq(cpcap->mb2irq, component);
+	free_irq(cpcap->hsirq, component);
 }
 
 static int cpcap_set_bias_level(struct snd_soc_component *component,
@@ -1739,11 +1757,24 @@ static int cpcap_codec_probe(struct platform_device *pdev)
 {
 	struct device_node *codec_node =
 		of_get_child_by_name(pdev->dev.parent->of_node, "audio-codec");
+	struct cpcap_audio *cpcap;
+
 	if (!codec_node)
 		return -ENODEV;
 
 	pdev->dev.of_node = codec_node;
 
+	cpcap = devm_kzalloc(&pdev->dev, sizeof(*cpcap), GFP_KERNEL);
+	if (!cpcap)
+		return -ENOMEM;
+
+	cpcap->vaudio = devm_regulator_get(&pdev->dev, "VAUDIO");
+	if (IS_ERR(cpcap->vaudio))
+		return dev_err_probe(&pdev->dev, PTR_ERR(cpcap->vaudio),
+				     "Cannot get VAUDIO regulator\n");
+
+	platform_set_drvdata(pdev, cpcap);
+
 	return devm_snd_soc_register_component(&pdev->dev, &soc_codec_dev_cpcap,
 				      cpcap_dai, ARRAY_SIZE(cpcap_dai));
 }
diff --git a/sound/soc/codecs/cs35l33.c b/sound/soc/codecs/cs35l33.c
index e217da124549a8..b8c19f17570197 100644
--- a/sound/soc/codecs/cs35l33.c
+++ b/sound/soc/codecs/cs35l33.c
@@ -578,7 +578,7 @@ static int cs35l33_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 			| CS35L33_X_LOC);
 	}
 
-	/* disconnect {vp,vbst}_mon routes: eanble later if set in tx_mask*/
+	/* disconnect {vp,vbst}_mon routes: enable later if set in tx_mask*/
 	snd_soc_dapm_del_routes(dapm, cs35l33_vp_vbst_mon_route,
 		ARRAY_SIZE(cs35l33_vp_vbst_mon_route));
 
@@ -663,12 +663,18 @@ static int cs35l33_component_set_sysclk(struct snd_soc_component *component,
 	return 0;
 }
 
+static const u64 cs35l33_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A;
+
 static const struct snd_soc_dai_ops cs35l33_ops = {
 	.startup = cs35l33_pcm_startup,
 	.set_tristate = cs35l33_set_tristate,
 	.set_fmt = cs35l33_set_dai_fmt,
 	.hw_params = cs35l33_pcm_hw_params,
 	.set_tdm_slot = cs35l33_set_tdm_slot,
+	.auto_selectable_formats = &cs35l33_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver cs35l33_dai = {
diff --git a/sound/soc/codecs/cs35l34.c b/sound/soc/codecs/cs35l34.c
index 3f265bd760f7c9..a72be629a31e55 100644
--- a/sound/soc/codecs/cs35l34.c
+++ b/sound/soc/codecs/cs35l34.c
@@ -297,7 +297,7 @@ static int cs35l34_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 				CS35L34_X_STATE | CS35L34_X_LOC,
 				CS35L34_X_STATE | CS35L34_X_LOC);
 
-	/* disconnect {vp,vbst}_mon routes: eanble later if set in tx_mask*/
+	/* disconnect {vp,vbst}_mon routes: enable later if set in tx_mask*/
 	while (slot >= 0) {
 		/* configure VMON_TX_LOC */
 		if (slot_num == 0)
diff --git a/sound/soc/codecs/cs35l35.c b/sound/soc/codecs/cs35l35.c
index b2439ec3c19fba..302d838f4c440f 100644
--- a/sound/soc/codecs/cs35l35.c
+++ b/sound/soc/codecs/cs35l35.c
@@ -657,17 +657,24 @@ static int cs35l35_dai_set_sysclk(struct snd_soc_dai *dai,
 	return 0;
 }
 
+static const u64 cs35l35_selectable_formats_i2s = SND_SOC_POSSIBLE_DAIFMT_I2S;
+static const u64 cs35l35_selectable_formats_pdm = SND_SOC_POSSIBLE_DAIFMT_PDM;
+
 static const struct snd_soc_dai_ops cs35l35_ops = {
 	.startup = cs35l35_pcm_startup,
 	.set_fmt = cs35l35_set_dai_fmt,
 	.hw_params = cs35l35_hw_params,
 	.set_sysclk = cs35l35_dai_set_sysclk,
+	.auto_selectable_formats = &cs35l35_selectable_formats_i2s,
+	.num_auto_selectable_formats = 1,
 };
 
 static const struct snd_soc_dai_ops cs35l35_pdm_ops = {
 	.startup = cs35l35_pdm_startup,
 	.set_fmt = cs35l35_set_dai_fmt,
 	.hw_params = cs35l35_hw_params,
+	.auto_selectable_formats	= &cs35l35_selectable_formats_pdm,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver cs35l35_dai[] = {
diff --git a/sound/soc/codecs/cs35l36.c b/sound/soc/codecs/cs35l36.c
index 89645327945f18..5fc8ec292e3c44 100644
--- a/sound/soc/codecs/cs35l36.c
+++ b/sound/soc/codecs/cs35l36.c
@@ -947,10 +947,66 @@ static const struct cs35l36_pll_config *cs35l36_get_clk_config(
 	return NULL;
 }
 
+static void cs35l36_mask_to_slots(struct cs35l36_private *cs35l36,
+				  unsigned long mask, unsigned int base_reg,
+				  unsigned int nchan)
+{
+	unsigned int chan = 0, shift;
+	int slot;
+
+	/* Two 6-bit slot fields per register, at bits 0 and 16 */
+	for_each_set_bit(slot, &mask, BITS_PER_TYPE(mask)) {
+		if (chan == nchan) {
+			dev_warn(cs35l36->dev,
+				 "Too many slots in TDM mask: %lx\n", mask);
+			return;
+		}
+
+		shift = (chan % 2) * CS35L36_ASP_TX2_SLOT_SHIFT;
+		regmap_update_bits(cs35l36->regmap, base_reg + (chan / 2) * 4,
+				   CS35L36_ASP_RX1_SLOT_MASK << shift,
+				   slot << shift);
+		chan++;
+	}
+}
+
+static int cs35l36_set_tdm_slot(struct snd_soc_dai *dai,
+				unsigned int tx_mask, unsigned int rx_mask,
+				int slots, int slot_width)
+{
+	struct cs35l36_private *cs35l36 =
+			snd_soc_component_get_drvdata(dai->component);
+
+	/* Note: rx/tx is from point of view of the CPU end */
+	if (!slots || !rx_mask)
+		rx_mask = BIT(0);		/* ASPRX1 in slot 0 */
+
+	if (!slots || !tx_mask)
+		tx_mask = GENMASK(7, 0);	/* ASPTX1..8 in slots 0..7 */
+
+	cs35l36_mask_to_slots(cs35l36, rx_mask, CS35L36_ASP_RX1_SLOT, 1);
+	cs35l36_mask_to_slots(cs35l36, tx_mask, CS35L36_ASP_TX1_TX2_SLOT, 8);
+
+	return 0;
+}
+
+static const u64 cs35l36_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_GATED	|
+	SND_SOC_POSSIBLE_DAIFMT_CONT	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs35l36_ops = {
 	.set_fmt = cs35l36_set_dai_fmt,
 	.hw_params = cs35l36_pcm_hw_params,
 	.set_sysclk = cs35l36_dai_set_sysclk,
+	.auto_selectable_formats = &cs35l36_selectable_formats,
+	.num_auto_selectable_formats = 1,
+	.set_tdm_slot = cs35l36_set_tdm_slot,
 };
 
 #define CS35L36_RATES (		    \
diff --git a/sound/soc/codecs/cs35l41-lib.c b/sound/soc/codecs/cs35l41-lib.c
index b0fa80705be7d2..c79b14f752f0c6 100644
--- a/sound/soc/codecs/cs35l41-lib.c
+++ b/sound/soc/codecs/cs35l41-lib.c
@@ -8,6 +8,7 @@
 // Author: Lucas Tanure <lucas.tanure@cirrus.com>
 
 #include <linux/dev_printk.h>
+#include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
diff --git a/sound/soc/codecs/cs35l41.c b/sound/soc/codecs/cs35l41.c
index b2a076706c796e..d286025183f466 100644
--- a/sound/soc/codecs/cs35l41.c
+++ b/sound/soc/codecs/cs35l41.c
@@ -953,11 +953,21 @@ static void cs35l41_component_remove(struct snd_soc_component *component)
 	wm_adsp2_component_remove(&cs35l41->dsp, component);
 }
 
+static const u64 cs35l41_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs35l41_ops = {
 	.set_fmt = cs35l41_set_dai_fmt,
 	.hw_params = cs35l41_pcm_hw_params,
 	.set_sysclk = cs35l41_dai_set_sysclk,
 	.set_channel_map = cs35l41_set_channel_map,
+	.auto_selectable_formats = &cs35l41_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 #define CS35L41_RATES (		    \
diff --git a/sound/soc/codecs/cs35l45-tables.c b/sound/soc/codecs/cs35l45-tables.c
index 764dbaa3504227..57982b8ae645b2 100644
--- a/sound/soc/codecs/cs35l45-tables.c
+++ b/sound/soc/codecs/cs35l45-tables.c
@@ -51,6 +51,8 @@ static const struct reg_default cs35l45_defaults[] = {
 	{ CS35L45_WKI2C_CTL,			0x00000030 },
 	{ CS35L45_REFCLK_INPUT,			0x00000510 },
 	{ CS35L45_GLOBAL_SAMPLE_RATE,		0x00000003 },
+	{ CS35L45_SYNC_TX_RX_ENABLES,		0x00000200 },
+	{ CS35L45_SYNC_SW_TX_ID,		0x00000000 },
 	{ CS35L45_ASP_ENABLES1,			0x00000000 },
 	{ CS35L45_ASP_CONTROL1,			0x00000028 },
 	{ CS35L45_ASP_CONTROL2,			0x18180200 },
@@ -134,6 +136,8 @@ static bool cs35l45_readable_reg(struct device *dev, unsigned int reg)
 	case CS35L45_PWRMGT_STS:
 	case CS35L45_REFCLK_INPUT:
 	case CS35L45_GLOBAL_SAMPLE_RATE:
+	case CS35L45_SYNC_TX_RX_ENABLES:
+	case CS35L45_SYNC_SW_TX_ID:
 	case CS35L45_ASP_ENABLES1:
 	case CS35L45_ASP_CONTROL1:
 	case CS35L45_ASP_CONTROL2:
diff --git a/sound/soc/codecs/cs35l45.c b/sound/soc/codecs/cs35l45.c
index a032bb23b4ac7d..9eef7c8b1837e7 100644
--- a/sound/soc/codecs/cs35l45.c
+++ b/sound/soc/codecs/cs35l45.c
@@ -6,6 +6,7 @@
 //
 // Author: James Schulman <james.schulman@cirrus.com>
 
+#include <linux/bitfield.h>
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
@@ -195,6 +196,44 @@ static int cs35l45_activate_ctl(struct snd_soc_component *component,
 	return 0;
 }
 
+static int cs35l45_sync_en_get(struct snd_kcontrol *kcontrol,
+				      struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct cs35l45_private *cs35l45 =
+			snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = cs35l45->sync_en;
+
+	return 0;
+}
+
+static int cs35l45_sync_en_put(struct snd_kcontrol *kcontrol,
+				      struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct cs35l45_private *cs35l45 =
+			snd_soc_component_get_drvdata(component);
+	struct snd_soc_dapm_context *dapm =
+			snd_soc_component_to_dapm(component);
+
+	snd_soc_dapm_mutex_lock(dapm);
+
+	if ((bool)ucontrol->value.integer.value[0] == cs35l45->sync_en) {
+		snd_soc_dapm_mutex_unlock(dapm);
+		return 0;
+	}
+
+	regmap_assign_bits(cs35l45->regmap, CS35L45_BLOCK_ENABLES2, CS35L45_SYNC_EN_MASK,
+			   (bool)ucontrol->value.integer.value[0]);
+
+	cs35l45->sync_en = (bool)ucontrol->value.integer.value[0];
+
+	snd_soc_dapm_mutex_unlock(dapm);
+
+	return 1;
+}
+
 static int cs35l45_amplifier_mode_get(struct snd_kcontrol *kcontrol,
 				      struct snd_ctl_elem_value *ucontrol)
 {
@@ -591,6 +630,18 @@ static SOC_ENUM_SINGLE_DECL(amplifier_mode_enum, SND_SOC_NOPM, 0,
 static DECLARE_TLV_DB_SCALE(amp_gain_tlv, 1000, 300, 0);
 static const DECLARE_TLV_DB_SCALE(cs35l45_dig_pcm_vol_tlv, -10225, 25, true);
 
+static const struct snd_kcontrol_new cs35l45_sync_controls[] = {
+	SOC_SINGLE_BOOL_EXT("SYNC Switch", 0, cs35l45_sync_en_get, cs35l45_sync_en_put),
+	SOC_SINGLE("SYNC LSW RX Switch", CS35L45_SYNC_TX_RX_ENABLES,
+			CS35L45_SYNC_LSW_RX_EN_SHIFT, 1, 0),
+	SOC_SINGLE("SYNC LSW TX Switch", CS35L45_SYNC_TX_RX_ENABLES,
+			CS35L45_SYNC_LSW_TX_EN_SHIFT, 1, 0),
+	SOC_SINGLE("SYNC SW RX Switch", CS35L45_SYNC_TX_RX_ENABLES,
+			CS35L45_SYNC_SW_RX_EN_SHIFT, 1, 0),
+	SOC_SINGLE("SYNC SW TX Switch", CS35L45_SYNC_TX_RX_ENABLES,
+			CS35L45_SYNC_SW_TX_EN_SHIFT, 1, 0),
+};
+
 static const struct snd_kcontrol_new cs35l45_controls[] = {
 	SOC_ENUM_EXT("Amplifier Mode", amplifier_mode_enum,
 		     cs35l45_amplifier_mode_get, cs35l45_amplifier_mode_put),
@@ -846,12 +897,22 @@ static int cs35l45_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
 	return 0;
 }
 
+static const u64 cs35l45_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs35l45_asp_dai_ops = {
 	.set_fmt = cs35l45_asp_set_fmt,
 	.hw_params = cs35l45_asp_hw_params,
 	.set_tdm_slot = cs35l45_asp_set_tdm_slot,
 	.set_sysclk = cs35l45_asp_set_sysclk,
 	.mute_stream = cs35l45_mute_stream,
+	.auto_selectable_formats	= &cs35l45_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver cs35l45_dai[] = {
@@ -880,6 +941,19 @@ static struct snd_soc_dai_driver cs35l45_dai[] = {
 static int cs35l45_component_probe(struct snd_soc_component *component)
 {
 	struct cs35l45_private *cs35l45 = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	ret = snd_soc_add_component_controls(component, cs35l45_controls,
+					     ARRAY_SIZE(cs35l45_controls));
+	if (ret < 0)
+		return ret;
+
+	if (cs35l45->sync_pin_set) {
+		ret = snd_soc_add_component_controls(component, cs35l45_sync_controls,
+						     ARRAY_SIZE(cs35l45_sync_controls));
+		if (ret < 0)
+			return ret;
+	}
 
 	return wm_adsp2_component_probe(&cs35l45->dsp, component);
 }
@@ -901,9 +975,6 @@ static const struct snd_soc_component_driver cs35l45_component = {
 	.dapm_routes = cs35l45_dapm_routes,
 	.num_dapm_routes = ARRAY_SIZE(cs35l45_dapm_routes),
 
-	.controls = cs35l45_controls,
-	.num_controls = ARRAY_SIZE(cs35l45_controls),
-
 	.name = "cs35l45",
 
 	.endianness = 1,
@@ -1064,6 +1135,19 @@ static int cs35l45_sys_resume(struct device *dev)
 	return 0;
 }
 
+static void cs35l45_apply_sync_property_config(struct cs35l45_private *cs35l45)
+{
+	unsigned int val;
+
+	if (device_property_read_u32(cs35l45->dev, "cirrus,sync-lsw-txid", &val) == 0)
+		regmap_update_bits(cs35l45->regmap, CS35L45_SYNC_SW_TX_ID,
+				   CS35L45_SYNC_LSW_TXID_MASK, val << CS35L45_SYNC_LSW_TXID_SHIFT);
+
+	if (device_property_read_u32(cs35l45->dev, "cirrus,sync-sw-txid", &val) == 0)
+		regmap_update_bits(cs35l45->regmap, CS35L45_SYNC_SW_TX_ID,
+				   CS35L45_SYNC_SW_TXID_MASK, val);
+}
+
 static int cs35l45_apply_property_config(struct cs35l45_private *cs35l45)
 {
 	struct device_node *node = cs35l45->dev->of_node;
@@ -1110,10 +1194,15 @@ static int cs35l45_apply_property_config(struct cs35l45_private *cs35l45)
 					   val << CS35L45_GPIO_POL_SHIFT);
 
 		ret = of_property_read_u32(child, "gpio-ctrl", &val);
-		if (!ret)
+		if (!ret) {
+			if ((i == 0) && (val == CS35L45_GP1_CTRL_MDSYNC)) {
+				cs35l45->sync_pin_set = true;
+				cs35l45_apply_sync_property_config(cs35l45);
+			}
 			regmap_update_bits(cs35l45->regmap, pad_regs[i],
 					   CS35L45_GPIO_CTRL_MASK,
 					   val << CS35L45_GPIO_CTRL_SHIFT);
+		}
 
 		ret = of_property_read_u32(child, "gpio-invert", &val);
 		if (!ret) {
diff --git a/sound/soc/codecs/cs35l45.h b/sound/soc/codecs/cs35l45.h
index 7a790d2acac7ff..f8a4713ae92f78 100644
--- a/sound/soc/codecs/cs35l45.h
+++ b/sound/soc/codecs/cs35l45.h
@@ -35,6 +35,8 @@
 #define CS35L45_PWRMGT_STS			0x0000290C
 #define CS35L45_REFCLK_INPUT			0x00002C04
 #define CS35L45_GLOBAL_SAMPLE_RATE		0x00002C0C
+#define CS35L45_SYNC_TX_RX_ENABLES		0x00003400
+#define CS35L45_SYNC_SW_TX_ID			0x00003408
 #define CS35L45_BOOST_CCM_CFG			0x00003808
 #define CS35L45_BOOST_DCM_CFG			0x0000380C
 #define CS35L45_BOOST_OV_CFG			0x0000382C
@@ -183,6 +185,8 @@
 #define CS35L45_ASP_EN_SHIFT			27
 #define CS35L45_AMP_DRE_EN_SHIFT		20
 #define CS35L45_AMP_DRE_EN_MASK		BIT(20)
+#define CS35L45_SYNC_EN_SHIFT			8
+#define CS35L45_SYNC_EN_MASK			BIT(8)
 #define CS35L45_MEM_RDY_SHIFT			1
 #define CS35L45_MEM_RDY_MASK			BIT(1)
 
@@ -220,6 +224,22 @@
 #define CS35L45_44P100_KHZ			0x0B
 #define CS35L45_88P200_KHZ			0x0C
 
+/* SYNC_TX_RX_ENABLES */
+#define CS35L45_SYNC_LSW_RX_EN_SHIFT		19
+#define CS35L45_SYNC_LSW_RX_EN_MASK		BIT(19)
+#define CS35L45_SYNC_LSW_TX_EN_SHIFT		18
+#define CS35L45_SYNC_LSW_TX_EN_MASK		BIT(18)
+#define CS35L45_SYNC_SW_RX_EN_SHIFT		17
+#define CS35L45_SYNC_SW_RX_EN_MASK		BIT(17)
+#define CS35L45_SYNC_SW_TX_EN_SHIFT		16
+#define CS35L45_SYNC_SW_TX_EN_MASK		BIT(16)
+
+/* SYNC_SW_TX_ID */
+#define CS35L45_SYNC_LSW_TXID_SHIFT		8
+#define CS35L45_SYNC_LSW_TXID_MASK		GENMASK(10, 8)
+#define CS35L45_SYNC_SW_TXID_SHIFT		0
+#define CS35L45_SYNC_SW_TXID_MASK		GENMASK(2, 0)
+
 /* ASP_ENABLES_1 */
 #define CS35L45_ASP_RX2_EN_SHIFT		17
 #define CS35L45_ASP_RX1_EN_SHIFT		16
@@ -320,6 +340,7 @@
 #define CS35L45_GPIO_CTRL_MASK			GENMASK(22, 20)
 #define CS35L45_GPIO_INVERT_SHIFT		19
 #define CS35L45_GPIO_INVERT_MASK		BIT(19)
+#define CS35L45_GP1_CTRL_MDSYNC		0x2
 
 /* CS35L45_IRQ1_EINT_1 */
 #define CS35L45_BST_UVP_ERR_SHIFT		7
@@ -493,6 +514,8 @@ struct cs35l45_private {
 	struct regulator *vdd_a;
 	bool initialized;
 	bool sysclk_set;
+	bool sync_en;
+	bool sync_pin_set;
 	u8 slot_width;
 	u8 slot_count;
 	int amplifier_mode;
diff --git a/sound/soc/codecs/cs35l56-shared-test.c b/sound/soc/codecs/cs35l56-shared-test.c
index 4f52c8a192e551..34bb3d44b51f10 100644
--- a/sound/soc/codecs/cs35l56-shared-test.c
+++ b/sound/soc/codecs/cs35l56-shared-test.c
@@ -29,6 +29,7 @@ struct cs35l56_shared_test_priv {
 	struct faux_device *gpio_dev;
 	struct cs35l56_shared_test_mock_gpio *gpio_priv;
 	struct regmap *registers;
+	const struct regmap_config *regmap_config;
 	unsigned int reg_offset;
 	struct cs35l56_base *cs35l56_base;
 	u8 applied_pad_pull_state[CS35L56_MAX_GPIO];
@@ -614,6 +615,55 @@ static void cs35l56_shared_test_get_speaker_id_from_host_gpio(struct kunit *test
 	KUNIT_EXPECT_EQ(test, cs35l56_get_speaker_id(priv->cs35l56_base), param->spkid);
 }
 
+static void cs35l56_shared_test_sorted_regmap_defaults(struct kunit *test)
+{
+	struct cs35l56_shared_test_priv *priv = test->priv;
+	const struct reg_default *defaults;
+	int i;
+
+	KUNIT_ASSERT_NOT_NULL(test, priv->regmap_config);
+	defaults = priv->regmap_config->reg_defaults;
+	KUNIT_ASSERT_NOT_NULL(test, defaults);
+	KUNIT_ASSERT_NE(test, priv->regmap_config->num_reg_defaults, 0);
+
+	for (i = 1; i < priv->regmap_config->num_reg_defaults; i++)
+		KUNIT_EXPECT_LT(test, defaults[i - 1].reg, defaults[i].reg);
+}
+
+static void cs35l56_shared_test_regmap_defaults_not_volatile(struct kunit *test)
+{
+	struct cs35l56_shared_test_priv *priv = test->priv;
+	struct cs35l56_base *cs35l56_base = priv->cs35l56_base;
+	const struct regmap_config *config = priv->regmap_config;
+	int i;
+
+	KUNIT_ASSERT_NOT_NULL(test, config);
+	KUNIT_ASSERT_NOT_NULL(test, config->volatile_reg);
+
+	for (i = 0; i < config->num_reg_defaults; i++) {
+		KUNIT_EXPECT_FALSE(test,
+				   config->volatile_reg(cs35l56_base->dev,
+							config->reg_defaults[i].reg));
+	}
+}
+
+static void cs35l56_shared_test_regmap_defaults_readable(struct kunit *test)
+{
+	struct cs35l56_shared_test_priv *priv = test->priv;
+	struct cs35l56_base *cs35l56_base = priv->cs35l56_base;
+	const struct regmap_config *config = priv->regmap_config;
+	int i;
+
+	KUNIT_ASSERT_NOT_NULL(test, config);
+	KUNIT_ASSERT_NOT_NULL(test, config->readable_reg);
+
+	for (i = 0; i < config->num_reg_defaults; i++) {
+		KUNIT_EXPECT_TRUE(test,
+				  config->readable_reg(cs35l56_base->dev,
+						       config->reg_defaults[i].reg));
+	}
+}
+
 static int cs35l56_shared_test_case_regmap_init(struct kunit *test,
 						const struct regmap_config *regmap_config)
 {
@@ -645,6 +695,8 @@ static int cs35l56_shared_test_case_regmap_init(struct kunit *test,
 			kunit_add_action_or_reset(test, regmap_exit_wrapper,
 						  cs35l56_base->regmap));
 
+	priv->regmap_config = regmap_config;
+
 	return 0;
 }
 
@@ -862,6 +914,10 @@ static struct kunit_case cs35l56_shared_test_cases[] = {
 			      cs35l56_shared_test_host_gpio_spkid_gen_params,
 			      { KUNIT_SPEED_SLOW }),
 
+	KUNIT_CASE(cs35l56_shared_test_sorted_regmap_defaults),
+	KUNIT_CASE(cs35l56_shared_test_regmap_defaults_not_volatile),
+	KUNIT_CASE(cs35l56_shared_test_regmap_defaults_readable),
+
 	{ }
 };
 
diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c
index 35d2106266279f..74129fcc79dce3 100644
--- a/sound/soc/codecs/cs35l56.c
+++ b/sound/soc/codecs/cs35l56.c
@@ -613,12 +613,22 @@ static int cs35l56_asp_dai_set_sysclk(struct snd_soc_dai *dai,
 	return 0;
 }
 
+static const u64 cs35l56_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs35l56_ops = {
 	.probe = cs35l56_asp_dai_probe,
 	.set_fmt = cs35l56_asp_dai_set_fmt,
 	.set_tdm_slot = cs35l56_asp_dai_set_tdm_slot,
 	.hw_params = cs35l56_asp_dai_hw_params,
 	.set_sysclk = cs35l56_asp_dai_set_sysclk,
+	.auto_selectable_formats = &cs35l56_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static void cs35l56_sdw_dai_shutdown(struct snd_pcm_substream *substream,
diff --git a/sound/soc/codecs/cs40l50-codec.c b/sound/soc/codecs/cs40l50-codec.c
index 67636716306761..698aa59194f4bf 100644
--- a/sound/soc/codecs/cs40l50-codec.c
+++ b/sound/soc/codecs/cs40l50-codec.c
@@ -233,10 +233,19 @@ static int cs40l50_set_dai_bclk_ratio(struct snd_soc_dai *dai, unsigned int rati
 	return 0;
 }
 
+static const u64 cs40l50_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs40l50_dai_ops = {
 	.set_fmt = cs40l50_set_dai_fmt,
 	.set_bclk_ratio = cs40l50_set_dai_bclk_ratio,
 	.hw_params = cs40l50_hw_params,
+	.auto_selectable_formats = &cs40l50_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver cs40l50_dai[] = {
diff --git a/sound/soc/codecs/cs4234.c b/sound/soc/codecs/cs4234.c
index a889fbd519a861..a6ead1ee370812 100644
--- a/sound/soc/codecs/cs4234.c
+++ b/sound/soc/codecs/cs4234.c
@@ -560,11 +560,20 @@ static int cs4234_dai_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask
 	return 0;
 }
 
+static const u64 cs4234_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF;
+
 static const struct snd_soc_dai_ops cs4234_dai_ops = {
 	.set_fmt	= cs4234_dai_set_fmt,
 	.hw_params	= cs4234_dai_hw_params,
 	.startup	= cs4234_dai_startup,
 	.set_tdm_slot	= cs4234_dai_set_tdm_slot,
+	.auto_selectable_formats	= &cs4234_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver cs4234_dai[] = {
diff --git a/sound/soc/codecs/cs4265.c b/sound/soc/codecs/cs4265.c
index 0b7d6826f4e913..97e32e6ccb8434 100644
--- a/sound/soc/codecs/cs4265.c
+++ b/sound/soc/codecs/cs4265.c
@@ -495,11 +495,23 @@ static int cs4265_set_bias_level(struct snd_soc_component *component,
 			SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_U24_LE | \
 			SNDRV_PCM_FMTBIT_S32_LE | SNDRV_PCM_FMTBIT_U32_LE)
 
+/*
+ * NOTE
+ *
+ * auto_select doesn't include _RIGHT_J, because capture can't use it
+ * (= cs4265_pcm_hw_params()). It can be selected via Card directly if needed.
+ */
+static const u64 cs4265_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops cs4265_ops = {
 	.hw_params	= cs4265_pcm_hw_params,
 	.mute_stream	= cs4265_mute,
 	.set_fmt	= cs4265_set_fmt,
 	.set_sysclk	= cs4265_set_sysclk,
+	.auto_selectable_formats	= &cs4265_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/cs4270.c b/sound/soc/codecs/cs4270.c
index 47b2f903a32c4b..aa3bbb95194514 100644
--- a/sound/soc/codecs/cs4270.c
+++ b/sound/soc/codecs/cs4270.c
@@ -456,11 +456,17 @@ static const struct snd_kcontrol_new cs4270_snd_controls[] = {
 		snd_soc_get_volsw, cs4270_soc_put_mute),
 };
 
+static const u64 cs4270_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops cs4270_dai_ops = {
 	.hw_params	= cs4270_hw_params,
 	.set_sysclk	= cs4270_set_dai_sysclk,
 	.set_fmt	= cs4270_set_dai_fmt,
 	.mute_stream	= cs4270_dai_mute,
+	.auto_selectable_formats	= &cs4270_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/cs4271.c b/sound/soc/codecs/cs4271.c
index d8cdd37e9112b2..398d8c24a9ce2b 100644
--- a/sound/soc/codecs/cs4271.c
+++ b/sound/soc/codecs/cs4271.c
@@ -455,11 +455,17 @@ static const struct snd_kcontrol_new cs4271_snd_controls[] = {
 		7, 1, 1),
 };
 
+static const u64 cs4271_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops cs4271_dai_ops = {
 	.hw_params	= cs4271_hw_params,
 	.set_sysclk	= cs4271_set_dai_sysclk,
 	.set_fmt	= cs4271_set_dai_fmt,
 	.mute_stream	= cs4271_mute_stream,
+	.auto_selectable_formats	= &cs4271_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static struct snd_soc_dai_driver cs4271_dai = {
diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c
index fadf688606014b..b5d69d5516a664 100644
--- a/sound/soc/codecs/cs42l42.c
+++ b/sound/soc/codecs/cs42l42.c
@@ -1122,6 +1122,13 @@ EXPORT_SYMBOL_NS_GPL(cs42l42_mute_stream, "SND_SOC_CS42L42_CORE");
 			 SNDRV_PCM_FMTBIT_S24_LE |\
 			 SNDRV_PCM_FMTBIT_S32_LE)
 
+static const u64 cs42l42_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs42l42_ops = {
 	.startup	= cs42l42_dai_startup,
 	.hw_params	= cs42l42_pcm_hw_params,
@@ -1129,6 +1136,8 @@ static const struct snd_soc_dai_ops cs42l42_ops = {
 	.set_sysclk	= cs42l42_set_sysclk,
 	.set_bclk_ratio	= cs42l42_set_bclk_ratio,
 	.mute_stream	= cs42l42_mute_stream,
+	.auto_selectable_formats	= &cs42l42_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 struct snd_soc_dai_driver cs42l42_dai = {
@@ -1673,7 +1682,7 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data)
 	if (cs42l42->suspended || !cs42l42->init_done)
 		return IRQ_NONE;
 
-	/* Read sticky registers to clear interurpt */
+	/* Read sticky registers to clear interrupt */
 	for (i = 0; i < ARRAY_SIZE(stickies); i++) {
 		regmap_read(cs42l42->regmap, irq_params_table[i].status_addr,
 				&(stickies[i]));
diff --git a/sound/soc/codecs/cs42l43.c b/sound/soc/codecs/cs42l43.c
index 1b817e1a6ea5e2..b6238f89c789c8 100644
--- a/sound/soc/codecs/cs42l43.c
+++ b/sound/soc/codecs/cs42l43.c
@@ -671,6 +671,16 @@ static int cs42l43_dai_remove(struct snd_soc_dai *dai)
 	return 0;
 }
 
+static const u64 cs42l43_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs42l43_asp_ops = {
 	.probe		= cs42l43_dai_probe,
 	.remove		= cs42l43_dai_remove,
@@ -678,6 +688,8 @@ static const struct snd_soc_dai_ops cs42l43_asp_ops = {
 	.hw_params	= cs42l43_asp_hw_params,
 	.set_fmt	= cs42l43_asp_set_fmt,
 	.set_tdm_slot	= cs42l43_asp_set_tdm_slot,
+	.auto_selectable_formats	= &cs42l43_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static int cs42l43_sdw_hw_params(struct snd_pcm_substream *substream,
diff --git a/sound/soc/codecs/cs42l51.c b/sound/soc/codecs/cs42l51.c
index 0519affe0ed8da..4b964c71339827 100644
--- a/sound/soc/codecs/cs42l51.c
+++ b/sound/soc/codecs/cs42l51.c
@@ -533,11 +533,18 @@ static int cs42l51_of_xlate_dai_id(struct snd_soc_component *component,
 	return 0;
 }
 
+static const u64 cs42l51_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops cs42l51_dai_ops = {
 	.hw_params      = cs42l51_hw_params,
 	.set_sysclk     = cs42l51_set_dai_sysclk,
 	.set_fmt        = cs42l51_set_dai_fmt,
 	.mute_stream    = cs42l51_dai_mute,
+	.auto_selectable_formats	= &cs42l51_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/cs42l52.c b/sound/soc/codecs/cs42l52.c
index 9d6bcfbbf0b201..5f033556099a20 100644
--- a/sound/soc/codecs/cs42l52.c
+++ b/sound/soc/codecs/cs42l52.c
@@ -883,11 +883,24 @@ static int cs42l52_set_bias_level(struct snd_soc_component *component,
 			SNDRV_PCM_FMTBIT_S20_3LE | SNDRV_PCM_FMTBIT_U20_3LE | \
 			SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_U24_LE)
 
+static const u64 cs42l52_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs42l52_ops = {
 	.hw_params	= cs42l52_pcm_hw_params,
 	.mute_stream	= cs42l52_mute,
 	.set_fmt	= cs42l52_set_fmt,
 	.set_sysclk	= cs42l52_set_sysclk,
+	.auto_selectable_formats	= &cs42l52_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
@@ -1000,7 +1013,7 @@ static void cs42l52_init_beep(struct snd_soc_component *component)
 	struct cs42l52_private *cs42l52 = snd_soc_component_get_drvdata(component);
 	int ret;
 
-	cs42l52->beep = devm_input_allocate_device(component->dev);
+	cs42l52->beep = input_allocate_device();
 	if (!cs42l52->beep) {
 		dev_err(component->dev, "Failed to allocate beep device\n");
 		return;
@@ -1021,8 +1034,10 @@ static void cs42l52_init_beep(struct snd_soc_component *component)
 
 	ret = input_register_device(cs42l52->beep);
 	if (ret != 0) {
+		input_free_device(cs42l52->beep);
 		cs42l52->beep = NULL;
 		dev_err(component->dev, "Failed to register beep device\n");
+		return;
 	}
 
 	ret = device_create_file(component->dev, &dev_attr_beep);
@@ -1038,7 +1053,10 @@ static void cs42l52_free_beep(struct snd_soc_component *component)
 
 	device_remove_file(component->dev, &dev_attr_beep);
 	cancel_work_sync(&cs42l52->beep_work);
-	cs42l52->beep = NULL;
+	if (cs42l52->beep) {
+		input_unregister_device(cs42l52->beep);
+		cs42l52->beep = NULL;
+	}
 
 	snd_soc_component_update_bits(component, CS42L52_BEEP_TONE_CTL,
 			    CS42L52_BEEP_EN_MASK, 0);
diff --git a/sound/soc/codecs/cs42l56.c b/sound/soc/codecs/cs42l56.c
index 4d9a22a1029c6b..ad45e8fdb39375 100644
--- a/sound/soc/codecs/cs42l56.c
+++ b/sound/soc/codecs/cs42l56.c
@@ -960,11 +960,19 @@ static int cs42l56_set_bias_level(struct snd_soc_component *component,
 			SNDRV_PCM_FMTBIT_S32_LE)
 
 
+static const u64 cs42l56_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF;
+
 static const struct snd_soc_dai_ops cs42l56_ops = {
 	.hw_params	= cs42l56_pcm_hw_params,
 	.mute_stream	= cs42l56_mute,
 	.set_fmt	= cs42l56_set_dai_fmt,
 	.set_sysclk	= cs42l56_set_sysclk,
+	.auto_selectable_formats	= &cs42l56_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
@@ -1077,7 +1085,7 @@ static void cs42l56_init_beep(struct snd_soc_component *component)
 	struct cs42l56_private *cs42l56 = snd_soc_component_get_drvdata(component);
 	int ret;
 
-	cs42l56->beep = devm_input_allocate_device(component->dev);
+	cs42l56->beep = input_allocate_device();
 	if (!cs42l56->beep) {
 		dev_err(component->dev, "Failed to allocate beep device\n");
 		return;
@@ -1098,8 +1106,10 @@ static void cs42l56_init_beep(struct snd_soc_component *component)
 
 	ret = input_register_device(cs42l56->beep);
 	if (ret != 0) {
+		input_free_device(cs42l56->beep);
 		cs42l56->beep = NULL;
 		dev_err(component->dev, "Failed to register beep device\n");
+		return;
 	}
 
 	ret = device_create_file(component->dev, &dev_attr_beep);
@@ -1115,7 +1125,10 @@ static void cs42l56_free_beep(struct snd_soc_component *component)
 
 	device_remove_file(component->dev, &dev_attr_beep);
 	cancel_work_sync(&cs42l56->beep_work);
-	cs42l56->beep = NULL;
+	if (cs42l56->beep) {
+		input_unregister_device(cs42l56->beep);
+		cs42l56->beep = NULL;
+	}
 
 	snd_soc_component_update_bits(component, CS42L56_BEEP_TONE_CFG,
 			    CS42L56_BEEP_EN_MASK, 0);
diff --git a/sound/soc/codecs/cs42l73.c b/sound/soc/codecs/cs42l73.c
index 6ddc6549492af2..72c4213ed74bf5 100644
--- a/sound/soc/codecs/cs42l73.c
+++ b/sound/soc/codecs/cs42l73.c
@@ -761,7 +761,7 @@ static const struct snd_soc_dapm_route cs42l73_audio_map[] = {
 	{"ASPOUTL", "ASP-IP Volume", "ASPL Output Mixer"},
 	{"ASPOUTR", "ASP-IP Volume", "ASPR Output Mixer"},
 
-	/* Auxillary Capture */
+	/* Auxiliary Capture */
 	{"XSPL Output Mixer", NULL, "Input Left Capture"},
 	{"XSPR Output Mixer", NULL, "Input Right Capture"},
 
@@ -1126,7 +1126,7 @@ static int cs42l73_set_bias_level(struct snd_soc_component *component,
 			mdelay(cs42l73->shutdwn_delay);
 			cs42l73->shutdwn_delay = 0;
 		} else {
-			mdelay(15); /* Min amount of time requred to power
+			mdelay(15); /* Min amount of time required to power
 				     * down.
 				     */
 		}
@@ -1163,12 +1163,19 @@ static int cs42l73_pcm_startup(struct snd_pcm_substream *substream,
 #define CS42L73_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
 	SNDRV_PCM_FMTBIT_S24_LE)
 
+static const u64 cs42l73_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B;
+
 static const struct snd_soc_dai_ops cs42l73_ops = {
 	.startup = cs42l73_pcm_startup,
 	.hw_params = cs42l73_pcm_hw_params,
 	.set_fmt = cs42l73_set_dai_fmt,
 	.set_sysclk = cs42l73_set_sysclk,
 	.set_tristate = cs42l73_set_tristate,
+	.auto_selectable_formats = &cs42l73_selectable_formats,
+	.num_auto_selectable_formats = 1,
 };
 
 static struct snd_soc_dai_driver cs42l73_dai[] = {
diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c
index a545730b7ed978..77015590368b64 100644
--- a/sound/soc/codecs/cs42l84.c
+++ b/sound/soc/codecs/cs42l84.c
@@ -662,11 +662,17 @@ static int cs42l84_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
 	return 0;
 }
 
+static const u64 cs42l84_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs42l84_ops = {
 	.hw_params	= cs42l84_pcm_hw_params,
 	.set_fmt	= cs42l84_set_dai_fmt,
 	.set_sysclk	= cs42l84_set_sysclk,
 	.mute_stream	= cs42l84_mute_stream,
+	.auto_selectable_formats	= &cs42l84_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 #define CS42L84_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
diff --git a/sound/soc/codecs/cs42xx8.c b/sound/soc/codecs/cs42xx8.c
index 5b689549c74ede..fb9f4aff6bd84a 100644
--- a/sound/soc/codecs/cs42xx8.c
+++ b/sound/soc/codecs/cs42xx8.c
@@ -393,12 +393,20 @@ static int cs42xx8_mute(struct snd_soc_dai *dai, int mute, int direction)
 	return 0;
 }
 
+static const u64 cs42xx8_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A;
+
 static const struct snd_soc_dai_ops cs42xx8_dai_ops = {
 	.set_fmt	= cs42xx8_set_dai_fmt,
 	.set_sysclk	= cs42xx8_set_dai_sysclk,
 	.hw_params	= cs42xx8_hw_params,
 	.hw_free	= cs42xx8_hw_free,
 	.mute_stream	= cs42xx8_mute,
+	.auto_selectable_formats	= &cs42xx8_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/cs43130.c b/sound/soc/codecs/cs43130.c
index 48d88dad4ceb08..7bbb2e83a1ffc9 100644
--- a/sound/soc/codecs/cs43130.c
+++ b/sound/soc/codecs/cs43130.c
@@ -1609,12 +1609,24 @@ static int cs43130_set_sysclk(struct snd_soc_dai *codec_dai,
 	return 0;
 }
 
+static const u64 cs43130_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs43130_pcm_ops = {
 	.startup	= cs43130_pcm_startup,
 	.hw_params	= cs43130_hw_params,
 	.hw_free	= cs43130_hw_free,
 	.set_sysclk	= cs43130_set_sysclk,
 	.set_fmt	= cs43130_pcm_set_fmt,
+	.auto_selectable_formats	= &cs43130_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static const struct snd_soc_dai_ops cs43130_dop_ops = {
@@ -1623,6 +1635,8 @@ static const struct snd_soc_dai_ops cs43130_dop_ops = {
 	.hw_free	= cs43130_hw_free,
 	.set_sysclk	= cs43130_set_sysclk,
 	.set_fmt	= cs43130_pcm_set_fmt,
+	.auto_selectable_formats	= &cs43130_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 };
 
 static const struct snd_soc_dai_ops cs43130_dsd_ops = {
diff --git a/sound/soc/codecs/cs4341.c b/sound/soc/codecs/cs4341.c
index a44e6b3c298e75..74544c58a41597 100644
--- a/sound/soc/codecs/cs4341.c
+++ b/sound/soc/codecs/cs4341.c
@@ -171,10 +171,18 @@ static const struct snd_kcontrol_new cs4341_controls[] = {
 	SOC_SINGLE("Popguard Transient Switch", CS4341_REG_MODE2, 1, 1, 0),
 };
 
+static const u64 cs4341_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF;
+
 static const struct snd_soc_dai_ops cs4341_dai_ops = {
 	.set_fmt	= cs4341_set_fmt,
 	.hw_params	= cs4341_hw_params,
 	.mute_stream	= cs4341_mute,
+	.auto_selectable_formats	= &cs4341_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/cs4349.c b/sound/soc/codecs/cs4349.c
index 9a7f7017571741..28320951e30902 100644
--- a/sound/soc/codecs/cs4349.c
+++ b/sound/soc/codecs/cs4349.c
@@ -228,10 +228,22 @@ static const struct snd_soc_dapm_route cs4349_routes[] = {
 
 #define CS4349_PCM_RATES SNDRV_PCM_RATE_8000_192000
 
+/*
+ * NOTE
+ *
+ * auto_select doesn't include _RIGHT_J, because 32bit width can't use it
+ * (= cs4349_pcm_hw_params()). It can be selected via Card directly if needed.
+ */
+static const u64 cs4349_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 static const struct snd_soc_dai_ops cs4349_dai_ops = {
 	.hw_params	= cs4349_pcm_hw_params,
 	.set_fmt	= cs4349_set_dai_fmt,
 	.mute_stream	= cs4349_mute,
+	.auto_selectable_formats	= &cs4349_selectable_formats,
+	.num_auto_selectable_formats	= 1,
 	.no_capture_mute = 1,
 };
 
diff --git a/sound/soc/codecs/cs48l32.c b/sound/soc/codecs/cs48l32.c
index a9967771124e5b..c11fc1a588c4e8 100644
--- a/sound/soc/codecs/cs48l32.c
+++ b/sound/soc/codecs/cs48l32.c
@@ -2361,6 +2361,27 @@ static int cs48l32_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 	return 0;
 }
 
+static const u64 cs48l32_selectable_formats[] = {
+	/* 1st priority */
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF,
+	/*
+	 * 2nd priority
+	 *
+	 * _DSP_B / _LEFT_J can't be clock consumer (= cs48l32_set_fmt()).
+	 */
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF,
+};
+
 static const struct snd_soc_dai_ops cs48l32_dai_ops = {
 	.probe = &cs48l32_asp_dai_probe,
 	.startup = &cs48l32_startup,
@@ -2368,6 +2389,8 @@ static const struct snd_soc_dai_ops cs48l32_dai_ops = {
 	.set_tdm_slot = &cs48l32_set_tdm_slot,
 	.hw_params = &cs48l32_hw_params,
 	.set_sysclk = &cs48l32_dai_set_sysclk,
+	.auto_selectable_formats = cs48l32_selectable_formats,
+	.num_auto_selectable_formats = ARRAY_SIZE(cs48l32_selectable_formats),
 };
 
 static int cs48l32_sysclk_ev(struct snd_soc_dapm_widget *w,
diff --git a/sound/soc/codecs/cs530x.c b/sound/soc/codecs/cs530x.c
index 2c7e331359117c..e0ea246edbd89a 100644
--- a/sound/soc/codecs/cs530x.c
+++ b/sound/soc/codecs/cs530x.c
@@ -945,10 +945,19 @@ static int cs530x_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
 				  val << CS530X_ASP_TDM_SLOT_SHIFT);
 }
 
+static const u64 cs530x_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs530x_dai_ops = {
 	.set_fmt = cs530x_set_fmt,
 	.hw_params = cs530x_hw_params,
 	.set_tdm_slot = cs530x_set_tdm_slot,
+	.auto_selectable_formats        = &cs530x_selectable_formats,
+	.num_auto_selectable_formats    = 1,
 };
 
 static const struct snd_soc_dai_driver cs530x_dai = {
diff --git a/sound/soc/codecs/cs53l30.c b/sound/soc/codecs/cs53l30.c
index 511453b4c3752d..c9c682a312c33f 100644
--- a/sound/soc/codecs/cs53l30.c
+++ b/sound/soc/codecs/cs53l30.c
@@ -833,6 +833,12 @@ static int cs53l30_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
 #define CS53L30_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
 			SNDRV_PCM_FMTBIT_S24_LE)
 
+static const u64 cs53l30_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_A	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops cs53l30_ops = {
 	.hw_params = cs53l30_pcm_hw_params,
 	.set_fmt = cs53l30_set_dai_fmt,
@@ -840,6 +846,8 @@ static const struct snd_soc_dai_ops cs53l30_ops = {
 	.set_tristate = cs53l30_set_tristate,
 	.set_tdm_slot = cs53l30_set_dai_tdm_slot,
 	.mute_stream = cs53l30_mute_stream,
+	.auto_selectable_formats        = &cs53l30_selectable_formats,
+	.num_auto_selectable_formats    = 1,
 };
 
 static struct snd_soc_dai_driver cs53l30_dai = {
diff --git a/sound/soc/codecs/cx20442.c b/sound/soc/codecs/cx20442.c
index 0dea84cd8ef2d9..d3c7a93702200b 100644
--- a/sound/soc/codecs/cx20442.c
+++ b/sound/soc/codecs/cx20442.c
@@ -200,7 +200,7 @@ static int cx20442_write(struct snd_soc_component *component, unsigned int reg,
  * Line discpline related code
  *
  * Any of the callback functions below can be used in two ways:
- * 1) registerd by a machine driver as one of line discipline operations,
+ * 1) registered by a machine driver as one of line discipline operations,
  * 2) called from a machine's provided line discipline callback function
  *    in case when extra machine specific code must be run as well.
  */
diff --git a/sound/soc/codecs/cx2072x.c b/sound/soc/codecs/cx2072x.c
index d02ab74e549bc1..5b9e9520ae0100 100644
--- a/sound/soc/codecs/cx2072x.c
+++ b/sound/soc/codecs/cx2072x.c
@@ -1521,6 +1521,16 @@ static const struct snd_soc_component_driver soc_codec_driver_cx2072x = {
 	.endianness = 1,
 };
 
+
+static const u64 cx2072x_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 /*
  * DAI ops
  */
@@ -1529,6 +1539,8 @@ static const struct snd_soc_dai_ops cx2072x_dai_ops = {
 	.set_fmt = cx2072x_set_dai_fmt,
 	.hw_params = cx2072x_hw_params,
 	.set_bclk_ratio = cx2072x_set_dai_bclk_ratio,
+	.auto_selectable_formats        = &cx2072x_selectable_formats,
+	.num_auto_selectable_formats    = 1,
 };
 
 static int cx2072x_dsp_dai_probe(struct snd_soc_dai *dai)
diff --git a/sound/soc/codecs/da7210.c b/sound/soc/codecs/da7210.c
index 4df1a25e318b6e..40d759938edcc2 100644
--- a/sound/soc/codecs/da7210.c
+++ b/sound/soc/codecs/da7210.c
@@ -1030,6 +1030,11 @@ static int da7210_set_dai_pll(struct snd_soc_dai *codec_dai, int pll_id,
 	return -EINVAL;
 }
 
+static const u64 da7210_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J;
+
 /* DAI operations */
 static const struct snd_soc_dai_ops da7210_dai_ops = {
 	.hw_params	= da7210_hw_params,
@@ -1037,6 +1042,8 @@ static const struct snd_soc_dai_ops da7210_dai_ops = {
 	.set_sysclk	= da7210_set_dai_sysclk,
 	.set_pll	= da7210_set_dai_pll,
 	.mute_stream	= da7210_mute,
+	.auto_selectable_formats        = &da7210_selectable_formats,
+	.num_auto_selectable_formats    = 1,
 	.no_capture_mute = 1,
 };
 
@@ -1151,7 +1158,7 @@ static int da7210_probe(struct snd_soc_component *component)
 	snd_soc_component_write(component, DA7210_PLL_DIV3, DA7210_MCLK_RANGE_10_20_MHZ |
 					      DA7210_PLL_BYP);
 
-	/* Diable PLL and bypass it */
+	/* Disable PLL and bypass it */
 	snd_soc_component_write(component, DA7210_PLL, DA7210_PLL_FS_48000);
 
 	/* Activate all enabled subsystem */
diff --git a/sound/soc/codecs/da7218.c b/sound/soc/codecs/da7218.c
index 93bd045ed87012..a538b3dd1635bc 100644
--- a/sound/soc/codecs/da7218.c
+++ b/sound/soc/codecs/da7218.c
@@ -2166,12 +2166,24 @@ static int da7218_hw_params(struct snd_pcm_substream *substream,
 	return 0;
 }
 
+static const u64 da7218_selectable_formats =
+	SND_SOC_POSSIBLE_DAIFMT_I2S	|
+	SND_SOC_POSSIBLE_DAIFMT_RIGHT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_LEFT_J	|
+	SND_SOC_POSSIBLE_DAIFMT_DSP_B	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_NB_IF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_NF	|
+	SND_SOC_POSSIBLE_DAIFMT_IB_IF;
+
 static const struct snd_soc_dai_ops da7218_dai_ops = {
 	.hw_params	= da7218_hw_params,
 	.set_sysclk	= da7218_set_dai_sysclk,
 	.set_pll	= da7218_set_dai_pll,
 	.set_fmt	= da7218_set_dai_fmt,
 	.set_tdm_slot	= da7218_set_dai_tdm_slot,
+	.auto_selectable_formats        = &da7218_selectable_formats,
+	.num_auto_selectable_formats    = 1,
 };
 
 #define DA7218_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
@@ -2286,7 +2298,7 @@ static const struct of_device_id da7218_of_match[] = {
 MODULE_DEVICE_TABLE(of, da7218_of_match);
 
 static enum da7218_micbias_voltage
-	da7218_of_micbias_lvl(struct snd_soc_component *component, u32 val)
+	da7218_of_micbias_lvl(struct device *dev, u32 val)
 {
 	switch (val) {
 	case 1200:
@@ -2308,13 +2320,13 @@ static enum da7218_micbias_voltage
 	case 3000:
 		return DA7218_MICBIAS_3_0V;
 	default:
-		dev_warn(component->dev, "Invalid micbias level");
+		dev_warn(dev, "Invalid micbias level");
 		return DA7218_MICBIAS_1_6V;
 	}
 }
 
 static enum da7218_mic_amp_in_sel
-	da7218_of_mic_amp_in_sel(struct snd_soc_component *component, const char *str)
+	da7218_of_mic_amp_in_sel(struct device *dev, const char *str)
 {
 	if (!strcmp(str, "diff")) {
 		return DA7218_MIC_AMP_IN_SEL_DIFF;
@@ -2323,39 +2335,39 @@ static enum da7218_mic_amp_in_sel
 	} else if (!strcmp(str, "se_n")) {
 		return DA7218_MIC_AMP_IN_SEL_SE_N;
 	} else {
-		dev_warn(component->dev, "Invalid mic input type selection");
+		dev_warn(dev, "Invalid mic input type selection");
 		return DA7218_MIC_AMP_IN_SEL_DIFF;
 	}
 }
 
 static enum da7218_dmic_data_sel
-	da7218_of_dmic_data_sel(struct snd_soc_component *component, const char *str)
+	da7218_of_dmic_data_sel(struct device *dev, const char *str)
 {
 	if (!strcmp(str, "lrise_rfall")) {
 		return DA7218_DMIC_DATA_LRISE_RFALL;
 	} else if (!strcmp(str, "lfall_rrise")) {
 		return DA7218_DMIC_DATA_LFALL_RRISE;
 	} else {
-		dev_warn(component->dev, "Invalid DMIC data type selection");
+		dev_warn(dev, "Invalid DMIC data type selection");
 		return DA7218_DMIC_DATA_LRISE_RFALL;
 	}
 }
 
 static enum da7218_dmic_samplephase
-	da7218_of_dmic_samplephase(struct snd_soc_component *component, const char *str)
+	da7218_of_dmic_samplephase(struct device *dev, const char *str)
 {
 	if (!strcmp(str, "on_clkedge")) {
 		return DA7218_DMIC_SAMPLE_ON_CLKEDGE;
 	} else if (!strcmp(str, "between_clkedge")) {
 		return DA7218_DMIC_SAMPLE_BETWEEN_CLKEDGE;
 	} else {
-		dev_warn(component->dev, "Invalid DMIC sample phase");
+		dev_warn(dev, "Invalid DMIC sample phase");
 		return DA7218_DMIC_SAMPLE_ON_CLKEDGE;
 	}
 }
 
 static enum da7218_dmic_clk_rate
-	da7218_of_dmic_clkrate(struct snd_soc_component *component, u32 val)
+	da7218_of_dmic_clkrate(struct device *dev, u32 val)
 {
 	switch (val) {
 	case 1500000:
@@ -2363,13 +2375,13 @@ static enum da7218_dmic_clk_rate
 	case 3000000:
 		return DA7218_DMIC_CLK_3_0MHZ;
 	default:
-		dev_warn(component->dev, "Invalid DMIC clock rate");
+		dev_warn(dev, "Invalid DMIC clock rate");
 		return DA7218_DMIC_CLK_3_0MHZ;
 	}
 }
 
 static enum da7218_hpldet_jack_rate
-	da7218_of_jack_rate(struct snd_soc_component *component, u32 val)
+	da7218_of_jack_rate(struct device *dev, u32 val)
 {
 	switch (val) {
 	case 5:
@@ -2389,13 +2401,13 @@ static enum da7218_hpldet_jack_rate
 	case 640:
 		return DA7218_HPLDET_JACK_RATE_640US;
 	default:
-		dev_warn(component->dev, "Invalid jack detect rate");
+		dev_warn(dev, "Invalid jack detect rate");
 		return DA7218_HPLDET_JACK_RATE_40US;
 	}
 }
 
 static enum da7218_hpldet_jack_debounce
-	da7218_of_jack_debounce(struct snd_soc_component *component, u32 val)
+	da7218_of_jack_debounce(struct device *dev, u32 val)
 {
 	switch (val) {
 	case 0:
@@ -2407,13 +2419,13 @@ static enum da7218_hpldet_jack_debounce
 	case 4:
 		return DA7218_HPLDET_JACK_DEBOUNCE_4;
 	default:
-		dev_warn(component->dev, "Invalid jack debounce");
+		dev_warn(dev, "Invalid jack debounce");
 		return DA7218_HPLDET_JACK_DEBOUNCE_2;
 	}
 }
 
 static enum da7218_hpldet_jack_thr
-	da7218_of_jack_thr(struct snd_soc_component *component, u32 val)
+	da7218_of_jack_thr(struct device *dev, u32 val)
 {
 	switch (val) {
 	case 84:
@@ -2425,76 +2437,76 @@ static enum da7218_hpldet_jack_thr
 	case 96:
 		return DA7218_HPLDET_JACK_THR_96PCT;
 	default:
-		dev_warn(component->dev, "Invalid jack threshold level");
+		dev_warn(dev, "Invalid jack threshold level");
 		return DA7218_HPLDET_JACK_THR_84PCT;
 	}
 }
 
-static struct da7218_pdata *da7218_of_to_pdata(struct snd_soc_component *component)
+static struct da7218_pdata *da7218_of_to_pdata(struct device *dev,
+					       struct da7218_priv *da7218)
 {
-	struct da7218_priv *da7218 = snd_soc_component_get_drvdata(component);
-	struct device_node *np = component->dev->of_node;
+	struct device_node *np = dev->of_node;
 	struct device_node *hpldet_np;
 	struct da7218_pdata *pdata;
 	struct da7218_hpldet_pdata *hpldet_pdata;
 	const char *of_str;
 	u32 of_val32;
 
-	pdata = devm_kzalloc(component->dev, sizeof(*pdata), GFP_KERNEL);
+	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
 		return NULL;
 
 	if (of_property_read_u32(np, "dlg,micbias1-lvl-millivolt", &of_val32) >= 0)
-		pdata->micbias1_lvl = da7218_of_micbias_lvl(component, of_val32);
+		pdata->micbias1_lvl = da7218_of_micbias_lvl(dev, of_val32);
 	else
 		pdata->micbias1_lvl = DA7218_MICBIAS_1_6V;
 
 	if (of_property_read_u32(np, "dlg,micbias2-lvl-millivolt", &of_val32) >= 0)
-		pdata->micbias2_lvl = da7218_of_micbias_lvl(component, of_val32);
+		pdata->micbias2_lvl = da7218_of_micbias_lvl(dev, of_val32);
 	else
 		pdata->micbias2_lvl = DA7218_MICBIAS_1_6V;
 
 	if (!of_property_read_string(np, "dlg,mic1-amp-in-sel", &of_str))
 		pdata->mic1_amp_in_sel =
-			da7218_of_mic_amp_in_sel(component, of_str);
+			da7218_of_mic_amp_in_sel(dev, of_str);
 	else
 		pdata->mic1_amp_in_sel = DA7218_MIC_AMP_IN_SEL_DIFF;
 
 	if (!of_property_read_string(np, "dlg,mic2-amp-in-sel", &of_str))
 		pdata->mic2_amp_in_sel =
-			da7218_of_mic_amp_in_sel(component, of_str);
+			da7218_of_mic_amp_in_sel(dev, of_str);
 	else
 		pdata->mic2_amp_in_sel = DA7218_MIC_AMP_IN_SEL_DIFF;
 
 	if (!of_property_read_string(np, "dlg,dmic1-data-sel", &of_str))
-		pdata->dmic1_data_sel =	da7218_of_dmic_data_sel(component, of_str);
+		pdata->dmic1_data_sel =	da7218_of_dmic_data_sel(dev, of_str);
 	else
 		pdata->dmic1_data_sel =	DA7218_DMIC_DATA_LRISE_RFALL;
 
 	if (!of_property_read_string(np, "dlg,dmic1-samplephase", &of_str))
 		pdata->dmic1_samplephase =
-			da7218_of_dmic_samplephase(component, of_str);
+			da7218_of_dmic_samplephase(dev, of_str);
 	else
 		pdata->dmic1_samplephase = DA7218_DMIC_SAMPLE_ON_CLKEDGE;
 
 	if (of_property_read_u32(np, "dlg,dmic1-clkrate-hz", &of_val32) >= 0)
-		pdata->dmic1_clk_rate = da7218_of_dmic_clkrate(component, of_val32);
+		pdata->dmic1_clk_rate = da7218_of_dmic_clkrate(dev, of_val32);
 	else
 		pdata->dmic1_clk_rate = DA7218_DMIC_CLK_3_0MHZ;
 
 	if (!of_property_read_string(np, "dlg,dmic2-data-sel", &of_str))
-		pdata->dmic2_data_sel = da7218_of_dmic_data_sel(component, of_str);
+		pdata->dmic2_data_sel = da7218_of_dmic_data_sel(dev, of_str);
 	else
 		pdata->dmic2_data_sel =	DA7218_DMIC_DATA_LRISE_RFALL;
 
 	if (!of_property_read_string(np, "dlg,dmic2-samplephase", &of_str))
 		pdata->dmic2_samplephase =
-			da7218_of_dmic_samplephase(component, of_str);
+			da7218_of_dmic_samplephase(dev, of_str);
 	else
 		pdata->dmic2_samplephase = DA7218_DMIC_SAMPLE_ON_CLKEDGE;
 
 	if (of_property_read_u32(np, "dlg,dmic2-clkrate-hz", &of_val32) >= 0)
-		pdata->dmic2_clk_rate = da7218_of_dmic_clkrate(component, of_val32);
+		pdata->dmic2_clk_rate = da7218_of_dmic_clkrate(dev, of_val32);
 	else
 		pdata->dmic2_clk_rate = DA7218_DMIC_CLK_3_0MHZ;
 
@@ -2508,7 +2520,7 @@ static struct da7218_pdata *da7218_of_to_pdata(struct snd_soc_component *compone
 		if (!hpldet_np)
 			return pdata;
 
-		hpldet_pdata = devm_kzalloc(component->dev, sizeof(*hpldet_pdata),
+		hpldet_pdata = devm_kzalloc(dev, sizeof(*hpldet_pdata),
 					    GFP_KERNEL);
 		if (!hpldet_pdata) {
 			of_node_put(hpldet_np);
@@ -2519,14 +2531,14 @@ static struct da7218_pdata *da7218_of_to_pdata(struct snd_soc_component *compone
 		if (of_property_read_u32(hpldet_np, "dlg,jack-rate-us",
 					 &of_val32) >= 0)
 			hpldet_pdata->jack_rate =
-				da7218_of_jack_rate(component, of_val32);
+				da7218_of_jack_rate(dev, of_val32);
 		else
 			hpldet_pdata->jack_rate = DA7218_HPLDET_JACK_RATE_40US;
 
 		if (of_property_read_u32(hpldet_np, "dlg,jack-debounce",
 					 &of_val32) >= 0)
 			hpldet_pdata->jack_debounce =
-				da7218_of_jack_debounce(component, of_val32);
+				da7218_of_jack_debounce(dev, of_val32);
 		else
 			hpldet_pdata->jack_debounce =
 				DA7218_HPLDET_JACK_DEBOUNCE_2;
@@ -2534,7 +2546,7 @@ static struct da7218_pdata *da7218_of_to_pdata(struct snd_soc_component *compone
 		if (of_property_read_u32(hpldet_np, "dlg,jack-threshold-pct",
 					 &of_val32) >= 0)
 			hpldet_pdata->jack_thr =
-				da7218_of_jack_thr(component, of_val32);
+				da7218_of_jack_thr(dev, of_val32);
 		else
 			hpldet_pdata->jack_thr = DA7218_HPLDET_JACK_THR_84PCT;
 
@@ -2626,18 +2638,7 @@ static int da7218_handle_supplies(struct snd_soc_component *component)
 	struct da7218_priv *da7218 = snd_soc_component_get_drvdata(component);
 	struct regulator *vddio;
 	u8 io_voltage_lvl = DA7218_IO_VOLTAGE_LEVEL_2_5V_3_6V;
-	int i, ret;
-
-	/* Get required supplies */
-	for (i = 0; i < DA7218_NUM_SUPPLIES; ++i)
-		da7218->supplies[i].supply = da7218_supply_names[i];
-
-	ret = devm_regulator_bulk_get(component->dev, DA7218_NUM_SUPPLIES,
-				      da7218->supplies);
-	if (ret) {
-		dev_err(component->dev, "Failed to get supplies\n");
-		return ret;
-	}
+	int ret;
 
 	/* Determine VDDIO voltage provided */
 	vddio = da7218->supplies[DA7218_SUPPLY_VDDIO].consumer;
@@ -2875,21 +2876,8 @@ static int da7218_probe(struct snd_soc_component *component)
 	if (ret)
 		return ret;
 
-	/* Handle DT/Platform data */
-	if (component->dev->of_node)
-		da7218->pdata = da7218_of_to_pdata(component);
-	else
-		da7218->pdata = dev_get_platdata(component->dev);
-
 	da7218_handle_pdata(component);
 
-	/* Check if MCLK provided, if not the clock is NULL */
-	da7218->mclk = devm_clk_get_optional(component->dev, "mclk");
-	if (IS_ERR(da7218->mclk)) {
-		ret = PTR_ERR(da7218->mclk);
-		goto err_disable_reg;
-	}
-
 	/* Default PC to free-running */
 	snd_soc_component_write(component, DA7218_PC_COUNT, DA7218_PC_FREERUN_MASK);
 
@@ -2953,10 +2941,10 @@ static int da7218_probe(struct snd_soc_component *component)
 	}
 
 	if (da7218->irq) {
-		ret = devm_request_threaded_irq(component->dev, da7218->irq, NULL,
-						da7218_irq_thread,
-						IRQF_TRIGGER_LOW | IRQF_ONESHOT,
-						"da7218", component);
+		ret = request_threaded_irq(da7218->irq, NU